diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..a6344aac8c09253b3b630fb776ae94478aa0275b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,35 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4f5eebd8fd4b2983dc052d3035ba6e02aa34d484 --- /dev/null +++ b/README.md @@ -0,0 +1,14 @@ +--- +title: Testapp2 +emoji: 📊 +colorFrom: indigo +colorTo: pink +sdk: gradio +sdk_version: 3.35.2 +app_file: app.py +pinned: false +license: mit +duplicated_from: atatakun/testapp2 +--- + +Check out the configuration reference at https://huggingface.co./docs/hub/spaces-config-reference diff --git a/annotator/annotator_path.py b/annotator/annotator_path.py new file mode 100644 index 0000000000000000000000000000000000000000..ba168e19cf0eb7f7dae6ac3d54c5977945e7386a --- /dev/null +++ b/annotator/annotator_path.py @@ -0,0 +1,22 @@ +import os +from modules import shared + +models_path = shared.opts.data.get('control_net_modules_path', None) +if not models_path: + models_path = getattr(shared.cmd_opts, 'controlnet_annotator_models_path', None) +if not models_path: + models_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'downloads') + +if not os.path.isabs(models_path): + models_path = os.path.join(shared.data_path, models_path) + +clip_vision_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'clip_vision') +# clip vision is always inside controlnet "extensions\sd-webui-controlnet" +# and any problem can be solved by removing controlnet and reinstall + +models_path = os.path.realpath(models_path) +os.makedirs(models_path, exist_ok=True) +print(f'ControlNet preprocessor location: {models_path}') +# Make sure that the default location is inside controlnet "extensions\sd-webui-controlnet" +# so that any problem can be solved by removing controlnet and reinstall +# if users do not change configs on their own (otherwise users will know what is wrong) diff --git a/annotator/binary/__init__.py b/annotator/binary/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2d13ad692ffc109ad95789334bb5524d52794acc --- /dev/null +++ b/annotator/binary/__init__.py @@ -0,0 +1,14 @@ +import cv2 + + +def apply_binary(img, bin_threshold): + img_gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) + + if bin_threshold == 0 or bin_threshold == 255: + # Otsu's threshold + otsu_threshold, img_bin = cv2.threshold(img_gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU) + print("Otsu threshold:", otsu_threshold) + else: + _, img_bin = cv2.threshold(img_gray, bin_threshold, 255, cv2.THRESH_BINARY_INV) + + return cv2.cvtColor(img_bin, cv2.COLOR_GRAY2RGB) diff --git a/annotator/canny/__init__.py b/annotator/canny/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cb0da951dc838ec9dec2131007e036113281800b --- /dev/null +++ b/annotator/canny/__init__.py @@ -0,0 +1,6 @@ +import cv2 + + +class CannyDetector: + def __call__(self, img, low_threshold, high_threshold): + return cv2.Canny(img, low_threshold, high_threshold) diff --git a/annotator/ckpts/ckpts.txt b/annotator/ckpts/ckpts.txt new file mode 100644 index 0000000000000000000000000000000000000000..1978551fb2a9226814eaf58459f414fcfac4e69b --- /dev/null +++ b/annotator/ckpts/ckpts.txt @@ -0,0 +1 @@ +Weights here. \ No newline at end of file diff --git a/annotator/clip/__init__.py b/annotator/clip/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e2c70a8eb5dcd06e1d233eaddd9b6d2c86344dec --- /dev/null +++ b/annotator/clip/__init__.py @@ -0,0 +1,39 @@ +import torch +from transformers import CLIPProcessor, CLIPVisionModel +from modules import devices +import os +from annotator.annotator_path import clip_vision_path + + +remote_model_path = "https://huggingface.co./openai/clip-vit-large-patch14/resolve/main/pytorch_model.bin" +clip_path = clip_vision_path +print(f'ControlNet ClipVision location: {clip_path}') + +clip_proc = None +clip_vision_model = None + + +def apply_clip(img): + global clip_proc, clip_vision_model + + if clip_vision_model is None: + modelpath = os.path.join(clip_path, 'pytorch_model.bin') + if not os.path.exists(modelpath): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(remote_model_path, model_dir=clip_path) + + clip_proc = CLIPProcessor.from_pretrained(clip_path) + clip_vision_model = CLIPVisionModel.from_pretrained(clip_path) + + with torch.no_grad(): + clip_vision_model = clip_vision_model.to(devices.get_device_for("controlnet")) + style_for_clip = clip_proc(images=img, return_tensors="pt")['pixel_values'] + style_feat = clip_vision_model(style_for_clip.to(devices.get_device_for("controlnet")))['last_hidden_state'] + + return style_feat + + +def unload_clip_model(): + global clip_proc, clip_vision_model + if clip_vision_model is not None: + clip_vision_model.cpu() \ No newline at end of file diff --git a/annotator/clip_vision/config.json b/annotator/clip_vision/config.json new file mode 100644 index 0000000000000000000000000000000000000000..2c19f6666e0e163c7954df66cb901353fcad088e --- /dev/null +++ b/annotator/clip_vision/config.json @@ -0,0 +1,171 @@ +{ + "_name_or_path": "clip-vit-large-patch14/", + "architectures": [ + "CLIPModel" + ], + "initializer_factor": 1.0, + "logit_scale_init_value": 2.6592, + "model_type": "clip", + "projection_dim": 768, + "text_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0.0, + "bad_words_ids": null, + "bos_token_id": 0, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dropout": 0.0, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": 2, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "quick_gelu", + "hidden_size": 768, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 3072, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_norm_eps": 1e-05, + "length_penalty": 1.0, + "max_length": 20, + "max_position_embeddings": 77, + "min_length": 0, + "model_type": "clip_text_model", + "no_repeat_ngram_size": 0, + "num_attention_heads": 12, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 12, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": 1, + "prefix": null, + "problem_type": null, + "projection_dim" : 768, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "task_specific_params": null, + "temperature": 1.0, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torch_dtype": null, + "torchscript": false, + "transformers_version": "4.16.0.dev0", + "use_bfloat16": false, + "vocab_size": 49408 + }, + "text_config_dict": { + "hidden_size": 768, + "intermediate_size": 3072, + "num_attention_heads": 12, + "num_hidden_layers": 12, + "projection_dim": 768 + }, + "torch_dtype": "float32", + "transformers_version": null, + "vision_config": { + "_name_or_path": "", + "add_cross_attention": false, + "architectures": null, + "attention_dropout": 0.0, + "bad_words_ids": null, + "bos_token_id": null, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dropout": 0.0, + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": null, + "finetuning_task": null, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "quick_gelu", + "hidden_size": 1024, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "image_size": 224, + "initializer_factor": 1.0, + "initializer_range": 0.02, + "intermediate_size": 4096, + "is_decoder": false, + "is_encoder_decoder": false, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "layer_norm_eps": 1e-05, + "length_penalty": 1.0, + "max_length": 20, + "min_length": 0, + "model_type": "clip_vision_model", + "no_repeat_ngram_size": 0, + "num_attention_heads": 16, + "num_beam_groups": 1, + "num_beams": 1, + "num_hidden_layers": 24, + "num_return_sequences": 1, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": null, + "patch_size": 14, + "prefix": null, + "problem_type": null, + "projection_dim" : 768, + "pruned_heads": {}, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "sep_token_id": null, + "task_specific_params": null, + "temperature": 1.0, + "tie_encoder_decoder": false, + "tie_word_embeddings": true, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "torch_dtype": null, + "torchscript": false, + "transformers_version": "4.16.0.dev0", + "use_bfloat16": false + }, + "vision_config_dict": { + "hidden_size": 1024, + "intermediate_size": 4096, + "num_attention_heads": 16, + "num_hidden_layers": 24, + "patch_size": 14, + "projection_dim": 768 + } +} diff --git a/annotator/clip_vision/merges.txt b/annotator/clip_vision/merges.txt new file mode 100644 index 0000000000000000000000000000000000000000..76e821f1b6f0a9709293c3b6b51ed90980b3166b --- /dev/null +++ b/annotator/clip_vision/merges.txt @@ -0,0 +1,48895 @@ +#version: 0.2 +i n +t h +a n +r e +a r +e r +th e +in g +o u +o n +s t +o r +e n +o n +a l +a t +e r +i t +i n +t o +r o +i s +l e +i c +a t +an d +e d +o f +c h +o r +e s +i l +e l +s t +a c +o m +a m +l o +a n +a y +s h +r i +l i +t i +f or +n e +ð Ł +r a +h a +d e +o l +v e +s i +u r +a l +s e +' s +u n +d i +b e +l a +w h +o o +d ay +e n +m a +n o +l e +t o +ou r +i r +g h +w it +i t +y o +a s +s p +th is +t s +at i +yo u +wit h +a d +i s +a b +l y +w e +th e +t e +a s +a g +v i +p p +s u +h o +m y +. . +b u +c om +s e +er s +m e +m e +al l +c on +m o +k e +g e +ou t +en t +c o +f e +v er +a r +f ro +a u +p o +c e +gh t +ar e +s s +fro m +c h +t r +ou n +on e +b y +d o +t h +w or +er e +k e +p ro +f or +d s +b o +t a +w e +g o +h e +t er +in g +d e +b e +ati on +m or +a y +e x +il l +p e +k s +s c +l u +f u +q u +v er +ðŁ ĺ +j u +m u +at e +an d +v e +k ing +m ar +o p +h i +.. . +p re +a d +r u +th at +j o +o f +c e +ne w +a m +a p +g re +s s +d u +no w +y e +t ing +y our +it y +n i +c i +p ar +g u +f i +a f +p er +t er +u p +s o +g i +on s +g r +g e +b r +p l +' t +m i +in e +we e +b i +u s +sh o +ha ve +to day +a v +m an +en t +ac k +ur e +ou r +â Ģ +c u +l d +lo o +i m +ic e +s om +f in +re d +re n +oo d +w as +ti on +p i +i r +th er +t y +p h +ar d +e c +! ! +m on +mor e +w ill +t ra +c an +c ol +p u +t e +w n +m b +s o +it i +ju st +n ing +h ere +t u +p a +p r +bu t +wh at +al ly +f ir +m in +c a +an t +s a +t ed +e v +m ent +f a +ge t +am e +ab out +g ra +no t +ha pp +ay s +m an +h is +ti me +li ke +g h +ha s +th an +lo ve +ar t +st e +d ing +h e +c re +w s +w at +d er +it e +s er +ac e +ag e +en d +st r +a w +st or +r e +c ar +el l +al l +p s +f ri +p ho +p or +d o +a k +w i +f re +wh o +sh i +b oo +s on +el l +wh en +il l +ho w +gre at +w in +e l +b l +s si +al i +som e +ðŁ Ĵ +t on +d er +le s +p la +ï ¸ +e d +s ch +h u +on g +d on +k i +s h +an n +c or +. . +oun d +a z +in e +ar y +fu l +st u +ou ld +st i +g o +se e +ab le +ar s +l l +m is +b er +c k +w a +en ts +n o +si g +f e +fir st +e t +sp e +ac k +i f +ou s +' m +st er +a pp +an g +an ce +an s +g ood +b re +e ver +the y +t ic +com e +of f +b ack +as e +ing s +ol d +i ght +f o +h er +happ y +p ic +it s +v ing +u s +m at +h om +d y +e m +s k +y ing +the ir +le d +r y +u l +h ar +c k +t on +on al +h el +r ic +b ir +vi e +w ay +t ri +d a +p le +b ro +st o +oo l +ni ght +tr u +b a +re ad +re s +ye ar +f r +t or +al s +c oun +c la +t ure +v el +at ed +le c +en d +th ing +v o +ic i +be st +c an +wor k +la st +af ter +en ce +p ri +p e +e s +i l +âĢ ¦ +d re +y s +o ver +i es +ðŁ ij +com m +t w +in k +s un +c l +li fe +t t +a ch +l and +s y +t re +t al +p ol +s m +du c +s al +f t +' re +ch e +w ar +t ur +ati ons +ac h +m s +il e +p m +ou gh +at e +st ar +wee k +! !! +c lu +th ere +n er +t om +s el +ï¸ ı +wor ld +v es +c am +go t +in ter +of f +u m +ton ight +o ther +h ou +loo k +j e +i d +si on +be au +at t +el i +or t +re c +f f +st er +su pp +g en +be en +il y +te am +m m +i c +pe op +it t +at s +on ly +mb er +en g +b ri +m p +k now +b ur +b ar +in s +lo w +sh e +ro w +â Ŀ +t ro +peop le +vi a +lo w +ag a +be t +x t +f ac +ch ar +e ar +w al +s en +f am +b le +n ati +is h +n or +g ame +li ve +s co +le y +d on +ic k +b all +ver y +the se +p an +i a +at ing +c r +a re +g ir +ma ke +st re +sho w +. " +f l +u p +d r +than ks +il li +w om +st s +i g +s ur +ever y +c ur +vie w +le t +in to +mo st +n a +in di +g ar +ha d +s ou +v ed +an t +iti on +ma de +f ol +un i +it ed +ðŁ ı +ic al +th r +read y +ch ec +d ra +k es +boo k +e p +si c +mor ning +ne ws +c au +c t +w ell +an c +pho to +th an +or s +bir th +g g +ou t +ne xt +som e +en ing +stor y +ch ri +do wn +hom e +f fe +fre e +d a +b or +f il +ci al +than k +si de +le ar +qu e +l ine +t en +at es +ye ars +m y +pho to +beau ti +ri ght +n u +for m +shi p +b an +th er +d ays +g am +as on +g y +ðŁ İ +birth day +se t +ic k +e t +st ill +com ing +ta ke +ðŁ ĩ +b b +s ol +s on +d en +e p +mu sic +the m +de n +wh y +f oo +c ra +am az +w n +h ol +t ting +w r +u e +ma g +c ro +l an +c lo +b ra +a k +s ing +c al +re ad +' ve +jo h +b ab +d ri +b lo +bi g +er ic +in t +t or +tr y +l a +le g +hou se +m ic +v al +beauti ful +l itt +chec k +ne w +ver s +s w +ar i +pla y +h er +âĢ ĵ +w in +m a +con gr +sch ool +f un +. @ +he al +ic h +d el +wh ere +l on +ke t +tw o +mu ch +wat ch +v en +d ed +a st +k ed +b as +go ing +m p +e ver +w ays +ro o +de sig +l y +s ed +to p +l in +ch an +to o +it ing +d ent +gh ts +t y +sp o +ne ed +b lu +in st +be ing +âĿ ¤ +w el +l s +hi m +m ay +st ing +n a +el y +litt le +g a +n at +tom or +m c +h on +w ant +a ir +pi c +am eric +p er +le ss +wee k +ve l +a h +c ap +ch am +g er +ti m +tomor row +ne ss +st ate +h al +ser v +z e +o s +p at +v is +ex c +s in +f f +c ity +c en +an y +b el +su mm +t in +w ould +loo king +k o +ce le +fam ily +m er +po w +hel p +bu s +c o +c le +sel f +en s +ic s +th o +an i +ch o +le ad +b s +t wee +th ink +for e +ch il +vi de +di d +al e +ch i +v il +en ds +w ing +p as +' ll +v ol +s a +g s +man y +j ec +be fore +gra ph +n y +ur ing +w il +d d +bu il +f av +st ed +tr an +l ing +ou d +d ge +fi el +nati onal +st a +c er +w ere +in a +se ason +c ou +n ed +amaz ing +ti ons +cele br +n s +a th +he ad +s day +d ar +lo c +v in +an other +g oo +s at +n y +jo in +pre s +s es +s ing +an a +in ing +.. .. +c our +ï¸ ı +ac t +cau se +li ght +am s +t a +b al +f c +hi gh +off ici +t t +chri st +d ic +d ay +ra l +h or +: ) +vi si +n am +o b +ma s +gh t +re ally +t un +fin d +thr ough +por t +u t +ti ve +st y +n e +or e +ðŁĺ Ĥ +supp ort +ne ver +ev en +ðŁ Ķ +h a +y a +l d +u k +r an +j am +wi th +me di +d es +ne y +ch ing +al e +h y +k in +! ! +d y +pl ace +al so +b le +wh ich +bl ack +b li +s ay +par k +pl ay +ir e +vide o +week end +a il +ke y +p t +w ard +fri day +d in +ine ss +g ro +b en +al ways +t ball +ag o +m il +c y +pro duc +di sc +un der +ple ase +sp or +fu ll +e y +ðŁ Ļ +is e +iti es +c at +k no +u se +fo re +k er +ar t +hi gh +op en +s an +e f +our s +sh ed +st ri +d ro +aga in +i m +ðŁ ĵ +en jo +fu n +ge tting +p en +g er +c li +an y +ever y +e u +wom en +â ľ +e st +c ould +r y +" @ +th ou +sh a +comm un +b er +d ents +di s +wh ile +aw ay +di o +h am +g la +d ate +k a +mis s +un ch +w on +in f +roo m +g a +re al +ex per +di rec +sh ould +sp r +g ol +l ong +bet ter +or i +e y +i ence +il s +z z +h an +f ound +v s +â Ļ +po st +ti c +par t +m en +ren ce +ce ss +v ic +s il +sho p +ðŁĺ Ĥ +f ood +v al +sti c +y ou +s ays +e lec +st ar +o c +l and +i d +c tion +fiel d +s of +st art +wat er +fri ends +on es +ðŁ Į +f la +f ar +wh ite +par ty +in st +gr ou +t v +every one +m ent +j a +ch a +pr in +an ts +d uring +l at +l ar +we st +th en +k a +y oun +in sp +in te +we en +visi t +aga inst +re le +he ad +c es +to wn +loo ks +th re +re gi +ren t +pro jec +gir l +se ar +w o +m om +c ar +h un +pu bli +d i +p le +c all +c ri +u m +for d +per fe +fri end +h ard +ssi on +te st +pla ying +ar ound +be cause +ke ts +me et +sat ur +ar ti +wor k +j un +v en +r un +me mber +por t +su per +t wit +s am +el s +t ly +ad v +ati ve +at h +s ure +av ail +la r +s qu +ar ds +ev ent +m en +l l +o ver +lo gy +it al +tim es +m al +b ack +c oo +ma king +st ru +â ģ +it u +sh ar +g an +c as +s n +summ er +pic ture +f an +h in +christ mas +c y +pr oud +cham pi +desig n +pp ing +ho pe +c a +avail able +ma y +we d +photo graph +spe cial +sal e +sto p +er y +a we +al ity +hi story +am a +pre si +b ru +wor king +d one +d r +k en +fe at +w ood +ate st +sun day +mo vi +vel y +s le +f ace +sp ec +stu dents +b y +ha m +sp on +bus iness +d at +i e +i p +so ci +g lo +h and +re cor +r s +me e +ke ep +p ur +heal th +sh e +com ple +go d +da vi +col lec +li st +r a +clu b +t ers +in clu +th ings +pl an +â ĺ +joh n +sh ing +at ul +so on +blu e +g or +satur day +w on +congr atul +se e +âĿ¤ ï¸ı +tho se +ðŁĺ į +fin al +d ou +it h +o wn +ro ad +t our +a st +indi a +ti l +n d +f er +fav or +su l +lear n +fir e +ju st +grou p +a h +r ac +bo dy +u r +c are +à ¸ +p lo +o h +po s +gi ve +te ch +su b +c ent +er ing +y m +il ity +f ic +lon don +v ir +gu ys +b a +ðŁ ¤ +bab y +sc re +ðŁĺ į +tru mp +un der +chan ge +i an +col le +ss es +l er +ss ed +n ice +ann oun +pow er +s ar +a king +min i +s li +s wee +k ar +fu l +c ru +ac tion +a ther +) . +st and +de vel +a a +g an +le ft +lo l +re l +tran s +m ents +in t +e f +man ag +di g +gen er +do wn +p au +ti v +k u +th ur +k en +st on +f ans +tal k +twee t +t oo +sty le +pro te +se con +fr on +awe some +g l +p al +ne t +s or +la u +g on +sin ce +t ty +ser ies +me mor +b eli +fil m +di d +di es +o t +congratul ations +p ra +e ve +w oo +offici al +su c +in cre +b on +par t +pp ed +cla ss +si ve +bo y +cu l +perfe ct +t ou +d am +wel come +foo tball +h i +p ap +wa it +ad a +congr ats +youn g +exc ited +re ce +j an +v a +re d +st ra +medi a +' d +do es +le t +mu l +ill s +gre en +m el +to ge +fu ture +ye ster +vers ity +for m +ta in +i de +ch es +ki ds +qu i +ha ha +de ta +bi g +favor ite +gir ls +con tin +do m +sear ch +u al +a ir +d ers +mon th +c er +yester day +commun ity +ad e +do g +vil le +ic es +d eli +sy ste +ru n +is m +he art +c up +en ti +fe w +presi dent +e ds +un til +fe sti +o k +f lo +sa id +ol e +me d +tra vel + £ +ph one +toge ther +fa st +lo t +gam es +sh ir +bet ween +y es +th ers +do ing +m ac +at or +b and +fol low +projec t +devel op +di ffe +con fe +spe ci +ca st +y s +bo ard +r d +i al +sh oo +r am +ha ving +sh are +fol low +on e +n ame +m r +pu t +disc u +or y +c ame +ou s +s ite +twit ter +t b +t it +fin ally +z ed +su per +com pan +us ing +all s +li st +r is +sho t +g al +t ar +de l +joh n +âĢ Ķ +some thing +ra m +inte re +wh e +b it +ðŁ į +stre et +oun d +a i +tic kets +movi e +re al +k y +ta king +o pp +c c +l am +m oun +in ve +bl ack +us ed +on line +y or +loc al +gu e +c ks +o w +ge st +bo ys +illi on +con t +re ci +in ed +eu ro +no w +se en +p h +te ach +de f +sou th +su ch +aw ard +mu st +is su +ca re +fe el +p lu +l atest +spor ts +we b +te x +e ment +s k +fi c +w an +te ch +o t +bo x +n er +fre e +t al +a sh +c ase +ho t +won der +mee ting +er a +ch all +ðŁ IJ +jo b +il i +c ool +j our +th s +m o +f el +di e +mic ha +e le +te am +serv ice +st and +ma kes +p ing +ear ly +com es +e k +ho li +v ers +ag ue +s au +thre e +mon day +fa shi +some one +th ro +se a +b ad +supp or +tur n +ur y +m ing +photograph y +n ic +mar k +pre tty +ss ing +wat ching +me mb +ar ri +coun ty +be ach +fr an +cen ter +pol ice +b at +publi c +t an +pre ss +s af +s y +ge ts +ro y +n ers +y our +bu y +st ers +sho w +as ed +chil dre +af ric +in es +sp ace +sc ri +h all +pa in +ar ing +hom e +m ur +heal th +ch ed +s and +rece i +gu y +e a +americ an +re si +childre n +- - +i ri +ing ton +coun try +ro ss +le n +ann a +boo ks +b c +e ce +d om +lo vely +k h +pe t +g y +g ri +st age +off ice +ro ck +m on +b ay +t able +su n +m ed +th in +l or +f low +( @ +uni versity +stor e +fron t +goo d +z a +vo te +nor th +he y +an im +or der +mi d +with out +a de +re member +mar ket +? ? +mu s +tra ining +e duc +bu t +co ver +st an +sc en +b la +bre ak +l ou +s ame +g old +a in +o s +bo th +l it +ver n +a i +al bu +p a +enjo y +be g +ell ing +thur sday +inf o +s an +americ a +ha ir +te l +mar ch +con cer +colle ge +confe rence +ap p +h our +ch ang +â ļ +s our +ol s +we ather +w ar +p hi +festi val +secon d +cu te +pr ac +en er +str y +le a +pol it +s av +se n +o w +m i +ne ar +ou ght +z e +co ffe +w illi +d an +se y +davi d +e se +f an +de ci +the at +no v +ati on +tr ac +sc i +re view +c el +e m +u n +ju ly +or ig +ti on +d ru +form er +st ay +af ter +in v +too k +dat a +b al +tu es +d an +ev ening +ðŁĺĤ ðŁĺĤ +d ol +u res +pro vi +t s +e st +sig n +j ac +u k +s ong +ye t +bo w +in du +j ap +h oo +po int +any one +z y +i st +h ur +it al +buil ding +wom an +ch ur +j er +per for +co ach +le ague +ce ss +ne t +i mag +nati on +br it +qu e +aw ards +ag es +wor ks +c ed +man ce +l ate +ig n +mon ey +tru e +i i +t ell +pl ac +p ac +as y +wor ld +be hin +im port +read ing +gra m +gi ving +me t +h it +for ward +st om +pres ent +jun e +so cial +no on +mar t +hal f +s we +go vern +k er +deta ils +li sh +_ _ +ac y +si a +ber t +f all +! !!! +) , +th i +d iti +sp ort +k ing +f it +st af +c at +mu se +cen tr +y er +con tro +b loo +wal k +ac tu +did n +li m +lear ning +re search +wed ne +au th +h ours +k y +f ar +h en +.. .. +it ch +ri l +str ong +sk y +que sti +jam es +r on +d g +f ur +c in +do es +app ro +mar ke +tu res +ful ly +ch at +behin d +te m +fin i +mis sion +b att +fe el +he av +every thing +b ar +w ish +pre mi +i ma +exper ience +e ach +re port +swee t +tic s +spr ing +re spon +syste m +vic tor +l in +sa w +al ready +gh ter +f le +ã ĥ +br ing +albu m +- - +ell s +st an +to m +inter national +w ent +an ni +mat ch +pp er +st one +sm all +ra in +fashi on +are a +v an +ag ram +k o +thou ght +wor th +v an +m er +coffe e +it es +g n +arti st +c on +ar ch +c ir +se cre +gr ound +is o +h and +co m +bri dge +h s +x i +l ink +pu l +sp l +r ace +f li +ri ver +g as +di sco +d al +play er +f it +photo s +it y +o k +j or +tr a +ap ril +ad s +a di +sol u +beau ty +do or +me ss +up date +ali a +sch o +en ed +mom ent +sco t +sc ience +i or +ti es +ac ross +ous ly +sh es +does n +p age +wat er +m illion +cla ssi +l ic +ca st +form ation +micha el +ell o +s mo +in ts +vi sion +op ening +ld n +au str +tues day +win ner +po ssi +r ound +shir t +di t +b o +u es +il led +al ong +tri p +star ting +im pro +k an +per son +no t +re co +ne eds +c le +li e +re st +r ing +win ter +si mp +mo m +be er +fac e +tor s +us a +collec tion +ge or +se ssion +tr ying +la s +la ke +j en +orig in +stu dent +se cur +v in +pic s +ex pe +com p +gon na +e qu +b ad +le y +a u +memb ers +bre ak +w all +gi c +din ner +bu l +insp ir +r i +min d +ic a +win ning +tal king +t ren +s is +t en +wonder ful +s now +he ar +th om +no thing +gu i +st in +blo g +fe st +b un +le e +war ds +ch ance +dre ss +re n +pau l +p es +tech no +ru ssi +c ard +e ast +mar i +w ine +t i +la w +str ic +k i +ap e +au gu +pro fe +as h +cour se +ma il +ren tly +d un +m un +lo ve +is land +dri ve +s l +end ed +ma in +lo st +nat ure +âĿ¤ ï¸ı +ch ic +re por +p in +pr o +st ation +ce p +ta kes +compan y +go es +on d +ma ch +ra dio +d ad +ro ck +j a +p ay +champi on +e e +in de +tt a +ati c +t ab +beli eve +ener gy +z i +t at +wor d +on ce +re sul +y l +and re +an o +inst agram +clo se +t am +cu stom +w a +con om +sho ws +li fe +k in +ro b +t age +n ation +al most +list en +sa ve +re li +ac e +mar y +tre e +for get +j ack +wa iting +direc tor +h ill +bor n +te mp +f l +st e +on a +sing le +wedne sday +un ited +in o +@ _ +ne l +celebr ate +en ding +de al +j i +can ada +hu ge +tr ack +âĢ ¢ +f y +fan ta +an g +yor k +rele ase +p un +ep iso +wor ds +t our +p ack +i gh +classi c +perfor mance +ke t +after noon +recor d +win s +pro ble +âĿ ¤ +f our +b ed +ban k +d ance +s la +cal led +mi ght +a p +pa st +ðŁ ļ +diffe rent +it e +gi ft +ssi ve +chur ch +c us +pro gram +ho tel +ic e +ma d +secur ity +en ge +d c +en ough +st a +e ty +de ad +g un +he ar +m ir +hu man +gre ss +oun ds +pi ece +bre aking +gar den +fi ght +vie ws +f ish +star ted +run ning +gre en +ser i +s m +as k +d or +de ath +e conom +er i +ir d +s er +l unch +âģ ¦ +bo x +nat u +ba se +b an +f al +glo bal +wil d +wo w +out side +mo ve +le ad +an al +muse um +on g +ha w +pow er +than k +b ac +char ac +cam pa +dig ital +r o +op er +de v +w ol +p ati +f a +m ale +pap er +ill ing +c s +â ĥ +educ ation +ta ken +e ffe +m ou +s ad +" . +bas ed +staf f +inclu ding +li ving +a c +ch ina +mo b +stor m +lu ck +ph il +o o +y n +tra vel +k el +ti al +pr ice +boo k +import ant +bi o +p ool +ny c +f ab +lo ad +? ! +chall enge +cr y +ser ve +we ar +bu s +ta in +nu mber +ro r +k at +i z +th ough +ho sp +m m +fa ir +ut es +ho t +po p +fi ed +cam p +develop ment +li br +c ali +em s +âģ¦ @ +b ol +is ed +stand ing +mo del +it a +g le +bro wn +ima ge +ve red +for ce +o il +par tic +sh u +da ily +la w +se c +cla ss +cam p +holi day +cl in +k ers +pres ent +gam e +incre di +er ship +inter view +b ill +du e +and y +ab o +in nov +ke y +ac ade +p il +mo der +st ars +br and +f er +wee ks +con si +pr e +sa fe +wr it +di um +la unch +marke ting +ann ual +as si +cour t +la dy +c ted +and a +in side +chil d +opp or +sm ith +centr e +gu e +âģ © +f ren +st y +for t +ent ly +is n +ke ep +to ber +on y +bo y +al d +col la +de mo +le vel +com pet +ad o +b our +fanta stic +m ate +s u +sou th +oppor tun +vers ary +lat er +bu d +face book +la un +ster n +p it +! " +ma j +gr am +tb t +fi re +happ y +a ks +wh ole +actu ally +ill er +ell a +lo ts +al ex +an ge +lan ds +ðŁĺ Ń +en ter +r ou +episo de +p ed +in ten +sh ire +wh o +pl an +h o +ca ke +we st +mag az +fre sh +c c +n ar +ch ris +wr iting +w er +n om +l o +mi dd +dre am +o l +ti onal +de b +> > +be come +s i +gr and +all ing +hi stor +ri de +i red +saf e +que en +ci l +in tro +vi l +d ani +.. . +ar tic +st at +sh ort +or ing +sel fi +mis si +do c +b it +g all +b om +i re +se lec +d ition +ðŁĶ ¥ +fri end +be at +gh ting +ðŁĺ Ĭ +pe ace +ex hi +ant a +ab ility +il lu +j on +qu ality +tri bu +m es +play ers +fa ir +cu t +c ab +suc cess +b i +su s +pro mo +sch e +an ge +ic o +comm it +cat ch +ill a +kin d +feel ing +qu o +s ay +anni versary +spo t +mo ther +an e +p end +your self +op s +app le +min utes +p o +gr and +ri es +ha ha +care er +ed ition +de c +ric k +am i +concer t +iti ve +ge ous +d ly +t te +adv ent +i g +li ghts +ak er +sk y +âĥ £ +r ay +fini shed +w ay +s d +ac coun +ðŁĴ ķ +ck y +ch el +lit er +pain ting +lo s +st un +techno logy +n as +ma r +b il +afric a +ki e +ey es +gol f +plu s +ni a +it ec +serv ices +wed ding +kno wn +te le +.. ... +star ts +pa ren +w ants +ati onal +mon ths +win do +fav our +er t +magaz ine +ex clu +re ve +b c +origin al +e ss +n al +an ti +st ro +t ice +stu dy +à ¤ +v ac +nation al +fi ve +ra in +ve ment +u te +ver se +em er +ar my +possi ble +gue ss +val ley +ther n +cro w +m r +col or +on to +pic k +cle ar +dar k +t ac +wan ted +it ting +can cer +govern ment +di e +ri se +z ing +col d +f oun +stu dio +str ation +bro ther +a head +sh el +mic ro +ic ally +d au +sig ned +vi ol +a x +as se +i o +w re +spl ay +ch ick +augu st +pl at +ti ps +sp i +hu man +e asy +lo gi +mi ke +gro w +ag re +w w +sh ad +mo tiv +wi de +tur ns +om g +v ar +de fin +su g +j im +ðŁĶ ¥ +t d +campa ign +nam ed +re tweet +co p +t v +le av +k is +dou ble +s mar +issu e +vil la +in formation +li es +sto ck +n t +di stric +sh or +mi x +er o +se p +me x +see ing +li ve +re min +co de +g ur +s c +wil d +l un +h ood +spo t +fa ther +fore ver +up d +tra f +f ly +ne ed +gra du +tra in +ma ke +s ab +be y +si ze +lead er +tal ks +e u +lo g +fo x +gor geous +le ss +le ts +sur pri +my self +no te +li ves +f ru +lo ved +se ver +de m +j i +so c +h old +do gs +n i +â ŀ +lea ve +air port +ben ef +ex pl +shi ps +comple te +ach i +gre at +vin tage +j ack +ro c +woo d +pri v +off er +ey e +ver sion +te a +co ach +off ic +w ell +g en +s at +h h +you th +o x +? " +m t +mi x +g g +d le +natu ral +buil d +break fast +thin king +theat re +mo on +ber g +go als +geor ge +en e +exc ell +il ing +tun e +y ed +g ate +m it +net work +jo e +h ello +f b +tu be +we aring +ath le +stru c +har d +gla ss +g ers +thro w +g es +b t +indu stry +manag ement +ali st +go al +stre am +y el +a vi +ici ous +o thers +s ki +chri sti +bir d +e sc +m in +tr o +l t +j an +im p +ri ghts +sh a +or gan +cent ral +ar a +ro ll +favour ite +che ster +el se +p ay +car s +m ine +ste p +prac tice +maj or +h ang +ðŁĺ ĺ +n on +v ari +eng ine +vol un +di a +i led +arch itec +p ink +d s +th y +wa sh +web site +ba g +contro l +el li +f ra +an sw +d ence +y u +r on +ol a +g in +dr in +li c +cou ple +sp ar +g on +cre ate +c t +celebr ating +de ep +e at +te e +vo ice +dro p +vis it +at ors +sta dium +f t +w is +ro l +gra de +fam il +po ints +re pre +w as +traf fic +jap an +or g +hon or +tex as +man u +âĻ ¥ +safe ty +re r +b ag +em plo +rele ased +re gu +ak a +n av +ro le +sen ior +spec t +cro ss +lin es +be st +p ack +s in +ti e +mis sing +sun set +li ber +is ing +j ay +sk i +champion ship +ac tiv +la dies +play ed +y y +pu bl +al o +pri de +s r +pa ki +lu x +sur vi +ck ed +e ts +cho col +austr alia +par is +mi les +h at +ment al +al a +me an +mob ile +en a +in si +f ound +chi ef +t ag +incredi ble +re turn +à © +goo gle +fren ch +cre w +hal lo +ali an +j az +ch er +sil ver +nor th +eng lish +base ball +c af +lim ited +follow ing +app reci +ear th +k ir +ve mber +w ed +p tion +g ed +oc tober +fl ori +c r +en cy +ga ve +lor d +stu ff +ber ry +po st +sm ile +bro ad +st ate +gg er +me ans +ic y +gu n +y o +ma ster +bur g +han ds +ni e +/ / +uni on +brit ish +big gest +distric t +am ing +h il +o ce +per son +pas s +en vir +scho ols +arri ved +anc es +insp ired +ex pla +be n +libr ary +bo tt +am p +ste ph +cont act +b ang +m s +cali for +t old +batt le +b b +chic ago +âľ ¨ +str ate +sh i +de ce +- ) +ad d +la b +j ones +leg end +cast le +ing er +st ance +be l +ur a +re fu +lead ers +po t +se x +h ic +artic le +ki d +fr ance +x x +ex e +gui de +volun te +pr int +al i +ce o +twee ts +w x +scen e +vol u +ant i +h an +as soci +shar ing +ro se +mini ster +sh er +in ste +cle an +demo cr +po ster +sk in +p sy +pro per +cra zy +i am +o re +in i +any thing +po d +mo ving +cl ick +ex plo +com b +cra ft +f i +bloo d +is ra +publ ic +d ent +ol ym +eng land +a si +ch er +fac t +envir on +har ry +g one +me dic +enjo ying +just ice +j r +indi an +wi fe +s ound +t es +dra wing +p al +ide a +cr it +ju li +il er +war m +cl ar +thou ghts +def en +coun cil +intro duc +di ed +jan u +an i +s end +li er +m l +intere sting +tra de +win d +b ay +s ac +anc y +sour ce +b es +org ani +ar ly +lar ge +ff ici +ta g +u t +de sp +o es +tit le +sy m +pic tures +op en +wom en +sho wing +ri a +le ast +lead ership +cur rent +elec tr +val ent +list ening +c key +gener al +de ser +du ce +; ) +c ent +ðŁĺį ðŁĺį +sco tt +po or +selfi e +ev ents +i on +wr ong +de v +h ill +sep te +cul ture +l ine +sor ry +s ent +si ster +ce pt +k ri +no vember +ar i +announ ce +z ation +br an +g ent +d u +l en +per s +f m +mart in +o p +e mb +om e +midd le +suc cess +pe ter +janu ary +f lu +rac ing +d av +bi ke +ðŁı » +pe t +shoo t +profe ssi +feat uring +septe mber +now playing +sta ur +z a +on ic +qu ick +bas ke +spe aking +mil it +z er +chick en +b ell +s ad +co ast +lo ving +y ers +d j +pan el +ver age +s wit +ic ks +b ou +califor nia +s am +paren ts +er o +k illed +ph ys +jo bs +mi gr +an th +e mo +hallo ween +and er +c m +compet ition +e ag +s ket +sp ir +may be +exclu sive +app e +jour ney +scre en +for d +i o +h ate +u g +sou l +her o +soci ety +sy n +gu it +n h +d j +as es +im pre +ti me +sal es +d d +f ts +summ it +stun ning +om s +tur ned +cle an +sof t +be at +re staur +de red +en ces +ma gic +di o +sh ine +gu est +health y +exhi b +stor ies +po pu +n is +el a +bel ow +fun ny +resul ts +s ne +cur rently +ar d +down load +f light +m al +f ine +p ad +ch u +ent ed +h at +ðŁij ı +ste ve +j o +mar k +r at +b all +p c +p on +b by +o li +ar ts +as ure +bow l +att ack +mi c +de ar +ran ge +en ter +chocol ate +br illi +ac cess +, " +? ?? +ch ap +con st +t n +mat ter +blu e +gall ery +em p +work shop +lead ing +y ours +baske tball +w anna +th u +_ _ +mar ri +sle ep +bi a +ch e +ma d +imp act +o wn +si r +chan nel +euro pe +e sp +k itch +hosp ital +w ra +roy al +f s +ne u +qu ar +ne y +ac ks +ch ase +pp y +st al +at ely +ti m +dece mber +r are +per form +cre am +we ight +ch oo +ni ght +ha ven +fr anc +kh an +buil t +hel ping +tru st +ty pe +gol den +ta x +s now +s wi +di sa +questi ons +ve y +li ght +c n +cl oud +thom as +ag ed +sh ou +te ams +gr an +re ason +a a +you tube +v p +pi zz +manag er +bur y +cre dit +tre at +ma x +i k +ma in +g ing +de ad +pro bab +ye ah +ã Ĥ +br and +so li +pl ant +ta yl +gir l +ðŁĺ Ń +nam ent +au to +mess age +ko re +n ur +ter r +ag u +ma p +sen ting +lo ves +gi ves +g ab +z en +ro bert +con fir +w ars +o m +sta in +cam era +and er +won der +a b +ca p +s old +su it +wal king +contin ue +effe c +dau ghter +d anc +cha in +mul ti +ki d +y an +champi on +v o +ta ins +ho st +min i +mis sed +re sc +ly n +fin ish +del icious +s as +tayl or +i b +pro mis +produc ts +moun tain +flori da +regi ster +tre at +rec ent +fe male +boo th +mat t +ve hic +s op +mo tor +suppor ting +phi c +ex tre +dr ink +lan e +th ird +p s +con stru +ce re +far m +ðŁİ ī +tu red +ðŁij ī +c ats +a j +gi e +shoo ting +as ked +paki stan +am e +m b +g il +leg al +squ are +in vol +dra w +oo oo +!! !! +opportun ity +p y +e i +b ts +teach er +charac ter +john son +br on +ly wood +ch ine +c ing +c ine +d ge +gam ing +russi a +ci a +quo te +ric h +go v +flow ers +sp iri +st in +grow th +ðŁı ¼ +comm er +j uni +mu m +r an +s na +a ren +c b +ac tor +col or +si t +pa ir +ch i +bo w +acade my +hel d +r ang +me tal +y l +ac tive +probab ly +t ch +need ed +spe e +cho ice +ital y +ry an +ðŁĩ º +flow er +v it +m n +found ation +b ak +si ons +ne igh +f loo +he ard +re mo +fre sh +ing ing +re f +to wn +cl ou +je sus +spiri t +cou ldn +z es +ðŁĴ Ļ +willi ams +pro ce +moder n +pro cess +sho es +cre ated +tri c +issu es +ann e +att en +de but +h r +n it +sti g +a po +e ps +z u +ã Ģ +si x +car ds +lan gu +fam ous +tour nament +se l +e bay +y n +st on +k ick +announ ced +k am +vo c +brilli ant +hou se +che ese +war ri +mus ic +ho ckey +ðŁĺĤ ðŁĺĤ +sk ills +au tom +smar t +med ical +mon y +e x +gu ar +gi ve +pers onal +ven tion +al li +pre ss +flo or +m c +victor y +hi m +simp le +th or +ðŁĩº ðŁĩ +ta il +lu cky +ale x +qu ite +bo t +ssi ons +chall eng +c ann +amaz on +h ell +b ought +) : +ed y +secre t +produc tion +inde pend +de fe +ad ded +p r +p ag +be d +gre atest +with in +j ay +ðŁ ¥ +ire land +re ly +s d +te xt +dri ving +pro gram +spe ed +col um +str on +à © +fore st +â ĸ +mach ine +co in +sc ar +oun t +bi e +¡ ï¸ı +por tra +comm on +wre st +recei ved +kno w +inve st +pl ans +ac cor +ad op +ter y +re ali +p p +k al +art work +me an +go d +inste ad +an ci +motiv ation +as ing +inspir ation +up coming +polit ical +euro pe +m ers +heav y +ðŁij į +fe bru +scot land +ou gh +b t +bo ss +sche du +spe ak +n ick +u red +in o +e k +ri sk +tor y +pres ents +b on +ru g +st ates +exhib ition +il o +m ill +br ought +: -) +tou ri +com e +offici ally +champi ons +do ors +re p +po se +ex tra +k ings +soc cer +squ ad +app lic +at a +some times +t ari +excell ent +ðŁĺ ĺ +stra ight +car ol +ri p +âĢ į +gra phic +m ol +elec tion +febru ary +as ons +l i +di r +m t +n ick +u su +m rs +com ics +inst itu +cor por +v i +ðŁĻ ı +tu ral +di se +ac ci +we are +am ong +sho pping +t ill +wh at +cha ir +sp an +chine se +innov ation +jo y +k it +cent ury +ob ama +ph ili +f c +re ach +c iti +ul ous +n on +d ang +happ ening +bur n +p el +or ange +d v +k ick +cla im +ing ham +ph y +no v +pod cast +wh i +ni ghts +ear lier +be ar +la h +exc iting +or a +gi ven +s lo +memor ies +contin ues +produc t +gh o +c d +kno ws +ðŁİ ī +publi shed +discu ss +y ard +i phone +tri es +w all +fe b +are n +tru th +win ners +tu re +diti onal +milit ary +proble m +m and +do g +lo ss +c ric +can adi +ve ter +villa ge +" , +y r +un g +don ald +ag ing +bir ds +sci enti +le s +th is +regi on +tic al +itt en +il a +ðŁĺ İ +d ad +di am +abo ve +st ren +li t +p ir +la b +fo cus +bus y +d ur +app ly +s ma +auth or +ac i +exe cu +dom in +re la +jack son +at o +wash ington +ðŁĻ Į +k ill +popu lar +ce ment +ro ad +e ating +loc ation +v ent +ar re +n an +cu sto +advent ure +or din +spor t +ul t +lo ck +questi on +dri ver +land sc +on i +k ins +p d +jor dan +te red +k k +a f +chil d +s p +just in +en i +s elling +z o +wh it +bo ston +partic ip +sig ning +happ ened +he at +m am +dre ams +lo ws +gra ph +the day +head ing +br o +ble ssed +vi c +ve gas +h d +in ning +ro man +and ro +den ti +u se +c it +pro gress +writ er +bo b +ff s +gro wing +b ly +aw are +ex am +sp ent +be t +sc ore +bey ond +do cu +ad el +s f +cou ra +colla bor +in c +priv ate +bo at +* * +z one +p ha +b ill +to tal +plan ning +to wards +plac es +pre view +cre ative +dam n +ide as +se ems +po ten +say ing +di splay +s w +a qu +lou is +by e +li l +e mail +we stern +ger many +ell er +re s +f ant +ment ary +de als +ric hard +jer sey +stren g +ra d +pizz a +mon d +w are +l ac +g i +ar chi +c d +yel low +rec ently +re ach +à ¹ +kitch en +desig ned +tr y +g al +restaur ant +at ure +w w +j as +l ma +ðŁij Į +pa in +av o +min ute +sch ol +ther ap +tic ket +d ry +jap an +diti ons +ter ri +sel ves +happ en +t up +ma g +cop y +sh er +free dom +f ile +speci ally +tor onto +lo ad +g ary +re y +answ er +lo y +cau ght +pri ze +u ne +fic ation +ni ger +sy d +tou ch +feat ure +jaz z +recor ds +him self +di sh +ro ber +spot ted +ma ster +wa ve +fin als +bu ll +for um +al d +re comm +ch a +a e +d oo +inst ru +tru ly +l g +in k +bro thers +de st +j im +m it +clo sed +is on +tri ed +s anta +af fe +w an +hor se +g row +camp us +rel ation +nati ve +jour n +go v +o ct +k it +b ound +part ner +re ma +crow d +! ) +c alls +ra il +qu ali +solu tion +con test +con vers +sn ap +b ase +in iti +ta x +y e +ent repre +it or +constru ction +foo d +present ed +n ings +cli mate +k m +mo del +b j +blo ck +present ation +dre am +fi x +c alling +bus ine +con gress +under stand +we b +val ue +ï¸ı âĥ£ +mex ico +it ely +ki m +char ity +ref lec +bl an +fl ying +anal y +famil ies +b and +reci pe +celebr ation +ac cep +ar y +to t +g b +intere sted +cap tain +âĻ ¥ +ti p +ab sol +bra z +inve stig +o logy +de c +tru ck +ver ing +c lear +don t +go tta +ad vis +beg ins +ma ss +de scri +blo ck +k im +davi d +son gs +memor ial +feat ures +su stain +' . +gra b +jo se +v a +con serv +se ts +man chester +fi ghting +de gre +ag a +in d +sle ep +pos ition +ha ir +sig ns +pol icy +it o +al ert +st am +sp end +w y +absol ut +d m +anim al +my ster +success ful +proble ms +ro bo +k ay +gar den +p d +may or +d ale +t ol +off ers +vis iting +friend ly +tre es +offic er +accoun t +ke vin +ðŁij į +gi ant +contin u +con su +tr act +n fl +ðŁĺ Ĭ +h q +b ility +a ar +dis ney +te en +on ed +wh ite +tra iler +de dic +al one +absolut ely +dig ital +willi am +in ation +s wa +e e +enti re +ger man +ro ll +h its +co st +st ay +th a +ali ve +accor ding +co t +liter ally +her it +re ti +haha ha +exper i +li kes +g t +ste el +__ __ +ch air +christi an +to wer +diffe rence +m d +tre ss +mi d +prin ce +afric an +fe der +foo t +car ri +ser ved +r ice +sh all +feat ured +ck er +rec ru +po e +sen se +ni fic +com edy +cont ent +f at +po sted +con tribu +tim ate +li ver +mb le +inter net +ag e +europe an +cl ing +gla d +ff ic +sc o +ak es +el le +ter min +ton y +p ale +col our +seri ous +pat ri +movi es +b m +professi onal +ad o +al u +br inging +f alls +isra el +ter m +langu age +bro ok +man n +commun ic +can not +ac ti +p he +y an +entrepre ne +tur key +log ical +lon g +ar m +ur s +work ers +ing ly +gg s +ri c +tu al +recei ve +op ens +ge ar +soci al +fe et +c king +ad ver +fin an +fe els +sp la +h r +ea ster +bra in +ã ģ +fi g +le dge +ne arly +prote ct +ma ssive +e th +aw a +ðŁĺ ģ +y rs +aware ness +defin itely +k n +imag ine +k u +syste ms +ðŁij ı +f as +li k +provi de +am o +disco ver +inf lu +ma ker +g az +fit ness +stre et +er s +te d +w c +ys is +pos itive +hel ped +que st +andre w +bra d +b in +hang ing +l ing +bri ght +se ction +ma ss +ðŁĻ Į +follow ers +ho sting +tem por +fla g +a ve +let ter +k ur +re qui +of ten +cry p +su ff +âļ ½ +russi an +treat ment +al le +ha y +l an +keep ing +hol y +power ful +pre dic +fun d +e specially +windo w +je wel +il y +ðŁĴ ľ +gener ation +app a +seri ously +o d +ðŁĺĤðŁĺĤ ðŁĺĤ +cer ti +iri sh +ðŁij Į +mi ami +be th +v ity +se cu +che f +cri me +graph y +ma x +arti sts +re volu +gu ard +spee ch +u c +upd ates +fac es +st ant +chang ed +repor ts +low er +pe ar +n c +k il +loo ked +spe aker +s f +re spect +ok ay +oce an +s itting +architec ture +tra il +se at +i ra +le g +japan ese +d am +u lar +sw im +polit ics +finan cial +ol d +mou th +at temp +de stin +fi shing +atten tion +me m +chang es +deci ded +reli gi +g in +c av +z z +ad am +ma c +wr ite +beg in +sc ul +al ter +is s +ath on +imag es +m oo +jo ined +ðŁĺ ī +âŀ ¡ï¸ı +pas sed +mu sli +h ir +lar gest +cam er +com ic +gh ted +rug by +bur gh +gg ing +te sting +pre par +lau gh +al ed +impro ve +beli ev +adv ice +sha res +he art +tur ning +s b +t el +caf e +n es +dani el +pat ter +t z +se tt +par k +c and +st ick +happ ens +bri an +ne west +e pic +ad or +ki es +war ning +anim als +custo m +ar c +di an +gol d +cor e +t f +c ity +pan ts +re ality +con fi +in ju +fo x +gu il +k new +âĺ º +cor rec +itu de +d den +. # +re duc +pas s +f on +y a +ow ner +re turns +n c +e ast +ap ol +in sur +th o +si m +juni or +be e +ang el +att le +elec tric +hor ror +cra sh +e ye +pat h +sou thern +emplo ye +ge o +t an +ha z +r ally +ðŁı » +proper ty +was n +enjo yed +gre y +g as +bre w +nor thern +hol ding +g p +ta ke +ch art +ly n +dr ama +z o +pa id +throw back +cu p +discu ssion +down town +w ill +le w +b is +t ary +bre ad +up on +r ate +teach ers +it ation +anc ed +cy cle +choo se +d c +ir an +co w +da ve +ra ise +prin cess +fa ith +- > +indu stri +sp ain +guit ar +fac ts +m n +sp en +cour te +go tt +projec ts +au di +o sc +pe ter +s and +intere st +happ iness +ven ue +sol di +surpri se +poten tial +per io +custom er +i i +g ni +manu fac +e co +bro ken +sing er +vel s +wal es +hu s +in j +f our +tal ent +d ying +mat the +fil m +jo ining +s ell +j ar +lma o +sur ger +bb c +sour ces +au stin +ni k +char les +f am +prin ci +ange l +cas h +lo t +o red +pla ys +pl ate +don e +memor y +br ings +n ba +solu tions +teach ing +gr ace +cir cu +hel ps +foun der +mar y +expl ore +de cor +par ts +ch o +inte gr +ha u +is es +pu tting +in er +r it +v y +mic hel +blu es +every day +for ms +bi o +ye ar +p in +t ter +spr ing +) ) +po t +al ing +perform ing +sh an +plan et +mus ical +head s +it alian +stru gg +âĢį âĻ +w ings +pu mp +h h +tr ou +a id +pri me +ear th +pa int +mon t +am y +bb c +fab ulous +fru it +andro id +bour ne +cere mony +enti al +? ? +deb ate +on ing +dra ft +sol ar +t x +j am +cor n +!! !!! +bro o +mil k +po sed +o hi +mo vement +b ren +part ner +p g +et te +ar ies +sh out +n g +leav ing +t ells +sen s +ta ste +kel ly +wor l +gy m +ric h +e gy +pi d +ma s +â Ĥ +courte sy +fran k +incre ase +wr itten +pp ers +re l +ha i +s as +s ound +tt i +w ich +ri ver +.. ." +a g +fel low +ro me +sm all +gen cy +ic an +lux ury +pro of +me t +wild life +mom ents +ra ther +cor ner +com pe +canadi an +lik ely +therap y +li am +econom ic +indi e +rou te +fi ght +ho pe +se tting +ant ly +cro ss +fant asy +de e +sket ch +comp li +ym i +ru les +engine ering +fig ure +ro w +. , +f w +syd ney +w ou +t ation +dre w +us es +the re +sp read +struc ture +pat rick +appa rently +ro s +h ills +w we +ann y +com mission +di v +f ying +con sul +anal ysis +ex i +ten nis +vehic le +ðŁĺŃ ðŁĺŃ +as s +high ly +op ened +b ann +ðŁĴ Ļ +mp h +wi shing +v or +fi f +give away +r r +ra y +je ss +g at +ic ymi +x it +high est +yor k +pi e +invol ved +high er +ri e +mal ay +int elli +desp ite +che e +sar ah +be an +reco gni +ar sen +tal ented +pas sion +ic h +ab c +lead s +dise ase +v is +se c +pre senting +m illi +hol e +sho ts +de part +surger y +gov t +b in +du al +e vi +lon ger +ev ol +scre en +portra it +et c +lo se +ch at +p en +p i +om a +s ick +er c +compan ies +en try +plan e +gr y +ven e +liver pool +premi ere +sha red +a red +fil ms +ir a +holi days +cric ket +ici an +v ing +. ) +ul timate +di vision +con duc +se pt +for ces +mon t +s mart +disa pp +sun shine +in d +b less +ma de +col ors +fran k +ir on +bott le +s go +m ood +j ason +er ic +bir th +te en +respon se +tar get +state ment +fe ar +th el +al um +ar ab +bl in +direc tion +ste ps +er ial +wor ked +at l +ðŁĴ ķ +fel t +pol i +scen es +hom es +b ell +e at +ate ful +t in +l ace +fol ks +p se +an n +wis dom +fa v +but ter +s r +are as +sm oo +bi z +dg es +app o +mo re +the m +effe ct +windo ws +sun ny +cap ital +tot ally +c ities +gr ant +mb ers +s low +au tu +il ities +w ro +ri sing +st ics +viol ence +i gh +qu ot +h it +t c +herit age +bu ff +ne s +z ar +den tial +ex ac +ed ge +de ep +aren a +be came +benef its +mar ks +mb er +a z +am es +pre ci +dra gon +re g +d ings +do s +ðŁĴ ª +n el +s ity +me al +di st +leg end +pur chase +pic al +st ick +f at +du ba +profe ss +car to +pro f +coun tries +respon si +se qu +fa b +tribu te +hon ored +prac tic +pur ple +an ton +pa red +t ough +summ er +environ ment +s ons +ðŁĻ ı +m ps +gi es +her oes +t elling +hen ry +f en +know ledge +Ģ ï¸ı +f r +ne g +u re +ac king +hear ts +s oo +hol lywood +ju mp +sau ce +schedu le +tur n +yo ga +cre ating +c ket +cre ek +â Ń +custom ers +ma dri +gu l +asse mb +moun t +c ell +to p +st al +dav is +t wi +sig n +premi er +iti ons +he aring +un k +pati ents +app ear +heav en +al ty +doc tor +a e +plat form +je ff +ðŁĵ · +regi onal +bi d +box ing +ex ten +or ity +a w +w ise +il le +sever al +bi e +s itu +sy ria +âľ ħ +remin der +enter tain +li on +part ners +in n +ph ar +f au +pl s +expe cted +sug ar +deci sion +s b +ch ron +associ ation +leav es +vis ited +sh ap +ðŁĴ ĸ +fur ther +h ann +w i +run s +l er +fun ding +fil led +.. .... +tin y +han g +or g +co ol +se min +ðŁı Ĩ +spon s +nav y +sa int +dru g +d al +r oun +co vered +tra ditional +invest ment +de te +al ism +f low +n is +sun rise +fe at +f ted +we ird +je re +ve gan +medic ine +an o +ac cu +deli very +temp le +chang ing +wil son +phili pp +re fe +n d +is er +g ay +r and +ati ves +t ely +p and +intelli g +g are +am bas +de mon +commit tee +strate gy +refu ge +bud get +prote c +pi er +ex press +nom in +econom y +al low +ic on +gal ax +o h +indi vi +dem and +vir gin +lu ke +ali sts +man i +s mi +ju dge +ent y +mic hi +resul t +am ed +spe aks +' , +hou ston +sh in +b ing +fl y +ch em +au to +v as +ge t +ar m +thank s +d in +gan g +x x +si on +loc ated +p l +jo sh +in fo +jo ins +adver ti +ot d +el d +si e +re asons +v ent +ðŁĩºðŁĩ ¸ +â ł +convers ation +stu di +ðŁĶ¥ ðŁĶ¥ +go s +s ounds +un it +mu sc +ge l +ack ed +pac i +co s +de re +u u +a o +la m +inspir ing +ar ms +tw are +mat ters +ad dic +du de +ex t +cri sis +b ath +me et +sing h +expe ct +del hi +resc ue +wor st +au g +shi pping +ser ving +st o +dar k +ac es +histor ic +landsc ape +desig ner +b illion +gr ateful +wa ke +e ve +m iller +hou sing +dy nam +is co +be ha +sh op +pr ou +e as +a sia +e ding +k on +depart ment +aw ar +mar ine +in ci +photograph er +ta pe +lo go +r ings +d it +-- -- +vin yl +w c +vo ting +se ven +ambas sad +dal las +t u +com ment +k ra +b les +w ag +u d +au dio +stri ke +offici al +o ts +me tho +to ols +ra di +al an +hun t +wat ched +a ke +fa ke +drin king +mer ry +m l +b day +ri o +ni ke +c ant +re pe +co stu +mur der +ak ers +ch ers +ou ts +beg inning +so s +ad es +n in +not es +wro te +sol o +c i +li ghting +ur ban +bre xit +att end +shir ts +pla yo +ac tress +pl ic +stand ard +quot es +par ade +anci ent + © +tur ing +re e +pri mary +fla sh +citi z +mat es +ste in +z i +clin ton +sk in +gen e +hu m +g ar +t le +y i +fo cu +de an +pl ants +cy ber +b u +om e +ho p +ad dress +ti x +gi fts +relation ship +sub scri +fe ed +exac tly +haw ks +ex o +stre ss +s n +arre sted +an e +sof tware +z ero +the me +mu mb +im migr +mi a +make up +ple asure +uni vers +har b +eng ine +ap er +r in +br a +institu te +le ather +al th +sing ing +co s +gh ty +me as +st ic +si de +insur ance +co t +pit ch +moun tains +cri min +su pre +valent ine +at er +wou ldn +sc ale +rel ated +re gar +star tup +pack ed +mi ke +week ly +p ts +coun t +ha r +gott en +min d +ber lin +con ditions +swit ch +cor n +sa ve +g li +emer gency +tun ed +sto ck +discu ssing +every body +s day +whe ther +wrest ling +ec es +gen der +ch en +ðŁij Ģ +madri d +mar athon +e gg +i er +th x +as king +kore a +wol f +ay a +g m +g au +at ory +v r +gra ss +k illing +b ble +ur o +un i +e th +sh ore +th en +re ale +bot tom +ex erc +k ar +or ies +ad ri +san ds +se x +. ' +volunte ers +per form +par liam +inclu de +deli ghted +execu tive +fu el +kis s +ã ħ +char ge +h u +ca kes +ve t +g lu +agre e +pr ices +n au +h l +g ru +ra j +streng th +b ic +sp ending +al es +av en +b last +: ( +yo f +nor mal +si x +qu ick +se a +d aw +mee ts +lo vers +upd ated +po tat +comple ted +coo k +opportun ities +p ure +organ ic +tem per +c am +avo id +par king +duba i +and o +di stri +to y +comple tely +don ald +tri al +bas s +b oun +back ground +v as +mar vel +lu m +ru s +t ool +com missi +throw back +fin ding +is lam +! ? +st op +e vil +or al +resi dents +i denti +o ak +ðŁİ ¶ +l il +span ish +chap ter +sto pped +direc t +ho sted +pic ked +lab our +lew is +defen se +à ® +health care +wh is +mat h +pe ak +ra ised +fi x +bu ll +th ir +chel sea +fol k +tr e +can di +pau l +ei ther +ad am +poe try +jewel ry +ðŁ ¦ +pr ay +Ø § +g c +o z +wi shes +fore ign +sun g +lear ned +en e +n ing +micha el +illu stration +legend ary +w av +b au +ðŁļ ¨ +cal end +stre ets +â Ĩ +mon ster +bu ck +g r +scho ol +ba th +wa ste +ne ck +ha wa +be ach +re plac +jec t +on er +fac tory +coun t +ðŁĵ ¸ +mor gan +der ing +se an +steph en +de p +no vel +vide os +ic al +press ure +arsen al +ex pre +ir s +tren ding +ss a +fla sh +re sear +thr ough +profess or +scul p +to s +gg ed +mm a +be e +a pe +hun ter +am i +he i +pla stic +bu cks +uni verse +le gen +niger ia +ple ased +ri s +thin ks +autu mn +i ds +d is +anth ony +ðŁı ½ +ak ed +gla sses +fin ance +z er +k as +con tract +nu mbers +sh aw +partner ship +t il +laun ched +s al +victor ia +theat er +usu al +nam es +perio d +eli za +i th +bar cel +ro cks +bag s +mat e +distri bu +j on +di ffic +ali zed +cur ren +sco red +b ha +du blin +ro se +in ted +soli d +beha vi +wal ker +simp ly +garden s +head ed +in i +ohi o +we ap +f o +gl en +e state +ran dom +th under +thr u +k ill +jac ket +it i +entertain ment +thanks giving +ent al +en coura +el o +a ther +tan k +high lights +f ting +ru le +model s +bor der +bj p +hus band +in done +ken ya +be ars +al o +n inten +pi x +str o +or ders +sal ad +ro ads +n or +l ation +sop hi +ðŁı ¼ +pi eces +b one +min s +inclu des +nu tr +phi l +s ent +fun dra +ga in +bor ough +n ad +mon day +activ ity +it ems +be coming +ken ne +de tro +car di +gue sts +u x +world wide +sever e +new s +thank ful +fic tion +ve ge +m all +si an +er al +inj ury +le e +men u +danc ing +scot ti +exam ple +( # +na i +studi os +ba i +ðŁĴ Ľ +j av +diam ond +vin ce +ric k +prote ction +lin col +cham ps +appro ach +d ar +m ile +clou ds +je ff +in fin +l ers +p les +pe ace +go p +âĻ ¡ +tech n +str a +a verage +ef fort +introduc ing +di versity +austr alian +am p +boo st +s ke +pati ent +appreci ate +ici ans +pu r +f ell +woo ds +illu str +ðŁ ĸ +ag ency +ac tions +brit ain +under way +se attle +el and +ag o +f ill +stre aming +pro test +challeng es +ky o +et sy +coo king +exper t +ru ss +rain bow +commer cial +sp in +be ats +c ry +val u +el i +th row +gr ams +le vels +michi gan +c ad +ador able +const itu +w s +pu b +mid night +th at +net fli +braz il +die go +regu lar +jo y +âĤ ¬ +li qu +ea stern +k ni +fl at +n p +bro wn +w er +se y +tt ers +ac ting +v anc +cy cling +program me +ra w +comple x +tat too +throwback thursday +se ssions +ro oms +si ght +speci es +bom b +lau gh +ke eps +mo on +offic ers +con ver +t r +ha sh +t ack +ri ous +ad ap +a j +reco gn +ex po +sug ge +confir med +rol ling +dre ssing +ic t +fri day +ph ones +ri dge +con cept +ro y +ke ys +ef for +c ate +k ne +ev en +l ay +commun ities +mo d +n az +every where +al ab +bit coin +ban ks +out door +feder al +sto res +h p +c al +m ely +sig nific +be ar +re public +clo ser +al lah +pic k +x d +pal ace +ch ill +b am +er ous +un a +al len +out standing +olym pic +supp ly +fi gu +v au +l p +char lie +un es +> >> +legen ds +ici al +co ast +benef it +mul ti +f its +far mers +am ount +si sters +har ve +hon ey +que en +b ers +pl ann +âŃ IJ +m u +barcel ona +al ber +stat us +re main +ex tra +c andy +vi ous +âľ Į +o v +warri ors +-- > +ju mp +am ar +x mas +stu dies +i ors +k or +don ate +pre p +fi sh +im a +pain ted +ad mini +co splay +spor ts +dro ps +fi ghter +evi dence +ðŁĴ ª +la ke +ro b +cine ma +pro file +à ± +stan ds +leg acy +sh ape +ro of +ci vil +i ans +sy l +sh am +vo ted +re tail +ph illi +li sted +du ty +n b +th es +f are +au ction +ffici al +stor ms +d p +l oun +sh ops +al y +ani me +multi ple +ðŁĺį ðŁĺį +psy cho +je an +ap art +candi date +gg y +con f +jose ph +w ick +me at +fr ame +c l +for got +ph y +f ing +li ed +re p +se ed +f all +u fc +nu t +lin d +mo de +fiel ds +en ce +s ley +ðŁ¤ Ķ +ch ill +follow ed +announ ces +cor ru +tro phy +them selves +ac le +al du +k ong +l on +s v +bro ke +ander son +ta i +stor y +tempor ary +activ ities +k ati +ari z +cry stal +spo ke +extre mely +tra ding +ðŁĴ ļ +à ¼ +in ch +ed in +out fit +equ ip +ma di +form ed +be ef +po p +ti ger +this day +ti red +neigh b +re tro +is a +un t +t as +kan sas +de st +secon ds +ta y +hur ric +o u +galax y +dad dy +bro w +bur ger +en ced +de sk +ac cur +secre tary +el ite +k ab +ch in +touri sm +bud dy +ici de +dre ssed +u d +vac ation +che ers +com for +charac ters +j et +bu ying +l ins +n ap +reale state +li e +af c +i ii +f ame +n r +b at +ag ent +ma kers +âĢ ¼ +sec tor +op ti +le on +di et +pra yer +hi p +mi r +le x +br y +an a +pas sing +w en +reco very +ak i +po pul +res ort +mar ia +stu ck +read s +ti er +perfe c +netfli x +p oo +cham p +o c +re duce +we red +comm ents +cla im +acci dent +s ag +h ack +sal t +kin da +k iller +i os +z y +ex change +lec ture +eng er +ic king +t au +reve als +pri son +z om +gh an +u l +jour nal +i ot +tr in +jon a +govern or +cap e +quar ter +spec tive +impre ssive +bab ies +t x +m ill +o y +har ri +jo int +su e +collabor ation +tren d +revolu tion +re new +alum ni +ge tt +sh ell +sun day +ent u +ni c +donald trump +block chain +paci fic +expla ins +sp y +ad voc +par adi +to f +star ring +p av +fe ed +br ac +smo ke +ham p +y am +to kyo +si mon +d h +e ffici +phys ical +n j +ell i +s low +gradu ate +americ ans +ti fy +f red +ap ore +fin ds +rob in +we t +not ice +se mi +un ve +k om +pil ot +scre ening +da ily +ðŁĴ Ĺ +roy al +sp a +vo tes +n ag +wh ate +att ending +exper im +ad dition +k ate +sto l +m ali +foo t +chri st +ch an +de e +lic en +glo bal +mo ore +ti a +bri gh +myster y +y ay +âĿ¤ï¸ı âĿ¤ï¸ı +cre ati +me chan +clo ck +di c +âĢ Ķ +pp er +al ph +through out +al low +re sources +selec tion +ham il +bb q +aa aa +virgin ia +dis ney +en g +so red +drin ks +f ancy +consi der +end a +jan e +hand made +du l +on tari +i us +s ville +color ado +whate ver +whe el +promis e +ne ver +desig ns +ab ly +sex ual +vanc ou +at i +con vention +cul tural +sing apore +pro mo +load ed +gla sgo +pp l +n oo +ke e +ste m +men tion +i do +cru ise +ri ding +be comes +be y +âļ½ ï¸ı +tw in +dedic ated +na sh +de si +work out +jen ni +i v +grou ps +rela x +pho eni +li ft +mix ed +m ck +p c +mu st +me tro +ci es +y ar +a im +ang er +i e +rec y +marri ed +dro pped +eng ag +le st +ambassad or +op h +de s +w ick +assi stant +nat ur +fa il +l td +shor t +k ap +sha w +bi gger +rema ins +crit ical +sur vey +co verage +er son +win d +n b +bil ly +let es +ac ts +jim my +at lan +al and +t c +import ance +dam age +f g +stor age +tw t +bon d +bal ance +cr ying +pu ppy +vo te +pu sh +ðŁĴ ľ +pol y +me l +lon don +terr ori +effec tive +corpor ate +atl anta +jac o +nas a +gre ek +sen ate +i sh +ev a +intellig ence +effor ts +al co +k un +h all +di ag +claim s +fir st +h b +ba e +v ul +pu ll + ° +se par +spe ed +vic ti +on thisday +audi ence +r ates +te ach +fil ming +bu sh +son g +y um +br un +ra ine +aw a +par ks +ð Ŀ +ra bb +ra ch +ra id +reach ed +ra il +mo ves +selec ted +fr i +ra ising +om y +st ones +su k +franc isco +cas es +cap it +con fu +w tf +po ke +equip ment +gre g +ess ential +off ering +ne x +pi es +be c +cre ation +chair man +cro wn +w al +john ny +shi ft +ne ck +ban g +bir d +ðŁĺ ı +du ck +re serve +de pu +ma sters +over all +no tic +ju ice +sne ak +che er +cla sses +eag les +n ca +car pet +ci vil +coach es +har ris +u ps +b alls +dec or +mar tin +ro s +v ice +announ cement +who se +ti gers +ste red +c ts +dr am +ste el +youn g +inst all +supp o +recor ding +de ck +se ats +l der +ang le +bo t +sty les +elec tions +for tun +n ab +but ter +ari an +ka sh +in ner +ou red +be ast +we i +ic onic +exper ts +ne cess +b eng +jam es +li a +gre ece +ðŁĵ · +ðŁĺ ģ +good bye +m itch +tw ice +mumb ai +ste am +ru sh +med al +ne tt +fashi on +t ar +r s +sav ing +ric ul +l m +sleep ing +brook lyn +mis s +sen ding +disco vered +sp here +of theday +k icks +missi ons +w right +er n +ght ly +i ous +mel bourne +star tu +mo ved +car ry +d ak +ag ues +bel gi +e ma +way ne +do t +er ie +pe l +it unes +matthe w +no body +est ab +cal m +win ds +lu c +prep are +tren ds +exerc ise +adv ant +ðŁĴ ¯ +athle tics +app s +c tions +adv ance +laun ches +litt le +real donaldtrump +eliza beth +carol ina +hu b +hi dden +n w +us er +pol l +great er +mo st +f ed +p at +life style +s ati +sco res +marri age +l r +aven ue +de serve +ri f +ðŁ Ĺ +wat ch +champion ships +gr ay +en ni +cot ton +g om +whe re +pack age +su m +ab solu +new ly +foo ds +ty ler +assemb ly +musli m +ban k +re memb +op tions +produc er +land o +fun ds +u pper +shad ow +pro gre +co p +ing e +leg s +detro it +hill ary +jo se +gi ants +sou p +sustain able +t us +clo thes +roc king +n z +min ne +mat eri +bru ce +ear t +ca sting +independ ent +thou sands +ta h +de cl +veter ans +li ons +wra p +âĢ ¦ +de ss +bl ing +st ine +e ggs +o on +clo sing +z ay +at t +bac on +fa il +ariz ona +de pre +gho st +new sp +w ers +vi p +li ked +id ent +volunte er +ad ult +pu pp +cir cle +mat erial +degre e +gro wn +boo m +calend ar +su r +vie wing +ath letes +ch and +re ll +asi an +en tr +vol ley +victi ms +bo dy +m ama +trans fer +ge ek +in dic +sav ed +ma i +g ent +it s +loun ge +k ol +the ory +situ ation +is lands +ar th +z oo +floo d +vi ously +show ed +parliam ent +ch ev +el ine +at trac +ab ad +ta il +h rs +lu s +por tu +gor y +provi des +to ys +de ath +in fe +an ce +g le +li am +lo ver +hu d +dv d +reve aled +g w +re ment +ca the +l ying +ra dio +der by +stor s +che mi +hosp it +âľ ¨ +' : +ilo ve +le mon +re public +s ni +ne ss +do or +re action +pre gn +fla v +schol ar +spo tify +is ation +vis ual +aw are +spon sored +jo ke +less ons +leg is +lo ck +si mil +ðŁĺ ĭ +kin d +la y +ma h +ho ping +vancou ver +as er +clean ing +gal a +thre at +la p +ach e +ro mance +ex pen +re post +z am +e pi +mir ror +o ak +ad ul +bat man +s lu +l c +vie wed +re views +d ates +indone sia +acti vi +off en +lea f +i si +ag ricul +costu me +s ites +spir itu +appear ance +ir y +st air +applic ation +spec tac +ic ity +ski es +hand le +pun k +paradi se +t n +de al +provi ding +do c +recei ving +bre w +micro soft +à ¶ +fer r +me tro +th ail +y um +car ter +à ¡ +gent le +bre aks +coo per +show case +cu tting +egy pt +bab y +semin ar +gl ori +ss on +fa ve +re hear +lo tte +la dy +al as +pre p +deli vered +nu clear +ir o +engag ement +at ta +con ven +z an +gl ory +hol ds +busine sses +str ange +sch e +it self +gra d +mar kets +f alling +st ats +ge on +bu dd +li s +she et +thi si +co lo +deser t +regi stration +ig n +expla in +inter ior +la ws +writ ers +spr ings +k r +fri ed +blo om +inf ra +a o +cre d +pa st +line up +bo o +bre a +boo ts +celebr ity +att acks +bro ok +ev es +ex cu +cher ry +oo p +fas cin +boy friend +se as +n ine +effec ts +po wered +k ha +ðŁĺ Ģ +sh out +con dition +i j +her o +enter pri +win ter +applic ations +sho e +g el +batt le +pro grams +w art +ðŁĴ ¥ +ra p +ho l +dang erous +di a +coun ter +ric s +i or +k night +co at +emo tional +at ures +d as +whe el +fore cast +tran sport +glasgo w +king dom +prepar ing +im medi +ff in +awar ded +prin ting +ro man +fight ers +any more +bel t +p ine +win e +x i +employe es +logi es +al led +de mo +birth day +ange les +lo g +dri vers +neck lace +k ath +s it +athle te +ef s +s burg +pur pose +resi stance +rele ases +t is +vari ous +deli ver +ch al +s anc +opp o +cra w +neu ro +dr a +suppor ters +sna p +diffic ult +swe ar +logi st +pa th +attemp t +à ¥ +swim ming +ste ve +hur t +inclu ded +b ap +wa re +ðŁĴ ĭ +end ers +ja ke +le eds +cli mb +l b +im ple +li sa +clo thing +ðŁĺ İ +d t +com pla +sw ing +stra w +v als +k le +us ers +stor m +cu ts +ontari o +p an +hand some +i ow +ar gu +chec king +scotti sh +Ķ ï¸ı +si er +em ma +po d +patter n +de sh +en h +ed ward +t ing +k h +hal f +lincol n +mo ther +al leg +r c +volley ball +d n +g ay +all y +le ton +gro ve +l oud +adv anced +re spec +cli ent +supre me +thail and +ho w +gi g +to i +do t +dol lar +ðŁij ĩ +p it +r b +h n +produc ed +gg ers +âĨ Ĵ +ml b +can vas +fin eart +us d +in the +p son +actu al +s l +t b +ip ad +en sure +u mb +w d +sk a +mar s +k end +f eli +th ing +count down +absolu te +r out +dra l +p y +inju red +min t +hun ting +mm er +s age +li gh +ac ity +ex pan +mur ray +ar o +sec ure +four th +eag le +reli ef +st akes +industri al +clar k +under standing +see m +pl enty +sil ver +cla u +thre at +sa il +pro duce +ab str +is is +b r +eng ers +wor ry +bie ber +s j +just in +reali ze +ky le +esp n +fil ter +s ch +ty pes +game dev +d ing +twit ter +soldi ers +p om +car bon +y ards +child hood +ri ed +ke l +ele ph +t ons +key note +qui et +wi re +po sting +is sa +repre senting +bac ks +alex ander +celebr ates +ta ining +| | +ch or +esc ape +pe ek +ti ves +fiel d +ssi e +im pac +spons or +r c +we dd +cann ab +si des +trac ks +com par +con trac +techn ical +bi ble +expl oring +sh are +tra v +n ate +ill o +sc ru +m ingham +gun s +of the +sh ame +se es +ca tho +ac cess +ce l +repor ted + » +mari o +p ad +hope fully +ou se +y on +disapp o +ol o +p itt +pa c +ga p +cru sh +s g +k le +ge m +emp ire +dir ty +a is +avi ation +ze aland +fac ing +high way +d anny +spi der +ot ta +ðŁĺ Ħ +w y +col ours +in fl +co sts +olym pics +au s +h m +ho ward +pas ses +lau ren +mu sh +op in +r ho +disc ount +oper ation +em ily +mm m +cham ber +d il +to yo +shi p +sam u +pic tured +un ic +po l +keep er +carto on +st en +ig nor +n ations +n l +ta sting +deta il +offici als +mo tor +franc is +ed itor +ðŁij ĩ +pe ts +rang ers +t g +r n +w ri +nic hol +i se +spo ts +ani e +chec k +tri ple +ku mar +spe akers +ic ing +pre pared +ab use +friend ship +mon th +swi m +air e +sc ent +hamil ton +indi an +j es +yum my +te ars +da wn +i zed +worl ds +ðŁ ķ +b illi +st one +n hs +ba sic +p or +st le +ir on +ol der +cle vel +e ing +ðŁĺįðŁĺį ðŁĺį +prin ts +fir m +air craft +fin est +devel op +aar on +t z +gra ham +own ers +fo li +less on +qu es +bab e +cra ft +ph en +ju n +bir mingham +v ine +ll er +i an +fineart america +evol u +st ab +im per +war d +com ic +wi z +inv ited +du ke +mat ch +por ts +ro ger +diag no +ke pt +te st +vis u +r hy +so c +to x +b aker +sur face +co vers +man s +b its +x box +ff le +n an +gar d +h art +wat ers +v illa +re tro +light ning +catho lic +democr acy +neigh bor +pen n +cr an +jona than +la ura +vi bes +su b +coach ing +clear ly +uk raine +bra ve +commit ment +t all +mar t +ra p +mo di +sco tt +bro s +show er +ðŁı ¾ +âĺº ï¸ı +cou sin +appro ach +br e +com pos +hil ari +phil ly +g ad +quick ly +ri an +t m +vir tual +hou ses +k t +phoeni x +w ire +ff y +b unch +anc ing +tal e +snap chat +star ter +h t +k icking +ap art +th y +) ! +blo gger +it z +com fort +ang els +w ash +" : +ar gent +re quest +hon est +mi ghty +bo bby +k g +ro l +thou se +ex po +h c +tab les +mag ical +po sts +de m +n w +or lando +ab er +* ** +ðŁĺ ľ +environ mental +trans formation +mi le +w ic +hir ing +ma ine +bo ar +r ying +ti s +nit ure +twee ted +anton io +opin ion +fin ale +di y +f is +th in +trou ble +le go +fi les +qu art +sp a +curren cy +cli mate +fan art +rail way +sp ace +ban ds +dani el +mo tion +l eng +hol der +oc cu +mar ie +cathe dral +bu zz +bi es +nas car +bm w +bat tery +char lotte +doc tor +zz le +se ven +in san +d dy +st en +lab or +thr illed +se ren +docu mentary +wav es +cer tain +can did +allow ed +ninten do +star wars +ta p +home made +d les +ther ing +bre e +emp ty +pi ano +pos iti +coun try +por k +pu ts +per ry +m atic +spot light +ti st +or ities +we alth +c p +bar bar +commit ted +as sau +pro fit +e ight +hu l +fini shing +run ner +ss o +insp ec +char ged +christ op +lo sing +co al +ho o +ele v +de le +mo ham +don ation +c able +clin ic +j in +manag ed +ter ing +â ¬ +ur ban +depu ty +bb er +bur n +acade mic +o tt +sta ke +it er +sto wn +ack er +advent ures +ad ams +gre g +pro m +vo l +ac qu +con gre +pa int +citiz ens +c all +af ford +v c +as ks +the tic +independ ence +â Ľ +h itting +bl on +fu ture +â ı +in no +gen e +bo ards +di stance +se t +re mem +th al +pre vent +l ang +ob jec +su sp +mat t +in duc +bor o +pi one +re di +vir tu +prin ted +sco pe +shar k +suc ce +a stron +il legal +j ag +c ting +ine e +at o +rob in +nutr ition +b f +du tch +b n +fur niture +for gotten +at ar +ru p +hy per +bran ch +communic ation +degre es +on ia +un cle +promo te +or che +wi i +j s +but ton +ma jor +c bs +bri stol +premi um +ordin ary +e dit +m g +we ed +st even +: ' +gu s +te s +cap tured +dru gs +do w +wr ites +bi shop +whe els +ali zation +disco very +w r +rach el +ne il +hy dr +cu test +entreprene ur +kore an +ore gon +ul ty +perfec tly +suppor ted +histor ical +t wins +ell y +we l +de vil +in come +scienti sts +de leg +h en +on i +ic ed +gi o +cur ry +reve al +e g +buff alo +n ol +op era +camer on +haha haha +j ab +gradu ation +cra ig +r al +i f +organi zation +le ge +g ang +su d +edin burgh +l ack +fli es +g ate +thr ones +q b +the real +e leg +pp in +c les +jam ie +tn am +cryp to +ou l +p ages +a se +roo ts +stu pid +a did +boo t +prote in +s ap +si um +su s +end or +fun ction +don t +en na +ch y +squ e +wor ker +m tv +e a +k an +ðŁĴ ļ +mu s +professi on +t to +oper ations +al lo +c tor +inv ite +sc and +ou th +z im +lin ks +cli ents +sam sung +discu sses +n ell +ul tra +some where +ste wart +ine t +de z +b out +fac tor +ti an +tr ans +jere my +d b +ðŁĩ ¬ +or n +develop ing +spo l +coo per +ma u +rememb ering +tre k +famil y +sen iors +fo ster +att ended +w ing +trans form +ele mentary +hor iz +li sting +malay sia +it ch +warri or +philipp ines +russ ell +m end +initi ative +cre ep +to ps +br iti +a ur +shar p +adverti sing +ug ly +achi ev +materi als +bu g +dev ice +bon us +fac ility +col e +nh l +y as +plann ed +pol e +excell ence +tr ick +con fl +r p +achi eve +lo an +swa g +jess ica +ho we +p our +sc u +z oo +r ated +dre sses +re bel +mex ican +co ordin +me ss +atlan tic +t l +osc ar +wal ks +phar mac +investig ation +... # +cc i +eas ily +monday motivation +y ment +au ti +for ced +ar med +colle agues +pap ers +pro per +sha ke +bu c +le an +exhi bit +e vement +co tt +bi z +sp er +k ent +sw an +/ @ +girl friend +haw k +âĺ Ģï¸ı +mon o +ðŁĴ Ľ +stat ue +ðŁĺ ³ +ra s +te eth +preci ous +t ile +p am +swi ft +v ali +no se +dr unk +experi ences +come back +gen ius +wor se +sh ef +ra d +ed it +hon our +au spol +lar ry +h ire +gor don +achi evement +.... .... +su icide +alter native +su p +sur roun +sha ke +ke ith +pe pper +tur k +crimin al +be ck +su m +w alls +cn n +an tic +of fe +col li +win es +high light +hawa ii +emb ar +l fc +ðŁĩ ® +m v +> > +at mo +wor d +car l +shout out +bre wing +ì Ŀ +do f +s ic +hot test +col on +hh h +shu t +low ing +volu me +apart ment +agre ement +de stro +we e +religi ous +iow a +ro d +land ing +re present +ðŁĵ· : +la s +usu ally +h l +c ac +sal v +al ong +laugh ing +be ans +remin ds +pha se +some body +ma sk +ran ked +dest roy +sc i +âĢ¼ ï¸ı +gab ri +le o +ro a +fa iled +si l +refuge es +re vi +r ing +ber ries +coo kies +y y +conserv ation +sh ab +human s +de termin +a in +ni all +as su +mb a +fro m +extre me +vic es +commer ce +ght ful +or dered +suppor ts +re cap +v or +dro pping +correc t +pay ing +mean ing +n j +qui z +" # +busine ss +ðŁĩ® ðŁĩ +indi gen +du st +box es +bl ind +x xx +zz y +ðŁĩ¬ ðŁĩ +ss els +s ant +dd le +hilari ous +desig n +wonder ing +vehic les +k re +ju d +rece ption +par ker +Ã Ń +pri vi +hy dro +sof tball +pol lu +lo cked +ba h +e ar +scri pt +di vi +br ace +geor ge +the ast +bel o +j al +tion ary +dent al +roc ket +pur ch +sh ak +manufac turing +e z +it is +con cep +tb all +ch s +direc ted +pra yers +oo k +phil os +vari ety +che ss +ser ver +g and +bal ti +ðŁĵ ¸ +sel y +cru z +spectac ular +bur ning +re present +i z +t one +mer ce +h ell +bed room +estab li +bo l +com mon +ãĥ » +ab or +kit ty +hei ghts +re pair +willi am +qu ake +alab ama +popul ation +re v +re tt +i sts +n ite +le m +a ha +clevel and +r m +po ver +ob se +mon tre +man ia + ® +con ne +car ni +sh ah +f y +u a +sc or +strugg le +bo b +' ' +appro pri +deci de +ff ed +ca ster +s ort +hun gry +dra g +ا Ù +gr ounds +d w +sli ghtly +car din +dead line +bron ze +web in +bar ry +sil ence +e uro +op tion +ear n +ðŁĴ ĸ +howe ver +na ren +na ils +bath room +v ine +ph d +min ing +gar age +( ) +shou lder +defe at +di r +o v +liber ty +ple as +x on +com pre +a v +j in +ab les +sil ent +fam ili +vis its +di pl +ha bit +milli ons +regar ding +innov ative +sen ator +r ts +v on +k l +wh il +requi red +âĿ Ħ +lu v +presi dential +po cket +hun dre +sho wn +fro zen +to ward +fa st +confi dence +r ough +indivi dual +qu et +ðŁı ½ +dom e +fi fa +engine er +z en +re mix +ðŁĺ ĥ +pl ant +min or +robin son +as y +pul led +cer tain +potat o +( : +pre s +oc ca +w it +it em +si e +d ating +thom pson +own ed +an u +vi e +te dly +good night +ex cept +ðŁĮ Ł +ira q +ki e +ren ces +li p +simil ar +sau di +vi g +arth ur +pic ks +mil an +hon da +ma xi +o g +ste st +ar ch +analy tics +ba sti +pear l +ter ry +hor se +ast ro +ac ce +laun ching +inter national +s no +ta sty +den ver +ir l +pe te +tor n +advant age +var sity +" " +sol e +g c +lan g +demon str +ol ds +un ity +ne ts +insp ire +cre te +nash ville +nel son +e ter +wal k +hy un +m ack +tre as +see king +ra ge +bru sh +ab and +whil st +co con +h ong +shel ter +i p +possi bly +so o +it ed +â Ħ +rac es +war ming +qu in +tele vision +mat ches +ra pi +ment al +pal m +jenni fer +rol ls +indi ana +b ars +cat ching +resc u +candid ates +fa re +âł Ģ +se o +vie tnam +alph a +michel le +visi ble +re gre +wn ed +app le +li p +f fe +li z +york shire +ha il +se asons +be gan +m d +k c +la p +fascin ating +hel p +ur y +u ms +nu ts +se m +along side +bri dge +ori al +o ve +world cup +briti sh +comfor table +i ve +hot els +fair s +hor ri +so x +d ining +stre am +bar ri +ss y +w im +ter ms +v u +pe re +l ens +wal ked +r or +l ars +shi eld +dou bt +pro to +cro ssing +me ant +medi um +ad ding +e b +che ap +fun c +pap er +bran ds +ry an +feed back +col lins +un known +tro pical +sand wich +fal len +for mu +selec t +lo ads +answ ers +or i +mag a +d or +du o +ali e +dru m +ur i +de er +sou l +sh ut +âĺ º +sto len +don ated +bu zz +patri ots +ha l +na sty +nomin ated +mon te +ki a +th ri +ing u +te sts +pe tro +ðŁij ij +ho sts +ne st +to pic +pat ch +m my +hu gh +ab ilities +ma the +s miles +g b +ag enda +insi ghts +chi p +ph an +fail ure +dg ers +ha i +signific ant +sho ck +ru ral +gl am +figu res +pot us +o ta +mini stry +appe ars +fe ar +r h +americ an +h att +son y +fi res +e di +n ou +e qui +wh en +univers al +mad ness +i x +sculp ture +b ach +t to +swe den +et a +en to +develop ed +month ly +ma ps +ra h +le d +del ta +sa ints +is lam +ben ch +fif th +v ard +so cks +wel coming +j e +tur ner +v b +ad i +nor way +ad y +hurric ane +por sche +tra dition +ex am +newsp aper +lu ci +a ver +ide al +d na +madi son +ðŁ § +wit ness +ac ou +insi ght +si mon +robo t +sna ke +n bc +ac o +ro ss +sh ment +religi on +ch ann +in su +camp bell +inst alled +we ather +hor ses +ol i +rober t +k az +ðŁı Ģ +veter an +th read +quar ter +ea sier +cap ture +hi pho +law rence +roman tic +pas sion +cl ay +ox ford +th ai +stu dying +fi a +elec ted +most ly +c b +tu mb +âĢįâĻ Ĥ +x l +sh an +fa ster +ev ans +sli de +sh ri +see k +mi es +chemi stry +pump kin +tu m +, , +ro om +fi red +li ps +pres ence +af f +brew ery +arri ve +sw ag +photo graph +pen gu +chi ps +at tor +val ues +accur ate +con temporary +princi pal +cannab is +ari o +any where +gi a +democr ats +buil dings +li ved +ap s +neg ative +m are +bal lo +li on +diam on +loo k +re form +tom my +il la +tre ats +hundre ds +port land +wor thy +ex cep +ar ia +ido l +be er +cd n +y u +aw k +ðŁĩ ¨ +c ells +à ³ +ident ity +dra wn +de vil +f inger +th am +ðŁij Ĭ +ear ned +fin tech +dol ph +twee ting +evolu tion +ðŁĵ į +est im +m vp +n one +ðŁĩºðŁĩ ¸ +toyo ta +au x +mar in +b old +l bs +ste ak +mur phy +it able +lou is +sol ve +pi a +sk ir +ill ino +webin ar +ban ana +lo v +th on +vo ters +afford able +defe ated +lm fa +air lines +super b +any way +deb t +bo red +ver si +me tal +responsi ble +m k +s se +f ay +cau sed +f p +recomm end +pla za +spor ting +alli ance +au stri +n n +t ours +surpri sed +arti f +th under +sur ve +wor e +bri ef +necess ary +z ie +ash ley +dra ke +r t +kni fe +im mun +char ges +a the +bri de +rep ly +g av +broad cast +pu er +brace let +cap acity +harve st +id k +perfor man +d ding +il ers +par a +jam a +pro vince +ch in +id ers +har i +te aser +ch en +re stor +r at +fl at +col om +ðŁĴ ŀ +ðŁĩ¨ ðŁĩ +smoo th +r t +p itch +stay ing +isra eli +t cot +per spective +do ck +open er +lo vel +x o +class room +l ington +go al +kenne dy +sh am +sp aces +mitch ell +home coming +uk i +claim ed +recru it +ing o +mu fc +mon it +g roo +resi dent +per cent +per man +otta wa +int ment +an xi +stand ards +wor ship +sche me +f x +pot ter +bi an +athle tic +af gh +s se +sat ell +par ties +âĿ¤ âĿ¤ +infra structure +rela x +mo du +wor n +smo king +y ach +practic es +wc w +am b +dome stic +tay lor +k entu +provi ded +mo di +ve g +" ... +ob serv +ðŁĺ © +be ard +m our +an gry +ðŁĺ ± +startu ps +woo den +di ve +na il +anti que +ro ses +torn ado +m at +^ ^ +su spect +far m +de vices +me ga +tu l +scholar ship +ge e +disa ster +arri val +po in +mar c +kati e +bb ed +fal se +deser ves +ric hard +ju ana +fre y +tion ed +hy bri +r w +sar ah +ach i +c ure +o le +mor ris +ch ic +broad way +la bel +pa k +pover ty +gol f +e red +f u +er ies +be es +alo gue +st el +wire less +je wish +ti de +blo cked +life time +b har +sp lit +am ster +th i +jo shu +br unch +ha ps +s for +oo ps +ka poor +hi king +suppo sed +ro of +re as +tra in +ti ght +tru mp +bas ically +r r +ea red +see ds +entr ance +c p +wi e +son ic +vic tim +he re +e h +ear rings +sal mon +arc tic +an ne +dou gla +corru ption +hann ah +ha sn +vo ices +con ce +att a +fle et +clin ical +democr atic +ton y +st ood +le f +twit ch +a il +honest ly +incre ased +dro me +don na +accep ted +visit ors +ap ar +ad or +p ar +jer ry +ra i +brand on +ab u +!! !!!! +me me +in gh +glori ous +b hu +pu mp +j ol +li ke +fi sher +ma z +ag an +destin ation +play list +le tters +gen u +br ace +celebr ated +bann er +r he +dra gon +ðŁĺ ħ +sig nature +gre y +âľ Ķï¸ı +al ice +be red +ph er +ber n +ca th +ga thering +sc oring +influ ence +sm iling +de pt +lo cal +a x +ac u +reti rement +hon or +her self +chem ical +asse ss +y all +fre qu +appreci ation +ac a +cho ir +cu z +so il +c il +repor ting +u h +enterpri se +gr at +jaco b +ru m +fe e +j ak +sp in +bi kes +phi a +ste re +p is +bloo d +t att +ra ft +war ren +sh eri +back stage +mar sh +hash tag +ther ine +re in +game day +guar an +reci pes +min ds +stron ger +issu ed +bic y +n ak +ment ed +sc ary +u x +pre vious +tt le +th ats +ac tors +u ma +tin a +bun ny +promo tion +u ss +oli ver +montre al +what s +appreci ated +la kes +excu se +kno wing +pri zes +musc le +shad es +sco t +ing redi +electr onic +ju an +comb at +s ri +e h +turk ish +l om +stri kes +pri son +re e +po pe +vi d +ol dest +dol l +sw iss +certi fied +cli p +re turning +lat or +le igh +tt es +wat son +heal ing +el im +per haps +ha ss +k au +d der +mou se +new castle +indigen ous +wel comes +co le +tau ght +no ise +appe ar +jo e +can on +wedne sday +u tah +c tive +dri ven +i v +c ell +stri p +ac c +focu sed +ar rest +sto cks +wo o +â Ĺ +notic ed +shad o +di spla +ter ror +bor ne +secon d +que ens +wo ke +ja il +no tt +cam bridge +har t +se af +fa x +ac cept +âĺ ħ +goo ds +k at +t win +h s +thou sand +s ins +su ite +amp ton +ar n +rele v +ric har +hoo ps +n bc +class ic +p ab +soldi er +de plo +le ans +install ation +cla sh +le ban +ee e +ti re +belo ved +fu sion +travel ing +ne i +coo kie +glo be +phys ics +s q +co l +wol ves +d l +ex it +" - +foo tball +le af +ster ling +hi de +minne so +fresh man +natu re +indi e +supp lies +bri s +iri sh +ink tober +doo dle +ic op +mess ages +adul ts +recor ded +fix ed +ar do +offe red +under ground +dr one +p ine +ma inten +and re +ham mer +s x +r ound +hi ke +bra d +ro me +fu ll +on ey +ro ws +colum bia +archi ves +appro ved +bat ch +illino is +recogn ition +shou ldn +fo g +nca a +ke vin +human ity +al though +pow ers +p ou +s ar +pe st +alco hol +con sci +phil adel +en o +t m +ok la +cate gory +particip ate +accu sed +bri ef +po em +clu bs +consul t +ja b +big data +amster dam +ac ing +certi fic +n u +d at +impro ved +and y +campa ig +pale stin +p ace +mo bi +feel ings +wol f +bra in +pro pos +inter active +prin ce +inde x +c is +cha e +peace ful +co vering +ac o +cour ses +mon key +re place +b l +bloo dy +tal es +brigh ton +neighbor hood +g ates +spiritu al +af raid +bre ast +b ones +ðŁij ī +vide o +w au +tou ch +inju ries +car l +ri x +une x +âĢ ¢ +fre d +consi dered +thu si +an ch +on y +u sa +graph ics +ac re +ðŁĺ © +com memor +com mod +go ti +guar dian +star bucks +pre vention +haha haha +admini stration +portu gal +fac ulty +bet a +ul a +al bert +bre ath +er i +le tting +tr ic +ment ation +incredi bly +ten nes +v d +ðŁĻ Ī +ed die +br ick +gr ill +bt w +wat ches +resear chers +t ney +ni e +p as +a ster +vi br +poke mon +ch rome +go at +pitt s +il ly +festi ve +y d +can al +ðŁ Ĩ +fi es +car los +re que +partic i +tra ins +sam ple +temper ature +sym ph +pic king +in door +z ers +playo ffs +____ ____ +ap es +ly rics +islam ic +performan ces +d ick +spar k +se as +hom a +gr ound +disc i +employe e +com mu +alas ka +al an +fe ast +dg ing +ban king +manu el +slow ly +tru cks +mc car +oo o +sc rat +orche stra +indivi du +m x +bre ath +stair s +equ ality +bla ke +loc ations +cocon ut +balti more +aa a +l c +ðŁı Ĩ +har vey +resi st +immigr ation +adid as +fil i +re f +lg bt +mo s +pp i +ken ny +terr or +ban e +apol is +s g +social media +ka i +hon est +as sas +bol lywood +âĢįâĻ Ģï¸ı +ferr ari +hor n +cryp to +bo om +mainten ance +i di +s man +w l +ext ended +in sul +ve s +go sp +tr i +pi g +tar ge +cel er +st ati +sm h +ri dic +appe al +? ) +con clu +cos me +she ep +christop her +en thusi +po lish +me ts +oun ded +sustain ability +creati vity +con crete +ra i +ali en +ble ss +te es +clu b +ro t +bo s +ex ist +perfe ction +lu ck +rock y +expen sive +mean while +happy birthday +pre t +thr iller +ca ve +playo ff +som er +l u +le x +def ence +am writing +home less +pro phe +ch et +past or +ðŁ¤ £ +land er +ww w +Ģ ï¸ı +tic a +! # +o tic +rad ar +po sters +pow der +po li +ha un +tra p +bl in +assau lt +shor ts +re y +sh y +squ ir +rac ist +gar lic +fu r +remo te +sm ell +impre ssed +fing ers +âł Ģ +din o +le ment +s nu +promo ting +str ing +produc tive +b age +ma son +ra z +direc tly +j k +ev al +ðŁij Ĭ +doc tors +co w +ri der +st v +re move +w u +na than +ro d +n r += > +affe cted +inve st +mp tion +g inger +o d +agricul ture +s que +mu g +coun ting +ke e +mag nific +coo k +ani stan +roo t +plac ed +sym po +gh ana +un d +che er +thro wing +secre ts +f illing +opti mi +butter fly +bu bb +ðŁĺ ī +terri ble +d g +sil k +obse ssed +lo u +ai de +sal ute +mon u +philadel phia +scienti fic +i st +u ae +dess ert +bott les +can yon +ðŁĺ Ī +car ib +o ther +w ich +re source +guil ty +un d +le on +e ss +kan e +el e +tra iner +he im +an te +man age +roo kie +tre ated +po ses +rs vp +cau ses +aw ak +je well +le tt +on ics +tit les +cardi ff +g aga +bu mp +use ful +? ! +loo se +bb ing +: : +argent ina +de bu +cy cl +wh el +dis gu +j el +k ills +bio logy +ex ter +tra sh +bo dies +tr am +circu it +expe ct +la ds +w ells +sho t +ge e +naren dr +fa stest +b ent +b ills +mar shall +h ats +intro duce +citi zen +im possible +gi b +az z +net working +r ant +thin k +in dy +st ops +f theday +bri an +* * +amo di +dom e +coura ge +pac king +af fairs +g n +si zed +ent ary +pol and +swit zer +afgh anistan +w u +ten der +subscri be +mo sco +att end +republic an +hon ey +âĢ ĭ +si mul +we ster +foo die +or o +midd le +ab t +co pies +ma je +narendr amodi +ty pical +inspir ational +vit am +wis con +cu bs +tiv ity +h ali +e ars +k ay +d are +mari juana +cu rious +an ia +tom ato +re mind +ðŁĩ · +sc ared +cou p +po et +land ed +ri d +wra pped +mor ri +climb ing +e ws +fe eding +con tra +tho logy +gri d +ti vely +read er +la ser +di ving +di g +lat in +ti ed +shake spe +o ci +ad m +show ers +chu ck +mar cus +oo s +kne e +o live +ow l +dy lan +an no +g ym +deci sions +well ness +arri ves +sati s +chri s +thur s +ðŁ¤ £ +inter views +thank you +switzer land +over night +journ alist +ser ves +vol can +.... ... +plo t +nic ol +car rying +mag ne +tre asure +ex p +be ver +ðŁĺ ¢ +mar ty +mo le +don ations +recogni zed +b h +du s +sh ann +al do +success fully +ent e +ðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤ +cab inet +cu is +tit led +d as +so l +strate gies +deli vering +ad ds +ani an +ne ther +ðŁĴ ĥ +con tain +su its +pa irs +to dd +rel la +ro pe +ci o +cro p +paint ings +su z +re jec +bu st +d h +fra ud +m h +contro l +je al +destroy ed +al lows +wo ol +minneso ta +om en +j u +sympo sium +d af +lim it +accoun ts +load ing +inter n +re solution +hol land +qu al +meet ings +gra ve +cam ping +v am +re nov +liber al +am ber +gre e +hu mb +fe ver +el ing +broo ks +à ² +be th +ad ed +al t +ro e +perform ed +jo sh +frank lin +nic ole +de ss +bb s +m g +net works +min im +al t +weap ons +gu y +jas on +g ha +harb our +at on +pra ise +kentu cky +bel fast +st icks +blo ss +ho pes +an thro +famili ar +wa it +ch ile +depre ssion +la x +je ts +le ice +recei ves +si er +an k +de x +inde ed +fle xi +fab ric +lam b +hel icop +am anda +âĢĶ âĢĶ +compe te +sn ack +techno logies +sy rian +mom s +mu ham +cho sen +an at +dev on +shar ks +re t +fundra iser +selfi es +st ations +communic ations +tennes see +tu tor +ro t +valu able +dynam ic +nur se +i ed +earth quake +deser ved +a ve +sar a +stre tch +dougla s +ne pal +à § +ob viously +d ame +ra pe +any body +k w +pat rol +hol ders +h anna +info graphic +ec o +be ating +stan ley +bo ats +ri bb +e z +wit ch +inv a +ac id +boar ding +- @ +gi l +da ve +care ers +opp os +l loy +in ter +do pe +re su +j agu +sh ade +in dy +on ist +rel ations +ag en +ab le +inci dent +me ter +shar ma +id r +pro ve +immedi ately +tro ops +am an +g low +gaz a +blo cks +person al +chron ic +all er +si d +sh r +whats app +lu cy +ar chae +ho u +journ alism +our selves +go t +the med +shap ed +we ak +cas ual +leng th +sla m +ab bey +e v +coun ter +est a +reci pi +cha pel +expan sion +sel f +suff ering +sp ice +n z +sp art +desp er +boo king +quart ers +y on +ðŁĴ Ĺ +p k +continu ed +- # +man hatt +tal ked +sh en +com bo +hybri d +je ans +liqu id +se al +re tweets +ac celer +collec tive +t as +: )) +profession als +ra w +o tt +su san +ir ing +okla homa +re ven +survi val +cre ator +tran sit +st ac +sur f +i k +ed iting +ch illing +bai ley +ste al +ra ble +pa rent +hun ger +sn app +collec t +philos oph +dedic ation +c f +c m +le ep +repe at +re ha +un fortun +a er +a ero +abstr act +mon itor +ag ents +bu l +sci ence +harb or +drag ons +floo ding +ac compli +d ash +juli a +the red +tues day +cy ber +b low +ta ined +le m +refe rence +pp o +ne goti +char le +con nor +au lt +access ories +commissi oner +rain y +re ar +advis ory +luc as +ma id +co al +k av +pol o +ðŁı ¾ +tran sport +mar gare +straw berry +bur ns +gre ens +ne v +partici pants +col in +belgi um +col our +in form +d ell +br on +cal y +kick off +strate gic +re union +hon ors +li b +egy p +âŃIJ ï¸ı +hy po +si zes +regi stered +bet es +relax ing +bloo m +inten se +valent ines +insan e +w wii +p x +tri o +bla de +wiscon sin +con e +plat in +ali ze +ra ven +incre asing +indi ans +il ian +bl u +rabb it +exten sion +je f +au di +fer ry +s ell +a day +us b +swe at +cham pag +metho d +mem ph +assi st +s by +ca pe +remo ved +mag n +v t +r ams +f bi +tack le +phe w +h on +motor cycle +su spec +eleph ant +sub ject +let te +da iry +whe at +awk ward +ac t +tro l +mit ted +zay n +sheri ff +ene my +con s +ke tt +bul ls +ev alu +bt c +satell ite +ho lo +por ter +dia betes +bet ter +rele asing +sur f +: - +se basti +collec ting +en cing +e thi +go ds +al ley +health y +m ills +sma sh +co pper +cr ack +read ers +sp ac +licen se +bas ket +bang la +en tic +om i +m ere +si vely +anim ation +lan es +dent ally +chill in +fi e +k aren +dep th +li pse +n g +ri p +mel o +sand y +ðŁijı ðŁijı +vin cent +nu t +hu g +who le +cre ates +? ??? +âĿ¤ï¸ı âĿ¤ï¸ı +bak ed +up grade +rober ts +har a +carib bean +auth entic +mb s +mosco w +attor ney +wi ki +ch lo +hu ll +cor k +" ! +sty lish +ðŁĵ¸ : +di ary +impro ving +ex pand +bri ght +pollu tion +k nights +person ality +chec ked +fac ilities +z el +bow ling +gu er +ðŁİ Ĥ +on going +un its +hoo k +be ck +confl ict +to dd +far ming +educ ational +k ak +cla y +stro ke +bel ly +explo re +mill enni +th m +loo p +sm s +consi st +cir ca +br yan +d ab +youn ger +soli dar +pp a +experi enced +b ella +bo ard +shef field +steph en +consu mer +sub mit +spon sor +t ang +ag gre +comb ined +trac king +sand ers +b az +survi ve +fer red +equ al +se p +re ed +str ong +priv acy +st ap +un g +ac ry +pa sta +pir ates +ag er +fair y +du p +introduc ed +wi p +let s +spr ay +ðŁĵ º +gre w +a sts +pitts burgh +new york +jo ey +lau ren +tra de +ch op +pi pe +cla ire +behavi or +v ap +cre ws +lap top +ðŁ¤ Ĺ +che ster +disci pl +d f +out doors +k s +go ver +super star +cas ino +far mer +; -) +re turned +ðŁı Ī +ma il +roa sted +co sta +v ill +pe z +gard ening +distribu tion +sh ining +inve stors +ra sp +dec ades +reali zed +bar n +p ti +st able +ut d +pan thers +m ens +b n +ca de +bu cket +yn n +when ever +wa ke +da is +ber nie +lo dge +ju lie +atmo sphere +ðŁĺĺ ðŁĺĺ +major ity +par ti +exc it +cu t +me h +musli ms +be gun +fli ghts +vene ss +ce me +po sing +so le +g ou +dark ness +pe ach +cel tic +auth ority +grand ma +ful ness +smi th +speci fic +gar cia +co ins +good ness +aldu b +recru iting +den nis +gar y +sle eve +weap on +pl z +disco ver +harri son +recruit ment +ja i +ch im +com pared +tom s +mo thers +am y +archi ve +t ask +ben jam +se g +law yer +al um +inve sting +mi e +che z +j p +a ke +fl am +wall paper +âĻ¥ ï¸ı +t ton +che st +favor ites +we igh +coo lest +r ating +relev ant +lo gan +ma ple +run ners +pri or +peop le +ma ur +terrori st +te sted +carni val +su spen +me asure +m v +cyber security +app ren +terror ism +o z +v ital +ni es +gon z +fun ded +twi st +assess ment +die sel +en for +colum n +ad dressing +ca sts +pay ment +x ton +fi er +, ' +la st +ne e +un less +clo se +sk ill +cuis ine +fun eral +ti les +a un +k ru +relation ships +ðŁĴ ¯ +ev ent +âĢįâĻĤ ï¸ı +kind ness +pro posed +acou stic +a es +defen der +dan ce +h tt +w at +vo y +ðŁ¤ ĺ +au s +cli ff +sear ching +beauti fully +in qu +at l +speci alist +ðŁIJ ¶ +da i +tra ils +class ics +inst ant +v ous +re venue +mar ch +kir k +fr inge +fire works +tri via +âĺ ħ +tr action +wal ter +mo to +l ily +att itude +cli mb +sc an +sav ings +c w +fa ith +cred its +ab led +gra ff +auto graph +he he +ran ch +ha d +ro gers +ðŁĮ ¹ +f in +re qu +fol k +ad ditional +lyn n +u ber +dol lars +lo gic +wor th +so m +the sis +p ound +bi c +st ur +cer am +spen cer +en tered +v amp +organi zed +âľ Ī +pp s +tr on +merce des +no ti +compet itive +do w +ous ness +vic tor +gr illed +na i +pu tin +ab ra +bl ame +alex and +anim al +dec ent +p ent +inter ior +:' ) +but ler +bal let +ðŁĴ Ķ +albu ms +down s +la d +si r +pla in +p ers +blon de +dis c +paki stan +se ment +ga a +w age +ch as +man i +co ps +terr it +lo l +lau ghter +ri vers +magnific ent +lam p +w b +new sle +char ts +ble ssing +p unch +lon gest +fl oral +cu tie +fare well +sto pping +mb b +bu d +chee se +de cla +si m +mc donald +de ter +you th +t ch +fre der +kin dle +fer n +at or +as leep +p ond +spr int +p ounds +la zy +gh e +fundra ising +dead ly +gran de +dou g +he y +lin da +consi dering +i um +gol den +vi k +auth ors +di ss +u ally +appropri ate +mor ning +y le +hon oring +foli o +be c +re bec +fin land +formu la +corn wall +sh ay +cau sing +bl end +sig nal +t ent +kash mir +nation als +har mony +sc out +acce ssi +he ight +medi eval +impro vement +ke es +prac tical +car d +de par +hu n +om ing +cal gary +ste l +bu bble +gur u +ma h +unex pe +n h +ed a +me at +i ge +si o +god dess +in ches +tun es +br itt +sti on +ra j +âĻ « +mer cy +ðŁĴ ĺ +sen ds +i est +pol ici +val e +reduc ed +as ap +vi jay +defen sive +celebr ations +ri ders +med itation +har mon +g ing + ¡ +program ming +in au +sud den +m h +replac ement +sk u +j ar +gra des +ta st +k itt +brand ing +k aw +boo t +f ought +p ays +g f +iz ation +ho p +k k +activi st +v end +coast al +cha os +ðŁĶ ´ +se me +bill board +li fting +cu mb +sc al +ðŁĸ ¤ +stru ck +l v +indie dev +beat en +jun gle +al right +destin y +m ing +k c +ch ances +om an +q atar +cra f +tra ined +pri x +char m +o tive +s mu +e c +and ers +hand ed +al ban +certain ly +arri ving +i ze +sa i +tr ack +pain ter +hu mble +appo intment +head line +manag ing +mo d +as pe +andre a +à ¤ +ethi op +un ited +exi st +bal i +k ad +n t +d red +re x +recogni ze +tam pa +be ers +ati a +he els +no te +transport ation +tur tle +re de +hipho p +sp icy +sp urs +⬠ĩ +cor p +ther n +to ast +hur ry +proper ties +ma ge +mar co +ele ments +bou ti +syn drome +ms g +develop er +gra ders +he im +re sil +off ices +del ay +di men +vin tag +barbar a +ðŁĺ ± +vene zu +cu lar +fac ed +bar n +ðŁĺ Ĩ +survi vor +wor m +confu sed +passion ate +Ø ± +identi fy +electr icity +sou ls +brad ley +repor tedly +lun ch +shel f +eli a +swee t +smoo th +emplo yment +am el +manhatt an +ste am +oun ts +ye p +li ving +un e +descri be +ca res +man ila +sha wn +ac ted +bas h +st even +re st +pet ition +div ine +wel sh +rac e +platin um +ðŁĮ ¸ +p b +extra ordinary +solidar ity +m all +on ion +schedu led +game of +fer gu +de ms +nor m +p k +tri als +polici es +publi shing +st ole +fron t +charac ter +van ia +ex ce +sti e +sc a +resi dential +sa iling +ðŁĶ¥ðŁĶ¥ ðŁĶ¥ +spons ors +th ick +champag ne +she pher +continu ing +ven ice +per th +na p +a ster +y ak +un limited +cho ices +ne o +hi v +repor ter +bru ssels +f old +dy s +se mi +la wn +it alia +wi fi +as k +em ed +fr ame +monit oring +ste ad +i da +gr in +is a +fli p +re stric +offen sive +atta ched +di sh +wh y +philli ps +gre et +p als +mix tape +v ou +fiel der +spar k +alber ta +g len +ca sh +s ri +u ri +ro dri +entreprene urs +climate change +p sy +d le +em ents +lin ked +nether lands +acci dentally +oppos ition +vel vet +ra ys +c w +om o +m f +lmfa o +newsle tter +: ) +toi let +liter ature +di sp +phili p +uni form +sudden ly +head er +cool er +-- - +prou d +bri g +nis san +scienti st +j ah +con centr +pac ks +appo inted +so ap +eng age +cho se +âĻ ¡ +se tup +jeal ous +har ry +g ation +tun nel +te mp +osc ars +dec ade +recomm ended +child ren +ab a +anxi ety +ve ments +sal on +pho too +organi z +mach ines +ab s +vil le +hy pe +ti ff +emer ging +av geek +[ # +contribu tion +bra dy +re sto +g mail +fit z +photo shoot +hel met +h t +eleg ant +ug anda +nur sing +or leans +pen n +na h +foo tage +em a +w o +w ad +concer ns +ve re +re mark +who ever +str ang +p t +qu it +sh ang +histor y +s ick +perman ent +ill ness +col d +visi on +he m +ar row +con vic +pin k +oc cup +bal d +ex hau +u of +am o +on t +ãĥ » +adop t +la id +smo ked +inter pre +ess enti +associ ated +b d +bb y +fi er +inst all +dipl om +con diti +c f +w ak +any a +gr aci +fi sher +s ss +ap r +il it +mus ician +symph ony +cor d +h ack +le gi +l v +bless ings +hum or +sc ra +e ti +min ster +trav elling +bu sh +jewell ery +li me +!! ! +pregn ant +pe e +lo b +cap ital +ip a +pen cil +la bor +duc ks +prou dly +wedd ing +dere k +m w +pe g +valent ine +an gu +re treat +pro spect +dang er +vul ner +up set +, # +sr k +x im +thur sday +n fl +kis ses +re ds +cr ack +re ward +c u +ko k +me te +aband oned +it t +me als +sp ell +stan bul +del ays +ru m +le op +gu m +no va +super man +ch ick +m is +dram atic +inno cent +r ounds +re c +auti sm +bangla desh +mor al +mo vie +sp oo +k la +âĥ £ +ou ting +mess i +ab road +loo kin +a im +q i +st ack +colla ge +à ¯ +hud son +sc an +ho e +ch au +oc cur +comm ander +ho les +ðŁİ Ħ +bi as +v on +stick er +ma k +responsi bility +colum bus +sa int +ed mon +rac ism +far ms +w en +gul f +may o +!!!! !!!! +corpor ation +ba chel +el a +inter nal +je ep +fol lows +di alogue +de rer +smart phone +he len +rich mond +equ ity +s land +b g +ne ar +av i +memph is +we ir +discu ssed +bad ge +p up +mi stake +phen omen +un ite +ðŁ Ľ +de pic +ri des +in augu +n at +sof twitter +comb ination +gosp el +âļ ¾ +ad mission +retro gaming +ðŁIJ ¾ +sch u +mb o +jun ction +al arm +à ¦ +gr ac +kh ali +k ul +m ale +cap tion +wi sh +te re +cor ps +ru bber +play station +er in +effici ent +l or +jo kes +in ary +nor man +lu is +inaugu ral +ch ed +âļ½ ï¸ı +di p +to e +str at +aa c +am u +pi er +co tt +comm and +tt en +sn oo +cu be +clo ses +class ical +s word +expre ssion +reach ing +n app +co st +affe ct +ric o +gi f +brea the +tri be +or tho +h ay +l g +fri es +n m +hi ding +richar ds +en de +mic ro +capit ol +cop y +ro m +regi me +mary land +tax i +di al +embar ra +un believ +ch t +v s +elim in +o dd +pen ny +sound track +l ings +trans ition +rema ining +a is +mali k +? !? +rand om +def end +ul tra +tru m +danc er +st ol +dri ve +a ver +ro ast +defin ition +se an +excit ement +partic ul +su rely +sh av +ber y +di shes +com m +is ol +i am +ob li +gho st +hugh es +chi efs +b as +conserv ative +speci al +fe min +sh ri +n ancy +inte l +tu ne +ðŁĩ ª +jo el +gg le +mo to +ðŁĺ Ķ +bu ck +d ag +antic ip +mont ana +gu id +fro g +ec raft +op e +dri ves +nu mer +x y +color ful +wednesday wisdom +illu min +bey on +inau gur +deep ly +pre fer +for tune +coo ked +ti ble +âĺ ķ +swe ater +it ter +tt y +u i +gi e +com plic +~ ~ +tax es +cu ps +di verse +sam anth +âłĢ âłĢ +ba king +sy mp +wa i +be half +mer cur +travel s +ðŁİī ðŁİ +or ia +eng aged +jump ing +reti red +n aked +p uni +speed way +sci ences +rehear sal +on ym +dy ou +pl ates +r ati +kri sh +jaz z +car ol +ra f +pen alty +tim eline +ru by +engine ers +ra f +bel le +do se +che on +esc ap +me g +ran k +or d +me gan +mer ch +ec lipse +âĺº ï¸ı +ple dge +kir k +per si +leice ster +sa k +w k +saf ely +yy y +je t +promis ed +j c +en ne +no ah +re no +re a +ðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤ +tra il +ðŁij Ģ +f d +soo o +ri min +w k +ภ² +i al +x ox +bis cu +d ale +fan dom +particip ating +fla g +privi lege +pe ach +mach ine +bo ston +gro ss +o g +mir acle +adop tion +u ss +mon sters +be ij +clar ke +pu shing +pra ying +ar o +d n +ell is +apol lo +od ds +refuge e +to w +b p +ðŁĩ¬ðŁĩ § +h end +app eared +memb ership +pe an +du m +viol ent +v y +potat oes +aw w +greet ings +t ts +ac on +sh ane +photograph ed +cra b +temper atures +cu ba +c fc +wel com +he l +in nings +m k +co de +kno ck +gra ss +swe dish +p ta +ick y +v at +lin ing +s q +sa p +ar c +announ cing +sk ins +cit yof +br ing +co x +gam er +it arian +i da +h d +ros se +sad ly +ge o +âļ ¡ï¸ı +tag s +fa ther +chan ge +l ance +whis key +adel aide +te c +stick ers +marke t +class y +bad ass +flo rence +lin er +fro st +k ate +ac on +scand al +es sex +ðŁĺ ı +vi vi +dr ill +blo ggers +recomm end +d ha +ac res +ro ma +bu y +gro cer +er ia +ma har +ff er +patter ns +ver i +com pu +st ev +ang a +ment or +do o +it ali +cdn poli +on ly +conduc t +elec tro +de f +wh ale +prepar ation +bicy cle +vi ral +turn out +bra ss +qu ad +hospit ality +pack aging +den cy +ceme tery +abo ard +dre aming +pic ture +t all +inv ent +ad mi +o e +tem ps +qu an +fun dam +pro mp +resi dence +mu d +sour i +âĦ ¢ +graff iti +gi f +d nd +com p +s war +pe eps +pale stine +devil s +san g +assi stance +bi ke +missi ssi +inter viewed +ne phew +dru ms +v and +gentle men +n sw +inst a +leban on +ee ee +oli via +ver y +rou gh +industri es +m ation +ðŁĺ Ĵ +bar rel +n ay +po ps +moder n +ill y +are st +on ents +protec ting +v ans +e o +vi kings +restaur ants +re ck +jac kie +andre w +w illing +he ath +citiz en +disc rimin +๠Ī +stu art +m ys +hi p +tran sp +" ? +te x +su shi +ke d +cro ssed +dist ur +pe dia +f ate +some how +mo th +proce ssing +is s +r in +u ts +yy c +ver t +lg bt +re id +on to +arab ia +habit at += = +stre ak +simp son +addic tion +wim ble +deli vers +challeng ing +ðŁİ ¶ +fran ch +e du +s me +ai ds +hur st +th am +tari an +remem bered +palestin ian +fe es +tru m +sket ch +ur u +fit ting +jes se +ðŁĶ¥ ðŁĶ¥ +---- ---- +ba ch +ici a +colo red +da h +associ ate +int el +s eller +p u +stu ffed +ac s +b s +sh in +cooper ation +certific ate +ab u +ingredi ents +re v +in ge +el der +christi an +bun dle +th ic +dir t +beij ing +comm it +ted dy +ed u +to day +s field +w yn +confir ms +lo o +j v +ene ss +al pha +vir us +ari um +gr ind +bri dges +introduc tion +pol ls +bac ter +z ach +termin al +ra iders +fla vor +zom bie +vo d +sp reading +gameof thrones +effici ency +lat ely +ale m +twee t +cri mes +cl er +de y +dg ed +hy un +pay ments +cir cus +ðŁĺŃ ðŁĺŃ +mis souri +lu b +episo des +c age +po s +mat ching +tumb lr +lin ed +ge st +am bi +nar r +ing ton +regu l +blo wn +is le +co co +on don +joshu a +tour ing +sm a +sau sage +best friend +bo eing +desi re +sav age +ra pper +de vo +te ar +take over +cow boys +po ker +par ag +pp e +h int +we ars +se th +ro les +l anc +man ga +form at +fl yer +c ay +mo or +ba ke +spla sh +v ad +ker ala +proce eds +sil ly +reflec tion +di str +wi d +su it +ci vic +yan kees +by n +migr ation +di stin +or ch +fe mini +quali fying +tu ri +o be +hun dred +cra p +wan g +mathe mat +bu re +expo sure +fergu son +seme ster +re serv +pl ym +a hu +fac ial +wa x +wor ried +ca b +vi o +as a +co d +to pics +p cs +hal o +rescu ed +horiz on +ar k +âļ ª +hol ly +el f +ul ti +pu p +quali fied +attend ance +ati vely +destro y +y c +for th +photoo ftheday +c ents +ic eland +meas ures +de sk +port folio +artic les +direc tors +dat ab +e w +creep y +oun ding +hon oured +mi st +j it +men tioned +port able +iti c +d ann +friday feeling +am id +ti ger +scri p +helicop ter +hard ware +expl or +work place +austri a +beat les +ber nar +spi der +disc o +cul t +lim its +shor tly +fin al +nin ja +lu ke +le bron +wal mart +o il +van illa +shi re +ye g +ak y +c s +bl er +collec ted +t g +rol led +speci als +b ff +pier re +sh im +vi er +flash back +restor ation +individu als +pro d +fre aking +tu rer +o a +re fre +mor oc +gre et +re yn +care ful +our ing +u sh +is d +g ill +vie w +thunder storm +b led +pic nic +guar di +pi g +ar k +syl vania +bann ed +u cl +vi jay +ori um +av engers +believ es +eu r +monu ment +concer ned +la bs +ber g +a ap +vi sh +sing les +can cel +z el +ar ab +ru th +too th +ar ta +sh af +chair s +r ack +dise ases +crow d +cl y +fle x +christ ma +artif icial +tom at +fin e +dra ws +advoc ate +fran ce +Ù Ĭ +ðŁĺ ³ +heav y +s our +compre hen +no ble +aa p +hin du +cor al +g ars +ow en +n l +st all +yel low +mar ina +in ver +suppor t +tou gh +promis es +pi e +master piece +sco re +for ce +mor tg +crypto currency +o x +r ors +rock in +pro vin +ho g +no stal +oak land +pat rick +inclu sion +tra ffic +ah med +a ha +lux ury +con secu +de mon +âĸ º +b lowing +st ag +: " +encoura ge +ben e +sku ll +do dge +bu ster +kin son +wit ne +er ror +lo west +fel low +à ° +sh re +bl ur +vir gin +compos er +sli p +mor nings +ga ins +tab le +gra in +ari st +braz ilian +w we +tu es +ribb on +an ag +di st +sac rif +em brace +entreprene ur +af fili +de o +t ali +touri st +fat al +ì Ĭ +autom atic +ðŁĩ µ +we ak +wel fare +confir m +benjam in +fi ghts +alleg ed +me ad +strugg ling +pro secu +che f +à ¨ +propos al +er n +ðŁĺ Ħ +dy k +on gs +hon g +m ack +mel on +on ent +ru sh +d ap +tol er +pro pag +c ze +trans lation +wal let +cott age +sa il +constitu tion +ðŁĴ Ģ +mun ici +fav or +storm hour +i h +ðŁĺ Į +approach ing +pin ned +j ed +niger ian +n ach +sh at +particul arly +mc don +camer as +anni e +admini str +he at +electr ical +char ming +gib son +bouti que +ex posed +ac tor +pil low +beach es +genu ine +margare t +ben nett +lou isi +pos itions +el y +shin y +ten tion +architec t +ren tal +ac qui +goo gle +sub way +mom ent +ðŁļ ¨ +ri m +metho ds +cy cli +nor folk +Ù Ī +over whel +ra pid +we ar +happy birthday +progre ssive +ðŁĴ ¥ +co gn +pap a +f ool +philosoph y +pol ar +jim my +wi g +ðŁĴ ĭ +oper ating +reduc tion +ph i +fla gs +to the +o di +a res +k oo +k ang +ar kansas +ash ton +wimble don +sci fi +attrac tive +mississi ppi +logi sts +ral ph +la bel +gradu ates +ma ha +home town +âľĮ ï¸ı +foun ded +on the +li z +trans l +mini mum +pre sti +ta m +gener ations +re bel +journ alists +par am +mc m +acry lic +death s +tes la +w t +bry ant +jer us +i stanbul +muham mad +ri ley +k ris +work shops +is o +coun ts +stre t +prote cted +trin ity +man ual +r hin +r il +pleas ant +le mon +ner d +har der +dar ren +bur y +ra h +bas is +mi gu +occa sion +li sts +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ı +e b +de cre +hamp ton +ìĿ ´ +tra vis +trans form +puer to +nh l +av oc +tri ps +unexpe cted +ve t +di dyou +bar ber +st ages +m son +re presented +for t +l al +pp le +nic ely +ignor e +qu il +qu inn +h k +carri er +remin ded +am ong +pass enger +el len +gue z +sc ape +mu ral +youn gest +ma sh +d ill +rout ine +stain less +jack son +gand hi +th al +on ers +edit orial +convers ations +sd ale +autom ation +i ke +า ภ+ðŁĩ ª +hau l +la ying +men tions +am en +abor tion +i bi +coun ties +ca therine +man ds +jam e +roll er +au t +n am +o logical +cep tion +ran king +tox ic +sn acks +victor ian +bang kok +psycho logy +re g +ang ela +respon d +sty le +sophi e +dak ota +achiev ed +mar ked +imper ial +in as +glo ves +sli m +confi dent +att acked +gg er +lon ely +valentine sday +re b +craft beer +orig in +zim bab +ce iling +te ens +other wise +w b +f ers +day sof +advis or +y ah +âĻ ª +en der +republic ans +av a +skir t +pi pel +chi e +jan e +ja x +ðŁĺ ĭ +âľ Ĭ +j ays +bre tt +bal o +cru cial +d har +as is +de au +lloy d +chat ting +âĿĦ ï¸ı +rel ay +remark able +n s +we t +bris bane +ðŁĶ ´ +tion ally +f k +la yer +house hold +consecu tive +es is +pend ant +st ir +crit ic +su gar +photo shop +pa res +arti stic +do dgers +c un +cra fted +am end +bo at +âŃIJ ï¸ı +egyp tian +sa w +tra ge +small er +ox y +pa ired +nex t +i res +tac o +o y +u c +st i +a erial +: // +dr o +dot com +gg ins +r pg +ay e +le an +stri ker +lo bby +prote sts +pri ority +congre ss +am ate +inv it +r ington +mom my +th us +allow ing +pione er +enfor cement +g ori +tal k +dra g +du mb +bul let +san ge +er y +tar gets +ðŁĩ ¦ +he ather +consi der +seaf ood +ve st +ris ks +% . +p g +sac red +he ating +kick ed +tto t +. - +chan di +co ven +po ol +pul se +i a +ro ster +shakespe are +es a +car go +pean ut +tro op +ac tion +tab let +home work +cast le +stru ction +mus icians +free zing +bu tt +justin bieber +j j +bah rain +an them +au dit +didyou know +na vig +guid ance +âĸ ¶ +tur f +n un +fic ations +ye men +char ging +x c +bron cos +su bur +p ale +bor ing +among st +for the +em per +om fg +p j +expe cting +ðŁĴ « +st l +ad min +expect ations +sw an +shoo t +oooo o +min ent +ãĢ IJ +wall ace +stan g +satur day +adop ted +dou bles +hom ie +ome z +d han +vent ure +surroun ding +fi le +mob ility +de es +w ski +broo ke +emb ro +re members +kar a +test im +bo tan +m tv +sacrif ice +jerus alem +d l + ´ +proper ly +ili on +as i +leg it +co pe +m cla +recy cling +lar ger +ðŁĴ ĵ +pat ric +gener ous +ja red +p f +mol ly +thom as +ju dges +h b +sor ts +bl vd +o ven +enter ing +plan es +be et +integr ation +boo ked +fre ed +ver n +ash es +to pped +de pot +welcom ed +ren a +m ick +d and +see ks +gam er +ran kings +ren e +mu t +whis ky +fire fighters +gu es +ga ther +tour ney +de men +y ang +new ton +autom otive +back yard +deta iled +mi st +to bac +fi ber +un usual +grat itude +sp are +ne ys +: * +per i +flo ating +fin alist +don ating +dre ss +bro ad +be the +econom ics +tai wan +ed wards +plu g +pra iri +val en +bab a +f ad +an as +har per +dis order +app lied +p att +bi kin +li ver +cu ri +carol ine +ann er +juli an +wal king +mal col +screen shot +co ding +skin care +activi sts +myster ious +ex act +blo cking +mercur y +bat ter +du mp +âľ Į +en se +li sh +ridic ulous +prote sters +ðŁĻ Ī +lu st +swe at +as s +ali ke +co dy +re ments +win ds +as pir +vi enna +pra y +.. .@ +bo i +cand le +assi sts +te e +der son +p ony +f ence +con spir +âĺħ âĺħ +oo th +e pic +ba rely +a unt +b am +diamon ds +end less +scre ens +can cer +gr o +p st +pro spec +mo sque +help ful +ou ri +bro ther +gu jar +cri sti +ine z +to wers +ad dresses +gra y +bur ton +re tweeted +ðŁ¤ Ķ +n ity +du ck +super vis +jo an +kin der +sanc tu +pi ed +âı ° +ł ï¸ı +m ati +reven ge +ce ster +eli fe +desig ners +back ed +bo li +wei ght +cou ch +su res +s its +shri mp +la gos +auth orities +os ity +hol ly +compu ting +fac tors +ab e +pan els +ram ad +sent ence +missi on +hol m +r b +d ads +shang hai +mon ey +she ets +sk ate +thre w +cup cakes +infin ite +l is +practic ing +ess ay +ka i +as ci +mo b +u gh +hol mes +re gg +ik h +mo ck +collec tions +pe p +o va +sal t +nan dez +co y +thre ats +tex ts +cin nam +pregn ancy +pen ding +stam p +flow er +g is +agre ed +pay ne +ro ver +ph ra +sof t +f fin +fa thers +pass engers +aw ays +al a +h es +li van +in s +samu el +ingu i +h of +j j +chen nai +cat al +om ic +he ath +ni ece +pump ed +integr ated +are l +no m +produc tivity +wan ting +vis a +di ana +tw il +it v +cam ps +ro wing +d ley +black and +gu ards +b ells +re verse +vi be +ric ky +mo ss +ny t +âĺ Ģï¸ı +el le +tro y +cu dd +ev an +women s +fo to +mi stakes +wick ed +mi l +c led +me mes +co smo +schol ar +ren o +ðŁĺ Ģ +v ents +# âĢ¦ +terrori sts +ca sey +cardin als +ðŁĺĬ ðŁĺĬ +venezu ela +bol a +liter acy +t w +en o +con tains +au stin +fin anci +ev an +har vard +origin ally +chev ro +her ald +nott ingham +manag ers +âŀ ¡ +accep ting +wal sh +tutor ial +entrepreneur ship +yach t +requi rements +glen n +pe de +unfortun ately +ach ing +dais y +gi an +night mare +âĿ Ĺ +r ina +b art +ema ils +oppo site +who m +sa ke +pu zzle +da shi +par ty +blan ket +bus es +lo re +beau ty +reas on +pun jab +winds or +func tional +exi sting +hel lo +gli mp +con vin +la k +scre aming +rebec ca +bli ss +north west +infin ity +cosme tics +pul ling +coffe e +pl ing +op ho +colom bia +interior design +( + +emo tions +sa c +sun glasses +sav es +d f +six th +al y +ðŁĺ » +de en +dev ast +polit icians +lac rosse +g u +pe i +jav a +comb ine +coal ition +er ts +survi v +ch ad +stri an +n n +de vi +coun c +concer n +contro ller +bre ast +j ury +tu m +introduc es +la di +mobi le +al z +ste ady +nur ses +h acking +on line +oce an +ðŁİ Ħ +a am +ju ven +ic c +louisi ana +ar te +street art +is on +wn s +fr m +p anda +no ir +main tain +del ay +symp toms +thor n +ge ome +ter n +carri ed +p ru +pan or +as sy +per u +clou d +sp ra +pe di +e ste +tag ged +ðŁĺ Ŀ +shado ws +naz i +ا٠Ħ +cor ri +âĻ¥ âĻ¥ +j ad +ðŁĩ « +form al +spo ken +ðŁĮ ŀ +enjo y +lo pez +out look +in ho +w ander +Ù ħ +ma ya +pe e +d ine +ãĢ ij +brief ing +suppor ter +ar ily +ght ers +natur ally +doctor who +j en +v ar +new year +re se +si mm +re x +con sequ +tomat oes +bur st +bra vo +bur gers +cr acking +nor theast +bi om +mush room +mar que +dou ble +ni er +v ag +tw enty +key board +win ni +jama ica +par ish +: - +mental health +ali zing +ren der +wa king +ðŁİ Ĥ +g ly +na than +wa shing +mel issa +jun g +loy al +chil i +song writer +guit arist +bo wie +neighb ors +onym ous +as set +ta i +head quarters +ðŁĮ Ī +i hear +ci gare +sur g +) " +re pl +dar ling +ðŁĻ Ħ +z ak +sa re +ãħ ĭ +mic key +ware house +mass age +ine es +did nt +i w +hur ts +eng aging +mag ic +women in +k itten +mor s +c art +tit ans +colle ague +compe ting +er an +k hal +mar ble +dem and +del ight +et ary +bli zz +lou ise +m ls +fini shes +experim ent +conduc ted +electr onics +itt ers +car ing +wh ats +sym bol +jun g +e cu +pi x +con text +char ger +ðŁĺ ĩ +re ig +fra g +ë ĭ +ch ad +tru e +ker ry +def ending +a int +au ton +check out +bar nes +less ly +d t +m me +clou dy +second ary +are z +_ : +app a +const ant +" ) +ve ts +jo b +i ent +ðŁĺŃðŁĺŃ ðŁĺŃ +m j +fren ch +di ver +davi es +hh hh +e book +๠ī +mar iti +bree ze +susp ended +mat o +vi et +ra hu +se i +bol t +en ary +le is +kar l +fr amed +expla ining +ab c +de aling +nat o +ja ke +exp and +leon ard +establi shed +du b +ar men +el led +voc al +nichol as +ori ent +k yo +illustr ated +ah h +danc ers +milli on +ge ta +po pp +as u +mur dered +gi ble +sto ked +gri ffin +maxi mum +adri an +en counter +ther o +david son +ðŁį » +holi day +ev o +asse ts +car son +memor able +âļ ½ +ob am +represent ative +cb d +tr icks +vo gue +vo ice +mm mm +sebasti an +cli f +ath y +par alle +ðŁ¤ · +pa k +ev acu +e ats +ا Ø +tou ched +organ ised +spir its +can ad +gui ded +frame work +ðŁĮ Ł +pe d +natur al +ag ar +replac ed +anch or +ti t +sha h +organ is +super ior +r n +ch ro +eric a +st ill +cor on +chu ck +loc ks +or gan +ro sen +sc am +ben ed +/ # +ke en +tre vor +vamp ire +sor ted +! ' +af ford +in tro +gr ace +ðŁĺ ľ +sau r +kick starter +influ en +v u +y up +po c +ðŁİ ¥ +a ar +s ang +tre k +et sy +tb h +scre am +chevro let +pix el +shepher d +an or +gabri el +tw ood +sd cc +me ters +develop ers +clo sure +v w +twit ch +ì Ĺ +se oul +pr ice +ho g +n ish +hill ary +scrat ch +in cen +wag on +dis ability +pan ther +ch ats +g d +wit z +sus sex +l ate +den mark +ger ald +cancel led +net te +i x +nav al +bap tist +te t +y ad +ma th +ho y +r andy +po int +intel lec +fru its +w ool +gu in +pr on +the ft +con dem +mar ry +n ola +architec ts +cin cin +roc kets +gentle man +ex plan +t ate +do e +ra ises +wild life +w l +insi der +blan c +w p +for sale +ny c +po well +unbeliev able +pen s +goo dies +mu stang +p ens +st ays +squ ash +xox o +near by +ever ton +co co +le agu +k han +stu d +south west +con struc +s worth +cro atia +le a +su ms +aim s +e an +van ess +iti ous +pa thy +arc ade +b end +sugge sts +sac ram +roy als +ri er +em ir +in cl +an k +clar k +ri ght +vac c +ठ¾ +tan e +li b +u sc +sal es +hu h +s ally +ver a +p ga +gro ws +dru m +tre e +eth ics +sug gest +is ab +se aled +pre viously +anim ated +ab du +ri ses +glo b +pre dat +scar f +del ic +om ar +ll i +sx sw +py thon +ne bra +fun k +reflec t +pav ilion +tic ally +ch asing +bak ery +inva sion +ko h +believ ed +co hen +con qu +cra fts +nat i +cle ver +govern ance +sam ples +fa ils +â Ķ +ti mo +r itu +stri king +inclu sive +sho cking +can t +requi res +dra wings +à¸ Ń +purch ased +du m +z ach +war ner +con sole +man sion +foun tain +circu m +e sh +is land +mil k +pro fits +hali fax +ri val +âľĪ ï¸ı +jen ny +sand ra +ny e +k elly +y al +qu ad +no s +inste in +fin alists +mid fielder +cu e +excep tional +a an +sa pp +gett in +sa a +f ati +sl ice +vol k +s wal +la sting +sum mary +it as +sm o +s z +âĺ Ĩ +ip l +fl ames +ene ws +ha v +hoo die +pitch er +win dy +re vol +centr al +ton ite +ðŁİī ðŁİī +sol ved +mil wau +organiz ations +wee ts +re fin +s th +ãĥ ¼ +el in +ton a +cinnam on +ðŁİ ¨ +ðŁİ ģ +ron aldo +pen insu +ome ga +el ds +desig ning +e igh +blu et +ben z +nu g +ash a +robo ts +su dan +choo sing +en do +ser ge +clo sely +hand y +fing er +be ing +ar te +survi ved +fl ame +mile stone +gu t +d war +fu tures +é e +el o +fri dge +eli c +ou ch +u b +p v +tit an +col lar +st ation +nev ada +aur ora +r d +dun can +âģ ł +bri en +mar sh +Ð ¾ +to tal +ch ry +s ers +su ffe +ra chel +colle ge +to days +cour ts +ch it +re united +gym na +gen esis +be side +re presentation +ch ant +collec tor +ra k +ath ens +ni gh +mun ich +langu ages +fl u +particip ation +__ _ +c v +spec trum +so da +co ver +refe ren +ab bo +ap a +public ation +ed m +mon ica +ar my +ðŁļ Ģ +div or +dr y +stre ams +robo tics +ci der +bull ying +appro val +sto ke +plat forms +sier ra +ex tin +i b +ha yes +succe ed +suff er +at ically +da i +lyn ch +h ound +del ines +ack now +d ated +exclu sively +he res +fac ilit +dam aged +char ter +la kers +fal con +unve iled +wel ove +e ase +pati ence +l one +gent le +gene tic +produc ing +g our +shann on +bil ities +zimbab we +p int +dau ghters +liter ary +bel le +cl am +surroun ded +k any +ne il +pir ate +rang er +hb d +nat alie +bel ong +olym pi +emb assy +sc ol +en er +ak in +lo ren +b h +: / +di va +den im +hi pp +ðŁĩµ ðŁĩ +arn old +? ' +we ren +em power +dis abled +man or +rasp berry +b af +aw ful +dru mmer +kar dashi +n ash +machine learning +ch u +rebel s +tim ing +mon roe +ton gue +ran ge +pup ils +re ss +amaz on +b z +har ley +pal mer +ballo on +s ings +ic ec +j b +c ers +g ps +whi st +ri se +l t +oo oo +c attle +shoo ter +vod ka +uc l +mt g +le sli +jon as +di spo +at ric +ste in +vintag e +fir ms +flo yd +cow boy +soo oo +is aac +war craft +disney land +beauti ful +be am +franch ise +bu n +k ag +an on +tur bo +swee p +made in +kar achi +dete ctive +penn sylvania +contro versi +vitam in +a side +chron ic +descri bes +remo val +ha h +ap er +ten ed +u to +bad ly +mir ac +f ry +ye a +in jec +ther mal +comp act +th or +te ed +ur gent +l ite +g illi +sop hom +ic o +che m +p m +for k +fre ak +ch ak +recipi ent +i y +ni k +model ing +c ans +ðŁı Ģ +del ux +se am +surviv ors +rad ical +investig ating +reli able +f m +tur t +ligh thouse +to ol +go wn +) ) +bo ts +auto graph +a id +bu ffe +h mm +horri ble +ssi onal +ann i +๠Ģ +k its +sch i +eter nal +hu ss +sens itive +r u +tast es +chec ks +im o +por tion +sk ate +e den +half time +fri ed +ri hanna +ti se +fl ick +ca in +s gt +âľ Ķ +sh au +sta ined +ra ffle +dro ve +sal man +princi ples +sh o +ar u +je ss +gu ine +gar bage +my an +jel ly +dis ru +z ia +q ld +ent ries +la v +fle w +ad mit +objec ts +comp are +ny times +cann es +p n +suff ol +ro c +d ana +e gg +hi st +coun sel +' ! +phy si +imag ination +ad just +explo sion +plym outh +hor ror +elli ott +bour ne +de x +bre ed +au dio +lob ster +disappo inted +nation wide +( ( +incre ases +austr ali +ce dar +star ing +rac ial +e is +g mt +visi ons +stay ed +discu ssions +de an +cur tis +mai den +stel lar +happ iest +h wy +pre season +car av +mon days +hospit als +glimp se +schol ars +ja i +ter race +ann a +goo se +gra ded +lot us +hun g +grocer y +stam ps +emper or +sc oop +in ser +c as +exist ence +he al +fal cons +mar vel +reduc ing +terri fic +magne tic +perfor ms +bar re +p us +tre ating +ic on +w h +decla red +tra uma +do d +come dian +nik on +bu gs +as m +mont gom +ibi za +comprehen sive +ha s +san ti +fellow ship +da sh +p sal +louis ville +sp y +fau lt +d the +fi led +vi sta +de sc +fe ars +you tu +sp s +es p +ri g +cri me +ber ger +wonder land +k ent +in formed +stev ens +my th +ast on +ir i +visit or +at ri +produc ers +al la +person ally +separ ate +agen cies +af ri +il an +spo ke +n ina +squ ad +di ves +de pend +li v +fier ce +enter taining +cha in +sc at +bor ders +pal ette +sp ro +os is +der by +tobac co +zi o +willi e +ju vent +zoo m +hol y +enti rely +af e +mart inez +be ds +pe a +bull dogs +ðŁĩª ðŁĩ +ib m +ne on +ethiop ia +team mates +plan ting +tw er +any time +for bes +ó n +run way +ner vous +ro ger +p ile +ch anc +apo caly +u w +o i +dr ought +territ ory +br ick +cre atures +go in +w aff +gre n +sou theast +je an +am bul +ed ited +stra p +c v +aar on +ãĥ» ãĥ» +t su +descri ption +kin dly +clu tch +im mer +en or +women sday +or ange +ra g +ob vious +hy der +chann els +man go +me yer +ra ining +ge tty +pil gri +coordin ator +up load +ninten do +don uts +san chez +app arel +j r +zz i +, @ +jeff erson +accessi ble +great ly +e id +initi al +budd ha +par is +ma scot +â¬ĩ ï¸ı +sch war +si ri +sp inning +mortg age +e cho +end ange +ge dly +chlo e +enh ance +kar nat +k ry +explo res +ðŁĴ ģ +af fair +ic als +all a +dar t +dolph ins +diffe rences +squir rel +au gh +dr ones +ell en +re store +pa w +un for +pi ke +hil ton +colla b +consu mers +co inci +out comes +pp p +a q +coup on +li est +si ms +k ho +av es +spo on +pu dding +cor byn +hat ers +ex ams +sla ve +. ! +p sa +app les +tam il +se d +co ke +zz o +lo sange +car bon +cla ir +... ) +k hu +cra ig +explor ation +sanctu ary +su e +al way +demen tia +won ders +super hero +pakistan i +brown s +bluet ooth +lo cker +mar c +ev entu +delux e +rodri guez +âĿ¤ âĿ¤ +ro bb +ðŁĴ ¦ +lin ux +ten s +intellig ent +se ed +vo ter +s ler +pe aks +inter n +teen age +peninsu la +hand ling +ti e +cou sins +wen dy +me e +à¹Ģ ภ+din o +ðŁĴ ° +ðŁĺ ĥ +ze e +s bury +trage dy +b k +bo re +z in +war ns +idi ot +tou ching +contin ental +tac os +saf ari +wa shed +po dium +morri son +fore sts +c bc +al on +partic ular +be ads +inv ented +lo ch +li ghter +where ver +i de +docu ments +a we +k r +no where +min er +st it +ro x +contribu te +har dy +cl an +ob ject +ca it +ðŁĴķ ðŁĴķ +happ ier +vege tables +t art +g ag +nom inee +heav ily +pan ic +j d +there sa +at m +u ph +s fc +su ri +drin k +n al +re vel +k l +avoc ado +nom ination +ma donna +shar on +malcol m +control led +sh ers +revi val +legis lation +shoo ts +n in +comm entary +pro s +human rights +str anger +mit ch +pipel ine +leg ally +th u +gil bert +tol l +gran ted +gh s +ir anian +refre shing +du k +ab i +pri me +jose ph +mo sa +stati stics +produc tions +mer ry +pat el +sa x +human itarian +struc tures +e missions +town s +fre el +ster ing +rat ings +alle gedly +cab in +st l +w ade +fl yers +tri m +promis ing +z u +bal lot +compar ison +free ze +ou ter +great ness +as sign +snow y +r ale +tor ies +med iter +kno ck +consult ant +cincin nati +analy st +sc oo +je ws +appro xim +pu re +portra its +cy rus +ation al +lo ans +acqu is +el u +accep table +uni on +water color +ru st +batt les +per fu +seas onal +ser ial +mind set +ri ot +fel d +enni al +clo set +pri est +tan ks +int l +scre w +bu m +ab dul +ou x +expla ined +ric a +imag ing +law yers +bu ried +ãĥ»ãĥ» ãĥ» +ear l +âĢ ķ +l ton +resto red +stri pes +fo ss +de mands +ste aling +alex is +mun d +ak er +ur us +war dro +hu gs +gen re +e go +Ù Ħ +particip ated +bab es +ban quet +ti ous +he mi +ds b +lo st +milwau kee +jen ner +ge m +ou tra +lo ses +id i +re ps +ðŁİ § +regu lation +fla w +f ang +vibr ant +ram p +ra ins +well being +so viet +vie wers +de po +libr aries +bi go +ser y +g ill +de struction +co z +c x +bri dal +al ds +plan ted +amate ur +lu d +che ering +show cas +pro file +i u +ver tical +pack ers +wiz ard +ski p +s light +be au +air ways +mu ch +re ra +ðŁĮ Ĭ +ab sor +pati o +pack ages +s ells +ment ally +ðŁĺ ¢ +reyn olds +k are +tri bun +wal t +kn it +ta ste +sur rey +boun ce +cre ature +b are +bet ting +su re +mi ley +laugh s +al ore +cy n +t l +arti st +ann ah +war mer +dynam ics +lunch time +mariti me +vulner able +ðŁĴ ĥ +wol ver +dur ham +const antly +am in +si bl +: @ +bul let +k ach +angel o +wil der +doo m +desk top +law suit +k ca +hen derson +inv iting +bet ty +ta wards +ra fa +le aked +and i +ge ms +af l +vel o +mediter ran +pro be +to tten +steph anie +sn ation +com be +q s +over come +assas sin +ra v +fil ip +winni peg +sh il +determin ed +k as +ou tre +regre t +gui des +aa a +ðŁĺ Ī +wi ves +mani fe +er ly +sm y +sh ima +x ing +pix el +jac ob +ac commod +to y +on o +po o +ti er +an swe +ðŁĴ ģ +ro sa +le ase +bel ongs +th ar +eventu ally +nei ther +go a +ski ing +at ra +ag h +broad casting +f ury +py ram +d ice +volk swag +wom ens +provi der +bom bs +miss ile +whi p +d ick +nor we +back up +el der +mat ure +concer ts +gi ous +sque e +good morning +bra ves +^ _ +au ssie +lun a +mal es +he ck +for tn +rome o +steel ers +p n +pe er +re presents + « +kat y +migu el +requ ire +cha ins +l ur +immedi ate +ti mber +âĸ¶ ï¸ı +advoc acy +ex port +an z +tiff any +auth or +ðŁİ Ī +du des +chil ly +hi d +har m +bu g +mon ster +terri er +tu c +story telling +ta k +in ti +immigr ants +b is +reach es +com passion +john ny +contribu tions +ðŁIJ ¶ +mechan ical +impre ssion +ran ks +ko be +men ting +bloss om +pab lo +buil der +bom bing +tw el +sul livan +om o +pe te +de mi +ku dos +w bb +t gif +mass ach +neighb or +che fs +eng ines +pun e +ga ined +phan tom +s days +ext end +gr an +cent ers +jac qu +dat asci +sleep y +el vis +answe red +s lot +con y +flexi ble +ti ally +le tics +% , +andre ws +si ble +mom ma +vin o +do x +invit ational +twil ight +j ade +ill ery +joh ns +f ou +p v +-- -> +break down +billi on +prin ter +mon d +c bc +mag gie +legi on +du b +kur t +po or +paren ting +regi ons +bikin i +be ware +si onal +au burn +kid ding +amp les +sp an +con tempor +c ic +ha bits +ak o +pre fe +bud dies +it z +em ily +person nel +moun tain +ver sus +ðŁĺ ¬ +ear ning +s ink +dar i +u u +s win +i ster +bru tal +n ac +kat a +clo th +am and +ðŁĶ Ĺ +ne o +alu min +week ends +nebra ska +co des +delay ed +brun o +pro ven +in c +i ght +fl an +or o +lam bert +regu lat +w f +massach use +kardashi an +bern ard +fi esta +volcan o +grand pa +anc a +d re +st itu +mean ing +fo am +au ck +at ed +r l +hot el +pers ons +dy nasty +ell or +ma i +am ne +sty ling +avi er +e g +vege tarian +, âĢ¦ +foun ders +sta in +g d +cy cles +sky line +trac tor +exi sts +tra l +kid ney +mar il +inst ag +se tte +addic t +tri angle +flash back +controversi al +z on +p ins +i as +tr ay +town ship +deleg ates +sp am +h ms +cr ane +peop les +o lo +fac tion +but es +on ica +deleg ation +new profile +eli er +mc a +w and +g ely +losange les +ber ke +ti ve +dis rup +zz a +cas a +jor dan +ford shire +ga thered +ic hi +atten dees +à¸Ń ภ+pe ppers +co in +bour bon +ern ity +ro tary +behavi our +jere my +team work +compli ance +tre mend +ðŁĩ § +bu hari +cam bo +bu yers +ha gen +bu ds +bay ern +mon te +sm ells +an za +ath lon +descri bed +work force +gi ving +ap i +invest ments +da il +sel ena +datab ase +th um +mor tal +stu dent +bu yer +do ver +gar ten +att le +loy alty +gen oci +holo cau +theat ers +ru ling +ven us +pat ent +ch un +ab by +awa ke +mass acre +bang alore +break ing +simm ons +ju sti +hal e +ed chat +gg les +haw k +mar king +head lines +stro m +co ve +breath taking +med als +hair cut +christ ine +tele graph +gujar at +ju ra +can e +sho re +propag anda +mu eller +.... .... +sa vi +stom ach +thro ws +ta b +war m +j ong +reno wned +hi r +ra is +mush rooms +guaran teed +bo a +m j +revolu tionary +certi fication +bru ins +jo in +w es +pas sport +c g +sex u +cap able +w v +ton es +jac kets +ac compan +spin ach +fore ver +bla ir +wat ts +g l +cou ples +prairi e +newprofile pic +logi stics +massachuse tts +jagu ar +o id +we al +under water +mo z +y i +ma ths +myan mar +pre ps +suffe red +tr ace +wal i +ah hh +bor g +st itch +cu lin +real ise +infe ction +discrimin ation +sh ame +an kle +hu mid +y t +brac ket +tru ck +tri u +ea ster +commun ity +post card +invol ving +ty ler +car amel +over view +ex amples +integr ity +base ment +instru ments +ani um +at us +gh er +laun dry +achi eve +gen eva +pr icing +hyder abad +beli ef +me ta +j aw +accoun ting +lead er +cristi ano +cou ture +cy p +vis ed +, ,, +k nu +h ick +break er +br am +ra b +mo or +ham as +gradu ating +pupp ies +ak h +ta h +ach es +ri e +op ini +g ta +re ign +tra gic +re ver +p ill +pine apple +tou ches +da re +le ys +il o +inter iors +sc outs +bar t +en zie +don o +bro ck +christi ans +ense mble + · +cine mas +new port +air line +win ston +le igh +cont ents +pre scri +ur ge +tr out +fic ally +il ia +sub si +are r +âļ¾ ï¸ı +w ounded +ðŁĻ Ĥ +pe pper +ðŁĴ ŀ +fit ted +af f +re sur +thursday thoughts +z ero +archae ology +di v +je e +i on +awa iting +co zy +beauti es +bal d +dat a +gri zz +stal k +kin ds +cle ared +jess ic +regu lar +ali ens +plac e +bo s +bi zar +thisi s +ðŁĴ Ģ +totten ham +ma fia +s lam +ari ana +car roll +back pack +care y +uni v +r g +pe p +dig it +tatt oos +ag on +volunte ering +diffe ren +consu mption +ka thr +head phones +t shirt +o b +ele ment +re tail +sh ru +al gori +contain er +consci ous +fi l +com ing +ra sh +u rope +def ine +gi or +femini st +flow ing +rout es +gl aci +fer t +somer set +ant es +twee ps +$ $ +h our +endange red +year sof +ro h +po pped +bac king +ba sil +bra ke +mon aco +lgbt q +pra gue +ut ility +cas si +gate way +haun ted +sch ul +ðŁİ µ +shou ld +walking dead +comple ting +dann y +montgom ery +pengu in +ss i +mer chandi +ðŁij ij +chur ch +h ates +cap tain +brea thing +ce t +fair ly +approach es +compan ion +surpri sing +kany e +pe y +hin di +targe ted +lor ds +de ut +di gging +ger man +ru t +ener gy +close st +y un +apo logi +ภ± +s ack +ru p +dd y +port al +d ough +b ats +ðŁĵ ° +at ur +graph er +pi res +mo tors +ðŁĮ ¹ +j c +dan g +tu k +clu e +us c +pag e +d less +bro ws +ju s +ad ing +re marks +oo m +car dio +ste fan +arm strong +âĢ¢ âĢ¢ +ni est +belgi an +bi op +so y +lo f +í ĥ +q t +flashback friday +ce e +ģ ภ+wre ck +mar ines +amend ment +wardro be +vo y +bur ned +guit ars +ra inf +li fel +ssi l +oun ce +exter nal +c key +me sh +she ikh +inv itation +sugge sti +pop corn +phenomen al +an onymous +tun a +chic ago +o val +del y +loc als +( & +pro f +no vel +fin der +spar ks +la ven +in fu +nic ks +qu ant +ra e +exe c +dist ingui +st ances +mu tual +sh al +unve ils +edmon ton +zan ia +a dio +vie wer +brad ford +audit orium +qu is +re act +htt p +l ero +chee ky +impac ts +ta k +ed t +desper ate +t ay +ì Ħ +sett le +bar gain +resu me +un ite +thro wn +ke st +se ys +mar ching +am it +decl ine +sch ar +me tr +stan ford +lin ke +ber ra +dol ls +rug by +jam i +b or +road trip +dino saur +mi k +sun der +re m +b k +over seas +nau ghty +imple mentation +iam srk +lun cheon +fir ing +mi ami +pere z +the e +z on +gi fted +con version +ceram ic +¡ ï¸ı +pe dro +ì Ĩ +v ick +! @ +he ed +si d +b w +docu ment +pl un +gr ants +fant asy +predic tions +vali d +car ved +gradu ated +ðŁijį ðŁı» +nation ally +ch y +af l +re sso +blan k +ri vals +j ig +e ties +om ics +une mp +b ound +sk o +inspec tion +par al +high s +cri sp +b ans +ob a +[ @ +co spla +costu mes +rec all +mou th +ni gel +b ts +ter a +ko v +do cs +west minster +dic t +gra vity +kar i +ro gue +t ted +war k +ida ho +w end +aw i +queen sland +proce sses +cli ffe +m ick +com pens +op ol +the y +cl ari +wiki pedia +salman khan +haz ard +pre ston +swee test +pd f +che es +tr ilo +south africa +bur nt +( $ +con tain +t p +sub mitted +sound cloud +at u +re z +word press +corru pt +n f +ma ker +í ķ +par as +adv ent +ri al +ca fe +fo ssil +!!!! !!! +co ws +c j +sp ur +institu tions +land mark +ent it +re ut +h is +alz heim +we mb +regg ae +mo squ +st at +identi fied +deal er +re am +re land +ten sion +ðŁĩ © +wra pping +deep er +fr at +red dit +ar is +moroc co +.. " +b low +ma pping +pri orities +ing a +swa p +re wards +conspir acy +creati ve +c j +congre ssional +vau lt +ple x +sophom ore +shad ow +ele ss +ðŁĺ ħ +dar ts +aldu b +anno ying +pro ps +n as +alumin um +h bo +offen se +j ill +oni ons +la ur +ta e +har dest +sh ro +ga ining +meas ure +ed tech +cyp rus +tar a +ang eli +car lo +go on +all i +im plic +ju pit +resil ience +ha il +bal anced +) ... +joy ce +gr a +th eli +defin ed +shi pped +main ly +min a +l m +sac ri +o ber +p im +claim ing +ent ers +co rey +bo k +cri ed +cool ing +dani elle +pharmac y +thor ough +ca ke +k lo +outre ach +z ens +digital marketing +val ent +sn p +her b +mr w +caf é +cap tures +no tre +triu mph +pan cakes +cu mber +spi ke +d ation +bi gg +sp er +crit ical +am al +too th +foun ding +a stro +' # +quan tum +th ames +un c +pri de +air bus +kno cked +un defeated +mediterran ean +cal cu +clo wn +sens or +ham mer +for give +cu shi +ber ry +maje stic +elec t +polit an +g ta +k ari +bur ke +sea hawks +volkswag en +re i +landsc apes +cas u +grand father +list ened +/ / +star trek +rainf all +fur ry +vi er +star k +rif le +ff a +leg es +hillary clinton +min us +correc tly +architec tural +pre ce +up side +box er +ðŁĻĮ ðŁı¼ +is ai +de t +pro vo +tis sue +spoo ky +ve led +re con +prospec ts +que bec +âļ « +ig no +anat omy +shap es +w p +p interest +hor e +an es +pick up +ti p +pra desh +hu gh +co e +po k +gram my +well ington +sti gate +ri gh +lea p +king ston +scen ic +go sh +v ani +au g +s ary +zi er +bure au +lin son +con te +fra gr +all an +g aw +lan a +colli sion +surve ill +ren ais +ar range +s ali +do in +br ance +bren dan +our se +in coming +suspen sion +à ´ +l la +educ ators +in tri +da e +bio graphy +bul gar +villa in +go thic +rw anda +e w +may or +meet up +democr at +mor gan +su dden +te sco +car rot +bom ber +mck in +re ne +fun day +agricul tural +haha h +show time +form ing +col a +scor pi +quo te +po ppy +s life +d az +tu b +ne n +mo t +ðŁĺ » +s ore +elder ly +o ve +skin ny +um i +anc o +man ship +we re +g v +k ah +fol ding +ne at +samanth a +dan ish +uk rain +humid ity +nu tri +jak arta +cand les +oooo oooo +at ile +streng th +i bra +bap ti +charle ston +fr ames +girl s +clear ing +glu ten +# # +super natural +ju bi +ph one +he in +dr un +le ak +invest or +y er +dom ain +ball room +mi sh +app li +off shore +bla ze +dor o +âĺķ ï¸ı +win ery +shar if +ad ore +n ir +saf er +si gh +as cri +strong ly +trac y +ck er +ol l +faith ful +ey ed +deli ghtful +vis m +karnat aka +tit an +wh ar +jer seys +re fur +heav en +gri p +pan ama +pre li +glu ten +o dd +cont ent +pon ti +tion ing +e commerce +feder ation +flaw less +ge ar +ti res +by r +pol ice +cu ban +tri butes +tic ul +chur ches +nur sery +di aries +muse ums +snapp ed +i van +wi ght +touri sts +ramad an +t rent +prophe t +won dered +focu sing +hi d +ic ons +i q +ambul ance +pi st +fun niest +time less +sr ilan +bu ys +ki ds +colour ful +a shi +ch ir +mu m +ðŁĵ ļ +let ter +x en +reut ers +pre serve +in ting +ste p +fu ji +uni ver +i u +show down +po ems +surveill ance +suspec ted +ta e +sol ving +tom b +mother sday +car pen +recru it +pil ots +bro c +mix ing +fri days +ty r +represent atives +tra pped +abdu l +free style +clu ster +âļ łï¸ı +k d +sk ill +pit t +ex o +commer ci +muse um +loc ally +g ina +no bel +immun e +fr ac +cap su +main ed +attemp ts +bull dog +be spoke +sing ers +sp elling +seg ment +nat ures +tic k +lip stick +clean er +gett able +preci sion +âĢ¼ ï¸ı +th ood +re ef +no pe +bill y +di gi +mu si +ri val +figu red +tal ity +sun ny +ber k +aw ww +awa its +un real +co pen +asy lum +ex otic +bu en +mo ck +en able +arch y +fr a +pla stic +al mond +amp li +displa ys +abbo tt +s me +x p +ðŁĻ ĥ +graph ic +i ved +mar a +cau tion +lea ks +en berg +ul u +unic orn +cann on +appren tic +ðŁĺĺ ðŁĺĺ +b ball +wil low +at ics +am as +manufac turer +campaig ns +port ers +flo ors +l su +ty pe +ke j +honor ary +it im +to le +min ecraft +d x +ma sh +ri o +consequ ences +ron ald +go ssi +suffol k +mu se +r bi +live music +i van +ðŁİ ¤ +le u +patri ot +man it +lan ca +home decor +de ar +sig ma +ti de +str ings +v ita +sequ el +try na +inve stigate +bor is +ve gan +barri er +mind fulness +web b +hu stle +in da +tan zania +str ay +tex as +c ag +diagno sis +wom an +g w +ob session +l ative +nu fc +fl ynn +moment um +sof a +wal d +vege table +tu cker +supp er +se ab +ar ro +se ag +ven ting +counc ill +sp lat +cal cul +.. # +com fy +odi sha +sto pp +war fare +ca es +à ¨ +co y +price less +in sec +ðŁĺ Ľ +contro ls +empower ment +datasci ence +per pe +gen ic +e res +tru deau +man o +sla very +expand ing +ma he +fa iling +s aga +photograph s +cre st +re on +surf ing +hi e +ðŁį Ģ +ja e +fel lows +south ampton +sol om +ce ster +tab ility +hor n +se ct +he e +cole man +at las +explo rer +consul tation +copy right +organi zing +den ied +mon keys +noo dles +br is +fl or +dou gh +bon ds +sho cked +eco system +care fully +w m +apart ments +cur ve +san diego +must ard +comm en +cere mon +e ch +ru th +ðŁĻĮ ðŁı» +hawa i +fil med +te ar +as ingly +ca ir +wat t +instru ment +ou tta +ye ol +river side +ë ° +. : +nor wich +alo g +migr ants +new man +ri de +spr ink +targe ting +beli eve +tor ch +reflec ts +per mission +ff man +ene mies +bas ics +se ized +sun days +le i +hass an +en do +h c +st ad +le ments +kk kk +nan o +shar k +man a +on ic +treat ments +ear ly +collabor ative +shu ttle +bran ches +mis ses +mained cm +ap ers +ky le +carri e +leis ure +sh et +bir ding +adv ances +ðŁĵ Ŀ +popu lar +di ane +a be +re war +neigh bour +k pop +remem brance +play ground +ru b +krish na +e bola +inqu iry +ep a +lu min +organ isation +abra ham +norm ally +pre ten +jan et +w t +ðŁĴ İ +encoura ging +a stic +bu mp +syd ney +s z +ss ss +gar rett +ðŁĵ » +consul ting +roman ia +spo tting +chanc ellor +ar ma +presti gious +ðĿ IJ +t ad +cry st +compe tit +rati o +cat aly +bro w +j ur +vi king +commu te +y day +la yers +du mb +esc al +genoci de +f ill +gu pta +ste pping +se i +fo to +wild cats +col i +projec t +ear nings +st r +ge ons +comple tion +b m +decor ated +craw ford +af ghan +sc are +visi bility +hi b +direc tion +stro ll +christ ina +alter nate +cl are +sty list +be hold +s ance +leop ard +acqui red +narr ative +ash i +the a +?? ?? +pe as +at ch +sli des +le en +renew able +eng lish +qu ir +co aster +r x +fo ols +match day +mis m +amaz ing +z ig +ke ting +won t +to wel +di ab +sta ke +n m +mel t +e than +gra pe +polit ician +sm en +í ĺ +re o +wedd ings +cat cher +or acle +me mo +ðŁĮ ´ +ec k +rob bie +norwe gian +oper ator +am or +se wing +ju l +x ie +u v +fif ty +me ga +tatt oo +liber als +u pri +traffic king +richard son +su v +ki p +mess y +tremend ous +gl ou +cour tney +la d +stere o +my ers +i dio +^_ ^ +man ning +dy e +w d +thr one +jun k +as u +provin cial +k ook +wr c +fine art +hamp shire +renais sance +b red +fall out +s j +sn l +al am +tor ture +fy i +sh ines +pa w +ch ar +hen ry +c row +aci ous +di an +pa ige +ba re +stock holm +scen ery +ðŁĩ · +jef frey +pu sh +decor ation +ne d +cu te +brig ade +laven der +inv ites +e sports +vo ir +dri ed +tran spl +sur geon +no vels +pul ls +son y +lun ar +man e +i vy +fru str +dor set +sa i +tor res +ssi on +shut down +suggesti ons +writ ing +e o +battle field +u ga +ðŁIJ ¾ +vac u +spl ac +g it +u g +high land +% ) +mer maid +sacram ento +ta ils +p w +ka h +t ell +enh anced +ì ķ +auck land +cru el +ðŁ¤ © +au dre +sail or +gram mar +g love +de on +infl am +fresh ly +k ell +zi p +christi e +mil d +di xon +instru ctor +g ence +ãħ ł +sub jec +constitu tional +crow ds +in visible +ru ins +da k +si p +pla que +p ouring +comple x +z ine +ste ad +f let +trans mission +lo way +ar un +incre asingly +au d +transp aren +cro wned +sc oun +blizz ard +lux u +fi ers +achieve ments +hun ters +rock ed +bas in +vio let +pro ves +achiev ing +pro sper +se ga +flo at +vi an +xi v +pol ic +tur a +approxim ately +wander lust +keep ers +geta way +co d +pol is +br yan +col ts +tal ents +yo gur +gluten free +wri st +gr y +cze ch +ðŁİ Ī +ev ille +ðŁı Ī +to x +dani els +am er +bi ds +weare one +me tab +g t +boy z +pd x +pos session +pu shed +shr ine +reali stic +tri gger +na vi +ru mors +n af +jen kins +tr un +comm uni +Ã Ĺ +gam ers +arm or +moham med +bal cony +y ah +stron gest +rhy thm +unfor gettable +k p +ho bb +custo dy +greg or +r ita +aes thetic +il ation +sponsor ing +n ay +kid napp +sh s +ra jas +me g +signific antly +butt ons +la c +ver sions +essenti als +opini ons +k ro +d printing +wi dely +d k +ur an +y al +reque sted +c n +cur ric +plu m +gr un +v m +dev on +m yo +rel ation +juvent us +rou ge +min ority +min es +jupit er +n ine +oxy gen +fran kie +une sco +fab ric +disgu sting +sal man +dete ction +lan ka +d ac +ðŁĩ« ðŁĩ· +argu ment +shel ves +cel tics +rober to +pi gs +he dge +fau l +pow ering +butter flies +fi r +re make +att i +com o +emp ha +kend all +poke mon +se ating +d ans +bald win +ðŁij » +lesli e +one direction +ti mber +im an +fon t +e der +di on +ste ph +for mat +gre gory +pro p +he x +ru in +sor y +inf er +n aw +bar ak +sd gs +kar ao +lu sh +v ander +end ent +g is +a fro +soc cer +ay an +t uni +lun g +da yof +alex a +mar ath +addic ted +ag ile +hy gi +light weight +ì § +mand ela +jo ey +anc y +hu m +bi r +memor ial +jim in +ging er +v ak +jav ascri +cro ps +orig ins +d ari +pi per +im port +aggre ssive +predic tion +re pairs +cr acker +voy age +ni ke +mu mmy +linke din +country side +bor der +gla ss +per t +s als +sho e +autograph ed +wal nut +colle gi +sal ary +pa iring +ðŁĮ ¸ +cath ol +swee the +defe ats +streng then +roof top +impro vements +barri ers +ur u +t ally +ru led +ðŁĨ ļ +nai ja +emo ji +per cent +gi o +pro bs +on ce +adm its +pa ths +li ar +day tona +pe ters +cal i +cal li +mu g +o sa +ap h +ab y +hy de +eth nic +pla ins +ol f +haha hahaha +holi c +?! ?! +su bli +bl acks +mo t +gh ton +lo vin +b rent +bar u +l ati +de w +ate au +q a +pain ful +bu sters +st atic +ðŁĩ¨ðŁĩ ¦ +note book +out fits +si es +r f +floo ds +Ñ Ģ +thro at +su ici +ro vers +beng al +pre pares +blo g +mini ature +Ø ¨ +am phi +com b +r sp +in timate +green e +Ì ĩ +al tar +surg ical +ves sel +... ? +gav in +g ator +threat ened +z ar +rob bery +di er +promo ted +y g +x s +su bs +inter viewing +threat ening +do zen +me ado +water fall +nintendo switch +cal um +mini sters +dro p +univers ities +war ned +tac tics +ðŁĩ ² +refu se +ad ju +v ast +ðŁĺ ´ +mc fc +lib ya +no filter +distribu ted +re ser +ron nie +de co +javascri pt +mon k +intere sts +fle x +mar tha +sti es +oo d +ðŁ¤£ ðŁ¤£ +e un +b ali +g omez +sti mul +moder ate +d ity +ir is +stra w +consist ent +direc tions +adop t +sal sa +cro o +reco vered +black friday +lan caster +accep t +weareone exo +buil ds +free man +air plane +diti on +bel ong +jam ie +pit ching +li f +om in +cri spy +pre pping +ve g +chan g +accompli shed +graci as +dolph in +elec tor +culin ary +super bowl +wal a +pur suit +black berry +be an +cardin al +pro ved +immigr ant +stric tly +holocau st +pass age +ha us +cou p +pur se +har ass +< < +le ed +ado be +st ad +legis lat +par ked +pri yan +sil va +kri st +s the +fun ky +ig a +sett lement +ph s +t mrw +stre ssed +hun t +ho ckey +treas ures +cham bers +ol u +hu t +mar ley +tex ture +wilder ness +mm ing +poten tially +om aha +ju dy +to es +spo iler +distingui shed +feli x +ah u +recommend ations +zom bies +hit ler +tri ple +colla pse +motiv ated +ulti mat +gg ling +so y +ci gar +fo ren +vine yard +gl itter +fin dings +colon ial +hun ter +eri k +den s +beet le +lot te +sub tle +s matter +tru sted +experim ental +nam ents +ðŁĺ Ĩ +regi on +acquis ition +bre eding +quarter back +am reading +oo td +ru de +initi atives +st out +hy ung +out come +al fred +mic s +exper tise +bacter ia +pengu ins +jump er +valen cia +bar k +ing day +sell ers +contrac ts +hou ston +commissi oned +adap tation +swan sea +santi ago +common wealth +ju dging +sub mission +sco rer +tom my +ñ o +ex quis +fil ing +explan ation +alli son +wemb ley +ri dge +chev y +san tos +own ership +cogn itive +favour ites +sh ed +phil anthro +dele ted +go dd +s nor +gui delines +ff ing +je ep +cli ps +sw amp +an or +guil d +bol ton +spring field +munici pal +goal keeper +ye on +ðŁĺįðŁĺį ðŁĺįðŁĺį +ãħĭ ãħĭ +water front +gra ve +contempor ary +ar ity +ÃŃ a +sle eps +sy rup +al am +pi re +co yo +moto gp +ty son +kej ri +cir cul +sing ly +cr unch +complic ated +nostal gia +k op +mo ve +k ale +mac ro +mid west +h ans +tri bal +nu de +௠į +bey once +congratul ate +cat er +leagu e +ðŁĻ Ĭ +la dder +cra shed +tech nic +karao ke +harass ment +ro ts +experi encing +kri sten +ðŁĩ ³ +ðŁ¤ Ĺ +reflec tions +guin ness +illustr ator +ðŁĻı ðŁı» +cen ter +nar row +comm ons +regul ations +Ù Ĩ +har m +cro ft +cu ssion +hong kong +st ical +intern ship +zo e +cho p +hoo ds +estim ated +batter ies +berke ley +smooth ie +shau n +cro s +~ ~ +cam pe +hu mp +b g +proto type +cl ick +shaw n +re viewed +tem pl +p f +jed i +blo gs +ray mond +as th +ba h +av ail +scot ch +leaf s +nik ki +to k +hol low +ur ges +of t +un like +lat in +u e +cat ering +mil i +alter nati +ma ver +Ð ¸ +ag le +pre order +lu x +cu cu +ðŁijı ðŁijı +t art +âĿ¤âĿ¤ âĿ¤ +arab ic +rapi dly +ar rang +all en +travel tuesday +pa ws +flo ws +st ability +flu id +ca pp +can berra +uu uu +sp ani +demon stration +m la +plac ement +m w +presi dents +awe som +bever ly +ani st +ne al +father sday +referen dum +la hore +o aks +deb bie +half way +gho sts +de bor +matthe ws +fi at +t fw +pre sen +rob i +de d +bro ck +laugh ed +am ounts +bam boo +kinder garten +eat en +mtv hottest +break out +u sic +fra ser +legis lative +p ang +modu le +sam my +go ver +ear ns +expe dition +gar h +concep ts +char lie +la va +bachel or +veg gies +deter mine +el lie +un locked +fru it +dal la +cou pe +wash ington +depo sit +iv ory +pau la +chic ag +gu cci +ðŁİ ĥ +cul tiv +pier ce +li fted +stu mb +re cover +musc les +conduc ting +cb s +mcla ren +sophi a +cel lu +oce ans +up loaded +game play +mal dives +kim ber +avo i +rac er +ca ine +cav s +h ana +li ga +ra ven +inter vention +inaugur ation +oo h +at traction +merchandi se +tune in +li king +juni ors +int ended +att acking +aqu arium +i wd +comp onents +sur ing +cent u +yogur t +ðŁı ĥ +show room +op tical +ty our +ju dge +yi eld +an to +pl c +transparen cy +recy cled +chi ef +ar om +ambassad ors +plan et +âĿĦ ï¸ı +om ed +vaness a +cour t +mar gar +hal ey +v r +reg ina +pd ates +hi span +live stream +âģ £ +ya hoo +gal la +secu red +w ir +bene ath +off l +n il +am b +ye g +out let +u te +pe ep +lind say +bent ley +... ! +he el +trilo gy +vo s +ty re +there fore +tor onto +ab i +simp li +ja e +exten sive +eleph ants +s or +orient ation +im peach +re play +constru cted +peter son +pa is +por ted +custom s +colla p +ad u +high lands +sal em +shel by +ko vic +stra in +ro sie +sen ators +snap s +bo bb +suz uki +bla des +k p +lo lo +gener ate +si ght +ma e +struc tural +predic t +jump ed +ah mad +sun g +just ice +gla m +vol vo +jubi lee +de tention +lo sses +pu ri +every time +Ð ° +ra o +ed ge +li mer +rese mb +har old +re tri +sacri fic +surpri ses +am c +srilan ka +bar bie +men s +fin n +ag s +ukrain ian +em brac +î IJ +flav ors +hom er +lau re +ou th +pr iced +ver de +fir m +ah s +cu b +tre y +par anor +pro fit +in dv +who a +har sh +al ot +crit ics +hu bby +fi gur +gi ra +ca stro +chan el +in put +origin als +ten ant +yy yy +ture rs +lincol n +co on +lear n +ch ou +ac are +o les +din er +hy p +bizar re +mc r +let sgo +decor ating +ðŁĮ İ +al ison +ar vin +f d +reha b +mccar thy +lot tery +da h +minne apolis +eli gible +diagno sed +emer ald +destin ations +s ans +or y +bla zers +n v +ba il +digital art +no c +mal ta +sol ar +pi pes +alleg ations +no ck +po pe +bri d +premi er +n x +present ations +ef a +bo ws +val ve +opp onent +Į ë +visu al +ing le +cate gor +e ter +po is +dan i +at tract +neu tral +th ene +cra shes +fred die +ut ili +c st +awak ening +slo ven +quali fy +pro of +fair y +le v +fre ight +enjo ys +cup cake +flav our +â ķ +protec tive +ðŁijı ðŁı» +is u +ad mir +h mmm +continu ous +ai res +rap tors +showcas ing +y uk +pa ste +follow er +instru ctions +sp ru +@ __ +the o +debu ts +ve tte +sto w +es of +ach ed +sul tan +sand wich +som alia +franc o +car ne +flu ffy +al pine +jas mine +he ated +viol in +ple ss +divor ce +per former +phi es +port sm +dar a +kir by +lo p +chill i +for th +sky pe +ðŁĩ®ðŁĩ ¹ +celebr ities +ed y +ve e +po ison +ey el +gra bs +ssi c +un o +wester n +rail road +am er +numer ous +s v +fo w +fi st +âĢ ĭ +reque sts +mar tial +em my +accept ance +lau ra +ภ´ +er up +hyun dai +out lander +u tt +wrest le +esp resso +demand ing +g dp +geo graphy +sas kat +tro ll +confe der +su es +se m +be ts +t ful +to sh +teach es +col oured +gal way +mac y +dis orders +bb cra +at em +fen der +lit ter +e sh +provi ders +renov ation +nomin ate +ps g +nomin ations +jen na +shar p +some day +z ur +bra ins +che shire +pre y +hu go + ¿ +to ken +r v +car r +tac tical +zel da +kay la +fern ando +photograph ers +j our +umb rella +woo dy +congress man +du mp +le vy +ju an +d azz +sign als +la in +an u +mic hel +por ch +al den +sibl ings +y ale +pe el +sw ick +gg in +ll c +k ale +s con +il d +pat reon +re el +qu in +wit t +mar ty +moo dy +ton i +der y +g ators +speci fically +dd in +ly on +tr ick +meado ws +p j +bor gh +vi k +tu r +bron x +pu ff +lan tern +ðŁ¤ ¦ +g ently +be stie +fac t +refu sed +fas ci +mp y +ðŁĶ µ +cross over +mead ow +indian apolis +duc ation +sle y +loo m +mix er +new music +film maker +prosper ity +li m +week end +cre amy +neu tr +lu ther +h v +nor thern +tw o +h ra +cat ches +appear ances +ha bit +kitt ens +n v +illa c +inf an +regar dless +liz ard +dun k +cur tain +ac om +in tu +ve z +e min +fl ats +calend ars +em power +ru ined +hun gary +vi d +we x +u lum +aber deen +o sa +k t +ma ssi +se emed +s den +' ? +tele phone +de fi +insp ires +me ow +z ones +bl ind +pl y +tuc son +advent ure +ge d +oy ster +ðŁijıðŁijı ðŁijı +out put +tt t +metal lic +sma sh +ucl a +sco ts +perfe ct +lu cy +regular ly +sp ic +rel ative +ath ers +mis e +batt ling +deci des +mat a +occu pied +random ly +cat softwitter +gi an +ball y +al ties +al lies +im men +sy rac +ðŁĴľ ðŁĴľ +l lan +au r +k ut +lam ar +affe cts +n ra +star war +ðŁ¤ ĺ +sc ram +en chan +pro cess +luxu rious +ar ray +sher lock +comp ati +dor f +stre ss +m su +s with +sal a +sof instagram +fo il +under stood +qu ay +r p +c ade +ja w +en ab +en coun +ðŁİī : +do ck +satur n +mu ll +lay out +ra rely +happ ily +fix ture +or ph +over looking +her bs +m itt +pil lar +nol an +pe tty +str y +u i +mu k +o res +o vers +á µ +re creation +we sley +ri t +kejri wal +sto cking +g v +subscri bers +moo se +ma e +ber t +opp re +assign ment +u ro +high lighting +cal vin +we igh +cambo dia +av on +ke m +dis abilities +read y +char gers +p ads +iz ing +illi an +tru ste +col leges +associ ates +alban y +mil ton +cr on +bu r +har dly +si ghts +anti ques +e cho +surpri singly +ha iti +cap t +ph p +op io +ine quality +equ al +ken y +sch mid +autograph s +ren t +qu er +cit rus +challeng ed +te c +epi de +fe st +z hou +li me +citizen ship +cry stal +convin ced +mess enger +copen hagen +âĿĹ ï¸ı +war ran +develop ments +ï¸ı âĥ£ +fore x +hi ro +sne akers +xi de +vi va +stere o +bat ting +ss el +ho st +beng al +critic ism +q c +cr un +attemp ted +ry e +determin ation +cre ations +d read +label s +pos se +anc er +joh an +si ster +partner ships +les bian +k st +guaran tee +bar o +fix ing +ma son +m ous +chem icals +t less +bio diversity +par o +bhar at +ac ol +refu ge +en te +t iti +dys sey +respon ds +lef to +in er +se vel +rahu l +ol ine +frank fur +cho reo +enjoy able +c to +strugg les +wood land +heavy weight +gen s +rece p +ac cred +ðŁĺ ¡ +trans formed +list en +at op +n k +sur ge +be re +gover nor +prison ers +clau de +t ill +mu lator +emo tion +water loo +star t +ðŁĩ º +clean ed +grand mother +fear less +afric an +astron omy +ðŁı ģ +ภĻ +the world +su itable +anth ony +k and +tt en +meaning ful +disc lo +jaco bs +à ¸ +tom linson +ghe tti +ty pho +sub stan +as co +te k +nag ar +mu d +am on +vacc ine +f ty +fle sh +no el +infl ation +portu gue +glam our +tra m +v re +te qu +roun dup +w yn +rejec ted +mosa ic +si ghting +cal f +o ta +com position +go pro +gonz ale +e ed +b ard +tu e +effec tively +we en +al to +ri bs +rel ate +thir sty +fu rious +di m +ch ard +perfu me +s ny +chur chill +k of +master class +wa ve +ðŁĶ µ +er in +own s +to be +sk illed +te m +go f +en i +tor i +cra zy +l ick +resi stant +ici al +ag ar +! : +g ali +del aware +bl itz +koh li +pu ck +avail ability +hi malay +influ ential +cro chet +victor i +read ing +ho bby +vie t +j as +en gra +sk ul +ðŁĩ² ðŁĩ +educ ate +tech no +distric ts +blu es +se tt +seven th +lear ns +ee ee +apocaly pse +hang out +cru el +mu tu +bru h +hel en +she er +c tion +kle in +tex ans +ce real +sh ine +ne red +gra s +am bro +f ella +hin du +matthe w +li ma +mir anda +je wel +so ho +euro vision +neighb ours +chand ler +be sides +ðŁ¥ ° +ast ros +thu mbs +ren ault +ra ve +hi red +ðŁĸ ¤ +it ary +z or +bla zer +k ine +ea u +kat y +dc comics +pe c +ro dgers +water proof +kill ers +super int +pre serv +as so +brew ers +promo tional +sc am +villa ges +sket ches +ju icy +for life +au dit +so lo +fundam ental +len e +philipp ine +t end +conserv atives +sponsor ship +dd le +a ine +h tc +os i +hul k +w af +ภĻ +evalu ation +ant ine +sle e +robert son +roo sevel +ag i +sophi stic +emplo yers +bubb les +ko wski +inter action +sh u +bou le +ic an +j are +han k +leg itim +k nicks +kar ma +recei ver +per ks +u h +sta ir +sun i +labor atory +gra ves +voc als +oo t +c ture +thri ve +tic o +ãĥ ³ +b w +carto ons +mcdon alds +dra w +y ung +pl er +li d +eth ical +groo ve +ent a +international womensday +pat ron +wor ries +ðŁİ ħ +ðŁij ĭ +ka therine +di az +tor i +bach chan +tru st +min eral +ic om +buil ders +bor n +col oring +lat te +ca se +revolu tion +tra der +ox id +chi pot +inst antly +sou thern +se hun +pro b +her nandez +lis bon +hu awe +p ong +me a +ro oney +wheel chair +ke en +be tt +cor in +regulat ory +di splac +ka ren +sch em +sun sets +wh ales +remin is +he p +hi de +mar cel +pand ora +do yle +th fc +ot to +no kia +trans gender +ko v +hawai ian +sha ve +so vere +exc er +nick i +pu g +st or +ro th +wee t +leg al +dig nity +po w +hom age +ðŁĩ³ ðŁĩ +s re +can on +la x +wo ah +quart z +ñ a +gree ting +flick r +nai robi +advoc ates +an c +vi i +eu gene +th ra +c re +el an +pen sion +th letics +ton i +re agan +x v +sto re +ben ch +har lem +todd ler +sent enced +âĻ¥ ï¸ı +glob ally +che aper +u f +ma m +nic o +ik u +tho u +ni st +dam i +th ala +rho des +sal e +bow ls +â Ī +las vegas +sanc tions +adm ire +mat ched +un able +travel er +ele ven +straw berries +âĢĶâĢĶ âĢĶâĢĶ +stu dio +jac ques +im s +valu ed +s no +cheese cake +n xt +e os +s x +f x +ton ic +hat ch +chic ks +gra ds +hand ic +r ory +as p +ri pped +denti st +n en +lu fc +âľ Ĭ +di ge +hop kins +sher man +f da +for all +ash ley +str and +h y +liqu or +buffe t +ess ence +phar ma +suri ya +ðŁĴĻ ðŁĴĻ +festi vals +z an +re fresh +pur ple +uni forms +kenne th += ) +as an +hel sin +transform ers +k ali +person alized +chal k +bo bby +â Į +the mes +depar ture +prin t +illustr ations +qui et +agre es +gri ff +Ø ³ +m iti +toge ther +conven ience +ab ar +car lo +turt les +info sec +some what +ar lington +scholar ships +emir ates +mu ms +st ella +auton om +fe ather +g ore +nom inees +fragr ance +Ñ Ĥ +w ong +thea stern +gr e +z illa +is i +bump er +go o +do zens +ab duc +âļª ï¸ı +o ils +don ors +sil icon +i pod +fortn ite +ðŁĴ ¨ +tor o +spark ling +consci ousness +pal a +nu m +moun ted +ffin s +thi eves +team mate +pra b +om er +ta pes +bo d +mit su +ste w +e re +p bs +tu sc +lo we +ra de +parliam entary +h m +ed gar +ðŁijĩ ðŁijĩ +to a +a gh +hon i +s late +ge ek +ap t +hard t +ta p +horiz on +grow th +make over +hi l +paper back +id an +reha bil +gi u +possi bilities +let tu +fran co +bo ss +ach er +does nt +mo e +ta ker +huss ain +ml k +di l +th ia +ham a +real ised +raven s +curric ulum +m ith +k night +ted x +r v +isai ah +cumb ria +birth days +f ing +pre z +mu barak +exquis ite +clear ance +y en +par i +ev o +à º +modi fied +app lying +imple ment +disco vering +chap man +indie game +dis k +crowd funding +mach in +li vel +sty led +âĿ Į +ma king +rehear sals +nutr iti +subscri ption +and ro +cre ators +car ries +ky lie +cam den +appren tice +tax pay +c ca +tuesday thoughts +pis sed +er man +dete c +freed om +mer i +.. ! +psal m +sun light +per spec +be ings +book store +rock star +fun ctions +p ence +fav es +z n +obam acare +sp ill +coven try +pi geon +pi vo +ba it +kol kata +av al +don or +wa h +privi leg +tra ditions +rajas than +ten ess +portugue se +yn es +tack les +de fic +tor n +pol ling +thor ne +in a +bened ict +bar ry +cal ories +ver dict +save the +nor ton +off ice +main stream +impro ves +fr on +respon ding +real tor +scotti sh +de clar +r l +shi v +supp lier +re sting +swee ts +qu i +. âĢ¦ +whit ney +startu p +thank you +teach er +h alls +ha ve +hand made +pro ving +quar tet +ro chester +li an +virtu al +mend es +of icial +mid lands +x box +meas uring +o vo +accommod ation +bri des +collegi ate +intellec tual +in car +ni ag +ðŁį · +sf w +coco a +co ats +civil ians +presi dency +mat rix +sweethe art +tri athlon +wag ner +ra dic +plann er +the o +execu tion +k um +the walkingdead +sc ar +ro tation +blo gging +bom b +re son +bb les +st are +assi sted +e do +brand ed +war nings +thor pe +acknow le +satis fied +sho res +ri d +dor a +phys ically +bi gh +appro ves +ha h +ric al +vers atile +pret end +lu m +ab hi +ye e +sp it +ãĢ Į +dj s +ash tra +j t +ven ues +gram mys +cy clo +tr acker +over watch +repl ica +el yn +nr l +lind sey +hom o +ballo ons +kitch en +si s +am os +ende av +ðŁĴ » +a rec +thu g +hoo ked +hr c +new york +bur gh +americ as +patric ia +ug u +ap athy +ha st +psy chi +cor k +petro l +ðŁİ ¬ +ak u +po pping +psycho logical +au x +g ma +cad illac +wa ste +auth ent +bri stol +nam e +que er +to ber +jer ry +com in +ch ant +privileg ed +op ar +lo ser +tex t +mar ker +stri es +equ ally +ak i +christ mas +gare th +ble w +em ma +imag in +se als +che at +conditi oning +j ana +ren s +dar ies +o asis +disc ounts +coun cil +i ka +shir ley +vou cher +al ps +w x +q r +dri ft +attemp ting +ut c +Ø ª +gonzale z +m f +jo ker +paralle l +pa re +aspe cts +proce du +n p +am a +rale igh +bright en +gu ire +radi ation +cre scent +ho b +il le +str and +v ore +n ard +che st +di wali +av atar +al der +d ling +pa thetic +ðŁĴ ĺ +spir it +jor ge +film making +ðŁĻı ðŁĻı +challeng er +b j +down town +ht ml +ade qu +twi sted +in ely +( ' +wra ps +oper ational +y ne +n us +mag net +market place +health ier +snap shot +dam on +inter ven +fe derer +ow ls +biscu its +j p +ro deo +blue berry +lec tion +fron tier +summ ers +re yes +pede strian +go l +caf fe +refur bi +bou lder +me ghan +speci alty +la ss +e i +suspec ts +appro x +rr r +ra th +st im +cru shed +he d +wh un +lo af +cr ore +river a +gene tics +so ck +wa sted +ny pd +answ ering +do ve +bel la +ol in +du n +fi ji +pre tty +spar kle +y un +j d +euro pa +li fts +am ber +mu r +te k +boy d +roy alty +in do +ri b +go tham +ti est +inst alling +ke mp +the photo +cos mic +) )) +whole sale +loy ment +eas y +su ing +sett led +af p +pro ver +suppor tive +re es +ne ath +deli ber +c é +wel come +pic oftheday +new born +pat ty +sun s +si est +fl int +diffe rently +spo ilers +troop er +g ins +cor y +look out +equi pped +ta pe +to by +resear cher +u sh +ke yes +al ma +induc tion +k w +k har +sl ick +bri de +e ur +cra ving +book ings +ch es +tr unk +vern on +sp her +cryst als +rel atively +pom pe +uni ons +val ley +par a +w ant +ok c +de af +ser gio +len non +sh ay +cr a +v at +he e +t we +liqu id +pol y +ðŁİ ģ +b ent +be aring +motor sport +bar be +te sti +han i +fin ancing +astron aut +water colour +ri sh +comic con +gar t +wr ong +ber n +it an +ste pped +fil ters +c low +me x +dem ons +all o +expand ed +comm and +et ers +go ats +si ri +y r +pot tery +mari on +i le +el an +san to +person a +du ke +hom eless +li ghted +wheel er +chang er +cab bage +sur real +ham burg +sma shed +str an +k not +i art +ob i +be dro +di al +th ick +b ingo +fu s +vacu um +con ve +ati ve +accur acy +accoun t +re fer +ri z +spider man +ban a +r ite +u b +ab s +medic al +lin k +si em +> >>> +be tra +g lowing +re actions +pupp et +spa ghetti +ang s +re medi +pray for +roy ce +char lotte +£ ï¸ı +gh et +affe cting +ro de +soci alist +mo ses +az i +o it +re porters +cd t +ap ing +s nat +minim al +wa ist +sie ge +>> >> +ri g +schmid t +h are +ec a +thor n +he mp +es the +cly de +th a +don ut +moham ed +ling erie +le gg +carpen ter +perform ers +de a +imag ined +cur se +la sh +ct r +agu a +ro ar +gr i +ro le +j fk +resur rec +roosevel t +maril yn +sm alle +will is +wa ited +char ities +the res +li k +origin al +car i +c ough +cru ci +la gun +contra st +k ou +arm our +re moving +t ent +maz da +bri ghter +thi ef +cor ner +tequ ila +buzz ing +al bi +p am +az ure +disc oun +pixel art +possi bility +ham ont +tra des +bu da +hi ve +vers y +fin ch +tran spa +em i +terri fying +in qui +g ba +sub stitu +collec ti +plac ing +cin dy +k ann +pa tho +diamon d +mour inho +guine a +anthro po +air s +pu mps +ì ļ +pas o +cur ling +an ita +resi dency +ne wh +jo on +cigare tte +que ue +ex trac +gam es +spl en +ex press +public ly +bon nie +tribun e +ba ek +reason able +c or +timo thy +she eran +Ä ± +f dn +su tton +concentr ation +carav an +x avier +al ger +cy lin +freder ick +ner ve +pe ak +lettu ce +j ail +pre game +kav an +up graded +eco logy +squad ron +gra pes +goo g +pa stry +ðŁĹ £ +ãĥ¼ ãĥ +mil ano +awa z +presen ter +ðŁĮ ¿ +her d +king s +tem plate +fl our +h v +k ley +i ya +spe c +at er +frankfur t +co ch +tex ting +del i +communi st +regi ment +ele anor +anticip ated +ðŁijĮ ðŁı» +thephoto hour +ran o +survi ving +simul ation +daw son +ar in +aqu a +m or +âĢ¦ . +cin o +ira qi +sh az +dun dee +we s +dra u +hann ah +s news +occup ation +ste en +x m +ang les +sett ings +gur u +kno x +or ca +shap ing +w ent +dr illing +zz ie +br i +kis sing +fin d +ma ine +âŃIJï¸ı âŃIJï¸ı +ðŁĮ į +lar ry +bu sted +ta vern +acti vely +- " +replac ing +no d +un lock +. " +âŀ ¤ +affili ate +to w +l n +happy newyear +di f +j m +green wich +contro versy +daw g +con dol +sav annah +compens ation +touch down +te o +amb itious +embro i +convic ted +iart g +bar ack +tr ance +testim ony +au dition +thum b +my ths +be x +que z +orch id +den y +entit led +hoo d +gr ant +in box +blue jays +r illa +smalle st +bur den +in famous +divi ded +boun daries +t ter +el t +wy oming +be verage +me sm +one ws +budd hist +y ana +as sad +is ms +bar rett +predic ted +back to +tw it +e there +cap tains +escap ed +ay o +lam borgh +gard ner +la ps +k al +adverti sement +insec ts +na po +am en +ac y +r and +g k +te h +k athle +tri dge +pan cake +at ro +pyram id +bu la +paral ym +gau ge +en cies +tom y +biscu it +but cher +quali fier +coun ty +ke i +po ols +dar ker +should ers +ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ +sp re +( " +writ ers +g m +ðŁİ ĵ +k nit +hu ff +mt b +philli es +o st +den is +g art +licen sed +inter face +ex cel +d well +from the +co fficial +az zi +appear ing +fore st +n ana +ke ith +manufac turers +beck ham +) ? +e se +col ony +delic ate +ut ter +mc in +transpl ant +pre ferred +par d +ari e +hu b +po ds +perspec tives +pic t +del u +app er +be than +p mo +crimin als +femin ism +sh ack +circum stances +fel las +prote sting +wa x +sugge sted +t ator +dre w +om ni +fa ke +kath y +re b +del ine +ber ni +mi sty +ðŁij © +er able +break through +men swear +millenni als +chan yeol +la z +inser t +rep lies +phra se +n x +ihear tawards +audre y +gran ite +rac ec +ori e +ter ra +innov ations +britt any +at eral +pe ar +bio logical +sh ments +institu tion +m sn +frequ ency +d man +neg lec +t f +ste fan +fox news +ty po +comm s +sequ ence +car men +wh ites +econom ist +exe ter +se um +re sorts +cas ually +bun de +divi de +Ø ¹ +ga g +cre ed +reti re +cau cus +rapi ds +wrestle mania +tul sa +sunder land +fundam ent +o di +yam aha +v ary +intri gu +el se +be acon +an gie +tra ded +tran sm +g ents +kn itting +gal ac +ðĿ Ĺ +u to +sea side +hol t +re rs +far go +train ers +mon soon +b ale +sou ght +mad die +h w +co li +fr an +fav s +ðŁĴ Ķ +int ent +r ally +s bs +lemon ade +barack obama +bre ad +stick y +explo sive +chel ten +t j +as soc +ram en +hom ies +v log +mi ster +lor d +âĢįâĻ Ģï¸ı +aly ssa +sketch book +ru mble +cat ch +migr ant +discipl ine +un likely +chronic les +fl ora +sl ams +am id +s boro +coo p +ju mps +tran qu +mel is +sof ia +en ri +gab e +sy ri +nicol as +cha i +w v +be cky +foo ty +ta o +suppo se +ðŁĺįðŁĺį ðŁĺįðŁĺį +plu sh +ri sh +ðŁ¤ ĵ +k ha +satur days +ac cent +he c +lim it +carl ton +wi red +taylor swift +ðŁĺ ij +sq l +har ro +recipi ents +g at +go p +th of +amaz ed +gh an +ðŁıĨ ðŁıĨ +por to +cla re +di stant +na c +ohi o +ðŁĻı ðŁı¼ +mt n +anti bio +dino sa +me sa +par tial +b v +lear nt +lov ato +questi on +ex tract +gossi p +gi bb +niag ara +ðŁij ¨ +displa yed +so oner +ste vie +nug gets +ml n +bro m +tur b +give aways +stu pi +bl ink +c ili +conven ient +mo h +vi ve +f ric +cau se +cham ber +cu les +ne arest +is se +small biz +t j +canadi ans +smar ter +bra sil +ra re +que tte +w ha +cand le +at omic +ðŁijį ðŁijį +warri or +relax ed +stri ps +ne ur +k ka +r fc +jen sen +reco vering +respon ses +sal am +ortho dox +acti ve +ell ers +n it +âŃ IJ +metro politan +centu ries +vi da +gra ding +transpa rent +sim ple +do ts +superint endent +elev ator +autom ated +red skins +ima m +summer time +jona than +ge aring +michel le +confl ic +m ice +to te +publi sh +pa x +) - +na iled +á ´ +tele scope +ser bia +ba b +ape u +st ically +sen ti +r ats +isol ated +grou p +hat red +paranor mal +stan ley +ali on +safe ty +l s +ठ° +nex us +alexand ra +mas ks ++ + +tr on +au k +brother hood +brow se +mix es +sim one +mu sk +appro ve +lo la +ex p +per th +fu turi +un seen +d m +chel se +sc outing +o we +portsm outh +k ram +mi ze +di spen +su p +d lc +adver t +tere sa +is le +cy cle +met all +shi elds +marin ers +ra z +ing en +fun d +an go +jon es +o ka +mad den +broc coli +domin ic +situ ations +mer o +cric ke +puni shment +d b +sha king +ðŁĺ ļ +m q +ari ans +le h +cla w +we ds +d ure +ni el +j elly +gour met +tra ders +le vi +w ages +kne es +wi se +heaven ly +avi d +melo dy +z ack +ban anas +apprentic e +pro p +fun ny +o de +respec ted +me gan +fe wer +dra fted +med it +gra pe +us army +cru sad +vo cali +prepar ations +non sense +us age +th r +ro th +wiz ards +insi de +promo tions +mon a +red sox +si g +eleg ance +ch ia +univer sal +ãĢ į +ra ja +un ga +pol lin +filip ino +ak a +t sun +ik on +bi king +decor ations +z ac +cade ts +hum our +ag m +re ppin +vac cin +elo ve +u w +dia be +galla gher +az er +do l +a while +pro minent +wel sh +t ann +' ) +bi en +wa g +in al +c wc +wic ket +ur st +q anon +x e +out door +dun n +star r +co logy +ric ky +u efa +reb ounds +s music +inf ant +ðŁĻ ĭ +so p +u mber +hand ing +beg in +sor ting +ha sh +sp ati +re k +buda pest +black hawks +dele te +ro m +can did +auth ori +de bris +spe cul +inter section +marri ott +im ran +ðŁĺģ ðŁĺģ +cru ises +ram sey +rafa el +aware ness +vas cular +beyon cé +ru g +ðŁĺ Į +festi v +ar am +s able +bas il +p ill +flo oring +un beaten +implic ations +u f +w ound +for ge +poin ting +po ts +popular ity +ðŁijı ðŁı» +mani pul +s lots +deb ates +abs ence +ver mont +never forget +wri st +gl oria +ren ce +hu sk +mel ting +ðŁİ Ł +br aces +tim ely +transform ing +am ps +ma k +po e +ah an +gener ally +nd p +ale ppo +unic ef +pro fs +nor d +ma sk +jackson ville +v v +sh ells +bloom ing +oper ators +char coal +ne ville +ma gi +chi p +sam a +ir an +re forms +accu mul +ru e +æ ľ +web sites +ga on +devast ating +sto s +glaci er +ra pp +chipot le +pr a +or ous +rom ney +seas on +decor ative +c isco +dit ch +compla in +ll o +assu me +ðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤ +n els +cent ric +ft w +car rots +tat a +can ter +per ience +li ers +demo s +bl unt +oper ate +reserv ations +le ah +sub stance +di son +an te +elec tion +v ue +squ are +non profit +ca a +f su +y am +ãĤ ¤ +v ladi +comple tes +mar i +philli p +ne ill +er as +ka it +men do +mahar ashtra +g p +dan e +provi dence +ther apeu +juven ile +me mo +in corpor +aa aa +seven teen +teen ager +à £ +or ns +wi de +cu teness +tw d +ff les +bar a +com edy +over time +y az +bar on +unemp loyment +ðŁij ĭ +exter ior +den se +cent res +match up +history month +artif icial +qu it +e sk +war n +cr itic +j af +ðŁĵ ² +inform ative +fu els +recy cle +nam ing +stri pe +sol ic +mole cular +dee pi +con vo +s sel +na e +de scent +ti z +accoun tability +ter ry +r ito +sl ay +em o +dem ol +sens ation +co v +tor e +round table +y ol +excu ses +ॠį +tur quo +hh hh +pod casts +cele b +me ssi +li o +man n +contribu ted +u z +gener ator +ele ts +veg gie +indu l +en suring +detro it +pun jab +tran spor +instru ction +ad d +por cel +pan eli +cir cles +persi st +clay ton +sp n +dog softwitter +is nt +sp r +retail ers +p w +hun gar +el ena +mon aster +gu atem +je ssie +an z +ra shi +fle e +car ving +fau x +l al +hen ri +d jo +du ll +s ana +lar a +glo be +cri mson +com pass +pau se +na b +lion el +ba ths +u fo +invent ory +sin gh +sat an +ðŁĩ ¸ +ce ments +in form +gener ated +bi den +av g +tas ks +de er +sa u +ja iled +pa stel +sc c +na il +steel e +per is +lamborgh ini +pur sue +mar gin +u ch +bo sch +dra in +cl ara +bo m +lat ino +web ster +rose mary +r ha +s oun +billion aire +not ch +percent age +con or +' " +hom es +earth day +h ort +big gest +di sin +wal ton +edit ors +im ma +om ar +equi valent +pharmac eu +ah med +cam eo +han ni +under rated +ge ment +micro bi +v oo +honor able +obe sity +âļ ¡ï¸ı +limer ick +invol vement +st agram +boule vard +bur g +blackand white +liber ation +fi ve +inter im +sm m +rival ry +cap abilities +stat ements +thu mb +ve d +sw ans +bar ber +e que +seren a +hel m +noo dle +sam pling +n awaz +sing le +thunder storms +sh on +in ev +ë ¯ +to pp +orch ard +bi an +ðŁĺ Ķ +door step +salv ation +marke ting +r ons +cle mson +ra vi +in take +stand with +sin a +ha iku +ple y +elector al +ph illy +la ys +electr ic +cap turing +u pp +er gy +believ ing +cul tures +es day +inva sive +ed ed +spee ch +end ur +viet nam +boy cott +pe de +deli ver +ðŁĴĸ ðŁĴĸ +mer chant +st ir +den ies +poc kets +o ti +cu ddle +ro land +mm ed +den ed +lear ners +hoo p +sour cing +h acked +di m +environ ments +ben son +jud icial +wor cester +pear ls +govern ments +arri vals +cor ners +tun ing +la bour +y m +or dering +le wi +i fe +hygi ene +thou ghtful +indone sian +campaig ning +princi ple +assau l +ru bb +at v +wil ly +en tre +il i +ph on +du ties +âĻ¥ âĻ¥ +sn akes +lo op +am ar +conver tible +bon ding +ment oring +max well +ethere um +destro ying +ax is +ca iro +fin nish +sho ck +ðŁĺ IJ +cal eb +com a +pe dal +co re +contin ent +el son +temp o +helsin ki +ac p +tack ling +st ated +bl a +dou b +sma shing +a ja +camer on +disru ption +warm th +being salmankhan +bullet in +o de +syrac use +ar an +mc gregor +bul k +an ton +confir mation +sp ine +im ran +instru c +jac ks +chi o +pal m +str e +embarra ssing +un t +elimin ate +to ss +c ise +a ws +oni sts +sh inee +jo s +ho se +li vely +opp onents +mo vements +recogni zing +sandwich es +sh akes +exerc ises +se at +profe ssion +merry christmas +lu gg +adopt dont +mar vin +byr ne +un le +he t +ku wait +rah man +aspe ct +humb led +gen es +f and +long time +) ; +cam pu +an gus +ðŁijį ðŁı¼ +q uran +sle eves +s lic +¸ ë +twel ve +your e +i ke +go gh +b st +dic tionary +reflec ting +to on +yar n +em bed +ðŁı ´ +re serves +floo ded +ver iz +du sk +estab lish +pro li +au d +ritu al +or bit +declar ation +recor dings +cam o +cas sette +good luck +cu tter +bo p +b ho +che ating +paci fic +ma res +tim er +col t +tr ous +tomor row +han sen +ci e +w ang +ban i +circu lar +ac ute +far mer +co ys +p se +ir ving +w j +haw kins +b ison +ur day +cru ising +o te +k ath +whi stle +your selves +ant is +sla sh +thorough ly +ke sh +ser ie +ex em +en ig +guil d +sh red +ho gan +ap o +ä ¸ +pu zz +ne tball +au ssi +panor ama +ws j +av is +ar ming +hum ph +brow ser +cri es +fo ggy +mat te +ðŁĮ » +it er +tal lest +by ron +cap tiv +je su +any ways +flag ship +p ton +we y +fay ette +financi al +f oul +solom on +jenni fer +cucu mber +ar gue +tex tile +wrest ler +john ston +pa stor +ðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃ +cac tus +edi ble +re served +ric hie +met res +ingredi ent +h ella +un to +ch ol +cele bs +po ets +gra ham +hay den +coinci dence +b aw +communic ate +flet cher +/ - +tole do +ecu ador +coun sel +s laughter +line ar +at p +os u +jo el +ev ed +conqu er +ru stic +plic ity +recogn ise +room mate +cr acked +jas per +ph er +ðŁĮ º +wo ven +mo ist +ff c +ste ering +ni sh +stand ings +frequ ent +ar di +haz el +as msg +bau m +d art +si dd +nat h +ch ero +card board +c ss +n sfw +pa ir +ðŁĺį ðŁĺĺ +occur red +homeless ness +mal one +ph e +xi a +pad dy +decl are +theat re +b f +per sian +ta d +ax e +susp icious +lam b +mu cho +sen ior +st as +k ite +st ing +gra d +k af +wat ering +Ø ¯ +spi ral +th ms +educ ator +jer ome +of c +clo ck +su l +pe mb +.... ..... +park way +de aux +restric tions +m ons +need le +e j +le agues +water melon +am an +pl enary +max im +w ab +coming soon +bry ce +vi gil +super market +fortun ate +turquo ise +presi dent +li v +inter ns +feel in +fix tures +stun t +st aged +premi eres +lo k +prac titi +shor tage +log ne +ve c +con cor +roc ke +li g +com posed +syn thetic +di p +cam ila +ch is +j ou +su san +eye brows +supp lement +satis faction +moham mad +ti bet +house of +pu n +as sam +shado whun +psy ched +se duc +mand atory +her bert +sc allo +stream ers +proto col +block buster +produc es +sch nei +lau rel +tri be +time hop +pl a +mod elling +tv time +mtv stars +wi dow +me tric +ch am +con do +flow ering +ale c +d ms +inten sity + ¨ +mccar tney +islam abad +k b +f fi +ph al +anal og +f ond +h acks +positi vity +treat y +sub marine +conne ct +sel en +categor ies +cu b +organi ze +si k +quote oftheday +remin ding +am or +loc king +ðŁijı ðŁı¼ +comp ound +et te +b out +rec ur +fe rence +mi zz +tren d +hip ster +for tress +forth coming +preli min +o dyssey +ang p +del ici +even ings +ðŁĶ ¹ +i q +d w +da ir +kathr yn +christian ity +moon light +ha b +wh oo +f bf +se th +genu inely +pa x +char ity +deplo yed +b nb +bu cs +ju dg +con ge +plant ation +im press +car a +sc lub +sco py +land ers +compla ints +b ama +re build +x y +real ism +sh our +le in +brac elets +mer a +assas sin +an chor +ðŁijĮ ðŁı¼ +lin en +con fron +chronic le +comm ent +cat alog +il les +gor ge +me try +jung kook +love my +sent in +se em +fit ness +alli ed +ts man +digital transformation +pr an +lo ft +min ton +alden richards +en vel +cher ish +certain ty +zz z +rhin o +per kins +en rich +cape town +ome ter +sec tions +ske leton +def enders +ðŁĺ Ŀ +pen c +bri t +ja h +capital ism +ðŁ¥ ĩ +baz aar +re me +ex t +kk k +conver t +stor my +b ye +kar an +chry sler +ad os +pre ssed +syn c +ation day +dang er +bad ges +refu ses +em powering +ly m +ex ports +adoptdont shop +ðŁĩ ¯ +th c +awa ited +focu ses +fin ed +o at +haha hah +âģ © +n family +fi ona +luck ily +thr illing +ty ping +out break +di es +he u +craw l +ne sses +o ath +scri pts +gee ks +ðŁIJ Ŀ +p b +mathemat ics +al is +________ ________ +gymna stics +acti vism +recommend ation +gre n +wa in +cour ty +n apol +cau li +hor nets +g als +jo ckey +dir ty +at ar +enor mous +pe st +greg ation +an os +ii ii +def ends +black historymonth +at x +mb c +lugg age +wit ch +co b +la sts +cu m +gg g +ba thing +n ar +ce bu +ðŁį ĥ +navig ation +min e +re jo +ðŁİ Ģ +gif tide +re ta +use less +pu ll +defic it +al lu +ati me +it v +tr illion +pu e +ac ies +proce dure +l ori +jen ny +c ad +ul ously +dr ac +promo tes +ing the +can u +woo hoo +na omi +zar dari +ts u +be ir +sd g +le ver +we ber +ab ud +lun d +crow ded +deplo yment +ter rain +ken ny +ho f +witne ssed +lo ch +j k +bul ly +w ren +poe try +do ff +ww i +mo red +din i +cul ture +promp t + ¥ +maur ice +to pps +r m +cor respon +ab out +jewel s +gi br +eag le +ðŁĺĺ ðŁĺĺðŁĺĺ +l ending +sou ven +ç Ķ +contemporary art +establi shment +j ong +âĢ¦ " +gat or +patri otic +mc coy +v ape +human e +feli z +coach ella +re posting +ste als +fu ller +n ering +at ra +( - +bla ke +he ather +wor ms +discipl inary +rede mption +y ard +am in +" @_ +d nc +t ds +k appa +ne wark +comm its +spe ars +j ams +t and +msn bc +inter medi +aim ed +at ic +teen th +observ ation +kash mir +kavan augh +ou l +san francisco +re u +bel ated +cho w +pass word +st ills +deta ined +sar i +day ton +dar ren +itali an +ar th +amu sic +ar bit +w m +v m +he m +dou g +my r +a sho +pre v +vin d +bra h +sta g +ภµ +pre views +gu k +con taining +leon ardo +sad dle +ru shing +st av +lon gh +gam bling +ve gas +reserv ation +end ale +bal a +fl a +vari ant +he dge +bulgar ia +nat ali +we aver +sol st +encoura ged +ap c +as parag +ne st +cycli sts +fe l +ìĬ ¤ +overwhel ming +pey ton +j it +a post +mb le +ble eding +neighbour hood +a very +expre ssions +mac donald +gi gs +mon ds +illu sion +n ct +cam ero +over head +my th +ol y +vi o +et v +lau rie +unve iling +pri or +con n +iron man +di ff +day in +crit ici +con go +re vision +wal e +direc tor +p ines +black pink +gar ner +cur ated +manit oba +h ac +common ly +bar ton +.... # +mor tality +live smatter +philos op +shor ter +con vince +fre ak +vend ors +insi ghtful +el ly +sens ors +e led +s berg +weight loss +u kip +sp ur +priv ate +qu a +ss c +, ... +supervis or +advis er +amaz ingly +less er +at es +mah on +oooo oo +sar as +pmo india +waff le +un ders +toler ance +sculp tures +her sh +kno cking +smo ke +cathol ic +gri m +tra veled +fli p +ge off +dinosa urs +sle pt +scar let +ok i +compla int +ob sc +nam i +la g +cross fit +u fc +mc cain +refe ree +sad ness +pen ny +li eu +mo de +ki er +vol s +w is +el on +she a +ba o +son ia +cla ire +em manuel +moist ure +di gest +vi ii +t eller +ch on +access ory +night club +foss il +aw an +hu sky +ab original +brand on +ffici ent +cou gars +ste d +ad mitted +igno red +content marketing +ag as +v ase +execu ted +negoti ations +she ad +n and +tab lets +go th +ts al +d fw +on ep +protec tor +sp ho +gaz ette +andre as +ss er +comp ilation +ha v +contain ers +bro ker +soc al +porcel ain +hy uk +air ing +ðŁĴ ° +publi sher +scen ario +spart ans +re viewing +itu des +ed el +pear son +ba sh +mau i +a ad +ðŁĮ Ĭ +li u +ul ate +program mes +fav our +web design +real ty +motiv ational +cro sses +' ... +bus ch +adjust able +ar jun +mist ak +dimen sion +pi stol +weigh s +en y +unve il +indy car +gor don +f ade +fran ken +qual ities +bet t +loc ate +ker r +sp c +confu sion +ne e +luck y +bas es +dep ends +fire fighter +ol a +re t +mar oon +ðŁĶ Ĭ +w am +defin ing +whe at +bi l +é s +b hai +psy ch +ta u +ic ans +thi k +ob ile +inspec tor +ìĨ Įë +ill on +go s +ev angel +fa i +si st +voc ation +bur ge +chi stan +renew ed +enthusi asm +en ting +ag ri +ike a +m sc +aero space +sens iti +memo ir +hosp ice +co caine +der ry +mechan ics +Ħ ภ+tin o +reduc es +collec tors +in justice +supp re +v ana +ab un +nap a +su sa +os lo +e ff +en core +lic ence +ched dar +z al +moun t +ðŁĴ IJ +threat ens +!! " +archi e +fu tsal +scu ba +jo s +gn on +se xi +s official +compar ing +domin ant +tof theday +fa it +propos als +gi ft +y as +cn c +l r +ha b +reser voir +beli efs +gener al +mar ti +t d +est e +ì ł +wi l +ðŁij ¯ +ðŁĶ « +sp x +et work +excer pt +e instein +hir o +sil hou +team ed +per ception +corri dor +mental health +hin ts +ben ny +induc ted +sw x +wi desp +spe ak +cher yl +dru g +ðŁĺ ķ +h f +asparag us +myster ies +fitz gerald +off er +therap ist +care er +dam aging +ts d +per u +wei bo +y ay +phoeni x +disc re +mac book +bar ker +stig ma +sp read +roc kies +kang ar +bri dg +pa i +bi shop +ta iled +capsu le +ðŁĴ ĵ +ge of +roy ale +short listed +o ste +ash amed +ch app +key e +cl a +screen shot +austri an +nati ve +en ight +juli et +michel e +ðŁĮ ´ +travel ers +pi l +football er +win chester +ðŁĻ Ħ +azer bai +gold eng +organis ations +interpre tation +predat or +ofthe week +lo gan +pok é +mari e +cal la +t nt +cin de +ge tic +fit fam +gra v +ow ens +ðŁĮ ± +shoot out +sal is +commissi ons +co he +p tic +ni xon +hi a +amb ition +mar ine +cruel ty +t k +cru de +sal ty +jim a +mon go +ir ony +on wards +arre sts +strang ers +ig er +cycli st +ra g +exten ds +tra dio +bour g +mo i +el la +e able +lex us +au l +der a +histor ian +mor ton +ti ff +man ner +ko t +d k +po inted +mar qu +a an +en ey +du blin +on poli +em ili +secre t +fl o +âļ ¡ +ba j +ste ep +accompan ied +rum ours +dev i +purch asing +fi g +pu b +sch oo +autonom ous +go alie +x ia +autom atically +re vers +ter o +fu ku +titan ic +shoo k +sand als +see kers +exc av +nor dic +bigo live +ba ke +r att +z ak +ne p +ðŁĺ ¤ +cand y +billi ons +book worm +pp et +à ³ +sur faces +sc ars +phil ip +do gg +ci gars +co te +transl ated +cur ator +sin dh +han gover +bre wer +on es +el ton +ðŁĴª ðŁı¼ +mar cu +elli ot +righ te +di oce +ru ss +rail ways +grand son +as cen +apo logy +awa it +mob ili +re spir +parti san +oli vi +stri ke +yo o +white house +expre ssed +pu ps +bed ford +cul tur +fro gs +fly ing +cav ali +c ds +fri ger +street photography +re solve +tali ban +kan g +cru shing +ju m +ðŁĺ Ĵ +william son +tan g +cur ly +t man +veter an +fa ire +artificial intelligence +un anim +pre n +back drop +fr ances +oc cer +doro thy +work ing +ar thr +conver ted +day light +serv ant +pad dle +compla ining +thir ty +nad al +ak u +ibra him +ad dressed +p iss +green house +batt alion +si mulator +out lets +embroi dery +ðŁĵ ± +fis cal +ger ard +sas sy +ðŁİī ðŁİīðŁİī +vent ures +mer it +public ity +ðŁij Ī +sophistic ated +c tu +conven tional +condol ences +isra el +tra dition +ar an +te ss +gla d +ðŁĺĬ ðŁĺĬ +correc tion +ge on +am d +or ship +be ast +ch ment +ì ŀ +nic o +wk nd +wel s +cushi on +beli e +vo c +idio ts +under neath +pu ma +corn ell +en ation +lu l +swa ch +ab ig +u rer +mi e +form erly +ca f +er nal +chor us +juli us +sen ator +âľ į +wh ir +salv ador +ph d +uni fied +boo ster +graph ical +w rec +son ny +mi z +dere rs +s all +ven s +tusc any +wi d +y ong +kur ds +w az +trol ls +mac ro +cat urday +pre ssing +sa sha +cent ennial +gu sts +em c +be fore +den ise +cu st +ðŁĵ ¢ +lo oo +base l +eng land +y olo +ar du +manife sto +do ha +ì ľ +kni ves +bourne mouth +bi bl +bar b +al icia +Ø © +com er +cycl one +g it +ane ws +character i +vent ura +in tra +sf giants +hu t +be a +dar win +ell er +al v +re ese +bl y +kar an +conclu sion +man ny +fla kes +unite blue +nad u +co pp +ed ges +lanca shire +i als +o tta +philipp e +l ent +che e +ment ors +festi val +an ism +compli mentary +r j +pu g +d ine +we i +cli ffs +sar my +ti veness +treas ury +il and +after math +rabb i +ou n +bou quet +herit age +zi on +sur render +shen an +in ks +kar l +gh ty +pol icing +exam ination +ce y +per su +measure ment +hydro gen +lu han +âłĢâłĢ âłĢâłĢ +war i +о Ð +j y +fow ler +mis h +al fre +âĺ ij +bb naija +cat alogue +recogn ised +sa ver +hu skies +col in +mun do +si va +p ng +discoun ted +man utd +fre sno +de vin +prelimin ary +tro phies +pla stics +du g +pro cu +indi go +g ard +dy lan +pit ches +ground breaking +in son +bl ac +an thology +f h +expl ic +r ard +admi ral +so chi +la shes +splen did +en vy +ad v +sex y +festiv ities +stic king +bi b +thr ill +op p +ari el +botan ical +endur ance +fe males +br icks +vat ican +black pool +ber mu +br ough +roll er +bi d +sue de +sloven ia +mm ing +ml b +med alist +di ans +rehabil itation +ne on +s go +li thu +ram os +z ed +pi anist +inten sive +broad band +stu dy +peter sburg +lu ca +ah hhh +phys ician +dill on +tele com +gri ef +mu n +ac ro +si ded +s ly +blo ws +classic cars +tri um +ar gy +? : +h ri +marsh mal +âĢ ĵ +to pping +war saw +tran sc +preserv ation +b av +re friger +experim ents +ä º +gl it +sli ga +g age +fac tor +flav ours +br ony +sp o +cook book +carri age +aw ay +ny fw +on ian +w g +simp sons +ro lex +ðŁı ¿ +cro sby +ãħ ¤ +cre di +syn dic +pu bs +ali fe +poor ly +mac ed +ðŁĺ ŀ +behin dthe +w enger +n ats +ðŁİ Ł +rubb ish +procedu res +typho on +opho bia +er do +fu el +vi era +bu mps +millenni um +new zealand +lec tures +it on +mil ky +respon ded +ê ° +landsc ape +.. @ +bo ther +âĸ ¶ +z hang +huawe i +tu ition +s worn +in u +y or +pa olo +au ditions +ab il +malay sian +ho ps +fe athers +mp le +au ts +ã o +boun ty +ic he +ì ĺ +sh q +pin ot +ge ars +disapp ear +video games +t na +alzheim er +ðŁĮ ŀ +a ji +under wear +swit ching +sign age +o scar +ec on +dro w +cl int +pl ated +gun dy +emb lem +ho es +ici st +nel ly +juni or +road show +miner als +at le +alexand ria +ac claimed +v ell +shi va +ad he +en ne +amne sty +h ounds +councill or +ðŁĴ ¦ +aes the +part nering +influ enced +mag no +fl are +extin ction +civil ian +maje sty +va il +law makers +rac ks +mc c +ori an +sp ices +er rors +may er +co ca +pa i +s ooooo +reti ring +ba thro +ðŁĻĮ ðŁĻĮ +âĸ ª +su f +endor sement +buil ding +broo ch +pal la +arvin d +ag ent +kar ate +r hi +c tv +ta ine +um m +ba x +reig ns +uni of +enterpri ses +adel e +fla ke +at tire +bru ce +ba hamas +gra vy +sa in +che ek +tri vi +lo v +e en +bb lo +lady gaga +itt a +. "- +du stin +observ atory +eigh th +bloom berg +kh s +f cc +gi st +commemor ate +ve er +sexu ality +ed c +nic ole +vac ancy +u ser +son a +:' ( +dipl oma +t end +up grades +Å Ł +jura ssic +cardi ac +dr s +widesp read +à ł +dail ies +vend or +sim plicity +wi der +len ses +supp lements +de pos +ob served +vin es +parti ally +renew al +collabor ate +ali g +fin ity +ph u +zz y +pe tit +ðŁĵ ħ +z in +i gu +sm ack +fall on +ðŁĵ £ +back wards +comp onent +o so +compati ble +bin ding +zur ich +thom e +w ounds +ly ric +fresh men +sne aky +fi bro +di et +emplo yer +in sect +h ated +sch er +raz or +n sw +boo ker +califor ni +av fc + ° +preten ding +pep si +al is +un titled +k art +grand parents +e the +o ck +lux emb +visu als +small business +abdul lah +min ho +su baru +h ra +reve aling +heart breaking +clar ity +am g +sl r +** ** +âŀ ĸ +recor d +ici ary +min ded +ye h +exce ssive +knu ck +icec ream +tru th +ev ic +ta stic +ant arc +ren dering +, , +mit t +loren zo +st patrick +bound ary +zi g +vo cab +osa ka +fur n +tu n +gu l +s ounding +blo gger +utter ly +g af +adv ancing +l cd +mar gin +lifel ong +solst ice +sh ra +wa its +ple ar +bre ach +en ligh +ad er +itt le +c ation +ho on +stu died +?? ??? +k ash +ev angeli +ps l +wei ghts +met als +ty res +tur no +wi e +car b +g ale +se al +sun ite +am ic +patter son +á n +eu ph +up stairs +quali fiers +khali fa +apple music +ìĨĮë ħ +vau ghan +al ter +cru iser +mu a +t ana +kat rina +id ols +spo iled +secre tly +fi bre +part nered +um es +gi ov +com et +screenshot saturday +k eller +fil tr +fe t +con way +pe u +bad minton +gi d +m ound +don key +bu ff +lea ther +lar gely +bro ch +int ments +am use +r k +sto ve +impac ted +con t +cr acks +prison er +bar i +contrac tor +ori oles +domin ate +pol ar +am elia +dr c +ðŁijĮ ðŁijĮ +vi st +su arez +injec tion +blo oms +ðŁļ¨ ðŁļ¨ +sti ff +pay pal +sno wing +thur sdays +goo se +we dge +educ ated +weak ness +de cker +abud ha +bree zy +Û Į +hope ful +o bi +rai der +gh am +de u +se ve +par tly +fu t +infu sed +mer ri +than e +some time +hu e +me in +cre dit +sli ding +ran de +cher ry +dead pool +sh ol +ar am +under wood +sky e +distur bing +m nt +poli shed +guardi ans +ha dn +pic asso +ari us +ak shay +ir ri +j h +happ en +la kh +dal ton +at the +s well +mar sha +re h +cour s +j kt +top us +serv ice +r ink +hack ers +dono van +hor o +tc m +may hem +cha se +dev ops +ken sing +sc up +sh ere +quali fication +c live +ton g +n ancy +mar is +der dale +ber man +cinde rella +jol ly +ci c +loo t +collecti bles +hom icide +g ge +epide mic +su ites +mu ddy +gi mme +e rec +- * +tal la +lis le +embro ide +ðŁĩ© ðŁĩª +veriz on +ve ctor +be anie +arti san +ga in +flo res +vi gil +u so +ðŁĻı ðŁı½ +grin ding +gh er +air ports +respon sive +shaf t +can cel +ceremon ies +e me +at ari +bru shes +eag er +bo hemi +children s +yan kee +ma a +suspen se +mor an +mac ar +sun flower +cre w +vo id +ke ar +fashi oned +jen nings +sunday funday +sub missions +me ad +her man +wa i +crit ically +le um +baek hyun +for cing +co bra +ãģ ® +acqu ire +al k +ge ology +pri mar +import antly +ire z +bunde sliga +curi osity +sen a +stric t +con soli +win ters +ven om +chelten ham +ðŁį º +cen a +t at +ba in +glo ver +under cover +as ses +car n +memorial day +am eli +i rene +ch on +syn thesis +spe edy +mitsu bi +sla yer +compos ite +under stands +pe w +inter rup +hen ri +mor row +an om +thof july +g lee +thre e +ðŁĺ ® +and hi +ch att +renew ables +ye s +trans fers +!!!! !!!! +bab u +du ter +lo ops +pe ers +o ilers +pau lo +ic ation +h mu +war a +mer cer +hom eland +fu ji +ale y +year book +re m +re en +ab sur +bo is +] : +caes ar +shot gun +kur dish +o ren +ra e +anci es +ty pic +f h +def ault +re plic +lu k +trans actions +r ys +infan try +ðŁį ¾ +cho w +chick ens +ba gh +wy att +ay e +gg i +bre ws +ed itions +mi ra +commen cement +pre su +peris cope +ic hi +guatem ala +zam bia +pain ts +wit ches +wan i +un dere +cro y +vo ws +us mc +hear ted +theat res +shu ffle +le vel +mul tic +squee ze +fer n +app et +post al +mal t +on board +ld nt +co o +s sc +k ac +ðŁĺ ĩ +sc rap +mar cos +deal ers +ann u +mill er +co ve +ul ary +vladi mir +be ef +th ur +pick led +se same +bengal uru +mo tt +kathle en +hi st +no tor +dr ank +du chess +snow fall +e ff +tin y +j n +sy our +speci alists +scot us +bay lor +eve rest +mali bu +pre m +harm ful +l ali +b ates +g ye +differen ti +and ra +geome try +el over +black out +== == +ko ta +inter act +asi an +la yo +samu rai +fi del +exhau sted +gla di +pd t +spher ic +anti qu +guit ar +stu ri +ho pper +ang le +f ills +sla p +mi th +rod ney +ong i +in som +pre venting +cassi dy +ap ho +ore gon +lo in +ham mond +contribu ting +f n +gar ri +ori on +comp elling +escap ing +aim ing +plu mb +bi stro +be asts +concer ning +bo e +do pp +shop local +stumb led +âĤ ¹ +naz is +âĢįâĻĤ ï¸ı +gest ure +war ts +us open +hi ggins +char li +hang s +bom bers +° : +fe eds +c ch +st il +nic ola +ðŁĵ º +clam ation +tro pic +af ro +ou k +expen ses +der rick +al ine +fa w +reg ard +im er +sat in +thi um +ry der +pear l +te ss +mm mmm +sen ses +ðŁĩ ¹ +positi ve +exhau st +occu r +nor ris +lil ly +is les +direc ting +yo fficial +count less +sam ar +on stage +flo ck +mir rors +arch er +mo i +k d +vi v +in os +si kh +le i +sen sory +br its +kno x +chest nut +op y +coli seum +z af +di vin +adap ter +:) )) +tem ple +ku n +hel mets +t df +gu ide +m old +o ids +lu ther +he is +monaster y +sp ree +k lu +brit ney +jagu ars +gre ats +c cc +ky rie +machin ery +cric ket +re ro +ab o +aspir ing +semi finals +ale ss +sig natures +var d +me th +her bal +hol den +king dom +ap or +reg gie +ore o +palestin ians +em mys +sec tional +ro i +ney mar +qu el +cu ll +l ka +haz el +estim ate +ul ties +go w +be a +purch ases +bel ts +protec ts +m é +gue ssing +bb o +clau dia +fr acking +jon ny +el k +cel tic +al mighty +ra je +courty ard +ig i +can es +ðŁĴª ðŁı» +bank rup +le thal +âľĮ ï¸ı +graphic design +vad er +penc ils +rough ly +dan te +m fg +const ell +cam el +j b +bloss oms +en to +balo chistan +cine mato +ill ard +jer sey +con sent +dent ed +con templ +sch er +hol i +lou gh +st our +a yo +begin ners +cur b +v hs +a jax +du ff +av eng +dom est +commit ting +ai red +cha p +hedge hog +disappo inting +freel ance +in land +char ms +ðŁĺį âĿ¤ï¸ı +ai sh +m x +buck le +ti dal +per mit +bo ating +ra cha +kend rick +b ello +b hi +ple a +estim ates +l b +apo logies +jay a +bb l +ast oni +inter state +main taining +el bow +mu p +ep it +ðŁĺ ¡ +viol ations +def end +be h +sl c +am ir +pur i +ti um +fi fa +blur ry +scri m +ðŁĻı ðŁı¾ +ma ple +rel atives +âĺ Ŀ +cho c +con nor +⾨ ⾨ +whi sp +list ings +ma ze +than king +ri dd +grass roots +shi fting +desper ately +gor illa +den i +ju les +stra th +g ley +ja in +bu ick +t anner +ðŁĴ Ŀ +ga e +pri m +it ors +n ano +separ ation +armen ia +bor deaux +ðŁ ħ +pj net +bu rial +e bon +glo ss +re new +gri er +spe eds +comic books +sym boli +pur poses +ãħł ãħł +spati al +no table +ci on +n ps +ho ffman +nor man +rt g +du sty +situ ated +tr an +k fc +em en +nic kel +hast ings +sett ling +gr it +l ena +w aw +art s +gu m +ca regi +le wis +sapp hire +rememb er +embed ded +t lc +bl at +serge ant +el sa +boot camp +bow man +photo graphic +pill ars +direction ers +classi fied +no is +ve er +barre ls +wh oop +ðŁĺ± ðŁĺ± +fe male +petro leum +medi a +e fc +poké mon +ठķ +enthusi astic +var un +pro files +pedi atric +acci dents +con rad +jan g +jo jo +ac or +ob server +l f +live stock +for gi +fo s +el m +an and +go e +c ere +avoi ding +gri t +om an +thank fully +scat tered +nick y +cylin der +chees y +di ver +mahe sh +cav es +ear liest +qu inte +subjec ts +b end +gul f +vocali st +glu e +pat ches +un stopp +sny der +demonstr ating +pi o +hor ns +wic kets +and the +r ama +yo on +stra ight +bed time +or ang +bul lets +sa urus +min ers +inci dents +! ... +ðŁİ ¸ +ag ers +hand les +stat es +in ity +d ons +incredi ble +emin em +avi v +ru dy +moz art +folk lore +appli ances +mt l +fre y +di as +hu a +page ant +stri ve +im prison +bul lish +r ana +al erts +bb mas +hy per +derby shire +re cre +re dd +debor ah +cosmo s +law son +mel anie +psy cho +ho or +doo dles +sni per +shad y +man tle +canadi an +new year +inter actions +separ ated +cor ds +spiritu ality +ap u +it o +p ct +pel osi +rebel lion +se iz +wor cester +sec tors +ul i +san ta +Ð µ +ðŁĩªðŁĩ ¸ +bi ased +class ical +gam ma +dee plear +emer ge +back er +sur ance +hand crafted +ðŁİ ¥ +franc is +mill an +ic i +cro wn +wo w +stri ped +un fair +relax ation +³ ï¸ı +embrac ing +she alth +pale o +martin i +dist illery +wr ink +or k +na th +hay ley +cour thouse +si ber +sa di +quiet ly +mel t +m sm +me h +smart phones +rel ent +pp ing +war wick +co logne +gli a +cot ton +pro g +lon e +ip sw +star ters +expan ds +u mp +su ed +ski pper +infe ctions +ing le +à ¡ +cler k +demonstr ate +ac ar +ðŁĺĤðŁĺĤ ðŁĺĤ +ti bet +bun s +alo m +demol ition +ssi a +g st +[ ] +so ar +âĺ Ģ +ðŁĺ ª +ðŁĵ Ĭ +dee pest +beyon d +are t +att ends +activ ated +di mit +âļª ï¸ı +high lighted +magaz ines +rum or +az za +steph ens +dol ph +sho ckey +mat s +we av +mel an +serv ers +tra um +ku sh +æ Ĺ +bab ys +pa z +a al +la use +break ers +canter bury +ul ture +mi ri +euro s +tane ous +impre ssions +du tch +il d +gh i +pur due +adequ ate +l p +sy ner +ang ler +du rable +gal ore +ro wn +mg mt +ðŁĵ Į +lu cia +âĺij ï¸ı +zay n +bor row +. ( +north umber +cru sh +eng a +su sh +extra vag +t out +ma hal +ali stic +ther mo +gall eries +es se +chi bi +attrac tions +lex ington +legislat ure +docu mented +resi den +brow nies +w f +st ool +plan ets +sho ppers +conduc tor +ms p +tr icky +fru ity +end ra +feel the +whi pped +hair style +re fer +oo k +oc topus +audi ences +ku mar +after no +op tim +c fl +ni p +gen i +alpha bet +ann ab +lam in +accep ts +l ng +ðŁĺ « +t ine +ac om +cheer leaders +t k +gr on +v g +k ung +ja x +dha bi +r ss +mack enzie +beir ut +clean up +gy psy +st ell +bur ger +hurric anes +educ ation +st ina +âĻ¡ âĻ¡ +unfortun ate +jere mi +bad ger +at ers +: âĢ¦ +ter ra +subli me +stu d +y mca +mr u +duter te +bren nan +bul b +mel o +yl on +hack er +c red +gu d +as an +pad illa +embroide red +vietnam ese +pione ers +projec tion +re boot +id c +an ey +pri mer +suff ers +win ding +p on +sto day +mor n +u ch +all in +adid as +eliza beth +tu ck +o graphy +ðŁļ Ģ +be g +os borne +ghet to +r h +cn n +ir ma +ma kin +cab les +mur ders +oc ks +inst a +al as +si k +cu ff +la re +foo dies +o vic +at om +geome tric +em pathy +ภµ +cent enary +newsp apers +administr ative +ðŁİ Ĭ +sti ve +contrac tors +le tt +tas mania +awesom eness +den sity +ve en +prince ton +frequ ently +re ject +gh i +modu lar +ceram ics +sh ag +ki wi +can vas +sweat shirt +an j +ti mm +napol i +il er +appe als +hamil ton +ma yo +we ave +arrang ed +whar f +occu py +b vb +as aki +ot ter +nor m +vi es +de tox +tion al +dere k +id ad +ad missions +constitu ency +u pper +woo t +allo y +se ve +lu b +un comfortable +ed win +ab re +d wight +ar che +virtu ally +sp ol +pri e +ai i +er r +swit ch +bar ack +se ok +cou l +wn t +pou l +o live +caffe ine +cardi ff +notor ious +de mp +ex cess +bar r +t ford +a jay +bump ed +my thology +shel ley +fal con +shakespe are +must angs +no ted +bon e +civil ization +sy d +par sons +un official +hy ped +sp ends +oppo sed +v ings +space x +noti fication +deci ding +bio tech +out si +sal ah +! . +fe d +ss y +c ms +bad gers +cr o +ela ine +n ba +dy our +n ant +honey moon +climb ed +conom y +ath a +m ell +ne bula +nature photography +juli e +bm x +inve sted +mon o +lieu tenant +wat kins +techn ician +o se +ka e +ì Ľ +mc queen +pre ach +trav eller +flexi bility +ze bra +reta iler +p ant +ben der +brand t +squ id +war rant +veri fied +cas s +pier cing +hon ours +t ying +mor ris +kis sed +op rah +panor amic +me i +splat oon +wich ita +ari as +gal li +indy ref +good times +athe ist +confe ssion +ow ski +re pping +ad ditions +mechan ism +z im +j ans +su f +cho pped +beg innings +vitam ins +ãħ¤ ãħ¤ +or th +po les +ru b +antarc tica +indie film +web cam +ket ch +bre tt +cle ment +her on +defe ating +hydr o +buc ket +wand ering +sid ney +future of +b inge +on ies +knock out +administr ator +syn the +l ent +jan i +bar ley +premier league +ner ds +cr m +bra s +bot any +evol ved +rot ter +ro wed +tum or +weal thy +Â Ń +mon arch +li shed +da hl +ðŁİ ĥ +bu ch +ken yan +Ø § +red ness +assemb led +se mit +hud der +shro p +ran i +lear ning +mor y +iti a +geo graphic +worl dof +f b +pho sp +boo gie +am ped +? ... +che w +dwar f +ar us +s sen +ru sty +recru its +h k +gar de +app lause +vol umes +invol ves +ta c +hand bag +trans late +ffe l +se ym +aqu atic +trans fer +zo di +and r +acade mia +cr ater +te z +ar se +adap t +col oni +snow man +mal i +hang in +di schar +oy sters +pho e +colon el +w ba +hispan ic +thri ving +sh y +ag les +sales force +cre me +so les +la fayette +â ī +ter ia +ach a +sp erson +go go +car ly +the ore +am ore +vo x +af t +ãĤ ¹ +stap le +mu ffin +di agram +ino x +su stained +av ent +me ta +arbit r +dec ay +ado le +Ð ½ +ec ol +ph o +n k +o cu +gr anny +ç a +luxemb our +stad t +alber to +le vit +am as +d x +or phan +co bb +as c +lo gy +immen se +chan ts +off line +p ent +bre x +w inger +plan e +i el +nichol s +ca thy +nar uto +low ed +/ // +ignor ance +cat astro +you ts +sch en +buil d +haz i +s ine +critical role +du g +dete ct +lo gs +en amel +stpatrick sday +ed die +co pa +cigare ttes +ho ff +kay a +la goon +ra pha +air borne +choo se +puer tor +ke v +gui ding +fro sty +bor ough +mir a +ðŁİ Ĭ +cade t +anu sh +yo gi +e ger +fl ing +slo pe +nin th +we ston +foot wear +f n +may weather +a am +pla in +stair case +witne sses +work outs +ro bust +dex ter +co hort +ðŁļ Ĺ +sp ell +ha ze +o om +organ ising +wild fire +cont acts +av on +min o +upd ating +ðŁį » +li thium +ing ual +k is +au ga +lo com +de duc +u da +th ak +boy le +mp er +hot tie +eri k +re vised +is la +travel photography +oo za +en qui +confe rences +clo ver +g room +cur ves +live on +per f +displac ed +bo log +xx xx +ðŁĺ© ðŁĺ© +te al +ve ssels +rain forest +cal ci +pan ther +gira ffe +ta sted +imag ery +pad res +day time +bas s +ri pe +opio id +nu e +vin yl +invent or +sen s +process or +mu t +gad gets +bibl ical +shann on +jacqu eline +car y +the resistance +ali en +n vi +co sy +bi har +fo ley +ren d +mu gs +fa ken +cl one +ni allo +gra bbed +chi hu +power house +n tt +chero kee +spon ge +imple menting +rh ine +le one +ðŁį Ģ +pret tiest +infra red +impro v +swit ched +tu bes +con tr +bl k +projec ted +be aver +yo t +bbcra dio +thi gh +per secu +apologi ze +w ack +po ster +oli ver +az a +lou d +( ?) +f the +women shi +spar row +blu sh +us able +sc ales +it ative +peu ge +ne eding +legg ings +glam orous +mat ur +c z +wat t +da b +tam ar +et sym +bau er +heart felt +h n +else where +bir ch +alu mini +hu ck +e me +j l +traf ford +d z +por tions +ana sta +arthr itis +esp n +ber gen +viol ation +yo shi +c z +northumber land +clo sures +ðŁĩ¯ ðŁĩ +smi ley +r w +tel ugu +inten si +gre gg +ve ga +dun geon +south bound +ba il +domin ican +semi final +chap ters +h itch +van ity +trans iti +recomm ends +sati sf +bar ca +queen s +( ( +de struc +stra it +ra vi +dess erts +in tru +har am +k os +fo e +fat ty +pais ley +magn itude +dri dge +com ey +schem es +vision ary +our t +down loaded +ðŁĻĮ ðŁı½ +gd pr +lan i +p wc +gu ad +nic est +stake holders +re ferred +george town +arvind kejriwal +schnei der +in doors +all star +strand ed +gen der +ze pp +ma sses +ðŁIJ ± +pati ently +bl dg +z ab +we arab +vi vid +he ck +d ella +sy mb +je opar +la ger +à ª +comb ines +ne c +br ay +flo p +tx wx +jo ys +pon t +pro found +sur round +mad hu +ma ble +ay r +te as +n sa +open ly +er nest +ãĥ © +to po +g na +anti oxid +ti an +e tr +c ello +ma thi +gener osity +b iting +man ic +kel sey +chee ks +ten der +w th +pron oun +ultimat ely +gu sta +ari anag +ger ry +ble ed +red dy +mic h +mitsubi shi +oper ated +sex ually +ma u +cl lr +vi ds +co c +mel ted +ðŁĮ Ī +q ld +ite ch +instru mental +end game +ðŁĵ ĸ +ener gi +brow nie +tam il +at in +domin ated +pra ises +fire place +sens ational +men a +k arti +un prece +ru pt +ori ental +mc cor +tour naments +scen ter +re eves +prescri ption +sam e +fra u +tru ffle +em bo +roman s +bla sts +techno logical +pr at +b sb +y ar +tren dy +ac l +al ad +ðŁį ģ +o hh +bankrup t +tho ven +regar ds +is er +war wick +vine yards +real m +niallo fficial +do ta +ge mini +to do +v able +¨ ¨ +la u +wre ath +ju ve +nat asha +le ver +lor i +hor ser +cc tv +air bnb +es anders +sin clair +ema biggest +high school +con test +optimi stic +t te +ðŁĴķ ðŁĴķ +ss d +ye e +hel ena +con sen +ric ks +jes se +an ic +ðŁİ ¯ +re acts +ro be +independ ence +vol tage +m ington +s ant +à¸Ļ ภ+-------- -------- +sentin el +ke tt +rehear sing +aaaa aaaa +sof the +stir ling +sear ch +wi gan +stand out +sna il +pent agon +Ä ģ +ch lor +cru st +net any +chemi st +disapp eared +ric ardo +sp iders +bo se +war ren +me ssing +bann ers +gu el +par ach +ma id +coun ted +epi le +bon fire +speech less +se tter +meas ured +rejec ts +nik ki +le ster +foren sic +fab rics +alo ha +pre served +wat ford +deta iling +dar th +bo u +car ly +... ' +tail gate +noti fications +å ¤ +pas sive +trous ers +balo ch +ro ther +typic ally +à ¥ +sp it +wi z +sic ily +technic ally +ex pose +st age +hu bb +cre am +cap s +po ke +sle ek +ju ne +tempor arily +de z +awak ens +l ame +_ - +ji ha +tues days +advis ed +advis ors +exi sted +dis agree +news room +lo sers +world tour +dr ying +al di +har ness +foot print +hobb it +p mln +i ro +que red +asse ss +gaz e +sa b +th ian +í Ĭ +ti f +ob serve +ev il +dra wer +swee p +cor y +co dy +kyo to +cal lum +n inj +lau rent +be i +sket ching +custom ized +du r +regre ts +knox ville +ìķ Ħ +mess aging +grac ie +abun dance +bi dding +bre wed +fl ouri +therapeu tic +alt itude +ho gs +bur ner +elec tro +wonder fully +he ater +post pon +li very +r all +ad as +a ac +sau l +brook lyn +play house +âĻ¥âĻ¥ âĻ¥ +char itable +in y +z ah +compet itions +be av +plu gged +o is +do om +astron om +speci alized +max i +ta ps +cellu lar +depre ssed +folklore thursday +cri b +e mul +ë° © +fi gh +ru z +car lisle +spe ar +side walk +de i +depend ent +lac es +nh s +ðŁĮ Ļ +reali zing +net work +ric he +re gin +re fresh +st ral +pa thology +pla id +psyched elic +hin d +u ka +algori thm +lin king +progre ssi +fe y +d ade +hydr ated +b ant +fam ed +cot sw +bo ise +as c +rac ing +ja vier +ww en +mar lins +poo p +swe pt +toni ghts +we f +ani me +slo vak +âŀĸ âŀĸ +cla us +lem me +cli ppers +re ls +arianag rande +r te +ko t +thal apathy +hungar ian +zu ma +y von +is u +jour neys +clin ics +be be +ww f +n ws +super heroes +er it +sle ague +identi fication +mo tto +ba i +sour ced +ill er +ap i +pri se +unprece dented +dam as +tuni sia +dra in +undere stim +e ther +quarter ly +rewar ding +al ham +wolver ine +cab ine +hyp no +nad ine +hav ana +da e +ðŁĵ Ī +dr on +read ings +b ati +pic o +mer ci +iti an +wal kers +el ope +mi key +god zilla +bur lington +abu ja +social ism +at ility +sh ell +harry potter +g no +ab ur +re leg +fel ici +ro gen +neuro science +inst in +ath am +vou chers +j arre +fu se +def ici +monte rey +de port +mid day +pp ard +fre ed +ame ter +wil t +n ingham +pr att +liber ty +slo gan +o to +pr i +co ated +c pd +ne tt +il las +mal awi +evol ve +accessi bility +ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ +or nament +b p +el is +son line +chi ro +fl ick +ib m +ar ak +en ables +gar land +san e +cu ties +tri p +rotter dam +n ys +lam ps +lu cas +bo g +ra ils +travel led +hic ks +en u +sab ha +scru b +hi er +hart ford +fo o +fer nandez +tre vor +mat tress +appo intments +ale j +fe i +o logist +saf ar +oc ta +sr c +sha un +ambi ent +dri c +bi ker +she e +must ache +h ta +bo one +her ty +car dio +bra kes +rec ital +consi sts +overwhel med +cau l +robb ins +im it +al th +ur l +bi bli +on ne +black livesmatter +diffic ulties +tel ang +tall er +ðŁĵ Ĩ +deb ating +bur rito +mo vember +strength ening +bo e +te stam +mirac les +base ball +re nee +ðŁijī ðŁı» +al fa +âĺ ĺ +unstopp able +ec s +g mo +giftide as +path way +fen cing +ðŁİ ¤ +b ham +ra s +sk o +d led +thel ast +magn um +bin ary +wil de +wil der +wh ati +barbe cue +h ism +can oe +kur di +eli ve +advant ages +mad ame +bi er +mis sing +enter tain +air force +y ama +c is +hash tags +j is +ve il +dream y +ten se +may ward +ch ateau +hunt ington +âļ ĵ +v all +up on +bl ouse +dun es +ðŁĺ ´ +fert ility +m ole +curren cies +st u +ber lin +toa sted +div as +wal t +lar k +por a +hit ter +um er +chil led +bal ancing +fa is +y in +or tiz +east enders +h ate +ur al +ap ril +tim el +à ± +per o +sto cked +respec ts +th t +best friends +giving tuesday +be ad +inv ent +im i +nap les +comb ining +tok ens +thir st +ma sc +par rot +sp u +dent on +* -* +t res +subur ban +wid th +si ve +con tender +siri us +lo k +troop ers +outra ge +tur bo +frag ile +me ssed +do h +disc ord +netany ahu +re sign +forgi veness +mo han +mun ch +cam ou +identi fying +enab ling +hot ter +thorn ton +jai pur +ar ya +ðŁı» âĢįâĻĢï¸ı +mu staf +maj ors +o ke +du ffy +roh ing +til t +ðŁĩ®ðŁĩ ³ +rock star +she ep +hend rix +ra v +in vention +do u +lagun a +gru mpy +sw is +im pe +) ' +you ths +bun ker +st ache +oppo se +indi es +acceler ate +ml p +ed en +w ann +k ail +akshay kumar +su pt +pol ym +midd leton +extra ordin +wil son +australi an +alumini um +way ne +alum nus +mat ics +gri m +er nie +opp a +competit ors +rand all +h ence +decla res +pre aching +sha he +can e +sustain able +stap les +le dge +ad ena +doctor al +bur gundy +decor ate +ren dered +ri sen +pr ank +di or +bee thoven +flo or +ac com +to t +ho dg +touri sm +say in +objec tive +mar kers +premi ership +en abled +camou fla +gi ant +Ñ ģ +smo key +ric ket +pan g +de pending +s ation +evol ving +inter cep +cen sus +tof the +re en +mendo za +trum pet +marke ters +an it +ðŁĻ Ĭ +north western +v la +foto gra +blackand white +che wan +wi g +tro om +ginger bread +k n +ro mero +n fc +or chi +fun ko +sour ce +f s +ra ped +o st +tar ot +ann ually +ðŁĺ ¬ +r ill +del av +.. !! +se s +can n +medic are +ph el +ape x +guardi an +rema ined +r pm +a ñ +story month +instag ood +neighb our +p ing +sem ite +my stic +as cot +mat er +hand ful +dang ers +ti d +ana heim +opol y +sh allow +nami bia +tor ia +procu rement +big bang +announ cements +prosecu tor +beng als +sal le +en roll +ga stro +sugge stion +ba k +ha ul +budd hism +berni esanders +flu te +fati gue +cyn thia +cho i +ir win +gu a +str ous +h p +ba p +satisf ying +play a +ðŁİ ¼ +inst ap +al ice +t p +irri gation +ðŁĩ¬ðŁĩ § +in tric +clu es +ple x +sa x +he pat +dump ed +signific ance +by u +medic ation +pro v +tough est +corn ish +âŀ ľ +kel ley +u v +si zz +si bling +me st +di stor +diplom atic +aun tie +b hat +son ic +bren da +pump kins +ro ch +black burn +ur ged +shi a +arrange ments +floo d +sa unders +lec turer +nou ri +popul ations +diplom acy +consist ently +ðŁ¤ Ļ +t mund +cauli flower +l ily +vocab ulary +vari eties +coo ker +up town +qu ent +mo sa +re inde +velo city +spru ce +social medi +i ber +volun tary +proce ssed +bal tic +y ang +leban ese +d p +dol ly +arrange ment +y uri +cran berry +kal yan +elev ation +cli ff +pu shes +ìĬ ¤ +sil ic +co wx +eter nity +sla ves +vine gar +glou cester +con tained +breaking news +aga inst +renov ated +norm andy +hero in +ys m +mo ds +gre ek +un di +tren ch +v h +encoura ges +head ache +gr ange +: ' +ever green +Ù Ĭ +reck on +ab used +th ru +cho ice +ti dy +col der +scho ice +ha in +bru m +li ars +bre it +yor ker +sh ack +he idi +micha els +sco pic +fasci st +play ful +ca c +yas ss +sh ad +.. ? +qu en +ram irez +clif ton +pr s +best fan +âģ ł +gener ating +head set +disappo intment +abstr act +bo iled +paren thood +azerbai jan +exhib iting +bom bay +oli vier +ko so +un lea +mat ernity +iz er +si ves +r hu +col l +saskat chewan +fre akin +de k +na g +stab ili +ðŁį ķ +organi zer +bo sses +ar u +u va +at able +ta un +after wards +fert ili +ver ge +az i +mor ph +๠ģภ+jer k +cosme tic +ko w +stru st +ap ache +post cards +for mul +ì ĭ +spin al +jack pot +elec tri +Ã Ń +lo y +gra der +diab lo +ar di +he sit +f w +arch ery +pa sh +the ories +repe al +re live +per cy +âĺ Ĩ +im in +syn chron +sham poo +coup ons +o to +la i +thou ght +luxembour g +mo v +ðŁĺ ¥ +ge mma +se ated +m ga +strat ford +un certainty +shi fts +est o +fo ol +fire arms +cor rie +ki ki +appa rent +p ills +olym pia +fi d +elev ated +de cks +ignor ing +av alan +ro v +whist le +p tsd +milit ants +robo tic +pac ers +quil t +bankrupt cy +lic h +per cussion +celebr ity +al s +( ; +su t +pokemon go +h g +off s +gibr altar +scre ams +billi e +gen ome +mar in +be ams +arch bishop +em in +bedro oms +g ated +ol ly +warran ty +at own +cudd les +gun na +k ic +vi ve +cy mru +nar row +pro b +le o +refe rences +manufac tured +cho pper +brun swick +sem is +don ia +r ye +man o +hur ting +? # +hol li +investig ations +c els +ðŁĵ ŀ +le ster +temp les +sto rey +mc mahon +toi lets +wo of +ï¸ İ +le verage +at om +night mares +victor ious +haun ting +custom er +ag i +yo ongi +mon ty +ver onica +w ur +inti mid +blan kets +volu tion +j m +âĺ İ +am on +jud ith +ðŁĺİ ðŁĺİ +distr acted +dri p +hurric ane +and es +revel ation +tro op +ab leg +col lin +tibet an +wor rying +inter nationally +eat er +camero on +brad or +y uk +ðŁĴĹ ðŁĴĹ +tra k +slo pes +ci er +ne a +ol er +ta ka +albi on +volcan ic +am n +a fi +ob stac +face time +ger ing +n pr +metall ica +organ ic +ðŁĴ ¡ +ki dd +d ances +pemb ro +wash er +m its +om er +emo tionally +tan go +ip o +do cks +scan ning +spec s +tho m +the ology +emer gen +om i +g pa +selec tions +un necessary +ima ge +ter s +induc ed +gi gan +rent als +supp lied +m fa +shan kar +lat er +pa jam +cla ve +Ù ģ +ma hin +carl son +avi an +ano va +kati e +aj ith +design ated +chocol ates +investig ators +gla zed +prin cess +er ry +ra gn +ou rable +hr u +sun dance +peuge ot +steam punk +gh lin +gre ase +hi res +z ap +per ce +j ill +tom e +he hehe +joy ful +mae stro +ni shed +gene alo +v ich +p its +fox es +good man +emer son +lo bes +con verse +o ats +thom son +ra him +mal ware +ah i +man kind +re sin +im g +sw ood +kin der +sc roll +ar a +sak ura +ro bbed +xi on +ny a +c ism +ce dar +be in +mour ning +tor to +heath row +done gal +bar b +hydr ation +k or +elim ination +su pdates +hill s +appe ti +star red +ko m +gw en +dd d +cra y +sc anner +personal ised +seren ity +re design +meta ph +box ed +judg ment +no se +ë ¹ +er ad +ac ne +supp liers +ener getic +v om +as ap +ðŁĶ ¸ +ir vine +hat ch +la ss +ad ren +waff les +accur ately +ici o +itt le +se un +occup y +web cam +thene w +ent es +ga i +j w +accoun table +vis or +ir rit +licen sing +hudder sfield +gen ie +ðŁİ ¾ +atmo spheric +ten sions +spart an +clif ford +ol an +north bound +ame en +cen sor +u el +ster y +$ $ +far rell +hy ster +cl t +se dan +rep lied +descri bing +micro wave +sla b +pro sp +assi sting +ru bio +e than +hh hhh +gu ay +z man +ra ise +roll ing +o e +n ile +ambro se +scar borough +hero ic +coo ks +mor t +chop ra +ðŁĮ · +to b +shav ing +stac ey +dor m +motor sports +wi ki +fol ds +sp iced +stress ful +liter al +fu dge +pe ggy +wa ite +tre sses +se sh +pr ic +ðŁİ ħ +fri ght +r va +mumb ai +po m +tt v +cel lar +tom e +andro id +dor is +tsun ami +tin der +o ec +m wc +dor tmund +no thin +l iti +so u +believe in +at u +kno cks +mag ni +ss sss +ro hit +ine ws +ang i +m andy +ke ttle +intermedi ate +av ant +cur l +endor sed +ori o +ur t +consider ation +wi res +shel ters +b ino +vik ram +imple mented +ly dia +bu k +paro dy +c news +under graduate +canu cks +sam i +polit ically +ro tten +gh z +tex tiles +over load +moder ni +recre ational +fli r +bat on +typo graphy +ov ation +intrigu ing +pilgri mage +al ge +ad ays +tcm party +sp elled +cur ls +boo ze +ste m +ann es +ir ls +spon ge +sho pper +sig nation +bra ss +mi stress +le ah +beg inner +lau derdale +augu st +pre school +ta ping +tai pei +execu tives +b d +rhe tor +esc or +immun o +deeplear ning +stat ues +it us +manu script +ly ric +cor vette +mol ly +la ge +de p +cn bc +le st +je ssi +fi fe +griff ith +oppo sing +ran g +dr ills +respec tful +p ity +d ell +har ding +play boy +blo ke +shut out +k ili +o sp +se attle +bc poli +mis es +journ als +team ing +es ther +fre ddy +Ķ ï¸ı +metr ics +no tre +gar ry +for ty +navi gate +perio ds +bened ic +j id +da w +ance stors +restor ing +con g +aller gy +tit anium +c ence +lean ing +ab bas +v ast +uc f +roof ing +e man +seve rely +vo gue +ve au +in bound +d z +tane ously +stret ching +man chester +dr yer +dav is +kan th +the game +it ted +re tain +el les +conge stion +frat ernity +ol lie +lo ki +fre ely +cho o +pon y +sc ep +tab ly +bal t +rock n +di me +lo gging +ðŁį · +ad u +ha voc +water ford +char is +swee tie +run ning +ner d +erdo gan +z ara +weigh ing +fif ty +pre cise +low ell +kurdi stan +r yo +or th +syn th +lin ers +phenomen on +art illery +il legally +constru ct +nostal gic +gar th +al ta +shel ton +a sean +w ander +dur ban +di versi +bon o +cl on +le man +sh un +obstac les +appet ite +fe eder +respir atory +di xie +formu la +an to +so ber +extin ct +au c +ing les +legitim ate +; ; +min nie +ipsw ich +dram atically +ðŁijı ðŁı¼ +ingh am +milit ary +mon et +us navy +for k +dun no +play er +q otd +st oo +ex or +ethiop ian +film fest +pe red +c ate +sau di +in ner +sin cere +tion ality +ale e +de eds +cooper ative +ir onic +cro cod +br ary +post season +cam per +can ary +e in +exten sions +nb d +sher wood +spo kane +hu mp +jit su +ê ¹ +dar yl +p si +stab bed +offer ings +expe cts +cav al +body building +fr aming +f ca +ye arly +bom bed +sk il +resear ching +jud iciary +gree ted +tu dor +mil o +innov ate +ðŁĺ Ľ +r hs +ru by +contribu tor +fam er +soci ally +m lin +fi ery +ut ter +beau t +it os +de voted +rain bow +bar ney +pe ren +ar jun +r na +gab by +ut i +hann ity +pick le +ser v +qu akes +pp e +fe m +wh itec +j n +victor ies +ðŁ§ ¡ +gol fer +congratul ates +resul ting +mechan ic +ur ve +cen tered +kie v +an s +in cub +< < +c mo +bestfan army +dap h +en ham +on cology +ku sh +t xt +ori ented +fashion able +c sr +sa hara +r ack +pd p +han son +ภĩ +ti ers +ra r +pan am +in sky +sa hi +testam ent +asth ma +in her +fisher ies +or der +ho we +gall on +ep is +suz anne +drow ning +paneli sts +ðŁĺ ² +ë ¦ +al ach +commemor ative +at tribu +ðŁij » +mo o +visi onal +week sary +gu st +ak in +poin te +ee e +di spar +ni pp +dent al +st all +pi an +bor e +ul ster +tic k +ir r +tae hyung +micro phone +bermu da +ga ard +el er +plumb ing +hu gely +âļ« ï¸ı +race way +cam bridge +mar cel +burn ley +to ast +holly wood +fa sting +me red +hib ition +ca pped +benef icial +ow ning +cont amin +arab ian +to on +cap ac +hul u +sm ir +nutri ents +se in +graph s +con ditional +ðŁij ħ +or ac +play in +nor the +tor nad +mar ian +ju mbo +lex i +incredible india +road to +uk one +confu sing +sp h +shan k +pi ed +mq m +positi vely +sher ry +path ways +consi ders +tof u +argu ments +resil ient +che tt +with dra +ter o +ated ly +sw ana +he b +fli ght +har ley +decre ase +kind le +book shop +³ ï¸ı +marty rs +sm ur +mc cl +concer to +sti me +rejo ice +app lau +cle ment +mer kel +jai me +im mortal +isle of +mar co +youtu ber +stal king +me too +st ack +sp ouse +u st +lu v +âļ¾ ï¸ı +eque strian +ev ing +fl in +nick name +the big +as ar +st acks +wal ker +bor a +kidnapp ed +hur ling +humb old +rec alls +co pper +ann is +se o +mer ger +mu ir +ad dy +ðŁĴª ðŁĴª +be x +cr acy +con an +congratul ation +mid st +âĻ ¬ +for bi +op tic +cr ate +crocod ile +mad agas +secur ing +ast on +o gue +savi or +salis bury +love it +fuji film +cast les +as st +ar rows +sp acious +tr s +poly vore +progre ssion +m ri +nel son +bi m +indic ator +o da +pe pe +re signation +gu t +sne aker +log ically +az y +are lla +te aring +jo shi +ssion ism +q pr +mari ah +p x +ble ed +mi an +med ley +we iss +ker ry +gat ory +at al +madi son +av enger +nab y +pl and +gi les +fresh water +d ington +ta j +demonstr ates +n tv +bul bs +sunday morning +pe ake +souven ir +wa h +ton nes +m kt +complex ity +con den +ross i +b ing +y ds +su k +n go +mid land +ol y +life is +ri pple +mo reno +dd ers +tu s +á ĥ +bou l +x a +hol dings +wn y +shadowhun ters +ke i +asp ire +m ous +ow en +so ak +skir ts +moun taine +stor ming +ch rome +ri ots +sar ato +amaz e +less ness +nav ar +crit eria +ra fa +indul ge +ay er +por to +nam o +........ ........ +yi elds +val le +j h +mac ron +sa ins +dur ant +tra ilers +wo t +confeder ate +sh rin +id ol +form ally +ten e +motor cycles +than g +no de +bang er +dal y +p ats +enroll ment +au ctions +at al +ar bor +lo gos +de arest +trans action +dom ingo +fle a +ser mon +de ck +sin cere +questi oning +juli o +was p +pre tz +armen ian +k ham +inflam mation +picture sque +acci dental +film makers +ðŁĺ ļ +ðŁĴ į +ca sey +so b +yee zy +good will +parag ra +ss ly +fe ather +dy ed +assassin ation +na de +b cs +app lies +femin ine +fe u +ext ent +depu ties +l ack +psy chic +go i +kill ings +pse u +ðŁ¤ ª +un c +mar l +tan e +mck enna +sur fer +influ ences +free way +hack ney +mal aria +el and +te au +rema stered +Ø ± +raz or +gg y +cor ro +lak sh +fla ir +honest y +hoor ay +de pp +am c +wedne sdays +q a +ed its +- $ +se villa +dou bled +human ities +c cot +som os +r ine +af a +si oux +re construction +wel ding +th reads +am ish +encoura gement +po der +bo ck +bal m +p tions +stand up +accompli shments +guar ding +convic tion +ac ion +napo leon +depic ting +att ack +su i +wear able +âĸª ï¸ı +pot ter +esc ort +vis e +to ts +bo on +event profs +angu lar +womenshi storymonth +bar row +sch i +ac comp +ti k +l end +kensing ton +wol fe +st acked +cra shing +exhi bit +wing ed +sab rina +ma sa +k ms +alway s +et t +pla sma +counsel ing +pick les +nfl draft +mr s +inev itable +coura geous +staf ford +writers life +ho s +e j +gh yun +trade mark +adri an +influen cer +coron ation +ra ging +explo red +usa f +excep tion +eu x +tan ker +sw ami +pac ket +ðŁij¨ âĢį +f en +she en +a ero +j l +re gal +nw t +au ster +meh ta +char ge +a ste +b ate +inf eld +racec ourse +collap sed +fle ece +z il +al lie +alternati ves +geor ges +ðŁĵ į +quir ky +fc b +nat geo +philanthro py +bra i +every day +ðŁIJ ° +ach ers +ja an +fin es +q i +fisher man +distin ct +gri mes +nation alist +comm ence +ro wn +âĢ ³ +z ing +f ter +hr w +baro que +bl ender +kitt y +hoo ks +c ited +w anda +consen sus +reinde er +an and +supp ly +me ds +v n +ol ph +rat chet +shel don +secur ities +ë°© íĥ +cro m +mosqu ito +j eric +im mac +dimen sions +â ¤ +di ssi +sponge bob +dami en +steven son +jo anne +del ish +yi kes +than x +surve ys +postpon ed +alco holic +al ised +ðŁĻı ðŁı» +do ch +sen tim +mered ith +com pares +b ago +happy days +mo ss +ãħ ĭ +ne c +gn ment +frustr ated +comb in +ri v +ec lec +col lo +compli ment +actor slife +ct to +nic ar +op hon +apar the +man t +ja de +trol ley +optimi zation +eye on +eco logical +qui st +ep he +ॠĩ +cin co +appo ints +old school +c pr +behavi oral +min aj +:- ( +tag ging +ev al +jo aqu +ðŁĺ « +ha k +de me +jama ican +so s +hy att +hand book +libr arian +hanni bal +pump ing +ch om +f man +ga i +hu ll +respon ders +green ville +n us +vau gh +ðŁİī ðŁİī +ta xi +gold berg +man tra +te ase +forbi dden +metho dist +ati vity +* *** +ec t +mc gr +Ħ ëĭ +se b +amid st +disapp ear +thy ro +phili ps +er ina +v icious +stream er +million aire +ma p +str ick +hack athon +gh a +ed ic +mi ka +pe ck +ill i +anto ine +ar ca +op tic +ma ure +ðŁĩ¦ ðŁĩº +cla shes +man ly +âĺ ģ +al var +and res +me i +el m +ww ww +al tered +l te +ê¹ Ģ +mo jo +for rest +thal ai +non t +spee ches +acknow ledge +ign ite +x factor +ðŁ¥ Ĥ +mead ow +disru pt +debu ted +scrim mage +pharmaceu tical +fi dd +found ations +philosop her +et al +publi shers +bo ys +c ke +ru gged +opti mism +re be +phil harmon +nar cis +ral lies +lu is +go blue +fol ded +un acceptable +optim al +li sa +pol aro ++ . +en za +âĿ £ï¸ı +mon opoly +grace ful +dair y +du a +diffic ulty +judge ment +o si +mer sey +flu x +new found +ter ns +dimen sional +in vic +al ba +am it +abudha bi +alger ia +autom obile +the ad +lo tion +acceler ator +vac ant +iti on +lu f +al ic +pl l +bla zing +ba z +sen e +ðŁij ¼ +villa ins +direc tory +eis en +to ck +broch ure +ri pp +hb d +zayn malik +nic he +lo lol +certific ates +mor se +fac up +x ham +un wanted +im ports +carne gie +fan sign +mo u +r alph +destroy er +sw ing +trek king +cili ation +pit bull +g aps +ho well +defin itive +mc le +f ps +et z +bol ly +lyn n +gan o +at ure +fur suit +co il +na v +but ts +tro jans +eu re +en ko +sch umer +horri fic +install ment +br b +subur bs +a bel +vi r +de sh +cun ningham +ðŁIJ » +span n +sch we +ke mp +tr u +ste alth +qu es +le w +deli ghts +ko ch +hu mili +cr iti +il t +sp ells +mi ley +car ic +ðŁį ´ +lc fc +substitu te +oun g +? !! +af fir +predic table +class of +er r +cy press +chand ra +age ing +__ __ +ther land +don caster +el in +yo shi +sail ors +har ris +jo anna +niger ians +h ers +pla gue +pro cra +k no +can ton +busine s +un h +pra kash +c in +bow en +co ating +m als +be gging +smith son +ponti ac +sp ies +dam ian +pl ine +und ant +al ta +one ss +shame less +da q +bb m +wal es +stam pede +ser um +Ù Ĩ +cataly st +x n +ab sc +free zer +ch un +ari os +mc cre +fore head +he ars +damas cus +tac oma +ardu ino +encoun ters +stan ton +lg b +ab as +" .. +ke te +drac ula +ele m +g ne +zepp elin +la brador +pul p +op tional +or n +russi ans +san itation +hil ary +etsym ntt +pen alties +au st +ig ans +olympi an +medic aid +vers ace +va pe +re stra +pe ep +sexi est +st alls +di le +the a +punjab i +pupp y +tuesday motivation +ðŁĵ ļ +the flash +roc ket +mo dest +chihu ahu +on na +k sa +hur dles +ca ve +fail ures +sp lit +bo ho +gur l +disappo int +ho ward +nug get +fran z +stal ert +kaz akh +for getting +sch ri +ag ate +am at +eve rett +du et +veter inary +juli an +ch ills +bra ve +ghost busters +lan do +gre ets +profit able +d é +ti r +ze e +om en +pd x +gray son +har i +fix es +stab bing +swim mer +symb ols +compli ments +po se +func tioning +th nx +gi r +corpor ations +bar low +lo e +off season +distin ctive +marvel ous +nik on +enri que +ky u +ja ws +amo to +lom bar +travel blogger +fa h +ouri sm +tri stan +so e +ce ase +ðŁı ħ +z ac +mck enzie +taxpay ers +swim suit +bl o +les ley +kan sas +w ks +ki el +provo king +my les +str ing +kangar oo +galac tic +fif th +s ke +we ir +ll is +mat ory +ðŁĩ ¿ +un ci +re productive +roo ting +ti des +gad get +.... ...... +alex ander +bow ler +scre w +apo log +eri ka +wal ters +shet ty +lan e +ban ter +as ant +me so +v ain +" "" +us i +fer din +accomp lish +man sfield +bom bar +collabor ating +cla p +it ure +s da +smo ky +na k +im person +car la +com ra +bur gl +lo co +ti es +in hi +trac ey +se is +diss er +rr rr +dra y +prote ct +cor ona +hun ger +ck en +c eli +trou bled +predat ors +fic tional +shav ed +riche st +metab oli +ful ham +gro oming +mono chrome +wa sting +as co +ast e +ti sta +remedi es +ung soo +south end +perman ently +bu mble +procra stin +ident ical +practic ally +ma scul +su ke +assu red +val erie +devi ant +grizz lies +thi er +pur a +ne pal +not ts +bil ateral +spo il +car mel +cine matic +ph l +ni fty +ma o +hypo cri +la ser +pan try +mathemat ical +el isa +coordin ation +bel mont +a it +radi ant +bo iler +man g +f ag +cr c +h ams +br in +â¬ĩ ï¸ı +famil ia +âĿ £ +sab er +ru pert +gg an +rit z +mic h +sal ford +le vi +gra l +ðŁĴ ¤ +n ino +ce d +business man +ul tr +sim ply +compre ssion +pa ins +hal t +ë°©íĥ Ħ +landsc aping +n f +croo ked +er d +itt in +ddle ston +sur passed +ino a +da g +bl en +exten ding +at ing +al gae +ball er +u mar +snoo ker +col lu +flo wn +thu b +ridic ulously +ki sh +op le +di re +as ser +ari sto +sc iss +h ating +trou ble +syl via +suc cul +plo ts +sincere ly +al er +laure ate +br ack +att n +rif les +me to +collec tible +cu omo +conte stant +consist ency +ant z +rang es +abig ail +de b +mini ster +grow ers +an oo +hoo ver +dream er +nu cle +resear ch +mi y +sha hid +ma v +d honi +cin i +do j +hin dus +part ying +dal i +alon so +inform al +clark son +it ton +ki an +cit yo +mor i +la sted +as pen +libr ary +susp ici +qu at +den ial +fol der +ch ori +swee ping +eni x +ðŁį Ĥ +Ø Ń +nas car +handmade hour +mou l +heat wave +em er +exam ine +ib n +gr ind +po v +tion ist +m bo +she ila +integr ate +om es +take away +cer v +con nie +tic ket +ce led +bi en +visu ally +madagas car +sor ry +gu i +park run +tra its +la be +pois oning +ॠĢ +vi able +bohemi an +denti stry +bad os +spr outs +mask ed +te ddy +ðŁĺ · +sa f +sa as +ji ang +ti ght +spe aker +withdra wal +bc n +as signed +class rooms +fle ming +ðŁĴ « +super girl +tot als +table top +e books +horizon tal +cra z +flu sh +j ard +c dc +er son +ãħ ł +green wood +ni h +co x +ad a +lit re +go ing +v icky +cur ved +lou ie +gra ins +hy e +lon ge +reme dy +tra inee +san jay +super stars +ma ser +man u +s age +wh l +ðŁĺĤ ðŁĺŃ +ðŁijį ðŁı» +m sd +en z +rab hu +j oo +gh u +ac er +e po +resurrec tion +justice for +bl ended +mo da +avalan che +france sco +re spective +g s +ye ast +wel ch +devo tion +ge tin +athe ism +am ic +carol yn +lo c +ld nont +ave c +us da +le gged +bra very +b lower +cow boy +he h +sti ble +buff al +chann el +run chat +âĺķ ï¸ı +ide ology +best seller +y oo +pe anu +bon ne +fel ic +edi son +fr actu +naren dra +pp ets +seym our +ri viera +he ctor +necess arily +bi anca +soci eties +the best +w g +sent ences +win k +vacc ines +pal ooza +jam ming +as f +mp us +agre ements +ec k +ba c +hon ore +com pul +wild cat +im posed +yo ga +hud son +can celed +l ich +fu zzy +es que +ch uk +w vu +se k +fli pping +r hon +wi shed +wh a +cap ability +len ovo +ìĨĮëħ Ħëĭ +vi vo +tv d +nor a +sil k +pas adena +yo semite +valu ation +clo cks +u ber +mr c +dar kest +au bre +ss o +bell y +wrest lers +kill in +lou der +buck ley +ge el +ad on +un s +appe aling +ðŁij ¯ +semit ism +list ens +fit z +ãĥ³ ãĥ +ny lon +ar ty +seem ingly +hal a +su ited +et y +she ds +mu ffins +ap ric +um ents +u ta +jam mu +chelse afc +star z +yo ko +roo t +clean sing +di ar +pione ering +ihear tradio +dig iti +fin dyour +can o +ðŁĴ İ +z ol +spac ecraft +six ers +moi sturi +b ile +ti sts +hor ton +rang ing +colum bi +mete oro +senti ment +ep l +foo th +text book +drain age +r ly +sc ue +imran khan +ðŁĴ ¸ +margar ita +ed dy +predic ts +gamer gate +advis e +growth hacking +love you +ug and +v f +beng hazi +s later +ne wor +ch el +independence day +p np +cul len +hoo dies +num bered +brit t +t sa +kl tu +s ages +mom o +onep lus +col l +gu ts +w ta +mesm eri +enh ancing +chiro prac +j is +teen agers +m one +constell ation +sweep stakes +e ze +slovak ia +la ye +pear ce +wa ver +po gba +k ron +sur geons +mar x +ti d +gg a +desc end +p ours +upri sing +wal la +sab bath +bachel ore +mack in +k am +peter borough +hor a +ðŁĮŁ ðŁĮŁ +think big +r j +hy drau +sp al +univers it +ðŁı ī +mail online +league of +ten ants +w ally +lan ce +heav ens +dd r +bol ts +am ir +i phone +ci gar +en du +re i +el abor +r inging +john son +characteri stics +sal oon +algori thms +tal kin +m tn +di ve +region als +ff ice +hat i +deviant art +so tto +shir o +l ama +k we +f aded +por ting +tu mmy +est ates +buen os +ðŁ¦ ģ +beli ever +pen etr +dar n +sp ite +can opy +fashi oni +t illa +pet als +eli jah +bra wl +marty r +ë°©íĥĦ ìĨĮëħĦëĭ +mid town +eric h +d apper +sm town +me gam +ww w +le le +on s +cat fish +fir th +fossil friday +ball park +th aw +pot ent +illi e +cre ep +car p +so ap +gun dam +infe c +yy yyy +ठ¨ +z ag +rit t +calcu lator +bo ca +ok o +to ad +threat en +refin ed +olym pic +accompli shment +bacter ial +a ji +tat um +feli z +she ed +j at +th ic +jam al +ðĿ ĺ +lin a +ðŁIJ ¯ +jo king +yot po +pin ch +ak ron +her b +motiv ation +li a +ho stage +cre ek +gam ble +russ ell +patt i +fo tos +c pc +bro ken +back the +cla ys +u mm +stock ton +mat ernal +ü r +la kel +cent ury +be k +infe cted +ภ¡ +smack down +man ned +ta hoe +sm es +bas a +su la +augu sta +. * +rohing ya +gre ed +counsel or +silhou ette +gra vit +cla use +' - +bo bc +occa sions +now adays +dic tat +be ard +n ally +brigh test +kab ul +inc india +dhan ush +archae ological +che ape +mizz ou +d hi +ov ski +bax ter +asse mble +à ¢ +gi gi +ac am +wis ely +haz ard +north ampton +âľĪ ï¸ı +me th +bla sting +re unite +mu lus +ali zes +t read +mil a +ed ward +ko va +pe sto +ðŁij ¶ +vit z +hydrau lic +refurbi shed +mo tel +isab ella +hom me +sever ance +uph ol +mis erable +f ari +lat ter +ef er +crack ers +es l +ac io +yy j +in an +ec b +z ind +pan as +tru cking +re ed +sh aker +burge ss +em pire +ag nes +n ington +art works +fr s +ti le +bi ome +eu n +ch ong +americ ana +god father +go blin +i shi +! ). +temp ted +gen omics +mand ate +ck y +ðŁĴĻ ðŁĴĽ +som ali +br andy +in ven +spoke sperson +pc b +yu an +h g +fa z +starwar s +ro wan +blue grass +don g +d day +trin idad +er ton +ban ning +re tention +cu red +tober fest +re set +we is +deta ched +behindthe scenes +immun ity +ph a +bra y +ðŁij ½ +ran cho +ram say +est onia +nd tv +] . +cab aret +tar o +d v +show cases +plu m +ðŁij ¸ +son oma +pre pa +memor ab +e stu +drive way +u les +magn us +x r +nn n +much as +en ge +stre amed +fore stry +audio book +tro y +reck less +kil om +ru ler +ra k +proce ssion +i ons +po ole +noc tur +wh s +farm house +per a +par me +hypocri sy +s ics +v ant +cas k +holi stic +au st +Ð ¿ +in do +ðŁij© âĢį +di so +disp atch +ol sen +make it +en nis +cent re +ar range +ðŁĮ ¼ +sal ted +ea siest +f ate +reg atta +mo zz +ac an +sin i +g ically +ch ops +chick en +work in +ha gg +invol ve +wee ds +book day +wake up +ky r +michel in +fu ss +re juven +vac ancies +incar cer +m st +sc ents +sovere ign +kick er +à § +bo d +âĢĶ > +sa h +mob il +shrop shire +oph one +dress er +mis suni +hep burn +i mo +foli age +diagno stic +as san +cycl ing +guil t +c sa +puertor ico +win elover +wake field +do ggy +k he +pa pp +co g +al lot +cu ck +poe tic +mi o +re vit +mag ician +ç ¥ +ant enna +west wood +mber g +lux e +oat meal +Ø ¬ +te at +ffe e +sear ches +l ly +plu to +el on +let tering +inno cence +fa i +ann on +telang ana +ma it +neu ral +can ni +ar oma +a stor +fe x +co cac +mon etary +f ent +un sure +' @ +indi rec +teh ran +isol ation +li bs +make up +merce des +ff y +he tero +de o +sco m +cur sed +veteran sday +franken stein +shre ws +de co +ge ese +lefto ver +ha did +vari able +acade mics +carol in +under going +vari ation +na h +ssi er +gamer sunite +pur suing +emer ged +ll ers +control ling +ro aring +mete or +vol t +daw gs +be aver +is life +bathro oms +aci onal +pre vent +lake district +in als +y ani +gra bbing +sac ks +le z +sw ay +k ool +time s +klo pp +la de +con cord +resul ted +revi ve +recon ciliation +ol and +az z +gir o +mand arin +de en +nutriti onal +is coming +van i +aw www +der ived +love your +stop the +shou ting +nov ak +ðŁĻĮ ðŁı¾ +lo af +displa ying +sunday with +ma guire +ch eri +ðŁı Ł +re match +qu ic +Ú © +y in +ðŁĺ ¹ +ili ve +z ip +our ke +down loads +sw at +missi ss +care rs +t ment +proper ty +hahahaha haha +gi bbs +sur rey +ar ise +tic ism +sti a +ir ling +fro g +co se +bas sist +fore ig +lea u +pil lows +hol la +eli e +disclo sure +peanu ts +inte ch +ww c +plun ge +trium ph +cor i +sli ppers +ðŁĻı ðŁĻı +neutr ality +ma re +hair y +gang ster +hu mming +cust ard +mer lin +ale a +s by +dam p +mo han +ver bal +j st +gu tted +b jor +un finished +ðŁĩ¯ðŁĩ µ +un happy +âļ« ï¸ı +by pass +at su +fis cher +sa v +afric ans +re use +mid way +demo lished +ger rard +her cules +Ä Ł +medic ines +cl icking +sur round +jo ong +wav ing +tri bes +wet lands +offici el +argu ing +l le +do va +su zy +club house +ne gro +ob tain +ga o +gl ance +assi st +ch os +ãĤ ¢ +âĺ ķ +adri d +occur s +st ans +par don +livel i +emplo yed +re visit +ff xiv +bb le +ne aring +min er +ðŁĺ ¹ +giov anni +up to +mar vell +mar se +to wels +cb n +engine ered +y elling +spart an +si ans +ðŁĻĮ ðŁı¼ +se v +coyo te +sta di +t cm +app en +shenan igans +open access +so aked +ma squ +le vine +stro kes +l k +aparthe id +hipho p +char don +may may +ha asan +stri pped +fr o +scri ption +f ton +h f +pri sons +marsh al +ķ ãĤ +an cho +com promise +classi fication +buzz feed +bblo ggers +deser ving +) / +s way +ob o +camp ers +poder nfamily +p oured +bri e +squir rels +se ize +: # +le k +ti mb +st acy +nas daq +repe atedly +br at +mi ghty +competit or +mah one +de si +o ke +bm w +shi e +f cb +cheape st +minim alist +par amount +n ate +har as +insan ity +lat eral +ment ality +mo zam +ta pped +yad av +u sp +b way +the od +bil t +ra ids +em press +adap ted +pat ron +nut shell +ag ra +be aded +sundaywith marsha +vi king +proce ed +main tained +thinkbig sundaywithmarsha +sn es +mus ica +to wer +ch ab +bo k +sm t +insul t +harve sting +windo w +ru ther +be ige +dec al +indic ate +ma iling +ri ft +po le +ander son +ch oral +sp ride +l ili +ev elyn +imrankhan pti +.... " +ke red +un dp +water falls +se ars +le mans +world series +ri el +ani e +app ar +score rs +lam p +a than +phys icians +qu inoa +refu sing +vu itton +unle ash +s la +pat i +shou ts +inten tions +fo amed +europe an +neighbor hoods +me er +man son +du h +br at +con es +bow l +kazakh stan +ठ¿ +in appropriate +del hi +ketch up +ful ton +s ys +consul t +gar field +to go +f ml +f led +b ds +facilit ate +ree bok +selfi e +elev ate +activ ate +bi ble +ca wx +b ys +cam ille +sy ou +sk ool +her t +w bc +ple dges +recor der +po sh +ac re +so aking +mat il +v sco +shoot ings +pla r +e con +ðŁĻĮ ðŁı» +rashi d +u bi +ðŁ¤ ¤ +sw inging +wi pe +rap tor +m su +music video +dur ham +at tic +apar ty +fe tus +activ ation +aa z +motiv ate +ðŁĴķ ðŁĴķðŁĴķ +j al +ठ® +ag on +sche er +stal ker +fo ster +az zo +tele gram +vi gor +s laugh +screen shots +entrepre neu +kri stin +inten tion +ch illi +fr action +don a +ge a +tc u +s ite +la k +em il +d nt +bor o +wil kinson +re cu +ato day +t anya +bl anco +cd n +brilli antly +g cc +ac c +evacu ated +ther ine +den ny +cait lin +she pard +pou ch +hand held +sou theastern +ha a +à ´ +re solutions +led ger +sr in +r ar +shat tered +chim ney +im with +mete or +hand led +ra ke +town send +en han +shi py +duc t +tw x +inflam matory +war hammer +theat rical +gro s +sk ar +sco tty +ni el +tit o +tin i +conne ction +_ . +goldeng lobes +sha q +ðŁı ³ï¸ı +hall way +fron ts +effec tiveness +gla ston +d hs +ex pi +to h +c pl +sc s +re o +ha g +resemb lance +hor an +abu sive +qu er +virtu e +cho lester +a q +shan e +m ce +carri ers +di stress +re wind + ¡ +voo doo +int act +ann o +ðŁĺ ¤ +pi led +adi a +ãĥ ³ +en ow +di gs +light ly +goo fy +turb ine +governor s +con te +re open +pa h +i ve +cra fting +swee ps +jo di +an de +zu cker +kaw aii +o ko +v ai +out line +kri sti +ts n +insp o +qu int +fil thy +lyn ne +listen ers +depar ting +or d +t weed +, & +ale k +sel fish +nor ther +recogni zes +i ps +be s +a ed +w ills +pe at +surround ings +mon uments +ais le +be cker +la v +quant ity +v ah +helicop ters +tu cked +alv arez +sha pe +o bey +ad diti +road side +m ite +bl ers +ep age +j au +ignor ant +b ins +lu lu +x o +c fo +ee eee +apprentice ship +shef fiel +to i +ho k +faken ews +deplo y +aid an +husk ers +ãĢ İ +west brook +mi ster +confi gur +car r +fic a +proceed ings +ha w +ste ak +mur derer +pay day +a jo +p vc +don ates +bi af +nom nom +be it +k ali +x rp +ahmed abad +se mic +che y +x tra +an twer +head lining +squ ares +roun ded +flu ore +bol d +disa sters +am oo +gener ic +cran es +brief ly +gi g +auster ity +anticip ation +for ti +treas urer +cann y +ce cil +dete cted +check list +ภ§ +pam ela +bar bados +an field +hear ty +tx lege +peren ni +arro g +ing ram +âĹ ı +ty ne +spo on +r ation +am ba +m be +cam el +h hs +york shire +reflec tive +fre aks +to k +ju do +partic les +du bs +ban jo +accred itation +prover bs +over dose +inte gral +gu ang +mc s +super car +af b +al vin +ail s +x tre +st aging +tw ent +rabb its +mar o +inste m +dol l +cr ay +sant ana +ble ach +mini ons +che ap +man t +di vers +catal onia +lo is +mat ri +cou gar +kay ak +e gre +p so +a ia +å ® +char lton +tr acked +sc ari +pe tt +f wd +x in +gra vel +br ic +bigg boss +ar den +hu gging +pal ms +st v +li mb +the movie +handic ap +ri me +z ai +stu b +indi a +lithu ania +rhy th +p ita +maced onia +high ered +brid get +schwar z +ske let +hi kes +ant arctic +c ps +mash up +Ð ° +n ell +chand ra +he ir +an us +sher idan +mi mi +muse u +bec ca +an ir +bar rie +dioce se +compar able +ðŁı³ï¸ı âĢį +yuk on +me p +hor mon +mer ic +al f +con quered +christ church +ðŁĴĻ ðŁĴĻ +hazard ous +poo h +cont ing +retro spective +par ame +na ir +con sor +ho tra +astoni shing +cater pillar +u man +ti sm +t vs +serv ic +croy don +mor ales +c g +cu m +te ur +scan ada +s all +magno lia +el ise +th our +à® ¿ +ag omez +phel ps +ë°©íĥĦìĨĮëħĦëĭ ¨ +wh os +weav ing +si sd +pro poses +cro ws +pre sale +econom ies +bernar do +sha hid +air show +mc cann +hor ticul +nr l +du el +mongo lia +tou lou +requi rement +struc tured +ed i +o lives +he a +cu ter +Ð º +enthusi ast +harri et +domin ion +sub mer +ðŁį ĥ +sa ab +nes burg +mo ff +def ended +bur t +rewar ded +gold man +op tics +khali d +house holds +buc kets +ce cil +che ss +substan tial +ef l +oper ation +evalu ate +st n +rece ssion +l ll +tom as +tru ths +ak bar +s words +p act +embarra ss +ha o +ay urve +scrip ture +ny cc +op t +di ameter +sc ented +organi zers +re lat +ha e +dream ers +de se +ðŁĮ » +restric ted +n ale +r hp +dol an +mun ster +ha ired +consult ants +jo ints +hu mil +d ill +relent less +t é +af il +ut ilities +japan ese +condem n +pet ite +colli de +q f +peach es +cou rier +l ore +âĺİ ï¸ı +reli ability +ch uk +ðŁĻ ĥ +stu res +ge ther +ho stel +bi er +- _- +â ĩ +e ze +ta ilo +di ent +blu ff +chu ffed +pil ip +mon arch +e em +bu chan +b ick +op au +ku ps +ภ¢ +pist ons +sp ins +m and +ce st +bur ne +v ile +cher ries +bec kett +need les +pan ch +ë Ĥ +haha h +trou bles +insi sts +do you +g mc +mor tar +deleg ate +in n +g anda +sin atra +ठ¤ +spee ding +pu pil +pre mises +ali gnment +pi kach +as us +j alan +Ø µ +lime stone +fol kl +parme san +ce il +mo y +shawn mendes +ac up +hu st +ot es +med ina +ma di +gta v +censor ship +ar g +swe eney +sy kes +col o +foot steps +cann ed +adv ance +gta online +healthy living +ðŁį ¾ +a ig +p ality +oc s +he brew +im minent +berk shire +jeremi ah +out going +bak er +entr ata +ma ids +gro ves +bo c +a del +m fw +con science +arm ys +nut ella +conte stalert +novel ist +la h +ban ker +marque z +ðŁı ¡ +to ff +out age +gr p +ðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃ +musc le +du dley +nvi dia +mi di +m uni +ess ays +dat ac +car ter +ภ£ +t ans +i ves +public ations +al er +ok wx +il u +cu tt +har p +out law +luther an +br ill +bo lic +do well +green land +be sties +path i +pay ton +gue st +har den +ðŁ¤ © +ann ed +evacu ation +po ised +mc der +b han +o i +envel ope +ci d +ca vi +ta pas +book review +grey hound +âĻ ª +fe ud +lun gs +for te +rai der +ff er +oni x +dep end +yn wa +rel ating +de vs +ðŁĴ IJ +acqui res +d ha +j yo +priv ati +can ine +k b +cra b +sar din +imag ining +k j +em por +down hill +ne z +ta eyeon +nick imin +gb p +à µ +w ap +sec co +ma shed +ðŁĴ¥ ðŁĴ¥ +augu stine +diss ol +dic tator +â ĵ +vi per +ed fringe +vau x +hard work +book let +no x +chi ff +ðŁĴ ¨ +observ ations +xbox one +u sher +ke er +lu p +dal las +cal gary +ma dra +di ous +k bs +wood ward +hero ine +lu mber +sea world +o ws +mc ke +maver ick +gu la +cross roads +fan g +s ade +nik ol +chee tah +me c +pp g +er ick +ðŁİ µ +tox ic +bj j +viol a +sp ire +ch ino +tra vis +institu tional +ha as +low ry +w ac +ea e +hu mid +mp ton +ru ck +je w +c ine +zim mer +se f +bhar at +fre es +aam ir +ðŁĴ ħ +z inc +wan e +multi player +royal wedding +e el +preci pit +qu ery +kimber ly +isa bel +ful fill +ig an +vau l +pan e +sc y +dig it +gun n +u tah +dog day +fi on +xia omi +da c +el ast +cha vez +ro blo +g ine +ten th +ab h +ke to +hur dle +na dia +memorab ilia +ha bs +qu an +h w +hv ac +pix ar +ec cle +kram er +accu ses +ðŁĴļ ðŁĴļ +per se +mean time +wa hl +atle tico +âĢ¢âĢ¢ âĢ¢âĢ¢ +ott oman +no vo +k us +conne cted +tru sts +d mv +spen cer +rahu lg +do ve +sto kes +bolog na +enthusi asts +à ª +rockstar games +ted cruz +du ras +s acked +late x +immer sive +cer t +lu cin +princi pals +fa res +sa ils +far n +am ent +saf fron +quent in +check point +fer ris +ex cur +ðŁijī ðŁı¼ +bai ley +se h +ter re +mad am +s band +wan derers +cumber batch +yy c +digit ally +blackandwhite photography +roll in +moroc can +ðŁĮ ħ +din ner +d well +to om +m ye +ez ra +cp fc +war hol +me er +jon ah +no aa +s gate +so on +secu lar +g ating +ti o +dri ver +si ssy +assan ge +ta th +ed mund +bobc ats +ra ji +po stage +stu ds +m gm +kat o +edin burgh +meet the +shir t +fa a +mens fashion +sp reads +wi m +car ts +phoe be +j ars +bot swana +Ù Ĥ +ed war +sk ar +ri ve +gu sty +c tv +ferdin and +su therland +nickimin aj +k v +si us +bee ch +re z +desi res +on ial +camp o +quar ry +lor raine +gil more +ig gy +µ ï¸ı +ho pping +avi z +ðŁĮ º +uni sex +dedic ate +att itudes +ste er +jun kie +rail way +y b +whi sper +key an +k us +ju g +di x +a ins +sum mon +ov ich +sy ed +her ald +ma ison +me ded +wild flower +main land +ri sky +ru kh +over looked +ki c +destro ys +nam an +ki p +z ano +champion sleague +ban dit +quin cy +smi le +cal vin +open ings +ta pp +ol ulu +spec tro +accred ited +ap k +pra ised +bar nett +pol len +premi ered +selen agomez +tou red +screen ings +uu u +mis o +en se +adam lambert +guel ph +har yana +hu tto +le ar +l tc +po ached +brex it +æ Ŀ +tt c +pa vement +mon gers +ro e +ad ers +ling ton +particip ant +ca red +ga il +y ates +lan tic +dash board +jo o +feli pe +ssi onist +bu m +s end +a eri +thu gs +luci fer +a he +dete ctor +fil ly +gas oline +ham per +hump day +the ta +the band +fore casts +o hhh +lo bb +hol l +cp u +az u +ad ar +hai ley +bu b +car t +quo ted +an archy +pan cre +twit art +al den +st ash +the less +or ni +belie bers +mor mon +partic le +avi ation +⬠Ĩ +webcam toy +sad dened +cru is +ham let +n ct +roll ins +marque e +saw yer +reli ance +a ura +di ec +soo thing +sig nings +ak is +à ³ +at kins +aer op +ðŁĮ ¿ +y ab +sh ari +con nol +du bbed +manufac ture +convin cing +feelthe bern +ra u +pu lit +on ec +gem stone +ur ging +bag u +ga h +aci ds +fi anc +zodi ac +sn oop +her rera +initi ated +ven ge +profess ors +pro di +stron ger +e mission +bb a +hal le +ta pp +haw an +wh im +compe ted +myr tle +ir port +cold play +ach e +ske p +m son +ss ic +calli graphy +swim mers +me y +pp c +thri ft +po c +re places +commu ter +âģ¦ âģ¦@ +go ers +lo gue +para dig +bas kets +sensiti vity +joh an +atl antis +& & +suit case +anxi ous +l h +str i +gal loway +stre ad +war den +gr ounded +ffici ency +li feat +reli c +disgu ise +island ers +f cofficial +classical music +b mc +en field +bi que +oak ley +bat man +sla ying +ner ves +mul tit +calci um +projec tor +scott sdale +ant ino +gri ps +kim mel +des mond +prote stors +hi atus +metaboli sm +conclu ded +press er +ti pping +sli de +e to +hun ting +aus open +ri k +pp ery +innov ators +pitch ers +ag ger +fun gi +z ad +proli fic +rockn roll +bl ames +ct ar +stam ford +q ad +mozz arella +insan ely +den ver +ph ouse +nom ad +ï ¿ +s ris +pro du +hen ley +pag an +am trak +ru bi +in cl +tu tor +sco tia +wo es +sing apo +fun nel +turn bull +know ledge +gri mm +real madrid +we are +missi les +con sol +emo jis +sne ak +smi ths +ru iz +br ou +i el +ha ver +ðŁĮ ļ +kin gof +basil ica +circul ation +prin ters +ta pping +ri dley +dra gged +ha j +writ er +fundament als +personal ities +me tre +stereo types +bur le +best of +n ffc +ha th +mini stries +a ali +trac ing +pav ed +ł ï¸ı +g ic +insp ire +tu g +ha re +repe ated +ex pon +lol li +rho de +pre cin +install ations +instag ram +az ar +i es +sole ly +du kes +mission ary +van guard +fursuit friday +on d +pol ari +ma st +har an +jos é +jack ed +ec oun +al ities +ne ph +ra vel +moder ated +sco w +s fb +uru guay +as o +ni g +au du +p ints +lat ina +ben z +m itting +char ted +mat ology +cit ro +biop ic +ðŁij Ń +djo kovic +fox y +agu il +so to +an ada +sin king +sc rap +hair s +bethan y +fact friday +ðŁIJ IJ +unlea shed +) ( +contra dic +ram on +coast line +y ong +sn sd +li gan +p ome +mit age +ge tt +wat i +ri sk +so aring +bru sh +f pl +av an +å Ĩ +lar son +sh ear +mul til +blu r +multi media +chun ky +par i +n ani +weir d +cholester ol +char les +dream ed +tan ning +puzz les +fr am +hand ball +ch ag +beli ze +al u +bang s +Ñ Ħ +detec tives +mc g +ish q +bo thered +saf c +mp ing +ten eri +g ays +sail or +an gi +mul ticul +gue ssed +ros é +high ways +bro om +chatt anoo +- ' +see ker +on ed +at f +lu c +> < +bar i +per cep +jewel ry +as ph +sor row +sl ing +mam moth +jac kie +ë § +wilt shire +sa o +can cell +im paired +tor ial +bre ed +guy en +jud ice +tit le +pro spective +applic ants +ðŁį Ĭ +epis cop +e id +b yo +stock ings +ðŁĴĥ ðŁĴĥ +ll p +sna g +keep it +l ough +ol son +matur ity +!! !" +cop ter +i sha +bl i +wil mington +tr youts +th ai +ðŁ¥ ³ +pe bble +kra ft +f p + º +ssi vely +li vin +contest ants +tex tures +jo an +h dr +film festival +prov ence +wi do +op end +c si +sto wn +cro ati +ad just +host ile +analy sts +il an +cu ppa +bru m +newfound land +good win +me tt +mall orca +plu gs +bu k +bb hutto +wrest le +sa ire +sho pped +for za +le head +vi vo +ba st +ro xy +reg is +hard working +hon olulu +desp air +young sters +ni g +impro mp +roll tide +de emed +tre ason +ru shed +for ged +ff f +pikach u +bri ggs +do it +ac cent +la us +gla ze +compet ent +a ho +photo g +mid field +le go +har vard +min orities +re illy +slic ed +once upon +initi ally +financi ally +landscape photography +har dro +qu o +mm ers +par kinson +smu gg +read iness +bru tally +glou cester +mp ed +bbhutto zardari +mur der +ye d +dat aviz +sr t +dow ning +bi ans +m ü +fle ck +fli pped +s ly +brilli ance +ri m +k um +bubb a +ko i +knit ted +sor g +ma is +ðŁĮ ² +ti ss +su stain +sen su +ak han +zi est +exam ines +chardon nay +user name +short list +re bs +on o +dar ing +hard wood +che que +righte ous +light ening +dir k +shra dd +du ra +down stairs +sh al +ami gos +ru ff +s law +ri es +red nation +man us +ðŁĩ§ ðŁĩ· +distin ction +u bun +dur an +mi gra +thi ans +la ver +domest ic +k x +jaz zy +justi fy +belong ing +insul ation +color stv +drun ken +chann eling +qu and +xi ii +enligh ten +kan o +fati ma +teen choice +terri fied +p ba +as ley +met museum +dun e +pack er +ki o +ðŁĴľ ðŁĴľ +bo iler +fas cism +ar mored +back grounds +in mates +embarra ssed +defin es +th d +we go +silic one +lo on +el ding +bor rowed +he mp +ak sh +kaw asaki +br y +de af +kill er +dispo sal +ðŁĩ ° +glaston bury +un covered +o xide +po ff +d ant +k j +ku ro +dri zzle +peop les +fe e +pro pri +dd lovato +pi ggy +ot is +aller gies +u bis +pengu in +ser a +vi z +prosp erous +ici des +tornad oes +sene gal +web cast +sto red +enchan ted +bb cone +bay area +entrepreneu rial +rednation rising +experim enting +ang an +lot to +they re +por e +er p +seren e +east wood +bro kers +bar ge +stal lion +timber lake +tailo red +dy stop +b ate +lat ors +di xit +bran son +dynam o +ky lie +shame ful +bt wn +spring time +mix ture +s ounded +lu ton +dad es +mal a +op ra +en ic +rahulg andhi +se wer +~~ ~~ +ky u +nor theastern +ca er +bc u +nir vana +kitch ens +ous y +al m +river dale +hid den +fl int +sp d +pat rons +katy perry +au gh +exhib itions +sm c +shu ts +at ore +da in +some thing +ber th +bo g +por ter +gen to +con cussion +ang lic +ro we +gr illing +scar lett +master ing +mor nin +comm ented +si me +si zing +christ y +ce os +st m +at ry +tari ffs +vac ation +pre judice +p su +paren tal +far age +can a +cap com +koso vo +you re +men stru +stal in +grape fruit +br an +che sa +dav en +exc el +!! ) +๠Į +distribu tor +ce a +bride sma +millenni al +wa in +ob serving +mis ery +plan etary +expo sing +bra ised +comp ton +don gha +q l +spring steen +th ul +syl ve +cab o +pal ad +niel sen +gaz ing +ba ja +r oud +orchi ds +johan nesburg +se man +d ji +oper ative +affe ction +eclec tic +at c +mut ant +aw x +nic e +mel bourne +indu lg +tu lip +dias pora +wel p +big gie +mississ auga +retri ever +or an +tam my +c ta +hipp o +seas oned +ger mans +eng v +marvell ous +im f +rela ys +mon tan +maur iti +me ister +as surance +reig ning +su fficient +han e +no thing +pos se +nav y +in love +brigh ton +en qu +ch ung +sweat y +es c +cal ed +man s +nicar agua +sl ices +mo cha +washington post +bb n +dam ned +grow ing +en burg +lo an +me s +wh oops +believ ers +spi el +vo daf +l at +s led +cricke ter +brown e +golf ers +bar ra +wat chers +lu igi +sw amy +mom s +pit ched +san tor +cr s +si re +sc amp +bo de +ste war +jon ny +ent ity +pac qui +mind ful +min india +bear ded +temp t +scorpi on +eat on +authori zed +ar to +s vp +op athy +cch ini +house music +disney world +âĢĶ @ +pro pose +di y +expen se +ten g +pupp ets +sm el +d aca +per ry +fin n +boo sting +lefto vers +cou gs +satell ites +man y +az e +g ong +fi e +metho do +fer ries +ðŁ¤Ķ ðŁ¤Ķ +explore rs +load er +attrac ted +il ton +godd amn +pi azza +doc tr +sav ing +paragra ph +visu alization +may ors +work flow +ack les +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ +ठ¸ +twer k +clu t +lo ver +te ases +si an +o te +deter ior +accor d +l fw +swar ovski +nat al +tra ps +k ina +analy ze +laye red +bever ages +un it +ran som +pe shaw +dest ined +astro logy +si pping +miley cyrus +cam ino +marshmal low +bli ss +out back +fa q +int oler +humil ity +po ppin +hallo ween +mon tene +op hy +nu n +tattoo ed +a as +ðŁĮ ³ +dale y +qual ity +du sa +fisher men +swi f +ter rac +st au +le in +trol ling +ship ment +garden er +march madness +head band +gr t +bur nett +w and +!!!! !!!!! +gh e +du x +hu d +war ner +ðŁĩ ¦ +ex ile +rescu e +rat a +d han +duc ati +dro wn +bl ends +spi e +alli gator +simul taneously +broo ke +u ke +k har +comm union +ri ka +ford fc +chin atown +you rown +me y +can al +syste matic +de pri +ox ford +an il +w ut +equ ation +be z +fle ur +the good +lang ley +ad ity +ed ith +al fie +о ÑĤ +en cry +br ill +ex emp +ce sar +mb ling +ab ri +sc icom +j ing +school ing +mi ka +mechan isms +impromp tu +rhe a +moo re +crime a +be sto +wri ght +el ders +ro ds +kam al +folkl ore +be et +mini on +reli eve +thr o +team usa +pas cal +made with +boli via +itt i +free bies +desi red +best selling +l iness +la den +ke ane +mi sts +hipp ie +atta chment +@ / +se w +flan agan +âĿĹ ï¸ı +supre mac +stl cards +si as +q u +rh ys +ste ep +val leys +v w +pav ing +disp at +al ison +por te +id u +new sc +soc ket +mo s +co star +re vo +prote ins +stanley cup +m cal +ear ring +se cs +mc lean +cap ric +nick elo +ad en +v c +shou se +adap tive +maxi mize +entertain er +pro se +gri ffi +six teen +lam ar +mi rage +saudi arabia +awe ather +ru st +in filtr +fashion week +ðŁĺĬðŁĺĬ ðŁĺĬ +selec tive +bubb le +a den +fen nel +deci sive +m ta +mock ing +mb les +st amp +mu le +bernar do +gr in +po tt +j ingle +vet tel +colom bian +cam o +motivation monday +ba han +p ly +dh ary +k ami +x men +sleep er +gar a +my sti +confi dential +conflic ts +p neu +ce s +insur tech +clean se +me rely +va is +tu x +the great +shar on +ma j +hol a +eco systems +aj ay +aa j +hu sh +har mon +backto school +wiki leaks +reflec ted +ðŁĺ ĵ +commemor ating +ac et +buck ingham +messi ah +tu ous +hor net +to be +d q +he ine +mi g +pl ate +nichol son +sp ie +cumber land +nor mal +pho bia +happy halloween +city fc +mc el +gilli an +ke to +lu de +de mise +su ga +str ate +mcgr ath +visit scotland +foo led +cb r +gc se +col ori +po td +missuni verse +fin ances +ma poli +for ks +Ø ´ +cann on +medic inal +ðŁĹ ĵ +kh o +wre ck +pan to +bag el +gu ll +syndic ate +ic y +pr c +ki en +zi ka +ti sh +pe ta +c co +li za +ch ut +ex traction +el g +gl i +fu eled +pos it +respec tively +leice ster +br ink +vulner ability +im ported +e sha +ðŁ¦ ħ +r ural +re ll +gam ing +atlan tic +aband on +no ah +re solved +pro state +aller gic +ps d +âĺ ¹ +dun geon +fang irl +illumin ated +m hs +white sox +d ently +ck o +endor se +over ly +dazz ling +prior iti +night life +ut il +be have +flam en +east bound +ðŁĴ Ł +ilove you +gov uk +mozam bique +alle gi +dr i +testim onial +ath s +ì§ Ģ +mm y +shab by +pro secco +friend ships +cal am +dam ages +off set +jura ssic +jun o +arre ll +ðŁĴ © +interven tions +dare devil +car ver +run away +ran e +truste es +ha ute +dep ths +ðŁİ Ń +me in +sacrific es +con cier +ne sting +i zzy +me tam +ilove my +ur ine +du lu +mal hotra +ve ins +night ly +co at +an di +he witt +lon el +ci ble +wr ite +jen nie +sant ac +ĸ ï¸ı +str ato +singapo re +sop rano +kri sten +cheer ful +flee twood +fa iri +m eli +wa st +tur nt +sfor sale +sc rolling +angel ina +ren dition +jeric ho +nick y +or b +fla vo +patri ot +ash eville +sick ness +re fund +aggre ssion +b pl +ãĥ ĥ +elu sive +thi story +hang er +bu ffs +vil las +at kinson +sp h +ja it +decl ined +wo k +supre macy +oo tball +ey ang +ðŁİ ĵ +s ford +ath i +consu me +road ster +e so +u pro +reci pe +au f +uc i +ar on +oo oh +cs go +re ich +mc d +min ute +ladi es +pun k +rut gers +mee k +ariz on +ta j +land lord +de gra +autu mn +lyn x +us f +b hi +fairy tale +dongha e +bet sy +explo ded +chen nai +op a +pro tag +br ant +ðŁĵ °: +g f +pal li +ðŁı¼ âĢįâĻĢï¸ı +su t +ill ini +colum nist +shir tless +de centr +sear ched +ec or +bu ggy +s ack +ðŁĺĤ ðŁĺŃ +de t +ther i +or naments +bring back +to v +quarter finals +ic he +con stra +gi er +buchan an +vi x +kay aking +mu stread +swal low +mel b +sc af +op al +may oral +har at +ðŁ¦ ĭ +schedu les +id f +ha gue +ro z +a ah +d mc +du plic +ca che +orph an +frac ture +rec on +ch av +bun nies +al ain +mustaf a +ðŁİ Ļ +vac ations +dynam ite +tex ted +broad caster +ðŁĴ £ +ste amed +rock er +di etary +luxury travel +inaugur ated +sa wards +vaugh n +lincoln shire +click ed +kra ja +f anc +remo ves +layo ffs +mc far +bre eds +win nie +jon ghyun +incen tive +vari ations +pat ton +atur day +persist ent +pr un +pi ers +dal es +æ ĸ +breast feeding +r ance +ta wa +Ĥ âĸ +mur doch +cap tive +thi stle +nic a +commod ity +cou ldnt +board walk +graci ous +practiti oners +n gc +scru m +ner o +camoufla ge +col on +he i +phys icist +saturday morning +ten er +si won +colum ns +bru ne +y vr +ba ir +reti res +hal am +cab er +shaz am +min u +cas cade +milk shake +gri d +d ren +vin cent +so dium +plat ter +cheer leader +chen ko +y ak +elimin ated +ty po +y man +re think +âĿ Ĺ +ts ville +bernardo kath +ex tr +ðŁĺģ ðŁĺģðŁĺģ +ta o +re per +mo ths +em powered +c iting +transpor ted +mon ks +san at +cle ars +bachelore tte +camp bell +racha el +har le +hand ler +climb s +inter ference +rele ase +sh and +r bs +hr h +ãģ ª +val le +r é +sli me +w akes +chu bby +slo an +el ves +ath en +attor neys +micro scope +ston er +sc aling +o be +c out +se man +mid week +bal sam +ðŁĺį âĿ¤ +ti ful +v ish +lo tta +ri pping +re mn +ti re +le ap +ha vent +la by +hi mach +whisp ers +we in +ðŁİ ¸ +wild flowers +se le +u cc +li ability +az ine +sw ings +k ya +ta ir +re main +e do +flo ps +poc ket +grand ad +exam iner +gr is +ffe ct +ðŁijĬ ðŁı» +stud ded +heart beat +de acon +firm ly +infec tious +ste f +out lines +le asing +cla ws +sen se +tab s +hoo t +mo sul +spa wn +co a +hog warts +ve in +alban ia +manu el +b ino +vaux hall +scot land +go bucks +mat ty +phy sio +tor ino +const able +investig ated +s lower +mistak en +bay er +wild fires +vo ic +x on +time to +chas sis +bar ric +pi on +bald head +woo k +regi str +dra fts +b hs +li gue +l ick +staf fordshire +baf ta +dar ry +je anne +ven ding +cor p +⼠³ï¸ı +kid dos +fen way +ca o +west bound +ðŁĺ Ļ +dv r +quick er +bla h +goo die +ðŁĴĭ ðŁĴĭ +vo x +esp er +fac ade +cor relation +red bull +rou p +decl ining +chi ve +mc gee +tur o +in der +f eller +fu g +il ysm +mar di +peshaw ar +ki eran +ine ma +meat balls +pe ck +depre ssing +sen sing +gi z +dd ington +spring watch +ro aming +yellow stone +horse shoe +am man +week day +ol or +ðŁ¥ ° +boo sts +spr int +scar ves +je e +bee tro +cl an +all the +ìĦ ¸ë +enlighten ment +ado be +re generation +? @ +cont ag +yach ts +to u +mor a +en voy +r ani +go li +dhanush kraja +wood working +streng ths +se di +disc s +ar ina +sc on +lit e +ano ther +ðŁ¥ Ĭ +ye men +gu ern +sav vy +lo yed +biom ed +heart break +comra des +milli e +pat ch +un f +jar vis +bl aming +commemor ation +ge y +å ¥ +cardio vascular +alig ned +docu ment +. ? +aesthe tics +em u +the irs +le h +ps ic +si f +pl ateau +ex pend +domin ating +rob es +mauriti us +excep tionally +hom er +discover ies +bra un +ten nant +insul in +ðŁİ ® +car bs +te as +? !" +zi e +franco is +brow sing +th ol +cla rence +hel per +ob tained +cas sie +le es +! , +pome gran +hu bs +presti ge +] [ +mach er +bott led +pun ch +pi pe +o ch +gall ons +deliver ies +u ra +un day +mon de +depic ts +re gency +outra geous +khal ed +car o +he arti +za g +develop mental +over coming +stati stical +flavo red +for ds +cre atives +lau rence +di as +sun screen +in ked +pre acher +n ul +impac ting +auti stic +âļ Ķï¸ı +o ss +pel icans +cele ste +v b +ru mp +mc gra +fair fax +hu mor +bbc news +row ling +cal der +seam less +ag ne +p ti +mix ed +t shirts +mer ci +b tob +women instem +genealo gy +pre ven +l our +cra dle +gi use +Ð ¾ +chron o +fair ness +chocol ate +tor y +as da +pre scott +stret ched +al man +u il +re charge +in tre +ob st +hosp ital +hay ward +teneri fe +fried man +vap ing +confe ssions +ye ah +bal li +luck now +cor pse +sculp tor +amp ton +t pp +indic ates +sur plus +tru man +ðĿ Ļ +sin ha +in vo +sovere ign +ke v +establi shing +engra ved +assu ming +ðŁı ģ +sou za +fab i +ton ed +oun ge +del oit +dow ney +no ble +om or +car tridge +ðŁı IJ +u hur +hol loway +succe sses +r sa +âĦ ¢ +ma zz +tw d +disc ourse +. < +y at +satis fy +com pri +ठ¹ +graph ite +disser tation +ar ter +í Ķ +b ally +zom bi +ly ons +a ic +u bc +pra da +e il +da x +cla i +grand daughter +extravag anza +chall enge +ðŁ¤ ŀ +po ver +primar ily +dad dy +man a +bi kers +inqui ries +da un +fel ine +gener ative +he f +benef iting +lind sey +pol ka +demonstr ated +al le +rand y +o su +low key +weir dest +red bull +our y +n ous +wood stock +cre denti +nic er +g ado +aly ss +ap h +prepa redness +station ary +incorpor ated +dy er +sarato ga +cele sti +: " +antibio tics +or gs +inde fin +ap ron +и Ð +fif teen +no f +ðŁĶ Ŀ +ph x +te ga +m z +organiz ational +on air +band ung +pleas ures +mor i +secre tari +rac coon +ca shi +pil ates +k on +geof frey +la o +kam p +depart ments +back packing +an am +à « +crack down +aun ty +on do +li zzie +ph ers +cu n +ðŁĩ ± +k pop +pu t +inten tional +connol ly +bar clays +hs fb +swin don +u ku +s ally +a int +âľ ħ +pen ang +up lifting +epile psy +inter ro +bun gal +go ku +blue berries +ठ¦ +u ssia +sil ky +mou red +i stic +bri efs +me ats +go b +ch aser +state wide +pra sad +gl itch +ar in +ban ff +memb er +ðŁĺŃ âĿ¤ï¸ı +lo ving +hall a +ภ¡ +smo kers +yak u +scicom m +physi o +sw ol +lem ons +gel ato +ch ool +capit als +ki stan +ti ghts +spi kes +trav ellers +ik lan +commissi oning +ar ine +emabiggest fans +empha sis +front line +pad dock +destruc tive +ba ha +l inger +je wish +shet land +mc gin +mon key +ko z +s one +raj ini +te h +y en +c vs +masqu er +gir ly +we sle +was nt +bro dy +termin ator +gil le +mag gi +bir die +jeopar dy +cu bic +vm ware +intric ate +an up +to pia +east on +sab res +investig ates +bu sting +bil ingual +valent ino +in format +fer re +advent ur +hydr ate +for sy +az iz +san to +e de +whist ler +continu ously +d ham +un used +ji had +addic tive +vi dy +do b +i do +fi ed +ni versary +n one +fu er +ðŁĺį ðŁĺĺ +coven ant +prin table +immac ulate +o em +cl t +serv ants +consu med +un released +sc um +pack aged +me re +ìĦ¸ë ¸ +to by +ta f +spo ons +me al +f ball +fair field +jan et +silver stone +dart mouth +follow me +voy ager +kom bat +anni ver +ene w +mag dal +ho ve +sa th +grizz ly +car di +gart ner +sand y +kan ye +post ure +po ign +im pulse +radio logy +horiz ons +si am +aish war += => +no che +tr is +el yn +com me +du i +ce c +councill ors +cudd ling +creep ing +loc ke +manag es +trans ferred +ne cks +di er +dan o +v ick +lun ches +d he +en sures +cri ss +ul ster +bann on +cont enders +sp am +sweet ness +med al +hon duras +arc tic +ultra sound +in fr +disco vers +ei ffel +ca sters +ru ben +du st +awe ed +atri um +lest we +se ared +ðŁĵº : +ty ne +ex changes +little mix +l le +astron auts +hersh ey +work day +kno b +so v +re signs +today show +der man +an th +af c +ta ster +sw oo +sa eed +per ing +narrow ly +rn li +best buy +panas onic +obst acle +farmer s +ðŁİ Ļ +pa wan +ki est +ang ers +absur d +oh my +sin o +pist achi +sp ice +giu li +prime time +ko w +k ens +ex agger +! ?! +u ba +midd les +ju dd +e jec +slam med +pen sions +of a +re create +b hp +xx l +liver pool +thre sh +pur ity +ni eu +hol ics +wr ath +ra do +gli o +am ma +dile mma +cr u +lets go +.... @ +âĿ ĵ +sugge sting +tru mps +hor us +f v +ic om +refer ring +predic tive +tar ts +ge tte +so ck +glo ssy +pin ky +al ec +thy me +ou ra +thero ad +pe tr +cr am +p fi +dv n +me ier +incen tives +tun nels +mobi l +rec ap +extra s +upri ght +rev amp +per severance +, - +ot p +mir ror +ar wx +ger ry +ma her +g or +hom epage +am is +ag ra +made le +best friend +sirius xm +bun dles +admir ing +t dsb +ðŁį ģ +ch as +slow ing +ro h +wall papers +âĢ¦ / +tek ken +gang s +tal a +lind say +shou l +line backer +tool kit +ur anium +caly p +ab rams +mat thi +ðŁı ¿ +hon ourable +da yo +ver sail +tan k +st c +fr itz +spl end +pat ag +anno yed +on day +devast ated +chattanoo ga +national ism +mas sey +jen n +tail or +dev gn +org ans +zu cchini +on fox +sat ire +wex ford +dis grace +no to +vol ta +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ı +à ¶ +home owners +poin ter +m cr +au sten +day sto +mo ons +pal ma +gra zing +e so +influen cers +shahid kapoor +compli ant +measure ments +develop s +y d +par l +p vt +rand olph +tor tured +ger ald +eli as +deepi kap +war mup +hick ory +g ap +co ffin +am our +re neg +moun ting +seven s +ig le +hi er +dec ad +tri ght +esc apes +wer ner +t fl +ful filled +ni ger +sour dough +re aper +choo ses +spin ner +week nd +fil tered +sh uk +kat i +old ham +open source +kh anna +at elier +conne c +opho bic +gla s +complic ations +ar son +counc ils +sm ol +as sy +lur king +ling ui +han ks +e in +Ù ħ +ru gs +n guyen +nou veau +men ace +le v +alad din +ru ining +round about +k m +con or +shoo ps +may day +traum atic +prab has +ka iser +k ita +rou ter +pe dro +re tar +stun ner +spani sh +distur bed +acade my +e learning +wit ty +sen g +fer al +av y +sta b +ke aton +ur du +ko to +hu i +coo ke +ari an +the personal +u ma +se ap +a sting +rhetor ic +hand writing +munici pality +consor tium +ðŁIJ Ł +glasgo w +ra ya +eli za +polym er +bro th +prac ti +correspon dent +addic ts +gay le +ail ing +o fe +p li +hear tw +st itch +sight ings +prie sts +sam o +slo th +good wood +roc co +sab c +summ it +l ace +pres ley +itt en +cin cy +thepersonal network +s week +pe gas +af con +regi stry +ci m +le th +dic ap +cand ice +flu ent +sm ack +pede stri +al oud +car ac +priyan kach +p gh +ir ons +dol ce +lat via +dece ased +thero ck +cla p +cen e +fo am +morris sey +gre t +essenti ally +com cast +be agle +argu es +ing ed +- âĢ¦ +sa g +ha san +ðŁĻ Ĩ +ðŁį ° +nh ra +kann ada +indic ators +on er +bri xton +at as +screen play +sor ority +sha heed +he em +class mates +tain ment +es i +breast cancer +zucker berg +aur or +en cia +ref ers +kae per +vor tex +com part +lym ph +photograph ing +ste ff +rest ling +par sley +mom ento +th man +lac king +du tt +ocu lus +fin o +fren zy +ra sc +der n +dis missed +noo k +met gala +sh ill +rapha el +maver icks +exhib its +eag erly +c pa +amen ities +. âłĢ +exo dus +ern st +lit a +deal t +womens march +i ain +score board +campe ones +c en +ti ki +garri son +fidel ity +bra g +road map +psy chop +lo e +ble u +ðŁijĬ ðŁı¼ +sau vi +spr inger +temp tation +ru dolph +ac ura +wic z +parach ute +stro l +len ny +zi k +dom s +nb af +al pac +vivi an +ro ve +pre et +perpe tu +sna ke +air soft +infl atable +prin ces +ati e +ffe y +pati ent +m ire +chel le +sl ack +groo vy +# : +up loading +!!!!!!!! !!!!!!!! +siem ens +provi sion +v fx +need y +f ats +to poli +bhu tto +sa thletics +alu ms +t winning +south western +adop ting +last night +man ne +la ga +tw ell +ac ia +-- -- +eye wear +hur ley +fle e +sa ch +pe cker +cost ly +is k +cr ates +polic y +ero sion +in go +wer k +ðŁIJ į +torto ise +therap ies +inter net +chihuahu a +ri ps +fre i +ed or +tai ji +t fc +do d +demp sey +christ in +chen g +hi ps +gra eme +com passionate +cavali ers +histor ic +soul ful +crimin al +ja c +vin ci +expi red +sur at +turi smo +k ona +se aweed +ber ts +le ica +expre ssing +a al +wor t +break fast +her ring +am used +rhu barb +mar tian +cospla yer +y ash +stri al +ra ul +refer ral +dw ts +j w +ad ler +cur tains +gu r +val ence +tyr one +sw fc +coach ed +re born +diabe tic +cho ke +nor folk +investig ative +ðŁĴ¯ ðŁĴ¯ +z id +v mas +phi e +objec tives +âľ ĭ +over due +di vers +mat su +ðŁİŁ ï¸ı +casu alties +ภ§ +al k +stand ardi +re alist +arti facts +pand or +ke x +in vin +( !) +ine y +par aly +mr t +fay e +the voice +on ga +de ed +skin ner +az wx +speci men +priyankach opra +nu evo +bar kley +toulou se +resu mes +football ers +cit i +fe tch +è re +lestwe forget +ðŁĻ ĭ +ch unk +dri fting +manipul ation +equ als +pu tt +ky ungsoo +âĿ¤ï¸ı # +ela stic +par ano +fo y +do ping +cin cy +ss ler +interrup ted +al ay +ado res +ame thy +con voy +ãĢ ı +Ĭ ãģ +black list +gener als +sa chin +bru shed +oun ces +non stop +illi ams +bt sarmy +u av +ru ff +bur ma +bi k +defen ce +schul tz +bo asts +lonel iness +go re +trans forms +alum na +@ @ +ra ppers +ne hru +car o +himalay an +wearab les +ge h +pepper mint +re development +flam ingo +cos by +big baldhead +ag ri +bare foot +sco pes +re gram +gh ana +ðŁİ « +i heart +sa die +carri e +microbi al +ku ala +sk ater +quer que +âĻ © +gen res +reas oning +ch ased +as o +sli pped +en can +vam os +ker s +ad verse +mo il +commod ities +with you +sil ent +hy pe +an de +am ination +whi spe +lit z +âļ½ï¸ı âļ½ï¸ı +ri ff +pp y +lam bs +gan esh +ab sent +regu lator +marse ille +en roll +par cel +wa p +by rd +ðŁĩ Ń +tu ber +country music +par l +contro llers +responsi bilities +we y +ch ate +montene gro +chic o +mil an +l ms +tra inees +appropri ately +un certain +popp ies +ed sheeran +nutr itious +gar o +deut sch +awe some +ãĥ ¼ +comfor tably +land marks +et i +re usable +daniel le +ro sal +co les +just ic +c cs +f anny +ni m +mc u +clin ch +at ene +mer ge +im db +ang lo +uc cino +pan ini +an not +bur berry +feat ure +predic ting +fashioni sta +s ask +imag inary +mm o +south sudan +spe ar +hu bble +jo inthe +coyo tes +sli go +ko dak +sit com +polaro id +roo ted +corru p +ðŁĻĮ ðŁĻĮ +bris ban +at z +ah l +re my +tal ent +aval on +ra da +pau line +locom otive +go ons +ne mo +maser ati +ic u +stu tt +histor ically +sm b +pres by +avo id +so oners +rhine stone +w ad +ri sing +tro t +mo des +reg ent +optimi ze +re ece +sm u +ver ti +newyork city +cor tez +ra c +in case +sin c +fiel ding +e tta +tiff any +al monds +sad dle +k rat +mat ter +g low +star ving +gl o +cra ppy +sl ur +st d +monit ors +recei pt +maymay entrata +mc il +un is +rain bows +cal dwell +pacqui ao +j op +a fe +hoo k +es sen +wiz ard +medi an +fla ws +com s +âĿ Ħ +ing h +ha ynes +anton io +tem plates +ou ter +na w +cardi gan +bel grade +ðŁĴ ī +hom o +a ise +ro pes +no ve +what you +tri gge +concep tion +ad ukone +na di +fri ars +sw er +adju sted +hot line +san ity +kau r +down loading +c gi +ten or +eth nic +app alach +ภ¸ +pa g +gol ds +on set +investig ator +car tel +peace fully +jarre tt +cat alan +poli o +n um +fru stration +dhar ma +my life +âľĮ ðŁı» +aber deen +mu sa +bin der +spark ly +fle eing +instin ct +co ping +domin ance +ill ers +er a +u conn +lo oms +living ston +gal i +he s +c ma +bel a +se ley +mon k +la ch +mar x + ´ +m erica +woman in +es sex +ra ina +jim i +nep tune +z ack +chine se +mart ins +chand elier +her n +with us +ear l +asph alt +modu les +st p +ul la +psychi atric +mile age +captiv ating +si der +men to +mor t +tran ce +tal bot +ab by +ì ĥ +âľĮ ðŁı¼ +j ak +daw n +turn up +scre wed +fe ds +blue print +ðŁĴĸ ðŁĴĸ +har sh +er os +insom nia +ban kers +ta emin +mis conduct +hu mber +gi di +edu ardo +con a +musc ular +consu ming +ra sh +don nie +di pped +col lie +samu el +melt down +ðŁĺįðŁĺį ðŁĺį +me z +exam ining +schwar tz +pri stine +ðŁIJ Ŀ +ve it +ful filling +an esthe +gue sses +dra ft +som me +soli d +pati onal +ho ped +evolu tionary +all er +enter tained +sli ps +lud wig +conclu des +sen sible +bon net +cra ze +tra s +haz ards +const antine +ed ics +star trek +to c +occu pational +in cheon +deepikap adukone +pizz as +new comer +de part +oppre ssion +ebon y +foss ils +tro jan +el en +ste aks +k hou +positi oning +ug by +red cross +ak h +dol ce +us mnt +pp en +dil ig +ma vs +call er +cost ello +⼠Ħ +dy n +thing s +rhin os +a xi +sar kar +con vocation +att ers +ss ss +fun gus +eu gen +russ o +squ at +w sb +eli on +william sburg +s off +defici ency +be arer +o kin +key stone +t wain +cal ming +break able +wa res +horser acing +com bs +bun ting +u it +t land +ðŁĴĻðŁĴĻ ðŁĴĻ +ga stron +sab ot +ick ers +commissi oners +sen ate +ii ot +ath ena +nit rogen +an tony +ero tic +di alo +mis sou +hypo cr +âľ Ī +kaeper nick +can v +d roo +clevel and +o sh +mon sta +stefan o +^ ) +sh ul +po ison +ha e +commerci als +ma ul +nit ro +co worker +alo e +vap or +t ents +russi an +qu id +question able +mid get +po ker +girl friends +sin the +erit rea +ten ure +depos its +buc keyes +spot ter +theod ore +trin ity +joaqu in +u cci +follow the +caf c +mp a +ðŁIJ » +plo tting +dom ino +ta ek +sion ally +dicap rio +pa p +car mel +ig er +bt cc +beth le +www bigbaldhead +foo die +bagh dad +mason ry +off ended +à · +ภģ +sc ro +vers es +ori ent +ar ches +pi yu +know your +gre e +ta kers +gu ard +dish on +bucket list +bha fc +war dly +ðŁİīðŁİ Ĭ +leigh ton +pe w +stra y +assaul ted +in hal +ly fe +amar keting +l x +kat z +ubun tu +me o +carto onist +turno ver +mi z +dis like +mul len +mo f +bl and +hi des +emer ges +chori zo +truste e +ma hog +lan sing +paralym pic +fa int +fa una +ch al +sn ar +cat h +bent on +cast illo +sli ppery +apric ot +oec d +bar o +l z +he ming +clow ns +co workers +peru vian +commu ters +y ell +ðŁļ ´ +under ing +v j +tt p +fli pk +w ana +soc ent +Ĥâĸ Ĥâĸ +ठĤ +oo sa +jag ger +di sm +e less +d ham +cali f +a official +ec lip +harro gate +gra pp +com rade +n tr +concentr ate +thi ghs +bit coin +bel arus +ë ĵ +end uring +now watching +industri al +pi p +ar on +ar at + ® +whit by +oooo ooo +sa ree +tic als +mis leading +yo on +year s +sle igh +roman ian +sciss ors +vam pires +ac up +ab ba +th weeksary +cent ri +fl ye +u o +c bi +bu ena +sin d +mar ino +bur r +re building +ठ² +anniver saire +ac ca +ðŁĴĢ ðŁĴĢ +gett ing +tu lips +wolf pack +âľį ï¸ı +more than +ta kin +ðŁ¤ĺ ðŁı» +u be +mon ic +dou bts +mo wer +co balt +don ne +specul ation +argu ably +kak u +htt ps +prosecu tion +din ah +stam atic +disclo sed +bever ly +fl wx +cra bs +extraordin aire +war mest +imper i +o logists +trac es +par c +lake side +am r +ter i +hour ly +domin ation +ar row +shrews bury +ance stry +wr angler +trigge red +pen sac +roo ster +survi ves +a on +bo ko +val or +love is +la g +pe y +fo cal +out laws +bl anc +artic ho +wit s +marsh all +die go +support small +u ca +sa h +je et +syn ago +gover ning +ðŁĴ ¬ +sal ads +cre ate +miri am +cen sored +ami de +no u +z eta +allegi ance +* ) +bl m +ric an +pa stors +oly mpus +blo c +whir l +star ry +pr one +y k +p ne +congratul ating +be v +so ber +love island +sa ir +an ing +tutor ials +q e +lun d +in ist +cle ver +taxpay er +ali z +wren ch +dd ling +cap ri +h pa +ðŁı» âĢįâĻĤï¸ı +na j +o j +futuri stic +jelly fish +ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ +cel ery +plan k +fil a +ne me +un healthy +lec tions +ðŁ§ ¡ +rit chie +n ws +mi kha +wonder woman +âĢ İ +hip stamatic +ka g +ðŁĴľðŁĴľ ðŁĴľ +poul try +mo w +wor ds +lo ff +ðŁ¤£ ðŁ¤£ +relat able +re mixes +keny atta +ke m +re signed +fo d +stra igh +j lo +hu tch +box ers +colle en +mag s +instruc tional +ko l +attrac ts +pra g +account ant +go ggles +br u +th ole +mar row +leu ke +oc to +pon ds +bubb ly +he ist +ìĹ ij +im p +a har +ha unt +hall mark +psy ch +kkkk kkkk +col umb +jump suit +cost co +si delines +ag gies +over turned +ni b +key chain +fu k +f af +mi am +assist ants +cy cled +ri der +dam mit +red wings +mag es +kin s +ì Ĥ +ho d +son t +carol ine +" ' +cu le +bra id +fel ony +ar ities +ruther ford +depic tion +isab elle +ro ach +k day +fifth harmony +em y +li gam +bari sta +albu querque +gro ss +ðŁį º +oo ks +ðŁij ¼ +dun can +try in +jag s +g ould +li tho +âģ £ +а Ð +sam my +tun g +cas ser +apo lo +aaaa a +man g +as ics +sh en +p ye +tur bul +ss p +saint sfc +on lin +n anny +he ster +do z +ภĶ +th read +ren ts +kh and +ðŁĴª ðŁı½ +un conditional +rob son +car re +ph on +sacrific ed + £ +auto s +par ker +oc a +log in +kee gan +hard cover +dough nuts +ðŁĮ İ +spit fire +refresh ments +saskat oon +commod ore +j f +rub ber +halam adrid +child care +stra da +io m +ri k +dak ar +ther mom +cro pped +gar u +ali k +ven i +i ft +si ka +ritu als +z ul +e ch + © +su dan +l land +i me +do cker +ì ¤ +fe ared +fa o +wal ter +no g +mutu als +l h +ali gn +mon ia +concep tart +ðŁĻı ðŁı¼ +sco e +compet ence +sw ine +ly me +laun ch +green er +abstract art +inqu is +gran ada +ga elic +flu ff +d backs +grave yard +ba be +acade mic +adventur ous +joh ann +~ ! +bi bi +| # +pl ings +gett y +as b +âĿ¤ï¸ı @ +staf f +religi ons +bang or +world bookday +me gh +de vin +ash ore +meri dian +gi thub +qui z +all stars +be stest +ir resi +ack er +do te +war rington +pol ly +newor leans +cr ou +wi gs +che y +smithson ian +la sag +de tour +bor is +stra ps +mari ah +inten tionally +ko h +ðŁį ¸ +ssi an +mar issa +cor al +episcop al +casu alty +tom o +supply chain +sam p +on go +ro o +cavi ar +p fw +clau dio +buff alo +s ations +mat ty +snap back +l ds +al arms +mat te +âĺ Ķï¸ı +conditi oner +d ors +he x +fi zz +a stri +sus sex +secur ity +qa eda +all star +cocac ola +as one +cl icks +sc ans +mu te +he avier +ðŁİ § +âĺ ŀ +lv l +book boost +youtu be +fla shes +f jor +c su +explo de +do dge +cair n +gonz ales +th ill +pel le +hart ley +renew able +re tin +e stre +costar ica +shipy ard +nc fc +pri ya +a ghan +an ath +plu gin +co rey +re bound +or u +kat rin +hor mone +gi m +mahin dra +s sus +park land +har per +fanta stic +infer no +ep ilo +wrest ling +fe ct +c it +ac oun +to ssed +monu mental +char tered +bu st +pe tra +âĮ ļ +wildflower hour +sweat ers +* . +bl er +ate ch +go wan +demo graphic +bra l +suici de +renov ations +vu el +sin ister +ar mani +miso gy +ph arrell +nap s +un iting +crusad ers +cor gi +insu red +than i +no or +g q +d ada +bicy cles +snu ggle +sch an +ten berg +ss al +fe mme +bo il +½ ï¸ı +re ap +occur ring +hus sein +divi d +sto ke +sh alom +na ia +o lic +frustr ating +Ù ĩ +ig s +gro ver +scen arios +n ds +bru tality +med alli +bu on +sas s +skate boarding +ony x +lor ry +ny u +gau tam +mm ings +gu g +end i +lo thian +comm ando +chal k +ph ora +asse ssing +ti gh +crun chy +ad ay +is l +ci ara +pilgri ms +kam al +p to +brit anni +t ani +sm c +l ure +app store +ab y +golf ing +cl c +fa u +an as +shu tting +regul ated +carn age +scow boys +all enge +c ma +humbold t +rel le +ku mb +her i +refin ery +sound check +d wayne +bos nia +i sp +the alth +anni v +relev ance +my a +bag gage +dre ad +s bc +th ed +bu h +hi jab +lo id +ke w +c te +respec t +lovel ies +cu bes +celebr ate +dir t +sav ers +_ , +gar ment +pulit zer +mas jid +beat port +al arts +encry ption +s ner +ple ads +found ry +sym metry +ru mi +birth place +scallo ps +supp le +pivo tal +t ati +no de +so d +pro xim +tr ics +col dest +bren t +mand u +cla ir +e ach +and alu +hi ddleston +ðŁIJ º +mel ts +v ance +pin n +se ments +scre ened +sa chs +o bl +ic ha +âĺĺ ï¸ı +school ers +heal ed +lo gged +ðŁ¤ĺ ðŁı¼ +ic us +bore dom +b ish +b ffs +tal king +sure sh +hoo kem +de on +de fl +ei leen +ðŁį ķ +women intech +ri sotto +rang er +adverti se +ภģภ+tel ly +la go +dart moor +d ong +sk ates +lo go +un ner +mail box +ma sala +lo oooo +amethy st +che wing +c bb +australi ans +rc mp +game art +# ... +kor n +extre mism +fruit ful +anci ent +pu bg +pol ite +wh it +mur als +m gr +line man +dav ao +ste ms +ten nis +av age +tu pac +gigan tic +hs bc +auto biography +up the +ี à¹Ī +re gal +fig uring +ku l +mis sy +hoo p +gra s +for ums +back lash +abduc ted +p nw +min ic +bu tt +bott oms +at on +ven g +ðŁĮ ı +del aney +prab hu +fan club +over haul +health ye +sy no +aa f +ren amed +kim i +un cle +man city +se u +qu anti +este em +um in +en zo +mel vin +under go +j har +far ah +coast ers +humph rey +mh z +children s +^ . +d hi +disrup tive +integr ating +r nb +over sized +a ide +ne au +docu mentation +ðŁijĢ ðŁijĢ +pal o +hear th +ri yad +pun ctu +abc news +secu res +boy band +bir ch +ju co +tra ff +legislat ors +bay a +ãĤ ¯ +no ises +collec ts +s warm +k ner +bi shops +stur geon +snapp ing +mo l +fre aky +chair person +tro p +lyn ch +car cin +art sy +e sto +cha i +fl ur +inv ali +sau sages +im el +j or +fun fact +wit ter +puni shed +ac ons +h ya +re versi +em c +dif fu +z x +sp aw +cla d +d mit +hol land +fre sco +pay roll +ab undant +stu ffing +mor o +c ny +boy cott +wend y +ele ven +pro voc +pil ot +tr x +be ad +climate action +ri on +assi e +ì ĸ +o sm +islam ic +ho ar +good reads +al ici +afterno ons +spoke sman +jo lie +it as +masc ara +âĻ© âĻ« +pre vail +beetro ot +lu jah +k li +dod ger + » +ru le +l n +scre am +ho bart +col bert +r tc +er m +pat ro +quo ting +s live +que st +non fiction +semin ary +prosecu tors +ve st +express way +g ge +nau tical +et f +ðŁİīðŁİ Ĭ +dur ation +cha ired +the film +fab io +she h +can o +ðŁĴª ðŁı» +with draw +! :) +cor pus +phen om +yel p +la wn +ent om +snapp er +but te +pin ball +pro xy +libr e +alle vi +n ada +gabri el +fo wl +eure ka +daph ne +tu nes +pun ched +wh ore +jo g +ren tial +man ners +o pe +wh ufc +gu th +revol t +sne aker +philharmon ic +ho ste +sovereign ty +ðŁĻıðŁĻı ðŁĻı +fish ing +sci art +fe ta +i pp +dump ing +kel own +gir i +dig its +sal u +san jay +twee ters +sp as +col chester +sc ab +ma dd +๠Ħภ+Ä ĩ +ged don +march for +do p +maure en +un plugged +di do +fashion blogger +up a +mex ic +tar y +pol ye +jame son +v t +grin der +mad dy +consult ancy +¬ ë +leagueof legends +ac cents +um ni +jane iro +tu ss +h ens +ampli fier +to shi +pret tier +pre vents +new town +red wood +vant age +ball ard +ar tof +a she +a sion +lac ey +ap at +gro ve +ภĦ +rw and +real tors +tra itor +bed ding +ö r +zi on +fla shing +cam pan +boom er +secretari at +ab ol +liti gation +cont amination +se dly +shred ded +in for +do herty +bench mark +ro che +skate board +sho vel +i zz +to pper +o ster +laby rin +autu m +k ong +hum mus +vi z +tech news +kla us +am using +socialmedi amarketing +i des +cast ell +ste e +underestim ate +cal ab +pa ign +b illing +unanim ously +g mb +fly fishing +hath away +commerci al +colour ing +skul ls +pivo t +te p +tb c +motor way +x press +construc tive +pu k +under lying +kir sten +mani ac +cha o +se ma +chiff on +ðŁijĮ ðŁı» +ver ona +kom o +stan doff +wi ped +c ated +bla ir +wor kin +m sc +bethle hem +swi pe +unexpe c +pe es +pe tri +orig ami +ðŁij ħ +mex ico +flav or +ru dd +cannab is +mar u +ri ddle +wor shi +sil on +sch at +ap se +tang er +bi ous +e er +questi oned +o zar +dan k +angle sey +char an +bak u +compe ten +re pri +bat ter +sa xon +cal ves +leng ths +$ $$ +âŀ ¡ï¸ı +immer sion +ga unt +car ry +cy to +b anda +shu tt +experi ence +el gin +mous se +ta z +ê µ +in correct +en z +b ham +mor on +so ver +ar un +ti pped +la ble +de arly +bau tista +í Ļ +mor tal +woo p +dt la +sho cks +dav os +ðŁĵ Ŀ +swim wear +her man +ðŁijĩ ðŁijĩ +z ir +neglec ted +grac ed +campu ses +av s +ar ora +swach hb +live pd +ac cra +enqui ries +shoo ters +kur t +vancou ver +brad ley +gar da +g ü +ol la +attrac ting +up ton +ne win +lu mia +furn ace +ev ers +e on +sw a +roo kies +a oc +v ss +bris ket +tor ch +yo da +heart land +tac o +ph ony +food bank +ab bey +bab ylon +u y +gre ate +expre sses +d andy +sc apes +survi vor +ron d +e ci +ha vin +ab el +chil dish +tor que +wav y +ur self +kanye west +year of +ale stine +o brien +al fon +sk ag +kore an +anchor age +val eri +de w +ðŁİ ¨ +land slide +car ole +christ en +go phers +af i +priyan ka +q q +power of +it te +pc so +tw ol +pr y +intellec tu +guer rero +pi les +wish list +w ren +time table +ë ı +prodi gy +gibb ons +. / +ne ur +anz ac +mur ray +vie st +pla ster +la ir +art gallery +inter continental +g br +bell ator +nam joon +mam mals +am el +y aw +saras ota +cam ar +bud ding +sum mari +aco sta +la sh +ey ou +post graduate +instruc tors +ti g +const ant +were wolf +ic os +cla s +glen n +bud ge +ðŁĻ Ĥ +er ta +sta ins +persecu tion +cumb ri +o ch +syner gy +hu ang +scand in +mid terms +comment ator +regar ded +perpe tual +bo iling +al p +lan ge +sch le +fac eli +twee ta +ri dden +ok toberfest +charlotte sville +ik lan +jo u +ch atham +b sc +ðŁį ¦ +stra uss +mel low +xx xx +happy hour +re actor +ww er +distr action +at orial +ðŁĴª ðŁı¼ +twin peaks +fay ette +a or +ko k +bro om +sy fy +ou se +am ag +Ø · +ubis oft +lu lu +hall mark +stu art +it ya +si deline +venge ance +re lu +sex ism +boun cing +un ites +gu stav +te ssa +stu mp +pro clamation +ima x +divid end +col by +ðŁį İ +play wright +un safe +co smo +ðŁĩ²ðŁĩ ½ +cup board +constitu ents +ang lia +ram page +ðŁĺįðŁĺį ðŁĺįðŁĺįðŁĺį +than ked +take aways +shro ff +de bat +kh ur +conduc ts +format s +à © +port age +graph ers +u ten +pre m +mo ines +condem ns +s ous +l ps +f cs +deal ership +leuke mia +bure au +ski d +guardi ola +ca ster +thir d +avoi ded +en cyclo +c sr +vi xx +analy zing +she ar +dulu th +shap iro +chan ting +stre sses +as be +mil itia +ãĥ ª +col lin +arsen e +sure sh +teach ings +yi xing +sh ill +nu des +sv u +clear water +war ped +pro life +artist son +it u +versail les +galax y +ax el +spring st +cal a +hu hu +sc u +commit ments +exe ter +poign ant +mo tion +conserv atory +row dy +rec alled +mu sk +emb elli +so the +âĺ Ģ +sto pper +sch ild +to pe +el mo +zi el +j om +barn sley +snow den +on tour +jour ney +hills borough +par ole +w ts +mo ving +ag ility +tiv o +ff ers +kindle unlimited +g wen +ann an +ah mad +tex tured +hepat itis +dra m +insi ders +tis sues +ãĥ Ħ +fc barcelona +cr atic +na acp +pe can +f gm +custom ize +concer t +g sm +pe g +p one +justin trudeau +super cars +happy holidays +bu lar +ado x +lap tops +digital health +destin ation +gradu ally +áĥ ¦ +popp y +ss l +inhi bit +star light +of fro +glo omy +x per +hal der +im plants +le to +hass el +a as +un told +en ci +liber ia +or an +con tests +il ah +sma g +sc out +mari anne +cr yo +schedu ling +lo s +kan e +stutt gart +ne se +law rence +da in +pho tom +car ou +ภ£ +g wy +national dogday +roa sting +band camp +kentu cky +stret ches +ke rel +ca she +ãĤ ¸ +sta x +tran si +dog gie +at ric +hal le +ci vic +brow ning +lein ster +cat day +high land +joy ous +in cumb +or lando +ro mo +col ton +del ta +car ab +ro tc +aster oid +goose bumps +mo logy +yo ko +an ds +tomor rows +red carpet +sm p +ca sio +ðŁ¤£ðŁ¤£ ðŁ¤£ +se au +rejec tion +rot ating +bi partisan +th un +mat i +bon i +ol l +ener gye +do it +l j +mother hood +lou ise +neck laces +el ite +ni x +l cs +en v +gl u +le sh +cran k +su sie +m clau +so tu +crow ley +rat ri +use d +bre ton +alfre do +ye o +travel pics +ti pp +elli son +sax ophone +me red +heu ghan +ta ine +f es +vi ro +suppo sedly +i as +dige stive +y le +li zzy +wildlife photography +bri anna +west field +ra ined +am her +ðŁĺĦ ðŁĺĦ +distribu te +bott om +pre serving +oil and +craf ty +de scen +col ling +shakespeare sunday +r wc +ang led +ci an +t ations +mon tage +me yers +france sca +ðŁĮ · +wi ggins +san ford +volunte er +car ra +bar k +vari ed +pl in +am u +kap il +rock ers +qu ind +br ane +in mate +ent al +impro vis +michi gan +re tweeting +progre ssing +mercedes benz +smo ker +physi ology +dor ado +watt pad +h wa +sr bachchan +w ga +vol atility +hi re +ac ap +wn ba +hein z +stit ches +kidnapp ing +bur ys +lim b +f itters +thumb nail +ton e +mir and +desi rable +ad dison +tar an +tamil nadu +spec tator +soci ology +amit shah +remo tely +âĻ ¦ +ham id +r ds +g lee +smooth ly +sch ro +er c +lali ga +he als +us f +ni shi +d hu +un il +h le +tro mb +bhu tan +pilip inas +se ung +whit man +te y +min ce +snow boarding +re au +k ker +av o +zach ary +ran veer +ti k +gover n +qu al +beck y +anthropo logy +att en +grocer ies +de bit +war p +sil icon +hawa ii +ðŁĴ ħ +pomegran ate +pe er +orang es +people schoice +end ure +ðŁĴĽ ðŁĴĽ +ãĤ¹ ãĥ +ac ial +a haha +stu k +imper ial +bl ond +pow der +kno ts +vin ce +wood lands +den a +watch in +mat cha +ma hat +galax ies +middles brough +k ö +stre e +resc ues +wal do +lero y +desp ic +real ities +tm nt +ha q +un o +pe c +bolly wood +blin ds +design thinking +he ms +and hra +ab sen +fan s +ste ch +shire hour +bla ine +shak ti +pu rely +ðŁı ı +tra fal +ke ynes +gr ate +to bias +spon taneous +satur ated +caval ry +pri sc +ðŁĺ ij +wh t +pas si +~~ ~ +vir at +patt inson +la o +weir do +sym pathy +ju da +occa sionally +cred ited +stat u +es co +hil ly +esc ape +dischar ge +se er +may nard +sud bury +z lat +or al +we er +encoun tered +sm elling +over sight +ê ¸ +that cher +mack ay +you can +fre ep +freed oms +prophe cy +ho e +ishq ba +dra ke +qu its +pel led +tur k +o vi +wesle yan +new music +leg g +ch eng +h illi +ay y +pan ties +ad versity +ad jac +vaccin ation +ju ke +ga c +exce ed +time sof +sta ining +ep cot +v ital +up ward +bethe sda +apar k +ma hi +camp fire +enchan ting +rha pso +h z +na ver +fa x +vali dation +ac ad +ny r +as ym +coordin ated +depar ted +all ery +var ies +spr ite +chap lin +ss occer +s wat +bre t +relu ct +tunes app +super star +reminis cing +o co +home grown +dough nut +un canny +la pd +thyro id +! âĿ¤ï¸ı +botan ic +bre s +sp ade +i ste +echo es +du lil +bur sting +qui ero +ðŁij İ +loy ola +amuse ment +ha ils +sleep y +burgl ary +âľ ı +ro gue +cot land +mo ors +low er +wic ked +ðŁĶ Ĭ +compet iti +argent ine +yvon ne +karti keyan +ili ary +gat sby +precin ct +six ty +na ji +cam s +practiti oner +ðŁĺ³ ðŁĺ³ +pu ne +neg li +juli en +inv aded +cali br +cla m +duba i +mu k +lan tic +produc t +fe dex +ï¸ı : +eu ra +dari us +s ling +virtual reality +home stead +ðŁı³ï¸ıâĢį ðŁĮĪ +pac ed +in ha +pul mon +la zy +premi ering +ma stered +in he +con gregation +ba jo +sport ing +new jersey +hor ny +lma oo +leng thy +du t +yo gh +swe aring +philosoph ical +pap ua +in ski +know les +dy ke +âĢ ² +to ken +mc guire +ri ot +probab ility +mc con +gro s +su mat +c ite +da a +on da +mad dow +che w +board games +spar ked +re claimed +ad hd +ny se +imwith her +equ inox +boo ths +balsam ic +ha zy +dor chester +ag os +se aw +moder ator +seri ea +ander sen +pilgri m +âŃIJ âŃIJ +itch en +hal li +x ton +nathan iel +mun ition +celesti al +ga f +zo om +mark le +pen thouse +cal e +s fa +bar king +tu cket +em ery +cal orie +li que +ad ar +mc nam +tor tilla +wood pecker +mo town +bad ger +ayr shire +scram ble +dd ay +cra ziest +per rie +cho co +cast e +i ot +wre cked +selec ting +uss r +gra ft +pun t +lab ou +ir st +ba ek +Û Į +su ki +que u +ach at +te ster +aug mented +wc vb +sin ks +ðŁĵ » +ra ke +inter ne +be cause +belle vue +une arth +light en +ðŁĺ £ +turn around +labe led +unemp loyed +twitter kurds +le ia +h ye +great er +ðŁIJ İ +tim ed +i red +e tt +limit ations +cab e +s out +bee ch +anni hil +re trac +yo ona +ang er +den nis +supp lying +di z +" ( +sc ur +gun man +su ho +sauvi gnon +ภ¥ +wi ley +land on +choreo graphy +pre historic +ðŁı ĥ +var gas +assess ments +pinn acle +di i +chamber lain +ì Ī +v p +present ers +deut sche +sun shine +sal utes +r one +bu siest +- .- +motor ists +hemi sphere +al wx +ps p +ow a +den ying +cho c +gu tier +han uk +mus kete +jait ley +se wage +t ame +thin kers +shi m +se quo +pap ar +middle east +k wa +ke g +patag onia +no y +bar ça +take off +he a +à ¬ +n sc +g dc +ðŁij Ī +mou stache +mel ania +thr a +â¬Ĩ ï¸ı +pier ced +ze us +fon ts +ber a +it iner +q atar +contr ary +ire land +i fy +ou los +commun al +fin s +un paid +pa a +ðŁijĩ ðŁı» +ri os +ou p +f iller +cafe teria +à¸ Ń +kas i +cali ber +z ulu +v sco +ts ford +dragon fly +smo kin +pi st +psycho logist +diplom at +we bs +buc cane +à® ¾ +motiv ational +du ne +ba e +c fs +with out +er on +i ac +ate e +pen sion +fra zier +en sis +sk is +par ting +ger y +territ ories +nach os +eni ght +ever lasting +msd honi +tel e +sp un +po di +sab ah +environ mentally +ce ase +beau mont +mar ta +kel vin +ho ff +sun il +n da +co b +sh ale +ree dus +un boxing +u bio +re opened +n all +capsu les +mar r +himalay as +swee ter +ja z +f mr +twee ter +dha ka +na u +de mi +d fs +ta urus +fad ing +it utes +ci p +over flow +jef frey +don ny +car tunesapp +ðŁį ij +prefe cture +danc ed +c pt +ple asing +ital k +earth quakes +ul ation +hi o +ãĢ ĭ +ant an +nutri ent +de ere +selec ts +enrich ment +r iti +tram pol +bl amed +j ia +contribu tors +chesa peake +pi geons +tribun al +mad uro +w su +ilo ve +effici ently +dar cy +war ms +ar ra +ec u +ho wer +strugg led +rajini kanth +ðŁĺ¢ ðŁĺ¢ +hou sing +str at +eli x +disp ro +raf fic +thi erry +na sty +c fb +staf fing +al ma +back ers +hen son +sky walker +reale state +roo s +ness y +chan ce +cair ns +c ci +pe dal +ly ft +cross word +wait er +only in +kru ger +k ir +alej andro +car tier +car rera +re paired +ou at +un clear +un breakable +today in +qu eries +jo dy +gen ital +win ner +to l +kelown a +fascin ated +ãĥ ¬ +sris ri +squ ared +spr ung +negoti ate +priv ately +av en +>> >>> +g ical +gav in +chester field +zu mba +or r +nat alia +impeach ment +mn l +car at +criti que +credi ble +trac y +tan i +musi k +jig saw +gam bia +tol kien +fe u +as per +sav ory +fo xx +f itt +mar lon +l rt +v ell +p br +imprison ed +i om +chu l +wind shield +kay e +ba a +chor d +s art +al gon +minister ial +nat geo +la zio +nor ms +ðŁijį ðŁijį +lic king +fut bol +un sung +dalla scowboys +sh red +distur b +dev ine +be ards +ch f +b day +ro sso +ig or +ay i +si ren +k air +sti les +ro f +mag nets +un cover +mou se +bang ing +si ghted +spe ople +impac t +row land +kir a +environ ment +love the +p sis +mish ra +gl endale +ca jun +o che +de ception +sex ist +stra ws +s ga +buff er +apost le +sp l +pop up +ðŁļ Ĺ +r g +up er +ball in +i dy +occa sional +national park +ðŁı Ĭ +u an +innov ation +ภ« +te aparty +re tte +counter fe +b ha +rec s +ig en +ðŁĮ IJ +humming bird +cu r +ha ven +la zar +pue blo +: : +zi onist +op ath +inver ness +promo ter +carto on +cabine ts +mahog any +surve ying +r ational +feel ing +testi fy +so w +oc on +ภ¢ +ne el +mar is +sol itary +che mo +rad cliffe +sim ons +ros ary +new er +jo die +re tali +pra wn +pad dy +hen ge +k ala +im plant +at y +bren twood +par adox +ene z +re designed +p our +wy d +al de +௠ģ +sol d +biomed ical +๠Ĥ +tt tt +mat teo +ys er +new ton +de bun +ner dy +loo l +wo on +elisa beth +ec c +wh i +ach o +salv age +sal aries +qu ity +navig ating +oph thal +con soles +re built +o pec +ast ers +sho red +set list +kathr yn +rhy mes +re visiting +ash ish +li ft +re post +sole il +âı ± +weal th +sa at +we c +king james +flipk art +field work +se gu +mo dal +bu b +are rs +ðŁį Ĵ +clo oney +pad dington +necess ity +guth rie +pen te +li mo +jo sie +ar tin +en c +l hs +betra yal +info graphics +i er +mo a +hear ings +bon jour +sym bolic +ag ro +wed ges +krist ina +wild flower +athle tic +photograph y +pe sh +ca hill +chi lean +gou l +fi oren +ðŁij ¶ +z il +sk im +bad oo +deli a +tre ble +n cc +ðŁĩ¦ ðŁĩ +a house +bul lock +sol itude +ا٠Ĩ +can cers +futureof work +hu tch +water shed +war mongers +sp illed +colom bo +mo th +associ ations +weigh ed +global goals +not just +christ i +tor g +swe ating +man eu +clu sters +âĢ¼ï¸ı âĢ¼ï¸ı +ta ped +ul y +tru sting +yu suf +te in +ra b +, ,,, +sin ai +audi ble +explic it +cro wns +sch iz +at least +ðŁĹ £ +de bra +je suit +ene gger +z hen +one sie +i it +ss f +gur gaon +chak ra +bear cats +k ran +k awa +reque sting +han over +g end +sor os +mer cy +lovel y +do omed +tim my +ku z +ul l +ab ram +sa ison +ãĥ « +clean ers +re mo +circu its +bar red +o th +mo ist +madele ine +gall o +u j +per mits +hea viest +car ols +az te +gior gio +flo ats +decl aring +us rc +min at +craf ts +pri ma +conven i +nickelo deon +danc ing +ceremon ial +blo gg +tw p +anglic an +she k +k nick +( (( +hubb ard +harve y +hit man +fen g +we some +for za +s word +op us +bro m +gi bility +z al +m unch +dance hall +gre edy +hd mi +re birth +ðŁĺĭ ðŁĺĭ +s world +figur ine +com post +k f +engra ving +gior no +st ana +k man +ham ster +compos ers +aj e +func tionality +pol k +is ons +air planes +te se +hor rors +musc at +gi ven +sp ence +ðŁĩ¸ ðŁĩ +eli ot +ach illes +fre ck +crypto currencies +sou ther +hal o +bor neo +polit ic +hahahaha h +up state +si ena +obsc ure +hau sen +lloy d +happy friday +motor bike +bon a +americ as +hol s +- ( +spor ty +un aware +reven ues +christop her +bank sy +av an +ev apor +com press +eyel iner +to dos +buff y +renewable energy +ly rical +ar chan +rapi st +fair trade +lma ooo +beat z +pro active +la pse +ir ical +revers al +po de +mcin tyre +mac au +ãĥ ķãĤ +nash grier +f sa +g all +çĶ Ł +perpe tr +il ya +configur ation +% ; +str ange +rac i +ภĩ +pic kups +kov sky +mam mal +w ps +g able +compar ative +z h +save our +da vey +on etsy +mu ssels +mis er +cri stina +electr on +cra ve +lo ren +precipit ation +m z +ðŁį « +vin cen +snow board +no ida +ah n +marin ated +g tr +town hall +min is +bethe l +adv an +su ra +shi el +fur ry +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤ +lyn d +so il +sc ence +sen eca +shar jah +dick ens +credenti als +av ar +per k +requ iring +pre fer +j ian +de ca +r ach +ing for +del e +be ep +ðŁĴ » +cis ely +hu ddle +green sboro +haw king +ho ax +hang ar +ç ľ +mis o +lo vin +gre ta +ab ad +logi e +at an +snow flake +mahe sh +fear the +al kal +bobb lehead +ba hn +ju dged +fu tu +feli x +ðŁį ĵ +pi ke +der iv +notic es +au er +dis super +or da +wi pes +am ino +stri kers +foo tb +dram as +pun ching +score less +heming way +bi h +bal lad +chat ter +am mo +kle in +fabric ation +kari m +z end +hi sto +vol ta +rock y +marke ter +xtre me +sequ encing +paradig m +cle ats +boom ing +âģł âģł +block ade +promp ts +yogh urt +pur pose +nu r +regu late +nois y +ing rid +bird watching +bar tender +Ù ĥ +wor dof +cha otic +shor ty +el dest +z app +onceupon atime +fl yo +rit os +mike quind +ðŁIJ ´ +regi stering +. ] +ad ol +gg gg +pur ge +kid lit +ar bor +val ves +synago gue +o th +unanim ous +veri fication +dar rell +ãģ Ħ +vander bilt +tape stry +pro sper +did dy +dra fting +de cep +marqu is +st int +michael jackson +pee led +men us +bb b +sc are +ema il +wri gley +it is +f ell +some thin +bar ra +ed gar +di pping +pu ddle +sla de +lear ner +jal en +ðŁ§ IJ +the daily +mikequind azzi +ju x +iq bal +mckin ney +ra iser +ef an +dr one +cat o +pic ket +cro we +l att +uk o +giuse ppe +hin i +synthe si +ponti fex +song writing +to d +swit ches +din ners +h q +gabri elle +pensac ola +cir cle +expo ses +ev s +riyad h +pro men +o ck +sa j +cit ation +brew co +jo si +ep aper +dri f +point less +tang led +cri pp +line ups +fairi es +daz e +mour n +bla dder +sal z +bur undi +book mark +the people +sub sequ +princi pal +sk er +court ney +a oki +rac ers +ad m +mom a +critical role +hou n +shed ding +sa ka +ace ous +mck ay +hus bands + ½ +me da +accu sations +ro sel +nc is +witne ssing +or ama +go ds +hil ton +el man +ÃŃ n +meg ap +cra ven +announ cer +crit eri +sheffiel dissuper +milit ant +consu l +hoo ded +aby ss +b x +ma dam +lo cu +mary am +manic ure +grat is +ac tresses +ros ario +this dayin +king ly +gn ome +cel ine +r ous +he el +lil ac +vish al +ab h +thor ns +s ls +ne al +construc ting +be ren +s lang +ma ins +far ra +sar ko +pai ge +gu iller +l ala +ice berg +nou n +plann ers +u mmm +ou ses +ill ary +ma an +box ing +zi pper +srin agar +migu el +o str +mp o +responsi bly +lan terns +appli ance +x b +gren ade +neglec t +dy sle +ham mock +ne ctar +wit cher +r gv +di ence +ser bian +seed ed +cru z +bi sh +sp he +e q +sky rim +alge bra +phil ately +bungal ow +ge off +y ves +demand ed +consider ations +the vamp +pawan kalyan +co ded +grit ty +erup tion +se infeld +uni denti +ëĭ Ī +wor m +ac us +se ung +dun g +ro land +su d +di visions +ab lanc +shor test +j f +p oun +plant based +be to +tough er +mc o +don et +mark us +v fl +ðŁı ł +open ing +co ward +caber net +o xi +burle sque +sand ra +su mo +consi st +tho t +cay man +motor ola +gutier rez +d slr +y w +no bel +nov ice +moms demand +grun ge +sp or +d cc +pre sses +sli st +allot ment +voc ational +ft c +pu ja +lo ven +utt arak +tan dem +sh ep +come dians +anat om +cant wait +healthye ating +west side +mar gins +chi ang +asbe stos +stupi dity +proble matic +fit bit +: $ +ceil ings +shu a +protec tions +bio tic +beng ali +re sts +bien nale +tim o +cul min +e minent +affe ction +unbeliev ably +individu ally +canvas sing +wh itt +nov asco +chin son +h pe +go w +gloucester shire +pa o +thresh old +chev ron +s ine +we ther +pp ie +aqu ino +antwer p +âĸ ¬ +po on +inst af +equ ine +cinemato graphy +nbaf inals +vali ant +kil kenny +te rence +syste mic +sr l +p ound +made ira +pl ough +tre cht +mat ed +mp d +ransom ware +ph in +li qui +bb ce +boom er +i standwith +con ju +r te +nar a +foo lish +da shing +vier nes +br ite +da u +juni per +ai da +you now +ra zer +de i +repe ating +comfor ting +adjac ent +e to +ca sted +chat ur +mu er +syn th +san itary +mac le +independ ent +law ful +e erie +h or +ðŁĴ Ń +am rit +vel o +station ery +mu f +may may +contempl ating +elabor ate +gre gor +dri es +ac col +ภļ +schwarz enegger +ill nesses +day break +follow back +collu sion +electr onic +jo vi +hiro shima +ta w +hom ec +mic ah +qu itting +fro sting +ben fica +hel i +s ical +pic cad +corpor ate +ment orship +you are +sing er +shi va +ru ne +ing er +ri um +play able +doo p +wil low +ter re +ni p +at d +war bler +profession ally +er ase +proce ed +pedestri ans +mis chief +ben ding +alas kan +c kett +mo p +dd les +shut ter +ge ared +atene o +ma deline +g ations +o sha +der ick +sw ild +an gry +pat ents +hun k +decre ased +fr y +ðŁĴĸðŁĴĸ ðŁĴĸ +sal on +quant ities +d ario +ni gel +ku ma +jen n +happ ye +xx x +rex perience +pro s +au sch +rele ssly +ham burger +fuku shima +er ne +stat ec +ren d +may field +j one +lef ty +bern stein +sm il +gener ates +fore station +band its +ta yo +r ca +ac ci +rodri go +kn app +elo vers +vege tation +u ral +le ft +ħ ï¸ı +worl dre +sur i +embar k +w son +ba you +mu ller +mo vers +ðŁķ º +presby ter +l f +cre e +bat b +sal am +demonstr ations +an ec +n pc +it ics +to graphy +re inst +thur st +tal e +off ences +smart city +bro tha +ofthe year +in valuable +ear n +ðŁijı ðŁı½ +kre mlin +gra dy +town fc +guern sey +ma ha +contag ious +dre x +be en +( £ +nati vity +k tm +somer halder +comp ounds +íķ ĺ +" âĢ¦ +af g +ott news +h ound +fire fly +cil an +donet sk +volunte ered +ak ira +è ª +sing ul +st h +dro wned +mand o +he ir +ðŁİīðŁİ Ī +tax is +y uki +vel d +k ans +el k +ran ts +hash tag +t eng +ro g +a at +gru b +e ber +in india +colo ssus +sig ni +so ever +mile stones +der o +differen tial +phu ket +master mind +an gh +mel ani +bro ker +actor vijay +stun ned +continu ity +af fl +vo cal +perenni al +fianc é +in complete +hun ts +re issue +domin ates +tur meric +ro am +ri on +bag ged +nas sau +fu t +x ox +national trust +jo ye +san o +hearth stone +dis respect +le es +h se +siber ian +offe e +re stock +wolf gang +re gan +plan o +un wind +re par +mil le +] , +skul l +fat ally +concep tual +ðŁĮ ² +f é +ber to +b ms +u a +mag na +notre dame +le te +la undering +heartw arming +buffe tt +go at +pe abo +wind mill +v ac +continu ally +az alea +mem brane +can cels +make yourown +athe red +p to +tor pe +ðŁĺ ł +ðŁĴ § +sc ares +le aking +z et +pix els +ac i +kh il +marath i +ðŁĻı ðŁı½ +u la +tam u +chandi garh +z agre +aa b +pronoun ced +aubre y +sand er +pun ta +har low +ic elan +celebr atory +so t +unci ation +stru ly +mc dowell +deepi ka +remin ders +my stical +ct c +chat ted +s ica +bar gains +ch hat +ru bin +m net +oiland gas +pel ican +o at +mor ality +k our +i h +nu clear +gc u +ric her +vene zia +m ma +le ith +ac company +rich mond +sports net +ba ahu +smu ggling +mm i +ðŁĩ®ðŁĩ ª +twi sts +sahi b +.... . +amb itions +il lo +histor ical +fo rec +show biz +pon ies +chas ers +remo del +will ing +prince sses +am ple +cushi ons +ac les +lot r +da ch +an the +in corporate +new bury +ki ri +fried rich +ab v +ball ers +alber t +ðŁij Ń +let i +nan op +ci de +anal o +n sf +)) )) +griffi ths +valen ci +ro ano +fun run +babys itting +ca day +ent re +u ck +slu g +tic al +the sims +ro ar +car ney +g am +sto we +fi d +bun ny +sham rock +pe cu +mol ina +go cougs +con tributes +transform ation +mo y +v aj +sever y +antioxid ants +thir teen +sight seeing +l j +reversi ble +odd ly +hoo kah +nou vel +hal al +fe i +stab les +mul t +ho pped +bra ids +inter change +ghana ian +ww ww +eth no +con junction +ago v +ye ti +earth and +ts p +con serve +heir loom +metaph or +woo f +tor io +self less +n wa +em ilia +yl ene +y xe +gi ar +moder ating +pro bz +b fi +ne er +du mmy +hanuk kah +we bber +k v +eye brow +dag ger +su mp +ra ges +ork ney +tb o +hal sey +assign ments +tr onic +scri b +co on +an war +# âĢİ +jal ape +flori da +qu aid +haw keyes +âĻ¡ âĻ¡ +street car +ro g +dat lantic +gran ola +un changed +expect ation +Ù ĩ +mar lin +gu mmy +ðŁĻı ðŁı¾ +awareness month +oil painting +mu th +per ch +jun to +villa gers +mor g +che ated +web comic +the future +d ps +la kings +men tioning +vo or +ident ities +accor d +mc gu +l pga +rum our +massi vely +m pls +heal y +d ate +sp oli +re visited +on t +al and +scru tiny +lakel and +bl ending +< / +an kara +jami edor +metab olic +f ences +ann y +å ħ +semic on +oo tt +space ship +wack y +le ta +ap ac +she e +in herit +do res +ðŁĩ¨ðŁĩ ¦ +gent e +tw ick +ri ms +gal ve +de ville +king fisher +scorpi o +ow l +al ar +vari an +ðŁĹ ĵ +vene tian +star dust +then orth +q ing +har rington +consul ate +spectac le +ho bbs +tur ks +gre er +mat ing +ðŁİ Ģ +ðŁĮ Ģ +direc ts +í ĭ +pompe o +vo iced +la os +tz u +pro me +pri sm +mer c +fortun ately +bc fc +mcdon nell +not sorry +smi led +t ba +for war +mid term +dar by +we instein +up grading +wol ff +bron co +cab ello +ðŁ¥ ĩ +fi able +shar pe +bat tered +sat o +myth ical +instap ic +pre pped +eni um +e spo +di aper +explan ations +who pping +ragn ar +pe el +antibio tic +l acks +harri son +li sm +au l +qu ail +martin a +sent encing +sc ams +di di +tr onics +ãħł ãħł +go ff +za in +param ore +cha ined +clin ton +li ff +cott ages +em on +reve rend +consu mer +ce an +t any +lum pur +e bay +sto ol +ðŁĺ» ðŁĺ» +ta pro +h ath +modern art +just ine +prover b +app y +tra x +mani fest +am bu +nai k +pe pp +r sd +mer chants +kitch ener +shi fted +li zz +âĺħâĺħ âĺħâĺħ +âĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶ +uto pia +tom o +ou ted +com ers +chiroprac tic +book club +cin dy +pro hibition +se uss +ë¯ ¼ +thin kin +rr rr +go fund +t ack +om b +catastro phic +ling u +guild ford +bo td +ॠĭ +plan ter +^ ^ +win k +kath mandu +sto ppers +smooth ies +re efs +hin d +bell amy +Ħ ë +waste water +vo or +nat l +! ] +re el +y ap +scoo by +work space +corin thians +bl un +obli gation +g bbo +dy son +cra vings +ell ington +dap l +wre xham +earthand clouds +uk runchat +positi oned +kal b +four square +jo ck +im pending +even ing +ath y +pro claimed +c ites +ann apolis +san i +mar th +ir l +accom mo +ka a +fin a +y aa +di sper +ec ar +bha k +will y +ðŁĺĢ ðŁĺĢ +mcder mott +mo j +gener ational +u said +train ing +lon ely +lo res +impe cc +âĢ IJ +beav ers +ma ki +he b +aap l +å ı +wolver hampton +leader board +me u +c fa +easter n +hu r +civil war +ou rage +hor ned +le high +awar ds +evi dent +gi gab +r ous +ma del +ro byn +ur gently +k ors +en as +heis man +bam bam +fab ian +f om +evalu ating +assemb ly +out sourcing +hun tsville +ðŁĶ ª +justi fied +cashi er +sp aper +buc keye +analy tical +illumin ati +au tho +o j +sha de +geel ong +wh ey +he aton +terri bly +ele k +un charted +sd live +moto cross +her mes +dar shan +dar lington +cash mere +gri pping +cilan tro +pun ish +... : +ðŁĴ Ħ +inst ance +der i +lo bal +muk her +sp ar +thin ker +fre mont +com piled +color ado +vig ne +sm d +whe ad +villa ge +le ek +formula e +ta res +persist ence +?? ???? +ped ago +he z +alzheim ers +vul ture +off ence +is great +suff ra +kick in +h mmmm +broad way +ï¸ı @ +art i +alli son +endor ses +ry u +lolli pop +soy bean +kend all +cer a +inv ade +( ðŁĵ·: +conver ter +car pets +ho bo +fr it +pe ac +es qu +ern an +ou f +an il +di ffer +ch ing +bre cht +sp g +daven port +stra va +sever n +n gos +stor ians +fe te +parame dic +j hb +al amo +sne aking +gold coast +roof s +isi l +depic ted +projec tions +nu mb +o ss +ep i +glu cose +zid ane +infin iti +íĺ Ħ +ran som +ton ics +fal k +g ler +ou tw +re ss +week ly +the on +n ole +ðŁĩªðŁĩ º +vol ley +sum mar +neg ativity +sam son +ye w +aus votes +ju l +ju dy +f art +pra yed +pal ate +multicul tural +double header +cycl ones +pier re +ãģ ¨ +âĺ łï¸ı +rt w +conver ting +wir ral +l ari +ir relevant +austin mahone +an che +ya an +sd f +$ . +explo ding +ulti mate +prof ici +gofund me +cell ence +ep stein +bul lied +sep tic +à® ¤ +lu mber +cu ff +vsco cam +pl or +ภ¥ +se ok +ro to +venezu elan +sor ta +spir ited +daniel padilla +team sisd +radio active +icelan dic +ðŁĴ ¤ +ver e +accommo date +shi pp +ot ter +ol ina +e go +su la +san antonio +de as +simil arities +âļ ¾ +y om +bro ward +å ° +can cun +veri fy +on te +candle light +ìł ķ +inf ants +az am +ðŁĺ ° +le ven +un stable +bloom ington +x ford +con tour +y p +innov ator +histor ies +po y +lolo lol +ex pires +cat alo +bill boards +an ab +el ic +novasco tia +fa ire +ìĿ ´ +rock well +gr ille +az tec +joh or +ur struly +fi ren +dun lop +id le +port man +jo es +tx hsfb +hol m +cham ele +under world +lo ss +ti em +therap ists +past ure +pa ste +ing now +vul can +ra gon +lar kin +o shi +ho co +child hood +umb rel +success or +kath y +iz en +° ï¸ı +share holders +ol ga +ai b +he ap +fl aming +ro u +air tel +rat t +z ane +vo w +thor ough +sn ag +par th +un conscious +ve y +new release +gh ee +croati an +facilit ating +swan son +astor ia +to logy +master y +ðŁ¤ ij +bil bao +trou pe +the ori +chey enne +ro tt +shore line +gra sso +master chef ++ ) +vi x +ellen show +as g +an ak +ku ya +safar ilive +debu ting +blu m +list ener +v ins +book shelf +smart cities +makeyourown lane +; ; +ðŁIJ ¯ +ri zz +on ward +bull dog +bear ish +vir uses +fri gh +lin den +we iser +sn t +gon a +dre sden +fl anders +cu k +wheel ing +ba u +atu esday +surf ers +swi ft +mc call +arbitr ation +aw d +mon c +b ine +at x +re fr +mi ro +po sey +n are +rit ter +âģ ¦ +play book +blow out +sports manship +s oooooo +malay alam +gri ms +bur bank +infin ity +sar gent +oit nb +joseph ine +ski pping +par kin +excur sion +semin ars +jo har +par tridge +post game +ll ll +blan che +temp ting +m na +lu ka +is ers +to ffee +bar ron +he mmings +sa e +go hawks +cu pid +li mbs +con se +un common +z ada +head shot +so ils +pione er +mam ma +sem itic +pan dey +jamiedor nan +spl its +vel a +son i +ra ff +t mobile +âŀ ĸ +pra wns +lit er +enjo yment +egg plant +tu b +cultur al +us ic +suspici on +sy cam +summ ed +ma du +ho ck +up wards +eye ing +ri ve +assas sins +âĤ ¬ +out fy +chi ves +t ner +la is +por ridge +sad dest +w cc +vick i +sna ils +biz italk +mill an +ðŁĮ į +sam oa +j ing +mi key +gu j +chel ms +eli gibility +arma da +thro p +surger ies +ãĤ ¿ +mo hawk +ex its +me m +is lington +c me +land fill +kait lyn +ðŁİ ¼ +combin ations +tomorrow land +ver b +cor a +pre cisely +na om +ðŁĨ ķ +shr ink +sof tly +merce de +mand el +poo dle +ball erina +sop h +jux ta +y at +ary an +hesit ate +lo wered +gu lar +dungeon sand +ron an +my ri +sp f +men opau +gra sp +pa thi +fe asi +fla w +shi story +ste ward +gg le +fay re +cli que +credi bility +yo g +sec tion +mu sko +se ville +no tt +cal m +mate o +indic ted +fi ba +by l +lin o +u kin +!! # +enig ma +siri us +bu sc +ðŁį Ĭ +mac kerel +psal ms +a at +tomorrow spaper +ðŁĺ ĸ +p fc +........ ... +shre k +mul let +o sh +danger ously +immen sely +am ur +ðŁį Ĥ +pro por +sy a +london marathon +abo ve +obli gatory +pro v +ra cha +alex is +pri mary +sh h +ether net +d stv +cou gar +un lucky +ni l +steak house +mel a +fc bayern +cause way +ca therine +fluore scent +nx t +to kyo +au sp +releg ation +qui zz +shored itch +proud tobe +promo s +inter acting +home brew +da esh +w pg +stead ily +provin ces +bal lots +i ah +al to +< << +you u +ri ley +prefe rence +tra verse +incen se +am munition +ho dges +# @ +hail state +tart an +witch craft +vent ilation +liber tarian +! âĢ¦ +ow es +% ! +ong chang +bru shing +le ic +fi ber +under attack +down load +ex pir +hy o +pompe y +mc bride +y ag +stre e +com bat +ten ding +ai ra +gug gen +ab ra +in na +fli ps +aw al +m ach +dol lar +inspir ations +z um +o du +it ty +video game +aqu aman +har u +bel fast +je b +but ch +us gs +calcu lus +go yal +mor gen +x finity +stand up +contrac ep +sab re +na be +in secure +gener ously +epit ome +l w +t ca +narr atives +don nell +pand as +ber gh +tu t +ker al +fel icity +br ampton +quinte t +nom ore +ðŁĶ ij +lo i +alham dulil +ðŁĶ¥ ðŁĶĹ +ston er +shaw l +clin ical +bren dan +gon e +fla wed +tri ppy +j g +al location +po aching +ve vo +mo cks +lef tist +bon uses +condem ned +abil ity +st ating +microbi ome +bio logist +for you +wahl berg +ss or +ift ar +w ul +ÑĦ оÑĤ +pom er +me me +ver te +tre ll +tra it +in let +hormon es +deliber ately +vill ar +battle ship +p bl +tw enti +ho kies +dal ail +say a +may fair +han s +die ts +⾨ ⾨ +od in +hot spur +pap i +k ana +k amp +fin na +flo tus +ti ans +unic orns +tribe ca +chang ers +fore ground +out a +inv aders +gett ys +tomorrowspaper stoday +mac millan +hand written +w fp +u de +state of +base d +âĺģ ï¸ı +cas m +psy ched +histor ians +fol d +d da +ag grav +p ans +green way +au sv +ðŁĺ ¶ +shradd ha +inde x +be sti +zim mer +t ness +eye shadow +ot te +go ts +distribu ting +pro min +yo l +ace a +tram rahim +hoo per +supre me +jam min +intu itive +quali fications +sli m +sid di +jay ne +tri pping +g tx +pun s +e manuel +om g +mid summer +in to +succul ent +ri en +new mexico +o or +hoo king +in f +ðŁ¤ Ŀ +flir ting +na hi +g friend +t ps +hel ix +z s +on ie +ct f +kri s +irresi stible +fla p +ðŁijıðŁı» ðŁijıðŁı» +us wnt +ru d +ram ps +pin oy +ot w +lol z +low ering +favor ite +t mc +phra ses +her mi +aver aging +em br +ben o +estu ary +sle eve +ribb ons +ta sh +ภ¹ +x f +aw gs +sun ited +brew eries +anir ud +pun ches +ol die +ip ads +wi fey +land lords +d ji +gun ner +íķ ´ +tex an +ex op +cas sandra +s off +ðŁļ « +igh ton +bak ers +awareness week +v all +ear p +bts bbmas +apologi zes +âļĵ ï¸ı +was ps +states man +snat ch +watch dog +ra fi +after party +spi ke +j er +peri ph +r nc +mu ll +le en +shi es +li eu +urstruly mahesh +mer ton +de sai +shi f +ðŁĮ ± +pe dic +gos ling +arrang ing +ww g +gen y +you uu +netfli x +e ttes +k wi +bernar dino +am iga +Ø ¨ +kashmir i +t ings +emer itus +de cat +ab domin +dc i +pha ses +d jan +be am +op ry +i shed +the ellenshow +the st +habit ats +to ons +mclau ghlin +ri pper +micro biology +tal aga +clu eless +ss u +cro che +bro mance +longe vity +zagre b +prev ented +tra ve +spo ilt +darry l +migra ine +al cat +dd dd +vi v +ser pent +mat tel +jam a +con quest +î Ħ +sam sung +presbyter ian +ket ch +fire fox +mo tif +le c +cho pping +cher no +j ann +ðŁIJ ° +pro lon +wake up +conver gence +mersey side +heart broken +lo oming +hal lucin +mai ze +commun ism +mo h +twitter storians +serge y +res eller +favor able +ed gy +re iter +mal aga +live me +ka hn +pul sion +big g +kim kardashian +ati o +tyr anny +ru ption +q ant +pro ven +by z +pu shaw +kri stin +e er +tar dis +ri z +awak en +mi ko +un documented +path finder +indirec t +resemb les +h ler +conce aled +scand al +re im +d nb +cr itters +attend ant +apprentice ships +aa u +scre amed +l su +fa h +har bour +ed d +bat sman +li ss +mi sha +spani el +it f +advan cement +fa c +close up +cecil ia +medi c +narcis si +lav ish +gi ac +ma ys +le it +wine wednesday +pushaw ard +let to +curren ts +bug atti +out ine +w j +un do +ler osis +devo tional +ðŁij « +on na +fais al +sa una +himach al +am ii +à® ® +di zzy +screen writing +ph x +sp n +ick i +ag irl +fi shes +wb z +pi m +bo ar +ac id +! .. +rocke feller +n ga +dra stically +simpli fy +dru mming +autum nal +gur mee +lor de +jo ann +give up +b our +am ura +der land +sim pler +wat son +tri dent +concor dia +bel lum +bre k +dum plings +vi on +dungeonsand dragons +sp ri +ascen sion +wil datlantic +u st +rob ins +legi on +insi st +jar o +gue ss +so b +bigh it +pool side +negoti ating +mc gill +bil d +techn icians +miti gation +ajay devgn +b to +ant en +cosmo politan +ðŁĺĬðŁĺĬ ðŁĺĬðŁĺĬ +patri oti +temp er +promen ade +nav ajo +nam m +wrink les +dc fc +le ach +bru nette +r f +cout inho +al ti +tradition ally +op tome +na z +accord ingly +rec ard +de ets +sw ell +po sure +whit ening +strang er +illi on +here ford +u wu +ro bber +cotsw olds +cl en +gor ge +nam aste +re lish +gri ff +adren aline +bla sio +val e +ê ² +toler ate +rail minindia +jen sen +ho ven +el lu +ob sole +eisen hower +unidenti fied +than niversary +body guard +Ø ¯ +i dge +sch al +stock port +sn i +re taining +po po +pix ie +oli thic +ki er +ha jj +sa z +cor bin +!!!! !!!!!! +v it +me gat +de h +circu it +af fleck +theore tical +hope less +u ab +slu mp +b ice +jam med +let stalk +can i +side ways +labyrin th +re fs +ha hn +jare d +ðŁį ¹ +jam bo +ph yl +enhan cement +c tr +ful lest +se ye +do ba +cho ic +yo s +cb j +andr é +re watch +pri ma +doctr ine +for gets +u hm +ar ound +u le +art lovers +shi raz +har th +ex tor +Å ¡ +unexpec tedly +eli us +y x +em my +se ac +ðŁijĩðŁijĩ ðŁijĩ +correc ted +com bu +wom anc +cou gh +what son +publi shes +divers ity +back bone +lock down +mesmeri zing +nor te +ma b +desig ner +í ģ +ra gh +mole cules +get outside +the beatles +semicon duc +nach o +lun es +ham mers +sul tan +o on +fe ren +att ach +ar qu +uttarak hand +s ash +; - +tre ad +i ko +ar thur +scandin avian +r ation +ga el +charge able +fish y +v ma +hand bags +char a +ay ne +de fam +sett lers +qad ri +pal ais +in wx +apocaly ptic +poo ja +a es +at ories +proof ing +n lp +ts la +v ina +li do +dee phouse +informat ics +v v +pp ings +di ss +à ¯ +uhur u +st ony +betra yed +b aff +my ra +as pen +allow ance +tam ara +ci f +cor bett +ser ge +di go +ambi gu +pain ters +p cr +p ca +nom s +lo ft +ve e +opend ata +ðŁIJ ± +alex andre +identi fies +fantasy football +re production +brom ley +ware agle +mm er +p ss +cu es +ay at +hut chinson +sar ac +jack man +ira h +ap ink +col s +aussi es +ex ecs +day ton +ðŁĻ Ĩ +im v +har am +chuck le +authent icity +ar do +incub ator +ภª +photo shopped +embrac ed +fight for +gor man +zz zz +schol astic +cri sps +te apo +mid night +ga ine +col lier +s ate +de tte +å Ń +imag ine +i ff +tw ili +i fication +teat ro +nor ma +es ur +emergen cies +rise up +r inger +hass le +cait lyn +tranqu il +vers a +se b +over look +gin i +bo go +se re +may ne +henri k +contamin ated +rhapso dy +pro portion +wildatlantic way +âģ© . +organis ers +tran e +stand ard +sper m +laun cher +ric ci +her ts +paper work +showcas ed +mer yl +pen a +p imp +disa strous +^. ^ +phar a +x is +fron tal +sw irl +sp ills +swag ger +smart watch +sizz ling +savi our +cat ar +bb cr +refurbi shment +dr is +citro en +absor b +patrioti sm +il leg +chro mo +fresh ers +ru s +lim iting +ef ish +down ed +man dir +hazel nut +p all +mac on +disappear ing +quali fies +bo on +bar racks +am ine +gen dere +ðŁļ ĺ +j es +ãĥ Ń +qu ito +middle weight +sch au +quad ru +aci ones +limit less +ðŁijĮ ðŁı½ +ch man +ar av +regulat ors +it up +batter sea +mil ford +g z +tic king +gh ou +cru shes +tu tu +dread ful +fam ine +for change +dalail ama +ðŁĴ į +whit aker +hash mi +h us +vo d +bet te +aa ah +iso o +ðŁ¥ Ī +ha ar +la ine +b v +all day +spr out +indie games +free bie +gree ks +but ler +ill in +ha al +ware ness +si ma +public health +gam a +wa a +oun g +goo oo +okin awa +off enders +im pose +ho c +young ster +story teller +sc ap +figh ter ++ , +whit es +music monday +re za +go ducks +bri a +mi um +cas per +cru mbs +a ad +marti alarts +ch p +ri gged +tn g +harve sted +sa k +do jo +mill wall +b nw +oc d +histor yof +t mr +si rens +fan ci +caregi vers +vir a +son i +recur ring +acknowle dged +ðŁı Ł +oph ile +bu cky +stre ssing +roo k +di gger +vi val +san do +fle et +si ers +sel caday +refre shed +anti fa +a que +po lo +disappear ance +de mb +âĮļ ï¸ı +ren ted +ber ger +g mb +cu la +ss al +goo dy +u hh +marcel o +w anna +soft ware +shop small +turt le +tom as +fri sco +ðŁĺį ðŁĴķ +jim enez +c su +day z +an do +wyn ne +choreo grapher +cerv ical +trail blazers +ed g +zend aya +travel blog +el s +whole some +co g +lab out +ar ney +del le +su isse +ma si +ine se +om be +fi ddle +re claim +pa u +wat cher +sla in +ber ty +opti mum +el ites +min is +tur key +patro ls +ger ard +au reli +wild ly +wal tz +br gy +w ob +cre st ++ ++ +ve z +fro sted +davi do +the x +param edics +p into +han k +du pont +ur g +fo stering +micro poetry +spec tre +---- > +ne uro +fri da +music al +galve ston +e ffic +sc ape +pal azzo +th all +pro visional +p js +au re +ðŁĶ ľ +mam amoo +kit ties +cre e +wa k +lo ool +lu pus +cn blue +à º +ðŁİ ¬ +rac ed +tro se +om as +stri de +co ors +⤠µï¸ı +in comparable +cy ril +broad er +arec lipse +ðŁį Ķ +inter val +ti ru +co working +w aco +a ham +a bee +flouri sh +the times +ol ini +kick boxing +lu cer +at la +as un +casser ole +mi aw +lobb ying +jan ice +cir que +re flex +le ary +sanat omy +tem pest +se mb +mur dering +us av +ro bo +on et +p cc +nati ves +life of +sa ha +ruth less +rel ates +appeti zer +pye ongchang +nor d +er u +a thing +ug ly +pl ying +bran ce +organ ise +kend ra +dat o +chees es +par ma +burn out +a stra +pre toria +adjust ment +uk u +sl o +li ken +fav ors +cli ve +be ets +snow donia +go tv +sy n +open house +pan i +portra yed +sl ated +me cca +ren al +supportsmall streamers +staf fs +da o +bi ker +vik tor +tit us +admi red +ðŁĵ ± +hurric an +he ats +gl ory +photo genic +mer i +de por +burn ham +or angu +dj ing +impre ssionism +ign ition +ca i +w ynn +de pe +cove ted +colla gen +sau s +or nam +administr ators +ss on +nh politics +hahahaha hahahaha +aspir ations +r gb +swol len +so we +sc r +diver gent +hou ghton +han oi +d ory +ni ki +land ry +b cci +ðŁijĮ ðŁijĮ +is mail +tri pod +her d +bhat t +dress age +tab by +ingu ish +hur on +à³ į +à ł +to das +evangel ical +chor ds +st john +slo ppy +marty r +face book +ali ght +sen sei +kath niel +r ites +zi one +u o +revel ations +weight lifting +pan o +nc wx +ac ton +à® ķ +Ø ² +som a +à¸ Ĺ +respec ting +mar che +fore man +be tty +ki k +shi bu +po on +argy le +k swx +et z +mar bella +brac kets +stand by +fire side +defi ance +v ex +britanni a +in habit +appo int +piyu sh +le ash +sci ento +fla sk +sen na +> : +at roc +sand erson +id lib +dhan ush +ðŁĺ Ļ +en thr +hit ch +de dly +al ley +dor k +mon do +cudd ly +mis sin +ye sss +night ing +j pn +w ary +ump ire +ma z +ê ³ +bab s +ĭ ãģ +stan ford +posse ssed +exce eded +ðŁĶ ¶ +wall art +tra p +j il +hi bis +sp ying +scri be +khali l +trans lator +lu mb +di zed +ch c +super vision +shut ter +ja g +_ * +yester days +ms f +hi hi +gonz aga +gille spie +vive k +ec static +this morning +ch us +ed es +ston ed +be es +ðŁĩ¹ ðŁĩ +tur in +ho ver +at rics +ster n +sam heughan +auti sm +mi ya +eye witness +writ ings +travel tips +chut ney +px rtg +keny ans +my stic +k rit +/ $ +red head +world ly +am us +op la +le ve +gab bana +se en +o clock +gang a +keen an +sc ent +ol dies +go green +corner stone +comp ly +con cours +ðŁİ¶ ðŁİ¶ +ha an +con fis +aw son +cle op +î Ģ +su zu +sau té +al gar +subscri ber +este emed +ãĤ¤ ãĥ +worth while +mel rose +flo ck +bri ghtly +viol inist +p ere +sli pping +and co +si gh +ha van +cu lo +m sa +fibro sis +matil da +ra fting +aw ard +ë ª +mm mm +ge aux +ste iner +sin n +help ers +beet les +ai mee +tai wan +pistachi o +mac beth +m zan +descend ants +on sale +in r +il m +grou se +sa ig +mo w +bi gre +adjust ments +tu la +mathe w +transl ates +mu h +bol lah +ðŁĴĽ ðŁĴĻ +amo res +ab outs +bomb shell +bla ster +x avi +s ns +k roger +ga ther +erad ic +daf t +chem o +ben ches +ðŁĩ© ðŁĩ +ut v +our a +n ko +gator ade +biaf ra +ok state +im danielpadilla +dom ains +open ingday +kid do +do i +ric e +day care +mac millan +ba thurst +cheer leading +ðŁ¦ ģ +cash back +k won +hob bies +exem pl +ries ling +âļ ª +ag les +ny s +every thing +nav is +ad di +magne sium +faceli ft +ark ham +grand es +extre mist +don at +vit ality +pump kin +be tta +sl td +arti san +li by +pe aked +ah hhhh +mary am +assi m +un sc +ment e +al aya +low ers +ar as +gri ev +le ip +gr ati +cri ses +spr ints +exe cute +w to +ms d +mag ical +re viewer +spark les +juke box +ðŁĺĤ âĿ¤ï¸ı +pay back +licen ses +dun kin +bel t +lake wood +h ateful +bud gets +rev amped +ph erson +ky iv +went worth +ro sen +cru ise +gi ggle +def star +assassin scre +ym outh +win kle +w fc +band wagon +b kk +w iring +kear ney +south side +pe tit +! ðŁĺį +nor dic +mir za +mu gabe +v l +scon es +k tv +sand al +du c +m alls +ðŁĴŀ ðŁĴŀ +it c +al ay +im pair +un rest +flo ss +c é +ab ou +var ying +muse o +ser ver +di ya +hibis cus +ero y +mer ritt +fin dom +f pp +un usually +go tt +conting ent +ali aa +ball on +jo l +hi ked +zy me +ay r +ag n +ga z +perio dic +spar ty +practi sing +lin ton +tal is +cy pri +womanin biz +radio disney +ðŁĮ ¼ +jump ers +endo cr +ðŁļ¨ ðŁļ¨ +and on +shar apo +mi er +ma sonic +fac tories +vi en +bb ers +ìĽ IJ +hol d +ke bab +be ak +approach ed +ac milan +mun ro +ko sher +excell ency +negoti ation +walt disneyworld +cr ouch +te asing +suppre ssion +en ya +b ce +transformation tuesday +cal lie +vis was +p gat +ic ted +end ings +esc u +recru ited +it fc +collabor ations +g ino +snu ck +ausch witz +i fc +x ii +ke sha +ger vais +clo ak +x l +sa ad +prob ation +pre cau +mac in +anasta si +le k +e azy +daysof code +mariah carey +yo g +stit ched +boy friends +sh ar +ph ile +ag u +twin kle +phi shing +week ender +ic ton +gurmee tramrahim +al ton +l eness +all an +pen ultimate +kry stal +go u +lan de +dis mant +ab using +nor se +pat erson +ed mun +ap an +xi umin +sk el +cat walk +re act +wal led +t angle +br yn +ve to +super moon +cas ablanc +appreci ates +ski d +bo th +catal ina +ele ague +cyber monday +cau tious +ðŁ¤ ĵ +nov o +hamp ton +ha ye +jose f +var an +lo bos +roano ke +orph ans +tt in +squ ads +ishqba aaz +black panther +e tu +k sh +cru mble +cess na +reli eved +scul ly +pollin ators +explore canada +ki es +kam loops +kir an +pri mal +sett lements +hot spot +brain storming +ce dric +bi ennial +sh ant +âĻ¡âĻ¡ âĻ¡ +do on +hear n +walk way +fe m +ve al +deport ation +tox ins +elimin ating +descen ding +by the +bla sphe +ha sta +comple ment +as cent +ri ga +provo st +âĸ ª +wee ping +anti semitism +employe e +unearth ed +pin o +natali e +bla d +ang ola +lock heed +in ian +ag r +ni ster +im pala +m ke +fan atic +âĺħ âĺħ +ðŁij ¸ +lu ch +simpli fied +gall ery +econom ic +cy borg +con i +sel ma +in ception +ko ala +dv ds +cre sted +m mor +visi ble +n sd +ðŁĻĮ ðŁı½ +w under +refriger ator +re opening +e era +carou sel +as p +balli stic +victor y +mo tive +tre y +sharapo va +si i +mon ter +int end +west chester +sp e +cy mb +vi dal +ll ama +uni v +fin er +crafts manship +jazz fest +b ch +ag gio +n cc +lamb da +tranqu ility +cis co +ba den +so bbing +of i +go ta +ru mored +war med +ore an +ac ton +mar ci +gh ani +âľ ĵ +as sorted +pembro ke +pen elope +da f +at ty +aim o +pretz el +carni val +than os +ko chi +mer sal +ham radio +ar twit +cas c +guer rilla +kush ner +k app +al ise +todd lers +steward ship +o tti +ter ri +tem pe +rest less +vit o +zay ed +rsp b +pi on +hi ppo +haw thorne +in as +am ily +nut cracker +lo p +d ali +tro pic +ðŁ¤ ł +ul o +jare dle +py rene +pale o +usa ir +m ould +it ated +gene tically +biom ass +ðŁĩ³ðŁĩ ± +do dd +practic ed +monarch s +un manned +m buhari +am al +photo gra +ko ol +bren don +ju ices +cu re +world bank +poin ters +ðŁĴ Ŀ +tur f +le ds +bor ussia +bapti sm +warwick shire +moun ts +gay o +be gg +co pied +asi ans +k g +moder nist +gi d +front man +concentr ated +y t +sc avenger +iron ically +adi c +ps n +ðŁ¥ ī +cultur ally +yu v +mac arthur +fertili zer +be withyou +ri gor +min ors +z oning +âĸ ł +ri r +adole scent +vin ny +ren g +sand stone +gu et +we sth +ple dged +lac ed +sp ide +v ai +ty coon +seiz ure +du p +appalach ian +ro k +cathol ics +sey chel +posse ss +la ger +jo di +cham p +stra s +d ina +cent uri +cal der +blur ay +ðŁĩ¨ðŁĩ ³ +mo do +an nette +youtu bers +chap s +ang ling +label ing +a qui +pk wy +ly le +bi sexual +lit ur +dug out +li bby +grey sanatomy +sub stances +august us +rall ying +fi del +ing ue +äº º +hallmark channel +tooth brush +m á +adi rond +ag gi +ðŁĵį : +cru sade +tax ation +k z +i ver +dou bling +room ie +wa b +en rolled +az on +a ju +grand children +as df +ðŁ¥ º +mat ic +ough ton +utili ze +ðŁĴ £ +pon der +rais in +dys function +co bain +butter nut +e man +su red +dri an +and friends +with the +on omy +heine ken +bri dal +leader ship +pyram ids +deutsch land +jo cel +bo wel +y qr +horse power +be acon +ing eni +gra dient +fer mented +mo om +thing y +pot assi +wrist band +bor d +bo died +ðŁĺŃ ðŁĺį +ma pp +ka u +cyber punk +ph ish +loo king +co ates +ap ur +am ie +uk labour +at in +g la +adop table +shel by +v illi +ri ya +m ingly +cli mber +bumble bee +ðŁĺ ¸ +c sd +âĿ ¥ +hospit alized +c ki +hat er +ch r +re tina +it a +fan base +beat rice +gwy ne +go ss +fo s +favor ited +swachhb harat +mal ade +mon mouth +" [ +si van +sh hh +command ing +sains burys +wee d +g man +ss w +rep tile +iv y +tro pics +roll ers +over cast +ex position +masquer ade +man crush +wa ist +spr inter +sle et +le vin +j pg +_ ( +o pel +explo it +ap a +po we +wrec king +jong in +or b +er ick +bo sco +pra ising +ber tr +to wing +in security +ku t +resto cked +rr p +prescri bed +trafal gar +per t +g ases +app rais +g har +music als +âĸ¬ âĸ¬ +mc fad +ag ony +conditi on +equi p +shi k +atra vel +ðŁĩ¿ ðŁĩ¦ +ke h +abduc tion +pe oria +wil kins +g ms +as d +ev i +ðŁĴĹ ðŁĴĹðŁĴĹ +u z +mo c +halle lujah +guad alu +lou vre +dra wing +go ve +ph ant +fri e +web dev +program mer +z able +games com +clari fy +li th +kin ky +âĿ £ +labour doorstep +son ata +ju ris +mai den +vi adu +buch arest +conditi oned +capit alist +u de +ps b +sp ca +lul la +footh ills +kay o +bon d +wom b +roun der +ce sar +bur sts +ap ra +sw oon +sab rin +fra grant +cle arer +ku brick +cli max +jour no +ag le +ðŁı½ âĢįâĻĢï¸ı +poo ch +hal e +sol it +sal mon +organis ms +bron son +art en +hodg son +alo ve +vent ure +bb i +ae a +ðŁIJ ¢ +ld n +d nr +o zone +el las +man ny +azz ur +un beat +tru ffles +th ong +ma ñ +las ers +ley e +gettys burg +back packs +or is +ma ison +craw ling +la bra +cl ing +dra gging +ste al +dou bt +de van +ck ers +agent sof +photo bomb +elon musk +abo y +dist ances +story line +sp i +nor than +europe ans +wh ale +ser pent +ðŁļ ² +fi or +tr it +ox o +awar ding +class mate +su fc +smar test +rich es +pr k +big foot +ar mb +bi polar +dw elling +om ars +k wan +gri me +m eng +freder ick +navar ro +sorry notsorry +jaredle to +pa ve +sl ack +barn sley +att ar +evic tion +accumul ation +o ir +cat chy +wel ter +vik as +has see +nik ita +mo yes +mathe ws +shi v +gat wick +pro filing +compan ions +mar rake +an tics +ðŁĻĮðŁĻĮ ðŁĻĮ +se se +bo i +bart lett +poison ous +ab uses +ym m +kam pala +guggen heim +imv kohli +dol om +bre e +thro ttle +gare th +fitz patrick +un ya +par ad +mar got +j nr +we a +potassi um +p nc +disgu ised +cra sh +ren ergy +ill ic +coup led +ni els +ci ones +æĹ ¥ +im ent +despic able +d ye +what cha +conne ctions +paralym pics +gaunt let +wait rose +suici dal +star ship +vap or +st ou +law maker +coo led +si mo +then o +offro ad +ja den +bas que +vick y +lu kaku +centr o +tri sh +strate gist +medic ations +hor st +b fc +gra il +sharp ly +ad itya +tom b +kau fman +tri pad +sam ba +pastor al +brit ney +sag an +hill side +mas ons +sar a +z one +x u +to tes +rob bie +app en +mon tag +der o +short film +charis matic +tat ors +ki ba +and ri +al arming +split ting +ic ar +th ug +scari est +sylve ster +an an +u trecht +a difference +me ade +bu ster +air strikes +cu ffs +account ants +ðŁĺ¡ ðŁĺ¡ +new t +bo tt +issu ing +cl ancy +wwen etwork +kyu hyun +rese mble +pajam as +sin k +kin ney +sul ph +or k +li es +la gh +or ton +ra hul +d sc +we will +re am +collo qui +shar ia +hec tic +sar casm +land er +tm z +endor f +ro z +ham mered +fri s +w adi +pope francis +he it +flash light +un born +op es +hol iness +ðŁIJ ¦ +nach t +im sa +gr acing +bj p +ver ts +c sc +home owner +a que +bigo try +anni e +bag h +âĿ¤ï¸ı ðŁĺį +car i +thom p +dispo sable +cardio logy +pat ented +hh hhhh +ld r +stephen son +cro res +fan ning +cli mat +ðŁijį ðŁijįðŁijį +ðŁijį ðŁı¼ +aer on +piccad illy +bank rupt +sil via +emplo y +don ny +commen ting +screen writer +io ta +ce an +anc ers +tu an +street wear +ठ¯ +sk ine +esp a +asi f +os ce +she ppard +more cam +bott le +der s +orac le +google play +aver aged +edmon ton +steph an +sister hood +cru sted +stag gering +methodo logy +congress woman +c abo +tri ggers +mil ky +gli de +tooth paste +room mates +nu ff +gu am +sprink les +alternati ve +wat fordfc +uof t +hal ey +cont acted +bun dy +pro stitu +gh ar +pre ston +on site +hil ar +g ts +c att +hamp stead +? ?! +ðŁĩ§ ðŁĩ +bbc qt +aless andro +resi st +ma idan +t ko +shad ing +pin up +gal lo +sin u +at ec +fun k +ac lu +stri des +rhy me +wet land +bbc springwatch +t ins +wild card +st our +flamen co +pau la +onto logy +gang sta +am ade +ãĤ « +t bs +skelet al +run ner +jard in +harri er +hun ted +z hen +believein film +de mean +au diti +re start +chon dri +âĿ¤ï¸ı ðŁĴĻ +mcla ren +ga b +sh um +au sa +lewi sham +y pg +k jv +fur nished +dor o +bon ded +mor ty +lat itude +_ ) +lo va +water ways +vin ai +shor th +drun k +c ay +ay ana +kap lan +capp uccino +spr o +life boat +has bro +spol ice +tor on +do ing +dam n +sh ree +foun tains +ent ation +mar u +boar der +to pless +j ada +chan ning +ul ls +en closure +gib son +fractu red +brit ton +à ¶ +t ous +por th +dra f +tra iling +mar gate +eli fe +down ward +lin n +gla des +girl power +ak rish +u ki +ron da +ts c +appreci ationday +vis ing +lo om +ðŁį ³ +mex ican +ar gos +y ya +jad ine +south port +d end +si sta +rede em +men g +bra xton +antioxid ant +s key +mp g +fin ding +vibr ation +ce u +kh art +di mini +cl ine +shel ly +hin es +ī ï¸ı +to pical +no ver +ma xx +prim itive +illustr ate +b ounds +tren ton +join tly +breed ers +u chi +wakeup america +b ada +ðŁĹ £ï¸ı +gu acam +sp heres +pere gr +youth ful +lo lo +bir min +t ly +jeremy corbyn +defe cts +co sm +a rent +v aa +bag els +medi ac +cori ander +ic ago +g haz +ab bas +re model +struc turing +pu m +out law +ad ani +r bc +gul ls +n li +confu se +ðŁijĩ ðŁı¼ +vil a +mcnam ara +correc tions +mug hal +ser i +re gain +ss b +lea ve +haha hah +gran de +di stressed +re chargeable +ho a +hou sed +sti l +attribu ted +opath ic +di ps +pri t +head phone +conclu de +pil o +he t +ut sa +nit in +je m +sni ppet +tutor ing +op er +sun k +en sla +cha u +ac orn +quinte ss +ran kin +affili ated +our lives +cl int +se ater +isa ac +ba shing +sme ar +nur se +doo dling +" ; +sa ku +atroc ities +im am +g fs +viol ating +comm end +brad shaw +er ville +b illed +b be +thul hu +i phones +moo se +di os +re w +me thane +strang ely +whis ky +ti ghtly +spiel berg +radi us +notic ing +wi f +ig nati +i fa +ap is +w ali +ha itian +bu shes +y z +v l +ex ited +asse l +tru ec +dom en +ash er +in king +newyear seve +hend ricks +bat i +ìĿ´ ì +rich ter +mon santo +con line +agre at +ðŁ¤ ¯ +master pieces +ar n +rough s +cle ve +se v +fashi ons +to ya +sh ail +cop eland +aqu ari +dec als +are you +y aya +a str +fon t +ml m +ar ca +pp or +pol lock +xper ia +conserv ation +chain saw +ag gie +?! ?!? +si le +sh on +ìĹ IJ +note books +marque tte +de us +bb led +spic er +mc cabe +nor wich +modi fication +boo sted +stru m +sales man +bang le +nis san +hez bollah +brea sts +a af +anth us +sk er +ow ed +her os +gi fs +fo sters +eat ers +du es +_ / +lymph oma +sf am +me gal +afri di +ag ic +p amp +jeal ousy +ðŁijĮ ðŁı¼ +calcul ate +napp ing +g ale +ðŁ¦ Ħ +lub bock +assu med +ren ting +íĥ ľ +subur b +ãĤ · +tech nic +u cla +in front +gar net +ster oids +stri ving +ho war +mo ver +le ton +bull do +is in +ci ao +sn z +fore front +d ams +mid wife +ma wards +cla pton +we in +subsi dies +spr oud +rother ham +phan tom +ar ach +spi el +rac ket +sel amat +no on +l bc +enti ally +ðŁĴ ¸ +sil ve +m oud +kine tic +y asi +ðŁİ © +o ol +mi ku +i za +fer a +flo ren +barber shop +groo t +z est +ne ars +stan is +z and +police man +juris dic +form ations +appar atus +sp d +arti fact +to sc +motiv ating +womanc rush +re dro +diagno stics +ra za +out fitters +el xn +dod gy +ry n +sh d +ortho don +ol de +jay anti +bal ances +quic kest +can ton +friday reads +! * +na a +a ak +ðŁĶ · +behavi ors +rasp berries +ä » +polit ical +cam il +å ľ +di k +ast ounding +lie be +novel ty +tur moil +sul ly +spring break +hon ouring +cc g +ðŁı Ĵ +my little +ky c +pro ms +ðŁķ Ĭ +à ¨ +bi ge +av ril +ðŁĩµðŁĩ ° +mari on +as ants +sur ya +oc tag +luf than +ac ron +fayette ville +ti que +love s +en ca +de kalb +ta ver +de vote +aux iliary +joh annes +tread mill +ay an +qu r +donald son +cher yl +" .... +s ven +kir sty +gun ners +ra dish +o ahu +v sky +i ble +con course +b ps +elo qu +ash ford +te bow +roblo x +ma da +dri ving +th day +spro ject +m ms +band ed +. !! +libr arians +flan nel +intoler ance +her al +ç µ +neme sis +list a +tar ak +cry pt +star plus +vish nu +sc ale +cr is +% ), +j illian +regg ae +pegas us +ol in +ip ment +man ic +l fc +godd ard +ite am +parl our +anch ors +lee minho +talla hassee +ant it +d ho +kid ney +y ash +batt led +az ad +gar is +faul kner +sni ff +papar azzi +ed m +phy llis +con tested +aa ay +se ca +k ton +vel ve +rain ier +for um +tam pab +ho sp +trac tors +ox fordshire +no tion +guang zhou +ðŁĺ ¯ +ref ill +wednesday motivation +sli der +mukher jee +pr att +fon taine +alph on +af ar +ts i +pest icides +fi ends +mo cking +bra w +tran sat +do ses +co res +hom ophobia +docu menting +zlat an +con doms +s é +sun set +kun st +ton ga +ภª +v ation +sp ray +chow der +ra ps +palla dium +nor wood +music history +hoo ker +si si +osp rey +ph ys +conce ded +bob cat +ar mad +ze it +Ù Ħ +ðŁĺģ ðŁĺģ +mer idi +ðŁĩ· ðŁĩº +corn wall +! ), +touch downs +ze it +chal et +mm m +al che +gor illa +fo ss +ati ku +lumin ous +ivan ka +be ek +sta res +sw iss +âĿ¤âĿ¤ âĿ¤âĿ¤ +scru bs +me ath +gusta v +jo gging +confe tti +as os +ers fc +breit bart +applic able +autho red +ya ho +h in +displac ement +j v +ðŁĮ¹ ðŁĮ¹ +ot c +non profits +diec ast +gu sto +inte stin +c ages +me en +lu kas +moon ey +ðŁĺ · +very day +tor ah +is sion +wa c +lever aging +ish able +cu se +le wood +may an +turn table +ju ice +tru sty +tu p +eti quette +supervis ors +stu n +gu zman +confe ren +ric o +fe ast +back ward +pol aris +mic he +jo g +h ing +field house +vel ing +sho cker +esc ence +ठ¾ +vi be +anasta sia +mar ched +kill ing +Ķ ë +fe tt +exop lan +... ( +snow day +lo h +ir ani +la khs +del a +po caly +boom ers +dictat orship +ac er +tur keys +quarter final +muskete ers +ðŁĴĽ ðŁĴļ +sf x +museum week +sc ala +ri sis +( ðŁĵ· +ãĢ Ĥ +z ies +bo eh +hu es +lu sci +dol a +impeach trump +roo d +don caster +tor re +hero es +fo yer +tar i +blur red +ke w +frank ly +dro id +ap al +Ð ¼ +y af +bre t +par agu +cac ao +ðŁĻĮ ðŁı¾ +ru e +head aches +shaw ty +char ley +pal er +go wns +correc tional +ðŁĺ© ðŁĺ© +breaking bad +ol ing +da p +endeav our +cit adel +tra d +incumb ent +medit ate +foo ted +ðŁĴ µ +shab bat +dayof the +wil lem +gal way +to red +marri age +f illion +sleeve less +aud itor +jin young +invin cible +kad una +a and +volcan oes +mon eti +indie gogo +buccane ers +ðŁijī ðŁı½ +ãĢ Ĥ +lay ton +cuck oo +hu mber +buzz er +Ï ī +to re +stra ins +sto m +pa ine +s we +du ff +z ou +si mi +li pp +ur n +se agu +ðŁĶ ® +sun dae +hi c +ðŁĺ ¨ +bull pen +u per +flyo ver +al dridge +glo bes +ali es +ken zie +ge es +y cle +sp lin +mag enta +j ha +bal u +gh orn +ti pper +wick er +taste of +con clave +ch ale +inv asi +cat er +dio xide +me gab +win n +at p +transform ative +nest led +hi g +bri dging +lil ies +chee red +bad dest +sc rolls +real is +dipl o +ðŁĶ « +conce ssion +prefe rences +explo des +er gon +introduc tory +ine au +ch af +som es +land rover +spir ation +sex y +sco recard +illustr ates +soul mate +wi en +inter disciplinary +fore casting +ent ities +glu ed +en lar +cur t +percep tions +boot leg +mi re +asho k +v az +hor ne +cal le +ac ulture +ther oy +night time +oc al +character design +ar mist +ðŁĺı ðŁĺı +yah oo +ac eae +to se +even to +sou t +nay anth +wh om +v are +ri gging +gen us +hi ve +com mands +sti e +day a +ethan ol +en f +hi fi +flu ence +cle mson +re invent +thermom eter +humor ous +emer ging +aci ón +ðŁĺĺ ðŁĺį +s ity +haw ke +accompan ying +t ility +ðŁĺ ª +re cess +protag onist +l ery +dun dal +int l +britt any +q bs +off the +marri ages +how to +viol ated +adel aide +wit t +lanc er +pak v +hu me +st ade +bra gging +ou tright +ad c +super st +real time +cu res +garden ers +ero ck +dale jr +ver o +bar tol +mo ti +mc fly +v pn +st ink +over rated +guer ra +e tis +ath ome +twd family +th ab +tn x +rafa el +family travel +x ley +sat anic +equ ations +ru dy +wal dorf +stan i +tu be +meas les +zimmer man +obli gations +i ously +bow ser +trans former +sho ppe +shak en +gh ouse +to d +ke tball +share holder +mar ca +kp mg +ak an +given chy +coast al +au th +roller coaster +mar ches +coordin ate +cine ma +apprentic es +par lor +mit o +men on +consider able +bar re +glo ss +enh ances +jaz eera +fal mouth +thra sh +stat en +k zn +eng el +samanth ap +flo ppy +sal om +ðŁıĨ ðŁıĨ +w ack +deliber ate +osc ill +herit ag +du sted +orni thology +pad dle +fer ns +bar un +cl ans +anticip ate +a ay +mat ically +é ĩ +tu mble +post man +unic ef +tro tter +op d +leaf let +ge ist +cease fire +scre ws +cre ation +wal nuts +longh orns +under statement +ab b +proxim ity +na x +un ity +turn pike +orda ined +dub step +chak ra +me ch +love her +look alike +donne in +vir on +Ù Ī +bang ers +vari ants +out dated +in ta +cri sto +sp elt +food and +f on +stefan i +margin al +hu tton +ti ara +tel ford +qu en +fair grounds +que tta +mikha il +heal er +v ball +ty re +under grad +gl end +hom ers +scri bed +main tains +po che +mis sal +mar ko +u as +á n +sh p +con vey +pad re +sab a +pu glia +madhu ri +pa xton +chap lain +n ago +ca si +... !!! +fli rt +sal eh +k are +di re +stam ped +extre me +ðŁĺĥ ðŁĺĥ +ho ppy +guadalu pe +advant aged +eu char +p low +un n +mac qu +port land +cla sh +pe s +lou bout +y p +keep ing +arca dia +fran kie +fi u +de th +encyclo pedia +si ze +inve sts +ðŁį © +geo logical +fran ç +con front +ðŁĺ ¥ +d ys +af m +tex an +graph ene +repost app +ac f +ur sula +gaz a +dd led +fu m +wsb tv +m be +fron tiers +chrono graph +ke s +inter faith +tab oo +spar ta +won do +flori st +em braces +ca w +no el +arch ers +ðŁIJ · +roman o +ban an +sh akers +melo dies +geo thermal +se phora +ìļ ° +оР´ +pro c +hand shake +pan de +popul ated +slow down +hor tons +registr ations +un deni +lan ts +pas sover +thak ur +li ef +adhe sive +pe tal +micro scopy +memph is +confir ming +air drop +mesm er +perce ived +ming le +lifel ine +gh j +worcester shire +pas sions +ach er +el lar +ah o +firen ze +bar ang +letter man +hat field +lu cha +je ter +e shop +william s +horo scope +pre de +east bourne +dur ga +di version +al trin +seis mic +premi osm +nar co +ti r +ori g +or m +land fall +ci ous +lin do +max ine +x ico +tra y +os wald +c ba +ric otta +n cr +mar au +ภ² +gladi ator +ch ery +lun g +u me +po psic +lon ging +can als +ta ya +decentr alized +sho pp +pres sures +mahar aj +eti had +wal greens +succe ssion +sign aling +li g +staf fer +north korea +def ying +as ma +de g +peri meter +oak ville +m sk +balti more +rece ip +de ple +ðŁĺŃ ðŁĺĤ +jambo ree +> .< +rsp b +puni sher +consider ably +in tothe +pari sian +acceler ated +polye ster +low es +fr ying +sauté ed +mou ths +seychel les +ra x +go dis +dak ota +house wives +the me +mat inee +black bird +ye sung +pre fers +pelle gr +in ated +trun ks +stronger together +re pet +re pairing +ped als +toler ant +her r +dun ne +indic ation +decat ur +b tv +exhibit ors +ik on +friday motivation +bra gg +live tweet +al ves +womens art +foreig ners +wal lets +min dy +lan ey +bb in +tv miaw +lif ter +tar get +tam e +dr ou +astro photography +mp c +g pu +nord strom +fric tion +run off +lov able +sp nfamily +ext ingui +bloo dy +sch el +arti stry +sw ish +scar ce +ph ils +max im +pos sum +com promised +sty li +sc fc +is sa +birmin gham +sket ched +angel ica +ordin ance +je ts +conqu er +ðŁĺ IJ +online shopping +s ori +reason ably +nue stro +ar turo +ch l +benef ici +spho to +wel t +ni kk +ðŁ¤ ŀ +dan ao +for mid +as se +af irst +âľ Ĥ +gil lette +as sor +an onym +sel ca +fe mi +bear able +y and +ar mory +cre pe +celtic fc +bra vo +in expensive +de lec +ge cko +new market +snow flakes +kab ir +con tra +can ning +mor pho +gar wal +ðŁĴĥ ðŁı» +fight ing +mu tation +woo dy +ju gg +gr aces +premiosm tvmiaw +kenne dy +gu p +sa e +op ha +off spring +fini sher +bet ts +span ning +mar j +h one +sh ing +contin ents +samanthap rabhu +un related +l acy +explo sions +benjam in +sophi e +no ting +micro soft +as sen +a hoy +i ker +ho fer +mo e +ah madi +yan n +an ak +ma hi +be u +aha h +creep er +baahu bali +am at +pri ory +haw keye +deloit te +sko da +print making +assemb ling +mirac ulous +no ch +sw o +leg a +oper ates +border lands +eli e +stron gh +rep tiles +pir ate +un fold + ¯ +qual comm +un predictable +ot r +rose wood +direc tional +counsel ors +corn ell +liber ated +j ad +ir regular +bulgar ian +high ness +vodaf one +sw ild +mini mize +gra zie +๠ĩ +r stats +stre ep +ome tric +humb le +lu mp +l ille +b ü +home depot +tripad visor +ki wan +a via +er z +ex ico +du f +blu men +mi zing +ar ma +in im +con stan +sor a +ju al +au n +tw ell +tren ches +her a +r k +po plar +recipe oftheday +ll an +bhu ban +short ages +ing don +bridge water +ðŁIJ ĺ +fortn ite +cam den +un cture +pro w +colon ies +t ks +n go +b hm +live pd +spl ace +sli ke +happye aster +ter rence +revol ver +j ed +yy yy +office of +m ts +exist ential +r ourke +explore bc +sse d +pri est +vix en +si ding +k pa +a har +ju ic +ob struc +foren sics +uk mfg +cancell ation +we ary +ab q +ele c +pri zed +deb ts +me zz +salv atore +m dc +gre tte +c gc +th on +snow storm +ts ch +cook ery +å ¹ +wa xing +n acional +mur s +ra ve +cap es +ger main +dri pping +sub mitting +ome lette +iter ation +aj es +shim mer +fu eling +ðŁĩ§ ðŁĩª +li po +bo bble +un follow +islam ist +hi ber +cat s +agentsof shield +sen si +____ _ +ster ia +inst al +ausp icious +har row +over land +femini sts +inst ant +char iot +blind ness +sp ed +sc arec +nu it +mini atures +ho seok +glo ck +fifa worldcup +e te +dis m +we iner +ex foli +ear ts +ภĶ +my art +man il +iss ant +form a +in cu +buffal ob +in tim +mc cul +anj ali +po po +un doub +hil a +fun gal +thank ful +fu tur +en dish +ren ds +th ar +she ff +ring o +nichol ls +io wa +po tom +cl ams +ãģ Ħ +acon f +stadi ums +di mp +di k +residen ces +do v +caric ature +seagu ll +kl m +confe ss +sla pped +cele b +turb ines +pp v +nur ture +el ab +.... .# +tu ff +de press +al far +amii bo +di spon +e wing +que er +friend s +for re +âĺ ¼ +sw t +aqu arius +head liner +cur d +fi gs +o tters +love fl +kare em +go vegan +fri yay +consol ation +at ri +ì§ Ħ +âĺĿ ï¸ı +poly ne +gu ed +o ya +la us +intestin al +cam illa +scal p +pi r +leed s +horri fying +bore tum +dand elion +fer rer +ell ic +as x +so ren +re loaded +ale ague +navig ator +ine tte +add ams +al chemist +ak shay +dystop ian +awe c +n aya +al isa +ai led +ag or +avi ator +ali zer +smo bile +findyour park +cop ying +to ddy +sh ti +mon ger +cal houn +nap kin +break up +y atra +se thu +ric hi +eras mus +fer ry +am ore +prac tise +bo bo +power point +oo se +li ffe +chin a +sh ka +fad navis +du ane +war on +fal se +ðŁļ Ĥ +wa shes +disc ip +==== ==== +g k +ab b +stub born +medi eval +p ci +ðŁį ª +maril yn +h yo +man di +cr i +prede cess +continu ation +om usic +s lat +wh al +mall ory +bon n +shen zhen +ca i +âĺ ĥ +sa fest +for wards +dra wers +bla sted +sle e +mor phe +mb ta +dumb ass +ÑĦоÑĤ о +alhamdulil lah +ec lub +al beit +heal ey +ayurve da +adverti sed +cro cs +itt les +bry son +be i +nj pw +honore e +fu sed +ðŁĶ ĺ +mul tin +n aga +de parts +ko p +kin o +jhar khand +ed na +ax le +mil ton +supremac ist +marrake ch +domin ic +tran script +] [# +: ). +wo c +sur rounds +o gil +leaf lets +co well +whe w +tru de +proli fer +succe s +sports man +con dom +po che +k up +imprison ment +{ } +scram bled +å Ľ +ka ine +cell phone +metam or +con i +remn ants +ee z +down pour +afterno on +exerc ising +ber ser +architec ture +wick low +m ns +is p +bo c +n iss +mn wild +stu mble +r si +lu ffy +sil en +dd ad +bul lies +haw ker +bb cc +scu ba +e pp +que ts +for aging +pal let +ha di +cinemato grapher +cat chers +to aster +k hi +lite coin +kid lit +amher st +maur icio +ip ad +mar malade +fe y +don nelly +g to +est as +cere bral +ant grasso +zz led +vir gil +swa pped +ðŁĺħ ðŁĺħ +no dapl +greate st +nhl bruins +fra ser +b mo +ane w +. âĿ¤ï¸ı +se gregation +remark ably +mccor mick +lo gger +er as +contrac ting +âłĢ âłĢ +yor ks +uku lele +touch screen +de cked +ben n +south wark +ra vin +nu mis +ðŁ¤ Ļ +ru t +gre co +eth ic +red neck +ar r +t cs +ih ri +ðŁĩ« ðŁĩ· +l k +inher ited +zy k +viadu ct +marty red +hi gu +ss n +be in +street style +fer gie +bank of +æĹ ¥ +stake holder +exempl ary +cre ss +ess a +ero tica +intre pid +gom es +bra un +bethan y +bang tan +pulmon ary +m illing +doctor ate +trump russia +ठ° +s ani +bl att +pla u +depri ved +t le +ful ly +bour n +st ak +lufthan sa +kio sk +far oo +def y +bad an +ðŁĺĺ âĿ¤ï¸ı +rit z +tri sha +ran ds +middle sex +arab s +pro j +sport scenter +repe ats +iv f +bleed blue +as sure +o bs +territ orial +ele n +bever ley +ann ah +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ı +z l +for good +science fiction +gla u +son ya +pri th +st weets +mix ers +mari o +ant elope +writing community +went z +den ham +be di +sf o +harley davidson +look book +immuno therapy +or phe +es ville +ed ged +tas k +sb ball +corro sion +kilom eters +co sting +play back +ke ke +di visi +u ter +re location +yel led +pen g +up beat +ser ve +âļ ł +hal en +stir ring +reh man +en v +schu macher +frag ment +alkal ine +sb k +resil i +share point +rol lover +tra sh +counter part +âĻ « +ob itu +à ½ +ãĤ ¹ +mul berry +ðŁİ Ĩ +auton omy +spra ying +nat l +love you +fran ki +nu k +esc ar +can teen +ali baba +de plor +mole cule +pu d +fort night +blon die +sp hin +portra yal +ta che +bu te +consi sting +freep alestine +c sp +im mort +d ns +ðŁĴ¥ ðŁĴ¥ +tour de +coo king +archi val +ga thers +bit t +b anc +pre mature +snow ball +poetry day +lou dly +fug itive +ed ay +em ra +ðŁĩ¸ ðŁĩª +sci en +node js +jur gen +je ong +band ana +un is +fox sports +v andy +pro visions +wee p +tu k +i ko +h oun +zig gy +z r +fil let +bat a +tin k +con e +we want +k ilo +hor ace +sl t +sc t +stay tuned +victor ia +umb ria +att acker +ingham shire +fright ening +no ir +fr at +con tempt +lia ison +ho i +br ink +tr ill +ni agar +kick ass +dun das +not my +rho de +bu mble +no xi +fa g +spec tators +mancrush monday +jin ping +distr act +dais y +wal den +portra it +ar thistory +vol tron +ev el +is c +ac m +r ite +na o +de ported +swe ats +ru fus +lo bo +labor day +gam o +ihri thik +bl it +abdomin al +ãħ¤ãħ¤ ãħ¤ãħ¤ +i it +e q +bu sy +allu arjun +un disclosed +de ton +pro create +ki l +ðŁİĤ ðŁİĤ +mitch ell +ki i +inherit ance +al p +jo burg +pat rolling +compul sory +un signed +ni am +l ga +eshop suk +tr illi +ma w +appreci ating +rock ab +mañ ana +an tal +mal vern +roy o +grand prix +sut ton +go ftheday +dig i +ãħĭãħĭ ãħĭãħĭ +t les +varan asi +erec ted +discip les +cont act +ðŁĺ µ +li d +⬠ĩ +scen tre +radi ator +ing tips +trans itions +thursday motivation +chem ical +separ ati +sal is +mi m +geo graphical +book fest +/ . +âľ ĭ +v ae +cur rie +ag garwal +acceler ation +the ses +lg m +u mass +pro portions +nat a +ani ans +ku ch +be acons +ap r +@ # +ðŁĴª ðŁı¾ +nu ke +sher aton +ki o +ma kati +polit ico +mor ale +ì Ļ +econom ically +gg ly +ss en +pa stries +intern ships +vic ente +fanta ken +aveng ers +accu se +slee pover +indic ated +the dream +ster one +ren ders +fro st +ou i +gre gg +d ore +⾨ ⾨⾨ +pu gs +sat y +nu mb +hems worth +tam i +la ssic +schi ff +igle sias +ag awa +] " +re shi +game stop +divor ced +theat er +clau di +un conventional +prophe ts +ac in +twel f +tow ering +t ml +sc lerosis +k wan +ge ts +distur b +na ira +ener g +pir acy +pru itt +noti fied +hen na +bra m +ground water +bl s +opti mis +$ ) +luci e +biz hour +fang irling +gr ills +or l +ver se +c ina +law less +artistson twitter +tele vised +marshmal lows +radio head +bar r +m fc +bre vi +mmor pg +g aya +âĸ « +sub titles +j t +disney land +to bago +nh m +groo ve +fi awec +" / +ba o +scra bble +om ni +ff l +um c +si mba +ali er +ter rell +plu me +mi di +dig nit +co c +bru t +ad ata +alche my +d sm +ðŁĺĨ ðŁĺĨ +win try +spa res +cu er +conclu sions +to ys +od or +fl ann +gar vey +scrip tions +inspec tions +cat ap +ang lo +st louis +heim er +at ay +tr ich +en yc +chil ds +vent il +mont p +guiller mo +circu lare +z ell +mode led +craf tsman +al ina +stimul ation +cashe w +ju das +best of +to ire +susp ends +scol lege +real ising +by tes +bloo ds +as si +ðŁĴ ¿ +o hs +ðŁį ĭ +scallo p +ठµ +gi fting +camo gie +wil kes +o zzy +ðŁ¤ ¤ +ver onic +sav oy +deme tri +baby girl +ðŁĺį ðŁĺŃ +so x +cly de +induc tee +count down +self care +ठľ +vi ka +tor re +phd chat +pe ars +aw h +suff rage +le sn +admir ation +mp p +shark week +schul z +santor ini +clo ver +( * +stras bourg +ex iting +so yu +finger print +che a +ãĢ ľ +vin dic +song writers +so a +prou der +nam a += )) +simple st +delici ously +gil les +u q +mn wx +ep p +sh un +ken nel +fall on +ðŁIJ £ +sin d +tra gically +out es +modern ism +co ke +gy n +spi on +âĺ¹ ï¸ı +le am +compress or +apolog ise +twent yon +fan atics +âĻ » +sco tsman +sa wa +ko u +as er +ภļ +welter weight +phen om +twick enham +stri a +p out +ka z +gi am +cd p +ho y +emplo y +red mond +ภĦภ+sm ere +trance family +proto cols +pie ce +lu iz +iter acy +carl s +united states +har med +phd life +ch aw +foot prints +l é +cho ker +z ana +sli pper +eric sson +insul ting +articho ke +advis ing +acquis itions +op or +mut ations +re ar +ॠģ +pod cast +wi ther +kun g +íĺ ¸ +win slow +di apers +ðŁĵ¸ @ +ec ker +col lar +hu ey +gi ro +mono gram +kas ich +si veness +malay si +arom atic +gre s +gali leo +u ji +rob b +dr m +none theless +as a +: > +lo a +l np +at work +ag t +laksh mi +pipel ines +id al +stre l +re all +chain z +stone wall +san sk +ðŁı ´ +pied mont +hoste ss +ci u +t é +analy ses +wil helm +scott y +rw by +mosqu it +use mb +qu ins +ðŁij İ +tu cker +s conf +speci fications +psychi atry +broo kes +s ils +ol af +de to +co di +cli p +fil th +womancrush wednesday +go to +ang erous +be ale +w tc +paneli st +ne x +lar sen +emili o +tab leau +h itters +conce ived +americ ani +or tega +mar di +Ñ ĥ +pain tball +thir sty +new yorker +etis ation +go ss +we aker +u gh +tro ll +har ga +du al +ght ning +at ine +ðŁĺİ ðŁĺİðŁĺİ +cook out +pyrene es +po ss +authent ication +sports wear +yun ho +kir o +archi pel +shen ko +ren der +nov ation +divin ity +ðŁij £ +su fi +humb ling +ge opol +devote es +wait ress +tr ough +py ro +i ba +bl ing +gra f +epilo ts +bt r +of tball +bas king +domin os +so om +r ath +sher yl +qu el +astronom ical +wel d +track list +sig nee +slee pless +com man +ch ron +summ on +pure michigan +cri spr +sli p +la gi +ra q +um u +thal ap +char med +scru mp +quad copter +ski p +peter sen +mun i +ðŁĮ ¾ +mon aghan +tra ys +ick ed +canad aday +te gr +ï¿ ½ +hot ness +heavy metal +ab ar +gop debate +az ul +spider man +sun flowers +ľ ë +web comics +bar d +Ð ² +nichol as +slu sh +ram an +mark ham +ffici al +ff ler +íĬ ¸ +ple ss +anush ka +to to +sk aters +pro wrestling +compet es +ay ala +myster y +thr ills +mp g +independ ently +y ul +imper ative +formid able +tire less +st acking +ton gues +mal tese +pot ts +mat ti +char ting +chill out +super nova +ome o +sky sports +nu tty +ðŁĹĵ ï¸ı +ro han +insp ired +concier ge +ser ra +ma kk +gal at +chi pp +ye v +ì £ +reim bur +op ul +kimber ley +i eee +bre men +ch itec +or in +nak u +bon kers +foo ty +emer gence +ðŁĨ ĺ +sti p +serge i +zo ey +ai me +wou ld +dy es +destin y +vinai grette +dri er +circulare conomy +an archi +ss r +sch el +cin er +gro om +determin ing +gar min +cal ais +incarcer ation +bu kit +no i +chelms ford +mckin ley +chi pped +belong ed +tu mors +str oud +mi i +influen za +wwen xt +tun dra +tele communications +cat sofinstagram +t ages +beat ty +o du +ml kday +oo per +dang le +ak ley +cru mb +anti gua +ti mbers +rou hani +ðŁĴª ðŁĴªðŁĴª +ha fi +... !! +w cs +coo p +sn c +lit res +ãĢ Ĭ +ha z +co z +k ant +green field +cur ti +y ale +flye agles +what soever +wor thing +rou lette +flyeagles fly +un da +a inted +stand ing +lusci ous +h pc +effic acy +ash land +me ghan +ky wx +n pr +bath tub +ac os +h ani +mar cor +man tis +da isi +bo ba +ab bie +mu til +vi al +spy der +po z +g ti +el fie +nigh tw +metro id +anton i +mad die +dh ry +dar lings +ten ds +taek wondo +atlan ta +me ow +chlo e +ãĥ İ +ym es +siber ia +k con +gu es +mar iner +fac il +azz le +[ ... +han nover +bav aria +vir go +te uk +u sps +) # +wall a +sam pson +need less +ver bally +hay ley +bow led +pi us +lam pard +ham string +vol vo +road safety +cho king +sor bet +a hem +healthy food +brai ded +horticul ture +cr ative +che ek +ad do +the force +ko ko +schiz oph +j ie +w ada +twentyon epilots +h bcu +pro ton +pau ls +lou isa +lat am +kyr gy +com pac +sd k +sap i +?? ? +liber alism +ep silon +ai den +w usa +spra yed +baske tball +kim ono +blue wave +ali as +ë§ Ī +mug shot +ce c +do gre +ad ora +ðŁĵ· @ +kra kow +intrigu ed +exhau sting +astron omer +ven ison +lady bug +ci v +bra e +us m +bri be +acup uncture +pembro ke +ke ating +chi e +y ad +t si +sm i +see ding +gate shead +lis boa +gy p +canv ass +ðŁĶ´ âļªï¸ı +op i +ni r +soci etal +ly te +ati es +c sm +ar tery +al in +aka poor +abstr acts +âĢ¦ âĢ¦ +teen wolf +ne we +travel gram +sentim ental +per ched +han del +ho ek +f ay +coordin ating +anim ate +man ian +effor t +jer ky +f ck +adri enne +ma bly +tra ding +my el +spi ro +sol a +stor ing +over drive +monday morning +dream team +pul se +bon di +ber nie +pgat our +tri poli +son am +plat t +âļ ¡ +ag roup +îIJ Ĵ +inv ading +v cu +k ell +ñ os +un dead +pod casting +mercede sam +mana fort +cor tex +que so +impecc able +pal mer +wil doz +sport sc +guacam ole +dispen ser +cate gori +stun ts +per il +invit ations +dune din +xi e +achi eves +saf er +pre ds +ph an +knuck les +k ak +igno res +lovemy job +aru ba +ound ation +datac enter +co vert +gr ing +cou ple +ا ر +vol i +mc cle +arti sans +lu do +kal am +arom a +under taker +hu la +wiz kid +gu mb +god frey +bakers field +ker n +engine er +car ve +pal in +guaran tees +pe bbles +b ays +zi eg +fin k +â¬ĩï¸ı â¬ĩï¸ı +down pours +ro chelle +rasp berry +ðŁĺ ® +gra phies +stom p +caf es +ari zed +utt ar +cal vary +dri e +crusad er +bus an +tux edo +si u +seam us +cul tured +blan chard +town house +ge red +butter milk +flu ctu +roger federer +hel i +ðŁ¦ ĥ +u ous +ram esh +mu ppets +email marketing +ye ss +br ice +ri zio +pel o +donnein arte +u rable +inve stin +bump ing +raji v +sav a +thro wer +fore x +o hhhh +th rust +pull man +r fid +sep sis +le ed +fri ght +roun ding +ne b +ph ins +ai sha +utili zing +squ ats +gold smith +j ic +bo ks +vau s +i po +exclu sion +tari ff +po kes +min al +land s +en force +washington dc +or char +g x +mar ys +ey our +aussi e +bak ers +un popular +latin os +lar ge +pu tnam +bol o +wa de +pel o +di zz +ob struction +fla ppy +weare the +depend ence +pajam a +e te +y ann +e wan +disc la +a ay +kar ina +e ic +an trim +w soc +neg atively +kai do +fotogra fia +dh ru +colo ssal +mcle od +k wang +mani pu +ex hilar +us atoday +summer slam +co les +tapro om +unbeat able +de ma +tic ks +k ling +fil s +campaig ners +ภķ +brew ster +audu bon +qu ay +ch s +ki gali +d ler +strength ens +som al +sign ingday +gol ds +pig ment +orche stral +g q +lin kin +ðŁı ĩ +ta w +algar ve +ho v +ear le +gold fish +am ig +ex er +ben in +dru id +ðŁIJ ¸ +she m +quat tro +mer cen +men te +incorpor ating +bon anza +state fair +en de +concep tions +e es +âĻ¥ï¸ı âĻ¥ï¸ı +d son +fire arm +orb ital +we h +multi p +fo b +requi em +p light +thou se +sa id +oc re +remem brance +n old +chi pping +be v +er t +ca thy +sy m +ri ggs +m ley +dialo gues +sl ender +how l +gau teng +wd w +to bi +smo kes +im plo +b pm +ad n +mom basa +cap sul +bloom field +artic ul +cle o +goog led +flu ffy +l ard +en zyme +ve sti +ibra hi +fl ame +e mea +out ages +dispro por +ble ak +an sel +ick er +st louis +stock market +good friday +sau lt +stal led +pro m +ep som +b é +the se +sau ces +me w +lit fest +pre d +re u +kar ak +si enna +ell in +bio technology +ï¸ıâĥ£ - +tac tic +sa in +por k +mon za +ka j +lu sh +compart ment +chang ing +shraddha kapoor +fo al +ar tem +cu ando +can ola +ori ente +me sse +d ited +br c +box er +bbc two +s st +ment day +em ing +de wey +kof i +âŀĸâŀĸ âŀĸâŀĸ +reali zation +smo l +tw ood +san je +flag staff +ber wick +cor set +can ary +whistle blower +et ched +com posing +squee zed +bow er +auto desk +ne h +mathi eu +ba ja +Å Ĥ +hy dra +da im +am eri +insi sted +mer lot +gar ros +heart news +gaine sville +cut ler +bo de +ðŁĺī ðŁĺī +lew es +scoun try +g sa +us u +cc m +god awgs +phara oh +cra e +mor ley +hyp noti +f ades +neur ons +fu zz +ing co +high landers +star k +vig ne +pac kets +amar illo +reu ben +insul ts +bas ic +vec tor +n me +ac ruz +tro s +transm itter +ðŁĺ ŀ +interpre t +ðŁĺ ² +pre quel +mc gowan +dis semin +ðŁĴĺ ðŁĴĺ +mascul inity +indie gamedev +ali ve +te t +pe tal +ema iled +ar med +ko o +he er +ba ird +super junior +metro polis +delav in +decl ines +stit utes +Û ģ +p tbo +g lan +cho res +e aling +chri ssy +ste mc +vi an +assassin ated +pron ounce +illeg als +discover y +cav ill +fri fotos +f al +so i +sabot age +t int +p dc +ðŁİīðŁİ Ī +ãĤ Ĭãģ +ji o +endeav or +in sig +commit tees +she arer +me tz +mar rying +h dd +g by +fre t +tri sh +pu l +scrip ted +sa ki +l w +ke ye +shim i +nan aimo +ca h +à « +tem pered +ici an +du gg +dish washer +air field +s rugby +gr inch +y st +r ms +mahat ma +lan kan +disc ar +dige stion +no des +l ls +om ic +gu tter +tis garh +feder ico +election day +bo he +master card +fire ball +âľ Ķï¸ı +oy ster +p ong +do k +en route +m vc +beat the +ali stair +shu b +sh aming +cherno byl +ghi bli +the s +pin ion +d bs +sal ts +ic tion +epi ph +nc pol +in convenience +whit ley +inspec ting +wood ley +wi ener +skil let +no les +m ca +h ina +a sha +willing ness +well ness +tam ed +show time +dis advantaged +ber nat +us n +mission aries +coun selling +arrog ant +quant itative +leg alization +ho dge +energye fficiency +cameron dallas +pos sessions +p bb +harris burg +v g +hindu ism +happy thanksgiving +fi b +re acting +tweeta picture +pol iti +mu ppet +hur rah +pac e +coast guard +guar ded +as am +par ry +fore very +x q +oom f +ke anu +j ind +ri st +customer service +sac red +ðŁĺ º +ton er +occur rence +mat u +val dez +red d +is ak +power rangers +pe asant +raj ini +abra ham +e mil +car do +tr il +hair styles +obsole te +sam pler +direc tive +delavin kisses +ver ton +glo s +sp ay +paler mo +com ets +man ziel +chicag of +ski pped +pic torial +h ant +b mi +a ol +re opens +pad dling +devo s +fra ud +bas eline +que ues +sp ired +sn are +eu ve +descri ptions +daisi es +ca ching +gall eria +tri mmed +stin o +recy cla +ic ular +bir ken +raw lings +fli x +chic as +b gt +lik eli +argy ll +thel ove +ga ston +bl anca +ha k +f one +sailor moon +h aci +ima c +fl yn +de can +bel les +ap ic +zo g +taun ton +con stance +lasag na +ker nel +in ka +har bor +collec tively +calcul ated +av ille +shil pa +pur du +gi mm +fun er +a est +pembroke shire +nighting ale +n unes +hyper tension +hu bert +sli ders +infer tility +comm ended +transat lantic +metr ical +!! @ +Å Ł +ss g +bac ca +inver ted +fun factfriday +it ans +albu m +acqu ainted +ri er +whel an +sar ab +mu e +snoo ze +pi ff +agre eing +sp itting +jer maine +n ye +âľı ï¸ı +am bush +ze ph +con greg +univers ity +s app +wann abe +pat rice +ib d +do glo +fri dges +sun d +king ston +ar gon +kam en +hardro ck +ds ley +do lores +ì ° +ota ku +pi ping +be having +âŃIJï¸ıâŃIJï¸ı âŃIJï¸ı +blue bird +an sari +teapo t +fire work +cro p +log ans +ty ped +thick ness +ig ers +c fp +dys functional +contra sting +et ty +aston martin +tx st +dra grace +at tributes +marath on +manu scripts +john stone +ðŁĺ± ðŁĺ± +bo er +ay u +aru gula +poo rest +con du +assu mption +anag h +no h +delav in +sit ter +g ö +mor ow +kick start +com i +gl acial +ghe ad +ba in +ker shaw +en dof +fre ud +om at +i af +hu g +sign up +each other +defin ite +tu bing +shak ira +ðŁijı ðŁı½ +uu uu +sw in +sham bles +ol as +sk ell +brit ain +kn w +clu tter +om y +j ens +hang ed +city scape +scra ps +un locking +dead liest +er no +breast cancer +a it +inspec t +fu ri +ðŁĴ Į +ku d +ju le +or ah +mi ds +m dt +bur gring +r attle +pu sa +stal k +cle ans +iss ance +z ek +worth it +nam eis +musko ka +council man +urban art +bar rac +un solved +tu l +g ita +white board +soy beans +em ent +cont i +saturday motivation +conveni ently +doc king +t ado +âı © +sp ino +puppy love +po f +fabric ated +robb ers +adop ts +ti fied +kk r +indulg ence +notic eable +macqu arie +chap el +sensu al +ki ko +melan oma +lore tta +li ance +ab en +sp lus +ga al +ac ele +lib dems +compar isons +ðŁĮ µ +rhy thms +mer y +en capsul +nap ier +ðŁijĮ ðŁijĮðŁijĮ +ðŁij IJ +plat z +fre sno +re formed +ran bir +el it +the best +bhu shan +vin nie +impro vised +s ittin +re created +e ba +ec ker +ac rob +pon te +cor d +gi ddy +eur usd +fe ver +intu ition +gar i +dum mies +bud weiser +amend ments +te tra +sch nit +ay as +mar ys +ci st +k ani +ker mit +ðŁĺ±ðŁĺ± ðŁĺ± +tin ker +strol ling +di visional +niger i +omin ous +menstru al +kar ab +k hy +bw fc +pan handle +l illi +well er +stra pped +son the +transfer ring +ethe real +sne aks +ru dol +gab les +jac king +cin code +for tune +canadi ens +con for +ab normal +frank lin +tit a +mu la +persi st +cu ties +ki el +ðŁĩ± ðŁĩ +her mann +aw k +fi asco +ko to +we ta +hi ker +budd y +preven tive +mcgra w +game boy +forsy th +top shop +si ob +sad h +in tram +follow art +so aps +dragon ball +ou x +morri son +๠ĥ +lu bric +adul thood +morri sons +âļ łï¸ı +her mo +ta ka +stall one +mis use +team gb +ra gha +con fined +at y +hom ophobic +nw o +sky news +ho ya +ac rosse +wi iu +pur ée +jed dah +ðŁ¤ § +advis ers +ph ine +an is +scrump tious +ë° ķ +c ke +vin y +ter m +s dc +o do +home school +vas c +leop ards +debor ah +illic it +cur ran +as roma +nau ght +mar ig +brand i +em p +ðŁĺį ðŁijĮ +î Į +su spend +lu z +initi ation +sch aft +jensen ackles +craw ler +post doc +des ks +trail blazer +den omin +tri x +no ise +po et +± ï¸ı +s mug +vol atile +proof s +pharmac ist +sardin ia +mash able +kim chi +co ed +schal ke +doo dled +c sw +sh ur +ro x +do k +chris brown +mathemat ician +ab ound +ang elic +rock ford +d ole +yor kers +ms n +g man +xavi er +bor rowing +mark ings +longh orn +k ja +diver ted +mm it +euph oria +ay yy +te a +pa h +ck i +un cut +li ven +ky ung +fan art +mer ing +red ding +amo vie +gri di +c thulhu +schol arly +ju dah +th bewithyou +eu calyp +ðŁIJ ķ +hert fordshire +cour troom +by u +auc tioned +ple ase +mar cia +ê° ĵ +succe eded +el as +arvin d +t lot +saig on +re tt +ra kesh +fd ny +as en +se bring +gladi ators +you know +v lad +gol a +par ap +ÑĢ и +sab cnews +one team +oh l +sun e +ri j +cd c +star gate +run down +plat o +ph c +chat ter +ra viol +mn f +mand ala +li et +ภķ +mari a +hun gover +consoli dation +fer rell +tradition al +ilove art +gal ap +ðŁı Į +que zon +espa ña +ðŁĩ¨ðŁĩ Ń +ho bby +steam boat +mali gn +guil lau +pro hi +its me +íĥ Ģ +in scription +al z +mari an +k ade +mm on +adju sting +ne sts +intern ally +ci r +vik ram +mal ala +k ph +fel icia +the real +cap tivity +at is +marcor ubio +kale ido +che v +mano j +le more +gent ri +vi ps +tro pe +" âĢĶ +pair ings +mal nutrition +fr ay +desig nation +brun omars +az e +tor rential +pan zer +ga il +under the +the ological +schizoph re +dazz le +freder ic +mo par +ad illa +so ggy +ra un +medi ocre +colo rec +i fe +p inst +blu ef + ² +world water +gir oud +clar inet +ad olf +tar antino +receip ts +assu mp +ðŁij Ł +coffe es +âľĬ ðŁı¾ +du plex +s of +r x +lin o +timber wolves +pan dit +mo tm +e ga +ay ama +ach s +outsi der +ll en +co er +til ly +cheese burger +ma ds +ple dis +emp ty +national parks +az iz +p mi +jun kies +f ener +sq n +è s +gener ation +cleop atra +bhuban es +mosqu es +ty free +popp ins +tw c +or well +n age +ka whi +hol low +dal ai +¨¨ ¨¨ +ou ro +m health +gi on +az o +vis as +reneg ade +re ic +w sop +ðŁĴļ ðŁĴĽ +e chel +tox icity +mü n +bun k +stimul ating +asth our +\ ' +ep h +ende mic +cn bc +shrin king +peabo dy +michel angelo +can yon +wal e +su mi +si ders +inu it +? . +profession alism +dr acing +plat oon +p ons +out bound +maple leafs +de sol +cen cy +a than +ver ma +ru bbing +ok an +ðŁij ł +mull ins +authent ic +Å į +alman ac +ga ia +bb q +on imo +ke h +ty a +tou ts +y av +re posit +, . +wi ght +se eyou +cal lof +done sia +bar gaining +gr anth +sd su +amphi theater +p su +re watching +wine tasting +peak district +dete cting +thur man +phe e +èª ķ +u mich +re r +sculp ted +go le +name sake +ðŁĶ ģ +serv icing +bau gh +pu gh +pen cil +dar th +munch kin +at orium +ten ers +sun y +rolling stones +mag ing +star rer +i dris +fe instein +ag ron +âĺºï¸ı âĺºï¸ı +supervis ed +chamele on +aggre gate +succe ssive +mo gul +inst yle +pol dark +custom e +ohio state +ha ya +ci des +broker age +angel ou +fifa wwc +de forestation +al ton +pam ph +hu gged +ho bo +change able +ku ber +bur roughs +demon etisation +cape cod +vers atility +or ice +le ila +womenin science +tu a +he dges +embarrass ment +ali fe +so ars +ni ghter +hy mn +gi pp +chas u +tech s +ni all +k illa +hi ka +cam els +valu e + ¢ +sc oops +mah moud +clu sive +adri ana +pac o +oz il +un as +transl ations +whispe rer +s bi +bu xton +bio tics +indi ffe +ken ney +k lar +et ching +barra best +inst ability +se ine +vo tel +blo gged +whis key +my space +t ant +lan dia +give back +illu s +aw ak +ac ab +f bloggers +cloud computing +blat ant +syri ans +band ra +sty n +an em +ke ted +kar thik +barun sob +pin ot +gu bernat +gay e +arti ste +i fied +conven tions +hu an +geni uses +eeee ee +fol ly +somer ville +pride month +ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ +chemo therapy +paul s +bak ar +ìĦ¸ë¸ IJ +taiwan ese +fol lo +c ss +re ign +nn nn +fla un +catastro phe +iti es +frag ments +extre mists +ym oun +car men +eze kiel +conne cting +se h +man ta +remodel ing +we ymouth +at oms +ce m +ne well +lu mi +the open +mo c +mili band +g land +z shq +mag gie +mani acs +m sp +ad y +cre ams +le anne +e sta +py g +af finity +pray er +dun bar +ligh troom +ac adi +wyn onna +roman tic +state dept +sick le +wh os +lam o +et our +fin ity +shru b +shar pen +pun dit +ed on +af ore +mar s +jeff ery +ter ps +medal list +kath arine +accu sing +ta z +roy d +from home +confron tation +alle gh +ðŁijī ðŁijī +refresh er +ran veer +never land +jo jo +lu crative +en am +ca ver +pa edi +man jaro +flu ids +the ssal +oppre ssed +mu ss +joh anna +Ø ® +cn g +buil dthe +sett les +s ith +fu ego +cl amp +ar ag +pay er +ted x +mand y +inter stellar +fr c +ch and +b cc +mo lo +len til +johan sson +grims by +nature lovers +ðŁļ¨ ðŁļ¨ðŁļ¨ +shin de +x in +international dayof +transiti onal +sat a +cad dy +wo d +if u +ha ys +holl yo +j ang +ir c +co im +grad able +" " +ðŁį ´ +ঠ¾ +a el +n yo +west lake +time out +sof i +phenom ena +cultiv ation +ag no +un armed +so t +con j +gen o +royal navy +nutriti on +fair mont +ti relessly +sn g +re ty +mic a +lu cent +slo ane +droo l +riz al +od ell +critici zed +. '" +la ze +deser ted +co der +pra s +l illian +itiner ary +dav y +an ap +whi pping +hobo ken +kare ena +çľ Ł +vi us +ter n +nan tucket +mis understood +bu laga +st ant +chin ook +z am +reli es +d ss +ed mond +sket chy +m ell +fe x +rec tor +dist ill +day dream +wine maker +ri pley +billion aires +hel ene +ati f +cul prit +bertr and +wou ldnt +ma pped +v ak +gla dly +parliam ent +kidlit art +ware ness +goli ath +âĨ ĵ +view point +tat ted +fu ls +dor sey +ang lers +li ds +ki ya +bow les +be h +b ite +compati bility +ance stral +pro x +beha ved +gubernat orial +ch field +sab an +z h +teen y +shibu ya +holli day +pan cy +âĿĦï¸ı âĿĦï¸ı +seun gri +? , +ðŁĩ¦ ðŁĩ· +im itation +impac tful +any i +gene vie +añ os +bate man +gli der +af ar +ra sheed +effor tless +sh war +dach sh +er un +at os +kin i +ch d +kha ki +k lin +felici dades +bel o +as l +to ppers +fin ley +stac ey +rigor ous +kar ting +le ppard +car michael +be ret +c se +ak hi +mer ingue +ab an +ha ke +ger i +er jee +re sto +comm anders +pr it +fl or +ad ven +ex termin +remain der +å IJ +es g +martin o +lulla by +| @ +mi gn +in store +big bang +cor di +cau ley +ante bellum +dg ate +cro ck +span dex +scaf folding +ore os +ê°ĵ ìĦ¸ë¸IJ +pom ona +ma uro +uni versi +re mi +af ootball +t ant +sm alls +ne h +worl do +tropic al +mor ph +jav elin +gla r +arqu itec +reminis cent +tu bs +spide y +make u +syl la +progressi ves +blo t +shor ten +keep in +ch ak +ang st +super food +decad ent +ston y +neuro logical +ar boretum +ann ak +fe ma +per cu +dis respectful +small biz +lo x +co om +c sc +bs bi +pre valence +him ss +esp an +mo ga +fr ampton +sky map +mas se +levi athan +( ). +noctur nal +car ameli +ang or +amne sia +outsi ders +she alth +rhin o +ant ag +ag io +ðŁĴ° ðŁĴ° +take me +kab addi +c si +m sh +coch rane +thessal oni +sil a +ha us +du sting +obe se +mack lemore +mani sh +len in +m dc +gro wn +shef field +s rs +ke le +car son +ch um +dah lia +can tore +opp o +how ling +cyber crime +sur realism +sc ran +fa iz +thre n +rac ists +r out +pk not +se mana +sin i +mc cull +ma chi +alfon so +y b +sar dar +kend rick +den g +reci pro +on f +doom sday +bri bery +custom iz +art is +c pi +ðŁĻĪ ðŁĻĪ +sla va +let te +en s +âĿ¤ï¸ı ðŁĺĺ +cra yon +ad an +tr c +migr ate +simp son +row ers +king sley +farmers market +shee han +ne phe +bor non +car ton +mic key +all ure +u lu +sli pknot +heb do +gui do +dog celebration +online marketing +acceler ating +) .. +origin ated +macar oni +ed tech +out field +mit z +disc us +adverti ser +man or +ha shi +descri p +cap ita +ful bright +recep tor +con n +con ey +spion age +r attle +pre st +u li +blog post +acker ay +) âĢ¦ +red velvet +mat th +inspir ing +b sd +ker ri +po con +mil lar +re pur +accent ure +ä ¹ +ram bo +ragnar ok +dele ting +british museum +pat ory +leip zig +flori an +sci fi +in ers +br ate +yo y +melis sa +ab er +ma sa +po te +mosquit oes +transpl ant +r pa +; )) +bast ille +yl an +joye ux +melo dic +cap tions +atri st +roch dale +gott i +pew die +cuties aturday +who is +aqu aculture +tiv a +sp el +he ss +ha ji +fred die +co per +brand o +v k +photo book +* , +my dayin +micha ela +brune i +sr ini +in te +Ä ± +de ol +d fc +separ ately +bun d +ve sts +to c +me ck +rein forced +constra ints +car roll +sq ft +re ver +cam per +bird man +in action +gener ators +triumph ant +pe sts +o vo +gy pt +al amo +sc aled +suresh pp +sd n +is mo +gi os +) @ +justic eleague +restaur ant +gab i +den gue +next gen +exemp li +ap ex +inspir ational +down side +kid z +u pl +et na +alvar o +fel dman +bar net +m ha +es ch +bloo ded +>>>> >>>> +kan i +ho fficial +casablanc a +bir ds +ty ga +sw amp +o day +new castle +nb ap +ci sion +cho ols +af lo +ne p +mon ton +ak b +super model +down time +th os +sc wx +snoo py +ag greg +yo ke +nor cal +we tt +prolon ged +me tast +beat er +f ta +t lap +disgu sted +y h +voice over +itch y +ip c +ðŁİ ¾ +phe asant +stra its +ram pant +j g +fer til +assu res +fortun es +sal inas +liz ards +kett le +i bs +cyn thi +he g +mc cr +soccer oos +happen ings +cor den +ðŁĺĤ ðŁijĮ +t ches +egre t +wolver ines +congratul ated +ho gg +bott ling +wr i +fer ri +bo sch +af ire +og den +s jo +j dm +sv t +con tex +tol lywood +min k +me se +super sonic +op oulos +å ¸ +âĶ ģ +knuck le +gu ise +gam i +chu cky +z inger +radi al +compla ined +bo da +fe tal +discipl ines +cor ro +ðŁĩ®ðŁĩ ¹ +op ted +filtr ation +ad nan +em cee +mi stre +insom ni +fer gus +tra jec +on don +med tech +tanger ine +madra s +gru e +cab s +z hu +sureshpp rabhu +insul ated +day swild +pp m +band ai +v day +s ff +squ id +lo thing +not dead +expre ssive +cu ll +ala stair +x u +up front +fish ers +en es +um d +dis missal +sti er +sel s +lu st +re active +prote ster +eyel ashes +al im +goo de +gre eng +da ir +com pen +anush ka +proto typing +ma pu +bear ings +ðŁIJ Ł +for me +bsbi botany +timo thy +out skirts +am bed +are tha +wend ell +stre aks +ni m +k pk +sne e +fit ter +quo ta +p ate +win ning +ðŁį Ń +sho pping +ma inst +cul ver +ste vie +mcfad den +counter parts +gren fell +fol som +dor set +tech crunch +⬠ħï¸ı +tip tuesday +us l +tre x +geor gie +ranveer official +lic ks +se wn +k f +' âĢ¦ +jap s +p ate +orth op +fe sta +stra s +mon tal +hammer smith +fore most +wido ws +mad re +ite z +mito chondri +lig ans +z ona +cari bou +m ss +andre i +weather channel +gh c +: ... +ta ft +awe ather +al isation +bru tal +bliss ful +nik ola +mal icious +q m +mpg vip +bro die +bl itz +applau d +dri bb +v ague +dog go +transl ating +interpre ted +hat ched +ge tyour +benefici aries +spar ring +caes ars +aw illiams +la hat +bro ke +ti mp +virtu es +rel ying +pie tro +k tn +ici sts +pab lo +lou i +a ag +pn pp +cha st +pul ses +fini sh +usair force +type writer +thomp son +dog s +ut to +ãģ į +sand al +new ly +do ge +z w +wan kers +ne gr +mu cha +determin es +black fish +sk unk +mu ps +instru ment +phy to +daysto go +skin ned +hai der +con ten +ðŁIJ¾ ðŁIJ¾ +we iler +undoub tedly +chair ing +wall is +sh ard +zind abad +adul t +absor ption +pre sto +deplo ying +drum mond +battle front +seag ulls +how dy +juda ism +des de +part ition +âľ Ŀ +no logy +national bestfriend +lesn ar +film fare +co asts +christen sen +ac an +mb u +co pped +ru bble +sw c +fun nier +far ther +where as +nano technology +with stand +pil low +bow ers +to pe +it ly +con fit +ma kar +comfor ts +bo sh +cli pper +bal la +sti k +mil b +safe guard +musi que +eas port +ya z +pad ded +bad er +fore ign +chop in +archi ve +o ka +tran sporting +tml talk +aj it +consequ ence +sc roo +ff o +collabor ated +pug chat +ye mi +jav ed +au burn +o of +ma w +sau cer +miti gate +i les +evangeli st +ter ie +re cl +indic tment +cat a +bright ness +may the +whim sical +un lv +key word +cu min +med way +west world +tra w +im posing +form ity +coul ter +ab z +ny pd +grass i +kel sey +qld pol +clock work +f dr +di anne +âĺ ij +ad h +p ann +bra vely +ae ge +un lawful +ver di +pocaly pse +phar o +kar la +reson ance +ma stiff +la dak +bu u +ma iled +hi i +craw ley +tor rent +mach ado +liby an +effort lessly +fal sely +q vist +ke ef +craf thour +cheri shed +val kyrie +s ari +kal amaz +be he +ðŁĮ Ļ +th im +ro ddy +col trane +but chers +ach im +wk end +awk ward +cab rera +:) ))) +fran c +decl an +con dos +a ja +pandor amusic +char ter +ph ill +mon trose +hatch back +handic app +gre aves +eucalyp tus +ut most +t son +bur ton +mid wives +in cur +ðŁĺį # +moo d +compre ssed +tom a +must ang +mo g +as ana +te stic +sho tel +in sol +cor sair +nh q +ben ny +sm ma +kap ur +in con +jon as +ener gies +don al +as ad +se z +n pa +archi ved +stimul ate +do p +hy d +gri eving +ãĥ Ī +ron a +why te +tree house +ss ell +sand ro +ko bo +ther most +se clu +hi ya +ge ez +mam as +prisc illa +flav oured +fas s +w old +maker space +cospla y +p tv +happy valentinesday +sequo ia +love craft +gu an +d tm +ci i +yoko hama +pos thum +re q +ðŁĶµ âļªï¸ı +galat asar +dol by +hamp tons +disturb ance +stone henge +ok c +disrup ting +month sary +jun gle +head lights +du stin +micro sof +happy mothersday +ko ko +gra zi +te sto +na idu +mal ay +ari al +ru mb +ab oo +har man +tra pe +spo ils +je ho +go dly +lock screen +z un +pi ous +ma gento +l enders +prob able +corpor al +m our +aw al +su a +call me +ton ne +go vin +devast ation +x j +gear box +war lock +per me +it ate +gaza underattack +du val +paras ite +clement e +le th +i va +fro zen +tho les +to bin +cair n +s ill +luc kiest +conver ts +st ale +pan cra +euro pale +wis dom +sch ur +ì ¶ +verti go +bi j +u bc +nu re +righte ousness +mt c +factor y +ver st +revers ed +hur i +hee chul +fab er +ar r +ul ous +ven om +ph at +green ery +bra dy +à ¦ +: (( +never giveup +di sha +mo ta +health care +dun ham +dex po +den zel +bb ins +f ics +wh am +mc g +eli an +wat a +str alia +tel lu +pe sky +spin off +ar moured +re acted +do fficial +te du +sag ar +mor ally +paralle led +fi os +dow ner +dau gh +re do +world cup +tari q +bar ne +glaci ers +oc cult +barbar ian +her mosa +!! !) +y ur +inter nation +p ss +sit u +p int +american air +sw am +dopp ler +ðŁĴĻ ðŁĴľ +cincode mayo +le van +hell enic +mc ne +ju di +yu h +st x +qu are +ðŁĺĤ . +sti g +g els +mot ley +hard work +euro zone +e ad +ç¥ Ń +seab ir +ci us +la id +alpac a +presu mably +pewdie pie +boo ted +am ari +tam ine +sol ace +bar row +acade mies +x ian +om ination +dun geons +b ma +de ity +ai k +stab il +hir a +affection ate +ving ne +new port +ãħĭ ãħĭ +thir ds +re tains +aroma therapy +ski er +ni ma +do pe +cr inge +con domin +to or +anim ator +sar aj +seas cape +minim alism +lake shore +calla way +berg man +à¤ Ĺ +whisp ering +stupi d +ri ghtful +requ is +ir n +se va +ut pol +tuber culo +squ ish +de but +govern mental +christ ine +all man +weap on +s ito +bur i +lo lita +leaf y +fu ch +tin ted +mck en +a hahaha +ðŁĩµðŁĩ ¹ +repe al +ne gan +ðŁķ Ĭ +tail gating +game insight +ðŁıŁ ï¸ı +yaku za +z t +ti ring +pro posing +bow lers +tra itors +ak shi +cler gy +cit o +up sets +tu scal +symph onic +sil ently +shu ff +black well +ðŁĺĤ ) +ko be +rober to +ri dg +dc u +mer ino +ft p +east side +. ~ +nb l +mn leg +ts for +frau dul +ca pping +in my +gymna st +ston es +ss in +twe aks +shag gy +oak land +dem sin +sang ria +mm va +hen nessy +down ton +ri ghtly +in it +aga ve +ob last +northe ast +friend ship +dal a +tro phy +ðŁij ½ +mag in +margar itas +ê · +ww fc +fa sh +di ke +cu d +char t +ðŁij ® +refuge es +jop lin +n cs +imp y +firm ware +pas cu +flam in +health tech +bell letstalk +w aka +ol ls +la go +co wan +bombar dier +sh ome +ðŁĻ ħ +mc master +na ve +well s +u ta +tell ers +mis fits +kap il +face off +af firm +a pro +whit epaper +super yacht +speci mens +al located +... , +- __ +ka w +dachsh und +djo ker +s work +qui ere +or um +ðŁIJ ł +som m +c mt +ingh our +skin ny +lgb ti +gi ggles +break away +resear ched +par ity +my al +ms l +re tained +si vity +make inindia +sol ves +defam ation +wal tham +sri racha +road way +concep tu +al in +iw ant +å Ī +del ft +tender loin +ga ins +faul ts +sw ire +st ellen +pol lo +dy ne +bornon thisday +asdf ghj +sq l +sali m +advis es +vo ip +ìĹij ìĨ +un touched +she il +ontari o +uph ill +so bre +de shi +nov ella +du tton +craw fish +ا٠Ĩ +ma a +tw ine +kal in +ðŁĩµðŁĩ Ń +ye ss +brook s +hoo siers +ton ka +umbrel las +ay ers +ate am +acqu iring +su ction +ä n +wi es +tari ans +soci o +mat tb +shepher ds +o so +charity tuesday +s logans +ninj as +al bat +by te +bash ir +trampol ine +mydayin la +i ja +bas el +ror y +gol die +fi rec +un noticed +pecu liar +sch a +ker son +mour ns +liquid ity +qu ipment +hi bs +ar s +aeron au +slide show +sla bs +delici ousness +sk itchen +hta fc +full erton +cre ighton +aer ob +procrastin ation +az ores +white hall +uss occer +medi ation +djoker nole +and me +um en +noxi ous +jo ss +ili fe +anni vers +sudan ese +et res +under mine +whole foods +diso be +kor i +ade le +eli z +can ti +al on +gymna sium +sarko die +meteoro logist +yl de +ste en +stamp collecting +nas al +lo tt +fran ks +ex ol +ack i +good year +animal rights +y les +vio lets +mm es +s thel +ra pping +tu scan +wai ver +tur ner +eat local +northe asthour +anim ations +tom morow +t sh +ff ame +bra e +pe tron +glam our +br yn +d cs +bal es +ðŁĶ ¶ +bro v +bre v +b ons +physi que +car ne +x e +elix ir +vol ved +l oma +ìľ ł +æ ĺ +van u +ri gs +bal ance +va res +bon ita +sprink le +perfec to +di on +le ak +calcu tta +o ba +d ma +c mon +tun er +pneu monia +bo gus +apolo ge +cl ough +bor ne +)) )) +revi ved +o varian +ner f +c legg +fan fest +cho u +reali zes +mc n +li gu +leg alize +just saying +for ster +bo sni +k hi +in dom +hei del +en cryp +si ss +ed di +mar bles +brisban e +y ing +pre paid +wal sall +cooper ate +orche str +mar isa +ho wie +che wy +bren ner +andro meda +e gan +sto cki +cav endish +ag an +ban o +de ir +go g +bl k +re thinking +ch ig +rhe u +sni p +p eng +semin ole +m swx +an nex +lyn da +lewisham ilton +cu mul +tb l +dolph in +agu ero +........ .... +pre lude +at our +gr anger +too ting +ro tun +dis ar +home items +da res +**** **** +ðŁij Ĩ +compre h +jin x +as well +iri e +circul ating +ðŁIJ ¥ +over board +cultiv ate +rhe tt +oriente ering +ca k +bal kans +s itt +jas min +britney spears +ro tor +se aling +g bc +oc ci +f as +eman cip +com er +war time +tic kle +son ny +pac es +log g +at rix +sr p +g win +do bbs +uz be +the wanted +dru sh +ex tru +m icky +honore es +dar win +re dux +mm j +ram i +jalape ño +io c +do ver +ju ju +whit ney +s eng +en ly +au ch +archipel ago +vigil ant +man gal +wil dest +parano id +hal i +bb ly +sanc tioned +real ms +con co +u ddin +c sk +play time +libr a +sav ag +oc tane +rec tan +re turn +par rish +mor rha +cc p +c mu +sa iled +se vent +ro sie +pil ing +he w +boar ded +seg ments +neph ro +( . +cr ats +bak es +ðŁį ¸ +back tothe +sibl ing +kirk land +ke o +gu wa +bre ads +ðŁĺľ ðŁĺľ +t q +haras sed +ga u +wil bur +j isoo +ep er +li sam +tri ppin +sh ino +ru kh +beast mode +cho a +inst aweather +rich land +gar i +fe z +cowboy snation +fur suit +k run +a en +sycam ore +se gun +ent ennial +di h +o ax +demsin philly +ðŁĻ Ģ +sn hl +pen nies +pass words +ma kin +ty e +d eng +kni gh +jeep life +hel pline +a for +zz zz +ste amy +pic ker +iter ate +happen ingnow +ki b +bloom berg +martyr dom +bul ly +assor tment +a hora +zo e +no i +illu stri +agar wal +p sc +electr onica +recruit er +gar diner +rad ha +naf ta +dot net +pi ero +geor g +bel s +ðŁĺĤ ðŁĺį +tuberculo sis +run nin +mor is +haul ing +ev oc +bre thren +sha ir +frame works +a stu +ri gid +ku ma +kre me +jin nah +insu rers +ny u +f ere +nol lywood +good vibes +- ... +toi le +sk ril +instaweather pro +cze ch +pa vel +one piece +nike plus +fi let +cav ity +ðŁı½ âĢįâĻĤï¸ı +ðŁİ £ +dra stic +dail ys +siam ese +re bu +oste o +lar k +f re +sh elling +p é +glad ys +ðŁıĢ ðŁıĢ +gusta ve +submer ged +grand stand +att u +won t +f pv +b ley +jon i +ang ames +weigh ted +al ou +ठ¶ +les bians +f j +anni es +am l +dor ia +dav in +be ta +can c +madewith unity +ha j +bad lands +mu l +blu ec +pa wn +cov ington +neuro logy +htt weets +dysle xia +thel ove +ne at +fork lift +autom ate +une ven +monte ss +he in +ha g +rel ics +competiti veness +can elo +mar tens +bullet proof +sk ittles +g ya +pri mo +americ afirst +woo o +abor tions +?? !! +ma che +ld ers +rl ly +preli ms +direc t +cour se +swa in +super cell +ec centric +sting ray +ple ts +wil cox +west in +okan agan +kir an +car bo +bomb ings +ra rest +bo h +gaw d +di gg +mo ana +enti rety +en closed +dodge ball +par ton +milky way +at r +thorough bred +re ally +qant as +epiph any +ine e +aero smith +spi eth +ar thro +ell ini +du bu +bra ving +âļ½ âļ½ +re structuring +illumin ate +equ ili +mp i +ash ton +pony tail +ma scots +flat tering +cru m +ast a +à® ° +stranger things +bar nab +ر ÙĬ +make shift +got cha +will am +cho irs +kilom etres +gho sh +eu than +dol ly +un ning +the ar +cre we +w sw +j ace +dis miss +ke an +ho ta +kh at +~ > +thir u +ren dez +hart man +tee ssi +cas ca +z ah +hydr ange +fo d +aw p +mzan si +thick er +nago ya +ne va +sti que +cast el +dam ian +there by +ji ang +ale k +music islife +ra q +calla han +gou ache +somal iland +sean hannity +ra heem +lo se +elo ve +whar ton +rectan gular +illustr ating +har ne +auti sma +scra pped +ell and +decre e +nag pur +ki pp +so re +n md +ma as +gun a +gart ner +bel li +then ight +je on +gendere quality +gi ver +a el +gar ments +ne u +mardi gras +mar sden +ro wer +pollu ted +camer aman +vin od +be asley +cro c +ji u +hollyo aks +anesthe sia +al les +ste ward +lati mes +ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ +tic ian +gor ia +come dic +ðŁ¤Ķ ðŁ¤ĶðŁ¤Ķ +nai ve +sli ons +ł Ī +bur glar +ðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃðŁĺŃ +york shi +se ñ +fan boy +lau rel +inci dence +potom ac +rober ta +presi den +pr yor +os bourne +w ku +te me +pal ae +ðŁ¥ º +re boun +itu de +red dish +k hand +coloni alism +north carolina +ðĿ Ĵ +manne quin +lady bird +ta sty +knowledge able +g shore +ðŁĮ Į +à® © +qu aker +salz burg +med alists +chy na +bridesma id +ma ori +ro p +outra ged +in adequate +truck ers +al ana +ìĿ ¼ +ri x +oooo oooo +command ments +lam beth +aa j +eco friendly +bla z +morecam be +boun cy +rou x +rai ded +mi zed +sh c +gaw x +labor atories +ru bs +rest room +consult ations +ca jun +virgin i +so ir +rev ue +ple in +wag er +ç ¹ +we do +growing up +! ðŁĺĬ +face ted +sin ners +ho vering +ti ene +seas oning +an ja +leg go +il is +fla x +dev o +ash ram +mati sse +ker i +go wer +bo tox +mar shes +unh cr +ts m +opti mus +dun i +stu ffs +so k +order ly +n bad +islam ophobia +raviol i +fab er +cre ds +won ka +in fusion +over weight +daily news +assi mil +acol lege +medalli on +kili manjaro +sti ff +tham es +sun ken +th ard +my dubai +hilari ously +han nel +plu mber +fair view +separ ating +rasc al +qui en +necess ities +confeder ation +ll ll +: ] +weak nesses +bron co +ra ffles +el ot +ãĤ¸ ãĥ +advent calendar +ðŁİ ¹ +stra vel +tun ic +k su +im peach +e spionage +! - +di ment +cur rant +bio de +commu ting +by ron +ðŁĴĵ ðŁĴĵ +shad ed +tr uro +cray ons +ar ne +h sc +fre aked +dram ati +fle ek +u cd +marl borough +^ - +cross ings +mal o +black ops +bin ance +cho ked +chen ey +pl o +ge stures +val edic +ryan air +rem ington +v cs +mc kee +ec z +be gs +nail art +mayor of +happy fathersday +war t +pet itions +n ingly +clean energy +bro x +sl alom +exist ent +ab ay +ug liest +tom p +stom a +sel by +goal scorer +ben ji +overwhel mingly +lan s +semiconduc tor +south korea +re scheduled +sk yl +en listed +dow ski +si del +rosen berg +nas ser +white head +pri us +har are +en n +ry der +í Ĥ +mon g +clas ico +transpor ter +po tty +is me +** *** +vic e +sk it +ode ssa +l mp +her n +raci ally +pin oy +paragu ay +obitu ary +go es +bu cha +side walks +angu lar +un constitutional +transiti oning +i bu +gu ys +un packing +oooo oo +black girl +ber gs + ¯ +wordof theday +trump train +thunder bolt +m si +fasci sts +ठ¬ +t sk +collap ses +raje sh +loveis love +migr ating +set back +ðŁĺĬ âĿ¤ï¸ı +t els +safety first +nar rated +jae joong +un answered +lique ur +en nes +dal go +bill ings +salt water +mer maids +lon gs +clap ham +we arec +pic collage +n ach +h ace +pois oned +lo th +ag na +adel rey +guar dia +poli shing +peace keeping +d all +p isa +la pland +process ors +de andre +so bs +p once +dra ins +c be +ðŁİ¥ : +spla sh +meat ball +fon tana +worcester shirehour +ne v +bri sk +b int +ac r +po x +cay enne +skril lex +j fc +hahahaha hahaha +gla s +en gul +tempor al +oni zed +con cre +com pose +vibr ations +plant ers +fer t +criticalrole fanart +t bli +sch allenge +huck abee +munici pal +iam bic +radi os +ne vis +dura bility +mc cla +horse back +inst itutes +ful fill +atta ch +ate ur +ak an +resi sting +illumin ation +hand le +hair care +om ent +macle od +ka iser +g no +bear down +ly f +gl omer +distor tion +z m +san k +roo sters +is now +as ports +ag en +wo ken +st george +ro mper +my le +econom ists +ru to +t will +health and +d ito +ws l +tair p +pra kash +mic heal +h ts +w rights +kat su +fioren tina +defen seman +d itch +var sity +texan scheer +ba ham +sc anned +we il +seduc tive +ðŁijį ðŁı½ +fu e +er win +dav ison +ter ran +moo ds +wool f +re source +@ . +cu sh +ðŁį ° +regre ssion +cur led +la zer +jo anne +ab bott +mo z +down ers +mm mmmm +valent ina +k hair +dream t +cro ok +che k +ste aming +nephe ws +cl eric +as ober +indefin itely +w ye +us news +joy ce +flu shing +wynonna earp +ron do +kis s +hot dog +bar ns +sax ophon +far ley +gas p +decre asing +al way +pe x +l sd +shi ft +p outine +ra zz +rescu ing +ni ko +ho ch +cc l +u aap +n ts +m car +il wx +conqu ering +ket tering +stur dy +delay ing +sto k +vani shed +cath ar +bin gham +in v +ic hiro +he mo +budge ting +[... ] +be ss +sebasti an +slow ed +ðĿ ij +musli m +stun s +acton climate +ve a +se ton +rose tta +oun t +hard in +flu id +ca w +ðŁ¥ Ĥ +yach t +un l +sp hy +provoc ative +or ic +is back +__ _ +nicol as +gy an +loo se +fl in +reb ate +: :: +! "@ +com icon +she ff +down stream +chic hester +beach life +mom life +diabe te +ar ra +van e +ok u +ye o +man go +try out +app ell +he irs +arjun a +dd u +na veen +movi c +soci alists +s back +criteri on +soyu z +k her +da z +yol anda +wine oclock +re ina +one w +leon ard +en dez +u bs +support local +facilit ated +carameli zed +b pa +vuel ta +my tho +m ami +spe are +nbap layoffs +fe vre +nick jonas +im print +c so +craig slist +la salle +gi deon +ha doop +dis regard +w ud +tu c +ma gee +acou stics +ta a +qui e +pol a +cr t +dw yer +dis sec +capit ol +men tion +kn oll +he igh +fin ders +plac ements +l se +indi ra +gur i +madhuri dixit +kingdom s +iambic pent +geor gina +je ky +conflic ting +bay an +aga tha +uph old +dr on +vic ar +ex pat +periph eral +pe ssi +fa f +ance stor +? .. +wid get +pun c +comm enced +beav s +air waves +ad dis +po a +de sses +co den +vu e +ru pee +kar in +spo ck +m sy +ภ° +pr ick +fill more +ti fication +thing sto +sar de +em ile +pere ira +n ad +bright ening +arre sting +wo king +usc g +sp ill +raspberry pi +hu go +ite c +is ma +cuff links +optimi zed +oc c +mi wx +en ka +el ited +afford able +sa kh +coron ado +ho h +at ul +ai oli +jim cantore +accoun ted +vin ay +her mit +groo ves +ran ch +r illa +we tter +ou tof +veter in +ni kov +ki an +fair banks +ram apho +n iti +k ko +ru sty +ne stle +tv xq +shahe er +âĿ¤âĿ¤ âĿ¤âĿ¤ +penn ant +gem stones +dem debate +ðŁIJ Ĭ +auton ews +support indiefilm +mach o +ve x +new sat +ne ti +conce ssions +can died +yof the +mac au +den ds +cricke ters +san iti +mari ano +gh at +ar toftheday +¡ ľ +e gos +gen oa +chat bots +bri er +al labout +mon ty +spi ed +r tr +comfor t +sni ppets +real time +gra in +exam ined +en lightening +tt u +god bless +release the +sing ular +ki ans +ha ka +sor ren +defe ct +mar g +equ ities +d orian +su ka +per l +aishwar ya +pul lover +preci sion +fair way +ne ve +rive ting +vill anova +en com +ak o +passion ately +europale ague +siem pre +x vi +enligh tened +c fr +âĺħâĺħ âĺħâĺħ +wast eland +is f +new comers +emergen cy +amphi theatre +- . +text books +figur ative +tre mb +pe sc +ab hin +ab bot +ac acia +har ds +por sche +kau ai +el isa +car rick +abo u +elli er +be ch +neu tron +galap agos +ru ben +in nis +how to +nun s +sab ine +i ac +clin ched +no tori +fi ves +cairn gor +per i +gr c +ðŁĴ¯ ðŁĴ¯ +mal m +twelf th +di ff +rout ines +marty n +lin den +synthesi zer +nu mber +game cube +fal kirk +byz antine +queu ing +gr ill +scal able +char red +rou ting +her bali +gri zz +ðŁĺŃðŁĺŃ ðŁĺŃ +tol l +termin als +l pc +ab d +war mups +remo vable +¯ \ +vi go +pap aya +ne ve +lov ingly +jo kers +ib les +sse tt +poten ti +pel e +gi gi +sadi q +leg acy +son o +ru pees +retar ded +ele e +par r +fi ance +ey re +say ers +pend ants +mak nae +al bans +adap ting +p ff +pu berty +ji u +ing rad +hypocr ite +diplom ats +phys ical +rob by +bon sai +ãģ · +f att +catal unya +âľ ĸï¸ı +ro ma +more land +so e +conver sions +stl blues +shol m +gra ssy +pra do +on u +assaul ting +> _ +sett es +dis graceful +aph ra +âļ½ï¸ı âļ½ï¸ı +ठª +kil n +goal tender +s ru +philanthro pist +b als +th n +stu den +sando val +dogre scue +eli ons +asse ssed +lar go +hec tares +sh rm +sa if +cle avage +no ches +n ene +fat alities +cur ing +clean ser +al es +p vp +south bank +pizz eria +marsh als +kni fe +an dover +tbli ghtning +sr sly +ou te +digi mon +timesof india +prome the +le bo +f su +wit z +rever e +man as +mam ba +ch ica +gu an +exhibit or +csr racing +d ere +xx xxx +gu sta +story time +ston ey +organ ics +and u +se am +min ogue +anushka sharma +ab a +ðŁİĻ ï¸ı +ugand an +chro matic +as sn +document aries +sh t +ru paul +loy d +k ats +e us +ite ch +me dusa +pan ty +kel logg +et to +talla de +sha a +do st +p ms +mari ana +je ster +croo ks +ðŁĶ ¬ +min danao +ind hoven +ðŁ¤ ª +le xi +tv n +jan is +co te +ãģ Ĩ +ser rano +iw m +ðŁIJ ¬ +k ke +distribu tors +cap u +counterfe it +camp site +ag gie +ðŁĺ ¼ +chhat tisgarh +~ @ +state u +san di +prevent able +cl s +can ne +mm c +i ver +sa haran +pal is +night out +do s +ap ia +absc bn +manag erial +aro se +mo wx +aro sa +ðŁĮ ³ +under dog +remo ver +astronom ers +lent ils +su scep +smoo ther +pend leton +fau cet +e mory +dal mati +af cb +tic us +exem pt +en rol +d heim +ðŁIJ º +restric tion +star fish +sto w +snor kel +thunder birds +she ad +homo sexual +dy n +as li +andre tti +dou che +dom o +tar mac +slu mber +pr onto +first dayof +mini ature +mari achi +argu s +recomm ending +mobi les +in ce +illustri ous +or c +adver ts +gr its +wea sel +pag oda +over pass +gre ys +maxi mus +arma gh +wood land +sun ni +ðŁĴ ī +ë Ŀ +ti one +soci o +ho s +ðŁ¤Ĺ ðŁ¤Ĺ +wind sor +subsequ ent +munch ies +id h +exclu ding +e mi +cu th +z ai +week days +law suits +barn ard +Ø ª +pe tting +net es +mul ligan +pharmac ists +ra quel +e ton +cran ston +gil ded +cle ary +ce ph +ra a +pam per +lombar di +as in +sher ry +pro d +for te +ari anism +buffalob ills +æľ ¬ +ðŁĶ¥ # +uu u +just ices +car ina +nat in +mas low +dro oling +cog nac +cam ber +el ong +r dr +in en +convic tions +am use +tro ck +harm less +visit ation +gen omic +bl and +beno it +chim p +tuscal oosa +gre asy +x po +gil t +se q +per mitted +christma seve +book s +mu e +old school +human right +be ati +ðŁĶ Ŀ +sh at +sculp ting +h wan +fern andes +sci utto +fu entes +endeav ors +maid stone +un paralleled +shou ted +queen of +mer c +band ic +ve da +sel angor +pi le +ja han +intimid ating +disapp ears +cl ich +za ha +w urst +hi v +fod ils +cor dless +aaaa aa +hy dra +bel inda +e els +bu f +su staining +rugby league +no c +brig itte +( ðŁĵ¸: +tromb one +soo the +smo g +ad p +stab le +ing ley +diagno se +ms g +we ss +tic keting +one e +nsw pol +e up +auto psy +adity anath +sun down +river front +si ya +p is +hier archy +dur ango +di jk +ren shaw +he aps +epide mi +david bowie +interne tof +dd i +nation ality +mb ar +air y +win der +w alia +elli ott +c x +bav arian +pl att +an tw +wi wx +sof ter +ne ha +h eller +th and +dani ela +bo ast +degra dation +ðŁĴ¦ ðŁĴ¦ +transform ing +man e +av ut +ðŁĺĪ ðŁĺĪ +vo ter +the e +t ate +pu ff +in door +sop roud +boy ce +boris johnson +wait in +immun ology +ðŁıĨðŁıĨ ðŁıĨ +âĿ Į +street food +liz asober +cavali er +c elia +need le +motor ing +g ato +, ) +ra de +harve st +t ms +jar pad +on ey +air men +v re +impair ment +abhi shek +snoo p +l ant +fam ously +bl ou +s ze +g ander +un touch +tu f +dee jay +col lateral +b ind +ðŁļ © +pin ning +ic n +' ; +the economist +ul tram +worldwater day +ti poff +the i +feed ers +campa ign +sc umb +day weekend +yo m +pe dic +h ough +ps v +pl in +on de +boston marathon +az zy +* _* +con ley +thi ago +hoo o +gal erie +luci d +je tt +gl itz +final fantasy +achiev ers +y ung +peregr ine +op hi +dam es +biom ar +âĺĢï¸ı âĺĢï¸ı +sk c +l ics +fl ank +ar rahman +ho of +uphol stery +t ats +wo z + ¿ +snor ing +ra er +l ju +ap d +pl ating +kan u +im ation +fragr ances +m ra +mor ay +mo tt +im muni +hearti es +bho pal +tim ers +g ata +color way +car nation +win get +si ghs +s ville +optimi st +chate au +olympi ans +ci o +singer songwriter +ny o +fi bers +bur ch +ag ro +mil ne +ig bo +cr amer +ation als +dan ube +pad ma +nor mani +en forced +bre ck +boeh ner +ar den +sur rendered +pros thetic +om a +ha iled +calcul ations +w fa +bi b +fcb live +fon da +west coast +que sts +friend ly +to wie +fit ch +bal ot +star dom +scrat ching +ho sa +thi ka +o ven +stro ke +out post +pharmaceu ticals +hi kari +mu y +af d +fallon tonight +squ at +or u +dra ined +chocol at +ë¯ ¼ +wor ths +ri b +mu j +that s +residen te +it el +boo st +mi gos +mul led +la a +etsy shop +don keys +me k +p tc +flin ders +e hs +ro hit +mu ir +g ad +compos itions +åĨ Ļ +combu stion +i kh +yemen i +wav ed +gar ci +ak os +oo ds +fu sion +se que +s lan +pl ur +kic chasu +shenan do +s ams +worl den +horo witz +with me +mic robes +k ki +ðŁĴĶ ðŁĴĶ +w su +patch work +fre er +y aki +the art +symboli sm +mil er +bt n +ma bu +side kick +motiv ates +sag itt +natur als +serv iced +ps ori +pa ola +qu ig +i badan +gi ggs +ë ³ +sciento logy +si oux +salam at +d res +cad bury +d hawan +ci ón +_ ' +swa pping +maris ka +james bond +explo sives +ay les +af er +s agu +cen sor +tom a +jeff erson +ring ed +par tist +ir responsible +aguil ar +vac ay +equ itable +altrin cham +ac ur +man ish +ger min +schoo led +pu tter +ed ad +nav al +toast y +sol areclipse +dish u +coy ne +ac co +mu ck +mar an +el os +len der +cro ix +worth less +ha ber +gun men +ðŁį ĵ +zen ith +t enders +hur st +hol tz +itali ans +car low +u cd +characteri stic +bun g +av l +u th +sa sia +rs l +red man +neighbor ing +green peace +sti ps +follow party +y gk +en os +omni bus +na issance +chri ssy +secu re +call back +ji hoon +memor y +block er +l anta +daf fodils +bil t +ffer ty +fau st +ie c +nipp les +so g +m nd +jagu ar +bol dly +ab poli +pro position +gun sense +evan sville +cu tters +we go +dou n +do x +stal lions +ka j +shi ppers +j awa +vol o +le ven +pap rika +kov ich +jor di +induc tees +app alling +dial ysis +allevi ate +âĢĶ âĢĶ +pie ter +mid wi +q tr +juli ette +inter mission +haw ks +act ment +one ill +k lin +vam ps +fam ous +cou ld +autom obi +da an +west end +elli p +nh c +mel anch +web series +ton gue +snat ched +smy th +tan gible +sl i +e asing +bar stool +over lay +afford ability +ting ed +ter as +ay ush +wanna one +rh ine +dan a +sh ana +kend al +fer tile +w ir +repl eni +lar vae +is ro +con vos +ab brevi +u cc +hun gry +bur rows +ag er +nav i +mat in +du per +cer n +ma don +ķ ï¸ı +é ģ +tu ps +hy att +sh ep +friday night +wis er +hei di +hat ton +p gh +foun tain +wrist bands +ahmadi yya +aeri al +subscri bed +so los +m ace +sla yed +for fe +dul ce +christ mass +arun jaitley +viol ate +ob stru +ni eces +w vu +idy l +fa ze +pre serves +infr inge +premi ers +inter vals +agen cy +( © +stand alone +di mes +bo er +param eters +ge tit +ðŁĺĺðŁĺĺ ðŁĺĺðŁĺĺ +tu lane +for given +scol l +mb ps +smash bros +rob bi +prima vera +ali st +ghost ly +ay at +ye ats +impre ssionist +ear phones +caul field +wai kiki +sal ute +sc ou +mu ay +louis vuitton +bak hta +ado g +inven tions +hur d +forec lo +stream line +thalai var +ch snews +will ard +t sn +euro parl +cru sher +my sore +gro wer +ra ping +pat ti +g den +sm w +muf ti +kid man +ab r +soun ders +skep tical +ðŁĶ İ +sun dar +i me +fer g +feather weight +ar lington +pas qu +ag azine +wearab le +nati c +mccl ure +inter mitt +hor de +six ties +car te +bha v +ze al +experi ential +ador ned +som mer +eno te +hypo thesis +stin ky +pro to +dead lines +vo gel +mus ings +monc ton +gu ter +f le +aci on +voice of +ta sha +inhabit ants +type face +s ba +bts x +ðŁĶ Ĵ +wor x +u hc +jo ko +cell ars +gor o +continu um +... & +weather cee +ha p +sr k +ris ers +lonely planet +un named +co eur +ðŁį Į +the world +ili ke +fa sten +ami go +ri ba +ramapho sa +staf fers +had ley +? ?" +fi ore +sal ut +hu ff +bez os +Ñ ĭ +ra der +kam ala +in line +fill ers +um atic +all in +shat ter +re in +o ku +ch ases +fla gged +baby metal +water stones +ts b +cut out +op hel +aam a +rockab illy +sto lic +jet blue +ich ick +down ton +uzbe kistan +pat na +la q +gr ange +) _/ +subsi di +sc p +newsc ast +it sa +twee tyour +e mor +archae ologists +uni fication +por ta +q x +protec tors +pro hib +charis ma +car tag +ren fre +scul pt +guwa hati +de ma +boo p +unf pa +dex ter +lay la +alleg es +sou ps +never again +l ys +cal c +bar oness +visu alize +ger ber +absor bed +i ers +a han +fon tein +detec tors +verst appen +sv c +formul ated +ac dc +li x +in competent +bh k +lour des +water house +snow ed +appreci ative +sig ma +lizasober ano +pen ned +pay check +tall inn +fanc afe +par isi +av alley +vi g +ru fc +hard ship +so cute +po ise +ì ¹ +roth schild +k ly +???? ???? +l hp +il ay +f hs +am ad +ide als +brad bury +bal boa +nic ot +kid nap +wol ve +tas manian +op t +matthi as +ãĥ³ ãĤ +super markets +mylittle pony +me lee +li ster +gr oun +fe dora +kind ness +en en +bra hms +¯\ _( +ros well +mar lene +ic u +re formation +or ail +he brides +dispar ities +terrac otta +swal lows +re id +influ encing +flu or +den e +tum our +blon des +thunder bird +sh eva +moga dishu +ka b +cre eps +i ving +ene ed +anno y +âĶ Ģ +intri gue +enqu iry +ar aj +tur al +kuber netes +end lessly +divi dends +tor a +ti sh +commemor ates +un ra +tri b +pon ty +ne m +diss ent +brew ingco +ðŁĺ ½ +nor mali +bi of +( ... +chil len +ì£ ¼ +mell on +av is +mccor mack +ing ra +enrich ed +custome rexperience +testo sterone +snu g +sett i +ger onimo +inqui rer +bre aches +very thing +bloom ing +mu ra +dispo s +bi de +de va +shade sof +in trin +sh ev +s ven +nayanth ara +gan esha +c ws +ber ta +label led +use um +nick named +ma han +car uso +ap ur +ðŁij Ĩ +w q +orphan age +discar ded +mag nu +lu e +je on +bridge port +pac ing +mercur y +( ðŁĵ¸ +marx ist +amphi bious +transplant ation +stit ching +then burg +gradu al +ãĤ Į +ro ft +ma ils +ine c +guy ana +dopp elg +ver o +re write +head less +harb augh +gate way +car sforsale +sw i +st is +mach t +un de +sura baya +stap leton +nur turing +mil ner +ya o +lma oooo +ko sh +arsen al +k ame +er ry +ar royo +dis misses +ru bbed +rc b +lew d +dil u +and or +vi de +ur in +inter sec +ha ar +al b +year swith +app leton +é al +ul livan +suc cu +monter rey +d mx +artem is +ron nie +farm land +s football +gro tto +anth i +ãĢ ģ +à® Ł +vid ya +jimmy fallon +ൠį +t zer +gravit ational +w thr +u hhh +e hr +tin ker +ti juana +scran ton +ram charan +bar clay +re van +m si +ka p +wr s +we thenorth +tor al +sat u +gro m +fac ep +erick son +z yn +se dge +oo dle +spur sofficial +ds p +sic ilian +soli hull +recei vers +ladak h +hend rick +ther i +presi ding +mc guinness +litt ers +gun nar +gh oul +wi b +n tv +kar o +fro ck +b lau +ampli fy +all is +ul lah +memo irs +kh loe +intercep tions +pet day +lo oney +con fin +ch ay +piyush goyal +frequ encies +ut z +event ual +warm ly +obli vion +an ka +ta it +âĿ¤ï¸ı . +director ial +ru lers +prince s +mu ck +stur ridge +deu ce +abri dged +bagu ette +un cles +pen du +min ding +forre ster +av ila +wall er +wall street +ment or +hin o +high way +crom well +fanart friday +mb i +co yle +a hi +tro ve +spie gel +pay tm +mcin tosh +jan sen +nit i +nash ville +len o +leicester shire +le gos +dic t +ðŁĵ ½ +sp ad +beverly hills +sy rah +separ ates +z ain +un fit +dra gs +tan ia +over flowing +hri thik +haw thorn +z ani +mac far +fi de +to tem +pe ds +fundament ally +cal ico +sin ner +j ä +hil de +ds d +ten ay +ta hit +mil f +lie b +inform ing +up lift +ra el +mortg ages +lec t +ii ii +guillau me +compos ites +old smobile +l end +gar th +com mish +bapti zed +scorpi ons +ru cker +bringback our +alli ance +thalap athy +tal i +sp ans +eri dge +wither spoon +lin da +sky lar +kor n +hom s +Ä į +sil enced +caf fe +ar ty +dist inguish +to wed +pun g +jessic a +ear nest +beau fort +t ama +study abroad +si khs +new bie +nav ratri +mar ble +loun ging +lit ter +dal it +so sa +iz es +gra de +com promising +tr iton +de tta +v j +chau ffe +spec tral +powe red +montess ori +artic ulate +hal ton +al co +ye y +mn twins +acoun ty +ðŁijı ðŁı¾ +âī Ī +mad men +kal a +gru m +chi k +ati s +su me +akh tar +job search +high lighter +bo ath +âĦ ¹ +tar zan +lam bo +âĽĦ ï¸ı +ox fam +dump ster +pretz els +mac os +incl ined +fac tual +adverti sers +shu i +pu ree +ml pfi +anti dote +cap o +pa str +merc ado +but ton +ar min +ag g +lol la +horri bly +er rands +christop he +time snow +monday motiv +li ss +scand als +mc i +dispropor tion +âĺ İ +sur pass +samar itan +so tho +pu rest +fl att +trivi atuesday +delec table +leop old +hermi one +chou dhary +en rich +¡ ¡ +subsi diary +ine qualities +bachel or +auto immune +la kota +i hop +ad jec +the simpsons +sh es +se k +gret chen +up stream +hin akhan +coper nic +x tina +lu g +tough ness +e ad +cli pped +bi us +sl v +fah ren +dee pak +ca u +x an +im mature +dig ni +bo bs +shred ding +but tery +accommod ations +de ven +chun ks +super league +sky bet +kil dare +je et +ë į +ce k +wrec ks +pro pane +oh l +tb d +quo i +trum pp +mi mo +reluct ant +ver ne +o ic +ma gh +ar nau +se ver +li dge +stair way +kicchasu deep +ðŁĶ º +mach ining +aama admi +ot i +c da +al it +pan y +inst alls +ac ct +e shop +di em +hard well +fulfill ment +sc afe +qu ack +extrac ts +swee tened +fi ghton +f di +d inger +wal tham +us ur +refe rees +seok jin +gran n +af rin +th n +sch af +par cels +bet is +amar ine +nom an +kh tar +mor itz +cou pling +bar ons +ðŁIJ ¸ +à ¸ +sl p +sad ler +x ander +tri ad +mc millan +kh z +divi ding +ìĹijìĨ Į +dar yl +zed d +le ys +pla ques +flu ori +tipper ary +on nell +di dier +lang ford +im c +the sun +bir dies +ar cha +ye ssss +t di +dar ia +cand ace +al tam +pal aces +ch it +sant am +event ful +book of +ad b +mon stax +cre ole +co el +âĸ ½ +we aren +sten nis +she ath +ati sm +gron ingen +mlpfi m +le pre +wrong ly +rsp ca +rendez vous +acknowle dging +pel vic +solic itor +sla ys +nue stra +lo d +is lander +fer oci +fashion show +ra ss +dge on +adole scents +sma shes +negli gence +grate ful +ved ere +sw oop +ing l +apol ice +vand alism +gan n +jo ao +di supdates +zimbab we +under age +radi ance +w of +bour geo +pla s +cr ani +gh ue +wrec kem +warran ts +re form +jim mie +at wood +ys l +neil himself +l bj +i man +tan to +nois se +ver bs +equip o +al together +mam ent +l ice +dou glass +tier ney +pri med +j hal +furn itu +braz ili +v ill +past els +n ison +u ff +paral ysis +jay e +im po +ðŁij ģ +strate gically +pakistan is +was sup +super bike +thank u +tru elove +sha ikh +israel is +vi p +to g +li en +la ker +grey hounds +cul ars +bian chi +balot elli +ar ran +loo s +str ates +he bron +ar vo +sunder land +the al +tomb stone +sand man +c pac +thanks giving +love him +lat ino +an in +aka if +ĭ ãĤ +tor quay +di est +alli anz +ðŁĺ ķ +golf club +cl lr +wal cott +sch nau +promp ted +nomin ating +len nox +val et +mon ro +may ward +e ph +ðŁĶ Ķ +inter oper +r da +re flex +arm chair +ê° ķ +stri pper +por ti +ph arm +ham za +ni reland +ne ue +h pv +port foli +sun burn +fris bee +be al +bapti ste +x h +ty m +pr ati +o vers +haz rat +deser t +der ry +us ky +em mett +ach arya +)_/ ¯ +shu d +may a +ham ill +ra im +nr c +fitt ings +cur vy +ðŁı ĩ +ster ling +ॠĢ +wal kin +short cuts +mil ly +ast ur +alpha be +pl i +pe z +miss you +rad ford +ml g +ta eyang +notjust lakes +du mps +seren dip +le ur +ra ving +e ster +de priv +absc bn +ðŁijĩ ðŁı» +scar city +o cr +mean ings +cap t +da hl +fer mentation +bri oche +to win +out lander +massi mo +en cro +ðŁ¥ ³ +buil t +po tam +kir i +tm w +monit ored +k ites +peoples vote +gray son +íģ ¬ +afri ka +a dies +i vote +gy ne +g annon +di x +c mc +ou ral +fox andfriends +bel i +ig ne +gl an +katrin akaif +co politics +qual itative +p si +lu cci +disc oura +âĺ ® +kel li +gau tam +carac as +reale st +pu la +in us +hill top +make aw +atten borough +tw y +r arity +peck ham +ma hon +corn elius +clin icians +ton line +tb i +paradi se +ka si +inev it +fresh ness +colling wood +lun atic +defen se +cop d +in fra +wain wright +sains bury +alab am +te ma +lac o +chec ker +releg ated +tren t +stal ks +huff post +bhubanes war +ast ral +share your +prim rose +hi me +cat an +end ment +en dow +cle mens +mal oney +hil ary +game time +den ise +collabor ators +b wo +radic als +gue tta +ici on +au a +snap matic +sat chel +excav ation +base man +s ão +gn ation +fel d +surve y +shah zad +ma st +anirud hofficial +tru cker +ot ago +geo graph +ethe l +âļ¡ï¸ı âļ¡ï¸ı +s ver +mu tt +internetof things +ancho red +wh ouse +bang la +bal main +ç¹ ĭãģ +break fa +á Ģ +twi ster +te tris +ca v +stag s +g z +au b +stor med +hel ens +yar mouth +st asy +gustav o +co sc +vin son +up p +sc ricket +assump tions +app e +nu h +u er +pre mise +n aga +e amon +coron ary +na f +north side +el mer +ro tar +out lining +el f +re surg +kat elyn +in can +hyster ia +ce e +am bani +pro lly +Į ãĤĬãģ +ax es +san jose +rem brandt +mag pie +even ly +scor sese +qu aint +f g +b buk +indian football +weare all +spd wy +pis ces +ec g +âĺħâĺħâĺħâĺħ âĺħ +pre orders +: | +ni pple +sal azar +ju me +jail break +min n +bas sett +ze tta +jef free +ad jun +tic on +san diego +drink local +chol era +solic itors +o bo +com post +ni an +wr a +tre ach +ic ic +profession al +del ve +leg ate +histor ia +cro issant +con noisse +nam o +palli ative +chem trails +i ority +global warming +comic art +behavi oural +re sted +li as +cli mates +Ł ãģĦ +rut land +nou rish +menopau se +hot ties +demen ti +ve spa +mel ville +anal ogue +tz man +str ung +im perfect +gl are +cir cling +ros berg +rec o +oc ity +lo ire +em be +do ssier +ne el +nan do +me a +gal vani +fin esse +ag p +berke ley +asi m +âĺº âĺº +quil ted +ish ere +un matched +po tion +for z +at re +selfi es +juli ana +ðŁļ ¶ +âĸ º +mel ton +âłĢâłĢâłĢâłĢ âłĢâłĢâłĢâłĢ +spin rilla +pur cell +ed p +at leti +tony awards +ra ja +pro gno +mol ten +stu ff +p ally +nobel prize +âĻ» ï¸ı +spiritu al +spe ake +sa sha +bri um +tru ss +critici ze +assassinscre ed +yor uba +u lo +fire man +workin progress +ef cc +fla res +ro bot +hi kers +cl l +shado wing +pat sy +leh man +c ns +å ± +guad al +à± į +ra pe +r honda +paralle ls +son ja +langu age +land ings +z ola +cr amps +bur ning +apprais al +jol la +ham m +kas a +gul ly +f go +uly sses +ri be +ðŁĴ Ħ +ib u +eti enne +bri ar +fin ely +comb ating +y ql +go tham +we chat +to paz +primar ies +l se +iz z +hel e +dispon ible +cy stic +bel ichick +th rush +kansas city +ge om +soli di +red bubble +by stand +cambridge shire +par fait +ast le +ow o +ind ore +stom ping +sm elly +ðŁ¤ ĸ +locom o +adm itting +hol me +clock wise +min sk +mc co +for get +ev p +cam ra +ab ella +yo tes +universit yof +mé xico +silver ado +ric ket +crom bie +pu j +eradic ate +deli ght +y go +glam ping +vic a +du ggan +coun ters +cf d +sc our +react js +pu ram +paras ites +in ki +vill en +stel la +li mbo +ang as +k cr +ðŁĴļðŁĴļ ðŁĴļ +vap ori +mum ford +oli gar +à ¼ +al oo +boo ties +ad r +k elli +dru mmers +av ici +nature uk +ron al +in trac +un splash +le che +g oma +el ine +envir o +bi onic +bu eno +mi k +av in +star ling +em powers +cake day +boy cot +ðŁĴļ ðŁĴļ +ðŁĮ¸ ðŁĮ¸ +v ach +m ci +fractu res +ger i +sk ing +exclu ded +lu ce +ja ve +ig gy +evi den +aki stan +a wn +mor als +luci fer +ha ban +tumb ling +sunday motivation +mo sley +captain america +sch icago +the one +mo td +d ts +ðŁIJ ¼ +rep ell +ii i +locu st +geo spatial +mer sey +immer se +desc end +ber nade +j s +boat sales +win der +cran k +sing leton +candid acy +ben a +ðŁı» âĢį +high lander +ol t +k prs +healthy lifestyle +four teen +end the +ith aca +circul ated +r ans +pre valent +ha vas +splend or +roo ster +kalamaz oo +jewell ers +enne dy +rou sey +es y +cann ons +ornam ental +// // +ren don +win ne +mol ding +eid mubarak +coun tess +simon a +ha wa +fo es +du ster +sb u +por tray +mar ries +goo dday +cho co +achi ever +ðŁĺ¹ ðŁĺ¹ +pre neur +tr amp +tom i +n bat +garden chat +farra khan +ever glades +ab ru +sou sa +se ce +homes wee +terre strial +bar it +sri devi +ol u +mel inda +f rick +can dies +ðŁĺŃ ðŁĴķ +qu reshi +family fun +exor cist +cardin al +ny t +dies el +cu mulus +capric orn +si ology +lor na +dou gie +an die +super sport +c fl +п ÑĢи +say ang +pe ek +ภĬ +lo be +j em +ing lis +gg led +c sn +amne sty +chu ps +ba es +sau er +ðŁı IJ +mongo lian +en et +back street +dr illed +acce ssing +ce o +b se +ai ken +pur r +wor sen +whe res +war k +testi fying +bu ri +bla st +aw g +ðŁĵ ĭ +re defining +hear ing +u ci +c mp +bon i +tail oring +ta ji +noc chi +em t +stephen king +ne et +compla ins +campaig ner +luci ano +twili ght +ti esto +pas sports +flo yd +cathe dr +na ked +caregi ver +b coz +ade cides +ku ri +ly k +br aries +dren ched +disc lose +ðŁĴª ðŁı½ +le blanc +je tty +gar ty +chip mun +b su +rhyth mic +ic z +fri d +anne x +ame x +solo ist +lanc ers +arro whead +speci fication +simul ated +na is +inver te +bo wing +wor ship +f z +abo ss +sha q +ì¶ ķ +challeng ers +an arch +aamaadmi party +ãħĭãħĭ ãħĭ +suffol k +so corro +sn ell +cla dding +absor bing +shaw a +particip ates +ðŁį Ķ +book stores +bak u +seap ort +ko jima +gab y +pack ard +electr ician +let it +mo wing +fa wad +young jae +hot mail +men ing +u rie +intim acy +con ti +: ") +lifeis good +in ciner +i dri +craz iness +jour nos +fran chi +bott len +al da +ff es +k x +south we +air a +clay ton +sco ti +f j +bri ga +ðŁ¤ĺ ðŁı» +demonstr ators +y z +stor k +na q +casc ades +travel chat +plat a +pad ma +fran ci +at tain +bat girl +lom bard +hoo s +d dos +neon atal +discla imer +r ss +r ant +di sen +tex aste +so cal +frac tal +cam ry +stri fe +sn acking +mu h +sant ander +mor ons +gra f +par ades +hu ston +dru pal +mi ento +kir stel +hy de +vom it +forti fied +sphin x +da v +bir yani +win nings +s baseball +mer ged +lovel ondon +ling ering +dream big +car leton +liveli hood +djan go +astri d +gri ds +down e +bru ised +s ne +scarec row +hel ium +f nc +bi ggs +an ter +restor ative +em pires +ab del +life style +kiwan is +colloqui um +me en +pr ick +anti que +ze b +mi mic +edmon ds +ðŁijĬ ðŁijĬ +q ing +pp el +mc gill +interpre ting +âŀ ķ +rash ad +do ka +narr ator +electro magnetic +ash by +sau ra +iran deal +âģ īï¸ı +krish nan +in di +ff en +bre a +os man +multin ational +chi ppe +recruit ers +aus biz +p ounding +re gen +cur sor +refu sal +mac s +in ak +ax ial +wa ifu +up cycled +hindu stan +cas sini +carly le +scrat ches +re ef +man atee +eat ery +ðŁĵ ¢ +un condition +sen pai +on ther +comic book +pro sciutto +de mar +mi se +ma ge +fre ec +aye sha +al der +android games +ley ton +ho ck +door way +chicagof ire +aali yah +sw elling +bi x +. ðŁĺĤ +evan kirstel +torpe do +kon stant +genevie ve +ma ia +ha user +do torg +hide ous +fi k +sp raw +e ek +z appa +wan dered +' ' +ra jan +bam bi +( $) +wid ening +tool box +sa ir +illumin ating +pra ys +out patient +i w +day o +lo b +sw fl +sha des +gu ms +coo kin +ko di +gri ffin +traum ati +ste a +slaugh tered +god bless +air time +pseu do +b sa +hau led +ar if +à¸Ńภĩ +le l +wc po +mil iti +char ters +worl da +ru k +k gs +digital india +is able +idyl lic +esp ino +marie tta +e bo +team canada +ab our +wil ton +rock stars +fav ored +phys ic +wrink le +tb r +d print +ball arat +ad al +z ey +ðŁĺį ðŁĶ¥ +tom lin +mt r +pal sy +fener bah +tight en +phil ia +ir oning +ry u +b ant +enqu ire +ca ir +abur ger +tru n +green berg +chau han +ir ina +sh ani +trend setter +pre tt +zaf ar +alo ve +v ici +pan ic +no o +lu stre +disrup ted +bal lis +son sof +mon si +inst ac +ake st +ëĭ ¤ +kw ame +horror movies +distric t +sau cy +mb an +ar mies +with drawn +med ics +loft us +er oom +be kind +ar ns +all on +un ison +davi ds +cr at +nicot ine +so or +sm x +on co +cospla ying +zombi es +har ms +e ger +ro sy +moon shine +fe in +ce tt +du brov +reg ents +ben itez +ðŁijıðŁı¼ ðŁijıðŁı¼ +ste c +m alia +prioriti ze +ic eland +ft se +v amo +lam ont +homo sexuality +bre es +regu i +cb p +te j +sky sports +deter gent +sha sta +de rel +conserv ancy +colori zed +accol ades +vis o +show your +nan ow +bice ps +us ability +bi m +dailys ketch +pearl jam +stran gest +mega deth +broad casts +bar ren +ar ton +chri ss +confi gu +lu res +is the +e ul +railway ana +global health +gi anni +u aap +s lum +consci ously +ab re +n up +bud get +v ada +e sch +real ness +er ased +th unt +be z +armist ice +ðŁij ¹ +sh run +o led +driver less +ðŁ¤· ðŁı»âĢįâĻĢï¸ı +won dr +sk an +sal aam +mother land +h wang +gen o +gang nam +tw right +endor sing +en ic +ador ation +pau sed +patric ks +do cked +plat te +ff xv +ethnic ity +auto show +side show +after life +re located +orphan ed +food network +dare to +and ra +sla ps +v live +swim s +re imagined +mist le +re vise +real ity +bhar ti +ðŁĴĻ ðŁĴĽ +late st +prou dest +gra sses +lan yard +fresh est +carcin oma +anom aly +zieg ler +sum ner +ly rix +gor g +is d +av el +swild life +me squ +john cena +euro league +sab er +master ful +yar ra +cogn ition +jacob son +abo lic +sir loin +shuk la +moj ito +su pere +st weet +me z +e sa +rudol f +gur a +where you +tt m +win s +trust worthy +ny k +bra den +table top +good food +es on +be k +lingui stic +gra ys +ch ath +h cs +mon i +de ans +cu ssions +ch ell +slo ws +he mi +d app +shar pie +boo sters +a os +str ack +se dona +mu eller +hard wick +or nate +thor a +sal ud +o twol +ch um +mi ho +for age +thel ittle +tear ful +ones elf +min dy +sm g +gmb h +emer ald +ðŁĶ´ âļªï¸ı +tu tti +recep tions +re vising +i brox +tope ka +sal ami +expan se +i books +dob son +cli o +at s +ðŁļ Į +mo ha +is ance +shu tters +moo t +jan ine +marvel comics +jor dani +pos er +kenne th +hy ung +de ja +ase ball +speci ality +eu ston +classic car +had ith +ðŁIJ ī +chas ing +iz o +gros ven +ag lia +thisdayin history +t row +om ile +hu ar +by n +sal ine +div ine +demon ic +ty ran +han dover +revit alization +pa ella +cryp tic +se dg +m end +dun kirk +bre d +wal d +sport scar +a ard +whe aton +da ener +k lan +br t +bakhta war +spi res +schu bert +ro ti +poli sh +o se +ag ame +wonder con +prote stant +bo sa +ðŁĺ Ł +d ü +joy ride +ger trude +âĿ Ŀ +gil a +v h +tw a +tra v +swal lowed +star ve +la in +ent ren +rei ki +su kh +cra ic +az u +web page +kee fe +hypo the +hir sch +hel le +camp ground +w amy +tra vi +sha hi +san deep +ru i +han uman +dw p +reposit ory +no or +no ff +un real +p ell +black history +har vick +ma scar +pay ee +pa sha +gastron omy +d ÃŃ +ai g +rosen thal +open day +embelli shed +t tip +sun bathing +go pack +end ome +ï¸ı # +invali d +final four +st fu +squish y +ra sta +mo sch +jam esc +die trich +sel a +mel b +el vi +t dp +sun i +sli t +j ha +bi za +spi ked +l li +l illard +vam pi +syno psis +az har +kendrick lamar +ĮãĤĬãģ ŁãģĦ +heart less +country file +air play +arrog ance +pre e +virtu oso +ãħłãħł ãħłãħł +raj u +le bu +for ward +tu g +dro s +mondaymotiv aton +concep cion +thel o +pad i +looo ol +ÑĢ од +it ss +eth ical +end uro +__ : +expend iture +mon ste +mas king +terri ers +ib is +e mber +cu mple +punctu ation +pi per +ir vin +ade e +yy yyyy +flash backs +cel sius +don nie +bo gota +ben evol +the script +shil pa +pro se +fin dia +ze ke +ne ko +do ves +blues lyrix +fro sh +sowe to +mp lo +al ai +sab i +raq qa +wf tv +stro ller +ian somerhalder +ðŁĶ ª +an on +mo seley +! ?!? +sta king +mol y +car tri +c sg +ast or +transc end +ma er +de ux +cow girl +sas k +pun ter +ma ken +o ates +love tt +grow ler +sag in +v n +ssi ble +officeof rg +y mc +sab ar +faul ty +ap ha +ak on +ðŁij « +snow don +ae w +raise the +ðĿ ĵ +grue some +clement ine +sp ing +lat a +worlden viron +mi mic +can aria +bakhtawar bz +ao a +fal a +ãĤ Ń +avi va +you uuu +thi gh +la dders +gu mbo +tz ky +fu zz +plastic pollution +est ate +strength ened +k ant +dr in +cal vert +transform ational +frigh tened +mac lean +elited angerous +ear thy +t son +to da +j nu +.. , +mic hal +i ban +je ong +is real +sim coe +exclu sives +blue bells +ben e +te u +pil sner +pens ke +athe ists +m pu +cartag ena +ðŁĴĹ ðŁĴĹ +million aires +kk kk +it ar +subscri ptions +remo te +ma fi +hin ton +w cc +ho k +ds b +ab leton +sevent y +pun ks +e indhoven +sh one +mcfar lane +lim popo +empha si +à ¼ +sin fo +pe tre +man grove +ch ino +ber tie +play lists +push awards +p af +deb bie +c do +r ino +ðŁı¾ âĢįâĻĤï¸ı +fol ke +bon nar +th ine +sl an +hal ter +evi e +aw some +vul tures +spar ky +seiz ures +âľ Ķ +ram one +ine ffe +al n +pro ctor +ast ra +the voice +gro te +sci on +dead line +am aya +tain ted +patter ned +exce eding +cross fit +kay lee +drop box +ru shes +tack led +mo by +retro gamer +n cbd +benef itting +shay kh +guild hall +gen try +dream cast +dread ed +bun dled +th aw +revol ving +n pt +kylie jenner +imagin ative +ron i +over came +family time +ds burg +car naval +relation ship +recogni zable +cor oner +ho le +fan fic +emir ates +bur ritos +analy se +thin ner +ne es +galli poli +bl r +cat woman +-- >> +au lt +ada ily +nau ghty +ili o +solit aire +mtv br +jocel yn +arun ach +rep ent +south gate +hy acin +essenti al +fent on +and um +it or +go pal +sl inger +po sei +aw il +wi elding +ra ila +eli as +a sto +à ¤ +tend ency +str ata +ker t +< - +im acele +da es +sti mulus +han ley +fit nes +ec stasy +lim ous +ha iling +ðŁ¤ Ń +chis wick +tar ies +sla v +pul i +moderni zation +black mail +b ingham +h fx ++ + +ðŁĩ®ðŁĩ ³ +ni v +we a +profess or +k off +bol ster +su ave +sequ ences +pepper oni +not te +dre n +ãģ¨ ç¹ĭãģ +hs v +o ga +ap tly +z ad +excel si +rin ka +mol dova +min n +ma bel +conferen cing +bas ing +of er +ob si +hamill himself +care less +brief ed +inhe rent +par ish +dub nation +town sville +sar awak +gee ky +doncaster isgreat +was abi +gu p +phen o +dra inthe +carrie underwood +ble eds +bbc world +ane w +alta f +dul wich +ani ston +w ti +sumat ra +gra fton +bl n +me ster +bode ga +re go +es q +an jo +sump tuous +mai sie +ï¿ ½ +wil t +jak ob +el vis +se pul +mu ster +air pollution +president e +happy monday +exten sively +fl ondon +t ls +play ing +pe ed +din ho +var dy +pi ka +n iro +au cus +ðŁį ¦ +nu ll +el ondon +juvent us +imag ines +dis ab +lit o +d ura +work places +promo te +mc caf +wood work +waw x +à® ª +tt ino +shar i +sem per +better together +ðŁijĬ ðŁı» +ze bra +pon dering +en chil +ho m +cosm ic +tan z +mo cked +ec cc +ath ed +abo lish +prop eller +paris agreement +assemb lies +indu stry +fraudul ent +pe sa +chang min +ax x +ðŁĴ µ +irr ational +cu sa +ramad han +octa via +on elove +jac ki +bar ak +taxi der +seri ous +nathan fillion +mc en +ch k +po part +grav ity +copp ola +reading fc +illu sions +j ig +ww x +re sh +ex porting +buzz ard +âĻ ¤ +p cm +lan apar +ko s +arom as +antal ya +ww dc +ven a +phil a +ball in +ðŁij Ħ +quin ta +ma o +f ery +eigh ty +sentim ents +safe guarding +r wa +pu ffs +luc ille +de cath +sl u +nu gent +de ter +braz il +ze iss +super bowl +subsi dy +alter n +hi dalgo +enz ymes +ä ½ +tag ne +hair dresser +adri en +walk out +oppo ses +can tina +bed side +af an +ðŁĶ Ĺ +prophe tic +dan es +un successful +super charged +pk k +exem ption +hart le +secu lar +cli pping +br s +united way +c net +pat chy +ha gan +e en +âļ ľ +var a +sym pathi +never trump +affir mation +om f +ny cfc +ma ja +sur ro +keer th +up scale +sandal wood +mon archy +kno bs +å ĭ +po tholes +hunger games +ter races +na sir +coun sell +welcome to +wa q +se aman +m ita +stun ningly +on theroad +in ability +) !! +bon go +ant v +sp ut +worldenviron mentday +resu sc +y td +fi m +eun hyuk +sa chin +rose anne +cler mont +ape c +am ina +v ening +n antes +al most +sin us +ex as +ty l +ti en +ple ad +lanc s +bur naby +re k +jo om +observ ers +disco graphy +cl g +âĻ ¦ +sn ack +r ti +o ily +crystal li +bru te +web development +topp ings +la f +an is +ad der +reli ving +car lin +battle of +we g +syri an +pon t +n dc +lagh ate +yu ma +sp p +p iti +ro bbing +mart ing +rey kja +raj put +nc ds +kie wicz +âĢ¢ âĢ¢ +vam pire +substan tially +opio ids +nepal i +k line +ar oo +under stand +lit t +u it +thro mbo +sar ies +qu ot +b alling +t tr +s gh +philip p +br ant +ac l +m ello +whit taker +. ; +defi ant +b gc +repl ying +mir ren +metamor pho +sch wab +bul ge +utili zed +pick ering +par don +d sa +ภĪ +doo ley +cumul ative +Ð » +ur gency +e mir ++ /- +¦ Ī +ot as +âı ³ +station ed +grape vine +ar ac +karan johar +f ancy +sau l +coo gs +lgbt q +ا٠ħ +jav i +u mmer +pl l +den is +dai pur +pu ffin +lewi sham +fand om +co pe +ves matter +s ve +hel pless +deo dor +ostr ich +kaz an +friday the +con dor +v x +sophom ores +rob les +cu tt +cli mbers +ë¦ ¬ +sle g +sn f +mac ys +hydr ating +grou pe +po yn +mou lin +hg tv +lmfa ooo +sulph ur +asdfghj kl +annab elle +hump back +bra ved +viswas am +multi purpose +hu midi +escor ted +barb ican +f ad +cor sa +ðŁ¤ « +pi ppa +here to +can y +ser gi +or cas +o vie +ed ou +s any +glob alization +man cini +food truck +f is +defi brill +sch re +sma fia +love wins +la ut +k aka +hol lande +game on +resurg ence +out side +olympi ad +int an +abstr action +rapi d +pal om +cal le +jas min +attack ers +swag g +mit ra +ky lo +à® ² +her mitage +gor do +e ira +so sfam +roll out +exc ite +sy nod +mer rill +c als +as sa +liveli hoods +ju ve +the black +gopack go +ant lers +alban ian +wool ly +qu iche +puri fication +are th +smar thome +ne k +all blacks +mex icans +is m +ger ms +comple xion +mar ck +u shi +ðŁIJ IJ +char l +ca stic +till erson +giuli ani +biode gradable +mal bec +bo is +ju bil +im es +r ame +gene tic +esp nu +ch ley +so ho +go pher +g sc +buu ren +cu be +bridesma ids +webin ars +to e +mani pur +viol ently +notic ias +ex changing +chi ev +replac eable +muay thai +bu ss +sp il +instal ment +div ya +cait lin +o lim +fil tering +whirl wind +sta red +prior it +pr am +pompe ii +mono logue +k ite +bu ka +âĢ¦ .. +vac cine +bre ro +woz ni +sol ent +re ferr +my rt +gridi ron +galatasar ay +fro ze +clare mont +ðŁ¥ ĥ +victori as +ssel dorf +pa stures +net neutrality +ch or +ðŁij ģ +ಠ¿ +we ho +symp tom +jo sel +in ous +dragon con +power ball +p te +four thofjuly +ec la +ear buds +where abouts +salt life +depriv ation +ch ter +wi ggle +syste m +ps st +ch az +d any +ri mo +oax aca +lanapar rilla +barcel on +melanch oly +way back +ho tro +n si +l illy +kur o +ja han +intellec t +board game +ðŁı Ĭ +sneak peek +k prc +jail s +cand el +zan zi +mor timer +star ch +ra gs +p fa +long live +k art +gir ona +cro cker +christop h +precau tions +war ship +per m +paren t +van gogh +gif ford +allegh eny +ra yn +ut m +sten cil +rec alling +pen ney +z azzle +ìĥ Ŀ +hin ds +aren as +nu ev +law ler +gu in +do this +ðŁij ķ +ì¶ķ íķĺ +we g +ti b +ri din +complex es +turbul ent +pe sos +de marcus +vall arta +sam sun +kis ses +hein rich +deport es +wil ms +ur d +then ext +inki gayo +ho wi +fir sts +carri age +clean liness +mas war +is ch +ax el +si zzle +road house +fr ans +ent ourage +co bble +boo th +benedic t +tal on +fc u +year ofthe +ray on +raider nation +fo yle +ko val +pi anos +l pg +bur mese +man ure +geo caching +cosc ino +b np +fer ra +stro phy +mar ais +ce es +legen dof +kat niss +eno ch +av ed +you know +d prk +ðŁĺ¢ ðŁĺ¢ +sp un +pro st +sor rows +cent red +ke a +gal icia +? ðŁ¤Ķ +ÑĢод а +bou chard +ðŁĴĻ ðŁĴľ +yu i +seed lings +jon ah +reco vers +ny rd +board room +su ma +my japs +tun g +sha i +ir gc +eli o +wag ons +ka shi +polic emen +john nie +ale coscino +shop ify +dot ted +de tri +va w +to fficial +in your +chal mers +trac ed +no vi +by es +ari el +nipp on +la pel +gri ez +b gs +fool ing +d ita +vijay sethu +nm wx +as ot +kr anti +hel m +ve di +sic kest +mo chi +k abo +shru bs +he red +b sp +sq m +ham r +dul kar +anth a +nr f +avoid ance +at en +publi x +be arers +nas i +ha p +h ells +ðŁĸ ¥ +ภ· +thelast jedi +oh wx +ðŁį « +wa hoo +there se +rec aps +ss nhq +bird photography +v ay +pet ti +pau lo +bel vedere +( * +gr l +du vet +c pec +sa it +por sch +meas urable +avi ators +fre mantle +bre en +on om +me and +life saving +eu ref +en don +embar as +aira sia +el is +dun kin +star magic +s ill +porto bello +ki efer +ex e +mu ted +ãģ ¦ +we thepeople +logi a +liber al +theforce awakens +min ed +haun ts +freck les +care taker +s india +âķ IJ +dev lin +list on +direction er +oh n +fi garo +em manuel +du bois +cl ones +bru ise +ðŁİĪ ðŁİī +disin fe +der matology +as r +s watch +dis comfort +tam anna +pi day +mack en +k atic +delu sional +shaw nee +gu d +al bino +p ali +din gh +cucu mbers +coffe y +anticip ating +treas ured +web summit +shel tered +sav or +pedago gy +m gs +sh ma +s bu +den ali +cam pos +bubble gum +o ir +le aps +y ler +r one +sansk rit +min t +meat less +futuri st +du de +a vel +prote sted +squ ire +z aki +sz n +har court +cycl one +bour dain +gather ings +d ant +advent urer +parag on +alt man +dd ing +ban erjee +snorkel ing +mother well +mis sy +en der +glo ws +ki wis +chick pea +por o +e fron +app t +u y +speci fied +gab by +e strada +com bos +bour bon +vin i +var un +steph ani +key words +car vings +amit abh +wr ought +tw al +re els +clu bbing +ubi quit +cri t +ambed kar +æ Ļ +prun ing +vaccin ated +boe ing +s ks +lo ona +hypno sis +edel man +pho l +he w +colo sse +mckin sey +u on +to te +sacrific ing +ox i +n ang +e mu +пÑĢи ÑĢода +m th +kers wednesday +argu ed +timel apse +ris king +regul ating +ni gh +likeli hood +cu bic +au ction +rein for +pi stor +no ses +ye l +snu ggles +pe i +jean ette +ta ku +ri th +guy z +ภŀ +y te +ver ted +pay soff +jau regui +hoo ligans +procedu ral +mi b +har dy +el eng +chec kers +all ine +the met +prou dof +keerth yofficial +collabor ator +ni u +infl icted +adv ani +re twee +memor iam +f icial +ti ghter +sal em +re viewers +br ics +ben digo +am ell +tur kish +sush maswar +paul son +pal awan +mol lie +stitch er +s burgh +ir u +hay dn +en ers +aro a +u zzi +saraj evo +hel a +apol lo +nine ty +vac a +sp on +vent u +jel ena +hei fer +avo ids +sp ine +pri ze +mar ist +re creating +me de +woo den +find lay +ro fl +n di +compreh end +yu go +y ü +to work +u fos +son ar +pi ston +recor ding +tent ative +art forsale +pel lets +fre do +ÙĪ ر +mu ses +custom ization +pro found +is ner +ide ally +si am +plan kton +cm dr +man ger +fran ken +customiz able +ठ® +walk away +swi vel +vast ly +no ton +lex a +ex moor +z as +tan te +reduc tions +lol ly +hip sters +benef ited +ë ² +ww www +mascul ine +fi ji +dre y +ph ill +ane ous +nic ol +men dez +disapp ro +ch ner +through s +shen mue +east man +ðŁIJ İ +yu ck +under tale +re ys +go beavs +eng en +c na +mer r +bir k +ãģ¨ç¹ĭãģ ĮãĤĬãģŁãģĦ +âĥ£ @ +yn na +ste ed +offen der +at um +vani shing +presi denti +love them +g nocchi +fri ggin +per il +mad hya +ag ne +dee jay +mar nock +m tb +fold able +@ ___ +stand re +bron x +bow ski +fin ite +cro ckett +b sf +ge tit +seren awilliams +mir o +ignati us +sla y +rin se +fon due +sel dom +s more +gan i +dy ce +dmit ry +cru mb +late post +pri mark +oh ana +flor als +do a +remembrance day +d ds +azi one +toon ami +air port +æĿ ± +th ad +fi st +dine sh +dr who +ad words +admi rer +pro je +kyrgy z +à « +manife station +le wan +j ic +thi bau +le ased +van ity +nouri shed +never theless +aug mente +fu elled +che ad +wil shere +ru di +p z +my co +mor ro +herbali fe +hardro ck +de man +dre ality +sp ades +ce vic +bha i +bar on +ultimat efan +hou news +to bi +stru t +ke el +affili ation +the masters +sm al +hu e +este ban +con v +om nic +datab ases +co v +ter ti +st g +snoop dogg +metab ol +leth bridge +ðŁı» âĢįâĻĢï¸ı +year ling +residente vil +nws l +iy aki +griez mann +c ous +ðŁĵĿ : +tor ian +sam i +ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ðŁĶ¥ +g are +alli ances +whit field +we ther +refin ing +coy i +kra ken +ðŁĺĺ âĿ¤ +singul arity +lil i +h ns +bol dand +waw rinka +misogy ny +lo vers +c q +b dg +ad ona +gar ter +women of +sc d +recogn ising +mun a +str ou +sign alling +lare do +hell boy +alek sand +un available +pedi atric +as in +mer ia +ri shi +futuri sm +w ye +polari zed +e we +pro pel +in forms +cre ase +~ " +arti ston +like for +heidel berg +er ra +life in +len ny +inter rupt +cohe rent +ca z +vick ers +le veled +f bs +cab ins +bu mmed +apost les +we h +ten don +souven irs +infu ri +pier ce +asse t +m las +go th +di ggin +ann as +yl or +th waite +sw el +pan era +mur derers +croo ked +bs go +ac u +a on +re an +one of +ko hl +bloo dh +pest icide +lost dog +fle xing +ëĤ ĺ +su pra +eter nally +ðŁļ Ļ +pa olo +ol an +mom o +is elle +captain marvel +s lou +mistak enly +akhi lesh +mer t +il inan +bu on +bal kan +mir ro +mill en +der ail +dam on +tit i +bi os +re don +pic ard +par te +ðŁ¤ Ł +Ø º +son ics +fir sth +dd c +veg ans +tur ban +ni gan +lot tie +lyn don +star buck +pink floyd +life styles +am ara +a she +r sc +val a +sm er +cw gc +cli ent +buen as +jag an +coo ps +ðŁijij ðŁijij +speci alizes +snag ged +g lar +ben net +wildlife wednesday +bow den +pi k +art in +empor ium +ar l +re ba +pas ser +disappo ints +additi ve +âľĬ ðŁı½ +bay er +missou la +ha skell +comm ences +ni x +ne man +explo ited +plastic surgery +cc d +aso cial +vo t +sie gel +fro ome +kap am +far a +e ha +pro bes +mw f +meet ing +p bb +ak ins +mistle toe +kingdom hearts +for kids +ec r +bal e +escor ts +adidas originals +k wa +k ts +hallo ffame +ðŁĺį . +wag s +pot ted +o wing +honey comb +he fty +uro logy +mer le +b pd +stri pping +re ich +k state +gu ay +yon ge +shak ti +g loom +bat t +son om +n ery +el ba +blan ks +hel le +triple ts +bom bay +ak arta +ab ia +transm itted +rol f +ja is +angular js +fi erc +m ss +trac e +ॠĩ +tom bs +old man +kom bucha +fo l +e health +cere als +are lli +in ari +ðŁĴ © +wo l +liber ties +fa wn +af firm +nun avut +hyster ical +k drama +art es +âĢ¢âĢ¢âĢ¢âĢ¢ âĢ¢âĢ¢âĢ¢âĢ¢ +valent in +man slaughter +gal es +eo in +energi zed +del s +with draws +st les +sar castic +ram esh +incredi bles +lock hart +ya wn +ultimatefan live +oooooooo oooooooo +mu en +guru dev +te er +pe eling +new snow +lingui stics +direc tv +ag end +uni lever +ru ger +han dedly +ero se +li mel +the c +royal ties +fini shers +nr g +m gt +fid get +com ps +bac on +aggre ssively +ab it +ch â +tar de +slu gger +q anda +gre ening +d ats +ensla ved +spec tor +o ye +fre ef +b hand +stop brexit +mis conceptions +cav a +ðŁĺįðŁĺįðŁĺįðŁĺį ðŁĺįðŁĺįðŁĺįðŁĺį +multit asking +hou sel +ferre ira +cen time +ank les +jo dh +hel ly +fro me +out tuesday +nar nia +bal aji +l bloggers +jyo ti +ðŁį ĩ +lan cia +cap ri +y ap +nat ash +down fall +." âĢĶ +à ® +ligam ent +coat ings +ai ded +hi ko +fall ing +encryp ted +yeg food +infringe ment +cu di +ce p +ðŁĺį ðŁĺĤ +tra d +super rugby +ed win +wh iche +vi meo +lay ne +in vigor +he he +dubrov nik +bie ber +u tr +sham an +op ers +ham ill +en ig +di f +ar um +scrap book +min h +diver gence +mckin non +life time +guter res +wil le +ple as +patt y +mic ron +k z +dom aine +ru sher +m ds +ches ney +screw driver +âģ© , +sle dge +hau er +chan a +stam ina +sprink ler +pl n +he ff +bol ton +om on +car rington +accor dion +jor ge +inter ception +in puts +gu ll +tran scription +vanu atu +it ical +eth os +tic h +spac ey +pee king +u mi +ha ger +psycho tic +illi an +illi a +bonnar oo +an ese +pu c +laghate parth +en hall +econom ical +dre dge +% - +u we +tu bular +scoun cil +pe asants +fl er +tumb ler +he p +ford ham +row ley +initi als +ev asion +er nation +plu gins +coch ran +c attle +acid ity +ðŁİĬ ðŁİī +re grann +jump man +ef ace +x ma +patri archy +esco bar +cristi an +tip ton +nu eva +hack ney +back seat +kill arney +aid an +sta dion +simul taneous +ida ho +a je +u th +figu re +clo s +bur k +volun tar +rec ite +macfar lane +cur few +bou do +w gn +sti x +sla p +scrat ched +philli p +jour ne +ex pelled +wa z +u ke +tati ana +ou e +ho pp +dimit ri +ðŁĵ £ +mato logist +electri fying +blu ffs +bill smafia +az cardinals +y aa +x mas +shar a +r ith +g ills +dre s +bar ton +authori zation +imperi alism +home of +to do +foot path +band width +visit spain +moh sin +erup ted +mi ki +insig nia +mike l +ss h +ger a +bank holiday +aw an +t weak +star craft +e al +construc tion +skelet ons +le ep +ine m +bar clay +ship wreck +monsi eur +yo h +ron t +form ative +ser o +le p +horse man +hoo sier +haz mat +cylin ders +cen ti +ðŁĴ¥ðŁĴ¥ ðŁĴ¥ +re em +na ire +mus ically +gras shopper +est onian +termin ology +ro main +blogger rt +tox in +stan ce +cultiv ated +an ast +ðŁIJ į +shi mano +go pher +ene i +recycla ble +gam ification +fight for +c q +avoc ados +ke ys +eli ke +gly cer +shak ur +mobili zation +gal ley +expla in +ex changed +pe th +obe dience +illa ge +en nis +ãĥ ŀ +wi v +walla bies +ma ar +ig ers +fin tech +fin alized +wo j +meaning less +in field +onna ise +e et +bron te +pass ages +ðŁij § +strick land +northern lights +lom ond +h tc +wr ay +shi fter +di alog +ðŁį į +>> >>>> +te atime +ste ch +sic huan +qu ill +fran ca +comple mentary +bar rington +marcu s +mal am +goo oo +for sa +elec tra +af s +âĹ Ĩ +tri fe +sn azzy +fo lia +and olan +after dark +wood son +stra de +litt lest +o gun +con wy +co wards +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤ +íĬ ¸ +se ul +mur phy +dun ks +kapil shar +jo achim +wom ack +equal ity +aver ages +a ine +ðŁ¦ Ī +tac ular +dis ability +u ked +mid century +bar thol +teas ers +tab ern +nj caa +sp out +op i +ku bball +bl om +so ar +popu lism +meth yl +ðŁijĬ ðŁı¼ +o spre +alo ils +ðŁĵ ĸ +ðŁĮ ļ +x er +sp illing +publ ica +car dam +adi sh +sa cha +p kg +bu da +lyric ist +i bc +gru mp +ho ver +hal ep +anti body +anem one +âĻ¥âĻ¥ âĻ¥âĻ¥ +m cl +litho graph +cc u +s fest +path ic +calli ster +otta wa +gun sn +rut ger +hali but +en vision +differenti ate +ðŁļĢ ðŁļĢ +pir an +lat el +uc n +trou bad +ra ine +fierc ely +learn english +lea se +wex mondays +em it +dray ton +bur rell +scuba diving +hol ler +dr u +clo cked +w ral +ap ro +trans lucent +w bo +patri arch +mo ja +lan nister +fish ery +ne derland +mil dly +mi rai +ma ko +ja p +ðŁĺ©ðŁĺ© ðŁĺ© +pro statec +p anna +ar ama +under taking +tomp kins +ne op +soli ds +sav oury +e ames +cut lery +wood bridge +steam er +ri zzo +wild cat +rat na +lamin ated +kin eni +jal ap +ai des +acknowle dges +?! ?!?! +! ðŁİī +w afc +mag gio +ha ves +dar je +of i +gr il +v asi +bru x +mo hd +fake speare +arn old +r mb +for be +wal leye +ro di +therapeu tics +strate gi +ob ste +mu dder +download able +dd ings +d ca +asi angames +campe on +appropri ation +th century +ram atta +dra ped +bul lion +mu c +one x +se greg +ophel ia +bod ily +âĿ¤ ðŁĺį +wi zar +te ased +ade my +to id +sur a +lazar us +sn ickers +ma se +lo h +bow ed +bibli o +x change +har lan +gho shal +flavor ful +bha gat +alle z +whiche ver +ten stein +disc er +organ iser +mt g +dream liner +t se +hok kaido +mo k +indulg ent +hick man +blin ded +al yn +aaa ah +sp ool +lough borough +inter pret +et v +aristo tle +optimi zing +avici i +madu rai +ju li +naw az +mat chups +ab ide +paint ing +w elling +vel i +octag on +in scribed +po king +plac er +life cycle +kili g +g sp +eli ves +cle ments +na sheed +me sut +incarcer ated +dist illed +wal ang +delic acy +del gado +che z +ch ita +ad ero +tu x +pati l +o do +abh cosmetics +tv c +p bc +in accurate +hardwork paysoff +ball er +quot ation +merchandi sing +ga stri +defen ses +dro gba +bex hill +ban kno +win ona +si eg +p gs +hahah ha +agu chi +su bram +mirac le +de sch +li bre +ba cher +ent ine +bbcra di +lou dest +r ps +pi erc +fr yer +storm trooper +rafael nadal +pas co +exhau stion +epic onetsy +rc tid +kel lie +ga ines +d bz +sm riti +s bridge +lim ited +cla w +technic al +bio graphical +ado red +ภ° +exclu de +ac adia +key boards +fur man +so ca +sur u +ni ps +sw aps +server less +run e +pu ffy +north ampton +nish ings +hen der +cartri dges +gun shot +ðŁĵ ¹ +fil ament +respon dents +pey ton +mountaine er +mer ging +life span +intimid ation +p afc +nl wx +expan sive +pur r +f ck +ca e +at ti +tele thon +so hn +mend el +lo pes +dor i +un broken +te red +tast ings +in active +disin tegr +t assel +share the +pi ano +is lay +air space +z awa +ricci ardo +ming ton +fresh er +cur ry +re vs +pharo ah +h mv +exhilar ating +wh oo +lin kin +kri spy +competen cy +ste wards +ne bu +kat su +ad mins +baz ar +as ar +giving back +s summit +song z +lin us +raj kumar +farm ington +fanta sia +ðŁĺ´ ðŁĺ´ +so bri +lis se +barry more +pri sm +blo b +sen ew +mono xide +exp ire +eigh teen +di pper +xi ao +kil t +hin ch +bbc sport +bam boo +p ter +ex al +ðŁ¦ ĭ +ham lin +expe ditions +star gazing +food security +wy lie +ul f +st ingly +on storm +lo eb +bro ome +bn ha +pancre atic +eli ve +!!!!!!!! !!! +ther apper +ortho pedic +avengers endgame +antit rust +ìļ ° +go te +om d +off side +gy llen +win eries +white water +ad l +lu pita +exce eds +consi sted +chew bacca +ash leigh +nhl jets +is san +sh ld +hay at +cran berries +ðŁ¤ĺ ðŁı½ +rock the +spring training +fall out +dairy free +wa j +un decided +so wn +rc n +north wales +htt r +fu mble +d its +comp elled +popu list +min ted +blan chett +. '' +pro pulsion +m illa +au berg +her tz +h ta +u daipur +serendip ity +azte cs +als ace +ðŁIJ ij +lu n +sho es +char li +gar za +ðŁĴ Ł +pro biotics +fox tv +ol is +mi ff +loc alized +diffu ser +si gue +fun ko +rend ous +ðŁĴ ij +jeky ll diff --git a/annotator/clip_vision/preprocessor_config.json b/annotator/clip_vision/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..5a12a1eb250987a4eee0e3e7d7338c4b22724be1 --- /dev/null +++ b/annotator/clip_vision/preprocessor_config.json @@ -0,0 +1,19 @@ +{ + "crop_size": 224, + "do_center_crop": true, + "do_normalize": true, + "do_resize": true, + "feature_extractor_type": "CLIPFeatureExtractor", + "image_mean": [ + 0.48145466, + 0.4578275, + 0.40821073 + ], + "image_std": [ + 0.26862954, + 0.26130258, + 0.27577711 + ], + "resample": 3, + "size": 224 +} diff --git a/annotator/clip_vision/tokenizer.json b/annotator/clip_vision/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..580c79c6862f31d1f9bd08dd1a415ba0d0502cd9 --- /dev/null +++ b/annotator/clip_vision/tokenizer.json @@ -0,0 +1,98393 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 49406, + "content": "<|startoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": true, + "special": true + }, + { + "id": 49407, + "content": "<|endoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": { + "type": "Sequence", + "normalizers": [ + { + "type": "NFC" + }, + { + "type": "Replace", + "pattern": { + "Regex": "\\s+" + }, + "content": " " + }, + { + "type": "Lowercase" + } + ] + }, + "pre_tokenizer": { + "type": "Sequence", + "pretokenizers": [ + { + "type": "Split", + "pattern": { + "Regex": "'s|'t|'re|'ve|'m|'ll|'d|[\\p{L}]+|[\\p{N}]|[^\\s\\p{L}\\p{N}]+" + }, + "behavior": "Removed", + "invert": true + }, + { + "type": "ByteLevel", + "add_prefix_space": false, + "trim_offsets": true + } + ] + }, + "post_processor": { + "type": "RobertaProcessing", + "sep": [ + "<|endoftext|>", + 49407 + ], + "cls": [ + "<|startoftext|>", + 49406 + ], + "trim_offsets": false, + "add_prefix_space": false + }, + "decoder": { + "type": "ByteLevel", + "add_prefix_space": true, + "trim_offsets": true + }, + "model": { + "type": "BPE", + "dropout": null, + "unk_token": "<|endoftext|>", + "continuing_subword_prefix": "", + "end_of_word_suffix": "", + "fuse_unk": false, + "vocab": { + "!": 0, + "\"": 1, + "#": 2, + "$": 3, + "%": 4, + "&": 5, + "'": 6, + "(": 7, + ")": 8, + "*": 9, + "+": 10, + ",": 11, + "-": 12, + ".": 13, + "/": 14, + "0": 15, + "1": 16, + "2": 17, + "3": 18, + "4": 19, + "5": 20, + "6": 21, + "7": 22, + "8": 23, + "9": 24, + ":": 25, + ";": 26, + "<": 27, + "=": 28, + ">": 29, + "?": 30, + "@": 31, + "A": 32, + "B": 33, + "C": 34, + "D": 35, + "E": 36, + "F": 37, + "G": 38, + "H": 39, + "I": 40, + "J": 41, + "K": 42, + "L": 43, + "M": 44, + "N": 45, + "O": 46, + "P": 47, + "Q": 48, + "R": 49, + "S": 50, + "T": 51, + "U": 52, + "V": 53, + "W": 54, + "X": 55, + "Y": 56, + "Z": 57, + "[": 58, + "\\": 59, + "]": 60, + "^": 61, + "_": 62, + "`": 63, + "a": 64, + "b": 65, + "c": 66, + "d": 67, + "e": 68, + "f": 69, + "g": 70, + "h": 71, + "i": 72, + "j": 73, + "k": 74, + "l": 75, + "m": 76, + "n": 77, + "o": 78, + "p": 79, + "q": 80, + "r": 81, + "s": 82, + "t": 83, + "u": 84, + "v": 85, + "w": 86, + "x": 87, + "y": 88, + "z": 89, + "{": 90, + "|": 91, + "}": 92, + "~": 93, + "¡": 94, + "¢": 95, + "£": 96, + "¤": 97, + "¥": 98, + "¦": 99, + "§": 100, + "¨": 101, + "©": 102, + "ª": 103, + "«": 104, + "¬": 105, + "®": 106, + "¯": 107, + "°": 108, + "±": 109, + "²": 110, + "³": 111, + "´": 112, + "µ": 113, + "¶": 114, + "·": 115, + "¸": 116, + "¹": 117, + "º": 118, + "»": 119, + "¼": 120, + "½": 121, + "¾": 122, + "¿": 123, + "À": 124, + "Á": 125, + "Â": 126, + "Ã": 127, + "Ä": 128, + "Å": 129, + "Æ": 130, + "Ç": 131, + "È": 132, + "É": 133, + "Ê": 134, + "Ë": 135, + "Ì": 136, + "Í": 137, + "Î": 138, + "Ï": 139, + "Ð": 140, + "Ñ": 141, + "Ò": 142, + "Ó": 143, + "Ô": 144, + "Õ": 145, + "Ö": 146, + "×": 147, + "Ø": 148, + "Ù": 149, + "Ú": 150, + "Û": 151, + "Ü": 152, + "Ý": 153, + "Þ": 154, + "ß": 155, + "à": 156, + "á": 157, + "â": 158, + "ã": 159, + "ä": 160, + "å": 161, + "æ": 162, + "ç": 163, + "è": 164, + "é": 165, + "ê": 166, + "ë": 167, + "ì": 168, + "í": 169, + "î": 170, + "ï": 171, + "ð": 172, + "ñ": 173, + "ò": 174, + "ó": 175, + "ô": 176, + "õ": 177, + "ö": 178, + "÷": 179, + "ø": 180, + "ù": 181, + "ú": 182, + "û": 183, + "ü": 184, + "ý": 185, + "þ": 186, + "ÿ": 187, + "Ā": 188, + "ā": 189, + "Ă": 190, + "ă": 191, + "Ą": 192, + "ą": 193, + "Ć": 194, + "ć": 195, + "Ĉ": 196, + "ĉ": 197, + "Ċ": 198, + "ċ": 199, + "Č": 200, + "č": 201, + "Ď": 202, + "ď": 203, + "Đ": 204, + "đ": 205, + "Ē": 206, + "ē": 207, + "Ĕ": 208, + "ĕ": 209, + "Ė": 210, + "ė": 211, + "Ę": 212, + "ę": 213, + "Ě": 214, + "ě": 215, + "Ĝ": 216, + "ĝ": 217, + "Ğ": 218, + "ğ": 219, + "Ġ": 220, + "ġ": 221, + "Ģ": 222, + "ģ": 223, + "Ĥ": 224, + "ĥ": 225, + "Ħ": 226, + "ħ": 227, + "Ĩ": 228, + "ĩ": 229, + "Ī": 230, + "ī": 231, + "Ĭ": 232, + "ĭ": 233, + "Į": 234, + "į": 235, + "İ": 236, + "ı": 237, + "IJ": 238, + "ij": 239, + "Ĵ": 240, + "ĵ": 241, + "Ķ": 242, + "ķ": 243, + "ĸ": 244, + "Ĺ": 245, + "ĺ": 246, + "Ļ": 247, + "ļ": 248, + "Ľ": 249, + "ľ": 250, + "Ŀ": 251, + "ŀ": 252, + "Ł": 253, + "ł": 254, + "Ń": 255, + "!": 256, + "\"": 257, + "#": 258, + "$": 259, + "%": 260, + "&": 261, + "'": 262, + "(": 263, + ")": 264, + "*": 265, + "+": 266, + ",": 267, + "-": 268, + ".": 269, + "/": 270, + "0": 271, + "1": 272, + "2": 273, + "3": 274, + "4": 275, + "5": 276, + "6": 277, + "7": 278, + "8": 279, + "9": 280, + ":": 281, + ";": 282, + "<": 283, + "=": 284, + ">": 285, + "?": 286, + "@": 287, + "A": 288, + "B": 289, + "C": 290, + "D": 291, + "E": 292, + "F": 293, + "G": 294, + "H": 295, + "I": 296, + "J": 297, + "K": 298, + "L": 299, + "M": 300, + "N": 301, + "O": 302, + "P": 303, + "Q": 304, + "R": 305, + "S": 306, + "T": 307, + "U": 308, + "V": 309, + "W": 310, + "X": 311, + "Y": 312, + "Z": 313, + "[": 314, + "\\": 315, + "]": 316, + "^": 317, + "_": 318, + "`": 319, + "a": 320, + "b": 321, + "c": 322, + "d": 323, + "e": 324, + "f": 325, + "g": 326, + "h": 327, + "i": 328, + "j": 329, + "k": 330, + "l": 331, + "m": 332, + "n": 333, + "o": 334, + "p": 335, + "q": 336, + "r": 337, + "s": 338, + "t": 339, + "u": 340, + "v": 341, + "w": 342, + "x": 343, + "y": 344, + "z": 345, + "{": 346, + "|": 347, + "}": 348, + "~": 349, + "¡": 350, + "¢": 351, + "£": 352, + "¤": 353, + "¥": 354, + "¦": 355, + "§": 356, + "¨": 357, + "©": 358, + "ª": 359, + "«": 360, + "¬": 361, + "®": 362, + "¯": 363, + "°": 364, + "±": 365, + "²": 366, + "³": 367, + "´": 368, + "µ": 369, + "¶": 370, + "·": 371, + "¸": 372, + "¹": 373, + "º": 374, + "»": 375, + "¼": 376, + "½": 377, + "¾": 378, + "¿": 379, + "À": 380, + "Á": 381, + "Â": 382, + "Ã": 383, + "Ä": 384, + "Å": 385, + "Æ": 386, + "Ç": 387, + "È": 388, + "É": 389, + "Ê": 390, + "Ë": 391, + "Ì": 392, + "Í": 393, + "Î": 394, + "Ï": 395, + "Ð": 396, + "Ñ": 397, + "Ò": 398, + "Ó": 399, + "Ô": 400, + "Õ": 401, + "Ö": 402, + "×": 403, + "Ø": 404, + "Ù": 405, + "Ú": 406, + "Û": 407, + "Ü": 408, + "Ý": 409, + "Þ": 410, + "ß": 411, + "à": 412, + "á": 413, + "â": 414, + "ã": 415, + "ä": 416, + "å": 417, + "æ": 418, + "ç": 419, + "è": 420, + "é": 421, + "ê": 422, + "ë": 423, + "ì": 424, + "í": 425, + "î": 426, + "ï": 427, + "ð": 428, + "ñ": 429, + "ò": 430, + "ó": 431, + "ô": 432, + "õ": 433, + "ö": 434, + "÷": 435, + "ø": 436, + "ù": 437, + "ú": 438, + "û": 439, + "ü": 440, + "ý": 441, + "þ": 442, + "ÿ": 443, + "Ā": 444, + "ā": 445, + "Ă": 446, + "ă": 447, + "Ą": 448, + "ą": 449, + "Ć": 450, + "ć": 451, + "Ĉ": 452, + "ĉ": 453, + "Ċ": 454, + "ċ": 455, + "Č": 456, + "č": 457, + "Ď": 458, + "ď": 459, + "Đ": 460, + "đ": 461, + "Ē": 462, + "ē": 463, + "Ĕ": 464, + "ĕ": 465, + "Ė": 466, + "ė": 467, + "Ę": 468, + "ę": 469, + "Ě": 470, + "ě": 471, + "Ĝ": 472, + "ĝ": 473, + "Ğ": 474, + "ğ": 475, + "Ġ": 476, + "ġ": 477, + "Ģ": 478, + "ģ": 479, + "Ĥ": 480, + "ĥ": 481, + "Ħ": 482, + "ħ": 483, + "Ĩ": 484, + "ĩ": 485, + "Ī": 486, + "ī": 487, + "Ĭ": 488, + "ĭ": 489, + "Į": 490, + "į": 491, + "İ": 492, + "ı": 493, + "IJ": 494, + "ij": 495, + "Ĵ": 496, + "ĵ": 497, + "Ķ": 498, + "ķ": 499, + "ĸ": 500, + "Ĺ": 501, + "ĺ": 502, + "Ļ": 503, + "ļ": 504, + "Ľ": 505, + "ľ": 506, + "Ŀ": 507, + "ŀ": 508, + "Ł": 509, + "ł": 510, + "Ń": 511, + "in": 512, + "th": 513, + "an": 514, + "re": 515, + "ar": 516, + "er": 517, + "the": 518, + "ing": 519, + "ou": 520, + "on": 521, + "st": 522, + "or": 523, + "en": 524, + "on": 525, + "al": 526, + "at": 527, + "er": 528, + "it": 529, + "in": 530, + "to": 531, + "ro": 532, + "is": 533, + "le": 534, + "ic": 535, + "at": 536, + "and": 537, + "ed": 538, + "of": 539, + "ch": 540, + "or": 541, + "es": 542, + "il": 543, + "el": 544, + "st": 545, + "ac": 546, + "om": 547, + "am": 548, + "lo": 549, + "an": 550, + "ay": 551, + "sh": 552, + "ri": 553, + "li": 554, + "ti": 555, + "for": 556, + "ne": 557, + "ðŁ": 558, + "ra": 559, + "ha": 560, + "de": 561, + "ol": 562, + "ve": 563, + "si": 564, + "ur": 565, + "al": 566, + "se": 567, + "'s": 568, + "un": 569, + "di": 570, + "be": 571, + "la": 572, + "wh": 573, + "oo": 574, + "day": 575, + "en": 576, + "ma": 577, + "no": 578, + "le": 579, + "to": 580, + "our": 581, + "ir": 582, + "gh": 583, + "wit": 584, + "it": 585, + "yo": 586, + "as": 587, + "sp": 588, + "this": 589, + "ts": 590, + "ati": 591, + "you": 592, + "with": 593, + "ad": 594, + "is": 595, + "ab": 596, + "ly": 597, + "we": 598, + "the": 599, + "te": 600, + "as": 601, + "ag": 602, + "vi": 603, + "pp": 604, + "su": 605, + "ho": 606, + "my": 607, + "..": 608, + "bu": 609, + "com": 610, + "se": 611, + "ers": 612, + "me": 613, + "me": 614, + "all": 615, + "con": 616, + "mo": 617, + "ke": 618, + "ge": 619, + "out": 620, + "ent": 621, + "co": 622, + "fe": 623, + "ver": 624, + "ar": 625, + "fro": 626, + "au": 627, + "po": 628, + "ce": 629, + "ght": 630, + "are": 631, + "ss": 632, + "from": 633, + "ch": 634, + "tr": 635, + "oun": 636, + "one": 637, + "by": 638, + "do": 639, + "th": 640, + "wor": 641, + "ere": 642, + "ke": 643, + "pro": 644, + "for": 645, + "ds": 646, + "bo": 647, + "ta": 648, + "we": 649, + "go": 650, + "he": 651, + "ter": 652, + "ing": 653, + "de": 654, + "be": 655, + "ation": 656, + "mor": 657, + "ay": 658, + "ex": 659, + "ill": 660, + "pe": 661, + "ks": 662, + "sc": 663, + "lu": 664, + "fu": 665, + "qu": 666, + "ver": 667, + "ðŁĺ": 668, + "ju": 669, + "mu": 670, + "ate": 671, + "and": 672, + "ve": 673, + "king": 674, + "mar": 675, + "op": 676, + "hi": 677, + "...": 678, + "pre": 679, + "ad": 680, + "ru": 681, + "that": 682, + "jo": 683, + "of": 684, + "ce": 685, + "new": 686, + "am": 687, + "ap": 688, + "gre": 689, + "ss": 690, + "du": 691, + "now": 692, + "ye": 693, + "ting": 694, + "your": 695, + "ity": 696, + "ni": 697, + "ci": 698, + "par": 699, + "gu": 700, + "fi": 701, + "af": 702, + "per": 703, + "ter": 704, + "up": 705, + "so": 706, + "gi": 707, + "ons": 708, + "gr": 709, + "ge": 710, + "br": 711, + "pl": 712, + "'t": 713, + "mi": 714, + "ine": 715, + "wee": 716, + "bi": 717, + "us": 718, + "sho": 719, + "have": 720, + "today": 721, + "av": 722, + "man": 723, + "ent": 724, + "ack": 725, + "ure": 726, + "our": 727, + "âĢ": 728, + "cu": 729, + "ld": 730, + "loo": 731, + "im": 732, + "ice": 733, + "som": 734, + "fin": 735, + "red": 736, + "ren": 737, + "ood": 738, + "was": 739, + "tion": 740, + "pi": 741, + "ir": 742, + "ther": 743, + "ty": 744, + "ph": 745, + "ard": 746, + "ec": 747, + "!!": 748, + "mon": 749, + "more": 750, + "will": 751, + "tra": 752, + "can": 753, + "col": 754, + "pu": 755, + "te": 756, + "wn": 757, + "mb": 758, + "so": 759, + "iti": 760, + "just": 761, + "ning": 762, + "here": 763, + "tu": 764, + "pa": 765, + "pr": 766, + "but": 767, + "what": 768, + "ally": 769, + "fir": 770, + "min": 771, + "ca": 772, + "ant": 773, + "sa": 774, + "ted": 775, + "ev": 776, + "ment": 777, + "fa": 778, + "get": 779, + "ame": 780, + "about": 781, + "gra": 782, + "not": 783, + "happ": 784, + "ays": 785, + "man": 786, + "his": 787, + "time": 788, + "like": 789, + "gh": 790, + "has": 791, + "than": 792, + "love": 793, + "art": 794, + "ste": 795, + "ding": 796, + "he": 797, + "cre": 798, + "ws": 799, + "wat": 800, + "der": 801, + "ite": 802, + "ser": 803, + "ace": 804, + "age": 805, + "end": 806, + "str": 807, + "aw": 808, + "stor": 809, + "re": 810, + "car": 811, + "ell": 812, + "all": 813, + "ps": 814, + "fri": 815, + "pho": 816, + "por": 817, + "do": 818, + "ak": 819, + "wi": 820, + "fre": 821, + "who": 822, + "shi": 823, + "boo": 824, + "son": 825, + "ell": 826, + "when": 827, + "ill": 828, + "how": 829, + "great": 830, + "win": 831, + "el": 832, + "bl": 833, + "ssi": 834, + "ali": 835, + "some": 836, + "ðŁĴ": 837, + "ton": 838, + "der": 839, + "les": 840, + "pla": 841, + "ï¸": 842, + "ed": 843, + "sch": 844, + "hu": 845, + "ong": 846, + "don": 847, + "ki": 848, + "sh": 849, + "ann": 850, + "cor": 851, + "..": 852, + "ound": 853, + "az": 854, + "ine": 855, + "ary": 856, + "ful": 857, + "stu": 858, + "ould": 859, + "sti": 860, + "go": 861, + "see": 862, + "able": 863, + "ars": 864, + "ll": 865, + "mis": 866, + "ber": 867, + "ck": 868, + "wa": 869, + "ents": 870, + "no": 871, + "sig": 872, + "fe": 873, + "first": 874, + "et": 875, + "spe": 876, + "ack": 877, + "if": 878, + "ous": 879, + "'m": 880, + "ster": 881, + "app": 882, + "ang": 883, + "ance": 884, + "ans": 885, + "good": 886, + "bre": 887, + "ever": 888, + "they": 889, + "tic": 890, + "come": 891, + "off": 892, + "back": 893, + "ase": 894, + "ings": 895, + "old": 896, + "ight": 897, + "fo": 898, + "her": 899, + "happy": 900, + "pic": 901, + "its": 902, + "ving": 903, + "us": 904, + "mat": 905, + "hom": 906, + "dy": 907, + "em": 908, + "sk": 909, + "ying": 910, + "their": 911, + "led": 912, + "ry": 913, + "ul": 914, + "har": 915, + "ck": 916, + "ton": 917, + "onal": 918, + "hel": 919, + "ric": 920, + "bir": 921, + "vie": 922, + "way": 923, + "tri": 924, + "da": 925, + "ple": 926, + "bro": 927, + "sto": 928, + "ool": 929, + "night": 930, + "tru": 931, + "ba": 932, + "read": 933, + "res": 934, + "year": 935, + "fr": 936, + "tor": 937, + "als": 938, + "coun": 939, + "cla": 940, + "ture": 941, + "vel": 942, + "ated": 943, + "lec": 944, + "end": 945, + "thing": 946, + "vo": 947, + "ici": 948, + "best": 949, + "can": 950, + "work": 951, + "last": 952, + "after": 953, + "ence": 954, + "pri": 955, + "pe": 956, + "es": 957, + "il": 958, + "âĢ¦": 959, + "dre": 960, + "ys": 961, + "over": 962, + "ies": 963, + "ðŁij": 964, + "comm": 965, + "tw": 966, + "ink": 967, + "sun": 968, + "cl": 969, + "life": 970, + "tt": 971, + "ach": 972, + "land": 973, + "sy": 974, + "tre": 975, + "tal": 976, + "pol": 977, + "sm": 978, + "duc": 979, + "sal": 980, + "ft": 981, + "'re": 982, + "che": 983, + "war": 984, + "tur": 985, + "ations": 986, + "ach": 987, + "ms": 988, + "ile": 989, + "pm": 990, + "ough": 991, + "ate": 992, + "star": 993, + "week": 994, + "!!!": 995, + "clu": 996, + "there": 997, + "ner": 998, + "tom": 999, + "sel": 1000, + "ï¸ı": 1001, + "world": 1002, + "ves": 1003, + "cam": 1004, + "got": 1005, + "inter": 1006, + "off": 1007, + "um": 1008, + "tonight": 1009, + "other": 1010, + "hou": 1011, + "look": 1012, + "je": 1013, + "id": 1014, + "sion": 1015, + "beau": 1016, + "att": 1017, + "eli": 1018, + "ort": 1019, + "rec": 1020, + "ff": 1021, + "ster": 1022, + "supp": 1023, + "gen": 1024, + "been": 1025, + "ily": 1026, + "team": 1027, + "mm": 1028, + "ic": 1029, + "peop": 1030, + "itt": 1031, + "ats": 1032, + "only": 1033, + "mber": 1034, + "eng": 1035, + "bri": 1036, + "mp": 1037, + "know": 1038, + "bur": 1039, + "bar": 1040, + "ins": 1041, + "low": 1042, + "she": 1043, + "row": 1044, + "âĿ": 1045, + "tro": 1046, + "people": 1047, + "via": 1048, + "low": 1049, + "aga": 1050, + "bet": 1051, + "xt": 1052, + "fac": 1053, + "char": 1054, + "ear": 1055, + "wal": 1056, + "sen": 1057, + "fam": 1058, + "ble": 1059, + "nati": 1060, + "ish": 1061, + "nor": 1062, + "game": 1063, + "live": 1064, + "sco": 1065, + "ley": 1066, + "don": 1067, + "ick": 1068, + "ball": 1069, + "very": 1070, + "these": 1071, + "pan": 1072, + "ia": 1073, + "ating": 1074, + "cr": 1075, + "are": 1076, + "gir": 1077, + "make": 1078, + "stre": 1079, + "show": 1080, + ".\"": 1081, + "fl": 1082, + "up": 1083, + "dr": 1084, + "thanks": 1085, + "illi": 1086, + "wom": 1087, + "sts": 1088, + "ig": 1089, + "sur": 1090, + "every": 1091, + "cur": 1092, + "view": 1093, + "let": 1094, + "into": 1095, + "most": 1096, + "na": 1097, + "indi": 1098, + "gar": 1099, + "had": 1100, + "sou": 1101, + "ved": 1102, + "ant": 1103, + "ition": 1104, + "made": 1105, + "fol": 1106, + "uni": 1107, + "ited": 1108, + "ðŁı": 1109, + "ical": 1110, + "thr": 1111, + "ready": 1112, + "chec": 1113, + "dra": 1114, + "kes": 1115, + "book": 1116, + "ep": 1117, + "sic": 1118, + "morning": 1119, + "news": 1120, + "cau": 1121, + "ct": 1122, + "well": 1123, + "anc": 1124, + "photo": 1125, + "than": 1126, + "ors": 1127, + "birth": 1128, + "gg": 1129, + "out": 1130, + "next": 1131, + "some": 1132, + "ening": 1133, + "story": 1134, + "chri": 1135, + "down": 1136, + "home": 1137, + "ffe": 1138, + "free": 1139, + "da": 1140, + "bor": 1141, + "fil": 1142, + "cial": 1143, + "thank": 1144, + "side": 1145, + "lear": 1146, + "que": 1147, + "line": 1148, + "ten": 1149, + "ates": 1150, + "years": 1151, + "my": 1152, + "photo": 1153, + "beauti": 1154, + "right": 1155, + "nu": 1156, + "form": 1157, + "ship": 1158, + "ban": 1159, + "ther": 1160, + "days": 1161, + "gam": 1162, + "ason": 1163, + "gy": 1164, + "ðŁİ": 1165, + "birthday": 1166, + "set": 1167, + "ick": 1168, + "et": 1169, + "still": 1170, + "coming": 1171, + "take": 1172, + "ðŁĩ": 1173, + "bb": 1174, + "sol": 1175, + "son": 1176, + "den": 1177, + "ep": 1178, + "music": 1179, + "them": 1180, + "den": 1181, + "why": 1182, + "foo": 1183, + "cra": 1184, + "amaz": 1185, + "wn": 1186, + "hol": 1187, + "tting": 1188, + "wr": 1189, + "ue": 1190, + "mag": 1191, + "cro": 1192, + "lan": 1193, + "clo": 1194, + "bra": 1195, + "ak": 1196, + "sing": 1197, + "cal": 1198, + "read": 1199, + "'ve": 1200, + "joh": 1201, + "bab": 1202, + "dri": 1203, + "blo": 1204, + "big": 1205, + "eric": 1206, + "int": 1207, + "tor": 1208, + "try": 1209, + "la": 1210, + "leg": 1211, + "house": 1212, + "mic": 1213, + "val": 1214, + "beautiful": 1215, + "litt": 1216, + "check": 1217, + "new": 1218, + "vers": 1219, + "sw": 1220, + "ari": 1221, + "play": 1222, + "her": 1223, + "âĢĵ": 1224, + "win": 1225, + "ma": 1226, + "congr": 1227, + "school": 1228, + "fun": 1229, + ".@": 1230, + "heal": 1231, + "ich": 1232, + "del": 1233, + "where": 1234, + "lon": 1235, + "ket": 1236, + "two": 1237, + "much": 1238, + "watch": 1239, + "ven": 1240, + "ded": 1241, + "ast": 1242, + "ked": 1243, + "bas": 1244, + "going": 1245, + "mp": 1246, + "ever": 1247, + "ways": 1248, + "roo": 1249, + "desig": 1250, + "ly": 1251, + "sed": 1252, + "top": 1253, + "lin": 1254, + "chan": 1255, + "too": 1256, + "iting": 1257, + "dent": 1258, + "ghts": 1259, + "ty": 1260, + "spo": 1261, + "need": 1262, + "blu": 1263, + "inst": 1264, + "being": 1265, + "âĿ¤": 1266, + "wel": 1267, + "ls": 1268, + "him": 1269, + "may": 1270, + "sting": 1271, + "na": 1272, + "ely": 1273, + "little": 1274, + "ga": 1275, + "nat": 1276, + "tomor": 1277, + "mc": 1278, + "hon": 1279, + "want": 1280, + "air": 1281, + "pic": 1282, + "americ": 1283, + "per": 1284, + "less": 1285, + "week": 1286, + "vel": 1287, + "ah": 1288, + "cap": 1289, + "cham": 1290, + "ger": 1291, + "tim": 1292, + "tomorrow": 1293, + "ness": 1294, + "state": 1295, + "hal": 1296, + "serv": 1297, + "ze": 1298, + "os": 1299, + "pat": 1300, + "vis": 1301, + "exc": 1302, + "sin": 1303, + "ff": 1304, + "city": 1305, + "cen": 1306, + "any": 1307, + "bel": 1308, + "summ": 1309, + "tin": 1310, + "would": 1311, + "looking": 1312, + "ko": 1313, + "cele": 1314, + "family": 1315, + "mer": 1316, + "pow": 1317, + "help": 1318, + "bus": 1319, + "co": 1320, + "cle": 1321, + "self": 1322, + "ens": 1323, + "ics": 1324, + "tho": 1325, + "ani": 1326, + "cho": 1327, + "lead": 1328, + "bs": 1329, + "twee": 1330, + "think": 1331, + "fore": 1332, + "chil": 1333, + "vide": 1334, + "did": 1335, + "ale": 1336, + "chi": 1337, + "vil": 1338, + "ends": 1339, + "wing": 1340, + "pas": 1341, + "'ll": 1342, + "vol": 1343, + "sa": 1344, + "gs": 1345, + "many": 1346, + "jec": 1347, + "before": 1348, + "graph": 1349, + "ny": 1350, + "uring": 1351, + "wil": 1352, + "dd": 1353, + "buil": 1354, + "fav": 1355, + "sted": 1356, + "tran": 1357, + "ling": 1358, + "oud": 1359, + "dge": 1360, + "fiel": 1361, + "national": 1362, + "sta": 1363, + "cer": 1364, + "were": 1365, + "ina": 1366, + "season": 1367, + "cou": 1368, + "ned": 1369, + "amazing": 1370, + "tions": 1371, + "celebr": 1372, + "ns": 1373, + "ath": 1374, + "head": 1375, + "sday": 1376, + "dar": 1377, + "loc": 1378, + "vin": 1379, + "another": 1380, + "goo": 1381, + "sat": 1382, + "ny": 1383, + "join": 1384, + "pres": 1385, + "ses": 1386, + "sing": 1387, + "ana": 1388, + "ining": 1389, + "....": 1390, + "cour": 1391, + "ï¸ı": 1392, + "act": 1393, + "cause": 1394, + "light": 1395, + "ams": 1396, + "ta": 1397, + "bal": 1398, + "fc": 1399, + "high": 1400, + "offici": 1401, + "tt": 1402, + "christ": 1403, + "dic": 1404, + "day": 1405, + "ral": 1406, + "hor": 1407, + ":)": 1408, + "visi": 1409, + "nam": 1410, + "ob": 1411, + "mas": 1412, + "ght": 1413, + "really": 1414, + "tun": 1415, + "find": 1416, + "through": 1417, + "port": 1418, + "ut": 1419, + "tive": 1420, + "sty": 1421, + "ne": 1422, + "ore": 1423, + "ðŁĺĤ": 1424, + "support": 1425, + "never": 1426, + "even": 1427, + "ðŁĶ": 1428, + "ha": 1429, + "ya": 1430, + "ld": 1431, + "uk": 1432, + "ran": 1433, + "jam": 1434, + "with": 1435, + "medi": 1436, + "des": 1437, + "ney": 1438, + "ching": 1439, + "ale": 1440, + "hy": 1441, + "kin": 1442, + "!!": 1443, + "dy": 1444, + "place": 1445, + "also": 1446, + "ble": 1447, + "which": 1448, + "black": 1449, + "bli": 1450, + "say": 1451, + "park": 1452, + "play": 1453, + "ire": 1454, + "video": 1455, + "weekend": 1456, + "ail": 1457, + "key": 1458, + "pt": 1459, + "ward": 1460, + "friday": 1461, + "din": 1462, + "iness": 1463, + "gro": 1464, + "ben": 1465, + "always": 1466, + "tball": 1467, + "ago": 1468, + "mil": 1469, + "cy": 1470, + "produc": 1471, + "disc": 1472, + "under": 1473, + "please": 1474, + "spor": 1475, + "full": 1476, + "ey": 1477, + "ðŁĻ": 1478, + "ise": 1479, + "ities": 1480, + "cat": 1481, + "kno": 1482, + "use": 1483, + "fore": 1484, + "ker": 1485, + "art": 1486, + "high": 1487, + "open": 1488, + "san": 1489, + "ef": 1490, + "ours": 1491, + "shed": 1492, + "stri": 1493, + "dro": 1494, + "again": 1495, + "im": 1496, + "ðŁĵ": 1497, + "enjo": 1498, + "fun": 1499, + "getting": 1500, + "pen": 1501, + "ger": 1502, + "cli": 1503, + "any": 1504, + "every": 1505, + "eu": 1506, + "women": 1507, + "âľ": 1508, + "est": 1509, + "could": 1510, + "ry": 1511, + "\"@": 1512, + "thou": 1513, + "sha": 1514, + "commun": 1515, + "ber": 1516, + "dents": 1517, + "dis": 1518, + "while": 1519, + "away": 1520, + "dio": 1521, + "ham": 1522, + "gla": 1523, + "date": 1524, + "ka": 1525, + "miss": 1526, + "unch": 1527, + "won": 1528, + "inf": 1529, + "room": 1530, + "ga": 1531, + "real": 1532, + "exper": 1533, + "direc": 1534, + "should": 1535, + "spr": 1536, + "gol": 1537, + "long": 1538, + "better": 1539, + "ori": 1540, + "ey": 1541, + "ience": 1542, + "ils": 1543, + "zz": 1544, + "han": 1545, + "found": 1546, + "vs": 1547, + "âĻ": 1548, + "post": 1549, + "tic": 1550, + "part": 1551, + "men": 1552, + "rence": 1553, + "cess": 1554, + "vic": 1555, + "sil": 1556, + "shop": 1557, + "ðŁĺĤ": 1558, + "food": 1559, + "val": 1560, + "stic": 1561, + "you": 1562, + "says": 1563, + "elec": 1564, + "star": 1565, + "oc": 1566, + "land": 1567, + "id": 1568, + "ction": 1569, + "field": 1570, + "sof": 1571, + "start": 1572, + "water": 1573, + "friends": 1574, + "ones": 1575, + "ðŁĮ": 1576, + "fla": 1577, + "far": 1578, + "white": 1579, + "party": 1580, + "inst": 1581, + "grou": 1582, + "tv": 1583, + "everyone": 1584, + "ment": 1585, + "ja": 1586, + "cha": 1587, + "prin": 1588, + "ants": 1589, + "during": 1590, + "lat": 1591, + "lar": 1592, + "west": 1593, + "then": 1594, + "ka": 1595, + "youn": 1596, + "insp": 1597, + "inte": 1598, + "ween": 1599, + "visit": 1600, + "against": 1601, + "rele": 1602, + "head": 1603, + "ces": 1604, + "town": 1605, + "looks": 1606, + "thre": 1607, + "regi": 1608, + "rent": 1609, + "projec": 1610, + "girl": 1611, + "sear": 1612, + "wo": 1613, + "mom": 1614, + "car": 1615, + "hun": 1616, + "publi": 1617, + "di": 1618, + "ple": 1619, + "call": 1620, + "cri": 1621, + "um": 1622, + "ford": 1623, + "perfe": 1624, + "friend": 1625, + "hard": 1626, + "ssion": 1627, + "test": 1628, + "playing": 1629, + "around": 1630, + "because": 1631, + "kets": 1632, + "meet": 1633, + "satur": 1634, + "arti": 1635, + "work": 1636, + "jun": 1637, + "ven": 1638, + "run": 1639, + "member": 1640, + "port": 1641, + "super": 1642, + "twit": 1643, + "sam": 1644, + "els": 1645, + "tly": 1646, + "adv": 1647, + "ative": 1648, + "ath": 1649, + "sure": 1650, + "avail": 1651, + "lar": 1652, + "squ": 1653, + "ards": 1654, + "event": 1655, + "men": 1656, + "ll": 1657, + "over": 1658, + "logy": 1659, + "ital": 1660, + "times": 1661, + "mal": 1662, + "back": 1663, + "coo": 1664, + "making": 1665, + "stru": 1666, + "âģ": 1667, + "itu": 1668, + "shar": 1669, + "gan": 1670, + "cas": 1671, + "sn": 1672, + "summer": 1673, + "picture": 1674, + "fan": 1675, + "hin": 1676, + "christmas": 1677, + "cy": 1678, + "proud": 1679, + "champi": 1680, + "design": 1681, + "pping": 1682, + "hope": 1683, + "ca": 1684, + "available": 1685, + "may": 1686, + "wed": 1687, + "photograph": 1688, + "special": 1689, + "sale": 1690, + "stop": 1691, + "ery": 1692, + "awe": 1693, + "ality": 1694, + "history": 1695, + "ama": 1696, + "presi": 1697, + "bru": 1698, + "working": 1699, + "done": 1700, + "dr": 1701, + "ken": 1702, + "feat": 1703, + "wood": 1704, + "atest": 1705, + "sunday": 1706, + "movi": 1707, + "vely": 1708, + "sle": 1709, + "face": 1710, + "spec": 1711, + "students": 1712, + "by": 1713, + "ham": 1714, + "spon": 1715, + "business": 1716, + "dat": 1717, + "ie": 1718, + "ip": 1719, + "soci": 1720, + "glo": 1721, + "hand": 1722, + "recor": 1723, + "rs": 1724, + "mee": 1725, + "keep": 1726, + "pur": 1727, + "health": 1728, + "she": 1729, + "comple": 1730, + "god": 1731, + "davi": 1732, + "collec": 1733, + "list": 1734, + "ra": 1735, + "club": 1736, + "ters": 1737, + "inclu": 1738, + "things": 1739, + "plan": 1740, + "âĺ": 1741, + "john": 1742, + "shing": 1743, + "atul": 1744, + "soon": 1745, + "blue": 1746, + "gor": 1747, + "saturday": 1748, + "won": 1749, + "congratul": 1750, + "see": 1751, + "âĿ¤ï¸ı": 1752, + "those": 1753, + "ðŁĺį": 1754, + "final": 1755, + "dou": 1756, + "ith": 1757, + "own": 1758, + "road": 1759, + "tour": 1760, + "ast": 1761, + "india": 1762, + "til": 1763, + "nd": 1764, + "fer": 1765, + "favor": 1766, + "sul": 1767, + "learn": 1768, + "fire": 1769, + "just": 1770, + "group": 1771, + "ah": 1772, + "rac": 1773, + "body": 1774, + "ur": 1775, + "care": 1776, + "à¸": 1777, + "plo": 1778, + "oh": 1779, + "pos": 1780, + "give": 1781, + "tech": 1782, + "sub": 1783, + "cent": 1784, + "ering": 1785, + "ym": 1786, + "ility": 1787, + "fic": 1788, + "london": 1789, + "vir": 1790, + "guys": 1791, + "ba": 1792, + "ðŁ¤": 1793, + "baby": 1794, + "scre": 1795, + "ðŁĺį": 1796, + "trump": 1797, + "under": 1798, + "change": 1799, + "ian": 1800, + "colle": 1801, + "sses": 1802, + "ler": 1803, + "ssed": 1804, + "nice": 1805, + "announ": 1806, + "power": 1807, + "sar": 1808, + "aking": 1809, + "mini": 1810, + "sli": 1811, + "swee": 1812, + "kar": 1813, + "ful": 1814, + "cru": 1815, + "action": 1816, + "ather": 1817, + ").": 1818, + "stand": 1819, + "devel": 1820, + "aa": 1821, + "gan": 1822, + "left": 1823, + "lol": 1824, + "rel": 1825, + "trans": 1826, + "ments": 1827, + "int": 1828, + "ef": 1829, + "manag": 1830, + "dig": 1831, + "gener": 1832, + "down": 1833, + "pau": 1834, + "tiv": 1835, + "ku": 1836, + "thur": 1837, + "ken": 1838, + "ston": 1839, + "fans": 1840, + "talk": 1841, + "tweet": 1842, + "too": 1843, + "style": 1844, + "prote": 1845, + "secon": 1846, + "fron": 1847, + "awesome": 1848, + "gl": 1849, + "pal": 1850, + "net": 1851, + "sor": 1852, + "lau": 1853, + "gon": 1854, + "since": 1855, + "tty": 1856, + "series": 1857, + "memor": 1858, + "beli": 1859, + "film": 1860, + "did": 1861, + "dies": 1862, + "ot": 1863, + "congratulations": 1864, + "pra": 1865, + "eve": 1866, + "woo": 1867, + "official": 1868, + "suc": 1869, + "incre": 1870, + "bon": 1871, + "part": 1872, + "pped": 1873, + "class": 1874, + "sive": 1875, + "boy": 1876, + "cul": 1877, + "perfect": 1878, + "tou": 1879, + "dam": 1880, + "welcome": 1881, + "football": 1882, + "hi": 1883, + "pap": 1884, + "wait": 1885, + "ada": 1886, + "congrats": 1887, + "young": 1888, + "excited": 1889, + "rece": 1890, + "jan": 1891, + "va": 1892, + "red": 1893, + "stra": 1894, + "media": 1895, + "'d": 1896, + "does": 1897, + "let": 1898, + "mul": 1899, + "ills": 1900, + "green": 1901, + "mel": 1902, + "toge": 1903, + "future": 1904, + "yester": 1905, + "versity": 1906, + "form": 1907, + "tain": 1908, + "ide": 1909, + "ches": 1910, + "kids": 1911, + "qui": 1912, + "haha": 1913, + "deta": 1914, + "big": 1915, + "favorite": 1916, + "girls": 1917, + "contin": 1918, + "dom": 1919, + "search": 1920, + "ual": 1921, + "air": 1922, + "ders": 1923, + "month": 1924, + "cer": 1925, + "yesterday": 1926, + "community": 1927, + "ade": 1928, + "dog": 1929, + "ville": 1930, + "ices": 1931, + "deli": 1932, + "syste": 1933, + "run": 1934, + "ism": 1935, + "heart": 1936, + "cup": 1937, + "enti": 1938, + "few": 1939, + "president": 1940, + "eds": 1941, + "until": 1942, + "festi": 1943, + "ok": 1944, + "flo": 1945, + "said": 1946, + "ole": 1947, + "med": 1948, + "travel": 1949, + "£": 1950, + "phone": 1951, + "together": 1952, + "fast": 1953, + "lot": 1954, + "games": 1955, + "shir": 1956, + "between": 1957, + "yes": 1958, + "thers": 1959, + "doing": 1960, + "mac": 1961, + "ator": 1962, + "band": 1963, + "follow": 1964, + "project": 1965, + "develop": 1966, + "diffe": 1967, + "confe": 1968, + "speci": 1969, + "cast": 1970, + "ys": 1971, + "board": 1972, + "rd": 1973, + "ial": 1974, + "shoo": 1975, + "ram": 1976, + "having": 1977, + "share": 1978, + "follow": 1979, + "one": 1980, + "name": 1981, + "mr": 1982, + "put": 1983, + "discu": 1984, + "ory": 1985, + "came": 1986, + "ous": 1987, + "site": 1988, + "twitter": 1989, + "tb": 1990, + "tit": 1991, + "finally": 1992, + "zed": 1993, + "super": 1994, + "compan": 1995, + "using": 1996, + "alls": 1997, + "list": 1998, + "ris": 1999, + "shot": 2000, + "gal": 2001, + "tar": 2002, + "del": 2003, + "john": 2004, + "âĢĶ": 2005, + "something": 2006, + "ram": 2007, + "intere": 2008, + "whe": 2009, + "bit": 2010, + "ðŁį": 2011, + "street": 2012, + "ound": 2013, + "ai": 2014, + "tickets": 2015, + "movie": 2016, + "real": 2017, + "ky": 2018, + "taking": 2019, + "opp": 2020, + "cc": 2021, + "lam": 2022, + "moun": 2023, + "inve": 2024, + "black": 2025, + "used": 2026, + "online": 2027, + "yor": 2028, + "local": 2029, + "gue": 2030, + "cks": 2031, + "ow": 2032, + "gest": 2033, + "boys": 2034, + "illion": 2035, + "cont": 2036, + "reci": 2037, + "ined": 2038, + "euro": 2039, + "now": 2040, + "seen": 2041, + "ph": 2042, + "teach": 2043, + "def": 2044, + "south": 2045, + "such": 2046, + "award": 2047, + "must": 2048, + "issu": 2049, + "care": 2050, + "feel": 2051, + "plu": 2052, + "latest": 2053, + "sports": 2054, + "web": 2055, + "tex": 2056, + "ement": 2057, + "sk": 2058, + "fic": 2059, + "wan": 2060, + "tech": 2061, + "ot": 2062, + "box": 2063, + "ner": 2064, + "free": 2065, + "tal": 2066, + "ash": 2067, + "case": 2068, + "hot": 2069, + "wonder": 2070, + "meeting": 2071, + "era": 2072, + "chall": 2073, + "ðŁIJ": 2074, + "job": 2075, + "ili": 2076, + "cool": 2077, + "jour": 2078, + "ths": 2079, + "mo": 2080, + "fel": 2081, + "die": 2082, + "micha": 2083, + "ele": 2084, + "team": 2085, + "service": 2086, + "stand": 2087, + "makes": 2088, + "ping": 2089, + "early": 2090, + "comes": 2091, + "ek": 2092, + "holi": 2093, + "vers": 2094, + "ague": 2095, + "sau": 2096, + "three": 2097, + "monday": 2098, + "fashi": 2099, + "someone": 2100, + "thro": 2101, + "sea": 2102, + "bad": 2103, + "suppor": 2104, + "turn": 2105, + "ury": 2106, + "ming": 2107, + "photography": 2108, + "nic": 2109, + "mark": 2110, + "pretty": 2111, + "ssing": 2112, + "watching": 2113, + "memb": 2114, + "arri": 2115, + "county": 2116, + "beach": 2117, + "fran": 2118, + "center": 2119, + "police": 2120, + "bat": 2121, + "public": 2122, + "tan": 2123, + "press": 2124, + "saf": 2125, + "sy": 2126, + "gets": 2127, + "roy": 2128, + "ners": 2129, + "your": 2130, + "buy": 2131, + "sters": 2132, + "show": 2133, + "ased": 2134, + "childre": 2135, + "afric": 2136, + "ines": 2137, + "space": 2138, + "scri": 2139, + "hall": 2140, + "pain": 2141, + "aring": 2142, + "home": 2143, + "mur": 2144, + "health": 2145, + "ched": 2146, + "sand": 2147, + "recei": 2148, + "guy": 2149, + "ea": 2150, + "american": 2151, + "resi": 2152, + "children": 2153, + "--": 2154, + "iri": 2155, + "ington": 2156, + "country": 2157, + "ross": 2158, + "len": 2159, + "anna": 2160, + "books": 2161, + "bc": 2162, + "ece": 2163, + "dom": 2164, + "lovely": 2165, + "kh": 2166, + "pet": 2167, + "gy": 2168, + "gri": 2169, + "stage": 2170, + "office": 2171, + "rock": 2172, + "mon": 2173, + "bay": 2174, + "table": 2175, + "sun": 2176, + "med": 2177, + "thin": 2178, + "lor": 2179, + "flow": 2180, + "(@": 2181, + "university": 2182, + "store": 2183, + "front": 2184, + "good": 2185, + "za": 2186, + "vote": 2187, + "north": 2188, + "hey": 2189, + "anim": 2190, + "order": 2191, + "mid": 2192, + "without": 2193, + "ade": 2194, + "remember": 2195, + "market": 2196, + "??": 2197, + "mus": 2198, + "training": 2199, + "educ": 2200, + "but": 2201, + "cover": 2202, + "stan": 2203, + "scen": 2204, + "bla": 2205, + "break": 2206, + "lou": 2207, + "same": 2208, + "gold": 2209, + "ain": 2210, + "os": 2211, + "both": 2212, + "lit": 2213, + "vern": 2214, + "ai": 2215, + "albu": 2216, + "pa": 2217, + "enjoy": 2218, + "beg": 2219, + "elling": 2220, + "thursday": 2221, + "info": 2222, + "san": 2223, + "america": 2224, + "hair": 2225, + "tel": 2226, + "march": 2227, + "concer": 2228, + "college": 2229, + "conference": 2230, + "app": 2231, + "hour": 2232, + "chang": 2233, + "âļ": 2234, + "sour": 2235, + "ols": 2236, + "weather": 2237, + "war": 2238, + "phi": 2239, + "festival": 2240, + "second": 2241, + "cute": 2242, + "prac": 2243, + "ener": 2244, + "stry": 2245, + "lea": 2246, + "polit": 2247, + "sav": 2248, + "sen": 2249, + "ow": 2250, + "mi": 2251, + "near": 2252, + "ought": 2253, + "ze": 2254, + "coffe": 2255, + "willi": 2256, + "dan": 2257, + "sey": 2258, + "david": 2259, + "ese": 2260, + "fan": 2261, + "deci": 2262, + "theat": 2263, + "nov": 2264, + "ation": 2265, + "trac": 2266, + "sci": 2267, + "review": 2268, + "cel": 2269, + "em": 2270, + "un": 2271, + "july": 2272, + "orig": 2273, + "tion": 2274, + "dru": 2275, + "former": 2276, + "stay": 2277, + "after": 2278, + "inv": 2279, + "took": 2280, + "data": 2281, + "bal": 2282, + "tues": 2283, + "dan": 2284, + "evening": 2285, + "ðŁĺĤðŁĺĤ": 2286, + "dol": 2287, + "ures": 2288, + "provi": 2289, + "ts": 2290, + "est": 2291, + "sign": 2292, + "jac": 2293, + "uk": 2294, + "song": 2295, + "yet": 2296, + "bow": 2297, + "indu": 2298, + "jap": 2299, + "hoo": 2300, + "point": 2301, + "anyone": 2302, + "zy": 2303, + "ist": 2304, + "hur": 2305, + "ital": 2306, + "building": 2307, + "woman": 2308, + "chur": 2309, + "jer": 2310, + "perfor": 2311, + "coach": 2312, + "league": 2313, + "cess": 2314, + "net": 2315, + "imag": 2316, + "nation": 2317, + "brit": 2318, + "que": 2319, + "awards": 2320, + "ages": 2321, + "works": 2322, + "ced": 2323, + "mance": 2324, + "late": 2325, + "ign": 2326, + "money": 2327, + "true": 2328, + "ii": 2329, + "tell": 2330, + "plac": 2331, + "pac": 2332, + "asy": 2333, + "world": 2334, + "behin": 2335, + "import": 2336, + "reading": 2337, + "gram": 2338, + "giving": 2339, + "met": 2340, + "hit": 2341, + "forward": 2342, + "stom": 2343, + "present": 2344, + "june": 2345, + "social": 2346, + "noon": 2347, + "mart": 2348, + "half": 2349, + "swe": 2350, + "govern": 2351, + "ker": 2352, + "details": 2353, + "lish": 2354, + "__": 2355, + "acy": 2356, + "sia": 2357, + "bert": 2358, + "fall": 2359, + "!!!!": 2360, + "),": 2361, + "thi": 2362, + "diti": 2363, + "sport": 2364, + "king": 2365, + "fit": 2366, + "staf": 2367, + "cat": 2368, + "muse": 2369, + "centr": 2370, + "yer": 2371, + "contro": 2372, + "bloo": 2373, + "walk": 2374, + "actu": 2375, + "didn": 2376, + "lim": 2377, + "learning": 2378, + "research": 2379, + "wedne": 2380, + "auth": 2381, + "hours": 2382, + "ky": 2383, + "far": 2384, + "hen": 2385, + "....": 2386, + "itch": 2387, + "ril": 2388, + "strong": 2389, + "sky": 2390, + "questi": 2391, + "james": 2392, + "ron": 2393, + "dg": 2394, + "fur": 2395, + "cin": 2396, + "does": 2397, + "appro": 2398, + "marke": 2399, + "tures": 2400, + "fully": 2401, + "chat": 2402, + "behind": 2403, + "tem": 2404, + "fini": 2405, + "mission": 2406, + "batt": 2407, + "feel": 2408, + "heav": 2409, + "everything": 2410, + "bar": 2411, + "wish": 2412, + "premi": 2413, + "ima": 2414, + "experience": 2415, + "each": 2416, + "report": 2417, + "sweet": 2418, + "tics": 2419, + "spring": 2420, + "respon": 2421, + "system": 2422, + "victor": 2423, + "lin": 2424, + "saw": 2425, + "already": 2426, + "ghter": 2427, + "fle": 2428, + "ãĥ": 2429, + "bring": 2430, + "album": 2431, + "--": 2432, + "ells": 2433, + "stan": 2434, + "tom": 2435, + "international": 2436, + "went": 2437, + "anni": 2438, + "match": 2439, + "pper": 2440, + "stone": 2441, + "small": 2442, + "rain": 2443, + "fashion": 2444, + "area": 2445, + "van": 2446, + "agram": 2447, + "ko": 2448, + "thought": 2449, + "worth": 2450, + "van": 2451, + "mer": 2452, + "coffee": 2453, + "ites": 2454, + "gn": 2455, + "artist": 2456, + "con": 2457, + "arch": 2458, + "cir": 2459, + "secre": 2460, + "ground": 2461, + "iso": 2462, + "hand": 2463, + "com": 2464, + "bridge": 2465, + "hs": 2466, + "xi": 2467, + "link": 2468, + "pul": 2469, + "spl": 2470, + "race": 2471, + "fli": 2472, + "river": 2473, + "gas": 2474, + "disco": 2475, + "dal": 2476, + "player": 2477, + "fit": 2478, + "photos": 2479, + "ity": 2480, + "ok": 2481, + "jor": 2482, + "tra": 2483, + "april": 2484, + "ads": 2485, + "adi": 2486, + "solu": 2487, + "beauty": 2488, + "door": 2489, + "mess": 2490, + "update": 2491, + "alia": 2492, + "scho": 2493, + "ened": 2494, + "moment": 2495, + "scot": 2496, + "science": 2497, + "ior": 2498, + "ties": 2499, + "across": 2500, + "ously": 2501, + "shes": 2502, + "doesn": 2503, + "page": 2504, + "water": 2505, + "million": 2506, + "classi": 2507, + "lic": 2508, + "cast": 2509, + "formation": 2510, + "michael": 2511, + "ello": 2512, + "smo": 2513, + "ints": 2514, + "vision": 2515, + "opening": 2516, + "ldn": 2517, + "austr": 2518, + "tuesday": 2519, + "winner": 2520, + "possi": 2521, + "round": 2522, + "shirt": 2523, + "dit": 2524, + "bo": 2525, + "ues": 2526, + "illed": 2527, + "along": 2528, + "trip": 2529, + "starting": 2530, + "impro": 2531, + "kan": 2532, + "person": 2533, + "not": 2534, + "reco": 2535, + "needs": 2536, + "cle": 2537, + "lie": 2538, + "rest": 2539, + "ring": 2540, + "winter": 2541, + "simp": 2542, + "mom": 2543, + "beer": 2544, + "face": 2545, + "tors": 2546, + "usa": 2547, + "collection": 2548, + "geor": 2549, + "session": 2550, + "trying": 2551, + "las": 2552, + "lake": 2553, + "jen": 2554, + "origin": 2555, + "student": 2556, + "secur": 2557, + "vin": 2558, + "pics": 2559, + "expe": 2560, + "comp": 2561, + "gonna": 2562, + "equ": 2563, + "bad": 2564, + "ley": 2565, + "au": 2566, + "members": 2567, + "break": 2568, + "wall": 2569, + "gic": 2570, + "dinner": 2571, + "bul": 2572, + "inspir": 2573, + "ri": 2574, + "mind": 2575, + "ica": 2576, + "winning": 2577, + "talking": 2578, + "tren": 2579, + "sis": 2580, + "ten": 2581, + "wonderful": 2582, + "snow": 2583, + "hear": 2584, + "thom": 2585, + "nothing": 2586, + "gui": 2587, + "stin": 2588, + "blog": 2589, + "fest": 2590, + "bun": 2591, + "lee": 2592, + "wards": 2593, + "chance": 2594, + "dress": 2595, + "ren": 2596, + "paul": 2597, + "pes": 2598, + "techno": 2599, + "russi": 2600, + "card": 2601, + "east": 2602, + "mari": 2603, + "wine": 2604, + "ti": 2605, + "law": 2606, + "stric": 2607, + "ki": 2608, + "ape": 2609, + "augu": 2610, + "profe": 2611, + "ash": 2612, + "course": 2613, + "mail": 2614, + "rently": 2615, + "dun": 2616, + "mun": 2617, + "love": 2618, + "island": 2619, + "drive": 2620, + "sl": 2621, + "ended": 2622, + "main": 2623, + "lost": 2624, + "nature": 2625, + "âĿ¤ï¸ı": 2626, + "chic": 2627, + "repor": 2628, + "pin": 2629, + "pro": 2630, + "station": 2631, + "cep": 2632, + "takes": 2633, + "company": 2634, + "goes": 2635, + "ond": 2636, + "mach": 2637, + "radio": 2638, + "dad": 2639, + "rock": 2640, + "ja": 2641, + "pay": 2642, + "champion": 2643, + "ee": 2644, + "inde": 2645, + "tta": 2646, + "atic": 2647, + "tab": 2648, + "believe": 2649, + "energy": 2650, + "zi": 2651, + "tat": 2652, + "word": 2653, + "once": 2654, + "resul": 2655, + "yl": 2656, + "andre": 2657, + "ano": 2658, + "instagram": 2659, + "close": 2660, + "tam": 2661, + "custom": 2662, + "wa": 2663, + "conom": 2664, + "shows": 2665, + "life": 2666, + "kin": 2667, + "rob": 2668, + "tage": 2669, + "nation": 2670, + "almost": 2671, + "listen": 2672, + "save": 2673, + "reli": 2674, + "ace": 2675, + "mary": 2676, + "tree": 2677, + "forget": 2678, + "jack": 2679, + "waiting": 2680, + "director": 2681, + "hill": 2682, + "born": 2683, + "temp": 2684, + "fl": 2685, + "ste": 2686, + "ona": 2687, + "single": 2688, + "wednesday": 2689, + "united": 2690, + "ino": 2691, + "@_": 2692, + "nel": 2693, + "celebrate": 2694, + "ending": 2695, + "deal": 2696, + "ji": 2697, + "canada": 2698, + "huge": 2699, + "track": 2700, + "âĢ¢": 2701, + "fy": 2702, + "fanta": 2703, + "ang": 2704, + "york": 2705, + "release": 2706, + "pun": 2707, + "episo": 2708, + "words": 2709, + "tour": 2710, + "pack": 2711, + "igh": 2712, + "classic": 2713, + "performance": 2714, + "ket": 2715, + "afternoon": 2716, + "record": 2717, + "wins": 2718, + "proble": 2719, + "âĿ¤": 2720, + "four": 2721, + "bed": 2722, + "bank": 2723, + "dance": 2724, + "sla": 2725, + "called": 2726, + "might": 2727, + "ap": 2728, + "past": 2729, + "ðŁļ": 2730, + "different": 2731, + "ite": 2732, + "gift": 2733, + "ssive": 2734, + "church": 2735, + "cus": 2736, + "program": 2737, + "hotel": 2738, + "ice": 2739, + "mad": 2740, + "security": 2741, + "enge": 2742, + "dc": 2743, + "enough": 2744, + "sta": 2745, + "ety": 2746, + "dead": 2747, + "gun": 2748, + "hear": 2749, + "mir": 2750, + "human": 2751, + "gress": 2752, + "ounds": 2753, + "piece": 2754, + "breaking": 2755, + "garden": 2756, + "fight": 2757, + "views": 2758, + "fish": 2759, + "started": 2760, + "running": 2761, + "green": 2762, + "seri": 2763, + "sm": 2764, + "ask": 2765, + "dor": 2766, + "death": 2767, + "econom": 2768, + "eri": 2769, + "ird": 2770, + "ser": 2771, + "lunch": 2772, + "âģ¦": 2773, + "box": 2774, + "natu": 2775, + "base": 2776, + "ban": 2777, + "fal": 2778, + "global": 2779, + "wild": 2780, + "wow": 2781, + "outside": 2782, + "move": 2783, + "lead": 2784, + "anal": 2785, + "museum": 2786, + "ong": 2787, + "haw": 2788, + "power": 2789, + "thank": 2790, + "bac": 2791, + "charac": 2792, + "campa": 2793, + "digital": 2794, + "ro": 2795, + "oper": 2796, + "dev": 2797, + "wol": 2798, + "pati": 2799, + "fa": 2800, + "male": 2801, + "paper": 2802, + "illing": 2803, + "cs": 2804, + "âĥ": 2805, + "education": 2806, + "taken": 2807, + "effe": 2808, + "mou": 2809, + "sad": 2810, + "\".": 2811, + "based": 2812, + "staff": 2813, + "including": 2814, + "living": 2815, + "ac": 2816, + "china": 2817, + "mob": 2818, + "storm": 2819, + "luck": 2820, + "phil": 2821, + "oo": 2822, + "yn": 2823, + "travel": 2824, + "kel": 2825, + "tial": 2826, + "price": 2827, + "book": 2828, + "important": 2829, + "bio": 2830, + "pool": 2831, + "nyc": 2832, + "fab": 2833, + "load": 2834, + "?!": 2835, + "challenge": 2836, + "cry": 2837, + "serve": 2838, + "wear": 2839, + "bus": 2840, + "tain": 2841, + "number": 2842, + "ror": 2843, + "kat": 2844, + "iz": 2845, + "though": 2846, + "hosp": 2847, + "mm": 2848, + "fair": 2849, + "utes": 2850, + "hot": 2851, + "pop": 2852, + "fied": 2853, + "camp": 2854, + "development": 2855, + "libr": 2856, + "cali": 2857, + "ems": 2858, + "âģ¦@": 2859, + "bol": 2860, + "ised": 2861, + "standing": 2862, + "model": 2863, + "ita": 2864, + "gle": 2865, + "brown": 2866, + "image": 2867, + "vered": 2868, + "force": 2869, + "oil": 2870, + "partic": 2871, + "shu": 2872, + "daily": 2873, + "law": 2874, + "sec": 2875, + "class": 2876, + "camp": 2877, + "holiday": 2878, + "clin": 2879, + "kers": 2880, + "present": 2881, + "game": 2882, + "incredi": 2883, + "ership": 2884, + "interview": 2885, + "bill": 2886, + "due": 2887, + "andy": 2888, + "abo": 2889, + "innov": 2890, + "key": 2891, + "acade": 2892, + "pil": 2893, + "moder": 2894, + "stars": 2895, + "brand": 2896, + "fer": 2897, + "weeks": 2898, + "consi": 2899, + "pre": 2900, + "safe": 2901, + "writ": 2902, + "dium": 2903, + "launch": 2904, + "marketing": 2905, + "annual": 2906, + "assi": 2907, + "court": 2908, + "lady": 2909, + "cted": 2910, + "anda": 2911, + "inside": 2912, + "child": 2913, + "oppor": 2914, + "smith": 2915, + "centre": 2916, + "gue": 2917, + "âģ©": 2918, + "fren": 2919, + "sty": 2920, + "fort": 2921, + "ently": 2922, + "isn": 2923, + "keep": 2924, + "tober": 2925, + "ony": 2926, + "boy": 2927, + "ald": 2928, + "colla": 2929, + "demo": 2930, + "level": 2931, + "compet": 2932, + "ado": 2933, + "bour": 2934, + "fantastic": 2935, + "mate": 2936, + "su": 2937, + "south": 2938, + "opportun": 2939, + "versary": 2940, + "later": 2941, + "bud": 2942, + "facebook": 2943, + "laun": 2944, + "stern": 2945, + "pit": 2946, + "!\"": 2947, + "maj": 2948, + "gram": 2949, + "tbt": 2950, + "fire": 2951, + "happy": 2952, + "aks": 2953, + "whole": 2954, + "actually": 2955, + "iller": 2956, + "ella": 2957, + "lots": 2958, + "alex": 2959, + "ange": 2960, + "lands": 2961, + "ðŁĺŃ": 2962, + "enter": 2963, + "rou": 2964, + "episode": 2965, + "ped": 2966, + "inten": 2967, + "shire": 2968, + "who": 2969, + "plan": 2970, + "ho": 2971, + "cake": 2972, + "west": 2973, + "magaz": 2974, + "fresh": 2975, + "cc": 2976, + "nar": 2977, + "chris": 2978, + "writing": 2979, + "wer": 2980, + "nom": 2981, + "lo": 2982, + "midd": 2983, + "dream": 2984, + "ol": 2985, + "tional": 2986, + "deb": 2987, + ">>": 2988, + "become": 2989, + "si": 2990, + "grand": 2991, + "alling": 2992, + "histor": 2993, + "ride": 2994, + "ired": 2995, + "safe": 2996, + "queen": 2997, + "cil": 2998, + "intro": 2999, + "vil": 3000, + "dani": 3001, + "...": 3002, + "artic": 3003, + "stat": 3004, + "short": 3005, + "oring": 3006, + "selfi": 3007, + "missi": 3008, + "doc": 3009, + "bit": 3010, + "gall": 3011, + "bom": 3012, + "ire": 3013, + "selec": 3014, + "dition": 3015, + "ðŁĶ¥": 3016, + "friend": 3017, + "beat": 3018, + "ghting": 3019, + "ðŁĺĬ": 3020, + "peace": 3021, + "exhi": 3022, + "anta": 3023, + "ability": 3024, + "illu": 3025, + "jon": 3026, + "quality": 3027, + "tribu": 3028, + "mes": 3029, + "players": 3030, + "fair": 3031, + "cut": 3032, + "cab": 3033, + "success": 3034, + "bi": 3035, + "sus": 3036, + "promo": 3037, + "sche": 3038, + "ange": 3039, + "ico": 3040, + "commit": 3041, + "catch": 3042, + "illa": 3043, + "kind": 3044, + "feeling": 3045, + "quo": 3046, + "say": 3047, + "anniversary": 3048, + "spot": 3049, + "mother": 3050, + "ane": 3051, + "pend": 3052, + "yourself": 3053, + "ops": 3054, + "apple": 3055, + "minutes": 3056, + "po": 3057, + "grand": 3058, + "ries": 3059, + "haha": 3060, + "career": 3061, + "edition": 3062, + "dec": 3063, + "rick": 3064, + "ami": 3065, + "concert": 3066, + "itive": 3067, + "geous": 3068, + "dly": 3069, + "tte": 3070, + "advent": 3071, + "ig": 3072, + "lights": 3073, + "aker": 3074, + "sky": 3075, + "âĥ£": 3076, + "ray": 3077, + "finished": 3078, + "way": 3079, + "sd": 3080, + "accoun": 3081, + "ðŁĴķ": 3082, + "cky": 3083, + "chel": 3084, + "liter": 3085, + "painting": 3086, + "los": 3087, + "stun": 3088, + "technology": 3089, + "nas": 3090, + "mar": 3091, + "bil": 3092, + "africa": 3093, + "kie": 3094, + "eyes": 3095, + "golf": 3096, + "plus": 3097, + "nia": 3098, + "itec": 3099, + "services": 3100, + "wedding": 3101, + "known": 3102, + "tele": 3103, + ".....": 3104, + "starts": 3105, + "paren": 3106, + "wants": 3107, + "ational": 3108, + "months": 3109, + "windo": 3110, + "favour": 3111, + "ert": 3112, + "magazine": 3113, + "exclu": 3114, + "reve": 3115, + "bc": 3116, + "original": 3117, + "ess": 3118, + "nal": 3119, + "anti": 3120, + "stro": 3121, + "tice": 3122, + "study": 3123, + "à¤": 3124, + "vac": 3125, + "national": 3126, + "five": 3127, + "rain": 3128, + "vement": 3129, + "ute": 3130, + "verse": 3131, + "emer": 3132, + "army": 3133, + "possible": 3134, + "guess": 3135, + "valley": 3136, + "thern": 3137, + "crow": 3138, + "mr": 3139, + "color": 3140, + "onto": 3141, + "pick": 3142, + "clear": 3143, + "dark": 3144, + "tac": 3145, + "wanted": 3146, + "itting": 3147, + "cancer": 3148, + "government": 3149, + "die": 3150, + "rise": 3151, + "zing": 3152, + "cold": 3153, + "foun": 3154, + "studio": 3155, + "stration": 3156, + "brother": 3157, + "ahead": 3158, + "shel": 3159, + "micro": 3160, + "ically": 3161, + "dau": 3162, + "signed": 3163, + "viol": 3164, + "ax": 3165, + "asse": 3166, + "io": 3167, + "wre": 3168, + "splay": 3169, + "chick": 3170, + "august": 3171, + "plat": 3172, + "tips": 3173, + "spi": 3174, + "human": 3175, + "easy": 3176, + "logi": 3177, + "mike": 3178, + "grow": 3179, + "agre": 3180, + "ww": 3181, + "shad": 3182, + "motiv": 3183, + "wide": 3184, + "turns": 3185, + "omg": 3186, + "var": 3187, + "defin": 3188, + "sug": 3189, + "jim": 3190, + "ðŁĶ¥": 3191, + "td": 3192, + "campaign": 3193, + "named": 3194, + "retweet": 3195, + "cop": 3196, + "tv": 3197, + "leav": 3198, + "kis": 3199, + "double": 3200, + "smar": 3201, + "issue": 3202, + "villa": 3203, + "information": 3204, + "lies": 3205, + "stock": 3206, + "nt": 3207, + "distric": 3208, + "shor": 3209, + "mix": 3210, + "ero": 3211, + "sep": 3212, + "mex": 3213, + "seeing": 3214, + "live": 3215, + "remin": 3216, + "code": 3217, + "gur": 3218, + "sc": 3219, + "wild": 3220, + "lun": 3221, + "hood": 3222, + "spot": 3223, + "father": 3224, + "forever": 3225, + "upd": 3226, + "traf": 3227, + "fly": 3228, + "need": 3229, + "gradu": 3230, + "train": 3231, + "make": 3232, + "sab": 3233, + "bey": 3234, + "size": 3235, + "leader": 3236, + "talks": 3237, + "eu": 3238, + "log": 3239, + "fox": 3240, + "gorgeous": 3241, + "less": 3242, + "lets": 3243, + "surpri": 3244, + "myself": 3245, + "note": 3246, + "lives": 3247, + "fru": 3248, + "loved": 3249, + "sever": 3250, + "dem": 3251, + "ji": 3252, + "soc": 3253, + "hold": 3254, + "dogs": 3255, + "ni": 3256, + "âŀ": 3257, + "leave": 3258, + "airport": 3259, + "benef": 3260, + "expl": 3261, + "ships": 3262, + "complete": 3263, + "achi": 3264, + "great": 3265, + "vintage": 3266, + "jack": 3267, + "roc": 3268, + "wood": 3269, + "priv": 3270, + "offer": 3271, + "eye": 3272, + "version": 3273, + "tea": 3274, + "coach": 3275, + "offic": 3276, + "well": 3277, + "gen": 3278, + "sat": 3279, + "hh": 3280, + "youth": 3281, + "ox": 3282, + "?\"": 3283, + "mt": 3284, + "mix": 3285, + "gg": 3286, + "dle": 3287, + "natural": 3288, + "build": 3289, + "breakfast": 3290, + "thinking": 3291, + "theatre": 3292, + "moon": 3293, + "berg": 3294, + "goals": 3295, + "george": 3296, + "ene": 3297, + "excell": 3298, + "iling": 3299, + "tune": 3300, + "yed": 3301, + "gate": 3302, + "mit": 3303, + "network": 3304, + "joe": 3305, + "hello": 3306, + "fb": 3307, + "tube": 3308, + "wearing": 3309, + "athle": 3310, + "struc": 3311, + "hard": 3312, + "glass": 3313, + "gers": 3314, + "throw": 3315, + "ges": 3316, + "bt": 3317, + "industry": 3318, + "management": 3319, + "alist": 3320, + "goal": 3321, + "stream": 3322, + "yel": 3323, + "avi": 3324, + "icious": 3325, + "others": 3326, + "ski": 3327, + "christi": 3328, + "bird": 3329, + "esc": 3330, + "min": 3331, + "tro": 3332, + "lt": 3333, + "jan": 3334, + "imp": 3335, + "rights": 3336, + "sha": 3337, + "organ": 3338, + "central": 3339, + "ara": 3340, + "roll": 3341, + "favourite": 3342, + "chester": 3343, + "else": 3344, + "pay": 3345, + "cars": 3346, + "mine": 3347, + "step": 3348, + "practice": 3349, + "major": 3350, + "hang": 3351, + "ðŁĺĺ": 3352, + "non": 3353, + "vari": 3354, + "engine": 3355, + "volun": 3356, + "dia": 3357, + "iled": 3358, + "architec": 3359, + "pink": 3360, + "ds": 3361, + "thy": 3362, + "wash": 3363, + "website": 3364, + "bag": 3365, + "control": 3366, + "elli": 3367, + "fra": 3368, + "answ": 3369, + "dence": 3370, + "yu": 3371, + "ron": 3372, + "ola": 3373, + "gin": 3374, + "drin": 3375, + "lic": 3376, + "couple": 3377, + "spar": 3378, + "gon": 3379, + "create": 3380, + "ct": 3381, + "celebrating": 3382, + "deep": 3383, + "eat": 3384, + "tee": 3385, + "voice": 3386, + "drop": 3387, + "visit": 3388, + "ators": 3389, + "stadium": 3390, + "ft": 3391, + "wis": 3392, + "rol": 3393, + "grade": 3394, + "famil": 3395, + "points": 3396, + "repre": 3397, + "was": 3398, + "traffic": 3399, + "japan": 3400, + "org": 3401, + "honor": 3402, + "texas": 3403, + "manu": 3404, + "âĻ¥": 3405, + "safety": 3406, + "rer": 3407, + "bag": 3408, + "emplo": 3409, + "released": 3410, + "regu": 3411, + "aka": 3412, + "nav": 3413, + "role": 3414, + "senior": 3415, + "spect": 3416, + "cross": 3417, + "lines": 3418, + "best": 3419, + "pack": 3420, + "sin": 3421, + "tie": 3422, + "missing": 3423, + "sunset": 3424, + "liber": 3425, + "ising": 3426, + "jay": 3427, + "ski": 3428, + "championship": 3429, + "activ": 3430, + "ladies": 3431, + "played": 3432, + "yy": 3433, + "publ": 3434, + "alo": 3435, + "pride": 3436, + "sr": 3437, + "paki": 3438, + "lux": 3439, + "survi": 3440, + "cked": 3441, + "ets": 3442, + "chocol": 3443, + "australia": 3444, + "paris": 3445, + "miles": 3446, + "hat": 3447, + "mental": 3448, + "ala": 3449, + "mean": 3450, + "mobile": 3451, + "ena": 3452, + "insi": 3453, + "found": 3454, + "chief": 3455, + "tag": 3456, + "incredible": 3457, + "return": 3458, + "é": 3459, + "google": 3460, + "french": 3461, + "crew": 3462, + "hallo": 3463, + "alian": 3464, + "jaz": 3465, + "cher": 3466, + "silver": 3467, + "north": 3468, + "english": 3469, + "baseball": 3470, + "caf": 3471, + "limited": 3472, + "following": 3473, + "appreci": 3474, + "earth": 3475, + "kir": 3476, + "vember": 3477, + "wed": 3478, + "ption": 3479, + "ged": 3480, + "october": 3481, + "flori": 3482, + "cr": 3483, + "ency": 3484, + "gave": 3485, + "lord": 3486, + "stuff": 3487, + "berry": 3488, + "post": 3489, + "smile": 3490, + "broad": 3491, + "state": 3492, + "gger": 3493, + "means": 3494, + "icy": 3495, + "gun": 3496, + "yo": 3497, + "master": 3498, + "burg": 3499, + "hands": 3500, + "nie": 3501, + "//": 3502, + "union": 3503, + "british": 3504, + "biggest": 3505, + "district": 3506, + "aming": 3507, + "hil": 3508, + "oce": 3509, + "person": 3510, + "pass": 3511, + "envir": 3512, + "schools": 3513, + "arrived": 3514, + "ances": 3515, + "inspired": 3516, + "expla": 3517, + "ben": 3518, + "library": 3519, + "bott": 3520, + "amp": 3521, + "steph": 3522, + "contact": 3523, + "bang": 3524, + "ms": 3525, + "califor": 3526, + "told": 3527, + "battle": 3528, + "bb": 3529, + "chicago": 3530, + "⾨": 3531, + "strate": 3532, + "shi": 3533, + "dece": 3534, + "-)": 3535, + "add": 3536, + "lab": 3537, + "jones": 3538, + "legend": 3539, + "castle": 3540, + "inger": 3541, + "stance": 3542, + "bel": 3543, + "ura": 3544, + "refu": 3545, + "leaders": 3546, + "pot": 3547, + "sex": 3548, + "hic": 3549, + "article": 3550, + "kid": 3551, + "france": 3552, + "xx": 3553, + "exe": 3554, + "guide": 3555, + "volunte": 3556, + "print": 3557, + "ali": 3558, + "ceo": 3559, + "tweets": 3560, + "wx": 3561, + "scene": 3562, + "volu": 3563, + "anti": 3564, + "han": 3565, + "associ": 3566, + "sharing": 3567, + "rose": 3568, + "minister": 3569, + "sher": 3570, + "inste": 3571, + "clean": 3572, + "democr": 3573, + "poster": 3574, + "skin": 3575, + "psy": 3576, + "proper": 3577, + "crazy": 3578, + "iam": 3579, + "ore": 3580, + "ini": 3581, + "anything": 3582, + "pod": 3583, + "moving": 3584, + "click": 3585, + "explo": 3586, + "comb": 3587, + "craft": 3588, + "fi": 3589, + "blood": 3590, + "isra": 3591, + "public": 3592, + "dent": 3593, + "olym": 3594, + "england": 3595, + "asi": 3596, + "cher": 3597, + "fact": 3598, + "environ": 3599, + "harry": 3600, + "gone": 3601, + "medic": 3602, + "enjoying": 3603, + "justice": 3604, + "jr": 3605, + "indian": 3606, + "wife": 3607, + "sound": 3608, + "tes": 3609, + "drawing": 3610, + "pal": 3611, + "idea": 3612, + "crit": 3613, + "juli": 3614, + "iler": 3615, + "warm": 3616, + "clar": 3617, + "thoughts": 3618, + "defen": 3619, + "council": 3620, + "introduc": 3621, + "died": 3622, + "janu": 3623, + "ani": 3624, + "send": 3625, + "lier": 3626, + "ml": 3627, + "interesting": 3628, + "trade": 3629, + "wind": 3630, + "bay": 3631, + "sac": 3632, + "ancy": 3633, + "source": 3634, + "bes": 3635, + "organi": 3636, + "arly": 3637, + "large": 3638, + "ffici": 3639, + "tag": 3640, + "ut": 3641, + "desp": 3642, + "oes": 3643, + "title": 3644, + "sym": 3645, + "pictures": 3646, + "open": 3647, + "women": 3648, + "showing": 3649, + "ria": 3650, + "least": 3651, + "leadership": 3652, + "current": 3653, + "electr": 3654, + "valent": 3655, + "listening": 3656, + "ckey": 3657, + "general": 3658, + "deser": 3659, + "duce": 3660, + ";)": 3661, + "cent": 3662, + "ðŁĺįðŁĺį": 3663, + "scott": 3664, + "poor": 3665, + "selfie": 3666, + "events": 3667, + "ion": 3668, + "wrong": 3669, + "dev": 3670, + "hill": 3671, + "septe": 3672, + "culture": 3673, + "line": 3674, + "sorry": 3675, + "sent": 3676, + "sister": 3677, + "cept": 3678, + "kri": 3679, + "november": 3680, + "ari": 3681, + "announce": 3682, + "zation": 3683, + "bran": 3684, + "gent": 3685, + "du": 3686, + "len": 3687, + "pers": 3688, + "fm": 3689, + "martin": 3690, + "op": 3691, + "emb": 3692, + "ome": 3693, + "middle": 3694, + "success": 3695, + "peter": 3696, + "january": 3697, + "flu": 3698, + "racing": 3699, + "dav": 3700, + "bike": 3701, + "ðŁı»": 3702, + "pet": 3703, + "shoot": 3704, + "professi": 3705, + "featuring": 3706, + "september": 3707, + "nowplaying": 3708, + "staur": 3709, + "za": 3710, + "onic": 3711, + "quick": 3712, + "baske": 3713, + "speaking": 3714, + "milit": 3715, + "zer": 3716, + "chicken": 3717, + "bell": 3718, + "sad": 3719, + "coast": 3720, + "loving": 3721, + "yers": 3722, + "dj": 3723, + "panel": 3724, + "verage": 3725, + "swit": 3726, + "icks": 3727, + "bou": 3728, + "california": 3729, + "sam": 3730, + "parents": 3731, + "ero": 3732, + "killed": 3733, + "phys": 3734, + "jobs": 3735, + "migr": 3736, + "anth": 3737, + "emo": 3738, + "halloween": 3739, + "ander": 3740, + "cm": 3741, + "competition": 3742, + "eag": 3743, + "sket": 3744, + "spir": 3745, + "maybe": 3746, + "exclusive": 3747, + "appe": 3748, + "journey": 3749, + "screen": 3750, + "ford": 3751, + "io": 3752, + "hate": 3753, + "ug": 3754, + "soul": 3755, + "hero": 3756, + "society": 3757, + "syn": 3758, + "guit": 3759, + "nh": 3760, + "dj": 3761, + "ases": 3762, + "impre": 3763, + "time": 3764, + "sales": 3765, + "dd": 3766, + "fts": 3767, + "summit": 3768, + "stunning": 3769, + "oms": 3770, + "turned": 3771, + "clean": 3772, + "soft": 3773, + "beat": 3774, + "restaur": 3775, + "dered": 3776, + "ences": 3777, + "magic": 3778, + "dio": 3779, + "shine": 3780, + "guest": 3781, + "healthy": 3782, + "exhib": 3783, + "stories": 3784, + "popu": 3785, + "nis": 3786, + "ela": 3787, + "below": 3788, + "funny": 3789, + "results": 3790, + "sne": 3791, + "currently": 3792, + "ard": 3793, + "download": 3794, + "flight": 3795, + "mal": 3796, + "fine": 3797, + "pad": 3798, + "chu": 3799, + "ented": 3800, + "hat": 3801, + "ðŁijı": 3802, + "steve": 3803, + "jo": 3804, + "mark": 3805, + "rat": 3806, + "ball": 3807, + "pc": 3808, + "pon": 3809, + "bby": 3810, + "oli": 3811, + "arts": 3812, + "asure": 3813, + "bowl": 3814, + "attack": 3815, + "mic": 3816, + "dear": 3817, + "range": 3818, + "enter": 3819, + "chocolate": 3820, + "brilli": 3821, + "access": 3822, + ",\"": 3823, + "???": 3824, + "chap": 3825, + "const": 3826, + "tn": 3827, + "matter": 3828, + "blue": 3829, + "gallery": 3830, + "emp": 3831, + "workshop": 3832, + "leading": 3833, + "yours": 3834, + "basketball": 3835, + "wanna": 3836, + "thu": 3837, + "__": 3838, + "marri": 3839, + "sleep": 3840, + "bia": 3841, + "che": 3842, + "mad": 3843, + "impact": 3844, + "own": 3845, + "sir": 3846, + "channel": 3847, + "europe": 3848, + "esp": 3849, + "kitch": 3850, + "hospital": 3851, + "wra": 3852, + "royal": 3853, + "fs": 3854, + "neu": 3855, + "quar": 3856, + "ney": 3857, + "acks": 3858, + "chase": 3859, + "ppy": 3860, + "stal": 3861, + "ately": 3862, + "tim": 3863, + "december": 3864, + "rare": 3865, + "perform": 3866, + "cream": 3867, + "weight": 3868, + "choo": 3869, + "night": 3870, + "haven": 3871, + "franc": 3872, + "khan": 3873, + "built": 3874, + "helping": 3875, + "trust": 3876, + "type": 3877, + "golden": 3878, + "tax": 3879, + "snow": 3880, + "swi": 3881, + "disa": 3882, + "questions": 3883, + "vey": 3884, + "light": 3885, + "cn": 3886, + "cloud": 3887, + "thomas": 3888, + "aged": 3889, + "shou": 3890, + "teams": 3891, + "gran": 3892, + "reason": 3893, + "aa": 3894, + "youtube": 3895, + "vp": 3896, + "pizz": 3897, + "manager": 3898, + "bury": 3899, + "credit": 3900, + "treat": 3901, + "max": 3902, + "ik": 3903, + "main": 3904, + "ging": 3905, + "dead": 3906, + "probab": 3907, + "yeah": 3908, + "ãĤ": 3909, + "brand": 3910, + "soli": 3911, + "plant": 3912, + "tayl": 3913, + "girl": 3914, + "ðŁĺŃ": 3915, + "nament": 3916, + "auto": 3917, + "message": 3918, + "kore": 3919, + "nur": 3920, + "terr": 3921, + "agu": 3922, + "map": 3923, + "senting": 3924, + "loves": 3925, + "gives": 3926, + "gab": 3927, + "zen": 3928, + "robert": 3929, + "confir": 3930, + "wars": 3931, + "om": 3932, + "stain": 3933, + "camera": 3934, + "ander": 3935, + "wonder": 3936, + "ab": 3937, + "cap": 3938, + "sold": 3939, + "suit": 3940, + "walking": 3941, + "continue": 3942, + "effec": 3943, + "daughter": 3944, + "danc": 3945, + "chain": 3946, + "multi": 3947, + "kid": 3948, + "yan": 3949, + "champion": 3950, + "vo": 3951, + "tains": 3952, + "host": 3953, + "mini": 3954, + "missed": 3955, + "resc": 3956, + "lyn": 3957, + "finish": 3958, + "delicious": 3959, + "sas": 3960, + "taylor": 3961, + "ib": 3962, + "promis": 3963, + "products": 3964, + "mountain": 3965, + "florida": 3966, + "register": 3967, + "treat": 3968, + "recent": 3969, + "female": 3970, + "booth": 3971, + "matt": 3972, + "vehic": 3973, + "sop": 3974, + "motor": 3975, + "supporting": 3976, + "phic": 3977, + "extre": 3978, + "drink": 3979, + "lane": 3980, + "third": 3981, + "ps": 3982, + "constru": 3983, + "cere": 3984, + "farm": 3985, + "ðŁİī": 3986, + "tured": 3987, + "ðŁijī": 3988, + "cats": 3989, + "aj": 3990, + "gie": 3991, + "shooting": 3992, + "asked": 3993, + "pakistan": 3994, + "ame": 3995, + "mb": 3996, + "gil": 3997, + "legal": 3998, + "square": 3999, + "invol": 4000, + "draw": 4001, + "oooo": 4002, + "!!!!": 4003, + "opportunity": 4004, + "py": 4005, + "ei": 4006, + "bts": 4007, + "teacher": 4008, + "character": 4009, + "johnson": 4010, + "bron": 4011, + "lywood": 4012, + "chine": 4013, + "cing": 4014, + "cine": 4015, + "dge": 4016, + "gaming": 4017, + "russia": 4018, + "cia": 4019, + "quote": 4020, + "rich": 4021, + "gov": 4022, + "flowers": 4023, + "spiri": 4024, + "stin": 4025, + "growth": 4026, + "ðŁı¼": 4027, + "commer": 4028, + "juni": 4029, + "mum": 4030, + "ran": 4031, + "sna": 4032, + "aren": 4033, + "cb": 4034, + "actor": 4035, + "color": 4036, + "sit": 4037, + "pair": 4038, + "chi": 4039, + "bow": 4040, + "academy": 4041, + "held": 4042, + "rang": 4043, + "metal": 4044, + "yl": 4045, + "active": 4046, + "probably": 4047, + "tch": 4048, + "needed": 4049, + "spee": 4050, + "choice": 4051, + "italy": 4052, + "ryan": 4053, + "ðŁĩº": 4054, + "flower": 4055, + "vit": 4056, + "mn": 4057, + "foundation": 4058, + "bak": 4059, + "sions": 4060, + "neigh": 4061, + "floo": 4062, + "heard": 4063, + "remo": 4064, + "fresh": 4065, + "inging": 4066, + "ref": 4067, + "town": 4068, + "clou": 4069, + "jesus": 4070, + "spirit": 4071, + "couldn": 4072, + "zes": 4073, + "ðŁĴĻ": 4074, + "williams": 4075, + "proce": 4076, + "modern": 4077, + "process": 4078, + "shoes": 4079, + "created": 4080, + "tric": 4081, + "issues": 4082, + "anne": 4083, + "atten": 4084, + "debut": 4085, + "hr": 4086, + "nit": 4087, + "stig": 4088, + "apo": 4089, + "eps": 4090, + "zu": 4091, + "ãĢ": 4092, + "six": 4093, + "cards": 4094, + "langu": 4095, + "famous": 4096, + "tournament": 4097, + "sel": 4098, + "ebay": 4099, + "yn": 4100, + "ston": 4101, + "kick": 4102, + "announced": 4103, + "kam": 4104, + "voc": 4105, + "brilliant": 4106, + "house": 4107, + "cheese": 4108, + "warri": 4109, + "music": 4110, + "hockey": 4111, + "ðŁĺĤðŁĺĤ": 4112, + "skills": 4113, + "autom": 4114, + "smart": 4115, + "medical": 4116, + "mony": 4117, + "ex": 4118, + "guar": 4119, + "give": 4120, + "personal": 4121, + "vention": 4122, + "alli": 4123, + "press": 4124, + "floor": 4125, + "mc": 4126, + "victory": 4127, + "him": 4128, + "simple": 4129, + "thor": 4130, + "ðŁĩºðŁĩ": 4131, + "tail": 4132, + "lucky": 4133, + "alex": 4134, + "quite": 4135, + "bot": 4136, + "ssions": 4137, + "challeng": 4138, + "cann": 4139, + "amazon": 4140, + "hell": 4141, + "bought": 4142, + "):": 4143, + "edy": 4144, + "secret": 4145, + "production": 4146, + "independ": 4147, + "defe": 4148, + "added": 4149, + "pr": 4150, + "pag": 4151, + "bed": 4152, + "greatest": 4153, + "within": 4154, + "jay": 4155, + "ðŁ¥": 4156, + "ireland": 4157, + "rely": 4158, + "sd": 4159, + "text": 4160, + "driving": 4161, + "program": 4162, + "speed": 4163, + "colum": 4164, + "stron": 4165, + "é": 4166, + "forest": 4167, + "âĸ": 4168, + "machine": 4169, + "coin": 4170, + "scar": 4171, + "ount": 4172, + "bie": 4173, + "¡ï¸ı": 4174, + "portra": 4175, + "common": 4176, + "wrest": 4177, + "received": 4178, + "know": 4179, + "invest": 4180, + "plans": 4181, + "accor": 4182, + "adop": 4183, + "tery": 4184, + "reali": 4185, + "pp": 4186, + "kal": 4187, + "artwork": 4188, + "mean": 4189, + "god": 4190, + "instead": 4191, + "anci": 4192, + "motivation": 4193, + "asing": 4194, + "inspiration": 4195, + "upcoming": 4196, + "political": 4197, + "europe": 4198, + "mers": 4199, + "heavy": 4200, + "ðŁijį": 4201, + "febru": 4202, + "scotland": 4203, + "ough": 4204, + "bt": 4205, + "boss": 4206, + "schedu": 4207, + "speak": 4208, + "nick": 4209, + "ured": 4210, + "ino": 4211, + "ek": 4212, + "risk": 4213, + "tory": 4214, + "presents": 4215, + "bon": 4216, + "rug": 4217, + "states": 4218, + "exhibition": 4219, + "ilo": 4220, + "mill": 4221, + "brought": 4222, + ":-)": 4223, + "touri": 4224, + "come": 4225, + "officially": 4226, + "champions": 4227, + "doors": 4228, + "rep": 4229, + "pose": 4230, + "extra": 4231, + "kings": 4232, + "soccer": 4233, + "squad": 4234, + "applic": 4235, + "ata": 4236, + "sometimes": 4237, + "tari": 4238, + "excellent": 4239, + "ðŁĺĺ": 4240, + "straight": 4241, + "carol": 4242, + "rip": 4243, + "âĢį": 4244, + "graphic": 4245, + "mol": 4246, + "election": 4247, + "february": 4248, + "asons": 4249, + "li": 4250, + "dir": 4251, + "mt": 4252, + "nick": 4253, + "usu": 4254, + "mrs": 4255, + "comics": 4256, + "institu": 4257, + "corpor": 4258, + "vi": 4259, + "ðŁĻı": 4260, + "tural": 4261, + "dise": 4262, + "acci": 4263, + "weare": 4264, + "among": 4265, + "shopping": 4266, + "till": 4267, + "what": 4268, + "chair": 4269, + "span": 4270, + "chinese": 4271, + "innovation": 4272, + "joy": 4273, + "kit": 4274, + "century": 4275, + "obama": 4276, + "phili": 4277, + "fc": 4278, + "reach": 4279, + "citi": 4280, + "ulous": 4281, + "non": 4282, + "dang": 4283, + "happening": 4284, + "burn": 4285, + "pel": 4286, + "orange": 4287, + "dv": 4288, + "kick": 4289, + "claim": 4290, + "ingham": 4291, + "phy": 4292, + "nov": 4293, + "podcast": 4294, + "whi": 4295, + "nights": 4296, + "earlier": 4297, + "bear": 4298, + "lah": 4299, + "exciting": 4300, + "ora": 4301, + "given": 4302, + "slo": 4303, + "memories": 4304, + "continues": 4305, + "product": 4306, + "gho": 4307, + "cd": 4308, + "knows": 4309, + "ðŁİī": 4310, + "published": 4311, + "discuss": 4312, + "yard": 4313, + "iphone": 4314, + "tries": 4315, + "wall": 4316, + "feb": 4317, + "aren": 4318, + "truth": 4319, + "winners": 4320, + "ture": 4321, + "ditional": 4322, + "military": 4323, + "problem": 4324, + "mand": 4325, + "dog": 4326, + "loss": 4327, + "cric": 4328, + "canadi": 4329, + "veter": 4330, + "village": 4331, + "\",": 4332, + "yr": 4333, + "ung": 4334, + "donald": 4335, + "aging": 4336, + "birds": 4337, + "scienti": 4338, + "les": 4339, + "this": 4340, + "region": 4341, + "tical": 4342, + "itten": 4343, + "ila": 4344, + "ðŁĺİ": 4345, + "dad": 4346, + "diam": 4347, + "above": 4348, + "stren": 4349, + "lit": 4350, + "pir": 4351, + "lab": 4352, + "focus": 4353, + "busy": 4354, + "dur": 4355, + "apply": 4356, + "sma": 4357, + "author": 4358, + "aci": 4359, + "execu": 4360, + "domin": 4361, + "rela": 4362, + "jackson": 4363, + "ato": 4364, + "washington": 4365, + "ðŁĻĮ": 4366, + "kill": 4367, + "popular": 4368, + "cement": 4369, + "road": 4370, + "eating": 4371, + "location": 4372, + "vent": 4373, + "arre": 4374, + "nan": 4375, + "custo": 4376, + "adventure": 4377, + "ordin": 4378, + "sport": 4379, + "ult": 4380, + "lock": 4381, + "question": 4382, + "driver": 4383, + "landsc": 4384, + "oni": 4385, + "kins": 4386, + "pd": 4387, + "jordan": 4388, + "tered": 4389, + "kk": 4390, + "af": 4391, + "child": 4392, + "sp": 4393, + "justin": 4394, + "eni": 4395, + "selling": 4396, + "zo": 4397, + "whit": 4398, + "boston": 4399, + "particip": 4400, + "signing": 4401, + "happened": 4402, + "heat": 4403, + "mam": 4404, + "dreams": 4405, + "lows": 4406, + "graph": 4407, + "theday": 4408, + "heading": 4409, + "bro": 4410, + "blessed": 4411, + "vic": 4412, + "vegas": 4413, + "hd": 4414, + "inning": 4415, + "roman": 4416, + "andro": 4417, + "denti": 4418, + "use": 4419, + "cit": 4420, + "progress": 4421, + "writer": 4422, + "bob": 4423, + "ffs": 4424, + "growing": 4425, + "bly": 4426, + "aware": 4427, + "exam": 4428, + "spent": 4429, + "bet": 4430, + "score": 4431, + "beyond": 4432, + "docu": 4433, + "adel": 4434, + "sf": 4435, + "coura": 4436, + "collabor": 4437, + "inc": 4438, + "private": 4439, + "boat": 4440, + "**": 4441, + "zone": 4442, + "pha": 4443, + "bill": 4444, + "total": 4445, + "planning": 4446, + "towards": 4447, + "places": 4448, + "preview": 4449, + "creative": 4450, + "damn": 4451, + "ideas": 4452, + "seems": 4453, + "poten": 4454, + "saying": 4455, + "display": 4456, + "sw": 4457, + "aqu": 4458, + "louis": 4459, + "bye": 4460, + "lil": 4461, + "email": 4462, + "western": 4463, + "germany": 4464, + "eller": 4465, + "res": 4466, + "fant": 4467, + "mentary": 4468, + "deals": 4469, + "richard": 4470, + "jersey": 4471, + "streng": 4472, + "rad": 4473, + "pizza": 4474, + "mond": 4475, + "ware": 4476, + "lac": 4477, + "gi": 4478, + "archi": 4479, + "cd": 4480, + "yellow": 4481, + "recently": 4482, + "reach": 4483, + "à¹": 4484, + "kitchen": 4485, + "designed": 4486, + "try": 4487, + "gal": 4488, + "restaurant": 4489, + "ature": 4490, + "ww": 4491, + "jas": 4492, + "lma": 4493, + "ðŁijĮ": 4494, + "pain": 4495, + "avo": 4496, + "minute": 4497, + "schol": 4498, + "therap": 4499, + "ticket": 4500, + "dry": 4501, + "japan": 4502, + "ditions": 4503, + "terri": 4504, + "selves": 4505, + "happen": 4506, + "tup": 4507, + "mag": 4508, + "copy": 4509, + "sher": 4510, + "freedom": 4511, + "file": 4512, + "specially": 4513, + "toronto": 4514, + "load": 4515, + "gary": 4516, + "rey": 4517, + "answer": 4518, + "loy": 4519, + "caught": 4520, + "prize": 4521, + "une": 4522, + "fication": 4523, + "niger": 4524, + "syd": 4525, + "touch": 4526, + "feature": 4527, + "jazz": 4528, + "records": 4529, + "himself": 4530, + "dish": 4531, + "rober": 4532, + "spotted": 4533, + "master": 4534, + "wave": 4535, + "finals": 4536, + "bull": 4537, + "forum": 4538, + "ald": 4539, + "recomm": 4540, + "cha": 4541, + "ae": 4542, + "doo": 4543, + "instru": 4544, + "truly": 4545, + "lg": 4546, + "ink": 4547, + "brothers": 4548, + "dest": 4549, + "jim": 4550, + "mit": 4551, + "closed": 4552, + "ison": 4553, + "tried": 4554, + "santa": 4555, + "affe": 4556, + "wan": 4557, + "horse": 4558, + "grow": 4559, + "campus": 4560, + "relation": 4561, + "native": 4562, + "journ": 4563, + "gov": 4564, + "oct": 4565, + "kit": 4566, + "bound": 4567, + "partner": 4568, + "rema": 4569, + "crowd": 4570, + "!)": 4571, + "calls": 4572, + "rail": 4573, + "quali": 4574, + "solution": 4575, + "contest": 4576, + "convers": 4577, + "snap": 4578, + "base": 4579, + "initi": 4580, + "tax": 4581, + "ye": 4582, + "entrepre": 4583, + "itor": 4584, + "construction": 4585, + "food": 4586, + "presented": 4587, + "nings": 4588, + "climate": 4589, + "km": 4590, + "model": 4591, + "bj": 4592, + "block": 4593, + "presentation": 4594, + "dream": 4595, + "fix": 4596, + "calling": 4597, + "busine": 4598, + "congress": 4599, + "understand": 4600, + "web": 4601, + "value": 4602, + "ï¸ıâĥ£": 4603, + "mexico": 4604, + "itely": 4605, + "kim": 4606, + "charity": 4607, + "reflec": 4608, + "blan": 4609, + "flying": 4610, + "analy": 4611, + "families": 4612, + "band": 4613, + "recipe": 4614, + "celebration": 4615, + "accep": 4616, + "ary": 4617, + "tot": 4618, + "gb": 4619, + "interested": 4620, + "captain": 4621, + "âĻ¥": 4622, + "tip": 4623, + "absol": 4624, + "braz": 4625, + "investig": 4626, + "ology": 4627, + "dec": 4628, + "truck": 4629, + "vering": 4630, + "clear": 4631, + "dont": 4632, + "gotta": 4633, + "advis": 4634, + "begins": 4635, + "mass": 4636, + "descri": 4637, + "block": 4638, + "kim": 4639, + "david": 4640, + "songs": 4641, + "memorial": 4642, + "features": 4643, + "sustain": 4644, + "'.": 4645, + "grab": 4646, + "jose": 4647, + "va": 4648, + "conserv": 4649, + "sets": 4650, + "manchester": 4651, + "fighting": 4652, + "degre": 4653, + "aga": 4654, + "ind": 4655, + "sleep": 4656, + "position": 4657, + "hair": 4658, + "signs": 4659, + "policy": 4660, + "ito": 4661, + "alert": 4662, + "stam": 4663, + "spend": 4664, + "wy": 4665, + "absolut": 4666, + "dm": 4667, + "animal": 4668, + "myster": 4669, + "successful": 4670, + "problems": 4671, + "robo": 4672, + "kay": 4673, + "garden": 4674, + "pd": 4675, + "mayor": 4676, + "dale": 4677, + "tol": 4678, + "offers": 4679, + "visiting": 4680, + "friendly": 4681, + "trees": 4682, + "officer": 4683, + "account": 4684, + "kevin": 4685, + "ðŁijį": 4686, + "giant": 4687, + "continu": 4688, + "consu": 4689, + "tract": 4690, + "nfl": 4691, + "ðŁĺĬ": 4692, + "hq": 4693, + "bility": 4694, + "aar": 4695, + "disney": 4696, + "teen": 4697, + "oned": 4698, + "white": 4699, + "trailer": 4700, + "dedic": 4701, + "alone": 4702, + "absolutely": 4703, + "digital": 4704, + "william": 4705, + "ination": 4706, + "swa": 4707, + "ee": 4708, + "entire": 4709, + "german": 4710, + "roll": 4711, + "hits": 4712, + "cost": 4713, + "stay": 4714, + "tha": 4715, + "alive": 4716, + "according": 4717, + "cot": 4718, + "literally": 4719, + "herit": 4720, + "reti": 4721, + "hahaha": 4722, + "experi": 4723, + "likes": 4724, + "gt": 4725, + "steel": 4726, + "____": 4727, + "chair": 4728, + "christian": 4729, + "tower": 4730, + "difference": 4731, + "md": 4732, + "tress": 4733, + "mid": 4734, + "prince": 4735, + "african": 4736, + "feder": 4737, + "foot": 4738, + "carri": 4739, + "served": 4740, + "rice": 4741, + "shall": 4742, + "featured": 4743, + "cker": 4744, + "recru": 4745, + "poe": 4746, + "sense": 4747, + "nific": 4748, + "comedy": 4749, + "content": 4750, + "fat": 4751, + "posted": 4752, + "contribu": 4753, + "timate": 4754, + "liver": 4755, + "mble": 4756, + "internet": 4757, + "age": 4758, + "european": 4759, + "cling": 4760, + "glad": 4761, + "ffic": 4762, + "sco": 4763, + "akes": 4764, + "elle": 4765, + "termin": 4766, + "tony": 4767, + "pale": 4768, + "colour": 4769, + "serious": 4770, + "patri": 4771, + "movies": 4772, + "bm": 4773, + "professional": 4774, + "ado": 4775, + "alu": 4776, + "bringing": 4777, + "falls": 4778, + "israel": 4779, + "term": 4780, + "language": 4781, + "brook": 4782, + "mann": 4783, + "communic": 4784, + "cannot": 4785, + "acti": 4786, + "phe": 4787, + "yan": 4788, + "entreprene": 4789, + "turkey": 4790, + "logical": 4791, + "long": 4792, + "arm": 4793, + "urs": 4794, + "workers": 4795, + "ingly": 4796, + "ggs": 4797, + "ric": 4798, + "tual": 4799, + "receive": 4800, + "opens": 4801, + "gear": 4802, + "social": 4803, + "feet": 4804, + "cking": 4805, + "adver": 4806, + "finan": 4807, + "feels": 4808, + "spla": 4809, + "hr": 4810, + "easter": 4811, + "brain": 4812, + "ãģ": 4813, + "fig": 4814, + "ledge": 4815, + "nearly": 4816, + "protect": 4817, + "massive": 4818, + "eth": 4819, + "awa": 4820, + "ðŁĺģ": 4821, + "yrs": 4822, + "awareness": 4823, + "definitely": 4824, + "kn": 4825, + "imagine": 4826, + "ku": 4827, + "systems": 4828, + "ðŁijı": 4829, + "fas": 4830, + "lik": 4831, + "provide": 4832, + "amo": 4833, + "discover": 4834, + "influ": 4835, + "maker": 4836, + "gaz": 4837, + "fitness": 4838, + "street": 4839, + "ers": 4840, + "ted": 4841, + "wc": 4842, + "ysis": 4843, + "positive": 4844, + "helped": 4845, + "quest": 4846, + "andrew": 4847, + "brad": 4848, + "bin": 4849, + "hanging": 4850, + "ling": 4851, + "bright": 4852, + "section": 4853, + "mass": 4854, + "ðŁĻĮ": 4855, + "followers": 4856, + "hosting": 4857, + "tempor": 4858, + "flag": 4859, + "ave": 4860, + "letter": 4861, + "kur": 4862, + "requi": 4863, + "often": 4864, + "cryp": 4865, + "suff": 4866, + "âļ½": 4867, + "russian": 4868, + "treatment": 4869, + "alle": 4870, + "hay": 4871, + "lan": 4872, + "keeping": 4873, + "holy": 4874, + "powerful": 4875, + "predic": 4876, + "fund": 4877, + "especially": 4878, + "window": 4879, + "jewel": 4880, + "ily": 4881, + "ðŁĴľ": 4882, + "generation": 4883, + "appa": 4884, + "seriously": 4885, + "od": 4886, + "ðŁĺĤðŁĺĤðŁĺĤ": 4887, + "certi": 4888, + "irish": 4889, + "ðŁijĮ": 4890, + "miami": 4891, + "beth": 4892, + "vity": 4893, + "secu": 4894, + "chef": 4895, + "crime": 4896, + "graphy": 4897, + "max": 4898, + "artists": 4899, + "revolu": 4900, + "guard": 4901, + "speech": 4902, + "uc": 4903, + "updates": 4904, + "faces": 4905, + "stant": 4906, + "changed": 4907, + "reports": 4908, + "lower": 4909, + "pear": 4910, + "nc": 4911, + "kil": 4912, + "looked": 4913, + "speaker": 4914, + "sf": 4915, + "respect": 4916, + "okay": 4917, + "ocean": 4918, + "sitting": 4919, + "architecture": 4920, + "trail": 4921, + "seat": 4922, + "ira": 4923, + "leg": 4924, + "japanese": 4925, + "dam": 4926, + "ular": 4927, + "swim": 4928, + "politics": 4929, + "financial": 4930, + "old": 4931, + "mouth": 4932, + "attemp": 4933, + "destin": 4934, + "fishing": 4935, + "attention": 4936, + "mem": 4937, + "changes": 4938, + "decided": 4939, + "religi": 4940, + "gin": 4941, + "cav": 4942, + "zz": 4943, + "adam": 4944, + "mac": 4945, + "write": 4946, + "begin": 4947, + "scul": 4948, + "alter": 4949, + "iss": 4950, + "athon": 4951, + "images": 4952, + "moo": 4953, + "joined": 4954, + "ðŁĺī": 4955, + "âŀ¡ï¸ı": 4956, + "passed": 4957, + "musli": 4958, + "hir": 4959, + "largest": 4960, + "camer": 4961, + "comic": 4962, + "ghted": 4963, + "rugby": 4964, + "burgh": 4965, + "gging": 4966, + "testing": 4967, + "prepar": 4968, + "laugh": 4969, + "aled": 4970, + "improve": 4971, + "believ": 4972, + "advice": 4973, + "shares": 4974, + "heart": 4975, + "turning": 4976, + "sb": 4977, + "tel": 4978, + "cafe": 4979, + "nes": 4980, + "daniel": 4981, + "patter": 4982, + "tz": 4983, + "sett": 4984, + "park": 4985, + "cand": 4986, + "stick": 4987, + "happens": 4988, + "brian": 4989, + "newest": 4990, + "epic": 4991, + "ador": 4992, + "kies": 4993, + "warning": 4994, + "animals": 4995, + "custom": 4996, + "arc": 4997, + "dian": 4998, + "gold": 4999, + "core": 5000, + "tf": 5001, + "city": 5002, + "pants": 5003, + "reality": 5004, + "confi": 5005, + "inju": 5006, + "fox": 5007, + "guil": 5008, + "knew": 5009, + "âĺº": 5010, + "correc": 5011, + "itude": 5012, + "dden": 5013, + ".#": 5014, + "reduc": 5015, + "pass": 5016, + "fon": 5017, + "ya": 5018, + "owner": 5019, + "returns": 5020, + "nc": 5021, + "east": 5022, + "apol": 5023, + "insur": 5024, + "tho": 5025, + "sim": 5026, + "junior": 5027, + "bee": 5028, + "angel": 5029, + "attle": 5030, + "electric": 5031, + "horror": 5032, + "crash": 5033, + "eye": 5034, + "path": 5035, + "southern": 5036, + "employe": 5037, + "geo": 5038, + "tan": 5039, + "haz": 5040, + "rally": 5041, + "ðŁı»": 5042, + "property": 5043, + "wasn": 5044, + "enjoyed": 5045, + "grey": 5046, + "gas": 5047, + "brew": 5048, + "northern": 5049, + "holding": 5050, + "gp": 5051, + "take": 5052, + "chart": 5053, + "lyn": 5054, + "drama": 5055, + "zo": 5056, + "paid": 5057, + "throwback": 5058, + "cup": 5059, + "discussion": 5060, + "downtown": 5061, + "will": 5062, + "lew": 5063, + "bis": 5064, + "tary": 5065, + "bread": 5066, + "upon": 5067, + "rate": 5068, + "teachers": 5069, + "itation": 5070, + "anced": 5071, + "cycle": 5072, + "choose": 5073, + "dc": 5074, + "iran": 5075, + "cow": 5076, + "dave": 5077, + "raise": 5078, + "princess": 5079, + "faith": 5080, + "->": 5081, + "industri": 5082, + "spain": 5083, + "guitar": 5084, + "facts": 5085, + "mn": 5086, + "spen": 5087, + "courte": 5088, + "gott": 5089, + "projects": 5090, + "audi": 5091, + "osc": 5092, + "peter": 5093, + "sand": 5094, + "interest": 5095, + "happiness": 5096, + "venue": 5097, + "soldi": 5098, + "surprise": 5099, + "potential": 5100, + "perio": 5101, + "customer": 5102, + "ii": 5103, + "gni": 5104, + "manufac": 5105, + "eco": 5106, + "broken": 5107, + "singer": 5108, + "vels": 5109, + "wales": 5110, + "hus": 5111, + "inj": 5112, + "four": 5113, + "talent": 5114, + "dying": 5115, + "matthe": 5116, + "film": 5117, + "joining": 5118, + "sell": 5119, + "jar": 5120, + "lmao": 5121, + "surger": 5122, + "bbc": 5123, + "sources": 5124, + "austin": 5125, + "nik": 5126, + "charles": 5127, + "fam": 5128, + "princi": 5129, + "angel": 5130, + "cash": 5131, + "lot": 5132, + "ored": 5133, + "plays": 5134, + "plate": 5135, + "done": 5136, + "memory": 5137, + "brings": 5138, + "nba": 5139, + "solutions": 5140, + "teaching": 5141, + "grace": 5142, + "circu": 5143, + "helps": 5144, + "founder": 5145, + "mary": 5146, + "explore": 5147, + "decor": 5148, + "parts": 5149, + "cho": 5150, + "integr": 5151, + "hau": 5152, + "ises": 5153, + "putting": 5154, + "iner": 5155, + "rit": 5156, + "vy": 5157, + "michel": 5158, + "blues": 5159, + "everyday": 5160, + "forms": 5161, + "bio": 5162, + "year": 5163, + "pin": 5164, + "tter": 5165, + "spring": 5166, + "))": 5167, + "pot": 5168, + "aling": 5169, + "performing": 5170, + "shan": 5171, + "planet": 5172, + "musical": 5173, + "heads": 5174, + "italian": 5175, + "strugg": 5176, + "âĢįâĻ": 5177, + "wings": 5178, + "pump": 5179, + "hh": 5180, + "trou": 5181, + "aid": 5182, + "prime": 5183, + "earth": 5184, + "paint": 5185, + "mont": 5186, + "amy": 5187, + "bbc": 5188, + "fabulous": 5189, + "fruit": 5190, + "android": 5191, + "bourne": 5192, + "ceremony": 5193, + "ential": 5194, + "??": 5195, + "debate": 5196, + "oning": 5197, + "draft": 5198, + "solar": 5199, + "tx": 5200, + "jam": 5201, + "corn": 5202, + "!!!!!": 5203, + "broo": 5204, + "milk": 5205, + "posed": 5206, + "ohi": 5207, + "movement": 5208, + "bren": 5209, + "partner": 5210, + "pg": 5211, + "ette": 5212, + "aries": 5213, + "shout": 5214, + "ng": 5215, + "leaving": 5216, + "tells": 5217, + "sens": 5218, + "taste": 5219, + "kelly": 5220, + "worl": 5221, + "gym": 5222, + "rich": 5223, + "egy": 5224, + "pid": 5225, + "mas": 5226, + "âĤ": 5227, + "courtesy": 5228, + "frank": 5229, + "increase": 5230, + "written": 5231, + "ppers": 5232, + "rel": 5233, + "hai": 5234, + "sas": 5235, + "sound": 5236, + "tti": 5237, + "wich": 5238, + "river": 5239, + "...\"": 5240, + "ag": 5241, + "fellow": 5242, + "rome": 5243, + "small": 5244, + "gency": 5245, + "ican": 5246, + "luxury": 5247, + "proof": 5248, + "met": 5249, + "wildlife": 5250, + "moments": 5251, + "rather": 5252, + "corner": 5253, + "compe": 5254, + "canadian": 5255, + "likely": 5256, + "therapy": 5257, + "liam": 5258, + "economic": 5259, + "indie": 5260, + "route": 5261, + "fight": 5262, + "hope": 5263, + "setting": 5264, + "antly": 5265, + "cross": 5266, + "fantasy": 5267, + "dee": 5268, + "sketch": 5269, + "compli": 5270, + "ymi": 5271, + "rules": 5272, + "engineering": 5273, + "figure": 5274, + "row": 5275, + ".,": 5276, + "fw": 5277, + "sydney": 5278, + "wou": 5279, + "tation": 5280, + "drew": 5281, + "uses": 5282, + "there": 5283, + "spread": 5284, + "structure": 5285, + "patrick": 5286, + "apparently": 5287, + "ros": 5288, + "hills": 5289, + "wwe": 5290, + "anny": 5291, + "commission": 5292, + "div": 5293, + "fying": 5294, + "consul": 5295, + "analysis": 5296, + "exi": 5297, + "tennis": 5298, + "vehicle": 5299, + "ðŁĺŃðŁĺŃ": 5300, + "ass": 5301, + "highly": 5302, + "opened": 5303, + "bann": 5304, + "ðŁĴĻ": 5305, + "mph": 5306, + "wishing": 5307, + "vor": 5308, + "fif": 5309, + "giveaway": 5310, + "rr": 5311, + "ray": 5312, + "jess": 5313, + "gat": 5314, + "icymi": 5315, + "xit": 5316, + "highest": 5317, + "york": 5318, + "pie": 5319, + "involved": 5320, + "higher": 5321, + "rie": 5322, + "malay": 5323, + "intelli": 5324, + "despite": 5325, + "chee": 5326, + "sarah": 5327, + "bean": 5328, + "recogni": 5329, + "arsen": 5330, + "talented": 5331, + "passion": 5332, + "ich": 5333, + "abc": 5334, + "leads": 5335, + "disease": 5336, + "vis": 5337, + "sec": 5338, + "presenting": 5339, + "milli": 5340, + "hole": 5341, + "shots": 5342, + "depart": 5343, + "surgery": 5344, + "govt": 5345, + "bin": 5346, + "dual": 5347, + "evi": 5348, + "longer": 5349, + "evol": 5350, + "screen": 5351, + "portrait": 5352, + "etc": 5353, + "lose": 5354, + "chat": 5355, + "pen": 5356, + "pi": 5357, + "oma": 5358, + "sick": 5359, + "erc": 5360, + "companies": 5361, + "entry": 5362, + "plane": 5363, + "gry": 5364, + "vene": 5365, + "liverpool": 5366, + "premiere": 5367, + "shared": 5368, + "ared": 5369, + "films": 5370, + "ira": 5371, + "holidays": 5372, + "cricket": 5373, + "ician": 5374, + "ving": 5375, + ".)": 5376, + "ultimate": 5377, + "division": 5378, + "conduc": 5379, + "sept": 5380, + "forces": 5381, + "mont": 5382, + "smart": 5383, + "disapp": 5384, + "sunshine": 5385, + "ind": 5386, + "bless": 5387, + "made": 5388, + "colors": 5389, + "frank": 5390, + "iron": 5391, + "bottle": 5392, + "sgo": 5393, + "mood": 5394, + "jason": 5395, + "eric": 5396, + "birth": 5397, + "teen": 5398, + "response": 5399, + "target": 5400, + "statement": 5401, + "fear": 5402, + "thel": 5403, + "alum": 5404, + "arab": 5405, + "blin": 5406, + "direction": 5407, + "steps": 5408, + "erial": 5409, + "worked": 5410, + "atl": 5411, + "ðŁĴķ": 5412, + "felt": 5413, + "poli": 5414, + "scenes": 5415, + "homes": 5416, + "bell": 5417, + "eat": 5418, + "ateful": 5419, + "tin": 5420, + "lace": 5421, + "folks": 5422, + "pse": 5423, + "ann": 5424, + "wisdom": 5425, + "fav": 5426, + "butter": 5427, + "sr": 5428, + "areas": 5429, + "smoo": 5430, + "biz": 5431, + "dges": 5432, + "appo": 5433, + "more": 5434, + "them": 5435, + "effect": 5436, + "windows": 5437, + "sunny": 5438, + "capital": 5439, + "totally": 5440, + "cities": 5441, + "grant": 5442, + "mbers": 5443, + "slow": 5444, + "autu": 5445, + "ilities": 5446, + "wro": 5447, + "rising": 5448, + "stics": 5449, + "violence": 5450, + "igh": 5451, + "quot": 5452, + "hit": 5453, + "tc": 5454, + "heritage": 5455, + "buff": 5456, + "nes": 5457, + "zar": 5458, + "dential": 5459, + "exac": 5460, + "edge": 5461, + "deep": 5462, + "arena": 5463, + "became": 5464, + "benefits": 5465, + "marks": 5466, + "mber": 5467, + "az": 5468, + "ames": 5469, + "preci": 5470, + "dragon": 5471, + "reg": 5472, + "dings": 5473, + "dos": 5474, + "ðŁĴª": 5475, + "nel": 5476, + "sity": 5477, + "meal": 5478, + "dist": 5479, + "legend": 5480, + "purchase": 5481, + "pical": 5482, + "stick": 5483, + "fat": 5484, + "duba": 5485, + "profess": 5486, + "carto": 5487, + "prof": 5488, + "countries": 5489, + "responsi": 5490, + "sequ": 5491, + "fab": 5492, + "tribute": 5493, + "honored": 5494, + "practic": 5495, + "purple": 5496, + "anton": 5497, + "pared": 5498, + "tough": 5499, + "summer": 5500, + "environment": 5501, + "sons": 5502, + "ðŁĻı": 5503, + "mps": 5504, + "gies": 5505, + "heroes": 5506, + "telling": 5507, + "henry": 5508, + "fen": 5509, + "knowledge": 5510, + "Ģï¸ı": 5511, + "fr": 5512, + "neg": 5513, + "ure": 5514, + "acking": 5515, + "hearts": 5516, + "soo": 5517, + "hollywood": 5518, + "jump": 5519, + "sauce": 5520, + "schedule": 5521, + "turn": 5522, + "yoga": 5523, + "creating": 5524, + "cket": 5525, + "creek": 5526, + "âŃ": 5527, + "customers": 5528, + "madri": 5529, + "gul": 5530, + "assemb": 5531, + "mount": 5532, + "cell": 5533, + "top": 5534, + "stal": 5535, + "davis": 5536, + "twi": 5537, + "sign": 5538, + "premier": 5539, + "itions": 5540, + "hearing": 5541, + "unk": 5542, + "patients": 5543, + "appear": 5544, + "heaven": 5545, + "alty": 5546, + "doctor": 5547, + "ae": 5548, + "platform": 5549, + "jeff": 5550, + "ðŁĵ·": 5551, + "regional": 5552, + "bid": 5553, + "boxing": 5554, + "exten": 5555, + "ority": 5556, + "aw": 5557, + "wise": 5558, + "ille": 5559, + "several": 5560, + "bie": 5561, + "situ": 5562, + "syria": 5563, + "âľħ": 5564, + "reminder": 5565, + "entertain": 5566, + "lion": 5567, + "partners": 5568, + "inn": 5569, + "phar": 5570, + "fau": 5571, + "pls": 5572, + "expected": 5573, + "sugar": 5574, + "decision": 5575, + "sb": 5576, + "chron": 5577, + "association": 5578, + "leaves": 5579, + "visited": 5580, + "shap": 5581, + "ðŁĴĸ": 5582, + "further": 5583, + "hann": 5584, + "wi": 5585, + "runs": 5586, + "ler": 5587, + "funding": 5588, + "filled": 5589, + "......": 5590, + "tiny": 5591, + "hang": 5592, + "org": 5593, + "cool": 5594, + "semin": 5595, + "ðŁıĨ": 5596, + "spons": 5597, + "navy": 5598, + "saint": 5599, + "drug": 5600, + "dal": 5601, + "roun": 5602, + "covered": 5603, + "traditional": 5604, + "investment": 5605, + "dete": 5606, + "alism": 5607, + "flow": 5608, + "nis": 5609, + "sunrise": 5610, + "feat": 5611, + "fted": 5612, + "weird": 5613, + "jere": 5614, + "vegan": 5615, + "medicine": 5616, + "ano": 5617, + "accu": 5618, + "delivery": 5619, + "temple": 5620, + "changing": 5621, + "wilson": 5622, + "philipp": 5623, + "refe": 5624, + "nd": 5625, + "iser": 5626, + "gay": 5627, + "rand": 5628, + "atives": 5629, + "tely": 5630, + "pand": 5631, + "intellig": 5632, + "gare": 5633, + "ambas": 5634, + "demon": 5635, + "committee": 5636, + "strategy": 5637, + "refuge": 5638, + "budget": 5639, + "protec": 5640, + "pier": 5641, + "express": 5642, + "nomin": 5643, + "economy": 5644, + "allow": 5645, + "icon": 5646, + "galax": 5647, + "oh": 5648, + "indivi": 5649, + "demand": 5650, + "virgin": 5651, + "luke": 5652, + "alists": 5653, + "mani": 5654, + "smi": 5655, + "judge": 5656, + "enty": 5657, + "michi": 5658, + "result": 5659, + "amed": 5660, + "speaks": 5661, + "',": 5662, + "houston": 5663, + "shin": 5664, + "bing": 5665, + "fly": 5666, + "chem": 5667, + "auto": 5668, + "vas": 5669, + "get": 5670, + "arm": 5671, + "thanks": 5672, + "din": 5673, + "gang": 5674, + "xx": 5675, + "sion": 5676, + "located": 5677, + "pl": 5678, + "josh": 5679, + "info": 5680, + "joins": 5681, + "adverti": 5682, + "otd": 5683, + "eld": 5684, + "sie": 5685, + "reasons": 5686, + "vent": 5687, + "ðŁĩºðŁĩ¸": 5688, + "âł": 5689, + "conversation": 5690, + "studi": 5691, + "ðŁĶ¥ðŁĶ¥": 5692, + "gos": 5693, + "sounds": 5694, + "unit": 5695, + "musc": 5696, + "gel": 5697, + "acked": 5698, + "paci": 5699, + "cos": 5700, + "dere": 5701, + "uu": 5702, + "ao": 5703, + "lam": 5704, + "inspiring": 5705, + "arms": 5706, + "tware": 5707, + "matters": 5708, + "addic": 5709, + "dude": 5710, + "ext": 5711, + "crisis": 5712, + "bath": 5713, + "meet": 5714, + "singh": 5715, + "expect": 5716, + "delhi": 5717, + "rescue": 5718, + "worst": 5719, + "aug": 5720, + "shipping": 5721, + "serving": 5722, + "sto": 5723, + "dark": 5724, + "aces": 5725, + "historic": 5726, + "landscape": 5727, + "designer": 5728, + "billion": 5729, + "grateful": 5730, + "wake": 5731, + "eve": 5732, + "miller": 5733, + "housing": 5734, + "dynam": 5735, + "isco": 5736, + "beha": 5737, + "shop": 5738, + "prou": 5739, + "eas": 5740, + "asia": 5741, + "eding": 5742, + "kon": 5743, + "department": 5744, + "awar": 5745, + "marine": 5746, + "inci": 5747, + "photographer": 5748, + "tape": 5749, + "logo": 5750, + "rings": 5751, + "dit": 5752, + "----": 5753, + "vinyl": 5754, + "wc": 5755, + "voting": 5756, + "seven": 5757, + "ambassad": 5758, + "dallas": 5759, + "tu": 5760, + "comment": 5761, + "kra": 5762, + "bles": 5763, + "wag": 5764, + "ud": 5765, + "audio": 5766, + "strike": 5767, + "official": 5768, + "ots": 5769, + "metho": 5770, + "tools": 5771, + "radi": 5772, + "alan": 5773, + "hunt": 5774, + "watched": 5775, + "ake": 5776, + "fake": 5777, + "drinking": 5778, + "merry": 5779, + "ml": 5780, + "bday": 5781, + "rio": 5782, + "nike": 5783, + "cant": 5784, + "repe": 5785, + "costu": 5786, + "murder": 5787, + "akers": 5788, + "chers": 5789, + "outs": 5790, + "beginning": 5791, + "sos": 5792, + "ades": 5793, + "nin": 5794, + "notes": 5795, + "wrote": 5796, + "solo": 5797, + "ci": 5798, + "lighting": 5799, + "urban": 5800, + "brexit": 5801, + "attend": 5802, + "shirts": 5803, + "playo": 5804, + "actress": 5805, + "plic": 5806, + "standard": 5807, + "quotes": 5808, + "parade": 5809, + "ancient": 5810, + "©": 5811, + "turing": 5812, + "ree": 5813, + "primary": 5814, + "flash": 5815, + "citiz": 5816, + "mates": 5817, + "stein": 5818, + "zi": 5819, + "clinton": 5820, + "skin": 5821, + "gene": 5822, + "hum": 5823, + "gar": 5824, + "tle": 5825, + "yi": 5826, + "focu": 5827, + "dean": 5828, + "plants": 5829, + "cyber": 5830, + "bu": 5831, + "ome": 5832, + "hop": 5833, + "address": 5834, + "tix": 5835, + "gifts": 5836, + "relationship": 5837, + "subscri": 5838, + "feed": 5839, + "exactly": 5840, + "hawks": 5841, + "exo": 5842, + "stress": 5843, + "sn": 5844, + "arrested": 5845, + "ane": 5846, + "software": 5847, + "zero": 5848, + "theme": 5849, + "mumb": 5850, + "immigr": 5851, + "mia": 5852, + "makeup": 5853, + "pleasure": 5854, + "univers": 5855, + "harb": 5856, + "engine": 5857, + "aper": 5858, + "rin": 5859, + "bra": 5860, + "institute": 5861, + "leather": 5862, + "alth": 5863, + "singing": 5864, + "cos": 5865, + "ghty": 5866, + "meas": 5867, + "stic": 5868, + "side": 5869, + "insurance": 5870, + "cot": 5871, + "pitch": 5872, + "mountains": 5873, + "crimin": 5874, + "supre": 5875, + "valentine": 5876, + "ater": 5877, + "wouldn": 5878, + "scale": 5879, + "related": 5880, + "regar": 5881, + "startup": 5882, + "packed": 5883, + "mike": 5884, + "weekly": 5885, + "pts": 5886, + "count": 5887, + "har": 5888, + "gotten": 5889, + "mind": 5890, + "berlin": 5891, + "conditions": 5892, + "switch": 5893, + "corn": 5894, + "save": 5895, + "gli": 5896, + "emergency": 5897, + "tuned": 5898, + "stock": 5899, + "discussing": 5900, + "everybody": 5901, + "sday": 5902, + "whether": 5903, + "wrestling": 5904, + "eces": 5905, + "gender": 5906, + "chen": 5907, + "ðŁijĢ": 5908, + "madrid": 5909, + "marathon": 5910, + "egg": 5911, + "ier": 5912, + "thx": 5913, + "asking": 5914, + "korea": 5915, + "wolf": 5916, + "aya": 5917, + "gm": 5918, + "gau": 5919, + "atory": 5920, + "vr": 5921, + "grass": 5922, + "killing": 5923, + "bble": 5924, + "uro": 5925, + "uni": 5926, + "eth": 5927, + "shore": 5928, + "then": 5929, + "reale": 5930, + "bottom": 5931, + "exerc": 5932, + "kar": 5933, + "ories": 5934, + "adri": 5935, + "sands": 5936, + "sex": 5937, + ".'": 5938, + "volunteers": 5939, + "perform": 5940, + "parliam": 5941, + "include": 5942, + "delighted": 5943, + "executive": 5944, + "fuel": 5945, + "kiss": 5946, + "ãħ": 5947, + "charge": 5948, + "hu": 5949, + "cakes": 5950, + "vet": 5951, + "glu": 5952, + "agree": 5953, + "prices": 5954, + "nau": 5955, + "hl": 5956, + "gru": 5957, + "raj": 5958, + "strength": 5959, + "bic": 5960, + "spending": 5961, + "ales": 5962, + "aven": 5963, + "blast": 5964, + ":(": 5965, + "yof": 5966, + "normal": 5967, + "six": 5968, + "quick": 5969, + "sea": 5970, + "daw": 5971, + "meets": 5972, + "lovers": 5973, + "updated": 5974, + "potat": 5975, + "completed": 5976, + "cook": 5977, + "opportunities": 5978, + "pure": 5979, + "organic": 5980, + "temper": 5981, + "cam": 5982, + "avoid": 5983, + "parking": 5984, + "dubai": 5985, + "ando": 5986, + "distri": 5987, + "toy": 5988, + "completely": 5989, + "donald": 5990, + "trial": 5991, + "bass": 5992, + "boun": 5993, + "background": 5994, + "vas": 5995, + "marvel": 5996, + "lum": 5997, + "rus": 5998, + "tool": 5999, + "commissi": 6000, + "throwback": 6001, + "finding": 6002, + "islam": 6003, + "!?": 6004, + "stop": 6005, + "evil": 6006, + "oral": 6007, + "residents": 6008, + "identi": 6009, + "oak": 6010, + "ðŁİ¶": 6011, + "lil": 6012, + "spanish": 6013, + "chapter": 6014, + "stopped": 6015, + "direct": 6016, + "hosted": 6017, + "picked": 6018, + "labour": 6019, + "lewis": 6020, + "defense": 6021, + "à®": 6022, + "healthcare": 6023, + "whis": 6024, + "math": 6025, + "peak": 6026, + "raised": 6027, + "fix": 6028, + "bull": 6029, + "thir": 6030, + "chelsea": 6031, + "folk": 6032, + "tre": 6033, + "candi": 6034, + "paul": 6035, + "either": 6036, + "adam": 6037, + "poetry": 6038, + "jewelry": 6039, + "ðŁ¦": 6040, + "pray": 6041, + "ا": 6042, + "gc": 6043, + "oz": 6044, + "wishes": 6045, + "foreign": 6046, + "sung": 6047, + "learned": 6048, + "ene": 6049, + "ning": 6050, + "michael": 6051, + "illustration": 6052, + "legendary": 6053, + "wav": 6054, + "bau": 6055, + "ðŁļ¨": 6056, + "calend": 6057, + "streets": 6058, + "âĨ": 6059, + "monster": 6060, + "buck": 6061, + "gr": 6062, + "school": 6063, + "bath": 6064, + "waste": 6065, + "neck": 6066, + "hawa": 6067, + "beach": 6068, + "replac": 6069, + "ject": 6070, + "oner": 6071, + "factory": 6072, + "count": 6073, + "ðŁĵ¸": 6074, + "morgan": 6075, + "dering": 6076, + "sean": 6077, + "stephen": 6078, + "dep": 6079, + "novel": 6080, + "videos": 6081, + "ical": 6082, + "pressure": 6083, + "arsenal": 6084, + "expre": 6085, + "irs": 6086, + "trending": 6087, + "ssa": 6088, + "flash": 6089, + "resear": 6090, + "through": 6091, + "professor": 6092, + "sculp": 6093, + "tos": 6094, + "gged": 6095, + "mma": 6096, + "bee": 6097, + "ape": 6098, + "hunter": 6099, + "ami": 6100, + "hei": 6101, + "plastic": 6102, + "bucks": 6103, + "universe": 6104, + "legen": 6105, + "nigeria": 6106, + "pleased": 6107, + "ris": 6108, + "thinks": 6109, + "autumn": 6110, + "ids": 6111, + "dis": 6112, + "anthony": 6113, + "ðŁı½": 6114, + "aked": 6115, + "glasses": 6116, + "finance": 6117, + "zer": 6118, + "kas": 6119, + "contract": 6120, + "numbers": 6121, + "shaw": 6122, + "partnership": 6123, + "til": 6124, + "launched": 6125, + "sal": 6126, + "victoria": 6127, + "theater": 6128, + "usual": 6129, + "names": 6130, + "period": 6131, + "eliza": 6132, + "ith": 6133, + "barcel": 6134, + "rocks": 6135, + "bags": 6136, + "mate": 6137, + "distribu": 6138, + "jon": 6139, + "diffic": 6140, + "alized": 6141, + "curren": 6142, + "scored": 6143, + "bha": 6144, + "dublin": 6145, + "rose": 6146, + "inted": 6147, + "solid": 6148, + "behavi": 6149, + "walker": 6150, + "simply": 6151, + "gardens": 6152, + "headed": 6153, + "ini": 6154, + "ohio": 6155, + "weap": 6156, + "fo": 6157, + "glen": 6158, + "estate": 6159, + "random": 6160, + "thunder": 6161, + "thru": 6162, + "kill": 6163, + "jacket": 6164, + "iti": 6165, + "entertainment": 6166, + "thanksgiving": 6167, + "ental": 6168, + "encoura": 6169, + "elo": 6170, + "ather": 6171, + "tank": 6172, + "highlights": 6173, + "fting": 6174, + "rule": 6175, + "models": 6176, + "border": 6177, + "bjp": 6178, + "husband": 6179, + "indone": 6180, + "kenya": 6181, + "bears": 6182, + "alo": 6183, + "ninten": 6184, + "pix": 6185, + "stro": 6186, + "orders": 6187, + "salad": 6188, + "roads": 6189, + "nor": 6190, + "lation": 6191, + "sophi": 6192, + "ðŁı¼": 6193, + "pieces": 6194, + "bone": 6195, + "mins": 6196, + "includes": 6197, + "nutr": 6198, + "phil": 6199, + "sent": 6200, + "fundra": 6201, + "gain": 6202, + "borough": 6203, + "nad": 6204, + "monday": 6205, + "activity": 6206, + "items": 6207, + "becoming": 6208, + "kenne": 6209, + "detro": 6210, + "cardi": 6211, + "guests": 6212, + "ux": 6213, + "worldwide": 6214, + "severe": 6215, + "news": 6216, + "thankful": 6217, + "fiction": 6218, + "vege": 6219, + "mall": 6220, + "sian": 6221, + "eral": 6222, + "injury": 6223, + "lee": 6224, + "menu": 6225, + "dancing": 6226, + "scotti": 6227, + "example": 6228, + "(#": 6229, + "nai": 6230, + "studios": 6231, + "bai": 6232, + "ðŁĴĽ": 6233, + "jav": 6234, + "diamond": 6235, + "vince": 6236, + "rick": 6237, + "protection": 6238, + "lincol": 6239, + "champs": 6240, + "approach": 6241, + "dar": 6242, + "mile": 6243, + "clouds": 6244, + "jeff": 6245, + "infin": 6246, + "lers": 6247, + "ples": 6248, + "peace": 6249, + "gop": 6250, + "âĻ¡": 6251, + "techn": 6252, + "stra": 6253, + "average": 6254, + "effort": 6255, + "introducing": 6256, + "diversity": 6257, + "australian": 6258, + "amp": 6259, + "boost": 6260, + "ske": 6261, + "patient": 6262, + "appreciate": 6263, + "icians": 6264, + "pur": 6265, + "fell": 6266, + "woods": 6267, + "illustr": 6268, + "ðŁĸ": 6269, + "agency": 6270, + "actions": 6271, + "britain": 6272, + "underway": 6273, + "seattle": 6274, + "eland": 6275, + "ago": 6276, + "fill": 6277, + "streaming": 6278, + "protest": 6279, + "challenges": 6280, + "kyo": 6281, + "etsy": 6282, + "cooking": 6283, + "expert": 6284, + "russ": 6285, + "rainbow": 6286, + "commercial": 6287, + "spin": 6288, + "beats": 6289, + "cry": 6290, + "valu": 6291, + "eli": 6292, + "throw": 6293, + "grams": 6294, + "levels": 6295, + "michigan": 6296, + "cad": 6297, + "adorable": 6298, + "constitu": 6299, + "ws": 6300, + "pub": 6301, + "midnight": 6302, + "that": 6303, + "netfli": 6304, + "brazil": 6305, + "diego": 6306, + "regular": 6307, + "joy": 6308, + "âĤ¬": 6309, + "liqu": 6310, + "eastern": 6311, + "kni": 6312, + "flat": 6313, + "np": 6314, + "brown": 6315, + "wer": 6316, + "sey": 6317, + "tters": 6318, + "acting": 6319, + "vanc": 6320, + "cycling": 6321, + "programme": 6322, + "raw": 6323, + "complex": 6324, + "tattoo": 6325, + "throwbackthursday": 6326, + "sessions": 6327, + "rooms": 6328, + "sight": 6329, + "species": 6330, + "bomb": 6331, + "laugh": 6332, + "keeps": 6333, + "moon": 6334, + "officers": 6335, + "conver": 6336, + "tr": 6337, + "hash": 6338, + "tack": 6339, + "rious": 6340, + "adap": 6341, + "aj": 6342, + "recogn": 6343, + "expo": 6344, + "sugge": 6345, + "confirmed": 6346, + "rolling": 6347, + "dressing": 6348, + "ict": 6349, + "friday": 6350, + "phones": 6351, + "ridge": 6352, + "concept": 6353, + "roy": 6354, + "keys": 6355, + "effor": 6356, + "cate": 6357, + "kne": 6358, + "even": 6359, + "lay": 6360, + "communities": 6361, + "mod": 6362, + "naz": 6363, + "everywhere": 6364, + "alab": 6365, + "bitcoin": 6366, + "banks": 6367, + "outdoor": 6368, + "federal": 6369, + "stores": 6370, + "hp": 6371, + "cal": 6372, + "mely": 6373, + "signific": 6374, + "bear": 6375, + "republic": 6376, + "closer": 6377, + "allah": 6378, + "pick": 6379, + "xd": 6380, + "palace": 6381, + "chill": 6382, + "bam": 6383, + "erous": 6384, + "una": 6385, + "allen": 6386, + "outstanding": 6387, + "olympic": 6388, + "supply": 6389, + "figu": 6390, + "vau": 6391, + "lp": 6392, + "charlie": 6393, + "unes": 6394, + ">>>": 6395, + "legends": 6396, + "icial": 6397, + "coast": 6398, + "benefit": 6399, + "multi": 6400, + "fits": 6401, + "farmers": 6402, + "amount": 6403, + "sisters": 6404, + "harve": 6405, + "honey": 6406, + "queen": 6407, + "bers": 6408, + "plann": 6409, + "âŃIJ": 6410, + "mu": 6411, + "barcelona": 6412, + "alber": 6413, + "status": 6414, + "remain": 6415, + "extra": 6416, + "candy": 6417, + "vious": 6418, + "âľĮ": 6419, + "ov": 6420, + "warriors": 6421, + "-->": 6422, + "jump": 6423, + "amar": 6424, + "xmas": 6425, + "studies": 6426, + "iors": 6427, + "kor": 6428, + "donate": 6429, + "prep": 6430, + "fish": 6431, + "ima": 6432, + "painted": 6433, + "admini": 6434, + "cosplay": 6435, + "sports": 6436, + "drops": 6437, + "fighter": 6438, + "evidence": 6439, + "ðŁĴª": 6440, + "lake": 6441, + "rob": 6442, + "cinema": 6443, + "profile": 6444, + "ñ": 6445, + "stands": 6446, + "legacy": 6447, + "shape": 6448, + "roof": 6449, + "civil": 6450, + "ians": 6451, + "syl": 6452, + "sham": 6453, + "voted": 6454, + "retail": 6455, + "philli": 6456, + "listed": 6457, + "duty": 6458, + "nb": 6459, + "thes": 6460, + "fare": 6461, + "auction": 6462, + "fficial": 6463, + "storms": 6464, + "dp": 6465, + "loun": 6466, + "shops": 6467, + "aly": 6468, + "anime": 6469, + "multiple": 6470, + "ðŁĺįðŁĺį": 6471, + "psycho": 6472, + "jean": 6473, + "apart": 6474, + "candidate": 6475, + "ggy": 6476, + "conf": 6477, + "joseph": 6478, + "wick": 6479, + "meat": 6480, + "frame": 6481, + "cl": 6482, + "forgot": 6483, + "phy": 6484, + "fing": 6485, + "lied": 6486, + "rep": 6487, + "seed": 6488, + "fall": 6489, + "ufc": 6490, + "nut": 6491, + "lind": 6492, + "mode": 6493, + "fields": 6494, + "ence": 6495, + "sley": 6496, + "ðŁ¤Ķ": 6497, + "chill": 6498, + "followed": 6499, + "announces": 6500, + "corru": 6501, + "trophy": 6502, + "themselves": 6503, + "acle": 6504, + "aldu": 6505, + "kong": 6506, + "lon": 6507, + "sv": 6508, + "broke": 6509, + "anderson": 6510, + "tai": 6511, + "story": 6512, + "temporary": 6513, + "activities": 6514, + "kati": 6515, + "ariz": 6516, + "crystal": 6517, + "spoke": 6518, + "extremely": 6519, + "trading": 6520, + "ðŁĴļ": 6521, + "ü": 6522, + "inch": 6523, + "edin": 6524, + "outfit": 6525, + "equip": 6526, + "madi": 6527, + "formed": 6528, + "beef": 6529, + "pop": 6530, + "tiger": 6531, + "thisday": 6532, + "tired": 6533, + "neighb": 6534, + "retro": 6535, + "isa": 6536, + "unt": 6537, + "tas": 6538, + "kansas": 6539, + "dest": 6540, + "seconds": 6541, + "tay": 6542, + "hurric": 6543, + "ou": 6544, + "galaxy": 6545, + "daddy": 6546, + "brow": 6547, + "burger": 6548, + "enced": 6549, + "desk": 6550, + "accur": 6551, + "secretary": 6552, + "elite": 6553, + "kab": 6554, + "chin": 6555, + "tourism": 6556, + "buddy": 6557, + "icide": 6558, + "dressed": 6559, + "ud": 6560, + "vacation": 6561, + "cheers": 6562, + "comfor": 6563, + "characters": 6564, + "jet": 6565, + "buying": 6566, + "lins": 6567, + "nap": 6568, + "realestate": 6569, + "lie": 6570, + "afc": 6571, + "iii": 6572, + "fame": 6573, + "nr": 6574, + "bat": 6575, + "agent": 6576, + "makers": 6577, + "âĢ¼": 6578, + "sector": 6579, + "opti": 6580, + "leon": 6581, + "diet": 6582, + "prayer": 6583, + "hip": 6584, + "mir": 6585, + "lex": 6586, + "bry": 6587, + "ana": 6588, + "passing": 6589, + "wen": 6590, + "recovery": 6591, + "aki": 6592, + "popul": 6593, + "resort": 6594, + "maria": 6595, + "stuck": 6596, + "reads": 6597, + "tier": 6598, + "perfec": 6599, + "netflix": 6600, + "poo": 6601, + "champ": 6602, + "oc": 6603, + "reduce": 6604, + "wered": 6605, + "comments": 6606, + "claim": 6607, + "accident": 6608, + "sag": 6609, + "hack": 6610, + "salt": 6611, + "kinda": 6612, + "killer": 6613, + "ios": 6614, + "zy": 6615, + "exchange": 6616, + "lecture": 6617, + "enger": 6618, + "icking": 6619, + "tau": 6620, + "reveals": 6621, + "prison": 6622, + "zom": 6623, + "ghan": 6624, + "ul": 6625, + "journal": 6626, + "iot": 6627, + "trin": 6628, + "jona": 6629, + "governor": 6630, + "cape": 6631, + "quarter": 6632, + "spective": 6633, + "impressive": 6634, + "babies": 6635, + "tx": 6636, + "mill": 6637, + "oy": 6638, + "harri": 6639, + "joint": 6640, + "sue": 6641, + "collaboration": 6642, + "trend": 6643, + "revolution": 6644, + "renew": 6645, + "alumni": 6646, + "gett": 6647, + "shell": 6648, + "sunday": 6649, + "entu": 6650, + "nic": 6651, + "donaldtrump": 6652, + "blockchain": 6653, + "pacific": 6654, + "explains": 6655, + "spy": 6656, + "advoc": 6657, + "paradi": 6658, + "tof": 6659, + "starring": 6660, + "pav": 6661, + "feed": 6662, + "brac": 6663, + "smoke": 6664, + "hamp": 6665, + "yam": 6666, + "tokyo": 6667, + "simon": 6668, + "dh": 6669, + "effici": 6670, + "physical": 6671, + "nj": 6672, + "elli": 6673, + "slow": 6674, + "graduate": 6675, + "americans": 6676, + "tify": 6677, + "fred": 6678, + "apore": 6679, + "finds": 6680, + "robin": 6681, + "wet": 6682, + "notice": 6683, + "semi": 6684, + "unve": 6685, + "kom": 6686, + "pilot": 6687, + "screening": 6688, + "daily": 6689, + "ðŁĴĹ": 6690, + "royal": 6691, + "spa": 6692, + "votes": 6693, + "nag": 6694, + "whate": 6695, + "attending": 6696, + "experim": 6697, + "addition": 6698, + "kate": 6699, + "stol": 6700, + "mali": 6701, + "foot": 6702, + "christ": 6703, + "chan": 6704, + "dee": 6705, + "licen": 6706, + "global": 6707, + "moore": 6708, + "tia": 6709, + "brigh": 6710, + "mystery": 6711, + "yay": 6712, + "âĿ¤ï¸ıâĿ¤ï¸ı": 6713, + "creati": 6714, + "mechan": 6715, + "clock": 6716, + "dic": 6717, + "âĢĶ": 6718, + "pper": 6719, + "alph": 6720, + "throughout": 6721, + "allow": 6722, + "resources": 6723, + "selection": 6724, + "hamil": 6725, + "bbq": 6726, + "aaaa": 6727, + "virginia": 6728, + "disney": 6729, + "eng": 6730, + "sored": 6731, + "drinks": 6732, + "fancy": 6733, + "consider": 6734, + "enda": 6735, + "jane": 6736, + "handmade": 6737, + "dul": 6738, + "ontari": 6739, + "ius": 6740, + "sville": 6741, + "colorado": 6742, + "whatever": 6743, + "wheel": 6744, + "promise": 6745, + "never": 6746, + "designs": 6747, + "ably": 6748, + "sexual": 6749, + "vancou": 6750, + "ati": 6751, + "convention": 6752, + "cultural": 6753, + "singapore": 6754, + "promo": 6755, + "loaded": 6756, + "glasgo": 6757, + "ppl": 6758, + "noo": 6759, + "kee": 6760, + "stem": 6761, + "mention": 6762, + "ido": 6763, + "cruise": 6764, + "riding": 6765, + "becomes": 6766, + "bey": 6767, + "âļ½ï¸ı": 6768, + "twin": 6769, + "dedicated": 6770, + "nash": 6771, + "desi": 6772, + "workout": 6773, + "jenni": 6774, + "iv": 6775, + "groups": 6776, + "relax": 6777, + "phoeni": 6778, + "lift": 6779, + "mixed": 6780, + "mck": 6781, + "pc": 6782, + "must": 6783, + "metro": 6784, + "cies": 6785, + "yar": 6786, + "aim": 6787, + "anger": 6788, + "ie": 6789, + "recy": 6790, + "married": 6791, + "dropped": 6792, + "engag": 6793, + "lest": 6794, + "ambassador": 6795, + "oph": 6796, + "des": 6797, + "wick": 6798, + "assistant": 6799, + "natur": 6800, + "fail": 6801, + "ltd": 6802, + "short": 6803, + "kap": 6804, + "shaw": 6805, + "bigger": 6806, + "remains": 6807, + "critical": 6808, + "survey": 6809, + "coverage": 6810, + "erson": 6811, + "wind": 6812, + "nb": 6813, + "billy": 6814, + "letes": 6815, + "acts": 6816, + "jimmy": 6817, + "atlan": 6818, + "aland": 6819, + "tc": 6820, + "importance": 6821, + "damage": 6822, + "fg": 6823, + "storage": 6824, + "twt": 6825, + "bond": 6826, + "balance": 6827, + "crying": 6828, + "puppy": 6829, + "vote": 6830, + "push": 6831, + "ðŁĴľ": 6832, + "poly": 6833, + "mel": 6834, + "london": 6835, + "terrori": 6836, + "effective": 6837, + "corporate": 6838, + "atlanta": 6839, + "jaco": 6840, + "nasa": 6841, + "greek": 6842, + "senate": 6843, + "ish": 6844, + "eva": 6845, + "intelligence": 6846, + "efforts": 6847, + "alco": 6848, + "kun": 6849, + "hall": 6850, + "diag": 6851, + "claims": 6852, + "first": 6853, + "hb": 6854, + "bae": 6855, + "vul": 6856, + "pull": 6857, + "°": 6858, + "separ": 6859, + "speed": 6860, + "victi": 6861, + "onthisday": 6862, + "audience": 6863, + "rates": 6864, + "teach": 6865, + "filming": 6866, + "bush": 6867, + "song": 6868, + "yum": 6869, + "brun": 6870, + "raine": 6871, + "awa": 6872, + "parks": 6873, + "ðĿ": 6874, + "rabb": 6875, + "rach": 6876, + "raid": 6877, + "reached": 6878, + "rail": 6879, + "moves": 6880, + "selected": 6881, + "fri": 6882, + "raising": 6883, + "omy": 6884, + "stones": 6885, + "suk": 6886, + "francisco": 6887, + "cases": 6888, + "capit": 6889, + "confu": 6890, + "wtf": 6891, + "poke": 6892, + "equipment": 6893, + "greg": 6894, + "essential": 6895, + "offering": 6896, + "nex": 6897, + "pies": 6898, + "bec": 6899, + "creation": 6900, + "chairman": 6901, + "crown": 6902, + "wal": 6903, + "johnny": 6904, + "shift": 6905, + "neck": 6906, + "bang": 6907, + "bird": 6908, + "ðŁĺı": 6909, + "duck": 6910, + "reserve": 6911, + "depu": 6912, + "masters": 6913, + "overall": 6914, + "notic": 6915, + "juice": 6916, + "sneak": 6917, + "cheer": 6918, + "classes": 6919, + "eagles": 6920, + "nca": 6921, + "carpet": 6922, + "civil": 6923, + "coaches": 6924, + "harris": 6925, + "ups": 6926, + "balls": 6927, + "decor": 6928, + "martin": 6929, + "ros": 6930, + "vice": 6931, + "announcement": 6932, + "whose": 6933, + "tigers": 6934, + "stered": 6935, + "cts": 6936, + "dram": 6937, + "steel": 6938, + "young": 6939, + "install": 6940, + "suppo": 6941, + "recording": 6942, + "deck": 6943, + "seats": 6944, + "lder": 6945, + "angle": 6946, + "bot": 6947, + "styles": 6948, + "elections": 6949, + "fortun": 6950, + "nab": 6951, + "butter": 6952, + "arian": 6953, + "kash": 6954, + "inner": 6955, + "oured": 6956, + "beast": 6957, + "wei": 6958, + "iconic": 6959, + "experts": 6960, + "necess": 6961, + "beng": 6962, + "james": 6963, + "lia": 6964, + "greece": 6965, + "ðŁĵ·": 6966, + "ðŁĺģ": 6967, + "goodbye": 6968, + "mitch": 6969, + "twice": 6970, + "mumbai": 6971, + "steam": 6972, + "rush": 6973, + "medal": 6974, + "nett": 6975, + "fashion": 6976, + "tar": 6977, + "rs": 6978, + "saving": 6979, + "ricul": 6980, + "lm": 6981, + "sleeping": 6982, + "brooklyn": 6983, + "miss": 6984, + "sending": 6985, + "discovered": 6986, + "sphere": 6987, + "oftheday": 6988, + "kicks": 6989, + "missions": 6990, + "wright": 6991, + "ern": 6992, + "ghtly": 6993, + "ious": 6994, + "melbourne": 6995, + "startu": 6996, + "moved": 6997, + "carry": 6998, + "dak": 6999, + "agues": 7000, + "belgi": 7001, + "ema": 7002, + "wayne": 7003, + "dot": 7004, + "erie": 7005, + "pel": 7006, + "itunes": 7007, + "matthew": 7008, + "nobody": 7009, + "estab": 7010, + "calm": 7011, + "winds": 7012, + "luc": 7013, + "prepare": 7014, + "trends": 7015, + "exercise": 7016, + "advant": 7017, + "ðŁĴ¯": 7018, + "athletics": 7019, + "apps": 7020, + "ctions": 7021, + "advance": 7022, + "launches": 7023, + "little": 7024, + "realdonaldtrump": 7025, + "elizabeth": 7026, + "carolina": 7027, + "hub": 7028, + "hidden": 7029, + "nw": 7030, + "user": 7031, + "poll": 7032, + "greater": 7033, + "most": 7034, + "fed": 7035, + "pat": 7036, + "lifestyle": 7037, + "sati": 7038, + "scores": 7039, + "marriage": 7040, + "lr": 7041, + "avenue": 7042, + "deserve": 7043, + "rif": 7044, + "ðŁĹ": 7045, + "watch": 7046, + "championships": 7047, + "gray": 7048, + "enni": 7049, + "cotton": 7050, + "gom": 7051, + "where": 7052, + "package": 7053, + "sum": 7054, + "absolu": 7055, + "newly": 7056, + "foods": 7057, + "tyler": 7058, + "assembly": 7059, + "muslim": 7060, + "bank": 7061, + "rememb": 7062, + "options": 7063, + "producer": 7064, + "lando": 7065, + "funds": 7066, + "upper": 7067, + "shadow": 7068, + "progre": 7069, + "cop": 7070, + "inge": 7071, + "legs": 7072, + "detroit": 7073, + "hillary": 7074, + "jose": 7075, + "giants": 7076, + "soup": 7077, + "sustainable": 7078, + "tus": 7079, + "clothes": 7080, + "rocking": 7081, + "nz": 7082, + "minne": 7083, + "materi": 7084, + "bruce": 7085, + "eart": 7086, + "casting": 7087, + "independent": 7088, + "thousands": 7089, + "tah": 7090, + "decl": 7091, + "veterans": 7092, + "lions": 7093, + "wrap": 7094, + "âĢ¦": 7095, + "dess": 7096, + "bling": 7097, + "stine": 7098, + "eggs": 7099, + "oon": 7100, + "closing": 7101, + "zay": 7102, + "att": 7103, + "bacon": 7104, + "fail": 7105, + "arizona": 7106, + "depre": 7107, + "ghost": 7108, + "newsp": 7109, + "wers": 7110, + "vip": 7111, + "liked": 7112, + "ident": 7113, + "volunteer": 7114, + "adult": 7115, + "pupp": 7116, + "circle": 7117, + "material": 7118, + "degree": 7119, + "grown": 7120, + "boom": 7121, + "calendar": 7122, + "sur": 7123, + "viewing": 7124, + "athletes": 7125, + "chand": 7126, + "rell": 7127, + "asian": 7128, + "entr": 7129, + "volley": 7130, + "victims": 7131, + "body": 7132, + "mama": 7133, + "transfer": 7134, + "geek": 7135, + "indic": 7136, + "saved": 7137, + "mai": 7138, + "gent": 7139, + "its": 7140, + "lounge": 7141, + "kol": 7142, + "theory": 7143, + "situation": 7144, + "islands": 7145, + "arth": 7146, + "zoo": 7147, + "flood": 7148, + "viously": 7149, + "showed": 7150, + "parliament": 7151, + "chev": 7152, + "eline": 7153, + "attrac": 7154, + "abad": 7155, + "tail": 7156, + "hrs": 7157, + "lus": 7158, + "portu": 7159, + "gory": 7160, + "provides": 7161, + "toys": 7162, + "death": 7163, + "infe": 7164, + "ance": 7165, + "gle": 7166, + "liam": 7167, + "lover": 7168, + "hud": 7169, + "dvd": 7170, + "revealed": 7171, + "gw": 7172, + "rement": 7173, + "cathe": 7174, + "lying": 7175, + "radio": 7176, + "derby": 7177, + "stors": 7178, + "chemi": 7179, + "hospit": 7180, + "⾨": 7181, + "':": 7182, + "ilove": 7183, + "lemon": 7184, + "republic": 7185, + "sni": 7186, + "ness": 7187, + "door": 7188, + "reaction": 7189, + "pregn": 7190, + "flav": 7191, + "scholar": 7192, + "spotify": 7193, + "isation": 7194, + "visual": 7195, + "aware": 7196, + "sponsored": 7197, + "joke": 7198, + "lessons": 7199, + "legis": 7200, + "lock": 7201, + "simil": 7202, + "ðŁĺĭ": 7203, + "kind": 7204, + "lay": 7205, + "mah": 7206, + "hoping": 7207, + "vancouver": 7208, + "aser": 7209, + "cleaning": 7210, + "gala": 7211, + "threat": 7212, + "lap": 7213, + "ache": 7214, + "romance": 7215, + "expen": 7216, + "repost": 7217, + "zam": 7218, + "epi": 7219, + "mirror": 7220, + "oak": 7221, + "adul": 7222, + "batman": 7223, + "slu": 7224, + "lc": 7225, + "viewed": 7226, + "reviews": 7227, + "dates": 7228, + "indonesia": 7229, + "activi": 7230, + "offen": 7231, + "leaf": 7232, + "isi": 7233, + "agricul": 7234, + "costume": 7235, + "sites": 7236, + "spiritu": 7237, + "appearance": 7238, + "iry": 7239, + "stair": 7240, + "application": 7241, + "spectac": 7242, + "icity": 7243, + "skies": 7244, + "handle": 7245, + "punk": 7246, + "paradise": 7247, + "tn": 7248, + "deal": 7249, + "providing": 7250, + "doc": 7251, + "receiving": 7252, + "brew": 7253, + "microsoft": 7254, + "ö": 7255, + "ferr": 7256, + "metro": 7257, + "thail": 7258, + "yum": 7259, + "carter": 7260, + "á": 7261, + "gentle": 7262, + "breaks": 7263, + "cooper": 7264, + "showcase": 7265, + "cutting": 7266, + "egypt": 7267, + "baby": 7268, + "seminar": 7269, + "glori": 7270, + "sson": 7271, + "fave": 7272, + "rehear": 7273, + "lotte": 7274, + "lady": 7275, + "alas": 7276, + "prep": 7277, + "delivered": 7278, + "nuclear": 7279, + "iro": 7280, + "engagement": 7281, + "atta": 7282, + "conven": 7283, + "zan": 7284, + "glory": 7285, + "holds": 7286, + "businesses": 7287, + "strange": 7288, + "sche": 7289, + "itself": 7290, + "grad": 7291, + "markets": 7292, + "falling": 7293, + "stats": 7294, + "geon": 7295, + "budd": 7296, + "lis": 7297, + "sheet": 7298, + "thisi": 7299, + "colo": 7300, + "desert": 7301, + "registration": 7302, + "ign": 7303, + "explain": 7304, + "interior": 7305, + "laws": 7306, + "writers": 7307, + "springs": 7308, + "kr": 7309, + "fried": 7310, + "bloom": 7311, + "infra": 7312, + "ao": 7313, + "cred": 7314, + "past": 7315, + "lineup": 7316, + "boo": 7317, + "brea": 7318, + "boots": 7319, + "celebrity": 7320, + "attacks": 7321, + "brook": 7322, + "eves": 7323, + "excu": 7324, + "cherry": 7325, + "oop": 7326, + "fascin": 7327, + "boyfriend": 7328, + "seas": 7329, + "nine": 7330, + "effects": 7331, + "powered": 7332, + "kha": 7333, + "ðŁĺĢ": 7334, + "shout": 7335, + "condition": 7336, + "ij": 7337, + "hero": 7338, + "enterpri": 7339, + "winter": 7340, + "applications": 7341, + "shoe": 7342, + "gel": 7343, + "battle": 7344, + "programs": 7345, + "wart": 7346, + "ðŁĴ¥": 7347, + "rap": 7348, + "hol": 7349, + "dangerous": 7350, + "dia": 7351, + "counter": 7352, + "rics": 7353, + "ior": 7354, + "knight": 7355, + "coat": 7356, + "emotional": 7357, + "atures": 7358, + "das": 7359, + "wheel": 7360, + "forecast": 7361, + "transport": 7362, + "glasgow": 7363, + "kingdom": 7364, + "preparing": 7365, + "immedi": 7366, + "ffin": 7367, + "awarded": 7368, + "printing": 7369, + "roman": 7370, + "fighters": 7371, + "anymore": 7372, + "belt": 7373, + "pine": 7374, + "wine": 7375, + "xi": 7376, + "employees": 7377, + "logies": 7378, + "alled": 7379, + "demo": 7380, + "birthday": 7381, + "angeles": 7382, + "log": 7383, + "drivers": 7384, + "necklace": 7385, + "kath": 7386, + "sit": 7387, + "athlete": 7388, + "efs": 7389, + "sburg": 7390, + "purpose": 7391, + "resistance": 7392, + "releases": 7393, + "tis": 7394, + "various": 7395, + "deliver": 7396, + "chal": 7397, + "sanc": 7398, + "oppo": 7399, + "craw": 7400, + "neuro": 7401, + "dra": 7402, + "supporters": 7403, + "snap": 7404, + "difficult": 7405, + "swear": 7406, + "logist": 7407, + "path": 7408, + "attempt": 7409, + "à¥": 7410, + "swimming": 7411, + "steve": 7412, + "hurt": 7413, + "included": 7414, + "bap": 7415, + "ware": 7416, + "ðŁĴĭ": 7417, + "enders": 7418, + "jake": 7419, + "leeds": 7420, + "climb": 7421, + "lb": 7422, + "imple": 7423, + "lisa": 7424, + "clothing": 7425, + "ðŁĺİ": 7426, + "dt": 7427, + "compla": 7428, + "swing": 7429, + "straw": 7430, + "vals": 7431, + "kle": 7432, + "users": 7433, + "storm": 7434, + "cuts": 7435, + "ontario": 7436, + "pan": 7437, + "handsome": 7438, + "iow": 7439, + "argu": 7440, + "checking": 7441, + "scottish": 7442, + "Ķï¸ı": 7443, + "sier": 7444, + "emma": 7445, + "pod": 7446, + "pattern": 7447, + "desh": 7448, + "enh": 7449, + "edward": 7450, + "ting": 7451, + "kh": 7452, + "half": 7453, + "lincoln": 7454, + "mother": 7455, + "alleg": 7456, + "rc": 7457, + "volleyball": 7458, + "dn": 7459, + "gay": 7460, + "ally": 7461, + "leton": 7462, + "grove": 7463, + "loud": 7464, + "advanced": 7465, + "respec": 7466, + "client": 7467, + "supreme": 7468, + "thailand": 7469, + "how": 7470, + "gig": 7471, + "toi": 7472, + "dot": 7473, + "dollar": 7474, + "ðŁijĩ": 7475, + "pit": 7476, + "rb": 7477, + "hn": 7478, + "produced": 7479, + "ggers": 7480, + "âĨĴ": 7481, + "mlb": 7482, + "canvas": 7483, + "fineart": 7484, + "usd": 7485, + "inthe": 7486, + "pson": 7487, + "actual": 7488, + "sl": 7489, + "tb": 7490, + "ipad": 7491, + "ensure": 7492, + "umb": 7493, + "wd": 7494, + "ska": 7495, + "mars": 7496, + "kend": 7497, + "feli": 7498, + "thing": 7499, + "countdown": 7500, + "absolute": 7501, + "rout": 7502, + "dral": 7503, + "py": 7504, + "injured": 7505, + "mint": 7506, + "hunting": 7507, + "mmer": 7508, + "sage": 7509, + "ligh": 7510, + "acity": 7511, + "expan": 7512, + "murray": 7513, + "aro": 7514, + "secure": 7515, + "fourth": 7516, + "eagle": 7517, + "relief": 7518, + "stakes": 7519, + "industrial": 7520, + "clark": 7521, + "understanding": 7522, + "seem": 7523, + "plenty": 7524, + "silver": 7525, + "clau": 7526, + "threat": 7527, + "sail": 7528, + "produce": 7529, + "abstr": 7530, + "isis": 7531, + "br": 7532, + "engers": 7533, + "worry": 7534, + "bieber": 7535, + "sj": 7536, + "justin": 7537, + "realize": 7538, + "kyle": 7539, + "espn": 7540, + "filter": 7541, + "sch": 7542, + "types": 7543, + "gamedev": 7544, + "ding": 7545, + "twitter": 7546, + "soldiers": 7547, + "pom": 7548, + "carbon": 7549, + "yards": 7550, + "childhood": 7551, + "ried": 7552, + "kel": 7553, + "eleph": 7554, + "tons": 7555, + "keynote": 7556, + "quiet": 7557, + "wire": 7558, + "posting": 7559, + "issa": 7560, + "representing": 7561, + "backs": 7562, + "alexander": 7563, + "celebrates": 7564, + "taining": 7565, + "||": 7566, + "chor": 7567, + "escape": 7568, + "peek": 7569, + "tives": 7570, + "field": 7571, + "ssie": 7572, + "impac": 7573, + "sponsor": 7574, + "rc": 7575, + "wedd": 7576, + "cannab": 7577, + "sides": 7578, + "tracks": 7579, + "compar": 7580, + "contrac": 7581, + "technical": 7582, + "bible": 7583, + "exploring": 7584, + "share": 7585, + "trav": 7586, + "nate": 7587, + "illo": 7588, + "scru": 7589, + "mingham": 7590, + "guns": 7591, + "ofthe": 7592, + "shame": 7593, + "sees": 7594, + "catho": 7595, + "access": 7596, + "cel": 7597, + "reported": 7598, + "»": 7599, + "mario": 7600, + "pad": 7601, + "hopefully": 7602, + "ouse": 7603, + "yon": 7604, + "disappo": 7605, + "olo": 7606, + "pitt": 7607, + "pac": 7608, + "gap": 7609, + "crush": 7610, + "sg": 7611, + "kle": 7612, + "gem": 7613, + "empire": 7614, + "dirty": 7615, + "ais": 7616, + "aviation": 7617, + "zealand": 7618, + "facing": 7619, + "highway": 7620, + "danny": 7621, + "spider": 7622, + "otta": 7623, + "ðŁĺĦ": 7624, + "wy": 7625, + "colours": 7626, + "infl": 7627, + "costs": 7628, + "olympics": 7629, + "aus": 7630, + "hm": 7631, + "howard": 7632, + "passes": 7633, + "lauren": 7634, + "mush": 7635, + "opin": 7636, + "rho": 7637, + "discount": 7638, + "operation": 7639, + "emily": 7640, + "mmm": 7641, + "chamber": 7642, + "dil": 7643, + "toyo": 7644, + "ship": 7645, + "samu": 7646, + "pictured": 7647, + "unic": 7648, + "pol": 7649, + "keeper": 7650, + "cartoon": 7651, + "sten": 7652, + "ignor": 7653, + "nations": 7654, + "nl": 7655, + "tasting": 7656, + "detail": 7657, + "officials": 7658, + "motor": 7659, + "francis": 7660, + "editor": 7661, + "ðŁijĩ": 7662, + "pets": 7663, + "rangers": 7664, + "tg": 7665, + "rn": 7666, + "wri": 7667, + "nichol": 7668, + "ise": 7669, + "spots": 7670, + "anie": 7671, + "check": 7672, + "triple": 7673, + "kumar": 7674, + "speakers": 7675, + "icing": 7676, + "prepared": 7677, + "abuse": 7678, + "friendship": 7679, + "month": 7680, + "swim": 7681, + "aire": 7682, + "scent": 7683, + "hamilton": 7684, + "indian": 7685, + "jes": 7686, + "yummy": 7687, + "tears": 7688, + "dawn": 7689, + "ized": 7690, + "worlds": 7691, + "ðŁķ": 7692, + "billi": 7693, + "stone": 7694, + "nhs": 7695, + "basic": 7696, + "por": 7697, + "stle": 7698, + "iron": 7699, + "older": 7700, + "clevel": 7701, + "eing": 7702, + "ðŁĺįðŁĺįðŁĺį": 7703, + "prints": 7704, + "firm": 7705, + "aircraft": 7706, + "finest": 7707, + "develop": 7708, + "aaron": 7709, + "tz": 7710, + "graham": 7711, + "owners": 7712, + "foli": 7713, + "lesson": 7714, + "ques": 7715, + "babe": 7716, + "craft": 7717, + "phen": 7718, + "jun": 7719, + "birmingham": 7720, + "vine": 7721, + "ller": 7722, + "ian": 7723, + "fineartamerica": 7724, + "evolu": 7725, + "stab": 7726, + "imper": 7727, + "ward": 7728, + "comic": 7729, + "wiz": 7730, + "invited": 7731, + "duke": 7732, + "match": 7733, + "ports": 7734, + "roger": 7735, + "diagno": 7736, + "kept": 7737, + "test": 7738, + "visu": 7739, + "rhy": 7740, + "soc": 7741, + "tox": 7742, + "baker": 7743, + "surface": 7744, + "covers": 7745, + "mans": 7746, + "bits": 7747, + "xbox": 7748, + "ffle": 7749, + "nan": 7750, + "gard": 7751, + "hart": 7752, + "waters": 7753, + "villa": 7754, + "retro": 7755, + "lightning": 7756, + "catholic": 7757, + "democracy": 7758, + "neighbor": 7759, + "penn": 7760, + "cran": 7761, + "jonathan": 7762, + "laura": 7763, + "vibes": 7764, + "sub": 7765, + "coaching": 7766, + "clearly": 7767, + "ukraine": 7768, + "brave": 7769, + "commitment": 7770, + "tall": 7771, + "mart": 7772, + "rap": 7773, + "modi": 7774, + "scott": 7775, + "bros": 7776, + "shower": 7777, + "ðŁı¾": 7778, + "âĺºï¸ı": 7779, + "cousin": 7780, + "approach": 7781, + "bre": 7782, + "compos": 7783, + "hilari": 7784, + "philly": 7785, + "gad": 7786, + "quickly": 7787, + "rian": 7788, + "tm": 7789, + "virtual": 7790, + "houses": 7791, + "kt": 7792, + "phoenix": 7793, + "wire": 7794, + "ffy": 7795, + "bunch": 7796, + "ancing": 7797, + "tale": 7798, + "snapchat": 7799, + "starter": 7800, + "ht": 7801, + "kicking": 7802, + "apart": 7803, + "thy": 7804, + ")!": 7805, + "blogger": 7806, + "itz": 7807, + "comfort": 7808, + "angels": 7809, + "wash": 7810, + "\":": 7811, + "argent": 7812, + "request": 7813, + "honest": 7814, + "mighty": 7815, + "bobby": 7816, + "kg": 7817, + "rol": 7818, + "thouse": 7819, + "expo": 7820, + "hc": 7821, + "tables": 7822, + "magical": 7823, + "posts": 7824, + "dem": 7825, + "nw": 7826, + "orlando": 7827, + "aber": 7828, + "***": 7829, + "ðŁĺľ": 7830, + "environmental": 7831, + "transformation": 7832, + "mile": 7833, + "wic": 7834, + "hiring": 7835, + "maine": 7836, + "boar": 7837, + "rying": 7838, + "tis": 7839, + "niture": 7840, + "tweeted": 7841, + "antonio": 7842, + "opinion": 7843, + "finale": 7844, + "diy": 7845, + "fis": 7846, + "thin": 7847, + "trouble": 7848, + "lego": 7849, + "files": 7850, + "quart": 7851, + "spa": 7852, + "currency": 7853, + "climate": 7854, + "fanart": 7855, + "railway": 7856, + "space": 7857, + "bands": 7858, + "daniel": 7859, + "motion": 7860, + "leng": 7861, + "holder": 7862, + "occu": 7863, + "marie": 7864, + "cathedral": 7865, + "buzz": 7866, + "bies": 7867, + "nascar": 7868, + "bmw": 7869, + "battery": 7870, + "charlotte": 7871, + "doctor": 7872, + "zzle": 7873, + "seven": 7874, + "insan": 7875, + "ddy": 7876, + "sten": 7877, + "labor": 7878, + "thrilled": 7879, + "seren": 7880, + "documentary": 7881, + "waves": 7882, + "certain": 7883, + "candid": 7884, + "allowed": 7885, + "nintendo": 7886, + "starwars": 7887, + "tap": 7888, + "homemade": 7889, + "dles": 7890, + "thering": 7891, + "bree": 7892, + "empty": 7893, + "piano": 7894, + "positi": 7895, + "country": 7896, + "pork": 7897, + "puts": 7898, + "perry": 7899, + "matic": 7900, + "spotlight": 7901, + "tist": 7902, + "orities": 7903, + "wealth": 7904, + "cp": 7905, + "barbar": 7906, + "committed": 7907, + "assau": 7908, + "profit": 7909, + "eight": 7910, + "hul": 7911, + "finishing": 7912, + "runner": 7913, + "sso": 7914, + "inspec": 7915, + "charged": 7916, + "christop": 7917, + "losing": 7918, + "coal": 7919, + "hoo": 7920, + "elev": 7921, + "dele": 7922, + "moham": 7923, + "donation": 7924, + "cable": 7925, + "clinic": 7926, + "jin": 7927, + "managed": 7928, + "tering": 7929, + "â¬": 7930, + "urban": 7931, + "deputy": 7932, + "bber": 7933, + "burn": 7934, + "academic": 7935, + "ott": 7936, + "stake": 7937, + "iter": 7938, + "stown": 7939, + "acker": 7940, + "adventures": 7941, + "adams": 7942, + "greg": 7943, + "prom": 7944, + "vol": 7945, + "acqu": 7946, + "congre": 7947, + "paint": 7948, + "citizens": 7949, + "call": 7950, + "afford": 7951, + "vc": 7952, + "asks": 7953, + "thetic": 7954, + "independence": 7955, + "âĽ": 7956, + "hitting": 7957, + "blon": 7958, + "future": 7959, + "âı": 7960, + "inno": 7961, + "gene": 7962, + "boards": 7963, + "distance": 7964, + "set": 7965, + "remem": 7966, + "thal": 7967, + "prevent": 7968, + "lang": 7969, + "objec": 7970, + "susp": 7971, + "matt": 7972, + "induc": 7973, + "boro": 7974, + "pione": 7975, + "redi": 7976, + "virtu": 7977, + "printed": 7978, + "scope": 7979, + "shark": 7980, + "succe": 7981, + "astron": 7982, + "illegal": 7983, + "jag": 7984, + "cting": 7985, + "inee": 7986, + "ato": 7987, + "robin": 7988, + "nutrition": 7989, + "bf": 7990, + "dutch": 7991, + "bn": 7992, + "furniture": 7993, + "forgotten": 7994, + "atar": 7995, + "rup": 7996, + "hyper": 7997, + "branch": 7998, + "communication": 7999, + "degrees": 8000, + "onia": 8001, + "uncle": 8002, + "promote": 8003, + "orche": 8004, + "wii": 8005, + "js": 8006, + "button": 8007, + "major": 8008, + "cbs": 8009, + "bristol": 8010, + "premium": 8011, + "ordinary": 8012, + "edit": 8013, + "mg": 8014, + "weed": 8015, + "steven": 8016, + ":'": 8017, + "gus": 8018, + "tes": 8019, + "captured": 8020, + "drugs": 8021, + "dow": 8022, + "writes": 8023, + "bishop": 8024, + "wheels": 8025, + "alization": 8026, + "discovery": 8027, + "wr": 8028, + "rachel": 8029, + "neil": 8030, + "hydr": 8031, + "cutest": 8032, + "entrepreneur": 8033, + "korean": 8034, + "oregon": 8035, + "ulty": 8036, + "perfectly": 8037, + "supported": 8038, + "historical": 8039, + "twins": 8040, + "elly": 8041, + "wel": 8042, + "devil": 8043, + "income": 8044, + "scientists": 8045, + "deleg": 8046, + "hen": 8047, + "oni": 8048, + "iced": 8049, + "gio": 8050, + "curry": 8051, + "reveal": 8052, + "eg": 8053, + "buffalo": 8054, + "nol": 8055, + "opera": 8056, + "cameron": 8057, + "hahahaha": 8058, + "jab": 8059, + "graduation": 8060, + "craig": 8061, + "ral": 8062, + "if": 8063, + "organization": 8064, + "lege": 8065, + "gang": 8066, + "sud": 8067, + "edinburgh": 8068, + "lack": 8069, + "flies": 8070, + "gate": 8071, + "thrones": 8072, + "qb": 8073, + "thereal": 8074, + "eleg": 8075, + "ppin": 8076, + "cles": 8077, + "jamie": 8078, + "tnam": 8079, + "crypto": 8080, + "oul": 8081, + "pages": 8082, + "ase": 8083, + "roots": 8084, + "stupid": 8085, + "adid": 8086, + "boot": 8087, + "protein": 8088, + "sap": 8089, + "sium": 8090, + "sus": 8091, + "endor": 8092, + "function": 8093, + "dont": 8094, + "enna": 8095, + "chy": 8096, + "sque": 8097, + "worker": 8098, + "mtv": 8099, + "ea": 8100, + "kan": 8101, + "ðŁĴļ": 8102, + "mus": 8103, + "profession": 8104, + "tto": 8105, + "operations": 8106, + "allo": 8107, + "ctor": 8108, + "invite": 8109, + "scand": 8110, + "outh": 8111, + "zim": 8112, + "links": 8113, + "clients": 8114, + "samsung": 8115, + "discusses": 8116, + "nell": 8117, + "ultra": 8118, + "somewhere": 8119, + "stewart": 8120, + "inet": 8121, + "dez": 8122, + "bout": 8123, + "factor": 8124, + "tian": 8125, + "trans": 8126, + "jeremy": 8127, + "db": 8128, + "ðŁĩ¬": 8129, + "orn": 8130, + "developing": 8131, + "spol": 8132, + "cooper": 8133, + "mau": 8134, + "remembering": 8135, + "trek": 8136, + "family": 8137, + "seniors": 8138, + "foster": 8139, + "attended": 8140, + "wing": 8141, + "transform": 8142, + "elementary": 8143, + "horiz": 8144, + "listing": 8145, + "malaysia": 8146, + "itch": 8147, + "warrior": 8148, + "philippines": 8149, + "russell": 8150, + "mend": 8151, + "initiative": 8152, + "creep": 8153, + "tops": 8154, + "briti": 8155, + "aur": 8156, + "sharp": 8157, + "advertising": 8158, + "ugly": 8159, + "achiev": 8160, + "materials": 8161, + "bug": 8162, + "device": 8163, + "bonus": 8164, + "facility": 8165, + "cole": 8166, + "nhl": 8167, + "yas": 8168, + "planned": 8169, + "pole": 8170, + "excellence": 8171, + "trick": 8172, + "confl": 8173, + "rp": 8174, + "achieve": 8175, + "loan": 8176, + "swag": 8177, + "jessica": 8178, + "howe": 8179, + "pour": 8180, + "scu": 8181, + "zoo": 8182, + "rated": 8183, + "dresses": 8184, + "rebel": 8185, + "mexican": 8186, + "coordin": 8187, + "mess": 8188, + "atlantic": 8189, + "tl": 8190, + "oscar": 8191, + "walks": 8192, + "pharmac": 8193, + "investigation": 8194, + "...#": 8195, + "cci": 8196, + "easily": 8197, + "mondaymotivation": 8198, + "yment": 8199, + "auti": 8200, + "forced": 8201, + "armed": 8202, + "colleagues": 8203, + "papers": 8204, + "proper": 8205, + "shake": 8206, + "buc": 8207, + "lean": 8208, + "exhibit": 8209, + "evement": 8210, + "cott": 8211, + "biz": 8212, + "sper": 8213, + "kent": 8214, + "swan": 8215, + "/@": 8216, + "girlfriend": 8217, + "hawk": 8218, + "âĺĢï¸ı": 8219, + "mono": 8220, + "ðŁĴĽ": 8221, + "statue": 8222, + "ðŁĺ³": 8223, + "ras": 8224, + "teeth": 8225, + "precious": 8226, + "tile": 8227, + "pam": 8228, + "swift": 8229, + "vali": 8230, + "nose": 8231, + "drunk": 8232, + "experiences": 8233, + "comeback": 8234, + "genius": 8235, + "worse": 8236, + "shef": 8237, + "rad": 8238, + "edit": 8239, + "honour": 8240, + "auspol": 8241, + "larry": 8242, + "hire": 8243, + "gordon": 8244, + "achievement": 8245, + "........": 8246, + "suicide": 8247, + "alternative": 8248, + "sup": 8249, + "surroun": 8250, + "shake": 8251, + "keith": 8252, + "pepper": 8253, + "turk": 8254, + "criminal": 8255, + "beck": 8256, + "sum": 8257, + "walls": 8258, + "cnn": 8259, + "antic": 8260, + "offe": 8261, + "colli": 8262, + "wines": 8263, + "highlight": 8264, + "hawaii": 8265, + "embar": 8266, + "lfc": 8267, + "ðŁĩ®": 8268, + "mv": 8269, + ">>": 8270, + "atmo": 8271, + "word": 8272, + "carl": 8273, + "shoutout": 8274, + "brewing": 8275, + "ìĿ": 8276, + "dof": 8277, + "sic": 8278, + "hottest": 8279, + "colon": 8280, + "hhh": 8281, + "shut": 8282, + "lowing": 8283, + "volume": 8284, + "apartment": 8285, + "agreement": 8286, + "destro": 8287, + "wee": 8288, + "religious": 8289, + "iowa": 8290, + "rod": 8291, + "landing": 8292, + "represent": 8293, + "ðŁĵ·:": 8294, + "las": 8295, + "usually": 8296, + "hl": 8297, + "cac": 8298, + "salv": 8299, + "along": 8300, + "laughing": 8301, + "beans": 8302, + "reminds": 8303, + "phase": 8304, + "somebody": 8305, + "mask": 8306, + "ranked": 8307, + "destroy": 8308, + "sci": 8309, + "âĢ¼ï¸ı": 8310, + "gabri": 8311, + "leo": 8312, + "roa": 8313, + "failed": 8314, + "sil": 8315, + "refugees": 8316, + "revi": 8317, + "ring": 8318, + "berries": 8319, + "cookies": 8320, + "yy": 8321, + "conservation": 8322, + "shab": 8323, + "humans": 8324, + "determin": 8325, + "ain": 8326, + "niall": 8327, + "assu": 8328, + "mba": 8329, + "from": 8330, + "extreme": 8331, + "vices": 8332, + "commerce": 8333, + "ghtful": 8334, + "ordered": 8335, + "supports": 8336, + "recap": 8337, + "vor": 8338, + "dropping": 8339, + "correct": 8340, + "paying": 8341, + "meaning": 8342, + "nj": 8343, + "quiz": 8344, + "\"#": 8345, + "business": 8346, + "ðŁĩ®ðŁĩ": 8347, + "indigen": 8348, + "dust": 8349, + "boxes": 8350, + "blind": 8351, + "xxx": 8352, + "zzy": 8353, + "ðŁĩ¬ðŁĩ": 8354, + "ssels": 8355, + "sant": 8356, + "ddle": 8357, + "hilarious": 8358, + "design": 8359, + "wondering": 8360, + "vehicles": 8361, + "kre": 8362, + "jud": 8363, + "reception": 8364, + "parker": 8365, + "ÃŃ": 8366, + "privi": 8367, + "hydro": 8368, + "softball": 8369, + "pollu": 8370, + "locked": 8371, + "bah": 8372, + "ear": 8373, + "script": 8374, + "divi": 8375, + "brace": 8376, + "george": 8377, + "theast": 8378, + "belo": 8379, + "jal": 8380, + "tionary": 8381, + "dental": 8382, + "rocket": 8383, + "purch": 8384, + "shak": 8385, + "manufacturing": 8386, + "ez": 8387, + "itis": 8388, + "concep": 8389, + "tball": 8390, + "chs": 8391, + "directed": 8392, + "prayers": 8393, + "ook": 8394, + "philos": 8395, + "variety": 8396, + "chess": 8397, + "server": 8398, + "gand": 8399, + "balti": 8400, + "ðŁĵ¸": 8401, + "sely": 8402, + "cruz": 8403, + "spectacular": 8404, + "burning": 8405, + "represent": 8406, + "iz": 8407, + "tone": 8408, + "merce": 8409, + "hell": 8410, + "bedroom": 8411, + "establi": 8412, + "bol": 8413, + "common": 8414, + "ãĥ»": 8415, + "abor": 8416, + "kitty": 8417, + "heights": 8418, + "repair": 8419, + "william": 8420, + "quake": 8421, + "alabama": 8422, + "population": 8423, + "rev": 8424, + "rett": 8425, + "ists": 8426, + "nite": 8427, + "lem": 8428, + "aha": 8429, + "cleveland": 8430, + "rm": 8431, + "pover": 8432, + "obse": 8433, + "montre": 8434, + "mania": 8435, + "®": 8436, + "conne": 8437, + "carni": 8438, + "shah": 8439, + "fy": 8440, + "ua": 8441, + "scor": 8442, + "struggle": 8443, + "bob": 8444, + "''": 8445, + "appropri": 8446, + "decide": 8447, + "ffed": 8448, + "caster": 8449, + "sort": 8450, + "hungry": 8451, + "drag": 8452, + "اÙ": 8453, + "grounds": 8454, + "dw": 8455, + "slightly": 8456, + "cardin": 8457, + "deadline": 8458, + "bronze": 8459, + "webin": 8460, + "barry": 8461, + "silence": 8462, + "euro": 8463, + "option": 8464, + "earn": 8465, + "ðŁĴĸ": 8466, + "however": 8467, + "naren": 8468, + "nails": 8469, + "bathroom": 8470, + "vine": 8471, + "phd": 8472, + "mining": 8473, + "garage": 8474, + "()": 8475, + "shoulder": 8476, + "defeat": 8477, + "dir": 8478, + "ov": 8479, + "liberty": 8480, + "pleas": 8481, + "xon": 8482, + "compre": 8483, + "av": 8484, + "jin": 8485, + "ables": 8486, + "silent": 8487, + "famili": 8488, + "visits": 8489, + "dipl": 8490, + "habit": 8491, + "millions": 8492, + "regarding": 8493, + "innovative": 8494, + "senator": 8495, + "rts": 8496, + "von": 8497, + "kl": 8498, + "whil": 8499, + "required": 8500, + "âĿĦ": 8501, + "luv": 8502, + "presidential": 8503, + "pocket": 8504, + "hundre": 8505, + "shown": 8506, + "frozen": 8507, + "toward": 8508, + "fast": 8509, + "confidence": 8510, + "rough": 8511, + "individual": 8512, + "quet": 8513, + "ðŁı½": 8514, + "dome": 8515, + "fifa": 8516, + "engineer": 8517, + "zen": 8518, + "remix": 8519, + "ðŁĺĥ": 8520, + "plant": 8521, + "minor": 8522, + "robinson": 8523, + "asy": 8524, + "pulled": 8525, + "certain": 8526, + "potato": 8527, + "(:": 8528, + "pres": 8529, + "occa": 8530, + "wit": 8531, + "item": 8532, + "sie": 8533, + "dating": 8534, + "thompson": 8535, + "owned": 8536, + "anu": 8537, + "vie": 8538, + "tedly": 8539, + "goodnight": 8540, + "except": 8541, + "ðŁĮŁ": 8542, + "iraq": 8543, + "kie": 8544, + "rences": 8545, + "lip": 8546, + "similar": 8547, + "saudi": 8548, + "vig": 8549, + "arthur": 8550, + "picks": 8551, + "milan": 8552, + "honda": 8553, + "maxi": 8554, + "og": 8555, + "stest": 8556, + "arch": 8557, + "analytics": 8558, + "basti": 8559, + "pearl": 8560, + "terry": 8561, + "horse": 8562, + "astro": 8563, + "acce": 8564, + "launching": 8565, + "international": 8566, + "sno": 8567, + "tasty": 8568, + "denver": 8569, + "irl": 8570, + "pete": 8571, + "torn": 8572, + "advantage": 8573, + "varsity": 8574, + "\"\"": 8575, + "sole": 8576, + "gc": 8577, + "lang": 8578, + "demonstr": 8579, + "olds": 8580, + "unity": 8581, + "nets": 8582, + "inspire": 8583, + "crete": 8584, + "nashville": 8585, + "nelson": 8586, + "eter": 8587, + "walk": 8588, + "hyun": 8589, + "mack": 8590, + "treas": 8591, + "seeking": 8592, + "rage": 8593, + "brush": 8594, + "aband": 8595, + "whilst": 8596, + "cocon": 8597, + "hong": 8598, + "shelter": 8599, + "ip": 8600, + "possibly": 8601, + "soo": 8602, + "ited": 8603, + "âĦ": 8604, + "races": 8605, + "warming": 8606, + "quin": 8607, + "television": 8608, + "matches": 8609, + "rapi": 8610, + "mental": 8611, + "palm": 8612, + "jennifer": 8613, + "rolls": 8614, + "indiana": 8615, + "bars": 8616, + "catching": 8617, + "rescu": 8618, + "candidates": 8619, + "fare": 8620, + "âłĢ": 8621, + "seo": 8622, + "vietnam": 8623, + "alpha": 8624, + "michelle": 8625, + "visible": 8626, + "regre": 8627, + "wned": 8628, + "apple": 8629, + "lip": 8630, + "ffe": 8631, + "liz": 8632, + "yorkshire": 8633, + "hail": 8634, + "seasons": 8635, + "began": 8636, + "md": 8637, + "kc": 8638, + "lap": 8639, + "fascinating": 8640, + "help": 8641, + "ury": 8642, + "ums": 8643, + "nuts": 8644, + "sem": 8645, + "alongside": 8646, + "bridge": 8647, + "orial": 8648, + "ove": 8649, + "worldcup": 8650, + "british": 8651, + "comfortable": 8652, + "ive": 8653, + "hotels": 8654, + "fairs": 8655, + "horri": 8656, + "sox": 8657, + "dining": 8658, + "stream": 8659, + "barri": 8660, + "ssy": 8661, + "wim": 8662, + "terms": 8663, + "vu": 8664, + "pere": 8665, + "lens": 8666, + "walked": 8667, + "ror": 8668, + "lars": 8669, + "shield": 8670, + "doubt": 8671, + "proto": 8672, + "crossing": 8673, + "meant": 8674, + "medium": 8675, + "adding": 8676, + "eb": 8677, + "cheap": 8678, + "func": 8679, + "paper": 8680, + "brands": 8681, + "ryan": 8682, + "feedback": 8683, + "collins": 8684, + "unknown": 8685, + "tropical": 8686, + "sandwich": 8687, + "fallen": 8688, + "formu": 8689, + "select": 8690, + "loads": 8691, + "answers": 8692, + "ori": 8693, + "maga": 8694, + "dor": 8695, + "duo": 8696, + "alie": 8697, + "drum": 8698, + "uri": 8699, + "deer": 8700, + "soul": 8701, + "shut": 8702, + "âĺº": 8703, + "stolen": 8704, + "donated": 8705, + "buzz": 8706, + "patriots": 8707, + "hal": 8708, + "nasty": 8709, + "nominated": 8710, + "monte": 8711, + "kia": 8712, + "thri": 8713, + "ingu": 8714, + "tests": 8715, + "petro": 8716, + "ðŁijij": 8717, + "hosts": 8718, + "nest": 8719, + "topic": 8720, + "patch": 8721, + "mmy": 8722, + "hugh": 8723, + "abilities": 8724, + "mathe": 8725, + "smiles": 8726, + "gb": 8727, + "agenda": 8728, + "insights": 8729, + "chip": 8730, + "phan": 8731, + "failure": 8732, + "dgers": 8733, + "hai": 8734, + "significant": 8735, + "shock": 8736, + "rural": 8737, + "glam": 8738, + "figures": 8739, + "potus": 8740, + "ota": 8741, + "ministry": 8742, + "appears": 8743, + "fear": 8744, + "rh": 8745, + "american": 8746, + "hatt": 8747, + "sony": 8748, + "fires": 8749, + "edi": 8750, + "nou": 8751, + "equi": 8752, + "when": 8753, + "universal": 8754, + "madness": 8755, + "ix": 8756, + "sculpture": 8757, + "bach": 8758, + "tto": 8759, + "sweden": 8760, + "eta": 8761, + "ento": 8762, + "developed": 8763, + "monthly": 8764, + "maps": 8765, + "rah": 8766, + "led": 8767, + "delta": 8768, + "saints": 8769, + "islam": 8770, + "bench": 8771, + "fifth": 8772, + "vard": 8773, + "socks": 8774, + "welcoming": 8775, + "je": 8776, + "turner": 8777, + "vb": 8778, + "adi": 8779, + "norway": 8780, + "ady": 8781, + "hurricane": 8782, + "porsche": 8783, + "tradition": 8784, + "exam": 8785, + "newspaper": 8786, + "luci": 8787, + "aver": 8788, + "ideal": 8789, + "dna": 8790, + "madison": 8791, + "ðŁ§": 8792, + "witness": 8793, + "acou": 8794, + "insight": 8795, + "simon": 8796, + "robot": 8797, + "snake": 8798, + "nbc": 8799, + "aco": 8800, + "ross": 8801, + "shment": 8802, + "religion": 8803, + "chann": 8804, + "insu": 8805, + "campbell": 8806, + "installed": 8807, + "weather": 8808, + "horses": 8809, + "oli": 8810, + "robert": 8811, + "kaz": 8812, + "ðŁıĢ": 8813, + "veteran": 8814, + "thread": 8815, + "quarter": 8816, + "easier": 8817, + "capture": 8818, + "hipho": 8819, + "lawrence": 8820, + "romantic": 8821, + "passion": 8822, + "clay": 8823, + "oxford": 8824, + "thai": 8825, + "studying": 8826, + "fia": 8827, + "elected": 8828, + "mostly": 8829, + "cb": 8830, + "tumb": 8831, + "âĢįâĻĤ": 8832, + "xl": 8833, + "shan": 8834, + "faster": 8835, + "evans": 8836, + "slide": 8837, + "shri": 8838, + "seek": 8839, + "mies": 8840, + "chemistry": 8841, + "pumpkin": 8842, + "tum": 8843, + ",,": 8844, + "room": 8845, + "fired": 8846, + "lips": 8847, + "presence": 8848, + "aff": 8849, + "brewery": 8850, + "arrive": 8851, + "swag": 8852, + "photograph": 8853, + "pengu": 8854, + "chips": 8855, + "attor": 8856, + "values": 8857, + "accurate": 8858, + "contemporary": 8859, + "principal": 8860, + "cannabis": 8861, + "ario": 8862, + "anywhere": 8863, + "gia": 8864, + "democrats": 8865, + "buildings": 8866, + "lived": 8867, + "aps": 8868, + "negative": 8869, + "mare": 8870, + "ballo": 8871, + "lion": 8872, + "diamon": 8873, + "look": 8874, + "reform": 8875, + "tommy": 8876, + "illa": 8877, + "treats": 8878, + "hundreds": 8879, + "portland": 8880, + "worthy": 8881, + "excep": 8882, + "aria": 8883, + "idol": 8884, + "beer": 8885, + "cdn": 8886, + "yu": 8887, + "awk": 8888, + "ðŁĩ¨": 8889, + "cells": 8890, + "ó": 8891, + "identity": 8892, + "drawn": 8893, + "devil": 8894, + "finger": 8895, + "tham": 8896, + "ðŁijĬ": 8897, + "earned": 8898, + "fintech": 8899, + "dolph": 8900, + "tweeting": 8901, + "evolution": 8902, + "ðŁĵį": 8903, + "estim": 8904, + "mvp": 8905, + "none": 8906, + "ðŁĩºðŁĩ¸": 8907, + "toyota": 8908, + "aux": 8909, + "marin": 8910, + "bold": 8911, + "lbs": 8912, + "steak": 8913, + "murphy": 8914, + "itable": 8915, + "louis": 8916, + "solve": 8917, + "pia": 8918, + "skir": 8919, + "illino": 8920, + "webinar": 8921, + "banana": 8922, + "lov": 8923, + "thon": 8924, + "voters": 8925, + "affordable": 8926, + "defeated": 8927, + "lmfa": 8928, + "airlines": 8929, + "superb": 8930, + "anyway": 8931, + "debt": 8932, + "bored": 8933, + "versi": 8934, + "metal": 8935, + "responsible": 8936, + "mk": 8937, + "sse": 8938, + "fay": 8939, + "caused": 8940, + "fp": 8941, + "recommend": 8942, + "plaza": 8943, + "sporting": 8944, + "alliance": 8945, + "austri": 8946, + "nn": 8947, + "tours": 8948, + "surprised": 8949, + "artif": 8950, + "thunder": 8951, + "surve": 8952, + "wore": 8953, + "brief": 8954, + "necessary": 8955, + "zie": 8956, + "ashley": 8957, + "drake": 8958, + "rt": 8959, + "knife": 8960, + "immun": 8961, + "charges": 8962, + "athe": 8963, + "bride": 8964, + "reply": 8965, + "gav": 8966, + "broadcast": 8967, + "puer": 8968, + "bracelet": 8969, + "capacity": 8970, + "harvest": 8971, + "idk": 8972, + "performan": 8973, + "dding": 8974, + "ilers": 8975, + "para": 8976, + "jama": 8977, + "province": 8978, + "chin": 8979, + "iders": 8980, + "hari": 8981, + "teaser": 8982, + "chen": 8983, + "restor": 8984, + "rat": 8985, + "flat": 8986, + "colom": 8987, + "ðŁĴŀ": 8988, + "ðŁĩ¨ðŁĩ": 8989, + "smooth": 8990, + "rt": 8991, + "pitch": 8992, + "staying": 8993, + "israeli": 8994, + "tcot": 8995, + "perspective": 8996, + "dock": 8997, + "opener": 8998, + "lovel": 8999, + "xo": 9000, + "classroom": 9001, + "lington": 9002, + "goal": 9003, + "kennedy": 9004, + "sham": 9005, + "spaces": 9006, + "mitchell": 9007, + "homecoming": 9008, + "uki": 9009, + "claimed": 9010, + "recruit": 9011, + "ingo": 9012, + "mufc": 9013, + "monit": 9014, + "groo": 9015, + "resident": 9016, + "percent": 9017, + "perman": 9018, + "ottawa": 9019, + "intment": 9020, + "anxi": 9021, + "standards": 9022, + "worship": 9023, + "scheme": 9024, + "fx": 9025, + "potter": 9026, + "bian": 9027, + "athletic": 9028, + "afgh": 9029, + "sse": 9030, + "satell": 9031, + "parties": 9032, + "âĿ¤âĿ¤": 9033, + "infrastructure": 9034, + "relax": 9035, + "modu": 9036, + "worn": 9037, + "smoking": 9038, + "yach": 9039, + "practices": 9040, + "wcw": 9041, + "amb": 9042, + "domestic": 9043, + "taylor": 9044, + "kentu": 9045, + "provided": 9046, + "modi": 9047, + "veg": 9048, + "\"...": 9049, + "observ": 9050, + "ðŁĺ©": 9051, + "beard": 9052, + "mour": 9053, + "angry": 9054, + "ðŁĺ±": 9055, + "startups": 9056, + "wooden": 9057, + "dive": 9058, + "nail": 9059, + "antique": 9060, + "roses": 9061, + "tornado": 9062, + "mat": 9063, + "^^": 9064, + "suspect": 9065, + "farm": 9066, + "devices": 9067, + "mega": 9068, + "tul": 9069, + "scholarship": 9070, + "gee": 9071, + "disaster": 9072, + "arrival": 9073, + "poin": 9074, + "marc": 9075, + "katie": 9076, + "bbed": 9077, + "false": 9078, + "deserves": 9079, + "richard": 9080, + "juana": 9081, + "frey": 9082, + "tioned": 9083, + "hybri": 9084, + "rw": 9085, + "sarah": 9086, + "achi": 9087, + "cure": 9088, + "ole": 9089, + "morris": 9090, + "chic": 9091, + "broadway": 9092, + "label": 9093, + "pak": 9094, + "poverty": 9095, + "golf": 9096, + "ered": 9097, + "fu": 9098, + "eries": 9099, + "bees": 9100, + "alogue": 9101, + "stel": 9102, + "wireless": 9103, + "jewish": 9104, + "tide": 9105, + "blocked": 9106, + "lifetime": 9107, + "bhar": 9108, + "split": 9109, + "amster": 9110, + "thi": 9111, + "joshu": 9112, + "brunch": 9113, + "haps": 9114, + "sfor": 9115, + "oops": 9116, + "kapoor": 9117, + "hiking": 9118, + "supposed": 9119, + "roof": 9120, + "reas": 9121, + "train": 9122, + "tight": 9123, + "trump": 9124, + "basically": 9125, + "rr": 9126, + "eared": 9127, + "seeds": 9128, + "entrance": 9129, + "cp": 9130, + "wie": 9131, + "sonic": 9132, + "victim": 9133, + "here": 9134, + "eh": 9135, + "earrings": 9136, + "salmon": 9137, + "arctic": 9138, + "anne": 9139, + "dougla": 9140, + "corruption": 9141, + "hannah": 9142, + "hasn": 9143, + "voices": 9144, + "conce": 9145, + "atta": 9146, + "fleet": 9147, + "clinical": 9148, + "democratic": 9149, + "tony": 9150, + "stood": 9151, + "lef": 9152, + "twitch": 9153, + "ail": 9154, + "honestly": 9155, + "increased": 9156, + "drome": 9157, + "donna": 9158, + "accepted": 9159, + "visitors": 9160, + "apar": 9161, + "ador": 9162, + "par": 9163, + "jerry": 9164, + "rai": 9165, + "brandon": 9166, + "abu": 9167, + "!!!!!!": 9168, + "meme": 9169, + "ingh": 9170, + "glorious": 9171, + "bhu": 9172, + "pump": 9173, + "jol": 9174, + "like": 9175, + "fisher": 9176, + "maz": 9177, + "agan": 9178, + "destination": 9179, + "playlist": 9180, + "letters": 9181, + "genu": 9182, + "brace": 9183, + "celebrated": 9184, + "banner": 9185, + "rhe": 9186, + "dragon": 9187, + "ðŁĺħ": 9188, + "signature": 9189, + "grey": 9190, + "âľĶï¸ı": 9191, + "alice": 9192, + "bered": 9193, + "pher": 9194, + "bern": 9195, + "cath": 9196, + "gathering": 9197, + "scoring": 9198, + "influence": 9199, + "smiling": 9200, + "dept": 9201, + "local": 9202, + "ax": 9203, + "acu": 9204, + "retirement": 9205, + "honor": 9206, + "herself": 9207, + "chemical": 9208, + "assess": 9209, + "yall": 9210, + "frequ": 9211, + "appreciation": 9212, + "aca": 9213, + "choir": 9214, + "cuz": 9215, + "soil": 9216, + "cil": 9217, + "reporting": 9218, + "uh": 9219, + "enterprise": 9220, + "grat": 9221, + "jacob": 9222, + "rum": 9223, + "fee": 9224, + "jak": 9225, + "spin": 9226, + "bikes": 9227, + "phia": 9228, + "stere": 9229, + "pis": 9230, + "blood": 9231, + "tatt": 9232, + "raft": 9233, + "warren": 9234, + "sheri": 9235, + "backstage": 9236, + "marsh": 9237, + "hashtag": 9238, + "therine": 9239, + "rein": 9240, + "gameday": 9241, + "guaran": 9242, + "recipes": 9243, + "minds": 9244, + "stronger": 9245, + "issued": 9246, + "bicy": 9247, + "nak": 9248, + "mented": 9249, + "scary": 9250, + "ux": 9251, + "previous": 9252, + "ttle": 9253, + "thats": 9254, + "actors": 9255, + "uma": 9256, + "tina": 9257, + "bunny": 9258, + "promotion": 9259, + "uss": 9260, + "oliver": 9261, + "montreal": 9262, + "whats": 9263, + "appreciated": 9264, + "lakes": 9265, + "excuse": 9266, + "knowing": 9267, + "prizes": 9268, + "muscle": 9269, + "shades": 9270, + "scot": 9271, + "ingredi": 9272, + "electronic": 9273, + "juan": 9274, + "combat": 9275, + "sri": 9276, + "eh": 9277, + "turkish": 9278, + "lom": 9279, + "strikes": 9280, + "prison": 9281, + "ree": 9282, + "pope": 9283, + "vid": 9284, + "oldest": 9285, + "doll": 9286, + "swiss": 9287, + "certified": 9288, + "clip": 9289, + "returning": 9290, + "lator": 9291, + "leigh": 9292, + "ttes": 9293, + "watson": 9294, + "healing": 9295, + "elim": 9296, + "perhaps": 9297, + "hass": 9298, + "kau": 9299, + "dder": 9300, + "mouse": 9301, + "newcastle": 9302, + "indigenous": 9303, + "welcomes": 9304, + "cole": 9305, + "taught": 9306, + "noise": 9307, + "appear": 9308, + "joe": 9309, + "canon": 9310, + "wednesday": 9311, + "utah": 9312, + "ctive": 9313, + "driven": 9314, + "iv": 9315, + "cell": 9316, + "strip": 9317, + "acc": 9318, + "focused": 9319, + "arrest": 9320, + "stocks": 9321, + "woo": 9322, + "âĹ": 9323, + "noticed": 9324, + "shado": 9325, + "displa": 9326, + "terror": 9327, + "borne": 9328, + "second": 9329, + "queens": 9330, + "woke": 9331, + "jail": 9332, + "nott": 9333, + "cambridge": 9334, + "hart": 9335, + "seaf": 9336, + "fax": 9337, + "accept": 9338, + "âĺħ": 9339, + "goods": 9340, + "kat": 9341, + "twin": 9342, + "hs": 9343, + "thousand": 9344, + "sins": 9345, + "suite": 9346, + "ampton": 9347, + "arn": 9348, + "relev": 9349, + "richar": 9350, + "hoops": 9351, + "nbc": 9352, + "classic": 9353, + "pab": 9354, + "soldier": 9355, + "deplo": 9356, + "leans": 9357, + "installation": 9358, + "clash": 9359, + "leban": 9360, + "eee": 9361, + "tire": 9362, + "beloved": 9363, + "fusion": 9364, + "traveling": 9365, + "nei": 9366, + "cookie": 9367, + "globe": 9368, + "physics": 9369, + "sq": 9370, + "col": 9371, + "wolves": 9372, + "dl": 9373, + "exit": 9374, + "\"-": 9375, + "football": 9376, + "leaf": 9377, + "sterling": 9378, + "hide": 9379, + "minneso": 9380, + "freshman": 9381, + "nature": 9382, + "indie": 9383, + "supplies": 9384, + "bris": 9385, + "irish": 9386, + "inktober": 9387, + "doodle": 9388, + "icop": 9389, + "messages": 9390, + "adults": 9391, + "recorded": 9392, + "fixed": 9393, + "ardo": 9394, + "offered": 9395, + "underground": 9396, + "drone": 9397, + "pine": 9398, + "mainten": 9399, + "andre": 9400, + "hammer": 9401, + "sx": 9402, + "round": 9403, + "hike": 9404, + "brad": 9405, + "rome": 9406, + "full": 9407, + "oney": 9408, + "rows": 9409, + "columbia": 9410, + "archives": 9411, + "approved": 9412, + "batch": 9413, + "illinois": 9414, + "recognition": 9415, + "shouldn": 9416, + "fog": 9417, + "ncaa": 9418, + "kevin": 9419, + "humanity": 9420, + "although": 9421, + "powers": 9422, + "pou": 9423, + "sar": 9424, + "pest": 9425, + "alcohol": 9426, + "consci": 9427, + "philadel": 9428, + "eno": 9429, + "tm": 9430, + "okla": 9431, + "category": 9432, + "participate": 9433, + "accused": 9434, + "brief": 9435, + "poem": 9436, + "clubs": 9437, + "consult": 9438, + "jab": 9439, + "bigdata": 9440, + "amsterdam": 9441, + "acing": 9442, + "certific": 9443, + "nu": 9444, + "dat": 9445, + "improved": 9446, + "andy": 9447, + "campaig": 9448, + "palestin": 9449, + "pace": 9450, + "mobi": 9451, + "feelings": 9452, + "wolf": 9453, + "brain": 9454, + "propos": 9455, + "interactive": 9456, + "prince": 9457, + "index": 9458, + "cis": 9459, + "chae": 9460, + "peaceful": 9461, + "covering": 9462, + "aco": 9463, + "courses": 9464, + "monkey": 9465, + "replace": 9466, + "bl": 9467, + "bloody": 9468, + "tales": 9469, + "brighton": 9470, + "neighborhood": 9471, + "gates": 9472, + "spiritual": 9473, + "afraid": 9474, + "breast": 9475, + "bones": 9476, + "ðŁijī": 9477, + "video": 9478, + "wau": 9479, + "touch": 9480, + "injuries": 9481, + "carl": 9482, + "rix": 9483, + "unex": 9484, + "âĢ¢": 9485, + "fred": 9486, + "considered": 9487, + "thusi": 9488, + "anch": 9489, + "ony": 9490, + "usa": 9491, + "graphics": 9492, + "acre": 9493, + "ðŁĺ©": 9494, + "commemor": 9495, + "commod": 9496, + "goti": 9497, + "guardian": 9498, + "starbucks": 9499, + "prevention": 9500, + "hahahaha": 9501, + "administration": 9502, + "portugal": 9503, + "faculty": 9504, + "beta": 9505, + "ula": 9506, + "albert": 9507, + "breath": 9508, + "eri": 9509, + "letting": 9510, + "tric": 9511, + "mentation": 9512, + "incredibly": 9513, + "tennes": 9514, + "vd": 9515, + "ðŁĻĪ": 9516, + "eddie": 9517, + "brick": 9518, + "grill": 9519, + "btw": 9520, + "watches": 9521, + "researchers": 9522, + "tney": 9523, + "nie": 9524, + "pas": 9525, + "aster": 9526, + "vibr": 9527, + "pokemon": 9528, + "chrome": 9529, + "goat": 9530, + "pitts": 9531, + "illy": 9532, + "festive": 9533, + "yd": 9534, + "canal": 9535, + "ðŁĨ": 9536, + "fies": 9537, + "carlos": 9538, + "reque": 9539, + "partici": 9540, + "trains": 9541, + "sample": 9542, + "temperature": 9543, + "symph": 9544, + "picking": 9545, + "indoor": 9546, + "zers": 9547, + "playoffs": 9548, + "________": 9549, + "apes": 9550, + "lyrics": 9551, + "islamic": 9552, + "performances": 9553, + "dick": 9554, + "spark": 9555, + "seas": 9556, + "homa": 9557, + "ground": 9558, + "disci": 9559, + "employee": 9560, + "commu": 9561, + "alaska": 9562, + "alan": 9563, + "feast": 9564, + "dging": 9565, + "banking": 9566, + "manuel": 9567, + "slowly": 9568, + "trucks": 9569, + "mccar": 9570, + "ooo": 9571, + "scrat": 9572, + "orchestra": 9573, + "individu": 9574, + "mx": 9575, + "breath": 9576, + "stairs": 9577, + "equality": 9578, + "blake": 9579, + "locations": 9580, + "coconut": 9581, + "baltimore": 9582, + "aaa": 9583, + "lc": 9584, + "ðŁıĨ": 9585, + "harvey": 9586, + "resist": 9587, + "immigration": 9588, + "adidas": 9589, + "fili": 9590, + "ref": 9591, + "lgbt": 9592, + "mos": 9593, + "ppi": 9594, + "kenny": 9595, + "terror": 9596, + "bane": 9597, + "apolis": 9598, + "sg": 9599, + "socialmedia": 9600, + "kai": 9601, + "honest": 9602, + "assas": 9603, + "bollywood": 9604, + "âĢįâĻĢï¸ı": 9605, + "ferrari": 9606, + "horn": 9607, + "crypto": 9608, + "boom": 9609, + "maintenance": 9610, + "idi": 9611, + "sman": 9612, + "wl": 9613, + "extended": 9614, + "insul": 9615, + "ves": 9616, + "gosp": 9617, + "tri": 9618, + "pig": 9619, + "targe": 9620, + "celer": 9621, + "stati": 9622, + "smh": 9623, + "ridic": 9624, + "appeal": 9625, + "?)": 9626, + "conclu": 9627, + "cosme": 9628, + "sheep": 9629, + "christopher": 9630, + "enthusi": 9631, + "polish": 9632, + "mets": 9633, + "ounded": 9634, + "sustainability": 9635, + "creativity": 9636, + "concrete": 9637, + "rai": 9638, + "alien": 9639, + "bless": 9640, + "tees": 9641, + "club": 9642, + "rot": 9643, + "bos": 9644, + "exist": 9645, + "perfection": 9646, + "luck": 9647, + "rocky": 9648, + "expensive": 9649, + "meanwhile": 9650, + "happybirthday": 9651, + "pret": 9652, + "thriller": 9653, + "cave": 9654, + "playoff": 9655, + "somer": 9656, + "lu": 9657, + "lex": 9658, + "defence": 9659, + "amwriting": 9660, + "homeless": 9661, + "prophe": 9662, + "chet": 9663, + "pastor": 9664, + "ðŁ¤£": 9665, + "lander": 9666, + "www": 9667, + "Ģï¸ı": 9668, + "tica": 9669, + "!#": 9670, + "otic": 9671, + "radar": 9672, + "posters": 9673, + "powder": 9674, + "poli": 9675, + "haun": 9676, + "trap": 9677, + "blin": 9678, + "assault": 9679, + "shorts": 9680, + "rey": 9681, + "shy": 9682, + "squir": 9683, + "racist": 9684, + "garlic": 9685, + "fur": 9686, + "remote": 9687, + "smell": 9688, + "impressed": 9689, + "fingers": 9690, + "âłĢ": 9691, + "dino": 9692, + "lement": 9693, + "snu": 9694, + "promoting": 9695, + "string": 9696, + "productive": 9697, + "bage": 9698, + "mason": 9699, + "raz": 9700, + "directly": 9701, + "jk": 9702, + "eval": 9703, + "ðŁijĬ": 9704, + "doctors": 9705, + "cow": 9706, + "rider": 9707, + "stv": 9708, + "remove": 9709, + "wu": 9710, + "nathan": 9711, + "rod": 9712, + "nr": 9713, + "=>": 9714, + "affected": 9715, + "invest": 9716, + "mption": 9717, + "ginger": 9718, + "od": 9719, + "agriculture": 9720, + "sque": 9721, + "mug": 9722, + "counting": 9723, + "kee": 9724, + "magnific": 9725, + "cook": 9726, + "anistan": 9727, + "root": 9728, + "placed": 9729, + "sympo": 9730, + "ghana": 9731, + "und": 9732, + "cheer": 9733, + "throwing": 9734, + "secrets": 9735, + "filling": 9736, + "optimi": 9737, + "butterfly": 9738, + "bubb": 9739, + "ðŁĺī": 9740, + "terrible": 9741, + "dg": 9742, + "silk": 9743, + "obsessed": 9744, + "lou": 9745, + "aide": 9746, + "salute": 9747, + "monu": 9748, + "philadelphia": 9749, + "scientific": 9750, + "ist": 9751, + "uae": 9752, + "dessert": 9753, + "bottles": 9754, + "canyon": 9755, + "ðŁĺĪ": 9756, + "carib": 9757, + "other": 9758, + "wich": 9759, + "resource": 9760, + "guilty": 9761, + "und": 9762, + "leon": 9763, + "ess": 9764, + "kane": 9765, + "ele": 9766, + "trainer": 9767, + "heim": 9768, + "ante": 9769, + "manage": 9770, + "rookie": 9771, + "treated": 9772, + "poses": 9773, + "rsvp": 9774, + "causes": 9775, + "awak": 9776, + "jewell": 9777, + "lett": 9778, + "onics": 9779, + "titles": 9780, + "cardiff": 9781, + "gaga": 9782, + "bump": 9783, + "useful": 9784, + "?!": 9785, + "loose": 9786, + "bbing": 9787, + "::": 9788, + "argentina": 9789, + "debu": 9790, + "cycl": 9791, + "whel": 9792, + "disgu": 9793, + "jel": 9794, + "kills": 9795, + "biology": 9796, + "exter": 9797, + "trash": 9798, + "bodies": 9799, + "tram": 9800, + "circuit": 9801, + "expect": 9802, + "lads": 9803, + "wells": 9804, + "shot": 9805, + "gee": 9806, + "narendr": 9807, + "fastest": 9808, + "bent": 9809, + "bills": 9810, + "marshall": 9811, + "hats": 9812, + "introduce": 9813, + "citizen": 9814, + "impossible": 9815, + "gib": 9816, + "azz": 9817, + "networking": 9818, + "rant": 9819, + "think": 9820, + "indy": 9821, + "stops": 9822, + "ftheday": 9823, + "brian": 9824, + "**": 9825, + "amodi": 9826, + "dome": 9827, + "courage": 9828, + "packing": 9829, + "affairs": 9830, + "gn": 9831, + "sized": 9832, + "entary": 9833, + "poland": 9834, + "switzer": 9835, + "afghanistan": 9836, + "wu": 9837, + "tender": 9838, + "subscribe": 9839, + "mosco": 9840, + "attend": 9841, + "republican": 9842, + "honey": 9843, + "âĢĭ": 9844, + "simul": 9845, + "wester": 9846, + "foodie": 9847, + "oro": 9848, + "middle": 9849, + "abt": 9850, + "copies": 9851, + "maje": 9852, + "narendramodi": 9853, + "typical": 9854, + "inspirational": 9855, + "vitam": 9856, + "wiscon": 9857, + "cubs": 9858, + "tivity": 9859, + "hali": 9860, + "ears": 9861, + "kay": 9862, + "dare": 9863, + "marijuana": 9864, + "curious": 9865, + "ania": 9866, + "tomato": 9867, + "remind": 9868, + "ðŁĩ·": 9869, + "scared": 9870, + "coup": 9871, + "poet": 9872, + "landed": 9873, + "rid": 9874, + "wrapped": 9875, + "morri": 9876, + "climbing": 9877, + "ews": 9878, + "feeding": 9879, + "contra": 9880, + "thology": 9881, + "grid": 9882, + "tively": 9883, + "reader": 9884, + "laser": 9885, + "diving": 9886, + "dig": 9887, + "latin": 9888, + "tied": 9889, + "shakespe": 9890, + "oci": 9891, + "adm": 9892, + "showers": 9893, + "chuck": 9894, + "marcus": 9895, + "oos": 9896, + "knee": 9897, + "olive": 9898, + "owl": 9899, + "dylan": 9900, + "anno": 9901, + "gym": 9902, + "decisions": 9903, + "wellness": 9904, + "arrives": 9905, + "satis": 9906, + "chris": 9907, + "thurs": 9908, + "ðŁ¤£": 9909, + "interviews": 9910, + "thankyou": 9911, + "switzerland": 9912, + "overnight": 9913, + "journalist": 9914, + "serves": 9915, + "volcan": 9916, + ".......": 9917, + "plot": 9918, + "nicol": 9919, + "carrying": 9920, + "magne": 9921, + "treasure": 9922, + "exp": 9923, + "bever": 9924, + "ðŁĺ¢": 9925, + "marty": 9926, + "mole": 9927, + "donations": 9928, + "recognized": 9929, + "bh": 9930, + "dus": 9931, + "shann": 9932, + "aldo": 9933, + "successfully": 9934, + "ente": 9935, + "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ": 9936, + "cabinet": 9937, + "cuis": 9938, + "titled": 9939, + "das": 9940, + "sol": 9941, + "strategies": 9942, + "delivering": 9943, + "adds": 9944, + "anian": 9945, + "nether": 9946, + "ðŁĴĥ": 9947, + "contain": 9948, + "suits": 9949, + "pairs": 9950, + "todd": 9951, + "rella": 9952, + "rope": 9953, + "cio": 9954, + "crop": 9955, + "paintings": 9956, + "suz": 9957, + "rejec": 9958, + "bust": 9959, + "dh": 9960, + "fraud": 9961, + "mh": 9962, + "control": 9963, + "jeal": 9964, + "destroyed": 9965, + "allows": 9966, + "wool": 9967, + "minnesota": 9968, + "omen": 9969, + "ju": 9970, + "symposium": 9971, + "daf": 9972, + "limit": 9973, + "accounts": 9974, + "loading": 9975, + "intern": 9976, + "resolution": 9977, + "holland": 9978, + "qual": 9979, + "meetings": 9980, + "grave": 9981, + "camping": 9982, + "vam": 9983, + "renov": 9984, + "liberal": 9985, + "amber": 9986, + "gree": 9987, + "humb": 9988, + "fever": 9989, + "eling": 9990, + "brooks": 9991, + "à²": 9992, + "beth": 9993, + "aded": 9994, + "alt": 9995, + "roe": 9996, + "performed": 9997, + "josh": 9998, + "franklin": 9999, + "nicole": 10000, + "dess": 10001, + "bbs": 10002, + "mg": 10003, + "networks": 10004, + "minim": 10005, + "alt": 10006, + "weapons": 10007, + "guy": 10008, + "jason": 10009, + "gha": 10010, + "harbour": 10011, + "aton": 10012, + "praise": 10013, + "kentucky": 10014, + "belfast": 10015, + "sticks": 10016, + "bloss": 10017, + "hopes": 10018, + "anthro": 10019, + "familiar": 10020, + "wait": 10021, + "chile": 10022, + "depression": 10023, + "lax": 10024, + "jets": 10025, + "leice": 10026, + "receives": 10027, + "sier": 10028, + "ank": 10029, + "dex": 10030, + "indeed": 10031, + "flexi": 10032, + "fabric": 10033, + "lamb": 10034, + "helicop": 10035, + "amanda": 10036, + "âĢĶâĢĶ": 10037, + "compete": 10038, + "snack": 10039, + "technologies": 10040, + "syrian": 10041, + "moms": 10042, + "muham": 10043, + "chosen": 10044, + "anat": 10045, + "devon": 10046, + "sharks": 10047, + "ret": 10048, + "fundraiser": 10049, + "selfies": 10050, + "stations": 10051, + "communications": 10052, + "tennessee": 10053, + "tutor": 10054, + "rot": 10055, + "valuable": 10056, + "dynamic": 10057, + "nurse": 10058, + "ied": 10059, + "earthquake": 10060, + "deserved": 10061, + "ave": 10062, + "sara": 10063, + "stretch": 10064, + "douglas": 10065, + "nepal": 10066, + "ç": 10067, + "obviously": 10068, + "dame": 10069, + "rape": 10070, + "anybody": 10071, + "kw": 10072, + "patrol": 10073, + "holders": 10074, + "hanna": 10075, + "infographic": 10076, + "eco": 10077, + "beating": 10078, + "stanley": 10079, + "boats": 10080, + "ribb": 10081, + "ez": 10082, + "witch": 10083, + "inva": 10084, + "acid": 10085, + "boarding": 10086, + "-@": 10087, + "gil": 10088, + "dave": 10089, + "careers": 10090, + "oppos": 10091, + "lloy": 10092, + "inter": 10093, + "dope": 10094, + "resu": 10095, + "jagu": 10096, + "shade": 10097, + "indy": 10098, + "onist": 10099, + "relations": 10100, + "agen": 10101, + "able": 10102, + "incident": 10103, + "meter": 10104, + "sharma": 10105, + "idr": 10106, + "prove": 10107, + "immediately": 10108, + "troops": 10109, + "aman": 10110, + "glow": 10111, + "gaza": 10112, + "blocks": 10113, + "personal": 10114, + "chronic": 10115, + "aller": 10116, + "sid": 10117, + "shr": 10118, + "whatsapp": 10119, + "lucy": 10120, + "archae": 10121, + "hou": 10122, + "journalism": 10123, + "ourselves": 10124, + "got": 10125, + "themed": 10126, + "shaped": 10127, + "weak": 10128, + "casual": 10129, + "length": 10130, + "slam": 10131, + "abbey": 10132, + "ev": 10133, + "counter": 10134, + "esta": 10135, + "recipi": 10136, + "chapel": 10137, + "expansion": 10138, + "self": 10139, + "suffering": 10140, + "spice": 10141, + "nz": 10142, + "spart": 10143, + "desper": 10144, + "booking": 10145, + "quarters": 10146, + "yon": 10147, + "ðŁĴĹ": 10148, + "pk": 10149, + "continued": 10150, + "-#": 10151, + "manhatt": 10152, + "talked": 10153, + "shen": 10154, + "combo": 10155, + "hybrid": 10156, + "jeans": 10157, + "liquid": 10158, + "seal": 10159, + "retweets": 10160, + "acceler": 10161, + "collective": 10162, + "tas": 10163, + ":))": 10164, + "professionals": 10165, + "raw": 10166, + "ott": 10167, + "susan": 10168, + "iring": 10169, + "oklahoma": 10170, + "reven": 10171, + "survival": 10172, + "creator": 10173, + "transit": 10174, + "stac": 10175, + "surf": 10176, + "ik": 10177, + "editing": 10178, + "chilling": 10179, + "bailey": 10180, + "steal": 10181, + "rable": 10182, + "parent": 10183, + "hunger": 10184, + "snapp": 10185, + "collect": 10186, + "philosoph": 10187, + "dedication": 10188, + "cf": 10189, + "cm": 10190, + "leep": 10191, + "repeat": 10192, + "reha": 10193, + "unfortun": 10194, + "aer": 10195, + "aero": 10196, + "abstract": 10197, + "monitor": 10198, + "agents": 10199, + "bul": 10200, + "science": 10201, + "harbor": 10202, + "dragons": 10203, + "flooding": 10204, + "accompli": 10205, + "dash": 10206, + "julia": 10207, + "thered": 10208, + "tuesday": 10209, + "cyber": 10210, + "blow": 10211, + "tained": 10212, + "lem": 10213, + "reference": 10214, + "ppo": 10215, + "negoti": 10216, + "charle": 10217, + "connor": 10218, + "ault": 10219, + "accessories": 10220, + "commissioner": 10221, + "rainy": 10222, + "rear": 10223, + "advisory": 10224, + "lucas": 10225, + "maid": 10226, + "coal": 10227, + "kav": 10228, + "polo": 10229, + "ðŁı¾": 10230, + "transport": 10231, + "margare": 10232, + "strawberry": 10233, + "burns": 10234, + "greens": 10235, + "nev": 10236, + "participants": 10237, + "colin": 10238, + "belgium": 10239, + "colour": 10240, + "inform": 10241, + "dell": 10242, + "bron": 10243, + "caly": 10244, + "kickoff": 10245, + "strategic": 10246, + "reunion": 10247, + "honors": 10248, + "lib": 10249, + "egyp": 10250, + "âŃIJï¸ı": 10251, + "hypo": 10252, + "sizes": 10253, + "registered": 10254, + "betes": 10255, + "relaxing": 10256, + "bloom": 10257, + "intense": 10258, + "valentines": 10259, + "insane": 10260, + "wwii": 10261, + "px": 10262, + "trio": 10263, + "blade": 10264, + "wisconsin": 10265, + "cone": 10266, + "platin": 10267, + "alize": 10268, + "raven": 10269, + "increasing": 10270, + "indians": 10271, + "ilian": 10272, + "blu": 10273, + "rabbit": 10274, + "extension": 10275, + "jef": 10276, + "audi": 10277, + "ferry": 10278, + "sell": 10279, + "aday": 10280, + "usb": 10281, + "sweat": 10282, + "champag": 10283, + "method": 10284, + "memph": 10285, + "assist": 10286, + "sby": 10287, + "cape": 10288, + "removed": 10289, + "magn": 10290, + "vt": 10291, + "rams": 10292, + "fbi": 10293, + "tackle": 10294, + "phew": 10295, + "hon": 10296, + "motorcycle": 10297, + "suspec": 10298, + "elephant": 10299, + "subject": 10300, + "lette": 10301, + "dairy": 10302, + "wheat": 10303, + "awkward": 10304, + "act": 10305, + "trol": 10306, + "mitted": 10307, + "zayn": 10308, + "sheriff": 10309, + "enemy": 10310, + "cons": 10311, + "kett": 10312, + "bulls": 10313, + "evalu": 10314, + "btc": 10315, + "satellite": 10316, + "holo": 10317, + "porter": 10318, + "diabetes": 10319, + "better": 10320, + "releasing": 10321, + "surf": 10322, + ":-": 10323, + "sebasti": 10324, + "collecting": 10325, + "encing": 10326, + "ethi": 10327, + "gods": 10328, + "alley": 10329, + "healthy": 10330, + "mills": 10331, + "smash": 10332, + "copper": 10333, + "crack": 10334, + "readers": 10335, + "spac": 10336, + "license": 10337, + "basket": 10338, + "bangla": 10339, + "entic": 10340, + "omi": 10341, + "mere": 10342, + "sively": 10343, + "animation": 10344, + "lanes": 10345, + "dentally": 10346, + "chillin": 10347, + "fie": 10348, + "karen": 10349, + "depth": 10350, + "lipse": 10351, + "ng": 10352, + "rip": 10353, + "melo": 10354, + "sandy": 10355, + "ðŁijıðŁijı": 10356, + "vincent": 10357, + "nut": 10358, + "hug": 10359, + "whole": 10360, + "creates": 10361, + "????": 10362, + "âĿ¤ï¸ıâĿ¤ï¸ı": 10363, + "baked": 10364, + "upgrade": 10365, + "roberts": 10366, + "hara": 10367, + "caribbean": 10368, + "authentic": 10369, + "mbs": 10370, + "moscow": 10371, + "attorney": 10372, + "wiki": 10373, + "chlo": 10374, + "hull": 10375, + "cork": 10376, + "\"!": 10377, + "stylish": 10378, + "ðŁĵ¸:": 10379, + "diary": 10380, + "improving": 10381, + "expand": 10382, + "bright": 10383, + "pollution": 10384, + "knights": 10385, + "personality": 10386, + "checked": 10387, + "facilities": 10388, + "zel": 10389, + "bowling": 10390, + "guer": 10391, + "ðŁİĤ": 10392, + "ongoing": 10393, + "units": 10394, + "hook": 10395, + "beck": 10396, + "conflict": 10397, + "todd": 10398, + "farming": 10399, + "educational": 10400, + "kak": 10401, + "clay": 10402, + "stroke": 10403, + "belly": 10404, + "explore": 10405, + "millenni": 10406, + "thm": 10407, + "loop": 10408, + "sms": 10409, + "consist": 10410, + "circa": 10411, + "bryan": 10412, + "dab": 10413, + "younger": 10414, + "solidar": 10415, + "ppa": 10416, + "experienced": 10417, + "bella": 10418, + "board": 10419, + "sheffield": 10420, + "stephen": 10421, + "consumer": 10422, + "submit": 10423, + "sponsor": 10424, + "tang": 10425, + "aggre": 10426, + "combined": 10427, + "tracking": 10428, + "sanders": 10429, + "baz": 10430, + "survive": 10431, + "ferred": 10432, + "equal": 10433, + "sep": 10434, + "reed": 10435, + "strong": 10436, + "privacy": 10437, + "stap": 10438, + "ung": 10439, + "acry": 10440, + "pasta": 10441, + "pirates": 10442, + "ager": 10443, + "fairy": 10444, + "dup": 10445, + "introduced": 10446, + "wip": 10447, + "lets": 10448, + "spray": 10449, + "ðŁĵº": 10450, + "grew": 10451, + "asts": 10452, + "pittsburgh": 10453, + "newyork": 10454, + "joey": 10455, + "lauren": 10456, + "trade": 10457, + "chop": 10458, + "pipe": 10459, + "claire": 10460, + "behavior": 10461, + "vap": 10462, + "crews": 10463, + "laptop": 10464, + "ðŁ¤Ĺ": 10465, + "chester": 10466, + "discipl": 10467, + "df": 10468, + "outdoors": 10469, + "ks": 10470, + "gover": 10471, + "superstar": 10472, + "casino": 10473, + "farmer": 10474, + ";-)": 10475, + "returned": 10476, + "ðŁıĪ": 10477, + "mail": 10478, + "roasted": 10479, + "costa": 10480, + "vill": 10481, + "pez": 10482, + "gardening": 10483, + "distribution": 10484, + "shining": 10485, + "investors": 10486, + "rasp": 10487, + "decades": 10488, + "realized": 10489, + "barn": 10490, + "pti": 10491, + "stable": 10492, + "utd": 10493, + "panthers": 10494, + "mens": 10495, + "bn": 10496, + "cade": 10497, + "bucket": 10498, + "ynn": 10499, + "whenever": 10500, + "wake": 10501, + "dais": 10502, + "bernie": 10503, + "lodge": 10504, + "julie": 10505, + "atmosphere": 10506, + "ðŁĺĺðŁĺĺ": 10507, + "majority": 10508, + "parti": 10509, + "excit": 10510, + "cut": 10511, + "meh": 10512, + "muslims": 10513, + "begun": 10514, + "flights": 10515, + "veness": 10516, + "ceme": 10517, + "posing": 10518, + "sole": 10519, + "gou": 10520, + "darkness": 10521, + "peach": 10522, + "celtic": 10523, + "authority": 10524, + "grandma": 10525, + "fulness": 10526, + "smith": 10527, + "specific": 10528, + "garcia": 10529, + "coins": 10530, + "goodness": 10531, + "aldub": 10532, + "recruiting": 10533, + "dennis": 10534, + "gary": 10535, + "sleeve": 10536, + "weapon": 10537, + "plz": 10538, + "discover": 10539, + "harrison": 10540, + "recruitment": 10541, + "jai": 10542, + "chim": 10543, + "compared": 10544, + "toms": 10545, + "mothers": 10546, + "amy": 10547, + "archive": 10548, + "task": 10549, + "benjam": 10550, + "seg": 10551, + "lawyer": 10552, + "alum": 10553, + "investing": 10554, + "mie": 10555, + "chez": 10556, + "jp": 10557, + "ake": 10558, + "flam": 10559, + "wallpaper": 10560, + "âĻ¥ï¸ı": 10561, + "tton": 10562, + "chest": 10563, + "favorites": 10564, + "weigh": 10565, + "coolest": 10566, + "rating": 10567, + "relevant": 10568, + "logan": 10569, + "maple": 10570, + "runners": 10571, + "prior": 10572, + "people": 10573, + "maur": 10574, + "terrorist": 10575, + "tested": 10576, + "carnival": 10577, + "suspen": 10578, + "measure": 10579, + "mv": 10580, + "cybersecurity": 10581, + "appren": 10582, + "terrorism": 10583, + "oz": 10584, + "vital": 10585, + "nies": 10586, + "gonz": 10587, + "funded": 10588, + "twist": 10589, + "assessment": 10590, + "diesel": 10591, + "enfor": 10592, + "column": 10593, + "addressing": 10594, + "casts": 10595, + "payment": 10596, + "xton": 10597, + "fier": 10598, + ",'": 10599, + "last": 10600, + "nee": 10601, + "unless": 10602, + "close": 10603, + "skill": 10604, + "cuisine": 10605, + "funeral": 10606, + "tiles": 10607, + "aun": 10608, + "kru": 10609, + "relationships": 10610, + "ðŁĴ¯": 10611, + "event": 10612, + "âĢįâĻĤï¸ı": 10613, + "kindness": 10614, + "proposed": 10615, + "acoustic": 10616, + "aes": 10617, + "defender": 10618, + "dance": 10619, + "htt": 10620, + "wat": 10621, + "voy": 10622, + "ðŁ¤ĺ": 10623, + "aus": 10624, + "cliff": 10625, + "searching": 10626, + "beautifully": 10627, + "inqu": 10628, + "atl": 10629, + "specialist": 10630, + "ðŁIJ¶": 10631, + "dai": 10632, + "trails": 10633, + "classics": 10634, + "instant": 10635, + "vous": 10636, + "revenue": 10637, + "march": 10638, + "kirk": 10639, + "fringe": 10640, + "fireworks": 10641, + "trivia": 10642, + "âĺħ": 10643, + "traction": 10644, + "walter": 10645, + "moto": 10646, + "lily": 10647, + "attitude": 10648, + "climb": 10649, + "scan": 10650, + "savings": 10651, + "cw": 10652, + "faith": 10653, + "credits": 10654, + "abled": 10655, + "graff": 10656, + "autograph": 10657, + "hehe": 10658, + "ranch": 10659, + "had": 10660, + "rogers": 10661, + "ðŁĮ¹": 10662, + "fin": 10663, + "requ": 10664, + "folk": 10665, + "additional": 10666, + "lynn": 10667, + "uber": 10668, + "dollars": 10669, + "logic": 10670, + "worth": 10671, + "som": 10672, + "thesis": 10673, + "pound": 10674, + "bic": 10675, + "stur": 10676, + "ceram": 10677, + "spencer": 10678, + "entered": 10679, + "vamp": 10680, + "organized": 10681, + "âľĪ": 10682, + "pps": 10683, + "tron": 10684, + "mercedes": 10685, + "noti": 10686, + "competitive": 10687, + "dow": 10688, + "ousness": 10689, + "victor": 10690, + "grilled": 10691, + "nai": 10692, + "putin": 10693, + "abra": 10694, + "blame": 10695, + "alexand": 10696, + "animal": 10697, + "decent": 10698, + "pent": 10699, + "interior": 10700, + ":')": 10701, + "butler": 10702, + "ballet": 10703, + "ðŁĴĶ": 10704, + "albums": 10705, + "downs": 10706, + "lad": 10707, + "sir": 10708, + "plain": 10709, + "pers": 10710, + "blonde": 10711, + "disc": 10712, + "pakistan": 10713, + "sement": 10714, + "gaa": 10715, + "wage": 10716, + "chas": 10717, + "mani": 10718, + "cops": 10719, + "territ": 10720, + "lol": 10721, + "laughter": 10722, + "rivers": 10723, + "magnificent": 10724, + "lamp": 10725, + "wb": 10726, + "newsle": 10727, + "charts": 10728, + "blessing": 10729, + "punch": 10730, + "longest": 10731, + "floral": 10732, + "cutie": 10733, + "farewell": 10734, + "stopping": 10735, + "mbb": 10736, + "bud": 10737, + "cheese": 10738, + "decla": 10739, + "sim": 10740, + "mcdonald": 10741, + "deter": 10742, + "youth": 10743, + "tch": 10744, + "freder": 10745, + "kindle": 10746, + "fern": 10747, + "ator": 10748, + "asleep": 10749, + "pond": 10750, + "sprint": 10751, + "pounds": 10752, + "lazy": 10753, + "ghe": 10754, + "fundraising": 10755, + "deadly": 10756, + "grande": 10757, + "doug": 10758, + "hey": 10759, + "linda": 10760, + "considering": 10761, + "ium": 10762, + "golden": 10763, + "vik": 10764, + "authors": 10765, + "diss": 10766, + "ually": 10767, + "appropriate": 10768, + "morning": 10769, + "yle": 10770, + "honoring": 10771, + "folio": 10772, + "bec": 10773, + "rebec": 10774, + "finland": 10775, + "formula": 10776, + "cornwall": 10777, + "shay": 10778, + "causing": 10779, + "blend": 10780, + "signal": 10781, + "tent": 10782, + "kashmir": 10783, + "nationals": 10784, + "harmony": 10785, + "scout": 10786, + "accessi": 10787, + "height": 10788, + "medieval": 10789, + "improvement": 10790, + "kees": 10791, + "practical": 10792, + "card": 10793, + "depar": 10794, + "hun": 10795, + "oming": 10796, + "calgary": 10797, + "stel": 10798, + "bubble": 10799, + "guru": 10800, + "mah": 10801, + "unexpe": 10802, + "nh": 10803, + "eda": 10804, + "meat": 10805, + "ige": 10806, + "sio": 10807, + "goddess": 10808, + "inches": 10809, + "tunes": 10810, + "britt": 10811, + "stion": 10812, + "raj": 10813, + "âĻ«": 10814, + "mercy": 10815, + "ðŁĴĺ": 10816, + "sends": 10817, + "iest": 10818, + "polici": 10819, + "vale": 10820, + "reduced": 10821, + "asap": 10822, + "vijay": 10823, + "defensive": 10824, + "celebrations": 10825, + "riders": 10826, + "meditation": 10827, + "harmon": 10828, + "ging": 10829, + "¡": 10830, + "programming": 10831, + "inau": 10832, + "sudden": 10833, + "mh": 10834, + "replacement": 10835, + "sku": 10836, + "jar": 10837, + "grades": 10838, + "tast": 10839, + "kitt": 10840, + "branding": 10841, + "kaw": 10842, + "boot": 10843, + "fought": 10844, + "pays": 10845, + "gf": 10846, + "ization": 10847, + "hop": 10848, + "kk": 10849, + "activist": 10850, + "vend": 10851, + "coastal": 10852, + "chaos": 10853, + "ðŁĶ´": 10854, + "seme": 10855, + "billboard": 10856, + "lifting": 10857, + "cumb": 10858, + "scal": 10859, + "ðŁĸ¤": 10860, + "struck": 10861, + "lv": 10862, + "indiedev": 10863, + "beaten": 10864, + "jungle": 10865, + "alright": 10866, + "destiny": 10867, + "ming": 10868, + "kc": 10869, + "chances": 10870, + "oman": 10871, + "qatar": 10872, + "craf": 10873, + "trained": 10874, + "prix": 10875, + "charm": 10876, + "otive": 10877, + "smu": 10878, + "ec": 10879, + "anders": 10880, + "handed": 10881, + "alban": 10882, + "certainly": 10883, + "arriving": 10884, + "ize": 10885, + "sai": 10886, + "track": 10887, + "painter": 10888, + "humble": 10889, + "appointment": 10890, + "headline": 10891, + "managing": 10892, + "mod": 10893, + "aspe": 10894, + "andrea": 10895, + "ä": 10896, + "ethiop": 10897, + "united": 10898, + "exist": 10899, + "bali": 10900, + "kad": 10901, + "nt": 10902, + "dred": 10903, + "rex": 10904, + "recognize": 10905, + "tampa": 10906, + "beers": 10907, + "atia": 10908, + "heels": 10909, + "note": 10910, + "transportation": 10911, + "turtle": 10912, + "rede": 10913, + "hiphop": 10914, + "spicy": 10915, + "spurs": 10916, + "â¬ĩ": 10917, + "corp": 10918, + "thern": 10919, + "toast": 10920, + "hurry": 10921, + "properties": 10922, + "mage": 10923, + "marco": 10924, + "elements": 10925, + "bouti": 10926, + "syndrome": 10927, + "msg": 10928, + "developer": 10929, + "graders": 10930, + "heim": 10931, + "resil": 10932, + "offices": 10933, + "delay": 10934, + "dimen": 10935, + "vintag": 10936, + "barbara": 10937, + "ðŁĺ±": 10938, + "venezu": 10939, + "cular": 10940, + "faced": 10941, + "barn": 10942, + "ðŁĺĨ": 10943, + "survivor": 10944, + "worm": 10945, + "confused": 10946, + "passionate": 10947, + "ر": 10948, + "identify": 10949, + "electricity": 10950, + "souls": 10951, + "bradley": 10952, + "reportedly": 10953, + "lunch": 10954, + "shelf": 10955, + "elia": 10956, + "sweet": 10957, + "smooth": 10958, + "employment": 10959, + "amel": 10960, + "manhattan": 10961, + "steam": 10962, + "ounts": 10963, + "yep": 10964, + "living": 10965, + "une": 10966, + "describe": 10967, + "cares": 10968, + "manila": 10969, + "shawn": 10970, + "acted": 10971, + "bash": 10972, + "steven": 10973, + "rest": 10974, + "petition": 10975, + "divine": 10976, + "welsh": 10977, + "race": 10978, + "platinum": 10979, + "ðŁĮ¸": 10980, + "pb": 10981, + "extraordinary": 10982, + "solidarity": 10983, + "mall": 10984, + "onion": 10985, + "scheduled": 10986, + "gameof": 10987, + "fergu": 10988, + "dems": 10989, + "norm": 10990, + "pk": 10991, + "trials": 10992, + "policies": 10993, + "publishing": 10994, + "stole": 10995, + "front": 10996, + "character": 10997, + "vania": 10998, + "exce": 10999, + "stie": 11000, + "sca": 11001, + "residential": 11002, + "sailing": 11003, + "ðŁĶ¥ðŁĶ¥ðŁĶ¥": 11004, + "sponsors": 11005, + "thick": 11006, + "champagne": 11007, + "shepher": 11008, + "continuing": 11009, + "venice": 11010, + "perth": 11011, + "nap": 11012, + "aster": 11013, + "yak": 11014, + "unlimited": 11015, + "choices": 11016, + "neo": 11017, + "hiv": 11018, + "reporter": 11019, + "brussels": 11020, + "fold": 11021, + "dys": 11022, + "semi": 11023, + "lawn": 11024, + "italia": 11025, + "wifi": 11026, + "ask": 11027, + "emed": 11028, + "frame": 11029, + "monitoring": 11030, + "stead": 11031, + "ida": 11032, + "grin": 11033, + "isa": 11034, + "flip": 11035, + "restric": 11036, + "offensive": 11037, + "attached": 11038, + "dish": 11039, + "why": 11040, + "phillips": 11041, + "greet": 11042, + "pals": 11043, + "mixtape": 11044, + "vou": 11045, + "fielder": 11046, + "spark": 11047, + "alberta": 11048, + "glen": 11049, + "cash": 11050, + "sri": 11051, + "uri": 11052, + "rodri": 11053, + "entrepreneurs": 11054, + "climatechange": 11055, + "psy": 11056, + "dle": 11057, + "ements": 11058, + "linked": 11059, + "netherlands": 11060, + "accidentally": 11061, + "opposition": 11062, + "velvet": 11063, + "rays": 11064, + "cw": 11065, + "omo": 11066, + "mf": 11067, + "lmfao": 11068, + "newsletter": 11069, + ":)": 11070, + "toilet": 11071, + "literature": 11072, + "disp": 11073, + "philip": 11074, + "uniform": 11075, + "suddenly": 11076, + "header": 11077, + "cooler": 11078, + "---": 11079, + "proud": 11080, + "brig": 11081, + "nissan": 11082, + "scientist": 11083, + "jah": 11084, + "concentr": 11085, + "packs": 11086, + "appointed": 11087, + "soap": 11088, + "engage": 11089, + "chose": 11090, + "âĻ¡": 11091, + "setup": 11092, + "jealous": 11093, + "harry": 11094, + "gation": 11095, + "tunnel": 11096, + "temp": 11097, + "oscars": 11098, + "decade": 11099, + "recommended": 11100, + "children": 11101, + "aba": 11102, + "anxiety": 11103, + "vements": 11104, + "salon": 11105, + "photoo": 11106, + "organiz": 11107, + "machines": 11108, + "abs": 11109, + "ville": 11110, + "hype": 11111, + "tiff": 11112, + "emerging": 11113, + "avgeek": 11114, + "[#": 11115, + "contribution": 11116, + "brady": 11117, + "resto": 11118, + "gmail": 11119, + "fitz": 11120, + "photoshoot": 11121, + "helmet": 11122, + "ht": 11123, + "elegant": 11124, + "uganda": 11125, + "nursing": 11126, + "orleans": 11127, + "penn": 11128, + "nah": 11129, + "footage": 11130, + "ema": 11131, + "wo": 11132, + "wad": 11133, + "concerns": 11134, + "vere": 11135, + "remark": 11136, + "whoever": 11137, + "strang": 11138, + "pt": 11139, + "quit": 11140, + "shang": 11141, + "history": 11142, + "sick": 11143, + "permanent": 11144, + "illness": 11145, + "cold": 11146, + "vision": 11147, + "hem": 11148, + "arrow": 11149, + "convic": 11150, + "pink": 11151, + "occup": 11152, + "bald": 11153, + "exhau": 11154, + "uof": 11155, + "amo": 11156, + "ont": 11157, + "ãĥ»": 11158, + "adopt": 11159, + "laid": 11160, + "smoked": 11161, + "interpre": 11162, + "essenti": 11163, + "associated": 11164, + "bd": 11165, + "bby": 11166, + "fier": 11167, + "install": 11168, + "diplom": 11169, + "conditi": 11170, + "cf": 11171, + "wak": 11172, + "anya": 11173, + "graci": 11174, + "fisher": 11175, + "sss": 11176, + "apr": 11177, + "ilit": 11178, + "musician": 11179, + "symphony": 11180, + "cord": 11181, + "hack": 11182, + "legi": 11183, + "lv": 11184, + "blessings": 11185, + "humor": 11186, + "scra": 11187, + "eti": 11188, + "minster": 11189, + "travelling": 11190, + "bush": 11191, + "jewellery": 11192, + "lime": 11193, + "!!!": 11194, + "pregnant": 11195, + "pee": 11196, + "lob": 11197, + "capital": 11198, + "ipa": 11199, + "pencil": 11200, + "labor": 11201, + "ducks": 11202, + "proudly": 11203, + "wedding": 11204, + "derek": 11205, + "mw": 11206, + "peg": 11207, + "valentine": 11208, + "angu": 11209, + "retreat": 11210, + "prospect": 11211, + "danger": 11212, + "vulner": 11213, + "upset": 11214, + ",#": 11215, + "srk": 11216, + "xim": 11217, + "thursday": 11218, + "nfl": 11219, + "kisses": 11220, + "reds": 11221, + "crack": 11222, + "reward": 11223, + "cu": 11224, + "kok": 11225, + "mete": 11226, + "abandoned": 11227, + "itt": 11228, + "meals": 11229, + "spell": 11230, + "stanbul": 11231, + "delays": 11232, + "rum": 11233, + "leop": 11234, + "gum": 11235, + "nova": 11236, + "superman": 11237, + "chick": 11238, + "mis": 11239, + "dramatic": 11240, + "innocent": 11241, + "rounds": 11242, + "rec": 11243, + "autism": 11244, + "bangladesh": 11245, + "moral": 11246, + "movie": 11247, + "spoo": 11248, + "kla": 11249, + "âĥ£": 11250, + "outing": 11251, + "messi": 11252, + "abroad": 11253, + "lookin": 11254, + "aim": 11255, + "qi": 11256, + "stack": 11257, + "collage": 11258, + "à¯": 11259, + "hudson": 11260, + "scan": 11261, + "hoe": 11262, + "chau": 11263, + "occur": 11264, + "commander": 11265, + "holes": 11266, + "ðŁİĦ": 11267, + "bias": 11268, + "von": 11269, + "sticker": 11270, + "mak": 11271, + "responsibility": 11272, + "columbus": 11273, + "saint": 11274, + "edmon": 11275, + "racism": 11276, + "farms": 11277, + "wen": 11278, + "gulf": 11279, + "mayo": 11280, + "!!!!!!!!": 11281, + "corporation": 11282, + "bachel": 11283, + "ela": 11284, + "internal": 11285, + "jeep": 11286, + "follows": 11287, + "dialogue": 11288, + "derer": 11289, + "smartphone": 11290, + "helen": 11291, + "richmond": 11292, + "equity": 11293, + "sland": 11294, + "bg": 11295, + "near": 11296, + "avi": 11297, + "memphis": 11298, + "weir": 11299, + "discussed": 11300, + "badge": 11301, + "pup": 11302, + "mistake": 11303, + "phenomen": 11304, + "unite": 11305, + "ðŁĽ": 11306, + "depic": 11307, + "rides": 11308, + "inaugu": 11309, + "nat": 11310, + "softwitter": 11311, + "combination": 11312, + "gospel": 11313, + "âļ¾": 11314, + "admission": 11315, + "retrogaming": 11316, + "ðŁIJ¾": 11317, + "schu": 11318, + "mbo": 11319, + "junction": 11320, + "alarm": 11321, + "à¦": 11322, + "grac": 11323, + "khali": 11324, + "kul": 11325, + "male": 11326, + "caption": 11327, + "wish": 11328, + "tere": 11329, + "corps": 11330, + "rubber": 11331, + "playstation": 11332, + "erin": 11333, + "efficient": 11334, + "lor": 11335, + "jokes": 11336, + "inary": 11337, + "norman": 11338, + "luis": 11339, + "inaugural": 11340, + "ched": 11341, + "âļ½ï¸ı": 11342, + "dip": 11343, + "toe": 11344, + "strat": 11345, + "aac": 11346, + "amu": 11347, + "pier": 11348, + "cott": 11349, + "command": 11350, + "tten": 11351, + "snoo": 11352, + "cube": 11353, + "closes": 11354, + "classical": 11355, + "sword": 11356, + "expression": 11357, + "reaching": 11358, + "napp": 11359, + "cost": 11360, + "affect": 11361, + "rico": 11362, + "gif": 11363, + "breathe": 11364, + "tribe": 11365, + "ortho": 11366, + "hay": 11367, + "lg": 11368, + "fries": 11369, + "nm": 11370, + "hiding": 11371, + "richards": 11372, + "ende": 11373, + "micro": 11374, + "capitol": 11375, + "copy": 11376, + "rom": 11377, + "regime": 11378, + "maryland": 11379, + "taxi": 11380, + "dial": 11381, + "embarra": 11382, + "unbeliev": 11383, + "cht": 11384, + "vs": 11385, + "elimin": 11386, + "odd": 11387, + "penny": 11388, + "soundtrack": 11389, + "lings": 11390, + "transition": 11391, + "remaining": 11392, + "ais": 11393, + "malik": 11394, + "?!?": 11395, + "random": 11396, + "defend": 11397, + "ultra": 11398, + "trum": 11399, + "dancer": 11400, + "stol": 11401, + "drive": 11402, + "aver": 11403, + "roast": 11404, + "definition": 11405, + "sean": 11406, + "excitement": 11407, + "particul": 11408, + "surely": 11409, + "shav": 11410, + "bery": 11411, + "dishes": 11412, + "comm": 11413, + "isol": 11414, + "iam": 11415, + "obli": 11416, + "ghost": 11417, + "hughes": 11418, + "chiefs": 11419, + "bas": 11420, + "conservative": 11421, + "special": 11422, + "femin": 11423, + "shri": 11424, + "nancy": 11425, + "intel": 11426, + "tune": 11427, + "ðŁĩª": 11428, + "joel": 11429, + "ggle": 11430, + "moto": 11431, + "ðŁĺĶ": 11432, + "buck": 11433, + "dag": 11434, + "anticip": 11435, + "montana": 11436, + "guid": 11437, + "frog": 11438, + "ecraft": 11439, + "ope": 11440, + "drives": 11441, + "numer": 11442, + "xy": 11443, + "colorful": 11444, + "wednesdaywisdom": 11445, + "illumin": 11446, + "beyon": 11447, + "inaugur": 11448, + "deeply": 11449, + "prefer": 11450, + "fortune": 11451, + "cooked": 11452, + "tible": 11453, + "âĺķ": 11454, + "sweater": 11455, + "itter": 11456, + "tty": 11457, + "ui": 11458, + "gie": 11459, + "complic": 11460, + "~~": 11461, + "taxes": 11462, + "cups": 11463, + "diverse": 11464, + "samanth": 11465, + "âłĢâłĢ": 11466, + "baking": 11467, + "symp": 11468, + "wai": 11469, + "behalf": 11470, + "mercur": 11471, + "travels": 11472, + "ðŁİīðŁİ": 11473, + "oria": 11474, + "engaged": 11475, + "jumping": 11476, + "retired": 11477, + "naked": 11478, + "puni": 11479, + "speedway": 11480, + "sciences": 11481, + "rehearsal": 11482, + "onym": 11483, + "dyou": 11484, + "plates": 11485, + "rati": 11486, + "krish": 11487, + "jazz": 11488, + "carol": 11489, + "raf": 11490, + "penalty": 11491, + "timeline": 11492, + "ruby": 11493, + "engineers": 11494, + "raf": 11495, + "belle": 11496, + "dose": 11497, + "cheon": 11498, + "escap": 11499, + "meg": 11500, + "rank": 11501, + "ord": 11502, + "megan": 11503, + "merch": 11504, + "eclipse": 11505, + "âĺºï¸ı": 11506, + "pledge": 11507, + "kirk": 11508, + "persi": 11509, + "leicester": 11510, + "sak": 11511, + "wk": 11512, + "safely": 11513, + "yyy": 11514, + "jet": 11515, + "promised": 11516, + "jc": 11517, + "enne": 11518, + "noah": 11519, + "reno": 11520, + "rea": 11521, + "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ": 11522, + "trail": 11523, + "ðŁijĢ": 11524, + "fd": 11525, + "sooo": 11526, + "rimin": 11527, + "wk": 11528, + "า": 11529, + "ial": 11530, + "xox": 11531, + "biscu": 11532, + "dale": 11533, + "fandom": 11534, + "participating": 11535, + "flag": 11536, + "privilege": 11537, + "peach": 11538, + "machine": 11539, + "boston": 11540, + "gross": 11541, + "og": 11542, + "miracle": 11543, + "adoption": 11544, + "uss": 11545, + "monsters": 11546, + "beij": 11547, + "clarke": 11548, + "pushing": 11549, + "praying": 11550, + "aro": 11551, + "dn": 11552, + "ellis": 11553, + "apollo": 11554, + "odds": 11555, + "refugee": 11556, + "tow": 11557, + "bp": 11558, + "ðŁĩ¬ðŁĩ§": 11559, + "hend": 11560, + "appeared": 11561, + "membership": 11562, + "pean": 11563, + "dum": 11564, + "violent": 11565, + "vy": 11566, + "potatoes": 11567, + "aww": 11568, + "greetings": 11569, + "tts": 11570, + "acon": 11571, + "shane": 11572, + "photographed": 11573, + "crab": 11574, + "temperatures": 11575, + "cuba": 11576, + "cfc": 11577, + "welcom": 11578, + "hel": 11579, + "innings": 11580, + "mk": 11581, + "code": 11582, + "knock": 11583, + "grass": 11584, + "swedish": 11585, + "pta": 11586, + "icky": 11587, + "vat": 11588, + "lining": 11589, + "sq": 11590, + "sap": 11591, + "arc": 11592, + "announcing": 11593, + "skins": 11594, + "cityof": 11595, + "bring": 11596, + "cox": 11597, + "gamer": 11598, + "itarian": 11599, + "ida": 11600, + "hd": 11601, + "rosse": 11602, + "sadly": 11603, + "geo": 11604, + "âļ¡ï¸ı": 11605, + "tags": 11606, + "father": 11607, + "change": 11608, + "lance": 11609, + "whiskey": 11610, + "adelaide": 11611, + "tec": 11612, + "stickers": 11613, + "market": 11614, + "classy": 11615, + "badass": 11616, + "florence": 11617, + "liner": 11618, + "frost": 11619, + "kate": 11620, + "acon": 11621, + "scandal": 11622, + "essex": 11623, + "ðŁĺı": 11624, + "vivi": 11625, + "drill": 11626, + "bloggers": 11627, + "recommend": 11628, + "dha": 11629, + "acres": 11630, + "roma": 11631, + "buy": 11632, + "grocer": 11633, + "eria": 11634, + "mahar": 11635, + "ffer": 11636, + "patterns": 11637, + "veri": 11638, + "compu": 11639, + "stev": 11640, + "anga": 11641, + "mentor": 11642, + "doo": 11643, + "itali": 11644, + "cdnpoli": 11645, + "only": 11646, + "conduct": 11647, + "electro": 11648, + "def": 11649, + "whale": 11650, + "preparation": 11651, + "bicycle": 11652, + "viral": 11653, + "turnout": 11654, + "brass": 11655, + "quad": 11656, + "hospitality": 11657, + "packaging": 11658, + "dency": 11659, + "cemetery": 11660, + "aboard": 11661, + "dreaming": 11662, + "picture": 11663, + "tall": 11664, + "invent": 11665, + "admi": 11666, + "oe": 11667, + "temps": 11668, + "quan": 11669, + "fundam": 11670, + "promp": 11671, + "residence": 11672, + "mud": 11673, + "souri": 11674, + "âĦ¢": 11675, + "graffiti": 11676, + "gif": 11677, + "dnd": 11678, + "comp": 11679, + "swar": 11680, + "peeps": 11681, + "palestine": 11682, + "devils": 11683, + "sang": 11684, + "assistance": 11685, + "bike": 11686, + "mississi": 11687, + "interviewed": 11688, + "nephew": 11689, + "drums": 11690, + "vand": 11691, + "gentlemen": 11692, + "nsw": 11693, + "insta": 11694, + "lebanon": 11695, + "eeee": 11696, + "olivia": 11697, + "very": 11698, + "rough": 11699, + "industries": 11700, + "mation": 11701, + "ðŁĺĴ": 11702, + "barrel": 11703, + "nay": 11704, + "pops": 11705, + "modern": 11706, + "illy": 11707, + "arest": 11708, + "onents": 11709, + "protecting": 11710, + "vans": 11711, + "eo": 11712, + "vikings": 11713, + "restaurants": 11714, + "reck": 11715, + "jackie": 11716, + "andrew": 11717, + "willing": 11718, + "heath": 11719, + "citizen": 11720, + "discrimin": 11721, + "à¹Ī": 11722, + "stuart": 11723, + "mys": 11724, + "hip": 11725, + "transp": 11726, + "\"?": 11727, + "tex": 11728, + "sushi": 11729, + "ked": 11730, + "crossed": 11731, + "distur": 11732, + "pedia": 11733, + "fate": 11734, + "somehow": 11735, + "moth": 11736, + "processing": 11737, + "iss": 11738, + "rin": 11739, + "uts": 11740, + "yyc": 11741, + "vert": 11742, + "lgbt": 11743, + "reid": 11744, + "onto": 11745, + "arabia": 11746, + "habitat": 11747, + "==": 11748, + "streak": 11749, + "simpson": 11750, + "addiction": 11751, + "wimble": 11752, + "delivers": 11753, + "challenging": 11754, + "ðŁİ¶": 11755, + "franch": 11756, + "edu": 11757, + "sme": 11758, + "aids": 11759, + "hurst": 11760, + "tham": 11761, + "tarian": 11762, + "remembered": 11763, + "palestinian": 11764, + "fees": 11765, + "trum": 11766, + "sketch": 11767, + "uru": 11768, + "fitting": 11769, + "jesse": 11770, + "ðŁĶ¥ðŁĶ¥": 11771, + "--------": 11772, + "bach": 11773, + "icia": 11774, + "colored": 11775, + "dah": 11776, + "associate": 11777, + "intel": 11778, + "seller": 11779, + "pu": 11780, + "stuffed": 11781, + "acs": 11782, + "bs": 11783, + "shin": 11784, + "cooperation": 11785, + "certificate": 11786, + "abu": 11787, + "ingredients": 11788, + "rev": 11789, + "inge": 11790, + "elder": 11791, + "christian": 11792, + "bundle": 11793, + "thic": 11794, + "dirt": 11795, + "beijing": 11796, + "commit": 11797, + "teddy": 11798, + "edu": 11799, + "today": 11800, + "sfield": 11801, + "wyn": 11802, + "confirms": 11803, + "loo": 11804, + "jv": 11805, + "eness": 11806, + "alpha": 11807, + "virus": 11808, + "arium": 11809, + "grind": 11810, + "bridges": 11811, + "introduction": 11812, + "polls": 11813, + "bacter": 11814, + "zach": 11815, + "terminal": 11816, + "raiders": 11817, + "flavor": 11818, + "zombie": 11819, + "vod": 11820, + "spreading": 11821, + "gameofthrones": 11822, + "efficiency": 11823, + "lately": 11824, + "alem": 11825, + "tweet": 11826, + "crimes": 11827, + "cler": 11828, + "dey": 11829, + "dged": 11830, + "hyun": 11831, + "payments": 11832, + "circus": 11833, + "ðŁĺŃðŁĺŃ": 11834, + "missouri": 11835, + "lub": 11836, + "episodes": 11837, + "cage": 11838, + "pos": 11839, + "matching": 11840, + "tumblr": 11841, + "lined": 11842, + "gest": 11843, + "ambi": 11844, + "narr": 11845, + "ington": 11846, + "regul": 11847, + "blown": 11848, + "isle": 11849, + "coco": 11850, + "ondon": 11851, + "joshua": 11852, + "touring": 11853, + "sma": 11854, + "sausage": 11855, + "bestfriend": 11856, + "boeing": 11857, + "desire": 11858, + "savage": 11859, + "rapper": 11860, + "devo": 11861, + "tear": 11862, + "takeover": 11863, + "cowboys": 11864, + "poker": 11865, + "parag": 11866, + "ppe": 11867, + "hint": 11868, + "wears": 11869, + "seth": 11870, + "roles": 11871, + "lanc": 11872, + "manga": 11873, + "format": 11874, + "flyer": 11875, + "cay": 11876, + "moor": 11877, + "bake": 11878, + "splash": 11879, + "vad": 11880, + "kerala": 11881, + "proceeds": 11882, + "silly": 11883, + "reflection": 11884, + "distr": 11885, + "wid": 11886, + "suit": 11887, + "civic": 11888, + "yankees": 11889, + "byn": 11890, + "migration": 11891, + "distin": 11892, + "orch": 11893, + "femini": 11894, + "qualifying": 11895, + "turi": 11896, + "obe": 11897, + "hundred": 11898, + "crap": 11899, + "wang": 11900, + "mathemat": 11901, + "bure": 11902, + "exposure": 11903, + "ferguson": 11904, + "semester": 11905, + "reserv": 11906, + "plym": 11907, + "ahu": 11908, + "facial": 11909, + "wax": 11910, + "worried": 11911, + "cab": 11912, + "vio": 11913, + "asa": 11914, + "cod": 11915, + "topics": 11916, + "pcs": 11917, + "halo": 11918, + "rescued": 11919, + "horizon": 11920, + "ark": 11921, + "âļª": 11922, + "holly": 11923, + "elf": 11924, + "ulti": 11925, + "pup": 11926, + "qualified": 11927, + "attendance": 11928, + "atively": 11929, + "destroy": 11930, + "yc": 11931, + "forth": 11932, + "photooftheday": 11933, + "cents": 11934, + "iceland": 11935, + "measures": 11936, + "desk": 11937, + "portfolio": 11938, + "articles": 11939, + "directors": 11940, + "datab": 11941, + "ew": 11942, + "creepy": 11943, + "ounding": 11944, + "honoured": 11945, + "mist": 11946, + "jit": 11947, + "mentioned": 11948, + "portable": 11949, + "itic": 11950, + "dann": 11951, + "fridayfeeling": 11952, + "amid": 11953, + "tiger": 11954, + "scrip": 11955, + "helicopter": 11956, + "hardware": 11957, + "explor": 11958, + "workplace": 11959, + "austria": 11960, + "beatles": 11961, + "bernar": 11962, + "spider": 11963, + "disco": 11964, + "cult": 11965, + "limits": 11966, + "shortly": 11967, + "final": 11968, + "ninja": 11969, + "luke": 11970, + "lebron": 11971, + "walmart": 11972, + "oil": 11973, + "vanilla": 11974, + "shire": 11975, + "yeg": 11976, + "aky": 11977, + "cs": 11978, + "bler": 11979, + "collected": 11980, + "tg": 11981, + "rolled": 11982, + "specials": 11983, + "bff": 11984, + "pierre": 11985, + "shim": 11986, + "vier": 11987, + "flashback": 11988, + "restoration": 11989, + "individuals": 11990, + "prod": 11991, + "freaking": 11992, + "turer": 11993, + "oa": 11994, + "refre": 11995, + "moroc": 11996, + "greet": 11997, + "reyn": 11998, + "careful": 11999, + "ouring": 12000, + "ush": 12001, + "isd": 12002, + "gill": 12003, + "view": 12004, + "thunderstorm": 12005, + "bled": 12006, + "picnic": 12007, + "guardi": 12008, + "pig": 12009, + "ark": 12010, + "sylvania": 12011, + "banned": 12012, + "ucl": 12013, + "vijay": 12014, + "orium": 12015, + "avengers": 12016, + "believes": 12017, + "eur": 12018, + "monument": 12019, + "concerned": 12020, + "labs": 12021, + "berg": 12022, + "aap": 12023, + "vish": 12024, + "singles": 12025, + "cancel": 12026, + "zel": 12027, + "arab": 12028, + "ruth": 12029, + "tooth": 12030, + "arta": 12031, + "shaf": 12032, + "chairs": 12033, + "rack": 12034, + "diseases": 12035, + "crowd": 12036, + "cly": 12037, + "flex": 12038, + "christma": 12039, + "artificial": 12040, + "tomat": 12041, + "fine": 12042, + "draws": 12043, + "advocate": 12044, + "france": 12045, + "ÙĬ": 12046, + "ðŁĺ³": 12047, + "heavy": 12048, + "sour": 12049, + "comprehen": 12050, + "noble": 12051, + "aap": 12052, + "hindu": 12053, + "coral": 12054, + "gars": 12055, + "owen": 12056, + "nl": 12057, + "stall": 12058, + "yellow": 12059, + "marina": 12060, + "inver": 12061, + "support": 12062, + "tough": 12063, + "promises": 12064, + "pie": 12065, + "masterpiece": 12066, + "score": 12067, + "force": 12068, + "mortg": 12069, + "cryptocurrency": 12070, + "ox": 12071, + "rors": 12072, + "rockin": 12073, + "provin": 12074, + "hog": 12075, + "nostal": 12076, + "oakland": 12077, + "patrick": 12078, + "inclusion": 12079, + "traffic": 12080, + "ahmed": 12081, + "aha": 12082, + "luxury": 12083, + "consecu": 12084, + "demon": 12085, + "âĸº": 12086, + "blowing": 12087, + "stag": 12088, + ":\"": 12089, + "encourage": 12090, + "bene": 12091, + "skull": 12092, + "dodge": 12093, + "buster": 12094, + "kinson": 12095, + "witne": 12096, + "error": 12097, + "lowest": 12098, + "fellow": 12099, + "à°": 12100, + "shre": 12101, + "blur": 12102, + "virgin": 12103, + "composer": 12104, + "slip": 12105, + "mornings": 12106, + "gains": 12107, + "table": 12108, + "grain": 12109, + "arist": 12110, + "brazilian": 12111, + "wwe": 12112, + "tues": 12113, + "ribbon": 12114, + "anag": 12115, + "dist": 12116, + "sacrif": 12117, + "embrace": 12118, + "entrepreneur": 12119, + "affili": 12120, + "deo": 12121, + "tali": 12122, + "tourist": 12123, + "fatal": 12124, + "ìĬ": 12125, + "automatic": 12126, + "ðŁĩµ": 12127, + "weak": 12128, + "welfare": 12129, + "confirm": 12130, + "benjamin": 12131, + "fights": 12132, + "alleged": 12133, + "mead": 12134, + "struggling": 12135, + "prosecu": 12136, + "chef": 12137, + "è": 12138, + "proposal": 12139, + "ern": 12140, + "ðŁĺĦ": 12141, + "dyk": 12142, + "ongs": 12143, + "hong": 12144, + "mack": 12145, + "melon": 12146, + "onent": 12147, + "rush": 12148, + "dap": 12149, + "toler": 12150, + "propag": 12151, + "cze": 12152, + "translation": 12153, + "wallet": 12154, + "cottage": 12155, + "sail": 12156, + "constitution": 12157, + "ðŁĴĢ": 12158, + "munici": 12159, + "favor": 12160, + "stormhour": 12161, + "ih": 12162, + "ðŁĺĮ": 12163, + "approaching": 12164, + "pinned": 12165, + "jed": 12166, + "nigerian": 12167, + "nach": 12168, + "shat": 12169, + "particularly": 12170, + "mcdon": 12171, + "cameras": 12172, + "annie": 12173, + "administr": 12174, + "heat": 12175, + "electrical": 12176, + "charming": 12177, + "gibson": 12178, + "boutique": 12179, + "exposed": 12180, + "actor": 12181, + "pillow": 12182, + "beaches": 12183, + "genuine": 12184, + "margaret": 12185, + "bennett": 12186, + "louisi": 12187, + "positions": 12188, + "ely": 12189, + "shiny": 12190, + "tention": 12191, + "architect": 12192, + "rental": 12193, + "acqui": 12194, + "google": 12195, + "subway": 12196, + "moment": 12197, + "ðŁļ¨": 12198, + "rim": 12199, + "methods": 12200, + "cycli": 12201, + "norfolk": 12202, + "ÙĪ": 12203, + "overwhel": 12204, + "rapid": 12205, + "wear": 12206, + "happybirthday": 12207, + "progressive": 12208, + "ðŁĴ¥": 12209, + "cogn": 12210, + "papa": 12211, + "fool": 12212, + "philosophy": 12213, + "polar": 12214, + "jimmy": 12215, + "wig": 12216, + "ðŁĴĭ": 12217, + "operating": 12218, + "reduction": 12219, + "phi": 12220, + "flags": 12221, + "tothe": 12222, + "odi": 12223, + "ares": 12224, + "koo": 12225, + "kang": 12226, + "arkansas": 12227, + "ashton": 12228, + "wimbledon": 12229, + "scifi": 12230, + "attractive": 12231, + "mississippi": 12232, + "logists": 12233, + "ralph": 12234, + "label": 12235, + "graduates": 12236, + "maha": 12237, + "hometown": 12238, + "âľĮï¸ı": 12239, + "founded": 12240, + "onthe": 12241, + "liz": 12242, + "transl": 12243, + "minimum": 12244, + "presti": 12245, + "tam": 12246, + "generations": 12247, + "rebel": 12248, + "journalists": 12249, + "param": 12250, + "mcm": 12251, + "acrylic": 12252, + "deaths": 12253, + "tesla": 12254, + "wt": 12255, + "bryant": 12256, + "jerus": 12257, + "istanbul": 12258, + "muhammad": 12259, + "riley": 12260, + "kris": 12261, + "workshops": 12262, + "iso": 12263, + "counts": 12264, + "stret": 12265, + "protected": 12266, + "trinity": 12267, + "manual": 12268, + "rhin": 12269, + "ril": 12270, + "pleasant": 12271, + "lemon": 12272, + "nerd": 12273, + "harder": 12274, + "darren": 12275, + "bury": 12276, + "rah": 12277, + "basis": 12278, + "migu": 12279, + "occasion": 12280, + "lists": 12281, + "âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı": 12282, + "eb": 12283, + "decre": 12284, + "hampton": 12285, + "ìĿ´": 12286, + "travis": 12287, + "transform": 12288, + "puerto": 12289, + "nhl": 12290, + "avoc": 12291, + "trips": 12292, + "unexpected": 12293, + "vet": 12294, + "didyou": 12295, + "barber": 12296, + "stages": 12297, + "mson": 12298, + "represented": 12299, + "fort": 12300, + "lal": 12301, + "pple": 12302, + "nicely": 12303, + "ignore": 12304, + "quil": 12305, + "quinn": 12306, + "hk": 12307, + "carrier": 12308, + "reminded": 12309, + "among": 12310, + "passenger": 12311, + "ellen": 12312, + "guez": 12313, + "scape": 12314, + "mural": 12315, + "youngest": 12316, + "mash": 12317, + "dill": 12318, + "routine": 12319, + "stainless": 12320, + "jackson": 12321, + "gandhi": 12322, + "thal": 12323, + "oners": 12324, + "editorial": 12325, + "conversations": 12326, + "sdale": 12327, + "automation": 12328, + "ike": 12329, + "าà¸": 12330, + "ðŁĩª": 12331, + "haul": 12332, + "laying": 12333, + "mentions": 12334, + "amen": 12335, + "abortion": 12336, + "ibi": 12337, + "counties": 12338, + "catherine": 12339, + "mands": 12340, + "jame": 12341, + "roller": 12342, + "aut": 12343, + "nam": 12344, + "ological": 12345, + "ception": 12346, + "ranking": 12347, + "toxic": 12348, + "snacks": 12349, + "victorian": 12350, + "bangkok": 12351, + "psychology": 12352, + "reg": 12353, + "angela": 12354, + "respond": 12355, + "style": 12356, + "sophie": 12357, + "dakota": 12358, + "achieved": 12359, + "marked": 12360, + "imperial": 12361, + "inas": 12362, + "gloves": 12363, + "slim": 12364, + "confident": 12365, + "attacked": 12366, + "gger": 12367, + "lonely": 12368, + "valentinesday": 12369, + "reb": 12370, + "craftbeer": 12371, + "origin": 12372, + "zimbab": 12373, + "ceiling": 12374, + "teens": 12375, + "otherwise": 12376, + "wb": 12377, + "fers": 12378, + "daysof": 12379, + "advisor": 12380, + "yah": 12381, + "âĻª": 12382, + "ender": 12383, + "republicans": 12384, + "ava": 12385, + "skirt": 12386, + "pipel": 12387, + "chie": 12388, + "jane": 12389, + "jax": 12390, + "ðŁĺĭ": 12391, + "âľĬ": 12392, + "jays": 12393, + "brett": 12394, + "balo": 12395, + "crucial": 12396, + "dhar": 12397, + "asis": 12398, + "deau": 12399, + "lloyd": 12400, + "chatting": 12401, + "âĿĦï¸ı": 12402, + "relay": 12403, + "remarkable": 12404, + "ns": 12405, + "wet": 12406, + "brisbane": 12407, + "ðŁĶ´": 12408, + "tionally": 12409, + "fk": 12410, + "layer": 12411, + "household": 12412, + "consecutive": 12413, + "esis": 12414, + "pendant": 12415, + "stir": 12416, + "critic": 12417, + "sugar": 12418, + "photoshop": 12419, + "pares": 12420, + "artistic": 12421, + "dodgers": 12422, + "cun": 12423, + "crafted": 12424, + "amend": 12425, + "boat": 12426, + "âŃIJï¸ı": 12427, + "egyptian": 12428, + "saw": 12429, + "trage": 12430, + "smaller": 12431, + "oxy": 12432, + "paired": 12433, + "next": 12434, + "ires": 12435, + "taco": 12436, + "oy": 12437, + "uc": 12438, + "sti": 12439, + "aerial": 12440, + "://": 12441, + "dro": 12442, + "dotcom": 12443, + "ggins": 12444, + "rpg": 12445, + "aye": 12446, + "lean": 12447, + "striker": 12448, + "lobby": 12449, + "protests": 12450, + "priority": 12451, + "congress": 12452, + "amate": 12453, + "invit": 12454, + "rington": 12455, + "mommy": 12456, + "thus": 12457, + "allowing": 12458, + "pioneer": 12459, + "enforcement": 12460, + "gori": 12461, + "talk": 12462, + "drag": 12463, + "dumb": 12464, + "bullet": 12465, + "sange": 12466, + "ery": 12467, + "targets": 12468, + "ðŁĩ¦": 12469, + "heather": 12470, + "consider": 12471, + "seafood": 12472, + "vest": 12473, + "risks": 12474, + "%.": 12475, + "pg": 12476, + "sacred": 12477, + "heating": 12478, + "kicked": 12479, + "ttot": 12480, + ".-": 12481, + "chandi": 12482, + "coven": 12483, + "pool": 12484, + "pulse": 12485, + "ia": 12486, + "roster": 12487, + "shakespeare": 12488, + "esa": 12489, + "cargo": 12490, + "peanut": 12491, + "troop": 12492, + "action": 12493, + "tablet": 12494, + "homework": 12495, + "castle": 12496, + "struction": 12497, + "musicians": 12498, + "freezing": 12499, + "butt": 12500, + "justinbieber": 12501, + "jj": 12502, + "bahrain": 12503, + "anthem": 12504, + "audit": 12505, + "didyouknow": 12506, + "navig": 12507, + "guidance": 12508, + "âĸ¶": 12509, + "turf": 12510, + "nun": 12511, + "fications": 12512, + "yemen": 12513, + "charging": 12514, + "xc": 12515, + "broncos": 12516, + "subur": 12517, + "pale": 12518, + "boring": 12519, + "amongst": 12520, + "forthe": 12521, + "emper": 12522, + "omfg": 12523, + "pj": 12524, + "expecting": 12525, + "ðŁĴ«": 12526, + "stl": 12527, + "admin": 12528, + "expectations": 12529, + "swan": 12530, + "shoot": 12531, + "ooooo": 12532, + "minent": 12533, + "ãĢIJ": 12534, + "wallace": 12535, + "stang": 12536, + "saturday": 12537, + "adopted": 12538, + "doubles": 12539, + "homie": 12540, + "omez": 12541, + "dhan": 12542, + "venture": 12543, + "surrounding": 12544, + "file": 12545, + "mobility": 12546, + "dees": 12547, + "wski": 12548, + "brooke": 12549, + "embro": 12550, + "remembers": 12551, + "kara": 12552, + "testim": 12553, + "botan": 12554, + "mtv": 12555, + "sacrifice": 12556, + "jerusalem": 12557, + "dl": 12558, + "´": 12559, + "properly": 12560, + "ilion": 12561, + "asi": 12562, + "legit": 12563, + "cope": 12564, + "mcla": 12565, + "recycling": 12566, + "larger": 12567, + "ðŁĴĵ": 12568, + "patric": 12569, + "generous": 12570, + "jared": 12571, + "pf": 12572, + "molly": 12573, + "thomas": 12574, + "judges": 12575, + "hb": 12576, + "sorts": 12577, + "blvd": 12578, + "oven": 12579, + "entering": 12580, + "planes": 12581, + "beet": 12582, + "integration": 12583, + "booked": 12584, + "freed": 12585, + "vern": 12586, + "ashes": 12587, + "topped": 12588, + "depot": 12589, + "welcomed": 12590, + "rena": 12591, + "mick": 12592, + "dand": 12593, + "seeks": 12594, + "gamer": 12595, + "rankings": 12596, + "rene": 12597, + "mut": 12598, + "whisky": 12599, + "firefighters": 12600, + "gues": 12601, + "gather": 12602, + "tourney": 12603, + "demen": 12604, + "yang": 12605, + "newton": 12606, + "automotive": 12607, + "backyard": 12608, + "detailed": 12609, + "mist": 12610, + "tobac": 12611, + "fiber": 12612, + "unusual": 12613, + "gratitude": 12614, + "spare": 12615, + "neys": 12616, + ":*": 12617, + "peri": 12618, + "floating": 12619, + "finalist": 12620, + "donating": 12621, + "dress": 12622, + "broad": 12623, + "bethe": 12624, + "economics": 12625, + "taiwan": 12626, + "edwards": 12627, + "plug": 12628, + "prairi": 12629, + "valen": 12630, + "baba": 12631, + "fad": 12632, + "anas": 12633, + "harper": 12634, + "disorder": 12635, + "applied": 12636, + "patt": 12637, + "bikin": 12638, + "liver": 12639, + "curi": 12640, + "caroline": 12641, + "anner": 12642, + "julian": 12643, + "walking": 12644, + "malcol": 12645, + "screenshot": 12646, + "coding": 12647, + "skincare": 12648, + "activists": 12649, + "mysterious": 12650, + "exact": 12651, + "blocking": 12652, + "mercury": 12653, + "batter": 12654, + "dump": 12655, + "âľĮ": 12656, + "ense": 12657, + "lish": 12658, + "ridiculous": 12659, + "protesters": 12660, + "ðŁĻĪ": 12661, + "lust": 12662, + "sweat": 12663, + "ass": 12664, + "alike": 12665, + "cody": 12666, + "rements": 12667, + "winds": 12668, + "aspir": 12669, + "vienna": 12670, + "pray": 12671, + "...@": 12672, + "boi": 12673, + "candle": 12674, + "assists": 12675, + "tee": 12676, + "derson": 12677, + "pony": 12678, + "fence": 12679, + "conspir": 12680, + "âĺħâĺħ": 12681, + "ooth": 12682, + "epic": 12683, + "barely": 12684, + "aunt": 12685, + "bam": 12686, + "diamonds": 12687, + "endless": 12688, + "screens": 12689, + "cancer": 12690, + "gro": 12691, + "pst": 12692, + "prospec": 12693, + "mosque": 12694, + "helpful": 12695, + "ouri": 12696, + "brother": 12697, + "gujar": 12698, + "cristi": 12699, + "inez": 12700, + "towers": 12701, + "addresses": 12702, + "gray": 12703, + "burton": 12704, + "retweeted": 12705, + "ðŁ¤Ķ": 12706, + "nity": 12707, + "duck": 12708, + "supervis": 12709, + "joan": 12710, + "kinder": 12711, + "sanctu": 12712, + "pied": 12713, + "âı°": 12714, + "łï¸ı": 12715, + "mati": 12716, + "revenge": 12717, + "cester": 12718, + "elife": 12719, + "designers": 12720, + "backed": 12721, + "boli": 12722, + "weight": 12723, + "couch": 12724, + "sures": 12725, + "sits": 12726, + "shrimp": 12727, + "lagos": 12728, + "authorities": 12729, + "osity": 12730, + "holly": 12731, + "computing": 12732, + "factors": 12733, + "abe": 12734, + "panels": 12735, + "ramad": 12736, + "sentence": 12737, + "mission": 12738, + "holm": 12739, + "rb": 12740, + "dads": 12741, + "shanghai": 12742, + "money": 12743, + "sheets": 12744, + "skate": 12745, + "threw": 12746, + "cupcakes": 12747, + "infinite": 12748, + "lis": 12749, + "practicing": 12750, + "essay": 12751, + "kai": 12752, + "asci": 12753, + "mob": 12754, + "ugh": 12755, + "holmes": 12756, + "regg": 12757, + "ikh": 12758, + "mock": 12759, + "collections": 12760, + "pep": 12761, + "ova": 12762, + "salt": 12763, + "nandez": 12764, + "coy": 12765, + "threats": 12766, + "texts": 12767, + "cinnam": 12768, + "pregnancy": 12769, + "pending": 12770, + "stamp": 12771, + "flower": 12772, + "gis": 12773, + "agreed": 12774, + "payne": 12775, + "rover": 12776, + "phra": 12777, + "soft": 12778, + "ffin": 12779, + "fathers": 12780, + "passengers": 12781, + "aways": 12782, + "ala": 12783, + "hes": 12784, + "livan": 12785, + "ins": 12786, + "samuel": 12787, + "ingui": 12788, + "hof": 12789, + "jj": 12790, + "chennai": 12791, + "catal": 12792, + "omic": 12793, + "heath": 12794, + "niece": 12795, + "pumped": 12796, + "integrated": 12797, + "arel": 12798, + "nom": 12799, + "productivity": 12800, + "wanting": 12801, + "visa": 12802, + "diana": 12803, + "twil": 12804, + "itv": 12805, + "camps": 12806, + "rowing": 12807, + "dley": 12808, + "blackand": 12809, + "guards": 12810, + "bells": 12811, + "reverse": 12812, + "vibe": 12813, + "ricky": 12814, + "moss": 12815, + "nyt": 12816, + "âĺĢï¸ı": 12817, + "elle": 12818, + "troy": 12819, + "cudd": 12820, + "evan": 12821, + "womens": 12822, + "foto": 12823, + "mistakes": 12824, + "wicked": 12825, + "mil": 12826, + "cled": 12827, + "memes": 12828, + "cosmo": 12829, + "scholar": 12830, + "reno": 12831, + "ðŁĺĢ": 12832, + "vents": 12833, + "#âĢ¦": 12834, + "terrorists": 12835, + "casey": 12836, + "cardinals": 12837, + "ðŁĺĬðŁĺĬ": 12838, + "venezuela": 12839, + "bola": 12840, + "literacy": 12841, + "tw": 12842, + "eno": 12843, + "contains": 12844, + "austin": 12845, + "financi": 12846, + "evan": 12847, + "harvard": 12848, + "originally": 12849, + "chevro": 12850, + "herald": 12851, + "nottingham": 12852, + "managers": 12853, + "âŀ¡": 12854, + "accepting": 12855, + "walsh": 12856, + "tutorial": 12857, + "entrepreneurship": 12858, + "yacht": 12859, + "requirements": 12860, + "glenn": 12861, + "pede": 12862, + "unfortunately": 12863, + "aching": 12864, + "daisy": 12865, + "gian": 12866, + "nightmare": 12867, + "âĿĹ": 12868, + "rina": 12869, + "bart": 12870, + "emails": 12871, + "opposite": 12872, + "whom": 12873, + "sake": 12874, + "puzzle": 12875, + "dashi": 12876, + "party": 12877, + "blanket": 12878, + "buses": 12879, + "lore": 12880, + "beauty": 12881, + "reason": 12882, + "punjab": 12883, + "windsor": 12884, + "functional": 12885, + "existing": 12886, + "hello": 12887, + "glimp": 12888, + "convin": 12889, + "lak": 12890, + "screaming": 12891, + "rebecca": 12892, + "bliss": 12893, + "northwest": 12894, + "infinity": 12895, + "cosmetics": 12896, + "pulling": 12897, + "coffee": 12898, + "pling": 12899, + "opho": 12900, + "colombia": 12901, + "interiordesign": 12902, + "(+": 12903, + "emotions": 12904, + "sac": 12905, + "sunglasses": 12906, + "saves": 12907, + "df": 12908, + "sixth": 12909, + "aly": 12910, + "ðŁĺ»": 12911, + "deen": 12912, + "devast": 12913, + "politicians": 12914, + "lacrosse": 12915, + "gu": 12916, + "pei": 12917, + "java": 12918, + "combine": 12919, + "coalition": 12920, + "erts": 12921, + "surviv": 12922, + "chad": 12923, + "strian": 12924, + "nn": 12925, + "devi": 12926, + "counc": 12927, + "concern": 12928, + "controller": 12929, + "breast": 12930, + "jury": 12931, + "tum": 12932, + "introduces": 12933, + "ladi": 12934, + "mobile": 12935, + "alz": 12936, + "steady": 12937, + "nurses": 12938, + "hacking": 12939, + "online": 12940, + "ocean": 12941, + "ðŁİĦ": 12942, + "aam": 12943, + "juven": 12944, + "icc": 12945, + "louisiana": 12946, + "arte": 12947, + "streetart": 12948, + "ison": 12949, + "wns": 12950, + "frm": 12951, + "panda": 12952, + "noir": 12953, + "maintain": 12954, + "delay": 12955, + "symptoms": 12956, + "thorn": 12957, + "geome": 12958, + "tern": 12959, + "carried": 12960, + "pru": 12961, + "panor": 12962, + "assy": 12963, + "peru": 12964, + "cloud": 12965, + "spra": 12966, + "pedi": 12967, + "este": 12968, + "tagged": 12969, + "ðŁĺĿ": 12970, + "shadows": 12971, + "nazi": 12972, + "اÙĦ": 12973, + "corri": 12974, + "âĻ¥âĻ¥": 12975, + "jad": 12976, + "ðŁĩ«": 12977, + "formal": 12978, + "spoken": 12979, + "ðŁĮŀ": 12980, + "enjoy": 12981, + "lopez": 12982, + "outlook": 12983, + "inho": 12984, + "wander": 12985, + "Ùħ": 12986, + "maya": 12987, + "pee": 12988, + "dine": 12989, + "ãĢij": 12990, + "briefing": 12991, + "supporter": 12992, + "arily": 12993, + "ghters": 12994, + "naturally": 12995, + "doctorwho": 12996, + "jen": 12997, + "var": 12998, + "newyear": 12999, + "rese": 13000, + "simm": 13001, + "rex": 13002, + "consequ": 13003, + "tomatoes": 13004, + "burst": 13005, + "bravo": 13006, + "burgers": 13007, + "cracking": 13008, + "northeast": 13009, + "biom": 13010, + "mushroom": 13011, + "marque": 13012, + "double": 13013, + "nier": 13014, + "vag": 13015, + "twenty": 13016, + "keyboard": 13017, + "winni": 13018, + "jamaica": 13019, + "parish": 13020, + ":-": 13021, + "mentalhealth": 13022, + "alizing": 13023, + "render": 13024, + "waking": 13025, + "ðŁİĤ": 13026, + "gly": 13027, + "nathan": 13028, + "washing": 13029, + "melissa": 13030, + "jung": 13031, + "loyal": 13032, + "chili": 13033, + "songwriter": 13034, + "guitarist": 13035, + "bowie": 13036, + "neighbors": 13037, + "onymous": 13038, + "asset": 13039, + "tai": 13040, + "headquarters": 13041, + "ðŁĮĪ": 13042, + "ihear": 13043, + "cigare": 13044, + "surg": 13045, + ")\"": 13046, + "repl": 13047, + "darling": 13048, + "ðŁĻĦ": 13049, + "zak": 13050, + "sare": 13051, + "ãħĭ": 13052, + "mickey": 13053, + "warehouse": 13054, + "massage": 13055, + "inees": 13056, + "didnt": 13057, + "iw": 13058, + "hurts": 13059, + "engaging": 13060, + "magic": 13061, + "womenin": 13062, + "kitten": 13063, + "mors": 13064, + "cart": 13065, + "titans": 13066, + "colleague": 13067, + "competing": 13068, + "eran": 13069, + "khal": 13070, + "marble": 13071, + "demand": 13072, + "delight": 13073, + "etary": 13074, + "blizz": 13075, + "louise": 13076, + "mls": 13077, + "finishes": 13078, + "experiment": 13079, + "conducted": 13080, + "electronics": 13081, + "itters": 13082, + "caring": 13083, + "whats": 13084, + "symbol": 13085, + "jung": 13086, + "ecu": 13087, + "pix": 13088, + "context": 13089, + "charger": 13090, + "ðŁĺĩ": 13091, + "reig": 13092, + "frag": 13093, + "ëĭ": 13094, + "chad": 13095, + "true": 13096, + "kerry": 13097, + "defending": 13098, + "aint": 13099, + "auton": 13100, + "checkout": 13101, + "barnes": 13102, + "lessly": 13103, + "dt": 13104, + "mme": 13105, + "cloudy": 13106, + "secondary": 13107, + "arez": 13108, + "_:": 13109, + "appa": 13110, + "constant": 13111, + "\")": 13112, + "vets": 13113, + "job": 13114, + "ient": 13115, + "ðŁĺŃðŁĺŃðŁĺŃ": 13116, + "mj": 13117, + "french": 13118, + "diver": 13119, + "davies": 13120, + "hhhh": 13121, + "ebook": 13122, + "à¹ī": 13123, + "mariti": 13124, + "breeze": 13125, + "suspended": 13126, + "mato": 13127, + "viet": 13128, + "rahu": 13129, + "sei": 13130, + "bolt": 13131, + "enary": 13132, + "leis": 13133, + "karl": 13134, + "framed": 13135, + "explaining": 13136, + "abc": 13137, + "dealing": 13138, + "nato": 13139, + "jake": 13140, + "expand": 13141, + "leonard": 13142, + "established": 13143, + "dub": 13144, + "armen": 13145, + "elled": 13146, + "vocal": 13147, + "nicholas": 13148, + "orient": 13149, + "kyo": 13150, + "illustrated": 13151, + "ahh": 13152, + "dancers": 13153, + "million": 13154, + "geta": 13155, + "popp": 13156, + "asu": 13157, + "murdered": 13158, + "gible": 13159, + "stoked": 13160, + "griffin": 13161, + "maximum": 13162, + "adrian": 13163, + "encounter": 13164, + "thero": 13165, + "davidson": 13166, + "ðŁį»": 13167, + "holiday": 13168, + "evo": 13169, + "assets": 13170, + "carson": 13171, + "memorable": 13172, + "âļ½": 13173, + "obam": 13174, + "representative": 13175, + "cbd": 13176, + "tricks": 13177, + "vogue": 13178, + "voice": 13179, + "mmmm": 13180, + "sebastian": 13181, + "clif": 13182, + "athy": 13183, + "paralle": 13184, + "ðŁ¤·": 13185, + "pak": 13186, + "evacu": 13187, + "eats": 13188, + "اØ": 13189, + "touched": 13190, + "organised": 13191, + "spirits": 13192, + "canad": 13193, + "guided": 13194, + "framework": 13195, + "ðŁĮŁ": 13196, + "ped": 13197, + "natural": 13198, + "agar": 13199, + "replaced": 13200, + "anchor": 13201, + "tit": 13202, + "shah": 13203, + "organis": 13204, + "superior": 13205, + "rn": 13206, + "chro": 13207, + "erica": 13208, + "still": 13209, + "coron": 13210, + "chuck": 13211, + "locks": 13212, + "organ": 13213, + "rosen": 13214, + "scam": 13215, + "bened": 13216, + "/#": 13217, + "keen": 13218, + "trevor": 13219, + "vampire": 13220, + "sorted": 13221, + "!'": 13222, + "afford": 13223, + "intro": 13224, + "grace": 13225, + "ðŁĺľ": 13226, + "saur": 13227, + "kickstarter": 13228, + "influen": 13229, + "vu": 13230, + "yup": 13231, + "poc": 13232, + "ðŁİ¥": 13233, + "aar": 13234, + "sang": 13235, + "trek": 13236, + "etsy": 13237, + "tbh": 13238, + "scream": 13239, + "chevrolet": 13240, + "pixel": 13241, + "shepherd": 13242, + "anor": 13243, + "gabriel": 13244, + "twood": 13245, + "sdcc": 13246, + "meters": 13247, + "developers": 13248, + "closure": 13249, + "vw": 13250, + "twitch": 13251, + "ìĹ": 13252, + "seoul": 13253, + "price": 13254, + "hog": 13255, + "nish": 13256, + "hillary": 13257, + "scratch": 13258, + "incen": 13259, + "wagon": 13260, + "disability": 13261, + "panther": 13262, + "chats": 13263, + "gd": 13264, + "witz": 13265, + "sussex": 13266, + "late": 13267, + "denmark": 13268, + "gerald": 13269, + "cancelled": 13270, + "nette": 13271, + "ix": 13272, + "naval": 13273, + "baptist": 13274, + "tet": 13275, + "yad": 13276, + "math": 13277, + "hoy": 13278, + "randy": 13279, + "point": 13280, + "intellec": 13281, + "fruits": 13282, + "wool": 13283, + "guin": 13284, + "pron": 13285, + "theft": 13286, + "condem": 13287, + "marry": 13288, + "nola": 13289, + "architects": 13290, + "cincin": 13291, + "rockets": 13292, + "gentleman": 13293, + "explan": 13294, + "tate": 13295, + "doe": 13296, + "raises": 13297, + "wildlife": 13298, + "wl": 13299, + "insider": 13300, + "blanc": 13301, + "wp": 13302, + "forsale": 13303, + "nyc": 13304, + "powell": 13305, + "unbelievable": 13306, + "pens": 13307, + "goodies": 13308, + "mustang": 13309, + "pens": 13310, + "stays": 13311, + "squash": 13312, + "xoxo": 13313, + "nearby": 13314, + "everton": 13315, + "coco": 13316, + "leagu": 13317, + "khan": 13318, + "stud": 13319, + "southwest": 13320, + "construc": 13321, + "sworth": 13322, + "croatia": 13323, + "lea": 13324, + "sums": 13325, + "aims": 13326, + "ean": 13327, + "vaness": 13328, + "itious": 13329, + "pathy": 13330, + "arcade": 13331, + "bend": 13332, + "suggests": 13333, + "sacram": 13334, + "royals": 13335, + "rier": 13336, + "emir": 13337, + "incl": 13338, + "ank": 13339, + "clark": 13340, + "right": 13341, + "vacc": 13342, + "ा": 13343, + "tane": 13344, + "lib": 13345, + "usc": 13346, + "sales": 13347, + "huh": 13348, + "sally": 13349, + "vera": 13350, + "pga": 13351, + "grows": 13352, + "drum": 13353, + "tree": 13354, + "ethics": 13355, + "suggest": 13356, + "isab": 13357, + "sealed": 13358, + "previously": 13359, + "animated": 13360, + "abdu": 13361, + "rises": 13362, + "glob": 13363, + "predat": 13364, + "scarf": 13365, + "delic": 13366, + "omar": 13367, + "lli": 13368, + "sxsw": 13369, + "python": 13370, + "nebra": 13371, + "funk": 13372, + "reflect": 13373, + "pavilion": 13374, + "tically": 13375, + "chasing": 13376, + "bakery": 13377, + "invasion": 13378, + "koh": 13379, + "believed": 13380, + "cohen": 13381, + "conqu": 13382, + "crafts": 13383, + "nati": 13384, + "clever": 13385, + "governance": 13386, + "samples": 13387, + "fails": 13388, + "âĶ": 13389, + "timo": 13390, + "ritu": 13391, + "striking": 13392, + "inclusive": 13393, + "shocking": 13394, + "cant": 13395, + "requires": 13396, + "drawings": 13397, + "à¸Ń": 13398, + "purchased": 13399, + "dum": 13400, + "zach": 13401, + "warner": 13402, + "console": 13403, + "mansion": 13404, + "fountain": 13405, + "circum": 13406, + "esh": 13407, + "island": 13408, + "milk": 13409, + "profits": 13410, + "halifax": 13411, + "rival": 13412, + "âľĪï¸ı": 13413, + "jenny": 13414, + "sandra": 13415, + "nye": 13416, + "kelly": 13417, + "yal": 13418, + "quad": 13419, + "nos": 13420, + "instein": 13421, + "finalists": 13422, + "midfielder": 13423, + "cue": 13424, + "exceptional": 13425, + "aan": 13426, + "sapp": 13427, + "gettin": 13428, + "saa": 13429, + "fati": 13430, + "slice": 13431, + "volk": 13432, + "swal": 13433, + "lasting": 13434, + "summary": 13435, + "itas": 13436, + "smo": 13437, + "sz": 13438, + "âĺĨ": 13439, + "ipl": 13440, + "flames": 13441, + "enews": 13442, + "hav": 13443, + "hoodie": 13444, + "pitcher": 13445, + "windy": 13446, + "revol": 13447, + "central": 13448, + "tonite": 13449, + "ðŁİīðŁİī": 13450, + "solved": 13451, + "milwau": 13452, + "organizations": 13453, + "weets": 13454, + "refin": 13455, + "sth": 13456, + "ãĥ¼": 13457, + "elin": 13458, + "tona": 13459, + "cinnamon": 13460, + "ðŁİ¨": 13461, + "ðŁİģ": 13462, + "ronaldo": 13463, + "peninsu": 13464, + "omega": 13465, + "elds": 13466, + "designing": 13467, + "eigh": 13468, + "bluet": 13469, + "benz": 13470, + "nug": 13471, + "asha": 13472, + "robots": 13473, + "sudan": 13474, + "choosing": 13475, + "endo": 13476, + "serge": 13477, + "closely": 13478, + "handy": 13479, + "finger": 13480, + "being": 13481, + "arte": 13482, + "survived": 13483, + "flame": 13484, + "milestone": 13485, + "gut": 13486, + "dwar": 13487, + "futures": 13488, + "ée": 13489, + "elo": 13490, + "fridge": 13491, + "elic": 13492, + "ouch": 13493, + "ub": 13494, + "pv": 13495, + "titan": 13496, + "collar": 13497, + "station": 13498, + "nevada": 13499, + "aurora": 13500, + "rd": 13501, + "duncan": 13502, + "âģł": 13503, + "brien": 13504, + "marsh": 13505, + "о": 13506, + "total": 13507, + "chry": 13508, + "sers": 13509, + "suffe": 13510, + "rachel": 13511, + "college": 13512, + "todays": 13513, + "courts": 13514, + "chit": 13515, + "reunited": 13516, + "gymna": 13517, + "genesis": 13518, + "beside": 13519, + "representation": 13520, + "chant": 13521, + "collector": 13522, + "rak": 13523, + "athens": 13524, + "nigh": 13525, + "munich": 13526, + "languages": 13527, + "flu": 13528, + "participation": 13529, + "___": 13530, + "cv": 13531, + "spectrum": 13532, + "soda": 13533, + "cover": 13534, + "referen": 13535, + "abbo": 13536, + "apa": 13537, + "publication": 13538, + "edm": 13539, + "monica": 13540, + "army": 13541, + "ðŁļĢ": 13542, + "divor": 13543, + "dry": 13544, + "streams": 13545, + "robotics": 13546, + "cider": 13547, + "bullying": 13548, + "approval": 13549, + "stoke": 13550, + "platforms": 13551, + "sierra": 13552, + "extin": 13553, + "ib": 13554, + "hayes": 13555, + "succeed": 13556, + "suffer": 13557, + "atically": 13558, + "dai": 13559, + "lynch": 13560, + "hound": 13561, + "delines": 13562, + "acknow": 13563, + "dated": 13564, + "exclusively": 13565, + "heres": 13566, + "facilit": 13567, + "damaged": 13568, + "charter": 13569, + "lakers": 13570, + "falcon": 13571, + "unveiled": 13572, + "welove": 13573, + "ease": 13574, + "patience": 13575, + "lone": 13576, + "gentle": 13577, + "genetic": 13578, + "producing": 13579, + "gour": 13580, + "shannon": 13581, + "bilities": 13582, + "zimbabwe": 13583, + "pint": 13584, + "daughters": 13585, + "literary": 13586, + "belle": 13587, + "clam": 13588, + "surrounded": 13589, + "kany": 13590, + "neil": 13591, + "pirate": 13592, + "ranger": 13593, + "hbd": 13594, + "natalie": 13595, + "belong": 13596, + "olympi": 13597, + "embassy": 13598, + "scol": 13599, + "ener": 13600, + "akin": 13601, + "loren": 13602, + "bh": 13603, + ":/": 13604, + "diva": 13605, + "denim": 13606, + "hipp": 13607, + "ðŁĩµðŁĩ": 13608, + "arnold": 13609, + "?'": 13610, + "weren": 13611, + "empower": 13612, + "disabled": 13613, + "manor": 13614, + "raspberry": 13615, + "baf": 13616, + "awful": 13617, + "drummer": 13618, + "kardashi": 13619, + "nash": 13620, + "machinelearning": 13621, + "chu": 13622, + "rebels": 13623, + "timing": 13624, + "monroe": 13625, + "tongue": 13626, + "range": 13627, + "pupils": 13628, + "ress": 13629, + "amazon": 13630, + "bz": 13631, + "harley": 13632, + "palmer": 13633, + "balloon": 13634, + "sings": 13635, + "icec": 13636, + "jb": 13637, + "cers": 13638, + "gps": 13639, + "whist": 13640, + "rise": 13641, + "lt": 13642, + "oooo": 13643, + "cattle": 13644, + "shooter": 13645, + "vodka": 13646, + "ucl": 13647, + "mtg": 13648, + "lesli": 13649, + "jonas": 13650, + "dispo": 13651, + "atric": 13652, + "stein": 13653, + "vintage": 13654, + "firms": 13655, + "floyd": 13656, + "cowboy": 13657, + "soooo": 13658, + "isaac": 13659, + "warcraft": 13660, + "disneyland": 13661, + "beautiful": 13662, + "beam": 13663, + "franchise": 13664, + "bun": 13665, + "kag": 13666, + "anon": 13667, + "turbo": 13668, + "sweep": 13669, + "madein": 13670, + "karachi": 13671, + "detective": 13672, + "pennsylvania": 13673, + "controversi": 13674, + "vitamin": 13675, + "aside": 13676, + "chronic": 13677, + "describes": 13678, + "removal": 13679, + "hah": 13680, + "aper": 13681, + "tened": 13682, + "uto": 13683, + "badly": 13684, + "mirac": 13685, + "fry": 13686, + "yea": 13687, + "injec": 13688, + "thermal": 13689, + "compact": 13690, + "thor": 13691, + "teed": 13692, + "urgent": 13693, + "lite": 13694, + "gilli": 13695, + "sophom": 13696, + "ico": 13697, + "chem": 13698, + "pm": 13699, + "fork": 13700, + "freak": 13701, + "chak": 13702, + "recipient": 13703, + "iy": 13704, + "nik": 13705, + "modeling": 13706, + "cans": 13707, + "ðŁıĢ": 13708, + "delux": 13709, + "seam": 13710, + "survivors": 13711, + "radical": 13712, + "investigating": 13713, + "reliable": 13714, + "fm": 13715, + "turt": 13716, + "lighthouse": 13717, + "tool": 13718, + "gown": 13719, + "))": 13720, + "bots": 13721, + "autograph": 13722, + "aid": 13723, + "buffe": 13724, + "hmm": 13725, + "horrible": 13726, + "ssional": 13727, + "anni": 13728, + "à¹Ģ": 13729, + "kits": 13730, + "schi": 13731, + "eternal": 13732, + "huss": 13733, + "sensitive": 13734, + "ru": 13735, + "tastes": 13736, + "checks": 13737, + "imo": 13738, + "portion": 13739, + "skate": 13740, + "eden": 13741, + "halftime": 13742, + "fried": 13743, + "rihanna": 13744, + "tise": 13745, + "flick": 13746, + "cain": 13747, + "sgt": 13748, + "âľĶ": 13749, + "shau": 13750, + "stained": 13751, + "raffle": 13752, + "drove": 13753, + "salman": 13754, + "principles": 13755, + "sho": 13756, + "aru": 13757, + "jess": 13758, + "guine": 13759, + "garbage": 13760, + "myan": 13761, + "jelly": 13762, + "disru": 13763, + "zia": 13764, + "qld": 13765, + "entries": 13766, + "lav": 13767, + "flew": 13768, + "admit": 13769, + "objects": 13770, + "compare": 13771, + "nytimes": 13772, + "cannes": 13773, + "pn": 13774, + "suffol": 13775, + "roc": 13776, + "dana": 13777, + "egg": 13778, + "hist": 13779, + "counsel": 13780, + "'!": 13781, + "physi": 13782, + "imagination": 13783, + "adjust": 13784, + "explosion": 13785, + "plymouth": 13786, + "horror": 13787, + "elliott": 13788, + "bourne": 13789, + "dex": 13790, + "breed": 13791, + "audio": 13792, + "lobster": 13793, + "disappointed": 13794, + "nationwide": 13795, + "((": 13796, + "increases": 13797, + "australi": 13798, + "cedar": 13799, + "staring": 13800, + "racial": 13801, + "eis": 13802, + "gmt": 13803, + "visions": 13804, + "stayed": 13805, + "discussions": 13806, + "dean": 13807, + "curtis": 13808, + "maiden": 13809, + "stellar": 13810, + "happiest": 13811, + "hwy": 13812, + "preseason": 13813, + "carav": 13814, + "mondays": 13815, + "hospitals": 13816, + "glimpse": 13817, + "scholars": 13818, + "jai": 13819, + "terrace": 13820, + "anna": 13821, + "goose": 13822, + "graded": 13823, + "lotus": 13824, + "hung": 13825, + "grocery": 13826, + "stamps": 13827, + "emperor": 13828, + "scoop": 13829, + "inser": 13830, + "cas": 13831, + "existence": 13832, + "heal": 13833, + "falcons": 13834, + "marvel": 13835, + "reducing": 13836, + "terrific": 13837, + "magnetic": 13838, + "performs": 13839, + "barre": 13840, + "pus": 13841, + "treating": 13842, + "icon": 13843, + "wh": 13844, + "declared": 13845, + "trauma": 13846, + "dod": 13847, + "comedian": 13848, + "nikon": 13849, + "bugs": 13850, + "asm": 13851, + "montgom": 13852, + "ibiza": 13853, + "comprehensive": 13854, + "has": 13855, + "santi": 13856, + "fellowship": 13857, + "dash": 13858, + "psal": 13859, + "louisville": 13860, + "spy": 13861, + "fault": 13862, + "dthe": 13863, + "filed": 13864, + "vista": 13865, + "desc": 13866, + "fears": 13867, + "youtu": 13868, + "sps": 13869, + "esp": 13870, + "rig": 13871, + "crime": 13872, + "berger": 13873, + "wonderland": 13874, + "kent": 13875, + "informed": 13876, + "stevens": 13877, + "myth": 13878, + "aston": 13879, + "iri": 13880, + "visitor": 13881, + "atri": 13882, + "producers": 13883, + "alla": 13884, + "personally": 13885, + "separate": 13886, + "agencies": 13887, + "afri": 13888, + "ilan": 13889, + "spoke": 13890, + "nina": 13891, + "squad": 13892, + "dives": 13893, + "depend": 13894, + "liv": 13895, + "fierce": 13896, + "entertaining": 13897, + "chain": 13898, + "scat": 13899, + "borders": 13900, + "palette": 13901, + "spro": 13902, + "osis": 13903, + "derby": 13904, + "tobacco": 13905, + "zio": 13906, + "willie": 13907, + "juvent": 13908, + "zoom": 13909, + "holy": 13910, + "entirely": 13911, + "afe": 13912, + "martinez": 13913, + "beds": 13914, + "pea": 13915, + "bulldogs": 13916, + "ðŁĩªðŁĩ": 13917, + "ibm": 13918, + "neon": 13919, + "ethiopia": 13920, + "teammates": 13921, + "planting": 13922, + "twer": 13923, + "anytime": 13924, + "forbes": 13925, + "ón": 13926, + "runway": 13927, + "nervous": 13928, + "roger": 13929, + "pile": 13930, + "chanc": 13931, + "apocaly": 13932, + "uw": 13933, + "oi": 13934, + "drought": 13935, + "territory": 13936, + "brick": 13937, + "creatures": 13938, + "goin": 13939, + "waff": 13940, + "gren": 13941, + "southeast": 13942, + "jean": 13943, + "ambul": 13944, + "edited": 13945, + "strap": 13946, + "cv": 13947, + "aaron": 13948, + "ãĥ»ãĥ»": 13949, + "tsu": 13950, + "description": 13951, + "kindly": 13952, + "clutch": 13953, + "immer": 13954, + "enor": 13955, + "womensday": 13956, + "orange": 13957, + "rag": 13958, + "obvious": 13959, + "hyder": 13960, + "channels": 13961, + "mango": 13962, + "meyer": 13963, + "raining": 13964, + "getty": 13965, + "pilgri": 13966, + "coordinator": 13967, + "upload": 13968, + "nintendo": 13969, + "donuts": 13970, + "sanchez": 13971, + "apparel": 13972, + "jr": 13973, + "zzi": 13974, + ",@": 13975, + "jefferson": 13976, + "accessible": 13977, + "greatly": 13978, + "eid": 13979, + "initial": 13980, + "buddha": 13981, + "paris": 13982, + "mascot": 13983, + "â¬ĩï¸ı": 13984, + "schwar": 13985, + "siri": 13986, + "spinning": 13987, + "mortgage": 13988, + "echo": 13989, + "endange": 13990, + "gedly": 13991, + "chloe": 13992, + "enhance": 13993, + "karnat": 13994, + "kry": 13995, + "explores": 13996, + "ðŁĴģ": 13997, + "affair": 13998, + "icals": 13999, + "alla": 14000, + "dart": 14001, + "dolphins": 14002, + "differences": 14003, + "squirrel": 14004, + "augh": 14005, + "drones": 14006, + "ellen": 14007, + "restore": 14008, + "paw": 14009, + "unfor": 14010, + "pike": 14011, + "hilton": 14012, + "collab": 14013, + "consumers": 14014, + "coinci": 14015, + "outcomes": 14016, + "ppp": 14017, + "aq": 14018, + "coupon": 14019, + "liest": 14020, + "sims": 14021, + "kho": 14022, + "aves": 14023, + "spoon": 14024, + "pudding": 14025, + "corbyn": 14026, + "haters": 14027, + "exams": 14028, + "slave": 14029, + ".!": 14030, + "psa": 14031, + "apples": 14032, + "tamil": 14033, + "sed": 14034, + "coke": 14035, + "zzo": 14036, + "losange": 14037, + "carbon": 14038, + "clair": 14039, + "...)": 14040, + "khu": 14041, + "craig": 14042, + "exploration": 14043, + "sanctuary": 14044, + "sue": 14045, + "alway": 14046, + "dementia": 14047, + "wonders": 14048, + "superhero": 14049, + "pakistani": 14050, + "browns": 14051, + "bluetooth": 14052, + "locker": 14053, + "marc": 14054, + "eventu": 14055, + "deluxe": 14056, + "rodriguez": 14057, + "âĿ¤âĿ¤": 14058, + "robb": 14059, + "ðŁĴ¦": 14060, + "linux": 14061, + "tens": 14062, + "intelligent": 14063, + "seed": 14064, + "voter": 14065, + "sler": 14066, + "peaks": 14067, + "intern": 14068, + "teenage": 14069, + "peninsula": 14070, + "handling": 14071, + "tie": 14072, + "cousins": 14073, + "wendy": 14074, + "mee": 14075, + "à¹Ģà¸": 14076, + "dino": 14077, + "ðŁĴ°": 14078, + "ðŁĺĥ": 14079, + "zee": 14080, + "sbury": 14081, + "tragedy": 14082, + "bk": 14083, + "bore": 14084, + "zin": 14085, + "warns": 14086, + "idiot": 14087, + "touching": 14088, + "continental": 14089, + "tacos": 14090, + "safari": 14091, + "washed": 14092, + "podium": 14093, + "morrison": 14094, + "forests": 14095, + "cbc": 14096, + "alon": 14097, + "particular": 14098, + "beads": 14099, + "invented": 14100, + "loch": 14101, + "lighter": 14102, + "wherever": 14103, + "ide": 14104, + "documents": 14105, + "awe": 14106, + "kr": 14107, + "nowhere": 14108, + "miner": 14109, + "stit": 14110, + "rox": 14111, + "contribute": 14112, + "hardy": 14113, + "clan": 14114, + "object": 14115, + "cait": 14116, + "ðŁĴķðŁĴķ": 14117, + "happier": 14118, + "vegetables": 14119, + "tart": 14120, + "gag": 14121, + "nominee": 14122, + "heavily": 14123, + "panic": 14124, + "jd": 14125, + "theresa": 14126, + "atm": 14127, + "uph": 14128, + "sfc": 14129, + "suri": 14130, + "drink": 14131, + "nal": 14132, + "revel": 14133, + "kl": 14134, + "avocado": 14135, + "nomination": 14136, + "madonna": 14137, + "sharon": 14138, + "malcolm": 14139, + "controlled": 14140, + "shers": 14141, + "revival": 14142, + "legislation": 14143, + "shoots": 14144, + "nin": 14145, + "commentary": 14146, + "pros": 14147, + "humanrights": 14148, + "stranger": 14149, + "mitch": 14150, + "pipeline": 14151, + "legally": 14152, + "thu": 14153, + "gilbert": 14154, + "toll": 14155, + "granted": 14156, + "ghs": 14157, + "iranian": 14158, + "refreshing": 14159, + "duk": 14160, + "abi": 14161, + "prime": 14162, + "joseph": 14163, + "mosa": 14164, + "statistics": 14165, + "productions": 14166, + "merry": 14167, + "patel": 14168, + "sax": 14169, + "humanitarian": 14170, + "structures": 14171, + "emissions": 14172, + "towns": 14173, + "freel": 14174, + "stering": 14175, + "ratings": 14176, + "allegedly": 14177, + "cabin": 14178, + "stl": 14179, + "wade": 14180, + "flyers": 14181, + "trim": 14182, + "promising": 14183, + "zu": 14184, + "ballot": 14185, + "comparison": 14186, + "freeze": 14187, + "outer": 14188, + "greatness": 14189, + "assign": 14190, + "snowy": 14191, + "rale": 14192, + "tories": 14193, + "mediter": 14194, + "knock": 14195, + "consultant": 14196, + "cincinnati": 14197, + "analyst": 14198, + "scoo": 14199, + "jews": 14200, + "approxim": 14201, + "pure": 14202, + "portraits": 14203, + "cyrus": 14204, + "ational": 14205, + "loans": 14206, + "acquis": 14207, + "elu": 14208, + "acceptable": 14209, + "union": 14210, + "watercolor": 14211, + "rust": 14212, + "battles": 14213, + "perfu": 14214, + "seasonal": 14215, + "serial": 14216, + "mindset": 14217, + "riot": 14218, + "feld": 14219, + "ennial": 14220, + "closet": 14221, + "priest": 14222, + "tanks": 14223, + "intl": 14224, + "screw": 14225, + "bum": 14226, + "abdul": 14227, + "oux": 14228, + "explained": 14229, + "rica": 14230, + "imaging": 14231, + "lawyers": 14232, + "buried": 14233, + "ãĥ»ãĥ»ãĥ»": 14234, + "earl": 14235, + "âĢķ": 14236, + "lton": 14237, + "restored": 14238, + "stripes": 14239, + "foss": 14240, + "demands": 14241, + "stealing": 14242, + "alexis": 14243, + "mund": 14244, + "aker": 14245, + "urus": 14246, + "wardro": 14247, + "hugs": 14248, + "genre": 14249, + "ego": 14250, + "ÙĦ": 14251, + "participated": 14252, + "babes": 14253, + "banquet": 14254, + "tious": 14255, + "hemi": 14256, + "dsb": 14257, + "lost": 14258, + "milwaukee": 14259, + "jenner": 14260, + "gem": 14261, + "outra": 14262, + "loses": 14263, + "idi": 14264, + "reps": 14265, + "ðŁİ§": 14266, + "regulation": 14267, + "flaw": 14268, + "fang": 14269, + "vibrant": 14270, + "ramp": 14271, + "rains": 14272, + "wellbeing": 14273, + "soviet": 14274, + "viewers": 14275, + "depo": 14276, + "libraries": 14277, + "bigo": 14278, + "sery": 14279, + "gill": 14280, + "destruction": 14281, + "coz": 14282, + "cx": 14283, + "bridal": 14284, + "alds": 14285, + "planted": 14286, + "amateur": 14287, + "lud": 14288, + "cheering": 14289, + "showcas": 14290, + "profile": 14291, + "iu": 14292, + "vertical": 14293, + "packers": 14294, + "wizard": 14295, + "skip": 14296, + "slight": 14297, + "beau": 14298, + "airways": 14299, + "much": 14300, + "rera": 14301, + "ðŁĮĬ": 14302, + "absor": 14303, + "patio": 14304, + "packages": 14305, + "sells": 14306, + "mentally": 14307, + "ðŁĺ¢": 14308, + "reynolds": 14309, + "kare": 14310, + "tribun": 14311, + "walt": 14312, + "knit": 14313, + "taste": 14314, + "surrey": 14315, + "bounce": 14316, + "creature": 14317, + "bare": 14318, + "betting": 14319, + "sure": 14320, + "miley": 14321, + "laughs": 14322, + "alore": 14323, + "cyn": 14324, + "tl": 14325, + "artist": 14326, + "annah": 14327, + "warmer": 14328, + "dynamics": 14329, + "lunchtime": 14330, + "maritime": 14331, + "vulnerable": 14332, + "ðŁĴĥ": 14333, + "wolver": 14334, + "durham": 14335, + "constantly": 14336, + "amin": 14337, + "sibl": 14338, + ":@": 14339, + "bullet": 14340, + "kach": 14341, + "angelo": 14342, + "wilder": 14343, + "doom": 14344, + "desktop": 14345, + "lawsuit": 14346, + "kca": 14347, + "henderson": 14348, + "inviting": 14349, + "betty": 14350, + "tawards": 14351, + "rafa": 14352, + "leaked": 14353, + "andi": 14354, + "gems": 14355, + "afl": 14356, + "velo": 14357, + "mediterran": 14358, + "probe": 14359, + "totten": 14360, + "stephanie": 14361, + "snation": 14362, + "combe": 14363, + "qs": 14364, + "overcome": 14365, + "assassin": 14366, + "rav": 14367, + "filip": 14368, + "winnipeg": 14369, + "shil": 14370, + "determined": 14371, + "kas": 14372, + "outre": 14373, + "regret": 14374, + "guides": 14375, + "aaa": 14376, + "ðŁĺĪ": 14377, + "wives": 14378, + "manife": 14379, + "erly": 14380, + "smy": 14381, + "shima": 14382, + "xing": 14383, + "pixel": 14384, + "jacob": 14385, + "accommod": 14386, + "toy": 14387, + "ono": 14388, + "poo": 14389, + "tier": 14390, + "answe": 14391, + "ðŁĴģ": 14392, + "rosa": 14393, + "lease": 14394, + "belongs": 14395, + "thar": 14396, + "eventually": 14397, + "neither": 14398, + "goa": 14399, + "skiing": 14400, + "atra": 14401, + "agh": 14402, + "broadcasting": 14403, + "fury": 14404, + "pyram": 14405, + "dice": 14406, + "volkswag": 14407, + "womens": 14408, + "provider": 14409, + "bombs": 14410, + "missile": 14411, + "whip": 14412, + "dick": 14413, + "norwe": 14414, + "backup": 14415, + "elder": 14416, + "mature": 14417, + "concerts": 14418, + "gious": 14419, + "squee": 14420, + "goodmorning": 14421, + "braves": 14422, + "^_": 14423, + "aussie": 14424, + "luna": 14425, + "males": 14426, + "heck": 14427, + "fortn": 14428, + "romeo": 14429, + "steelers": 14430, + "pn": 14431, + "peer": 14432, + "represents": 14433, + "«": 14434, + "katy": 14435, + "miguel": 14436, + "require": 14437, + "chains": 14438, + "lur": 14439, + "immediate": 14440, + "timber": 14441, + "âĸ¶ï¸ı": 14442, + "advocacy": 14443, + "export": 14444, + "anz": 14445, + "tiffany": 14446, + "author": 14447, + "ðŁİĪ": 14448, + "dudes": 14449, + "chilly": 14450, + "hid": 14451, + "harm": 14452, + "bug": 14453, + "monster": 14454, + "terrier": 14455, + "tuc": 14456, + "storytelling": 14457, + "tak": 14458, + "inti": 14459, + "immigrants": 14460, + "bis": 14461, + "reaches": 14462, + "compassion": 14463, + "johnny": 14464, + "contributions": 14465, + "ðŁIJ¶": 14466, + "mechanical": 14467, + "impression": 14468, + "ranks": 14469, + "kobe": 14470, + "menting": 14471, + "blossom": 14472, + "pablo": 14473, + "builder": 14474, + "bombing": 14475, + "twel": 14476, + "sullivan": 14477, + "omo": 14478, + "pete": 14479, + "demi": 14480, + "kudos": 14481, + "wbb": 14482, + "tgif": 14483, + "massach": 14484, + "neighbor": 14485, + "chefs": 14486, + "engines": 14487, + "pune": 14488, + "gained": 14489, + "phantom": 14490, + "sdays": 14491, + "extend": 14492, + "gran": 14493, + "centers": 14494, + "jacqu": 14495, + "datasci": 14496, + "sleepy": 14497, + "elvis": 14498, + "answered": 14499, + "slot": 14500, + "cony": 14501, + "flexible": 14502, + "tially": 14503, + "letics": 14504, + "%,": 14505, + "andrews": 14506, + "sible": 14507, + "momma": 14508, + "vino": 14509, + "dox": 14510, + "invitational": 14511, + "twilight": 14512, + "jade": 14513, + "illery": 14514, + "johns": 14515, + "fou": 14516, + "pv": 14517, + "--->": 14518, + "breakdown": 14519, + "billion": 14520, + "printer": 14521, + "mond": 14522, + "cbc": 14523, + "maggie": 14524, + "legion": 14525, + "dub": 14526, + "kurt": 14527, + "poor": 14528, + "parenting": 14529, + "regions": 14530, + "bikini": 14531, + "beware": 14532, + "sional": 14533, + "auburn": 14534, + "kidding": 14535, + "amples": 14536, + "span": 14537, + "contempor": 14538, + "cic": 14539, + "habits": 14540, + "ako": 14541, + "prefe": 14542, + "buddies": 14543, + "itz": 14544, + "emily": 14545, + "personnel": 14546, + "mountain": 14547, + "versus": 14548, + "ðŁĺ¬": 14549, + "earning": 14550, + "sink": 14551, + "dari": 14552, + "uu": 14553, + "swin": 14554, + "ister": 14555, + "brutal": 14556, + "nac": 14557, + "kata": 14558, + "cloth": 14559, + "amand": 14560, + "ðŁĶĹ": 14561, + "neo": 14562, + "alumin": 14563, + "weekends": 14564, + "nebraska": 14565, + "codes": 14566, + "delayed": 14567, + "bruno": 14568, + "proven": 14569, + "inc": 14570, + "ight": 14571, + "flan": 14572, + "oro": 14573, + "lambert": 14574, + "regulat": 14575, + "wf": 14576, + "massachuse": 14577, + "kardashian": 14578, + "bernard": 14579, + "fiesta": 14580, + "volcano": 14581, + "grandpa": 14582, + "anca": 14583, + "dre": 14584, + "stitu": 14585, + "meaning": 14586, + "foam": 14587, + "auck": 14588, + "ated": 14589, + "rl": 14590, + "hotel": 14591, + "persons": 14592, + "dynasty": 14593, + "ellor": 14594, + "mai": 14595, + "amne": 14596, + "styling": 14597, + "avier": 14598, + "eg": 14599, + "vegetarian": 14600, + ",âĢ¦": 14601, + "founders": 14602, + "stain": 14603, + "gd": 14604, + "cycles": 14605, + "skyline": 14606, + "tractor": 14607, + "exists": 14608, + "tral": 14609, + "kidney": 14610, + "maril": 14611, + "instag": 14612, + "sette": 14613, + "addict": 14614, + "triangle": 14615, + "flashback": 14616, + "controversial": 14617, + "zon": 14618, + "pins": 14619, + "ias": 14620, + "tray": 14621, + "township": 14622, + "delegates": 14623, + "spam": 14624, + "hms": 14625, + "crane": 14626, + "peoples": 14627, + "olo": 14628, + "faction": 14629, + "butes": 14630, + "onica": 14631, + "delegation": 14632, + "newprofile": 14633, + "elier": 14634, + "mca": 14635, + "wand": 14636, + "gely": 14637, + "losangeles": 14638, + "berke": 14639, + "tive": 14640, + "disrup": 14641, + "zza": 14642, + "casa": 14643, + "jordan": 14644, + "fordshire": 14645, + "gathered": 14646, + "ichi": 14647, + "attendees": 14648, + "à¸Ńà¸": 14649, + "peppers": 14650, + "coin": 14651, + "bourbon": 14652, + "ernity": 14653, + "rotary": 14654, + "behaviour": 14655, + "jeremy": 14656, + "teamwork": 14657, + "compliance": 14658, + "tremend": 14659, + "ðŁĩ§": 14660, + "buhari": 14661, + "cambo": 14662, + "buyers": 14663, + "hagen": 14664, + "buds": 14665, + "bayern": 14666, + "monte": 14667, + "smells": 14668, + "anza": 14669, + "athlon": 14670, + "described": 14671, + "workforce": 14672, + "giving": 14673, + "api": 14674, + "investments": 14675, + "dail": 14676, + "selena": 14677, + "database": 14678, + "thum": 14679, + "mortal": 14680, + "student": 14681, + "buyer": 14682, + "dover": 14683, + "garten": 14684, + "attle": 14685, + "loyalty": 14686, + "genoci": 14687, + "holocau": 14688, + "theaters": 14689, + "ruling": 14690, + "venus": 14691, + "patent": 14692, + "chun": 14693, + "abby": 14694, + "awake": 14695, + "massacre": 14696, + "bangalore": 14697, + "breaking": 14698, + "simmons": 14699, + "justi": 14700, + "hale": 14701, + "edchat": 14702, + "ggles": 14703, + "hawk": 14704, + "marking": 14705, + "headlines": 14706, + "strom": 14707, + "cove": 14708, + "breathtaking": 14709, + "medals": 14710, + "haircut": 14711, + "christine": 14712, + "telegraph": 14713, + "gujarat": 14714, + "jura": 14715, + "cane": 14716, + "shore": 14717, + "propaganda": 14718, + "mueller": 14719, + "........": 14720, + "savi": 14721, + "stomach": 14722, + "throws": 14723, + "tab": 14724, + "warm": 14725, + "jong": 14726, + "renowned": 14727, + "hir": 14728, + "rais": 14729, + "mushrooms": 14730, + "guaranteed": 14731, + "boa": 14732, + "mj": 14733, + "revolutionary": 14734, + "certification": 14735, + "bruins": 14736, + "join": 14737, + "wes": 14738, + "passport": 14739, + "cg": 14740, + "sexu": 14741, + "capable": 14742, + "wv": 14743, + "tones": 14744, + "jackets": 14745, + "accompan": 14746, + "spinach": 14747, + "forever": 14748, + "blair": 14749, + "watts": 14750, + "gl": 14751, + "couples": 14752, + "prairie": 14753, + "newprofilepic": 14754, + "logistics": 14755, + "massachusetts": 14756, + "jaguar": 14757, + "oid": 14758, + "weal": 14759, + "underwater": 14760, + "moz": 14761, + "yi": 14762, + "maths": 14763, + "myanmar": 14764, + "preps": 14765, + "suffered": 14766, + "trace": 14767, + "wali": 14768, + "ahhh": 14769, + "borg": 14770, + "stitch": 14771, + "culin": 14772, + "realise": 14773, + "infection": 14774, + "discrimination": 14775, + "shame": 14776, + "ankle": 14777, + "humid": 14778, + "yt": 14779, + "bracket": 14780, + "truck": 14781, + "triu": 14782, + "easter": 14783, + "community": 14784, + "postcard": 14785, + "involving": 14786, + "tyler": 14787, + "caramel": 14788, + "overview": 14789, + "examples": 14790, + "integrity": 14791, + "basement": 14792, + "instruments": 14793, + "anium": 14794, + "atus": 14795, + "gher": 14796, + "laundry": 14797, + "achieve": 14798, + "geneva": 14799, + "pricing": 14800, + "hyderabad": 14801, + "belief": 14802, + "meta": 14803, + "jaw": 14804, + "accounting": 14805, + "leader": 14806, + "cristiano": 14807, + "couture": 14808, + "cyp": 14809, + "vised": 14810, + ",,,": 14811, + "knu": 14812, + "hick": 14813, + "breaker": 14814, + "bram": 14815, + "rab": 14816, + "moor": 14817, + "hamas": 14818, + "graduating": 14819, + "puppies": 14820, + "akh": 14821, + "tah": 14822, + "aches": 14823, + "rie": 14824, + "opini": 14825, + "gta": 14826, + "reign": 14827, + "tragic": 14828, + "rever": 14829, + "pill": 14830, + "pineapple": 14831, + "touches": 14832, + "dare": 14833, + "leys": 14834, + "ilo": 14835, + "interiors": 14836, + "scouts": 14837, + "bart": 14838, + "enzie": 14839, + "dono": 14840, + "brock": 14841, + "christians": 14842, + "ensemble": 14843, + "·": 14844, + "cinemas": 14845, + "newport": 14846, + "airline": 14847, + "winston": 14848, + "leigh": 14849, + "contents": 14850, + "prescri": 14851, + "urge": 14852, + "trout": 14853, + "fically": 14854, + "ilia": 14855, + "subsi": 14856, + "arer": 14857, + "âļ¾ï¸ı": 14858, + "wounded": 14859, + "ðŁĻĤ": 14860, + "pepper": 14861, + "ðŁĴŀ": 14862, + "fitted": 14863, + "aff": 14864, + "resur": 14865, + "thursdaythoughts": 14866, + "zero": 14867, + "archaeology": 14868, + "div": 14869, + "jee": 14870, + "ion": 14871, + "awaiting": 14872, + "cozy": 14873, + "beauties": 14874, + "bald": 14875, + "data": 14876, + "grizz": 14877, + "stalk": 14878, + "kinds": 14879, + "cleared": 14880, + "jessic": 14881, + "regular": 14882, + "aliens": 14883, + "place": 14884, + "bos": 14885, + "bizar": 14886, + "thisis": 14887, + "ðŁĴĢ": 14888, + "tottenham": 14889, + "mafia": 14890, + "slam": 14891, + "ariana": 14892, + "carroll": 14893, + "backpack": 14894, + "carey": 14895, + "univ": 14896, + "rg": 14897, + "pep": 14898, + "digit": 14899, + "tattoos": 14900, + "agon": 14901, + "volunteering": 14902, + "differen": 14903, + "consumption": 14904, + "kathr": 14905, + "headphones": 14906, + "tshirt": 14907, + "ob": 14908, + "element": 14909, + "retail": 14910, + "shru": 14911, + "algori": 14912, + "container": 14913, + "conscious": 14914, + "fil": 14915, + "coming": 14916, + "rash": 14917, + "urope": 14918, + "define": 14919, + "gior": 14920, + "feminist": 14921, + "flowing": 14922, + "routes": 14923, + "glaci": 14924, + "fert": 14925, + "somerset": 14926, + "antes": 14927, + "tweeps": 14928, + "$$": 14929, + "hour": 14930, + "endangered": 14931, + "yearsof": 14932, + "roh": 14933, + "popped": 14934, + "backing": 14935, + "basil": 14936, + "brake": 14937, + "monaco": 14938, + "lgbtq": 14939, + "prague": 14940, + "utility": 14941, + "cassi": 14942, + "gateway": 14943, + "haunted": 14944, + "schul": 14945, + "ðŁİµ": 14946, + "should": 14947, + "walkingdead": 14948, + "completing": 14949, + "danny": 14950, + "montgomery": 14951, + "penguin": 14952, + "ssi": 14953, + "merchandi": 14954, + "ðŁijij": 14955, + "church": 14956, + "hates": 14957, + "captain": 14958, + "breathing": 14959, + "cet": 14960, + "fairly": 14961, + "approaches": 14962, + "companion": 14963, + "surprising": 14964, + "kanye": 14965, + "pey": 14966, + "hindi": 14967, + "targeted": 14968, + "lords": 14969, + "deut": 14970, + "digging": 14971, + "german": 14972, + "rut": 14973, + "energy": 14974, + "closest": 14975, + "yun": 14976, + "apologi": 14977, + "ั": 14978, + "sack": 14979, + "rup": 14980, + "ddy": 14981, + "portal": 14982, + "dough": 14983, + "bats": 14984, + "ðŁĵ°": 14985, + "atur": 14986, + "grapher": 14987, + "pires": 14988, + "motors": 14989, + "ðŁĮ¹": 14990, + "jc": 14991, + "dang": 14992, + "tuk": 14993, + "clue": 14994, + "usc": 14995, + "page": 14996, + "dless": 14997, + "brows": 14998, + "jus": 14999, + "ading": 15000, + "remarks": 15001, + "oom": 15002, + "cardio": 15003, + "stefan": 15004, + "armstrong": 15005, + "âĢ¢âĢ¢": 15006, + "niest": 15007, + "belgian": 15008, + "biop": 15009, + "soy": 15010, + "lof": 15011, + "íĥ": 15012, + "qt": 15013, + "flashbackfriday": 15014, + "cee": 15015, + "ģà¸": 15016, + "wreck": 15017, + "marines": 15018, + "amendment": 15019, + "wardrobe": 15020, + "voy": 15021, + "burned": 15022, + "guitars": 15023, + "rainf": 15024, + "lifel": 15025, + "ssil": 15026, + "ounce": 15027, + "external": 15028, + "ckey": 15029, + "mesh": 15030, + "sheikh": 15031, + "invitation": 15032, + "suggesti": 15033, + "popcorn": 15034, + "phenomenal": 15035, + "anonymous": 15036, + "tuna": 15037, + "chicago": 15038, + "oval": 15039, + "dely": 15040, + "locals": 15041, + "(&": 15042, + "prof": 15043, + "novel": 15044, + "finder": 15045, + "sparks": 15046, + "laven": 15047, + "infu": 15048, + "nicks": 15049, + "quant": 15050, + "rae": 15051, + "exec": 15052, + "distingui": 15053, + "stances": 15054, + "mutual": 15055, + "shal": 15056, + "unveils": 15057, + "edmonton": 15058, + "zania": 15059, + "adio": 15060, + "viewer": 15061, + "bradford": 15062, + "auditorium": 15063, + "quis": 15064, + "react": 15065, + "http": 15066, + "lero": 15067, + "cheeky": 15068, + "impacts": 15069, + "tak": 15070, + "edt": 15071, + "desperate": 15072, + "tay": 15073, + "ìĦ": 15074, + "settle": 15075, + "bargain": 15076, + "resume": 15077, + "unite": 15078, + "thrown": 15079, + "kest": 15080, + "seys": 15081, + "marching": 15082, + "amit": 15083, + "decline": 15084, + "schar": 15085, + "metr": 15086, + "stanford": 15087, + "linke": 15088, + "berra": 15089, + "dolls": 15090, + "rugby": 15091, + "jami": 15092, + "bor": 15093, + "roadtrip": 15094, + "dinosaur": 15095, + "mik": 15096, + "sunder": 15097, + "rem": 15098, + "bk": 15099, + "overseas": 15100, + "naughty": 15101, + "implementation": 15102, + "iamsrk": 15103, + "luncheon": 15104, + "firing": 15105, + "miami": 15106, + "perez": 15107, + "thee": 15108, + "zon": 15109, + "gifted": 15110, + "conversion": 15111, + "ceramic": 15112, + "¡ï¸ı": 15113, + "pedro": 15114, + "ìĨ": 15115, + "vick": 15116, + "!@": 15117, + "heed": 15118, + "sid": 15119, + "bw": 15120, + "document": 15121, + "plun": 15122, + "grants": 15123, + "fantasy": 15124, + "predictions": 15125, + "valid": 15126, + "carved": 15127, + "graduated": 15128, + "ðŁijįðŁı»": 15129, + "nationally": 15130, + "chy": 15131, + "afl": 15132, + "resso": 15133, + "blank": 15134, + "rivals": 15135, + "jig": 15136, + "eties": 15137, + "omics": 15138, + "unemp": 15139, + "bound": 15140, + "sko": 15141, + "inspection": 15142, + "paral": 15143, + "highs": 15144, + "crisp": 15145, + "bans": 15146, + "oba": 15147, + "[@": 15148, + "cospla": 15149, + "costumes": 15150, + "recall": 15151, + "mouth": 15152, + "nigel": 15153, + "bts": 15154, + "tera": 15155, + "kov": 15156, + "docs": 15157, + "westminster": 15158, + "dict": 15159, + "gravity": 15160, + "kari": 15161, + "rogue": 15162, + "tted": 15163, + "wark": 15164, + "idaho": 15165, + "wend": 15166, + "awi": 15167, + "queensland": 15168, + "processes": 15169, + "cliffe": 15170, + "mick": 15171, + "compens": 15172, + "opol": 15173, + "they": 15174, + "clari": 15175, + "wikipedia": 15176, + "salmankhan": 15177, + "hazard": 15178, + "preston": 15179, + "sweetest": 15180, + "pdf": 15181, + "chees": 15182, + "trilo": 15183, + "southafrica": 15184, + "burnt": 15185, + "($": 15186, + "contain": 15187, + "tp": 15188, + "submitted": 15189, + "soundcloud": 15190, + "atu": 15191, + "rez": 15192, + "wordpress": 15193, + "corrupt": 15194, + "nf": 15195, + "maker": 15196, + "íķ": 15197, + "paras": 15198, + "advent": 15199, + "rial": 15200, + "cafe": 15201, + "fossil": 15202, + "!!!!!!!": 15203, + "cows": 15204, + "cj": 15205, + "spur": 15206, + "institutions": 15207, + "landmark": 15208, + "entit": 15209, + "reut": 15210, + "his": 15211, + "alzheim": 15212, + "wemb": 15213, + "reggae": 15214, + "mosqu": 15215, + "stat": 15216, + "identified": 15217, + "dealer": 15218, + "ream": 15219, + "reland": 15220, + "tension": 15221, + "ðŁĩ©": 15222, + "wrapping": 15223, + "deeper": 15224, + "frat": 15225, + "reddit": 15226, + "aris": 15227, + "morocco": 15228, + "..\"": 15229, + "blow": 15230, + "mapping": 15231, + "priorities": 15232, + "inga": 15233, + "swap": 15234, + "rewards": 15235, + "conspiracy": 15236, + "creative": 15237, + "cj": 15238, + "congressional": 15239, + "vault": 15240, + "plex": 15241, + "sophomore": 15242, + "shadow": 15243, + "eless": 15244, + "ðŁĺħ": 15245, + "darts": 15246, + "aldub": 15247, + "annoying": 15248, + "props": 15249, + "nas": 15250, + "aluminum": 15251, + "hbo": 15252, + "offense": 15253, + "jill": 15254, + "onions": 15255, + "laur": 15256, + "tae": 15257, + "hardest": 15258, + "shro": 15259, + "gaining": 15260, + "measure": 15261, + "edtech": 15262, + "cyprus": 15263, + "tara": 15264, + "angeli": 15265, + "carlo": 15266, + "goon": 15267, + "alli": 15268, + "implic": 15269, + "jupit": 15270, + "resilience": 15271, + "hail": 15272, + "balanced": 15273, + ")...": 15274, + "joyce": 15275, + "gra": 15276, + "theli": 15277, + "defined": 15278, + "shipped": 15279, + "mainly": 15280, + "mina": 15281, + "lm": 15282, + "sacri": 15283, + "ober": 15284, + "pim": 15285, + "claiming": 15286, + "enters": 15287, + "corey": 15288, + "bok": 15289, + "cried": 15290, + "cooling": 15291, + "danielle": 15292, + "pharmacy": 15293, + "thorough": 15294, + "cake": 15295, + "klo": 15296, + "outreach": 15297, + "zens": 15298, + "digitalmarketing": 15299, + "valent": 15300, + "snp": 15301, + "herb": 15302, + "mrw": 15303, + "café": 15304, + "captures": 15305, + "notre": 15306, + "triumph": 15307, + "pancakes": 15308, + "cumber": 15309, + "spike": 15310, + "dation": 15311, + "bigg": 15312, + "sper": 15313, + "critical": 15314, + "amal": 15315, + "tooth": 15316, + "founding": 15317, + "astro": 15318, + "'#": 15319, + "quantum": 15320, + "thames": 15321, + "unc": 15322, + "pride": 15323, + "airbus": 15324, + "knocked": 15325, + "undefeated": 15326, + "mediterranean": 15327, + "calcu": 15328, + "clown": 15329, + "sensor": 15330, + "hammer": 15331, + "forgive": 15332, + "cushi": 15333, + "berry": 15334, + "majestic": 15335, + "elect": 15336, + "politan": 15337, + "gta": 15338, + "kari": 15339, + "burke": 15340, + "seahawks": 15341, + "volkswagen": 15342, + "rei": 15343, + "landscapes": 15344, + "casu": 15345, + "grandfather": 15346, + "listened": 15347, + "//": 15348, + "startrek": 15349, + "rainfall": 15350, + "furry": 15351, + "vier": 15352, + "stark": 15353, + "rifle": 15354, + "ffa": 15355, + "leges": 15356, + "hillaryclinton": 15357, + "minus": 15358, + "correctly": 15359, + "architectural": 15360, + "prece": 15361, + "upside": 15362, + "boxer": 15363, + "ðŁĻĮðŁı¼": 15364, + "isai": 15365, + "det": 15366, + "provo": 15367, + "tissue": 15368, + "spooky": 15369, + "veled": 15370, + "recon": 15371, + "prospects": 15372, + "quebec": 15373, + "âļ«": 15374, + "igno": 15375, + "anatomy": 15376, + "shapes": 15377, + "wp": 15378, + "pinterest": 15379, + "hore": 15380, + "anes": 15381, + "pickup": 15382, + "tip": 15383, + "pradesh": 15384, + "hugh": 15385, + "coe": 15386, + "pok": 15387, + "grammy": 15388, + "wellington": 15389, + "stigate": 15390, + "righ": 15391, + "leap": 15392, + "kingston": 15393, + "scenic": 15394, + "gosh": 15395, + "vani": 15396, + "aug": 15397, + "sary": 15398, + "zier": 15399, + "bureau": 15400, + "linson": 15401, + "conte": 15402, + "fragr": 15403, + "allan": 15404, + "gaw": 15405, + "lana": 15406, + "collision": 15407, + "surveill": 15408, + "renais": 15409, + "arrange": 15410, + "sali": 15411, + "doin": 15412, + "brance": 15413, + "brendan": 15414, + "ourse": 15415, + "incoming": 15416, + "suspension": 15417, + "à´": 15418, + "lla": 15419, + "educators": 15420, + "intri": 15421, + "dae": 15422, + "biography": 15423, + "bulgar": 15424, + "villain": 15425, + "gothic": 15426, + "rwanda": 15427, + "ew": 15428, + "mayor": 15429, + "meetup": 15430, + "democrat": 15431, + "morgan": 15432, + "sudden": 15433, + "tesco": 15434, + "carrot": 15435, + "bomber": 15436, + "mckin": 15437, + "rene": 15438, + "funday": 15439, + "agricultural": 15440, + "hahah": 15441, + "showtime": 15442, + "forming": 15443, + "cola": 15444, + "scorpi": 15445, + "quote": 15446, + "poppy": 15447, + "slife": 15448, + "daz": 15449, + "tub": 15450, + "nen": 15451, + "mot": 15452, + "ðŁĺ»": 15453, + "sore": 15454, + "elderly": 15455, + "ove": 15456, + "skinny": 15457, + "umi": 15458, + "anco": 15459, + "manship": 15460, + "were": 15461, + "gv": 15462, + "kah": 15463, + "folding": 15464, + "neat": 15465, + "samantha": 15466, + "danish": 15467, + "ukrain": 15468, + "humidity": 15469, + "nutri": 15470, + "jakarta": 15471, + "candles": 15472, + "oooooooo": 15473, + "atile": 15474, + "strength": 15475, + "ibra": 15476, + "bapti": 15477, + "charleston": 15478, + "frames": 15479, + "girls": 15480, + "clearing": 15481, + "gluten": 15482, + "##": 15483, + "supernatural": 15484, + "jubi": 15485, + "phone": 15486, + "hein": 15487, + "drun": 15488, + "leak": 15489, + "investor": 15490, + "yer": 15491, + "domain": 15492, + "ballroom": 15493, + "mish": 15494, + "appli": 15495, + "offshore": 15496, + "blaze": 15497, + "doro": 15498, + "âĺķï¸ı": 15499, + "winery": 15500, + "sharif": 15501, + "adore": 15502, + "nir": 15503, + "safer": 15504, + "sigh": 15505, + "ascri": 15506, + "strongly": 15507, + "tracy": 15508, + "cker": 15509, + "oll": 15510, + "faithful": 15511, + "eyed": 15512, + "delightful": 15513, + "vism": 15514, + "karnataka": 15515, + "titan": 15516, + "whar": 15517, + "jerseys": 15518, + "refur": 15519, + "heaven": 15520, + "grip": 15521, + "panama": 15522, + "preli": 15523, + "gluten": 15524, + "odd": 15525, + "content": 15526, + "ponti": 15527, + "tioning": 15528, + "ecommerce": 15529, + "federation": 15530, + "flawless": 15531, + "gear": 15532, + "tires": 15533, + "byr": 15534, + "police": 15535, + "cuban": 15536, + "tributes": 15537, + "ticul": 15538, + "churches": 15539, + "nursery": 15540, + "diaries": 15541, + "museums": 15542, + "snapped": 15543, + "ivan": 15544, + "wight": 15545, + "tourists": 15546, + "ramadan": 15547, + "trent": 15548, + "prophet": 15549, + "wondered": 15550, + "focusing": 15551, + "hid": 15552, + "icons": 15553, + "iq": 15554, + "ambulance": 15555, + "pist": 15556, + "funniest": 15557, + "timeless": 15558, + "srilan": 15559, + "buys": 15560, + "kids": 15561, + "colourful": 15562, + "ashi": 15563, + "chir": 15564, + "mum": 15565, + "ðŁĵļ": 15566, + "letter": 15567, + "xen": 15568, + "reuters": 15569, + "preserve": 15570, + "inting": 15571, + "step": 15572, + "fuji": 15573, + "univer": 15574, + "iu": 15575, + "showdown": 15576, + "poems": 15577, + "surveillance": 15578, + "suspected": 15579, + "tae": 15580, + "solving": 15581, + "tomb": 15582, + "mothersday": 15583, + "carpen": 15584, + "recruit": 15585, + "pilots": 15586, + "broc": 15587, + "mixing": 15588, + "fridays": 15589, + "tyr": 15590, + "representatives": 15591, + "trapped": 15592, + "abdul": 15593, + "freestyle": 15594, + "cluster": 15595, + "âļłï¸ı": 15596, + "kd": 15597, + "skill": 15598, + "pitt": 15599, + "exo": 15600, + "commerci": 15601, + "museum": 15602, + "locally": 15603, + "gina": 15604, + "nobel": 15605, + "immune": 15606, + "frac": 15607, + "capsu": 15608, + "mained": 15609, + "attempts": 15610, + "bulldog": 15611, + "bespoke": 15612, + "singers": 15613, + "spelling": 15614, + "segment": 15615, + "natures": 15616, + "tick": 15617, + "lipstick": 15618, + "cleaner": 15619, + "gettable": 15620, + "precision": 15621, + "âĢ¼ï¸ı": 15622, + "thood": 15623, + "reef": 15624, + "nope": 15625, + "billy": 15626, + "digi": 15627, + "musi": 15628, + "rival": 15629, + "figured": 15630, + "tality": 15631, + "sunny": 15632, + "berk": 15633, + "awww": 15634, + "awaits": 15635, + "unreal": 15636, + "copen": 15637, + "asylum": 15638, + "exotic": 15639, + "buen": 15640, + "mock": 15641, + "enable": 15642, + "archy": 15643, + "fra": 15644, + "plastic": 15645, + "almond": 15646, + "ampli": 15647, + "displays": 15648, + "abbott": 15649, + "sme": 15650, + "xp": 15651, + "ðŁĻĥ": 15652, + "graphic": 15653, + "ived": 15654, + "mara": 15655, + "caution": 15656, + "leaks": 15657, + "enberg": 15658, + "ulu": 15659, + "unicorn": 15660, + "cannon": 15661, + "apprentic": 15662, + "ðŁĺĺðŁĺĺ": 15663, + "bball": 15664, + "willow": 15665, + "atics": 15666, + "amas": 15667, + "manufacturer": 15668, + "campaigns": 15669, + "porters": 15670, + "floors": 15671, + "lsu": 15672, + "type": 15673, + "kej": 15674, + "honorary": 15675, + "itim": 15676, + "tole": 15677, + "minecraft": 15678, + "dx": 15679, + "mash": 15680, + "rio": 15681, + "consequences": 15682, + "ronald": 15683, + "gossi": 15684, + "suffolk": 15685, + "muse": 15686, + "rbi": 15687, + "livemusic": 15688, + "ivan": 15689, + "ðŁİ¤": 15690, + "leu": 15691, + "patriot": 15692, + "manit": 15693, + "lanca": 15694, + "homedecor": 15695, + "dear": 15696, + "sigma": 15697, + "tide": 15698, + "strings": 15699, + "vita": 15700, + "sequel": 15701, + "tryna": 15702, + "investigate": 15703, + "boris": 15704, + "vegan": 15705, + "barrier": 15706, + "mindfulness": 15707, + "webb": 15708, + "hustle": 15709, + "inda": 15710, + "tanzania": 15711, + "stray": 15712, + "texas": 15713, + "cag": 15714, + "diagnosis": 15715, + "woman": 15716, + "gw": 15717, + "obsession": 15718, + "lative": 15719, + "nufc": 15720, + "flynn": 15721, + "momentum": 15722, + "sofa": 15723, + "wald": 15724, + "vegetable": 15725, + "tucker": 15726, + "supper": 15727, + "seab": 15728, + "arro": 15729, + "seag": 15730, + "venting": 15731, + "councill": 15732, + "splat": 15733, + "calcul": 15734, + "..#": 15735, + "comfy": 15736, + "odisha": 15737, + "stopp": 15738, + "warfare": 15739, + "caes": 15740, + "à¨": 15741, + "coy": 15742, + "priceless": 15743, + "insec": 15744, + "ðŁĺĽ": 15745, + "controls": 15746, + "empowerment": 15747, + "datascience": 15748, + "perpe": 15749, + "genic": 15750, + "eres": 15751, + "trudeau": 15752, + "mano": 15753, + "slavery": 15754, + "expanding": 15755, + "mahe": 15756, + "failing": 15757, + "saga": 15758, + "photographs": 15759, + "crest": 15760, + "reon": 15761, + "surfing": 15762, + "hie": 15763, + "ðŁįĢ": 15764, + "jae": 15765, + "fellows": 15766, + "southampton": 15767, + "solom": 15768, + "cester": 15769, + "tability": 15770, + "horn": 15771, + "sect": 15772, + "hee": 15773, + "coleman": 15774, + "atlas": 15775, + "explorer": 15776, + "consultation": 15777, + "copyright": 15778, + "organizing": 15779, + "denied": 15780, + "monkeys": 15781, + "noodles": 15782, + "bris": 15783, + "flor": 15784, + "dough": 15785, + "bonds": 15786, + "shocked": 15787, + "ecosystem": 15788, + "carefully": 15789, + "wm": 15790, + "apartments": 15791, + "curve": 15792, + "sandiego": 15793, + "mustard": 15794, + "commen": 15795, + "ceremon": 15796, + "ech": 15797, + "ruth": 15798, + "ðŁĻĮðŁı»": 15799, + "hawai": 15800, + "filmed": 15801, + "tear": 15802, + "asingly": 15803, + "cair": 15804, + "watt": 15805, + "instrument": 15806, + "outta": 15807, + "yeol": 15808, + "riverside": 15809, + "ë°": 15810, + ".:": 15811, + "norwich": 15812, + "alog": 15813, + "migrants": 15814, + "newman": 15815, + "ride": 15816, + "sprink": 15817, + "targeting": 15818, + "believe": 15819, + "torch": 15820, + "reflects": 15821, + "permission": 15822, + "ffman": 15823, + "enemies": 15824, + "basics": 15825, + "seized": 15826, + "sundays": 15827, + "lei": 15828, + "hassan": 15829, + "endo": 15830, + "hc": 15831, + "stad": 15832, + "lements": 15833, + "kkkk": 15834, + "nano": 15835, + "shark": 15836, + "mana": 15837, + "onic": 15838, + "treatments": 15839, + "early": 15840, + "collaborative": 15841, + "shuttle": 15842, + "branches": 15843, + "misses": 15844, + "mainedcm": 15845, + "apers": 15846, + "kyle": 15847, + "carrie": 15848, + "leisure": 15849, + "shet": 15850, + "birding": 15851, + "advances": 15852, + "ðŁĵĿ": 15853, + "popular": 15854, + "diane": 15855, + "abe": 15856, + "rewar": 15857, + "neighbour": 15858, + "kpop": 15859, + "remembrance": 15860, + "playground": 15861, + "rub": 15862, + "krishna": 15863, + "ebola": 15864, + "inquiry": 15865, + "epa": 15866, + "lumin": 15867, + "organisation": 15868, + "abraham": 15869, + "normally": 15870, + "preten": 15871, + "janet": 15872, + "wt": 15873, + "ðŁĴİ": 15874, + "encouraging": 15875, + "astic": 15876, + "bump": 15877, + "sydney": 15878, + "sz": 15879, + "ssss": 15880, + "garrett": 15881, + "ðŁĵ»": 15882, + "consulting": 15883, + "romania": 15884, + "spotting": 15885, + "chancellor": 15886, + "arma": 15887, + "prestigious": 15888, + "ðĿIJ": 15889, + "tad": 15890, + "cryst": 15891, + "competit": 15892, + "ratio": 15893, + "cataly": 15894, + "brow": 15895, + "jur": 15896, + "viking": 15897, + "commute": 15898, + "yday": 15899, + "layers": 15900, + "dumb": 15901, + "escal": 15902, + "genocide": 15903, + "fill": 15904, + "gupta": 15905, + "stepping": 15906, + "sei": 15907, + "foto": 15908, + "wildcats": 15909, + "coli": 15910, + "project": 15911, + "earnings": 15912, + "str": 15913, + "geons": 15914, + "completion": 15915, + "bm": 15916, + "decorated": 15917, + "crawford": 15918, + "afghan": 15919, + "scare": 15920, + "visibility": 15921, + "hib": 15922, + "direction": 15923, + "stroll": 15924, + "christina": 15925, + "alternate": 15926, + "clare": 15927, + "stylist": 15928, + "behold": 15929, + "sance": 15930, + "leopard": 15931, + "acquired": 15932, + "narrative": 15933, + "ashi": 15934, + "thea": 15935, + "????": 15936, + "peas": 15937, + "atch": 15938, + "slides": 15939, + "leen": 15940, + "renewable": 15941, + "english": 15942, + "quir": 15943, + "coaster": 15944, + "rx": 15945, + "fools": 15946, + "matchday": 15947, + "mism": 15948, + "amazing": 15949, + "zig": 15950, + "keting": 15951, + "wont": 15952, + "towel": 15953, + "diab": 15954, + "stake": 15955, + "nm": 15956, + "melt": 15957, + "ethan": 15958, + "grape": 15959, + "politician": 15960, + "smen": 15961, + "íĺ": 15962, + "reo": 15963, + "weddings": 15964, + "catcher": 15965, + "oracle": 15966, + "memo": 15967, + "ðŁĮ´": 15968, + "eck": 15969, + "robbie": 15970, + "norwegian": 15971, + "operator": 15972, + "amor": 15973, + "sewing": 15974, + "jul": 15975, + "xie": 15976, + "uv": 15977, + "fifty": 15978, + "mega": 15979, + "tattoo": 15980, + "liberals": 15981, + "upri": 15982, + "trafficking": 15983, + "richardson": 15984, + "suv": 15985, + "kip": 15986, + "messy": 15987, + "tremendous": 15988, + "glou": 15989, + "courtney": 15990, + "lad": 15991, + "stereo": 15992, + "myers": 15993, + "idio": 15994, + "^_^": 15995, + "manning": 15996, + "dye": 15997, + "wd": 15998, + "throne": 15999, + "junk": 16000, + "asu": 16001, + "provincial": 16002, + "kook": 16003, + "wrc": 16004, + "fineart": 16005, + "hampshire": 16006, + "renaissance": 16007, + "bred": 16008, + "fallout": 16009, + "sj": 16010, + "snl": 16011, + "alam": 16012, + "torture": 16013, + "fyi": 16014, + "shines": 16015, + "paw": 16016, + "char": 16017, + "henry": 16018, + "crow": 16019, + "acious": 16020, + "dian": 16021, + "paige": 16022, + "bare": 16023, + "stockholm": 16024, + "scenery": 16025, + "ðŁĩ·": 16026, + "jeffrey": 16027, + "push": 16028, + "decoration": 16029, + "ned": 16030, + "cute": 16031, + "brigade": 16032, + "lavender": 16033, + "invites": 16034, + "esports": 16035, + "voir": 16036, + "dried": 16037, + "transpl": 16038, + "surgeon": 16039, + "novels": 16040, + "pulls": 16041, + "sony": 16042, + "lunar": 16043, + "mane": 16044, + "ivy": 16045, + "frustr": 16046, + "dorset": 16047, + "sai": 16048, + "torres": 16049, + "ssion": 16050, + "shutdown": 16051, + "suggestions": 16052, + "writing": 16053, + "eo": 16054, + "battlefield": 16055, + "uga": 16056, + "ðŁIJ¾": 16057, + "vacu": 16058, + "splac": 16059, + "git": 16060, + "ug": 16061, + "highland": 16062, + "%)": 16063, + "mermaid": 16064, + "sacramento": 16065, + "tails": 16066, + "pw": 16067, + "kah": 16068, + "tell": 16069, + "enhanced": 16070, + "ìķ": 16071, + "auckland": 16072, + "cruel": 16073, + "ðŁ¤©": 16074, + "audre": 16075, + "sailor": 16076, + "grammar": 16077, + "glove": 16078, + "deon": 16079, + "inflam": 16080, + "freshly": 16081, + "kell": 16082, + "zip": 16083, + "christie": 16084, + "mild": 16085, + "dixon": 16086, + "instructor": 16087, + "gence": 16088, + "ãħł": 16089, + "subjec": 16090, + "constitutional": 16091, + "crowds": 16092, + "invisible": 16093, + "ruins": 16094, + "dak": 16095, + "sip": 16096, + "plaque": 16097, + "pouring": 16098, + "complex": 16099, + "zine": 16100, + "stead": 16101, + "flet": 16102, + "transmission": 16103, + "loway": 16104, + "arun": 16105, + "increasingly": 16106, + "aud": 16107, + "transparen": 16108, + "crowned": 16109, + "scoun": 16110, + "blizzard": 16111, + "luxu": 16112, + "fiers": 16113, + "achievements": 16114, + "hunters": 16115, + "rocked": 16116, + "basin": 16117, + "violet": 16118, + "proves": 16119, + "achieving": 16120, + "prosper": 16121, + "sega": 16122, + "float": 16123, + "vian": 16124, + "xiv": 16125, + "polic": 16126, + "tura": 16127, + "approximately": 16128, + "wanderlust": 16129, + "keepers": 16130, + "getaway": 16131, + "cod": 16132, + "polis": 16133, + "bryan": 16134, + "colts": 16135, + "talents": 16136, + "yogur": 16137, + "glutenfree": 16138, + "wrist": 16139, + "gry": 16140, + "czech": 16141, + "ðŁİĪ": 16142, + "eville": 16143, + "ðŁıĪ": 16144, + "tox": 16145, + "daniels": 16146, + "amer": 16147, + "bids": 16148, + "weareone": 16149, + "metab": 16150, + "gt": 16151, + "boyz": 16152, + "pdx": 16153, + "possession": 16154, + "pushed": 16155, + "shrine": 16156, + "realistic": 16157, + "trigger": 16158, + "navi": 16159, + "rumors": 16160, + "naf": 16161, + "jenkins": 16162, + "trun": 16163, + "communi": 16164, + "ÃĹ": 16165, + "gamers": 16166, + "armor": 16167, + "mohammed": 16168, + "balcony": 16169, + "yah": 16170, + "strongest": 16171, + "rhythm": 16172, + "unforgettable": 16173, + "kp": 16174, + "hobb": 16175, + "custody": 16176, + "gregor": 16177, + "rita": 16178, + "aesthetic": 16179, + "ilation": 16180, + "sponsoring": 16181, + "nay": 16182, + "kidnapp": 16183, + "shs": 16184, + "rajas": 16185, + "meg": 16186, + "significantly": 16187, + "buttons": 16188, + "lac": 16189, + "versions": 16190, + "essentials": 16191, + "opinions": 16192, + "kro": 16193, + "dprinting": 16194, + "widely": 16195, + "dk": 16196, + "uran": 16197, + "yal": 16198, + "requested": 16199, + "cn": 16200, + "curric": 16201, + "plum": 16202, + "grun": 16203, + "vm": 16204, + "devon": 16205, + "myo": 16206, + "relation": 16207, + "juventus": 16208, + "rouge": 16209, + "minority": 16210, + "mines": 16211, + "jupiter": 16212, + "nine": 16213, + "oxygen": 16214, + "frankie": 16215, + "unesco": 16216, + "fabric": 16217, + "disgusting": 16218, + "salman": 16219, + "detection": 16220, + "lanka": 16221, + "dac": 16222, + "ðŁĩ«ðŁĩ·": 16223, + "argument": 16224, + "shelves": 16225, + "celtics": 16226, + "roberto": 16227, + "pigs": 16228, + "hedge": 16229, + "faul": 16230, + "powering": 16231, + "butterflies": 16232, + "fir": 16233, + "remake": 16234, + "atti": 16235, + "como": 16236, + "empha": 16237, + "kendall": 16238, + "pokemon": 16239, + "seating": 16240, + "dans": 16241, + "baldwin": 16242, + "ðŁij»": 16243, + "leslie": 16244, + "onedirection": 16245, + "timber": 16246, + "iman": 16247, + "font": 16248, + "eder": 16249, + "dion": 16250, + "steph": 16251, + "format": 16252, + "gregory": 16253, + "prop": 16254, + "hex": 16255, + "ruin": 16256, + "sory": 16257, + "infer": 16258, + "naw": 16259, + "barak": 16260, + "sdgs": 16261, + "karao": 16262, + "lush": 16263, + "vander": 16264, + "endent": 16265, + "gis": 16266, + "afro": 16267, + "soccer": 16268, + "ayan": 16269, + "tuni": 16270, + "lung": 16271, + "dayof": 16272, + "alexa": 16273, + "marath": 16274, + "addicted": 16275, + "agile": 16276, + "hygi": 16277, + "lightweight": 16278, + "ì§": 16279, + "mandela": 16280, + "joey": 16281, + "ancy": 16282, + "hum": 16283, + "bir": 16284, + "memorial": 16285, + "jimin": 16286, + "ginger": 16287, + "vak": 16288, + "javascri": 16289, + "crops": 16290, + "origins": 16291, + "dari": 16292, + "piper": 16293, + "import": 16294, + "aggressive": 16295, + "prediction": 16296, + "repairs": 16297, + "cracker": 16298, + "voyage": 16299, + "nike": 16300, + "mummy": 16301, + "linkedin": 16302, + "countryside": 16303, + "border": 16304, + "glass": 16305, + "pert": 16306, + "sals": 16307, + "shoe": 16308, + "autographed": 16309, + "walnut": 16310, + "collegi": 16311, + "salary": 16312, + "pairing": 16313, + "ðŁĮ¸": 16314, + "cathol": 16315, + "sweethe": 16316, + "defeats": 16317, + "strengthen": 16318, + "rooftop": 16319, + "improvements": 16320, + "barriers": 16321, + "uru": 16322, + "tally": 16323, + "ruled": 16324, + "ðŁĨļ": 16325, + "naija": 16326, + "emoji": 16327, + "percent": 16328, + "gio": 16329, + "probs": 16330, + "once": 16331, + "admits": 16332, + "paths": 16333, + "liar": 16334, + "daytona": 16335, + "peters": 16336, + "cali": 16337, + "calli": 16338, + "mug": 16339, + "osa": 16340, + "aph": 16341, + "aby": 16342, + "hyde": 16343, + "ethnic": 16344, + "plains": 16345, + "olf": 16346, + "hahahahaha": 16347, + "holic": 16348, + "?!?!": 16349, + "subli": 16350, + "blacks": 16351, + "mot": 16352, + "ghton": 16353, + "lovin": 16354, + "brent": 16355, + "baru": 16356, + "lati": 16357, + "dew": 16358, + "ateau": 16359, + "qa": 16360, + "painful": 16361, + "busters": 16362, + "static": 16363, + "ðŁĩ¨ðŁĩ¦": 16364, + "notebook": 16365, + "outfits": 16366, + "sies": 16367, + "rf": 16368, + "floods": 16369, + "ÑĢ": 16370, + "throat": 16371, + "suici": 16372, + "rovers": 16373, + "bengal": 16374, + "prepares": 16375, + "blog": 16376, + "miniature": 16377, + "ب": 16378, + "amphi": 16379, + "comb": 16380, + "rsp": 16381, + "intimate": 16382, + "greene": 16383, + "Ìĩ": 16384, + "altar": 16385, + "surgical": 16386, + "vessel": 16387, + "...?": 16388, + "gavin": 16389, + "gator": 16390, + "threatened": 16391, + "zar": 16392, + "robbery": 16393, + "dier": 16394, + "promoted": 16395, + "yg": 16396, + "xs": 16397, + "subs": 16398, + "interviewing": 16399, + "threatening": 16400, + "dozen": 16401, + "meado": 16402, + "waterfall": 16403, + "nintendoswitch": 16404, + "calum": 16405, + "ministers": 16406, + "drop": 16407, + "universities": 16408, + "warned": 16409, + "tactics": 16410, + "ðŁĩ²": 16411, + "refuse": 16412, + "adju": 16413, + "vast": 16414, + "ðŁĺ´": 16415, + "mcfc": 16416, + "libya": 16417, + "nofilter": 16418, + "distributed": 16419, + "reser": 16420, + "ronnie": 16421, + "deco": 16422, + "javascript": 16423, + "monk": 16424, + "interests": 16425, + "flex": 16426, + "martha": 16427, + "sties": 16428, + "ood": 16429, + "ðŁ¤£ðŁ¤£": 16430, + "eun": 16431, + "bali": 16432, + "gomez": 16433, + "stimul": 16434, + "moderate": 16435, + "dity": 16436, + "iris": 16437, + "straw": 16438, + "consistent": 16439, + "directions": 16440, + "adopt": 16441, + "salsa": 16442, + "croo": 16443, + "recovered": 16444, + "blackfriday": 16445, + "lancaster": 16446, + "accept": 16447, + "weareoneexo": 16448, + "builds": 16449, + "freeman": 16450, + "airplane": 16451, + "dition": 16452, + "belong": 16453, + "jamie": 16454, + "pitching": 16455, + "lif": 16456, + "omin": 16457, + "crispy": 16458, + "prepping": 16459, + "veg": 16460, + "chang": 16461, + "accomplished": 16462, + "gracias": 16463, + "dolphin": 16464, + "elector": 16465, + "culinary": 16466, + "superbowl": 16467, + "wala": 16468, + "pursuit": 16469, + "blackberry": 16470, + "bean": 16471, + "cardinal": 16472, + "proved": 16473, + "immigrant": 16474, + "strictly": 16475, + "holocaust": 16476, + "passage": 16477, + "haus": 16478, + "coup": 16479, + "purse": 16480, + "harass": 16481, + "<<": 16482, + "leed": 16483, + "adobe": 16484, + "stad": 16485, + "legislat": 16486, + "parked": 16487, + "priyan": 16488, + "silva": 16489, + "krist": 16490, + "sthe": 16491, + "funky": 16492, + "iga": 16493, + "settlement": 16494, + "phs": 16495, + "tmrw": 16496, + "stressed": 16497, + "hunt": 16498, + "hockey": 16499, + "treasures": 16500, + "chambers": 16501, + "olu": 16502, + "hut": 16503, + "marley": 16504, + "texture": 16505, + "wilderness": 16506, + "mming": 16507, + "potentially": 16508, + "omaha": 16509, + "judy": 16510, + "toes": 16511, + "spoiler": 16512, + "distinguished": 16513, + "felix": 16514, + "ahu": 16515, + "recommendations": 16516, + "zombies": 16517, + "hitler": 16518, + "triple": 16519, + "collapse": 16520, + "motivated": 16521, + "ultimat": 16522, + "ggling": 16523, + "soy": 16524, + "cigar": 16525, + "foren": 16526, + "vineyard": 16527, + "glitter": 16528, + "findings": 16529, + "colonial": 16530, + "hunter": 16531, + "erik": 16532, + "dens": 16533, + "beetle": 16534, + "lotte": 16535, + "subtle": 16536, + "smatter": 16537, + "trusted": 16538, + "experimental": 16539, + "naments": 16540, + "ðŁĺĨ": 16541, + "region": 16542, + "acquisition": 16543, + "breeding": 16544, + "quarterback": 16545, + "amreading": 16546, + "ootd": 16547, + "rude": 16548, + "initiatives": 16549, + "stout": 16550, + "hyung": 16551, + "outcome": 16552, + "alfred": 16553, + "mics": 16554, + "expertise": 16555, + "bacteria": 16556, + "penguins": 16557, + "jumper": 16558, + "valencia": 16559, + "bark": 16560, + "ingday": 16561, + "sellers": 16562, + "contracts": 16563, + "houston": 16564, + "commissioned": 16565, + "adaptation": 16566, + "swansea": 16567, + "santiago": 16568, + "commonwealth": 16569, + "judging": 16570, + "submission": 16571, + "scorer": 16572, + "tommy": 16573, + "ño": 16574, + "exquis": 16575, + "filing": 16576, + "explanation": 16577, + "allison": 16578, + "wembley": 16579, + "ridge": 16580, + "chevy": 16581, + "santos": 16582, + "ownership": 16583, + "cognitive": 16584, + "favourites": 16585, + "shed": 16586, + "philanthro": 16587, + "deleted": 16588, + "godd": 16589, + "snor": 16590, + "guidelines": 16591, + "ffing": 16592, + "jeep": 16593, + "clips": 16594, + "swamp": 16595, + "anor": 16596, + "guild": 16597, + "bolton": 16598, + "springfield": 16599, + "municipal": 16600, + "goalkeeper": 16601, + "yeon": 16602, + "ðŁĺįðŁĺįðŁĺįðŁĺį": 16603, + "ãħĭãħĭ": 16604, + "waterfront": 16605, + "grave": 16606, + "contemporary": 16607, + "arity": 16608, + "ÃŃa": 16609, + "sleeps": 16610, + "syrup": 16611, + "alam": 16612, + "pire": 16613, + "coyo": 16614, + "motogp": 16615, + "tyson": 16616, + "kejri": 16617, + "circul": 16618, + "singly": 16619, + "crunch": 16620, + "complicated": 16621, + "nostalgia": 16622, + "kop": 16623, + "move": 16624, + "kale": 16625, + "macro": 16626, + "midwest": 16627, + "hans": 16628, + "tribal": 16629, + "nude": 16630, + "à¯į": 16631, + "beyonce": 16632, + "congratulate": 16633, + "cater": 16634, + "league": 16635, + "ðŁĻĬ": 16636, + "ladder": 16637, + "crashed": 16638, + "technic": 16639, + "karaoke": 16640, + "harassment": 16641, + "rots": 16642, + "experiencing": 16643, + "kristen": 16644, + "ðŁĩ³": 16645, + "ðŁ¤Ĺ": 16646, + "reflections": 16647, + "guinness": 16648, + "illustrator": 16649, + "ðŁĻıðŁı»": 16650, + "center": 16651, + "narrow": 16652, + "commons": 16653, + "regulations": 16654, + "ÙĨ": 16655, + "harm": 16656, + "croft": 16657, + "cussion": 16658, + "hongkong": 16659, + "stical": 16660, + "internship": 16661, + "zoe": 16662, + "chop": 16663, + "hoods": 16664, + "estimated": 16665, + "batteries": 16666, + "berkeley": 16667, + "smoothie": 16668, + "shaun": 16669, + "cros": 16670, + "~~": 16671, + "campe": 16672, + "hump": 16673, + "bg": 16674, + "prototype": 16675, + "click": 16676, + "shawn": 16677, + "reviewed": 16678, + "templ": 16679, + "pf": 16680, + "jedi": 16681, + "blogs": 16682, + "raymond": 16683, + "asth": 16684, + "bah": 16685, + "avail": 16686, + "scotch": 16687, + "leafs": 16688, + "nikki": 16689, + "tok": 16690, + "hollow": 16691, + "urges": 16692, + "oft": 16693, + "unlike": 16694, + "latin": 16695, + "ue": 16696, + "catering": 16697, + "mili": 16698, + "alternati": 16699, + "maver": 16700, + "и": 16701, + "agle": 16702, + "preorder": 16703, + "lux": 16704, + "cucu": 16705, + "ðŁijıðŁijı": 16706, + "tart": 16707, + "âĿ¤âĿ¤âĿ¤": 16708, + "arabic": 16709, + "rapidly": 16710, + "arrang": 16711, + "allen": 16712, + "traveltuesday": 16713, + "paws": 16714, + "flows": 16715, + "stability": 16716, + "fluid": 16717, + "capp": 16718, + "canberra": 16719, + "uuuu": 16720, + "spani": 16721, + "demonstration": 16722, + "mla": 16723, + "placement": 16724, + "mw": 16725, + "presidents": 16726, + "awesom": 16727, + "beverly": 16728, + "anist": 16729, + "neal": 16730, + "fathersday": 16731, + "referendum": 16732, + "lahore": 16733, + "oaks": 16734, + "debbie": 16735, + "halfway": 16736, + "ghosts": 16737, + "debor": 16738, + "matthews": 16739, + "fiat": 16740, + "tfw": 16741, + "presen": 16742, + "robi": 16743, + "ded": 16744, + "brock": 16745, + "laughed": 16746, + "amounts": 16747, + "bamboo": 16748, + "kindergarten": 16749, + "eaten": 16750, + "mtvhottest": 16751, + "breakout": 16752, + "usic": 16753, + "fraser": 16754, + "legislative": 16755, + "pang": 16756, + "module": 16757, + "sammy": 16758, + "gover": 16759, + "earns": 16760, + "expedition": 16761, + "garh": 16762, + "concepts": 16763, + "charlie": 16764, + "lava": 16765, + "bachelor": 16766, + "veggies": 16767, + "determine": 16768, + "ellie": 16769, + "unlocked": 16770, + "fruit": 16771, + "dalla": 16772, + "coupe": 16773, + "washington": 16774, + "deposit": 16775, + "ivory": 16776, + "paula": 16777, + "chicag": 16778, + "gucci": 16779, + "ðŁİĥ": 16780, + "cultiv": 16781, + "pierce": 16782, + "lifted": 16783, + "stumb": 16784, + "recover": 16785, + "muscles": 16786, + "conducting": 16787, + "cbs": 16788, + "mclaren": 16789, + "sophia": 16790, + "cellu": 16791, + "oceans": 16792, + "uploaded": 16793, + "gameplay": 16794, + "maldives": 16795, + "kimber": 16796, + "avoi": 16797, + "racer": 16798, + "caine": 16799, + "cavs": 16800, + "hana": 16801, + "liga": 16802, + "raven": 16803, + "intervention": 16804, + "inauguration": 16805, + "ooh": 16806, + "attraction": 16807, + "merchandise": 16808, + "tunein": 16809, + "liking": 16810, + "juniors": 16811, + "intended": 16812, + "attacking": 16813, + "aquarium": 16814, + "iwd": 16815, + "components": 16816, + "suring": 16817, + "centu": 16818, + "yogurt": 16819, + "ðŁıĥ": 16820, + "showroom": 16821, + "optical": 16822, + "tyour": 16823, + "judge": 16824, + "yield": 16825, + "anto": 16826, + "plc": 16827, + "transparency": 16828, + "recycled": 16829, + "chief": 16830, + "arom": 16831, + "ambassadors": 16832, + "planet": 16833, + "âĿĦï¸ı": 16834, + "omed": 16835, + "vanessa": 16836, + "court": 16837, + "margar": 16838, + "haley": 16839, + "vr": 16840, + "regina": 16841, + "pdates": 16842, + "hispan": 16843, + "livestream": 16844, + "âģ£": 16845, + "yahoo": 16846, + "galla": 16847, + "secured": 16848, + "wir": 16849, + "beneath": 16850, + "offl": 16851, + "nil": 16852, + "amb": 16853, + "yeg": 16854, + "outlet": 16855, + "ute": 16856, + "peep": 16857, + "lindsay": 16858, + "bentley": 16859, + "...!": 16860, + "heel": 16861, + "trilogy": 16862, + "vos": 16863, + "tyre": 16864, + "therefore": 16865, + "toronto": 16866, + "abi": 16867, + "simpli": 16868, + "jae": 16869, + "extensive": 16870, + "elephants": 16871, + "sor": 16872, + "orientation": 16873, + "impeach": 16874, + "replay": 16875, + "constructed": 16876, + "peterson": 16877, + "pais": 16878, + "ported": 16879, + "customs": 16880, + "collap": 16881, + "adu": 16882, + "highlands": 16883, + "salem": 16884, + "shelby": 16885, + "kovic": 16886, + "strain": 16887, + "rosie": 16888, + "senators": 16889, + "snaps": 16890, + "bobb": 16891, + "suzuki": 16892, + "blades": 16893, + "kp": 16894, + "lolo": 16895, + "generate": 16896, + "sight": 16897, + "mae": 16898, + "structural": 16899, + "predict": 16900, + "jumped": 16901, + "ahmad": 16902, + "sung": 16903, + "justice": 16904, + "glam": 16905, + "volvo": 16906, + "jubilee": 16907, + "detention": 16908, + "losses": 16909, + "puri": 16910, + "everytime": 16911, + "а": 16912, + "rao": 16913, + "edge": 16914, + "limer": 16915, + "resemb": 16916, + "harold": 16917, + "retri": 16918, + "sacrific": 16919, + "surprises": 16920, + "amc": 16921, + "srilanka": 16922, + "barbie": 16923, + "mens": 16924, + "finn": 16925, + "ags": 16926, + "ukrainian": 16927, + "embrac": 16928, + "îIJ": 16929, + "flavors": 16930, + "homer": 16931, + "laure": 16932, + "outh": 16933, + "priced": 16934, + "verde": 16935, + "firm": 16936, + "ahs": 16937, + "cub": 16938, + "trey": 16939, + "paranor": 16940, + "profit": 16941, + "indv": 16942, + "whoa": 16943, + "harsh": 16944, + "alot": 16945, + "critics": 16946, + "hubby": 16947, + "figur": 16948, + "gira": 16949, + "castro": 16950, + "chanel": 16951, + "input": 16952, + "originals": 16953, + "tenant": 16954, + "yyyy": 16955, + "turers": 16956, + "lincoln": 16957, + "coon": 16958, + "learn": 16959, + "chou": 16960, + "acare": 16961, + "oles": 16962, + "diner": 16963, + "hyp": 16964, + "bizarre": 16965, + "mcr": 16966, + "letsgo": 16967, + "decorating": 16968, + "ðŁĮİ": 16969, + "alison": 16970, + "arvin": 16971, + "fd": 16972, + "rehab": 16973, + "mccarthy": 16974, + "lottery": 16975, + "dah": 16976, + "minneapolis": 16977, + "eligible": 16978, + "diagnosed": 16979, + "emerald": 16980, + "destinations": 16981, + "sans": 16982, + "ory": 16983, + "blazers": 16984, + "nv": 16985, + "bail": 16986, + "digitalart": 16987, + "noc": 16988, + "malta": 16989, + "solar": 16990, + "pipes": 16991, + "allegations": 16992, + "nock": 16993, + "pope": 16994, + "brid": 16995, + "premier": 16996, + "nx": 16997, + "presentations": 16998, + "efa": 16999, + "bows": 17000, + "valve": 17001, + "opponent": 17002, + "Įë": 17003, + "visual": 17004, + "ingle": 17005, + "categor": 17006, + "eter": 17007, + "pois": 17008, + "dani": 17009, + "attract": 17010, + "neutral": 17011, + "thene": 17012, + "crashes": 17013, + "freddie": 17014, + "utili": 17015, + "cst": 17016, + "awakening": 17017, + "sloven": 17018, + "qualify": 17019, + "proof": 17020, + "fairy": 17021, + "lev": 17022, + "freight": 17023, + "enjoys": 17024, + "cupcake": 17025, + "flavour": 17026, + "âķ": 17027, + "protective": 17028, + "ðŁijıðŁı»": 17029, + "isu": 17030, + "admir": 17031, + "hmmm": 17032, + "continuous": 17033, + "aires": 17034, + "raptors": 17035, + "showcasing": 17036, + "yuk": 17037, + "paste": 17038, + "follower": 17039, + "instructions": 17040, + "spru": 17041, + "@__": 17042, + "theo": 17043, + "debuts": 17044, + "vette": 17045, + "stow": 17046, + "esof": 17047, + "ached": 17048, + "sultan": 17049, + "sandwich": 17050, + "somalia": 17051, + "franco": 17052, + "carne": 17053, + "fluffy": 17054, + "alpine": 17055, + "jasmine": 17056, + "heated": 17057, + "violin": 17058, + "pless": 17059, + "divorce": 17060, + "performer": 17061, + "phies": 17062, + "portsm": 17063, + "dara": 17064, + "kirby": 17065, + "lop": 17066, + "chilli": 17067, + "forth": 17068, + "skype": 17069, + "ðŁĩ®ðŁĩ¹": 17070, + "celebrities": 17071, + "edy": 17072, + "vee": 17073, + "poison": 17074, + "eyel": 17075, + "grabs": 17076, + "ssic": 17077, + "uno": 17078, + "western": 17079, + "railroad": 17080, + "amer": 17081, + "numerous": 17082, + "sv": 17083, + "fow": 17084, + "fist": 17085, + "âĢĭ": 17086, + "requests": 17087, + "martial": 17088, + "emmy": 17089, + "acceptance": 17090, + "laura": 17091, + "ิ": 17092, + "erup": 17093, + "hyundai": 17094, + "outlander": 17095, + "utt": 17096, + "wrestle": 17097, + "espresso": 17098, + "demanding": 17099, + "gdp": 17100, + "geography": 17101, + "saskat": 17102, + "troll": 17103, + "confeder": 17104, + "sues": 17105, + "sem": 17106, + "bets": 17107, + "tful": 17108, + "tosh": 17109, + "teaches": 17110, + "coloured": 17111, + "galway": 17112, + "macy": 17113, + "disorders": 17114, + "bbcra": 17115, + "atem": 17116, + "fender": 17117, + "litter": 17118, + "esh": 17119, + "providers": 17120, + "renovation": 17121, + "nominate": 17122, + "psg": 17123, + "nominations": 17124, + "jenna": 17125, + "sharp": 17126, + "someday": 17127, + "zur": 17128, + "brains": 17129, + "cheshire": 17130, + "prey": 17131, + "hugo": 17132, + "¿": 17133, + "token": 17134, + "rv": 17135, + "carr": 17136, + "tactical": 17137, + "zelda": 17138, + "kayla": 17139, + "fernando": 17140, + "photographers": 17141, + "jour": 17142, + "umbrella": 17143, + "woody": 17144, + "congressman": 17145, + "dump": 17146, + "levy": 17147, + "juan": 17148, + "dazz": 17149, + "signals": 17150, + "lain": 17151, + "anu": 17152, + "michel": 17153, + "porch": 17154, + "alden": 17155, + "siblings": 17156, + "yale": 17157, + "peel": 17158, + "swick": 17159, + "ggin": 17160, + "llc": 17161, + "kale": 17162, + "scon": 17163, + "ild": 17164, + "patreon": 17165, + "reel": 17166, + "quin": 17167, + "witt": 17168, + "marty": 17169, + "moody": 17170, + "toni": 17171, + "dery": 17172, + "gators": 17173, + "specifically": 17174, + "ddin": 17175, + "lyon": 17176, + "trick": 17177, + "meadows": 17178, + "pj": 17179, + "borgh": 17180, + "vik": 17181, + "tur": 17182, + "bronx": 17183, + "puff": 17184, + "lantern": 17185, + "ðŁ¤¦": 17186, + "gently": 17187, + "bestie": 17188, + "fact": 17189, + "refused": 17190, + "fasci": 17191, + "mpy": 17192, + "ðŁĶµ": 17193, + "crossover": 17194, + "meadow": 17195, + "indianapolis": 17196, + "ducation": 17197, + "sley": 17198, + "loom": 17199, + "mixer": 17200, + "newmusic": 17201, + "filmmaker": 17202, + "prosperity": 17203, + "lim": 17204, + "weekend": 17205, + "creamy": 17206, + "neutr": 17207, + "luther": 17208, + "hv": 17209, + "northern": 17210, + "two": 17211, + "hra": 17212, + "catches": 17213, + "appearances": 17214, + "habit": 17215, + "kittens": 17216, + "nv": 17217, + "illac": 17218, + "infan": 17219, + "regardless": 17220, + "lizard": 17221, + "dunk": 17222, + "curtain": 17223, + "acom": 17224, + "intu": 17225, + "vez": 17226, + "emin": 17227, + "flats": 17228, + "calendars": 17229, + "empower": 17230, + "ruined": 17231, + "hungary": 17232, + "vid": 17233, + "wex": 17234, + "ulum": 17235, + "aberdeen": 17236, + "osa": 17237, + "kt": 17238, + "massi": 17239, + "seemed": 17240, + "sden": 17241, + "'?": 17242, + "telephone": 17243, + "defi": 17244, + "inspires": 17245, + "meow": 17246, + "zones": 17247, + "blind": 17248, + "ply": 17249, + "tucson": 17250, + "adventure": 17251, + "ged": 17252, + "oyster": 17253, + "ðŁijıðŁijıðŁijı": 17254, + "output": 17255, + "ttt": 17256, + "metallic": 17257, + "smash": 17258, + "ucla": 17259, + "scots": 17260, + "perfect": 17261, + "lucy": 17262, + "regularly": 17263, + "spic": 17264, + "relative": 17265, + "athers": 17266, + "mise": 17267, + "battling": 17268, + "decides": 17269, + "mata": 17270, + "occupied": 17271, + "randomly": 17272, + "catsoftwitter": 17273, + "gian": 17274, + "bally": 17275, + "alties": 17276, + "allies": 17277, + "immen": 17278, + "syrac": 17279, + "ðŁĴľðŁĴľ": 17280, + "llan": 17281, + "aur": 17282, + "kut": 17283, + "lamar": 17284, + "affects": 17285, + "nra": 17286, + "starwar": 17287, + "ðŁ¤ĺ": 17288, + "scram": 17289, + "enchan": 17290, + "process": 17291, + "luxurious": 17292, + "array": 17293, + "sherlock": 17294, + "compati": 17295, + "dorf": 17296, + "stress": 17297, + "msu": 17298, + "swith": 17299, + "sala": 17300, + "sofinstagram": 17301, + "foil": 17302, + "understood": 17303, + "quay": 17304, + "rp": 17305, + "cade": 17306, + "jaw": 17307, + "enab": 17308, + "encoun": 17309, + "ðŁİī:": 17310, + "dock": 17311, + "saturn": 17312, + "mull": 17313, + "layout": 17314, + "rarely": 17315, + "happily": 17316, + "fixture": 17317, + "orph": 17318, + "overlooking": 17319, + "herbs": 17320, + "mitt": 17321, + "pillar": 17322, + "nolan": 17323, + "petty": 17324, + "stry": 17325, + "ui": 17326, + "muk": 17327, + "ores": 17328, + "overs": 17329, + "áµ": 17330, + "recreation": 17331, + "wesley": 17332, + "rit": 17333, + "kejriwal": 17334, + "stocking": 17335, + "gv": 17336, + "subscribers": 17337, + "moose": 17338, + "mae": 17339, + "bert": 17340, + "oppre": 17341, + "assignment": 17342, + "uro": 17343, + "highlighting": 17344, + "calvin": 17345, + "weigh": 17346, + "cambodia": 17347, + "avon": 17348, + "kem": 17349, + "disabilities": 17350, + "ready": 17351, + "chargers": 17352, + "pads": 17353, + "izing": 17354, + "illian": 17355, + "truste": 17356, + "colleges": 17357, + "associates": 17358, + "albany": 17359, + "milton": 17360, + "cron": 17361, + "bur": 17362, + "hardly": 17363, + "sights": 17364, + "antiques": 17365, + "echo": 17366, + "surprisingly": 17367, + "haiti": 17368, + "capt": 17369, + "php": 17370, + "opio": 17371, + "inequality": 17372, + "equal": 17373, + "keny": 17374, + "schmid": 17375, + "autographs": 17376, + "rent": 17377, + "quer": 17378, + "citrus": 17379, + "challenged": 17380, + "tec": 17381, + "epide": 17382, + "fest": 17383, + "zhou": 17384, + "lime": 17385, + "citizenship": 17386, + "crystal": 17387, + "convinced": 17388, + "messenger": 17389, + "copenhagen": 17390, + "âĿĹï¸ı": 17391, + "warran": 17392, + "developments": 17393, + "ï¸ıâĥ£": 17394, + "forex": 17395, + "hiro": 17396, + "sneakers": 17397, + "xide": 17398, + "viva": 17399, + "stereo": 17400, + "batting": 17401, + "ssel": 17402, + "host": 17403, + "bengal": 17404, + "criticism": 17405, + "qc": 17406, + "crun": 17407, + "attempted": 17408, + "rye": 17409, + "determination": 17410, + "creations": 17411, + "dread": 17412, + "labels": 17413, + "posse": 17414, + "ancer": 17415, + "johan": 17416, + "sister": 17417, + "partnerships": 17418, + "lesbian": 17419, + "kst": 17420, + "guarantee": 17421, + "baro": 17422, + "fixing": 17423, + "mason": 17424, + "mous": 17425, + "chemicals": 17426, + "tless": 17427, + "biodiversity": 17428, + "paro": 17429, + "bharat": 17430, + "acol": 17431, + "refuge": 17432, + "ente": 17433, + "titi": 17434, + "dyssey": 17435, + "responds": 17436, + "lefto": 17437, + "iner": 17438, + "sevel": 17439, + "rahul": 17440, + "oline": 17441, + "frankfur": 17442, + "choreo": 17443, + "enjoyable": 17444, + "cto": 17445, + "struggles": 17446, + "woodland": 17447, + "heavyweight": 17448, + "gens": 17449, + "recep": 17450, + "accred": 17451, + "ðŁĺ¡": 17452, + "transformed": 17453, + "listen": 17454, + "atop": 17455, + "nk": 17456, + "surge": 17457, + "bere": 17458, + "governor": 17459, + "prisoners": 17460, + "claude": 17461, + "till": 17462, + "mulator": 17463, + "emotion": 17464, + "waterloo": 17465, + "start": 17466, + "ðŁĩº": 17467, + "cleaned": 17468, + "grandmother": 17469, + "fearless": 17470, + "african": 17471, + "astronomy": 17472, + "ðŁıģ": 17473, + "à¸Ļ": 17474, + "theworld": 17475, + "suitable": 17476, + "anthony": 17477, + "kand": 17478, + "tten": 17479, + "meaningful": 17480, + "disclo": 17481, + "jacobs": 17482, + "ø": 17483, + "tomlinson": 17484, + "ghetti": 17485, + "typho": 17486, + "substan": 17487, + "asco": 17488, + "tek": 17489, + "nagar": 17490, + "mud": 17491, + "amon": 17492, + "vaccine": 17493, + "fty": 17494, + "flesh": 17495, + "noel": 17496, + "inflation": 17497, + "portugue": 17498, + "glamour": 17499, + "tram": 17500, + "vre": 17501, + "tequ": 17502, + "roundup": 17503, + "wyn": 17504, + "rejected": 17505, + "mosaic": 17506, + "sighting": 17507, + "calf": 17508, + "ota": 17509, + "composition": 17510, + "gopro": 17511, + "gonzale": 17512, + "eed": 17513, + "bard": 17514, + "tue": 17515, + "effectively": 17516, + "ween": 17517, + "alto": 17518, + "ribs": 17519, + "relate": 17520, + "thirsty": 17521, + "furious": 17522, + "dim": 17523, + "chard": 17524, + "perfume": 17525, + "sny": 17526, + "churchill": 17527, + "kof": 17528, + "masterclass": 17529, + "wave": 17530, + "ðŁĶµ": 17531, + "erin": 17532, + "owns": 17533, + "tobe": 17534, + "skilled": 17535, + "tem": 17536, + "gof": 17537, + "eni": 17538, + "tori": 17539, + "crazy": 17540, + "lick": 17541, + "resistant": 17542, + "icial": 17543, + "agar": 17544, + "!:": 17545, + "gali": 17546, + "delaware": 17547, + "blitz": 17548, + "kohli": 17549, + "puck": 17550, + "availability": 17551, + "himalay": 17552, + "influential": 17553, + "crochet": 17554, + "victori": 17555, + "reading": 17556, + "hobby": 17557, + "viet": 17558, + "jas": 17559, + "engra": 17560, + "skul": 17561, + "ðŁĩ²ðŁĩ": 17562, + "educate": 17563, + "techno": 17564, + "districts": 17565, + "blues": 17566, + "sett": 17567, + "seventh": 17568, + "learns": 17569, + "eeee": 17570, + "apocalypse": 17571, + "hangout": 17572, + "cruel": 17573, + "mutu": 17574, + "bruh": 17575, + "helen": 17576, + "sheer": 17577, + "ction": 17578, + "klein": 17579, + "texans": 17580, + "cereal": 17581, + "shine": 17582, + "nered": 17583, + "gras": 17584, + "ambro": 17585, + "fella": 17586, + "hindu": 17587, + "matthew": 17588, + "lima": 17589, + "miranda": 17590, + "jewel": 17591, + "soho": 17592, + "eurovision": 17593, + "neighbours": 17594, + "chandler": 17595, + "besides": 17596, + "ðŁ¥°": 17597, + "astros": 17598, + "thumbs": 17599, + "renault": 17600, + "rave": 17601, + "hired": 17602, + "ðŁĸ¤": 17603, + "itary": 17604, + "zor": 17605, + "blazer": 17606, + "kine": 17607, + "eau": 17608, + "katy": 17609, + "dccomics": 17610, + "pec": 17611, + "rodgers": 17612, + "waterproof": 17613, + "killers": 17614, + "superint": 17615, + "preserv": 17616, + "asso": 17617, + "brewers": 17618, + "promotional": 17619, + "scam": 17620, + "villages": 17621, + "sketches": 17622, + "juicy": 17623, + "forlife": 17624, + "audit": 17625, + "solo": 17626, + "fundamental": 17627, + "lene": 17628, + "philippine": 17629, + "tend": 17630, + "conservatives": 17631, + "sponsorship": 17632, + "ddle": 17633, + "aine": 17634, + "htc": 17635, + "osi": 17636, + "hulk": 17637, + "waf": 17638, + "à¸Ļ": 17639, + "evaluation": 17640, + "antine": 17641, + "slee": 17642, + "robertson": 17643, + "roosevel": 17644, + "agi": 17645, + "sophistic": 17646, + "employers": 17647, + "bubbles": 17648, + "kowski": 17649, + "interaction": 17650, + "shu": 17651, + "boule": 17652, + "ican": 17653, + "jare": 17654, + "hank": 17655, + "legitim": 17656, + "knicks": 17657, + "karma": 17658, + "receiver": 17659, + "perks": 17660, + "uh": 17661, + "stair": 17662, + "suni": 17663, + "laboratory": 17664, + "graves": 17665, + "vocals": 17666, + "oot": 17667, + "cture": 17668, + "thrive": 17669, + "tico": 17670, + "ãĥ³": 17671, + "bw": 17672, + "cartoons": 17673, + "mcdonalds": 17674, + "draw": 17675, + "yung": 17676, + "pler": 17677, + "lid": 17678, + "ethical": 17679, + "groove": 17680, + "enta": 17681, + "internationalwomensday": 17682, + "patron": 17683, + "worries": 17684, + "ðŁİħ": 17685, + "ðŁijĭ": 17686, + "katherine": 17687, + "diaz": 17688, + "tori": 17689, + "bachchan": 17690, + "trust": 17691, + "mineral": 17692, + "icom": 17693, + "builders": 17694, + "born": 17695, + "coloring": 17696, + "latte": 17697, + "case": 17698, + "revolution": 17699, + "trader": 17700, + "oxid": 17701, + "chipot": 17702, + "instantly": 17703, + "southern": 17704, + "sehun": 17705, + "prob": 17706, + "hernandez": 17707, + "lisbon": 17708, + "huawe": 17709, + "pong": 17710, + "mea": 17711, + "rooney": 17712, + "wheelchair": 17713, + "keen": 17714, + "bett": 17715, + "corin": 17716, + "regulatory": 17717, + "displac": 17718, + "karen": 17719, + "schem": 17720, + "sunsets": 17721, + "whales": 17722, + "reminis": 17723, + "hep": 17724, + "hide": 17725, + "marcel": 17726, + "pandora": 17727, + "doyle": 17728, + "thfc": 17729, + "otto": 17730, + "nokia": 17731, + "transgender": 17732, + "kov": 17733, + "hawaiian": 17734, + "shave": 17735, + "sovere": 17736, + "excer": 17737, + "nicki": 17738, + "pug": 17739, + "stor": 17740, + "roth": 17741, + "weet": 17742, + "legal": 17743, + "dignity": 17744, + "pow": 17745, + "homage": 17746, + "ðŁĩ³ðŁĩ": 17747, + "sre": 17748, + "canon": 17749, + "lax": 17750, + "woah": 17751, + "quartz": 17752, + "ña": 17753, + "greeting": 17754, + "flickr": 17755, + "nairobi": 17756, + "advocates": 17757, + "anc": 17758, + "vii": 17759, + "eugene": 17760, + "thra": 17761, + "cre": 17762, + "elan": 17763, + "pension": 17764, + "thletics": 17765, + "toni": 17766, + "reagan": 17767, + "xv": 17768, + "store": 17769, + "bench": 17770, + "harlem": 17771, + "toddler": 17772, + "sentenced": 17773, + "âĻ¥ï¸ı": 17774, + "globally": 17775, + "cheaper": 17776, + "uf": 17777, + "mam": 17778, + "nico": 17779, + "iku": 17780, + "thou": 17781, + "nist": 17782, + "dami": 17783, + "thala": 17784, + "rhodes": 17785, + "sale": 17786, + "bowls": 17787, + "âĪ": 17788, + "lasvegas": 17789, + "sanctions": 17790, + "admire": 17791, + "matched": 17792, + "unable": 17793, + "traveler": 17794, + "eleven": 17795, + "strawberries": 17796, + "âĢĶâĢĶâĢĶâĢĶ": 17797, + "studio": 17798, + "jacques": 17799, + "ims": 17800, + "valued": 17801, + "sno": 17802, + "cheesecake": 17803, + "nxt": 17804, + "eos": 17805, + "sx": 17806, + "fx": 17807, + "tonic": 17808, + "hatch": 17809, + "chicks": 17810, + "grads": 17811, + "handic": 17812, + "rory": 17813, + "asp": 17814, + "ripped": 17815, + "dentist": 17816, + "nen": 17817, + "lufc": 17818, + "âľĬ": 17819, + "dige": 17820, + "hopkins": 17821, + "sherman": 17822, + "fda": 17823, + "forall": 17824, + "ashley": 17825, + "strand": 17826, + "hy": 17827, + "liquor": 17828, + "buffet": 17829, + "essence": 17830, + "pharma": 17831, + "suriya": 17832, + "ðŁĴĻðŁĴĻ": 17833, + "festivals": 17834, + "zan": 17835, + "refresh": 17836, + "purple": 17837, + "uniforms": 17838, + "kenneth": 17839, + "=)": 17840, + "asan": 17841, + "helsin": 17842, + "transformers": 17843, + "kali": 17844, + "personalized": 17845, + "chalk": 17846, + "bobby": 17847, + "âĮ": 17848, + "themes": 17849, + "departure": 17850, + "print": 17851, + "illustrations": 17852, + "quiet": 17853, + "agrees": 17854, + "griff": 17855, + "س": 17856, + "miti": 17857, + "together": 17858, + "convenience": 17859, + "abar": 17860, + "carlo": 17861, + "turtles": 17862, + "infosec": 17863, + "somewhat": 17864, + "arlington": 17865, + "scholarships": 17866, + "emirates": 17867, + "mums": 17868, + "stella": 17869, + "autonom": 17870, + "feather": 17871, + "gore": 17872, + "nominees": 17873, + "fragrance": 17874, + "ÑĤ": 17875, + "wong": 17876, + "theastern": 17877, + "gre": 17878, + "zilla": 17879, + "isi": 17880, + "bumper": 17881, + "goo": 17882, + "dozens": 17883, + "abduc": 17884, + "âļªï¸ı": 17885, + "oils": 17886, + "donors": 17887, + "silicon": 17888, + "ipod": 17889, + "fortnite": 17890, + "ðŁĴ¨": 17891, + "toro": 17892, + "sparkling": 17893, + "consciousness": 17894, + "pala": 17895, + "num": 17896, + "mounted": 17897, + "ffins": 17898, + "thieves": 17899, + "teammate": 17900, + "prab": 17901, + "omer": 17902, + "tapes": 17903, + "bod": 17904, + "mitsu": 17905, + "stew": 17906, + "ere": 17907, + "pbs": 17908, + "tusc": 17909, + "lowe": 17910, + "rade": 17911, + "parliamentary": 17912, + "hm": 17913, + "edgar": 17914, + "ðŁijĩðŁijĩ": 17915, + "toa": 17916, + "agh": 17917, + "honi": 17918, + "slate": 17919, + "geek": 17920, + "apt": 17921, + "hardt": 17922, + "tap": 17923, + "horizon": 17924, + "growth": 17925, + "makeover": 17926, + "hil": 17927, + "paperback": 17928, + "idan": 17929, + "rehabil": 17930, + "giu": 17931, + "possibilities": 17932, + "lettu": 17933, + "franco": 17934, + "boss": 17935, + "acher": 17936, + "doesnt": 17937, + "moe": 17938, + "taker": 17939, + "hussain": 17940, + "mlk": 17941, + "dil": 17942, + "thia": 17943, + "hama": 17944, + "realised": 17945, + "ravens": 17946, + "curriculum": 17947, + "mith": 17948, + "knight": 17949, + "tedx": 17950, + "rv": 17951, + "isaiah": 17952, + "cumbria": 17953, + "birthdays": 17954, + "fing": 17955, + "prez": 17956, + "mubarak": 17957, + "exquisite": 17958, + "clearance": 17959, + "yen": 17960, + "pari": 17961, + "evo": 17962, + "ú": 17963, + "modified": 17964, + "applying": 17965, + "implement": 17966, + "discovering": 17967, + "chapman": 17968, + "indiegame": 17969, + "disk": 17970, + "crowdfunding": 17971, + "machin": 17972, + "livel": 17973, + "styled": 17974, + "âĿĮ": 17975, + "making": 17976, + "rehearsals": 17977, + "nutriti": 17978, + "subscription": 17979, + "andro": 17980, + "creators": 17981, + "carries": 17982, + "kylie": 17983, + "camden": 17984, + "apprentice": 17985, + "taxpay": 17986, + "cca": 17987, + "tuesdaythoughts": 17988, + "pissed": 17989, + "erman": 17990, + "detec": 17991, + "freedom": 17992, + "meri": 17993, + "..!": 17994, + "psalm": 17995, + "sunlight": 17996, + "perspec": 17997, + "beings": 17998, + "bookstore": 17999, + "rockstar": 18000, + "functions": 18001, + "pence": 18002, + "faves": 18003, + "zn": 18004, + "obamacare": 18005, + "spill": 18006, + "coventry": 18007, + "pigeon": 18008, + "pivo": 18009, + "bait": 18010, + "kolkata": 18011, + "aval": 18012, + "donor": 18013, + "wah": 18014, + "privileg": 18015, + "traditions": 18016, + "rajasthan": 18017, + "teness": 18018, + "portuguese": 18019, + "ynes": 18020, + "tackles": 18021, + "defic": 18022, + "torn": 18023, + "polling": 18024, + "thorne": 18025, + "ina": 18026, + "benedict": 18027, + "barry": 18028, + "calories": 18029, + "verdict": 18030, + "savethe": 18031, + "norton": 18032, + "office": 18033, + "mainstream": 18034, + "improves": 18035, + "fron": 18036, + "responding": 18037, + "realtor": 18038, + "scottish": 18039, + "declar": 18040, + "rl": 18041, + "shiv": 18042, + "supplier": 18043, + "resting": 18044, + "sweets": 18045, + "qui": 18046, + ".âĢ¦": 18047, + "whitney": 18048, + "startup": 18049, + "thankyou": 18050, + "teacher": 18051, + "halls": 18052, + "have": 18053, + "handmade": 18054, + "proving": 18055, + "quartet": 18056, + "rochester": 18057, + "lian": 18058, + "virtual": 18059, + "mendes": 18060, + "oficial": 18061, + "midlands": 18062, + "xbox": 18063, + "measuring": 18064, + "ovo": 18065, + "accommodation": 18066, + "brides": 18067, + "collegiate": 18068, + "intellectual": 18069, + "incar": 18070, + "niag": 18071, + "ðŁį·": 18072, + "sfw": 18073, + "cocoa": 18074, + "coats": 18075, + "civilians": 18076, + "presidency": 18077, + "matrix": 18078, + "sweetheart": 18079, + "triathlon": 18080, + "wagner": 18081, + "radic": 18082, + "planner": 18083, + "theo": 18084, + "execution": 18085, + "kum": 18086, + "thewalkingdead": 18087, + "scar": 18088, + "rotation": 18089, + "blogging": 18090, + "bomb": 18091, + "reson": 18092, + "bbles": 18093, + "stare": 18094, + "assisted": 18095, + "edo": 18096, + "branded": 18097, + "warnings": 18098, + "thorpe": 18099, + "acknowle": 18100, + "satisfied": 18101, + "shores": 18102, + "rid": 18103, + "dora": 18104, + "physically": 18105, + "bigh": 18106, + "approves": 18107, + "hah": 18108, + "rical": 18109, + "versatile": 18110, + "pretend": 18111, + "lum": 18112, + "abhi": 18113, + "yee": 18114, + "spit": 18115, + "ãĢĮ": 18116, + "djs": 18117, + "ashtra": 18118, + "jt": 18119, + "venues": 18120, + "grammys": 18121, + "cyclo": 18122, + "tracker": 18123, + "overwatch": 18124, + "replica": 18125, + "elyn": 18126, + "nrl": 18127, + "lindsey": 18128, + "homo": 18129, + "balloons": 18130, + "kitchen": 18131, + "sis": 18132, + "amos": 18133, + "endeav": 18134, + "ðŁĴ»": 18135, + "arec": 18136, + "thug": 18137, + "hooked": 18138, + "hrc": 18139, + "newyork": 18140, + "burgh": 18141, + "americas": 18142, + "patricia": 18143, + "ugu": 18144, + "apathy": 18145, + "hast": 18146, + "psychi": 18147, + "cork": 18148, + "petrol": 18149, + "ðŁİ¬": 18150, + "aku": 18151, + "popping": 18152, + "psychological": 18153, + "aux": 18154, + "gma": 18155, + "cadillac": 18156, + "waste": 18157, + "authent": 18158, + "bristol": 18159, + "name": 18160, + "queer": 18161, + "tober": 18162, + "jerry": 18163, + "comin": 18164, + "chant": 18165, + "privileged": 18166, + "opar": 18167, + "loser": 18168, + "text": 18169, + "marker": 18170, + "stries": 18171, + "equally": 18172, + "aki": 18173, + "christmas": 18174, + "gareth": 18175, + "blew": 18176, + "emma": 18177, + "imagin": 18178, + "seals": 18179, + "cheat": 18180, + "conditioning": 18181, + "jana": 18182, + "rens": 18183, + "daries": 18184, + "oasis": 18185, + "discounts": 18186, + "council": 18187, + "ika": 18188, + "shirley": 18189, + "voucher": 18190, + "alps": 18191, + "wx": 18192, + "qr": 18193, + "drift": 18194, + "attempting": 18195, + "utc": 18196, + "ت": 18197, + "gonzalez": 18198, + "mf": 18199, + "joker": 18200, + "parallel": 18201, + "pare": 18202, + "aspects": 18203, + "procedu": 18204, + "np": 18205, + "ama": 18206, + "raleigh": 18207, + "brighten": 18208, + "guire": 18209, + "radiation": 18210, + "crescent": 18211, + "hob": 18212, + "ille": 18213, + "strand": 18214, + "vore": 18215, + "nard": 18216, + "chest": 18217, + "diwali": 18218, + "avatar": 18219, + "alder": 18220, + "dling": 18221, + "pathetic": 18222, + "ðŁĴĺ": 18223, + "spirit": 18224, + "jorge": 18225, + "filmmaking": 18226, + "ðŁĻıðŁĻı": 18227, + "challenger": 18228, + "bj": 18229, + "downtown": 18230, + "html": 18231, + "adequ": 18232, + "twisted": 18233, + "inely": 18234, + "('": 18235, + "wraps": 18236, + "operational": 18237, + "yne": 18238, + "nus": 18239, + "magnet": 18240, + "marketplace": 18241, + "healthier": 18242, + "snapshot": 18243, + "damon": 18244, + "interven": 18245, + "federer": 18246, + "owls": 18247, + "biscuits": 18248, + "jp": 18249, + "rodeo": 18250, + "blueberry": 18251, + "lection": 18252, + "frontier": 18253, + "summers": 18254, + "reyes": 18255, + "pedestrian": 18256, + "gol": 18257, + "caffe": 18258, + "refurbi": 18259, + "boulder": 18260, + "meghan": 18261, + "specialty": 18262, + "lass": 18263, + "ei": 18264, + "suspects": 18265, + "approx": 18266, + "rrr": 18267, + "rath": 18268, + "stim": 18269, + "crushed": 18270, + "hed": 18271, + "whun": 18272, + "loaf": 18273, + "crore": 18274, + "rivera": 18275, + "genetics": 18276, + "sock": 18277, + "wasted": 18278, + "nypd": 18279, + "answering": 18280, + "dove": 18281, + "bella": 18282, + "olin": 18283, + "dun": 18284, + "fiji": 18285, + "pretty": 18286, + "sparkle": 18287, + "yun": 18288, + "jd": 18289, + "europa": 18290, + "lifts": 18291, + "amber": 18292, + "mur": 18293, + "tek": 18294, + "boyd": 18295, + "royalty": 18296, + "indo": 18297, + "rib": 18298, + "gotham": 18299, + "tiest": 18300, + "installing": 18301, + "kemp": 18302, + "thephoto": 18303, + "cosmic": 18304, + ")))": 18305, + "wholesale": 18306, + "loyment": 18307, + "easy": 18308, + "suing": 18309, + "settled": 18310, + "afp": 18311, + "prover": 18312, + "supportive": 18313, + "rees": 18314, + "neath": 18315, + "deliber": 18316, + "cé": 18317, + "welcome": 18318, + "picoftheday": 18319, + "newborn": 18320, + "patty": 18321, + "suns": 18322, + "siest": 18323, + "flint": 18324, + "differently": 18325, + "spoilers": 18326, + "trooper": 18327, + "gins": 18328, + "cory": 18329, + "lookout": 18330, + "equipped": 18331, + "tape": 18332, + "toby": 18333, + "researcher": 18334, + "ush": 18335, + "keyes": 18336, + "alma": 18337, + "induction": 18338, + "kw": 18339, + "khar": 18340, + "slick": 18341, + "bride": 18342, + "eur": 18343, + "craving": 18344, + "bookings": 18345, + "ches": 18346, + "trunk": 18347, + "vernon": 18348, + "spher": 18349, + "crystals": 18350, + "relatively": 18351, + "pompe": 18352, + "unions": 18353, + "valley": 18354, + "para": 18355, + "want": 18356, + "okc": 18357, + "deaf": 18358, + "sergio": 18359, + "lennon": 18360, + "shay": 18361, + "cra": 18362, + "vat": 18363, + "hee": 18364, + "twe": 18365, + "liquid": 18366, + "poly": 18367, + "ðŁİģ": 18368, + "bent": 18369, + "bearing": 18370, + "motorsport": 18371, + "barbe": 18372, + "testi": 18373, + "hani": 18374, + "financing": 18375, + "astronaut": 18376, + "watercolour": 18377, + "rish": 18378, + "comiccon": 18379, + "gart": 18380, + "wrong": 18381, + "bern": 18382, + "itan": 18383, + "stepped": 18384, + "filters": 18385, + "clow": 18386, + "mex": 18387, + "demons": 18388, + "allo": 18389, + "expanded": 18390, + "command": 18391, + "eters": 18392, + "goats": 18393, + "siri": 18394, + "yr": 18395, + "pottery": 18396, + "marion": 18397, + "ile": 18398, + "elan": 18399, + "santo": 18400, + "persona": 18401, + "duke": 18402, + "homeless": 18403, + "lighted": 18404, + "wheeler": 18405, + "changer": 18406, + "cabbage": 18407, + "surreal": 18408, + "hamburg": 18409, + "smashed": 18410, + "stran": 18411, + "knot": 18412, + "iart": 18413, + "obi": 18414, + "bedro": 18415, + "dial": 18416, + "thick": 18417, + "bingo": 18418, + "fus": 18419, + "vacuum": 18420, + "conve": 18421, + "ative": 18422, + "accuracy": 18423, + "account": 18424, + "refer": 18425, + "riz": 18426, + "spiderman": 18427, + "bana": 18428, + "rite": 18429, + "ub": 18430, + "abs": 18431, + "medical": 18432, + "link": 18433, + "siem": 18434, + ">>>>": 18435, + "betra": 18436, + "glowing": 18437, + "reactions": 18438, + "puppet": 18439, + "spaghetti": 18440, + "angs": 18441, + "remedi": 18442, + "prayfor": 18443, + "royce": 18444, + "charlotte": 18445, + "£ï¸ı": 18446, + "ghet": 18447, + "affecting": 18448, + "rode": 18449, + "socialist": 18450, + "moses": 18451, + "azi": 18452, + "oit": 18453, + "reporters": 18454, + "cdt": 18455, + "aping": 18456, + "snat": 18457, + "minimal": 18458, + "waist": 18459, + "siege": 18460, + ">>>>": 18461, + "rig": 18462, + "schmidt": 18463, + "hare": 18464, + "eca": 18465, + "thorn": 18466, + "hemp": 18467, + "esthe": 18468, + "clyde": 18469, + "tha": 18470, + "donut": 18471, + "mohamed": 18472, + "lingerie": 18473, + "legg": 18474, + "carpenter": 18475, + "performers": 18476, + "dea": 18477, + "imagined": 18478, + "curse": 18479, + "lash": 18480, + "ctr": 18481, + "agua": 18482, + "roar": 18483, + "gri": 18484, + "role": 18485, + "jfk": 18486, + "resurrec": 18487, + "roosevelt": 18488, + "marilyn": 18489, + "smalle": 18490, + "willis": 18491, + "waited": 18492, + "charities": 18493, + "theres": 18494, + "lik": 18495, + "original": 18496, + "cari": 18497, + "cough": 18498, + "cruci": 18499, + "lagun": 18500, + "contrast": 18501, + "kou": 18502, + "armour": 18503, + "removing": 18504, + "tent": 18505, + "mazda": 18506, + "brighter": 18507, + "thief": 18508, + "corner": 18509, + "tequila": 18510, + "buzzing": 18511, + "albi": 18512, + "pam": 18513, + "azure": 18514, + "discoun": 18515, + "pixelart": 18516, + "possibility": 18517, + "hamont": 18518, + "trades": 18519, + "buda": 18520, + "hive": 18521, + "versy": 18522, + "finch": 18523, + "transpa": 18524, + "emi": 18525, + "terrifying": 18526, + "inqui": 18527, + "gba": 18528, + "substitu": 18529, + "collecti": 18530, + "placing": 18531, + "cindy": 18532, + "kann": 18533, + "patho": 18534, + "diamond": 18535, + "mourinho": 18536, + "guinea": 18537, + "anthropo": 18538, + "airs": 18539, + "pumps": 18540, + "ìļ": 18541, + "paso": 18542, + "curling": 18543, + "anita": 18544, + "residency": 18545, + "newh": 18546, + "joon": 18547, + "cigarette": 18548, + "queue": 18549, + "extrac": 18550, + "games": 18551, + "splen": 18552, + "express": 18553, + "publicly": 18554, + "bonnie": 18555, + "tribune": 18556, + "baek": 18557, + "reasonable": 18558, + "cor": 18559, + "timothy": 18560, + "sheeran": 18561, + "ı": 18562, + "fdn": 18563, + "sutton": 18564, + "concentration": 18565, + "caravan": 18566, + "xavier": 18567, + "alger": 18568, + "cylin": 18569, + "frederick": 18570, + "nerve": 18571, + "peak": 18572, + "lettuce": 18573, + "jail": 18574, + "pregame": 18575, + "kavan": 18576, + "upgraded": 18577, + "ecology": 18578, + "squadron": 18579, + "grapes": 18580, + "goog": 18581, + "pastry": 18582, + "ðŁĹ£": 18583, + "ãĥ¼ãĥ": 18584, + "milano": 18585, + "awaz": 18586, + "presenter": 18587, + "ðŁĮ¿": 18588, + "herd": 18589, + "kings": 18590, + "template": 18591, + "flour": 18592, + "hv": 18593, + "kley": 18594, + "iya": 18595, + "spec": 18596, + "ater": 18597, + "frankfurt": 18598, + "coch": 18599, + "texting": 18600, + "deli": 18601, + "communist": 18602, + "regiment": 18603, + "eleanor": 18604, + "anticipated": 18605, + "ðŁijĮðŁı»": 18606, + "thephotohour": 18607, + "rano": 18608, + "surviving": 18609, + "simulation": 18610, + "dawson": 18611, + "arin": 18612, + "aqua": 18613, + "mor": 18614, + "âĢ¦.": 18615, + "cino": 18616, + "iraqi": 18617, + "shaz": 18618, + "dundee": 18619, + "wes": 18620, + "drau": 18621, + "hannah": 18622, + "snews": 18623, + "occupation": 18624, + "steen": 18625, + "xm": 18626, + "angles": 18627, + "settings": 18628, + "guru": 18629, + "knox": 18630, + "orca": 18631, + "shaping": 18632, + "went": 18633, + "drilling": 18634, + "zzie": 18635, + "bri": 18636, + "kissing": 18637, + "find": 18638, + "maine": 18639, + "âŃIJï¸ıâŃIJï¸ı": 18640, + "ðŁĮį": 18641, + "larry": 18642, + "busted": 18643, + "tavern": 18644, + "actively": 18645, + "-\"": 18646, + "replacing": 18647, + "nod": 18648, + "unlock": 18649, + ".\"": 18650, + "âŀ¤": 18651, + "affiliate": 18652, + "tow": 18653, + "ln": 18654, + "happynewyear": 18655, + "dif": 18656, + "jm": 18657, + "greenwich": 18658, + "controversy": 18659, + "dawg": 18660, + "condol": 18661, + "savannah": 18662, + "compensation": 18663, + "touchdown": 18664, + "teo": 18665, + "ambitious": 18666, + "embroi": 18667, + "convicted": 18668, + "iartg": 18669, + "barack": 18670, + "trance": 18671, + "testimony": 18672, + "audition": 18673, + "thumb": 18674, + "myths": 18675, + "bex": 18676, + "quez": 18677, + "orchid": 18678, + "deny": 18679, + "entitled": 18680, + "hood": 18681, + "grant": 18682, + "inbox": 18683, + "bluejays": 18684, + "rilla": 18685, + "smallest": 18686, + "burden": 18687, + "infamous": 18688, + "divided": 18689, + "boundaries": 18690, + "tter": 18691, + "elt": 18692, + "wyoming": 18693, + "beverage": 18694, + "mesm": 18695, + "onews": 18696, + "buddhist": 18697, + "yana": 18698, + "assad": 18699, + "isms": 18700, + "barrett": 18701, + "predicted": 18702, + "backto": 18703, + "twit": 18704, + "ethere": 18705, + "captains": 18706, + "escaped": 18707, + "ayo": 18708, + "lamborgh": 18709, + "gardner": 18710, + "laps": 18711, + "kal": 18712, + "advertisement": 18713, + "insects": 18714, + "napo": 18715, + "amen": 18716, + "acy": 18717, + "rand": 18718, + "gk": 18719, + "teh": 18720, + "kathle": 18721, + "tridge": 18722, + "pancake": 18723, + "atro": 18724, + "pyramid": 18725, + "bula": 18726, + "paralym": 18727, + "gauge": 18728, + "encies": 18729, + "tomy": 18730, + "biscuit": 18731, + "butcher": 18732, + "qualifier": 18733, + "county": 18734, + "kei": 18735, + "pools": 18736, + "darker": 18737, + "shoulders": 18738, + "ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸": 18739, + "spre": 18740, + "(\"": 18741, + "writers": 18742, + "gm": 18743, + "ðŁİĵ": 18744, + "knit": 18745, + "huff": 18746, + "mtb": 18747, + "phillies": 18748, + "ost": 18749, + "denis": 18750, + "gart": 18751, + "licensed": 18752, + "interface": 18753, + "excel": 18754, + "dwell": 18755, + "fromthe": 18756, + "cofficial": 18757, + "azzi": 18758, + "appearing": 18759, + "forest": 18760, + "nana": 18761, + "keith": 18762, + "manufacturers": 18763, + "beckham": 18764, + ")?": 18765, + "ese": 18766, + "colony": 18767, + "delicate": 18768, + "utter": 18769, + "mcin": 18770, + "transplant": 18771, + "preferred": 18772, + "pard": 18773, + "arie": 18774, + "hub": 18775, + "pods": 18776, + "perspectives": 18777, + "pict": 18778, + "delu": 18779, + "apper": 18780, + "bethan": 18781, + "pmo": 18782, + "criminals": 18783, + "feminism": 18784, + "shack": 18785, + "circumstances": 18786, + "fellas": 18787, + "protesting": 18788, + "wax": 18789, + "suggested": 18790, + "tator": 18791, + "drew": 18792, + "omni": 18793, + "fake": 18794, + "kathy": 18795, + "reb": 18796, + "deline": 18797, + "berni": 18798, + "misty": 18799, + "ðŁij©": 18800, + "erable": 18801, + "breakthrough": 18802, + "menswear": 18803, + "millennials": 18804, + "chanyeol": 18805, + "laz": 18806, + "insert": 18807, + "replies": 18808, + "phrase": 18809, + "nx": 18810, + "iheartawards": 18811, + "audrey": 18812, + "granite": 18813, + "racec": 18814, + "orie": 18815, + "terra": 18816, + "innovations": 18817, + "brittany": 18818, + "ateral": 18819, + "pear": 18820, + "biological": 18821, + "shments": 18822, + "institution": 18823, + "msn": 18824, + "frequency": 18825, + "dman": 18826, + "neglec": 18827, + "tf": 18828, + "stefan": 18829, + "foxnews": 18830, + "typo": 18831, + "comms": 18832, + "sequence": 18833, + "carmen": 18834, + "whites": 18835, + "economist": 18836, + "exeter": 18837, + "seum": 18838, + "resorts": 18839, + "casually": 18840, + "bunde": 18841, + "divide": 18842, + "ع": 18843, + "gag": 18844, + "creed": 18845, + "retire": 18846, + "caucus": 18847, + "rapids": 18848, + "wrestlemania": 18849, + "tulsa": 18850, + "sunderland": 18851, + "fundament": 18852, + "odi": 18853, + "yamaha": 18854, + "vary": 18855, + "intrigu": 18856, + "else": 18857, + "beacon": 18858, + "angie": 18859, + "traded": 18860, + "transm": 18861, + "gents": 18862, + "knitting": 18863, + "galac": 18864, + "ðĿĹ": 18865, + "uto": 18866, + "seaside": 18867, + "holt": 18868, + "rers": 18869, + "fargo": 18870, + "trainers": 18871, + "monsoon": 18872, + "bale": 18873, + "sought": 18874, + "maddie": 18875, + "hw": 18876, + "coli": 18877, + "fran": 18878, + "favs": 18879, + "ðŁĴĶ": 18880, + "intent": 18881, + "rally": 18882, + "sbs": 18883, + "lemonade": 18884, + "barackobama": 18885, + "bread": 18886, + "sticky": 18887, + "explosive": 18888, + "chelten": 18889, + "tj": 18890, + "assoc": 18891, + "ramen": 18892, + "homies": 18893, + "vlog": 18894, + "mister": 18895, + "lord": 18896, + "âĢįâĻĢï¸ı": 18897, + "alyssa": 18898, + "sketchbook": 18899, + "rumble": 18900, + "catch": 18901, + "migrant": 18902, + "discipline": 18903, + "unlikely": 18904, + "chronicles": 18905, + "flora": 18906, + "slams": 18907, + "amid": 18908, + "sboro": 18909, + "coop": 18910, + "jumps": 18911, + "tranqu": 18912, + "melis": 18913, + "sofia": 18914, + "enri": 18915, + "gabe": 18916, + "syri": 18917, + "nicolas": 18918, + "chai": 18919, + "wv": 18920, + "becky": 18921, + "footy": 18922, + "tao": 18923, + "suppose": 18924, + "ðŁĺįðŁĺįðŁĺįðŁĺį": 18925, + "plush": 18926, + "rish": 18927, + "ðŁ¤ĵ": 18928, + "kha": 18929, + "saturdays": 18930, + "accent": 18931, + "hec": 18932, + "limit": 18933, + "carlton": 18934, + "wired": 18935, + "taylorswift": 18936, + "ðŁĺij": 18937, + "sql": 18938, + "harro": 18939, + "recipients": 18940, + "gat": 18941, + "gop": 18942, + "thof": 18943, + "amazed": 18944, + "ghan": 18945, + "ðŁıĨðŁıĨ": 18946, + "porto": 18947, + "clare": 18948, + "distant": 18949, + "nac": 18950, + "ohio": 18951, + "ðŁĻıðŁı¼": 18952, + "mtn": 18953, + "antibio": 18954, + "dinosa": 18955, + "mesa": 18956, + "partial": 18957, + "bv": 18958, + "learnt": 18959, + "lovato": 18960, + "question": 18961, + "extract": 18962, + "gossip": 18963, + "gibb": 18964, + "niagara": 18965, + "ðŁij¨": 18966, + "displayed": 18967, + "sooner": 18968, + "stevie": 18969, + "nuggets": 18970, + "mln": 18971, + "brom": 18972, + "turb": 18973, + "giveaways": 18974, + "stupi": 18975, + "blink": 18976, + "cili": 18977, + "convenient": 18978, + "moh": 18979, + "vive": 18980, + "fric": 18981, + "cause": 18982, + "chamber": 18983, + "cules": 18984, + "nearest": 18985, + "isse": 18986, + "smallbiz": 18987, + "tj": 18988, + "canadians": 18989, + "smarter": 18990, + "brasil": 18991, + "rare": 18992, + "quette": 18993, + "wha": 18994, + "candle": 18995, + "atomic": 18996, + "ðŁijįðŁijį": 18997, + "warrior": 18998, + "relaxed": 18999, + "strips": 19000, + "neur": 19001, + "kka": 19002, + "rfc": 19003, + "jensen": 19004, + "recovering": 19005, + "responses": 19006, + "salam": 19007, + "orthodox": 19008, + "active": 19009, + "ellers": 19010, + "nit": 19011, + "âŃIJ": 19012, + "metropolitan": 19013, + "centuries": 19014, + "vida": 19015, + "grading": 19016, + "transparent": 19017, + "simple": 19018, + "dots": 19019, + "superintendent": 19020, + "elevator": 19021, + "automated": 19022, + "redskins": 19023, + "imam": 19024, + "summertime": 19025, + "jonathan": 19026, + "gearing": 19027, + "michelle": 19028, + "conflic": 19029, + "mice": 19030, + "tote": 19031, + "publish": 19032, + "pax": 19033, + ")-": 19034, + "nailed": 19035, + "á´": 19036, + "telescope": 19037, + "serbia": 19038, + "bab": 19039, + "apeu": 19040, + "stically": 19041, + "senti": 19042, + "rats": 19043, + "isolated": 19044, + "group": 19045, + "hatred": 19046, + "paranormal": 19047, + "stanley": 19048, + "alion": 19049, + "safety": 19050, + "ls": 19051, + "र": 19052, + "nexus": 19053, + "alexandra": 19054, + "masks": 19055, + "++": 19056, + "tron": 19057, + "auk": 19058, + "brotherhood": 19059, + "browse": 19060, + "mixes": 19061, + "simone": 19062, + "musk": 19063, + "approve": 19064, + "lola": 19065, + "exp": 19066, + "perth": 19067, + "futuri": 19068, + "unseen": 19069, + "dm": 19070, + "chelse": 19071, + "scouting": 19072, + "owe": 19073, + "portsmouth": 19074, + "kram": 19075, + "mize": 19076, + "dispen": 19077, + "sup": 19078, + "dlc": 19079, + "advert": 19080, + "teresa": 19081, + "isle": 19082, + "cycle": 19083, + "metall": 19084, + "shields": 19085, + "mariners": 19086, + "raz": 19087, + "ingen": 19088, + "fund": 19089, + "ango": 19090, + "jones": 19091, + "oka": 19092, + "madden": 19093, + "broccoli": 19094, + "dominic": 19095, + "situations": 19096, + "mero": 19097, + "cricke": 19098, + "punishment": 19099, + "db": 19100, + "shaking": 19101, + "ðŁĺļ": 19102, + "mq": 19103, + "arians": 19104, + "leh": 19105, + "claw": 19106, + "weds": 19107, + "dure": 19108, + "niel": 19109, + "jelly": 19110, + "gourmet": 19111, + "traders": 19112, + "levi": 19113, + "wages": 19114, + "knees": 19115, + "wise": 19116, + "heavenly": 19117, + "avid": 19118, + "melody": 19119, + "zack": 19120, + "bananas": 19121, + "apprentice": 19122, + "prop": 19123, + "funny": 19124, + "ode": 19125, + "respected": 19126, + "megan": 19127, + "fewer": 19128, + "drafted": 19129, + "medit": 19130, + "grape": 19131, + "usarmy": 19132, + "crusad": 19133, + "vocali": 19134, + "preparations": 19135, + "nonsense": 19136, + "usage": 19137, + "thr": 19138, + "roth": 19139, + "wizards": 19140, + "inside": 19141, + "promotions": 19142, + "mona": 19143, + "redsox": 19144, + "sig": 19145, + "elegance": 19146, + "chia": 19147, + "universal": 19148, + "ãĢį": 19149, + "raja": 19150, + "unga": 19151, + "pollin": 19152, + "filipino": 19153, + "aka": 19154, + "tsun": 19155, + "ikon": 19156, + "biking": 19157, + "decorations": 19158, + "zac": 19159, + "cadets": 19160, + "humour": 19161, + "agm": 19162, + "reppin": 19163, + "vaccin": 19164, + "elove": 19165, + "uw": 19166, + "diabe": 19167, + "gallagher": 19168, + "azer": 19169, + "dol": 19170, + "awhile": 19171, + "prominent": 19172, + "welsh": 19173, + "tann": 19174, + "')": 19175, + "bien": 19176, + "wag": 19177, + "inal": 19178, + "cwc": 19179, + "wicket": 19180, + "urst": 19181, + "qanon": 19182, + "xe": 19183, + "outdoor": 19184, + "dunn": 19185, + "starr": 19186, + "cology": 19187, + "ricky": 19188, + "uefa": 19189, + "rebounds": 19190, + "smusic": 19191, + "infant": 19192, + "ðŁĻĭ": 19193, + "sop": 19194, + "umber": 19195, + "handing": 19196, + "begin": 19197, + "sorting": 19198, + "hash": 19199, + "spati": 19200, + "rek": 19201, + "budapest": 19202, + "blackhawks": 19203, + "delete": 19204, + "rom": 19205, + "candid": 19206, + "authori": 19207, + "debris": 19208, + "specul": 19209, + "intersection": 19210, + "marriott": 19211, + "imran": 19212, + "ðŁĺģðŁĺģ": 19213, + "cruises": 19214, + "ramsey": 19215, + "rafael": 19216, + "awareness": 19217, + "vascular": 19218, + "beyoncé": 19219, + "rug": 19220, + "ðŁĺĮ": 19221, + "festiv": 19222, + "aram": 19223, + "sable": 19224, + "basil": 19225, + "pill": 19226, + "flooring": 19227, + "unbeaten": 19228, + "implications": 19229, + "uf": 19230, + "wound": 19231, + "forge": 19232, + "pointing": 19233, + "pots": 19234, + "popularity": 19235, + "ðŁijıðŁı»": 19236, + "manipul": 19237, + "slots": 19238, + "debates": 19239, + "absence": 19240, + "vermont": 19241, + "neverforget": 19242, + "wrist": 19243, + "gloria": 19244, + "rence": 19245, + "husk": 19246, + "melting": 19247, + "ðŁİŁ": 19248, + "braces": 19249, + "timely": 19250, + "transforming": 19251, + "amps": 19252, + "mak": 19253, + "poe": 19254, + "ahan": 19255, + "generally": 19256, + "ndp": 19257, + "aleppo": 19258, + "unicef": 19259, + "profs": 19260, + "nord": 19261, + "mask": 19262, + "jacksonville": 19263, + "vv": 19264, + "shells": 19265, + "blooming": 19266, + "operators": 19267, + "charcoal": 19268, + "neville": 19269, + "magi": 19270, + "chip": 19271, + "sama": 19272, + "iran": 19273, + "reforms": 19274, + "accumul": 19275, + "rue": 19276, + "æľ": 19277, + "websites": 19278, + "gaon": 19279, + "devastating": 19280, + "stos": 19281, + "glacier": 19282, + "rapp": 19283, + "chipotle": 19284, + "pra": 19285, + "orous": 19286, + "romney": 19287, + "season": 19288, + "decorative": 19289, + "cisco": 19290, + "ditch": 19291, + "complain": 19292, + "llo": 19293, + "assume": 19294, + "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ": 19295, + "nels": 19296, + "centric": 19297, + "ftw": 19298, + "carrots": 19299, + "tata": 19300, + "canter": 19301, + "perience": 19302, + "liers": 19303, + "demos": 19304, + "blunt": 19305, + "operate": 19306, + "reservations": 19307, + "leah": 19308, + "substance": 19309, + "dison": 19310, + "ante": 19311, + "election": 19312, + "vue": 19313, + "square": 19314, + "nonprofit": 19315, + "caa": 19316, + "fsu": 19317, + "yam": 19318, + "ãĤ¤": 19319, + "vladi": 19320, + "completes": 19321, + "mari": 19322, + "phillip": 19323, + "neill": 19324, + "eras": 19325, + "kait": 19326, + "mendo": 19327, + "maharashtra": 19328, + "gp": 19329, + "dane": 19330, + "providence": 19331, + "therapeu": 19332, + "juvenile": 19333, + "memo": 19334, + "incorpor": 19335, + "aaaa": 19336, + "seventeen": 19337, + "teenager": 19338, + "ã": 19339, + "orns": 19340, + "wide": 19341, + "cuteness": 19342, + "twd": 19343, + "ffles": 19344, + "bara": 19345, + "comedy": 19346, + "overtime": 19347, + "yaz": 19348, + "baron": 19349, + "unemployment": 19350, + "ðŁijĭ": 19351, + "exterior": 19352, + "dense": 19353, + "centres": 19354, + "matchup": 19355, + "historymonth": 19356, + "artificial": 19357, + "quit": 19358, + "esk": 19359, + "warn": 19360, + "critic": 19361, + "jaf": 19362, + "ðŁĵ²": 19363, + "informative": 19364, + "fuels": 19365, + "recycle": 19366, + "naming": 19367, + "stripe": 19368, + "solic": 19369, + "molecular": 19370, + "deepi": 19371, + "convo": 19372, + "ssel": 19373, + "nae": 19374, + "descent": 19375, + "tiz": 19376, + "accountability": 19377, + "terry": 19378, + "rito": 19379, + "slay": 19380, + "emo": 19381, + "demol": 19382, + "sensation": 19383, + "cov": 19384, + "tore": 19385, + "roundtable": 19386, + "yol": 19387, + "excuses": 19388, + "à¥į": 19389, + "turquo": 19390, + "hhhh": 19391, + "podcasts": 19392, + "celeb": 19393, + "messi": 19394, + "lio": 19395, + "mann": 19396, + "contributed": 19397, + "uz": 19398, + "generator": 19399, + "elets": 19400, + "veggie": 19401, + "indul": 19402, + "ensuring": 19403, + "detroit": 19404, + "punjab": 19405, + "transpor": 19406, + "instruction": 19407, + "add": 19408, + "porcel": 19409, + "paneli": 19410, + "circles": 19411, + "persist": 19412, + "clayton": 19413, + "spn": 19414, + "dogsoftwitter": 19415, + "isnt": 19416, + "spr": 19417, + "retailers": 19418, + "pw": 19419, + "hungar": 19420, + "elena": 19421, + "monaster": 19422, + "guatem": 19423, + "jessie": 19424, + "anz": 19425, + "rashi": 19426, + "flee": 19427, + "carving": 19428, + "faux": 19429, + "lal": 19430, + "henri": 19431, + "djo": 19432, + "dull": 19433, + "sana": 19434, + "lara": 19435, + "globe": 19436, + "crimson": 19437, + "compass": 19438, + "pause": 19439, + "nab": 19440, + "lionel": 19441, + "baths": 19442, + "ufo": 19443, + "inventory": 19444, + "singh": 19445, + "satan": 19446, + "ðŁĩ¸": 19447, + "cements": 19448, + "inform": 19449, + "generated": 19450, + "biden": 19451, + "avg": 19452, + "tasks": 19453, + "deer": 19454, + "sau": 19455, + "jailed": 19456, + "pastel": 19457, + "scc": 19458, + "nail": 19459, + "steele": 19460, + "peris": 19461, + "lamborghini": 19462, + "pursue": 19463, + "margin": 19464, + "uch": 19465, + "bosch": 19466, + "drain": 19467, + "clara": 19468, + "bom": 19469, + "latino": 19470, + "webster": 19471, + "rosemary": 19472, + "rha": 19473, + "soun": 19474, + "billionaire": 19475, + "notch": 19476, + "percentage": 19477, + "conor": 19478, + "'\"": 19479, + "homes": 19480, + "earthday": 19481, + "hort": 19482, + "biggest": 19483, + "disin": 19484, + "walton": 19485, + "editors": 19486, + "imma": 19487, + "omar": 19488, + "equivalent": 19489, + "pharmaceu": 19490, + "ahmed": 19491, + "cameo": 19492, + "hanni": 19493, + "underrated": 19494, + "gement": 19495, + "microbi": 19496, + "voo": 19497, + "honorable": 19498, + "obesity": 19499, + "âļ¡ï¸ı": 19500, + "limerick": 19501, + "involvement": 19502, + "stagram": 19503, + "boulevard": 19504, + "burg": 19505, + "blackandwhite": 19506, + "liberation": 19507, + "five": 19508, + "interim": 19509, + "smm": 19510, + "rivalry": 19511, + "capabilities": 19512, + "statements": 19513, + "thumb": 19514, + "ved": 19515, + "swans": 19516, + "barber": 19517, + "eque": 19518, + "serena": 19519, + "helm": 19520, + "noodle": 19521, + "sampling": 19522, + "nawaz": 19523, + "single": 19524, + "thunderstorms": 19525, + "shon": 19526, + "inev": 19527, + "ë¯": 19528, + "topp": 19529, + "orchard": 19530, + "bian": 19531, + "ðŁĺĶ": 19532, + "doorstep": 19533, + "salvation": 19534, + "marketing": 19535, + "rons": 19536, + "clemson": 19537, + "ravi": 19538, + "intake": 19539, + "standwith": 19540, + "sina": 19541, + "haiku": 19542, + "pley": 19543, + "electoral": 19544, + "philly": 19545, + "lays": 19546, + "electric": 19547, + "capturing": 19548, + "upp": 19549, + "ergy": 19550, + "believing": 19551, + "cultures": 19552, + "esday": 19553, + "invasive": 19554, + "eded": 19555, + "speech": 19556, + "endur": 19557, + "vietnam": 19558, + "boycott": 19559, + "pede": 19560, + "deliver": 19561, + "ðŁĴĸðŁĴĸ": 19562, + "merchant": 19563, + "stir": 19564, + "denies": 19565, + "pockets": 19566, + "oti": 19567, + "cuddle": 19568, + "roland": 19569, + "mmed": 19570, + "dened": 19571, + "learners": 19572, + "hoop": 19573, + "sourcing": 19574, + "hacked": 19575, + "dim": 19576, + "environments": 19577, + "benson": 19578, + "judicial": 19579, + "worcester": 19580, + "pearls": 19581, + "governments": 19582, + "arrivals": 19583, + "corners": 19584, + "tuning": 19585, + "labour": 19586, + "ym": 19587, + "ordering": 19588, + "lewi": 19589, + "ife": 19590, + "hygiene": 19591, + "thoughtful": 19592, + "indonesian": 19593, + "campaigning": 19594, + "principle": 19595, + "assaul": 19596, + "rubb": 19597, + "atv": 19598, + "willy": 19599, + "entre": 19600, + "ili": 19601, + "phon": 19602, + "duties": 19603, + "âĻ¥âĻ¥": 19604, + "snakes": 19605, + "loop": 19606, + "amar": 19607, + "convertible": 19608, + "bonding": 19609, + "mentoring": 19610, + "maxwell": 19611, + "ethereum": 19612, + "destroying": 19613, + "axis": 19614, + "cairo": 19615, + "finnish": 19616, + "shock": 19617, + "ðŁĺIJ": 19618, + "caleb": 19619, + "coma": 19620, + "pedal": 19621, + "core": 19622, + "continent": 19623, + "elson": 19624, + "tempo": 19625, + "helsinki": 19626, + "acp": 19627, + "tackling": 19628, + "stated": 19629, + "bla": 19630, + "doub": 19631, + "smashing": 19632, + "aja": 19633, + "cameron": 19634, + "disruption": 19635, + "warmth": 19636, + "beingsalmankhan": 19637, + "bulletin": 19638, + "ode": 19639, + "syracuse": 19640, + "aran": 19641, + "mcgregor": 19642, + "bulk": 19643, + "anton": 19644, + "confirmation": 19645, + "spine": 19646, + "imran": 19647, + "instruc": 19648, + "jacks": 19649, + "chio": 19650, + "palm": 19651, + "stre": 19652, + "embarrassing": 19653, + "unt": 19654, + "eliminate": 19655, + "toss": 19656, + "cise": 19657, + "aws": 19658, + "onists": 19659, + "shinee": 19660, + "jos": 19661, + "hose": 19662, + "lively": 19663, + "opponents": 19664, + "movements": 19665, + "recognizing": 19666, + "sandwiches": 19667, + "shakes": 19668, + "exercises": 19669, + "seat": 19670, + "profession": 19671, + "merrychristmas": 19672, + "lugg": 19673, + "adoptdont": 19674, + "marvin": 19675, + "byrne": 19676, + "unle": 19677, + "het": 19678, + "kuwait": 19679, + "rahman": 19680, + "aspect": 19681, + "humbled": 19682, + "genes": 19683, + "fand": 19684, + "longtime": 19685, + ");": 19686, + "campu": 19687, + "angus": 19688, + "ðŁijįðŁı¼": 19689, + "quran": 19690, + "sleeves": 19691, + "slic": 19692, + "¸ë": 19693, + "twelve": 19694, + "youre": 19695, + "ike": 19696, + "gogh": 19697, + "bst": 19698, + "dictionary": 19699, + "reflecting": 19700, + "toon": 19701, + "yarn": 19702, + "embed": 19703, + "ðŁı´": 19704, + "reserves": 19705, + "flooded": 19706, + "veriz": 19707, + "dusk": 19708, + "establish": 19709, + "proli": 19710, + "aud": 19711, + "ritual": 19712, + "orbit": 19713, + "declaration": 19714, + "recordings": 19715, + "camo": 19716, + "cassette": 19717, + "goodluck": 19718, + "cutter": 19719, + "bop": 19720, + "bho": 19721, + "cheating": 19722, + "pacific": 19723, + "mares": 19724, + "timer": 19725, + "colt": 19726, + "trous": 19727, + "tomorrow": 19728, + "hansen": 19729, + "cie": 19730, + "wang": 19731, + "bani": 19732, + "circular": 19733, + "acute": 19734, + "farmer": 19735, + "coys": 19736, + "pse": 19737, + "irving": 19738, + "wj": 19739, + "hawkins": 19740, + "bison": 19741, + "urday": 19742, + "cruising": 19743, + "ote": 19744, + "kath": 19745, + "whistle": 19746, + "yourselves": 19747, + "antis": 19748, + "slash": 19749, + "thoroughly": 19750, + "kesh": 19751, + "serie": 19752, + "exem": 19753, + "enig": 19754, + "guild": 19755, + "shred": 19756, + "hogan": 19757, + "apo": 19758, + "ä¸": 19759, + "puzz": 19760, + "netball": 19761, + "aussi": 19762, + "panorama": 19763, + "wsj": 19764, + "avis": 19765, + "arming": 19766, + "humph": 19767, + "browser": 19768, + "cries": 19769, + "foggy": 19770, + "matte": 19771, + "ðŁĮ»": 19772, + "iter": 19773, + "tallest": 19774, + "byron": 19775, + "captiv": 19776, + "jesu": 19777, + "anyways": 19778, + "flagship": 19779, + "pton": 19780, + "wey": 19781, + "fayette": 19782, + "financial": 19783, + "foul": 19784, + "solomon": 19785, + "jennifer": 19786, + "cucumber": 19787, + "argue": 19788, + "textile": 19789, + "wrestler": 19790, + "johnston": 19791, + "pastor": 19792, + "ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ": 19793, + "cactus": 19794, + "edible": 19795, + "reserved": 19796, + "richie": 19797, + "metres": 19798, + "ingredient": 19799, + "hella": 19800, + "unto": 19801, + "chol": 19802, + "celebs": 19803, + "poets": 19804, + "graham": 19805, + "hayden": 19806, + "coincidence": 19807, + "baw": 19808, + "communicate": 19809, + "fletcher": 19810, + "/-": 19811, + "toledo": 19812, + "ecuador": 19813, + "counsel": 19814, + "slaughter": 19815, + "linear": 19816, + "atp": 19817, + "osu": 19818, + "joel": 19819, + "eved": 19820, + "conquer": 19821, + "rustic": 19822, + "plicity": 19823, + "recognise": 19824, + "roommate": 19825, + "cracked": 19826, + "jasper": 19827, + "pher": 19828, + "ðŁĮº": 19829, + "woven": 19830, + "moist": 19831, + "ffc": 19832, + "steering": 19833, + "nish": 19834, + "standings": 19835, + "frequent": 19836, + "ardi": 19837, + "hazel": 19838, + "asmsg": 19839, + "baum": 19840, + "dart": 19841, + "sidd": 19842, + "nath": 19843, + "chero": 19844, + "cardboard": 19845, + "css": 19846, + "nsfw": 19847, + "pair": 19848, + "ðŁĺįðŁĺĺ": 19849, + "occurred": 19850, + "homelessness": 19851, + "malone": 19852, + "phe": 19853, + "xia": 19854, + "paddy": 19855, + "declare": 19856, + "theatre": 19857, + "bf": 19858, + "persian": 19859, + "tad": 19860, + "axe": 19861, + "suspicious": 19862, + "lamb": 19863, + "mucho": 19864, + "senior": 19865, + "stas": 19866, + "kite": 19867, + "sting": 19868, + "grad": 19869, + "kaf": 19870, + "watering": 19871, + "د": 19872, + "spiral": 19873, + "thms": 19874, + "educator": 19875, + "jerome": 19876, + "ofc": 19877, + "clock": 19878, + "sul": 19879, + "pemb": 19880, + ".........": 19881, + "parkway": 19882, + "deaux": 19883, + "restrictions": 19884, + "mons": 19885, + "needle": 19886, + "ej": 19887, + "leagues": 19888, + "watermelon": 19889, + "aman": 19890, + "plenary": 19891, + "maxim": 19892, + "wab": 19893, + "comingsoon": 19894, + "bryce": 19895, + "vigil": 19896, + "supermarket": 19897, + "fortunate": 19898, + "turquoise": 19899, + "president": 19900, + "liv": 19901, + "interns": 19902, + "feelin": 19903, + "fixtures": 19904, + "stunt": 19905, + "staged": 19906, + "premieres": 19907, + "lok": 19908, + "practiti": 19909, + "shortage": 19910, + "logne": 19911, + "vec": 19912, + "concor": 19913, + "rocke": 19914, + "lig": 19915, + "composed": 19916, + "synthetic": 19917, + "dip": 19918, + "camila": 19919, + "chis": 19920, + "jou": 19921, + "susan": 19922, + "eyebrows": 19923, + "supplement": 19924, + "satisfaction": 19925, + "mohammad": 19926, + "tibet": 19927, + "houseof": 19928, + "pun": 19929, + "assam": 19930, + "shadowhun": 19931, + "psyched": 19932, + "seduc": 19933, + "mandatory": 19934, + "herbert": 19935, + "scallo": 19936, + "streamers": 19937, + "protocol": 19938, + "blockbuster": 19939, + "produces": 19940, + "schnei": 19941, + "laurel": 19942, + "tribe": 19943, + "timehop": 19944, + "pla": 19945, + "modelling": 19946, + "tvtime": 19947, + "mtvstars": 19948, + "widow": 19949, + "metric": 19950, + "cham": 19951, + "condo": 19952, + "flowering": 19953, + "alec": 19954, + "dms": 19955, + "intensity": 19956, + "¨": 19957, + "mccartney": 19958, + "islamabad": 19959, + "kb": 19960, + "ffi": 19961, + "phal": 19962, + "analog": 19963, + "fond": 19964, + "hacks": 19965, + "positivity": 19966, + "treaty": 19967, + "submarine": 19968, + "connect": 19969, + "selen": 19970, + "categories": 19971, + "cub": 19972, + "organize": 19973, + "sik": 19974, + "quoteoftheday": 19975, + "reminding": 19976, + "amor": 19977, + "locking": 19978, + "ðŁijıðŁı¼": 19979, + "compound": 19980, + "ette": 19981, + "bout": 19982, + "recur": 19983, + "ference": 19984, + "mizz": 19985, + "trend": 19986, + "hipster": 19987, + "fortress": 19988, + "forthcoming": 19989, + "prelimin": 19990, + "odyssey": 19991, + "angp": 19992, + "delici": 19993, + "evenings": 19994, + "ðŁĶ¹": 19995, + "iq": 19996, + "dw": 19997, + "dair": 19998, + "kathryn": 19999, + "christianity": 20000, + "moonlight": 20001, + "hab": 20002, + "whoo": 20003, + "fbf": 20004, + "seth": 20005, + "genuinely": 20006, + "pax": 20007, + "charity": 20008, + "deployed": 20009, + "bnb": 20010, + "bucs": 20011, + "judg": 20012, + "conge": 20013, + "plantation": 20014, + "impress": 20015, + "cara": 20016, + "sclub": 20017, + "scopy": 20018, + "landers": 20019, + "complaints": 20020, + "bama": 20021, + "rebuild": 20022, + "xy": 20023, + "realism": 20024, + "shour": 20025, + "lein": 20026, + "bracelets": 20027, + "mera": 20028, + "assassin": 20029, + "anchor": 20030, + "ðŁijĮðŁı¼": 20031, + "linen": 20032, + "confron": 20033, + "chronicle": 20034, + "comment": 20035, + "catalog": 20036, + "illes": 20037, + "gorge": 20038, + "metry": 20039, + "jungkook": 20040, + "lovemy": 20041, + "sentin": 20042, + "seem": 20043, + "fitness": 20044, + "allied": 20045, + "tsman": 20046, + "digitaltransformation": 20047, + "pran": 20048, + "loft": 20049, + "minton": 20050, + "aldenrichards": 20051, + "envel": 20052, + "cherish": 20053, + "certainty": 20054, + "zzz": 20055, + "rhino": 20056, + "perkins": 20057, + "enrich": 20058, + "capetown": 20059, + "ometer": 20060, + "sections": 20061, + "skeleton": 20062, + "defenders": 20063, + "ðŁĺĿ": 20064, + "penc": 20065, + "brit": 20066, + "jah": 20067, + "capitalism": 20068, + "ðŁ¥ĩ": 20069, + "bazaar": 20070, + "reme": 20071, + "ext": 20072, + "kkk": 20073, + "convert": 20074, + "stormy": 20075, + "bye": 20076, + "karan": 20077, + "chrysler": 20078, + "ados": 20079, + "pressed": 20080, + "sync": 20081, + "ationday": 20082, + "danger": 20083, + "badges": 20084, + "refuses": 20085, + "empowering": 20086, + "lym": 20087, + "exports": 20088, + "adoptdontshop": 20089, + "ðŁĩ¯": 20090, + "thc": 20091, + "awaited": 20092, + "focuses": 20093, + "fined": 20094, + "oat": 20095, + "hahahah": 20096, + "âģ©": 20097, + "nfamily": 20098, + "fiona": 20099, + "luckily": 20100, + "thrilling": 20101, + "typing": 20102, + "outbreak": 20103, + "dies": 20104, + "heu": 20105, + "crawl": 20106, + "nesses": 20107, + "oath": 20108, + "scripts": 20109, + "geeks": 20110, + "ðŁIJĿ": 20111, + "pb": 20112, + "mathematics": 20113, + "alis": 20114, + "________________": 20115, + "gymnastics": 20116, + "activism": 20117, + "recommendation": 20118, + "gren": 20119, + "wain": 20120, + "courty": 20121, + "napol": 20122, + "cauli": 20123, + "hornets": 20124, + "gals": 20125, + "jockey": 20126, + "dirty": 20127, + "atar": 20128, + "enormous": 20129, + "pest": 20130, + "gregation": 20131, + "anos": 20132, + "iiii": 20133, + "defends": 20134, + "blackhistorymonth": 20135, + "atx": 20136, + "mbc": 20137, + "luggage": 20138, + "witch": 20139, + "cob": 20140, + "lasts": 20141, + "cum": 20142, + "ggg": 20143, + "bathing": 20144, + "nar": 20145, + "cebu": 20146, + "ðŁįĥ": 20147, + "navigation": 20148, + "mine": 20149, + "rejo": 20150, + "ðŁİĢ": 20151, + "giftide": 20152, + "reta": 20153, + "useless": 20154, + "pull": 20155, + "deficit": 20156, + "allu": 20157, + "atime": 20158, + "itv": 20159, + "trillion": 20160, + "pue": 20161, + "acies": 20162, + "procedure": 20163, + "lori": 20164, + "jenny": 20165, + "cad": 20166, + "ulously": 20167, + "drac": 20168, + "promotes": 20169, + "ingthe": 20170, + "canu": 20171, + "woohoo": 20172, + "naomi": 20173, + "zardari": 20174, + "tsu": 20175, + "beir": 20176, + "sdg": 20177, + "lever": 20178, + "weber": 20179, + "abud": 20180, + "lund": 20181, + "crowded": 20182, + "deployment": 20183, + "terrain": 20184, + "kenny": 20185, + "hof": 20186, + "witnessed": 20187, + "loch": 20188, + "jk": 20189, + "bully": 20190, + "wren": 20191, + "poetry": 20192, + "doff": 20193, + "wwi": 20194, + "mored": 20195, + "dini": 20196, + "culture": 20197, + "prompt": 20198, + "Â¥": 20199, + "maurice": 20200, + "topps": 20201, + "rm": 20202, + "correspon": 20203, + "about": 20204, + "jewels": 20205, + "gibr": 20206, + "eagle": 20207, + "ðŁĺĺðŁĺĺðŁĺĺ": 20208, + "lending": 20209, + "souven": 20210, + "çĶ": 20211, + "contemporaryart": 20212, + "establishment": 20213, + "jong": 20214, + "âĢ¦\"": 20215, + "gator": 20216, + "patriotic": 20217, + "mccoy": 20218, + "vape": 20219, + "humane": 20220, + "feliz": 20221, + "coachella": 20222, + "reposting": 20223, + "steals": 20224, + "fuller": 20225, + "nering": 20226, + "atra": 20227, + "(-": 20228, + "blake": 20229, + "heather": 20230, + "worms": 20231, + "disciplinary": 20232, + "redemption": 20233, + "yard": 20234, + "amin": 20235, + "\"@_": 20236, + "dnc": 20237, + "tds": 20238, + "kappa": 20239, + "newark": 20240, + "commits": 20241, + "spears": 20242, + "jams": 20243, + "tand": 20244, + "msnbc": 20245, + "intermedi": 20246, + "aimed": 20247, + "atic": 20248, + "teenth": 20249, + "observation": 20250, + "kashmir": 20251, + "kavanaugh": 20252, + "oul": 20253, + "sanfrancisco": 20254, + "reu": 20255, + "belated": 20256, + "chow": 20257, + "password": 20258, + "stills": 20259, + "detained": 20260, + "sari": 20261, + "dayton": 20262, + "darren": 20263, + "italian": 20264, + "arth": 20265, + "amusic": 20266, + "arbit": 20267, + "wm": 20268, + "vm": 20269, + "hem": 20270, + "doug": 20271, + "myr": 20272, + "asho": 20273, + "prev": 20274, + "vind": 20275, + "brah": 20276, + "stag": 20277, + "ี": 20278, + "previews": 20279, + "guk": 20280, + "containing": 20281, + "leonardo": 20282, + "saddle": 20283, + "rushing": 20284, + "stav": 20285, + "longh": 20286, + "gambling": 20287, + "vegas": 20288, + "reservation": 20289, + "endale": 20290, + "bala": 20291, + "fla": 20292, + "variant": 20293, + "hedge": 20294, + "bulgaria": 20295, + "natali": 20296, + "weaver": 20297, + "solst": 20298, + "encouraged": 20299, + "apc": 20300, + "asparag": 20301, + "nest": 20302, + "cyclists": 20303, + "fel": 20304, + "ìĬ¤": 20305, + "overwhelming": 20306, + "peyton": 20307, + "jit": 20308, + "apost": 20309, + "mble": 20310, + "bleeding": 20311, + "neighbourhood": 20312, + "avery": 20313, + "expressions": 20314, + "macdonald": 20315, + "gigs": 20316, + "monds": 20317, + "illusion": 20318, + "nct": 20319, + "camero": 20320, + "overhead": 20321, + "myth": 20322, + "oly": 20323, + "vio": 20324, + "etv": 20325, + "laurie": 20326, + "unveiling": 20327, + "prior": 20328, + "conn": 20329, + "ironman": 20330, + "diff": 20331, + "dayin": 20332, + "critici": 20333, + "congo": 20334, + "revision": 20335, + "wale": 20336, + "director": 20337, + "pines": 20338, + "blackpink": 20339, + "garner": 20340, + "curated": 20341, + "manitoba": 20342, + "hac": 20343, + "commonly": 20344, + "barton": 20345, + "....#": 20346, + "mortality": 20347, + "livesmatter": 20348, + "philosop": 20349, + "shorter": 20350, + "convince": 20351, + "freak": 20352, + "vendors": 20353, + "insightful": 20354, + "elly": 20355, + "sensors": 20356, + "eled": 20357, + "sberg": 20358, + "weightloss": 20359, + "ukip": 20360, + "spur": 20361, + "private": 20362, + "qua": 20363, + "ssc": 20364, + ",...": 20365, + "supervisor": 20366, + "adviser": 20367, + "amazingly": 20368, + "lesser": 20369, + "ates": 20370, + "mahon": 20371, + "oooooo": 20372, + "saras": 20373, + "pmoindia": 20374, + "waffle": 20375, + "unders": 20376, + "tolerance": 20377, + "sculptures": 20378, + "hersh": 20379, + "knocking": 20380, + "smoke": 20381, + "catholic": 20382, + "grim": 20383, + "traveled": 20384, + "flip": 20385, + "geoff": 20386, + "dinosaurs": 20387, + "slept": 20388, + "scarlet": 20389, + "oki": 20390, + "complaint": 20391, + "obsc": 20392, + "nami": 20393, + "lag": 20394, + "crossfit": 20395, + "ufc": 20396, + "mccain": 20397, + "referee": 20398, + "sadness": 20399, + "penny": 20400, + "lieu": 20401, + "mode": 20402, + "kier": 20403, + "vols": 20404, + "wis": 20405, + "elon": 20406, + "shea": 20407, + "bao": 20408, + "sonia": 20409, + "claire": 20410, + "emmanuel": 20411, + "moisture": 20412, + "digest": 20413, + "viii": 20414, + "teller": 20415, + "chon": 20416, + "accessory": 20417, + "nightclub": 20418, + "fossil": 20419, + "awan": 20420, + "husky": 20421, + "aboriginal": 20422, + "brandon": 20423, + "fficient": 20424, + "cougars": 20425, + "sted": 20426, + "admitted": 20427, + "ignored": 20428, + "contentmarketing": 20429, + "agas": 20430, + "vase": 20431, + "executed": 20432, + "negotiations": 20433, + "shead": 20434, + "nand": 20435, + "tablets": 20436, + "goth": 20437, + "tsal": 20438, + "dfw": 20439, + "onep": 20440, + "protector": 20441, + "spho": 20442, + "gazette": 20443, + "andreas": 20444, + "sser": 20445, + "compilation": 20446, + "hav": 20447, + "containers": 20448, + "broker": 20449, + "socal": 20450, + "porcelain": 20451, + "hyuk": 20452, + "airing": 20453, + "ðŁĴ°": 20454, + "publisher": 20455, + "scenario": 20456, + "spartans": 20457, + "reviewing": 20458, + "itudes": 20459, + "edel": 20460, + "pearson": 20461, + "bash": 20462, + "maui": 20463, + "aad": 20464, + "ðŁĮĬ": 20465, + "liu": 20466, + "ulate": 20467, + "programmes": 20468, + "favour": 20469, + "webdesign": 20470, + "realty": 20471, + "motivational": 20472, + "crosses": 20473, + "'...": 20474, + "busch": 20475, + "adjustable": 20476, + "arjun": 20477, + "mistak": 20478, + "dimension": 20479, + "pistol": 20480, + "weighs": 20481, + "eny": 20482, + "unveil": 20483, + "indycar": 20484, + "gordon": 20485, + "fade": 20486, + "franken": 20487, + "qualities": 20488, + "bett": 20489, + "locate": 20490, + "kerr": 20491, + "spc": 20492, + "confusion": 20493, + "nee": 20494, + "lucky": 20495, + "bases": 20496, + "depends": 20497, + "firefighter": 20498, + "ola": 20499, + "ret": 20500, + "maroon": 20501, + "ðŁĶĬ": 20502, + "wam": 20503, + "defining": 20504, + "wheat": 20505, + "bil": 20506, + "és": 20507, + "bhai": 20508, + "psych": 20509, + "tau": 20510, + "icans": 20511, + "thik": 20512, + "obile": 20513, + "inspector": 20514, + "ìĨĮë": 20515, + "illon": 20516, + "gos": 20517, + "evangel": 20518, + "fai": 20519, + "sist": 20520, + "vocation": 20521, + "burge": 20522, + "chistan": 20523, + "renewed": 20524, + "enthusiasm": 20525, + "enting": 20526, + "agri": 20527, + "ikea": 20528, + "msc": 20529, + "aerospace": 20530, + "sensiti": 20531, + "memoir": 20532, + "hospice": 20533, + "cocaine": 20534, + "derry": 20535, + "mechanics": 20536, + "Ħà¸": 20537, + "tino": 20538, + "reduces": 20539, + "collectors": 20540, + "injustice": 20541, + "suppre": 20542, + "vana": 20543, + "abun": 20544, + "napa": 20545, + "susa": 20546, + "oslo": 20547, + "eff": 20548, + "encore": 20549, + "licence": 20550, + "cheddar": 20551, + "zal": 20552, + "mount": 20553, + "ðŁĴIJ": 20554, + "threatens": 20555, + "!!\"": 20556, + "archie": 20557, + "futsal": 20558, + "scuba": 20559, + "jos": 20560, + "gnon": 20561, + "sexi": 20562, + "sofficial": 20563, + "comparing": 20564, + "dominant": 20565, + "toftheday": 20566, + "fait": 20567, + "proposals": 20568, + "gift": 20569, + "yas": 20570, + "cnc": 20571, + "lr": 20572, + "hab": 20573, + "reservoir": 20574, + "beliefs": 20575, + "general": 20576, + "marti": 20577, + "td": 20578, + "este": 20579, + "ìł": 20580, + "wil": 20581, + "ðŁij¯": 20582, + "ðŁĶ«": 20583, + "spx": 20584, + "etwork": 20585, + "excerpt": 20586, + "einstein": 20587, + "hiro": 20588, + "silhou": 20589, + "teamed": 20590, + "perception": 20591, + "corridor": 20592, + "mentalhealth": 20593, + "hints": 20594, + "benny": 20595, + "inducted": 20596, + "swx": 20597, + "widesp": 20598, + "speak": 20599, + "cheryl": 20600, + "drug": 20601, + "ðŁĺķ": 20602, + "hf": 20603, + "asparagus": 20604, + "mysteries": 20605, + "fitzgerald": 20606, + "offer": 20607, + "therapist": 20608, + "career": 20609, + "damaging": 20610, + "tsd": 20611, + "peru": 20612, + "weibo": 20613, + "yay": 20614, + "phoenix": 20615, + "discre": 20616, + "macbook": 20617, + "barker": 20618, + "stigma": 20619, + "spread": 20620, + "rockies": 20621, + "kangar": 20622, + "bridg": 20623, + "pai": 20624, + "bishop": 20625, + "tailed": 20626, + "capsule": 20627, + "ðŁĴĵ": 20628, + "geof": 20629, + "royale": 20630, + "shortlisted": 20631, + "oste": 20632, + "ashamed": 20633, + "chapp": 20634, + "keye": 20635, + "cla": 20636, + "screenshot": 20637, + "austrian": 20638, + "native": 20639, + "enight": 20640, + "juliet": 20641, + "michele": 20642, + "ðŁĮ´": 20643, + "travelers": 20644, + "pil": 20645, + "footballer": 20646, + "winchester": 20647, + "ðŁĻĦ": 20648, + "azerbai": 20649, + "goldeng": 20650, + "organisations": 20651, + "interpretation": 20652, + "predator": 20653, + "oftheweek": 20654, + "logan": 20655, + "poké": 20656, + "marie": 20657, + "calla": 20658, + "tnt": 20659, + "cinde": 20660, + "getic": 20661, + "fitfam": 20662, + "grav": 20663, + "owens": 20664, + "ðŁĮ±": 20665, + "shootout": 20666, + "salis": 20667, + "commissions": 20668, + "cohe": 20669, + "ptic": 20670, + "nixon": 20671, + "hia": 20672, + "ambition": 20673, + "marine": 20674, + "cruelty": 20675, + "tk": 20676, + "crude": 20677, + "salty": 20678, + "jima": 20679, + "mongo": 20680, + "irony": 20681, + "onwards": 20682, + "arrests": 20683, + "strangers": 20684, + "iger": 20685, + "cyclist": 20686, + "rag": 20687, + "extends": 20688, + "tradio": 20689, + "bourg": 20690, + "moi": 20691, + "ella": 20692, + "eable": 20693, + "lexus": 20694, + "aul": 20695, + "dera": 20696, + "historian": 20697, + "morton": 20698, + "tiff": 20699, + "manner": 20700, + "kot": 20701, + "dk": 20702, + "pointed": 20703, + "marqu": 20704, + "aan": 20705, + "eney": 20706, + "dublin": 20707, + "onpoli": 20708, + "emili": 20709, + "secret": 20710, + "flo": 20711, + "âļ¡": 20712, + "baj": 20713, + "steep": 20714, + "accompanied": 20715, + "rumours": 20716, + "devi": 20717, + "purchasing": 20718, + "fig": 20719, + "pub": 20720, + "schoo": 20721, + "autonomous": 20722, + "goalie": 20723, + "xia": 20724, + "automatically": 20725, + "revers": 20726, + "tero": 20727, + "fuku": 20728, + "titanic": 20729, + "shook": 20730, + "sandals": 20731, + "seekers": 20732, + "excav": 20733, + "nordic": 20734, + "bigolive": 20735, + "bake": 20736, + "ratt": 20737, + "zak": 20738, + "nep": 20739, + "ðŁĺ¤": 20740, + "candy": 20741, + "billions": 20742, + "bookworm": 20743, + "ppet": 20744, + "à³": 20745, + "surfaces": 20746, + "scars": 20747, + "philip": 20748, + "dogg": 20749, + "cigars": 20750, + "cote": 20751, + "translated": 20752, + "curator": 20753, + "sindh": 20754, + "hangover": 20755, + "brewer": 20756, + "ones": 20757, + "elton": 20758, + "ðŁĴªðŁı¼": 20759, + "marcu": 20760, + "elliot": 20761, + "righte": 20762, + "dioce": 20763, + "russ": 20764, + "railways": 20765, + "grandson": 20766, + "ascen": 20767, + "apology": 20768, + "await": 20769, + "mobili": 20770, + "respir": 20771, + "partisan": 20772, + "olivi": 20773, + "strike": 20774, + "yoo": 20775, + "whitehouse": 20776, + "expressed": 20777, + "pups": 20778, + "bedford": 20779, + "cultur": 20780, + "frogs": 20781, + "flying": 20782, + "cavali": 20783, + "cds": 20784, + "friger": 20785, + "streetphotography": 20786, + "resolve": 20787, + "taliban": 20788, + "kang": 20789, + "crushing": 20790, + "jum": 20791, + "ðŁĺĴ": 20792, + "williamson": 20793, + "tang": 20794, + "curly": 20795, + "tman": 20796, + "veteran": 20797, + "faire": 20798, + "artificialintelligence": 20799, + "unanim": 20800, + "pren": 20801, + "backdrop": 20802, + "frances": 20803, + "occer": 20804, + "dorothy": 20805, + "working": 20806, + "arthr": 20807, + "converted": 20808, + "daylight": 20809, + "servant": 20810, + "paddle": 20811, + "complaining": 20812, + "thirty": 20813, + "nadal": 20814, + "aku": 20815, + "ibrahim": 20816, + "addressed": 20817, + "piss": 20818, + "greenhouse": 20819, + "battalion": 20820, + "simulator": 20821, + "outlets": 20822, + "embroidery": 20823, + "ðŁĵ±": 20824, + "fiscal": 20825, + "gerard": 20826, + "sassy": 20827, + "ðŁİīðŁİīðŁİī": 20828, + "ventures": 20829, + "merit": 20830, + "publicity": 20831, + "ðŁijĪ": 20832, + "sophisticated": 20833, + "ctu": 20834, + "conventional": 20835, + "condolences": 20836, + "israel": 20837, + "tradition": 20838, + "aran": 20839, + "tess": 20840, + "glad": 20841, + "ðŁĺĬðŁĺĬ": 20842, + "correction": 20843, + "geon": 20844, + "amd": 20845, + "orship": 20846, + "beast": 20847, + "chment": 20848, + "ìŀ": 20849, + "nico": 20850, + "wknd": 20851, + "wels": 20852, + "cushion": 20853, + "belie": 20854, + "voc": 20855, + "idiots": 20856, + "underneath": 20857, + "puma": 20858, + "cornell": 20859, + "enation": 20860, + "lul": 20861, + "swach": 20862, + "abig": 20863, + "urer": 20864, + "mie": 20865, + "formerly": 20866, + "caf": 20867, + "ernal": 20868, + "chorus": 20869, + "julius": 20870, + "senator": 20871, + "âľį": 20872, + "whir": 20873, + "salvador": 20874, + "phd": 20875, + "unified": 20876, + "booster": 20877, + "graphical": 20878, + "wrec": 20879, + "sonny": 20880, + "miz": 20881, + "derers": 20882, + "sall": 20883, + "vens": 20884, + "tuscany": 20885, + "wid": 20886, + "yong": 20887, + "kurds": 20888, + "waz": 20889, + "trolls": 20890, + "macro": 20891, + "caturday": 20892, + "pressing": 20893, + "sasha": 20894, + "centennial": 20895, + "gusts": 20896, + "emc": 20897, + "before": 20898, + "denise": 20899, + "cust": 20900, + "ðŁĵ¢": 20901, + "looo": 20902, + "basel": 20903, + "england": 20904, + "yolo": 20905, + "ardu": 20906, + "manifesto": 20907, + "doha": 20908, + "ìľ": 20909, + "knives": 20910, + "bournemouth": 20911, + "bibl": 20912, + "barb": 20913, + "alicia": 20914, + "Ø©": 20915, + "comer": 20916, + "cyclone": 20917, + "git": 20918, + "anews": 20919, + "characteri": 20920, + "ventura": 20921, + "intra": 20922, + "sfgiants": 20923, + "hut": 20924, + "bea": 20925, + "darwin": 20926, + "eller": 20927, + "alv": 20928, + "reese": 20929, + "bly": 20930, + "karan": 20931, + "conclusion": 20932, + "manny": 20933, + "flakes": 20934, + "uniteblue": 20935, + "nadu": 20936, + "copp": 20937, + "edges": 20938, + "lancashire": 20939, + "ials": 20940, + "otta": 20941, + "philippe": 20942, + "lent": 20943, + "chee": 20944, + "mentors": 20945, + "festival": 20946, + "anism": 20947, + "complimentary": 20948, + "rj": 20949, + "pug": 20950, + "dine": 20951, + "wei": 20952, + "cliffs": 20953, + "sarmy": 20954, + "tiveness": 20955, + "treasury": 20956, + "iland": 20957, + "aftermath": 20958, + "rabbi": 20959, + "oun": 20960, + "bouquet": 20961, + "heritage": 20962, + "zion": 20963, + "surrender": 20964, + "shenan": 20965, + "inks": 20966, + "karl": 20967, + "ghty": 20968, + "policing": 20969, + "examination": 20970, + "cey": 20971, + "persu": 20972, + "measurement": 20973, + "hydrogen": 20974, + "luhan": 20975, + "âłĢâłĢâłĢâłĢ": 20976, + "wari": 20977, + "оÐ": 20978, + "jy": 20979, + "fowler": 20980, + "mish": 20981, + "alfre": 20982, + "âĺij": 20983, + "bbnaija": 20984, + "catalogue": 20985, + "recognised": 20986, + "saver": 20987, + "huskies": 20988, + "colin": 20989, + "mundo": 20990, + "siva": 20991, + "png": 20992, + "discounted": 20993, + "manutd": 20994, + "fresno": 20995, + "devin": 20996, + "preliminary": 20997, + "trophies": 20998, + "plastics": 20999, + "dug": 21000, + "procu": 21001, + "indigo": 21002, + "gard": 21003, + "dylan": 21004, + "pitches": 21005, + "groundbreaking": 21006, + "inson": 21007, + "blac": 21008, + "anthology": 21009, + "fh": 21010, + "explic": 21011, + "rard": 21012, + "admiral": 21013, + "sochi": 21014, + "lashes": 21015, + "splendid": 21016, + "envy": 21017, + "adv": 21018, + "sexy": 21019, + "festivities": 21020, + "sticking": 21021, + "bib": 21022, + "thrill": 21023, + "opp": 21024, + "ariel": 21025, + "botanical": 21026, + "endurance": 21027, + "females": 21028, + "bricks": 21029, + "vatican": 21030, + "blackpool": 21031, + "bermu": 21032, + "brough": 21033, + "roller": 21034, + "bid": 21035, + "suede": 21036, + "slovenia": 21037, + "mming": 21038, + "mlb": 21039, + "medalist": 21040, + "dians": 21041, + "rehabilitation": 21042, + "neon": 21043, + "sgo": 21044, + "lithu": 21045, + "ramos": 21046, + "zed": 21047, + "pianist": 21048, + "intensive": 21049, + "broadband": 21050, + "study": 21051, + "petersburg": 21052, + "luca": 21053, + "ahhhh": 21054, + "physician": 21055, + "dillon": 21056, + "telecom": 21057, + "grief": 21058, + "mun": 21059, + "acro": 21060, + "sided": 21061, + "sly": 21062, + "blows": 21063, + "classiccars": 21064, + "trium": 21065, + "argy": 21066, + "?:": 21067, + "hri": 21068, + "marshmal": 21069, + "âĢĵ": 21070, + "topping": 21071, + "warsaw": 21072, + "transc": 21073, + "preservation": 21074, + "bav": 21075, + "refriger": 21076, + "experiments": 21077, + "äº": 21078, + "glit": 21079, + "sliga": 21080, + "gage": 21081, + "factor": 21082, + "flavours": 21083, + "brony": 21084, + "spo": 21085, + "cookbook": 21086, + "carriage": 21087, + "away": 21088, + "nyfw": 21089, + "onian": 21090, + "wg": 21091, + "simpsons": 21092, + "rolex": 21093, + "ðŁı¿": 21094, + "crosby": 21095, + "ãħ¤": 21096, + "credi": 21097, + "syndic": 21098, + "pubs": 21099, + "alife": 21100, + "poorly": 21101, + "maced": 21102, + "ðŁĺŀ": 21103, + "behindthe": 21104, + "wenger": 21105, + "nats": 21106, + "ðŁİŁ": 21107, + "rubbish": 21108, + "procedures": 21109, + "typhoon": 21110, + "ophobia": 21111, + "erdo": 21112, + "fuel": 21113, + "viera": 21114, + "bumps": 21115, + "millennium": 21116, + "newzealand": 21117, + "lectures": 21118, + "iton": 21119, + "milky": 21120, + "responded": 21121, + "ê°": 21122, + "landscape": 21123, + "..@": 21124, + "bother": 21125, + "âĸ¶": 21126, + "zhang": 21127, + "huawei": 21128, + "tuition": 21129, + "sworn": 21130, + "inu": 21131, + "yor": 21132, + "paolo": 21133, + "auditions": 21134, + "abil": 21135, + "malaysian": 21136, + "hops": 21137, + "feathers": 21138, + "mple": 21139, + "auts": 21140, + "ão": 21141, + "bounty": 21142, + "iche": 21143, + "ìĺ": 21144, + "shq": 21145, + "pinot": 21146, + "gears": 21147, + "disappear": 21148, + "videogames": 21149, + "tna": 21150, + "alzheimer": 21151, + "ðŁĮŀ": 21152, + "aji": 21153, + "underwear": 21154, + "switching": 21155, + "signage": 21156, + "oscar": 21157, + "econ": 21158, + "drow": 21159, + "clint": 21160, + "plated": 21161, + "gundy": 21162, + "emblem": 21163, + "hoes": 21164, + "icist": 21165, + "nelly": 21166, + "junior": 21167, + "roadshow": 21168, + "minerals": 21169, + "atle": 21170, + "alexandria": 21171, + "acclaimed": 21172, + "vell": 21173, + "shiva": 21174, + "adhe": 21175, + "enne": 21176, + "amnesty": 21177, + "hounds": 21178, + "councillor": 21179, + "ðŁĴ¦": 21180, + "aesthe": 21181, + "partnering": 21182, + "influenced": 21183, + "magno": 21184, + "flare": 21185, + "extinction": 21186, + "civilian": 21187, + "majesty": 21188, + "vail": 21189, + "lawmakers": 21190, + "racks": 21191, + "mcc": 21192, + "orian": 21193, + "spices": 21194, + "errors": 21195, + "mayer": 21196, + "coca": 21197, + "pai": 21198, + "sooooo": 21199, + "retiring": 21200, + "bathro": 21201, + "ðŁĻĮðŁĻĮ": 21202, + "âĸª": 21203, + "suf": 21204, + "endorsement": 21205, + "building": 21206, + "brooch": 21207, + "palla": 21208, + "arvind": 21209, + "agent": 21210, + "karate": 21211, + "rhi": 21212, + "ctv": 21213, + "taine": 21214, + "umm": 21215, + "bax": 21216, + "reigns": 21217, + "uniof": 21218, + "enterprises": 21219, + "adele": 21220, + "flake": 21221, + "attire": 21222, + "bruce": 21223, + "bahamas": 21224, + "gravy": 21225, + "sain": 21226, + "cheek": 21227, + "trivi": 21228, + "lov": 21229, + "een": 21230, + "bblo": 21231, + "ladygaga": 21232, + "itta": 21233, + ".\"-": 21234, + "dustin": 21235, + "observatory": 21236, + "eighth": 21237, + "bloomberg": 21238, + "khs": 21239, + "fcc": 21240, + "gist": 21241, + "commemorate": 21242, + "veer": 21243, + "sexuality": 21244, + "edc": 21245, + "nicole": 21246, + "vacancy": 21247, + "user": 21248, + "sona": 21249, + ":'(": 21250, + "diploma": 21251, + "tend": 21252, + "upgrades": 21253, + "ÅŁ": 21254, + "jurassic": 21255, + "cardiac": 21256, + "drs": 21257, + "widespread": 21258, + "Ãł": 21259, + "dailies": 21260, + "vendor": 21261, + "simplicity": 21262, + "wider": 21263, + "lenses": 21264, + "supplements": 21265, + "depos": 21266, + "observed": 21267, + "vines": 21268, + "partially": 21269, + "renewal": 21270, + "collaborate": 21271, + "alig": 21272, + "finity": 21273, + "phu": 21274, + "zzy": 21275, + "petit": 21276, + "ðŁĵħ": 21277, + "zin": 21278, + "igu": 21279, + "smack": 21280, + "fallon": 21281, + "ðŁĵ£": 21282, + "backwards": 21283, + "component": 21284, + "oso": 21285, + "compatible": 21286, + "binding": 21287, + "zurich": 21288, + "thome": 21289, + "wounds": 21290, + "lyric": 21291, + "freshmen": 21292, + "sneaky": 21293, + "fibro": 21294, + "diet": 21295, + "employer": 21296, + "insect": 21297, + "hated": 21298, + "scher": 21299, + "razor": 21300, + "nsw": 21301, + "booker": 21302, + "californi": 21303, + "avfc": 21304, + "°": 21305, + "pretending": 21306, + "pepsi": 21307, + "alis": 21308, + "untitled": 21309, + "kart": 21310, + "grandparents": 21311, + "ethe": 21312, + "ock": 21313, + "luxemb": 21314, + "visuals": 21315, + "smallbusiness": 21316, + "abdullah": 21317, + "minho": 21318, + "subaru": 21319, + "hra": 21320, + "revealing": 21321, + "heartbreaking": 21322, + "clarity": 21323, + "amg": 21324, + "slr": 21325, + "****": 21326, + "âŀĸ": 21327, + "record": 21328, + "iciary": 21329, + "minded": 21330, + "yeh": 21331, + "excessive": 21332, + "knuck": 21333, + "icecream": 21334, + "truth": 21335, + "evic": 21336, + "tastic": 21337, + "antarc": 21338, + "rendering": 21339, + ",,": 21340, + "mitt": 21341, + "lorenzo": 21342, + "stpatrick": 21343, + "boundary": 21344, + "zig": 21345, + "vocab": 21346, + "osaka": 21347, + "furn": 21348, + "tun": 21349, + "gul": 21350, + "sounding": 21351, + "blogger": 21352, + "utterly": 21353, + "gaf": 21354, + "advancing": 21355, + "lcd": 21356, + "margin": 21357, + "lifelong": 21358, + "solstice": 21359, + "shra": 21360, + "waits": 21361, + "plear": 21362, + "breach": 21363, + "enligh": 21364, + "ader": 21365, + "ittle": 21366, + "cation": 21367, + "hoon": 21368, + "studied": 21369, + "?????": 21370, + "kash": 21371, + "evangeli": 21372, + "psl": 21373, + "weights": 21374, + "metals": 21375, + "tyres": 21376, + "turno": 21377, + "wie": 21378, + "carb": 21379, + "gale": 21380, + "seal": 21381, + "sunite": 21382, + "amic": 21383, + "patterson": 21384, + "án": 21385, + "euph": 21386, + "upstairs": 21387, + "qualifiers": 21388, + "khalifa": 21389, + "applemusic": 21390, + "ìĨĮëħ": 21391, + "vaughan": 21392, + "alter": 21393, + "cruiser": 21394, + "mua": 21395, + "tana": 21396, + "katrina": 21397, + "idols": 21398, + "spoiled": 21399, + "secretly": 21400, + "fibre": 21401, + "partnered": 21402, + "umes": 21403, + "giov": 21404, + "comet": 21405, + "screenshotsaturday": 21406, + "keller": 21407, + "filtr": 21408, + "fet": 21409, + "conway": 21410, + "peu": 21411, + "badminton": 21412, + "gid": 21413, + "mound": 21414, + "donkey": 21415, + "buff": 21416, + "leather": 21417, + "largely": 21418, + "broch": 21419, + "intments": 21420, + "amuse": 21421, + "rk": 21422, + "stove": 21423, + "impacted": 21424, + "cont": 21425, + "cracks": 21426, + "prisoner": 21427, + "bari": 21428, + "contractor": 21429, + "orioles": 21430, + "dominate": 21431, + "polar": 21432, + "amelia": 21433, + "drc": 21434, + "ðŁijĮðŁijĮ": 21435, + "vist": 21436, + "suarez": 21437, + "injection": 21438, + "blooms": 21439, + "ðŁļ¨ðŁļ¨": 21440, + "stiff": 21441, + "paypal": 21442, + "snowing": 21443, + "thursdays": 21444, + "goose": 21445, + "wedge": 21446, + "educated": 21447, + "weakness": 21448, + "decker": 21449, + "abudha": 21450, + "breezy": 21451, + "ÛĮ": 21452, + "hopeful": 21453, + "obi": 21454, + "raider": 21455, + "gham": 21456, + "deu": 21457, + "seve": 21458, + "partly": 21459, + "fut": 21460, + "infused": 21461, + "merri": 21462, + "thane": 21463, + "sometime": 21464, + "hue": 21465, + "mein": 21466, + "credit": 21467, + "sliding": 21468, + "rande": 21469, + "cherry": 21470, + "deadpool": 21471, + "shol": 21472, + "aram": 21473, + "underwood": 21474, + "skye": 21475, + "disturbing": 21476, + "mnt": 21477, + "polished": 21478, + "guardians": 21479, + "hadn": 21480, + "picasso": 21481, + "arius": 21482, + "akshay": 21483, + "irri": 21484, + "jh": 21485, + "happen": 21486, + "lakh": 21487, + "dalton": 21488, + "atthe": 21489, + "swell": 21490, + "marsha": 21491, + "reh": 21492, + "cours": 21493, + "jkt": 21494, + "topus": 21495, + "service": 21496, + "rink": 21497, + "hackers": 21498, + "donovan": 21499, + "horo": 21500, + "tcm": 21501, + "mayhem": 21502, + "chase": 21503, + "devops": 21504, + "kensing": 21505, + "scup": 21506, + "shere": 21507, + "qualification": 21508, + "clive": 21509, + "tong": 21510, + "nancy": 21511, + "maris": 21512, + "derdale": 21513, + "berman": 21514, + "cinderella": 21515, + "jolly": 21516, + "cic": 21517, + "loot": 21518, + "collectibles": 21519, + "homicide": 21520, + "gge": 21521, + "epidemic": 21522, + "suites": 21523, + "muddy": 21524, + "gimme": 21525, + "erec": 21526, + "-*": 21527, + "talla": 21528, + "lisle": 21529, + "embroide": 21530, + "ðŁĩ©ðŁĩª": 21531, + "verizon": 21532, + "vector": 21533, + "beanie": 21534, + "artisan": 21535, + "gain": 21536, + "flores": 21537, + "vigil": 21538, + "uso": 21539, + "ðŁĻıðŁı½": 21540, + "grinding": 21541, + "gher": 21542, + "airports": 21543, + "responsive": 21544, + "shaft": 21545, + "cancel": 21546, + "ceremonies": 21547, + "eme": 21548, + "atari": 21549, + "brushes": 21550, + "eager": 21551, + "bohemi": 21552, + "childrens": 21553, + "yankee": 21554, + "maa": 21555, + "suspense": 21556, + "moran": 21557, + "macar": 21558, + "sunflower": 21559, + "crew": 21560, + "void": 21561, + "kear": 21562, + "fashioned": 21563, + "jennings": 21564, + "sundayfunday": 21565, + "submissions": 21566, + "mead": 21567, + "herman": 21568, + "wai": 21569, + "critically": 21570, + "leum": 21571, + "baekhyun": 21572, + "forcing": 21573, + "cobra": 21574, + "ãģ®": 21575, + "acquire": 21576, + "alk": 21577, + "geology": 21578, + "primar": 21579, + "importantly": 21580, + "irez": 21581, + "bundesliga": 21582, + "curiosity": 21583, + "sena": 21584, + "strict": 21585, + "consoli": 21586, + "winters": 21587, + "venom": 21588, + "cheltenham": 21589, + "ðŁįº": 21590, + "cena": 21591, + "tat": 21592, + "bain": 21593, + "glover": 21594, + "undercover": 21595, + "asses": 21596, + "carn": 21597, + "memorialday": 21598, + "ameli": 21599, + "irene": 21600, + "chon": 21601, + "synthesis": 21602, + "speedy": 21603, + "mitsubi": 21604, + "slayer": 21605, + "composite": 21606, + "understands": 21607, + "pew": 21608, + "interrup": 21609, + "henri": 21610, + "morrow": 21611, + "anom": 21612, + "thofjuly": 21613, + "glee": 21614, + "three": 21615, + "ðŁĺ®": 21616, + "andhi": 21617, + "chatt": 21618, + "renewables": 21619, + "yes": 21620, + "transfers": 21621, + "!!!!!!!!": 21622, + "babu": 21623, + "duter": 21624, + "loops": 21625, + "peers": 21626, + "oilers": 21627, + "paulo": 21628, + "ication": 21629, + "hmu": 21630, + "wara": 21631, + "mercer": 21632, + "homeland": 21633, + "fuji": 21634, + "aley": 21635, + "yearbook": 21636, + "rem": 21637, + "reen": 21638, + "absur": 21639, + "bois": 21640, + "]:": 21641, + "caesar": 21642, + "shotgun": 21643, + "kurdish": 21644, + "oren": 21645, + "rae": 21646, + "ancies": 21647, + "typic": 21648, + "fh": 21649, + "default": 21650, + "replic": 21651, + "luk": 21652, + "transactions": 21653, + "rys": 21654, + "infantry": 21655, + "ðŁį¾": 21656, + "chow": 21657, + "chickens": 21658, + "bagh": 21659, + "wyatt": 21660, + "aye": 21661, + "ggi": 21662, + "brews": 21663, + "editions": 21664, + "mira": 21665, + "commencement": 21666, + "presu": 21667, + "periscope": 21668, + "ichi": 21669, + "guatemala": 21670, + "zambia": 21671, + "paints": 21672, + "witches": 21673, + "wani": 21674, + "undere": 21675, + "croy": 21676, + "vows": 21677, + "usmc": 21678, + "hearted": 21679, + "theatres": 21680, + "shuffle": 21681, + "level": 21682, + "multic": 21683, + "squeeze": 21684, + "fern": 21685, + "appet": 21686, + "postal": 21687, + "malt": 21688, + "onboard": 21689, + "ldnt": 21690, + "coo": 21691, + "ssc": 21692, + "kac": 21693, + "ðŁĺĩ": 21694, + "scrap": 21695, + "marcos": 21696, + "dealers": 21697, + "annu": 21698, + "miller": 21699, + "cove": 21700, + "ulary": 21701, + "vladimir": 21702, + "beef": 21703, + "thur": 21704, + "pickled": 21705, + "sesame": 21706, + "bengaluru": 21707, + "mott": 21708, + "kathleen": 21709, + "hist": 21710, + "notor": 21711, + "drank": 21712, + "duchess": 21713, + "snowfall": 21714, + "eff": 21715, + "tiny": 21716, + "jn": 21717, + "syour": 21718, + "specialists": 21719, + "scotus": 21720, + "baylor": 21721, + "everest": 21722, + "malibu": 21723, + "prem": 21724, + "harmful": 21725, + "lali": 21726, + "bates": 21727, + "gye": 21728, + "differenti": 21729, + "andra": 21730, + "geometry": 21731, + "elover": 21732, + "blackout": 21733, + "====": 21734, + "kota": 21735, + "interact": 21736, + "asian": 21737, + "layo": 21738, + "samurai": 21739, + "fidel": 21740, + "exhausted": 21741, + "gladi": 21742, + "pdt": 21743, + "spheric": 21744, + "antiqu": 21745, + "guitar": 21746, + "sturi": 21747, + "hopper": 21748, + "angle": 21749, + "fills": 21750, + "slap": 21751, + "mith": 21752, + "rodney": 21753, + "ongi": 21754, + "insom": 21755, + "preventing": 21756, + "cassidy": 21757, + "apho": 21758, + "oregon": 21759, + "loin": 21760, + "hammond": 21761, + "contributing": 21762, + "fn": 21763, + "garri": 21764, + "orion": 21765, + "compelling": 21766, + "escaping": 21767, + "aiming": 21768, + "plumb": 21769, + "bistro": 21770, + "beasts": 21771, + "concerning": 21772, + "boe": 21773, + "dopp": 21774, + "shoplocal": 21775, + "stumbled": 21776, + "âĤ¹": 21777, + "nazis": 21778, + "âĢįâĻĤï¸ı": 21779, + "gesture": 21780, + "warts": 21781, + "usopen": 21782, + "higgins": 21783, + "charli": 21784, + "hangs": 21785, + "bombers": 21786, + "°:": 21787, + "feeds": 21788, + "cch": 21789, + "stil": 21790, + "nicola": 21791, + "ðŁĵº": 21792, + "clamation": 21793, + "tropic": 21794, + "afro": 21795, + "ouk": 21796, + "expenses": 21797, + "derrick": 21798, + "aline": 21799, + "faw": 21800, + "regard": 21801, + "imer": 21802, + "satin": 21803, + "thium": 21804, + "ryder": 21805, + "pearl": 21806, + "tess": 21807, + "mmmmm": 21808, + "senses": 21809, + "ðŁĩ¹": 21810, + "positive": 21811, + "exhaust": 21812, + "occur": 21813, + "norris": 21814, + "lilly": 21815, + "isles": 21816, + "directing": 21817, + "yofficial": 21818, + "countless": 21819, + "samar": 21820, + "onstage": 21821, + "flock": 21822, + "mirrors": 21823, + "archer": 21824, + "moi": 21825, + "kd": 21826, + "viv": 21827, + "inos": 21828, + "sikh": 21829, + "lei": 21830, + "sensory": 21831, + "brits": 21832, + "knox": 21833, + "chestnut": 21834, + "opy": 21835, + "coliseum": 21836, + "zaf": 21837, + "divin": 21838, + "adapter": 21839, + ":)))": 21840, + "temple": 21841, + "kun": 21842, + "helmets": 21843, + "tdf": 21844, + "guide": 21845, + "mold": 21846, + "oids": 21847, + "luther": 21848, + "heis": 21849, + "monastery": 21850, + "spree": 21851, + "klu": 21852, + "britney": 21853, + "jaguars": 21854, + "greats": 21855, + "ccc": 21856, + "kyrie": 21857, + "machinery": 21858, + "cricket": 21859, + "rero": 21860, + "abo": 21861, + "aspiring": 21862, + "semifinals": 21863, + "aless": 21864, + "signatures": 21865, + "vard": 21866, + "meth": 21867, + "herbal": 21868, + "holden": 21869, + "kingdom": 21870, + "apor": 21871, + "reggie": 21872, + "oreo": 21873, + "palestinians": 21874, + "emmys": 21875, + "sectional": 21876, + "roi": 21877, + "neymar": 21878, + "quel": 21879, + "cull": 21880, + "lka": 21881, + "hazel": 21882, + "estimate": 21883, + "ulties": 21884, + "gow": 21885, + "bea": 21886, + "purchases": 21887, + "belts": 21888, + "protects": 21889, + "mé": 21890, + "guessing": 21891, + "bbo": 21892, + "claudia": 21893, + "fracking": 21894, + "jonny": 21895, + "elk": 21896, + "celtic": 21897, + "almighty": 21898, + "raje": 21899, + "courtyard": 21900, + "igi": 21901, + "canes": 21902, + "ðŁĴªðŁı»": 21903, + "bankrup": 21904, + "lethal": 21905, + "âľĮï¸ı": 21906, + "graphicdesign": 21907, + "vader": 21908, + "pencils": 21909, + "roughly": 21910, + "dante": 21911, + "mfg": 21912, + "constell": 21913, + "camel": 21914, + "jb": 21915, + "blossoms": 21916, + "ento": 21917, + "balochistan": 21918, + "cinemato": 21919, + "illard": 21920, + "jersey": 21921, + "consent": 21922, + "dented": 21923, + "contempl": 21924, + "scher": 21925, + "holi": 21926, + "lough": 21927, + "stour": 21928, + "ayo": 21929, + "beginners": 21930, + "curb": 21931, + "vhs": 21932, + "ajax": 21933, + "duff": 21934, + "aveng": 21935, + "domest": 21936, + "committing": 21937, + "aired": 21938, + "chap": 21939, + "hedgehog": 21940, + "disappointing": 21941, + "freelance": 21942, + "inland": 21943, + "charms": 21944, + "ðŁĺįâĿ¤ï¸ı": 21945, + "aish": 21946, + "mx": 21947, + "buckle": 21948, + "tidal": 21949, + "permit": 21950, + "boating": 21951, + "racha": 21952, + "kendrick": 21953, + "bello": 21954, + "bhi": 21955, + "plea": 21956, + "estimates": 21957, + "lb": 21958, + "apologies": 21959, + "jaya": 21960, + "bbl": 21961, + "astoni": 21962, + "interstate": 21963, + "maintaining": 21964, + "elbow": 21965, + "mup": 21966, + "epit": 21967, + "ðŁĺ¡": 21968, + "violations": 21969, + "defend": 21970, + "beh": 21971, + "slc": 21972, + "amir": 21973, + "puri": 21974, + "tium": 21975, + "fifa": 21976, + "blurry": 21977, + "scrim": 21978, + "ðŁĻıðŁı¾": 21979, + "maple": 21980, + "relatives": 21981, + "âĺĿ": 21982, + "choc": 21983, + "connor": 21984, + "⾨⾨": 21985, + "whisp": 21986, + "listings": 21987, + "maze": 21988, + "thanking": 21989, + "ridd": 21990, + "grassroots": 21991, + "shifting": 21992, + "desperately": 21993, + "gorilla": 21994, + "deni": 21995, + "jules": 21996, + "strath": 21997, + "gley": 21998, + "jain": 21999, + "buick": 22000, + "tanner": 22001, + "ðŁĴĿ": 22002, + "gae": 22003, + "prim": 22004, + "itors": 22005, + "nano": 22006, + "separation": 22007, + "armenia": 22008, + "bordeaux": 22009, + "ðŁħ": 22010, + "pjnet": 22011, + "burial": 22012, + "ebon": 22013, + "gloss": 22014, + "renew": 22015, + "grier": 22016, + "speeds": 22017, + "comicbooks": 22018, + "symboli": 22019, + "purposes": 22020, + "ãħłãħł": 22021, + "spatial": 22022, + "notable": 22023, + "cion": 22024, + "nps": 22025, + "hoffman": 22026, + "norman": 22027, + "rtg": 22028, + "dusty": 22029, + "situated": 22030, + "tran": 22031, + "kfc": 22032, + "emen": 22033, + "nickel": 22034, + "hastings": 22035, + "settling": 22036, + "grit": 22037, + "lena": 22038, + "waw": 22039, + "arts": 22040, + "gum": 22041, + "caregi": 22042, + "lewis": 22043, + "sapphire": 22044, + "remember": 22045, + "embedded": 22046, + "tlc": 22047, + "blat": 22048, + "sergeant": 22049, + "elsa": 22050, + "bootcamp": 22051, + "bowman": 22052, + "photographic": 22053, + "pillars": 22054, + "directioners": 22055, + "classified": 22056, + "nois": 22057, + "veer": 22058, + "barrels": 22059, + "whoop": 22060, + "ðŁĺ±ðŁĺ±": 22061, + "female": 22062, + "petroleum": 22063, + "media": 22064, + "efc": 22065, + "pokémon": 22066, + "à¤ķ": 22067, + "enthusiastic": 22068, + "varun": 22069, + "profiles": 22070, + "pediatric": 22071, + "accidents": 22072, + "conrad": 22073, + "jang": 22074, + "jojo": 22075, + "acor": 22076, + "observer": 22077, + "lf": 22078, + "livestock": 22079, + "forgi": 22080, + "fos": 22081, + "elm": 22082, + "anand": 22083, + "goe": 22084, + "cere": 22085, + "avoiding": 22086, + "grit": 22087, + "oman": 22088, + "thankfully": 22089, + "scattered": 22090, + "nicky": 22091, + "cylinder": 22092, + "cheesy": 22093, + "diver": 22094, + "mahesh": 22095, + "caves": 22096, + "earliest": 22097, + "quinte": 22098, + "subjects": 22099, + "bend": 22100, + "gulf": 22101, + "vocalist": 22102, + "glue": 22103, + "patches": 22104, + "unstopp": 22105, + "snyder": 22106, + "demonstrating": 22107, + "pio": 22108, + "horns": 22109, + "wickets": 22110, + "andthe": 22111, + "rama": 22112, + "yoon": 22113, + "straight": 22114, + "bedtime": 22115, + "orang": 22116, + "bullets": 22117, + "saurus": 22118, + "miners": 22119, + "incidents": 22120, + "!...": 22121, + "ðŁİ¸": 22122, + "agers": 22123, + "handles": 22124, + "states": 22125, + "inity": 22126, + "dons": 22127, + "incredible": 22128, + "eminem": 22129, + "aviv": 22130, + "rudy": 22131, + "mozart": 22132, + "folklore": 22133, + "appliances": 22134, + "mtl": 22135, + "frey": 22136, + "dias": 22137, + "hua": 22138, + "pageant": 22139, + "strive": 22140, + "imprison": 22141, + "bullish": 22142, + "rana": 22143, + "alerts": 22144, + "bbmas": 22145, + "hyper": 22146, + "derbyshire": 22147, + "recre": 22148, + "redd": 22149, + "deborah": 22150, + "cosmos": 22151, + "lawson": 22152, + "melanie": 22153, + "psycho": 22154, + "hoor": 22155, + "doodles": 22156, + "sniper": 22157, + "shady": 22158, + "mantle": 22159, + "canadian": 22160, + "newyear": 22161, + "interactions": 22162, + "separated": 22163, + "cords": 22164, + "spirituality": 22165, + "apu": 22166, + "ito": 22167, + "pct": 22168, + "pelosi": 22169, + "rebellion": 22170, + "seiz": 22171, + "worcester": 22172, + "sectors": 22173, + "uli": 22174, + "santa": 22175, + "е": 22176, + "ðŁĩªðŁĩ¸": 22177, + "biased": 22178, + "classical": 22179, + "gamma": 22180, + "deeplear": 22181, + "emerge": 22182, + "backer": 22183, + "surance": 22184, + "handcrafted": 22185, + "ðŁİ¥": 22186, + "francis": 22187, + "millan": 22188, + "ici": 22189, + "crown": 22190, + "wow": 22191, + "striped": 22192, + "unfair": 22193, + "relaxation": 22194, + "³ï¸ı": 22195, + "embracing": 22196, + "shealth": 22197, + "paleo": 22198, + "martini": 22199, + "distillery": 22200, + "wrink": 22201, + "ork": 22202, + "nath": 22203, + "hayley": 22204, + "courthouse": 22205, + "siber": 22206, + "sadi": 22207, + "quietly": 22208, + "melt": 22209, + "msm": 22210, + "meh": 22211, + "smartphones": 22212, + "relent": 22213, + "pping": 22214, + "warwick": 22215, + "cologne": 22216, + "glia": 22217, + "cotton": 22218, + "prog": 22219, + "lone": 22220, + "ipsw": 22221, + "starters": 22222, + "expands": 22223, + "ump": 22224, + "sued": 22225, + "skipper": 22226, + "infections": 22227, + "ingle": 22228, + "á": 22229, + "clerk": 22230, + "demonstrate": 22231, + "acar": 22232, + "ðŁĺĤðŁĺĤðŁĺĤ": 22233, + "tibet": 22234, + "buns": 22235, + "alom": 22236, + "demolition": 22237, + "ssia": 22238, + "gst": 22239, + "[]": 22240, + "soar": 22241, + "âĺĢ": 22242, + "ðŁĺª": 22243, + "ðŁĵĬ": 22244, + "deepest": 22245, + "beyond": 22246, + "aret": 22247, + "attends": 22248, + "activated": 22249, + "dimit": 22250, + "âļªï¸ı": 22251, + "highlighted": 22252, + "magazines": 22253, + "rumor": 22254, + "azza": 22255, + "stephens": 22256, + "dolph": 22257, + "shockey": 22258, + "mats": 22259, + "weav": 22260, + "melan": 22261, + "servers": 22262, + "traum": 22263, + "kush": 22264, + "æĹ": 22265, + "babys": 22266, + "paz": 22267, + "aal": 22268, + "lause": 22269, + "breakers": 22270, + "canterbury": 22271, + "ulture": 22272, + "miri": 22273, + "euros": 22274, + "taneous": 22275, + "impressions": 22276, + "dutch": 22277, + "ild": 22278, + "ghi": 22279, + "purdue": 22280, + "adequate": 22281, + "lp": 22282, + "syner": 22283, + "angler": 22284, + "durable": 22285, + "galore": 22286, + "rown": 22287, + "mgmt": 22288, + "ðŁĵĮ": 22289, + "lucia": 22290, + "âĺijï¸ı": 22291, + "zayn": 22292, + "borrow": 22293, + ".(": 22294, + "northumber": 22295, + "crush": 22296, + "enga": 22297, + "sush": 22298, + "extravag": 22299, + "tout": 22300, + "mahal": 22301, + "alistic": 22302, + "thermo": 22303, + "galleries": 22304, + "esse": 22305, + "chibi": 22306, + "attractions": 22307, + "lexington": 22308, + "legislature": 22309, + "documented": 22310, + "residen": 22311, + "brownies": 22312, + "wf": 22313, + "stool": 22314, + "planets": 22315, + "shoppers": 22316, + "conductor": 22317, + "msp": 22318, + "tricky": 22319, + "fruity": 22320, + "endra": 22321, + "feelthe": 22322, + "whipped": 22323, + "hairstyle": 22324, + "refer": 22325, + "ook": 22326, + "octopus": 22327, + "audiences": 22328, + "kumar": 22329, + "afterno": 22330, + "optim": 22331, + "cfl": 22332, + "nip": 22333, + "geni": 22334, + "alphabet": 22335, + "annab": 22336, + "lamin": 22337, + "accepts": 22338, + "lng": 22339, + "ðŁĺ«": 22340, + "tine": 22341, + "acom": 22342, + "cheerleaders": 22343, + "tk": 22344, + "gron": 22345, + "vg": 22346, + "kung": 22347, + "jax": 22348, + "dhabi": 22349, + "rss": 22350, + "mackenzie": 22351, + "beirut": 22352, + "cleanup": 22353, + "gypsy": 22354, + "stell": 22355, + "burger": 22356, + "hurricanes": 22357, + "education": 22358, + "stina": 22359, + "âĻ¡âĻ¡": 22360, + "unfortunate": 22361, + "jeremi": 22362, + "badger": 22363, + "aters": 22364, + ":âĢ¦": 22365, + "terra": 22366, + "sublime": 22367, + "stud": 22368, + "ymca": 22369, + "mru": 22370, + "duterte": 22371, + "brennan": 22372, + "bulb": 22373, + "melo": 22374, + "ylon": 22375, + "hacker": 22376, + "cred": 22377, + "gud": 22378, + "asan": 22379, + "padilla": 22380, + "embroidered": 22381, + "vietnamese": 22382, + "pioneers": 22383, + "projection": 22384, + "reboot": 22385, + "idc": 22386, + "aney": 22387, + "primer": 22388, + "suffers": 22389, + "winding": 22390, + "pon": 22391, + "stoday": 22392, + "morn": 22393, + "uch": 22394, + "allin": 22395, + "adidas": 22396, + "elizabeth": 22397, + "tuck": 22398, + "ography": 22399, + "ðŁļĢ": 22400, + "beg": 22401, + "osborne": 22402, + "ghetto": 22403, + "rh": 22404, + "cnn": 22405, + "irma": 22406, + "makin": 22407, + "cables": 22408, + "murders": 22409, + "ocks": 22410, + "insta": 22411, + "alas": 22412, + "sik": 22413, + "cuff": 22414, + "lare": 22415, + "foodies": 22416, + "ovic": 22417, + "atom": 22418, + "geometric": 22419, + "empathy": 22420, + "ี": 22421, + "centenary": 22422, + "newspapers": 22423, + "administrative": 22424, + "ðŁİĬ": 22425, + "stive": 22426, + "contractors": 22427, + "lett": 22428, + "tasmania": 22429, + "awesomeness": 22430, + "density": 22431, + "veen": 22432, + "princeton": 22433, + "frequently": 22434, + "reject": 22435, + "ghi": 22436, + "modular": 22437, + "ceramics": 22438, + "shag": 22439, + "kiwi": 22440, + "canvas": 22441, + "sweatshirt": 22442, + "anj": 22443, + "timm": 22444, + "napoli": 22445, + "iler": 22446, + "appeals": 22447, + "hamilton": 22448, + "mayo": 22449, + "weave": 22450, + "arranged": 22451, + "wharf": 22452, + "occupy": 22453, + "bvb": 22454, + "asaki": 22455, + "otter": 22456, + "norm": 22457, + "vies": 22458, + "detox": 22459, + "tional": 22460, + "derek": 22461, + "idad": 22462, + "admissions": 22463, + "constituency": 22464, + "upper": 22465, + "woot": 22466, + "alloy": 22467, + "seve": 22468, + "lub": 22469, + "uncomfortable": 22470, + "edwin": 22471, + "abre": 22472, + "dwight": 22473, + "arche": 22474, + "virtually": 22475, + "spol": 22476, + "prie": 22477, + "aii": 22478, + "err": 22479, + "switch": 22480, + "barack": 22481, + "seok": 22482, + "coul": 22483, + "wnt": 22484, + "poul": 22485, + "olive": 22486, + "caffeine": 22487, + "cardiff": 22488, + "notorious": 22489, + "demp": 22490, + "excess": 22491, + "barr": 22492, + "tford": 22493, + "ajay": 22494, + "bumped": 22495, + "mythology": 22496, + "shelley": 22497, + "falcon": 22498, + "shakespeare": 22499, + "mustangs": 22500, + "noted": 22501, + "bone": 22502, + "civilization": 22503, + "syd": 22504, + "parsons": 22505, + "unofficial": 22506, + "hyped": 22507, + "spends": 22508, + "opposed": 22509, + "vings": 22510, + "spacex": 22511, + "notification": 22512, + "deciding": 22513, + "biotech": 22514, + "outsi": 22515, + "salah": 22516, + "!.": 22517, + "fed": 22518, + "ssy": 22519, + "cms": 22520, + "badgers": 22521, + "cro": 22522, + "elaine": 22523, + "nba": 22524, + "dyour": 22525, + "nant": 22526, + "honeymoon": 22527, + "climbed": 22528, + "conomy": 22529, + "atha": 22530, + "mell": 22531, + "nebula": 22532, + "naturephotography": 22533, + "julie": 22534, + "bmx": 22535, + "invested": 22536, + "mono": 22537, + "lieutenant": 22538, + "watkins": 22539, + "technician": 22540, + "ose": 22541, + "kae": 22542, + "ìĽ": 22543, + "mcqueen": 22544, + "preach": 22545, + "traveller": 22546, + "flexibility": 22547, + "zebra": 22548, + "retailer": 22549, + "pant": 22550, + "bender": 22551, + "brandt": 22552, + "squid": 22553, + "warrant": 22554, + "verified": 22555, + "cass": 22556, + "piercing": 22557, + "honours": 22558, + "tying": 22559, + "morris": 22560, + "kissed": 22561, + "oprah": 22562, + "panoramic": 22563, + "mei": 22564, + "splatoon": 22565, + "wichita": 22566, + "arias": 22567, + "galli": 22568, + "indyref": 22569, + "goodtimes": 22570, + "atheist": 22571, + "confession": 22572, + "owski": 22573, + "repping": 22574, + "additions": 22575, + "mechanism": 22576, + "zim": 22577, + "jans": 22578, + "suf": 22579, + "chopped": 22580, + "beginnings": 22581, + "vitamins": 22582, + "ãħ¤ãħ¤": 22583, + "orth": 22584, + "poles": 22585, + "rub": 22586, + "antarctica": 22587, + "indiefilm": 22588, + "webcam": 22589, + "ketch": 22590, + "brett": 22591, + "clement": 22592, + "heron": 22593, + "defeating": 22594, + "hydro": 22595, + "bucket": 22596, + "wandering": 22597, + "sidney": 22598, + "futureof": 22599, + "binge": 22600, + "onies": 22601, + "knockout": 22602, + "administrator": 22603, + "synthe": 22604, + "lent": 22605, + "jani": 22606, + "barley": 22607, + "premierleague": 22608, + "nerds": 22609, + "crm": 22610, + "bras": 22611, + "botany": 22612, + "evolved": 22613, + "rotter": 22614, + "rowed": 22615, + "tumor": 22616, + "wealthy": 22617, + "ÂŃ": 22618, + "monarch": 22619, + "lished": 22620, + "dahl": 22621, + "ðŁİĥ": 22622, + "buch": 22623, + "kenyan": 22624, + "ا": 22625, + "redness": 22626, + "assembled": 22627, + "semit": 22628, + "hudder": 22629, + "shrop": 22630, + "rani": 22631, + "learning": 22632, + "mory": 22633, + "itia": 22634, + "geographic": 22635, + "worldof": 22636, + "fb": 22637, + "phosp": 22638, + "boogie": 22639, + "amped": 22640, + "?...": 22641, + "chew": 22642, + "dwarf": 22643, + "arus": 22644, + "ssen": 22645, + "rusty": 22646, + "recruits": 22647, + "hk": 22648, + "garde": 22649, + "applause": 22650, + "volumes": 22651, + "involves": 22652, + "tac": 22653, + "handbag": 22654, + "translate": 22655, + "ffel": 22656, + "seym": 22657, + "aquatic": 22658, + "transfer": 22659, + "zodi": 22660, + "andr": 22661, + "academia": 22662, + "crater": 22663, + "tez": 22664, + "arse": 22665, + "adapt": 22666, + "coloni": 22667, + "snowman": 22668, + "mali": 22669, + "hangin": 22670, + "dischar": 22671, + "oysters": 22672, + "phoe": 22673, + "colonel": 22674, + "wba": 22675, + "hispanic": 22676, + "thriving": 22677, + "shy": 22678, + "agles": 22679, + "salesforce": 22680, + "creme": 22681, + "soles": 22682, + "lafayette": 22683, + "âī": 22684, + "teria": 22685, + "acha": 22686, + "sperson": 22687, + "gogo": 22688, + "carly": 22689, + "theore": 22690, + "amore": 22691, + "vox": 22692, + "aft": 22693, + "ãĤ¹": 22694, + "staple": 22695, + "muffin": 22696, + "diagram": 22697, + "inox": 22698, + "sustained": 22699, + "avent": 22700, + "meta": 22701, + "arbitr": 22702, + "decay": 22703, + "adole": 22704, + "н": 22705, + "ecol": 22706, + "pho": 22707, + "nk": 22708, + "ocu": 22709, + "granny": 22710, + "ça": 22711, + "luxembour": 22712, + "stadt": 22713, + "alberto": 22714, + "levit": 22715, + "amas": 22716, + "dx": 22717, + "orphan": 22718, + "cobb": 22719, + "asc": 22720, + "logy": 22721, + "immense": 22722, + "chants": 22723, + "offline": 22724, + "pent": 22725, + "brex": 22726, + "winger": 22727, + "plane": 22728, + "iel": 22729, + "nichols": 22730, + "cathy": 22731, + "naruto": 22732, + "lowed": 22733, + "///": 22734, + "ignorance": 22735, + "catastro": 22736, + "youts": 22737, + "schen": 22738, + "build": 22739, + "hazi": 22740, + "sine": 22741, + "criticalrole": 22742, + "dug": 22743, + "detect": 22744, + "logs": 22745, + "enamel": 22746, + "stpatricksday": 22747, + "eddie": 22748, + "copa": 22749, + "cigarettes": 22750, + "hoff": 22751, + "kaya": 22752, + "lagoon": 22753, + "rapha": 22754, + "airborne": 22755, + "choose": 22756, + "puertor": 22757, + "kev": 22758, + "guiding": 22759, + "frosty": 22760, + "borough": 22761, + "mira": 22762, + "ðŁİĬ": 22763, + "cadet": 22764, + "anush": 22765, + "yogi": 22766, + "eger": 22767, + "fling": 22768, + "slope": 22769, + "ninth": 22770, + "weston": 22771, + "footwear": 22772, + "fn": 22773, + "mayweather": 22774, + "aam": 22775, + "plain": 22776, + "staircase": 22777, + "witnesses": 22778, + "workouts": 22779, + "robust": 22780, + "dexter": 22781, + "cohort": 22782, + "ðŁļĹ": 22783, + "spell": 22784, + "haze": 22785, + "oom": 22786, + "organising": 22787, + "wildfire": 22788, + "contacts": 22789, + "avon": 22790, + "mino": 22791, + "updating": 22792, + "ðŁį»": 22793, + "lithium": 22794, + "ingual": 22795, + "kis": 22796, + "auga": 22797, + "locom": 22798, + "deduc": 22799, + "uda": 22800, + "thak": 22801, + "boyle": 22802, + "mper": 22803, + "hottie": 22804, + "erik": 22805, + "revised": 22806, + "isla": 22807, + "travelphotography": 22808, + "ooza": 22809, + "enqui": 22810, + "conferences": 22811, + "clover": 22812, + "groom": 22813, + "curves": 22814, + "liveon": 22815, + "perf": 22816, + "displaced": 22817, + "bolog": 22818, + "xxxx": 22819, + "ðŁĺ©ðŁĺ©": 22820, + "teal": 22821, + "vessels": 22822, + "rainforest": 22823, + "calci": 22824, + "panther": 22825, + "giraffe": 22826, + "tasted": 22827, + "imagery": 22828, + "padres": 22829, + "daytime": 22830, + "bass": 22831, + "ripe": 22832, + "opioid": 22833, + "nue": 22834, + "vinyl": 22835, + "inventor": 22836, + "sens": 22837, + "processor": 22838, + "mut": 22839, + "gadgets": 22840, + "biblical": 22841, + "shannon": 22842, + "jacqueline": 22843, + "cary": 22844, + "theresistance": 22845, + "alien": 22846, + "nvi": 22847, + "cosy": 22848, + "bihar": 22849, + "foley": 22850, + "rend": 22851, + "mugs": 22852, + "faken": 22853, + "clone": 22854, + "niallo": 22855, + "grabbed": 22856, + "chihu": 22857, + "powerhouse": 22858, + "ntt": 22859, + "cherokee": 22860, + "sponge": 22861, + "implementing": 22862, + "rhine": 22863, + "leone": 22864, + "ðŁįĢ": 22865, + "prettiest": 22866, + "infrared": 22867, + "improv": 22868, + "switched": 22869, + "tubes": 22870, + "contr": 22871, + "blk": 22872, + "projected": 22873, + "beaver": 22874, + "yot": 22875, + "bbcradio": 22876, + "thigh": 22877, + "persecu": 22878, + "apologize": 22879, + "wack": 22880, + "poster": 22881, + "oliver": 22882, + "aza": 22883, + "loud": 22884, + "(?)": 22885, + "fthe": 22886, + "womenshi": 22887, + "sparrow": 22888, + "blush": 22889, + "usable": 22890, + "scales": 22891, + "itative": 22892, + "peuge": 22893, + "needing": 22894, + "leggings": 22895, + "glamorous": 22896, + "matur": 22897, + "cz": 22898, + "watt": 22899, + "dab": 22900, + "tamar": 22901, + "etsym": 22902, + "bauer": 22903, + "heartfelt": 22904, + "hn": 22905, + "elsewhere": 22906, + "birch": 22907, + "alumini": 22908, + "huck": 22909, + "eme": 22910, + "jl": 22911, + "trafford": 22912, + "dz": 22913, + "portions": 22914, + "anasta": 22915, + "arthritis": 22916, + "espn": 22917, + "bergen": 22918, + "violation": 22919, + "yoshi": 22920, + "cz": 22921, + "northumberland": 22922, + "closures": 22923, + "ðŁĩ¯ðŁĩ": 22924, + "smiley": 22925, + "rw": 22926, + "telugu": 22927, + "intensi": 22928, + "gregg": 22929, + "vega": 22930, + "dungeon": 22931, + "southbound": 22932, + "bail": 22933, + "dominican": 22934, + "semifinal": 22935, + "chapters": 22936, + "hitch": 22937, + "vanity": 22938, + "transiti": 22939, + "recommends": 22940, + "satisf": 22941, + "barca": 22942, + "queens": 22943, + "((": 22944, + "destruc": 22945, + "strait": 22946, + "ravi": 22947, + "desserts": 22948, + "intru": 22949, + "haram": 22950, + "kos": 22951, + "foe": 22952, + "fatty": 22953, + "paisley": 22954, + "magnitude": 22955, + "dridge": 22956, + "comey": 22957, + "schemes": 22958, + "visionary": 22959, + "ourt": 22960, + "downloaded": 22961, + "ðŁĻĮðŁı½": 22962, + "gdpr": 22963, + "lani": 22964, + "pwc": 22965, + "guad": 22966, + "nicest": 22967, + "stakeholders": 22968, + "referred": 22969, + "georgetown": 22970, + "arvindkejriwal": 22971, + "schneider": 22972, + "indoors": 22973, + "allstar": 22974, + "stranded": 22975, + "gender": 22976, + "zepp": 22977, + "masses": 22978, + "ðŁIJ±": 22979, + "patiently": 22980, + "bldg": 22981, + "zab": 22982, + "wearab": 22983, + "vivid": 22984, + "heck": 22985, + "della": 22986, + "symb": 22987, + "jeopar": 22988, + "lager": 22989, + "àª": 22990, + "combines": 22991, + "nec": 22992, + "bray": 22993, + "flop": 22994, + "txwx": 22995, + "joys": 22996, + "pont": 22997, + "profound": 22998, + "surround": 22999, + "madhu": 23000, + "mable": 23001, + "ayr": 23002, + "teas": 23003, + "nsa": 23004, + "openly": 23005, + "ernest": 23006, + "ãĥ©": 23007, + "topo": 23008, + "gna": 23009, + "antioxid": 23010, + "tian": 23011, + "etr": 23012, + "cello": 23013, + "mathi": 23014, + "generosity": 23015, + "biting": 23016, + "manic": 23017, + "kelsey": 23018, + "cheeks": 23019, + "tender": 23020, + "wth": 23021, + "pronoun": 23022, + "ultimately": 23023, + "gusta": 23024, + "arianag": 23025, + "gerry": 23026, + "bleed": 23027, + "reddy": 23028, + "mich": 23029, + "mitsubishi": 23030, + "operated": 23031, + "sexually": 23032, + "mau": 23033, + "cllr": 23034, + "vids": 23035, + "coc": 23036, + "melted": 23037, + "ðŁĮĪ": 23038, + "qld": 23039, + "itech": 23040, + "instrumental": 23041, + "endgame": 23042, + "ðŁĵĸ": 23043, + "energi": 23044, + "brownie": 23045, + "tamil": 23046, + "atin": 23047, + "dominated": 23048, + "praises": 23049, + "fireplace": 23050, + "sensational": 23051, + "mena": 23052, + "karti": 23053, + "unprece": 23054, + "rupt": 23055, + "oriental": 23056, + "mccor": 23057, + "tournaments": 23058, + "scenter": 23059, + "reeves": 23060, + "prescription": 23061, + "same": 23062, + "frau": 23063, + "truffle": 23064, + "embo": 23065, + "romans": 23066, + "blasts": 23067, + "technological": 23068, + "prat": 23069, + "bsb": 23070, + "yar": 23071, + "trendy": 23072, + "acl": 23073, + "alad": 23074, + "ðŁįģ": 23075, + "ohh": 23076, + "bankrupt": 23077, + "thoven": 23078, + "regards": 23079, + "iser": 23080, + "warwick": 23081, + "vineyards": 23082, + "realm": 23083, + "niallofficial": 23084, + "dota": 23085, + "gemini": 23086, + "todo": 23087, + "vable": 23088, + "¨¨": 23089, + "lau": 23090, + "wreath": 23091, + "juve": 23092, + "natasha": 23093, + "lever": 23094, + "lori": 23095, + "horser": 23096, + "cctv": 23097, + "airbnb": 23098, + "esanders": 23099, + "sinclair": 23100, + "emabiggest": 23101, + "highschool": 23102, + "contest": 23103, + "optimistic": 23104, + "tte": 23105, + "ðŁĴķðŁĴķ": 23106, + "ssd": 23107, + "yee": 23108, + "helena": 23109, + "consen": 23110, + "ricks": 23111, + "jesse": 23112, + "anic": 23113, + "ðŁİ¯": 23114, + "reacts": 23115, + "robe": 23116, + "independence": 23117, + "voltage": 23118, + "mington": 23119, + "sant": 23120, + "à¸Ļà¸": 23121, + "----------------": 23122, + "sentinel": 23123, + "kett": 23124, + "rehearsing": 23125, + "aaaaaaaa": 23126, + "softhe": 23127, + "stirling": 23128, + "search": 23129, + "wigan": 23130, + "standout": 23131, + "snail": 23132, + "pentagon": 23133, + "Äģ": 23134, + "chlor": 23135, + "crust": 23136, + "netany": 23137, + "chemist": 23138, + "disappeared": 23139, + "ricardo": 23140, + "spiders": 23141, + "bose": 23142, + "warren": 23143, + "messing": 23144, + "banners": 23145, + "guel": 23146, + "parach": 23147, + "maid": 23148, + "counted": 23149, + "epile": 23150, + "bonfire": 23151, + "speechless": 23152, + "setter": 23153, + "measured": 23154, + "rejects": 23155, + "nikki": 23156, + "lester": 23157, + "forensic": 23158, + "fabrics": 23159, + "aloha": 23160, + "preserved": 23161, + "watford": 23162, + "detailing": 23163, + "darth": 23164, + "bou": 23165, + "carly": 23166, + "...'": 23167, + "tailgate": 23168, + "notifications": 23169, + "å¤": 23170, + "passive": 23171, + "trousers": 23172, + "baloch": 23173, + "rother": 23174, + "typically": 23175, + "Ã¥": 23176, + "spit": 23177, + "wiz": 23178, + "sicily": 23179, + "technically": 23180, + "expose": 23181, + "stage": 23182, + "hubb": 23183, + "cream": 23184, + "caps": 23185, + "poke": 23186, + "sleek": 23187, + "june": 23188, + "temporarily": 23189, + "dez": 23190, + "awakens": 23191, + "lame": 23192, + "_-": 23193, + "jiha": 23194, + "tuesdays": 23195, + "advised": 23196, + "advisors": 23197, + "existed": 23198, + "disagree": 23199, + "newsroom": 23200, + "losers": 23201, + "worldtour": 23202, + "drying": 23203, + "aldi": 23204, + "harness": 23205, + "footprint": 23206, + "hobbit": 23207, + "pmln": 23208, + "iro": 23209, + "quered": 23210, + "assess": 23211, + "gaze": 23212, + "sab": 23213, + "thian": 23214, + "íĬ": 23215, + "tif": 23216, + "observe": 23217, + "evil": 23218, + "drawer": 23219, + "sweep": 23220, + "cory": 23221, + "cody": 23222, + "kyoto": 23223, + "callum": 23224, + "ninj": 23225, + "laurent": 23226, + "bei": 23227, + "sketching": 23228, + "customized": 23229, + "dur": 23230, + "regrets": 23231, + "knoxville": 23232, + "ìķĦ": 23233, + "messaging": 23234, + "gracie": 23235, + "abundance": 23236, + "bidding": 23237, + "brewed": 23238, + "flouri": 23239, + "therapeutic": 23240, + "altitude": 23241, + "hogs": 23242, + "burner": 23243, + "electro": 23244, + "wonderfully": 23245, + "heater": 23246, + "postpon": 23247, + "livery": 23248, + "rall": 23249, + "adas": 23250, + "aac": 23251, + "saul": 23252, + "brooklyn": 23253, + "playhouse": 23254, + "âĻ¥âĻ¥âĻ¥": 23255, + "charitable": 23256, + "iny": 23257, + "zah": 23258, + "competitions": 23259, + "beav": 23260, + "plugged": 23261, + "ois": 23262, + "doom": 23263, + "astronom": 23264, + "specialized": 23265, + "maxi": 23266, + "taps": 23267, + "cellular": 23268, + "depressed": 23269, + "folklorethursday": 23270, + "crib": 23271, + "emul": 23272, + "ë°©": 23273, + "figh": 23274, + "ruz": 23275, + "carlisle": 23276, + "spear": 23277, + "sidewalk": 23278, + "dei": 23279, + "dependent": 23280, + "laces": 23281, + "nhs": 23282, + "ðŁĮĻ": 23283, + "realizing": 23284, + "network": 23285, + "riche": 23286, + "regin": 23287, + "refresh": 23288, + "stral": 23289, + "pathology": 23290, + "plaid": 23291, + "psychedelic": 23292, + "hind": 23293, + "uka": 23294, + "algorithm": 23295, + "linking": 23296, + "progressi": 23297, + "fey": 23298, + "dade": 23299, + "hydrated": 23300, + "bant": 23301, + "famed": 23302, + "cotsw": 23303, + "boise": 23304, + "asc": 23305, + "racing": 23306, + "javier": 23307, + "wwen": 23308, + "marlins": 23309, + "poop": 23310, + "swept": 23311, + "tonights": 23312, + "wef": 23313, + "anime": 23314, + "slovak": 23315, + "âŀĸâŀĸ": 23316, + "claus": 23317, + "lemme": 23318, + "clippers": 23319, + "rels": 23320, + "arianagrande": 23321, + "rte": 23322, + "kot": 23323, + "thalapathy": 23324, + "hungarian": 23325, + "zuma": 23326, + "yvon": 23327, + "isu": 23328, + "journeys": 23329, + "clinics": 23330, + "bebe": 23331, + "wwf": 23332, + "nws": 23333, + "superheroes": 23334, + "erit": 23335, + "sleague": 23336, + "identification": 23337, + "motto": 23338, + "bai": 23339, + "sourced": 23340, + "iller": 23341, + "api": 23342, + "prise": 23343, + "unprecedented": 23344, + "damas": 23345, + "tunisia": 23346, + "drain": 23347, + "underestim": 23348, + "ether": 23349, + "quarterly": 23350, + "rewarding": 23351, + "alham": 23352, + "wolverine": 23353, + "cabine": 23354, + "hypno": 23355, + "nadine": 23356, + "havana": 23357, + "dae": 23358, + "ðŁĵĪ": 23359, + "dron": 23360, + "readings": 23361, + "bati": 23362, + "pico": 23363, + "merci": 23364, + "itian": 23365, + "walkers": 23366, + "elope": 23367, + "mikey": 23368, + "godzilla": 23369, + "burlington": 23370, + "abuja": 23371, + "socialism": 23372, + "atility": 23373, + "shell": 23374, + "harrypotter": 23375, + "gno": 23376, + "abur": 23377, + "releg": 23378, + "felici": 23379, + "rogen": 23380, + "neuroscience": 23381, + "instin": 23382, + "atham": 23383, + "vouchers": 23384, + "jarre": 23385, + "fuse": 23386, + "defici": 23387, + "monterey": 23388, + "deport": 23389, + "midday": 23390, + "ppard": 23391, + "freed": 23392, + "ameter": 23393, + "wilt": 23394, + "ningham": 23395, + "pratt": 23396, + "liberty": 23397, + "slogan": 23398, + "oto": 23399, + "pri": 23400, + "coated": 23401, + "cpd": 23402, + "nett": 23403, + "illas": 23404, + "malawi": 23405, + "evolve": 23406, + "accessibility": 23407, + "ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥": 23408, + "ornament": 23409, + "bp": 23410, + "elis": 23411, + "sonline": 23412, + "chiro": 23413, + "flick": 23414, + "ibm": 23415, + "arak": 23416, + "enables": 23417, + "garland": 23418, + "sane": 23419, + "cuties": 23420, + "trip": 23421, + "rotterdam": 23422, + "nys": 23423, + "lamps": 23424, + "lucas": 23425, + "bog": 23426, + "rails": 23427, + "travelled": 23428, + "hicks": 23429, + "enu": 23430, + "sabha": 23431, + "scrub": 23432, + "hier": 23433, + "hartford": 23434, + "foo": 23435, + "fernandez": 23436, + "trevor": 23437, + "mattress": 23438, + "appointments": 23439, + "alej": 23440, + "fei": 23441, + "ologist": 23442, + "safar": 23443, + "octa": 23444, + "src": 23445, + "shaun": 23446, + "ambient": 23447, + "dric": 23448, + "biker": 23449, + "shee": 23450, + "mustache": 23451, + "hta": 23452, + "boone": 23453, + "herty": 23454, + "cardio": 23455, + "brakes": 23456, + "recital": 23457, + "consists": 23458, + "overwhelmed": 23459, + "caul": 23460, + "robbins": 23461, + "imit": 23462, + "alth": 23463, + "url": 23464, + "bibli": 23465, + "onne": 23466, + "blacklivesmatter": 23467, + "difficulties": 23468, + "telang": 23469, + "taller": 23470, + "ðŁĵĨ": 23471, + "debating": 23472, + "burrito": 23473, + "movember": 23474, + "strengthening": 23475, + "boe": 23476, + "testam": 23477, + "miracles": 23478, + "baseball": 23479, + "renee": 23480, + "ðŁijīðŁı»": 23481, + "alfa": 23482, + "âĺĺ": 23483, + "unstoppable": 23484, + "ecs": 23485, + "gmo": 23486, + "giftideas": 23487, + "pathway": 23488, + "fencing": 23489, + "ðŁİ¤": 23490, + "bham": 23491, + "ras": 23492, + "sko": 23493, + "dled": 23494, + "thelast": 23495, + "magnum": 23496, + "binary": 23497, + "wilde": 23498, + "wilder": 23499, + "whati": 23500, + "barbecue": 23501, + "hism": 23502, + "canoe": 23503, + "kurdi": 23504, + "elive": 23505, + "advantages": 23506, + "madame": 23507, + "bier": 23508, + "missing": 23509, + "entertain": 23510, + "airforce": 23511, + "yama": 23512, + "cis": 23513, + "hashtags": 23514, + "jis": 23515, + "veil": 23516, + "dreamy": 23517, + "tense": 23518, + "mayward": 23519, + "chateau": 23520, + "huntington": 23521, + "âļĵ": 23522, + "vall": 23523, + "upon": 23524, + "blouse": 23525, + "dunes": 23526, + "ðŁĺ´": 23527, + "fertility": 23528, + "mole": 23529, + "currencies": 23530, + "stu": 23531, + "berlin": 23532, + "toasted": 23533, + "divas": 23534, + "walt": 23535, + "lark": 23536, + "pora": 23537, + "hitter": 23538, + "umer": 23539, + "chilled": 23540, + "balancing": 23541, + "fais": 23542, + "yin": 23543, + "ortiz": 23544, + "eastenders": 23545, + "hate": 23546, + "ural": 23547, + "april": 23548, + "timel": 23549, + "à±": 23550, + "pero": 23551, + "stocked": 23552, + "respects": 23553, + "tht": 23554, + "bestfriends": 23555, + "givingtuesday": 23556, + "bead": 23557, + "invent": 23558, + "imi": 23559, + "naples": 23560, + "combining": 23561, + "tokens": 23562, + "thirst": 23563, + "masc": 23564, + "parrot": 23565, + "spu": 23566, + "denton": 23567, + "*-*": 23568, + "tres": 23569, + "suburban": 23570, + "width": 23571, + "sive": 23572, + "contender": 23573, + "sirius": 23574, + "lok": 23575, + "troopers": 23576, + "outrage": 23577, + "turbo": 23578, + "fragile": 23579, + "messed": 23580, + "doh": 23581, + "discord": 23582, + "netanyahu": 23583, + "resign": 23584, + "forgiveness": 23585, + "mohan": 23586, + "munch": 23587, + "camou": 23588, + "identifying": 23589, + "enabling": 23590, + "hotter": 23591, + "thornton": 23592, + "jaipur": 23593, + "arya": 23594, + "ðŁı»âĢįâĻĢï¸ı": 23595, + "mustaf": 23596, + "majors": 23597, + "oke": 23598, + "duffy": 23599, + "rohing": 23600, + "tilt": 23601, + "ðŁĩ®ðŁĩ³": 23602, + "rockstar": 23603, + "sheep": 23604, + "hendrix": 23605, + "rav": 23606, + "invention": 23607, + "dou": 23608, + "laguna": 23609, + "grumpy": 23610, + "swis": 23611, + "impe": 23612, + ")'": 23613, + "youths": 23614, + "bunker": 23615, + "stache": 23616, + "oppose": 23617, + "indies": 23618, + "accelerate": 23619, + "mlp": 23620, + "eden": 23621, + "wann": 23622, + "kail": 23623, + "akshaykumar": 23624, + "supt": 23625, + "polym": 23626, + "middleton": 23627, + "extraordin": 23628, + "wilson": 23629, + "australian": 23630, + "aluminium": 23631, + "wayne": 23632, + "alumnus": 23633, + "matics": 23634, + "grim": 23635, + "ernie": 23636, + "oppa": 23637, + "competitors": 23638, + "randall": 23639, + "hence": 23640, + "declares": 23641, + "preaching": 23642, + "shahe": 23643, + "cane": 23644, + "sustainable": 23645, + "staples": 23646, + "ledge": 23647, + "adena": 23648, + "doctoral": 23649, + "burgundy": 23650, + "decorate": 23651, + "rendered": 23652, + "risen": 23653, + "prank": 23654, + "dior": 23655, + "beethoven": 23656, + "floor": 23657, + "accom": 23658, + "tot": 23659, + "hodg": 23660, + "tourism": 23661, + "sayin": 23662, + "objective": 23663, + "markers": 23664, + "premiership": 23665, + "enabled": 23666, + "camoufla": 23667, + "giant": 23668, + "Ñģ": 23669, + "smokey": 23670, + "ricket": 23671, + "pang": 23672, + "depending": 23673, + "sation": 23674, + "evolving": 23675, + "intercep": 23676, + "census": 23677, + "tofthe": 23678, + "reen": 23679, + "mendoza": 23680, + "trumpet": 23681, + "marketers": 23682, + "anit": 23683, + "ðŁĻĬ": 23684, + "northwestern": 23685, + "vla": 23686, + "fotogra": 23687, + "blackandwhite": 23688, + "chewan": 23689, + "wig": 23690, + "troom": 23691, + "gingerbread": 23692, + "kn": 23693, + "romero": 23694, + "nfc": 23695, + "orchi": 23696, + "funko": 23697, + "source": 23698, + "fs": 23699, + "raped": 23700, + "ost": 23701, + "tarot": 23702, + "annually": 23703, + "ðŁĺ¬": 23704, + "rill": 23705, + "delav": 23706, + "..!!": 23707, + "ses": 23708, + "cann": 23709, + "medicare": 23710, + "phel": 23711, + "apex": 23712, + "guardian": 23713, + "remained": 23714, + "rpm": 23715, + "añ": 23716, + "storymonth": 23717, + "instagood": 23718, + "neighbour": 23719, + "ping": 23720, + "semite": 23721, + "mystic": 23722, + "ascot": 23723, + "mater": 23724, + "handful": 23725, + "dangers": 23726, + "tid": 23727, + "anaheim": 23728, + "opoly": 23729, + "shallow": 23730, + "namibia": 23731, + "toria": 23732, + "procurement": 23733, + "bigbang": 23734, + "announcements": 23735, + "prosecutor": 23736, + "bengals": 23737, + "salle": 23738, + "enroll": 23739, + "gastro": 23740, + "suggestion": 23741, + "bak": 23742, + "haul": 23743, + "buddhism": 23744, + "berniesanders": 23745, + "flute": 23746, + "fatigue": 23747, + "cynthia": 23748, + "choi": 23749, + "irwin": 23750, + "gua": 23751, + "strous": 23752, + "hp": 23753, + "bap": 23754, + "satisfying": 23755, + "playa": 23756, + "ðŁİ¼": 23757, + "instap": 23758, + "alice": 23759, + "tp": 23760, + "irrigation": 23761, + "ðŁĩ¬ðŁĩ§": 23762, + "intric": 23763, + "clues": 23764, + "plex": 23765, + "sax": 23766, + "hepat": 23767, + "dumped": 23768, + "significance": 23769, + "byu": 23770, + "medication": 23771, + "prov": 23772, + "toughest": 23773, + "cornish": 23774, + "âŀľ": 23775, + "kelley": 23776, + "uv": 23777, + "sizz": 23778, + "sibling": 23779, + "mest": 23780, + "distor": 23781, + "diplomatic": 23782, + "auntie": 23783, + "bhat": 23784, + "sonic": 23785, + "brenda": 23786, + "pumpkins": 23787, + "roch": 23788, + "blackburn": 23789, + "urged": 23790, + "shia": 23791, + "arrangements": 23792, + "flood": 23793, + "saunders": 23794, + "lecturer": 23795, + "nouri": 23796, + "populations": 23797, + "diplomacy": 23798, + "consistently": 23799, + "ðŁ¤Ļ": 23800, + "tmund": 23801, + "cauliflower": 23802, + "lily": 23803, + "vocabulary": 23804, + "varieties": 23805, + "cooker": 23806, + "uptown": 23807, + "quent": 23808, + "mosa": 23809, + "reinde": 23810, + "velocity": 23811, + "spruce": 23812, + "socialmedi": 23813, + "iber": 23814, + "voluntary": 23815, + "processed": 23816, + "baltic": 23817, + "yang": 23818, + "lebanese": 23819, + "dp": 23820, + "dolly": 23821, + "arrangement": 23822, + "yuri": 23823, + "cranberry": 23824, + "kalyan": 23825, + "elevation": 23826, + "cliff": 23827, + "pushes": 23828, + "ìĬ¤": 23829, + "silic": 23830, + "cowx": 23831, + "eternity": 23832, + "slaves": 23833, + "vinegar": 23834, + "gloucester": 23835, + "contained": 23836, + "breakingnews": 23837, + "against": 23838, + "renovated": 23839, + "normandy": 23840, + "heroin": 23841, + "ysm": 23842, + "mods": 23843, + "greek": 23844, + "undi": 23845, + "trench": 23846, + "vh": 23847, + "encourages": 23848, + "headache": 23849, + "grange": 23850, + ":'": 23851, + "evergreen": 23852, + "ÙĬ": 23853, + "reckon": 23854, + "abused": 23855, + "thru": 23856, + "choice": 23857, + "tidy": 23858, + "colder": 23859, + "schoice": 23860, + "hain": 23861, + "brum": 23862, + "liars": 23863, + "breit": 23864, + "yorker": 23865, + "shack": 23866, + "heidi": 23867, + "michaels": 23868, + "scopic": 23869, + "fascist": 23870, + "playful": 23871, + "cac": 23872, + "yasss": 23873, + "shad": 23874, + "..?": 23875, + "quen": 23876, + "ramirez": 23877, + "clifton": 23878, + "prs": 23879, + "bestfan": 23880, + "âģł": 23881, + "generating": 23882, + "headset": 23883, + "disappointment": 23884, + "abstract": 23885, + "boiled": 23886, + "parenthood": 23887, + "azerbaijan": 23888, + "exhibiting": 23889, + "bombay": 23890, + "olivier": 23891, + "koso": 23892, + "unlea": 23893, + "maternity": 23894, + "izer": 23895, + "sives": 23896, + "rhu": 23897, + "coll": 23898, + "saskatchewan": 23899, + "freakin": 23900, + "dek": 23901, + "nag": 23902, + "stabili": 23903, + "ðŁįķ": 23904, + "organizer": 23905, + "bosses": 23906, + "aru": 23907, + "uva": 23908, + "atable": 23909, + "taun": 23910, + "afterwards": 23911, + "fertili": 23912, + "verge": 23913, + "azi": 23914, + "morph": 23915, + "à¹ģà¸": 23916, + "jerk": 23917, + "cosmetic": 23918, + "kow": 23919, + "strust": 23920, + "apache": 23921, + "postcards": 23922, + "formul": 23923, + "ìĭ": 23924, + "spinal": 23925, + "jackpot": 23926, + "electri": 23927, + "ÃŃ": 23928, + "loy": 23929, + "grader": 23930, + "diablo": 23931, + "ardi": 23932, + "hesit": 23933, + "fw": 23934, + "archery": 23935, + "pash": 23936, + "theories": 23937, + "repeal": 23938, + "relive": 23939, + "percy": 23940, + "âĺĨ": 23941, + "imin": 23942, + "synchron": 23943, + "shampoo": 23944, + "coupons": 23945, + "oto": 23946, + "lai": 23947, + "thought": 23948, + "luxembourg": 23949, + "mov": 23950, + "ðŁĺ¥": 23951, + "gemma": 23952, + "seated": 23953, + "mga": 23954, + "stratford": 23955, + "uncertainty": 23956, + "shifts": 23957, + "esto": 23958, + "fool": 23959, + "firearms": 23960, + "corrie": 23961, + "kiki": 23962, + "apparent": 23963, + "pills": 23964, + "olympia": 23965, + "fid": 23966, + "elevated": 23967, + "decks": 23968, + "ignoring": 23969, + "avalan": 23970, + "rov": 23971, + "whistle": 23972, + "ptsd": 23973, + "militants": 23974, + "robotic": 23975, + "pacers": 23976, + "quilt": 23977, + "bankruptcy": 23978, + "lich": 23979, + "percussion": 23980, + "celebrity": 23981, + "als": 23982, + "(;": 23983, + "sut": 23984, + "pokemongo": 23985, + "hg": 23986, + "offs": 23987, + "gibraltar": 23988, + "screams": 23989, + "billie": 23990, + "genome": 23991, + "marin": 23992, + "beams": 23993, + "archbishop": 23994, + "emin": 23995, + "bedrooms": 23996, + "gated": 23997, + "olly": 23998, + "warranty": 23999, + "atown": 24000, + "cuddles": 24001, + "gunna": 24002, + "kic": 24003, + "vive": 24004, + "cymru": 24005, + "narrow": 24006, + "prob": 24007, + "leo": 24008, + "references": 24009, + "manufactured": 24010, + "chopper": 24011, + "brunswick": 24012, + "semis": 24013, + "donia": 24014, + "rye": 24015, + "mano": 24016, + "hurting": 24017, + "?#": 24018, + "holli": 24019, + "investigations": 24020, + "cels": 24021, + "ðŁĵŀ": 24022, + "lester": 24023, + "temples": 24024, + "storey": 24025, + "mcmahon": 24026, + "toilets": 24027, + "woof": 24028, + "ï¸İ": 24029, + "leverage": 24030, + "atom": 24031, + "nightmares": 24032, + "victorious": 24033, + "haunting": 24034, + "customer": 24035, + "agi": 24036, + "yoongi": 24037, + "monty": 24038, + "veronica": 24039, + "wur": 24040, + "intimid": 24041, + "blankets": 24042, + "volution": 24043, + "jm": 24044, + "âĺİ": 24045, + "amon": 24046, + "judith": 24047, + "ðŁĺİðŁĺİ": 24048, + "distracted": 24049, + "drip": 24050, + "hurricane": 24051, + "andes": 24052, + "revelation": 24053, + "troop": 24054, + "ableg": 24055, + "collin": 24056, + "tibetan": 24057, + "worrying": 24058, + "internationally": 24059, + "eater": 24060, + "cameroon": 24061, + "brador": 24062, + "yuk": 24063, + "ðŁĴĹðŁĴĹ": 24064, + "trak": 24065, + "slopes": 24066, + "cier": 24067, + "nea": 24068, + "oler": 24069, + "taka": 24070, + "albion": 24071, + "volcanic": 24072, + "amn": 24073, + "afi": 24074, + "obstac": 24075, + "facetime": 24076, + "gering": 24077, + "npr": 24078, + "metallica": 24079, + "organic": 24080, + "ðŁĴ¡": 24081, + "kidd": 24082, + "dances": 24083, + "pembro": 24084, + "washer": 24085, + "mits": 24086, + "omer": 24087, + "emotionally": 24088, + "tango": 24089, + "ipo": 24090, + "docks": 24091, + "scanning": 24092, + "specs": 24093, + "thom": 24094, + "theology": 24095, + "emergen": 24096, + "omi": 24097, + "gpa": 24098, + "selections": 24099, + "unnecessary": 24100, + "image": 24101, + "ters": 24102, + "induced": 24103, + "gigan": 24104, + "rentals": 24105, + "supplied": 24106, + "mfa": 24107, + "shankar": 24108, + "later": 24109, + "pajam": 24110, + "clave": 24111, + "Ùģ": 24112, + "mahin": 24113, + "carlson": 24114, + "avian": 24115, + "anova": 24116, + "katie": 24117, + "ajith": 24118, + "designated": 24119, + "chocolates": 24120, + "investigators": 24121, + "glazed": 24122, + "princess": 24123, + "erry": 24124, + "ragn": 24125, + "ourable": 24126, + "hru": 24127, + "sundance": 24128, + "peugeot": 24129, + "steampunk": 24130, + "ghlin": 24131, + "grease": 24132, + "hires": 24133, + "zap": 24134, + "perce": 24135, + "jill": 24136, + "tome": 24137, + "hehehe": 24138, + "joyful": 24139, + "maestro": 24140, + "nished": 24141, + "genealo": 24142, + "vich": 24143, + "pits": 24144, + "foxes": 24145, + "goodman": 24146, + "emerson": 24147, + "lobes": 24148, + "converse": 24149, + "oats": 24150, + "thomson": 24151, + "rahim": 24152, + "malware": 24153, + "ahi": 24154, + "mankind": 24155, + "resin": 24156, + "img": 24157, + "swood": 24158, + "kinder": 24159, + "scroll": 24160, + "ara": 24161, + "sakura": 24162, + "robbed": 24163, + "xion": 24164, + "nya": 24165, + "cism": 24166, + "cedar": 24167, + "bein": 24168, + "mourning": 24169, + "torto": 24170, + "heathrow": 24171, + "donegal": 24172, + "barb": 24173, + "hydration": 24174, + "kor": 24175, + "elimination": 24176, + "supdates": 24177, + "hills": 24178, + "appeti": 24179, + "starred": 24180, + "kom": 24181, + "gwen": 24182, + "ddd": 24183, + "cray": 24184, + "scanner": 24185, + "personalised": 24186, + "serenity": 24187, + "redesign": 24188, + "metaph": 24189, + "boxed": 24190, + "judgment": 24191, + "nose": 24192, + "ë¹": 24193, + "erad": 24194, + "acne": 24195, + "suppliers": 24196, + "energetic": 24197, + "vom": 24198, + "asap": 24199, + "ðŁĶ¸": 24200, + "irvine": 24201, + "hatch": 24202, + "lass": 24203, + "adren": 24204, + "waffles": 24205, + "accurately": 24206, + "icio": 24207, + "ittle": 24208, + "seun": 24209, + "occupy": 24210, + "webcam": 24211, + "thenew": 24212, + "entes": 24213, + "gai": 24214, + "jw": 24215, + "accountable": 24216, + "visor": 24217, + "irrit": 24218, + "licensing": 24219, + "huddersfield": 24220, + "genie": 24221, + "ðŁİ¾": 24222, + "atmospheric": 24223, + "tensions": 24224, + "spartan": 24225, + "clifford": 24226, + "olan": 24227, + "northbound": 24228, + "ameen": 24229, + "censor": 24230, + "uel": 24231, + "stery": 24232, + "$$": 24233, + "farrell": 24234, + "hyster": 24235, + "clt": 24236, + "sedan": 24237, + "replied": 24238, + "describing": 24239, + "microwave": 24240, + "slab": 24241, + "prosp": 24242, + "assisting": 24243, + "rubio": 24244, + "ethan": 24245, + "hhhhh": 24246, + "guay": 24247, + "zman": 24248, + "raise": 24249, + "rolling": 24250, + "oe": 24251, + "nile": 24252, + "ambrose": 24253, + "scarborough": 24254, + "heroic": 24255, + "cooks": 24256, + "mort": 24257, + "chopra": 24258, + "ðŁĮ·": 24259, + "tob": 24260, + "shaving": 24261, + "stacey": 24262, + "dorm": 24263, + "motorsports": 24264, + "wiki": 24265, + "folds": 24266, + "spiced": 24267, + "stressful": 24268, + "literal": 24269, + "fudge": 24270, + "peggy": 24271, + "waite": 24272, + "tresses": 24273, + "sesh": 24274, + "pric": 24275, + "ðŁİħ": 24276, + "fright": 24277, + "rva": 24278, + "mumbai": 24279, + "pom": 24280, + "ttv": 24281, + "cellar": 24282, + "tome": 24283, + "android": 24284, + "doris": 24285, + "tsunami": 24286, + "tinder": 24287, + "oec": 24288, + "mwc": 24289, + "dortmund": 24290, + "nothin": 24291, + "liti": 24292, + "sou": 24293, + "believein": 24294, + "atu": 24295, + "knocks": 24296, + "magni": 24297, + "sssss": 24298, + "rohit": 24299, + "inews": 24300, + "angi": 24301, + "mandy": 24302, + "kettle": 24303, + "intermediate": 24304, + "avant": 24305, + "curl": 24306, + "endorsed": 24307, + "orio": 24308, + "urt": 24309, + "consideration": 24310, + "wires": 24311, + "shelters": 24312, + "bino": 24313, + "vikram": 24314, + "implemented": 24315, + "lydia": 24316, + "buk": 24317, + "parody": 24318, + "cnews": 24319, + "undergraduate": 24320, + "canucks": 24321, + "sami": 24322, + "politically": 24323, + "rotten": 24324, + "ghz": 24325, + "textiles": 24326, + "overload": 24327, + "moderni": 24328, + "recreational": 24329, + "flir": 24330, + "baton": 24331, + "typography": 24332, + "ovation": 24333, + "intriguing": 24334, + "pilgrimage": 24335, + "alge": 24336, + "adays": 24337, + "tcmparty": 24338, + "spelled": 24339, + "curls": 24340, + "booze": 24341, + "stem": 24342, + "annes": 24343, + "irls": 24344, + "sponge": 24345, + "shopper": 24346, + "signation": 24347, + "brass": 24348, + "mistress": 24349, + "leah": 24350, + "beginner": 24351, + "lauderdale": 24352, + "august": 24353, + "preschool": 24354, + "taping": 24355, + "taipei": 24356, + "executives": 24357, + "bd": 24358, + "rhetor": 24359, + "escor": 24360, + "immuno": 24361, + "deeplearning": 24362, + "statues": 24363, + "itus": 24364, + "manuscript": 24365, + "lyric": 24366, + "corvette": 24367, + "molly": 24368, + "lage": 24369, + "dep": 24370, + "cnbc": 24371, + "lest": 24372, + "jessi": 24373, + "fife": 24374, + "griffith": 24375, + "opposing": 24376, + "rang": 24377, + "drills": 24378, + "respectful": 24379, + "pity": 24380, + "dell": 24381, + "harding": 24382, + "playboy": 24383, + "bloke": 24384, + "shutout": 24385, + "kili": 24386, + "osp": 24387, + "seattle": 24388, + "bcpoli": 24389, + "mises": 24390, + "journals": 24391, + "teaming": 24392, + "esther": 24393, + "freddy": 24394, + "Ķï¸ı": 24395, + "metrics": 24396, + "notre": 24397, + "garry": 24398, + "forty": 24399, + "navigate": 24400, + "periods": 24401, + "benedic": 24402, + "jid": 24403, + "daw": 24404, + "ancestors": 24405, + "restoring": 24406, + "cong": 24407, + "allergy": 24408, + "titanium": 24409, + "cence": 24410, + "leaning": 24411, + "abbas": 24412, + "vast": 24413, + "ucf": 24414, + "roofing": 24415, + "eman": 24416, + "severely": 24417, + "vogue": 24418, + "veau": 24419, + "inbound": 24420, + "dz": 24421, + "taneously": 24422, + "stretching": 24423, + "manchester": 24424, + "dryer": 24425, + "davis": 24426, + "kanth": 24427, + "thegame": 24428, + "itted": 24429, + "retain": 24430, + "elles": 24431, + "congestion": 24432, + "fraternity": 24433, + "ollie": 24434, + "loki": 24435, + "freely": 24436, + "choo": 24437, + "pony": 24438, + "scep": 24439, + "tably": 24440, + "balt": 24441, + "rockn": 24442, + "dime": 24443, + "logging": 24444, + "ðŁį·": 24445, + "adu": 24446, + "havoc": 24447, + "waterford": 24448, + "charis": 24449, + "sweetie": 24450, + "running": 24451, + "nerd": 24452, + "erdogan": 24453, + "zara": 24454, + "weighing": 24455, + "fifty": 24456, + "precise": 24457, + "lowell": 24458, + "kurdistan": 24459, + "ryo": 24460, + "orth": 24461, + "synth": 24462, + "liners": 24463, + "phenomenon": 24464, + "artillery": 24465, + "illegally": 24466, + "construct": 24467, + "nostalgic": 24468, + "garth": 24469, + "alta": 24470, + "shelton": 24471, + "asean": 24472, + "wander": 24473, + "durban": 24474, + "diversi": 24475, + "bono": 24476, + "clon": 24477, + "leman": 24478, + "shun": 24479, + "obstacles": 24480, + "appetite": 24481, + "feeder": 24482, + "respiratory": 24483, + "dixie": 24484, + "formula": 24485, + "anto": 24486, + "sober": 24487, + "extinct": 24488, + "auc": 24489, + "ingles": 24490, + "legitimate": 24491, + ";;": 24492, + "minnie": 24493, + "ipswich": 24494, + "dramatically": 24495, + "ðŁijıðŁı¼": 24496, + "ingham": 24497, + "military": 24498, + "monet": 24499, + "usnavy": 24500, + "fork": 24501, + "dunno": 24502, + "player": 24503, + "qotd": 24504, + "stoo": 24505, + "exor": 24506, + "ethiopian": 24507, + "filmfest": 24508, + "pered": 24509, + "cate": 24510, + "saudi": 24511, + "inner": 24512, + "sincere": 24513, + "tionality": 24514, + "alee": 24515, + "deeds": 24516, + "cooperative": 24517, + "ironic": 24518, + "crocod": 24519, + "brary": 24520, + "postseason": 24521, + "camper": 24522, + "canary": 24523, + "ein": 24524, + "extensions": 24525, + "nbd": 24526, + "sherwood": 24527, + "spokane": 24528, + "hump": 24529, + "jitsu": 24530, + "ê¹": 24531, + "daryl": 24532, + "psi": 24533, + "stabbed": 24534, + "offerings": 24535, + "expects": 24536, + "caval": 24537, + "bodybuilding": 24538, + "framing": 24539, + "fca": 24540, + "yearly": 24541, + "bombed": 24542, + "skil": 24543, + "researching": 24544, + "judiciary": 24545, + "greeted": 24546, + "tudor": 24547, + "milo": 24548, + "innovate": 24549, + "ðŁĺĽ": 24550, + "rhs": 24551, + "ruby": 24552, + "contributor": 24553, + "famer": 24554, + "socially": 24555, + "mlin": 24556, + "fiery": 24557, + "utter": 24558, + "beaut": 24559, + "itos": 24560, + "devoted": 24561, + "rainbow": 24562, + "barney": 24563, + "peren": 24564, + "arjun": 24565, + "rna": 24566, + "gabby": 24567, + "uti": 24568, + "hannity": 24569, + "pickle": 24570, + "serv": 24571, + "quakes": 24572, + "ppe": 24573, + "fem": 24574, + "whitec": 24575, + "jn": 24576, + "victories": 24577, + "ðŁ§¡": 24578, + "golfer": 24579, + "congratulates": 24580, + "resulting": 24581, + "mechanic": 24582, + "urve": 24583, + "centered": 24584, + "kiev": 24585, + "ans": 24586, + "incub": 24587, + "<<": 24588, + "cmo": 24589, + "bestfanarmy": 24590, + "daph": 24591, + "enham": 24592, + "oncology": 24593, + "kush": 24594, + "txt": 24595, + "oriented": 24596, + "fashionable": 24597, + "csr": 24598, + "sahara": 24599, + "rack": 24600, + "pdp": 24601, + "hanson": 24602, + "à¸ĩ": 24603, + "tiers": 24604, + "rar": 24605, + "panam": 24606, + "insky": 24607, + "sahi": 24608, + "testament": 24609, + "asthma": 24610, + "inher": 24611, + "fisheries": 24612, + "order": 24613, + "howe": 24614, + "gallon": 24615, + "epis": 24616, + "suzanne": 24617, + "drowning": 24618, + "panelists": 24619, + "ðŁĺ²": 24620, + "ë¦": 24621, + "alach": 24622, + "commemorative": 24623, + "attribu": 24624, + "ðŁij»": 24625, + "moo": 24626, + "visional": 24627, + "weeksary": 24628, + "gust": 24629, + "akin": 24630, + "pointe": 24631, + "eee": 24632, + "dispar": 24633, + "nipp": 24634, + "dental": 24635, + "stall": 24636, + "pian": 24637, + "bore": 24638, + "ulster": 24639, + "tick": 24640, + "irr": 24641, + "taehyung": 24642, + "microphone": 24643, + "bermuda": 24644, + "gaard": 24645, + "eler": 24646, + "plumbing": 24647, + "hugely": 24648, + "âļ«ï¸ı": 24649, + "raceway": 24650, + "cambridge": 24651, + "marcel": 24652, + "burnley": 24653, + "toast": 24654, + "hollywood": 24655, + "fasting": 24656, + "mered": 24657, + "hibition": 24658, + "capped": 24659, + "beneficial": 24660, + "owning": 24661, + "contamin": 24662, + "arabian": 24663, + "toon": 24664, + "capac": 24665, + "hulu": 24666, + "smir": 24667, + "nutrients": 24668, + "sein": 24669, + "graphs": 24670, + "conditional": 24671, + "ðŁijħ": 24672, + "orac": 24673, + "playin": 24674, + "northe": 24675, + "tornad": 24676, + "marian": 24677, + "jumbo": 24678, + "lexi": 24679, + "incredibleindia": 24680, + "roadto": 24681, + "ukone": 24682, + "confusing": 24683, + "sph": 24684, + "shank": 24685, + "pied": 24686, + "mqm": 24687, + "positively": 24688, + "sherry": 24689, + "pathways": 24690, + "considers": 24691, + "tofu": 24692, + "arguments": 24693, + "resilient": 24694, + "chett": 24695, + "withdra": 24696, + "tero": 24697, + "atedly": 24698, + "swana": 24699, + "heb": 24700, + "flight": 24701, + "harley": 24702, + "decrease": 24703, + "kindle": 24704, + "bookshop": 24705, + "³ï¸ı": 24706, + "martyrs": 24707, + "smur": 24708, + "mccl": 24709, + "concerto": 24710, + "stime": 24711, + "rejoice": 24712, + "applau": 24713, + "clement": 24714, + "merkel": 24715, + "jaime": 24716, + "immortal": 24717, + "isleof": 24718, + "marco": 24719, + "youtuber": 24720, + "stalking": 24721, + "metoo": 24722, + "stack": 24723, + "spouse": 24724, + "ust": 24725, + "luv": 24726, + "âļ¾ï¸ı": 24727, + "equestrian": 24728, + "eving": 24729, + "flin": 24730, + "nickname": 24731, + "thebig": 24732, + "asar": 24733, + "stacks": 24734, + "walker": 24735, + "bora": 24736, + "kidnapped": 24737, + "hurling": 24738, + "humbold": 24739, + "recalls": 24740, + "copper": 24741, + "annis": 24742, + "seo": 24743, + "merger": 24744, + "muir": 24745, + "addy": 24746, + "ðŁĴªðŁĴª": 24747, + "bex": 24748, + "cracy": 24749, + "conan": 24750, + "congratulation": 24751, + "midst": 24752, + "âĻ¬": 24753, + "forbi": 24754, + "optic": 24755, + "crate": 24756, + "crocodile": 24757, + "madagas": 24758, + "securing": 24759, + "aston": 24760, + "ogue": 24761, + "savior": 24762, + "salisbury": 24763, + "loveit": 24764, + "fujifilm": 24765, + "castles": 24766, + "asst": 24767, + "arrows": 24768, + "spacious": 24769, + "trs": 24770, + "polyvore": 24771, + "progression": 24772, + "mri": 24773, + "nelson": 24774, + "bim": 24775, + "indicator": 24776, + "oda": 24777, + "pepe": 24778, + "resignation": 24779, + "gut": 24780, + "sneaker": 24781, + "logically": 24782, + "azy": 24783, + "arella": 24784, + "tearing": 24785, + "joshi": 24786, + "ssionism": 24787, + "qpr": 24788, + "mariah": 24789, + "px": 24790, + "bleed": 24791, + "mian": 24792, + "medley": 24793, + "weiss": 24794, + "kerry": 24795, + "gatory": 24796, + "atal": 24797, + "madison": 24798, + "avenger": 24799, + "naby": 24800, + "pland": 24801, + "giles": 24802, + "freshwater": 24803, + "dington": 24804, + "taj": 24805, + "demonstrates": 24806, + "ntv": 24807, + "bulbs": 24808, + "sundaymorning": 24809, + "peake": 24810, + "souvenir": 24811, + "wah": 24812, + "tonnes": 24813, + "mkt": 24814, + "complexity": 24815, + "conden": 24816, + "rossi": 24817, + "bing": 24818, + "yds": 24819, + "suk": 24820, + "ngo": 24821, + "midland": 24822, + "oly": 24823, + "lifeis": 24824, + "ripple": 24825, + "moreno": 24826, + "dders": 24827, + "tus": 24828, + "áĥ": 24829, + "boul": 24830, + "xa": 24831, + "holdings": 24832, + "wny": 24833, + "shadowhunters": 24834, + "kei": 24835, + "aspire": 24836, + "mous": 24837, + "owen": 24838, + "soak": 24839, + "skirts": 24840, + "mountaine": 24841, + "storming": 24842, + "chrome": 24843, + "riots": 24844, + "sarato": 24845, + "amaze": 24846, + "lessness": 24847, + "navar": 24848, + "criteria": 24849, + "rafa": 24850, + "indulge": 24851, + "ayer": 24852, + "porto": 24853, + "namo": 24854, + "................": 24855, + "yields": 24856, + "valle": 24857, + "jh": 24858, + "macron": 24859, + "sains": 24860, + "durant": 24861, + "trailers": 24862, + "wot": 24863, + "confederate": 24864, + "shrin": 24865, + "idol": 24866, + "formally": 24867, + "tene": 24868, + "motorcycles": 24869, + "thang": 24870, + "node": 24871, + "banger": 24872, + "daly": 24873, + "pats": 24874, + "enrollment": 24875, + "auctions": 24876, + "atal": 24877, + "arbor": 24878, + "logos": 24879, + "dearest": 24880, + "transaction": 24881, + "domingo": 24882, + "flea": 24883, + "sermon": 24884, + "deck": 24885, + "sincere": 24886, + "questioning": 24887, + "julio": 24888, + "wasp": 24889, + "pretz": 24890, + "armenian": 24891, + "kham": 24892, + "inflammation": 24893, + "picturesque": 24894, + "accidental": 24895, + "filmmakers": 24896, + "ðŁĺļ": 24897, + "ðŁĴį": 24898, + "casey": 24899, + "sob": 24900, + "yeezy": 24901, + "goodwill": 24902, + "paragra": 24903, + "ssly": 24904, + "feather": 24905, + "dyed": 24906, + "assassination": 24907, + "nade": 24908, + "bcs": 24909, + "applies": 24910, + "feminine": 24911, + "feu": 24912, + "extent": 24913, + "deputies": 24914, + "lack": 24915, + "psychic": 24916, + "goi": 24917, + "killings": 24918, + "pseu": 24919, + "ðŁ¤ª": 24920, + "unc": 24921, + "marl": 24922, + "tane": 24923, + "mckenna": 24924, + "surfer": 24925, + "influences": 24926, + "freeway": 24927, + "hackney": 24928, + "malaria": 24929, + "eland": 24930, + "teau": 24931, + "remastered": 24932, + "ر": 24933, + "razor": 24934, + "ggy": 24935, + "corro": 24936, + "laksh": 24937, + "flair": 24938, + "honesty": 24939, + "hooray": 24940, + "depp": 24941, + "amc": 24942, + "wednesdays": 24943, + "qa": 24944, + "edits": 24945, + "-$": 24946, + "sevilla": 24947, + "doubled": 24948, + "humanities": 24949, + "ccot": 24950, + "somos": 24951, + "rine": 24952, + "afa": 24953, + "sioux": 24954, + "reconstruction": 24955, + "welding": 24956, + "threads": 24957, + "amish": 24958, + "encouragement": 24959, + "poder": 24960, + "bock": 24961, + "balm": 24962, + "ptions": 24963, + "standup": 24964, + "accomplishments": 24965, + "guarding": 24966, + "conviction": 24967, + "acion": 24968, + "napoleon": 24969, + "depicting": 24970, + "attack": 24971, + "sui": 24972, + "wearable": 24973, + "âĸªï¸ı": 24974, + "potter": 24975, + "escort": 24976, + "vise": 24977, + "tots": 24978, + "boon": 24979, + "eventprofs": 24980, + "angular": 24981, + "womenshistorymonth": 24982, + "barrow": 24983, + "schi": 24984, + "accomp": 24985, + "tik": 24986, + "lend": 24987, + "kensington": 24988, + "wolfe": 24989, + "stacked": 24990, + "crashing": 24991, + "exhibit": 24992, + "winged": 24993, + "sabrina": 24994, + "masa": 24995, + "kms": 24996, + "always": 24997, + "ett": 24998, + "plasma": 24999, + "counseling": 25000, + "pickles": 25001, + "nfldraft": 25002, + "mrs": 25003, + "inevitable": 25004, + "courageous": 25005, + "stafford": 25006, + "writerslife": 25007, + "hos": 25008, + "ej": 25009, + "ghyun": 25010, + "trademark": 25011, + "adrian": 25012, + "influencer": 25013, + "coronation": 25014, + "raging": 25015, + "explored": 25016, + "usaf": 25017, + "exception": 25018, + "eux": 25019, + "tanker": 25020, + "swami": 25021, + "packet": 25022, + "ðŁij¨âĢį": 25023, + "fen": 25024, + "sheen": 25025, + "aero": 25026, + "jl": 25027, + "regal": 25028, + "nwt": 25029, + "auster": 25030, + "mehta": 25031, + "charge": 25032, + "aste": 25033, + "bate": 25034, + "infeld": 25035, + "racecourse": 25036, + "collapsed": 25037, + "fleece": 25038, + "zil": 25039, + "allie": 25040, + "alternatives": 25041, + "georges": 25042, + "ðŁĵį": 25043, + "quirky": 25044, + "fcb": 25045, + "natgeo": 25046, + "philanthropy": 25047, + "brai": 25048, + "everyday": 25049, + "ðŁIJ°": 25050, + "achers": 25051, + "jaan": 25052, + "fines": 25053, + "qi": 25054, + "fisherman": 25055, + "distinct": 25056, + "grimes": 25057, + "nationalist": 25058, + "commence": 25059, + "rown": 25060, + "âĢ³": 25061, + "zing": 25062, + "fter": 25063, + "hrw": 25064, + "baroque": 25065, + "blender": 25066, + "kitty": 25067, + "hooks": 25068, + "cited": 25069, + "wanda": 25070, + "consensus": 25071, + "reindeer": 25072, + "anand": 25073, + "supply": 25074, + "meds": 25075, + "vn": 25076, + "olph": 25077, + "ratchet": 25078, + "sheldon": 25079, + "securities": 25080, + "ë°©íĥ": 25081, + "crom": 25082, + "mosquito": 25083, + "jeric": 25084, + "immac": 25085, + "dimensions": 25086, + "â¤": 25087, + "dissi": 25088, + "spongebob": 25089, + "damien": 25090, + "stevenson": 25091, + "joanne": 25092, + "delish": 25093, + "yikes": 25094, + "thanx": 25095, + "surveys": 25096, + "postponed": 25097, + "alcoholic": 25098, + "alised": 25099, + "ðŁĻıðŁı»": 25100, + "doch": 25101, + "sentim": 25102, + "meredith": 25103, + "compares": 25104, + "bago": 25105, + "happydays": 25106, + "moss": 25107, + "ãħĭ": 25108, + "nec": 25109, + "gnment": 25110, + "frustrated": 25111, + "combin": 25112, + "riv": 25113, + "eclec": 25114, + "collo": 25115, + "compliment": 25116, + "actorslife": 25117, + "ctto": 25118, + "nicar": 25119, + "ophon": 25120, + "aparthe": 25121, + "mant": 25122, + "jade": 25123, + "trolley": 25124, + "optimization": 25125, + "eyeon": 25126, + "ecological": 25127, + "quist": 25128, + "ephe": 25129, + "à¥ĩ": 25130, + "cinco": 25131, + "appoints": 25132, + "oldschool": 25133, + "cpr": 25134, + "behavioral": 25135, + "minaj": 25136, + ":-(": 25137, + "tagging": 25138, + "eval": 25139, + "joaqu": 25140, + "ðŁĺ«": 25141, + "hak": 25142, + "deme": 25143, + "jamaican": 25144, + "sos": 25145, + "hyatt": 25146, + "handbook": 25147, + "librarian": 25148, + "hannibal": 25149, + "pumping": 25150, + "chom": 25151, + "fman": 25152, + "gai": 25153, + "hull": 25154, + "responders": 25155, + "greenville": 25156, + "nus": 25157, + "vaugh": 25158, + "ðŁİīðŁİī": 25159, + "taxi": 25160, + "goldberg": 25161, + "mantra": 25162, + "tease": 25163, + "forbidden": 25164, + "methodist": 25165, + "ativity": 25166, + "****": 25167, + "ect": 25168, + "mcgr": 25169, + "Ħëĭ": 25170, + "seb": 25171, + "amidst": 25172, + "disappear": 25173, + "thyro": 25174, + "philips": 25175, + "erina": 25176, + "vicious": 25177, + "streamer": 25178, + "millionaire": 25179, + "map": 25180, + "strick": 25181, + "hackathon": 25182, + "gha": 25183, + "edic": 25184, + "mika": 25185, + "peck": 25186, + "illi": 25187, + "antoine": 25188, + "arca": 25189, + "optic": 25190, + "maure": 25191, + "ðŁĩ¦ðŁĩº": 25192, + "clashes": 25193, + "manly": 25194, + "âĺģ": 25195, + "alvar": 25196, + "andres": 25197, + "mei": 25198, + "elm": 25199, + "wwww": 25200, + "altered": 25201, + "lte": 25202, + "ê¹Ģ": 25203, + "mojo": 25204, + "forrest": 25205, + "thalai": 25206, + "nont": 25207, + "speeches": 25208, + "acknowledge": 25209, + "ignite": 25210, + "xfactor": 25211, + "ðŁ¥Ĥ": 25212, + "meadow": 25213, + "disrupt": 25214, + "debuted": 25215, + "scrimmage": 25216, + "pharmaceutical": 25217, + "fidd": 25218, + "foundations": 25219, + "philosopher": 25220, + "etal": 25221, + "publishers": 25222, + "boys": 25223, + "cke": 25224, + "rugged": 25225, + "optimism": 25226, + "rebe": 25227, + "philharmon": 25228, + "narcis": 25229, + "rallies": 25230, + "luis": 25231, + "goblue": 25232, + "folded": 25233, + "unacceptable": 25234, + "optimal": 25235, + "lisa": 25236, + "polaro": 25237, + "+.": 25238, + "enza": 25239, + "âĿ£ï¸ı": 25240, + "monopoly": 25241, + "graceful": 25242, + "dairy": 25243, + "dua": 25244, + "difficulty": 25245, + "judgement": 25246, + "osi": 25247, + "mersey": 25248, + "flux": 25249, + "newfound": 25250, + "terns": 25251, + "dimensional": 25252, + "invic": 25253, + "alba": 25254, + "amit": 25255, + "abudhabi": 25256, + "algeria": 25257, + "automobile": 25258, + "thead": 25259, + "lotion": 25260, + "accelerator": 25261, + "vacant": 25262, + "ition": 25263, + "luf": 25264, + "alic": 25265, + "pll": 25266, + "blazing": 25267, + "baz": 25268, + "sene": 25269, + "ðŁij¼": 25270, + "villains": 25271, + "directory": 25272, + "eisen": 25273, + "tock": 25274, + "brochure": 25275, + "ripp": 25276, + "hbd": 25277, + "zaynmalik": 25278, + "niche": 25279, + "lolol": 25280, + "certificates": 25281, + "morse": 25282, + "facup": 25283, + "xham": 25284, + "unwanted": 25285, + "imports": 25286, + "carnegie": 25287, + "fansign": 25288, + "mou": 25289, + "ralph": 25290, + "destroyer": 25291, + "swing": 25292, + "trekking": 25293, + "ciliation": 25294, + "pitbull": 25295, + "gaps": 25296, + "howell": 25297, + "definitive": 25298, + "mcle": 25299, + "fps": 25300, + "etz": 25301, + "bolly": 25302, + "lynn": 25303, + "gano": 25304, + "ature": 25305, + "fursuit": 25306, + "coil": 25307, + "nav": 25308, + "butts": 25309, + "trojans": 25310, + "eure": 25311, + "enko": 25312, + "schumer": 25313, + "horrific": 25314, + "installment": 25315, + "brb": 25316, + "suburbs": 25317, + "abel": 25318, + "vir": 25319, + "desh": 25320, + "cunningham": 25321, + "ðŁIJ»": 25322, + "spann": 25323, + "schwe": 25324, + "kemp": 25325, + "tru": 25326, + "stealth": 25327, + "ques": 25328, + "lew": 25329, + "delights": 25330, + "koch": 25331, + "humili": 25332, + "criti": 25333, + "ilt": 25334, + "spells": 25335, + "miley": 25336, + "caric": 25337, + "ðŁį´": 25338, + "lcfc": 25339, + "substitute": 25340, + "oung": 25341, + "?!!": 25342, + "affir": 25343, + "predictable": 25344, + "classof": 25345, + "err": 25346, + "cypress": 25347, + "chandra": 25348, + "ageing": 25349, + "____": 25350, + "therland": 25351, + "doncaster": 25352, + "elin": 25353, + "yoshi": 25354, + "sailors": 25355, + "harris": 25356, + "joanna": 25357, + "nigerians": 25358, + "hers": 25359, + "plague": 25360, + "procra": 25361, + "kno": 25362, + "canton": 25363, + "busines": 25364, + "unh": 25365, + "prakash": 25366, + "cin": 25367, + "bowen": 25368, + "coating": 25369, + "mals": 25370, + "begging": 25371, + "smithson": 25372, + "pontiac": 25373, + "spies": 25374, + "damian": 25375, + "pline": 25376, + "undant": 25377, + "alta": 25378, + "oness": 25379, + "shameless": 25380, + "daq": 25381, + "bbm": 25382, + "wales": 25383, + "stampede": 25384, + "serum": 25385, + "ÙĨ": 25386, + "catalyst": 25387, + "xn": 25388, + "absc": 25389, + "freezer": 25390, + "chun": 25391, + "arios": 25392, + "mccre": 25393, + "forehead": 25394, + "hears": 25395, + "damascus": 25396, + "tacoma": 25397, + "arduino": 25398, + "encounters": 25399, + "stanton": 25400, + "lgb": 25401, + "abas": 25402, + "\"..": 25403, + "kete": 25404, + "dracula": 25405, + "elem": 25406, + "gne": 25407, + "zeppelin": 25408, + "labrador": 25409, + "pulp": 25410, + "optional": 25411, + "orn": 25412, + "russians": 25413, + "sanitation": 25414, + "hilary": 25415, + "etsymntt": 25416, + "penalties": 25417, + "aust": 25418, + "igans": 25419, + "olympian": 25420, + "medicaid": 25421, + "versace": 25422, + "vape": 25423, + "restra": 25424, + "peep": 25425, + "sexiest": 25426, + "stalls": 25427, + "dile": 25428, + "thea": 25429, + "punjabi": 25430, + "puppy": 25431, + "tuesdaymotivation": 25432, + "ðŁĵļ": 25433, + "theflash": 25434, + "rocket": 25435, + "modest": 25436, + "chihuahu": 25437, + "onna": 25438, + "ksa": 25439, + "hurdles": 25440, + "cave": 25441, + "failures": 25442, + "split": 25443, + "boho": 25444, + "gurl": 25445, + "disappoint": 25446, + "howard": 25447, + "nugget": 25448, + "franz": 25449, + "stalert": 25450, + "kazakh": 25451, + "forgetting": 25452, + "schri": 25453, + "agate": 25454, + "amat": 25455, + "everett": 25456, + "duet": 25457, + "veterinary": 25458, + "julian": 25459, + "chills": 25460, + "brave": 25461, + "ghostbusters": 25462, + "lando": 25463, + "greets": 25464, + "profitable": 25465, + "dé": 25466, + "tir": 25467, + "zee": 25468, + "omen": 25469, + "pdx": 25470, + "grayson": 25471, + "hari": 25472, + "fixes": 25473, + "stabbing": 25474, + "swimmer": 25475, + "symbols": 25476, + "compliments": 25477, + "pose": 25478, + "functioning": 25479, + "thnx": 25480, + "gir": 25481, + "corporations": 25482, + "barlow": 25483, + "loe": 25484, + "offseason": 25485, + "distinctive": 25486, + "marvelous": 25487, + "nikon": 25488, + "enrique": 25489, + "kyu": 25490, + "jaws": 25491, + "amoto": 25492, + "lombar": 25493, + "travelblogger": 25494, + "fah": 25495, + "ourism": 25496, + "tristan": 25497, + "soe": 25498, + "cease": 25499, + "ðŁıħ": 25500, + "zac": 25501, + "mckenzie": 25502, + "taxpayers": 25503, + "swimsuit": 25504, + "blo": 25505, + "lesley": 25506, + "kansas": 25507, + "wks": 25508, + "kiel": 25509, + "provoking": 25510, + "myles": 25511, + "string": 25512, + "kangaroo": 25513, + "galactic": 25514, + "fifth": 25515, + "ske": 25516, + "weir": 25517, + "llis": 25518, + "matory": 25519, + "ðŁĩ¿": 25520, + "unci": 25521, + "reproductive": 25522, + "rooting": 25523, + "tides": 25524, + "gadget": 25525, + "..........": 25526, + "alexander": 25527, + "bowler": 25528, + "screw": 25529, + "apolog": 25530, + "erika": 25531, + "walters": 25532, + "shetty": 25533, + "lane": 25534, + "banter": 25535, + "asant": 25536, + "meso": 25537, + "vain": 25538, + "\"\"\"": 25539, + "usi": 25540, + "ferdin": 25541, + "accomplish": 25542, + "mansfield": 25543, + "bombar": 25544, + "collaborating": 25545, + "clap": 25546, + "iture": 25547, + "sda": 25548, + "smoky": 25549, + "nak": 25550, + "imperson": 25551, + "carla": 25552, + "comra": 25553, + "burgl": 25554, + "loco": 25555, + "ties": 25556, + "inhi": 25557, + "tracey": 25558, + "seis": 25559, + "disser": 25560, + "rrrr": 25561, + "dray": 25562, + "protect": 25563, + "corona": 25564, + "hunger": 25565, + "cken": 25566, + "celi": 25567, + "troubled": 25568, + "predators": 25569, + "fictional": 25570, + "shaved": 25571, + "richest": 25572, + "metaboli": 25573, + "fulham": 25574, + "grooming": 25575, + "monochrome": 25576, + "wasting": 25577, + "asco": 25578, + "aste": 25579, + "tista": 25580, + "remedies": 25581, + "ungsoo": 25582, + "southend": 25583, + "permanently": 25584, + "bumble": 25585, + "procrastin": 25586, + "identical": 25587, + "practically": 25588, + "mascul": 25589, + "suke": 25590, + "assured": 25591, + "valerie": 25592, + "deviant": 25593, + "grizzlies": 25594, + "thier": 25595, + "pura": 25596, + "nepal": 25597, + "notts": 25598, + "bilateral": 25599, + "spoil": 25600, + "carmel": 25601, + "cinematic": 25602, + "phl": 25603, + "nifty": 25604, + "mao": 25605, + "hypocri": 25606, + "laser": 25607, + "pantry": 25608, + "mathematical": 25609, + "elisa": 25610, + "coordination": 25611, + "belmont": 25612, + "ait": 25613, + "radiant": 25614, + "boiler": 25615, + "mang": 25616, + "fag": 25617, + "crc": 25618, + "hams": 25619, + "brin": 25620, + "â¬ĩï¸ı": 25621, + "familia": 25622, + "âĿ£": 25623, + "saber": 25624, + "rupert": 25625, + "ggan": 25626, + "ritz": 25627, + "mich": 25628, + "salford": 25629, + "levi": 25630, + "gral": 25631, + "ðŁĴ¤": 25632, + "nino": 25633, + "ced": 25634, + "businessman": 25635, + "ultr": 25636, + "simply": 25637, + "compression": 25638, + "pains": 25639, + "halt": 25640, + "ë°©íĥĦ": 25641, + "landscaping": 25642, + "nf": 25643, + "crooked": 25644, + "erd": 25645, + "ittin": 25646, + "ddleston": 25647, + "surpassed": 25648, + "inoa": 25649, + "dag": 25650, + "blen": 25651, + "extending": 25652, + "ating": 25653, + "algae": 25654, + "baller": 25655, + "umar": 25656, + "snooker": 25657, + "collu": 25658, + "flown": 25659, + "thub": 25660, + "ridiculously": 25661, + "kish": 25662, + "ople": 25663, + "dire": 25664, + "asser": 25665, + "aristo": 25666, + "sciss": 25667, + "hating": 25668, + "trouble": 25669, + "sylvia": 25670, + "succul": 25671, + "plots": 25672, + "sincerely": 25673, + "aler": 25674, + "laureate": 25675, + "brack": 25676, + "attn": 25677, + "rifles": 25678, + "meto": 25679, + "collectible": 25680, + "cuomo": 25681, + "contestant": 25682, + "consistency": 25683, + "antz": 25684, + "ranges": 25685, + "abigail": 25686, + "deb": 25687, + "minister": 25688, + "growers": 25689, + "anoo": 25690, + "hoover": 25691, + "dreamer": 25692, + "nucle": 25693, + "research": 25694, + "miy": 25695, + "shahid": 25696, + "mav": 25697, + "dhoni": 25698, + "cini": 25699, + "doj": 25700, + "hindus": 25701, + "partying": 25702, + "dali": 25703, + "alonso": 25704, + "informal": 25705, + "clarkson": 25706, + "itton": 25707, + "kian": 25708, + "cityo": 25709, + "mori": 25710, + "lasted": 25711, + "aspen": 25712, + "library": 25713, + "suspici": 25714, + "quat": 25715, + "denial": 25716, + "folder": 25717, + "chori": 25718, + "sweeping": 25719, + "enix": 25720, + "ðŁįĤ": 25721, + "ØŃ": 25722, + "nascar": 25723, + "handmadehour": 25724, + "moul": 25725, + "heatwave": 25726, + "emer": 25727, + "examine": 25728, + "ibn": 25729, + "grind": 25730, + "pov": 25731, + "tionist": 25732, + "mbo": 25733, + "sheila": 25734, + "integrate": 25735, + "omes": 25736, + "takeaway": 25737, + "cerv": 25738, + "connie": 25739, + "ticket": 25740, + "celed": 25741, + "bien": 25742, + "visually": 25743, + "madagascar": 25744, + "sorry": 25745, + "gui": 25746, + "parkrun": 25747, + "traits": 25748, + "labe": 25749, + "poisoning": 25750, + "à¥Ģ": 25751, + "viable": 25752, + "bohemian": 25753, + "dentistry": 25754, + "bados": 25755, + "sprouts": 25756, + "masked": 25757, + "teddy": 25758, + "ðŁĺ·": 25759, + "saf": 25760, + "saas": 25761, + "jiang": 25762, + "tight": 25763, + "speaker": 25764, + "withdrawal": 25765, + "bcn": 25766, + "assigned": 25767, + "classrooms": 25768, + "fleming": 25769, + "ðŁĴ«": 25770, + "supergirl": 25771, + "totals": 25772, + "tabletop": 25773, + "ebooks": 25774, + "horizontal": 25775, + "craz": 25776, + "flush": 25777, + "jard": 25778, + "cdc": 25779, + "erson": 25780, + "ãħł": 25781, + "greenwood": 25782, + "nih": 25783, + "cox": 25784, + "ada": 25785, + "litre": 25786, + "going": 25787, + "vicky": 25788, + "curved": 25789, + "louie": 25790, + "grains": 25791, + "hye": 25792, + "longe": 25793, + "remedy": 25794, + "trainee": 25795, + "sanjay": 25796, + "superstars": 25797, + "maser": 25798, + "manu": 25799, + "sage": 25800, + "whl": 25801, + "ðŁĺĤðŁĺŃ": 25802, + "ðŁijįðŁı»": 25803, + "msd": 25804, + "enz": 25805, + "rabhu": 25806, + "joo": 25807, + "ghu": 25808, + "acer": 25809, + "epo": 25810, + "resurrection": 25811, + "justicefor": 25812, + "blended": 25813, + "moda": 25814, + "avalanche": 25815, + "francesco": 25816, + "respective": 25817, + "gs": 25818, + "yeast": 25819, + "welch": 25820, + "devotion": 25821, + "getin": 25822, + "atheism": 25823, + "amic": 25824, + "carolyn": 25825, + "loc": 25826, + "ldnont": 25827, + "avec": 25828, + "usda": 25829, + "legged": 25830, + "bravery": 25831, + "blower": 25832, + "cowboy": 25833, + "heh": 25834, + "stible": 25835, + "buffal": 25836, + "channel": 25837, + "runchat": 25838, + "âĺķï¸ı": 25839, + "ideology": 25840, + "bestseller": 25841, + "yoo": 25842, + "peanu": 25843, + "bonne": 25844, + "felic": 25845, + "edison": 25846, + "fractu": 25847, + "narendra": 25848, + "ppets": 25849, + "seymour": 25850, + "riviera": 25851, + "hector": 25852, + "necessarily": 25853, + "bianca": 25854, + "societies": 25855, + "thebest": 25856, + "wg": 25857, + "sentences": 25858, + "wink": 25859, + "vaccines": 25860, + "palooza": 25861, + "jamming": 25862, + "asf": 25863, + "mpus": 25864, + "agreements": 25865, + "eck": 25866, + "bac": 25867, + "honore": 25868, + "compul": 25869, + "wildcat": 25870, + "imposed": 25871, + "yoga": 25872, + "hudson": 25873, + "canceled": 25874, + "lich": 25875, + "fuzzy": 25876, + "esque": 25877, + "chuk": 25878, + "wvu": 25879, + "sek": 25880, + "flipping": 25881, + "rhon": 25882, + "wished": 25883, + "wha": 25884, + "capability": 25885, + "lenovo": 25886, + "ìĨĮëħĦëĭ": 25887, + "vivo": 25888, + "tvd": 25889, + "nora": 25890, + "silk": 25891, + "pasadena": 25892, + "yosemite": 25893, + "valuation": 25894, + "clocks": 25895, + "uber": 25896, + "mrc": 25897, + "darkest": 25898, + "aubre": 25899, + "sso": 25900, + "belly": 25901, + "wrestlers": 25902, + "killin": 25903, + "louder": 25904, + "buckley": 25905, + "geel": 25906, + "adon": 25907, + "uns": 25908, + "appealing": 25909, + "ðŁij¯": 25910, + "semitism": 25911, + "listens": 25912, + "fitz": 25913, + "ãĥ³ãĥ": 25914, + "nylon": 25915, + "arty": 25916, + "seemingly": 25917, + "hala": 25918, + "suited": 25919, + "ety": 25920, + "sheds": 25921, + "muffins": 25922, + "apric": 25923, + "uments": 25924, + "uta": 25925, + "jammu": 25926, + "chelseafc": 25927, + "starz": 25928, + "yoko": 25929, + "root": 25930, + "cleansing": 25931, + "diar": 25932, + "pioneering": 25933, + "iheartradio": 25934, + "digiti": 25935, + "findyour": 25936, + "cano": 25937, + "ðŁĴİ": 25938, + "zol": 25939, + "spacecraft": 25940, + "sixers": 25941, + "moisturi": 25942, + "bile": 25943, + "tists": 25944, + "horton": 25945, + "ranging": 25946, + "columbi": 25947, + "meteoro": 25948, + "sentiment": 25949, + "epl": 25950, + "footh": 25951, + "textbook": 25952, + "drainage": 25953, + "rly": 25954, + "scue": 25955, + "imrankhan": 25956, + "ðŁĴ¸": 25957, + "margarita": 25958, + "eddy": 25959, + "predicts": 25960, + "gamergate": 25961, + "advise": 25962, + "growthhacking": 25963, + "loveyou": 25964, + "ugand": 25965, + "vf": 25966, + "benghazi": 25967, + "slater": 25968, + "newor": 25969, + "chel": 25970, + "independenceday": 25971, + "pnp": 25972, + "cullen": 25973, + "hoodies": 25974, + "numbered": 25975, + "britt": 25976, + "tsa": 25977, + "kltu": 25978, + "sages": 25979, + "momo": 25980, + "oneplus": 25981, + "coll": 25982, + "guts": 25983, + "wta": 25984, + "mesmeri": 25985, + "enhancing": 25986, + "chiroprac": 25987, + "jis": 25988, + "teenagers": 25989, + "mone": 25990, + "constellation": 25991, + "sweepstakes": 25992, + "eze": 25993, + "slovakia": 25994, + "laye": 25995, + "pearce": 25996, + "waver": 25997, + "pogba": 25998, + "kron": 25999, + "surgeons": 26000, + "marx": 26001, + "tid": 26002, + "gga": 26003, + "descend": 26004, + "pours": 26005, + "uprising": 26006, + "walla": 26007, + "sabbath": 26008, + "bachelore": 26009, + "mackin": 26010, + "kam": 26011, + "peterborough": 26012, + "hora": 26013, + "ðŁĮŁðŁĮŁ": 26014, + "thinkbig": 26015, + "rj": 26016, + "hydrau": 26017, + "spal": 26018, + "universit": 26019, + "ðŁıī": 26020, + "mailonline": 26021, + "leagueof": 26022, + "tenants": 26023, + "wally": 26024, + "lance": 26025, + "heavens": 26026, + "ddr": 26027, + "bolts": 26028, + "amir": 26029, + "iphone": 26030, + "cigar": 26031, + "endu": 26032, + "rei": 26033, + "elabor": 26034, + "ringing": 26035, + "johnson": 26036, + "characteristics": 26037, + "saloon": 26038, + "algorithms": 26039, + "talkin": 26040, + "mtn": 26041, + "dive": 26042, + "regionals": 26043, + "ffice": 26044, + "hati": 26045, + "deviantart": 26046, + "sotto": 26047, + "shiro": 26048, + "lama": 26049, + "kwe": 26050, + "faded": 26051, + "porting": 26052, + "tummy": 26053, + "estates": 26054, + "buenos": 26055, + "ðŁ¦ģ": 26056, + "believer": 26057, + "penetr": 26058, + "darn": 26059, + "spite": 26060, + "canopy": 26061, + "fashioni": 26062, + "tilla": 26063, + "petals": 26064, + "elijah": 26065, + "brawl": 26066, + "martyr": 26067, + "ë°©íĥĦìĨĮëħĦëĭ": 26068, + "midtown": 26069, + "erich": 26070, + "dapper": 26071, + "smtown": 26072, + "megam": 26073, + "www": 26074, + "lele": 26075, + "ons": 26076, + "catfish": 26077, + "firth": 26078, + "fossilfriday": 26079, + "ballpark": 26080, + "thaw": 26081, + "potent": 26082, + "illie": 26083, + "creep": 26084, + "carp": 26085, + "soap": 26086, + "gundam": 26087, + "infec": 26088, + "yyyyy": 26089, + "न": 26090, + "zag": 26091, + "ritt": 26092, + "calculator": 26093, + "boca": 26094, + "oko": 26095, + "toad": 26096, + "threaten": 26097, + "refined": 26098, + "olympic": 26099, + "accomplishment": 26100, + "bacterial": 26101, + "aji": 26102, + "tatum": 26103, + "feliz": 26104, + "sheed": 26105, + "jat": 26106, + "thic": 26107, + "jamal": 26108, + "ðĿĺ": 26109, + "lina": 26110, + "ðŁIJ¯": 26111, + "joking": 26112, + "yotpo": 26113, + "pinch": 26114, + "akron": 26115, + "herb": 26116, + "motivation": 26117, + "lia": 26118, + "hostage": 26119, + "creek": 26120, + "gamble": 26121, + "russell": 26122, + "patti": 26123, + "fotos": 26124, + "cpc": 26125, + "broken": 26126, + "backthe": 26127, + "clays": 26128, + "umm": 26129, + "stockton": 26130, + "maternal": 26131, + "ür": 26132, + "lakel": 26133, + "century": 26134, + "bek": 26135, + "infected": 26136, + "ม": 26137, + "smackdown": 26138, + "manned": 26139, + "tahoe": 26140, + "smes": 26141, + "basa": 26142, + "sula": 26143, + "augusta": 26144, + ".*": 26145, + "rohingya": 26146, + "greed": 26147, + "counselor": 26148, + "silhouette": 26149, + "gravit": 26150, + "clause": 26151, + "'-": 26152, + "bobc": 26153, + "occasions": 26154, + "nowadays": 26155, + "dictat": 26156, + "beard": 26157, + "nally": 26158, + "brightest": 26159, + "kabul": 26160, + "incindia": 26161, + "dhanush": 26162, + "archaeological": 26163, + "cheape": 26164, + "mizzou": 26165, + "dhi": 26166, + "ovski": 26167, + "baxter": 26168, + "assemble": 26169, + "â": 26170, + "gigi": 26171, + "acam": 26172, + "wisely": 26173, + "hazard": 26174, + "northampton": 26175, + "âľĪï¸ı": 26176, + "meth": 26177, + "blasting": 26178, + "reunite": 26179, + "mulus": 26180, + "alizes": 26181, + "tread": 26182, + "mila": 26183, + "edward": 26184, + "kova": 26185, + "pesto": 26186, + "ðŁij¶": 26187, + "vitz": 26188, + "hydraulic": 26189, + "refurbished": 26190, + "motel": 26191, + "isabella": 26192, + "homme": 26193, + "severance": 26194, + "uphol": 26195, + "miserable": 26196, + "fari": 26197, + "latter": 26198, + "efer": 26199, + "crackers": 26200, + "esl": 26201, + "acio": 26202, + "yyj": 26203, + "inan": 26204, + "ecb": 26205, + "zind": 26206, + "panas": 26207, + "trucking": 26208, + "reed": 26209, + "shaker": 26210, + "burgess": 26211, + "empire": 26212, + "agnes": 26213, + "nington": 26214, + "artworks": 26215, + "frs": 26216, + "tile": 26217, + "biome": 26218, + "eun": 26219, + "chong": 26220, + "americana": 26221, + "godfather": 26222, + "goblin": 26223, + "ishi": 26224, + "!).": 26225, + "tempted": 26226, + "genomics": 26227, + "mandate": 26228, + "cky": 26229, + "ðŁĴĻðŁĴĽ": 26230, + "somali": 26231, + "brandy": 26232, + "inven": 26233, + "spokesperson": 26234, + "pcb": 26235, + "yuan": 26236, + "hg": 26237, + "faz": 26238, + "starwars": 26239, + "rowan": 26240, + "bluegrass": 26241, + "dong": 26242, + "dday": 26243, + "trinidad": 26244, + "erton": 26245, + "banning": 26246, + "retention": 26247, + "cured": 26248, + "toberfest": 26249, + "reset": 26250, + "weis": 26251, + "detached": 26252, + "behindthescenes": 26253, + "immunity": 26254, + "pha": 26255, + "bray": 26256, + "ðŁij½": 26257, + "rancho": 26258, + "ramsay": 26259, + "estonia": 26260, + "ndtv": 26261, + "].": 26262, + "cabaret": 26263, + "taro": 26264, + "dv": 26265, + "showcases": 26266, + "plum": 26267, + "ðŁij¸": 26268, + "sonoma": 26269, + "prepa": 26270, + "memorab": 26271, + "estu": 26272, + "driveway": 26273, + "ules": 26274, + "magnus": 26275, + "xr": 26276, + "nnn": 26277, + "muchas": 26278, + "enge": 26279, + "streamed": 26280, + "forestry": 26281, + "audiobook": 26282, + "troy": 26283, + "reckless": 26284, + "kilom": 26285, + "ruler": 26286, + "rak": 26287, + "procession": 26288, + "ions": 26289, + "poole": 26290, + "noctur": 26291, + "whs": 26292, + "farmhouse": 26293, + "pera": 26294, + "parme": 26295, + "hypocrisy": 26296, + "sics": 26297, + "vant": 26298, + "cask": 26299, + "holistic": 26300, + "aust": 26301, + "п": 26302, + "indo": 26303, + "ðŁij©âĢį": 26304, + "diso": 26305, + "dispatch": 26306, + "olsen": 26307, + "makeit": 26308, + "ennis": 26309, + "centre": 26310, + "arrange": 26311, + "ðŁĮ¼": 26312, + "salted": 26313, + "easiest": 26314, + "fate": 26315, + "regatta": 26316, + "mozz": 26317, + "acan": 26318, + "sini": 26319, + "gically": 26320, + "chops": 26321, + "chicken": 26322, + "workin": 26323, + "hagg": 26324, + "involve": 26325, + "weeds": 26326, + "bookday": 26327, + "wakeup": 26328, + "kyr": 26329, + "michelin": 26330, + "fuss": 26331, + "rejuven": 26332, + "vacancies": 26333, + "incarcer": 26334, + "mst": 26335, + "scents": 26336, + "sovereign": 26337, + "kicker": 26338, + "à§": 26339, + "bod": 26340, + "âĢĶ>": 26341, + "sah": 26342, + "mobil": 26343, + "shropshire": 26344, + "ophone": 26345, + "dresser": 26346, + "missuni": 26347, + "hepburn": 26348, + "imo": 26349, + "foliage": 26350, + "diagnostic": 26351, + "assan": 26352, + "cycling": 26353, + "guilt": 26354, + "csa": 26355, + "puertorico": 26356, + "winelover": 26357, + "wakefield": 26358, + "doggy": 26359, + "khe": 26360, + "papp": 26361, + "cog": 26362, + "allot": 26363, + "cuck": 26364, + "poetic": 26365, + "mio": 26366, + "revit": 26367, + "magician": 26368, + "ç¥": 26369, + "antenna": 26370, + "westwood": 26371, + "mberg": 26372, + "luxe": 26373, + "oatmeal": 26374, + "ج": 26375, + "teat": 26376, + "ffee": 26377, + "searches": 26378, + "lly": 26379, + "pluto": 26380, + "elon": 26381, + "lettering": 26382, + "innocence": 26383, + "fai": 26384, + "annon": 26385, + "telangana": 26386, + "mait": 26387, + "neural": 26388, + "canni": 26389, + "aroma": 26390, + "astor": 26391, + "fex": 26392, + "cocac": 26393, + "monetary": 26394, + "fent": 26395, + "unsure": 26396, + "'@": 26397, + "indirec": 26398, + "tehran": 26399, + "isolation": 26400, + "libs": 26401, + "makeup": 26402, + "mercedes": 26403, + "ffy": 26404, + "hetero": 26405, + "deo": 26406, + "scom": 26407, + "cursed": 26408, + "veteransday": 26409, + "frankenstein": 26410, + "shrews": 26411, + "deco": 26412, + "geese": 26413, + "leftover": 26414, + "hadid": 26415, + "variable": 26416, + "academics": 26417, + "carolin": 26418, + "undergoing": 26419, + "variation": 26420, + "nah": 26421, + "ssier": 26422, + "gamersunite": 26423, + "pursuing": 26424, + "emerged": 26425, + "llers": 26426, + "controlling": 26427, + "roaring": 26428, + "meteor": 26429, + "volt": 26430, + "dawgs": 26431, + "beaver": 26432, + "islife": 26433, + "bathrooms": 26434, + "acional": 26435, + "prevent": 26436, + "lakedistrict": 26437, + "inals": 26438, + "yani": 26439, + "grabbing": 26440, + "sacks": 26441, + "lez": 26442, + "sway": 26443, + "kool": 26444, + "times": 26445, + "klopp": 26446, + "lade": 26447, + "concord": 26448, + "resulted": 26449, + "revive": 26450, + "reconciliation": 26451, + "oland": 26452, + "azz": 26453, + "giro": 26454, + "mandarin": 26455, + "deen": 26456, + "nutritional": 26457, + "iscoming": 26458, + "vani": 26459, + "awwww": 26460, + "derived": 26461, + "loveyour": 26462, + "stopthe": 26463, + "shouting": 26464, + "novak": 26465, + "ðŁĻĮðŁı¾": 26466, + "loaf": 26467, + "displaying": 26468, + "sundaywith": 26469, + "maguire": 26470, + "cheri": 26471, + "ðŁıŁ": 26472, + "rematch": 26473, + "quic": 26474, + "Ú©": 26475, + "yin": 26476, + "ðŁĺ¹": 26477, + "ilive": 26478, + "zip": 26479, + "ourke": 26480, + "downloads": 26481, + "swat": 26482, + "mississ": 26483, + "carers": 26484, + "tment": 26485, + "property": 26486, + "hahahahahaha": 26487, + "gibbs": 26488, + "surrey": 26489, + "arise": 26490, + "ticism": 26491, + "stia": 26492, + "irling": 26493, + "frog": 26494, + "cose": 26495, + "bassist": 26496, + "foreig": 26497, + "leau": 26498, + "pillows": 26499, + "holla": 26500, + "elie": 26501, + "disclosure": 26502, + "peanuts": 26503, + "intech": 26504, + "wwc": 26505, + "plunge": 26506, + "triumph": 26507, + "cori": 26508, + "slippers": 26509, + "ðŁĻıðŁĻı": 26510, + "neutrality": 26511, + "mare": 26512, + "hairy": 26513, + "gangster": 26514, + "humming": 26515, + "custard": 26516, + "merlin": 26517, + "alea": 26518, + "sby": 26519, + "damp": 26520, + "mohan": 26521, + "verbal": 26522, + "jst": 26523, + "gutted": 26524, + "bjor": 26525, + "unfinished": 26526, + "ðŁĩ¯ðŁĩµ": 26527, + "unhappy": 26528, + "âļ«ï¸ı": 26529, + "bypass": 26530, + "atsu": 26531, + "fischer": 26532, + "sav": 26533, + "africans": 26534, + "reuse": 26535, + "midway": 26536, + "demolished": 26537, + "gerrard": 26538, + "hercules": 26539, + "ÄŁ": 26540, + "medicines": 26541, + "clicking": 26542, + "surround": 26543, + "joong": 26544, + "waving": 26545, + "tribes": 26546, + "wetlands": 26547, + "officiel": 26548, + "arguing": 26549, + "lle": 26550, + "dova": 26551, + "suzy": 26552, + "clubhouse": 26553, + "negro": 26554, + "obtain": 26555, + "gao": 26556, + "glance": 26557, + "assist": 26558, + "chos": 26559, + "ãĤ¢": 26560, + "âĺķ": 26561, + "adrid": 26562, + "occurs": 26563, + "stans": 26564, + "pardon": 26565, + "liveli": 26566, + "employed": 26567, + "revisit": 26568, + "ffxiv": 26569, + "bble": 26570, + "nearing": 26571, + "miner": 26572, + "ðŁĺ¹": 26573, + "giovanni": 26574, + "upto": 26575, + "marvell": 26576, + "marse": 26577, + "towels": 26578, + "cbn": 26579, + "engineered": 26580, + "yelling": 26581, + "spartan": 26582, + "sians": 26583, + "ðŁĻĮðŁı¼": 26584, + "sev": 26585, + "coyote": 26586, + "stadi": 26587, + "tcm": 26588, + "appen": 26589, + "shenanigans": 26590, + "openaccess": 26591, + "soaked": 26592, + "masqu": 26593, + "levine": 26594, + "strokes": 26595, + "lk": 26596, + "apartheid": 26597, + "hiphop": 26598, + "chardon": 26599, + "maymay": 26600, + "haasan": 26601, + "stripped": 26602, + "fro": 26603, + "scription": 26604, + "fton": 26605, + "hf": 26606, + "prisons": 26607, + "marshal": 26608, + "ķãĤ": 26609, + "ancho": 26610, + "compromise": 26611, + "classification": 26612, + "buzzfeed": 26613, + "bbloggers": 26614, + "deserving": 26615, + ")/": 26616, + "sway": 26617, + "obo": 26618, + "campers": 26619, + "podernfamily": 26620, + "poured": 26621, + "brie": 26622, + "squirrels": 26623, + "seize": 26624, + ":#": 26625, + "lek": 26626, + "timb": 26627, + "stacy": 26628, + "nasdaq": 26629, + "repeatedly": 26630, + "brat": 26631, + "mighty": 26632, + "competitor": 26633, + "mahone": 26634, + "desi": 26635, + "oke": 26636, + "bmw": 26637, + "shie": 26638, + "fcb": 26639, + "cheapest": 26640, + "minimalist": 26641, + "paramount": 26642, + "nate": 26643, + "haras": 26644, + "insanity": 26645, + "lateral": 26646, + "mentality": 26647, + "mozam": 26648, + "tapped": 26649, + "yadav": 26650, + "usp": 26651, + "bway": 26652, + "theod": 26653, + "bilt": 26654, + "raids": 26655, + "empress": 26656, + "adapted": 26657, + "patron": 26658, + "nutshell": 26659, + "agra": 26660, + "beaded": 26661, + "sundaywithmarsha": 26662, + "viking": 26663, + "proceed": 26664, + "maintained": 26665, + "thinkbigsundaywithmarsha": 26666, + "snes": 26667, + "musica": 26668, + "tower": 26669, + "chab": 26670, + "bok": 26671, + "smt": 26672, + "insult": 26673, + "harvesting": 26674, + "window": 26675, + "ruther": 26676, + "beige": 26677, + "decal": 26678, + "indicate": 26679, + "mailing": 26680, + "rift": 26681, + "pole": 26682, + "anderson": 26683, + "choral": 26684, + "spride": 26685, + "lili": 26686, + "evelyn": 26687, + "imrankhanpti": 26688, + "....\"": 26689, + "kered": 26690, + "undp": 26691, + "waterfalls": 26692, + "sears": 26693, + "lemans": 26694, + "worldseries": 26695, + "riel": 26696, + "anie": 26697, + "appar": 26698, + "scorers": 26699, + "lamp": 26700, + "athan": 26701, + "physicians": 26702, + "quinoa": 26703, + "refusing": 26704, + "vuitton": 26705, + "unleash": 26706, + "sla": 26707, + "pati": 26708, + "shouts": 26709, + "intentions": 26710, + "foamed": 26711, + "european": 26712, + "neighborhoods": 26713, + "meer": 26714, + "manson": 26715, + "duh": 26716, + "brat": 26717, + "cones": 26718, + "bowl": 26719, + "kazakhstan": 26720, + "ि": 26721, + "inappropriate": 26722, + "delhi": 26723, + "ketchup": 26724, + "fulton": 26725, + "sys": 26726, + "consult": 26727, + "garfield": 26728, + "togo": 26729, + "fml": 26730, + "fled": 26731, + "bds": 26732, + "facilitate": 26733, + "reebok": 26734, + "selfie": 26735, + "elevate": 26736, + "activate": 26737, + "bible": 26738, + "cawx": 26739, + "bys": 26740, + "camille": 26741, + "syou": 26742, + "skool": 26743, + "hert": 26744, + "wbc": 26745, + "pledges": 26746, + "recorder": 26747, + "posh": 26748, + "acre": 26749, + "soaking": 26750, + "matil": 26751, + "vsco": 26752, + "shootings": 26753, + "plar": 26754, + "econ": 26755, + "ðŁĻĮðŁı»": 26756, + "rashid": 26757, + "ubi": 26758, + "ðŁ¤¤": 26759, + "swinging": 26760, + "wipe": 26761, + "raptor": 26762, + "msu": 26763, + "musicvideo": 26764, + "durham": 26765, + "attic": 26766, + "aparty": 26767, + "fetus": 26768, + "activation": 26769, + "aaz": 26770, + "motivate": 26771, + "ðŁĴķðŁĴķðŁĴķ": 26772, + "jal": 26773, + "म": 26774, + "agon": 26775, + "scheer": 26776, + "stalker": 26777, + "foster": 26778, + "azzo": 26779, + "telegram": 26780, + "vigor": 26781, + "slaugh": 26782, + "screenshots": 26783, + "entrepreneu": 26784, + "kristin": 26785, + "intention": 26786, + "chilli": 26787, + "fraction": 26788, + "dona": 26789, + "gea": 26790, + "tcu": 26791, + "site": 26792, + "lak": 26793, + "emil": 26794, + "dnt": 26795, + "boro": 26796, + "wilkinson": 26797, + "recu": 26798, + "atoday": 26799, + "tanya": 26800, + "blanco": 26801, + "cdn": 26802, + "brilliantly": 26803, + "gcc": 26804, + "acc": 26805, + "evacuated": 26806, + "therine": 26807, + "denny": 26808, + "caitlin": 26809, + "shepard": 26810, + "pouch": 26811, + "handheld": 26812, + "southeastern": 26813, + "haa": 26814, + "ô": 26815, + "resolutions": 26816, + "ledger": 26817, + "srin": 26818, + "rar": 26819, + "shattered": 26820, + "chimney": 26821, + "imwith": 26822, + "meteor": 26823, + "handled": 26824, + "rake": 26825, + "townsend": 26826, + "enhan": 26827, + "shipy": 26828, + "duct": 26829, + "twx": 26830, + "inflammatory": 26831, + "warhammer": 26832, + "theatrical": 26833, + "gros": 26834, + "skar": 26835, + "scotty": 26836, + "niel": 26837, + "tito": 26838, + "tini": 26839, + "connection": 26840, + "_.": 26841, + "goldenglobes": 26842, + "shaq": 26843, + "ðŁı³ï¸ı": 26844, + "hallway": 26845, + "fronts": 26846, + "effectiveness": 26847, + "glaston": 26848, + "dhs": 26849, + "expi": 26850, + "toh": 26851, + "cpl": 26852, + "scs": 26853, + "reo": 26854, + "hag": 26855, + "resemblance": 26856, + "horan": 26857, + "abusive": 26858, + "quer": 26859, + "virtue": 26860, + "cholester": 26861, + "aq": 26862, + "shane": 26863, + "mce": 26864, + "carriers": 26865, + "distress": 26866, + "rewind": 26867, + "¡": 26868, + "voodoo": 26869, + "intact": 26870, + "anno": 26871, + "ðŁĺ¤": 26872, + "piled": 26873, + "adia": 26874, + "ãĥ³": 26875, + "enow": 26876, + "digs": 26877, + "lightly": 26878, + "goofy": 26879, + "turbine": 26880, + "governors": 26881, + "conte": 26882, + "reopen": 26883, + "pah": 26884, + "ive": 26885, + "crafting": 26886, + "sweeps": 26887, + "jodi": 26888, + "ande": 26889, + "zucker": 26890, + "kawaii": 26891, + "oko": 26892, + "vai": 26893, + "outline": 26894, + "kristi": 26895, + "tsn": 26896, + "inspo": 26897, + "quint": 26898, + "filthy": 26899, + "lynne": 26900, + "listeners": 26901, + "departing": 26902, + "ord": 26903, + "tweed": 26904, + ",&": 26905, + "alek": 26906, + "selfish": 26907, + "norther": 26908, + "recognizes": 26909, + "ips": 26910, + "bes": 26911, + "aed": 26912, + "wills": 26913, + "peat": 26914, + "surroundings": 26915, + "monuments": 26916, + "aisle": 26917, + "becker": 26918, + "lav": 26919, + "quantity": 26920, + "vah": 26921, + "helicopters": 26922, + "tucked": 26923, + "alvarez": 26924, + "shape": 26925, + "obey": 26926, + "additi": 26927, + "roadside": 26928, + "mite": 26929, + "blers": 26930, + "epage": 26931, + "jau": 26932, + "ignorant": 26933, + "bins": 26934, + "lulu": 26935, + "xo": 26936, + "cfo": 26937, + "eeeee": 26938, + "apprenticeship": 26939, + "sheffiel": 26940, + "toi": 26941, + "hok": 26942, + "fakenews": 26943, + "deploy": 26944, + "aidan": 26945, + "huskers": 26946, + "ãĢİ": 26947, + "westbrook": 26948, + "mister": 26949, + "configur": 26950, + "carr": 26951, + "fica": 26952, + "proceedings": 26953, + "haw": 26954, + "steak": 26955, + "murderer": 26956, + "payday": 26957, + "ajo": 26958, + "pvc": 26959, + "donates": 26960, + "biaf": 26961, + "nomnom": 26962, + "beit": 26963, + "kali": 26964, + "xrp": 26965, + "ahmedabad": 26966, + "semic": 26967, + "chey": 26968, + "xtra": 26969, + "antwer": 26970, + "headlining": 26971, + "squares": 26972, + "rounded": 26973, + "fluore": 26974, + "bold": 26975, + "disasters": 26976, + "amoo": 26977, + "generic": 26978, + "cranes": 26979, + "briefly": 26980, + "gig": 26981, + "austerity": 26982, + "anticipation": 26983, + "forti": 26984, + "treasurer": 26985, + "canny": 26986, + "cecil": 26987, + "detected": 26988, + "checklist": 26989, + "ว": 26990, + "pamela": 26991, + "barbados": 26992, + "anfield": 26993, + "hearty": 26994, + "txlege": 26995, + "perenni": 26996, + "arrog": 26997, + "ingram": 26998, + "âĹı": 26999, + "tyne": 27000, + "spoon": 27001, + "ration": 27002, + "amba": 27003, + "mbe": 27004, + "camel": 27005, + "hhs": 27006, + "yorkshire": 27007, + "reflective": 27008, + "freaks": 27009, + "tok": 27010, + "judo": 27011, + "particles": 27012, + "dubs": 27013, + "banjo": 27014, + "accreditation": 27015, + "proverbs": 27016, + "overdose": 27017, + "integral": 27018, + "guang": 27019, + "mcs": 27020, + "supercar": 27021, + "afb": 27022, + "alvin": 27023, + "ails": 27024, + "xtre": 27025, + "staging": 27026, + "twent": 27027, + "rabbits": 27028, + "maro": 27029, + "instem": 27030, + "doll": 27031, + "cray": 27032, + "santana": 27033, + "bleach": 27034, + "minions": 27035, + "cheap": 27036, + "mant": 27037, + "divers": 27038, + "catalonia": 27039, + "lois": 27040, + "matri": 27041, + "cougar": 27042, + "kayak": 27043, + "egre": 27044, + "pso": 27045, + "aia": 27046, + "å®": 27047, + "charlton": 27048, + "tracked": 27049, + "scari": 27050, + "pett": 27051, + "fwd": 27052, + "xin": 27053, + "gravel": 27054, + "bric": 27055, + "biggboss": 27056, + "arden": 27057, + "hugging": 27058, + "palms": 27059, + "stv": 27060, + "limb": 27061, + "themovie": 27062, + "handicap": 27063, + "rime": 27064, + "zai": 27065, + "stub": 27066, + "india": 27067, + "lithuania": 27068, + "rhyth": 27069, + "pita": 27070, + "macedonia": 27071, + "highered": 27072, + "bridget": 27073, + "schwarz": 27074, + "skelet": 27075, + "hikes": 27076, + "antarctic": 27077, + "cps": 27078, + "mashup": 27079, + "а": 27080, + "nell": 27081, + "chandra": 27082, + "heir": 27083, + "anus": 27084, + "sheridan": 27085, + "mimi": 27086, + "museu": 27087, + "becca": 27088, + "anir": 27089, + "barrie": 27090, + "diocese": 27091, + "comparable": 27092, + "ðŁı³ï¸ıâĢį": 27093, + "yukon": 27094, + "mep": 27095, + "hormon": 27096, + "meric": 27097, + "alf": 27098, + "conquered": 27099, + "christchurch": 27100, + "ðŁĴĻðŁĴĻ": 27101, + "hazardous": 27102, + "pooh": 27103, + "conting": 27104, + "retrospective": 27105, + "parame": 27106, + "nair": 27107, + "consor": 27108, + "hotra": 27109, + "astonishing": 27110, + "caterpillar": 27111, + "uman": 27112, + "tism": 27113, + "tvs": 27114, + "servic": 27115, + "croydon": 27116, + "morales": 27117, + "cg": 27118, + "cum": 27119, + "teur": 27120, + "scanada": 27121, + "sall": 27122, + "magnolia": 27123, + "elise": 27124, + "thour": 27125, + "ி": 27126, + "agomez": 27127, + "phelps": 27128, + "ë°©íĥĦìĨĮëħĦëĭ¨": 27129, + "whos": 27130, + "weaving": 27131, + "sisd": 27132, + "proposes": 27133, + "crows": 27134, + "presale": 27135, + "economies": 27136, + "bernardo": 27137, + "shahid": 27138, + "airshow": 27139, + "mccann": 27140, + "horticul": 27141, + "nrl": 27142, + "duel": 27143, + "mongolia": 27144, + "toulou": 27145, + "requirement": 27146, + "structured": 27147, + "edi": 27148, + "olives": 27149, + "hea": 27150, + "cuter": 27151, + "к": 27152, + "enthusiast": 27153, + "harriet": 27154, + "dominion": 27155, + "submer": 27156, + "ðŁįĥ": 27157, + "saab": 27158, + "nesburg": 27159, + "moff": 27160, + "defended": 27161, + "burt": 27162, + "rewarded": 27163, + "goldman": 27164, + "optics": 27165, + "khalid": 27166, + "households": 27167, + "buckets": 27168, + "cecil": 27169, + "chess": 27170, + "substantial": 27171, + "efl": 27172, + "operation": 27173, + "evaluate": 27174, + "stn": 27175, + "recession": 27176, + "lll": 27177, + "tomas": 27178, + "truths": 27179, + "akbar": 27180, + "swords": 27181, + "pact": 27182, + "embarrass": 27183, + "hao": 27184, + "ayurve": 27185, + "scripture": 27186, + "nycc": 27187, + "opt": 27188, + "diameter": 27189, + "scented": 27190, + "organizers": 27191, + "relat": 27192, + "hae": 27193, + "dreamers": 27194, + "dese": 27195, + "ðŁĮ»": 27196, + "restricted": 27197, + "nale": 27198, + "rhp": 27199, + "dolan": 27200, + "munster": 27201, + "haired": 27202, + "consultants": 27203, + "joints": 27204, + "humil": 27205, + "dill": 27206, + "relentless": 27207, + "té": 27208, + "afil": 27209, + "utilities": 27210, + "japanese": 27211, + "condemn": 27212, + "petite": 27213, + "collide": 27214, + "qf": 27215, + "peaches": 27216, + "courier": 27217, + "lore": 27218, + "âĺİï¸ı": 27219, + "reliability": 27220, + "chuk": 27221, + "ðŁĻĥ": 27222, + "stures": 27223, + "gether": 27224, + "hostel": 27225, + "bier": 27226, + "-_-": 27227, + "âĩ": 27228, + "eze": 27229, + "tailo": 27230, + "dient": 27231, + "bluff": 27232, + "chuffed": 27233, + "pilip": 27234, + "monarch": 27235, + "eem": 27236, + "buchan": 27237, + "bick": 27238, + "opau": 27239, + "kups": 27240, + "ย": 27241, + "pistons": 27242, + "spins": 27243, + "mand": 27244, + "cest": 27245, + "burne": 27246, + "vile": 27247, + "cherries": 27248, + "beckett": 27249, + "needles": 27250, + "panch": 27251, + "ëĤ": 27252, + "hahah": 27253, + "troubles": 27254, + "insists": 27255, + "doyou": 27256, + "gmc": 27257, + "mortar": 27258, + "delegate": 27259, + "inn": 27260, + "ganda": 27261, + "sinatra": 27262, + "त": 27263, + "speeding": 27264, + "pupil": 27265, + "premises": 27266, + "alignment": 27267, + "pikach": 27268, + "asus": 27269, + "jalan": 27270, + "ص": 27271, + "limestone": 27272, + "folkl": 27273, + "parmesan": 27274, + "ceil": 27275, + "moy": 27276, + "shawnmendes": 27277, + "acup": 27278, + "hust": 27279, + "otes": 27280, + "medina": 27281, + "madi": 27282, + "gtav": 27283, + "censorship": 27284, + "arg": 27285, + "sweeney": 27286, + "sykes": 27287, + "colo": 27288, + "footsteps": 27289, + "canned": 27290, + "advance": 27291, + "gtaonline": 27292, + "healthyliving": 27293, + "ðŁį¾": 27294, + "aig": 27295, + "pality": 27296, + "ocs": 27297, + "hebrew": 27298, + "imminent": 27299, + "berkshire": 27300, + "jeremiah": 27301, + "outgoing": 27302, + "baker": 27303, + "entrata": 27304, + "maids": 27305, + "groves": 27306, + "boc": 27307, + "adel": 27308, + "mfw": 27309, + "conscience": 27310, + "armys": 27311, + "nutella": 27312, + "contestalert": 27313, + "novelist": 27314, + "lah": 27315, + "banker": 27316, + "marquez": 27317, + "ðŁı¡": 27318, + "toff": 27319, + "outage": 27320, + "grp": 27321, + "ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ": 27322, + "muscle": 27323, + "dudley": 27324, + "nvidia": 27325, + "midi": 27326, + "muni": 27327, + "essays": 27328, + "datac": 27329, + "carter": 27330, + "ร": 27331, + "tans": 27332, + "ives": 27333, + "publications": 27334, + "aler": 27335, + "okwx": 27336, + "ilu": 27337, + "cutt": 27338, + "harp": 27339, + "outlaw": 27340, + "lutheran": 27341, + "brill": 27342, + "bolic": 27343, + "dowell": 27344, + "greenland": 27345, + "besties": 27346, + "pathi": 27347, + "payton": 27348, + "guest": 27349, + "harden": 27350, + "ðŁ¤©": 27351, + "anned": 27352, + "evacuation": 27353, + "poised": 27354, + "mcder": 27355, + "bhan": 27356, + "oi": 27357, + "envelope": 27358, + "cid": 27359, + "cavi": 27360, + "tapas": 27361, + "bookreview": 27362, + "greyhound": 27363, + "âĻª": 27364, + "feud": 27365, + "lungs": 27366, + "forte": 27367, + "raider": 27368, + "ffer": 27369, + "onix": 27370, + "depend": 27371, + "ynwa": 27372, + "relating": 27373, + "devs": 27374, + "ðŁĴIJ": 27375, + "acquires": 27376, + "dha": 27377, + "jyo": 27378, + "privati": 27379, + "canine": 27380, + "kb": 27381, + "crab": 27382, + "sardin": 27383, + "imagining": 27384, + "kj": 27385, + "empor": 27386, + "downhill": 27387, + "nez": 27388, + "taeyeon": 27389, + "nickimin": 27390, + "gbp": 27391, + "àµ": 27392, + "wap": 27393, + "secco": 27394, + "mashed": 27395, + "ðŁĴ¥ðŁĴ¥": 27396, + "augustine": 27397, + "dissol": 27398, + "dictator": 27399, + "âĵ": 27400, + "viper": 27401, + "edfringe": 27402, + "vaux": 27403, + "hardwork": 27404, + "booklet": 27405, + "nox": 27406, + "chiff": 27407, + "ðŁĴ¨": 27408, + "observations": 27409, + "xboxone": 27410, + "usher": 27411, + "keer": 27412, + "lup": 27413, + "dallas": 27414, + "calgary": 27415, + "madra": 27416, + "dious": 27417, + "kbs": 27418, + "woodward": 27419, + "heroine": 27420, + "lumber": 27421, + "seaworld": 27422, + "ows": 27423, + "mcke": 27424, + "maverick": 27425, + "gula": 27426, + "crossroads": 27427, + "fang": 27428, + "sade": 27429, + "nikol": 27430, + "cheetah": 27431, + "mec": 27432, + "ppg": 27433, + "erick": 27434, + "ðŁİµ": 27435, + "toxic": 27436, + "bjj": 27437, + "viola": 27438, + "spire": 27439, + "chino": 27440, + "travis": 27441, + "institutional": 27442, + "haas": 27443, + "lowry": 27444, + "wac": 27445, + "eae": 27446, + "humid": 27447, + "mpton": 27448, + "ruck": 27449, + "jew": 27450, + "cine": 27451, + "zimmer": 27452, + "sef": 27453, + "bharat": 27454, + "frees": 27455, + "aamir": 27456, + "ðŁĴħ": 27457, + "zinc": 27458, + "wane": 27459, + "multiplayer": 27460, + "royalwedding": 27461, + "eel": 27462, + "precipit": 27463, + "query": 27464, + "kimberly": 27465, + "isabel": 27466, + "fulfill": 27467, + "igan": 27468, + "vaul": 27469, + "pane": 27470, + "scy": 27471, + "digit": 27472, + "gunn": 27473, + "utah": 27474, + "dogday": 27475, + "fion": 27476, + "xiaomi": 27477, + "dac": 27478, + "elast": 27479, + "chavez": 27480, + "roblo": 27481, + "gine": 27482, + "tenth": 27483, + "abh": 27484, + "keto": 27485, + "hurdle": 27486, + "nadia": 27487, + "memorabilia": 27488, + "habs": 27489, + "quan": 27490, + "hw": 27491, + "hvac": 27492, + "pixar": 27493, + "eccle": 27494, + "kramer": 27495, + "accuses": 27496, + "ðŁĴļðŁĴļ": 27497, + "perse": 27498, + "meantime": 27499, + "wahl": 27500, + "atletico": 27501, + "âĢ¢âĢ¢âĢ¢âĢ¢": 27502, + "ottoman": 27503, + "novo": 27504, + "kus": 27505, + "connected": 27506, + "trusts": 27507, + "dmv": 27508, + "spencer": 27509, + "rahulg": 27510, + "dove": 27511, + "stokes": 27512, + "bologna": 27513, + "enthusiasts": 27514, + "ê": 27515, + "rockstargames": 27516, + "tedcruz": 27517, + "duras": 27518, + "sacked": 27519, + "latex": 27520, + "immersive": 27521, + "cert": 27522, + "lucin": 27523, + "principals": 27524, + "fares": 27525, + "sails": 27526, + "farn": 27527, + "ament": 27528, + "saffron": 27529, + "quentin": 27530, + "checkpoint": 27531, + "ferris": 27532, + "excur": 27533, + "ðŁijīðŁı¼": 27534, + "bailey": 27535, + "seh": 27536, + "terre": 27537, + "madam": 27538, + "sband": 27539, + "wanderers": 27540, + "cumberbatch": 27541, + "yyc": 27542, + "digitally": 27543, + "blackandwhitephotography": 27544, + "rollin": 27545, + "moroccan": 27546, + "ðŁĮħ": 27547, + "dinner": 27548, + "dwell": 27549, + "toom": 27550, + "mye": 27551, + "ezra": 27552, + "cpfc": 27553, + "warhol": 27554, + "meer": 27555, + "jonah": 27556, + "noaa": 27557, + "sgate": 27558, + "soon": 27559, + "secular": 27560, + "gating": 27561, + "tio": 27562, + "driver": 27563, + "sissy": 27564, + "assange": 27565, + "tath": 27566, + "edmund": 27567, + "bobcats": 27568, + "raji": 27569, + "postage": 27570, + "studs": 27571, + "mgm": 27572, + "kato": 27573, + "edinburgh": 27574, + "meetthe": 27575, + "shirt": 27576, + "faa": 27577, + "mensfashion": 27578, + "spreads": 27579, + "wim": 27580, + "carts": 27581, + "phoebe": 27582, + "jars": 27583, + "botswana": 27584, + "ÙĤ": 27585, + "edwar": 27586, + "skar": 27587, + "rive": 27588, + "gusty": 27589, + "ctv": 27590, + "ferdinand": 27591, + "sutherland": 27592, + "nickiminaj": 27593, + "kv": 27594, + "sius": 27595, + "beech": 27596, + "rez": 27597, + "desires": 27598, + "onial": 27599, + "campo": 27600, + "quarry": 27601, + "lorraine": 27602, + "gilmore": 27603, + "iggy": 27604, + "µï¸ı": 27605, + "hopping": 27606, + "aviz": 27607, + "ðŁĮº": 27608, + "unisex": 27609, + "dedicate": 27610, + "attitudes": 27611, + "steer": 27612, + "junkie": 27613, + "railway": 27614, + "yb": 27615, + "whisper": 27616, + "keyan": 27617, + "kus": 27618, + "jug": 27619, + "dix": 27620, + "ains": 27621, + "summon": 27622, + "ovich": 27623, + "syed": 27624, + "herald": 27625, + "maison": 27626, + "meded": 27627, + "wildflower": 27628, + "mainland": 27629, + "risky": 27630, + "rukh": 27631, + "overlooked": 27632, + "kic": 27633, + "destroys": 27634, + "naman": 27635, + "kip": 27636, + "zano": 27637, + "championsleague": 27638, + "bandit": 27639, + "quincy": 27640, + "smile": 27641, + "calvin": 27642, + "openings": 27643, + "tapp": 27644, + "olulu": 27645, + "spectro": 27646, + "accredited": 27647, + "apk": 27648, + "praised": 27649, + "barnett": 27650, + "pollen": 27651, + "premiered": 27652, + "selenagomez": 27653, + "toured": 27654, + "screenings": 27655, + "uuu": 27656, + "miso": 27657, + "ense": 27658, + "adamlambert": 27659, + "guelph": 27660, + "haryana": 27661, + "hutto": 27662, + "lear": 27663, + "ltc": 27664, + "poached": 27665, + "brexit": 27666, + "æĿ": 27667, + "ttc": 27668, + "pavement": 27669, + "mongers": 27670, + "roe": 27671, + "aders": 27672, + "lington": 27673, + "participant": 27674, + "cared": 27675, + "gail": 27676, + "yates": 27677, + "lantic": 27678, + "dashboard": 27679, + "joo": 27680, + "felipe": 27681, + "ssionist": 27682, + "bum": 27683, + "send": 27684, + "aeri": 27685, + "thugs": 27686, + "lucifer": 27687, + "ahe": 27688, + "detector": 27689, + "filly": 27690, + "gasoline": 27691, + "hamper": 27692, + "humpday": 27693, + "theta": 27694, + "theband": 27695, + "forecasts": 27696, + "ohhh": 27697, + "lobb": 27698, + "holl": 27699, + "cpu": 27700, + "azu": 27701, + "adar": 27702, + "hailey": 27703, + "bub": 27704, + "cart": 27705, + "quoted": 27706, + "anarchy": 27707, + "pancre": 27708, + "twitart": 27709, + "alden": 27710, + "stash": 27711, + "theless": 27712, + "orni": 27713, + "beliebers": 27714, + "mormon": 27715, + "particle": 27716, + "aviation": 27717, + "â¬Ĩ": 27718, + "webcamtoy": 27719, + "saddened": 27720, + "cruis": 27721, + "hamlet": 27722, + "nct": 27723, + "rollins": 27724, + "marquee": 27725, + "sawyer": 27726, + "reliance": 27727, + "aura": 27728, + "diec": 27729, + "soothing": 27730, + "signings": 27731, + "akis": 27732, + "ó": 27733, + "atkins": 27734, + "aerop": 27735, + "ðŁĮ¿": 27736, + "yab": 27737, + "shari": 27738, + "connol": 27739, + "dubbed": 27740, + "manufacture": 27741, + "convincing": 27742, + "feelthebern": 27743, + "rau": 27744, + "pulit": 27745, + "onec": 27746, + "gemstone": 27747, + "urging": 27748, + "bagu": 27749, + "gah": 27750, + "acids": 27751, + "fianc": 27752, + "zodiac": 27753, + "snoop": 27754, + "herrera": 27755, + "initiated": 27756, + "venge": 27757, + "professors": 27758, + "prodi": 27759, + "stronger": 27760, + "emission": 27761, + "bba": 27762, + "halle": 27763, + "tapp": 27764, + "hawan": 27765, + "whim": 27766, + "competed": 27767, + "myrtle": 27768, + "irport": 27769, + "coldplay": 27770, + "ache": 27771, + "skep": 27772, + "mson": 27773, + "ssic": 27774, + "calligraphy": 27775, + "swimmers": 27776, + "mey": 27777, + "ppc": 27778, + "thrift": 27779, + "poc": 27780, + "replaces": 27781, + "commuter": 27782, + "âģ¦âģ¦@": 27783, + "goers": 27784, + "logue": 27785, + "paradig": 27786, + "baskets": 27787, + "sensitivity": 27788, + "johan": 27789, + "atlantis": 27790, + "&&": 27791, + "suitcase": 27792, + "anxious": 27793, + "lh": 27794, + "stri": 27795, + "galloway": 27796, + "stread": 27797, + "warden": 27798, + "grounded": 27799, + "fficiency": 27800, + "lifeat": 27801, + "relic": 27802, + "disguise": 27803, + "islanders": 27804, + "fcofficial": 27805, + "classicalmusic": 27806, + "bmc": 27807, + "enfield": 27808, + "bique": 27809, + "oakley": 27810, + "batman": 27811, + "slaying": 27812, + "nerves": 27813, + "multit": 27814, + "calcium": 27815, + "projector": 27816, + "scottsdale": 27817, + "antino": 27818, + "grips": 27819, + "kimmel": 27820, + "desmond": 27821, + "protestors": 27822, + "hiatus": 27823, + "metabolism": 27824, + "concluded": 27825, + "presser": 27826, + "tipping": 27827, + "slide": 27828, + "eto": 27829, + "hunting": 27830, + "ausopen": 27831, + "rik": 27832, + "ppery": 27833, + "innovators": 27834, + "pitchers": 27835, + "agger": 27836, + "fungi": 27837, + "zad": 27838, + "prolific": 27839, + "rocknroll": 27840, + "blames": 27841, + "ctar": 27842, + "stamford": 27843, + "qad": 27844, + "mozzarella": 27845, + "insanely": 27846, + "denver": 27847, + "phouse": 27848, + "nomad": 27849, + "ï¿": 27850, + "sris": 27851, + "produ": 27852, + "henley": 27853, + "pagan": 27854, + "amtrak": 27855, + "rubi": 27856, + "incl": 27857, + "tutor": 27858, + "scotia": 27859, + "woes": 27860, + "singapo": 27861, + "funnel": 27862, + "turnbull": 27863, + "knowledge": 27864, + "grimm": 27865, + "realmadrid": 27866, + "weare": 27867, + "missiles": 27868, + "consol": 27869, + "emojis": 27870, + "sneak": 27871, + "smiths": 27872, + "ruiz": 27873, + "brou": 27874, + "iel": 27875, + "haver": 27876, + "ðŁĮļ": 27877, + "kingof": 27878, + "basilica": 27879, + "circulation": 27880, + "printers": 27881, + "tapping": 27882, + "ridley": 27883, + "dragged": 27884, + "haj": 27885, + "writer": 27886, + "fundamentals": 27887, + "personalities": 27888, + "metre": 27889, + "stereotypes": 27890, + "burle": 27891, + "bestof": 27892, + "nffc": 27893, + "hath": 27894, + "ministries": 27895, + "aali": 27896, + "tracing": 27897, + "paved": 27898, + "łï¸ı": 27899, + "gic": 27900, + "inspire": 27901, + "tug": 27902, + "hare": 27903, + "repeated": 27904, + "expon": 27905, + "lolli": 27906, + "rhode": 27907, + "precin": 27908, + "installations": 27909, + "instagram": 27910, + "azar": 27911, + "ies": 27912, + "solely": 27913, + "dukes": 27914, + "missionary": 27915, + "vanguard": 27916, + "fursuitfriday": 27917, + "ond": 27918, + "polari": 27919, + "mast": 27920, + "haran": 27921, + "josé": 27922, + "jacked": 27923, + "ecoun": 27924, + "alities": 27925, + "neph": 27926, + "ravel": 27927, + "moderated": 27928, + "scow": 27929, + "sfb": 27930, + "uruguay": 27931, + "aso": 27932, + "nig": 27933, + "audu": 27934, + "pints": 27935, + "latina": 27936, + "benz": 27937, + "mitting": 27938, + "charted": 27939, + "matology": 27940, + "citro": 27941, + "biopic": 27942, + "ðŁijŃ": 27943, + "djokovic": 27944, + "foxy": 27945, + "aguil": 27946, + "soto": 27947, + "anada": 27948, + "sinking": 27949, + "scrap": 27950, + "hairs": 27951, + "bethany": 27952, + "factfriday": 27953, + "ðŁIJIJ": 27954, + "unleashed": 27955, + ")(": 27956, + "contradic": 27957, + "ramon": 27958, + "coastline": 27959, + "yong": 27960, + "snsd": 27961, + "ligan": 27962, + "pome": 27963, + "mitage": 27964, + "gett": 27965, + "wati": 27966, + "risk": 27967, + "soaring": 27968, + "brush": 27969, + "fpl": 27970, + "avan": 27971, + "åĨ": 27972, + "larson": 27973, + "shear": 27974, + "multil": 27975, + "blur": 27976, + "multimedia": 27977, + "chunky": 27978, + "pari": 27979, + "nani": 27980, + "weird": 27981, + "cholesterol": 27982, + "charles": 27983, + "dreamed": 27984, + "tanning": 27985, + "puzzles": 27986, + "fram": 27987, + "handball": 27988, + "chag": 27989, + "belize": 27990, + "alu": 27991, + "bangs": 27992, + "ÑĦ": 27993, + "detectives": 27994, + "mcg": 27995, + "ishq": 27996, + "bothered": 27997, + "safc": 27998, + "mping": 27999, + "teneri": 28000, + "gays": 28001, + "sailor": 28002, + "angi": 28003, + "multicul": 28004, + "guessed": 28005, + "rosé": 28006, + "highways": 28007, + "broom": 28008, + "chattanoo": 28009, + "-'": 28010, + "seeker": 28011, + "oned": 28012, + "atf": 28013, + "luc": 28014, + "><": 28015, + "bari": 28016, + "percep": 28017, + "jewelry": 28018, + "asph": 28019, + "sorrow": 28020, + "sling": 28021, + "mammoth": 28022, + "jackie": 28023, + "ë§": 28024, + "wiltshire": 28025, + "sao": 28026, + "cancell": 28027, + "impaired": 28028, + "torial": 28029, + "breed": 28030, + "guyen": 28031, + "judice": 28032, + "title": 28033, + "prospective": 28034, + "applicants": 28035, + "ðŁįĬ": 28036, + "episcop": 28037, + "eid": 28038, + "byo": 28039, + "stockings": 28040, + "ðŁĴĥðŁĴĥ": 28041, + "llp": 28042, + "snag": 28043, + "keepit": 28044, + "lough": 28045, + "olson": 28046, + "maturity": 28047, + "!!!\"": 28048, + "copter": 28049, + "isha": 28050, + "bli": 28051, + "wilmington": 28052, + "tryouts": 28053, + "thai": 28054, + "ðŁ¥³": 28055, + "pebble": 28056, + "kraft": 28057, + "fp": 28058, + "º": 28059, + "ssively": 28060, + "livin": 28061, + "contestants": 28062, + "textures": 28063, + "joan": 28064, + "hdr": 28065, + "filmfestival": 28066, + "provence": 28067, + "wido": 28068, + "opend": 28069, + "csi": 28070, + "stown": 28071, + "croati": 28072, + "adjust": 28073, + "hostile": 28074, + "analysts": 28075, + "ilan": 28076, + "cuppa": 28077, + "brum": 28078, + "newfoundland": 28079, + "goodwin": 28080, + "mett": 28081, + "mallorca": 28082, + "plugs": 28083, + "buk": 28084, + "bbhutto": 28085, + "wrestle": 28086, + "saire": 28087, + "shopped": 28088, + "forza": 28089, + "lehead": 28090, + "vivo": 28091, + "bast": 28092, + "roxy": 28093, + "regis": 28094, + "hardworking": 28095, + "honolulu": 28096, + "despair": 28097, + "youngsters": 28098, + "nig": 28099, + "impromp": 28100, + "rolltide": 28101, + "deemed": 28102, + "treason": 28103, + "rushed": 28104, + "forged": 28105, + "fff": 28106, + "pikachu": 28107, + "briggs": 28108, + "doit": 28109, + "accent": 28110, + "laus": 28111, + "glaze": 28112, + "competent": 28113, + "aho": 28114, + "photog": 28115, + "midfield": 28116, + "lego": 28117, + "harvard": 28118, + "minorities": 28119, + "reilly": 28120, + "sliced": 28121, + "onceupon": 28122, + "initially": 28123, + "financially": 28124, + "landscapephotography": 28125, + "hardro": 28126, + "quo": 28127, + "mmers": 28128, + "parkinson": 28129, + "smugg": 28130, + "readiness": 28131, + "brutally": 28132, + "gloucester": 28133, + "mped": 28134, + "bbhuttozardari": 28135, + "murder": 28136, + "yed": 28137, + "dataviz": 28138, + "srt": 28139, + "downing": 28140, + "bians": 28141, + "mü": 28142, + "fleck": 28143, + "flipped": 28144, + "sly": 28145, + "brilliance": 28146, + "rim": 28147, + "kum": 28148, + "bubba": 28149, + "koi": 28150, + "knitted": 28151, + "sorg": 28152, + "mais": 28153, + "ðŁĮ²": 28154, + "tiss": 28155, + "sustain": 28156, + "sensu": 28157, + "akhan": 28158, + "ziest": 28159, + "examines": 28160, + "chardonnay": 28161, + "username": 28162, + "shortlist": 28163, + "rebs": 28164, + "ono": 28165, + "daring": 28166, + "hardwood": 28167, + "cheque": 28168, + "righteous": 28169, + "lightening": 28170, + "dirk": 28171, + "shradd": 28172, + "dura": 28173, + "downstairs": 28174, + "shal": 28175, + "amigos": 28176, + "ruff": 28177, + "slaw": 28178, + "ries": 28179, + "rednation": 28180, + "manus": 28181, + "ðŁĩ§ðŁĩ·": 28182, + "distinction": 28183, + "ubun": 28184, + "duran": 28185, + "migra": 28186, + "thians": 28187, + "laver": 28188, + "domestic": 28189, + "kx": 28190, + "jazzy": 28191, + "justify": 28192, + "belonging": 28193, + "insulation": 28194, + "colorstv": 28195, + "drunken": 28196, + "channeling": 28197, + "quand": 28198, + "xiii": 28199, + "enlighten": 28200, + "kano": 28201, + "fatima": 28202, + "teenchoice": 28203, + "terrified": 28204, + "pba": 28205, + "asley": 28206, + "metmuseum": 28207, + "dune": 28208, + "packer": 28209, + "kio": 28210, + "ðŁĴľðŁĴľ": 28211, + "boiler": 28212, + "fascism": 28213, + "armored": 28214, + "backgrounds": 28215, + "inmates": 28216, + "embarrassed": 28217, + "defines": 28218, + "thd": 28219, + "wego": 28220, + "silicone": 28221, + "loon": 28222, + "elding": 28223, + "borrowed": 28224, + "hemp": 28225, + "aksh": 28226, + "kawasaki": 28227, + "bry": 28228, + "deaf": 28229, + "killer": 28230, + "disposal": 28231, + "ðŁĩ°": 28232, + "glastonbury": 28233, + "uncovered": 28234, + "oxide": 28235, + "poff": 28236, + "dant": 28237, + "kj": 28238, + "kuro": 28239, + "drizzle": 28240, + "peoples": 28241, + "fee": 28242, + "propri": 28243, + "ddlovato": 28244, + "piggy": 28245, + "otis": 28246, + "allergies": 28247, + "ubis": 28248, + "penguin": 28249, + "sera": 28250, + "viz": 28251, + "prosperous": 28252, + "icides": 28253, + "tornadoes": 28254, + "senegal": 28255, + "webcast": 28256, + "stored": 28257, + "enchanted": 28258, + "bbcone": 28259, + "bayarea": 28260, + "entrepreneurial": 28261, + "rednationrising": 28262, + "experimenting": 28263, + "angan": 28264, + "lotto": 28265, + "theyre": 28266, + "pore": 28267, + "erp": 28268, + "serene": 28269, + "eastwood": 28270, + "brokers": 28271, + "barge": 28272, + "stallion": 28273, + "timberlake": 28274, + "tailored": 28275, + "dystop": 28276, + "bate": 28277, + "lators": 28278, + "dixit": 28279, + "branson": 28280, + "dynamo": 28281, + "kylie": 28282, + "shameful": 28283, + "btwn": 28284, + "springtime": 28285, + "mixture": 28286, + "sounded": 28287, + "luton": 28288, + "dades": 28289, + "mala": 28290, + "opra": 28291, + "enic": 28292, + "rahulgandhi": 28293, + "sewer": 28294, + "~~~~": 28295, + "kyu": 28296, + "northeastern": 28297, + "caer": 28298, + "bcu": 28299, + "nirvana": 28300, + "kitchens": 28301, + "ousy": 28302, + "alm": 28303, + "riverdale": 28304, + "hidden": 28305, + "flint": 28306, + "spd": 28307, + "patrons": 28308, + "katyperry": 28309, + "augh": 28310, + "exhibitions": 28311, + "smc": 28312, + "shuts": 28313, + "atore": 28314, + "dain": 28315, + "something": 28316, + "berth": 28317, + "bog": 28318, + "porter": 28319, + "gento": 28320, + "concussion": 28321, + "anglic": 28322, + "rowe": 28323, + "grilling": 28324, + "scarlett": 28325, + "mastering": 28326, + "mornin": 28327, + "commented": 28328, + "sime": 28329, + "sizing": 28330, + "christy": 28331, + "ceos": 28332, + "stm": 28333, + "atry": 28334, + "tariffs": 28335, + "vacation": 28336, + "prejudice": 28337, + "psu": 28338, + "parental": 28339, + "farage": 28340, + "cana": 28341, + "capcom": 28342, + "kosovo": 28343, + "youre": 28344, + "menstru": 28345, + "stalin": 28346, + "grapefruit": 28347, + "bran": 28348, + "chesa": 28349, + "daven": 28350, + "excel": 28351, + "!!)": 28352, + "à¹Į": 28353, + "distributor": 28354, + "cea": 28355, + "bridesma": 28356, + "millennial": 28357, + "wain": 28358, + "observing": 28359, + "misery": 28360, + "planetary": 28361, + "exposing": 28362, + "braised": 28363, + "compton": 28364, + "dongha": 28365, + "ql": 28366, + "springsteen": 28367, + "thul": 28368, + "sylve": 28369, + "cabo": 28370, + "palad": 28371, + "nielsen": 28372, + "gazing": 28373, + "baja": 28374, + "roud": 28375, + "orchids": 28376, + "johannesburg": 28377, + "seman": 28378, + "dji": 28379, + "operative": 28380, + "affection": 28381, + "eclectic": 28382, + "atc": 28383, + "mutant": 28384, + "awx": 28385, + "nice": 28386, + "melbourne": 28387, + "indulg": 28388, + "tulip": 28389, + "diaspora": 28390, + "welp": 28391, + "biggie": 28392, + "mississauga": 28393, + "retriever": 28394, + "oran": 28395, + "tammy": 28396, + "cta": 28397, + "hippo": 28398, + "seasoned": 28399, + "germans": 28400, + "engv": 28401, + "marvellous": 28402, + "imf": 28403, + "relays": 28404, + "montan": 28405, + "mauriti": 28406, + "meister": 28407, + "assurance": 28408, + "reigning": 28409, + "sufficient": 28410, + "hane": 28411, + "nothing": 28412, + "posse": 28413, + "navy": 28414, + "inlove": 28415, + "brighton": 28416, + "enqu": 28417, + "chung": 28418, + "sweaty": 28419, + "esc": 28420, + "caled": 28421, + "mans": 28422, + "nicaragua": 28423, + "slices": 28424, + "mocha": 28425, + "washingtonpost": 28426, + "bbn": 28427, + "damned": 28428, + "growing": 28429, + "enburg": 28430, + "loan": 28431, + "mes": 28432, + "whoops": 28433, + "believers": 28434, + "spiel": 28435, + "vodaf": 28436, + "lat": 28437, + "sled": 28438, + "cricketer": 28439, + "browne": 28440, + "golfers": 28441, + "barra": 28442, + "watchers": 28443, + "luigi": 28444, + "swamy": 28445, + "moms": 28446, + "pitched": 28447, + "santor": 28448, + "crs": 28449, + "sire": 28450, + "scamp": 28451, + "bode": 28452, + "stewar": 28453, + "jonny": 28454, + "entity": 28455, + "pacqui": 28456, + "mindful": 28457, + "minindia": 28458, + "bearded": 28459, + "tempt": 28460, + "scorpion": 28461, + "eaton": 28462, + "authorized": 28463, + "arto": 28464, + "svp": 28465, + "opathy": 28466, + "cchini": 28467, + "housemusic": 28468, + "disneyworld": 28469, + "âĢĶ@": 28470, + "propose": 28471, + "diy": 28472, + "expense": 28473, + "teng": 28474, + "puppets": 28475, + "smel": 28476, + "daca": 28477, + "perry": 28478, + "finn": 28479, + "boosting": 28480, + "leftovers": 28481, + "cougs": 28482, + "satellites": 28483, + "many": 28484, + "aze": 28485, + "gong": 28486, + "fie": 28487, + "methodo": 28488, + "ferries": 28489, + "ðŁ¤ĶðŁ¤Ķ": 28490, + "explorers": 28491, + "loader": 28492, + "attracted": 28493, + "ilton": 28494, + "goddamn": 28495, + "piazza": 28496, + "doctr": 28497, + "saving": 28498, + "paragraph": 28499, + "visualization": 28500, + "mayors": 28501, + "workflow": 28502, + "ackles": 28503, + "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ": 28504, + "स": 28505, + "twerk": 28506, + "clut": 28507, + "lover": 28508, + "teases": 28509, + "sian": 28510, + "ote": 28511, + "deterior": 28512, + "accord": 28513, + "lfw": 28514, + "swarovski": 28515, + "natal": 28516, + "traps": 28517, + "kina": 28518, + "analyze": 28519, + "layered": 28520, + "beverages": 28521, + "unit": 28522, + "ransom": 28523, + "peshaw": 28524, + "destined": 28525, + "astrology": 28526, + "sipping": 28527, + "mileycyrus": 28528, + "camino": 28529, + "marshmallow": 28530, + "bliss": 28531, + "outback": 28532, + "faq": 28533, + "intoler": 28534, + "humility": 28535, + "poppin": 28536, + "halloween": 28537, + "montene": 28538, + "ophy": 28539, + "nun": 28540, + "tattooed": 28541, + "aas": 28542, + "ðŁĮ³": 28543, + "daley": 28544, + "quality": 28545, + "dusa": 28546, + "fishermen": 28547, + "swif": 28548, + "terrac": 28549, + "stau": 28550, + "lein": 28551, + "trolling": 28552, + "shipment": 28553, + "gardener": 28554, + "marchmadness": 28555, + "headband": 28556, + "grt": 28557, + "burnett": 28558, + "wand": 28559, + "!!!!!!!!!": 28560, + "ghe": 28561, + "dux": 28562, + "hud": 28563, + "warner": 28564, + "ðŁĩ¦": 28565, + "exile": 28566, + "rescue": 28567, + "rata": 28568, + "dhan": 28569, + "ducati": 28570, + "drown": 28571, + "blends": 28572, + "spie": 28573, + "alligator": 28574, + "simultaneously": 28575, + "brooke": 28576, + "uke": 28577, + "khar": 28578, + "communion": 28579, + "rika": 28580, + "fordfc": 28581, + "chinatown": 28582, + "yourown": 28583, + "mey": 28584, + "canal": 28585, + "systematic": 28586, + "depri": 28587, + "oxford": 28588, + "anil": 28589, + "wut": 28590, + "equation": 28591, + "bez": 28592, + "fleur": 28593, + "thegood": 28594, + "langley": 28595, + "adity": 28596, + "edith": 28597, + "alfie": 28598, + "оÑĤ": 28599, + "encry": 28600, + "brill": 28601, + "exemp": 28602, + "cesar": 28603, + "mbling": 28604, + "abri": 28605, + "scicom": 28606, + "jing": 28607, + "schooling": 28608, + "mika": 28609, + "mechanisms": 28610, + "impromptu": 28611, + "rhea": 28612, + "moore": 28613, + "crimea": 28614, + "besto": 28615, + "wright": 28616, + "elders": 28617, + "rods": 28618, + "kamal": 28619, + "folklore": 28620, + "beet": 28621, + "minion": 28622, + "relieve": 28623, + "thro": 28624, + "teamusa": 28625, + "pascal": 28626, + "madewith": 28627, + "bolivia": 28628, + "itti": 28629, + "freebies": 28630, + "desired": 28631, + "bestselling": 28632, + "liness": 28633, + "laden": 28634, + "keane": 28635, + "mists": 28636, + "hippie": 28637, + "attachment": 28638, + "@/": 28639, + "sew": 28640, + "flanagan": 28641, + "âĿĹï¸ı": 28642, + "supremac": 28643, + "stlcards": 28644, + "sias": 28645, + "qu": 28646, + "rhys": 28647, + "steep": 28648, + "valleys": 28649, + "vw": 28650, + "paving": 28651, + "dispat": 28652, + "alison": 28653, + "porte": 28654, + "idu": 28655, + "newsc": 28656, + "socket": 28657, + "mos": 28658, + "costar": 28659, + "revo": 28660, + "proteins": 28661, + "stanleycup": 28662, + "mcal": 28663, + "earring": 28664, + "secs": 28665, + "mclean": 28666, + "capric": 28667, + "nickelo": 28668, + "aden": 28669, + "vc": 28670, + "shouse": 28671, + "adaptive": 28672, + "maximize": 28673, + "entertainer": 28674, + "prose": 28675, + "griffi": 28676, + "sixteen": 28677, + "lamar": 28678, + "mirage": 28679, + "saudiarabia": 28680, + "aweather": 28681, + "rust": 28682, + "infiltr": 28683, + "fashionweek": 28684, + "ðŁĺĬðŁĺĬðŁĺĬ": 28685, + "selective": 28686, + "bubble": 28687, + "aden": 28688, + "fennel": 28689, + "decisive": 28690, + "mta": 28691, + "mocking": 28692, + "mbles": 28693, + "stamp": 28694, + "mule": 28695, + "bernardo": 28696, + "grin": 28697, + "pott": 28698, + "jingle": 28699, + "vettel": 28700, + "colombian": 28701, + "camo": 28702, + "motivationmonday": 28703, + "bahan": 28704, + "ply": 28705, + "dhary": 28706, + "kami": 28707, + "xmen": 28708, + "sleeper": 28709, + "gara": 28710, + "mysti": 28711, + "confidential": 28712, + "conflicts": 28713, + "pneu": 28714, + "ces": 28715, + "insurtech": 28716, + "cleanse": 28717, + "merely": 28718, + "vais": 28719, + "tux": 28720, + "thegreat": 28721, + "sharon": 28722, + "maj": 28723, + "hola": 28724, + "ecosystems": 28725, + "ajay": 28726, + "aaj": 28727, + "hush": 28728, + "harmon": 28729, + "backtoschool": 28730, + "wikileaks": 28731, + "reflected": 28732, + "ðŁĺĵ": 28733, + "commemorating": 28734, + "acet": 28735, + "buckingham": 28736, + "messiah": 28737, + "tuous": 28738, + "hornet": 28739, + "tobe": 28740, + "dq": 28741, + "heine": 28742, + "mig": 28743, + "plate": 28744, + "nicholson": 28745, + "spie": 28746, + "cumberland": 28747, + "normal": 28748, + "phobia": 28749, + "happyhalloween": 28750, + "cityfc": 28751, + "mcel": 28752, + "gillian": 28753, + "keto": 28754, + "lude": 28755, + "demise": 28756, + "suga": 28757, + "strate": 28758, + "mcgrath": 28759, + "visitscotland": 28760, + "fooled": 28761, + "cbr": 28762, + "gcse": 28763, + "colori": 28764, + "potd": 28765, + "missuniverse": 28766, + "finances": 28767, + "mapoli": 28768, + "forks": 28769, + "Ø´": 28770, + "cannon": 28771, + "medicinal": 28772, + "ðŁĹĵ": 28773, + "kho": 28774, + "wreck": 28775, + "panto": 28776, + "bagel": 28777, + "gull": 28778, + "syndicate": 28779, + "icy": 28780, + "prc": 28781, + "kien": 28782, + "zika": 28783, + "tish": 28784, + "peta": 28785, + "cco": 28786, + "liza": 28787, + "chut": 28788, + "extraction": 28789, + "elg": 28790, + "gli": 28791, + "fueled": 28792, + "posit": 28793, + "respectively": 28794, + "leicester": 28795, + "brink": 28796, + "vulnerability": 28797, + "imported": 28798, + "esha": 28799, + "ðŁ¦ħ": 28800, + "rural": 28801, + "rell": 28802, + "gaming": 28803, + "atlantic": 28804, + "abandon": 28805, + "noah": 28806, + "resolved": 28807, + "prostate": 28808, + "allergic": 28809, + "psd": 28810, + "âĺ¹": 28811, + "dungeon": 28812, + "fangirl": 28813, + "illuminated": 28814, + "mhs": 28815, + "whitesox": 28816, + "dently": 28817, + "cko": 28818, + "endorse": 28819, + "overly": 28820, + "dazzling": 28821, + "prioriti": 28822, + "nightlife": 28823, + "util": 28824, + "behave": 28825, + "flamen": 28826, + "eastbound": 28827, + "ðŁĴŁ": 28828, + "iloveyou": 28829, + "govuk": 28830, + "mozambique": 28831, + "allegi": 28832, + "dri": 28833, + "testimonial": 28834, + "aths": 28835, + "ì§Ģ": 28836, + "mmy": 28837, + "shabby": 28838, + "prosecco": 28839, + "friendships": 28840, + "calam": 28841, + "damages": 28842, + "offset": 28843, + "jurassic": 28844, + "juno": 28845, + "arrell": 28846, + "ðŁĴ©": 28847, + "interventions": 28848, + "daredevil": 28849, + "carver": 28850, + "runaway": 28851, + "rane": 28852, + "trustees": 28853, + "haute": 28854, + "depths": 28855, + "ðŁİŃ": 28856, + "mein": 28857, + "sacrifices": 28858, + "concier": 28859, + "nesting": 28860, + "izzy": 28861, + "metam": 28862, + "ilovemy": 28863, + "urine": 28864, + "dulu": 28865, + "malhotra": 28866, + "veins": 28867, + "nightly": 28868, + "coat": 28869, + "andi": 28870, + "hewitt": 28871, + "lonel": 28872, + "cible": 28873, + "write": 28874, + "jennie": 28875, + "santac": 28876, + "ĸï¸ı": 28877, + "strato": 28878, + "singapore": 28879, + "soprano": 28880, + "kristen": 28881, + "cheerful": 28882, + "fleetwood": 28883, + "fairi": 28884, + "meli": 28885, + "wast": 28886, + "turnt": 28887, + "sforsale": 28888, + "scrolling": 28889, + "angelina": 28890, + "rendition": 28891, + "jericho": 28892, + "nicky": 28893, + "orb": 28894, + "flavo": 28895, + "patriot": 28896, + "asheville": 28897, + "sickness": 28898, + "refund": 28899, + "aggression": 28900, + "bpl": 28901, + "ãĥĥ": 28902, + "elusive": 28903, + "thistory": 28904, + "hanger": 28905, + "buffs": 28906, + "villas": 28907, + "atkinson": 28908, + "sph": 28909, + "jait": 28910, + "declined": 28911, + "wok": 28912, + "supremacy": 28913, + "ootball": 28914, + "eyang": 28915, + "ðŁİĵ": 28916, + "sford": 28917, + "athi": 28918, + "consume": 28919, + "roadster": 28920, + "eso": 28921, + "upro": 28922, + "recipe": 28923, + "auf": 28924, + "uci": 28925, + "aron": 28926, + "oooh": 28927, + "csgo": 28928, + "reich": 28929, + "mcd": 28930, + "minute": 28931, + "ladies": 28932, + "punk": 28933, + "rutgers": 28934, + "meek": 28935, + "arizon": 28936, + "taj": 28937, + "landlord": 28938, + "degra": 28939, + "autumn": 28940, + "lynx": 28941, + "usf": 28942, + "bhi": 28943, + "fairytale": 28944, + "donghae": 28945, + "betsy": 28946, + "exploded": 28947, + "chennai": 28948, + "opa": 28949, + "protag": 28950, + "brant": 28951, + "ðŁĵ°:": 28952, + "gf": 28953, + "palli": 28954, + "ðŁı¼âĢįâĻĢï¸ı": 28955, + "sut": 28956, + "illini": 28957, + "columnist": 28958, + "shirtless": 28959, + "decentr": 28960, + "searched": 28961, + "ecor": 28962, + "buggy": 28963, + "sack": 28964, + "ðŁĺĤðŁĺŃ": 28965, + "det": 28966, + "theri": 28967, + "ornaments": 28968, + "bringback": 28969, + "tov": 28970, + "quarterfinals": 28971, + "iche": 28972, + "constra": 28973, + "gier": 28974, + "buchanan": 28975, + "vix": 28976, + "kayaking": 28977, + "mustread": 28978, + "swallow": 28979, + "melb": 28980, + "scaf": 28981, + "opal": 28982, + "mayoral": 28983, + "harat": 28984, + "ðŁ¦ĭ": 28985, + "schedules": 28986, + "idf": 28987, + "hague": 28988, + "roz": 28989, + "aah": 28990, + "dmc": 28991, + "duplic": 28992, + "cache": 28993, + "orphan": 28994, + "fracture": 28995, + "recon": 28996, + "chav": 28997, + "bunnies": 28998, + "alain": 28999, + "mustafa": 29000, + "ðŁİĻ": 29001, + "vacations": 29002, + "dynamite": 29003, + "texted": 29004, + "broadcaster": 29005, + "ðŁĴ£": 29006, + "steamed": 29007, + "rocker": 29008, + "dietary": 29009, + "luxurytravel": 29010, + "inaugurated": 29011, + "sawards": 29012, + "vaughn": 29013, + "lincolnshire": 29014, + "clicked": 29015, + "kraja": 29016, + "fanc": 29017, + "removes": 29018, + "layoffs": 29019, + "mcfar": 29020, + "breeds": 29021, + "winnie": 29022, + "jonghyun": 29023, + "incentive": 29024, + "variations": 29025, + "patton": 29026, + "aturday": 29027, + "persistent": 29028, + "prun": 29029, + "piers": 29030, + "dales": 29031, + "æĸ": 29032, + "breastfeeding": 29033, + "rance": 29034, + "tawa": 29035, + "Ĥâĸ": 29036, + "murdoch": 29037, + "captive": 29038, + "thistle": 29039, + "nica": 29040, + "commodity": 29041, + "couldnt": 29042, + "boardwalk": 29043, + "gracious": 29044, + "practitioners": 29045, + "ngc": 29046, + "scrum": 29047, + "nero": 29048, + "camouflage": 29049, + "colon": 29050, + "hei": 29051, + "physicist": 29052, + "saturdaymorning": 29053, + "tener": 29054, + "siwon": 29055, + "columns": 29056, + "brune": 29057, + "yvr": 29058, + "bair": 29059, + "retires": 29060, + "halam": 29061, + "caber": 29062, + "shazam": 29063, + "minu": 29064, + "cascade": 29065, + "milkshake": 29066, + "grid": 29067, + "dren": 29068, + "vincent": 29069, + "sodium": 29070, + "platter": 29071, + "cheerleader": 29072, + "chenko": 29073, + "yak": 29074, + "eliminated": 29075, + "typo": 29076, + "yman": 29077, + "rethink": 29078, + "âĿĹ": 29079, + "tsville": 29080, + "bernardokath": 29081, + "extr": 29082, + "ðŁĺģðŁĺģðŁĺģ": 29083, + "tao": 29084, + "reper": 29085, + "moths": 29086, + "empowered": 29087, + "citing": 29088, + "transported": 29089, + "monks": 29090, + "sanat": 29091, + "clears": 29092, + "bachelorette": 29093, + "campbell": 29094, + "rachael": 29095, + "harle": 29096, + "handler": 29097, + "climbs": 29098, + "interference": 29099, + "release": 29100, + "shand": 29101, + "rbs": 29102, + "hrh": 29103, + "ãģª": 29104, + "valle": 29105, + "ré": 29106, + "slime": 29107, + "wakes": 29108, + "chubby": 29109, + "sloan": 29110, + "elves": 29111, + "athen": 29112, + "attorneys": 29113, + "microscope": 29114, + "stoner": 29115, + "scaling": 29116, + "obe": 29117, + "cout": 29118, + "seman": 29119, + "midweek": 29120, + "balsam": 29121, + "ðŁĺįâĿ¤": 29122, + "tiful": 29123, + "vish": 29124, + "lotta": 29125, + "ripping": 29126, + "remn": 29127, + "tire": 29128, + "leap": 29129, + "havent": 29130, + "laby": 29131, + "himach": 29132, + "whispers": 29133, + "wein": 29134, + "ðŁİ¸": 29135, + "wildflowers": 29136, + "sele": 29137, + "ucc": 29138, + "liability": 29139, + "azine": 29140, + "swings": 29141, + "kya": 29142, + "tair": 29143, + "remain": 29144, + "edo": 29145, + "flops": 29146, + "pocket": 29147, + "grandad": 29148, + "examiner": 29149, + "gris": 29150, + "ffect": 29151, + "ðŁijĬðŁı»": 29152, + "studded": 29153, + "heartbeat": 29154, + "deacon": 29155, + "firmly": 29156, + "infectious": 29157, + "stef": 29158, + "outlines": 29159, + "leasing": 29160, + "claws": 29161, + "sense": 29162, + "tabs": 29163, + "hoot": 29164, + "mosul": 29165, + "spawn": 29166, + "coa": 29167, + "hogwarts": 29168, + "vein": 29169, + "albania": 29170, + "manuel": 29171, + "bino": 29172, + "vauxhall": 29173, + "scotland": 29174, + "gobucks": 29175, + "matty": 29176, + "physio": 29177, + "torino": 29178, + "constable": 29179, + "investigated": 29180, + "slower": 29181, + "mistaken": 29182, + "bayer": 29183, + "wildfires": 29184, + "voic": 29185, + "xon": 29186, + "timeto": 29187, + "chassis": 29188, + "barric": 29189, + "pion": 29190, + "baldhead": 29191, + "wook": 29192, + "registr": 29193, + "drafts": 29194, + "bhs": 29195, + "ligue": 29196, + "lick": 29197, + "staffordshire": 29198, + "bafta": 29199, + "darry": 29200, + "jeanne": 29201, + "vending": 29202, + "corp": 29203, + "âĽ³ï¸ı": 29204, + "kiddos": 29205, + "fenway": 29206, + "cao": 29207, + "westbound": 29208, + "ðŁĺĻ": 29209, + "dvr": 29210, + "quicker": 29211, + "blah": 29212, + "goodie": 29213, + "ðŁĴĭðŁĴĭ": 29214, + "vox": 29215, + "esper": 29216, + "facade": 29217, + "correlation": 29218, + "redbull": 29219, + "roup": 29220, + "declining": 29221, + "chive": 29222, + "mcgee": 29223, + "turo": 29224, + "inder": 29225, + "feller": 29226, + "fug": 29227, + "ilysm": 29228, + "mardi": 29229, + "peshawar": 29230, + "kieran": 29231, + "inema": 29232, + "meatballs": 29233, + "peck": 29234, + "depressing": 29235, + "sensing": 29236, + "giz": 29237, + "ddington": 29238, + "springwatch": 29239, + "roaming": 29240, + "yellowstone": 29241, + "horseshoe": 29242, + "amman": 29243, + "weekday": 29244, + "olor": 29245, + "ðŁ¥°": 29246, + "boosts": 29247, + "sprint": 29248, + "scarves": 29249, + "jee": 29250, + "beetro": 29251, + "clan": 29252, + "allthe": 29253, + "ìĦ¸ë": 29254, + "enlightenment": 29255, + "adobe": 29256, + "regeneration": 29257, + "?@": 29258, + "contag": 29259, + "yachts": 29260, + "tou": 29261, + "mora": 29262, + "envoy": 29263, + "rani": 29264, + "goli": 29265, + "dhanushkraja": 29266, + "woodworking": 29267, + "strengths": 29268, + "sedi": 29269, + "discs": 29270, + "arina": 29271, + "scon": 29272, + "lite": 29273, + "another": 29274, + "ðŁ¥Ĭ": 29275, + "yemen": 29276, + "guern": 29277, + "savvy": 29278, + "loyed": 29279, + "biomed": 29280, + "heartbreak": 29281, + "comrades": 29282, + "millie": 29283, + "patch": 29284, + "unf": 29285, + "jarvis": 29286, + "blaming": 29287, + "commemoration": 29288, + "gey": 29289, + "å¥": 29290, + "cardiovascular": 29291, + "aligned": 29292, + "document": 29293, + ".?": 29294, + "aesthetics": 29295, + "emu": 29296, + "theirs": 29297, + "leh": 29298, + "psic": 29299, + "sif": 29300, + "plateau": 29301, + "expend": 29302, + "dominating": 29303, + "robes": 29304, + "mauritius": 29305, + "exceptionally": 29306, + "homer": 29307, + "discoveries": 29308, + "braun": 29309, + "tennant": 29310, + "insulin": 29311, + "ðŁİ®": 29312, + "carbs": 29313, + "teas": 29314, + "?!\"": 29315, + "zie": 29316, + "francois": 29317, + "browsing": 29318, + "thol": 29319, + "clarence": 29320, + "helper": 29321, + "obtained": 29322, + "cassie": 29323, + "lees": 29324, + "!,": 29325, + "pomegran": 29326, + "hubs": 29327, + "prestige": 29328, + "][": 29329, + "macher": 29330, + "bottled": 29331, + "punch": 29332, + "pipe": 29333, + "och": 29334, + "gallons": 29335, + "deliveries": 29336, + "ura": 29337, + "unday": 29338, + "monde": 29339, + "depicts": 29340, + "regency": 29341, + "outrageous": 29342, + "khaled": 29343, + "caro": 29344, + "hearti": 29345, + "zag": 29346, + "developmental": 29347, + "overcoming": 29348, + "statistical": 29349, + "flavored": 29350, + "fords": 29351, + "creatives": 29352, + "laurence": 29353, + "dias": 29354, + "sunscreen": 29355, + "inked": 29356, + "preacher": 29357, + "nul": 29358, + "impacting": 29359, + "autistic": 29360, + "âļĶï¸ı": 29361, + "oss": 29362, + "pelicans": 29363, + "celeste": 29364, + "vb": 29365, + "rump": 29366, + "mcgra": 29367, + "fairfax": 29368, + "humor": 29369, + "bbcnews": 29370, + "rowling": 29371, + "calder": 29372, + "seamless": 29373, + "agne": 29374, + "pti": 29375, + "mixed": 29376, + "tshirts": 29377, + "merci": 29378, + "btob": 29379, + "womeninstem": 29380, + "genealogy": 29381, + "preven": 29382, + "lour": 29383, + "cradle": 29384, + "giuse": 29385, + "о": 29386, + "chrono": 29387, + "fairness": 29388, + "chocolate": 29389, + "tory": 29390, + "asda": 29391, + "prescott": 29392, + "stretched": 29393, + "alman": 29394, + "uil": 29395, + "recharge": 29396, + "intre": 29397, + "obst": 29398, + "hospital": 29399, + "hayward": 29400, + "tenerife": 29401, + "friedman": 29402, + "vaping": 29403, + "confessions": 29404, + "yeah": 29405, + "balli": 29406, + "lucknow": 29407, + "corpse": 29408, + "sculptor": 29409, + "ampton": 29410, + "tpp": 29411, + "indicates": 29412, + "surplus": 29413, + "truman": 29414, + "ðĿĻ": 29415, + "sinha": 29416, + "invo": 29417, + "sovereign": 29418, + "kev": 29419, + "establishing": 29420, + "engraved": 29421, + "assuming": 29422, + "ðŁıģ": 29423, + "souza": 29424, + "fabi": 29425, + "toned": 29426, + "ounge": 29427, + "deloit": 29428, + "downey": 29429, + "noble": 29430, + "omor": 29431, + "cartridge": 29432, + "ðŁıIJ": 29433, + "uhur": 29434, + "holloway": 29435, + "successes": 29436, + "rsa": 29437, + "âĦ¢": 29438, + "mazz": 29439, + "twd": 29440, + "discourse": 29441, + ".<": 29442, + "yat": 29443, + "satisfy": 29444, + "compri": 29445, + "ह": 29446, + "graphite": 29447, + "dissertation": 29448, + "arter": 29449, + "íĶ": 29450, + "bally": 29451, + "zombi": 29452, + "lyons": 29453, + "aic": 29454, + "ubc": 29455, + "prada": 29456, + "eil": 29457, + "dax": 29458, + "clai": 29459, + "granddaughter": 29460, + "extravaganza": 29461, + "challenge": 29462, + "ðŁ¤ŀ": 29463, + "pover": 29464, + "primarily": 29465, + "daddy": 29466, + "mana": 29467, + "bikers": 29468, + "inquiries": 29469, + "daun": 29470, + "feline": 29471, + "generative": 29472, + "hef": 29473, + "benefiting": 29474, + "lindsey": 29475, + "polka": 29476, + "demonstrated": 29477, + "alle": 29478, + "randy": 29479, + "osu": 29480, + "lowkey": 29481, + "weirdest": 29482, + "redbull": 29483, + "oury": 29484, + "nous": 29485, + "woodstock": 29486, + "credenti": 29487, + "nicer": 29488, + "gado": 29489, + "alyss": 29490, + "aph": 29491, + "preparedness": 29492, + "stationary": 29493, + "incorporated": 29494, + "dyer": 29495, + "saratoga": 29496, + "celesti": 29497, + ":\"": 29498, + "antibiotics": 29499, + "orgs": 29500, + "indefin": 29501, + "apron": 29502, + "иÐ": 29503, + "fifteen": 29504, + "nof": 29505, + "ðŁĶĿ": 29506, + "phx": 29507, + "tega": 29508, + "mz": 29509, + "organizational": 29510, + "onair": 29511, + "bandung": 29512, + "pleasures": 29513, + "mori": 29514, + "secretari": 29515, + "raccoon": 29516, + "cashi": 29517, + "pilates": 29518, + "kon": 29519, + "geoffrey": 29520, + "lao": 29521, + "kamp": 29522, + "departments": 29523, + "backpacking": 29524, + "anam": 29525, + "ë": 29526, + "crackdown": 29527, + "aunty": 29528, + "ondo": 29529, + "lizzie": 29530, + "phers": 29531, + "cun": 29532, + "ðŁĩ±": 29533, + "kpop": 29534, + "put": 29535, + "intentional": 29536, + "connolly": 29537, + "barclays": 29538, + "hsfb": 29539, + "swindon": 29540, + "uku": 29541, + "sally": 29542, + "aint": 29543, + "âľħ": 29544, + "penang": 29545, + "uplifting": 29546, + "epilepsy": 29547, + "interro": 29548, + "bungal": 29549, + "goku": 29550, + "blueberries": 29551, + "द": 29552, + "ussia": 29553, + "silky": 29554, + "moured": 29555, + "istic": 29556, + "briefs": 29557, + "meats": 29558, + "gob": 29559, + "chaser": 29560, + "statewide": 29561, + "prasad": 29562, + "glitch": 29563, + "arin": 29564, + "banff": 29565, + "member": 29566, + "ðŁĺŃâĿ¤ï¸ı": 29567, + "loving": 29568, + "halla": 29569, + "ม": 29570, + "smokers": 29571, + "yaku": 29572, + "scicomm": 29573, + "physio": 29574, + "swol": 29575, + "lemons": 29576, + "gelato": 29577, + "chool": 29578, + "capitals": 29579, + "kistan": 29580, + "tights": 29581, + "spikes": 29582, + "travellers": 29583, + "iklan": 29584, + "commissioning": 29585, + "arine": 29586, + "emabiggestfans": 29587, + "emphasis": 29588, + "frontline": 29589, + "paddock": 29590, + "destructive": 29591, + "baha": 29592, + "linger": 29593, + "jewish": 29594, + "shetland": 29595, + "mcgin": 29596, + "monkey": 29597, + "koz": 29598, + "sone": 29599, + "rajini": 29600, + "teh": 29601, + "yen": 29602, + "cvs": 29603, + "masquer": 29604, + "girly": 29605, + "wesle": 29606, + "wasnt": 29607, + "brody": 29608, + "terminator": 29609, + "gille": 29610, + "maggi": 29611, + "birdie": 29612, + "jeopardy": 29613, + "cubic": 29614, + "vmware": 29615, + "intricate": 29616, + "anup": 29617, + "topia": 29618, + "easton": 29619, + "sabres": 29620, + "investigates": 29621, + "busting": 29622, + "bilingual": 29623, + "valentino": 29624, + "informat": 29625, + "ferre": 29626, + "adventur": 29627, + "hydrate": 29628, + "forsy": 29629, + "aziz": 29630, + "santo": 29631, + "ede": 29632, + "whistler": 29633, + "continuously": 29634, + "dham": 29635, + "unused": 29636, + "jihad": 29637, + "addictive": 29638, + "vidy": 29639, + "dob": 29640, + "ido": 29641, + "fied": 29642, + "niversary": 29643, + "none": 29644, + "fuer": 29645, + "ðŁĺįðŁĺĺ": 29646, + "covenant": 29647, + "printable": 29648, + "immaculate": 29649, + "oem": 29650, + "clt": 29651, + "servants": 29652, + "consumed": 29653, + "unreleased": 29654, + "scum": 29655, + "packaged": 29656, + "mere": 29657, + "ìĦ¸ë¸": 29658, + "toby": 29659, + "taf": 29660, + "spoons": 29661, + "meal": 29662, + "fball": 29663, + "fairfield": 29664, + "janet": 29665, + "silverstone": 29666, + "dartmouth": 29667, + "followme": 29668, + "voyager": 29669, + "kombat": 29670, + "anniver": 29671, + "enew": 29672, + "magdal": 29673, + "hove": 29674, + "sath": 29675, + "grizzly": 29676, + "cardi": 29677, + "gartner": 29678, + "sandy": 29679, + "kanye": 29680, + "posture": 29681, + "poign": 29682, + "impulse": 29683, + "radiology": 29684, + "horizons": 29685, + "siam": 29686, + "aishwar": 29687, + "==>": 29688, + "noche": 29689, + "tris": 29690, + "elyn": 29691, + "comme": 29692, + "dui": 29693, + "cec": 29694, + "councillors": 29695, + "cuddling": 29696, + "creeping": 29697, + "locke": 29698, + "manages": 29699, + "transferred": 29700, + "necks": 29701, + "dier": 29702, + "dano": 29703, + "vick": 29704, + "lunches": 29705, + "dhe": 29706, + "ensures": 29707, + "criss": 29708, + "ulster": 29709, + "bannon": 29710, + "contenders": 29711, + "spam": 29712, + "sweetness": 29713, + "medal": 29714, + "honduras": 29715, + "arctic": 29716, + "ultrasound": 29717, + "infr": 29718, + "discovers": 29719, + "eiffel": 29720, + "casters": 29721, + "ruben": 29722, + "dust": 29723, + "aweed": 29724, + "atrium": 29725, + "lestwe": 29726, + "seared": 29727, + "ðŁĵº:": 29728, + "tyne": 29729, + "exchanges": 29730, + "littlemix": 29731, + "lle": 29732, + "astronauts": 29733, + "hershey": 29734, + "workday": 29735, + "knob": 29736, + "sov": 29737, + "resigns": 29738, + "todayshow": 29739, + "derman": 29740, + "anth": 29741, + "afc": 29742, + "taster": 29743, + "swoo": 29744, + "saeed": 29745, + "pering": 29746, + "narrowly": 29747, + "rnli": 29748, + "bestbuy": 29749, + "panasonic": 29750, + "obstacle": 29751, + "farmers": 29752, + "ðŁİĻ": 29753, + "pawan": 29754, + "kiest": 29755, + "angers": 29756, + "absurd": 29757, + "ohmy": 29758, + "sino": 29759, + "pistachi": 29760, + "spice": 29761, + "giuli": 29762, + "primetime": 29763, + "kow": 29764, + "kens": 29765, + "exagger": 29766, + "!?!": 29767, + "uba": 29768, + "middles": 29769, + "judd": 29770, + "ejec": 29771, + "slammed": 29772, + "pensions": 29773, + "ofa": 29774, + "recreate": 29775, + "bhp": 29776, + "xxl": 29777, + "liverpool": 29778, + "thresh": 29779, + "purity": 29780, + "nieu": 29781, + "holics": 29782, + "wrath": 29783, + "rado": 29784, + "glio": 29785, + "amma": 29786, + "dilemma": 29787, + "cru": 29788, + "letsgo": 29789, + "....@": 29790, + "âĿĵ": 29791, + "suggesting": 29792, + "trumps": 29793, + "horus": 29794, + "fv": 29795, + "icom": 29796, + "referring": 29797, + "predictive": 29798, + "tarts": 29799, + "gette": 29800, + "sock": 29801, + "glossy": 29802, + "pinky": 29803, + "alec": 29804, + "thyme": 29805, + "oura": 29806, + "theroad": 29807, + "petr": 29808, + "cram": 29809, + "pfi": 29810, + "dvn": 29811, + "meier": 29812, + "incentives": 29813, + "tunnels": 29814, + "mobil": 29815, + "recap": 29816, + "extras": 29817, + "upright": 29818, + "revamp": 29819, + "perseverance": 29820, + ",-": 29821, + "otp": 29822, + "mirror": 29823, + "arwx": 29824, + "gerry": 29825, + "maher": 29826, + "gor": 29827, + "homepage": 29828, + "amis": 29829, + "agra": 29830, + "madele": 29831, + "bestfriend": 29832, + "siriusxm": 29833, + "bundles": 29834, + "admiring": 29835, + "tdsb": 29836, + "ðŁįģ": 29837, + "chas": 29838, + "slowing": 29839, + "roh": 29840, + "wallpapers": 29841, + "âĢ¦/": 29842, + "tekken": 29843, + "gangs": 29844, + "tala": 29845, + "lindsay": 29846, + "shoul": 29847, + "linebacker": 29848, + "toolkit": 29849, + "uranium": 29850, + "calyp": 29851, + "abrams": 29852, + "matthi": 29853, + "ðŁı¿": 29854, + "honourable": 29855, + "dayo": 29856, + "versail": 29857, + "tank": 29858, + "stc": 29859, + "fritz": 29860, + "splend": 29861, + "patag": 29862, + "annoyed": 29863, + "onday": 29864, + "devastated": 29865, + "chattanooga": 29866, + "nationalism": 29867, + "massey": 29868, + "jenn": 29869, + "tailor": 29870, + "devgn": 29871, + "organs": 29872, + "zucchini": 29873, + "onfox": 29874, + "satire": 29875, + "wexford": 29876, + "disgrace": 29877, + "noto": 29878, + "volta": 29879, + "âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı": 29880, + "à¶": 29881, + "homeowners": 29882, + "pointer": 29883, + "mcr": 29884, + "austen": 29885, + "daysto": 29886, + "moons": 29887, + "palma": 29888, + "grazing": 29889, + "eso": 29890, + "influencers": 29891, + "shahidkapoor": 29892, + "compliant": 29893, + "measurements": 29894, + "develops": 29895, + "yd": 29896, + "parl": 29897, + "pvt": 29898, + "randolph": 29899, + "tortured": 29900, + "gerald": 29901, + "elias": 29902, + "deepikap": 29903, + "warmup": 29904, + "hickory": 29905, + "gap": 29906, + "coffin": 29907, + "amour": 29908, + "reneg": 29909, + "mounting": 29910, + "sevens": 29911, + "igle": 29912, + "hier": 29913, + "decad": 29914, + "tright": 29915, + "escapes": 29916, + "werner": 29917, + "tfl": 29918, + "fulfilled": 29919, + "niger": 29920, + "sourdough": 29921, + "reaper": 29922, + "chooses": 29923, + "spinner": 29924, + "weeknd": 29925, + "filtered": 29926, + "shuk": 29927, + "kati": 29928, + "oldham": 29929, + "opensource": 29930, + "khanna": 29931, + "atelier": 29932, + "connec": 29933, + "ophobic": 29934, + "glas": 29935, + "complications": 29936, + "arson": 29937, + "councils": 29938, + "smol": 29939, + "assy": 29940, + "lurking": 29941, + "lingui": 29942, + "hanks": 29943, + "ein": 29944, + "Ùħ": 29945, + "rugs": 29946, + "nguyen": 29947, + "nouveau": 29948, + "menace": 29949, + "lev": 29950, + "aladdin": 29951, + "ruining": 29952, + "roundabout": 29953, + "km": 29954, + "conor": 29955, + "shoops": 29956, + "mayday": 29957, + "traumatic": 29958, + "prabhas": 29959, + "kaiser": 29960, + "kita": 29961, + "router": 29962, + "pedro": 29963, + "retar": 29964, + "stunner": 29965, + "spanish": 29966, + "disturbed": 29967, + "academy": 29968, + "elearning": 29969, + "witty": 29970, + "seng": 29971, + "feral": 29972, + "avy": 29973, + "stab": 29974, + "keaton": 29975, + "urdu": 29976, + "koto": 29977, + "hui": 29978, + "cooke": 29979, + "arian": 29980, + "thepersonal": 29981, + "uma": 29982, + "seap": 29983, + "asting": 29984, + "rhetoric": 29985, + "handwriting": 29986, + "municipality": 29987, + "consortium": 29988, + "ðŁIJŁ": 29989, + "glasgow": 29990, + "raya": 29991, + "eliza": 29992, + "polymer": 29993, + "broth": 29994, + "practi": 29995, + "correspondent": 29996, + "addicts": 29997, + "gayle": 29998, + "ailing": 29999, + "ofe": 30000, + "pli": 30001, + "heartw": 30002, + "stitch": 30003, + "sightings": 30004, + "priests": 30005, + "samo": 30006, + "sloth": 30007, + "goodwood": 30008, + "rocco": 30009, + "sabc": 30010, + "summit": 30011, + "lace": 30012, + "presley": 30013, + "itten": 30014, + "cincy": 30015, + "thepersonalnetwork": 30016, + "sweek": 30017, + "pegas": 30018, + "afcon": 30019, + "registry": 30020, + "cim": 30021, + "leth": 30022, + "dicap": 30023, + "candice": 30024, + "fluent": 30025, + "smack": 30026, + "pedestri": 30027, + "aloud": 30028, + "carac": 30029, + "priyankach": 30030, + "pgh": 30031, + "irons": 30032, + "dolce": 30033, + "latvia": 30034, + "deceased": 30035, + "therock": 30036, + "clap": 30037, + "cene": 30038, + "foam": 30039, + "morrissey": 30040, + "gret": 30041, + "essentially": 30042, + "comcast": 30043, + "beagle": 30044, + "argues": 30045, + "inged": 30046, + "-âĢ¦": 30047, + "sag": 30048, + "hasan": 30049, + "ðŁĻĨ": 30050, + "ðŁį°": 30051, + "nhra": 30052, + "kannada": 30053, + "indicators": 30054, + "oner": 30055, + "brixton": 30056, + "atas": 30057, + "screenplay": 30058, + "sorority": 30059, + "shaheed": 30060, + "heem": 30061, + "classmates": 30062, + "tainment": 30063, + "esi": 30064, + "breastcancer": 30065, + "zuckerberg": 30066, + "auror": 30067, + "encia": 30068, + "refers": 30069, + "kaeper": 30070, + "vortex": 30071, + "compart": 30072, + "lymph": 30073, + "photographing": 30074, + "steff": 30075, + "restling": 30076, + "parsley": 30077, + "momento": 30078, + "thman": 30079, + "lacking": 30080, + "dutt": 30081, + "oculus": 30082, + "fino": 30083, + "frenzy": 30084, + "rasc": 30085, + "dern": 30086, + "dismissed": 30087, + "nook": 30088, + "metgala": 30089, + "shill": 30090, + "raphael": 30091, + "mavericks": 30092, + "exhibits": 30093, + "eagerly": 30094, + "cpa": 30095, + "amenities": 30096, + ".âłĢ": 30097, + "exodus": 30098, + "ernst": 30099, + "lita": 30100, + "dealt": 30101, + "womensmarch": 30102, + "iain": 30103, + "scoreboard": 30104, + "campeones": 30105, + "cen": 30106, + "tiki": 30107, + "garrison": 30108, + "fidelity": 30109, + "brag": 30110, + "roadmap": 30111, + "psychop": 30112, + "loe": 30113, + "bleu": 30114, + "ðŁijĬðŁı¼": 30115, + "sauvi": 30116, + "springer": 30117, + "temptation": 30118, + "rudolph": 30119, + "acura": 30120, + "wicz": 30121, + "parachute": 30122, + "strol": 30123, + "lenny": 30124, + "zik": 30125, + "doms": 30126, + "nbaf": 30127, + "alpac": 30128, + "vivian": 30129, + "rove": 30130, + "preet": 30131, + "perpetu": 30132, + "snake": 30133, + "airsoft": 30134, + "inflatable": 30135, + "princes": 30136, + "atie": 30137, + "ffey": 30138, + "patient": 30139, + "mire": 30140, + "chelle": 30141, + "slack": 30142, + "groovy": 30143, + "#:": 30144, + "uploading": 30145, + "!!!!!!!!!!!!!!!!": 30146, + "siemens": 30147, + "provision": 30148, + "vfx": 30149, + "needy": 30150, + "fats": 30151, + "topoli": 30152, + "bhutto": 30153, + "sathletics": 30154, + "alums": 30155, + "twinning": 30156, + "southwestern": 30157, + "adopting": 30158, + "lastnight": 30159, + "manne": 30160, + "laga": 30161, + "twell": 30162, + "acia": 30163, + "----": 30164, + "eyewear": 30165, + "hurley": 30166, + "flee": 30167, + "sach": 30168, + "pecker": 30169, + "costly": 30170, + "isk": 30171, + "crates": 30172, + "policy": 30173, + "erosion": 30174, + "ingo": 30175, + "werk": 30176, + "ðŁIJį": 30177, + "tortoise": 30178, + "therapies": 30179, + "internet": 30180, + "chihuahua": 30181, + "rips": 30182, + "frei": 30183, + "edor": 30184, + "taiji": 30185, + "tfc": 30186, + "dod": 30187, + "dempsey": 30188, + "christin": 30189, + "cheng": 30190, + "hips": 30191, + "graeme": 30192, + "compassionate": 30193, + "cavaliers": 30194, + "historic": 30195, + "soulful": 30196, + "criminal": 30197, + "jac": 30198, + "vinci": 30199, + "expired": 30200, + "surat": 30201, + "turismo": 30202, + "kona": 30203, + "seaweed": 30204, + "berts": 30205, + "leica": 30206, + "expressing": 30207, + "aal": 30208, + "wort": 30209, + "breakfast": 30210, + "herring": 30211, + "amused": 30212, + "rhubarb": 30213, + "martian": 30214, + "cosplayer": 30215, + "yash": 30216, + "strial": 30217, + "raul": 30218, + "referral": 30219, + "dwts": 30220, + "jw": 30221, + "adler": 30222, + "curtains": 30223, + "gur": 30224, + "valence": 30225, + "tyrone": 30226, + "swfc": 30227, + "coached": 30228, + "reborn": 30229, + "diabetic": 30230, + "choke": 30231, + "norfolk": 30232, + "investigative": 30233, + "ðŁĴ¯ðŁĴ¯": 30234, + "zid": 30235, + "vmas": 30236, + "phie": 30237, + "objectives": 30238, + "âľĭ": 30239, + "overdue": 30240, + "divers": 30241, + "matsu": 30242, + "ðŁİŁï¸ı": 30243, + "casualties": 30244, + "ว": 30245, + "alk": 30246, + "standardi": 30247, + "realist": 30248, + "artifacts": 30249, + "pandor": 30250, + "kex": 30251, + "invin": 30252, + "(!)": 30253, + "iney": 30254, + "paraly": 30255, + "mrt": 30256, + "faye": 30257, + "thevoice": 30258, + "onga": 30259, + "deed": 30260, + "skinner": 30261, + "azwx": 30262, + "specimen": 30263, + "priyankachopra": 30264, + "nuevo": 30265, + "barkley": 30266, + "toulouse": 30267, + "resumes": 30268, + "footballers": 30269, + "citi": 30270, + "fetch": 30271, + "ère": 30272, + "lestweforget": 30273, + "ðŁĻĭ": 30274, + "chunk": 30275, + "drifting": 30276, + "manipulation": 30277, + "equals": 30278, + "putt": 30279, + "kyungsoo": 30280, + "âĿ¤ï¸ı#": 30281, + "elastic": 30282, + "parano": 30283, + "foy": 30284, + "doping": 30285, + "cincy": 30286, + "ssler": 30287, + "interrupted": 30288, + "alay": 30289, + "adores": 30290, + "amethy": 30291, + "convoy": 30292, + "ãĢı": 30293, + "Ĭãģ": 30294, + "blacklist": 30295, + "generals": 30296, + "sachin": 30297, + "brushed": 30298, + "ounces": 30299, + "nonstop": 30300, + "illiams": 30301, + "btsarmy": 30302, + "uav": 30303, + "ruff": 30304, + "burma": 30305, + "bik": 30306, + "defence": 30307, + "schultz": 30308, + "boasts": 30309, + "loneliness": 30310, + "gore": 30311, + "transforms": 30312, + "alumna": 30313, + "@@": 30314, + "rappers": 30315, + "nehru": 30316, + "caro": 30317, + "himalayan": 30318, + "wearables": 30319, + "geh": 30320, + "peppermint": 30321, + "redevelopment": 30322, + "flamingo": 30323, + "cosby": 30324, + "bigbaldhead": 30325, + "agri": 30326, + "barefoot": 30327, + "scopes": 30328, + "regram": 30329, + "ghana": 30330, + "ðŁİ«": 30331, + "iheart": 30332, + "sadie": 30333, + "carrie": 30334, + "microbial": 30335, + "kuala": 30336, + "skater": 30337, + "querque": 30338, + "âĻ©": 30339, + "genres": 30340, + "reasoning": 30341, + "chased": 30342, + "aso": 30343, + "slipped": 30344, + "encan": 30345, + "vamos": 30346, + "kers": 30347, + "adverse": 30348, + "moil": 30349, + "commodities": 30350, + "withyou": 30351, + "silent": 30352, + "hype": 30353, + "ande": 30354, + "amination": 30355, + "whispe": 30356, + "litz": 30357, + "âļ½ï¸ıâļ½ï¸ı": 30358, + "riff": 30359, + "ppy": 30360, + "lambs": 30361, + "ganesh": 30362, + "absent": 30363, + "regulator": 30364, + "marseille": 30365, + "enroll": 30366, + "parcel": 30367, + "wap": 30368, + "byrd": 30369, + "ðŁĩŃ": 30370, + "tuber": 30371, + "countrymusic": 30372, + "parl": 30373, + "controllers": 30374, + "responsibilities": 30375, + "wey": 30376, + "chate": 30377, + "montenegro": 30378, + "chico": 30379, + "milan": 30380, + "lms": 30381, + "trainees": 30382, + "appropriately": 30383, + "uncertain": 30384, + "poppies": 30385, + "edsheeran": 30386, + "nutritious": 30387, + "garo": 30388, + "deutsch": 30389, + "awesome": 30390, + "ãĥ¼": 30391, + "comfortably": 30392, + "landmarks": 30393, + "eti": 30394, + "reusable": 30395, + "danielle": 30396, + "rosal": 30397, + "coles": 30398, + "justic": 30399, + "ccs": 30400, + "fanny": 30401, + "nim": 30402, + "mcu": 30403, + "clinch": 30404, + "atene": 30405, + "merge": 30406, + "imdb": 30407, + "anglo": 30408, + "uccino": 30409, + "panini": 30410, + "annot": 30411, + "burberry": 30412, + "feature": 30413, + "predicting": 30414, + "fashionista": 30415, + "sask": 30416, + "imaginary": 30417, + "mmo": 30418, + "southsudan": 30419, + "spear": 30420, + "hubble": 30421, + "jointhe": 30422, + "coyotes": 30423, + "sligo": 30424, + "kodak": 30425, + "sitcom": 30426, + "polaroid": 30427, + "rooted": 30428, + "corrup": 30429, + "ðŁĻĮðŁĻĮ": 30430, + "brisban": 30431, + "atz": 30432, + "ahl": 30433, + "remy": 30434, + "talent": 30435, + "avalon": 30436, + "rada": 30437, + "pauline": 30438, + "locomotive": 30439, + "goons": 30440, + "nemo": 30441, + "maserati": 30442, + "icu": 30443, + "stutt": 30444, + "historically": 30445, + "smb": 30446, + "presby": 30447, + "avoid": 30448, + "sooners": 30449, + "rhinestone": 30450, + "wad": 30451, + "rising": 30452, + "trot": 30453, + "modes": 30454, + "regent": 30455, + "optimize": 30456, + "reece": 30457, + "smu": 30458, + "verti": 30459, + "newyorkcity": 30460, + "cortez": 30461, + "rac": 30462, + "incase": 30463, + "sinc": 30464, + "fielding": 30465, + "etta": 30466, + "tiffany": 30467, + "almonds": 30468, + "saddle": 30469, + "krat": 30470, + "matter": 30471, + "glow": 30472, + "starving": 30473, + "glo": 30474, + "crappy": 30475, + "slur": 30476, + "std": 30477, + "monitors": 30478, + "receipt": 30479, + "maymayentrata": 30480, + "mcil": 30481, + "unis": 30482, + "rainbows": 30483, + "caldwell": 30484, + "pacquiao": 30485, + "jop": 30486, + "afe": 30487, + "hook": 30488, + "essen": 30489, + "wizard": 30490, + "median": 30491, + "flaws": 30492, + "coms": 30493, + "âĿĦ": 30494, + "ingh": 30495, + "haynes": 30496, + "antonio": 30497, + "templates": 30498, + "outer": 30499, + "naw": 30500, + "cardigan": 30501, + "belgrade": 30502, + "ðŁĴī": 30503, + "homo": 30504, + "aise": 30505, + "ropes": 30506, + "nove": 30507, + "whatyou": 30508, + "trigge": 30509, + "conception": 30510, + "adukone": 30511, + "nadi": 30512, + "friars": 30513, + "swer": 30514, + "adjusted": 30515, + "hotline": 30516, + "sanity": 30517, + "kaur": 30518, + "downloading": 30519, + "cgi": 30520, + "tenor": 30521, + "ethnic": 30522, + "appalach": 30523, + "ุ": 30524, + "pag": 30525, + "golds": 30526, + "onset": 30527, + "investigator": 30528, + "cartel": 30529, + "peacefully": 30530, + "jarrett": 30531, + "catalan": 30532, + "polio": 30533, + "num": 30534, + "frustration": 30535, + "dharma": 30536, + "mylife": 30537, + "âľĮðŁı»": 30538, + "aberdeen": 30539, + "musa": 30540, + "binder": 30541, + "sparkly": 30542, + "fleeing": 30543, + "instinct": 30544, + "coping": 30545, + "dominance": 30546, + "illers": 30547, + "era": 30548, + "uconn": 30549, + "looms": 30550, + "livingston": 30551, + "gali": 30552, + "hes": 30553, + "cma": 30554, + "bela": 30555, + "seley": 30556, + "monk": 30557, + "lach": 30558, + "marx": 30559, + "´": 30560, + "merica": 30561, + "womanin": 30562, + "essex": 30563, + "raina": 30564, + "jimi": 30565, + "neptune": 30566, + "zack": 30567, + "chinese": 30568, + "martins": 30569, + "chandelier": 30570, + "hern": 30571, + "withus": 30572, + "earl": 30573, + "asphalt": 30574, + "modules": 30575, + "stp": 30576, + "ulla": 30577, + "psychiatric": 30578, + "mileage": 30579, + "captivating": 30580, + "sider": 30581, + "mento": 30582, + "mort": 30583, + "trance": 30584, + "talbot": 30585, + "abby": 30586, + "ìĥ": 30587, + "âľĮðŁı¼": 30588, + "jak": 30589, + "dawn": 30590, + "turnup": 30591, + "screwed": 30592, + "feds": 30593, + "blueprint": 30594, + "ðŁĴĸðŁĴĸ": 30595, + "harsh": 30596, + "eros": 30597, + "insomnia": 30598, + "bankers": 30599, + "taemin": 30600, + "misconduct": 30601, + "humber": 30602, + "gidi": 30603, + "eduardo": 30604, + "cona": 30605, + "muscular": 30606, + "consuming": 30607, + "rash": 30608, + "donnie": 30609, + "dipped": 30610, + "collie": 30611, + "samuel": 30612, + "meltdown": 30613, + "ðŁĺįðŁĺįðŁĺį": 30614, + "mez": 30615, + "examining": 30616, + "schwartz": 30617, + "pristine": 30618, + "ðŁIJĿ": 30619, + "veit": 30620, + "fulfilling": 30621, + "anesthe": 30622, + "guesses": 30623, + "draft": 30624, + "somme": 30625, + "solid": 30626, + "pational": 30627, + "hoped": 30628, + "evolutionary": 30629, + "aller": 30630, + "entertained": 30631, + "slips": 30632, + "ludwig": 30633, + "concludes": 30634, + "sensible": 30635, + "bonnet": 30636, + "craze": 30637, + "tras": 30638, + "hazards": 30639, + "constantine": 30640, + "edics": 30641, + "startrek": 30642, + "toc": 30643, + "occupational": 30644, + "incheon": 30645, + "deepikapadukone": 30646, + "pizzas": 30647, + "newcomer": 30648, + "depart": 30649, + "oppression": 30650, + "ebony": 30651, + "fossils": 30652, + "trojan": 30653, + "elen": 30654, + "steaks": 30655, + "khou": 30656, + "positioning": 30657, + "ugby": 30658, + "redcross": 30659, + "akh": 30660, + "dolce": 30661, + "usmnt": 30662, + "ppen": 30663, + "dilig": 30664, + "mavs": 30665, + "caller": 30666, + "costello": 30667, + "âĽĦ": 30668, + "dyn": 30669, + "things": 30670, + "rhinos": 30671, + "axi": 30672, + "sarkar": 30673, + "convocation": 30674, + "atters": 30675, + "ssss": 30676, + "fungus": 30677, + "eugen": 30678, + "russo": 30679, + "squat": 30680, + "wsb": 30681, + "elion": 30682, + "williamsburg": 30683, + "soff": 30684, + "deficiency": 30685, + "bearer": 30686, + "okin": 30687, + "keystone": 30688, + "twain": 30689, + "calming": 30690, + "breakable": 30691, + "wares": 30692, + "horseracing": 30693, + "combs": 30694, + "bunting": 30695, + "uit": 30696, + "tland": 30697, + "ðŁĴĻðŁĴĻðŁĴĻ": 30698, + "gastron": 30699, + "sabot": 30700, + "ickers": 30701, + "commissioners": 30702, + "senate": 30703, + "iiot": 30704, + "athena": 30705, + "nitrogen": 30706, + "antony": 30707, + "erotic": 30708, + "dialo": 30709, + "missou": 30710, + "hypocr": 30711, + "âľĪ": 30712, + "kaepernick": 30713, + "canv": 30714, + "droo": 30715, + "cleveland": 30716, + "osh": 30717, + "monsta": 30718, + "stefano": 30719, + "^)": 30720, + "shul": 30721, + "poison": 30722, + "hae": 30723, + "commercials": 30724, + "maul": 30725, + "nitro": 30726, + "coworker": 30727, + "aloe": 30728, + "vapor": 30729, + "tents": 30730, + "russian": 30731, + "quid": 30732, + "questionable": 30733, + "midget": 30734, + "poker": 30735, + "girlfriends": 30736, + "sinthe": 30737, + "eritrea": 30738, + "tenure": 30739, + "deposits": 30740, + "buckeyes": 30741, + "spotter": 30742, + "theodore": 30743, + "trinity": 30744, + "joaquin": 30745, + "ucci": 30746, + "followthe": 30747, + "cafc": 30748, + "mpa": 30749, + "ðŁIJ»": 30750, + "plotting": 30751, + "domino": 30752, + "taek": 30753, + "sionally": 30754, + "dicaprio": 30755, + "pap": 30756, + "carmel": 30757, + "iger": 30758, + "btcc": 30759, + "bethle": 30760, + "wwwbigbaldhead": 30761, + "foodie": 30762, + "baghdad": 30763, + "masonry": 30764, + "offended": 30765, + "à·": 30766, + "à¸ģ": 30767, + "scro": 30768, + "verses": 30769, + "orient": 30770, + "arches": 30771, + "piyu": 30772, + "knowyour": 30773, + "gree": 30774, + "takers": 30775, + "guard": 30776, + "dishon": 30777, + "bucketlist": 30778, + "bhafc": 30779, + "wardly": 30780, + "ðŁİīðŁİĬ": 30781, + "leighton": 30782, + "pew": 30783, + "stray": 30784, + "assaulted": 30785, + "inhal": 30786, + "lyfe": 30787, + "amarketing": 30788, + "lx": 30789, + "katz": 30790, + "ubuntu": 30791, + "meo": 30792, + "cartoonist": 30793, + "turnover": 30794, + "miz": 30795, + "dislike": 30796, + "mullen": 30797, + "mof": 30798, + "bland": 30799, + "hides": 30800, + "emerges": 30801, + "chorizo": 30802, + "trustee": 30803, + "mahog": 30804, + "lansing": 30805, + "paralympic": 30806, + "faint": 30807, + "fauna": 30808, + "chal": 30809, + "snar": 30810, + "cath": 30811, + "benton": 30812, + "castillo": 30813, + "slippery": 30814, + "apricot": 30815, + "oecd": 30816, + "baro": 30817, + "lz": 30818, + "heming": 30819, + "clowns": 30820, + "coworkers": 30821, + "peruvian": 30822, + "commuters": 30823, + "yell": 30824, + "ðŁļ´": 30825, + "undering": 30826, + "vj": 30827, + "ttp": 30828, + "flipk": 30829, + "wana": 30830, + "socent": 30831, + "ĤâĸĤâĸ": 30832, + "à¤Ĥ": 30833, + "oosa": 30834, + "jagger": 30835, + "dism": 30836, + "eless": 30837, + "dham": 30838, + "calif": 30839, + "aofficial": 30840, + "eclip": 30841, + "harrogate": 30842, + "grapp": 30843, + "comrade": 30844, + "ntr": 30845, + "concentrate": 30846, + "thighs": 30847, + "bitcoin": 30848, + "belarus": 30849, + "ëĵ": 30850, + "enduring": 30851, + "nowwatching": 30852, + "industrial": 30853, + "pip": 30854, + "aron": 30855, + "arat": 30856, + "®": 30857, + "whitby": 30858, + "ooooooo": 30859, + "saree": 30860, + "ticals": 30861, + "misleading": 30862, + "yoon": 30863, + "years": 30864, + "sleigh": 30865, + "romanian": 30866, + "scissors": 30867, + "vampires": 30868, + "acup": 30869, + "abba": 30870, + "thweeksary": 30871, + "centri": 30872, + "flye": 30873, + "uo": 30874, + "cbi": 30875, + "buena": 30876, + "sind": 30877, + "marino": 30878, + "burr": 30879, + "rebuilding": 30880, + "ल": 30881, + "anniversaire": 30882, + "acca": 30883, + "ðŁĴĢðŁĴĢ": 30884, + "getting": 30885, + "tulips": 30886, + "wolfpack": 30887, + "âľįï¸ı": 30888, + "morethan": 30889, + "takin": 30890, + "ðŁ¤ĺðŁı»": 30891, + "ube": 30892, + "monic": 30893, + "doubts": 30894, + "mower": 30895, + "cobalt": 30896, + "donne": 30897, + "speculation": 30898, + "arguably": 30899, + "kaku": 30900, + "https": 30901, + "prosecution": 30902, + "dinah": 30903, + "stamatic": 30904, + "disclosed": 30905, + "beverly": 30906, + "flwx": 30907, + "crabs": 30908, + "extraordinaire": 30909, + "warmest": 30910, + "imperi": 30911, + "ologists": 30912, + "traces": 30913, + "parc": 30914, + "lakeside": 30915, + "amr": 30916, + "teri": 30917, + "hourly": 30918, + "domination": 30919, + "arrow": 30920, + "shrewsbury": 30921, + "ancestry": 30922, + "wrangler": 30923, + "triggered": 30924, + "pensac": 30925, + "rooster": 30926, + "survives": 30927, + "aon": 30928, + "boko": 30929, + "valor": 30930, + "loveis": 30931, + "lag": 30932, + "pey": 30933, + "focal": 30934, + "outlaws": 30935, + "blanc": 30936, + "articho": 30937, + "wits": 30938, + "marshall": 30939, + "diego": 30940, + "supportsmall": 30941, + "uca": 30942, + "sah": 30943, + "jeet": 30944, + "synago": 30945, + "governing": 30946, + "ðŁĴ¬": 30947, + "salads": 30948, + "create": 30949, + "miriam": 30950, + "censored": 30951, + "amide": 30952, + "nou": 30953, + "zeta": 30954, + "allegiance": 30955, + "*)": 30956, + "blm": 30957, + "rican": 30958, + "pastors": 30959, + "olympus": 30960, + "bloc": 30961, + "whirl": 30962, + "starry": 30963, + "prone": 30964, + "yk": 30965, + "pne": 30966, + "congratulating": 30967, + "bev": 30968, + "sober": 30969, + "loveisland": 30970, + "sair": 30971, + "aning": 30972, + "tutorials": 30973, + "qe": 30974, + "lund": 30975, + "inist": 30976, + "clever": 30977, + "taxpayer": 30978, + "aliz": 30979, + "wrench": 30980, + "ddling": 30981, + "capri": 30982, + "hpa": 30983, + "ðŁı»âĢįâĻĤï¸ı": 30984, + "naj": 30985, + "oj": 30986, + "futuristic": 30987, + "jellyfish": 30988, + "ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥": 30989, + "celery": 30990, + "plank": 30991, + "fila": 30992, + "neme": 30993, + "unhealthy": 30994, + "lections": 30995, + "ðŁ§¡": 30996, + "ritchie": 30997, + "nws": 30998, + "mikha": 30999, + "wonderwoman": 31000, + "âĢİ": 31001, + "hipstamatic": 31002, + "kag": 31003, + "ðŁĴľðŁĴľðŁĴľ": 31004, + "poultry": 31005, + "mow": 31006, + "words": 31007, + "loff": 31008, + "ðŁ¤£ðŁ¤£": 31009, + "relatable": 31010, + "remixes": 31011, + "kenyatta": 31012, + "kem": 31013, + "resigned": 31014, + "fod": 31015, + "straigh": 31016, + "jlo": 31017, + "hutch": 31018, + "boxers": 31019, + "colleen": 31020, + "mags": 31021, + "instructional": 31022, + "kol": 31023, + "attracts": 31024, + "prag": 31025, + "accountant": 31026, + "goggles": 31027, + "bru": 31028, + "thole": 31029, + "marrow": 31030, + "leuke": 31031, + "octo": 31032, + "ponds": 31033, + "bubbly": 31034, + "heist": 31035, + "ìĹij": 31036, + "imp": 31037, + "ahar": 31038, + "haunt": 31039, + "hallmark": 31040, + "psych": 31041, + "kkkkkkkk": 31042, + "columb": 31043, + "jumpsuit": 31044, + "costco": 31045, + "sidelines": 31046, + "aggies": 31047, + "overturned": 31048, + "nib": 31049, + "keychain": 31050, + "fuk": 31051, + "faf": 31052, + "miam": 31053, + "assistants": 31054, + "cycled": 31055, + "rider": 31056, + "dammit": 31057, + "redwings": 31058, + "mages": 31059, + "kins": 31060, + "ìĤ": 31061, + "hod": 31062, + "sont": 31063, + "caroline": 31064, + "\"'": 31065, + "cule": 31066, + "braid": 31067, + "felony": 31068, + "arities": 31069, + "rutherford": 31070, + "depiction": 31071, + "isabelle": 31072, + "roach": 31073, + "kday": 31074, + "fifthharmony": 31075, + "emy": 31076, + "ligam": 31077, + "barista": 31078, + "albuquerque": 31079, + "gross": 31080, + "ðŁįº": 31081, + "ooks": 31082, + "ðŁij¼": 31083, + "duncan": 31084, + "tryin": 31085, + "jags": 31086, + "gould": 31087, + "litho": 31088, + "âģ£": 31089, + "аÐ": 31090, + "sammy": 31091, + "tung": 31092, + "casser": 31093, + "apolo": 31094, + "aaaaa": 31095, + "mang": 31096, + "asics": 31097, + "shen": 31098, + "pye": 31099, + "turbul": 31100, + "ssp": 31101, + "saintsfc": 31102, + "onlin": 31103, + "nanny": 31104, + "hester": 31105, + "doz": 31106, + "à¸Ķ": 31107, + "thread": 31108, + "rents": 31109, + "khand": 31110, + "ðŁĴªðŁı½": 31111, + "unconditional": 31112, + "robson": 31113, + "carre": 31114, + "phon": 31115, + "sacrificed": 31116, + "£": 31117, + "autos": 31118, + "parker": 31119, + "oca": 31120, + "login": 31121, + "keegan": 31122, + "hardcover": 31123, + "doughnuts": 31124, + "ðŁĮİ": 31125, + "spitfire": 31126, + "refreshments": 31127, + "saskatoon": 31128, + "commodore": 31129, + "jf": 31130, + "rubber": 31131, + "halamadrid": 31132, + "childcare": 31133, + "strada": 31134, + "iom": 31135, + "rik": 31136, + "dakar": 31137, + "thermom": 31138, + "cropped": 31139, + "garu": 31140, + "alik": 31141, + "veni": 31142, + "ift": 31143, + "sika": 31144, + "rituals": 31145, + "zul": 31146, + "ech": 31147, + "©": 31148, + "sudan": 31149, + "lland": 31150, + "ime": 31151, + "docker": 31152, + "ì¤": 31153, + "feared": 31154, + "fao": 31155, + "walter": 31156, + "nog": 31157, + "mutuals": 31158, + "lh": 31159, + "align": 31160, + "monia": 31161, + "conceptart": 31162, + "ðŁĻıðŁı¼": 31163, + "scoe": 31164, + "competence": 31165, + "swine": 31166, + "lyme": 31167, + "launch": 31168, + "greener": 31169, + "abstractart": 31170, + "inquis": 31171, + "granada": 31172, + "gaelic": 31173, + "fluff": 31174, + "dbacks": 31175, + "graveyard": 31176, + "babe": 31177, + "academic": 31178, + "adventurous": 31179, + "johann": 31180, + "~!": 31181, + "bibi": 31182, + "|#": 31183, + "plings": 31184, + "getty": 31185, + "asb": 31186, + "âĿ¤ï¸ı@": 31187, + "staff": 31188, + "religions": 31189, + "bangor": 31190, + "worldbookday": 31191, + "megh": 31192, + "devin": 31193, + "ashore": 31194, + "meridian": 31195, + "github": 31196, + "quiz": 31197, + "allstars": 31198, + "bestest": 31199, + "irresi": 31200, + "acker": 31201, + "dote": 31202, + "warrington": 31203, + "polly": 31204, + "neworleans": 31205, + "crou": 31206, + "wigs": 31207, + "chey": 31208, + "smithsonian": 31209, + "lasag": 31210, + "detour": 31211, + "boris": 31212, + "straps": 31213, + "mariah": 31214, + "intentionally": 31215, + "koh": 31216, + "ðŁį¸": 31217, + "ssian": 31218, + "marissa": 31219, + "coral": 31220, + "episcopal": 31221, + "casualty": 31222, + "tomo": 31223, + "supplychain": 31224, + "samp": 31225, + "ongo": 31226, + "roo": 31227, + "caviar": 31228, + "pfw": 31229, + "claudio": 31230, + "buffalo": 31231, + "sations": 31232, + "matty": 31233, + "snapback": 31234, + "lds": 31235, + "alarms": 31236, + "matte": 31237, + "âĺĶï¸ı": 31238, + "conditioner": 31239, + "dors": 31240, + "hex": 31241, + "fizz": 31242, + "astri": 31243, + "sussex": 31244, + "security": 31245, + "qaeda": 31246, + "allstar": 31247, + "cocacola": 31248, + "asone": 31249, + "clicks": 31250, + "scans": 31251, + "mute": 31252, + "heavier": 31253, + "ðŁİ§": 31254, + "âĺŀ": 31255, + "lvl": 31256, + "bookboost": 31257, + "youtube": 31258, + "flashes": 31259, + "fjor": 31260, + "csu": 31261, + "explode": 31262, + "dodge": 31263, + "cairn": 31264, + "gonzales": 31265, + "thill": 31266, + "pelle": 31267, + "hartley": 31268, + "renewable": 31269, + "retin": 31270, + "estre": 31271, + "costarica": 31272, + "shipyard": 31273, + "ncfc": 31274, + "priya": 31275, + "aghan": 31276, + "anath": 31277, + "plugin": 31278, + "corey": 31279, + "rebound": 31280, + "oru": 31281, + "katrin": 31282, + "hormone": 31283, + "gim": 31284, + "mahindra": 31285, + "ssus": 31286, + "parkland": 31287, + "harper": 31288, + "fantastic": 31289, + "inferno": 31290, + "epilo": 31291, + "wrestling": 31292, + "fect": 31293, + "cit": 31294, + "acoun": 31295, + "tossed": 31296, + "monumental": 31297, + "chartered": 31298, + "bust": 31299, + "petra": 31300, + "âĮļ": 31301, + "wildflowerhour": 31302, + "sweaters": 31303, + "*.": 31304, + "bler": 31305, + "atech": 31306, + "gowan": 31307, + "demographic": 31308, + "bral": 31309, + "suicide": 31310, + "renovations": 31311, + "vuel": 31312, + "sinister": 31313, + "armani": 31314, + "misogy": 31315, + "pharrell": 31316, + "naps": 31317, + "uniting": 31318, + "crusaders": 31319, + "corgi": 31320, + "insured": 31321, + "thani": 31322, + "noor": 31323, + "gq": 31324, + "dada": 31325, + "bicycles": 31326, + "snuggle": 31327, + "schan": 31328, + "tenberg": 31329, + "ssal": 31330, + "femme": 31331, + "boil": 31332, + "½ï¸ı": 31333, + "reap": 31334, + "occurring": 31335, + "hussein": 31336, + "divid": 31337, + "stoke": 31338, + "shalom": 31339, + "naia": 31340, + "olic": 31341, + "frustrating": 31342, + "Ùĩ": 31343, + "igs": 31344, + "grover": 31345, + "scenarios": 31346, + "nds": 31347, + "brutality": 31348, + "medalli": 31349, + "buon": 31350, + "sass": 31351, + "skateboarding": 31352, + "onyx": 31353, + "lorry": 31354, + "nyu": 31355, + "gautam": 31356, + "mmings": 31357, + "gug": 31358, + "endi": 31359, + "lothian": 31360, + "commando": 31361, + "chalk": 31362, + "phora": 31363, + "assessing": 31364, + "tigh": 31365, + "crunchy": 31366, + "aday": 31367, + "isl": 31368, + "ciara": 31369, + "pilgrims": 31370, + "kamal": 31371, + "pto": 31372, + "britanni": 31373, + "tani": 31374, + "smc": 31375, + "lure": 31376, + "appstore": 31377, + "aby": 31378, + "golfing": 31379, + "clc": 31380, + "fau": 31381, + "anas": 31382, + "shutting": 31383, + "regulated": 31384, + "carnage": 31385, + "scowboys": 31386, + "allenge": 31387, + "cma": 31388, + "humboldt": 31389, + "relle": 31390, + "kumb": 31391, + "heri": 31392, + "refinery": 31393, + "soundcheck": 31394, + "dwayne": 31395, + "bosnia": 31396, + "isp": 31397, + "thealth": 31398, + "anniv": 31399, + "relevance": 31400, + "mya": 31401, + "baggage": 31402, + "dread": 31403, + "sbc": 31404, + "thed": 31405, + "buh": 31406, + "hijab": 31407, + "loid": 31408, + "kew": 31409, + "cte": 31410, + "respect": 31411, + "lovelies": 31412, + "cubes": 31413, + "celebrate": 31414, + "dirt": 31415, + "savers": 31416, + "_,": 31417, + "garment": 31418, + "pulitzer": 31419, + "masjid": 31420, + "beatport": 31421, + "alarts": 31422, + "encryption": 31423, + "sner": 31424, + "pleads": 31425, + "foundry": 31426, + "symmetry": 31427, + "rumi": 31428, + "birthplace": 31429, + "scallops": 31430, + "supple": 31431, + "pivotal": 31432, + "tati": 31433, + "node": 31434, + "sod": 31435, + "proxim": 31436, + "trics": 31437, + "coldest": 31438, + "brent": 31439, + "mandu": 31440, + "clair": 31441, + "each": 31442, + "andalu": 31443, + "hiddleston": 31444, + "ðŁIJº": 31445, + "melts": 31446, + "vance": 31447, + "pinn": 31448, + "sements": 31449, + "screened": 31450, + "sachs": 31451, + "obl": 31452, + "icha": 31453, + "âĺĺï¸ı": 31454, + "schoolers": 31455, + "healed": 31456, + "logged": 31457, + "ðŁ¤ĺðŁı¼": 31458, + "icus": 31459, + "boredom": 31460, + "bish": 31461, + "bffs": 31462, + "talking": 31463, + "suresh": 31464, + "hookem": 31465, + "deon": 31466, + "defl": 31467, + "eileen": 31468, + "ðŁįķ": 31469, + "womenintech": 31470, + "risotto": 31471, + "ranger": 31472, + "advertise": 31473, + "à¸ģà¸": 31474, + "telly": 31475, + "lago": 31476, + "dartmoor": 31477, + "dong": 31478, + "skates": 31479, + "logo": 31480, + "unner": 31481, + "mailbox": 31482, + "masala": 31483, + "looooo": 31484, + "amethyst": 31485, + "chewing": 31486, + "cbb": 31487, + "australians": 31488, + "rcmp": 31489, + "gameart": 31490, + "#...": 31491, + "korn": 31492, + "extremism": 31493, + "fruitful": 31494, + "ancient": 31495, + "pubg": 31496, + "polite": 31497, + "whit": 31498, + "murals": 31499, + "mgr": 31500, + "lineman": 31501, + "davao": 31502, + "stems": 31503, + "tennis": 31504, + "avage": 31505, + "tupac": 31506, + "gigantic": 31507, + "hsbc": 31508, + "autobiography": 31509, + "upthe": 31510, + "ีà¹Ī": 31511, + "regal": 31512, + "figuring": 31513, + "kul": 31514, + "missy": 31515, + "hoop": 31516, + "gras": 31517, + "forums": 31518, + "backlash": 31519, + "abducted": 31520, + "pnw": 31521, + "minic": 31522, + "butt": 31523, + "bottoms": 31524, + "aton": 31525, + "veng": 31526, + "ðŁĮı": 31527, + "delaney": 31528, + "prabhu": 31529, + "fanclub": 31530, + "overhaul": 31531, + "healthye": 31532, + "syno": 31533, + "aaf": 31534, + "renamed": 31535, + "kimi": 31536, + "uncle": 31537, + "mancity": 31538, + "seu": 31539, + "quanti": 31540, + "esteem": 31541, + "umin": 31542, + "enzo": 31543, + "melvin": 31544, + "undergo": 31545, + "jhar": 31546, + "farah": 31547, + "coasters": 31548, + "humphrey": 31549, + "mhz": 31550, + "childrens": 31551, + "^.": 31552, + "dhi": 31553, + "disruptive": 31554, + "integrating": 31555, + "rnb": 31556, + "oversized": 31557, + "aide": 31558, + "neau": 31559, + "documentation": 31560, + "ðŁijĢðŁijĢ": 31561, + "palo": 31562, + "hearth": 31563, + "riyad": 31564, + "punctu": 31565, + "abcnews": 31566, + "secures": 31567, + "boyband": 31568, + "birch": 31569, + "juco": 31570, + "traff": 31571, + "legislators": 31572, + "baya": 31573, + "ãĤ¯": 31574, + "noises": 31575, + "collects": 31576, + "swarm": 31577, + "kner": 31578, + "bishops": 31579, + "sturgeon": 31580, + "snapping": 31581, + "mol": 31582, + "freaky": 31583, + "chairperson": 31584, + "trop": 31585, + "lynch": 31586, + "carcin": 31587, + "artsy": 31588, + "esto": 31589, + "chai": 31590, + "flur": 31591, + "invali": 31592, + "sausages": 31593, + "imel": 31594, + "jor": 31595, + "funfact": 31596, + "witter": 31597, + "punished": 31598, + "acons": 31599, + "hya": 31600, + "reversi": 31601, + "emc": 31602, + "diffu": 31603, + "zx": 31604, + "spaw": 31605, + "clad": 31606, + "dmit": 31607, + "holland": 31608, + "fresco": 31609, + "payroll": 31610, + "abundant": 31611, + "stuffing": 31612, + "moro": 31613, + "cny": 31614, + "boycott": 31615, + "wendy": 31616, + "eleven": 31617, + "provoc": 31618, + "pilot": 31619, + "trx": 31620, + "bead": 31621, + "climateaction": 31622, + "rion": 31623, + "assie": 31624, + "ìĸ": 31625, + "osm": 31626, + "islamic": 31627, + "hoar": 31628, + "goodreads": 31629, + "alici": 31630, + "afternoons": 31631, + "spokesman": 31632, + "jolie": 31633, + "itas": 31634, + "mascara": 31635, + "âĻ©âĻ«": 31636, + "prevail": 31637, + "beetroot": 31638, + "lujah": 31639, + "kli": 31640, + "dodger": 31641, + "»": 31642, + "rule": 31643, + "ln": 31644, + "scream": 31645, + "hobart": 31646, + "colbert": 31647, + "rtc": 31648, + "erm": 31649, + "patro": 31650, + "quoting": 31651, + "slive": 31652, + "quest": 31653, + "nonfiction": 31654, + "seminary": 31655, + "prosecutors": 31656, + "vest": 31657, + "expressway": 31658, + "gge": 31659, + "nautical": 31660, + "etf": 31661, + "ðŁİīðŁİĬ": 31662, + "duration": 31663, + "chaired": 31664, + "thefilm": 31665, + "fabio": 31666, + "sheh": 31667, + "cano": 31668, + "ðŁĴªðŁı»": 31669, + "withdraw": 31670, + "!:)": 31671, + "corpus": 31672, + "phenom": 31673, + "yelp": 31674, + "lawn": 31675, + "entom": 31676, + "snapper": 31677, + "butte": 31678, + "pinball": 31679, + "proxy": 31680, + "libre": 31681, + "allevi": 31682, + "nada": 31683, + "gabriel": 31684, + "fowl": 31685, + "eureka": 31686, + "daphne": 31687, + "tunes": 31688, + "punched": 31689, + "whore": 31690, + "jog": 31691, + "rential": 31692, + "manners": 31693, + "ope": 31694, + "whufc": 31695, + "guth": 31696, + "revolt": 31697, + "sneaker": 31698, + "philharmonic": 31699, + "hoste": 31700, + "sovereignty": 31701, + "ðŁĻıðŁĻıðŁĻı": 31702, + "fishing": 31703, + "sciart": 31704, + "feta": 31705, + "ipp": 31706, + "dumping": 31707, + "kelown": 31708, + "giri": 31709, + "digits": 31710, + "salu": 31711, + "sanjay": 31712, + "tweeters": 31713, + "spas": 31714, + "colchester": 31715, + "scab": 31716, + "madd": 31717, + "à¹Ħà¸": 31718, + "Äĩ": 31719, + "geddon": 31720, + "marchfor": 31721, + "dop": 31722, + "maureen": 31723, + "unplugged": 31724, + "dido": 31725, + "fashionblogger": 31726, + "upa": 31727, + "mexic": 31728, + "tary": 31729, + "polye": 31730, + "jameson": 31731, + "vt": 31732, + "grinder": 31733, + "maddy": 31734, + "consultancy": 31735, + "¬ë": 31736, + "leagueoflegends": 31737, + "accents": 31738, + "umni": 31739, + "janeiro": 31740, + "tuss": 31741, + "hens": 31742, + "amplifier": 31743, + "toshi": 31744, + "prettier": 31745, + "prevents": 31746, + "newtown": 31747, + "redwood": 31748, + "vantage": 31749, + "ballard": 31750, + "artof": 31751, + "ashe": 31752, + "asion": 31753, + "lacey": 31754, + "apat": 31755, + "grove": 31756, + "à¸Ħ": 31757, + "rwand": 31758, + "realtors": 31759, + "traitor": 31760, + "bedding": 31761, + "ör": 31762, + "zion": 31763, + "flashing": 31764, + "campan": 31765, + "boomer": 31766, + "secretariat": 31767, + "abol": 31768, + "litigation": 31769, + "contamination": 31770, + "sedly": 31771, + "shredded": 31772, + "infor": 31773, + "doherty": 31774, + "benchmark": 31775, + "roche": 31776, + "skateboard": 31777, + "shovel": 31778, + "izz": 31779, + "topper": 31780, + "oster": 31781, + "labyrin": 31782, + "autum": 31783, + "kong": 31784, + "hummus": 31785, + "viz": 31786, + "technews": 31787, + "klaus": 31788, + "amusing": 31789, + "socialmediamarketing": 31790, + "ides": 31791, + "castell": 31792, + "stee": 31793, + "underestimate": 31794, + "calab": 31795, + "paign": 31796, + "billing": 31797, + "unanimously": 31798, + "gmb": 31799, + "flyfishing": 31800, + "hathaway": 31801, + "commercial": 31802, + "colouring": 31803, + "skulls": 31804, + "pivot": 31805, + "tep": 31806, + "tbc": 31807, + "motorway": 31808, + "xpress": 31809, + "constructive": 31810, + "puk": 31811, + "underlying": 31812, + "kirsten": 31813, + "maniac": 31814, + "chao": 31815, + "sema": 31816, + "chiffon": 31817, + "ðŁijĮðŁı»": 31818, + "verona": 31819, + "komo": 31820, + "standoff": 31821, + "wiped": 31822, + "cated": 31823, + "blair": 31824, + "workin": 31825, + "msc": 31826, + "bethlehem": 31827, + "swipe": 31828, + "unexpec": 31829, + "pees": 31830, + "petri": 31831, + "origami": 31832, + "ðŁijħ": 31833, + "mexico": 31834, + "flavor": 31835, + "rudd": 31836, + "cannabis": 31837, + "maru": 31838, + "riddle": 31839, + "worshi": 31840, + "silon": 31841, + "schat": 31842, + "apse": 31843, + "tanger": 31844, + "bious": 31845, + "eer": 31846, + "questioned": 31847, + "ozar": 31848, + "dank": 31849, + "anglesey": 31850, + "charan": 31851, + "baku": 31852, + "competen": 31853, + "repri": 31854, + "batter": 31855, + "saxon": 31856, + "calves": 31857, + "lengths": 31858, + "$$$": 31859, + "âŀ¡ï¸ı": 31860, + "immersion": 31861, + "gaunt": 31862, + "carry": 31863, + "cyto": 31864, + "banda": 31865, + "shutt": 31866, + "experience": 31867, + "elgin": 31868, + "mousse": 31869, + "taz": 31870, + "êµ": 31871, + "incorrect": 31872, + "enz": 31873, + "bham": 31874, + "moron": 31875, + "sover": 31876, + "arun": 31877, + "tipped": 31878, + "lable": 31879, + "dearly": 31880, + "bautista": 31881, + "íĻ": 31882, + "mortal": 31883, + "woop": 31884, + "dtla": 31885, + "shocks": 31886, + "davos": 31887, + "ðŁĵĿ": 31888, + "swimwear": 31889, + "herman": 31890, + "ðŁijĩðŁijĩ": 31891, + "zir": 31892, + "neglected": 31893, + "graced": 31894, + "campuses": 31895, + "avs": 31896, + "arora": 31897, + "swachhb": 31898, + "livepd": 31899, + "accra": 31900, + "enquiries": 31901, + "shooters": 31902, + "kurt": 31903, + "vancouver": 31904, + "bradley": 31905, + "garda": 31906, + "gü": 31907, + "olla": 31908, + "attracting": 31909, + "upton": 31910, + "newin": 31911, + "lumia": 31912, + "furnace": 31913, + "evers": 31914, + "eon": 31915, + "swa": 31916, + "rookies": 31917, + "aoc": 31918, + "vss": 31919, + "brisket": 31920, + "torch": 31921, + "yoda": 31922, + "heartland": 31923, + "taco": 31924, + "phony": 31925, + "foodbank": 31926, + "abbey": 31927, + "babylon": 31928, + "uy": 31929, + "greate": 31930, + "expresses": 31931, + "dandy": 31932, + "scapes": 31933, + "survivor": 31934, + "rond": 31935, + "eci": 31936, + "havin": 31937, + "abel": 31938, + "childish": 31939, + "torque": 31940, + "wavy": 31941, + "urself": 31942, + "kanyewest": 31943, + "yearof": 31944, + "alestine": 31945, + "obrien": 31946, + "alfon": 31947, + "skag": 31948, + "korean": 31949, + "anchorage": 31950, + "valeri": 31951, + "dew": 31952, + "ðŁİ¨": 31953, + "landslide": 31954, + "carole": 31955, + "christen": 31956, + "gophers": 31957, + "afi": 31958, + "priyanka": 31959, + "qq": 31960, + "powerof": 31961, + "itte": 31962, + "pcso": 31963, + "twol": 31964, + "pry": 31965, + "intellectu": 31966, + "guerrero": 31967, + "piles": 31968, + "wishlist": 31969, + "wren": 31970, + "timetable": 31971, + "ëı": 31972, + "prodigy": 31973, + "gibbons": 31974, + "./": 31975, + "neur": 31976, + "anzac": 31977, + "murray": 31978, + "viest": 31979, + "plaster": 31980, + "lair": 31981, + "artgallery": 31982, + "intercontinental": 31983, + "gbr": 31984, + "bellator": 31985, + "namjoon": 31986, + "mammals": 31987, + "amel": 31988, + "yaw": 31989, + "sarasota": 31990, + "camar": 31991, + "budding": 31992, + "summari": 31993, + "acosta": 31994, + "lash": 31995, + "eyou": 31996, + "postgraduate": 31997, + "instructors": 31998, + "tig": 31999, + "constant": 32000, + "werewolf": 32001, + "icos": 32002, + "clas": 32003, + "glenn": 32004, + "budge": 32005, + "ðŁĻĤ": 32006, + "erta": 32007, + "stains": 32008, + "persecution": 32009, + "cumbri": 32010, + "och": 32011, + "synergy": 32012, + "huang": 32013, + "scandin": 32014, + "midterms": 32015, + "commentator": 32016, + "regarded": 32017, + "perpetual": 32018, + "boiling": 32019, + "alp": 32020, + "lange": 32021, + "schle": 32022, + "faceli": 32023, + "tweeta": 32024, + "ridden": 32025, + "oktoberfest": 32026, + "charlottesville": 32027, + "iklan": 32028, + "jou": 32029, + "chatham": 32030, + "bsc": 32031, + "ðŁį¦": 32032, + "strauss": 32033, + "mellow": 32034, + "xxxx": 32035, + "happyhour": 32036, + "reactor": 32037, + "wwer": 32038, + "distraction": 32039, + "atorial": 32040, + "ðŁĴªðŁı¼": 32041, + "twinpeaks": 32042, + "fayette": 32043, + "aor": 32044, + "kok": 32045, + "broom": 32046, + "syfy": 32047, + "ouse": 32048, + "amag": 32049, + "Ø·": 32050, + "ubisoft": 32051, + "lulu": 32052, + "hallmark": 32053, + "stuart": 32054, + "itya": 32055, + "sideline": 32056, + "vengeance": 32057, + "relu": 32058, + "sexism": 32059, + "bouncing": 32060, + "unites": 32061, + "gustav": 32062, + "tessa": 32063, + "stump": 32064, + "proclamation": 32065, + "imax": 32066, + "dividend": 32067, + "colby": 32068, + "ðŁįİ": 32069, + "playwright": 32070, + "unsafe": 32071, + "cosmo": 32072, + "ðŁĩ²ðŁĩ½": 32073, + "cupboard": 32074, + "constituents": 32075, + "anglia": 32076, + "rampage": 32077, + "ðŁĺįðŁĺįðŁĺįðŁĺįðŁĺį": 32078, + "thanked": 32079, + "takeaways": 32080, + "shroff": 32081, + "debat": 32082, + "khur": 32083, + "conducts": 32084, + "formats": 32085, + "à©": 32086, + "portage": 32087, + "graphers": 32088, + "uten": 32089, + "prem": 32090, + "moines": 32091, + "condemns": 32092, + "sous": 32093, + "lps": 32094, + "fcs": 32095, + "dealership": 32096, + "leukemia": 32097, + "bureau": 32098, + "skid": 32099, + "guardiola": 32100, + "caster": 32101, + "third": 32102, + "avoided": 32103, + "encyclo": 32104, + "csr": 32105, + "vixx": 32106, + "analyzing": 32107, + "shear": 32108, + "duluth": 32109, + "shapiro": 32110, + "chanting": 32111, + "stresses": 32112, + "asbe": 32113, + "militia": 32114, + "ãĥª": 32115, + "collin": 32116, + "arsene": 32117, + "suresh": 32118, + "teachings": 32119, + "yixing": 32120, + "shill": 32121, + "nudes": 32122, + "svu": 32123, + "clearwater": 32124, + "warped": 32125, + "prolife": 32126, + "artistson": 32127, + "itu": 32128, + "versailles": 32129, + "galaxy": 32130, + "axel": 32131, + "springst": 32132, + "cala": 32133, + "huhu": 32134, + "scu": 32135, + "commitments": 32136, + "exeter": 32137, + "poignant": 32138, + "motion": 32139, + "conservatory": 32140, + "rowdy": 32141, + "recalled": 32142, + "musk": 32143, + "embelli": 32144, + "sothe": 32145, + "âĺĢ": 32146, + "stopper": 32147, + "schild": 32148, + "tope": 32149, + "elmo": 32150, + "ziel": 32151, + "jom": 32152, + "barnsley": 32153, + "snowden": 32154, + "ontour": 32155, + "journey": 32156, + "hillsborough": 32157, + "parole": 32158, + "wts": 32159, + "moving": 32160, + "agility": 32161, + "tivo": 32162, + "ffers": 32163, + "kindleunlimited": 32164, + "gwen": 32165, + "annan": 32166, + "ahmad": 32167, + "textured": 32168, + "hepatitis": 32169, + "dram": 32170, + "insiders": 32171, + "tissues": 32172, + "ãĥĦ": 32173, + "fcbarcelona": 32174, + "cratic": 32175, + "naacp": 32176, + "pecan": 32177, + "fgm": 32178, + "customize": 32179, + "concert": 32180, + "gsm": 32181, + "peg": 32182, + "pone": 32183, + "justintrudeau": 32184, + "supercars": 32185, + "happyholidays": 32186, + "bular": 32187, + "adox": 32188, + "laptops": 32189, + "digitalhealth": 32190, + "destination": 32191, + "gradually": 32192, + "áĥ¦": 32193, + "poppy": 32194, + "ssl": 32195, + "inhibit": 32196, + "starlight": 32197, + "offro": 32198, + "gloomy": 32199, + "xper": 32200, + "halder": 32201, + "implants": 32202, + "leto": 32203, + "hassel": 32204, + "aas": 32205, + "untold": 32206, + "enci": 32207, + "liberia": 32208, + "oran": 32209, + "contests": 32210, + "ilah": 32211, + "smag": 32212, + "scout": 32213, + "marianne": 32214, + "cryo": 32215, + "scheduling": 32216, + "los": 32217, + "kane": 32218, + "stuttgart": 32219, + "nese": 32220, + "lawrence": 32221, + "dain": 32222, + "photom": 32223, + "carou": 32224, + "ร": 32225, + "gwy": 32226, + "nationaldogday": 32227, + "roasting": 32228, + "bandcamp": 32229, + "kentucky": 32230, + "stretches": 32231, + "kerel": 32232, + "cashe": 32233, + "ãĤ¸": 32234, + "stax": 32235, + "transi": 32236, + "doggie": 32237, + "atric": 32238, + "halle": 32239, + "civic": 32240, + "browning": 32241, + "leinster": 32242, + "catday": 32243, + "highland": 32244, + "joyous": 32245, + "incumb": 32246, + "orlando": 32247, + "romo": 32248, + "colton": 32249, + "delta": 32250, + "carab": 32251, + "rotc": 32252, + "asteroid": 32253, + "goosebumps": 32254, + "mology": 32255, + "yoko": 32256, + "ands": 32257, + "tomorrows": 32258, + "redcarpet": 32259, + "smp": 32260, + "casio": 32261, + "ðŁ¤£ðŁ¤£ðŁ¤£": 32262, + "seau": 32263, + "rejection": 32264, + "rotating": 32265, + "bipartisan": 32266, + "thun": 32267, + "mati": 32268, + "boni": 32269, + "oll": 32270, + "energye": 32271, + "doit": 32272, + "lj": 32273, + "motherhood": 32274, + "louise": 32275, + "necklaces": 32276, + "elite": 32277, + "nix": 32278, + "lcs": 32279, + "env": 32280, + "glu": 32281, + "lesh": 32282, + "crank": 32283, + "susie": 32284, + "mclau": 32285, + "sotu": 32286, + "crowley": 32287, + "ratri": 32288, + "used": 32289, + "breton": 32290, + "alfredo": 32291, + "yeo": 32292, + "travelpics": 32293, + "tipp": 32294, + "ellison": 32295, + "saxophone": 32296, + "mered": 32297, + "heughan": 32298, + "taine": 32299, + "fes": 32300, + "viro": 32301, + "supposedly": 32302, + "ias": 32303, + "digestive": 32304, + "yle": 32305, + "lizzy": 32306, + "wildlifephotography": 32307, + "brianna": 32308, + "westfield": 32309, + "rained": 32310, + "amher": 32311, + "ðŁĺĦðŁĺĦ": 32312, + "distribute": 32313, + "bottom": 32314, + "preserving": 32315, + "oiland": 32316, + "crafty": 32317, + "descen": 32318, + "colling": 32319, + "shakespearesunday": 32320, + "rwc": 32321, + "angled": 32322, + "cian": 32323, + "tations": 32324, + "montage": 32325, + "meyers": 32326, + "francesca": 32327, + "ðŁĮ·": 32328, + "wiggins": 32329, + "sanford": 32330, + "volunteer": 32331, + "carra": 32332, + "bark": 32333, + "varied": 32334, + "plin": 32335, + "amu": 32336, + "kapil": 32337, + "rockers": 32338, + "quind": 32339, + "brane": 32340, + "inmate": 32341, + "ental": 32342, + "improvis": 32343, + "michigan": 32344, + "retweeting": 32345, + "progressing": 32346, + "mercedesbenz": 32347, + "smoker": 32348, + "physiology": 32349, + "dorado": 32350, + "wattpad": 32351, + "hwa": 32352, + "srbachchan": 32353, + "wga": 32354, + "volatility": 32355, + "hire": 32356, + "acap": 32357, + "wnba": 32358, + "heinz": 32359, + "stitches": 32360, + "kidnapping": 32361, + "burys": 32362, + "limb": 32363, + "fitters": 32364, + "thumbnail": 32365, + "tone": 32366, + "mirand": 32367, + "desirable": 32368, + "addison": 32369, + "taran": 32370, + "tamilnadu": 32371, + "spectator": 32372, + "sociology": 32373, + "amitshah": 32374, + "remotely": 32375, + "âĻ¦": 32376, + "hamid": 32377, + "rds": 32378, + "glee": 32379, + "smoothly": 32380, + "schro": 32381, + "erc": 32382, + "laliga": 32383, + "heals": 32384, + "usf": 32385, + "nishi": 32386, + "dhu": 32387, + "unil": 32388, + "hle": 32389, + "tromb": 32390, + "bhutan": 32391, + "pilipinas": 32392, + "seung": 32393, + "whitman": 32394, + "tey": 32395, + "mince": 32396, + "snowboarding": 32397, + "reau": 32398, + "kker": 32399, + "avo": 32400, + "zachary": 32401, + "ranveer": 32402, + "tik": 32403, + "govern": 32404, + "qual": 32405, + "becky": 32406, + "anthropology": 32407, + "atten": 32408, + "groceries": 32409, + "debit": 32410, + "warp": 32411, + "silicon": 32412, + "hawaii": 32413, + "ðŁĴħ": 32414, + "pomegranate": 32415, + "peer": 32416, + "oranges": 32417, + "peopleschoice": 32418, + "endure": 32419, + "ðŁĴĽðŁĴĽ": 32420, + "ãĤ¹ãĥ": 32421, + "acial": 32422, + "ahaha": 32423, + "stuk": 32424, + "imperial": 32425, + "blond": 32426, + "powder": 32427, + "knots": 32428, + "vince": 32429, + "woodlands": 32430, + "dena": 32431, + "watchin": 32432, + "matcha": 32433, + "mahat": 32434, + "galaxies": 32435, + "middlesbrough": 32436, + "kö": 32437, + "stree": 32438, + "rescues": 32439, + "waldo": 32440, + "leroy": 32441, + "despic": 32442, + "realities": 32443, + "tmnt": 32444, + "haq": 32445, + "uno": 32446, + "pec": 32447, + "bollywood": 32448, + "blinds": 32449, + "designthinking": 32450, + "hems": 32451, + "andhra": 32452, + "absen": 32453, + "fans": 32454, + "stech": 32455, + "shirehour": 32456, + "blaine": 32457, + "shakti": 32458, + "purely": 32459, + "ðŁıı": 32460, + "trafal": 32461, + "keynes": 32462, + "grate": 32463, + "tobias": 32464, + "spontaneous": 32465, + "saturated": 32466, + "cavalry": 32467, + "prisc": 32468, + "ðŁĺij": 32469, + "wht": 32470, + "passi": 32471, + "~~~": 32472, + "virat": 32473, + "pattinson": 32474, + "lao": 32475, + "weirdo": 32476, + "sympathy": 32477, + "juda": 32478, + "occasionally": 32479, + "credited": 32480, + "statu": 32481, + "esco": 32482, + "hilly": 32483, + "escape": 32484, + "discharge": 32485, + "seer": 32486, + "maynard": 32487, + "sudbury": 32488, + "zlat": 32489, + "oral": 32490, + "weer": 32491, + "encountered": 32492, + "smelling": 32493, + "oversight": 32494, + "ê¸": 32495, + "thatcher": 32496, + "mackay": 32497, + "youcan": 32498, + "freep": 32499, + "freedoms": 32500, + "prophecy": 32501, + "hoe": 32502, + "ishqba": 32503, + "drake": 32504, + "quits": 32505, + "pelled": 32506, + "turk": 32507, + "ovi": 32508, + "wesleyan": 32509, + "newmusic": 32510, + "legg": 32511, + "cheng": 32512, + "hilli": 32513, + "ayy": 32514, + "panties": 32515, + "adversity": 32516, + "adjac": 32517, + "vaccination": 32518, + "juke": 32519, + "gac": 32520, + "exceed": 32521, + "timesof": 32522, + "staining": 32523, + "epcot": 32524, + "vital": 32525, + "upward": 32526, + "bethesda": 32527, + "apark": 32528, + "mahi": 32529, + "campfire": 32530, + "enchanting": 32531, + "rhapso": 32532, + "hz": 32533, + "naver": 32534, + "fax": 32535, + "validation": 32536, + "acad": 32537, + "nyr": 32538, + "asym": 32539, + "coordinated": 32540, + "departed": 32541, + "allery": 32542, + "varies": 32543, + "sprite": 32544, + "chaplin": 32545, + "ssoccer": 32546, + "swat": 32547, + "bret": 32548, + "reluct": 32549, + "tunesapp": 32550, + "superstar": 32551, + "reminiscing": 32552, + "oco": 32553, + "homegrown": 32554, + "doughnut": 32555, + "uncanny": 32556, + "lapd": 32557, + "thyroid": 32558, + "!âĿ¤ï¸ı": 32559, + "botanic": 32560, + "bres": 32561, + "spade": 32562, + "iste": 32563, + "echoes": 32564, + "dulil": 32565, + "bursting": 32566, + "quiero": 32567, + "ðŁijİ": 32568, + "loyola": 32569, + "amusement": 32570, + "hails": 32571, + "sleepy": 32572, + "burglary": 32573, + "âľı": 32574, + "rogue": 32575, + "cotland": 32576, + "moors": 32577, + "lower": 32578, + "wicked": 32579, + "ðŁĶĬ": 32580, + "competiti": 32581, + "argentine": 32582, + "yvonne": 32583, + "kartikeyan": 32584, + "iliary": 32585, + "gatsby": 32586, + "precinct": 32587, + "sixty": 32588, + "naji": 32589, + "cams": 32590, + "practitioner": 32591, + "ðŁĺ³ðŁĺ³": 32592, + "pune": 32593, + "negli": 32594, + "julien": 32595, + "invaded": 32596, + "calibr": 32597, + "clam": 32598, + "dubai": 32599, + "muk": 32600, + "lantic": 32601, + "product": 32602, + "fedex": 32603, + "ï¸ı:": 32604, + "eura": 32605, + "darius": 32606, + "sling": 32607, + "virtualreality": 32608, + "homestead": 32609, + "ðŁı³ï¸ıâĢįðŁĮĪ": 32610, + "paced": 32611, + "inha": 32612, + "pulmon": 32613, + "lazy": 32614, + "premiering": 32615, + "mastered": 32616, + "inhe": 32617, + "congregation": 32618, + "bajo": 32619, + "sporting": 32620, + "newjersey": 32621, + "horny": 32622, + "lmaoo": 32623, + "lengthy": 32624, + "dut": 32625, + "yogh": 32626, + "swearing": 32627, + "philosophical": 32628, + "papua": 32629, + "inski": 32630, + "knowles": 32631, + "dyke": 32632, + "âĢ²": 32633, + "token": 32634, + "mcguire": 32635, + "riot": 32636, + "probability": 32637, + "mccon": 32638, + "gros": 32639, + "sumat": 32640, + "cite": 32641, + "daa": 32642, + "onda": 32643, + "maddow": 32644, + "chew": 32645, + "boardgames": 32646, + "sparked": 32647, + "reclaimed": 32648, + "adhd": 32649, + "nyse": 32650, + "imwithher": 32651, + "equinox": 32652, + "booths": 32653, + "balsamic": 32654, + "hazy": 32655, + "dorchester": 32656, + "agos": 32657, + "seaw": 32658, + "moderator": 32659, + "seriea": 32660, + "andersen": 32661, + "pilgrim": 32662, + "âŃIJâŃIJ": 32663, + "itchen": 32664, + "halli": 32665, + "xton": 32666, + "nathaniel": 32667, + "munition": 32668, + "celestial": 32669, + "gaf": 32670, + "zoom": 32671, + "markle": 32672, + "penthouse": 32673, + "cale": 32674, + "sfa": 32675, + "barking": 32676, + "tucket": 32677, + "emery": 32678, + "calorie": 32679, + "lique": 32680, + "adar": 32681, + "mcnam": 32682, + "tortilla": 32683, + "woodpecker": 32684, + "motown": 32685, + "badger": 32686, + "ayrshire": 32687, + "scramble": 32688, + "dday": 32689, + "craziest": 32690, + "perrie": 32691, + "choco": 32692, + "caste": 32693, + "iot": 32694, + "wrecked": 32695, + "selecting": 32696, + "ussr": 32697, + "graft": 32698, + "punt": 32699, + "labou": 32700, + "irst": 32701, + "baek": 32702, + "ÛĮ": 32703, + "suki": 32704, + "queu": 32705, + "achat": 32706, + "tester": 32707, + "augmented": 32708, + "wcvb": 32709, + "sinks": 32710, + "ðŁĵ»": 32711, + "rake": 32712, + "interne": 32713, + "because": 32714, + "bellevue": 32715, + "unearth": 32716, + "lighten": 32717, + "ðŁĺ£": 32718, + "turnaround": 32719, + "labeled": 32720, + "unemployed": 32721, + "twitterkurds": 32722, + "leia": 32723, + "hye": 32724, + "greater": 32725, + "ðŁIJİ": 32726, + "timed": 32727, + "ired": 32728, + "ett": 32729, + "limitations": 32730, + "cabe": 32731, + "sout": 32732, + "beech": 32733, + "annihil": 32734, + "retrac": 32735, + "yoona": 32736, + "anger": 32737, + "dennis": 32738, + "supplying": 32739, + "diz": 32740, + "\"(": 32741, + "scur": 32742, + "gunman": 32743, + "suho": 32744, + "sauvignon": 32745, + "ล": 32746, + "wiley": 32747, + "landon": 32748, + "choreography": 32749, + "prehistoric": 32750, + "ðŁıĥ": 32751, + "vargas": 32752, + "assessments": 32753, + "pinnacle": 32754, + "dii": 32755, + "chamberlain": 32756, + "ìĪ": 32757, + "vp": 32758, + "presenters": 32759, + "deutsche": 32760, + "sunshine": 32761, + "salutes": 32762, + "rone": 32763, + "busiest": 32764, + "-.-": 32765, + "motorists": 32766, + "hemisphere": 32767, + "alwx": 32768, + "psp": 32769, + "owa": 32770, + "denying": 32771, + "choc": 32772, + "gutier": 32773, + "hanuk": 32774, + "muskete": 32775, + "jaitley": 32776, + "sewage": 32777, + "tame": 32778, + "thinkers": 32779, + "shim": 32780, + "sequo": 32781, + "papar": 32782, + "middleeast": 32783, + "kwa": 32784, + "keg": 32785, + "patagonia": 32786, + "noy": 32787, + "barça": 32788, + "takeoff": 32789, + "hea": 32790, + "à¬": 32791, + "nsc": 32792, + "gdc": 32793, + "ðŁijĪ": 32794, + "moustache": 32795, + "melania": 32796, + "thra": 32797, + "â¬Ĩï¸ı": 32798, + "pierced": 32799, + "zeus": 32800, + "fonts": 32801, + "bera": 32802, + "itiner": 32803, + "qatar": 32804, + "contrary": 32805, + "ireland": 32806, + "ify": 32807, + "oulos": 32808, + "communal": 32809, + "fins": 32810, + "unpaid": 32811, + "paa": 32812, + "ðŁijĩðŁı»": 32813, + "rios": 32814, + "oup": 32815, + "filler": 32816, + "cafeteria": 32817, + "à¸Ń": 32818, + "kasi": 32819, + "caliber": 32820, + "zulu": 32821, + "vsco": 32822, + "tsford": 32823, + "dragonfly": 32824, + "smokin": 32825, + "pist": 32826, + "psychologist": 32827, + "diplomat": 32828, + "webs": 32829, + "buccane": 32830, + "ா": 32831, + "motivational": 32832, + "dune": 32833, + "bae": 32834, + "cfs": 32835, + "without": 32836, + "eron": 32837, + "iac": 32838, + "atee": 32839, + "pension": 32840, + "frazier": 32841, + "ensis": 32842, + "skis": 32843, + "parting": 32844, + "gery": 32845, + "territories": 32846, + "nachos": 32847, + "enight": 32848, + "everlasting": 32849, + "msdhoni": 32850, + "tele": 32851, + "spun": 32852, + "podi": 32853, + "sabah": 32854, + "environmentally": 32855, + "cease": 32856, + "beaumont": 32857, + "marta": 32858, + "kelvin": 32859, + "hoff": 32860, + "sunil": 32861, + "nda": 32862, + "cob": 32863, + "shale": 32864, + "reedus": 32865, + "unboxing": 32866, + "ubio": 32867, + "reopened": 32868, + "nall": 32869, + "capsules": 32870, + "marr": 32871, + "himalayas": 32872, + "sweeter": 32873, + "jaz": 32874, + "fmr": 32875, + "tweeter": 32876, + "dhaka": 32877, + "nau": 32878, + "demi": 32879, + "dfs": 32880, + "taurus": 32881, + "fading": 32882, + "itutes": 32883, + "cip": 32884, + "overflow": 32885, + "jeffrey": 32886, + "donny": 32887, + "cartunesapp": 32888, + "ðŁįij": 32889, + "prefecture": 32890, + "danced": 32891, + "cpt": 32892, + "pleasing": 32893, + "italk": 32894, + "earthquakes": 32895, + "ulation": 32896, + "hio": 32897, + "ãĢĭ": 32898, + "antan": 32899, + "nutrient": 32900, + "deere": 32901, + "selects": 32902, + "enrichment": 32903, + "riti": 32904, + "trampol": 32905, + "blamed": 32906, + "jia": 32907, + "contributors": 32908, + "chesapeake": 32909, + "pigeons": 32910, + "tribunal": 32911, + "maduro": 32912, + "wsu": 32913, + "ilove": 32914, + "efficiently": 32915, + "darcy": 32916, + "warms": 32917, + "arra": 32918, + "ecu": 32919, + "hower": 32920, + "struggled": 32921, + "rajinikanth": 32922, + "ðŁĺ¢ðŁĺ¢": 32923, + "housing": 32924, + "strat": 32925, + "elix": 32926, + "dispro": 32927, + "raffic": 32928, + "thierry": 32929, + "nasty": 32930, + "cfb": 32931, + "staffing": 32932, + "alma": 32933, + "backers": 32934, + "henson": 32935, + "skywalker": 32936, + "realestate": 32937, + "roos": 32938, + "nessy": 32939, + "chance": 32940, + "cairns": 32941, + "cci": 32942, + "pedal": 32943, + "lyft": 32944, + "crossword": 32945, + "waiter": 32946, + "onlyin": 32947, + "kruger": 32948, + "kir": 32949, + "alejandro": 32950, + "cartier": 32951, + "carrera": 32952, + "repaired": 32953, + "ouat": 32954, + "unclear": 32955, + "unbreakable": 32956, + "todayin": 32957, + "queries": 32958, + "jody": 32959, + "genital": 32960, + "winner": 32961, + "tol": 32962, + "kelowna": 32963, + "fascinated": 32964, + "ãĥ¬": 32965, + "srisri": 32966, + "squared": 32967, + "sprung": 32968, + "negotiate": 32969, + "privately": 32970, + "aven": 32971, + ">>>>>": 32972, + "gical": 32973, + "gavin": 32974, + "chesterfield": 32975, + "zumba": 32976, + "orr": 32977, + "natalia": 32978, + "impeachment": 32979, + "mnl": 32980, + "carat": 32981, + "critique": 32982, + "credible": 32983, + "tracy": 32984, + "tani": 32985, + "musik": 32986, + "jigsaw": 32987, + "gambia": 32988, + "tolkien": 32989, + "feu": 32990, + "asper": 32991, + "savory": 32992, + "foxx": 32993, + "fitt": 32994, + "marlon": 32995, + "lrt": 32996, + "vell": 32997, + "pbr": 32998, + "imprisoned": 32999, + "iom": 33000, + "chul": 33001, + "windshield": 33002, + "kaye": 33003, + "baa": 33004, + "chord": 33005, + "sart": 33006, + "algon": 33007, + "ministerial": 33008, + "natgeo": 33009, + "lazio": 33010, + "norms": 33011, + "ðŁijįðŁijį": 33012, + "licking": 33013, + "futbol": 33014, + "unsung": 33015, + "dallascowboys": 33016, + "shred": 33017, + "disturb": 33018, + "devine": 33019, + "beards": 33020, + "chf": 33021, + "bday": 33022, + "rosso": 33023, + "igor": 33024, + "ayi": 33025, + "siren": 33026, + "kair": 33027, + "stiles": 33028, + "rof": 33029, + "magnets": 33030, + "uncover": 33031, + "mouse": 33032, + "banging": 33033, + "sighted": 33034, + "speople": 33035, + "impact": 33036, + "rowland": 33037, + "kira": 33038, + "environment": 33039, + "lovethe": 33040, + "psis": 33041, + "mishra": 33042, + "glendale": 33043, + "cajun": 33044, + "oche": 33045, + "deception": 33046, + "sexist": 33047, + "straws": 33048, + "sga": 33049, + "buffer": 33050, + "apostle": 33051, + "spl": 33052, + "popup": 33053, + "ðŁļĹ": 33054, + "rg": 33055, + "uper": 33056, + "ballin": 33057, + "idy": 33058, + "occasional": 33059, + "nationalpark": 33060, + "ðŁıĬ": 33061, + "uan": 33062, + "innovation": 33063, + "ห": 33064, + "teaparty": 33065, + "rette": 33066, + "counterfe": 33067, + "bha": 33068, + "recs": 33069, + "igen": 33070, + "ðŁĮIJ": 33071, + "hummingbird": 33072, + "cur": 33073, + "haven": 33074, + "lazar": 33075, + "pueblo": 33076, + "::": 33077, + "zionist": 33078, + "opath": 33079, + "inverness": 33080, + "promoter": 33081, + "cartoon": 33082, + "cabinets": 33083, + "mahogany": 33084, + "surveying": 33085, + "rational": 33086, + "feeling": 33087, + "testify": 33088, + "sow": 33089, + "ocon": 33090, + "ย": 33091, + "neel": 33092, + "maris": 33093, + "solitary": 33094, + "chemo": 33095, + "radcliffe": 33096, + "simons": 33097, + "rosary": 33098, + "newer": 33099, + "jodie": 33100, + "retali": 33101, + "prawn": 33102, + "paddy": 33103, + "henge": 33104, + "kala": 33105, + "implant": 33106, + "aty": 33107, + "brentwood": 33108, + "paradox": 33109, + "enez": 33110, + "redesigned": 33111, + "pour": 33112, + "wyd": 33113, + "alde": 33114, + "à¯ģ": 33115, + "sold": 33116, + "biomedical": 33117, + "à¹Ĥ": 33118, + "tttt": 33119, + "matteo": 33120, + "yser": 33121, + "newton": 33122, + "debun": 33123, + "nerdy": 33124, + "lool": 33125, + "woon": 33126, + "elisabeth": 33127, + "ecc": 33128, + "whi": 33129, + "acho": 33130, + "salvage": 33131, + "salaries": 33132, + "quity": 33133, + "navigating": 33134, + "ophthal": 33135, + "consoles": 33136, + "rebuilt": 33137, + "opec": 33138, + "asters": 33139, + "shored": 33140, + "setlist": 33141, + "kathryn": 33142, + "rhymes": 33143, + "revisiting": 33144, + "ashish": 33145, + "lift": 33146, + "repost": 33147, + "soleil": 33148, + "âı±": 33149, + "wealth": 33150, + "saat": 33151, + "wec": 33152, + "kingjames": 33153, + "flipkart": 33154, + "fieldwork": 33155, + "segu": 33156, + "modal": 33157, + "bub": 33158, + "arers": 33159, + "ðŁįĴ": 33160, + "clooney": 33161, + "paddington": 33162, + "necessity": 33163, + "guthrie": 33164, + "pente": 33165, + "limo": 33166, + "josie": 33167, + "artin": 33168, + "enc": 33169, + "lhs": 33170, + "betrayal": 33171, + "infographics": 33172, + "ier": 33173, + "moa": 33174, + "hearings": 33175, + "bonjour": 33176, + "symbolic": 33177, + "agro": 33178, + "wedges": 33179, + "kristina": 33180, + "wildflower": 33181, + "athletic": 33182, + "photography": 33183, + "pesh": 33184, + "cahill": 33185, + "chilean": 33186, + "goul": 33187, + "fioren": 33188, + "ðŁij¶": 33189, + "zil": 33190, + "skim": 33191, + "badoo": 33192, + "delia": 33193, + "treble": 33194, + "ncc": 33195, + "ðŁĩ¦ðŁĩ": 33196, + "ahouse": 33197, + "bullock": 33198, + "solitude": 33199, + "اÙĨ": 33200, + "cancers": 33201, + "futureofwork": 33202, + "hutch": 33203, + "watershed": 33204, + "warmongers": 33205, + "spilled": 33206, + "colombo": 33207, + "moth": 33208, + "associations": 33209, + "weighed": 33210, + "globalgoals": 33211, + "notjust": 33212, + "christi": 33213, + "torg": 33214, + "sweating": 33215, + "maneu": 33216, + "clusters": 33217, + "âĢ¼ï¸ıâĢ¼ï¸ı": 33218, + "taped": 33219, + "uly": 33220, + "trusting": 33221, + "yusuf": 33222, + "tein": 33223, + "rab": 33224, + ",,,,": 33225, + "sinai": 33226, + "audible": 33227, + "explicit": 33228, + "crowns": 33229, + "schiz": 33230, + "atleast": 33231, + "ðŁĹ£": 33232, + "debra": 33233, + "jesuit": 33234, + "enegger": 33235, + "zhen": 33236, + "onesie": 33237, + "iit": 33238, + "ssf": 33239, + "gurgaon": 33240, + "chakra": 33241, + "bearcats": 33242, + "kran": 33243, + "kawa": 33244, + "requesting": 33245, + "hanover": 33246, + "gend": 33247, + "soros": 33248, + "mercy": 33249, + "lovely": 33250, + "doomed": 33251, + "timmy": 33252, + "kuz": 33253, + "ull": 33254, + "abram": 33255, + "saison": 33256, + "ãĥ«": 33257, + "cleaners": 33258, + "remo": 33259, + "circuits": 33260, + "barred": 33261, + "oth": 33262, + "moist": 33263, + "madeleine": 33264, + "gallo": 33265, + "uj": 33266, + "permits": 33267, + "heaviest": 33268, + "carols": 33269, + "azte": 33270, + "giorgio": 33271, + "floats": 33272, + "declaring": 33273, + "usrc": 33274, + "minat": 33275, + "crafts": 33276, + "prima": 33277, + "conveni": 33278, + "nickelodeon": 33279, + "dancing": 33280, + "ceremonial": 33281, + "blogg": 33282, + "twp": 33283, + "anglican": 33284, + "shek": 33285, + "knick": 33286, + "(((": 33287, + "hubbard": 33288, + "harvey": 33289, + "hitman": 33290, + "feng": 33291, + "wesome": 33292, + "forza": 33293, + "sword": 33294, + "opus": 33295, + "brom": 33296, + "gibility": 33297, + "zal": 33298, + "munch": 33299, + "dancehall": 33300, + "greedy": 33301, + "hdmi": 33302, + "rebirth": 33303, + "ðŁĺĭðŁĺĭ": 33304, + "sworld": 33305, + "figurine": 33306, + "compost": 33307, + "kf": 33308, + "engraving": 33309, + "giorno": 33310, + "stana": 33311, + "kman": 33312, + "hamster": 33313, + "composers": 33314, + "aje": 33315, + "functionality": 33316, + "polk": 33317, + "isons": 33318, + "airplanes": 33319, + "tese": 33320, + "horrors": 33321, + "muscat": 33322, + "given": 33323, + "spence": 33324, + "ðŁĩ¸ðŁĩ": 33325, + "eliot": 33326, + "achilles": 33327, + "freck": 33328, + "cryptocurrencies": 33329, + "souther": 33330, + "halo": 33331, + "borneo": 33332, + "politic": 33333, + "hahahahah": 33334, + "upstate": 33335, + "siena": 33336, + "obscure": 33337, + "hausen": 33338, + "lloyd": 33339, + "happyfriday": 33340, + "motorbike": 33341, + "bona": 33342, + "americas": 33343, + "hols": 33344, + "-(": 33345, + "sporty": 33346, + "unaware": 33347, + "revenues": 33348, + "christopher": 33349, + "banksy": 33350, + "avan": 33351, + "evapor": 33352, + "compress": 33353, + "eyeliner": 33354, + "todos": 33355, + "buffy": 33356, + "renewableenergy": 33357, + "lyrical": 33358, + "archan": 33359, + "rapist": 33360, + "fairtrade": 33361, + "lmaooo": 33362, + "beatz": 33363, + "proactive": 33364, + "lapse": 33365, + "irical": 33366, + "reversal": 33367, + "pode": 33368, + "mcintyre": 33369, + "macau": 33370, + "ãĥķãĤ": 33371, + "nashgrier": 33372, + "fsa": 33373, + "gall": 33374, + "çĶŁ": 33375, + "perpetr": 33376, + "ilya": 33377, + "configuration": 33378, + "%;": 33379, + "strange": 33380, + "raci": 33381, + "à¸ĩ": 33382, + "pickups": 33383, + "kovsky": 33384, + "mammal": 33385, + "wps": 33386, + "gable": 33387, + "comparative": 33388, + "zh": 33389, + "saveour": 33390, + "davey": 33391, + "onetsy": 33392, + "mussels": 33393, + "miser": 33394, + "cristina": 33395, + "electron": 33396, + "crave": 33397, + "loren": 33398, + "precipitation": 33399, + "mz": 33400, + "ðŁį«": 33401, + "vincen": 33402, + "snowboard": 33403, + "noida": 33404, + "ahn": 33405, + "marinated": 33406, + "gtr": 33407, + "townhall": 33408, + "minis": 33409, + "bethel": 33410, + "advan": 33411, + "sura": 33412, + "shiel": 33413, + "furry": 33414, + "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ": 33415, + "lynd": 33416, + "soil": 33417, + "scence": 33418, + "seneca": 33419, + "sharjah": 33420, + "dickens": 33421, + "credentials": 33422, + "avar": 33423, + "perk": 33424, + "requiring": 33425, + "prefer": 33426, + "jian": 33427, + "deca": 33428, + "rach": 33429, + "ingfor": 33430, + "dele": 33431, + "beep": 33432, + "ðŁĴ»": 33433, + "cisely": 33434, + "huddle": 33435, + "greensboro": 33436, + "hawking": 33437, + "hoax": 33438, + "hangar": 33439, + "çľ": 33440, + "miso": 33441, + "lovin": 33442, + "greta": 33443, + "abad": 33444, + "logie": 33445, + "atan": 33446, + "snowflake": 33447, + "mahesh": 33448, + "fearthe": 33449, + "alkal": 33450, + "bobblehead": 33451, + "bahn": 33452, + "judged": 33453, + "futu": 33454, + "felix": 33455, + "ðŁįĵ": 33456, + "pike": 33457, + "deriv": 33458, + "notices": 33459, + "auer": 33460, + "dissuper": 33461, + "orda": 33462, + "wipes": 33463, + "amino": 33464, + "strikers": 33465, + "footb": 33466, + "dramas": 33467, + "punching": 33468, + "scoreless": 33469, + "hemingway": 33470, + "bih": 33471, + "ballad": 33472, + "chatter": 33473, + "ammo": 33474, + "klein": 33475, + "fabrication": 33476, + "karim": 33477, + "zend": 33478, + "histo": 33479, + "volta": 33480, + "rocky": 33481, + "marketer": 33482, + "xtreme": 33483, + "sequencing": 33484, + "paradigm": 33485, + "cleats": 33486, + "booming": 33487, + "âģłâģł": 33488, + "blockade": 33489, + "prompts": 33490, + "yoghurt": 33491, + "purpose": 33492, + "nur": 33493, + "regulate": 33494, + "noisy": 33495, + "ingrid": 33496, + "birdwatching": 33497, + "bartender": 33498, + "Ùĥ": 33499, + "wordof": 33500, + "chaotic": 33501, + "shorty": 33502, + "eldest": 33503, + "zapp": 33504, + "onceuponatime": 33505, + "flyo": 33506, + "ritos": 33507, + "mikequind": 33508, + "ðŁIJ´": 33509, + "registering": 33510, + ".]": 33511, + "adol": 33512, + "gggg": 33513, + "purge": 33514, + "kidlit": 33515, + "arbor": 33516, + "valves": 33517, + "synagogue": 33518, + "oth": 33519, + "unanimous": 33520, + "verification": 33521, + "darrell": 33522, + "ãģĦ": 33523, + "vanderbilt": 33524, + "tapestry": 33525, + "prosper": 33526, + "diddy": 33527, + "drafting": 33528, + "decep": 33529, + "marquis": 33530, + "stint": 33531, + "michaeljackson": 33532, + "peeled": 33533, + "menus": 33534, + "bbb": 33535, + "scare": 33536, + "email": 33537, + "wrigley": 33538, + "itis": 33539, + "fell": 33540, + "somethin": 33541, + "barra": 33542, + "edgar": 33543, + "dipping": 33544, + "puddle": 33545, + "slade": 33546, + "learner": 33547, + "jalen": 33548, + "ðŁ§IJ": 33549, + "thedaily": 33550, + "mikequindazzi": 33551, + "jux": 33552, + "iqbal": 33553, + "mckinney": 33554, + "raiser": 33555, + "efan": 33556, + "drone": 33557, + "cato": 33558, + "picket": 33559, + "crowe": 33560, + "latt": 33561, + "uko": 33562, + "giuseppe": 33563, + "hini": 33564, + "synthesi": 33565, + "pontifex": 33566, + "songwriting": 33567, + "tod": 33568, + "switches": 33569, + "dinners": 33570, + "hq": 33571, + "gabrielle": 33572, + "pensacola": 33573, + "circle": 33574, + "exposes": 33575, + "evs": 33576, + "riyadh": 33577, + "promen": 33578, + "ock": 33579, + "saj": 33580, + "citation": 33581, + "brewco": 33582, + "josi": 33583, + "epaper": 33584, + "drif": 33585, + "pointless": 33586, + "tangled": 33587, + "cripp": 33588, + "lineups": 33589, + "fairies": 33590, + "daze": 33591, + "mourn": 33592, + "bladder": 33593, + "salz": 33594, + "burundi": 33595, + "bookmark": 33596, + "thepeople": 33597, + "subsequ": 33598, + "principal": 33599, + "sker": 33600, + "courtney": 33601, + "aoki": 33602, + "racers": 33603, + "adm": 33604, + "moma": 33605, + "criticalrole": 33606, + "houn": 33607, + "shedding": 33608, + "saka": 33609, + "aceous": 33610, + "mckay": 33611, + "husbands": 33612, + "½": 33613, + "meda": 33614, + "accusations": 33615, + "rosel": 33616, + "ncis": 33617, + "witnessing": 33618, + "orama": 33619, + "gods": 33620, + "hilton": 33621, + "elman": 33622, + "ÃŃn": 33623, + "megap": 33624, + "craven": 33625, + "announcer": 33626, + "criteri": 33627, + "sheffieldissuper": 33628, + "militant": 33629, + "consul": 33630, + "hooded": 33631, + "abyss": 33632, + "bx": 33633, + "madam": 33634, + "locu": 33635, + "maryam": 33636, + "manicure": 33637, + "gratis": 33638, + "actresses": 33639, + "rosario": 33640, + "thisdayin": 33641, + "kingly": 33642, + "gnome": 33643, + "celine": 33644, + "rous": 33645, + "heel": 33646, + "lilac": 33647, + "vishal": 33648, + "abh": 33649, + "thorns": 33650, + "sls": 33651, + "neal": 33652, + "constructing": 33653, + "beren": 33654, + "slang": 33655, + "mains": 33656, + "farra": 33657, + "sarko": 33658, + "paige": 33659, + "guiller": 33660, + "lala": 33661, + "iceberg": 33662, + "noun": 33663, + "planners": 33664, + "ummm": 33665, + "ouses": 33666, + "illary": 33667, + "maan": 33668, + "boxing": 33669, + "zipper": 33670, + "srinagar": 33671, + "miguel": 33672, + "ostr": 33673, + "mpo": 33674, + "responsibly": 33675, + "lanterns": 33676, + "appliance": 33677, + "xb": 33678, + "grenade": 33679, + "neglect": 33680, + "dysle": 33681, + "hammock": 33682, + "nectar": 33683, + "witcher": 33684, + "rgv": 33685, + "dience": 33686, + "serbian": 33687, + "seeded": 33688, + "cruz": 33689, + "bish": 33690, + "sphe": 33691, + "eq": 33692, + "skyrim": 33693, + "algebra": 33694, + "philately": 33695, + "bungalow": 33696, + "geoff": 33697, + "yves": 33698, + "demanded": 33699, + "considerations": 33700, + "thevamp": 33701, + "pawankalyan": 33702, + "coded": 33703, + "gritty": 33704, + "eruption": 33705, + "seinfeld": 33706, + "unidenti": 33707, + "ëĭĪ": 33708, + "worm": 33709, + "acus": 33710, + "seung": 33711, + "dung": 33712, + "roland": 33713, + "sud": 33714, + "divisions": 33715, + "ablanc": 33716, + "shortest": 33717, + "jf": 33718, + "poun": 33719, + "plantbased": 33720, + "beto": 33721, + "tougher": 33722, + "mco": 33723, + "donet": 33724, + "markus": 33725, + "vfl": 33726, + "ðŁıł": 33727, + "opening": 33728, + "coward": 33729, + "cabernet": 33730, + "oxi": 33731, + "burlesque": 33732, + "sandra": 33733, + "sumo": 33734, + "consist": 33735, + "thot": 33736, + "cayman": 33737, + "motorola": 33738, + "gutierrez": 33739, + "dslr": 33740, + "yw": 33741, + "nobel": 33742, + "novice": 33743, + "momsdemand": 33744, + "grunge": 33745, + "spor": 33746, + "dcc": 33747, + "presses": 33748, + "slist": 33749, + "allotment": 33750, + "vocational": 33751, + "ftc": 33752, + "puja": 33753, + "loven": 33754, + "uttarak": 33755, + "tandem": 33756, + "shep": 33757, + "comedians": 33758, + "anatom": 33759, + "cantwait": 33760, + "healthyeating": 33761, + "westside": 33762, + "margins": 33763, + "chiang": 33764, + "asbestos": 33765, + "stupidity": 33766, + "problematic": 33767, + "fitbit": 33768, + ":$": 33769, + "ceilings": 33770, + "shua": 33771, + "protections": 33772, + "biotic": 33773, + "bengali": 33774, + "rests": 33775, + "biennale": 33776, + "timo": 33777, + "culmin": 33778, + "eminent": 33779, + "affection": 33780, + "unbelievably": 33781, + "individually": 33782, + "canvassing": 33783, + "whitt": 33784, + "novasco": 33785, + "chinson": 33786, + "hpe": 33787, + "gow": 33788, + "gloucestershire": 33789, + "pao": 33790, + "threshold": 33791, + "chevron": 33792, + "sine": 33793, + "wether": 33794, + "ppie": 33795, + "aquino": 33796, + "antwerp": 33797, + "âĸ¬": 33798, + "poon": 33799, + "instaf": 33800, + "equine": 33801, + "cinematography": 33802, + "nbafinals": 33803, + "valiant": 33804, + "kilkenny": 33805, + "terence": 33806, + "systemic": 33807, + "srl": 33808, + "pound": 33809, + "madeira": 33810, + "plough": 33811, + "trecht": 33812, + "mated": 33813, + "mpd": 33814, + "ransomware": 33815, + "phin": 33816, + "liqui": 33817, + "bbce": 33818, + "boomer": 33819, + "istandwith": 33820, + "conju": 33821, + "rte": 33822, + "nara": 33823, + "foolish": 33824, + "dashing": 33825, + "viernes": 33826, + "brite": 33827, + "dau": 33828, + "juniper": 33829, + "aida": 33830, + "younow": 33831, + "razer": 33832, + "dei": 33833, + "repeating": 33834, + "comforting": 33835, + "adjacent": 33836, + "eto": 33837, + "casted": 33838, + "chatur": 33839, + "muer": 33840, + "synth": 33841, + "sanitary": 33842, + "macle": 33843, + "independent": 33844, + "lawful": 33845, + "eerie": 33846, + "hor": 33847, + "ðŁĴŃ": 33848, + "amrit": 33849, + "velo": 33850, + "stationery": 33851, + "muf": 33852, + "maymay": 33853, + "contemplating": 33854, + "elaborate": 33855, + "gregor": 33856, + "dries": 33857, + "accol": 33858, + "à¸ļ": 33859, + "schwarzenegger": 33860, + "illnesses": 33861, + "daybreak": 33862, + "followback": 33863, + "collusion": 33864, + "electronic": 33865, + "jovi": 33866, + "hiroshima": 33867, + "taw": 33868, + "homec": 33869, + "micah": 33870, + "quitting": 33871, + "frosting": 33872, + "benfica": 33873, + "heli": 33874, + "sical": 33875, + "piccad": 33876, + "corporate": 33877, + "mentorship": 33878, + "youare": 33879, + "singer": 33880, + "shiva": 33881, + "rune": 33882, + "inger": 33883, + "rium": 33884, + "playable": 33885, + "doop": 33886, + "willow": 33887, + "terre": 33888, + "nip": 33889, + "atd": 33890, + "warbler": 33891, + "professionally": 33892, + "erase": 33893, + "proceed": 33894, + "pedestrians": 33895, + "mischief": 33896, + "bending": 33897, + "alaskan": 33898, + "ckett": 33899, + "mop": 33900, + "ddles": 33901, + "shutter": 33902, + "geared": 33903, + "ateneo": 33904, + "madeline": 33905, + "gations": 33906, + "osha": 33907, + "derick": 33908, + "swild": 33909, + "angry": 33910, + "patents": 33911, + "hunk": 33912, + "decreased": 33913, + "fry": 33914, + "ðŁĴĸðŁĴĸðŁĴĸ": 33915, + "salon": 33916, + "quantities": 33917, + "dario": 33918, + "nigel": 33919, + "kuma": 33920, + "jenn": 33921, + "happye": 33922, + "xxx": 33923, + "rexperience": 33924, + "pros": 33925, + "ausch": 33926, + "relessly": 33927, + "hamburger": 33928, + "fukushima": 33929, + "erne": 33930, + "statec": 33931, + "rend": 33932, + "mayfield": 33933, + "jone": 33934, + "lefty": 33935, + "bernstein": 33936, + "smil": 33937, + "generates": 33938, + "forestation": 33939, + "bandits": 33940, + "tayo": 33941, + "rca": 33942, + "acci": 33943, + "rodrigo": 33944, + "knapp": 33945, + "elovers": 33946, + "vegetation": 33947, + "ural": 33948, + "left": 33949, + "ħï¸ı": 33950, + "worldre": 33951, + "suri": 33952, + "embark": 33953, + "wson": 33954, + "bayou": 33955, + "muller": 33956, + "movers": 33957, + "ðŁķº": 33958, + "presbyter": 33959, + "lf": 33960, + "cree": 33961, + "batb": 33962, + "salam": 33963, + "demonstrations": 33964, + "anec": 33965, + "npc": 33966, + "itics": 33967, + "tography": 33968, + "reinst": 33969, + "thurst": 33970, + "tale": 33971, + "offences": 33972, + "smartcity": 33973, + "brotha": 33974, + "oftheyear": 33975, + "invaluable": 33976, + "earn": 33977, + "ðŁijıðŁı½": 33978, + "kremlin": 33979, + "grady": 33980, + "townfc": 33981, + "guernsey": 33982, + "maha": 33983, + "contagious": 33984, + "drex": 33985, + "been": 33986, + "(£": 33987, + "nativity": 33988, + "ktm": 33989, + "somerhalder": 33990, + "compounds": 33991, + "íķĺ": 33992, + "\"âĢ¦": 33993, + "afg": 33994, + "ottnews": 33995, + "hound": 33996, + "firefly": 33997, + "cilan": 33998, + "donetsk": 33999, + "volunteered": 34000, + "akira": 34001, + "èª": 34002, + "singul": 34003, + "sth": 34004, + "drowned": 34005, + "mando": 34006, + "heir": 34007, + "ðŁİīðŁİĪ": 34008, + "taxis": 34009, + "yuki": 34010, + "veld": 34011, + "kans": 34012, + "elk": 34013, + "rants": 34014, + "hashtag": 34015, + "teng": 34016, + "rog": 34017, + "aat": 34018, + "grub": 34019, + "eber": 34020, + "inindia": 34021, + "colossus": 34022, + "signi": 34023, + "soever": 34024, + "milestones": 34025, + "dero": 34026, + "differential": 34027, + "phuket": 34028, + "mastermind": 34029, + "angh": 34030, + "melani": 34031, + "broker": 34032, + "actorvijay": 34033, + "stunned": 34034, + "continuity": 34035, + "affl": 34036, + "vocal": 34037, + "perennial": 34038, + "fiancé": 34039, + "incomplete": 34040, + "hunts": 34041, + "reissue": 34042, + "dominates": 34043, + "turmeric": 34044, + "roam": 34045, + "rion": 34046, + "bagged": 34047, + "nassau": 34048, + "fut": 34049, + "xox": 34050, + "nationaltrust": 34051, + "joye": 34052, + "sano": 34053, + "hearthstone": 34054, + "disrespect": 34055, + "lees": 34056, + "hse": 34057, + "siberian": 34058, + "offee": 34059, + "restock": 34060, + "wolfgang": 34061, + "regan": 34062, + "plano": 34063, + "unwind": 34064, + "repar": 34065, + "mille": 34066, + "],": 34067, + "skull": 34068, + "fatally": 34069, + "conceptual": 34070, + "ðŁĮ²": 34071, + "fé": 34072, + "berto": 34073, + "bms": 34074, + "ua": 34075, + "magna": 34076, + "notredame": 34077, + "lete": 34078, + "laundering": 34079, + "heartwarming": 34080, + "buffett": 34081, + "goat": 34082, + "peabo": 34083, + "windmill": 34084, + "vac": 34085, + "continually": 34086, + "azalea": 34087, + "membrane": 34088, + "cancels": 34089, + "makeyourown": 34090, + "athered": 34091, + "pto": 34092, + "torpe": 34093, + "ðŁĺł": 34094, + "ðŁĴ§": 34095, + "scares": 34096, + "leaking": 34097, + "zet": 34098, + "pixels": 34099, + "aci": 34100, + "khil": 34101, + "marathi": 34102, + "ðŁĻıðŁı½": 34103, + "ula": 34104, + "tamu": 34105, + "chandigarh": 34106, + "zagre": 34107, + "aab": 34108, + "pronounced": 34109, + "aubrey": 34110, + "sander": 34111, + "punta": 34112, + "harlow": 34113, + "icelan": 34114, + "celebratory": 34115, + "sot": 34116, + "unciation": 34117, + "struly": 34118, + "mcdowell": 34119, + "deepika": 34120, + "reminders": 34121, + "mystical": 34122, + "ctc": 34123, + "chatted": 34124, + "sica": 34125, + "bargains": 34126, + "chhat": 34127, + "rubin": 34128, + "mnet": 34129, + "oilandgas": 34130, + "pelican": 34131, + "oat": 34132, + "morality": 34133, + "kour": 34134, + "ih": 34135, + "nuclear": 34136, + "gcu": 34137, + "richer": 34138, + "venezia": 34139, + "mma": 34140, + "leith": 34141, + "accompany": 34142, + "richmond": 34143, + "sportsnet": 34144, + "baahu": 34145, + "smuggling": 34146, + "mmi": 34147, + "ðŁĩ®ðŁĩª": 34148, + "twists": 34149, + "sahib": 34150, + ".....": 34151, + "ambitions": 34152, + "illo": 34153, + "historical": 34154, + "forec": 34155, + "showbiz": 34156, + "ponies": 34157, + "chasers": 34158, + "remodel": 34159, + "willing": 34160, + "princesses": 34161, + "ample": 34162, + "cushions": 34163, + "acles": 34164, + "lotr": 34165, + "dach": 34166, + "anthe": 34167, + "incorporate": 34168, + "newbury": 34169, + "kiri": 34170, + "friedrich": 34171, + "abv": 34172, + "ballers": 34173, + "albert": 34174, + "ðŁijŃ": 34175, + "leti": 34176, + "nanop": 34177, + "cide": 34178, + "analo": 34179, + "nsf": 34180, + "))))": 34181, + "griffiths": 34182, + "valenci": 34183, + "roano": 34184, + "funrun": 34185, + "babysitting": 34186, + "caday": 34187, + "entre": 34188, + "uck": 34189, + "slug": 34190, + "tical": 34191, + "thesims": 34192, + "roar": 34193, + "carney": 34194, + "gam": 34195, + "stowe": 34196, + "fid": 34197, + "bunny": 34198, + "shamrock": 34199, + "pecu": 34200, + "molina": 34201, + "gocougs": 34202, + "contributes": 34203, + "transformation": 34204, + "moy": 34205, + "vaj": 34206, + "severy": 34207, + "antioxidants": 34208, + "thirteen": 34209, + "sightseeing": 34210, + "lj": 34211, + "reversible": 34212, + "oddly": 34213, + "hookah": 34214, + "nouvel": 34215, + "halal": 34216, + "fei": 34217, + "stables": 34218, + "mult": 34219, + "hopped": 34220, + "braids": 34221, + "interchange": 34222, + "ghanaian": 34223, + "wwww": 34224, + "ethno": 34225, + "conjunction": 34226, + "agov": 34227, + "yeti": 34228, + "earthand": 34229, + "tsp": 34230, + "conserve": 34231, + "heirloom": 34232, + "metaphor": 34233, + "woof": 34234, + "torio": 34235, + "selfless": 34236, + "nwa": 34237, + "emilia": 34238, + "ylene": 34239, + "yxe": 34240, + "giar": 34241, + "moderating": 34242, + "probz": 34243, + "bfi": 34244, + "neer": 34245, + "dummy": 34246, + "hanukkah": 34247, + "webber": 34248, + "kv": 34249, + "eyebrow": 34250, + "dagger": 34251, + "sump": 34252, + "rages": 34253, + "orkney": 34254, + "tbo": 34255, + "halsey": 34256, + "assignments": 34257, + "tronic": 34258, + "scrib": 34259, + "coon": 34260, + "anwar": 34261, + "#âĢİ": 34262, + "jalape": 34263, + "florida": 34264, + "quaid": 34265, + "hawkeyes": 34266, + "âĻ¡âĻ¡": 34267, + "streetcar": 34268, + "rog": 34269, + "datlantic": 34270, + "granola": 34271, + "unchanged": 34272, + "expectation": 34273, + "Ùĩ": 34274, + "marlin": 34275, + "gummy": 34276, + "ðŁĻıðŁı¾": 34277, + "awarenessmonth": 34278, + "oilpainting": 34279, + "muth": 34280, + "perch": 34281, + "junto": 34282, + "villagers": 34283, + "morg": 34284, + "cheated": 34285, + "webcomic": 34286, + "thefuture": 34287, + "dps": 34288, + "lakings": 34289, + "mentioning": 34290, + "voor": 34291, + "identities": 34292, + "accord": 34293, + "mcgu": 34294, + "lpga": 34295, + "rumour": 34296, + "massively": 34297, + "mpls": 34298, + "healy": 34299, + "date": 34300, + "spoli": 34301, + "revisited": 34302, + "ont": 34303, + "aland": 34304, + "scrutiny": 34305, + "lakeland": 34306, + "blending": 34307, + "": 34308, + "ankara": 34309, + "jamiedor": 34310, + "metabolic": 34311, + "fences": 34312, + "anny": 34313, + "åħ": 34314, + "semicon": 34315, + "oott": 34316, + "spaceship": 34317, + "wacky": 34318, + "leta": 34319, + "apac": 34320, + "shee": 34321, + "inherit": 34322, + "dores": 34323, + "ðŁĩ¨ðŁĩ¦": 34324, + "gente": 34325, + "twick": 34326, + "rims": 34327, + "galve": 34328, + "deville": 34329, + "kingfisher": 34330, + "scorpio": 34331, + "owl": 34332, + "alar": 34333, + "varian": 34334, + "ðŁĹĵ": 34335, + "venetian": 34336, + "stardust": 34337, + "thenorth": 34338, + "qing": 34339, + "harrington": 34340, + "consulate": 34341, + "spectacle": 34342, + "hobbs": 34343, + "turks": 34344, + "greer": 34345, + "mating": 34346, + "ðŁİĢ": 34347, + "ðŁĮĢ": 34348, + "directs": 34349, + "íĭ": 34350, + "pompeo": 34351, + "voiced": 34352, + "laos": 34353, + "tzu": 34354, + "prome": 34355, + "prism": 34356, + "merc": 34357, + "fortunately": 34358, + "bcfc": 34359, + "mcdonnell": 34360, + "notsorry": 34361, + "smiled": 34362, + "tba": 34363, + "forwar": 34364, + "midterm": 34365, + "darby": 34366, + "weinstein": 34367, + "upgrading": 34368, + "wolff": 34369, + "bronco": 34370, + "cabello": 34371, + "ðŁ¥ĩ": 34372, + "fiable": 34373, + "sharpe": 34374, + "battered": 34375, + "sato": 34376, + "mythical": 34377, + "instapic": 34378, + "prepped": 34379, + "enium": 34380, + "espo": 34381, + "diaper": 34382, + "explanations": 34383, + "whopping": 34384, + "ragnar": 34385, + "peel": 34386, + "antibiotic": 34387, + "lacks": 34388, + "harrison": 34389, + "lism": 34390, + "aul": 34391, + "quail": 34392, + "martina": 34393, + "sentencing": 34394, + "scams": 34395, + "didi": 34396, + "tronics": 34397, + "ãħłãħł": 34398, + "goff": 34399, + "zain": 34400, + "paramore": 34401, + "chained": 34402, + "clinton": 34403, + "liff": 34404, + "cottages": 34405, + "emon": 34406, + "reverend": 34407, + "consumer": 34408, + "cean": 34409, + "tany": 34410, + "lumpur": 34411, + "ebay": 34412, + "stool": 34413, + "ðŁĺ»ðŁĺ»": 34414, + "tapro": 34415, + "hath": 34416, + "modernart": 34417, + "justine": 34418, + "proverb": 34419, + "appy": 34420, + "trax": 34421, + "manifest": 34422, + "ambu": 34423, + "naik": 34424, + "pepp": 34425, + "rsd": 34426, + "merchants": 34427, + "kitchener": 34428, + "shifted": 34429, + "lizz": 34430, + "âĺħâĺħâĺħâĺħ": 34431, + "âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ": 34432, + "utopia": 34433, + "tomo": 34434, + "outed": 34435, + "comers": 34436, + "chiropractic": 34437, + "bookclub": 34438, + "cindy": 34439, + "prohibition": 34440, + "seuss": 34441, + "민": 34442, + "thinkin": 34443, + "rrrr": 34444, + "gofund": 34445, + "tack": 34446, + "omb": 34447, + "catastrophic": 34448, + "lingu": 34449, + "guildford": 34450, + "botd": 34451, + "à¥ĭ": 34452, + "planter": 34453, + "^^": 34454, + "wink": 34455, + "kathmandu": 34456, + "stoppers": 34457, + "smoothies": 34458, + "reefs": 34459, + "hind": 34460, + "bellamy": 34461, + "Ħë": 34462, + "wastewater": 34463, + "voor": 34464, + "natl": 34465, + "!]": 34466, + "reel": 34467, + "yap": 34468, + "scooby": 34469, + "workspace": 34470, + "corinthians": 34471, + "blun": 34472, + "obligation": 34473, + "gbbo": 34474, + "dyson": 34475, + "cravings": 34476, + "ellington": 34477, + "dapl": 34478, + "wrexham": 34479, + "earthandclouds": 34480, + "ukrunchat": 34481, + "positioned": 34482, + "kalb": 34483, + "foursquare": 34484, + "jock": 34485, + "impending": 34486, + "evening": 34487, + "athy": 34488, + "proclaimed": 34489, + "cites": 34490, + "annapolis": 34491, + "sani": 34492, + "marth": 34493, + "irl": 34494, + "accommo": 34495, + "kaa": 34496, + "fina": 34497, + "yaa": 34498, + "disper": 34499, + "ecar": 34500, + "bhak": 34501, + "willy": 34502, + "ðŁĺĢðŁĺĢ": 34503, + "mcdermott": 34504, + "moj": 34505, + "generational": 34506, + "usaid": 34507, + "training": 34508, + "lonely": 34509, + "lores": 34510, + "impecc": 34511, + "âĢIJ": 34512, + "beavers": 34513, + "maki": 34514, + "heb": 34515, + "aapl": 34516, + "åı": 34517, + "wolverhampton": 34518, + "leaderboard": 34519, + "meu": 34520, + "cfa": 34521, + "eastern": 34522, + "hur": 34523, + "civilwar": 34524, + "ourage": 34525, + "horned": 34526, + "lehigh": 34527, + "awards": 34528, + "evident": 34529, + "gigab": 34530, + "rous": 34531, + "madel": 34532, + "robyn": 34533, + "urgently": 34534, + "kors": 34535, + "enas": 34536, + "heisman": 34537, + "bambam": 34538, + "fabian": 34539, + "fom": 34540, + "evaluating": 34541, + "assembly": 34542, + "outsourcing": 34543, + "huntsville": 34544, + "ðŁĶª": 34545, + "justified": 34546, + "cashier": 34547, + "spaper": 34548, + "buckeye": 34549, + "analytical": 34550, + "illuminati": 34551, + "autho": 34552, + "oj": 34553, + "shade": 34554, + "geelong": 34555, + "whey": 34556, + "heaton": 34557, + "terribly": 34558, + "elek": 34559, + "uncharted": 34560, + "sdlive": 34561, + "motocross": 34562, + "hermes": 34563, + "darshan": 34564, + "darlington": 34565, + "cashmere": 34566, + "gripping": 34567, + "cilantro": 34568, + "punish": 34569, + "...:": 34570, + "ðŁĴĦ": 34571, + "instance": 34572, + "deri": 34573, + "lobal": 34574, + "mukher": 34575, + "spar": 34576, + "thinker": 34577, + "fremont": 34578, + "compiled": 34579, + "colorado": 34580, + "vigne": 34581, + "smd": 34582, + "whead": 34583, + "village": 34584, + "leek": 34585, + "formulae": 34586, + "tares": 34587, + "persistence": 34588, + "??????": 34589, + "pedago": 34590, + "hez": 34591, + "alzheimers": 34592, + "vulture": 34593, + "offence": 34594, + "isgreat": 34595, + "suffra": 34596, + "kickin": 34597, + "hmmmm": 34598, + "broadway": 34599, + "ï¸ı@": 34600, + "arti": 34601, + "allison": 34602, + "endorses": 34603, + "ryu": 34604, + "lollipop": 34605, + "soybean": 34606, + "kendall": 34607, + "cera": 34608, + "invade": 34609, + "(ðŁĵ·:": 34610, + "converter": 34611, + "carpets": 34612, + "hobo": 34613, + "frit": 34614, + "peac": 34615, + "esqu": 34616, + "ernan": 34617, + "ouf": 34618, + "anil": 34619, + "differ": 34620, + "ching": 34621, + "brecht": 34622, + "spg": 34623, + "davenport": 34624, + "strava": 34625, + "severn": 34626, + "ngos": 34627, + "storians": 34628, + "fete": 34629, + "paramedic": 34630, + "jhb": 34631, + "alamo": 34632, + "sneaking": 34633, + "goldcoast": 34634, + "roofs": 34635, + "isil": 34636, + "depicted": 34637, + "projections": 34638, + "numb": 34639, + "oss": 34640, + "epi": 34641, + "glucose": 34642, + "zidane": 34643, + "infiniti": 34644, + "íĺĦ": 34645, + "ransom": 34646, + "tonics": 34647, + "falk": 34648, + "gler": 34649, + "outw": 34650, + "ress": 34651, + "weekly": 34652, + "theon": 34653, + "nole": 34654, + "ðŁĩªðŁĩº": 34655, + "volley": 34656, + "summar": 34657, + "negativity": 34658, + "samson": 34659, + "yew": 34660, + "ausvotes": 34661, + "jul": 34662, + "judy": 34663, + "fart": 34664, + "prayed": 34665, + "palate": 34666, + "multicultural": 34667, + "doubleheader": 34668, + "cyclones": 34669, + "pierre": 34670, + "ãģ¨": 34671, + "âĺłï¸ı": 34672, + "rtw": 34673, + "converting": 34674, + "wirral": 34675, + "lari": 34676, + "irrelevant": 34677, + "austinmahone": 34678, + "anche": 34679, + "yaan": 34680, + "sdf": 34681, + "$.": 34682, + "exploding": 34683, + "ultimate": 34684, + "profici": 34685, + "gofundme": 34686, + "cellence": 34687, + "epstein": 34688, + "bullied": 34689, + "septic": 34690, + "த": 34691, + "lumber": 34692, + "cuff": 34693, + "vscocam": 34694, + "plor": 34695, + "ล": 34696, + "seok": 34697, + "roto": 34698, + "venezuelan": 34699, + "sorta": 34700, + "spirited": 34701, + "danielpadilla": 34702, + "teamsisd": 34703, + "radioactive": 34704, + "icelandic": 34705, + "ðŁĴ¤": 34706, + "vere": 34707, + "accommodate": 34708, + "shipp": 34709, + "otter": 34710, + "olina": 34711, + "ego": 34712, + "sula": 34713, + "sanantonio": 34714, + "deas": 34715, + "similarities": 34716, + "âļ¾": 34717, + "yom": 34718, + "broward": 34719, + "å°": 34720, + "cancun": 34721, + "verify": 34722, + "onte": 34723, + "candlelight": 34724, + "ìłķ": 34725, + "infants": 34726, + "azam": 34727, + "ðŁĺ°": 34728, + "leven": 34729, + "unstable": 34730, + "bloomington": 34731, + "xford": 34732, + "contour": 34733, + "yp": 34734, + "innovator": 34735, + "histories": 34736, + "poy": 34737, + "lololol": 34738, + "expires": 34739, + "catalo": 34740, + "billboards": 34741, + "anab": 34742, + "elic": 34743, + "novascotia": 34744, + "faire": 34745, + "ìĿ´": 34746, + "rockwell": 34747, + "grille": 34748, + "aztec": 34749, + "johor": 34750, + "urstruly": 34751, + "firen": 34752, + "dunlop": 34753, + "idle": 34754, + "portman": 34755, + "joes": 34756, + "txhsfb": 34757, + "holm": 34758, + "chamele": 34759, + "underworld": 34760, + "loss": 34761, + "tiem": 34762, + "therapists": 34763, + "pasture": 34764, + "paste": 34765, + "ingnow": 34766, + "vulcan": 34767, + "ragon": 34768, + "larkin": 34769, + "oshi": 34770, + "hoco": 34771, + "childhood": 34772, + "umbrel": 34773, + "successor": 34774, + "kathy": 34775, + "izen": 34776, + "°ï¸ı": 34777, + "shareholders": 34778, + "olga": 34779, + "aib": 34780, + "heap": 34781, + "flaming": 34782, + "rou": 34783, + "airtel": 34784, + "ratt": 34785, + "zane": 34786, + "vow": 34787, + "thorough": 34788, + "snag": 34789, + "parth": 34790, + "unconscious": 34791, + "vey": 34792, + "newrelease": 34793, + "ghee": 34794, + "croatian": 34795, + "facilitating": 34796, + "swanson": 34797, + "astoria": 34798, + "tology": 34799, + "mastery": 34800, + "ðŁ¤ij": 34801, + "bilbao": 34802, + "troupe": 34803, + "theori": 34804, + "cheyenne": 34805, + "rott": 34806, + "shoreline": 34807, + "grasso": 34808, + "masterchef": 34809, + "+)": 34810, + "vix": 34811, + "ellenshow": 34812, + "asg": 34813, + "anak": 34814, + "kuya": 34815, + "safarilive": 34816, + "debuting": 34817, + "blum": 34818, + "listener": 34819, + "vins": 34820, + "bookshelf": 34821, + "smartcities": 34822, + "makeyourownlane": 34823, + ";;": 34824, + "ðŁIJ¯": 34825, + "rizz": 34826, + "onward": 34827, + "bulldog": 34828, + "bearish": 34829, + "viruses": 34830, + "frigh": 34831, + "linden": 34832, + "weiser": 34833, + "snt": 34834, + "gona": 34835, + "dresden": 34836, + "flanders": 34837, + "cuk": 34838, + "wheeling": 34839, + "bau": 34840, + "atuesday": 34841, + "surfers": 34842, + "swift": 34843, + "mccall": 34844, + "arbitration": 34845, + "awd": 34846, + "monc": 34847, + "bine": 34848, + "atx": 34849, + "refr": 34850, + "miro": 34851, + "posey": 34852, + "nare": 34853, + "ritter": 34854, + "âģ¦": 34855, + "playbook": 34856, + "blowout": 34857, + "sportsmanship": 34858, + "soooooo": 34859, + "malayalam": 34860, + "grims": 34861, + "burbank": 34862, + "infinity": 34863, + "sargent": 34864, + "oitnb": 34865, + "josephine": 34866, + "skipping": 34867, + "parkin": 34868, + "excursion": 34869, + "seminars": 34870, + "johar": 34871, + "partridge": 34872, + "postgame": 34873, + "llll": 34874, + "blanche": 34875, + "tempting": 34876, + "mna": 34877, + "luka": 34878, + "isers": 34879, + "toffee": 34880, + "barron": 34881, + "hemmings": 34882, + "sae": 34883, + "gohawks": 34884, + "cupid": 34885, + "limbs": 34886, + "conse": 34887, + "uncommon": 34888, + "zada": 34889, + "headshot": 34890, + "soils": 34891, + "pioneer": 34892, + "mamma": 34893, + "semitic": 34894, + "pandey": 34895, + "jamiedornan": 34896, + "splits": 34897, + "vela": 34898, + "soni": 34899, + "raff": 34900, + "tmobile": 34901, + "âŀĸ": 34902, + "prawns": 34903, + "liter": 34904, + "enjoyment": 34905, + "eggplant": 34906, + "tub": 34907, + "cultural": 34908, + "usic": 34909, + "suspicion": 34910, + "sycam": 34911, + "summed": 34912, + "madu": 34913, + "hock": 34914, + "upwards": 34915, + "eyeing": 34916, + "rive": 34917, + "assassins": 34918, + "âĤ¬": 34919, + "outfy": 34920, + "chives": 34921, + "tner": 34922, + "lais": 34923, + "porridge": 34924, + "saddest": 34925, + "wcc": 34926, + "vicki": 34927, + "snails": 34928, + "bizitalk": 34929, + "millan": 34930, + "ðŁĮį": 34931, + "samoa": 34932, + "jing": 34933, + "mikey": 34934, + "guj": 34935, + "chelms": 34936, + "eligibility": 34937, + "armada": 34938, + "throp": 34939, + "surgeries": 34940, + "ãĤ¿": 34941, + "mohawk": 34942, + "exits": 34943, + "mem": 34944, + "islington": 34945, + "cme": 34946, + "landfill": 34947, + "kaitlyn": 34948, + "ðŁİ¼": 34949, + "combinations": 34950, + "tomorrowland": 34951, + "verb": 34952, + "cora": 34953, + "precisely": 34954, + "naom": 34955, + "ðŁĨķ": 34956, + "shrink": 34957, + "softly": 34958, + "mercede": 34959, + "mandel": 34960, + "poodle": 34961, + "ballerina": 34962, + "soph": 34963, + "juxta": 34964, + "yat": 34965, + "aryan": 34966, + "hesitate": 34967, + "lowered": 34968, + "gular": 34969, + "dungeonsand": 34970, + "ronan": 34971, + "myri": 34972, + "spf": 34973, + "menopau": 34974, + "grasp": 34975, + "pathi": 34976, + "feasi": 34977, + "flaw": 34978, + "shistory": 34979, + "steward": 34980, + "ggle": 34981, + "fayre": 34982, + "clique": 34983, + "credibility": 34984, + "yog": 34985, + "section": 34986, + "musko": 34987, + "seville": 34988, + "nott": 34989, + "calm": 34990, + "mateo": 34991, + "indicted": 34992, + "fiba": 34993, + "byl": 34994, + "lino": 34995, + "ukin": 34996, + "!!#": 34997, + "enigma": 34998, + "sirius": 34999, + "busc": 35000, + "ðŁįĬ": 35001, + "mackerel": 35002, + "psalms": 35003, + "aat": 35004, + "tomorrowspaper": 35005, + "ðŁĺĸ": 35006, + "pfc": 35007, + "...........": 35008, + "shrek": 35009, + "mullet": 35010, + "osh": 35011, + "dangerously": 35012, + "immensely": 35013, + "amur": 35014, + "ðŁįĤ": 35015, + "propor": 35016, + "sya": 35017, + "londonmarathon": 35018, + "above": 35019, + "obligatory": 35020, + "prov": 35021, + "racha": 35022, + "alexis": 35023, + "primary": 35024, + "shh": 35025, + "ethernet": 35026, + "dstv": 35027, + "cougar": 35028, + "unlucky": 35029, + "nil": 35030, + "steakhouse": 35031, + "mela": 35032, + "fcbayern": 35033, + "causeway": 35034, + "catherine": 35035, + "fluorescent": 35036, + "nxt": 35037, + "tokyo": 35038, + "ausp": 35039, + "relegation": 35040, + "quizz": 35041, + "shoreditch": 35042, + "proudtobe": 35043, + "promos": 35044, + "interacting": 35045, + "homebrew": 35046, + "daesh": 35047, + "wpg": 35048, + "steadily": 35049, + "provinces": 35050, + "ballots": 35051, + "iah": 35052, + "alto": 35053, + "<<<": 35054, + "youu": 35055, + "riley": 35056, + "preference": 35057, + "traverse": 35058, + "incense": 35059, + "ammunition": 35060, + "hodges": 35061, + "#@": 35062, + "hailstate": 35063, + "tartan": 35064, + "witchcraft": 35065, + "ventilation": 35066, + "libertarian": 35067, + "!âĢ¦": 35068, + "owes": 35069, + "%!": 35070, + "ongchang": 35071, + "brushing": 35072, + "leic": 35073, + "fiber": 35074, + "underattack": 35075, + "download": 35076, + "expir": 35077, + "hyo": 35078, + "pompey": 35079, + "mcbride": 35080, + "yag": 35081, + "stree": 35082, + "combat": 35083, + "tending": 35084, + "aira": 35085, + "guggen": 35086, + "abra": 35087, + "inna": 35088, + "flips": 35089, + "awal": 35090, + "mach": 35091, + "dollar": 35092, + "inspirations": 35093, + "zum": 35094, + "odu": 35095, + "itty": 35096, + "videogame": 35097, + "aquaman": 35098, + "haru": 35099, + "belfast": 35100, + "jeb": 35101, + "butch": 35102, + "usgs": 35103, + "calculus": 35104, + "goyal": 35105, + "morgen": 35106, + "xfinity": 35107, + "standup": 35108, + "contracep": 35109, + "sabre": 35110, + "nabe": 35111, + "insecure": 35112, + "generously": 35113, + "epitome": 35114, + "lw": 35115, + "tca": 35116, + "narratives": 35117, + "donnell": 35118, + "pandas": 35119, + "bergh": 35120, + "tut": 35121, + "keral": 35122, + "felicity": 35123, + "brampton": 35124, + "quintet": 35125, + "nomore": 35126, + "ðŁĶij": 35127, + "loi": 35128, + "alhamdulil": 35129, + "ðŁĶ¥ðŁĶĹ": 35130, + "stoner": 35131, + "shawl": 35132, + "clinical": 35133, + "brendan": 35134, + "gone": 35135, + "flawed": 35136, + "trippy": 35137, + "jg": 35138, + "allocation": 35139, + "poaching": 35140, + "vevo": 35141, + "mocks": 35142, + "leftist": 35143, + "bonuses": 35144, + "condemned": 35145, + "ability": 35146, + "stating": 35147, + "microbiome": 35148, + "biologist": 35149, + "foryou": 35150, + "wahlberg": 35151, + "ssor": 35152, + "iftar": 35153, + "wul": 35154, + "ÑĦоÑĤ": 35155, + "pomer": 35156, + "meme": 35157, + "verte": 35158, + "trell": 35159, + "trait": 35160, + "inlet": 35161, + "hormones": 35162, + "deliberately": 35163, + "villar": 35164, + "battleship": 35165, + "pbl": 35166, + "twenti": 35167, + "hokies": 35168, + "dalail": 35169, + "saya": 35170, + "mayfair": 35171, + "hans": 35172, + "diets": 35173, + "⾨⾨": 35174, + "odin": 35175, + "hotspur": 35176, + "papi": 35177, + "kana": 35178, + "kamp": 35179, + "finna": 35180, + "flotus": 35181, + "tians": 35182, + "unicorns": 35183, + "tribeca": 35184, + "changers": 35185, + "foreground": 35186, + "outa": 35187, + "invaders": 35188, + "gettys": 35189, + "tomorrowspaperstoday": 35190, + "macmillan": 35191, + "handwritten": 35192, + "wfp": 35193, + "ude": 35194, + "stateof": 35195, + "based": 35196, + "âĺģï¸ı": 35197, + "casm": 35198, + "psyched": 35199, + "historians": 35200, + "fold": 35201, + "dda": 35202, + "aggrav": 35203, + "pans": 35204, + "greenway": 35205, + "ausv": 35206, + "ðŁĺ¶": 35207, + "shraddha": 35208, + "index": 35209, + "besti": 35210, + "zimmer": 35211, + "tness": 35212, + "eyeshadow": 35213, + "otte": 35214, + "gots": 35215, + "distributing": 35216, + "promin": 35217, + "yol": 35218, + "acea": 35219, + "tramrahim": 35220, + "hooper": 35221, + "supreme": 35222, + "jammin": 35223, + "intuitive": 35224, + "qualifications": 35225, + "slim": 35226, + "siddi": 35227, + "jayne": 35228, + "tripping": 35229, + "gtx": 35230, + "puns": 35231, + "emanuel": 35232, + "omg": 35233, + "midsummer": 35234, + "into": 35235, + "succulent": 35236, + "rien": 35237, + "newmexico": 35238, + "oor": 35239, + "hooking": 35240, + "inf": 35241, + "ðŁ¤Ŀ": 35242, + "flirting": 35243, + "nahi": 35244, + "gfriend": 35245, + "tps": 35246, + "helix": 35247, + "zs": 35248, + "onie": 35249, + "ctf": 35250, + "kris": 35251, + "irresistible": 35252, + "flap": 35253, + "ðŁijıðŁı»ðŁijıðŁı»": 35254, + "uswnt": 35255, + "rud": 35256, + "ramps": 35257, + "pinoy": 35258, + "otw": 35259, + "lolz": 35260, + "lowering": 35261, + "favorite": 35262, + "tmc": 35263, + "phrases": 35264, + "hermi": 35265, + "averaging": 35266, + "embr": 35267, + "beno": 35268, + "estuary": 35269, + "sleeve": 35270, + "ribbons": 35271, + "tash": 35272, + "ู": 35273, + "xf": 35274, + "awgs": 35275, + "sunited": 35276, + "breweries": 35277, + "anirud": 35278, + "punches": 35279, + "oldie": 35280, + "ipads": 35281, + "wifey": 35282, + "landlords": 35283, + "dji": 35284, + "gunner": 35285, + "íķ´": 35286, + "texan": 35287, + "exop": 35288, + "cassandra": 35289, + "soff": 35290, + "ðŁļ«": 35291, + "ighton": 35292, + "bakers": 35293, + "awarenessweek": 35294, + "vall": 35295, + "earp": 35296, + "btsbbmas": 35297, + "apologizes": 35298, + "âļĵï¸ı": 35299, + "wasps": 35300, + "statesman": 35301, + "snatch": 35302, + "watchdog": 35303, + "rafi": 35304, + "afterparty": 35305, + "spike": 35306, + "jer": 35307, + "periph": 35308, + "rnc": 35309, + "mull": 35310, + "leen": 35311, + "shies": 35312, + "lieu": 35313, + "urstrulymahesh": 35314, + "merton": 35315, + "desai": 35316, + "shif": 35317, + "ðŁĮ±": 35318, + "pedic": 35319, + "gosling": 35320, + "arranging": 35321, + "wwg": 35322, + "geny": 35323, + "youuu": 35324, + "netflix": 35325, + "ettes": 35326, + "kwi": 35327, + "bernardino": 35328, + "amiga": 35329, + "ب": 35330, + "kashmiri": 35331, + "tings": 35332, + "emeritus": 35333, + "decat": 35334, + "abdomin": 35335, + "dci": 35336, + "phases": 35337, + "djan": 35338, + "beam": 35339, + "opry": 35340, + "ished": 35341, + "theellenshow": 35342, + "thest": 35343, + "habitats": 35344, + "toons": 35345, + "mclaughlin": 35346, + "ripper": 35347, + "microbiology": 35348, + "talaga": 35349, + "clueless": 35350, + "ssu": 35351, + "croche": 35352, + "bromance": 35353, + "longevity": 35354, + "zagreb": 35355, + "prevented": 35356, + "trave": 35357, + "spoilt": 35358, + "darryl": 35359, + "migraine": 35360, + "alcat": 35361, + "dddd": 35362, + "viv": 35363, + "serpent": 35364, + "mattel": 35365, + "jama": 35366, + "conquest": 35367, + "îĦ": 35368, + "samsung": 35369, + "presbyterian": 35370, + "ketch": 35371, + "firefox": 35372, + "motif": 35373, + "lec": 35374, + "chopping": 35375, + "cherno": 35376, + "jann": 35377, + "ðŁIJ°": 35378, + "prolon": 35379, + "wakeup": 35380, + "convergence": 35381, + "merseyside": 35382, + "heartbroken": 35383, + "looming": 35384, + "hallucin": 35385, + "maize": 35386, + "communism": 35387, + "moh": 35388, + "twitterstorians": 35389, + "sergey": 35390, + "reseller": 35391, + "favorable": 35392, + "edgy": 35393, + "reiter": 35394, + "malaga": 35395, + "liveme": 35396, + "kahn": 35397, + "pulsion": 35398, + "bigg": 35399, + "kimkardashian": 35400, + "atio": 35401, + "tyranny": 35402, + "ruption": 35403, + "qant": 35404, + "proven": 35405, + "byz": 35406, + "pushaw": 35407, + "kristin": 35408, + "eer": 35409, + "tardis": 35410, + "riz": 35411, + "awaken": 35412, + "miko": 35413, + "undocumented": 35414, + "pathfinder": 35415, + "indirect": 35416, + "resembles": 35417, + "hler": 35418, + "concealed": 35419, + "scandal": 35420, + "reim": 35421, + "dnb": 35422, + "critters": 35423, + "attendant": 35424, + "apprenticeships": 35425, + "aau": 35426, + "screamed": 35427, + "lsu": 35428, + "fah": 35429, + "harbour": 35430, + "edd": 35431, + "batsman": 35432, + "liss": 35433, + "misha": 35434, + "spaniel": 35435, + "itf": 35436, + "advancement": 35437, + "fac": 35438, + "closeup": 35439, + "cecilia": 35440, + "medic": 35441, + "narcissi": 35442, + "lavish": 35443, + "giac": 35444, + "mays": 35445, + "leit": 35446, + "winewednesday": 35447, + "pushaward": 35448, + "letto": 35449, + "currents": 35450, + "bugatti": 35451, + "outine": 35452, + "wj": 35453, + "undo": 35454, + "lerosis": 35455, + "devotional": 35456, + "ðŁij«": 35457, + "onna": 35458, + "faisal": 35459, + "sauna": 35460, + "himachal": 35461, + "amii": 35462, + "à®®": 35463, + "dizzy": 35464, + "screenwriting": 35465, + "phx": 35466, + "spn": 35467, + "icki": 35468, + "agirl": 35469, + "fishes": 35470, + "wbz": 35471, + "pim": 35472, + "boar": 35473, + "acid": 35474, + "!..": 35475, + "rockefeller": 35476, + "nga": 35477, + "drastically": 35478, + "simplify": 35479, + "drumming": 35480, + "autumnal": 35481, + "gurmee": 35482, + "lorde": 35483, + "joann": 35484, + "giveup": 35485, + "bour": 35486, + "amura": 35487, + "derland": 35488, + "simpler": 35489, + "watson": 35490, + "trident": 35491, + "concordia": 35492, + "bellum": 35493, + "brek": 35494, + "dumplings": 35495, + "vion": 35496, + "dungeonsanddragons": 35497, + "spri": 35498, + "ascension": 35499, + "wildatlantic": 35500, + "ust": 35501, + "robins": 35502, + "legion": 35503, + "insist": 35504, + "jaro": 35505, + "guess": 35506, + "sob": 35507, + "bighit": 35508, + "poolside": 35509, + "negotiating": 35510, + "mcgill": 35511, + "bild": 35512, + "technicians": 35513, + "mitigation": 35514, + "ajaydevgn": 35515, + "bto": 35516, + "anten": 35517, + "cosmopolitan": 35518, + "ðŁĺĬðŁĺĬðŁĺĬðŁĺĬ": 35519, + "patrioti": 35520, + "temper": 35521, + "promenade": 35522, + "navajo": 35523, + "namm": 35524, + "wrinkles": 35525, + "dcfc": 35526, + "leach": 35527, + "brunette": 35528, + "rf": 35529, + "coutinho": 35530, + "alti": 35531, + "traditionally": 35532, + "optome": 35533, + "naz": 35534, + "accordingly": 35535, + "recard": 35536, + "deets": 35537, + "swell": 35538, + "posure": 35539, + "whitening": 35540, + "stranger": 35541, + "illion": 35542, + "hereford": 35543, + "uwu": 35544, + "robber": 35545, + "cotswolds": 35546, + "clen": 35547, + "gorge": 35548, + "namaste": 35549, + "relish": 35550, + "griff": 35551, + "adrenaline": 35552, + "blasio": 35553, + "vale": 35554, + "ê²": 35555, + "tolerate": 35556, + "railminindia": 35557, + "jensen": 35558, + "hoven": 35559, + "ellu": 35560, + "obsole": 35561, + "eisenhower": 35562, + "unidentified": 35563, + "thanniversary": 35564, + "bodyguard": 35565, + "د": 35566, + "idge": 35567, + "schal": 35568, + "stockport": 35569, + "sni": 35570, + "retaining": 35571, + "popo": 35572, + "pixie": 35573, + "olithic": 35574, + "kier": 35575, + "hajj": 35576, + "saz": 35577, + "corbin": 35578, + "!!!!!!!!!!": 35579, + "vit": 35580, + "megat": 35581, + "deh": 35582, + "circuit": 35583, + "affleck": 35584, + "theoretical": 35585, + "hopeless": 35586, + "uab": 35587, + "slump": 35588, + "bice": 35589, + "jammed": 35590, + "letstalk": 35591, + "cani": 35592, + "sideways": 35593, + "labyrinth": 35594, + "refs": 35595, + "hahn": 35596, + "jared": 35597, + "ðŁį¹": 35598, + "jambo": 35599, + "phyl": 35600, + "enhancement": 35601, + "ctr": 35602, + "fullest": 35603, + "seye": 35604, + "doba": 35605, + "choic": 35606, + "yos": 35607, + "cbj": 35608, + "andré": 35609, + "rewatch": 35610, + "prima": 35611, + "doctrine": 35612, + "forgets": 35613, + "uhm": 35614, + "around": 35615, + "ule": 35616, + "artlovers": 35617, + "shiraz": 35618, + "harth": 35619, + "extor": 35620, + "Å¡": 35621, + "unexpectedly": 35622, + "elius": 35623, + "yx": 35624, + "emmy": 35625, + "seac": 35626, + "ðŁijĩðŁijĩðŁijĩ": 35627, + "corrected": 35628, + "combu": 35629, + "womanc": 35630, + "cough": 35631, + "whatson": 35632, + "publishes": 35633, + "diversity": 35634, + "backbone": 35635, + "lockdown": 35636, + "mesmerizing": 35637, + "norte": 35638, + "mab": 35639, + "designer": 35640, + "íģ": 35641, + "ragh": 35642, + "molecules": 35643, + "getoutside": 35644, + "thebeatles": 35645, + "semiconduc": 35646, + "nacho": 35647, + "lunes": 35648, + "hammers": 35649, + "sultan": 35650, + "oon": 35651, + "feren": 35652, + "attach": 35653, + "arqu": 35654, + "uttarakhand": 35655, + "sash": 35656, + ";-": 35657, + "tread": 35658, + "iko": 35659, + "arthur": 35660, + "scandinavian": 35661, + "ration": 35662, + "gael": 35663, + "chargeable": 35664, + "fishy": 35665, + "vma": 35666, + "handbags": 35667, + "chara": 35668, + "ayne": 35669, + "defam": 35670, + "settlers": 35671, + "qadri": 35672, + "palais": 35673, + "inwx": 35674, + "apocalyptic": 35675, + "pooja": 35676, + "aes": 35677, + "atories": 35678, + "proofing": 35679, + "nlp": 35680, + "tsla": 35681, + "vina": 35682, + "lido": 35683, + "deephouse": 35684, + "informatics": 35685, + "vv": 35686, + "ppings": 35687, + "diss": 35688, + "ï": 35689, + "uhuru": 35690, + "stony": 35691, + "betrayed": 35692, + "baff": 35693, + "myra": 35694, + "aspen": 35695, + "allowance": 35696, + "tamara": 35697, + "cif": 35698, + "corbett": 35699, + "serge": 35700, + "digo": 35701, + "ambigu": 35702, + "painters": 35703, + "pcr": 35704, + "pca": 35705, + "noms": 35706, + "loft": 35707, + "vee": 35708, + "opendata": 35709, + "ðŁIJ±": 35710, + "alexandre": 35711, + "identifies": 35712, + "fantasyfootball": 35713, + "reproduction": 35714, + "bromley": 35715, + "wareagle": 35716, + "mmer": 35717, + "pss": 35718, + "cues": 35719, + "ayat": 35720, + "hutchinson": 35721, + "sarac": 35722, + "jackman": 35723, + "irah": 35724, + "apink": 35725, + "cols": 35726, + "aussies": 35727, + "execs": 35728, + "dayton": 35729, + "ðŁĻĨ": 35730, + "imv": 35731, + "haram": 35732, + "chuckle": 35733, + "authenticity": 35734, + "ardo": 35735, + "incubator": 35736, + "ส": 35737, + "photoshopped": 35738, + "embraced": 35739, + "fightfor": 35740, + "gorman": 35741, + "zzzz": 35742, + "scholastic": 35743, + "crisps": 35744, + "teapo": 35745, + "midnight": 35746, + "gaine": 35747, + "collier": 35748, + "sate": 35749, + "dette": 35750, + "åŃ": 35751, + "imagine": 35752, + "iff": 35753, + "twili": 35754, + "ification": 35755, + "teatro": 35756, + "norma": 35757, + "esur": 35758, + "emergencies": 35759, + "riseup": 35760, + "ringer": 35761, + "hassle": 35762, + "caitlyn": 35763, + "tranquil": 35764, + "versa": 35765, + "seb": 35766, + "overlook": 35767, + "gini": 35768, + "bogo": 35769, + "sere": 35770, + "mayne": 35771, + "henrik": 35772, + "contaminated": 35773, + "rhapsody": 35774, + "proportion": 35775, + "wildatlanticway": 35776, + "âģ©.": 35777, + "organisers": 35778, + "trane": 35779, + "standard": 35780, + "sperm": 35781, + "launcher": 35782, + "ricci": 35783, + "herts": 35784, + "paperwork": 35785, + "showcased": 35786, + "meryl": 35787, + "pena": 35788, + "pimp": 35789, + "disastrous": 35790, + "^.^": 35791, + "phara": 35792, + "xis": 35793, + "frontal": 35794, + "swirl": 35795, + "spills": 35796, + "swagger": 35797, + "smartwatch": 35798, + "sizzling": 35799, + "saviour": 35800, + "catar": 35801, + "bbcr": 35802, + "refurbishment": 35803, + "dris": 35804, + "citroen": 35805, + "absorb": 35806, + "patriotism": 35807, + "illeg": 35808, + "chromo": 35809, + "freshers": 35810, + "rus": 35811, + "limiting": 35812, + "efish": 35813, + "downed": 35814, + "mandir": 35815, + "hazelnut": 35816, + "pall": 35817, + "macon": 35818, + "disappearing": 35819, + "qualifies": 35820, + "boon": 35821, + "barracks": 35822, + "amine": 35823, + "gendere": 35824, + "ðŁļĺ": 35825, + "jes": 35826, + "ãĥŃ": 35827, + "quito": 35828, + "middleweight": 35829, + "schau": 35830, + "quadru": 35831, + "aciones": 35832, + "limitless": 35833, + "ðŁijĮðŁı½": 35834, + "chman": 35835, + "arav": 35836, + "regulators": 35837, + "itup": 35838, + "battersea": 35839, + "milford": 35840, + "gz": 35841, + "ticking": 35842, + "ghou": 35843, + "crushes": 35844, + "tutu": 35845, + "dreadful": 35846, + "famine": 35847, + "forchange": 35848, + "dalailama": 35849, + "ðŁĴį": 35850, + "whitaker": 35851, + "hashmi": 35852, + "hus": 35853, + "vod": 35854, + "bette": 35855, + "aaah": 35856, + "isoo": 35857, + "ðŁ¥Ī": 35858, + "haar": 35859, + "laine": 35860, + "bv": 35861, + "allday": 35862, + "sprout": 35863, + "indiegames": 35864, + "freebie": 35865, + "greeks": 35866, + "butler": 35867, + "illin": 35868, + "haal": 35869, + "wareness": 35870, + "sima": 35871, + "publichealth": 35872, + "gama": 35873, + "waa": 35874, + "oung": 35875, + "goooo": 35876, + "okinawa": 35877, + "offenders": 35878, + "impose": 35879, + "hoc": 35880, + "youngster": 35881, + "storyteller": 35882, + "scap": 35883, + "fighter": 35884, + "+,": 35885, + "whites": 35886, + "musicmonday": 35887, + "reza": 35888, + "goducks": 35889, + "bria": 35890, + "mium": 35891, + "casper": 35892, + "crumbs": 35893, + "aad": 35894, + "martialarts": 35895, + "chp": 35896, + "rigged": 35897, + "tng": 35898, + "harvested": 35899, + "sak": 35900, + "dojo": 35901, + "millwall": 35902, + "bnw": 35903, + "ocd": 35904, + "historyof": 35905, + "tmr": 35906, + "sirens": 35907, + "fanci": 35908, + "caregivers": 35909, + "vira": 35910, + "soni": 35911, + "recurring": 35912, + "acknowledged": 35913, + "ðŁıŁ": 35914, + "ophile": 35915, + "bucky": 35916, + "stressing": 35917, + "rook": 35918, + "digger": 35919, + "vival": 35920, + "sando": 35921, + "fleet": 35922, + "siers": 35923, + "selcaday": 35924, + "refreshed": 35925, + "antifa": 35926, + "aque": 35927, + "polo": 35928, + "disappearance": 35929, + "demb": 35930, + "âĮļï¸ı": 35931, + "rented": 35932, + "berger": 35933, + "gmb": 35934, + "cula": 35935, + "ssal": 35936, + "goody": 35937, + "uhh": 35938, + "marcelo": 35939, + "wanna": 35940, + "software": 35941, + "shopsmall": 35942, + "turtle": 35943, + "tomas": 35944, + "frisco": 35945, + "ðŁĺįðŁĴķ": 35946, + "jimenez": 35947, + "csu": 35948, + "dayz": 35949, + "ando": 35950, + "wynne": 35951, + "choreographer": 35952, + "cervical": 35953, + "trailblazers": 35954, + "edg": 35955, + "zendaya": 35956, + "travelblog": 35957, + "els": 35958, + "wholesome": 35959, + "cog": 35960, + "labout": 35961, + "arney": 35962, + "delle": 35963, + "suisse": 35964, + "masi": 35965, + "inese": 35966, + "ombe": 35967, + "fiddle": 35968, + "reclaim": 35969, + "pau": 35970, + "watcher": 35971, + "slain": 35972, + "berty": 35973, + "optimum": 35974, + "elites": 35975, + "minis": 35976, + "turkey": 35977, + "patrols": 35978, + "gerard": 35979, + "aureli": 35980, + "wildly": 35981, + "waltz": 35982, + "brgy": 35983, + "wob": 35984, + "crest": 35985, + "+++": 35986, + "vez": 35987, + "frosted": 35988, + "davido": 35989, + "thex": 35990, + "paramedics": 35991, + "pinto": 35992, + "hank": 35993, + "dupont": 35994, + "urg": 35995, + "fostering": 35996, + "micropoetry": 35997, + "spectre": 35998, + "---->": 35999, + "neuro": 36000, + "frida": 36001, + "musical": 36002, + "galveston": 36003, + "effic": 36004, + "scape": 36005, + "palazzo": 36006, + "thall": 36007, + "provisional": 36008, + "pjs": 36009, + "aure": 36010, + "ðŁĶľ": 36011, + "mamamoo": 36012, + "kitties": 36013, + "cree": 36014, + "wak": 36015, + "loool": 36016, + "lupus": 36017, + "cnblue": 36018, + "ú": 36019, + "ðŁİ¬": 36020, + "raced": 36021, + "trose": 36022, + "omas": 36023, + "stride": 36024, + "coors": 36025, + "⤵ï¸ı": 36026, + "incomparable": 36027, + "cyril": 36028, + "broader": 36029, + "areclipse": 36030, + "ðŁįĶ": 36031, + "interval": 36032, + "tiru": 36033, + "coworking": 36034, + "waco": 36035, + "aham": 36036, + "abee": 36037, + "flourish": 36038, + "thetimes": 36039, + "olini": 36040, + "kickboxing": 36041, + "lucer": 36042, + "atla": 36043, + "asun": 36044, + "casserole": 36045, + "miaw": 36046, + "lobbying": 36047, + "janice": 36048, + "cirque": 36049, + "reflex": 36050, + "leary": 36051, + "sanatomy": 36052, + "tempest": 36053, + "semb": 36054, + "murdering": 36055, + "usav": 36056, + "robo": 36057, + "onet": 36058, + "pcc": 36059, + "natives": 36060, + "lifeof": 36061, + "saha": 36062, + "ruthless": 36063, + "relates": 36064, + "appetizer": 36065, + "pyeongchang": 36066, + "nord": 36067, + "eru": 36068, + "athing": 36069, + "ugly": 36070, + "plying": 36071, + "brance": 36072, + "organise": 36073, + "kendra": 36074, + "dato": 36075, + "cheeses": 36076, + "parma": 36077, + "burnout": 36078, + "astra": 36079, + "pretoria": 36080, + "adjustment": 36081, + "uku": 36082, + "slo": 36083, + "liken": 36084, + "favors": 36085, + "clive": 36086, + "beets": 36087, + "snowdonia": 36088, + "gotv": 36089, + "syn": 36090, + "openhouse": 36091, + "pani": 36092, + "portrayed": 36093, + "slated": 36094, + "mecca": 36095, + "renal": 36096, + "supportsmallstreamers": 36097, + "staffs": 36098, + "dao": 36099, + "biker": 36100, + "viktor": 36101, + "titus": 36102, + "admired": 36103, + "ðŁĵ±": 36104, + "hurrican": 36105, + "heats": 36106, + "glory": 36107, + "photogenic": 36108, + "meri": 36109, + "depor": 36110, + "burnham": 36111, + "orangu": 36112, + "djing": 36113, + "impressionism": 36114, + "ignition": 36115, + "cai": 36116, + "wynn": 36117, + "depe": 36118, + "coveted": 36119, + "collagen": 36120, + "saus": 36121, + "ornam": 36122, + "administrators": 36123, + "sson": 36124, + "nhpolitics": 36125, + "hahahahahahahaha": 36126, + "aspirations": 36127, + "rgb": 36128, + "swollen": 36129, + "sowe": 36130, + "scr": 36131, + "divergent": 36132, + "houghton": 36133, + "hanoi": 36134, + "dory": 36135, + "niki": 36136, + "landry": 36137, + "bcci": 36138, + "ðŁijĮðŁijĮ": 36139, + "ismail": 36140, + "tripod": 36141, + "herd": 36142, + "bhatt": 36143, + "dressage": 36144, + "tabby": 36145, + "inguish": 36146, + "huron": 36147, + "à³į": 36148, + "Ãł": 36149, + "todas": 36150, + "evangelical": 36151, + "chords": 36152, + "stjohn": 36153, + "sloppy": 36154, + "martyr": 36155, + "facebook": 36156, + "alight": 36157, + "sensei": 36158, + "kathniel": 36159, + "rites": 36160, + "zione": 36161, + "uo": 36162, + "revelations": 36163, + "weightlifting": 36164, + "pano": 36165, + "ncwx": 36166, + "acton": 36167, + "à®ķ": 36168, + "ز": 36169, + "soma": 36170, + "à¸Ĺ": 36171, + "respecting": 36172, + "marche": 36173, + "foreman": 36174, + "betty": 36175, + "kik": 36176, + "shibu": 36177, + "poon": 36178, + "argyle": 36179, + "kswx": 36180, + "etz": 36181, + "marbella": 36182, + "brackets": 36183, + "standby": 36184, + "fireside": 36185, + "defiance": 36186, + "vex": 36187, + "britannia": 36188, + "inhabit": 36189, + "appoint": 36190, + "piyush": 36191, + "leash": 36192, + "sciento": 36193, + "flask": 36194, + "senna": 36195, + ">:": 36196, + "atroc": 36197, + "sanderson": 36198, + "idlib": 36199, + "dhanush": 36200, + "ðŁĺĻ": 36201, + "enthr": 36202, + "hitch": 36203, + "dedly": 36204, + "alley": 36205, + "dork": 36206, + "mondo": 36207, + "cuddly": 36208, + "missin": 36209, + "yesss": 36210, + "nighting": 36211, + "jpn": 36212, + "wary": 36213, + "umpire": 36214, + "maz": 36215, + "ê³": 36216, + "babs": 36217, + "ĭãģ": 36218, + "stanford": 36219, + "possessed": 36220, + "exceeded": 36221, + "ðŁĶ¶": 36222, + "wallart": 36223, + "trap": 36224, + "jil": 36225, + "hibis": 36226, + "spying": 36227, + "scribe": 36228, + "khalil": 36229, + "translator": 36230, + "lumb": 36231, + "dized": 36232, + "chc": 36233, + "supervision": 36234, + "shutter": 36235, + "jag": 36236, + "_*": 36237, + "yesterdays": 36238, + "msf": 36239, + "hihi": 36240, + "gonzaga": 36241, + "gillespie": 36242, + "vivek": 36243, + "ecstatic": 36244, + "thismorning": 36245, + "chus": 36246, + "edes": 36247, + "stoned": 36248, + "bees": 36249, + "ðŁĩ¹ðŁĩ": 36250, + "turin": 36251, + "hover": 36252, + "atrics": 36253, + "stern": 36254, + "samheughan": 36255, + "autism": 36256, + "miya": 36257, + "eyewitness": 36258, + "writings": 36259, + "traveltips": 36260, + "chutney": 36261, + "pxrtg": 36262, + "kenyans": 36263, + "mystic": 36264, + "krit": 36265, + "/$": 36266, + "redhead": 36267, + "worldly": 36268, + "amus": 36269, + "opla": 36270, + "leve": 36271, + "gabbana": 36272, + "seen": 36273, + "oclock": 36274, + "ganga": 36275, + "keenan": 36276, + "scent": 36277, + "oldies": 36278, + "gogreen": 36279, + "cornerstone": 36280, + "comply": 36281, + "concours": 36282, + "ðŁİ¶ðŁİ¶": 36283, + "haan": 36284, + "confis": 36285, + "awson": 36286, + "cleop": 36287, + "îĢ": 36288, + "suzu": 36289, + "sauté": 36290, + "algar": 36291, + "subscriber": 36292, + "esteemed": 36293, + "ãĤ¤ãĥ": 36294, + "worthwhile": 36295, + "melrose": 36296, + "flock": 36297, + "brightly": 36298, + "violinist": 36299, + "pere": 36300, + "slipping": 36301, + "andco": 36302, + "sigh": 36303, + "havan": 36304, + "culo": 36305, + "msa": 36306, + "fibrosis": 36307, + "matilda": 36308, + "rafting": 36309, + "award": 36310, + "ëª": 36311, + "mmmm": 36312, + "geaux": 36313, + "steiner": 36314, + "sinn": 36315, + "helpers": 36316, + "beetles": 36317, + "aimee": 36318, + "taiwan": 36319, + "pistachio": 36320, + "macbeth": 36321, + "mzan": 36322, + "descendants": 36323, + "onsale": 36324, + "inr": 36325, + "ilm": 36326, + "grouse": 36327, + "saig": 36328, + "mow": 36329, + "bigre": 36330, + "adjustments": 36331, + "tula": 36332, + "mathew": 36333, + "translates": 36334, + "muh": 36335, + "bollah": 36336, + "ðŁĴĽðŁĴĻ": 36337, + "amores": 36338, + "abouts": 36339, + "bombshell": 36340, + "blaster": 36341, + "xavi": 36342, + "sns": 36343, + "kroger": 36344, + "gather": 36345, + "eradic": 36346, + "daft": 36347, + "chemo": 36348, + "benches": 36349, + "ðŁĩ©ðŁĩ": 36350, + "utv": 36351, + "oura": 36352, + "nko": 36353, + "gatorade": 36354, + "biafra": 36355, + "okstate": 36356, + "imdanielpadilla": 36357, + "domains": 36358, + "openingday": 36359, + "kiddo": 36360, + "doi": 36361, + "rice": 36362, + "daycare": 36363, + "macmillan": 36364, + "bathurst": 36365, + "cheerleading": 36366, + "ðŁ¦ģ": 36367, + "cashback": 36368, + "kwon": 36369, + "hobbies": 36370, + "exempl": 36371, + "riesling": 36372, + "âļª": 36373, + "agles": 36374, + "nys": 36375, + "everything": 36376, + "navis": 36377, + "addi": 36378, + "magnesium": 36379, + "facelift": 36380, + "arkham": 36381, + "grandes": 36382, + "extremist": 36383, + "donat": 36384, + "vitality": 36385, + "pumpkin": 36386, + "betta": 36387, + "sltd": 36388, + "artisan": 36389, + "liby": 36390, + "peaked": 36391, + "ahhhhh": 36392, + "maryam": 36393, + "assim": 36394, + "unsc": 36395, + "mente": 36396, + "alaya": 36397, + "lowers": 36398, + "aras": 36399, + "griev": 36400, + "leip": 36401, + "grati": 36402, + "crises": 36403, + "sprints": 36404, + "execute": 36405, + "wto": 36406, + "msd": 36407, + "magical": 36408, + "reviewer": 36409, + "sparkles": 36410, + "jukebox": 36411, + "ðŁĺĤâĿ¤ï¸ı": 36412, + "payback": 36413, + "licenses": 36414, + "dunkin": 36415, + "belt": 36416, + "lakewood": 36417, + "hateful": 36418, + "budgets": 36419, + "revamped": 36420, + "pherson": 36421, + "kyiv": 36422, + "wentworth": 36423, + "rosen": 36424, + "cruise": 36425, + "giggle": 36426, + "defstar": 36427, + "assassinscre": 36428, + "ymouth": 36429, + "winkle": 36430, + "wfc": 36431, + "bandwagon": 36432, + "bkk": 36433, + "wiring": 36434, + "kearney": 36435, + "southside": 36436, + "petit": 36437, + "!ðŁĺį": 36438, + "nordic": 36439, + "mirza": 36440, + "mugabe": 36441, + "vl": 36442, + "scones": 36443, + "ktv": 36444, + "sandal": 36445, + "duc": 36446, + "malls": 36447, + "ðŁĴŀðŁĴŀ": 36448, + "itc": 36449, + "alay": 36450, + "impair": 36451, + "unrest": 36452, + "floss": 36453, + "cé": 36454, + "abou": 36455, + "varying": 36456, + "museo": 36457, + "server": 36458, + "diya": 36459, + "hibiscus": 36460, + "eroy": 36461, + "merritt": 36462, + "findom": 36463, + "fpp": 36464, + "unusually": 36465, + "gott": 36466, + "contingent": 36467, + "aliaa": 36468, + "ballon": 36469, + "jol": 36470, + "hiked": 36471, + "zyme": 36472, + "ayr": 36473, + "agn": 36474, + "gaz": 36475, + "periodic": 36476, + "sparty": 36477, + "practising": 36478, + "linton": 36479, + "talis": 36480, + "cypri": 36481, + "womaninbiz": 36482, + "radiodisney": 36483, + "ðŁĮ¼": 36484, + "jumpers": 36485, + "endocr": 36486, + "ðŁļ¨ðŁļ¨": 36487, + "andon": 36488, + "sharapo": 36489, + "mier": 36490, + "masonic": 36491, + "factories": 36492, + "vien": 36493, + "bbers": 36494, + "ìĽIJ": 36495, + "hold": 36496, + "kebab": 36497, + "beak": 36498, + "approached": 36499, + "acmilan": 36500, + "munro": 36501, + "kosher": 36502, + "excellency": 36503, + "negotiation": 36504, + "waltdisneyworld": 36505, + "crouch": 36506, + "teasing": 36507, + "suppression": 36508, + "enya": 36509, + "bce": 36510, + "transformationtuesday": 36511, + "callie": 36512, + "viswas": 36513, + "pgat": 36514, + "icted": 36515, + "endings": 36516, + "escu": 36517, + "recruited": 36518, + "itfc": 36519, + "collaborations": 36520, + "gino": 36521, + "snuck": 36522, + "auschwitz": 36523, + "ifc": 36524, + "xii": 36525, + "kesha": 36526, + "gervais": 36527, + "cloak": 36528, + "xl": 36529, + "saad": 36530, + "probation": 36531, + "precau": 36532, + "macin": 36533, + "anastasi": 36534, + "lek": 36535, + "eazy": 36536, + "daysofcode": 36537, + "mariahcarey": 36538, + "yog": 36539, + "stitched": 36540, + "boyfriends": 36541, + "shar": 36542, + "phile": 36543, + "agu": 36544, + "twinkle": 36545, + "phishing": 36546, + "weekender": 36547, + "icton": 36548, + "gurmeetramrahim": 36549, + "alton": 36550, + "leness": 36551, + "allan": 36552, + "penultimate": 36553, + "krystal": 36554, + "gou": 36555, + "lande": 36556, + "dismant": 36557, + "abusing": 36558, + "norse": 36559, + "paterson": 36560, + "edmun": 36561, + "apan": 36562, + "xiumin": 36563, + "skel": 36564, + "catwalk": 36565, + "react": 36566, + "walled": 36567, + "tangle": 36568, + "bryn": 36569, + "veto": 36570, + "supermoon": 36571, + "casablanc": 36572, + "appreciates": 36573, + "skid": 36574, + "both": 36575, + "catalina": 36576, + "eleague": 36577, + "cybermonday": 36578, + "cautious": 36579, + "ðŁ¤ĵ": 36580, + "novo": 36581, + "hampton": 36582, + "haye": 36583, + "josef": 36584, + "varan": 36585, + "lobos": 36586, + "roanoke": 36587, + "orphans": 36588, + "ttin": 36589, + "squads": 36590, + "ishqbaaaz": 36591, + "blackpanther": 36592, + "etu": 36593, + "ksh": 36594, + "crumble": 36595, + "cessna": 36596, + "relieved": 36597, + "scully": 36598, + "pollinators": 36599, + "explorecanada": 36600, + "kies": 36601, + "kamloops": 36602, + "kiran": 36603, + "primal": 36604, + "settlements": 36605, + "hotspot": 36606, + "brainstorming": 36607, + "cedric": 36608, + "biennial": 36609, + "shant": 36610, + "âĻ¡âĻ¡âĻ¡": 36611, + "doon": 36612, + "hearn": 36613, + "walkway": 36614, + "fem": 36615, + "veal": 36616, + "deportation": 36617, + "toxins": 36618, + "eliminating": 36619, + "descending": 36620, + "bythe": 36621, + "blasphe": 36622, + "hasta": 36623, + "complement": 36624, + "ascent": 36625, + "riga": 36626, + "provost": 36627, + "âĸª": 36628, + "weeping": 36629, + "antisemitism": 36630, + "employee": 36631, + "unearthed": 36632, + "pino": 36633, + "natalie": 36634, + "blad": 36635, + "angola": 36636, + "lockheed": 36637, + "inian": 36638, + "agr": 36639, + "nister": 36640, + "impala": 36641, + "mke": 36642, + "fanatic": 36643, + "âĺħâĺħ": 36644, + "ðŁij¸": 36645, + "luch": 36646, + "simplified": 36647, + "gallery": 36648, + "economic": 36649, + "cyborg": 36650, + "coni": 36651, + "selma": 36652, + "inception": 36653, + "koala": 36654, + "dvds": 36655, + "crested": 36656, + "mmor": 36657, + "visible": 36658, + "nsd": 36659, + "ðŁĻĮðŁı½": 36660, + "wunder": 36661, + "refrigerator": 36662, + "reopening": 36663, + "eera": 36664, + "carousel": 36665, + "asp": 36666, + "ballistic": 36667, + "victory": 36668, + "motive": 36669, + "trey": 36670, + "sharapova": 36671, + "sii": 36672, + "monter": 36673, + "intend": 36674, + "westchester": 36675, + "spe": 36676, + "cymb": 36677, + "vidal": 36678, + "llama": 36679, + "univ": 36680, + "finer": 36681, + "craftsmanship": 36682, + "jazzfest": 36683, + "bch": 36684, + "aggio": 36685, + "ncc": 36686, + "lambda": 36687, + "tranquility": 36688, + "cisco": 36689, + "baden": 36690, + "sobbing": 36691, + "ofi": 36692, + "gota": 36693, + "rumored": 36694, + "warmed": 36695, + "orean": 36696, + "acton": 36697, + "marci": 36698, + "ghani": 36699, + "âľĵ": 36700, + "assorted": 36701, + "pembroke": 36702, + "penelope": 36703, + "daf": 36704, + "atty": 36705, + "aimo": 36706, + "pretzel": 36707, + "carnival": 36708, + "thanos": 36709, + "kochi": 36710, + "mersal": 36711, + "hamradio": 36712, + "artwit": 36713, + "casc": 36714, + "guerrilla": 36715, + "kushner": 36716, + "kapp": 36717, + "alise": 36718, + "toddlers": 36719, + "stewardship": 36720, + "otti": 36721, + "terri": 36722, + "tempe": 36723, + "restless": 36724, + "vito": 36725, + "zayed": 36726, + "rspb": 36727, + "pion": 36728, + "hippo": 36729, + "hawthorne": 36730, + "inas": 36731, + "amily": 36732, + "nutcracker": 36733, + "lop": 36734, + "dali": 36735, + "tropic": 36736, + "ðŁ¤ł": 36737, + "ulo": 36738, + "jaredle": 36739, + "pyrene": 36740, + "paleo": 36741, + "usair": 36742, + "mould": 36743, + "itated": 36744, + "genetically": 36745, + "biomass": 36746, + "ðŁĩ³ðŁĩ±": 36747, + "dodd": 36748, + "practiced": 36749, + "monarchs": 36750, + "unmanned": 36751, + "mbuhari": 36752, + "amal": 36753, + "photogra": 36754, + "kool": 36755, + "brendon": 36756, + "juices": 36757, + "cure": 36758, + "worldbank": 36759, + "pointers": 36760, + "ðŁĴĿ": 36761, + "turf": 36762, + "leds": 36763, + "borussia": 36764, + "baptism": 36765, + "warwickshire": 36766, + "mounts": 36767, + "gayo": 36768, + "begg": 36769, + "copied": 36770, + "asians": 36771, + "kg": 36772, + "modernist": 36773, + "gid": 36774, + "frontman": 36775, + "concentrated": 36776, + "yt": 36777, + "scavenger": 36778, + "ironically": 36779, + "adic": 36780, + "psn": 36781, + "ðŁ¥ī": 36782, + "culturally": 36783, + "yuv": 36784, + "macarthur": 36785, + "fertilizer": 36786, + "bewithyou": 36787, + "rigor": 36788, + "minors": 36789, + "zoning": 36790, + "âĸł": 36791, + "rir": 36792, + "adolescent": 36793, + "vinny": 36794, + "reng": 36795, + "sandstone": 36796, + "guet": 36797, + "westh": 36798, + "pledged": 36799, + "laced": 36800, + "spide": 36801, + "vai": 36802, + "tycoon": 36803, + "seizure": 36804, + "dup": 36805, + "appalachian": 36806, + "rok": 36807, + "catholics": 36808, + "seychel": 36809, + "possess": 36810, + "lager": 36811, + "jodi": 36812, + "champ": 36813, + "stras": 36814, + "dina": 36815, + "centuri": 36816, + "calder": 36817, + "bluray": 36818, + "ðŁĩ¨ðŁĩ³": 36819, + "modo": 36820, + "annette": 36821, + "youtubers": 36822, + "chaps": 36823, + "angling": 36824, + "labeling": 36825, + "aqui": 36826, + "pkwy": 36827, + "lyle": 36828, + "bisexual": 36829, + "litur": 36830, + "dugout": 36831, + "libby": 36832, + "greysanatomy": 36833, + "substances": 36834, + "augustus": 36835, + "rallying": 36836, + "fidel": 36837, + "ingue": 36838, + "人": 36839, + "hallmarkchannel": 36840, + "toothbrush": 36841, + "má": 36842, + "adirond": 36843, + "aggi": 36844, + "ðŁĵį:": 36845, + "crusade": 36846, + "taxation": 36847, + "kz": 36848, + "iver": 36849, + "doubling": 36850, + "roomie": 36851, + "wab": 36852, + "enrolled": 36853, + "azon": 36854, + "aju": 36855, + "grandchildren": 36856, + "asdf": 36857, + "ðŁ¥º": 36858, + "matic": 36859, + "oughton": 36860, + "utilize": 36861, + "ðŁĴ£": 36862, + "ponder": 36863, + "raisin": 36864, + "dysfunction": 36865, + "cobain": 36866, + "butternut": 36867, + "eman": 36868, + "sured": 36869, + "drian": 36870, + "andfriends": 36871, + "withthe": 36872, + "onomy": 36873, + "heineken": 36874, + "bridal": 36875, + "leadership": 36876, + "pyramids": 36877, + "deutschland": 36878, + "jocel": 36879, + "bowel": 36880, + "yqr": 36881, + "horsepower": 36882, + "beacon": 36883, + "ingeni": 36884, + "gradient": 36885, + "fermented": 36886, + "moom": 36887, + "thingy": 36888, + "potassi": 36889, + "wristband": 36890, + "bord": 36891, + "bodied": 36892, + "ðŁĺŃðŁĺį": 36893, + "mapp": 36894, + "kau": 36895, + "cyberpunk": 36896, + "phish": 36897, + "looking": 36898, + "coates": 36899, + "apur": 36900, + "amie": 36901, + "uklabour": 36902, + "atin": 36903, + "gla": 36904, + "adoptable": 36905, + "shelby": 36906, + "villi": 36907, + "riya": 36908, + "mingly": 36909, + "climber": 36910, + "bumblebee": 36911, + "ðŁĺ¸": 36912, + "csd": 36913, + "âĿ¥": 36914, + "hospitalized": 36915, + "cki": 36916, + "hater": 36917, + "chr": 36918, + "retina": 36919, + "ita": 36920, + "fanbase": 36921, + "beatrice": 36922, + "gwyne": 36923, + "goss": 36924, + "fos": 36925, + "favorited": 36926, + "swachhbharat": 36927, + "malade": 36928, + "monmouth": 36929, + "\"[": 36930, + "sivan": 36931, + "shhh": 36932, + "commanding": 36933, + "sainsburys": 36934, + "weed": 36935, + "gman": 36936, + "ssw": 36937, + "reptile": 36938, + "ivy": 36939, + "tropics": 36940, + "rollers": 36941, + "overcast": 36942, + "exposition": 36943, + "masquerade": 36944, + "mancrush": 36945, + "waist": 36946, + "sprinter": 36947, + "sleet": 36948, + "levin": 36949, + "jpg": 36950, + "_(": 36951, + "opel": 36952, + "exploit": 36953, + "apa": 36954, + "powe": 36955, + "wrecking": 36956, + "jongin": 36957, + "orb": 36958, + "erick": 36959, + "bosco": 36960, + "praising": 36961, + "bertr": 36962, + "towing": 36963, + "insecurity": 36964, + "kut": 36965, + "restocked": 36966, + "rrp": 36967, + "prescribed": 36968, + "trafalgar": 36969, + "pert": 36970, + "gases": 36971, + "apprais": 36972, + "ghar": 36973, + "musicals": 36974, + "âĸ¬âĸ¬": 36975, + "mcfad": 36976, + "agony": 36977, + "condition": 36978, + "equip": 36979, + "shik": 36980, + "atravel": 36981, + "ðŁĩ¿ðŁĩ¦": 36982, + "keh": 36983, + "abduction": 36984, + "peoria": 36985, + "wilkins": 36986, + "gms": 36987, + "asd": 36988, + "evi": 36989, + "ðŁĴĹðŁĴĹðŁĴĹ": 36990, + "uz": 36991, + "moc": 36992, + "hallelujah": 36993, + "guadalu": 36994, + "louvre": 36995, + "drawing": 36996, + "gove": 36997, + "phant": 36998, + "frie": 36999, + "webdev": 37000, + "programmer": 37001, + "zable": 37002, + "gamescom": 37003, + "clarify": 37004, + "lith": 37005, + "kinky": 37006, + "âĿ£": 37007, + "labourdoorstep": 37008, + "sonata": 37009, + "juris": 37010, + "maiden": 37011, + "viadu": 37012, + "bucharest": 37013, + "conditioned": 37014, + "capitalist": 37015, + "ude": 37016, + "psb": 37017, + "spca": 37018, + "lulla": 37019, + "foothills": 37020, + "kayo": 37021, + "bond": 37022, + "womb": 37023, + "rounder": 37024, + "cesar": 37025, + "bursts": 37026, + "apra": 37027, + "swoon": 37028, + "sabrin": 37029, + "fragrant": 37030, + "clearer": 37031, + "kubrick": 37032, + "climax": 37033, + "journo": 37034, + "agle": 37035, + "ðŁı½âĢįâĻĢï¸ı": 37036, + "pooch": 37037, + "hale": 37038, + "solit": 37039, + "salmon": 37040, + "organisms": 37041, + "bronson": 37042, + "arten": 37043, + "hodgson": 37044, + "alove": 37045, + "venture": 37046, + "bbi": 37047, + "aea": 37048, + "ðŁIJ¢": 37049, + "ldn": 37050, + "dnr": 37051, + "ozone": 37052, + "ellas": 37053, + "manny": 37054, + "azzur": 37055, + "unbeat": 37056, + "truffles": 37057, + "thong": 37058, + "mañ": 37059, + "lasers": 37060, + "leye": 37061, + "gettysburg": 37062, + "backpacks": 37063, + "oris": 37064, + "maison": 37065, + "crawling": 37066, + "labra": 37067, + "cling": 37068, + "dragging": 37069, + "steal": 37070, + "doubt": 37071, + "devan": 37072, + "ckers": 37073, + "agentsof": 37074, + "photobomb": 37075, + "elonmusk": 37076, + "aboy": 37077, + "distances": 37078, + "storyline": 37079, + "spi": 37080, + "northan": 37081, + "europeans": 37082, + "whale": 37083, + "serpent": 37084, + "ðŁļ²": 37085, + "fior": 37086, + "trit": 37087, + "oxo": 37088, + "awarding": 37089, + "classmate": 37090, + "sufc": 37091, + "smartest": 37092, + "riches": 37093, + "prk": 37094, + "bigfoot": 37095, + "armb": 37096, + "bipolar": 37097, + "dwelling": 37098, + "omars": 37099, + "kwan": 37100, + "grime": 37101, + "meng": 37102, + "frederick": 37103, + "navarro": 37104, + "sorrynotsorry": 37105, + "jaredleto": 37106, + "pave": 37107, + "slack": 37108, + "barnsley": 37109, + "attar": 37110, + "eviction": 37111, + "accumulation": 37112, + "oir": 37113, + "catchy": 37114, + "welter": 37115, + "vikas": 37116, + "hassee": 37117, + "nikita": 37118, + "moyes": 37119, + "mathews": 37120, + "shiv": 37121, + "gatwick": 37122, + "profiling": 37123, + "companions": 37124, + "marrake": 37125, + "antics": 37126, + "ðŁĻĮðŁĻĮðŁĻĮ": 37127, + "sese": 37128, + "boi": 37129, + "bartlett": 37130, + "poisonous": 37131, + "abuses": 37132, + "ymm": 37133, + "kampala": 37134, + "guggenheim": 37135, + "imvkohli": 37136, + "dolom": 37137, + "bree": 37138, + "throttle": 37139, + "gareth": 37140, + "fitzpatrick": 37141, + "unya": 37142, + "parad": 37143, + "margot": 37144, + "jnr": 37145, + "wea": 37146, + "potassium": 37147, + "pnc": 37148, + "disguised": 37149, + "crash": 37150, + "renergy": 37151, + "illic": 37152, + "coupled": 37153, + "niels": 37154, + "ciones": 37155, + "æĹ¥": 37156, + "iment": 37157, + "despicable": 37158, + "dye": 37159, + "whatcha": 37160, + "connections": 37161, + "paralympics": 37162, + "gauntlet": 37163, + "waitrose": 37164, + "suicidal": 37165, + "starship": 37166, + "vapor": 37167, + "stou": 37168, + "lawmaker": 37169, + "cooled": 37170, + "simo": 37171, + "theno": 37172, + "offroad": 37173, + "jaden": 37174, + "basque": 37175, + "vicky": 37176, + "lukaku": 37177, + "centro": 37178, + "trish": 37179, + "strategist": 37180, + "medications": 37181, + "horst": 37182, + "bfc": 37183, + "grail": 37184, + "sharply": 37185, + "aditya": 37186, + "tomb": 37187, + "kaufman": 37188, + "tripad": 37189, + "samba": 37190, + "pastoral": 37191, + "britney": 37192, + "sagan": 37193, + "hillside": 37194, + "masons": 37195, + "sara": 37196, + "zone": 37197, + "xu": 37198, + "totes": 37199, + "robbie": 37200, + "appen": 37201, + "montag": 37202, + "dero": 37203, + "shortfilm": 37204, + "charismatic": 37205, + "tators": 37206, + "kiba": 37207, + "andri": 37208, + "alarming": 37209, + "splitting": 37210, + "icar": 37211, + "thug": 37212, + "scariest": 37213, + "sylvester": 37214, + "anan": 37215, + "utrecht": 37216, + "adifference": 37217, + "meade": 37218, + "buster": 37219, + "airstrikes": 37220, + "cuffs": 37221, + "accountants": 37222, + "ðŁĺ¡ðŁĺ¡": 37223, + "newt": 37224, + "bott": 37225, + "issuing": 37226, + "clancy": 37227, + "wwenetwork": 37228, + "kyuhyun": 37229, + "resemble": 37230, + "pajamas": 37231, + "sink": 37232, + "kinney": 37233, + "sulph": 37234, + "ork": 37235, + "lies": 37236, + "lagh": 37237, + "orton": 37238, + "rahul": 37239, + "dsc": 37240, + "wewill": 37241, + "ream": 37242, + "colloqui": 37243, + "sharia": 37244, + "hectic": 37245, + "sarcasm": 37246, + "lander": 37247, + "tmz": 37248, + "endorf": 37249, + "roz": 37250, + "hammered": 37251, + "fris": 37252, + "wadi": 37253, + "popefrancis": 37254, + "heit": 37255, + "flashlight": 37256, + "unborn": 37257, + "opes": 37258, + "holiness": 37259, + "ðŁIJ¦": 37260, + "nacht": 37261, + "imsa": 37262, + "gracing": 37263, + "bjp": 37264, + "verts": 37265, + "csc": 37266, + "homeowner": 37267, + "aque": 37268, + "bigotry": 37269, + "annie": 37270, + "bagh": 37271, + "âĿ¤ï¸ıðŁĺį": 37272, + "cari": 37273, + "thomp": 37274, + "disposable": 37275, + "cardiology": 37276, + "patented": 37277, + "hhhhhh": 37278, + "ldr": 37279, + "stephenson": 37280, + "crores": 37281, + "fanning": 37282, + "climat": 37283, + "ðŁijįðŁijįðŁijį": 37284, + "ðŁijįðŁı¼": 37285, + "aeron": 37286, + "piccadilly": 37287, + "bankrupt": 37288, + "silvia": 37289, + "employ": 37290, + "donny": 37291, + "commenting": 37292, + "screenwriter": 37293, + "iota": 37294, + "cean": 37295, + "ancers": 37296, + "tuan": 37297, + "streetwear": 37298, + "य": 37299, + "skine": 37300, + "espa": 37301, + "asif": 37302, + "osce": 37303, + "sheppard": 37304, + "morecam": 37305, + "bottle": 37306, + "ders": 37307, + "oracle": 37308, + "googleplay": 37309, + "averaged": 37310, + "edmonton": 37311, + "stephan": 37312, + "sisterhood": 37313, + "crusted": 37314, + "staggering": 37315, + "methodology": 37316, + "congresswoman": 37317, + "cabo": 37318, + "triggers": 37319, + "milky": 37320, + "glide": 37321, + "toothpaste": 37322, + "roommates": 37323, + "nuff": 37324, + "guam": 37325, + "sprinkles": 37326, + "alternative": 37327, + "watfordfc": 37328, + "uoft": 37329, + "haley": 37330, + "contacted": 37331, + "bundy": 37332, + "prostitu": 37333, + "ghar": 37334, + "preston": 37335, + "onsite": 37336, + "hilar": 37337, + "gts": 37338, + "catt": 37339, + "hampstead": 37340, + "??!": 37341, + "ðŁĩ§ðŁĩ": 37342, + "bbcqt": 37343, + "alessandro": 37344, + "resist": 37345, + "maidan": 37346, + "tko": 37347, + "shading": 37348, + "pinup": 37349, + "gallo": 37350, + "sinu": 37351, + "atec": 37352, + "funk": 37353, + "aclu": 37354, + "strides": 37355, + "rhyme": 37356, + "wetland": 37357, + "bbcspringwatch": 37358, + "tins": 37359, + "wildcard": 37360, + "stour": 37361, + "flamenco": 37362, + "paula": 37363, + "ontology": 37364, + "gangsta": 37365, + "amade": 37366, + "ãĤ«": 37367, + "tbs": 37368, + "skeletal": 37369, + "runner": 37370, + "jardin": 37371, + "harrier": 37372, + "hunted": 37373, + "zhen": 37374, + "believeinfilm": 37375, + "demean": 37376, + "auditi": 37377, + "restart": 37378, + "chondri": 37379, + "âĿ¤ï¸ıðŁĴĻ": 37380, + "mclaren": 37381, + "gab": 37382, + "shum": 37383, + "ausa": 37384, + "lewisham": 37385, + "ypg": 37386, + "kjv": 37387, + "furnished": 37388, + "doro": 37389, + "bonded": 37390, + "morty": 37391, + "latitude": 37392, + "_)": 37393, + "lova": 37394, + "waterways": 37395, + "vinai": 37396, + "shorth": 37397, + "drunk": 37398, + "cay": 37399, + "ayana": 37400, + "kaplan": 37401, + "cappuccino": 37402, + "spro": 37403, + "lifeboat": 37404, + "hasbro": 37405, + "spolice": 37406, + "toron": 37407, + "doing": 37408, + "damn": 37409, + "shree": 37410, + "fountains": 37411, + "entation": 37412, + "maru": 37413, + "boarder": 37414, + "topless": 37415, + "jada": 37416, + "channing": 37417, + "ulls": 37418, + "enclosure": 37419, + "gibson": 37420, + "fractured": 37421, + "britton": 37422, + "ö": 37423, + "tous": 37424, + "porth": 37425, + "draf": 37426, + "trailing": 37427, + "margate": 37428, + "elife": 37429, + "downward": 37430, + "linn": 37431, + "glades": 37432, + "girlpower": 37433, + "akrish": 37434, + "uki": 37435, + "ronda": 37436, + "tsc": 37437, + "appreciationday": 37438, + "vising": 37439, + "loom": 37440, + "ðŁį³": 37441, + "mexican": 37442, + "argos": 37443, + "yya": 37444, + "jadine": 37445, + "southport": 37446, + "dend": 37447, + "sista": 37448, + "redeem": 37449, + "meng": 37450, + "braxton": 37451, + "antioxidant": 37452, + "skey": 37453, + "mpg": 37454, + "finding": 37455, + "vibration": 37456, + "ceu": 37457, + "khart": 37458, + "dimini": 37459, + "cline": 37460, + "shelly": 37461, + "hines": 37462, + "īï¸ı": 37463, + "topical": 37464, + "nover": 37465, + "maxx": 37466, + "primitive": 37467, + "illustrate": 37468, + "bounds": 37469, + "trenton": 37470, + "jointly": 37471, + "breeders": 37472, + "uchi": 37473, + "wakeupamerica": 37474, + "bada": 37475, + "ðŁĹ£ï¸ı": 37476, + "guacam": 37477, + "spheres": 37478, + "peregr": 37479, + "youthful": 37480, + "lolo": 37481, + "birmin": 37482, + "tly": 37483, + "jeremycorbyn": 37484, + "defects": 37485, + "cosm": 37486, + "arent": 37487, + "vaa": 37488, + "bagels": 37489, + "mediac": 37490, + "coriander": 37491, + "icago": 37492, + "ghaz": 37493, + "abbas": 37494, + "remodel": 37495, + "structuring": 37496, + "pum": 37497, + "outlaw": 37498, + "adani": 37499, + "rbc": 37500, + "gulls": 37501, + "nli": 37502, + "confuse": 37503, + "ðŁijĩðŁı¼": 37504, + "vila": 37505, + "mcnamara": 37506, + "corrections": 37507, + "mughal": 37508, + "seri": 37509, + "regain": 37510, + "ssb": 37511, + "leave": 37512, + "hahahah": 37513, + "grande": 37514, + "distressed": 37515, + "rechargeable": 37516, + "hoa": 37517, + "housed": 37518, + "stil": 37519, + "attributed": 37520, + "opathic": 37521, + "dips": 37522, + "prit": 37523, + "headphone": 37524, + "conclude": 37525, + "pilo": 37526, + "het": 37527, + "utsa": 37528, + "nitin": 37529, + "jem": 37530, + "snippet": 37531, + "tutoring": 37532, + "oper": 37533, + "sunk": 37534, + "ensla": 37535, + "chau": 37536, + "acorn": 37537, + "quintess": 37538, + "rankin": 37539, + "affiliated": 37540, + "ourlives": 37541, + "clint": 37542, + "seater": 37543, + "isaac": 37544, + "bashing": 37545, + "smear": 37546, + "nurse": 37547, + "doodling": 37548, + "\";": 37549, + "saku": 37550, + "atrocities": 37551, + "imam": 37552, + "gfs": 37553, + "violating": 37554, + "commend": 37555, + "bradshaw": 37556, + "erville": 37557, + "billed": 37558, + "bbe": 37559, + "thulhu": 37560, + "iphones": 37561, + "moose": 37562, + "dios": 37563, + "rew": 37564, + "methane": 37565, + "strangely": 37566, + "whisky": 37567, + "tightly": 37568, + "spielberg": 37569, + "radius": 37570, + "noticing": 37571, + "wif": 37572, + "ignati": 37573, + "ifa": 37574, + "apis": 37575, + "wali": 37576, + "haitian": 37577, + "bushes": 37578, + "yz": 37579, + "vl": 37580, + "exited": 37581, + "assel": 37582, + "truec": 37583, + "domen": 37584, + "asher": 37585, + "inking": 37586, + "newyearseve": 37587, + "hendricks": 37588, + "bati": 37589, + "ìĿ´ì": 37590, + "richter": 37591, + "monsanto": 37592, + "conline": 37593, + "agreat": 37594, + "ðŁ¤¯": 37595, + "masterpieces": 37596, + "arn": 37597, + "roughs": 37598, + "cleve": 37599, + "sev": 37600, + "fashions": 37601, + "toya": 37602, + "shail": 37603, + "copeland": 37604, + "aquari": 37605, + "decals": 37606, + "areyou": 37607, + "yaya": 37608, + "astr": 37609, + "font": 37610, + "mlm": 37611, + "arca": 37612, + "ppor": 37613, + "pollock": 37614, + "xperia": 37615, + "conservation": 37616, + "chainsaw": 37617, + "aggie": 37618, + "?!?!?": 37619, + "sile": 37620, + "shon": 37621, + "ìĹIJ": 37622, + "notebooks": 37623, + "marquette": 37624, + "deus": 37625, + "bbled": 37626, + "spicer": 37627, + "mccabe": 37628, + "norwich": 37629, + "modification": 37630, + "boosted": 37631, + "strum": 37632, + "salesman": 37633, + "bangle": 37634, + "nissan": 37635, + "hezbollah": 37636, + "breasts": 37637, + "aaf": 37638, + "anthus": 37639, + "sker": 37640, + "owed": 37641, + "heros": 37642, + "gifs": 37643, + "fosters": 37644, + "eaters": 37645, + "dues": 37646, + "_/": 37647, + "lymphoma": 37648, + "sfam": 37649, + "megal": 37650, + "afridi": 37651, + "agic": 37652, + "pamp": 37653, + "jealousy": 37654, + "ðŁijĮðŁı¼": 37655, + "calculate": 37656, + "napping": 37657, + "gale": 37658, + "ðŁ¦Ħ": 37659, + "lubbock": 37660, + "assumed": 37661, + "renting": 37662, + "íĥľ": 37663, + "suburb": 37664, + "ãĤ·": 37665, + "technic": 37666, + "ucla": 37667, + "infront": 37668, + "garnet": 37669, + "steroids": 37670, + "striving": 37671, + "howar": 37672, + "mover": 37673, + "leton": 37674, + "bulldo": 37675, + "isin": 37676, + "ciao": 37677, + "snz": 37678, + "forefront": 37679, + "dams": 37680, + "midwife": 37681, + "mawards": 37682, + "clapton": 37683, + "wein": 37684, + "subsidies": 37685, + "sproud": 37686, + "rotherham": 37687, + "phantom": 37688, + "arach": 37689, + "spiel": 37690, + "racket": 37691, + "selamat": 37692, + "noon": 37693, + "lbc": 37694, + "entially": 37695, + "ðŁĴ¸": 37696, + "silve": 37697, + "moud": 37698, + "kinetic": 37699, + "yasi": 37700, + "ðŁİ©": 37701, + "ool": 37702, + "miku": 37703, + "iza": 37704, + "fera": 37705, + "floren": 37706, + "barbershop": 37707, + "groot": 37708, + "zest": 37709, + "nears": 37710, + "stanis": 37711, + "zand": 37712, + "policeman": 37713, + "jurisdic": 37714, + "formations": 37715, + "apparatus": 37716, + "spd": 37717, + "artifact": 37718, + "tosc": 37719, + "motivating": 37720, + "womancrush": 37721, + "redro": 37722, + "diagnostics": 37723, + "raza": 37724, + "outfitters": 37725, + "elxn": 37726, + "dodgy": 37727, + "ryn": 37728, + "shd": 37729, + "orthodon": 37730, + "olde": 37731, + "jayanti": 37732, + "balances": 37733, + "quickest": 37734, + "canton": 37735, + "fridayreads": 37736, + "!*": 37737, + "naa": 37738, + "aak": 37739, + "ðŁĶ·": 37740, + "behaviors": 37741, + "raspberries": 37742, + "ä»": 37743, + "political": 37744, + "camil": 37745, + "åľ": 37746, + "dik": 37747, + "astounding": 37748, + "liebe": 37749, + "novelty": 37750, + "turmoil": 37751, + "sully": 37752, + "springbreak": 37753, + "honouring": 37754, + "ccg": 37755, + "ðŁıĴ": 37756, + "mylittle": 37757, + "kyc": 37758, + "proms": 37759, + "ðŁķĬ": 37760, + "è": 37761, + "bige": 37762, + "avril": 37763, + "ðŁĩµðŁĩ°": 37764, + "marion": 37765, + "asants": 37766, + "surya": 37767, + "octag": 37768, + "lufthan": 37769, + "acron": 37770, + "fayetteville": 37771, + "tique": 37772, + "loves": 37773, + "enca": 37774, + "dekalb": 37775, + "taver": 37776, + "devote": 37777, + "auxiliary": 37778, + "johannes": 37779, + "treadmill": 37780, + "ayan": 37781, + "qur": 37782, + "donaldson": 37783, + "cheryl": 37784, + "\"....": 37785, + "sven": 37786, + "kirsty": 37787, + "gunners": 37788, + "radish": 37789, + "oahu": 37790, + "vsky": 37791, + "ible": 37792, + "concourse": 37793, + "bps": 37794, + "eloqu": 37795, + "ashford": 37796, + "tebow": 37797, + "roblox": 37798, + "mada": 37799, + "driving": 37800, + "thday": 37801, + "sproject": 37802, + "mms": 37803, + "banded": 37804, + ".!!": 37805, + "librarians": 37806, + "flannel": 37807, + "intolerance": 37808, + "heral": 37809, + "çµ": 37810, + "nemesis": 37811, + "lista": 37812, + "tarak": 37813, + "crypt": 37814, + "starplus": 37815, + "vishnu": 37816, + "scale": 37817, + "cris": 37818, + "%),": 37819, + "jillian": 37820, + "reggae": 37821, + "pegasus": 37822, + "olin": 37823, + "ipment": 37824, + "manic": 37825, + "lfc": 37826, + "goddard": 37827, + "iteam": 37828, + "parlour": 37829, + "anchors": 37830, + "leeminho": 37831, + "tallahassee": 37832, + "antit": 37833, + "dho": 37834, + "kidney": 37835, + "yash": 37836, + "battled": 37837, + "azad": 37838, + "garis": 37839, + "faulkner": 37840, + "sniff": 37841, + "paparazzi": 37842, + "edm": 37843, + "phyllis": 37844, + "contested": 37845, + "aaay": 37846, + "seca": 37847, + "kton": 37848, + "velve": 37849, + "rainier": 37850, + "forum": 37851, + "tampab": 37852, + "hosp": 37853, + "tractors": 37854, + "oxfordshire": 37855, + "notion": 37856, + "guangzhou": 37857, + "ðŁĺ¯": 37858, + "refill": 37859, + "wednesdaymotivation": 37860, + "slider": 37861, + "mukherjee": 37862, + "pratt": 37863, + "fontaine": 37864, + "alphon": 37865, + "afar": 37866, + "tsi": 37867, + "pesticides": 37868, + "fiends": 37869, + "mocking": 37870, + "braw": 37871, + "transat": 37872, + "doses": 37873, + "cores": 37874, + "homophobia": 37875, + "documenting": 37876, + "zlatan": 37877, + "condoms": 37878, + "sé": 37879, + "sunset": 37880, + "kunst": 37881, + "tonga": 37882, + "ส": 37883, + "vation": 37884, + "spray": 37885, + "chowder": 37886, + "raps": 37887, + "palladium": 37888, + "norwood": 37889, + "musichistory": 37890, + "hooker": 37891, + "sisi": 37892, + "osprey": 37893, + "phys": 37894, + "conceded": 37895, + "bobcat": 37896, + "armad": 37897, + "zeit": 37898, + "ÙĦ": 37899, + "ðŁĺģðŁĺģ": 37900, + "meridi": 37901, + "ðŁĩ·ðŁĩº": 37902, + "cornwall": 37903, + "!),": 37904, + "touchdowns": 37905, + "zeit": 37906, + "chalet": 37907, + "mmm": 37908, + "alche": 37909, + "gorilla": 37910, + "foss": 37911, + "atiku": 37912, + "luminous": 37913, + "ivanka": 37914, + "beek": 37915, + "stares": 37916, + "swiss": 37917, + "âĿ¤âĿ¤âĿ¤âĿ¤": 37918, + "scrubs": 37919, + "meath": 37920, + "gustav": 37921, + "jogging": 37922, + "confetti": 37923, + "asos": 37924, + "ersfc": 37925, + "breitbart": 37926, + "applicable": 37927, + "authored": 37928, + "yaho": 37929, + "hin": 37930, + "displacement": 37931, + "jv": 37932, + "ðŁĮ¹ðŁĮ¹": 37933, + "otc": 37934, + "nonprofits": 37935, + "diecast": 37936, + "gusto": 37937, + "intestin": 37938, + "cages": 37939, + "meen": 37940, + "lukas": 37941, + "mooney": 37942, + "ðŁĺ·": 37943, + "veryday": 37944, + "torah": 37945, + "ission": 37946, + "wac": 37947, + "leveraging": 37948, + "ishable": 37949, + "cuse": 37950, + "lewood": 37951, + "mayan": 37952, + "turntable": 37953, + "juice": 37954, + "trusty": 37955, + "tup": 37956, + "etiquette": 37957, + "supervisors": 37958, + "stun": 37959, + "guzman": 37960, + "conferen": 37961, + "rico": 37962, + "feast": 37963, + "backward": 37964, + "polaris": 37965, + "miche": 37966, + "jog": 37967, + "hing": 37968, + "fieldhouse": 37969, + "veling": 37970, + "shocker": 37971, + "escence": 37972, + "ा": 37973, + "vibe": 37974, + "anastasia": 37975, + "marched": 37976, + "killing": 37977, + "Ķë": 37978, + "fett": 37979, + "exoplan": 37980, + "...(": 37981, + "snowday": 37982, + "loh": 37983, + "irani": 37984, + "lakhs": 37985, + "dela": 37986, + "pocaly": 37987, + "boomers": 37988, + "dictatorship": 37989, + "acer": 37990, + "turkeys": 37991, + "quarterfinal": 37992, + "musketeers": 37993, + "ðŁĴĽðŁĴļ": 37994, + "sfx": 37995, + "museumweek": 37996, + "scala": 37997, + "risis": 37998, + "(ðŁĵ·": 37999, + "ãĢĤ": 38000, + "zies": 38001, + "boeh": 38002, + "hues": 38003, + "lusci": 38004, + "dola": 38005, + "impeachtrump": 38006, + "rood": 38007, + "doncaster": 38008, + "torre": 38009, + "heroes": 38010, + "foyer": 38011, + "tari": 38012, + "blurred": 38013, + "kew": 38014, + "frankly": 38015, + "droid": 38016, + "apal": 38017, + "м": 38018, + "yaf": 38019, + "bret": 38020, + "paragu": 38021, + "cacao": 38022, + "ðŁĻĮðŁı¾": 38023, + "rue": 38024, + "headaches": 38025, + "shawty": 38026, + "charley": 38027, + "paler": 38028, + "gowns": 38029, + "correctional": 38030, + "ðŁĺ©ðŁĺ©": 38031, + "breakingbad": 38032, + "oling": 38033, + "dap": 38034, + "endeavour": 38035, + "citadel": 38036, + "trad": 38037, + "incumbent": 38038, + "meditate": 38039, + "footed": 38040, + "ðŁĴµ": 38041, + "shabbat": 38042, + "dayofthe": 38043, + "willem": 38044, + "galway": 38045, + "tored": 38046, + "marriage": 38047, + "fillion": 38048, + "sleeveless": 38049, + "auditor": 38050, + "jinyoung": 38051, + "invincible": 38052, + "kaduna": 38053, + "aand": 38054, + "volcanoes": 38055, + "moneti": 38056, + "indiegogo": 38057, + "buccaneers": 38058, + "ðŁijīðŁı½": 38059, + "ãĢĤ": 38060, + "layton": 38061, + "cuckoo": 38062, + "humber": 38063, + "buzzer": 38064, + "Ïī": 38065, + "tore": 38066, + "strains": 38067, + "stom": 38068, + "paine": 38069, + "swe": 38070, + "duff": 38071, + "zou": 38072, + "simi": 38073, + "lipp": 38074, + "urn": 38075, + "seagu": 38076, + "ðŁĶ®": 38077, + "sundae": 38078, + "hic": 38079, + "ðŁĺ¨": 38080, + "bullpen": 38081, + "uper": 38082, + "flyover": 38083, + "aldridge": 38084, + "globes": 38085, + "alies": 38086, + "kenzie": 38087, + "gees": 38088, + "ycle": 38089, + "splin": 38090, + "magenta": 38091, + "jha": 38092, + "balu": 38093, + "ghorn": 38094, + "tipper": 38095, + "wicker": 38096, + "tasteof": 38097, + "conclave": 38098, + "chale": 38099, + "invasi": 38100, + "cater": 38101, + "dioxide": 38102, + "megab": 38103, + "winn": 38104, + "atp": 38105, + "transformative": 38106, + "nestled": 38107, + "hig": 38108, + "bridging": 38109, + "lilies": 38110, + "cheered": 38111, + "baddest": 38112, + "scrolls": 38113, + "realis": 38114, + "diplo": 38115, + "ðŁĶ«": 38116, + "concession": 38117, + "preferences": 38118, + "explodes": 38119, + "ergon": 38120, + "introductory": 38121, + "ineau": 38122, + "chaf": 38123, + "somes": 38124, + "landrover": 38125, + "spiration": 38126, + "sexy": 38127, + "scorecard": 38128, + "illustrates": 38129, + "soulmate": 38130, + "wien": 38131, + "interdisciplinary": 38132, + "forecasting": 38133, + "entities": 38134, + "glued": 38135, + "enlar": 38136, + "curt": 38137, + "perceptions": 38138, + "bootleg": 38139, + "mire": 38140, + "ashok": 38141, + "vaz": 38142, + "horne": 38143, + "calle": 38144, + "aculture": 38145, + "theroy": 38146, + "nighttime": 38147, + "ocal": 38148, + "characterdesign": 38149, + "armist": 38150, + "ðŁĺıðŁĺı": 38151, + "yahoo": 38152, + "aceae": 38153, + "tose": 38154, + "evento": 38155, + "sout": 38156, + "nayanth": 38157, + "whom": 38158, + "vare": 38159, + "rigging": 38160, + "genus": 38161, + "hive": 38162, + "commands": 38163, + "stie": 38164, + "daya": 38165, + "ethanol": 38166, + "enf": 38167, + "hifi": 38168, + "fluence": 38169, + "clemson": 38170, + "reinvent": 38171, + "thermometer": 38172, + "humorous": 38173, + "emerging": 38174, + "ación": 38175, + "ðŁĺĺðŁĺį": 38176, + "sity": 38177, + "hawke": 38178, + "accompanying": 38179, + "tility": 38180, + "ðŁĺª": 38181, + "recess": 38182, + "protagonist": 38183, + "lery": 38184, + "dundal": 38185, + "intl": 38186, + "brittany": 38187, + "qbs": 38188, + "offthe": 38189, + "marriages": 38190, + "howto": 38191, + "violated": 38192, + "adelaide": 38193, + "witt": 38194, + "lancer": 38195, + "pakv": 38196, + "hume": 38197, + "stade": 38198, + "bragging": 38199, + "outright": 38200, + "adc": 38201, + "superst": 38202, + "realtime": 38203, + "cures": 38204, + "gardeners": 38205, + "erock": 38206, + "dalejr": 38207, + "vero": 38208, + "bartol": 38209, + "moti": 38210, + "mcfly": 38211, + "vpn": 38212, + "stink": 38213, + "overrated": 38214, + "guerra": 38215, + "etis": 38216, + "athome": 38217, + "twdfamily": 38218, + "thab": 38219, + "tnx": 38220, + "rafael": 38221, + "familytravel": 38222, + "xley": 38223, + "satanic": 38224, + "equations": 38225, + "rudy": 38226, + "waldorf": 38227, + "stani": 38228, + "tube": 38229, + "measles": 38230, + "zimmerman": 38231, + "obligations": 38232, + "iously": 38233, + "bowser": 38234, + "transformer": 38235, + "shoppe": 38236, + "shaken": 38237, + "ghouse": 38238, + "tod": 38239, + "ketball": 38240, + "shareholder": 38241, + "marca": 38242, + "kpmg": 38243, + "akan": 38244, + "givenchy": 38245, + "coastal": 38246, + "auth": 38247, + "rollercoaster": 38248, + "marches": 38249, + "coordinate": 38250, + "cinema": 38251, + "apprentices": 38252, + "parlor": 38253, + "mito": 38254, + "menon": 38255, + "considerable": 38256, + "barre": 38257, + "gloss": 38258, + "enhances": 38259, + "jazeera": 38260, + "falmouth": 38261, + "thrash": 38262, + "staten": 38263, + "kzn": 38264, + "engel": 38265, + "samanthap": 38266, + "floppy": 38267, + "salom": 38268, + "ðŁıĨðŁıĨ": 38269, + "wack": 38270, + "deliberate": 38271, + "oscill": 38272, + "heritag": 38273, + "dusted": 38274, + "ornithology": 38275, + "paddle": 38276, + "ferns": 38277, + "barun": 38278, + "clans": 38279, + "anticipate": 38280, + "aay": 38281, + "matically": 38282, + "éĩ": 38283, + "tumble": 38284, + "postman": 38285, + "unicef": 38286, + "trotter": 38287, + "opd": 38288, + "leaflet": 38289, + "geist": 38290, + "ceasefire": 38291, + "screws": 38292, + "creation": 38293, + "walnuts": 38294, + "longhorns": 38295, + "understatement": 38296, + "abb": 38297, + "proximity": 38298, + "nax": 38299, + "unity": 38300, + "turnpike": 38301, + "ordained": 38302, + "dubstep": 38303, + "chakra": 38304, + "mech": 38305, + "loveher": 38306, + "lookalike": 38307, + "donnein": 38308, + "viron": 38309, + "ÙĪ": 38310, + "bangers": 38311, + "variants": 38312, + "outdated": 38313, + "inta": 38314, + "cristo": 38315, + "spelt": 38316, + "foodand": 38317, + "fon": 38318, + "stefani": 38319, + "marginal": 38320, + "hutton": 38321, + "tiara": 38322, + "telford": 38323, + "quen": 38324, + "fairgrounds": 38325, + "quetta": 38326, + "mikhail": 38327, + "healer": 38328, + "vball": 38329, + "tyre": 38330, + "undergrad": 38331, + "glend": 38332, + "homers": 38333, + "scribed": 38334, + "maintains": 38335, + "poche": 38336, + "missal": 38337, + "marko": 38338, + "uas": 38339, + "án": 38340, + "shp": 38341, + "convey": 38342, + "padre": 38343, + "saba": 38344, + "puglia": 38345, + "madhuri": 38346, + "paxton": 38347, + "chaplain": 38348, + "nago": 38349, + "casi": 38350, + "...!!!": 38351, + "flirt": 38352, + "saleh": 38353, + "kare": 38354, + "dire": 38355, + "stamped": 38356, + "extreme": 38357, + "ðŁĺĥðŁĺĥ": 38358, + "hoppy": 38359, + "guadalupe": 38360, + "advantaged": 38361, + "euchar": 38362, + "plow": 38363, + "unn": 38364, + "macqu": 38365, + "portland": 38366, + "clash": 38367, + "pes": 38368, + "loubout": 38369, + "yp": 38370, + "keeping": 38371, + "arcadia": 38372, + "frankie": 38373, + "fiu": 38374, + "deth": 38375, + "encyclopedia": 38376, + "size": 38377, + "invests": 38378, + "ðŁį©": 38379, + "geological": 38380, + "franç": 38381, + "confront": 38382, + "ðŁĺ¥": 38383, + "dys": 38384, + "afm": 38385, + "texan": 38386, + "graphene": 38387, + "repostapp": 38388, + "acf": 38389, + "ursula": 38390, + "gaza": 38391, + "ddled": 38392, + "fum": 38393, + "wsbtv": 38394, + "mbe": 38395, + "frontiers": 38396, + "chronograph": 38397, + "kes": 38398, + "interfaith": 38399, + "taboo": 38400, + "sparta": 38401, + "wondo": 38402, + "florist": 38403, + "embraces": 38404, + "caw": 38405, + "noel": 38406, + "archers": 38407, + "ðŁIJ·": 38408, + "romano": 38409, + "banan": 38410, + "shakers": 38411, + "melodies": 38412, + "geothermal": 38413, + "sephora": 38414, + "ìļ°": 38415, + "од": 38416, + "proc": 38417, + "handshake": 38418, + "pande": 38419, + "populated": 38420, + "slowdown": 38421, + "hortons": 38422, + "registrations": 38423, + "undeni": 38424, + "lants": 38425, + "passover": 38426, + "thakur": 38427, + "lief": 38428, + "adhesive": 38429, + "petal": 38430, + "microscopy": 38431, + "memphis": 38432, + "confirming": 38433, + "airdrop": 38434, + "mesmer": 38435, + "perceived": 38436, + "mingle": 38437, + "lifeline": 38438, + "ghj": 38439, + "worcestershire": 38440, + "passions": 38441, + "acher": 38442, + "ellar": 38443, + "aho": 38444, + "firenze": 38445, + "barang": 38446, + "letterman": 38447, + "hatfield": 38448, + "lucha": 38449, + "jeter": 38450, + "eshop": 38451, + "williams": 38452, + "horoscope": 38453, + "prede": 38454, + "eastbourne": 38455, + "durga": 38456, + "diversion": 38457, + "altrin": 38458, + "seismic": 38459, + "premiosm": 38460, + "narco": 38461, + "tir": 38462, + "orig": 38463, + "orm": 38464, + "landfall": 38465, + "cious": 38466, + "lindo": 38467, + "maxine": 38468, + "xico": 38469, + "tray": 38470, + "oswald": 38471, + "cba": 38472, + "ricotta": 38473, + "ncr": 38474, + "marau": 38475, + "า": 38476, + "gladiator": 38477, + "chery": 38478, + "lung": 38479, + "ume": 38480, + "popsic": 38481, + "longing": 38482, + "canals": 38483, + "taya": 38484, + "decentralized": 38485, + "shopp": 38486, + "pressures": 38487, + "maharaj": 38488, + "etihad": 38489, + "walgreens": 38490, + "succession": 38491, + "signaling": 38492, + "lig": 38493, + "staffer": 38494, + "northkorea": 38495, + "defying": 38496, + "asma": 38497, + "deg": 38498, + "perimeter": 38499, + "oakville": 38500, + "msk": 38501, + "baltimore": 38502, + "receip": 38503, + "deple": 38504, + "ðŁĺŃðŁĺĤ": 38505, + "jamboree": 38506, + ">.<": 38507, + "rspb": 38508, + "punisher": 38509, + "considerably": 38510, + "intothe": 38511, + "parisian": 38512, + "accelerated": 38513, + "polyester": 38514, + "lowes": 38515, + "frying": 38516, + "sautéed": 38517, + "mouths": 38518, + "seychelles": 38519, + "rax": 38520, + "godis": 38521, + "dakota": 38522, + "housewives": 38523, + "theme": 38524, + "matinee": 38525, + "blackbird": 38526, + "yesung": 38527, + "prefers": 38528, + "pellegr": 38529, + "inated": 38530, + "trunks": 38531, + "strongertogether": 38532, + "repet": 38533, + "repairing": 38534, + "pedals": 38535, + "tolerant": 38536, + "herr": 38537, + "dunne": 38538, + "indication": 38539, + "decatur": 38540, + "btv": 38541, + "exhibitors": 38542, + "ikon": 38543, + "fridaymotivation": 38544, + "bragg": 38545, + "livetweet": 38546, + "alves": 38547, + "womensart": 38548, + "foreigners": 38549, + "wallets": 38550, + "mindy": 38551, + "laney": 38552, + "bbin": 38553, + "tvmiaw": 38554, + "lifter": 38555, + "target": 38556, + "tame": 38557, + "drou": 38558, + "astrophotography": 38559, + "mpc": 38560, + "gpu": 38561, + "nordstrom": 38562, + "friction": 38563, + "runoff": 38564, + "lovable": 38565, + "spnfamily": 38566, + "extingui": 38567, + "bloody": 38568, + "schel": 38569, + "artistry": 38570, + "swish": 38571, + "scarce": 38572, + "phils": 38573, + "maxim": 38574, + "possum": 38575, + "compromised": 38576, + "styli": 38577, + "scfc": 38578, + "issa": 38579, + "birmingham": 38580, + "sketched": 38581, + "angelica": 38582, + "ordinance": 38583, + "jets": 38584, + "conquer": 38585, + "ðŁĺIJ": 38586, + "onlineshopping": 38587, + "sori": 38588, + "reasonably": 38589, + "nuestro": 38590, + "arturo": 38591, + "chl": 38592, + "benefici": 38593, + "sphoto": 38594, + "welt": 38595, + "nikk": 38596, + "ðŁ¤ŀ": 38597, + "danao": 38598, + "formid": 38599, + "asse": 38600, + "afirst": 38601, + "âľĤ": 38602, + "gillette": 38603, + "assor": 38604, + "anonym": 38605, + "selca": 38606, + "femi": 38607, + "bearable": 38608, + "yand": 38609, + "armory": 38610, + "crepe": 38611, + "celticfc": 38612, + "bravo": 38613, + "inexpensive": 38614, + "delec": 38615, + "gecko": 38616, + "newmarket": 38617, + "snowflakes": 38618, + "kabir": 38619, + "contra": 38620, + "canning": 38621, + "morpho": 38622, + "garwal": 38623, + "ðŁĴĥðŁı»": 38624, + "fighting": 38625, + "mutation": 38626, + "woody": 38627, + "jugg": 38628, + "graces": 38629, + "premiosmtvmiaw": 38630, + "kennedy": 38631, + "gup": 38632, + "sae": 38633, + "opha": 38634, + "offspring": 38635, + "finisher": 38636, + "betts": 38637, + "spanning": 38638, + "marj": 38639, + "hone": 38640, + "shing": 38641, + "continents": 38642, + "samanthaprabhu": 38643, + "unrelated": 38644, + "lacy": 38645, + "explosions": 38646, + "benjamin": 38647, + "sophie": 38648, + "noting": 38649, + "microsoft": 38650, + "assen": 38651, + "ahoy": 38652, + "iker": 38653, + "hofer": 38654, + "moe": 38655, + "ahmadi": 38656, + "yann": 38657, + "anak": 38658, + "mahi": 38659, + "beu": 38660, + "ahah": 38661, + "creeper": 38662, + "baahubali": 38663, + "amat": 38664, + "priory": 38665, + "hawkeye": 38666, + "deloitte": 38667, + "skoda": 38668, + "printmaking": 38669, + "assembling": 38670, + "miraculous": 38671, + "noch": 38672, + "swo": 38673, + "lega": 38674, + "operates": 38675, + "borderlands": 38676, + "elie": 38677, + "strongh": 38678, + "reptiles": 38679, + "pirate": 38680, + "unfold": 38681, + "¯": 38682, + "qualcomm": 38683, + "unpredictable": 38684, + "otr": 38685, + "rosewood": 38686, + "directional": 38687, + "counselors": 38688, + "cornell": 38689, + "liberated": 38690, + "jad": 38691, + "irregular": 38692, + "bulgarian": 38693, + "highness": 38694, + "vodafone": 38695, + "swild": 38696, + "minimize": 38697, + "grazie": 38698, + "à¹ĩ": 38699, + "rstats": 38700, + "streep": 38701, + "ometric": 38702, + "humble": 38703, + "lump": 38704, + "lille": 38705, + "bü": 38706, + "homedepot": 38707, + "tripadvisor": 38708, + "kiwan": 38709, + "avia": 38710, + "erz": 38711, + "exico": 38712, + "duf": 38713, + "blumen": 38714, + "mizing": 38715, + "arma": 38716, + "inim": 38717, + "constan": 38718, + "sora": 38719, + "jual": 38720, + "aun": 38721, + "twell": 38722, + "trenches": 38723, + "hera": 38724, + "rk": 38725, + "poplar": 38726, + "recipeoftheday": 38727, + "llan": 38728, + "bhuban": 38729, + "shortages": 38730, + "ingdon": 38731, + "bridgewater": 38732, + "ðŁIJĺ": 38733, + "fortnite": 38734, + "camden": 38735, + "uncture": 38736, + "prow": 38737, + "colonies": 38738, + "tks": 38739, + "ngo": 38740, + "bhm": 38741, + "livepd": 38742, + "splace": 38743, + "slike": 38744, + "happyeaster": 38745, + "terrence": 38746, + "revolver": 38747, + "jed": 38748, + "yyyy": 38749, + "officeof": 38750, + "mts": 38751, + "existential": 38752, + "rourke": 38753, + "explorebc": 38754, + "ssed": 38755, + "priest": 38756, + "vixen": 38757, + "siding": 38758, + "kpa": 38759, + "ahar": 38760, + "juic": 38761, + "obstruc": 38762, + "forensics": 38763, + "ukmfg": 38764, + "cancellation": 38765, + "weary": 38766, + "abq": 38767, + "elec": 38768, + "prized": 38769, + "debts": 38770, + "mezz": 38771, + "salvatore": 38772, + "mdc": 38773, + "grette": 38774, + "cgc": 38775, + "thon": 38776, + "snowstorm": 38777, + "tsch": 38778, + "cookery": 38779, + "å¹": 38780, + "waxing": 38781, + "nacional": 38782, + "murs": 38783, + "rave": 38784, + "capes": 38785, + "germain": 38786, + "dripping": 38787, + "submitting": 38788, + "omelette": 38789, + "iteration": 38790, + "ajes": 38791, + "shimmer": 38792, + "fueling": 38793, + "ðŁĩ§ðŁĩª": 38794, + "lipo": 38795, + "bobble": 38796, + "unfollow": 38797, + "islamist": 38798, + "hiber": 38799, + "cats": 38800, + "agentsofshield": 38801, + "sensi": 38802, + "_____": 38803, + "steria": 38804, + "instal": 38805, + "auspicious": 38806, + "harrow": 38807, + "overland": 38808, + "feminists": 38809, + "instant": 38810, + "chariot": 38811, + "blindness": 38812, + "sped": 38813, + "scarec": 38814, + "nuit": 38815, + "miniatures": 38816, + "hoseok": 38817, + "glock": 38818, + "fifaworldcup": 38819, + "ete": 38820, + "dism": 38821, + "weiner": 38822, + "exfoli": 38823, + "earts": 38824, + "à¸Ķ": 38825, + "myart": 38826, + "manil": 38827, + "issant": 38828, + "forma": 38829, + "incu": 38830, + "buffalob": 38831, + "intim": 38832, + "mccul": 38833, + "anjali": 38834, + "popo": 38835, + "undoub": 38836, + "hila": 38837, + "fungal": 38838, + "thankful": 38839, + "futur": 38840, + "endish": 38841, + "rends": 38842, + "thar": 38843, + "sheff": 38844, + "ringo": 38845, + "nicholls": 38846, + "iowa": 38847, + "potom": 38848, + "clams": 38849, + "ãģĦ": 38850, + "aconf": 38851, + "stadiums": 38852, + "dimp": 38853, + "dik": 38854, + "residences": 38855, + "dov": 38856, + "caricature": 38857, + "seagull": 38858, + "klm": 38859, + "confess": 38860, + "slapped": 38861, + "celeb": 38862, + "turbines": 38863, + "ppv": 38864, + "nurture": 38865, + "elab": 38866, + ".....#": 38867, + "tuff": 38868, + "depress": 38869, + "alfar": 38870, + "amiibo": 38871, + "dispon": 38872, + "ewing": 38873, + "queer": 38874, + "friends": 38875, + "forre": 38876, + "âĺ¼": 38877, + "swt": 38878, + "aquarius": 38879, + "headliner": 38880, + "curd": 38881, + "figs": 38882, + "otters": 38883, + "lovefl": 38884, + "kareem": 38885, + "govegan": 38886, + "friyay": 38887, + "consolation": 38888, + "atri": 38889, + "ì§Ħ": 38890, + "âĺĿï¸ı": 38891, + "polyne": 38892, + "gued": 38893, + "oya": 38894, + "laus": 38895, + "intestinal": 38896, + "camilla": 38897, + "scalp": 38898, + "pir": 38899, + "leeds": 38900, + "horrifying": 38901, + "boretum": 38902, + "dandelion": 38903, + "ferrer": 38904, + "ellic": 38905, + "asx": 38906, + "soren": 38907, + "reloaded": 38908, + "aleague": 38909, + "navigator": 38910, + "inette": 38911, + "addams": 38912, + "alchemist": 38913, + "akshay": 38914, + "dystopian": 38915, + "awec": 38916, + "naya": 38917, + "alisa": 38918, + "ailed": 38919, + "agor": 38920, + "aviator": 38921, + "alizer": 38922, + "smobile": 38923, + "findyourpark": 38924, + "copying": 38925, + "toddy": 38926, + "shti": 38927, + "monger": 38928, + "calhoun": 38929, + "napkin": 38930, + "breakup": 38931, + "yatra": 38932, + "sethu": 38933, + "richi": 38934, + "erasmus": 38935, + "ferry": 38936, + "amore": 38937, + "practise": 38938, + "bobo": 38939, + "powerpoint": 38940, + "oose": 38941, + "liffe": 38942, + "china": 38943, + "shka": 38944, + "fadnavis": 38945, + "duane": 38946, + "waron": 38947, + "false": 38948, + "ðŁļĤ": 38949, + "washes": 38950, + "discip": 38951, + "========": 38952, + "gk": 38953, + "abb": 38954, + "stubborn": 38955, + "medieval": 38956, + "pci": 38957, + "ðŁįª": 38958, + "marilyn": 38959, + "hyo": 38960, + "mandi": 38961, + "cri": 38962, + "predecess": 38963, + "continuation": 38964, + "omusic": 38965, + "slat": 38966, + "whal": 38967, + "mallory": 38968, + "bonn": 38969, + "shenzhen": 38970, + "cai": 38971, + "âĺĥ": 38972, + "safest": 38973, + "forwards": 38974, + "drawers": 38975, + "blasted": 38976, + "slee": 38977, + "morphe": 38978, + "mbta": 38979, + "dumbass": 38980, + "ÑĦоÑĤо": 38981, + "alhamdulillah": 38982, + "eclub": 38983, + "albeit": 38984, + "healey": 38985, + "ayurveda": 38986, + "advertised": 38987, + "crocs": 38988, + "ittles": 38989, + "bryson": 38990, + "bei": 38991, + "njpw": 38992, + "honoree": 38993, + "fused": 38994, + "ðŁĶĺ": 38995, + "multin": 38996, + "naga": 38997, + "departs": 38998, + "kop": 38999, + "kino": 39000, + "jharkhand": 39001, + "edna": 39002, + "axle": 39003, + "milton": 39004, + "supremacist": 39005, + "marrakech": 39006, + "dominic": 39007, + "transcript": 39008, + "][#": 39009, + ":).": 39010, + "woc": 39011, + "surrounds": 39012, + "ogil": 39013, + "leaflets": 39014, + "cowell": 39015, + "whew": 39016, + "trude": 39017, + "prolifer": 39018, + "succes": 39019, + "sportsman": 39020, + "condom": 39021, + "poche": 39022, + "kup": 39023, + "imprisonment": 39024, + "{}": 39025, + "scrambled": 39026, + "åĽ": 39027, + "kaine": 39028, + "cellphone": 39029, + "metamor": 39030, + "coni": 39031, + "remnants": 39032, + "eez": 39033, + "downpour": 39034, + "afternoon": 39035, + "exercising": 39036, + "berser": 39037, + "architecture": 39038, + "wicklow": 39039, + "mns": 39040, + "isp": 39041, + "boc": 39042, + "niss": 39043, + "mnwild": 39044, + "stumble": 39045, + "rsi": 39046, + "luffy": 39047, + "silen": 39048, + "ddad": 39049, + "bullies": 39050, + "hawker": 39051, + "bbcc": 39052, + "scuba": 39053, + "epp": 39054, + "quets": 39055, + "foraging": 39056, + "pallet": 39057, + "hadi": 39058, + "cinematographer": 39059, + "catchers": 39060, + "toaster": 39061, + "khi": 39062, + "litecoin": 39063, + "kidlit": 39064, + "amherst": 39065, + "mauricio": 39066, + "ipad": 39067, + "marmalade": 39068, + "fey": 39069, + "donnelly": 39070, + "gto": 39071, + "estas": 39072, + "cerebral": 39073, + "antgrasso": 39074, + "zzled": 39075, + "virgil": 39076, + "swapped": 39077, + "ðŁĺħðŁĺħ": 39078, + "nodapl": 39079, + "greatest": 39080, + "nhlbruins": 39081, + "fraser": 39082, + "bmo": 39083, + "anew": 39084, + ".âĿ¤ï¸ı": 39085, + "segregation": 39086, + "remarkably": 39087, + "mccormick": 39088, + "logger": 39089, + "eras": 39090, + "contracting": 39091, + "âłĢâłĢ": 39092, + "yorks": 39093, + "ukulele": 39094, + "touchscreen": 39095, + "decked": 39096, + "benn": 39097, + "southwark": 39098, + "ravin": 39099, + "numis": 39100, + "ðŁ¤Ļ": 39101, + "rut": 39102, + "greco": 39103, + "ethic": 39104, + "redneck": 39105, + "arr": 39106, + "tcs": 39107, + "ihri": 39108, + "ðŁĩ«ðŁĩ·": 39109, + "lk": 39110, + "inherited": 39111, + "zyk": 39112, + "viaduct": 39113, + "martyred": 39114, + "higu": 39115, + "ssn": 39116, + "bein": 39117, + "streetstyle": 39118, + "fergie": 39119, + "bankof": 39120, + "æĹ¥": 39121, + "stakeholder": 39122, + "exemplary": 39123, + "cress": 39124, + "essa": 39125, + "erotica": 39126, + "intrepid": 39127, + "gomes": 39128, + "braun": 39129, + "bethany": 39130, + "bangtan": 39131, + "pulmonary": 39132, + "milling": 39133, + "doctorate": 39134, + "trumprussia": 39135, + "र": 39136, + "sani": 39137, + "blatt": 39138, + "plau": 39139, + "deprived": 39140, + "tle": 39141, + "fully": 39142, + "bourn": 39143, + "stak": 39144, + "lufthansa": 39145, + "kiosk": 39146, + "faroo": 39147, + "defy": 39148, + "badan": 39149, + "ðŁĺĺâĿ¤ï¸ı": 39150, + "ritz": 39151, + "trisha": 39152, + "rands": 39153, + "middlesex": 39154, + "arabs": 39155, + "proj": 39156, + "sportscenter": 39157, + "repeats": 39158, + "ivf": 39159, + "bleedblue": 39160, + "assure": 39161, + "obs": 39162, + "territorial": 39163, + "elen": 39164, + "beverley": 39165, + "annah": 39166, + "âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı": 39167, + "zl": 39168, + "forgood": 39169, + "sciencefiction": 39170, + "glau": 39171, + "sonya": 39172, + "prith": 39173, + "stweets": 39174, + "mixers": 39175, + "mario": 39176, + "antelope": 39177, + "writingcommunity": 39178, + "wentz": 39179, + "denham": 39180, + "bedi": 39181, + "sfo": 39182, + "harleydavidson": 39183, + "lookbook": 39184, + "immunotherapy": 39185, + "orphe": 39186, + "esville": 39187, + "edged": 39188, + "task": 39189, + "sbball": 39190, + "corrosion": 39191, + "kilometers": 39192, + "costing": 39193, + "playback": 39194, + "keke": 39195, + "divisi": 39196, + "uter": 39197, + "relocation": 39198, + "yelled": 39199, + "peng": 39200, + "upbeat": 39201, + "serve": 39202, + "âļł": 39203, + "halen": 39204, + "stirring": 39205, + "rehman": 39206, + "env": 39207, + "schumacher": 39208, + "fragment": 39209, + "alkaline": 39210, + "sbk": 39211, + "resili": 39212, + "sharepoint": 39213, + "rollover": 39214, + "trash": 39215, + "counterpart": 39216, + "âĻ«": 39217, + "obitu": 39218, + "à½": 39219, + "ãĤ¹": 39220, + "mulberry": 39221, + "ðŁİĨ": 39222, + "autonomy": 39223, + "spraying": 39224, + "natl": 39225, + "loveyou": 39226, + "franki": 39227, + "nuk": 39228, + "escar": 39229, + "canteen": 39230, + "alibaba": 39231, + "deplor": 39232, + "molecule": 39233, + "pud": 39234, + "fortnight": 39235, + "blondie": 39236, + "sphin": 39237, + "portrayal": 39238, + "tache": 39239, + "bute": 39240, + "consisting": 39241, + "freepalestine": 39242, + "csp": 39243, + "immort": 39244, + "dns": 39245, + "ðŁĴ¥ðŁĴ¥": 39246, + "tourde": 39247, + "cooking": 39248, + "archival": 39249, + "gathers": 39250, + "bitt": 39251, + "banc": 39252, + "premature": 39253, + "snowball": 39254, + "poetryday": 39255, + "loudly": 39256, + "fugitive": 39257, + "eday": 39258, + "emra": 39259, + "ðŁĩ¸ðŁĩª": 39260, + "scien": 39261, + "nodejs": 39262, + "jurgen": 39263, + "jeong": 39264, + "bandana": 39265, + "unis": 39266, + "foxsports": 39267, + "vandy": 39268, + "provisions": 39269, + "weep": 39270, + "tuk": 39271, + "iko": 39272, + "houn": 39273, + "ziggy": 39274, + "zr": 39275, + "fillet": 39276, + "bata": 39277, + "tink": 39278, + "cone": 39279, + "wewant": 39280, + "kilo": 39281, + "horace": 39282, + "slt": 39283, + "sct": 39284, + "staytuned": 39285, + "victoria": 39286, + "umbria": 39287, + "attacker": 39288, + "inghamshire": 39289, + "frightening": 39290, + "noir": 39291, + "frat": 39292, + "contempt": 39293, + "liaison": 39294, + "hoi": 39295, + "brink": 39296, + "trill": 39297, + "niagar": 39298, + "kickass": 39299, + "dundas": 39300, + "notmy": 39301, + "rhode": 39302, + "bumble": 39303, + "noxi": 39304, + "fag": 39305, + "spectators": 39306, + "mancrushmonday": 39307, + "jinping": 39308, + "distract": 39309, + "daisy": 39310, + "walden": 39311, + "portrait": 39312, + "arthistory": 39313, + "voltron": 39314, + "evel": 39315, + "isc": 39316, + "acm": 39317, + "rite": 39318, + "nao": 39319, + "deported": 39320, + "sweats": 39321, + "rufus": 39322, + "lobo": 39323, + "laborday": 39324, + "gamo": 39325, + "ihrithik": 39326, + "blit": 39327, + "abdominal": 39328, + "ãħ¤ãħ¤ãħ¤ãħ¤": 39329, + "iit": 39330, + "eq": 39331, + "busy": 39332, + "alluarjun": 39333, + "undisclosed": 39334, + "deton": 39335, + "procreate": 39336, + "kil": 39337, + "ðŁİĤðŁİĤ": 39338, + "mitchell": 39339, + "kii": 39340, + "inheritance": 39341, + "alp": 39342, + "joburg": 39343, + "patrolling": 39344, + "compulsory": 39345, + "unsigned": 39346, + "niam": 39347, + "lga": 39348, + "eshopsuk": 39349, + "trilli": 39350, + "maw": 39351, + "appreciating": 39352, + "rockab": 39353, + "mañana": 39354, + "antal": 39355, + "malvern": 39356, + "royo": 39357, + "grandprix": 39358, + "sutton": 39359, + "goftheday": 39360, + "digi": 39361, + "ãħĭãħĭãħĭãħĭ": 39362, + "tles": 39363, + "varanasi": 39364, + "erected": 39365, + "disciples": 39366, + "contact": 39367, + "ðŁĺµ": 39368, + "lid": 39369, + "â¬ĩ": 39370, + "scentre": 39371, + "radiator": 39372, + "ingtips": 39373, + "transitions": 39374, + "thursdaymotivation": 39375, + "chemical": 39376, + "separati": 39377, + "salis": 39378, + "mim": 39379, + "geographical": 39380, + "bookfest": 39381, + "/.": 39382, + "âľĭ": 39383, + "vae": 39384, + "currie": 39385, + "aggarwal": 39386, + "acceleration": 39387, + "theses": 39388, + "lgm": 39389, + "umass": 39390, + "proportions": 39391, + "nata": 39392, + "anians": 39393, + "kuch": 39394, + "beacons": 39395, + "apr": 39396, + "@#": 39397, + "ðŁĴªðŁı¾": 39398, + "nuke": 39399, + "sheraton": 39400, + "kio": 39401, + "makati": 39402, + "politico": 39403, + "morale": 39404, + "ìĻ": 39405, + "economically": 39406, + "ggly": 39407, + "ssen": 39408, + "pastries": 39409, + "internships": 39410, + "vicente": 39411, + "fantaken": 39412, + "avengers": 39413, + "accuse": 39414, + "sleepover": 39415, + "indicated": 39416, + "thedream": 39417, + "sterone": 39418, + "renders": 39419, + "frost": 39420, + "oui": 39421, + "gregg": 39422, + "dore": 39423, + "⾨⾨⾨": 39424, + "pugs": 39425, + "saty": 39426, + "numb": 39427, + "hemsworth": 39428, + "tami": 39429, + "lassic": 39430, + "schiff": 39431, + "iglesias": 39432, + "agawa": 39433, + "]\"": 39434, + "reshi": 39435, + "gamestop": 39436, + "divorced": 39437, + "theater": 39438, + "claudi": 39439, + "unconventional": 39440, + "prophets": 39441, + "acin": 39442, + "twelf": 39443, + "towering": 39444, + "tml": 39445, + "sclerosis": 39446, + "kwan": 39447, + "gets": 39448, + "disturb": 39449, + "naira": 39450, + "energ": 39451, + "piracy": 39452, + "pruitt": 39453, + "notified": 39454, + "henna": 39455, + "bram": 39456, + "groundwater": 39457, + "bls": 39458, + "optimis": 39459, + "$)": 39460, + "lucie": 39461, + "bizhour": 39462, + "fangirling": 39463, + "grills": 39464, + "orl": 39465, + "verse": 39466, + "cina": 39467, + "lawless": 39468, + "artistsontwitter": 39469, + "televised": 39470, + "marshmallows": 39471, + "radiohead": 39472, + "barr": 39473, + "mfc": 39474, + "brevi": 39475, + "mmorpg": 39476, + "gaya": 39477, + "âĸ«": 39478, + "subtitles": 39479, + "jt": 39480, + "disneyland": 39481, + "tobago": 39482, + "nhm": 39483, + "groove": 39484, + "fiawec": 39485, + "\"/": 39486, + "bao": 39487, + "scrabble": 39488, + "omni": 39489, + "ffl": 39490, + "umc": 39491, + "simba": 39492, + "alier": 39493, + "terrell": 39494, + "plume": 39495, + "midi": 39496, + "dignit": 39497, + "coc": 39498, + "brut": 39499, + "adata": 39500, + "alchemy": 39501, + "dsm": 39502, + "ðŁĺĨðŁĺĨ": 39503, + "wintry": 39504, + "spares": 39505, + "cuer": 39506, + "conclusions": 39507, + "toys": 39508, + "odor": 39509, + "flann": 39510, + "garvey": 39511, + "scriptions": 39512, + "inspections": 39513, + "catap": 39514, + "anglo": 39515, + "stlouis": 39516, + "heimer": 39517, + "atay": 39518, + "trich": 39519, + "enyc": 39520, + "childs": 39521, + "ventil": 39522, + "montp": 39523, + "guillermo": 39524, + "circulare": 39525, + "zell": 39526, + "modeled": 39527, + "craftsman": 39528, + "alina": 39529, + "stimulation": 39530, + "cashew": 39531, + "judas": 39532, + "bestof": 39533, + "toire": 39534, + "suspends": 39535, + "scollege": 39536, + "realising": 39537, + "bytes": 39538, + "bloods": 39539, + "assi": 39540, + "ðŁĴ¿": 39541, + "ohs": 39542, + "ðŁįĭ": 39543, + "scallop": 39544, + "व": 39545, + "gifting": 39546, + "camogie": 39547, + "wilkes": 39548, + "ozzy": 39549, + "ðŁ¤¤": 39550, + "veronic": 39551, + "savoy": 39552, + "demetri": 39553, + "babygirl": 39554, + "ðŁĺįðŁĺŃ": 39555, + "sox": 39556, + "clyde": 39557, + "inductee": 39558, + "countdown": 39559, + "selfcare": 39560, + "à¤ľ": 39561, + "vika": 39562, + "torre": 39563, + "phdchat": 39564, + "pears": 39565, + "awh": 39566, + "suffrage": 39567, + "lesn": 39568, + "admiration": 39569, + "mpp": 39570, + "sharkweek": 39571, + "schulz": 39572, + "santorini": 39573, + "clover": 39574, + "(*": 39575, + "strasbourg": 39576, + "exiting": 39577, + "soyu": 39578, + "fingerprint": 39579, + "chea": 39580, + "ãĢľ": 39581, + "vindic": 39582, + "songwriters": 39583, + "soa": 39584, + "prouder": 39585, + "nama": 39586, + "=))": 39587, + "simplest": 39588, + "deliciously": 39589, + "gilles": 39590, + "uq": 39591, + "mnwx": 39592, + "epp": 39593, + "shun": 39594, + "kennel": 39595, + "fallon": 39596, + "ðŁIJ£": 39597, + "sind": 39598, + "tragically": 39599, + "outes": 39600, + "modernism": 39601, + "coke": 39602, + "gyn": 39603, + "spion": 39604, + "âĺ¹ï¸ı": 39605, + "leam": 39606, + "compressor": 39607, + "apologise": 39608, + "twentyon": 39609, + "fanatics": 39610, + "âĻ»": 39611, + "scotsman": 39612, + "sawa": 39613, + "kou": 39614, + "aser": 39615, + "à¸ļ": 39616, + "welterweight": 39617, + "phenom": 39618, + "twickenham": 39619, + "stria": 39620, + "pout": 39621, + "kaz": 39622, + "giam": 39623, + "cdp": 39624, + "hoy": 39625, + "employ": 39626, + "redmond": 39627, + "à¸Ħà¸": 39628, + "smere": 39629, + "trancefamily": 39630, + "protocols": 39631, + "piece": 39632, + "luiz": 39633, + "iteracy": 39634, + "carls": 39635, + "unitedstates": 39636, + "harmed": 39637, + "phdlife": 39638, + "chaw": 39639, + "footprints": 39640, + "lé": 39641, + "choker": 39642, + "zana": 39643, + "slipper": 39644, + "ericsson": 39645, + "insulting": 39646, + "artichoke": 39647, + "advising": 39648, + "acquisitions": 39649, + "opor": 39650, + "mutations": 39651, + "rear": 39652, + "à¥ģ": 39653, + "podcast": 39654, + "wither": 39655, + "kung": 39656, + "íĺ¸": 39657, + "winslow": 39658, + "diapers": 39659, + "ðŁĵ¸@": 39660, + "ecker": 39661, + "collar": 39662, + "huey": 39663, + "giro": 39664, + "monogram": 39665, + "kasich": 39666, + "siveness": 39667, + "malaysi": 39668, + "aromatic": 39669, + "gres": 39670, + "galileo": 39671, + "uji": 39672, + "robb": 39673, + "drm": 39674, + "nonetheless": 39675, + "asa": 39676, + ":>": 39677, + "loa": 39678, + "lnp": 39679, + "atwork": 39680, + "agt": 39681, + "lakshmi": 39682, + "pipelines": 39683, + "idal": 39684, + "strel": 39685, + "reall": 39686, + "chainz": 39687, + "stonewall": 39688, + "sansk": 39689, + "ðŁı´": 39690, + "piedmont": 39691, + "hostess": 39692, + "ciu": 39693, + "té": 39694, + "analyses": 39695, + "wilhelm": 39696, + "scotty": 39697, + "rwby": 39698, + "mosquit": 39699, + "usemb": 39700, + "quins": 39701, + "ðŁijİ": 39702, + "tucker": 39703, + "sconf": 39704, + "specifications": 39705, + "psychiatry": 39706, + "brookes": 39707, + "sils": 39708, + "olaf": 39709, + "deto": 39710, + "codi": 39711, + "clip": 39712, + "filth": 39713, + "womancrushwednesday": 39714, + "goto": 39715, + "angerous": 39716, + "beale": 39717, + "wtc": 39718, + "panelist": 39719, + "nex": 39720, + "larsen": 39721, + "emilio": 39722, + "tableau": 39723, + "hitters": 39724, + "conceived": 39725, + "americani": 39726, + "ortega": 39727, + "mardi": 39728, + "Ñĥ": 39729, + "paintball": 39730, + "thirsty": 39731, + "newyorker": 39732, + "etisation": 39733, + "goss": 39734, + "weaker": 39735, + "ugh": 39736, + "troll": 39737, + "harga": 39738, + "dual": 39739, + "ghtning": 39740, + "atine": 39741, + "ðŁĺİðŁĺİðŁĺİ": 39742, + "cookout": 39743, + "pyrenees": 39744, + "poss": 39745, + "authentication": 39746, + "sportswear": 39747, + "yunho": 39748, + "kiro": 39749, + "archipel": 39750, + "shenko": 39751, + "render": 39752, + "novation": 39753, + "divinity": 39754, + "ðŁij£": 39755, + "sufi": 39756, + "humbling": 39757, + "geopol": 39758, + "devotees": 39759, + "waitress": 39760, + "trough": 39761, + "pyro": 39762, + "iba": 39763, + "bling": 39764, + "graf": 39765, + "epilots": 39766, + "btr": 39767, + "oftball": 39768, + "basking": 39769, + "dominos": 39770, + "soom": 39771, + "rath": 39772, + "sheryl": 39773, + "quel": 39774, + "astronomical": 39775, + "weld": 39776, + "tracklist": 39777, + "signee": 39778, + "sleepless": 39779, + "comman": 39780, + "chron": 39781, + "summon": 39782, + "puremichigan": 39783, + "crispr": 39784, + "slip": 39785, + "lagi": 39786, + "raq": 39787, + "umu": 39788, + "thalap": 39789, + "charmed": 39790, + "scrump": 39791, + "quadcopter": 39792, + "skip": 39793, + "petersen": 39794, + "muni": 39795, + "ðŁĮ¾": 39796, + "monaghan": 39797, + "trays": 39798, + "icked": 39799, + "canadaday": 39800, + "tegr": 39801, + "�": 39802, + "hotness": 39803, + "heavymetal": 39804, + "abar": 39805, + "gopdebate": 39806, + "azul": 39807, + "spiderman": 39808, + "sunflowers": 39809, + "ľë": 39810, + "webcomics": 39811, + "bard": 39812, + "в": 39813, + "nicholas": 39814, + "slush": 39815, + "raman": 39816, + "markham": 39817, + "fficial": 39818, + "ffler": 39819, + "íĬ¸": 39820, + "pless": 39821, + "anushka": 39822, + "toto": 39823, + "skaters": 39824, + "prowrestling": 39825, + "competes": 39826, + "ayala": 39827, + "mystery": 39828, + "thrills": 39829, + "mpg": 39830, + "independently": 39831, + "yul": 39832, + "imperative": 39833, + "formidable": 39834, + "tireless": 39835, + "stacking": 39836, + "tongues": 39837, + "maltese": 39838, + "potts": 39839, + "matti": 39840, + "charting": 39841, + "chillout": 39842, + "supernova": 39843, + "omeo": 39844, + "skysports": 39845, + "nutty": 39846, + "ðŁĹĵï¸ı": 39847, + "rohan": 39848, + "inspired": 39849, + "concierge": 39850, + "serra": 39851, + "makk": 39852, + "galat": 39853, + "chipp": 39854, + "yev": 39855, + "ì£": 39856, + "reimbur": 39857, + "opul": 39858, + "kimberley": 39859, + "ieee": 39860, + "bremen": 39861, + "chitec": 39862, + "orin": 39863, + "naku": 39864, + "bonkers": 39865, + "footy": 39866, + "emergence": 39867, + "ðŁĨĺ": 39868, + "stip": 39869, + "sergei": 39870, + "zoey": 39871, + "aime": 39872, + "would": 39873, + "dyes": 39874, + "destiny": 39875, + "vinaigrette": 39876, + "drier": 39877, + "circulareconomy": 39878, + "anarchi": 39879, + "ssr": 39880, + "schel": 39881, + "ciner": 39882, + "groom": 39883, + "determining": 39884, + "garmin": 39885, + "calais": 39886, + "incarceration": 39887, + "bukit": 39888, + "noi": 39889, + "chelmsford": 39890, + "mckinley": 39891, + "chipped": 39892, + "belonged": 39893, + "tumors": 39894, + "stroud": 39895, + "mii": 39896, + "influenza": 39897, + "wwenxt": 39898, + "tundra": 39899, + "telecommunications": 39900, + "catsofinstagram": 39901, + "tages": 39902, + "beatty": 39903, + "odu": 39904, + "mlkday": 39905, + "ooper": 39906, + "dangle": 39907, + "akley": 39908, + "crumb": 39909, + "antigua": 39910, + "timbers": 39911, + "rouhani": 39912, + "ðŁĴªðŁĴªðŁĴª": 39913, + "hafi": 39914, + "...!!": 39915, + "wcs": 39916, + "coop": 39917, + "snc": 39918, + "litres": 39919, + "ãĢĬ": 39920, + "haz": 39921, + "coz": 39922, + "kant": 39923, + "greenfield": 39924, + "curti": 39925, + "yale": 39926, + "flyeagles": 39927, + "whatsoever": 39928, + "worthing": 39929, + "roulette": 39930, + "flyeaglesfly": 39931, + "unda": 39932, + "ainted": 39933, + "standing": 39934, + "luscious": 39935, + "hpc": 39936, + "efficacy": 39937, + "ashland": 39938, + "meghan": 39939, + "kywx": 39940, + "npr": 39941, + "bathtub": 39942, + "acos": 39943, + "hani": 39944, + "marcor": 39945, + "mantis": 39946, + "daisi": 39947, + "boba": 39948, + "abbie": 39949, + "mutil": 39950, + "vial": 39951, + "spyder": 39952, + "poz": 39953, + "gti": 39954, + "elfie": 39955, + "nightw": 39956, + "metroid": 39957, + "antoni": 39958, + "maddie": 39959, + "dhry": 39960, + "darlings": 39961, + "tends": 39962, + "taekwondo": 39963, + "atlanta": 39964, + "meow": 39965, + "chloe": 39966, + "ãĥİ": 39967, + "ymes": 39968, + "siberia": 39969, + "kcon": 39970, + "gues": 39971, + "mariner": 39972, + "facil": 39973, + "azzle": 39974, + "[...": 39975, + "hannover": 39976, + "bavaria": 39977, + "virgo": 39978, + "teuk": 39979, + "usps": 39980, + ")#": 39981, + "walla": 39982, + "sampson": 39983, + "needless": 39984, + "verbally": 39985, + "hayley": 39986, + "bowled": 39987, + "pius": 39988, + "lampard": 39989, + "hamstring": 39990, + "volvo": 39991, + "roadsafety": 39992, + "choking": 39993, + "sorbet": 39994, + "ahem": 39995, + "healthyfood": 39996, + "braided": 39997, + "horticulture": 39998, + "crative": 39999, + "cheek": 40000, + "addo": 40001, + "theforce": 40002, + "koko": 40003, + "schizoph": 40004, + "jie": 40005, + "wada": 40006, + "twentyonepilots": 40007, + "hbcu": 40008, + "proton": 40009, + "pauls": 40010, + "louisa": 40011, + "latam": 40012, + "kyrgy": 40013, + "compac": 40014, + "sdk": 40015, + "sapi": 40016, + "???": 40017, + "liberalism": 40018, + "epsilon": 40019, + "aiden": 40020, + "wusa": 40021, + "sprayed": 40022, + "basketball": 40023, + "kimono": 40024, + "bluewave": 40025, + "alias": 40026, + "ë§Ī": 40027, + "mugshot": 40028, + "cec": 40029, + "dogre": 40030, + "adora": 40031, + "ðŁĵ·@": 40032, + "krakow": 40033, + "intrigued": 40034, + "exhausting": 40035, + "astronomer": 40036, + "venison": 40037, + "ladybug": 40038, + "civ": 40039, + "brae": 40040, + "usm": 40041, + "bribe": 40042, + "acupuncture": 40043, + "pembroke": 40044, + "keating": 40045, + "chie": 40046, + "yad": 40047, + "tsi": 40048, + "smi": 40049, + "seeding": 40050, + "gateshead": 40051, + "lisboa": 40052, + "gyp": 40053, + "canvass": 40054, + "ðŁĶ´âļªï¸ı": 40055, + "opi": 40056, + "nir": 40057, + "societal": 40058, + "lyte": 40059, + "aties": 40060, + "csm": 40061, + "artery": 40062, + "alin": 40063, + "akapoor": 40064, + "abstracts": 40065, + "âĢ¦âĢ¦": 40066, + "teenwolf": 40067, + "newe": 40068, + "travelgram": 40069, + "sentimental": 40070, + "perched": 40071, + "handel": 40072, + "hoek": 40073, + "fay": 40074, + "coordinating": 40075, + "animate": 40076, + "manian": 40077, + "effort": 40078, + "jerky": 40079, + "fck": 40080, + "adrienne": 40081, + "mably": 40082, + "trading": 40083, + "myel": 40084, + "spiro": 40085, + "sola": 40086, + "storing": 40087, + "overdrive": 40088, + "mondaymorning": 40089, + "dreamteam": 40090, + "pulse": 40091, + "bondi": 40092, + "bernie": 40093, + "pgatour": 40094, + "tripoli": 40095, + "sonam": 40096, + "platt": 40097, + "âļ¡": 40098, + "agroup": 40099, + "îIJĴ": 40100, + "invading": 40101, + "vcu": 40102, + "kell": 40103, + "ños": 40104, + "undead": 40105, + "podcasting": 40106, + "mercedesam": 40107, + "manafort": 40108, + "cortex": 40109, + "queso": 40110, + "impeccable": 40111, + "palmer": 40112, + "wildoz": 40113, + "sportsc": 40114, + "guacamole": 40115, + "dispenser": 40116, + "categori": 40117, + "stunts": 40118, + "peril": 40119, + "invitations": 40120, + "dunedin": 40121, + "xie": 40122, + "achieves": 40123, + "safer": 40124, + "preds": 40125, + "phan": 40126, + "knuckles": 40127, + "kak": 40128, + "ignores": 40129, + "lovemyjob": 40130, + "aruba": 40131, + "oundation": 40132, + "datacenter": 40133, + "covert": 40134, + "gring": 40135, + "couple": 40136, + "ار": 40137, + "voli": 40138, + "mccle": 40139, + "artisans": 40140, + "ludo": 40141, + "kalam": 40142, + "aroma": 40143, + "undertaker": 40144, + "hula": 40145, + "wizkid": 40146, + "gumb": 40147, + "godfrey": 40148, + "bakersfield": 40149, + "kern": 40150, + "engineer": 40151, + "carve": 40152, + "palin": 40153, + "guarantees": 40154, + "pebbles": 40155, + "bays": 40156, + "zieg": 40157, + "fink": 40158, + "â¬ĩï¸ıâ¬ĩï¸ı": 40159, + "downpours": 40160, + "rochelle": 40161, + "raspberry": 40162, + "ðŁĺ®": 40163, + "graphies": 40164, + "stomp": 40165, + "cafes": 40166, + "arized": 40167, + "uttar": 40168, + "calvary": 40169, + "drie": 40170, + "crusader": 40171, + "busan": 40172, + "tuxedo": 40173, + "siu": 40174, + "seamus": 40175, + "cultured": 40176, + "blanchard": 40177, + "townhouse": 40178, + "gered": 40179, + "buttermilk": 40180, + "fluctu": 40181, + "rogerfederer": 40182, + "heli": 40183, + "ðŁ¦ĥ": 40184, + "uous": 40185, + "ramesh": 40186, + "muppets": 40187, + "emailmarketing": 40188, + "yess": 40189, + "brice": 40190, + "rizio": 40191, + "pelo": 40192, + "donneinarte": 40193, + "urable": 40194, + "investin": 40195, + "bumping": 40196, + "rajiv": 40197, + "sava": 40198, + "thrower": 40199, + "forex": 40200, + "ohhhh": 40201, + "thrust": 40202, + "pullman": 40203, + "rfid": 40204, + "sepsis": 40205, + "leed": 40206, + "fright": 40207, + "rounding": 40208, + "neb": 40209, + "phins": 40210, + "aisha": 40211, + "utilizing": 40212, + "squats": 40213, + "goldsmith": 40214, + "jic": 40215, + "boks": 40216, + "vaus": 40217, + "ipo": 40218, + "exclusion": 40219, + "tariff": 40220, + "pokes": 40221, + "minal": 40222, + "lands": 40223, + "enforce": 40224, + "washingtondc": 40225, + "orchar": 40226, + "gx": 40227, + "marys": 40228, + "eyour": 40229, + "aussie": 40230, + "bakers": 40231, + "unpopular": 40232, + "latinos": 40233, + "large": 40234, + "putnam": 40235, + "bolo": 40236, + "wade": 40237, + "pelo": 40238, + "dizz": 40239, + "obstruction": 40240, + "flappy": 40241, + "wearethe": 40242, + "dependence": 40243, + "pajama": 40244, + "ete": 40245, + "yann": 40246, + "ewan": 40247, + "discla": 40248, + "aay": 40249, + "karina": 40250, + "eic": 40251, + "antrim": 40252, + "wsoc": 40253, + "negatively": 40254, + "kaido": 40255, + "fotografia": 40256, + "dhru": 40257, + "colossal": 40258, + "mcleod": 40259, + "kwang": 40260, + "manipu": 40261, + "exhilar": 40262, + "usatoday": 40263, + "summerslam": 40264, + "coles": 40265, + "taproom": 40266, + "unbeatable": 40267, + "dema": 40268, + "ticks": 40269, + "kling": 40270, + "fils": 40271, + "campaigners": 40272, + "à¸ķ": 40273, + "brewster": 40274, + "audubon": 40275, + "quay": 40276, + "chs": 40277, + "kigali": 40278, + "dler": 40279, + "strengthens": 40280, + "somal": 40281, + "signingday": 40282, + "golds": 40283, + "pigment": 40284, + "orchestral": 40285, + "gq": 40286, + "linkin": 40287, + "ðŁıĩ": 40288, + "taw": 40289, + "algarve": 40290, + "hov": 40291, + "earle": 40292, + "goldfish": 40293, + "amig": 40294, + "exer": 40295, + "benin": 40296, + "druid": 40297, + "ðŁIJ¸": 40298, + "shem": 40299, + "quattro": 40300, + "mercen": 40301, + "mente": 40302, + "incorporating": 40303, + "bonanza": 40304, + "statefair": 40305, + "ende": 40306, + "conceptions": 40307, + "ees": 40308, + "âĻ¥ï¸ıâĻ¥ï¸ı": 40309, + "dson": 40310, + "firearm": 40311, + "orbital": 40312, + "weh": 40313, + "multip": 40314, + "fob": 40315, + "requiem": 40316, + "plight": 40317, + "thouse": 40318, + "said": 40319, + "ocre": 40320, + "remembrance": 40321, + "nold": 40322, + "chipping": 40323, + "bev": 40324, + "ert": 40325, + "cathy": 40326, + "sym": 40327, + "riggs": 40328, + "mley": 40329, + "dialogues": 40330, + "slender": 40331, + "howl": 40332, + "gauteng": 40333, + "wdw": 40334, + "tobi": 40335, + "smokes": 40336, + "implo": 40337, + "bpm": 40338, + "adn": 40339, + "mombasa": 40340, + "capsul": 40341, + "bloomfield": 40342, + "articul": 40343, + "cleo": 40344, + "googled": 40345, + "fluffy": 40346, + "lard": 40347, + "enzyme": 40348, + "vesti": 40349, + "ibrahi": 40350, + "flame": 40351, + "emea": 40352, + "outages": 40353, + "dispropor": 40354, + "bleak": 40355, + "ansel": 40356, + "icker": 40357, + "stlouis": 40358, + "stockmarket": 40359, + "goodfriday": 40360, + "sault": 40361, + "stalled": 40362, + "prom": 40363, + "epsom": 40364, + "bé": 40365, + "these": 40366, + "sauces": 40367, + "mew": 40368, + "litfest": 40369, + "pred": 40370, + "reu": 40371, + "karak": 40372, + "sienna": 40373, + "ellin": 40374, + "biotechnology": 40375, + "ï¸ıâĥ£-": 40376, + "tactic": 40377, + "sain": 40378, + "pork": 40379, + "monza": 40380, + "kaj": 40381, + "lush": 40382, + "compartment": 40383, + "changing": 40384, + "shraddhakapoor": 40385, + "foal": 40386, + "artem": 40387, + "cuando": 40388, + "canola": 40389, + "oriente": 40390, + "messe": 40391, + "dited": 40392, + "brc": 40393, + "boxer": 40394, + "bbctwo": 40395, + "sst": 40396, + "mentday": 40397, + "eming": 40398, + "dewey": 40399, + "kofi": 40400, + "âŀĸâŀĸâŀĸâŀĸ": 40401, + "realization": 40402, + "smol": 40403, + "twood": 40404, + "sanje": 40405, + "flagstaff": 40406, + "berwick": 40407, + "corset": 40408, + "canary": 40409, + "whistleblower": 40410, + "etched": 40411, + "composing": 40412, + "squeezed": 40413, + "bower": 40414, + "autodesk": 40415, + "neh": 40416, + "mathieu": 40417, + "baja": 40418, + "ÅĤ": 40419, + "hydra": 40420, + "daim": 40421, + "ameri": 40422, + "insisted": 40423, + "merlot": 40424, + "garros": 40425, + "heartnews": 40426, + "gainesville": 40427, + "cutler": 40428, + "bode": 40429, + "ðŁĺīðŁĺī": 40430, + "lewes": 40431, + "scountry": 40432, + "gsa": 40433, + "usu": 40434, + "ccm": 40435, + "godawgs": 40436, + "pharaoh": 40437, + "crae": 40438, + "morley": 40439, + "hypnoti": 40440, + "fades": 40441, + "neurons": 40442, + "fuzz": 40443, + "ingco": 40444, + "highlanders": 40445, + "stark": 40446, + "vigne": 40447, + "packets": 40448, + "amarillo": 40449, + "reuben": 40450, + "insults": 40451, + "basic": 40452, + "vector": 40453, + "nme": 40454, + "acruz": 40455, + "tros": 40456, + "transmitter": 40457, + "ðŁĺŀ": 40458, + "interpret": 40459, + "ðŁĺ²": 40460, + "prequel": 40461, + "mcgowan": 40462, + "dissemin": 40463, + "ðŁĴĺðŁĴĺ": 40464, + "masculinity": 40465, + "indiegamedev": 40466, + "alive": 40467, + "tet": 40468, + "petal": 40469, + "emailed": 40470, + "armed": 40471, + "koo": 40472, + "heer": 40473, + "baird": 40474, + "superjunior": 40475, + "metropolis": 40476, + "delavin": 40477, + "declines": 40478, + "stitutes": 40479, + "Ûģ": 40480, + "ptbo": 40481, + "glan": 40482, + "chores": 40483, + "ealing": 40484, + "chrissy": 40485, + "stemc": 40486, + "vian": 40487, + "assassinated": 40488, + "pronounce": 40489, + "illegals": 40490, + "discovery": 40491, + "cavill": 40492, + "frifotos": 40493, + "fal": 40494, + "soi": 40495, + "sabotage": 40496, + "tint": 40497, + "pdc": 40498, + "ðŁİīðŁİĪ": 40499, + "ãĤĬãģ": 40500, + "jio": 40501, + "endeavor": 40502, + "insig": 40503, + "committees": 40504, + "shearer": 40505, + "metz": 40506, + "marrying": 40507, + "hdd": 40508, + "gby": 40509, + "fret": 40510, + "trish": 40511, + "pul": 40512, + "scripted": 40513, + "saki": 40514, + "lw": 40515, + "keye": 40516, + "shimi": 40517, + "nanaimo": 40518, + "cah": 40519, + "ë": 40520, + "tempered": 40521, + "ician": 40522, + "dugg": 40523, + "dishwasher": 40524, + "airfield": 40525, + "srugby": 40526, + "grinch": 40527, + "yst": 40528, + "rms": 40529, + "mahatma": 40530, + "lankan": 40531, + "discar": 40532, + "digestion": 40533, + "nodes": 40534, + "lls": 40535, + "omic": 40536, + "gutter": 40537, + "tisgarh": 40538, + "federico": 40539, + "electionday": 40540, + "bohe": 40541, + "mastercard": 40542, + "fireball": 40543, + "âľĶï¸ı": 40544, + "oyster": 40545, + "pong": 40546, + "dok": 40547, + "enroute": 40548, + "mvc": 40549, + "beatthe": 40550, + "alistair": 40551, + "shub": 40552, + "shaming": 40553, + "chernobyl": 40554, + "ghibli": 40555, + "thes": 40556, + "pinion": 40557, + "dbs": 40558, + "salts": 40559, + "iction": 40560, + "epiph": 40561, + "ncpol": 40562, + "inconvenience": 40563, + "whitley": 40564, + "inspecting": 40565, + "woodley": 40566, + "wiener": 40567, + "skillet": 40568, + "noles": 40569, + "mca": 40570, + "hina": 40571, + "asha": 40572, + "willingness": 40573, + "wellness": 40574, + "tamed": 40575, + "showtime": 40576, + "disadvantaged": 40577, + "bernat": 40578, + "usn": 40579, + "missionaries": 40580, + "counselling": 40581, + "arrogant": 40582, + "quantitative": 40583, + "legalization": 40584, + "hodge": 40585, + "energyefficiency": 40586, + "camerondallas": 40587, + "possessions": 40588, + "pbb": 40589, + "harrisburg": 40590, + "vg": 40591, + "hinduism": 40592, + "happythanksgiving": 40593, + "fib": 40594, + "reacting": 40595, + "tweetapicture": 40596, + "politi": 40597, + "muppet": 40598, + "hurrah": 40599, + "pace": 40600, + "coastguard": 40601, + "guarded": 40602, + "asam": 40603, + "parry": 40604, + "forevery": 40605, + "xq": 40606, + "oomf": 40607, + "keanu": 40608, + "jind": 40609, + "rist": 40610, + "customerservice": 40611, + "sacred": 40612, + "ðŁĺº": 40613, + "toner": 40614, + "occurrence": 40615, + "matu": 40616, + "valdez": 40617, + "redd": 40618, + "isak": 40619, + "powerrangers": 40620, + "peasant": 40621, + "rajini": 40622, + "abraham": 40623, + "emil": 40624, + "cardo": 40625, + "tril": 40626, + "hairstyles": 40627, + "obsolete": 40628, + "sampler": 40629, + "directive": 40630, + "delavinkisses": 40631, + "verton": 40632, + "glos": 40633, + "spay": 40634, + "palermo": 40635, + "comets": 40636, + "manziel": 40637, + "chicagof": 40638, + "skipped": 40639, + "pictorial": 40640, + "hant": 40641, + "bmi": 40642, + "aol": 40643, + "reopens": 40644, + "paddling": 40645, + "devos": 40646, + "fraud": 40647, + "baseline": 40648, + "queues": 40649, + "spired": 40650, + "snare": 40651, + "euve": 40652, + "descriptions": 40653, + "daisies": 40654, + "caching": 40655, + "galleria": 40656, + "trimmed": 40657, + "stino": 40658, + "recycla": 40659, + "icular": 40660, + "birken": 40661, + "rawlings": 40662, + "flix": 40663, + "chicas": 40664, + "bgt": 40665, + "likeli": 40666, + "argyll": 40667, + "thelove": 40668, + "gaston": 40669, + "blanca": 40670, + "hak": 40671, + "fone": 40672, + "sailormoon": 40673, + "haci": 40674, + "imac": 40675, + "flyn": 40676, + "decan": 40677, + "belles": 40678, + "apic": 40679, + "zog": 40680, + "taunton": 40681, + "constance": 40682, + "lasagna": 40683, + "kernel": 40684, + "inka": 40685, + "harbor": 40686, + "collectively": 40687, + "calculated": 40688, + "aville": 40689, + "shilpa": 40690, + "purdu": 40691, + "gimm": 40692, + "funer": 40693, + "aest": 40694, + "pembrokeshire": 40695, + "nightingale": 40696, + "nunes": 40697, + "hypertension": 40698, + "hubert": 40699, + "sliders": 40700, + "infertility": 40701, + "commended": 40702, + "transatlantic": 40703, + "metrical": 40704, + "!!@": 40705, + "ÅŁ": 40706, + "ssg": 40707, + "bacca": 40708, + "inverted": 40709, + "funfactfriday": 40710, + "itans": 40711, + "album": 40712, + "acquainted": 40713, + "rier": 40714, + "whelan": 40715, + "sarab": 40716, + "mue": 40717, + "snooze": 40718, + "piff": 40719, + "agreeing": 40720, + "spitting": 40721, + "jermaine": 40722, + "nye": 40723, + "âľıï¸ı": 40724, + "ambush": 40725, + "zeph": 40726, + "congreg": 40727, + "university": 40728, + "sapp": 40729, + "wannabe": 40730, + "patrice": 40731, + "ibd": 40732, + "doglo": 40733, + "fridges": 40734, + "sund": 40735, + "kingston": 40736, + "argon": 40737, + "kamen": 40738, + "hardrock": 40739, + "dsley": 40740, + "dolores": 40741, + "ì°": 40742, + "otaku": 40743, + "piping": 40744, + "behaving": 40745, + "âŃIJï¸ıâŃIJï¸ıâŃIJï¸ı": 40746, + "bluebird": 40747, + "ansari": 40748, + "teapot": 40749, + "firework": 40750, + "crop": 40751, + "logans": 40752, + "typed": 40753, + "thickness": 40754, + "igers": 40755, + "cfp": 40756, + "dysfunctional": 40757, + "contrasting": 40758, + "etty": 40759, + "astonmartin": 40760, + "txst": 40761, + "dragrace": 40762, + "attributes": 40763, + "marathon": 40764, + "manuscripts": 40765, + "johnstone": 40766, + "ðŁĺ±ðŁĺ±": 40767, + "boer": 40768, + "ayu": 40769, + "arugula": 40770, + "poorest": 40771, + "condu": 40772, + "assumption": 40773, + "anagh": 40774, + "noh": 40775, + "delavin": 40776, + "sitter": 40777, + "gö": 40778, + "morow": 40779, + "kickstart": 40780, + "comi": 40781, + "glacial": 40782, + "ghead": 40783, + "bain": 40784, + "kershaw": 40785, + "endof": 40786, + "freud": 40787, + "omat": 40788, + "iaf": 40789, + "hug": 40790, + "signup": 40791, + "eachother": 40792, + "definite": 40793, + "tubing": 40794, + "shakira": 40795, + "ðŁijıðŁı½": 40796, + "uuuu": 40797, + "swin": 40798, + "shambles": 40799, + "olas": 40800, + "skell": 40801, + "britain": 40802, + "knw": 40803, + "clutter": 40804, + "omy": 40805, + "jens": 40806, + "hanged": 40807, + "cityscape": 40808, + "scraps": 40809, + "unlocking": 40810, + "deadliest": 40811, + "erno": 40812, + "breastcancer": 40813, + "ait": 40814, + "inspect": 40815, + "furi": 40816, + "ðŁĴĮ": 40817, + "kud": 40818, + "jule": 40819, + "orah": 40820, + "mids": 40821, + "mdt": 40822, + "burgring": 40823, + "rattle": 40824, + "pusa": 40825, + "stalk": 40826, + "cleans": 40827, + "issance": 40828, + "zek": 40829, + "worthit": 40830, + "nameis": 40831, + "muskoka": 40832, + "councilman": 40833, + "urbanart": 40834, + "barrac": 40835, + "unsolved": 40836, + "tul": 40837, + "gita": 40838, + "whiteboard": 40839, + "soybeans": 40840, + "ement": 40841, + "conti": 40842, + "saturdaymotivation": 40843, + "conveniently": 40844, + "docking": 40845, + "tado": 40846, + "âı©": 40847, + "spino": 40848, + "puppylove": 40849, + "pof": 40850, + "fabricated": 40851, + "robbers": 40852, + "adopts": 40853, + "tified": 40854, + "kkr": 40855, + "indulgence": 40856, + "noticeable": 40857, + "macquarie": 40858, + "chapel": 40859, + "sensual": 40860, + "kiko": 40861, + "melanoma": 40862, + "loretta": 40863, + "liance": 40864, + "aben": 40865, + "splus": 40866, + "gaal": 40867, + "acele": 40868, + "libdems": 40869, + "comparisons": 40870, + "ðŁĮµ": 40871, + "rhythms": 40872, + "mery": 40873, + "encapsul": 40874, + "napier": 40875, + "ðŁijĮðŁijĮðŁijĮ": 40876, + "ðŁijIJ": 40877, + "platz": 40878, + "fresno": 40879, + "reformed": 40880, + "ranbir": 40881, + "elit": 40882, + "thebest": 40883, + "bhushan": 40884, + "vinnie": 40885, + "improvised": 40886, + "sittin": 40887, + "recreated": 40888, + "eba": 40889, + "ecker": 40890, + "acrob": 40891, + "ponte": 40892, + "cord": 40893, + "giddy": 40894, + "eurusd": 40895, + "fever": 40896, + "intuition": 40897, + "gari": 40898, + "dummies": 40899, + "budweiser": 40900, + "amendments": 40901, + "tetra": 40902, + "schnit": 40903, + "ayas": 40904, + "marys": 40905, + "cist": 40906, + "kani": 40907, + "kermit": 40908, + "ðŁĺ±ðŁĺ±ðŁĺ±": 40909, + "tinker": 40910, + "strolling": 40911, + "divisional": 40912, + "nigeri": 40913, + "ominous": 40914, + "menstrual": 40915, + "karab": 40916, + "khy": 40917, + "bwfc": 40918, + "panhandle": 40919, + "lilli": 40920, + "weller": 40921, + "strapped": 40922, + "sonthe": 40923, + "transferring": 40924, + "ethereal": 40925, + "sneaks": 40926, + "rudol": 40927, + "gables": 40928, + "jacking": 40929, + "cincode": 40930, + "fortune": 40931, + "canadiens": 40932, + "confor": 40933, + "abnormal": 40934, + "franklin": 40935, + "tita": 40936, + "mula": 40937, + "persist": 40938, + "cuties": 40939, + "kiel": 40940, + "ðŁĩ±ðŁĩ": 40941, + "hermann": 40942, + "awk": 40943, + "fiasco": 40944, + "koto": 40945, + "weta": 40946, + "hiker": 40947, + "buddy": 40948, + "preventive": 40949, + "mcgraw": 40950, + "gameboy": 40951, + "forsyth": 40952, + "topshop": 40953, + "siob": 40954, + "sadh": 40955, + "intram": 40956, + "followart": 40957, + "soaps": 40958, + "dragonball": 40959, + "oux": 40960, + "morrison": 40961, + "à¹ĥ": 40962, + "lubric": 40963, + "adulthood": 40964, + "morrisons": 40965, + "âļłï¸ı": 40966, + "hermo": 40967, + "taka": 40968, + "stallone": 40969, + "misuse": 40970, + "teamgb": 40971, + "ragha": 40972, + "confined": 40973, + "aty": 40974, + "homophobic": 40975, + "nwo": 40976, + "skynews": 40977, + "hoya": 40978, + "acrosse": 40979, + "wiiu": 40980, + "purée": 40981, + "jeddah": 40982, + "ðŁ¤§": 40983, + "advisers": 40984, + "phine": 40985, + "anis": 40986, + "scrumptious": 40987, + "ë°ķ": 40988, + "cke": 40989, + "viny": 40990, + "term": 40991, + "sdc": 40992, + "odo": 40993, + "homeschool": 40994, + "vasc": 40995, + "leopards": 40996, + "deborah": 40997, + "illicit": 40998, + "curran": 40999, + "asroma": 41000, + "naught": 41001, + "marig": 41002, + "brandi": 41003, + "emp": 41004, + "ðŁĺįðŁijĮ": 41005, + "îĮ": 41006, + "suspend": 41007, + "luz": 41008, + "initiation": 41009, + "schaft": 41010, + "jensenackles": 41011, + "crawler": 41012, + "postdoc": 41013, + "desks": 41014, + "trailblazer": 41015, + "denomin": 41016, + "trix": 41017, + "noise": 41018, + "poet": 41019, + "±ï¸ı": 41020, + "smug": 41021, + "volatile": 41022, + "proofs": 41023, + "pharmacist": 41024, + "sardinia": 41025, + "mashable": 41026, + "kimchi": 41027, + "coed": 41028, + "schalke": 41029, + "doodled": 41030, + "csw": 41031, + "shur": 41032, + "rox": 41033, + "dok": 41034, + "chrisbrown": 41035, + "mathematician": 41036, + "abound": 41037, + "angelic": 41038, + "rockford": 41039, + "dole": 41040, + "yorkers": 41041, + "msn": 41042, + "gman": 41043, + "xavier": 41044, + "borrowing": 41045, + "markings": 41046, + "longhorn": 41047, + "kja": 41048, + "diverted": 41049, + "mmit": 41050, + "euphoria": 41051, + "ayyy": 41052, + "tea": 41053, + "pah": 41054, + "cki": 41055, + "uncut": 41056, + "liven": 41057, + "kyung": 41058, + "fanart": 41059, + "mering": 41060, + "redding": 41061, + "amovie": 41062, + "gridi": 41063, + "cthulhu": 41064, + "scholarly": 41065, + "judah": 41066, + "thbewithyou": 41067, + "eucalyp": 41068, + "ðŁIJķ": 41069, + "hertfordshire": 41070, + "courtroom": 41071, + "byu": 41072, + "auctioned": 41073, + "please": 41074, + "marcia": 41075, + "ê°ĵ": 41076, + "succeeded": 41077, + "elas": 41078, + "arvind": 41079, + "tlot": 41080, + "saigon": 41081, + "rett": 41082, + "rakesh": 41083, + "fdny": 41084, + "asen": 41085, + "sebring": 41086, + "gladiators": 41087, + "youknow": 41088, + "vlad": 41089, + "gola": 41090, + "parap": 41091, + "ÑĢи": 41092, + "sabcnews": 41093, + "oneteam": 41094, + "ohl": 41095, + "sune": 41096, + "rij": 41097, + "cdc": 41098, + "stargate": 41099, + "rundown": 41100, + "plato": 41101, + "phc": 41102, + "chatter": 41103, + "raviol": 41104, + "mnf": 41105, + "mandala": 41106, + "liet": 41107, + "à¸ķ": 41108, + "maria": 41109, + "hungover": 41110, + "consolidation": 41111, + "ferrell": 41112, + "traditional": 41113, + "iloveart": 41114, + "galap": 41115, + "ðŁıĮ": 41116, + "quezon": 41117, + "españa": 41118, + "ðŁĩ¨ðŁĩŃ": 41119, + "hobby": 41120, + "steamboat": 41121, + "malign": 41122, + "guillau": 41123, + "prohi": 41124, + "itsme": 41125, + "íĥĢ": 41126, + "inscription": 41127, + "alz": 41128, + "marian": 41129, + "kade": 41130, + "mmon": 41131, + "adjusting": 41132, + "nests": 41133, + "internally": 41134, + "cir": 41135, + "vikram": 41136, + "malala": 41137, + "kph": 41138, + "felicia": 41139, + "thereal": 41140, + "captivity": 41141, + "atis": 41142, + "marcorubio": 41143, + "kaleido": 41144, + "chev": 41145, + "manoj": 41146, + "lemore": 41147, + "gentri": 41148, + "vips": 41149, + "trope": 41150, + "\"âĢĶ": 41151, + "pairings": 41152, + "malnutrition": 41153, + "fray": 41154, + "designation": 41155, + "brunomars": 41156, + "aze": 41157, + "torrential": 41158, + "panzer": 41159, + "gail": 41160, + "underthe": 41161, + "theological": 41162, + "schizophre": 41163, + "dazzle": 41164, + "frederic": 41165, + "mopar": 41166, + "adilla": 41167, + "soggy": 41168, + "raun": 41169, + "mediocre": 41170, + "colorec": 41171, + "ife": 41172, + "pinst": 41173, + "bluef": 41174, + "²": 41175, + "worldwater": 41176, + "giroud": 41177, + "clarinet": 41178, + "adolf": 41179, + "tarantino": 41180, + "receipts": 41181, + "assump": 41182, + "ðŁijŁ": 41183, + "coffees": 41184, + "âľĬðŁı¾": 41185, + "duplex": 41186, + "sof": 41187, + "rx": 41188, + "lino": 41189, + "timberwolves": 41190, + "pandit": 41191, + "motm": 41192, + "ega": 41193, + "ayama": 41194, + "achs": 41195, + "outsider": 41196, + "llen": 41197, + "coer": 41198, + "tilly": 41199, + "cheeseburger": 41200, + "mads": 41201, + "pledis": 41202, + "empty": 41203, + "nationalparks": 41204, + "aziz": 41205, + "pmi": 41206, + "junkies": 41207, + "fener": 41208, + "sqn": 41209, + "ès": 41210, + "generation": 41211, + "cleopatra": 41212, + "bhubanes": 41213, + "mosques": 41214, + "tyfree": 41215, + "poppins": 41216, + "twc": 41217, + "orwell": 41218, + "nage": 41219, + "kawhi": 41220, + "hollow": 41221, + "dalai": 41222, + "¨¨¨¨": 41223, + "ouro": 41224, + "mhealth": 41225, + "gion": 41226, + "azo": 41227, + "visas": 41228, + "renegade": 41229, + "reic": 41230, + "wsop": 41231, + "ðŁĴļðŁĴĽ": 41232, + "echel": 41233, + "toxicity": 41234, + "mün": 41235, + "bunk": 41236, + "stimulating": 41237, + "asthour": 41238, + "\\'": 41239, + "eph": 41240, + "endemic": 41241, + "cnbc": 41242, + "shrinking": 41243, + "peabody": 41244, + "michelangelo": 41245, + "canyon": 41246, + "wale": 41247, + "sumi": 41248, + "siders": 41249, + "inuit": 41250, + "?.": 41251, + "professionalism": 41252, + "dracing": 41253, + "platoon": 41254, + "pons": 41255, + "outbound": 41256, + "mapleleafs": 41257, + "desol": 41258, + "cency": 41259, + "athan": 41260, + "verma": 41261, + "rubbing": 41262, + "okan": 41263, + "ðŁijł": 41264, + "mullins": 41265, + "authentic": 41266, + "Åį": 41267, + "almanac": 41268, + "gaia": 41269, + "bbq": 41270, + "onimo": 41271, + "keh": 41272, + "tya": 41273, + "touts": 41274, + "yav": 41275, + "reposit": 41276, + ",.": 41277, + "wight": 41278, + "seeyou": 41279, + "callof": 41280, + "donesia": 41281, + "bargaining": 41282, + "granth": 41283, + "sdsu": 41284, + "amphitheater": 41285, + "psu": 41286, + "rewatching": 41287, + "winetasting": 41288, + "peakdistrict": 41289, + "detecting": 41290, + "thurman": 41291, + "phee": 41292, + "èªķ": 41293, + "umich": 41294, + "rer": 41295, + "sculpted": 41296, + "gole": 41297, + "namesake": 41298, + "ðŁĶģ": 41299, + "servicing": 41300, + "baugh": 41301, + "pugh": 41302, + "pencil": 41303, + "darth": 41304, + "munchkin": 41305, + "atorium": 41306, + "teners": 41307, + "suny": 41308, + "rollingstones": 41309, + "maging": 41310, + "starrer": 41311, + "idris": 41312, + "feinstein": 41313, + "agron": 41314, + "âĺºï¸ıâĺºï¸ı": 41315, + "supervised": 41316, + "chameleon": 41317, + "aggregate": 41318, + "successive": 41319, + "mogul": 41320, + "instyle": 41321, + "poldark": 41322, + "custome": 41323, + "ohiostate": 41324, + "haya": 41325, + "cides": 41326, + "brokerage": 41327, + "angelou": 41328, + "fifawwc": 41329, + "deforestation": 41330, + "alton": 41331, + "pamph": 41332, + "hugged": 41333, + "hobo": 41334, + "changeable": 41335, + "kuber": 41336, + "burroughs": 41337, + "demonetisation": 41338, + "capecod": 41339, + "versatility": 41340, + "orice": 41341, + "leila": 41342, + "womeninscience": 41343, + "tua": 41344, + "hedges": 41345, + "embarrassment": 41346, + "alife": 41347, + "soars": 41348, + "nighter": 41349, + "hymn": 41350, + "gipp": 41351, + "chasu": 41352, + "techs": 41353, + "niall": 41354, + "killa": 41355, + "hika": 41356, + "camels": 41357, + "value": 41358, + "¢": 41359, + "scoops": 41360, + "mahmoud": 41361, + "clusive": 41362, + "adriana": 41363, + "paco": 41364, + "ozil": 41365, + "unas": 41366, + "translations": 41367, + "whisperer": 41368, + "sbi": 41369, + "buxton": 41370, + "biotics": 41371, + "indiffe": 41372, + "kenney": 41373, + "klar": 41374, + "etching": 41375, + "barrabest": 41376, + "instability": 41377, + "seine": 41378, + "votel": 41379, + "blogged": 41380, + "whiskey": 41381, + "myspace": 41382, + "tant": 41383, + "landia": 41384, + "giveback": 41385, + "illus": 41386, + "awak": 41387, + "acab": 41388, + "fbloggers": 41389, + "cloudcomputing": 41390, + "blatant": 41391, + "syrians": 41392, + "bandra": 41393, + "styn": 41394, + "anem": 41395, + "keted": 41396, + "karthik": 41397, + "barunsob": 41398, + "pinot": 41399, + "gubernat": 41400, + "gaye": 41401, + "artiste": 41402, + "ified": 41403, + "conventions": 41404, + "huan": 41405, + "geniuses": 41406, + "eeeeee": 41407, + "folly": 41408, + "somerville": 41409, + "pridemonth": 41410, + "ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸": 41411, + "chemotherapy": 41412, + "pauls": 41413, + "bakar": 41414, + "ìĦ¸ë¸IJ": 41415, + "taiwanese": 41416, + "follo": 41417, + "css": 41418, + "reign": 41419, + "nnnn": 41420, + "flaun": 41421, + "catastrophe": 41422, + "ities": 41423, + "fragments": 41424, + "extremists": 41425, + "ymoun": 41426, + "carmen": 41427, + "ezekiel": 41428, + "connecting": 41429, + "seh": 41430, + "manta": 41431, + "remodeling": 41432, + "weymouth": 41433, + "atoms": 41434, + "cem": 41435, + "newell": 41436, + "lumi": 41437, + "theopen": 41438, + "moc": 41439, + "miliband": 41440, + "gland": 41441, + "zshq": 41442, + "maggie": 41443, + "maniacs": 41444, + "msp": 41445, + "ady": 41446, + "creams": 41447, + "leanne": 41448, + "esta": 41449, + "pyg": 41450, + "affinity": 41451, + "prayer": 41452, + "dunbar": 41453, + "lightroom": 41454, + "acadi": 41455, + "wynonna": 41456, + "romantic": 41457, + "statedept": 41458, + "sickle": 41459, + "whos": 41460, + "lamo": 41461, + "etour": 41462, + "finity": 41463, + "shrub": 41464, + "sharpen": 41465, + "pundit": 41466, + "edon": 41467, + "afore": 41468, + "mars": 41469, + "jeffery": 41470, + "terps": 41471, + "medallist": 41472, + "katharine": 41473, + "accusing": 41474, + "taz": 41475, + "royd": 41476, + "fromhome": 41477, + "confrontation": 41478, + "allegh": 41479, + "ðŁijīðŁijī": 41480, + "refresher": 41481, + "ranveer": 41482, + "neverland": 41483, + "jojo": 41484, + "lucrative": 41485, + "enam": 41486, + "caver": 41487, + "paedi": 41488, + "manjaro": 41489, + "fluids": 41490, + "thessal": 41491, + "oppressed": 41492, + "muss": 41493, + "johanna": 41494, + "Ø®": 41495, + "cng": 41496, + "buildthe": 41497, + "settles": 41498, + "sith": 41499, + "fuego": 41500, + "clamp": 41501, + "arag": 41502, + "payer": 41503, + "tedx": 41504, + "mandy": 41505, + "interstellar": 41506, + "frc": 41507, + "chand": 41508, + "bcc": 41509, + "molo": 41510, + "lentil": 41511, + "johansson": 41512, + "grimsby": 41513, + "naturelovers": 41514, + "ðŁļ¨ðŁļ¨ðŁļ¨": 41515, + "shinde": 41516, + "xin": 41517, + "internationaldayof": 41518, + "transitional": 41519, + "sata": 41520, + "caddy": 41521, + "wod": 41522, + "ifu": 41523, + "hays": 41524, + "hollyo": 41525, + "jang": 41526, + "irc": 41527, + "coim": 41528, + "gradable": 41529, + "\"\"": 41530, + "ðŁį´": 41531, + "া": 41532, + "ael": 41533, + "nyo": 41534, + "westlake": 41535, + "timeout": 41536, + "sofi": 41537, + "phenomena": 41538, + "cultivation": 41539, + "agno": 41540, + "unarmed": 41541, + "sot": 41542, + "conj": 41543, + "geno": 41544, + "royalnavy": 41545, + "nutrition": 41546, + "fairmont": 41547, + "tirelessly": 41548, + "sng": 41549, + "rety": 41550, + "mica": 41551, + "lucent": 41552, + "sloane": 41553, + "drool": 41554, + "rizal": 41555, + "odell": 41556, + "criticized": 41557, + ".'\"": 41558, + "laze": 41559, + "deserted": 41560, + "coder": 41561, + "pras": 41562, + "lillian": 41563, + "itinerary": 41564, + "davy": 41565, + "anap": 41566, + "whipping": 41567, + "hoboken": 41568, + "kareena": 41569, + "羣": 41570, + "vius": 41571, + "tern": 41572, + "nantucket": 41573, + "misunderstood": 41574, + "bulaga": 41575, + "stant": 41576, + "chinook": 41577, + "zam": 41578, + "relies": 41579, + "dss": 41580, + "edmond": 41581, + "sketchy": 41582, + "mell": 41583, + "fex": 41584, + "rector": 41585, + "distill": 41586, + "daydream": 41587, + "winemaker": 41588, + "ripley": 41589, + "billionaires": 41590, + "helene": 41591, + "atif": 41592, + "culprit": 41593, + "bertrand": 41594, + "wouldnt": 41595, + "mapped": 41596, + "vak": 41597, + "gladly": 41598, + "parliament": 41599, + "kidlitart": 41600, + "wareness": 41601, + "goliath": 41602, + "âĨĵ": 41603, + "viewpoint": 41604, + "tatted": 41605, + "fuls": 41606, + "dorsey": 41607, + "anglers": 41608, + "lids": 41609, + "kiya": 41610, + "bowles": 41611, + "beh": 41612, + "bite": 41613, + "compatibility": 41614, + "ancestral": 41615, + "prox": 41616, + "behaved": 41617, + "gubernatorial": 41618, + "chfield": 41619, + "saban": 41620, + "zh": 41621, + "teeny": 41622, + "shibuya": 41623, + "holliday": 41624, + "pancy": 41625, + "âĿĦï¸ıâĿĦï¸ı": 41626, + "seungri": 41627, + "?,": 41628, + "ðŁĩ¦ðŁĩ·": 41629, + "imitation": 41630, + "impactful": 41631, + "anyi": 41632, + "genevie": 41633, + "años": 41634, + "bateman": 41635, + "glider": 41636, + "afar": 41637, + "rasheed": 41638, + "effortless": 41639, + "shwar": 41640, + "dachsh": 41641, + "erun": 41642, + "atos": 41643, + "kini": 41644, + "chd": 41645, + "khaki": 41646, + "klin": 41647, + "felicidades": 41648, + "belo": 41649, + "asl": 41650, + "toppers": 41651, + "finley": 41652, + "stacey": 41653, + "rigorous": 41654, + "karting": 41655, + "leppard": 41656, + "carmichael": 41657, + "beret": 41658, + "cse": 41659, + "akhi": 41660, + "meringue": 41661, + "aban": 41662, + "hake": 41663, + "geri": 41664, + "erjee": 41665, + "resto": 41666, + "commanders": 41667, + "prit": 41668, + "flor": 41669, + "adven": 41670, + "extermin": 41671, + "remainder": 41672, + "åIJ": 41673, + "esg": 41674, + "martino": 41675, + "lullaby": 41676, + "|@": 41677, + "mign": 41678, + "instore": 41679, + "bigbang": 41680, + "cordi": 41681, + "cauley": 41682, + "antebellum": 41683, + "dgate": 41684, + "crock": 41685, + "spandex": 41686, + "scaffolding": 41687, + "oreos": 41688, + "ê°ĵìĦ¸ë¸IJ": 41689, + "pomona": 41690, + "mauro": 41691, + "universi": 41692, + "remi": 41693, + "afootball": 41694, + "tant": 41695, + "smalls": 41696, + "neh": 41697, + "worldo": 41698, + "tropical": 41699, + "morph": 41700, + "javelin": 41701, + "glar": 41702, + "arquitec": 41703, + "reminiscent": 41704, + "tubs": 41705, + "spidey": 41706, + "makeu": 41707, + "sylla": 41708, + "progressives": 41709, + "blot": 41710, + "shorten": 41711, + "keepin": 41712, + "chak": 41713, + "angst": 41714, + "superfood": 41715, + "decadent": 41716, + "stony": 41717, + "neurological": 41718, + "arboretum": 41719, + "annak": 41720, + "fema": 41721, + "percu": 41722, + "disrespectful": 41723, + "smallbiz": 41724, + "lox": 41725, + "coom": 41726, + "csc": 41727, + "bsbi": 41728, + "prevalence": 41729, + "himss": 41730, + "espan": 41731, + "moga": 41732, + "frampton": 41733, + "skymap": 41734, + "masse": 41735, + "leviathan": 41736, + "().": 41737, + "nocturnal": 41738, + "carameli": 41739, + "angor": 41740, + "amnesia": 41741, + "outsiders": 41742, + "shealth": 41743, + "rhino": 41744, + "antag": 41745, + "agio": 41746, + "ðŁĴ°ðŁĴ°": 41747, + "takeme": 41748, + "kabaddi": 41749, + "csi": 41750, + "msh": 41751, + "cochrane": 41752, + "thessaloni": 41753, + "sila": 41754, + "haus": 41755, + "dusting": 41756, + "obese": 41757, + "macklemore": 41758, + "manish": 41759, + "lenin": 41760, + "mdc": 41761, + "grown": 41762, + "sheffield": 41763, + "srs": 41764, + "kele": 41765, + "carson": 41766, + "chum": 41767, + "dahlia": 41768, + "cantore": 41769, + "oppo": 41770, + "howling": 41771, + "cybercrime": 41772, + "surrealism": 41773, + "scran": 41774, + "faiz": 41775, + "thren": 41776, + "racists": 41777, + "rout": 41778, + "pknot": 41779, + "semana": 41780, + "sini": 41781, + "mccull": 41782, + "machi": 41783, + "alfonso": 41784, + "yb": 41785, + "sardar": 41786, + "kendrick": 41787, + "deng": 41788, + "recipro": 41789, + "onf": 41790, + "doomsday": 41791, + "bribery": 41792, + "customiz": 41793, + "artis": 41794, + "cpi": 41795, + "ðŁĻĪðŁĻĪ": 41796, + "slava": 41797, + "lette": 41798, + "ens": 41799, + "âĿ¤ï¸ıðŁĺĺ": 41800, + "crayon": 41801, + "adan": 41802, + "trc": 41803, + "migrate": 41804, + "simpson": 41805, + "rowers": 41806, + "kingsley": 41807, + "farmersmarket": 41808, + "sheehan": 41809, + "nephe": 41810, + "bornon": 41811, + "carton": 41812, + "mickey": 41813, + "allure": 41814, + "ulu": 41815, + "slipknot": 41816, + "hebdo": 41817, + "guido": 41818, + "dogcelebration": 41819, + "onlinemarketing": 41820, + "accelerating": 41821, + ")..": 41822, + "originated": 41823, + "macaroni": 41824, + "edtech": 41825, + "outfield": 41826, + "mitz": 41827, + "discus": 41828, + "advertiser": 41829, + "manor": 41830, + "hashi": 41831, + "descrip": 41832, + "capita": 41833, + "fulbright": 41834, + "receptor": 41835, + "conn": 41836, + "coney": 41837, + "spionage": 41838, + "rattle": 41839, + "prest": 41840, + "uli": 41841, + "blogpost": 41842, + "ackeray": 41843, + ")âĢ¦": 41844, + "redvelvet": 41845, + "matth": 41846, + "inspiring": 41847, + "bsd": 41848, + "kerri": 41849, + "pocon": 41850, + "millar": 41851, + "repur": 41852, + "accenture": 41853, + "ä¹": 41854, + "rambo": 41855, + "ragnarok": 41856, + "deleting": 41857, + "britishmuseum": 41858, + "patory": 41859, + "leipzig": 41860, + "florian": 41861, + "scifi": 41862, + "iners": 41863, + "brate": 41864, + "yoy": 41865, + "melissa": 41866, + "aber": 41867, + "masa": 41868, + "pote": 41869, + "mosquitoes": 41870, + "transplant": 41871, + "rpa": 41872, + ";))": 41873, + "bastille": 41874, + "ylan": 41875, + "joyeux": 41876, + "melodic": 41877, + "captions": 41878, + "atrist": 41879, + "rochdale": 41880, + "gotti": 41881, + "pewdie": 41882, + "cutiesaturday": 41883, + "whois": 41884, + "aquaculture": 41885, + "tiva": 41886, + "spel": 41887, + "hess": 41888, + "haji": 41889, + "freddie": 41890, + "coper": 41891, + "brando": 41892, + "vk": 41893, + "photobook": 41894, + "*,": 41895, + "mydayin": 41896, + "michaela": 41897, + "brunei": 41898, + "srini": 41899, + "inte": 41900, + "ı": 41901, + "deol": 41902, + "dfc": 41903, + "separately": 41904, + "bund": 41905, + "vests": 41906, + "toc": 41907, + "meck": 41908, + "reinforced": 41909, + "constraints": 41910, + "carroll": 41911, + "sqft": 41912, + "rever": 41913, + "camper": 41914, + "birdman": 41915, + "inaction": 41916, + "generators": 41917, + "triumphant": 41918, + "pests": 41919, + "ovo": 41920, + "gypt": 41921, + "alamo": 41922, + "scaled": 41923, + "sureshpp": 41924, + "sdn": 41925, + "ismo": 41926, + "gios": 41927, + ")@": 41928, + "justiceleague": 41929, + "restaurant": 41930, + "gabi": 41931, + "dengue": 41932, + "nextgen": 41933, + "exempli": 41934, + "apex": 41935, + "inspirational": 41936, + "downside": 41937, + "kidz": 41938, + "upl": 41939, + "etna": 41940, + "alvaro": 41941, + "feldman": 41942, + "barnet": 41943, + "mha": 41944, + "esch": 41945, + "blooded": 41946, + ">>>>>>>>": 41947, + "kani": 41948, + "hofficial": 41949, + "casablanca": 41950, + "birds": 41951, + "tyga": 41952, + "swamp": 41953, + "oday": 41954, + "newcastle": 41955, + "nbap": 41956, + "cision": 41957, + "chools": 41958, + "aflo": 41959, + "nep": 41960, + "monton": 41961, + "akb": 41962, + "supermodel": 41963, + "downtime": 41964, + "thos": 41965, + "scwx": 41966, + "snoopy": 41967, + "aggreg": 41968, + "yoke": 41969, + "norcal": 41970, + "wett": 41971, + "prolonged": 41972, + "metast": 41973, + "beater": 41974, + "fta": 41975, + "tlap": 41976, + "disgusted": 41977, + "yh": 41978, + "voiceover": 41979, + "itchy": 41980, + "ipc": 41981, + "ðŁİ¾": 41982, + "pheasant": 41983, + "straits": 41984, + "rampant": 41985, + "jg": 41986, + "fertil": 41987, + "assures": 41988, + "fortunes": 41989, + "salinas": 41990, + "lizards": 41991, + "kettle": 41992, + "ibs": 41993, + "cynthi": 41994, + "heg": 41995, + "mccr": 41996, + "socceroos": 41997, + "happenings": 41998, + "corden": 41999, + "ðŁĺĤðŁijĮ": 42000, + "tches": 42001, + "egret": 42002, + "wolverines": 42003, + "congratulated": 42004, + "hogg": 42005, + "bottling": 42006, + "wri": 42007, + "ferri": 42008, + "bosch": 42009, + "afire": 42010, + "ogden": 42011, + "sjo": 42012, + "jdm": 42013, + "svt": 42014, + "contex": 42015, + "tollywood": 42016, + "mink": 42017, + "mese": 42018, + "supersonic": 42019, + "opoulos": 42020, + "å¸": 42021, + "âĶģ": 42022, + "knuckle": 42023, + "guise": 42024, + "gami": 42025, + "chucky": 42026, + "zinger": 42027, + "radial": 42028, + "complained": 42029, + "boda": 42030, + "fetal": 42031, + "disciplines": 42032, + "corro": 42033, + "ðŁĩ®ðŁĩ¹": 42034, + "opted": 42035, + "filtration": 42036, + "adnan": 42037, + "emcee": 42038, + "mistre": 42039, + "insomni": 42040, + "fergus": 42041, + "trajec": 42042, + "ondon": 42043, + "medtech": 42044, + "tangerine": 42045, + "madras": 42046, + "grue": 42047, + "cabs": 42048, + "zhu": 42049, + "sureshpprabhu": 42050, + "insulated": 42051, + "dayswild": 42052, + "ppm": 42053, + "bandai": 42054, + "vday": 42055, + "sff": 42056, + "squid": 42057, + "lothing": 42058, + "notdead": 42059, + "expressive": 42060, + "cull": 42061, + "alastair": 42062, + "xu": 42063, + "upfront": 42064, + "fishers": 42065, + "enes": 42066, + "umd": 42067, + "dismissal": 42068, + "stier": 42069, + "sels": 42070, + "lust": 42071, + "reactive": 42072, + "protester": 42073, + "eyelashes": 42074, + "alim": 42075, + "goode": 42076, + "greeng": 42077, + "dair": 42078, + "compen": 42079, + "anushka": 42080, + "prototyping": 42081, + "mapu": 42082, + "bearings": 42083, + "ðŁIJŁ": 42084, + "forme": 42085, + "bsbibotany": 42086, + "timothy": 42087, + "outskirts": 42088, + "ambed": 42089, + "aretha": 42090, + "wendell": 42091, + "streaks": 42092, + "nim": 42093, + "kpk": 42094, + "snee": 42095, + "fitter": 42096, + "quota": 42097, + "pate": 42098, + "winning": 42099, + "ðŁįŃ": 42100, + "shopping": 42101, + "mainst": 42102, + "culver": 42103, + "stevie": 42104, + "mcfadden": 42105, + "counterparts": 42106, + "grenfell": 42107, + "folsom": 42108, + "dorset": 42109, + "techcrunch": 42110, + "â¬ħï¸ı": 42111, + "tiptuesday": 42112, + "usl": 42113, + "trex": 42114, + "georgie": 42115, + "ranveerofficial": 42116, + "licks": 42117, + "sewn": 42118, + "kf": 42119, + "'âĢ¦": 42120, + "japs": 42121, + "pate": 42122, + "orthop": 42123, + "festa": 42124, + "stras": 42125, + "montal": 42126, + "hammersmith": 42127, + "foremost": 42128, + "widows": 42129, + "madre": 42130, + "itez": 42131, + "mitochondri": 42132, + "ligans": 42133, + "zona": 42134, + "caribou": 42135, + "mss": 42136, + "andrei": 42137, + "weatherchannel": 42138, + "ghc": 42139, + ":...": 42140, + "taft": 42141, + "aweather": 42142, + "alisation": 42143, + "brutal": 42144, + "blissful": 42145, + "nikola": 42146, + "malicious": 42147, + "qm": 42148, + "mpgvip": 42149, + "brodie": 42150, + "blitz": 42151, + "applaud": 42152, + "dribb": 42153, + "vague": 42154, + "doggo": 42155, + "translating": 42156, + "interpreted": 42157, + "hatched": 42158, + "getyour": 42159, + "beneficiaries": 42160, + "sparring": 42161, + "caesars": 42162, + "awilliams": 42163, + "lahat": 42164, + "broke": 42165, + "timp": 42166, + "virtues": 42167, + "relying": 42168, + "pietro": 42169, + "ktn": 42170, + "icists": 42171, + "pablo": 42172, + "loui": 42173, + "aag": 42174, + "pnpp": 42175, + "chast": 42176, + "pulses": 42177, + "finish": 42178, + "usairforce": 42179, + "typewriter": 42180, + "thompson": 42181, + "dogs": 42182, + "utto": 42183, + "ãģį": 42184, + "sandal": 42185, + "newly": 42186, + "doge": 42187, + "zw": 42188, + "wankers": 42189, + "negr": 42190, + "mucha": 42191, + "determines": 42192, + "blackfish": 42193, + "skunk": 42194, + "mups": 42195, + "instrument": 42196, + "phyto": 42197, + "daystogo": 42198, + "skinned": 42199, + "haider": 42200, + "conten": 42201, + "ðŁIJ¾ðŁIJ¾": 42202, + "weiler": 42203, + "undoubtedly": 42204, + "chairing": 42205, + "wallis": 42206, + "shard": 42207, + "zindabad": 42208, + "adult": 42209, + "absorption": 42210, + "presto": 42211, + "deploying": 42212, + "drummond": 42213, + "battlefront": 42214, + "seagulls": 42215, + "howdy": 42216, + "judaism": 42217, + "desde": 42218, + "partition": 42219, + "âľĿ": 42220, + "nology": 42221, + "nationalbestfriend": 42222, + "lesnar": 42223, + "filmfare": 42224, + "coasts": 42225, + "christensen": 42226, + "acan": 42227, + "mbu": 42228, + "copped": 42229, + "rubble": 42230, + "swc": 42231, + "funnier": 42232, + "farther": 42233, + "whereas": 42234, + "nanotechnology": 42235, + "withstand": 42236, + "pillow": 42237, + "bowers": 42238, + "tope": 42239, + "itly": 42240, + "confit": 42241, + "makar": 42242, + "comforts": 42243, + "bosh": 42244, + "clipper": 42245, + "balla": 42246, + "stik": 42247, + "milb": 42248, + "safeguard": 42249, + "musique": 42250, + "easport": 42251, + "yaz": 42252, + "padded": 42253, + "bader": 42254, + "foreign": 42255, + "chopin": 42256, + "archive": 42257, + "oka": 42258, + "transporting": 42259, + "tmltalk": 42260, + "ajit": 42261, + "consequence": 42262, + "scroo": 42263, + "ffo": 42264, + "collaborated": 42265, + "pugchat": 42266, + "yemi": 42267, + "javed": 42268, + "auburn": 42269, + "oof": 42270, + "maw": 42271, + "saucer": 42272, + "mitigate": 42273, + "iles": 42274, + "evangelist": 42275, + "terie": 42276, + "recl": 42277, + "indictment": 42278, + "cata": 42279, + "brightness": 42280, + "maythe": 42281, + "whimsical": 42282, + "unlv": 42283, + "keyword": 42284, + "cumin": 42285, + "medway": 42286, + "westworld": 42287, + "traw": 42288, + "imposing": 42289, + "formity": 42290, + "coulter": 42291, + "abz": 42292, + "nypd": 42293, + "grassi": 42294, + "kelsey": 42295, + "qldpol": 42296, + "clockwork": 42297, + "fdr": 42298, + "dianne": 42299, + "âĺij": 42300, + "adh": 42301, + "pann": 42302, + "bravely": 42303, + "aege": 42304, + "unlawful": 42305, + "verdi": 42306, + "pocalypse": 42307, + "pharo": 42308, + "karla": 42309, + "resonance": 42310, + "mastiff": 42311, + "ladak": 42312, + "buu": 42313, + "mailed": 42314, + "hii": 42315, + "crawley": 42316, + "torrent": 42317, + "machado": 42318, + "libyan": 42319, + "effortlessly": 42320, + "falsely": 42321, + "qvist": 42322, + "keef": 42323, + "crafthour": 42324, + "cherished": 42325, + "valkyrie": 42326, + "sari": 42327, + "kalamaz": 42328, + "behe": 42329, + "ðŁĮĻ": 42330, + "thim": 42331, + "roddy": 42332, + "coltrane": 42333, + "butchers": 42334, + "achim": 42335, + "wkend": 42336, + "awkward": 42337, + "cabrera": 42338, + ":))))": 42339, + "franc": 42340, + "declan": 42341, + "condos": 42342, + "aja": 42343, + "pandoramusic": 42344, + "charter": 42345, + "phill": 42346, + "montrose": 42347, + "hatchback": 42348, + "handicapp": 42349, + "greaves": 42350, + "eucalyptus": 42351, + "utmost": 42352, + "tson": 42353, + "burton": 42354, + "midwives": 42355, + "incur": 42356, + "ðŁĺį#": 42357, + "mood": 42358, + "compressed": 42359, + "toma": 42360, + "mustang": 42361, + "mog": 42362, + "asana": 42363, + "testic": 42364, + "shotel": 42365, + "insol": 42366, + "corsair": 42367, + "nhq": 42368, + "benny": 42369, + "smma": 42370, + "kapur": 42371, + "incon": 42372, + "jonas": 42373, + "energies": 42374, + "donal": 42375, + "asad": 42376, + "sez": 42377, + "npa": 42378, + "archived": 42379, + "stimulate": 42380, + "dop": 42381, + "hyd": 42382, + "grieving": 42383, + "ãĥĪ": 42384, + "rona": 42385, + "whyte": 42386, + "treehouse": 42387, + "ssell": 42388, + "sandro": 42389, + "kobo": 42390, + "thermost": 42391, + "seclu": 42392, + "hiya": 42393, + "geez": 42394, + "mamas": 42395, + "priscilla": 42396, + "flavoured": 42397, + "fass": 42398, + "wold": 42399, + "makerspace": 42400, + "cosplay": 42401, + "ptv": 42402, + "happyvalentinesday": 42403, + "sequoia": 42404, + "lovecraft": 42405, + "guan": 42406, + "dtm": 42407, + "cii": 42408, + "yokohama": 42409, + "posthum": 42410, + "req": 42411, + "ðŁĶµâļªï¸ı": 42412, + "galatasar": 42413, + "dolby": 42414, + "hamptons": 42415, + "disturbance": 42416, + "stonehenge": 42417, + "okc": 42418, + "disrupting": 42419, + "monthsary": 42420, + "jungle": 42421, + "headlights": 42422, + "dustin": 42423, + "microsof": 42424, + "happymothersday": 42425, + "koko": 42426, + "grazi": 42427, + "testo": 42428, + "naidu": 42429, + "malay": 42430, + "arial": 42431, + "rumb": 42432, + "aboo": 42433, + "harman": 42434, + "trape": 42435, + "spoils": 42436, + "jeho": 42437, + "godly": 42438, + "lockscreen": 42439, + "zun": 42440, + "pious": 42441, + "magento": 42442, + "lenders": 42443, + "probable": 42444, + "corporal": 42445, + "mour": 42446, + "awal": 42447, + "sua": 42448, + "callme": 42449, + "tonne": 42450, + "govin": 42451, + "devastation": 42452, + "xj": 42453, + "gearbox": 42454, + "warlock": 42455, + "perme": 42456, + "itate": 42457, + "gazaunderattack": 42458, + "duval": 42459, + "parasite": 42460, + "clemente": 42461, + "leth": 42462, + "iva": 42463, + "frozen": 42464, + "tholes": 42465, + "tobin": 42466, + "cairn": 42467, + "sill": 42468, + "luckiest": 42469, + "converts": 42470, + "stale": 42471, + "pancra": 42472, + "europale": 42473, + "wisdom": 42474, + "schur": 42475, + "ì¶": 42476, + "vertigo": 42477, + "bij": 42478, + "ubc": 42479, + "nure": 42480, + "righteousness": 42481, + "mtc": 42482, + "factory": 42483, + "verst": 42484, + "reversed": 42485, + "huri": 42486, + "heechul": 42487, + "faber": 42488, + "arr": 42489, + "ulous": 42490, + "venom": 42491, + "phat": 42492, + "greenery": 42493, + "brady": 42494, + "æ": 42495, + ":((": 42496, + "nevergiveup": 42497, + "disha": 42498, + "mota": 42499, + "healthcare": 42500, + "dunham": 42501, + "dexpo": 42502, + "denzel": 42503, + "bbins": 42504, + "fics": 42505, + "wham": 42506, + "mcg": 42507, + "elian": 42508, + "wata": 42509, + "stralia": 42510, + "tellu": 42511, + "pesky": 42512, + "spinoff": 42513, + "armoured": 42514, + "reacted": 42515, + "dofficial": 42516, + "tedu": 42517, + "sagar": 42518, + "morally": 42519, + "paralleled": 42520, + "fios": 42521, + "downer": 42522, + "daugh": 42523, + "redo": 42524, + "worldcup": 42525, + "tariq": 42526, + "barne": 42527, + "glaciers": 42528, + "occult": 42529, + "barbarian": 42530, + "hermosa": 42531, + "!!!)": 42532, + "yur": 42533, + "internation": 42534, + "pss": 42535, + "situ": 42536, + "pint": 42537, + "americanair": 42538, + "swam": 42539, + "doppler": 42540, + "ðŁĴĻðŁĴľ": 42541, + "cincodemayo": 42542, + "levan": 42543, + "hellenic": 42544, + "mcne": 42545, + "judi": 42546, + "yuh": 42547, + "stx": 42548, + "quare": 42549, + "ðŁĺĤ.": 42550, + "stig": 42551, + "gels": 42552, + "motley": 42553, + "hardwork": 42554, + "eurozone": 42555, + "ead": 42556, + "ç¥Ń": 42557, + "seabir": 42558, + "cius": 42559, + "laid": 42560, + "alpaca": 42561, + "presumably": 42562, + "pewdiepie": 42563, + "booted": 42564, + "amari": 42565, + "tamine": 42566, + "solace": 42567, + "barrow": 42568, + "academies": 42569, + "xian": 42570, + "omination": 42571, + "dungeons": 42572, + "bma": 42573, + "deity": 42574, + "aik": 42575, + "stabil": 42576, + "hira": 42577, + "affectionate": 42578, + "vingne": 42579, + "newport": 42580, + "ãħĭãħĭ": 42581, + "thirds": 42582, + "retains": 42583, + "aromatherapy": 42584, + "skier": 42585, + "nima": 42586, + "dope": 42587, + "cringe": 42588, + "condomin": 42589, + "toor": 42590, + "animator": 42591, + "saraj": 42592, + "seascape": 42593, + "minimalism": 42594, + "lakeshore": 42595, + "callaway": 42596, + "bergman": 42597, + "à¤Ĺ": 42598, + "whispering": 42599, + "stupid": 42600, + "rightful": 42601, + "requis": 42602, + "irn": 42603, + "seva": 42604, + "utpol": 42605, + "tuberculo": 42606, + "squish": 42607, + "debut": 42608, + "governmental": 42609, + "christine": 42610, + "allman": 42611, + "weapon": 42612, + "sito": 42613, + "buri": 42614, + "lolita": 42615, + "leafy": 42616, + "fuch": 42617, + "tinted": 42618, + "mcken": 42619, + "ahahaha": 42620, + "ðŁĩµðŁĩ¹": 42621, + "repeal": 42622, + "negan": 42623, + "ðŁķĬ": 42624, + "tailgating": 42625, + "gameinsight": 42626, + "ðŁıŁï¸ı": 42627, + "yakuza": 42628, + "zt": 42629, + "tiring": 42630, + "proposing": 42631, + "bowlers": 42632, + "traitors": 42633, + "akshi": 42634, + "clergy": 42635, + "cito": 42636, + "upsets": 42637, + "tuscal": 42638, + "symphonic": 42639, + "silently": 42640, + "shuff": 42641, + "blackwell": 42642, + "ðŁĺĤ)": 42643, + "kobe": 42644, + "roberto": 42645, + "ridg": 42646, + "dcu": 42647, + "merino": 42648, + "ftp": 42649, + "eastside": 42650, + ".~": 42651, + "nbl": 42652, + "mnleg": 42653, + "tsfor": 42654, + "fraudul": 42655, + "capping": 42656, + "inmy": 42657, + "gymnast": 42658, + "stones": 42659, + "ssin": 42660, + "tweaks": 42661, + "shaggy": 42662, + "oakland": 42663, + "demsin": 42664, + "sangria": 42665, + "mmva": 42666, + "hennessy": 42667, + "downton": 42668, + "rightly": 42669, + "init": 42670, + "agave": 42671, + "oblast": 42672, + "northeast": 42673, + "friendship": 42674, + "dala": 42675, + "trophy": 42676, + "ðŁij½": 42677, + "magin": 42678, + "margaritas": 42679, + "ê·": 42680, + "wwfc": 42681, + "fash": 42682, + "dike": 42683, + "cud": 42684, + "chart": 42685, + "ðŁij®": 42686, + "refugees": 42687, + "joplin": 42688, + "ncs": 42689, + "impy": 42690, + "firmware": 42691, + "pascu": 42692, + "flamin": 42693, + "healthtech": 42694, + "bellletstalk": 42695, + "waka": 42696, + "olls": 42697, + "lago": 42698, + "cowan": 42699, + "bombardier": 42700, + "shome": 42701, + "ðŁĻħ": 42702, + "mcmaster": 42703, + "nave": 42704, + "wells": 42705, + "uta": 42706, + "tellers": 42707, + "misfits": 42708, + "kapil": 42709, + "faceoff": 42710, + "affirm": 42711, + "apro": 42712, + "whitepaper": 42713, + "superyacht": 42714, + "specimens": 42715, + "allocated": 42716, + "...,": 42717, + "-__": 42718, + "kaw": 42719, + "dachshund": 42720, + "djoker": 42721, + "swork": 42722, + "quiere": 42723, + "orum": 42724, + "ðŁIJł": 42725, + "somm": 42726, + "cmt": 42727, + "inghour": 42728, + "skinny": 42729, + "lgbti": 42730, + "giggles": 42731, + "breakaway": 42732, + "researched": 42733, + "parity": 42734, + "myal": 42735, + "msl": 42736, + "retained": 42737, + "sivity": 42738, + "makeinindia": 42739, + "solves": 42740, + "defamation": 42741, + "waltham": 42742, + "sriracha": 42743, + "roadway": 42744, + "conceptu": 42745, + "alin": 42746, + "iwant": 42747, + "åĪ": 42748, + "delft": 42749, + "tenderloin": 42750, + "gains": 42751, + "faults": 42752, + "swire": 42753, + "stellen": 42754, + "pollo": 42755, + "dyne": 42756, + "bornonthisday": 42757, + "asdfghj": 42758, + "sql": 42759, + "salim": 42760, + "advises": 42761, + "voip": 42762, + "ìĹijìĨ": 42763, + "untouched": 42764, + "sheil": 42765, + "ontario": 42766, + "uphill": 42767, + "sobre": 42768, + "deshi": 42769, + "novella": 42770, + "dutton": 42771, + "crawfish": 42772, + "اÙĨ": 42773, + "maa": 42774, + "twine": 42775, + "kalin": 42776, + "ðŁĩµðŁĩŃ": 42777, + "yess": 42778, + "brooks": 42779, + "hoosiers": 42780, + "tonka": 42781, + "umbrellas": 42782, + "ayers": 42783, + "ateam": 42784, + "acquiring": 42785, + "suction": 42786, + "än": 42787, + "wies": 42788, + "tarians": 42789, + "socio": 42790, + "mattb": 42791, + "shepherds": 42792, + "oso": 42793, + "charitytuesday": 42794, + "slogans": 42795, + "ninjas": 42796, + "albat": 42797, + "byte": 42798, + "bashir": 42799, + "trampoline": 42800, + "mydayinla": 42801, + "ija": 42802, + "basel": 42803, + "rory": 42804, + "goldie": 42805, + "firec": 42806, + "unnoticed": 42807, + "peculiar": 42808, + "scha": 42809, + "kerson": 42810, + "mourns": 42811, + "liquidity": 42812, + "quipment": 42813, + "hibs": 42814, + "ars": 42815, + "aeronau": 42816, + "slideshow": 42817, + "slabs": 42818, + "deliciousness": 42819, + "skitchen": 42820, + "htafc": 42821, + "fullerton": 42822, + "creighton": 42823, + "aerob": 42824, + "procrastination": 42825, + "azores": 42826, + "whitehall": 42827, + "ussoccer": 42828, + "mediation": 42829, + "djokernole": 42830, + "andme": 42831, + "umen": 42832, + "noxious": 42833, + "joss": 42834, + "ilife": 42835, + "annivers": 42836, + "sudanese": 42837, + "etres": 42838, + "undermine": 42839, + "wholefoods": 42840, + "disobe": 42841, + "kori": 42842, + "adele": 42843, + "eliz": 42844, + "canti": 42845, + "alon": 42846, + "gymnasium": 42847, + "sarkodie": 42848, + "meteorologist": 42849, + "ylde": 42850, + "steen": 42851, + "stampcollecting": 42852, + "nasal": 42853, + "lott": 42854, + "franks": 42855, + "exol": 42856, + "acki": 42857, + "goodyear": 42858, + "animalrights": 42859, + "yles": 42860, + "violets": 42861, + "mmes": 42862, + "sthel": 42863, + "rapping": 42864, + "tuscan": 42865, + "waiver": 42866, + "turner": 42867, + "eatlocal": 42868, + "northeasthour": 42869, + "animations": 42870, + "tommorow": 42871, + "tsh": 42872, + "ffame": 42873, + "brae": 42874, + "petron": 42875, + "glamour": 42876, + "bryn": 42877, + "dcs": 42878, + "bales": 42879, + "ðŁĶ¶": 42880, + "brov": 42881, + "brev": 42882, + "bons": 42883, + "physique": 42884, + "carne": 42885, + "xe": 42886, + "elixir": 42887, + "volved": 42888, + "loma": 42889, + "ìľł": 42890, + "æĺ": 42891, + "vanu": 42892, + "rigs": 42893, + "balance": 42894, + "vares": 42895, + "bonita": 42896, + "sprinkle": 42897, + "perfecto": 42898, + "dion": 42899, + "leak": 42900, + "calcutta": 42901, + "oba": 42902, + "dma": 42903, + "cmon": 42904, + "tuner": 42905, + "pneumonia": 42906, + "bogus": 42907, + "apologe": 42908, + "clough": 42909, + "borne": 42910, + "))))": 42911, + "revived": 42912, + "ovarian": 42913, + "nerf": 42914, + "clegg": 42915, + "fanfest": 42916, + "chou": 42917, + "realizes": 42918, + "mcn": 42919, + "ligu": 42920, + "legalize": 42921, + "justsaying": 42922, + "forster": 42923, + "bosni": 42924, + "khi": 42925, + "indom": 42926, + "heidel": 42927, + "encryp": 42928, + "siss": 42929, + "eddi": 42930, + "marbles": 42931, + "brisbane": 42932, + "ying": 42933, + "prepaid": 42934, + "walsall": 42935, + "cooperate": 42936, + "orchestr": 42937, + "marisa": 42938, + "howie": 42939, + "chewy": 42940, + "brenner": 42941, + "andromeda": 42942, + "egan": 42943, + "stocki": 42944, + "cavendish": 42945, + "agan": 42946, + "bano": 42947, + "deir": 42948, + "gog": 42949, + "blk": 42950, + "rethinking": 42951, + "chig": 42952, + "rheu": 42953, + "snip": 42954, + "peng": 42955, + "seminole": 42956, + "mswx": 42957, + "annex": 42958, + "lynda": 42959, + "lewishamilton": 42960, + "cumul": 42961, + "tbl": 42962, + "dolphin": 42963, + "aguero": 42964, + "............": 42965, + "prelude": 42966, + "atour": 42967, + "granger": 42968, + "tooting": 42969, + "rotun": 42970, + "disar": 42971, + "homeitems": 42972, + "dares": 42973, + "********": 42974, + "ðŁijĨ": 42975, + "compreh": 42976, + "jinx": 42977, + "aswell": 42978, + "irie": 42979, + "circulating": 42980, + "ðŁIJ¥": 42981, + "overboard": 42982, + "cultivate": 42983, + "rhett": 42984, + "orienteering": 42985, + "cak": 42986, + "balkans": 42987, + "sitt": 42988, + "jasmin": 42989, + "britneyspears": 42990, + "rotor": 42991, + "sealing": 42992, + "gbc": 42993, + "occi": 42994, + "fas": 42995, + "emancip": 42996, + "comer": 42997, + "wartime": 42998, + "tickle": 42999, + "sonny": 43000, + "paces": 43001, + "logg": 43002, + "atrix": 43003, + "srp": 43004, + "gwin": 43005, + "dobbs": 43006, + "uzbe": 43007, + "thewanted": 43008, + "drush": 43009, + "extru": 43010, + "micky": 43011, + "honorees": 43012, + "darwin": 43013, + "redux": 43014, + "mmj": 43015, + "rami": 43016, + "jalapeño": 43017, + "ioc": 43018, + "dover": 43019, + "juju": 43020, + "whitney": 43021, + "seng": 43022, + "enly": 43023, + "auch": 43024, + "archipelago": 43025, + "vigilant": 43026, + "mangal": 43027, + "wildest": 43028, + "paranoid": 43029, + "hali": 43030, + "bbly": 43031, + "sanctioned": 43032, + "realms": 43033, + "conco": 43034, + "uddin": 43035, + "csk": 43036, + "playtime": 43037, + "libra": 43038, + "savag": 43039, + "octane": 43040, + "rectan": 43041, + "return": 43042, + "parrish": 43043, + "morrha": 43044, + "ccp": 43045, + "cmu": 43046, + "sailed": 43047, + "sevent": 43048, + "rosie": 43049, + "piling": 43050, + "hew": 43051, + "boarded": 43052, + "segments": 43053, + "nephro": 43054, + "(.": 43055, + "crats": 43056, + "bakes": 43057, + "ðŁį¸": 43058, + "backtothe": 43059, + "sibling": 43060, + "kirkland": 43061, + "keo": 43062, + "guwa": 43063, + "breads": 43064, + "ðŁĺľðŁĺľ": 43065, + "tq": 43066, + "harassed": 43067, + "gau": 43068, + "wilbur": 43069, + "jisoo": 43070, + "eper": 43071, + "lisam": 43072, + "trippin": 43073, + "shino": 43074, + "rukh": 43075, + "beastmode": 43076, + "choa": 43077, + "instaweather": 43078, + "richland": 43079, + "gari": 43080, + "fez": 43081, + "cowboysnation": 43082, + "fursuit": 43083, + "krun": 43084, + "aen": 43085, + "sycamore": 43086, + "segun": 43087, + "entennial": 43088, + "dih": 43089, + "oax": 43090, + "demsinphilly": 43091, + "ðŁĻĢ": 43092, + "snhl": 43093, + "pennies": 43094, + "passwords": 43095, + "makin": 43096, + "tye": 43097, + "deng": 43098, + "knigh": 43099, + "jeeplife": 43100, + "helpline": 43101, + "afor": 43102, + "zzzz": 43103, + "steamy": 43104, + "picker": 43105, + "iterate": 43106, + "happeningnow": 43107, + "kib": 43108, + "bloomberg": 43109, + "martyrdom": 43110, + "bully": 43111, + "assortment": 43112, + "ahora": 43113, + "zoe": 43114, + "noi": 43115, + "illustri": 43116, + "agarwal": 43117, + "psc": 43118, + "electronica": 43119, + "recruiter": 43120, + "gardiner": 43121, + "radha": 43122, + "nafta": 43123, + "dotnet": 43124, + "piero": 43125, + "georg": 43126, + "bels": 43127, + "ðŁĺĤðŁĺį": 43128, + "tuberculosis": 43129, + "runnin": 43130, + "moris": 43131, + "hauling": 43132, + "evoc": 43133, + "brethren": 43134, + "shair": 43135, + "frameworks": 43136, + "astu": 43137, + "rigid": 43138, + "kuma": 43139, + "kreme": 43140, + "jinnah": 43141, + "insurers": 43142, + "nyu": 43143, + "fere": 43144, + "nollywood": 43145, + "goodvibes": 43146, + "-...": 43147, + "toile": 43148, + "skril": 43149, + "instaweatherpro": 43150, + "czech": 43151, + "pavel": 43152, + "onepiece": 43153, + "nikeplus": 43154, + "filet": 43155, + "cavity": 43156, + "ðŁı½âĢįâĻĤï¸ı": 43157, + "ðŁİ£": 43158, + "drastic": 43159, + "dailys": 43160, + "siamese": 43161, + "rebu": 43162, + "osteo": 43163, + "lark": 43164, + "fre": 43165, + "shelling": 43166, + "pé": 43167, + "gladys": 43168, + "ðŁıĢðŁıĢ": 43169, + "gustave": 43170, + "submerged": 43171, + "grandstand": 43172, + "attu": 43173, + "wont": 43174, + "fpv": 43175, + "bley": 43176, + "joni": 43177, + "angames": 43178, + "weighted": 43179, + "alou": 43180, + "श": 43181, + "lesbians": 43182, + "fj": 43183, + "annies": 43184, + "aml": 43185, + "doria": 43186, + "davin": 43187, + "beta": 43188, + "canc": 43189, + "madewithunity": 43190, + "haj": 43191, + "badlands": 43192, + "mul": 43193, + "bluec": 43194, + "pawn": 43195, + "covington": 43196, + "neurology": 43197, + "httweets": 43198, + "dyslexia": 43199, + "thelove": 43200, + "neat": 43201, + "forklift": 43202, + "automate": 43203, + "uneven": 43204, + "montess": 43205, + "hein": 43206, + "hag": 43207, + "relics": 43208, + "competitiveness": 43209, + "canelo": 43210, + "martens": 43211, + "bulletproof": 43212, + "skittles": 43213, + "gya": 43214, + "primo": 43215, + "americafirst": 43216, + "wooo": 43217, + "abortions": 43218, + "??!!": 43219, + "mache": 43220, + "lders": 43221, + "rlly": 43222, + "prelims": 43223, + "direct": 43224, + "course": 43225, + "swain": 43226, + "supercell": 43227, + "eccentric": 43228, + "stingray": 43229, + "plets": 43230, + "wilcox": 43231, + "westin": 43232, + "okanagan": 43233, + "kiran": 43234, + "carbo": 43235, + "bombings": 43236, + "rarest": 43237, + "boh": 43238, + "gawd": 43239, + "digg": 43240, + "moana": 43241, + "entirety": 43242, + "enclosed": 43243, + "dodgeball": 43244, + "parton": 43245, + "milkyway": 43246, + "atr": 43247, + "thoroughbred": 43248, + "really": 43249, + "qantas": 43250, + "epiphany": 43251, + "inee": 43252, + "aerosmith": 43253, + "spieth": 43254, + "arthro": 43255, + "ellini": 43256, + "dubu": 43257, + "braving": 43258, + "âļ½âļ½": 43259, + "restructuring": 43260, + "illuminate": 43261, + "equili": 43262, + "mpi": 43263, + "ashton": 43264, + "ponytail": 43265, + "mascots": 43266, + "flattering": 43267, + "crum": 43268, + "asta": 43269, + "à®°": 43270, + "strangerthings": 43271, + "barnab": 43272, + "رÙĬ": 43273, + "makeshift": 43274, + "gotcha": 43275, + "willam": 43276, + "choirs": 43277, + "kilometres": 43278, + "ghosh": 43279, + "euthan": 43280, + "dolly": 43281, + "unning": 43282, + "thear": 43283, + "crewe": 43284, + "wsw": 43285, + "jace": 43286, + "dismiss": 43287, + "kean": 43288, + "hota": 43289, + "khat": 43290, + "~>": 43291, + "thiru": 43292, + "rendez": 43293, + "hartman": 43294, + "teessi": 43295, + "casca": 43296, + "zah": 43297, + "hydrange": 43298, + "fod": 43299, + "awp": 43300, + "mzansi": 43301, + "thicker": 43302, + "nagoya": 43303, + "neva": 43304, + "stique": 43305, + "castel": 43306, + "damian": 43307, + "thereby": 43308, + "jiang": 43309, + "alek": 43310, + "musicislife": 43311, + "raq": 43312, + "callahan": 43313, + "gouache": 43314, + "somaliland": 43315, + "seanhannity": 43316, + "raheem": 43317, + "lose": 43318, + "elove": 43319, + "wharton": 43320, + "rectangular": 43321, + "illustrating": 43322, + "harne": 43323, + "autisma": 43324, + "scrapped": 43325, + "elland": 43326, + "decree": 43327, + "nagpur": 43328, + "kipp": 43329, + "sore": 43330, + "nmd": 43331, + "maas": 43332, + "guna": 43333, + "gartner": 43334, + "belli": 43335, + "thenight": 43336, + "jeon": 43337, + "genderequality": 43338, + "giver": 43339, + "ael": 43340, + "garments": 43341, + "neu": 43342, + "mardigras": 43343, + "marsden": 43344, + "rower": 43345, + "polluted": 43346, + "cameraman": 43347, + "vinod": 43348, + "beasley": 43349, + "croc": 43350, + "jiu": 43351, + "hollyoaks": 43352, + "anesthesia": 43353, + "alles": 43354, + "steward": 43355, + "latimes": 43356, + "ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸": 43357, + "tician": 43358, + "goria": 43359, + "comedic": 43360, + "ðŁ¤ĶðŁ¤ĶðŁ¤Ķ": 43361, + "naive": 43362, + "slions": 43363, + "łĪ": 43364, + "burglar": 43365, + "ðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃ": 43366, + "yorkshi": 43367, + "señ": 43368, + "fanboy": 43369, + "laurel": 43370, + "incidence": 43371, + "potomac": 43372, + "roberta": 43373, + "presiden": 43374, + "pryor": 43375, + "osbourne": 43376, + "wku": 43377, + "teme": 43378, + "palae": 43379, + "ðŁ¥º": 43380, + "reboun": 43381, + "itude": 43382, + "reddish": 43383, + "khand": 43384, + "colonialism": 43385, + "northcarolina": 43386, + "ðĿĴ": 43387, + "mannequin": 43388, + "ladybird": 43389, + "tasty": 43390, + "knowledgeable": 43391, + "gshore": 43392, + "ðŁĮĮ": 43393, + "ன": 43394, + "quaker": 43395, + "salzburg": 43396, + "medalists": 43397, + "chyna": 43398, + "bridesmaid": 43399, + "maori": 43400, + "rop": 43401, + "outraged": 43402, + "inadequate": 43403, + "truckers": 43404, + "alana": 43405, + "ìĿ¼": 43406, + "rix": 43407, + "oooooooo": 43408, + "commandments": 43409, + "lambeth": 43410, + "aaj": 43411, + "ecofriendly": 43412, + "blaz": 43413, + "morecambe": 43414, + "bouncy": 43415, + "roux": 43416, + "raided": 43417, + "mized": 43418, + "shc": 43419, + "gawx": 43420, + "laboratories": 43421, + "rubs": 43422, + "restroom": 43423, + "consultations": 43424, + "cajun": 43425, + "virgini": 43426, + "soir": 43427, + "revue": 43428, + "plein": 43429, + "wager": 43430, + "ç¹": 43431, + "wedo": 43432, + "growingup": 43433, + "!ðŁĺĬ": 43434, + "faceted": 43435, + "sinners": 43436, + "hovering": 43437, + "tiene": 43438, + "seasoning": 43439, + "anja": 43440, + "leggo": 43441, + "ilis": 43442, + "flax": 43443, + "devo": 43444, + "ashram": 43445, + "matisse": 43446, + "keri": 43447, + "gower": 43448, + "botox": 43449, + "marshes": 43450, + "unhcr": 43451, + "tsm": 43452, + "optimus": 43453, + "duni": 43454, + "stuffs": 43455, + "sok": 43456, + "orderly": 43457, + "nbad": 43458, + "islamophobia": 43459, + "ravioli": 43460, + "faber": 43461, + "creds": 43462, + "wonka": 43463, + "infusion": 43464, + "overweight": 43465, + "dailynews": 43466, + "assimil": 43467, + "acollege": 43468, + "medallion": 43469, + "kilimanjaro": 43470, + "stiff": 43471, + "thames": 43472, + "sunken": 43473, + "thard": 43474, + "mydubai": 43475, + "hilariously": 43476, + "hannel": 43477, + "plumber": 43478, + "fairview": 43479, + "separating": 43480, + "rascal": 43481, + "quien": 43482, + "necessities": 43483, + "confederation": 43484, + "llll": 43485, + ":]": 43486, + "weaknesses": 43487, + "bronco": 43488, + "raffles": 43489, + "elot": 43490, + "ãĤ¸ãĥ": 43491, + "adventcalendar": 43492, + "ðŁİ¹": 43493, + "stravel": 43494, + "tunic": 43495, + "ksu": 43496, + "impeach": 43497, + "espionage": 43498, + "!-": 43499, + "diment": 43500, + "currant": 43501, + "biode": 43502, + "commuting": 43503, + "byron": 43504, + "ðŁĴĵðŁĴĵ": 43505, + "shaded": 43506, + "truro": 43507, + "crayons": 43508, + "arne": 43509, + "hsc": 43510, + "freaked": 43511, + "dramati": 43512, + "fleek": 43513, + "ucd": 43514, + "marlborough": 43515, + "^-": 43516, + "crossings": 43517, + "malo": 43518, + "blackops": 43519, + "binance": 43520, + "choked": 43521, + "cheney": 43522, + "plo": 43523, + "gestures": 43524, + "valedic": 43525, + "ryanair": 43526, + "remington": 43527, + "vcs": 43528, + "mckee": 43529, + "ecz": 43530, + "begs": 43531, + "nailart": 43532, + "mayorof": 43533, + "happyfathersday": 43534, + "wart": 43535, + "petitions": 43536, + "ningly": 43537, + "cleanenergy": 43538, + "brox": 43539, + "slalom": 43540, + "existent": 43541, + "abay": 43542, + "ugliest": 43543, + "tomp": 43544, + "stoma": 43545, + "selby": 43546, + "goalscorer": 43547, + "benji": 43548, + "overwhelmingly": 43549, + "lans": 43550, + "semiconductor": 43551, + "southkorea": 43552, + "rescheduled": 43553, + "skyl": 43554, + "enlisted": 43555, + "dowski": 43556, + "sidel": 43557, + "rosenberg": 43558, + "nasser": 43559, + "whitehead": 43560, + "prius": 43561, + "harare": 43562, + "enn": 43563, + "ryder": 43564, + "íĤ": 43565, + "mong": 43566, + "clasico": 43567, + "transporter": 43568, + "potty": 43569, + "isme": 43570, + "*****": 43571, + "vice": 43572, + "skit": 43573, + "odessa": 43574, + "lmp": 43575, + "hern": 43576, + "racially": 43577, + "pinoy": 43578, + "paraguay": 43579, + "obituary": 43580, + "goes": 43581, + "bucha": 43582, + "sidewalks": 43583, + "angular": 43584, + "unconstitutional": 43585, + "transitioning": 43586, + "ibu": 43587, + "guys": 43588, + "unpacking": 43589, + "oooooo": 43590, + "blackgirl": 43591, + "bergs": 43592, + "¯": 43593, + "wordoftheday": 43594, + "trumptrain": 43595, + "thunderbolt": 43596, + "msi": 43597, + "fascists": 43598, + "ब": 43599, + "tsk": 43600, + "collapses": 43601, + "rajesh": 43602, + "loveislove": 43603, + "migrating": 43604, + "setback": 43605, + "ðŁĺĬâĿ¤ï¸ı": 43606, + "tels": 43607, + "safetyfirst": 43608, + "narrated": 43609, + "jaejoong": 43610, + "unanswered": 43611, + "liqueur": 43612, + "ennes": 43613, + "dalgo": 43614, + "billings": 43615, + "saltwater": 43616, + "mermaids": 43617, + "longs": 43618, + "clapham": 43619, + "wearec": 43620, + "piccollage": 43621, + "nach": 43622, + "hace": 43623, + "poisoned": 43624, + "loth": 43625, + "agna": 43626, + "adelrey": 43627, + "guardia": 43628, + "polishing": 43629, + "peacekeeping": 43630, + "dall": 43631, + "pisa": 43632, + "lapland": 43633, + "processors": 43634, + "deandre": 43635, + "sobs": 43636, + "ponce": 43637, + "drains": 43638, + "cbe": 43639, + "ðŁİ¥:": 43640, + "splash": 43641, + "meatball": 43642, + "fontana": 43643, + "worcestershirehour": 43644, + "nev": 43645, + "brisk": 43646, + "bint": 43647, + "acr": 43648, + "pox": 43649, + "cayenne": 43650, + "skrillex": 43651, + "jfc": 43652, + "hahahahahahaha": 43653, + "glas": 43654, + "engul": 43655, + "temporal": 43656, + "onized": 43657, + "concre": 43658, + "compose": 43659, + "vibrations": 43660, + "planters": 43661, + "fert": 43662, + "criticalrolefanart": 43663, + "tbli": 43664, + "schallenge": 43665, + "huckabee": 43666, + "municipal": 43667, + "iambic": 43668, + "radios": 43669, + "nevis": 43670, + "durability": 43671, + "mccla": 43672, + "horseback": 43673, + "institutes": 43674, + "fulfill": 43675, + "attach": 43676, + "ateur": 43677, + "akan": 43678, + "resisting": 43679, + "illumination": 43680, + "handle": 43681, + "haircare": 43682, + "oment": 43683, + "macleod": 43684, + "kaiser": 43685, + "gno": 43686, + "beardown": 43687, + "lyf": 43688, + "glomer": 43689, + "distortion": 43690, + "zm": 43691, + "sank": 43692, + "roosters": 43693, + "isnow": 43694, + "asports": 43695, + "agen": 43696, + "woken": 43697, + "stgeorge": 43698, + "romper": 43699, + "myle": 43700, + "economists": 43701, + "ruto": 43702, + "twill": 43703, + "healthand": 43704, + "dito": 43705, + "wsl": 43706, + "tairp": 43707, + "prakash": 43708, + "micheal": 43709, + "hts": 43710, + "wrights": 43711, + "katsu": 43712, + "fiorentina": 43713, + "defenseman": 43714, + "ditch": 43715, + "varsity": 43716, + "texanscheer": 43717, + "baham": 43718, + "scanned": 43719, + "weil": 43720, + "seductive": 43721, + "ðŁijįðŁı½": 43722, + "fue": 43723, + "erwin": 43724, + "davison": 43725, + "terran": 43726, + "moods": 43727, + "woolf": 43728, + "resource": 43729, + "@.": 43730, + "cush": 43731, + "ðŁį°": 43732, + "regression": 43733, + "curled": 43734, + "lazer": 43735, + "joanne": 43736, + "abbott": 43737, + "moz": 43738, + "downers": 43739, + "mmmmmm": 43740, + "valentina": 43741, + "khair": 43742, + "dreamt": 43743, + "crook": 43744, + "chek": 43745, + "steaming": 43746, + "nephews": 43747, + "cleric": 43748, + "asober": 43749, + "indefinitely": 43750, + "wye": 43751, + "usnews": 43752, + "joyce": 43753, + "flushing": 43754, + "wynonnaearp": 43755, + "rondo": 43756, + "kiss": 43757, + "hotdog": 43758, + "barns": 43759, + "saxophon": 43760, + "farley": 43761, + "gasp": 43762, + "decreasing": 43763, + "alway": 43764, + "pex": 43765, + "lsd": 43766, + "shift": 43767, + "poutine": 43768, + "razz": 43769, + "rescuing": 43770, + "niko": 43771, + "hoch": 43772, + "ccl": 43773, + "uaap": 43774, + "nts": 43775, + "mcar": 43776, + "ilwx": 43777, + "conquering": 43778, + "kettering": 43779, + "sturdy": 43780, + "delaying": 43781, + "stok": 43782, + "vanished": 43783, + "cathar": 43784, + "bingham": 43785, + "inv": 43786, + "ichiro": 43787, + "hemo": 43788, + "budgeting": 43789, + "[...]": 43790, + "bess": 43791, + "sebastian": 43792, + "slowed": 43793, + "ðĿij": 43794, + "muslim": 43795, + "stuns": 43796, + "actonclimate": 43797, + "vea": 43798, + "seton": 43799, + "rosetta": 43800, + "ount": 43801, + "hardin": 43802, + "fluid": 43803, + "caw": 43804, + "ðŁ¥Ĥ": 43805, + "yacht": 43806, + "unl": 43807, + "sphy": 43808, + "provocative": 43809, + "oric": 43810, + "isback": 43811, + "___": 43812, + "nicolas": 43813, + "gyan": 43814, + "loose": 43815, + "flin": 43816, + "rebate": 43817, + ":::": 43818, + "!\"@": 43819, + "comicon": 43820, + "sheff": 43821, + "downstream": 43822, + "chichester": 43823, + "beachlife": 43824, + "momlife": 43825, + "diabete": 43826, + "arra": 43827, + "vane": 43828, + "oku": 43829, + "yeo": 43830, + "mango": 43831, + "tryout": 43832, + "appell": 43833, + "heirs": 43834, + "arjuna": 43835, + "ddu": 43836, + "naveen": 43837, + "movic": 43838, + "socialists": 43839, + "sback": 43840, + "criterion": 43841, + "soyuz": 43842, + "kher": 43843, + "daz": 43844, + "yolanda": 43845, + "wineoclock": 43846, + "reina": 43847, + "onew": 43848, + "leonard": 43849, + "endez": 43850, + "ubs": 43851, + "supportlocal": 43852, + "facilitated": 43853, + "caramelized": 43854, + "bpa": 43855, + "vuelta": 43856, + "mytho": 43857, + "mami": 43858, + "speare": 43859, + "nbaplayoffs": 43860, + "fevre": 43861, + "nickjonas": 43862, + "imprint": 43863, + "cso": 43864, + "craigslist": 43865, + "lasalle": 43866, + "gideon": 43867, + "hadoop": 43868, + "disregard": 43869, + "wud": 43870, + "tuc": 43871, + "magee": 43872, + "acoustics": 43873, + "taa": 43874, + "quie": 43875, + "pola": 43876, + "crt": 43877, + "dwyer": 43878, + "dissec": 43879, + "capitol": 43880, + "mention": 43881, + "knoll": 43882, + "heigh": 43883, + "finders": 43884, + "placements": 43885, + "lse": 43886, + "indira": 43887, + "guri": 43888, + "madhuridixit": 43889, + "kingdoms": 43890, + "iambicpent": 43891, + "georgina": 43892, + "jeky": 43893, + "conflicting": 43894, + "bayan": 43895, + "agatha": 43896, + "uphold": 43897, + "dron": 43898, + "vicar": 43899, + "expat": 43900, + "peripheral": 43901, + "pessi": 43902, + "faf": 43903, + "ancestor": 43904, + "?..": 43905, + "widget": 43906, + "punc": 43907, + "commenced": 43908, + "beavs": 43909, + "airwaves": 43910, + "addis": 43911, + "poa": 43912, + "desses": 43913, + "coden": 43914, + "vue": 43915, + "rupee": 43916, + "karin": 43917, + "spock": 43918, + "msy": 43919, + "ะ": 43920, + "prick": 43921, + "fillmore": 43922, + "tification": 43923, + "thingsto": 43924, + "sarde": 43925, + "emile": 43926, + "pereira": 43927, + "nad": 43928, + "brightening": 43929, + "arresting": 43930, + "woking": 43931, + "uscg": 43932, + "spill": 43933, + "raspberrypi": 43934, + "hugo": 43935, + "itec": 43936, + "isma": 43937, + "cufflinks": 43938, + "optimized": 43939, + "occ": 43940, + "miwx": 43941, + "enka": 43942, + "elited": 43943, + "affordable": 43944, + "sakh": 43945, + "coronado": 43946, + "hoh": 43947, + "atul": 43948, + "aioli": 43949, + "jimcantore": 43950, + "accounted": 43951, + "vinay": 43952, + "hermit": 43953, + "grooves": 43954, + "ranch": 43955, + "rilla": 43956, + "wetter": 43957, + "outof": 43958, + "veterin": 43959, + "nikov": 43960, + "kian": 43961, + "fairbanks": 43962, + "ramapho": 43963, + "niti": 43964, + "kko": 43965, + "rusty": 43966, + "nestle": 43967, + "tvxq": 43968, + "shaheer": 43969, + "âĿ¤âĿ¤âĿ¤âĿ¤": 43970, + "pennant": 43971, + "gemstones": 43972, + "demdebate": 43973, + "ðŁIJĬ": 43974, + "autonews": 43975, + "supportindiefilm": 43976, + "macho": 43977, + "vex": 43978, + "newsat": 43979, + "neti": 43980, + "concessions": 43981, + "candied": 43982, + "yofthe": 43983, + "macau": 43984, + "dends": 43985, + "cricketers": 43986, + "saniti": 43987, + "mariano": 43988, + "ghat": 43989, + "artoftheday": 43990, + "¡ľ": 43991, + "egos": 43992, + "genoa": 43993, + "chatbots": 43994, + "brier": 43995, + "allabout": 43996, + "monty": 43997, + "spied": 43998, + "rtr": 43999, + "comfort": 44000, + "snippets": 44001, + "realtime": 44002, + "grain": 44003, + "examined": 44004, + "enlightening": 44005, + "ttu": 44006, + "godbless": 44007, + "releasethe": 44008, + "singular": 44009, + "kians": 44010, + "haka": 44011, + "sorren": 44012, + "defect": 44013, + "marg": 44014, + "equities": 44015, + "dorian": 44016, + "suka": 44017, + "perl": 44018, + "aishwarya": 44019, + "pullover": 44020, + "precision": 44021, + "fairway": 44022, + "neve": 44023, + "riveting": 44024, + "villanova": 44025, + "encom": 44026, + "ako": 44027, + "passionately": 44028, + "europaleague": 44029, + "siempre": 44030, + "xvi": 44031, + "enlightened": 44032, + "cfr": 44033, + "âĺħâĺħâĺħâĺħ": 44034, + "wasteland": 44035, + "isf": 44036, + "newcomers": 44037, + "emergency": 44038, + "amphitheatre": 44039, + "-.": 44040, + "textbooks": 44041, + "figurative": 44042, + "tremb": 44043, + "pesc": 44044, + "abhin": 44045, + "abbot": 44046, + "acacia": 44047, + "hards": 44048, + "porsche": 44049, + "kauai": 44050, + "elisa": 44051, + "carrick": 44052, + "abou": 44053, + "ellier": 44054, + "bech": 44055, + "neutron": 44056, + "galapagos": 44057, + "ruben": 44058, + "innis": 44059, + "howto": 44060, + "nuns": 44061, + "sabine": 44062, + "iac": 44063, + "clinched": 44064, + "notori": 44065, + "fives": 44066, + "cairngor": 44067, + "peri": 44068, + "grc": 44069, + "ðŁĴ¯ðŁĴ¯": 44070, + "malm": 44071, + "twelfth": 44072, + "diff": 44073, + "routines": 44074, + "martyn": 44075, + "linden": 44076, + "synthesizer": 44077, + "number": 44078, + "gamecube": 44079, + "falkirk": 44080, + "byzantine": 44081, + "queuing": 44082, + "grill": 44083, + "scalable": 44084, + "charred": 44085, + "routing": 44086, + "herbali": 44087, + "grizz": 44088, + "ðŁĺŃðŁĺŃðŁĺŃ": 44089, + "toll": 44090, + "terminals": 44091, + "lpc": 44092, + "abd": 44093, + "warmups": 44094, + "removable": 44095, + "¯\\": 44096, + "vigo": 44097, + "papaya": 44098, + "neve": 44099, + "lovingly": 44100, + "jokers": 44101, + "ibles": 44102, + "ssett": 44103, + "potenti": 44104, + "pele": 44105, + "gigi": 44106, + "sadiq": 44107, + "legacy": 44108, + "sono": 44109, + "rupees": 44110, + "retarded": 44111, + "elee": 44112, + "parr": 44113, + "fiance": 44114, + "eyre": 44115, + "sayers": 44116, + "pendants": 44117, + "maknae": 44118, + "albans": 44119, + "adapting": 44120, + "pff": 44121, + "puberty": 44122, + "jiu": 44123, + "ingrad": 44124, + "hypocrite": 44125, + "diplomats": 44126, + "physical": 44127, + "robby": 44128, + "bonsai": 44129, + "ãģ·": 44130, + "fatt": 44131, + "catalunya": 44132, + "âľĸï¸ı": 44133, + "roma": 44134, + "moreland": 44135, + "soe": 44136, + "conversions": 44137, + "stlblues": 44138, + "sholm": 44139, + "grassy": 44140, + "prado": 44141, + "onu": 44142, + "assaulting": 44143, + ">_": 44144, + "settes": 44145, + "disgraceful": 44146, + "aphra": 44147, + "âļ½ï¸ıâļ½ï¸ı": 44148, + "प": 44149, + "kiln": 44150, + "goaltender": 44151, + "sru": 44152, + "philanthropist": 44153, + "bals": 44154, + "thn": 44155, + "studen": 44156, + "sandoval": 44157, + "dogrescue": 44158, + "elions": 44159, + "assessed": 44160, + "largo": 44161, + "hectares": 44162, + "shrm": 44163, + "saif": 44164, + "cleavage": 44165, + "noches": 44166, + "nene": 44167, + "fatalities": 44168, + "curing": 44169, + "cleanser": 44170, + "ales": 44171, + "pvp": 44172, + "southbank": 44173, + "pizzeria": 44174, + "marshals": 44175, + "knife": 44176, + "andover": 44177, + "tblightning": 44178, + "srsly": 44179, + "oute": 44180, + "digimon": 44181, + "timesofindia": 44182, + "promethe": 44183, + "lebo": 44184, + "fsu": 44185, + "witz": 44186, + "revere": 44187, + "manas": 44188, + "mamba": 44189, + "chica": 44190, + "guan": 44191, + "exhibitor": 44192, + "csrracing": 44193, + "dere": 44194, + "xxxxx": 44195, + "gusta": 44196, + "storytime": 44197, + "stoney": 44198, + "organics": 44199, + "andu": 44200, + "seam": 44201, + "minogue": 44202, + "anushkasharma": 44203, + "aba": 44204, + "ðŁİĻï¸ı": 44205, + "ugandan": 44206, + "chromatic": 44207, + "assn": 44208, + "documentaries": 44209, + "sht": 44210, + "rupaul": 44211, + "loyd": 44212, + "kats": 44213, + "eus": 44214, + "itech": 44215, + "medusa": 44216, + "panty": 44217, + "kellogg": 44218, + "etto": 44219, + "tallade": 44220, + "shaa": 44221, + "dost": 44222, + "pms": 44223, + "mariana": 44224, + "jester": 44225, + "crooks": 44226, + "ðŁĶ¬": 44227, + "mindanao": 44228, + "indhoven": 44229, + "ðŁ¤ª": 44230, + "lexi": 44231, + "tvn": 44232, + "janis": 44233, + "cote": 44234, + "ãģĨ": 44235, + "serrano": 44236, + "iwm": 44237, + "ðŁIJ¬": 44238, + "kke": 44239, + "distributors": 44240, + "capu": 44241, + "counterfeit": 44242, + "campsite": 44243, + "aggie": 44244, + "ðŁĺ¼": 44245, + "chhattisgarh": 44246, + "~@": 44247, + "stateu": 44248, + "sandi": 44249, + "preventable": 44250, + "cls": 44251, + "canne": 44252, + "mmc": 44253, + "iver": 44254, + "saharan": 44255, + "palis": 44256, + "nightout": 44257, + "dos": 44258, + "apia": 44259, + "abscbn": 44260, + "managerial": 44261, + "arose": 44262, + "mowx": 44263, + "arosa": 44264, + "ðŁĮ³": 44265, + "underdog": 44266, + "remover": 44267, + "astronomers": 44268, + "lentils": 44269, + "suscep": 44270, + "smoother": 44271, + "pendleton": 44272, + "faucet": 44273, + "emory": 44274, + "dalmati": 44275, + "afcb": 44276, + "ticus": 44277, + "exempt": 44278, + "enrol": 44279, + "dheim": 44280, + "ðŁIJº": 44281, + "restriction": 44282, + "starfish": 44283, + "stow": 44284, + "snorkel": 44285, + "thunderbirds": 44286, + "shead": 44287, + "homosexual": 44288, + "dyn": 44289, + "asli": 44290, + "andretti": 44291, + "douche": 44292, + "domo": 44293, + "tarmac": 44294, + "slumber": 44295, + "pronto": 44296, + "firstdayof": 44297, + "miniature": 44298, + "mariachi": 44299, + "argus": 44300, + "recommending": 44301, + "mobiles": 44302, + "ince": 44303, + "illustrious": 44304, + "orc": 44305, + "adverts": 44306, + "grits": 44307, + "weasel": 44308, + "pagoda": 44309, + "overpass": 44310, + "greys": 44311, + "maximus": 44312, + "armagh": 44313, + "woodland": 44314, + "sunni": 44315, + "ðŁĴī": 44316, + "ëĿ": 44317, + "tione": 44318, + "socio": 44319, + "hos": 44320, + "ðŁ¤ĹðŁ¤Ĺ": 44321, + "windsor": 44322, + "subsequent": 44323, + "munchies": 44324, + "idh": 44325, + "excluding": 44326, + "emi": 44327, + "cuth": 44328, + "zai": 44329, + "weekdays": 44330, + "lawsuits": 44331, + "barnard": 44332, + "ت": 44333, + "petting": 44334, + "netes": 44335, + "mulligan": 44336, + "pharmacists": 44337, + "raquel": 44338, + "eton": 44339, + "cranston": 44340, + "gilded": 44341, + "cleary": 44342, + "ceph": 44343, + "raa": 44344, + "pamper": 44345, + "lombardi": 44346, + "asin": 44347, + "sherry": 44348, + "prod": 44349, + "forte": 44350, + "arianism": 44351, + "buffalobills": 44352, + "æľ¬": 44353, + "ðŁĶ¥#": 44354, + "uuu": 44355, + "justices": 44356, + "carina": 44357, + "natin": 44358, + "maslow": 44359, + "drooling": 44360, + "cognac": 44361, + "camber": 44362, + "elong": 44363, + "rdr": 44364, + "inen": 44365, + "convictions": 44366, + "amuse": 44367, + "trock": 44368, + "harmless": 44369, + "visitation": 44370, + "genomic": 44371, + "bland": 44372, + "benoit": 44373, + "chimp": 44374, + "tuscaloosa": 44375, + "greasy": 44376, + "xpo": 44377, + "gilt": 44378, + "seq": 44379, + "permitted": 44380, + "christmaseve": 44381, + "books": 44382, + "mue": 44383, + "oldschool": 44384, + "humanright": 44385, + "beati": 44386, + "ðŁĶĿ": 44387, + "shat": 44388, + "sculpting": 44389, + "hwan": 44390, + "fernandes": 44391, + "sciutto": 44392, + "fuentes": 44393, + "endeavors": 44394, + "maidstone": 44395, + "unparalleled": 44396, + "shouted": 44397, + "queenof": 44398, + "merc": 44399, + "bandic": 44400, + "veda": 44401, + "selangor": 44402, + "pile": 44403, + "jahan": 44404, + "intimidating": 44405, + "disappears": 44406, + "clich": 44407, + "zaha": 44408, + "wurst": 44409, + "hiv": 44410, + "fodils": 44411, + "cordless": 44412, + "aaaaaa": 44413, + "hydra": 44414, + "belinda": 44415, + "eels": 44416, + "buf": 44417, + "sustaining": 44418, + "rugbyleague": 44419, + "noc": 44420, + "brigitte": 44421, + "(ðŁĵ¸:": 44422, + "trombone": 44423, + "soothe": 44424, + "smog": 44425, + "adp": 44426, + "stable": 44427, + "ingley": 44428, + "diagnose": 44429, + "msg": 44430, + "wess": 44431, + "ticketing": 44432, + "onee": 44433, + "nswpol": 44434, + "eup": 44435, + "autopsy": 44436, + "adityanath": 44437, + "sundown": 44438, + "riverfront": 44439, + "siya": 44440, + "pis": 44441, + "hierarchy": 44442, + "durango": 44443, + "dijk": 44444, + "renshaw": 44445, + "heaps": 44446, + "epidemi": 44447, + "davidbowie": 44448, + "internetof": 44449, + "ddi": 44450, + "nationality": 44451, + "mbar": 44452, + "airy": 44453, + "winder": 44454, + "walia": 44455, + "elliott": 44456, + "cx": 44457, + "bavarian": 44458, + "platt": 44459, + "antw": 44460, + "wiwx": 44461, + "softer": 44462, + "neha": 44463, + "heller": 44464, + "thand": 44465, + "daniela": 44466, + "boast": 44467, + "degradation": 44468, + "ðŁĴ¦ðŁĴ¦": 44469, + "transforming": 44470, + "mane": 44471, + "avut": 44472, + "ðŁĺĪðŁĺĪ": 44473, + "voter": 44474, + "thee": 44475, + "tate": 44476, + "puff": 44477, + "indoor": 44478, + "soproud": 44479, + "boyce": 44480, + "borisjohnson": 44481, + "waitin": 44482, + "immunology": 44483, + "ðŁıĨðŁıĨðŁıĨ": 44484, + "âĿĮ": 44485, + "streetfood": 44486, + "lizasober": 44487, + "cavalier": 44488, + "celia": 44489, + "needle": 44490, + "motoring": 44491, + "gato": 44492, + ",)": 44493, + "rade": 44494, + "harvest": 44495, + "tms": 44496, + "jarpad": 44497, + "oney": 44498, + "airmen": 44499, + "vre": 44500, + "impairment": 44501, + "abhishek": 44502, + "snoop": 44503, + "lant": 44504, + "famously": 44505, + "blou": 44506, + "sze": 44507, + "gander": 44508, + "untouch": 44509, + "tuf": 44510, + "deejay": 44511, + "collateral": 44512, + "bind": 44513, + "ðŁļ©": 44514, + "pinning": 44515, + "icn": 44516, + "';": 44517, + "theeconomist": 44518, + "ultram": 44519, + "worldwaterday": 44520, + "tipoff": 44521, + "thei": 44522, + "feeders": 44523, + "campaign": 44524, + "scumb": 44525, + "dayweekend": 44526, + "yom": 44527, + "pedic": 44528, + "hough": 44529, + "psv": 44530, + "plin": 44531, + "onde": 44532, + "bostonmarathon": 44533, + "azzy": 44534, + "*_*": 44535, + "conley": 44536, + "thiago": 44537, + "hooo": 44538, + "galerie": 44539, + "lucid": 44540, + "jett": 44541, + "glitz": 44542, + "finalfantasy": 44543, + "achievers": 44544, + "yung": 44545, + "peregrine": 44546, + "ophi": 44547, + "dames": 44548, + "biomar": 44549, + "âĺĢï¸ıâĺĢï¸ı": 44550, + "skc": 44551, + "lics": 44552, + "flank": 44553, + "arrahman": 44554, + "hoof": 44555, + "upholstery": 44556, + "tats": 44557, + "woz": 44558, + "¿": 44559, + "snoring": 44560, + "raer": 44561, + "lju": 44562, + "apd": 44563, + "plating": 44564, + "kanu": 44565, + "imation": 44566, + "fragrances": 44567, + "mra": 44568, + "moray": 44569, + "mott": 44570, + "immuni": 44571, + "hearties": 44572, + "bhopal": 44573, + "timers": 44574, + "gata": 44575, + "colorway": 44576, + "carnation": 44577, + "winget": 44578, + "sighs": 44579, + "sville": 44580, + "optimist": 44581, + "chateau": 44582, + "olympians": 44583, + "cio": 44584, + "singersongwriter": 44585, + "nyo": 44586, + "fibers": 44587, + "burch": 44588, + "agro": 44589, + "milne": 44590, + "igbo": 44591, + "cramer": 44592, + "ationals": 44593, + "danube": 44594, + "padma": 44595, + "normani": 44596, + "enforced": 44597, + "breck": 44598, + "boehner": 44599, + "arden": 44600, + "surrendered": 44601, + "prosthetic": 44602, + "oma": 44603, + "hailed": 44604, + "calculations": 44605, + "wfa": 44606, + "bib": 44607, + "fcblive": 44608, + "fonda": 44609, + "westcoast": 44610, + "quests": 44611, + "friendly": 44612, + "towie": 44613, + "fitch": 44614, + "balot": 44615, + "stardom": 44616, + "scratching": 44617, + "hosa": 44618, + "thika": 44619, + "oven": 44620, + "stroke": 44621, + "outpost": 44622, + "pharmaceuticals": 44623, + "hikari": 44624, + "muy": 44625, + "afd": 44626, + "fallontonight": 44627, + "squat": 44628, + "oru": 44629, + "drained": 44630, + "chocolat": 44631, + "민": 44632, + "worths": 44633, + "rib": 44634, + "muj": 44635, + "thats": 44636, + "residente": 44637, + "itel": 44638, + "boost": 44639, + "migos": 44640, + "mulled": 44641, + "laa": 44642, + "etsyshop": 44643, + "donkeys": 44644, + "mek": 44645, + "ptc": 44646, + "flinders": 44647, + "ehs": 44648, + "rohit": 44649, + "muir": 44650, + "gad": 44651, + "compositions": 44652, + "åĨĻ": 44653, + "combustion": 44654, + "ikh": 44655, + "yemeni": 44656, + "waved": 44657, + "garci": 44658, + "akos": 44659, + "oods": 44660, + "fusion": 44661, + "seque": 44662, + "slan": 44663, + "plur": 44664, + "kicchasu": 44665, + "shenando": 44666, + "sams": 44667, + "worlden": 44668, + "horowitz": 44669, + "withme": 44670, + "microbes": 44671, + "kki": 44672, + "ðŁĴĶðŁĴĶ": 44673, + "wsu": 44674, + "patchwork": 44675, + "freer": 44676, + "yaki": 44677, + "theart": 44678, + "symbolism": 44679, + "miler": 44680, + "btn": 44681, + "mabu": 44682, + "sidekick": 44683, + "motivates": 44684, + "sagitt": 44685, + "naturals": 44686, + "serviced": 44687, + "psori": 44688, + "paola": 44689, + "quig": 44690, + "ibadan": 44691, + "giggs": 44692, + "ë³": 44693, + "scientology": 44694, + "sioux": 44695, + "salamat": 44696, + "dres": 44697, + "cadbury": 44698, + "dhawan": 44699, + "ción": 44700, + "_'": 44701, + "swapping": 44702, + "mariska": 44703, + "jamesbond": 44704, + "explosives": 44705, + "ayles": 44706, + "afer": 44707, + "sagu": 44708, + "censor": 44709, + "toma": 44710, + "jefferson": 44711, + "ringed": 44712, + "partist": 44713, + "irresponsible": 44714, + "aguilar": 44715, + "vacay": 44716, + "equitable": 44717, + "altrincham": 44718, + "acur": 44719, + "manish": 44720, + "germin": 44721, + "schooled": 44722, + "putter": 44723, + "edad": 44724, + "naval": 44725, + "toasty": 44726, + "solareclipse": 44727, + "dishu": 44728, + "coyne": 44729, + "acco": 44730, + "muck": 44731, + "maran": 44732, + "elos": 44733, + "lender": 44734, + "croix": 44735, + "worthless": 44736, + "haber": 44737, + "gunmen": 44738, + "ðŁįĵ": 44739, + "zenith": 44740, + "tenders": 44741, + "hurst": 44742, + "holtz": 44743, + "italians": 44744, + "carlow": 44745, + "ucd": 44746, + "characteristic": 44747, + "bung": 44748, + "avl": 44749, + "uth": 44750, + "sasia": 44751, + "rsl": 44752, + "redman": 44753, + "neighboring": 44754, + "greenpeace": 44755, + "stips": 44756, + "followparty": 44757, + "ygk": 44758, + "enos": 44759, + "omnibus": 44760, + "naissance": 44761, + "chrissy": 44762, + "secure": 44763, + "callback": 44764, + "jihoon": 44765, + "memory": 44766, + "blocker": 44767, + "lanta": 44768, + "daffodils": 44769, + "bilt": 44770, + "fferty": 44771, + "faust": 44772, + "iec": 44773, + "nipples": 44774, + "sog": 44775, + "mnd": 44776, + "jaguar": 44777, + "boldly": 44778, + "abpoli": 44779, + "proposition": 44780, + "gunsense": 44781, + "evansville": 44782, + "cutters": 44783, + "wego": 44784, + "doun": 44785, + "dox": 44786, + "stallions": 44787, + "kaj": 44788, + "shippers": 44789, + "jawa": 44790, + "volo": 44791, + "leven": 44792, + "paprika": 44793, + "kovich": 44794, + "jordi": 44795, + "inductees": 44796, + "appalling": 44797, + "dialysis": 44798, + "alleviate": 44799, + "âĢĶâĢĶ": 44800, + "pieter": 44801, + "midwi": 44802, + "qtr": 44803, + "juliette": 44804, + "intermission": 44805, + "hawks": 44806, + "actment": 44807, + "oneill": 44808, + "klin": 44809, + "vamps": 44810, + "famous": 44811, + "could": 44812, + "automobi": 44813, + "daan": 44814, + "westend": 44815, + "ellip": 44816, + "nhc": 44817, + "melanch": 44818, + "webseries": 44819, + "tongue": 44820, + "snatched": 44821, + "smyth": 44822, + "tangible": 44823, + "sli": 44824, + "easing": 44825, + "barstool": 44826, + "overlay": 44827, + "affordability": 44828, + "tinged": 44829, + "teras": 44830, + "ayush": 44831, + "wannaone": 44832, + "rhine": 44833, + "dana": 44834, + "shana": 44835, + "kendal": 44836, + "fertile": 44837, + "wir": 44838, + "repleni": 44839, + "larvae": 44840, + "isro": 44841, + "convos": 44842, + "abbrevi": 44843, + "ucc": 44844, + "hungry": 44845, + "burrows": 44846, + "ager": 44847, + "navi": 44848, + "matin": 44849, + "duper": 44850, + "cern": 44851, + "madon": 44852, + "ķï¸ı": 44853, + "éģ": 44854, + "tups": 44855, + "hyatt": 44856, + "shep": 44857, + "fridaynight": 44858, + "wiser": 44859, + "heidi": 44860, + "hatton": 44861, + "pgh": 44862, + "fountain": 44863, + "wristbands": 44864, + "ahmadiyya": 44865, + "aerial": 44866, + "subscribed": 44867, + "solos": 44868, + "mace": 44869, + "slayed": 44870, + "forfe": 44871, + "dulce": 44872, + "christmass": 44873, + "arunjaitley": 44874, + "violate": 44875, + "obstru": 44876, + "nieces": 44877, + "wvu": 44878, + "idyl": 44879, + "faze": 44880, + "preserves": 44881, + "infringe": 44882, + "premiers": 44883, + "intervals": 44884, + "agency": 44885, + "(©": 44886, + "standalone": 44887, + "dimes": 44888, + "boer": 44889, + "parameters": 44890, + "getit": 44891, + "ðŁĺĺðŁĺĺðŁĺĺðŁĺĺ": 44892, + "tulane": 44893, + "forgiven": 44894, + "scoll": 44895, + "mbps": 44896, + "smashbros": 44897, + "robbi": 44898, + "primavera": 44899, + "alist": 44900, + "ghostly": 44901, + "ayat": 44902, + "yeats": 44903, + "impressionist": 44904, + "earphones": 44905, + "caulfield": 44906, + "waikiki": 44907, + "salute": 44908, + "scou": 44909, + "muay": 44910, + "louisvuitton": 44911, + "bakhta": 44912, + "adog": 44913, + "inventions": 44914, + "hurd": 44915, + "foreclo": 44916, + "streamline": 44917, + "thalaivar": 44918, + "chsnews": 44919, + "willard": 44920, + "tsn": 44921, + "europarl": 44922, + "crusher": 44923, + "mysore": 44924, + "grower": 44925, + "raping": 44926, + "patti": 44927, + "gden": 44928, + "smw": 44929, + "mufti": 44930, + "kidman": 44931, + "abr": 44932, + "sounders": 44933, + "skeptical": 44934, + "ðŁĶİ": 44935, + "sundar": 44936, + "ime": 44937, + "ferg": 44938, + "featherweight": 44939, + "arlington": 44940, + "pasqu": 44941, + "agazine": 44942, + "wearable": 44943, + "natic": 44944, + "mcclure": 44945, + "intermitt": 44946, + "horde": 44947, + "sixties": 44948, + "carte": 44949, + "bhav": 44950, + "zeal": 44951, + "experiential": 44952, + "adorned": 44953, + "sommer": 44954, + "enote": 44955, + "hypothesis": 44956, + "stinky": 44957, + "proto": 44958, + "deadlines": 44959, + "vogel": 44960, + "musings": 44961, + "moncton": 44962, + "guter": 44963, + "fle": 44964, + "acion": 44965, + "voiceof": 44966, + "tasha": 44967, + "inhabitants": 44968, + "typeface": 44969, + "sba": 44970, + "btsx": 44971, + "ðŁĶĴ": 44972, + "worx": 44973, + "uhc": 44974, + "joko": 44975, + "cellars": 44976, + "goro": 44977, + "continuum": 44978, + "...&": 44979, + "weathercee": 44980, + "hap": 44981, + "srk": 44982, + "risers": 44983, + "lonelyplanet": 44984, + "unnamed": 44985, + "coeur": 44986, + "ðŁįĮ": 44987, + "theworld": 44988, + "ilike": 44989, + "fasten": 44990, + "amigo": 44991, + "riba": 44992, + "ramaphosa": 44993, + "staffers": 44994, + "hadley": 44995, + "??\"": 44996, + "fiore": 44997, + "salut": 44998, + "huff": 44999, + "bezos": 45000, + "Ñĭ": 45001, + "rader": 45002, + "kamala": 45003, + "inline": 45004, + "fillers": 45005, + "umatic": 45006, + "allin": 45007, + "shatter": 45008, + "rein": 45009, + "oku": 45010, + "chases": 45011, + "flagged": 45012, + "babymetal": 45013, + "waterstones": 45014, + "tsb": 45015, + "cutout": 45016, + "ophel": 45017, + "aama": 45018, + "rockabilly": 45019, + "stolic": 45020, + "jetblue": 45021, + "ichick": 45022, + "downton": 45023, + "uzbekistan": 45024, + "patna": 45025, + "laq": 45026, + "grange": 45027, + ")_/": 45028, + "subsidi": 45029, + "scp": 45030, + "newscast": 45031, + "itsa": 45032, + "tweetyour": 45033, + "emor": 45034, + "archaeologists": 45035, + "unification": 45036, + "porta": 45037, + "qx": 45038, + "protectors": 45039, + "prohib": 45040, + "charisma": 45041, + "cartag": 45042, + "renfre": 45043, + "sculpt": 45044, + "guwahati": 45045, + "dema": 45046, + "boop": 45047, + "unfpa": 45048, + "dexter": 45049, + "layla": 45050, + "alleges": 45051, + "soups": 45052, + "neveragain": 45053, + "lys": 45054, + "calc": 45055, + "baroness": 45056, + "visualize": 45057, + "gerber": 45058, + "absorbed": 45059, + "iers": 45060, + "ahan": 45061, + "fontein": 45062, + "detectors": 45063, + "verstappen": 45064, + "svc": 45065, + "formulated": 45066, + "acdc": 45067, + "lix": 45068, + "incompetent": 45069, + "bhk": 45070, + "lourdes": 45071, + "waterhouse": 45072, + "snowed": 45073, + "appreciative": 45074, + "sigma": 45075, + "lizasoberano": 45076, + "penned": 45077, + "paycheck": 45078, + "tallinn": 45079, + "fancafe": 45080, + "parisi": 45081, + "avalley": 45082, + "vig": 45083, + "rufc": 45084, + "hardship": 45085, + "socute": 45086, + "poise": 45087, + "ì¹": 45088, + "rothschild": 45089, + "kly": 45090, + "????????": 45091, + "lhp": 45092, + "ilay": 45093, + "fhs": 45094, + "amad": 45095, + "ideals": 45096, + "bradbury": 45097, + "balboa": 45098, + "nicot": 45099, + "kidnap": 45100, + "wolve": 45101, + "tasmanian": 45102, + "opt": 45103, + "matthias": 45104, + "ãĥ³ãĤ": 45105, + "supermarkets": 45106, + "mylittlepony": 45107, + "melee": 45108, + "lister": 45109, + "groun": 45110, + "fedora": 45111, + "kindness": 45112, + "enen": 45113, + "brahms": 45114, + "¯\\_(": 45115, + "roswell": 45116, + "marlene": 45117, + "icu": 45118, + "reformation": 45119, + "orail": 45120, + "hebrides": 45121, + "disparities": 45122, + "terracotta": 45123, + "swallows": 45124, + "reid": 45125, + "influencing": 45126, + "fluor": 45127, + "dene": 45128, + "tumour": 45129, + "blondes": 45130, + "thunderbird": 45131, + "sheva": 45132, + "mogadishu": 45133, + "kab": 45134, + "creeps": 45135, + "iving": 45136, + "eneed": 45137, + "annoy": 45138, + "âĶĢ": 45139, + "intrigue": 45140, + "enquiry": 45141, + "araj": 45142, + "tural": 45143, + "kubernetes": 45144, + "endlessly": 45145, + "dividends": 45146, + "tora": 45147, + "tish": 45148, + "commemorates": 45149, + "unra": 45150, + "trib": 45151, + "ponty": 45152, + "nem": 45153, + "dissent": 45154, + "brewingco": 45155, + "ðŁĺ½": 45156, + "normali": 45157, + "biof": 45158, + "(...": 45159, + "chillen": 45160, + "주": 45161, + "mellon": 45162, + "avis": 45163, + "mccormack": 45164, + "ingra": 45165, + "enriched": 45166, + "customerexperience": 45167, + "testosterone": 45168, + "snug": 45169, + "setti": 45170, + "geronimo": 45171, + "inquirer": 45172, + "breaches": 45173, + "verything": 45174, + "blooming": 45175, + "mura": 45176, + "dispos": 45177, + "bide": 45178, + "deva": 45179, + "shadesof": 45180, + "intrin": 45181, + "shev": 45182, + "sven": 45183, + "nayanthara": 45184, + "ganesha": 45185, + "cws": 45186, + "berta": 45187, + "labelled": 45188, + "useum": 45189, + "nicknamed": 45190, + "mahan": 45191, + "caruso": 45192, + "apur": 45193, + "ðŁijĨ": 45194, + "wq": 45195, + "orphanage": 45196, + "discarded": 45197, + "magnu": 45198, + "lue": 45199, + "jeon": 45200, + "bridgeport": 45201, + "pacing": 45202, + "mercury": 45203, + "(ðŁĵ¸": 45204, + "marxist": 45205, + "amphibious": 45206, + "transplantation": 45207, + "stitching": 45208, + "thenburg": 45209, + "gradual": 45210, + "ãĤĮ": 45211, + "roft": 45212, + "mails": 45213, + "inec": 45214, + "guyana": 45215, + "doppelg": 45216, + "vero": 45217, + "rewrite": 45218, + "headless": 45219, + "harbaugh": 45220, + "gateway": 45221, + "carsforsale": 45222, + "swi": 45223, + "stis": 45224, + "macht": 45225, + "unde": 45226, + "surabaya": 45227, + "stapleton": 45228, + "nurturing": 45229, + "milner": 45230, + "yao": 45231, + "lmaoooo": 45232, + "kosh": 45233, + "arsenal": 45234, + "kame": 45235, + "erry": 45236, + "arroyo": 45237, + "dismisses": 45238, + "rubbed": 45239, + "rcb": 45240, + "lewd": 45241, + "dilu": 45242, + "andor": 45243, + "vide": 45244, + "urin": 45245, + "intersec": 45246, + "haar": 45247, + "alb": 45248, + "yearswith": 45249, + "appleton": 45250, + "éal": 45251, + "ullivan": 45252, + "succu": 45253, + "monterrey": 45254, + "dmx": 45255, + "artemis": 45256, + "ronnie": 45257, + "farmland": 45258, + "sfootball": 45259, + "grotto": 45260, + "anthi": 45261, + "ãĢģ": 45262, + "à®Ł": 45263, + "vidya": 45264, + "jimmyfallon": 45265, + "àµį": 45266, + "tzer": 45267, + "gravitational": 45268, + "wthr": 45269, + "uhhh": 45270, + "ehr": 45271, + "tinker": 45272, + "tijuana": 45273, + "scranton": 45274, + "ramcharan": 45275, + "barclay": 45276, + "revan": 45277, + "msi": 45278, + "kap": 45279, + "wrs": 45280, + "wethenorth": 45281, + "toral": 45282, + "satu": 45283, + "grom": 45284, + "facep": 45285, + "erickson": 45286, + "zyn": 45287, + "sedge": 45288, + "oodle": 45289, + "spursofficial": 45290, + "dsp": 45291, + "sicilian": 45292, + "solihull": 45293, + "receivers": 45294, + "ladakh": 45295, + "hendrick": 45296, + "theri": 45297, + "presiding": 45298, + "mcguinness": 45299, + "litters": 45300, + "gunnar": 45301, + "ghoul": 45302, + "wib": 45303, + "ntv": 45304, + "karo": 45305, + "frock": 45306, + "blau": 45307, + "amplify": 45308, + "allis": 45309, + "ullah": 45310, + "memoirs": 45311, + "khloe": 45312, + "interceptions": 45313, + "petday": 45314, + "looney": 45315, + "confin": 45316, + "chay": 45317, + "piyushgoyal": 45318, + "frequencies": 45319, + "utz": 45320, + "eventual": 45321, + "warmly": 45322, + "oblivion": 45323, + "anka": 45324, + "tait": 45325, + "âĿ¤ï¸ı.": 45326, + "directorial": 45327, + "rulers": 45328, + "princes": 45329, + "muck": 45330, + "sturridge": 45331, + "deuce": 45332, + "abridged": 45333, + "baguette": 45334, + "uncles": 45335, + "pendu": 45336, + "minding": 45337, + "forrester": 45338, + "avila": 45339, + "waller": 45340, + "wallstreet": 45341, + "mentor": 45342, + "hino": 45343, + "highway": 45344, + "cromwell": 45345, + "fanartfriday": 45346, + "mbi": 45347, + "coyle": 45348, + "ahi": 45349, + "trove": 45350, + "spiegel": 45351, + "paytm": 45352, + "mcintosh": 45353, + "jansen": 45354, + "niti": 45355, + "nashville": 45356, + "leno": 45357, + "leicestershire": 45358, + "legos": 45359, + "dict": 45360, + "ðŁĵ½": 45361, + "spad": 45362, + "beverlyhills": 45363, + "syrah": 45364, + "separates": 45365, + "zain": 45366, + "unfit": 45367, + "drags": 45368, + "tania": 45369, + "overflowing": 45370, + "hrithik": 45371, + "hawthorn": 45372, + "zani": 45373, + "macfar": 45374, + "fide": 45375, + "totem": 45376, + "peds": 45377, + "fundamentally": 45378, + "calico": 45379, + "sinner": 45380, + "jä": 45381, + "hilde": 45382, + "dsd": 45383, + "tenay": 45384, + "tahit": 45385, + "milf": 45386, + "lieb": 45387, + "informing": 45388, + "uplift": 45389, + "rael": 45390, + "mortgages": 45391, + "lect": 45392, + "iiii": 45393, + "guillaume": 45394, + "composites": 45395, + "oldsmobile": 45396, + "lend": 45397, + "garth": 45398, + "commish": 45399, + "baptized": 45400, + "scorpions": 45401, + "rucker": 45402, + "bringbackour": 45403, + "alliance": 45404, + "thalapathy": 45405, + "tali": 45406, + "spans": 45407, + "eridge": 45408, + "witherspoon": 45409, + "linda": 45410, + "skylar": 45411, + "korn": 45412, + "homs": 45413, + "Äį": 45414, + "silenced": 45415, + "caffe": 45416, + "arty": 45417, + "distinguish": 45418, + "towed": 45419, + "pung": 45420, + "jessica": 45421, + "earnest": 45422, + "beaufort": 45423, + "tama": 45424, + "studyabroad": 45425, + "sikhs": 45426, + "newbie": 45427, + "navratri": 45428, + "marble": 45429, + "lounging": 45430, + "litter": 45431, + "dalit": 45432, + "sosa": 45433, + "izes": 45434, + "grade": 45435, + "compromising": 45436, + "triton": 45437, + "detta": 45438, + "vj": 45439, + "chauffe": 45440, + "spectral": 45441, + "powered": 45442, + "montessori": 45443, + "articulate": 45444, + "halton": 45445, + "alco": 45446, + "yey": 45447, + "mntwins": 45448, + "acounty": 45449, + "ðŁijıðŁı¾": 45450, + "âīĪ": 45451, + "madmen": 45452, + "kala": 45453, + "grum": 45454, + "chik": 45455, + "atis": 45456, + "sume": 45457, + "akhtar": 45458, + "jobsearch": 45459, + "highlighter": 45460, + "boath": 45461, + "âĦ¹": 45462, + "tarzan": 45463, + "lambo": 45464, + "âĽĦï¸ı": 45465, + "oxfam": 45466, + "dumpster": 45467, + "pretzels": 45468, + "macos": 45469, + "inclined": 45470, + "factual": 45471, + "advertisers": 45472, + "shui": 45473, + "puree": 45474, + "mlpfi": 45475, + "antidote": 45476, + "capo": 45477, + "pastr": 45478, + "mercado": 45479, + "button": 45480, + "armin": 45481, + "agg": 45482, + "lolla": 45483, + "horribly": 45484, + "errands": 45485, + "christophe": 45486, + "timesnow": 45487, + "mondaymotiv": 45488, + "liss": 45489, + "scandals": 45490, + "mci": 45491, + "disproportion": 45492, + "âĺİ": 45493, + "surpass": 45494, + "samaritan": 45495, + "sotho": 45496, + "purest": 45497, + "flatt": 45498, + "triviatuesday": 45499, + "delectable": 45500, + "leopold": 45501, + "hermione": 45502, + "choudhary": 45503, + "enrich": 45504, + "¡¡": 45505, + "subsidiary": 45506, + "inequalities": 45507, + "bachelor": 45508, + "autoimmune": 45509, + "lakota": 45510, + "ihop": 45511, + "adjec": 45512, + "thesimpsons": 45513, + "shes": 45514, + "sek": 45515, + "gretchen": 45516, + "upstream": 45517, + "hinakhan": 45518, + "copernic": 45519, + "xtina": 45520, + "lug": 45521, + "toughness": 45522, + "ead": 45523, + "clipped": 45524, + "bius": 45525, + "slv": 45526, + "fahren": 45527, + "deepak": 45528, + "cau": 45529, + "xan": 45530, + "immature": 45531, + "digni": 45532, + "bobs": 45533, + "shredding": 45534, + "buttery": 45535, + "accommodations": 45536, + "deven": 45537, + "chunks": 45538, + "superleague": 45539, + "skybet": 45540, + "kildare": 45541, + "jeet": 45542, + "ëį": 45543, + "cek": 45544, + "wrecks": 45545, + "propane": 45546, + "ohl": 45547, + "tbd": 45548, + "quoi": 45549, + "trumpp": 45550, + "mimo": 45551, + "reluctant": 45552, + "verne": 45553, + "oic": 45554, + "magh": 45555, + "arnau": 45556, + "sever": 45557, + "lidge": 45558, + "stairway": 45559, + "kicchasudeep": 45560, + "ðŁĶº": 45561, + "machining": 45562, + "aamaadmi": 45563, + "oti": 45564, + "cda": 45565, + "alit": 45566, + "pany": 45567, + "installs": 45568, + "acct": 45569, + "eshop": 45570, + "diem": 45571, + "hardwell": 45572, + "fulfillment": 45573, + "scafe": 45574, + "quack": 45575, + "extracts": 45576, + "sweetened": 45577, + "fighton": 45578, + "fdi": 45579, + "dinger": 45580, + "waltham": 45581, + "usur": 45582, + "referees": 45583, + "seokjin": 45584, + "grann": 45585, + "afrin": 45586, + "thn": 45587, + "schaf": 45588, + "parcels": 45589, + "betis": 45590, + "amarine": 45591, + "noman": 45592, + "khtar": 45593, + "moritz": 45594, + "coupling": 45595, + "barons": 45596, + "ðŁIJ¸": 45597, + "ø": 45598, + "slp": 45599, + "sadler": 45600, + "xander": 45601, + "triad": 45602, + "mcmillan": 45603, + "khz": 45604, + "dividing": 45605, + "ìĹijìĨĮ": 45606, + "daryl": 45607, + "zedd": 45608, + "leys": 45609, + "plaques": 45610, + "fluori": 45611, + "tipperary": 45612, + "onnell": 45613, + "didier": 45614, + "langford": 45615, + "imc": 45616, + "thesun": 45617, + "birdies": 45618, + "archa": 45619, + "yessss": 45620, + "tdi": 45621, + "daria": 45622, + "candace": 45623, + "altam": 45624, + "palaces": 45625, + "chit": 45626, + "santam": 45627, + "eventful": 45628, + "bookof": 45629, + "adb": 45630, + "monstax": 45631, + "creole": 45632, + "coel": 45633, + "âĸ½": 45634, + "wearen": 45635, + "stennis": 45636, + "sheath": 45637, + "atism": 45638, + "groningen": 45639, + "mlpfim": 45640, + "lepre": 45641, + "wrongly": 45642, + "rspca": 45643, + "rendezvous": 45644, + "acknowledging": 45645, + "pelvic": 45646, + "solicitor": 45647, + "slays": 45648, + "nuestra": 45649, + "lod": 45650, + "islander": 45651, + "feroci": 45652, + "fashionshow": 45653, + "rass": 45654, + "dgeon": 45655, + "adolescents": 45656, + "smashes": 45657, + "negligence": 45658, + "grateful": 45659, + "vedere": 45660, + "swoop": 45661, + "ingl": 45662, + "apolice": 45663, + "vandalism": 45664, + "gann": 45665, + "joao": 45666, + "disupdates": 45667, + "zimbabwe": 45668, + "underage": 45669, + "radiance": 45670, + "wof": 45671, + "bourgeo": 45672, + "plas": 45673, + "crani": 45674, + "ghue": 45675, + "wreckem": 45676, + "warrants": 45677, + "reform": 45678, + "jimmie": 45679, + "atwood": 45680, + "ysl": 45681, + "neilhimself": 45682, + "lbj": 45683, + "iman": 45684, + "tanto": 45685, + "noisse": 45686, + "verbs": 45687, + "equipo": 45688, + "altogether": 45689, + "mament": 45690, + "lice": 45691, + "douglass": 45692, + "tierney": 45693, + "primed": 45694, + "jhal": 45695, + "furnitu": 45696, + "brazili": 45697, + "vill": 45698, + "pastels": 45699, + "nison": 45700, + "uff": 45701, + "paralysis": 45702, + "jaye": 45703, + "impo": 45704, + "ðŁijģ": 45705, + "strategically": 45706, + "pakistanis": 45707, + "wassup": 45708, + "superbike": 45709, + "thanku": 45710, + "truelove": 45711, + "shaikh": 45712, + "israelis": 45713, + "vip": 45714, + "tog": 45715, + "lien": 45716, + "laker": 45717, + "greyhounds": 45718, + "culars": 45719, + "bianchi": 45720, + "balotelli": 45721, + "arran": 45722, + "loos": 45723, + "strates": 45724, + "hebron": 45725, + "arvo": 45726, + "sunderland": 45727, + "theal": 45728, + "tombstone": 45729, + "sandman": 45730, + "cpac": 45731, + "thanksgiving": 45732, + "lovehim": 45733, + "latino": 45734, + "anin": 45735, + "akaif": 45736, + "ĭãĤ": 45737, + "torquay": 45738, + "diest": 45739, + "allianz": 45740, + "ðŁĺķ": 45741, + "golfclub": 45742, + "cllr": 45743, + "walcott": 45744, + "schnau": 45745, + "prompted": 45746, + "nominating": 45747, + "lennox": 45748, + "valet": 45749, + "monro": 45750, + "mayward": 45751, + "eph": 45752, + "ðŁĶĶ": 45753, + "interoper": 45754, + "rda": 45755, + "reflex": 45756, + "armchair": 45757, + "ê°ķ": 45758, + "stripper": 45759, + "porti": 45760, + "pharm": 45761, + "hamza": 45762, + "nireland": 45763, + "neue": 45764, + "hpv": 45765, + "portfoli": 45766, + "sunburn": 45767, + "frisbee": 45768, + "beal": 45769, + "baptiste": 45770, + "xh": 45771, + "tym": 45772, + "prati": 45773, + "overs": 45774, + "hazrat": 45775, + "desert": 45776, + "derry": 45777, + "usky": 45778, + "emmett": 45779, + "acharya": 45780, + ")_/¯": 45781, + "shud": 45782, + "maya": 45783, + "hamill": 45784, + "raim": 45785, + "nrc": 45786, + "fittings": 45787, + "curvy": 45788, + "ðŁıĩ": 45789, + "sterling": 45790, + "à¥Ģ": 45791, + "walkin": 45792, + "shortcuts": 45793, + "milly": 45794, + "astur": 45795, + "alphabe": 45796, + "pli": 45797, + "pez": 45798, + "missyou": 45799, + "radford": 45800, + "mlg": 45801, + "taeyang": 45802, + "notjustlakes": 45803, + "dumps": 45804, + "serendip": 45805, + "leur": 45806, + "raving": 45807, + "ester": 45808, + "depriv": 45809, + "abscbn": 45810, + "ðŁijĩðŁı»": 45811, + "scarcity": 45812, + "ocr": 45813, + "meanings": 45814, + "capt": 45815, + "dahl": 45816, + "fermentation": 45817, + "brioche": 45818, + "towin": 45819, + "outlander": 45820, + "massimo": 45821, + "encro": 45822, + "ðŁ¥³": 45823, + "built": 45824, + "potam": 45825, + "kiri": 45826, + "tmw": 45827, + "monitored": 45828, + "kites": 45829, + "peoplesvote": 45830, + "grayson": 45831, + "íģ¬": 45832, + "afrika": 45833, + "adies": 45834, + "ivote": 45835, + "gyne": 45836, + "gannon": 45837, + "dix": 45838, + "cmc": 45839, + "oural": 45840, + "foxandfriends": 45841, + "beli": 45842, + "igne": 45843, + "glan": 45844, + "katrinakaif": 45845, + "copolitics": 45846, + "qualitative": 45847, + "psi": 45848, + "lucci": 45849, + "discoura": 45850, + "âĺ®": 45851, + "kelli": 45852, + "gautam": 45853, + "caracas": 45854, + "realest": 45855, + "pula": 45856, + "inus": 45857, + "hilltop": 45858, + "makeaw": 45859, + "attenborough": 45860, + "twy": 45861, + "rarity": 45862, + "peckham": 45863, + "mahon": 45864, + "cornelius": 45865, + "clinicians": 45866, + "tonline": 45867, + "tbi": 45868, + "paradise": 45869, + "kasi": 45870, + "inevit": 45871, + "freshness": 45872, + "collingwood": 45873, + "lunatic": 45874, + "defense": 45875, + "copd": 45876, + "infra": 45877, + "wainwright": 45878, + "sainsbury": 45879, + "alabam": 45880, + "tema": 45881, + "laco": 45882, + "checker": 45883, + "relegated": 45884, + "trent": 45885, + "stalks": 45886, + "huffpost": 45887, + "bhubaneswar": 45888, + "astral": 45889, + "shareyour": 45890, + "primrose": 45891, + "hime": 45892, + "catan": 45893, + "endment": 45894, + "endow": 45895, + "clemens": 45896, + "maloney": 45897, + "hilary": 45898, + "gametime": 45899, + "denise": 45900, + "collaborators": 45901, + "bwo": 45902, + "radicals": 45903, + "guetta": 45904, + "icion": 45905, + "aua": 45906, + "snapmatic": 45907, + "satchel": 45908, + "excavation": 45909, + "baseman": 45910, + "são": 45911, + "gnation": 45912, + "feld": 45913, + "survey": 45914, + "shahzad": 45915, + "mast": 45916, + "anirudhofficial": 45917, + "trucker": 45918, + "otago": 45919, + "geograph": 45920, + "ethel": 45921, + "âļ¡ï¸ıâļ¡ï¸ı": 45922, + "sver": 45923, + "mutt": 45924, + "internetofthings": 45925, + "anchored": 45926, + "whouse": 45927, + "bangla": 45928, + "balmain": 45929, + "ç¹ĭãģ": 45930, + "breakfa": 45931, + "áĢ": 45932, + "twister": 45933, + "tetris": 45934, + "cav": 45935, + "stags": 45936, + "gz": 45937, + "aub": 45938, + "stormed": 45939, + "helens": 45940, + "yarmouth": 45941, + "stasy": 45942, + "gustavo": 45943, + "cosc": 45944, + "vinson": 45945, + "upp": 45946, + "scricket": 45947, + "assumptions": 45948, + "appe": 45949, + "nuh": 45950, + "uer": 45951, + "premise": 45952, + "naga": 45953, + "eamon": 45954, + "coronary": 45955, + "naf": 45956, + "northside": 45957, + "elmer": 45958, + "rotar": 45959, + "outlining": 45960, + "elf": 45961, + "resurg": 45962, + "katelyn": 45963, + "incan": 45964, + "hysteria": 45965, + "cee": 45966, + "ambani": 45967, + "prolly": 45968, + "ĮãĤĬãģ": 45969, + "axes": 45970, + "sanjose": 45971, + "rembrandt": 45972, + "magpie": 45973, + "evenly": 45974, + "scorsese": 45975, + "quaint": 45976, + "fg": 45977, + "bbuk": 45978, + "indianfootball": 45979, + "weareall": 45980, + "spdwy": 45981, + "pisces": 45982, + "ecg": 45983, + "âĺħâĺħâĺħâĺħâĺħ": 45984, + "preorders": 45985, + ":|": 45986, + "nipple": 45987, + "salazar": 45988, + "jume": 45989, + "jailbreak": 45990, + "minn": 45991, + "bassett": 45992, + "zetta": 45993, + "jeffree": 45994, + "adjun": 45995, + "ticon": 45996, + "sandiego": 45997, + "drinklocal": 45998, + "cholera": 45999, + "solicitors": 46000, + "obo": 46001, + "compost": 46002, + "nian": 46003, + "wra": 46004, + "treach": 46005, + "icic": 46006, + "professional": 46007, + "delve": 46008, + "legate": 46009, + "historia": 46010, + "croissant": 46011, + "connoisse": 46012, + "namo": 46013, + "palliative": 46014, + "chemtrails": 46015, + "iority": 46016, + "globalwarming": 46017, + "comicart": 46018, + "behavioural": 46019, + "rested": 46020, + "lias": 46021, + "climates": 46022, + "ŁãģĦ": 46023, + "rutland": 46024, + "nourish": 46025, + "menopause": 46026, + "hotties": 46027, + "dementi": 46028, + "vespa": 46029, + "melville": 46030, + "analogue": 46031, + "tzman": 46032, + "strung": 46033, + "imperfect": 46034, + "glare": 46035, + "circling": 46036, + "rosberg": 46037, + "reco": 46038, + "ocity": 46039, + "loire": 46040, + "embe": 46041, + "dossier": 46042, + "neel": 46043, + "nando": 46044, + "mea": 46045, + "galvani": 46046, + "finesse": 46047, + "agp": 46048, + "berkeley": 46049, + "asim": 46050, + "âĺºâĺº": 46051, + "quilted": 46052, + "ishere": 46053, + "unmatched": 46054, + "potion": 46055, + "forz": 46056, + "atre": 46057, + "selfies": 46058, + "juliana": 46059, + "ðŁļ¶": 46060, + "âĸº": 46061, + "melton": 46062, + "âłĢâłĢâłĢâłĢâłĢâłĢâłĢâłĢ": 46063, + "spinrilla": 46064, + "purcell": 46065, + "edp": 46066, + "atleti": 46067, + "tonyawards": 46068, + "raja": 46069, + "progno": 46070, + "molten": 46071, + "stuff": 46072, + "pally": 46073, + "nobelprize": 46074, + "âĻ»ï¸ı": 46075, + "spiritual": 46076, + "speake": 46077, + "sasha": 46078, + "brium": 46079, + "truss": 46080, + "criticize": 46081, + "assassinscreed": 46082, + "yoruba": 46083, + "ulo": 46084, + "fireman": 46085, + "workinprogress": 46086, + "efcc": 46087, + "flares": 46088, + "robot": 46089, + "hikers": 46090, + "cll": 46091, + "shadowing": 46092, + "patsy": 46093, + "lehman": 46094, + "cns": 46095, + "å±": 46096, + "guadal": 46097, + "à±į": 46098, + "rape": 46099, + "rhonda": 46100, + "parallels": 46101, + "sonja": 46102, + "language": 46103, + "landings": 46104, + "zola": 46105, + "cramps": 46106, + "burning": 46107, + "appraisal": 46108, + "jolla": 46109, + "hamm": 46110, + "kasa": 46111, + "gully": 46112, + "fgo": 46113, + "ulysses": 46114, + "ribe": 46115, + "ðŁĴĦ": 46116, + "ibu": 46117, + "etienne": 46118, + "briar": 46119, + "finely": 46120, + "combating": 46121, + "yql": 46122, + "gotham": 46123, + "wechat": 46124, + "topaz": 46125, + "primaries": 46126, + "lse": 46127, + "izz": 46128, + "hele": 46129, + "disponible": 46130, + "cystic": 46131, + "belichick": 46132, + "thrush": 46133, + "kansascity": 46134, + "geom": 46135, + "solidi": 46136, + "redbubble": 46137, + "bystand": 46138, + "cambridgeshire": 46139, + "parfait": 46140, + "astle": 46141, + "owo": 46142, + "indore": 46143, + "stomping": 46144, + "smelly": 46145, + "ðŁ¤ĸ": 46146, + "locomo": 46147, + "admitting": 46148, + "holme": 46149, + "clockwise": 46150, + "minsk": 46151, + "mcco": 46152, + "forget": 46153, + "evp": 46154, + "camra": 46155, + "abella": 46156, + "yotes": 46157, + "universityof": 46158, + "méxico": 46159, + "silverado": 46160, + "ricket": 46161, + "crombie": 46162, + "puj": 46163, + "eradicate": 46164, + "delight": 46165, + "ygo": 46166, + "glamping": 46167, + "vica": 46168, + "duggan": 46169, + "counters": 46170, + "cfd": 46171, + "scour": 46172, + "reactjs": 46173, + "puram": 46174, + "parasites": 46175, + "inki": 46176, + "villen": 46177, + "stella": 46178, + "limbo": 46179, + "angas": 46180, + "kcr": 46181, + "ðŁĴļðŁĴļðŁĴļ": 46182, + "vapori": 46183, + "mumford": 46184, + "oligar": 46185, + "à¼": 46186, + "aloo": 46187, + "booties": 46188, + "adr": 46189, + "kelli": 46190, + "drummers": 46191, + "avici": 46192, + "natureuk": 46193, + "ronal": 46194, + "intrac": 46195, + "unsplash": 46196, + "leche": 46197, + "goma": 46198, + "eline": 46199, + "enviro": 46200, + "bionic": 46201, + "bueno": 46202, + "mik": 46203, + "avin": 46204, + "starling": 46205, + "empowers": 46206, + "cakeday": 46207, + "boycot": 46208, + "ðŁĴļðŁĴļ": 46209, + "ðŁĮ¸ðŁĮ¸": 46210, + "vach": 46211, + "mci": 46212, + "fractures": 46213, + "geri": 46214, + "sking": 46215, + "excluded": 46216, + "luce": 46217, + "jave": 46218, + "iggy": 46219, + "eviden": 46220, + "akistan": 46221, + "awn": 46222, + "morals": 46223, + "lucifer": 46224, + "haban": 46225, + "tumbling": 46226, + "sundaymotivation": 46227, + "mosley": 46228, + "captainamerica": 46229, + "schicago": 46230, + "theone": 46231, + "motd": 46232, + "dts": 46233, + "ðŁIJ¼": 46234, + "repell": 46235, + "iii": 46236, + "locust": 46237, + "geospatial": 46238, + "mersey": 46239, + "immerse": 46240, + "descend": 46241, + "bernade": 46242, + "js": 46243, + "boatsales": 46244, + "winder": 46245, + "crank": 46246, + "singleton": 46247, + "candidacy": 46248, + "bena": 46249, + "ðŁı»âĢį": 46250, + "highlander": 46251, + "olt": 46252, + "kprs": 46253, + "healthylifestyle": 46254, + "fourteen": 46255, + "endthe": 46256, + "ithaca": 46257, + "circulated": 46258, + "rans": 46259, + "prevalent": 46260, + "havas": 46261, + "splendor": 46262, + "rooster": 46263, + "kalamazoo": 46264, + "jewellers": 46265, + "ennedy": 46266, + "rousey": 46267, + "esy": 46268, + "cannons": 46269, + "ornamental": 46270, + "////": 46271, + "rendon": 46272, + "winne": 46273, + "molding": 46274, + "eidmubarak": 46275, + "countess": 46276, + "simona": 46277, + "hawa": 46278, + "foes": 46279, + "duster": 46280, + "sbu": 46281, + "portray": 46282, + "marries": 46283, + "goodday": 46284, + "choco": 46285, + "achiever": 46286, + "ðŁĺ¹ðŁĺ¹": 46287, + "preneur": 46288, + "tramp": 46289, + "tomi": 46290, + "nbat": 46291, + "gardenchat": 46292, + "farrakhan": 46293, + "everglades": 46294, + "abru": 46295, + "sousa": 46296, + "sece": 46297, + "homeswee": 46298, + "terrestrial": 46299, + "barit": 46300, + "sridevi": 46301, + "olu": 46302, + "melinda": 46303, + "frick": 46304, + "candies": 46305, + "ðŁĺŃðŁĴķ": 46306, + "qureshi": 46307, + "familyfun": 46308, + "exorcist": 46309, + "cardinal": 46310, + "nyt": 46311, + "diesel": 46312, + "cumulus": 46313, + "capricorn": 46314, + "siology": 46315, + "lorna": 46316, + "dougie": 46317, + "andie": 46318, + "supersport": 46319, + "cfl": 46320, + "пÑĢи": 46321, + "sayang": 46322, + "peek": 46323, + "à¸Ĭ": 46324, + "lobe": 46325, + "jem": 46326, + "inglis": 46327, + "ggled": 46328, + "csn": 46329, + "amnesty": 46330, + "chups": 46331, + "baes": 46332, + "sauer": 46333, + "ðŁıIJ": 46334, + "mongolian": 46335, + "enet": 46336, + "backstreet": 46337, + "drilled": 46338, + "accessing": 46339, + "ceo": 46340, + "bse": 46341, + "aiken": 46342, + "purr": 46343, + "worsen": 46344, + "wheres": 46345, + "wark": 46346, + "testifying": 46347, + "buri": 46348, + "blast": 46349, + "awg": 46350, + "ðŁĵĭ": 46351, + "redefining": 46352, + "hearing": 46353, + "uci": 46354, + "cmp": 46355, + "boni": 46356, + "tailoring": 46357, + "taji": 46358, + "nocchi": 46359, + "emt": 46360, + "stephenking": 46361, + "neet": 46362, + "complains": 46363, + "campaigner": 46364, + "luciano": 46365, + "twilight": 46366, + "tiesto": 46367, + "passports": 46368, + "floyd": 46369, + "cathedr": 46370, + "naked": 46371, + "caregiver": 46372, + "bcoz": 46373, + "adecides": 46374, + "kuri": 46375, + "lyk": 46376, + "braries": 46377, + "drenched": 46378, + "disclose": 46379, + "ðŁĴªðŁı½": 46380, + "leblanc": 46381, + "jetty": 46382, + "garty": 46383, + "chipmun": 46384, + "bsu": 46385, + "rhythmic": 46386, + "icz": 46387, + "frid": 46388, + "annex": 46389, + "amex": 46390, + "soloist": 46391, + "lancers": 46392, + "arrowhead": 46393, + "specification": 46394, + "simulated": 46395, + "nais": 46396, + "inverte": 46397, + "bowing": 46398, + "worship": 46399, + "fz": 46400, + "aboss": 46401, + "shaq": 46402, + "ì¶ķ": 46403, + "challengers": 46404, + "anarch": 46405, + "aamaadmiparty": 46406, + "ãħĭãħĭãħĭ": 46407, + "suffolk": 46408, + "socorro": 46409, + "snell": 46410, + "cladding": 46411, + "absorbing": 46412, + "shawa": 46413, + "participates": 46414, + "ðŁįĶ": 46415, + "bookstores": 46416, + "baku": 46417, + "seaport": 46418, + "kojima": 46419, + "gaby": 46420, + "packard": 46421, + "electrician": 46422, + "letit": 46423, + "mowing": 46424, + "fawad": 46425, + "youngjae": 46426, + "hotmail": 46427, + "mening": 46428, + "urie": 46429, + "intimacy": 46430, + "conti": 46431, + ":\")": 46432, + "lifeisgood": 46433, + "inciner": 46434, + "idri": 46435, + "craziness": 46436, + "journos": 46437, + "franchi": 46438, + "bottlen": 46439, + "alda": 46440, + "ffes": 46441, + "kx": 46442, + "southwe": 46443, + "aira": 46444, + "clayton": 46445, + "scoti": 46446, + "fj": 46447, + "briga": 46448, + "ðŁ¤ĺðŁı»": 46449, + "demonstrators": 46450, + "yz": 46451, + "stork": 46452, + "naq": 46453, + "cascades": 46454, + "travelchat": 46455, + "plata": 46456, + "padma": 46457, + "franci": 46458, + "attain": 46459, + "batgirl": 46460, + "lombard": 46461, + "hoos": 46462, + "ddos": 46463, + "neonatal": 46464, + "disclaimer": 46465, + "rss": 46466, + "rant": 46467, + "disen": 46468, + "texaste": 46469, + "socal": 46470, + "fractal": 46471, + "camry": 46472, + "strife": 46473, + "snacking": 46474, + "muh": 46475, + "santander": 46476, + "morons": 46477, + "graf": 46478, + "parades": 46479, + "huston": 46480, + "drupal": 46481, + "miento": 46482, + "kirstel": 46483, + "hyde": 46484, + "vomit": 46485, + "fortified": 46486, + "sphinx": 46487, + "dav": 46488, + "biryani": 46489, + "winnings": 46490, + "sbaseball": 46491, + "merged": 46492, + "lovelondon": 46493, + "lingering": 46494, + "dreambig": 46495, + "carleton": 46496, + "livelihood": 46497, + "django": 46498, + "astrid": 46499, + "grids": 46500, + "downe": 46501, + "bruised": 46502, + "sne": 46503, + "scarecrow": 46504, + "helium": 46505, + "fnc": 46506, + "biggs": 46507, + "anter": 46508, + "restorative": 46509, + "empires": 46510, + "abdel": 46511, + "lifestyle": 46512, + "kiwanis": 46513, + "colloquium": 46514, + "meen": 46515, + "prick": 46516, + "antique": 46517, + "zeb": 46518, + "mimic": 46519, + "edmonds": 46520, + "ðŁijĬðŁijĬ": 46521, + "qing": 46522, + "ppel": 46523, + "mcgill": 46524, + "interpreting": 46525, + "âŀķ": 46526, + "rashad": 46527, + "doka": 46528, + "narrator": 46529, + "electromagnetic": 46530, + "ashby": 46531, + "saura": 46532, + "irandeal": 46533, + "âģīï¸ı": 46534, + "krishnan": 46535, + "indi": 46536, + "ffen": 46537, + "brea": 46538, + "osman": 46539, + "multinational": 46540, + "chippe": 46541, + "recruiters": 46542, + "ausbiz": 46543, + "pounding": 46544, + "regen": 46545, + "cursor": 46546, + "refusal": 46547, + "macs": 46548, + "inak": 46549, + "axial": 46550, + "waifu": 46551, + "upcycled": 46552, + "hindustan": 46553, + "cassini": 46554, + "carlyle": 46555, + "scratches": 46556, + "reef": 46557, + "manatee": 46558, + "eatery": 46559, + "ðŁĵ¢": 46560, + "uncondition": 46561, + "senpai": 46562, + "onther": 46563, + "comicbook": 46564, + "prosciutto": 46565, + "demar": 46566, + "mise": 46567, + "mage": 46568, + "freec": 46569, + "ayesha": 46570, + "alder": 46571, + "androidgames": 46572, + "leyton": 46573, + "hock": 46574, + "doorway": 46575, + "chicagofire": 46576, + "aaliyah": 46577, + "swelling": 46578, + "bix": 46579, + ".ðŁĺĤ": 46580, + "evankirstel": 46581, + "torpedo": 46582, + "konstant": 46583, + "genevieve": 46584, + "maia": 46585, + "hauser": 46586, + "dotorg": 46587, + "hideous": 46588, + "fik": 46589, + "spraw": 46590, + "eek": 46591, + "zappa": 46592, + "wandered": 46593, + "''": 46594, + "rajan": 46595, + "bambi": 46596, + "($)": 46597, + "widening": 46598, + "toolbox": 46599, + "sair": 46600, + "illuminating": 46601, + "prays": 46602, + "outpatient": 46603, + "iw": 46604, + "dayo": 46605, + "lob": 46606, + "swfl": 46607, + "shades": 46608, + "gums": 46609, + "cookin": 46610, + "kodi": 46611, + "griffin": 46612, + "traumati": 46613, + "stea": 46614, + "slaughtered": 46615, + "godbless": 46616, + "airtime": 46617, + "pseudo": 46618, + "bsa": 46619, + "hauled": 46620, + "arif": 46621, + "à¸Ńà¸ĩ": 46622, + "lel": 46623, + "wcpo": 46624, + "militi": 46625, + "charters": 46626, + "worlda": 46627, + "ruk": 46628, + "kgs": 46629, + "digitalindia": 46630, + "isable": 46631, + "idyllic": 46632, + "espino": 46633, + "marietta": 46634, + "ebo": 46635, + "teamcanada": 46636, + "abour": 46637, + "wilton": 46638, + "rockstars": 46639, + "favored": 46640, + "physic": 46641, + "wrinkle": 46642, + "tbr": 46643, + "dprint": 46644, + "ballarat": 46645, + "adal": 46646, + "zey": 46647, + "ðŁĺįðŁĶ¥": 46648, + "tomlin": 46649, + "mtr": 46650, + "palsy": 46651, + "fenerbah": 46652, + "tighten": 46653, + "philia": 46654, + "ironing": 46655, + "ryu": 46656, + "bant": 46657, + "enquire": 46658, + "cair": 46659, + "aburger": 46660, + "trun": 46661, + "greenberg": 46662, + "chauhan": 46663, + "irina": 46664, + "shani": 46665, + "trendsetter": 46666, + "prett": 46667, + "zafar": 46668, + "alove": 46669, + "vici": 46670, + "panic": 46671, + "noo": 46672, + "lustre": 46673, + "disrupted": 46674, + "ballis": 46675, + "sonsof": 46676, + "monsi": 46677, + "instac": 46678, + "akest": 46679, + "ëĭ¤": 46680, + "kwame": 46681, + "horrormovies": 46682, + "district": 46683, + "saucy": 46684, + "mban": 46685, + "armies": 46686, + "withdrawn": 46687, + "medics": 46688, + "loftus": 46689, + "eroom": 46690, + "bekind": 46691, + "arns": 46692, + "allon": 46693, + "unison": 46694, + "davids": 46695, + "crat": 46696, + "nicotine": 46697, + "soor": 46698, + "smx": 46699, + "onco": 46700, + "cosplaying": 46701, + "zombies": 46702, + "harms": 46703, + "eger": 46704, + "rosy": 46705, + "moonshine": 46706, + "fein": 46707, + "cett": 46708, + "dubrov": 46709, + "regents": 46710, + "benitez": 46711, + "ðŁijıðŁı¼ðŁijıðŁı¼": 46712, + "stec": 46713, + "malia": 46714, + "prioritize": 46715, + "iceland": 46716, + "ftse": 46717, + "vamo": 46718, + "lamont": 46719, + "homosexuality": 46720, + "brees": 46721, + "regui": 46722, + "cbp": 46723, + "tej": 46724, + "skysports": 46725, + "detergent": 46726, + "shasta": 46727, + "derel": 46728, + "conservancy": 46729, + "colorized": 46730, + "accolades": 46731, + "viso": 46732, + "showyour": 46733, + "nanow": 46734, + "biceps": 46735, + "usability": 46736, + "bim": 46737, + "dailysketch": 46738, + "pearljam": 46739, + "strangest": 46740, + "megadeth": 46741, + "broadcasts": 46742, + "barren": 46743, + "arton": 46744, + "chriss": 46745, + "configu": 46746, + "lures": 46747, + "isthe": 46748, + "eul": 46749, + "railwayana": 46750, + "globalhealth": 46751, + "gianni": 46752, + "uaap": 46753, + "slum": 46754, + "consciously": 46755, + "abre": 46756, + "nup": 46757, + "budget": 46758, + "vada": 46759, + "esch": 46760, + "realness": 46761, + "erased": 46762, + "thunt": 46763, + "bez": 46764, + "armistice": 46765, + "ðŁij¹": 46766, + "shrun": 46767, + "oled": 46768, + "driverless": 46769, + "ðŁ¤·ðŁı»âĢįâĻĢï¸ı": 46770, + "wondr": 46771, + "skan": 46772, + "salaam": 46773, + "motherland": 46774, + "hwang": 46775, + "geno": 46776, + "gangnam": 46777, + "twright": 46778, + "endorsing": 46779, + "enic": 46780, + "adoration": 46781, + "paused": 46782, + "patricks": 46783, + "docked": 46784, + "platte": 46785, + "ffxv": 46786, + "ethnicity": 46787, + "autoshow": 46788, + "sideshow": 46789, + "afterlife": 46790, + "relocated": 46791, + "orphaned": 46792, + "foodnetwork": 46793, + "dareto": 46794, + "andra": 46795, + "slaps": 46796, + "vlive": 46797, + "swims": 46798, + "reimagined": 46799, + "mistle": 46800, + "revise": 46801, + "reality": 46802, + "bharti": 46803, + "ðŁĴĻðŁĴĽ": 46804, + "latest": 46805, + "proudest": 46806, + "grasses": 46807, + "lanyard": 46808, + "freshest": 46809, + "carcinoma": 46810, + "anomaly": 46811, + "ziegler": 46812, + "sumner": 46813, + "lyrix": 46814, + "gorg": 46815, + "isd": 46816, + "avel": 46817, + "swildlife": 46818, + "mesqu": 46819, + "johncena": 46820, + "euroleague": 46821, + "saber": 46822, + "masterful": 46823, + "yarra": 46824, + "cognition": 46825, + "jacobson": 46826, + "abolic": 46827, + "sirloin": 46828, + "shukla": 46829, + "mojito": 46830, + "supere": 46831, + "stweet": 46832, + "mez": 46833, + "esa": 46834, + "rudolf": 46835, + "gura": 46836, + "whereyou": 46837, + "ttm": 46838, + "wins": 46839, + "trustworthy": 46840, + "nyk": 46841, + "braden": 46842, + "tabletop": 46843, + "goodfood": 46844, + "eson": 46845, + "bek": 46846, + "linguistic": 46847, + "grays": 46848, + "chath": 46849, + "hcs": 46850, + "moni": 46851, + "deans": 46852, + "cussions": 46853, + "chell": 46854, + "slows": 46855, + "hemi": 46856, + "dapp": 46857, + "sharpie": 46858, + "boosters": 46859, + "aos": 46860, + "strack": 46861, + "sedona": 46862, + "mueller": 46863, + "hardwick": 46864, + "ornate": 46865, + "thora": 46866, + "salud": 46867, + "otwol": 46868, + "chum": 46869, + "miho": 46870, + "forage": 46871, + "thelittle": 46872, + "tearful": 46873, + "oneself": 46874, + "mindy": 46875, + "smg": 46876, + "gmbh": 46877, + "emerald": 46878, + "ðŁĶ´âļªï¸ı": 46879, + "tutti": 46880, + "receptions": 46881, + "revising": 46882, + "ibrox": 46883, + "topeka": 46884, + "salami": 46885, + "expanse": 46886, + "ibooks": 46887, + "dobson": 46888, + "clio": 46889, + "ats": 46890, + "ðŁļĮ": 46891, + "moha": 46892, + "isance": 46893, + "shutters": 46894, + "moot": 46895, + "janine": 46896, + "marvelcomics": 46897, + "jordani": 46898, + "poser": 46899, + "kenneth": 46900, + "hyung": 46901, + "deja": 46902, + "aseball": 46903, + "speciality": 46904, + "euston": 46905, + "classiccar": 46906, + "hadith": 46907, + "ðŁIJī": 46908, + "chasing": 46909, + "izo": 46910, + "grosven": 46911, + "aglia": 46912, + "thisdayinhistory": 46913, + "trow": 46914, + "omile": 46915, + "huar": 46916, + "byn": 46917, + "saline": 46918, + "divine": 46919, + "demonic": 46920, + "tyran": 46921, + "handover": 46922, + "revitalization": 46923, + "paella": 46924, + "cryptic": 46925, + "sedg": 46926, + "mend": 46927, + "dunkirk": 46928, + "bred": 46929, + "wald": 46930, + "sportscar": 46931, + "aard": 46932, + "wheaton": 46933, + "daener": 46934, + "klan": 46935, + "brt": 46936, + "bakhtawar": 46937, + "spires": 46938, + "schubert": 46939, + "roti": 46940, + "polish": 46941, + "ose": 46942, + "agame": 46943, + "wondercon": 46944, + "protestant": 46945, + "bosa": 46946, + "ðŁĺŁ": 46947, + "dü": 46948, + "joyride": 46949, + "gertrude": 46950, + "âĿĿ": 46951, + "gila": 46952, + "vh": 46953, + "twa": 46954, + "trav": 46955, + "swallowed": 46956, + "starve": 46957, + "lain": 46958, + "entren": 46959, + "reiki": 46960, + "sukh": 46961, + "craic": 46962, + "azu": 46963, + "webpage": 46964, + "keefe": 46965, + "hypothe": 46966, + "hirsch": 46967, + "helle": 46968, + "campground": 46969, + "wamy": 46970, + "travi": 46971, + "shahi": 46972, + "sandeep": 46973, + "rui": 46974, + "hanuman": 46975, + "dwp": 46976, + "repository": 46977, + "noor": 46978, + "noff": 46979, + "unreal": 46980, + "pell": 46981, + "blackhistory": 46982, + "harvick": 46983, + "mascar": 46984, + "payee": 46985, + "pasha": 46986, + "gastronomy": 46987, + "dÃŃ": 46988, + "aig": 46989, + "rosenthal": 46990, + "openday": 46991, + "embellished": 46992, + "ttip": 46993, + "sunbathing": 46994, + "gopack": 46995, + "endome": 46996, + "ï¸ı#": 46997, + "invalid": 46998, + "finalfour": 46999, + "stfu": 47000, + "squishy": 47001, + "rasta": 47002, + "mosch": 47003, + "jamesc": 47004, + "dietrich": 47005, + "sela": 47006, + "melb": 47007, + "elvi": 47008, + "tdp": 47009, + "suni": 47010, + "slit": 47011, + "jha": 47012, + "biza": 47013, + "spiked": 47014, + "lli": 47015, + "lillard": 47016, + "vampi": 47017, + "synopsis": 47018, + "azhar": 47019, + "kendricklamar": 47020, + "ĮãĤĬãģŁãģĦ": 47021, + "heartless": 47022, + "countryfile": 47023, + "airplay": 47024, + "arrogance": 47025, + "pree": 47026, + "virtuoso": 47027, + "ãħłãħłãħłãħł": 47028, + "raju": 47029, + "lebu": 47030, + "forward": 47031, + "tug": 47032, + "dros": 47033, + "mondaymotivaton": 47034, + "concepcion": 47035, + "thelo": 47036, + "padi": 47037, + "looool": 47038, + "ÑĢод": 47039, + "itss": 47040, + "ethical": 47041, + "enduro": 47042, + "__:": 47043, + "expenditure": 47044, + "monste": 47045, + "masking": 47046, + "terriers": 47047, + "ibis": 47048, + "ember": 47049, + "cumple": 47050, + "punctuation": 47051, + "piper": 47052, + "irvin": 47053, + "adee": 47054, + "yyyyyy": 47055, + "flashbacks": 47056, + "celsius": 47057, + "donnie": 47058, + "bogota": 47059, + "benevol": 47060, + "thescript": 47061, + "shilpa": 47062, + "prose": 47063, + "findia": 47064, + "zeke": 47065, + "neko": 47066, + "doves": 47067, + "blueslyrix": 47068, + "frosh": 47069, + "soweto": 47070, + "mplo": 47071, + "alai": 47072, + "sabi": 47073, + "raqqa": 47074, + "wftv": 47075, + "stroller": 47076, + "iansomerhalder": 47077, + "ðŁĶª": 47078, + "anon": 47079, + "moseley": 47080, + "!?!?": 47081, + "staking": 47082, + "moly": 47083, + "cartri": 47084, + "csg": 47085, + "astor": 47086, + "transcend": 47087, + "maer": 47088, + "deux": 47089, + "cowgirl": 47090, + "sask": 47091, + "punter": 47092, + "maken": 47093, + "oates": 47094, + "lovett": 47095, + "growler": 47096, + "sagin": 47097, + "vn": 47098, + "ssible": 47099, + "officeofrg": 47100, + "ymc": 47101, + "sabar": 47102, + "faulty": 47103, + "apha": 47104, + "akon": 47105, + "ðŁij«": 47106, + "snowdon": 47107, + "aew": 47108, + "raisethe": 47109, + "ðĿĵ": 47110, + "gruesome": 47111, + "clementine": 47112, + "sping": 47113, + "lata": 47114, + "worldenviron": 47115, + "mimic": 47116, + "canaria": 47117, + "bakhtawarbz": 47118, + "aoa": 47119, + "fala": 47120, + "ãĤŃ": 47121, + "aviva": 47122, + "youuuu": 47123, + "thigh": 47124, + "ladders": 47125, + "gumbo": 47126, + "tzky": 47127, + "fuzz": 47128, + "plasticpollution": 47129, + "estate": 47130, + "strengthened": 47131, + "kant": 47132, + "drin": 47133, + "calvert": 47134, + "transformational": 47135, + "frightened": 47136, + "maclean": 47137, + "elitedangerous": 47138, + "earthy": 47139, + "tson": 47140, + "toda": 47141, + "jnu": 47142, + "..,": 47143, + "michal": 47144, + "iban": 47145, + "jeong": 47146, + "isreal": 47147, + "simcoe": 47148, + "exclusives": 47149, + "bluebells": 47150, + "bene": 47151, + "teu": 47152, + "pilsner": 47153, + "penske": 47154, + "atheists": 47155, + "mpu": 47156, + "cartagena": 47157, + "ðŁĴĹðŁĴĹ": 47158, + "millionaires": 47159, + "kkkk": 47160, + "itar": 47161, + "subscriptions": 47162, + "remote": 47163, + "mafi": 47164, + "hinton": 47165, + "wcc": 47166, + "hok": 47167, + "dsb": 47168, + "ableton": 47169, + "seventy": 47170, + "punks": 47171, + "eindhoven": 47172, + "shone": 47173, + "mcfarlane": 47174, + "limpopo": 47175, + "emphasi": 47176, + "ü": 47177, + "sinfo": 47178, + "petre": 47179, + "mangrove": 47180, + "chino": 47181, + "bertie": 47182, + "playlists": 47183, + "pushawards": 47184, + "paf": 47185, + "debbie": 47186, + "cdo": 47187, + "rino": 47188, + "ðŁı¾âĢįâĻĤï¸ı": 47189, + "folke": 47190, + "bonnar": 47191, + "thine": 47192, + "slan": 47193, + "halter": 47194, + "evie": 47195, + "awsome": 47196, + "vultures": 47197, + "sparky": 47198, + "seizures": 47199, + "âľĶ": 47200, + "ramone": 47201, + "ineffe": 47202, + "aln": 47203, + "proctor": 47204, + "astra": 47205, + "thevoice": 47206, + "grote": 47207, + "scion": 47208, + "deadline": 47209, + "amaya": 47210, + "tainted": 47211, + "patterned": 47212, + "exceeding": 47213, + "crossfit": 47214, + "kaylee": 47215, + "dropbox": 47216, + "rushes": 47217, + "tackled": 47218, + "moby": 47219, + "retrogamer": 47220, + "ncbd": 47221, + "benefitting": 47222, + "shaykh": 47223, + "guildhall": 47224, + "gentry": 47225, + "dreamcast": 47226, + "dreaded": 47227, + "bundled": 47228, + "thaw": 47229, + "revolving": 47230, + "npt": 47231, + "kyliejenner": 47232, + "imaginative": 47233, + "roni": 47234, + "overcame": 47235, + "familytime": 47236, + "dsburg": 47237, + "carnaval": 47238, + "relationship": 47239, + "recognizable": 47240, + "coroner": 47241, + "hole": 47242, + "fanfic": 47243, + "emirates": 47244, + "burritos": 47245, + "analyse": 47246, + "thinner": 47247, + "nees": 47248, + "gallipoli": 47249, + "blr": 47250, + "catwoman": 47251, + "-->>": 47252, + "ault": 47253, + "adaily": 47254, + "naughty": 47255, + "ilio": 47256, + "solitaire": 47257, + "mtvbr": 47258, + "jocelyn": 47259, + "arunach": 47260, + "repent": 47261, + "southgate": 47262, + "hyacin": 47263, + "essential": 47264, + "fenton": 47265, + "andum": 47266, + "itor": 47267, + "gopal": 47268, + "slinger": 47269, + "posei": 47270, + "awil": 47271, + "wielding": 47272, + "raila": 47273, + "elias": 47274, + "asto": 47275, + "ä": 47276, + "tendency": 47277, + "strata": 47278, + "kert": 47279, + "<-": 47280, + "imacele": 47281, + "daes": 47282, + "stimulus": 47283, + "hanley": 47284, + "fitnes": 47285, + "ecstasy": 47286, + "limous": 47287, + "hailing": 47288, + "ðŁ¤Ń": 47289, + "chiswick": 47290, + "taries": 47291, + "slav": 47292, + "puli": 47293, + "modernization": 47294, + "blackmail": 47295, + "bingham": 47296, + "hfx": 47297, + "++": 47298, + "ðŁĩ®ðŁĩ³": 47299, + "niv": 47300, + "wea": 47301, + "professor": 47302, + "koff": 47303, + "bolster": 47304, + "suave": 47305, + "sequences": 47306, + "pepperoni": 47307, + "notte": 47308, + "dren": 47309, + "ãģ¨ç¹ĭãģ": 47310, + "hsv": 47311, + "oga": 47312, + "aptly": 47313, + "zad": 47314, + "excelsi": 47315, + "rinka": 47316, + "moldova": 47317, + "minn": 47318, + "mabel": 47319, + "conferencing": 47320, + "basing": 47321, + "ofer": 47322, + "obsi": 47323, + "hamillhimself": 47324, + "careless": 47325, + "briefed": 47326, + "inherent": 47327, + "parish": 47328, + "dubnation": 47329, + "townsville": 47330, + "sarawak": 47331, + "geeky": 47332, + "doncasterisgreat": 47333, + "wasabi": 47334, + "gup": 47335, + "pheno": 47336, + "drainthe": 47337, + "carrieunderwood": 47338, + "bleeds": 47339, + "bbcworld": 47340, + "anew": 47341, + "altaf": 47342, + "dulwich": 47343, + "aniston": 47344, + "wti": 47345, + "sumatra": 47346, + "grafton": 47347, + "bln": 47348, + "mester": 47349, + "bodega": 47350, + "rego": 47351, + "esq": 47352, + "anjo": 47353, + "sumptuous": 47354, + "maisie": 47355, + "�": 47356, + "wilt": 47357, + "jakob": 47358, + "elvis": 47359, + "sepul": 47360, + "muster": 47361, + "airpollution": 47362, + "presidente": 47363, + "happymonday": 47364, + "extensively": 47365, + "flondon": 47366, + "tls": 47367, + "playing": 47368, + "peed": 47369, + "dinho": 47370, + "vardy": 47371, + "pika": 47372, + "niro": 47373, + "aucus": 47374, + "ðŁį¦": 47375, + "null": 47376, + "elondon": 47377, + "juventus": 47378, + "imagines": 47379, + "disab": 47380, + "lito": 47381, + "dura": 47382, + "workplaces": 47383, + "promote": 47384, + "mccaf": 47385, + "woodwork": 47386, + "wawx": 47387, + "ப": 47388, + "ttino": 47389, + "shari": 47390, + "semper": 47391, + "bettertogether": 47392, + "ðŁijĬðŁı»": 47393, + "zebra": 47394, + "pondering": 47395, + "enchil": 47396, + "hom": 47397, + "cosmic": 47398, + "tanz": 47399, + "mocked": 47400, + "eccc": 47401, + "athed": 47402, + "abolish": 47403, + "propeller": 47404, + "parisagreement": 47405, + "assemblies": 47406, + "industry": 47407, + "fraudulent": 47408, + "pesa": 47409, + "changmin": 47410, + "axx": 47411, + "ðŁĴµ": 47412, + "irrational": 47413, + "cusa": 47414, + "ramadhan": 47415, + "octavia": 47416, + "onelove": 47417, + "jacki": 47418, + "barak": 47419, + "taxider": 47420, + "serious": 47421, + "nathanfillion": 47422, + "mcen": 47423, + "chk": 47424, + "popart": 47425, + "gravity": 47426, + "coppola": 47427, + "readingfc": 47428, + "illusions": 47429, + "jig": 47430, + "wwx": 47431, + "resh": 47432, + "exporting": 47433, + "buzzard": 47434, + "âĻ¤": 47435, + "pcm": 47436, + "lanapar": 47437, + "kos": 47438, + "aromas": 47439, + "antalya": 47440, + "wwdc": 47441, + "vena": 47442, + "phila": 47443, + "ballin": 47444, + "ðŁijĦ": 47445, + "quinta": 47446, + "mao": 47447, + "fery": 47448, + "eighty": 47449, + "sentiments": 47450, + "safeguarding": 47451, + "rwa": 47452, + "puffs": 47453, + "lucille": 47454, + "decath": 47455, + "slu": 47456, + "nugent": 47457, + "deter": 47458, + "brazil": 47459, + "zeiss": 47460, + "superbowl": 47461, + "subsidy": 47462, + "altern": 47463, + "hidalgo": 47464, + "enzymes": 47465, + "ä½": 47466, + "tagne": 47467, + "hairdresser": 47468, + "adrien": 47469, + "walkout": 47470, + "opposes": 47471, + "cantina": 47472, + "bedside": 47473, + "afan": 47474, + "ðŁĶĹ": 47475, + "prophetic": 47476, + "danes": 47477, + "unsuccessful": 47478, + "supercharged": 47479, + "pkk": 47480, + "exemption": 47481, + "hartle": 47482, + "secular": 47483, + "clipping": 47484, + "brs": 47485, + "unitedway": 47486, + "cnet": 47487, + "patchy": 47488, + "hagan": 47489, + "een": 47490, + "âļľ": 47491, + "vara": 47492, + "sympathi": 47493, + "nevertrump": 47494, + "affirmation": 47495, + "omf": 47496, + "nycfc": 47497, + "maja": 47498, + "surro": 47499, + "keerth": 47500, + "upscale": 47501, + "sandalwood": 47502, + "monarchy": 47503, + "knobs": 47504, + "åĭ": 47505, + "potholes": 47506, + "hungergames": 47507, + "terraces": 47508, + "nasir": 47509, + "counsell": 47510, + "welcometo": 47511, + "waq": 47512, + "seaman": 47513, + "mita": 47514, + "stunningly": 47515, + "ontheroad": 47516, + "inability": 47517, + ")!!": 47518, + "bongo": 47519, + "antv": 47520, + "sput": 47521, + "worldenvironmentday": 47522, + "resusc": 47523, + "ytd": 47524, + "fim": 47525, + "eunhyuk": 47526, + "sachin": 47527, + "roseanne": 47528, + "clermont": 47529, + "apec": 47530, + "amina": 47531, + "vening": 47532, + "nantes": 47533, + "almost": 47534, + "sinus": 47535, + "exas": 47536, + "tyl": 47537, + "tien": 47538, + "plead": 47539, + "lancs": 47540, + "burnaby": 47541, + "rek": 47542, + "joom": 47543, + "observers": 47544, + "discography": 47545, + "clg": 47546, + "âĻ¦": 47547, + "snack": 47548, + "rti": 47549, + "oily": 47550, + "crystalli": 47551, + "brute": 47552, + "webdevelopment": 47553, + "toppings": 47554, + "laf": 47555, + "anis": 47556, + "adder": 47557, + "reliving": 47558, + "carlin": 47559, + "battleof": 47560, + "weg": 47561, + "syrian": 47562, + "pont": 47563, + "ndc": 47564, + "laghate": 47565, + "yuma": 47566, + "spp": 47567, + "piti": 47568, + "robbing": 47569, + "marting": 47570, + "reykja": 47571, + "rajput": 47572, + "ncds": 47573, + "kiewicz": 47574, + "âĢ¢âĢ¢": 47575, + "vampire": 47576, + "substantially": 47577, + "opioids": 47578, + "nepali": 47579, + "kline": 47580, + "aroo": 47581, + "understand": 47582, + "litt": 47583, + "uit": 47584, + "thrombo": 47585, + "saries": 47586, + "quot": 47587, + "balling": 47588, + "ttr": 47589, + "sgh": 47590, + "philipp": 47591, + "brant": 47592, + "acl": 47593, + "mello": 47594, + "whittaker": 47595, + ".;": 47596, + "defiant": 47597, + "bgc": 47598, + "replying": 47599, + "mirren": 47600, + "metamorpho": 47601, + "schwab": 47602, + "bulge": 47603, + "utilized": 47604, + "pickering": 47605, + "pardon": 47606, + "dsa": 47607, + "à¸Ī": 47608, + "dooley": 47609, + "cumulative": 47610, + "л": 47611, + "urgency": 47612, + "emir": 47613, + "+/-": 47614, + "¦Ī": 47615, + "otas": 47616, + "âı³": 47617, + "stationed": 47618, + "grapevine": 47619, + "arac": 47620, + "karanjohar": 47621, + "fancy": 47622, + "saul": 47623, + "coogs": 47624, + "lgbtq": 47625, + "اÙħ": 47626, + "javi": 47627, + "ummer": 47628, + "pll": 47629, + "denis": 47630, + "daipur": 47631, + "puffin": 47632, + "lewisham": 47633, + "fandom": 47634, + "cope": 47635, + "vesmatter": 47636, + "sve": 47637, + "helpless": 47638, + "deodor": 47639, + "ostrich": 47640, + "kazan": 47641, + "fridaythe": 47642, + "condor": 47643, + "vx": 47644, + "sophomores": 47645, + "robles": 47646, + "cutt": 47647, + "climbers": 47648, + "리": 47649, + "sleg": 47650, + "snf": 47651, + "macys": 47652, + "hydrating": 47653, + "groupe": 47654, + "poyn": 47655, + "moulin": 47656, + "hgtv": 47657, + "lmfaooo": 47658, + "sulphur": 47659, + "asdfghjkl": 47660, + "annabelle": 47661, + "humpback": 47662, + "braved": 47663, + "viswasam": 47664, + "multipurpose": 47665, + "humidi": 47666, + "escorted": 47667, + "barbican": 47668, + "fad": 47669, + "corsa": 47670, + "ðŁ¤«": 47671, + "pippa": 47672, + "hereto": 47673, + "cany": 47674, + "sergi": 47675, + "orcas": 47676, + "ovie": 47677, + "edou": 47678, + "sany": 47679, + "globalization": 47680, + "mancini": 47681, + "foodtruck": 47682, + "fis": 47683, + "defibrill": 47684, + "schre": 47685, + "smafia": 47686, + "lovewins": 47687, + "laut": 47688, + "kaka": 47689, + "hollande": 47690, + "gameon": 47691, + "resurgence": 47692, + "outside": 47693, + "olympiad": 47694, + "intan": 47695, + "abstraction": 47696, + "rapid": 47697, + "palom": 47698, + "calle": 47699, + "jasmin": 47700, + "attackers": 47701, + "swagg": 47702, + "mitra": 47703, + "kylo": 47704, + "ல": 47705, + "hermitage": 47706, + "gordo": 47707, + "eira": 47708, + "sosfam": 47709, + "rollout": 47710, + "excite": 47711, + "synod": 47712, + "merrill": 47713, + "cals": 47714, + "assa": 47715, + "livelihoods": 47716, + "juve": 47717, + "theblack": 47718, + "gopackgo": 47719, + "antlers": 47720, + "albanian": 47721, + "woolly": 47722, + "quiche": 47723, + "purification": 47724, + "areth": 47725, + "smarthome": 47726, + "nek": 47727, + "allblacks": 47728, + "mexicans": 47729, + "ism": 47730, + "germs": 47731, + "complexion": 47732, + "marck": 47733, + "ushi": 47734, + "ðŁIJIJ": 47735, + "charl": 47736, + "castic": 47737, + "tillerson": 47738, + "giuliani": 47739, + "biodegradable": 47740, + "malbec": 47741, + "bois": 47742, + "jubil": 47743, + "imes": 47744, + "rame": 47745, + "genetic": 47746, + "espnu": 47747, + "chley": 47748, + "soho": 47749, + "gopher": 47750, + "gsc": 47751, + "buuren": 47752, + "cube": 47753, + "bridesmaids": 47754, + "webinars": 47755, + "toe": 47756, + "manipur": 47757, + "violently": 47758, + "noticias": 47759, + "exchanging": 47760, + "chiev": 47761, + "replaceable": 47762, + "muaythai": 47763, + "buss": 47764, + "spil": 47765, + "instalment": 47766, + "divya": 47767, + "caitlin": 47768, + "olim": 47769, + "filtering": 47770, + "whirlwind": 47771, + "stared": 47772, + "priorit": 47773, + "pram": 47774, + "pompeii": 47775, + "monologue": 47776, + "kite": 47777, + "buka": 47778, + "âĢ¦..": 47779, + "vaccine": 47780, + "brero": 47781, + "wozni": 47782, + "solent": 47783, + "referr": 47784, + "myrt": 47785, + "gridiron": 47786, + "galatasaray": 47787, + "froze": 47788, + "claremont": 47789, + "ðŁ¥ĥ": 47790, + "victorias": 47791, + "sseldorf": 47792, + "pastures": 47793, + "netneutrality": 47794, + "chor": 47795, + "ðŁijģ": 47796, + "ಿ": 47797, + "weho": 47798, + "symptom": 47799, + "josel": 47800, + "inous": 47801, + "dragoncon": 47802, + "powerball": 47803, + "pte": 47804, + "fourthofjuly": 47805, + "ecla": 47806, + "earbuds": 47807, + "whereabouts": 47808, + "saltlife": 47809, + "deprivation": 47810, + "chter": 47811, + "wiggle": 47812, + "system": 47813, + "psst": 47814, + "chaz": 47815, + "dany": 47816, + "rimo": 47817, + "oaxaca": 47818, + "lanaparrilla": 47819, + "barcelon": 47820, + "melancholy": 47821, + "wayback": 47822, + "hotro": 47823, + "nsi": 47824, + "lilly": 47825, + "kuro": 47826, + "jahan": 47827, + "intellect": 47828, + "boardgame": 47829, + "ðŁıĬ": 47830, + "sneakpeek": 47831, + "kprc": 47832, + "jails": 47833, + "candel": 47834, + "zanzi": 47835, + "mortimer": 47836, + "starch": 47837, + "rags": 47838, + "pfa": 47839, + "longlive": 47840, + "kart": 47841, + "girona": 47842, + "crocker": 47843, + "christoph": 47844, + "precautions": 47845, + "warship": 47846, + "perm": 47847, + "parent": 47848, + "vangogh": 47849, + "gifford": 47850, + "allegheny": 47851, + "rayn": 47852, + "utm": 47853, + "stencil": 47854, + "recalling": 47855, + "penney": 47856, + "zazzle": 47857, + "ìĥĿ": 47858, + "hinds": 47859, + "arenas": 47860, + "nuev": 47861, + "lawler": 47862, + "guin": 47863, + "dothis": 47864, + "ðŁijķ": 47865, + "ì¶ķíķĺ": 47866, + "weg": 47867, + "tib": 47868, + "ridin": 47869, + "complexes": 47870, + "turbulent": 47871, + "pesos": 47872, + "demarcus": 47873, + "vallarta": 47874, + "samsun": 47875, + "kisses": 47876, + "heinrich": 47877, + "deportes": 47878, + "wilms": 47879, + "urd": 47880, + "thenext": 47881, + "inkigayo": 47882, + "howi": 47883, + "firsts": 47884, + "carriage": 47885, + "cleanliness": 47886, + "maswar": 47887, + "isch": 47888, + "axel": 47889, + "sizzle": 47890, + "roadhouse": 47891, + "frans": 47892, + "entourage": 47893, + "cobble": 47894, + "booth": 47895, + "benedict": 47896, + "talon": 47897, + "fcu": 47898, + "yearofthe": 47899, + "rayon": 47900, + "raidernation": 47901, + "foyle": 47902, + "koval": 47903, + "pianos": 47904, + "lpg": 47905, + "burmese": 47906, + "manure": 47907, + "geocaching": 47908, + "coscino": 47909, + "bnp": 47910, + "ferra": 47911, + "strophy": 47912, + "marais": 47913, + "cees": 47914, + "legendof": 47915, + "katniss": 47916, + "enoch": 47917, + "aved": 47918, + "youknow": 47919, + "dprk": 47920, + "ðŁĺ¢ðŁĺ¢": 47921, + "spun": 47922, + "prost": 47923, + "sorrows": 47924, + "centred": 47925, + "kea": 47926, + "galicia": 47927, + "?ðŁ¤Ķ": 47928, + "ÑĢода": 47929, + "bouchard": 47930, + "ðŁĴĻðŁĴľ": 47931, + "yui": 47932, + "seedlings": 47933, + "jonah": 47934, + "recovers": 47935, + "nyrd": 47936, + "boardroom": 47937, + "suma": 47938, + "myjaps": 47939, + "tung": 47940, + "shai": 47941, + "irgc": 47942, + "elio": 47943, + "wagons": 47944, + "kashi": 47945, + "policemen": 47946, + "johnnie": 47947, + "alecoscino": 47948, + "shopify": 47949, + "dotted": 47950, + "detri": 47951, + "vaw": 47952, + "tofficial": 47953, + "inyour": 47954, + "chalmers": 47955, + "traced": 47956, + "novi": 47957, + "byes": 47958, + "ariel": 47959, + "nippon": 47960, + "lapel": 47961, + "griez": 47962, + "bgs": 47963, + "fooling": 47964, + "dita": 47965, + "vijaysethu": 47966, + "nmwx": 47967, + "asot": 47968, + "kranti": 47969, + "helm": 47970, + "vedi": 47971, + "sickest": 47972, + "mochi": 47973, + "kabo": 47974, + "shrubs": 47975, + "hered": 47976, + "bsp": 47977, + "sqm": 47978, + "hamr": 47979, + "dulkar": 47980, + "antha": 47981, + "nrf": 47982, + "avoidance": 47983, + "aten": 47984, + "publix": 47985, + "bearers": 47986, + "nasi": 47987, + "hap": 47988, + "hells": 47989, + "ðŁĸ¥": 47990, + "ื": 47991, + "thelastjedi": 47992, + "ohwx": 47993, + "ðŁį«": 47994, + "wahoo": 47995, + "therese": 47996, + "recaps": 47997, + "ssnhq": 47998, + "birdphotography": 47999, + "vay": 48000, + "petti": 48001, + "paulo": 48002, + "belvedere": 48003, + "(*": 48004, + "grl": 48005, + "duvet": 48006, + "cpec": 48007, + "sait": 48008, + "porsch": 48009, + "measurable": 48010, + "aviators": 48011, + "fremantle": 48012, + "breen": 48013, + "onom": 48014, + "meand": 48015, + "lifesaving": 48016, + "euref": 48017, + "endon": 48018, + "embaras": 48019, + "airasia": 48020, + "elis": 48021, + "dunkin": 48022, + "starmagic": 48023, + "sill": 48024, + "portobello": 48025, + "kiefer": 48026, + "exe": 48027, + "muted": 48028, + "ãģ¦": 48029, + "wethepeople": 48030, + "logia": 48031, + "liberal": 48032, + "theforceawakens": 48033, + "mined": 48034, + "haunts": 48035, + "freckles": 48036, + "caretaker": 48037, + "sindia": 48038, + "âķIJ": 48039, + "devlin": 48040, + "liston": 48041, + "directioner": 48042, + "ohn": 48043, + "figaro": 48044, + "emmanuel": 48045, + "dubois": 48046, + "clones": 48047, + "bruise": 48048, + "ðŁİĪðŁİī": 48049, + "disinfe": 48050, + "dermatology": 48051, + "asr": 48052, + "swatch": 48053, + "discomfort": 48054, + "tamanna": 48055, + "piday": 48056, + "macken": 48057, + "katic": 48058, + "delusional": 48059, + "shawnee": 48060, + "gud": 48061, + "albino": 48062, + "pali": 48063, + "dingh": 48064, + "cucumbers": 48065, + "coffey": 48066, + "anticipating": 48067, + "treasured": 48068, + "websummit": 48069, + "sheltered": 48070, + "savor": 48071, + "pedagogy": 48072, + "mgs": 48073, + "shma": 48074, + "sbu": 48075, + "denali": 48076, + "campos": 48077, + "bubblegum": 48078, + "oir": 48079, + "leaps": 48080, + "yler": 48081, + "rone": 48082, + "sanskrit": 48083, + "mint": 48084, + "meatless": 48085, + "futurist": 48086, + "dude": 48087, + "avel": 48088, + "protested": 48089, + "squire": 48090, + "zaki": 48091, + "szn": 48092, + "harcourt": 48093, + "cyclone": 48094, + "bourdain": 48095, + "gatherings": 48096, + "dant": 48097, + "adventurer": 48098, + "paragon": 48099, + "altman": 48100, + "dding": 48101, + "banerjee": 48102, + "snorkeling": 48103, + "motherwell": 48104, + "missy": 48105, + "ender": 48106, + "glows": 48107, + "kiwis": 48108, + "chickpea": 48109, + "poro": 48110, + "efron": 48111, + "appt": 48112, + "uy": 48113, + "specified": 48114, + "gabby": 48115, + "estrada": 48116, + "combos": 48117, + "bourbon": 48118, + "vini": 48119, + "varun": 48120, + "stephani": 48121, + "keywords": 48122, + "carvings": 48123, + "amitabh": 48124, + "wrought": 48125, + "twal": 48126, + "reels": 48127, + "clubbing": 48128, + "ubiquit": 48129, + "crit": 48130, + "ambedkar": 48131, + "æĻ": 48132, + "pruning": 48133, + "vaccinated": 48134, + "boeing": 48135, + "sks": 48136, + "loona": 48137, + "hypnosis": 48138, + "edelman": 48139, + "phol": 48140, + "hew": 48141, + "colosse": 48142, + "mckinsey": 48143, + "uon": 48144, + "tote": 48145, + "sacrificing": 48146, + "oxi": 48147, + "nang": 48148, + "emu": 48149, + "пÑĢиÑĢода": 48150, + "mth": 48151, + "kerswednesday": 48152, + "argued": 48153, + "timelapse": 48154, + "risking": 48155, + "regulating": 48156, + "nigh": 48157, + "likelihood": 48158, + "cubic": 48159, + "auction": 48160, + "reinfor": 48161, + "pistor": 48162, + "noses": 48163, + "yel": 48164, + "snuggles": 48165, + "pei": 48166, + "jeanette": 48167, + "taku": 48168, + "rith": 48169, + "guyz": 48170, + "à¸ŀ": 48171, + "yte": 48172, + "verted": 48173, + "paysoff": 48174, + "jauregui": 48175, + "hooligans": 48176, + "procedural": 48177, + "mib": 48178, + "hardy": 48179, + "eleng": 48180, + "checkers": 48181, + "alline": 48182, + "themet": 48183, + "proudof": 48184, + "keerthyofficial": 48185, + "collaborator": 48186, + "niu": 48187, + "inflicted": 48188, + "advani": 48189, + "retwee": 48190, + "memoriam": 48191, + "ficial": 48192, + "tighter": 48193, + "salem": 48194, + "reviewers": 48195, + "brics": 48196, + "bendigo": 48197, + "amell": 48198, + "turkish": 48199, + "sushmaswar": 48200, + "paulson": 48201, + "palawan": 48202, + "mollie": 48203, + "stitcher": 48204, + "sburgh": 48205, + "iru": 48206, + "haydn": 48207, + "eners": 48208, + "aroa": 48209, + "uzzi": 48210, + "sarajevo": 48211, + "hela": 48212, + "apollo": 48213, + "ninety": 48214, + "vaca": 48215, + "spon": 48216, + "ventu": 48217, + "jelena": 48218, + "heifer": 48219, + "avoids": 48220, + "spine": 48221, + "prize": 48222, + "marist": 48223, + "recreating": 48224, + "mede": 48225, + "wooden": 48226, + "findlay": 48227, + "rofl": 48228, + "ndi": 48229, + "comprehend": 48230, + "yugo": 48231, + "yü": 48232, + "towork": 48233, + "ufos": 48234, + "sonar": 48235, + "piston": 48236, + "recording": 48237, + "tentative": 48238, + "artforsale": 48239, + "pellets": 48240, + "fredo": 48241, + "ÙĪر": 48242, + "muses": 48243, + "customization": 48244, + "profound": 48245, + "isner": 48246, + "ideally": 48247, + "siam": 48248, + "plankton": 48249, + "cmdr": 48250, + "manger": 48251, + "franken": 48252, + "customizable": 48253, + "म": 48254, + "walkaway": 48255, + "swivel": 48256, + "vastly": 48257, + "noton": 48258, + "lexa": 48259, + "exmoor": 48260, + "zas": 48261, + "tante": 48262, + "reductions": 48263, + "lolly": 48264, + "hipsters": 48265, + "benefited": 48266, + "ë²": 48267, + "wwwww": 48268, + "masculine": 48269, + "fiji": 48270, + "drey": 48271, + "phill": 48272, + "aneous": 48273, + "nicol": 48274, + "mendez": 48275, + "disappro": 48276, + "chner": 48277, + "throughs": 48278, + "shenmue": 48279, + "eastman": 48280, + "ðŁIJİ": 48281, + "yuck": 48282, + "undertale": 48283, + "reys": 48284, + "gobeavs": 48285, + "engen": 48286, + "cna": 48287, + "merr": 48288, + "birk": 48289, + "ãģ¨ç¹ĭãģĮãĤĬãģŁãģĦ": 48290, + "âĥ£@": 48291, + "ynna": 48292, + "steed": 48293, + "offender": 48294, + "atum": 48295, + "vanishing": 48296, + "presidenti": 48297, + "lovethem": 48298, + "gnocchi": 48299, + "friggin": 48300, + "peril": 48301, + "madhya": 48302, + "agne": 48303, + "deejay": 48304, + "marnock": 48305, + "mtb": 48306, + "foldable": 48307, + "@___": 48308, + "standre": 48309, + "bronx": 48310, + "bowski": 48311, + "finite": 48312, + "crockett": 48313, + "bsf": 48314, + "getit": 48315, + "serenawilliams": 48316, + "miro": 48317, + "ignatius": 48318, + "slay": 48319, + "rinse": 48320, + "fondue": 48321, + "seldom": 48322, + "smore": 48323, + "gani": 48324, + "dyce": 48325, + "dmitry": 48326, + "crumb": 48327, + "latepost": 48328, + "primark": 48329, + "ohana": 48330, + "florals": 48331, + "doa": 48332, + "remembranceday": 48333, + "dds": 48334, + "azione": 48335, + "toonami": 48336, + "airport": 48337, + "æĿ±": 48338, + "thad": 48339, + "fist": 48340, + "dinesh": 48341, + "drwho": 48342, + "adwords": 48343, + "admirer": 48344, + "proje": 48345, + "kyrgyz": 48346, + "à«": 48347, + "manifestation": 48348, + "lewan": 48349, + "jic": 48350, + "thibau": 48351, + "leased": 48352, + "vanity": 48353, + "nourished": 48354, + "nevertheless": 48355, + "augmente": 48356, + "fuelled": 48357, + "chead": 48358, + "wilshere": 48359, + "rudi": 48360, + "pz": 48361, + "myco": 48362, + "morro": 48363, + "herbalife": 48364, + "hardrock": 48365, + "deman": 48366, + "dreality": 48367, + "spades": 48368, + "cevic": 48369, + "bhai": 48370, + "baron": 48371, + "ultimatefan": 48372, + "hounews": 48373, + "tobi": 48374, + "strut": 48375, + "keel": 48376, + "affiliation": 48377, + "themasters": 48378, + "smal": 48379, + "hue": 48380, + "esteban": 48381, + "conv": 48382, + "omnic": 48383, + "databases": 48384, + "cov": 48385, + "terti": 48386, + "stg": 48387, + "snoopdogg": 48388, + "metabol": 48389, + "lethbridge": 48390, + "ðŁı»âĢįâĻĢï¸ı": 48391, + "yearling": 48392, + "residentevil": 48393, + "nwsl": 48394, + "iyaki": 48395, + "griezmann": 48396, + "cous": 48397, + "ðŁĵĿ:": 48398, + "torian": 48399, + "sami": 48400, + "ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥": 48401, + "gare": 48402, + "alliances": 48403, + "whitfield": 48404, + "wether": 48405, + "refining": 48406, + "coyi": 48407, + "kraken": 48408, + "ðŁĺĺâĿ¤": 48409, + "singularity": 48410, + "lili": 48411, + "hns": 48412, + "boldand": 48413, + "wawrinka": 48414, + "misogyny": 48415, + "lovers": 48416, + "cq": 48417, + "bdg": 48418, + "adona": 48419, + "garter": 48420, + "womenof": 48421, + "scd": 48422, + "recognising": 48423, + "muna": 48424, + "strou": 48425, + "signalling": 48426, + "laredo": 48427, + "hellboy": 48428, + "aleksand": 48429, + "unavailable": 48430, + "pediatric": 48431, + "asin": 48432, + "meria": 48433, + "rishi": 48434, + "futurism": 48435, + "wye": 48436, + "polarized": 48437, + "ewe": 48438, + "propel": 48439, + "informs": 48440, + "crease": 48441, + "~\"": 48442, + "artiston": 48443, + "likefor": 48444, + "heidelberg": 48445, + "erra": 48446, + "lifein": 48447, + "lenny": 48448, + "interrupt": 48449, + "coherent": 48450, + "caz": 48451, + "vickers": 48452, + "leveled": 48453, + "fbs": 48454, + "cabins": 48455, + "bummed": 48456, + "apostles": 48457, + "weh": 48458, + "tendon": 48459, + "souvenirs": 48460, + "infuri": 48461, + "pierce": 48462, + "asset": 48463, + "mlas": 48464, + "goth": 48465, + "diggin": 48466, + "annas": 48467, + "ylor": 48468, + "thwaite": 48469, + "swel": 48470, + "panera": 48471, + "murderers": 48472, + "crooked": 48473, + "bsgo": 48474, + "acu": 48475, + "aon": 48476, + "rean": 48477, + "oneof": 48478, + "kohl": 48479, + "bloodh": 48480, + "pesticide": 48481, + "lostdog": 48482, + "flexing": 48483, + "ëĤĺ": 48484, + "supra": 48485, + "eternally": 48486, + "ðŁļĻ": 48487, + "paolo": 48488, + "olan": 48489, + "momo": 48490, + "iselle": 48491, + "captainmarvel": 48492, + "slou": 48493, + "mistakenly": 48494, + "akhilesh": 48495, + "mert": 48496, + "ilinan": 48497, + "buon": 48498, + "balkan": 48499, + "mirro": 48500, + "millen": 48501, + "derail": 48502, + "damon": 48503, + "titi": 48504, + "bios": 48505, + "redon": 48506, + "picard": 48507, + "parte": 48508, + "ðŁ¤Ł": 48509, + "غ": 48510, + "sonics": 48511, + "firsth": 48512, + "ddc": 48513, + "vegans": 48514, + "turban": 48515, + "nigan": 48516, + "lottie": 48517, + "lyndon": 48518, + "starbuck": 48519, + "pinkfloyd": 48520, + "lifestyles": 48521, + "amara": 48522, + "ashe": 48523, + "rsc": 48524, + "vala": 48525, + "smer": 48526, + "cwgc": 48527, + "client": 48528, + "buenas": 48529, + "jagan": 48530, + "coops": 48531, + "ðŁijijðŁijij": 48532, + "specializes": 48533, + "snagged": 48534, + "glar": 48535, + "bennet": 48536, + "wildlifewednesday": 48537, + "bowden": 48538, + "pik": 48539, + "artin": 48540, + "emporium": 48541, + "arl": 48542, + "reba": 48543, + "passer": 48544, + "disappoints": 48545, + "additive": 48546, + "âľĬðŁı½": 48547, + "bayer": 48548, + "missoula": 48549, + "haskell": 48550, + "commences": 48551, + "nix": 48552, + "neman": 48553, + "exploited": 48554, + "plasticsurgery": 48555, + "ccd": 48556, + "asocial": 48557, + "vot": 48558, + "siegel": 48559, + "froome": 48560, + "kapam": 48561, + "fara": 48562, + "eha": 48563, + "probes": 48564, + "mwf": 48565, + "meeting": 48566, + "pbb": 48567, + "akins": 48568, + "mistletoe": 48569, + "kingdomhearts": 48570, + "forkids": 48571, + "ecr": 48572, + "bale": 48573, + "escorts": 48574, + "adidasoriginals": 48575, + "kwa": 48576, + "kts": 48577, + "halloffame": 48578, + "ðŁĺį.": 48579, + "wags": 48580, + "potted": 48581, + "owing": 48582, + "honeycomb": 48583, + "hefty": 48584, + "urology": 48585, + "merle": 48586, + "bpd": 48587, + "stripping": 48588, + "reich": 48589, + "kstate": 48590, + "guay": 48591, + "yonge": 48592, + "shakti": 48593, + "gloom": 48594, + "batt": 48595, + "sonom": 48596, + "nery": 48597, + "elba": 48598, + "blanks": 48599, + "helle": 48600, + "triplets": 48601, + "bombay": 48602, + "akarta": 48603, + "abia": 48604, + "transmitted": 48605, + "rolf": 48606, + "jais": 48607, + "angularjs": 48608, + "fierc": 48609, + "mss": 48610, + "trace": 48611, + "à¥ĩ": 48612, + "tombs": 48613, + "oldman": 48614, + "kombucha": 48615, + "fol": 48616, + "ehealth": 48617, + "cereals": 48618, + "arelli": 48619, + "inari": 48620, + "ðŁĴ©": 48621, + "wol": 48622, + "liberties": 48623, + "fawn": 48624, + "affirm": 48625, + "nunavut": 48626, + "hysterical": 48627, + "kdrama": 48628, + "artes": 48629, + "âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢": 48630, + "valentin": 48631, + "manslaughter": 48632, + "gales": 48633, + "eoin": 48634, + "energized": 48635, + "dels": 48636, + "withdraws": 48637, + "stles": 48638, + "sarcastic": 48639, + "ramesh": 48640, + "incredibles": 48641, + "lockhart": 48642, + "yawn": 48643, + "ultimatefanlive": 48644, + "oooooooooooooooo": 48645, + "muen": 48646, + "gurudev": 48647, + "teer": 48648, + "peeling": 48649, + "newsnow": 48650, + "linguistics": 48651, + "directv": 48652, + "agend": 48653, + "unilever": 48654, + "ruger": 48655, + "handedly": 48656, + "erose": 48657, + "limel": 48658, + "thec": 48659, + "royalties": 48660, + "finishers": 48661, + "nrg": 48662, + "mgt": 48663, + "fidget": 48664, + "comps": 48665, + "bacon": 48666, + "aggressively": 48667, + "abit": 48668, + "châ": 48669, + "tarde": 48670, + "slugger": 48671, + "qanda": 48672, + "greening": 48673, + "dats": 48674, + "enslaved": 48675, + "spector": 48676, + "oye": 48677, + "freef": 48678, + "bhand": 48679, + "stopbrexit": 48680, + "misconceptions": 48681, + "cava": 48682, + "ðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺį": 48683, + "multitasking": 48684, + "housel": 48685, + "ferreira": 48686, + "centime": 48687, + "ankles": 48688, + "jodh": 48689, + "helly": 48690, + "frome": 48691, + "outtuesday": 48692, + "narnia": 48693, + "balaji": 48694, + "lbloggers": 48695, + "jyoti": 48696, + "ðŁįĩ": 48697, + "lancia": 48698, + "capri": 48699, + "yap": 48700, + "natash": 48701, + "downfall": 48702, + ".\"âĢĶ": 48703, + "î": 48704, + "ligament": 48705, + "coatings": 48706, + "aided": 48707, + "hiko": 48708, + "falling": 48709, + "encrypted": 48710, + "yegfood": 48711, + "infringement": 48712, + "cudi": 48713, + "cep": 48714, + "ðŁĺįðŁĺĤ": 48715, + "trad": 48716, + "superrugby": 48717, + "edwin": 48718, + "whiche": 48719, + "vimeo": 48720, + "layne": 48721, + "invigor": 48722, + "hehe": 48723, + "dubrovnik": 48724, + "bieber": 48725, + "utr": 48726, + "shaman": 48727, + "opers": 48728, + "hamill": 48729, + "enig": 48730, + "dif": 48731, + "arum": 48732, + "scrapbook": 48733, + "minh": 48734, + "divergence": 48735, + "mckinnon": 48736, + "lifetime": 48737, + "guterres": 48738, + "wille": 48739, + "pleas": 48740, + "patty": 48741, + "micron": 48742, + "kz": 48743, + "domaine": 48744, + "rusher": 48745, + "mds": 48746, + "chesney": 48747, + "screwdriver": 48748, + "âģ©,": 48749, + "sledge": 48750, + "hauer": 48751, + "chana": 48752, + "stamina": 48753, + "sprinkler": 48754, + "pln": 48755, + "heff": 48756, + "bolton": 48757, + "omon": 48758, + "carrington": 48759, + "accordion": 48760, + "jorge": 48761, + "interception": 48762, + "inputs": 48763, + "gull": 48764, + "transcription": 48765, + "vanuatu": 48766, + "itical": 48767, + "ethos": 48768, + "tich": 48769, + "spacey": 48770, + "peeking": 48771, + "umi": 48772, + "hager": 48773, + "psychotic": 48774, + "illian": 48775, + "illia": 48776, + "bonnaroo": 48777, + "anese": 48778, + "puc": 48779, + "laghateparth": 48780, + "enhall": 48781, + "economical": 48782, + "dredge": 48783, + "%-": 48784, + "uwe": 48785, + "tubular": 48786, + "scouncil": 48787, + "peasants": 48788, + "fler": 48789, + "tumbler": 48790, + "hep": 48791, + "fordham": 48792, + "rowley": 48793, + "initials": 48794, + "evasion": 48795, + "ernation": 48796, + "plugins": 48797, + "cochran": 48798, + "cattle": 48799, + "acidity": 48800, + "ðŁİĬðŁİī": 48801, + "regrann": 48802, + "jumpman": 48803, + "eface": 48804, + "xma": 48805, + "patriarchy": 48806, + "escobar": 48807, + "cristian": 48808, + "tipton": 48809, + "nueva": 48810, + "hackney": 48811, + "backseat": 48812, + "killarney": 48813, + "aidan": 48814, + "stadion": 48815, + "simultaneous": 48816, + "idaho": 48817, + "aje": 48818, + "uth": 48819, + "figure": 48820, + "clos": 48821, + "burk": 48822, + "voluntar": 48823, + "recite": 48824, + "macfarlane": 48825, + "curfew": 48826, + "boudo": 48827, + "wgn": 48828, + "stix": 48829, + "slap": 48830, + "scratched": 48831, + "phillip": 48832, + "journe": 48833, + "expelled": 48834, + "waz": 48835, + "uke": 48836, + "tatiana": 48837, + "oue": 48838, + "hopp": 48839, + "dimitri": 48840, + "ðŁĵ£": 48841, + "matologist": 48842, + "electrifying": 48843, + "bluffs": 48844, + "billsmafia": 48845, + "azcardinals": 48846, + "yaa": 48847, + "xmas": 48848, + "shara": 48849, + "rith": 48850, + "gills": 48851, + "dres": 48852, + "barton": 48853, + "authorization": 48854, + "imperialism": 48855, + "homeof": 48856, + "todo": 48857, + "footpath": 48858, + "bandwidth": 48859, + "visitspain": 48860, + "mohsin": 48861, + "erupted": 48862, + "miki": 48863, + "insignia": 48864, + "mikel": 48865, + "ssh": 48866, + "gera": 48867, + "bankholiday": 48868, + "awan": 48869, + "tweak": 48870, + "starcraft": 48871, + "eal": 48872, + "construction": 48873, + "skeletons": 48874, + "leep": 48875, + "inem": 48876, + "barclay": 48877, + "shipwreck": 48878, + "monsieur": 48879, + "yoh": 48880, + "ront": 48881, + "formative": 48882, + "sero": 48883, + "lep": 48884, + "horseman": 48885, + "hoosier": 48886, + "hazmat": 48887, + "cylinders": 48888, + "centi": 48889, + "ðŁĴ¥ðŁĴ¥ðŁĴ¥": 48890, + "reem": 48891, + "naire": 48892, + "musically": 48893, + "grasshopper": 48894, + "estonian": 48895, + "terminology": 48896, + "romain": 48897, + "bloggerrt": 48898, + "toxin": 48899, + "stance": 48900, + "cultivated": 48901, + "anast": 48902, + "ðŁIJį": 48903, + "shimano": 48904, + "gopher": 48905, + "enei": 48906, + "recyclable": 48907, + "gamification": 48908, + "fightfor": 48909, + "cq": 48910, + "avocados": 48911, + "keys": 48912, + "elike": 48913, + "glycer": 48914, + "shakur": 48915, + "mobilization": 48916, + "galley": 48917, + "explain": 48918, + "exchanged": 48919, + "peth": 48920, + "obedience": 48921, + "illage": 48922, + "ennis": 48923, + "ãĥŀ": 48924, + "wiv": 48925, + "wallabies": 48926, + "maar": 48927, + "igers": 48928, + "fintech": 48929, + "finalized": 48930, + "woj": 48931, + "meaningless": 48932, + "infield": 48933, + "onnaise": 48934, + "eet": 48935, + "bronte": 48936, + "passages": 48937, + "ðŁij§": 48938, + "strickland": 48939, + "northernlights": 48940, + "lomond": 48941, + "htc": 48942, + "wray": 48943, + "shifter": 48944, + "dialog": 48945, + "ðŁįį": 48946, + ">>>>>>": 48947, + "teatime": 48948, + "stech": 48949, + "sichuan": 48950, + "quill": 48951, + "franca": 48952, + "complementary": 48953, + "barrington": 48954, + "marcus": 48955, + "malam": 48956, + "goooo": 48957, + "forsa": 48958, + "electra": 48959, + "afs": 48960, + "âĹĨ": 48961, + "trife": 48962, + "snazzy": 48963, + "folia": 48964, + "andolan": 48965, + "afterdark": 48966, + "woodson": 48967, + "strade": 48968, + "littlest": 48969, + "ogun": 48970, + "conwy": 48971, + "cowards": 48972, + "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ": 48973, + "íĬ¸": 48974, + "seul": 48975, + "murphy": 48976, + "dunks": 48977, + "kapilshar": 48978, + "joachim": 48979, + "womack": 48980, + "equality": 48981, + "averages": 48982, + "aine": 48983, + "ðŁ¦Ī": 48984, + "tacular": 48985, + "disability": 48986, + "uked": 48987, + "midcentury": 48988, + "barthol": 48989, + "teasers": 48990, + "tabern": 48991, + "njcaa": 48992, + "spout": 48993, + "opi": 48994, + "kubball": 48995, + "blom": 48996, + "soar": 48997, + "populism": 48998, + "methyl": 48999, + "ðŁijĬðŁı¼": 49000, + "ospre": 49001, + "aloils": 49002, + "ðŁĵĸ": 49003, + "ðŁĮļ": 49004, + "xer": 49005, + "spilling": 49006, + "publica": 49007, + "cardam": 49008, + "adish": 49009, + "sacha": 49010, + "pkg": 49011, + "buda": 49012, + "lyricist": 49013, + "ibc": 49014, + "grump": 49015, + "hover": 49016, + "halep": 49017, + "antibody": 49018, + "anemone": 49019, + "âĻ¥âĻ¥âĻ¥âĻ¥": 49020, + "mcl": 49021, + "lithograph": 49022, + "ccu": 49023, + "sfest": 49024, + "pathic": 49025, + "callister": 49026, + "ottawa": 49027, + "gunsn": 49028, + "rutger": 49029, + "halibut": 49030, + "envision": 49031, + "differentiate": 49032, + "ðŁļĢðŁļĢ": 49033, + "piran": 49034, + "latel": 49035, + "ucn": 49036, + "troubad": 49037, + "raine": 49038, + "fiercely": 49039, + "learnenglish": 49040, + "lease": 49041, + "wexmondays": 49042, + "emit": 49043, + "drayton": 49044, + "burrell": 49045, + "scubadiving": 49046, + "holler": 49047, + "dru": 49048, + "clocked": 49049, + "wral": 49050, + "apro": 49051, + "translucent": 49052, + "wbo": 49053, + "patriarch": 49054, + "moja": 49055, + "lannister": 49056, + "fishery": 49057, + "nederland": 49058, + "mildly": 49059, + "mirai": 49060, + "mako": 49061, + "jap": 49062, + "ðŁĺ©ðŁĺ©ðŁĺ©": 49063, + "prostatec": 49064, + "panna": 49065, + "arama": 49066, + "undertaking": 49067, + "tompkins": 49068, + "neop": 49069, + "solids": 49070, + "savoury": 49071, + "eames": 49072, + "cutlery": 49073, + "woodbridge": 49074, + "steamer": 49075, + "rizzo": 49076, + "wildcat": 49077, + "ratna": 49078, + "laminated": 49079, + "kineni": 49080, + "jalap": 49081, + "aides": 49082, + "acknowledges": 49083, + "?!?!?!": 49084, + "!ðŁİī": 49085, + "wafc": 49086, + "maggio": 49087, + "haves": 49088, + "darje": 49089, + "ofi": 49090, + "gril": 49091, + "vasi": 49092, + "brux": 49093, + "mohd": 49094, + "fakespeare": 49095, + "arnold": 49096, + "rmb": 49097, + "forbe": 49098, + "walleye": 49099, + "rodi": 49100, + "therapeutics": 49101, + "strategi": 49102, + "obste": 49103, + "mudder": 49104, + "downloadable": 49105, + "ddings": 49106, + "dca": 49107, + "asiangames": 49108, + "campeon": 49109, + "appropriation": 49110, + "thcentury": 49111, + "ramatta": 49112, + "draped": 49113, + "bullion": 49114, + "muc": 49115, + "onex": 49116, + "segreg": 49117, + "ophelia": 49118, + "bodily": 49119, + "âĿ¤ðŁĺį": 49120, + "wizar": 49121, + "teased": 49122, + "ademy": 49123, + "toid": 49124, + "sura": 49125, + "lazarus": 49126, + "snickers": 49127, + "mase": 49128, + "loh": 49129, + "bowed": 49130, + "biblio": 49131, + "xchange": 49132, + "harlan": 49133, + "ghoshal": 49134, + "flavorful": 49135, + "bhagat": 49136, + "allez": 49137, + "whichever": 49138, + "tenstein": 49139, + "discer": 49140, + "organiser": 49141, + "mtg": 49142, + "dreamliner": 49143, + "tse": 49144, + "hokkaido": 49145, + "mok": 49146, + "indulgent": 49147, + "hickman": 49148, + "blinded": 49149, + "alyn": 49150, + "aaaah": 49151, + "spool": 49152, + "loughborough": 49153, + "interpret": 49154, + "etv": 49155, + "aristotle": 49156, + "optimizing": 49157, + "avicii": 49158, + "madurai": 49159, + "juli": 49160, + "nawaz": 49161, + "matchups": 49162, + "abide": 49163, + "painting": 49164, + "welling": 49165, + "veli": 49166, + "octagon": 49167, + "inscribed": 49168, + "poking": 49169, + "placer": 49170, + "lifecycle": 49171, + "kilig": 49172, + "gsp": 49173, + "elives": 49174, + "clements": 49175, + "nasheed": 49176, + "mesut": 49177, + "incarcerated": 49178, + "distilled": 49179, + "walang": 49180, + "delicacy": 49181, + "delgado": 49182, + "chez": 49183, + "chita": 49184, + "adero": 49185, + "tux": 49186, + "patil": 49187, + "odo": 49188, + "abhcosmetics": 49189, + "tvc": 49190, + "pbc": 49191, + "inaccurate": 49192, + "hardworkpaysoff": 49193, + "baller": 49194, + "quotation": 49195, + "merchandising": 49196, + "gastri": 49197, + "defenses": 49198, + "drogba": 49199, + "bexhill": 49200, + "bankno": 49201, + "winona": 49202, + "sieg": 49203, + "pgs": 49204, + "hahahha": 49205, + "aguchi": 49206, + "subram": 49207, + "miracle": 49208, + "desch": 49209, + "libre": 49210, + "bacher": 49211, + "entine": 49212, + "bbcradi": 49213, + "loudest": 49214, + "rps": 49215, + "pierc": 49216, + "fryer": 49217, + "stormtrooper": 49218, + "rafaelnadal": 49219, + "pasco": 49220, + "exhaustion": 49221, + "epiconetsy": 49222, + "rctid": 49223, + "kellie": 49224, + "gaines": 49225, + "dbz": 49226, + "smriti": 49227, + "sbridge": 49228, + "limited": 49229, + "claw": 49230, + "technical": 49231, + "biographical": 49232, + "adored": 49233, + "ะ": 49234, + "exclude": 49235, + "acadia": 49236, + "keyboards": 49237, + "furman": 49238, + "soca": 49239, + "suru": 49240, + "nips": 49241, + "swaps": 49242, + "serverless": 49243, + "rune": 49244, + "puffy": 49245, + "northampton": 49246, + "nishings": 49247, + "hender": 49248, + "cartridges": 49249, + "gunshot": 49250, + "ðŁĵ¹": 49251, + "filament": 49252, + "respondents": 49253, + "peyton": 49254, + "mountaineer": 49255, + "merging": 49256, + "lifespan": 49257, + "intimidation": 49258, + "pafc": 49259, + "nlwx": 49260, + "expansive": 49261, + "purr": 49262, + "fck": 49263, + "cae": 49264, + "atti": 49265, + "telethon": 49266, + "sohn": 49267, + "mendel": 49268, + "lopes": 49269, + "dori": 49270, + "unbroken": 49271, + "tered": 49272, + "tastings": 49273, + "inactive": 49274, + "disintegr": 49275, + "tassel": 49276, + "sharethe": 49277, + "piano": 49278, + "islay": 49279, + "airspace": 49280, + "zawa": 49281, + "ricciardo": 49282, + "mington": 49283, + "fresher": 49284, + "curry": 49285, + "revs": 49286, + "pharoah": 49287, + "hmv": 49288, + "exhilarating": 49289, + "whoo": 49290, + "linkin": 49291, + "krispy": 49292, + "competency": 49293, + "stewards": 49294, + "nebu": 49295, + "katsu": 49296, + "admins": 49297, + "bazar": 49298, + "asar": 49299, + "givingback": 49300, + "ssummit": 49301, + "songz": 49302, + "linus": 49303, + "rajkumar": 49304, + "farmington": 49305, + "fantasia": 49306, + "ðŁĺ´ðŁĺ´": 49307, + "sobri": 49308, + "lisse": 49309, + "barrymore": 49310, + "prism": 49311, + "blob": 49312, + "senew": 49313, + "monoxide": 49314, + "expire": 49315, + "eighteen": 49316, + "dipper": 49317, + "xiao": 49318, + "kilt": 49319, + "hinch": 49320, + "bbcsport": 49321, + "bamboo": 49322, + "pter": 49323, + "exal": 49324, + "ðŁ¦ĭ": 49325, + "hamlin": 49326, + "expeditions": 49327, + "stargazing": 49328, + "foodsecurity": 49329, + "wylie": 49330, + "ulf": 49331, + "stingly": 49332, + "onstorm": 49333, + "loeb": 49334, + "broome": 49335, + "bnha": 49336, + "pancreatic": 49337, + "elive": 49338, + "!!!!!!!!!!!": 49339, + "therapper": 49340, + "orthopedic": 49341, + "avengersendgame": 49342, + "antitrust": 49343, + "ìļ°": 49344, + "gote": 49345, + "omd": 49346, + "offside": 49347, + "gyllen": 49348, + "wineries": 49349, + "whitewater": 49350, + "adl": 49351, + "lupita": 49352, + "exceeds": 49353, + "consisted": 49354, + "chewbacca": 49355, + "ashleigh": 49356, + "nhljets": 49357, + "issan": 49358, + "shld": 49359, + "hayat": 49360, + "cranberries": 49361, + "ðŁ¤ĺðŁı½": 49362, + "rockthe": 49363, + "springtraining": 49364, + "fallout": 49365, + "dairyfree": 49366, + "waj": 49367, + "undecided": 49368, + "sown": 49369, + "rcn": 49370, + "northwales": 49371, + "httr": 49372, + "fumble": 49373, + "dits": 49374, + "compelled": 49375, + "populist": 49376, + "minted": 49377, + "blanchett": 49378, + ".''": 49379, + "propulsion": 49380, + "milla": 49381, + "auberg": 49382, + "hertz": 49383, + "hta": 49384, + "udaipur": 49385, + "serendipity": 49386, + "aztecs": 49387, + "alsace": 49388, + "ðŁIJij": 49389, + "lun": 49390, + "shoes": 49391, + "charli": 49392, + "garza": 49393, + "ðŁĴŁ": 49394, + "probiotics": 49395, + "foxtv": 49396, + "olis": 49397, + "miff": 49398, + "localized": 49399, + "diffuser": 49400, + "sigue": 49401, + "funko": 49402, + "rendous": 49403, + "ðŁĴij": 49404, + "jekyll": 49405, + "<|startoftext|>": 49406, + "<|endoftext|>": 49407 + }, + "merges": [ + "i n", + "t h", + "a n", + "r e", + "a r", + "e r", + "th e", + "in g", + "o u", + "o n", + "s t", + "o r", + "e n", + "o n", + "a l", + "a t", + "e r", + "i t", + "i n", + "t o", + "r o", + "i s", + "l e", + "i c", + "a t", + "an d", + "e d", + "o f", + "c h", + "o r", + "e s", + "i l", + "e l", + "s t", + "a c", + "o m", + "a m", + "l o", + "a n", + "a y", + "s h", + "r i", + "l i", + "t i", + "f or", + "n e", + "ð Ł", + "r a", + "h a", + "d e", + "o l", + "v e", + "s i", + "u r", + "a l", + "s e", + "' s", + "u n", + "d i", + "b e", + "l a", + "w h", + "o o", + "d ay", + "e n", + "m a", + "n o", + "l e", + "t o", + "ou r", + "i r", + "g h", + "w it", + "i t", + "y o", + "a s", + "s p", + "th is", + "t s", + "at i", + "yo u", + "wit h", + "a d", + "i s", + "a b", + "l y", + "w e", + "th e", + "t e", + "a s", + "a g", + "v i", + "p p", + "s u", + "h o", + "m y", + ". .", + "b u", + "c om", + "s e", + "er s", + "m e", + "m e", + "al l", + "c on", + "m o", + "k e", + "g e", + "ou t", + "en t", + "c o", + "f e", + "v er", + "a r", + "f ro", + "a u", + "p o", + "c e", + "gh t", + "ar e", + "s s", + "fro m", + "c h", + "t r", + "ou n", + "on e", + "b y", + "d o", + "t h", + "w or", + "er e", + "k e", + "p ro", + "f or", + "d s", + "b o", + "t a", + "w e", + "g o", + "h e", + "t er", + "in g", + "d e", + "b e", + "ati on", + "m or", + "a y", + "e x", + "il l", + "p e", + "k s", + "s c", + "l u", + "f u", + "q u", + "v er", + "ðŁ ĺ", + "j u", + "m u", + "at e", + "an d", + "v e", + "k ing", + "m ar", + "o p", + "h i", + ".. .", + "p re", + "a d", + "r u", + "th at", + "j o", + "o f", + "c e", + "ne w", + "a m", + "a p", + "g re", + "s s", + "d u", + "no w", + "y e", + "t ing", + "y our", + "it y", + "n i", + "c i", + "p ar", + "g u", + "f i", + "a f", + "p er", + "t er", + "u p", + "s o", + "g i", + "on s", + "g r", + "g e", + "b r", + "p l", + "' t", + "m i", + "in e", + "we e", + "b i", + "u s", + "sh o", + "ha ve", + "to day", + "a v", + "m an", + "en t", + "ac k", + "ur e", + "ou r", + "â Ģ", + "c u", + "l d", + "lo o", + "i m", + "ic e", + "s om", + "f in", + "re d", + "re n", + "oo d", + "w as", + "ti on", + "p i", + "i r", + "th er", + "t y", + "p h", + "ar d", + "e c", + "! !", + "m on", + "mor e", + "w ill", + "t ra", + "c an", + "c ol", + "p u", + "t e", + "w n", + "m b", + "s o", + "it i", + "ju st", + "n ing", + "h ere", + "t u", + "p a", + "p r", + "bu t", + "wh at", + "al ly", + "f ir", + "m in", + "c a", + "an t", + "s a", + "t ed", + "e v", + "m ent", + "f a", + "ge t", + "am e", + "ab out", + "g ra", + "no t", + "ha pp", + "ay s", + "m an", + "h is", + "ti me", + "li ke", + "g h", + "ha s", + "th an", + "lo ve", + "ar t", + "st e", + "d ing", + "h e", + "c re", + "w s", + "w at", + "d er", + "it e", + "s er", + "ac e", + "ag e", + "en d", + "st r", + "a w", + "st or", + "r e", + "c ar", + "el l", + "al l", + "p s", + "f ri", + "p ho", + "p or", + "d o", + "a k", + "w i", + "f re", + "wh o", + "sh i", + "b oo", + "s on", + "el l", + "wh en", + "il l", + "ho w", + "gre at", + "w in", + "e l", + "b l", + "s si", + "al i", + "som e", + "ðŁ Ĵ", + "t on", + "d er", + "le s", + "p la", + "ï ¸", + "e d", + "s ch", + "h u", + "on g", + "d on", + "k i", + "s h", + "an n", + "c or", + ". .", + "oun d", + "a z", + "in e", + "ar y", + "fu l", + "st u", + "ou ld", + "st i", + "g o", + "se e", + "ab le", + "ar s", + "l l", + "m is", + "b er", + "c k", + "w a", + "en ts", + "n o", + "si g", + "f e", + "fir st", + "e t", + "sp e", + "ac k", + "i f", + "ou s", + "' m", + "st er", + "a pp", + "an g", + "an ce", + "an s", + "g ood", + "b re", + "e ver", + "the y", + "t ic", + "com e", + "of f", + "b ack", + "as e", + "ing s", + "ol d", + "i ght", + "f o", + "h er", + "happ y", + "p ic", + "it s", + "v ing", + "u s", + "m at", + "h om", + "d y", + "e m", + "s k", + "y ing", + "the ir", + "le d", + "r y", + "u l", + "h ar", + "c k", + "t on", + "on al", + "h el", + "r ic", + "b ir", + "vi e", + "w ay", + "t ri", + "d a", + "p le", + "b ro", + "st o", + "oo l", + "ni ght", + "tr u", + "b a", + "re ad", + "re s", + "ye ar", + "f r", + "t or", + "al s", + "c oun", + "c la", + "t ure", + "v el", + "at ed", + "le c", + "en d", + "th ing", + "v o", + "ic i", + "be st", + "c an", + "wor k", + "la st", + "af ter", + "en ce", + "p ri", + "p e", + "e s", + "i l", + "âĢ ¦", + "d re", + "y s", + "o ver", + "i es", + "ðŁ ij", + "com m", + "t w", + "in k", + "s un", + "c l", + "li fe", + "t t", + "a ch", + "l and", + "s y", + "t re", + "t al", + "p ol", + "s m", + "du c", + "s al", + "f t", + "' re", + "ch e", + "w ar", + "t ur", + "ati ons", + "ac h", + "m s", + "il e", + "p m", + "ou gh", + "at e", + "st ar", + "wee k", + "! !!", + "c lu", + "th ere", + "n er", + "t om", + "s el", + "ï¸ ı", + "wor ld", + "v es", + "c am", + "go t", + "in ter", + "of f", + "u m", + "ton ight", + "o ther", + "h ou", + "loo k", + "j e", + "i d", + "si on", + "be au", + "at t", + "el i", + "or t", + "re c", + "f f", + "st er", + "su pp", + "g en", + "be en", + "il y", + "te am", + "m m", + "i c", + "pe op", + "it t", + "at s", + "on ly", + "mb er", + "en g", + "b ri", + "m p", + "k now", + "b ur", + "b ar", + "in s", + "lo w", + "sh e", + "ro w", + "â Ŀ", + "t ro", + "peop le", + "vi a", + "lo w", + "ag a", + "be t", + "x t", + "f ac", + "ch ar", + "e ar", + "w al", + "s en", + "f am", + "b le", + "n ati", + "is h", + "n or", + "g ame", + "li ve", + "s co", + "le y", + "d on", + "ic k", + "b all", + "ver y", + "the se", + "p an", + "i a", + "at ing", + "c r", + "a re", + "g ir", + "ma ke", + "st re", + "sho w", + ". \"", + "f l", + "u p", + "d r", + "than ks", + "il li", + "w om", + "st s", + "i g", + "s ur", + "ever y", + "c ur", + "vie w", + "le t", + "in to", + "mo st", + "n a", + "in di", + "g ar", + "ha d", + "s ou", + "v ed", + "an t", + "iti on", + "ma de", + "f ol", + "un i", + "it ed", + "ðŁ ı", + "ic al", + "th r", + "read y", + "ch ec", + "d ra", + "k es", + "boo k", + "e p", + "si c", + "mor ning", + "ne ws", + "c au", + "c t", + "w ell", + "an c", + "pho to", + "th an", + "or s", + "bir th", + "g g", + "ou t", + "ne xt", + "som e", + "en ing", + "stor y", + "ch ri", + "do wn", + "hom e", + "f fe", + "fre e", + "d a", + "b or", + "f il", + "ci al", + "than k", + "si de", + "le ar", + "qu e", + "l ine", + "t en", + "at es", + "ye ars", + "m y", + "pho to", + "beau ti", + "ri ght", + "n u", + "for m", + "shi p", + "b an", + "th er", + "d ays", + "g am", + "as on", + "g y", + "ðŁ İ", + "birth day", + "se t", + "ic k", + "e t", + "st ill", + "com ing", + "ta ke", + "ðŁ ĩ", + "b b", + "s ol", + "s on", + "d en", + "e p", + "mu sic", + "the m", + "de n", + "wh y", + "f oo", + "c ra", + "am az", + "w n", + "h ol", + "t ting", + "w r", + "u e", + "ma g", + "c ro", + "l an", + "c lo", + "b ra", + "a k", + "s ing", + "c al", + "re ad", + "' ve", + "jo h", + "b ab", + "d ri", + "b lo", + "bi g", + "er ic", + "in t", + "t or", + "tr y", + "l a", + "le g", + "hou se", + "m ic", + "v al", + "beauti ful", + "l itt", + "chec k", + "ne w", + "ver s", + "s w", + "ar i", + "pla y", + "h er", + "âĢ ĵ", + "w in", + "m a", + "con gr", + "sch ool", + "f un", + ". @", + "he al", + "ic h", + "d el", + "wh ere", + "l on", + "ke t", + "tw o", + "mu ch", + "wat ch", + "v en", + "d ed", + "a st", + "k ed", + "b as", + "go ing", + "m p", + "e ver", + "w ays", + "ro o", + "de sig", + "l y", + "s ed", + "to p", + "l in", + "ch an", + "to o", + "it ing", + "d ent", + "gh ts", + "t y", + "sp o", + "ne ed", + "b lu", + "in st", + "be ing", + "âĿ ¤", + "w el", + "l s", + "hi m", + "m ay", + "st ing", + "n a", + "el y", + "litt le", + "g a", + "n at", + "tom or", + "m c", + "h on", + "w ant", + "a ir", + "pi c", + "am eric", + "p er", + "le ss", + "wee k", + "ve l", + "a h", + "c ap", + "ch am", + "g er", + "ti m", + "tomor row", + "ne ss", + "st ate", + "h al", + "ser v", + "z e", + "o s", + "p at", + "v is", + "ex c", + "s in", + "f f", + "c ity", + "c en", + "an y", + "b el", + "su mm", + "t in", + "w ould", + "loo king", + "k o", + "ce le", + "fam ily", + "m er", + "po w", + "hel p", + "bu s", + "c o", + "c le", + "sel f", + "en s", + "ic s", + "th o", + "an i", + "ch o", + "le ad", + "b s", + "t wee", + "th ink", + "for e", + "ch il", + "vi de", + "di d", + "al e", + "ch i", + "v il", + "en ds", + "w ing", + "p as", + "' ll", + "v ol", + "s a", + "g s", + "man y", + "j ec", + "be fore", + "gra ph", + "n y", + "ur ing", + "w il", + "d d", + "bu il", + "f av", + "st ed", + "tr an", + "l ing", + "ou d", + "d ge", + "fi el", + "nati onal", + "st a", + "c er", + "w ere", + "in a", + "se ason", + "c ou", + "n ed", + "amaz ing", + "ti ons", + "cele br", + "n s", + "a th", + "he ad", + "s day", + "d ar", + "lo c", + "v in", + "an other", + "g oo", + "s at", + "n y", + "jo in", + "pre s", + "s es", + "s ing", + "an a", + "in ing", + ".. ..", + "c our", + "ï¸ ı", + "ac t", + "cau se", + "li ght", + "am s", + "t a", + "b al", + "f c", + "hi gh", + "off ici", + "t t", + "chri st", + "d ic", + "d ay", + "ra l", + "h or", + ": )", + "vi si", + "n am", + "o b", + "ma s", + "gh t", + "re ally", + "t un", + "fin d", + "thr ough", + "por t", + "u t", + "ti ve", + "st y", + "n e", + "or e", + "ðŁĺ Ĥ", + "supp ort", + "ne ver", + "ev en", + "ðŁ Ķ", + "h a", + "y a", + "l d", + "u k", + "r an", + "j am", + "wi th", + "me di", + "d es", + "ne y", + "ch ing", + "al e", + "h y", + "k in", + "! !", + "d y", + "pl ace", + "al so", + "b le", + "wh ich", + "bl ack", + "b li", + "s ay", + "par k", + "pl ay", + "ir e", + "vide o", + "week end", + "a il", + "ke y", + "p t", + "w ard", + "fri day", + "d in", + "ine ss", + "g ro", + "b en", + "al ways", + "t ball", + "ag o", + "m il", + "c y", + "pro duc", + "di sc", + "un der", + "ple ase", + "sp or", + "fu ll", + "e y", + "ðŁ Ļ", + "is e", + "iti es", + "c at", + "k no", + "u se", + "fo re", + "k er", + "ar t", + "hi gh", + "op en", + "s an", + "e f", + "our s", + "sh ed", + "st ri", + "d ro", + "aga in", + "i m", + "ðŁ ĵ", + "en jo", + "fu n", + "ge tting", + "p en", + "g er", + "c li", + "an y", + "ever y", + "e u", + "wom en", + "â ľ", + "e st", + "c ould", + "r y", + "\" @", + "th ou", + "sh a", + "comm un", + "b er", + "d ents", + "di s", + "wh ile", + "aw ay", + "di o", + "h am", + "g la", + "d ate", + "k a", + "mis s", + "un ch", + "w on", + "in f", + "roo m", + "g a", + "re al", + "ex per", + "di rec", + "sh ould", + "sp r", + "g ol", + "l ong", + "bet ter", + "or i", + "e y", + "i ence", + "il s", + "z z", + "h an", + "f ound", + "v s", + "â Ļ", + "po st", + "ti c", + "par t", + "m en", + "ren ce", + "ce ss", + "v ic", + "s il", + "sho p", + "ðŁĺ Ĥ", + "f ood", + "v al", + "sti c", + "y ou", + "s ays", + "e lec", + "st ar", + "o c", + "l and", + "i d", + "c tion", + "fiel d", + "s of", + "st art", + "wat er", + "fri ends", + "on es", + "ðŁ Į", + "f la", + "f ar", + "wh ite", + "par ty", + "in st", + "gr ou", + "t v", + "every one", + "m ent", + "j a", + "ch a", + "pr in", + "an ts", + "d uring", + "l at", + "l ar", + "we st", + "th en", + "k a", + "y oun", + "in sp", + "in te", + "we en", + "visi t", + "aga inst", + "re le", + "he ad", + "c es", + "to wn", + "loo ks", + "th re", + "re gi", + "ren t", + "pro jec", + "gir l", + "se ar", + "w o", + "m om", + "c ar", + "h un", + "pu bli", + "d i", + "p le", + "c all", + "c ri", + "u m", + "for d", + "per fe", + "fri end", + "h ard", + "ssi on", + "te st", + "pla ying", + "ar ound", + "be cause", + "ke ts", + "me et", + "sat ur", + "ar ti", + "wor k", + "j un", + "v en", + "r un", + "me mber", + "por t", + "su per", + "t wit", + "s am", + "el s", + "t ly", + "ad v", + "ati ve", + "at h", + "s ure", + "av ail", + "la r", + "s qu", + "ar ds", + "ev ent", + "m en", + "l l", + "o ver", + "lo gy", + "it al", + "tim es", + "m al", + "b ack", + "c oo", + "ma king", + "st ru", + "â ģ", + "it u", + "sh ar", + "g an", + "c as", + "s n", + "summ er", + "pic ture", + "f an", + "h in", + "christ mas", + "c y", + "pr oud", + "cham pi", + "desig n", + "pp ing", + "ho pe", + "c a", + "avail able", + "ma y", + "we d", + "photo graph", + "spe cial", + "sal e", + "sto p", + "er y", + "a we", + "al ity", + "hi story", + "am a", + "pre si", + "b ru", + "wor king", + "d one", + "d r", + "k en", + "fe at", + "w ood", + "ate st", + "sun day", + "mo vi", + "vel y", + "s le", + "f ace", + "sp ec", + "stu dents", + "b y", + "ha m", + "sp on", + "bus iness", + "d at", + "i e", + "i p", + "so ci", + "g lo", + "h and", + "re cor", + "r s", + "me e", + "ke ep", + "p ur", + "heal th", + "sh e", + "com ple", + "go d", + "da vi", + "col lec", + "li st", + "r a", + "clu b", + "t ers", + "in clu", + "th ings", + "pl an", + "â ĺ", + "joh n", + "sh ing", + "at ul", + "so on", + "blu e", + "g or", + "satur day", + "w on", + "congr atul", + "se e", + "âĿ¤ ï¸ı", + "tho se", + "ðŁĺ į", + "fin al", + "d ou", + "it h", + "o wn", + "ro ad", + "t our", + "a st", + "indi a", + "ti l", + "n d", + "f er", + "fav or", + "su l", + "lear n", + "fir e", + "ju st", + "grou p", + "a h", + "r ac", + "bo dy", + "u r", + "c are", + "à ¸", + "p lo", + "o h", + "po s", + "gi ve", + "te ch", + "su b", + "c ent", + "er ing", + "y m", + "il ity", + "f ic", + "lon don", + "v ir", + "gu ys", + "b a", + "ðŁ ¤", + "bab y", + "sc re", + "ðŁĺ į", + "tru mp", + "un der", + "chan ge", + "i an", + "col le", + "ss es", + "l er", + "ss ed", + "n ice", + "ann oun", + "pow er", + "s ar", + "a king", + "min i", + "s li", + "s wee", + "k ar", + "fu l", + "c ru", + "ac tion", + "a ther", + ") .", + "st and", + "de vel", + "a a", + "g an", + "le ft", + "lo l", + "re l", + "tran s", + "m ents", + "in t", + "e f", + "man ag", + "di g", + "gen er", + "do wn", + "p au", + "ti v", + "k u", + "th ur", + "k en", + "st on", + "f ans", + "tal k", + "twee t", + "t oo", + "sty le", + "pro te", + "se con", + "fr on", + "awe some", + "g l", + "p al", + "ne t", + "s or", + "la u", + "g on", + "sin ce", + "t ty", + "ser ies", + "me mor", + "b eli", + "fil m", + "di d", + "di es", + "o t", + "congratul ations", + "p ra", + "e ve", + "w oo", + "offici al", + "su c", + "in cre", + "b on", + "par t", + "pp ed", + "cla ss", + "si ve", + "bo y", + "cu l", + "perfe ct", + "t ou", + "d am", + "wel come", + "foo tball", + "h i", + "p ap", + "wa it", + "ad a", + "congr ats", + "youn g", + "exc ited", + "re ce", + "j an", + "v a", + "re d", + "st ra", + "medi a", + "' d", + "do es", + "le t", + "mu l", + "ill s", + "gre en", + "m el", + "to ge", + "fu ture", + "ye ster", + "vers ity", + "for m", + "ta in", + "i de", + "ch es", + "ki ds", + "qu i", + "ha ha", + "de ta", + "bi g", + "favor ite", + "gir ls", + "con tin", + "do m", + "sear ch", + "u al", + "a ir", + "d ers", + "mon th", + "c er", + "yester day", + "commun ity", + "ad e", + "do g", + "vil le", + "ic es", + "d eli", + "sy ste", + "ru n", + "is m", + "he art", + "c up", + "en ti", + "fe w", + "presi dent", + "e ds", + "un til", + "fe sti", + "o k", + "f lo", + "sa id", + "ol e", + "me d", + "tra vel", + " £", + "ph one", + "toge ther", + "fa st", + "lo t", + "gam es", + "sh ir", + "bet ween", + "y es", + "th ers", + "do ing", + "m ac", + "at or", + "b and", + "fol low", + "projec t", + "devel op", + "di ffe", + "con fe", + "spe ci", + "ca st", + "y s", + "bo ard", + "r d", + "i al", + "sh oo", + "r am", + "ha ving", + "sh are", + "fol low", + "on e", + "n ame", + "m r", + "pu t", + "disc u", + "or y", + "c ame", + "ou s", + "s ite", + "twit ter", + "t b", + "t it", + "fin ally", + "z ed", + "su per", + "com pan", + "us ing", + "all s", + "li st", + "r is", + "sho t", + "g al", + "t ar", + "de l", + "joh n", + "âĢ Ķ", + "some thing", + "ra m", + "inte re", + "wh e", + "b it", + "ðŁ į", + "stre et", + "oun d", + "a i", + "tic kets", + "movi e", + "re al", + "k y", + "ta king", + "o pp", + "c c", + "l am", + "m oun", + "in ve", + "bl ack", + "us ed", + "on line", + "y or", + "loc al", + "gu e", + "c ks", + "o w", + "ge st", + "bo ys", + "illi on", + "con t", + "re ci", + "in ed", + "eu ro", + "no w", + "se en", + "p h", + "te ach", + "de f", + "sou th", + "su ch", + "aw ard", + "mu st", + "is su", + "ca re", + "fe el", + "p lu", + "l atest", + "spor ts", + "we b", + "te x", + "e ment", + "s k", + "fi c", + "w an", + "te ch", + "o t", + "bo x", + "n er", + "fre e", + "t al", + "a sh", + "c ase", + "ho t", + "won der", + "mee ting", + "er a", + "ch all", + "ðŁ IJ", + "jo b", + "il i", + "c ool", + "j our", + "th s", + "m o", + "f el", + "di e", + "mic ha", + "e le", + "te am", + "serv ice", + "st and", + "ma kes", + "p ing", + "ear ly", + "com es", + "e k", + "ho li", + "v ers", + "ag ue", + "s au", + "thre e", + "mon day", + "fa shi", + "some one", + "th ro", + "se a", + "b ad", + "supp or", + "tur n", + "ur y", + "m ing", + "photograph y", + "n ic", + "mar k", + "pre tty", + "ss ing", + "wat ching", + "me mb", + "ar ri", + "coun ty", + "be ach", + "fr an", + "cen ter", + "pol ice", + "b at", + "publi c", + "t an", + "pre ss", + "s af", + "s y", + "ge ts", + "ro y", + "n ers", + "y our", + "bu y", + "st ers", + "sho w", + "as ed", + "chil dre", + "af ric", + "in es", + "sp ace", + "sc ri", + "h all", + "pa in", + "ar ing", + "hom e", + "m ur", + "heal th", + "ch ed", + "s and", + "rece i", + "gu y", + "e a", + "americ an", + "re si", + "childre n", + "- -", + "i ri", + "ing ton", + "coun try", + "ro ss", + "le n", + "ann a", + "boo ks", + "b c", + "e ce", + "d om", + "lo vely", + "k h", + "pe t", + "g y", + "g ri", + "st age", + "off ice", + "ro ck", + "m on", + "b ay", + "t able", + "su n", + "m ed", + "th in", + "l or", + "f low", + "( @", + "uni versity", + "stor e", + "fron t", + "goo d", + "z a", + "vo te", + "nor th", + "he y", + "an im", + "or der", + "mi d", + "with out", + "a de", + "re member", + "mar ket", + "? ?", + "mu s", + "tra ining", + "e duc", + "bu t", + "co ver", + "st an", + "sc en", + "b la", + "bre ak", + "l ou", + "s ame", + "g old", + "a in", + "o s", + "bo th", + "l it", + "ver n", + "a i", + "al bu", + "p a", + "enjo y", + "be g", + "ell ing", + "thur sday", + "inf o", + "s an", + "americ a", + "ha ir", + "te l", + "mar ch", + "con cer", + "colle ge", + "confe rence", + "ap p", + "h our", + "ch ang", + "â ļ", + "s our", + "ol s", + "we ather", + "w ar", + "p hi", + "festi val", + "secon d", + "cu te", + "pr ac", + "en er", + "str y", + "le a", + "pol it", + "s av", + "se n", + "o w", + "m i", + "ne ar", + "ou ght", + "z e", + "co ffe", + "w illi", + "d an", + "se y", + "davi d", + "e se", + "f an", + "de ci", + "the at", + "no v", + "ati on", + "tr ac", + "sc i", + "re view", + "c el", + "e m", + "u n", + "ju ly", + "or ig", + "ti on", + "d ru", + "form er", + "st ay", + "af ter", + "in v", + "too k", + "dat a", + "b al", + "tu es", + "d an", + "ev ening", + "ðŁĺĤ ðŁĺĤ", + "d ol", + "u res", + "pro vi", + "t s", + "e st", + "sig n", + "j ac", + "u k", + "s ong", + "ye t", + "bo w", + "in du", + "j ap", + "h oo", + "po int", + "any one", + "z y", + "i st", + "h ur", + "it al", + "buil ding", + "wom an", + "ch ur", + "j er", + "per for", + "co ach", + "le ague", + "ce ss", + "ne t", + "i mag", + "nati on", + "br it", + "qu e", + "aw ards", + "ag es", + "wor ks", + "c ed", + "man ce", + "l ate", + "ig n", + "mon ey", + "tru e", + "i i", + "t ell", + "pl ac", + "p ac", + "as y", + "wor ld", + "be hin", + "im port", + "read ing", + "gra m", + "gi ving", + "me t", + "h it", + "for ward", + "st om", + "pres ent", + "jun e", + "so cial", + "no on", + "mar t", + "hal f", + "s we", + "go vern", + "k er", + "deta ils", + "li sh", + "_ _", + "ac y", + "si a", + "ber t", + "f all", + "! !!!", + ") ,", + "th i", + "d iti", + "sp ort", + "k ing", + "f it", + "st af", + "c at", + "mu se", + "cen tr", + "y er", + "con tro", + "b loo", + "wal k", + "ac tu", + "did n", + "li m", + "lear ning", + "re search", + "wed ne", + "au th", + "h ours", + "k y", + "f ar", + "h en", + ".. ..", + "it ch", + "ri l", + "str ong", + "sk y", + "que sti", + "jam es", + "r on", + "d g", + "f ur", + "c in", + "do es", + "app ro", + "mar ke", + "tu res", + "ful ly", + "ch at", + "behin d", + "te m", + "fin i", + "mis sion", + "b att", + "fe el", + "he av", + "every thing", + "b ar", + "w ish", + "pre mi", + "i ma", + "exper ience", + "e ach", + "re port", + "swee t", + "tic s", + "spr ing", + "re spon", + "syste m", + "vic tor", + "l in", + "sa w", + "al ready", + "gh ter", + "f le", + "ã ĥ", + "br ing", + "albu m", + "- -", + "ell s", + "st an", + "to m", + "inter national", + "w ent", + "an ni", + "mat ch", + "pp er", + "st one", + "sm all", + "ra in", + "fashi on", + "are a", + "v an", + "ag ram", + "k o", + "thou ght", + "wor th", + "v an", + "m er", + "coffe e", + "it es", + "g n", + "arti st", + "c on", + "ar ch", + "c ir", + "se cre", + "gr ound", + "is o", + "h and", + "co m", + "bri dge", + "h s", + "x i", + "l ink", + "pu l", + "sp l", + "r ace", + "f li", + "ri ver", + "g as", + "di sco", + "d al", + "play er", + "f it", + "photo s", + "it y", + "o k", + "j or", + "tr a", + "ap ril", + "ad s", + "a di", + "sol u", + "beau ty", + "do or", + "me ss", + "up date", + "ali a", + "sch o", + "en ed", + "mom ent", + "sco t", + "sc ience", + "i or", + "ti es", + "ac ross", + "ous ly", + "sh es", + "does n", + "p age", + "wat er", + "m illion", + "cla ssi", + "l ic", + "ca st", + "form ation", + "micha el", + "ell o", + "s mo", + "in ts", + "vi sion", + "op ening", + "ld n", + "au str", + "tues day", + "win ner", + "po ssi", + "r ound", + "shir t", + "di t", + "b o", + "u es", + "il led", + "al ong", + "tri p", + "star ting", + "im pro", + "k an", + "per son", + "no t", + "re co", + "ne eds", + "c le", + "li e", + "re st", + "r ing", + "win ter", + "si mp", + "mo m", + "be er", + "fac e", + "tor s", + "us a", + "collec tion", + "ge or", + "se ssion", + "tr ying", + "la s", + "la ke", + "j en", + "orig in", + "stu dent", + "se cur", + "v in", + "pic s", + "ex pe", + "com p", + "gon na", + "e qu", + "b ad", + "le y", + "a u", + "memb ers", + "bre ak", + "w all", + "gi c", + "din ner", + "bu l", + "insp ir", + "r i", + "min d", + "ic a", + "win ning", + "tal king", + "t ren", + "s is", + "t en", + "wonder ful", + "s now", + "he ar", + "th om", + "no thing", + "gu i", + "st in", + "blo g", + "fe st", + "b un", + "le e", + "war ds", + "ch ance", + "dre ss", + "re n", + "pau l", + "p es", + "tech no", + "ru ssi", + "c ard", + "e ast", + "mar i", + "w ine", + "t i", + "la w", + "str ic", + "k i", + "ap e", + "au gu", + "pro fe", + "as h", + "cour se", + "ma il", + "ren tly", + "d un", + "m un", + "lo ve", + "is land", + "dri ve", + "s l", + "end ed", + "ma in", + "lo st", + "nat ure", + "âĿ¤ ï¸ı", + "ch ic", + "re por", + "p in", + "pr o", + "st ation", + "ce p", + "ta kes", + "compan y", + "go es", + "on d", + "ma ch", + "ra dio", + "d ad", + "ro ck", + "j a", + "p ay", + "champi on", + "e e", + "in de", + "tt a", + "ati c", + "t ab", + "beli eve", + "ener gy", + "z i", + "t at", + "wor d", + "on ce", + "re sul", + "y l", + "and re", + "an o", + "inst agram", + "clo se", + "t am", + "cu stom", + "w a", + "con om", + "sho ws", + "li fe", + "k in", + "ro b", + "t age", + "n ation", + "al most", + "list en", + "sa ve", + "re li", + "ac e", + "mar y", + "tre e", + "for get", + "j ack", + "wa iting", + "direc tor", + "h ill", + "bor n", + "te mp", + "f l", + "st e", + "on a", + "sing le", + "wedne sday", + "un ited", + "in o", + "@ _", + "ne l", + "celebr ate", + "en ding", + "de al", + "j i", + "can ada", + "hu ge", + "tr ack", + "âĢ ¢", + "f y", + "fan ta", + "an g", + "yor k", + "rele ase", + "p un", + "ep iso", + "wor ds", + "t our", + "p ack", + "i gh", + "classi c", + "perfor mance", + "ke t", + "after noon", + "recor d", + "win s", + "pro ble", + "âĿ ¤", + "f our", + "b ed", + "ban k", + "d ance", + "s la", + "cal led", + "mi ght", + "a p", + "pa st", + "ðŁ ļ", + "diffe rent", + "it e", + "gi ft", + "ssi ve", + "chur ch", + "c us", + "pro gram", + "ho tel", + "ic e", + "ma d", + "secur ity", + "en ge", + "d c", + "en ough", + "st a", + "e ty", + "de ad", + "g un", + "he ar", + "m ir", + "hu man", + "gre ss", + "oun ds", + "pi ece", + "bre aking", + "gar den", + "fi ght", + "vie ws", + "f ish", + "star ted", + "run ning", + "gre en", + "ser i", + "s m", + "as k", + "d or", + "de ath", + "e conom", + "er i", + "ir d", + "s er", + "l unch", + "âģ ¦", + "bo x", + "nat u", + "ba se", + "b an", + "f al", + "glo bal", + "wil d", + "wo w", + "out side", + "mo ve", + "le ad", + "an al", + "muse um", + "on g", + "ha w", + "pow er", + "than k", + "b ac", + "char ac", + "cam pa", + "dig ital", + "r o", + "op er", + "de v", + "w ol", + "p ati", + "f a", + "m ale", + "pap er", + "ill ing", + "c s", + "â ĥ", + "educ ation", + "ta ken", + "e ffe", + "m ou", + "s ad", + "\" .", + "bas ed", + "staf f", + "inclu ding", + "li ving", + "a c", + "ch ina", + "mo b", + "stor m", + "lu ck", + "ph il", + "o o", + "y n", + "tra vel", + "k el", + "ti al", + "pr ice", + "boo k", + "import ant", + "bi o", + "p ool", + "ny c", + "f ab", + "lo ad", + "? !", + "chall enge", + "cr y", + "ser ve", + "we ar", + "bu s", + "ta in", + "nu mber", + "ro r", + "k at", + "i z", + "th ough", + "ho sp", + "m m", + "fa ir", + "ut es", + "ho t", + "po p", + "fi ed", + "cam p", + "develop ment", + "li br", + "c ali", + "em s", + "âģ¦ @", + "b ol", + "is ed", + "stand ing", + "mo del", + "it a", + "g le", + "bro wn", + "ima ge", + "ve red", + "for ce", + "o il", + "par tic", + "sh u", + "da ily", + "la w", + "se c", + "cla ss", + "cam p", + "holi day", + "cl in", + "k ers", + "pres ent", + "gam e", + "incre di", + "er ship", + "inter view", + "b ill", + "du e", + "and y", + "ab o", + "in nov", + "ke y", + "ac ade", + "p il", + "mo der", + "st ars", + "br and", + "f er", + "wee ks", + "con si", + "pr e", + "sa fe", + "wr it", + "di um", + "la unch", + "marke ting", + "ann ual", + "as si", + "cour t", + "la dy", + "c ted", + "and a", + "in side", + "chil d", + "opp or", + "sm ith", + "centr e", + "gu e", + "âģ ©", + "f ren", + "st y", + "for t", + "ent ly", + "is n", + "ke ep", + "to ber", + "on y", + "bo y", + "al d", + "col la", + "de mo", + "le vel", + "com pet", + "ad o", + "b our", + "fanta stic", + "m ate", + "s u", + "sou th", + "oppor tun", + "vers ary", + "lat er", + "bu d", + "face book", + "la un", + "ster n", + "p it", + "! \"", + "ma j", + "gr am", + "tb t", + "fi re", + "happ y", + "a ks", + "wh ole", + "actu ally", + "ill er", + "ell a", + "lo ts", + "al ex", + "an ge", + "lan ds", + "ðŁĺ Ń", + "en ter", + "r ou", + "episo de", + "p ed", + "in ten", + "sh ire", + "wh o", + "pl an", + "h o", + "ca ke", + "we st", + "mag az", + "fre sh", + "c c", + "n ar", + "ch ris", + "wr iting", + "w er", + "n om", + "l o", + "mi dd", + "dre am", + "o l", + "ti onal", + "de b", + "> >", + "be come", + "s i", + "gr and", + "all ing", + "hi stor", + "ri de", + "i red", + "saf e", + "que en", + "ci l", + "in tro", + "vi l", + "d ani", + ".. .", + "ar tic", + "st at", + "sh ort", + "or ing", + "sel fi", + "mis si", + "do c", + "b it", + "g all", + "b om", + "i re", + "se lec", + "d ition", + "ðŁĶ ¥", + "fri end", + "be at", + "gh ting", + "ðŁĺ Ĭ", + "pe ace", + "ex hi", + "ant a", + "ab ility", + "il lu", + "j on", + "qu ality", + "tri bu", + "m es", + "play ers", + "fa ir", + "cu t", + "c ab", + "suc cess", + "b i", + "su s", + "pro mo", + "sch e", + "an ge", + "ic o", + "comm it", + "cat ch", + "ill a", + "kin d", + "feel ing", + "qu o", + "s ay", + "anni versary", + "spo t", + "mo ther", + "an e", + "p end", + "your self", + "op s", + "app le", + "min utes", + "p o", + "gr and", + "ri es", + "ha ha", + "care er", + "ed ition", + "de c", + "ric k", + "am i", + "concer t", + "iti ve", + "ge ous", + "d ly", + "t te", + "adv ent", + "i g", + "li ghts", + "ak er", + "sk y", + "âĥ £", + "r ay", + "fini shed", + "w ay", + "s d", + "ac coun", + "ðŁĴ ķ", + "ck y", + "ch el", + "lit er", + "pain ting", + "lo s", + "st un", + "techno logy", + "n as", + "ma r", + "b il", + "afric a", + "ki e", + "ey es", + "gol f", + "plu s", + "ni a", + "it ec", + "serv ices", + "wed ding", + "kno wn", + "te le", + ".. ...", + "star ts", + "pa ren", + "w ants", + "ati onal", + "mon ths", + "win do", + "fav our", + "er t", + "magaz ine", + "ex clu", + "re ve", + "b c", + "origin al", + "e ss", + "n al", + "an ti", + "st ro", + "t ice", + "stu dy", + "à ¤", + "v ac", + "nation al", + "fi ve", + "ra in", + "ve ment", + "u te", + "ver se", + "em er", + "ar my", + "possi ble", + "gue ss", + "val ley", + "ther n", + "cro w", + "m r", + "col or", + "on to", + "pic k", + "cle ar", + "dar k", + "t ac", + "wan ted", + "it ting", + "can cer", + "govern ment", + "di e", + "ri se", + "z ing", + "col d", + "f oun", + "stu dio", + "str ation", + "bro ther", + "a head", + "sh el", + "mic ro", + "ic ally", + "d au", + "sig ned", + "vi ol", + "a x", + "as se", + "i o", + "w re", + "spl ay", + "ch ick", + "augu st", + "pl at", + "ti ps", + "sp i", + "hu man", + "e asy", + "lo gi", + "mi ke", + "gro w", + "ag re", + "w w", + "sh ad", + "mo tiv", + "wi de", + "tur ns", + "om g", + "v ar", + "de fin", + "su g", + "j im", + "ðŁĶ ¥", + "t d", + "campa ign", + "nam ed", + "re tweet", + "co p", + "t v", + "le av", + "k is", + "dou ble", + "s mar", + "issu e", + "vil la", + "in formation", + "li es", + "sto ck", + "n t", + "di stric", + "sh or", + "mi x", + "er o", + "se p", + "me x", + "see ing", + "li ve", + "re min", + "co de", + "g ur", + "s c", + "wil d", + "l un", + "h ood", + "spo t", + "fa ther", + "fore ver", + "up d", + "tra f", + "f ly", + "ne ed", + "gra du", + "tra in", + "ma ke", + "s ab", + "be y", + "si ze", + "lead er", + "tal ks", + "e u", + "lo g", + "fo x", + "gor geous", + "le ss", + "le ts", + "sur pri", + "my self", + "no te", + "li ves", + "f ru", + "lo ved", + "se ver", + "de m", + "j i", + "so c", + "h old", + "do gs", + "n i", + "â ŀ", + "lea ve", + "air port", + "ben ef", + "ex pl", + "shi ps", + "comple te", + "ach i", + "gre at", + "vin tage", + "j ack", + "ro c", + "woo d", + "pri v", + "off er", + "ey e", + "ver sion", + "te a", + "co ach", + "off ic", + "w ell", + "g en", + "s at", + "h h", + "you th", + "o x", + "? \"", + "m t", + "mi x", + "g g", + "d le", + "natu ral", + "buil d", + "break fast", + "thin king", + "theat re", + "mo on", + "ber g", + "go als", + "geor ge", + "en e", + "exc ell", + "il ing", + "tun e", + "y ed", + "g ate", + "m it", + "net work", + "jo e", + "h ello", + "f b", + "tu be", + "we aring", + "ath le", + "stru c", + "har d", + "gla ss", + "g ers", + "thro w", + "g es", + "b t", + "indu stry", + "manag ement", + "ali st", + "go al", + "stre am", + "y el", + "a vi", + "ici ous", + "o thers", + "s ki", + "chri sti", + "bir d", + "e sc", + "m in", + "tr o", + "l t", + "j an", + "im p", + "ri ghts", + "sh a", + "or gan", + "cent ral", + "ar a", + "ro ll", + "favour ite", + "che ster", + "el se", + "p ay", + "car s", + "m ine", + "ste p", + "prac tice", + "maj or", + "h ang", + "ðŁĺ ĺ", + "n on", + "v ari", + "eng ine", + "vol un", + "di a", + "i led", + "arch itec", + "p ink", + "d s", + "th y", + "wa sh", + "web site", + "ba g", + "contro l", + "el li", + "f ra", + "an sw", + "d ence", + "y u", + "r on", + "ol a", + "g in", + "dr in", + "li c", + "cou ple", + "sp ar", + "g on", + "cre ate", + "c t", + "celebr ating", + "de ep", + "e at", + "te e", + "vo ice", + "dro p", + "vis it", + "at ors", + "sta dium", + "f t", + "w is", + "ro l", + "gra de", + "fam il", + "po ints", + "re pre", + "w as", + "traf fic", + "jap an", + "or g", + "hon or", + "tex as", + "man u", + "âĻ ¥", + "safe ty", + "re r", + "b ag", + "em plo", + "rele ased", + "re gu", + "ak a", + "n av", + "ro le", + "sen ior", + "spec t", + "cro ss", + "lin es", + "be st", + "p ack", + "s in", + "ti e", + "mis sing", + "sun set", + "li ber", + "is ing", + "j ay", + "sk i", + "champion ship", + "ac tiv", + "la dies", + "play ed", + "y y", + "pu bl", + "al o", + "pri de", + "s r", + "pa ki", + "lu x", + "sur vi", + "ck ed", + "e ts", + "cho col", + "austr alia", + "par is", + "mi les", + "h at", + "ment al", + "al a", + "me an", + "mob ile", + "en a", + "in si", + "f ound", + "chi ef", + "t ag", + "incredi ble", + "re turn", + "à ©", + "goo gle", + "fren ch", + "cre w", + "hal lo", + "ali an", + "j az", + "ch er", + "sil ver", + "nor th", + "eng lish", + "base ball", + "c af", + "lim ited", + "follow ing", + "app reci", + "ear th", + "k ir", + "ve mber", + "w ed", + "p tion", + "g ed", + "oc tober", + "fl ori", + "c r", + "en cy", + "ga ve", + "lor d", + "stu ff", + "ber ry", + "po st", + "sm ile", + "bro ad", + "st ate", + "gg er", + "me ans", + "ic y", + "gu n", + "y o", + "ma ster", + "bur g", + "han ds", + "ni e", + "/ /", + "uni on", + "brit ish", + "big gest", + "distric t", + "am ing", + "h il", + "o ce", + "per son", + "pas s", + "en vir", + "scho ols", + "arri ved", + "anc es", + "insp ired", + "ex pla", + "be n", + "libr ary", + "bo tt", + "am p", + "ste ph", + "cont act", + "b ang", + "m s", + "cali for", + "t old", + "batt le", + "b b", + "chic ago", + "âľ ¨", + "str ate", + "sh i", + "de ce", + "- )", + "ad d", + "la b", + "j ones", + "leg end", + "cast le", + "ing er", + "st ance", + "be l", + "ur a", + "re fu", + "lead ers", + "po t", + "se x", + "h ic", + "artic le", + "ki d", + "fr ance", + "x x", + "ex e", + "gui de", + "volun te", + "pr int", + "al i", + "ce o", + "twee ts", + "w x", + "scen e", + "vol u", + "ant i", + "h an", + "as soci", + "shar ing", + "ro se", + "mini ster", + "sh er", + "in ste", + "cle an", + "demo cr", + "po ster", + "sk in", + "p sy", + "pro per", + "cra zy", + "i am", + "o re", + "in i", + "any thing", + "po d", + "mo ving", + "cl ick", + "ex plo", + "com b", + "cra ft", + "f i", + "bloo d", + "is ra", + "publ ic", + "d ent", + "ol ym", + "eng land", + "a si", + "ch er", + "fac t", + "envir on", + "har ry", + "g one", + "me dic", + "enjo ying", + "just ice", + "j r", + "indi an", + "wi fe", + "s ound", + "t es", + "dra wing", + "p al", + "ide a", + "cr it", + "ju li", + "il er", + "war m", + "cl ar", + "thou ghts", + "def en", + "coun cil", + "intro duc", + "di ed", + "jan u", + "an i", + "s end", + "li er", + "m l", + "intere sting", + "tra de", + "win d", + "b ay", + "s ac", + "anc y", + "sour ce", + "b es", + "org ani", + "ar ly", + "lar ge", + "ff ici", + "ta g", + "u t", + "de sp", + "o es", + "tit le", + "sy m", + "pic tures", + "op en", + "wom en", + "sho wing", + "ri a", + "le ast", + "lead ership", + "cur rent", + "elec tr", + "val ent", + "list ening", + "c key", + "gener al", + "de ser", + "du ce", + "; )", + "c ent", + "ðŁĺį ðŁĺį", + "sco tt", + "po or", + "selfi e", + "ev ents", + "i on", + "wr ong", + "de v", + "h ill", + "sep te", + "cul ture", + "l ine", + "sor ry", + "s ent", + "si ster", + "ce pt", + "k ri", + "no vember", + "ar i", + "announ ce", + "z ation", + "br an", + "g ent", + "d u", + "l en", + "per s", + "f m", + "mart in", + "o p", + "e mb", + "om e", + "midd le", + "suc cess", + "pe ter", + "janu ary", + "f lu", + "rac ing", + "d av", + "bi ke", + "ðŁı »", + "pe t", + "shoo t", + "profe ssi", + "feat uring", + "septe mber", + "now playing", + "sta ur", + "z a", + "on ic", + "qu ick", + "bas ke", + "spe aking", + "mil it", + "z er", + "chick en", + "b ell", + "s ad", + "co ast", + "lo ving", + "y ers", + "d j", + "pan el", + "ver age", + "s wit", + "ic ks", + "b ou", + "califor nia", + "s am", + "paren ts", + "er o", + "k illed", + "ph ys", + "jo bs", + "mi gr", + "an th", + "e mo", + "hallo ween", + "and er", + "c m", + "compet ition", + "e ag", + "s ket", + "sp ir", + "may be", + "exclu sive", + "app e", + "jour ney", + "scre en", + "for d", + "i o", + "h ate", + "u g", + "sou l", + "her o", + "soci ety", + "sy n", + "gu it", + "n h", + "d j", + "as es", + "im pre", + "ti me", + "sal es", + "d d", + "f ts", + "summ it", + "stun ning", + "om s", + "tur ned", + "cle an", + "sof t", + "be at", + "re staur", + "de red", + "en ces", + "ma gic", + "di o", + "sh ine", + "gu est", + "health y", + "exhi b", + "stor ies", + "po pu", + "n is", + "el a", + "bel ow", + "fun ny", + "resul ts", + "s ne", + "cur rently", + "ar d", + "down load", + "f light", + "m al", + "f ine", + "p ad", + "ch u", + "ent ed", + "h at", + "ðŁij ı", + "ste ve", + "j o", + "mar k", + "r at", + "b all", + "p c", + "p on", + "b by", + "o li", + "ar ts", + "as ure", + "bow l", + "att ack", + "mi c", + "de ar", + "ran ge", + "en ter", + "chocol ate", + "br illi", + "ac cess", + ", \"", + "? ??", + "ch ap", + "con st", + "t n", + "mat ter", + "blu e", + "gall ery", + "em p", + "work shop", + "lead ing", + "y ours", + "baske tball", + "w anna", + "th u", + "_ _", + "mar ri", + "sle ep", + "bi a", + "ch e", + "ma d", + "imp act", + "o wn", + "si r", + "chan nel", + "euro pe", + "e sp", + "k itch", + "hosp ital", + "w ra", + "roy al", + "f s", + "ne u", + "qu ar", + "ne y", + "ac ks", + "ch ase", + "pp y", + "st al", + "at ely", + "ti m", + "dece mber", + "r are", + "per form", + "cre am", + "we ight", + "ch oo", + "ni ght", + "ha ven", + "fr anc", + "kh an", + "buil t", + "hel ping", + "tru st", + "ty pe", + "gol den", + "ta x", + "s now", + "s wi", + "di sa", + "questi ons", + "ve y", + "li ght", + "c n", + "cl oud", + "thom as", + "ag ed", + "sh ou", + "te ams", + "gr an", + "re ason", + "a a", + "you tube", + "v p", + "pi zz", + "manag er", + "bur y", + "cre dit", + "tre at", + "ma x", + "i k", + "ma in", + "g ing", + "de ad", + "pro bab", + "ye ah", + "ã Ĥ", + "br and", + "so li", + "pl ant", + "ta yl", + "gir l", + "ðŁĺ Ń", + "nam ent", + "au to", + "mess age", + "ko re", + "n ur", + "ter r", + "ag u", + "ma p", + "sen ting", + "lo ves", + "gi ves", + "g ab", + "z en", + "ro bert", + "con fir", + "w ars", + "o m", + "sta in", + "cam era", + "and er", + "won der", + "a b", + "ca p", + "s old", + "su it", + "wal king", + "contin ue", + "effe c", + "dau ghter", + "d anc", + "cha in", + "mul ti", + "ki d", + "y an", + "champi on", + "v o", + "ta ins", + "ho st", + "min i", + "mis sed", + "re sc", + "ly n", + "fin ish", + "del icious", + "s as", + "tayl or", + "i b", + "pro mis", + "produc ts", + "moun tain", + "flori da", + "regi ster", + "tre at", + "rec ent", + "fe male", + "boo th", + "mat t", + "ve hic", + "s op", + "mo tor", + "suppor ting", + "phi c", + "ex tre", + "dr ink", + "lan e", + "th ird", + "p s", + "con stru", + "ce re", + "far m", + "ðŁİ ī", + "tu red", + "ðŁij ī", + "c ats", + "a j", + "gi e", + "shoo ting", + "as ked", + "paki stan", + "am e", + "m b", + "g il", + "leg al", + "squ are", + "in vol", + "dra w", + "oo oo", + "!! !!", + "opportun ity", + "p y", + "e i", + "b ts", + "teach er", + "charac ter", + "john son", + "br on", + "ly wood", + "ch ine", + "c ing", + "c ine", + "d ge", + "gam ing", + "russi a", + "ci a", + "quo te", + "ric h", + "go v", + "flow ers", + "sp iri", + "st in", + "grow th", + "ðŁı ¼", + "comm er", + "j uni", + "mu m", + "r an", + "s na", + "a ren", + "c b", + "ac tor", + "col or", + "si t", + "pa ir", + "ch i", + "bo w", + "acade my", + "hel d", + "r ang", + "me tal", + "y l", + "ac tive", + "probab ly", + "t ch", + "need ed", + "spe e", + "cho ice", + "ital y", + "ry an", + "ðŁĩ º", + "flow er", + "v it", + "m n", + "found ation", + "b ak", + "si ons", + "ne igh", + "f loo", + "he ard", + "re mo", + "fre sh", + "ing ing", + "re f", + "to wn", + "cl ou", + "je sus", + "spiri t", + "cou ldn", + "z es", + "ðŁĴ Ļ", + "willi ams", + "pro ce", + "moder n", + "pro cess", + "sho es", + "cre ated", + "tri c", + "issu es", + "ann e", + "att en", + "de but", + "h r", + "n it", + "sti g", + "a po", + "e ps", + "z u", + "ã Ģ", + "si x", + "car ds", + "lan gu", + "fam ous", + "tour nament", + "se l", + "e bay", + "y n", + "st on", + "k ick", + "announ ced", + "k am", + "vo c", + "brilli ant", + "hou se", + "che ese", + "war ri", + "mus ic", + "ho ckey", + "ðŁĺĤ ðŁĺĤ", + "sk ills", + "au tom", + "smar t", + "med ical", + "mon y", + "e x", + "gu ar", + "gi ve", + "pers onal", + "ven tion", + "al li", + "pre ss", + "flo or", + "m c", + "victor y", + "hi m", + "simp le", + "th or", + "ðŁĩº ðŁĩ", + "ta il", + "lu cky", + "ale x", + "qu ite", + "bo t", + "ssi ons", + "chall eng", + "c ann", + "amaz on", + "h ell", + "b ought", + ") :", + "ed y", + "secre t", + "produc tion", + "inde pend", + "de fe", + "ad ded", + "p r", + "p ag", + "be d", + "gre atest", + "with in", + "j ay", + "ðŁ ¥", + "ire land", + "re ly", + "s d", + "te xt", + "dri ving", + "pro gram", + "spe ed", + "col um", + "str on", + "à ©", + "fore st", + "â ĸ", + "mach ine", + "co in", + "sc ar", + "oun t", + "bi e", + "¡ ï¸ı", + "por tra", + "comm on", + "wre st", + "recei ved", + "kno w", + "inve st", + "pl ans", + "ac cor", + "ad op", + "ter y", + "re ali", + "p p", + "k al", + "art work", + "me an", + "go d", + "inste ad", + "an ci", + "motiv ation", + "as ing", + "inspir ation", + "up coming", + "polit ical", + "euro pe", + "m ers", + "heav y", + "ðŁij į", + "fe bru", + "scot land", + "ou gh", + "b t", + "bo ss", + "sche du", + "spe ak", + "n ick", + "u red", + "in o", + "e k", + "ri sk", + "tor y", + "pres ents", + "b on", + "ru g", + "st ates", + "exhib ition", + "il o", + "m ill", + "br ought", + ": -)", + "tou ri", + "com e", + "offici ally", + "champi ons", + "do ors", + "re p", + "po se", + "ex tra", + "k ings", + "soc cer", + "squ ad", + "app lic", + "at a", + "some times", + "t ari", + "excell ent", + "ðŁĺ ĺ", + "stra ight", + "car ol", + "ri p", + "âĢ į", + "gra phic", + "m ol", + "elec tion", + "febru ary", + "as ons", + "l i", + "di r", + "m t", + "n ick", + "u su", + "m rs", + "com ics", + "inst itu", + "cor por", + "v i", + "ðŁĻ ı", + "tu ral", + "di se", + "ac ci", + "we are", + "am ong", + "sho pping", + "t ill", + "wh at", + "cha ir", + "sp an", + "chine se", + "innov ation", + "jo y", + "k it", + "cent ury", + "ob ama", + "ph ili", + "f c", + "re ach", + "c iti", + "ul ous", + "n on", + "d ang", + "happ ening", + "bur n", + "p el", + "or ange", + "d v", + "k ick", + "cla im", + "ing ham", + "ph y", + "no v", + "pod cast", + "wh i", + "ni ghts", + "ear lier", + "be ar", + "la h", + "exc iting", + "or a", + "gi ven", + "s lo", + "memor ies", + "contin ues", + "produc t", + "gh o", + "c d", + "kno ws", + "ðŁİ ī", + "publi shed", + "discu ss", + "y ard", + "i phone", + "tri es", + "w all", + "fe b", + "are n", + "tru th", + "win ners", + "tu re", + "diti onal", + "milit ary", + "proble m", + "m and", + "do g", + "lo ss", + "c ric", + "can adi", + "ve ter", + "villa ge", + "\" ,", + "y r", + "un g", + "don ald", + "ag ing", + "bir ds", + "sci enti", + "le s", + "th is", + "regi on", + "tic al", + "itt en", + "il a", + "ðŁĺ İ", + "d ad", + "di am", + "abo ve", + "st ren", + "li t", + "p ir", + "la b", + "fo cus", + "bus y", + "d ur", + "app ly", + "s ma", + "auth or", + "ac i", + "exe cu", + "dom in", + "re la", + "jack son", + "at o", + "wash ington", + "ðŁĻ Į", + "k ill", + "popu lar", + "ce ment", + "ro ad", + "e ating", + "loc ation", + "v ent", + "ar re", + "n an", + "cu sto", + "advent ure", + "or din", + "spor t", + "ul t", + "lo ck", + "questi on", + "dri ver", + "land sc", + "on i", + "k ins", + "p d", + "jor dan", + "te red", + "k k", + "a f", + "chil d", + "s p", + "just in", + "en i", + "s elling", + "z o", + "wh it", + "bo ston", + "partic ip", + "sig ning", + "happ ened", + "he at", + "m am", + "dre ams", + "lo ws", + "gra ph", + "the day", + "head ing", + "br o", + "ble ssed", + "vi c", + "ve gas", + "h d", + "in ning", + "ro man", + "and ro", + "den ti", + "u se", + "c it", + "pro gress", + "writ er", + "bo b", + "ff s", + "gro wing", + "b ly", + "aw are", + "ex am", + "sp ent", + "be t", + "sc ore", + "bey ond", + "do cu", + "ad el", + "s f", + "cou ra", + "colla bor", + "in c", + "priv ate", + "bo at", + "* *", + "z one", + "p ha", + "b ill", + "to tal", + "plan ning", + "to wards", + "plac es", + "pre view", + "cre ative", + "dam n", + "ide as", + "se ems", + "po ten", + "say ing", + "di splay", + "s w", + "a qu", + "lou is", + "by e", + "li l", + "e mail", + "we stern", + "ger many", + "ell er", + "re s", + "f ant", + "ment ary", + "de als", + "ric hard", + "jer sey", + "stren g", + "ra d", + "pizz a", + "mon d", + "w are", + "l ac", + "g i", + "ar chi", + "c d", + "yel low", + "rec ently", + "re ach", + "à ¹", + "kitch en", + "desig ned", + "tr y", + "g al", + "restaur ant", + "at ure", + "w w", + "j as", + "l ma", + "ðŁij Į", + "pa in", + "av o", + "min ute", + "sch ol", + "ther ap", + "tic ket", + "d ry", + "jap an", + "diti ons", + "ter ri", + "sel ves", + "happ en", + "t up", + "ma g", + "cop y", + "sh er", + "free dom", + "f ile", + "speci ally", + "tor onto", + "lo ad", + "g ary", + "re y", + "answ er", + "lo y", + "cau ght", + "pri ze", + "u ne", + "fic ation", + "ni ger", + "sy d", + "tou ch", + "feat ure", + "jaz z", + "recor ds", + "him self", + "di sh", + "ro ber", + "spot ted", + "ma ster", + "wa ve", + "fin als", + "bu ll", + "for um", + "al d", + "re comm", + "ch a", + "a e", + "d oo", + "inst ru", + "tru ly", + "l g", + "in k", + "bro thers", + "de st", + "j im", + "m it", + "clo sed", + "is on", + "tri ed", + "s anta", + "af fe", + "w an", + "hor se", + "g row", + "camp us", + "rel ation", + "nati ve", + "jour n", + "go v", + "o ct", + "k it", + "b ound", + "part ner", + "re ma", + "crow d", + "! )", + "c alls", + "ra il", + "qu ali", + "solu tion", + "con test", + "con vers", + "sn ap", + "b ase", + "in iti", + "ta x", + "y e", + "ent repre", + "it or", + "constru ction", + "foo d", + "present ed", + "n ings", + "cli mate", + "k m", + "mo del", + "b j", + "blo ck", + "present ation", + "dre am", + "fi x", + "c alling", + "bus ine", + "con gress", + "under stand", + "we b", + "val ue", + "ï¸ı âĥ£", + "mex ico", + "it ely", + "ki m", + "char ity", + "ref lec", + "bl an", + "fl ying", + "anal y", + "famil ies", + "b and", + "reci pe", + "celebr ation", + "ac cep", + "ar y", + "to t", + "g b", + "intere sted", + "cap tain", + "âĻ ¥", + "ti p", + "ab sol", + "bra z", + "inve stig", + "o logy", + "de c", + "tru ck", + "ver ing", + "c lear", + "don t", + "go tta", + "ad vis", + "beg ins", + "ma ss", + "de scri", + "blo ck", + "k im", + "davi d", + "son gs", + "memor ial", + "feat ures", + "su stain", + "' .", + "gra b", + "jo se", + "v a", + "con serv", + "se ts", + "man chester", + "fi ghting", + "de gre", + "ag a", + "in d", + "sle ep", + "pos ition", + "ha ir", + "sig ns", + "pol icy", + "it o", + "al ert", + "st am", + "sp end", + "w y", + "absol ut", + "d m", + "anim al", + "my ster", + "success ful", + "proble ms", + "ro bo", + "k ay", + "gar den", + "p d", + "may or", + "d ale", + "t ol", + "off ers", + "vis iting", + "friend ly", + "tre es", + "offic er", + "accoun t", + "ke vin", + "ðŁij į", + "gi ant", + "contin u", + "con su", + "tr act", + "n fl", + "ðŁĺ Ĭ", + "h q", + "b ility", + "a ar", + "dis ney", + "te en", + "on ed", + "wh ite", + "tra iler", + "de dic", + "al one", + "absolut ely", + "dig ital", + "willi am", + "in ation", + "s wa", + "e e", + "enti re", + "ger man", + "ro ll", + "h its", + "co st", + "st ay", + "th a", + "ali ve", + "accor ding", + "co t", + "liter ally", + "her it", + "re ti", + "haha ha", + "exper i", + "li kes", + "g t", + "ste el", + "__ __", + "ch air", + "christi an", + "to wer", + "diffe rence", + "m d", + "tre ss", + "mi d", + "prin ce", + "afric an", + "fe der", + "foo t", + "car ri", + "ser ved", + "r ice", + "sh all", + "feat ured", + "ck er", + "rec ru", + "po e", + "sen se", + "ni fic", + "com edy", + "cont ent", + "f at", + "po sted", + "con tribu", + "tim ate", + "li ver", + "mb le", + "inter net", + "ag e", + "europe an", + "cl ing", + "gla d", + "ff ic", + "sc o", + "ak es", + "el le", + "ter min", + "ton y", + "p ale", + "col our", + "seri ous", + "pat ri", + "movi es", + "b m", + "professi onal", + "ad o", + "al u", + "br inging", + "f alls", + "isra el", + "ter m", + "langu age", + "bro ok", + "man n", + "commun ic", + "can not", + "ac ti", + "p he", + "y an", + "entrepre ne", + "tur key", + "log ical", + "lon g", + "ar m", + "ur s", + "work ers", + "ing ly", + "gg s", + "ri c", + "tu al", + "recei ve", + "op ens", + "ge ar", + "soci al", + "fe et", + "c king", + "ad ver", + "fin an", + "fe els", + "sp la", + "h r", + "ea ster", + "bra in", + "ã ģ", + "fi g", + "le dge", + "ne arly", + "prote ct", + "ma ssive", + "e th", + "aw a", + "ðŁĺ ģ", + "y rs", + "aware ness", + "defin itely", + "k n", + "imag ine", + "k u", + "syste ms", + "ðŁij ı", + "f as", + "li k", + "provi de", + "am o", + "disco ver", + "inf lu", + "ma ker", + "g az", + "fit ness", + "stre et", + "er s", + "te d", + "w c", + "ys is", + "pos itive", + "hel ped", + "que st", + "andre w", + "bra d", + "b in", + "hang ing", + "l ing", + "bri ght", + "se ction", + "ma ss", + "ðŁĻ Į", + "follow ers", + "ho sting", + "tem por", + "fla g", + "a ve", + "let ter", + "k ur", + "re qui", + "of ten", + "cry p", + "su ff", + "âļ ½", + "russi an", + "treat ment", + "al le", + "ha y", + "l an", + "keep ing", + "hol y", + "power ful", + "pre dic", + "fun d", + "e specially", + "windo w", + "je wel", + "il y", + "ðŁĴ ľ", + "gener ation", + "app a", + "seri ously", + "o d", + "ðŁĺĤðŁĺĤ ðŁĺĤ", + "cer ti", + "iri sh", + "ðŁij Į", + "mi ami", + "be th", + "v ity", + "se cu", + "che f", + "cri me", + "graph y", + "ma x", + "arti sts", + "re volu", + "gu ard", + "spee ch", + "u c", + "upd ates", + "fac es", + "st ant", + "chang ed", + "repor ts", + "low er", + "pe ar", + "n c", + "k il", + "loo ked", + "spe aker", + "s f", + "re spect", + "ok ay", + "oce an", + "s itting", + "architec ture", + "tra il", + "se at", + "i ra", + "le g", + "japan ese", + "d am", + "u lar", + "sw im", + "polit ics", + "finan cial", + "ol d", + "mou th", + "at temp", + "de stin", + "fi shing", + "atten tion", + "me m", + "chang es", + "deci ded", + "reli gi", + "g in", + "c av", + "z z", + "ad am", + "ma c", + "wr ite", + "beg in", + "sc ul", + "al ter", + "is s", + "ath on", + "imag es", + "m oo", + "jo ined", + "ðŁĺ ī", + "âŀ ¡ï¸ı", + "pas sed", + "mu sli", + "h ir", + "lar gest", + "cam er", + "com ic", + "gh ted", + "rug by", + "bur gh", + "gg ing", + "te sting", + "pre par", + "lau gh", + "al ed", + "impro ve", + "beli ev", + "adv ice", + "sha res", + "he art", + "tur ning", + "s b", + "t el", + "caf e", + "n es", + "dani el", + "pat ter", + "t z", + "se tt", + "par k", + "c and", + "st ick", + "happ ens", + "bri an", + "ne west", + "e pic", + "ad or", + "ki es", + "war ning", + "anim als", + "custo m", + "ar c", + "di an", + "gol d", + "cor e", + "t f", + "c ity", + "pan ts", + "re ality", + "con fi", + "in ju", + "fo x", + "gu il", + "k new", + "âĺ º", + "cor rec", + "itu de", + "d den", + ". #", + "re duc", + "pas s", + "f on", + "y a", + "ow ner", + "re turns", + "n c", + "e ast", + "ap ol", + "in sur", + "th o", + "si m", + "juni or", + "be e", + "ang el", + "att le", + "elec tric", + "hor ror", + "cra sh", + "e ye", + "pat h", + "sou thern", + "emplo ye", + "ge o", + "t an", + "ha z", + "r ally", + "ðŁı »", + "proper ty", + "was n", + "enjo yed", + "gre y", + "g as", + "bre w", + "nor thern", + "hol ding", + "g p", + "ta ke", + "ch art", + "ly n", + "dr ama", + "z o", + "pa id", + "throw back", + "cu p", + "discu ssion", + "down town", + "w ill", + "le w", + "b is", + "t ary", + "bre ad", + "up on", + "r ate", + "teach ers", + "it ation", + "anc ed", + "cy cle", + "choo se", + "d c", + "ir an", + "co w", + "da ve", + "ra ise", + "prin cess", + "fa ith", + "- >", + "indu stri", + "sp ain", + "guit ar", + "fac ts", + "m n", + "sp en", + "cour te", + "go tt", + "projec ts", + "au di", + "o sc", + "pe ter", + "s and", + "intere st", + "happ iness", + "ven ue", + "sol di", + "surpri se", + "poten tial", + "per io", + "custom er", + "i i", + "g ni", + "manu fac", + "e co", + "bro ken", + "sing er", + "vel s", + "wal es", + "hu s", + "in j", + "f our", + "tal ent", + "d ying", + "mat the", + "fil m", + "jo ining", + "s ell", + "j ar", + "lma o", + "sur ger", + "bb c", + "sour ces", + "au stin", + "ni k", + "char les", + "f am", + "prin ci", + "ange l", + "cas h", + "lo t", + "o red", + "pla ys", + "pl ate", + "don e", + "memor y", + "br ings", + "n ba", + "solu tions", + "teach ing", + "gr ace", + "cir cu", + "hel ps", + "foun der", + "mar y", + "expl ore", + "de cor", + "par ts", + "ch o", + "inte gr", + "ha u", + "is es", + "pu tting", + "in er", + "r it", + "v y", + "mic hel", + "blu es", + "every day", + "for ms", + "bi o", + "ye ar", + "p in", + "t ter", + "spr ing", + ") )", + "po t", + "al ing", + "perform ing", + "sh an", + "plan et", + "mus ical", + "head s", + "it alian", + "stru gg", + "âĢį âĻ", + "w ings", + "pu mp", + "h h", + "tr ou", + "a id", + "pri me", + "ear th", + "pa int", + "mon t", + "am y", + "bb c", + "fab ulous", + "fru it", + "andro id", + "bour ne", + "cere mony", + "enti al", + "? ?", + "deb ate", + "on ing", + "dra ft", + "sol ar", + "t x", + "j am", + "cor n", + "!! !!!", + "bro o", + "mil k", + "po sed", + "o hi", + "mo vement", + "b ren", + "part ner", + "p g", + "et te", + "ar ies", + "sh out", + "n g", + "leav ing", + "t ells", + "sen s", + "ta ste", + "kel ly", + "wor l", + "gy m", + "ric h", + "e gy", + "pi d", + "ma s", + "â Ĥ", + "courte sy", + "fran k", + "incre ase", + "wr itten", + "pp ers", + "re l", + "ha i", + "s as", + "s ound", + "tt i", + "w ich", + "ri ver", + ".. .\"", + "a g", + "fel low", + "ro me", + "sm all", + "gen cy", + "ic an", + "lux ury", + "pro of", + "me t", + "wild life", + "mom ents", + "ra ther", + "cor ner", + "com pe", + "canadi an", + "lik ely", + "therap y", + "li am", + "econom ic", + "indi e", + "rou te", + "fi ght", + "ho pe", + "se tting", + "ant ly", + "cro ss", + "fant asy", + "de e", + "sket ch", + "comp li", + "ym i", + "ru les", + "engine ering", + "fig ure", + "ro w", + ". ,", + "f w", + "syd ney", + "w ou", + "t ation", + "dre w", + "us es", + "the re", + "sp read", + "struc ture", + "pat rick", + "appa rently", + "ro s", + "h ills", + "w we", + "ann y", + "com mission", + "di v", + "f ying", + "con sul", + "anal ysis", + "ex i", + "ten nis", + "vehic le", + "ðŁĺŃ ðŁĺŃ", + "as s", + "high ly", + "op ened", + "b ann", + "ðŁĴ Ļ", + "mp h", + "wi shing", + "v or", + "fi f", + "give away", + "r r", + "ra y", + "je ss", + "g at", + "ic ymi", + "x it", + "high est", + "yor k", + "pi e", + "invol ved", + "high er", + "ri e", + "mal ay", + "int elli", + "desp ite", + "che e", + "sar ah", + "be an", + "reco gni", + "ar sen", + "tal ented", + "pas sion", + "ic h", + "ab c", + "lead s", + "dise ase", + "v is", + "se c", + "pre senting", + "m illi", + "hol e", + "sho ts", + "de part", + "surger y", + "gov t", + "b in", + "du al", + "e vi", + "lon ger", + "ev ol", + "scre en", + "portra it", + "et c", + "lo se", + "ch at", + "p en", + "p i", + "om a", + "s ick", + "er c", + "compan ies", + "en try", + "plan e", + "gr y", + "ven e", + "liver pool", + "premi ere", + "sha red", + "a red", + "fil ms", + "ir a", + "holi days", + "cric ket", + "ici an", + "v ing", + ". )", + "ul timate", + "di vision", + "con duc", + "se pt", + "for ces", + "mon t", + "s mart", + "disa pp", + "sun shine", + "in d", + "b less", + "ma de", + "col ors", + "fran k", + "ir on", + "bott le", + "s go", + "m ood", + "j ason", + "er ic", + "bir th", + "te en", + "respon se", + "tar get", + "state ment", + "fe ar", + "th el", + "al um", + "ar ab", + "bl in", + "direc tion", + "ste ps", + "er ial", + "wor ked", + "at l", + "ðŁĴ ķ", + "fel t", + "pol i", + "scen es", + "hom es", + "b ell", + "e at", + "ate ful", + "t in", + "l ace", + "fol ks", + "p se", + "an n", + "wis dom", + "fa v", + "but ter", + "s r", + "are as", + "sm oo", + "bi z", + "dg es", + "app o", + "mo re", + "the m", + "effe ct", + "windo ws", + "sun ny", + "cap ital", + "tot ally", + "c ities", + "gr ant", + "mb ers", + "s low", + "au tu", + "il ities", + "w ro", + "ri sing", + "st ics", + "viol ence", + "i gh", + "qu ot", + "h it", + "t c", + "herit age", + "bu ff", + "ne s", + "z ar", + "den tial", + "ex ac", + "ed ge", + "de ep", + "aren a", + "be came", + "benef its", + "mar ks", + "mb er", + "a z", + "am es", + "pre ci", + "dra gon", + "re g", + "d ings", + "do s", + "ðŁĴ ª", + "n el", + "s ity", + "me al", + "di st", + "leg end", + "pur chase", + "pic al", + "st ick", + "f at", + "du ba", + "profe ss", + "car to", + "pro f", + "coun tries", + "respon si", + "se qu", + "fa b", + "tribu te", + "hon ored", + "prac tic", + "pur ple", + "an ton", + "pa red", + "t ough", + "summ er", + "environ ment", + "s ons", + "ðŁĻ ı", + "m ps", + "gi es", + "her oes", + "t elling", + "hen ry", + "f en", + "know ledge", + "Ģ ï¸ı", + "f r", + "ne g", + "u re", + "ac king", + "hear ts", + "s oo", + "hol lywood", + "ju mp", + "sau ce", + "schedu le", + "tur n", + "yo ga", + "cre ating", + "c ket", + "cre ek", + "â Ń", + "custom ers", + "ma dri", + "gu l", + "asse mb", + "moun t", + "c ell", + "to p", + "st al", + "dav is", + "t wi", + "sig n", + "premi er", + "iti ons", + "he aring", + "un k", + "pati ents", + "app ear", + "heav en", + "al ty", + "doc tor", + "a e", + "plat form", + "je ff", + "ðŁĵ ·", + "regi onal", + "bi d", + "box ing", + "ex ten", + "or ity", + "a w", + "w ise", + "il le", + "sever al", + "bi e", + "s itu", + "sy ria", + "âľ ħ", + "remin der", + "enter tain", + "li on", + "part ners", + "in n", + "ph ar", + "f au", + "pl s", + "expe cted", + "sug ar", + "deci sion", + "s b", + "ch ron", + "associ ation", + "leav es", + "vis ited", + "sh ap", + "ðŁĴ ĸ", + "fur ther", + "h ann", + "w i", + "run s", + "l er", + "fun ding", + "fil led", + ".. ....", + "tin y", + "han g", + "or g", + "co ol", + "se min", + "ðŁı Ĩ", + "spon s", + "nav y", + "sa int", + "dru g", + "d al", + "r oun", + "co vered", + "tra ditional", + "invest ment", + "de te", + "al ism", + "f low", + "n is", + "sun rise", + "fe at", + "f ted", + "we ird", + "je re", + "ve gan", + "medic ine", + "an o", + "ac cu", + "deli very", + "temp le", + "chang ing", + "wil son", + "phili pp", + "re fe", + "n d", + "is er", + "g ay", + "r and", + "ati ves", + "t ely", + "p and", + "intelli g", + "g are", + "am bas", + "de mon", + "commit tee", + "strate gy", + "refu ge", + "bud get", + "prote c", + "pi er", + "ex press", + "nom in", + "econom y", + "al low", + "ic on", + "gal ax", + "o h", + "indi vi", + "dem and", + "vir gin", + "lu ke", + "ali sts", + "man i", + "s mi", + "ju dge", + "ent y", + "mic hi", + "resul t", + "am ed", + "spe aks", + "' ,", + "hou ston", + "sh in", + "b ing", + "fl y", + "ch em", + "au to", + "v as", + "ge t", + "ar m", + "thank s", + "d in", + "gan g", + "x x", + "si on", + "loc ated", + "p l", + "jo sh", + "in fo", + "jo ins", + "adver ti", + "ot d", + "el d", + "si e", + "re asons", + "v ent", + "ðŁĩºðŁĩ ¸", + "â ł", + "convers ation", + "stu di", + "ðŁĶ¥ ðŁĶ¥", + "go s", + "s ounds", + "un it", + "mu sc", + "ge l", + "ack ed", + "pac i", + "co s", + "de re", + "u u", + "a o", + "la m", + "inspir ing", + "ar ms", + "tw are", + "mat ters", + "ad dic", + "du de", + "ex t", + "cri sis", + "b ath", + "me et", + "sing h", + "expe ct", + "del hi", + "resc ue", + "wor st", + "au g", + "shi pping", + "ser ving", + "st o", + "dar k", + "ac es", + "histor ic", + "landsc ape", + "desig ner", + "b illion", + "gr ateful", + "wa ke", + "e ve", + "m iller", + "hou sing", + "dy nam", + "is co", + "be ha", + "sh op", + "pr ou", + "e as", + "a sia", + "e ding", + "k on", + "depart ment", + "aw ar", + "mar ine", + "in ci", + "photograph er", + "ta pe", + "lo go", + "r ings", + "d it", + "-- --", + "vin yl", + "w c", + "vo ting", + "se ven", + "ambas sad", + "dal las", + "t u", + "com ment", + "k ra", + "b les", + "w ag", + "u d", + "au dio", + "stri ke", + "offici al", + "o ts", + "me tho", + "to ols", + "ra di", + "al an", + "hun t", + "wat ched", + "a ke", + "fa ke", + "drin king", + "mer ry", + "m l", + "b day", + "ri o", + "ni ke", + "c ant", + "re pe", + "co stu", + "mur der", + "ak ers", + "ch ers", + "ou ts", + "beg inning", + "so s", + "ad es", + "n in", + "not es", + "wro te", + "sol o", + "c i", + "li ghting", + "ur ban", + "bre xit", + "att end", + "shir ts", + "pla yo", + "ac tress", + "pl ic", + "stand ard", + "quot es", + "par ade", + "anci ent", + " ©", + "tur ing", + "re e", + "pri mary", + "fla sh", + "citi z", + "mat es", + "ste in", + "z i", + "clin ton", + "sk in", + "gen e", + "hu m", + "g ar", + "t le", + "y i", + "fo cu", + "de an", + "pl ants", + "cy ber", + "b u", + "om e", + "ho p", + "ad dress", + "ti x", + "gi fts", + "relation ship", + "sub scri", + "fe ed", + "exac tly", + "haw ks", + "ex o", + "stre ss", + "s n", + "arre sted", + "an e", + "sof tware", + "z ero", + "the me", + "mu mb", + "im migr", + "mi a", + "make up", + "ple asure", + "uni vers", + "har b", + "eng ine", + "ap er", + "r in", + "br a", + "institu te", + "le ather", + "al th", + "sing ing", + "co s", + "gh ty", + "me as", + "st ic", + "si de", + "insur ance", + "co t", + "pit ch", + "moun tains", + "cri min", + "su pre", + "valent ine", + "at er", + "wou ldn", + "sc ale", + "rel ated", + "re gar", + "star tup", + "pack ed", + "mi ke", + "week ly", + "p ts", + "coun t", + "ha r", + "gott en", + "min d", + "ber lin", + "con ditions", + "swit ch", + "cor n", + "sa ve", + "g li", + "emer gency", + "tun ed", + "sto ck", + "discu ssing", + "every body", + "s day", + "whe ther", + "wrest ling", + "ec es", + "gen der", + "ch en", + "ðŁij Ģ", + "madri d", + "mar athon", + "e gg", + "i er", + "th x", + "as king", + "kore a", + "wol f", + "ay a", + "g m", + "g au", + "at ory", + "v r", + "gra ss", + "k illing", + "b ble", + "ur o", + "un i", + "e th", + "sh ore", + "th en", + "re ale", + "bot tom", + "ex erc", + "k ar", + "or ies", + "ad ri", + "san ds", + "se x", + ". '", + "volunte ers", + "per form", + "par liam", + "inclu de", + "deli ghted", + "execu tive", + "fu el", + "kis s", + "ã ħ", + "char ge", + "h u", + "ca kes", + "ve t", + "g lu", + "agre e", + "pr ices", + "n au", + "h l", + "g ru", + "ra j", + "streng th", + "b ic", + "sp ending", + "al es", + "av en", + "b last", + ": (", + "yo f", + "nor mal", + "si x", + "qu ick", + "se a", + "d aw", + "mee ts", + "lo vers", + "upd ated", + "po tat", + "comple ted", + "coo k", + "opportun ities", + "p ure", + "organ ic", + "tem per", + "c am", + "avo id", + "par king", + "duba i", + "and o", + "di stri", + "to y", + "comple tely", + "don ald", + "tri al", + "bas s", + "b oun", + "back ground", + "v as", + "mar vel", + "lu m", + "ru s", + "t ool", + "com missi", + "throw back", + "fin ding", + "is lam", + "! ?", + "st op", + "e vil", + "or al", + "resi dents", + "i denti", + "o ak", + "ðŁİ ¶", + "l il", + "span ish", + "chap ter", + "sto pped", + "direc t", + "ho sted", + "pic ked", + "lab our", + "lew is", + "defen se", + "à ®", + "health care", + "wh is", + "mat h", + "pe ak", + "ra ised", + "fi x", + "bu ll", + "th ir", + "chel sea", + "fol k", + "tr e", + "can di", + "pau l", + "ei ther", + "ad am", + "poe try", + "jewel ry", + "ðŁ ¦", + "pr ay", + "Ø §", + "g c", + "o z", + "wi shes", + "fore ign", + "sun g", + "lear ned", + "en e", + "n ing", + "micha el", + "illu stration", + "legend ary", + "w av", + "b au", + "ðŁļ ¨", + "cal end", + "stre ets", + "â Ĩ", + "mon ster", + "bu ck", + "g r", + "scho ol", + "ba th", + "wa ste", + "ne ck", + "ha wa", + "be ach", + "re plac", + "jec t", + "on er", + "fac tory", + "coun t", + "ðŁĵ ¸", + "mor gan", + "der ing", + "se an", + "steph en", + "de p", + "no vel", + "vide os", + "ic al", + "press ure", + "arsen al", + "ex pre", + "ir s", + "tren ding", + "ss a", + "fla sh", + "re sear", + "thr ough", + "profess or", + "scul p", + "to s", + "gg ed", + "mm a", + "be e", + "a pe", + "hun ter", + "am i", + "he i", + "pla stic", + "bu cks", + "uni verse", + "le gen", + "niger ia", + "ple ased", + "ri s", + "thin ks", + "autu mn", + "i ds", + "d is", + "anth ony", + "ðŁı ½", + "ak ed", + "gla sses", + "fin ance", + "z er", + "k as", + "con tract", + "nu mbers", + "sh aw", + "partner ship", + "t il", + "laun ched", + "s al", + "victor ia", + "theat er", + "usu al", + "nam es", + "perio d", + "eli za", + "i th", + "bar cel", + "ro cks", + "bag s", + "mat e", + "distri bu", + "j on", + "di ffic", + "ali zed", + "cur ren", + "sco red", + "b ha", + "du blin", + "ro se", + "in ted", + "soli d", + "beha vi", + "wal ker", + "simp ly", + "garden s", + "head ed", + "in i", + "ohi o", + "we ap", + "f o", + "gl en", + "e state", + "ran dom", + "th under", + "thr u", + "k ill", + "jac ket", + "it i", + "entertain ment", + "thanks giving", + "ent al", + "en coura", + "el o", + "a ther", + "tan k", + "high lights", + "f ting", + "ru le", + "model s", + "bor der", + "bj p", + "hus band", + "in done", + "ken ya", + "be ars", + "al o", + "n inten", + "pi x", + "str o", + "or ders", + "sal ad", + "ro ads", + "n or", + "l ation", + "sop hi", + "ðŁı ¼", + "pi eces", + "b one", + "min s", + "inclu des", + "nu tr", + "phi l", + "s ent", + "fun dra", + "ga in", + "bor ough", + "n ad", + "mon day", + "activ ity", + "it ems", + "be coming", + "ken ne", + "de tro", + "car di", + "gue sts", + "u x", + "world wide", + "sever e", + "new s", + "thank ful", + "fic tion", + "ve ge", + "m all", + "si an", + "er al", + "inj ury", + "le e", + "men u", + "danc ing", + "scot ti", + "exam ple", + "( #", + "na i", + "studi os", + "ba i", + "ðŁĴ Ľ", + "j av", + "diam ond", + "vin ce", + "ric k", + "prote ction", + "lin col", + "cham ps", + "appro ach", + "d ar", + "m ile", + "clou ds", + "je ff", + "in fin", + "l ers", + "p les", + "pe ace", + "go p", + "âĻ ¡", + "tech n", + "str a", + "a verage", + "ef fort", + "introduc ing", + "di versity", + "austr alian", + "am p", + "boo st", + "s ke", + "pati ent", + "appreci ate", + "ici ans", + "pu r", + "f ell", + "woo ds", + "illu str", + "ðŁ ĸ", + "ag ency", + "ac tions", + "brit ain", + "under way", + "se attle", + "el and", + "ag o", + "f ill", + "stre aming", + "pro test", + "challeng es", + "ky o", + "et sy", + "coo king", + "exper t", + "ru ss", + "rain bow", + "commer cial", + "sp in", + "be ats", + "c ry", + "val u", + "el i", + "th row", + "gr ams", + "le vels", + "michi gan", + "c ad", + "ador able", + "const itu", + "w s", + "pu b", + "mid night", + "th at", + "net fli", + "braz il", + "die go", + "regu lar", + "jo y", + "âĤ ¬", + "li qu", + "ea stern", + "k ni", + "fl at", + "n p", + "bro wn", + "w er", + "se y", + "tt ers", + "ac ting", + "v anc", + "cy cling", + "program me", + "ra w", + "comple x", + "tat too", + "throwback thursday", + "se ssions", + "ro oms", + "si ght", + "speci es", + "bom b", + "lau gh", + "ke eps", + "mo on", + "offic ers", + "con ver", + "t r", + "ha sh", + "t ack", + "ri ous", + "ad ap", + "a j", + "reco gn", + "ex po", + "sug ge", + "confir med", + "rol ling", + "dre ssing", + "ic t", + "fri day", + "ph ones", + "ri dge", + "con cept", + "ro y", + "ke ys", + "ef for", + "c ate", + "k ne", + "ev en", + "l ay", + "commun ities", + "mo d", + "n az", + "every where", + "al ab", + "bit coin", + "ban ks", + "out door", + "feder al", + "sto res", + "h p", + "c al", + "m ely", + "sig nific", + "be ar", + "re public", + "clo ser", + "al lah", + "pic k", + "x d", + "pal ace", + "ch ill", + "b am", + "er ous", + "un a", + "al len", + "out standing", + "olym pic", + "supp ly", + "fi gu", + "v au", + "l p", + "char lie", + "un es", + "> >>", + "legen ds", + "ici al", + "co ast", + "benef it", + "mul ti", + "f its", + "far mers", + "am ount", + "si sters", + "har ve", + "hon ey", + "que en", + "b ers", + "pl ann", + "âŃ IJ", + "m u", + "barcel ona", + "al ber", + "stat us", + "re main", + "ex tra", + "c andy", + "vi ous", + "âľ Į", + "o v", + "warri ors", + "-- >", + "ju mp", + "am ar", + "x mas", + "stu dies", + "i ors", + "k or", + "don ate", + "pre p", + "fi sh", + "im a", + "pain ted", + "ad mini", + "co splay", + "spor ts", + "dro ps", + "fi ghter", + "evi dence", + "ðŁĴ ª", + "la ke", + "ro b", + "cine ma", + "pro file", + "à ±", + "stan ds", + "leg acy", + "sh ape", + "ro of", + "ci vil", + "i ans", + "sy l", + "sh am", + "vo ted", + "re tail", + "ph illi", + "li sted", + "du ty", + "n b", + "th es", + "f are", + "au ction", + "ffici al", + "stor ms", + "d p", + "l oun", + "sh ops", + "al y", + "ani me", + "multi ple", + "ðŁĺį ðŁĺį", + "psy cho", + "je an", + "ap art", + "candi date", + "gg y", + "con f", + "jose ph", + "w ick", + "me at", + "fr ame", + "c l", + "for got", + "ph y", + "f ing", + "li ed", + "re p", + "se ed", + "f all", + "u fc", + "nu t", + "lin d", + "mo de", + "fiel ds", + "en ce", + "s ley", + "ðŁ¤ Ķ", + "ch ill", + "follow ed", + "announ ces", + "cor ru", + "tro phy", + "them selves", + "ac le", + "al du", + "k ong", + "l on", + "s v", + "bro ke", + "ander son", + "ta i", + "stor y", + "tempor ary", + "activ ities", + "k ati", + "ari z", + "cry stal", + "spo ke", + "extre mely", + "tra ding", + "ðŁĴ ļ", + "à ¼", + "in ch", + "ed in", + "out fit", + "equ ip", + "ma di", + "form ed", + "be ef", + "po p", + "ti ger", + "this day", + "ti red", + "neigh b", + "re tro", + "is a", + "un t", + "t as", + "kan sas", + "de st", + "secon ds", + "ta y", + "hur ric", + "o u", + "galax y", + "dad dy", + "bro w", + "bur ger", + "en ced", + "de sk", + "ac cur", + "secre tary", + "el ite", + "k ab", + "ch in", + "touri sm", + "bud dy", + "ici de", + "dre ssed", + "u d", + "vac ation", + "che ers", + "com for", + "charac ters", + "j et", + "bu ying", + "l ins", + "n ap", + "reale state", + "li e", + "af c", + "i ii", + "f ame", + "n r", + "b at", + "ag ent", + "ma kers", + "âĢ ¼", + "sec tor", + "op ti", + "le on", + "di et", + "pra yer", + "hi p", + "mi r", + "le x", + "br y", + "an a", + "pas sing", + "w en", + "reco very", + "ak i", + "po pul", + "res ort", + "mar ia", + "stu ck", + "read s", + "ti er", + "perfe c", + "netfli x", + "p oo", + "cham p", + "o c", + "re duce", + "we red", + "comm ents", + "cla im", + "acci dent", + "s ag", + "h ack", + "sal t", + "kin da", + "k iller", + "i os", + "z y", + "ex change", + "lec ture", + "eng er", + "ic king", + "t au", + "reve als", + "pri son", + "z om", + "gh an", + "u l", + "jour nal", + "i ot", + "tr in", + "jon a", + "govern or", + "cap e", + "quar ter", + "spec tive", + "impre ssive", + "bab ies", + "t x", + "m ill", + "o y", + "har ri", + "jo int", + "su e", + "collabor ation", + "tren d", + "revolu tion", + "re new", + "alum ni", + "ge tt", + "sh ell", + "sun day", + "ent u", + "ni c", + "donald trump", + "block chain", + "paci fic", + "expla ins", + "sp y", + "ad voc", + "par adi", + "to f", + "star ring", + "p av", + "fe ed", + "br ac", + "smo ke", + "ham p", + "y am", + "to kyo", + "si mon", + "d h", + "e ffici", + "phys ical", + "n j", + "ell i", + "s low", + "gradu ate", + "americ ans", + "ti fy", + "f red", + "ap ore", + "fin ds", + "rob in", + "we t", + "not ice", + "se mi", + "un ve", + "k om", + "pil ot", + "scre ening", + "da ily", + "ðŁĴ Ĺ", + "roy al", + "sp a", + "vo tes", + "n ag", + "wh ate", + "att ending", + "exper im", + "ad dition", + "k ate", + "sto l", + "m ali", + "foo t", + "chri st", + "ch an", + "de e", + "lic en", + "glo bal", + "mo ore", + "ti a", + "bri gh", + "myster y", + "y ay", + "âĿ¤ï¸ı âĿ¤ï¸ı", + "cre ati", + "me chan", + "clo ck", + "di c", + "âĢ Ķ", + "pp er", + "al ph", + "through out", + "al low", + "re sources", + "selec tion", + "ham il", + "bb q", + "aa aa", + "virgin ia", + "dis ney", + "en g", + "so red", + "drin ks", + "f ancy", + "consi der", + "end a", + "jan e", + "hand made", + "du l", + "on tari", + "i us", + "s ville", + "color ado", + "whate ver", + "whe el", + "promis e", + "ne ver", + "desig ns", + "ab ly", + "sex ual", + "vanc ou", + "at i", + "con vention", + "cul tural", + "sing apore", + "pro mo", + "load ed", + "gla sgo", + "pp l", + "n oo", + "ke e", + "ste m", + "men tion", + "i do", + "cru ise", + "ri ding", + "be comes", + "be y", + "âļ½ ï¸ı", + "tw in", + "dedic ated", + "na sh", + "de si", + "work out", + "jen ni", + "i v", + "grou ps", + "rela x", + "pho eni", + "li ft", + "mix ed", + "m ck", + "p c", + "mu st", + "me tro", + "ci es", + "y ar", + "a im", + "ang er", + "i e", + "rec y", + "marri ed", + "dro pped", + "eng ag", + "le st", + "ambassad or", + "op h", + "de s", + "w ick", + "assi stant", + "nat ur", + "fa il", + "l td", + "shor t", + "k ap", + "sha w", + "bi gger", + "rema ins", + "crit ical", + "sur vey", + "co verage", + "er son", + "win d", + "n b", + "bil ly", + "let es", + "ac ts", + "jim my", + "at lan", + "al and", + "t c", + "import ance", + "dam age", + "f g", + "stor age", + "tw t", + "bon d", + "bal ance", + "cr ying", + "pu ppy", + "vo te", + "pu sh", + "ðŁĴ ľ", + "pol y", + "me l", + "lon don", + "terr ori", + "effec tive", + "corpor ate", + "atl anta", + "jac o", + "nas a", + "gre ek", + "sen ate", + "i sh", + "ev a", + "intellig ence", + "effor ts", + "al co", + "k un", + "h all", + "di ag", + "claim s", + "fir st", + "h b", + "ba e", + "v ul", + "pu ll", + " °", + "se par", + "spe ed", + "vic ti", + "on thisday", + "audi ence", + "r ates", + "te ach", + "fil ming", + "bu sh", + "son g", + "y um", + "br un", + "ra ine", + "aw a", + "par ks", + "ð Ŀ", + "ra bb", + "ra ch", + "ra id", + "reach ed", + "ra il", + "mo ves", + "selec ted", + "fr i", + "ra ising", + "om y", + "st ones", + "su k", + "franc isco", + "cas es", + "cap it", + "con fu", + "w tf", + "po ke", + "equip ment", + "gre g", + "ess ential", + "off ering", + "ne x", + "pi es", + "be c", + "cre ation", + "chair man", + "cro wn", + "w al", + "john ny", + "shi ft", + "ne ck", + "ban g", + "bir d", + "ðŁĺ ı", + "du ck", + "re serve", + "de pu", + "ma sters", + "over all", + "no tic", + "ju ice", + "sne ak", + "che er", + "cla sses", + "eag les", + "n ca", + "car pet", + "ci vil", + "coach es", + "har ris", + "u ps", + "b alls", + "dec or", + "mar tin", + "ro s", + "v ice", + "announ cement", + "who se", + "ti gers", + "ste red", + "c ts", + "dr am", + "ste el", + "youn g", + "inst all", + "supp o", + "recor ding", + "de ck", + "se ats", + "l der", + "ang le", + "bo t", + "sty les", + "elec tions", + "for tun", + "n ab", + "but ter", + "ari an", + "ka sh", + "in ner", + "ou red", + "be ast", + "we i", + "ic onic", + "exper ts", + "ne cess", + "b eng", + "jam es", + "li a", + "gre ece", + "ðŁĵ ·", + "ðŁĺ ģ", + "good bye", + "m itch", + "tw ice", + "mumb ai", + "ste am", + "ru sh", + "med al", + "ne tt", + "fashi on", + "t ar", + "r s", + "sav ing", + "ric ul", + "l m", + "sleep ing", + "brook lyn", + "mis s", + "sen ding", + "disco vered", + "sp here", + "of theday", + "k icks", + "missi ons", + "w right", + "er n", + "ght ly", + "i ous", + "mel bourne", + "star tu", + "mo ved", + "car ry", + "d ak", + "ag ues", + "bel gi", + "e ma", + "way ne", + "do t", + "er ie", + "pe l", + "it unes", + "matthe w", + "no body", + "est ab", + "cal m", + "win ds", + "lu c", + "prep are", + "tren ds", + "exerc ise", + "adv ant", + "ðŁĴ ¯", + "athle tics", + "app s", + "c tions", + "adv ance", + "laun ches", + "litt le", + "real donaldtrump", + "eliza beth", + "carol ina", + "hu b", + "hi dden", + "n w", + "us er", + "pol l", + "great er", + "mo st", + "f ed", + "p at", + "life style", + "s ati", + "sco res", + "marri age", + "l r", + "aven ue", + "de serve", + "ri f", + "ðŁ Ĺ", + "wat ch", + "champion ships", + "gr ay", + "en ni", + "cot ton", + "g om", + "whe re", + "pack age", + "su m", + "ab solu", + "new ly", + "foo ds", + "ty ler", + "assemb ly", + "musli m", + "ban k", + "re memb", + "op tions", + "produc er", + "land o", + "fun ds", + "u pper", + "shad ow", + "pro gre", + "co p", + "ing e", + "leg s", + "detro it", + "hill ary", + "jo se", + "gi ants", + "sou p", + "sustain able", + "t us", + "clo thes", + "roc king", + "n z", + "min ne", + "mat eri", + "bru ce", + "ear t", + "ca sting", + "independ ent", + "thou sands", + "ta h", + "de cl", + "veter ans", + "li ons", + "wra p", + "âĢ ¦", + "de ss", + "bl ing", + "st ine", + "e ggs", + "o on", + "clo sing", + "z ay", + "at t", + "bac on", + "fa il", + "ariz ona", + "de pre", + "gho st", + "new sp", + "w ers", + "vi p", + "li ked", + "id ent", + "volunte er", + "ad ult", + "pu pp", + "cir cle", + "mat erial", + "degre e", + "gro wn", + "boo m", + "calend ar", + "su r", + "vie wing", + "ath letes", + "ch and", + "re ll", + "asi an", + "en tr", + "vol ley", + "victi ms", + "bo dy", + "m ama", + "trans fer", + "ge ek", + "in dic", + "sav ed", + "ma i", + "g ent", + "it s", + "loun ge", + "k ol", + "the ory", + "situ ation", + "is lands", + "ar th", + "z oo", + "floo d", + "vi ously", + "show ed", + "parliam ent", + "ch ev", + "el ine", + "at trac", + "ab ad", + "ta il", + "h rs", + "lu s", + "por tu", + "gor y", + "provi des", + "to ys", + "de ath", + "in fe", + "an ce", + "g le", + "li am", + "lo ver", + "hu d", + "dv d", + "reve aled", + "g w", + "re ment", + "ca the", + "l ying", + "ra dio", + "der by", + "stor s", + "che mi", + "hosp it", + "âľ ¨", + "' :", + "ilo ve", + "le mon", + "re public", + "s ni", + "ne ss", + "do or", + "re action", + "pre gn", + "fla v", + "schol ar", + "spo tify", + "is ation", + "vis ual", + "aw are", + "spon sored", + "jo ke", + "less ons", + "leg is", + "lo ck", + "si mil", + "ðŁĺ ĭ", + "kin d", + "la y", + "ma h", + "ho ping", + "vancou ver", + "as er", + "clean ing", + "gal a", + "thre at", + "la p", + "ach e", + "ro mance", + "ex pen", + "re post", + "z am", + "e pi", + "mir ror", + "o ak", + "ad ul", + "bat man", + "s lu", + "l c", + "vie wed", + "re views", + "d ates", + "indone sia", + "acti vi", + "off en", + "lea f", + "i si", + "ag ricul", + "costu me", + "s ites", + "spir itu", + "appear ance", + "ir y", + "st air", + "applic ation", + "spec tac", + "ic ity", + "ski es", + "hand le", + "pun k", + "paradi se", + "t n", + "de al", + "provi ding", + "do c", + "recei ving", + "bre w", + "micro soft", + "à ¶", + "fer r", + "me tro", + "th ail", + "y um", + "car ter", + "à ¡", + "gent le", + "bre aks", + "coo per", + "show case", + "cu tting", + "egy pt", + "bab y", + "semin ar", + "gl ori", + "ss on", + "fa ve", + "re hear", + "lo tte", + "la dy", + "al as", + "pre p", + "deli vered", + "nu clear", + "ir o", + "engag ement", + "at ta", + "con ven", + "z an", + "gl ory", + "hol ds", + "busine sses", + "str ange", + "sch e", + "it self", + "gra d", + "mar kets", + "f alling", + "st ats", + "ge on", + "bu dd", + "li s", + "she et", + "thi si", + "co lo", + "deser t", + "regi stration", + "ig n", + "expla in", + "inter ior", + "la ws", + "writ ers", + "spr ings", + "k r", + "fri ed", + "blo om", + "inf ra", + "a o", + "cre d", + "pa st", + "line up", + "bo o", + "bre a", + "boo ts", + "celebr ity", + "att acks", + "bro ok", + "ev es", + "ex cu", + "cher ry", + "oo p", + "fas cin", + "boy friend", + "se as", + "n ine", + "effec ts", + "po wered", + "k ha", + "ðŁĺ Ģ", + "sh out", + "con dition", + "i j", + "her o", + "enter pri", + "win ter", + "applic ations", + "sho e", + "g el", + "batt le", + "pro grams", + "w art", + "ðŁĴ ¥", + "ra p", + "ho l", + "dang erous", + "di a", + "coun ter", + "ric s", + "i or", + "k night", + "co at", + "emo tional", + "at ures", + "d as", + "whe el", + "fore cast", + "tran sport", + "glasgo w", + "king dom", + "prepar ing", + "im medi", + "ff in", + "awar ded", + "prin ting", + "ro man", + "fight ers", + "any more", + "bel t", + "p ine", + "win e", + "x i", + "employe es", + "logi es", + "al led", + "de mo", + "birth day", + "ange les", + "lo g", + "dri vers", + "neck lace", + "k ath", + "s it", + "athle te", + "ef s", + "s burg", + "pur pose", + "resi stance", + "rele ases", + "t is", + "vari ous", + "deli ver", + "ch al", + "s anc", + "opp o", + "cra w", + "neu ro", + "dr a", + "suppor ters", + "sna p", + "diffic ult", + "swe ar", + "logi st", + "pa th", + "attemp t", + "à ¥", + "swim ming", + "ste ve", + "hur t", + "inclu ded", + "b ap", + "wa re", + "ðŁĴ ĭ", + "end ers", + "ja ke", + "le eds", + "cli mb", + "l b", + "im ple", + "li sa", + "clo thing", + "ðŁĺ İ", + "d t", + "com pla", + "sw ing", + "stra w", + "v als", + "k le", + "us ers", + "stor m", + "cu ts", + "ontari o", + "p an", + "hand some", + "i ow", + "ar gu", + "chec king", + "scotti sh", + "Ķ ï¸ı", + "si er", + "em ma", + "po d", + "patter n", + "de sh", + "en h", + "ed ward", + "t ing", + "k h", + "hal f", + "lincol n", + "mo ther", + "al leg", + "r c", + "volley ball", + "d n", + "g ay", + "all y", + "le ton", + "gro ve", + "l oud", + "adv anced", + "re spec", + "cli ent", + "supre me", + "thail and", + "ho w", + "gi g", + "to i", + "do t", + "dol lar", + "ðŁij ĩ", + "p it", + "r b", + "h n", + "produc ed", + "gg ers", + "âĨ Ĵ", + "ml b", + "can vas", + "fin eart", + "us d", + "in the", + "p son", + "actu al", + "s l", + "t b", + "ip ad", + "en sure", + "u mb", + "w d", + "sk a", + "mar s", + "k end", + "f eli", + "th ing", + "count down", + "absolu te", + "r out", + "dra l", + "p y", + "inju red", + "min t", + "hun ting", + "mm er", + "s age", + "li gh", + "ac ity", + "ex pan", + "mur ray", + "ar o", + "sec ure", + "four th", + "eag le", + "reli ef", + "st akes", + "industri al", + "clar k", + "under standing", + "see m", + "pl enty", + "sil ver", + "cla u", + "thre at", + "sa il", + "pro duce", + "ab str", + "is is", + "b r", + "eng ers", + "wor ry", + "bie ber", + "s j", + "just in", + "reali ze", + "ky le", + "esp n", + "fil ter", + "s ch", + "ty pes", + "game dev", + "d ing", + "twit ter", + "soldi ers", + "p om", + "car bon", + "y ards", + "child hood", + "ri ed", + "ke l", + "ele ph", + "t ons", + "key note", + "qui et", + "wi re", + "po sting", + "is sa", + "repre senting", + "bac ks", + "alex ander", + "celebr ates", + "ta ining", + "| |", + "ch or", + "esc ape", + "pe ek", + "ti ves", + "fiel d", + "ssi e", + "im pac", + "spons or", + "r c", + "we dd", + "cann ab", + "si des", + "trac ks", + "com par", + "con trac", + "techn ical", + "bi ble", + "expl oring", + "sh are", + "tra v", + "n ate", + "ill o", + "sc ru", + "m ingham", + "gun s", + "of the", + "sh ame", + "se es", + "ca tho", + "ac cess", + "ce l", + "repor ted", + " »", + "mari o", + "p ad", + "hope fully", + "ou se", + "y on", + "disapp o", + "ol o", + "p itt", + "pa c", + "ga p", + "cru sh", + "s g", + "k le", + "ge m", + "emp ire", + "dir ty", + "a is", + "avi ation", + "ze aland", + "fac ing", + "high way", + "d anny", + "spi der", + "ot ta", + "ðŁĺ Ħ", + "w y", + "col ours", + "in fl", + "co sts", + "olym pics", + "au s", + "h m", + "ho ward", + "pas ses", + "lau ren", + "mu sh", + "op in", + "r ho", + "disc ount", + "oper ation", + "em ily", + "mm m", + "cham ber", + "d il", + "to yo", + "shi p", + "sam u", + "pic tured", + "un ic", + "po l", + "keep er", + "carto on", + "st en", + "ig nor", + "n ations", + "n l", + "ta sting", + "deta il", + "offici als", + "mo tor", + "franc is", + "ed itor", + "ðŁij ĩ", + "pe ts", + "rang ers", + "t g", + "r n", + "w ri", + "nic hol", + "i se", + "spo ts", + "ani e", + "chec k", + "tri ple", + "ku mar", + "spe akers", + "ic ing", + "pre pared", + "ab use", + "friend ship", + "mon th", + "swi m", + "air e", + "sc ent", + "hamil ton", + "indi an", + "j es", + "yum my", + "te ars", + "da wn", + "i zed", + "worl ds", + "ðŁ ķ", + "b illi", + "st one", + "n hs", + "ba sic", + "p or", + "st le", + "ir on", + "ol der", + "cle vel", + "e ing", + "ðŁĺįðŁĺį ðŁĺį", + "prin ts", + "fir m", + "air craft", + "fin est", + "devel op", + "aar on", + "t z", + "gra ham", + "own ers", + "fo li", + "less on", + "qu es", + "bab e", + "cra ft", + "ph en", + "ju n", + "bir mingham", + "v ine", + "ll er", + "i an", + "fineart america", + "evol u", + "st ab", + "im per", + "war d", + "com ic", + "wi z", + "inv ited", + "du ke", + "mat ch", + "por ts", + "ro ger", + "diag no", + "ke pt", + "te st", + "vis u", + "r hy", + "so c", + "to x", + "b aker", + "sur face", + "co vers", + "man s", + "b its", + "x box", + "ff le", + "n an", + "gar d", + "h art", + "wat ers", + "v illa", + "re tro", + "light ning", + "catho lic", + "democr acy", + "neigh bor", + "pen n", + "cr an", + "jona than", + "la ura", + "vi bes", + "su b", + "coach ing", + "clear ly", + "uk raine", + "bra ve", + "commit ment", + "t all", + "mar t", + "ra p", + "mo di", + "sco tt", + "bro s", + "show er", + "ðŁı ¾", + "âĺº ï¸ı", + "cou sin", + "appro ach", + "br e", + "com pos", + "hil ari", + "phil ly", + "g ad", + "quick ly", + "ri an", + "t m", + "vir tual", + "hou ses", + "k t", + "phoeni x", + "w ire", + "ff y", + "b unch", + "anc ing", + "tal e", + "snap chat", + "star ter", + "h t", + "k icking", + "ap art", + "th y", + ") !", + "blo gger", + "it z", + "com fort", + "ang els", + "w ash", + "\" :", + "ar gent", + "re quest", + "hon est", + "mi ghty", + "bo bby", + "k g", + "ro l", + "thou se", + "ex po", + "h c", + "tab les", + "mag ical", + "po sts", + "de m", + "n w", + "or lando", + "ab er", + "* **", + "ðŁĺ ľ", + "environ mental", + "trans formation", + "mi le", + "w ic", + "hir ing", + "ma ine", + "bo ar", + "r ying", + "ti s", + "nit ure", + "twee ted", + "anton io", + "opin ion", + "fin ale", + "di y", + "f is", + "th in", + "trou ble", + "le go", + "fi les", + "qu art", + "sp a", + "curren cy", + "cli mate", + "fan art", + "rail way", + "sp ace", + "ban ds", + "dani el", + "mo tion", + "l eng", + "hol der", + "oc cu", + "mar ie", + "cathe dral", + "bu zz", + "bi es", + "nas car", + "bm w", + "bat tery", + "char lotte", + "doc tor", + "zz le", + "se ven", + "in san", + "d dy", + "st en", + "lab or", + "thr illed", + "se ren", + "docu mentary", + "wav es", + "cer tain", + "can did", + "allow ed", + "ninten do", + "star wars", + "ta p", + "home made", + "d les", + "ther ing", + "bre e", + "emp ty", + "pi ano", + "pos iti", + "coun try", + "por k", + "pu ts", + "per ry", + "m atic", + "spot light", + "ti st", + "or ities", + "we alth", + "c p", + "bar bar", + "commit ted", + "as sau", + "pro fit", + "e ight", + "hu l", + "fini shing", + "run ner", + "ss o", + "insp ec", + "char ged", + "christ op", + "lo sing", + "co al", + "ho o", + "ele v", + "de le", + "mo ham", + "don ation", + "c able", + "clin ic", + "j in", + "manag ed", + "ter ing", + "â ¬", + "ur ban", + "depu ty", + "bb er", + "bur n", + "acade mic", + "o tt", + "sta ke", + "it er", + "sto wn", + "ack er", + "advent ures", + "ad ams", + "gre g", + "pro m", + "vo l", + "ac qu", + "con gre", + "pa int", + "citiz ens", + "c all", + "af ford", + "v c", + "as ks", + "the tic", + "independ ence", + "â Ľ", + "h itting", + "bl on", + "fu ture", + "â ı", + "in no", + "gen e", + "bo ards", + "di stance", + "se t", + "re mem", + "th al", + "pre vent", + "l ang", + "ob jec", + "su sp", + "mat t", + "in duc", + "bor o", + "pi one", + "re di", + "vir tu", + "prin ted", + "sco pe", + "shar k", + "suc ce", + "a stron", + "il legal", + "j ag", + "c ting", + "ine e", + "at o", + "rob in", + "nutr ition", + "b f", + "du tch", + "b n", + "fur niture", + "for gotten", + "at ar", + "ru p", + "hy per", + "bran ch", + "communic ation", + "degre es", + "on ia", + "un cle", + "promo te", + "or che", + "wi i", + "j s", + "but ton", + "ma jor", + "c bs", + "bri stol", + "premi um", + "ordin ary", + "e dit", + "m g", + "we ed", + "st even", + ": '", + "gu s", + "te s", + "cap tured", + "dru gs", + "do w", + "wr ites", + "bi shop", + "whe els", + "ali zation", + "disco very", + "w r", + "rach el", + "ne il", + "hy dr", + "cu test", + "entreprene ur", + "kore an", + "ore gon", + "ul ty", + "perfec tly", + "suppor ted", + "histor ical", + "t wins", + "ell y", + "we l", + "de vil", + "in come", + "scienti sts", + "de leg", + "h en", + "on i", + "ic ed", + "gi o", + "cur ry", + "reve al", + "e g", + "buff alo", + "n ol", + "op era", + "camer on", + "haha haha", + "j ab", + "gradu ation", + "cra ig", + "r al", + "i f", + "organi zation", + "le ge", + "g ang", + "su d", + "edin burgh", + "l ack", + "fli es", + "g ate", + "thr ones", + "q b", + "the real", + "e leg", + "pp in", + "c les", + "jam ie", + "tn am", + "cryp to", + "ou l", + "p ages", + "a se", + "roo ts", + "stu pid", + "a did", + "boo t", + "prote in", + "s ap", + "si um", + "su s", + "end or", + "fun ction", + "don t", + "en na", + "ch y", + "squ e", + "wor ker", + "m tv", + "e a", + "k an", + "ðŁĴ ļ", + "mu s", + "professi on", + "t to", + "oper ations", + "al lo", + "c tor", + "inv ite", + "sc and", + "ou th", + "z im", + "lin ks", + "cli ents", + "sam sung", + "discu sses", + "n ell", + "ul tra", + "some where", + "ste wart", + "ine t", + "de z", + "b out", + "fac tor", + "ti an", + "tr ans", + "jere my", + "d b", + "ðŁĩ ¬", + "or n", + "develop ing", + "spo l", + "coo per", + "ma u", + "rememb ering", + "tre k", + "famil y", + "sen iors", + "fo ster", + "att ended", + "w ing", + "trans form", + "ele mentary", + "hor iz", + "li sting", + "malay sia", + "it ch", + "warri or", + "philipp ines", + "russ ell", + "m end", + "initi ative", + "cre ep", + "to ps", + "br iti", + "a ur", + "shar p", + "adverti sing", + "ug ly", + "achi ev", + "materi als", + "bu g", + "dev ice", + "bon us", + "fac ility", + "col e", + "nh l", + "y as", + "plann ed", + "pol e", + "excell ence", + "tr ick", + "con fl", + "r p", + "achi eve", + "lo an", + "swa g", + "jess ica", + "ho we", + "p our", + "sc u", + "z oo", + "r ated", + "dre sses", + "re bel", + "mex ican", + "co ordin", + "me ss", + "atlan tic", + "t l", + "osc ar", + "wal ks", + "phar mac", + "investig ation", + "... #", + "cc i", + "eas ily", + "monday motivation", + "y ment", + "au ti", + "for ced", + "ar med", + "colle agues", + "pap ers", + "pro per", + "sha ke", + "bu c", + "le an", + "exhi bit", + "e vement", + "co tt", + "bi z", + "sp er", + "k ent", + "sw an", + "/ @", + "girl friend", + "haw k", + "âĺ Ģï¸ı", + "mon o", + "ðŁĴ Ľ", + "stat ue", + "ðŁĺ ³", + "ra s", + "te eth", + "preci ous", + "t ile", + "p am", + "swi ft", + "v ali", + "no se", + "dr unk", + "experi ences", + "come back", + "gen ius", + "wor se", + "sh ef", + "ra d", + "ed it", + "hon our", + "au spol", + "lar ry", + "h ire", + "gor don", + "achi evement", + ".... ....", + "su icide", + "alter native", + "su p", + "sur roun", + "sha ke", + "ke ith", + "pe pper", + "tur k", + "crimin al", + "be ck", + "su m", + "w alls", + "cn n", + "an tic", + "of fe", + "col li", + "win es", + "high light", + "hawa ii", + "emb ar", + "l fc", + "ðŁĩ ®", + "m v", + "> >", + "at mo", + "wor d", + "car l", + "shout out", + "bre wing", + "ì Ŀ", + "do f", + "s ic", + "hot test", + "col on", + "hh h", + "shu t", + "low ing", + "volu me", + "apart ment", + "agre ement", + "de stro", + "we e", + "religi ous", + "iow a", + "ro d", + "land ing", + "re present", + "ðŁĵ· :", + "la s", + "usu ally", + "h l", + "c ac", + "sal v", + "al ong", + "laugh ing", + "be ans", + "remin ds", + "pha se", + "some body", + "ma sk", + "ran ked", + "dest roy", + "sc i", + "âĢ¼ ï¸ı", + "gab ri", + "le o", + "ro a", + "fa iled", + "si l", + "refuge es", + "re vi", + "r ing", + "ber ries", + "coo kies", + "y y", + "conserv ation", + "sh ab", + "human s", + "de termin", + "a in", + "ni all", + "as su", + "mb a", + "fro m", + "extre me", + "vic es", + "commer ce", + "ght ful", + "or dered", + "suppor ts", + "re cap", + "v or", + "dro pping", + "correc t", + "pay ing", + "mean ing", + "n j", + "qui z", + "\" #", + "busine ss", + "ðŁĩ® ðŁĩ", + "indi gen", + "du st", + "box es", + "bl ind", + "x xx", + "zz y", + "ðŁĩ¬ ðŁĩ", + "ss els", + "s ant", + "dd le", + "hilari ous", + "desig n", + "wonder ing", + "vehic les", + "k re", + "ju d", + "rece ption", + "par ker", + "à Ń", + "pri vi", + "hy dro", + "sof tball", + "pol lu", + "lo cked", + "ba h", + "e ar", + "scri pt", + "di vi", + "br ace", + "geor ge", + "the ast", + "bel o", + "j al", + "tion ary", + "dent al", + "roc ket", + "pur ch", + "sh ak", + "manufac turing", + "e z", + "it is", + "con cep", + "tb all", + "ch s", + "direc ted", + "pra yers", + "oo k", + "phil os", + "vari ety", + "che ss", + "ser ver", + "g and", + "bal ti", + "ðŁĵ ¸", + "sel y", + "cru z", + "spectac ular", + "bur ning", + "re present", + "i z", + "t one", + "mer ce", + "h ell", + "bed room", + "estab li", + "bo l", + "com mon", + "ãĥ »", + "ab or", + "kit ty", + "hei ghts", + "re pair", + "willi am", + "qu ake", + "alab ama", + "popul ation", + "re v", + "re tt", + "i sts", + "n ite", + "le m", + "a ha", + "clevel and", + "r m", + "po ver", + "ob se", + "mon tre", + "man ia", + " ®", + "con ne", + "car ni", + "sh ah", + "f y", + "u a", + "sc or", + "strugg le", + "bo b", + "' '", + "appro pri", + "deci de", + "ff ed", + "ca ster", + "s ort", + "hun gry", + "dra g", + "ا Ù", + "gr ounds", + "d w", + "sli ghtly", + "car din", + "dead line", + "bron ze", + "web in", + "bar ry", + "sil ence", + "e uro", + "op tion", + "ear n", + "ðŁĴ ĸ", + "howe ver", + "na ren", + "na ils", + "bath room", + "v ine", + "ph d", + "min ing", + "gar age", + "( )", + "shou lder", + "defe at", + "di r", + "o v", + "liber ty", + "ple as", + "x on", + "com pre", + "a v", + "j in", + "ab les", + "sil ent", + "fam ili", + "vis its", + "di pl", + "ha bit", + "milli ons", + "regar ding", + "innov ative", + "sen ator", + "r ts", + "v on", + "k l", + "wh il", + "requi red", + "âĿ Ħ", + "lu v", + "presi dential", + "po cket", + "hun dre", + "sho wn", + "fro zen", + "to ward", + "fa st", + "confi dence", + "r ough", + "indivi dual", + "qu et", + "ðŁı ½", + "dom e", + "fi fa", + "engine er", + "z en", + "re mix", + "ðŁĺ ĥ", + "pl ant", + "min or", + "robin son", + "as y", + "pul led", + "cer tain", + "potat o", + "( :", + "pre s", + "oc ca", + "w it", + "it em", + "si e", + "d ating", + "thom pson", + "own ed", + "an u", + "vi e", + "te dly", + "good night", + "ex cept", + "ðŁĮ Ł", + "ira q", + "ki e", + "ren ces", + "li p", + "simil ar", + "sau di", + "vi g", + "arth ur", + "pic ks", + "mil an", + "hon da", + "ma xi", + "o g", + "ste st", + "ar ch", + "analy tics", + "ba sti", + "pear l", + "ter ry", + "hor se", + "ast ro", + "ac ce", + "laun ching", + "inter national", + "s no", + "ta sty", + "den ver", + "ir l", + "pe te", + "tor n", + "advant age", + "var sity", + "\" \"", + "sol e", + "g c", + "lan g", + "demon str", + "ol ds", + "un ity", + "ne ts", + "insp ire", + "cre te", + "nash ville", + "nel son", + "e ter", + "wal k", + "hy un", + "m ack", + "tre as", + "see king", + "ra ge", + "bru sh", + "ab and", + "whil st", + "co con", + "h ong", + "shel ter", + "i p", + "possi bly", + "so o", + "it ed", + "â Ħ", + "rac es", + "war ming", + "qu in", + "tele vision", + "mat ches", + "ra pi", + "ment al", + "pal m", + "jenni fer", + "rol ls", + "indi ana", + "b ars", + "cat ching", + "resc u", + "candid ates", + "fa re", + "âł Ģ", + "se o", + "vie tnam", + "alph a", + "michel le", + "visi ble", + "re gre", + "wn ed", + "app le", + "li p", + "f fe", + "li z", + "york shire", + "ha il", + "se asons", + "be gan", + "m d", + "k c", + "la p", + "fascin ating", + "hel p", + "ur y", + "u ms", + "nu ts", + "se m", + "along side", + "bri dge", + "ori al", + "o ve", + "world cup", + "briti sh", + "comfor table", + "i ve", + "hot els", + "fair s", + "hor ri", + "so x", + "d ining", + "stre am", + "bar ri", + "ss y", + "w im", + "ter ms", + "v u", + "pe re", + "l ens", + "wal ked", + "r or", + "l ars", + "shi eld", + "dou bt", + "pro to", + "cro ssing", + "me ant", + "medi um", + "ad ding", + "e b", + "che ap", + "fun c", + "pap er", + "bran ds", + "ry an", + "feed back", + "col lins", + "un known", + "tro pical", + "sand wich", + "fal len", + "for mu", + "selec t", + "lo ads", + "answ ers", + "or i", + "mag a", + "d or", + "du o", + "ali e", + "dru m", + "ur i", + "de er", + "sou l", + "sh ut", + "âĺ º", + "sto len", + "don ated", + "bu zz", + "patri ots", + "ha l", + "na sty", + "nomin ated", + "mon te", + "ki a", + "th ri", + "ing u", + "te sts", + "pe tro", + "ðŁij ij", + "ho sts", + "ne st", + "to pic", + "pat ch", + "m my", + "hu gh", + "ab ilities", + "ma the", + "s miles", + "g b", + "ag enda", + "insi ghts", + "chi p", + "ph an", + "fail ure", + "dg ers", + "ha i", + "signific ant", + "sho ck", + "ru ral", + "gl am", + "figu res", + "pot us", + "o ta", + "mini stry", + "appe ars", + "fe ar", + "r h", + "americ an", + "h att", + "son y", + "fi res", + "e di", + "n ou", + "e qui", + "wh en", + "univers al", + "mad ness", + "i x", + "sculp ture", + "b ach", + "t to", + "swe den", + "et a", + "en to", + "develop ed", + "month ly", + "ma ps", + "ra h", + "le d", + "del ta", + "sa ints", + "is lam", + "ben ch", + "fif th", + "v ard", + "so cks", + "wel coming", + "j e", + "tur ner", + "v b", + "ad i", + "nor way", + "ad y", + "hurric ane", + "por sche", + "tra dition", + "ex am", + "newsp aper", + "lu ci", + "a ver", + "ide al", + "d na", + "madi son", + "ðŁ §", + "wit ness", + "ac ou", + "insi ght", + "si mon", + "robo t", + "sna ke", + "n bc", + "ac o", + "ro ss", + "sh ment", + "religi on", + "ch ann", + "in su", + "camp bell", + "inst alled", + "we ather", + "hor ses", + "ol i", + "rober t", + "k az", + "ðŁı Ģ", + "veter an", + "th read", + "quar ter", + "ea sier", + "cap ture", + "hi pho", + "law rence", + "roman tic", + "pas sion", + "cl ay", + "ox ford", + "th ai", + "stu dying", + "fi a", + "elec ted", + "most ly", + "c b", + "tu mb", + "âĢįâĻ Ĥ", + "x l", + "sh an", + "fa ster", + "ev ans", + "sli de", + "sh ri", + "see k", + "mi es", + "chemi stry", + "pump kin", + "tu m", + ", ,", + "ro om", + "fi red", + "li ps", + "pres ence", + "af f", + "brew ery", + "arri ve", + "sw ag", + "photo graph", + "pen gu", + "chi ps", + "at tor", + "val ues", + "accur ate", + "con temporary", + "princi pal", + "cannab is", + "ari o", + "any where", + "gi a", + "democr ats", + "buil dings", + "li ved", + "ap s", + "neg ative", + "m are", + "bal lo", + "li on", + "diam on", + "loo k", + "re form", + "tom my", + "il la", + "tre ats", + "hundre ds", + "port land", + "wor thy", + "ex cep", + "ar ia", + "ido l", + "be er", + "cd n", + "y u", + "aw k", + "ðŁĩ ¨", + "c ells", + "à ³", + "ident ity", + "dra wn", + "de vil", + "f inger", + "th am", + "ðŁij Ĭ", + "ear ned", + "fin tech", + "dol ph", + "twee ting", + "evolu tion", + "ðŁĵ į", + "est im", + "m vp", + "n one", + "ðŁĩºðŁĩ ¸", + "toyo ta", + "au x", + "mar in", + "b old", + "l bs", + "ste ak", + "mur phy", + "it able", + "lou is", + "sol ve", + "pi a", + "sk ir", + "ill ino", + "webin ar", + "ban ana", + "lo v", + "th on", + "vo ters", + "afford able", + "defe ated", + "lm fa", + "air lines", + "super b", + "any way", + "deb t", + "bo red", + "ver si", + "me tal", + "responsi ble", + "m k", + "s se", + "f ay", + "cau sed", + "f p", + "recomm end", + "pla za", + "spor ting", + "alli ance", + "au stri", + "n n", + "t ours", + "surpri sed", + "arti f", + "th under", + "sur ve", + "wor e", + "bri ef", + "necess ary", + "z ie", + "ash ley", + "dra ke", + "r t", + "kni fe", + "im mun", + "char ges", + "a the", + "bri de", + "rep ly", + "g av", + "broad cast", + "pu er", + "brace let", + "cap acity", + "harve st", + "id k", + "perfor man", + "d ding", + "il ers", + "par a", + "jam a", + "pro vince", + "ch in", + "id ers", + "har i", + "te aser", + "ch en", + "re stor", + "r at", + "fl at", + "col om", + "ðŁĴ ŀ", + "ðŁĩ¨ ðŁĩ", + "smoo th", + "r t", + "p itch", + "stay ing", + "isra eli", + "t cot", + "per spective", + "do ck", + "open er", + "lo vel", + "x o", + "class room", + "l ington", + "go al", + "kenne dy", + "sh am", + "sp aces", + "mitch ell", + "home coming", + "uk i", + "claim ed", + "recru it", + "ing o", + "mu fc", + "mon it", + "g roo", + "resi dent", + "per cent", + "per man", + "otta wa", + "int ment", + "an xi", + "stand ards", + "wor ship", + "sche me", + "f x", + "pot ter", + "bi an", + "athle tic", + "af gh", + "s se", + "sat ell", + "par ties", + "âĿ¤ âĿ¤", + "infra structure", + "rela x", + "mo du", + "wor n", + "smo king", + "y ach", + "practic es", + "wc w", + "am b", + "dome stic", + "tay lor", + "k entu", + "provi ded", + "mo di", + "ve g", + "\" ...", + "ob serv", + "ðŁĺ ©", + "be ard", + "m our", + "an gry", + "ðŁĺ ±", + "startu ps", + "woo den", + "di ve", + "na il", + "anti que", + "ro ses", + "torn ado", + "m at", + "^ ^", + "su spect", + "far m", + "de vices", + "me ga", + "tu l", + "scholar ship", + "ge e", + "disa ster", + "arri val", + "po in", + "mar c", + "kati e", + "bb ed", + "fal se", + "deser ves", + "ric hard", + "ju ana", + "fre y", + "tion ed", + "hy bri", + "r w", + "sar ah", + "ach i", + "c ure", + "o le", + "mor ris", + "ch ic", + "broad way", + "la bel", + "pa k", + "pover ty", + "gol f", + "e red", + "f u", + "er ies", + "be es", + "alo gue", + "st el", + "wire less", + "je wish", + "ti de", + "blo cked", + "life time", + "b har", + "sp lit", + "am ster", + "th i", + "jo shu", + "br unch", + "ha ps", + "s for", + "oo ps", + "ka poor", + "hi king", + "suppo sed", + "ro of", + "re as", + "tra in", + "ti ght", + "tru mp", + "bas ically", + "r r", + "ea red", + "see ds", + "entr ance", + "c p", + "wi e", + "son ic", + "vic tim", + "he re", + "e h", + "ear rings", + "sal mon", + "arc tic", + "an ne", + "dou gla", + "corru ption", + "hann ah", + "ha sn", + "vo ices", + "con ce", + "att a", + "fle et", + "clin ical", + "democr atic", + "ton y", + "st ood", + "le f", + "twit ch", + "a il", + "honest ly", + "incre ased", + "dro me", + "don na", + "accep ted", + "visit ors", + "ap ar", + "ad or", + "p ar", + "jer ry", + "ra i", + "brand on", + "ab u", + "!! !!!!", + "me me", + "in gh", + "glori ous", + "b hu", + "pu mp", + "j ol", + "li ke", + "fi sher", + "ma z", + "ag an", + "destin ation", + "play list", + "le tters", + "gen u", + "br ace", + "celebr ated", + "bann er", + "r he", + "dra gon", + "ðŁĺ ħ", + "sig nature", + "gre y", + "âľ Ķï¸ı", + "al ice", + "be red", + "ph er", + "ber n", + "ca th", + "ga thering", + "sc oring", + "influ ence", + "sm iling", + "de pt", + "lo cal", + "a x", + "ac u", + "reti rement", + "hon or", + "her self", + "chem ical", + "asse ss", + "y all", + "fre qu", + "appreci ation", + "ac a", + "cho ir", + "cu z", + "so il", + "c il", + "repor ting", + "u h", + "enterpri se", + "gr at", + "jaco b", + "ru m", + "fe e", + "j ak", + "sp in", + "bi kes", + "phi a", + "ste re", + "p is", + "bloo d", + "t att", + "ra ft", + "war ren", + "sh eri", + "back stage", + "mar sh", + "hash tag", + "ther ine", + "re in", + "game day", + "guar an", + "reci pes", + "min ds", + "stron ger", + "issu ed", + "bic y", + "n ak", + "ment ed", + "sc ary", + "u x", + "pre vious", + "tt le", + "th ats", + "ac tors", + "u ma", + "tin a", + "bun ny", + "promo tion", + "u ss", + "oli ver", + "montre al", + "what s", + "appreci ated", + "la kes", + "excu se", + "kno wing", + "pri zes", + "musc le", + "shad es", + "sco t", + "ing redi", + "electr onic", + "ju an", + "comb at", + "s ri", + "e h", + "turk ish", + "l om", + "stri kes", + "pri son", + "re e", + "po pe", + "vi d", + "ol dest", + "dol l", + "sw iss", + "certi fied", + "cli p", + "re turning", + "lat or", + "le igh", + "tt es", + "wat son", + "heal ing", + "el im", + "per haps", + "ha ss", + "k au", + "d der", + "mou se", + "new castle", + "indigen ous", + "wel comes", + "co le", + "tau ght", + "no ise", + "appe ar", + "jo e", + "can on", + "wedne sday", + "u tah", + "c tive", + "dri ven", + "i v", + "c ell", + "stri p", + "ac c", + "focu sed", + "ar rest", + "sto cks", + "wo o", + "â Ĺ", + "notic ed", + "shad o", + "di spla", + "ter ror", + "bor ne", + "secon d", + "que ens", + "wo ke", + "ja il", + "no tt", + "cam bridge", + "har t", + "se af", + "fa x", + "ac cept", + "âĺ ħ", + "goo ds", + "k at", + "t win", + "h s", + "thou sand", + "s ins", + "su ite", + "amp ton", + "ar n", + "rele v", + "ric har", + "hoo ps", + "n bc", + "class ic", + "p ab", + "soldi er", + "de plo", + "le ans", + "install ation", + "cla sh", + "le ban", + "ee e", + "ti re", + "belo ved", + "fu sion", + "travel ing", + "ne i", + "coo kie", + "glo be", + "phys ics", + "s q", + "co l", + "wol ves", + "d l", + "ex it", + "\" -", + "foo tball", + "le af", + "ster ling", + "hi de", + "minne so", + "fresh man", + "natu re", + "indi e", + "supp lies", + "bri s", + "iri sh", + "ink tober", + "doo dle", + "ic op", + "mess ages", + "adul ts", + "recor ded", + "fix ed", + "ar do", + "offe red", + "under ground", + "dr one", + "p ine", + "ma inten", + "and re", + "ham mer", + "s x", + "r ound", + "hi ke", + "bra d", + "ro me", + "fu ll", + "on ey", + "ro ws", + "colum bia", + "archi ves", + "appro ved", + "bat ch", + "illino is", + "recogn ition", + "shou ldn", + "fo g", + "nca a", + "ke vin", + "human ity", + "al though", + "pow ers", + "p ou", + "s ar", + "pe st", + "alco hol", + "con sci", + "phil adel", + "en o", + "t m", + "ok la", + "cate gory", + "particip ate", + "accu sed", + "bri ef", + "po em", + "clu bs", + "consul t", + "ja b", + "big data", + "amster dam", + "ac ing", + "certi fic", + "n u", + "d at", + "impro ved", + "and y", + "campa ig", + "pale stin", + "p ace", + "mo bi", + "feel ings", + "wol f", + "bra in", + "pro pos", + "inter active", + "prin ce", + "inde x", + "c is", + "cha e", + "peace ful", + "co vering", + "ac o", + "cour ses", + "mon key", + "re place", + "b l", + "bloo dy", + "tal es", + "brigh ton", + "neighbor hood", + "g ates", + "spiritu al", + "af raid", + "bre ast", + "b ones", + "ðŁij ī", + "vide o", + "w au", + "tou ch", + "inju ries", + "car l", + "ri x", + "une x", + "âĢ ¢", + "fre d", + "consi dered", + "thu si", + "an ch", + "on y", + "u sa", + "graph ics", + "ac re", + "ðŁĺ ©", + "com memor", + "com mod", + "go ti", + "guar dian", + "star bucks", + "pre vention", + "haha haha", + "admini stration", + "portu gal", + "fac ulty", + "bet a", + "ul a", + "al bert", + "bre ath", + "er i", + "le tting", + "tr ic", + "ment ation", + "incredi bly", + "ten nes", + "v d", + "ðŁĻ Ī", + "ed die", + "br ick", + "gr ill", + "bt w", + "wat ches", + "resear chers", + "t ney", + "ni e", + "p as", + "a ster", + "vi br", + "poke mon", + "ch rome", + "go at", + "pitt s", + "il ly", + "festi ve", + "y d", + "can al", + "ðŁ Ĩ", + "fi es", + "car los", + "re que", + "partic i", + "tra ins", + "sam ple", + "temper ature", + "sym ph", + "pic king", + "in door", + "z ers", + "playo ffs", + "____ ____", + "ap es", + "ly rics", + "islam ic", + "performan ces", + "d ick", + "spar k", + "se as", + "hom a", + "gr ound", + "disc i", + "employe e", + "com mu", + "alas ka", + "al an", + "fe ast", + "dg ing", + "ban king", + "manu el", + "slow ly", + "tru cks", + "mc car", + "oo o", + "sc rat", + "orche stra", + "indivi du", + "m x", + "bre ath", + "stair s", + "equ ality", + "bla ke", + "loc ations", + "cocon ut", + "balti more", + "aa a", + "l c", + "ðŁı Ĩ", + "har vey", + "resi st", + "immigr ation", + "adid as", + "fil i", + "re f", + "lg bt", + "mo s", + "pp i", + "ken ny", + "terr or", + "ban e", + "apol is", + "s g", + "social media", + "ka i", + "hon est", + "as sas", + "bol lywood", + "âĢįâĻ Ģï¸ı", + "ferr ari", + "hor n", + "cryp to", + "bo om", + "mainten ance", + "i di", + "s man", + "w l", + "ext ended", + "in sul", + "ve s", + "go sp", + "tr i", + "pi g", + "tar ge", + "cel er", + "st ati", + "sm h", + "ri dic", + "appe al", + "? )", + "con clu", + "cos me", + "she ep", + "christop her", + "en thusi", + "po lish", + "me ts", + "oun ded", + "sustain ability", + "creati vity", + "con crete", + "ra i", + "ali en", + "ble ss", + "te es", + "clu b", + "ro t", + "bo s", + "ex ist", + "perfe ction", + "lu ck", + "rock y", + "expen sive", + "mean while", + "happy birthday", + "pre t", + "thr iller", + "ca ve", + "playo ff", + "som er", + "l u", + "le x", + "def ence", + "am writing", + "home less", + "pro phe", + "ch et", + "past or", + "ðŁ¤ £", + "land er", + "ww w", + "Ģ ï¸ı", + "tic a", + "! #", + "o tic", + "rad ar", + "po sters", + "pow der", + "po li", + "ha un", + "tra p", + "bl in", + "assau lt", + "shor ts", + "re y", + "sh y", + "squ ir", + "rac ist", + "gar lic", + "fu r", + "remo te", + "sm ell", + "impre ssed", + "fing ers", + "âł Ģ", + "din o", + "le ment", + "s nu", + "promo ting", + "str ing", + "produc tive", + "b age", + "ma son", + "ra z", + "direc tly", + "j k", + "ev al", + "ðŁij Ĭ", + "doc tors", + "co w", + "ri der", + "st v", + "re move", + "w u", + "na than", + "ro d", + "n r", + "= >", + "affe cted", + "inve st", + "mp tion", + "g inger", + "o d", + "agricul ture", + "s que", + "mu g", + "coun ting", + "ke e", + "mag nific", + "coo k", + "ani stan", + "roo t", + "plac ed", + "sym po", + "gh ana", + "un d", + "che er", + "thro wing", + "secre ts", + "f illing", + "opti mi", + "butter fly", + "bu bb", + "ðŁĺ ī", + "terri ble", + "d g", + "sil k", + "obse ssed", + "lo u", + "ai de", + "sal ute", + "mon u", + "philadel phia", + "scienti fic", + "i st", + "u ae", + "dess ert", + "bott les", + "can yon", + "ðŁĺ Ī", + "car ib", + "o ther", + "w ich", + "re source", + "guil ty", + "un d", + "le on", + "e ss", + "kan e", + "el e", + "tra iner", + "he im", + "an te", + "man age", + "roo kie", + "tre ated", + "po ses", + "rs vp", + "cau ses", + "aw ak", + "je well", + "le tt", + "on ics", + "tit les", + "cardi ff", + "g aga", + "bu mp", + "use ful", + "? !", + "loo se", + "bb ing", + ": :", + "argent ina", + "de bu", + "cy cl", + "wh el", + "dis gu", + "j el", + "k ills", + "bio logy", + "ex ter", + "tra sh", + "bo dies", + "tr am", + "circu it", + "expe ct", + "la ds", + "w ells", + "sho t", + "ge e", + "naren dr", + "fa stest", + "b ent", + "b ills", + "mar shall", + "h ats", + "intro duce", + "citi zen", + "im possible", + "gi b", + "az z", + "net working", + "r ant", + "thin k", + "in dy", + "st ops", + "f theday", + "bri an", + "* *", + "amo di", + "dom e", + "coura ge", + "pac king", + "af fairs", + "g n", + "si zed", + "ent ary", + "pol and", + "swit zer", + "afgh anistan", + "w u", + "ten der", + "subscri be", + "mo sco", + "att end", + "republic an", + "hon ey", + "âĢ ĭ", + "si mul", + "we ster", + "foo die", + "or o", + "midd le", + "ab t", + "co pies", + "ma je", + "narendr amodi", + "ty pical", + "inspir ational", + "vit am", + "wis con", + "cu bs", + "tiv ity", + "h ali", + "e ars", + "k ay", + "d are", + "mari juana", + "cu rious", + "an ia", + "tom ato", + "re mind", + "ðŁĩ ·", + "sc ared", + "cou p", + "po et", + "land ed", + "ri d", + "wra pped", + "mor ri", + "climb ing", + "e ws", + "fe eding", + "con tra", + "tho logy", + "gri d", + "ti vely", + "read er", + "la ser", + "di ving", + "di g", + "lat in", + "ti ed", + "shake spe", + "o ci", + "ad m", + "show ers", + "chu ck", + "mar cus", + "oo s", + "kne e", + "o live", + "ow l", + "dy lan", + "an no", + "g ym", + "deci sions", + "well ness", + "arri ves", + "sati s", + "chri s", + "thur s", + "ðŁ¤ £", + "inter views", + "thank you", + "switzer land", + "over night", + "journ alist", + "ser ves", + "vol can", + ".... ...", + "plo t", + "nic ol", + "car rying", + "mag ne", + "tre asure", + "ex p", + "be ver", + "ðŁĺ ¢", + "mar ty", + "mo le", + "don ations", + "recogni zed", + "b h", + "du s", + "sh ann", + "al do", + "success fully", + "ent e", + "ðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤ", + "cab inet", + "cu is", + "tit led", + "d as", + "so l", + "strate gies", + "deli vering", + "ad ds", + "ani an", + "ne ther", + "ðŁĴ ĥ", + "con tain", + "su its", + "pa irs", + "to dd", + "rel la", + "ro pe", + "ci o", + "cro p", + "paint ings", + "su z", + "re jec", + "bu st", + "d h", + "fra ud", + "m h", + "contro l", + "je al", + "destroy ed", + "al lows", + "wo ol", + "minneso ta", + "om en", + "j u", + "sympo sium", + "d af", + "lim it", + "accoun ts", + "load ing", + "inter n", + "re solution", + "hol land", + "qu al", + "meet ings", + "gra ve", + "cam ping", + "v am", + "re nov", + "liber al", + "am ber", + "gre e", + "hu mb", + "fe ver", + "el ing", + "broo ks", + "à ²", + "be th", + "ad ed", + "al t", + "ro e", + "perform ed", + "jo sh", + "frank lin", + "nic ole", + "de ss", + "bb s", + "m g", + "net works", + "min im", + "al t", + "weap ons", + "gu y", + "jas on", + "g ha", + "harb our", + "at on", + "pra ise", + "kentu cky", + "bel fast", + "st icks", + "blo ss", + "ho pes", + "an thro", + "famili ar", + "wa it", + "ch ile", + "depre ssion", + "la x", + "je ts", + "le ice", + "recei ves", + "si er", + "an k", + "de x", + "inde ed", + "fle xi", + "fab ric", + "lam b", + "hel icop", + "am anda", + "âĢĶ âĢĶ", + "compe te", + "sn ack", + "techno logies", + "sy rian", + "mom s", + "mu ham", + "cho sen", + "an at", + "dev on", + "shar ks", + "re t", + "fundra iser", + "selfi es", + "st ations", + "communic ations", + "tennes see", + "tu tor", + "ro t", + "valu able", + "dynam ic", + "nur se", + "i ed", + "earth quake", + "deser ved", + "a ve", + "sar a", + "stre tch", + "dougla s", + "ne pal", + "à §", + "ob viously", + "d ame", + "ra pe", + "any body", + "k w", + "pat rol", + "hol ders", + "h anna", + "info graphic", + "ec o", + "be ating", + "stan ley", + "bo ats", + "ri bb", + "e z", + "wit ch", + "inv a", + "ac id", + "boar ding", + "- @", + "gi l", + "da ve", + "care ers", + "opp os", + "l loy", + "in ter", + "do pe", + "re su", + "j agu", + "sh ade", + "in dy", + "on ist", + "rel ations", + "ag en", + "ab le", + "inci dent", + "me ter", + "shar ma", + "id r", + "pro ve", + "immedi ately", + "tro ops", + "am an", + "g low", + "gaz a", + "blo cks", + "person al", + "chron ic", + "all er", + "si d", + "sh r", + "whats app", + "lu cy", + "ar chae", + "ho u", + "journ alism", + "our selves", + "go t", + "the med", + "shap ed", + "we ak", + "cas ual", + "leng th", + "sla m", + "ab bey", + "e v", + "coun ter", + "est a", + "reci pi", + "cha pel", + "expan sion", + "sel f", + "suff ering", + "sp ice", + "n z", + "sp art", + "desp er", + "boo king", + "quart ers", + "y on", + "ðŁĴ Ĺ", + "p k", + "continu ed", + "- #", + "man hatt", + "tal ked", + "sh en", + "com bo", + "hybri d", + "je ans", + "liqu id", + "se al", + "re tweets", + "ac celer", + "collec tive", + "t as", + ": ))", + "profession als", + "ra w", + "o tt", + "su san", + "ir ing", + "okla homa", + "re ven", + "survi val", + "cre ator", + "tran sit", + "st ac", + "sur f", + "i k", + "ed iting", + "ch illing", + "bai ley", + "ste al", + "ra ble", + "pa rent", + "hun ger", + "sn app", + "collec t", + "philos oph", + "dedic ation", + "c f", + "c m", + "le ep", + "repe at", + "re ha", + "un fortun", + "a er", + "a ero", + "abstr act", + "mon itor", + "ag ents", + "bu l", + "sci ence", + "harb or", + "drag ons", + "floo ding", + "ac compli", + "d ash", + "juli a", + "the red", + "tues day", + "cy ber", + "b low", + "ta ined", + "le m", + "refe rence", + "pp o", + "ne goti", + "char le", + "con nor", + "au lt", + "access ories", + "commissi oner", + "rain y", + "re ar", + "advis ory", + "luc as", + "ma id", + "co al", + "k av", + "pol o", + "ðŁı ¾", + "tran sport", + "mar gare", + "straw berry", + "bur ns", + "gre ens", + "ne v", + "partici pants", + "col in", + "belgi um", + "col our", + "in form", + "d ell", + "br on", + "cal y", + "kick off", + "strate gic", + "re union", + "hon ors", + "li b", + "egy p", + "âŃIJ ï¸ı", + "hy po", + "si zes", + "regi stered", + "bet es", + "relax ing", + "bloo m", + "inten se", + "valent ines", + "insan e", + "w wii", + "p x", + "tri o", + "bla de", + "wiscon sin", + "con e", + "plat in", + "ali ze", + "ra ven", + "incre asing", + "indi ans", + "il ian", + "bl u", + "rabb it", + "exten sion", + "je f", + "au di", + "fer ry", + "s ell", + "a day", + "us b", + "swe at", + "cham pag", + "metho d", + "mem ph", + "assi st", + "s by", + "ca pe", + "remo ved", + "mag n", + "v t", + "r ams", + "f bi", + "tack le", + "phe w", + "h on", + "motor cycle", + "su spec", + "eleph ant", + "sub ject", + "let te", + "da iry", + "whe at", + "awk ward", + "ac t", + "tro l", + "mit ted", + "zay n", + "sheri ff", + "ene my", + "con s", + "ke tt", + "bul ls", + "ev alu", + "bt c", + "satell ite", + "ho lo", + "por ter", + "dia betes", + "bet ter", + "rele asing", + "sur f", + ": -", + "se basti", + "collec ting", + "en cing", + "e thi", + "go ds", + "al ley", + "health y", + "m ills", + "sma sh", + "co pper", + "cr ack", + "read ers", + "sp ac", + "licen se", + "bas ket", + "bang la", + "en tic", + "om i", + "m ere", + "si vely", + "anim ation", + "lan es", + "dent ally", + "chill in", + "fi e", + "k aren", + "dep th", + "li pse", + "n g", + "ri p", + "mel o", + "sand y", + "ðŁijı ðŁijı", + "vin cent", + "nu t", + "hu g", + "who le", + "cre ates", + "? ???", + "âĿ¤ï¸ı âĿ¤ï¸ı", + "bak ed", + "up grade", + "rober ts", + "har a", + "carib bean", + "auth entic", + "mb s", + "mosco w", + "attor ney", + "wi ki", + "ch lo", + "hu ll", + "cor k", + "\" !", + "sty lish", + "ðŁĵ¸ :", + "di ary", + "impro ving", + "ex pand", + "bri ght", + "pollu tion", + "k nights", + "person ality", + "chec ked", + "fac ilities", + "z el", + "bow ling", + "gu er", + "ðŁİ Ĥ", + "on going", + "un its", + "hoo k", + "be ck", + "confl ict", + "to dd", + "far ming", + "educ ational", + "k ak", + "cla y", + "stro ke", + "bel ly", + "explo re", + "mill enni", + "th m", + "loo p", + "sm s", + "consi st", + "cir ca", + "br yan", + "d ab", + "youn ger", + "soli dar", + "pp a", + "experi enced", + "b ella", + "bo ard", + "shef field", + "steph en", + "consu mer", + "sub mit", + "spon sor", + "t ang", + "ag gre", + "comb ined", + "trac king", + "sand ers", + "b az", + "survi ve", + "fer red", + "equ al", + "se p", + "re ed", + "str ong", + "priv acy", + "st ap", + "un g", + "ac ry", + "pa sta", + "pir ates", + "ag er", + "fair y", + "du p", + "introduc ed", + "wi p", + "let s", + "spr ay", + "ðŁĵ º", + "gre w", + "a sts", + "pitts burgh", + "new york", + "jo ey", + "lau ren", + "tra de", + "ch op", + "pi pe", + "cla ire", + "behavi or", + "v ap", + "cre ws", + "lap top", + "ðŁ¤ Ĺ", + "che ster", + "disci pl", + "d f", + "out doors", + "k s", + "go ver", + "super star", + "cas ino", + "far mer", + "; -)", + "re turned", + "ðŁı Ī", + "ma il", + "roa sted", + "co sta", + "v ill", + "pe z", + "gard ening", + "distribu tion", + "sh ining", + "inve stors", + "ra sp", + "dec ades", + "reali zed", + "bar n", + "p ti", + "st able", + "ut d", + "pan thers", + "m ens", + "b n", + "ca de", + "bu cket", + "yn n", + "when ever", + "wa ke", + "da is", + "ber nie", + "lo dge", + "ju lie", + "atmo sphere", + "ðŁĺĺ ðŁĺĺ", + "major ity", + "par ti", + "exc it", + "cu t", + "me h", + "musli ms", + "be gun", + "fli ghts", + "vene ss", + "ce me", + "po sing", + "so le", + "g ou", + "dark ness", + "pe ach", + "cel tic", + "auth ority", + "grand ma", + "ful ness", + "smi th", + "speci fic", + "gar cia", + "co ins", + "good ness", + "aldu b", + "recru iting", + "den nis", + "gar y", + "sle eve", + "weap on", + "pl z", + "disco ver", + "harri son", + "recruit ment", + "ja i", + "ch im", + "com pared", + "tom s", + "mo thers", + "am y", + "archi ve", + "t ask", + "ben jam", + "se g", + "law yer", + "al um", + "inve sting", + "mi e", + "che z", + "j p", + "a ke", + "fl am", + "wall paper", + "âĻ¥ ï¸ı", + "t ton", + "che st", + "favor ites", + "we igh", + "coo lest", + "r ating", + "relev ant", + "lo gan", + "ma ple", + "run ners", + "pri or", + "peop le", + "ma ur", + "terrori st", + "te sted", + "carni val", + "su spen", + "me asure", + "m v", + "cyber security", + "app ren", + "terror ism", + "o z", + "v ital", + "ni es", + "gon z", + "fun ded", + "twi st", + "assess ment", + "die sel", + "en for", + "colum n", + "ad dressing", + "ca sts", + "pay ment", + "x ton", + "fi er", + ", '", + "la st", + "ne e", + "un less", + "clo se", + "sk ill", + "cuis ine", + "fun eral", + "ti les", + "a un", + "k ru", + "relation ships", + "ðŁĴ ¯", + "ev ent", + "âĢįâĻĤ ï¸ı", + "kind ness", + "pro posed", + "acou stic", + "a es", + "defen der", + "dan ce", + "h tt", + "w at", + "vo y", + "ðŁ¤ ĺ", + "au s", + "cli ff", + "sear ching", + "beauti fully", + "in qu", + "at l", + "speci alist", + "ðŁIJ ¶", + "da i", + "tra ils", + "class ics", + "inst ant", + "v ous", + "re venue", + "mar ch", + "kir k", + "fr inge", + "fire works", + "tri via", + "âĺ ħ", + "tr action", + "wal ter", + "mo to", + "l ily", + "att itude", + "cli mb", + "sc an", + "sav ings", + "c w", + "fa ith", + "cred its", + "ab led", + "gra ff", + "auto graph", + "he he", + "ran ch", + "ha d", + "ro gers", + "ðŁĮ ¹", + "f in", + "re qu", + "fol k", + "ad ditional", + "lyn n", + "u ber", + "dol lars", + "lo gic", + "wor th", + "so m", + "the sis", + "p ound", + "bi c", + "st ur", + "cer am", + "spen cer", + "en tered", + "v amp", + "organi zed", + "âľ Ī", + "pp s", + "tr on", + "merce des", + "no ti", + "compet itive", + "do w", + "ous ness", + "vic tor", + "gr illed", + "na i", + "pu tin", + "ab ra", + "bl ame", + "alex and", + "anim al", + "dec ent", + "p ent", + "inter ior", + ":' )", + "but ler", + "bal let", + "ðŁĴ Ķ", + "albu ms", + "down s", + "la d", + "si r", + "pla in", + "p ers", + "blon de", + "dis c", + "paki stan", + "se ment", + "ga a", + "w age", + "ch as", + "man i", + "co ps", + "terr it", + "lo l", + "lau ghter", + "ri vers", + "magnific ent", + "lam p", + "w b", + "new sle", + "char ts", + "ble ssing", + "p unch", + "lon gest", + "fl oral", + "cu tie", + "fare well", + "sto pping", + "mb b", + "bu d", + "chee se", + "de cla", + "si m", + "mc donald", + "de ter", + "you th", + "t ch", + "fre der", + "kin dle", + "fer n", + "at or", + "as leep", + "p ond", + "spr int", + "p ounds", + "la zy", + "gh e", + "fundra ising", + "dead ly", + "gran de", + "dou g", + "he y", + "lin da", + "consi dering", + "i um", + "gol den", + "vi k", + "auth ors", + "di ss", + "u ally", + "appropri ate", + "mor ning", + "y le", + "hon oring", + "foli o", + "be c", + "re bec", + "fin land", + "formu la", + "corn wall", + "sh ay", + "cau sing", + "bl end", + "sig nal", + "t ent", + "kash mir", + "nation als", + "har mony", + "sc out", + "acce ssi", + "he ight", + "medi eval", + "impro vement", + "ke es", + "prac tical", + "car d", + "de par", + "hu n", + "om ing", + "cal gary", + "ste l", + "bu bble", + "gur u", + "ma h", + "unex pe", + "n h", + "ed a", + "me at", + "i ge", + "si o", + "god dess", + "in ches", + "tun es", + "br itt", + "sti on", + "ra j", + "âĻ «", + "mer cy", + "ðŁĴ ĺ", + "sen ds", + "i est", + "pol ici", + "val e", + "reduc ed", + "as ap", + "vi jay", + "defen sive", + "celebr ations", + "ri ders", + "med itation", + "har mon", + "g ing", + " ¡", + "program ming", + "in au", + "sud den", + "m h", + "replac ement", + "sk u", + "j ar", + "gra des", + "ta st", + "k itt", + "brand ing", + "k aw", + "boo t", + "f ought", + "p ays", + "g f", + "iz ation", + "ho p", + "k k", + "activi st", + "v end", + "coast al", + "cha os", + "ðŁĶ ´", + "se me", + "bill board", + "li fting", + "cu mb", + "sc al", + "ðŁĸ ¤", + "stru ck", + "l v", + "indie dev", + "beat en", + "jun gle", + "al right", + "destin y", + "m ing", + "k c", + "ch ances", + "om an", + "q atar", + "cra f", + "tra ined", + "pri x", + "char m", + "o tive", + "s mu", + "e c", + "and ers", + "hand ed", + "al ban", + "certain ly", + "arri ving", + "i ze", + "sa i", + "tr ack", + "pain ter", + "hu mble", + "appo intment", + "head line", + "manag ing", + "mo d", + "as pe", + "andre a", + "à ¤", + "ethi op", + "un ited", + "exi st", + "bal i", + "k ad", + "n t", + "d red", + "re x", + "recogni ze", + "tam pa", + "be ers", + "ati a", + "he els", + "no te", + "transport ation", + "tur tle", + "re de", + "hipho p", + "sp icy", + "sp urs", + "⬠ĩ", + "cor p", + "ther n", + "to ast", + "hur ry", + "proper ties", + "ma ge", + "mar co", + "ele ments", + "bou ti", + "syn drome", + "ms g", + "develop er", + "gra ders", + "he im", + "re sil", + "off ices", + "del ay", + "di men", + "vin tag", + "barbar a", + "ðŁĺ ±", + "vene zu", + "cu lar", + "fac ed", + "bar n", + "ðŁĺ Ĩ", + "survi vor", + "wor m", + "confu sed", + "passion ate", + "Ø ±", + "identi fy", + "electr icity", + "sou ls", + "brad ley", + "repor tedly", + "lun ch", + "shel f", + "eli a", + "swee t", + "smoo th", + "emplo yment", + "am el", + "manhatt an", + "ste am", + "oun ts", + "ye p", + "li ving", + "un e", + "descri be", + "ca res", + "man ila", + "sha wn", + "ac ted", + "bas h", + "st even", + "re st", + "pet ition", + "div ine", + "wel sh", + "rac e", + "platin um", + "ðŁĮ ¸", + "p b", + "extra ordinary", + "solidar ity", + "m all", + "on ion", + "schedu led", + "game of", + "fer gu", + "de ms", + "nor m", + "p k", + "tri als", + "polici es", + "publi shing", + "st ole", + "fron t", + "charac ter", + "van ia", + "ex ce", + "sti e", + "sc a", + "resi dential", + "sa iling", + "ðŁĶ¥ðŁĶ¥ ðŁĶ¥", + "spons ors", + "th ick", + "champag ne", + "she pher", + "continu ing", + "ven ice", + "per th", + "na p", + "a ster", + "y ak", + "un limited", + "cho ices", + "ne o", + "hi v", + "repor ter", + "bru ssels", + "f old", + "dy s", + "se mi", + "la wn", + "it alia", + "wi fi", + "as k", + "em ed", + "fr ame", + "monit oring", + "ste ad", + "i da", + "gr in", + "is a", + "fli p", + "re stric", + "offen sive", + "atta ched", + "di sh", + "wh y", + "philli ps", + "gre et", + "p als", + "mix tape", + "v ou", + "fiel der", + "spar k", + "alber ta", + "g len", + "ca sh", + "s ri", + "u ri", + "ro dri", + "entreprene urs", + "climate change", + "p sy", + "d le", + "em ents", + "lin ked", + "nether lands", + "acci dentally", + "oppos ition", + "vel vet", + "ra ys", + "c w", + "om o", + "m f", + "lmfa o", + "newsle tter", + ": )", + "toi let", + "liter ature", + "di sp", + "phili p", + "uni form", + "sudden ly", + "head er", + "cool er", + "-- -", + "prou d", + "bri g", + "nis san", + "scienti st", + "j ah", + "con centr", + "pac ks", + "appo inted", + "so ap", + "eng age", + "cho se", + "âĻ ¡", + "se tup", + "jeal ous", + "har ry", + "g ation", + "tun nel", + "te mp", + "osc ars", + "dec ade", + "recomm ended", + "child ren", + "ab a", + "anxi ety", + "ve ments", + "sal on", + "pho too", + "organi z", + "mach ines", + "ab s", + "vil le", + "hy pe", + "ti ff", + "emer ging", + "av geek", + "[ #", + "contribu tion", + "bra dy", + "re sto", + "g mail", + "fit z", + "photo shoot", + "hel met", + "h t", + "eleg ant", + "ug anda", + "nur sing", + "or leans", + "pen n", + "na h", + "foo tage", + "em a", + "w o", + "w ad", + "concer ns", + "ve re", + "re mark", + "who ever", + "str ang", + "p t", + "qu it", + "sh ang", + "histor y", + "s ick", + "perman ent", + "ill ness", + "col d", + "visi on", + "he m", + "ar row", + "con vic", + "pin k", + "oc cup", + "bal d", + "ex hau", + "u of", + "am o", + "on t", + "ãĥ »", + "adop t", + "la id", + "smo ked", + "inter pre", + "ess enti", + "associ ated", + "b d", + "bb y", + "fi er", + "inst all", + "dipl om", + "con diti", + "c f", + "w ak", + "any a", + "gr aci", + "fi sher", + "s ss", + "ap r", + "il it", + "mus ician", + "symph ony", + "cor d", + "h ack", + "le gi", + "l v", + "bless ings", + "hum or", + "sc ra", + "e ti", + "min ster", + "trav elling", + "bu sh", + "jewell ery", + "li me", + "!! !", + "pregn ant", + "pe e", + "lo b", + "cap ital", + "ip a", + "pen cil", + "la bor", + "duc ks", + "prou dly", + "wedd ing", + "dere k", + "m w", + "pe g", + "valent ine", + "an gu", + "re treat", + "pro spect", + "dang er", + "vul ner", + "up set", + ", #", + "sr k", + "x im", + "thur sday", + "n fl", + "kis ses", + "re ds", + "cr ack", + "re ward", + "c u", + "ko k", + "me te", + "aband oned", + "it t", + "me als", + "sp ell", + "stan bul", + "del ays", + "ru m", + "le op", + "gu m", + "no va", + "super man", + "ch ick", + "m is", + "dram atic", + "inno cent", + "r ounds", + "re c", + "auti sm", + "bangla desh", + "mor al", + "mo vie", + "sp oo", + "k la", + "âĥ £", + "ou ting", + "mess i", + "ab road", + "loo kin", + "a im", + "q i", + "st ack", + "colla ge", + "à ¯", + "hud son", + "sc an", + "ho e", + "ch au", + "oc cur", + "comm ander", + "ho les", + "ðŁİ Ħ", + "bi as", + "v on", + "stick er", + "ma k", + "responsi bility", + "colum bus", + "sa int", + "ed mon", + "rac ism", + "far ms", + "w en", + "gul f", + "may o", + "!!!! !!!!", + "corpor ation", + "ba chel", + "el a", + "inter nal", + "je ep", + "fol lows", + "di alogue", + "de rer", + "smart phone", + "he len", + "rich mond", + "equ ity", + "s land", + "b g", + "ne ar", + "av i", + "memph is", + "we ir", + "discu ssed", + "bad ge", + "p up", + "mi stake", + "phen omen", + "un ite", + "ðŁ Ľ", + "de pic", + "ri des", + "in augu", + "n at", + "sof twitter", + "comb ination", + "gosp el", + "âļ ¾", + "ad mission", + "retro gaming", + "ðŁIJ ¾", + "sch u", + "mb o", + "jun ction", + "al arm", + "à ¦", + "gr ac", + "kh ali", + "k ul", + "m ale", + "cap tion", + "wi sh", + "te re", + "cor ps", + "ru bber", + "play station", + "er in", + "effici ent", + "l or", + "jo kes", + "in ary", + "nor man", + "lu is", + "inaugu ral", + "ch ed", + "âļ½ ï¸ı", + "di p", + "to e", + "str at", + "aa c", + "am u", + "pi er", + "co tt", + "comm and", + "tt en", + "sn oo", + "cu be", + "clo ses", + "class ical", + "s word", + "expre ssion", + "reach ing", + "n app", + "co st", + "affe ct", + "ric o", + "gi f", + "brea the", + "tri be", + "or tho", + "h ay", + "l g", + "fri es", + "n m", + "hi ding", + "richar ds", + "en de", + "mic ro", + "capit ol", + "cop y", + "ro m", + "regi me", + "mary land", + "tax i", + "di al", + "embar ra", + "un believ", + "ch t", + "v s", + "elim in", + "o dd", + "pen ny", + "sound track", + "l ings", + "trans ition", + "rema ining", + "a is", + "mali k", + "? !?", + "rand om", + "def end", + "ul tra", + "tru m", + "danc er", + "st ol", + "dri ve", + "a ver", + "ro ast", + "defin ition", + "se an", + "excit ement", + "partic ul", + "su rely", + "sh av", + "ber y", + "di shes", + "com m", + "is ol", + "i am", + "ob li", + "gho st", + "hugh es", + "chi efs", + "b as", + "conserv ative", + "speci al", + "fe min", + "sh ri", + "n ancy", + "inte l", + "tu ne", + "ðŁĩ ª", + "jo el", + "gg le", + "mo to", + "ðŁĺ Ķ", + "bu ck", + "d ag", + "antic ip", + "mont ana", + "gu id", + "fro g", + "ec raft", + "op e", + "dri ves", + "nu mer", + "x y", + "color ful", + "wednesday wisdom", + "illu min", + "bey on", + "inau gur", + "deep ly", + "pre fer", + "for tune", + "coo ked", + "ti ble", + "âĺ ķ", + "swe ater", + "it ter", + "tt y", + "u i", + "gi e", + "com plic", + "~ ~", + "tax es", + "cu ps", + "di verse", + "sam anth", + "âłĢ âłĢ", + "ba king", + "sy mp", + "wa i", + "be half", + "mer cur", + "travel s", + "ðŁİī ðŁİ", + "or ia", + "eng aged", + "jump ing", + "reti red", + "n aked", + "p uni", + "speed way", + "sci ences", + "rehear sal", + "on ym", + "dy ou", + "pl ates", + "r ati", + "kri sh", + "jaz z", + "car ol", + "ra f", + "pen alty", + "tim eline", + "ru by", + "engine ers", + "ra f", + "bel le", + "do se", + "che on", + "esc ap", + "me g", + "ran k", + "or d", + "me gan", + "mer ch", + "ec lipse", + "âĺº ï¸ı", + "ple dge", + "kir k", + "per si", + "leice ster", + "sa k", + "w k", + "saf ely", + "yy y", + "je t", + "promis ed", + "j c", + "en ne", + "no ah", + "re no", + "re a", + "ðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤ", + "tra il", + "ðŁij Ģ", + "f d", + "soo o", + "ri min", + "w k", + "ภ²", + "i al", + "x ox", + "bis cu", + "d ale", + "fan dom", + "particip ating", + "fla g", + "privi lege", + "pe ach", + "mach ine", + "bo ston", + "gro ss", + "o g", + "mir acle", + "adop tion", + "u ss", + "mon sters", + "be ij", + "clar ke", + "pu shing", + "pra ying", + "ar o", + "d n", + "ell is", + "apol lo", + "od ds", + "refuge e", + "to w", + "b p", + "ðŁĩ¬ðŁĩ §", + "h end", + "app eared", + "memb ership", + "pe an", + "du m", + "viol ent", + "v y", + "potat oes", + "aw w", + "greet ings", + "t ts", + "ac on", + "sh ane", + "photograph ed", + "cra b", + "temper atures", + "cu ba", + "c fc", + "wel com", + "he l", + "in nings", + "m k", + "co de", + "kno ck", + "gra ss", + "swe dish", + "p ta", + "ick y", + "v at", + "lin ing", + "s q", + "sa p", + "ar c", + "announ cing", + "sk ins", + "cit yof", + "br ing", + "co x", + "gam er", + "it arian", + "i da", + "h d", + "ros se", + "sad ly", + "ge o", + "âļ ¡ï¸ı", + "tag s", + "fa ther", + "chan ge", + "l ance", + "whis key", + "adel aide", + "te c", + "stick ers", + "marke t", + "class y", + "bad ass", + "flo rence", + "lin er", + "fro st", + "k ate", + "ac on", + "scand al", + "es sex", + "ðŁĺ ı", + "vi vi", + "dr ill", + "blo ggers", + "recomm end", + "d ha", + "ac res", + "ro ma", + "bu y", + "gro cer", + "er ia", + "ma har", + "ff er", + "patter ns", + "ver i", + "com pu", + "st ev", + "ang a", + "ment or", + "do o", + "it ali", + "cdn poli", + "on ly", + "conduc t", + "elec tro", + "de f", + "wh ale", + "prepar ation", + "bicy cle", + "vi ral", + "turn out", + "bra ss", + "qu ad", + "hospit ality", + "pack aging", + "den cy", + "ceme tery", + "abo ard", + "dre aming", + "pic ture", + "t all", + "inv ent", + "ad mi", + "o e", + "tem ps", + "qu an", + "fun dam", + "pro mp", + "resi dence", + "mu d", + "sour i", + "âĦ ¢", + "graff iti", + "gi f", + "d nd", + "com p", + "s war", + "pe eps", + "pale stine", + "devil s", + "san g", + "assi stance", + "bi ke", + "missi ssi", + "inter viewed", + "ne phew", + "dru ms", + "v and", + "gentle men", + "n sw", + "inst a", + "leban on", + "ee ee", + "oli via", + "ver y", + "rou gh", + "industri es", + "m ation", + "ðŁĺ Ĵ", + "bar rel", + "n ay", + "po ps", + "moder n", + "ill y", + "are st", + "on ents", + "protec ting", + "v ans", + "e o", + "vi kings", + "restaur ants", + "re ck", + "jac kie", + "andre w", + "w illing", + "he ath", + "citiz en", + "disc rimin", + "๠Ī", + "stu art", + "m ys", + "hi p", + "tran sp", + "\" ?", + "te x", + "su shi", + "ke d", + "cro ssed", + "dist ur", + "pe dia", + "f ate", + "some how", + "mo th", + "proce ssing", + "is s", + "r in", + "u ts", + "yy c", + "ver t", + "lg bt", + "re id", + "on to", + "arab ia", + "habit at", + "= =", + "stre ak", + "simp son", + "addic tion", + "wim ble", + "deli vers", + "challeng ing", + "ðŁİ ¶", + "fran ch", + "e du", + "s me", + "ai ds", + "hur st", + "th am", + "tari an", + "remem bered", + "palestin ian", + "fe es", + "tru m", + "sket ch", + "ur u", + "fit ting", + "jes se", + "ðŁĶ¥ ðŁĶ¥", + "---- ----", + "ba ch", + "ici a", + "colo red", + "da h", + "associ ate", + "int el", + "s eller", + "p u", + "stu ffed", + "ac s", + "b s", + "sh in", + "cooper ation", + "certific ate", + "ab u", + "ingredi ents", + "re v", + "in ge", + "el der", + "christi an", + "bun dle", + "th ic", + "dir t", + "beij ing", + "comm it", + "ted dy", + "ed u", + "to day", + "s field", + "w yn", + "confir ms", + "lo o", + "j v", + "ene ss", + "al pha", + "vir us", + "ari um", + "gr ind", + "bri dges", + "introduc tion", + "pol ls", + "bac ter", + "z ach", + "termin al", + "ra iders", + "fla vor", + "zom bie", + "vo d", + "sp reading", + "gameof thrones", + "effici ency", + "lat ely", + "ale m", + "twee t", + "cri mes", + "cl er", + "de y", + "dg ed", + "hy un", + "pay ments", + "cir cus", + "ðŁĺŃ ðŁĺŃ", + "mis souri", + "lu b", + "episo des", + "c age", + "po s", + "mat ching", + "tumb lr", + "lin ed", + "ge st", + "am bi", + "nar r", + "ing ton", + "regu l", + "blo wn", + "is le", + "co co", + "on don", + "joshu a", + "tour ing", + "sm a", + "sau sage", + "best friend", + "bo eing", + "desi re", + "sav age", + "ra pper", + "de vo", + "te ar", + "take over", + "cow boys", + "po ker", + "par ag", + "pp e", + "h int", + "we ars", + "se th", + "ro les", + "l anc", + "man ga", + "form at", + "fl yer", + "c ay", + "mo or", + "ba ke", + "spla sh", + "v ad", + "ker ala", + "proce eds", + "sil ly", + "reflec tion", + "di str", + "wi d", + "su it", + "ci vic", + "yan kees", + "by n", + "migr ation", + "di stin", + "or ch", + "fe mini", + "quali fying", + "tu ri", + "o be", + "hun dred", + "cra p", + "wan g", + "mathe mat", + "bu re", + "expo sure", + "fergu son", + "seme ster", + "re serv", + "pl ym", + "a hu", + "fac ial", + "wa x", + "wor ried", + "ca b", + "vi o", + "as a", + "co d", + "to pics", + "p cs", + "hal o", + "rescu ed", + "horiz on", + "ar k", + "âļ ª", + "hol ly", + "el f", + "ul ti", + "pu p", + "quali fied", + "attend ance", + "ati vely", + "destro y", + "y c", + "for th", + "photoo ftheday", + "c ents", + "ic eland", + "meas ures", + "de sk", + "port folio", + "artic les", + "direc tors", + "dat ab", + "e w", + "creep y", + "oun ding", + "hon oured", + "mi st", + "j it", + "men tioned", + "port able", + "iti c", + "d ann", + "friday feeling", + "am id", + "ti ger", + "scri p", + "helicop ter", + "hard ware", + "expl or", + "work place", + "austri a", + "beat les", + "ber nar", + "spi der", + "disc o", + "cul t", + "lim its", + "shor tly", + "fin al", + "nin ja", + "lu ke", + "le bron", + "wal mart", + "o il", + "van illa", + "shi re", + "ye g", + "ak y", + "c s", + "bl er", + "collec ted", + "t g", + "rol led", + "speci als", + "b ff", + "pier re", + "sh im", + "vi er", + "flash back", + "restor ation", + "individu als", + "pro d", + "fre aking", + "tu rer", + "o a", + "re fre", + "mor oc", + "gre et", + "re yn", + "care ful", + "our ing", + "u sh", + "is d", + "g ill", + "vie w", + "thunder storm", + "b led", + "pic nic", + "guar di", + "pi g", + "ar k", + "syl vania", + "bann ed", + "u cl", + "vi jay", + "ori um", + "av engers", + "believ es", + "eu r", + "monu ment", + "concer ned", + "la bs", + "ber g", + "a ap", + "vi sh", + "sing les", + "can cel", + "z el", + "ar ab", + "ru th", + "too th", + "ar ta", + "sh af", + "chair s", + "r ack", + "dise ases", + "crow d", + "cl y", + "fle x", + "christ ma", + "artif icial", + "tom at", + "fin e", + "dra ws", + "advoc ate", + "fran ce", + "Ù Ĭ", + "ðŁĺ ³", + "heav y", + "s our", + "compre hen", + "no ble", + "aa p", + "hin du", + "cor al", + "g ars", + "ow en", + "n l", + "st all", + "yel low", + "mar ina", + "in ver", + "suppor t", + "tou gh", + "promis es", + "pi e", + "master piece", + "sco re", + "for ce", + "mor tg", + "crypto currency", + "o x", + "r ors", + "rock in", + "pro vin", + "ho g", + "no stal", + "oak land", + "pat rick", + "inclu sion", + "tra ffic", + "ah med", + "a ha", + "lux ury", + "con secu", + "de mon", + "âĸ º", + "b lowing", + "st ag", + ": \"", + "encoura ge", + "ben e", + "sku ll", + "do dge", + "bu ster", + "kin son", + "wit ne", + "er ror", + "lo west", + "fel low", + "à °", + "sh re", + "bl ur", + "vir gin", + "compos er", + "sli p", + "mor nings", + "ga ins", + "tab le", + "gra in", + "ari st", + "braz ilian", + "w we", + "tu es", + "ribb on", + "an ag", + "di st", + "sac rif", + "em brace", + "entreprene ur", + "af fili", + "de o", + "t ali", + "touri st", + "fat al", + "ì Ĭ", + "autom atic", + "ðŁĩ µ", + "we ak", + "wel fare", + "confir m", + "benjam in", + "fi ghts", + "alleg ed", + "me ad", + "strugg ling", + "pro secu", + "che f", + "à ¨", + "propos al", + "er n", + "ðŁĺ Ħ", + "dy k", + "on gs", + "hon g", + "m ack", + "mel on", + "on ent", + "ru sh", + "d ap", + "tol er", + "pro pag", + "c ze", + "trans lation", + "wal let", + "cott age", + "sa il", + "constitu tion", + "ðŁĴ Ģ", + "mun ici", + "fav or", + "storm hour", + "i h", + "ðŁĺ Į", + "approach ing", + "pin ned", + "j ed", + "niger ian", + "n ach", + "sh at", + "particul arly", + "mc don", + "camer as", + "anni e", + "admini str", + "he at", + "electr ical", + "char ming", + "gib son", + "bouti que", + "ex posed", + "ac tor", + "pil low", + "beach es", + "genu ine", + "margare t", + "ben nett", + "lou isi", + "pos itions", + "el y", + "shin y", + "ten tion", + "architec t", + "ren tal", + "ac qui", + "goo gle", + "sub way", + "mom ent", + "ðŁļ ¨", + "ri m", + "metho ds", + "cy cli", + "nor folk", + "Ù Ī", + "over whel", + "ra pid", + "we ar", + "happy birthday", + "progre ssive", + "ðŁĴ ¥", + "co gn", + "pap a", + "f ool", + "philosoph y", + "pol ar", + "jim my", + "wi g", + "ðŁĴ ĭ", + "oper ating", + "reduc tion", + "ph i", + "fla gs", + "to the", + "o di", + "a res", + "k oo", + "k ang", + "ar kansas", + "ash ton", + "wimble don", + "sci fi", + "attrac tive", + "mississi ppi", + "logi sts", + "ral ph", + "la bel", + "gradu ates", + "ma ha", + "home town", + "âľĮ ï¸ı", + "foun ded", + "on the", + "li z", + "trans l", + "mini mum", + "pre sti", + "ta m", + "gener ations", + "re bel", + "journ alists", + "par am", + "mc m", + "acry lic", + "death s", + "tes la", + "w t", + "bry ant", + "jer us", + "i stanbul", + "muham mad", + "ri ley", + "k ris", + "work shops", + "is o", + "coun ts", + "stre t", + "prote cted", + "trin ity", + "man ual", + "r hin", + "r il", + "pleas ant", + "le mon", + "ner d", + "har der", + "dar ren", + "bur y", + "ra h", + "bas is", + "mi gu", + "occa sion", + "li sts", + "âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ı", + "e b", + "de cre", + "hamp ton", + "ìĿ ´", + "tra vis", + "trans form", + "puer to", + "nh l", + "av oc", + "tri ps", + "unexpe cted", + "ve t", + "di dyou", + "bar ber", + "st ages", + "m son", + "re presented", + "for t", + "l al", + "pp le", + "nic ely", + "ignor e", + "qu il", + "qu inn", + "h k", + "carri er", + "remin ded", + "am ong", + "pass enger", + "el len", + "gue z", + "sc ape", + "mu ral", + "youn gest", + "ma sh", + "d ill", + "rout ine", + "stain less", + "jack son", + "gand hi", + "th al", + "on ers", + "edit orial", + "convers ations", + "sd ale", + "autom ation", + "i ke", + "า à¸", + "ðŁĩ ª", + "hau l", + "la ying", + "men tions", + "am en", + "abor tion", + "i bi", + "coun ties", + "ca therine", + "man ds", + "jam e", + "roll er", + "au t", + "n am", + "o logical", + "cep tion", + "ran king", + "tox ic", + "sn acks", + "victor ian", + "bang kok", + "psycho logy", + "re g", + "ang ela", + "respon d", + "sty le", + "sophi e", + "dak ota", + "achiev ed", + "mar ked", + "imper ial", + "in as", + "glo ves", + "sli m", + "confi dent", + "att acked", + "gg er", + "lon ely", + "valentine sday", + "re b", + "craft beer", + "orig in", + "zim bab", + "ce iling", + "te ens", + "other wise", + "w b", + "f ers", + "day sof", + "advis or", + "y ah", + "âĻ ª", + "en der", + "republic ans", + "av a", + "skir t", + "pi pel", + "chi e", + "jan e", + "ja x", + "ðŁĺ ĭ", + "âľ Ĭ", + "j ays", + "bre tt", + "bal o", + "cru cial", + "d har", + "as is", + "de au", + "lloy d", + "chat ting", + "âĿĦ ï¸ı", + "rel ay", + "remark able", + "n s", + "we t", + "bris bane", + "ðŁĶ ´", + "tion ally", + "f k", + "la yer", + "house hold", + "consecu tive", + "es is", + "pend ant", + "st ir", + "crit ic", + "su gar", + "photo shop", + "pa res", + "arti stic", + "do dgers", + "c un", + "cra fted", + "am end", + "bo at", + "âŃIJ ï¸ı", + "egyp tian", + "sa w", + "tra ge", + "small er", + "ox y", + "pa ired", + "nex t", + "i res", + "tac o", + "o y", + "u c", + "st i", + "a erial", + ": //", + "dr o", + "dot com", + "gg ins", + "r pg", + "ay e", + "le an", + "stri ker", + "lo bby", + "prote sts", + "pri ority", + "congre ss", + "am ate", + "inv it", + "r ington", + "mom my", + "th us", + "allow ing", + "pione er", + "enfor cement", + "g ori", + "tal k", + "dra g", + "du mb", + "bul let", + "san ge", + "er y", + "tar gets", + "ðŁĩ ¦", + "he ather", + "consi der", + "seaf ood", + "ve st", + "ris ks", + "% .", + "p g", + "sac red", + "he ating", + "kick ed", + "tto t", + ". -", + "chan di", + "co ven", + "po ol", + "pul se", + "i a", + "ro ster", + "shakespe are", + "es a", + "car go", + "pean ut", + "tro op", + "ac tion", + "tab let", + "home work", + "cast le", + "stru ction", + "mus icians", + "free zing", + "bu tt", + "justin bieber", + "j j", + "bah rain", + "an them", + "au dit", + "didyou know", + "na vig", + "guid ance", + "âĸ ¶", + "tur f", + "n un", + "fic ations", + "ye men", + "char ging", + "x c", + "bron cos", + "su bur", + "p ale", + "bor ing", + "among st", + "for the", + "em per", + "om fg", + "p j", + "expe cting", + "ðŁĴ «", + "st l", + "ad min", + "expect ations", + "sw an", + "shoo t", + "oooo o", + "min ent", + "ãĢ IJ", + "wall ace", + "stan g", + "satur day", + "adop ted", + "dou bles", + "hom ie", + "ome z", + "d han", + "vent ure", + "surroun ding", + "fi le", + "mob ility", + "de es", + "w ski", + "broo ke", + "emb ro", + "re members", + "kar a", + "test im", + "bo tan", + "m tv", + "sacrif ice", + "jerus alem", + "d l", + " ´", + "proper ly", + "ili on", + "as i", + "leg it", + "co pe", + "m cla", + "recy cling", + "lar ger", + "ðŁĴ ĵ", + "pat ric", + "gener ous", + "ja red", + "p f", + "mol ly", + "thom as", + "ju dges", + "h b", + "sor ts", + "bl vd", + "o ven", + "enter ing", + "plan es", + "be et", + "integr ation", + "boo ked", + "fre ed", + "ver n", + "ash es", + "to pped", + "de pot", + "welcom ed", + "ren a", + "m ick", + "d and", + "see ks", + "gam er", + "ran kings", + "ren e", + "mu t", + "whis ky", + "fire fighters", + "gu es", + "ga ther", + "tour ney", + "de men", + "y ang", + "new ton", + "autom otive", + "back yard", + "deta iled", + "mi st", + "to bac", + "fi ber", + "un usual", + "grat itude", + "sp are", + "ne ys", + ": *", + "per i", + "flo ating", + "fin alist", + "don ating", + "dre ss", + "bro ad", + "be the", + "econom ics", + "tai wan", + "ed wards", + "plu g", + "pra iri", + "val en", + "bab a", + "f ad", + "an as", + "har per", + "dis order", + "app lied", + "p att", + "bi kin", + "li ver", + "cu ri", + "carol ine", + "ann er", + "juli an", + "wal king", + "mal col", + "screen shot", + "co ding", + "skin care", + "activi sts", + "myster ious", + "ex act", + "blo cking", + "mercur y", + "bat ter", + "du mp", + "âľ Į", + "en se", + "li sh", + "ridic ulous", + "prote sters", + "ðŁĻ Ī", + "lu st", + "swe at", + "as s", + "ali ke", + "co dy", + "re ments", + "win ds", + "as pir", + "vi enna", + "pra y", + ".. .@", + "bo i", + "cand le", + "assi sts", + "te e", + "der son", + "p ony", + "f ence", + "con spir", + "âĺħ âĺħ", + "oo th", + "e pic", + "ba rely", + "a unt", + "b am", + "diamon ds", + "end less", + "scre ens", + "can cer", + "gr o", + "p st", + "pro spec", + "mo sque", + "help ful", + "ou ri", + "bro ther", + "gu jar", + "cri sti", + "ine z", + "to wers", + "ad dresses", + "gra y", + "bur ton", + "re tweeted", + "ðŁ¤ Ķ", + "n ity", + "du ck", + "super vis", + "jo an", + "kin der", + "sanc tu", + "pi ed", + "âı °", + "ł ï¸ı", + "m ati", + "reven ge", + "ce ster", + "eli fe", + "desig ners", + "back ed", + "bo li", + "wei ght", + "cou ch", + "su res", + "s its", + "shri mp", + "la gos", + "auth orities", + "os ity", + "hol ly", + "compu ting", + "fac tors", + "ab e", + "pan els", + "ram ad", + "sent ence", + "missi on", + "hol m", + "r b", + "d ads", + "shang hai", + "mon ey", + "she ets", + "sk ate", + "thre w", + "cup cakes", + "infin ite", + "l is", + "practic ing", + "ess ay", + "ka i", + "as ci", + "mo b", + "u gh", + "hol mes", + "re gg", + "ik h", + "mo ck", + "collec tions", + "pe p", + "o va", + "sal t", + "nan dez", + "co y", + "thre ats", + "tex ts", + "cin nam", + "pregn ancy", + "pen ding", + "stam p", + "flow er", + "g is", + "agre ed", + "pay ne", + "ro ver", + "ph ra", + "sof t", + "f fin", + "fa thers", + "pass engers", + "aw ays", + "al a", + "h es", + "li van", + "in s", + "samu el", + "ingu i", + "h of", + "j j", + "chen nai", + "cat al", + "om ic", + "he ath", + "ni ece", + "pump ed", + "integr ated", + "are l", + "no m", + "produc tivity", + "wan ting", + "vis a", + "di ana", + "tw il", + "it v", + "cam ps", + "ro wing", + "d ley", + "black and", + "gu ards", + "b ells", + "re verse", + "vi be", + "ric ky", + "mo ss", + "ny t", + "âĺ Ģï¸ı", + "el le", + "tro y", + "cu dd", + "ev an", + "women s", + "fo to", + "mi stakes", + "wick ed", + "mi l", + "c led", + "me mes", + "co smo", + "schol ar", + "ren o", + "ðŁĺ Ģ", + "v ents", + "# âĢ¦", + "terrori sts", + "ca sey", + "cardin als", + "ðŁĺĬ ðŁĺĬ", + "venezu ela", + "bol a", + "liter acy", + "t w", + "en o", + "con tains", + "au stin", + "fin anci", + "ev an", + "har vard", + "origin ally", + "chev ro", + "her ald", + "nott ingham", + "manag ers", + "âŀ ¡", + "accep ting", + "wal sh", + "tutor ial", + "entrepreneur ship", + "yach t", + "requi rements", + "glen n", + "pe de", + "unfortun ately", + "ach ing", + "dais y", + "gi an", + "night mare", + "âĿ Ĺ", + "r ina", + "b art", + "ema ils", + "oppo site", + "who m", + "sa ke", + "pu zzle", + "da shi", + "par ty", + "blan ket", + "bus es", + "lo re", + "beau ty", + "reas on", + "pun jab", + "winds or", + "func tional", + "exi sting", + "hel lo", + "gli mp", + "con vin", + "la k", + "scre aming", + "rebec ca", + "bli ss", + "north west", + "infin ity", + "cosme tics", + "pul ling", + "coffe e", + "pl ing", + "op ho", + "colom bia", + "interior design", + "( +", + "emo tions", + "sa c", + "sun glasses", + "sav es", + "d f", + "six th", + "al y", + "ðŁĺ »", + "de en", + "dev ast", + "polit icians", + "lac rosse", + "g u", + "pe i", + "jav a", + "comb ine", + "coal ition", + "er ts", + "survi v", + "ch ad", + "stri an", + "n n", + "de vi", + "coun c", + "concer n", + "contro ller", + "bre ast", + "j ury", + "tu m", + "introduc es", + "la di", + "mobi le", + "al z", + "ste ady", + "nur ses", + "h acking", + "on line", + "oce an", + "ðŁİ Ħ", + "a am", + "ju ven", + "ic c", + "louisi ana", + "ar te", + "street art", + "is on", + "wn s", + "fr m", + "p anda", + "no ir", + "main tain", + "del ay", + "symp toms", + "thor n", + "ge ome", + "ter n", + "carri ed", + "p ru", + "pan or", + "as sy", + "per u", + "clou d", + "sp ra", + "pe di", + "e ste", + "tag ged", + "ðŁĺ Ŀ", + "shado ws", + "naz i", + "ا٠Ħ", + "cor ri", + "âĻ¥ âĻ¥", + "j ad", + "ðŁĩ «", + "form al", + "spo ken", + "ðŁĮ ŀ", + "enjo y", + "lo pez", + "out look", + "in ho", + "w ander", + "Ù ħ", + "ma ya", + "pe e", + "d ine", + "ãĢ ij", + "brief ing", + "suppor ter", + "ar ily", + "ght ers", + "natur ally", + "doctor who", + "j en", + "v ar", + "new year", + "re se", + "si mm", + "re x", + "con sequ", + "tomat oes", + "bur st", + "bra vo", + "bur gers", + "cr acking", + "nor theast", + "bi om", + "mush room", + "mar que", + "dou ble", + "ni er", + "v ag", + "tw enty", + "key board", + "win ni", + "jama ica", + "par ish", + ": -", + "mental health", + "ali zing", + "ren der", + "wa king", + "ðŁİ Ĥ", + "g ly", + "na than", + "wa shing", + "mel issa", + "jun g", + "loy al", + "chil i", + "song writer", + "guit arist", + "bo wie", + "neighb ors", + "onym ous", + "as set", + "ta i", + "head quarters", + "ðŁĮ Ī", + "i hear", + "ci gare", + "sur g", + ") \"", + "re pl", + "dar ling", + "ðŁĻ Ħ", + "z ak", + "sa re", + "ãħ ĭ", + "mic key", + "ware house", + "mass age", + "ine es", + "did nt", + "i w", + "hur ts", + "eng aging", + "mag ic", + "women in", + "k itten", + "mor s", + "c art", + "tit ans", + "colle ague", + "compe ting", + "er an", + "k hal", + "mar ble", + "dem and", + "del ight", + "et ary", + "bli zz", + "lou ise", + "m ls", + "fini shes", + "experim ent", + "conduc ted", + "electr onics", + "itt ers", + "car ing", + "wh ats", + "sym bol", + "jun g", + "e cu", + "pi x", + "con text", + "char ger", + "ðŁĺ ĩ", + "re ig", + "fra g", + "ë ĭ", + "ch ad", + "tru e", + "ker ry", + "def ending", + "a int", + "au ton", + "check out", + "bar nes", + "less ly", + "d t", + "m me", + "clou dy", + "second ary", + "are z", + "_ :", + "app a", + "const ant", + "\" )", + "ve ts", + "jo b", + "i ent", + "ðŁĺŃðŁĺŃ ðŁĺŃ", + "m j", + "fren ch", + "di ver", + "davi es", + "hh hh", + "e book", + "๠ī", + "mar iti", + "bree ze", + "susp ended", + "mat o", + "vi et", + "ra hu", + "se i", + "bol t", + "en ary", + "le is", + "kar l", + "fr amed", + "expla ining", + "ab c", + "de aling", + "nat o", + "ja ke", + "exp and", + "leon ard", + "establi shed", + "du b", + "ar men", + "el led", + "voc al", + "nichol as", + "ori ent", + "k yo", + "illustr ated", + "ah h", + "danc ers", + "milli on", + "ge ta", + "po pp", + "as u", + "mur dered", + "gi ble", + "sto ked", + "gri ffin", + "maxi mum", + "adri an", + "en counter", + "ther o", + "david son", + "ðŁį »", + "holi day", + "ev o", + "asse ts", + "car son", + "memor able", + "âļ ½", + "ob am", + "represent ative", + "cb d", + "tr icks", + "vo gue", + "vo ice", + "mm mm", + "sebasti an", + "cli f", + "ath y", + "par alle", + "ðŁ¤ ·", + "pa k", + "ev acu", + "e ats", + "ا Ø", + "tou ched", + "organ ised", + "spir its", + "can ad", + "gui ded", + "frame work", + "ðŁĮ Ł", + "pe d", + "natur al", + "ag ar", + "replac ed", + "anch or", + "ti t", + "sha h", + "organ is", + "super ior", + "r n", + "ch ro", + "eric a", + "st ill", + "cor on", + "chu ck", + "loc ks", + "or gan", + "ro sen", + "sc am", + "ben ed", + "/ #", + "ke en", + "tre vor", + "vamp ire", + "sor ted", + "! '", + "af ford", + "in tro", + "gr ace", + "ðŁĺ ľ", + "sau r", + "kick starter", + "influ en", + "v u", + "y up", + "po c", + "ðŁİ ¥", + "a ar", + "s ang", + "tre k", + "et sy", + "tb h", + "scre am", + "chevro let", + "pix el", + "shepher d", + "an or", + "gabri el", + "tw ood", + "sd cc", + "me ters", + "develop ers", + "clo sure", + "v w", + "twit ch", + "ì Ĺ", + "se oul", + "pr ice", + "ho g", + "n ish", + "hill ary", + "scrat ch", + "in cen", + "wag on", + "dis ability", + "pan ther", + "ch ats", + "g d", + "wit z", + "sus sex", + "l ate", + "den mark", + "ger ald", + "cancel led", + "net te", + "i x", + "nav al", + "bap tist", + "te t", + "y ad", + "ma th", + "ho y", + "r andy", + "po int", + "intel lec", + "fru its", + "w ool", + "gu in", + "pr on", + "the ft", + "con dem", + "mar ry", + "n ola", + "architec ts", + "cin cin", + "roc kets", + "gentle man", + "ex plan", + "t ate", + "do e", + "ra ises", + "wild life", + "w l", + "insi der", + "blan c", + "w p", + "for sale", + "ny c", + "po well", + "unbeliev able", + "pen s", + "goo dies", + "mu stang", + "p ens", + "st ays", + "squ ash", + "xox o", + "near by", + "ever ton", + "co co", + "le agu", + "k han", + "stu d", + "south west", + "con struc", + "s worth", + "cro atia", + "le a", + "su ms", + "aim s", + "e an", + "van ess", + "iti ous", + "pa thy", + "arc ade", + "b end", + "sugge sts", + "sac ram", + "roy als", + "ri er", + "em ir", + "in cl", + "an k", + "clar k", + "ri ght", + "vac c", + "ठ¾", + "tan e", + "li b", + "u sc", + "sal es", + "hu h", + "s ally", + "ver a", + "p ga", + "gro ws", + "dru m", + "tre e", + "eth ics", + "sug gest", + "is ab", + "se aled", + "pre viously", + "anim ated", + "ab du", + "ri ses", + "glo b", + "pre dat", + "scar f", + "del ic", + "om ar", + "ll i", + "sx sw", + "py thon", + "ne bra", + "fun k", + "reflec t", + "pav ilion", + "tic ally", + "ch asing", + "bak ery", + "inva sion", + "ko h", + "believ ed", + "co hen", + "con qu", + "cra fts", + "nat i", + "cle ver", + "govern ance", + "sam ples", + "fa ils", + "â Ķ", + "ti mo", + "r itu", + "stri king", + "inclu sive", + "sho cking", + "can t", + "requi res", + "dra wings", + "ภŃ", + "purch ased", + "du m", + "z ach", + "war ner", + "con sole", + "man sion", + "foun tain", + "circu m", + "e sh", + "is land", + "mil k", + "pro fits", + "hali fax", + "ri val", + "âľĪ ï¸ı", + "jen ny", + "sand ra", + "ny e", + "k elly", + "y al", + "qu ad", + "no s", + "inste in", + "fin alists", + "mid fielder", + "cu e", + "excep tional", + "a an", + "sa pp", + "gett in", + "sa a", + "f ati", + "sl ice", + "vol k", + "s wal", + "la sting", + "sum mary", + "it as", + "sm o", + "s z", + "âĺ Ĩ", + "ip l", + "fl ames", + "ene ws", + "ha v", + "hoo die", + "pitch er", + "win dy", + "re vol", + "centr al", + "ton ite", + "ðŁİī ðŁİī", + "sol ved", + "mil wau", + "organiz ations", + "wee ts", + "re fin", + "s th", + "ãĥ ¼", + "el in", + "ton a", + "cinnam on", + "ðŁİ ¨", + "ðŁİ ģ", + "ron aldo", + "pen insu", + "ome ga", + "el ds", + "desig ning", + "e igh", + "blu et", + "ben z", + "nu g", + "ash a", + "robo ts", + "su dan", + "choo sing", + "en do", + "ser ge", + "clo sely", + "hand y", + "fing er", + "be ing", + "ar te", + "survi ved", + "fl ame", + "mile stone", + "gu t", + "d war", + "fu tures", + "é e", + "el o", + "fri dge", + "eli c", + "ou ch", + "u b", + "p v", + "tit an", + "col lar", + "st ation", + "nev ada", + "aur ora", + "r d", + "dun can", + "âģ ł", + "bri en", + "mar sh", + "Ð ¾", + "to tal", + "ch ry", + "s ers", + "su ffe", + "ra chel", + "colle ge", + "to days", + "cour ts", + "ch it", + "re united", + "gym na", + "gen esis", + "be side", + "re presentation", + "ch ant", + "collec tor", + "ra k", + "ath ens", + "ni gh", + "mun ich", + "langu ages", + "fl u", + "particip ation", + "__ _", + "c v", + "spec trum", + "so da", + "co ver", + "refe ren", + "ab bo", + "ap a", + "public ation", + "ed m", + "mon ica", + "ar my", + "ðŁļ Ģ", + "div or", + "dr y", + "stre ams", + "robo tics", + "ci der", + "bull ying", + "appro val", + "sto ke", + "plat forms", + "sier ra", + "ex tin", + "i b", + "ha yes", + "succe ed", + "suff er", + "at ically", + "da i", + "lyn ch", + "h ound", + "del ines", + "ack now", + "d ated", + "exclu sively", + "he res", + "fac ilit", + "dam aged", + "char ter", + "la kers", + "fal con", + "unve iled", + "wel ove", + "e ase", + "pati ence", + "l one", + "gent le", + "gene tic", + "produc ing", + "g our", + "shann on", + "bil ities", + "zimbab we", + "p int", + "dau ghters", + "liter ary", + "bel le", + "cl am", + "surroun ded", + "k any", + "ne il", + "pir ate", + "rang er", + "hb d", + "nat alie", + "bel ong", + "olym pi", + "emb assy", + "sc ol", + "en er", + "ak in", + "lo ren", + "b h", + ": /", + "di va", + "den im", + "hi pp", + "ðŁĩµ ðŁĩ", + "arn old", + "? '", + "we ren", + "em power", + "dis abled", + "man or", + "rasp berry", + "b af", + "aw ful", + "dru mmer", + "kar dashi", + "n ash", + "machine learning", + "ch u", + "rebel s", + "tim ing", + "mon roe", + "ton gue", + "ran ge", + "pup ils", + "re ss", + "amaz on", + "b z", + "har ley", + "pal mer", + "ballo on", + "s ings", + "ic ec", + "j b", + "c ers", + "g ps", + "whi st", + "ri se", + "l t", + "oo oo", + "c attle", + "shoo ter", + "vod ka", + "uc l", + "mt g", + "le sli", + "jon as", + "di spo", + "at ric", + "ste in", + "vintag e", + "fir ms", + "flo yd", + "cow boy", + "soo oo", + "is aac", + "war craft", + "disney land", + "beauti ful", + "be am", + "franch ise", + "bu n", + "k ag", + "an on", + "tur bo", + "swee p", + "made in", + "kar achi", + "dete ctive", + "penn sylvania", + "contro versi", + "vitam in", + "a side", + "chron ic", + "descri bes", + "remo val", + "ha h", + "ap er", + "ten ed", + "u to", + "bad ly", + "mir ac", + "f ry", + "ye a", + "in jec", + "ther mal", + "comp act", + "th or", + "te ed", + "ur gent", + "l ite", + "g illi", + "sop hom", + "ic o", + "che m", + "p m", + "for k", + "fre ak", + "ch ak", + "recipi ent", + "i y", + "ni k", + "model ing", + "c ans", + "ðŁı Ģ", + "del ux", + "se am", + "surviv ors", + "rad ical", + "investig ating", + "reli able", + "f m", + "tur t", + "ligh thouse", + "to ol", + "go wn", + ") )", + "bo ts", + "auto graph", + "a id", + "bu ffe", + "h mm", + "horri ble", + "ssi onal", + "ann i", + "๠Ģ", + "k its", + "sch i", + "eter nal", + "hu ss", + "sens itive", + "r u", + "tast es", + "chec ks", + "im o", + "por tion", + "sk ate", + "e den", + "half time", + "fri ed", + "ri hanna", + "ti se", + "fl ick", + "ca in", + "s gt", + "âľ Ķ", + "sh au", + "sta ined", + "ra ffle", + "dro ve", + "sal man", + "princi ples", + "sh o", + "ar u", + "je ss", + "gu ine", + "gar bage", + "my an", + "jel ly", + "dis ru", + "z ia", + "q ld", + "ent ries", + "la v", + "fle w", + "ad mit", + "objec ts", + "comp are", + "ny times", + "cann es", + "p n", + "suff ol", + "ro c", + "d ana", + "e gg", + "hi st", + "coun sel", + "' !", + "phy si", + "imag ination", + "ad just", + "explo sion", + "plym outh", + "hor ror", + "elli ott", + "bour ne", + "de x", + "bre ed", + "au dio", + "lob ster", + "disappo inted", + "nation wide", + "( (", + "incre ases", + "austr ali", + "ce dar", + "star ing", + "rac ial", + "e is", + "g mt", + "visi ons", + "stay ed", + "discu ssions", + "de an", + "cur tis", + "mai den", + "stel lar", + "happ iest", + "h wy", + "pre season", + "car av", + "mon days", + "hospit als", + "glimp se", + "schol ars", + "ja i", + "ter race", + "ann a", + "goo se", + "gra ded", + "lot us", + "hun g", + "grocer y", + "stam ps", + "emper or", + "sc oop", + "in ser", + "c as", + "exist ence", + "he al", + "fal cons", + "mar vel", + "reduc ing", + "terri fic", + "magne tic", + "perfor ms", + "bar re", + "p us", + "tre ating", + "ic on", + "w h", + "decla red", + "tra uma", + "do d", + "come dian", + "nik on", + "bu gs", + "as m", + "mont gom", + "ibi za", + "comprehen sive", + "ha s", + "san ti", + "fellow ship", + "da sh", + "p sal", + "louis ville", + "sp y", + "fau lt", + "d the", + "fi led", + "vi sta", + "de sc", + "fe ars", + "you tu", + "sp s", + "es p", + "ri g", + "cri me", + "ber ger", + "wonder land", + "k ent", + "in formed", + "stev ens", + "my th", + "ast on", + "ir i", + "visit or", + "at ri", + "produc ers", + "al la", + "person ally", + "separ ate", + "agen cies", + "af ri", + "il an", + "spo ke", + "n ina", + "squ ad", + "di ves", + "de pend", + "li v", + "fier ce", + "enter taining", + "cha in", + "sc at", + "bor ders", + "pal ette", + "sp ro", + "os is", + "der by", + "tobac co", + "zi o", + "willi e", + "ju vent", + "zoo m", + "hol y", + "enti rely", + "af e", + "mart inez", + "be ds", + "pe a", + "bull dogs", + "ðŁĩª ðŁĩ", + "ib m", + "ne on", + "ethiop ia", + "team mates", + "plan ting", + "tw er", + "any time", + "for bes", + "ó n", + "run way", + "ner vous", + "ro ger", + "p ile", + "ch anc", + "apo caly", + "u w", + "o i", + "dr ought", + "territ ory", + "br ick", + "cre atures", + "go in", + "w aff", + "gre n", + "sou theast", + "je an", + "am bul", + "ed ited", + "stra p", + "c v", + "aar on", + "ãĥ» ãĥ»", + "t su", + "descri ption", + "kin dly", + "clu tch", + "im mer", + "en or", + "women sday", + "or ange", + "ra g", + "ob vious", + "hy der", + "chann els", + "man go", + "me yer", + "ra ining", + "ge tty", + "pil gri", + "coordin ator", + "up load", + "ninten do", + "don uts", + "san chez", + "app arel", + "j r", + "zz i", + ", @", + "jeff erson", + "accessi ble", + "great ly", + "e id", + "initi al", + "budd ha", + "par is", + "ma scot", + "â¬ĩ ï¸ı", + "sch war", + "si ri", + "sp inning", + "mortg age", + "e cho", + "end ange", + "ge dly", + "chlo e", + "enh ance", + "kar nat", + "k ry", + "explo res", + "ðŁĴ ģ", + "af fair", + "ic als", + "all a", + "dar t", + "dolph ins", + "diffe rences", + "squir rel", + "au gh", + "dr ones", + "ell en", + "re store", + "pa w", + "un for", + "pi ke", + "hil ton", + "colla b", + "consu mers", + "co inci", + "out comes", + "pp p", + "a q", + "coup on", + "li est", + "si ms", + "k ho", + "av es", + "spo on", + "pu dding", + "cor byn", + "hat ers", + "ex ams", + "sla ve", + ". !", + "p sa", + "app les", + "tam il", + "se d", + "co ke", + "zz o", + "lo sange", + "car bon", + "cla ir", + "... )", + "k hu", + "cra ig", + "explor ation", + "sanctu ary", + "su e", + "al way", + "demen tia", + "won ders", + "super hero", + "pakistan i", + "brown s", + "bluet ooth", + "lo cker", + "mar c", + "ev entu", + "delux e", + "rodri guez", + "âĿ¤ âĿ¤", + "ro bb", + "ðŁĴ ¦", + "lin ux", + "ten s", + "intellig ent", + "se ed", + "vo ter", + "s ler", + "pe aks", + "inter n", + "teen age", + "peninsu la", + "hand ling", + "ti e", + "cou sins", + "wen dy", + "me e", + "à¹Ģ à¸", + "din o", + "ðŁĴ °", + "ðŁĺ ĥ", + "ze e", + "s bury", + "trage dy", + "b k", + "bo re", + "z in", + "war ns", + "idi ot", + "tou ching", + "contin ental", + "tac os", + "saf ari", + "wa shed", + "po dium", + "morri son", + "fore sts", + "c bc", + "al on", + "partic ular", + "be ads", + "inv ented", + "lo ch", + "li ghter", + "where ver", + "i de", + "docu ments", + "a we", + "k r", + "no where", + "min er", + "st it", + "ro x", + "contribu te", + "har dy", + "cl an", + "ob ject", + "ca it", + "ðŁĴķ ðŁĴķ", + "happ ier", + "vege tables", + "t art", + "g ag", + "nom inee", + "heav ily", + "pan ic", + "j d", + "there sa", + "at m", + "u ph", + "s fc", + "su ri", + "drin k", + "n al", + "re vel", + "k l", + "avoc ado", + "nom ination", + "ma donna", + "shar on", + "malcol m", + "control led", + "sh ers", + "revi val", + "legis lation", + "shoo ts", + "n in", + "comm entary", + "pro s", + "human rights", + "str anger", + "mit ch", + "pipel ine", + "leg ally", + "th u", + "gil bert", + "tol l", + "gran ted", + "gh s", + "ir anian", + "refre shing", + "du k", + "ab i", + "pri me", + "jose ph", + "mo sa", + "stati stics", + "produc tions", + "mer ry", + "pat el", + "sa x", + "human itarian", + "struc tures", + "e missions", + "town s", + "fre el", + "ster ing", + "rat ings", + "alle gedly", + "cab in", + "st l", + "w ade", + "fl yers", + "tri m", + "promis ing", + "z u", + "bal lot", + "compar ison", + "free ze", + "ou ter", + "great ness", + "as sign", + "snow y", + "r ale", + "tor ies", + "med iter", + "kno ck", + "consult ant", + "cincin nati", + "analy st", + "sc oo", + "je ws", + "appro xim", + "pu re", + "portra its", + "cy rus", + "ation al", + "lo ans", + "acqu is", + "el u", + "accep table", + "uni on", + "water color", + "ru st", + "batt les", + "per fu", + "seas onal", + "ser ial", + "mind set", + "ri ot", + "fel d", + "enni al", + "clo set", + "pri est", + "tan ks", + "int l", + "scre w", + "bu m", + "ab dul", + "ou x", + "expla ined", + "ric a", + "imag ing", + "law yers", + "bu ried", + "ãĥ»ãĥ» ãĥ»", + "ear l", + "âĢ ķ", + "l ton", + "resto red", + "stri pes", + "fo ss", + "de mands", + "ste aling", + "alex is", + "mun d", + "ak er", + "ur us", + "war dro", + "hu gs", + "gen re", + "e go", + "Ù Ħ", + "particip ated", + "bab es", + "ban quet", + "ti ous", + "he mi", + "ds b", + "lo st", + "milwau kee", + "jen ner", + "ge m", + "ou tra", + "lo ses", + "id i", + "re ps", + "ðŁİ §", + "regu lation", + "fla w", + "f ang", + "vibr ant", + "ram p", + "ra ins", + "well being", + "so viet", + "vie wers", + "de po", + "libr aries", + "bi go", + "ser y", + "g ill", + "de struction", + "co z", + "c x", + "bri dal", + "al ds", + "plan ted", + "amate ur", + "lu d", + "che ering", + "show cas", + "pro file", + "i u", + "ver tical", + "pack ers", + "wiz ard", + "ski p", + "s light", + "be au", + "air ways", + "mu ch", + "re ra", + "ðŁĮ Ĭ", + "ab sor", + "pati o", + "pack ages", + "s ells", + "ment ally", + "ðŁĺ ¢", + "reyn olds", + "k are", + "tri bun", + "wal t", + "kn it", + "ta ste", + "sur rey", + "boun ce", + "cre ature", + "b are", + "bet ting", + "su re", + "mi ley", + "laugh s", + "al ore", + "cy n", + "t l", + "arti st", + "ann ah", + "war mer", + "dynam ics", + "lunch time", + "mariti me", + "vulner able", + "ðŁĴ ĥ", + "wol ver", + "dur ham", + "const antly", + "am in", + "si bl", + ": @", + "bul let", + "k ach", + "angel o", + "wil der", + "doo m", + "desk top", + "law suit", + "k ca", + "hen derson", + "inv iting", + "bet ty", + "ta wards", + "ra fa", + "le aked", + "and i", + "ge ms", + "af l", + "vel o", + "mediter ran", + "pro be", + "to tten", + "steph anie", + "sn ation", + "com be", + "q s", + "over come", + "assas sin", + "ra v", + "fil ip", + "winni peg", + "sh il", + "determin ed", + "k as", + "ou tre", + "regre t", + "gui des", + "aa a", + "ðŁĺ Ī", + "wi ves", + "mani fe", + "er ly", + "sm y", + "sh ima", + "x ing", + "pix el", + "jac ob", + "ac commod", + "to y", + "on o", + "po o", + "ti er", + "an swe", + "ðŁĴ ģ", + "ro sa", + "le ase", + "bel ongs", + "th ar", + "eventu ally", + "nei ther", + "go a", + "ski ing", + "at ra", + "ag h", + "broad casting", + "f ury", + "py ram", + "d ice", + "volk swag", + "wom ens", + "provi der", + "bom bs", + "miss ile", + "whi p", + "d ick", + "nor we", + "back up", + "el der", + "mat ure", + "concer ts", + "gi ous", + "sque e", + "good morning", + "bra ves", + "^ _", + "au ssie", + "lun a", + "mal es", + "he ck", + "for tn", + "rome o", + "steel ers", + "p n", + "pe er", + "re presents", + " «", + "kat y", + "migu el", + "requ ire", + "cha ins", + "l ur", + "immedi ate", + "ti mber", + "âĸ¶ ï¸ı", + "advoc acy", + "ex port", + "an z", + "tiff any", + "auth or", + "ðŁİ Ī", + "du des", + "chil ly", + "hi d", + "har m", + "bu g", + "mon ster", + "terri er", + "tu c", + "story telling", + "ta k", + "in ti", + "immigr ants", + "b is", + "reach es", + "com passion", + "john ny", + "contribu tions", + "ðŁIJ ¶", + "mechan ical", + "impre ssion", + "ran ks", + "ko be", + "men ting", + "bloss om", + "pab lo", + "buil der", + "bom bing", + "tw el", + "sul livan", + "om o", + "pe te", + "de mi", + "ku dos", + "w bb", + "t gif", + "mass ach", + "neighb or", + "che fs", + "eng ines", + "pun e", + "ga ined", + "phan tom", + "s days", + "ext end", + "gr an", + "cent ers", + "jac qu", + "dat asci", + "sleep y", + "el vis", + "answe red", + "s lot", + "con y", + "flexi ble", + "ti ally", + "le tics", + "% ,", + "andre ws", + "si ble", + "mom ma", + "vin o", + "do x", + "invit ational", + "twil ight", + "j ade", + "ill ery", + "joh ns", + "f ou", + "p v", + "-- ->", + "break down", + "billi on", + "prin ter", + "mon d", + "c bc", + "mag gie", + "legi on", + "du b", + "kur t", + "po or", + "paren ting", + "regi ons", + "bikin i", + "be ware", + "si onal", + "au burn", + "kid ding", + "amp les", + "sp an", + "con tempor", + "c ic", + "ha bits", + "ak o", + "pre fe", + "bud dies", + "it z", + "em ily", + "person nel", + "moun tain", + "ver sus", + "ðŁĺ ¬", + "ear ning", + "s ink", + "dar i", + "u u", + "s win", + "i ster", + "bru tal", + "n ac", + "kat a", + "clo th", + "am and", + "ðŁĶ Ĺ", + "ne o", + "alu min", + "week ends", + "nebra ska", + "co des", + "delay ed", + "brun o", + "pro ven", + "in c", + "i ght", + "fl an", + "or o", + "lam bert", + "regu lat", + "w f", + "massach use", + "kardashi an", + "bern ard", + "fi esta", + "volcan o", + "grand pa", + "anc a", + "d re", + "st itu", + "mean ing", + "fo am", + "au ck", + "at ed", + "r l", + "hot el", + "pers ons", + "dy nasty", + "ell or", + "ma i", + "am ne", + "sty ling", + "avi er", + "e g", + "vege tarian", + ", âĢ¦", + "foun ders", + "sta in", + "g d", + "cy cles", + "sky line", + "trac tor", + "exi sts", + "tra l", + "kid ney", + "mar il", + "inst ag", + "se tte", + "addic t", + "tri angle", + "flash back", + "controversi al", + "z on", + "p ins", + "i as", + "tr ay", + "town ship", + "deleg ates", + "sp am", + "h ms", + "cr ane", + "peop les", + "o lo", + "fac tion", + "but es", + "on ica", + "deleg ation", + "new profile", + "eli er", + "mc a", + "w and", + "g ely", + "losange les", + "ber ke", + "ti ve", + "dis rup", + "zz a", + "cas a", + "jor dan", + "ford shire", + "ga thered", + "ic hi", + "atten dees", + "à¸Ń à¸", + "pe ppers", + "co in", + "bour bon", + "ern ity", + "ro tary", + "behavi our", + "jere my", + "team work", + "compli ance", + "tre mend", + "ðŁĩ §", + "bu hari", + "cam bo", + "bu yers", + "ha gen", + "bu ds", + "bay ern", + "mon te", + "sm ells", + "an za", + "ath lon", + "descri bed", + "work force", + "gi ving", + "ap i", + "invest ments", + "da il", + "sel ena", + "datab ase", + "th um", + "mor tal", + "stu dent", + "bu yer", + "do ver", + "gar ten", + "att le", + "loy alty", + "gen oci", + "holo cau", + "theat ers", + "ru ling", + "ven us", + "pat ent", + "ch un", + "ab by", + "awa ke", + "mass acre", + "bang alore", + "break ing", + "simm ons", + "ju sti", + "hal e", + "ed chat", + "gg les", + "haw k", + "mar king", + "head lines", + "stro m", + "co ve", + "breath taking", + "med als", + "hair cut", + "christ ine", + "tele graph", + "gujar at", + "ju ra", + "can e", + "sho re", + "propag anda", + "mu eller", + ".... ....", + "sa vi", + "stom ach", + "thro ws", + "ta b", + "war m", + "j ong", + "reno wned", + "hi r", + "ra is", + "mush rooms", + "guaran teed", + "bo a", + "m j", + "revolu tionary", + "certi fication", + "bru ins", + "jo in", + "w es", + "pas sport", + "c g", + "sex u", + "cap able", + "w v", + "ton es", + "jac kets", + "ac compan", + "spin ach", + "fore ver", + "bla ir", + "wat ts", + "g l", + "cou ples", + "prairi e", + "newprofile pic", + "logi stics", + "massachuse tts", + "jagu ar", + "o id", + "we al", + "under water", + "mo z", + "y i", + "ma ths", + "myan mar", + "pre ps", + "suffe red", + "tr ace", + "wal i", + "ah hh", + "bor g", + "st itch", + "cu lin", + "real ise", + "infe ction", + "discrimin ation", + "sh ame", + "an kle", + "hu mid", + "y t", + "brac ket", + "tru ck", + "tri u", + "ea ster", + "commun ity", + "post card", + "invol ving", + "ty ler", + "car amel", + "over view", + "ex amples", + "integr ity", + "base ment", + "instru ments", + "ani um", + "at us", + "gh er", + "laun dry", + "achi eve", + "gen eva", + "pr icing", + "hyder abad", + "beli ef", + "me ta", + "j aw", + "accoun ting", + "lead er", + "cristi ano", + "cou ture", + "cy p", + "vis ed", + ", ,,", + "k nu", + "h ick", + "break er", + "br am", + "ra b", + "mo or", + "ham as", + "gradu ating", + "pupp ies", + "ak h", + "ta h", + "ach es", + "ri e", + "op ini", + "g ta", + "re ign", + "tra gic", + "re ver", + "p ill", + "pine apple", + "tou ches", + "da re", + "le ys", + "il o", + "inter iors", + "sc outs", + "bar t", + "en zie", + "don o", + "bro ck", + "christi ans", + "ense mble", + " ·", + "cine mas", + "new port", + "air line", + "win ston", + "le igh", + "cont ents", + "pre scri", + "ur ge", + "tr out", + "fic ally", + "il ia", + "sub si", + "are r", + "âļ¾ ï¸ı", + "w ounded", + "ðŁĻ Ĥ", + "pe pper", + "ðŁĴ ŀ", + "fit ted", + "af f", + "re sur", + "thursday thoughts", + "z ero", + "archae ology", + "di v", + "je e", + "i on", + "awa iting", + "co zy", + "beauti es", + "bal d", + "dat a", + "gri zz", + "stal k", + "kin ds", + "cle ared", + "jess ic", + "regu lar", + "ali ens", + "plac e", + "bo s", + "bi zar", + "thisi s", + "ðŁĴ Ģ", + "totten ham", + "ma fia", + "s lam", + "ari ana", + "car roll", + "back pack", + "care y", + "uni v", + "r g", + "pe p", + "dig it", + "tatt oos", + "ag on", + "volunte ering", + "diffe ren", + "consu mption", + "ka thr", + "head phones", + "t shirt", + "o b", + "ele ment", + "re tail", + "sh ru", + "al gori", + "contain er", + "consci ous", + "fi l", + "com ing", + "ra sh", + "u rope", + "def ine", + "gi or", + "femini st", + "flow ing", + "rout es", + "gl aci", + "fer t", + "somer set", + "ant es", + "twee ps", + "$ $", + "h our", + "endange red", + "year sof", + "ro h", + "po pped", + "bac king", + "ba sil", + "bra ke", + "mon aco", + "lgbt q", + "pra gue", + "ut ility", + "cas si", + "gate way", + "haun ted", + "sch ul", + "ðŁİ µ", + "shou ld", + "walking dead", + "comple ting", + "dann y", + "montgom ery", + "pengu in", + "ss i", + "mer chandi", + "ðŁij ij", + "chur ch", + "h ates", + "cap tain", + "brea thing", + "ce t", + "fair ly", + "approach es", + "compan ion", + "surpri sing", + "kany e", + "pe y", + "hin di", + "targe ted", + "lor ds", + "de ut", + "di gging", + "ger man", + "ru t", + "ener gy", + "close st", + "y un", + "apo logi", + "ภ±", + "s ack", + "ru p", + "dd y", + "port al", + "d ough", + "b ats", + "ðŁĵ °", + "at ur", + "graph er", + "pi res", + "mo tors", + "ðŁĮ ¹", + "j c", + "dan g", + "tu k", + "clu e", + "us c", + "pag e", + "d less", + "bro ws", + "ju s", + "ad ing", + "re marks", + "oo m", + "car dio", + "ste fan", + "arm strong", + "âĢ¢ âĢ¢", + "ni est", + "belgi an", + "bi op", + "so y", + "lo f", + "í ĥ", + "q t", + "flashback friday", + "ce e", + "ģ à¸", + "wre ck", + "mar ines", + "amend ment", + "wardro be", + "vo y", + "bur ned", + "guit ars", + "ra inf", + "li fel", + "ssi l", + "oun ce", + "exter nal", + "c key", + "me sh", + "she ikh", + "inv itation", + "sugge sti", + "pop corn", + "phenomen al", + "an onymous", + "tun a", + "chic ago", + "o val", + "del y", + "loc als", + "( &", + "pro f", + "no vel", + "fin der", + "spar ks", + "la ven", + "in fu", + "nic ks", + "qu ant", + "ra e", + "exe c", + "dist ingui", + "st ances", + "mu tual", + "sh al", + "unve ils", + "edmon ton", + "zan ia", + "a dio", + "vie wer", + "brad ford", + "audit orium", + "qu is", + "re act", + "htt p", + "l ero", + "chee ky", + "impac ts", + "ta k", + "ed t", + "desper ate", + "t ay", + "ì Ħ", + "sett le", + "bar gain", + "resu me", + "un ite", + "thro wn", + "ke st", + "se ys", + "mar ching", + "am it", + "decl ine", + "sch ar", + "me tr", + "stan ford", + "lin ke", + "ber ra", + "dol ls", + "rug by", + "jam i", + "b or", + "road trip", + "dino saur", + "mi k", + "sun der", + "re m", + "b k", + "over seas", + "nau ghty", + "imple mentation", + "iam srk", + "lun cheon", + "fir ing", + "mi ami", + "pere z", + "the e", + "z on", + "gi fted", + "con version", + "ceram ic", + "¡ ï¸ı", + "pe dro", + "ì Ĩ", + "v ick", + "! @", + "he ed", + "si d", + "b w", + "docu ment", + "pl un", + "gr ants", + "fant asy", + "predic tions", + "vali d", + "car ved", + "gradu ated", + "ðŁijį ðŁı»", + "nation ally", + "ch y", + "af l", + "re sso", + "blan k", + "ri vals", + "j ig", + "e ties", + "om ics", + "une mp", + "b ound", + "sk o", + "inspec tion", + "par al", + "high s", + "cri sp", + "b ans", + "ob a", + "[ @", + "co spla", + "costu mes", + "rec all", + "mou th", + "ni gel", + "b ts", + "ter a", + "ko v", + "do cs", + "west minster", + "dic t", + "gra vity", + "kar i", + "ro gue", + "t ted", + "war k", + "ida ho", + "w end", + "aw i", + "queen sland", + "proce sses", + "cli ffe", + "m ick", + "com pens", + "op ol", + "the y", + "cl ari", + "wiki pedia", + "salman khan", + "haz ard", + "pre ston", + "swee test", + "pd f", + "che es", + "tr ilo", + "south africa", + "bur nt", + "( $", + "con tain", + "t p", + "sub mitted", + "sound cloud", + "at u", + "re z", + "word press", + "corru pt", + "n f", + "ma ker", + "í ķ", + "par as", + "adv ent", + "ri al", + "ca fe", + "fo ssil", + "!!!! !!!", + "co ws", + "c j", + "sp ur", + "institu tions", + "land mark", + "ent it", + "re ut", + "h is", + "alz heim", + "we mb", + "regg ae", + "mo squ", + "st at", + "identi fied", + "deal er", + "re am", + "re land", + "ten sion", + "ðŁĩ ©", + "wra pping", + "deep er", + "fr at", + "red dit", + "ar is", + "moroc co", + ".. \"", + "b low", + "ma pping", + "pri orities", + "ing a", + "swa p", + "re wards", + "conspir acy", + "creati ve", + "c j", + "congre ssional", + "vau lt", + "ple x", + "sophom ore", + "shad ow", + "ele ss", + "ðŁĺ ħ", + "dar ts", + "aldu b", + "anno ying", + "pro ps", + "n as", + "alumin um", + "h bo", + "offen se", + "j ill", + "oni ons", + "la ur", + "ta e", + "har dest", + "sh ro", + "ga ining", + "meas ure", + "ed tech", + "cyp rus", + "tar a", + "ang eli", + "car lo", + "go on", + "all i", + "im plic", + "ju pit", + "resil ience", + "ha il", + "bal anced", + ") ...", + "joy ce", + "gr a", + "th eli", + "defin ed", + "shi pped", + "main ly", + "min a", + "l m", + "sac ri", + "o ber", + "p im", + "claim ing", + "ent ers", + "co rey", + "bo k", + "cri ed", + "cool ing", + "dani elle", + "pharmac y", + "thor ough", + "ca ke", + "k lo", + "outre ach", + "z ens", + "digital marketing", + "val ent", + "sn p", + "her b", + "mr w", + "caf é", + "cap tures", + "no tre", + "triu mph", + "pan cakes", + "cu mber", + "spi ke", + "d ation", + "bi gg", + "sp er", + "crit ical", + "am al", + "too th", + "foun ding", + "a stro", + "' #", + "quan tum", + "th ames", + "un c", + "pri de", + "air bus", + "kno cked", + "un defeated", + "mediterran ean", + "cal cu", + "clo wn", + "sens or", + "ham mer", + "for give", + "cu shi", + "ber ry", + "maje stic", + "elec t", + "polit an", + "g ta", + "k ari", + "bur ke", + "sea hawks", + "volkswag en", + "re i", + "landsc apes", + "cas u", + "grand father", + "list ened", + "/ /", + "star trek", + "rainf all", + "fur ry", + "vi er", + "star k", + "rif le", + "ff a", + "leg es", + "hillary clinton", + "min us", + "correc tly", + "architec tural", + "pre ce", + "up side", + "box er", + "ðŁĻĮ ðŁı¼", + "is ai", + "de t", + "pro vo", + "tis sue", + "spoo ky", + "ve led", + "re con", + "prospec ts", + "que bec", + "âļ «", + "ig no", + "anat omy", + "shap es", + "w p", + "p interest", + "hor e", + "an es", + "pick up", + "ti p", + "pra desh", + "hu gh", + "co e", + "po k", + "gram my", + "well ington", + "sti gate", + "ri gh", + "lea p", + "king ston", + "scen ic", + "go sh", + "v ani", + "au g", + "s ary", + "zi er", + "bure au", + "lin son", + "con te", + "fra gr", + "all an", + "g aw", + "lan a", + "colli sion", + "surve ill", + "ren ais", + "ar range", + "s ali", + "do in", + "br ance", + "bren dan", + "our se", + "in coming", + "suspen sion", + "à ´", + "l la", + "educ ators", + "in tri", + "da e", + "bio graphy", + "bul gar", + "villa in", + "go thic", + "rw anda", + "e w", + "may or", + "meet up", + "democr at", + "mor gan", + "su dden", + "te sco", + "car rot", + "bom ber", + "mck in", + "re ne", + "fun day", + "agricul tural", + "haha h", + "show time", + "form ing", + "col a", + "scor pi", + "quo te", + "po ppy", + "s life", + "d az", + "tu b", + "ne n", + "mo t", + "ðŁĺ »", + "s ore", + "elder ly", + "o ve", + "skin ny", + "um i", + "anc o", + "man ship", + "we re", + "g v", + "k ah", + "fol ding", + "ne at", + "samanth a", + "dan ish", + "uk rain", + "humid ity", + "nu tri", + "jak arta", + "cand les", + "oooo oooo", + "at ile", + "streng th", + "i bra", + "bap ti", + "charle ston", + "fr ames", + "girl s", + "clear ing", + "glu ten", + "# #", + "super natural", + "ju bi", + "ph one", + "he in", + "dr un", + "le ak", + "invest or", + "y er", + "dom ain", + "ball room", + "mi sh", + "app li", + "off shore", + "bla ze", + "dor o", + "âĺķ ï¸ı", + "win ery", + "shar if", + "ad ore", + "n ir", + "saf er", + "si gh", + "as cri", + "strong ly", + "trac y", + "ck er", + "ol l", + "faith ful", + "ey ed", + "deli ghtful", + "vis m", + "karnat aka", + "tit an", + "wh ar", + "jer seys", + "re fur", + "heav en", + "gri p", + "pan ama", + "pre li", + "glu ten", + "o dd", + "cont ent", + "pon ti", + "tion ing", + "e commerce", + "feder ation", + "flaw less", + "ge ar", + "ti res", + "by r", + "pol ice", + "cu ban", + "tri butes", + "tic ul", + "chur ches", + "nur sery", + "di aries", + "muse ums", + "snapp ed", + "i van", + "wi ght", + "touri sts", + "ramad an", + "t rent", + "prophe t", + "won dered", + "focu sing", + "hi d", + "ic ons", + "i q", + "ambul ance", + "pi st", + "fun niest", + "time less", + "sr ilan", + "bu ys", + "ki ds", + "colour ful", + "a shi", + "ch ir", + "mu m", + "ðŁĵ ļ", + "let ter", + "x en", + "reut ers", + "pre serve", + "in ting", + "ste p", + "fu ji", + "uni ver", + "i u", + "show down", + "po ems", + "surveill ance", + "suspec ted", + "ta e", + "sol ving", + "tom b", + "mother sday", + "car pen", + "recru it", + "pil ots", + "bro c", + "mix ing", + "fri days", + "ty r", + "represent atives", + "tra pped", + "abdu l", + "free style", + "clu ster", + "âļ łï¸ı", + "k d", + "sk ill", + "pit t", + "ex o", + "commer ci", + "muse um", + "loc ally", + "g ina", + "no bel", + "immun e", + "fr ac", + "cap su", + "main ed", + "attemp ts", + "bull dog", + "be spoke", + "sing ers", + "sp elling", + "seg ment", + "nat ures", + "tic k", + "lip stick", + "clean er", + "gett able", + "preci sion", + "âĢ¼ ï¸ı", + "th ood", + "re ef", + "no pe", + "bill y", + "di gi", + "mu si", + "ri val", + "figu red", + "tal ity", + "sun ny", + "ber k", + "aw ww", + "awa its", + "un real", + "co pen", + "asy lum", + "ex otic", + "bu en", + "mo ck", + "en able", + "arch y", + "fr a", + "pla stic", + "al mond", + "amp li", + "displa ys", + "abbo tt", + "s me", + "x p", + "ðŁĻ ĥ", + "graph ic", + "i ved", + "mar a", + "cau tion", + "lea ks", + "en berg", + "ul u", + "unic orn", + "cann on", + "appren tic", + "ðŁĺĺ ðŁĺĺ", + "b ball", + "wil low", + "at ics", + "am as", + "manufac turer", + "campaig ns", + "port ers", + "flo ors", + "l su", + "ty pe", + "ke j", + "honor ary", + "it im", + "to le", + "min ecraft", + "d x", + "ma sh", + "ri o", + "consequ ences", + "ron ald", + "go ssi", + "suffol k", + "mu se", + "r bi", + "live music", + "i van", + "ðŁİ ¤", + "le u", + "patri ot", + "man it", + "lan ca", + "home decor", + "de ar", + "sig ma", + "ti de", + "str ings", + "v ita", + "sequ el", + "try na", + "inve stigate", + "bor is", + "ve gan", + "barri er", + "mind fulness", + "web b", + "hu stle", + "in da", + "tan zania", + "str ay", + "tex as", + "c ag", + "diagno sis", + "wom an", + "g w", + "ob session", + "l ative", + "nu fc", + "fl ynn", + "moment um", + "sof a", + "wal d", + "vege table", + "tu cker", + "supp er", + "se ab", + "ar ro", + "se ag", + "ven ting", + "counc ill", + "sp lat", + "cal cul", + ".. #", + "com fy", + "odi sha", + "sto pp", + "war fare", + "ca es", + "à ¨", + "co y", + "price less", + "in sec", + "ðŁĺ Ľ", + "contro ls", + "empower ment", + "datasci ence", + "per pe", + "gen ic", + "e res", + "tru deau", + "man o", + "sla very", + "expand ing", + "ma he", + "fa iling", + "s aga", + "photograph s", + "cre st", + "re on", + "surf ing", + "hi e", + "ðŁį Ģ", + "ja e", + "fel lows", + "south ampton", + "sol om", + "ce ster", + "tab ility", + "hor n", + "se ct", + "he e", + "cole man", + "at las", + "explo rer", + "consul tation", + "copy right", + "organi zing", + "den ied", + "mon keys", + "noo dles", + "br is", + "fl or", + "dou gh", + "bon ds", + "sho cked", + "eco system", + "care fully", + "w m", + "apart ments", + "cur ve", + "san diego", + "must ard", + "comm en", + "cere mon", + "e ch", + "ru th", + "ðŁĻĮ ðŁı»", + "hawa i", + "fil med", + "te ar", + "as ingly", + "ca ir", + "wat t", + "instru ment", + "ou tta", + "ye ol", + "river side", + "ë °", + ". :", + "nor wich", + "alo g", + "migr ants", + "new man", + "ri de", + "spr ink", + "targe ting", + "beli eve", + "tor ch", + "reflec ts", + "per mission", + "ff man", + "ene mies", + "bas ics", + "se ized", + "sun days", + "le i", + "hass an", + "en do", + "h c", + "st ad", + "le ments", + "kk kk", + "nan o", + "shar k", + "man a", + "on ic", + "treat ments", + "ear ly", + "collabor ative", + "shu ttle", + "bran ches", + "mis ses", + "mained cm", + "ap ers", + "ky le", + "carri e", + "leis ure", + "sh et", + "bir ding", + "adv ances", + "ðŁĵ Ŀ", + "popu lar", + "di ane", + "a be", + "re war", + "neigh bour", + "k pop", + "remem brance", + "play ground", + "ru b", + "krish na", + "e bola", + "inqu iry", + "ep a", + "lu min", + "organ isation", + "abra ham", + "norm ally", + "pre ten", + "jan et", + "w t", + "ðŁĴ İ", + "encoura ging", + "a stic", + "bu mp", + "syd ney", + "s z", + "ss ss", + "gar rett", + "ðŁĵ »", + "consul ting", + "roman ia", + "spo tting", + "chanc ellor", + "ar ma", + "presti gious", + "ðĿ IJ", + "t ad", + "cry st", + "compe tit", + "rati o", + "cat aly", + "bro w", + "j ur", + "vi king", + "commu te", + "y day", + "la yers", + "du mb", + "esc al", + "genoci de", + "f ill", + "gu pta", + "ste pping", + "se i", + "fo to", + "wild cats", + "col i", + "projec t", + "ear nings", + "st r", + "ge ons", + "comple tion", + "b m", + "decor ated", + "craw ford", + "af ghan", + "sc are", + "visi bility", + "hi b", + "direc tion", + "stro ll", + "christ ina", + "alter nate", + "cl are", + "sty list", + "be hold", + "s ance", + "leop ard", + "acqui red", + "narr ative", + "ash i", + "the a", + "?? ??", + "pe as", + "at ch", + "sli des", + "le en", + "renew able", + "eng lish", + "qu ir", + "co aster", + "r x", + "fo ols", + "match day", + "mis m", + "amaz ing", + "z ig", + "ke ting", + "won t", + "to wel", + "di ab", + "sta ke", + "n m", + "mel t", + "e than", + "gra pe", + "polit ician", + "sm en", + "í ĺ", + "re o", + "wedd ings", + "cat cher", + "or acle", + "me mo", + "ðŁĮ ´", + "ec k", + "rob bie", + "norwe gian", + "oper ator", + "am or", + "se wing", + "ju l", + "x ie", + "u v", + "fif ty", + "me ga", + "tatt oo", + "liber als", + "u pri", + "traffic king", + "richard son", + "su v", + "ki p", + "mess y", + "tremend ous", + "gl ou", + "cour tney", + "la d", + "stere o", + "my ers", + "i dio", + "^_ ^", + "man ning", + "dy e", + "w d", + "thr one", + "jun k", + "as u", + "provin cial", + "k ook", + "wr c", + "fine art", + "hamp shire", + "renais sance", + "b red", + "fall out", + "s j", + "sn l", + "al am", + "tor ture", + "fy i", + "sh ines", + "pa w", + "ch ar", + "hen ry", + "c row", + "aci ous", + "di an", + "pa ige", + "ba re", + "stock holm", + "scen ery", + "ðŁĩ ·", + "jef frey", + "pu sh", + "decor ation", + "ne d", + "cu te", + "brig ade", + "laven der", + "inv ites", + "e sports", + "vo ir", + "dri ed", + "tran spl", + "sur geon", + "no vels", + "pul ls", + "son y", + "lun ar", + "man e", + "i vy", + "fru str", + "dor set", + "sa i", + "tor res", + "ssi on", + "shut down", + "suggesti ons", + "writ ing", + "e o", + "battle field", + "u ga", + "ðŁIJ ¾", + "vac u", + "spl ac", + "g it", + "u g", + "high land", + "% )", + "mer maid", + "sacram ento", + "ta ils", + "p w", + "ka h", + "t ell", + "enh anced", + "ì ķ", + "auck land", + "cru el", + "ðŁ¤ ©", + "au dre", + "sail or", + "gram mar", + "g love", + "de on", + "infl am", + "fresh ly", + "k ell", + "zi p", + "christi e", + "mil d", + "di xon", + "instru ctor", + "g ence", + "ãħ ł", + "sub jec", + "constitu tional", + "crow ds", + "in visible", + "ru ins", + "da k", + "si p", + "pla que", + "p ouring", + "comple x", + "z ine", + "ste ad", + "f let", + "trans mission", + "lo way", + "ar un", + "incre asingly", + "au d", + "transp aren", + "cro wned", + "sc oun", + "blizz ard", + "lux u", + "fi ers", + "achieve ments", + "hun ters", + "rock ed", + "bas in", + "vio let", + "pro ves", + "achiev ing", + "pro sper", + "se ga", + "flo at", + "vi an", + "xi v", + "pol ic", + "tur a", + "approxim ately", + "wander lust", + "keep ers", + "geta way", + "co d", + "pol is", + "br yan", + "col ts", + "tal ents", + "yo gur", + "gluten free", + "wri st", + "gr y", + "cze ch", + "ðŁİ Ī", + "ev ille", + "ðŁı Ī", + "to x", + "dani els", + "am er", + "bi ds", + "weare one", + "me tab", + "g t", + "boy z", + "pd x", + "pos session", + "pu shed", + "shr ine", + "reali stic", + "tri gger", + "na vi", + "ru mors", + "n af", + "jen kins", + "tr un", + "comm uni", + "à Ĺ", + "gam ers", + "arm or", + "moham med", + "bal cony", + "y ah", + "stron gest", + "rhy thm", + "unfor gettable", + "k p", + "ho bb", + "custo dy", + "greg or", + "r ita", + "aes thetic", + "il ation", + "sponsor ing", + "n ay", + "kid napp", + "sh s", + "ra jas", + "me g", + "signific antly", + "butt ons", + "la c", + "ver sions", + "essenti als", + "opini ons", + "k ro", + "d printing", + "wi dely", + "d k", + "ur an", + "y al", + "reque sted", + "c n", + "cur ric", + "plu m", + "gr un", + "v m", + "dev on", + "m yo", + "rel ation", + "juvent us", + "rou ge", + "min ority", + "min es", + "jupit er", + "n ine", + "oxy gen", + "fran kie", + "une sco", + "fab ric", + "disgu sting", + "sal man", + "dete ction", + "lan ka", + "d ac", + "ðŁĩ« ðŁĩ·", + "argu ment", + "shel ves", + "cel tics", + "rober to", + "pi gs", + "he dge", + "fau l", + "pow ering", + "butter flies", + "fi r", + "re make", + "att i", + "com o", + "emp ha", + "kend all", + "poke mon", + "se ating", + "d ans", + "bald win", + "ðŁij »", + "lesli e", + "one direction", + "ti mber", + "im an", + "fon t", + "e der", + "di on", + "ste ph", + "for mat", + "gre gory", + "pro p", + "he x", + "ru in", + "sor y", + "inf er", + "n aw", + "bar ak", + "sd gs", + "kar ao", + "lu sh", + "v ander", + "end ent", + "g is", + "a fro", + "soc cer", + "ay an", + "t uni", + "lun g", + "da yof", + "alex a", + "mar ath", + "addic ted", + "ag ile", + "hy gi", + "light weight", + "ì §", + "mand ela", + "jo ey", + "anc y", + "hu m", + "bi r", + "memor ial", + "jim in", + "ging er", + "v ak", + "jav ascri", + "cro ps", + "orig ins", + "d ari", + "pi per", + "im port", + "aggre ssive", + "predic tion", + "re pairs", + "cr acker", + "voy age", + "ni ke", + "mu mmy", + "linke din", + "country side", + "bor der", + "gla ss", + "per t", + "s als", + "sho e", + "autograph ed", + "wal nut", + "colle gi", + "sal ary", + "pa iring", + "ðŁĮ ¸", + "cath ol", + "swee the", + "defe ats", + "streng then", + "roof top", + "impro vements", + "barri ers", + "ur u", + "t ally", + "ru led", + "ðŁĨ ļ", + "nai ja", + "emo ji", + "per cent", + "gi o", + "pro bs", + "on ce", + "adm its", + "pa ths", + "li ar", + "day tona", + "pe ters", + "cal i", + "cal li", + "mu g", + "o sa", + "ap h", + "ab y", + "hy de", + "eth nic", + "pla ins", + "ol f", + "haha hahaha", + "holi c", + "?! ?!", + "su bli", + "bl acks", + "mo t", + "gh ton", + "lo vin", + "b rent", + "bar u", + "l ati", + "de w", + "ate au", + "q a", + "pain ful", + "bu sters", + "st atic", + "ðŁĩ¨ðŁĩ ¦", + "note book", + "out fits", + "si es", + "r f", + "floo ds", + "Ñ Ģ", + "thro at", + "su ici", + "ro vers", + "beng al", + "pre pares", + "blo g", + "mini ature", + "Ø ¨", + "am phi", + "com b", + "r sp", + "in timate", + "green e", + "Ì ĩ", + "al tar", + "surg ical", + "ves sel", + "... ?", + "gav in", + "g ator", + "threat ened", + "z ar", + "rob bery", + "di er", + "promo ted", + "y g", + "x s", + "su bs", + "inter viewing", + "threat ening", + "do zen", + "me ado", + "water fall", + "nintendo switch", + "cal um", + "mini sters", + "dro p", + "univers ities", + "war ned", + "tac tics", + "ðŁĩ ²", + "refu se", + "ad ju", + "v ast", + "ðŁĺ ´", + "mc fc", + "lib ya", + "no filter", + "distribu ted", + "re ser", + "ron nie", + "de co", + "javascri pt", + "mon k", + "intere sts", + "fle x", + "mar tha", + "sti es", + "oo d", + "ðŁ¤£ ðŁ¤£", + "e un", + "b ali", + "g omez", + "sti mul", + "moder ate", + "d ity", + "ir is", + "stra w", + "consist ent", + "direc tions", + "adop t", + "sal sa", + "cro o", + "reco vered", + "black friday", + "lan caster", + "accep t", + "weareone exo", + "buil ds", + "free man", + "air plane", + "diti on", + "bel ong", + "jam ie", + "pit ching", + "li f", + "om in", + "cri spy", + "pre pping", + "ve g", + "chan g", + "accompli shed", + "graci as", + "dolph in", + "elec tor", + "culin ary", + "super bowl", + "wal a", + "pur suit", + "black berry", + "be an", + "cardin al", + "pro ved", + "immigr ant", + "stric tly", + "holocau st", + "pass age", + "ha us", + "cou p", + "pur se", + "har ass", + "< <", + "le ed", + "ado be", + "st ad", + "legis lat", + "par ked", + "pri yan", + "sil va", + "kri st", + "s the", + "fun ky", + "ig a", + "sett lement", + "ph s", + "t mrw", + "stre ssed", + "hun t", + "ho ckey", + "treas ures", + "cham bers", + "ol u", + "hu t", + "mar ley", + "tex ture", + "wilder ness", + "mm ing", + "poten tially", + "om aha", + "ju dy", + "to es", + "spo iler", + "distingui shed", + "feli x", + "ah u", + "recommend ations", + "zom bies", + "hit ler", + "tri ple", + "colla pse", + "motiv ated", + "ulti mat", + "gg ling", + "so y", + "ci gar", + "fo ren", + "vine yard", + "gl itter", + "fin dings", + "colon ial", + "hun ter", + "eri k", + "den s", + "beet le", + "lot te", + "sub tle", + "s matter", + "tru sted", + "experim ental", + "nam ents", + "ðŁĺ Ĩ", + "regi on", + "acquis ition", + "bre eding", + "quarter back", + "am reading", + "oo td", + "ru de", + "initi atives", + "st out", + "hy ung", + "out come", + "al fred", + "mic s", + "exper tise", + "bacter ia", + "pengu ins", + "jump er", + "valen cia", + "bar k", + "ing day", + "sell ers", + "contrac ts", + "hou ston", + "commissi oned", + "adap tation", + "swan sea", + "santi ago", + "common wealth", + "ju dging", + "sub mission", + "sco rer", + "tom my", + "ñ o", + "ex quis", + "fil ing", + "explan ation", + "alli son", + "wemb ley", + "ri dge", + "chev y", + "san tos", + "own ership", + "cogn itive", + "favour ites", + "sh ed", + "phil anthro", + "dele ted", + "go dd", + "s nor", + "gui delines", + "ff ing", + "je ep", + "cli ps", + "sw amp", + "an or", + "guil d", + "bol ton", + "spring field", + "munici pal", + "goal keeper", + "ye on", + "ðŁĺįðŁĺį ðŁĺįðŁĺį", + "ãħĭ ãħĭ", + "water front", + "gra ve", + "contempor ary", + "ar ity", + "ÃŃ a", + "sle eps", + "sy rup", + "al am", + "pi re", + "co yo", + "moto gp", + "ty son", + "kej ri", + "cir cul", + "sing ly", + "cr unch", + "complic ated", + "nostal gia", + "k op", + "mo ve", + "k ale", + "mac ro", + "mid west", + "h ans", + "tri bal", + "nu de", + "௠į", + "bey once", + "congratul ate", + "cat er", + "leagu e", + "ðŁĻ Ĭ", + "la dder", + "cra shed", + "tech nic", + "karao ke", + "harass ment", + "ro ts", + "experi encing", + "kri sten", + "ðŁĩ ³", + "ðŁ¤ Ĺ", + "reflec tions", + "guin ness", + "illustr ator", + "ðŁĻı ðŁı»", + "cen ter", + "nar row", + "comm ons", + "regul ations", + "Ù Ĩ", + "har m", + "cro ft", + "cu ssion", + "hong kong", + "st ical", + "intern ship", + "zo e", + "cho p", + "hoo ds", + "estim ated", + "batter ies", + "berke ley", + "smooth ie", + "shau n", + "cro s", + "~ ~", + "cam pe", + "hu mp", + "b g", + "proto type", + "cl ick", + "shaw n", + "re viewed", + "tem pl", + "p f", + "jed i", + "blo gs", + "ray mond", + "as th", + "ba h", + "av ail", + "scot ch", + "leaf s", + "nik ki", + "to k", + "hol low", + "ur ges", + "of t", + "un like", + "lat in", + "u e", + "cat ering", + "mil i", + "alter nati", + "ma ver", + "Ð ¸", + "ag le", + "pre order", + "lu x", + "cu cu", + "ðŁijı ðŁijı", + "t art", + "âĿ¤âĿ¤ âĿ¤", + "arab ic", + "rapi dly", + "ar rang", + "all en", + "travel tuesday", + "pa ws", + "flo ws", + "st ability", + "flu id", + "ca pp", + "can berra", + "uu uu", + "sp ani", + "demon stration", + "m la", + "plac ement", + "m w", + "presi dents", + "awe som", + "bever ly", + "ani st", + "ne al", + "father sday", + "referen dum", + "la hore", + "o aks", + "deb bie", + "half way", + "gho sts", + "de bor", + "matthe ws", + "fi at", + "t fw", + "pre sen", + "rob i", + "de d", + "bro ck", + "laugh ed", + "am ounts", + "bam boo", + "kinder garten", + "eat en", + "mtv hottest", + "break out", + "u sic", + "fra ser", + "legis lative", + "p ang", + "modu le", + "sam my", + "go ver", + "ear ns", + "expe dition", + "gar h", + "concep ts", + "char lie", + "la va", + "bachel or", + "veg gies", + "deter mine", + "el lie", + "un locked", + "fru it", + "dal la", + "cou pe", + "wash ington", + "depo sit", + "iv ory", + "pau la", + "chic ag", + "gu cci", + "ðŁİ ĥ", + "cul tiv", + "pier ce", + "li fted", + "stu mb", + "re cover", + "musc les", + "conduc ting", + "cb s", + "mcla ren", + "sophi a", + "cel lu", + "oce ans", + "up loaded", + "game play", + "mal dives", + "kim ber", + "avo i", + "rac er", + "ca ine", + "cav s", + "h ana", + "li ga", + "ra ven", + "inter vention", + "inaugur ation", + "oo h", + "at traction", + "merchandi se", + "tune in", + "li king", + "juni ors", + "int ended", + "att acking", + "aqu arium", + "i wd", + "comp onents", + "sur ing", + "cent u", + "yogur t", + "ðŁı ĥ", + "show room", + "op tical", + "ty our", + "ju dge", + "yi eld", + "an to", + "pl c", + "transparen cy", + "recy cled", + "chi ef", + "ar om", + "ambassad ors", + "plan et", + "âĿĦ ï¸ı", + "om ed", + "vaness a", + "cour t", + "mar gar", + "hal ey", + "v r", + "reg ina", + "pd ates", + "hi span", + "live stream", + "âģ £", + "ya hoo", + "gal la", + "secu red", + "w ir", + "bene ath", + "off l", + "n il", + "am b", + "ye g", + "out let", + "u te", + "pe ep", + "lind say", + "bent ley", + "... !", + "he el", + "trilo gy", + "vo s", + "ty re", + "there fore", + "tor onto", + "ab i", + "simp li", + "ja e", + "exten sive", + "eleph ants", + "s or", + "orient ation", + "im peach", + "re play", + "constru cted", + "peter son", + "pa is", + "por ted", + "custom s", + "colla p", + "ad u", + "high lands", + "sal em", + "shel by", + "ko vic", + "stra in", + "ro sie", + "sen ators", + "snap s", + "bo bb", + "suz uki", + "bla des", + "k p", + "lo lo", + "gener ate", + "si ght", + "ma e", + "struc tural", + "predic t", + "jump ed", + "ah mad", + "sun g", + "just ice", + "gla m", + "vol vo", + "jubi lee", + "de tention", + "lo sses", + "pu ri", + "every time", + "Ð °", + "ra o", + "ed ge", + "li mer", + "rese mb", + "har old", + "re tri", + "sacri fic", + "surpri ses", + "am c", + "srilan ka", + "bar bie", + "men s", + "fin n", + "ag s", + "ukrain ian", + "em brac", + "î IJ", + "flav ors", + "hom er", + "lau re", + "ou th", + "pr iced", + "ver de", + "fir m", + "ah s", + "cu b", + "tre y", + "par anor", + "pro fit", + "in dv", + "who a", + "har sh", + "al ot", + "crit ics", + "hu bby", + "fi gur", + "gi ra", + "ca stro", + "chan el", + "in put", + "origin als", + "ten ant", + "yy yy", + "ture rs", + "lincol n", + "co on", + "lear n", + "ch ou", + "ac are", + "o les", + "din er", + "hy p", + "bizar re", + "mc r", + "let sgo", + "decor ating", + "ðŁĮ İ", + "al ison", + "ar vin", + "f d", + "reha b", + "mccar thy", + "lot tery", + "da h", + "minne apolis", + "eli gible", + "diagno sed", + "emer ald", + "destin ations", + "s ans", + "or y", + "bla zers", + "n v", + "ba il", + "digital art", + "no c", + "mal ta", + "sol ar", + "pi pes", + "alleg ations", + "no ck", + "po pe", + "bri d", + "premi er", + "n x", + "present ations", + "ef a", + "bo ws", + "val ve", + "opp onent", + "Į ë", + "visu al", + "ing le", + "cate gor", + "e ter", + "po is", + "dan i", + "at tract", + "neu tral", + "th ene", + "cra shes", + "fred die", + "ut ili", + "c st", + "awak ening", + "slo ven", + "quali fy", + "pro of", + "fair y", + "le v", + "fre ight", + "enjo ys", + "cup cake", + "flav our", + "â ķ", + "protec tive", + "ðŁijı ðŁı»", + "is u", + "ad mir", + "h mmm", + "continu ous", + "ai res", + "rap tors", + "showcas ing", + "y uk", + "pa ste", + "follow er", + "instru ctions", + "sp ru", + "@ __", + "the o", + "debu ts", + "ve tte", + "sto w", + "es of", + "ach ed", + "sul tan", + "sand wich", + "som alia", + "franc o", + "car ne", + "flu ffy", + "al pine", + "jas mine", + "he ated", + "viol in", + "ple ss", + "divor ce", + "per former", + "phi es", + "port sm", + "dar a", + "kir by", + "lo p", + "chill i", + "for th", + "sky pe", + "ðŁĩ®ðŁĩ ¹", + "celebr ities", + "ed y", + "ve e", + "po ison", + "ey el", + "gra bs", + "ssi c", + "un o", + "wester n", + "rail road", + "am er", + "numer ous", + "s v", + "fo w", + "fi st", + "âĢ ĭ", + "reque sts", + "mar tial", + "em my", + "accept ance", + "lau ra", + "ภ´", + "er up", + "hyun dai", + "out lander", + "u tt", + "wrest le", + "esp resso", + "demand ing", + "g dp", + "geo graphy", + "sas kat", + "tro ll", + "confe der", + "su es", + "se m", + "be ts", + "t ful", + "to sh", + "teach es", + "col oured", + "gal way", + "mac y", + "dis orders", + "bb cra", + "at em", + "fen der", + "lit ter", + "e sh", + "provi ders", + "renov ation", + "nomin ate", + "ps g", + "nomin ations", + "jen na", + "shar p", + "some day", + "z ur", + "bra ins", + "che shire", + "pre y", + "hu go", + " ¿", + "to ken", + "r v", + "car r", + "tac tical", + "zel da", + "kay la", + "fern ando", + "photograph ers", + "j our", + "umb rella", + "woo dy", + "congress man", + "du mp", + "le vy", + "ju an", + "d azz", + "sign als", + "la in", + "an u", + "mic hel", + "por ch", + "al den", + "sibl ings", + "y ale", + "pe el", + "sw ick", + "gg in", + "ll c", + "k ale", + "s con", + "il d", + "pat reon", + "re el", + "qu in", + "wit t", + "mar ty", + "moo dy", + "ton i", + "der y", + "g ators", + "speci fically", + "dd in", + "ly on", + "tr ick", + "meado ws", + "p j", + "bor gh", + "vi k", + "tu r", + "bron x", + "pu ff", + "lan tern", + "ðŁ¤ ¦", + "g ently", + "be stie", + "fac t", + "refu sed", + "fas ci", + "mp y", + "ðŁĶ µ", + "cross over", + "mead ow", + "indian apolis", + "duc ation", + "sle y", + "loo m", + "mix er", + "new music", + "film maker", + "prosper ity", + "li m", + "week end", + "cre amy", + "neu tr", + "lu ther", + "h v", + "nor thern", + "tw o", + "h ra", + "cat ches", + "appear ances", + "ha bit", + "kitt ens", + "n v", + "illa c", + "inf an", + "regar dless", + "liz ard", + "dun k", + "cur tain", + "ac om", + "in tu", + "ve z", + "e min", + "fl ats", + "calend ars", + "em power", + "ru ined", + "hun gary", + "vi d", + "we x", + "u lum", + "aber deen", + "o sa", + "k t", + "ma ssi", + "se emed", + "s den", + "' ?", + "tele phone", + "de fi", + "insp ires", + "me ow", + "z ones", + "bl ind", + "pl y", + "tuc son", + "advent ure", + "ge d", + "oy ster", + "ðŁijıðŁijı ðŁijı", + "out put", + "tt t", + "metal lic", + "sma sh", + "ucl a", + "sco ts", + "perfe ct", + "lu cy", + "regular ly", + "sp ic", + "rel ative", + "ath ers", + "mis e", + "batt ling", + "deci des", + "mat a", + "occu pied", + "random ly", + "cat softwitter", + "gi an", + "ball y", + "al ties", + "al lies", + "im men", + "sy rac", + "ðŁĴľ ðŁĴľ", + "l lan", + "au r", + "k ut", + "lam ar", + "affe cts", + "n ra", + "star war", + "ðŁ¤ ĺ", + "sc ram", + "en chan", + "pro cess", + "luxu rious", + "ar ray", + "sher lock", + "comp ati", + "dor f", + "stre ss", + "m su", + "s with", + "sal a", + "sof instagram", + "fo il", + "under stood", + "qu ay", + "r p", + "c ade", + "ja w", + "en ab", + "en coun", + "ðŁİī :", + "do ck", + "satur n", + "mu ll", + "lay out", + "ra rely", + "happ ily", + "fix ture", + "or ph", + "over looking", + "her bs", + "m itt", + "pil lar", + "nol an", + "pe tty", + "str y", + "u i", + "mu k", + "o res", + "o vers", + "á µ", + "re creation", + "we sley", + "ri t", + "kejri wal", + "sto cking", + "g v", + "subscri bers", + "moo se", + "ma e", + "ber t", + "opp re", + "assign ment", + "u ro", + "high lighting", + "cal vin", + "we igh", + "cambo dia", + "av on", + "ke m", + "dis abilities", + "read y", + "char gers", + "p ads", + "iz ing", + "illi an", + "tru ste", + "col leges", + "associ ates", + "alban y", + "mil ton", + "cr on", + "bu r", + "har dly", + "si ghts", + "anti ques", + "e cho", + "surpri singly", + "ha iti", + "cap t", + "ph p", + "op io", + "ine quality", + "equ al", + "ken y", + "sch mid", + "autograph s", + "ren t", + "qu er", + "cit rus", + "challeng ed", + "te c", + "epi de", + "fe st", + "z hou", + "li me", + "citizen ship", + "cry stal", + "convin ced", + "mess enger", + "copen hagen", + "âĿĹ ï¸ı", + "war ran", + "develop ments", + "ï¸ı âĥ£", + "fore x", + "hi ro", + "sne akers", + "xi de", + "vi va", + "stere o", + "bat ting", + "ss el", + "ho st", + "beng al", + "critic ism", + "q c", + "cr un", + "attemp ted", + "ry e", + "determin ation", + "cre ations", + "d read", + "label s", + "pos se", + "anc er", + "joh an", + "si ster", + "partner ships", + "les bian", + "k st", + "guaran tee", + "bar o", + "fix ing", + "ma son", + "m ous", + "chem icals", + "t less", + "bio diversity", + "par o", + "bhar at", + "ac ol", + "refu ge", + "en te", + "t iti", + "dys sey", + "respon ds", + "lef to", + "in er", + "se vel", + "rahu l", + "ol ine", + "frank fur", + "cho reo", + "enjoy able", + "c to", + "strugg les", + "wood land", + "heavy weight", + "gen s", + "rece p", + "ac cred", + "ðŁĺ ¡", + "trans formed", + "list en", + "at op", + "n k", + "sur ge", + "be re", + "gover nor", + "prison ers", + "clau de", + "t ill", + "mu lator", + "emo tion", + "water loo", + "star t", + "ðŁĩ º", + "clean ed", + "grand mother", + "fear less", + "afric an", + "astron omy", + "ðŁı ģ", + "ภĻ", + "the world", + "su itable", + "anth ony", + "k and", + "tt en", + "meaning ful", + "disc lo", + "jaco bs", + "à ¸", + "tom linson", + "ghe tti", + "ty pho", + "sub stan", + "as co", + "te k", + "nag ar", + "mu d", + "am on", + "vacc ine", + "f ty", + "fle sh", + "no el", + "infl ation", + "portu gue", + "glam our", + "tra m", + "v re", + "te qu", + "roun dup", + "w yn", + "rejec ted", + "mosa ic", + "si ghting", + "cal f", + "o ta", + "com position", + "go pro", + "gonz ale", + "e ed", + "b ard", + "tu e", + "effec tively", + "we en", + "al to", + "ri bs", + "rel ate", + "thir sty", + "fu rious", + "di m", + "ch ard", + "perfu me", + "s ny", + "chur chill", + "k of", + "master class", + "wa ve", + "ðŁĶ µ", + "er in", + "own s", + "to be", + "sk illed", + "te m", + "go f", + "en i", + "tor i", + "cra zy", + "l ick", + "resi stant", + "ici al", + "ag ar", + "! :", + "g ali", + "del aware", + "bl itz", + "koh li", + "pu ck", + "avail ability", + "hi malay", + "influ ential", + "cro chet", + "victor i", + "read ing", + "ho bby", + "vie t", + "j as", + "en gra", + "sk ul", + "ðŁĩ² ðŁĩ", + "educ ate", + "tech no", + "distric ts", + "blu es", + "se tt", + "seven th", + "lear ns", + "ee ee", + "apocaly pse", + "hang out", + "cru el", + "mu tu", + "bru h", + "hel en", + "she er", + "c tion", + "kle in", + "tex ans", + "ce real", + "sh ine", + "ne red", + "gra s", + "am bro", + "f ella", + "hin du", + "matthe w", + "li ma", + "mir anda", + "je wel", + "so ho", + "euro vision", + "neighb ours", + "chand ler", + "be sides", + "ðŁ¥ °", + "ast ros", + "thu mbs", + "ren ault", + "ra ve", + "hi red", + "ðŁĸ ¤", + "it ary", + "z or", + "bla zer", + "k ine", + "ea u", + "kat y", + "dc comics", + "pe c", + "ro dgers", + "water proof", + "kill ers", + "super int", + "pre serv", + "as so", + "brew ers", + "promo tional", + "sc am", + "villa ges", + "sket ches", + "ju icy", + "for life", + "au dit", + "so lo", + "fundam ental", + "len e", + "philipp ine", + "t end", + "conserv atives", + "sponsor ship", + "dd le", + "a ine", + "h tc", + "os i", + "hul k", + "w af", + "ภĻ", + "evalu ation", + "ant ine", + "sle e", + "robert son", + "roo sevel", + "ag i", + "sophi stic", + "emplo yers", + "bubb les", + "ko wski", + "inter action", + "sh u", + "bou le", + "ic an", + "j are", + "han k", + "leg itim", + "k nicks", + "kar ma", + "recei ver", + "per ks", + "u h", + "sta ir", + "sun i", + "labor atory", + "gra ves", + "voc als", + "oo t", + "c ture", + "thri ve", + "tic o", + "ãĥ ³", + "b w", + "carto ons", + "mcdon alds", + "dra w", + "y ung", + "pl er", + "li d", + "eth ical", + "groo ve", + "ent a", + "international womensday", + "pat ron", + "wor ries", + "ðŁİ ħ", + "ðŁij ĭ", + "ka therine", + "di az", + "tor i", + "bach chan", + "tru st", + "min eral", + "ic om", + "buil ders", + "bor n", + "col oring", + "lat te", + "ca se", + "revolu tion", + "tra der", + "ox id", + "chi pot", + "inst antly", + "sou thern", + "se hun", + "pro b", + "her nandez", + "lis bon", + "hu awe", + "p ong", + "me a", + "ro oney", + "wheel chair", + "ke en", + "be tt", + "cor in", + "regulat ory", + "di splac", + "ka ren", + "sch em", + "sun sets", + "wh ales", + "remin is", + "he p", + "hi de", + "mar cel", + "pand ora", + "do yle", + "th fc", + "ot to", + "no kia", + "trans gender", + "ko v", + "hawai ian", + "sha ve", + "so vere", + "exc er", + "nick i", + "pu g", + "st or", + "ro th", + "wee t", + "leg al", + "dig nity", + "po w", + "hom age", + "ðŁĩ³ ðŁĩ", + "s re", + "can on", + "la x", + "wo ah", + "quart z", + "ñ a", + "gree ting", + "flick r", + "nai robi", + "advoc ates", + "an c", + "vi i", + "eu gene", + "th ra", + "c re", + "el an", + "pen sion", + "th letics", + "ton i", + "re agan", + "x v", + "sto re", + "ben ch", + "har lem", + "todd ler", + "sent enced", + "âĻ¥ ï¸ı", + "glob ally", + "che aper", + "u f", + "ma m", + "nic o", + "ik u", + "tho u", + "ni st", + "dam i", + "th ala", + "rho des", + "sal e", + "bow ls", + "â Ī", + "las vegas", + "sanc tions", + "adm ire", + "mat ched", + "un able", + "travel er", + "ele ven", + "straw berries", + "âĢĶâĢĶ âĢĶâĢĶ", + "stu dio", + "jac ques", + "im s", + "valu ed", + "s no", + "cheese cake", + "n xt", + "e os", + "s x", + "f x", + "ton ic", + "hat ch", + "chic ks", + "gra ds", + "hand ic", + "r ory", + "as p", + "ri pped", + "denti st", + "n en", + "lu fc", + "âľ Ĭ", + "di ge", + "hop kins", + "sher man", + "f da", + "for all", + "ash ley", + "str and", + "h y", + "liqu or", + "buffe t", + "ess ence", + "phar ma", + "suri ya", + "ðŁĴĻ ðŁĴĻ", + "festi vals", + "z an", + "re fresh", + "pur ple", + "uni forms", + "kenne th", + "= )", + "as an", + "hel sin", + "transform ers", + "k ali", + "person alized", + "chal k", + "bo bby", + "â Į", + "the mes", + "depar ture", + "prin t", + "illustr ations", + "qui et", + "agre es", + "gri ff", + "Ø ³", + "m iti", + "toge ther", + "conven ience", + "ab ar", + "car lo", + "turt les", + "info sec", + "some what", + "ar lington", + "scholar ships", + "emir ates", + "mu ms", + "st ella", + "auton om", + "fe ather", + "g ore", + "nom inees", + "fragr ance", + "Ñ Ĥ", + "w ong", + "thea stern", + "gr e", + "z illa", + "is i", + "bump er", + "go o", + "do zens", + "ab duc", + "âļª ï¸ı", + "o ils", + "don ors", + "sil icon", + "i pod", + "fortn ite", + "ðŁĴ ¨", + "tor o", + "spark ling", + "consci ousness", + "pal a", + "nu m", + "moun ted", + "ffin s", + "thi eves", + "team mate", + "pra b", + "om er", + "ta pes", + "bo d", + "mit su", + "ste w", + "e re", + "p bs", + "tu sc", + "lo we", + "ra de", + "parliam entary", + "h m", + "ed gar", + "ðŁijĩ ðŁijĩ", + "to a", + "a gh", + "hon i", + "s late", + "ge ek", + "ap t", + "hard t", + "ta p", + "horiz on", + "grow th", + "make over", + "hi l", + "paper back", + "id an", + "reha bil", + "gi u", + "possi bilities", + "let tu", + "fran co", + "bo ss", + "ach er", + "does nt", + "mo e", + "ta ker", + "huss ain", + "ml k", + "di l", + "th ia", + "ham a", + "real ised", + "raven s", + "curric ulum", + "m ith", + "k night", + "ted x", + "r v", + "isai ah", + "cumb ria", + "birth days", + "f ing", + "pre z", + "mu barak", + "exquis ite", + "clear ance", + "y en", + "par i", + "ev o", + "à º", + "modi fied", + "app lying", + "imple ment", + "disco vering", + "chap man", + "indie game", + "dis k", + "crowd funding", + "mach in", + "li vel", + "sty led", + "âĿ Į", + "ma king", + "rehear sals", + "nutr iti", + "subscri ption", + "and ro", + "cre ators", + "car ries", + "ky lie", + "cam den", + "appren tice", + "tax pay", + "c ca", + "tuesday thoughts", + "pis sed", + "er man", + "dete c", + "freed om", + "mer i", + ".. !", + "psal m", + "sun light", + "per spec", + "be ings", + "book store", + "rock star", + "fun ctions", + "p ence", + "fav es", + "z n", + "obam acare", + "sp ill", + "coven try", + "pi geon", + "pi vo", + "ba it", + "kol kata", + "av al", + "don or", + "wa h", + "privi leg", + "tra ditions", + "rajas than", + "ten ess", + "portugue se", + "yn es", + "tack les", + "de fic", + "tor n", + "pol ling", + "thor ne", + "in a", + "bened ict", + "bar ry", + "cal ories", + "ver dict", + "save the", + "nor ton", + "off ice", + "main stream", + "impro ves", + "fr on", + "respon ding", + "real tor", + "scotti sh", + "de clar", + "r l", + "shi v", + "supp lier", + "re sting", + "swee ts", + "qu i", + ". âĢ¦", + "whit ney", + "startu p", + "thank you", + "teach er", + "h alls", + "ha ve", + "hand made", + "pro ving", + "quar tet", + "ro chester", + "li an", + "virtu al", + "mend es", + "of icial", + "mid lands", + "x box", + "meas uring", + "o vo", + "accommod ation", + "bri des", + "collegi ate", + "intellec tual", + "in car", + "ni ag", + "ðŁį ·", + "sf w", + "coco a", + "co ats", + "civil ians", + "presi dency", + "mat rix", + "sweethe art", + "tri athlon", + "wag ner", + "ra dic", + "plann er", + "the o", + "execu tion", + "k um", + "the walkingdead", + "sc ar", + "ro tation", + "blo gging", + "bom b", + "re son", + "bb les", + "st are", + "assi sted", + "e do", + "brand ed", + "war nings", + "thor pe", + "acknow le", + "satis fied", + "sho res", + "ri d", + "dor a", + "phys ically", + "bi gh", + "appro ves", + "ha h", + "ric al", + "vers atile", + "pret end", + "lu m", + "ab hi", + "ye e", + "sp it", + "ãĢ Į", + "dj s", + "ash tra", + "j t", + "ven ues", + "gram mys", + "cy clo", + "tr acker", + "over watch", + "repl ica", + "el yn", + "nr l", + "lind sey", + "hom o", + "ballo ons", + "kitch en", + "si s", + "am os", + "ende av", + "ðŁĴ »", + "a rec", + "thu g", + "hoo ked", + "hr c", + "new york", + "bur gh", + "americ as", + "patric ia", + "ug u", + "ap athy", + "ha st", + "psy chi", + "cor k", + "petro l", + "ðŁİ ¬", + "ak u", + "po pping", + "psycho logical", + "au x", + "g ma", + "cad illac", + "wa ste", + "auth ent", + "bri stol", + "nam e", + "que er", + "to ber", + "jer ry", + "com in", + "ch ant", + "privileg ed", + "op ar", + "lo ser", + "tex t", + "mar ker", + "stri es", + "equ ally", + "ak i", + "christ mas", + "gare th", + "ble w", + "em ma", + "imag in", + "se als", + "che at", + "conditi oning", + "j ana", + "ren s", + "dar ies", + "o asis", + "disc ounts", + "coun cil", + "i ka", + "shir ley", + "vou cher", + "al ps", + "w x", + "q r", + "dri ft", + "attemp ting", + "ut c", + "Ø ª", + "gonzale z", + "m f", + "jo ker", + "paralle l", + "pa re", + "aspe cts", + "proce du", + "n p", + "am a", + "rale igh", + "bright en", + "gu ire", + "radi ation", + "cre scent", + "ho b", + "il le", + "str and", + "v ore", + "n ard", + "che st", + "di wali", + "av atar", + "al der", + "d ling", + "pa thetic", + "ðŁĴ ĺ", + "spir it", + "jor ge", + "film making", + "ðŁĻı ðŁĻı", + "challeng er", + "b j", + "down town", + "ht ml", + "ade qu", + "twi sted", + "in ely", + "( '", + "wra ps", + "oper ational", + "y ne", + "n us", + "mag net", + "market place", + "health ier", + "snap shot", + "dam on", + "inter ven", + "fe derer", + "ow ls", + "biscu its", + "j p", + "ro deo", + "blue berry", + "lec tion", + "fron tier", + "summ ers", + "re yes", + "pede strian", + "go l", + "caf fe", + "refur bi", + "bou lder", + "me ghan", + "speci alty", + "la ss", + "e i", + "suspec ts", + "appro x", + "rr r", + "ra th", + "st im", + "cru shed", + "he d", + "wh un", + "lo af", + "cr ore", + "river a", + "gene tics", + "so ck", + "wa sted", + "ny pd", + "answ ering", + "do ve", + "bel la", + "ol in", + "du n", + "fi ji", + "pre tty", + "spar kle", + "y un", + "j d", + "euro pa", + "li fts", + "am ber", + "mu r", + "te k", + "boy d", + "roy alty", + "in do", + "ri b", + "go tham", + "ti est", + "inst alling", + "ke mp", + "the photo", + "cos mic", + ") ))", + "whole sale", + "loy ment", + "eas y", + "su ing", + "sett led", + "af p", + "pro ver", + "suppor tive", + "re es", + "ne ath", + "deli ber", + "c é", + "wel come", + "pic oftheday", + "new born", + "pat ty", + "sun s", + "si est", + "fl int", + "diffe rently", + "spo ilers", + "troop er", + "g ins", + "cor y", + "look out", + "equi pped", + "ta pe", + "to by", + "resear cher", + "u sh", + "ke yes", + "al ma", + "induc tion", + "k w", + "k har", + "sl ick", + "bri de", + "e ur", + "cra ving", + "book ings", + "ch es", + "tr unk", + "vern on", + "sp her", + "cryst als", + "rel atively", + "pom pe", + "uni ons", + "val ley", + "par a", + "w ant", + "ok c", + "de af", + "ser gio", + "len non", + "sh ay", + "cr a", + "v at", + "he e", + "t we", + "liqu id", + "pol y", + "ðŁİ ģ", + "b ent", + "be aring", + "motor sport", + "bar be", + "te sti", + "han i", + "fin ancing", + "astron aut", + "water colour", + "ri sh", + "comic con", + "gar t", + "wr ong", + "ber n", + "it an", + "ste pped", + "fil ters", + "c low", + "me x", + "dem ons", + "all o", + "expand ed", + "comm and", + "et ers", + "go ats", + "si ri", + "y r", + "pot tery", + "mari on", + "i le", + "el an", + "san to", + "person a", + "du ke", + "hom eless", + "li ghted", + "wheel er", + "chang er", + "cab bage", + "sur real", + "ham burg", + "sma shed", + "str an", + "k not", + "i art", + "ob i", + "be dro", + "di al", + "th ick", + "b ingo", + "fu s", + "vacu um", + "con ve", + "ati ve", + "accur acy", + "accoun t", + "re fer", + "ri z", + "spider man", + "ban a", + "r ite", + "u b", + "ab s", + "medic al", + "lin k", + "si em", + "> >>>", + "be tra", + "g lowing", + "re actions", + "pupp et", + "spa ghetti", + "ang s", + "re medi", + "pray for", + "roy ce", + "char lotte", + "£ ï¸ı", + "gh et", + "affe cting", + "ro de", + "soci alist", + "mo ses", + "az i", + "o it", + "re porters", + "cd t", + "ap ing", + "s nat", + "minim al", + "wa ist", + "sie ge", + ">> >>", + "ri g", + "schmid t", + "h are", + "ec a", + "thor n", + "he mp", + "es the", + "cly de", + "th a", + "don ut", + "moham ed", + "ling erie", + "le gg", + "carpen ter", + "perform ers", + "de a", + "imag ined", + "cur se", + "la sh", + "ct r", + "agu a", + "ro ar", + "gr i", + "ro le", + "j fk", + "resur rec", + "roosevel t", + "maril yn", + "sm alle", + "will is", + "wa ited", + "char ities", + "the res", + "li k", + "origin al", + "car i", + "c ough", + "cru ci", + "la gun", + "contra st", + "k ou", + "arm our", + "re moving", + "t ent", + "maz da", + "bri ghter", + "thi ef", + "cor ner", + "tequ ila", + "buzz ing", + "al bi", + "p am", + "az ure", + "disc oun", + "pixel art", + "possi bility", + "ham ont", + "tra des", + "bu da", + "hi ve", + "vers y", + "fin ch", + "tran spa", + "em i", + "terri fying", + "in qui", + "g ba", + "sub stitu", + "collec ti", + "plac ing", + "cin dy", + "k ann", + "pa tho", + "diamon d", + "mour inho", + "guine a", + "anthro po", + "air s", + "pu mps", + "ì ļ", + "pas o", + "cur ling", + "an ita", + "resi dency", + "ne wh", + "jo on", + "cigare tte", + "que ue", + "ex trac", + "gam es", + "spl en", + "ex press", + "public ly", + "bon nie", + "tribun e", + "ba ek", + "reason able", + "c or", + "timo thy", + "she eran", + "Ä ±", + "f dn", + "su tton", + "concentr ation", + "carav an", + "x avier", + "al ger", + "cy lin", + "freder ick", + "ner ve", + "pe ak", + "lettu ce", + "j ail", + "pre game", + "kav an", + "up graded", + "eco logy", + "squad ron", + "gra pes", + "goo g", + "pa stry", + "ðŁĹ £", + "ãĥ¼ ãĥ", + "mil ano", + "awa z", + "presen ter", + "ðŁĮ ¿", + "her d", + "king s", + "tem plate", + "fl our", + "h v", + "k ley", + "i ya", + "spe c", + "at er", + "frankfur t", + "co ch", + "tex ting", + "del i", + "communi st", + "regi ment", + "ele anor", + "anticip ated", + "ðŁijĮ ðŁı»", + "thephoto hour", + "ran o", + "survi ving", + "simul ation", + "daw son", + "ar in", + "aqu a", + "m or", + "âĢ¦ .", + "cin o", + "ira qi", + "sh az", + "dun dee", + "we s", + "dra u", + "hann ah", + "s news", + "occup ation", + "ste en", + "x m", + "ang les", + "sett ings", + "gur u", + "kno x", + "or ca", + "shap ing", + "w ent", + "dr illing", + "zz ie", + "br i", + "kis sing", + "fin d", + "ma ine", + "âŃIJï¸ı âŃIJï¸ı", + "ðŁĮ į", + "lar ry", + "bu sted", + "ta vern", + "acti vely", + "- \"", + "replac ing", + "no d", + "un lock", + ". \"", + "âŀ ¤", + "affili ate", + "to w", + "l n", + "happy newyear", + "di f", + "j m", + "green wich", + "contro versy", + "daw g", + "con dol", + "sav annah", + "compens ation", + "touch down", + "te o", + "amb itious", + "embro i", + "convic ted", + "iart g", + "bar ack", + "tr ance", + "testim ony", + "au dition", + "thum b", + "my ths", + "be x", + "que z", + "orch id", + "den y", + "entit led", + "hoo d", + "gr ant", + "in box", + "blue jays", + "r illa", + "smalle st", + "bur den", + "in famous", + "divi ded", + "boun daries", + "t ter", + "el t", + "wy oming", + "be verage", + "me sm", + "one ws", + "budd hist", + "y ana", + "as sad", + "is ms", + "bar rett", + "predic ted", + "back to", + "tw it", + "e there", + "cap tains", + "escap ed", + "ay o", + "lam borgh", + "gard ner", + "la ps", + "k al", + "adverti sement", + "insec ts", + "na po", + "am en", + "ac y", + "r and", + "g k", + "te h", + "k athle", + "tri dge", + "pan cake", + "at ro", + "pyram id", + "bu la", + "paral ym", + "gau ge", + "en cies", + "tom y", + "biscu it", + "but cher", + "quali fier", + "coun ty", + "ke i", + "po ols", + "dar ker", + "should ers", + "ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸", + "sp re", + "( \"", + "writ ers", + "g m", + "ðŁİ ĵ", + "k nit", + "hu ff", + "mt b", + "philli es", + "o st", + "den is", + "g art", + "licen sed", + "inter face", + "ex cel", + "d well", + "from the", + "co fficial", + "az zi", + "appear ing", + "fore st", + "n ana", + "ke ith", + "manufac turers", + "beck ham", + ") ?", + "e se", + "col ony", + "delic ate", + "ut ter", + "mc in", + "transpl ant", + "pre ferred", + "par d", + "ari e", + "hu b", + "po ds", + "perspec tives", + "pic t", + "del u", + "app er", + "be than", + "p mo", + "crimin als", + "femin ism", + "sh ack", + "circum stances", + "fel las", + "prote sting", + "wa x", + "sugge sted", + "t ator", + "dre w", + "om ni", + "fa ke", + "kath y", + "re b", + "del ine", + "ber ni", + "mi sty", + "ðŁij ©", + "er able", + "break through", + "men swear", + "millenni als", + "chan yeol", + "la z", + "inser t", + "rep lies", + "phra se", + "n x", + "ihear tawards", + "audre y", + "gran ite", + "rac ec", + "ori e", + "ter ra", + "innov ations", + "britt any", + "at eral", + "pe ar", + "bio logical", + "sh ments", + "institu tion", + "m sn", + "frequ ency", + "d man", + "neg lec", + "t f", + "ste fan", + "fox news", + "ty po", + "comm s", + "sequ ence", + "car men", + "wh ites", + "econom ist", + "exe ter", + "se um", + "re sorts", + "cas ually", + "bun de", + "divi de", + "Ø ¹", + "ga g", + "cre ed", + "reti re", + "cau cus", + "rapi ds", + "wrestle mania", + "tul sa", + "sunder land", + "fundam ent", + "o di", + "yam aha", + "v ary", + "intri gu", + "el se", + "be acon", + "an gie", + "tra ded", + "tran sm", + "g ents", + "kn itting", + "gal ac", + "ðĿ Ĺ", + "u to", + "sea side", + "hol t", + "re rs", + "far go", + "train ers", + "mon soon", + "b ale", + "sou ght", + "mad die", + "h w", + "co li", + "fr an", + "fav s", + "ðŁĴ Ķ", + "int ent", + "r ally", + "s bs", + "lemon ade", + "barack obama", + "bre ad", + "stick y", + "explo sive", + "chel ten", + "t j", + "as soc", + "ram en", + "hom ies", + "v log", + "mi ster", + "lor d", + "âĢįâĻ Ģï¸ı", + "aly ssa", + "sketch book", + "ru mble", + "cat ch", + "migr ant", + "discipl ine", + "un likely", + "chronic les", + "fl ora", + "sl ams", + "am id", + "s boro", + "coo p", + "ju mps", + "tran qu", + "mel is", + "sof ia", + "en ri", + "gab e", + "sy ri", + "nicol as", + "cha i", + "w v", + "be cky", + "foo ty", + "ta o", + "suppo se", + "ðŁĺįðŁĺį ðŁĺįðŁĺį", + "plu sh", + "ri sh", + "ðŁ¤ ĵ", + "k ha", + "satur days", + "ac cent", + "he c", + "lim it", + "carl ton", + "wi red", + "taylor swift", + "ðŁĺ ij", + "sq l", + "har ro", + "recipi ents", + "g at", + "go p", + "th of", + "amaz ed", + "gh an", + "ðŁıĨ ðŁıĨ", + "por to", + "cla re", + "di stant", + "na c", + "ohi o", + "ðŁĻı ðŁı¼", + "mt n", + "anti bio", + "dino sa", + "me sa", + "par tial", + "b v", + "lear nt", + "lov ato", + "questi on", + "ex tract", + "gossi p", + "gi bb", + "niag ara", + "ðŁij ¨", + "displa yed", + "so oner", + "ste vie", + "nug gets", + "ml n", + "bro m", + "tur b", + "give aways", + "stu pi", + "bl ink", + "c ili", + "conven ient", + "mo h", + "vi ve", + "f ric", + "cau se", + "cham ber", + "cu les", + "ne arest", + "is se", + "small biz", + "t j", + "canadi ans", + "smar ter", + "bra sil", + "ra re", + "que tte", + "w ha", + "cand le", + "at omic", + "ðŁijį ðŁijį", + "warri or", + "relax ed", + "stri ps", + "ne ur", + "k ka", + "r fc", + "jen sen", + "reco vering", + "respon ses", + "sal am", + "ortho dox", + "acti ve", + "ell ers", + "n it", + "âŃ IJ", + "metro politan", + "centu ries", + "vi da", + "gra ding", + "transpa rent", + "sim ple", + "do ts", + "superint endent", + "elev ator", + "autom ated", + "red skins", + "ima m", + "summer time", + "jona than", + "ge aring", + "michel le", + "confl ic", + "m ice", + "to te", + "publi sh", + "pa x", + ") -", + "na iled", + "á ´", + "tele scope", + "ser bia", + "ba b", + "ape u", + "st ically", + "sen ti", + "r ats", + "isol ated", + "grou p", + "hat red", + "paranor mal", + "stan ley", + "ali on", + "safe ty", + "l s", + "ठ°", + "nex us", + "alexand ra", + "mas ks", + "+ +", + "tr on", + "au k", + "brother hood", + "brow se", + "mix es", + "sim one", + "mu sk", + "appro ve", + "lo la", + "ex p", + "per th", + "fu turi", + "un seen", + "d m", + "chel se", + "sc outing", + "o we", + "portsm outh", + "k ram", + "mi ze", + "di spen", + "su p", + "d lc", + "adver t", + "tere sa", + "is le", + "cy cle", + "met all", + "shi elds", + "marin ers", + "ra z", + "ing en", + "fun d", + "an go", + "jon es", + "o ka", + "mad den", + "broc coli", + "domin ic", + "situ ations", + "mer o", + "cric ke", + "puni shment", + "d b", + "sha king", + "ðŁĺ ļ", + "m q", + "ari ans", + "le h", + "cla w", + "we ds", + "d ure", + "ni el", + "j elly", + "gour met", + "tra ders", + "le vi", + "w ages", + "kne es", + "wi se", + "heaven ly", + "avi d", + "melo dy", + "z ack", + "ban anas", + "apprentic e", + "pro p", + "fun ny", + "o de", + "respec ted", + "me gan", + "fe wer", + "dra fted", + "med it", + "gra pe", + "us army", + "cru sad", + "vo cali", + "prepar ations", + "non sense", + "us age", + "th r", + "ro th", + "wiz ards", + "insi de", + "promo tions", + "mon a", + "red sox", + "si g", + "eleg ance", + "ch ia", + "univer sal", + "ãĢ į", + "ra ja", + "un ga", + "pol lin", + "filip ino", + "ak a", + "t sun", + "ik on", + "bi king", + "decor ations", + "z ac", + "cade ts", + "hum our", + "ag m", + "re ppin", + "vac cin", + "elo ve", + "u w", + "dia be", + "galla gher", + "az er", + "do l", + "a while", + "pro minent", + "wel sh", + "t ann", + "' )", + "bi en", + "wa g", + "in al", + "c wc", + "wic ket", + "ur st", + "q anon", + "x e", + "out door", + "dun n", + "star r", + "co logy", + "ric ky", + "u efa", + "reb ounds", + "s music", + "inf ant", + "ðŁĻ ĭ", + "so p", + "u mber", + "hand ing", + "beg in", + "sor ting", + "ha sh", + "sp ati", + "re k", + "buda pest", + "black hawks", + "dele te", + "ro m", + "can did", + "auth ori", + "de bris", + "spe cul", + "inter section", + "marri ott", + "im ran", + "ðŁĺģ ðŁĺģ", + "cru ises", + "ram sey", + "rafa el", + "aware ness", + "vas cular", + "beyon cé", + "ru g", + "ðŁĺ Į", + "festi v", + "ar am", + "s able", + "bas il", + "p ill", + "flo oring", + "un beaten", + "implic ations", + "u f", + "w ound", + "for ge", + "poin ting", + "po ts", + "popular ity", + "ðŁijı ðŁı»", + "mani pul", + "s lots", + "deb ates", + "abs ence", + "ver mont", + "never forget", + "wri st", + "gl oria", + "ren ce", + "hu sk", + "mel ting", + "ðŁİ Ł", + "br aces", + "tim ely", + "transform ing", + "am ps", + "ma k", + "po e", + "ah an", + "gener ally", + "nd p", + "ale ppo", + "unic ef", + "pro fs", + "nor d", + "ma sk", + "jackson ville", + "v v", + "sh ells", + "bloom ing", + "oper ators", + "char coal", + "ne ville", + "ma gi", + "chi p", + "sam a", + "ir an", + "re forms", + "accu mul", + "ru e", + "æ ľ", + "web sites", + "ga on", + "devast ating", + "sto s", + "glaci er", + "ra pp", + "chipot le", + "pr a", + "or ous", + "rom ney", + "seas on", + "decor ative", + "c isco", + "dit ch", + "compla in", + "ll o", + "assu me", + "ðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤ", + "n els", + "cent ric", + "ft w", + "car rots", + "tat a", + "can ter", + "per ience", + "li ers", + "demo s", + "bl unt", + "oper ate", + "reserv ations", + "le ah", + "sub stance", + "di son", + "an te", + "elec tion", + "v ue", + "squ are", + "non profit", + "ca a", + "f su", + "y am", + "ãĤ ¤", + "v ladi", + "comple tes", + "mar i", + "philli p", + "ne ill", + "er as", + "ka it", + "men do", + "mahar ashtra", + "g p", + "dan e", + "provi dence", + "ther apeu", + "juven ile", + "me mo", + "in corpor", + "aa aa", + "seven teen", + "teen ager", + "à £", + "or ns", + "wi de", + "cu teness", + "tw d", + "ff les", + "bar a", + "com edy", + "over time", + "y az", + "bar on", + "unemp loyment", + "ðŁij ĭ", + "exter ior", + "den se", + "cent res", + "match up", + "history month", + "artif icial", + "qu it", + "e sk", + "war n", + "cr itic", + "j af", + "ðŁĵ ²", + "inform ative", + "fu els", + "recy cle", + "nam ing", + "stri pe", + "sol ic", + "mole cular", + "dee pi", + "con vo", + "s sel", + "na e", + "de scent", + "ti z", + "accoun tability", + "ter ry", + "r ito", + "sl ay", + "em o", + "dem ol", + "sens ation", + "co v", + "tor e", + "round table", + "y ol", + "excu ses", + "ॠį", + "tur quo", + "hh hh", + "pod casts", + "cele b", + "me ssi", + "li o", + "man n", + "contribu ted", + "u z", + "gener ator", + "ele ts", + "veg gie", + "indu l", + "en suring", + "detro it", + "pun jab", + "tran spor", + "instru ction", + "ad d", + "por cel", + "pan eli", + "cir cles", + "persi st", + "clay ton", + "sp n", + "dog softwitter", + "is nt", + "sp r", + "retail ers", + "p w", + "hun gar", + "el ena", + "mon aster", + "gu atem", + "je ssie", + "an z", + "ra shi", + "fle e", + "car ving", + "fau x", + "l al", + "hen ri", + "d jo", + "du ll", + "s ana", + "lar a", + "glo be", + "cri mson", + "com pass", + "pau se", + "na b", + "lion el", + "ba ths", + "u fo", + "invent ory", + "sin gh", + "sat an", + "ðŁĩ ¸", + "ce ments", + "in form", + "gener ated", + "bi den", + "av g", + "tas ks", + "de er", + "sa u", + "ja iled", + "pa stel", + "sc c", + "na il", + "steel e", + "per is", + "lamborgh ini", + "pur sue", + "mar gin", + "u ch", + "bo sch", + "dra in", + "cl ara", + "bo m", + "lat ino", + "web ster", + "rose mary", + "r ha", + "s oun", + "billion aire", + "not ch", + "percent age", + "con or", + "' \"", + "hom es", + "earth day", + "h ort", + "big gest", + "di sin", + "wal ton", + "edit ors", + "im ma", + "om ar", + "equi valent", + "pharmac eu", + "ah med", + "cam eo", + "han ni", + "under rated", + "ge ment", + "micro bi", + "v oo", + "honor able", + "obe sity", + "âļ ¡ï¸ı", + "limer ick", + "invol vement", + "st agram", + "boule vard", + "bur g", + "blackand white", + "liber ation", + "fi ve", + "inter im", + "sm m", + "rival ry", + "cap abilities", + "stat ements", + "thu mb", + "ve d", + "sw ans", + "bar ber", + "e que", + "seren a", + "hel m", + "noo dle", + "sam pling", + "n awaz", + "sing le", + "thunder storms", + "sh on", + "in ev", + "ë ¯", + "to pp", + "orch ard", + "bi an", + "ðŁĺ Ķ", + "door step", + "salv ation", + "marke ting", + "r ons", + "cle mson", + "ra vi", + "in take", + "stand with", + "sin a", + "ha iku", + "ple y", + "elector al", + "ph illy", + "la ys", + "electr ic", + "cap turing", + "u pp", + "er gy", + "believ ing", + "cul tures", + "es day", + "inva sive", + "ed ed", + "spee ch", + "end ur", + "viet nam", + "boy cott", + "pe de", + "deli ver", + "ðŁĴĸ ðŁĴĸ", + "mer chant", + "st ir", + "den ies", + "poc kets", + "o ti", + "cu ddle", + "ro land", + "mm ed", + "den ed", + "lear ners", + "hoo p", + "sour cing", + "h acked", + "di m", + "environ ments", + "ben son", + "jud icial", + "wor cester", + "pear ls", + "govern ments", + "arri vals", + "cor ners", + "tun ing", + "la bour", + "y m", + "or dering", + "le wi", + "i fe", + "hygi ene", + "thou ghtful", + "indone sian", + "campaig ning", + "princi ple", + "assau l", + "ru bb", + "at v", + "wil ly", + "en tre", + "il i", + "ph on", + "du ties", + "âĻ¥ âĻ¥", + "sn akes", + "lo op", + "am ar", + "conver tible", + "bon ding", + "ment oring", + "max well", + "ethere um", + "destro ying", + "ax is", + "ca iro", + "fin nish", + "sho ck", + "ðŁĺ IJ", + "cal eb", + "com a", + "pe dal", + "co re", + "contin ent", + "el son", + "temp o", + "helsin ki", + "ac p", + "tack ling", + "st ated", + "bl a", + "dou b", + "sma shing", + "a ja", + "camer on", + "disru ption", + "warm th", + "being salmankhan", + "bullet in", + "o de", + "syrac use", + "ar an", + "mc gregor", + "bul k", + "an ton", + "confir mation", + "sp ine", + "im ran", + "instru c", + "jac ks", + "chi o", + "pal m", + "str e", + "embarra ssing", + "un t", + "elimin ate", + "to ss", + "c ise", + "a ws", + "oni sts", + "sh inee", + "jo s", + "ho se", + "li vely", + "opp onents", + "mo vements", + "recogni zing", + "sandwich es", + "sh akes", + "exerc ises", + "se at", + "profe ssion", + "merry christmas", + "lu gg", + "adopt dont", + "mar vin", + "byr ne", + "un le", + "he t", + "ku wait", + "rah man", + "aspe ct", + "humb led", + "gen es", + "f and", + "long time", + ") ;", + "cam pu", + "an gus", + "ðŁijį ðŁı¼", + "q uran", + "sle eves", + "s lic", + "¸ ë", + "twel ve", + "your e", + "i ke", + "go gh", + "b st", + "dic tionary", + "reflec ting", + "to on", + "yar n", + "em bed", + "ðŁı ´", + "re serves", + "floo ded", + "ver iz", + "du sk", + "estab lish", + "pro li", + "au d", + "ritu al", + "or bit", + "declar ation", + "recor dings", + "cam o", + "cas sette", + "good luck", + "cu tter", + "bo p", + "b ho", + "che ating", + "paci fic", + "ma res", + "tim er", + "col t", + "tr ous", + "tomor row", + "han sen", + "ci e", + "w ang", + "ban i", + "circu lar", + "ac ute", + "far mer", + "co ys", + "p se", + "ir ving", + "w j", + "haw kins", + "b ison", + "ur day", + "cru ising", + "o te", + "k ath", + "whi stle", + "your selves", + "ant is", + "sla sh", + "thorough ly", + "ke sh", + "ser ie", + "ex em", + "en ig", + "guil d", + "sh red", + "ho gan", + "ap o", + "ä ¸", + "pu zz", + "ne tball", + "au ssi", + "panor ama", + "ws j", + "av is", + "ar ming", + "hum ph", + "brow ser", + "cri es", + "fo ggy", + "mat te", + "ðŁĮ »", + "it er", + "tal lest", + "by ron", + "cap tiv", + "je su", + "any ways", + "flag ship", + "p ton", + "we y", + "fay ette", + "financi al", + "f oul", + "solom on", + "jenni fer", + "cucu mber", + "ar gue", + "tex tile", + "wrest ler", + "john ston", + "pa stor", + "ðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃ", + "cac tus", + "edi ble", + "re served", + "ric hie", + "met res", + "ingredi ent", + "h ella", + "un to", + "ch ol", + "cele bs", + "po ets", + "gra ham", + "hay den", + "coinci dence", + "b aw", + "communic ate", + "flet cher", + "/ -", + "tole do", + "ecu ador", + "coun sel", + "s laughter", + "line ar", + "at p", + "os u", + "jo el", + "ev ed", + "conqu er", + "ru stic", + "plic ity", + "recogn ise", + "room mate", + "cr acked", + "jas per", + "ph er", + "ðŁĮ º", + "wo ven", + "mo ist", + "ff c", + "ste ering", + "ni sh", + "stand ings", + "frequ ent", + "ar di", + "haz el", + "as msg", + "bau m", + "d art", + "si dd", + "nat h", + "ch ero", + "card board", + "c ss", + "n sfw", + "pa ir", + "ðŁĺį ðŁĺĺ", + "occur red", + "homeless ness", + "mal one", + "ph e", + "xi a", + "pad dy", + "decl are", + "theat re", + "b f", + "per sian", + "ta d", + "ax e", + "susp icious", + "lam b", + "mu cho", + "sen ior", + "st as", + "k ite", + "st ing", + "gra d", + "k af", + "wat ering", + "Ø ¯", + "spi ral", + "th ms", + "educ ator", + "jer ome", + "of c", + "clo ck", + "su l", + "pe mb", + ".... .....", + "park way", + "de aux", + "restric tions", + "m ons", + "need le", + "e j", + "le agues", + "water melon", + "am an", + "pl enary", + "max im", + "w ab", + "coming soon", + "bry ce", + "vi gil", + "super market", + "fortun ate", + "turquo ise", + "presi dent", + "li v", + "inter ns", + "feel in", + "fix tures", + "stun t", + "st aged", + "premi eres", + "lo k", + "prac titi", + "shor tage", + "log ne", + "ve c", + "con cor", + "roc ke", + "li g", + "com posed", + "syn thetic", + "di p", + "cam ila", + "ch is", + "j ou", + "su san", + "eye brows", + "supp lement", + "satis faction", + "moham mad", + "ti bet", + "house of", + "pu n", + "as sam", + "shado whun", + "psy ched", + "se duc", + "mand atory", + "her bert", + "sc allo", + "stream ers", + "proto col", + "block buster", + "produc es", + "sch nei", + "lau rel", + "tri be", + "time hop", + "pl a", + "mod elling", + "tv time", + "mtv stars", + "wi dow", + "me tric", + "ch am", + "con do", + "flow ering", + "ale c", + "d ms", + "inten sity", + " ¨", + "mccar tney", + "islam abad", + "k b", + "f fi", + "ph al", + "anal og", + "f ond", + "h acks", + "positi vity", + "treat y", + "sub marine", + "conne ct", + "sel en", + "categor ies", + "cu b", + "organi ze", + "si k", + "quote oftheday", + "remin ding", + "am or", + "loc king", + "ðŁijı ðŁı¼", + "comp ound", + "et te", + "b out", + "rec ur", + "fe rence", + "mi zz", + "tren d", + "hip ster", + "for tress", + "forth coming", + "preli min", + "o dyssey", + "ang p", + "del ici", + "even ings", + "ðŁĶ ¹", + "i q", + "d w", + "da ir", + "kathr yn", + "christian ity", + "moon light", + "ha b", + "wh oo", + "f bf", + "se th", + "genu inely", + "pa x", + "char ity", + "deplo yed", + "b nb", + "bu cs", + "ju dg", + "con ge", + "plant ation", + "im press", + "car a", + "sc lub", + "sco py", + "land ers", + "compla ints", + "b ama", + "re build", + "x y", + "real ism", + "sh our", + "le in", + "brac elets", + "mer a", + "assas sin", + "an chor", + "ðŁijĮ ðŁı¼", + "lin en", + "con fron", + "chronic le", + "comm ent", + "cat alog", + "il les", + "gor ge", + "me try", + "jung kook", + "love my", + "sent in", + "se em", + "fit ness", + "alli ed", + "ts man", + "digital transformation", + "pr an", + "lo ft", + "min ton", + "alden richards", + "en vel", + "cher ish", + "certain ty", + "zz z", + "rhin o", + "per kins", + "en rich", + "cape town", + "ome ter", + "sec tions", + "ske leton", + "def enders", + "ðŁĺ Ŀ", + "pen c", + "bri t", + "ja h", + "capital ism", + "ðŁ¥ ĩ", + "baz aar", + "re me", + "ex t", + "kk k", + "conver t", + "stor my", + "b ye", + "kar an", + "chry sler", + "ad os", + "pre ssed", + "syn c", + "ation day", + "dang er", + "bad ges", + "refu ses", + "em powering", + "ly m", + "ex ports", + "adoptdont shop", + "ðŁĩ ¯", + "th c", + "awa ited", + "focu ses", + "fin ed", + "o at", + "haha hah", + "âģ ©", + "n family", + "fi ona", + "luck ily", + "thr illing", + "ty ping", + "out break", + "di es", + "he u", + "craw l", + "ne sses", + "o ath", + "scri pts", + "gee ks", + "ðŁIJ Ŀ", + "p b", + "mathemat ics", + "al is", + "________ ________", + "gymna stics", + "acti vism", + "recommend ation", + "gre n", + "wa in", + "cour ty", + "n apol", + "cau li", + "hor nets", + "g als", + "jo ckey", + "dir ty", + "at ar", + "enor mous", + "pe st", + "greg ation", + "an os", + "ii ii", + "def ends", + "black historymonth", + "at x", + "mb c", + "lugg age", + "wit ch", + "co b", + "la sts", + "cu m", + "gg g", + "ba thing", + "n ar", + "ce bu", + "ðŁį ĥ", + "navig ation", + "min e", + "re jo", + "ðŁİ Ģ", + "gif tide", + "re ta", + "use less", + "pu ll", + "defic it", + "al lu", + "ati me", + "it v", + "tr illion", + "pu e", + "ac ies", + "proce dure", + "l ori", + "jen ny", + "c ad", + "ul ously", + "dr ac", + "promo tes", + "ing the", + "can u", + "woo hoo", + "na omi", + "zar dari", + "ts u", + "be ir", + "sd g", + "le ver", + "we ber", + "ab ud", + "lun d", + "crow ded", + "deplo yment", + "ter rain", + "ken ny", + "ho f", + "witne ssed", + "lo ch", + "j k", + "bul ly", + "w ren", + "poe try", + "do ff", + "ww i", + "mo red", + "din i", + "cul ture", + "promp t", + " ¥", + "maur ice", + "to pps", + "r m", + "cor respon", + "ab out", + "jewel s", + "gi br", + "eag le", + "ðŁĺĺ ðŁĺĺðŁĺĺ", + "l ending", + "sou ven", + "ç Ķ", + "contemporary art", + "establi shment", + "j ong", + "âĢ¦ \"", + "gat or", + "patri otic", + "mc coy", + "v ape", + "human e", + "feli z", + "coach ella", + "re posting", + "ste als", + "fu ller", + "n ering", + "at ra", + "( -", + "bla ke", + "he ather", + "wor ms", + "discipl inary", + "rede mption", + "y ard", + "am in", + "\" @_", + "d nc", + "t ds", + "k appa", + "ne wark", + "comm its", + "spe ars", + "j ams", + "t and", + "msn bc", + "inter medi", + "aim ed", + "at ic", + "teen th", + "observ ation", + "kash mir", + "kavan augh", + "ou l", + "san francisco", + "re u", + "bel ated", + "cho w", + "pass word", + "st ills", + "deta ined", + "sar i", + "day ton", + "dar ren", + "itali an", + "ar th", + "amu sic", + "ar bit", + "w m", + "v m", + "he m", + "dou g", + "my r", + "a sho", + "pre v", + "vin d", + "bra h", + "sta g", + "ภµ", + "pre views", + "gu k", + "con taining", + "leon ardo", + "sad dle", + "ru shing", + "st av", + "lon gh", + "gam bling", + "ve gas", + "reserv ation", + "end ale", + "bal a", + "fl a", + "vari ant", + "he dge", + "bulgar ia", + "nat ali", + "we aver", + "sol st", + "encoura ged", + "ap c", + "as parag", + "ne st", + "cycli sts", + "fe l", + "ìĬ ¤", + "overwhel ming", + "pey ton", + "j it", + "a post", + "mb le", + "ble eding", + "neighbour hood", + "a very", + "expre ssions", + "mac donald", + "gi gs", + "mon ds", + "illu sion", + "n ct", + "cam ero", + "over head", + "my th", + "ol y", + "vi o", + "et v", + "lau rie", + "unve iling", + "pri or", + "con n", + "iron man", + "di ff", + "day in", + "crit ici", + "con go", + "re vision", + "wal e", + "direc tor", + "p ines", + "black pink", + "gar ner", + "cur ated", + "manit oba", + "h ac", + "common ly", + "bar ton", + ".... #", + "mor tality", + "live smatter", + "philos op", + "shor ter", + "con vince", + "fre ak", + "vend ors", + "insi ghtful", + "el ly", + "sens ors", + "e led", + "s berg", + "weight loss", + "u kip", + "sp ur", + "priv ate", + "qu a", + "ss c", + ", ...", + "supervis or", + "advis er", + "amaz ingly", + "less er", + "at es", + "mah on", + "oooo oo", + "sar as", + "pmo india", + "waff le", + "un ders", + "toler ance", + "sculp tures", + "her sh", + "kno cking", + "smo ke", + "cathol ic", + "gri m", + "tra veled", + "fli p", + "ge off", + "dinosa urs", + "sle pt", + "scar let", + "ok i", + "compla int", + "ob sc", + "nam i", + "la g", + "cross fit", + "u fc", + "mc cain", + "refe ree", + "sad ness", + "pen ny", + "li eu", + "mo de", + "ki er", + "vol s", + "w is", + "el on", + "she a", + "ba o", + "son ia", + "cla ire", + "em manuel", + "moist ure", + "di gest", + "vi ii", + "t eller", + "ch on", + "access ory", + "night club", + "foss il", + "aw an", + "hu sky", + "ab original", + "brand on", + "ffici ent", + "cou gars", + "ste d", + "ad mitted", + "igno red", + "content marketing", + "ag as", + "v ase", + "execu ted", + "negoti ations", + "she ad", + "n and", + "tab lets", + "go th", + "ts al", + "d fw", + "on ep", + "protec tor", + "sp ho", + "gaz ette", + "andre as", + "ss er", + "comp ilation", + "ha v", + "contain ers", + "bro ker", + "soc al", + "porcel ain", + "hy uk", + "air ing", + "ðŁĴ °", + "publi sher", + "scen ario", + "spart ans", + "re viewing", + "itu des", + "ed el", + "pear son", + "ba sh", + "mau i", + "a ad", + "ðŁĮ Ĭ", + "li u", + "ul ate", + "program mes", + "fav our", + "web design", + "real ty", + "motiv ational", + "cro sses", + "' ...", + "bus ch", + "adjust able", + "ar jun", + "mist ak", + "dimen sion", + "pi stol", + "weigh s", + "en y", + "unve il", + "indy car", + "gor don", + "f ade", + "fran ken", + "qual ities", + "bet t", + "loc ate", + "ker r", + "sp c", + "confu sion", + "ne e", + "luck y", + "bas es", + "dep ends", + "fire fighter", + "ol a", + "re t", + "mar oon", + "ðŁĶ Ĭ", + "w am", + "defin ing", + "whe at", + "bi l", + "é s", + "b hai", + "psy ch", + "ta u", + "ic ans", + "thi k", + "ob ile", + "inspec tor", + "ìĨ Įë", + "ill on", + "go s", + "ev angel", + "fa i", + "si st", + "voc ation", + "bur ge", + "chi stan", + "renew ed", + "enthusi asm", + "en ting", + "ag ri", + "ike a", + "m sc", + "aero space", + "sens iti", + "memo ir", + "hosp ice", + "co caine", + "der ry", + "mechan ics", + "Ħ à¸", + "tin o", + "reduc es", + "collec tors", + "in justice", + "supp re", + "v ana", + "ab un", + "nap a", + "su sa", + "os lo", + "e ff", + "en core", + "lic ence", + "ched dar", + "z al", + "moun t", + "ðŁĴ IJ", + "threat ens", + "!! \"", + "archi e", + "fu tsal", + "scu ba", + "jo s", + "gn on", + "se xi", + "s official", + "compar ing", + "domin ant", + "tof theday", + "fa it", + "propos als", + "gi ft", + "y as", + "cn c", + "l r", + "ha b", + "reser voir", + "beli efs", + "gener al", + "mar ti", + "t d", + "est e", + "ì ł", + "wi l", + "ðŁij ¯", + "ðŁĶ «", + "sp x", + "et work", + "excer pt", + "e instein", + "hir o", + "sil hou", + "team ed", + "per ception", + "corri dor", + "mental health", + "hin ts", + "ben ny", + "induc ted", + "sw x", + "wi desp", + "spe ak", + "cher yl", + "dru g", + "ðŁĺ ķ", + "h f", + "asparag us", + "myster ies", + "fitz gerald", + "off er", + "therap ist", + "care er", + "dam aging", + "ts d", + "per u", + "wei bo", + "y ay", + "phoeni x", + "disc re", + "mac book", + "bar ker", + "stig ma", + "sp read", + "roc kies", + "kang ar", + "bri dg", + "pa i", + "bi shop", + "ta iled", + "capsu le", + "ðŁĴ ĵ", + "ge of", + "roy ale", + "short listed", + "o ste", + "ash amed", + "ch app", + "key e", + "cl a", + "screen shot", + "austri an", + "nati ve", + "en ight", + "juli et", + "michel e", + "ðŁĮ ´", + "travel ers", + "pi l", + "football er", + "win chester", + "ðŁĻ Ħ", + "azer bai", + "gold eng", + "organis ations", + "interpre tation", + "predat or", + "ofthe week", + "lo gan", + "pok é", + "mari e", + "cal la", + "t nt", + "cin de", + "ge tic", + "fit fam", + "gra v", + "ow ens", + "ðŁĮ ±", + "shoot out", + "sal is", + "commissi ons", + "co he", + "p tic", + "ni xon", + "hi a", + "amb ition", + "mar ine", + "cruel ty", + "t k", + "cru de", + "sal ty", + "jim a", + "mon go", + "ir ony", + "on wards", + "arre sts", + "strang ers", + "ig er", + "cycli st", + "ra g", + "exten ds", + "tra dio", + "bour g", + "mo i", + "el la", + "e able", + "lex us", + "au l", + "der a", + "histor ian", + "mor ton", + "ti ff", + "man ner", + "ko t", + "d k", + "po inted", + "mar qu", + "a an", + "en ey", + "du blin", + "on poli", + "em ili", + "secre t", + "fl o", + "âļ ¡", + "ba j", + "ste ep", + "accompan ied", + "rum ours", + "dev i", + "purch asing", + "fi g", + "pu b", + "sch oo", + "autonom ous", + "go alie", + "x ia", + "autom atically", + "re vers", + "ter o", + "fu ku", + "titan ic", + "shoo k", + "sand als", + "see kers", + "exc av", + "nor dic", + "bigo live", + "ba ke", + "r att", + "z ak", + "ne p", + "ðŁĺ ¤", + "cand y", + "billi ons", + "book worm", + "pp et", + "à ³", + "sur faces", + "sc ars", + "phil ip", + "do gg", + "ci gars", + "co te", + "transl ated", + "cur ator", + "sin dh", + "han gover", + "bre wer", + "on es", + "el ton", + "ðŁĴª ðŁı¼", + "mar cu", + "elli ot", + "righ te", + "di oce", + "ru ss", + "rail ways", + "grand son", + "as cen", + "apo logy", + "awa it", + "mob ili", + "re spir", + "parti san", + "oli vi", + "stri ke", + "yo o", + "white house", + "expre ssed", + "pu ps", + "bed ford", + "cul tur", + "fro gs", + "fly ing", + "cav ali", + "c ds", + "fri ger", + "street photography", + "re solve", + "tali ban", + "kan g", + "cru shing", + "ju m", + "ðŁĺ Ĵ", + "william son", + "tan g", + "cur ly", + "t man", + "veter an", + "fa ire", + "artificial intelligence", + "un anim", + "pre n", + "back drop", + "fr ances", + "oc cer", + "doro thy", + "work ing", + "ar thr", + "conver ted", + "day light", + "serv ant", + "pad dle", + "compla ining", + "thir ty", + "nad al", + "ak u", + "ibra him", + "ad dressed", + "p iss", + "green house", + "batt alion", + "si mulator", + "out lets", + "embroi dery", + "ðŁĵ ±", + "fis cal", + "ger ard", + "sas sy", + "ðŁİī ðŁİīðŁİī", + "vent ures", + "mer it", + "public ity", + "ðŁij Ī", + "sophistic ated", + "c tu", + "conven tional", + "condol ences", + "isra el", + "tra dition", + "ar an", + "te ss", + "gla d", + "ðŁĺĬ ðŁĺĬ", + "correc tion", + "ge on", + "am d", + "or ship", + "be ast", + "ch ment", + "ì ŀ", + "nic o", + "wk nd", + "wel s", + "cushi on", + "beli e", + "vo c", + "idio ts", + "under neath", + "pu ma", + "corn ell", + "en ation", + "lu l", + "swa ch", + "ab ig", + "u rer", + "mi e", + "form erly", + "ca f", + "er nal", + "chor us", + "juli us", + "sen ator", + "âľ į", + "wh ir", + "salv ador", + "ph d", + "uni fied", + "boo ster", + "graph ical", + "w rec", + "son ny", + "mi z", + "dere rs", + "s all", + "ven s", + "tusc any", + "wi d", + "y ong", + "kur ds", + "w az", + "trol ls", + "mac ro", + "cat urday", + "pre ssing", + "sa sha", + "cent ennial", + "gu sts", + "em c", + "be fore", + "den ise", + "cu st", + "ðŁĵ ¢", + "lo oo", + "base l", + "eng land", + "y olo", + "ar du", + "manife sto", + "do ha", + "ì ľ", + "kni ves", + "bourne mouth", + "bi bl", + "bar b", + "al icia", + "Ø ©", + "com er", + "cycl one", + "g it", + "ane ws", + "character i", + "vent ura", + "in tra", + "sf giants", + "hu t", + "be a", + "dar win", + "ell er", + "al v", + "re ese", + "bl y", + "kar an", + "conclu sion", + "man ny", + "fla kes", + "unite blue", + "nad u", + "co pp", + "ed ges", + "lanca shire", + "i als", + "o tta", + "philipp e", + "l ent", + "che e", + "ment ors", + "festi val", + "an ism", + "compli mentary", + "r j", + "pu g", + "d ine", + "we i", + "cli ffs", + "sar my", + "ti veness", + "treas ury", + "il and", + "after math", + "rabb i", + "ou n", + "bou quet", + "herit age", + "zi on", + "sur render", + "shen an", + "in ks", + "kar l", + "gh ty", + "pol icing", + "exam ination", + "ce y", + "per su", + "measure ment", + "hydro gen", + "lu han", + "âłĢâłĢ âłĢâłĢ", + "war i", + "о Ð", + "j y", + "fow ler", + "mis h", + "al fre", + "âĺ ij", + "bb naija", + "cat alogue", + "recogn ised", + "sa ver", + "hu skies", + "col in", + "mun do", + "si va", + "p ng", + "discoun ted", + "man utd", + "fre sno", + "de vin", + "prelimin ary", + "tro phies", + "pla stics", + "du g", + "pro cu", + "indi go", + "g ard", + "dy lan", + "pit ches", + "ground breaking", + "in son", + "bl ac", + "an thology", + "f h", + "expl ic", + "r ard", + "admi ral", + "so chi", + "la shes", + "splen did", + "en vy", + "ad v", + "sex y", + "festiv ities", + "stic king", + "bi b", + "thr ill", + "op p", + "ari el", + "botan ical", + "endur ance", + "fe males", + "br icks", + "vat ican", + "black pool", + "ber mu", + "br ough", + "roll er", + "bi d", + "sue de", + "sloven ia", + "mm ing", + "ml b", + "med alist", + "di ans", + "rehabil itation", + "ne on", + "s go", + "li thu", + "ram os", + "z ed", + "pi anist", + "inten sive", + "broad band", + "stu dy", + "peter sburg", + "lu ca", + "ah hhh", + "phys ician", + "dill on", + "tele com", + "gri ef", + "mu n", + "ac ro", + "si ded", + "s ly", + "blo ws", + "classic cars", + "tri um", + "ar gy", + "? :", + "h ri", + "marsh mal", + "âĢ ĵ", + "to pping", + "war saw", + "tran sc", + "preserv ation", + "b av", + "re friger", + "experim ents", + "ä º", + "gl it", + "sli ga", + "g age", + "fac tor", + "flav ours", + "br ony", + "sp o", + "cook book", + "carri age", + "aw ay", + "ny fw", + "on ian", + "w g", + "simp sons", + "ro lex", + "ðŁı ¿", + "cro sby", + "ãħ ¤", + "cre di", + "syn dic", + "pu bs", + "ali fe", + "poor ly", + "mac ed", + "ðŁĺ ŀ", + "behin dthe", + "w enger", + "n ats", + "ðŁİ Ł", + "rubb ish", + "procedu res", + "typho on", + "opho bia", + "er do", + "fu el", + "vi era", + "bu mps", + "millenni um", + "new zealand", + "lec tures", + "it on", + "mil ky", + "respon ded", + "ê °", + "landsc ape", + ".. @", + "bo ther", + "âĸ ¶", + "z hang", + "huawe i", + "tu ition", + "s worn", + "in u", + "y or", + "pa olo", + "au ditions", + "ab il", + "malay sian", + "ho ps", + "fe athers", + "mp le", + "au ts", + "ã o", + "boun ty", + "ic he", + "ì ĺ", + "sh q", + "pin ot", + "ge ars", + "disapp ear", + "video games", + "t na", + "alzheim er", + "ðŁĮ ŀ", + "a ji", + "under wear", + "swit ching", + "sign age", + "o scar", + "ec on", + "dro w", + "cl int", + "pl ated", + "gun dy", + "emb lem", + "ho es", + "ici st", + "nel ly", + "juni or", + "road show", + "miner als", + "at le", + "alexand ria", + "ac claimed", + "v ell", + "shi va", + "ad he", + "en ne", + "amne sty", + "h ounds", + "councill or", + "ðŁĴ ¦", + "aes the", + "part nering", + "influ enced", + "mag no", + "fl are", + "extin ction", + "civil ian", + "maje sty", + "va il", + "law makers", + "rac ks", + "mc c", + "ori an", + "sp ices", + "er rors", + "may er", + "co ca", + "pa i", + "s ooooo", + "reti ring", + "ba thro", + "ðŁĻĮ ðŁĻĮ", + "âĸ ª", + "su f", + "endor sement", + "buil ding", + "broo ch", + "pal la", + "arvin d", + "ag ent", + "kar ate", + "r hi", + "c tv", + "ta ine", + "um m", + "ba x", + "reig ns", + "uni of", + "enterpri ses", + "adel e", + "fla ke", + "at tire", + "bru ce", + "ba hamas", + "gra vy", + "sa in", + "che ek", + "tri vi", + "lo v", + "e en", + "bb lo", + "lady gaga", + "itt a", + ". \"-", + "du stin", + "observ atory", + "eigh th", + "bloom berg", + "kh s", + "f cc", + "gi st", + "commemor ate", + "ve er", + "sexu ality", + "ed c", + "nic ole", + "vac ancy", + "u ser", + "son a", + ":' (", + "dipl oma", + "t end", + "up grades", + "Å Ł", + "jura ssic", + "cardi ac", + "dr s", + "widesp read", + "à ł", + "dail ies", + "vend or", + "sim plicity", + "wi der", + "len ses", + "supp lements", + "de pos", + "ob served", + "vin es", + "parti ally", + "renew al", + "collabor ate", + "ali g", + "fin ity", + "ph u", + "zz y", + "pe tit", + "ðŁĵ ħ", + "z in", + "i gu", + "sm ack", + "fall on", + "ðŁĵ £", + "back wards", + "comp onent", + "o so", + "compati ble", + "bin ding", + "zur ich", + "thom e", + "w ounds", + "ly ric", + "fresh men", + "sne aky", + "fi bro", + "di et", + "emplo yer", + "in sect", + "h ated", + "sch er", + "raz or", + "n sw", + "boo ker", + "califor ni", + "av fc", + " °", + "preten ding", + "pep si", + "al is", + "un titled", + "k art", + "grand parents", + "e the", + "o ck", + "lux emb", + "visu als", + "small business", + "abdul lah", + "min ho", + "su baru", + "h ra", + "reve aling", + "heart breaking", + "clar ity", + "am g", + "sl r", + "** **", + "âŀ ĸ", + "recor d", + "ici ary", + "min ded", + "ye h", + "exce ssive", + "knu ck", + "icec ream", + "tru th", + "ev ic", + "ta stic", + "ant arc", + "ren dering", + ", ,", + "mit t", + "loren zo", + "st patrick", + "bound ary", + "zi g", + "vo cab", + "osa ka", + "fur n", + "tu n", + "gu l", + "s ounding", + "blo gger", + "utter ly", + "g af", + "adv ancing", + "l cd", + "mar gin", + "lifel ong", + "solst ice", + "sh ra", + "wa its", + "ple ar", + "bre ach", + "en ligh", + "ad er", + "itt le", + "c ation", + "ho on", + "stu died", + "?? ???", + "k ash", + "ev angeli", + "ps l", + "wei ghts", + "met als", + "ty res", + "tur no", + "wi e", + "car b", + "g ale", + "se al", + "sun ite", + "am ic", + "patter son", + "á n", + "eu ph", + "up stairs", + "quali fiers", + "khali fa", + "apple music", + "ìĨĮë ħ", + "vau ghan", + "al ter", + "cru iser", + "mu a", + "t ana", + "kat rina", + "id ols", + "spo iled", + "secre tly", + "fi bre", + "part nered", + "um es", + "gi ov", + "com et", + "screenshot saturday", + "k eller", + "fil tr", + "fe t", + "con way", + "pe u", + "bad minton", + "gi d", + "m ound", + "don key", + "bu ff", + "lea ther", + "lar gely", + "bro ch", + "int ments", + "am use", + "r k", + "sto ve", + "impac ted", + "con t", + "cr acks", + "prison er", + "bar i", + "contrac tor", + "ori oles", + "domin ate", + "pol ar", + "am elia", + "dr c", + "ðŁijĮ ðŁijĮ", + "vi st", + "su arez", + "injec tion", + "blo oms", + "ðŁļ¨ ðŁļ¨", + "sti ff", + "pay pal", + "sno wing", + "thur sdays", + "goo se", + "we dge", + "educ ated", + "weak ness", + "de cker", + "abud ha", + "bree zy", + "Û Į", + "hope ful", + "o bi", + "rai der", + "gh am", + "de u", + "se ve", + "par tly", + "fu t", + "infu sed", + "mer ri", + "than e", + "some time", + "hu e", + "me in", + "cre dit", + "sli ding", + "ran de", + "cher ry", + "dead pool", + "sh ol", + "ar am", + "under wood", + "sky e", + "distur bing", + "m nt", + "poli shed", + "guardi ans", + "ha dn", + "pic asso", + "ari us", + "ak shay", + "ir ri", + "j h", + "happ en", + "la kh", + "dal ton", + "at the", + "s well", + "mar sha", + "re h", + "cour s", + "j kt", + "top us", + "serv ice", + "r ink", + "hack ers", + "dono van", + "hor o", + "tc m", + "may hem", + "cha se", + "dev ops", + "ken sing", + "sc up", + "sh ere", + "quali fication", + "c live", + "ton g", + "n ancy", + "mar is", + "der dale", + "ber man", + "cinde rella", + "jol ly", + "ci c", + "loo t", + "collecti bles", + "hom icide", + "g ge", + "epide mic", + "su ites", + "mu ddy", + "gi mme", + "e rec", + "- *", + "tal la", + "lis le", + "embro ide", + "ðŁĩ© ðŁĩª", + "veriz on", + "ve ctor", + "be anie", + "arti san", + "ga in", + "flo res", + "vi gil", + "u so", + "ðŁĻı ðŁı½", + "grin ding", + "gh er", + "air ports", + "respon sive", + "shaf t", + "can cel", + "ceremon ies", + "e me", + "at ari", + "bru shes", + "eag er", + "bo hemi", + "children s", + "yan kee", + "ma a", + "suspen se", + "mor an", + "mac ar", + "sun flower", + "cre w", + "vo id", + "ke ar", + "fashi oned", + "jen nings", + "sunday funday", + "sub missions", + "me ad", + "her man", + "wa i", + "crit ically", + "le um", + "baek hyun", + "for cing", + "co bra", + "ãģ ®", + "acqu ire", + "al k", + "ge ology", + "pri mar", + "import antly", + "ire z", + "bunde sliga", + "curi osity", + "sen a", + "stric t", + "con soli", + "win ters", + "ven om", + "chelten ham", + "ðŁį º", + "cen a", + "t at", + "ba in", + "glo ver", + "under cover", + "as ses", + "car n", + "memorial day", + "am eli", + "i rene", + "ch on", + "syn thesis", + "spe edy", + "mitsu bi", + "sla yer", + "compos ite", + "under stands", + "pe w", + "inter rup", + "hen ri", + "mor row", + "an om", + "thof july", + "g lee", + "thre e", + "ðŁĺ ®", + "and hi", + "ch att", + "renew ables", + "ye s", + "trans fers", + "!!!! !!!!", + "bab u", + "du ter", + "lo ops", + "pe ers", + "o ilers", + "pau lo", + "ic ation", + "h mu", + "war a", + "mer cer", + "hom eland", + "fu ji", + "ale y", + "year book", + "re m", + "re en", + "ab sur", + "bo is", + "] :", + "caes ar", + "shot gun", + "kur dish", + "o ren", + "ra e", + "anci es", + "ty pic", + "f h", + "def ault", + "re plic", + "lu k", + "trans actions", + "r ys", + "infan try", + "ðŁį ¾", + "cho w", + "chick ens", + "ba gh", + "wy att", + "ay e", + "gg i", + "bre ws", + "ed itions", + "mi ra", + "commen cement", + "pre su", + "peris cope", + "ic hi", + "guatem ala", + "zam bia", + "pain ts", + "wit ches", + "wan i", + "un dere", + "cro y", + "vo ws", + "us mc", + "hear ted", + "theat res", + "shu ffle", + "le vel", + "mul tic", + "squee ze", + "fer n", + "app et", + "post al", + "mal t", + "on board", + "ld nt", + "co o", + "s sc", + "k ac", + "ðŁĺ ĩ", + "sc rap", + "mar cos", + "deal ers", + "ann u", + "mill er", + "co ve", + "ul ary", + "vladi mir", + "be ef", + "th ur", + "pick led", + "se same", + "bengal uru", + "mo tt", + "kathle en", + "hi st", + "no tor", + "dr ank", + "du chess", + "snow fall", + "e ff", + "tin y", + "j n", + "sy our", + "speci alists", + "scot us", + "bay lor", + "eve rest", + "mali bu", + "pre m", + "harm ful", + "l ali", + "b ates", + "g ye", + "differen ti", + "and ra", + "geome try", + "el over", + "black out", + "== ==", + "ko ta", + "inter act", + "asi an", + "la yo", + "samu rai", + "fi del", + "exhau sted", + "gla di", + "pd t", + "spher ic", + "anti qu", + "guit ar", + "stu ri", + "ho pper", + "ang le", + "f ills", + "sla p", + "mi th", + "rod ney", + "ong i", + "in som", + "pre venting", + "cassi dy", + "ap ho", + "ore gon", + "lo in", + "ham mond", + "contribu ting", + "f n", + "gar ri", + "ori on", + "comp elling", + "escap ing", + "aim ing", + "plu mb", + "bi stro", + "be asts", + "concer ning", + "bo e", + "do pp", + "shop local", + "stumb led", + "âĤ ¹", + "naz is", + "âĢįâĻĤ ï¸ı", + "gest ure", + "war ts", + "us open", + "hi ggins", + "char li", + "hang s", + "bom bers", + "° :", + "fe eds", + "c ch", + "st il", + "nic ola", + "ðŁĵ º", + "clam ation", + "tro pic", + "af ro", + "ou k", + "expen ses", + "der rick", + "al ine", + "fa w", + "reg ard", + "im er", + "sat in", + "thi um", + "ry der", + "pear l", + "te ss", + "mm mmm", + "sen ses", + "ðŁĩ ¹", + "positi ve", + "exhau st", + "occu r", + "nor ris", + "lil ly", + "is les", + "direc ting", + "yo fficial", + "count less", + "sam ar", + "on stage", + "flo ck", + "mir rors", + "arch er", + "mo i", + "k d", + "vi v", + "in os", + "si kh", + "le i", + "sen sory", + "br its", + "kno x", + "chest nut", + "op y", + "coli seum", + "z af", + "di vin", + "adap ter", + ":) ))", + "tem ple", + "ku n", + "hel mets", + "t df", + "gu ide", + "m old", + "o ids", + "lu ther", + "he is", + "monaster y", + "sp ree", + "k lu", + "brit ney", + "jagu ars", + "gre ats", + "c cc", + "ky rie", + "machin ery", + "cric ket", + "re ro", + "ab o", + "aspir ing", + "semi finals", + "ale ss", + "sig natures", + "var d", + "me th", + "her bal", + "hol den", + "king dom", + "ap or", + "reg gie", + "ore o", + "palestin ians", + "em mys", + "sec tional", + "ro i", + "ney mar", + "qu el", + "cu ll", + "l ka", + "haz el", + "estim ate", + "ul ties", + "go w", + "be a", + "purch ases", + "bel ts", + "protec ts", + "m é", + "gue ssing", + "bb o", + "clau dia", + "fr acking", + "jon ny", + "el k", + "cel tic", + "al mighty", + "ra je", + "courty ard", + "ig i", + "can es", + "ðŁĴª ðŁı»", + "bank rup", + "le thal", + "âľĮ ï¸ı", + "graphic design", + "vad er", + "penc ils", + "rough ly", + "dan te", + "m fg", + "const ell", + "cam el", + "j b", + "bloss oms", + "en to", + "balo chistan", + "cine mato", + "ill ard", + "jer sey", + "con sent", + "dent ed", + "con templ", + "sch er", + "hol i", + "lou gh", + "st our", + "a yo", + "begin ners", + "cur b", + "v hs", + "a jax", + "du ff", + "av eng", + "dom est", + "commit ting", + "ai red", + "cha p", + "hedge hog", + "disappo inting", + "freel ance", + "in land", + "char ms", + "ðŁĺį âĿ¤ï¸ı", + "ai sh", + "m x", + "buck le", + "ti dal", + "per mit", + "bo ating", + "ra cha", + "kend rick", + "b ello", + "b hi", + "ple a", + "estim ates", + "l b", + "apo logies", + "jay a", + "bb l", + "ast oni", + "inter state", + "main taining", + "el bow", + "mu p", + "ep it", + "ðŁĺ ¡", + "viol ations", + "def end", + "be h", + "sl c", + "am ir", + "pur i", + "ti um", + "fi fa", + "blur ry", + "scri m", + "ðŁĻı ðŁı¾", + "ma ple", + "rel atives", + "âĺ Ŀ", + "cho c", + "con nor", + "⾨ ⾨", + "whi sp", + "list ings", + "ma ze", + "than king", + "ri dd", + "grass roots", + "shi fting", + "desper ately", + "gor illa", + "den i", + "ju les", + "stra th", + "g ley", + "ja in", + "bu ick", + "t anner", + "ðŁĴ Ŀ", + "ga e", + "pri m", + "it ors", + "n ano", + "separ ation", + "armen ia", + "bor deaux", + "ðŁ ħ", + "pj net", + "bu rial", + "e bon", + "glo ss", + "re new", + "gri er", + "spe eds", + "comic books", + "sym boli", + "pur poses", + "ãħł ãħł", + "spati al", + "no table", + "ci on", + "n ps", + "ho ffman", + "nor man", + "rt g", + "du sty", + "situ ated", + "tr an", + "k fc", + "em en", + "nic kel", + "hast ings", + "sett ling", + "gr it", + "l ena", + "w aw", + "art s", + "gu m", + "ca regi", + "le wis", + "sapp hire", + "rememb er", + "embed ded", + "t lc", + "bl at", + "serge ant", + "el sa", + "boot camp", + "bow man", + "photo graphic", + "pill ars", + "direction ers", + "classi fied", + "no is", + "ve er", + "barre ls", + "wh oop", + "ðŁĺ± ðŁĺ±", + "fe male", + "petro leum", + "medi a", + "e fc", + "poké mon", + "ठķ", + "enthusi astic", + "var un", + "pro files", + "pedi atric", + "acci dents", + "con rad", + "jan g", + "jo jo", + "ac or", + "ob server", + "l f", + "live stock", + "for gi", + "fo s", + "el m", + "an and", + "go e", + "c ere", + "avoi ding", + "gri t", + "om an", + "thank fully", + "scat tered", + "nick y", + "cylin der", + "chees y", + "di ver", + "mahe sh", + "cav es", + "ear liest", + "qu inte", + "subjec ts", + "b end", + "gul f", + "vocali st", + "glu e", + "pat ches", + "un stopp", + "sny der", + "demonstr ating", + "pi o", + "hor ns", + "wic kets", + "and the", + "r ama", + "yo on", + "stra ight", + "bed time", + "or ang", + "bul lets", + "sa urus", + "min ers", + "inci dents", + "! ...", + "ðŁİ ¸", + "ag ers", + "hand les", + "stat es", + "in ity", + "d ons", + "incredi ble", + "emin em", + "avi v", + "ru dy", + "moz art", + "folk lore", + "appli ances", + "mt l", + "fre y", + "di as", + "hu a", + "page ant", + "stri ve", + "im prison", + "bul lish", + "r ana", + "al erts", + "bb mas", + "hy per", + "derby shire", + "re cre", + "re dd", + "debor ah", + "cosmo s", + "law son", + "mel anie", + "psy cho", + "ho or", + "doo dles", + "sni per", + "shad y", + "man tle", + "canadi an", + "new year", + "inter actions", + "separ ated", + "cor ds", + "spiritu ality", + "ap u", + "it o", + "p ct", + "pel osi", + "rebel lion", + "se iz", + "wor cester", + "sec tors", + "ul i", + "san ta", + "Ð µ", + "ðŁĩªðŁĩ ¸", + "bi ased", + "class ical", + "gam ma", + "dee plear", + "emer ge", + "back er", + "sur ance", + "hand crafted", + "ðŁİ ¥", + "franc is", + "mill an", + "ic i", + "cro wn", + "wo w", + "stri ped", + "un fair", + "relax ation", + "³ ï¸ı", + "embrac ing", + "she alth", + "pale o", + "martin i", + "dist illery", + "wr ink", + "or k", + "na th", + "hay ley", + "cour thouse", + "si ber", + "sa di", + "quiet ly", + "mel t", + "m sm", + "me h", + "smart phones", + "rel ent", + "pp ing", + "war wick", + "co logne", + "gli a", + "cot ton", + "pro g", + "lon e", + "ip sw", + "star ters", + "expan ds", + "u mp", + "su ed", + "ski pper", + "infe ctions", + "ing le", + "à ¡", + "cler k", + "demonstr ate", + "ac ar", + "ðŁĺĤðŁĺĤ ðŁĺĤ", + "ti bet", + "bun s", + "alo m", + "demol ition", + "ssi a", + "g st", + "[ ]", + "so ar", + "âĺ Ģ", + "ðŁĺ ª", + "ðŁĵ Ĭ", + "dee pest", + "beyon d", + "are t", + "att ends", + "activ ated", + "di mit", + "âļª ï¸ı", + "high lighted", + "magaz ines", + "rum or", + "az za", + "steph ens", + "dol ph", + "sho ckey", + "mat s", + "we av", + "mel an", + "serv ers", + "tra um", + "ku sh", + "æ Ĺ", + "bab ys", + "pa z", + "a al", + "la use", + "break ers", + "canter bury", + "ul ture", + "mi ri", + "euro s", + "tane ous", + "impre ssions", + "du tch", + "il d", + "gh i", + "pur due", + "adequ ate", + "l p", + "sy ner", + "ang ler", + "du rable", + "gal ore", + "ro wn", + "mg mt", + "ðŁĵ Į", + "lu cia", + "âĺij ï¸ı", + "zay n", + "bor row", + ". (", + "north umber", + "cru sh", + "eng a", + "su sh", + "extra vag", + "t out", + "ma hal", + "ali stic", + "ther mo", + "gall eries", + "es se", + "chi bi", + "attrac tions", + "lex ington", + "legislat ure", + "docu mented", + "resi den", + "brow nies", + "w f", + "st ool", + "plan ets", + "sho ppers", + "conduc tor", + "ms p", + "tr icky", + "fru ity", + "end ra", + "feel the", + "whi pped", + "hair style", + "re fer", + "oo k", + "oc topus", + "audi ences", + "ku mar", + "after no", + "op tim", + "c fl", + "ni p", + "gen i", + "alpha bet", + "ann ab", + "lam in", + "accep ts", + "l ng", + "ðŁĺ «", + "t ine", + "ac om", + "cheer leaders", + "t k", + "gr on", + "v g", + "k ung", + "ja x", + "dha bi", + "r ss", + "mack enzie", + "beir ut", + "clean up", + "gy psy", + "st ell", + "bur ger", + "hurric anes", + "educ ation", + "st ina", + "âĻ¡ âĻ¡", + "unfortun ate", + "jere mi", + "bad ger", + "at ers", + ": âĢ¦", + "ter ra", + "subli me", + "stu d", + "y mca", + "mr u", + "duter te", + "bren nan", + "bul b", + "mel o", + "yl on", + "hack er", + "c red", + "gu d", + "as an", + "pad illa", + "embroide red", + "vietnam ese", + "pione ers", + "projec tion", + "re boot", + "id c", + "an ey", + "pri mer", + "suff ers", + "win ding", + "p on", + "sto day", + "mor n", + "u ch", + "all in", + "adid as", + "eliza beth", + "tu ck", + "o graphy", + "ðŁļ Ģ", + "be g", + "os borne", + "ghet to", + "r h", + "cn n", + "ir ma", + "ma kin", + "cab les", + "mur ders", + "oc ks", + "inst a", + "al as", + "si k", + "cu ff", + "la re", + "foo dies", + "o vic", + "at om", + "geome tric", + "em pathy", + "ภµ", + "cent enary", + "newsp apers", + "administr ative", + "ðŁİ Ĭ", + "sti ve", + "contrac tors", + "le tt", + "tas mania", + "awesom eness", + "den sity", + "ve en", + "prince ton", + "frequ ently", + "re ject", + "gh i", + "modu lar", + "ceram ics", + "sh ag", + "ki wi", + "can vas", + "sweat shirt", + "an j", + "ti mm", + "napol i", + "il er", + "appe als", + "hamil ton", + "ma yo", + "we ave", + "arrang ed", + "whar f", + "occu py", + "b vb", + "as aki", + "ot ter", + "nor m", + "vi es", + "de tox", + "tion al", + "dere k", + "id ad", + "ad missions", + "constitu ency", + "u pper", + "woo t", + "allo y", + "se ve", + "lu b", + "un comfortable", + "ed win", + "ab re", + "d wight", + "ar che", + "virtu ally", + "sp ol", + "pri e", + "ai i", + "er r", + "swit ch", + "bar ack", + "se ok", + "cou l", + "wn t", + "pou l", + "o live", + "caffe ine", + "cardi ff", + "notor ious", + "de mp", + "ex cess", + "bar r", + "t ford", + "a jay", + "bump ed", + "my thology", + "shel ley", + "fal con", + "shakespe are", + "must angs", + "no ted", + "bon e", + "civil ization", + "sy d", + "par sons", + "un official", + "hy ped", + "sp ends", + "oppo sed", + "v ings", + "space x", + "noti fication", + "deci ding", + "bio tech", + "out si", + "sal ah", + "! .", + "fe d", + "ss y", + "c ms", + "bad gers", + "cr o", + "ela ine", + "n ba", + "dy our", + "n ant", + "honey moon", + "climb ed", + "conom y", + "ath a", + "m ell", + "ne bula", + "nature photography", + "juli e", + "bm x", + "inve sted", + "mon o", + "lieu tenant", + "wat kins", + "techn ician", + "o se", + "ka e", + "ì Ľ", + "mc queen", + "pre ach", + "trav eller", + "flexi bility", + "ze bra", + "reta iler", + "p ant", + "ben der", + "brand t", + "squ id", + "war rant", + "veri fied", + "cas s", + "pier cing", + "hon ours", + "t ying", + "mor ris", + "kis sed", + "op rah", + "panor amic", + "me i", + "splat oon", + "wich ita", + "ari as", + "gal li", + "indy ref", + "good times", + "athe ist", + "confe ssion", + "ow ski", + "re pping", + "ad ditions", + "mechan ism", + "z im", + "j ans", + "su f", + "cho pped", + "beg innings", + "vitam ins", + "ãħ¤ ãħ¤", + "or th", + "po les", + "ru b", + "antarc tica", + "indie film", + "web cam", + "ket ch", + "bre tt", + "cle ment", + "her on", + "defe ating", + "hydr o", + "buc ket", + "wand ering", + "sid ney", + "future of", + "b inge", + "on ies", + "knock out", + "administr ator", + "syn the", + "l ent", + "jan i", + "bar ley", + "premier league", + "ner ds", + "cr m", + "bra s", + "bot any", + "evol ved", + "rot ter", + "ro wed", + "tum or", + "weal thy", + " Ń", + "mon arch", + "li shed", + "da hl", + "ðŁİ ĥ", + "bu ch", + "ken yan", + "Ø §", + "red ness", + "assemb led", + "se mit", + "hud der", + "shro p", + "ran i", + "lear ning", + "mor y", + "iti a", + "geo graphic", + "worl dof", + "f b", + "pho sp", + "boo gie", + "am ped", + "? ...", + "che w", + "dwar f", + "ar us", + "s sen", + "ru sty", + "recru its", + "h k", + "gar de", + "app lause", + "vol umes", + "invol ves", + "ta c", + "hand bag", + "trans late", + "ffe l", + "se ym", + "aqu atic", + "trans fer", + "zo di", + "and r", + "acade mia", + "cr ater", + "te z", + "ar se", + "adap t", + "col oni", + "snow man", + "mal i", + "hang in", + "di schar", + "oy sters", + "pho e", + "colon el", + "w ba", + "hispan ic", + "thri ving", + "sh y", + "ag les", + "sales force", + "cre me", + "so les", + "la fayette", + "â ī", + "ter ia", + "ach a", + "sp erson", + "go go", + "car ly", + "the ore", + "am ore", + "vo x", + "af t", + "ãĤ ¹", + "stap le", + "mu ffin", + "di agram", + "ino x", + "su stained", + "av ent", + "me ta", + "arbit r", + "dec ay", + "ado le", + "Ð ½", + "ec ol", + "ph o", + "n k", + "o cu", + "gr anny", + "ç a", + "luxemb our", + "stad t", + "alber to", + "le vit", + "am as", + "d x", + "or phan", + "co bb", + "as c", + "lo gy", + "immen se", + "chan ts", + "off line", + "p ent", + "bre x", + "w inger", + "plan e", + "i el", + "nichol s", + "ca thy", + "nar uto", + "low ed", + "/ //", + "ignor ance", + "cat astro", + "you ts", + "sch en", + "buil d", + "haz i", + "s ine", + "critical role", + "du g", + "dete ct", + "lo gs", + "en amel", + "stpatrick sday", + "ed die", + "co pa", + "cigare ttes", + "ho ff", + "kay a", + "la goon", + "ra pha", + "air borne", + "choo se", + "puer tor", + "ke v", + "gui ding", + "fro sty", + "bor ough", + "mir a", + "ðŁİ Ĭ", + "cade t", + "anu sh", + "yo gi", + "e ger", + "fl ing", + "slo pe", + "nin th", + "we ston", + "foot wear", + "f n", + "may weather", + "a am", + "pla in", + "stair case", + "witne sses", + "work outs", + "ro bust", + "dex ter", + "co hort", + "ðŁļ Ĺ", + "sp ell", + "ha ze", + "o om", + "organ ising", + "wild fire", + "cont acts", + "av on", + "min o", + "upd ating", + "ðŁį »", + "li thium", + "ing ual", + "k is", + "au ga", + "lo com", + "de duc", + "u da", + "th ak", + "boy le", + "mp er", + "hot tie", + "eri k", + "re vised", + "is la", + "travel photography", + "oo za", + "en qui", + "confe rences", + "clo ver", + "g room", + "cur ves", + "live on", + "per f", + "displac ed", + "bo log", + "xx xx", + "ðŁĺ© ðŁĺ©", + "te al", + "ve ssels", + "rain forest", + "cal ci", + "pan ther", + "gira ffe", + "ta sted", + "imag ery", + "pad res", + "day time", + "bas s", + "ri pe", + "opio id", + "nu e", + "vin yl", + "invent or", + "sen s", + "process or", + "mu t", + "gad gets", + "bibl ical", + "shann on", + "jacqu eline", + "car y", + "the resistance", + "ali en", + "n vi", + "co sy", + "bi har", + "fo ley", + "ren d", + "mu gs", + "fa ken", + "cl one", + "ni allo", + "gra bbed", + "chi hu", + "power house", + "n tt", + "chero kee", + "spon ge", + "imple menting", + "rh ine", + "le one", + "ðŁį Ģ", + "pret tiest", + "infra red", + "impro v", + "swit ched", + "tu bes", + "con tr", + "bl k", + "projec ted", + "be aver", + "yo t", + "bbcra dio", + "thi gh", + "per secu", + "apologi ze", + "w ack", + "po ster", + "oli ver", + "az a", + "lou d", + "( ?)", + "f the", + "women shi", + "spar row", + "blu sh", + "us able", + "sc ales", + "it ative", + "peu ge", + "ne eding", + "legg ings", + "glam orous", + "mat ur", + "c z", + "wat t", + "da b", + "tam ar", + "et sym", + "bau er", + "heart felt", + "h n", + "else where", + "bir ch", + "alu mini", + "hu ck", + "e me", + "j l", + "traf ford", + "d z", + "por tions", + "ana sta", + "arthr itis", + "esp n", + "ber gen", + "viol ation", + "yo shi", + "c z", + "northumber land", + "clo sures", + "ðŁĩ¯ ðŁĩ", + "smi ley", + "r w", + "tel ugu", + "inten si", + "gre gg", + "ve ga", + "dun geon", + "south bound", + "ba il", + "domin ican", + "semi final", + "chap ters", + "h itch", + "van ity", + "trans iti", + "recomm ends", + "sati sf", + "bar ca", + "queen s", + "( (", + "de struc", + "stra it", + "ra vi", + "dess erts", + "in tru", + "har am", + "k os", + "fo e", + "fat ty", + "pais ley", + "magn itude", + "dri dge", + "com ey", + "schem es", + "vision ary", + "our t", + "down loaded", + "ðŁĻĮ ðŁı½", + "gd pr", + "lan i", + "p wc", + "gu ad", + "nic est", + "stake holders", + "re ferred", + "george town", + "arvind kejriwal", + "schnei der", + "in doors", + "all star", + "strand ed", + "gen der", + "ze pp", + "ma sses", + "ðŁIJ ±", + "pati ently", + "bl dg", + "z ab", + "we arab", + "vi vid", + "he ck", + "d ella", + "sy mb", + "je opar", + "la ger", + "à ª", + "comb ines", + "ne c", + "br ay", + "flo p", + "tx wx", + "jo ys", + "pon t", + "pro found", + "sur round", + "mad hu", + "ma ble", + "ay r", + "te as", + "n sa", + "open ly", + "er nest", + "ãĥ ©", + "to po", + "g na", + "anti oxid", + "ti an", + "e tr", + "c ello", + "ma thi", + "gener osity", + "b iting", + "man ic", + "kel sey", + "chee ks", + "ten der", + "w th", + "pron oun", + "ultimat ely", + "gu sta", + "ari anag", + "ger ry", + "ble ed", + "red dy", + "mic h", + "mitsubi shi", + "oper ated", + "sex ually", + "ma u", + "cl lr", + "vi ds", + "co c", + "mel ted", + "ðŁĮ Ī", + "q ld", + "ite ch", + "instru mental", + "end game", + "ðŁĵ ĸ", + "ener gi", + "brow nie", + "tam il", + "at in", + "domin ated", + "pra ises", + "fire place", + "sens ational", + "men a", + "k arti", + "un prece", + "ru pt", + "ori ental", + "mc cor", + "tour naments", + "scen ter", + "re eves", + "prescri ption", + "sam e", + "fra u", + "tru ffle", + "em bo", + "roman s", + "bla sts", + "techno logical", + "pr at", + "b sb", + "y ar", + "tren dy", + "ac l", + "al ad", + "ðŁį ģ", + "o hh", + "bankrup t", + "tho ven", + "regar ds", + "is er", + "war wick", + "vine yards", + "real m", + "niallo fficial", + "do ta", + "ge mini", + "to do", + "v able", + "¨ ¨", + "la u", + "wre ath", + "ju ve", + "nat asha", + "le ver", + "lor i", + "hor ser", + "cc tv", + "air bnb", + "es anders", + "sin clair", + "ema biggest", + "high school", + "con test", + "optimi stic", + "t te", + "ðŁĴķ ðŁĴķ", + "ss d", + "ye e", + "hel ena", + "con sen", + "ric ks", + "jes se", + "an ic", + "ðŁİ ¯", + "re acts", + "ro be", + "independ ence", + "vol tage", + "m ington", + "s ant", + "à¸Ļ à¸", + "-------- --------", + "sentin el", + "ke tt", + "rehear sing", + "aaaa aaaa", + "sof the", + "stir ling", + "sear ch", + "wi gan", + "stand out", + "sna il", + "pent agon", + "Ä ģ", + "ch lor", + "cru st", + "net any", + "chemi st", + "disapp eared", + "ric ardo", + "sp iders", + "bo se", + "war ren", + "me ssing", + "bann ers", + "gu el", + "par ach", + "ma id", + "coun ted", + "epi le", + "bon fire", + "speech less", + "se tter", + "meas ured", + "rejec ts", + "nik ki", + "le ster", + "foren sic", + "fab rics", + "alo ha", + "pre served", + "wat ford", + "deta iling", + "dar th", + "bo u", + "car ly", + "... '", + "tail gate", + "noti fications", + "å ¤", + "pas sive", + "trous ers", + "balo ch", + "ro ther", + "typic ally", + "à ¥", + "sp it", + "wi z", + "sic ily", + "technic ally", + "ex pose", + "st age", + "hu bb", + "cre am", + "cap s", + "po ke", + "sle ek", + "ju ne", + "tempor arily", + "de z", + "awak ens", + "l ame", + "_ -", + "ji ha", + "tues days", + "advis ed", + "advis ors", + "exi sted", + "dis agree", + "news room", + "lo sers", + "world tour", + "dr ying", + "al di", + "har ness", + "foot print", + "hobb it", + "p mln", + "i ro", + "que red", + "asse ss", + "gaz e", + "sa b", + "th ian", + "í Ĭ", + "ti f", + "ob serve", + "ev il", + "dra wer", + "swee p", + "cor y", + "co dy", + "kyo to", + "cal lum", + "n inj", + "lau rent", + "be i", + "sket ching", + "custom ized", + "du r", + "regre ts", + "knox ville", + "ìķ Ħ", + "mess aging", + "grac ie", + "abun dance", + "bi dding", + "bre wed", + "fl ouri", + "therapeu tic", + "alt itude", + "ho gs", + "bur ner", + "elec tro", + "wonder fully", + "he ater", + "post pon", + "li very", + "r all", + "ad as", + "a ac", + "sau l", + "brook lyn", + "play house", + "âĻ¥âĻ¥ âĻ¥", + "char itable", + "in y", + "z ah", + "compet itions", + "be av", + "plu gged", + "o is", + "do om", + "astron om", + "speci alized", + "max i", + "ta ps", + "cellu lar", + "depre ssed", + "folklore thursday", + "cri b", + "e mul", + "ë° ©", + "fi gh", + "ru z", + "car lisle", + "spe ar", + "side walk", + "de i", + "depend ent", + "lac es", + "nh s", + "ðŁĮ Ļ", + "reali zing", + "net work", + "ric he", + "re gin", + "re fresh", + "st ral", + "pa thology", + "pla id", + "psyched elic", + "hin d", + "u ka", + "algori thm", + "lin king", + "progre ssi", + "fe y", + "d ade", + "hydr ated", + "b ant", + "fam ed", + "cot sw", + "bo ise", + "as c", + "rac ing", + "ja vier", + "ww en", + "mar lins", + "poo p", + "swe pt", + "toni ghts", + "we f", + "ani me", + "slo vak", + "âŀĸ âŀĸ", + "cla us", + "lem me", + "cli ppers", + "re ls", + "arianag rande", + "r te", + "ko t", + "thal apathy", + "hungar ian", + "zu ma", + "y von", + "is u", + "jour neys", + "clin ics", + "be be", + "ww f", + "n ws", + "super heroes", + "er it", + "sle ague", + "identi fication", + "mo tto", + "ba i", + "sour ced", + "ill er", + "ap i", + "pri se", + "unprece dented", + "dam as", + "tuni sia", + "dra in", + "undere stim", + "e ther", + "quarter ly", + "rewar ding", + "al ham", + "wolver ine", + "cab ine", + "hyp no", + "nad ine", + "hav ana", + "da e", + "ðŁĵ Ī", + "dr on", + "read ings", + "b ati", + "pic o", + "mer ci", + "iti an", + "wal kers", + "el ope", + "mi key", + "god zilla", + "bur lington", + "abu ja", + "social ism", + "at ility", + "sh ell", + "harry potter", + "g no", + "ab ur", + "re leg", + "fel ici", + "ro gen", + "neuro science", + "inst in", + "ath am", + "vou chers", + "j arre", + "fu se", + "def ici", + "monte rey", + "de port", + "mid day", + "pp ard", + "fre ed", + "ame ter", + "wil t", + "n ingham", + "pr att", + "liber ty", + "slo gan", + "o to", + "pr i", + "co ated", + "c pd", + "ne tt", + "il las", + "mal awi", + "evol ve", + "accessi bility", + "ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥", + "or nament", + "b p", + "el is", + "son line", + "chi ro", + "fl ick", + "ib m", + "ar ak", + "en ables", + "gar land", + "san e", + "cu ties", + "tri p", + "rotter dam", + "n ys", + "lam ps", + "lu cas", + "bo g", + "ra ils", + "travel led", + "hic ks", + "en u", + "sab ha", + "scru b", + "hi er", + "hart ford", + "fo o", + "fer nandez", + "tre vor", + "mat tress", + "appo intments", + "ale j", + "fe i", + "o logist", + "saf ar", + "oc ta", + "sr c", + "sha un", + "ambi ent", + "dri c", + "bi ker", + "she e", + "must ache", + "h ta", + "bo one", + "her ty", + "car dio", + "bra kes", + "rec ital", + "consi sts", + "overwhel med", + "cau l", + "robb ins", + "im it", + "al th", + "ur l", + "bi bli", + "on ne", + "black livesmatter", + "diffic ulties", + "tel ang", + "tall er", + "ðŁĵ Ĩ", + "deb ating", + "bur rito", + "mo vember", + "strength ening", + "bo e", + "te stam", + "mirac les", + "base ball", + "re nee", + "ðŁijī ðŁı»", + "al fa", + "âĺ ĺ", + "unstopp able", + "ec s", + "g mo", + "giftide as", + "path way", + "fen cing", + "ðŁİ ¤", + "b ham", + "ra s", + "sk o", + "d led", + "thel ast", + "magn um", + "bin ary", + "wil de", + "wil der", + "wh ati", + "barbe cue", + "h ism", + "can oe", + "kur di", + "eli ve", + "advant ages", + "mad ame", + "bi er", + "mis sing", + "enter tain", + "air force", + "y ama", + "c is", + "hash tags", + "j is", + "ve il", + "dream y", + "ten se", + "may ward", + "ch ateau", + "hunt ington", + "âļ ĵ", + "v all", + "up on", + "bl ouse", + "dun es", + "ðŁĺ ´", + "fert ility", + "m ole", + "curren cies", + "st u", + "ber lin", + "toa sted", + "div as", + "wal t", + "lar k", + "por a", + "hit ter", + "um er", + "chil led", + "bal ancing", + "fa is", + "y in", + "or tiz", + "east enders", + "h ate", + "ur al", + "ap ril", + "tim el", + "à ±", + "per o", + "sto cked", + "respec ts", + "th t", + "best friends", + "giving tuesday", + "be ad", + "inv ent", + "im i", + "nap les", + "comb ining", + "tok ens", + "thir st", + "ma sc", + "par rot", + "sp u", + "dent on", + "* -*", + "t res", + "subur ban", + "wid th", + "si ve", + "con tender", + "siri us", + "lo k", + "troop ers", + "outra ge", + "tur bo", + "frag ile", + "me ssed", + "do h", + "disc ord", + "netany ahu", + "re sign", + "forgi veness", + "mo han", + "mun ch", + "cam ou", + "identi fying", + "enab ling", + "hot ter", + "thorn ton", + "jai pur", + "ar ya", + "ðŁı» âĢįâĻĢï¸ı", + "mu staf", + "maj ors", + "o ke", + "du ffy", + "roh ing", + "til t", + "ðŁĩ®ðŁĩ ³", + "rock star", + "she ep", + "hend rix", + "ra v", + "in vention", + "do u", + "lagun a", + "gru mpy", + "sw is", + "im pe", + ") '", + "you ths", + "bun ker", + "st ache", + "oppo se", + "indi es", + "acceler ate", + "ml p", + "ed en", + "w ann", + "k ail", + "akshay kumar", + "su pt", + "pol ym", + "midd leton", + "extra ordin", + "wil son", + "australi an", + "alumini um", + "way ne", + "alum nus", + "mat ics", + "gri m", + "er nie", + "opp a", + "competit ors", + "rand all", + "h ence", + "decla res", + "pre aching", + "sha he", + "can e", + "sustain able", + "stap les", + "le dge", + "ad ena", + "doctor al", + "bur gundy", + "decor ate", + "ren dered", + "ri sen", + "pr ank", + "di or", + "bee thoven", + "flo or", + "ac com", + "to t", + "ho dg", + "touri sm", + "say in", + "objec tive", + "mar kers", + "premi ership", + "en abled", + "camou fla", + "gi ant", + "Ñ ģ", + "smo key", + "ric ket", + "pan g", + "de pending", + "s ation", + "evol ving", + "inter cep", + "cen sus", + "tof the", + "re en", + "mendo za", + "trum pet", + "marke ters", + "an it", + "ðŁĻ Ĭ", + "north western", + "v la", + "foto gra", + "blackand white", + "che wan", + "wi g", + "tro om", + "ginger bread", + "k n", + "ro mero", + "n fc", + "or chi", + "fun ko", + "sour ce", + "f s", + "ra ped", + "o st", + "tar ot", + "ann ually", + "ðŁĺ ¬", + "r ill", + "del av", + ".. !!", + "se s", + "can n", + "medic are", + "ph el", + "ape x", + "guardi an", + "rema ined", + "r pm", + "a ñ", + "story month", + "instag ood", + "neighb our", + "p ing", + "sem ite", + "my stic", + "as cot", + "mat er", + "hand ful", + "dang ers", + "ti d", + "ana heim", + "opol y", + "sh allow", + "nami bia", + "tor ia", + "procu rement", + "big bang", + "announ cements", + "prosecu tor", + "beng als", + "sal le", + "en roll", + "ga stro", + "sugge stion", + "ba k", + "ha ul", + "budd hism", + "berni esanders", + "flu te", + "fati gue", + "cyn thia", + "cho i", + "ir win", + "gu a", + "str ous", + "h p", + "ba p", + "satisf ying", + "play a", + "ðŁİ ¼", + "inst ap", + "al ice", + "t p", + "irri gation", + "ðŁĩ¬ðŁĩ §", + "in tric", + "clu es", + "ple x", + "sa x", + "he pat", + "dump ed", + "signific ance", + "by u", + "medic ation", + "pro v", + "tough est", + "corn ish", + "âŀ ľ", + "kel ley", + "u v", + "si zz", + "si bling", + "me st", + "di stor", + "diplom atic", + "aun tie", + "b hat", + "son ic", + "bren da", + "pump kins", + "ro ch", + "black burn", + "ur ged", + "shi a", + "arrange ments", + "floo d", + "sa unders", + "lec turer", + "nou ri", + "popul ations", + "diplom acy", + "consist ently", + "ðŁ¤ Ļ", + "t mund", + "cauli flower", + "l ily", + "vocab ulary", + "vari eties", + "coo ker", + "up town", + "qu ent", + "mo sa", + "re inde", + "velo city", + "spru ce", + "social medi", + "i ber", + "volun tary", + "proce ssed", + "bal tic", + "y ang", + "leban ese", + "d p", + "dol ly", + "arrange ment", + "y uri", + "cran berry", + "kal yan", + "elev ation", + "cli ff", + "pu shes", + "ìĬ ¤", + "sil ic", + "co wx", + "eter nity", + "sla ves", + "vine gar", + "glou cester", + "con tained", + "breaking news", + "aga inst", + "renov ated", + "norm andy", + "hero in", + "ys m", + "mo ds", + "gre ek", + "un di", + "tren ch", + "v h", + "encoura ges", + "head ache", + "gr ange", + ": '", + "ever green", + "Ù Ĭ", + "reck on", + "ab used", + "th ru", + "cho ice", + "ti dy", + "col der", + "scho ice", + "ha in", + "bru m", + "li ars", + "bre it", + "yor ker", + "sh ack", + "he idi", + "micha els", + "sco pic", + "fasci st", + "play ful", + "ca c", + "yas ss", + "sh ad", + ".. ?", + "qu en", + "ram irez", + "clif ton", + "pr s", + "best fan", + "âģ ł", + "gener ating", + "head set", + "disappo intment", + "abstr act", + "bo iled", + "paren thood", + "azerbai jan", + "exhib iting", + "bom bay", + "oli vier", + "ko so", + "un lea", + "mat ernity", + "iz er", + "si ves", + "r hu", + "col l", + "saskat chewan", + "fre akin", + "de k", + "na g", + "stab ili", + "ðŁį ķ", + "organi zer", + "bo sses", + "ar u", + "u va", + "at able", + "ta un", + "after wards", + "fert ili", + "ver ge", + "az i", + "mor ph", + "๠ģà¸", + "jer k", + "cosme tic", + "ko w", + "stru st", + "ap ache", + "post cards", + "for mul", + "ì ĭ", + "spin al", + "jack pot", + "elec tri", + "à Ń", + "lo y", + "gra der", + "diab lo", + "ar di", + "he sit", + "f w", + "arch ery", + "pa sh", + "the ories", + "repe al", + "re live", + "per cy", + "âĺ Ĩ", + "im in", + "syn chron", + "sham poo", + "coup ons", + "o to", + "la i", + "thou ght", + "luxembour g", + "mo v", + "ðŁĺ ¥", + "ge mma", + "se ated", + "m ga", + "strat ford", + "un certainty", + "shi fts", + "est o", + "fo ol", + "fire arms", + "cor rie", + "ki ki", + "appa rent", + "p ills", + "olym pia", + "fi d", + "elev ated", + "de cks", + "ignor ing", + "av alan", + "ro v", + "whist le", + "p tsd", + "milit ants", + "robo tic", + "pac ers", + "quil t", + "bankrupt cy", + "lic h", + "per cussion", + "celebr ity", + "al s", + "( ;", + "su t", + "pokemon go", + "h g", + "off s", + "gibr altar", + "scre ams", + "billi e", + "gen ome", + "mar in", + "be ams", + "arch bishop", + "em in", + "bedro oms", + "g ated", + "ol ly", + "warran ty", + "at own", + "cudd les", + "gun na", + "k ic", + "vi ve", + "cy mru", + "nar row", + "pro b", + "le o", + "refe rences", + "manufac tured", + "cho pper", + "brun swick", + "sem is", + "don ia", + "r ye", + "man o", + "hur ting", + "? #", + "hol li", + "investig ations", + "c els", + "ðŁĵ ŀ", + "le ster", + "temp les", + "sto rey", + "mc mahon", + "toi lets", + "wo of", + "ï¸ İ", + "le verage", + "at om", + "night mares", + "victor ious", + "haun ting", + "custom er", + "ag i", + "yo ongi", + "mon ty", + "ver onica", + "w ur", + "inti mid", + "blan kets", + "volu tion", + "j m", + "âĺ İ", + "am on", + "jud ith", + "ðŁĺİ ðŁĺİ", + "distr acted", + "dri p", + "hurric ane", + "and es", + "revel ation", + "tro op", + "ab leg", + "col lin", + "tibet an", + "wor rying", + "inter nationally", + "eat er", + "camero on", + "brad or", + "y uk", + "ðŁĴĹ ðŁĴĹ", + "tra k", + "slo pes", + "ci er", + "ne a", + "ol er", + "ta ka", + "albi on", + "volcan ic", + "am n", + "a fi", + "ob stac", + "face time", + "ger ing", + "n pr", + "metall ica", + "organ ic", + "ðŁĴ ¡", + "ki dd", + "d ances", + "pemb ro", + "wash er", + "m its", + "om er", + "emo tionally", + "tan go", + "ip o", + "do cks", + "scan ning", + "spec s", + "tho m", + "the ology", + "emer gen", + "om i", + "g pa", + "selec tions", + "un necessary", + "ima ge", + "ter s", + "induc ed", + "gi gan", + "rent als", + "supp lied", + "m fa", + "shan kar", + "lat er", + "pa jam", + "cla ve", + "Ù ģ", + "ma hin", + "carl son", + "avi an", + "ano va", + "kati e", + "aj ith", + "design ated", + "chocol ates", + "investig ators", + "gla zed", + "prin cess", + "er ry", + "ra gn", + "ou rable", + "hr u", + "sun dance", + "peuge ot", + "steam punk", + "gh lin", + "gre ase", + "hi res", + "z ap", + "per ce", + "j ill", + "tom e", + "he hehe", + "joy ful", + "mae stro", + "ni shed", + "gene alo", + "v ich", + "p its", + "fox es", + "good man", + "emer son", + "lo bes", + "con verse", + "o ats", + "thom son", + "ra him", + "mal ware", + "ah i", + "man kind", + "re sin", + "im g", + "sw ood", + "kin der", + "sc roll", + "ar a", + "sak ura", + "ro bbed", + "xi on", + "ny a", + "c ism", + "ce dar", + "be in", + "mour ning", + "tor to", + "heath row", + "done gal", + "bar b", + "hydr ation", + "k or", + "elim ination", + "su pdates", + "hill s", + "appe ti", + "star red", + "ko m", + "gw en", + "dd d", + "cra y", + "sc anner", + "personal ised", + "seren ity", + "re design", + "meta ph", + "box ed", + "judg ment", + "no se", + "ë ¹", + "er ad", + "ac ne", + "supp liers", + "ener getic", + "v om", + "as ap", + "ðŁĶ ¸", + "ir vine", + "hat ch", + "la ss", + "ad ren", + "waff les", + "accur ately", + "ici o", + "itt le", + "se un", + "occup y", + "web cam", + "thene w", + "ent es", + "ga i", + "j w", + "accoun table", + "vis or", + "ir rit", + "licen sing", + "hudder sfield", + "gen ie", + "ðŁİ ¾", + "atmo spheric", + "ten sions", + "spart an", + "clif ford", + "ol an", + "north bound", + "ame en", + "cen sor", + "u el", + "ster y", + "$ $", + "far rell", + "hy ster", + "cl t", + "se dan", + "rep lied", + "descri bing", + "micro wave", + "sla b", + "pro sp", + "assi sting", + "ru bio", + "e than", + "hh hhh", + "gu ay", + "z man", + "ra ise", + "roll ing", + "o e", + "n ile", + "ambro se", + "scar borough", + "hero ic", + "coo ks", + "mor t", + "chop ra", + "ðŁĮ ·", + "to b", + "shav ing", + "stac ey", + "dor m", + "motor sports", + "wi ki", + "fol ds", + "sp iced", + "stress ful", + "liter al", + "fu dge", + "pe ggy", + "wa ite", + "tre sses", + "se sh", + "pr ic", + "ðŁİ ħ", + "fri ght", + "r va", + "mumb ai", + "po m", + "tt v", + "cel lar", + "tom e", + "andro id", + "dor is", + "tsun ami", + "tin der", + "o ec", + "m wc", + "dor tmund", + "no thin", + "l iti", + "so u", + "believe in", + "at u", + "kno cks", + "mag ni", + "ss sss", + "ro hit", + "ine ws", + "ang i", + "m andy", + "ke ttle", + "intermedi ate", + "av ant", + "cur l", + "endor sed", + "ori o", + "ur t", + "consider ation", + "wi res", + "shel ters", + "b ino", + "vik ram", + "imple mented", + "ly dia", + "bu k", + "paro dy", + "c news", + "under graduate", + "canu cks", + "sam i", + "polit ically", + "ro tten", + "gh z", + "tex tiles", + "over load", + "moder ni", + "recre ational", + "fli r", + "bat on", + "typo graphy", + "ov ation", + "intrigu ing", + "pilgri mage", + "al ge", + "ad ays", + "tcm party", + "sp elled", + "cur ls", + "boo ze", + "ste m", + "ann es", + "ir ls", + "spon ge", + "sho pper", + "sig nation", + "bra ss", + "mi stress", + "le ah", + "beg inner", + "lau derdale", + "augu st", + "pre school", + "ta ping", + "tai pei", + "execu tives", + "b d", + "rhe tor", + "esc or", + "immun o", + "deeplear ning", + "stat ues", + "it us", + "manu script", + "ly ric", + "cor vette", + "mol ly", + "la ge", + "de p", + "cn bc", + "le st", + "je ssi", + "fi fe", + "griff ith", + "oppo sing", + "ran g", + "dr ills", + "respec tful", + "p ity", + "d ell", + "har ding", + "play boy", + "blo ke", + "shut out", + "k ili", + "o sp", + "se attle", + "bc poli", + "mis es", + "journ als", + "team ing", + "es ther", + "fre ddy", + "Ķ ï¸ı", + "metr ics", + "no tre", + "gar ry", + "for ty", + "navi gate", + "perio ds", + "bened ic", + "j id", + "da w", + "ance stors", + "restor ing", + "con g", + "aller gy", + "tit anium", + "c ence", + "lean ing", + "ab bas", + "v ast", + "uc f", + "roof ing", + "e man", + "seve rely", + "vo gue", + "ve au", + "in bound", + "d z", + "tane ously", + "stret ching", + "man chester", + "dr yer", + "dav is", + "kan th", + "the game", + "it ted", + "re tain", + "el les", + "conge stion", + "frat ernity", + "ol lie", + "lo ki", + "fre ely", + "cho o", + "pon y", + "sc ep", + "tab ly", + "bal t", + "rock n", + "di me", + "lo gging", + "ðŁį ·", + "ad u", + "ha voc", + "water ford", + "char is", + "swee tie", + "run ning", + "ner d", + "erdo gan", + "z ara", + "weigh ing", + "fif ty", + "pre cise", + "low ell", + "kurdi stan", + "r yo", + "or th", + "syn th", + "lin ers", + "phenomen on", + "art illery", + "il legally", + "constru ct", + "nostal gic", + "gar th", + "al ta", + "shel ton", + "a sean", + "w ander", + "dur ban", + "di versi", + "bon o", + "cl on", + "le man", + "sh un", + "obstac les", + "appet ite", + "fe eder", + "respir atory", + "di xie", + "formu la", + "an to", + "so ber", + "extin ct", + "au c", + "ing les", + "legitim ate", + "; ;", + "min nie", + "ipsw ich", + "dram atically", + "ðŁijı ðŁı¼", + "ingh am", + "milit ary", + "mon et", + "us navy", + "for k", + "dun no", + "play er", + "q otd", + "st oo", + "ex or", + "ethiop ian", + "film fest", + "pe red", + "c ate", + "sau di", + "in ner", + "sin cere", + "tion ality", + "ale e", + "de eds", + "cooper ative", + "ir onic", + "cro cod", + "br ary", + "post season", + "cam per", + "can ary", + "e in", + "exten sions", + "nb d", + "sher wood", + "spo kane", + "hu mp", + "jit su", + "ê ¹", + "dar yl", + "p si", + "stab bed", + "offer ings", + "expe cts", + "cav al", + "body building", + "fr aming", + "f ca", + "ye arly", + "bom bed", + "sk il", + "resear ching", + "jud iciary", + "gree ted", + "tu dor", + "mil o", + "innov ate", + "ðŁĺ Ľ", + "r hs", + "ru by", + "contribu tor", + "fam er", + "soci ally", + "m lin", + "fi ery", + "ut ter", + "beau t", + "it os", + "de voted", + "rain bow", + "bar ney", + "pe ren", + "ar jun", + "r na", + "gab by", + "ut i", + "hann ity", + "pick le", + "ser v", + "qu akes", + "pp e", + "fe m", + "wh itec", + "j n", + "victor ies", + "ðŁ§ ¡", + "gol fer", + "congratul ates", + "resul ting", + "mechan ic", + "ur ve", + "cen tered", + "kie v", + "an s", + "in cub", + "< <", + "c mo", + "bestfan army", + "dap h", + "en ham", + "on cology", + "ku sh", + "t xt", + "ori ented", + "fashion able", + "c sr", + "sa hara", + "r ack", + "pd p", + "han son", + "ภĩ", + "ti ers", + "ra r", + "pan am", + "in sky", + "sa hi", + "testam ent", + "asth ma", + "in her", + "fisher ies", + "or der", + "ho we", + "gall on", + "ep is", + "suz anne", + "drow ning", + "paneli sts", + "ðŁĺ ²", + "ë ¦", + "al ach", + "commemor ative", + "at tribu", + "ðŁij »", + "mo o", + "visi onal", + "week sary", + "gu st", + "ak in", + "poin te", + "ee e", + "di spar", + "ni pp", + "dent al", + "st all", + "pi an", + "bor e", + "ul ster", + "tic k", + "ir r", + "tae hyung", + "micro phone", + "bermu da", + "ga ard", + "el er", + "plumb ing", + "hu gely", + "âļ« ï¸ı", + "race way", + "cam bridge", + "mar cel", + "burn ley", + "to ast", + "holly wood", + "fa sting", + "me red", + "hib ition", + "ca pped", + "benef icial", + "ow ning", + "cont amin", + "arab ian", + "to on", + "cap ac", + "hul u", + "sm ir", + "nutri ents", + "se in", + "graph s", + "con ditional", + "ðŁij ħ", + "or ac", + "play in", + "nor the", + "tor nad", + "mar ian", + "ju mbo", + "lex i", + "incredible india", + "road to", + "uk one", + "confu sing", + "sp h", + "shan k", + "pi ed", + "mq m", + "positi vely", + "sher ry", + "path ways", + "consi ders", + "tof u", + "argu ments", + "resil ient", + "che tt", + "with dra", + "ter o", + "ated ly", + "sw ana", + "he b", + "fli ght", + "har ley", + "decre ase", + "kind le", + "book shop", + "³ ï¸ı", + "marty rs", + "sm ur", + "mc cl", + "concer to", + "sti me", + "rejo ice", + "app lau", + "cle ment", + "mer kel", + "jai me", + "im mortal", + "isle of", + "mar co", + "youtu ber", + "stal king", + "me too", + "st ack", + "sp ouse", + "u st", + "lu v", + "âļ¾ ï¸ı", + "eque strian", + "ev ing", + "fl in", + "nick name", + "the big", + "as ar", + "st acks", + "wal ker", + "bor a", + "kidnapp ed", + "hur ling", + "humb old", + "rec alls", + "co pper", + "ann is", + "se o", + "mer ger", + "mu ir", + "ad dy", + "ðŁĴª ðŁĴª", + "be x", + "cr acy", + "con an", + "congratul ation", + "mid st", + "âĻ ¬", + "for bi", + "op tic", + "cr ate", + "crocod ile", + "mad agas", + "secur ing", + "ast on", + "o gue", + "savi or", + "salis bury", + "love it", + "fuji film", + "cast les", + "as st", + "ar rows", + "sp acious", + "tr s", + "poly vore", + "progre ssion", + "m ri", + "nel son", + "bi m", + "indic ator", + "o da", + "pe pe", + "re signation", + "gu t", + "sne aker", + "log ically", + "az y", + "are lla", + "te aring", + "jo shi", + "ssion ism", + "q pr", + "mari ah", + "p x", + "ble ed", + "mi an", + "med ley", + "we iss", + "ker ry", + "gat ory", + "at al", + "madi son", + "av enger", + "nab y", + "pl and", + "gi les", + "fresh water", + "d ington", + "ta j", + "demonstr ates", + "n tv", + "bul bs", + "sunday morning", + "pe ake", + "souven ir", + "wa h", + "ton nes", + "m kt", + "complex ity", + "con den", + "ross i", + "b ing", + "y ds", + "su k", + "n go", + "mid land", + "ol y", + "life is", + "ri pple", + "mo reno", + "dd ers", + "tu s", + "á ĥ", + "bou l", + "x a", + "hol dings", + "wn y", + "shadowhun ters", + "ke i", + "asp ire", + "m ous", + "ow en", + "so ak", + "skir ts", + "moun taine", + "stor ming", + "ch rome", + "ri ots", + "sar ato", + "amaz e", + "less ness", + "nav ar", + "crit eria", + "ra fa", + "indul ge", + "ay er", + "por to", + "nam o", + "........ ........", + "yi elds", + "val le", + "j h", + "mac ron", + "sa ins", + "dur ant", + "tra ilers", + "wo t", + "confeder ate", + "sh rin", + "id ol", + "form ally", + "ten e", + "motor cycles", + "than g", + "no de", + "bang er", + "dal y", + "p ats", + "enroll ment", + "au ctions", + "at al", + "ar bor", + "lo gos", + "de arest", + "trans action", + "dom ingo", + "fle a", + "ser mon", + "de ck", + "sin cere", + "questi oning", + "juli o", + "was p", + "pre tz", + "armen ian", + "k ham", + "inflam mation", + "picture sque", + "acci dental", + "film makers", + "ðŁĺ ļ", + "ðŁĴ į", + "ca sey", + "so b", + "yee zy", + "good will", + "parag ra", + "ss ly", + "fe ather", + "dy ed", + "assassin ation", + "na de", + "b cs", + "app lies", + "femin ine", + "fe u", + "ext ent", + "depu ties", + "l ack", + "psy chic", + "go i", + "kill ings", + "pse u", + "ðŁ¤ ª", + "un c", + "mar l", + "tan e", + "mck enna", + "sur fer", + "influ ences", + "free way", + "hack ney", + "mal aria", + "el and", + "te au", + "rema stered", + "Ø ±", + "raz or", + "gg y", + "cor ro", + "lak sh", + "fla ir", + "honest y", + "hoor ay", + "de pp", + "am c", + "wedne sdays", + "q a", + "ed its", + "- $", + "se villa", + "dou bled", + "human ities", + "c cot", + "som os", + "r ine", + "af a", + "si oux", + "re construction", + "wel ding", + "th reads", + "am ish", + "encoura gement", + "po der", + "bo ck", + "bal m", + "p tions", + "stand up", + "accompli shments", + "guar ding", + "convic tion", + "ac ion", + "napo leon", + "depic ting", + "att ack", + "su i", + "wear able", + "âĸª ï¸ı", + "pot ter", + "esc ort", + "vis e", + "to ts", + "bo on", + "event profs", + "angu lar", + "womenshi storymonth", + "bar row", + "sch i", + "ac comp", + "ti k", + "l end", + "kensing ton", + "wol fe", + "st acked", + "cra shing", + "exhi bit", + "wing ed", + "sab rina", + "ma sa", + "k ms", + "alway s", + "et t", + "pla sma", + "counsel ing", + "pick les", + "nfl draft", + "mr s", + "inev itable", + "coura geous", + "staf ford", + "writers life", + "ho s", + "e j", + "gh yun", + "trade mark", + "adri an", + "influen cer", + "coron ation", + "ra ging", + "explo red", + "usa f", + "excep tion", + "eu x", + "tan ker", + "sw ami", + "pac ket", + "ðŁij¨ âĢį", + "f en", + "she en", + "a ero", + "j l", + "re gal", + "nw t", + "au ster", + "meh ta", + "char ge", + "a ste", + "b ate", + "inf eld", + "racec ourse", + "collap sed", + "fle ece", + "z il", + "al lie", + "alternati ves", + "geor ges", + "ðŁĵ į", + "quir ky", + "fc b", + "nat geo", + "philanthro py", + "bra i", + "every day", + "ðŁIJ °", + "ach ers", + "ja an", + "fin es", + "q i", + "fisher man", + "distin ct", + "gri mes", + "nation alist", + "comm ence", + "ro wn", + "âĢ ³", + "z ing", + "f ter", + "hr w", + "baro que", + "bl ender", + "kitt y", + "hoo ks", + "c ited", + "w anda", + "consen sus", + "reinde er", + "an and", + "supp ly", + "me ds", + "v n", + "ol ph", + "rat chet", + "shel don", + "secur ities", + "ë°© íĥ", + "cro m", + "mosqu ito", + "j eric", + "im mac", + "dimen sions", + "â ¤", + "di ssi", + "sponge bob", + "dami en", + "steven son", + "jo anne", + "del ish", + "yi kes", + "than x", + "surve ys", + "postpon ed", + "alco holic", + "al ised", + "ðŁĻı ðŁı»", + "do ch", + "sen tim", + "mered ith", + "com pares", + "b ago", + "happy days", + "mo ss", + "ãħ ĭ", + "ne c", + "gn ment", + "frustr ated", + "comb in", + "ri v", + "ec lec", + "col lo", + "compli ment", + "actor slife", + "ct to", + "nic ar", + "op hon", + "apar the", + "man t", + "ja de", + "trol ley", + "optimi zation", + "eye on", + "eco logical", + "qui st", + "ep he", + "ॠĩ", + "cin co", + "appo ints", + "old school", + "c pr", + "behavi oral", + "min aj", + ":- (", + "tag ging", + "ev al", + "jo aqu", + "ðŁĺ «", + "ha k", + "de me", + "jama ican", + "so s", + "hy att", + "hand book", + "libr arian", + "hanni bal", + "pump ing", + "ch om", + "f man", + "ga i", + "hu ll", + "respon ders", + "green ville", + "n us", + "vau gh", + "ðŁİī ðŁİī", + "ta xi", + "gold berg", + "man tra", + "te ase", + "forbi dden", + "metho dist", + "ati vity", + "* ***", + "ec t", + "mc gr", + "Ħ ëĭ", + "se b", + "amid st", + "disapp ear", + "thy ro", + "phili ps", + "er ina", + "v icious", + "stream er", + "million aire", + "ma p", + "str ick", + "hack athon", + "gh a", + "ed ic", + "mi ka", + "pe ck", + "ill i", + "anto ine", + "ar ca", + "op tic", + "ma ure", + "ðŁĩ¦ ðŁĩº", + "cla shes", + "man ly", + "âĺ ģ", + "al var", + "and res", + "me i", + "el m", + "ww ww", + "al tered", + "l te", + "ê¹ Ģ", + "mo jo", + "for rest", + "thal ai", + "non t", + "spee ches", + "acknow ledge", + "ign ite", + "x factor", + "ðŁ¥ Ĥ", + "mead ow", + "disru pt", + "debu ted", + "scrim mage", + "pharmaceu tical", + "fi dd", + "found ations", + "philosop her", + "et al", + "publi shers", + "bo ys", + "c ke", + "ru gged", + "opti mism", + "re be", + "phil harmon", + "nar cis", + "ral lies", + "lu is", + "go blue", + "fol ded", + "un acceptable", + "optim al", + "li sa", + "pol aro", + "+ .", + "en za", + "âĿ £ï¸ı", + "mon opoly", + "grace ful", + "dair y", + "du a", + "diffic ulty", + "judge ment", + "o si", + "mer sey", + "flu x", + "new found", + "ter ns", + "dimen sional", + "in vic", + "al ba", + "am it", + "abudha bi", + "alger ia", + "autom obile", + "the ad", + "lo tion", + "acceler ator", + "vac ant", + "iti on", + "lu f", + "al ic", + "pl l", + "bla zing", + "ba z", + "sen e", + "ðŁij ¼", + "villa ins", + "direc tory", + "eis en", + "to ck", + "broch ure", + "ri pp", + "hb d", + "zayn malik", + "nic he", + "lo lol", + "certific ates", + "mor se", + "fac up", + "x ham", + "un wanted", + "im ports", + "carne gie", + "fan sign", + "mo u", + "r alph", + "destroy er", + "sw ing", + "trek king", + "cili ation", + "pit bull", + "g aps", + "ho well", + "defin itive", + "mc le", + "f ps", + "et z", + "bol ly", + "lyn n", + "gan o", + "at ure", + "fur suit", + "co il", + "na v", + "but ts", + "tro jans", + "eu re", + "en ko", + "sch umer", + "horri fic", + "install ment", + "br b", + "subur bs", + "a bel", + "vi r", + "de sh", + "cun ningham", + "ðŁIJ »", + "span n", + "sch we", + "ke mp", + "tr u", + "ste alth", + "qu es", + "le w", + "deli ghts", + "ko ch", + "hu mili", + "cr iti", + "il t", + "sp ells", + "mi ley", + "car ic", + "ðŁį ´", + "lc fc", + "substitu te", + "oun g", + "? !!", + "af fir", + "predic table", + "class of", + "er r", + "cy press", + "chand ra", + "age ing", + "__ __", + "ther land", + "don caster", + "el in", + "yo shi", + "sail ors", + "har ris", + "jo anna", + "niger ians", + "h ers", + "pla gue", + "pro cra", + "k no", + "can ton", + "busine s", + "un h", + "pra kash", + "c in", + "bow en", + "co ating", + "m als", + "be gging", + "smith son", + "ponti ac", + "sp ies", + "dam ian", + "pl ine", + "und ant", + "al ta", + "one ss", + "shame less", + "da q", + "bb m", + "wal es", + "stam pede", + "ser um", + "Ù Ĩ", + "cataly st", + "x n", + "ab sc", + "free zer", + "ch un", + "ari os", + "mc cre", + "fore head", + "he ars", + "damas cus", + "tac oma", + "ardu ino", + "encoun ters", + "stan ton", + "lg b", + "ab as", + "\" ..", + "ke te", + "drac ula", + "ele m", + "g ne", + "zepp elin", + "la brador", + "pul p", + "op tional", + "or n", + "russi ans", + "san itation", + "hil ary", + "etsym ntt", + "pen alties", + "au st", + "ig ans", + "olympi an", + "medic aid", + "vers ace", + "va pe", + "re stra", + "pe ep", + "sexi est", + "st alls", + "di le", + "the a", + "punjab i", + "pupp y", + "tuesday motivation", + "ðŁĵ ļ", + "the flash", + "roc ket", + "mo dest", + "chihu ahu", + "on na", + "k sa", + "hur dles", + "ca ve", + "fail ures", + "sp lit", + "bo ho", + "gur l", + "disappo int", + "ho ward", + "nug get", + "fran z", + "stal ert", + "kaz akh", + "for getting", + "sch ri", + "ag ate", + "am at", + "eve rett", + "du et", + "veter inary", + "juli an", + "ch ills", + "bra ve", + "ghost busters", + "lan do", + "gre ets", + "profit able", + "d é", + "ti r", + "ze e", + "om en", + "pd x", + "gray son", + "har i", + "fix es", + "stab bing", + "swim mer", + "symb ols", + "compli ments", + "po se", + "func tioning", + "th nx", + "gi r", + "corpor ations", + "bar low", + "lo e", + "off season", + "distin ctive", + "marvel ous", + "nik on", + "enri que", + "ky u", + "ja ws", + "amo to", + "lom bar", + "travel blogger", + "fa h", + "ouri sm", + "tri stan", + "so e", + "ce ase", + "ðŁı ħ", + "z ac", + "mck enzie", + "taxpay ers", + "swim suit", + "bl o", + "les ley", + "kan sas", + "w ks", + "ki el", + "provo king", + "my les", + "str ing", + "kangar oo", + "galac tic", + "fif th", + "s ke", + "we ir", + "ll is", + "mat ory", + "ðŁĩ ¿", + "un ci", + "re productive", + "roo ting", + "ti des", + "gad get", + ".... ......", + "alex ander", + "bow ler", + "scre w", + "apo log", + "eri ka", + "wal ters", + "shet ty", + "lan e", + "ban ter", + "as ant", + "me so", + "v ain", + "\" \"\"", + "us i", + "fer din", + "accomp lish", + "man sfield", + "bom bar", + "collabor ating", + "cla p", + "it ure", + "s da", + "smo ky", + "na k", + "im person", + "car la", + "com ra", + "bur gl", + "lo co", + "ti es", + "in hi", + "trac ey", + "se is", + "diss er", + "rr rr", + "dra y", + "prote ct", + "cor ona", + "hun ger", + "ck en", + "c eli", + "trou bled", + "predat ors", + "fic tional", + "shav ed", + "riche st", + "metab oli", + "ful ham", + "gro oming", + "mono chrome", + "wa sting", + "as co", + "ast e", + "ti sta", + "remedi es", + "ung soo", + "south end", + "perman ently", + "bu mble", + "procra stin", + "ident ical", + "practic ally", + "ma scul", + "su ke", + "assu red", + "val erie", + "devi ant", + "grizz lies", + "thi er", + "pur a", + "ne pal", + "not ts", + "bil ateral", + "spo il", + "car mel", + "cine matic", + "ph l", + "ni fty", + "ma o", + "hypo cri", + "la ser", + "pan try", + "mathemat ical", + "el isa", + "coordin ation", + "bel mont", + "a it", + "radi ant", + "bo iler", + "man g", + "f ag", + "cr c", + "h ams", + "br in", + "â¬ĩ ï¸ı", + "famil ia", + "âĿ £", + "sab er", + "ru pert", + "gg an", + "rit z", + "mic h", + "sal ford", + "le vi", + "gra l", + "ðŁĴ ¤", + "n ino", + "ce d", + "business man", + "ul tr", + "sim ply", + "compre ssion", + "pa ins", + "hal t", + "ë°©íĥ Ħ", + "landsc aping", + "n f", + "croo ked", + "er d", + "itt in", + "ddle ston", + "sur passed", + "ino a", + "da g", + "bl en", + "exten ding", + "at ing", + "al gae", + "ball er", + "u mar", + "snoo ker", + "col lu", + "flo wn", + "thu b", + "ridic ulously", + "ki sh", + "op le", + "di re", + "as ser", + "ari sto", + "sc iss", + "h ating", + "trou ble", + "syl via", + "suc cul", + "plo ts", + "sincere ly", + "al er", + "laure ate", + "br ack", + "att n", + "rif les", + "me to", + "collec tible", + "cu omo", + "conte stant", + "consist ency", + "ant z", + "rang es", + "abig ail", + "de b", + "mini ster", + "grow ers", + "an oo", + "hoo ver", + "dream er", + "nu cle", + "resear ch", + "mi y", + "sha hid", + "ma v", + "d honi", + "cin i", + "do j", + "hin dus", + "part ying", + "dal i", + "alon so", + "inform al", + "clark son", + "it ton", + "ki an", + "cit yo", + "mor i", + "la sted", + "as pen", + "libr ary", + "susp ici", + "qu at", + "den ial", + "fol der", + "ch ori", + "swee ping", + "eni x", + "ðŁį Ĥ", + "Ø Ń", + "nas car", + "handmade hour", + "mou l", + "heat wave", + "em er", + "exam ine", + "ib n", + "gr ind", + "po v", + "tion ist", + "m bo", + "she ila", + "integr ate", + "om es", + "take away", + "cer v", + "con nie", + "tic ket", + "ce led", + "bi en", + "visu ally", + "madagas car", + "sor ry", + "gu i", + "park run", + "tra its", + "la be", + "pois oning", + "ॠĢ", + "vi able", + "bohemi an", + "denti stry", + "bad os", + "spr outs", + "mask ed", + "te ddy", + "ðŁĺ ·", + "sa f", + "sa as", + "ji ang", + "ti ght", + "spe aker", + "withdra wal", + "bc n", + "as signed", + "class rooms", + "fle ming", + "ðŁĴ «", + "super girl", + "tot als", + "table top", + "e books", + "horizon tal", + "cra z", + "flu sh", + "j ard", + "c dc", + "er son", + "ãħ ł", + "green wood", + "ni h", + "co x", + "ad a", + "lit re", + "go ing", + "v icky", + "cur ved", + "lou ie", + "gra ins", + "hy e", + "lon ge", + "reme dy", + "tra inee", + "san jay", + "super stars", + "ma ser", + "man u", + "s age", + "wh l", + "ðŁĺĤ ðŁĺŃ", + "ðŁijį ðŁı»", + "m sd", + "en z", + "rab hu", + "j oo", + "gh u", + "ac er", + "e po", + "resurrec tion", + "justice for", + "bl ended", + "mo da", + "avalan che", + "france sco", + "re spective", + "g s", + "ye ast", + "wel ch", + "devo tion", + "ge tin", + "athe ism", + "am ic", + "carol yn", + "lo c", + "ld nont", + "ave c", + "us da", + "le gged", + "bra very", + "b lower", + "cow boy", + "he h", + "sti ble", + "buff al", + "chann el", + "run chat", + "âĺķ ï¸ı", + "ide ology", + "best seller", + "y oo", + "pe anu", + "bon ne", + "fel ic", + "edi son", + "fr actu", + "naren dra", + "pp ets", + "seym our", + "ri viera", + "he ctor", + "necess arily", + "bi anca", + "soci eties", + "the best", + "w g", + "sent ences", + "win k", + "vacc ines", + "pal ooza", + "jam ming", + "as f", + "mp us", + "agre ements", + "ec k", + "ba c", + "hon ore", + "com pul", + "wild cat", + "im posed", + "yo ga", + "hud son", + "can celed", + "l ich", + "fu zzy", + "es que", + "ch uk", + "w vu", + "se k", + "fli pping", + "r hon", + "wi shed", + "wh a", + "cap ability", + "len ovo", + "ìĨĮëħ Ħëĭ", + "vi vo", + "tv d", + "nor a", + "sil k", + "pas adena", + "yo semite", + "valu ation", + "clo cks", + "u ber", + "mr c", + "dar kest", + "au bre", + "ss o", + "bell y", + "wrest lers", + "kill in", + "lou der", + "buck ley", + "ge el", + "ad on", + "un s", + "appe aling", + "ðŁij ¯", + "semit ism", + "list ens", + "fit z", + "ãĥ³ ãĥ", + "ny lon", + "ar ty", + "seem ingly", + "hal a", + "su ited", + "et y", + "she ds", + "mu ffins", + "ap ric", + "um ents", + "u ta", + "jam mu", + "chelse afc", + "star z", + "yo ko", + "roo t", + "clean sing", + "di ar", + "pione ering", + "ihear tradio", + "dig iti", + "fin dyour", + "can o", + "ðŁĴ İ", + "z ol", + "spac ecraft", + "six ers", + "moi sturi", + "b ile", + "ti sts", + "hor ton", + "rang ing", + "colum bi", + "mete oro", + "senti ment", + "ep l", + "foo th", + "text book", + "drain age", + "r ly", + "sc ue", + "imran khan", + "ðŁĴ ¸", + "margar ita", + "ed dy", + "predic ts", + "gamer gate", + "advis e", + "growth hacking", + "love you", + "ug and", + "v f", + "beng hazi", + "s later", + "ne wor", + "ch el", + "independence day", + "p np", + "cul len", + "hoo dies", + "num bered", + "brit t", + "t sa", + "kl tu", + "s ages", + "mom o", + "onep lus", + "col l", + "gu ts", + "w ta", + "mesm eri", + "enh ancing", + "chiro prac", + "j is", + "teen agers", + "m one", + "constell ation", + "sweep stakes", + "e ze", + "slovak ia", + "la ye", + "pear ce", + "wa ver", + "po gba", + "k ron", + "sur geons", + "mar x", + "ti d", + "gg a", + "desc end", + "p ours", + "upri sing", + "wal la", + "sab bath", + "bachel ore", + "mack in", + "k am", + "peter borough", + "hor a", + "ðŁĮŁ ðŁĮŁ", + "think big", + "r j", + "hy drau", + "sp al", + "univers it", + "ðŁı ī", + "mail online", + "league of", + "ten ants", + "w ally", + "lan ce", + "heav ens", + "dd r", + "bol ts", + "am ir", + "i phone", + "ci gar", + "en du", + "re i", + "el abor", + "r inging", + "john son", + "characteri stics", + "sal oon", + "algori thms", + "tal kin", + "m tn", + "di ve", + "region als", + "ff ice", + "hat i", + "deviant art", + "so tto", + "shir o", + "l ama", + "k we", + "f aded", + "por ting", + "tu mmy", + "est ates", + "buen os", + "ðŁ¦ ģ", + "beli ever", + "pen etr", + "dar n", + "sp ite", + "can opy", + "fashi oni", + "t illa", + "pet als", + "eli jah", + "bra wl", + "marty r", + "ë°©íĥĦ ìĨĮëħĦëĭ", + "mid town", + "eric h", + "d apper", + "sm town", + "me gam", + "ww w", + "le le", + "on s", + "cat fish", + "fir th", + "fossil friday", + "ball park", + "th aw", + "pot ent", + "illi e", + "cre ep", + "car p", + "so ap", + "gun dam", + "infe c", + "yy yyy", + "ठ¨", + "z ag", + "rit t", + "calcu lator", + "bo ca", + "ok o", + "to ad", + "threat en", + "refin ed", + "olym pic", + "accompli shment", + "bacter ial", + "a ji", + "tat um", + "feli z", + "she ed", + "j at", + "th ic", + "jam al", + "ðĿ ĺ", + "lin a", + "ðŁIJ ¯", + "jo king", + "yot po", + "pin ch", + "ak ron", + "her b", + "motiv ation", + "li a", + "ho stage", + "cre ek", + "gam ble", + "russ ell", + "patt i", + "fo tos", + "c pc", + "bro ken", + "back the", + "cla ys", + "u mm", + "stock ton", + "mat ernal", + "ü r", + "la kel", + "cent ury", + "be k", + "infe cted", + "ภ¡", + "smack down", + "man ned", + "ta hoe", + "sm es", + "bas a", + "su la", + "augu sta", + ". *", + "rohing ya", + "gre ed", + "counsel or", + "silhou ette", + "gra vit", + "cla use", + "' -", + "bo bc", + "occa sions", + "now adays", + "dic tat", + "be ard", + "n ally", + "brigh test", + "kab ul", + "inc india", + "dhan ush", + "archae ological", + "che ape", + "mizz ou", + "d hi", + "ov ski", + "bax ter", + "asse mble", + "à ¢", + "gi gi", + "ac am", + "wis ely", + "haz ard", + "north ampton", + "âľĪ ï¸ı", + "me th", + "bla sting", + "re unite", + "mu lus", + "ali zes", + "t read", + "mil a", + "ed ward", + "ko va", + "pe sto", + "ðŁij ¶", + "vit z", + "hydrau lic", + "refurbi shed", + "mo tel", + "isab ella", + "hom me", + "sever ance", + "uph ol", + "mis erable", + "f ari", + "lat ter", + "ef er", + "crack ers", + "es l", + "ac io", + "yy j", + "in an", + "ec b", + "z ind", + "pan as", + "tru cking", + "re ed", + "sh aker", + "burge ss", + "em pire", + "ag nes", + "n ington", + "art works", + "fr s", + "ti le", + "bi ome", + "eu n", + "ch ong", + "americ ana", + "god father", + "go blin", + "i shi", + "! ).", + "temp ted", + "gen omics", + "mand ate", + "ck y", + "ðŁĴĻ ðŁĴĽ", + "som ali", + "br andy", + "in ven", + "spoke sperson", + "pc b", + "yu an", + "h g", + "fa z", + "starwar s", + "ro wan", + "blue grass", + "don g", + "d day", + "trin idad", + "er ton", + "ban ning", + "re tention", + "cu red", + "tober fest", + "re set", + "we is", + "deta ched", + "behindthe scenes", + "immun ity", + "ph a", + "bra y", + "ðŁij ½", + "ran cho", + "ram say", + "est onia", + "nd tv", + "] .", + "cab aret", + "tar o", + "d v", + "show cases", + "plu m", + "ðŁij ¸", + "son oma", + "pre pa", + "memor ab", + "e stu", + "drive way", + "u les", + "magn us", + "x r", + "nn n", + "much as", + "en ge", + "stre amed", + "fore stry", + "audio book", + "tro y", + "reck less", + "kil om", + "ru ler", + "ra k", + "proce ssion", + "i ons", + "po ole", + "noc tur", + "wh s", + "farm house", + "per a", + "par me", + "hypocri sy", + "s ics", + "v ant", + "cas k", + "holi stic", + "au st", + "Ð ¿", + "in do", + "ðŁij© âĢį", + "di so", + "disp atch", + "ol sen", + "make it", + "en nis", + "cent re", + "ar range", + "ðŁĮ ¼", + "sal ted", + "ea siest", + "f ate", + "reg atta", + "mo zz", + "ac an", + "sin i", + "g ically", + "ch ops", + "chick en", + "work in", + "ha gg", + "invol ve", + "wee ds", + "book day", + "wake up", + "ky r", + "michel in", + "fu ss", + "re juven", + "vac ancies", + "incar cer", + "m st", + "sc ents", + "sovere ign", + "kick er", + "à §", + "bo d", + "âĢĶ >", + "sa h", + "mob il", + "shrop shire", + "oph one", + "dress er", + "mis suni", + "hep burn", + "i mo", + "foli age", + "diagno stic", + "as san", + "cycl ing", + "guil t", + "c sa", + "puertor ico", + "win elover", + "wake field", + "do ggy", + "k he", + "pa pp", + "co g", + "al lot", + "cu ck", + "poe tic", + "mi o", + "re vit", + "mag ician", + "ç ¥", + "ant enna", + "west wood", + "mber g", + "lux e", + "oat meal", + "Ø ¬", + "te at", + "ffe e", + "sear ches", + "l ly", + "plu to", + "el on", + "let tering", + "inno cence", + "fa i", + "ann on", + "telang ana", + "ma it", + "neu ral", + "can ni", + "ar oma", + "a stor", + "fe x", + "co cac", + "mon etary", + "f ent", + "un sure", + "' @", + "indi rec", + "teh ran", + "isol ation", + "li bs", + "make up", + "merce des", + "ff y", + "he tero", + "de o", + "sco m", + "cur sed", + "veteran sday", + "franken stein", + "shre ws", + "de co", + "ge ese", + "lefto ver", + "ha did", + "vari able", + "acade mics", + "carol in", + "under going", + "vari ation", + "na h", + "ssi er", + "gamer sunite", + "pur suing", + "emer ged", + "ll ers", + "control ling", + "ro aring", + "mete or", + "vol t", + "daw gs", + "be aver", + "is life", + "bathro oms", + "aci onal", + "pre vent", + "lake district", + "in als", + "y ani", + "gra bbing", + "sac ks", + "le z", + "sw ay", + "k ool", + "time s", + "klo pp", + "la de", + "con cord", + "resul ted", + "revi ve", + "recon ciliation", + "ol and", + "az z", + "gir o", + "mand arin", + "de en", + "nutriti onal", + "is coming", + "van i", + "aw www", + "der ived", + "love your", + "stop the", + "shou ting", + "nov ak", + "ðŁĻĮ ðŁı¾", + "lo af", + "displa ying", + "sunday with", + "ma guire", + "ch eri", + "ðŁı Ł", + "re match", + "qu ic", + "Ú ©", + "y in", + "ðŁĺ ¹", + "ili ve", + "z ip", + "our ke", + "down loads", + "sw at", + "missi ss", + "care rs", + "t ment", + "proper ty", + "hahahaha haha", + "gi bbs", + "sur rey", + "ar ise", + "tic ism", + "sti a", + "ir ling", + "fro g", + "co se", + "bas sist", + "fore ig", + "lea u", + "pil lows", + "hol la", + "eli e", + "disclo sure", + "peanu ts", + "inte ch", + "ww c", + "plun ge", + "trium ph", + "cor i", + "sli ppers", + "ðŁĻı ðŁĻı", + "neutr ality", + "ma re", + "hair y", + "gang ster", + "hu mming", + "cust ard", + "mer lin", + "ale a", + "s by", + "dam p", + "mo han", + "ver bal", + "j st", + "gu tted", + "b jor", + "un finished", + "ðŁĩ¯ðŁĩ µ", + "un happy", + "âļ« ï¸ı", + "by pass", + "at su", + "fis cher", + "sa v", + "afric ans", + "re use", + "mid way", + "demo lished", + "ger rard", + "her cules", + "Ä Ł", + "medic ines", + "cl icking", + "sur round", + "jo ong", + "wav ing", + "tri bes", + "wet lands", + "offici el", + "argu ing", + "l le", + "do va", + "su zy", + "club house", + "ne gro", + "ob tain", + "ga o", + "gl ance", + "assi st", + "ch os", + "ãĤ ¢", + "âĺ ķ", + "adri d", + "occur s", + "st ans", + "par don", + "livel i", + "emplo yed", + "re visit", + "ff xiv", + "bb le", + "ne aring", + "min er", + "ðŁĺ ¹", + "giov anni", + "up to", + "mar vell", + "mar se", + "to wels", + "cb n", + "engine ered", + "y elling", + "spart an", + "si ans", + "ðŁĻĮ ðŁı¼", + "se v", + "coyo te", + "sta di", + "t cm", + "app en", + "shenan igans", + "open access", + "so aked", + "ma squ", + "le vine", + "stro kes", + "l k", + "aparthe id", + "hipho p", + "char don", + "may may", + "ha asan", + "stri pped", + "fr o", + "scri ption", + "f ton", + "h f", + "pri sons", + "marsh al", + "ķ ãĤ", + "an cho", + "com promise", + "classi fication", + "buzz feed", + "bblo ggers", + "deser ving", + ") /", + "s way", + "ob o", + "camp ers", + "poder nfamily", + "p oured", + "bri e", + "squir rels", + "se ize", + ": #", + "le k", + "ti mb", + "st acy", + "nas daq", + "repe atedly", + "br at", + "mi ghty", + "competit or", + "mah one", + "de si", + "o ke", + "bm w", + "shi e", + "f cb", + "cheape st", + "minim alist", + "par amount", + "n ate", + "har as", + "insan ity", + "lat eral", + "ment ality", + "mo zam", + "ta pped", + "yad av", + "u sp", + "b way", + "the od", + "bil t", + "ra ids", + "em press", + "adap ted", + "pat ron", + "nut shell", + "ag ra", + "be aded", + "sundaywith marsha", + "vi king", + "proce ed", + "main tained", + "thinkbig sundaywithmarsha", + "sn es", + "mus ica", + "to wer", + "ch ab", + "bo k", + "sm t", + "insul t", + "harve sting", + "windo w", + "ru ther", + "be ige", + "dec al", + "indic ate", + "ma iling", + "ri ft", + "po le", + "ander son", + "ch oral", + "sp ride", + "l ili", + "ev elyn", + "imrankhan pti", + ".... \"", + "ke red", + "un dp", + "water falls", + "se ars", + "le mans", + "world series", + "ri el", + "ani e", + "app ar", + "score rs", + "lam p", + "a than", + "phys icians", + "qu inoa", + "refu sing", + "vu itton", + "unle ash", + "s la", + "pat i", + "shou ts", + "inten tions", + "fo amed", + "europe an", + "neighbor hoods", + "me er", + "man son", + "du h", + "br at", + "con es", + "bow l", + "kazakh stan", + "ठ¿", + "in appropriate", + "del hi", + "ketch up", + "ful ton", + "s ys", + "consul t", + "gar field", + "to go", + "f ml", + "f led", + "b ds", + "facilit ate", + "ree bok", + "selfi e", + "elev ate", + "activ ate", + "bi ble", + "ca wx", + "b ys", + "cam ille", + "sy ou", + "sk ool", + "her t", + "w bc", + "ple dges", + "recor der", + "po sh", + "ac re", + "so aking", + "mat il", + "v sco", + "shoot ings", + "pla r", + "e con", + "ðŁĻĮ ðŁı»", + "rashi d", + "u bi", + "ðŁ¤ ¤", + "sw inging", + "wi pe", + "rap tor", + "m su", + "music video", + "dur ham", + "at tic", + "apar ty", + "fe tus", + "activ ation", + "aa z", + "motiv ate", + "ðŁĴķ ðŁĴķðŁĴķ", + "j al", + "ठ®", + "ag on", + "sche er", + "stal ker", + "fo ster", + "az zo", + "tele gram", + "vi gor", + "s laugh", + "screen shots", + "entrepre neu", + "kri stin", + "inten tion", + "ch illi", + "fr action", + "don a", + "ge a", + "tc u", + "s ite", + "la k", + "em il", + "d nt", + "bor o", + "wil kinson", + "re cu", + "ato day", + "t anya", + "bl anco", + "cd n", + "brilli antly", + "g cc", + "ac c", + "evacu ated", + "ther ine", + "den ny", + "cait lin", + "she pard", + "pou ch", + "hand held", + "sou theastern", + "ha a", + "à ´", + "re solutions", + "led ger", + "sr in", + "r ar", + "shat tered", + "chim ney", + "im with", + "mete or", + "hand led", + "ra ke", + "town send", + "en han", + "shi py", + "duc t", + "tw x", + "inflam matory", + "war hammer", + "theat rical", + "gro s", + "sk ar", + "sco tty", + "ni el", + "tit o", + "tin i", + "conne ction", + "_ .", + "goldeng lobes", + "sha q", + "ðŁı ³ï¸ı", + "hall way", + "fron ts", + "effec tiveness", + "gla ston", + "d hs", + "ex pi", + "to h", + "c pl", + "sc s", + "re o", + "ha g", + "resemb lance", + "hor an", + "abu sive", + "qu er", + "virtu e", + "cho lester", + "a q", + "shan e", + "m ce", + "carri ers", + "di stress", + "re wind", + " ¡", + "voo doo", + "int act", + "ann o", + "ðŁĺ ¤", + "pi led", + "adi a", + "ãĥ ³", + "en ow", + "di gs", + "light ly", + "goo fy", + "turb ine", + "governor s", + "con te", + "re open", + "pa h", + "i ve", + "cra fting", + "swee ps", + "jo di", + "an de", + "zu cker", + "kaw aii", + "o ko", + "v ai", + "out line", + "kri sti", + "ts n", + "insp o", + "qu int", + "fil thy", + "lyn ne", + "listen ers", + "depar ting", + "or d", + "t weed", + ", &", + "ale k", + "sel fish", + "nor ther", + "recogni zes", + "i ps", + "be s", + "a ed", + "w ills", + "pe at", + "surround ings", + "mon uments", + "ais le", + "be cker", + "la v", + "quant ity", + "v ah", + "helicop ters", + "tu cked", + "alv arez", + "sha pe", + "o bey", + "ad diti", + "road side", + "m ite", + "bl ers", + "ep age", + "j au", + "ignor ant", + "b ins", + "lu lu", + "x o", + "c fo", + "ee eee", + "apprentice ship", + "shef fiel", + "to i", + "ho k", + "faken ews", + "deplo y", + "aid an", + "husk ers", + "ãĢ İ", + "west brook", + "mi ster", + "confi gur", + "car r", + "fic a", + "proceed ings", + "ha w", + "ste ak", + "mur derer", + "pay day", + "a jo", + "p vc", + "don ates", + "bi af", + "nom nom", + "be it", + "k ali", + "x rp", + "ahmed abad", + "se mic", + "che y", + "x tra", + "an twer", + "head lining", + "squ ares", + "roun ded", + "flu ore", + "bol d", + "disa sters", + "am oo", + "gener ic", + "cran es", + "brief ly", + "gi g", + "auster ity", + "anticip ation", + "for ti", + "treas urer", + "cann y", + "ce cil", + "dete cted", + "check list", + "ภ§", + "pam ela", + "bar bados", + "an field", + "hear ty", + "tx lege", + "peren ni", + "arro g", + "ing ram", + "âĹ ı", + "ty ne", + "spo on", + "r ation", + "am ba", + "m be", + "cam el", + "h hs", + "york shire", + "reflec tive", + "fre aks", + "to k", + "ju do", + "partic les", + "du bs", + "ban jo", + "accred itation", + "prover bs", + "over dose", + "inte gral", + "gu ang", + "mc s", + "super car", + "af b", + "al vin", + "ail s", + "x tre", + "st aging", + "tw ent", + "rabb its", + "mar o", + "inste m", + "dol l", + "cr ay", + "sant ana", + "ble ach", + "mini ons", + "che ap", + "man t", + "di vers", + "catal onia", + "lo is", + "mat ri", + "cou gar", + "kay ak", + "e gre", + "p so", + "a ia", + "å ®", + "char lton", + "tr acked", + "sc ari", + "pe tt", + "f wd", + "x in", + "gra vel", + "br ic", + "bigg boss", + "ar den", + "hu gging", + "pal ms", + "st v", + "li mb", + "the movie", + "handic ap", + "ri me", + "z ai", + "stu b", + "indi a", + "lithu ania", + "rhy th", + "p ita", + "maced onia", + "high ered", + "brid get", + "schwar z", + "ske let", + "hi kes", + "ant arctic", + "c ps", + "mash up", + "Ð °", + "n ell", + "chand ra", + "he ir", + "an us", + "sher idan", + "mi mi", + "muse u", + "bec ca", + "an ir", + "bar rie", + "dioce se", + "compar able", + "ðŁı³ï¸ı âĢį", + "yuk on", + "me p", + "hor mon", + "mer ic", + "al f", + "con quered", + "christ church", + "ðŁĴĻ ðŁĴĻ", + "hazard ous", + "poo h", + "cont ing", + "retro spective", + "par ame", + "na ir", + "con sor", + "ho tra", + "astoni shing", + "cater pillar", + "u man", + "ti sm", + "t vs", + "serv ic", + "croy don", + "mor ales", + "c g", + "cu m", + "te ur", + "scan ada", + "s all", + "magno lia", + "el ise", + "th our", + "à® ¿", + "ag omez", + "phel ps", + "ë°©íĥĦìĨĮëħĦëĭ ¨", + "wh os", + "weav ing", + "si sd", + "pro poses", + "cro ws", + "pre sale", + "econom ies", + "bernar do", + "sha hid", + "air show", + "mc cann", + "hor ticul", + "nr l", + "du el", + "mongo lia", + "tou lou", + "requi rement", + "struc tured", + "ed i", + "o lives", + "he a", + "cu ter", + "Ð º", + "enthusi ast", + "harri et", + "domin ion", + "sub mer", + "ðŁį ĥ", + "sa ab", + "nes burg", + "mo ff", + "def ended", + "bur t", + "rewar ded", + "gold man", + "op tics", + "khali d", + "house holds", + "buc kets", + "ce cil", + "che ss", + "substan tial", + "ef l", + "oper ation", + "evalu ate", + "st n", + "rece ssion", + "l ll", + "tom as", + "tru ths", + "ak bar", + "s words", + "p act", + "embarra ss", + "ha o", + "ay urve", + "scrip ture", + "ny cc", + "op t", + "di ameter", + "sc ented", + "organi zers", + "re lat", + "ha e", + "dream ers", + "de se", + "ðŁĮ »", + "restric ted", + "n ale", + "r hp", + "dol an", + "mun ster", + "ha ired", + "consult ants", + "jo ints", + "hu mil", + "d ill", + "relent less", + "t é", + "af il", + "ut ilities", + "japan ese", + "condem n", + "pet ite", + "colli de", + "q f", + "peach es", + "cou rier", + "l ore", + "âĺİ ï¸ı", + "reli ability", + "ch uk", + "ðŁĻ ĥ", + "stu res", + "ge ther", + "ho stel", + "bi er", + "- _-", + "â ĩ", + "e ze", + "ta ilo", + "di ent", + "blu ff", + "chu ffed", + "pil ip", + "mon arch", + "e em", + "bu chan", + "b ick", + "op au", + "ku ps", + "ภ¢", + "pist ons", + "sp ins", + "m and", + "ce st", + "bur ne", + "v ile", + "cher ries", + "bec kett", + "need les", + "pan ch", + "ë Ĥ", + "haha h", + "trou bles", + "insi sts", + "do you", + "g mc", + "mor tar", + "deleg ate", + "in n", + "g anda", + "sin atra", + "ठ¤", + "spee ding", + "pu pil", + "pre mises", + "ali gnment", + "pi kach", + "as us", + "j alan", + "Ø µ", + "lime stone", + "fol kl", + "parme san", + "ce il", + "mo y", + "shawn mendes", + "ac up", + "hu st", + "ot es", + "med ina", + "ma di", + "gta v", + "censor ship", + "ar g", + "swe eney", + "sy kes", + "col o", + "foot steps", + "cann ed", + "adv ance", + "gta online", + "healthy living", + "ðŁį ¾", + "a ig", + "p ality", + "oc s", + "he brew", + "im minent", + "berk shire", + "jeremi ah", + "out going", + "bak er", + "entr ata", + "ma ids", + "gro ves", + "bo c", + "a del", + "m fw", + "con science", + "arm ys", + "nut ella", + "conte stalert", + "novel ist", + "la h", + "ban ker", + "marque z", + "ðŁı ¡", + "to ff", + "out age", + "gr p", + "ðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃ", + "musc le", + "du dley", + "nvi dia", + "mi di", + "m uni", + "ess ays", + "dat ac", + "car ter", + "ภ£", + "t ans", + "i ves", + "public ations", + "al er", + "ok wx", + "il u", + "cu tt", + "har p", + "out law", + "luther an", + "br ill", + "bo lic", + "do well", + "green land", + "be sties", + "path i", + "pay ton", + "gue st", + "har den", + "ðŁ¤ ©", + "ann ed", + "evacu ation", + "po ised", + "mc der", + "b han", + "o i", + "envel ope", + "ci d", + "ca vi", + "ta pas", + "book review", + "grey hound", + "âĻ ª", + "fe ud", + "lun gs", + "for te", + "rai der", + "ff er", + "oni x", + "dep end", + "yn wa", + "rel ating", + "de vs", + "ðŁĴ IJ", + "acqui res", + "d ha", + "j yo", + "priv ati", + "can ine", + "k b", + "cra b", + "sar din", + "imag ining", + "k j", + "em por", + "down hill", + "ne z", + "ta eyeon", + "nick imin", + "gb p", + "à µ", + "w ap", + "sec co", + "ma shed", + "ðŁĴ¥ ðŁĴ¥", + "augu stine", + "diss ol", + "dic tator", + "â ĵ", + "vi per", + "ed fringe", + "vau x", + "hard work", + "book let", + "no x", + "chi ff", + "ðŁĴ ¨", + "observ ations", + "xbox one", + "u sher", + "ke er", + "lu p", + "dal las", + "cal gary", + "ma dra", + "di ous", + "k bs", + "wood ward", + "hero ine", + "lu mber", + "sea world", + "o ws", + "mc ke", + "maver ick", + "gu la", + "cross roads", + "fan g", + "s ade", + "nik ol", + "chee tah", + "me c", + "pp g", + "er ick", + "ðŁİ µ", + "tox ic", + "bj j", + "viol a", + "sp ire", + "ch ino", + "tra vis", + "institu tional", + "ha as", + "low ry", + "w ac", + "ea e", + "hu mid", + "mp ton", + "ru ck", + "je w", + "c ine", + "zim mer", + "se f", + "bhar at", + "fre es", + "aam ir", + "ðŁĴ ħ", + "z inc", + "wan e", + "multi player", + "royal wedding", + "e el", + "preci pit", + "qu ery", + "kimber ly", + "isa bel", + "ful fill", + "ig an", + "vau l", + "pan e", + "sc y", + "dig it", + "gun n", + "u tah", + "dog day", + "fi on", + "xia omi", + "da c", + "el ast", + "cha vez", + "ro blo", + "g ine", + "ten th", + "ab h", + "ke to", + "hur dle", + "na dia", + "memorab ilia", + "ha bs", + "qu an", + "h w", + "hv ac", + "pix ar", + "ec cle", + "kram er", + "accu ses", + "ðŁĴļ ðŁĴļ", + "per se", + "mean time", + "wa hl", + "atle tico", + "âĢ¢âĢ¢ âĢ¢âĢ¢", + "ott oman", + "no vo", + "k us", + "conne cted", + "tru sts", + "d mv", + "spen cer", + "rahu lg", + "do ve", + "sto kes", + "bolog na", + "enthusi asts", + "à ª", + "rockstar games", + "ted cruz", + "du ras", + "s acked", + "late x", + "immer sive", + "cer t", + "lu cin", + "princi pals", + "fa res", + "sa ils", + "far n", + "am ent", + "saf fron", + "quent in", + "check point", + "fer ris", + "ex cur", + "ðŁijī ðŁı¼", + "bai ley", + "se h", + "ter re", + "mad am", + "s band", + "wan derers", + "cumber batch", + "yy c", + "digit ally", + "blackandwhite photography", + "roll in", + "moroc can", + "ðŁĮ ħ", + "din ner", + "d well", + "to om", + "m ye", + "ez ra", + "cp fc", + "war hol", + "me er", + "jon ah", + "no aa", + "s gate", + "so on", + "secu lar", + "g ating", + "ti o", + "dri ver", + "si ssy", + "assan ge", + "ta th", + "ed mund", + "bobc ats", + "ra ji", + "po stage", + "stu ds", + "m gm", + "kat o", + "edin burgh", + "meet the", + "shir t", + "fa a", + "mens fashion", + "sp reads", + "wi m", + "car ts", + "phoe be", + "j ars", + "bot swana", + "Ù Ĥ", + "ed war", + "sk ar", + "ri ve", + "gu sty", + "c tv", + "ferdin and", + "su therland", + "nickimin aj", + "k v", + "si us", + "bee ch", + "re z", + "desi res", + "on ial", + "camp o", + "quar ry", + "lor raine", + "gil more", + "ig gy", + "µ ï¸ı", + "ho pping", + "avi z", + "ðŁĮ º", + "uni sex", + "dedic ate", + "att itudes", + "ste er", + "jun kie", + "rail way", + "y b", + "whi sper", + "key an", + "k us", + "ju g", + "di x", + "a ins", + "sum mon", + "ov ich", + "sy ed", + "her ald", + "ma ison", + "me ded", + "wild flower", + "main land", + "ri sky", + "ru kh", + "over looked", + "ki c", + "destro ys", + "nam an", + "ki p", + "z ano", + "champion sleague", + "ban dit", + "quin cy", + "smi le", + "cal vin", + "open ings", + "ta pp", + "ol ulu", + "spec tro", + "accred ited", + "ap k", + "pra ised", + "bar nett", + "pol len", + "premi ered", + "selen agomez", + "tou red", + "screen ings", + "uu u", + "mis o", + "en se", + "adam lambert", + "guel ph", + "har yana", + "hu tto", + "le ar", + "l tc", + "po ached", + "brex it", + "æ Ŀ", + "tt c", + "pa vement", + "mon gers", + "ro e", + "ad ers", + "ling ton", + "particip ant", + "ca red", + "ga il", + "y ates", + "lan tic", + "dash board", + "jo o", + "feli pe", + "ssi onist", + "bu m", + "s end", + "a eri", + "thu gs", + "luci fer", + "a he", + "dete ctor", + "fil ly", + "gas oline", + "ham per", + "hump day", + "the ta", + "the band", + "fore casts", + "o hhh", + "lo bb", + "hol l", + "cp u", + "az u", + "ad ar", + "hai ley", + "bu b", + "car t", + "quo ted", + "an archy", + "pan cre", + "twit art", + "al den", + "st ash", + "the less", + "or ni", + "belie bers", + "mor mon", + "partic le", + "avi ation", + "⬠Ĩ", + "webcam toy", + "sad dened", + "cru is", + "ham let", + "n ct", + "roll ins", + "marque e", + "saw yer", + "reli ance", + "a ura", + "di ec", + "soo thing", + "sig nings", + "ak is", + "à ³", + "at kins", + "aer op", + "ðŁĮ ¿", + "y ab", + "sh ari", + "con nol", + "du bbed", + "manufac ture", + "convin cing", + "feelthe bern", + "ra u", + "pu lit", + "on ec", + "gem stone", + "ur ging", + "bag u", + "ga h", + "aci ds", + "fi anc", + "zodi ac", + "sn oop", + "her rera", + "initi ated", + "ven ge", + "profess ors", + "pro di", + "stron ger", + "e mission", + "bb a", + "hal le", + "ta pp", + "haw an", + "wh im", + "compe ted", + "myr tle", + "ir port", + "cold play", + "ach e", + "ske p", + "m son", + "ss ic", + "calli graphy", + "swim mers", + "me y", + "pp c", + "thri ft", + "po c", + "re places", + "commu ter", + "âģ¦ âģ¦@", + "go ers", + "lo gue", + "para dig", + "bas kets", + "sensiti vity", + "joh an", + "atl antis", + "& &", + "suit case", + "anxi ous", + "l h", + "str i", + "gal loway", + "stre ad", + "war den", + "gr ounded", + "ffici ency", + "li feat", + "reli c", + "disgu ise", + "island ers", + "f cofficial", + "classical music", + "b mc", + "en field", + "bi que", + "oak ley", + "bat man", + "sla ying", + "ner ves", + "mul tit", + "calci um", + "projec tor", + "scott sdale", + "ant ino", + "gri ps", + "kim mel", + "des mond", + "prote stors", + "hi atus", + "metaboli sm", + "conclu ded", + "press er", + "ti pping", + "sli de", + "e to", + "hun ting", + "aus open", + "ri k", + "pp ery", + "innov ators", + "pitch ers", + "ag ger", + "fun gi", + "z ad", + "proli fic", + "rockn roll", + "bl ames", + "ct ar", + "stam ford", + "q ad", + "mozz arella", + "insan ely", + "den ver", + "ph ouse", + "nom ad", + "ï ¿", + "s ris", + "pro du", + "hen ley", + "pag an", + "am trak", + "ru bi", + "in cl", + "tu tor", + "sco tia", + "wo es", + "sing apo", + "fun nel", + "turn bull", + "know ledge", + "gri mm", + "real madrid", + "we are", + "missi les", + "con sol", + "emo jis", + "sne ak", + "smi ths", + "ru iz", + "br ou", + "i el", + "ha ver", + "ðŁĮ ļ", + "kin gof", + "basil ica", + "circul ation", + "prin ters", + "ta pping", + "ri dley", + "dra gged", + "ha j", + "writ er", + "fundament als", + "personal ities", + "me tre", + "stereo types", + "bur le", + "best of", + "n ffc", + "ha th", + "mini stries", + "a ali", + "trac ing", + "pav ed", + "ł ï¸ı", + "g ic", + "insp ire", + "tu g", + "ha re", + "repe ated", + "ex pon", + "lol li", + "rho de", + "pre cin", + "install ations", + "instag ram", + "az ar", + "i es", + "sole ly", + "du kes", + "mission ary", + "van guard", + "fursuit friday", + "on d", + "pol ari", + "ma st", + "har an", + "jos é", + "jack ed", + "ec oun", + "al ities", + "ne ph", + "ra vel", + "moder ated", + "sco w", + "s fb", + "uru guay", + "as o", + "ni g", + "au du", + "p ints", + "lat ina", + "ben z", + "m itting", + "char ted", + "mat ology", + "cit ro", + "biop ic", + "ðŁij Ń", + "djo kovic", + "fox y", + "agu il", + "so to", + "an ada", + "sin king", + "sc rap", + "hair s", + "bethan y", + "fact friday", + "ðŁIJ IJ", + "unlea shed", + ") (", + "contra dic", + "ram on", + "coast line", + "y ong", + "sn sd", + "li gan", + "p ome", + "mit age", + "ge tt", + "wat i", + "ri sk", + "so aring", + "bru sh", + "f pl", + "av an", + "å Ĩ", + "lar son", + "sh ear", + "mul til", + "blu r", + "multi media", + "chun ky", + "par i", + "n ani", + "weir d", + "cholester ol", + "char les", + "dream ed", + "tan ning", + "puzz les", + "fr am", + "hand ball", + "ch ag", + "beli ze", + "al u", + "bang s", + "Ñ Ħ", + "detec tives", + "mc g", + "ish q", + "bo thered", + "saf c", + "mp ing", + "ten eri", + "g ays", + "sail or", + "an gi", + "mul ticul", + "gue ssed", + "ros é", + "high ways", + "bro om", + "chatt anoo", + "- '", + "see ker", + "on ed", + "at f", + "lu c", + "> <", + "bar i", + "per cep", + "jewel ry", + "as ph", + "sor row", + "sl ing", + "mam moth", + "jac kie", + "ë §", + "wilt shire", + "sa o", + "can cell", + "im paired", + "tor ial", + "bre ed", + "guy en", + "jud ice", + "tit le", + "pro spective", + "applic ants", + "ðŁį Ĭ", + "epis cop", + "e id", + "b yo", + "stock ings", + "ðŁĴĥ ðŁĴĥ", + "ll p", + "sna g", + "keep it", + "l ough", + "ol son", + "matur ity", + "!! !\"", + "cop ter", + "i sha", + "bl i", + "wil mington", + "tr youts", + "th ai", + "ðŁ¥ ³", + "pe bble", + "kra ft", + "f p", + " º", + "ssi vely", + "li vin", + "contest ants", + "tex tures", + "jo an", + "h dr", + "film festival", + "prov ence", + "wi do", + "op end", + "c si", + "sto wn", + "cro ati", + "ad just", + "host ile", + "analy sts", + "il an", + "cu ppa", + "bru m", + "newfound land", + "good win", + "me tt", + "mall orca", + "plu gs", + "bu k", + "bb hutto", + "wrest le", + "sa ire", + "sho pped", + "for za", + "le head", + "vi vo", + "ba st", + "ro xy", + "reg is", + "hard working", + "hon olulu", + "desp air", + "young sters", + "ni g", + "impro mp", + "roll tide", + "de emed", + "tre ason", + "ru shed", + "for ged", + "ff f", + "pikach u", + "bri ggs", + "do it", + "ac cent", + "la us", + "gla ze", + "compet ent", + "a ho", + "photo g", + "mid field", + "le go", + "har vard", + "min orities", + "re illy", + "slic ed", + "once upon", + "initi ally", + "financi ally", + "landscape photography", + "har dro", + "qu o", + "mm ers", + "par kinson", + "smu gg", + "read iness", + "bru tally", + "glou cester", + "mp ed", + "bbhutto zardari", + "mur der", + "ye d", + "dat aviz", + "sr t", + "dow ning", + "bi ans", + "m ü", + "fle ck", + "fli pped", + "s ly", + "brilli ance", + "ri m", + "k um", + "bubb a", + "ko i", + "knit ted", + "sor g", + "ma is", + "ðŁĮ ²", + "ti ss", + "su stain", + "sen su", + "ak han", + "zi est", + "exam ines", + "chardon nay", + "user name", + "short list", + "re bs", + "on o", + "dar ing", + "hard wood", + "che que", + "righte ous", + "light ening", + "dir k", + "shra dd", + "du ra", + "down stairs", + "sh al", + "ami gos", + "ru ff", + "s law", + "ri es", + "red nation", + "man us", + "ðŁĩ§ ðŁĩ·", + "distin ction", + "u bun", + "dur an", + "mi gra", + "thi ans", + "la ver", + "domest ic", + "k x", + "jaz zy", + "justi fy", + "belong ing", + "insul ation", + "color stv", + "drun ken", + "chann eling", + "qu and", + "xi ii", + "enligh ten", + "kan o", + "fati ma", + "teen choice", + "terri fied", + "p ba", + "as ley", + "met museum", + "dun e", + "pack er", + "ki o", + "ðŁĴľ ðŁĴľ", + "bo iler", + "fas cism", + "ar mored", + "back grounds", + "in mates", + "embarra ssed", + "defin es", + "th d", + "we go", + "silic one", + "lo on", + "el ding", + "bor rowed", + "he mp", + "ak sh", + "kaw asaki", + "br y", + "de af", + "kill er", + "dispo sal", + "ðŁĩ °", + "glaston bury", + "un covered", + "o xide", + "po ff", + "d ant", + "k j", + "ku ro", + "dri zzle", + "peop les", + "fe e", + "pro pri", + "dd lovato", + "pi ggy", + "ot is", + "aller gies", + "u bis", + "pengu in", + "ser a", + "vi z", + "prosp erous", + "ici des", + "tornad oes", + "sene gal", + "web cast", + "sto red", + "enchan ted", + "bb cone", + "bay area", + "entrepreneu rial", + "rednation rising", + "experim enting", + "ang an", + "lot to", + "they re", + "por e", + "er p", + "seren e", + "east wood", + "bro kers", + "bar ge", + "stal lion", + "timber lake", + "tailo red", + "dy stop", + "b ate", + "lat ors", + "di xit", + "bran son", + "dynam o", + "ky lie", + "shame ful", + "bt wn", + "spring time", + "mix ture", + "s ounded", + "lu ton", + "dad es", + "mal a", + "op ra", + "en ic", + "rahulg andhi", + "se wer", + "~~ ~~", + "ky u", + "nor theastern", + "ca er", + "bc u", + "nir vana", + "kitch ens", + "ous y", + "al m", + "river dale", + "hid den", + "fl int", + "sp d", + "pat rons", + "katy perry", + "au gh", + "exhib itions", + "sm c", + "shu ts", + "at ore", + "da in", + "some thing", + "ber th", + "bo g", + "por ter", + "gen to", + "con cussion", + "ang lic", + "ro we", + "gr illing", + "scar lett", + "master ing", + "mor nin", + "comm ented", + "si me", + "si zing", + "christ y", + "ce os", + "st m", + "at ry", + "tari ffs", + "vac ation", + "pre judice", + "p su", + "paren tal", + "far age", + "can a", + "cap com", + "koso vo", + "you re", + "men stru", + "stal in", + "grape fruit", + "br an", + "che sa", + "dav en", + "exc el", + "!! )", + "๠Į", + "distribu tor", + "ce a", + "bride sma", + "millenni al", + "wa in", + "ob serving", + "mis ery", + "plan etary", + "expo sing", + "bra ised", + "comp ton", + "don gha", + "q l", + "spring steen", + "th ul", + "syl ve", + "cab o", + "pal ad", + "niel sen", + "gaz ing", + "ba ja", + "r oud", + "orchi ds", + "johan nesburg", + "se man", + "d ji", + "oper ative", + "affe ction", + "eclec tic", + "at c", + "mut ant", + "aw x", + "nic e", + "mel bourne", + "indu lg", + "tu lip", + "dias pora", + "wel p", + "big gie", + "mississ auga", + "retri ever", + "or an", + "tam my", + "c ta", + "hipp o", + "seas oned", + "ger mans", + "eng v", + "marvell ous", + "im f", + "rela ys", + "mon tan", + "maur iti", + "me ister", + "as surance", + "reig ning", + "su fficient", + "han e", + "no thing", + "pos se", + "nav y", + "in love", + "brigh ton", + "en qu", + "ch ung", + "sweat y", + "es c", + "cal ed", + "man s", + "nicar agua", + "sl ices", + "mo cha", + "washington post", + "bb n", + "dam ned", + "grow ing", + "en burg", + "lo an", + "me s", + "wh oops", + "believ ers", + "spi el", + "vo daf", + "l at", + "s led", + "cricke ter", + "brown e", + "golf ers", + "bar ra", + "wat chers", + "lu igi", + "sw amy", + "mom s", + "pit ched", + "san tor", + "cr s", + "si re", + "sc amp", + "bo de", + "ste war", + "jon ny", + "ent ity", + "pac qui", + "mind ful", + "min india", + "bear ded", + "temp t", + "scorpi on", + "eat on", + "authori zed", + "ar to", + "s vp", + "op athy", + "cch ini", + "house music", + "disney world", + "âĢĶ @", + "pro pose", + "di y", + "expen se", + "ten g", + "pupp ets", + "sm el", + "d aca", + "per ry", + "fin n", + "boo sting", + "lefto vers", + "cou gs", + "satell ites", + "man y", + "az e", + "g ong", + "fi e", + "metho do", + "fer ries", + "ðŁ¤Ķ ðŁ¤Ķ", + "explore rs", + "load er", + "attrac ted", + "il ton", + "godd amn", + "pi azza", + "doc tr", + "sav ing", + "paragra ph", + "visu alization", + "may ors", + "work flow", + "ack les", + "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ", + "ठ¸", + "twer k", + "clu t", + "lo ver", + "te ases", + "si an", + "o te", + "deter ior", + "accor d", + "l fw", + "swar ovski", + "nat al", + "tra ps", + "k ina", + "analy ze", + "laye red", + "bever ages", + "un it", + "ran som", + "pe shaw", + "dest ined", + "astro logy", + "si pping", + "miley cyrus", + "cam ino", + "marshmal low", + "bli ss", + "out back", + "fa q", + "int oler", + "humil ity", + "po ppin", + "hallo ween", + "mon tene", + "op hy", + "nu n", + "tattoo ed", + "a as", + "ðŁĮ ³", + "dale y", + "qual ity", + "du sa", + "fisher men", + "swi f", + "ter rac", + "st au", + "le in", + "trol ling", + "ship ment", + "garden er", + "march madness", + "head band", + "gr t", + "bur nett", + "w and", + "!!!! !!!!!", + "gh e", + "du x", + "hu d", + "war ner", + "ðŁĩ ¦", + "ex ile", + "rescu e", + "rat a", + "d han", + "duc ati", + "dro wn", + "bl ends", + "spi e", + "alli gator", + "simul taneously", + "broo ke", + "u ke", + "k har", + "comm union", + "ri ka", + "ford fc", + "chin atown", + "you rown", + "me y", + "can al", + "syste matic", + "de pri", + "ox ford", + "an il", + "w ut", + "equ ation", + "be z", + "fle ur", + "the good", + "lang ley", + "ad ity", + "ed ith", + "al fie", + "о ÑĤ", + "en cry", + "br ill", + "ex emp", + "ce sar", + "mb ling", + "ab ri", + "sc icom", + "j ing", + "school ing", + "mi ka", + "mechan isms", + "impromp tu", + "rhe a", + "moo re", + "crime a", + "be sto", + "wri ght", + "el ders", + "ro ds", + "kam al", + "folkl ore", + "be et", + "mini on", + "reli eve", + "thr o", + "team usa", + "pas cal", + "made with", + "boli via", + "itt i", + "free bies", + "desi red", + "best selling", + "l iness", + "la den", + "ke ane", + "mi sts", + "hipp ie", + "atta chment", + "@ /", + "se w", + "flan agan", + "âĿĹ ï¸ı", + "supre mac", + "stl cards", + "si as", + "q u", + "rh ys", + "ste ep", + "val leys", + "v w", + "pav ing", + "disp at", + "al ison", + "por te", + "id u", + "new sc", + "soc ket", + "mo s", + "co star", + "re vo", + "prote ins", + "stanley cup", + "m cal", + "ear ring", + "se cs", + "mc lean", + "cap ric", + "nick elo", + "ad en", + "v c", + "shou se", + "adap tive", + "maxi mize", + "entertain er", + "pro se", + "gri ffi", + "six teen", + "lam ar", + "mi rage", + "saudi arabia", + "awe ather", + "ru st", + "in filtr", + "fashion week", + "ðŁĺĬðŁĺĬ ðŁĺĬ", + "selec tive", + "bubb le", + "a den", + "fen nel", + "deci sive", + "m ta", + "mock ing", + "mb les", + "st amp", + "mu le", + "bernar do", + "gr in", + "po tt", + "j ingle", + "vet tel", + "colom bian", + "cam o", + "motivation monday", + "ba han", + "p ly", + "dh ary", + "k ami", + "x men", + "sleep er", + "gar a", + "my sti", + "confi dential", + "conflic ts", + "p neu", + "ce s", + "insur tech", + "clean se", + "me rely", + "va is", + "tu x", + "the great", + "shar on", + "ma j", + "hol a", + "eco systems", + "aj ay", + "aa j", + "hu sh", + "har mon", + "backto school", + "wiki leaks", + "reflec ted", + "ðŁĺ ĵ", + "commemor ating", + "ac et", + "buck ingham", + "messi ah", + "tu ous", + "hor net", + "to be", + "d q", + "he ine", + "mi g", + "pl ate", + "nichol son", + "sp ie", + "cumber land", + "nor mal", + "pho bia", + "happy halloween", + "city fc", + "mc el", + "gilli an", + "ke to", + "lu de", + "de mise", + "su ga", + "str ate", + "mcgr ath", + "visit scotland", + "foo led", + "cb r", + "gc se", + "col ori", + "po td", + "missuni verse", + "fin ances", + "ma poli", + "for ks", + "Ø ´", + "cann on", + "medic inal", + "ðŁĹ ĵ", + "kh o", + "wre ck", + "pan to", + "bag el", + "gu ll", + "syndic ate", + "ic y", + "pr c", + "ki en", + "zi ka", + "ti sh", + "pe ta", + "c co", + "li za", + "ch ut", + "ex traction", + "el g", + "gl i", + "fu eled", + "pos it", + "respec tively", + "leice ster", + "br ink", + "vulner ability", + "im ported", + "e sha", + "ðŁ¦ ħ", + "r ural", + "re ll", + "gam ing", + "atlan tic", + "aband on", + "no ah", + "re solved", + "pro state", + "aller gic", + "ps d", + "âĺ ¹", + "dun geon", + "fang irl", + "illumin ated", + "m hs", + "white sox", + "d ently", + "ck o", + "endor se", + "over ly", + "dazz ling", + "prior iti", + "night life", + "ut il", + "be have", + "flam en", + "east bound", + "ðŁĴ Ł", + "ilove you", + "gov uk", + "mozam bique", + "alle gi", + "dr i", + "testim onial", + "ath s", + "ì§ Ģ", + "mm y", + "shab by", + "pro secco", + "friend ships", + "cal am", + "dam ages", + "off set", + "jura ssic", + "jun o", + "arre ll", + "ðŁĴ ©", + "interven tions", + "dare devil", + "car ver", + "run away", + "ran e", + "truste es", + "ha ute", + "dep ths", + "ðŁİ Ń", + "me in", + "sacrific es", + "con cier", + "ne sting", + "i zzy", + "me tam", + "ilove my", + "ur ine", + "du lu", + "mal hotra", + "ve ins", + "night ly", + "co at", + "an di", + "he witt", + "lon el", + "ci ble", + "wr ite", + "jen nie", + "sant ac", + "ĸ ï¸ı", + "str ato", + "singapo re", + "sop rano", + "kri sten", + "cheer ful", + "flee twood", + "fa iri", + "m eli", + "wa st", + "tur nt", + "sfor sale", + "sc rolling", + "angel ina", + "ren dition", + "jeric ho", + "nick y", + "or b", + "fla vo", + "patri ot", + "ash eville", + "sick ness", + "re fund", + "aggre ssion", + "b pl", + "ãĥ ĥ", + "elu sive", + "thi story", + "hang er", + "bu ffs", + "vil las", + "at kinson", + "sp h", + "ja it", + "decl ined", + "wo k", + "supre macy", + "oo tball", + "ey ang", + "ðŁİ ĵ", + "s ford", + "ath i", + "consu me", + "road ster", + "e so", + "u pro", + "reci pe", + "au f", + "uc i", + "ar on", + "oo oh", + "cs go", + "re ich", + "mc d", + "min ute", + "ladi es", + "pun k", + "rut gers", + "mee k", + "ariz on", + "ta j", + "land lord", + "de gra", + "autu mn", + "lyn x", + "us f", + "b hi", + "fairy tale", + "dongha e", + "bet sy", + "explo ded", + "chen nai", + "op a", + "pro tag", + "br ant", + "ðŁĵ °:", + "g f", + "pal li", + "ðŁı¼ âĢįâĻĢï¸ı", + "su t", + "ill ini", + "colum nist", + "shir tless", + "de centr", + "sear ched", + "ec or", + "bu ggy", + "s ack", + "ðŁĺĤ ðŁĺŃ", + "de t", + "ther i", + "or naments", + "bring back", + "to v", + "quarter finals", + "ic he", + "con stra", + "gi er", + "buchan an", + "vi x", + "kay aking", + "mu stread", + "swal low", + "mel b", + "sc af", + "op al", + "may oral", + "har at", + "ðŁ¦ ĭ", + "schedu les", + "id f", + "ha gue", + "ro z", + "a ah", + "d mc", + "du plic", + "ca che", + "orph an", + "frac ture", + "rec on", + "ch av", + "bun nies", + "al ain", + "mustaf a", + "ðŁİ Ļ", + "vac ations", + "dynam ite", + "tex ted", + "broad caster", + "ðŁĴ £", + "ste amed", + "rock er", + "di etary", + "luxury travel", + "inaugur ated", + "sa wards", + "vaugh n", + "lincoln shire", + "click ed", + "kra ja", + "f anc", + "remo ves", + "layo ffs", + "mc far", + "bre eds", + "win nie", + "jon ghyun", + "incen tive", + "vari ations", + "pat ton", + "atur day", + "persist ent", + "pr un", + "pi ers", + "dal es", + "æ ĸ", + "breast feeding", + "r ance", + "ta wa", + "Ĥ âĸ", + "mur doch", + "cap tive", + "thi stle", + "nic a", + "commod ity", + "cou ldnt", + "board walk", + "graci ous", + "practiti oners", + "n gc", + "scru m", + "ner o", + "camoufla ge", + "col on", + "he i", + "phys icist", + "saturday morning", + "ten er", + "si won", + "colum ns", + "bru ne", + "y vr", + "ba ir", + "reti res", + "hal am", + "cab er", + "shaz am", + "min u", + "cas cade", + "milk shake", + "gri d", + "d ren", + "vin cent", + "so dium", + "plat ter", + "cheer leader", + "chen ko", + "y ak", + "elimin ated", + "ty po", + "y man", + "re think", + "âĿ Ĺ", + "ts ville", + "bernardo kath", + "ex tr", + "ðŁĺģ ðŁĺģðŁĺģ", + "ta o", + "re per", + "mo ths", + "em powered", + "c iting", + "transpor ted", + "mon ks", + "san at", + "cle ars", + "bachelore tte", + "camp bell", + "racha el", + "har le", + "hand ler", + "climb s", + "inter ference", + "rele ase", + "sh and", + "r bs", + "hr h", + "ãģ ª", + "val le", + "r é", + "sli me", + "w akes", + "chu bby", + "slo an", + "el ves", + "ath en", + "attor neys", + "micro scope", + "ston er", + "sc aling", + "o be", + "c out", + "se man", + "mid week", + "bal sam", + "ðŁĺį âĿ¤", + "ti ful", + "v ish", + "lo tta", + "ri pping", + "re mn", + "ti re", + "le ap", + "ha vent", + "la by", + "hi mach", + "whisp ers", + "we in", + "ðŁİ ¸", + "wild flowers", + "se le", + "u cc", + "li ability", + "az ine", + "sw ings", + "k ya", + "ta ir", + "re main", + "e do", + "flo ps", + "poc ket", + "grand ad", + "exam iner", + "gr is", + "ffe ct", + "ðŁijĬ ðŁı»", + "stud ded", + "heart beat", + "de acon", + "firm ly", + "infec tious", + "ste f", + "out lines", + "le asing", + "cla ws", + "sen se", + "tab s", + "hoo t", + "mo sul", + "spa wn", + "co a", + "hog warts", + "ve in", + "alban ia", + "manu el", + "b ino", + "vaux hall", + "scot land", + "go bucks", + "mat ty", + "phy sio", + "tor ino", + "const able", + "investig ated", + "s lower", + "mistak en", + "bay er", + "wild fires", + "vo ic", + "x on", + "time to", + "chas sis", + "bar ric", + "pi on", + "bald head", + "woo k", + "regi str", + "dra fts", + "b hs", + "li gue", + "l ick", + "staf fordshire", + "baf ta", + "dar ry", + "je anne", + "ven ding", + "cor p", + "⼠³ï¸ı", + "kid dos", + "fen way", + "ca o", + "west bound", + "ðŁĺ Ļ", + "dv r", + "quick er", + "bla h", + "goo die", + "ðŁĴĭ ðŁĴĭ", + "vo x", + "esp er", + "fac ade", + "cor relation", + "red bull", + "rou p", + "decl ining", + "chi ve", + "mc gee", + "tur o", + "in der", + "f eller", + "fu g", + "il ysm", + "mar di", + "peshaw ar", + "ki eran", + "ine ma", + "meat balls", + "pe ck", + "depre ssing", + "sen sing", + "gi z", + "dd ington", + "spring watch", + "ro aming", + "yellow stone", + "horse shoe", + "am man", + "week day", + "ol or", + "ðŁ¥ °", + "boo sts", + "spr int", + "scar ves", + "je e", + "bee tro", + "cl an", + "all the", + "ìĦ ¸ë", + "enlighten ment", + "ado be", + "re generation", + "? @", + "cont ag", + "yach ts", + "to u", + "mor a", + "en voy", + "r ani", + "go li", + "dhanush kraja", + "wood working", + "streng ths", + "se di", + "disc s", + "ar ina", + "sc on", + "lit e", + "ano ther", + "ðŁ¥ Ĭ", + "ye men", + "gu ern", + "sav vy", + "lo yed", + "biom ed", + "heart break", + "comra des", + "milli e", + "pat ch", + "un f", + "jar vis", + "bl aming", + "commemor ation", + "ge y", + "å ¥", + "cardio vascular", + "alig ned", + "docu ment", + ". ?", + "aesthe tics", + "em u", + "the irs", + "le h", + "ps ic", + "si f", + "pl ateau", + "ex pend", + "domin ating", + "rob es", + "mauriti us", + "excep tionally", + "hom er", + "discover ies", + "bra un", + "ten nant", + "insul in", + "ðŁİ ®", + "car bs", + "te as", + "? !\"", + "zi e", + "franco is", + "brow sing", + "th ol", + "cla rence", + "hel per", + "ob tained", + "cas sie", + "le es", + "! ,", + "pome gran", + "hu bs", + "presti ge", + "] [", + "mach er", + "bott led", + "pun ch", + "pi pe", + "o ch", + "gall ons", + "deliver ies", + "u ra", + "un day", + "mon de", + "depic ts", + "re gency", + "outra geous", + "khal ed", + "car o", + "he arti", + "za g", + "develop mental", + "over coming", + "stati stical", + "flavo red", + "for ds", + "cre atives", + "lau rence", + "di as", + "sun screen", + "in ked", + "pre acher", + "n ul", + "impac ting", + "auti stic", + "âļ Ķï¸ı", + "o ss", + "pel icans", + "cele ste", + "v b", + "ru mp", + "mc gra", + "fair fax", + "hu mor", + "bbc news", + "row ling", + "cal der", + "seam less", + "ag ne", + "p ti", + "mix ed", + "t shirts", + "mer ci", + "b tob", + "women instem", + "genealo gy", + "pre ven", + "l our", + "cra dle", + "gi use", + "Ð ¾", + "chron o", + "fair ness", + "chocol ate", + "tor y", + "as da", + "pre scott", + "stret ched", + "al man", + "u il", + "re charge", + "in tre", + "ob st", + "hosp ital", + "hay ward", + "teneri fe", + "fried man", + "vap ing", + "confe ssions", + "ye ah", + "bal li", + "luck now", + "cor pse", + "sculp tor", + "amp ton", + "t pp", + "indic ates", + "sur plus", + "tru man", + "ðĿ Ļ", + "sin ha", + "in vo", + "sovere ign", + "ke v", + "establi shing", + "engra ved", + "assu ming", + "ðŁı ģ", + "sou za", + "fab i", + "ton ed", + "oun ge", + "del oit", + "dow ney", + "no ble", + "om or", + "car tridge", + "ðŁı IJ", + "u hur", + "hol loway", + "succe sses", + "r sa", + "âĦ ¢", + "ma zz", + "tw d", + "disc ourse", + ". <", + "y at", + "satis fy", + "com pri", + "ठ¹", + "graph ite", + "disser tation", + "ar ter", + "í Ķ", + "b ally", + "zom bi", + "ly ons", + "a ic", + "u bc", + "pra da", + "e il", + "da x", + "cla i", + "grand daughter", + "extravag anza", + "chall enge", + "ðŁ¤ ŀ", + "po ver", + "primar ily", + "dad dy", + "man a", + "bi kers", + "inqui ries", + "da un", + "fel ine", + "gener ative", + "he f", + "benef iting", + "lind sey", + "pol ka", + "demonstr ated", + "al le", + "rand y", + "o su", + "low key", + "weir dest", + "red bull", + "our y", + "n ous", + "wood stock", + "cre denti", + "nic er", + "g ado", + "aly ss", + "ap h", + "prepa redness", + "station ary", + "incorpor ated", + "dy er", + "sarato ga", + "cele sti", + ": \"", + "antibio tics", + "or gs", + "inde fin", + "ap ron", + "и Ð", + "fif teen", + "no f", + "ðŁĶ Ŀ", + "ph x", + "te ga", + "m z", + "organiz ational", + "on air", + "band ung", + "pleas ures", + "mor i", + "secre tari", + "rac coon", + "ca shi", + "pil ates", + "k on", + "geof frey", + "la o", + "kam p", + "depart ments", + "back packing", + "an am", + "à «", + "crack down", + "aun ty", + "on do", + "li zzie", + "ph ers", + "cu n", + "ðŁĩ ±", + "k pop", + "pu t", + "inten tional", + "connol ly", + "bar clays", + "hs fb", + "swin don", + "u ku", + "s ally", + "a int", + "âľ ħ", + "pen ang", + "up lifting", + "epile psy", + "inter ro", + "bun gal", + "go ku", + "blue berries", + "ठ¦", + "u ssia", + "sil ky", + "mou red", + "i stic", + "bri efs", + "me ats", + "go b", + "ch aser", + "state wide", + "pra sad", + "gl itch", + "ar in", + "ban ff", + "memb er", + "ðŁĺŃ âĿ¤ï¸ı", + "lo ving", + "hall a", + "ภ¡", + "smo kers", + "yak u", + "scicom m", + "physi o", + "sw ol", + "lem ons", + "gel ato", + "ch ool", + "capit als", + "ki stan", + "ti ghts", + "spi kes", + "trav ellers", + "ik lan", + "commissi oning", + "ar ine", + "emabiggest fans", + "empha sis", + "front line", + "pad dock", + "destruc tive", + "ba ha", + "l inger", + "je wish", + "shet land", + "mc gin", + "mon key", + "ko z", + "s one", + "raj ini", + "te h", + "y en", + "c vs", + "masqu er", + "gir ly", + "we sle", + "was nt", + "bro dy", + "termin ator", + "gil le", + "mag gi", + "bir die", + "jeopar dy", + "cu bic", + "vm ware", + "intric ate", + "an up", + "to pia", + "east on", + "sab res", + "investig ates", + "bu sting", + "bil ingual", + "valent ino", + "in format", + "fer re", + "advent ur", + "hydr ate", + "for sy", + "az iz", + "san to", + "e de", + "whist ler", + "continu ously", + "d ham", + "un used", + "ji had", + "addic tive", + "vi dy", + "do b", + "i do", + "fi ed", + "ni versary", + "n one", + "fu er", + "ðŁĺį ðŁĺĺ", + "coven ant", + "prin table", + "immac ulate", + "o em", + "cl t", + "serv ants", + "consu med", + "un released", + "sc um", + "pack aged", + "me re", + "ìĦ¸ë ¸", + "to by", + "ta f", + "spo ons", + "me al", + "f ball", + "fair field", + "jan et", + "silver stone", + "dart mouth", + "follow me", + "voy ager", + "kom bat", + "anni ver", + "ene w", + "mag dal", + "ho ve", + "sa th", + "grizz ly", + "car di", + "gart ner", + "sand y", + "kan ye", + "post ure", + "po ign", + "im pulse", + "radio logy", + "horiz ons", + "si am", + "aish war", + "= =>", + "no che", + "tr is", + "el yn", + "com me", + "du i", + "ce c", + "councill ors", + "cudd ling", + "creep ing", + "loc ke", + "manag es", + "trans ferred", + "ne cks", + "di er", + "dan o", + "v ick", + "lun ches", + "d he", + "en sures", + "cri ss", + "ul ster", + "bann on", + "cont enders", + "sp am", + "sweet ness", + "med al", + "hon duras", + "arc tic", + "ultra sound", + "in fr", + "disco vers", + "ei ffel", + "ca sters", + "ru ben", + "du st", + "awe ed", + "atri um", + "lest we", + "se ared", + "ðŁĵº :", + "ty ne", + "ex changes", + "little mix", + "l le", + "astron auts", + "hersh ey", + "work day", + "kno b", + "so v", + "re signs", + "today show", + "der man", + "an th", + "af c", + "ta ster", + "sw oo", + "sa eed", + "per ing", + "narrow ly", + "rn li", + "best buy", + "panas onic", + "obst acle", + "farmer s", + "ðŁİ Ļ", + "pa wan", + "ki est", + "ang ers", + "absur d", + "oh my", + "sin o", + "pist achi", + "sp ice", + "giu li", + "prime time", + "ko w", + "k ens", + "ex agger", + "! ?!", + "u ba", + "midd les", + "ju dd", + "e jec", + "slam med", + "pen sions", + "of a", + "re create", + "b hp", + "xx l", + "liver pool", + "thre sh", + "pur ity", + "ni eu", + "hol ics", + "wr ath", + "ra do", + "gli o", + "am ma", + "dile mma", + "cr u", + "lets go", + ".... @", + "âĿ ĵ", + "sugge sting", + "tru mps", + "hor us", + "f v", + "ic om", + "refer ring", + "predic tive", + "tar ts", + "ge tte", + "so ck", + "glo ssy", + "pin ky", + "al ec", + "thy me", + "ou ra", + "thero ad", + "pe tr", + "cr am", + "p fi", + "dv n", + "me ier", + "incen tives", + "tun nels", + "mobi l", + "rec ap", + "extra s", + "upri ght", + "rev amp", + "per severance", + ", -", + "ot p", + "mir ror", + "ar wx", + "ger ry", + "ma her", + "g or", + "hom epage", + "am is", + "ag ra", + "made le", + "best friend", + "sirius xm", + "bun dles", + "admir ing", + "t dsb", + "ðŁį ģ", + "ch as", + "slow ing", + "ro h", + "wall papers", + "âĢ¦ /", + "tek ken", + "gang s", + "tal a", + "lind say", + "shou l", + "line backer", + "tool kit", + "ur anium", + "caly p", + "ab rams", + "mat thi", + "ðŁı ¿", + "hon ourable", + "da yo", + "ver sail", + "tan k", + "st c", + "fr itz", + "spl end", + "pat ag", + "anno yed", + "on day", + "devast ated", + "chattanoo ga", + "national ism", + "mas sey", + "jen n", + "tail or", + "dev gn", + "org ans", + "zu cchini", + "on fox", + "sat ire", + "wex ford", + "dis grace", + "no to", + "vol ta", + "âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ı", + "à ¶", + "home owners", + "poin ter", + "m cr", + "au sten", + "day sto", + "mo ons", + "pal ma", + "gra zing", + "e so", + "influen cers", + "shahid kapoor", + "compli ant", + "measure ments", + "develop s", + "y d", + "par l", + "p vt", + "rand olph", + "tor tured", + "ger ald", + "eli as", + "deepi kap", + "war mup", + "hick ory", + "g ap", + "co ffin", + "am our", + "re neg", + "moun ting", + "seven s", + "ig le", + "hi er", + "dec ad", + "tri ght", + "esc apes", + "wer ner", + "t fl", + "ful filled", + "ni ger", + "sour dough", + "re aper", + "choo ses", + "spin ner", + "week nd", + "fil tered", + "sh uk", + "kat i", + "old ham", + "open source", + "kh anna", + "at elier", + "conne c", + "opho bic", + "gla s", + "complic ations", + "ar son", + "counc ils", + "sm ol", + "as sy", + "lur king", + "ling ui", + "han ks", + "e in", + "Ù ħ", + "ru gs", + "n guyen", + "nou veau", + "men ace", + "le v", + "alad din", + "ru ining", + "round about", + "k m", + "con or", + "shoo ps", + "may day", + "traum atic", + "prab has", + "ka iser", + "k ita", + "rou ter", + "pe dro", + "re tar", + "stun ner", + "spani sh", + "distur bed", + "acade my", + "e learning", + "wit ty", + "sen g", + "fer al", + "av y", + "sta b", + "ke aton", + "ur du", + "ko to", + "hu i", + "coo ke", + "ari an", + "the personal", + "u ma", + "se ap", + "a sting", + "rhetor ic", + "hand writing", + "munici pality", + "consor tium", + "ðŁIJ Ł", + "glasgo w", + "ra ya", + "eli za", + "polym er", + "bro th", + "prac ti", + "correspon dent", + "addic ts", + "gay le", + "ail ing", + "o fe", + "p li", + "hear tw", + "st itch", + "sight ings", + "prie sts", + "sam o", + "slo th", + "good wood", + "roc co", + "sab c", + "summ it", + "l ace", + "pres ley", + "itt en", + "cin cy", + "thepersonal network", + "s week", + "pe gas", + "af con", + "regi stry", + "ci m", + "le th", + "dic ap", + "cand ice", + "flu ent", + "sm ack", + "pede stri", + "al oud", + "car ac", + "priyan kach", + "p gh", + "ir ons", + "dol ce", + "lat via", + "dece ased", + "thero ck", + "cla p", + "cen e", + "fo am", + "morris sey", + "gre t", + "essenti ally", + "com cast", + "be agle", + "argu es", + "ing ed", + "- âĢ¦", + "sa g", + "ha san", + "ðŁĻ Ĩ", + "ðŁį °", + "nh ra", + "kann ada", + "indic ators", + "on er", + "bri xton", + "at as", + "screen play", + "sor ority", + "sha heed", + "he em", + "class mates", + "tain ment", + "es i", + "breast cancer", + "zucker berg", + "aur or", + "en cia", + "ref ers", + "kae per", + "vor tex", + "com part", + "lym ph", + "photograph ing", + "ste ff", + "rest ling", + "par sley", + "mom ento", + "th man", + "lac king", + "du tt", + "ocu lus", + "fin o", + "fren zy", + "ra sc", + "der n", + "dis missed", + "noo k", + "met gala", + "sh ill", + "rapha el", + "maver icks", + "exhib its", + "eag erly", + "c pa", + "amen ities", + ". âłĢ", + "exo dus", + "ern st", + "lit a", + "deal t", + "womens march", + "i ain", + "score board", + "campe ones", + "c en", + "ti ki", + "garri son", + "fidel ity", + "bra g", + "road map", + "psy chop", + "lo e", + "ble u", + "ðŁijĬ ðŁı¼", + "sau vi", + "spr inger", + "temp tation", + "ru dolph", + "ac ura", + "wic z", + "parach ute", + "stro l", + "len ny", + "zi k", + "dom s", + "nb af", + "al pac", + "vivi an", + "ro ve", + "pre et", + "perpe tu", + "sna ke", + "air soft", + "infl atable", + "prin ces", + "ati e", + "ffe y", + "pati ent", + "m ire", + "chel le", + "sl ack", + "groo vy", + "# :", + "up loading", + "!!!!!!!! !!!!!!!!", + "siem ens", + "provi sion", + "v fx", + "need y", + "f ats", + "to poli", + "bhu tto", + "sa thletics", + "alu ms", + "t winning", + "south western", + "adop ting", + "last night", + "man ne", + "la ga", + "tw ell", + "ac ia", + "-- --", + "eye wear", + "hur ley", + "fle e", + "sa ch", + "pe cker", + "cost ly", + "is k", + "cr ates", + "polic y", + "ero sion", + "in go", + "wer k", + "ðŁIJ į", + "torto ise", + "therap ies", + "inter net", + "chihuahu a", + "ri ps", + "fre i", + "ed or", + "tai ji", + "t fc", + "do d", + "demp sey", + "christ in", + "chen g", + "hi ps", + "gra eme", + "com passionate", + "cavali ers", + "histor ic", + "soul ful", + "crimin al", + "ja c", + "vin ci", + "expi red", + "sur at", + "turi smo", + "k ona", + "se aweed", + "ber ts", + "le ica", + "expre ssing", + "a al", + "wor t", + "break fast", + "her ring", + "am used", + "rhu barb", + "mar tian", + "cospla yer", + "y ash", + "stri al", + "ra ul", + "refer ral", + "dw ts", + "j w", + "ad ler", + "cur tains", + "gu r", + "val ence", + "tyr one", + "sw fc", + "coach ed", + "re born", + "diabe tic", + "cho ke", + "nor folk", + "investig ative", + "ðŁĴ¯ ðŁĴ¯", + "z id", + "v mas", + "phi e", + "objec tives", + "âľ ĭ", + "over due", + "di vers", + "mat su", + "ðŁİŁ ï¸ı", + "casu alties", + "ภ§", + "al k", + "stand ardi", + "re alist", + "arti facts", + "pand or", + "ke x", + "in vin", + "( !)", + "ine y", + "par aly", + "mr t", + "fay e", + "the voice", + "on ga", + "de ed", + "skin ner", + "az wx", + "speci men", + "priyankach opra", + "nu evo", + "bar kley", + "toulou se", + "resu mes", + "football ers", + "cit i", + "fe tch", + "è re", + "lestwe forget", + "ðŁĻ ĭ", + "ch unk", + "dri fting", + "manipul ation", + "equ als", + "pu tt", + "ky ungsoo", + "âĿ¤ï¸ı #", + "ela stic", + "par ano", + "fo y", + "do ping", + "cin cy", + "ss ler", + "interrup ted", + "al ay", + "ado res", + "ame thy", + "con voy", + "ãĢ ı", + "Ĭ ãģ", + "black list", + "gener als", + "sa chin", + "bru shed", + "oun ces", + "non stop", + "illi ams", + "bt sarmy", + "u av", + "ru ff", + "bur ma", + "bi k", + "defen ce", + "schul tz", + "bo asts", + "lonel iness", + "go re", + "trans forms", + "alum na", + "@ @", + "ra ppers", + "ne hru", + "car o", + "himalay an", + "wearab les", + "ge h", + "pepper mint", + "re development", + "flam ingo", + "cos by", + "big baldhead", + "ag ri", + "bare foot", + "sco pes", + "re gram", + "gh ana", + "ðŁİ «", + "i heart", + "sa die", + "carri e", + "microbi al", + "ku ala", + "sk ater", + "quer que", + "âĻ ©", + "gen res", + "reas oning", + "ch ased", + "as o", + "sli pped", + "en can", + "vam os", + "ker s", + "ad verse", + "mo il", + "commod ities", + "with you", + "sil ent", + "hy pe", + "an de", + "am ination", + "whi spe", + "lit z", + "âļ½ï¸ı âļ½ï¸ı", + "ri ff", + "pp y", + "lam bs", + "gan esh", + "ab sent", + "regu lator", + "marse ille", + "en roll", + "par cel", + "wa p", + "by rd", + "ðŁĩ Ń", + "tu ber", + "country music", + "par l", + "contro llers", + "responsi bilities", + "we y", + "ch ate", + "montene gro", + "chic o", + "mil an", + "l ms", + "tra inees", + "appropri ately", + "un certain", + "popp ies", + "ed sheeran", + "nutr itious", + "gar o", + "deut sch", + "awe some", + "ãĥ ¼", + "comfor tably", + "land marks", + "et i", + "re usable", + "daniel le", + "ro sal", + "co les", + "just ic", + "c cs", + "f anny", + "ni m", + "mc u", + "clin ch", + "at ene", + "mer ge", + "im db", + "ang lo", + "uc cino", + "pan ini", + "an not", + "bur berry", + "feat ure", + "predic ting", + "fashioni sta", + "s ask", + "imag inary", + "mm o", + "south sudan", + "spe ar", + "hu bble", + "jo inthe", + "coyo tes", + "sli go", + "ko dak", + "sit com", + "polaro id", + "roo ted", + "corru p", + "ðŁĻĮ ðŁĻĮ", + "bris ban", + "at z", + "ah l", + "re my", + "tal ent", + "aval on", + "ra da", + "pau line", + "locom otive", + "go ons", + "ne mo", + "maser ati", + "ic u", + "stu tt", + "histor ically", + "sm b", + "pres by", + "avo id", + "so oners", + "rhine stone", + "w ad", + "ri sing", + "tro t", + "mo des", + "reg ent", + "optimi ze", + "re ece", + "sm u", + "ver ti", + "newyork city", + "cor tez", + "ra c", + "in case", + "sin c", + "fiel ding", + "e tta", + "tiff any", + "al monds", + "sad dle", + "k rat", + "mat ter", + "g low", + "star ving", + "gl o", + "cra ppy", + "sl ur", + "st d", + "monit ors", + "recei pt", + "maymay entrata", + "mc il", + "un is", + "rain bows", + "cal dwell", + "pacqui ao", + "j op", + "a fe", + "hoo k", + "es sen", + "wiz ard", + "medi an", + "fla ws", + "com s", + "âĿ Ħ", + "ing h", + "ha ynes", + "anton io", + "tem plates", + "ou ter", + "na w", + "cardi gan", + "bel grade", + "ðŁĴ ī", + "hom o", + "a ise", + "ro pes", + "no ve", + "what you", + "tri gge", + "concep tion", + "ad ukone", + "na di", + "fri ars", + "sw er", + "adju sted", + "hot line", + "san ity", + "kau r", + "down loading", + "c gi", + "ten or", + "eth nic", + "app alach", + "ภ¸", + "pa g", + "gol ds", + "on set", + "investig ator", + "car tel", + "peace fully", + "jarre tt", + "cat alan", + "poli o", + "n um", + "fru stration", + "dhar ma", + "my life", + "âľĮ ðŁı»", + "aber deen", + "mu sa", + "bin der", + "spark ly", + "fle eing", + "instin ct", + "co ping", + "domin ance", + "ill ers", + "er a", + "u conn", + "lo oms", + "living ston", + "gal i", + "he s", + "c ma", + "bel a", + "se ley", + "mon k", + "la ch", + "mar x", + " ´", + "m erica", + "woman in", + "es sex", + "ra ina", + "jim i", + "nep tune", + "z ack", + "chine se", + "mart ins", + "chand elier", + "her n", + "with us", + "ear l", + "asph alt", + "modu les", + "st p", + "ul la", + "psychi atric", + "mile age", + "captiv ating", + "si der", + "men to", + "mor t", + "tran ce", + "tal bot", + "ab by", + "ì ĥ", + "âľĮ ðŁı¼", + "j ak", + "daw n", + "turn up", + "scre wed", + "fe ds", + "blue print", + "ðŁĴĸ ðŁĴĸ", + "har sh", + "er os", + "insom nia", + "ban kers", + "ta emin", + "mis conduct", + "hu mber", + "gi di", + "edu ardo", + "con a", + "musc ular", + "consu ming", + "ra sh", + "don nie", + "di pped", + "col lie", + "samu el", + "melt down", + "ðŁĺįðŁĺį ðŁĺį", + "me z", + "exam ining", + "schwar tz", + "pri stine", + "ðŁIJ Ŀ", + "ve it", + "ful filling", + "an esthe", + "gue sses", + "dra ft", + "som me", + "soli d", + "pati onal", + "ho ped", + "evolu tionary", + "all er", + "enter tained", + "sli ps", + "lud wig", + "conclu des", + "sen sible", + "bon net", + "cra ze", + "tra s", + "haz ards", + "const antine", + "ed ics", + "star trek", + "to c", + "occu pational", + "in cheon", + "deepikap adukone", + "pizz as", + "new comer", + "de part", + "oppre ssion", + "ebon y", + "foss ils", + "tro jan", + "el en", + "ste aks", + "k hou", + "positi oning", + "ug by", + "red cross", + "ak h", + "dol ce", + "us mnt", + "pp en", + "dil ig", + "ma vs", + "call er", + "cost ello", + "⼠Ħ", + "dy n", + "thing s", + "rhin os", + "a xi", + "sar kar", + "con vocation", + "att ers", + "ss ss", + "fun gus", + "eu gen", + "russ o", + "squ at", + "w sb", + "eli on", + "william sburg", + "s off", + "defici ency", + "be arer", + "o kin", + "key stone", + "t wain", + "cal ming", + "break able", + "wa res", + "horser acing", + "com bs", + "bun ting", + "u it", + "t land", + "ðŁĴĻðŁĴĻ ðŁĴĻ", + "ga stron", + "sab ot", + "ick ers", + "commissi oners", + "sen ate", + "ii ot", + "ath ena", + "nit rogen", + "an tony", + "ero tic", + "di alo", + "mis sou", + "hypo cr", + "âľ Ī", + "kaeper nick", + "can v", + "d roo", + "clevel and", + "o sh", + "mon sta", + "stefan o", + "^ )", + "sh ul", + "po ison", + "ha e", + "commerci als", + "ma ul", + "nit ro", + "co worker", + "alo e", + "vap or", + "t ents", + "russi an", + "qu id", + "question able", + "mid get", + "po ker", + "girl friends", + "sin the", + "erit rea", + "ten ure", + "depos its", + "buc keyes", + "spot ter", + "theod ore", + "trin ity", + "joaqu in", + "u cci", + "follow the", + "caf c", + "mp a", + "ðŁIJ »", + "plo tting", + "dom ino", + "ta ek", + "sion ally", + "dicap rio", + "pa p", + "car mel", + "ig er", + "bt cc", + "beth le", + "www bigbaldhead", + "foo die", + "bagh dad", + "mason ry", + "off ended", + "à ·", + "ภģ", + "sc ro", + "vers es", + "ori ent", + "ar ches", + "pi yu", + "know your", + "gre e", + "ta kers", + "gu ard", + "dish on", + "bucket list", + "bha fc", + "war dly", + "ðŁİīðŁİ Ĭ", + "leigh ton", + "pe w", + "stra y", + "assaul ted", + "in hal", + "ly fe", + "amar keting", + "l x", + "kat z", + "ubun tu", + "me o", + "carto onist", + "turno ver", + "mi z", + "dis like", + "mul len", + "mo f", + "bl and", + "hi des", + "emer ges", + "chori zo", + "truste e", + "ma hog", + "lan sing", + "paralym pic", + "fa int", + "fa una", + "ch al", + "sn ar", + "cat h", + "bent on", + "cast illo", + "sli ppery", + "apric ot", + "oec d", + "bar o", + "l z", + "he ming", + "clow ns", + "co workers", + "peru vian", + "commu ters", + "y ell", + "ðŁļ ´", + "under ing", + "v j", + "tt p", + "fli pk", + "w ana", + "soc ent", + "Ĥâĸ Ĥâĸ", + "ठĤ", + "oo sa", + "jag ger", + "di sm", + "e less", + "d ham", + "cali f", + "a official", + "ec lip", + "harro gate", + "gra pp", + "com rade", + "n tr", + "concentr ate", + "thi ghs", + "bit coin", + "bel arus", + "ë ĵ", + "end uring", + "now watching", + "industri al", + "pi p", + "ar on", + "ar at", + " ®", + "whit by", + "oooo ooo", + "sa ree", + "tic als", + "mis leading", + "yo on", + "year s", + "sle igh", + "roman ian", + "sciss ors", + "vam pires", + "ac up", + "ab ba", + "th weeksary", + "cent ri", + "fl ye", + "u o", + "c bi", + "bu ena", + "sin d", + "mar ino", + "bur r", + "re building", + "ठ²", + "anniver saire", + "ac ca", + "ðŁĴĢ ðŁĴĢ", + "gett ing", + "tu lips", + "wolf pack", + "âľį ï¸ı", + "more than", + "ta kin", + "ðŁ¤ĺ ðŁı»", + "u be", + "mon ic", + "dou bts", + "mo wer", + "co balt", + "don ne", + "specul ation", + "argu ably", + "kak u", + "htt ps", + "prosecu tion", + "din ah", + "stam atic", + "disclo sed", + "bever ly", + "fl wx", + "cra bs", + "extraordin aire", + "war mest", + "imper i", + "o logists", + "trac es", + "par c", + "lake side", + "am r", + "ter i", + "hour ly", + "domin ation", + "ar row", + "shrews bury", + "ance stry", + "wr angler", + "trigge red", + "pen sac", + "roo ster", + "survi ves", + "a on", + "bo ko", + "val or", + "love is", + "la g", + "pe y", + "fo cal", + "out laws", + "bl anc", + "artic ho", + "wit s", + "marsh all", + "die go", + "support small", + "u ca", + "sa h", + "je et", + "syn ago", + "gover ning", + "ðŁĴ ¬", + "sal ads", + "cre ate", + "miri am", + "cen sored", + "ami de", + "no u", + "z eta", + "allegi ance", + "* )", + "bl m", + "ric an", + "pa stors", + "oly mpus", + "blo c", + "whir l", + "star ry", + "pr one", + "y k", + "p ne", + "congratul ating", + "be v", + "so ber", + "love island", + "sa ir", + "an ing", + "tutor ials", + "q e", + "lun d", + "in ist", + "cle ver", + "taxpay er", + "ali z", + "wren ch", + "dd ling", + "cap ri", + "h pa", + "ðŁı» âĢįâĻĤï¸ı", + "na j", + "o j", + "futuri stic", + "jelly fish", + "ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥", + "cel ery", + "plan k", + "fil a", + "ne me", + "un healthy", + "lec tions", + "ðŁ§ ¡", + "rit chie", + "n ws", + "mi kha", + "wonder woman", + "âĢ İ", + "hip stamatic", + "ka g", + "ðŁĴľðŁĴľ ðŁĴľ", + "poul try", + "mo w", + "wor ds", + "lo ff", + "ðŁ¤£ ðŁ¤£", + "relat able", + "re mixes", + "keny atta", + "ke m", + "re signed", + "fo d", + "stra igh", + "j lo", + "hu tch", + "box ers", + "colle en", + "mag s", + "instruc tional", + "ko l", + "attrac ts", + "pra g", + "account ant", + "go ggles", + "br u", + "th ole", + "mar row", + "leu ke", + "oc to", + "pon ds", + "bubb ly", + "he ist", + "ìĹ ij", + "im p", + "a har", + "ha unt", + "hall mark", + "psy ch", + "kkkk kkkk", + "col umb", + "jump suit", + "cost co", + "si delines", + "ag gies", + "over turned", + "ni b", + "key chain", + "fu k", + "f af", + "mi am", + "assist ants", + "cy cled", + "ri der", + "dam mit", + "red wings", + "mag es", + "kin s", + "ì Ĥ", + "ho d", + "son t", + "carol ine", + "\" '", + "cu le", + "bra id", + "fel ony", + "ar ities", + "ruther ford", + "depic tion", + "isab elle", + "ro ach", + "k day", + "fifth harmony", + "em y", + "li gam", + "bari sta", + "albu querque", + "gro ss", + "ðŁį º", + "oo ks", + "ðŁij ¼", + "dun can", + "try in", + "jag s", + "g ould", + "li tho", + "âģ £", + "а Ð", + "sam my", + "tun g", + "cas ser", + "apo lo", + "aaaa a", + "man g", + "as ics", + "sh en", + "p ye", + "tur bul", + "ss p", + "saint sfc", + "on lin", + "n anny", + "he ster", + "do z", + "ภĶ", + "th read", + "ren ts", + "kh and", + "ðŁĴª ðŁı½", + "un conditional", + "rob son", + "car re", + "ph on", + "sacrific ed", + " £", + "auto s", + "par ker", + "oc a", + "log in", + "kee gan", + "hard cover", + "dough nuts", + "ðŁĮ İ", + "spit fire", + "refresh ments", + "saskat oon", + "commod ore", + "j f", + "rub ber", + "halam adrid", + "child care", + "stra da", + "io m", + "ri k", + "dak ar", + "ther mom", + "cro pped", + "gar u", + "ali k", + "ven i", + "i ft", + "si ka", + "ritu als", + "z ul", + "e ch", + " ©", + "su dan", + "l land", + "i me", + "do cker", + "ì ¤", + "fe ared", + "fa o", + "wal ter", + "no g", + "mutu als", + "l h", + "ali gn", + "mon ia", + "concep tart", + "ðŁĻı ðŁı¼", + "sco e", + "compet ence", + "sw ine", + "ly me", + "laun ch", + "green er", + "abstract art", + "inqu is", + "gran ada", + "ga elic", + "flu ff", + "d backs", + "grave yard", + "ba be", + "acade mic", + "adventur ous", + "joh ann", + "~ !", + "bi bi", + "| #", + "pl ings", + "gett y", + "as b", + "âĿ¤ï¸ı @", + "staf f", + "religi ons", + "bang or", + "world bookday", + "me gh", + "de vin", + "ash ore", + "meri dian", + "gi thub", + "qui z", + "all stars", + "be stest", + "ir resi", + "ack er", + "do te", + "war rington", + "pol ly", + "newor leans", + "cr ou", + "wi gs", + "che y", + "smithson ian", + "la sag", + "de tour", + "bor is", + "stra ps", + "mari ah", + "inten tionally", + "ko h", + "ðŁį ¸", + "ssi an", + "mar issa", + "cor al", + "episcop al", + "casu alty", + "tom o", + "supply chain", + "sam p", + "on go", + "ro o", + "cavi ar", + "p fw", + "clau dio", + "buff alo", + "s ations", + "mat ty", + "snap back", + "l ds", + "al arms", + "mat te", + "âĺ Ķï¸ı", + "conditi oner", + "d ors", + "he x", + "fi zz", + "a stri", + "sus sex", + "secur ity", + "qa eda", + "all star", + "cocac ola", + "as one", + "cl icks", + "sc ans", + "mu te", + "he avier", + "ðŁİ §", + "âĺ ŀ", + "lv l", + "book boost", + "youtu be", + "fla shes", + "f jor", + "c su", + "explo de", + "do dge", + "cair n", + "gonz ales", + "th ill", + "pel le", + "hart ley", + "renew able", + "re tin", + "e stre", + "costar ica", + "shipy ard", + "nc fc", + "pri ya", + "a ghan", + "an ath", + "plu gin", + "co rey", + "re bound", + "or u", + "kat rin", + "hor mone", + "gi m", + "mahin dra", + "s sus", + "park land", + "har per", + "fanta stic", + "infer no", + "ep ilo", + "wrest ling", + "fe ct", + "c it", + "ac oun", + "to ssed", + "monu mental", + "char tered", + "bu st", + "pe tra", + "âĮ ļ", + "wildflower hour", + "sweat ers", + "* .", + "bl er", + "ate ch", + "go wan", + "demo graphic", + "bra l", + "suici de", + "renov ations", + "vu el", + "sin ister", + "ar mani", + "miso gy", + "ph arrell", + "nap s", + "un iting", + "crusad ers", + "cor gi", + "insu red", + "than i", + "no or", + "g q", + "d ada", + "bicy cles", + "snu ggle", + "sch an", + "ten berg", + "ss al", + "fe mme", + "bo il", + "½ ï¸ı", + "re ap", + "occur ring", + "hus sein", + "divi d", + "sto ke", + "sh alom", + "na ia", + "o lic", + "frustr ating", + "Ù ĩ", + "ig s", + "gro ver", + "scen arios", + "n ds", + "bru tality", + "med alli", + "bu on", + "sas s", + "skate boarding", + "ony x", + "lor ry", + "ny u", + "gau tam", + "mm ings", + "gu g", + "end i", + "lo thian", + "comm ando", + "chal k", + "ph ora", + "asse ssing", + "ti gh", + "crun chy", + "ad ay", + "is l", + "ci ara", + "pilgri ms", + "kam al", + "p to", + "brit anni", + "t ani", + "sm c", + "l ure", + "app store", + "ab y", + "golf ing", + "cl c", + "fa u", + "an as", + "shu tting", + "regul ated", + "carn age", + "scow boys", + "all enge", + "c ma", + "humbold t", + "rel le", + "ku mb", + "her i", + "refin ery", + "sound check", + "d wayne", + "bos nia", + "i sp", + "the alth", + "anni v", + "relev ance", + "my a", + "bag gage", + "dre ad", + "s bc", + "th ed", + "bu h", + "hi jab", + "lo id", + "ke w", + "c te", + "respec t", + "lovel ies", + "cu bes", + "celebr ate", + "dir t", + "sav ers", + "_ ,", + "gar ment", + "pulit zer", + "mas jid", + "beat port", + "al arts", + "encry ption", + "s ner", + "ple ads", + "found ry", + "sym metry", + "ru mi", + "birth place", + "scallo ps", + "supp le", + "pivo tal", + "t ati", + "no de", + "so d", + "pro xim", + "tr ics", + "col dest", + "bren t", + "mand u", + "cla ir", + "e ach", + "and alu", + "hi ddleston", + "ðŁIJ º", + "mel ts", + "v ance", + "pin n", + "se ments", + "scre ened", + "sa chs", + "o bl", + "ic ha", + "âĺĺ ï¸ı", + "school ers", + "heal ed", + "lo gged", + "ðŁ¤ĺ ðŁı¼", + "ic us", + "bore dom", + "b ish", + "b ffs", + "tal king", + "sure sh", + "hoo kem", + "de on", + "de fl", + "ei leen", + "ðŁį ķ", + "women intech", + "ri sotto", + "rang er", + "adverti se", + "ภģà¸", + "tel ly", + "la go", + "dart moor", + "d ong", + "sk ates", + "lo go", + "un ner", + "mail box", + "ma sala", + "lo oooo", + "amethy st", + "che wing", + "c bb", + "australi ans", + "rc mp", + "game art", + "# ...", + "kor n", + "extre mism", + "fruit ful", + "anci ent", + "pu bg", + "pol ite", + "wh it", + "mur als", + "m gr", + "line man", + "dav ao", + "ste ms", + "ten nis", + "av age", + "tu pac", + "gigan tic", + "hs bc", + "auto biography", + "up the", + "ี à¹Ī", + "re gal", + "fig uring", + "ku l", + "mis sy", + "hoo p", + "gra s", + "for ums", + "back lash", + "abduc ted", + "p nw", + "min ic", + "bu tt", + "bott oms", + "at on", + "ven g", + "ðŁĮ ı", + "del aney", + "prab hu", + "fan club", + "over haul", + "health ye", + "sy no", + "aa f", + "ren amed", + "kim i", + "un cle", + "man city", + "se u", + "qu anti", + "este em", + "um in", + "en zo", + "mel vin", + "under go", + "j har", + "far ah", + "coast ers", + "humph rey", + "mh z", + "children s", + "^ .", + "d hi", + "disrup tive", + "integr ating", + "r nb", + "over sized", + "a ide", + "ne au", + "docu mentation", + "ðŁijĢ ðŁijĢ", + "pal o", + "hear th", + "ri yad", + "pun ctu", + "abc news", + "secu res", + "boy band", + "bir ch", + "ju co", + "tra ff", + "legislat ors", + "bay a", + "ãĤ ¯", + "no ises", + "collec ts", + "s warm", + "k ner", + "bi shops", + "stur geon", + "snapp ing", + "mo l", + "fre aky", + "chair person", + "tro p", + "lyn ch", + "car cin", + "art sy", + "e sto", + "cha i", + "fl ur", + "inv ali", + "sau sages", + "im el", + "j or", + "fun fact", + "wit ter", + "puni shed", + "ac ons", + "h ya", + "re versi", + "em c", + "dif fu", + "z x", + "sp aw", + "cla d", + "d mit", + "hol land", + "fre sco", + "pay roll", + "ab undant", + "stu ffing", + "mor o", + "c ny", + "boy cott", + "wend y", + "ele ven", + "pro voc", + "pil ot", + "tr x", + "be ad", + "climate action", + "ri on", + "assi e", + "ì ĸ", + "o sm", + "islam ic", + "ho ar", + "good reads", + "al ici", + "afterno ons", + "spoke sman", + "jo lie", + "it as", + "masc ara", + "âĻ© âĻ«", + "pre vail", + "beetro ot", + "lu jah", + "k li", + "dod ger", + " »", + "ru le", + "l n", + "scre am", + "ho bart", + "col bert", + "r tc", + "er m", + "pat ro", + "quo ting", + "s live", + "que st", + "non fiction", + "semin ary", + "prosecu tors", + "ve st", + "express way", + "g ge", + "nau tical", + "et f", + "ðŁİīðŁİ Ĭ", + "dur ation", + "cha ired", + "the film", + "fab io", + "she h", + "can o", + "ðŁĴª ðŁı»", + "with draw", + "! :)", + "cor pus", + "phen om", + "yel p", + "la wn", + "ent om", + "snapp er", + "but te", + "pin ball", + "pro xy", + "libr e", + "alle vi", + "n ada", + "gabri el", + "fo wl", + "eure ka", + "daph ne", + "tu nes", + "pun ched", + "wh ore", + "jo g", + "ren tial", + "man ners", + "o pe", + "wh ufc", + "gu th", + "revol t", + "sne aker", + "philharmon ic", + "ho ste", + "sovereign ty", + "ðŁĻıðŁĻı ðŁĻı", + "fish ing", + "sci art", + "fe ta", + "i pp", + "dump ing", + "kel own", + "gir i", + "dig its", + "sal u", + "san jay", + "twee ters", + "sp as", + "col chester", + "sc ab", + "ma dd", + "๠Ħà¸", + "Ä ĩ", + "ged don", + "march for", + "do p", + "maure en", + "un plugged", + "di do", + "fashion blogger", + "up a", + "mex ic", + "tar y", + "pol ye", + "jame son", + "v t", + "grin der", + "mad dy", + "consult ancy", + "¬ ë", + "leagueof legends", + "ac cents", + "um ni", + "jane iro", + "tu ss", + "h ens", + "ampli fier", + "to shi", + "pret tier", + "pre vents", + "new town", + "red wood", + "vant age", + "ball ard", + "ar tof", + "a she", + "a sion", + "lac ey", + "ap at", + "gro ve", + "ภĦ", + "rw and", + "real tors", + "tra itor", + "bed ding", + "ö r", + "zi on", + "fla shing", + "cam pan", + "boom er", + "secretari at", + "ab ol", + "liti gation", + "cont amination", + "se dly", + "shred ded", + "in for", + "do herty", + "bench mark", + "ro che", + "skate board", + "sho vel", + "i zz", + "to pper", + "o ster", + "laby rin", + "autu m", + "k ong", + "hum mus", + "vi z", + "tech news", + "kla us", + "am using", + "socialmedi amarketing", + "i des", + "cast ell", + "ste e", + "underestim ate", + "cal ab", + "pa ign", + "b illing", + "unanim ously", + "g mb", + "fly fishing", + "hath away", + "commerci al", + "colour ing", + "skul ls", + "pivo t", + "te p", + "tb c", + "motor way", + "x press", + "construc tive", + "pu k", + "under lying", + "kir sten", + "mani ac", + "cha o", + "se ma", + "chiff on", + "ðŁijĮ ðŁı»", + "ver ona", + "kom o", + "stan doff", + "wi ped", + "c ated", + "bla ir", + "wor kin", + "m sc", + "bethle hem", + "swi pe", + "unexpe c", + "pe es", + "pe tri", + "orig ami", + "ðŁij ħ", + "mex ico", + "flav or", + "ru dd", + "cannab is", + "mar u", + "ri ddle", + "wor shi", + "sil on", + "sch at", + "ap se", + "tang er", + "bi ous", + "e er", + "questi oned", + "o zar", + "dan k", + "angle sey", + "char an", + "bak u", + "compe ten", + "re pri", + "bat ter", + "sa xon", + "cal ves", + "leng ths", + "$ $$", + "âŀ ¡ï¸ı", + "immer sion", + "ga unt", + "car ry", + "cy to", + "b anda", + "shu tt", + "experi ence", + "el gin", + "mous se", + "ta z", + "ê µ", + "in correct", + "en z", + "b ham", + "mor on", + "so ver", + "ar un", + "ti pped", + "la ble", + "de arly", + "bau tista", + "í Ļ", + "mor tal", + "woo p", + "dt la", + "sho cks", + "dav os", + "ðŁĵ Ŀ", + "swim wear", + "her man", + "ðŁijĩ ðŁijĩ", + "z ir", + "neglec ted", + "grac ed", + "campu ses", + "av s", + "ar ora", + "swach hb", + "live pd", + "ac cra", + "enqui ries", + "shoo ters", + "kur t", + "vancou ver", + "brad ley", + "gar da", + "g ü", + "ol la", + "attrac ting", + "up ton", + "ne win", + "lu mia", + "furn ace", + "ev ers", + "e on", + "sw a", + "roo kies", + "a oc", + "v ss", + "bris ket", + "tor ch", + "yo da", + "heart land", + "tac o", + "ph ony", + "food bank", + "ab bey", + "bab ylon", + "u y", + "gre ate", + "expre sses", + "d andy", + "sc apes", + "survi vor", + "ron d", + "e ci", + "ha vin", + "ab el", + "chil dish", + "tor que", + "wav y", + "ur self", + "kanye west", + "year of", + "ale stine", + "o brien", + "al fon", + "sk ag", + "kore an", + "anchor age", + "val eri", + "de w", + "ðŁİ ¨", + "land slide", + "car ole", + "christ en", + "go phers", + "af i", + "priyan ka", + "q q", + "power of", + "it te", + "pc so", + "tw ol", + "pr y", + "intellec tu", + "guer rero", + "pi les", + "wish list", + "w ren", + "time table", + "ë ı", + "prodi gy", + "gibb ons", + ". /", + "ne ur", + "anz ac", + "mur ray", + "vie st", + "pla ster", + "la ir", + "art gallery", + "inter continental", + "g br", + "bell ator", + "nam joon", + "mam mals", + "am el", + "y aw", + "saras ota", + "cam ar", + "bud ding", + "sum mari", + "aco sta", + "la sh", + "ey ou", + "post graduate", + "instruc tors", + "ti g", + "const ant", + "were wolf", + "ic os", + "cla s", + "glen n", + "bud ge", + "ðŁĻ Ĥ", + "er ta", + "sta ins", + "persecu tion", + "cumb ri", + "o ch", + "syner gy", + "hu ang", + "scand in", + "mid terms", + "comment ator", + "regar ded", + "perpe tual", + "bo iling", + "al p", + "lan ge", + "sch le", + "fac eli", + "twee ta", + "ri dden", + "ok toberfest", + "charlotte sville", + "ik lan", + "jo u", + "ch atham", + "b sc", + "ðŁį ¦", + "stra uss", + "mel low", + "xx xx", + "happy hour", + "re actor", + "ww er", + "distr action", + "at orial", + "ðŁĴª ðŁı¼", + "twin peaks", + "fay ette", + "a or", + "ko k", + "bro om", + "sy fy", + "ou se", + "am ag", + "Ø ·", + "ubis oft", + "lu lu", + "hall mark", + "stu art", + "it ya", + "si deline", + "venge ance", + "re lu", + "sex ism", + "boun cing", + "un ites", + "gu stav", + "te ssa", + "stu mp", + "pro clamation", + "ima x", + "divid end", + "col by", + "ðŁį İ", + "play wright", + "un safe", + "co smo", + "ðŁĩ²ðŁĩ ½", + "cup board", + "constitu ents", + "ang lia", + "ram page", + "ðŁĺįðŁĺį ðŁĺįðŁĺįðŁĺį", + "than ked", + "take aways", + "shro ff", + "de bat", + "kh ur", + "conduc ts", + "format s", + "à ©", + "port age", + "graph ers", + "u ten", + "pre m", + "mo ines", + "condem ns", + "s ous", + "l ps", + "f cs", + "deal ership", + "leuke mia", + "bure au", + "ski d", + "guardi ola", + "ca ster", + "thir d", + "avoi ded", + "en cyclo", + "c sr", + "vi xx", + "analy zing", + "she ar", + "dulu th", + "shap iro", + "chan ting", + "stre sses", + "as be", + "mil itia", + "ãĥ ª", + "col lin", + "arsen e", + "sure sh", + "teach ings", + "yi xing", + "sh ill", + "nu des", + "sv u", + "clear water", + "war ped", + "pro life", + "artist son", + "it u", + "versail les", + "galax y", + "ax el", + "spring st", + "cal a", + "hu hu", + "sc u", + "commit ments", + "exe ter", + "poign ant", + "mo tion", + "conserv atory", + "row dy", + "rec alled", + "mu sk", + "emb elli", + "so the", + "âĺ Ģ", + "sto pper", + "sch ild", + "to pe", + "el mo", + "zi el", + "j om", + "barn sley", + "snow den", + "on tour", + "jour ney", + "hills borough", + "par ole", + "w ts", + "mo ving", + "ag ility", + "tiv o", + "ff ers", + "kindle unlimited", + "g wen", + "ann an", + "ah mad", + "tex tured", + "hepat itis", + "dra m", + "insi ders", + "tis sues", + "ãĥ Ħ", + "fc barcelona", + "cr atic", + "na acp", + "pe can", + "f gm", + "custom ize", + "concer t", + "g sm", + "pe g", + "p one", + "justin trudeau", + "super cars", + "happy holidays", + "bu lar", + "ado x", + "lap tops", + "digital health", + "destin ation", + "gradu ally", + "áĥ ¦", + "popp y", + "ss l", + "inhi bit", + "star light", + "of fro", + "glo omy", + "x per", + "hal der", + "im plants", + "le to", + "hass el", + "a as", + "un told", + "en ci", + "liber ia", + "or an", + "con tests", + "il ah", + "sma g", + "sc out", + "mari anne", + "cr yo", + "schedu ling", + "lo s", + "kan e", + "stutt gart", + "ne se", + "law rence", + "da in", + "pho tom", + "car ou", + "ภ£", + "g wy", + "national dogday", + "roa sting", + "band camp", + "kentu cky", + "stret ches", + "ke rel", + "ca she", + "ãĤ ¸", + "sta x", + "tran si", + "dog gie", + "at ric", + "hal le", + "ci vic", + "brow ning", + "lein ster", + "cat day", + "high land", + "joy ous", + "in cumb", + "or lando", + "ro mo", + "col ton", + "del ta", + "car ab", + "ro tc", + "aster oid", + "goose bumps", + "mo logy", + "yo ko", + "an ds", + "tomor rows", + "red carpet", + "sm p", + "ca sio", + "ðŁ¤£ðŁ¤£ ðŁ¤£", + "se au", + "rejec tion", + "rot ating", + "bi partisan", + "th un", + "mat i", + "bon i", + "ol l", + "ener gye", + "do it", + "l j", + "mother hood", + "lou ise", + "neck laces", + "el ite", + "ni x", + "l cs", + "en v", + "gl u", + "le sh", + "cran k", + "su sie", + "m clau", + "so tu", + "crow ley", + "rat ri", + "use d", + "bre ton", + "alfre do", + "ye o", + "travel pics", + "ti pp", + "elli son", + "sax ophone", + "me red", + "heu ghan", + "ta ine", + "f es", + "vi ro", + "suppo sedly", + "i as", + "dige stive", + "y le", + "li zzy", + "wildlife photography", + "bri anna", + "west field", + "ra ined", + "am her", + "ðŁĺĦ ðŁĺĦ", + "distribu te", + "bott om", + "pre serving", + "oil and", + "craf ty", + "de scen", + "col ling", + "shakespeare sunday", + "r wc", + "ang led", + "ci an", + "t ations", + "mon tage", + "me yers", + "france sca", + "ðŁĮ ·", + "wi ggins", + "san ford", + "volunte er", + "car ra", + "bar k", + "vari ed", + "pl in", + "am u", + "kap il", + "rock ers", + "qu ind", + "br ane", + "in mate", + "ent al", + "impro vis", + "michi gan", + "re tweeting", + "progre ssing", + "mercedes benz", + "smo ker", + "physi ology", + "dor ado", + "watt pad", + "h wa", + "sr bachchan", + "w ga", + "vol atility", + "hi re", + "ac ap", + "wn ba", + "hein z", + "stit ches", + "kidnapp ing", + "bur ys", + "lim b", + "f itters", + "thumb nail", + "ton e", + "mir and", + "desi rable", + "ad dison", + "tar an", + "tamil nadu", + "spec tator", + "soci ology", + "amit shah", + "remo tely", + "âĻ ¦", + "ham id", + "r ds", + "g lee", + "smooth ly", + "sch ro", + "er c", + "lali ga", + "he als", + "us f", + "ni shi", + "d hu", + "un il", + "h le", + "tro mb", + "bhu tan", + "pilip inas", + "se ung", + "whit man", + "te y", + "min ce", + "snow boarding", + "re au", + "k ker", + "av o", + "zach ary", + "ran veer", + "ti k", + "gover n", + "qu al", + "beck y", + "anthropo logy", + "att en", + "grocer ies", + "de bit", + "war p", + "sil icon", + "hawa ii", + "ðŁĴ ħ", + "pomegran ate", + "pe er", + "orang es", + "people schoice", + "end ure", + "ðŁĴĽ ðŁĴĽ", + "ãĤ¹ ãĥ", + "ac ial", + "a haha", + "stu k", + "imper ial", + "bl ond", + "pow der", + "kno ts", + "vin ce", + "wood lands", + "den a", + "watch in", + "mat cha", + "ma hat", + "galax ies", + "middles brough", + "k ö", + "stre e", + "resc ues", + "wal do", + "lero y", + "desp ic", + "real ities", + "tm nt", + "ha q", + "un o", + "pe c", + "bolly wood", + "blin ds", + "design thinking", + "he ms", + "and hra", + "ab sen", + "fan s", + "ste ch", + "shire hour", + "bla ine", + "shak ti", + "pu rely", + "ðŁı ı", + "tra fal", + "ke ynes", + "gr ate", + "to bias", + "spon taneous", + "satur ated", + "caval ry", + "pri sc", + "ðŁĺ ij", + "wh t", + "pas si", + "~~ ~", + "vir at", + "patt inson", + "la o", + "weir do", + "sym pathy", + "ju da", + "occa sionally", + "cred ited", + "stat u", + "es co", + "hil ly", + "esc ape", + "dischar ge", + "se er", + "may nard", + "sud bury", + "z lat", + "or al", + "we er", + "encoun tered", + "sm elling", + "over sight", + "ê ¸", + "that cher", + "mack ay", + "you can", + "fre ep", + "freed oms", + "prophe cy", + "ho e", + "ishq ba", + "dra ke", + "qu its", + "pel led", + "tur k", + "o vi", + "wesle yan", + "new music", + "leg g", + "ch eng", + "h illi", + "ay y", + "pan ties", + "ad versity", + "ad jac", + "vaccin ation", + "ju ke", + "ga c", + "exce ed", + "time sof", + "sta ining", + "ep cot", + "v ital", + "up ward", + "bethe sda", + "apar k", + "ma hi", + "camp fire", + "enchan ting", + "rha pso", + "h z", + "na ver", + "fa x", + "vali dation", + "ac ad", + "ny r", + "as ym", + "coordin ated", + "depar ted", + "all ery", + "var ies", + "spr ite", + "chap lin", + "ss occer", + "s wat", + "bre t", + "relu ct", + "tunes app", + "super star", + "reminis cing", + "o co", + "home grown", + "dough nut", + "un canny", + "la pd", + "thyro id", + "! âĿ¤ï¸ı", + "botan ic", + "bre s", + "sp ade", + "i ste", + "echo es", + "du lil", + "bur sting", + "qui ero", + "ðŁij İ", + "loy ola", + "amuse ment", + "ha ils", + "sleep y", + "burgl ary", + "âľ ı", + "ro gue", + "cot land", + "mo ors", + "low er", + "wic ked", + "ðŁĶ Ĭ", + "compet iti", + "argent ine", + "yvon ne", + "karti keyan", + "ili ary", + "gat sby", + "precin ct", + "six ty", + "na ji", + "cam s", + "practiti oner", + "ðŁĺ³ ðŁĺ³", + "pu ne", + "neg li", + "juli en", + "inv aded", + "cali br", + "cla m", + "duba i", + "mu k", + "lan tic", + "produc t", + "fe dex", + "ï¸ı :", + "eu ra", + "dari us", + "s ling", + "virtual reality", + "home stead", + "ðŁı³ï¸ıâĢį ðŁĮĪ", + "pac ed", + "in ha", + "pul mon", + "la zy", + "premi ering", + "ma stered", + "in he", + "con gregation", + "ba jo", + "sport ing", + "new jersey", + "hor ny", + "lma oo", + "leng thy", + "du t", + "yo gh", + "swe aring", + "philosoph ical", + "pap ua", + "in ski", + "know les", + "dy ke", + "âĢ ²", + "to ken", + "mc guire", + "ri ot", + "probab ility", + "mc con", + "gro s", + "su mat", + "c ite", + "da a", + "on da", + "mad dow", + "che w", + "board games", + "spar ked", + "re claimed", + "ad hd", + "ny se", + "imwith her", + "equ inox", + "boo ths", + "balsam ic", + "ha zy", + "dor chester", + "ag os", + "se aw", + "moder ator", + "seri ea", + "ander sen", + "pilgri m", + "âŃIJ âŃIJ", + "itch en", + "hal li", + "x ton", + "nathan iel", + "mun ition", + "celesti al", + "ga f", + "zo om", + "mark le", + "pen thouse", + "cal e", + "s fa", + "bar king", + "tu cket", + "em ery", + "cal orie", + "li que", + "ad ar", + "mc nam", + "tor tilla", + "wood pecker", + "mo town", + "bad ger", + "ayr shire", + "scram ble", + "dd ay", + "cra ziest", + "per rie", + "cho co", + "cast e", + "i ot", + "wre cked", + "selec ting", + "uss r", + "gra ft", + "pun t", + "lab ou", + "ir st", + "ba ek", + "Û Į", + "su ki", + "que u", + "ach at", + "te ster", + "aug mented", + "wc vb", + "sin ks", + "ðŁĵ »", + "ra ke", + "inter ne", + "be cause", + "belle vue", + "une arth", + "light en", + "ðŁĺ £", + "turn around", + "labe led", + "unemp loyed", + "twitter kurds", + "le ia", + "h ye", + "great er", + "ðŁIJ İ", + "tim ed", + "i red", + "e tt", + "limit ations", + "cab e", + "s out", + "bee ch", + "anni hil", + "re trac", + "yo ona", + "ang er", + "den nis", + "supp lying", + "di z", + "\" (", + "sc ur", + "gun man", + "su ho", + "sauvi gnon", + "ภ¥", + "wi ley", + "land on", + "choreo graphy", + "pre historic", + "ðŁı ĥ", + "var gas", + "assess ments", + "pinn acle", + "di i", + "chamber lain", + "ì Ī", + "v p", + "present ers", + "deut sche", + "sun shine", + "sal utes", + "r one", + "bu siest", + "- .-", + "motor ists", + "hemi sphere", + "al wx", + "ps p", + "ow a", + "den ying", + "cho c", + "gu tier", + "han uk", + "mus kete", + "jait ley", + "se wage", + "t ame", + "thin kers", + "shi m", + "se quo", + "pap ar", + "middle east", + "k wa", + "ke g", + "patag onia", + "no y", + "bar ça", + "take off", + "he a", + "à ¬", + "n sc", + "g dc", + "ðŁij Ī", + "mou stache", + "mel ania", + "thr a", + "â¬Ĩ ï¸ı", + "pier ced", + "ze us", + "fon ts", + "ber a", + "it iner", + "q atar", + "contr ary", + "ire land", + "i fy", + "ou los", + "commun al", + "fin s", + "un paid", + "pa a", + "ðŁijĩ ðŁı»", + "ri os", + "ou p", + "f iller", + "cafe teria", + "ภŃ", + "kas i", + "cali ber", + "z ulu", + "v sco", + "ts ford", + "dragon fly", + "smo kin", + "pi st", + "psycho logist", + "diplom at", + "we bs", + "buc cane", + "à® ¾", + "motiv ational", + "du ne", + "ba e", + "c fs", + "with out", + "er on", + "i ac", + "ate e", + "pen sion", + "fra zier", + "en sis", + "sk is", + "par ting", + "ger y", + "territ ories", + "nach os", + "eni ght", + "ever lasting", + "msd honi", + "tel e", + "sp un", + "po di", + "sab ah", + "environ mentally", + "ce ase", + "beau mont", + "mar ta", + "kel vin", + "ho ff", + "sun il", + "n da", + "co b", + "sh ale", + "ree dus", + "un boxing", + "u bio", + "re opened", + "n all", + "capsu les", + "mar r", + "himalay as", + "swee ter", + "ja z", + "f mr", + "twee ter", + "dha ka", + "na u", + "de mi", + "d fs", + "ta urus", + "fad ing", + "it utes", + "ci p", + "over flow", + "jef frey", + "don ny", + "car tunesapp", + "ðŁį ij", + "prefe cture", + "danc ed", + "c pt", + "ple asing", + "ital k", + "earth quakes", + "ul ation", + "hi o", + "ãĢ ĭ", + "ant an", + "nutri ent", + "de ere", + "selec ts", + "enrich ment", + "r iti", + "tram pol", + "bl amed", + "j ia", + "contribu tors", + "chesa peake", + "pi geons", + "tribun al", + "mad uro", + "w su", + "ilo ve", + "effici ently", + "dar cy", + "war ms", + "ar ra", + "ec u", + "ho wer", + "strugg led", + "rajini kanth", + "ðŁĺ¢ ðŁĺ¢", + "hou sing", + "str at", + "eli x", + "disp ro", + "raf fic", + "thi erry", + "na sty", + "c fb", + "staf fing", + "al ma", + "back ers", + "hen son", + "sky walker", + "reale state", + "roo s", + "ness y", + "chan ce", + "cair ns", + "c ci", + "pe dal", + "ly ft", + "cross word", + "wait er", + "only in", + "kru ger", + "k ir", + "alej andro", + "car tier", + "car rera", + "re paired", + "ou at", + "un clear", + "un breakable", + "today in", + "qu eries", + "jo dy", + "gen ital", + "win ner", + "to l", + "kelown a", + "fascin ated", + "ãĥ ¬", + "sris ri", + "squ ared", + "spr ung", + "negoti ate", + "priv ately", + "av en", + ">> >>>", + "g ical", + "gav in", + "chester field", + "zu mba", + "or r", + "nat alia", + "impeach ment", + "mn l", + "car at", + "criti que", + "credi ble", + "trac y", + "tan i", + "musi k", + "jig saw", + "gam bia", + "tol kien", + "fe u", + "as per", + "sav ory", + "fo xx", + "f itt", + "mar lon", + "l rt", + "v ell", + "p br", + "imprison ed", + "i om", + "chu l", + "wind shield", + "kay e", + "ba a", + "chor d", + "s art", + "al gon", + "minister ial", + "nat geo", + "la zio", + "nor ms", + "ðŁijį ðŁijį", + "lic king", + "fut bol", + "un sung", + "dalla scowboys", + "sh red", + "distur b", + "dev ine", + "be ards", + "ch f", + "b day", + "ro sso", + "ig or", + "ay i", + "si ren", + "k air", + "sti les", + "ro f", + "mag nets", + "un cover", + "mou se", + "bang ing", + "si ghted", + "spe ople", + "impac t", + "row land", + "kir a", + "environ ment", + "love the", + "p sis", + "mish ra", + "gl endale", + "ca jun", + "o che", + "de ception", + "sex ist", + "stra ws", + "s ga", + "buff er", + "apost le", + "sp l", + "pop up", + "ðŁļ Ĺ", + "r g", + "up er", + "ball in", + "i dy", + "occa sional", + "national park", + "ðŁı Ĭ", + "u an", + "innov ation", + "ภ«", + "te aparty", + "re tte", + "counter fe", + "b ha", + "rec s", + "ig en", + "ðŁĮ IJ", + "humming bird", + "cu r", + "ha ven", + "la zar", + "pue blo", + ": :", + "zi onist", + "op ath", + "inver ness", + "promo ter", + "carto on", + "cabine ts", + "mahog any", + "surve ying", + "r ational", + "feel ing", + "testi fy", + "so w", + "oc on", + "ภ¢", + "ne el", + "mar is", + "sol itary", + "che mo", + "rad cliffe", + "sim ons", + "ros ary", + "new er", + "jo die", + "re tali", + "pra wn", + "pad dy", + "hen ge", + "k ala", + "im plant", + "at y", + "bren twood", + "par adox", + "ene z", + "re designed", + "p our", + "wy d", + "al de", + "௠ģ", + "sol d", + "biomed ical", + "๠Ĥ", + "tt tt", + "mat teo", + "ys er", + "new ton", + "de bun", + "ner dy", + "loo l", + "wo on", + "elisa beth", + "ec c", + "wh i", + "ach o", + "salv age", + "sal aries", + "qu ity", + "navig ating", + "oph thal", + "con soles", + "re built", + "o pec", + "ast ers", + "sho red", + "set list", + "kathr yn", + "rhy mes", + "re visiting", + "ash ish", + "li ft", + "re post", + "sole il", + "âı ±", + "weal th", + "sa at", + "we c", + "king james", + "flipk art", + "field work", + "se gu", + "mo dal", + "bu b", + "are rs", + "ðŁį Ĵ", + "clo oney", + "pad dington", + "necess ity", + "guth rie", + "pen te", + "li mo", + "jo sie", + "ar tin", + "en c", + "l hs", + "betra yal", + "info graphics", + "i er", + "mo a", + "hear ings", + "bon jour", + "sym bolic", + "ag ro", + "wed ges", + "krist ina", + "wild flower", + "athle tic", + "photograph y", + "pe sh", + "ca hill", + "chi lean", + "gou l", + "fi oren", + "ðŁij ¶", + "z il", + "sk im", + "bad oo", + "deli a", + "tre ble", + "n cc", + "ðŁĩ¦ ðŁĩ", + "a house", + "bul lock", + "sol itude", + "ا٠Ĩ", + "can cers", + "futureof work", + "hu tch", + "water shed", + "war mongers", + "sp illed", + "colom bo", + "mo th", + "associ ations", + "weigh ed", + "global goals", + "not just", + "christ i", + "tor g", + "swe ating", + "man eu", + "clu sters", + "âĢ¼ï¸ı âĢ¼ï¸ı", + "ta ped", + "ul y", + "tru sting", + "yu suf", + "te in", + "ra b", + ", ,,,", + "sin ai", + "audi ble", + "explic it", + "cro wns", + "sch iz", + "at least", + "ðŁĹ £", + "de bra", + "je suit", + "ene gger", + "z hen", + "one sie", + "i it", + "ss f", + "gur gaon", + "chak ra", + "bear cats", + "k ran", + "k awa", + "reque sting", + "han over", + "g end", + "sor os", + "mer cy", + "lovel y", + "do omed", + "tim my", + "ku z", + "ul l", + "ab ram", + "sa ison", + "ãĥ «", + "clean ers", + "re mo", + "circu its", + "bar red", + "o th", + "mo ist", + "madele ine", + "gall o", + "u j", + "per mits", + "hea viest", + "car ols", + "az te", + "gior gio", + "flo ats", + "decl aring", + "us rc", + "min at", + "craf ts", + "pri ma", + "conven i", + "nickelo deon", + "danc ing", + "ceremon ial", + "blo gg", + "tw p", + "anglic an", + "she k", + "k nick", + "( ((", + "hubb ard", + "harve y", + "hit man", + "fen g", + "we some", + "for za", + "s word", + "op us", + "bro m", + "gi bility", + "z al", + "m unch", + "dance hall", + "gre edy", + "hd mi", + "re birth", + "ðŁĺĭ ðŁĺĭ", + "s world", + "figur ine", + "com post", + "k f", + "engra ving", + "gior no", + "st ana", + "k man", + "ham ster", + "compos ers", + "aj e", + "func tionality", + "pol k", + "is ons", + "air planes", + "te se", + "hor rors", + "musc at", + "gi ven", + "sp ence", + "ðŁĩ¸ ðŁĩ", + "eli ot", + "ach illes", + "fre ck", + "crypto currencies", + "sou ther", + "hal o", + "bor neo", + "polit ic", + "hahahaha h", + "up state", + "si ena", + "obsc ure", + "hau sen", + "lloy d", + "happy friday", + "motor bike", + "bon a", + "americ as", + "hol s", + "- (", + "spor ty", + "un aware", + "reven ues", + "christop her", + "bank sy", + "av an", + "ev apor", + "com press", + "eyel iner", + "to dos", + "buff y", + "renewable energy", + "ly rical", + "ar chan", + "rapi st", + "fair trade", + "lma ooo", + "beat z", + "pro active", + "la pse", + "ir ical", + "revers al", + "po de", + "mcin tyre", + "mac au", + "ãĥ ķãĤ", + "nash grier", + "f sa", + "g all", + "çĶ Ł", + "perpe tr", + "il ya", + "configur ation", + "% ;", + "str ange", + "rac i", + "ภĩ", + "pic kups", + "kov sky", + "mam mal", + "w ps", + "g able", + "compar ative", + "z h", + "save our", + "da vey", + "on etsy", + "mu ssels", + "mis er", + "cri stina", + "electr on", + "cra ve", + "lo ren", + "precipit ation", + "m z", + "ðŁį «", + "vin cen", + "snow board", + "no ida", + "ah n", + "marin ated", + "g tr", + "town hall", + "min is", + "bethe l", + "adv an", + "su ra", + "shi el", + "fur ry", + "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤ", + "lyn d", + "so il", + "sc ence", + "sen eca", + "shar jah", + "dick ens", + "credenti als", + "av ar", + "per k", + "requ iring", + "pre fer", + "j ian", + "de ca", + "r ach", + "ing for", + "del e", + "be ep", + "ðŁĴ »", + "cis ely", + "hu ddle", + "green sboro", + "haw king", + "ho ax", + "hang ar", + "ç ľ", + "mis o", + "lo vin", + "gre ta", + "ab ad", + "logi e", + "at an", + "snow flake", + "mahe sh", + "fear the", + "al kal", + "bobb lehead", + "ba hn", + "ju dged", + "fu tu", + "feli x", + "ðŁį ĵ", + "pi ke", + "der iv", + "notic es", + "au er", + "dis super", + "or da", + "wi pes", + "am ino", + "stri kers", + "foo tb", + "dram as", + "pun ching", + "score less", + "heming way", + "bi h", + "bal lad", + "chat ter", + "am mo", + "kle in", + "fabric ation", + "kari m", + "z end", + "hi sto", + "vol ta", + "rock y", + "marke ter", + "xtre me", + "sequ encing", + "paradig m", + "cle ats", + "boom ing", + "âģł âģł", + "block ade", + "promp ts", + "yogh urt", + "pur pose", + "nu r", + "regu late", + "nois y", + "ing rid", + "bird watching", + "bar tender", + "Ù ĥ", + "wor dof", + "cha otic", + "shor ty", + "el dest", + "z app", + "onceupon atime", + "fl yo", + "rit os", + "mike quind", + "ðŁIJ ´", + "regi stering", + ". ]", + "ad ol", + "gg gg", + "pur ge", + "kid lit", + "ar bor", + "val ves", + "synago gue", + "o th", + "unanim ous", + "veri fication", + "dar rell", + "ãģ Ħ", + "vander bilt", + "tape stry", + "pro sper", + "did dy", + "dra fting", + "de cep", + "marqu is", + "st int", + "michael jackson", + "pee led", + "men us", + "bb b", + "sc are", + "ema il", + "wri gley", + "it is", + "f ell", + "some thin", + "bar ra", + "ed gar", + "di pping", + "pu ddle", + "sla de", + "lear ner", + "jal en", + "ðŁ§ IJ", + "the daily", + "mikequind azzi", + "ju x", + "iq bal", + "mckin ney", + "ra iser", + "ef an", + "dr one", + "cat o", + "pic ket", + "cro we", + "l att", + "uk o", + "giuse ppe", + "hin i", + "synthe si", + "ponti fex", + "song writing", + "to d", + "swit ches", + "din ners", + "h q", + "gabri elle", + "pensac ola", + "cir cle", + "expo ses", + "ev s", + "riyad h", + "pro men", + "o ck", + "sa j", + "cit ation", + "brew co", + "jo si", + "ep aper", + "dri f", + "point less", + "tang led", + "cri pp", + "line ups", + "fairi es", + "daz e", + "mour n", + "bla dder", + "sal z", + "bur undi", + "book mark", + "the people", + "sub sequ", + "princi pal", + "sk er", + "court ney", + "a oki", + "rac ers", + "ad m", + "mom a", + "critical role", + "hou n", + "shed ding", + "sa ka", + "ace ous", + "mck ay", + "hus bands", + " ½", + "me da", + "accu sations", + "ro sel", + "nc is", + "witne ssing", + "or ama", + "go ds", + "hil ton", + "el man", + "ÃŃ n", + "meg ap", + "cra ven", + "announ cer", + "crit eri", + "sheffiel dissuper", + "milit ant", + "consu l", + "hoo ded", + "aby ss", + "b x", + "ma dam", + "lo cu", + "mary am", + "manic ure", + "grat is", + "ac tresses", + "ros ario", + "this dayin", + "king ly", + "gn ome", + "cel ine", + "r ous", + "he el", + "lil ac", + "vish al", + "ab h", + "thor ns", + "s ls", + "ne al", + "construc ting", + "be ren", + "s lang", + "ma ins", + "far ra", + "sar ko", + "pai ge", + "gu iller", + "l ala", + "ice berg", + "nou n", + "plann ers", + "u mmm", + "ou ses", + "ill ary", + "ma an", + "box ing", + "zi pper", + "srin agar", + "migu el", + "o str", + "mp o", + "responsi bly", + "lan terns", + "appli ance", + "x b", + "gren ade", + "neglec t", + "dy sle", + "ham mock", + "ne ctar", + "wit cher", + "r gv", + "di ence", + "ser bian", + "seed ed", + "cru z", + "bi sh", + "sp he", + "e q", + "sky rim", + "alge bra", + "phil ately", + "bungal ow", + "ge off", + "y ves", + "demand ed", + "consider ations", + "the vamp", + "pawan kalyan", + "co ded", + "grit ty", + "erup tion", + "se infeld", + "uni denti", + "ëĭ Ī", + "wor m", + "ac us", + "se ung", + "dun g", + "ro land", + "su d", + "di visions", + "ab lanc", + "shor test", + "j f", + "p oun", + "plant based", + "be to", + "tough er", + "mc o", + "don et", + "mark us", + "v fl", + "ðŁı ł", + "open ing", + "co ward", + "caber net", + "o xi", + "burle sque", + "sand ra", + "su mo", + "consi st", + "tho t", + "cay man", + "motor ola", + "gutier rez", + "d slr", + "y w", + "no bel", + "nov ice", + "moms demand", + "grun ge", + "sp or", + "d cc", + "pre sses", + "sli st", + "allot ment", + "voc ational", + "ft c", + "pu ja", + "lo ven", + "utt arak", + "tan dem", + "sh ep", + "come dians", + "anat om", + "cant wait", + "healthye ating", + "west side", + "mar gins", + "chi ang", + "asbe stos", + "stupi dity", + "proble matic", + "fit bit", + ": $", + "ceil ings", + "shu a", + "protec tions", + "bio tic", + "beng ali", + "re sts", + "bien nale", + "tim o", + "cul min", + "e minent", + "affe ction", + "unbeliev ably", + "individu ally", + "canvas sing", + "wh itt", + "nov asco", + "chin son", + "h pe", + "go w", + "gloucester shire", + "pa o", + "thresh old", + "chev ron", + "s ine", + "we ther", + "pp ie", + "aqu ino", + "antwer p", + "âĸ ¬", + "po on", + "inst af", + "equ ine", + "cinemato graphy", + "nbaf inals", + "vali ant", + "kil kenny", + "te rence", + "syste mic", + "sr l", + "p ound", + "made ira", + "pl ough", + "tre cht", + "mat ed", + "mp d", + "ransom ware", + "ph in", + "li qui", + "bb ce", + "boom er", + "i standwith", + "con ju", + "r te", + "nar a", + "foo lish", + "da shing", + "vier nes", + "br ite", + "da u", + "juni per", + "ai da", + "you now", + "ra zer", + "de i", + "repe ating", + "comfor ting", + "adjac ent", + "e to", + "ca sted", + "chat ur", + "mu er", + "syn th", + "san itary", + "mac le", + "independ ent", + "law ful", + "e erie", + "h or", + "ðŁĴ Ń", + "am rit", + "vel o", + "station ery", + "mu f", + "may may", + "contempl ating", + "elabor ate", + "gre gor", + "dri es", + "ac col", + "ภļ", + "schwarz enegger", + "ill nesses", + "day break", + "follow back", + "collu sion", + "electr onic", + "jo vi", + "hiro shima", + "ta w", + "hom ec", + "mic ah", + "qu itting", + "fro sting", + "ben fica", + "hel i", + "s ical", + "pic cad", + "corpor ate", + "ment orship", + "you are", + "sing er", + "shi va", + "ru ne", + "ing er", + "ri um", + "play able", + "doo p", + "wil low", + "ter re", + "ni p", + "at d", + "war bler", + "profession ally", + "er ase", + "proce ed", + "pedestri ans", + "mis chief", + "ben ding", + "alas kan", + "c kett", + "mo p", + "dd les", + "shut ter", + "ge ared", + "atene o", + "ma deline", + "g ations", + "o sha", + "der ick", + "sw ild", + "an gry", + "pat ents", + "hun k", + "decre ased", + "fr y", + "ðŁĴĸðŁĴĸ ðŁĴĸ", + "sal on", + "quant ities", + "d ario", + "ni gel", + "ku ma", + "jen n", + "happ ye", + "xx x", + "rex perience", + "pro s", + "au sch", + "rele ssly", + "ham burger", + "fuku shima", + "er ne", + "stat ec", + "ren d", + "may field", + "j one", + "lef ty", + "bern stein", + "sm il", + "gener ates", + "fore station", + "band its", + "ta yo", + "r ca", + "ac ci", + "rodri go", + "kn app", + "elo vers", + "vege tation", + "u ral", + "le ft", + "ħ ï¸ı", + "worl dre", + "sur i", + "embar k", + "w son", + "ba you", + "mu ller", + "mo vers", + "ðŁķ º", + "presby ter", + "l f", + "cre e", + "bat b", + "sal am", + "demonstr ations", + "an ec", + "n pc", + "it ics", + "to graphy", + "re inst", + "thur st", + "tal e", + "off ences", + "smart city", + "bro tha", + "ofthe year", + "in valuable", + "ear n", + "ðŁijı ðŁı½", + "kre mlin", + "gra dy", + "town fc", + "guern sey", + "ma ha", + "contag ious", + "dre x", + "be en", + "( £", + "nati vity", + "k tm", + "somer halder", + "comp ounds", + "íķ ĺ", + "\" âĢ¦", + "af g", + "ott news", + "h ound", + "fire fly", + "cil an", + "donet sk", + "volunte ered", + "ak ira", + "è ª", + "sing ul", + "st h", + "dro wned", + "mand o", + "he ir", + "ðŁİīðŁİ Ī", + "tax is", + "y uki", + "vel d", + "k ans", + "el k", + "ran ts", + "hash tag", + "t eng", + "ro g", + "a at", + "gru b", + "e ber", + "in india", + "colo ssus", + "sig ni", + "so ever", + "mile stones", + "der o", + "differen tial", + "phu ket", + "master mind", + "an gh", + "mel ani", + "bro ker", + "actor vijay", + "stun ned", + "continu ity", + "af fl", + "vo cal", + "perenni al", + "fianc é", + "in complete", + "hun ts", + "re issue", + "domin ates", + "tur meric", + "ro am", + "ri on", + "bag ged", + "nas sau", + "fu t", + "x ox", + "national trust", + "jo ye", + "san o", + "hearth stone", + "dis respect", + "le es", + "h se", + "siber ian", + "offe e", + "re stock", + "wolf gang", + "re gan", + "plan o", + "un wind", + "re par", + "mil le", + "] ,", + "skul l", + "fat ally", + "concep tual", + "ðŁĮ ²", + "f é", + "ber to", + "b ms", + "u a", + "mag na", + "notre dame", + "le te", + "la undering", + "heartw arming", + "buffe tt", + "go at", + "pe abo", + "wind mill", + "v ac", + "continu ally", + "az alea", + "mem brane", + "can cels", + "make yourown", + "athe red", + "p to", + "tor pe", + "ðŁĺ ł", + "ðŁĴ §", + "sc ares", + "le aking", + "z et", + "pix els", + "ac i", + "kh il", + "marath i", + "ðŁĻı ðŁı½", + "u la", + "tam u", + "chandi garh", + "z agre", + "aa b", + "pronoun ced", + "aubre y", + "sand er", + "pun ta", + "har low", + "ic elan", + "celebr atory", + "so t", + "unci ation", + "stru ly", + "mc dowell", + "deepi ka", + "remin ders", + "my stical", + "ct c", + "chat ted", + "s ica", + "bar gains", + "ch hat", + "ru bin", + "m net", + "oiland gas", + "pel ican", + "o at", + "mor ality", + "k our", + "i h", + "nu clear", + "gc u", + "ric her", + "vene zia", + "m ma", + "le ith", + "ac company", + "rich mond", + "sports net", + "ba ahu", + "smu ggling", + "mm i", + "ðŁĩ®ðŁĩ ª", + "twi sts", + "sahi b", + ".... .", + "amb itions", + "il lo", + "histor ical", + "fo rec", + "show biz", + "pon ies", + "chas ers", + "remo del", + "will ing", + "prince sses", + "am ple", + "cushi ons", + "ac les", + "lot r", + "da ch", + "an the", + "in corporate", + "new bury", + "ki ri", + "fried rich", + "ab v", + "ball ers", + "alber t", + "ðŁij Ń", + "let i", + "nan op", + "ci de", + "anal o", + "n sf", + ")) ))", + "griffi ths", + "valen ci", + "ro ano", + "fun run", + "babys itting", + "ca day", + "ent re", + "u ck", + "slu g", + "tic al", + "the sims", + "ro ar", + "car ney", + "g am", + "sto we", + "fi d", + "bun ny", + "sham rock", + "pe cu", + "mol ina", + "go cougs", + "con tributes", + "transform ation", + "mo y", + "v aj", + "sever y", + "antioxid ants", + "thir teen", + "sight seeing", + "l j", + "reversi ble", + "odd ly", + "hoo kah", + "nou vel", + "hal al", + "fe i", + "stab les", + "mul t", + "ho pped", + "bra ids", + "inter change", + "ghana ian", + "ww ww", + "eth no", + "con junction", + "ago v", + "ye ti", + "earth and", + "ts p", + "con serve", + "heir loom", + "metaph or", + "woo f", + "tor io", + "self less", + "n wa", + "em ilia", + "yl ene", + "y xe", + "gi ar", + "moder ating", + "pro bz", + "b fi", + "ne er", + "du mmy", + "hanuk kah", + "we bber", + "k v", + "eye brow", + "dag ger", + "su mp", + "ra ges", + "ork ney", + "tb o", + "hal sey", + "assign ments", + "tr onic", + "scri b", + "co on", + "an war", + "# âĢİ", + "jal ape", + "flori da", + "qu aid", + "haw keyes", + "âĻ¡ âĻ¡", + "street car", + "ro g", + "dat lantic", + "gran ola", + "un changed", + "expect ation", + "Ù ĩ", + "mar lin", + "gu mmy", + "ðŁĻı ðŁı¾", + "awareness month", + "oil painting", + "mu th", + "per ch", + "jun to", + "villa gers", + "mor g", + "che ated", + "web comic", + "the future", + "d ps", + "la kings", + "men tioning", + "vo or", + "ident ities", + "accor d", + "mc gu", + "l pga", + "rum our", + "massi vely", + "m pls", + "heal y", + "d ate", + "sp oli", + "re visited", + "on t", + "al and", + "scru tiny", + "lakel and", + "bl ending", + "< /", + "an kara", + "jami edor", + "metab olic", + "f ences", + "ann y", + "å ħ", + "semic on", + "oo tt", + "space ship", + "wack y", + "le ta", + "ap ac", + "she e", + "in herit", + "do res", + "ðŁĩ¨ðŁĩ ¦", + "gent e", + "tw ick", + "ri ms", + "gal ve", + "de ville", + "king fisher", + "scorpi o", + "ow l", + "al ar", + "vari an", + "ðŁĹ ĵ", + "vene tian", + "star dust", + "then orth", + "q ing", + "har rington", + "consul ate", + "spectac le", + "ho bbs", + "tur ks", + "gre er", + "mat ing", + "ðŁİ Ģ", + "ðŁĮ Ģ", + "direc ts", + "í ĭ", + "pompe o", + "vo iced", + "la os", + "tz u", + "pro me", + "pri sm", + "mer c", + "fortun ately", + "bc fc", + "mcdon nell", + "not sorry", + "smi led", + "t ba", + "for war", + "mid term", + "dar by", + "we instein", + "up grading", + "wol ff", + "bron co", + "cab ello", + "ðŁ¥ ĩ", + "fi able", + "shar pe", + "bat tered", + "sat o", + "myth ical", + "instap ic", + "pre pped", + "eni um", + "e spo", + "di aper", + "explan ations", + "who pping", + "ragn ar", + "pe el", + "antibio tic", + "l acks", + "harri son", + "li sm", + "au l", + "qu ail", + "martin a", + "sent encing", + "sc ams", + "di di", + "tr onics", + "ãħł ãħł", + "go ff", + "za in", + "param ore", + "cha ined", + "clin ton", + "li ff", + "cott ages", + "em on", + "reve rend", + "consu mer", + "ce an", + "t any", + "lum pur", + "e bay", + "sto ol", + "ðŁĺ» ðŁĺ»", + "ta pro", + "h ath", + "modern art", + "just ine", + "prover b", + "app y", + "tra x", + "mani fest", + "am bu", + "nai k", + "pe pp", + "r sd", + "mer chants", + "kitch ener", + "shi fted", + "li zz", + "âĺħâĺħ âĺħâĺħ", + "âĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶ", + "uto pia", + "tom o", + "ou ted", + "com ers", + "chiroprac tic", + "book club", + "cin dy", + "pro hibition", + "se uss", + "ë¯ ¼", + "thin kin", + "rr rr", + "go fund", + "t ack", + "om b", + "catastro phic", + "ling u", + "guild ford", + "bo td", + "ॠĭ", + "plan ter", + "^ ^", + "win k", + "kath mandu", + "sto ppers", + "smooth ies", + "re efs", + "hin d", + "bell amy", + "Ħ ë", + "waste water", + "vo or", + "nat l", + "! ]", + "re el", + "y ap", + "scoo by", + "work space", + "corin thians", + "bl un", + "obli gation", + "g bbo", + "dy son", + "cra vings", + "ell ington", + "dap l", + "wre xham", + "earthand clouds", + "uk runchat", + "positi oned", + "kal b", + "four square", + "jo ck", + "im pending", + "even ing", + "ath y", + "pro claimed", + "c ites", + "ann apolis", + "san i", + "mar th", + "ir l", + "accom mo", + "ka a", + "fin a", + "y aa", + "di sper", + "ec ar", + "bha k", + "will y", + "ðŁĺĢ ðŁĺĢ", + "mcder mott", + "mo j", + "gener ational", + "u said", + "train ing", + "lon ely", + "lo res", + "impe cc", + "âĢ IJ", + "beav ers", + "ma ki", + "he b", + "aap l", + "å ı", + "wolver hampton", + "leader board", + "me u", + "c fa", + "easter n", + "hu r", + "civil war", + "ou rage", + "hor ned", + "le high", + "awar ds", + "evi dent", + "gi gab", + "r ous", + "ma del", + "ro byn", + "ur gently", + "k ors", + "en as", + "heis man", + "bam bam", + "fab ian", + "f om", + "evalu ating", + "assemb ly", + "out sourcing", + "hun tsville", + "ðŁĶ ª", + "justi fied", + "cashi er", + "sp aper", + "buc keye", + "analy tical", + "illumin ati", + "au tho", + "o j", + "sha de", + "geel ong", + "wh ey", + "he aton", + "terri bly", + "ele k", + "un charted", + "sd live", + "moto cross", + "her mes", + "dar shan", + "dar lington", + "cash mere", + "gri pping", + "cilan tro", + "pun ish", + "... :", + "ðŁĴ Ħ", + "inst ance", + "der i", + "lo bal", + "muk her", + "sp ar", + "thin ker", + "fre mont", + "com piled", + "color ado", + "vig ne", + "sm d", + "whe ad", + "villa ge", + "le ek", + "formula e", + "ta res", + "persist ence", + "?? ????", + "ped ago", + "he z", + "alzheim ers", + "vul ture", + "off ence", + "is great", + "suff ra", + "kick in", + "h mmmm", + "broad way", + "ï¸ı @", + "art i", + "alli son", + "endor ses", + "ry u", + "lolli pop", + "soy bean", + "kend all", + "cer a", + "inv ade", + "( ðŁĵ·:", + "conver ter", + "car pets", + "ho bo", + "fr it", + "pe ac", + "es qu", + "ern an", + "ou f", + "an il", + "di ffer", + "ch ing", + "bre cht", + "sp g", + "daven port", + "stra va", + "sever n", + "n gos", + "stor ians", + "fe te", + "parame dic", + "j hb", + "al amo", + "sne aking", + "gold coast", + "roof s", + "isi l", + "depic ted", + "projec tions", + "nu mb", + "o ss", + "ep i", + "glu cose", + "zid ane", + "infin iti", + "íĺ Ħ", + "ran som", + "ton ics", + "fal k", + "g ler", + "ou tw", + "re ss", + "week ly", + "the on", + "n ole", + "ðŁĩªðŁĩ º", + "vol ley", + "sum mar", + "neg ativity", + "sam son", + "ye w", + "aus votes", + "ju l", + "ju dy", + "f art", + "pra yed", + "pal ate", + "multicul tural", + "double header", + "cycl ones", + "pier re", + "ãģ ¨", + "âĺ łï¸ı", + "rt w", + "conver ting", + "wir ral", + "l ari", + "ir relevant", + "austin mahone", + "an che", + "ya an", + "sd f", + "$ .", + "explo ding", + "ulti mate", + "prof ici", + "gofund me", + "cell ence", + "ep stein", + "bul lied", + "sep tic", + "à® ¤", + "lu mber", + "cu ff", + "vsco cam", + "pl or", + "ภ¥", + "se ok", + "ro to", + "venezu elan", + "sor ta", + "spir ited", + "daniel padilla", + "team sisd", + "radio active", + "icelan dic", + "ðŁĴ ¤", + "ver e", + "accommo date", + "shi pp", + "ot ter", + "ol ina", + "e go", + "su la", + "san antonio", + "de as", + "simil arities", + "âļ ¾", + "y om", + "bro ward", + "å °", + "can cun", + "veri fy", + "on te", + "candle light", + "ìł ķ", + "inf ants", + "az am", + "ðŁĺ °", + "le ven", + "un stable", + "bloom ington", + "x ford", + "con tour", + "y p", + "innov ator", + "histor ies", + "po y", + "lolo lol", + "ex pires", + "cat alo", + "bill boards", + "an ab", + "el ic", + "novasco tia", + "fa ire", + "ìĿ ´", + "rock well", + "gr ille", + "az tec", + "joh or", + "ur struly", + "fi ren", + "dun lop", + "id le", + "port man", + "jo es", + "tx hsfb", + "hol m", + "cham ele", + "under world", + "lo ss", + "ti em", + "therap ists", + "past ure", + "pa ste", + "ing now", + "vul can", + "ra gon", + "lar kin", + "o shi", + "ho co", + "child hood", + "umb rel", + "success or", + "kath y", + "iz en", + "° ï¸ı", + "share holders", + "ol ga", + "ai b", + "he ap", + "fl aming", + "ro u", + "air tel", + "rat t", + "z ane", + "vo w", + "thor ough", + "sn ag", + "par th", + "un conscious", + "ve y", + "new release", + "gh ee", + "croati an", + "facilit ating", + "swan son", + "astor ia", + "to logy", + "master y", + "ðŁ¤ ij", + "bil bao", + "trou pe", + "the ori", + "chey enne", + "ro tt", + "shore line", + "gra sso", + "master chef", + "+ )", + "vi x", + "ellen show", + "as g", + "an ak", + "ku ya", + "safar ilive", + "debu ting", + "blu m", + "list ener", + "v ins", + "book shelf", + "smart cities", + "makeyourown lane", + "; ;", + "ðŁIJ ¯", + "ri zz", + "on ward", + "bull dog", + "bear ish", + "vir uses", + "fri gh", + "lin den", + "we iser", + "sn t", + "gon a", + "dre sden", + "fl anders", + "cu k", + "wheel ing", + "ba u", + "atu esday", + "surf ers", + "swi ft", + "mc call", + "arbitr ation", + "aw d", + "mon c", + "b ine", + "at x", + "re fr", + "mi ro", + "po sey", + "n are", + "rit ter", + "âģ ¦", + "play book", + "blow out", + "sports manship", + "s oooooo", + "malay alam", + "gri ms", + "bur bank", + "infin ity", + "sar gent", + "oit nb", + "joseph ine", + "ski pping", + "par kin", + "excur sion", + "semin ars", + "jo har", + "par tridge", + "post game", + "ll ll", + "blan che", + "temp ting", + "m na", + "lu ka", + "is ers", + "to ffee", + "bar ron", + "he mmings", + "sa e", + "go hawks", + "cu pid", + "li mbs", + "con se", + "un common", + "z ada", + "head shot", + "so ils", + "pione er", + "mam ma", + "sem itic", + "pan dey", + "jamiedor nan", + "spl its", + "vel a", + "son i", + "ra ff", + "t mobile", + "âŀ ĸ", + "pra wns", + "lit er", + "enjo yment", + "egg plant", + "tu b", + "cultur al", + "us ic", + "suspici on", + "sy cam", + "summ ed", + "ma du", + "ho ck", + "up wards", + "eye ing", + "ri ve", + "assas sins", + "âĤ ¬", + "out fy", + "chi ves", + "t ner", + "la is", + "por ridge", + "sad dest", + "w cc", + "vick i", + "sna ils", + "biz italk", + "mill an", + "ðŁĮ į", + "sam oa", + "j ing", + "mi key", + "gu j", + "chel ms", + "eli gibility", + "arma da", + "thro p", + "surger ies", + "ãĤ ¿", + "mo hawk", + "ex its", + "me m", + "is lington", + "c me", + "land fill", + "kait lyn", + "ðŁİ ¼", + "combin ations", + "tomorrow land", + "ver b", + "cor a", + "pre cisely", + "na om", + "ðŁĨ ķ", + "shr ink", + "sof tly", + "merce de", + "mand el", + "poo dle", + "ball erina", + "sop h", + "jux ta", + "y at", + "ary an", + "hesit ate", + "lo wered", + "gu lar", + "dungeon sand", + "ron an", + "my ri", + "sp f", + "men opau", + "gra sp", + "pa thi", + "fe asi", + "fla w", + "shi story", + "ste ward", + "gg le", + "fay re", + "cli que", + "credi bility", + "yo g", + "sec tion", + "mu sko", + "se ville", + "no tt", + "cal m", + "mate o", + "indic ted", + "fi ba", + "by l", + "lin o", + "u kin", + "!! #", + "enig ma", + "siri us", + "bu sc", + "ðŁį Ĭ", + "mac kerel", + "psal ms", + "a at", + "tomorrow spaper", + "ðŁĺ ĸ", + "p fc", + "........ ...", + "shre k", + "mul let", + "o sh", + "danger ously", + "immen sely", + "am ur", + "ðŁį Ĥ", + "pro por", + "sy a", + "london marathon", + "abo ve", + "obli gatory", + "pro v", + "ra cha", + "alex is", + "pri mary", + "sh h", + "ether net", + "d stv", + "cou gar", + "un lucky", + "ni l", + "steak house", + "mel a", + "fc bayern", + "cause way", + "ca therine", + "fluore scent", + "nx t", + "to kyo", + "au sp", + "releg ation", + "qui zz", + "shored itch", + "proud tobe", + "promo s", + "inter acting", + "home brew", + "da esh", + "w pg", + "stead ily", + "provin ces", + "bal lots", + "i ah", + "al to", + "< <<", + "you u", + "ri ley", + "prefe rence", + "tra verse", + "incen se", + "am munition", + "ho dges", + "# @", + "hail state", + "tart an", + "witch craft", + "vent ilation", + "liber tarian", + "! âĢ¦", + "ow es", + "% !", + "ong chang", + "bru shing", + "le ic", + "fi ber", + "under attack", + "down load", + "ex pir", + "hy o", + "pompe y", + "mc bride", + "y ag", + "stre e", + "com bat", + "ten ding", + "ai ra", + "gug gen", + "ab ra", + "in na", + "fli ps", + "aw al", + "m ach", + "dol lar", + "inspir ations", + "z um", + "o du", + "it ty", + "video game", + "aqu aman", + "har u", + "bel fast", + "je b", + "but ch", + "us gs", + "calcu lus", + "go yal", + "mor gen", + "x finity", + "stand up", + "contrac ep", + "sab re", + "na be", + "in secure", + "gener ously", + "epit ome", + "l w", + "t ca", + "narr atives", + "don nell", + "pand as", + "ber gh", + "tu t", + "ker al", + "fel icity", + "br ampton", + "quinte t", + "nom ore", + "ðŁĶ ij", + "lo i", + "alham dulil", + "ðŁĶ¥ ðŁĶĹ", + "ston er", + "shaw l", + "clin ical", + "bren dan", + "gon e", + "fla wed", + "tri ppy", + "j g", + "al location", + "po aching", + "ve vo", + "mo cks", + "lef tist", + "bon uses", + "condem ned", + "abil ity", + "st ating", + "microbi ome", + "bio logist", + "for you", + "wahl berg", + "ss or", + "ift ar", + "w ul", + "ÑĦ оÑĤ", + "pom er", + "me me", + "ver te", + "tre ll", + "tra it", + "in let", + "hormon es", + "deliber ately", + "vill ar", + "battle ship", + "p bl", + "tw enti", + "ho kies", + "dal ail", + "say a", + "may fair", + "han s", + "die ts", + "⾨ ⾨", + "od in", + "hot spur", + "pap i", + "k ana", + "k amp", + "fin na", + "flo tus", + "ti ans", + "unic orns", + "tribe ca", + "chang ers", + "fore ground", + "out a", + "inv aders", + "gett ys", + "tomorrowspaper stoday", + "mac millan", + "hand written", + "w fp", + "u de", + "state of", + "base d", + "âĺģ ï¸ı", + "cas m", + "psy ched", + "histor ians", + "fol d", + "d da", + "ag grav", + "p ans", + "green way", + "au sv", + "ðŁĺ ¶", + "shradd ha", + "inde x", + "be sti", + "zim mer", + "t ness", + "eye shadow", + "ot te", + "go ts", + "distribu ting", + "pro min", + "yo l", + "ace a", + "tram rahim", + "hoo per", + "supre me", + "jam min", + "intu itive", + "quali fications", + "sli m", + "sid di", + "jay ne", + "tri pping", + "g tx", + "pun s", + "e manuel", + "om g", + "mid summer", + "in to", + "succul ent", + "ri en", + "new mexico", + "o or", + "hoo king", + "in f", + "ðŁ¤ Ŀ", + "flir ting", + "na hi", + "g friend", + "t ps", + "hel ix", + "z s", + "on ie", + "ct f", + "kri s", + "irresi stible", + "fla p", + "ðŁijıðŁı» ðŁijıðŁı»", + "us wnt", + "ru d", + "ram ps", + "pin oy", + "ot w", + "lol z", + "low ering", + "favor ite", + "t mc", + "phra ses", + "her mi", + "aver aging", + "em br", + "ben o", + "estu ary", + "sle eve", + "ribb ons", + "ta sh", + "ภ¹", + "x f", + "aw gs", + "sun ited", + "brew eries", + "anir ud", + "pun ches", + "ol die", + "ip ads", + "wi fey", + "land lords", + "d ji", + "gun ner", + "íķ ´", + "tex an", + "ex op", + "cas sandra", + "s off", + "ðŁļ «", + "igh ton", + "bak ers", + "awareness week", + "v all", + "ear p", + "bts bbmas", + "apologi zes", + "âļĵ ï¸ı", + "was ps", + "states man", + "snat ch", + "watch dog", + "ra fi", + "after party", + "spi ke", + "j er", + "peri ph", + "r nc", + "mu ll", + "le en", + "shi es", + "li eu", + "urstruly mahesh", + "mer ton", + "de sai", + "shi f", + "ðŁĮ ±", + "pe dic", + "gos ling", + "arrang ing", + "ww g", + "gen y", + "you uu", + "netfli x", + "e ttes", + "k wi", + "bernar dino", + "am iga", + "Ø ¨", + "kashmir i", + "t ings", + "emer itus", + "de cat", + "ab domin", + "dc i", + "pha ses", + "d jan", + "be am", + "op ry", + "i shed", + "the ellenshow", + "the st", + "habit ats", + "to ons", + "mclau ghlin", + "ri pper", + "micro biology", + "tal aga", + "clu eless", + "ss u", + "cro che", + "bro mance", + "longe vity", + "zagre b", + "prev ented", + "tra ve", + "spo ilt", + "darry l", + "migra ine", + "al cat", + "dd dd", + "vi v", + "ser pent", + "mat tel", + "jam a", + "con quest", + "î Ħ", + "sam sung", + "presbyter ian", + "ket ch", + "fire fox", + "mo tif", + "le c", + "cho pping", + "cher no", + "j ann", + "ðŁIJ °", + "pro lon", + "wake up", + "conver gence", + "mersey side", + "heart broken", + "lo oming", + "hal lucin", + "mai ze", + "commun ism", + "mo h", + "twitter storians", + "serge y", + "res eller", + "favor able", + "ed gy", + "re iter", + "mal aga", + "live me", + "ka hn", + "pul sion", + "big g", + "kim kardashian", + "ati o", + "tyr anny", + "ru ption", + "q ant", + "pro ven", + "by z", + "pu shaw", + "kri stin", + "e er", + "tar dis", + "ri z", + "awak en", + "mi ko", + "un documented", + "path finder", + "indirec t", + "resemb les", + "h ler", + "conce aled", + "scand al", + "re im", + "d nb", + "cr itters", + "attend ant", + "apprentice ships", + "aa u", + "scre amed", + "l su", + "fa h", + "har bour", + "ed d", + "bat sman", + "li ss", + "mi sha", + "spani el", + "it f", + "advan cement", + "fa c", + "close up", + "cecil ia", + "medi c", + "narcis si", + "lav ish", + "gi ac", + "ma ys", + "le it", + "wine wednesday", + "pushaw ard", + "let to", + "curren ts", + "bug atti", + "out ine", + "w j", + "un do", + "ler osis", + "devo tional", + "ðŁij «", + "on na", + "fais al", + "sa una", + "himach al", + "am ii", + "à® ®", + "di zzy", + "screen writing", + "ph x", + "sp n", + "ick i", + "ag irl", + "fi shes", + "wb z", + "pi m", + "bo ar", + "ac id", + "! ..", + "rocke feller", + "n ga", + "dra stically", + "simpli fy", + "dru mming", + "autum nal", + "gur mee", + "lor de", + "jo ann", + "give up", + "b our", + "am ura", + "der land", + "sim pler", + "wat son", + "tri dent", + "concor dia", + "bel lum", + "bre k", + "dum plings", + "vi on", + "dungeonsand dragons", + "sp ri", + "ascen sion", + "wil datlantic", + "u st", + "rob ins", + "legi on", + "insi st", + "jar o", + "gue ss", + "so b", + "bigh it", + "pool side", + "negoti ating", + "mc gill", + "bil d", + "techn icians", + "miti gation", + "ajay devgn", + "b to", + "ant en", + "cosmo politan", + "ðŁĺĬðŁĺĬ ðŁĺĬðŁĺĬ", + "patri oti", + "temp er", + "promen ade", + "nav ajo", + "nam m", + "wrink les", + "dc fc", + "le ach", + "bru nette", + "r f", + "cout inho", + "al ti", + "tradition ally", + "op tome", + "na z", + "accord ingly", + "rec ard", + "de ets", + "sw ell", + "po sure", + "whit ening", + "strang er", + "illi on", + "here ford", + "u wu", + "ro bber", + "cotsw olds", + "cl en", + "gor ge", + "nam aste", + "re lish", + "gri ff", + "adren aline", + "bla sio", + "val e", + "ê ²", + "toler ate", + "rail minindia", + "jen sen", + "ho ven", + "el lu", + "ob sole", + "eisen hower", + "unidenti fied", + "than niversary", + "body guard", + "Ø ¯", + "i dge", + "sch al", + "stock port", + "sn i", + "re taining", + "po po", + "pix ie", + "oli thic", + "ki er", + "ha jj", + "sa z", + "cor bin", + "!!!! !!!!!!", + "v it", + "me gat", + "de h", + "circu it", + "af fleck", + "theore tical", + "hope less", + "u ab", + "slu mp", + "b ice", + "jam med", + "let stalk", + "can i", + "side ways", + "labyrin th", + "re fs", + "ha hn", + "jare d", + "ðŁį ¹", + "jam bo", + "ph yl", + "enhan cement", + "c tr", + "ful lest", + "se ye", + "do ba", + "cho ic", + "yo s", + "cb j", + "andr é", + "re watch", + "pri ma", + "doctr ine", + "for gets", + "u hm", + "ar ound", + "u le", + "art lovers", + "shi raz", + "har th", + "ex tor", + "Å ¡", + "unexpec tedly", + "eli us", + "y x", + "em my", + "se ac", + "ðŁijĩðŁijĩ ðŁijĩ", + "correc ted", + "com bu", + "wom anc", + "cou gh", + "what son", + "publi shes", + "divers ity", + "back bone", + "lock down", + "mesmeri zing", + "nor te", + "ma b", + "desig ner", + "í ģ", + "ra gh", + "mole cules", + "get outside", + "the beatles", + "semicon duc", + "nach o", + "lun es", + "ham mers", + "sul tan", + "o on", + "fe ren", + "att ach", + "ar qu", + "uttarak hand", + "s ash", + "; -", + "tre ad", + "i ko", + "ar thur", + "scandin avian", + "r ation", + "ga el", + "charge able", + "fish y", + "v ma", + "hand bags", + "char a", + "ay ne", + "de fam", + "sett lers", + "qad ri", + "pal ais", + "in wx", + "apocaly ptic", + "poo ja", + "a es", + "at ories", + "proof ing", + "n lp", + "ts la", + "v ina", + "li do", + "dee phouse", + "informat ics", + "v v", + "pp ings", + "di ss", + "à ¯", + "uhur u", + "st ony", + "betra yed", + "b aff", + "my ra", + "as pen", + "allow ance", + "tam ara", + "ci f", + "cor bett", + "ser ge", + "di go", + "ambi gu", + "pain ters", + "p cr", + "p ca", + "nom s", + "lo ft", + "ve e", + "opend ata", + "ðŁIJ ±", + "alex andre", + "identi fies", + "fantasy football", + "re production", + "brom ley", + "ware agle", + "mm er", + "p ss", + "cu es", + "ay at", + "hut chinson", + "sar ac", + "jack man", + "ira h", + "ap ink", + "col s", + "aussi es", + "ex ecs", + "day ton", + "ðŁĻ Ĩ", + "im v", + "har am", + "chuck le", + "authent icity", + "ar do", + "incub ator", + "ภª", + "photo shopped", + "embrac ed", + "fight for", + "gor man", + "zz zz", + "schol astic", + "cri sps", + "te apo", + "mid night", + "ga ine", + "col lier", + "s ate", + "de tte", + "å Ń", + "imag ine", + "i ff", + "tw ili", + "i fication", + "teat ro", + "nor ma", + "es ur", + "emergen cies", + "rise up", + "r inger", + "hass le", + "cait lyn", + "tranqu il", + "vers a", + "se b", + "over look", + "gin i", + "bo go", + "se re", + "may ne", + "henri k", + "contamin ated", + "rhapso dy", + "pro portion", + "wildatlantic way", + "âģ© .", + "organis ers", + "tran e", + "stand ard", + "sper m", + "laun cher", + "ric ci", + "her ts", + "paper work", + "showcas ed", + "mer yl", + "pen a", + "p imp", + "disa strous", + "^. ^", + "phar a", + "x is", + "fron tal", + "sw irl", + "sp ills", + "swag ger", + "smart watch", + "sizz ling", + "savi our", + "cat ar", + "bb cr", + "refurbi shment", + "dr is", + "citro en", + "absor b", + "patrioti sm", + "il leg", + "chro mo", + "fresh ers", + "ru s", + "lim iting", + "ef ish", + "down ed", + "man dir", + "hazel nut", + "p all", + "mac on", + "disappear ing", + "quali fies", + "bo on", + "bar racks", + "am ine", + "gen dere", + "ðŁļ ĺ", + "j es", + "ãĥ Ń", + "qu ito", + "middle weight", + "sch au", + "quad ru", + "aci ones", + "limit less", + "ðŁijĮ ðŁı½", + "ch man", + "ar av", + "regulat ors", + "it up", + "batter sea", + "mil ford", + "g z", + "tic king", + "gh ou", + "cru shes", + "tu tu", + "dread ful", + "fam ine", + "for change", + "dalail ama", + "ðŁĴ į", + "whit aker", + "hash mi", + "h us", + "vo d", + "bet te", + "aa ah", + "iso o", + "ðŁ¥ Ī", + "ha ar", + "la ine", + "b v", + "all day", + "spr out", + "indie games", + "free bie", + "gree ks", + "but ler", + "ill in", + "ha al", + "ware ness", + "si ma", + "public health", + "gam a", + "wa a", + "oun g", + "goo oo", + "okin awa", + "off enders", + "im pose", + "ho c", + "young ster", + "story teller", + "sc ap", + "figh ter", + "+ ,", + "whit es", + "music monday", + "re za", + "go ducks", + "bri a", + "mi um", + "cas per", + "cru mbs", + "a ad", + "marti alarts", + "ch p", + "ri gged", + "tn g", + "harve sted", + "sa k", + "do jo", + "mill wall", + "b nw", + "oc d", + "histor yof", + "t mr", + "si rens", + "fan ci", + "caregi vers", + "vir a", + "son i", + "recur ring", + "acknowle dged", + "ðŁı Ł", + "oph ile", + "bu cky", + "stre ssing", + "roo k", + "di gger", + "vi val", + "san do", + "fle et", + "si ers", + "sel caday", + "refre shed", + "anti fa", + "a que", + "po lo", + "disappear ance", + "de mb", + "âĮļ ï¸ı", + "ren ted", + "ber ger", + "g mb", + "cu la", + "ss al", + "goo dy", + "u hh", + "marcel o", + "w anna", + "soft ware", + "shop small", + "turt le", + "tom as", + "fri sco", + "ðŁĺį ðŁĴķ", + "jim enez", + "c su", + "day z", + "an do", + "wyn ne", + "choreo grapher", + "cerv ical", + "trail blazers", + "ed g", + "zend aya", + "travel blog", + "el s", + "whole some", + "co g", + "lab out", + "ar ney", + "del le", + "su isse", + "ma si", + "ine se", + "om be", + "fi ddle", + "re claim", + "pa u", + "wat cher", + "sla in", + "ber ty", + "opti mum", + "el ites", + "min is", + "tur key", + "patro ls", + "ger ard", + "au reli", + "wild ly", + "wal tz", + "br gy", + "w ob", + "cre st", + "+ ++", + "ve z", + "fro sted", + "davi do", + "the x", + "param edics", + "p into", + "han k", + "du pont", + "ur g", + "fo stering", + "micro poetry", + "spec tre", + "---- >", + "ne uro", + "fri da", + "music al", + "galve ston", + "e ffic", + "sc ape", + "pal azzo", + "th all", + "pro visional", + "p js", + "au re", + "ðŁĶ ľ", + "mam amoo", + "kit ties", + "cre e", + "wa k", + "lo ool", + "lu pus", + "cn blue", + "à º", + "ðŁİ ¬", + "rac ed", + "tro se", + "om as", + "stri de", + "co ors", + "⤠µï¸ı", + "in comparable", + "cy ril", + "broad er", + "arec lipse", + "ðŁį Ķ", + "inter val", + "ti ru", + "co working", + "w aco", + "a ham", + "a bee", + "flouri sh", + "the times", + "ol ini", + "kick boxing", + "lu cer", + "at la", + "as un", + "casser ole", + "mi aw", + "lobb ying", + "jan ice", + "cir que", + "re flex", + "le ary", + "sanat omy", + "tem pest", + "se mb", + "mur dering", + "us av", + "ro bo", + "on et", + "p cc", + "nati ves", + "life of", + "sa ha", + "ruth less", + "rel ates", + "appeti zer", + "pye ongchang", + "nor d", + "er u", + "a thing", + "ug ly", + "pl ying", + "bran ce", + "organ ise", + "kend ra", + "dat o", + "chees es", + "par ma", + "burn out", + "a stra", + "pre toria", + "adjust ment", + "uk u", + "sl o", + "li ken", + "fav ors", + "cli ve", + "be ets", + "snow donia", + "go tv", + "sy n", + "open house", + "pan i", + "portra yed", + "sl ated", + "me cca", + "ren al", + "supportsmall streamers", + "staf fs", + "da o", + "bi ker", + "vik tor", + "tit us", + "admi red", + "ðŁĵ ±", + "hurric an", + "he ats", + "gl ory", + "photo genic", + "mer i", + "de por", + "burn ham", + "or angu", + "dj ing", + "impre ssionism", + "ign ition", + "ca i", + "w ynn", + "de pe", + "cove ted", + "colla gen", + "sau s", + "or nam", + "administr ators", + "ss on", + "nh politics", + "hahahaha hahahaha", + "aspir ations", + "r gb", + "swol len", + "so we", + "sc r", + "diver gent", + "hou ghton", + "han oi", + "d ory", + "ni ki", + "land ry", + "b cci", + "ðŁijĮ ðŁijĮ", + "is mail", + "tri pod", + "her d", + "bhat t", + "dress age", + "tab by", + "ingu ish", + "hur on", + "à³ į", + "à ł", + "to das", + "evangel ical", + "chor ds", + "st john", + "slo ppy", + "marty r", + "face book", + "ali ght", + "sen sei", + "kath niel", + "r ites", + "zi one", + "u o", + "revel ations", + "weight lifting", + "pan o", + "nc wx", + "ac ton", + "à® ķ", + "Ø ²", + "som a", + "ภĹ", + "respec ting", + "mar che", + "fore man", + "be tty", + "ki k", + "shi bu", + "po on", + "argy le", + "k swx", + "et z", + "mar bella", + "brac kets", + "stand by", + "fire side", + "defi ance", + "v ex", + "britanni a", + "in habit", + "appo int", + "piyu sh", + "le ash", + "sci ento", + "fla sk", + "sen na", + "> :", + "at roc", + "sand erson", + "id lib", + "dhan ush", + "ðŁĺ Ļ", + "en thr", + "hit ch", + "de dly", + "al ley", + "dor k", + "mon do", + "cudd ly", + "mis sin", + "ye sss", + "night ing", + "j pn", + "w ary", + "ump ire", + "ma z", + "ê ³", + "bab s", + "ĭ ãģ", + "stan ford", + "posse ssed", + "exce eded", + "ðŁĶ ¶", + "wall art", + "tra p", + "j il", + "hi bis", + "sp ying", + "scri be", + "khali l", + "trans lator", + "lu mb", + "di zed", + "ch c", + "super vision", + "shut ter", + "ja g", + "_ *", + "yester days", + "ms f", + "hi hi", + "gonz aga", + "gille spie", + "vive k", + "ec static", + "this morning", + "ch us", + "ed es", + "ston ed", + "be es", + "ðŁĩ¹ ðŁĩ", + "tur in", + "ho ver", + "at rics", + "ster n", + "sam heughan", + "auti sm", + "mi ya", + "eye witness", + "writ ings", + "travel tips", + "chut ney", + "px rtg", + "keny ans", + "my stic", + "k rit", + "/ $", + "red head", + "world ly", + "am us", + "op la", + "le ve", + "gab bana", + "se en", + "o clock", + "gang a", + "keen an", + "sc ent", + "ol dies", + "go green", + "corner stone", + "comp ly", + "con cours", + "ðŁİ¶ ðŁİ¶", + "ha an", + "con fis", + "aw son", + "cle op", + "î Ģ", + "su zu", + "sau té", + "al gar", + "subscri ber", + "este emed", + "ãĤ¤ ãĥ", + "worth while", + "mel rose", + "flo ck", + "bri ghtly", + "viol inist", + "p ere", + "sli pping", + "and co", + "si gh", + "ha van", + "cu lo", + "m sa", + "fibro sis", + "matil da", + "ra fting", + "aw ard", + "ë ª", + "mm mm", + "ge aux", + "ste iner", + "sin n", + "help ers", + "beet les", + "ai mee", + "tai wan", + "pistachi o", + "mac beth", + "m zan", + "descend ants", + "on sale", + "in r", + "il m", + "grou se", + "sa ig", + "mo w", + "bi gre", + "adjust ments", + "tu la", + "mathe w", + "transl ates", + "mu h", + "bol lah", + "ðŁĴĽ ðŁĴĻ", + "amo res", + "ab outs", + "bomb shell", + "bla ster", + "x avi", + "s ns", + "k roger", + "ga ther", + "erad ic", + "daf t", + "chem o", + "ben ches", + "ðŁĩ© ðŁĩ", + "ut v", + "our a", + "n ko", + "gator ade", + "biaf ra", + "ok state", + "im danielpadilla", + "dom ains", + "open ingday", + "kid do", + "do i", + "ric e", + "day care", + "mac millan", + "ba thurst", + "cheer leading", + "ðŁ¦ ģ", + "cash back", + "k won", + "hob bies", + "exem pl", + "ries ling", + "âļ ª", + "ag les", + "ny s", + "every thing", + "nav is", + "ad di", + "magne sium", + "faceli ft", + "ark ham", + "grand es", + "extre mist", + "don at", + "vit ality", + "pump kin", + "be tta", + "sl td", + "arti san", + "li by", + "pe aked", + "ah hhhh", + "mary am", + "assi m", + "un sc", + "ment e", + "al aya", + "low ers", + "ar as", + "gri ev", + "le ip", + "gr ati", + "cri ses", + "spr ints", + "exe cute", + "w to", + "ms d", + "mag ical", + "re viewer", + "spark les", + "juke box", + "ðŁĺĤ âĿ¤ï¸ı", + "pay back", + "licen ses", + "dun kin", + "bel t", + "lake wood", + "h ateful", + "bud gets", + "rev amped", + "ph erson", + "ky iv", + "went worth", + "ro sen", + "cru ise", + "gi ggle", + "def star", + "assassin scre", + "ym outh", + "win kle", + "w fc", + "band wagon", + "b kk", + "w iring", + "kear ney", + "south side", + "pe tit", + "! ðŁĺį", + "nor dic", + "mir za", + "mu gabe", + "v l", + "scon es", + "k tv", + "sand al", + "du c", + "m alls", + "ðŁĴŀ ðŁĴŀ", + "it c", + "al ay", + "im pair", + "un rest", + "flo ss", + "c é", + "ab ou", + "var ying", + "muse o", + "ser ver", + "di ya", + "hibis cus", + "ero y", + "mer ritt", + "fin dom", + "f pp", + "un usually", + "go tt", + "conting ent", + "ali aa", + "ball on", + "jo l", + "hi ked", + "zy me", + "ay r", + "ag n", + "ga z", + "perio dic", + "spar ty", + "practi sing", + "lin ton", + "tal is", + "cy pri", + "womanin biz", + "radio disney", + "ðŁĮ ¼", + "jump ers", + "endo cr", + "ðŁļ¨ ðŁļ¨", + "and on", + "shar apo", + "mi er", + "ma sonic", + "fac tories", + "vi en", + "bb ers", + "ìĽ IJ", + "hol d", + "ke bab", + "be ak", + "approach ed", + "ac milan", + "mun ro", + "ko sher", + "excell ency", + "negoti ation", + "walt disneyworld", + "cr ouch", + "te asing", + "suppre ssion", + "en ya", + "b ce", + "transformation tuesday", + "cal lie", + "vis was", + "p gat", + "ic ted", + "end ings", + "esc u", + "recru ited", + "it fc", + "collabor ations", + "g ino", + "snu ck", + "ausch witz", + "i fc", + "x ii", + "ke sha", + "ger vais", + "clo ak", + "x l", + "sa ad", + "prob ation", + "pre cau", + "mac in", + "anasta si", + "le k", + "e azy", + "daysof code", + "mariah carey", + "yo g", + "stit ched", + "boy friends", + "sh ar", + "ph ile", + "ag u", + "twin kle", + "phi shing", + "week ender", + "ic ton", + "gurmee tramrahim", + "al ton", + "l eness", + "all an", + "pen ultimate", + "kry stal", + "go u", + "lan de", + "dis mant", + "ab using", + "nor se", + "pat erson", + "ed mun", + "ap an", + "xi umin", + "sk el", + "cat walk", + "re act", + "wal led", + "t angle", + "br yn", + "ve to", + "super moon", + "cas ablanc", + "appreci ates", + "ski d", + "bo th", + "catal ina", + "ele ague", + "cyber monday", + "cau tious", + "ðŁ¤ ĵ", + "nov o", + "hamp ton", + "ha ye", + "jose f", + "var an", + "lo bos", + "roano ke", + "orph ans", + "tt in", + "squ ads", + "ishqba aaz", + "black panther", + "e tu", + "k sh", + "cru mble", + "cess na", + "reli eved", + "scul ly", + "pollin ators", + "explore canada", + "ki es", + "kam loops", + "kir an", + "pri mal", + "sett lements", + "hot spot", + "brain storming", + "ce dric", + "bi ennial", + "sh ant", + "âĻ¡âĻ¡ âĻ¡", + "do on", + "hear n", + "walk way", + "fe m", + "ve al", + "deport ation", + "tox ins", + "elimin ating", + "descen ding", + "by the", + "bla sphe", + "ha sta", + "comple ment", + "as cent", + "ri ga", + "provo st", + "âĸ ª", + "wee ping", + "anti semitism", + "employe e", + "unearth ed", + "pin o", + "natali e", + "bla d", + "ang ola", + "lock heed", + "in ian", + "ag r", + "ni ster", + "im pala", + "m ke", + "fan atic", + "âĺħ âĺħ", + "ðŁij ¸", + "lu ch", + "simpli fied", + "gall ery", + "econom ic", + "cy borg", + "con i", + "sel ma", + "in ception", + "ko ala", + "dv ds", + "cre sted", + "m mor", + "visi ble", + "n sd", + "ðŁĻĮ ðŁı½", + "w under", + "refriger ator", + "re opening", + "e era", + "carou sel", + "as p", + "balli stic", + "victor y", + "mo tive", + "tre y", + "sharapo va", + "si i", + "mon ter", + "int end", + "west chester", + "sp e", + "cy mb", + "vi dal", + "ll ama", + "uni v", + "fin er", + "crafts manship", + "jazz fest", + "b ch", + "ag gio", + "n cc", + "lamb da", + "tranqu ility", + "cis co", + "ba den", + "so bbing", + "of i", + "go ta", + "ru mored", + "war med", + "ore an", + "ac ton", + "mar ci", + "gh ani", + "âľ ĵ", + "as sorted", + "pembro ke", + "pen elope", + "da f", + "at ty", + "aim o", + "pretz el", + "carni val", + "than os", + "ko chi", + "mer sal", + "ham radio", + "ar twit", + "cas c", + "guer rilla", + "kush ner", + "k app", + "al ise", + "todd lers", + "steward ship", + "o tti", + "ter ri", + "tem pe", + "rest less", + "vit o", + "zay ed", + "rsp b", + "pi on", + "hi ppo", + "haw thorne", + "in as", + "am ily", + "nut cracker", + "lo p", + "d ali", + "tro pic", + "ðŁ¤ ł", + "ul o", + "jare dle", + "py rene", + "pale o", + "usa ir", + "m ould", + "it ated", + "gene tically", + "biom ass", + "ðŁĩ³ðŁĩ ±", + "do dd", + "practic ed", + "monarch s", + "un manned", + "m buhari", + "am al", + "photo gra", + "ko ol", + "bren don", + "ju ices", + "cu re", + "world bank", + "poin ters", + "ðŁĴ Ŀ", + "tur f", + "le ds", + "bor ussia", + "bapti sm", + "warwick shire", + "moun ts", + "gay o", + "be gg", + "co pied", + "asi ans", + "k g", + "moder nist", + "gi d", + "front man", + "concentr ated", + "y t", + "sc avenger", + "iron ically", + "adi c", + "ps n", + "ðŁ¥ ī", + "cultur ally", + "yu v", + "mac arthur", + "fertili zer", + "be withyou", + "ri gor", + "min ors", + "z oning", + "âĸ ł", + "ri r", + "adole scent", + "vin ny", + "ren g", + "sand stone", + "gu et", + "we sth", + "ple dged", + "lac ed", + "sp ide", + "v ai", + "ty coon", + "seiz ure", + "du p", + "appalach ian", + "ro k", + "cathol ics", + "sey chel", + "posse ss", + "la ger", + "jo di", + "cham p", + "stra s", + "d ina", + "cent uri", + "cal der", + "blur ay", + "ðŁĩ¨ðŁĩ ³", + "mo do", + "an nette", + "youtu bers", + "chap s", + "ang ling", + "label ing", + "a qui", + "pk wy", + "ly le", + "bi sexual", + "lit ur", + "dug out", + "li bby", + "grey sanatomy", + "sub stances", + "august us", + "rall ying", + "fi del", + "ing ue", + "äº º", + "hallmark channel", + "tooth brush", + "m á", + "adi rond", + "ag gi", + "ðŁĵį :", + "cru sade", + "tax ation", + "k z", + "i ver", + "dou bling", + "room ie", + "wa b", + "en rolled", + "az on", + "a ju", + "grand children", + "as df", + "ðŁ¥ º", + "mat ic", + "ough ton", + "utili ze", + "ðŁĴ £", + "pon der", + "rais in", + "dys function", + "co bain", + "butter nut", + "e man", + "su red", + "dri an", + "and friends", + "with the", + "on omy", + "heine ken", + "bri dal", + "leader ship", + "pyram ids", + "deutsch land", + "jo cel", + "bo wel", + "y qr", + "horse power", + "be acon", + "ing eni", + "gra dient", + "fer mented", + "mo om", + "thing y", + "pot assi", + "wrist band", + "bor d", + "bo died", + "ðŁĺŃ ðŁĺį", + "ma pp", + "ka u", + "cyber punk", + "ph ish", + "loo king", + "co ates", + "ap ur", + "am ie", + "uk labour", + "at in", + "g la", + "adop table", + "shel by", + "v illi", + "ri ya", + "m ingly", + "cli mber", + "bumble bee", + "ðŁĺ ¸", + "c sd", + "âĿ ¥", + "hospit alized", + "c ki", + "hat er", + "ch r", + "re tina", + "it a", + "fan base", + "beat rice", + "gwy ne", + "go ss", + "fo s", + "favor ited", + "swachhb harat", + "mal ade", + "mon mouth", + "\" [", + "si van", + "sh hh", + "command ing", + "sains burys", + "wee d", + "g man", + "ss w", + "rep tile", + "iv y", + "tro pics", + "roll ers", + "over cast", + "ex position", + "masquer ade", + "man crush", + "wa ist", + "spr inter", + "sle et", + "le vin", + "j pg", + "_ (", + "o pel", + "explo it", + "ap a", + "po we", + "wrec king", + "jong in", + "or b", + "er ick", + "bo sco", + "pra ising", + "ber tr", + "to wing", + "in security", + "ku t", + "resto cked", + "rr p", + "prescri bed", + "trafal gar", + "per t", + "g ases", + "app rais", + "g har", + "music als", + "âĸ¬ âĸ¬", + "mc fad", + "ag ony", + "conditi on", + "equi p", + "shi k", + "atra vel", + "ðŁĩ¿ ðŁĩ¦", + "ke h", + "abduc tion", + "pe oria", + "wil kins", + "g ms", + "as d", + "ev i", + "ðŁĴĹ ðŁĴĹðŁĴĹ", + "u z", + "mo c", + "halle lujah", + "guad alu", + "lou vre", + "dra wing", + "go ve", + "ph ant", + "fri e", + "web dev", + "program mer", + "z able", + "games com", + "clari fy", + "li th", + "kin ky", + "âĿ £", + "labour doorstep", + "son ata", + "ju ris", + "mai den", + "vi adu", + "buch arest", + "conditi oned", + "capit alist", + "u de", + "ps b", + "sp ca", + "lul la", + "footh ills", + "kay o", + "bon d", + "wom b", + "roun der", + "ce sar", + "bur sts", + "ap ra", + "sw oon", + "sab rin", + "fra grant", + "cle arer", + "ku brick", + "cli max", + "jour no", + "ag le", + "ðŁı½ âĢįâĻĢï¸ı", + "poo ch", + "hal e", + "sol it", + "sal mon", + "organis ms", + "bron son", + "art en", + "hodg son", + "alo ve", + "vent ure", + "bb i", + "ae a", + "ðŁIJ ¢", + "ld n", + "d nr", + "o zone", + "el las", + "man ny", + "azz ur", + "un beat", + "tru ffles", + "th ong", + "ma ñ", + "las ers", + "ley e", + "gettys burg", + "back packs", + "or is", + "ma ison", + "craw ling", + "la bra", + "cl ing", + "dra gging", + "ste al", + "dou bt", + "de van", + "ck ers", + "agent sof", + "photo bomb", + "elon musk", + "abo y", + "dist ances", + "story line", + "sp i", + "nor than", + "europe ans", + "wh ale", + "ser pent", + "ðŁļ ²", + "fi or", + "tr it", + "ox o", + "awar ding", + "class mate", + "su fc", + "smar test", + "rich es", + "pr k", + "big foot", + "ar mb", + "bi polar", + "dw elling", + "om ars", + "k wan", + "gri me", + "m eng", + "freder ick", + "navar ro", + "sorry notsorry", + "jaredle to", + "pa ve", + "sl ack", + "barn sley", + "att ar", + "evic tion", + "accumul ation", + "o ir", + "cat chy", + "wel ter", + "vik as", + "has see", + "nik ita", + "mo yes", + "mathe ws", + "shi v", + "gat wick", + "pro filing", + "compan ions", + "mar rake", + "an tics", + "ðŁĻĮðŁĻĮ ðŁĻĮ", + "se se", + "bo i", + "bart lett", + "poison ous", + "ab uses", + "ym m", + "kam pala", + "guggen heim", + "imv kohli", + "dol om", + "bre e", + "thro ttle", + "gare th", + "fitz patrick", + "un ya", + "par ad", + "mar got", + "j nr", + "we a", + "potassi um", + "p nc", + "disgu ised", + "cra sh", + "ren ergy", + "ill ic", + "coup led", + "ni els", + "ci ones", + "æĹ ¥", + "im ent", + "despic able", + "d ye", + "what cha", + "conne ctions", + "paralym pics", + "gaunt let", + "wait rose", + "suici dal", + "star ship", + "vap or", + "st ou", + "law maker", + "coo led", + "si mo", + "then o", + "offro ad", + "ja den", + "bas que", + "vick y", + "lu kaku", + "centr o", + "tri sh", + "strate gist", + "medic ations", + "hor st", + "b fc", + "gra il", + "sharp ly", + "ad itya", + "tom b", + "kau fman", + "tri pad", + "sam ba", + "pastor al", + "brit ney", + "sag an", + "hill side", + "mas ons", + "sar a", + "z one", + "x u", + "to tes", + "rob bie", + "app en", + "mon tag", + "der o", + "short film", + "charis matic", + "tat ors", + "ki ba", + "and ri", + "al arming", + "split ting", + "ic ar", + "th ug", + "scari est", + "sylve ster", + "an an", + "u trecht", + "a difference", + "me ade", + "bu ster", + "air strikes", + "cu ffs", + "account ants", + "ðŁĺ¡ ðŁĺ¡", + "new t", + "bo tt", + "issu ing", + "cl ancy", + "wwen etwork", + "kyu hyun", + "rese mble", + "pajam as", + "sin k", + "kin ney", + "sul ph", + "or k", + "li es", + "la gh", + "or ton", + "ra hul", + "d sc", + "we will", + "re am", + "collo qui", + "shar ia", + "hec tic", + "sar casm", + "land er", + "tm z", + "endor f", + "ro z", + "ham mered", + "fri s", + "w adi", + "pope francis", + "he it", + "flash light", + "un born", + "op es", + "hol iness", + "ðŁIJ ¦", + "nach t", + "im sa", + "gr acing", + "bj p", + "ver ts", + "c sc", + "home owner", + "a que", + "bigo try", + "anni e", + "bag h", + "âĿ¤ï¸ı ðŁĺį", + "car i", + "thom p", + "dispo sable", + "cardio logy", + "pat ented", + "hh hhhh", + "ld r", + "stephen son", + "cro res", + "fan ning", + "cli mat", + "ðŁijį ðŁijįðŁijį", + "ðŁijį ðŁı¼", + "aer on", + "piccad illy", + "bank rupt", + "sil via", + "emplo y", + "don ny", + "commen ting", + "screen writer", + "io ta", + "ce an", + "anc ers", + "tu an", + "street wear", + "ठ¯", + "sk ine", + "esp a", + "asi f", + "os ce", + "she ppard", + "more cam", + "bott le", + "der s", + "orac le", + "google play", + "aver aged", + "edmon ton", + "steph an", + "sister hood", + "cru sted", + "stag gering", + "methodo logy", + "congress woman", + "c abo", + "tri ggers", + "mil ky", + "gli de", + "tooth paste", + "room mates", + "nu ff", + "gu am", + "sprink les", + "alternati ve", + "wat fordfc", + "uof t", + "hal ey", + "cont acted", + "bun dy", + "pro stitu", + "gh ar", + "pre ston", + "on site", + "hil ar", + "g ts", + "c att", + "hamp stead", + "? ?!", + "ðŁĩ§ ðŁĩ", + "bbc qt", + "aless andro", + "resi st", + "ma idan", + "t ko", + "shad ing", + "pin up", + "gal lo", + "sin u", + "at ec", + "fun k", + "ac lu", + "stri des", + "rhy me", + "wet land", + "bbc springwatch", + "t ins", + "wild card", + "st our", + "flamen co", + "pau la", + "onto logy", + "gang sta", + "am ade", + "ãĤ «", + "t bs", + "skelet al", + "run ner", + "jard in", + "harri er", + "hun ted", + "z hen", + "believein film", + "de mean", + "au diti", + "re start", + "chon dri", + "âĿ¤ï¸ı ðŁĴĻ", + "mcla ren", + "ga b", + "sh um", + "au sa", + "lewi sham", + "y pg", + "k jv", + "fur nished", + "dor o", + "bon ded", + "mor ty", + "lat itude", + "_ )", + "lo va", + "water ways", + "vin ai", + "shor th", + "drun k", + "c ay", + "ay ana", + "kap lan", + "capp uccino", + "spr o", + "life boat", + "has bro", + "spol ice", + "tor on", + "do ing", + "dam n", + "sh ree", + "foun tains", + "ent ation", + "mar u", + "boar der", + "to pless", + "j ada", + "chan ning", + "ul ls", + "en closure", + "gib son", + "fractu red", + "brit ton", + "à ¶", + "t ous", + "por th", + "dra f", + "tra iling", + "mar gate", + "eli fe", + "down ward", + "lin n", + "gla des", + "girl power", + "ak rish", + "u ki", + "ron da", + "ts c", + "appreci ationday", + "vis ing", + "lo om", + "ðŁį ³", + "mex ican", + "ar gos", + "y ya", + "jad ine", + "south port", + "d end", + "si sta", + "rede em", + "men g", + "bra xton", + "antioxid ant", + "s key", + "mp g", + "fin ding", + "vibr ation", + "ce u", + "kh art", + "di mini", + "cl ine", + "shel ly", + "hin es", + "ī ï¸ı", + "to pical", + "no ver", + "ma xx", + "prim itive", + "illustr ate", + "b ounds", + "tren ton", + "join tly", + "breed ers", + "u chi", + "wakeup america", + "b ada", + "ðŁĹ £ï¸ı", + "gu acam", + "sp heres", + "pere gr", + "youth ful", + "lo lo", + "bir min", + "t ly", + "jeremy corbyn", + "defe cts", + "co sm", + "a rent", + "v aa", + "bag els", + "medi ac", + "cori ander", + "ic ago", + "g haz", + "ab bas", + "re model", + "struc turing", + "pu m", + "out law", + "ad ani", + "r bc", + "gul ls", + "n li", + "confu se", + "ðŁijĩ ðŁı¼", + "vil a", + "mcnam ara", + "correc tions", + "mug hal", + "ser i", + "re gain", + "ss b", + "lea ve", + "haha hah", + "gran de", + "di stressed", + "re chargeable", + "ho a", + "hou sed", + "sti l", + "attribu ted", + "opath ic", + "di ps", + "pri t", + "head phone", + "conclu de", + "pil o", + "he t", + "ut sa", + "nit in", + "je m", + "sni ppet", + "tutor ing", + "op er", + "sun k", + "en sla", + "cha u", + "ac orn", + "quinte ss", + "ran kin", + "affili ated", + "our lives", + "cl int", + "se ater", + "isa ac", + "ba shing", + "sme ar", + "nur se", + "doo dling", + "\" ;", + "sa ku", + "atroc ities", + "im am", + "g fs", + "viol ating", + "comm end", + "brad shaw", + "er ville", + "b illed", + "b be", + "thul hu", + "i phones", + "moo se", + "di os", + "re w", + "me thane", + "strang ely", + "whis ky", + "ti ghtly", + "spiel berg", + "radi us", + "notic ing", + "wi f", + "ig nati", + "i fa", + "ap is", + "w ali", + "ha itian", + "bu shes", + "y z", + "v l", + "ex ited", + "asse l", + "tru ec", + "dom en", + "ash er", + "in king", + "newyear seve", + "hend ricks", + "bat i", + "ìĿ´ ì", + "rich ter", + "mon santo", + "con line", + "agre at", + "ðŁ¤ ¯", + "master pieces", + "ar n", + "rough s", + "cle ve", + "se v", + "fashi ons", + "to ya", + "sh ail", + "cop eland", + "aqu ari", + "dec als", + "are you", + "y aya", + "a str", + "fon t", + "ml m", + "ar ca", + "pp or", + "pol lock", + "xper ia", + "conserv ation", + "chain saw", + "ag gie", + "?! ?!?", + "si le", + "sh on", + "ìĹ IJ", + "note books", + "marque tte", + "de us", + "bb led", + "spic er", + "mc cabe", + "nor wich", + "modi fication", + "boo sted", + "stru m", + "sales man", + "bang le", + "nis san", + "hez bollah", + "brea sts", + "a af", + "anth us", + "sk er", + "ow ed", + "her os", + "gi fs", + "fo sters", + "eat ers", + "du es", + "_ /", + "lymph oma", + "sf am", + "me gal", + "afri di", + "ag ic", + "p amp", + "jeal ousy", + "ðŁijĮ ðŁı¼", + "calcul ate", + "napp ing", + "g ale", + "ðŁ¦ Ħ", + "lub bock", + "assu med", + "ren ting", + "íĥ ľ", + "subur b", + "ãĤ ·", + "tech nic", + "u cla", + "in front", + "gar net", + "ster oids", + "stri ving", + "ho war", + "mo ver", + "le ton", + "bull do", + "is in", + "ci ao", + "sn z", + "fore front", + "d ams", + "mid wife", + "ma wards", + "cla pton", + "we in", + "subsi dies", + "spr oud", + "rother ham", + "phan tom", + "ar ach", + "spi el", + "rac ket", + "sel amat", + "no on", + "l bc", + "enti ally", + "ðŁĴ ¸", + "sil ve", + "m oud", + "kine tic", + "y asi", + "ðŁİ ©", + "o ol", + "mi ku", + "i za", + "fer a", + "flo ren", + "barber shop", + "groo t", + "z est", + "ne ars", + "stan is", + "z and", + "police man", + "juris dic", + "form ations", + "appar atus", + "sp d", + "arti fact", + "to sc", + "motiv ating", + "womanc rush", + "re dro", + "diagno stics", + "ra za", + "out fitters", + "el xn", + "dod gy", + "ry n", + "sh d", + "ortho don", + "ol de", + "jay anti", + "bal ances", + "quic kest", + "can ton", + "friday reads", + "! *", + "na a", + "a ak", + "ðŁĶ ·", + "behavi ors", + "rasp berries", + "ä »", + "polit ical", + "cam il", + "å ľ", + "di k", + "ast ounding", + "lie be", + "novel ty", + "tur moil", + "sul ly", + "spring break", + "hon ouring", + "cc g", + "ðŁı Ĵ", + "my little", + "ky c", + "pro ms", + "ðŁķ Ĭ", + "à ¨", + "bi ge", + "av ril", + "ðŁĩµðŁĩ °", + "mari on", + "as ants", + "sur ya", + "oc tag", + "luf than", + "ac ron", + "fayette ville", + "ti que", + "love s", + "en ca", + "de kalb", + "ta ver", + "de vote", + "aux iliary", + "joh annes", + "tread mill", + "ay an", + "qu r", + "donald son", + "cher yl", + "\" ....", + "s ven", + "kir sty", + "gun ners", + "ra dish", + "o ahu", + "v sky", + "i ble", + "con course", + "b ps", + "elo qu", + "ash ford", + "te bow", + "roblo x", + "ma da", + "dri ving", + "th day", + "spro ject", + "m ms", + "band ed", + ". !!", + "libr arians", + "flan nel", + "intoler ance", + "her al", + "ç µ", + "neme sis", + "list a", + "tar ak", + "cry pt", + "star plus", + "vish nu", + "sc ale", + "cr is", + "% ),", + "j illian", + "regg ae", + "pegas us", + "ol in", + "ip ment", + "man ic", + "l fc", + "godd ard", + "ite am", + "parl our", + "anch ors", + "lee minho", + "talla hassee", + "ant it", + "d ho", + "kid ney", + "y ash", + "batt led", + "az ad", + "gar is", + "faul kner", + "sni ff", + "papar azzi", + "ed m", + "phy llis", + "con tested", + "aa ay", + "se ca", + "k ton", + "vel ve", + "rain ier", + "for um", + "tam pab", + "ho sp", + "trac tors", + "ox fordshire", + "no tion", + "guang zhou", + "ðŁĺ ¯", + "ref ill", + "wednesday motivation", + "sli der", + "mukher jee", + "pr att", + "fon taine", + "alph on", + "af ar", + "ts i", + "pest icides", + "fi ends", + "mo cking", + "bra w", + "tran sat", + "do ses", + "co res", + "hom ophobia", + "docu menting", + "zlat an", + "con doms", + "s é", + "sun set", + "kun st", + "ton ga", + "ภª", + "v ation", + "sp ray", + "chow der", + "ra ps", + "palla dium", + "nor wood", + "music history", + "hoo ker", + "si si", + "osp rey", + "ph ys", + "conce ded", + "bob cat", + "ar mad", + "ze it", + "Ù Ħ", + "ðŁĺģ ðŁĺģ", + "mer idi", + "ðŁĩ· ðŁĩº", + "corn wall", + "! ),", + "touch downs", + "ze it", + "chal et", + "mm m", + "al che", + "gor illa", + "fo ss", + "ati ku", + "lumin ous", + "ivan ka", + "be ek", + "sta res", + "sw iss", + "âĿ¤âĿ¤ âĿ¤âĿ¤", + "scru bs", + "me ath", + "gusta v", + "jo gging", + "confe tti", + "as os", + "ers fc", + "breit bart", + "applic able", + "autho red", + "ya ho", + "h in", + "displac ement", + "j v", + "ðŁĮ¹ ðŁĮ¹", + "ot c", + "non profits", + "diec ast", + "gu sto", + "inte stin", + "c ages", + "me en", + "lu kas", + "moon ey", + "ðŁĺ ·", + "very day", + "tor ah", + "is sion", + "wa c", + "lever aging", + "ish able", + "cu se", + "le wood", + "may an", + "turn table", + "ju ice", + "tru sty", + "tu p", + "eti quette", + "supervis ors", + "stu n", + "gu zman", + "confe ren", + "ric o", + "fe ast", + "back ward", + "pol aris", + "mic he", + "jo g", + "h ing", + "field house", + "vel ing", + "sho cker", + "esc ence", + "ठ¾", + "vi be", + "anasta sia", + "mar ched", + "kill ing", + "Ķ ë", + "fe tt", + "exop lan", + "... (", + "snow day", + "lo h", + "ir ani", + "la khs", + "del a", + "po caly", + "boom ers", + "dictat orship", + "ac er", + "tur keys", + "quarter final", + "muskete ers", + "ðŁĴĽ ðŁĴļ", + "sf x", + "museum week", + "sc ala", + "ri sis", + "( ðŁĵ·", + "ãĢ Ĥ", + "z ies", + "bo eh", + "hu es", + "lu sci", + "dol a", + "impeach trump", + "roo d", + "don caster", + "tor re", + "hero es", + "fo yer", + "tar i", + "blur red", + "ke w", + "frank ly", + "dro id", + "ap al", + "Ð ¼", + "y af", + "bre t", + "par agu", + "cac ao", + "ðŁĻĮ ðŁı¾", + "ru e", + "head aches", + "shaw ty", + "char ley", + "pal er", + "go wns", + "correc tional", + "ðŁĺ© ðŁĺ©", + "breaking bad", + "ol ing", + "da p", + "endeav our", + "cit adel", + "tra d", + "incumb ent", + "medit ate", + "foo ted", + "ðŁĴ µ", + "shab bat", + "dayof the", + "wil lem", + "gal way", + "to red", + "marri age", + "f illion", + "sleeve less", + "aud itor", + "jin young", + "invin cible", + "kad una", + "a and", + "volcan oes", + "mon eti", + "indie gogo", + "buccane ers", + "ðŁijī ðŁı½", + "ãĢ Ĥ", + "lay ton", + "cuck oo", + "hu mber", + "buzz er", + "Ï ī", + "to re", + "stra ins", + "sto m", + "pa ine", + "s we", + "du ff", + "z ou", + "si mi", + "li pp", + "ur n", + "se agu", + "ðŁĶ ®", + "sun dae", + "hi c", + "ðŁĺ ¨", + "bull pen", + "u per", + "flyo ver", + "al dridge", + "glo bes", + "ali es", + "ken zie", + "ge es", + "y cle", + "sp lin", + "mag enta", + "j ha", + "bal u", + "gh orn", + "ti pper", + "wick er", + "taste of", + "con clave", + "ch ale", + "inv asi", + "cat er", + "dio xide", + "me gab", + "win n", + "at p", + "transform ative", + "nest led", + "hi g", + "bri dging", + "lil ies", + "chee red", + "bad dest", + "sc rolls", + "real is", + "dipl o", + "ðŁĶ «", + "conce ssion", + "prefe rences", + "explo des", + "er gon", + "introduc tory", + "ine au", + "ch af", + "som es", + "land rover", + "spir ation", + "sex y", + "sco recard", + "illustr ates", + "soul mate", + "wi en", + "inter disciplinary", + "fore casting", + "ent ities", + "glu ed", + "en lar", + "cur t", + "percep tions", + "boot leg", + "mi re", + "asho k", + "v az", + "hor ne", + "cal le", + "ac ulture", + "ther oy", + "night time", + "oc al", + "character design", + "ar mist", + "ðŁĺı ðŁĺı", + "yah oo", + "ac eae", + "to se", + "even to", + "sou t", + "nay anth", + "wh om", + "v are", + "ri gging", + "gen us", + "hi ve", + "com mands", + "sti e", + "day a", + "ethan ol", + "en f", + "hi fi", + "flu ence", + "cle mson", + "re invent", + "thermom eter", + "humor ous", + "emer ging", + "aci ón", + "ðŁĺĺ ðŁĺį", + "s ity", + "haw ke", + "accompan ying", + "t ility", + "ðŁĺ ª", + "re cess", + "protag onist", + "l ery", + "dun dal", + "int l", + "britt any", + "q bs", + "off the", + "marri ages", + "how to", + "viol ated", + "adel aide", + "wit t", + "lanc er", + "pak v", + "hu me", + "st ade", + "bra gging", + "ou tright", + "ad c", + "super st", + "real time", + "cu res", + "garden ers", + "ero ck", + "dale jr", + "ver o", + "bar tol", + "mo ti", + "mc fly", + "v pn", + "st ink", + "over rated", + "guer ra", + "e tis", + "ath ome", + "twd family", + "th ab", + "tn x", + "rafa el", + "family travel", + "x ley", + "sat anic", + "equ ations", + "ru dy", + "wal dorf", + "stan i", + "tu be", + "meas les", + "zimmer man", + "obli gations", + "i ously", + "bow ser", + "trans former", + "sho ppe", + "shak en", + "gh ouse", + "to d", + "ke tball", + "share holder", + "mar ca", + "kp mg", + "ak an", + "given chy", + "coast al", + "au th", + "roller coaster", + "mar ches", + "coordin ate", + "cine ma", + "apprentic es", + "par lor", + "mit o", + "men on", + "consider able", + "bar re", + "glo ss", + "enh ances", + "jaz eera", + "fal mouth", + "thra sh", + "stat en", + "k zn", + "eng el", + "samanth ap", + "flo ppy", + "sal om", + "ðŁıĨ ðŁıĨ", + "w ack", + "deliber ate", + "osc ill", + "herit ag", + "du sted", + "orni thology", + "pad dle", + "fer ns", + "bar un", + "cl ans", + "anticip ate", + "a ay", + "mat ically", + "é ĩ", + "tu mble", + "post man", + "unic ef", + "tro tter", + "op d", + "leaf let", + "ge ist", + "cease fire", + "scre ws", + "cre ation", + "wal nuts", + "longh orns", + "under statement", + "ab b", + "proxim ity", + "na x", + "un ity", + "turn pike", + "orda ined", + "dub step", + "chak ra", + "me ch", + "love her", + "look alike", + "donne in", + "vir on", + "Ù Ī", + "bang ers", + "vari ants", + "out dated", + "in ta", + "cri sto", + "sp elt", + "food and", + "f on", + "stefan i", + "margin al", + "hu tton", + "ti ara", + "tel ford", + "qu en", + "fair grounds", + "que tta", + "mikha il", + "heal er", + "v ball", + "ty re", + "under grad", + "gl end", + "hom ers", + "scri bed", + "main tains", + "po che", + "mis sal", + "mar ko", + "u as", + "á n", + "sh p", + "con vey", + "pad re", + "sab a", + "pu glia", + "madhu ri", + "pa xton", + "chap lain", + "n ago", + "ca si", + "... !!!", + "fli rt", + "sal eh", + "k are", + "di re", + "stam ped", + "extre me", + "ðŁĺĥ ðŁĺĥ", + "ho ppy", + "guadalu pe", + "advant aged", + "eu char", + "p low", + "un n", + "mac qu", + "port land", + "cla sh", + "pe s", + "lou bout", + "y p", + "keep ing", + "arca dia", + "fran kie", + "fi u", + "de th", + "encyclo pedia", + "si ze", + "inve sts", + "ðŁį ©", + "geo logical", + "fran ç", + "con front", + "ðŁĺ ¥", + "d ys", + "af m", + "tex an", + "graph ene", + "repost app", + "ac f", + "ur sula", + "gaz a", + "dd led", + "fu m", + "wsb tv", + "m be", + "fron tiers", + "chrono graph", + "ke s", + "inter faith", + "tab oo", + "spar ta", + "won do", + "flori st", + "em braces", + "ca w", + "no el", + "arch ers", + "ðŁIJ ·", + "roman o", + "ban an", + "sh akers", + "melo dies", + "geo thermal", + "se phora", + "ìļ °", + "оР´", + "pro c", + "hand shake", + "pan de", + "popul ated", + "slow down", + "hor tons", + "registr ations", + "un deni", + "lan ts", + "pas sover", + "thak ur", + "li ef", + "adhe sive", + "pe tal", + "micro scopy", + "memph is", + "confir ming", + "air drop", + "mesm er", + "perce ived", + "ming le", + "lifel ine", + "gh j", + "worcester shire", + "pas sions", + "ach er", + "el lar", + "ah o", + "firen ze", + "bar ang", + "letter man", + "hat field", + "lu cha", + "je ter", + "e shop", + "william s", + "horo scope", + "pre de", + "east bourne", + "dur ga", + "di version", + "al trin", + "seis mic", + "premi osm", + "nar co", + "ti r", + "ori g", + "or m", + "land fall", + "ci ous", + "lin do", + "max ine", + "x ico", + "tra y", + "os wald", + "c ba", + "ric otta", + "n cr", + "mar au", + "ภ²", + "gladi ator", + "ch ery", + "lun g", + "u me", + "po psic", + "lon ging", + "can als", + "ta ya", + "decentr alized", + "sho pp", + "pres sures", + "mahar aj", + "eti had", + "wal greens", + "succe ssion", + "sign aling", + "li g", + "staf fer", + "north korea", + "def ying", + "as ma", + "de g", + "peri meter", + "oak ville", + "m sk", + "balti more", + "rece ip", + "de ple", + "ðŁĺŃ ðŁĺĤ", + "jambo ree", + "> .<", + "rsp b", + "puni sher", + "consider ably", + "in tothe", + "pari sian", + "acceler ated", + "polye ster", + "low es", + "fr ying", + "sauté ed", + "mou ths", + "seychel les", + "ra x", + "go dis", + "dak ota", + "house wives", + "the me", + "mat inee", + "black bird", + "ye sung", + "pre fers", + "pelle gr", + "in ated", + "trun ks", + "stronger together", + "re pet", + "re pairing", + "ped als", + "toler ant", + "her r", + "dun ne", + "indic ation", + "decat ur", + "b tv", + "exhibit ors", + "ik on", + "friday motivation", + "bra gg", + "live tweet", + "al ves", + "womens art", + "foreig ners", + "wal lets", + "min dy", + "lan ey", + "bb in", + "tv miaw", + "lif ter", + "tar get", + "tam e", + "dr ou", + "astro photography", + "mp c", + "g pu", + "nord strom", + "fric tion", + "run off", + "lov able", + "sp nfamily", + "ext ingui", + "bloo dy", + "sch el", + "arti stry", + "sw ish", + "scar ce", + "ph ils", + "max im", + "pos sum", + "com promised", + "sty li", + "sc fc", + "is sa", + "birmin gham", + "sket ched", + "angel ica", + "ordin ance", + "je ts", + "conqu er", + "ðŁĺ IJ", + "online shopping", + "s ori", + "reason ably", + "nue stro", + "ar turo", + "ch l", + "benef ici", + "spho to", + "wel t", + "ni kk", + "ðŁ¤ ŀ", + "dan ao", + "for mid", + "as se", + "af irst", + "âľ Ĥ", + "gil lette", + "as sor", + "an onym", + "sel ca", + "fe mi", + "bear able", + "y and", + "ar mory", + "cre pe", + "celtic fc", + "bra vo", + "in expensive", + "de lec", + "ge cko", + "new market", + "snow flakes", + "kab ir", + "con tra", + "can ning", + "mor pho", + "gar wal", + "ðŁĴĥ ðŁı»", + "fight ing", + "mu tation", + "woo dy", + "ju gg", + "gr aces", + "premiosm tvmiaw", + "kenne dy", + "gu p", + "sa e", + "op ha", + "off spring", + "fini sher", + "bet ts", + "span ning", + "mar j", + "h one", + "sh ing", + "contin ents", + "samanthap rabhu", + "un related", + "l acy", + "explo sions", + "benjam in", + "sophi e", + "no ting", + "micro soft", + "as sen", + "a hoy", + "i ker", + "ho fer", + "mo e", + "ah madi", + "yan n", + "an ak", + "ma hi", + "be u", + "aha h", + "creep er", + "baahu bali", + "am at", + "pri ory", + "haw keye", + "deloit te", + "sko da", + "print making", + "assemb ling", + "mirac ulous", + "no ch", + "sw o", + "leg a", + "oper ates", + "border lands", + "eli e", + "stron gh", + "rep tiles", + "pir ate", + "un fold", + " ¯", + "qual comm", + "un predictable", + "ot r", + "rose wood", + "direc tional", + "counsel ors", + "corn ell", + "liber ated", + "j ad", + "ir regular", + "bulgar ian", + "high ness", + "vodaf one", + "sw ild", + "mini mize", + "gra zie", + "๠ĩ", + "r stats", + "stre ep", + "ome tric", + "humb le", + "lu mp", + "l ille", + "b ü", + "home depot", + "tripad visor", + "ki wan", + "a via", + "er z", + "ex ico", + "du f", + "blu men", + "mi zing", + "ar ma", + "in im", + "con stan", + "sor a", + "ju al", + "au n", + "tw ell", + "tren ches", + "her a", + "r k", + "po plar", + "recipe oftheday", + "ll an", + "bhu ban", + "short ages", + "ing don", + "bridge water", + "ðŁIJ ĺ", + "fortn ite", + "cam den", + "un cture", + "pro w", + "colon ies", + "t ks", + "n go", + "b hm", + "live pd", + "spl ace", + "sli ke", + "happye aster", + "ter rence", + "revol ver", + "j ed", + "yy yy", + "office of", + "m ts", + "exist ential", + "r ourke", + "explore bc", + "sse d", + "pri est", + "vix en", + "si ding", + "k pa", + "a har", + "ju ic", + "ob struc", + "foren sics", + "uk mfg", + "cancell ation", + "we ary", + "ab q", + "ele c", + "pri zed", + "deb ts", + "me zz", + "salv atore", + "m dc", + "gre tte", + "c gc", + "th on", + "snow storm", + "ts ch", + "cook ery", + "å ¹", + "wa xing", + "n acional", + "mur s", + "ra ve", + "cap es", + "ger main", + "dri pping", + "sub mitting", + "ome lette", + "iter ation", + "aj es", + "shim mer", + "fu eling", + "ðŁĩ§ ðŁĩª", + "li po", + "bo bble", + "un follow", + "islam ist", + "hi ber", + "cat s", + "agentsof shield", + "sen si", + "____ _", + "ster ia", + "inst al", + "ausp icious", + "har row", + "over land", + "femini sts", + "inst ant", + "char iot", + "blind ness", + "sp ed", + "sc arec", + "nu it", + "mini atures", + "ho seok", + "glo ck", + "fifa worldcup", + "e te", + "dis m", + "we iner", + "ex foli", + "ear ts", + "ภĶ", + "my art", + "man il", + "iss ant", + "form a", + "in cu", + "buffal ob", + "in tim", + "mc cul", + "anj ali", + "po po", + "un doub", + "hil a", + "fun gal", + "thank ful", + "fu tur", + "en dish", + "ren ds", + "th ar", + "she ff", + "ring o", + "nichol ls", + "io wa", + "po tom", + "cl ams", + "ãģ Ħ", + "acon f", + "stadi ums", + "di mp", + "di k", + "residen ces", + "do v", + "caric ature", + "seagu ll", + "kl m", + "confe ss", + "sla pped", + "cele b", + "turb ines", + "pp v", + "nur ture", + "el ab", + ".... .#", + "tu ff", + "de press", + "al far", + "amii bo", + "di spon", + "e wing", + "que er", + "friend s", + "for re", + "âĺ ¼", + "sw t", + "aqu arius", + "head liner", + "cur d", + "fi gs", + "o tters", + "love fl", + "kare em", + "go vegan", + "fri yay", + "consol ation", + "at ri", + "ì§ Ħ", + "âĺĿ ï¸ı", + "poly ne", + "gu ed", + "o ya", + "la us", + "intestin al", + "cam illa", + "scal p", + "pi r", + "leed s", + "horri fying", + "bore tum", + "dand elion", + "fer rer", + "ell ic", + "as x", + "so ren", + "re loaded", + "ale ague", + "navig ator", + "ine tte", + "add ams", + "al chemist", + "ak shay", + "dystop ian", + "awe c", + "n aya", + "al isa", + "ai led", + "ag or", + "avi ator", + "ali zer", + "smo bile", + "findyour park", + "cop ying", + "to ddy", + "sh ti", + "mon ger", + "cal houn", + "nap kin", + "break up", + "y atra", + "se thu", + "ric hi", + "eras mus", + "fer ry", + "am ore", + "prac tise", + "bo bo", + "power point", + "oo se", + "li ffe", + "chin a", + "sh ka", + "fad navis", + "du ane", + "war on", + "fal se", + "ðŁļ Ĥ", + "wa shes", + "disc ip", + "==== ====", + "g k", + "ab b", + "stub born", + "medi eval", + "p ci", + "ðŁį ª", + "maril yn", + "h yo", + "man di", + "cr i", + "prede cess", + "continu ation", + "om usic", + "s lat", + "wh al", + "mall ory", + "bon n", + "shen zhen", + "ca i", + "âĺ ĥ", + "sa fest", + "for wards", + "dra wers", + "bla sted", + "sle e", + "mor phe", + "mb ta", + "dumb ass", + "ÑĦоÑĤ о", + "alhamdulil lah", + "ec lub", + "al beit", + "heal ey", + "ayurve da", + "adverti sed", + "cro cs", + "itt les", + "bry son", + "be i", + "nj pw", + "honore e", + "fu sed", + "ðŁĶ ĺ", + "mul tin", + "n aga", + "de parts", + "ko p", + "kin o", + "jhar khand", + "ed na", + "ax le", + "mil ton", + "supremac ist", + "marrake ch", + "domin ic", + "tran script", + "] [#", + ": ).", + "wo c", + "sur rounds", + "o gil", + "leaf lets", + "co well", + "whe w", + "tru de", + "proli fer", + "succe s", + "sports man", + "con dom", + "po che", + "k up", + "imprison ment", + "{ }", + "scram bled", + "å Ľ", + "ka ine", + "cell phone", + "metam or", + "con i", + "remn ants", + "ee z", + "down pour", + "afterno on", + "exerc ising", + "ber ser", + "architec ture", + "wick low", + "m ns", + "is p", + "bo c", + "n iss", + "mn wild", + "stu mble", + "r si", + "lu ffy", + "sil en", + "dd ad", + "bul lies", + "haw ker", + "bb cc", + "scu ba", + "e pp", + "que ts", + "for aging", + "pal let", + "ha di", + "cinemato grapher", + "cat chers", + "to aster", + "k hi", + "lite coin", + "kid lit", + "amher st", + "maur icio", + "ip ad", + "mar malade", + "fe y", + "don nelly", + "g to", + "est as", + "cere bral", + "ant grasso", + "zz led", + "vir gil", + "swa pped", + "ðŁĺħ ðŁĺħ", + "no dapl", + "greate st", + "nhl bruins", + "fra ser", + "b mo", + "ane w", + ". âĿ¤ï¸ı", + "se gregation", + "remark ably", + "mccor mick", + "lo gger", + "er as", + "contrac ting", + "âłĢ âłĢ", + "yor ks", + "uku lele", + "touch screen", + "de cked", + "ben n", + "south wark", + "ra vin", + "nu mis", + "ðŁ¤ Ļ", + "ru t", + "gre co", + "eth ic", + "red neck", + "ar r", + "t cs", + "ih ri", + "ðŁĩ« ðŁĩ·", + "l k", + "inher ited", + "zy k", + "viadu ct", + "marty red", + "hi gu", + "ss n", + "be in", + "street style", + "fer gie", + "bank of", + "æĹ ¥", + "stake holder", + "exempl ary", + "cre ss", + "ess a", + "ero tica", + "intre pid", + "gom es", + "bra un", + "bethan y", + "bang tan", + "pulmon ary", + "m illing", + "doctor ate", + "trump russia", + "ठ°", + "s ani", + "bl att", + "pla u", + "depri ved", + "t le", + "ful ly", + "bour n", + "st ak", + "lufthan sa", + "kio sk", + "far oo", + "def y", + "bad an", + "ðŁĺĺ âĿ¤ï¸ı", + "rit z", + "tri sha", + "ran ds", + "middle sex", + "arab s", + "pro j", + "sport scenter", + "repe ats", + "iv f", + "bleed blue", + "as sure", + "o bs", + "territ orial", + "ele n", + "bever ley", + "ann ah", + "âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ı", + "z l", + "for good", + "science fiction", + "gla u", + "son ya", + "pri th", + "st weets", + "mix ers", + "mari o", + "ant elope", + "writing community", + "went z", + "den ham", + "be di", + "sf o", + "harley davidson", + "look book", + "immuno therapy", + "or phe", + "es ville", + "ed ged", + "tas k", + "sb ball", + "corro sion", + "kilom eters", + "co sting", + "play back", + "ke ke", + "di visi", + "u ter", + "re location", + "yel led", + "pen g", + "up beat", + "ser ve", + "âļ ł", + "hal en", + "stir ring", + "reh man", + "en v", + "schu macher", + "frag ment", + "alkal ine", + "sb k", + "resil i", + "share point", + "rol lover", + "tra sh", + "counter part", + "âĻ «", + "ob itu", + "à ½", + "ãĤ ¹", + "mul berry", + "ðŁİ Ĩ", + "auton omy", + "spra ying", + "nat l", + "love you", + "fran ki", + "nu k", + "esc ar", + "can teen", + "ali baba", + "de plor", + "mole cule", + "pu d", + "fort night", + "blon die", + "sp hin", + "portra yal", + "ta che", + "bu te", + "consi sting", + "freep alestine", + "c sp", + "im mort", + "d ns", + "ðŁĴ¥ ðŁĴ¥", + "tour de", + "coo king", + "archi val", + "ga thers", + "bit t", + "b anc", + "pre mature", + "snow ball", + "poetry day", + "lou dly", + "fug itive", + "ed ay", + "em ra", + "ðŁĩ¸ ðŁĩª", + "sci en", + "node js", + "jur gen", + "je ong", + "band ana", + "un is", + "fox sports", + "v andy", + "pro visions", + "wee p", + "tu k", + "i ko", + "h oun", + "zig gy", + "z r", + "fil let", + "bat a", + "tin k", + "con e", + "we want", + "k ilo", + "hor ace", + "sl t", + "sc t", + "stay tuned", + "victor ia", + "umb ria", + "att acker", + "ingham shire", + "fright ening", + "no ir", + "fr at", + "con tempt", + "lia ison", + "ho i", + "br ink", + "tr ill", + "ni agar", + "kick ass", + "dun das", + "not my", + "rho de", + "bu mble", + "no xi", + "fa g", + "spec tators", + "mancrush monday", + "jin ping", + "distr act", + "dais y", + "wal den", + "portra it", + "ar thistory", + "vol tron", + "ev el", + "is c", + "ac m", + "r ite", + "na o", + "de ported", + "swe ats", + "ru fus", + "lo bo", + "labor day", + "gam o", + "ihri thik", + "bl it", + "abdomin al", + "ãħ¤ãħ¤ ãħ¤ãħ¤", + "i it", + "e q", + "bu sy", + "allu arjun", + "un disclosed", + "de ton", + "pro create", + "ki l", + "ðŁİĤ ðŁİĤ", + "mitch ell", + "ki i", + "inherit ance", + "al p", + "jo burg", + "pat rolling", + "compul sory", + "un signed", + "ni am", + "l ga", + "eshop suk", + "tr illi", + "ma w", + "appreci ating", + "rock ab", + "mañ ana", + "an tal", + "mal vern", + "roy o", + "grand prix", + "sut ton", + "go ftheday", + "dig i", + "ãħĭãħĭ ãħĭãħĭ", + "t les", + "varan asi", + "erec ted", + "discip les", + "cont act", + "ðŁĺ µ", + "li d", + "⬠ĩ", + "scen tre", + "radi ator", + "ing tips", + "trans itions", + "thursday motivation", + "chem ical", + "separ ati", + "sal is", + "mi m", + "geo graphical", + "book fest", + "/ .", + "âľ ĭ", + "v ae", + "cur rie", + "ag garwal", + "acceler ation", + "the ses", + "lg m", + "u mass", + "pro portions", + "nat a", + "ani ans", + "ku ch", + "be acons", + "ap r", + "@ #", + "ðŁĴª ðŁı¾", + "nu ke", + "sher aton", + "ki o", + "ma kati", + "polit ico", + "mor ale", + "ì Ļ", + "econom ically", + "gg ly", + "ss en", + "pa stries", + "intern ships", + "vic ente", + "fanta ken", + "aveng ers", + "accu se", + "slee pover", + "indic ated", + "the dream", + "ster one", + "ren ders", + "fro st", + "ou i", + "gre gg", + "d ore", + "⾨ ⾨⾨", + "pu gs", + "sat y", + "nu mb", + "hems worth", + "tam i", + "la ssic", + "schi ff", + "igle sias", + "ag awa", + "] \"", + "re shi", + "game stop", + "divor ced", + "theat er", + "clau di", + "un conventional", + "prophe ts", + "ac in", + "twel f", + "tow ering", + "t ml", + "sc lerosis", + "k wan", + "ge ts", + "distur b", + "na ira", + "ener g", + "pir acy", + "pru itt", + "noti fied", + "hen na", + "bra m", + "ground water", + "bl s", + "opti mis", + "$ )", + "luci e", + "biz hour", + "fang irling", + "gr ills", + "or l", + "ver se", + "c ina", + "law less", + "artistson twitter", + "tele vised", + "marshmal lows", + "radio head", + "bar r", + "m fc", + "bre vi", + "mmor pg", + "g aya", + "âĸ «", + "sub titles", + "j t", + "disney land", + "to bago", + "nh m", + "groo ve", + "fi awec", + "\" /", + "ba o", + "scra bble", + "om ni", + "ff l", + "um c", + "si mba", + "ali er", + "ter rell", + "plu me", + "mi di", + "dig nit", + "co c", + "bru t", + "ad ata", + "alche my", + "d sm", + "ðŁĺĨ ðŁĺĨ", + "win try", + "spa res", + "cu er", + "conclu sions", + "to ys", + "od or", + "fl ann", + "gar vey", + "scrip tions", + "inspec tions", + "cat ap", + "ang lo", + "st louis", + "heim er", + "at ay", + "tr ich", + "en yc", + "chil ds", + "vent il", + "mont p", + "guiller mo", + "circu lare", + "z ell", + "mode led", + "craf tsman", + "al ina", + "stimul ation", + "cashe w", + "ju das", + "best of", + "to ire", + "susp ends", + "scol lege", + "real ising", + "by tes", + "bloo ds", + "as si", + "ðŁĴ ¿", + "o hs", + "ðŁį ĭ", + "scallo p", + "ठµ", + "gi fting", + "camo gie", + "wil kes", + "o zzy", + "ðŁ¤ ¤", + "ver onic", + "sav oy", + "deme tri", + "baby girl", + "ðŁĺį ðŁĺŃ", + "so x", + "cly de", + "induc tee", + "count down", + "self care", + "ठľ", + "vi ka", + "tor re", + "phd chat", + "pe ars", + "aw h", + "suff rage", + "le sn", + "admir ation", + "mp p", + "shark week", + "schul z", + "santor ini", + "clo ver", + "( *", + "stras bourg", + "ex iting", + "so yu", + "finger print", + "che a", + "ãĢ ľ", + "vin dic", + "song writers", + "so a", + "prou der", + "nam a", + "= ))", + "simple st", + "delici ously", + "gil les", + "u q", + "mn wx", + "ep p", + "sh un", + "ken nel", + "fall on", + "ðŁIJ £", + "sin d", + "tra gically", + "out es", + "modern ism", + "co ke", + "gy n", + "spi on", + "âĺ¹ ï¸ı", + "le am", + "compress or", + "apolog ise", + "twent yon", + "fan atics", + "âĻ »", + "sco tsman", + "sa wa", + "ko u", + "as er", + "ภļ", + "welter weight", + "phen om", + "twick enham", + "stri a", + "p out", + "ka z", + "gi am", + "cd p", + "ho y", + "emplo y", + "red mond", + "ภĦà¸", + "sm ere", + "trance family", + "proto cols", + "pie ce", + "lu iz", + "iter acy", + "carl s", + "united states", + "har med", + "phd life", + "ch aw", + "foot prints", + "l é", + "cho ker", + "z ana", + "sli pper", + "eric sson", + "insul ting", + "articho ke", + "advis ing", + "acquis itions", + "op or", + "mut ations", + "re ar", + "ॠģ", + "pod cast", + "wi ther", + "kun g", + "íĺ ¸", + "win slow", + "di apers", + "ðŁĵ¸ @", + "ec ker", + "col lar", + "hu ey", + "gi ro", + "mono gram", + "kas ich", + "si veness", + "malay si", + "arom atic", + "gre s", + "gali leo", + "u ji", + "rob b", + "dr m", + "none theless", + "as a", + ": >", + "lo a", + "l np", + "at work", + "ag t", + "laksh mi", + "pipel ines", + "id al", + "stre l", + "re all", + "chain z", + "stone wall", + "san sk", + "ðŁı ´", + "pied mont", + "hoste ss", + "ci u", + "t é", + "analy ses", + "wil helm", + "scott y", + "rw by", + "mosqu it", + "use mb", + "qu ins", + "ðŁij İ", + "tu cker", + "s conf", + "speci fications", + "psychi atry", + "broo kes", + "s ils", + "ol af", + "de to", + "co di", + "cli p", + "fil th", + "womancrush wednesday", + "go to", + "ang erous", + "be ale", + "w tc", + "paneli st", + "ne x", + "lar sen", + "emili o", + "tab leau", + "h itters", + "conce ived", + "americ ani", + "or tega", + "mar di", + "Ñ ĥ", + "pain tball", + "thir sty", + "new yorker", + "etis ation", + "go ss", + "we aker", + "u gh", + "tro ll", + "har ga", + "du al", + "ght ning", + "at ine", + "ðŁĺİ ðŁĺİðŁĺİ", + "cook out", + "pyrene es", + "po ss", + "authent ication", + "sports wear", + "yun ho", + "kir o", + "archi pel", + "shen ko", + "ren der", + "nov ation", + "divin ity", + "ðŁij £", + "su fi", + "humb ling", + "ge opol", + "devote es", + "wait ress", + "tr ough", + "py ro", + "i ba", + "bl ing", + "gra f", + "epilo ts", + "bt r", + "of tball", + "bas king", + "domin os", + "so om", + "r ath", + "sher yl", + "qu el", + "astronom ical", + "wel d", + "track list", + "sig nee", + "slee pless", + "com man", + "ch ron", + "summ on", + "pure michigan", + "cri spr", + "sli p", + "la gi", + "ra q", + "um u", + "thal ap", + "char med", + "scru mp", + "quad copter", + "ski p", + "peter sen", + "mun i", + "ðŁĮ ¾", + "mon aghan", + "tra ys", + "ick ed", + "canad aday", + "te gr", + "ï¿ ½", + "hot ness", + "heavy metal", + "ab ar", + "gop debate", + "az ul", + "spider man", + "sun flowers", + "ľ ë", + "web comics", + "bar d", + "Ð ²", + "nichol as", + "slu sh", + "ram an", + "mark ham", + "ffici al", + "ff ler", + "íĬ ¸", + "ple ss", + "anush ka", + "to to", + "sk aters", + "pro wrestling", + "compet es", + "ay ala", + "myster y", + "thr ills", + "mp g", + "independ ently", + "y ul", + "imper ative", + "formid able", + "tire less", + "st acking", + "ton gues", + "mal tese", + "pot ts", + "mat ti", + "char ting", + "chill out", + "super nova", + "ome o", + "sky sports", + "nu tty", + "ðŁĹĵ ï¸ı", + "ro han", + "insp ired", + "concier ge", + "ser ra", + "ma kk", + "gal at", + "chi pp", + "ye v", + "ì £", + "reim bur", + "op ul", + "kimber ley", + "i eee", + "bre men", + "ch itec", + "or in", + "nak u", + "bon kers", + "foo ty", + "emer gence", + "ðŁĨ ĺ", + "sti p", + "serge i", + "zo ey", + "ai me", + "wou ld", + "dy es", + "destin y", + "vinai grette", + "dri er", + "circulare conomy", + "an archi", + "ss r", + "sch el", + "cin er", + "gro om", + "determin ing", + "gar min", + "cal ais", + "incarcer ation", + "bu kit", + "no i", + "chelms ford", + "mckin ley", + "chi pped", + "belong ed", + "tu mors", + "str oud", + "mi i", + "influen za", + "wwen xt", + "tun dra", + "tele communications", + "cat sofinstagram", + "t ages", + "beat ty", + "o du", + "ml kday", + "oo per", + "dang le", + "ak ley", + "cru mb", + "anti gua", + "ti mbers", + "rou hani", + "ðŁĴª ðŁĴªðŁĴª", + "ha fi", + "... !!", + "w cs", + "coo p", + "sn c", + "lit res", + "ãĢ Ĭ", + "ha z", + "co z", + "k ant", + "green field", + "cur ti", + "y ale", + "flye agles", + "what soever", + "wor thing", + "rou lette", + "flyeagles fly", + "un da", + "a inted", + "stand ing", + "lusci ous", + "h pc", + "effic acy", + "ash land", + "me ghan", + "ky wx", + "n pr", + "bath tub", + "ac os", + "h ani", + "mar cor", + "man tis", + "da isi", + "bo ba", + "ab bie", + "mu til", + "vi al", + "spy der", + "po z", + "g ti", + "el fie", + "nigh tw", + "metro id", + "anton i", + "mad die", + "dh ry", + "dar lings", + "ten ds", + "taek wondo", + "atlan ta", + "me ow", + "chlo e", + "ãĥ İ", + "ym es", + "siber ia", + "k con", + "gu es", + "mar iner", + "fac il", + "azz le", + "[ ...", + "han nover", + "bav aria", + "vir go", + "te uk", + "u sps", + ") #", + "wall a", + "sam pson", + "need less", + "ver bally", + "hay ley", + "bow led", + "pi us", + "lam pard", + "ham string", + "vol vo", + "road safety", + "cho king", + "sor bet", + "a hem", + "healthy food", + "brai ded", + "horticul ture", + "cr ative", + "che ek", + "ad do", + "the force", + "ko ko", + "schiz oph", + "j ie", + "w ada", + "twentyon epilots", + "h bcu", + "pro ton", + "pau ls", + "lou isa", + "lat am", + "kyr gy", + "com pac", + "sd k", + "sap i", + "?? ?", + "liber alism", + "ep silon", + "ai den", + "w usa", + "spra yed", + "baske tball", + "kim ono", + "blue wave", + "ali as", + "ë§ Ī", + "mug shot", + "ce c", + "do gre", + "ad ora", + "ðŁĵ· @", + "kra kow", + "intrigu ed", + "exhau sting", + "astron omer", + "ven ison", + "lady bug", + "ci v", + "bra e", + "us m", + "bri be", + "acup uncture", + "pembro ke", + "ke ating", + "chi e", + "y ad", + "t si", + "sm i", + "see ding", + "gate shead", + "lis boa", + "gy p", + "canv ass", + "ðŁĶ´ âļªï¸ı", + "op i", + "ni r", + "soci etal", + "ly te", + "ati es", + "c sm", + "ar tery", + "al in", + "aka poor", + "abstr acts", + "âĢ¦ âĢ¦", + "teen wolf", + "ne we", + "travel gram", + "sentim ental", + "per ched", + "han del", + "ho ek", + "f ay", + "coordin ating", + "anim ate", + "man ian", + "effor t", + "jer ky", + "f ck", + "adri enne", + "ma bly", + "tra ding", + "my el", + "spi ro", + "sol a", + "stor ing", + "over drive", + "monday morning", + "dream team", + "pul se", + "bon di", + "ber nie", + "pgat our", + "tri poli", + "son am", + "plat t", + "âļ ¡", + "ag roup", + "îIJ Ĵ", + "inv ading", + "v cu", + "k ell", + "ñ os", + "un dead", + "pod casting", + "mercede sam", + "mana fort", + "cor tex", + "que so", + "impecc able", + "pal mer", + "wil doz", + "sport sc", + "guacam ole", + "dispen ser", + "cate gori", + "stun ts", + "per il", + "invit ations", + "dune din", + "xi e", + "achi eves", + "saf er", + "pre ds", + "ph an", + "knuck les", + "k ak", + "igno res", + "lovemy job", + "aru ba", + "ound ation", + "datac enter", + "co vert", + "gr ing", + "cou ple", + "ا ر", + "vol i", + "mc cle", + "arti sans", + "lu do", + "kal am", + "arom a", + "under taker", + "hu la", + "wiz kid", + "gu mb", + "god frey", + "bakers field", + "ker n", + "engine er", + "car ve", + "pal in", + "guaran tees", + "pe bbles", + "b ays", + "zi eg", + "fin k", + "â¬ĩï¸ı â¬ĩï¸ı", + "down pours", + "ro chelle", + "rasp berry", + "ðŁĺ ®", + "gra phies", + "stom p", + "caf es", + "ari zed", + "utt ar", + "cal vary", + "dri e", + "crusad er", + "bus an", + "tux edo", + "si u", + "seam us", + "cul tured", + "blan chard", + "town house", + "ge red", + "butter milk", + "flu ctu", + "roger federer", + "hel i", + "ðŁ¦ ĥ", + "u ous", + "ram esh", + "mu ppets", + "email marketing", + "ye ss", + "br ice", + "ri zio", + "pel o", + "donnein arte", + "u rable", + "inve stin", + "bump ing", + "raji v", + "sav a", + "thro wer", + "fore x", + "o hhhh", + "th rust", + "pull man", + "r fid", + "sep sis", + "le ed", + "fri ght", + "roun ding", + "ne b", + "ph ins", + "ai sha", + "utili zing", + "squ ats", + "gold smith", + "j ic", + "bo ks", + "vau s", + "i po", + "exclu sion", + "tari ff", + "po kes", + "min al", + "land s", + "en force", + "washington dc", + "or char", + "g x", + "mar ys", + "ey our", + "aussi e", + "bak ers", + "un popular", + "latin os", + "lar ge", + "pu tnam", + "bol o", + "wa de", + "pel o", + "di zz", + "ob struction", + "fla ppy", + "weare the", + "depend ence", + "pajam a", + "e te", + "y ann", + "e wan", + "disc la", + "a ay", + "kar ina", + "e ic", + "an trim", + "w soc", + "neg atively", + "kai do", + "fotogra fia", + "dh ru", + "colo ssal", + "mcle od", + "k wang", + "mani pu", + "ex hilar", + "us atoday", + "summer slam", + "co les", + "tapro om", + "unbeat able", + "de ma", + "tic ks", + "k ling", + "fil s", + "campaig ners", + "ภķ", + "brew ster", + "audu bon", + "qu ay", + "ch s", + "ki gali", + "d ler", + "strength ens", + "som al", + "sign ingday", + "gol ds", + "pig ment", + "orche stral", + "g q", + "lin kin", + "ðŁı ĩ", + "ta w", + "algar ve", + "ho v", + "ear le", + "gold fish", + "am ig", + "ex er", + "ben in", + "dru id", + "ðŁIJ ¸", + "she m", + "quat tro", + "mer cen", + "men te", + "incorpor ating", + "bon anza", + "state fair", + "en de", + "concep tions", + "e es", + "âĻ¥ï¸ı âĻ¥ï¸ı", + "d son", + "fire arm", + "orb ital", + "we h", + "multi p", + "fo b", + "requi em", + "p light", + "thou se", + "sa id", + "oc re", + "remem brance", + "n old", + "chi pping", + "be v", + "er t", + "ca thy", + "sy m", + "ri ggs", + "m ley", + "dialo gues", + "sl ender", + "how l", + "gau teng", + "wd w", + "to bi", + "smo kes", + "im plo", + "b pm", + "ad n", + "mom basa", + "cap sul", + "bloom field", + "artic ul", + "cle o", + "goog led", + "flu ffy", + "l ard", + "en zyme", + "ve sti", + "ibra hi", + "fl ame", + "e mea", + "out ages", + "dispro por", + "ble ak", + "an sel", + "ick er", + "st louis", + "stock market", + "good friday", + "sau lt", + "stal led", + "pro m", + "ep som", + "b é", + "the se", + "sau ces", + "me w", + "lit fest", + "pre d", + "re u", + "kar ak", + "si enna", + "ell in", + "bio technology", + "ï¸ıâĥ£ -", + "tac tic", + "sa in", + "por k", + "mon za", + "ka j", + "lu sh", + "compart ment", + "chang ing", + "shraddha kapoor", + "fo al", + "ar tem", + "cu ando", + "can ola", + "ori ente", + "me sse", + "d ited", + "br c", + "box er", + "bbc two", + "s st", + "ment day", + "em ing", + "de wey", + "kof i", + "âŀĸâŀĸ âŀĸâŀĸ", + "reali zation", + "smo l", + "tw ood", + "san je", + "flag staff", + "ber wick", + "cor set", + "can ary", + "whistle blower", + "et ched", + "com posing", + "squee zed", + "bow er", + "auto desk", + "ne h", + "mathi eu", + "ba ja", + "Å Ĥ", + "hy dra", + "da im", + "am eri", + "insi sted", + "mer lot", + "gar ros", + "heart news", + "gaine sville", + "cut ler", + "bo de", + "ðŁĺī ðŁĺī", + "lew es", + "scoun try", + "g sa", + "us u", + "cc m", + "god awgs", + "phara oh", + "cra e", + "mor ley", + "hyp noti", + "f ades", + "neur ons", + "fu zz", + "ing co", + "high landers", + "star k", + "vig ne", + "pac kets", + "amar illo", + "reu ben", + "insul ts", + "bas ic", + "vec tor", + "n me", + "ac ruz", + "tro s", + "transm itter", + "ðŁĺ ŀ", + "interpre t", + "ðŁĺ ²", + "pre quel", + "mc gowan", + "dis semin", + "ðŁĴĺ ðŁĴĺ", + "mascul inity", + "indie gamedev", + "ali ve", + "te t", + "pe tal", + "ema iled", + "ar med", + "ko o", + "he er", + "ba ird", + "super junior", + "metro polis", + "delav in", + "decl ines", + "stit utes", + "Û ģ", + "p tbo", + "g lan", + "cho res", + "e aling", + "chri ssy", + "ste mc", + "vi an", + "assassin ated", + "pron ounce", + "illeg als", + "discover y", + "cav ill", + "fri fotos", + "f al", + "so i", + "sabot age", + "t int", + "p dc", + "ðŁİīðŁİ Ī", + "ãĤ Ĭãģ", + "ji o", + "endeav or", + "in sig", + "commit tees", + "she arer", + "me tz", + "mar rying", + "h dd", + "g by", + "fre t", + "tri sh", + "pu l", + "scrip ted", + "sa ki", + "l w", + "ke ye", + "shim i", + "nan aimo", + "ca h", + "à «", + "tem pered", + "ici an", + "du gg", + "dish washer", + "air field", + "s rugby", + "gr inch", + "y st", + "r ms", + "mahat ma", + "lan kan", + "disc ar", + "dige stion", + "no des", + "l ls", + "om ic", + "gu tter", + "tis garh", + "feder ico", + "election day", + "bo he", + "master card", + "fire ball", + "âľ Ķï¸ı", + "oy ster", + "p ong", + "do k", + "en route", + "m vc", + "beat the", + "ali stair", + "shu b", + "sh aming", + "cherno byl", + "ghi bli", + "the s", + "pin ion", + "d bs", + "sal ts", + "ic tion", + "epi ph", + "nc pol", + "in convenience", + "whit ley", + "inspec ting", + "wood ley", + "wi ener", + "skil let", + "no les", + "m ca", + "h ina", + "a sha", + "willing ness", + "well ness", + "tam ed", + "show time", + "dis advantaged", + "ber nat", + "us n", + "mission aries", + "coun selling", + "arrog ant", + "quant itative", + "leg alization", + "ho dge", + "energye fficiency", + "cameron dallas", + "pos sessions", + "p bb", + "harris burg", + "v g", + "hindu ism", + "happy thanksgiving", + "fi b", + "re acting", + "tweeta picture", + "pol iti", + "mu ppet", + "hur rah", + "pac e", + "coast guard", + "guar ded", + "as am", + "par ry", + "fore very", + "x q", + "oom f", + "ke anu", + "j ind", + "ri st", + "customer service", + "sac red", + "ðŁĺ º", + "ton er", + "occur rence", + "mat u", + "val dez", + "red d", + "is ak", + "power rangers", + "pe asant", + "raj ini", + "abra ham", + "e mil", + "car do", + "tr il", + "hair styles", + "obsole te", + "sam pler", + "direc tive", + "delavin kisses", + "ver ton", + "glo s", + "sp ay", + "paler mo", + "com ets", + "man ziel", + "chicag of", + "ski pped", + "pic torial", + "h ant", + "b mi", + "a ol", + "re opens", + "pad dling", + "devo s", + "fra ud", + "bas eline", + "que ues", + "sp ired", + "sn are", + "eu ve", + "descri ptions", + "daisi es", + "ca ching", + "gall eria", + "tri mmed", + "stin o", + "recy cla", + "ic ular", + "bir ken", + "raw lings", + "fli x", + "chic as", + "b gt", + "lik eli", + "argy ll", + "thel ove", + "ga ston", + "bl anca", + "ha k", + "f one", + "sailor moon", + "h aci", + "ima c", + "fl yn", + "de can", + "bel les", + "ap ic", + "zo g", + "taun ton", + "con stance", + "lasag na", + "ker nel", + "in ka", + "har bor", + "collec tively", + "calcul ated", + "av ille", + "shil pa", + "pur du", + "gi mm", + "fun er", + "a est", + "pembroke shire", + "nighting ale", + "n unes", + "hyper tension", + "hu bert", + "sli ders", + "infer tility", + "comm ended", + "transat lantic", + "metr ical", + "!! @", + "Å Ł", + "ss g", + "bac ca", + "inver ted", + "fun factfriday", + "it ans", + "albu m", + "acqu ainted", + "ri er", + "whel an", + "sar ab", + "mu e", + "snoo ze", + "pi ff", + "agre eing", + "sp itting", + "jer maine", + "n ye", + "âľı ï¸ı", + "am bush", + "ze ph", + "con greg", + "univers ity", + "s app", + "wann abe", + "pat rice", + "ib d", + "do glo", + "fri dges", + "sun d", + "king ston", + "ar gon", + "kam en", + "hardro ck", + "ds ley", + "do lores", + "ì °", + "ota ku", + "pi ping", + "be having", + "âŃIJï¸ıâŃIJï¸ı âŃIJï¸ı", + "blue bird", + "an sari", + "teapo t", + "fire work", + "cro p", + "log ans", + "ty ped", + "thick ness", + "ig ers", + "c fp", + "dys functional", + "contra sting", + "et ty", + "aston martin", + "tx st", + "dra grace", + "at tributes", + "marath on", + "manu scripts", + "john stone", + "ðŁĺ± ðŁĺ±", + "bo er", + "ay u", + "aru gula", + "poo rest", + "con du", + "assu mption", + "anag h", + "no h", + "delav in", + "sit ter", + "g ö", + "mor ow", + "kick start", + "com i", + "gl acial", + "ghe ad", + "ba in", + "ker shaw", + "en dof", + "fre ud", + "om at", + "i af", + "hu g", + "sign up", + "each other", + "defin ite", + "tu bing", + "shak ira", + "ðŁijı ðŁı½", + "uu uu", + "sw in", + "sham bles", + "ol as", + "sk ell", + "brit ain", + "kn w", + "clu tter", + "om y", + "j ens", + "hang ed", + "city scape", + "scra ps", + "un locking", + "dead liest", + "er no", + "breast cancer", + "a it", + "inspec t", + "fu ri", + "ðŁĴ Į", + "ku d", + "ju le", + "or ah", + "mi ds", + "m dt", + "bur gring", + "r attle", + "pu sa", + "stal k", + "cle ans", + "iss ance", + "z ek", + "worth it", + "nam eis", + "musko ka", + "council man", + "urban art", + "bar rac", + "un solved", + "tu l", + "g ita", + "white board", + "soy beans", + "em ent", + "cont i", + "saturday motivation", + "conveni ently", + "doc king", + "t ado", + "âı ©", + "sp ino", + "puppy love", + "po f", + "fabric ated", + "robb ers", + "adop ts", + "ti fied", + "kk r", + "indulg ence", + "notic eable", + "macqu arie", + "chap el", + "sensu al", + "ki ko", + "melan oma", + "lore tta", + "li ance", + "ab en", + "sp lus", + "ga al", + "ac ele", + "lib dems", + "compar isons", + "ðŁĮ µ", + "rhy thms", + "mer y", + "en capsul", + "nap ier", + "ðŁijĮ ðŁijĮðŁijĮ", + "ðŁij IJ", + "plat z", + "fre sno", + "re formed", + "ran bir", + "el it", + "the best", + "bhu shan", + "vin nie", + "impro vised", + "s ittin", + "re created", + "e ba", + "ec ker", + "ac rob", + "pon te", + "cor d", + "gi ddy", + "eur usd", + "fe ver", + "intu ition", + "gar i", + "dum mies", + "bud weiser", + "amend ments", + "te tra", + "sch nit", + "ay as", + "mar ys", + "ci st", + "k ani", + "ker mit", + "ðŁĺ±ðŁĺ± ðŁĺ±", + "tin ker", + "strol ling", + "di visional", + "niger i", + "omin ous", + "menstru al", + "kar ab", + "k hy", + "bw fc", + "pan handle", + "l illi", + "well er", + "stra pped", + "son the", + "transfer ring", + "ethe real", + "sne aks", + "ru dol", + "gab les", + "jac king", + "cin code", + "for tune", + "canadi ens", + "con for", + "ab normal", + "frank lin", + "tit a", + "mu la", + "persi st", + "cu ties", + "ki el", + "ðŁĩ± ðŁĩ", + "her mann", + "aw k", + "fi asco", + "ko to", + "we ta", + "hi ker", + "budd y", + "preven tive", + "mcgra w", + "game boy", + "forsy th", + "top shop", + "si ob", + "sad h", + "in tram", + "follow art", + "so aps", + "dragon ball", + "ou x", + "morri son", + "๠ĥ", + "lu bric", + "adul thood", + "morri sons", + "âļ łï¸ı", + "her mo", + "ta ka", + "stall one", + "mis use", + "team gb", + "ra gha", + "con fined", + "at y", + "hom ophobic", + "nw o", + "sky news", + "ho ya", + "ac rosse", + "wi iu", + "pur ée", + "jed dah", + "ðŁ¤ §", + "advis ers", + "ph ine", + "an is", + "scrump tious", + "ë° ķ", + "c ke", + "vin y", + "ter m", + "s dc", + "o do", + "home school", + "vas c", + "leop ards", + "debor ah", + "illic it", + "cur ran", + "as roma", + "nau ght", + "mar ig", + "brand i", + "em p", + "ðŁĺį ðŁijĮ", + "î Į", + "su spend", + "lu z", + "initi ation", + "sch aft", + "jensen ackles", + "craw ler", + "post doc", + "des ks", + "trail blazer", + "den omin", + "tri x", + "no ise", + "po et", + "± ï¸ı", + "s mug", + "vol atile", + "proof s", + "pharmac ist", + "sardin ia", + "mash able", + "kim chi", + "co ed", + "schal ke", + "doo dled", + "c sw", + "sh ur", + "ro x", + "do k", + "chris brown", + "mathemat ician", + "ab ound", + "ang elic", + "rock ford", + "d ole", + "yor kers", + "ms n", + "g man", + "xavi er", + "bor rowing", + "mark ings", + "longh orn", + "k ja", + "diver ted", + "mm it", + "euph oria", + "ay yy", + "te a", + "pa h", + "ck i", + "un cut", + "li ven", + "ky ung", + "fan art", + "mer ing", + "red ding", + "amo vie", + "gri di", + "c thulhu", + "schol arly", + "ju dah", + "th bewithyou", + "eu calyp", + "ðŁIJ ķ", + "hert fordshire", + "cour troom", + "by u", + "auc tioned", + "ple ase", + "mar cia", + "ê° ĵ", + "succe eded", + "el as", + "arvin d", + "t lot", + "saig on", + "re tt", + "ra kesh", + "fd ny", + "as en", + "se bring", + "gladi ators", + "you know", + "v lad", + "gol a", + "par ap", + "ÑĢ и", + "sab cnews", + "one team", + "oh l", + "sun e", + "ri j", + "cd c", + "star gate", + "run down", + "plat o", + "ph c", + "chat ter", + "ra viol", + "mn f", + "mand ala", + "li et", + "ภķ", + "mari a", + "hun gover", + "consoli dation", + "fer rell", + "tradition al", + "ilove art", + "gal ap", + "ðŁı Į", + "que zon", + "espa ña", + "ðŁĩ¨ðŁĩ Ń", + "ho bby", + "steam boat", + "mali gn", + "guil lau", + "pro hi", + "its me", + "íĥ Ģ", + "in scription", + "al z", + "mari an", + "k ade", + "mm on", + "adju sting", + "ne sts", + "intern ally", + "ci r", + "vik ram", + "mal ala", + "k ph", + "fel icia", + "the real", + "cap tivity", + "at is", + "marcor ubio", + "kale ido", + "che v", + "mano j", + "le more", + "gent ri", + "vi ps", + "tro pe", + "\" âĢĶ", + "pair ings", + "mal nutrition", + "fr ay", + "desig nation", + "brun omars", + "az e", + "tor rential", + "pan zer", + "ga il", + "under the", + "the ological", + "schizoph re", + "dazz le", + "freder ic", + "mo par", + "ad illa", + "so ggy", + "ra un", + "medi ocre", + "colo rec", + "i fe", + "p inst", + "blu ef", + " ²", + "world water", + "gir oud", + "clar inet", + "ad olf", + "tar antino", + "receip ts", + "assu mp", + "ðŁij Ł", + "coffe es", + "âľĬ ðŁı¾", + "du plex", + "s of", + "r x", + "lin o", + "timber wolves", + "pan dit", + "mo tm", + "e ga", + "ay ama", + "ach s", + "outsi der", + "ll en", + "co er", + "til ly", + "cheese burger", + "ma ds", + "ple dis", + "emp ty", + "national parks", + "az iz", + "p mi", + "jun kies", + "f ener", + "sq n", + "è s", + "gener ation", + "cleop atra", + "bhuban es", + "mosqu es", + "ty free", + "popp ins", + "tw c", + "or well", + "n age", + "ka whi", + "hol low", + "dal ai", + "¨¨ ¨¨", + "ou ro", + "m health", + "gi on", + "az o", + "vis as", + "reneg ade", + "re ic", + "w sop", + "ðŁĴļ ðŁĴĽ", + "e chel", + "tox icity", + "mü n", + "bun k", + "stimul ating", + "asth our", + "\\ '", + "ep h", + "ende mic", + "cn bc", + "shrin king", + "peabo dy", + "michel angelo", + "can yon", + "wal e", + "su mi", + "si ders", + "inu it", + "? .", + "profession alism", + "dr acing", + "plat oon", + "p ons", + "out bound", + "maple leafs", + "de sol", + "cen cy", + "a than", + "ver ma", + "ru bbing", + "ok an", + "ðŁij ł", + "mull ins", + "authent ic", + "Å į", + "alman ac", + "ga ia", + "bb q", + "on imo", + "ke h", + "ty a", + "tou ts", + "y av", + "re posit", + ", .", + "wi ght", + "se eyou", + "cal lof", + "done sia", + "bar gaining", + "gr anth", + "sd su", + "amphi theater", + "p su", + "re watching", + "wine tasting", + "peak district", + "dete cting", + "thur man", + "phe e", + "èª ķ", + "u mich", + "re r", + "sculp ted", + "go le", + "name sake", + "ðŁĶ ģ", + "serv icing", + "bau gh", + "pu gh", + "pen cil", + "dar th", + "munch kin", + "at orium", + "ten ers", + "sun y", + "rolling stones", + "mag ing", + "star rer", + "i dris", + "fe instein", + "ag ron", + "âĺºï¸ı âĺºï¸ı", + "supervis ed", + "chamele on", + "aggre gate", + "succe ssive", + "mo gul", + "inst yle", + "pol dark", + "custom e", + "ohio state", + "ha ya", + "ci des", + "broker age", + "angel ou", + "fifa wwc", + "de forestation", + "al ton", + "pam ph", + "hu gged", + "ho bo", + "change able", + "ku ber", + "bur roughs", + "demon etisation", + "cape cod", + "vers atility", + "or ice", + "le ila", + "womenin science", + "tu a", + "he dges", + "embarrass ment", + "ali fe", + "so ars", + "ni ghter", + "hy mn", + "gi pp", + "chas u", + "tech s", + "ni all", + "k illa", + "hi ka", + "cam els", + "valu e", + " ¢", + "sc oops", + "mah moud", + "clu sive", + "adri ana", + "pac o", + "oz il", + "un as", + "transl ations", + "whispe rer", + "s bi", + "bu xton", + "bio tics", + "indi ffe", + "ken ney", + "k lar", + "et ching", + "barra best", + "inst ability", + "se ine", + "vo tel", + "blo gged", + "whis key", + "my space", + "t ant", + "lan dia", + "give back", + "illu s", + "aw ak", + "ac ab", + "f bloggers", + "cloud computing", + "blat ant", + "syri ans", + "band ra", + "sty n", + "an em", + "ke ted", + "kar thik", + "barun sob", + "pin ot", + "gu bernat", + "gay e", + "arti ste", + "i fied", + "conven tions", + "hu an", + "geni uses", + "eeee ee", + "fol ly", + "somer ville", + "pride month", + "ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸", + "chemo therapy", + "paul s", + "bak ar", + "ìĦ¸ë¸ IJ", + "taiwan ese", + "fol lo", + "c ss", + "re ign", + "nn nn", + "fla un", + "catastro phe", + "iti es", + "frag ments", + "extre mists", + "ym oun", + "car men", + "eze kiel", + "conne cting", + "se h", + "man ta", + "remodel ing", + "we ymouth", + "at oms", + "ce m", + "ne well", + "lu mi", + "the open", + "mo c", + "mili band", + "g land", + "z shq", + "mag gie", + "mani acs", + "m sp", + "ad y", + "cre ams", + "le anne", + "e sta", + "py g", + "af finity", + "pray er", + "dun bar", + "ligh troom", + "ac adi", + "wyn onna", + "roman tic", + "state dept", + "sick le", + "wh os", + "lam o", + "et our", + "fin ity", + "shru b", + "shar pen", + "pun dit", + "ed on", + "af ore", + "mar s", + "jeff ery", + "ter ps", + "medal list", + "kath arine", + "accu sing", + "ta z", + "roy d", + "from home", + "confron tation", + "alle gh", + "ðŁijī ðŁijī", + "refresh er", + "ran veer", + "never land", + "jo jo", + "lu crative", + "en am", + "ca ver", + "pa edi", + "man jaro", + "flu ids", + "the ssal", + "oppre ssed", + "mu ss", + "joh anna", + "Ø ®", + "cn g", + "buil dthe", + "sett les", + "s ith", + "fu ego", + "cl amp", + "ar ag", + "pay er", + "ted x", + "mand y", + "inter stellar", + "fr c", + "ch and", + "b cc", + "mo lo", + "len til", + "johan sson", + "grims by", + "nature lovers", + "ðŁļ¨ ðŁļ¨ðŁļ¨", + "shin de", + "x in", + "international dayof", + "transiti onal", + "sat a", + "cad dy", + "wo d", + "if u", + "ha ys", + "holl yo", + "j ang", + "ir c", + "co im", + "grad able", + "\" \"", + "ðŁį ´", + "ঠ¾", + "a el", + "n yo", + "west lake", + "time out", + "sof i", + "phenom ena", + "cultiv ation", + "ag no", + "un armed", + "so t", + "con j", + "gen o", + "royal navy", + "nutriti on", + "fair mont", + "ti relessly", + "sn g", + "re ty", + "mic a", + "lu cent", + "slo ane", + "droo l", + "riz al", + "od ell", + "critici zed", + ". '\"", + "la ze", + "deser ted", + "co der", + "pra s", + "l illian", + "itiner ary", + "dav y", + "an ap", + "whi pping", + "hobo ken", + "kare ena", + "çľ Ł", + "vi us", + "ter n", + "nan tucket", + "mis understood", + "bu laga", + "st ant", + "chin ook", + "z am", + "reli es", + "d ss", + "ed mond", + "sket chy", + "m ell", + "fe x", + "rec tor", + "dist ill", + "day dream", + "wine maker", + "ri pley", + "billion aires", + "hel ene", + "ati f", + "cul prit", + "bertr and", + "wou ldnt", + "ma pped", + "v ak", + "gla dly", + "parliam ent", + "kidlit art", + "ware ness", + "goli ath", + "âĨ ĵ", + "view point", + "tat ted", + "fu ls", + "dor sey", + "ang lers", + "li ds", + "ki ya", + "bow les", + "be h", + "b ite", + "compati bility", + "ance stral", + "pro x", + "beha ved", + "gubernat orial", + "ch field", + "sab an", + "z h", + "teen y", + "shibu ya", + "holli day", + "pan cy", + "âĿĦï¸ı âĿĦï¸ı", + "seun gri", + "? ,", + "ðŁĩ¦ ðŁĩ·", + "im itation", + "impac tful", + "any i", + "gene vie", + "añ os", + "bate man", + "gli der", + "af ar", + "ra sheed", + "effor tless", + "sh war", + "dach sh", + "er un", + "at os", + "kin i", + "ch d", + "kha ki", + "k lin", + "felici dades", + "bel o", + "as l", + "to ppers", + "fin ley", + "stac ey", + "rigor ous", + "kar ting", + "le ppard", + "car michael", + "be ret", + "c se", + "ak hi", + "mer ingue", + "ab an", + "ha ke", + "ger i", + "er jee", + "re sto", + "comm anders", + "pr it", + "fl or", + "ad ven", + "ex termin", + "remain der", + "å IJ", + "es g", + "martin o", + "lulla by", + "| @", + "mi gn", + "in store", + "big bang", + "cor di", + "cau ley", + "ante bellum", + "dg ate", + "cro ck", + "span dex", + "scaf folding", + "ore os", + "ê°ĵ ìĦ¸ë¸IJ", + "pom ona", + "ma uro", + "uni versi", + "re mi", + "af ootball", + "t ant", + "sm alls", + "ne h", + "worl do", + "tropic al", + "mor ph", + "jav elin", + "gla r", + "arqu itec", + "reminis cent", + "tu bs", + "spide y", + "make u", + "syl la", + "progressi ves", + "blo t", + "shor ten", + "keep in", + "ch ak", + "ang st", + "super food", + "decad ent", + "ston y", + "neuro logical", + "ar boretum", + "ann ak", + "fe ma", + "per cu", + "dis respectful", + "small biz", + "lo x", + "co om", + "c sc", + "bs bi", + "pre valence", + "him ss", + "esp an", + "mo ga", + "fr ampton", + "sky map", + "mas se", + "levi athan", + "( ).", + "noctur nal", + "car ameli", + "ang or", + "amne sia", + "outsi ders", + "she alth", + "rhin o", + "ant ag", + "ag io", + "ðŁĴ° ðŁĴ°", + "take me", + "kab addi", + "c si", + "m sh", + "coch rane", + "thessal oni", + "sil a", + "ha us", + "du sting", + "obe se", + "mack lemore", + "mani sh", + "len in", + "m dc", + "gro wn", + "shef field", + "s rs", + "ke le", + "car son", + "ch um", + "dah lia", + "can tore", + "opp o", + "how ling", + "cyber crime", + "sur realism", + "sc ran", + "fa iz", + "thre n", + "rac ists", + "r out", + "pk not", + "se mana", + "sin i", + "mc cull", + "ma chi", + "alfon so", + "y b", + "sar dar", + "kend rick", + "den g", + "reci pro", + "on f", + "doom sday", + "bri bery", + "custom iz", + "art is", + "c pi", + "ðŁĻĪ ðŁĻĪ", + "sla va", + "let te", + "en s", + "âĿ¤ï¸ı ðŁĺĺ", + "cra yon", + "ad an", + "tr c", + "migr ate", + "simp son", + "row ers", + "king sley", + "farmers market", + "shee han", + "ne phe", + "bor non", + "car ton", + "mic key", + "all ure", + "u lu", + "sli pknot", + "heb do", + "gui do", + "dog celebration", + "online marketing", + "acceler ating", + ") ..", + "origin ated", + "macar oni", + "ed tech", + "out field", + "mit z", + "disc us", + "adverti ser", + "man or", + "ha shi", + "descri p", + "cap ita", + "ful bright", + "recep tor", + "con n", + "con ey", + "spion age", + "r attle", + "pre st", + "u li", + "blog post", + "acker ay", + ") âĢ¦", + "red velvet", + "mat th", + "inspir ing", + "b sd", + "ker ri", + "po con", + "mil lar", + "re pur", + "accent ure", + "ä ¹", + "ram bo", + "ragnar ok", + "dele ting", + "british museum", + "pat ory", + "leip zig", + "flori an", + "sci fi", + "in ers", + "br ate", + "yo y", + "melis sa", + "ab er", + "ma sa", + "po te", + "mosquit oes", + "transpl ant", + "r pa", + "; ))", + "bast ille", + "yl an", + "joye ux", + "melo dic", + "cap tions", + "atri st", + "roch dale", + "gott i", + "pew die", + "cuties aturday", + "who is", + "aqu aculture", + "tiv a", + "sp el", + "he ss", + "ha ji", + "fred die", + "co per", + "brand o", + "v k", + "photo book", + "* ,", + "my dayin", + "micha ela", + "brune i", + "sr ini", + "in te", + "Ä ±", + "de ol", + "d fc", + "separ ately", + "bun d", + "ve sts", + "to c", + "me ck", + "rein forced", + "constra ints", + "car roll", + "sq ft", + "re ver", + "cam per", + "bird man", + "in action", + "gener ators", + "triumph ant", + "pe sts", + "o vo", + "gy pt", + "al amo", + "sc aled", + "suresh pp", + "sd n", + "is mo", + "gi os", + ") @", + "justic eleague", + "restaur ant", + "gab i", + "den gue", + "next gen", + "exemp li", + "ap ex", + "inspir ational", + "down side", + "kid z", + "u pl", + "et na", + "alvar o", + "fel dman", + "bar net", + "m ha", + "es ch", + "bloo ded", + ">>>> >>>>", + "kan i", + "ho fficial", + "casablanc a", + "bir ds", + "ty ga", + "sw amp", + "o day", + "new castle", + "nb ap", + "ci sion", + "cho ols", + "af lo", + "ne p", + "mon ton", + "ak b", + "super model", + "down time", + "th os", + "sc wx", + "snoo py", + "ag greg", + "yo ke", + "nor cal", + "we tt", + "prolon ged", + "me tast", + "beat er", + "f ta", + "t lap", + "disgu sted", + "y h", + "voice over", + "itch y", + "ip c", + "ðŁİ ¾", + "phe asant", + "stra its", + "ram pant", + "j g", + "fer til", + "assu res", + "fortun es", + "sal inas", + "liz ards", + "kett le", + "i bs", + "cyn thi", + "he g", + "mc cr", + "soccer oos", + "happen ings", + "cor den", + "ðŁĺĤ ðŁijĮ", + "t ches", + "egre t", + "wolver ines", + "congratul ated", + "ho gg", + "bott ling", + "wr i", + "fer ri", + "bo sch", + "af ire", + "og den", + "s jo", + "j dm", + "sv t", + "con tex", + "tol lywood", + "min k", + "me se", + "super sonic", + "op oulos", + "å ¸", + "âĶ ģ", + "knuck le", + "gu ise", + "gam i", + "chu cky", + "z inger", + "radi al", + "compla ined", + "bo da", + "fe tal", + "discipl ines", + "cor ro", + "ðŁĩ®ðŁĩ ¹", + "op ted", + "filtr ation", + "ad nan", + "em cee", + "mi stre", + "insom ni", + "fer gus", + "tra jec", + "on don", + "med tech", + "tanger ine", + "madra s", + "gru e", + "cab s", + "z hu", + "sureshpp rabhu", + "insul ated", + "day swild", + "pp m", + "band ai", + "v day", + "s ff", + "squ id", + "lo thing", + "not dead", + "expre ssive", + "cu ll", + "ala stair", + "x u", + "up front", + "fish ers", + "en es", + "um d", + "dis missal", + "sti er", + "sel s", + "lu st", + "re active", + "prote ster", + "eyel ashes", + "al im", + "goo de", + "gre eng", + "da ir", + "com pen", + "anush ka", + "proto typing", + "ma pu", + "bear ings", + "ðŁIJ Ł", + "for me", + "bsbi botany", + "timo thy", + "out skirts", + "am bed", + "are tha", + "wend ell", + "stre aks", + "ni m", + "k pk", + "sne e", + "fit ter", + "quo ta", + "p ate", + "win ning", + "ðŁį Ń", + "sho pping", + "ma inst", + "cul ver", + "ste vie", + "mcfad den", + "counter parts", + "gren fell", + "fol som", + "dor set", + "tech crunch", + "⬠ħï¸ı", + "tip tuesday", + "us l", + "tre x", + "geor gie", + "ranveer official", + "lic ks", + "se wn", + "k f", + "' âĢ¦", + "jap s", + "p ate", + "orth op", + "fe sta", + "stra s", + "mon tal", + "hammer smith", + "fore most", + "wido ws", + "mad re", + "ite z", + "mito chondri", + "lig ans", + "z ona", + "cari bou", + "m ss", + "andre i", + "weather channel", + "gh c", + ": ...", + "ta ft", + "awe ather", + "al isation", + "bru tal", + "bliss ful", + "nik ola", + "mal icious", + "q m", + "mpg vip", + "bro die", + "bl itz", + "applau d", + "dri bb", + "v ague", + "dog go", + "transl ating", + "interpre ted", + "hat ched", + "ge tyour", + "benefici aries", + "spar ring", + "caes ars", + "aw illiams", + "la hat", + "bro ke", + "ti mp", + "virtu es", + "rel ying", + "pie tro", + "k tn", + "ici sts", + "pab lo", + "lou i", + "a ag", + "pn pp", + "cha st", + "pul ses", + "fini sh", + "usair force", + "type writer", + "thomp son", + "dog s", + "ut to", + "ãģ į", + "sand al", + "new ly", + "do ge", + "z w", + "wan kers", + "ne gr", + "mu cha", + "determin es", + "black fish", + "sk unk", + "mu ps", + "instru ment", + "phy to", + "daysto go", + "skin ned", + "hai der", + "con ten", + "ðŁIJ¾ ðŁIJ¾", + "we iler", + "undoub tedly", + "chair ing", + "wall is", + "sh ard", + "zind abad", + "adul t", + "absor ption", + "pre sto", + "deplo ying", + "drum mond", + "battle front", + "seag ulls", + "how dy", + "juda ism", + "des de", + "part ition", + "âľ Ŀ", + "no logy", + "national bestfriend", + "lesn ar", + "film fare", + "co asts", + "christen sen", + "ac an", + "mb u", + "co pped", + "ru bble", + "sw c", + "fun nier", + "far ther", + "where as", + "nano technology", + "with stand", + "pil low", + "bow ers", + "to pe", + "it ly", + "con fit", + "ma kar", + "comfor ts", + "bo sh", + "cli pper", + "bal la", + "sti k", + "mil b", + "safe guard", + "musi que", + "eas port", + "ya z", + "pad ded", + "bad er", + "fore ign", + "chop in", + "archi ve", + "o ka", + "tran sporting", + "tml talk", + "aj it", + "consequ ence", + "sc roo", + "ff o", + "collabor ated", + "pug chat", + "ye mi", + "jav ed", + "au burn", + "o of", + "ma w", + "sau cer", + "miti gate", + "i les", + "evangeli st", + "ter ie", + "re cl", + "indic tment", + "cat a", + "bright ness", + "may the", + "whim sical", + "un lv", + "key word", + "cu min", + "med way", + "west world", + "tra w", + "im posing", + "form ity", + "coul ter", + "ab z", + "ny pd", + "grass i", + "kel sey", + "qld pol", + "clock work", + "f dr", + "di anne", + "âĺ ij", + "ad h", + "p ann", + "bra vely", + "ae ge", + "un lawful", + "ver di", + "pocaly pse", + "phar o", + "kar la", + "reson ance", + "ma stiff", + "la dak", + "bu u", + "ma iled", + "hi i", + "craw ley", + "tor rent", + "mach ado", + "liby an", + "effort lessly", + "fal sely", + "q vist", + "ke ef", + "craf thour", + "cheri shed", + "val kyrie", + "s ari", + "kal amaz", + "be he", + "ðŁĮ Ļ", + "th im", + "ro ddy", + "col trane", + "but chers", + "ach im", + "wk end", + "awk ward", + "cab rera", + ":) )))", + "fran c", + "decl an", + "con dos", + "a ja", + "pandor amusic", + "char ter", + "ph ill", + "mon trose", + "hatch back", + "handic app", + "gre aves", + "eucalyp tus", + "ut most", + "t son", + "bur ton", + "mid wives", + "in cur", + "ðŁĺį #", + "moo d", + "compre ssed", + "tom a", + "must ang", + "mo g", + "as ana", + "te stic", + "sho tel", + "in sol", + "cor sair", + "nh q", + "ben ny", + "sm ma", + "kap ur", + "in con", + "jon as", + "ener gies", + "don al", + "as ad", + "se z", + "n pa", + "archi ved", + "stimul ate", + "do p", + "hy d", + "gri eving", + "ãĥ Ī", + "ron a", + "why te", + "tree house", + "ss ell", + "sand ro", + "ko bo", + "ther most", + "se clu", + "hi ya", + "ge ez", + "mam as", + "prisc illa", + "flav oured", + "fas s", + "w old", + "maker space", + "cospla y", + "p tv", + "happy valentinesday", + "sequo ia", + "love craft", + "gu an", + "d tm", + "ci i", + "yoko hama", + "pos thum", + "re q", + "ðŁĶµ âļªï¸ı", + "galat asar", + "dol by", + "hamp tons", + "disturb ance", + "stone henge", + "ok c", + "disrup ting", + "month sary", + "jun gle", + "head lights", + "du stin", + "micro sof", + "happy mothersday", + "ko ko", + "gra zi", + "te sto", + "na idu", + "mal ay", + "ari al", + "ru mb", + "ab oo", + "har man", + "tra pe", + "spo ils", + "je ho", + "go dly", + "lock screen", + "z un", + "pi ous", + "ma gento", + "l enders", + "prob able", + "corpor al", + "m our", + "aw al", + "su a", + "call me", + "ton ne", + "go vin", + "devast ation", + "x j", + "gear box", + "war lock", + "per me", + "it ate", + "gaza underattack", + "du val", + "paras ite", + "clement e", + "le th", + "i va", + "fro zen", + "tho les", + "to bin", + "cair n", + "s ill", + "luc kiest", + "conver ts", + "st ale", + "pan cra", + "euro pale", + "wis dom", + "sch ur", + "ì ¶", + "verti go", + "bi j", + "u bc", + "nu re", + "righte ousness", + "mt c", + "factor y", + "ver st", + "revers ed", + "hur i", + "hee chul", + "fab er", + "ar r", + "ul ous", + "ven om", + "ph at", + "green ery", + "bra dy", + "à ¦", + ": ((", + "never giveup", + "di sha", + "mo ta", + "health care", + "dun ham", + "dex po", + "den zel", + "bb ins", + "f ics", + "wh am", + "mc g", + "eli an", + "wat a", + "str alia", + "tel lu", + "pe sky", + "spin off", + "ar moured", + "re acted", + "do fficial", + "te du", + "sag ar", + "mor ally", + "paralle led", + "fi os", + "dow ner", + "dau gh", + "re do", + "world cup", + "tari q", + "bar ne", + "glaci ers", + "oc cult", + "barbar ian", + "her mosa", + "!! !)", + "y ur", + "inter nation", + "p ss", + "sit u", + "p int", + "american air", + "sw am", + "dopp ler", + "ðŁĴĻ ðŁĴľ", + "cincode mayo", + "le van", + "hell enic", + "mc ne", + "ju di", + "yu h", + "st x", + "qu are", + "ðŁĺĤ .", + "sti g", + "g els", + "mot ley", + "hard work", + "euro zone", + "e ad", + "ç¥ Ń", + "seab ir", + "ci us", + "la id", + "alpac a", + "presu mably", + "pewdie pie", + "boo ted", + "am ari", + "tam ine", + "sol ace", + "bar row", + "acade mies", + "x ian", + "om ination", + "dun geons", + "b ma", + "de ity", + "ai k", + "stab il", + "hir a", + "affection ate", + "ving ne", + "new port", + "ãħĭ ãħĭ", + "thir ds", + "re tains", + "aroma therapy", + "ski er", + "ni ma", + "do pe", + "cr inge", + "con domin", + "to or", + "anim ator", + "sar aj", + "seas cape", + "minim alism", + "lake shore", + "calla way", + "berg man", + "ठĹ", + "whisp ering", + "stupi d", + "ri ghtful", + "requ is", + "ir n", + "se va", + "ut pol", + "tuber culo", + "squ ish", + "de but", + "govern mental", + "christ ine", + "all man", + "weap on", + "s ito", + "bur i", + "lo lita", + "leaf y", + "fu ch", + "tin ted", + "mck en", + "a hahaha", + "ðŁĩµðŁĩ ¹", + "repe al", + "ne gan", + "ðŁķ Ĭ", + "tail gating", + "game insight", + "ðŁıŁ ï¸ı", + "yaku za", + "z t", + "ti ring", + "pro posing", + "bow lers", + "tra itors", + "ak shi", + "cler gy", + "cit o", + "up sets", + "tu scal", + "symph onic", + "sil ently", + "shu ff", + "black well", + "ðŁĺĤ )", + "ko be", + "rober to", + "ri dg", + "dc u", + "mer ino", + "ft p", + "east side", + ". ~", + "nb l", + "mn leg", + "ts for", + "frau dul", + "ca pping", + "in my", + "gymna st", + "ston es", + "ss in", + "twe aks", + "shag gy", + "oak land", + "dem sin", + "sang ria", + "mm va", + "hen nessy", + "down ton", + "ri ghtly", + "in it", + "aga ve", + "ob last", + "northe ast", + "friend ship", + "dal a", + "tro phy", + "ðŁij ½", + "mag in", + "margar itas", + "ê ·", + "ww fc", + "fa sh", + "di ke", + "cu d", + "char t", + "ðŁij ®", + "refuge es", + "jop lin", + "n cs", + "imp y", + "firm ware", + "pas cu", + "flam in", + "health tech", + "bell letstalk", + "w aka", + "ol ls", + "la go", + "co wan", + "bombar dier", + "sh ome", + "ðŁĻ ħ", + "mc master", + "na ve", + "well s", + "u ta", + "tell ers", + "mis fits", + "kap il", + "face off", + "af firm", + "a pro", + "whit epaper", + "super yacht", + "speci mens", + "al located", + "... ,", + "- __", + "ka w", + "dachsh und", + "djo ker", + "s work", + "qui ere", + "or um", + "ðŁIJ ł", + "som m", + "c mt", + "ingh our", + "skin ny", + "lgb ti", + "gi ggles", + "break away", + "resear ched", + "par ity", + "my al", + "ms l", + "re tained", + "si vity", + "make inindia", + "sol ves", + "defam ation", + "wal tham", + "sri racha", + "road way", + "concep tu", + "al in", + "iw ant", + "å Ī", + "del ft", + "tender loin", + "ga ins", + "faul ts", + "sw ire", + "st ellen", + "pol lo", + "dy ne", + "bornon thisday", + "asdf ghj", + "sq l", + "sali m", + "advis es", + "vo ip", + "ìĹij ìĨ", + "un touched", + "she il", + "ontari o", + "uph ill", + "so bre", + "de shi", + "nov ella", + "du tton", + "craw fish", + "ا٠Ĩ", + "ma a", + "tw ine", + "kal in", + "ðŁĩµðŁĩ Ń", + "ye ss", + "brook s", + "hoo siers", + "ton ka", + "umbrel las", + "ay ers", + "ate am", + "acqu iring", + "su ction", + "ä n", + "wi es", + "tari ans", + "soci o", + "mat tb", + "shepher ds", + "o so", + "charity tuesday", + "s logans", + "ninj as", + "al bat", + "by te", + "bash ir", + "trampol ine", + "mydayin la", + "i ja", + "bas el", + "ror y", + "gol die", + "fi rec", + "un noticed", + "pecu liar", + "sch a", + "ker son", + "mour ns", + "liquid ity", + "qu ipment", + "hi bs", + "ar s", + "aeron au", + "slide show", + "sla bs", + "delici ousness", + "sk itchen", + "hta fc", + "full erton", + "cre ighton", + "aer ob", + "procrastin ation", + "az ores", + "white hall", + "uss occer", + "medi ation", + "djoker nole", + "and me", + "um en", + "noxi ous", + "jo ss", + "ili fe", + "anni vers", + "sudan ese", + "et res", + "under mine", + "whole foods", + "diso be", + "kor i", + "ade le", + "eli z", + "can ti", + "al on", + "gymna sium", + "sarko die", + "meteoro logist", + "yl de", + "ste en", + "stamp collecting", + "nas al", + "lo tt", + "fran ks", + "ex ol", + "ack i", + "good year", + "animal rights", + "y les", + "vio lets", + "mm es", + "s thel", + "ra pping", + "tu scan", + "wai ver", + "tur ner", + "eat local", + "northe asthour", + "anim ations", + "tom morow", + "t sh", + "ff ame", + "bra e", + "pe tron", + "glam our", + "br yn", + "d cs", + "bal es", + "ðŁĶ ¶", + "bro v", + "bre v", + "b ons", + "physi que", + "car ne", + "x e", + "elix ir", + "vol ved", + "l oma", + "ìľ ł", + "æ ĺ", + "van u", + "ri gs", + "bal ance", + "va res", + "bon ita", + "sprink le", + "perfec to", + "di on", + "le ak", + "calcu tta", + "o ba", + "d ma", + "c mon", + "tun er", + "pneu monia", + "bo gus", + "apolo ge", + "cl ough", + "bor ne", + ")) ))", + "revi ved", + "o varian", + "ner f", + "c legg", + "fan fest", + "cho u", + "reali zes", + "mc n", + "li gu", + "leg alize", + "just saying", + "for ster", + "bo sni", + "k hi", + "in dom", + "hei del", + "en cryp", + "si ss", + "ed di", + "mar bles", + "brisban e", + "y ing", + "pre paid", + "wal sall", + "cooper ate", + "orche str", + "mar isa", + "ho wie", + "che wy", + "bren ner", + "andro meda", + "e gan", + "sto cki", + "cav endish", + "ag an", + "ban o", + "de ir", + "go g", + "bl k", + "re thinking", + "ch ig", + "rhe u", + "sni p", + "p eng", + "semin ole", + "m swx", + "an nex", + "lyn da", + "lewisham ilton", + "cu mul", + "tb l", + "dolph in", + "agu ero", + "........ ....", + "pre lude", + "at our", + "gr anger", + "too ting", + "ro tun", + "dis ar", + "home items", + "da res", + "**** ****", + "ðŁij Ĩ", + "compre h", + "jin x", + "as well", + "iri e", + "circul ating", + "ðŁIJ ¥", + "over board", + "cultiv ate", + "rhe tt", + "oriente ering", + "ca k", + "bal kans", + "s itt", + "jas min", + "britney spears", + "ro tor", + "se aling", + "g bc", + "oc ci", + "f as", + "eman cip", + "com er", + "war time", + "tic kle", + "son ny", + "pac es", + "log g", + "at rix", + "sr p", + "g win", + "do bbs", + "uz be", + "the wanted", + "dru sh", + "ex tru", + "m icky", + "honore es", + "dar win", + "re dux", + "mm j", + "ram i", + "jalape ño", + "io c", + "do ver", + "ju ju", + "whit ney", + "s eng", + "en ly", + "au ch", + "archipel ago", + "vigil ant", + "man gal", + "wil dest", + "parano id", + "hal i", + "bb ly", + "sanc tioned", + "real ms", + "con co", + "u ddin", + "c sk", + "play time", + "libr a", + "sav ag", + "oc tane", + "rec tan", + "re turn", + "par rish", + "mor rha", + "cc p", + "c mu", + "sa iled", + "se vent", + "ro sie", + "pil ing", + "he w", + "boar ded", + "seg ments", + "neph ro", + "( .", + "cr ats", + "bak es", + "ðŁį ¸", + "back tothe", + "sibl ing", + "kirk land", + "ke o", + "gu wa", + "bre ads", + "ðŁĺľ ðŁĺľ", + "t q", + "haras sed", + "ga u", + "wil bur", + "j isoo", + "ep er", + "li sam", + "tri ppin", + "sh ino", + "ru kh", + "beast mode", + "cho a", + "inst aweather", + "rich land", + "gar i", + "fe z", + "cowboy snation", + "fur suit", + "k run", + "a en", + "sycam ore", + "se gun", + "ent ennial", + "di h", + "o ax", + "demsin philly", + "ðŁĻ Ģ", + "sn hl", + "pen nies", + "pass words", + "ma kin", + "ty e", + "d eng", + "kni gh", + "jeep life", + "hel pline", + "a for", + "zz zz", + "ste amy", + "pic ker", + "iter ate", + "happen ingnow", + "ki b", + "bloom berg", + "martyr dom", + "bul ly", + "assor tment", + "a hora", + "zo e", + "no i", + "illu stri", + "agar wal", + "p sc", + "electr onica", + "recruit er", + "gar diner", + "rad ha", + "naf ta", + "dot net", + "pi ero", + "geor g", + "bel s", + "ðŁĺĤ ðŁĺį", + "tuberculo sis", + "run nin", + "mor is", + "haul ing", + "ev oc", + "bre thren", + "sha ir", + "frame works", + "a stu", + "ri gid", + "ku ma", + "kre me", + "jin nah", + "insu rers", + "ny u", + "f ere", + "nol lywood", + "good vibes", + "- ...", + "toi le", + "sk ril", + "instaweather pro", + "cze ch", + "pa vel", + "one piece", + "nike plus", + "fi let", + "cav ity", + "ðŁı½ âĢįâĻĤï¸ı", + "ðŁİ £", + "dra stic", + "dail ys", + "siam ese", + "re bu", + "oste o", + "lar k", + "f re", + "sh elling", + "p é", + "glad ys", + "ðŁıĢ ðŁıĢ", + "gusta ve", + "submer ged", + "grand stand", + "att u", + "won t", + "f pv", + "b ley", + "jon i", + "ang ames", + "weigh ted", + "al ou", + "ठ¶", + "les bians", + "f j", + "anni es", + "am l", + "dor ia", + "dav in", + "be ta", + "can c", + "madewith unity", + "ha j", + "bad lands", + "mu l", + "blu ec", + "pa wn", + "cov ington", + "neuro logy", + "htt weets", + "dysle xia", + "thel ove", + "ne at", + "fork lift", + "autom ate", + "une ven", + "monte ss", + "he in", + "ha g", + "rel ics", + "competiti veness", + "can elo", + "mar tens", + "bullet proof", + "sk ittles", + "g ya", + "pri mo", + "americ afirst", + "woo o", + "abor tions", + "?? !!", + "ma che", + "ld ers", + "rl ly", + "preli ms", + "direc t", + "cour se", + "swa in", + "super cell", + "ec centric", + "sting ray", + "ple ts", + "wil cox", + "west in", + "okan agan", + "kir an", + "car bo", + "bomb ings", + "ra rest", + "bo h", + "gaw d", + "di gg", + "mo ana", + "enti rety", + "en closed", + "dodge ball", + "par ton", + "milky way", + "at r", + "thorough bred", + "re ally", + "qant as", + "epiph any", + "ine e", + "aero smith", + "spi eth", + "ar thro", + "ell ini", + "du bu", + "bra ving", + "âļ½ âļ½", + "re structuring", + "illumin ate", + "equ ili", + "mp i", + "ash ton", + "pony tail", + "ma scots", + "flat tering", + "cru m", + "ast a", + "à® °", + "stranger things", + "bar nab", + "ر ÙĬ", + "make shift", + "got cha", + "will am", + "cho irs", + "kilom etres", + "gho sh", + "eu than", + "dol ly", + "un ning", + "the ar", + "cre we", + "w sw", + "j ace", + "dis miss", + "ke an", + "ho ta", + "kh at", + "~ >", + "thir u", + "ren dez", + "hart man", + "tee ssi", + "cas ca", + "z ah", + "hydr ange", + "fo d", + "aw p", + "mzan si", + "thick er", + "nago ya", + "ne va", + "sti que", + "cast el", + "dam ian", + "there by", + "ji ang", + "ale k", + "music islife", + "ra q", + "calla han", + "gou ache", + "somal iland", + "sean hannity", + "ra heem", + "lo se", + "elo ve", + "whar ton", + "rectan gular", + "illustr ating", + "har ne", + "auti sma", + "scra pped", + "ell and", + "decre e", + "nag pur", + "ki pp", + "so re", + "n md", + "ma as", + "gun a", + "gart ner", + "bel li", + "then ight", + "je on", + "gendere quality", + "gi ver", + "a el", + "gar ments", + "ne u", + "mardi gras", + "mar sden", + "ro wer", + "pollu ted", + "camer aman", + "vin od", + "be asley", + "cro c", + "ji u", + "hollyo aks", + "anesthe sia", + "al les", + "ste ward", + "lati mes", + "ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸", + "tic ian", + "gor ia", + "come dic", + "ðŁ¤Ķ ðŁ¤ĶðŁ¤Ķ", + "nai ve", + "sli ons", + "ł Ī", + "bur glar", + "ðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃðŁĺŃ", + "york shi", + "se ñ", + "fan boy", + "lau rel", + "inci dence", + "potom ac", + "rober ta", + "presi den", + "pr yor", + "os bourne", + "w ku", + "te me", + "pal ae", + "ðŁ¥ º", + "re boun", + "itu de", + "red dish", + "k hand", + "coloni alism", + "north carolina", + "ðĿ Ĵ", + "manne quin", + "lady bird", + "ta sty", + "knowledge able", + "g shore", + "ðŁĮ Į", + "à® ©", + "qu aker", + "salz burg", + "med alists", + "chy na", + "bridesma id", + "ma ori", + "ro p", + "outra ged", + "in adequate", + "truck ers", + "al ana", + "ìĿ ¼", + "ri x", + "oooo oooo", + "command ments", + "lam beth", + "aa j", + "eco friendly", + "bla z", + "morecam be", + "boun cy", + "rou x", + "rai ded", + "mi zed", + "sh c", + "gaw x", + "labor atories", + "ru bs", + "rest room", + "consult ations", + "ca jun", + "virgin i", + "so ir", + "rev ue", + "ple in", + "wag er", + "ç ¹", + "we do", + "growing up", + "! ðŁĺĬ", + "face ted", + "sin ners", + "ho vering", + "ti ene", + "seas oning", + "an ja", + "leg go", + "il is", + "fla x", + "dev o", + "ash ram", + "mati sse", + "ker i", + "go wer", + "bo tox", + "mar shes", + "unh cr", + "ts m", + "opti mus", + "dun i", + "stu ffs", + "so k", + "order ly", + "n bad", + "islam ophobia", + "raviol i", + "fab er", + "cre ds", + "won ka", + "in fusion", + "over weight", + "daily news", + "assi mil", + "acol lege", + "medalli on", + "kili manjaro", + "sti ff", + "tham es", + "sun ken", + "th ard", + "my dubai", + "hilari ously", + "han nel", + "plu mber", + "fair view", + "separ ating", + "rasc al", + "qui en", + "necess ities", + "confeder ation", + "ll ll", + ": ]", + "weak nesses", + "bron co", + "ra ffles", + "el ot", + "ãĤ¸ ãĥ", + "advent calendar", + "ðŁİ ¹", + "stra vel", + "tun ic", + "k su", + "im peach", + "e spionage", + "! -", + "di ment", + "cur rant", + "bio de", + "commu ting", + "by ron", + "ðŁĴĵ ðŁĴĵ", + "shad ed", + "tr uro", + "cray ons", + "ar ne", + "h sc", + "fre aked", + "dram ati", + "fle ek", + "u cd", + "marl borough", + "^ -", + "cross ings", + "mal o", + "black ops", + "bin ance", + "cho ked", + "chen ey", + "pl o", + "ge stures", + "val edic", + "ryan air", + "rem ington", + "v cs", + "mc kee", + "ec z", + "be gs", + "nail art", + "mayor of", + "happy fathersday", + "war t", + "pet itions", + "n ingly", + "clean energy", + "bro x", + "sl alom", + "exist ent", + "ab ay", + "ug liest", + "tom p", + "stom a", + "sel by", + "goal scorer", + "ben ji", + "overwhel mingly", + "lan s", + "semiconduc tor", + "south korea", + "re scheduled", + "sk yl", + "en listed", + "dow ski", + "si del", + "rosen berg", + "nas ser", + "white head", + "pri us", + "har are", + "en n", + "ry der", + "í Ĥ", + "mon g", + "clas ico", + "transpor ter", + "po tty", + "is me", + "** ***", + "vic e", + "sk it", + "ode ssa", + "l mp", + "her n", + "raci ally", + "pin oy", + "paragu ay", + "obitu ary", + "go es", + "bu cha", + "side walks", + "angu lar", + "un constitutional", + "transiti oning", + "i bu", + "gu ys", + "un packing", + "oooo oo", + "black girl", + "ber gs", + " ¯", + "wordof theday", + "trump train", + "thunder bolt", + "m si", + "fasci sts", + "ठ¬", + "t sk", + "collap ses", + "raje sh", + "loveis love", + "migr ating", + "set back", + "ðŁĺĬ âĿ¤ï¸ı", + "t els", + "safety first", + "nar rated", + "jae joong", + "un answered", + "lique ur", + "en nes", + "dal go", + "bill ings", + "salt water", + "mer maids", + "lon gs", + "clap ham", + "we arec", + "pic collage", + "n ach", + "h ace", + "pois oned", + "lo th", + "ag na", + "adel rey", + "guar dia", + "poli shing", + "peace keeping", + "d all", + "p isa", + "la pland", + "process ors", + "de andre", + "so bs", + "p once", + "dra ins", + "c be", + "ðŁİ¥ :", + "spla sh", + "meat ball", + "fon tana", + "worcester shirehour", + "ne v", + "bri sk", + "b int", + "ac r", + "po x", + "cay enne", + "skril lex", + "j fc", + "hahahaha hahaha", + "gla s", + "en gul", + "tempor al", + "oni zed", + "con cre", + "com pose", + "vibr ations", + "plant ers", + "fer t", + "criticalrole fanart", + "t bli", + "sch allenge", + "huck abee", + "munici pal", + "iam bic", + "radi os", + "ne vis", + "dura bility", + "mc cla", + "horse back", + "inst itutes", + "ful fill", + "atta ch", + "ate ur", + "ak an", + "resi sting", + "illumin ation", + "hand le", + "hair care", + "om ent", + "macle od", + "ka iser", + "g no", + "bear down", + "ly f", + "gl omer", + "distor tion", + "z m", + "san k", + "roo sters", + "is now", + "as ports", + "ag en", + "wo ken", + "st george", + "ro mper", + "my le", + "econom ists", + "ru to", + "t will", + "health and", + "d ito", + "ws l", + "tair p", + "pra kash", + "mic heal", + "h ts", + "w rights", + "kat su", + "fioren tina", + "defen seman", + "d itch", + "var sity", + "texan scheer", + "ba ham", + "sc anned", + "we il", + "seduc tive", + "ðŁijį ðŁı½", + "fu e", + "er win", + "dav ison", + "ter ran", + "moo ds", + "wool f", + "re source", + "@ .", + "cu sh", + "ðŁį °", + "regre ssion", + "cur led", + "la zer", + "jo anne", + "ab bott", + "mo z", + "down ers", + "mm mmmm", + "valent ina", + "k hair", + "dream t", + "cro ok", + "che k", + "ste aming", + "nephe ws", + "cl eric", + "as ober", + "indefin itely", + "w ye", + "us news", + "joy ce", + "flu shing", + "wynonna earp", + "ron do", + "kis s", + "hot dog", + "bar ns", + "sax ophon", + "far ley", + "gas p", + "decre asing", + "al way", + "pe x", + "l sd", + "shi ft", + "p outine", + "ra zz", + "rescu ing", + "ni ko", + "ho ch", + "cc l", + "u aap", + "n ts", + "m car", + "il wx", + "conqu ering", + "ket tering", + "stur dy", + "delay ing", + "sto k", + "vani shed", + "cath ar", + "bin gham", + "in v", + "ic hiro", + "he mo", + "budge ting", + "[... ]", + "be ss", + "sebasti an", + "slow ed", + "ðĿ ij", + "musli m", + "stun s", + "acton climate", + "ve a", + "se ton", + "rose tta", + "oun t", + "hard in", + "flu id", + "ca w", + "ðŁ¥ Ĥ", + "yach t", + "un l", + "sp hy", + "provoc ative", + "or ic", + "is back", + "__ _", + "nicol as", + "gy an", + "loo se", + "fl in", + "reb ate", + ": ::", + "! \"@", + "com icon", + "she ff", + "down stream", + "chic hester", + "beach life", + "mom life", + "diabe te", + "ar ra", + "van e", + "ok u", + "ye o", + "man go", + "try out", + "app ell", + "he irs", + "arjun a", + "dd u", + "na veen", + "movi c", + "soci alists", + "s back", + "criteri on", + "soyu z", + "k her", + "da z", + "yol anda", + "wine oclock", + "re ina", + "one w", + "leon ard", + "en dez", + "u bs", + "support local", + "facilit ated", + "carameli zed", + "b pa", + "vuel ta", + "my tho", + "m ami", + "spe are", + "nbap layoffs", + "fe vre", + "nick jonas", + "im print", + "c so", + "craig slist", + "la salle", + "gi deon", + "ha doop", + "dis regard", + "w ud", + "tu c", + "ma gee", + "acou stics", + "ta a", + "qui e", + "pol a", + "cr t", + "dw yer", + "dis sec", + "capit ol", + "men tion", + "kn oll", + "he igh", + "fin ders", + "plac ements", + "l se", + "indi ra", + "gur i", + "madhuri dixit", + "kingdom s", + "iambic pent", + "geor gina", + "je ky", + "conflic ting", + "bay an", + "aga tha", + "uph old", + "dr on", + "vic ar", + "ex pat", + "periph eral", + "pe ssi", + "fa f", + "ance stor", + "? ..", + "wid get", + "pun c", + "comm enced", + "beav s", + "air waves", + "ad dis", + "po a", + "de sses", + "co den", + "vu e", + "ru pee", + "kar in", + "spo ck", + "m sy", + "ภ°", + "pr ick", + "fill more", + "ti fication", + "thing sto", + "sar de", + "em ile", + "pere ira", + "n ad", + "bright ening", + "arre sting", + "wo king", + "usc g", + "sp ill", + "raspberry pi", + "hu go", + "ite c", + "is ma", + "cuff links", + "optimi zed", + "oc c", + "mi wx", + "en ka", + "el ited", + "afford able", + "sa kh", + "coron ado", + "ho h", + "at ul", + "ai oli", + "jim cantore", + "accoun ted", + "vin ay", + "her mit", + "groo ves", + "ran ch", + "r illa", + "we tter", + "ou tof", + "veter in", + "ni kov", + "ki an", + "fair banks", + "ram apho", + "n iti", + "k ko", + "ru sty", + "ne stle", + "tv xq", + "shahe er", + "âĿ¤âĿ¤ âĿ¤âĿ¤", + "penn ant", + "gem stones", + "dem debate", + "ðŁIJ Ĭ", + "auton ews", + "support indiefilm", + "mach o", + "ve x", + "new sat", + "ne ti", + "conce ssions", + "can died", + "yof the", + "mac au", + "den ds", + "cricke ters", + "san iti", + "mari ano", + "gh at", + "ar toftheday", + "¡ ľ", + "e gos", + "gen oa", + "chat bots", + "bri er", + "al labout", + "mon ty", + "spi ed", + "r tr", + "comfor t", + "sni ppets", + "real time", + "gra in", + "exam ined", + "en lightening", + "tt u", + "god bless", + "release the", + "sing ular", + "ki ans", + "ha ka", + "sor ren", + "defe ct", + "mar g", + "equ ities", + "d orian", + "su ka", + "per l", + "aishwar ya", + "pul lover", + "preci sion", + "fair way", + "ne ve", + "rive ting", + "vill anova", + "en com", + "ak o", + "passion ately", + "europale ague", + "siem pre", + "x vi", + "enligh tened", + "c fr", + "âĺħâĺħ âĺħâĺħ", + "wast eland", + "is f", + "new comers", + "emergen cy", + "amphi theatre", + "- .", + "text books", + "figur ative", + "tre mb", + "pe sc", + "ab hin", + "ab bot", + "ac acia", + "har ds", + "por sche", + "kau ai", + "el isa", + "car rick", + "abo u", + "elli er", + "be ch", + "neu tron", + "galap agos", + "ru ben", + "in nis", + "how to", + "nun s", + "sab ine", + "i ac", + "clin ched", + "no tori", + "fi ves", + "cairn gor", + "per i", + "gr c", + "ðŁĴ¯ ðŁĴ¯", + "mal m", + "twelf th", + "di ff", + "rout ines", + "marty n", + "lin den", + "synthesi zer", + "nu mber", + "game cube", + "fal kirk", + "byz antine", + "queu ing", + "gr ill", + "scal able", + "char red", + "rou ting", + "her bali", + "gri zz", + "ðŁĺŃðŁĺŃ ðŁĺŃ", + "tol l", + "termin als", + "l pc", + "ab d", + "war mups", + "remo vable", + "¯ \\", + "vi go", + "pap aya", + "ne ve", + "lov ingly", + "jo kers", + "ib les", + "sse tt", + "poten ti", + "pel e", + "gi gi", + "sadi q", + "leg acy", + "son o", + "ru pees", + "retar ded", + "ele e", + "par r", + "fi ance", + "ey re", + "say ers", + "pend ants", + "mak nae", + "al bans", + "adap ting", + "p ff", + "pu berty", + "ji u", + "ing rad", + "hypocr ite", + "diplom ats", + "phys ical", + "rob by", + "bon sai", + "ãģ ·", + "f att", + "catal unya", + "âľ ĸï¸ı", + "ro ma", + "more land", + "so e", + "conver sions", + "stl blues", + "shol m", + "gra ssy", + "pra do", + "on u", + "assaul ting", + "> _", + "sett es", + "dis graceful", + "aph ra", + "âļ½ï¸ı âļ½ï¸ı", + "ठª", + "kil n", + "goal tender", + "s ru", + "philanthro pist", + "b als", + "th n", + "stu den", + "sando val", + "dogre scue", + "eli ons", + "asse ssed", + "lar go", + "hec tares", + "sh rm", + "sa if", + "cle avage", + "no ches", + "n ene", + "fat alities", + "cur ing", + "clean ser", + "al es", + "p vp", + "south bank", + "pizz eria", + "marsh als", + "kni fe", + "an dover", + "tbli ghtning", + "sr sly", + "ou te", + "digi mon", + "timesof india", + "prome the", + "le bo", + "f su", + "wit z", + "rever e", + "man as", + "mam ba", + "ch ica", + "gu an", + "exhibit or", + "csr racing", + "d ere", + "xx xxx", + "gu sta", + "story time", + "ston ey", + "organ ics", + "and u", + "se am", + "min ogue", + "anushka sharma", + "ab a", + "ðŁİĻ ï¸ı", + "ugand an", + "chro matic", + "as sn", + "document aries", + "sh t", + "ru paul", + "loy d", + "k ats", + "e us", + "ite ch", + "me dusa", + "pan ty", + "kel logg", + "et to", + "talla de", + "sha a", + "do st", + "p ms", + "mari ana", + "je ster", + "croo ks", + "ðŁĶ ¬", + "min danao", + "ind hoven", + "ðŁ¤ ª", + "le xi", + "tv n", + "jan is", + "co te", + "ãģ Ĩ", + "ser rano", + "iw m", + "ðŁIJ ¬", + "k ke", + "distribu tors", + "cap u", + "counterfe it", + "camp site", + "ag gie", + "ðŁĺ ¼", + "chhat tisgarh", + "~ @", + "state u", + "san di", + "prevent able", + "cl s", + "can ne", + "mm c", + "i ver", + "sa haran", + "pal is", + "night out", + "do s", + "ap ia", + "absc bn", + "manag erial", + "aro se", + "mo wx", + "aro sa", + "ðŁĮ ³", + "under dog", + "remo ver", + "astronom ers", + "lent ils", + "su scep", + "smoo ther", + "pend leton", + "fau cet", + "e mory", + "dal mati", + "af cb", + "tic us", + "exem pt", + "en rol", + "d heim", + "ðŁIJ º", + "restric tion", + "star fish", + "sto w", + "snor kel", + "thunder birds", + "she ad", + "homo sexual", + "dy n", + "as li", + "andre tti", + "dou che", + "dom o", + "tar mac", + "slu mber", + "pr onto", + "first dayof", + "mini ature", + "mari achi", + "argu s", + "recomm ending", + "mobi les", + "in ce", + "illustri ous", + "or c", + "adver ts", + "gr its", + "wea sel", + "pag oda", + "over pass", + "gre ys", + "maxi mus", + "arma gh", + "wood land", + "sun ni", + "ðŁĴ ī", + "ë Ŀ", + "ti one", + "soci o", + "ho s", + "ðŁ¤Ĺ ðŁ¤Ĺ", + "wind sor", + "subsequ ent", + "munch ies", + "id h", + "exclu ding", + "e mi", + "cu th", + "z ai", + "week days", + "law suits", + "barn ard", + "Ø ª", + "pe tting", + "net es", + "mul ligan", + "pharmac ists", + "ra quel", + "e ton", + "cran ston", + "gil ded", + "cle ary", + "ce ph", + "ra a", + "pam per", + "lombar di", + "as in", + "sher ry", + "pro d", + "for te", + "ari anism", + "buffalob ills", + "æľ ¬", + "ðŁĶ¥ #", + "uu u", + "just ices", + "car ina", + "nat in", + "mas low", + "dro oling", + "cog nac", + "cam ber", + "el ong", + "r dr", + "in en", + "convic tions", + "am use", + "tro ck", + "harm less", + "visit ation", + "gen omic", + "bl and", + "beno it", + "chim p", + "tuscal oosa", + "gre asy", + "x po", + "gil t", + "se q", + "per mitted", + "christma seve", + "book s", + "mu e", + "old school", + "human right", + "be ati", + "ðŁĶ Ŀ", + "sh at", + "sculp ting", + "h wan", + "fern andes", + "sci utto", + "fu entes", + "endeav ors", + "maid stone", + "un paralleled", + "shou ted", + "queen of", + "mer c", + "band ic", + "ve da", + "sel angor", + "pi le", + "ja han", + "intimid ating", + "disapp ears", + "cl ich", + "za ha", + "w urst", + "hi v", + "fod ils", + "cor dless", + "aaaa aa", + "hy dra", + "bel inda", + "e els", + "bu f", + "su staining", + "rugby league", + "no c", + "brig itte", + "( ðŁĵ¸:", + "tromb one", + "soo the", + "smo g", + "ad p", + "stab le", + "ing ley", + "diagno se", + "ms g", + "we ss", + "tic keting", + "one e", + "nsw pol", + "e up", + "auto psy", + "adity anath", + "sun down", + "river front", + "si ya", + "p is", + "hier archy", + "dur ango", + "di jk", + "ren shaw", + "he aps", + "epide mi", + "david bowie", + "interne tof", + "dd i", + "nation ality", + "mb ar", + "air y", + "win der", + "w alia", + "elli ott", + "c x", + "bav arian", + "pl att", + "an tw", + "wi wx", + "sof ter", + "ne ha", + "h eller", + "th and", + "dani ela", + "bo ast", + "degra dation", + "ðŁĴ¦ ðŁĴ¦", + "transform ing", + "man e", + "av ut", + "ðŁĺĪ ðŁĺĪ", + "vo ter", + "the e", + "t ate", + "pu ff", + "in door", + "sop roud", + "boy ce", + "boris johnson", + "wait in", + "immun ology", + "ðŁıĨðŁıĨ ðŁıĨ", + "âĿ Į", + "street food", + "liz asober", + "cavali er", + "c elia", + "need le", + "motor ing", + "g ato", + ", )", + "ra de", + "harve st", + "t ms", + "jar pad", + "on ey", + "air men", + "v re", + "impair ment", + "abhi shek", + "snoo p", + "l ant", + "fam ously", + "bl ou", + "s ze", + "g ander", + "un touch", + "tu f", + "dee jay", + "col lateral", + "b ind", + "ðŁļ ©", + "pin ning", + "ic n", + "' ;", + "the economist", + "ul tram", + "worldwater day", + "ti poff", + "the i", + "feed ers", + "campa ign", + "sc umb", + "day weekend", + "yo m", + "pe dic", + "h ough", + "ps v", + "pl in", + "on de", + "boston marathon", + "az zy", + "* _*", + "con ley", + "thi ago", + "hoo o", + "gal erie", + "luci d", + "je tt", + "gl itz", + "final fantasy", + "achiev ers", + "y ung", + "peregr ine", + "op hi", + "dam es", + "biom ar", + "âĺĢï¸ı âĺĢï¸ı", + "sk c", + "l ics", + "fl ank", + "ar rahman", + "ho of", + "uphol stery", + "t ats", + "wo z", + " ¿", + "snor ing", + "ra er", + "l ju", + "ap d", + "pl ating", + "kan u", + "im ation", + "fragr ances", + "m ra", + "mor ay", + "mo tt", + "im muni", + "hearti es", + "bho pal", + "tim ers", + "g ata", + "color way", + "car nation", + "win get", + "si ghs", + "s ville", + "optimi st", + "chate au", + "olympi ans", + "ci o", + "singer songwriter", + "ny o", + "fi bers", + "bur ch", + "ag ro", + "mil ne", + "ig bo", + "cr amer", + "ation als", + "dan ube", + "pad ma", + "nor mani", + "en forced", + "bre ck", + "boeh ner", + "ar den", + "sur rendered", + "pros thetic", + "om a", + "ha iled", + "calcul ations", + "w fa", + "bi b", + "fcb live", + "fon da", + "west coast", + "que sts", + "friend ly", + "to wie", + "fit ch", + "bal ot", + "star dom", + "scrat ching", + "ho sa", + "thi ka", + "o ven", + "stro ke", + "out post", + "pharmaceu ticals", + "hi kari", + "mu y", + "af d", + "fallon tonight", + "squ at", + "or u", + "dra ined", + "chocol at", + "ë¯ ¼", + "wor ths", + "ri b", + "mu j", + "that s", + "residen te", + "it el", + "boo st", + "mi gos", + "mul led", + "la a", + "etsy shop", + "don keys", + "me k", + "p tc", + "flin ders", + "e hs", + "ro hit", + "mu ir", + "g ad", + "compos itions", + "åĨ Ļ", + "combu stion", + "i kh", + "yemen i", + "wav ed", + "gar ci", + "ak os", + "oo ds", + "fu sion", + "se que", + "s lan", + "pl ur", + "kic chasu", + "shenan do", + "s ams", + "worl den", + "horo witz", + "with me", + "mic robes", + "k ki", + "ðŁĴĶ ðŁĴĶ", + "w su", + "patch work", + "fre er", + "y aki", + "the art", + "symboli sm", + "mil er", + "bt n", + "ma bu", + "side kick", + "motiv ates", + "sag itt", + "natur als", + "serv iced", + "ps ori", + "pa ola", + "qu ig", + "i badan", + "gi ggs", + "ë ³", + "sciento logy", + "si oux", + "salam at", + "d res", + "cad bury", + "d hawan", + "ci ón", + "_ '", + "swa pping", + "maris ka", + "james bond", + "explo sives", + "ay les", + "af er", + "s agu", + "cen sor", + "tom a", + "jeff erson", + "ring ed", + "par tist", + "ir responsible", + "aguil ar", + "vac ay", + "equ itable", + "altrin cham", + "ac ur", + "man ish", + "ger min", + "schoo led", + "pu tter", + "ed ad", + "nav al", + "toast y", + "sol areclipse", + "dish u", + "coy ne", + "ac co", + "mu ck", + "mar an", + "el os", + "len der", + "cro ix", + "worth less", + "ha ber", + "gun men", + "ðŁį ĵ", + "zen ith", + "t enders", + "hur st", + "hol tz", + "itali ans", + "car low", + "u cd", + "characteri stic", + "bun g", + "av l", + "u th", + "sa sia", + "rs l", + "red man", + "neighbor ing", + "green peace", + "sti ps", + "follow party", + "y gk", + "en os", + "omni bus", + "na issance", + "chri ssy", + "secu re", + "call back", + "ji hoon", + "memor y", + "block er", + "l anta", + "daf fodils", + "bil t", + "ffer ty", + "fau st", + "ie c", + "nipp les", + "so g", + "m nd", + "jagu ar", + "bol dly", + "ab poli", + "pro position", + "gun sense", + "evan sville", + "cu tters", + "we go", + "dou n", + "do x", + "stal lions", + "ka j", + "shi ppers", + "j awa", + "vol o", + "le ven", + "pap rika", + "kov ich", + "jor di", + "induc tees", + "app alling", + "dial ysis", + "allevi ate", + "âĢĶ âĢĶ", + "pie ter", + "mid wi", + "q tr", + "juli ette", + "inter mission", + "haw ks", + "act ment", + "one ill", + "k lin", + "vam ps", + "fam ous", + "cou ld", + "autom obi", + "da an", + "west end", + "elli p", + "nh c", + "mel anch", + "web series", + "ton gue", + "snat ched", + "smy th", + "tan gible", + "sl i", + "e asing", + "bar stool", + "over lay", + "afford ability", + "ting ed", + "ter as", + "ay ush", + "wanna one", + "rh ine", + "dan a", + "sh ana", + "kend al", + "fer tile", + "w ir", + "repl eni", + "lar vae", + "is ro", + "con vos", + "ab brevi", + "u cc", + "hun gry", + "bur rows", + "ag er", + "nav i", + "mat in", + "du per", + "cer n", + "ma don", + "ķ ï¸ı", + "é ģ", + "tu ps", + "hy att", + "sh ep", + "friday night", + "wis er", + "hei di", + "hat ton", + "p gh", + "foun tain", + "wrist bands", + "ahmadi yya", + "aeri al", + "subscri bed", + "so los", + "m ace", + "sla yed", + "for fe", + "dul ce", + "christ mass", + "arun jaitley", + "viol ate", + "ob stru", + "ni eces", + "w vu", + "idy l", + "fa ze", + "pre serves", + "infr inge", + "premi ers", + "inter vals", + "agen cy", + "( ©", + "stand alone", + "di mes", + "bo er", + "param eters", + "ge tit", + "ðŁĺĺðŁĺĺ ðŁĺĺðŁĺĺ", + "tu lane", + "for given", + "scol l", + "mb ps", + "smash bros", + "rob bi", + "prima vera", + "ali st", + "ghost ly", + "ay at", + "ye ats", + "impre ssionist", + "ear phones", + "caul field", + "wai kiki", + "sal ute", + "sc ou", + "mu ay", + "louis vuitton", + "bak hta", + "ado g", + "inven tions", + "hur d", + "forec lo", + "stream line", + "thalai var", + "ch snews", + "will ard", + "t sn", + "euro parl", + "cru sher", + "my sore", + "gro wer", + "ra ping", + "pat ti", + "g den", + "sm w", + "muf ti", + "kid man", + "ab r", + "soun ders", + "skep tical", + "ðŁĶ İ", + "sun dar", + "i me", + "fer g", + "feather weight", + "ar lington", + "pas qu", + "ag azine", + "wearab le", + "nati c", + "mccl ure", + "inter mitt", + "hor de", + "six ties", + "car te", + "bha v", + "ze al", + "experi ential", + "ador ned", + "som mer", + "eno te", + "hypo thesis", + "stin ky", + "pro to", + "dead lines", + "vo gel", + "mus ings", + "monc ton", + "gu ter", + "f le", + "aci on", + "voice of", + "ta sha", + "inhabit ants", + "type face", + "s ba", + "bts x", + "ðŁĶ Ĵ", + "wor x", + "u hc", + "jo ko", + "cell ars", + "gor o", + "continu um", + "... &", + "weather cee", + "ha p", + "sr k", + "ris ers", + "lonely planet", + "un named", + "co eur", + "ðŁį Į", + "the world", + "ili ke", + "fa sten", + "ami go", + "ri ba", + "ramapho sa", + "staf fers", + "had ley", + "? ?\"", + "fi ore", + "sal ut", + "hu ff", + "bez os", + "Ñ ĭ", + "ra der", + "kam ala", + "in line", + "fill ers", + "um atic", + "all in", + "shat ter", + "re in", + "o ku", + "ch ases", + "fla gged", + "baby metal", + "water stones", + "ts b", + "cut out", + "op hel", + "aam a", + "rockab illy", + "sto lic", + "jet blue", + "ich ick", + "down ton", + "uzbe kistan", + "pat na", + "la q", + "gr ange", + ") _/", + "subsi di", + "sc p", + "newsc ast", + "it sa", + "twee tyour", + "e mor", + "archae ologists", + "uni fication", + "por ta", + "q x", + "protec tors", + "pro hib", + "charis ma", + "car tag", + "ren fre", + "scul pt", + "guwa hati", + "de ma", + "boo p", + "unf pa", + "dex ter", + "lay la", + "alleg es", + "sou ps", + "never again", + "l ys", + "cal c", + "bar oness", + "visu alize", + "ger ber", + "absor bed", + "i ers", + "a han", + "fon tein", + "detec tors", + "verst appen", + "sv c", + "formul ated", + "ac dc", + "li x", + "in competent", + "bh k", + "lour des", + "water house", + "snow ed", + "appreci ative", + "sig ma", + "lizasober ano", + "pen ned", + "pay check", + "tall inn", + "fanc afe", + "par isi", + "av alley", + "vi g", + "ru fc", + "hard ship", + "so cute", + "po ise", + "ì ¹", + "roth schild", + "k ly", + "???? ????", + "l hp", + "il ay", + "f hs", + "am ad", + "ide als", + "brad bury", + "bal boa", + "nic ot", + "kid nap", + "wol ve", + "tas manian", + "op t", + "matthi as", + "ãĥ³ ãĤ", + "super markets", + "mylittle pony", + "me lee", + "li ster", + "gr oun", + "fe dora", + "kind ness", + "en en", + "bra hms", + "¯\\ _(", + "ros well", + "mar lene", + "ic u", + "re formation", + "or ail", + "he brides", + "dispar ities", + "terrac otta", + "swal lows", + "re id", + "influ encing", + "flu or", + "den e", + "tum our", + "blon des", + "thunder bird", + "sh eva", + "moga dishu", + "ka b", + "cre eps", + "i ving", + "ene ed", + "anno y", + "âĶ Ģ", + "intri gue", + "enqu iry", + "ar aj", + "tur al", + "kuber netes", + "end lessly", + "divi dends", + "tor a", + "ti sh", + "commemor ates", + "un ra", + "tri b", + "pon ty", + "ne m", + "diss ent", + "brew ingco", + "ðŁĺ ½", + "nor mali", + "bi of", + "( ...", + "chil len", + "ì£ ¼", + "mell on", + "av is", + "mccor mack", + "ing ra", + "enrich ed", + "custome rexperience", + "testo sterone", + "snu g", + "sett i", + "ger onimo", + "inqui rer", + "bre aches", + "very thing", + "bloom ing", + "mu ra", + "dispo s", + "bi de", + "de va", + "shade sof", + "in trin", + "sh ev", + "s ven", + "nayanth ara", + "gan esha", + "c ws", + "ber ta", + "label led", + "use um", + "nick named", + "ma han", + "car uso", + "ap ur", + "ðŁij Ĩ", + "w q", + "orphan age", + "discar ded", + "mag nu", + "lu e", + "je on", + "bridge port", + "pac ing", + "mercur y", + "( ðŁĵ¸", + "marx ist", + "amphi bious", + "transplant ation", + "stit ching", + "then burg", + "gradu al", + "ãĤ Į", + "ro ft", + "ma ils", + "ine c", + "guy ana", + "dopp elg", + "ver o", + "re write", + "head less", + "harb augh", + "gate way", + "car sforsale", + "sw i", + "st is", + "mach t", + "un de", + "sura baya", + "stap leton", + "nur turing", + "mil ner", + "ya o", + "lma oooo", + "ko sh", + "arsen al", + "k ame", + "er ry", + "ar royo", + "dis misses", + "ru bbed", + "rc b", + "lew d", + "dil u", + "and or", + "vi de", + "ur in", + "inter sec", + "ha ar", + "al b", + "year swith", + "app leton", + "é al", + "ul livan", + "suc cu", + "monter rey", + "d mx", + "artem is", + "ron nie", + "farm land", + "s football", + "gro tto", + "anth i", + "ãĢ ģ", + "à® Ł", + "vid ya", + "jimmy fallon", + "ൠį", + "t zer", + "gravit ational", + "w thr", + "u hhh", + "e hr", + "tin ker", + "ti juana", + "scran ton", + "ram charan", + "bar clay", + "re van", + "m si", + "ka p", + "wr s", + "we thenorth", + "tor al", + "sat u", + "gro m", + "fac ep", + "erick son", + "z yn", + "se dge", + "oo dle", + "spur sofficial", + "ds p", + "sic ilian", + "soli hull", + "recei vers", + "ladak h", + "hend rick", + "ther i", + "presi ding", + "mc guinness", + "litt ers", + "gun nar", + "gh oul", + "wi b", + "n tv", + "kar o", + "fro ck", + "b lau", + "ampli fy", + "all is", + "ul lah", + "memo irs", + "kh loe", + "intercep tions", + "pet day", + "lo oney", + "con fin", + "ch ay", + "piyush goyal", + "frequ encies", + "ut z", + "event ual", + "warm ly", + "obli vion", + "an ka", + "ta it", + "âĿ¤ï¸ı .", + "director ial", + "ru lers", + "prince s", + "mu ck", + "stur ridge", + "deu ce", + "abri dged", + "bagu ette", + "un cles", + "pen du", + "min ding", + "forre ster", + "av ila", + "wall er", + "wall street", + "ment or", + "hin o", + "high way", + "crom well", + "fanart friday", + "mb i", + "co yle", + "a hi", + "tro ve", + "spie gel", + "pay tm", + "mcin tosh", + "jan sen", + "nit i", + "nash ville", + "len o", + "leicester shire", + "le gos", + "dic t", + "ðŁĵ ½", + "sp ad", + "beverly hills", + "sy rah", + "separ ates", + "z ain", + "un fit", + "dra gs", + "tan ia", + "over flowing", + "hri thik", + "haw thorn", + "z ani", + "mac far", + "fi de", + "to tem", + "pe ds", + "fundament ally", + "cal ico", + "sin ner", + "j ä", + "hil de", + "ds d", + "ten ay", + "ta hit", + "mil f", + "lie b", + "inform ing", + "up lift", + "ra el", + "mortg ages", + "lec t", + "ii ii", + "guillau me", + "compos ites", + "old smobile", + "l end", + "gar th", + "com mish", + "bapti zed", + "scorpi ons", + "ru cker", + "bringback our", + "alli ance", + "thalap athy", + "tal i", + "sp ans", + "eri dge", + "wither spoon", + "lin da", + "sky lar", + "kor n", + "hom s", + "Ä į", + "sil enced", + "caf fe", + "ar ty", + "dist inguish", + "to wed", + "pun g", + "jessic a", + "ear nest", + "beau fort", + "t ama", + "study abroad", + "si khs", + "new bie", + "nav ratri", + "mar ble", + "loun ging", + "lit ter", + "dal it", + "so sa", + "iz es", + "gra de", + "com promising", + "tr iton", + "de tta", + "v j", + "chau ffe", + "spec tral", + "powe red", + "montess ori", + "artic ulate", + "hal ton", + "al co", + "ye y", + "mn twins", + "acoun ty", + "ðŁijı ðŁı¾", + "âī Ī", + "mad men", + "kal a", + "gru m", + "chi k", + "ati s", + "su me", + "akh tar", + "job search", + "high lighter", + "bo ath", + "âĦ ¹", + "tar zan", + "lam bo", + "âĽĦ ï¸ı", + "ox fam", + "dump ster", + "pretz els", + "mac os", + "incl ined", + "fac tual", + "adverti sers", + "shu i", + "pu ree", + "ml pfi", + "anti dote", + "cap o", + "pa str", + "merc ado", + "but ton", + "ar min", + "ag g", + "lol la", + "horri bly", + "er rands", + "christop he", + "time snow", + "monday motiv", + "li ss", + "scand als", + "mc i", + "dispropor tion", + "âĺ İ", + "sur pass", + "samar itan", + "so tho", + "pu rest", + "fl att", + "trivi atuesday", + "delec table", + "leop old", + "hermi one", + "chou dhary", + "en rich", + "¡ ¡", + "subsi diary", + "ine qualities", + "bachel or", + "auto immune", + "la kota", + "i hop", + "ad jec", + "the simpsons", + "sh es", + "se k", + "gret chen", + "up stream", + "hin akhan", + "coper nic", + "x tina", + "lu g", + "tough ness", + "e ad", + "cli pped", + "bi us", + "sl v", + "fah ren", + "dee pak", + "ca u", + "x an", + "im mature", + "dig ni", + "bo bs", + "shred ding", + "but tery", + "accommod ations", + "de ven", + "chun ks", + "super league", + "sky bet", + "kil dare", + "je et", + "ë į", + "ce k", + "wrec ks", + "pro pane", + "oh l", + "tb d", + "quo i", + "trum pp", + "mi mo", + "reluct ant", + "ver ne", + "o ic", + "ma gh", + "ar nau", + "se ver", + "li dge", + "stair way", + "kicchasu deep", + "ðŁĶ º", + "mach ining", + "aama admi", + "ot i", + "c da", + "al it", + "pan y", + "inst alls", + "ac ct", + "e shop", + "di em", + "hard well", + "fulfill ment", + "sc afe", + "qu ack", + "extrac ts", + "swee tened", + "fi ghton", + "f di", + "d inger", + "wal tham", + "us ur", + "refe rees", + "seok jin", + "gran n", + "af rin", + "th n", + "sch af", + "par cels", + "bet is", + "amar ine", + "nom an", + "kh tar", + "mor itz", + "cou pling", + "bar ons", + "ðŁIJ ¸", + "à ¸", + "sl p", + "sad ler", + "x ander", + "tri ad", + "mc millan", + "kh z", + "divi ding", + "ìĹijìĨ Į", + "dar yl", + "zed d", + "le ys", + "pla ques", + "flu ori", + "tipper ary", + "on nell", + "di dier", + "lang ford", + "im c", + "the sun", + "bir dies", + "ar cha", + "ye ssss", + "t di", + "dar ia", + "cand ace", + "al tam", + "pal aces", + "ch it", + "sant am", + "event ful", + "book of", + "ad b", + "mon stax", + "cre ole", + "co el", + "âĸ ½", + "we aren", + "sten nis", + "she ath", + "ati sm", + "gron ingen", + "mlpfi m", + "le pre", + "wrong ly", + "rsp ca", + "rendez vous", + "acknowle dging", + "pel vic", + "solic itor", + "sla ys", + "nue stra", + "lo d", + "is lander", + "fer oci", + "fashion show", + "ra ss", + "dge on", + "adole scents", + "sma shes", + "negli gence", + "grate ful", + "ved ere", + "sw oop", + "ing l", + "apol ice", + "vand alism", + "gan n", + "jo ao", + "di supdates", + "zimbab we", + "under age", + "radi ance", + "w of", + "bour geo", + "pla s", + "cr ani", + "gh ue", + "wrec kem", + "warran ts", + "re form", + "jim mie", + "at wood", + "ys l", + "neil himself", + "l bj", + "i man", + "tan to", + "nois se", + "ver bs", + "equip o", + "al together", + "mam ent", + "l ice", + "dou glass", + "tier ney", + "pri med", + "j hal", + "furn itu", + "braz ili", + "v ill", + "past els", + "n ison", + "u ff", + "paral ysis", + "jay e", + "im po", + "ðŁij ģ", + "strate gically", + "pakistan is", + "was sup", + "super bike", + "thank u", + "tru elove", + "sha ikh", + "israel is", + "vi p", + "to g", + "li en", + "la ker", + "grey hounds", + "cul ars", + "bian chi", + "balot elli", + "ar ran", + "loo s", + "str ates", + "he bron", + "ar vo", + "sunder land", + "the al", + "tomb stone", + "sand man", + "c pac", + "thanks giving", + "love him", + "lat ino", + "an in", + "aka if", + "ĭ ãĤ", + "tor quay", + "di est", + "alli anz", + "ðŁĺ ķ", + "golf club", + "cl lr", + "wal cott", + "sch nau", + "promp ted", + "nomin ating", + "len nox", + "val et", + "mon ro", + "may ward", + "e ph", + "ðŁĶ Ķ", + "inter oper", + "r da", + "re flex", + "arm chair", + "ê° ķ", + "stri pper", + "por ti", + "ph arm", + "ham za", + "ni reland", + "ne ue", + "h pv", + "port foli", + "sun burn", + "fris bee", + "be al", + "bapti ste", + "x h", + "ty m", + "pr ati", + "o vers", + "haz rat", + "deser t", + "der ry", + "us ky", + "em mett", + "ach arya", + ")_/ ¯", + "shu d", + "may a", + "ham ill", + "ra im", + "nr c", + "fitt ings", + "cur vy", + "ðŁı ĩ", + "ster ling", + "ॠĢ", + "wal kin", + "short cuts", + "mil ly", + "ast ur", + "alpha be", + "pl i", + "pe z", + "miss you", + "rad ford", + "ml g", + "ta eyang", + "notjust lakes", + "du mps", + "seren dip", + "le ur", + "ra ving", + "e ster", + "de priv", + "absc bn", + "ðŁijĩ ðŁı»", + "scar city", + "o cr", + "mean ings", + "cap t", + "da hl", + "fer mentation", + "bri oche", + "to win", + "out lander", + "massi mo", + "en cro", + "ðŁ¥ ³", + "buil t", + "po tam", + "kir i", + "tm w", + "monit ored", + "k ites", + "peoples vote", + "gray son", + "íģ ¬", + "afri ka", + "a dies", + "i vote", + "gy ne", + "g annon", + "di x", + "c mc", + "ou ral", + "fox andfriends", + "bel i", + "ig ne", + "gl an", + "katrin akaif", + "co politics", + "qual itative", + "p si", + "lu cci", + "disc oura", + "âĺ ®", + "kel li", + "gau tam", + "carac as", + "reale st", + "pu la", + "in us", + "hill top", + "make aw", + "atten borough", + "tw y", + "r arity", + "peck ham", + "ma hon", + "corn elius", + "clin icians", + "ton line", + "tb i", + "paradi se", + "ka si", + "inev it", + "fresh ness", + "colling wood", + "lun atic", + "defen se", + "cop d", + "in fra", + "wain wright", + "sains bury", + "alab am", + "te ma", + "lac o", + "chec ker", + "releg ated", + "tren t", + "stal ks", + "huff post", + "bhubanes war", + "ast ral", + "share your", + "prim rose", + "hi me", + "cat an", + "end ment", + "en dow", + "cle mens", + "mal oney", + "hil ary", + "game time", + "den ise", + "collabor ators", + "b wo", + "radic als", + "gue tta", + "ici on", + "au a", + "snap matic", + "sat chel", + "excav ation", + "base man", + "s ão", + "gn ation", + "fel d", + "surve y", + "shah zad", + "ma st", + "anirud hofficial", + "tru cker", + "ot ago", + "geo graph", + "ethe l", + "âļ¡ï¸ı âļ¡ï¸ı", + "s ver", + "mu tt", + "internetof things", + "ancho red", + "wh ouse", + "bang la", + "bal main", + "ç¹ ĭãģ", + "break fa", + "á Ģ", + "twi ster", + "te tris", + "ca v", + "stag s", + "g z", + "au b", + "stor med", + "hel ens", + "yar mouth", + "st asy", + "gustav o", + "co sc", + "vin son", + "up p", + "sc ricket", + "assump tions", + "app e", + "nu h", + "u er", + "pre mise", + "n aga", + "e amon", + "coron ary", + "na f", + "north side", + "el mer", + "ro tar", + "out lining", + "el f", + "re surg", + "kat elyn", + "in can", + "hyster ia", + "ce e", + "am bani", + "pro lly", + "Į ãĤĬãģ", + "ax es", + "san jose", + "rem brandt", + "mag pie", + "even ly", + "scor sese", + "qu aint", + "f g", + "b buk", + "indian football", + "weare all", + "spd wy", + "pis ces", + "ec g", + "âĺħâĺħâĺħâĺħ âĺħ", + "pre orders", + ": |", + "ni pple", + "sal azar", + "ju me", + "jail break", + "min n", + "bas sett", + "ze tta", + "jef free", + "ad jun", + "tic on", + "san diego", + "drink local", + "chol era", + "solic itors", + "o bo", + "com post", + "ni an", + "wr a", + "tre ach", + "ic ic", + "profession al", + "del ve", + "leg ate", + "histor ia", + "cro issant", + "con noisse", + "nam o", + "palli ative", + "chem trails", + "i ority", + "global warming", + "comic art", + "behavi oural", + "re sted", + "li as", + "cli mates", + "Ł ãģĦ", + "rut land", + "nou rish", + "menopau se", + "hot ties", + "demen ti", + "ve spa", + "mel ville", + "anal ogue", + "tz man", + "str ung", + "im perfect", + "gl are", + "cir cling", + "ros berg", + "rec o", + "oc ity", + "lo ire", + "em be", + "do ssier", + "ne el", + "nan do", + "me a", + "gal vani", + "fin esse", + "ag p", + "berke ley", + "asi m", + "âĺº âĺº", + "quil ted", + "ish ere", + "un matched", + "po tion", + "for z", + "at re", + "selfi es", + "juli ana", + "ðŁļ ¶", + "âĸ º", + "mel ton", + "âłĢâłĢâłĢâłĢ âłĢâłĢâłĢâłĢ", + "spin rilla", + "pur cell", + "ed p", + "at leti", + "tony awards", + "ra ja", + "pro gno", + "mol ten", + "stu ff", + "p ally", + "nobel prize", + "âĻ» ï¸ı", + "spiritu al", + "spe ake", + "sa sha", + "bri um", + "tru ss", + "critici ze", + "assassinscre ed", + "yor uba", + "u lo", + "fire man", + "workin progress", + "ef cc", + "fla res", + "ro bot", + "hi kers", + "cl l", + "shado wing", + "pat sy", + "leh man", + "c ns", + "å ±", + "guad al", + "à± į", + "ra pe", + "r honda", + "paralle ls", + "son ja", + "langu age", + "land ings", + "z ola", + "cr amps", + "bur ning", + "apprais al", + "jol la", + "ham m", + "kas a", + "gul ly", + "f go", + "uly sses", + "ri be", + "ðŁĴ Ħ", + "ib u", + "eti enne", + "bri ar", + "fin ely", + "comb ating", + "y ql", + "go tham", + "we chat", + "to paz", + "primar ies", + "l se", + "iz z", + "hel e", + "dispon ible", + "cy stic", + "bel ichick", + "th rush", + "kansas city", + "ge om", + "soli di", + "red bubble", + "by stand", + "cambridge shire", + "par fait", + "ast le", + "ow o", + "ind ore", + "stom ping", + "sm elly", + "ðŁ¤ ĸ", + "locom o", + "adm itting", + "hol me", + "clock wise", + "min sk", + "mc co", + "for get", + "ev p", + "cam ra", + "ab ella", + "yo tes", + "universit yof", + "mé xico", + "silver ado", + "ric ket", + "crom bie", + "pu j", + "eradic ate", + "deli ght", + "y go", + "glam ping", + "vic a", + "du ggan", + "coun ters", + "cf d", + "sc our", + "react js", + "pu ram", + "paras ites", + "in ki", + "vill en", + "stel la", + "li mbo", + "ang as", + "k cr", + "ðŁĴļðŁĴļ ðŁĴļ", + "vap ori", + "mum ford", + "oli gar", + "à ¼", + "al oo", + "boo ties", + "ad r", + "k elli", + "dru mmers", + "av ici", + "nature uk", + "ron al", + "in trac", + "un splash", + "le che", + "g oma", + "el ine", + "envir o", + "bi onic", + "bu eno", + "mi k", + "av in", + "star ling", + "em powers", + "cake day", + "boy cot", + "ðŁĴļ ðŁĴļ", + "ðŁĮ¸ ðŁĮ¸", + "v ach", + "m ci", + "fractu res", + "ger i", + "sk ing", + "exclu ded", + "lu ce", + "ja ve", + "ig gy", + "evi den", + "aki stan", + "a wn", + "mor als", + "luci fer", + "ha ban", + "tumb ling", + "sunday motivation", + "mo sley", + "captain america", + "sch icago", + "the one", + "mo td", + "d ts", + "ðŁIJ ¼", + "rep ell", + "ii i", + "locu st", + "geo spatial", + "mer sey", + "immer se", + "desc end", + "ber nade", + "j s", + "boat sales", + "win der", + "cran k", + "sing leton", + "candid acy", + "ben a", + "ðŁı» âĢį", + "high lander", + "ol t", + "k prs", + "healthy lifestyle", + "four teen", + "end the", + "ith aca", + "circul ated", + "r ans", + "pre valent", + "ha vas", + "splend or", + "roo ster", + "kalamaz oo", + "jewell ers", + "enne dy", + "rou sey", + "es y", + "cann ons", + "ornam ental", + "// //", + "ren don", + "win ne", + "mol ding", + "eid mubarak", + "coun tess", + "simon a", + "ha wa", + "fo es", + "du ster", + "sb u", + "por tray", + "mar ries", + "goo dday", + "cho co", + "achi ever", + "ðŁĺ¹ ðŁĺ¹", + "pre neur", + "tr amp", + "tom i", + "n bat", + "garden chat", + "farra khan", + "ever glades", + "ab ru", + "sou sa", + "se ce", + "homes wee", + "terre strial", + "bar it", + "sri devi", + "ol u", + "mel inda", + "f rick", + "can dies", + "ðŁĺŃ ðŁĴķ", + "qu reshi", + "family fun", + "exor cist", + "cardin al", + "ny t", + "dies el", + "cu mulus", + "capric orn", + "si ology", + "lor na", + "dou gie", + "an die", + "super sport", + "c fl", + "п ÑĢи", + "say ang", + "pe ek", + "ภĬ", + "lo be", + "j em", + "ing lis", + "gg led", + "c sn", + "amne sty", + "chu ps", + "ba es", + "sau er", + "ðŁı IJ", + "mongo lian", + "en et", + "back street", + "dr illed", + "acce ssing", + "ce o", + "b se", + "ai ken", + "pur r", + "wor sen", + "whe res", + "war k", + "testi fying", + "bu ri", + "bla st", + "aw g", + "ðŁĵ ĭ", + "re defining", + "hear ing", + "u ci", + "c mp", + "bon i", + "tail oring", + "ta ji", + "noc chi", + "em t", + "stephen king", + "ne et", + "compla ins", + "campaig ner", + "luci ano", + "twili ght", + "ti esto", + "pas sports", + "flo yd", + "cathe dr", + "na ked", + "caregi ver", + "b coz", + "ade cides", + "ku ri", + "ly k", + "br aries", + "dren ched", + "disc lose", + "ðŁĴª ðŁı½", + "le blanc", + "je tty", + "gar ty", + "chip mun", + "b su", + "rhyth mic", + "ic z", + "fri d", + "anne x", + "ame x", + "solo ist", + "lanc ers", + "arro whead", + "speci fication", + "simul ated", + "na is", + "inver te", + "bo wing", + "wor ship", + "f z", + "abo ss", + "sha q", + "ì¶ ķ", + "challeng ers", + "an arch", + "aamaadmi party", + "ãħĭãħĭ ãħĭ", + "suffol k", + "so corro", + "sn ell", + "cla dding", + "absor bing", + "shaw a", + "particip ates", + "ðŁį Ķ", + "book stores", + "bak u", + "seap ort", + "ko jima", + "gab y", + "pack ard", + "electr ician", + "let it", + "mo wing", + "fa wad", + "young jae", + "hot mail", + "men ing", + "u rie", + "intim acy", + "con ti", + ": \")", + "lifeis good", + "in ciner", + "i dri", + "craz iness", + "jour nos", + "fran chi", + "bott len", + "al da", + "ff es", + "k x", + "south we", + "air a", + "clay ton", + "sco ti", + "f j", + "bri ga", + "ðŁ¤ĺ ðŁı»", + "demonstr ators", + "y z", + "stor k", + "na q", + "casc ades", + "travel chat", + "plat a", + "pad ma", + "fran ci", + "at tain", + "bat girl", + "lom bard", + "hoo s", + "d dos", + "neon atal", + "discla imer", + "r ss", + "r ant", + "di sen", + "tex aste", + "so cal", + "frac tal", + "cam ry", + "stri fe", + "sn acking", + "mu h", + "sant ander", + "mor ons", + "gra f", + "par ades", + "hu ston", + "dru pal", + "mi ento", + "kir stel", + "hy de", + "vom it", + "forti fied", + "sphin x", + "da v", + "bir yani", + "win nings", + "s baseball", + "mer ged", + "lovel ondon", + "ling ering", + "dream big", + "car leton", + "liveli hood", + "djan go", + "astri d", + "gri ds", + "down e", + "bru ised", + "s ne", + "scarec row", + "hel ium", + "f nc", + "bi ggs", + "an ter", + "restor ative", + "em pires", + "ab del", + "life style", + "kiwan is", + "colloqui um", + "me en", + "pr ick", + "anti que", + "ze b", + "mi mic", + "edmon ds", + "ðŁijĬ ðŁijĬ", + "q ing", + "pp el", + "mc gill", + "interpre ting", + "âŀ ķ", + "rash ad", + "do ka", + "narr ator", + "electro magnetic", + "ash by", + "sau ra", + "iran deal", + "âģ īï¸ı", + "krish nan", + "in di", + "ff en", + "bre a", + "os man", + "multin ational", + "chi ppe", + "recruit ers", + "aus biz", + "p ounding", + "re gen", + "cur sor", + "refu sal", + "mac s", + "in ak", + "ax ial", + "wa ifu", + "up cycled", + "hindu stan", + "cas sini", + "carly le", + "scrat ches", + "re ef", + "man atee", + "eat ery", + "ðŁĵ ¢", + "un condition", + "sen pai", + "on ther", + "comic book", + "pro sciutto", + "de mar", + "mi se", + "ma ge", + "fre ec", + "aye sha", + "al der", + "android games", + "ley ton", + "ho ck", + "door way", + "chicagof ire", + "aali yah", + "sw elling", + "bi x", + ". ðŁĺĤ", + "evan kirstel", + "torpe do", + "kon stant", + "genevie ve", + "ma ia", + "ha user", + "do torg", + "hide ous", + "fi k", + "sp raw", + "e ek", + "z appa", + "wan dered", + "' '", + "ra jan", + "bam bi", + "( $)", + "wid ening", + "tool box", + "sa ir", + "illumin ating", + "pra ys", + "out patient", + "i w", + "day o", + "lo b", + "sw fl", + "sha des", + "gu ms", + "coo kin", + "ko di", + "gri ffin", + "traum ati", + "ste a", + "slaugh tered", + "god bless", + "air time", + "pseu do", + "b sa", + "hau led", + "ar if", + "à¸Ńภĩ", + "le l", + "wc po", + "mil iti", + "char ters", + "worl da", + "ru k", + "k gs", + "digital india", + "is able", + "idyl lic", + "esp ino", + "marie tta", + "e bo", + "team canada", + "ab our", + "wil ton", + "rock stars", + "fav ored", + "phys ic", + "wrink le", + "tb r", + "d print", + "ball arat", + "ad al", + "z ey", + "ðŁĺį ðŁĶ¥", + "tom lin", + "mt r", + "pal sy", + "fener bah", + "tight en", + "phil ia", + "ir oning", + "ry u", + "b ant", + "enqu ire", + "ca ir", + "abur ger", + "tru n", + "green berg", + "chau han", + "ir ina", + "sh ani", + "trend setter", + "pre tt", + "zaf ar", + "alo ve", + "v ici", + "pan ic", + "no o", + "lu stre", + "disrup ted", + "bal lis", + "son sof", + "mon si", + "inst ac", + "ake st", + "ëĭ ¤", + "kw ame", + "horror movies", + "distric t", + "sau cy", + "mb an", + "ar mies", + "with drawn", + "med ics", + "loft us", + "er oom", + "be kind", + "ar ns", + "all on", + "un ison", + "davi ds", + "cr at", + "nicot ine", + "so or", + "sm x", + "on co", + "cospla ying", + "zombi es", + "har ms", + "e ger", + "ro sy", + "moon shine", + "fe in", + "ce tt", + "du brov", + "reg ents", + "ben itez", + "ðŁijıðŁı¼ ðŁijıðŁı¼", + "ste c", + "m alia", + "prioriti ze", + "ic eland", + "ft se", + "v amo", + "lam ont", + "homo sexuality", + "bre es", + "regu i", + "cb p", + "te j", + "sky sports", + "deter gent", + "sha sta", + "de rel", + "conserv ancy", + "colori zed", + "accol ades", + "vis o", + "show your", + "nan ow", + "bice ps", + "us ability", + "bi m", + "dailys ketch", + "pearl jam", + "stran gest", + "mega deth", + "broad casts", + "bar ren", + "ar ton", + "chri ss", + "confi gu", + "lu res", + "is the", + "e ul", + "railway ana", + "global health", + "gi anni", + "u aap", + "s lum", + "consci ously", + "ab re", + "n up", + "bud get", + "v ada", + "e sch", + "real ness", + "er ased", + "th unt", + "be z", + "armist ice", + "ðŁij ¹", + "sh run", + "o led", + "driver less", + "ðŁ¤· ðŁı»âĢįâĻĢï¸ı", + "won dr", + "sk an", + "sal aam", + "mother land", + "h wang", + "gen o", + "gang nam", + "tw right", + "endor sing", + "en ic", + "ador ation", + "pau sed", + "patric ks", + "do cked", + "plat te", + "ff xv", + "ethnic ity", + "auto show", + "side show", + "after life", + "re located", + "orphan ed", + "food network", + "dare to", + "and ra", + "sla ps", + "v live", + "swim s", + "re imagined", + "mist le", + "re vise", + "real ity", + "bhar ti", + "ðŁĴĻ ðŁĴĽ", + "late st", + "prou dest", + "gra sses", + "lan yard", + "fresh est", + "carcin oma", + "anom aly", + "zieg ler", + "sum ner", + "ly rix", + "gor g", + "is d", + "av el", + "swild life", + "me squ", + "john cena", + "euro league", + "sab er", + "master ful", + "yar ra", + "cogn ition", + "jacob son", + "abo lic", + "sir loin", + "shuk la", + "moj ito", + "su pere", + "st weet", + "me z", + "e sa", + "rudol f", + "gur a", + "where you", + "tt m", + "win s", + "trust worthy", + "ny k", + "bra den", + "table top", + "good food", + "es on", + "be k", + "lingui stic", + "gra ys", + "ch ath", + "h cs", + "mon i", + "de ans", + "cu ssions", + "ch ell", + "slo ws", + "he mi", + "d app", + "shar pie", + "boo sters", + "a os", + "str ack", + "se dona", + "mu eller", + "hard wick", + "or nate", + "thor a", + "sal ud", + "o twol", + "ch um", + "mi ho", + "for age", + "thel ittle", + "tear ful", + "ones elf", + "min dy", + "sm g", + "gmb h", + "emer ald", + "ðŁĶ´ âļªï¸ı", + "tu tti", + "recep tions", + "re vising", + "i brox", + "tope ka", + "sal ami", + "expan se", + "i books", + "dob son", + "cli o", + "at s", + "ðŁļ Į", + "mo ha", + "is ance", + "shu tters", + "moo t", + "jan ine", + "marvel comics", + "jor dani", + "pos er", + "kenne th", + "hy ung", + "de ja", + "ase ball", + "speci ality", + "eu ston", + "classic car", + "had ith", + "ðŁIJ ī", + "chas ing", + "iz o", + "gros ven", + "ag lia", + "thisdayin history", + "t row", + "om ile", + "hu ar", + "by n", + "sal ine", + "div ine", + "demon ic", + "ty ran", + "han dover", + "revit alization", + "pa ella", + "cryp tic", + "se dg", + "m end", + "dun kirk", + "bre d", + "wal d", + "sport scar", + "a ard", + "whe aton", + "da ener", + "k lan", + "br t", + "bakhta war", + "spi res", + "schu bert", + "ro ti", + "poli sh", + "o se", + "ag ame", + "wonder con", + "prote stant", + "bo sa", + "ðŁĺ Ł", + "d ü", + "joy ride", + "ger trude", + "âĿ Ŀ", + "gil a", + "v h", + "tw a", + "tra v", + "swal lowed", + "star ve", + "la in", + "ent ren", + "rei ki", + "su kh", + "cra ic", + "az u", + "web page", + "kee fe", + "hypo the", + "hir sch", + "hel le", + "camp ground", + "w amy", + "tra vi", + "sha hi", + "san deep", + "ru i", + "han uman", + "dw p", + "reposit ory", + "no or", + "no ff", + "un real", + "p ell", + "black history", + "har vick", + "ma scar", + "pay ee", + "pa sha", + "gastron omy", + "d ÃŃ", + "ai g", + "rosen thal", + "open day", + "embelli shed", + "t tip", + "sun bathing", + "go pack", + "end ome", + "ï¸ı #", + "invali d", + "final four", + "st fu", + "squish y", + "ra sta", + "mo sch", + "jam esc", + "die trich", + "sel a", + "mel b", + "el vi", + "t dp", + "sun i", + "sli t", + "j ha", + "bi za", + "spi ked", + "l li", + "l illard", + "vam pi", + "syno psis", + "az har", + "kendrick lamar", + "ĮãĤĬãģ ŁãģĦ", + "heart less", + "country file", + "air play", + "arrog ance", + "pre e", + "virtu oso", + "ãħłãħł ãħłãħł", + "raj u", + "le bu", + "for ward", + "tu g", + "dro s", + "mondaymotiv aton", + "concep cion", + "thel o", + "pad i", + "looo ol", + "ÑĢ од", + "it ss", + "eth ical", + "end uro", + "__ :", + "expend iture", + "mon ste", + "mas king", + "terri ers", + "ib is", + "e mber", + "cu mple", + "punctu ation", + "pi per", + "ir vin", + "ade e", + "yy yyyy", + "flash backs", + "cel sius", + "don nie", + "bo gota", + "ben evol", + "the script", + "shil pa", + "pro se", + "fin dia", + "ze ke", + "ne ko", + "do ves", + "blues lyrix", + "fro sh", + "sowe to", + "mp lo", + "al ai", + "sab i", + "raq qa", + "wf tv", + "stro ller", + "ian somerhalder", + "ðŁĶ ª", + "an on", + "mo seley", + "! ?!?", + "sta king", + "mol y", + "car tri", + "c sg", + "ast or", + "transc end", + "ma er", + "de ux", + "cow girl", + "sas k", + "pun ter", + "ma ken", + "o ates", + "love tt", + "grow ler", + "sag in", + "v n", + "ssi ble", + "officeof rg", + "y mc", + "sab ar", + "faul ty", + "ap ha", + "ak on", + "ðŁij «", + "snow don", + "ae w", + "raise the", + "ðĿ ĵ", + "grue some", + "clement ine", + "sp ing", + "lat a", + "worlden viron", + "mi mic", + "can aria", + "bakhtawar bz", + "ao a", + "fal a", + "ãĤ Ń", + "avi va", + "you uuu", + "thi gh", + "la dders", + "gu mbo", + "tz ky", + "fu zz", + "plastic pollution", + "est ate", + "strength ened", + "k ant", + "dr in", + "cal vert", + "transform ational", + "frigh tened", + "mac lean", + "elited angerous", + "ear thy", + "t son", + "to da", + "j nu", + ".. ,", + "mic hal", + "i ban", + "je ong", + "is real", + "sim coe", + "exclu sives", + "blue bells", + "ben e", + "te u", + "pil sner", + "pens ke", + "athe ists", + "m pu", + "cartag ena", + "ðŁĴĹ ðŁĴĹ", + "million aires", + "kk kk", + "it ar", + "subscri ptions", + "remo te", + "ma fi", + "hin ton", + "w cc", + "ho k", + "ds b", + "ab leton", + "sevent y", + "pun ks", + "e indhoven", + "sh one", + "mcfar lane", + "lim popo", + "empha si", + "à ¼", + "sin fo", + "pe tre", + "man grove", + "ch ino", + "ber tie", + "play lists", + "push awards", + "p af", + "deb bie", + "c do", + "r ino", + "ðŁı¾ âĢįâĻĤï¸ı", + "fol ke", + "bon nar", + "th ine", + "sl an", + "hal ter", + "evi e", + "aw some", + "vul tures", + "spar ky", + "seiz ures", + "âľ Ķ", + "ram one", + "ine ffe", + "al n", + "pro ctor", + "ast ra", + "the voice", + "gro te", + "sci on", + "dead line", + "am aya", + "tain ted", + "patter ned", + "exce eding", + "cross fit", + "kay lee", + "drop box", + "ru shes", + "tack led", + "mo by", + "retro gamer", + "n cbd", + "benef itting", + "shay kh", + "guild hall", + "gen try", + "dream cast", + "dread ed", + "bun dled", + "th aw", + "revol ving", + "n pt", + "kylie jenner", + "imagin ative", + "ron i", + "over came", + "family time", + "ds burg", + "car naval", + "relation ship", + "recogni zable", + "cor oner", + "ho le", + "fan fic", + "emir ates", + "bur ritos", + "analy se", + "thin ner", + "ne es", + "galli poli", + "bl r", + "cat woman", + "-- >>", + "au lt", + "ada ily", + "nau ghty", + "ili o", + "solit aire", + "mtv br", + "jocel yn", + "arun ach", + "rep ent", + "south gate", + "hy acin", + "essenti al", + "fent on", + "and um", + "it or", + "go pal", + "sl inger", + "po sei", + "aw il", + "wi elding", + "ra ila", + "eli as", + "a sto", + "à ¤", + "tend ency", + "str ata", + "ker t", + "< -", + "im acele", + "da es", + "sti mulus", + "han ley", + "fit nes", + "ec stasy", + "lim ous", + "ha iling", + "ðŁ¤ Ń", + "chis wick", + "tar ies", + "sla v", + "pul i", + "moderni zation", + "black mail", + "b ingham", + "h fx", + "+ +", + "ðŁĩ®ðŁĩ ³", + "ni v", + "we a", + "profess or", + "k off", + "bol ster", + "su ave", + "sequ ences", + "pepper oni", + "not te", + "dre n", + "ãģ¨ ç¹ĭãģ", + "hs v", + "o ga", + "ap tly", + "z ad", + "excel si", + "rin ka", + "mol dova", + "min n", + "ma bel", + "conferen cing", + "bas ing", + "of er", + "ob si", + "hamill himself", + "care less", + "brief ed", + "inhe rent", + "par ish", + "dub nation", + "town sville", + "sar awak", + "gee ky", + "doncaster isgreat", + "was abi", + "gu p", + "phen o", + "dra inthe", + "carrie underwood", + "ble eds", + "bbc world", + "ane w", + "alta f", + "dul wich", + "ani ston", + "w ti", + "sumat ra", + "gra fton", + "bl n", + "me ster", + "bode ga", + "re go", + "es q", + "an jo", + "sump tuous", + "mai sie", + "ï¿ ½", + "wil t", + "jak ob", + "el vis", + "se pul", + "mu ster", + "air pollution", + "president e", + "happy monday", + "exten sively", + "fl ondon", + "t ls", + "play ing", + "pe ed", + "din ho", + "var dy", + "pi ka", + "n iro", + "au cus", + "ðŁį ¦", + "nu ll", + "el ondon", + "juvent us", + "imag ines", + "dis ab", + "lit o", + "d ura", + "work places", + "promo te", + "mc caf", + "wood work", + "waw x", + "à® ª", + "tt ino", + "shar i", + "sem per", + "better together", + "ðŁijĬ ðŁı»", + "ze bra", + "pon dering", + "en chil", + "ho m", + "cosm ic", + "tan z", + "mo cked", + "ec cc", + "ath ed", + "abo lish", + "prop eller", + "paris agreement", + "assemb lies", + "indu stry", + "fraudul ent", + "pe sa", + "chang min", + "ax x", + "ðŁĴ µ", + "irr ational", + "cu sa", + "ramad han", + "octa via", + "on elove", + "jac ki", + "bar ak", + "taxi der", + "seri ous", + "nathan fillion", + "mc en", + "ch k", + "po part", + "grav ity", + "copp ola", + "reading fc", + "illu sions", + "j ig", + "ww x", + "re sh", + "ex porting", + "buzz ard", + "âĻ ¤", + "p cm", + "lan apar", + "ko s", + "arom as", + "antal ya", + "ww dc", + "ven a", + "phil a", + "ball in", + "ðŁij Ħ", + "quin ta", + "ma o", + "f ery", + "eigh ty", + "sentim ents", + "safe guarding", + "r wa", + "pu ffs", + "luc ille", + "de cath", + "sl u", + "nu gent", + "de ter", + "braz il", + "ze iss", + "super bowl", + "subsi dy", + "alter n", + "hi dalgo", + "enz ymes", + "ä ½", + "tag ne", + "hair dresser", + "adri en", + "walk out", + "oppo ses", + "can tina", + "bed side", + "af an", + "ðŁĶ Ĺ", + "prophe tic", + "dan es", + "un successful", + "super charged", + "pk k", + "exem ption", + "hart le", + "secu lar", + "cli pping", + "br s", + "united way", + "c net", + "pat chy", + "ha gan", + "e en", + "âļ ľ", + "var a", + "sym pathi", + "never trump", + "affir mation", + "om f", + "ny cfc", + "ma ja", + "sur ro", + "keer th", + "up scale", + "sandal wood", + "mon archy", + "kno bs", + "å ĭ", + "po tholes", + "hunger games", + "ter races", + "na sir", + "coun sell", + "welcome to", + "wa q", + "se aman", + "m ita", + "stun ningly", + "on theroad", + "in ability", + ") !!", + "bon go", + "ant v", + "sp ut", + "worldenviron mentday", + "resu sc", + "y td", + "fi m", + "eun hyuk", + "sa chin", + "rose anne", + "cler mont", + "ape c", + "am ina", + "v ening", + "n antes", + "al most", + "sin us", + "ex as", + "ty l", + "ti en", + "ple ad", + "lanc s", + "bur naby", + "re k", + "jo om", + "observ ers", + "disco graphy", + "cl g", + "âĻ ¦", + "sn ack", + "r ti", + "o ily", + "crystal li", + "bru te", + "web development", + "topp ings", + "la f", + "an is", + "ad der", + "reli ving", + "car lin", + "battle of", + "we g", + "syri an", + "pon t", + "n dc", + "lagh ate", + "yu ma", + "sp p", + "p iti", + "ro bbing", + "mart ing", + "rey kja", + "raj put", + "nc ds", + "kie wicz", + "âĢ¢ âĢ¢", + "vam pire", + "substan tially", + "opio ids", + "nepal i", + "k line", + "ar oo", + "under stand", + "lit t", + "u it", + "thro mbo", + "sar ies", + "qu ot", + "b alling", + "t tr", + "s gh", + "philip p", + "br ant", + "ac l", + "m ello", + "whit taker", + ". ;", + "defi ant", + "b gc", + "repl ying", + "mir ren", + "metamor pho", + "sch wab", + "bul ge", + "utili zed", + "pick ering", + "par don", + "d sa", + "ภĪ", + "doo ley", + "cumul ative", + "Ð »", + "ur gency", + "e mir", + "+ /-", + "¦ Ī", + "ot as", + "âı ³", + "station ed", + "grape vine", + "ar ac", + "karan johar", + "f ancy", + "sau l", + "coo gs", + "lgbt q", + "ا٠ħ", + "jav i", + "u mmer", + "pl l", + "den is", + "dai pur", + "pu ffin", + "lewi sham", + "fand om", + "co pe", + "ves matter", + "s ve", + "hel pless", + "deo dor", + "ostr ich", + "kaz an", + "friday the", + "con dor", + "v x", + "sophom ores", + "rob les", + "cu tt", + "cli mbers", + "ë¦ ¬", + "sle g", + "sn f", + "mac ys", + "hydr ating", + "grou pe", + "po yn", + "mou lin", + "hg tv", + "lmfa ooo", + "sulph ur", + "asdfghj kl", + "annab elle", + "hump back", + "bra ved", + "viswas am", + "multi purpose", + "hu midi", + "escor ted", + "barb ican", + "f ad", + "cor sa", + "ðŁ¤ «", + "pi ppa", + "here to", + "can y", + "ser gi", + "or cas", + "o vie", + "ed ou", + "s any", + "glob alization", + "man cini", + "food truck", + "f is", + "defi brill", + "sch re", + "sma fia", + "love wins", + "la ut", + "k aka", + "hol lande", + "game on", + "resurg ence", + "out side", + "olympi ad", + "int an", + "abstr action", + "rapi d", + "pal om", + "cal le", + "jas min", + "attack ers", + "swag g", + "mit ra", + "ky lo", + "à® ²", + "her mitage", + "gor do", + "e ira", + "so sfam", + "roll out", + "exc ite", + "sy nod", + "mer rill", + "c als", + "as sa", + "liveli hoods", + "ju ve", + "the black", + "gopack go", + "ant lers", + "alban ian", + "wool ly", + "qu iche", + "puri fication", + "are th", + "smar thome", + "ne k", + "all blacks", + "mex icans", + "is m", + "ger ms", + "comple xion", + "mar ck", + "u shi", + "ðŁIJ IJ", + "char l", + "ca stic", + "till erson", + "giuli ani", + "biode gradable", + "mal bec", + "bo is", + "ju bil", + "im es", + "r ame", + "gene tic", + "esp nu", + "ch ley", + "so ho", + "go pher", + "g sc", + "buu ren", + "cu be", + "bridesma ids", + "webin ars", + "to e", + "mani pur", + "viol ently", + "notic ias", + "ex changing", + "chi ev", + "replac eable", + "muay thai", + "bu ss", + "sp il", + "instal ment", + "div ya", + "cait lin", + "o lim", + "fil tering", + "whirl wind", + "sta red", + "prior it", + "pr am", + "pompe ii", + "mono logue", + "k ite", + "bu ka", + "âĢ¦ ..", + "vac cine", + "bre ro", + "woz ni", + "sol ent", + "re ferr", + "my rt", + "gridi ron", + "galatasar ay", + "fro ze", + "clare mont", + "ðŁ¥ ĥ", + "victori as", + "ssel dorf", + "pa stures", + "net neutrality", + "ch or", + "ðŁij ģ", + "ಠ¿", + "we ho", + "symp tom", + "jo sel", + "in ous", + "dragon con", + "power ball", + "p te", + "four thofjuly", + "ec la", + "ear buds", + "where abouts", + "salt life", + "depriv ation", + "ch ter", + "wi ggle", + "syste m", + "ps st", + "ch az", + "d any", + "ri mo", + "oax aca", + "lanapar rilla", + "barcel on", + "melanch oly", + "way back", + "ho tro", + "n si", + "l illy", + "kur o", + "ja han", + "intellec t", + "board game", + "ðŁı Ĭ", + "sneak peek", + "k prc", + "jail s", + "cand el", + "zan zi", + "mor timer", + "star ch", + "ra gs", + "p fa", + "long live", + "k art", + "gir ona", + "cro cker", + "christop h", + "precau tions", + "war ship", + "per m", + "paren t", + "van gogh", + "gif ford", + "allegh eny", + "ra yn", + "ut m", + "sten cil", + "rec alling", + "pen ney", + "z azzle", + "ìĥ Ŀ", + "hin ds", + "aren as", + "nu ev", + "law ler", + "gu in", + "do this", + "ðŁij ķ", + "ì¶ķ íķĺ", + "we g", + "ti b", + "ri din", + "complex es", + "turbul ent", + "pe sos", + "de marcus", + "vall arta", + "sam sun", + "kis ses", + "hein rich", + "deport es", + "wil ms", + "ur d", + "then ext", + "inki gayo", + "ho wi", + "fir sts", + "carri age", + "clean liness", + "mas war", + "is ch", + "ax el", + "si zzle", + "road house", + "fr ans", + "ent ourage", + "co bble", + "boo th", + "benedic t", + "tal on", + "fc u", + "year ofthe", + "ray on", + "raider nation", + "fo yle", + "ko val", + "pi anos", + "l pg", + "bur mese", + "man ure", + "geo caching", + "cosc ino", + "b np", + "fer ra", + "stro phy", + "mar ais", + "ce es", + "legen dof", + "kat niss", + "eno ch", + "av ed", + "you know", + "d prk", + "ðŁĺ¢ ðŁĺ¢", + "sp un", + "pro st", + "sor rows", + "cent red", + "ke a", + "gal icia", + "? ðŁ¤Ķ", + "ÑĢод а", + "bou chard", + "ðŁĴĻ ðŁĴľ", + "yu i", + "seed lings", + "jon ah", + "reco vers", + "ny rd", + "board room", + "su ma", + "my japs", + "tun g", + "sha i", + "ir gc", + "eli o", + "wag ons", + "ka shi", + "polic emen", + "john nie", + "ale coscino", + "shop ify", + "dot ted", + "de tri", + "va w", + "to fficial", + "in your", + "chal mers", + "trac ed", + "no vi", + "by es", + "ari el", + "nipp on", + "la pel", + "gri ez", + "b gs", + "fool ing", + "d ita", + "vijay sethu", + "nm wx", + "as ot", + "kr anti", + "hel m", + "ve di", + "sic kest", + "mo chi", + "k abo", + "shru bs", + "he red", + "b sp", + "sq m", + "ham r", + "dul kar", + "anth a", + "nr f", + "avoid ance", + "at en", + "publi x", + "be arers", + "nas i", + "ha p", + "h ells", + "ðŁĸ ¥", + "ภ·", + "thelast jedi", + "oh wx", + "ðŁį «", + "wa hoo", + "there se", + "rec aps", + "ss nhq", + "bird photography", + "v ay", + "pet ti", + "pau lo", + "bel vedere", + "( *", + "gr l", + "du vet", + "c pec", + "sa it", + "por sch", + "meas urable", + "avi ators", + "fre mantle", + "bre en", + "on om", + "me and", + "life saving", + "eu ref", + "en don", + "embar as", + "aira sia", + "el is", + "dun kin", + "star magic", + "s ill", + "porto bello", + "ki efer", + "ex e", + "mu ted", + "ãģ ¦", + "we thepeople", + "logi a", + "liber al", + "theforce awakens", + "min ed", + "haun ts", + "freck les", + "care taker", + "s india", + "âķ IJ", + "dev lin", + "list on", + "direction er", + "oh n", + "fi garo", + "em manuel", + "du bois", + "cl ones", + "bru ise", + "ðŁİĪ ðŁİī", + "disin fe", + "der matology", + "as r", + "s watch", + "dis comfort", + "tam anna", + "pi day", + "mack en", + "k atic", + "delu sional", + "shaw nee", + "gu d", + "al bino", + "p ali", + "din gh", + "cucu mbers", + "coffe y", + "anticip ating", + "treas ured", + "web summit", + "shel tered", + "sav or", + "pedago gy", + "m gs", + "sh ma", + "s bu", + "den ali", + "cam pos", + "bubble gum", + "o ir", + "le aps", + "y ler", + "r one", + "sansk rit", + "min t", + "meat less", + "futuri st", + "du de", + "a vel", + "prote sted", + "squ ire", + "z aki", + "sz n", + "har court", + "cycl one", + "bour dain", + "gather ings", + "d ant", + "advent urer", + "parag on", + "alt man", + "dd ing", + "ban erjee", + "snorkel ing", + "mother well", + "mis sy", + "en der", + "glo ws", + "ki wis", + "chick pea", + "por o", + "e fron", + "app t", + "u y", + "speci fied", + "gab by", + "e strada", + "com bos", + "bour bon", + "vin i", + "var un", + "steph ani", + "key words", + "car vings", + "amit abh", + "wr ought", + "tw al", + "re els", + "clu bbing", + "ubi quit", + "cri t", + "ambed kar", + "æ Ļ", + "prun ing", + "vaccin ated", + "boe ing", + "s ks", + "lo ona", + "hypno sis", + "edel man", + "pho l", + "he w", + "colo sse", + "mckin sey", + "u on", + "to te", + "sacrific ing", + "ox i", + "n ang", + "e mu", + "пÑĢи ÑĢода", + "m th", + "kers wednesday", + "argu ed", + "timel apse", + "ris king", + "regul ating", + "ni gh", + "likeli hood", + "cu bic", + "au ction", + "rein for", + "pi stor", + "no ses", + "ye l", + "snu ggles", + "pe i", + "jean ette", + "ta ku", + "ri th", + "guy z", + "ภŀ", + "y te", + "ver ted", + "pay soff", + "jau regui", + "hoo ligans", + "procedu ral", + "mi b", + "har dy", + "el eng", + "chec kers", + "all ine", + "the met", + "prou dof", + "keerth yofficial", + "collabor ator", + "ni u", + "infl icted", + "adv ani", + "re twee", + "memor iam", + "f icial", + "ti ghter", + "sal em", + "re viewers", + "br ics", + "ben digo", + "am ell", + "tur kish", + "sush maswar", + "paul son", + "pal awan", + "mol lie", + "stitch er", + "s burgh", + "ir u", + "hay dn", + "en ers", + "aro a", + "u zzi", + "saraj evo", + "hel a", + "apol lo", + "nine ty", + "vac a", + "sp on", + "vent u", + "jel ena", + "hei fer", + "avo ids", + "sp ine", + "pri ze", + "mar ist", + "re creating", + "me de", + "woo den", + "find lay", + "ro fl", + "n di", + "compreh end", + "yu go", + "y ü", + "to work", + "u fos", + "son ar", + "pi ston", + "recor ding", + "tent ative", + "art forsale", + "pel lets", + "fre do", + "ÙĪ ر", + "mu ses", + "custom ization", + "pro found", + "is ner", + "ide ally", + "si am", + "plan kton", + "cm dr", + "man ger", + "fran ken", + "customiz able", + "ठ®", + "walk away", + "swi vel", + "vast ly", + "no ton", + "lex a", + "ex moor", + "z as", + "tan te", + "reduc tions", + "lol ly", + "hip sters", + "benef ited", + "ë ²", + "ww www", + "mascul ine", + "fi ji", + "dre y", + "ph ill", + "ane ous", + "nic ol", + "men dez", + "disapp ro", + "ch ner", + "through s", + "shen mue", + "east man", + "ðŁIJ İ", + "yu ck", + "under tale", + "re ys", + "go beavs", + "eng en", + "c na", + "mer r", + "bir k", + "ãģ¨ç¹ĭãģ ĮãĤĬãģŁãģĦ", + "âĥ£ @", + "yn na", + "ste ed", + "offen der", + "at um", + "vani shing", + "presi denti", + "love them", + "g nocchi", + "fri ggin", + "per il", + "mad hya", + "ag ne", + "dee jay", + "mar nock", + "m tb", + "fold able", + "@ ___", + "stand re", + "bron x", + "bow ski", + "fin ite", + "cro ckett", + "b sf", + "ge tit", + "seren awilliams", + "mir o", + "ignati us", + "sla y", + "rin se", + "fon due", + "sel dom", + "s more", + "gan i", + "dy ce", + "dmit ry", + "cru mb", + "late post", + "pri mark", + "oh ana", + "flor als", + "do a", + "remembrance day", + "d ds", + "azi one", + "toon ami", + "air port", + "æĿ ±", + "th ad", + "fi st", + "dine sh", + "dr who", + "ad words", + "admi rer", + "pro je", + "kyrgy z", + "à «", + "manife station", + "le wan", + "j ic", + "thi bau", + "le ased", + "van ity", + "nouri shed", + "never theless", + "aug mente", + "fu elled", + "che ad", + "wil shere", + "ru di", + "p z", + "my co", + "mor ro", + "herbali fe", + "hardro ck", + "de man", + "dre ality", + "sp ades", + "ce vic", + "bha i", + "bar on", + "ultimat efan", + "hou news", + "to bi", + "stru t", + "ke el", + "affili ation", + "the masters", + "sm al", + "hu e", + "este ban", + "con v", + "om nic", + "datab ases", + "co v", + "ter ti", + "st g", + "snoop dogg", + "metab ol", + "leth bridge", + "ðŁı» âĢįâĻĢï¸ı", + "year ling", + "residente vil", + "nws l", + "iy aki", + "griez mann", + "c ous", + "ðŁĵĿ :", + "tor ian", + "sam i", + "ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ðŁĶ¥", + "g are", + "alli ances", + "whit field", + "we ther", + "refin ing", + "coy i", + "kra ken", + "ðŁĺĺ âĿ¤", + "singul arity", + "lil i", + "h ns", + "bol dand", + "waw rinka", + "misogy ny", + "lo vers", + "c q", + "b dg", + "ad ona", + "gar ter", + "women of", + "sc d", + "recogn ising", + "mun a", + "str ou", + "sign alling", + "lare do", + "hell boy", + "alek sand", + "un available", + "pedi atric", + "as in", + "mer ia", + "ri shi", + "futuri sm", + "w ye", + "polari zed", + "e we", + "pro pel", + "in forms", + "cre ase", + "~ \"", + "arti ston", + "like for", + "heidel berg", + "er ra", + "life in", + "len ny", + "inter rupt", + "cohe rent", + "ca z", + "vick ers", + "le veled", + "f bs", + "cab ins", + "bu mmed", + "apost les", + "we h", + "ten don", + "souven irs", + "infu ri", + "pier ce", + "asse t", + "m las", + "go th", + "di ggin", + "ann as", + "yl or", + "th waite", + "sw el", + "pan era", + "mur derers", + "croo ked", + "bs go", + "ac u", + "a on", + "re an", + "one of", + "ko hl", + "bloo dh", + "pest icide", + "lost dog", + "fle xing", + "ëĤ ĺ", + "su pra", + "eter nally", + "ðŁļ Ļ", + "pa olo", + "ol an", + "mom o", + "is elle", + "captain marvel", + "s lou", + "mistak enly", + "akhi lesh", + "mer t", + "il inan", + "bu on", + "bal kan", + "mir ro", + "mill en", + "der ail", + "dam on", + "tit i", + "bi os", + "re don", + "pic ard", + "par te", + "ðŁ¤ Ł", + "Ø º", + "son ics", + "fir sth", + "dd c", + "veg ans", + "tur ban", + "ni gan", + "lot tie", + "lyn don", + "star buck", + "pink floyd", + "life styles", + "am ara", + "a she", + "r sc", + "val a", + "sm er", + "cw gc", + "cli ent", + "buen as", + "jag an", + "coo ps", + "ðŁijij ðŁijij", + "speci alizes", + "snag ged", + "g lar", + "ben net", + "wildlife wednesday", + "bow den", + "pi k", + "art in", + "empor ium", + "ar l", + "re ba", + "pas ser", + "disappo ints", + "additi ve", + "âľĬ ðŁı½", + "bay er", + "missou la", + "ha skell", + "comm ences", + "ni x", + "ne man", + "explo ited", + "plastic surgery", + "cc d", + "aso cial", + "vo t", + "sie gel", + "fro ome", + "kap am", + "far a", + "e ha", + "pro bes", + "mw f", + "meet ing", + "p bb", + "ak ins", + "mistle toe", + "kingdom hearts", + "for kids", + "ec r", + "bal e", + "escor ts", + "adidas originals", + "k wa", + "k ts", + "hallo ffame", + "ðŁĺį .", + "wag s", + "pot ted", + "o wing", + "honey comb", + "he fty", + "uro logy", + "mer le", + "b pd", + "stri pping", + "re ich", + "k state", + "gu ay", + "yon ge", + "shak ti", + "g loom", + "bat t", + "son om", + "n ery", + "el ba", + "blan ks", + "hel le", + "triple ts", + "bom bay", + "ak arta", + "ab ia", + "transm itted", + "rol f", + "ja is", + "angular js", + "fi erc", + "m ss", + "trac e", + "ॠĩ", + "tom bs", + "old man", + "kom bucha", + "fo l", + "e health", + "cere als", + "are lli", + "in ari", + "ðŁĴ ©", + "wo l", + "liber ties", + "fa wn", + "af firm", + "nun avut", + "hyster ical", + "k drama", + "art es", + "âĢ¢âĢ¢âĢ¢âĢ¢ âĢ¢âĢ¢âĢ¢âĢ¢", + "valent in", + "man slaughter", + "gal es", + "eo in", + "energi zed", + "del s", + "with draws", + "st les", + "sar castic", + "ram esh", + "incredi bles", + "lock hart", + "ya wn", + "ultimatefan live", + "oooooooo oooooooo", + "mu en", + "guru dev", + "te er", + "pe eling", + "new snow", + "lingui stics", + "direc tv", + "ag end", + "uni lever", + "ru ger", + "han dedly", + "ero se", + "li mel", + "the c", + "royal ties", + "fini shers", + "nr g", + "m gt", + "fid get", + "com ps", + "bac on", + "aggre ssively", + "ab it", + "ch â", + "tar de", + "slu gger", + "q anda", + "gre ening", + "d ats", + "ensla ved", + "spec tor", + "o ye", + "fre ef", + "b hand", + "stop brexit", + "mis conceptions", + "cav a", + "ðŁĺįðŁĺįðŁĺįðŁĺį ðŁĺįðŁĺįðŁĺįðŁĺį", + "multit asking", + "hou sel", + "ferre ira", + "cen time", + "ank les", + "jo dh", + "hel ly", + "fro me", + "out tuesday", + "nar nia", + "bal aji", + "l bloggers", + "jyo ti", + "ðŁį ĩ", + "lan cia", + "cap ri", + "y ap", + "nat ash", + "down fall", + ".\" âĢĶ", + "à ®", + "ligam ent", + "coat ings", + "ai ded", + "hi ko", + "fall ing", + "encryp ted", + "yeg food", + "infringe ment", + "cu di", + "ce p", + "ðŁĺį ðŁĺĤ", + "tra d", + "super rugby", + "ed win", + "wh iche", + "vi meo", + "lay ne", + "in vigor", + "he he", + "dubrov nik", + "bie ber", + "u tr", + "sham an", + "op ers", + "ham ill", + "en ig", + "di f", + "ar um", + "scrap book", + "min h", + "diver gence", + "mckin non", + "life time", + "guter res", + "wil le", + "ple as", + "patt y", + "mic ron", + "k z", + "dom aine", + "ru sher", + "m ds", + "ches ney", + "screw driver", + "âģ© ,", + "sle dge", + "hau er", + "chan a", + "stam ina", + "sprink ler", + "pl n", + "he ff", + "bol ton", + "om on", + "car rington", + "accor dion", + "jor ge", + "inter ception", + "in puts", + "gu ll", + "tran scription", + "vanu atu", + "it ical", + "eth os", + "tic h", + "spac ey", + "pee king", + "u mi", + "ha ger", + "psycho tic", + "illi an", + "illi a", + "bonnar oo", + "an ese", + "pu c", + "laghate parth", + "en hall", + "econom ical", + "dre dge", + "% -", + "u we", + "tu bular", + "scoun cil", + "pe asants", + "fl er", + "tumb ler", + "he p", + "ford ham", + "row ley", + "initi als", + "ev asion", + "er nation", + "plu gins", + "coch ran", + "c attle", + "acid ity", + "ðŁİĬ ðŁİī", + "re grann", + "jump man", + "ef ace", + "x ma", + "patri archy", + "esco bar", + "cristi an", + "tip ton", + "nu eva", + "hack ney", + "back seat", + "kill arney", + "aid an", + "sta dion", + "simul taneous", + "ida ho", + "a je", + "u th", + "figu re", + "clo s", + "bur k", + "volun tar", + "rec ite", + "macfar lane", + "cur few", + "bou do", + "w gn", + "sti x", + "sla p", + "scrat ched", + "philli p", + "jour ne", + "ex pelled", + "wa z", + "u ke", + "tati ana", + "ou e", + "ho pp", + "dimit ri", + "ðŁĵ £", + "mato logist", + "electri fying", + "blu ffs", + "bill smafia", + "az cardinals", + "y aa", + "x mas", + "shar a", + "r ith", + "g ills", + "dre s", + "bar ton", + "authori zation", + "imperi alism", + "home of", + "to do", + "foot path", + "band width", + "visit spain", + "moh sin", + "erup ted", + "mi ki", + "insig nia", + "mike l", + "ss h", + "ger a", + "bank holiday", + "aw an", + "t weak", + "star craft", + "e al", + "construc tion", + "skelet ons", + "le ep", + "ine m", + "bar clay", + "ship wreck", + "monsi eur", + "yo h", + "ron t", + "form ative", + "ser o", + "le p", + "horse man", + "hoo sier", + "haz mat", + "cylin ders", + "cen ti", + "ðŁĴ¥ðŁĴ¥ ðŁĴ¥", + "re em", + "na ire", + "mus ically", + "gras shopper", + "est onian", + "termin ology", + "ro main", + "blogger rt", + "tox in", + "stan ce", + "cultiv ated", + "an ast", + "ðŁIJ į", + "shi mano", + "go pher", + "ene i", + "recycla ble", + "gam ification", + "fight for", + "c q", + "avoc ados", + "ke ys", + "eli ke", + "gly cer", + "shak ur", + "mobili zation", + "gal ley", + "expla in", + "ex changed", + "pe th", + "obe dience", + "illa ge", + "en nis", + "ãĥ ŀ", + "wi v", + "walla bies", + "ma ar", + "ig ers", + "fin tech", + "fin alized", + "wo j", + "meaning less", + "in field", + "onna ise", + "e et", + "bron te", + "pass ages", + "ðŁij §", + "strick land", + "northern lights", + "lom ond", + "h tc", + "wr ay", + "shi fter", + "di alog", + "ðŁį į", + ">> >>>>", + "te atime", + "ste ch", + "sic huan", + "qu ill", + "fran ca", + "comple mentary", + "bar rington", + "marcu s", + "mal am", + "goo oo", + "for sa", + "elec tra", + "af s", + "âĹ Ĩ", + "tri fe", + "sn azzy", + "fo lia", + "and olan", + "after dark", + "wood son", + "stra de", + "litt lest", + "o gun", + "con wy", + "co wards", + "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤ", + "íĬ ¸", + "se ul", + "mur phy", + "dun ks", + "kapil shar", + "jo achim", + "wom ack", + "equal ity", + "aver ages", + "a ine", + "ðŁ¦ Ī", + "tac ular", + "dis ability", + "u ked", + "mid century", + "bar thol", + "teas ers", + "tab ern", + "nj caa", + "sp out", + "op i", + "ku bball", + "bl om", + "so ar", + "popu lism", + "meth yl", + "ðŁijĬ ðŁı¼", + "o spre", + "alo ils", + "ðŁĵ ĸ", + "ðŁĮ ļ", + "x er", + "sp illing", + "publ ica", + "car dam", + "adi sh", + "sa cha", + "p kg", + "bu da", + "lyric ist", + "i bc", + "gru mp", + "ho ver", + "hal ep", + "anti body", + "anem one", + "âĻ¥âĻ¥ âĻ¥âĻ¥", + "m cl", + "litho graph", + "cc u", + "s fest", + "path ic", + "calli ster", + "otta wa", + "gun sn", + "rut ger", + "hali but", + "en vision", + "differenti ate", + "ðŁļĢ ðŁļĢ", + "pir an", + "lat el", + "uc n", + "trou bad", + "ra ine", + "fierc ely", + "learn english", + "lea se", + "wex mondays", + "em it", + "dray ton", + "bur rell", + "scuba diving", + "hol ler", + "dr u", + "clo cked", + "w ral", + "ap ro", + "trans lucent", + "w bo", + "patri arch", + "mo ja", + "lan nister", + "fish ery", + "ne derland", + "mil dly", + "mi rai", + "ma ko", + "ja p", + "ðŁĺ©ðŁĺ© ðŁĺ©", + "pro statec", + "p anna", + "ar ama", + "under taking", + "tomp kins", + "ne op", + "soli ds", + "sav oury", + "e ames", + "cut lery", + "wood bridge", + "steam er", + "ri zzo", + "wild cat", + "rat na", + "lamin ated", + "kin eni", + "jal ap", + "ai des", + "acknowle dges", + "?! ?!?!", + "! ðŁİī", + "w afc", + "mag gio", + "ha ves", + "dar je", + "of i", + "gr il", + "v asi", + "bru x", + "mo hd", + "fake speare", + "arn old", + "r mb", + "for be", + "wal leye", + "ro di", + "therapeu tics", + "strate gi", + "ob ste", + "mu dder", + "download able", + "dd ings", + "d ca", + "asi angames", + "campe on", + "appropri ation", + "th century", + "ram atta", + "dra ped", + "bul lion", + "mu c", + "one x", + "se greg", + "ophel ia", + "bod ily", + "âĿ¤ ðŁĺį", + "wi zar", + "te ased", + "ade my", + "to id", + "sur a", + "lazar us", + "sn ickers", + "ma se", + "lo h", + "bow ed", + "bibli o", + "x change", + "har lan", + "gho shal", + "flavor ful", + "bha gat", + "alle z", + "whiche ver", + "ten stein", + "disc er", + "organ iser", + "mt g", + "dream liner", + "t se", + "hok kaido", + "mo k", + "indulg ent", + "hick man", + "blin ded", + "al yn", + "aaa ah", + "sp ool", + "lough borough", + "inter pret", + "et v", + "aristo tle", + "optimi zing", + "avici i", + "madu rai", + "ju li", + "naw az", + "mat chups", + "ab ide", + "paint ing", + "w elling", + "vel i", + "octag on", + "in scribed", + "po king", + "plac er", + "life cycle", + "kili g", + "g sp", + "eli ves", + "cle ments", + "na sheed", + "me sut", + "incarcer ated", + "dist illed", + "wal ang", + "delic acy", + "del gado", + "che z", + "ch ita", + "ad ero", + "tu x", + "pati l", + "o do", + "abh cosmetics", + "tv c", + "p bc", + "in accurate", + "hardwork paysoff", + "ball er", + "quot ation", + "merchandi sing", + "ga stri", + "defen ses", + "dro gba", + "bex hill", + "ban kno", + "win ona", + "si eg", + "p gs", + "hahah ha", + "agu chi", + "su bram", + "mirac le", + "de sch", + "li bre", + "ba cher", + "ent ine", + "bbcra di", + "lou dest", + "r ps", + "pi erc", + "fr yer", + "storm trooper", + "rafael nadal", + "pas co", + "exhau stion", + "epic onetsy", + "rc tid", + "kel lie", + "ga ines", + "d bz", + "sm riti", + "s bridge", + "lim ited", + "cla w", + "technic al", + "bio graphical", + "ado red", + "ภ°", + "exclu de", + "ac adia", + "key boards", + "fur man", + "so ca", + "sur u", + "ni ps", + "sw aps", + "server less", + "run e", + "pu ffy", + "north ampton", + "nish ings", + "hen der", + "cartri dges", + "gun shot", + "ðŁĵ ¹", + "fil ament", + "respon dents", + "pey ton", + "mountaine er", + "mer ging", + "life span", + "intimid ation", + "p afc", + "nl wx", + "expan sive", + "pur r", + "f ck", + "ca e", + "at ti", + "tele thon", + "so hn", + "mend el", + "lo pes", + "dor i", + "un broken", + "te red", + "tast ings", + "in active", + "disin tegr", + "t assel", + "share the", + "pi ano", + "is lay", + "air space", + "z awa", + "ricci ardo", + "ming ton", + "fresh er", + "cur ry", + "re vs", + "pharo ah", + "h mv", + "exhilar ating", + "wh oo", + "lin kin", + "kri spy", + "competen cy", + "ste wards", + "ne bu", + "kat su", + "ad mins", + "baz ar", + "as ar", + "giving back", + "s summit", + "song z", + "lin us", + "raj kumar", + "farm ington", + "fanta sia", + "ðŁĺ´ ðŁĺ´", + "so bri", + "lis se", + "barry more", + "pri sm", + "blo b", + "sen ew", + "mono xide", + "exp ire", + "eigh teen", + "di pper", + "xi ao", + "kil t", + "hin ch", + "bbc sport", + "bam boo", + "p ter", + "ex al", + "ðŁ¦ ĭ", + "ham lin", + "expe ditions", + "star gazing", + "food security", + "wy lie", + "ul f", + "st ingly", + "on storm", + "lo eb", + "bro ome", + "bn ha", + "pancre atic", + "eli ve", + "!!!!!!!! !!!", + "ther apper", + "ortho pedic", + "avengers endgame", + "antit rust", + "ìļ °", + "go te", + "om d", + "off side", + "gy llen", + "win eries", + "white water", + "ad l", + "lu pita", + "exce eds", + "consi sted", + "chew bacca", + "ash leigh", + "nhl jets", + "is san", + "sh ld", + "hay at", + "cran berries", + "ðŁ¤ĺ ðŁı½", + "rock the", + "spring training", + "fall out", + "dairy free", + "wa j", + "un decided", + "so wn", + "rc n", + "north wales", + "htt r", + "fu mble", + "d its", + "comp elled", + "popu list", + "min ted", + "blan chett", + ". ''", + "pro pulsion", + "m illa", + "au berg", + "her tz", + "h ta", + "u daipur", + "serendip ity", + "azte cs", + "als ace", + "ðŁIJ ij", + "lu n", + "sho es", + "char li", + "gar za", + "ðŁĴ Ł", + "pro biotics", + "fox tv", + "ol is", + "mi ff", + "loc alized", + "diffu ser", + "si gue", + "fun ko", + "rend ous", + "ðŁĴ ij", + "jeky ll" + ] + } +} \ No newline at end of file diff --git a/annotator/clip_vision/tokenizer_config.json b/annotator/clip_vision/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..702bb12920b291cade3706cf215c1604d2255d93 --- /dev/null +++ b/annotator/clip_vision/tokenizer_config.json @@ -0,0 +1,34 @@ +{ + "unk_token": { + "content": "<|endoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": true, + "__type": "AddedToken" + }, + "bos_token": { + "content": "<|startoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": true, + "__type": "AddedToken" + }, + "eos_token": { + "content": "<|endoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": true, + "__type": "AddedToken" + }, + "pad_token": "<|endoftext|>", + "add_prefix_space": false, + "errors": "replace", + "do_lower_case": true, + "name_or_path": "openai/clip-vit-base-patch32", + "model_max_length": 77, + "special_tokens_map_file": "./special_tokens_map.json", + "tokenizer_class": "CLIPTokenizer" +} diff --git a/annotator/clip_vision/vocab.json b/annotator/clip_vision/vocab.json new file mode 100644 index 0000000000000000000000000000000000000000..4297ea6a8d2bae1fea8f48b45e257814dcb11f69 --- /dev/null +++ b/annotator/clip_vision/vocab.json @@ -0,0 +1 @@ +{"!": 0, "\"": 1, "#": 2, "$": 3, "%": 4, "&": 5, "'": 6, "(": 7, ")": 8, "*": 9, "+": 10, ",": 11, "-": 12, ".": 13, "/": 14, "0": 15, "1": 16, "2": 17, "3": 18, "4": 19, "5": 20, "6": 21, "7": 22, "8": 23, "9": 24, ":": 25, ";": 26, "<": 27, "=": 28, ">": 29, "?": 30, "@": 31, "A": 32, "B": 33, "C": 34, "D": 35, "E": 36, "F": 37, "G": 38, "H": 39, "I": 40, "J": 41, "K": 42, "L": 43, "M": 44, "N": 45, "O": 46, "P": 47, "Q": 48, "R": 49, "S": 50, "T": 51, "U": 52, "V": 53, "W": 54, "X": 55, "Y": 56, "Z": 57, "[": 58, "\\": 59, "]": 60, "^": 61, "_": 62, "`": 63, "a": 64, "b": 65, "c": 66, "d": 67, "e": 68, "f": 69, "g": 70, "h": 71, "i": 72, "j": 73, "k": 74, "l": 75, "m": 76, "n": 77, "o": 78, "p": 79, "q": 80, "r": 81, "s": 82, "t": 83, "u": 84, "v": 85, "w": 86, "x": 87, "y": 88, "z": 89, "{": 90, "|": 91, "}": 92, "~": 93, "¡": 94, "¢": 95, "£": 96, "¤": 97, "¥": 98, "¦": 99, "§": 100, "¨": 101, "©": 102, "ª": 103, "«": 104, "¬": 105, "®": 106, "¯": 107, "°": 108, "±": 109, "²": 110, "³": 111, "´": 112, "µ": 113, "¶": 114, "·": 115, "¸": 116, "¹": 117, "º": 118, "»": 119, "¼": 120, "½": 121, "¾": 122, "¿": 123, "À": 124, "Á": 125, "Â": 126, "Ã": 127, "Ä": 128, "Å": 129, "Æ": 130, "Ç": 131, "È": 132, "É": 133, "Ê": 134, "Ë": 135, "Ì": 136, "Í": 137, "Î": 138, "Ï": 139, "Ð": 140, "Ñ": 141, "Ò": 142, "Ó": 143, "Ô": 144, "Õ": 145, "Ö": 146, "×": 147, "Ø": 148, "Ù": 149, "Ú": 150, "Û": 151, "Ü": 152, "Ý": 153, "Þ": 154, "ß": 155, "à": 156, "á": 157, "â": 158, "ã": 159, "ä": 160, "å": 161, "æ": 162, "ç": 163, "è": 164, "é": 165, "ê": 166, "ë": 167, "ì": 168, "í": 169, "î": 170, "ï": 171, "ð": 172, "ñ": 173, "ò": 174, "ó": 175, "ô": 176, "õ": 177, "ö": 178, "÷": 179, "ø": 180, "ù": 181, "ú": 182, "û": 183, "ü": 184, "ý": 185, "þ": 186, "ÿ": 187, "Ā": 188, "ā": 189, "Ă": 190, "ă": 191, "Ą": 192, "ą": 193, "Ć": 194, "ć": 195, "Ĉ": 196, "ĉ": 197, "Ċ": 198, "ċ": 199, "Č": 200, "č": 201, "Ď": 202, "ď": 203, "Đ": 204, "đ": 205, "Ē": 206, "ē": 207, "Ĕ": 208, "ĕ": 209, "Ė": 210, "ė": 211, "Ę": 212, "ę": 213, "Ě": 214, "ě": 215, "Ĝ": 216, "ĝ": 217, "Ğ": 218, "ğ": 219, "Ġ": 220, "ġ": 221, "Ģ": 222, "ģ": 223, "Ĥ": 224, "ĥ": 225, "Ħ": 226, "ħ": 227, "Ĩ": 228, "ĩ": 229, "Ī": 230, "ī": 231, "Ĭ": 232, "ĭ": 233, "Į": 234, "į": 235, "İ": 236, "ı": 237, "IJ": 238, "ij": 239, "Ĵ": 240, "ĵ": 241, "Ķ": 242, "ķ": 243, "ĸ": 244, "Ĺ": 245, "ĺ": 246, "Ļ": 247, "ļ": 248, "Ľ": 249, "ľ": 250, "Ŀ": 251, "ŀ": 252, "Ł": 253, "ł": 254, "Ń": 255, "!": 256, "\"": 257, "#": 258, "$": 259, "%": 260, "&": 261, "'": 262, "(": 263, ")": 264, "*": 265, "+": 266, ",": 267, "-": 268, ".": 269, "/": 270, "0": 271, "1": 272, "2": 273, "3": 274, "4": 275, "5": 276, "6": 277, "7": 278, "8": 279, "9": 280, ":": 281, ";": 282, "<": 283, "=": 284, ">": 285, "?": 286, "@": 287, "A": 288, "B": 289, "C": 290, "D": 291, "E": 292, "F": 293, "G": 294, "H": 295, "I": 296, "J": 297, "K": 298, "L": 299, "M": 300, "N": 301, "O": 302, "P": 303, "Q": 304, "R": 305, "S": 306, "T": 307, "U": 308, "V": 309, "W": 310, "X": 311, "Y": 312, "Z": 313, "[": 314, "\\": 315, "]": 316, "^": 317, "_": 318, "`": 319, "a": 320, "b": 321, "c": 322, "d": 323, "e": 324, "f": 325, "g": 326, "h": 327, "i": 328, "j": 329, "k": 330, "l": 331, "m": 332, "n": 333, "o": 334, "p": 335, "q": 336, "r": 337, "s": 338, "t": 339, "u": 340, "v": 341, "w": 342, "x": 343, "y": 344, "z": 345, "{": 346, "|": 347, "}": 348, "~": 349, "¡": 350, "¢": 351, "£": 352, "¤": 353, "¥": 354, "¦": 355, "§": 356, "¨": 357, "©": 358, "ª": 359, "«": 360, "¬": 361, "®": 362, "¯": 363, "°": 364, "±": 365, "²": 366, "³": 367, "´": 368, "µ": 369, "¶": 370, "·": 371, "¸": 372, "¹": 373, "º": 374, "»": 375, "¼": 376, "½": 377, "¾": 378, "¿": 379, "À": 380, "Á": 381, "Â": 382, "Ã": 383, "Ä": 384, "Å": 385, "Æ": 386, "Ç": 387, "È": 388, "É": 389, "Ê": 390, "Ë": 391, "Ì": 392, "Í": 393, "Î": 394, "Ï": 395, "Ð": 396, "Ñ": 397, "Ò": 398, "Ó": 399, "Ô": 400, "Õ": 401, "Ö": 402, "×": 403, "Ø": 404, "Ù": 405, "Ú": 406, "Û": 407, "Ü": 408, "Ý": 409, "Þ": 410, "ß": 411, "à": 412, "á": 413, "â": 414, "ã": 415, "ä": 416, "å": 417, "æ": 418, "ç": 419, "è": 420, "é": 421, "ê": 422, "ë": 423, "ì": 424, "í": 425, "î": 426, "ï": 427, "ð": 428, "ñ": 429, "ò": 430, "ó": 431, "ô": 432, "õ": 433, "ö": 434, "÷": 435, "ø": 436, "ù": 437, "ú": 438, "û": 439, "ü": 440, "ý": 441, "þ": 442, "ÿ": 443, "Ā": 444, "ā": 445, "Ă": 446, "ă": 447, "Ą": 448, "ą": 449, "Ć": 450, "ć": 451, "Ĉ": 452, "ĉ": 453, "Ċ": 454, "ċ": 455, "Č": 456, "č": 457, "Ď": 458, "ď": 459, "Đ": 460, "đ": 461, "Ē": 462, "ē": 463, "Ĕ": 464, "ĕ": 465, "Ė": 466, "ė": 467, "Ę": 468, "ę": 469, "Ě": 470, "ě": 471, "Ĝ": 472, "ĝ": 473, "Ğ": 474, "ğ": 475, "Ġ": 476, "ġ": 477, "Ģ": 478, "ģ": 479, "Ĥ": 480, "ĥ": 481, "Ħ": 482, "ħ": 483, "Ĩ": 484, "ĩ": 485, "Ī": 486, "ī": 487, "Ĭ": 488, "ĭ": 489, "Į": 490, "į": 491, "İ": 492, "ı": 493, "IJ": 494, "ij": 495, "Ĵ": 496, "ĵ": 497, "Ķ": 498, "ķ": 499, "ĸ": 500, "Ĺ": 501, "ĺ": 502, "Ļ": 503, "ļ": 504, "Ľ": 505, "ľ": 506, "Ŀ": 507, "ŀ": 508, "Ł": 509, "ł": 510, "Ń": 511, "in": 512, "th": 513, "an": 514, "re": 515, "ar": 516, "er": 517, "the": 518, "ing": 519, "ou": 520, "on": 521, "st": 522, "or": 523, "en": 524, "on": 525, "al": 526, "at": 527, "er": 528, "it": 529, "in": 530, "to": 531, "ro": 532, "is": 533, "le": 534, "ic": 535, "at": 536, "and": 537, "ed": 538, "of": 539, "ch": 540, "or": 541, "es": 542, "il": 543, "el": 544, "st": 545, "ac": 546, "om": 547, "am": 548, "lo": 549, "an": 550, "ay": 551, "sh": 552, "ri": 553, "li": 554, "ti": 555, "for": 556, "ne": 557, "ðŁ": 558, "ra": 559, "ha": 560, "de": 561, "ol": 562, "ve": 563, "si": 564, "ur": 565, "al": 566, "se": 567, "'s": 568, "un": 569, "di": 570, "be": 571, "la": 572, "wh": 573, "oo": 574, "day": 575, "en": 576, "ma": 577, "no": 578, "le": 579, "to": 580, "our": 581, "ir": 582, "gh": 583, "wit": 584, "it": 585, "yo": 586, "as": 587, "sp": 588, "this": 589, "ts": 590, "ati": 591, "you": 592, "with": 593, "ad": 594, "is": 595, "ab": 596, "ly": 597, "we": 598, "the": 599, "te": 600, "as": 601, "ag": 602, "vi": 603, "pp": 604, "su": 605, "ho": 606, "my": 607, "..": 608, "bu": 609, "com": 610, "se": 611, "ers": 612, "me": 613, "me": 614, "all": 615, "con": 616, "mo": 617, "ke": 618, "ge": 619, "out": 620, "ent": 621, "co": 622, "fe": 623, "ver": 624, "ar": 625, "fro": 626, "au": 627, "po": 628, "ce": 629, "ght": 630, "are": 631, "ss": 632, "from": 633, "ch": 634, "tr": 635, "oun": 636, "one": 637, "by": 638, "do": 639, "th": 640, "wor": 641, "ere": 642, "ke": 643, "pro": 644, "for": 645, "ds": 646, "bo": 647, "ta": 648, "we": 649, "go": 650, "he": 651, "ter": 652, "ing": 653, "de": 654, "be": 655, "ation": 656, "mor": 657, "ay": 658, "ex": 659, "ill": 660, "pe": 661, "ks": 662, "sc": 663, "lu": 664, "fu": 665, "qu": 666, "ver": 667, "ðŁĺ": 668, "ju": 669, "mu": 670, "ate": 671, "and": 672, "ve": 673, "king": 674, "mar": 675, "op": 676, "hi": 677, "...": 678, "pre": 679, "ad": 680, "ru": 681, "that": 682, "jo": 683, "of": 684, "ce": 685, "new": 686, "am": 687, "ap": 688, "gre": 689, "ss": 690, "du": 691, "now": 692, "ye": 693, "ting": 694, "your": 695, "ity": 696, "ni": 697, "ci": 698, "par": 699, "gu": 700, "fi": 701, "af": 702, "per": 703, "ter": 704, "up": 705, "so": 706, "gi": 707, "ons": 708, "gr": 709, "ge": 710, "br": 711, "pl": 712, "'t": 713, "mi": 714, "ine": 715, "wee": 716, "bi": 717, "us": 718, "sho": 719, "have": 720, "today": 721, "av": 722, "man": 723, "ent": 724, "ack": 725, "ure": 726, "our": 727, "âĢ": 728, "cu": 729, "ld": 730, "loo": 731, "im": 732, "ice": 733, "som": 734, "fin": 735, "red": 736, "ren": 737, "ood": 738, "was": 739, "tion": 740, "pi": 741, "ir": 742, "ther": 743, "ty": 744, "ph": 745, "ard": 746, "ec": 747, "!!": 748, "mon": 749, "more": 750, "will": 751, "tra": 752, "can": 753, "col": 754, "pu": 755, "te": 756, "wn": 757, "mb": 758, "so": 759, "iti": 760, "just": 761, "ning": 762, "here": 763, "tu": 764, "pa": 765, "pr": 766, "but": 767, "what": 768, "ally": 769, "fir": 770, "min": 771, "ca": 772, "ant": 773, "sa": 774, "ted": 775, "ev": 776, "ment": 777, "fa": 778, "get": 779, "ame": 780, "about": 781, "gra": 782, "not": 783, "happ": 784, "ays": 785, "man": 786, "his": 787, "time": 788, "like": 789, "gh": 790, "has": 791, "than": 792, "love": 793, "art": 794, "ste": 795, "ding": 796, "he": 797, "cre": 798, "ws": 799, "wat": 800, "der": 801, "ite": 802, "ser": 803, "ace": 804, "age": 805, "end": 806, "str": 807, "aw": 808, "stor": 809, "re": 810, "car": 811, "ell": 812, "all": 813, "ps": 814, "fri": 815, "pho": 816, "por": 817, "do": 818, "ak": 819, "wi": 820, "fre": 821, "who": 822, "shi": 823, "boo": 824, "son": 825, "ell": 826, "when": 827, "ill": 828, "how": 829, "great": 830, "win": 831, "el": 832, "bl": 833, "ssi": 834, "ali": 835, "some": 836, "ðŁĴ": 837, "ton": 838, "der": 839, "les": 840, "pla": 841, "ï¸": 842, "ed": 843, "sch": 844, "hu": 845, "ong": 846, "don": 847, "ki": 848, "sh": 849, "ann": 850, "cor": 851, "..": 852, "ound": 853, "az": 854, "ine": 855, "ary": 856, "ful": 857, "stu": 858, "ould": 859, "sti": 860, "go": 861, "see": 862, "able": 863, "ars": 864, "ll": 865, "mis": 866, "ber": 867, "ck": 868, "wa": 869, "ents": 870, "no": 871, "sig": 872, "fe": 873, "first": 874, "et": 875, "spe": 876, "ack": 877, "if": 878, "ous": 879, "'m": 880, "ster": 881, "app": 882, "ang": 883, "ance": 884, "ans": 885, "good": 886, "bre": 887, "ever": 888, "they": 889, "tic": 890, "come": 891, "off": 892, "back": 893, "ase": 894, "ings": 895, "old": 896, "ight": 897, "fo": 898, "her": 899, "happy": 900, "pic": 901, "its": 902, "ving": 903, "us": 904, "mat": 905, "hom": 906, "dy": 907, "em": 908, "sk": 909, "ying": 910, "their": 911, "led": 912, "ry": 913, "ul": 914, "har": 915, "ck": 916, "ton": 917, "onal": 918, "hel": 919, "ric": 920, "bir": 921, "vie": 922, "way": 923, "tri": 924, "da": 925, "ple": 926, "bro": 927, "sto": 928, "ool": 929, "night": 930, "tru": 931, "ba": 932, "read": 933, "res": 934, "year": 935, "fr": 936, "tor": 937, "als": 938, "coun": 939, "cla": 940, "ture": 941, "vel": 942, "ated": 943, "lec": 944, "end": 945, "thing": 946, "vo": 947, "ici": 948, "best": 949, "can": 950, "work": 951, "last": 952, "after": 953, "ence": 954, "pri": 955, "pe": 956, "es": 957, "il": 958, "âĢ¦": 959, "dre": 960, "ys": 961, "over": 962, "ies": 963, "ðŁij": 964, "comm": 965, "tw": 966, "ink": 967, "sun": 968, "cl": 969, "life": 970, "tt": 971, "ach": 972, "land": 973, "sy": 974, "tre": 975, "tal": 976, "pol": 977, "sm": 978, "duc": 979, "sal": 980, "ft": 981, "'re": 982, "che": 983, "war": 984, "tur": 985, "ations": 986, "ach": 987, "ms": 988, "ile": 989, "pm": 990, "ough": 991, "ate": 992, "star": 993, "week": 994, "!!!": 995, "clu": 996, "there": 997, "ner": 998, "tom": 999, "sel": 1000, "ï¸ı": 1001, "world": 1002, "ves": 1003, "cam": 1004, "got": 1005, "inter": 1006, "off": 1007, "um": 1008, "tonight": 1009, "other": 1010, "hou": 1011, "look": 1012, "je": 1013, "id": 1014, "sion": 1015, "beau": 1016, "att": 1017, "eli": 1018, "ort": 1019, "rec": 1020, "ff": 1021, "ster": 1022, "supp": 1023, "gen": 1024, "been": 1025, "ily": 1026, "team": 1027, "mm": 1028, "ic": 1029, "peop": 1030, "itt": 1031, "ats": 1032, "only": 1033, "mber": 1034, "eng": 1035, "bri": 1036, "mp": 1037, "know": 1038, "bur": 1039, "bar": 1040, "ins": 1041, "low": 1042, "she": 1043, "row": 1044, "âĿ": 1045, "tro": 1046, "people": 1047, "via": 1048, "low": 1049, "aga": 1050, "bet": 1051, "xt": 1052, "fac": 1053, "char": 1054, "ear": 1055, "wal": 1056, "sen": 1057, "fam": 1058, "ble": 1059, "nati": 1060, "ish": 1061, "nor": 1062, "game": 1063, "live": 1064, "sco": 1065, "ley": 1066, "don": 1067, "ick": 1068, "ball": 1069, "very": 1070, "these": 1071, "pan": 1072, "ia": 1073, "ating": 1074, "cr": 1075, "are": 1076, "gir": 1077, "make": 1078, "stre": 1079, "show": 1080, ".\"": 1081, "fl": 1082, "up": 1083, "dr": 1084, "thanks": 1085, "illi": 1086, "wom": 1087, "sts": 1088, "ig": 1089, "sur": 1090, "every": 1091, "cur": 1092, "view": 1093, "let": 1094, "into": 1095, "most": 1096, "na": 1097, "indi": 1098, "gar": 1099, "had": 1100, "sou": 1101, "ved": 1102, "ant": 1103, "ition": 1104, "made": 1105, "fol": 1106, "uni": 1107, "ited": 1108, "ðŁı": 1109, "ical": 1110, "thr": 1111, "ready": 1112, "chec": 1113, "dra": 1114, "kes": 1115, "book": 1116, "ep": 1117, "sic": 1118, "morning": 1119, "news": 1120, "cau": 1121, "ct": 1122, "well": 1123, "anc": 1124, "photo": 1125, "than": 1126, "ors": 1127, "birth": 1128, "gg": 1129, "out": 1130, "next": 1131, "some": 1132, "ening": 1133, "story": 1134, "chri": 1135, "down": 1136, "home": 1137, "ffe": 1138, "free": 1139, "da": 1140, "bor": 1141, "fil": 1142, "cial": 1143, "thank": 1144, "side": 1145, "lear": 1146, "que": 1147, "line": 1148, "ten": 1149, "ates": 1150, "years": 1151, "my": 1152, "photo": 1153, "beauti": 1154, "right": 1155, "nu": 1156, "form": 1157, "ship": 1158, "ban": 1159, "ther": 1160, "days": 1161, "gam": 1162, "ason": 1163, "gy": 1164, "ðŁİ": 1165, "birthday": 1166, "set": 1167, "ick": 1168, "et": 1169, "still": 1170, "coming": 1171, "take": 1172, "ðŁĩ": 1173, "bb": 1174, "sol": 1175, "son": 1176, "den": 1177, "ep": 1178, "music": 1179, "them": 1180, "den": 1181, "why": 1182, "foo": 1183, "cra": 1184, "amaz": 1185, "wn": 1186, "hol": 1187, "tting": 1188, "wr": 1189, "ue": 1190, "mag": 1191, "cro": 1192, "lan": 1193, "clo": 1194, "bra": 1195, "ak": 1196, "sing": 1197, "cal": 1198, "read": 1199, "'ve": 1200, "joh": 1201, "bab": 1202, "dri": 1203, "blo": 1204, "big": 1205, "eric": 1206, "int": 1207, "tor": 1208, "try": 1209, "la": 1210, "leg": 1211, "house": 1212, "mic": 1213, "val": 1214, "beautiful": 1215, "litt": 1216, "check": 1217, "new": 1218, "vers": 1219, "sw": 1220, "ari": 1221, "play": 1222, "her": 1223, "âĢĵ": 1224, "win": 1225, "ma": 1226, "congr": 1227, "school": 1228, "fun": 1229, ".@": 1230, "heal": 1231, "ich": 1232, "del": 1233, "where": 1234, "lon": 1235, "ket": 1236, "two": 1237, "much": 1238, "watch": 1239, "ven": 1240, "ded": 1241, "ast": 1242, "ked": 1243, "bas": 1244, "going": 1245, "mp": 1246, "ever": 1247, "ways": 1248, "roo": 1249, "desig": 1250, "ly": 1251, "sed": 1252, "top": 1253, "lin": 1254, "chan": 1255, "too": 1256, "iting": 1257, "dent": 1258, "ghts": 1259, "ty": 1260, "spo": 1261, "need": 1262, "blu": 1263, "inst": 1264, "being": 1265, "âĿ¤": 1266, "wel": 1267, "ls": 1268, "him": 1269, "may": 1270, "sting": 1271, "na": 1272, "ely": 1273, "little": 1274, "ga": 1275, "nat": 1276, "tomor": 1277, "mc": 1278, "hon": 1279, "want": 1280, "air": 1281, "pic": 1282, "americ": 1283, "per": 1284, "less": 1285, "week": 1286, "vel": 1287, "ah": 1288, "cap": 1289, "cham": 1290, "ger": 1291, "tim": 1292, "tomorrow": 1293, "ness": 1294, "state": 1295, "hal": 1296, "serv": 1297, "ze": 1298, "os": 1299, "pat": 1300, "vis": 1301, "exc": 1302, "sin": 1303, "ff": 1304, "city": 1305, "cen": 1306, "any": 1307, "bel": 1308, "summ": 1309, "tin": 1310, "would": 1311, "looking": 1312, "ko": 1313, "cele": 1314, "family": 1315, "mer": 1316, "pow": 1317, "help": 1318, "bus": 1319, "co": 1320, "cle": 1321, "self": 1322, "ens": 1323, "ics": 1324, "tho": 1325, "ani": 1326, "cho": 1327, "lead": 1328, "bs": 1329, "twee": 1330, "think": 1331, "fore": 1332, "chil": 1333, "vide": 1334, "did": 1335, "ale": 1336, "chi": 1337, "vil": 1338, "ends": 1339, "wing": 1340, "pas": 1341, "'ll": 1342, "vol": 1343, "sa": 1344, "gs": 1345, "many": 1346, "jec": 1347, "before": 1348, "graph": 1349, "ny": 1350, "uring": 1351, "wil": 1352, "dd": 1353, "buil": 1354, "fav": 1355, "sted": 1356, "tran": 1357, "ling": 1358, "oud": 1359, "dge": 1360, "fiel": 1361, "national": 1362, "sta": 1363, "cer": 1364, "were": 1365, "ina": 1366, "season": 1367, "cou": 1368, "ned": 1369, "amazing": 1370, "tions": 1371, "celebr": 1372, "ns": 1373, "ath": 1374, "head": 1375, "sday": 1376, "dar": 1377, "loc": 1378, "vin": 1379, "another": 1380, "goo": 1381, "sat": 1382, "ny": 1383, "join": 1384, "pres": 1385, "ses": 1386, "sing": 1387, "ana": 1388, "ining": 1389, "....": 1390, "cour": 1391, "ï¸ı": 1392, "act": 1393, "cause": 1394, "light": 1395, "ams": 1396, "ta": 1397, "bal": 1398, "fc": 1399, "high": 1400, "offici": 1401, "tt": 1402, "christ": 1403, "dic": 1404, "day": 1405, "ral": 1406, "hor": 1407, ":)": 1408, "visi": 1409, "nam": 1410, "ob": 1411, "mas": 1412, "ght": 1413, "really": 1414, "tun": 1415, "find": 1416, "through": 1417, "port": 1418, "ut": 1419, "tive": 1420, "sty": 1421, "ne": 1422, "ore": 1423, "ðŁĺĤ": 1424, "support": 1425, "never": 1426, "even": 1427, "ðŁĶ": 1428, "ha": 1429, "ya": 1430, "ld": 1431, "uk": 1432, "ran": 1433, "jam": 1434, "with": 1435, "medi": 1436, "des": 1437, "ney": 1438, "ching": 1439, "ale": 1440, "hy": 1441, "kin": 1442, "!!": 1443, "dy": 1444, "place": 1445, "also": 1446, "ble": 1447, "which": 1448, "black": 1449, "bli": 1450, "say": 1451, "park": 1452, "play": 1453, "ire": 1454, "video": 1455, "weekend": 1456, "ail": 1457, "key": 1458, "pt": 1459, "ward": 1460, "friday": 1461, "din": 1462, "iness": 1463, "gro": 1464, "ben": 1465, "always": 1466, "tball": 1467, "ago": 1468, "mil": 1469, "cy": 1470, "produc": 1471, "disc": 1472, "under": 1473, "please": 1474, "spor": 1475, "full": 1476, "ey": 1477, "ðŁĻ": 1478, "ise": 1479, "ities": 1480, "cat": 1481, "kno": 1482, "use": 1483, "fore": 1484, "ker": 1485, "art": 1486, "high": 1487, "open": 1488, "san": 1489, "ef": 1490, "ours": 1491, "shed": 1492, "stri": 1493, "dro": 1494, "again": 1495, "im": 1496, "ðŁĵ": 1497, "enjo": 1498, "fun": 1499, "getting": 1500, "pen": 1501, "ger": 1502, "cli": 1503, "any": 1504, "every": 1505, "eu": 1506, "women": 1507, "âľ": 1508, "est": 1509, "could": 1510, "ry": 1511, "\"@": 1512, "thou": 1513, "sha": 1514, "commun": 1515, "ber": 1516, "dents": 1517, "dis": 1518, "while": 1519, "away": 1520, "dio": 1521, "ham": 1522, "gla": 1523, "date": 1524, "ka": 1525, "miss": 1526, "unch": 1527, "won": 1528, "inf": 1529, "room": 1530, "ga": 1531, "real": 1532, "exper": 1533, "direc": 1534, "should": 1535, "spr": 1536, "gol": 1537, "long": 1538, "better": 1539, "ori": 1540, "ey": 1541, "ience": 1542, "ils": 1543, "zz": 1544, "han": 1545, "found": 1546, "vs": 1547, "âĻ": 1548, "post": 1549, "tic": 1550, "part": 1551, "men": 1552, "rence": 1553, "cess": 1554, "vic": 1555, "sil": 1556, "shop": 1557, "ðŁĺĤ": 1558, "food": 1559, "val": 1560, "stic": 1561, "you": 1562, "says": 1563, "elec": 1564, "star": 1565, "oc": 1566, "land": 1567, "id": 1568, "ction": 1569, "field": 1570, "sof": 1571, "start": 1572, "water": 1573, "friends": 1574, "ones": 1575, "ðŁĮ": 1576, "fla": 1577, "far": 1578, "white": 1579, "party": 1580, "inst": 1581, "grou": 1582, "tv": 1583, "everyone": 1584, "ment": 1585, "ja": 1586, "cha": 1587, "prin": 1588, "ants": 1589, "during": 1590, "lat": 1591, "lar": 1592, "west": 1593, "then": 1594, "ka": 1595, "youn": 1596, "insp": 1597, "inte": 1598, "ween": 1599, "visit": 1600, "against": 1601, "rele": 1602, "head": 1603, "ces": 1604, "town": 1605, "looks": 1606, "thre": 1607, "regi": 1608, "rent": 1609, "projec": 1610, "girl": 1611, "sear": 1612, "wo": 1613, "mom": 1614, "car": 1615, "hun": 1616, "publi": 1617, "di": 1618, "ple": 1619, "call": 1620, "cri": 1621, "um": 1622, "ford": 1623, "perfe": 1624, "friend": 1625, "hard": 1626, "ssion": 1627, "test": 1628, "playing": 1629, "around": 1630, "because": 1631, "kets": 1632, "meet": 1633, "satur": 1634, "arti": 1635, "work": 1636, "jun": 1637, "ven": 1638, "run": 1639, "member": 1640, "port": 1641, "super": 1642, "twit": 1643, "sam": 1644, "els": 1645, "tly": 1646, "adv": 1647, "ative": 1648, "ath": 1649, "sure": 1650, "avail": 1651, "lar": 1652, "squ": 1653, "ards": 1654, "event": 1655, "men": 1656, "ll": 1657, "over": 1658, "logy": 1659, "ital": 1660, "times": 1661, "mal": 1662, "back": 1663, "coo": 1664, "making": 1665, "stru": 1666, "âģ": 1667, "itu": 1668, "shar": 1669, "gan": 1670, "cas": 1671, "sn": 1672, "summer": 1673, "picture": 1674, "fan": 1675, "hin": 1676, "christmas": 1677, "cy": 1678, "proud": 1679, "champi": 1680, "design": 1681, "pping": 1682, "hope": 1683, "ca": 1684, "available": 1685, "may": 1686, "wed": 1687, "photograph": 1688, "special": 1689, "sale": 1690, "stop": 1691, "ery": 1692, "awe": 1693, "ality": 1694, "history": 1695, "ama": 1696, "presi": 1697, "bru": 1698, "working": 1699, "done": 1700, "dr": 1701, "ken": 1702, "feat": 1703, "wood": 1704, "atest": 1705, "sunday": 1706, "movi": 1707, "vely": 1708, "sle": 1709, "face": 1710, "spec": 1711, "students": 1712, "by": 1713, "ham": 1714, "spon": 1715, "business": 1716, "dat": 1717, "ie": 1718, "ip": 1719, "soci": 1720, "glo": 1721, "hand": 1722, "recor": 1723, "rs": 1724, "mee": 1725, "keep": 1726, "pur": 1727, "health": 1728, "she": 1729, "comple": 1730, "god": 1731, "davi": 1732, "collec": 1733, "list": 1734, "ra": 1735, "club": 1736, "ters": 1737, "inclu": 1738, "things": 1739, "plan": 1740, "âĺ": 1741, "john": 1742, "shing": 1743, "atul": 1744, "soon": 1745, "blue": 1746, "gor": 1747, "saturday": 1748, "won": 1749, "congratul": 1750, "see": 1751, "âĿ¤ï¸ı": 1752, "those": 1753, "ðŁĺį": 1754, "final": 1755, "dou": 1756, "ith": 1757, "own": 1758, "road": 1759, "tour": 1760, "ast": 1761, "india": 1762, "til": 1763, "nd": 1764, "fer": 1765, "favor": 1766, "sul": 1767, "learn": 1768, "fire": 1769, "just": 1770, "group": 1771, "ah": 1772, "rac": 1773, "body": 1774, "ur": 1775, "care": 1776, "à¸": 1777, "plo": 1778, "oh": 1779, "pos": 1780, "give": 1781, "tech": 1782, "sub": 1783, "cent": 1784, "ering": 1785, "ym": 1786, "ility": 1787, "fic": 1788, "london": 1789, "vir": 1790, "guys": 1791, "ba": 1792, "ðŁ¤": 1793, "baby": 1794, "scre": 1795, "ðŁĺį": 1796, "trump": 1797, "under": 1798, "change": 1799, "ian": 1800, "colle": 1801, "sses": 1802, "ler": 1803, "ssed": 1804, "nice": 1805, "announ": 1806, "power": 1807, "sar": 1808, "aking": 1809, "mini": 1810, "sli": 1811, "swee": 1812, "kar": 1813, "ful": 1814, "cru": 1815, "action": 1816, "ather": 1817, ").": 1818, "stand": 1819, "devel": 1820, "aa": 1821, "gan": 1822, "left": 1823, "lol": 1824, "rel": 1825, "trans": 1826, "ments": 1827, "int": 1828, "ef": 1829, "manag": 1830, "dig": 1831, "gener": 1832, "down": 1833, "pau": 1834, "tiv": 1835, "ku": 1836, "thur": 1837, "ken": 1838, "ston": 1839, "fans": 1840, "talk": 1841, "tweet": 1842, "too": 1843, "style": 1844, "prote": 1845, "secon": 1846, "fron": 1847, "awesome": 1848, "gl": 1849, "pal": 1850, "net": 1851, "sor": 1852, "lau": 1853, "gon": 1854, "since": 1855, "tty": 1856, "series": 1857, "memor": 1858, "beli": 1859, "film": 1860, "did": 1861, "dies": 1862, "ot": 1863, "congratulations": 1864, "pra": 1865, "eve": 1866, "woo": 1867, "official": 1868, "suc": 1869, "incre": 1870, "bon": 1871, "part": 1872, "pped": 1873, "class": 1874, "sive": 1875, "boy": 1876, "cul": 1877, "perfect": 1878, "tou": 1879, "dam": 1880, "welcome": 1881, "football": 1882, "hi": 1883, "pap": 1884, "wait": 1885, "ada": 1886, "congrats": 1887, "young": 1888, "excited": 1889, "rece": 1890, "jan": 1891, "va": 1892, "red": 1893, "stra": 1894, "media": 1895, "'d": 1896, "does": 1897, "let": 1898, "mul": 1899, "ills": 1900, "green": 1901, "mel": 1902, "toge": 1903, "future": 1904, "yester": 1905, "versity": 1906, "form": 1907, "tain": 1908, "ide": 1909, "ches": 1910, "kids": 1911, "qui": 1912, "haha": 1913, "deta": 1914, "big": 1915, "favorite": 1916, "girls": 1917, "contin": 1918, "dom": 1919, "search": 1920, "ual": 1921, "air": 1922, "ders": 1923, "month": 1924, "cer": 1925, "yesterday": 1926, "community": 1927, "ade": 1928, "dog": 1929, "ville": 1930, "ices": 1931, "deli": 1932, "syste": 1933, "run": 1934, "ism": 1935, "heart": 1936, "cup": 1937, "enti": 1938, "few": 1939, "president": 1940, "eds": 1941, "until": 1942, "festi": 1943, "ok": 1944, "flo": 1945, "said": 1946, "ole": 1947, "med": 1948, "travel": 1949, "£": 1950, "phone": 1951, "together": 1952, "fast": 1953, "lot": 1954, "games": 1955, "shir": 1956, "between": 1957, "yes": 1958, "thers": 1959, "doing": 1960, "mac": 1961, "ator": 1962, "band": 1963, "follow": 1964, "project": 1965, "develop": 1966, "diffe": 1967, "confe": 1968, "speci": 1969, "cast": 1970, "ys": 1971, "board": 1972, "rd": 1973, "ial": 1974, "shoo": 1975, "ram": 1976, "having": 1977, "share": 1978, "follow": 1979, "one": 1980, "name": 1981, "mr": 1982, "put": 1983, "discu": 1984, "ory": 1985, "came": 1986, "ous": 1987, "site": 1988, "twitter": 1989, "tb": 1990, "tit": 1991, "finally": 1992, "zed": 1993, "super": 1994, "compan": 1995, "using": 1996, "alls": 1997, "list": 1998, "ris": 1999, "shot": 2000, "gal": 2001, "tar": 2002, "del": 2003, "john": 2004, "âĢĶ": 2005, "something": 2006, "ram": 2007, "intere": 2008, "whe": 2009, "bit": 2010, "ðŁį": 2011, "street": 2012, "ound": 2013, "ai": 2014, "tickets": 2015, "movie": 2016, "real": 2017, "ky": 2018, "taking": 2019, "opp": 2020, "cc": 2021, "lam": 2022, "moun": 2023, "inve": 2024, "black": 2025, "used": 2026, "online": 2027, "yor": 2028, "local": 2029, "gue": 2030, "cks": 2031, "ow": 2032, "gest": 2033, "boys": 2034, "illion": 2035, "cont": 2036, "reci": 2037, "ined": 2038, "euro": 2039, "now": 2040, "seen": 2041, "ph": 2042, "teach": 2043, "def": 2044, "south": 2045, "such": 2046, "award": 2047, "must": 2048, "issu": 2049, "care": 2050, "feel": 2051, "plu": 2052, "latest": 2053, "sports": 2054, "web": 2055, "tex": 2056, "ement": 2057, "sk": 2058, "fic": 2059, "wan": 2060, "tech": 2061, "ot": 2062, "box": 2063, "ner": 2064, "free": 2065, "tal": 2066, "ash": 2067, "case": 2068, "hot": 2069, "wonder": 2070, "meeting": 2071, "era": 2072, "chall": 2073, "ðŁIJ": 2074, "job": 2075, "ili": 2076, "cool": 2077, "jour": 2078, "ths": 2079, "mo": 2080, "fel": 2081, "die": 2082, "micha": 2083, "ele": 2084, "team": 2085, "service": 2086, "stand": 2087, "makes": 2088, "ping": 2089, "early": 2090, "comes": 2091, "ek": 2092, "holi": 2093, "vers": 2094, "ague": 2095, "sau": 2096, "three": 2097, "monday": 2098, "fashi": 2099, "someone": 2100, "thro": 2101, "sea": 2102, "bad": 2103, "suppor": 2104, "turn": 2105, "ury": 2106, "ming": 2107, "photography": 2108, "nic": 2109, "mark": 2110, "pretty": 2111, "ssing": 2112, "watching": 2113, "memb": 2114, "arri": 2115, "county": 2116, "beach": 2117, "fran": 2118, "center": 2119, "police": 2120, "bat": 2121, "public": 2122, "tan": 2123, "press": 2124, "saf": 2125, "sy": 2126, "gets": 2127, "roy": 2128, "ners": 2129, "your": 2130, "buy": 2131, "sters": 2132, "show": 2133, "ased": 2134, "childre": 2135, "afric": 2136, "ines": 2137, "space": 2138, "scri": 2139, "hall": 2140, "pain": 2141, "aring": 2142, "home": 2143, "mur": 2144, "health": 2145, "ched": 2146, "sand": 2147, "recei": 2148, "guy": 2149, "ea": 2150, "american": 2151, "resi": 2152, "children": 2153, "--": 2154, "iri": 2155, "ington": 2156, "country": 2157, "ross": 2158, "len": 2159, "anna": 2160, "books": 2161, "bc": 2162, "ece": 2163, "dom": 2164, "lovely": 2165, "kh": 2166, "pet": 2167, "gy": 2168, "gri": 2169, "stage": 2170, "office": 2171, "rock": 2172, "mon": 2173, "bay": 2174, "table": 2175, "sun": 2176, "med": 2177, "thin": 2178, "lor": 2179, "flow": 2180, "(@": 2181, "university": 2182, "store": 2183, "front": 2184, "good": 2185, "za": 2186, "vote": 2187, "north": 2188, "hey": 2189, "anim": 2190, "order": 2191, "mid": 2192, "without": 2193, "ade": 2194, "remember": 2195, "market": 2196, "??": 2197, "mus": 2198, "training": 2199, "educ": 2200, "but": 2201, "cover": 2202, "stan": 2203, "scen": 2204, "bla": 2205, "break": 2206, "lou": 2207, "same": 2208, "gold": 2209, "ain": 2210, "os": 2211, "both": 2212, "lit": 2213, "vern": 2214, "ai": 2215, "albu": 2216, "pa": 2217, "enjoy": 2218, "beg": 2219, "elling": 2220, "thursday": 2221, "info": 2222, "san": 2223, "america": 2224, "hair": 2225, "tel": 2226, "march": 2227, "concer": 2228, "college": 2229, "conference": 2230, "app": 2231, "hour": 2232, "chang": 2233, "âļ": 2234, "sour": 2235, "ols": 2236, "weather": 2237, "war": 2238, "phi": 2239, "festival": 2240, "second": 2241, "cute": 2242, "prac": 2243, "ener": 2244, "stry": 2245, "lea": 2246, "polit": 2247, "sav": 2248, "sen": 2249, "ow": 2250, "mi": 2251, "near": 2252, "ought": 2253, "ze": 2254, "coffe": 2255, "willi": 2256, "dan": 2257, "sey": 2258, "david": 2259, "ese": 2260, "fan": 2261, "deci": 2262, "theat": 2263, "nov": 2264, "ation": 2265, "trac": 2266, "sci": 2267, "review": 2268, "cel": 2269, "em": 2270, "un": 2271, "july": 2272, "orig": 2273, "tion": 2274, "dru": 2275, "former": 2276, "stay": 2277, "after": 2278, "inv": 2279, "took": 2280, "data": 2281, "bal": 2282, "tues": 2283, "dan": 2284, "evening": 2285, "ðŁĺĤðŁĺĤ": 2286, "dol": 2287, "ures": 2288, "provi": 2289, "ts": 2290, "est": 2291, "sign": 2292, "jac": 2293, "uk": 2294, "song": 2295, "yet": 2296, "bow": 2297, "indu": 2298, "jap": 2299, "hoo": 2300, "point": 2301, "anyone": 2302, "zy": 2303, "ist": 2304, "hur": 2305, "ital": 2306, "building": 2307, "woman": 2308, "chur": 2309, "jer": 2310, "perfor": 2311, "coach": 2312, "league": 2313, "cess": 2314, "net": 2315, "imag": 2316, "nation": 2317, "brit": 2318, "que": 2319, "awards": 2320, "ages": 2321, "works": 2322, "ced": 2323, "mance": 2324, "late": 2325, "ign": 2326, "money": 2327, "true": 2328, "ii": 2329, "tell": 2330, "plac": 2331, "pac": 2332, "asy": 2333, "world": 2334, "behin": 2335, "import": 2336, "reading": 2337, "gram": 2338, "giving": 2339, "met": 2340, "hit": 2341, "forward": 2342, "stom": 2343, "present": 2344, "june": 2345, "social": 2346, "noon": 2347, "mart": 2348, "half": 2349, "swe": 2350, "govern": 2351, "ker": 2352, "details": 2353, "lish": 2354, "__": 2355, "acy": 2356, "sia": 2357, "bert": 2358, "fall": 2359, "!!!!": 2360, "),": 2361, "thi": 2362, "diti": 2363, "sport": 2364, "king": 2365, "fit": 2366, "staf": 2367, "cat": 2368, "muse": 2369, "centr": 2370, "yer": 2371, "contro": 2372, "bloo": 2373, "walk": 2374, "actu": 2375, "didn": 2376, "lim": 2377, "learning": 2378, "research": 2379, "wedne": 2380, "auth": 2381, "hours": 2382, "ky": 2383, "far": 2384, "hen": 2385, "....": 2386, "itch": 2387, "ril": 2388, "strong": 2389, "sky": 2390, "questi": 2391, "james": 2392, "ron": 2393, "dg": 2394, "fur": 2395, "cin": 2396, "does": 2397, "appro": 2398, "marke": 2399, "tures": 2400, "fully": 2401, "chat": 2402, "behind": 2403, "tem": 2404, "fini": 2405, "mission": 2406, "batt": 2407, "feel": 2408, "heav": 2409, "everything": 2410, "bar": 2411, "wish": 2412, "premi": 2413, "ima": 2414, "experience": 2415, "each": 2416, "report": 2417, "sweet": 2418, "tics": 2419, "spring": 2420, "respon": 2421, "system": 2422, "victor": 2423, "lin": 2424, "saw": 2425, "already": 2426, "ghter": 2427, "fle": 2428, "ãĥ": 2429, "bring": 2430, "album": 2431, "--": 2432, "ells": 2433, "stan": 2434, "tom": 2435, "international": 2436, "went": 2437, "anni": 2438, "match": 2439, "pper": 2440, "stone": 2441, "small": 2442, "rain": 2443, "fashion": 2444, "area": 2445, "van": 2446, "agram": 2447, "ko": 2448, "thought": 2449, "worth": 2450, "van": 2451, "mer": 2452, "coffee": 2453, "ites": 2454, "gn": 2455, "artist": 2456, "con": 2457, "arch": 2458, "cir": 2459, "secre": 2460, "ground": 2461, "iso": 2462, "hand": 2463, "com": 2464, "bridge": 2465, "hs": 2466, "xi": 2467, "link": 2468, "pul": 2469, "spl": 2470, "race": 2471, "fli": 2472, "river": 2473, "gas": 2474, "disco": 2475, "dal": 2476, "player": 2477, "fit": 2478, "photos": 2479, "ity": 2480, "ok": 2481, "jor": 2482, "tra": 2483, "april": 2484, "ads": 2485, "adi": 2486, "solu": 2487, "beauty": 2488, "door": 2489, "mess": 2490, "update": 2491, "alia": 2492, "scho": 2493, "ened": 2494, "moment": 2495, "scot": 2496, "science": 2497, "ior": 2498, "ties": 2499, "across": 2500, "ously": 2501, "shes": 2502, "doesn": 2503, "page": 2504, "water": 2505, "million": 2506, "classi": 2507, "lic": 2508, "cast": 2509, "formation": 2510, "michael": 2511, "ello": 2512, "smo": 2513, "ints": 2514, "vision": 2515, "opening": 2516, "ldn": 2517, "austr": 2518, "tuesday": 2519, "winner": 2520, "possi": 2521, "round": 2522, "shirt": 2523, "dit": 2524, "bo": 2525, "ues": 2526, "illed": 2527, "along": 2528, "trip": 2529, "starting": 2530, "impro": 2531, "kan": 2532, "person": 2533, "not": 2534, "reco": 2535, "needs": 2536, "cle": 2537, "lie": 2538, "rest": 2539, "ring": 2540, "winter": 2541, "simp": 2542, "mom": 2543, "beer": 2544, "face": 2545, "tors": 2546, "usa": 2547, "collection": 2548, "geor": 2549, "session": 2550, "trying": 2551, "las": 2552, "lake": 2553, "jen": 2554, "origin": 2555, "student": 2556, "secur": 2557, "vin": 2558, "pics": 2559, "expe": 2560, "comp": 2561, "gonna": 2562, "equ": 2563, "bad": 2564, "ley": 2565, "au": 2566, "members": 2567, "break": 2568, "wall": 2569, "gic": 2570, "dinner": 2571, "bul": 2572, "inspir": 2573, "ri": 2574, "mind": 2575, "ica": 2576, "winning": 2577, "talking": 2578, "tren": 2579, "sis": 2580, "ten": 2581, "wonderful": 2582, "snow": 2583, "hear": 2584, "thom": 2585, "nothing": 2586, "gui": 2587, "stin": 2588, "blog": 2589, "fest": 2590, "bun": 2591, "lee": 2592, "wards": 2593, "chance": 2594, "dress": 2595, "ren": 2596, "paul": 2597, "pes": 2598, "techno": 2599, "russi": 2600, "card": 2601, "east": 2602, "mari": 2603, "wine": 2604, "ti": 2605, "law": 2606, "stric": 2607, "ki": 2608, "ape": 2609, "augu": 2610, "profe": 2611, "ash": 2612, "course": 2613, "mail": 2614, "rently": 2615, "dun": 2616, "mun": 2617, "love": 2618, "island": 2619, "drive": 2620, "sl": 2621, "ended": 2622, "main": 2623, "lost": 2624, "nature": 2625, "âĿ¤ï¸ı": 2626, "chic": 2627, "repor": 2628, "pin": 2629, "pro": 2630, "station": 2631, "cep": 2632, "takes": 2633, "company": 2634, "goes": 2635, "ond": 2636, "mach": 2637, "radio": 2638, "dad": 2639, "rock": 2640, "ja": 2641, "pay": 2642, "champion": 2643, "ee": 2644, "inde": 2645, "tta": 2646, "atic": 2647, "tab": 2648, "believe": 2649, "energy": 2650, "zi": 2651, "tat": 2652, "word": 2653, "once": 2654, "resul": 2655, "yl": 2656, "andre": 2657, "ano": 2658, "instagram": 2659, "close": 2660, "tam": 2661, "custom": 2662, "wa": 2663, "conom": 2664, "shows": 2665, "life": 2666, "kin": 2667, "rob": 2668, "tage": 2669, "nation": 2670, "almost": 2671, "listen": 2672, "save": 2673, "reli": 2674, "ace": 2675, "mary": 2676, "tree": 2677, "forget": 2678, "jack": 2679, "waiting": 2680, "director": 2681, "hill": 2682, "born": 2683, "temp": 2684, "fl": 2685, "ste": 2686, "ona": 2687, "single": 2688, "wednesday": 2689, "united": 2690, "ino": 2691, "@_": 2692, "nel": 2693, "celebrate": 2694, "ending": 2695, "deal": 2696, "ji": 2697, "canada": 2698, "huge": 2699, "track": 2700, "âĢ¢": 2701, "fy": 2702, "fanta": 2703, "ang": 2704, "york": 2705, "release": 2706, "pun": 2707, "episo": 2708, "words": 2709, "tour": 2710, "pack": 2711, "igh": 2712, "classic": 2713, "performance": 2714, "ket": 2715, "afternoon": 2716, "record": 2717, "wins": 2718, "proble": 2719, "âĿ¤": 2720, "four": 2721, "bed": 2722, "bank": 2723, "dance": 2724, "sla": 2725, "called": 2726, "might": 2727, "ap": 2728, "past": 2729, "ðŁļ": 2730, "different": 2731, "ite": 2732, "gift": 2733, "ssive": 2734, "church": 2735, "cus": 2736, "program": 2737, "hotel": 2738, "ice": 2739, "mad": 2740, "security": 2741, "enge": 2742, "dc": 2743, "enough": 2744, "sta": 2745, "ety": 2746, "dead": 2747, "gun": 2748, "hear": 2749, "mir": 2750, "human": 2751, "gress": 2752, "ounds": 2753, "piece": 2754, "breaking": 2755, "garden": 2756, "fight": 2757, "views": 2758, "fish": 2759, "started": 2760, "running": 2761, "green": 2762, "seri": 2763, "sm": 2764, "ask": 2765, "dor": 2766, "death": 2767, "econom": 2768, "eri": 2769, "ird": 2770, "ser": 2771, "lunch": 2772, "âģ¦": 2773, "box": 2774, "natu": 2775, "base": 2776, "ban": 2777, "fal": 2778, "global": 2779, "wild": 2780, "wow": 2781, "outside": 2782, "move": 2783, "lead": 2784, "anal": 2785, "museum": 2786, "ong": 2787, "haw": 2788, "power": 2789, "thank": 2790, "bac": 2791, "charac": 2792, "campa": 2793, "digital": 2794, "ro": 2795, "oper": 2796, "dev": 2797, "wol": 2798, "pati": 2799, "fa": 2800, "male": 2801, "paper": 2802, "illing": 2803, "cs": 2804, "âĥ": 2805, "education": 2806, "taken": 2807, "effe": 2808, "mou": 2809, "sad": 2810, "\".": 2811, "based": 2812, "staff": 2813, "including": 2814, "living": 2815, "ac": 2816, "china": 2817, "mob": 2818, "storm": 2819, "luck": 2820, "phil": 2821, "oo": 2822, "yn": 2823, "travel": 2824, "kel": 2825, "tial": 2826, "price": 2827, "book": 2828, "important": 2829, "bio": 2830, "pool": 2831, "nyc": 2832, "fab": 2833, "load": 2834, "?!": 2835, "challenge": 2836, "cry": 2837, "serve": 2838, "wear": 2839, "bus": 2840, "tain": 2841, "number": 2842, "ror": 2843, "kat": 2844, "iz": 2845, "though": 2846, "hosp": 2847, "mm": 2848, "fair": 2849, "utes": 2850, "hot": 2851, "pop": 2852, "fied": 2853, "camp": 2854, "development": 2855, "libr": 2856, "cali": 2857, "ems": 2858, "âģ¦@": 2859, "bol": 2860, "ised": 2861, "standing": 2862, "model": 2863, "ita": 2864, "gle": 2865, "brown": 2866, "image": 2867, "vered": 2868, "force": 2869, "oil": 2870, "partic": 2871, "shu": 2872, "daily": 2873, "law": 2874, "sec": 2875, "class": 2876, "camp": 2877, "holiday": 2878, "clin": 2879, "kers": 2880, "present": 2881, "game": 2882, "incredi": 2883, "ership": 2884, "interview": 2885, "bill": 2886, "due": 2887, "andy": 2888, "abo": 2889, "innov": 2890, "key": 2891, "acade": 2892, "pil": 2893, "moder": 2894, "stars": 2895, "brand": 2896, "fer": 2897, "weeks": 2898, "consi": 2899, "pre": 2900, "safe": 2901, "writ": 2902, "dium": 2903, "launch": 2904, "marketing": 2905, "annual": 2906, "assi": 2907, "court": 2908, "lady": 2909, "cted": 2910, "anda": 2911, "inside": 2912, "child": 2913, "oppor": 2914, "smith": 2915, "centre": 2916, "gue": 2917, "âģ©": 2918, "fren": 2919, "sty": 2920, "fort": 2921, "ently": 2922, "isn": 2923, "keep": 2924, "tober": 2925, "ony": 2926, "boy": 2927, "ald": 2928, "colla": 2929, "demo": 2930, "level": 2931, "compet": 2932, "ado": 2933, "bour": 2934, "fantastic": 2935, "mate": 2936, "su": 2937, "south": 2938, "opportun": 2939, "versary": 2940, "later": 2941, "bud": 2942, "facebook": 2943, "laun": 2944, "stern": 2945, "pit": 2946, "!\"": 2947, "maj": 2948, "gram": 2949, "tbt": 2950, "fire": 2951, "happy": 2952, "aks": 2953, "whole": 2954, "actually": 2955, "iller": 2956, "ella": 2957, "lots": 2958, "alex": 2959, "ange": 2960, "lands": 2961, "ðŁĺŃ": 2962, "enter": 2963, "rou": 2964, "episode": 2965, "ped": 2966, "inten": 2967, "shire": 2968, "who": 2969, "plan": 2970, "ho": 2971, "cake": 2972, "west": 2973, "magaz": 2974, "fresh": 2975, "cc": 2976, "nar": 2977, "chris": 2978, "writing": 2979, "wer": 2980, "nom": 2981, "lo": 2982, "midd": 2983, "dream": 2984, "ol": 2985, "tional": 2986, "deb": 2987, ">>": 2988, "become": 2989, "si": 2990, "grand": 2991, "alling": 2992, "histor": 2993, "ride": 2994, "ired": 2995, "safe": 2996, "queen": 2997, "cil": 2998, "intro": 2999, "vil": 3000, "dani": 3001, "...": 3002, "artic": 3003, "stat": 3004, "short": 3005, "oring": 3006, "selfi": 3007, "missi": 3008, "doc": 3009, "bit": 3010, "gall": 3011, "bom": 3012, "ire": 3013, "selec": 3014, "dition": 3015, "ðŁĶ¥": 3016, "friend": 3017, "beat": 3018, "ghting": 3019, "ðŁĺĬ": 3020, "peace": 3021, "exhi": 3022, "anta": 3023, "ability": 3024, "illu": 3025, "jon": 3026, "quality": 3027, "tribu": 3028, "mes": 3029, "players": 3030, "fair": 3031, "cut": 3032, "cab": 3033, "success": 3034, "bi": 3035, "sus": 3036, "promo": 3037, "sche": 3038, "ange": 3039, "ico": 3040, "commit": 3041, "catch": 3042, "illa": 3043, "kind": 3044, "feeling": 3045, "quo": 3046, "say": 3047, "anniversary": 3048, "spot": 3049, "mother": 3050, "ane": 3051, "pend": 3052, "yourself": 3053, "ops": 3054, "apple": 3055, "minutes": 3056, "po": 3057, "grand": 3058, "ries": 3059, "haha": 3060, "career": 3061, "edition": 3062, "dec": 3063, "rick": 3064, "ami": 3065, "concert": 3066, "itive": 3067, "geous": 3068, "dly": 3069, "tte": 3070, "advent": 3071, "ig": 3072, "lights": 3073, "aker": 3074, "sky": 3075, "âĥ£": 3076, "ray": 3077, "finished": 3078, "way": 3079, "sd": 3080, "accoun": 3081, "ðŁĴķ": 3082, "cky": 3083, "chel": 3084, "liter": 3085, "painting": 3086, "los": 3087, "stun": 3088, "technology": 3089, "nas": 3090, "mar": 3091, "bil": 3092, "africa": 3093, "kie": 3094, "eyes": 3095, "golf": 3096, "plus": 3097, "nia": 3098, "itec": 3099, "services": 3100, "wedding": 3101, "known": 3102, "tele": 3103, ".....": 3104, "starts": 3105, "paren": 3106, "wants": 3107, "ational": 3108, "months": 3109, "windo": 3110, "favour": 3111, "ert": 3112, "magazine": 3113, "exclu": 3114, "reve": 3115, "bc": 3116, "original": 3117, "ess": 3118, "nal": 3119, "anti": 3120, "stro": 3121, "tice": 3122, "study": 3123, "à¤": 3124, "vac": 3125, "national": 3126, "five": 3127, "rain": 3128, "vement": 3129, "ute": 3130, "verse": 3131, "emer": 3132, "army": 3133, "possible": 3134, "guess": 3135, "valley": 3136, "thern": 3137, "crow": 3138, "mr": 3139, "color": 3140, "onto": 3141, "pick": 3142, "clear": 3143, "dark": 3144, "tac": 3145, "wanted": 3146, "itting": 3147, "cancer": 3148, "government": 3149, "die": 3150, "rise": 3151, "zing": 3152, "cold": 3153, "foun": 3154, "studio": 3155, "stration": 3156, "brother": 3157, "ahead": 3158, "shel": 3159, "micro": 3160, "ically": 3161, "dau": 3162, "signed": 3163, "viol": 3164, "ax": 3165, "asse": 3166, "io": 3167, "wre": 3168, "splay": 3169, "chick": 3170, "august": 3171, "plat": 3172, "tips": 3173, "spi": 3174, "human": 3175, "easy": 3176, "logi": 3177, "mike": 3178, "grow": 3179, "agre": 3180, "ww": 3181, "shad": 3182, "motiv": 3183, "wide": 3184, "turns": 3185, "omg": 3186, "var": 3187, "defin": 3188, "sug": 3189, "jim": 3190, "ðŁĶ¥": 3191, "td": 3192, "campaign": 3193, "named": 3194, "retweet": 3195, "cop": 3196, "tv": 3197, "leav": 3198, "kis": 3199, "double": 3200, "smar": 3201, "issue": 3202, "villa": 3203, "information": 3204, "lies": 3205, "stock": 3206, "nt": 3207, "distric": 3208, "shor": 3209, "mix": 3210, "ero": 3211, "sep": 3212, "mex": 3213, "seeing": 3214, "live": 3215, "remin": 3216, "code": 3217, "gur": 3218, "sc": 3219, "wild": 3220, "lun": 3221, "hood": 3222, "spot": 3223, "father": 3224, "forever": 3225, "upd": 3226, "traf": 3227, "fly": 3228, "need": 3229, "gradu": 3230, "train": 3231, "make": 3232, "sab": 3233, "bey": 3234, "size": 3235, "leader": 3236, "talks": 3237, "eu": 3238, "log": 3239, "fox": 3240, "gorgeous": 3241, "less": 3242, "lets": 3243, "surpri": 3244, "myself": 3245, "note": 3246, "lives": 3247, "fru": 3248, "loved": 3249, "sever": 3250, "dem": 3251, "ji": 3252, "soc": 3253, "hold": 3254, "dogs": 3255, "ni": 3256, "âŀ": 3257, "leave": 3258, "airport": 3259, "benef": 3260, "expl": 3261, "ships": 3262, "complete": 3263, "achi": 3264, "great": 3265, "vintage": 3266, "jack": 3267, "roc": 3268, "wood": 3269, "priv": 3270, "offer": 3271, "eye": 3272, "version": 3273, "tea": 3274, "coach": 3275, "offic": 3276, "well": 3277, "gen": 3278, "sat": 3279, "hh": 3280, "youth": 3281, "ox": 3282, "?\"": 3283, "mt": 3284, "mix": 3285, "gg": 3286, "dle": 3287, "natural": 3288, "build": 3289, "breakfast": 3290, "thinking": 3291, "theatre": 3292, "moon": 3293, "berg": 3294, "goals": 3295, "george": 3296, "ene": 3297, "excell": 3298, "iling": 3299, "tune": 3300, "yed": 3301, "gate": 3302, "mit": 3303, "network": 3304, "joe": 3305, "hello": 3306, "fb": 3307, "tube": 3308, "wearing": 3309, "athle": 3310, "struc": 3311, "hard": 3312, "glass": 3313, "gers": 3314, "throw": 3315, "ges": 3316, "bt": 3317, "industry": 3318, "management": 3319, "alist": 3320, "goal": 3321, "stream": 3322, "yel": 3323, "avi": 3324, "icious": 3325, "others": 3326, "ski": 3327, "christi": 3328, "bird": 3329, "esc": 3330, "min": 3331, "tro": 3332, "lt": 3333, "jan": 3334, "imp": 3335, "rights": 3336, "sha": 3337, "organ": 3338, "central": 3339, "ara": 3340, "roll": 3341, "favourite": 3342, "chester": 3343, "else": 3344, "pay": 3345, "cars": 3346, "mine": 3347, "step": 3348, "practice": 3349, "major": 3350, "hang": 3351, "ðŁĺĺ": 3352, "non": 3353, "vari": 3354, "engine": 3355, "volun": 3356, "dia": 3357, "iled": 3358, "architec": 3359, "pink": 3360, "ds": 3361, "thy": 3362, "wash": 3363, "website": 3364, "bag": 3365, "control": 3366, "elli": 3367, "fra": 3368, "answ": 3369, "dence": 3370, "yu": 3371, "ron": 3372, "ola": 3373, "gin": 3374, "drin": 3375, "lic": 3376, "couple": 3377, "spar": 3378, "gon": 3379, "create": 3380, "ct": 3381, "celebrating": 3382, "deep": 3383, "eat": 3384, "tee": 3385, "voice": 3386, "drop": 3387, "visit": 3388, "ators": 3389, "stadium": 3390, "ft": 3391, "wis": 3392, "rol": 3393, "grade": 3394, "famil": 3395, "points": 3396, "repre": 3397, "was": 3398, "traffic": 3399, "japan": 3400, "org": 3401, "honor": 3402, "texas": 3403, "manu": 3404, "âĻ¥": 3405, "safety": 3406, "rer": 3407, "bag": 3408, "emplo": 3409, "released": 3410, "regu": 3411, "aka": 3412, "nav": 3413, "role": 3414, "senior": 3415, "spect": 3416, "cross": 3417, "lines": 3418, "best": 3419, "pack": 3420, "sin": 3421, "tie": 3422, "missing": 3423, "sunset": 3424, "liber": 3425, "ising": 3426, "jay": 3427, "ski": 3428, "championship": 3429, "activ": 3430, "ladies": 3431, "played": 3432, "yy": 3433, "publ": 3434, "alo": 3435, "pride": 3436, "sr": 3437, "paki": 3438, "lux": 3439, "survi": 3440, "cked": 3441, "ets": 3442, "chocol": 3443, "australia": 3444, "paris": 3445, "miles": 3446, "hat": 3447, "mental": 3448, "ala": 3449, "mean": 3450, "mobile": 3451, "ena": 3452, "insi": 3453, "found": 3454, "chief": 3455, "tag": 3456, "incredible": 3457, "return": 3458, "é": 3459, "google": 3460, "french": 3461, "crew": 3462, "hallo": 3463, "alian": 3464, "jaz": 3465, "cher": 3466, "silver": 3467, "north": 3468, "english": 3469, "baseball": 3470, "caf": 3471, "limited": 3472, "following": 3473, "appreci": 3474, "earth": 3475, "kir": 3476, "vember": 3477, "wed": 3478, "ption": 3479, "ged": 3480, "october": 3481, "flori": 3482, "cr": 3483, "ency": 3484, "gave": 3485, "lord": 3486, "stuff": 3487, "berry": 3488, "post": 3489, "smile": 3490, "broad": 3491, "state": 3492, "gger": 3493, "means": 3494, "icy": 3495, "gun": 3496, "yo": 3497, "master": 3498, "burg": 3499, "hands": 3500, "nie": 3501, "//": 3502, "union": 3503, "british": 3504, "biggest": 3505, "district": 3506, "aming": 3507, "hil": 3508, "oce": 3509, "person": 3510, "pass": 3511, "envir": 3512, "schools": 3513, "arrived": 3514, "ances": 3515, "inspired": 3516, "expla": 3517, "ben": 3518, "library": 3519, "bott": 3520, "amp": 3521, "steph": 3522, "contact": 3523, "bang": 3524, "ms": 3525, "califor": 3526, "told": 3527, "battle": 3528, "bb": 3529, "chicago": 3530, "⾨": 3531, "strate": 3532, "shi": 3533, "dece": 3534, "-)": 3535, "add": 3536, "lab": 3537, "jones": 3538, "legend": 3539, "castle": 3540, "inger": 3541, "stance": 3542, "bel": 3543, "ura": 3544, "refu": 3545, "leaders": 3546, "pot": 3547, "sex": 3548, "hic": 3549, "article": 3550, "kid": 3551, "france": 3552, "xx": 3553, "exe": 3554, "guide": 3555, "volunte": 3556, "print": 3557, "ali": 3558, "ceo": 3559, "tweets": 3560, "wx": 3561, "scene": 3562, "volu": 3563, "anti": 3564, "han": 3565, "associ": 3566, "sharing": 3567, "rose": 3568, "minister": 3569, "sher": 3570, "inste": 3571, "clean": 3572, "democr": 3573, "poster": 3574, "skin": 3575, "psy": 3576, "proper": 3577, "crazy": 3578, "iam": 3579, "ore": 3580, "ini": 3581, "anything": 3582, "pod": 3583, "moving": 3584, "click": 3585, "explo": 3586, "comb": 3587, "craft": 3588, "fi": 3589, "blood": 3590, "isra": 3591, "public": 3592, "dent": 3593, "olym": 3594, "england": 3595, "asi": 3596, "cher": 3597, "fact": 3598, "environ": 3599, "harry": 3600, "gone": 3601, "medic": 3602, "enjoying": 3603, "justice": 3604, "jr": 3605, "indian": 3606, "wife": 3607, "sound": 3608, "tes": 3609, "drawing": 3610, "pal": 3611, "idea": 3612, "crit": 3613, "juli": 3614, "iler": 3615, "warm": 3616, "clar": 3617, "thoughts": 3618, "defen": 3619, "council": 3620, "introduc": 3621, "died": 3622, "janu": 3623, "ani": 3624, "send": 3625, "lier": 3626, "ml": 3627, "interesting": 3628, "trade": 3629, "wind": 3630, "bay": 3631, "sac": 3632, "ancy": 3633, "source": 3634, "bes": 3635, "organi": 3636, "arly": 3637, "large": 3638, "ffici": 3639, "tag": 3640, "ut": 3641, "desp": 3642, "oes": 3643, "title": 3644, "sym": 3645, "pictures": 3646, "open": 3647, "women": 3648, "showing": 3649, "ria": 3650, "least": 3651, "leadership": 3652, "current": 3653, "electr": 3654, "valent": 3655, "listening": 3656, "ckey": 3657, "general": 3658, "deser": 3659, "duce": 3660, ";)": 3661, "cent": 3662, "ðŁĺįðŁĺį": 3663, "scott": 3664, "poor": 3665, "selfie": 3666, "events": 3667, "ion": 3668, "wrong": 3669, "dev": 3670, "hill": 3671, "septe": 3672, "culture": 3673, "line": 3674, "sorry": 3675, "sent": 3676, "sister": 3677, "cept": 3678, "kri": 3679, "november": 3680, "ari": 3681, "announce": 3682, "zation": 3683, "bran": 3684, "gent": 3685, "du": 3686, "len": 3687, "pers": 3688, "fm": 3689, "martin": 3690, "op": 3691, "emb": 3692, "ome": 3693, "middle": 3694, "success": 3695, "peter": 3696, "january": 3697, "flu": 3698, "racing": 3699, "dav": 3700, "bike": 3701, "ðŁı»": 3702, "pet": 3703, "shoot": 3704, "professi": 3705, "featuring": 3706, "september": 3707, "nowplaying": 3708, "staur": 3709, "za": 3710, "onic": 3711, "quick": 3712, "baske": 3713, "speaking": 3714, "milit": 3715, "zer": 3716, "chicken": 3717, "bell": 3718, "sad": 3719, "coast": 3720, "loving": 3721, "yers": 3722, "dj": 3723, "panel": 3724, "verage": 3725, "swit": 3726, "icks": 3727, "bou": 3728, "california": 3729, "sam": 3730, "parents": 3731, "ero": 3732, "killed": 3733, "phys": 3734, "jobs": 3735, "migr": 3736, "anth": 3737, "emo": 3738, "halloween": 3739, "ander": 3740, "cm": 3741, "competition": 3742, "eag": 3743, "sket": 3744, "spir": 3745, "maybe": 3746, "exclusive": 3747, "appe": 3748, "journey": 3749, "screen": 3750, "ford": 3751, "io": 3752, "hate": 3753, "ug": 3754, "soul": 3755, "hero": 3756, "society": 3757, "syn": 3758, "guit": 3759, "nh": 3760, "dj": 3761, "ases": 3762, "impre": 3763, "time": 3764, "sales": 3765, "dd": 3766, "fts": 3767, "summit": 3768, "stunning": 3769, "oms": 3770, "turned": 3771, "clean": 3772, "soft": 3773, "beat": 3774, "restaur": 3775, "dered": 3776, "ences": 3777, "magic": 3778, "dio": 3779, "shine": 3780, "guest": 3781, "healthy": 3782, "exhib": 3783, "stories": 3784, "popu": 3785, "nis": 3786, "ela": 3787, "below": 3788, "funny": 3789, "results": 3790, "sne": 3791, "currently": 3792, "ard": 3793, "download": 3794, "flight": 3795, "mal": 3796, "fine": 3797, "pad": 3798, "chu": 3799, "ented": 3800, "hat": 3801, "ðŁijı": 3802, "steve": 3803, "jo": 3804, "mark": 3805, "rat": 3806, "ball": 3807, "pc": 3808, "pon": 3809, "bby": 3810, "oli": 3811, "arts": 3812, "asure": 3813, "bowl": 3814, "attack": 3815, "mic": 3816, "dear": 3817, "range": 3818, "enter": 3819, "chocolate": 3820, "brilli": 3821, "access": 3822, ",\"": 3823, "???": 3824, "chap": 3825, "const": 3826, "tn": 3827, "matter": 3828, "blue": 3829, "gallery": 3830, "emp": 3831, "workshop": 3832, "leading": 3833, "yours": 3834, "basketball": 3835, "wanna": 3836, "thu": 3837, "__": 3838, "marri": 3839, "sleep": 3840, "bia": 3841, "che": 3842, "mad": 3843, "impact": 3844, "own": 3845, "sir": 3846, "channel": 3847, "europe": 3848, "esp": 3849, "kitch": 3850, "hospital": 3851, "wra": 3852, "royal": 3853, "fs": 3854, "neu": 3855, "quar": 3856, "ney": 3857, "acks": 3858, "chase": 3859, "ppy": 3860, "stal": 3861, "ately": 3862, "tim": 3863, "december": 3864, "rare": 3865, "perform": 3866, "cream": 3867, "weight": 3868, "choo": 3869, "night": 3870, "haven": 3871, "franc": 3872, "khan": 3873, "built": 3874, "helping": 3875, "trust": 3876, "type": 3877, "golden": 3878, "tax": 3879, "snow": 3880, "swi": 3881, "disa": 3882, "questions": 3883, "vey": 3884, "light": 3885, "cn": 3886, "cloud": 3887, "thomas": 3888, "aged": 3889, "shou": 3890, "teams": 3891, "gran": 3892, "reason": 3893, "aa": 3894, "youtube": 3895, "vp": 3896, "pizz": 3897, "manager": 3898, "bury": 3899, "credit": 3900, "treat": 3901, "max": 3902, "ik": 3903, "main": 3904, "ging": 3905, "dead": 3906, "probab": 3907, "yeah": 3908, "ãĤ": 3909, "brand": 3910, "soli": 3911, "plant": 3912, "tayl": 3913, "girl": 3914, "ðŁĺŃ": 3915, "nament": 3916, "auto": 3917, "message": 3918, "kore": 3919, "nur": 3920, "terr": 3921, "agu": 3922, "map": 3923, "senting": 3924, "loves": 3925, "gives": 3926, "gab": 3927, "zen": 3928, "robert": 3929, "confir": 3930, "wars": 3931, "om": 3932, "stain": 3933, "camera": 3934, "ander": 3935, "wonder": 3936, "ab": 3937, "cap": 3938, "sold": 3939, "suit": 3940, "walking": 3941, "continue": 3942, "effec": 3943, "daughter": 3944, "danc": 3945, "chain": 3946, "multi": 3947, "kid": 3948, "yan": 3949, "champion": 3950, "vo": 3951, "tains": 3952, "host": 3953, "mini": 3954, "missed": 3955, "resc": 3956, "lyn": 3957, "finish": 3958, "delicious": 3959, "sas": 3960, "taylor": 3961, "ib": 3962, "promis": 3963, "products": 3964, "mountain": 3965, "florida": 3966, "register": 3967, "treat": 3968, "recent": 3969, "female": 3970, "booth": 3971, "matt": 3972, "vehic": 3973, "sop": 3974, "motor": 3975, "supporting": 3976, "phic": 3977, "extre": 3978, "drink": 3979, "lane": 3980, "third": 3981, "ps": 3982, "constru": 3983, "cere": 3984, "farm": 3985, "ðŁİī": 3986, "tured": 3987, "ðŁijī": 3988, "cats": 3989, "aj": 3990, "gie": 3991, "shooting": 3992, "asked": 3993, "pakistan": 3994, "ame": 3995, "mb": 3996, "gil": 3997, "legal": 3998, "square": 3999, "invol": 4000, "draw": 4001, "oooo": 4002, "!!!!": 4003, "opportunity": 4004, "py": 4005, "ei": 4006, "bts": 4007, "teacher": 4008, "character": 4009, "johnson": 4010, "bron": 4011, "lywood": 4012, "chine": 4013, "cing": 4014, "cine": 4015, "dge": 4016, "gaming": 4017, "russia": 4018, "cia": 4019, "quote": 4020, "rich": 4021, "gov": 4022, "flowers": 4023, "spiri": 4024, "stin": 4025, "growth": 4026, "ðŁı¼": 4027, "commer": 4028, "juni": 4029, "mum": 4030, "ran": 4031, "sna": 4032, "aren": 4033, "cb": 4034, "actor": 4035, "color": 4036, "sit": 4037, "pair": 4038, "chi": 4039, "bow": 4040, "academy": 4041, "held": 4042, "rang": 4043, "metal": 4044, "yl": 4045, "active": 4046, "probably": 4047, "tch": 4048, "needed": 4049, "spee": 4050, "choice": 4051, "italy": 4052, "ryan": 4053, "ðŁĩº": 4054, "flower": 4055, "vit": 4056, "mn": 4057, "foundation": 4058, "bak": 4059, "sions": 4060, "neigh": 4061, "floo": 4062, "heard": 4063, "remo": 4064, "fresh": 4065, "inging": 4066, "ref": 4067, "town": 4068, "clou": 4069, "jesus": 4070, "spirit": 4071, "couldn": 4072, "zes": 4073, "ðŁĴĻ": 4074, "williams": 4075, "proce": 4076, "modern": 4077, "process": 4078, "shoes": 4079, "created": 4080, "tric": 4081, "issues": 4082, "anne": 4083, "atten": 4084, "debut": 4085, "hr": 4086, "nit": 4087, "stig": 4088, "apo": 4089, "eps": 4090, "zu": 4091, "ãĢ": 4092, "six": 4093, "cards": 4094, "langu": 4095, "famous": 4096, "tournament": 4097, "sel": 4098, "ebay": 4099, "yn": 4100, "ston": 4101, "kick": 4102, "announced": 4103, "kam": 4104, "voc": 4105, "brilliant": 4106, "house": 4107, "cheese": 4108, "warri": 4109, "music": 4110, "hockey": 4111, "ðŁĺĤðŁĺĤ": 4112, "skills": 4113, "autom": 4114, "smart": 4115, "medical": 4116, "mony": 4117, "ex": 4118, "guar": 4119, "give": 4120, "personal": 4121, "vention": 4122, "alli": 4123, "press": 4124, "floor": 4125, "mc": 4126, "victory": 4127, "him": 4128, "simple": 4129, "thor": 4130, "ðŁĩºðŁĩ": 4131, "tail": 4132, "lucky": 4133, "alex": 4134, "quite": 4135, "bot": 4136, "ssions": 4137, "challeng": 4138, "cann": 4139, "amazon": 4140, "hell": 4141, "bought": 4142, "):": 4143, "edy": 4144, "secret": 4145, "production": 4146, "independ": 4147, "defe": 4148, "added": 4149, "pr": 4150, "pag": 4151, "bed": 4152, "greatest": 4153, "within": 4154, "jay": 4155, "ðŁ¥": 4156, "ireland": 4157, "rely": 4158, "sd": 4159, "text": 4160, "driving": 4161, "program": 4162, "speed": 4163, "colum": 4164, "stron": 4165, "é": 4166, "forest": 4167, "âĸ": 4168, "machine": 4169, "coin": 4170, "scar": 4171, "ount": 4172, "bie": 4173, "¡ï¸ı": 4174, "portra": 4175, "common": 4176, "wrest": 4177, "received": 4178, "know": 4179, "invest": 4180, "plans": 4181, "accor": 4182, "adop": 4183, "tery": 4184, "reali": 4185, "pp": 4186, "kal": 4187, "artwork": 4188, "mean": 4189, "god": 4190, "instead": 4191, "anci": 4192, "motivation": 4193, "asing": 4194, "inspiration": 4195, "upcoming": 4196, "political": 4197, "europe": 4198, "mers": 4199, "heavy": 4200, "ðŁijį": 4201, "febru": 4202, "scotland": 4203, "ough": 4204, "bt": 4205, "boss": 4206, "schedu": 4207, "speak": 4208, "nick": 4209, "ured": 4210, "ino": 4211, "ek": 4212, "risk": 4213, "tory": 4214, "presents": 4215, "bon": 4216, "rug": 4217, "states": 4218, "exhibition": 4219, "ilo": 4220, "mill": 4221, "brought": 4222, ":-)": 4223, "touri": 4224, "come": 4225, "officially": 4226, "champions": 4227, "doors": 4228, "rep": 4229, "pose": 4230, "extra": 4231, "kings": 4232, "soccer": 4233, "squad": 4234, "applic": 4235, "ata": 4236, "sometimes": 4237, "tari": 4238, "excellent": 4239, "ðŁĺĺ": 4240, "straight": 4241, "carol": 4242, "rip": 4243, "âĢį": 4244, "graphic": 4245, "mol": 4246, "election": 4247, "february": 4248, "asons": 4249, "li": 4250, "dir": 4251, "mt": 4252, "nick": 4253, "usu": 4254, "mrs": 4255, "comics": 4256, "institu": 4257, "corpor": 4258, "vi": 4259, "ðŁĻı": 4260, "tural": 4261, "dise": 4262, "acci": 4263, "weare": 4264, "among": 4265, "shopping": 4266, "till": 4267, "what": 4268, "chair": 4269, "span": 4270, "chinese": 4271, "innovation": 4272, "joy": 4273, "kit": 4274, "century": 4275, "obama": 4276, "phili": 4277, "fc": 4278, "reach": 4279, "citi": 4280, "ulous": 4281, "non": 4282, "dang": 4283, "happening": 4284, "burn": 4285, "pel": 4286, "orange": 4287, "dv": 4288, "kick": 4289, "claim": 4290, "ingham": 4291, "phy": 4292, "nov": 4293, "podcast": 4294, "whi": 4295, "nights": 4296, "earlier": 4297, "bear": 4298, "lah": 4299, "exciting": 4300, "ora": 4301, "given": 4302, "slo": 4303, "memories": 4304, "continues": 4305, "product": 4306, "gho": 4307, "cd": 4308, "knows": 4309, "ðŁİī": 4310, "published": 4311, "discuss": 4312, "yard": 4313, "iphone": 4314, "tries": 4315, "wall": 4316, "feb": 4317, "aren": 4318, "truth": 4319, "winners": 4320, "ture": 4321, "ditional": 4322, "military": 4323, "problem": 4324, "mand": 4325, "dog": 4326, "loss": 4327, "cric": 4328, "canadi": 4329, "veter": 4330, "village": 4331, "\",": 4332, "yr": 4333, "ung": 4334, "donald": 4335, "aging": 4336, "birds": 4337, "scienti": 4338, "les": 4339, "this": 4340, "region": 4341, "tical": 4342, "itten": 4343, "ila": 4344, "ðŁĺİ": 4345, "dad": 4346, "diam": 4347, "above": 4348, "stren": 4349, "lit": 4350, "pir": 4351, "lab": 4352, "focus": 4353, "busy": 4354, "dur": 4355, "apply": 4356, "sma": 4357, "author": 4358, "aci": 4359, "execu": 4360, "domin": 4361, "rela": 4362, "jackson": 4363, "ato": 4364, "washington": 4365, "ðŁĻĮ": 4366, "kill": 4367, "popular": 4368, "cement": 4369, "road": 4370, "eating": 4371, "location": 4372, "vent": 4373, "arre": 4374, "nan": 4375, "custo": 4376, "adventure": 4377, "ordin": 4378, "sport": 4379, "ult": 4380, "lock": 4381, "question": 4382, "driver": 4383, "landsc": 4384, "oni": 4385, "kins": 4386, "pd": 4387, "jordan": 4388, "tered": 4389, "kk": 4390, "af": 4391, "child": 4392, "sp": 4393, "justin": 4394, "eni": 4395, "selling": 4396, "zo": 4397, "whit": 4398, "boston": 4399, "particip": 4400, "signing": 4401, "happened": 4402, "heat": 4403, "mam": 4404, "dreams": 4405, "lows": 4406, "graph": 4407, "theday": 4408, "heading": 4409, "bro": 4410, "blessed": 4411, "vic": 4412, "vegas": 4413, "hd": 4414, "inning": 4415, "roman": 4416, "andro": 4417, "denti": 4418, "use": 4419, "cit": 4420, "progress": 4421, "writer": 4422, "bob": 4423, "ffs": 4424, "growing": 4425, "bly": 4426, "aware": 4427, "exam": 4428, "spent": 4429, "bet": 4430, "score": 4431, "beyond": 4432, "docu": 4433, "adel": 4434, "sf": 4435, "coura": 4436, "collabor": 4437, "inc": 4438, "private": 4439, "boat": 4440, "**": 4441, "zone": 4442, "pha": 4443, "bill": 4444, "total": 4445, "planning": 4446, "towards": 4447, "places": 4448, "preview": 4449, "creative": 4450, "damn": 4451, "ideas": 4452, "seems": 4453, "poten": 4454, "saying": 4455, "display": 4456, "sw": 4457, "aqu": 4458, "louis": 4459, "bye": 4460, "lil": 4461, "email": 4462, "western": 4463, "germany": 4464, "eller": 4465, "res": 4466, "fant": 4467, "mentary": 4468, "deals": 4469, "richard": 4470, "jersey": 4471, "streng": 4472, "rad": 4473, "pizza": 4474, "mond": 4475, "ware": 4476, "lac": 4477, "gi": 4478, "archi": 4479, "cd": 4480, "yellow": 4481, "recently": 4482, "reach": 4483, "à¹": 4484, "kitchen": 4485, "designed": 4486, "try": 4487, "gal": 4488, "restaurant": 4489, "ature": 4490, "ww": 4491, "jas": 4492, "lma": 4493, "ðŁijĮ": 4494, "pain": 4495, "avo": 4496, "minute": 4497, "schol": 4498, "therap": 4499, "ticket": 4500, "dry": 4501, "japan": 4502, "ditions": 4503, "terri": 4504, "selves": 4505, "happen": 4506, "tup": 4507, "mag": 4508, "copy": 4509, "sher": 4510, "freedom": 4511, "file": 4512, "specially": 4513, "toronto": 4514, "load": 4515, "gary": 4516, "rey": 4517, "answer": 4518, "loy": 4519, "caught": 4520, "prize": 4521, "une": 4522, "fication": 4523, "niger": 4524, "syd": 4525, "touch": 4526, "feature": 4527, "jazz": 4528, "records": 4529, "himself": 4530, "dish": 4531, "rober": 4532, "spotted": 4533, "master": 4534, "wave": 4535, "finals": 4536, "bull": 4537, "forum": 4538, "ald": 4539, "recomm": 4540, "cha": 4541, "ae": 4542, "doo": 4543, "instru": 4544, "truly": 4545, "lg": 4546, "ink": 4547, "brothers": 4548, "dest": 4549, "jim": 4550, "mit": 4551, "closed": 4552, "ison": 4553, "tried": 4554, "santa": 4555, "affe": 4556, "wan": 4557, "horse": 4558, "grow": 4559, "campus": 4560, "relation": 4561, "native": 4562, "journ": 4563, "gov": 4564, "oct": 4565, "kit": 4566, "bound": 4567, "partner": 4568, "rema": 4569, "crowd": 4570, "!)": 4571, "calls": 4572, "rail": 4573, "quali": 4574, "solution": 4575, "contest": 4576, "convers": 4577, "snap": 4578, "base": 4579, "initi": 4580, "tax": 4581, "ye": 4582, "entrepre": 4583, "itor": 4584, "construction": 4585, "food": 4586, "presented": 4587, "nings": 4588, "climate": 4589, "km": 4590, "model": 4591, "bj": 4592, "block": 4593, "presentation": 4594, "dream": 4595, "fix": 4596, "calling": 4597, "busine": 4598, "congress": 4599, "understand": 4600, "web": 4601, "value": 4602, "ï¸ıâĥ£": 4603, "mexico": 4604, "itely": 4605, "kim": 4606, "charity": 4607, "reflec": 4608, "blan": 4609, "flying": 4610, "analy": 4611, "families": 4612, "band": 4613, "recipe": 4614, "celebration": 4615, "accep": 4616, "ary": 4617, "tot": 4618, "gb": 4619, "interested": 4620, "captain": 4621, "âĻ¥": 4622, "tip": 4623, "absol": 4624, "braz": 4625, "investig": 4626, "ology": 4627, "dec": 4628, "truck": 4629, "vering": 4630, "clear": 4631, "dont": 4632, "gotta": 4633, "advis": 4634, "begins": 4635, "mass": 4636, "descri": 4637, "block": 4638, "kim": 4639, "david": 4640, "songs": 4641, "memorial": 4642, "features": 4643, "sustain": 4644, "'.": 4645, "grab": 4646, "jose": 4647, "va": 4648, "conserv": 4649, "sets": 4650, "manchester": 4651, "fighting": 4652, "degre": 4653, "aga": 4654, "ind": 4655, "sleep": 4656, "position": 4657, "hair": 4658, "signs": 4659, "policy": 4660, "ito": 4661, "alert": 4662, "stam": 4663, "spend": 4664, "wy": 4665, "absolut": 4666, "dm": 4667, "animal": 4668, "myster": 4669, "successful": 4670, "problems": 4671, "robo": 4672, "kay": 4673, "garden": 4674, "pd": 4675, "mayor": 4676, "dale": 4677, "tol": 4678, "offers": 4679, "visiting": 4680, "friendly": 4681, "trees": 4682, "officer": 4683, "account": 4684, "kevin": 4685, "ðŁijį": 4686, "giant": 4687, "continu": 4688, "consu": 4689, "tract": 4690, "nfl": 4691, "ðŁĺĬ": 4692, "hq": 4693, "bility": 4694, "aar": 4695, "disney": 4696, "teen": 4697, "oned": 4698, "white": 4699, "trailer": 4700, "dedic": 4701, "alone": 4702, "absolutely": 4703, "digital": 4704, "william": 4705, "ination": 4706, "swa": 4707, "ee": 4708, "entire": 4709, "german": 4710, "roll": 4711, "hits": 4712, "cost": 4713, "stay": 4714, "tha": 4715, "alive": 4716, "according": 4717, "cot": 4718, "literally": 4719, "herit": 4720, "reti": 4721, "hahaha": 4722, "experi": 4723, "likes": 4724, "gt": 4725, "steel": 4726, "____": 4727, "chair": 4728, "christian": 4729, "tower": 4730, "difference": 4731, "md": 4732, "tress": 4733, "mid": 4734, "prince": 4735, "african": 4736, "feder": 4737, "foot": 4738, "carri": 4739, "served": 4740, "rice": 4741, "shall": 4742, "featured": 4743, "cker": 4744, "recru": 4745, "poe": 4746, "sense": 4747, "nific": 4748, "comedy": 4749, "content": 4750, "fat": 4751, "posted": 4752, "contribu": 4753, "timate": 4754, "liver": 4755, "mble": 4756, "internet": 4757, "age": 4758, "european": 4759, "cling": 4760, "glad": 4761, "ffic": 4762, "sco": 4763, "akes": 4764, "elle": 4765, "termin": 4766, "tony": 4767, "pale": 4768, "colour": 4769, "serious": 4770, "patri": 4771, "movies": 4772, "bm": 4773, "professional": 4774, "ado": 4775, "alu": 4776, "bringing": 4777, "falls": 4778, "israel": 4779, "term": 4780, "language": 4781, "brook": 4782, "mann": 4783, "communic": 4784, "cannot": 4785, "acti": 4786, "phe": 4787, "yan": 4788, "entreprene": 4789, "turkey": 4790, "logical": 4791, "long": 4792, "arm": 4793, "urs": 4794, "workers": 4795, "ingly": 4796, "ggs": 4797, "ric": 4798, "tual": 4799, "receive": 4800, "opens": 4801, "gear": 4802, "social": 4803, "feet": 4804, "cking": 4805, "adver": 4806, "finan": 4807, "feels": 4808, "spla": 4809, "hr": 4810, "easter": 4811, "brain": 4812, "ãģ": 4813, "fig": 4814, "ledge": 4815, "nearly": 4816, "protect": 4817, "massive": 4818, "eth": 4819, "awa": 4820, "ðŁĺģ": 4821, "yrs": 4822, "awareness": 4823, "definitely": 4824, "kn": 4825, "imagine": 4826, "ku": 4827, "systems": 4828, "ðŁijı": 4829, "fas": 4830, "lik": 4831, "provide": 4832, "amo": 4833, "discover": 4834, "influ": 4835, "maker": 4836, "gaz": 4837, "fitness": 4838, "street": 4839, "ers": 4840, "ted": 4841, "wc": 4842, "ysis": 4843, "positive": 4844, "helped": 4845, "quest": 4846, "andrew": 4847, "brad": 4848, "bin": 4849, "hanging": 4850, "ling": 4851, "bright": 4852, "section": 4853, "mass": 4854, "ðŁĻĮ": 4855, "followers": 4856, "hosting": 4857, "tempor": 4858, "flag": 4859, "ave": 4860, "letter": 4861, "kur": 4862, "requi": 4863, "often": 4864, "cryp": 4865, "suff": 4866, "âļ½": 4867, "russian": 4868, "treatment": 4869, "alle": 4870, "hay": 4871, "lan": 4872, "keeping": 4873, "holy": 4874, "powerful": 4875, "predic": 4876, "fund": 4877, "especially": 4878, "window": 4879, "jewel": 4880, "ily": 4881, "ðŁĴľ": 4882, "generation": 4883, "appa": 4884, "seriously": 4885, "od": 4886, "ðŁĺĤðŁĺĤðŁĺĤ": 4887, "certi": 4888, "irish": 4889, "ðŁijĮ": 4890, "miami": 4891, "beth": 4892, "vity": 4893, "secu": 4894, "chef": 4895, "crime": 4896, "graphy": 4897, "max": 4898, "artists": 4899, "revolu": 4900, "guard": 4901, "speech": 4902, "uc": 4903, "updates": 4904, "faces": 4905, "stant": 4906, "changed": 4907, "reports": 4908, "lower": 4909, "pear": 4910, "nc": 4911, "kil": 4912, "looked": 4913, "speaker": 4914, "sf": 4915, "respect": 4916, "okay": 4917, "ocean": 4918, "sitting": 4919, "architecture": 4920, "trail": 4921, "seat": 4922, "ira": 4923, "leg": 4924, "japanese": 4925, "dam": 4926, "ular": 4927, "swim": 4928, "politics": 4929, "financial": 4930, "old": 4931, "mouth": 4932, "attemp": 4933, "destin": 4934, "fishing": 4935, "attention": 4936, "mem": 4937, "changes": 4938, "decided": 4939, "religi": 4940, "gin": 4941, "cav": 4942, "zz": 4943, "adam": 4944, "mac": 4945, "write": 4946, "begin": 4947, "scul": 4948, "alter": 4949, "iss": 4950, "athon": 4951, "images": 4952, "moo": 4953, "joined": 4954, "ðŁĺī": 4955, "âŀ¡ï¸ı": 4956, "passed": 4957, "musli": 4958, "hir": 4959, "largest": 4960, "camer": 4961, "comic": 4962, "ghted": 4963, "rugby": 4964, "burgh": 4965, "gging": 4966, "testing": 4967, "prepar": 4968, "laugh": 4969, "aled": 4970, "improve": 4971, "believ": 4972, "advice": 4973, "shares": 4974, "heart": 4975, "turning": 4976, "sb": 4977, "tel": 4978, "cafe": 4979, "nes": 4980, "daniel": 4981, "patter": 4982, "tz": 4983, "sett": 4984, "park": 4985, "cand": 4986, "stick": 4987, "happens": 4988, "brian": 4989, "newest": 4990, "epic": 4991, "ador": 4992, "kies": 4993, "warning": 4994, "animals": 4995, "custom": 4996, "arc": 4997, "dian": 4998, "gold": 4999, "core": 5000, "tf": 5001, "city": 5002, "pants": 5003, "reality": 5004, "confi": 5005, "inju": 5006, "fox": 5007, "guil": 5008, "knew": 5009, "âĺº": 5010, "correc": 5011, "itude": 5012, "dden": 5013, ".#": 5014, "reduc": 5015, "pass": 5016, "fon": 5017, "ya": 5018, "owner": 5019, "returns": 5020, "nc": 5021, "east": 5022, "apol": 5023, "insur": 5024, "tho": 5025, "sim": 5026, "junior": 5027, "bee": 5028, "angel": 5029, "attle": 5030, "electric": 5031, "horror": 5032, "crash": 5033, "eye": 5034, "path": 5035, "southern": 5036, "employe": 5037, "geo": 5038, "tan": 5039, "haz": 5040, "rally": 5041, "ðŁı»": 5042, "property": 5043, "wasn": 5044, "enjoyed": 5045, "grey": 5046, "gas": 5047, "brew": 5048, "northern": 5049, "holding": 5050, "gp": 5051, "take": 5052, "chart": 5053, "lyn": 5054, "drama": 5055, "zo": 5056, "paid": 5057, "throwback": 5058, "cup": 5059, "discussion": 5060, "downtown": 5061, "will": 5062, "lew": 5063, "bis": 5064, "tary": 5065, "bread": 5066, "upon": 5067, "rate": 5068, "teachers": 5069, "itation": 5070, "anced": 5071, "cycle": 5072, "choose": 5073, "dc": 5074, "iran": 5075, "cow": 5076, "dave": 5077, "raise": 5078, "princess": 5079, "faith": 5080, "->": 5081, "industri": 5082, "spain": 5083, "guitar": 5084, "facts": 5085, "mn": 5086, "spen": 5087, "courte": 5088, "gott": 5089, "projects": 5090, "audi": 5091, "osc": 5092, "peter": 5093, "sand": 5094, "interest": 5095, "happiness": 5096, "venue": 5097, "soldi": 5098, "surprise": 5099, "potential": 5100, "perio": 5101, "customer": 5102, "ii": 5103, "gni": 5104, "manufac": 5105, "eco": 5106, "broken": 5107, "singer": 5108, "vels": 5109, "wales": 5110, "hus": 5111, "inj": 5112, "four": 5113, "talent": 5114, "dying": 5115, "matthe": 5116, "film": 5117, "joining": 5118, "sell": 5119, "jar": 5120, "lmao": 5121, "surger": 5122, "bbc": 5123, "sources": 5124, "austin": 5125, "nik": 5126, "charles": 5127, "fam": 5128, "princi": 5129, "angel": 5130, "cash": 5131, "lot": 5132, "ored": 5133, "plays": 5134, "plate": 5135, "done": 5136, "memory": 5137, "brings": 5138, "nba": 5139, "solutions": 5140, "teaching": 5141, "grace": 5142, "circu": 5143, "helps": 5144, "founder": 5145, "mary": 5146, "explore": 5147, "decor": 5148, "parts": 5149, "cho": 5150, "integr": 5151, "hau": 5152, "ises": 5153, "putting": 5154, "iner": 5155, "rit": 5156, "vy": 5157, "michel": 5158, "blues": 5159, "everyday": 5160, "forms": 5161, "bio": 5162, "year": 5163, "pin": 5164, "tter": 5165, "spring": 5166, "))": 5167, "pot": 5168, "aling": 5169, "performing": 5170, "shan": 5171, "planet": 5172, "musical": 5173, "heads": 5174, "italian": 5175, "strugg": 5176, "âĢįâĻ": 5177, "wings": 5178, "pump": 5179, "hh": 5180, "trou": 5181, "aid": 5182, "prime": 5183, "earth": 5184, "paint": 5185, "mont": 5186, "amy": 5187, "bbc": 5188, "fabulous": 5189, "fruit": 5190, "android": 5191, "bourne": 5192, "ceremony": 5193, "ential": 5194, "??": 5195, "debate": 5196, "oning": 5197, "draft": 5198, "solar": 5199, "tx": 5200, "jam": 5201, "corn": 5202, "!!!!!": 5203, "broo": 5204, "milk": 5205, "posed": 5206, "ohi": 5207, "movement": 5208, "bren": 5209, "partner": 5210, "pg": 5211, "ette": 5212, "aries": 5213, "shout": 5214, "ng": 5215, "leaving": 5216, "tells": 5217, "sens": 5218, "taste": 5219, "kelly": 5220, "worl": 5221, "gym": 5222, "rich": 5223, "egy": 5224, "pid": 5225, "mas": 5226, "âĤ": 5227, "courtesy": 5228, "frank": 5229, "increase": 5230, "written": 5231, "ppers": 5232, "rel": 5233, "hai": 5234, "sas": 5235, "sound": 5236, "tti": 5237, "wich": 5238, "river": 5239, "...\"": 5240, "ag": 5241, "fellow": 5242, "rome": 5243, "small": 5244, "gency": 5245, "ican": 5246, "luxury": 5247, "proof": 5248, "met": 5249, "wildlife": 5250, "moments": 5251, "rather": 5252, "corner": 5253, "compe": 5254, "canadian": 5255, "likely": 5256, "therapy": 5257, "liam": 5258, "economic": 5259, "indie": 5260, "route": 5261, "fight": 5262, "hope": 5263, "setting": 5264, "antly": 5265, "cross": 5266, "fantasy": 5267, "dee": 5268, "sketch": 5269, "compli": 5270, "ymi": 5271, "rules": 5272, "engineering": 5273, "figure": 5274, "row": 5275, ".,": 5276, "fw": 5277, "sydney": 5278, "wou": 5279, "tation": 5280, "drew": 5281, "uses": 5282, "there": 5283, "spread": 5284, "structure": 5285, "patrick": 5286, "apparently": 5287, "ros": 5288, "hills": 5289, "wwe": 5290, "anny": 5291, "commission": 5292, "div": 5293, "fying": 5294, "consul": 5295, "analysis": 5296, "exi": 5297, "tennis": 5298, "vehicle": 5299, "ðŁĺŃðŁĺŃ": 5300, "ass": 5301, "highly": 5302, "opened": 5303, "bann": 5304, "ðŁĴĻ": 5305, "mph": 5306, "wishing": 5307, "vor": 5308, "fif": 5309, "giveaway": 5310, "rr": 5311, "ray": 5312, "jess": 5313, "gat": 5314, "icymi": 5315, "xit": 5316, "highest": 5317, "york": 5318, "pie": 5319, "involved": 5320, "higher": 5321, "rie": 5322, "malay": 5323, "intelli": 5324, "despite": 5325, "chee": 5326, "sarah": 5327, "bean": 5328, "recogni": 5329, "arsen": 5330, "talented": 5331, "passion": 5332, "ich": 5333, "abc": 5334, "leads": 5335, "disease": 5336, "vis": 5337, "sec": 5338, "presenting": 5339, "milli": 5340, "hole": 5341, "shots": 5342, "depart": 5343, "surgery": 5344, "govt": 5345, "bin": 5346, "dual": 5347, "evi": 5348, "longer": 5349, "evol": 5350, "screen": 5351, "portrait": 5352, "etc": 5353, "lose": 5354, "chat": 5355, "pen": 5356, "pi": 5357, "oma": 5358, "sick": 5359, "erc": 5360, "companies": 5361, "entry": 5362, "plane": 5363, "gry": 5364, "vene": 5365, "liverpool": 5366, "premiere": 5367, "shared": 5368, "ared": 5369, "films": 5370, "ira": 5371, "holidays": 5372, "cricket": 5373, "ician": 5374, "ving": 5375, ".)": 5376, "ultimate": 5377, "division": 5378, "conduc": 5379, "sept": 5380, "forces": 5381, "mont": 5382, "smart": 5383, "disapp": 5384, "sunshine": 5385, "ind": 5386, "bless": 5387, "made": 5388, "colors": 5389, "frank": 5390, "iron": 5391, "bottle": 5392, "sgo": 5393, "mood": 5394, "jason": 5395, "eric": 5396, "birth": 5397, "teen": 5398, "response": 5399, "target": 5400, "statement": 5401, "fear": 5402, "thel": 5403, "alum": 5404, "arab": 5405, "blin": 5406, "direction": 5407, "steps": 5408, "erial": 5409, "worked": 5410, "atl": 5411, "ðŁĴķ": 5412, "felt": 5413, "poli": 5414, "scenes": 5415, "homes": 5416, "bell": 5417, "eat": 5418, "ateful": 5419, "tin": 5420, "lace": 5421, "folks": 5422, "pse": 5423, "ann": 5424, "wisdom": 5425, "fav": 5426, "butter": 5427, "sr": 5428, "areas": 5429, "smoo": 5430, "biz": 5431, "dges": 5432, "appo": 5433, "more": 5434, "them": 5435, "effect": 5436, "windows": 5437, "sunny": 5438, "capital": 5439, "totally": 5440, "cities": 5441, "grant": 5442, "mbers": 5443, "slow": 5444, "autu": 5445, "ilities": 5446, "wro": 5447, "rising": 5448, "stics": 5449, "violence": 5450, "igh": 5451, "quot": 5452, "hit": 5453, "tc": 5454, "heritage": 5455, "buff": 5456, "nes": 5457, "zar": 5458, "dential": 5459, "exac": 5460, "edge": 5461, "deep": 5462, "arena": 5463, "became": 5464, "benefits": 5465, "marks": 5466, "mber": 5467, "az": 5468, "ames": 5469, "preci": 5470, "dragon": 5471, "reg": 5472, "dings": 5473, "dos": 5474, "ðŁĴª": 5475, "nel": 5476, "sity": 5477, "meal": 5478, "dist": 5479, "legend": 5480, "purchase": 5481, "pical": 5482, "stick": 5483, "fat": 5484, "duba": 5485, "profess": 5486, "carto": 5487, "prof": 5488, "countries": 5489, "responsi": 5490, "sequ": 5491, "fab": 5492, "tribute": 5493, "honored": 5494, "practic": 5495, "purple": 5496, "anton": 5497, "pared": 5498, "tough": 5499, "summer": 5500, "environment": 5501, "sons": 5502, "ðŁĻı": 5503, "mps": 5504, "gies": 5505, "heroes": 5506, "telling": 5507, "henry": 5508, "fen": 5509, "knowledge": 5510, "Ģï¸ı": 5511, "fr": 5512, "neg": 5513, "ure": 5514, "acking": 5515, "hearts": 5516, "soo": 5517, "hollywood": 5518, "jump": 5519, "sauce": 5520, "schedule": 5521, "turn": 5522, "yoga": 5523, "creating": 5524, "cket": 5525, "creek": 5526, "âŃ": 5527, "customers": 5528, "madri": 5529, "gul": 5530, "assemb": 5531, "mount": 5532, "cell": 5533, "top": 5534, "stal": 5535, "davis": 5536, "twi": 5537, "sign": 5538, "premier": 5539, "itions": 5540, "hearing": 5541, "unk": 5542, "patients": 5543, "appear": 5544, "heaven": 5545, "alty": 5546, "doctor": 5547, "ae": 5548, "platform": 5549, "jeff": 5550, "ðŁĵ·": 5551, "regional": 5552, "bid": 5553, "boxing": 5554, "exten": 5555, "ority": 5556, "aw": 5557, "wise": 5558, "ille": 5559, "several": 5560, "bie": 5561, "situ": 5562, "syria": 5563, "âľħ": 5564, "reminder": 5565, "entertain": 5566, "lion": 5567, "partners": 5568, "inn": 5569, "phar": 5570, "fau": 5571, "pls": 5572, "expected": 5573, "sugar": 5574, "decision": 5575, "sb": 5576, "chron": 5577, "association": 5578, "leaves": 5579, "visited": 5580, "shap": 5581, "ðŁĴĸ": 5582, "further": 5583, "hann": 5584, "wi": 5585, "runs": 5586, "ler": 5587, "funding": 5588, "filled": 5589, "......": 5590, "tiny": 5591, "hang": 5592, "org": 5593, "cool": 5594, "semin": 5595, "ðŁıĨ": 5596, "spons": 5597, "navy": 5598, "saint": 5599, "drug": 5600, "dal": 5601, "roun": 5602, "covered": 5603, "traditional": 5604, "investment": 5605, "dete": 5606, "alism": 5607, "flow": 5608, "nis": 5609, "sunrise": 5610, "feat": 5611, "fted": 5612, "weird": 5613, "jere": 5614, "vegan": 5615, "medicine": 5616, "ano": 5617, "accu": 5618, "delivery": 5619, "temple": 5620, "changing": 5621, "wilson": 5622, "philipp": 5623, "refe": 5624, "nd": 5625, "iser": 5626, "gay": 5627, "rand": 5628, "atives": 5629, "tely": 5630, "pand": 5631, "intellig": 5632, "gare": 5633, "ambas": 5634, "demon": 5635, "committee": 5636, "strategy": 5637, "refuge": 5638, "budget": 5639, "protec": 5640, "pier": 5641, "express": 5642, "nomin": 5643, "economy": 5644, "allow": 5645, "icon": 5646, "galax": 5647, "oh": 5648, "indivi": 5649, "demand": 5650, "virgin": 5651, "luke": 5652, "alists": 5653, "mani": 5654, "smi": 5655, "judge": 5656, "enty": 5657, "michi": 5658, "result": 5659, "amed": 5660, "speaks": 5661, "',": 5662, "houston": 5663, "shin": 5664, "bing": 5665, "fly": 5666, "chem": 5667, "auto": 5668, "vas": 5669, "get": 5670, "arm": 5671, "thanks": 5672, "din": 5673, "gang": 5674, "xx": 5675, "sion": 5676, "located": 5677, "pl": 5678, "josh": 5679, "info": 5680, "joins": 5681, "adverti": 5682, "otd": 5683, "eld": 5684, "sie": 5685, "reasons": 5686, "vent": 5687, "ðŁĩºðŁĩ¸": 5688, "âł": 5689, "conversation": 5690, "studi": 5691, "ðŁĶ¥ðŁĶ¥": 5692, "gos": 5693, "sounds": 5694, "unit": 5695, "musc": 5696, "gel": 5697, "acked": 5698, "paci": 5699, "cos": 5700, "dere": 5701, "uu": 5702, "ao": 5703, "lam": 5704, "inspiring": 5705, "arms": 5706, "tware": 5707, "matters": 5708, "addic": 5709, "dude": 5710, "ext": 5711, "crisis": 5712, "bath": 5713, "meet": 5714, "singh": 5715, "expect": 5716, "delhi": 5717, "rescue": 5718, "worst": 5719, "aug": 5720, "shipping": 5721, "serving": 5722, "sto": 5723, "dark": 5724, "aces": 5725, "historic": 5726, "landscape": 5727, "designer": 5728, "billion": 5729, "grateful": 5730, "wake": 5731, "eve": 5732, "miller": 5733, "housing": 5734, "dynam": 5735, "isco": 5736, "beha": 5737, "shop": 5738, "prou": 5739, "eas": 5740, "asia": 5741, "eding": 5742, "kon": 5743, "department": 5744, "awar": 5745, "marine": 5746, "inci": 5747, "photographer": 5748, "tape": 5749, "logo": 5750, "rings": 5751, "dit": 5752, "----": 5753, "vinyl": 5754, "wc": 5755, "voting": 5756, "seven": 5757, "ambassad": 5758, "dallas": 5759, "tu": 5760, "comment": 5761, "kra": 5762, "bles": 5763, "wag": 5764, "ud": 5765, "audio": 5766, "strike": 5767, "official": 5768, "ots": 5769, "metho": 5770, "tools": 5771, "radi": 5772, "alan": 5773, "hunt": 5774, "watched": 5775, "ake": 5776, "fake": 5777, "drinking": 5778, "merry": 5779, "ml": 5780, "bday": 5781, "rio": 5782, "nike": 5783, "cant": 5784, "repe": 5785, "costu": 5786, "murder": 5787, "akers": 5788, "chers": 5789, "outs": 5790, "beginning": 5791, "sos": 5792, "ades": 5793, "nin": 5794, "notes": 5795, "wrote": 5796, "solo": 5797, "ci": 5798, "lighting": 5799, "urban": 5800, "brexit": 5801, "attend": 5802, "shirts": 5803, "playo": 5804, "actress": 5805, "plic": 5806, "standard": 5807, "quotes": 5808, "parade": 5809, "ancient": 5810, "©": 5811, "turing": 5812, "ree": 5813, "primary": 5814, "flash": 5815, "citiz": 5816, "mates": 5817, "stein": 5818, "zi": 5819, "clinton": 5820, "skin": 5821, "gene": 5822, "hum": 5823, "gar": 5824, "tle": 5825, "yi": 5826, "focu": 5827, "dean": 5828, "plants": 5829, "cyber": 5830, "bu": 5831, "ome": 5832, "hop": 5833, "address": 5834, "tix": 5835, "gifts": 5836, "relationship": 5837, "subscri": 5838, "feed": 5839, "exactly": 5840, "hawks": 5841, "exo": 5842, "stress": 5843, "sn": 5844, "arrested": 5845, "ane": 5846, "software": 5847, "zero": 5848, "theme": 5849, "mumb": 5850, "immigr": 5851, "mia": 5852, "makeup": 5853, "pleasure": 5854, "univers": 5855, "harb": 5856, "engine": 5857, "aper": 5858, "rin": 5859, "bra": 5860, "institute": 5861, "leather": 5862, "alth": 5863, "singing": 5864, "cos": 5865, "ghty": 5866, "meas": 5867, "stic": 5868, "side": 5869, "insurance": 5870, "cot": 5871, "pitch": 5872, "mountains": 5873, "crimin": 5874, "supre": 5875, "valentine": 5876, "ater": 5877, "wouldn": 5878, "scale": 5879, "related": 5880, "regar": 5881, "startup": 5882, "packed": 5883, "mike": 5884, "weekly": 5885, "pts": 5886, "count": 5887, "har": 5888, "gotten": 5889, "mind": 5890, "berlin": 5891, "conditions": 5892, "switch": 5893, "corn": 5894, "save": 5895, "gli": 5896, "emergency": 5897, "tuned": 5898, "stock": 5899, "discussing": 5900, "everybody": 5901, "sday": 5902, "whether": 5903, "wrestling": 5904, "eces": 5905, "gender": 5906, "chen": 5907, "ðŁijĢ": 5908, "madrid": 5909, "marathon": 5910, "egg": 5911, "ier": 5912, "thx": 5913, "asking": 5914, "korea": 5915, "wolf": 5916, "aya": 5917, "gm": 5918, "gau": 5919, "atory": 5920, "vr": 5921, "grass": 5922, "killing": 5923, "bble": 5924, "uro": 5925, "uni": 5926, "eth": 5927, "shore": 5928, "then": 5929, "reale": 5930, "bottom": 5931, "exerc": 5932, "kar": 5933, "ories": 5934, "adri": 5935, "sands": 5936, "sex": 5937, ".'": 5938, "volunteers": 5939, "perform": 5940, "parliam": 5941, "include": 5942, "delighted": 5943, "executive": 5944, "fuel": 5945, "kiss": 5946, "ãħ": 5947, "charge": 5948, "hu": 5949, "cakes": 5950, "vet": 5951, "glu": 5952, "agree": 5953, "prices": 5954, "nau": 5955, "hl": 5956, "gru": 5957, "raj": 5958, "strength": 5959, "bic": 5960, "spending": 5961, "ales": 5962, "aven": 5963, "blast": 5964, ":(": 5965, "yof": 5966, "normal": 5967, "six": 5968, "quick": 5969, "sea": 5970, "daw": 5971, "meets": 5972, "lovers": 5973, "updated": 5974, "potat": 5975, "completed": 5976, "cook": 5977, "opportunities": 5978, "pure": 5979, "organic": 5980, "temper": 5981, "cam": 5982, "avoid": 5983, "parking": 5984, "dubai": 5985, "ando": 5986, "distri": 5987, "toy": 5988, "completely": 5989, "donald": 5990, "trial": 5991, "bass": 5992, "boun": 5993, "background": 5994, "vas": 5995, "marvel": 5996, "lum": 5997, "rus": 5998, "tool": 5999, "commissi": 6000, "throwback": 6001, "finding": 6002, "islam": 6003, "!?": 6004, "stop": 6005, "evil": 6006, "oral": 6007, "residents": 6008, "identi": 6009, "oak": 6010, "ðŁİ¶": 6011, "lil": 6012, "spanish": 6013, "chapter": 6014, "stopped": 6015, "direct": 6016, "hosted": 6017, "picked": 6018, "labour": 6019, "lewis": 6020, "defense": 6021, "à®": 6022, "healthcare": 6023, "whis": 6024, "math": 6025, "peak": 6026, "raised": 6027, "fix": 6028, "bull": 6029, "thir": 6030, "chelsea": 6031, "folk": 6032, "tre": 6033, "candi": 6034, "paul": 6035, "either": 6036, "adam": 6037, "poetry": 6038, "jewelry": 6039, "ðŁ¦": 6040, "pray": 6041, "ا": 6042, "gc": 6043, "oz": 6044, "wishes": 6045, "foreign": 6046, "sung": 6047, "learned": 6048, "ene": 6049, "ning": 6050, "michael": 6051, "illustration": 6052, "legendary": 6053, "wav": 6054, "bau": 6055, "ðŁļ¨": 6056, "calend": 6057, "streets": 6058, "âĨ": 6059, "monster": 6060, "buck": 6061, "gr": 6062, "school": 6063, "bath": 6064, "waste": 6065, "neck": 6066, "hawa": 6067, "beach": 6068, "replac": 6069, "ject": 6070, "oner": 6071, "factory": 6072, "count": 6073, "ðŁĵ¸": 6074, "morgan": 6075, "dering": 6076, "sean": 6077, "stephen": 6078, "dep": 6079, "novel": 6080, "videos": 6081, "ical": 6082, "pressure": 6083, "arsenal": 6084, "expre": 6085, "irs": 6086, "trending": 6087, "ssa": 6088, "flash": 6089, "resear": 6090, "through": 6091, "professor": 6092, "sculp": 6093, "tos": 6094, "gged": 6095, "mma": 6096, "bee": 6097, "ape": 6098, "hunter": 6099, "ami": 6100, "hei": 6101, "plastic": 6102, "bucks": 6103, "universe": 6104, "legen": 6105, "nigeria": 6106, "pleased": 6107, "ris": 6108, "thinks": 6109, "autumn": 6110, "ids": 6111, "dis": 6112, "anthony": 6113, "ðŁı½": 6114, "aked": 6115, "glasses": 6116, "finance": 6117, "zer": 6118, "kas": 6119, "contract": 6120, "numbers": 6121, "shaw": 6122, "partnership": 6123, "til": 6124, "launched": 6125, "sal": 6126, "victoria": 6127, "theater": 6128, "usual": 6129, "names": 6130, "period": 6131, "eliza": 6132, "ith": 6133, "barcel": 6134, "rocks": 6135, "bags": 6136, "mate": 6137, "distribu": 6138, "jon": 6139, "diffic": 6140, "alized": 6141, "curren": 6142, "scored": 6143, "bha": 6144, "dublin": 6145, "rose": 6146, "inted": 6147, "solid": 6148, "behavi": 6149, "walker": 6150, "simply": 6151, "gardens": 6152, "headed": 6153, "ini": 6154, "ohio": 6155, "weap": 6156, "fo": 6157, "glen": 6158, "estate": 6159, "random": 6160, "thunder": 6161, "thru": 6162, "kill": 6163, "jacket": 6164, "iti": 6165, "entertainment": 6166, "thanksgiving": 6167, "ental": 6168, "encoura": 6169, "elo": 6170, "ather": 6171, "tank": 6172, "highlights": 6173, "fting": 6174, "rule": 6175, "models": 6176, "border": 6177, "bjp": 6178, "husband": 6179, "indone": 6180, "kenya": 6181, "bears": 6182, "alo": 6183, "ninten": 6184, "pix": 6185, "stro": 6186, "orders": 6187, "salad": 6188, "roads": 6189, "nor": 6190, "lation": 6191, "sophi": 6192, "ðŁı¼": 6193, "pieces": 6194, "bone": 6195, "mins": 6196, "includes": 6197, "nutr": 6198, "phil": 6199, "sent": 6200, "fundra": 6201, "gain": 6202, "borough": 6203, "nad": 6204, "monday": 6205, "activity": 6206, "items": 6207, "becoming": 6208, "kenne": 6209, "detro": 6210, "cardi": 6211, "guests": 6212, "ux": 6213, "worldwide": 6214, "severe": 6215, "news": 6216, "thankful": 6217, "fiction": 6218, "vege": 6219, "mall": 6220, "sian": 6221, "eral": 6222, "injury": 6223, "lee": 6224, "menu": 6225, "dancing": 6226, "scotti": 6227, "example": 6228, "(#": 6229, "nai": 6230, "studios": 6231, "bai": 6232, "ðŁĴĽ": 6233, "jav": 6234, "diamond": 6235, "vince": 6236, "rick": 6237, "protection": 6238, "lincol": 6239, "champs": 6240, "approach": 6241, "dar": 6242, "mile": 6243, "clouds": 6244, "jeff": 6245, "infin": 6246, "lers": 6247, "ples": 6248, "peace": 6249, "gop": 6250, "âĻ¡": 6251, "techn": 6252, "stra": 6253, "average": 6254, "effort": 6255, "introducing": 6256, "diversity": 6257, "australian": 6258, "amp": 6259, "boost": 6260, "ske": 6261, "patient": 6262, "appreciate": 6263, "icians": 6264, "pur": 6265, "fell": 6266, "woods": 6267, "illustr": 6268, "ðŁĸ": 6269, "agency": 6270, "actions": 6271, "britain": 6272, "underway": 6273, "seattle": 6274, "eland": 6275, "ago": 6276, "fill": 6277, "streaming": 6278, "protest": 6279, "challenges": 6280, "kyo": 6281, "etsy": 6282, "cooking": 6283, "expert": 6284, "russ": 6285, "rainbow": 6286, "commercial": 6287, "spin": 6288, "beats": 6289, "cry": 6290, "valu": 6291, "eli": 6292, "throw": 6293, "grams": 6294, "levels": 6295, "michigan": 6296, "cad": 6297, "adorable": 6298, "constitu": 6299, "ws": 6300, "pub": 6301, "midnight": 6302, "that": 6303, "netfli": 6304, "brazil": 6305, "diego": 6306, "regular": 6307, "joy": 6308, "âĤ¬": 6309, "liqu": 6310, "eastern": 6311, "kni": 6312, "flat": 6313, "np": 6314, "brown": 6315, "wer": 6316, "sey": 6317, "tters": 6318, "acting": 6319, "vanc": 6320, "cycling": 6321, "programme": 6322, "raw": 6323, "complex": 6324, "tattoo": 6325, "throwbackthursday": 6326, "sessions": 6327, "rooms": 6328, "sight": 6329, "species": 6330, "bomb": 6331, "laugh": 6332, "keeps": 6333, "moon": 6334, "officers": 6335, "conver": 6336, "tr": 6337, "hash": 6338, "tack": 6339, "rious": 6340, "adap": 6341, "aj": 6342, "recogn": 6343, "expo": 6344, "sugge": 6345, "confirmed": 6346, "rolling": 6347, "dressing": 6348, "ict": 6349, "friday": 6350, "phones": 6351, "ridge": 6352, "concept": 6353, "roy": 6354, "keys": 6355, "effor": 6356, "cate": 6357, "kne": 6358, "even": 6359, "lay": 6360, "communities": 6361, "mod": 6362, "naz": 6363, "everywhere": 6364, "alab": 6365, "bitcoin": 6366, "banks": 6367, "outdoor": 6368, "federal": 6369, "stores": 6370, "hp": 6371, "cal": 6372, "mely": 6373, "signific": 6374, "bear": 6375, "republic": 6376, "closer": 6377, "allah": 6378, "pick": 6379, "xd": 6380, "palace": 6381, "chill": 6382, "bam": 6383, "erous": 6384, "una": 6385, "allen": 6386, "outstanding": 6387, "olympic": 6388, "supply": 6389, "figu": 6390, "vau": 6391, "lp": 6392, "charlie": 6393, "unes": 6394, ">>>": 6395, "legends": 6396, "icial": 6397, "coast": 6398, "benefit": 6399, "multi": 6400, "fits": 6401, "farmers": 6402, "amount": 6403, "sisters": 6404, "harve": 6405, "honey": 6406, "queen": 6407, "bers": 6408, "plann": 6409, "âŃIJ": 6410, "mu": 6411, "barcelona": 6412, "alber": 6413, "status": 6414, "remain": 6415, "extra": 6416, "candy": 6417, "vious": 6418, "âľĮ": 6419, "ov": 6420, "warriors": 6421, "-->": 6422, "jump": 6423, "amar": 6424, "xmas": 6425, "studies": 6426, "iors": 6427, "kor": 6428, "donate": 6429, "prep": 6430, "fish": 6431, "ima": 6432, "painted": 6433, "admini": 6434, "cosplay": 6435, "sports": 6436, "drops": 6437, "fighter": 6438, "evidence": 6439, "ðŁĴª": 6440, "lake": 6441, "rob": 6442, "cinema": 6443, "profile": 6444, "ñ": 6445, "stands": 6446, "legacy": 6447, "shape": 6448, "roof": 6449, "civil": 6450, "ians": 6451, "syl": 6452, "sham": 6453, "voted": 6454, "retail": 6455, "philli": 6456, "listed": 6457, "duty": 6458, "nb": 6459, "thes": 6460, "fare": 6461, "auction": 6462, "fficial": 6463, "storms": 6464, "dp": 6465, "loun": 6466, "shops": 6467, "aly": 6468, "anime": 6469, "multiple": 6470, "ðŁĺįðŁĺį": 6471, "psycho": 6472, "jean": 6473, "apart": 6474, "candidate": 6475, "ggy": 6476, "conf": 6477, "joseph": 6478, "wick": 6479, "meat": 6480, "frame": 6481, "cl": 6482, "forgot": 6483, "phy": 6484, "fing": 6485, "lied": 6486, "rep": 6487, "seed": 6488, "fall": 6489, "ufc": 6490, "nut": 6491, "lind": 6492, "mode": 6493, "fields": 6494, "ence": 6495, "sley": 6496, "ðŁ¤Ķ": 6497, "chill": 6498, "followed": 6499, "announces": 6500, "corru": 6501, "trophy": 6502, "themselves": 6503, "acle": 6504, "aldu": 6505, "kong": 6506, "lon": 6507, "sv": 6508, "broke": 6509, "anderson": 6510, "tai": 6511, "story": 6512, "temporary": 6513, "activities": 6514, "kati": 6515, "ariz": 6516, "crystal": 6517, "spoke": 6518, "extremely": 6519, "trading": 6520, "ðŁĴļ": 6521, "ü": 6522, "inch": 6523, "edin": 6524, "outfit": 6525, "equip": 6526, "madi": 6527, "formed": 6528, "beef": 6529, "pop": 6530, "tiger": 6531, "thisday": 6532, "tired": 6533, "neighb": 6534, "retro": 6535, "isa": 6536, "unt": 6537, "tas": 6538, "kansas": 6539, "dest": 6540, "seconds": 6541, "tay": 6542, "hurric": 6543, "ou": 6544, "galaxy": 6545, "daddy": 6546, "brow": 6547, "burger": 6548, "enced": 6549, "desk": 6550, "accur": 6551, "secretary": 6552, "elite": 6553, "kab": 6554, "chin": 6555, "tourism": 6556, "buddy": 6557, "icide": 6558, "dressed": 6559, "ud": 6560, "vacation": 6561, "cheers": 6562, "comfor": 6563, "characters": 6564, "jet": 6565, "buying": 6566, "lins": 6567, "nap": 6568, "realestate": 6569, "lie": 6570, "afc": 6571, "iii": 6572, "fame": 6573, "nr": 6574, "bat": 6575, "agent": 6576, "makers": 6577, "âĢ¼": 6578, "sector": 6579, "opti": 6580, "leon": 6581, "diet": 6582, "prayer": 6583, "hip": 6584, "mir": 6585, "lex": 6586, "bry": 6587, "ana": 6588, "passing": 6589, "wen": 6590, "recovery": 6591, "aki": 6592, "popul": 6593, "resort": 6594, "maria": 6595, "stuck": 6596, "reads": 6597, "tier": 6598, "perfec": 6599, "netflix": 6600, "poo": 6601, "champ": 6602, "oc": 6603, "reduce": 6604, "wered": 6605, "comments": 6606, "claim": 6607, "accident": 6608, "sag": 6609, "hack": 6610, "salt": 6611, "kinda": 6612, "killer": 6613, "ios": 6614, "zy": 6615, "exchange": 6616, "lecture": 6617, "enger": 6618, "icking": 6619, "tau": 6620, "reveals": 6621, "prison": 6622, "zom": 6623, "ghan": 6624, "ul": 6625, "journal": 6626, "iot": 6627, "trin": 6628, "jona": 6629, "governor": 6630, "cape": 6631, "quarter": 6632, "spective": 6633, "impressive": 6634, "babies": 6635, "tx": 6636, "mill": 6637, "oy": 6638, "harri": 6639, "joint": 6640, "sue": 6641, "collaboration": 6642, "trend": 6643, "revolution": 6644, "renew": 6645, "alumni": 6646, "gett": 6647, "shell": 6648, "sunday": 6649, "entu": 6650, "nic": 6651, "donaldtrump": 6652, "blockchain": 6653, "pacific": 6654, "explains": 6655, "spy": 6656, "advoc": 6657, "paradi": 6658, "tof": 6659, "starring": 6660, "pav": 6661, "feed": 6662, "brac": 6663, "smoke": 6664, "hamp": 6665, "yam": 6666, "tokyo": 6667, "simon": 6668, "dh": 6669, "effici": 6670, "physical": 6671, "nj": 6672, "elli": 6673, "slow": 6674, "graduate": 6675, "americans": 6676, "tify": 6677, "fred": 6678, "apore": 6679, "finds": 6680, "robin": 6681, "wet": 6682, "notice": 6683, "semi": 6684, "unve": 6685, "kom": 6686, "pilot": 6687, "screening": 6688, "daily": 6689, "ðŁĴĹ": 6690, "royal": 6691, "spa": 6692, "votes": 6693, "nag": 6694, "whate": 6695, "attending": 6696, "experim": 6697, "addition": 6698, "kate": 6699, "stol": 6700, "mali": 6701, "foot": 6702, "christ": 6703, "chan": 6704, "dee": 6705, "licen": 6706, "global": 6707, "moore": 6708, "tia": 6709, "brigh": 6710, "mystery": 6711, "yay": 6712, "âĿ¤ï¸ıâĿ¤ï¸ı": 6713, "creati": 6714, "mechan": 6715, "clock": 6716, "dic": 6717, "âĢĶ": 6718, "pper": 6719, "alph": 6720, "throughout": 6721, "allow": 6722, "resources": 6723, "selection": 6724, "hamil": 6725, "bbq": 6726, "aaaa": 6727, "virginia": 6728, "disney": 6729, "eng": 6730, "sored": 6731, "drinks": 6732, "fancy": 6733, "consider": 6734, "enda": 6735, "jane": 6736, "handmade": 6737, "dul": 6738, "ontari": 6739, "ius": 6740, "sville": 6741, "colorado": 6742, "whatever": 6743, "wheel": 6744, "promise": 6745, "never": 6746, "designs": 6747, "ably": 6748, "sexual": 6749, "vancou": 6750, "ati": 6751, "convention": 6752, "cultural": 6753, "singapore": 6754, "promo": 6755, "loaded": 6756, "glasgo": 6757, "ppl": 6758, "noo": 6759, "kee": 6760, "stem": 6761, "mention": 6762, "ido": 6763, "cruise": 6764, "riding": 6765, "becomes": 6766, "bey": 6767, "âļ½ï¸ı": 6768, "twin": 6769, "dedicated": 6770, "nash": 6771, "desi": 6772, "workout": 6773, "jenni": 6774, "iv": 6775, "groups": 6776, "relax": 6777, "phoeni": 6778, "lift": 6779, "mixed": 6780, "mck": 6781, "pc": 6782, "must": 6783, "metro": 6784, "cies": 6785, "yar": 6786, "aim": 6787, "anger": 6788, "ie": 6789, "recy": 6790, "married": 6791, "dropped": 6792, "engag": 6793, "lest": 6794, "ambassador": 6795, "oph": 6796, "des": 6797, "wick": 6798, "assistant": 6799, "natur": 6800, "fail": 6801, "ltd": 6802, "short": 6803, "kap": 6804, "shaw": 6805, "bigger": 6806, "remains": 6807, "critical": 6808, "survey": 6809, "coverage": 6810, "erson": 6811, "wind": 6812, "nb": 6813, "billy": 6814, "letes": 6815, "acts": 6816, "jimmy": 6817, "atlan": 6818, "aland": 6819, "tc": 6820, "importance": 6821, "damage": 6822, "fg": 6823, "storage": 6824, "twt": 6825, "bond": 6826, "balance": 6827, "crying": 6828, "puppy": 6829, "vote": 6830, "push": 6831, "ðŁĴľ": 6832, "poly": 6833, "mel": 6834, "london": 6835, "terrori": 6836, "effective": 6837, "corporate": 6838, "atlanta": 6839, "jaco": 6840, "nasa": 6841, "greek": 6842, "senate": 6843, "ish": 6844, "eva": 6845, "intelligence": 6846, "efforts": 6847, "alco": 6848, "kun": 6849, "hall": 6850, "diag": 6851, "claims": 6852, "first": 6853, "hb": 6854, "bae": 6855, "vul": 6856, "pull": 6857, "°": 6858, "separ": 6859, "speed": 6860, "victi": 6861, "onthisday": 6862, "audience": 6863, "rates": 6864, "teach": 6865, "filming": 6866, "bush": 6867, "song": 6868, "yum": 6869, "brun": 6870, "raine": 6871, "awa": 6872, "parks": 6873, "ðĿ": 6874, "rabb": 6875, "rach": 6876, "raid": 6877, "reached": 6878, "rail": 6879, "moves": 6880, "selected": 6881, "fri": 6882, "raising": 6883, "omy": 6884, "stones": 6885, "suk": 6886, "francisco": 6887, "cases": 6888, "capit": 6889, "confu": 6890, "wtf": 6891, "poke": 6892, "equipment": 6893, "greg": 6894, "essential": 6895, "offering": 6896, "nex": 6897, "pies": 6898, "bec": 6899, "creation": 6900, "chairman": 6901, "crown": 6902, "wal": 6903, "johnny": 6904, "shift": 6905, "neck": 6906, "bang": 6907, "bird": 6908, "ðŁĺı": 6909, "duck": 6910, "reserve": 6911, "depu": 6912, "masters": 6913, "overall": 6914, "notic": 6915, "juice": 6916, "sneak": 6917, "cheer": 6918, "classes": 6919, "eagles": 6920, "nca": 6921, "carpet": 6922, "civil": 6923, "coaches": 6924, "harris": 6925, "ups": 6926, "balls": 6927, "decor": 6928, "martin": 6929, "ros": 6930, "vice": 6931, "announcement": 6932, "whose": 6933, "tigers": 6934, "stered": 6935, "cts": 6936, "dram": 6937, "steel": 6938, "young": 6939, "install": 6940, "suppo": 6941, "recording": 6942, "deck": 6943, "seats": 6944, "lder": 6945, "angle": 6946, "bot": 6947, "styles": 6948, "elections": 6949, "fortun": 6950, "nab": 6951, "butter": 6952, "arian": 6953, "kash": 6954, "inner": 6955, "oured": 6956, "beast": 6957, "wei": 6958, "iconic": 6959, "experts": 6960, "necess": 6961, "beng": 6962, "james": 6963, "lia": 6964, "greece": 6965, "ðŁĵ·": 6966, "ðŁĺģ": 6967, "goodbye": 6968, "mitch": 6969, "twice": 6970, "mumbai": 6971, "steam": 6972, "rush": 6973, "medal": 6974, "nett": 6975, "fashion": 6976, "tar": 6977, "rs": 6978, "saving": 6979, "ricul": 6980, "lm": 6981, "sleeping": 6982, "brooklyn": 6983, "miss": 6984, "sending": 6985, "discovered": 6986, "sphere": 6987, "oftheday": 6988, "kicks": 6989, "missions": 6990, "wright": 6991, "ern": 6992, "ghtly": 6993, "ious": 6994, "melbourne": 6995, "startu": 6996, "moved": 6997, "carry": 6998, "dak": 6999, "agues": 7000, "belgi": 7001, "ema": 7002, "wayne": 7003, "dot": 7004, "erie": 7005, "pel": 7006, "itunes": 7007, "matthew": 7008, "nobody": 7009, "estab": 7010, "calm": 7011, "winds": 7012, "luc": 7013, "prepare": 7014, "trends": 7015, "exercise": 7016, "advant": 7017, "ðŁĴ¯": 7018, "athletics": 7019, "apps": 7020, "ctions": 7021, "advance": 7022, "launches": 7023, "little": 7024, "realdonaldtrump": 7025, "elizabeth": 7026, "carolina": 7027, "hub": 7028, "hidden": 7029, "nw": 7030, "user": 7031, "poll": 7032, "greater": 7033, "most": 7034, "fed": 7035, "pat": 7036, "lifestyle": 7037, "sati": 7038, "scores": 7039, "marriage": 7040, "lr": 7041, "avenue": 7042, "deserve": 7043, "rif": 7044, "ðŁĹ": 7045, "watch": 7046, "championships": 7047, "gray": 7048, "enni": 7049, "cotton": 7050, "gom": 7051, "where": 7052, "package": 7053, "sum": 7054, "absolu": 7055, "newly": 7056, "foods": 7057, "tyler": 7058, "assembly": 7059, "muslim": 7060, "bank": 7061, "rememb": 7062, "options": 7063, "producer": 7064, "lando": 7065, "funds": 7066, "upper": 7067, "shadow": 7068, "progre": 7069, "cop": 7070, "inge": 7071, "legs": 7072, "detroit": 7073, "hillary": 7074, "jose": 7075, "giants": 7076, "soup": 7077, "sustainable": 7078, "tus": 7079, "clothes": 7080, "rocking": 7081, "nz": 7082, "minne": 7083, "materi": 7084, "bruce": 7085, "eart": 7086, "casting": 7087, "independent": 7088, "thousands": 7089, "tah": 7090, "decl": 7091, "veterans": 7092, "lions": 7093, "wrap": 7094, "âĢ¦": 7095, "dess": 7096, "bling": 7097, "stine": 7098, "eggs": 7099, "oon": 7100, "closing": 7101, "zay": 7102, "att": 7103, "bacon": 7104, "fail": 7105, "arizona": 7106, "depre": 7107, "ghost": 7108, "newsp": 7109, "wers": 7110, "vip": 7111, "liked": 7112, "ident": 7113, "volunteer": 7114, "adult": 7115, "pupp": 7116, "circle": 7117, "material": 7118, "degree": 7119, "grown": 7120, "boom": 7121, "calendar": 7122, "sur": 7123, "viewing": 7124, "athletes": 7125, "chand": 7126, "rell": 7127, "asian": 7128, "entr": 7129, "volley": 7130, "victims": 7131, "body": 7132, "mama": 7133, "transfer": 7134, "geek": 7135, "indic": 7136, "saved": 7137, "mai": 7138, "gent": 7139, "its": 7140, "lounge": 7141, "kol": 7142, "theory": 7143, "situation": 7144, "islands": 7145, "arth": 7146, "zoo": 7147, "flood": 7148, "viously": 7149, "showed": 7150, "parliament": 7151, "chev": 7152, "eline": 7153, "attrac": 7154, "abad": 7155, "tail": 7156, "hrs": 7157, "lus": 7158, "portu": 7159, "gory": 7160, "provides": 7161, "toys": 7162, "death": 7163, "infe": 7164, "ance": 7165, "gle": 7166, "liam": 7167, "lover": 7168, "hud": 7169, "dvd": 7170, "revealed": 7171, "gw": 7172, "rement": 7173, "cathe": 7174, "lying": 7175, "radio": 7176, "derby": 7177, "stors": 7178, "chemi": 7179, "hospit": 7180, "⾨": 7181, "':": 7182, "ilove": 7183, "lemon": 7184, "republic": 7185, "sni": 7186, "ness": 7187, "door": 7188, "reaction": 7189, "pregn": 7190, "flav": 7191, "scholar": 7192, "spotify": 7193, "isation": 7194, "visual": 7195, "aware": 7196, "sponsored": 7197, "joke": 7198, "lessons": 7199, "legis": 7200, "lock": 7201, "simil": 7202, "ðŁĺĭ": 7203, "kind": 7204, "lay": 7205, "mah": 7206, "hoping": 7207, "vancouver": 7208, "aser": 7209, "cleaning": 7210, "gala": 7211, "threat": 7212, "lap": 7213, "ache": 7214, "romance": 7215, "expen": 7216, "repost": 7217, "zam": 7218, "epi": 7219, "mirror": 7220, "oak": 7221, "adul": 7222, "batman": 7223, "slu": 7224, "lc": 7225, "viewed": 7226, "reviews": 7227, "dates": 7228, "indonesia": 7229, "activi": 7230, "offen": 7231, "leaf": 7232, "isi": 7233, "agricul": 7234, "costume": 7235, "sites": 7236, "spiritu": 7237, "appearance": 7238, "iry": 7239, "stair": 7240, "application": 7241, "spectac": 7242, "icity": 7243, "skies": 7244, "handle": 7245, "punk": 7246, "paradise": 7247, "tn": 7248, "deal": 7249, "providing": 7250, "doc": 7251, "receiving": 7252, "brew": 7253, "microsoft": 7254, "ö": 7255, "ferr": 7256, "metro": 7257, "thail": 7258, "yum": 7259, "carter": 7260, "á": 7261, "gentle": 7262, "breaks": 7263, "cooper": 7264, "showcase": 7265, "cutting": 7266, "egypt": 7267, "baby": 7268, "seminar": 7269, "glori": 7270, "sson": 7271, "fave": 7272, "rehear": 7273, "lotte": 7274, "lady": 7275, "alas": 7276, "prep": 7277, "delivered": 7278, "nuclear": 7279, "iro": 7280, "engagement": 7281, "atta": 7282, "conven": 7283, "zan": 7284, "glory": 7285, "holds": 7286, "businesses": 7287, "strange": 7288, "sche": 7289, "itself": 7290, "grad": 7291, "markets": 7292, "falling": 7293, "stats": 7294, "geon": 7295, "budd": 7296, "lis": 7297, "sheet": 7298, "thisi": 7299, "colo": 7300, "desert": 7301, "registration": 7302, "ign": 7303, "explain": 7304, "interior": 7305, "laws": 7306, "writers": 7307, "springs": 7308, "kr": 7309, "fried": 7310, "bloom": 7311, "infra": 7312, "ao": 7313, "cred": 7314, "past": 7315, "lineup": 7316, "boo": 7317, "brea": 7318, "boots": 7319, "celebrity": 7320, "attacks": 7321, "brook": 7322, "eves": 7323, "excu": 7324, "cherry": 7325, "oop": 7326, "fascin": 7327, "boyfriend": 7328, "seas": 7329, "nine": 7330, "effects": 7331, "powered": 7332, "kha": 7333, "ðŁĺĢ": 7334, "shout": 7335, "condition": 7336, "ij": 7337, "hero": 7338, "enterpri": 7339, "winter": 7340, "applications": 7341, "shoe": 7342, "gel": 7343, "battle": 7344, "programs": 7345, "wart": 7346, "ðŁĴ¥": 7347, "rap": 7348, "hol": 7349, "dangerous": 7350, "dia": 7351, "counter": 7352, "rics": 7353, "ior": 7354, "knight": 7355, "coat": 7356, "emotional": 7357, "atures": 7358, "das": 7359, "wheel": 7360, "forecast": 7361, "transport": 7362, "glasgow": 7363, "kingdom": 7364, "preparing": 7365, "immedi": 7366, "ffin": 7367, "awarded": 7368, "printing": 7369, "roman": 7370, "fighters": 7371, "anymore": 7372, "belt": 7373, "pine": 7374, "wine": 7375, "xi": 7376, "employees": 7377, "logies": 7378, "alled": 7379, "demo": 7380, "birthday": 7381, "angeles": 7382, "log": 7383, "drivers": 7384, "necklace": 7385, "kath": 7386, "sit": 7387, "athlete": 7388, "efs": 7389, "sburg": 7390, "purpose": 7391, "resistance": 7392, "releases": 7393, "tis": 7394, "various": 7395, "deliver": 7396, "chal": 7397, "sanc": 7398, "oppo": 7399, "craw": 7400, "neuro": 7401, "dra": 7402, "supporters": 7403, "snap": 7404, "difficult": 7405, "swear": 7406, "logist": 7407, "path": 7408, "attempt": 7409, "à¥": 7410, "swimming": 7411, "steve": 7412, "hurt": 7413, "included": 7414, "bap": 7415, "ware": 7416, "ðŁĴĭ": 7417, "enders": 7418, "jake": 7419, "leeds": 7420, "climb": 7421, "lb": 7422, "imple": 7423, "lisa": 7424, "clothing": 7425, "ðŁĺİ": 7426, "dt": 7427, "compla": 7428, "swing": 7429, "straw": 7430, "vals": 7431, "kle": 7432, "users": 7433, "storm": 7434, "cuts": 7435, "ontario": 7436, "pan": 7437, "handsome": 7438, "iow": 7439, "argu": 7440, "checking": 7441, "scottish": 7442, "Ķï¸ı": 7443, "sier": 7444, "emma": 7445, "pod": 7446, "pattern": 7447, "desh": 7448, "enh": 7449, "edward": 7450, "ting": 7451, "kh": 7452, "half": 7453, "lincoln": 7454, "mother": 7455, "alleg": 7456, "rc": 7457, "volleyball": 7458, "dn": 7459, "gay": 7460, "ally": 7461, "leton": 7462, "grove": 7463, "loud": 7464, "advanced": 7465, "respec": 7466, "client": 7467, "supreme": 7468, "thailand": 7469, "how": 7470, "gig": 7471, "toi": 7472, "dot": 7473, "dollar": 7474, "ðŁijĩ": 7475, "pit": 7476, "rb": 7477, "hn": 7478, "produced": 7479, "ggers": 7480, "âĨĴ": 7481, "mlb": 7482, "canvas": 7483, "fineart": 7484, "usd": 7485, "inthe": 7486, "pson": 7487, "actual": 7488, "sl": 7489, "tb": 7490, "ipad": 7491, "ensure": 7492, "umb": 7493, "wd": 7494, "ska": 7495, "mars": 7496, "kend": 7497, "feli": 7498, "thing": 7499, "countdown": 7500, "absolute": 7501, "rout": 7502, "dral": 7503, "py": 7504, "injured": 7505, "mint": 7506, "hunting": 7507, "mmer": 7508, "sage": 7509, "ligh": 7510, "acity": 7511, "expan": 7512, "murray": 7513, "aro": 7514, "secure": 7515, "fourth": 7516, "eagle": 7517, "relief": 7518, "stakes": 7519, "industrial": 7520, "clark": 7521, "understanding": 7522, "seem": 7523, "plenty": 7524, "silver": 7525, "clau": 7526, "threat": 7527, "sail": 7528, "produce": 7529, "abstr": 7530, "isis": 7531, "br": 7532, "engers": 7533, "worry": 7534, "bieber": 7535, "sj": 7536, "justin": 7537, "realize": 7538, "kyle": 7539, "espn": 7540, "filter": 7541, "sch": 7542, "types": 7543, "gamedev": 7544, "ding": 7545, "twitter": 7546, "soldiers": 7547, "pom": 7548, "carbon": 7549, "yards": 7550, "childhood": 7551, "ried": 7552, "kel": 7553, "eleph": 7554, "tons": 7555, "keynote": 7556, "quiet": 7557, "wire": 7558, "posting": 7559, "issa": 7560, "representing": 7561, "backs": 7562, "alexander": 7563, "celebrates": 7564, "taining": 7565, "||": 7566, "chor": 7567, "escape": 7568, "peek": 7569, "tives": 7570, "field": 7571, "ssie": 7572, "impac": 7573, "sponsor": 7574, "rc": 7575, "wedd": 7576, "cannab": 7577, "sides": 7578, "tracks": 7579, "compar": 7580, "contrac": 7581, "technical": 7582, "bible": 7583, "exploring": 7584, "share": 7585, "trav": 7586, "nate": 7587, "illo": 7588, "scru": 7589, "mingham": 7590, "guns": 7591, "ofthe": 7592, "shame": 7593, "sees": 7594, "catho": 7595, "access": 7596, "cel": 7597, "reported": 7598, "»": 7599, "mario": 7600, "pad": 7601, "hopefully": 7602, "ouse": 7603, "yon": 7604, "disappo": 7605, "olo": 7606, "pitt": 7607, "pac": 7608, "gap": 7609, "crush": 7610, "sg": 7611, "kle": 7612, "gem": 7613, "empire": 7614, "dirty": 7615, "ais": 7616, "aviation": 7617, "zealand": 7618, "facing": 7619, "highway": 7620, "danny": 7621, "spider": 7622, "otta": 7623, "ðŁĺĦ": 7624, "wy": 7625, "colours": 7626, "infl": 7627, "costs": 7628, "olympics": 7629, "aus": 7630, "hm": 7631, "howard": 7632, "passes": 7633, "lauren": 7634, "mush": 7635, "opin": 7636, "rho": 7637, "discount": 7638, "operation": 7639, "emily": 7640, "mmm": 7641, "chamber": 7642, "dil": 7643, "toyo": 7644, "ship": 7645, "samu": 7646, "pictured": 7647, "unic": 7648, "pol": 7649, "keeper": 7650, "cartoon": 7651, "sten": 7652, "ignor": 7653, "nations": 7654, "nl": 7655, "tasting": 7656, "detail": 7657, "officials": 7658, "motor": 7659, "francis": 7660, "editor": 7661, "ðŁijĩ": 7662, "pets": 7663, "rangers": 7664, "tg": 7665, "rn": 7666, "wri": 7667, "nichol": 7668, "ise": 7669, "spots": 7670, "anie": 7671, "check": 7672, "triple": 7673, "kumar": 7674, "speakers": 7675, "icing": 7676, "prepared": 7677, "abuse": 7678, "friendship": 7679, "month": 7680, "swim": 7681, "aire": 7682, "scent": 7683, "hamilton": 7684, "indian": 7685, "jes": 7686, "yummy": 7687, "tears": 7688, "dawn": 7689, "ized": 7690, "worlds": 7691, "ðŁķ": 7692, "billi": 7693, "stone": 7694, "nhs": 7695, "basic": 7696, "por": 7697, "stle": 7698, "iron": 7699, "older": 7700, "clevel": 7701, "eing": 7702, "ðŁĺįðŁĺįðŁĺį": 7703, "prints": 7704, "firm": 7705, "aircraft": 7706, "finest": 7707, "develop": 7708, "aaron": 7709, "tz": 7710, "graham": 7711, "owners": 7712, "foli": 7713, "lesson": 7714, "ques": 7715, "babe": 7716, "craft": 7717, "phen": 7718, "jun": 7719, "birmingham": 7720, "vine": 7721, "ller": 7722, "ian": 7723, "fineartamerica": 7724, "evolu": 7725, "stab": 7726, "imper": 7727, "ward": 7728, "comic": 7729, "wiz": 7730, "invited": 7731, "duke": 7732, "match": 7733, "ports": 7734, "roger": 7735, "diagno": 7736, "kept": 7737, "test": 7738, "visu": 7739, "rhy": 7740, "soc": 7741, "tox": 7742, "baker": 7743, "surface": 7744, "covers": 7745, "mans": 7746, "bits": 7747, "xbox": 7748, "ffle": 7749, "nan": 7750, "gard": 7751, "hart": 7752, "waters": 7753, "villa": 7754, "retro": 7755, "lightning": 7756, "catholic": 7757, "democracy": 7758, "neighbor": 7759, "penn": 7760, "cran": 7761, "jonathan": 7762, "laura": 7763, "vibes": 7764, "sub": 7765, "coaching": 7766, "clearly": 7767, "ukraine": 7768, "brave": 7769, "commitment": 7770, "tall": 7771, "mart": 7772, "rap": 7773, "modi": 7774, "scott": 7775, "bros": 7776, "shower": 7777, "ðŁı¾": 7778, "âĺºï¸ı": 7779, "cousin": 7780, "approach": 7781, "bre": 7782, "compos": 7783, "hilari": 7784, "philly": 7785, "gad": 7786, "quickly": 7787, "rian": 7788, "tm": 7789, "virtual": 7790, "houses": 7791, "kt": 7792, "phoenix": 7793, "wire": 7794, "ffy": 7795, "bunch": 7796, "ancing": 7797, "tale": 7798, "snapchat": 7799, "starter": 7800, "ht": 7801, "kicking": 7802, "apart": 7803, "thy": 7804, ")!": 7805, "blogger": 7806, "itz": 7807, "comfort": 7808, "angels": 7809, "wash": 7810, "\":": 7811, "argent": 7812, "request": 7813, "honest": 7814, "mighty": 7815, "bobby": 7816, "kg": 7817, "rol": 7818, "thouse": 7819, "expo": 7820, "hc": 7821, "tables": 7822, "magical": 7823, "posts": 7824, "dem": 7825, "nw": 7826, "orlando": 7827, "aber": 7828, "***": 7829, "ðŁĺľ": 7830, "environmental": 7831, "transformation": 7832, "mile": 7833, "wic": 7834, "hiring": 7835, "maine": 7836, "boar": 7837, "rying": 7838, "tis": 7839, "niture": 7840, "tweeted": 7841, "antonio": 7842, "opinion": 7843, "finale": 7844, "diy": 7845, "fis": 7846, "thin": 7847, "trouble": 7848, "lego": 7849, "files": 7850, "quart": 7851, "spa": 7852, "currency": 7853, "climate": 7854, "fanart": 7855, "railway": 7856, "space": 7857, "bands": 7858, "daniel": 7859, "motion": 7860, "leng": 7861, "holder": 7862, "occu": 7863, "marie": 7864, "cathedral": 7865, "buzz": 7866, "bies": 7867, "nascar": 7868, "bmw": 7869, "battery": 7870, "charlotte": 7871, "doctor": 7872, "zzle": 7873, "seven": 7874, "insan": 7875, "ddy": 7876, "sten": 7877, "labor": 7878, "thrilled": 7879, "seren": 7880, "documentary": 7881, "waves": 7882, "certain": 7883, "candid": 7884, "allowed": 7885, "nintendo": 7886, "starwars": 7887, "tap": 7888, "homemade": 7889, "dles": 7890, "thering": 7891, "bree": 7892, "empty": 7893, "piano": 7894, "positi": 7895, "country": 7896, "pork": 7897, "puts": 7898, "perry": 7899, "matic": 7900, "spotlight": 7901, "tist": 7902, "orities": 7903, "wealth": 7904, "cp": 7905, "barbar": 7906, "committed": 7907, "assau": 7908, "profit": 7909, "eight": 7910, "hul": 7911, "finishing": 7912, "runner": 7913, "sso": 7914, "inspec": 7915, "charged": 7916, "christop": 7917, "losing": 7918, "coal": 7919, "hoo": 7920, "elev": 7921, "dele": 7922, "moham": 7923, "donation": 7924, "cable": 7925, "clinic": 7926, "jin": 7927, "managed": 7928, "tering": 7929, "â¬": 7930, "urban": 7931, "deputy": 7932, "bber": 7933, "burn": 7934, "academic": 7935, "ott": 7936, "stake": 7937, "iter": 7938, "stown": 7939, "acker": 7940, "adventures": 7941, "adams": 7942, "greg": 7943, "prom": 7944, "vol": 7945, "acqu": 7946, "congre": 7947, "paint": 7948, "citizens": 7949, "call": 7950, "afford": 7951, "vc": 7952, "asks": 7953, "thetic": 7954, "independence": 7955, "âĽ": 7956, "hitting": 7957, "blon": 7958, "future": 7959, "âı": 7960, "inno": 7961, "gene": 7962, "boards": 7963, "distance": 7964, "set": 7965, "remem": 7966, "thal": 7967, "prevent": 7968, "lang": 7969, "objec": 7970, "susp": 7971, "matt": 7972, "induc": 7973, "boro": 7974, "pione": 7975, "redi": 7976, "virtu": 7977, "printed": 7978, "scope": 7979, "shark": 7980, "succe": 7981, "astron": 7982, "illegal": 7983, "jag": 7984, "cting": 7985, "inee": 7986, "ato": 7987, "robin": 7988, "nutrition": 7989, "bf": 7990, "dutch": 7991, "bn": 7992, "furniture": 7993, "forgotten": 7994, "atar": 7995, "rup": 7996, "hyper": 7997, "branch": 7998, "communication": 7999, "degrees": 8000, "onia": 8001, "uncle": 8002, "promote": 8003, "orche": 8004, "wii": 8005, "js": 8006, "button": 8007, "major": 8008, "cbs": 8009, "bristol": 8010, "premium": 8011, "ordinary": 8012, "edit": 8013, "mg": 8014, "weed": 8015, "steven": 8016, ":'": 8017, "gus": 8018, "tes": 8019, "captured": 8020, "drugs": 8021, "dow": 8022, "writes": 8023, "bishop": 8024, "wheels": 8025, "alization": 8026, "discovery": 8027, "wr": 8028, "rachel": 8029, "neil": 8030, "hydr": 8031, "cutest": 8032, "entrepreneur": 8033, "korean": 8034, "oregon": 8035, "ulty": 8036, "perfectly": 8037, "supported": 8038, "historical": 8039, "twins": 8040, "elly": 8041, "wel": 8042, "devil": 8043, "income": 8044, "scientists": 8045, "deleg": 8046, "hen": 8047, "oni": 8048, "iced": 8049, "gio": 8050, "curry": 8051, "reveal": 8052, "eg": 8053, "buffalo": 8054, "nol": 8055, "opera": 8056, "cameron": 8057, "hahahaha": 8058, "jab": 8059, "graduation": 8060, "craig": 8061, "ral": 8062, "if": 8063, "organization": 8064, "lege": 8065, "gang": 8066, "sud": 8067, "edinburgh": 8068, "lack": 8069, "flies": 8070, "gate": 8071, "thrones": 8072, "qb": 8073, "thereal": 8074, "eleg": 8075, "ppin": 8076, "cles": 8077, "jamie": 8078, "tnam": 8079, "crypto": 8080, "oul": 8081, "pages": 8082, "ase": 8083, "roots": 8084, "stupid": 8085, "adid": 8086, "boot": 8087, "protein": 8088, "sap": 8089, "sium": 8090, "sus": 8091, "endor": 8092, "function": 8093, "dont": 8094, "enna": 8095, "chy": 8096, "sque": 8097, "worker": 8098, "mtv": 8099, "ea": 8100, "kan": 8101, "ðŁĴļ": 8102, "mus": 8103, "profession": 8104, "tto": 8105, "operations": 8106, "allo": 8107, "ctor": 8108, "invite": 8109, "scand": 8110, "outh": 8111, "zim": 8112, "links": 8113, "clients": 8114, "samsung": 8115, "discusses": 8116, "nell": 8117, "ultra": 8118, "somewhere": 8119, "stewart": 8120, "inet": 8121, "dez": 8122, "bout": 8123, "factor": 8124, "tian": 8125, "trans": 8126, "jeremy": 8127, "db": 8128, "ðŁĩ¬": 8129, "orn": 8130, "developing": 8131, "spol": 8132, "cooper": 8133, "mau": 8134, "remembering": 8135, "trek": 8136, "family": 8137, "seniors": 8138, "foster": 8139, "attended": 8140, "wing": 8141, "transform": 8142, "elementary": 8143, "horiz": 8144, "listing": 8145, "malaysia": 8146, "itch": 8147, "warrior": 8148, "philippines": 8149, "russell": 8150, "mend": 8151, "initiative": 8152, "creep": 8153, "tops": 8154, "briti": 8155, "aur": 8156, "sharp": 8157, "advertising": 8158, "ugly": 8159, "achiev": 8160, "materials": 8161, "bug": 8162, "device": 8163, "bonus": 8164, "facility": 8165, "cole": 8166, "nhl": 8167, "yas": 8168, "planned": 8169, "pole": 8170, "excellence": 8171, "trick": 8172, "confl": 8173, "rp": 8174, "achieve": 8175, "loan": 8176, "swag": 8177, "jessica": 8178, "howe": 8179, "pour": 8180, "scu": 8181, "zoo": 8182, "rated": 8183, "dresses": 8184, "rebel": 8185, "mexican": 8186, "coordin": 8187, "mess": 8188, "atlantic": 8189, "tl": 8190, "oscar": 8191, "walks": 8192, "pharmac": 8193, "investigation": 8194, "...#": 8195, "cci": 8196, "easily": 8197, "mondaymotivation": 8198, "yment": 8199, "auti": 8200, "forced": 8201, "armed": 8202, "colleagues": 8203, "papers": 8204, "proper": 8205, "shake": 8206, "buc": 8207, "lean": 8208, "exhibit": 8209, "evement": 8210, "cott": 8211, "biz": 8212, "sper": 8213, "kent": 8214, "swan": 8215, "/@": 8216, "girlfriend": 8217, "hawk": 8218, "âĺĢï¸ı": 8219, "mono": 8220, "ðŁĴĽ": 8221, "statue": 8222, "ðŁĺ³": 8223, "ras": 8224, "teeth": 8225, "precious": 8226, "tile": 8227, "pam": 8228, "swift": 8229, "vali": 8230, "nose": 8231, "drunk": 8232, "experiences": 8233, "comeback": 8234, "genius": 8235, "worse": 8236, "shef": 8237, "rad": 8238, "edit": 8239, "honour": 8240, "auspol": 8241, "larry": 8242, "hire": 8243, "gordon": 8244, "achievement": 8245, "........": 8246, "suicide": 8247, "alternative": 8248, "sup": 8249, "surroun": 8250, "shake": 8251, "keith": 8252, "pepper": 8253, "turk": 8254, "criminal": 8255, "beck": 8256, "sum": 8257, "walls": 8258, "cnn": 8259, "antic": 8260, "offe": 8261, "colli": 8262, "wines": 8263, "highlight": 8264, "hawaii": 8265, "embar": 8266, "lfc": 8267, "ðŁĩ®": 8268, "mv": 8269, ">>": 8270, "atmo": 8271, "word": 8272, "carl": 8273, "shoutout": 8274, "brewing": 8275, "ìĿ": 8276, "dof": 8277, "sic": 8278, "hottest": 8279, "colon": 8280, "hhh": 8281, "shut": 8282, "lowing": 8283, "volume": 8284, "apartment": 8285, "agreement": 8286, "destro": 8287, "wee": 8288, "religious": 8289, "iowa": 8290, "rod": 8291, "landing": 8292, "represent": 8293, "ðŁĵ·:": 8294, "las": 8295, "usually": 8296, "hl": 8297, "cac": 8298, "salv": 8299, "along": 8300, "laughing": 8301, "beans": 8302, "reminds": 8303, "phase": 8304, "somebody": 8305, "mask": 8306, "ranked": 8307, "destroy": 8308, "sci": 8309, "âĢ¼ï¸ı": 8310, "gabri": 8311, "leo": 8312, "roa": 8313, "failed": 8314, "sil": 8315, "refugees": 8316, "revi": 8317, "ring": 8318, "berries": 8319, "cookies": 8320, "yy": 8321, "conservation": 8322, "shab": 8323, "humans": 8324, "determin": 8325, "ain": 8326, "niall": 8327, "assu": 8328, "mba": 8329, "from": 8330, "extreme": 8331, "vices": 8332, "commerce": 8333, "ghtful": 8334, "ordered": 8335, "supports": 8336, "recap": 8337, "vor": 8338, "dropping": 8339, "correct": 8340, "paying": 8341, "meaning": 8342, "nj": 8343, "quiz": 8344, "\"#": 8345, "business": 8346, "ðŁĩ®ðŁĩ": 8347, "indigen": 8348, "dust": 8349, "boxes": 8350, "blind": 8351, "xxx": 8352, "zzy": 8353, "ðŁĩ¬ðŁĩ": 8354, "ssels": 8355, "sant": 8356, "ddle": 8357, "hilarious": 8358, "design": 8359, "wondering": 8360, "vehicles": 8361, "kre": 8362, "jud": 8363, "reception": 8364, "parker": 8365, "ÃŃ": 8366, "privi": 8367, "hydro": 8368, "softball": 8369, "pollu": 8370, "locked": 8371, "bah": 8372, "ear": 8373, "script": 8374, "divi": 8375, "brace": 8376, "george": 8377, "theast": 8378, "belo": 8379, "jal": 8380, "tionary": 8381, "dental": 8382, "rocket": 8383, "purch": 8384, "shak": 8385, "manufacturing": 8386, "ez": 8387, "itis": 8388, "concep": 8389, "tball": 8390, "chs": 8391, "directed": 8392, "prayers": 8393, "ook": 8394, "philos": 8395, "variety": 8396, "chess": 8397, "server": 8398, "gand": 8399, "balti": 8400, "ðŁĵ¸": 8401, "sely": 8402, "cruz": 8403, "spectacular": 8404, "burning": 8405, "represent": 8406, "iz": 8407, "tone": 8408, "merce": 8409, "hell": 8410, "bedroom": 8411, "establi": 8412, "bol": 8413, "common": 8414, "ãĥ»": 8415, "abor": 8416, "kitty": 8417, "heights": 8418, "repair": 8419, "william": 8420, "quake": 8421, "alabama": 8422, "population": 8423, "rev": 8424, "rett": 8425, "ists": 8426, "nite": 8427, "lem": 8428, "aha": 8429, "cleveland": 8430, "rm": 8431, "pover": 8432, "obse": 8433, "montre": 8434, "mania": 8435, "®": 8436, "conne": 8437, "carni": 8438, "shah": 8439, "fy": 8440, "ua": 8441, "scor": 8442, "struggle": 8443, "bob": 8444, "''": 8445, "appropri": 8446, "decide": 8447, "ffed": 8448, "caster": 8449, "sort": 8450, "hungry": 8451, "drag": 8452, "اÙ": 8453, "grounds": 8454, "dw": 8455, "slightly": 8456, "cardin": 8457, "deadline": 8458, "bronze": 8459, "webin": 8460, "barry": 8461, "silence": 8462, "euro": 8463, "option": 8464, "earn": 8465, "ðŁĴĸ": 8466, "however": 8467, "naren": 8468, "nails": 8469, "bathroom": 8470, "vine": 8471, "phd": 8472, "mining": 8473, "garage": 8474, "()": 8475, "shoulder": 8476, "defeat": 8477, "dir": 8478, "ov": 8479, "liberty": 8480, "pleas": 8481, "xon": 8482, "compre": 8483, "av": 8484, "jin": 8485, "ables": 8486, "silent": 8487, "famili": 8488, "visits": 8489, "dipl": 8490, "habit": 8491, "millions": 8492, "regarding": 8493, "innovative": 8494, "senator": 8495, "rts": 8496, "von": 8497, "kl": 8498, "whil": 8499, "required": 8500, "âĿĦ": 8501, "luv": 8502, "presidential": 8503, "pocket": 8504, "hundre": 8505, "shown": 8506, "frozen": 8507, "toward": 8508, "fast": 8509, "confidence": 8510, "rough": 8511, "individual": 8512, "quet": 8513, "ðŁı½": 8514, "dome": 8515, "fifa": 8516, "engineer": 8517, "zen": 8518, "remix": 8519, "ðŁĺĥ": 8520, "plant": 8521, "minor": 8522, "robinson": 8523, "asy": 8524, "pulled": 8525, "certain": 8526, "potato": 8527, "(:": 8528, "pres": 8529, "occa": 8530, "wit": 8531, "item": 8532, "sie": 8533, "dating": 8534, "thompson": 8535, "owned": 8536, "anu": 8537, "vie": 8538, "tedly": 8539, "goodnight": 8540, "except": 8541, "ðŁĮŁ": 8542, "iraq": 8543, "kie": 8544, "rences": 8545, "lip": 8546, "similar": 8547, "saudi": 8548, "vig": 8549, "arthur": 8550, "picks": 8551, "milan": 8552, "honda": 8553, "maxi": 8554, "og": 8555, "stest": 8556, "arch": 8557, "analytics": 8558, "basti": 8559, "pearl": 8560, "terry": 8561, "horse": 8562, "astro": 8563, "acce": 8564, "launching": 8565, "international": 8566, "sno": 8567, "tasty": 8568, "denver": 8569, "irl": 8570, "pete": 8571, "torn": 8572, "advantage": 8573, "varsity": 8574, "\"\"": 8575, "sole": 8576, "gc": 8577, "lang": 8578, "demonstr": 8579, "olds": 8580, "unity": 8581, "nets": 8582, "inspire": 8583, "crete": 8584, "nashville": 8585, "nelson": 8586, "eter": 8587, "walk": 8588, "hyun": 8589, "mack": 8590, "treas": 8591, "seeking": 8592, "rage": 8593, "brush": 8594, "aband": 8595, "whilst": 8596, "cocon": 8597, "hong": 8598, "shelter": 8599, "ip": 8600, "possibly": 8601, "soo": 8602, "ited": 8603, "âĦ": 8604, "races": 8605, "warming": 8606, "quin": 8607, "television": 8608, "matches": 8609, "rapi": 8610, "mental": 8611, "palm": 8612, "jennifer": 8613, "rolls": 8614, "indiana": 8615, "bars": 8616, "catching": 8617, "rescu": 8618, "candidates": 8619, "fare": 8620, "âłĢ": 8621, "seo": 8622, "vietnam": 8623, "alpha": 8624, "michelle": 8625, "visible": 8626, "regre": 8627, "wned": 8628, "apple": 8629, "lip": 8630, "ffe": 8631, "liz": 8632, "yorkshire": 8633, "hail": 8634, "seasons": 8635, "began": 8636, "md": 8637, "kc": 8638, "lap": 8639, "fascinating": 8640, "help": 8641, "ury": 8642, "ums": 8643, "nuts": 8644, "sem": 8645, "alongside": 8646, "bridge": 8647, "orial": 8648, "ove": 8649, "worldcup": 8650, "british": 8651, "comfortable": 8652, "ive": 8653, "hotels": 8654, "fairs": 8655, "horri": 8656, "sox": 8657, "dining": 8658, "stream": 8659, "barri": 8660, "ssy": 8661, "wim": 8662, "terms": 8663, "vu": 8664, "pere": 8665, "lens": 8666, "walked": 8667, "ror": 8668, "lars": 8669, "shield": 8670, "doubt": 8671, "proto": 8672, "crossing": 8673, "meant": 8674, "medium": 8675, "adding": 8676, "eb": 8677, "cheap": 8678, "func": 8679, "paper": 8680, "brands": 8681, "ryan": 8682, "feedback": 8683, "collins": 8684, "unknown": 8685, "tropical": 8686, "sandwich": 8687, "fallen": 8688, "formu": 8689, "select": 8690, "loads": 8691, "answers": 8692, "ori": 8693, "maga": 8694, "dor": 8695, "duo": 8696, "alie": 8697, "drum": 8698, "uri": 8699, "deer": 8700, "soul": 8701, "shut": 8702, "âĺº": 8703, "stolen": 8704, "donated": 8705, "buzz": 8706, "patriots": 8707, "hal": 8708, "nasty": 8709, "nominated": 8710, "monte": 8711, "kia": 8712, "thri": 8713, "ingu": 8714, "tests": 8715, "petro": 8716, "ðŁijij": 8717, "hosts": 8718, "nest": 8719, "topic": 8720, "patch": 8721, "mmy": 8722, "hugh": 8723, "abilities": 8724, "mathe": 8725, "smiles": 8726, "gb": 8727, "agenda": 8728, "insights": 8729, "chip": 8730, "phan": 8731, "failure": 8732, "dgers": 8733, "hai": 8734, "significant": 8735, "shock": 8736, "rural": 8737, "glam": 8738, "figures": 8739, "potus": 8740, "ota": 8741, "ministry": 8742, "appears": 8743, "fear": 8744, "rh": 8745, "american": 8746, "hatt": 8747, "sony": 8748, "fires": 8749, "edi": 8750, "nou": 8751, "equi": 8752, "when": 8753, "universal": 8754, "madness": 8755, "ix": 8756, "sculpture": 8757, "bach": 8758, "tto": 8759, "sweden": 8760, "eta": 8761, "ento": 8762, "developed": 8763, "monthly": 8764, "maps": 8765, "rah": 8766, "led": 8767, "delta": 8768, "saints": 8769, "islam": 8770, "bench": 8771, "fifth": 8772, "vard": 8773, "socks": 8774, "welcoming": 8775, "je": 8776, "turner": 8777, "vb": 8778, "adi": 8779, "norway": 8780, "ady": 8781, "hurricane": 8782, "porsche": 8783, "tradition": 8784, "exam": 8785, "newspaper": 8786, "luci": 8787, "aver": 8788, "ideal": 8789, "dna": 8790, "madison": 8791, "ðŁ§": 8792, "witness": 8793, "acou": 8794, "insight": 8795, "simon": 8796, "robot": 8797, "snake": 8798, "nbc": 8799, "aco": 8800, "ross": 8801, "shment": 8802, "religion": 8803, "chann": 8804, "insu": 8805, "campbell": 8806, "installed": 8807, "weather": 8808, "horses": 8809, "oli": 8810, "robert": 8811, "kaz": 8812, "ðŁıĢ": 8813, "veteran": 8814, "thread": 8815, "quarter": 8816, "easier": 8817, "capture": 8818, "hipho": 8819, "lawrence": 8820, "romantic": 8821, "passion": 8822, "clay": 8823, "oxford": 8824, "thai": 8825, "studying": 8826, "fia": 8827, "elected": 8828, "mostly": 8829, "cb": 8830, "tumb": 8831, "âĢįâĻĤ": 8832, "xl": 8833, "shan": 8834, "faster": 8835, "evans": 8836, "slide": 8837, "shri": 8838, "seek": 8839, "mies": 8840, "chemistry": 8841, "pumpkin": 8842, "tum": 8843, ",,": 8844, "room": 8845, "fired": 8846, "lips": 8847, "presence": 8848, "aff": 8849, "brewery": 8850, "arrive": 8851, "swag": 8852, "photograph": 8853, "pengu": 8854, "chips": 8855, "attor": 8856, "values": 8857, "accurate": 8858, "contemporary": 8859, "principal": 8860, "cannabis": 8861, "ario": 8862, "anywhere": 8863, "gia": 8864, "democrats": 8865, "buildings": 8866, "lived": 8867, "aps": 8868, "negative": 8869, "mare": 8870, "ballo": 8871, "lion": 8872, "diamon": 8873, "look": 8874, "reform": 8875, "tommy": 8876, "illa": 8877, "treats": 8878, "hundreds": 8879, "portland": 8880, "worthy": 8881, "excep": 8882, "aria": 8883, "idol": 8884, "beer": 8885, "cdn": 8886, "yu": 8887, "awk": 8888, "ðŁĩ¨": 8889, "cells": 8890, "ó": 8891, "identity": 8892, "drawn": 8893, "devil": 8894, "finger": 8895, "tham": 8896, "ðŁijĬ": 8897, "earned": 8898, "fintech": 8899, "dolph": 8900, "tweeting": 8901, "evolution": 8902, "ðŁĵį": 8903, "estim": 8904, "mvp": 8905, "none": 8906, "ðŁĩºðŁĩ¸": 8907, "toyota": 8908, "aux": 8909, "marin": 8910, "bold": 8911, "lbs": 8912, "steak": 8913, "murphy": 8914, "itable": 8915, "louis": 8916, "solve": 8917, "pia": 8918, "skir": 8919, "illino": 8920, "webinar": 8921, "banana": 8922, "lov": 8923, "thon": 8924, "voters": 8925, "affordable": 8926, "defeated": 8927, "lmfa": 8928, "airlines": 8929, "superb": 8930, "anyway": 8931, "debt": 8932, "bored": 8933, "versi": 8934, "metal": 8935, "responsible": 8936, "mk": 8937, "sse": 8938, "fay": 8939, "caused": 8940, "fp": 8941, "recommend": 8942, "plaza": 8943, "sporting": 8944, "alliance": 8945, "austri": 8946, "nn": 8947, "tours": 8948, "surprised": 8949, "artif": 8950, "thunder": 8951, "surve": 8952, "wore": 8953, "brief": 8954, "necessary": 8955, "zie": 8956, "ashley": 8957, "drake": 8958, "rt": 8959, "knife": 8960, "immun": 8961, "charges": 8962, "athe": 8963, "bride": 8964, "reply": 8965, "gav": 8966, "broadcast": 8967, "puer": 8968, "bracelet": 8969, "capacity": 8970, "harvest": 8971, "idk": 8972, "performan": 8973, "dding": 8974, "ilers": 8975, "para": 8976, "jama": 8977, "province": 8978, "chin": 8979, "iders": 8980, "hari": 8981, "teaser": 8982, "chen": 8983, "restor": 8984, "rat": 8985, "flat": 8986, "colom": 8987, "ðŁĴŀ": 8988, "ðŁĩ¨ðŁĩ": 8989, "smooth": 8990, "rt": 8991, "pitch": 8992, "staying": 8993, "israeli": 8994, "tcot": 8995, "perspective": 8996, "dock": 8997, "opener": 8998, "lovel": 8999, "xo": 9000, "classroom": 9001, "lington": 9002, "goal": 9003, "kennedy": 9004, "sham": 9005, "spaces": 9006, "mitchell": 9007, "homecoming": 9008, "uki": 9009, "claimed": 9010, "recruit": 9011, "ingo": 9012, "mufc": 9013, "monit": 9014, "groo": 9015, "resident": 9016, "percent": 9017, "perman": 9018, "ottawa": 9019, "intment": 9020, "anxi": 9021, "standards": 9022, "worship": 9023, "scheme": 9024, "fx": 9025, "potter": 9026, "bian": 9027, "athletic": 9028, "afgh": 9029, "sse": 9030, "satell": 9031, "parties": 9032, "âĿ¤âĿ¤": 9033, "infrastructure": 9034, "relax": 9035, "modu": 9036, "worn": 9037, "smoking": 9038, "yach": 9039, "practices": 9040, "wcw": 9041, "amb": 9042, "domestic": 9043, "taylor": 9044, "kentu": 9045, "provided": 9046, "modi": 9047, "veg": 9048, "\"...": 9049, "observ": 9050, "ðŁĺ©": 9051, "beard": 9052, "mour": 9053, "angry": 9054, "ðŁĺ±": 9055, "startups": 9056, "wooden": 9057, "dive": 9058, "nail": 9059, "antique": 9060, "roses": 9061, "tornado": 9062, "mat": 9063, "^^": 9064, "suspect": 9065, "farm": 9066, "devices": 9067, "mega": 9068, "tul": 9069, "scholarship": 9070, "gee": 9071, "disaster": 9072, "arrival": 9073, "poin": 9074, "marc": 9075, "katie": 9076, "bbed": 9077, "false": 9078, "deserves": 9079, "richard": 9080, "juana": 9081, "frey": 9082, "tioned": 9083, "hybri": 9084, "rw": 9085, "sarah": 9086, "achi": 9087, "cure": 9088, "ole": 9089, "morris": 9090, "chic": 9091, "broadway": 9092, "label": 9093, "pak": 9094, "poverty": 9095, "golf": 9096, "ered": 9097, "fu": 9098, "eries": 9099, "bees": 9100, "alogue": 9101, "stel": 9102, "wireless": 9103, "jewish": 9104, "tide": 9105, "blocked": 9106, "lifetime": 9107, "bhar": 9108, "split": 9109, "amster": 9110, "thi": 9111, "joshu": 9112, "brunch": 9113, "haps": 9114, "sfor": 9115, "oops": 9116, "kapoor": 9117, "hiking": 9118, "supposed": 9119, "roof": 9120, "reas": 9121, "train": 9122, "tight": 9123, "trump": 9124, "basically": 9125, "rr": 9126, "eared": 9127, "seeds": 9128, "entrance": 9129, "cp": 9130, "wie": 9131, "sonic": 9132, "victim": 9133, "here": 9134, "eh": 9135, "earrings": 9136, "salmon": 9137, "arctic": 9138, "anne": 9139, "dougla": 9140, "corruption": 9141, "hannah": 9142, "hasn": 9143, "voices": 9144, "conce": 9145, "atta": 9146, "fleet": 9147, "clinical": 9148, "democratic": 9149, "tony": 9150, "stood": 9151, "lef": 9152, "twitch": 9153, "ail": 9154, "honestly": 9155, "increased": 9156, "drome": 9157, "donna": 9158, "accepted": 9159, "visitors": 9160, "apar": 9161, "ador": 9162, "par": 9163, "jerry": 9164, "rai": 9165, "brandon": 9166, "abu": 9167, "!!!!!!": 9168, "meme": 9169, "ingh": 9170, "glorious": 9171, "bhu": 9172, "pump": 9173, "jol": 9174, "like": 9175, "fisher": 9176, "maz": 9177, "agan": 9178, "destination": 9179, "playlist": 9180, "letters": 9181, "genu": 9182, "brace": 9183, "celebrated": 9184, "banner": 9185, "rhe": 9186, "dragon": 9187, "ðŁĺħ": 9188, "signature": 9189, "grey": 9190, "âľĶï¸ı": 9191, "alice": 9192, "bered": 9193, "pher": 9194, "bern": 9195, "cath": 9196, "gathering": 9197, "scoring": 9198, "influence": 9199, "smiling": 9200, "dept": 9201, "local": 9202, "ax": 9203, "acu": 9204, "retirement": 9205, "honor": 9206, "herself": 9207, "chemical": 9208, "assess": 9209, "yall": 9210, "frequ": 9211, "appreciation": 9212, "aca": 9213, "choir": 9214, "cuz": 9215, "soil": 9216, "cil": 9217, "reporting": 9218, "uh": 9219, "enterprise": 9220, "grat": 9221, "jacob": 9222, "rum": 9223, "fee": 9224, "jak": 9225, "spin": 9226, "bikes": 9227, "phia": 9228, "stere": 9229, "pis": 9230, "blood": 9231, "tatt": 9232, "raft": 9233, "warren": 9234, "sheri": 9235, "backstage": 9236, "marsh": 9237, "hashtag": 9238, "therine": 9239, "rein": 9240, "gameday": 9241, "guaran": 9242, "recipes": 9243, "minds": 9244, "stronger": 9245, "issued": 9246, "bicy": 9247, "nak": 9248, "mented": 9249, "scary": 9250, "ux": 9251, "previous": 9252, "ttle": 9253, "thats": 9254, "actors": 9255, "uma": 9256, "tina": 9257, "bunny": 9258, "promotion": 9259, "uss": 9260, "oliver": 9261, "montreal": 9262, "whats": 9263, "appreciated": 9264, "lakes": 9265, "excuse": 9266, "knowing": 9267, "prizes": 9268, "muscle": 9269, "shades": 9270, "scot": 9271, "ingredi": 9272, "electronic": 9273, "juan": 9274, "combat": 9275, "sri": 9276, "eh": 9277, "turkish": 9278, "lom": 9279, "strikes": 9280, "prison": 9281, "ree": 9282, "pope": 9283, "vid": 9284, "oldest": 9285, "doll": 9286, "swiss": 9287, "certified": 9288, "clip": 9289, "returning": 9290, "lator": 9291, "leigh": 9292, "ttes": 9293, "watson": 9294, "healing": 9295, "elim": 9296, "perhaps": 9297, "hass": 9298, "kau": 9299, "dder": 9300, "mouse": 9301, "newcastle": 9302, "indigenous": 9303, "welcomes": 9304, "cole": 9305, "taught": 9306, "noise": 9307, "appear": 9308, "joe": 9309, "canon": 9310, "wednesday": 9311, "utah": 9312, "ctive": 9313, "driven": 9314, "iv": 9315, "cell": 9316, "strip": 9317, "acc": 9318, "focused": 9319, "arrest": 9320, "stocks": 9321, "woo": 9322, "âĹ": 9323, "noticed": 9324, "shado": 9325, "displa": 9326, "terror": 9327, "borne": 9328, "second": 9329, "queens": 9330, "woke": 9331, "jail": 9332, "nott": 9333, "cambridge": 9334, "hart": 9335, "seaf": 9336, "fax": 9337, "accept": 9338, "âĺħ": 9339, "goods": 9340, "kat": 9341, "twin": 9342, "hs": 9343, "thousand": 9344, "sins": 9345, "suite": 9346, "ampton": 9347, "arn": 9348, "relev": 9349, "richar": 9350, "hoops": 9351, "nbc": 9352, "classic": 9353, "pab": 9354, "soldier": 9355, "deplo": 9356, "leans": 9357, "installation": 9358, "clash": 9359, "leban": 9360, "eee": 9361, "tire": 9362, "beloved": 9363, "fusion": 9364, "traveling": 9365, "nei": 9366, "cookie": 9367, "globe": 9368, "physics": 9369, "sq": 9370, "col": 9371, "wolves": 9372, "dl": 9373, "exit": 9374, "\"-": 9375, "football": 9376, "leaf": 9377, "sterling": 9378, "hide": 9379, "minneso": 9380, "freshman": 9381, "nature": 9382, "indie": 9383, "supplies": 9384, "bris": 9385, "irish": 9386, "inktober": 9387, "doodle": 9388, "icop": 9389, "messages": 9390, "adults": 9391, "recorded": 9392, "fixed": 9393, "ardo": 9394, "offered": 9395, "underground": 9396, "drone": 9397, "pine": 9398, "mainten": 9399, "andre": 9400, "hammer": 9401, "sx": 9402, "round": 9403, "hike": 9404, "brad": 9405, "rome": 9406, "full": 9407, "oney": 9408, "rows": 9409, "columbia": 9410, "archives": 9411, "approved": 9412, "batch": 9413, "illinois": 9414, "recognition": 9415, "shouldn": 9416, "fog": 9417, "ncaa": 9418, "kevin": 9419, "humanity": 9420, "although": 9421, "powers": 9422, "pou": 9423, "sar": 9424, "pest": 9425, "alcohol": 9426, "consci": 9427, "philadel": 9428, "eno": 9429, "tm": 9430, "okla": 9431, "category": 9432, "participate": 9433, "accused": 9434, "brief": 9435, "poem": 9436, "clubs": 9437, "consult": 9438, "jab": 9439, "bigdata": 9440, "amsterdam": 9441, "acing": 9442, "certific": 9443, "nu": 9444, "dat": 9445, "improved": 9446, "andy": 9447, "campaig": 9448, "palestin": 9449, "pace": 9450, "mobi": 9451, "feelings": 9452, "wolf": 9453, "brain": 9454, "propos": 9455, "interactive": 9456, "prince": 9457, "index": 9458, "cis": 9459, "chae": 9460, "peaceful": 9461, "covering": 9462, "aco": 9463, "courses": 9464, "monkey": 9465, "replace": 9466, "bl": 9467, "bloody": 9468, "tales": 9469, "brighton": 9470, "neighborhood": 9471, "gates": 9472, "spiritual": 9473, "afraid": 9474, "breast": 9475, "bones": 9476, "ðŁijī": 9477, "video": 9478, "wau": 9479, "touch": 9480, "injuries": 9481, "carl": 9482, "rix": 9483, "unex": 9484, "âĢ¢": 9485, "fred": 9486, "considered": 9487, "thusi": 9488, "anch": 9489, "ony": 9490, "usa": 9491, "graphics": 9492, "acre": 9493, "ðŁĺ©": 9494, "commemor": 9495, "commod": 9496, "goti": 9497, "guardian": 9498, "starbucks": 9499, "prevention": 9500, "hahahaha": 9501, "administration": 9502, "portugal": 9503, "faculty": 9504, "beta": 9505, "ula": 9506, "albert": 9507, "breath": 9508, "eri": 9509, "letting": 9510, "tric": 9511, "mentation": 9512, "incredibly": 9513, "tennes": 9514, "vd": 9515, "ðŁĻĪ": 9516, "eddie": 9517, "brick": 9518, "grill": 9519, "btw": 9520, "watches": 9521, "researchers": 9522, "tney": 9523, "nie": 9524, "pas": 9525, "aster": 9526, "vibr": 9527, "pokemon": 9528, "chrome": 9529, "goat": 9530, "pitts": 9531, "illy": 9532, "festive": 9533, "yd": 9534, "canal": 9535, "ðŁĨ": 9536, "fies": 9537, "carlos": 9538, "reque": 9539, "partici": 9540, "trains": 9541, "sample": 9542, "temperature": 9543, "symph": 9544, "picking": 9545, "indoor": 9546, "zers": 9547, "playoffs": 9548, "________": 9549, "apes": 9550, "lyrics": 9551, "islamic": 9552, "performances": 9553, "dick": 9554, "spark": 9555, "seas": 9556, "homa": 9557, "ground": 9558, "disci": 9559, "employee": 9560, "commu": 9561, "alaska": 9562, "alan": 9563, "feast": 9564, "dging": 9565, "banking": 9566, "manuel": 9567, "slowly": 9568, "trucks": 9569, "mccar": 9570, "ooo": 9571, "scrat": 9572, "orchestra": 9573, "individu": 9574, "mx": 9575, "breath": 9576, "stairs": 9577, "equality": 9578, "blake": 9579, "locations": 9580, "coconut": 9581, "baltimore": 9582, "aaa": 9583, "lc": 9584, "ðŁıĨ": 9585, "harvey": 9586, "resist": 9587, "immigration": 9588, "adidas": 9589, "fili": 9590, "ref": 9591, "lgbt": 9592, "mos": 9593, "ppi": 9594, "kenny": 9595, "terror": 9596, "bane": 9597, "apolis": 9598, "sg": 9599, "socialmedia": 9600, "kai": 9601, "honest": 9602, "assas": 9603, "bollywood": 9604, "âĢįâĻĢï¸ı": 9605, "ferrari": 9606, "horn": 9607, "crypto": 9608, "boom": 9609, "maintenance": 9610, "idi": 9611, "sman": 9612, "wl": 9613, "extended": 9614, "insul": 9615, "ves": 9616, "gosp": 9617, "tri": 9618, "pig": 9619, "targe": 9620, "celer": 9621, "stati": 9622, "smh": 9623, "ridic": 9624, "appeal": 9625, "?)": 9626, "conclu": 9627, "cosme": 9628, "sheep": 9629, "christopher": 9630, "enthusi": 9631, "polish": 9632, "mets": 9633, "ounded": 9634, "sustainability": 9635, "creativity": 9636, "concrete": 9637, "rai": 9638, "alien": 9639, "bless": 9640, "tees": 9641, "club": 9642, "rot": 9643, "bos": 9644, "exist": 9645, "perfection": 9646, "luck": 9647, "rocky": 9648, "expensive": 9649, "meanwhile": 9650, "happybirthday": 9651, "pret": 9652, "thriller": 9653, "cave": 9654, "playoff": 9655, "somer": 9656, "lu": 9657, "lex": 9658, "defence": 9659, "amwriting": 9660, "homeless": 9661, "prophe": 9662, "chet": 9663, "pastor": 9664, "ðŁ¤£": 9665, "lander": 9666, "www": 9667, "Ģï¸ı": 9668, "tica": 9669, "!#": 9670, "otic": 9671, "radar": 9672, "posters": 9673, "powder": 9674, "poli": 9675, "haun": 9676, "trap": 9677, "blin": 9678, "assault": 9679, "shorts": 9680, "rey": 9681, "shy": 9682, "squir": 9683, "racist": 9684, "garlic": 9685, "fur": 9686, "remote": 9687, "smell": 9688, "impressed": 9689, "fingers": 9690, "âłĢ": 9691, "dino": 9692, "lement": 9693, "snu": 9694, "promoting": 9695, "string": 9696, "productive": 9697, "bage": 9698, "mason": 9699, "raz": 9700, "directly": 9701, "jk": 9702, "eval": 9703, "ðŁijĬ": 9704, "doctors": 9705, "cow": 9706, "rider": 9707, "stv": 9708, "remove": 9709, "wu": 9710, "nathan": 9711, "rod": 9712, "nr": 9713, "=>": 9714, "affected": 9715, "invest": 9716, "mption": 9717, "ginger": 9718, "od": 9719, "agriculture": 9720, "sque": 9721, "mug": 9722, "counting": 9723, "kee": 9724, "magnific": 9725, "cook": 9726, "anistan": 9727, "root": 9728, "placed": 9729, "sympo": 9730, "ghana": 9731, "und": 9732, "cheer": 9733, "throwing": 9734, "secrets": 9735, "filling": 9736, "optimi": 9737, "butterfly": 9738, "bubb": 9739, "ðŁĺī": 9740, "terrible": 9741, "dg": 9742, "silk": 9743, "obsessed": 9744, "lou": 9745, "aide": 9746, "salute": 9747, "monu": 9748, "philadelphia": 9749, "scientific": 9750, "ist": 9751, "uae": 9752, "dessert": 9753, "bottles": 9754, "canyon": 9755, "ðŁĺĪ": 9756, "carib": 9757, "other": 9758, "wich": 9759, "resource": 9760, "guilty": 9761, "und": 9762, "leon": 9763, "ess": 9764, "kane": 9765, "ele": 9766, "trainer": 9767, "heim": 9768, "ante": 9769, "manage": 9770, "rookie": 9771, "treated": 9772, "poses": 9773, "rsvp": 9774, "causes": 9775, "awak": 9776, "jewell": 9777, "lett": 9778, "onics": 9779, "titles": 9780, "cardiff": 9781, "gaga": 9782, "bump": 9783, "useful": 9784, "?!": 9785, "loose": 9786, "bbing": 9787, "::": 9788, "argentina": 9789, "debu": 9790, "cycl": 9791, "whel": 9792, "disgu": 9793, "jel": 9794, "kills": 9795, "biology": 9796, "exter": 9797, "trash": 9798, "bodies": 9799, "tram": 9800, "circuit": 9801, "expect": 9802, "lads": 9803, "wells": 9804, "shot": 9805, "gee": 9806, "narendr": 9807, "fastest": 9808, "bent": 9809, "bills": 9810, "marshall": 9811, "hats": 9812, "introduce": 9813, "citizen": 9814, "impossible": 9815, "gib": 9816, "azz": 9817, "networking": 9818, "rant": 9819, "think": 9820, "indy": 9821, "stops": 9822, "ftheday": 9823, "brian": 9824, "**": 9825, "amodi": 9826, "dome": 9827, "courage": 9828, "packing": 9829, "affairs": 9830, "gn": 9831, "sized": 9832, "entary": 9833, "poland": 9834, "switzer": 9835, "afghanistan": 9836, "wu": 9837, "tender": 9838, "subscribe": 9839, "mosco": 9840, "attend": 9841, "republican": 9842, "honey": 9843, "âĢĭ": 9844, "simul": 9845, "wester": 9846, "foodie": 9847, "oro": 9848, "middle": 9849, "abt": 9850, "copies": 9851, "maje": 9852, "narendramodi": 9853, "typical": 9854, "inspirational": 9855, "vitam": 9856, "wiscon": 9857, "cubs": 9858, "tivity": 9859, "hali": 9860, "ears": 9861, "kay": 9862, "dare": 9863, "marijuana": 9864, "curious": 9865, "ania": 9866, "tomato": 9867, "remind": 9868, "ðŁĩ·": 9869, "scared": 9870, "coup": 9871, "poet": 9872, "landed": 9873, "rid": 9874, "wrapped": 9875, "morri": 9876, "climbing": 9877, "ews": 9878, "feeding": 9879, "contra": 9880, "thology": 9881, "grid": 9882, "tively": 9883, "reader": 9884, "laser": 9885, "diving": 9886, "dig": 9887, "latin": 9888, "tied": 9889, "shakespe": 9890, "oci": 9891, "adm": 9892, "showers": 9893, "chuck": 9894, "marcus": 9895, "oos": 9896, "knee": 9897, "olive": 9898, "owl": 9899, "dylan": 9900, "anno": 9901, "gym": 9902, "decisions": 9903, "wellness": 9904, "arrives": 9905, "satis": 9906, "chris": 9907, "thurs": 9908, "ðŁ¤£": 9909, "interviews": 9910, "thankyou": 9911, "switzerland": 9912, "overnight": 9913, "journalist": 9914, "serves": 9915, "volcan": 9916, ".......": 9917, "plot": 9918, "nicol": 9919, "carrying": 9920, "magne": 9921, "treasure": 9922, "exp": 9923, "bever": 9924, "ðŁĺ¢": 9925, "marty": 9926, "mole": 9927, "donations": 9928, "recognized": 9929, "bh": 9930, "dus": 9931, "shann": 9932, "aldo": 9933, "successfully": 9934, "ente": 9935, "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ": 9936, "cabinet": 9937, "cuis": 9938, "titled": 9939, "das": 9940, "sol": 9941, "strategies": 9942, "delivering": 9943, "adds": 9944, "anian": 9945, "nether": 9946, "ðŁĴĥ": 9947, "contain": 9948, "suits": 9949, "pairs": 9950, "todd": 9951, "rella": 9952, "rope": 9953, "cio": 9954, "crop": 9955, "paintings": 9956, "suz": 9957, "rejec": 9958, "bust": 9959, "dh": 9960, "fraud": 9961, "mh": 9962, "control": 9963, "jeal": 9964, "destroyed": 9965, "allows": 9966, "wool": 9967, "minnesota": 9968, "omen": 9969, "ju": 9970, "symposium": 9971, "daf": 9972, "limit": 9973, "accounts": 9974, "loading": 9975, "intern": 9976, "resolution": 9977, "holland": 9978, "qual": 9979, "meetings": 9980, "grave": 9981, "camping": 9982, "vam": 9983, "renov": 9984, "liberal": 9985, "amber": 9986, "gree": 9987, "humb": 9988, "fever": 9989, "eling": 9990, "brooks": 9991, "à²": 9992, "beth": 9993, "aded": 9994, "alt": 9995, "roe": 9996, "performed": 9997, "josh": 9998, "franklin": 9999, "nicole": 10000, "dess": 10001, "bbs": 10002, "mg": 10003, "networks": 10004, "minim": 10005, "alt": 10006, "weapons": 10007, "guy": 10008, "jason": 10009, "gha": 10010, "harbour": 10011, "aton": 10012, "praise": 10013, "kentucky": 10014, "belfast": 10015, "sticks": 10016, "bloss": 10017, "hopes": 10018, "anthro": 10019, "familiar": 10020, "wait": 10021, "chile": 10022, "depression": 10023, "lax": 10024, "jets": 10025, "leice": 10026, "receives": 10027, "sier": 10028, "ank": 10029, "dex": 10030, "indeed": 10031, "flexi": 10032, "fabric": 10033, "lamb": 10034, "helicop": 10035, "amanda": 10036, "âĢĶâĢĶ": 10037, "compete": 10038, "snack": 10039, "technologies": 10040, "syrian": 10041, "moms": 10042, "muham": 10043, "chosen": 10044, "anat": 10045, "devon": 10046, "sharks": 10047, "ret": 10048, "fundraiser": 10049, "selfies": 10050, "stations": 10051, "communications": 10052, "tennessee": 10053, "tutor": 10054, "rot": 10055, "valuable": 10056, "dynamic": 10057, "nurse": 10058, "ied": 10059, "earthquake": 10060, "deserved": 10061, "ave": 10062, "sara": 10063, "stretch": 10064, "douglas": 10065, "nepal": 10066, "ç": 10067, "obviously": 10068, "dame": 10069, "rape": 10070, "anybody": 10071, "kw": 10072, "patrol": 10073, "holders": 10074, "hanna": 10075, "infographic": 10076, "eco": 10077, "beating": 10078, "stanley": 10079, "boats": 10080, "ribb": 10081, "ez": 10082, "witch": 10083, "inva": 10084, "acid": 10085, "boarding": 10086, "-@": 10087, "gil": 10088, "dave": 10089, "careers": 10090, "oppos": 10091, "lloy": 10092, "inter": 10093, "dope": 10094, "resu": 10095, "jagu": 10096, "shade": 10097, "indy": 10098, "onist": 10099, "relations": 10100, "agen": 10101, "able": 10102, "incident": 10103, "meter": 10104, "sharma": 10105, "idr": 10106, "prove": 10107, "immediately": 10108, "troops": 10109, "aman": 10110, "glow": 10111, "gaza": 10112, "blocks": 10113, "personal": 10114, "chronic": 10115, "aller": 10116, "sid": 10117, "shr": 10118, "whatsapp": 10119, "lucy": 10120, "archae": 10121, "hou": 10122, "journalism": 10123, "ourselves": 10124, "got": 10125, "themed": 10126, "shaped": 10127, "weak": 10128, "casual": 10129, "length": 10130, "slam": 10131, "abbey": 10132, "ev": 10133, "counter": 10134, "esta": 10135, "recipi": 10136, "chapel": 10137, "expansion": 10138, "self": 10139, "suffering": 10140, "spice": 10141, "nz": 10142, "spart": 10143, "desper": 10144, "booking": 10145, "quarters": 10146, "yon": 10147, "ðŁĴĹ": 10148, "pk": 10149, "continued": 10150, "-#": 10151, "manhatt": 10152, "talked": 10153, "shen": 10154, "combo": 10155, "hybrid": 10156, "jeans": 10157, "liquid": 10158, "seal": 10159, "retweets": 10160, "acceler": 10161, "collective": 10162, "tas": 10163, ":))": 10164, "professionals": 10165, "raw": 10166, "ott": 10167, "susan": 10168, "iring": 10169, "oklahoma": 10170, "reven": 10171, "survival": 10172, "creator": 10173, "transit": 10174, "stac": 10175, "surf": 10176, "ik": 10177, "editing": 10178, "chilling": 10179, "bailey": 10180, "steal": 10181, "rable": 10182, "parent": 10183, "hunger": 10184, "snapp": 10185, "collect": 10186, "philosoph": 10187, "dedication": 10188, "cf": 10189, "cm": 10190, "leep": 10191, "repeat": 10192, "reha": 10193, "unfortun": 10194, "aer": 10195, "aero": 10196, "abstract": 10197, "monitor": 10198, "agents": 10199, "bul": 10200, "science": 10201, "harbor": 10202, "dragons": 10203, "flooding": 10204, "accompli": 10205, "dash": 10206, "julia": 10207, "thered": 10208, "tuesday": 10209, "cyber": 10210, "blow": 10211, "tained": 10212, "lem": 10213, "reference": 10214, "ppo": 10215, "negoti": 10216, "charle": 10217, "connor": 10218, "ault": 10219, "accessories": 10220, "commissioner": 10221, "rainy": 10222, "rear": 10223, "advisory": 10224, "lucas": 10225, "maid": 10226, "coal": 10227, "kav": 10228, "polo": 10229, "ðŁı¾": 10230, "transport": 10231, "margare": 10232, "strawberry": 10233, "burns": 10234, "greens": 10235, "nev": 10236, "participants": 10237, "colin": 10238, "belgium": 10239, "colour": 10240, "inform": 10241, "dell": 10242, "bron": 10243, "caly": 10244, "kickoff": 10245, "strategic": 10246, "reunion": 10247, "honors": 10248, "lib": 10249, "egyp": 10250, "âŃIJï¸ı": 10251, "hypo": 10252, "sizes": 10253, "registered": 10254, "betes": 10255, "relaxing": 10256, "bloom": 10257, "intense": 10258, "valentines": 10259, "insane": 10260, "wwii": 10261, "px": 10262, "trio": 10263, "blade": 10264, "wisconsin": 10265, "cone": 10266, "platin": 10267, "alize": 10268, "raven": 10269, "increasing": 10270, "indians": 10271, "ilian": 10272, "blu": 10273, "rabbit": 10274, "extension": 10275, "jef": 10276, "audi": 10277, "ferry": 10278, "sell": 10279, "aday": 10280, "usb": 10281, "sweat": 10282, "champag": 10283, "method": 10284, "memph": 10285, "assist": 10286, "sby": 10287, "cape": 10288, "removed": 10289, "magn": 10290, "vt": 10291, "rams": 10292, "fbi": 10293, "tackle": 10294, "phew": 10295, "hon": 10296, "motorcycle": 10297, "suspec": 10298, "elephant": 10299, "subject": 10300, "lette": 10301, "dairy": 10302, "wheat": 10303, "awkward": 10304, "act": 10305, "trol": 10306, "mitted": 10307, "zayn": 10308, "sheriff": 10309, "enemy": 10310, "cons": 10311, "kett": 10312, "bulls": 10313, "evalu": 10314, "btc": 10315, "satellite": 10316, "holo": 10317, "porter": 10318, "diabetes": 10319, "better": 10320, "releasing": 10321, "surf": 10322, ":-": 10323, "sebasti": 10324, "collecting": 10325, "encing": 10326, "ethi": 10327, "gods": 10328, "alley": 10329, "healthy": 10330, "mills": 10331, "smash": 10332, "copper": 10333, "crack": 10334, "readers": 10335, "spac": 10336, "license": 10337, "basket": 10338, "bangla": 10339, "entic": 10340, "omi": 10341, "mere": 10342, "sively": 10343, "animation": 10344, "lanes": 10345, "dentally": 10346, "chillin": 10347, "fie": 10348, "karen": 10349, "depth": 10350, "lipse": 10351, "ng": 10352, "rip": 10353, "melo": 10354, "sandy": 10355, "ðŁijıðŁijı": 10356, "vincent": 10357, "nut": 10358, "hug": 10359, "whole": 10360, "creates": 10361, "????": 10362, "âĿ¤ï¸ıâĿ¤ï¸ı": 10363, "baked": 10364, "upgrade": 10365, "roberts": 10366, "hara": 10367, "caribbean": 10368, "authentic": 10369, "mbs": 10370, "moscow": 10371, "attorney": 10372, "wiki": 10373, "chlo": 10374, "hull": 10375, "cork": 10376, "\"!": 10377, "stylish": 10378, "ðŁĵ¸:": 10379, "diary": 10380, "improving": 10381, "expand": 10382, "bright": 10383, "pollution": 10384, "knights": 10385, "personality": 10386, "checked": 10387, "facilities": 10388, "zel": 10389, "bowling": 10390, "guer": 10391, "ðŁİĤ": 10392, "ongoing": 10393, "units": 10394, "hook": 10395, "beck": 10396, "conflict": 10397, "todd": 10398, "farming": 10399, "educational": 10400, "kak": 10401, "clay": 10402, "stroke": 10403, "belly": 10404, "explore": 10405, "millenni": 10406, "thm": 10407, "loop": 10408, "sms": 10409, "consist": 10410, "circa": 10411, "bryan": 10412, "dab": 10413, "younger": 10414, "solidar": 10415, "ppa": 10416, "experienced": 10417, "bella": 10418, "board": 10419, "sheffield": 10420, "stephen": 10421, "consumer": 10422, "submit": 10423, "sponsor": 10424, "tang": 10425, "aggre": 10426, "combined": 10427, "tracking": 10428, "sanders": 10429, "baz": 10430, "survive": 10431, "ferred": 10432, "equal": 10433, "sep": 10434, "reed": 10435, "strong": 10436, "privacy": 10437, "stap": 10438, "ung": 10439, "acry": 10440, "pasta": 10441, "pirates": 10442, "ager": 10443, "fairy": 10444, "dup": 10445, "introduced": 10446, "wip": 10447, "lets": 10448, "spray": 10449, "ðŁĵº": 10450, "grew": 10451, "asts": 10452, "pittsburgh": 10453, "newyork": 10454, "joey": 10455, "lauren": 10456, "trade": 10457, "chop": 10458, "pipe": 10459, "claire": 10460, "behavior": 10461, "vap": 10462, "crews": 10463, "laptop": 10464, "ðŁ¤Ĺ": 10465, "chester": 10466, "discipl": 10467, "df": 10468, "outdoors": 10469, "ks": 10470, "gover": 10471, "superstar": 10472, "casino": 10473, "farmer": 10474, ";-)": 10475, "returned": 10476, "ðŁıĪ": 10477, "mail": 10478, "roasted": 10479, "costa": 10480, "vill": 10481, "pez": 10482, "gardening": 10483, "distribution": 10484, "shining": 10485, "investors": 10486, "rasp": 10487, "decades": 10488, "realized": 10489, "barn": 10490, "pti": 10491, "stable": 10492, "utd": 10493, "panthers": 10494, "mens": 10495, "bn": 10496, "cade": 10497, "bucket": 10498, "ynn": 10499, "whenever": 10500, "wake": 10501, "dais": 10502, "bernie": 10503, "lodge": 10504, "julie": 10505, "atmosphere": 10506, "ðŁĺĺðŁĺĺ": 10507, "majority": 10508, "parti": 10509, "excit": 10510, "cut": 10511, "meh": 10512, "muslims": 10513, "begun": 10514, "flights": 10515, "veness": 10516, "ceme": 10517, "posing": 10518, "sole": 10519, "gou": 10520, "darkness": 10521, "peach": 10522, "celtic": 10523, "authority": 10524, "grandma": 10525, "fulness": 10526, "smith": 10527, "specific": 10528, "garcia": 10529, "coins": 10530, "goodness": 10531, "aldub": 10532, "recruiting": 10533, "dennis": 10534, "gary": 10535, "sleeve": 10536, "weapon": 10537, "plz": 10538, "discover": 10539, "harrison": 10540, "recruitment": 10541, "jai": 10542, "chim": 10543, "compared": 10544, "toms": 10545, "mothers": 10546, "amy": 10547, "archive": 10548, "task": 10549, "benjam": 10550, "seg": 10551, "lawyer": 10552, "alum": 10553, "investing": 10554, "mie": 10555, "chez": 10556, "jp": 10557, "ake": 10558, "flam": 10559, "wallpaper": 10560, "âĻ¥ï¸ı": 10561, "tton": 10562, "chest": 10563, "favorites": 10564, "weigh": 10565, "coolest": 10566, "rating": 10567, "relevant": 10568, "logan": 10569, "maple": 10570, "runners": 10571, "prior": 10572, "people": 10573, "maur": 10574, "terrorist": 10575, "tested": 10576, "carnival": 10577, "suspen": 10578, "measure": 10579, "mv": 10580, "cybersecurity": 10581, "appren": 10582, "terrorism": 10583, "oz": 10584, "vital": 10585, "nies": 10586, "gonz": 10587, "funded": 10588, "twist": 10589, "assessment": 10590, "diesel": 10591, "enfor": 10592, "column": 10593, "addressing": 10594, "casts": 10595, "payment": 10596, "xton": 10597, "fier": 10598, ",'": 10599, "last": 10600, "nee": 10601, "unless": 10602, "close": 10603, "skill": 10604, "cuisine": 10605, "funeral": 10606, "tiles": 10607, "aun": 10608, "kru": 10609, "relationships": 10610, "ðŁĴ¯": 10611, "event": 10612, "âĢįâĻĤï¸ı": 10613, "kindness": 10614, "proposed": 10615, "acoustic": 10616, "aes": 10617, "defender": 10618, "dance": 10619, "htt": 10620, "wat": 10621, "voy": 10622, "ðŁ¤ĺ": 10623, "aus": 10624, "cliff": 10625, "searching": 10626, "beautifully": 10627, "inqu": 10628, "atl": 10629, "specialist": 10630, "ðŁIJ¶": 10631, "dai": 10632, "trails": 10633, "classics": 10634, "instant": 10635, "vous": 10636, "revenue": 10637, "march": 10638, "kirk": 10639, "fringe": 10640, "fireworks": 10641, "trivia": 10642, "âĺħ": 10643, "traction": 10644, "walter": 10645, "moto": 10646, "lily": 10647, "attitude": 10648, "climb": 10649, "scan": 10650, "savings": 10651, "cw": 10652, "faith": 10653, "credits": 10654, "abled": 10655, "graff": 10656, "autograph": 10657, "hehe": 10658, "ranch": 10659, "had": 10660, "rogers": 10661, "ðŁĮ¹": 10662, "fin": 10663, "requ": 10664, "folk": 10665, "additional": 10666, "lynn": 10667, "uber": 10668, "dollars": 10669, "logic": 10670, "worth": 10671, "som": 10672, "thesis": 10673, "pound": 10674, "bic": 10675, "stur": 10676, "ceram": 10677, "spencer": 10678, "entered": 10679, "vamp": 10680, "organized": 10681, "âľĪ": 10682, "pps": 10683, "tron": 10684, "mercedes": 10685, "noti": 10686, "competitive": 10687, "dow": 10688, "ousness": 10689, "victor": 10690, "grilled": 10691, "nai": 10692, "putin": 10693, "abra": 10694, "blame": 10695, "alexand": 10696, "animal": 10697, "decent": 10698, "pent": 10699, "interior": 10700, ":')": 10701, "butler": 10702, "ballet": 10703, "ðŁĴĶ": 10704, "albums": 10705, "downs": 10706, "lad": 10707, "sir": 10708, "plain": 10709, "pers": 10710, "blonde": 10711, "disc": 10712, "pakistan": 10713, "sement": 10714, "gaa": 10715, "wage": 10716, "chas": 10717, "mani": 10718, "cops": 10719, "territ": 10720, "lol": 10721, "laughter": 10722, "rivers": 10723, "magnificent": 10724, "lamp": 10725, "wb": 10726, "newsle": 10727, "charts": 10728, "blessing": 10729, "punch": 10730, "longest": 10731, "floral": 10732, "cutie": 10733, "farewell": 10734, "stopping": 10735, "mbb": 10736, "bud": 10737, "cheese": 10738, "decla": 10739, "sim": 10740, "mcdonald": 10741, "deter": 10742, "youth": 10743, "tch": 10744, "freder": 10745, "kindle": 10746, "fern": 10747, "ator": 10748, "asleep": 10749, "pond": 10750, "sprint": 10751, "pounds": 10752, "lazy": 10753, "ghe": 10754, "fundraising": 10755, "deadly": 10756, "grande": 10757, "doug": 10758, "hey": 10759, "linda": 10760, "considering": 10761, "ium": 10762, "golden": 10763, "vik": 10764, "authors": 10765, "diss": 10766, "ually": 10767, "appropriate": 10768, "morning": 10769, "yle": 10770, "honoring": 10771, "folio": 10772, "bec": 10773, "rebec": 10774, "finland": 10775, "formula": 10776, "cornwall": 10777, "shay": 10778, "causing": 10779, "blend": 10780, "signal": 10781, "tent": 10782, "kashmir": 10783, "nationals": 10784, "harmony": 10785, "scout": 10786, "accessi": 10787, "height": 10788, "medieval": 10789, "improvement": 10790, "kees": 10791, "practical": 10792, "card": 10793, "depar": 10794, "hun": 10795, "oming": 10796, "calgary": 10797, "stel": 10798, "bubble": 10799, "guru": 10800, "mah": 10801, "unexpe": 10802, "nh": 10803, "eda": 10804, "meat": 10805, "ige": 10806, "sio": 10807, "goddess": 10808, "inches": 10809, "tunes": 10810, "britt": 10811, "stion": 10812, "raj": 10813, "âĻ«": 10814, "mercy": 10815, "ðŁĴĺ": 10816, "sends": 10817, "iest": 10818, "polici": 10819, "vale": 10820, "reduced": 10821, "asap": 10822, "vijay": 10823, "defensive": 10824, "celebrations": 10825, "riders": 10826, "meditation": 10827, "harmon": 10828, "ging": 10829, "¡": 10830, "programming": 10831, "inau": 10832, "sudden": 10833, "mh": 10834, "replacement": 10835, "sku": 10836, "jar": 10837, "grades": 10838, "tast": 10839, "kitt": 10840, "branding": 10841, "kaw": 10842, "boot": 10843, "fought": 10844, "pays": 10845, "gf": 10846, "ization": 10847, "hop": 10848, "kk": 10849, "activist": 10850, "vend": 10851, "coastal": 10852, "chaos": 10853, "ðŁĶ´": 10854, "seme": 10855, "billboard": 10856, "lifting": 10857, "cumb": 10858, "scal": 10859, "ðŁĸ¤": 10860, "struck": 10861, "lv": 10862, "indiedev": 10863, "beaten": 10864, "jungle": 10865, "alright": 10866, "destiny": 10867, "ming": 10868, "kc": 10869, "chances": 10870, "oman": 10871, "qatar": 10872, "craf": 10873, "trained": 10874, "prix": 10875, "charm": 10876, "otive": 10877, "smu": 10878, "ec": 10879, "anders": 10880, "handed": 10881, "alban": 10882, "certainly": 10883, "arriving": 10884, "ize": 10885, "sai": 10886, "track": 10887, "painter": 10888, "humble": 10889, "appointment": 10890, "headline": 10891, "managing": 10892, "mod": 10893, "aspe": 10894, "andrea": 10895, "ä": 10896, "ethiop": 10897, "united": 10898, "exist": 10899, "bali": 10900, "kad": 10901, "nt": 10902, "dred": 10903, "rex": 10904, "recognize": 10905, "tampa": 10906, "beers": 10907, "atia": 10908, "heels": 10909, "note": 10910, "transportation": 10911, "turtle": 10912, "rede": 10913, "hiphop": 10914, "spicy": 10915, "spurs": 10916, "â¬ĩ": 10917, "corp": 10918, "thern": 10919, "toast": 10920, "hurry": 10921, "properties": 10922, "mage": 10923, "marco": 10924, "elements": 10925, "bouti": 10926, "syndrome": 10927, "msg": 10928, "developer": 10929, "graders": 10930, "heim": 10931, "resil": 10932, "offices": 10933, "delay": 10934, "dimen": 10935, "vintag": 10936, "barbara": 10937, "ðŁĺ±": 10938, "venezu": 10939, "cular": 10940, "faced": 10941, "barn": 10942, "ðŁĺĨ": 10943, "survivor": 10944, "worm": 10945, "confused": 10946, "passionate": 10947, "ر": 10948, "identify": 10949, "electricity": 10950, "souls": 10951, "bradley": 10952, "reportedly": 10953, "lunch": 10954, "shelf": 10955, "elia": 10956, "sweet": 10957, "smooth": 10958, "employment": 10959, "amel": 10960, "manhattan": 10961, "steam": 10962, "ounts": 10963, "yep": 10964, "living": 10965, "une": 10966, "describe": 10967, "cares": 10968, "manila": 10969, "shawn": 10970, "acted": 10971, "bash": 10972, "steven": 10973, "rest": 10974, "petition": 10975, "divine": 10976, "welsh": 10977, "race": 10978, "platinum": 10979, "ðŁĮ¸": 10980, "pb": 10981, "extraordinary": 10982, "solidarity": 10983, "mall": 10984, "onion": 10985, "scheduled": 10986, "gameof": 10987, "fergu": 10988, "dems": 10989, "norm": 10990, "pk": 10991, "trials": 10992, "policies": 10993, "publishing": 10994, "stole": 10995, "front": 10996, "character": 10997, "vania": 10998, "exce": 10999, "stie": 11000, "sca": 11001, "residential": 11002, "sailing": 11003, "ðŁĶ¥ðŁĶ¥ðŁĶ¥": 11004, "sponsors": 11005, "thick": 11006, "champagne": 11007, "shepher": 11008, "continuing": 11009, "venice": 11010, "perth": 11011, "nap": 11012, "aster": 11013, "yak": 11014, "unlimited": 11015, "choices": 11016, "neo": 11017, "hiv": 11018, "reporter": 11019, "brussels": 11020, "fold": 11021, "dys": 11022, "semi": 11023, "lawn": 11024, "italia": 11025, "wifi": 11026, "ask": 11027, "emed": 11028, "frame": 11029, "monitoring": 11030, "stead": 11031, "ida": 11032, "grin": 11033, "isa": 11034, "flip": 11035, "restric": 11036, "offensive": 11037, "attached": 11038, "dish": 11039, "why": 11040, "phillips": 11041, "greet": 11042, "pals": 11043, "mixtape": 11044, "vou": 11045, "fielder": 11046, "spark": 11047, "alberta": 11048, "glen": 11049, "cash": 11050, "sri": 11051, "uri": 11052, "rodri": 11053, "entrepreneurs": 11054, "climatechange": 11055, "psy": 11056, "dle": 11057, "ements": 11058, "linked": 11059, "netherlands": 11060, "accidentally": 11061, "opposition": 11062, "velvet": 11063, "rays": 11064, "cw": 11065, "omo": 11066, "mf": 11067, "lmfao": 11068, "newsletter": 11069, ":)": 11070, "toilet": 11071, "literature": 11072, "disp": 11073, "philip": 11074, "uniform": 11075, "suddenly": 11076, "header": 11077, "cooler": 11078, "---": 11079, "proud": 11080, "brig": 11081, "nissan": 11082, "scientist": 11083, "jah": 11084, "concentr": 11085, "packs": 11086, "appointed": 11087, "soap": 11088, "engage": 11089, "chose": 11090, "âĻ¡": 11091, "setup": 11092, "jealous": 11093, "harry": 11094, "gation": 11095, "tunnel": 11096, "temp": 11097, "oscars": 11098, "decade": 11099, "recommended": 11100, "children": 11101, "aba": 11102, "anxiety": 11103, "vements": 11104, "salon": 11105, "photoo": 11106, "organiz": 11107, "machines": 11108, "abs": 11109, "ville": 11110, "hype": 11111, "tiff": 11112, "emerging": 11113, "avgeek": 11114, "[#": 11115, "contribution": 11116, "brady": 11117, "resto": 11118, "gmail": 11119, "fitz": 11120, "photoshoot": 11121, "helmet": 11122, "ht": 11123, "elegant": 11124, "uganda": 11125, "nursing": 11126, "orleans": 11127, "penn": 11128, "nah": 11129, "footage": 11130, "ema": 11131, "wo": 11132, "wad": 11133, "concerns": 11134, "vere": 11135, "remark": 11136, "whoever": 11137, "strang": 11138, "pt": 11139, "quit": 11140, "shang": 11141, "history": 11142, "sick": 11143, "permanent": 11144, "illness": 11145, "cold": 11146, "vision": 11147, "hem": 11148, "arrow": 11149, "convic": 11150, "pink": 11151, "occup": 11152, "bald": 11153, "exhau": 11154, "uof": 11155, "amo": 11156, "ont": 11157, "ãĥ»": 11158, "adopt": 11159, "laid": 11160, "smoked": 11161, "interpre": 11162, "essenti": 11163, "associated": 11164, "bd": 11165, "bby": 11166, "fier": 11167, "install": 11168, "diplom": 11169, "conditi": 11170, "cf": 11171, "wak": 11172, "anya": 11173, "graci": 11174, "fisher": 11175, "sss": 11176, "apr": 11177, "ilit": 11178, "musician": 11179, "symphony": 11180, "cord": 11181, "hack": 11182, "legi": 11183, "lv": 11184, "blessings": 11185, "humor": 11186, "scra": 11187, "eti": 11188, "minster": 11189, "travelling": 11190, "bush": 11191, "jewellery": 11192, "lime": 11193, "!!!": 11194, "pregnant": 11195, "pee": 11196, "lob": 11197, "capital": 11198, "ipa": 11199, "pencil": 11200, "labor": 11201, "ducks": 11202, "proudly": 11203, "wedding": 11204, "derek": 11205, "mw": 11206, "peg": 11207, "valentine": 11208, "angu": 11209, "retreat": 11210, "prospect": 11211, "danger": 11212, "vulner": 11213, "upset": 11214, ",#": 11215, "srk": 11216, "xim": 11217, "thursday": 11218, "nfl": 11219, "kisses": 11220, "reds": 11221, "crack": 11222, "reward": 11223, "cu": 11224, "kok": 11225, "mete": 11226, "abandoned": 11227, "itt": 11228, "meals": 11229, "spell": 11230, "stanbul": 11231, "delays": 11232, "rum": 11233, "leop": 11234, "gum": 11235, "nova": 11236, "superman": 11237, "chick": 11238, "mis": 11239, "dramatic": 11240, "innocent": 11241, "rounds": 11242, "rec": 11243, "autism": 11244, "bangladesh": 11245, "moral": 11246, "movie": 11247, "spoo": 11248, "kla": 11249, "âĥ£": 11250, "outing": 11251, "messi": 11252, "abroad": 11253, "lookin": 11254, "aim": 11255, "qi": 11256, "stack": 11257, "collage": 11258, "à¯": 11259, "hudson": 11260, "scan": 11261, "hoe": 11262, "chau": 11263, "occur": 11264, "commander": 11265, "holes": 11266, "ðŁİĦ": 11267, "bias": 11268, "von": 11269, "sticker": 11270, "mak": 11271, "responsibility": 11272, "columbus": 11273, "saint": 11274, "edmon": 11275, "racism": 11276, "farms": 11277, "wen": 11278, "gulf": 11279, "mayo": 11280, "!!!!!!!!": 11281, "corporation": 11282, "bachel": 11283, "ela": 11284, "internal": 11285, "jeep": 11286, "follows": 11287, "dialogue": 11288, "derer": 11289, "smartphone": 11290, "helen": 11291, "richmond": 11292, "equity": 11293, "sland": 11294, "bg": 11295, "near": 11296, "avi": 11297, "memphis": 11298, "weir": 11299, "discussed": 11300, "badge": 11301, "pup": 11302, "mistake": 11303, "phenomen": 11304, "unite": 11305, "ðŁĽ": 11306, "depic": 11307, "rides": 11308, "inaugu": 11309, "nat": 11310, "softwitter": 11311, "combination": 11312, "gospel": 11313, "âļ¾": 11314, "admission": 11315, "retrogaming": 11316, "ðŁIJ¾": 11317, "schu": 11318, "mbo": 11319, "junction": 11320, "alarm": 11321, "à¦": 11322, "grac": 11323, "khali": 11324, "kul": 11325, "male": 11326, "caption": 11327, "wish": 11328, "tere": 11329, "corps": 11330, "rubber": 11331, "playstation": 11332, "erin": 11333, "efficient": 11334, "lor": 11335, "jokes": 11336, "inary": 11337, "norman": 11338, "luis": 11339, "inaugural": 11340, "ched": 11341, "âļ½ï¸ı": 11342, "dip": 11343, "toe": 11344, "strat": 11345, "aac": 11346, "amu": 11347, "pier": 11348, "cott": 11349, "command": 11350, "tten": 11351, "snoo": 11352, "cube": 11353, "closes": 11354, "classical": 11355, "sword": 11356, "expression": 11357, "reaching": 11358, "napp": 11359, "cost": 11360, "affect": 11361, "rico": 11362, "gif": 11363, "breathe": 11364, "tribe": 11365, "ortho": 11366, "hay": 11367, "lg": 11368, "fries": 11369, "nm": 11370, "hiding": 11371, "richards": 11372, "ende": 11373, "micro": 11374, "capitol": 11375, "copy": 11376, "rom": 11377, "regime": 11378, "maryland": 11379, "taxi": 11380, "dial": 11381, "embarra": 11382, "unbeliev": 11383, "cht": 11384, "vs": 11385, "elimin": 11386, "odd": 11387, "penny": 11388, "soundtrack": 11389, "lings": 11390, "transition": 11391, "remaining": 11392, "ais": 11393, "malik": 11394, "?!?": 11395, "random": 11396, "defend": 11397, "ultra": 11398, "trum": 11399, "dancer": 11400, "stol": 11401, "drive": 11402, "aver": 11403, "roast": 11404, "definition": 11405, "sean": 11406, "excitement": 11407, "particul": 11408, "surely": 11409, "shav": 11410, "bery": 11411, "dishes": 11412, "comm": 11413, "isol": 11414, "iam": 11415, "obli": 11416, "ghost": 11417, "hughes": 11418, "chiefs": 11419, "bas": 11420, "conservative": 11421, "special": 11422, "femin": 11423, "shri": 11424, "nancy": 11425, "intel": 11426, "tune": 11427, "ðŁĩª": 11428, "joel": 11429, "ggle": 11430, "moto": 11431, "ðŁĺĶ": 11432, "buck": 11433, "dag": 11434, "anticip": 11435, "montana": 11436, "guid": 11437, "frog": 11438, "ecraft": 11439, "ope": 11440, "drives": 11441, "numer": 11442, "xy": 11443, "colorful": 11444, "wednesdaywisdom": 11445, "illumin": 11446, "beyon": 11447, "inaugur": 11448, "deeply": 11449, "prefer": 11450, "fortune": 11451, "cooked": 11452, "tible": 11453, "âĺķ": 11454, "sweater": 11455, "itter": 11456, "tty": 11457, "ui": 11458, "gie": 11459, "complic": 11460, "~~": 11461, "taxes": 11462, "cups": 11463, "diverse": 11464, "samanth": 11465, "âłĢâłĢ": 11466, "baking": 11467, "symp": 11468, "wai": 11469, "behalf": 11470, "mercur": 11471, "travels": 11472, "ðŁİīðŁİ": 11473, "oria": 11474, "engaged": 11475, "jumping": 11476, "retired": 11477, "naked": 11478, "puni": 11479, "speedway": 11480, "sciences": 11481, "rehearsal": 11482, "onym": 11483, "dyou": 11484, "plates": 11485, "rati": 11486, "krish": 11487, "jazz": 11488, "carol": 11489, "raf": 11490, "penalty": 11491, "timeline": 11492, "ruby": 11493, "engineers": 11494, "raf": 11495, "belle": 11496, "dose": 11497, "cheon": 11498, "escap": 11499, "meg": 11500, "rank": 11501, "ord": 11502, "megan": 11503, "merch": 11504, "eclipse": 11505, "âĺºï¸ı": 11506, "pledge": 11507, "kirk": 11508, "persi": 11509, "leicester": 11510, "sak": 11511, "wk": 11512, "safely": 11513, "yyy": 11514, "jet": 11515, "promised": 11516, "jc": 11517, "enne": 11518, "noah": 11519, "reno": 11520, "rea": 11521, "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ": 11522, "trail": 11523, "ðŁijĢ": 11524, "fd": 11525, "sooo": 11526, "rimin": 11527, "wk": 11528, "า": 11529, "ial": 11530, "xox": 11531, "biscu": 11532, "dale": 11533, "fandom": 11534, "participating": 11535, "flag": 11536, "privilege": 11537, "peach": 11538, "machine": 11539, "boston": 11540, "gross": 11541, "og": 11542, "miracle": 11543, "adoption": 11544, "uss": 11545, "monsters": 11546, "beij": 11547, "clarke": 11548, "pushing": 11549, "praying": 11550, "aro": 11551, "dn": 11552, "ellis": 11553, "apollo": 11554, "odds": 11555, "refugee": 11556, "tow": 11557, "bp": 11558, "ðŁĩ¬ðŁĩ§": 11559, "hend": 11560, "appeared": 11561, "membership": 11562, "pean": 11563, "dum": 11564, "violent": 11565, "vy": 11566, "potatoes": 11567, "aww": 11568, "greetings": 11569, "tts": 11570, "acon": 11571, "shane": 11572, "photographed": 11573, "crab": 11574, "temperatures": 11575, "cuba": 11576, "cfc": 11577, "welcom": 11578, "hel": 11579, "innings": 11580, "mk": 11581, "code": 11582, "knock": 11583, "grass": 11584, "swedish": 11585, "pta": 11586, "icky": 11587, "vat": 11588, "lining": 11589, "sq": 11590, "sap": 11591, "arc": 11592, "announcing": 11593, "skins": 11594, "cityof": 11595, "bring": 11596, "cox": 11597, "gamer": 11598, "itarian": 11599, "ida": 11600, "hd": 11601, "rosse": 11602, "sadly": 11603, "geo": 11604, "âļ¡ï¸ı": 11605, "tags": 11606, "father": 11607, "change": 11608, "lance": 11609, "whiskey": 11610, "adelaide": 11611, "tec": 11612, "stickers": 11613, "market": 11614, "classy": 11615, "badass": 11616, "florence": 11617, "liner": 11618, "frost": 11619, "kate": 11620, "acon": 11621, "scandal": 11622, "essex": 11623, "ðŁĺı": 11624, "vivi": 11625, "drill": 11626, "bloggers": 11627, "recommend": 11628, "dha": 11629, "acres": 11630, "roma": 11631, "buy": 11632, "grocer": 11633, "eria": 11634, "mahar": 11635, "ffer": 11636, "patterns": 11637, "veri": 11638, "compu": 11639, "stev": 11640, "anga": 11641, "mentor": 11642, "doo": 11643, "itali": 11644, "cdnpoli": 11645, "only": 11646, "conduct": 11647, "electro": 11648, "def": 11649, "whale": 11650, "preparation": 11651, "bicycle": 11652, "viral": 11653, "turnout": 11654, "brass": 11655, "quad": 11656, "hospitality": 11657, "packaging": 11658, "dency": 11659, "cemetery": 11660, "aboard": 11661, "dreaming": 11662, "picture": 11663, "tall": 11664, "invent": 11665, "admi": 11666, "oe": 11667, "temps": 11668, "quan": 11669, "fundam": 11670, "promp": 11671, "residence": 11672, "mud": 11673, "souri": 11674, "âĦ¢": 11675, "graffiti": 11676, "gif": 11677, "dnd": 11678, "comp": 11679, "swar": 11680, "peeps": 11681, "palestine": 11682, "devils": 11683, "sang": 11684, "assistance": 11685, "bike": 11686, "mississi": 11687, "interviewed": 11688, "nephew": 11689, "drums": 11690, "vand": 11691, "gentlemen": 11692, "nsw": 11693, "insta": 11694, "lebanon": 11695, "eeee": 11696, "olivia": 11697, "very": 11698, "rough": 11699, "industries": 11700, "mation": 11701, "ðŁĺĴ": 11702, "barrel": 11703, "nay": 11704, "pops": 11705, "modern": 11706, "illy": 11707, "arest": 11708, "onents": 11709, "protecting": 11710, "vans": 11711, "eo": 11712, "vikings": 11713, "restaurants": 11714, "reck": 11715, "jackie": 11716, "andrew": 11717, "willing": 11718, "heath": 11719, "citizen": 11720, "discrimin": 11721, "à¹Ī": 11722, "stuart": 11723, "mys": 11724, "hip": 11725, "transp": 11726, "\"?": 11727, "tex": 11728, "sushi": 11729, "ked": 11730, "crossed": 11731, "distur": 11732, "pedia": 11733, "fate": 11734, "somehow": 11735, "moth": 11736, "processing": 11737, "iss": 11738, "rin": 11739, "uts": 11740, "yyc": 11741, "vert": 11742, "lgbt": 11743, "reid": 11744, "onto": 11745, "arabia": 11746, "habitat": 11747, "==": 11748, "streak": 11749, "simpson": 11750, "addiction": 11751, "wimble": 11752, "delivers": 11753, "challenging": 11754, "ðŁİ¶": 11755, "franch": 11756, "edu": 11757, "sme": 11758, "aids": 11759, "hurst": 11760, "tham": 11761, "tarian": 11762, "remembered": 11763, "palestinian": 11764, "fees": 11765, "trum": 11766, "sketch": 11767, "uru": 11768, "fitting": 11769, "jesse": 11770, "ðŁĶ¥ðŁĶ¥": 11771, "--------": 11772, "bach": 11773, "icia": 11774, "colored": 11775, "dah": 11776, "associate": 11777, "intel": 11778, "seller": 11779, "pu": 11780, "stuffed": 11781, "acs": 11782, "bs": 11783, "shin": 11784, "cooperation": 11785, "certificate": 11786, "abu": 11787, "ingredients": 11788, "rev": 11789, "inge": 11790, "elder": 11791, "christian": 11792, "bundle": 11793, "thic": 11794, "dirt": 11795, "beijing": 11796, "commit": 11797, "teddy": 11798, "edu": 11799, "today": 11800, "sfield": 11801, "wyn": 11802, "confirms": 11803, "loo": 11804, "jv": 11805, "eness": 11806, "alpha": 11807, "virus": 11808, "arium": 11809, "grind": 11810, "bridges": 11811, "introduction": 11812, "polls": 11813, "bacter": 11814, "zach": 11815, "terminal": 11816, "raiders": 11817, "flavor": 11818, "zombie": 11819, "vod": 11820, "spreading": 11821, "gameofthrones": 11822, "efficiency": 11823, "lately": 11824, "alem": 11825, "tweet": 11826, "crimes": 11827, "cler": 11828, "dey": 11829, "dged": 11830, "hyun": 11831, "payments": 11832, "circus": 11833, "ðŁĺŃðŁĺŃ": 11834, "missouri": 11835, "lub": 11836, "episodes": 11837, "cage": 11838, "pos": 11839, "matching": 11840, "tumblr": 11841, "lined": 11842, "gest": 11843, "ambi": 11844, "narr": 11845, "ington": 11846, "regul": 11847, "blown": 11848, "isle": 11849, "coco": 11850, "ondon": 11851, "joshua": 11852, "touring": 11853, "sma": 11854, "sausage": 11855, "bestfriend": 11856, "boeing": 11857, "desire": 11858, "savage": 11859, "rapper": 11860, "devo": 11861, "tear": 11862, "takeover": 11863, "cowboys": 11864, "poker": 11865, "parag": 11866, "ppe": 11867, "hint": 11868, "wears": 11869, "seth": 11870, "roles": 11871, "lanc": 11872, "manga": 11873, "format": 11874, "flyer": 11875, "cay": 11876, "moor": 11877, "bake": 11878, "splash": 11879, "vad": 11880, "kerala": 11881, "proceeds": 11882, "silly": 11883, "reflection": 11884, "distr": 11885, "wid": 11886, "suit": 11887, "civic": 11888, "yankees": 11889, "byn": 11890, "migration": 11891, "distin": 11892, "orch": 11893, "femini": 11894, "qualifying": 11895, "turi": 11896, "obe": 11897, "hundred": 11898, "crap": 11899, "wang": 11900, "mathemat": 11901, "bure": 11902, "exposure": 11903, "ferguson": 11904, "semester": 11905, "reserv": 11906, "plym": 11907, "ahu": 11908, "facial": 11909, "wax": 11910, "worried": 11911, "cab": 11912, "vio": 11913, "asa": 11914, "cod": 11915, "topics": 11916, "pcs": 11917, "halo": 11918, "rescued": 11919, "horizon": 11920, "ark": 11921, "âļª": 11922, "holly": 11923, "elf": 11924, "ulti": 11925, "pup": 11926, "qualified": 11927, "attendance": 11928, "atively": 11929, "destroy": 11930, "yc": 11931, "forth": 11932, "photooftheday": 11933, "cents": 11934, "iceland": 11935, "measures": 11936, "desk": 11937, "portfolio": 11938, "articles": 11939, "directors": 11940, "datab": 11941, "ew": 11942, "creepy": 11943, "ounding": 11944, "honoured": 11945, "mist": 11946, "jit": 11947, "mentioned": 11948, "portable": 11949, "itic": 11950, "dann": 11951, "fridayfeeling": 11952, "amid": 11953, "tiger": 11954, "scrip": 11955, "helicopter": 11956, "hardware": 11957, "explor": 11958, "workplace": 11959, "austria": 11960, "beatles": 11961, "bernar": 11962, "spider": 11963, "disco": 11964, "cult": 11965, "limits": 11966, "shortly": 11967, "final": 11968, "ninja": 11969, "luke": 11970, "lebron": 11971, "walmart": 11972, "oil": 11973, "vanilla": 11974, "shire": 11975, "yeg": 11976, "aky": 11977, "cs": 11978, "bler": 11979, "collected": 11980, "tg": 11981, "rolled": 11982, "specials": 11983, "bff": 11984, "pierre": 11985, "shim": 11986, "vier": 11987, "flashback": 11988, "restoration": 11989, "individuals": 11990, "prod": 11991, "freaking": 11992, "turer": 11993, "oa": 11994, "refre": 11995, "moroc": 11996, "greet": 11997, "reyn": 11998, "careful": 11999, "ouring": 12000, "ush": 12001, "isd": 12002, "gill": 12003, "view": 12004, "thunderstorm": 12005, "bled": 12006, "picnic": 12007, "guardi": 12008, "pig": 12009, "ark": 12010, "sylvania": 12011, "banned": 12012, "ucl": 12013, "vijay": 12014, "orium": 12015, "avengers": 12016, "believes": 12017, "eur": 12018, "monument": 12019, "concerned": 12020, "labs": 12021, "berg": 12022, "aap": 12023, "vish": 12024, "singles": 12025, "cancel": 12026, "zel": 12027, "arab": 12028, "ruth": 12029, "tooth": 12030, "arta": 12031, "shaf": 12032, "chairs": 12033, "rack": 12034, "diseases": 12035, "crowd": 12036, "cly": 12037, "flex": 12038, "christma": 12039, "artificial": 12040, "tomat": 12041, "fine": 12042, "draws": 12043, "advocate": 12044, "france": 12045, "ÙĬ": 12046, "ðŁĺ³": 12047, "heavy": 12048, "sour": 12049, "comprehen": 12050, "noble": 12051, "aap": 12052, "hindu": 12053, "coral": 12054, "gars": 12055, "owen": 12056, "nl": 12057, "stall": 12058, "yellow": 12059, "marina": 12060, "inver": 12061, "support": 12062, "tough": 12063, "promises": 12064, "pie": 12065, "masterpiece": 12066, "score": 12067, "force": 12068, "mortg": 12069, "cryptocurrency": 12070, "ox": 12071, "rors": 12072, "rockin": 12073, "provin": 12074, "hog": 12075, "nostal": 12076, "oakland": 12077, "patrick": 12078, "inclusion": 12079, "traffic": 12080, "ahmed": 12081, "aha": 12082, "luxury": 12083, "consecu": 12084, "demon": 12085, "âĸº": 12086, "blowing": 12087, "stag": 12088, ":\"": 12089, "encourage": 12090, "bene": 12091, "skull": 12092, "dodge": 12093, "buster": 12094, "kinson": 12095, "witne": 12096, "error": 12097, "lowest": 12098, "fellow": 12099, "à°": 12100, "shre": 12101, "blur": 12102, "virgin": 12103, "composer": 12104, "slip": 12105, "mornings": 12106, "gains": 12107, "table": 12108, "grain": 12109, "arist": 12110, "brazilian": 12111, "wwe": 12112, "tues": 12113, "ribbon": 12114, "anag": 12115, "dist": 12116, "sacrif": 12117, "embrace": 12118, "entrepreneur": 12119, "affili": 12120, "deo": 12121, "tali": 12122, "tourist": 12123, "fatal": 12124, "ìĬ": 12125, "automatic": 12126, "ðŁĩµ": 12127, "weak": 12128, "welfare": 12129, "confirm": 12130, "benjamin": 12131, "fights": 12132, "alleged": 12133, "mead": 12134, "struggling": 12135, "prosecu": 12136, "chef": 12137, "è": 12138, "proposal": 12139, "ern": 12140, "ðŁĺĦ": 12141, "dyk": 12142, "ongs": 12143, "hong": 12144, "mack": 12145, "melon": 12146, "onent": 12147, "rush": 12148, "dap": 12149, "toler": 12150, "propag": 12151, "cze": 12152, "translation": 12153, "wallet": 12154, "cottage": 12155, "sail": 12156, "constitution": 12157, "ðŁĴĢ": 12158, "munici": 12159, "favor": 12160, "stormhour": 12161, "ih": 12162, "ðŁĺĮ": 12163, "approaching": 12164, "pinned": 12165, "jed": 12166, "nigerian": 12167, "nach": 12168, "shat": 12169, "particularly": 12170, "mcdon": 12171, "cameras": 12172, "annie": 12173, "administr": 12174, "heat": 12175, "electrical": 12176, "charming": 12177, "gibson": 12178, "boutique": 12179, "exposed": 12180, "actor": 12181, "pillow": 12182, "beaches": 12183, "genuine": 12184, "margaret": 12185, "bennett": 12186, "louisi": 12187, "positions": 12188, "ely": 12189, "shiny": 12190, "tention": 12191, "architect": 12192, "rental": 12193, "acqui": 12194, "google": 12195, "subway": 12196, "moment": 12197, "ðŁļ¨": 12198, "rim": 12199, "methods": 12200, "cycli": 12201, "norfolk": 12202, "ÙĪ": 12203, "overwhel": 12204, "rapid": 12205, "wear": 12206, "happybirthday": 12207, "progressive": 12208, "ðŁĴ¥": 12209, "cogn": 12210, "papa": 12211, "fool": 12212, "philosophy": 12213, "polar": 12214, "jimmy": 12215, "wig": 12216, "ðŁĴĭ": 12217, "operating": 12218, "reduction": 12219, "phi": 12220, "flags": 12221, "tothe": 12222, "odi": 12223, "ares": 12224, "koo": 12225, "kang": 12226, "arkansas": 12227, "ashton": 12228, "wimbledon": 12229, "scifi": 12230, "attractive": 12231, "mississippi": 12232, "logists": 12233, "ralph": 12234, "label": 12235, "graduates": 12236, "maha": 12237, "hometown": 12238, "âľĮï¸ı": 12239, "founded": 12240, "onthe": 12241, "liz": 12242, "transl": 12243, "minimum": 12244, "presti": 12245, "tam": 12246, "generations": 12247, "rebel": 12248, "journalists": 12249, "param": 12250, "mcm": 12251, "acrylic": 12252, "deaths": 12253, "tesla": 12254, "wt": 12255, "bryant": 12256, "jerus": 12257, "istanbul": 12258, "muhammad": 12259, "riley": 12260, "kris": 12261, "workshops": 12262, "iso": 12263, "counts": 12264, "stret": 12265, "protected": 12266, "trinity": 12267, "manual": 12268, "rhin": 12269, "ril": 12270, "pleasant": 12271, "lemon": 12272, "nerd": 12273, "harder": 12274, "darren": 12275, "bury": 12276, "rah": 12277, "basis": 12278, "migu": 12279, "occasion": 12280, "lists": 12281, "âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı": 12282, "eb": 12283, "decre": 12284, "hampton": 12285, "ìĿ´": 12286, "travis": 12287, "transform": 12288, "puerto": 12289, "nhl": 12290, "avoc": 12291, "trips": 12292, "unexpected": 12293, "vet": 12294, "didyou": 12295, "barber": 12296, "stages": 12297, "mson": 12298, "represented": 12299, "fort": 12300, "lal": 12301, "pple": 12302, "nicely": 12303, "ignore": 12304, "quil": 12305, "quinn": 12306, "hk": 12307, "carrier": 12308, "reminded": 12309, "among": 12310, "passenger": 12311, "ellen": 12312, "guez": 12313, "scape": 12314, "mural": 12315, "youngest": 12316, "mash": 12317, "dill": 12318, "routine": 12319, "stainless": 12320, "jackson": 12321, "gandhi": 12322, "thal": 12323, "oners": 12324, "editorial": 12325, "conversations": 12326, "sdale": 12327, "automation": 12328, "ike": 12329, "าà¸": 12330, "ðŁĩª": 12331, "haul": 12332, "laying": 12333, "mentions": 12334, "amen": 12335, "abortion": 12336, "ibi": 12337, "counties": 12338, "catherine": 12339, "mands": 12340, "jame": 12341, "roller": 12342, "aut": 12343, "nam": 12344, "ological": 12345, "ception": 12346, "ranking": 12347, "toxic": 12348, "snacks": 12349, "victorian": 12350, "bangkok": 12351, "psychology": 12352, "reg": 12353, "angela": 12354, "respond": 12355, "style": 12356, "sophie": 12357, "dakota": 12358, "achieved": 12359, "marked": 12360, "imperial": 12361, "inas": 12362, "gloves": 12363, "slim": 12364, "confident": 12365, "attacked": 12366, "gger": 12367, "lonely": 12368, "valentinesday": 12369, "reb": 12370, "craftbeer": 12371, "origin": 12372, "zimbab": 12373, "ceiling": 12374, "teens": 12375, "otherwise": 12376, "wb": 12377, "fers": 12378, "daysof": 12379, "advisor": 12380, "yah": 12381, "âĻª": 12382, "ender": 12383, "republicans": 12384, "ava": 12385, "skirt": 12386, "pipel": 12387, "chie": 12388, "jane": 12389, "jax": 12390, "ðŁĺĭ": 12391, "âľĬ": 12392, "jays": 12393, "brett": 12394, "balo": 12395, "crucial": 12396, "dhar": 12397, "asis": 12398, "deau": 12399, "lloyd": 12400, "chatting": 12401, "âĿĦï¸ı": 12402, "relay": 12403, "remarkable": 12404, "ns": 12405, "wet": 12406, "brisbane": 12407, "ðŁĶ´": 12408, "tionally": 12409, "fk": 12410, "layer": 12411, "household": 12412, "consecutive": 12413, "esis": 12414, "pendant": 12415, "stir": 12416, "critic": 12417, "sugar": 12418, "photoshop": 12419, "pares": 12420, "artistic": 12421, "dodgers": 12422, "cun": 12423, "crafted": 12424, "amend": 12425, "boat": 12426, "âŃIJï¸ı": 12427, "egyptian": 12428, "saw": 12429, "trage": 12430, "smaller": 12431, "oxy": 12432, "paired": 12433, "next": 12434, "ires": 12435, "taco": 12436, "oy": 12437, "uc": 12438, "sti": 12439, "aerial": 12440, "://": 12441, "dro": 12442, "dotcom": 12443, "ggins": 12444, "rpg": 12445, "aye": 12446, "lean": 12447, "striker": 12448, "lobby": 12449, "protests": 12450, "priority": 12451, "congress": 12452, "amate": 12453, "invit": 12454, "rington": 12455, "mommy": 12456, "thus": 12457, "allowing": 12458, "pioneer": 12459, "enforcement": 12460, "gori": 12461, "talk": 12462, "drag": 12463, "dumb": 12464, "bullet": 12465, "sange": 12466, "ery": 12467, "targets": 12468, "ðŁĩ¦": 12469, "heather": 12470, "consider": 12471, "seafood": 12472, "vest": 12473, "risks": 12474, "%.": 12475, "pg": 12476, "sacred": 12477, "heating": 12478, "kicked": 12479, "ttot": 12480, ".-": 12481, "chandi": 12482, "coven": 12483, "pool": 12484, "pulse": 12485, "ia": 12486, "roster": 12487, "shakespeare": 12488, "esa": 12489, "cargo": 12490, "peanut": 12491, "troop": 12492, "action": 12493, "tablet": 12494, "homework": 12495, "castle": 12496, "struction": 12497, "musicians": 12498, "freezing": 12499, "butt": 12500, "justinbieber": 12501, "jj": 12502, "bahrain": 12503, "anthem": 12504, "audit": 12505, "didyouknow": 12506, "navig": 12507, "guidance": 12508, "âĸ¶": 12509, "turf": 12510, "nun": 12511, "fications": 12512, "yemen": 12513, "charging": 12514, "xc": 12515, "broncos": 12516, "subur": 12517, "pale": 12518, "boring": 12519, "amongst": 12520, "forthe": 12521, "emper": 12522, "omfg": 12523, "pj": 12524, "expecting": 12525, "ðŁĴ«": 12526, "stl": 12527, "admin": 12528, "expectations": 12529, "swan": 12530, "shoot": 12531, "ooooo": 12532, "minent": 12533, "ãĢIJ": 12534, "wallace": 12535, "stang": 12536, "saturday": 12537, "adopted": 12538, "doubles": 12539, "homie": 12540, "omez": 12541, "dhan": 12542, "venture": 12543, "surrounding": 12544, "file": 12545, "mobility": 12546, "dees": 12547, "wski": 12548, "brooke": 12549, "embro": 12550, "remembers": 12551, "kara": 12552, "testim": 12553, "botan": 12554, "mtv": 12555, "sacrifice": 12556, "jerusalem": 12557, "dl": 12558, "´": 12559, "properly": 12560, "ilion": 12561, "asi": 12562, "legit": 12563, "cope": 12564, "mcla": 12565, "recycling": 12566, "larger": 12567, "ðŁĴĵ": 12568, "patric": 12569, "generous": 12570, "jared": 12571, "pf": 12572, "molly": 12573, "thomas": 12574, "judges": 12575, "hb": 12576, "sorts": 12577, "blvd": 12578, "oven": 12579, "entering": 12580, "planes": 12581, "beet": 12582, "integration": 12583, "booked": 12584, "freed": 12585, "vern": 12586, "ashes": 12587, "topped": 12588, "depot": 12589, "welcomed": 12590, "rena": 12591, "mick": 12592, "dand": 12593, "seeks": 12594, "gamer": 12595, "rankings": 12596, "rene": 12597, "mut": 12598, "whisky": 12599, "firefighters": 12600, "gues": 12601, "gather": 12602, "tourney": 12603, "demen": 12604, "yang": 12605, "newton": 12606, "automotive": 12607, "backyard": 12608, "detailed": 12609, "mist": 12610, "tobac": 12611, "fiber": 12612, "unusual": 12613, "gratitude": 12614, "spare": 12615, "neys": 12616, ":*": 12617, "peri": 12618, "floating": 12619, "finalist": 12620, "donating": 12621, "dress": 12622, "broad": 12623, "bethe": 12624, "economics": 12625, "taiwan": 12626, "edwards": 12627, "plug": 12628, "prairi": 12629, "valen": 12630, "baba": 12631, "fad": 12632, "anas": 12633, "harper": 12634, "disorder": 12635, "applied": 12636, "patt": 12637, "bikin": 12638, "liver": 12639, "curi": 12640, "caroline": 12641, "anner": 12642, "julian": 12643, "walking": 12644, "malcol": 12645, "screenshot": 12646, "coding": 12647, "skincare": 12648, "activists": 12649, "mysterious": 12650, "exact": 12651, "blocking": 12652, "mercury": 12653, "batter": 12654, "dump": 12655, "âľĮ": 12656, "ense": 12657, "lish": 12658, "ridiculous": 12659, "protesters": 12660, "ðŁĻĪ": 12661, "lust": 12662, "sweat": 12663, "ass": 12664, "alike": 12665, "cody": 12666, "rements": 12667, "winds": 12668, "aspir": 12669, "vienna": 12670, "pray": 12671, "...@": 12672, "boi": 12673, "candle": 12674, "assists": 12675, "tee": 12676, "derson": 12677, "pony": 12678, "fence": 12679, "conspir": 12680, "âĺħâĺħ": 12681, "ooth": 12682, "epic": 12683, "barely": 12684, "aunt": 12685, "bam": 12686, "diamonds": 12687, "endless": 12688, "screens": 12689, "cancer": 12690, "gro": 12691, "pst": 12692, "prospec": 12693, "mosque": 12694, "helpful": 12695, "ouri": 12696, "brother": 12697, "gujar": 12698, "cristi": 12699, "inez": 12700, "towers": 12701, "addresses": 12702, "gray": 12703, "burton": 12704, "retweeted": 12705, "ðŁ¤Ķ": 12706, "nity": 12707, "duck": 12708, "supervis": 12709, "joan": 12710, "kinder": 12711, "sanctu": 12712, "pied": 12713, "âı°": 12714, "łï¸ı": 12715, "mati": 12716, "revenge": 12717, "cester": 12718, "elife": 12719, "designers": 12720, "backed": 12721, "boli": 12722, "weight": 12723, "couch": 12724, "sures": 12725, "sits": 12726, "shrimp": 12727, "lagos": 12728, "authorities": 12729, "osity": 12730, "holly": 12731, "computing": 12732, "factors": 12733, "abe": 12734, "panels": 12735, "ramad": 12736, "sentence": 12737, "mission": 12738, "holm": 12739, "rb": 12740, "dads": 12741, "shanghai": 12742, "money": 12743, "sheets": 12744, "skate": 12745, "threw": 12746, "cupcakes": 12747, "infinite": 12748, "lis": 12749, "practicing": 12750, "essay": 12751, "kai": 12752, "asci": 12753, "mob": 12754, "ugh": 12755, "holmes": 12756, "regg": 12757, "ikh": 12758, "mock": 12759, "collections": 12760, "pep": 12761, "ova": 12762, "salt": 12763, "nandez": 12764, "coy": 12765, "threats": 12766, "texts": 12767, "cinnam": 12768, "pregnancy": 12769, "pending": 12770, "stamp": 12771, "flower": 12772, "gis": 12773, "agreed": 12774, "payne": 12775, "rover": 12776, "phra": 12777, "soft": 12778, "ffin": 12779, "fathers": 12780, "passengers": 12781, "aways": 12782, "ala": 12783, "hes": 12784, "livan": 12785, "ins": 12786, "samuel": 12787, "ingui": 12788, "hof": 12789, "jj": 12790, "chennai": 12791, "catal": 12792, "omic": 12793, "heath": 12794, "niece": 12795, "pumped": 12796, "integrated": 12797, "arel": 12798, "nom": 12799, "productivity": 12800, "wanting": 12801, "visa": 12802, "diana": 12803, "twil": 12804, "itv": 12805, "camps": 12806, "rowing": 12807, "dley": 12808, "blackand": 12809, "guards": 12810, "bells": 12811, "reverse": 12812, "vibe": 12813, "ricky": 12814, "moss": 12815, "nyt": 12816, "âĺĢï¸ı": 12817, "elle": 12818, "troy": 12819, "cudd": 12820, "evan": 12821, "womens": 12822, "foto": 12823, "mistakes": 12824, "wicked": 12825, "mil": 12826, "cled": 12827, "memes": 12828, "cosmo": 12829, "scholar": 12830, "reno": 12831, "ðŁĺĢ": 12832, "vents": 12833, "#âĢ¦": 12834, "terrorists": 12835, "casey": 12836, "cardinals": 12837, "ðŁĺĬðŁĺĬ": 12838, "venezuela": 12839, "bola": 12840, "literacy": 12841, "tw": 12842, "eno": 12843, "contains": 12844, "austin": 12845, "financi": 12846, "evan": 12847, "harvard": 12848, "originally": 12849, "chevro": 12850, "herald": 12851, "nottingham": 12852, "managers": 12853, "âŀ¡": 12854, "accepting": 12855, "walsh": 12856, "tutorial": 12857, "entrepreneurship": 12858, "yacht": 12859, "requirements": 12860, "glenn": 12861, "pede": 12862, "unfortunately": 12863, "aching": 12864, "daisy": 12865, "gian": 12866, "nightmare": 12867, "âĿĹ": 12868, "rina": 12869, "bart": 12870, "emails": 12871, "opposite": 12872, "whom": 12873, "sake": 12874, "puzzle": 12875, "dashi": 12876, "party": 12877, "blanket": 12878, "buses": 12879, "lore": 12880, "beauty": 12881, "reason": 12882, "punjab": 12883, "windsor": 12884, "functional": 12885, "existing": 12886, "hello": 12887, "glimp": 12888, "convin": 12889, "lak": 12890, "screaming": 12891, "rebecca": 12892, "bliss": 12893, "northwest": 12894, "infinity": 12895, "cosmetics": 12896, "pulling": 12897, "coffee": 12898, "pling": 12899, "opho": 12900, "colombia": 12901, "interiordesign": 12902, "(+": 12903, "emotions": 12904, "sac": 12905, "sunglasses": 12906, "saves": 12907, "df": 12908, "sixth": 12909, "aly": 12910, "ðŁĺ»": 12911, "deen": 12912, "devast": 12913, "politicians": 12914, "lacrosse": 12915, "gu": 12916, "pei": 12917, "java": 12918, "combine": 12919, "coalition": 12920, "erts": 12921, "surviv": 12922, "chad": 12923, "strian": 12924, "nn": 12925, "devi": 12926, "counc": 12927, "concern": 12928, "controller": 12929, "breast": 12930, "jury": 12931, "tum": 12932, "introduces": 12933, "ladi": 12934, "mobile": 12935, "alz": 12936, "steady": 12937, "nurses": 12938, "hacking": 12939, "online": 12940, "ocean": 12941, "ðŁİĦ": 12942, "aam": 12943, "juven": 12944, "icc": 12945, "louisiana": 12946, "arte": 12947, "streetart": 12948, "ison": 12949, "wns": 12950, "frm": 12951, "panda": 12952, "noir": 12953, "maintain": 12954, "delay": 12955, "symptoms": 12956, "thorn": 12957, "geome": 12958, "tern": 12959, "carried": 12960, "pru": 12961, "panor": 12962, "assy": 12963, "peru": 12964, "cloud": 12965, "spra": 12966, "pedi": 12967, "este": 12968, "tagged": 12969, "ðŁĺĿ": 12970, "shadows": 12971, "nazi": 12972, "اÙĦ": 12973, "corri": 12974, "âĻ¥âĻ¥": 12975, "jad": 12976, "ðŁĩ«": 12977, "formal": 12978, "spoken": 12979, "ðŁĮŀ": 12980, "enjoy": 12981, "lopez": 12982, "outlook": 12983, "inho": 12984, "wander": 12985, "Ùħ": 12986, "maya": 12987, "pee": 12988, "dine": 12989, "ãĢij": 12990, "briefing": 12991, "supporter": 12992, "arily": 12993, "ghters": 12994, "naturally": 12995, "doctorwho": 12996, "jen": 12997, "var": 12998, "newyear": 12999, "rese": 13000, "simm": 13001, "rex": 13002, "consequ": 13003, "tomatoes": 13004, "burst": 13005, "bravo": 13006, "burgers": 13007, "cracking": 13008, "northeast": 13009, "biom": 13010, "mushroom": 13011, "marque": 13012, "double": 13013, "nier": 13014, "vag": 13015, "twenty": 13016, "keyboard": 13017, "winni": 13018, "jamaica": 13019, "parish": 13020, ":-": 13021, "mentalhealth": 13022, "alizing": 13023, "render": 13024, "waking": 13025, "ðŁİĤ": 13026, "gly": 13027, "nathan": 13028, "washing": 13029, "melissa": 13030, "jung": 13031, "loyal": 13032, "chili": 13033, "songwriter": 13034, "guitarist": 13035, "bowie": 13036, "neighbors": 13037, "onymous": 13038, "asset": 13039, "tai": 13040, "headquarters": 13041, "ðŁĮĪ": 13042, "ihear": 13043, "cigare": 13044, "surg": 13045, ")\"": 13046, "repl": 13047, "darling": 13048, "ðŁĻĦ": 13049, "zak": 13050, "sare": 13051, "ãħĭ": 13052, "mickey": 13053, "warehouse": 13054, "massage": 13055, "inees": 13056, "didnt": 13057, "iw": 13058, "hurts": 13059, "engaging": 13060, "magic": 13061, "womenin": 13062, "kitten": 13063, "mors": 13064, "cart": 13065, "titans": 13066, "colleague": 13067, "competing": 13068, "eran": 13069, "khal": 13070, "marble": 13071, "demand": 13072, "delight": 13073, "etary": 13074, "blizz": 13075, "louise": 13076, "mls": 13077, "finishes": 13078, "experiment": 13079, "conducted": 13080, "electronics": 13081, "itters": 13082, "caring": 13083, "whats": 13084, "symbol": 13085, "jung": 13086, "ecu": 13087, "pix": 13088, "context": 13089, "charger": 13090, "ðŁĺĩ": 13091, "reig": 13092, "frag": 13093, "ëĭ": 13094, "chad": 13095, "true": 13096, "kerry": 13097, "defending": 13098, "aint": 13099, "auton": 13100, "checkout": 13101, "barnes": 13102, "lessly": 13103, "dt": 13104, "mme": 13105, "cloudy": 13106, "secondary": 13107, "arez": 13108, "_:": 13109, "appa": 13110, "constant": 13111, "\")": 13112, "vets": 13113, "job": 13114, "ient": 13115, "ðŁĺŃðŁĺŃðŁĺŃ": 13116, "mj": 13117, "french": 13118, "diver": 13119, "davies": 13120, "hhhh": 13121, "ebook": 13122, "à¹ī": 13123, "mariti": 13124, "breeze": 13125, "suspended": 13126, "mato": 13127, "viet": 13128, "rahu": 13129, "sei": 13130, "bolt": 13131, "enary": 13132, "leis": 13133, "karl": 13134, "framed": 13135, "explaining": 13136, "abc": 13137, "dealing": 13138, "nato": 13139, "jake": 13140, "expand": 13141, "leonard": 13142, "established": 13143, "dub": 13144, "armen": 13145, "elled": 13146, "vocal": 13147, "nicholas": 13148, "orient": 13149, "kyo": 13150, "illustrated": 13151, "ahh": 13152, "dancers": 13153, "million": 13154, "geta": 13155, "popp": 13156, "asu": 13157, "murdered": 13158, "gible": 13159, "stoked": 13160, "griffin": 13161, "maximum": 13162, "adrian": 13163, "encounter": 13164, "thero": 13165, "davidson": 13166, "ðŁį»": 13167, "holiday": 13168, "evo": 13169, "assets": 13170, "carson": 13171, "memorable": 13172, "âļ½": 13173, "obam": 13174, "representative": 13175, "cbd": 13176, "tricks": 13177, "vogue": 13178, "voice": 13179, "mmmm": 13180, "sebastian": 13181, "clif": 13182, "athy": 13183, "paralle": 13184, "ðŁ¤·": 13185, "pak": 13186, "evacu": 13187, "eats": 13188, "اØ": 13189, "touched": 13190, "organised": 13191, "spirits": 13192, "canad": 13193, "guided": 13194, "framework": 13195, "ðŁĮŁ": 13196, "ped": 13197, "natural": 13198, "agar": 13199, "replaced": 13200, "anchor": 13201, "tit": 13202, "shah": 13203, "organis": 13204, "superior": 13205, "rn": 13206, "chro": 13207, "erica": 13208, "still": 13209, "coron": 13210, "chuck": 13211, "locks": 13212, "organ": 13213, "rosen": 13214, "scam": 13215, "bened": 13216, "/#": 13217, "keen": 13218, "trevor": 13219, "vampire": 13220, "sorted": 13221, "!'": 13222, "afford": 13223, "intro": 13224, "grace": 13225, "ðŁĺľ": 13226, "saur": 13227, "kickstarter": 13228, "influen": 13229, "vu": 13230, "yup": 13231, "poc": 13232, "ðŁİ¥": 13233, "aar": 13234, "sang": 13235, "trek": 13236, "etsy": 13237, "tbh": 13238, "scream": 13239, "chevrolet": 13240, "pixel": 13241, "shepherd": 13242, "anor": 13243, "gabriel": 13244, "twood": 13245, "sdcc": 13246, "meters": 13247, "developers": 13248, "closure": 13249, "vw": 13250, "twitch": 13251, "ìĹ": 13252, "seoul": 13253, "price": 13254, "hog": 13255, "nish": 13256, "hillary": 13257, "scratch": 13258, "incen": 13259, "wagon": 13260, "disability": 13261, "panther": 13262, "chats": 13263, "gd": 13264, "witz": 13265, "sussex": 13266, "late": 13267, "denmark": 13268, "gerald": 13269, "cancelled": 13270, "nette": 13271, "ix": 13272, "naval": 13273, "baptist": 13274, "tet": 13275, "yad": 13276, "math": 13277, "hoy": 13278, "randy": 13279, "point": 13280, "intellec": 13281, "fruits": 13282, "wool": 13283, "guin": 13284, "pron": 13285, "theft": 13286, "condem": 13287, "marry": 13288, "nola": 13289, "architects": 13290, "cincin": 13291, "rockets": 13292, "gentleman": 13293, "explan": 13294, "tate": 13295, "doe": 13296, "raises": 13297, "wildlife": 13298, "wl": 13299, "insider": 13300, "blanc": 13301, "wp": 13302, "forsale": 13303, "nyc": 13304, "powell": 13305, "unbelievable": 13306, "pens": 13307, "goodies": 13308, "mustang": 13309, "pens": 13310, "stays": 13311, "squash": 13312, "xoxo": 13313, "nearby": 13314, "everton": 13315, "coco": 13316, "leagu": 13317, "khan": 13318, "stud": 13319, "southwest": 13320, "construc": 13321, "sworth": 13322, "croatia": 13323, "lea": 13324, "sums": 13325, "aims": 13326, "ean": 13327, "vaness": 13328, "itious": 13329, "pathy": 13330, "arcade": 13331, "bend": 13332, "suggests": 13333, "sacram": 13334, "royals": 13335, "rier": 13336, "emir": 13337, "incl": 13338, "ank": 13339, "clark": 13340, "right": 13341, "vacc": 13342, "ा": 13343, "tane": 13344, "lib": 13345, "usc": 13346, "sales": 13347, "huh": 13348, "sally": 13349, "vera": 13350, "pga": 13351, "grows": 13352, "drum": 13353, "tree": 13354, "ethics": 13355, "suggest": 13356, "isab": 13357, "sealed": 13358, "previously": 13359, "animated": 13360, "abdu": 13361, "rises": 13362, "glob": 13363, "predat": 13364, "scarf": 13365, "delic": 13366, "omar": 13367, "lli": 13368, "sxsw": 13369, "python": 13370, "nebra": 13371, "funk": 13372, "reflect": 13373, "pavilion": 13374, "tically": 13375, "chasing": 13376, "bakery": 13377, "invasion": 13378, "koh": 13379, "believed": 13380, "cohen": 13381, "conqu": 13382, "crafts": 13383, "nati": 13384, "clever": 13385, "governance": 13386, "samples": 13387, "fails": 13388, "âĶ": 13389, "timo": 13390, "ritu": 13391, "striking": 13392, "inclusive": 13393, "shocking": 13394, "cant": 13395, "requires": 13396, "drawings": 13397, "à¸Ń": 13398, "purchased": 13399, "dum": 13400, "zach": 13401, "warner": 13402, "console": 13403, "mansion": 13404, "fountain": 13405, "circum": 13406, "esh": 13407, "island": 13408, "milk": 13409, "profits": 13410, "halifax": 13411, "rival": 13412, "âľĪï¸ı": 13413, "jenny": 13414, "sandra": 13415, "nye": 13416, "kelly": 13417, "yal": 13418, "quad": 13419, "nos": 13420, "instein": 13421, "finalists": 13422, "midfielder": 13423, "cue": 13424, "exceptional": 13425, "aan": 13426, "sapp": 13427, "gettin": 13428, "saa": 13429, "fati": 13430, "slice": 13431, "volk": 13432, "swal": 13433, "lasting": 13434, "summary": 13435, "itas": 13436, "smo": 13437, "sz": 13438, "âĺĨ": 13439, "ipl": 13440, "flames": 13441, "enews": 13442, "hav": 13443, "hoodie": 13444, "pitcher": 13445, "windy": 13446, "revol": 13447, "central": 13448, "tonite": 13449, "ðŁİīðŁİī": 13450, "solved": 13451, "milwau": 13452, "organizations": 13453, "weets": 13454, "refin": 13455, "sth": 13456, "ãĥ¼": 13457, "elin": 13458, "tona": 13459, "cinnamon": 13460, "ðŁİ¨": 13461, "ðŁİģ": 13462, "ronaldo": 13463, "peninsu": 13464, "omega": 13465, "elds": 13466, "designing": 13467, "eigh": 13468, "bluet": 13469, "benz": 13470, "nug": 13471, "asha": 13472, "robots": 13473, "sudan": 13474, "choosing": 13475, "endo": 13476, "serge": 13477, "closely": 13478, "handy": 13479, "finger": 13480, "being": 13481, "arte": 13482, "survived": 13483, "flame": 13484, "milestone": 13485, "gut": 13486, "dwar": 13487, "futures": 13488, "ée": 13489, "elo": 13490, "fridge": 13491, "elic": 13492, "ouch": 13493, "ub": 13494, "pv": 13495, "titan": 13496, "collar": 13497, "station": 13498, "nevada": 13499, "aurora": 13500, "rd": 13501, "duncan": 13502, "âģł": 13503, "brien": 13504, "marsh": 13505, "о": 13506, "total": 13507, "chry": 13508, "sers": 13509, "suffe": 13510, "rachel": 13511, "college": 13512, "todays": 13513, "courts": 13514, "chit": 13515, "reunited": 13516, "gymna": 13517, "genesis": 13518, "beside": 13519, "representation": 13520, "chant": 13521, "collector": 13522, "rak": 13523, "athens": 13524, "nigh": 13525, "munich": 13526, "languages": 13527, "flu": 13528, "participation": 13529, "___": 13530, "cv": 13531, "spectrum": 13532, "soda": 13533, "cover": 13534, "referen": 13535, "abbo": 13536, "apa": 13537, "publication": 13538, "edm": 13539, "monica": 13540, "army": 13541, "ðŁļĢ": 13542, "divor": 13543, "dry": 13544, "streams": 13545, "robotics": 13546, "cider": 13547, "bullying": 13548, "approval": 13549, "stoke": 13550, "platforms": 13551, "sierra": 13552, "extin": 13553, "ib": 13554, "hayes": 13555, "succeed": 13556, "suffer": 13557, "atically": 13558, "dai": 13559, "lynch": 13560, "hound": 13561, "delines": 13562, "acknow": 13563, "dated": 13564, "exclusively": 13565, "heres": 13566, "facilit": 13567, "damaged": 13568, "charter": 13569, "lakers": 13570, "falcon": 13571, "unveiled": 13572, "welove": 13573, "ease": 13574, "patience": 13575, "lone": 13576, "gentle": 13577, "genetic": 13578, "producing": 13579, "gour": 13580, "shannon": 13581, "bilities": 13582, "zimbabwe": 13583, "pint": 13584, "daughters": 13585, "literary": 13586, "belle": 13587, "clam": 13588, "surrounded": 13589, "kany": 13590, "neil": 13591, "pirate": 13592, "ranger": 13593, "hbd": 13594, "natalie": 13595, "belong": 13596, "olympi": 13597, "embassy": 13598, "scol": 13599, "ener": 13600, "akin": 13601, "loren": 13602, "bh": 13603, ":/": 13604, "diva": 13605, "denim": 13606, "hipp": 13607, "ðŁĩµðŁĩ": 13608, "arnold": 13609, "?'": 13610, "weren": 13611, "empower": 13612, "disabled": 13613, "manor": 13614, "raspberry": 13615, "baf": 13616, "awful": 13617, "drummer": 13618, "kardashi": 13619, "nash": 13620, "machinelearning": 13621, "chu": 13622, "rebels": 13623, "timing": 13624, "monroe": 13625, "tongue": 13626, "range": 13627, "pupils": 13628, "ress": 13629, "amazon": 13630, "bz": 13631, "harley": 13632, "palmer": 13633, "balloon": 13634, "sings": 13635, "icec": 13636, "jb": 13637, "cers": 13638, "gps": 13639, "whist": 13640, "rise": 13641, "lt": 13642, "oooo": 13643, "cattle": 13644, "shooter": 13645, "vodka": 13646, "ucl": 13647, "mtg": 13648, "lesli": 13649, "jonas": 13650, "dispo": 13651, "atric": 13652, "stein": 13653, "vintage": 13654, "firms": 13655, "floyd": 13656, "cowboy": 13657, "soooo": 13658, "isaac": 13659, "warcraft": 13660, "disneyland": 13661, "beautiful": 13662, "beam": 13663, "franchise": 13664, "bun": 13665, "kag": 13666, "anon": 13667, "turbo": 13668, "sweep": 13669, "madein": 13670, "karachi": 13671, "detective": 13672, "pennsylvania": 13673, "controversi": 13674, "vitamin": 13675, "aside": 13676, "chronic": 13677, "describes": 13678, "removal": 13679, "hah": 13680, "aper": 13681, "tened": 13682, "uto": 13683, "badly": 13684, "mirac": 13685, "fry": 13686, "yea": 13687, "injec": 13688, "thermal": 13689, "compact": 13690, "thor": 13691, "teed": 13692, "urgent": 13693, "lite": 13694, "gilli": 13695, "sophom": 13696, "ico": 13697, "chem": 13698, "pm": 13699, "fork": 13700, "freak": 13701, "chak": 13702, "recipient": 13703, "iy": 13704, "nik": 13705, "modeling": 13706, "cans": 13707, "ðŁıĢ": 13708, "delux": 13709, "seam": 13710, "survivors": 13711, "radical": 13712, "investigating": 13713, "reliable": 13714, "fm": 13715, "turt": 13716, "lighthouse": 13717, "tool": 13718, "gown": 13719, "))": 13720, "bots": 13721, "autograph": 13722, "aid": 13723, "buffe": 13724, "hmm": 13725, "horrible": 13726, "ssional": 13727, "anni": 13728, "à¹Ģ": 13729, "kits": 13730, "schi": 13731, "eternal": 13732, "huss": 13733, "sensitive": 13734, "ru": 13735, "tastes": 13736, "checks": 13737, "imo": 13738, "portion": 13739, "skate": 13740, "eden": 13741, "halftime": 13742, "fried": 13743, "rihanna": 13744, "tise": 13745, "flick": 13746, "cain": 13747, "sgt": 13748, "âľĶ": 13749, "shau": 13750, "stained": 13751, "raffle": 13752, "drove": 13753, "salman": 13754, "principles": 13755, "sho": 13756, "aru": 13757, "jess": 13758, "guine": 13759, "garbage": 13760, "myan": 13761, "jelly": 13762, "disru": 13763, "zia": 13764, "qld": 13765, "entries": 13766, "lav": 13767, "flew": 13768, "admit": 13769, "objects": 13770, "compare": 13771, "nytimes": 13772, "cannes": 13773, "pn": 13774, "suffol": 13775, "roc": 13776, "dana": 13777, "egg": 13778, "hist": 13779, "counsel": 13780, "'!": 13781, "physi": 13782, "imagination": 13783, "adjust": 13784, "explosion": 13785, "plymouth": 13786, "horror": 13787, "elliott": 13788, "bourne": 13789, "dex": 13790, "breed": 13791, "audio": 13792, "lobster": 13793, "disappointed": 13794, "nationwide": 13795, "((": 13796, "increases": 13797, "australi": 13798, "cedar": 13799, "staring": 13800, "racial": 13801, "eis": 13802, "gmt": 13803, "visions": 13804, "stayed": 13805, "discussions": 13806, "dean": 13807, "curtis": 13808, "maiden": 13809, "stellar": 13810, "happiest": 13811, "hwy": 13812, "preseason": 13813, "carav": 13814, "mondays": 13815, "hospitals": 13816, "glimpse": 13817, "scholars": 13818, "jai": 13819, "terrace": 13820, "anna": 13821, "goose": 13822, "graded": 13823, "lotus": 13824, "hung": 13825, "grocery": 13826, "stamps": 13827, "emperor": 13828, "scoop": 13829, "inser": 13830, "cas": 13831, "existence": 13832, "heal": 13833, "falcons": 13834, "marvel": 13835, "reducing": 13836, "terrific": 13837, "magnetic": 13838, "performs": 13839, "barre": 13840, "pus": 13841, "treating": 13842, "icon": 13843, "wh": 13844, "declared": 13845, "trauma": 13846, "dod": 13847, "comedian": 13848, "nikon": 13849, "bugs": 13850, "asm": 13851, "montgom": 13852, "ibiza": 13853, "comprehensive": 13854, "has": 13855, "santi": 13856, "fellowship": 13857, "dash": 13858, "psal": 13859, "louisville": 13860, "spy": 13861, "fault": 13862, "dthe": 13863, "filed": 13864, "vista": 13865, "desc": 13866, "fears": 13867, "youtu": 13868, "sps": 13869, "esp": 13870, "rig": 13871, "crime": 13872, "berger": 13873, "wonderland": 13874, "kent": 13875, "informed": 13876, "stevens": 13877, "myth": 13878, "aston": 13879, "iri": 13880, "visitor": 13881, "atri": 13882, "producers": 13883, "alla": 13884, "personally": 13885, "separate": 13886, "agencies": 13887, "afri": 13888, "ilan": 13889, "spoke": 13890, "nina": 13891, "squad": 13892, "dives": 13893, "depend": 13894, "liv": 13895, "fierce": 13896, "entertaining": 13897, "chain": 13898, "scat": 13899, "borders": 13900, "palette": 13901, "spro": 13902, "osis": 13903, "derby": 13904, "tobacco": 13905, "zio": 13906, "willie": 13907, "juvent": 13908, "zoom": 13909, "holy": 13910, "entirely": 13911, "afe": 13912, "martinez": 13913, "beds": 13914, "pea": 13915, "bulldogs": 13916, "ðŁĩªðŁĩ": 13917, "ibm": 13918, "neon": 13919, "ethiopia": 13920, "teammates": 13921, "planting": 13922, "twer": 13923, "anytime": 13924, "forbes": 13925, "ón": 13926, "runway": 13927, "nervous": 13928, "roger": 13929, "pile": 13930, "chanc": 13931, "apocaly": 13932, "uw": 13933, "oi": 13934, "drought": 13935, "territory": 13936, "brick": 13937, "creatures": 13938, "goin": 13939, "waff": 13940, "gren": 13941, "southeast": 13942, "jean": 13943, "ambul": 13944, "edited": 13945, "strap": 13946, "cv": 13947, "aaron": 13948, "ãĥ»ãĥ»": 13949, "tsu": 13950, "description": 13951, "kindly": 13952, "clutch": 13953, "immer": 13954, "enor": 13955, "womensday": 13956, "orange": 13957, "rag": 13958, "obvious": 13959, "hyder": 13960, "channels": 13961, "mango": 13962, "meyer": 13963, "raining": 13964, "getty": 13965, "pilgri": 13966, "coordinator": 13967, "upload": 13968, "nintendo": 13969, "donuts": 13970, "sanchez": 13971, "apparel": 13972, "jr": 13973, "zzi": 13974, ",@": 13975, "jefferson": 13976, "accessible": 13977, "greatly": 13978, "eid": 13979, "initial": 13980, "buddha": 13981, "paris": 13982, "mascot": 13983, "â¬ĩï¸ı": 13984, "schwar": 13985, "siri": 13986, "spinning": 13987, "mortgage": 13988, "echo": 13989, "endange": 13990, "gedly": 13991, "chloe": 13992, "enhance": 13993, "karnat": 13994, "kry": 13995, "explores": 13996, "ðŁĴģ": 13997, "affair": 13998, "icals": 13999, "alla": 14000, "dart": 14001, "dolphins": 14002, "differences": 14003, "squirrel": 14004, "augh": 14005, "drones": 14006, "ellen": 14007, "restore": 14008, "paw": 14009, "unfor": 14010, "pike": 14011, "hilton": 14012, "collab": 14013, "consumers": 14014, "coinci": 14015, "outcomes": 14016, "ppp": 14017, "aq": 14018, "coupon": 14019, "liest": 14020, "sims": 14021, "kho": 14022, "aves": 14023, "spoon": 14024, "pudding": 14025, "corbyn": 14026, "haters": 14027, "exams": 14028, "slave": 14029, ".!": 14030, "psa": 14031, "apples": 14032, "tamil": 14033, "sed": 14034, "coke": 14035, "zzo": 14036, "losange": 14037, "carbon": 14038, "clair": 14039, "...)": 14040, "khu": 14041, "craig": 14042, "exploration": 14043, "sanctuary": 14044, "sue": 14045, "alway": 14046, "dementia": 14047, "wonders": 14048, "superhero": 14049, "pakistani": 14050, "browns": 14051, "bluetooth": 14052, "locker": 14053, "marc": 14054, "eventu": 14055, "deluxe": 14056, "rodriguez": 14057, "âĿ¤âĿ¤": 14058, "robb": 14059, "ðŁĴ¦": 14060, "linux": 14061, "tens": 14062, "intelligent": 14063, "seed": 14064, "voter": 14065, "sler": 14066, "peaks": 14067, "intern": 14068, "teenage": 14069, "peninsula": 14070, "handling": 14071, "tie": 14072, "cousins": 14073, "wendy": 14074, "mee": 14075, "à¹Ģà¸": 14076, "dino": 14077, "ðŁĴ°": 14078, "ðŁĺĥ": 14079, "zee": 14080, "sbury": 14081, "tragedy": 14082, "bk": 14083, "bore": 14084, "zin": 14085, "warns": 14086, "idiot": 14087, "touching": 14088, "continental": 14089, "tacos": 14090, "safari": 14091, "washed": 14092, "podium": 14093, "morrison": 14094, "forests": 14095, "cbc": 14096, "alon": 14097, "particular": 14098, "beads": 14099, "invented": 14100, "loch": 14101, "lighter": 14102, "wherever": 14103, "ide": 14104, "documents": 14105, "awe": 14106, "kr": 14107, "nowhere": 14108, "miner": 14109, "stit": 14110, "rox": 14111, "contribute": 14112, "hardy": 14113, "clan": 14114, "object": 14115, "cait": 14116, "ðŁĴķðŁĴķ": 14117, "happier": 14118, "vegetables": 14119, "tart": 14120, "gag": 14121, "nominee": 14122, "heavily": 14123, "panic": 14124, "jd": 14125, "theresa": 14126, "atm": 14127, "uph": 14128, "sfc": 14129, "suri": 14130, "drink": 14131, "nal": 14132, "revel": 14133, "kl": 14134, "avocado": 14135, "nomination": 14136, "madonna": 14137, "sharon": 14138, "malcolm": 14139, "controlled": 14140, "shers": 14141, "revival": 14142, "legislation": 14143, "shoots": 14144, "nin": 14145, "commentary": 14146, "pros": 14147, "humanrights": 14148, "stranger": 14149, "mitch": 14150, "pipeline": 14151, "legally": 14152, "thu": 14153, "gilbert": 14154, "toll": 14155, "granted": 14156, "ghs": 14157, "iranian": 14158, "refreshing": 14159, "duk": 14160, "abi": 14161, "prime": 14162, "joseph": 14163, "mosa": 14164, "statistics": 14165, "productions": 14166, "merry": 14167, "patel": 14168, "sax": 14169, "humanitarian": 14170, "structures": 14171, "emissions": 14172, "towns": 14173, "freel": 14174, "stering": 14175, "ratings": 14176, "allegedly": 14177, "cabin": 14178, "stl": 14179, "wade": 14180, "flyers": 14181, "trim": 14182, "promising": 14183, "zu": 14184, "ballot": 14185, "comparison": 14186, "freeze": 14187, "outer": 14188, "greatness": 14189, "assign": 14190, "snowy": 14191, "rale": 14192, "tories": 14193, "mediter": 14194, "knock": 14195, "consultant": 14196, "cincinnati": 14197, "analyst": 14198, "scoo": 14199, "jews": 14200, "approxim": 14201, "pure": 14202, "portraits": 14203, "cyrus": 14204, "ational": 14205, "loans": 14206, "acquis": 14207, "elu": 14208, "acceptable": 14209, "union": 14210, "watercolor": 14211, "rust": 14212, "battles": 14213, "perfu": 14214, "seasonal": 14215, "serial": 14216, "mindset": 14217, "riot": 14218, "feld": 14219, "ennial": 14220, "closet": 14221, "priest": 14222, "tanks": 14223, "intl": 14224, "screw": 14225, "bum": 14226, "abdul": 14227, "oux": 14228, "explained": 14229, "rica": 14230, "imaging": 14231, "lawyers": 14232, "buried": 14233, "ãĥ»ãĥ»ãĥ»": 14234, "earl": 14235, "âĢķ": 14236, "lton": 14237, "restored": 14238, "stripes": 14239, "foss": 14240, "demands": 14241, "stealing": 14242, "alexis": 14243, "mund": 14244, "aker": 14245, "urus": 14246, "wardro": 14247, "hugs": 14248, "genre": 14249, "ego": 14250, "ÙĦ": 14251, "participated": 14252, "babes": 14253, "banquet": 14254, "tious": 14255, "hemi": 14256, "dsb": 14257, "lost": 14258, "milwaukee": 14259, "jenner": 14260, "gem": 14261, "outra": 14262, "loses": 14263, "idi": 14264, "reps": 14265, "ðŁİ§": 14266, "regulation": 14267, "flaw": 14268, "fang": 14269, "vibrant": 14270, "ramp": 14271, "rains": 14272, "wellbeing": 14273, "soviet": 14274, "viewers": 14275, "depo": 14276, "libraries": 14277, "bigo": 14278, "sery": 14279, "gill": 14280, "destruction": 14281, "coz": 14282, "cx": 14283, "bridal": 14284, "alds": 14285, "planted": 14286, "amateur": 14287, "lud": 14288, "cheering": 14289, "showcas": 14290, "profile": 14291, "iu": 14292, "vertical": 14293, "packers": 14294, "wizard": 14295, "skip": 14296, "slight": 14297, "beau": 14298, "airways": 14299, "much": 14300, "rera": 14301, "ðŁĮĬ": 14302, "absor": 14303, "patio": 14304, "packages": 14305, "sells": 14306, "mentally": 14307, "ðŁĺ¢": 14308, "reynolds": 14309, "kare": 14310, "tribun": 14311, "walt": 14312, "knit": 14313, "taste": 14314, "surrey": 14315, "bounce": 14316, "creature": 14317, "bare": 14318, "betting": 14319, "sure": 14320, "miley": 14321, "laughs": 14322, "alore": 14323, "cyn": 14324, "tl": 14325, "artist": 14326, "annah": 14327, "warmer": 14328, "dynamics": 14329, "lunchtime": 14330, "maritime": 14331, "vulnerable": 14332, "ðŁĴĥ": 14333, "wolver": 14334, "durham": 14335, "constantly": 14336, "amin": 14337, "sibl": 14338, ":@": 14339, "bullet": 14340, "kach": 14341, "angelo": 14342, "wilder": 14343, "doom": 14344, "desktop": 14345, "lawsuit": 14346, "kca": 14347, "henderson": 14348, "inviting": 14349, "betty": 14350, "tawards": 14351, "rafa": 14352, "leaked": 14353, "andi": 14354, "gems": 14355, "afl": 14356, "velo": 14357, "mediterran": 14358, "probe": 14359, "totten": 14360, "stephanie": 14361, "snation": 14362, "combe": 14363, "qs": 14364, "overcome": 14365, "assassin": 14366, "rav": 14367, "filip": 14368, "winnipeg": 14369, "shil": 14370, "determined": 14371, "kas": 14372, "outre": 14373, "regret": 14374, "guides": 14375, "aaa": 14376, "ðŁĺĪ": 14377, "wives": 14378, "manife": 14379, "erly": 14380, "smy": 14381, "shima": 14382, "xing": 14383, "pixel": 14384, "jacob": 14385, "accommod": 14386, "toy": 14387, "ono": 14388, "poo": 14389, "tier": 14390, "answe": 14391, "ðŁĴģ": 14392, "rosa": 14393, "lease": 14394, "belongs": 14395, "thar": 14396, "eventually": 14397, "neither": 14398, "goa": 14399, "skiing": 14400, "atra": 14401, "agh": 14402, "broadcasting": 14403, "fury": 14404, "pyram": 14405, "dice": 14406, "volkswag": 14407, "womens": 14408, "provider": 14409, "bombs": 14410, "missile": 14411, "whip": 14412, "dick": 14413, "norwe": 14414, "backup": 14415, "elder": 14416, "mature": 14417, "concerts": 14418, "gious": 14419, "squee": 14420, "goodmorning": 14421, "braves": 14422, "^_": 14423, "aussie": 14424, "luna": 14425, "males": 14426, "heck": 14427, "fortn": 14428, "romeo": 14429, "steelers": 14430, "pn": 14431, "peer": 14432, "represents": 14433, "«": 14434, "katy": 14435, "miguel": 14436, "require": 14437, "chains": 14438, "lur": 14439, "immediate": 14440, "timber": 14441, "âĸ¶ï¸ı": 14442, "advocacy": 14443, "export": 14444, "anz": 14445, "tiffany": 14446, "author": 14447, "ðŁİĪ": 14448, "dudes": 14449, "chilly": 14450, "hid": 14451, "harm": 14452, "bug": 14453, "monster": 14454, "terrier": 14455, "tuc": 14456, "storytelling": 14457, "tak": 14458, "inti": 14459, "immigrants": 14460, "bis": 14461, "reaches": 14462, "compassion": 14463, "johnny": 14464, "contributions": 14465, "ðŁIJ¶": 14466, "mechanical": 14467, "impression": 14468, "ranks": 14469, "kobe": 14470, "menting": 14471, "blossom": 14472, "pablo": 14473, "builder": 14474, "bombing": 14475, "twel": 14476, "sullivan": 14477, "omo": 14478, "pete": 14479, "demi": 14480, "kudos": 14481, "wbb": 14482, "tgif": 14483, "massach": 14484, "neighbor": 14485, "chefs": 14486, "engines": 14487, "pune": 14488, "gained": 14489, "phantom": 14490, "sdays": 14491, "extend": 14492, "gran": 14493, "centers": 14494, "jacqu": 14495, "datasci": 14496, "sleepy": 14497, "elvis": 14498, "answered": 14499, "slot": 14500, "cony": 14501, "flexible": 14502, "tially": 14503, "letics": 14504, "%,": 14505, "andrews": 14506, "sible": 14507, "momma": 14508, "vino": 14509, "dox": 14510, "invitational": 14511, "twilight": 14512, "jade": 14513, "illery": 14514, "johns": 14515, "fou": 14516, "pv": 14517, "--->": 14518, "breakdown": 14519, "billion": 14520, "printer": 14521, "mond": 14522, "cbc": 14523, "maggie": 14524, "legion": 14525, "dub": 14526, "kurt": 14527, "poor": 14528, "parenting": 14529, "regions": 14530, "bikini": 14531, "beware": 14532, "sional": 14533, "auburn": 14534, "kidding": 14535, "amples": 14536, "span": 14537, "contempor": 14538, "cic": 14539, "habits": 14540, "ako": 14541, "prefe": 14542, "buddies": 14543, "itz": 14544, "emily": 14545, "personnel": 14546, "mountain": 14547, "versus": 14548, "ðŁĺ¬": 14549, "earning": 14550, "sink": 14551, "dari": 14552, "uu": 14553, "swin": 14554, "ister": 14555, "brutal": 14556, "nac": 14557, "kata": 14558, "cloth": 14559, "amand": 14560, "ðŁĶĹ": 14561, "neo": 14562, "alumin": 14563, "weekends": 14564, "nebraska": 14565, "codes": 14566, "delayed": 14567, "bruno": 14568, "proven": 14569, "inc": 14570, "ight": 14571, "flan": 14572, "oro": 14573, "lambert": 14574, "regulat": 14575, "wf": 14576, "massachuse": 14577, "kardashian": 14578, "bernard": 14579, "fiesta": 14580, "volcano": 14581, "grandpa": 14582, "anca": 14583, "dre": 14584, "stitu": 14585, "meaning": 14586, "foam": 14587, "auck": 14588, "ated": 14589, "rl": 14590, "hotel": 14591, "persons": 14592, "dynasty": 14593, "ellor": 14594, "mai": 14595, "amne": 14596, "styling": 14597, "avier": 14598, "eg": 14599, "vegetarian": 14600, ",âĢ¦": 14601, "founders": 14602, "stain": 14603, "gd": 14604, "cycles": 14605, "skyline": 14606, "tractor": 14607, "exists": 14608, "tral": 14609, "kidney": 14610, "maril": 14611, "instag": 14612, "sette": 14613, "addict": 14614, "triangle": 14615, "flashback": 14616, "controversial": 14617, "zon": 14618, "pins": 14619, "ias": 14620, "tray": 14621, "township": 14622, "delegates": 14623, "spam": 14624, "hms": 14625, "crane": 14626, "peoples": 14627, "olo": 14628, "faction": 14629, "butes": 14630, "onica": 14631, "delegation": 14632, "newprofile": 14633, "elier": 14634, "mca": 14635, "wand": 14636, "gely": 14637, "losangeles": 14638, "berke": 14639, "tive": 14640, "disrup": 14641, "zza": 14642, "casa": 14643, "jordan": 14644, "fordshire": 14645, "gathered": 14646, "ichi": 14647, "attendees": 14648, "à¸Ńà¸": 14649, "peppers": 14650, "coin": 14651, "bourbon": 14652, "ernity": 14653, "rotary": 14654, "behaviour": 14655, "jeremy": 14656, "teamwork": 14657, "compliance": 14658, "tremend": 14659, "ðŁĩ§": 14660, "buhari": 14661, "cambo": 14662, "buyers": 14663, "hagen": 14664, "buds": 14665, "bayern": 14666, "monte": 14667, "smells": 14668, "anza": 14669, "athlon": 14670, "described": 14671, "workforce": 14672, "giving": 14673, "api": 14674, "investments": 14675, "dail": 14676, "selena": 14677, "database": 14678, "thum": 14679, "mortal": 14680, "student": 14681, "buyer": 14682, "dover": 14683, "garten": 14684, "attle": 14685, "loyalty": 14686, "genoci": 14687, "holocau": 14688, "theaters": 14689, "ruling": 14690, "venus": 14691, "patent": 14692, "chun": 14693, "abby": 14694, "awake": 14695, "massacre": 14696, "bangalore": 14697, "breaking": 14698, "simmons": 14699, "justi": 14700, "hale": 14701, "edchat": 14702, "ggles": 14703, "hawk": 14704, "marking": 14705, "headlines": 14706, "strom": 14707, "cove": 14708, "breathtaking": 14709, "medals": 14710, "haircut": 14711, "christine": 14712, "telegraph": 14713, "gujarat": 14714, "jura": 14715, "cane": 14716, "shore": 14717, "propaganda": 14718, "mueller": 14719, "........": 14720, "savi": 14721, "stomach": 14722, "throws": 14723, "tab": 14724, "warm": 14725, "jong": 14726, "renowned": 14727, "hir": 14728, "rais": 14729, "mushrooms": 14730, "guaranteed": 14731, "boa": 14732, "mj": 14733, "revolutionary": 14734, "certification": 14735, "bruins": 14736, "join": 14737, "wes": 14738, "passport": 14739, "cg": 14740, "sexu": 14741, "capable": 14742, "wv": 14743, "tones": 14744, "jackets": 14745, "accompan": 14746, "spinach": 14747, "forever": 14748, "blair": 14749, "watts": 14750, "gl": 14751, "couples": 14752, "prairie": 14753, "newprofilepic": 14754, "logistics": 14755, "massachusetts": 14756, "jaguar": 14757, "oid": 14758, "weal": 14759, "underwater": 14760, "moz": 14761, "yi": 14762, "maths": 14763, "myanmar": 14764, "preps": 14765, "suffered": 14766, "trace": 14767, "wali": 14768, "ahhh": 14769, "borg": 14770, "stitch": 14771, "culin": 14772, "realise": 14773, "infection": 14774, "discrimination": 14775, "shame": 14776, "ankle": 14777, "humid": 14778, "yt": 14779, "bracket": 14780, "truck": 14781, "triu": 14782, "easter": 14783, "community": 14784, "postcard": 14785, "involving": 14786, "tyler": 14787, "caramel": 14788, "overview": 14789, "examples": 14790, "integrity": 14791, "basement": 14792, "instruments": 14793, "anium": 14794, "atus": 14795, "gher": 14796, "laundry": 14797, "achieve": 14798, "geneva": 14799, "pricing": 14800, "hyderabad": 14801, "belief": 14802, "meta": 14803, "jaw": 14804, "accounting": 14805, "leader": 14806, "cristiano": 14807, "couture": 14808, "cyp": 14809, "vised": 14810, ",,,": 14811, "knu": 14812, "hick": 14813, "breaker": 14814, "bram": 14815, "rab": 14816, "moor": 14817, "hamas": 14818, "graduating": 14819, "puppies": 14820, "akh": 14821, "tah": 14822, "aches": 14823, "rie": 14824, "opini": 14825, "gta": 14826, "reign": 14827, "tragic": 14828, "rever": 14829, "pill": 14830, "pineapple": 14831, "touches": 14832, "dare": 14833, "leys": 14834, "ilo": 14835, "interiors": 14836, "scouts": 14837, "bart": 14838, "enzie": 14839, "dono": 14840, "brock": 14841, "christians": 14842, "ensemble": 14843, "·": 14844, "cinemas": 14845, "newport": 14846, "airline": 14847, "winston": 14848, "leigh": 14849, "contents": 14850, "prescri": 14851, "urge": 14852, "trout": 14853, "fically": 14854, "ilia": 14855, "subsi": 14856, "arer": 14857, "âļ¾ï¸ı": 14858, "wounded": 14859, "ðŁĻĤ": 14860, "pepper": 14861, "ðŁĴŀ": 14862, "fitted": 14863, "aff": 14864, "resur": 14865, "thursdaythoughts": 14866, "zero": 14867, "archaeology": 14868, "div": 14869, "jee": 14870, "ion": 14871, "awaiting": 14872, "cozy": 14873, "beauties": 14874, "bald": 14875, "data": 14876, "grizz": 14877, "stalk": 14878, "kinds": 14879, "cleared": 14880, "jessic": 14881, "regular": 14882, "aliens": 14883, "place": 14884, "bos": 14885, "bizar": 14886, "thisis": 14887, "ðŁĴĢ": 14888, "tottenham": 14889, "mafia": 14890, "slam": 14891, "ariana": 14892, "carroll": 14893, "backpack": 14894, "carey": 14895, "univ": 14896, "rg": 14897, "pep": 14898, "digit": 14899, "tattoos": 14900, "agon": 14901, "volunteering": 14902, "differen": 14903, "consumption": 14904, "kathr": 14905, "headphones": 14906, "tshirt": 14907, "ob": 14908, "element": 14909, "retail": 14910, "shru": 14911, "algori": 14912, "container": 14913, "conscious": 14914, "fil": 14915, "coming": 14916, "rash": 14917, "urope": 14918, "define": 14919, "gior": 14920, "feminist": 14921, "flowing": 14922, "routes": 14923, "glaci": 14924, "fert": 14925, "somerset": 14926, "antes": 14927, "tweeps": 14928, "$$": 14929, "hour": 14930, "endangered": 14931, "yearsof": 14932, "roh": 14933, "popped": 14934, "backing": 14935, "basil": 14936, "brake": 14937, "monaco": 14938, "lgbtq": 14939, "prague": 14940, "utility": 14941, "cassi": 14942, "gateway": 14943, "haunted": 14944, "schul": 14945, "ðŁİµ": 14946, "should": 14947, "walkingdead": 14948, "completing": 14949, "danny": 14950, "montgomery": 14951, "penguin": 14952, "ssi": 14953, "merchandi": 14954, "ðŁijij": 14955, "church": 14956, "hates": 14957, "captain": 14958, "breathing": 14959, "cet": 14960, "fairly": 14961, "approaches": 14962, "companion": 14963, "surprising": 14964, "kanye": 14965, "pey": 14966, "hindi": 14967, "targeted": 14968, "lords": 14969, "deut": 14970, "digging": 14971, "german": 14972, "rut": 14973, "energy": 14974, "closest": 14975, "yun": 14976, "apologi": 14977, "ั": 14978, "sack": 14979, "rup": 14980, "ddy": 14981, "portal": 14982, "dough": 14983, "bats": 14984, "ðŁĵ°": 14985, "atur": 14986, "grapher": 14987, "pires": 14988, "motors": 14989, "ðŁĮ¹": 14990, "jc": 14991, "dang": 14992, "tuk": 14993, "clue": 14994, "usc": 14995, "page": 14996, "dless": 14997, "brows": 14998, "jus": 14999, "ading": 15000, "remarks": 15001, "oom": 15002, "cardio": 15003, "stefan": 15004, "armstrong": 15005, "âĢ¢âĢ¢": 15006, "niest": 15007, "belgian": 15008, "biop": 15009, "soy": 15010, "lof": 15011, "íĥ": 15012, "qt": 15013, "flashbackfriday": 15014, "cee": 15015, "ģà¸": 15016, "wreck": 15017, "marines": 15018, "amendment": 15019, "wardrobe": 15020, "voy": 15021, "burned": 15022, "guitars": 15023, "rainf": 15024, "lifel": 15025, "ssil": 15026, "ounce": 15027, "external": 15028, "ckey": 15029, "mesh": 15030, "sheikh": 15031, "invitation": 15032, "suggesti": 15033, "popcorn": 15034, "phenomenal": 15035, "anonymous": 15036, "tuna": 15037, "chicago": 15038, "oval": 15039, "dely": 15040, "locals": 15041, "(&": 15042, "prof": 15043, "novel": 15044, "finder": 15045, "sparks": 15046, "laven": 15047, "infu": 15048, "nicks": 15049, "quant": 15050, "rae": 15051, "exec": 15052, "distingui": 15053, "stances": 15054, "mutual": 15055, "shal": 15056, "unveils": 15057, "edmonton": 15058, "zania": 15059, "adio": 15060, "viewer": 15061, "bradford": 15062, "auditorium": 15063, "quis": 15064, "react": 15065, "http": 15066, "lero": 15067, "cheeky": 15068, "impacts": 15069, "tak": 15070, "edt": 15071, "desperate": 15072, "tay": 15073, "ìĦ": 15074, "settle": 15075, "bargain": 15076, "resume": 15077, "unite": 15078, "thrown": 15079, "kest": 15080, "seys": 15081, "marching": 15082, "amit": 15083, "decline": 15084, "schar": 15085, "metr": 15086, "stanford": 15087, "linke": 15088, "berra": 15089, "dolls": 15090, "rugby": 15091, "jami": 15092, "bor": 15093, "roadtrip": 15094, "dinosaur": 15095, "mik": 15096, "sunder": 15097, "rem": 15098, "bk": 15099, "overseas": 15100, "naughty": 15101, "implementation": 15102, "iamsrk": 15103, "luncheon": 15104, "firing": 15105, "miami": 15106, "perez": 15107, "thee": 15108, "zon": 15109, "gifted": 15110, "conversion": 15111, "ceramic": 15112, "¡ï¸ı": 15113, "pedro": 15114, "ìĨ": 15115, "vick": 15116, "!@": 15117, "heed": 15118, "sid": 15119, "bw": 15120, "document": 15121, "plun": 15122, "grants": 15123, "fantasy": 15124, "predictions": 15125, "valid": 15126, "carved": 15127, "graduated": 15128, "ðŁijįðŁı»": 15129, "nationally": 15130, "chy": 15131, "afl": 15132, "resso": 15133, "blank": 15134, "rivals": 15135, "jig": 15136, "eties": 15137, "omics": 15138, "unemp": 15139, "bound": 15140, "sko": 15141, "inspection": 15142, "paral": 15143, "highs": 15144, "crisp": 15145, "bans": 15146, "oba": 15147, "[@": 15148, "cospla": 15149, "costumes": 15150, "recall": 15151, "mouth": 15152, "nigel": 15153, "bts": 15154, "tera": 15155, "kov": 15156, "docs": 15157, "westminster": 15158, "dict": 15159, "gravity": 15160, "kari": 15161, "rogue": 15162, "tted": 15163, "wark": 15164, "idaho": 15165, "wend": 15166, "awi": 15167, "queensland": 15168, "processes": 15169, "cliffe": 15170, "mick": 15171, "compens": 15172, "opol": 15173, "they": 15174, "clari": 15175, "wikipedia": 15176, "salmankhan": 15177, "hazard": 15178, "preston": 15179, "sweetest": 15180, "pdf": 15181, "chees": 15182, "trilo": 15183, "southafrica": 15184, "burnt": 15185, "($": 15186, "contain": 15187, "tp": 15188, "submitted": 15189, "soundcloud": 15190, "atu": 15191, "rez": 15192, "wordpress": 15193, "corrupt": 15194, "nf": 15195, "maker": 15196, "íķ": 15197, "paras": 15198, "advent": 15199, "rial": 15200, "cafe": 15201, "fossil": 15202, "!!!!!!!": 15203, "cows": 15204, "cj": 15205, "spur": 15206, "institutions": 15207, "landmark": 15208, "entit": 15209, "reut": 15210, "his": 15211, "alzheim": 15212, "wemb": 15213, "reggae": 15214, "mosqu": 15215, "stat": 15216, "identified": 15217, "dealer": 15218, "ream": 15219, "reland": 15220, "tension": 15221, "ðŁĩ©": 15222, "wrapping": 15223, "deeper": 15224, "frat": 15225, "reddit": 15226, "aris": 15227, "morocco": 15228, "..\"": 15229, "blow": 15230, "mapping": 15231, "priorities": 15232, "inga": 15233, "swap": 15234, "rewards": 15235, "conspiracy": 15236, "creative": 15237, "cj": 15238, "congressional": 15239, "vault": 15240, "plex": 15241, "sophomore": 15242, "shadow": 15243, "eless": 15244, "ðŁĺħ": 15245, "darts": 15246, "aldub": 15247, "annoying": 15248, "props": 15249, "nas": 15250, "aluminum": 15251, "hbo": 15252, "offense": 15253, "jill": 15254, "onions": 15255, "laur": 15256, "tae": 15257, "hardest": 15258, "shro": 15259, "gaining": 15260, "measure": 15261, "edtech": 15262, "cyprus": 15263, "tara": 15264, "angeli": 15265, "carlo": 15266, "goon": 15267, "alli": 15268, "implic": 15269, "jupit": 15270, "resilience": 15271, "hail": 15272, "balanced": 15273, ")...": 15274, "joyce": 15275, "gra": 15276, "theli": 15277, "defined": 15278, "shipped": 15279, "mainly": 15280, "mina": 15281, "lm": 15282, "sacri": 15283, "ober": 15284, "pim": 15285, "claiming": 15286, "enters": 15287, "corey": 15288, "bok": 15289, "cried": 15290, "cooling": 15291, "danielle": 15292, "pharmacy": 15293, "thorough": 15294, "cake": 15295, "klo": 15296, "outreach": 15297, "zens": 15298, "digitalmarketing": 15299, "valent": 15300, "snp": 15301, "herb": 15302, "mrw": 15303, "café": 15304, "captures": 15305, "notre": 15306, "triumph": 15307, "pancakes": 15308, "cumber": 15309, "spike": 15310, "dation": 15311, "bigg": 15312, "sper": 15313, "critical": 15314, "amal": 15315, "tooth": 15316, "founding": 15317, "astro": 15318, "'#": 15319, "quantum": 15320, "thames": 15321, "unc": 15322, "pride": 15323, "airbus": 15324, "knocked": 15325, "undefeated": 15326, "mediterranean": 15327, "calcu": 15328, "clown": 15329, "sensor": 15330, "hammer": 15331, "forgive": 15332, "cushi": 15333, "berry": 15334, "majestic": 15335, "elect": 15336, "politan": 15337, "gta": 15338, "kari": 15339, "burke": 15340, "seahawks": 15341, "volkswagen": 15342, "rei": 15343, "landscapes": 15344, "casu": 15345, "grandfather": 15346, "listened": 15347, "//": 15348, "startrek": 15349, "rainfall": 15350, "furry": 15351, "vier": 15352, "stark": 15353, "rifle": 15354, "ffa": 15355, "leges": 15356, "hillaryclinton": 15357, "minus": 15358, "correctly": 15359, "architectural": 15360, "prece": 15361, "upside": 15362, "boxer": 15363, "ðŁĻĮðŁı¼": 15364, "isai": 15365, "det": 15366, "provo": 15367, "tissue": 15368, "spooky": 15369, "veled": 15370, "recon": 15371, "prospects": 15372, "quebec": 15373, "âļ«": 15374, "igno": 15375, "anatomy": 15376, "shapes": 15377, "wp": 15378, "pinterest": 15379, "hore": 15380, "anes": 15381, "pickup": 15382, "tip": 15383, "pradesh": 15384, "hugh": 15385, "coe": 15386, "pok": 15387, "grammy": 15388, "wellington": 15389, "stigate": 15390, "righ": 15391, "leap": 15392, "kingston": 15393, "scenic": 15394, "gosh": 15395, "vani": 15396, "aug": 15397, "sary": 15398, "zier": 15399, "bureau": 15400, "linson": 15401, "conte": 15402, "fragr": 15403, "allan": 15404, "gaw": 15405, "lana": 15406, "collision": 15407, "surveill": 15408, "renais": 15409, "arrange": 15410, "sali": 15411, "doin": 15412, "brance": 15413, "brendan": 15414, "ourse": 15415, "incoming": 15416, "suspension": 15417, "à´": 15418, "lla": 15419, "educators": 15420, "intri": 15421, "dae": 15422, "biography": 15423, "bulgar": 15424, "villain": 15425, "gothic": 15426, "rwanda": 15427, "ew": 15428, "mayor": 15429, "meetup": 15430, "democrat": 15431, "morgan": 15432, "sudden": 15433, "tesco": 15434, "carrot": 15435, "bomber": 15436, "mckin": 15437, "rene": 15438, "funday": 15439, "agricultural": 15440, "hahah": 15441, "showtime": 15442, "forming": 15443, "cola": 15444, "scorpi": 15445, "quote": 15446, "poppy": 15447, "slife": 15448, "daz": 15449, "tub": 15450, "nen": 15451, "mot": 15452, "ðŁĺ»": 15453, "sore": 15454, "elderly": 15455, "ove": 15456, "skinny": 15457, "umi": 15458, "anco": 15459, "manship": 15460, "were": 15461, "gv": 15462, "kah": 15463, "folding": 15464, "neat": 15465, "samantha": 15466, "danish": 15467, "ukrain": 15468, "humidity": 15469, "nutri": 15470, "jakarta": 15471, "candles": 15472, "oooooooo": 15473, "atile": 15474, "strength": 15475, "ibra": 15476, "bapti": 15477, "charleston": 15478, "frames": 15479, "girls": 15480, "clearing": 15481, "gluten": 15482, "##": 15483, "supernatural": 15484, "jubi": 15485, "phone": 15486, "hein": 15487, "drun": 15488, "leak": 15489, "investor": 15490, "yer": 15491, "domain": 15492, "ballroom": 15493, "mish": 15494, "appli": 15495, "offshore": 15496, "blaze": 15497, "doro": 15498, "âĺķï¸ı": 15499, "winery": 15500, "sharif": 15501, "adore": 15502, "nir": 15503, "safer": 15504, "sigh": 15505, "ascri": 15506, "strongly": 15507, "tracy": 15508, "cker": 15509, "oll": 15510, "faithful": 15511, "eyed": 15512, "delightful": 15513, "vism": 15514, "karnataka": 15515, "titan": 15516, "whar": 15517, "jerseys": 15518, "refur": 15519, "heaven": 15520, "grip": 15521, "panama": 15522, "preli": 15523, "gluten": 15524, "odd": 15525, "content": 15526, "ponti": 15527, "tioning": 15528, "ecommerce": 15529, "federation": 15530, "flawless": 15531, "gear": 15532, "tires": 15533, "byr": 15534, "police": 15535, "cuban": 15536, "tributes": 15537, "ticul": 15538, "churches": 15539, "nursery": 15540, "diaries": 15541, "museums": 15542, "snapped": 15543, "ivan": 15544, "wight": 15545, "tourists": 15546, "ramadan": 15547, "trent": 15548, "prophet": 15549, "wondered": 15550, "focusing": 15551, "hid": 15552, "icons": 15553, "iq": 15554, "ambulance": 15555, "pist": 15556, "funniest": 15557, "timeless": 15558, "srilan": 15559, "buys": 15560, "kids": 15561, "colourful": 15562, "ashi": 15563, "chir": 15564, "mum": 15565, "ðŁĵļ": 15566, "letter": 15567, "xen": 15568, "reuters": 15569, "preserve": 15570, "inting": 15571, "step": 15572, "fuji": 15573, "univer": 15574, "iu": 15575, "showdown": 15576, "poems": 15577, "surveillance": 15578, "suspected": 15579, "tae": 15580, "solving": 15581, "tomb": 15582, "mothersday": 15583, "carpen": 15584, "recruit": 15585, "pilots": 15586, "broc": 15587, "mixing": 15588, "fridays": 15589, "tyr": 15590, "representatives": 15591, "trapped": 15592, "abdul": 15593, "freestyle": 15594, "cluster": 15595, "âļłï¸ı": 15596, "kd": 15597, "skill": 15598, "pitt": 15599, "exo": 15600, "commerci": 15601, "museum": 15602, "locally": 15603, "gina": 15604, "nobel": 15605, "immune": 15606, "frac": 15607, "capsu": 15608, "mained": 15609, "attempts": 15610, "bulldog": 15611, "bespoke": 15612, "singers": 15613, "spelling": 15614, "segment": 15615, "natures": 15616, "tick": 15617, "lipstick": 15618, "cleaner": 15619, "gettable": 15620, "precision": 15621, "âĢ¼ï¸ı": 15622, "thood": 15623, "reef": 15624, "nope": 15625, "billy": 15626, "digi": 15627, "musi": 15628, "rival": 15629, "figured": 15630, "tality": 15631, "sunny": 15632, "berk": 15633, "awww": 15634, "awaits": 15635, "unreal": 15636, "copen": 15637, "asylum": 15638, "exotic": 15639, "buen": 15640, "mock": 15641, "enable": 15642, "archy": 15643, "fra": 15644, "plastic": 15645, "almond": 15646, "ampli": 15647, "displays": 15648, "abbott": 15649, "sme": 15650, "xp": 15651, "ðŁĻĥ": 15652, "graphic": 15653, "ived": 15654, "mara": 15655, "caution": 15656, "leaks": 15657, "enberg": 15658, "ulu": 15659, "unicorn": 15660, "cannon": 15661, "apprentic": 15662, "ðŁĺĺðŁĺĺ": 15663, "bball": 15664, "willow": 15665, "atics": 15666, "amas": 15667, "manufacturer": 15668, "campaigns": 15669, "porters": 15670, "floors": 15671, "lsu": 15672, "type": 15673, "kej": 15674, "honorary": 15675, "itim": 15676, "tole": 15677, "minecraft": 15678, "dx": 15679, "mash": 15680, "rio": 15681, "consequences": 15682, "ronald": 15683, "gossi": 15684, "suffolk": 15685, "muse": 15686, "rbi": 15687, "livemusic": 15688, "ivan": 15689, "ðŁİ¤": 15690, "leu": 15691, "patriot": 15692, "manit": 15693, "lanca": 15694, "homedecor": 15695, "dear": 15696, "sigma": 15697, "tide": 15698, "strings": 15699, "vita": 15700, "sequel": 15701, "tryna": 15702, "investigate": 15703, "boris": 15704, "vegan": 15705, "barrier": 15706, "mindfulness": 15707, "webb": 15708, "hustle": 15709, "inda": 15710, "tanzania": 15711, "stray": 15712, "texas": 15713, "cag": 15714, "diagnosis": 15715, "woman": 15716, "gw": 15717, "obsession": 15718, "lative": 15719, "nufc": 15720, "flynn": 15721, "momentum": 15722, "sofa": 15723, "wald": 15724, "vegetable": 15725, "tucker": 15726, "supper": 15727, "seab": 15728, "arro": 15729, "seag": 15730, "venting": 15731, "councill": 15732, "splat": 15733, "calcul": 15734, "..#": 15735, "comfy": 15736, "odisha": 15737, "stopp": 15738, "warfare": 15739, "caes": 15740, "à¨": 15741, "coy": 15742, "priceless": 15743, "insec": 15744, "ðŁĺĽ": 15745, "controls": 15746, "empowerment": 15747, "datascience": 15748, "perpe": 15749, "genic": 15750, "eres": 15751, "trudeau": 15752, "mano": 15753, "slavery": 15754, "expanding": 15755, "mahe": 15756, "failing": 15757, "saga": 15758, "photographs": 15759, "crest": 15760, "reon": 15761, "surfing": 15762, "hie": 15763, "ðŁįĢ": 15764, "jae": 15765, "fellows": 15766, "southampton": 15767, "solom": 15768, "cester": 15769, "tability": 15770, "horn": 15771, "sect": 15772, "hee": 15773, "coleman": 15774, "atlas": 15775, "explorer": 15776, "consultation": 15777, "copyright": 15778, "organizing": 15779, "denied": 15780, "monkeys": 15781, "noodles": 15782, "bris": 15783, "flor": 15784, "dough": 15785, "bonds": 15786, "shocked": 15787, "ecosystem": 15788, "carefully": 15789, "wm": 15790, "apartments": 15791, "curve": 15792, "sandiego": 15793, "mustard": 15794, "commen": 15795, "ceremon": 15796, "ech": 15797, "ruth": 15798, "ðŁĻĮðŁı»": 15799, "hawai": 15800, "filmed": 15801, "tear": 15802, "asingly": 15803, "cair": 15804, "watt": 15805, "instrument": 15806, "outta": 15807, "yeol": 15808, "riverside": 15809, "ë°": 15810, ".:": 15811, "norwich": 15812, "alog": 15813, "migrants": 15814, "newman": 15815, "ride": 15816, "sprink": 15817, "targeting": 15818, "believe": 15819, "torch": 15820, "reflects": 15821, "permission": 15822, "ffman": 15823, "enemies": 15824, "basics": 15825, "seized": 15826, "sundays": 15827, "lei": 15828, "hassan": 15829, "endo": 15830, "hc": 15831, "stad": 15832, "lements": 15833, "kkkk": 15834, "nano": 15835, "shark": 15836, "mana": 15837, "onic": 15838, "treatments": 15839, "early": 15840, "collaborative": 15841, "shuttle": 15842, "branches": 15843, "misses": 15844, "mainedcm": 15845, "apers": 15846, "kyle": 15847, "carrie": 15848, "leisure": 15849, "shet": 15850, "birding": 15851, "advances": 15852, "ðŁĵĿ": 15853, "popular": 15854, "diane": 15855, "abe": 15856, "rewar": 15857, "neighbour": 15858, "kpop": 15859, "remembrance": 15860, "playground": 15861, "rub": 15862, "krishna": 15863, "ebola": 15864, "inquiry": 15865, "epa": 15866, "lumin": 15867, "organisation": 15868, "abraham": 15869, "normally": 15870, "preten": 15871, "janet": 15872, "wt": 15873, "ðŁĴİ": 15874, "encouraging": 15875, "astic": 15876, "bump": 15877, "sydney": 15878, "sz": 15879, "ssss": 15880, "garrett": 15881, "ðŁĵ»": 15882, "consulting": 15883, "romania": 15884, "spotting": 15885, "chancellor": 15886, "arma": 15887, "prestigious": 15888, "ðĿIJ": 15889, "tad": 15890, "cryst": 15891, "competit": 15892, "ratio": 15893, "cataly": 15894, "brow": 15895, "jur": 15896, "viking": 15897, "commute": 15898, "yday": 15899, "layers": 15900, "dumb": 15901, "escal": 15902, "genocide": 15903, "fill": 15904, "gupta": 15905, "stepping": 15906, "sei": 15907, "foto": 15908, "wildcats": 15909, "coli": 15910, "project": 15911, "earnings": 15912, "str": 15913, "geons": 15914, "completion": 15915, "bm": 15916, "decorated": 15917, "crawford": 15918, "afghan": 15919, "scare": 15920, "visibility": 15921, "hib": 15922, "direction": 15923, "stroll": 15924, "christina": 15925, "alternate": 15926, "clare": 15927, "stylist": 15928, "behold": 15929, "sance": 15930, "leopard": 15931, "acquired": 15932, "narrative": 15933, "ashi": 15934, "thea": 15935, "????": 15936, "peas": 15937, "atch": 15938, "slides": 15939, "leen": 15940, "renewable": 15941, "english": 15942, "quir": 15943, "coaster": 15944, "rx": 15945, "fools": 15946, "matchday": 15947, "mism": 15948, "amazing": 15949, "zig": 15950, "keting": 15951, "wont": 15952, "towel": 15953, "diab": 15954, "stake": 15955, "nm": 15956, "melt": 15957, "ethan": 15958, "grape": 15959, "politician": 15960, "smen": 15961, "íĺ": 15962, "reo": 15963, "weddings": 15964, "catcher": 15965, "oracle": 15966, "memo": 15967, "ðŁĮ´": 15968, "eck": 15969, "robbie": 15970, "norwegian": 15971, "operator": 15972, "amor": 15973, "sewing": 15974, "jul": 15975, "xie": 15976, "uv": 15977, "fifty": 15978, "mega": 15979, "tattoo": 15980, "liberals": 15981, "upri": 15982, "trafficking": 15983, "richardson": 15984, "suv": 15985, "kip": 15986, "messy": 15987, "tremendous": 15988, "glou": 15989, "courtney": 15990, "lad": 15991, "stereo": 15992, "myers": 15993, "idio": 15994, "^_^": 15995, "manning": 15996, "dye": 15997, "wd": 15998, "throne": 15999, "junk": 16000, "asu": 16001, "provincial": 16002, "kook": 16003, "wrc": 16004, "fineart": 16005, "hampshire": 16006, "renaissance": 16007, "bred": 16008, "fallout": 16009, "sj": 16010, "snl": 16011, "alam": 16012, "torture": 16013, "fyi": 16014, "shines": 16015, "paw": 16016, "char": 16017, "henry": 16018, "crow": 16019, "acious": 16020, "dian": 16021, "paige": 16022, "bare": 16023, "stockholm": 16024, "scenery": 16025, "ðŁĩ·": 16026, "jeffrey": 16027, "push": 16028, "decoration": 16029, "ned": 16030, "cute": 16031, "brigade": 16032, "lavender": 16033, "invites": 16034, "esports": 16035, "voir": 16036, "dried": 16037, "transpl": 16038, "surgeon": 16039, "novels": 16040, "pulls": 16041, "sony": 16042, "lunar": 16043, "mane": 16044, "ivy": 16045, "frustr": 16046, "dorset": 16047, "sai": 16048, "torres": 16049, "ssion": 16050, "shutdown": 16051, "suggestions": 16052, "writing": 16053, "eo": 16054, "battlefield": 16055, "uga": 16056, "ðŁIJ¾": 16057, "vacu": 16058, "splac": 16059, "git": 16060, "ug": 16061, "highland": 16062, "%)": 16063, "mermaid": 16064, "sacramento": 16065, "tails": 16066, "pw": 16067, "kah": 16068, "tell": 16069, "enhanced": 16070, "ìķ": 16071, "auckland": 16072, "cruel": 16073, "ðŁ¤©": 16074, "audre": 16075, "sailor": 16076, "grammar": 16077, "glove": 16078, "deon": 16079, "inflam": 16080, "freshly": 16081, "kell": 16082, "zip": 16083, "christie": 16084, "mild": 16085, "dixon": 16086, "instructor": 16087, "gence": 16088, "ãħł": 16089, "subjec": 16090, "constitutional": 16091, "crowds": 16092, "invisible": 16093, "ruins": 16094, "dak": 16095, "sip": 16096, "plaque": 16097, "pouring": 16098, "complex": 16099, "zine": 16100, "stead": 16101, "flet": 16102, "transmission": 16103, "loway": 16104, "arun": 16105, "increasingly": 16106, "aud": 16107, "transparen": 16108, "crowned": 16109, "scoun": 16110, "blizzard": 16111, "luxu": 16112, "fiers": 16113, "achievements": 16114, "hunters": 16115, "rocked": 16116, "basin": 16117, "violet": 16118, "proves": 16119, "achieving": 16120, "prosper": 16121, "sega": 16122, "float": 16123, "vian": 16124, "xiv": 16125, "polic": 16126, "tura": 16127, "approximately": 16128, "wanderlust": 16129, "keepers": 16130, "getaway": 16131, "cod": 16132, "polis": 16133, "bryan": 16134, "colts": 16135, "talents": 16136, "yogur": 16137, "glutenfree": 16138, "wrist": 16139, "gry": 16140, "czech": 16141, "ðŁİĪ": 16142, "eville": 16143, "ðŁıĪ": 16144, "tox": 16145, "daniels": 16146, "amer": 16147, "bids": 16148, "weareone": 16149, "metab": 16150, "gt": 16151, "boyz": 16152, "pdx": 16153, "possession": 16154, "pushed": 16155, "shrine": 16156, "realistic": 16157, "trigger": 16158, "navi": 16159, "rumors": 16160, "naf": 16161, "jenkins": 16162, "trun": 16163, "communi": 16164, "ÃĹ": 16165, "gamers": 16166, "armor": 16167, "mohammed": 16168, "balcony": 16169, "yah": 16170, "strongest": 16171, "rhythm": 16172, "unforgettable": 16173, "kp": 16174, "hobb": 16175, "custody": 16176, "gregor": 16177, "rita": 16178, "aesthetic": 16179, "ilation": 16180, "sponsoring": 16181, "nay": 16182, "kidnapp": 16183, "shs": 16184, "rajas": 16185, "meg": 16186, "significantly": 16187, "buttons": 16188, "lac": 16189, "versions": 16190, "essentials": 16191, "opinions": 16192, "kro": 16193, "dprinting": 16194, "widely": 16195, "dk": 16196, "uran": 16197, "yal": 16198, "requested": 16199, "cn": 16200, "curric": 16201, "plum": 16202, "grun": 16203, "vm": 16204, "devon": 16205, "myo": 16206, "relation": 16207, "juventus": 16208, "rouge": 16209, "minority": 16210, "mines": 16211, "jupiter": 16212, "nine": 16213, "oxygen": 16214, "frankie": 16215, "unesco": 16216, "fabric": 16217, "disgusting": 16218, "salman": 16219, "detection": 16220, "lanka": 16221, "dac": 16222, "ðŁĩ«ðŁĩ·": 16223, "argument": 16224, "shelves": 16225, "celtics": 16226, "roberto": 16227, "pigs": 16228, "hedge": 16229, "faul": 16230, "powering": 16231, "butterflies": 16232, "fir": 16233, "remake": 16234, "atti": 16235, "como": 16236, "empha": 16237, "kendall": 16238, "pokemon": 16239, "seating": 16240, "dans": 16241, "baldwin": 16242, "ðŁij»": 16243, "leslie": 16244, "onedirection": 16245, "timber": 16246, "iman": 16247, "font": 16248, "eder": 16249, "dion": 16250, "steph": 16251, "format": 16252, "gregory": 16253, "prop": 16254, "hex": 16255, "ruin": 16256, "sory": 16257, "infer": 16258, "naw": 16259, "barak": 16260, "sdgs": 16261, "karao": 16262, "lush": 16263, "vander": 16264, "endent": 16265, "gis": 16266, "afro": 16267, "soccer": 16268, "ayan": 16269, "tuni": 16270, "lung": 16271, "dayof": 16272, "alexa": 16273, "marath": 16274, "addicted": 16275, "agile": 16276, "hygi": 16277, "lightweight": 16278, "ì§": 16279, "mandela": 16280, "joey": 16281, "ancy": 16282, "hum": 16283, "bir": 16284, "memorial": 16285, "jimin": 16286, "ginger": 16287, "vak": 16288, "javascri": 16289, "crops": 16290, "origins": 16291, "dari": 16292, "piper": 16293, "import": 16294, "aggressive": 16295, "prediction": 16296, "repairs": 16297, "cracker": 16298, "voyage": 16299, "nike": 16300, "mummy": 16301, "linkedin": 16302, "countryside": 16303, "border": 16304, "glass": 16305, "pert": 16306, "sals": 16307, "shoe": 16308, "autographed": 16309, "walnut": 16310, "collegi": 16311, "salary": 16312, "pairing": 16313, "ðŁĮ¸": 16314, "cathol": 16315, "sweethe": 16316, "defeats": 16317, "strengthen": 16318, "rooftop": 16319, "improvements": 16320, "barriers": 16321, "uru": 16322, "tally": 16323, "ruled": 16324, "ðŁĨļ": 16325, "naija": 16326, "emoji": 16327, "percent": 16328, "gio": 16329, "probs": 16330, "once": 16331, "admits": 16332, "paths": 16333, "liar": 16334, "daytona": 16335, "peters": 16336, "cali": 16337, "calli": 16338, "mug": 16339, "osa": 16340, "aph": 16341, "aby": 16342, "hyde": 16343, "ethnic": 16344, "plains": 16345, "olf": 16346, "hahahahaha": 16347, "holic": 16348, "?!?!": 16349, "subli": 16350, "blacks": 16351, "mot": 16352, "ghton": 16353, "lovin": 16354, "brent": 16355, "baru": 16356, "lati": 16357, "dew": 16358, "ateau": 16359, "qa": 16360, "painful": 16361, "busters": 16362, "static": 16363, "ðŁĩ¨ðŁĩ¦": 16364, "notebook": 16365, "outfits": 16366, "sies": 16367, "rf": 16368, "floods": 16369, "ÑĢ": 16370, "throat": 16371, "suici": 16372, "rovers": 16373, "bengal": 16374, "prepares": 16375, "blog": 16376, "miniature": 16377, "ب": 16378, "amphi": 16379, "comb": 16380, "rsp": 16381, "intimate": 16382, "greene": 16383, "Ìĩ": 16384, "altar": 16385, "surgical": 16386, "vessel": 16387, "...?": 16388, "gavin": 16389, "gator": 16390, "threatened": 16391, "zar": 16392, "robbery": 16393, "dier": 16394, "promoted": 16395, "yg": 16396, "xs": 16397, "subs": 16398, "interviewing": 16399, "threatening": 16400, "dozen": 16401, "meado": 16402, "waterfall": 16403, "nintendoswitch": 16404, "calum": 16405, "ministers": 16406, "drop": 16407, "universities": 16408, "warned": 16409, "tactics": 16410, "ðŁĩ²": 16411, "refuse": 16412, "adju": 16413, "vast": 16414, "ðŁĺ´": 16415, "mcfc": 16416, "libya": 16417, "nofilter": 16418, "distributed": 16419, "reser": 16420, "ronnie": 16421, "deco": 16422, "javascript": 16423, "monk": 16424, "interests": 16425, "flex": 16426, "martha": 16427, "sties": 16428, "ood": 16429, "ðŁ¤£ðŁ¤£": 16430, "eun": 16431, "bali": 16432, "gomez": 16433, "stimul": 16434, "moderate": 16435, "dity": 16436, "iris": 16437, "straw": 16438, "consistent": 16439, "directions": 16440, "adopt": 16441, "salsa": 16442, "croo": 16443, "recovered": 16444, "blackfriday": 16445, "lancaster": 16446, "accept": 16447, "weareoneexo": 16448, "builds": 16449, "freeman": 16450, "airplane": 16451, "dition": 16452, "belong": 16453, "jamie": 16454, "pitching": 16455, "lif": 16456, "omin": 16457, "crispy": 16458, "prepping": 16459, "veg": 16460, "chang": 16461, "accomplished": 16462, "gracias": 16463, "dolphin": 16464, "elector": 16465, "culinary": 16466, "superbowl": 16467, "wala": 16468, "pursuit": 16469, "blackberry": 16470, "bean": 16471, "cardinal": 16472, "proved": 16473, "immigrant": 16474, "strictly": 16475, "holocaust": 16476, "passage": 16477, "haus": 16478, "coup": 16479, "purse": 16480, "harass": 16481, "<<": 16482, "leed": 16483, "adobe": 16484, "stad": 16485, "legislat": 16486, "parked": 16487, "priyan": 16488, "silva": 16489, "krist": 16490, "sthe": 16491, "funky": 16492, "iga": 16493, "settlement": 16494, "phs": 16495, "tmrw": 16496, "stressed": 16497, "hunt": 16498, "hockey": 16499, "treasures": 16500, "chambers": 16501, "olu": 16502, "hut": 16503, "marley": 16504, "texture": 16505, "wilderness": 16506, "mming": 16507, "potentially": 16508, "omaha": 16509, "judy": 16510, "toes": 16511, "spoiler": 16512, "distinguished": 16513, "felix": 16514, "ahu": 16515, "recommendations": 16516, "zombies": 16517, "hitler": 16518, "triple": 16519, "collapse": 16520, "motivated": 16521, "ultimat": 16522, "ggling": 16523, "soy": 16524, "cigar": 16525, "foren": 16526, "vineyard": 16527, "glitter": 16528, "findings": 16529, "colonial": 16530, "hunter": 16531, "erik": 16532, "dens": 16533, "beetle": 16534, "lotte": 16535, "subtle": 16536, "smatter": 16537, "trusted": 16538, "experimental": 16539, "naments": 16540, "ðŁĺĨ": 16541, "region": 16542, "acquisition": 16543, "breeding": 16544, "quarterback": 16545, "amreading": 16546, "ootd": 16547, "rude": 16548, "initiatives": 16549, "stout": 16550, "hyung": 16551, "outcome": 16552, "alfred": 16553, "mics": 16554, "expertise": 16555, "bacteria": 16556, "penguins": 16557, "jumper": 16558, "valencia": 16559, "bark": 16560, "ingday": 16561, "sellers": 16562, "contracts": 16563, "houston": 16564, "commissioned": 16565, "adaptation": 16566, "swansea": 16567, "santiago": 16568, "commonwealth": 16569, "judging": 16570, "submission": 16571, "scorer": 16572, "tommy": 16573, "ño": 16574, "exquis": 16575, "filing": 16576, "explanation": 16577, "allison": 16578, "wembley": 16579, "ridge": 16580, "chevy": 16581, "santos": 16582, "ownership": 16583, "cognitive": 16584, "favourites": 16585, "shed": 16586, "philanthro": 16587, "deleted": 16588, "godd": 16589, "snor": 16590, "guidelines": 16591, "ffing": 16592, "jeep": 16593, "clips": 16594, "swamp": 16595, "anor": 16596, "guild": 16597, "bolton": 16598, "springfield": 16599, "municipal": 16600, "goalkeeper": 16601, "yeon": 16602, "ðŁĺįðŁĺįðŁĺįðŁĺį": 16603, "ãħĭãħĭ": 16604, "waterfront": 16605, "grave": 16606, "contemporary": 16607, "arity": 16608, "ÃŃa": 16609, "sleeps": 16610, "syrup": 16611, "alam": 16612, "pire": 16613, "coyo": 16614, "motogp": 16615, "tyson": 16616, "kejri": 16617, "circul": 16618, "singly": 16619, "crunch": 16620, "complicated": 16621, "nostalgia": 16622, "kop": 16623, "move": 16624, "kale": 16625, "macro": 16626, "midwest": 16627, "hans": 16628, "tribal": 16629, "nude": 16630, "à¯į": 16631, "beyonce": 16632, "congratulate": 16633, "cater": 16634, "league": 16635, "ðŁĻĬ": 16636, "ladder": 16637, "crashed": 16638, "technic": 16639, "karaoke": 16640, "harassment": 16641, "rots": 16642, "experiencing": 16643, "kristen": 16644, "ðŁĩ³": 16645, "ðŁ¤Ĺ": 16646, "reflections": 16647, "guinness": 16648, "illustrator": 16649, "ðŁĻıðŁı»": 16650, "center": 16651, "narrow": 16652, "commons": 16653, "regulations": 16654, "ÙĨ": 16655, "harm": 16656, "croft": 16657, "cussion": 16658, "hongkong": 16659, "stical": 16660, "internship": 16661, "zoe": 16662, "chop": 16663, "hoods": 16664, "estimated": 16665, "batteries": 16666, "berkeley": 16667, "smoothie": 16668, "shaun": 16669, "cros": 16670, "~~": 16671, "campe": 16672, "hump": 16673, "bg": 16674, "prototype": 16675, "click": 16676, "shawn": 16677, "reviewed": 16678, "templ": 16679, "pf": 16680, "jedi": 16681, "blogs": 16682, "raymond": 16683, "asth": 16684, "bah": 16685, "avail": 16686, "scotch": 16687, "leafs": 16688, "nikki": 16689, "tok": 16690, "hollow": 16691, "urges": 16692, "oft": 16693, "unlike": 16694, "latin": 16695, "ue": 16696, "catering": 16697, "mili": 16698, "alternati": 16699, "maver": 16700, "и": 16701, "agle": 16702, "preorder": 16703, "lux": 16704, "cucu": 16705, "ðŁijıðŁijı": 16706, "tart": 16707, "âĿ¤âĿ¤âĿ¤": 16708, "arabic": 16709, "rapidly": 16710, "arrang": 16711, "allen": 16712, "traveltuesday": 16713, "paws": 16714, "flows": 16715, "stability": 16716, "fluid": 16717, "capp": 16718, "canberra": 16719, "uuuu": 16720, "spani": 16721, "demonstration": 16722, "mla": 16723, "placement": 16724, "mw": 16725, "presidents": 16726, "awesom": 16727, "beverly": 16728, "anist": 16729, "neal": 16730, "fathersday": 16731, "referendum": 16732, "lahore": 16733, "oaks": 16734, "debbie": 16735, "halfway": 16736, "ghosts": 16737, "debor": 16738, "matthews": 16739, "fiat": 16740, "tfw": 16741, "presen": 16742, "robi": 16743, "ded": 16744, "brock": 16745, "laughed": 16746, "amounts": 16747, "bamboo": 16748, "kindergarten": 16749, "eaten": 16750, "mtvhottest": 16751, "breakout": 16752, "usic": 16753, "fraser": 16754, "legislative": 16755, "pang": 16756, "module": 16757, "sammy": 16758, "gover": 16759, "earns": 16760, "expedition": 16761, "garh": 16762, "concepts": 16763, "charlie": 16764, "lava": 16765, "bachelor": 16766, "veggies": 16767, "determine": 16768, "ellie": 16769, "unlocked": 16770, "fruit": 16771, "dalla": 16772, "coupe": 16773, "washington": 16774, "deposit": 16775, "ivory": 16776, "paula": 16777, "chicag": 16778, "gucci": 16779, "ðŁİĥ": 16780, "cultiv": 16781, "pierce": 16782, "lifted": 16783, "stumb": 16784, "recover": 16785, "muscles": 16786, "conducting": 16787, "cbs": 16788, "mclaren": 16789, "sophia": 16790, "cellu": 16791, "oceans": 16792, "uploaded": 16793, "gameplay": 16794, "maldives": 16795, "kimber": 16796, "avoi": 16797, "racer": 16798, "caine": 16799, "cavs": 16800, "hana": 16801, "liga": 16802, "raven": 16803, "intervention": 16804, "inauguration": 16805, "ooh": 16806, "attraction": 16807, "merchandise": 16808, "tunein": 16809, "liking": 16810, "juniors": 16811, "intended": 16812, "attacking": 16813, "aquarium": 16814, "iwd": 16815, "components": 16816, "suring": 16817, "centu": 16818, "yogurt": 16819, "ðŁıĥ": 16820, "showroom": 16821, "optical": 16822, "tyour": 16823, "judge": 16824, "yield": 16825, "anto": 16826, "plc": 16827, "transparency": 16828, "recycled": 16829, "chief": 16830, "arom": 16831, "ambassadors": 16832, "planet": 16833, "âĿĦï¸ı": 16834, "omed": 16835, "vanessa": 16836, "court": 16837, "margar": 16838, "haley": 16839, "vr": 16840, "regina": 16841, "pdates": 16842, "hispan": 16843, "livestream": 16844, "âģ£": 16845, "yahoo": 16846, "galla": 16847, "secured": 16848, "wir": 16849, "beneath": 16850, "offl": 16851, "nil": 16852, "amb": 16853, "yeg": 16854, "outlet": 16855, "ute": 16856, "peep": 16857, "lindsay": 16858, "bentley": 16859, "...!": 16860, "heel": 16861, "trilogy": 16862, "vos": 16863, "tyre": 16864, "therefore": 16865, "toronto": 16866, "abi": 16867, "simpli": 16868, "jae": 16869, "extensive": 16870, "elephants": 16871, "sor": 16872, "orientation": 16873, "impeach": 16874, "replay": 16875, "constructed": 16876, "peterson": 16877, "pais": 16878, "ported": 16879, "customs": 16880, "collap": 16881, "adu": 16882, "highlands": 16883, "salem": 16884, "shelby": 16885, "kovic": 16886, "strain": 16887, "rosie": 16888, "senators": 16889, "snaps": 16890, "bobb": 16891, "suzuki": 16892, "blades": 16893, "kp": 16894, "lolo": 16895, "generate": 16896, "sight": 16897, "mae": 16898, "structural": 16899, "predict": 16900, "jumped": 16901, "ahmad": 16902, "sung": 16903, "justice": 16904, "glam": 16905, "volvo": 16906, "jubilee": 16907, "detention": 16908, "losses": 16909, "puri": 16910, "everytime": 16911, "а": 16912, "rao": 16913, "edge": 16914, "limer": 16915, "resemb": 16916, "harold": 16917, "retri": 16918, "sacrific": 16919, "surprises": 16920, "amc": 16921, "srilanka": 16922, "barbie": 16923, "mens": 16924, "finn": 16925, "ags": 16926, "ukrainian": 16927, "embrac": 16928, "îIJ": 16929, "flavors": 16930, "homer": 16931, "laure": 16932, "outh": 16933, "priced": 16934, "verde": 16935, "firm": 16936, "ahs": 16937, "cub": 16938, "trey": 16939, "paranor": 16940, "profit": 16941, "indv": 16942, "whoa": 16943, "harsh": 16944, "alot": 16945, "critics": 16946, "hubby": 16947, "figur": 16948, "gira": 16949, "castro": 16950, "chanel": 16951, "input": 16952, "originals": 16953, "tenant": 16954, "yyyy": 16955, "turers": 16956, "lincoln": 16957, "coon": 16958, "learn": 16959, "chou": 16960, "acare": 16961, "oles": 16962, "diner": 16963, "hyp": 16964, "bizarre": 16965, "mcr": 16966, "letsgo": 16967, "decorating": 16968, "ðŁĮİ": 16969, "alison": 16970, "arvin": 16971, "fd": 16972, "rehab": 16973, "mccarthy": 16974, "lottery": 16975, "dah": 16976, "minneapolis": 16977, "eligible": 16978, "diagnosed": 16979, "emerald": 16980, "destinations": 16981, "sans": 16982, "ory": 16983, "blazers": 16984, "nv": 16985, "bail": 16986, "digitalart": 16987, "noc": 16988, "malta": 16989, "solar": 16990, "pipes": 16991, "allegations": 16992, "nock": 16993, "pope": 16994, "brid": 16995, "premier": 16996, "nx": 16997, "presentations": 16998, "efa": 16999, "bows": 17000, "valve": 17001, "opponent": 17002, "Įë": 17003, "visual": 17004, "ingle": 17005, "categor": 17006, "eter": 17007, "pois": 17008, "dani": 17009, "attract": 17010, "neutral": 17011, "thene": 17012, "crashes": 17013, "freddie": 17014, "utili": 17015, "cst": 17016, "awakening": 17017, "sloven": 17018, "qualify": 17019, "proof": 17020, "fairy": 17021, "lev": 17022, "freight": 17023, "enjoys": 17024, "cupcake": 17025, "flavour": 17026, "âķ": 17027, "protective": 17028, "ðŁijıðŁı»": 17029, "isu": 17030, "admir": 17031, "hmmm": 17032, "continuous": 17033, "aires": 17034, "raptors": 17035, "showcasing": 17036, "yuk": 17037, "paste": 17038, "follower": 17039, "instructions": 17040, "spru": 17041, "@__": 17042, "theo": 17043, "debuts": 17044, "vette": 17045, "stow": 17046, "esof": 17047, "ached": 17048, "sultan": 17049, "sandwich": 17050, "somalia": 17051, "franco": 17052, "carne": 17053, "fluffy": 17054, "alpine": 17055, "jasmine": 17056, "heated": 17057, "violin": 17058, "pless": 17059, "divorce": 17060, "performer": 17061, "phies": 17062, "portsm": 17063, "dara": 17064, "kirby": 17065, "lop": 17066, "chilli": 17067, "forth": 17068, "skype": 17069, "ðŁĩ®ðŁĩ¹": 17070, "celebrities": 17071, "edy": 17072, "vee": 17073, "poison": 17074, "eyel": 17075, "grabs": 17076, "ssic": 17077, "uno": 17078, "western": 17079, "railroad": 17080, "amer": 17081, "numerous": 17082, "sv": 17083, "fow": 17084, "fist": 17085, "âĢĭ": 17086, "requests": 17087, "martial": 17088, "emmy": 17089, "acceptance": 17090, "laura": 17091, "ิ": 17092, "erup": 17093, "hyundai": 17094, "outlander": 17095, "utt": 17096, "wrestle": 17097, "espresso": 17098, "demanding": 17099, "gdp": 17100, "geography": 17101, "saskat": 17102, "troll": 17103, "confeder": 17104, "sues": 17105, "sem": 17106, "bets": 17107, "tful": 17108, "tosh": 17109, "teaches": 17110, "coloured": 17111, "galway": 17112, "macy": 17113, "disorders": 17114, "bbcra": 17115, "atem": 17116, "fender": 17117, "litter": 17118, "esh": 17119, "providers": 17120, "renovation": 17121, "nominate": 17122, "psg": 17123, "nominations": 17124, "jenna": 17125, "sharp": 17126, "someday": 17127, "zur": 17128, "brains": 17129, "cheshire": 17130, "prey": 17131, "hugo": 17132, "¿": 17133, "token": 17134, "rv": 17135, "carr": 17136, "tactical": 17137, "zelda": 17138, "kayla": 17139, "fernando": 17140, "photographers": 17141, "jour": 17142, "umbrella": 17143, "woody": 17144, "congressman": 17145, "dump": 17146, "levy": 17147, "juan": 17148, "dazz": 17149, "signals": 17150, "lain": 17151, "anu": 17152, "michel": 17153, "porch": 17154, "alden": 17155, "siblings": 17156, "yale": 17157, "peel": 17158, "swick": 17159, "ggin": 17160, "llc": 17161, "kale": 17162, "scon": 17163, "ild": 17164, "patreon": 17165, "reel": 17166, "quin": 17167, "witt": 17168, "marty": 17169, "moody": 17170, "toni": 17171, "dery": 17172, "gators": 17173, "specifically": 17174, "ddin": 17175, "lyon": 17176, "trick": 17177, "meadows": 17178, "pj": 17179, "borgh": 17180, "vik": 17181, "tur": 17182, "bronx": 17183, "puff": 17184, "lantern": 17185, "ðŁ¤¦": 17186, "gently": 17187, "bestie": 17188, "fact": 17189, "refused": 17190, "fasci": 17191, "mpy": 17192, "ðŁĶµ": 17193, "crossover": 17194, "meadow": 17195, "indianapolis": 17196, "ducation": 17197, "sley": 17198, "loom": 17199, "mixer": 17200, "newmusic": 17201, "filmmaker": 17202, "prosperity": 17203, "lim": 17204, "weekend": 17205, "creamy": 17206, "neutr": 17207, "luther": 17208, "hv": 17209, "northern": 17210, "two": 17211, "hra": 17212, "catches": 17213, "appearances": 17214, "habit": 17215, "kittens": 17216, "nv": 17217, "illac": 17218, "infan": 17219, "regardless": 17220, "lizard": 17221, "dunk": 17222, "curtain": 17223, "acom": 17224, "intu": 17225, "vez": 17226, "emin": 17227, "flats": 17228, "calendars": 17229, "empower": 17230, "ruined": 17231, "hungary": 17232, "vid": 17233, "wex": 17234, "ulum": 17235, "aberdeen": 17236, "osa": 17237, "kt": 17238, "massi": 17239, "seemed": 17240, "sden": 17241, "'?": 17242, "telephone": 17243, "defi": 17244, "inspires": 17245, "meow": 17246, "zones": 17247, "blind": 17248, "ply": 17249, "tucson": 17250, "adventure": 17251, "ged": 17252, "oyster": 17253, "ðŁijıðŁijıðŁijı": 17254, "output": 17255, "ttt": 17256, "metallic": 17257, "smash": 17258, "ucla": 17259, "scots": 17260, "perfect": 17261, "lucy": 17262, "regularly": 17263, "spic": 17264, "relative": 17265, "athers": 17266, "mise": 17267, "battling": 17268, "decides": 17269, "mata": 17270, "occupied": 17271, "randomly": 17272, "catsoftwitter": 17273, "gian": 17274, "bally": 17275, "alties": 17276, "allies": 17277, "immen": 17278, "syrac": 17279, "ðŁĴľðŁĴľ": 17280, "llan": 17281, "aur": 17282, "kut": 17283, "lamar": 17284, "affects": 17285, "nra": 17286, "starwar": 17287, "ðŁ¤ĺ": 17288, "scram": 17289, "enchan": 17290, "process": 17291, "luxurious": 17292, "array": 17293, "sherlock": 17294, "compati": 17295, "dorf": 17296, "stress": 17297, "msu": 17298, "swith": 17299, "sala": 17300, "sofinstagram": 17301, "foil": 17302, "understood": 17303, "quay": 17304, "rp": 17305, "cade": 17306, "jaw": 17307, "enab": 17308, "encoun": 17309, "ðŁİī:": 17310, "dock": 17311, "saturn": 17312, "mull": 17313, "layout": 17314, "rarely": 17315, "happily": 17316, "fixture": 17317, "orph": 17318, "overlooking": 17319, "herbs": 17320, "mitt": 17321, "pillar": 17322, "nolan": 17323, "petty": 17324, "stry": 17325, "ui": 17326, "muk": 17327, "ores": 17328, "overs": 17329, "áµ": 17330, "recreation": 17331, "wesley": 17332, "rit": 17333, "kejriwal": 17334, "stocking": 17335, "gv": 17336, "subscribers": 17337, "moose": 17338, "mae": 17339, "bert": 17340, "oppre": 17341, "assignment": 17342, "uro": 17343, "highlighting": 17344, "calvin": 17345, "weigh": 17346, "cambodia": 17347, "avon": 17348, "kem": 17349, "disabilities": 17350, "ready": 17351, "chargers": 17352, "pads": 17353, "izing": 17354, "illian": 17355, "truste": 17356, "colleges": 17357, "associates": 17358, "albany": 17359, "milton": 17360, "cron": 17361, "bur": 17362, "hardly": 17363, "sights": 17364, "antiques": 17365, "echo": 17366, "surprisingly": 17367, "haiti": 17368, "capt": 17369, "php": 17370, "opio": 17371, "inequality": 17372, "equal": 17373, "keny": 17374, "schmid": 17375, "autographs": 17376, "rent": 17377, "quer": 17378, "citrus": 17379, "challenged": 17380, "tec": 17381, "epide": 17382, "fest": 17383, "zhou": 17384, "lime": 17385, "citizenship": 17386, "crystal": 17387, "convinced": 17388, "messenger": 17389, "copenhagen": 17390, "âĿĹï¸ı": 17391, "warran": 17392, "developments": 17393, "ï¸ıâĥ£": 17394, "forex": 17395, "hiro": 17396, "sneakers": 17397, "xide": 17398, "viva": 17399, "stereo": 17400, "batting": 17401, "ssel": 17402, "host": 17403, "bengal": 17404, "criticism": 17405, "qc": 17406, "crun": 17407, "attempted": 17408, "rye": 17409, "determination": 17410, "creations": 17411, "dread": 17412, "labels": 17413, "posse": 17414, "ancer": 17415, "johan": 17416, "sister": 17417, "partnerships": 17418, "lesbian": 17419, "kst": 17420, "guarantee": 17421, "baro": 17422, "fixing": 17423, "mason": 17424, "mous": 17425, "chemicals": 17426, "tless": 17427, "biodiversity": 17428, "paro": 17429, "bharat": 17430, "acol": 17431, "refuge": 17432, "ente": 17433, "titi": 17434, "dyssey": 17435, "responds": 17436, "lefto": 17437, "iner": 17438, "sevel": 17439, "rahul": 17440, "oline": 17441, "frankfur": 17442, "choreo": 17443, "enjoyable": 17444, "cto": 17445, "struggles": 17446, "woodland": 17447, "heavyweight": 17448, "gens": 17449, "recep": 17450, "accred": 17451, "ðŁĺ¡": 17452, "transformed": 17453, "listen": 17454, "atop": 17455, "nk": 17456, "surge": 17457, "bere": 17458, "governor": 17459, "prisoners": 17460, "claude": 17461, "till": 17462, "mulator": 17463, "emotion": 17464, "waterloo": 17465, "start": 17466, "ðŁĩº": 17467, "cleaned": 17468, "grandmother": 17469, "fearless": 17470, "african": 17471, "astronomy": 17472, "ðŁıģ": 17473, "à¸Ļ": 17474, "theworld": 17475, "suitable": 17476, "anthony": 17477, "kand": 17478, "tten": 17479, "meaningful": 17480, "disclo": 17481, "jacobs": 17482, "ø": 17483, "tomlinson": 17484, "ghetti": 17485, "typho": 17486, "substan": 17487, "asco": 17488, "tek": 17489, "nagar": 17490, "mud": 17491, "amon": 17492, "vaccine": 17493, "fty": 17494, "flesh": 17495, "noel": 17496, "inflation": 17497, "portugue": 17498, "glamour": 17499, "tram": 17500, "vre": 17501, "tequ": 17502, "roundup": 17503, "wyn": 17504, "rejected": 17505, "mosaic": 17506, "sighting": 17507, "calf": 17508, "ota": 17509, "composition": 17510, "gopro": 17511, "gonzale": 17512, "eed": 17513, "bard": 17514, "tue": 17515, "effectively": 17516, "ween": 17517, "alto": 17518, "ribs": 17519, "relate": 17520, "thirsty": 17521, "furious": 17522, "dim": 17523, "chard": 17524, "perfume": 17525, "sny": 17526, "churchill": 17527, "kof": 17528, "masterclass": 17529, "wave": 17530, "ðŁĶµ": 17531, "erin": 17532, "owns": 17533, "tobe": 17534, "skilled": 17535, "tem": 17536, "gof": 17537, "eni": 17538, "tori": 17539, "crazy": 17540, "lick": 17541, "resistant": 17542, "icial": 17543, "agar": 17544, "!:": 17545, "gali": 17546, "delaware": 17547, "blitz": 17548, "kohli": 17549, "puck": 17550, "availability": 17551, "himalay": 17552, "influential": 17553, "crochet": 17554, "victori": 17555, "reading": 17556, "hobby": 17557, "viet": 17558, "jas": 17559, "engra": 17560, "skul": 17561, "ðŁĩ²ðŁĩ": 17562, "educate": 17563, "techno": 17564, "districts": 17565, "blues": 17566, "sett": 17567, "seventh": 17568, "learns": 17569, "eeee": 17570, "apocalypse": 17571, "hangout": 17572, "cruel": 17573, "mutu": 17574, "bruh": 17575, "helen": 17576, "sheer": 17577, "ction": 17578, "klein": 17579, "texans": 17580, "cereal": 17581, "shine": 17582, "nered": 17583, "gras": 17584, "ambro": 17585, "fella": 17586, "hindu": 17587, "matthew": 17588, "lima": 17589, "miranda": 17590, "jewel": 17591, "soho": 17592, "eurovision": 17593, "neighbours": 17594, "chandler": 17595, "besides": 17596, "ðŁ¥°": 17597, "astros": 17598, "thumbs": 17599, "renault": 17600, "rave": 17601, "hired": 17602, "ðŁĸ¤": 17603, "itary": 17604, "zor": 17605, "blazer": 17606, "kine": 17607, "eau": 17608, "katy": 17609, "dccomics": 17610, "pec": 17611, "rodgers": 17612, "waterproof": 17613, "killers": 17614, "superint": 17615, "preserv": 17616, "asso": 17617, "brewers": 17618, "promotional": 17619, "scam": 17620, "villages": 17621, "sketches": 17622, "juicy": 17623, "forlife": 17624, "audit": 17625, "solo": 17626, "fundamental": 17627, "lene": 17628, "philippine": 17629, "tend": 17630, "conservatives": 17631, "sponsorship": 17632, "ddle": 17633, "aine": 17634, "htc": 17635, "osi": 17636, "hulk": 17637, "waf": 17638, "à¸Ļ": 17639, "evaluation": 17640, "antine": 17641, "slee": 17642, "robertson": 17643, "roosevel": 17644, "agi": 17645, "sophistic": 17646, "employers": 17647, "bubbles": 17648, "kowski": 17649, "interaction": 17650, "shu": 17651, "boule": 17652, "ican": 17653, "jare": 17654, "hank": 17655, "legitim": 17656, "knicks": 17657, "karma": 17658, "receiver": 17659, "perks": 17660, "uh": 17661, "stair": 17662, "suni": 17663, "laboratory": 17664, "graves": 17665, "vocals": 17666, "oot": 17667, "cture": 17668, "thrive": 17669, "tico": 17670, "ãĥ³": 17671, "bw": 17672, "cartoons": 17673, "mcdonalds": 17674, "draw": 17675, "yung": 17676, "pler": 17677, "lid": 17678, "ethical": 17679, "groove": 17680, "enta": 17681, "internationalwomensday": 17682, "patron": 17683, "worries": 17684, "ðŁİħ": 17685, "ðŁijĭ": 17686, "katherine": 17687, "diaz": 17688, "tori": 17689, "bachchan": 17690, "trust": 17691, "mineral": 17692, "icom": 17693, "builders": 17694, "born": 17695, "coloring": 17696, "latte": 17697, "case": 17698, "revolution": 17699, "trader": 17700, "oxid": 17701, "chipot": 17702, "instantly": 17703, "southern": 17704, "sehun": 17705, "prob": 17706, "hernandez": 17707, "lisbon": 17708, "huawe": 17709, "pong": 17710, "mea": 17711, "rooney": 17712, "wheelchair": 17713, "keen": 17714, "bett": 17715, "corin": 17716, "regulatory": 17717, "displac": 17718, "karen": 17719, "schem": 17720, "sunsets": 17721, "whales": 17722, "reminis": 17723, "hep": 17724, "hide": 17725, "marcel": 17726, "pandora": 17727, "doyle": 17728, "thfc": 17729, "otto": 17730, "nokia": 17731, "transgender": 17732, "kov": 17733, "hawaiian": 17734, "shave": 17735, "sovere": 17736, "excer": 17737, "nicki": 17738, "pug": 17739, "stor": 17740, "roth": 17741, "weet": 17742, "legal": 17743, "dignity": 17744, "pow": 17745, "homage": 17746, "ðŁĩ³ðŁĩ": 17747, "sre": 17748, "canon": 17749, "lax": 17750, "woah": 17751, "quartz": 17752, "ña": 17753, "greeting": 17754, "flickr": 17755, "nairobi": 17756, "advocates": 17757, "anc": 17758, "vii": 17759, "eugene": 17760, "thra": 17761, "cre": 17762, "elan": 17763, "pension": 17764, "thletics": 17765, "toni": 17766, "reagan": 17767, "xv": 17768, "store": 17769, "bench": 17770, "harlem": 17771, "toddler": 17772, "sentenced": 17773, "âĻ¥ï¸ı": 17774, "globally": 17775, "cheaper": 17776, "uf": 17777, "mam": 17778, "nico": 17779, "iku": 17780, "thou": 17781, "nist": 17782, "dami": 17783, "thala": 17784, "rhodes": 17785, "sale": 17786, "bowls": 17787, "âĪ": 17788, "lasvegas": 17789, "sanctions": 17790, "admire": 17791, "matched": 17792, "unable": 17793, "traveler": 17794, "eleven": 17795, "strawberries": 17796, "âĢĶâĢĶâĢĶâĢĶ": 17797, "studio": 17798, "jacques": 17799, "ims": 17800, "valued": 17801, "sno": 17802, "cheesecake": 17803, "nxt": 17804, "eos": 17805, "sx": 17806, "fx": 17807, "tonic": 17808, "hatch": 17809, "chicks": 17810, "grads": 17811, "handic": 17812, "rory": 17813, "asp": 17814, "ripped": 17815, "dentist": 17816, "nen": 17817, "lufc": 17818, "âľĬ": 17819, "dige": 17820, "hopkins": 17821, "sherman": 17822, "fda": 17823, "forall": 17824, "ashley": 17825, "strand": 17826, "hy": 17827, "liquor": 17828, "buffet": 17829, "essence": 17830, "pharma": 17831, "suriya": 17832, "ðŁĴĻðŁĴĻ": 17833, "festivals": 17834, "zan": 17835, "refresh": 17836, "purple": 17837, "uniforms": 17838, "kenneth": 17839, "=)": 17840, "asan": 17841, "helsin": 17842, "transformers": 17843, "kali": 17844, "personalized": 17845, "chalk": 17846, "bobby": 17847, "âĮ": 17848, "themes": 17849, "departure": 17850, "print": 17851, "illustrations": 17852, "quiet": 17853, "agrees": 17854, "griff": 17855, "س": 17856, "miti": 17857, "together": 17858, "convenience": 17859, "abar": 17860, "carlo": 17861, "turtles": 17862, "infosec": 17863, "somewhat": 17864, "arlington": 17865, "scholarships": 17866, "emirates": 17867, "mums": 17868, "stella": 17869, "autonom": 17870, "feather": 17871, "gore": 17872, "nominees": 17873, "fragrance": 17874, "ÑĤ": 17875, "wong": 17876, "theastern": 17877, "gre": 17878, "zilla": 17879, "isi": 17880, "bumper": 17881, "goo": 17882, "dozens": 17883, "abduc": 17884, "âļªï¸ı": 17885, "oils": 17886, "donors": 17887, "silicon": 17888, "ipod": 17889, "fortnite": 17890, "ðŁĴ¨": 17891, "toro": 17892, "sparkling": 17893, "consciousness": 17894, "pala": 17895, "num": 17896, "mounted": 17897, "ffins": 17898, "thieves": 17899, "teammate": 17900, "prab": 17901, "omer": 17902, "tapes": 17903, "bod": 17904, "mitsu": 17905, "stew": 17906, "ere": 17907, "pbs": 17908, "tusc": 17909, "lowe": 17910, "rade": 17911, "parliamentary": 17912, "hm": 17913, "edgar": 17914, "ðŁijĩðŁijĩ": 17915, "toa": 17916, "agh": 17917, "honi": 17918, "slate": 17919, "geek": 17920, "apt": 17921, "hardt": 17922, "tap": 17923, "horizon": 17924, "growth": 17925, "makeover": 17926, "hil": 17927, "paperback": 17928, "idan": 17929, "rehabil": 17930, "giu": 17931, "possibilities": 17932, "lettu": 17933, "franco": 17934, "boss": 17935, "acher": 17936, "doesnt": 17937, "moe": 17938, "taker": 17939, "hussain": 17940, "mlk": 17941, "dil": 17942, "thia": 17943, "hama": 17944, "realised": 17945, "ravens": 17946, "curriculum": 17947, "mith": 17948, "knight": 17949, "tedx": 17950, "rv": 17951, "isaiah": 17952, "cumbria": 17953, "birthdays": 17954, "fing": 17955, "prez": 17956, "mubarak": 17957, "exquisite": 17958, "clearance": 17959, "yen": 17960, "pari": 17961, "evo": 17962, "ú": 17963, "modified": 17964, "applying": 17965, "implement": 17966, "discovering": 17967, "chapman": 17968, "indiegame": 17969, "disk": 17970, "crowdfunding": 17971, "machin": 17972, "livel": 17973, "styled": 17974, "âĿĮ": 17975, "making": 17976, "rehearsals": 17977, "nutriti": 17978, "subscription": 17979, "andro": 17980, "creators": 17981, "carries": 17982, "kylie": 17983, "camden": 17984, "apprentice": 17985, "taxpay": 17986, "cca": 17987, "tuesdaythoughts": 17988, "pissed": 17989, "erman": 17990, "detec": 17991, "freedom": 17992, "meri": 17993, "..!": 17994, "psalm": 17995, "sunlight": 17996, "perspec": 17997, "beings": 17998, "bookstore": 17999, "rockstar": 18000, "functions": 18001, "pence": 18002, "faves": 18003, "zn": 18004, "obamacare": 18005, "spill": 18006, "coventry": 18007, "pigeon": 18008, "pivo": 18009, "bait": 18010, "kolkata": 18011, "aval": 18012, "donor": 18013, "wah": 18014, "privileg": 18015, "traditions": 18016, "rajasthan": 18017, "teness": 18018, "portuguese": 18019, "ynes": 18020, "tackles": 18021, "defic": 18022, "torn": 18023, "polling": 18024, "thorne": 18025, "ina": 18026, "benedict": 18027, "barry": 18028, "calories": 18029, "verdict": 18030, "savethe": 18031, "norton": 18032, "office": 18033, "mainstream": 18034, "improves": 18035, "fron": 18036, "responding": 18037, "realtor": 18038, "scottish": 18039, "declar": 18040, "rl": 18041, "shiv": 18042, "supplier": 18043, "resting": 18044, "sweets": 18045, "qui": 18046, ".âĢ¦": 18047, "whitney": 18048, "startup": 18049, "thankyou": 18050, "teacher": 18051, "halls": 18052, "have": 18053, "handmade": 18054, "proving": 18055, "quartet": 18056, "rochester": 18057, "lian": 18058, "virtual": 18059, "mendes": 18060, "oficial": 18061, "midlands": 18062, "xbox": 18063, "measuring": 18064, "ovo": 18065, "accommodation": 18066, "brides": 18067, "collegiate": 18068, "intellectual": 18069, "incar": 18070, "niag": 18071, "ðŁį·": 18072, "sfw": 18073, "cocoa": 18074, "coats": 18075, "civilians": 18076, "presidency": 18077, "matrix": 18078, "sweetheart": 18079, "triathlon": 18080, "wagner": 18081, "radic": 18082, "planner": 18083, "theo": 18084, "execution": 18085, "kum": 18086, "thewalkingdead": 18087, "scar": 18088, "rotation": 18089, "blogging": 18090, "bomb": 18091, "reson": 18092, "bbles": 18093, "stare": 18094, "assisted": 18095, "edo": 18096, "branded": 18097, "warnings": 18098, "thorpe": 18099, "acknowle": 18100, "satisfied": 18101, "shores": 18102, "rid": 18103, "dora": 18104, "physically": 18105, "bigh": 18106, "approves": 18107, "hah": 18108, "rical": 18109, "versatile": 18110, "pretend": 18111, "lum": 18112, "abhi": 18113, "yee": 18114, "spit": 18115, "ãĢĮ": 18116, "djs": 18117, "ashtra": 18118, "jt": 18119, "venues": 18120, "grammys": 18121, "cyclo": 18122, "tracker": 18123, "overwatch": 18124, "replica": 18125, "elyn": 18126, "nrl": 18127, "lindsey": 18128, "homo": 18129, "balloons": 18130, "kitchen": 18131, "sis": 18132, "amos": 18133, "endeav": 18134, "ðŁĴ»": 18135, "arec": 18136, "thug": 18137, "hooked": 18138, "hrc": 18139, "newyork": 18140, "burgh": 18141, "americas": 18142, "patricia": 18143, "ugu": 18144, "apathy": 18145, "hast": 18146, "psychi": 18147, "cork": 18148, "petrol": 18149, "ðŁİ¬": 18150, "aku": 18151, "popping": 18152, "psychological": 18153, "aux": 18154, "gma": 18155, "cadillac": 18156, "waste": 18157, "authent": 18158, "bristol": 18159, "name": 18160, "queer": 18161, "tober": 18162, "jerry": 18163, "comin": 18164, "chant": 18165, "privileged": 18166, "opar": 18167, "loser": 18168, "text": 18169, "marker": 18170, "stries": 18171, "equally": 18172, "aki": 18173, "christmas": 18174, "gareth": 18175, "blew": 18176, "emma": 18177, "imagin": 18178, "seals": 18179, "cheat": 18180, "conditioning": 18181, "jana": 18182, "rens": 18183, "daries": 18184, "oasis": 18185, "discounts": 18186, "council": 18187, "ika": 18188, "shirley": 18189, "voucher": 18190, "alps": 18191, "wx": 18192, "qr": 18193, "drift": 18194, "attempting": 18195, "utc": 18196, "ت": 18197, "gonzalez": 18198, "mf": 18199, "joker": 18200, "parallel": 18201, "pare": 18202, "aspects": 18203, "procedu": 18204, "np": 18205, "ama": 18206, "raleigh": 18207, "brighten": 18208, "guire": 18209, "radiation": 18210, "crescent": 18211, "hob": 18212, "ille": 18213, "strand": 18214, "vore": 18215, "nard": 18216, "chest": 18217, "diwali": 18218, "avatar": 18219, "alder": 18220, "dling": 18221, "pathetic": 18222, "ðŁĴĺ": 18223, "spirit": 18224, "jorge": 18225, "filmmaking": 18226, "ðŁĻıðŁĻı": 18227, "challenger": 18228, "bj": 18229, "downtown": 18230, "html": 18231, "adequ": 18232, "twisted": 18233, "inely": 18234, "('": 18235, "wraps": 18236, "operational": 18237, "yne": 18238, "nus": 18239, "magnet": 18240, "marketplace": 18241, "healthier": 18242, "snapshot": 18243, "damon": 18244, "interven": 18245, "federer": 18246, "owls": 18247, "biscuits": 18248, "jp": 18249, "rodeo": 18250, "blueberry": 18251, "lection": 18252, "frontier": 18253, "summers": 18254, "reyes": 18255, "pedestrian": 18256, "gol": 18257, "caffe": 18258, "refurbi": 18259, "boulder": 18260, "meghan": 18261, "specialty": 18262, "lass": 18263, "ei": 18264, "suspects": 18265, "approx": 18266, "rrr": 18267, "rath": 18268, "stim": 18269, "crushed": 18270, "hed": 18271, "whun": 18272, "loaf": 18273, "crore": 18274, "rivera": 18275, "genetics": 18276, "sock": 18277, "wasted": 18278, "nypd": 18279, "answering": 18280, "dove": 18281, "bella": 18282, "olin": 18283, "dun": 18284, "fiji": 18285, "pretty": 18286, "sparkle": 18287, "yun": 18288, "jd": 18289, "europa": 18290, "lifts": 18291, "amber": 18292, "mur": 18293, "tek": 18294, "boyd": 18295, "royalty": 18296, "indo": 18297, "rib": 18298, "gotham": 18299, "tiest": 18300, "installing": 18301, "kemp": 18302, "thephoto": 18303, "cosmic": 18304, ")))": 18305, "wholesale": 18306, "loyment": 18307, "easy": 18308, "suing": 18309, "settled": 18310, "afp": 18311, "prover": 18312, "supportive": 18313, "rees": 18314, "neath": 18315, "deliber": 18316, "cé": 18317, "welcome": 18318, "picoftheday": 18319, "newborn": 18320, "patty": 18321, "suns": 18322, "siest": 18323, "flint": 18324, "differently": 18325, "spoilers": 18326, "trooper": 18327, "gins": 18328, "cory": 18329, "lookout": 18330, "equipped": 18331, "tape": 18332, "toby": 18333, "researcher": 18334, "ush": 18335, "keyes": 18336, "alma": 18337, "induction": 18338, "kw": 18339, "khar": 18340, "slick": 18341, "bride": 18342, "eur": 18343, "craving": 18344, "bookings": 18345, "ches": 18346, "trunk": 18347, "vernon": 18348, "spher": 18349, "crystals": 18350, "relatively": 18351, "pompe": 18352, "unions": 18353, "valley": 18354, "para": 18355, "want": 18356, "okc": 18357, "deaf": 18358, "sergio": 18359, "lennon": 18360, "shay": 18361, "cra": 18362, "vat": 18363, "hee": 18364, "twe": 18365, "liquid": 18366, "poly": 18367, "ðŁİģ": 18368, "bent": 18369, "bearing": 18370, "motorsport": 18371, "barbe": 18372, "testi": 18373, "hani": 18374, "financing": 18375, "astronaut": 18376, "watercolour": 18377, "rish": 18378, "comiccon": 18379, "gart": 18380, "wrong": 18381, "bern": 18382, "itan": 18383, "stepped": 18384, "filters": 18385, "clow": 18386, "mex": 18387, "demons": 18388, "allo": 18389, "expanded": 18390, "command": 18391, "eters": 18392, "goats": 18393, "siri": 18394, "yr": 18395, "pottery": 18396, "marion": 18397, "ile": 18398, "elan": 18399, "santo": 18400, "persona": 18401, "duke": 18402, "homeless": 18403, "lighted": 18404, "wheeler": 18405, "changer": 18406, "cabbage": 18407, "surreal": 18408, "hamburg": 18409, "smashed": 18410, "stran": 18411, "knot": 18412, "iart": 18413, "obi": 18414, "bedro": 18415, "dial": 18416, "thick": 18417, "bingo": 18418, "fus": 18419, "vacuum": 18420, "conve": 18421, "ative": 18422, "accuracy": 18423, "account": 18424, "refer": 18425, "riz": 18426, "spiderman": 18427, "bana": 18428, "rite": 18429, "ub": 18430, "abs": 18431, "medical": 18432, "link": 18433, "siem": 18434, ">>>>": 18435, "betra": 18436, "glowing": 18437, "reactions": 18438, "puppet": 18439, "spaghetti": 18440, "angs": 18441, "remedi": 18442, "prayfor": 18443, "royce": 18444, "charlotte": 18445, "£ï¸ı": 18446, "ghet": 18447, "affecting": 18448, "rode": 18449, "socialist": 18450, "moses": 18451, "azi": 18452, "oit": 18453, "reporters": 18454, "cdt": 18455, "aping": 18456, "snat": 18457, "minimal": 18458, "waist": 18459, "siege": 18460, ">>>>": 18461, "rig": 18462, "schmidt": 18463, "hare": 18464, "eca": 18465, "thorn": 18466, "hemp": 18467, "esthe": 18468, "clyde": 18469, "tha": 18470, "donut": 18471, "mohamed": 18472, "lingerie": 18473, "legg": 18474, "carpenter": 18475, "performers": 18476, "dea": 18477, "imagined": 18478, "curse": 18479, "lash": 18480, "ctr": 18481, "agua": 18482, "roar": 18483, "gri": 18484, "role": 18485, "jfk": 18486, "resurrec": 18487, "roosevelt": 18488, "marilyn": 18489, "smalle": 18490, "willis": 18491, "waited": 18492, "charities": 18493, "theres": 18494, "lik": 18495, "original": 18496, "cari": 18497, "cough": 18498, "cruci": 18499, "lagun": 18500, "contrast": 18501, "kou": 18502, "armour": 18503, "removing": 18504, "tent": 18505, "mazda": 18506, "brighter": 18507, "thief": 18508, "corner": 18509, "tequila": 18510, "buzzing": 18511, "albi": 18512, "pam": 18513, "azure": 18514, "discoun": 18515, "pixelart": 18516, "possibility": 18517, "hamont": 18518, "trades": 18519, "buda": 18520, "hive": 18521, "versy": 18522, "finch": 18523, "transpa": 18524, "emi": 18525, "terrifying": 18526, "inqui": 18527, "gba": 18528, "substitu": 18529, "collecti": 18530, "placing": 18531, "cindy": 18532, "kann": 18533, "patho": 18534, "diamond": 18535, "mourinho": 18536, "guinea": 18537, "anthropo": 18538, "airs": 18539, "pumps": 18540, "ìļ": 18541, "paso": 18542, "curling": 18543, "anita": 18544, "residency": 18545, "newh": 18546, "joon": 18547, "cigarette": 18548, "queue": 18549, "extrac": 18550, "games": 18551, "splen": 18552, "express": 18553, "publicly": 18554, "bonnie": 18555, "tribune": 18556, "baek": 18557, "reasonable": 18558, "cor": 18559, "timothy": 18560, "sheeran": 18561, "ı": 18562, "fdn": 18563, "sutton": 18564, "concentration": 18565, "caravan": 18566, "xavier": 18567, "alger": 18568, "cylin": 18569, "frederick": 18570, "nerve": 18571, "peak": 18572, "lettuce": 18573, "jail": 18574, "pregame": 18575, "kavan": 18576, "upgraded": 18577, "ecology": 18578, "squadron": 18579, "grapes": 18580, "goog": 18581, "pastry": 18582, "ðŁĹ£": 18583, "ãĥ¼ãĥ": 18584, "milano": 18585, "awaz": 18586, "presenter": 18587, "ðŁĮ¿": 18588, "herd": 18589, "kings": 18590, "template": 18591, "flour": 18592, "hv": 18593, "kley": 18594, "iya": 18595, "spec": 18596, "ater": 18597, "frankfurt": 18598, "coch": 18599, "texting": 18600, "deli": 18601, "communist": 18602, "regiment": 18603, "eleanor": 18604, "anticipated": 18605, "ðŁijĮðŁı»": 18606, "thephotohour": 18607, "rano": 18608, "surviving": 18609, "simulation": 18610, "dawson": 18611, "arin": 18612, "aqua": 18613, "mor": 18614, "âĢ¦.": 18615, "cino": 18616, "iraqi": 18617, "shaz": 18618, "dundee": 18619, "wes": 18620, "drau": 18621, "hannah": 18622, "snews": 18623, "occupation": 18624, "steen": 18625, "xm": 18626, "angles": 18627, "settings": 18628, "guru": 18629, "knox": 18630, "orca": 18631, "shaping": 18632, "went": 18633, "drilling": 18634, "zzie": 18635, "bri": 18636, "kissing": 18637, "find": 18638, "maine": 18639, "âŃIJï¸ıâŃIJï¸ı": 18640, "ðŁĮį": 18641, "larry": 18642, "busted": 18643, "tavern": 18644, "actively": 18645, "-\"": 18646, "replacing": 18647, "nod": 18648, "unlock": 18649, ".\"": 18650, "âŀ¤": 18651, "affiliate": 18652, "tow": 18653, "ln": 18654, "happynewyear": 18655, "dif": 18656, "jm": 18657, "greenwich": 18658, "controversy": 18659, "dawg": 18660, "condol": 18661, "savannah": 18662, "compensation": 18663, "touchdown": 18664, "teo": 18665, "ambitious": 18666, "embroi": 18667, "convicted": 18668, "iartg": 18669, "barack": 18670, "trance": 18671, "testimony": 18672, "audition": 18673, "thumb": 18674, "myths": 18675, "bex": 18676, "quez": 18677, "orchid": 18678, "deny": 18679, "entitled": 18680, "hood": 18681, "grant": 18682, "inbox": 18683, "bluejays": 18684, "rilla": 18685, "smallest": 18686, "burden": 18687, "infamous": 18688, "divided": 18689, "boundaries": 18690, "tter": 18691, "elt": 18692, "wyoming": 18693, "beverage": 18694, "mesm": 18695, "onews": 18696, "buddhist": 18697, "yana": 18698, "assad": 18699, "isms": 18700, "barrett": 18701, "predicted": 18702, "backto": 18703, "twit": 18704, "ethere": 18705, "captains": 18706, "escaped": 18707, "ayo": 18708, "lamborgh": 18709, "gardner": 18710, "laps": 18711, "kal": 18712, "advertisement": 18713, "insects": 18714, "napo": 18715, "amen": 18716, "acy": 18717, "rand": 18718, "gk": 18719, "teh": 18720, "kathle": 18721, "tridge": 18722, "pancake": 18723, "atro": 18724, "pyramid": 18725, "bula": 18726, "paralym": 18727, "gauge": 18728, "encies": 18729, "tomy": 18730, "biscuit": 18731, "butcher": 18732, "qualifier": 18733, "county": 18734, "kei": 18735, "pools": 18736, "darker": 18737, "shoulders": 18738, "ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸": 18739, "spre": 18740, "(\"": 18741, "writers": 18742, "gm": 18743, "ðŁİĵ": 18744, "knit": 18745, "huff": 18746, "mtb": 18747, "phillies": 18748, "ost": 18749, "denis": 18750, "gart": 18751, "licensed": 18752, "interface": 18753, "excel": 18754, "dwell": 18755, "fromthe": 18756, "cofficial": 18757, "azzi": 18758, "appearing": 18759, "forest": 18760, "nana": 18761, "keith": 18762, "manufacturers": 18763, "beckham": 18764, ")?": 18765, "ese": 18766, "colony": 18767, "delicate": 18768, "utter": 18769, "mcin": 18770, "transplant": 18771, "preferred": 18772, "pard": 18773, "arie": 18774, "hub": 18775, "pods": 18776, "perspectives": 18777, "pict": 18778, "delu": 18779, "apper": 18780, "bethan": 18781, "pmo": 18782, "criminals": 18783, "feminism": 18784, "shack": 18785, "circumstances": 18786, "fellas": 18787, "protesting": 18788, "wax": 18789, "suggested": 18790, "tator": 18791, "drew": 18792, "omni": 18793, "fake": 18794, "kathy": 18795, "reb": 18796, "deline": 18797, "berni": 18798, "misty": 18799, "ðŁij©": 18800, "erable": 18801, "breakthrough": 18802, "menswear": 18803, "millennials": 18804, "chanyeol": 18805, "laz": 18806, "insert": 18807, "replies": 18808, "phrase": 18809, "nx": 18810, "iheartawards": 18811, "audrey": 18812, "granite": 18813, "racec": 18814, "orie": 18815, "terra": 18816, "innovations": 18817, "brittany": 18818, "ateral": 18819, "pear": 18820, "biological": 18821, "shments": 18822, "institution": 18823, "msn": 18824, "frequency": 18825, "dman": 18826, "neglec": 18827, "tf": 18828, "stefan": 18829, "foxnews": 18830, "typo": 18831, "comms": 18832, "sequence": 18833, "carmen": 18834, "whites": 18835, "economist": 18836, "exeter": 18837, "seum": 18838, "resorts": 18839, "casually": 18840, "bunde": 18841, "divide": 18842, "ع": 18843, "gag": 18844, "creed": 18845, "retire": 18846, "caucus": 18847, "rapids": 18848, "wrestlemania": 18849, "tulsa": 18850, "sunderland": 18851, "fundament": 18852, "odi": 18853, "yamaha": 18854, "vary": 18855, "intrigu": 18856, "else": 18857, "beacon": 18858, "angie": 18859, "traded": 18860, "transm": 18861, "gents": 18862, "knitting": 18863, "galac": 18864, "ðĿĹ": 18865, "uto": 18866, "seaside": 18867, "holt": 18868, "rers": 18869, "fargo": 18870, "trainers": 18871, "monsoon": 18872, "bale": 18873, "sought": 18874, "maddie": 18875, "hw": 18876, "coli": 18877, "fran": 18878, "favs": 18879, "ðŁĴĶ": 18880, "intent": 18881, "rally": 18882, "sbs": 18883, "lemonade": 18884, "barackobama": 18885, "bread": 18886, "sticky": 18887, "explosive": 18888, "chelten": 18889, "tj": 18890, "assoc": 18891, "ramen": 18892, "homies": 18893, "vlog": 18894, "mister": 18895, "lord": 18896, "âĢįâĻĢï¸ı": 18897, "alyssa": 18898, "sketchbook": 18899, "rumble": 18900, "catch": 18901, "migrant": 18902, "discipline": 18903, "unlikely": 18904, "chronicles": 18905, "flora": 18906, "slams": 18907, "amid": 18908, "sboro": 18909, "coop": 18910, "jumps": 18911, "tranqu": 18912, "melis": 18913, "sofia": 18914, "enri": 18915, "gabe": 18916, "syri": 18917, "nicolas": 18918, "chai": 18919, "wv": 18920, "becky": 18921, "footy": 18922, "tao": 18923, "suppose": 18924, "ðŁĺįðŁĺįðŁĺįðŁĺį": 18925, "plush": 18926, "rish": 18927, "ðŁ¤ĵ": 18928, "kha": 18929, "saturdays": 18930, "accent": 18931, "hec": 18932, "limit": 18933, "carlton": 18934, "wired": 18935, "taylorswift": 18936, "ðŁĺij": 18937, "sql": 18938, "harro": 18939, "recipients": 18940, "gat": 18941, "gop": 18942, "thof": 18943, "amazed": 18944, "ghan": 18945, "ðŁıĨðŁıĨ": 18946, "porto": 18947, "clare": 18948, "distant": 18949, "nac": 18950, "ohio": 18951, "ðŁĻıðŁı¼": 18952, "mtn": 18953, "antibio": 18954, "dinosa": 18955, "mesa": 18956, "partial": 18957, "bv": 18958, "learnt": 18959, "lovato": 18960, "question": 18961, "extract": 18962, "gossip": 18963, "gibb": 18964, "niagara": 18965, "ðŁij¨": 18966, "displayed": 18967, "sooner": 18968, "stevie": 18969, "nuggets": 18970, "mln": 18971, "brom": 18972, "turb": 18973, "giveaways": 18974, "stupi": 18975, "blink": 18976, "cili": 18977, "convenient": 18978, "moh": 18979, "vive": 18980, "fric": 18981, "cause": 18982, "chamber": 18983, "cules": 18984, "nearest": 18985, "isse": 18986, "smallbiz": 18987, "tj": 18988, "canadians": 18989, "smarter": 18990, "brasil": 18991, "rare": 18992, "quette": 18993, "wha": 18994, "candle": 18995, "atomic": 18996, "ðŁijįðŁijį": 18997, "warrior": 18998, "relaxed": 18999, "strips": 19000, "neur": 19001, "kka": 19002, "rfc": 19003, "jensen": 19004, "recovering": 19005, "responses": 19006, "salam": 19007, "orthodox": 19008, "active": 19009, "ellers": 19010, "nit": 19011, "âŃIJ": 19012, "metropolitan": 19013, "centuries": 19014, "vida": 19015, "grading": 19016, "transparent": 19017, "simple": 19018, "dots": 19019, "superintendent": 19020, "elevator": 19021, "automated": 19022, "redskins": 19023, "imam": 19024, "summertime": 19025, "jonathan": 19026, "gearing": 19027, "michelle": 19028, "conflic": 19029, "mice": 19030, "tote": 19031, "publish": 19032, "pax": 19033, ")-": 19034, "nailed": 19035, "á´": 19036, "telescope": 19037, "serbia": 19038, "bab": 19039, "apeu": 19040, "stically": 19041, "senti": 19042, "rats": 19043, "isolated": 19044, "group": 19045, "hatred": 19046, "paranormal": 19047, "stanley": 19048, "alion": 19049, "safety": 19050, "ls": 19051, "र": 19052, "nexus": 19053, "alexandra": 19054, "masks": 19055, "++": 19056, "tron": 19057, "auk": 19058, "brotherhood": 19059, "browse": 19060, "mixes": 19061, "simone": 19062, "musk": 19063, "approve": 19064, "lola": 19065, "exp": 19066, "perth": 19067, "futuri": 19068, "unseen": 19069, "dm": 19070, "chelse": 19071, "scouting": 19072, "owe": 19073, "portsmouth": 19074, "kram": 19075, "mize": 19076, "dispen": 19077, "sup": 19078, "dlc": 19079, "advert": 19080, "teresa": 19081, "isle": 19082, "cycle": 19083, "metall": 19084, "shields": 19085, "mariners": 19086, "raz": 19087, "ingen": 19088, "fund": 19089, "ango": 19090, "jones": 19091, "oka": 19092, "madden": 19093, "broccoli": 19094, "dominic": 19095, "situations": 19096, "mero": 19097, "cricke": 19098, "punishment": 19099, "db": 19100, "shaking": 19101, "ðŁĺļ": 19102, "mq": 19103, "arians": 19104, "leh": 19105, "claw": 19106, "weds": 19107, "dure": 19108, "niel": 19109, "jelly": 19110, "gourmet": 19111, "traders": 19112, "levi": 19113, "wages": 19114, "knees": 19115, "wise": 19116, "heavenly": 19117, "avid": 19118, "melody": 19119, "zack": 19120, "bananas": 19121, "apprentice": 19122, "prop": 19123, "funny": 19124, "ode": 19125, "respected": 19126, "megan": 19127, "fewer": 19128, "drafted": 19129, "medit": 19130, "grape": 19131, "usarmy": 19132, "crusad": 19133, "vocali": 19134, "preparations": 19135, "nonsense": 19136, "usage": 19137, "thr": 19138, "roth": 19139, "wizards": 19140, "inside": 19141, "promotions": 19142, "mona": 19143, "redsox": 19144, "sig": 19145, "elegance": 19146, "chia": 19147, "universal": 19148, "ãĢį": 19149, "raja": 19150, "unga": 19151, "pollin": 19152, "filipino": 19153, "aka": 19154, "tsun": 19155, "ikon": 19156, "biking": 19157, "decorations": 19158, "zac": 19159, "cadets": 19160, "humour": 19161, "agm": 19162, "reppin": 19163, "vaccin": 19164, "elove": 19165, "uw": 19166, "diabe": 19167, "gallagher": 19168, "azer": 19169, "dol": 19170, "awhile": 19171, "prominent": 19172, "welsh": 19173, "tann": 19174, "')": 19175, "bien": 19176, "wag": 19177, "inal": 19178, "cwc": 19179, "wicket": 19180, "urst": 19181, "qanon": 19182, "xe": 19183, "outdoor": 19184, "dunn": 19185, "starr": 19186, "cology": 19187, "ricky": 19188, "uefa": 19189, "rebounds": 19190, "smusic": 19191, "infant": 19192, "ðŁĻĭ": 19193, "sop": 19194, "umber": 19195, "handing": 19196, "begin": 19197, "sorting": 19198, "hash": 19199, "spati": 19200, "rek": 19201, "budapest": 19202, "blackhawks": 19203, "delete": 19204, "rom": 19205, "candid": 19206, "authori": 19207, "debris": 19208, "specul": 19209, "intersection": 19210, "marriott": 19211, "imran": 19212, "ðŁĺģðŁĺģ": 19213, "cruises": 19214, "ramsey": 19215, "rafael": 19216, "awareness": 19217, "vascular": 19218, "beyoncé": 19219, "rug": 19220, "ðŁĺĮ": 19221, "festiv": 19222, "aram": 19223, "sable": 19224, "basil": 19225, "pill": 19226, "flooring": 19227, "unbeaten": 19228, "implications": 19229, "uf": 19230, "wound": 19231, "forge": 19232, "pointing": 19233, "pots": 19234, "popularity": 19235, "ðŁijıðŁı»": 19236, "manipul": 19237, "slots": 19238, "debates": 19239, "absence": 19240, "vermont": 19241, "neverforget": 19242, "wrist": 19243, "gloria": 19244, "rence": 19245, "husk": 19246, "melting": 19247, "ðŁİŁ": 19248, "braces": 19249, "timely": 19250, "transforming": 19251, "amps": 19252, "mak": 19253, "poe": 19254, "ahan": 19255, "generally": 19256, "ndp": 19257, "aleppo": 19258, "unicef": 19259, "profs": 19260, "nord": 19261, "mask": 19262, "jacksonville": 19263, "vv": 19264, "shells": 19265, "blooming": 19266, "operators": 19267, "charcoal": 19268, "neville": 19269, "magi": 19270, "chip": 19271, "sama": 19272, "iran": 19273, "reforms": 19274, "accumul": 19275, "rue": 19276, "æľ": 19277, "websites": 19278, "gaon": 19279, "devastating": 19280, "stos": 19281, "glacier": 19282, "rapp": 19283, "chipotle": 19284, "pra": 19285, "orous": 19286, "romney": 19287, "season": 19288, "decorative": 19289, "cisco": 19290, "ditch": 19291, "complain": 19292, "llo": 19293, "assume": 19294, "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ": 19295, "nels": 19296, "centric": 19297, "ftw": 19298, "carrots": 19299, "tata": 19300, "canter": 19301, "perience": 19302, "liers": 19303, "demos": 19304, "blunt": 19305, "operate": 19306, "reservations": 19307, "leah": 19308, "substance": 19309, "dison": 19310, "ante": 19311, "election": 19312, "vue": 19313, "square": 19314, "nonprofit": 19315, "caa": 19316, "fsu": 19317, "yam": 19318, "ãĤ¤": 19319, "vladi": 19320, "completes": 19321, "mari": 19322, "phillip": 19323, "neill": 19324, "eras": 19325, "kait": 19326, "mendo": 19327, "maharashtra": 19328, "gp": 19329, "dane": 19330, "providence": 19331, "therapeu": 19332, "juvenile": 19333, "memo": 19334, "incorpor": 19335, "aaaa": 19336, "seventeen": 19337, "teenager": 19338, "ã": 19339, "orns": 19340, "wide": 19341, "cuteness": 19342, "twd": 19343, "ffles": 19344, "bara": 19345, "comedy": 19346, "overtime": 19347, "yaz": 19348, "baron": 19349, "unemployment": 19350, "ðŁijĭ": 19351, "exterior": 19352, "dense": 19353, "centres": 19354, "matchup": 19355, "historymonth": 19356, "artificial": 19357, "quit": 19358, "esk": 19359, "warn": 19360, "critic": 19361, "jaf": 19362, "ðŁĵ²": 19363, "informative": 19364, "fuels": 19365, "recycle": 19366, "naming": 19367, "stripe": 19368, "solic": 19369, "molecular": 19370, "deepi": 19371, "convo": 19372, "ssel": 19373, "nae": 19374, "descent": 19375, "tiz": 19376, "accountability": 19377, "terry": 19378, "rito": 19379, "slay": 19380, "emo": 19381, "demol": 19382, "sensation": 19383, "cov": 19384, "tore": 19385, "roundtable": 19386, "yol": 19387, "excuses": 19388, "à¥į": 19389, "turquo": 19390, "hhhh": 19391, "podcasts": 19392, "celeb": 19393, "messi": 19394, "lio": 19395, "mann": 19396, "contributed": 19397, "uz": 19398, "generator": 19399, "elets": 19400, "veggie": 19401, "indul": 19402, "ensuring": 19403, "detroit": 19404, "punjab": 19405, "transpor": 19406, "instruction": 19407, "add": 19408, "porcel": 19409, "paneli": 19410, "circles": 19411, "persist": 19412, "clayton": 19413, "spn": 19414, "dogsoftwitter": 19415, "isnt": 19416, "spr": 19417, "retailers": 19418, "pw": 19419, "hungar": 19420, "elena": 19421, "monaster": 19422, "guatem": 19423, "jessie": 19424, "anz": 19425, "rashi": 19426, "flee": 19427, "carving": 19428, "faux": 19429, "lal": 19430, "henri": 19431, "djo": 19432, "dull": 19433, "sana": 19434, "lara": 19435, "globe": 19436, "crimson": 19437, "compass": 19438, "pause": 19439, "nab": 19440, "lionel": 19441, "baths": 19442, "ufo": 19443, "inventory": 19444, "singh": 19445, "satan": 19446, "ðŁĩ¸": 19447, "cements": 19448, "inform": 19449, "generated": 19450, "biden": 19451, "avg": 19452, "tasks": 19453, "deer": 19454, "sau": 19455, "jailed": 19456, "pastel": 19457, "scc": 19458, "nail": 19459, "steele": 19460, "peris": 19461, "lamborghini": 19462, "pursue": 19463, "margin": 19464, "uch": 19465, "bosch": 19466, "drain": 19467, "clara": 19468, "bom": 19469, "latino": 19470, "webster": 19471, "rosemary": 19472, "rha": 19473, "soun": 19474, "billionaire": 19475, "notch": 19476, "percentage": 19477, "conor": 19478, "'\"": 19479, "homes": 19480, "earthday": 19481, "hort": 19482, "biggest": 19483, "disin": 19484, "walton": 19485, "editors": 19486, "imma": 19487, "omar": 19488, "equivalent": 19489, "pharmaceu": 19490, "ahmed": 19491, "cameo": 19492, "hanni": 19493, "underrated": 19494, "gement": 19495, "microbi": 19496, "voo": 19497, "honorable": 19498, "obesity": 19499, "âļ¡ï¸ı": 19500, "limerick": 19501, "involvement": 19502, "stagram": 19503, "boulevard": 19504, "burg": 19505, "blackandwhite": 19506, "liberation": 19507, "five": 19508, "interim": 19509, "smm": 19510, "rivalry": 19511, "capabilities": 19512, "statements": 19513, "thumb": 19514, "ved": 19515, "swans": 19516, "barber": 19517, "eque": 19518, "serena": 19519, "helm": 19520, "noodle": 19521, "sampling": 19522, "nawaz": 19523, "single": 19524, "thunderstorms": 19525, "shon": 19526, "inev": 19527, "ë¯": 19528, "topp": 19529, "orchard": 19530, "bian": 19531, "ðŁĺĶ": 19532, "doorstep": 19533, "salvation": 19534, "marketing": 19535, "rons": 19536, "clemson": 19537, "ravi": 19538, "intake": 19539, "standwith": 19540, "sina": 19541, "haiku": 19542, "pley": 19543, "electoral": 19544, "philly": 19545, "lays": 19546, "electric": 19547, "capturing": 19548, "upp": 19549, "ergy": 19550, "believing": 19551, "cultures": 19552, "esday": 19553, "invasive": 19554, "eded": 19555, "speech": 19556, "endur": 19557, "vietnam": 19558, "boycott": 19559, "pede": 19560, "deliver": 19561, "ðŁĴĸðŁĴĸ": 19562, "merchant": 19563, "stir": 19564, "denies": 19565, "pockets": 19566, "oti": 19567, "cuddle": 19568, "roland": 19569, "mmed": 19570, "dened": 19571, "learners": 19572, "hoop": 19573, "sourcing": 19574, "hacked": 19575, "dim": 19576, "environments": 19577, "benson": 19578, "judicial": 19579, "worcester": 19580, "pearls": 19581, "governments": 19582, "arrivals": 19583, "corners": 19584, "tuning": 19585, "labour": 19586, "ym": 19587, "ordering": 19588, "lewi": 19589, "ife": 19590, "hygiene": 19591, "thoughtful": 19592, "indonesian": 19593, "campaigning": 19594, "principle": 19595, "assaul": 19596, "rubb": 19597, "atv": 19598, "willy": 19599, "entre": 19600, "ili": 19601, "phon": 19602, "duties": 19603, "âĻ¥âĻ¥": 19604, "snakes": 19605, "loop": 19606, "amar": 19607, "convertible": 19608, "bonding": 19609, "mentoring": 19610, "maxwell": 19611, "ethereum": 19612, "destroying": 19613, "axis": 19614, "cairo": 19615, "finnish": 19616, "shock": 19617, "ðŁĺIJ": 19618, "caleb": 19619, "coma": 19620, "pedal": 19621, "core": 19622, "continent": 19623, "elson": 19624, "tempo": 19625, "helsinki": 19626, "acp": 19627, "tackling": 19628, "stated": 19629, "bla": 19630, "doub": 19631, "smashing": 19632, "aja": 19633, "cameron": 19634, "disruption": 19635, "warmth": 19636, "beingsalmankhan": 19637, "bulletin": 19638, "ode": 19639, "syracuse": 19640, "aran": 19641, "mcgregor": 19642, "bulk": 19643, "anton": 19644, "confirmation": 19645, "spine": 19646, "imran": 19647, "instruc": 19648, "jacks": 19649, "chio": 19650, "palm": 19651, "stre": 19652, "embarrassing": 19653, "unt": 19654, "eliminate": 19655, "toss": 19656, "cise": 19657, "aws": 19658, "onists": 19659, "shinee": 19660, "jos": 19661, "hose": 19662, "lively": 19663, "opponents": 19664, "movements": 19665, "recognizing": 19666, "sandwiches": 19667, "shakes": 19668, "exercises": 19669, "seat": 19670, "profession": 19671, "merrychristmas": 19672, "lugg": 19673, "adoptdont": 19674, "marvin": 19675, "byrne": 19676, "unle": 19677, "het": 19678, "kuwait": 19679, "rahman": 19680, "aspect": 19681, "humbled": 19682, "genes": 19683, "fand": 19684, "longtime": 19685, ");": 19686, "campu": 19687, "angus": 19688, "ðŁijįðŁı¼": 19689, "quran": 19690, "sleeves": 19691, "slic": 19692, "¸ë": 19693, "twelve": 19694, "youre": 19695, "ike": 19696, "gogh": 19697, "bst": 19698, "dictionary": 19699, "reflecting": 19700, "toon": 19701, "yarn": 19702, "embed": 19703, "ðŁı´": 19704, "reserves": 19705, "flooded": 19706, "veriz": 19707, "dusk": 19708, "establish": 19709, "proli": 19710, "aud": 19711, "ritual": 19712, "orbit": 19713, "declaration": 19714, "recordings": 19715, "camo": 19716, "cassette": 19717, "goodluck": 19718, "cutter": 19719, "bop": 19720, "bho": 19721, "cheating": 19722, "pacific": 19723, "mares": 19724, "timer": 19725, "colt": 19726, "trous": 19727, "tomorrow": 19728, "hansen": 19729, "cie": 19730, "wang": 19731, "bani": 19732, "circular": 19733, "acute": 19734, "farmer": 19735, "coys": 19736, "pse": 19737, "irving": 19738, "wj": 19739, "hawkins": 19740, "bison": 19741, "urday": 19742, "cruising": 19743, "ote": 19744, "kath": 19745, "whistle": 19746, "yourselves": 19747, "antis": 19748, "slash": 19749, "thoroughly": 19750, "kesh": 19751, "serie": 19752, "exem": 19753, "enig": 19754, "guild": 19755, "shred": 19756, "hogan": 19757, "apo": 19758, "ä¸": 19759, "puzz": 19760, "netball": 19761, "aussi": 19762, "panorama": 19763, "wsj": 19764, "avis": 19765, "arming": 19766, "humph": 19767, "browser": 19768, "cries": 19769, "foggy": 19770, "matte": 19771, "ðŁĮ»": 19772, "iter": 19773, "tallest": 19774, "byron": 19775, "captiv": 19776, "jesu": 19777, "anyways": 19778, "flagship": 19779, "pton": 19780, "wey": 19781, "fayette": 19782, "financial": 19783, "foul": 19784, "solomon": 19785, "jennifer": 19786, "cucumber": 19787, "argue": 19788, "textile": 19789, "wrestler": 19790, "johnston": 19791, "pastor": 19792, "ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ": 19793, "cactus": 19794, "edible": 19795, "reserved": 19796, "richie": 19797, "metres": 19798, "ingredient": 19799, "hella": 19800, "unto": 19801, "chol": 19802, "celebs": 19803, "poets": 19804, "graham": 19805, "hayden": 19806, "coincidence": 19807, "baw": 19808, "communicate": 19809, "fletcher": 19810, "/-": 19811, "toledo": 19812, "ecuador": 19813, "counsel": 19814, "slaughter": 19815, "linear": 19816, "atp": 19817, "osu": 19818, "joel": 19819, "eved": 19820, "conquer": 19821, "rustic": 19822, "plicity": 19823, "recognise": 19824, "roommate": 19825, "cracked": 19826, "jasper": 19827, "pher": 19828, "ðŁĮº": 19829, "woven": 19830, "moist": 19831, "ffc": 19832, "steering": 19833, "nish": 19834, "standings": 19835, "frequent": 19836, "ardi": 19837, "hazel": 19838, "asmsg": 19839, "baum": 19840, "dart": 19841, "sidd": 19842, "nath": 19843, "chero": 19844, "cardboard": 19845, "css": 19846, "nsfw": 19847, "pair": 19848, "ðŁĺįðŁĺĺ": 19849, "occurred": 19850, "homelessness": 19851, "malone": 19852, "phe": 19853, "xia": 19854, "paddy": 19855, "declare": 19856, "theatre": 19857, "bf": 19858, "persian": 19859, "tad": 19860, "axe": 19861, "suspicious": 19862, "lamb": 19863, "mucho": 19864, "senior": 19865, "stas": 19866, "kite": 19867, "sting": 19868, "grad": 19869, "kaf": 19870, "watering": 19871, "د": 19872, "spiral": 19873, "thms": 19874, "educator": 19875, "jerome": 19876, "ofc": 19877, "clock": 19878, "sul": 19879, "pemb": 19880, ".........": 19881, "parkway": 19882, "deaux": 19883, "restrictions": 19884, "mons": 19885, "needle": 19886, "ej": 19887, "leagues": 19888, "watermelon": 19889, "aman": 19890, "plenary": 19891, "maxim": 19892, "wab": 19893, "comingsoon": 19894, "bryce": 19895, "vigil": 19896, "supermarket": 19897, "fortunate": 19898, "turquoise": 19899, "president": 19900, "liv": 19901, "interns": 19902, "feelin": 19903, "fixtures": 19904, "stunt": 19905, "staged": 19906, "premieres": 19907, "lok": 19908, "practiti": 19909, "shortage": 19910, "logne": 19911, "vec": 19912, "concor": 19913, "rocke": 19914, "lig": 19915, "composed": 19916, "synthetic": 19917, "dip": 19918, "camila": 19919, "chis": 19920, "jou": 19921, "susan": 19922, "eyebrows": 19923, "supplement": 19924, "satisfaction": 19925, "mohammad": 19926, "tibet": 19927, "houseof": 19928, "pun": 19929, "assam": 19930, "shadowhun": 19931, "psyched": 19932, "seduc": 19933, "mandatory": 19934, "herbert": 19935, "scallo": 19936, "streamers": 19937, "protocol": 19938, "blockbuster": 19939, "produces": 19940, "schnei": 19941, "laurel": 19942, "tribe": 19943, "timehop": 19944, "pla": 19945, "modelling": 19946, "tvtime": 19947, "mtvstars": 19948, "widow": 19949, "metric": 19950, "cham": 19951, "condo": 19952, "flowering": 19953, "alec": 19954, "dms": 19955, "intensity": 19956, "¨": 19957, "mccartney": 19958, "islamabad": 19959, "kb": 19960, "ffi": 19961, "phal": 19962, "analog": 19963, "fond": 19964, "hacks": 19965, "positivity": 19966, "treaty": 19967, "submarine": 19968, "connect": 19969, "selen": 19970, "categories": 19971, "cub": 19972, "organize": 19973, "sik": 19974, "quoteoftheday": 19975, "reminding": 19976, "amor": 19977, "locking": 19978, "ðŁijıðŁı¼": 19979, "compound": 19980, "ette": 19981, "bout": 19982, "recur": 19983, "ference": 19984, "mizz": 19985, "trend": 19986, "hipster": 19987, "fortress": 19988, "forthcoming": 19989, "prelimin": 19990, "odyssey": 19991, "angp": 19992, "delici": 19993, "evenings": 19994, "ðŁĶ¹": 19995, "iq": 19996, "dw": 19997, "dair": 19998, "kathryn": 19999, "christianity": 20000, "moonlight": 20001, "hab": 20002, "whoo": 20003, "fbf": 20004, "seth": 20005, "genuinely": 20006, "pax": 20007, "charity": 20008, "deployed": 20009, "bnb": 20010, "bucs": 20011, "judg": 20012, "conge": 20013, "plantation": 20014, "impress": 20015, "cara": 20016, "sclub": 20017, "scopy": 20018, "landers": 20019, "complaints": 20020, "bama": 20021, "rebuild": 20022, "xy": 20023, "realism": 20024, "shour": 20025, "lein": 20026, "bracelets": 20027, "mera": 20028, "assassin": 20029, "anchor": 20030, "ðŁijĮðŁı¼": 20031, "linen": 20032, "confron": 20033, "chronicle": 20034, "comment": 20035, "catalog": 20036, "illes": 20037, "gorge": 20038, "metry": 20039, "jungkook": 20040, "lovemy": 20041, "sentin": 20042, "seem": 20043, "fitness": 20044, "allied": 20045, "tsman": 20046, "digitaltransformation": 20047, "pran": 20048, "loft": 20049, "minton": 20050, "aldenrichards": 20051, "envel": 20052, "cherish": 20053, "certainty": 20054, "zzz": 20055, "rhino": 20056, "perkins": 20057, "enrich": 20058, "capetown": 20059, "ometer": 20060, "sections": 20061, "skeleton": 20062, "defenders": 20063, "ðŁĺĿ": 20064, "penc": 20065, "brit": 20066, "jah": 20067, "capitalism": 20068, "ðŁ¥ĩ": 20069, "bazaar": 20070, "reme": 20071, "ext": 20072, "kkk": 20073, "convert": 20074, "stormy": 20075, "bye": 20076, "karan": 20077, "chrysler": 20078, "ados": 20079, "pressed": 20080, "sync": 20081, "ationday": 20082, "danger": 20083, "badges": 20084, "refuses": 20085, "empowering": 20086, "lym": 20087, "exports": 20088, "adoptdontshop": 20089, "ðŁĩ¯": 20090, "thc": 20091, "awaited": 20092, "focuses": 20093, "fined": 20094, "oat": 20095, "hahahah": 20096, "âģ©": 20097, "nfamily": 20098, "fiona": 20099, "luckily": 20100, "thrilling": 20101, "typing": 20102, "outbreak": 20103, "dies": 20104, "heu": 20105, "crawl": 20106, "nesses": 20107, "oath": 20108, "scripts": 20109, "geeks": 20110, "ðŁIJĿ": 20111, "pb": 20112, "mathematics": 20113, "alis": 20114, "________________": 20115, "gymnastics": 20116, "activism": 20117, "recommendation": 20118, "gren": 20119, "wain": 20120, "courty": 20121, "napol": 20122, "cauli": 20123, "hornets": 20124, "gals": 20125, "jockey": 20126, "dirty": 20127, "atar": 20128, "enormous": 20129, "pest": 20130, "gregation": 20131, "anos": 20132, "iiii": 20133, "defends": 20134, "blackhistorymonth": 20135, "atx": 20136, "mbc": 20137, "luggage": 20138, "witch": 20139, "cob": 20140, "lasts": 20141, "cum": 20142, "ggg": 20143, "bathing": 20144, "nar": 20145, "cebu": 20146, "ðŁįĥ": 20147, "navigation": 20148, "mine": 20149, "rejo": 20150, "ðŁİĢ": 20151, "giftide": 20152, "reta": 20153, "useless": 20154, "pull": 20155, "deficit": 20156, "allu": 20157, "atime": 20158, "itv": 20159, "trillion": 20160, "pue": 20161, "acies": 20162, "procedure": 20163, "lori": 20164, "jenny": 20165, "cad": 20166, "ulously": 20167, "drac": 20168, "promotes": 20169, "ingthe": 20170, "canu": 20171, "woohoo": 20172, "naomi": 20173, "zardari": 20174, "tsu": 20175, "beir": 20176, "sdg": 20177, "lever": 20178, "weber": 20179, "abud": 20180, "lund": 20181, "crowded": 20182, "deployment": 20183, "terrain": 20184, "kenny": 20185, "hof": 20186, "witnessed": 20187, "loch": 20188, "jk": 20189, "bully": 20190, "wren": 20191, "poetry": 20192, "doff": 20193, "wwi": 20194, "mored": 20195, "dini": 20196, "culture": 20197, "prompt": 20198, "Â¥": 20199, "maurice": 20200, "topps": 20201, "rm": 20202, "correspon": 20203, "about": 20204, "jewels": 20205, "gibr": 20206, "eagle": 20207, "ðŁĺĺðŁĺĺðŁĺĺ": 20208, "lending": 20209, "souven": 20210, "çĶ": 20211, "contemporaryart": 20212, "establishment": 20213, "jong": 20214, "âĢ¦\"": 20215, "gator": 20216, "patriotic": 20217, "mccoy": 20218, "vape": 20219, "humane": 20220, "feliz": 20221, "coachella": 20222, "reposting": 20223, "steals": 20224, "fuller": 20225, "nering": 20226, "atra": 20227, "(-": 20228, "blake": 20229, "heather": 20230, "worms": 20231, "disciplinary": 20232, "redemption": 20233, "yard": 20234, "amin": 20235, "\"@_": 20236, "dnc": 20237, "tds": 20238, "kappa": 20239, "newark": 20240, "commits": 20241, "spears": 20242, "jams": 20243, "tand": 20244, "msnbc": 20245, "intermedi": 20246, "aimed": 20247, "atic": 20248, "teenth": 20249, "observation": 20250, "kashmir": 20251, "kavanaugh": 20252, "oul": 20253, "sanfrancisco": 20254, "reu": 20255, "belated": 20256, "chow": 20257, "password": 20258, "stills": 20259, "detained": 20260, "sari": 20261, "dayton": 20262, "darren": 20263, "italian": 20264, "arth": 20265, "amusic": 20266, "arbit": 20267, "wm": 20268, "vm": 20269, "hem": 20270, "doug": 20271, "myr": 20272, "asho": 20273, "prev": 20274, "vind": 20275, "brah": 20276, "stag": 20277, "ี": 20278, "previews": 20279, "guk": 20280, "containing": 20281, "leonardo": 20282, "saddle": 20283, "rushing": 20284, "stav": 20285, "longh": 20286, "gambling": 20287, "vegas": 20288, "reservation": 20289, "endale": 20290, "bala": 20291, "fla": 20292, "variant": 20293, "hedge": 20294, "bulgaria": 20295, "natali": 20296, "weaver": 20297, "solst": 20298, "encouraged": 20299, "apc": 20300, "asparag": 20301, "nest": 20302, "cyclists": 20303, "fel": 20304, "ìĬ¤": 20305, "overwhelming": 20306, "peyton": 20307, "jit": 20308, "apost": 20309, "mble": 20310, "bleeding": 20311, "neighbourhood": 20312, "avery": 20313, "expressions": 20314, "macdonald": 20315, "gigs": 20316, "monds": 20317, "illusion": 20318, "nct": 20319, "camero": 20320, "overhead": 20321, "myth": 20322, "oly": 20323, "vio": 20324, "etv": 20325, "laurie": 20326, "unveiling": 20327, "prior": 20328, "conn": 20329, "ironman": 20330, "diff": 20331, "dayin": 20332, "critici": 20333, "congo": 20334, "revision": 20335, "wale": 20336, "director": 20337, "pines": 20338, "blackpink": 20339, "garner": 20340, "curated": 20341, "manitoba": 20342, "hac": 20343, "commonly": 20344, "barton": 20345, "....#": 20346, "mortality": 20347, "livesmatter": 20348, "philosop": 20349, "shorter": 20350, "convince": 20351, "freak": 20352, "vendors": 20353, "insightful": 20354, "elly": 20355, "sensors": 20356, "eled": 20357, "sberg": 20358, "weightloss": 20359, "ukip": 20360, "spur": 20361, "private": 20362, "qua": 20363, "ssc": 20364, ",...": 20365, "supervisor": 20366, "adviser": 20367, "amazingly": 20368, "lesser": 20369, "ates": 20370, "mahon": 20371, "oooooo": 20372, "saras": 20373, "pmoindia": 20374, "waffle": 20375, "unders": 20376, "tolerance": 20377, "sculptures": 20378, "hersh": 20379, "knocking": 20380, "smoke": 20381, "catholic": 20382, "grim": 20383, "traveled": 20384, "flip": 20385, "geoff": 20386, "dinosaurs": 20387, "slept": 20388, "scarlet": 20389, "oki": 20390, "complaint": 20391, "obsc": 20392, "nami": 20393, "lag": 20394, "crossfit": 20395, "ufc": 20396, "mccain": 20397, "referee": 20398, "sadness": 20399, "penny": 20400, "lieu": 20401, "mode": 20402, "kier": 20403, "vols": 20404, "wis": 20405, "elon": 20406, "shea": 20407, "bao": 20408, "sonia": 20409, "claire": 20410, "emmanuel": 20411, "moisture": 20412, "digest": 20413, "viii": 20414, "teller": 20415, "chon": 20416, "accessory": 20417, "nightclub": 20418, "fossil": 20419, "awan": 20420, "husky": 20421, "aboriginal": 20422, "brandon": 20423, "fficient": 20424, "cougars": 20425, "sted": 20426, "admitted": 20427, "ignored": 20428, "contentmarketing": 20429, "agas": 20430, "vase": 20431, "executed": 20432, "negotiations": 20433, "shead": 20434, "nand": 20435, "tablets": 20436, "goth": 20437, "tsal": 20438, "dfw": 20439, "onep": 20440, "protector": 20441, "spho": 20442, "gazette": 20443, "andreas": 20444, "sser": 20445, "compilation": 20446, "hav": 20447, "containers": 20448, "broker": 20449, "socal": 20450, "porcelain": 20451, "hyuk": 20452, "airing": 20453, "ðŁĴ°": 20454, "publisher": 20455, "scenario": 20456, "spartans": 20457, "reviewing": 20458, "itudes": 20459, "edel": 20460, "pearson": 20461, "bash": 20462, "maui": 20463, "aad": 20464, "ðŁĮĬ": 20465, "liu": 20466, "ulate": 20467, "programmes": 20468, "favour": 20469, "webdesign": 20470, "realty": 20471, "motivational": 20472, "crosses": 20473, "'...": 20474, "busch": 20475, "adjustable": 20476, "arjun": 20477, "mistak": 20478, "dimension": 20479, "pistol": 20480, "weighs": 20481, "eny": 20482, "unveil": 20483, "indycar": 20484, "gordon": 20485, "fade": 20486, "franken": 20487, "qualities": 20488, "bett": 20489, "locate": 20490, "kerr": 20491, "spc": 20492, "confusion": 20493, "nee": 20494, "lucky": 20495, "bases": 20496, "depends": 20497, "firefighter": 20498, "ola": 20499, "ret": 20500, "maroon": 20501, "ðŁĶĬ": 20502, "wam": 20503, "defining": 20504, "wheat": 20505, "bil": 20506, "és": 20507, "bhai": 20508, "psych": 20509, "tau": 20510, "icans": 20511, "thik": 20512, "obile": 20513, "inspector": 20514, "ìĨĮë": 20515, "illon": 20516, "gos": 20517, "evangel": 20518, "fai": 20519, "sist": 20520, "vocation": 20521, "burge": 20522, "chistan": 20523, "renewed": 20524, "enthusiasm": 20525, "enting": 20526, "agri": 20527, "ikea": 20528, "msc": 20529, "aerospace": 20530, "sensiti": 20531, "memoir": 20532, "hospice": 20533, "cocaine": 20534, "derry": 20535, "mechanics": 20536, "Ħà¸": 20537, "tino": 20538, "reduces": 20539, "collectors": 20540, "injustice": 20541, "suppre": 20542, "vana": 20543, "abun": 20544, "napa": 20545, "susa": 20546, "oslo": 20547, "eff": 20548, "encore": 20549, "licence": 20550, "cheddar": 20551, "zal": 20552, "mount": 20553, "ðŁĴIJ": 20554, "threatens": 20555, "!!\"": 20556, "archie": 20557, "futsal": 20558, "scuba": 20559, "jos": 20560, "gnon": 20561, "sexi": 20562, "sofficial": 20563, "comparing": 20564, "dominant": 20565, "toftheday": 20566, "fait": 20567, "proposals": 20568, "gift": 20569, "yas": 20570, "cnc": 20571, "lr": 20572, "hab": 20573, "reservoir": 20574, "beliefs": 20575, "general": 20576, "marti": 20577, "td": 20578, "este": 20579, "ìł": 20580, "wil": 20581, "ðŁij¯": 20582, "ðŁĶ«": 20583, "spx": 20584, "etwork": 20585, "excerpt": 20586, "einstein": 20587, "hiro": 20588, "silhou": 20589, "teamed": 20590, "perception": 20591, "corridor": 20592, "mentalhealth": 20593, "hints": 20594, "benny": 20595, "inducted": 20596, "swx": 20597, "widesp": 20598, "speak": 20599, "cheryl": 20600, "drug": 20601, "ðŁĺķ": 20602, "hf": 20603, "asparagus": 20604, "mysteries": 20605, "fitzgerald": 20606, "offer": 20607, "therapist": 20608, "career": 20609, "damaging": 20610, "tsd": 20611, "peru": 20612, "weibo": 20613, "yay": 20614, "phoenix": 20615, "discre": 20616, "macbook": 20617, "barker": 20618, "stigma": 20619, "spread": 20620, "rockies": 20621, "kangar": 20622, "bridg": 20623, "pai": 20624, "bishop": 20625, "tailed": 20626, "capsule": 20627, "ðŁĴĵ": 20628, "geof": 20629, "royale": 20630, "shortlisted": 20631, "oste": 20632, "ashamed": 20633, "chapp": 20634, "keye": 20635, "cla": 20636, "screenshot": 20637, "austrian": 20638, "native": 20639, "enight": 20640, "juliet": 20641, "michele": 20642, "ðŁĮ´": 20643, "travelers": 20644, "pil": 20645, "footballer": 20646, "winchester": 20647, "ðŁĻĦ": 20648, "azerbai": 20649, "goldeng": 20650, "organisations": 20651, "interpretation": 20652, "predator": 20653, "oftheweek": 20654, "logan": 20655, "poké": 20656, "marie": 20657, "calla": 20658, "tnt": 20659, "cinde": 20660, "getic": 20661, "fitfam": 20662, "grav": 20663, "owens": 20664, "ðŁĮ±": 20665, "shootout": 20666, "salis": 20667, "commissions": 20668, "cohe": 20669, "ptic": 20670, "nixon": 20671, "hia": 20672, "ambition": 20673, "marine": 20674, "cruelty": 20675, "tk": 20676, "crude": 20677, "salty": 20678, "jima": 20679, "mongo": 20680, "irony": 20681, "onwards": 20682, "arrests": 20683, "strangers": 20684, "iger": 20685, "cyclist": 20686, "rag": 20687, "extends": 20688, "tradio": 20689, "bourg": 20690, "moi": 20691, "ella": 20692, "eable": 20693, "lexus": 20694, "aul": 20695, "dera": 20696, "historian": 20697, "morton": 20698, "tiff": 20699, "manner": 20700, "kot": 20701, "dk": 20702, "pointed": 20703, "marqu": 20704, "aan": 20705, "eney": 20706, "dublin": 20707, "onpoli": 20708, "emili": 20709, "secret": 20710, "flo": 20711, "âļ¡": 20712, "baj": 20713, "steep": 20714, "accompanied": 20715, "rumours": 20716, "devi": 20717, "purchasing": 20718, "fig": 20719, "pub": 20720, "schoo": 20721, "autonomous": 20722, "goalie": 20723, "xia": 20724, "automatically": 20725, "revers": 20726, "tero": 20727, "fuku": 20728, "titanic": 20729, "shook": 20730, "sandals": 20731, "seekers": 20732, "excav": 20733, "nordic": 20734, "bigolive": 20735, "bake": 20736, "ratt": 20737, "zak": 20738, "nep": 20739, "ðŁĺ¤": 20740, "candy": 20741, "billions": 20742, "bookworm": 20743, "ppet": 20744, "à³": 20745, "surfaces": 20746, "scars": 20747, "philip": 20748, "dogg": 20749, "cigars": 20750, "cote": 20751, "translated": 20752, "curator": 20753, "sindh": 20754, "hangover": 20755, "brewer": 20756, "ones": 20757, "elton": 20758, "ðŁĴªðŁı¼": 20759, "marcu": 20760, "elliot": 20761, "righte": 20762, "dioce": 20763, "russ": 20764, "railways": 20765, "grandson": 20766, "ascen": 20767, "apology": 20768, "await": 20769, "mobili": 20770, "respir": 20771, "partisan": 20772, "olivi": 20773, "strike": 20774, "yoo": 20775, "whitehouse": 20776, "expressed": 20777, "pups": 20778, "bedford": 20779, "cultur": 20780, "frogs": 20781, "flying": 20782, "cavali": 20783, "cds": 20784, "friger": 20785, "streetphotography": 20786, "resolve": 20787, "taliban": 20788, "kang": 20789, "crushing": 20790, "jum": 20791, "ðŁĺĴ": 20792, "williamson": 20793, "tang": 20794, "curly": 20795, "tman": 20796, "veteran": 20797, "faire": 20798, "artificialintelligence": 20799, "unanim": 20800, "pren": 20801, "backdrop": 20802, "frances": 20803, "occer": 20804, "dorothy": 20805, "working": 20806, "arthr": 20807, "converted": 20808, "daylight": 20809, "servant": 20810, "paddle": 20811, "complaining": 20812, "thirty": 20813, "nadal": 20814, "aku": 20815, "ibrahim": 20816, "addressed": 20817, "piss": 20818, "greenhouse": 20819, "battalion": 20820, "simulator": 20821, "outlets": 20822, "embroidery": 20823, "ðŁĵ±": 20824, "fiscal": 20825, "gerard": 20826, "sassy": 20827, "ðŁİīðŁİīðŁİī": 20828, "ventures": 20829, "merit": 20830, "publicity": 20831, "ðŁijĪ": 20832, "sophisticated": 20833, "ctu": 20834, "conventional": 20835, "condolences": 20836, "israel": 20837, "tradition": 20838, "aran": 20839, "tess": 20840, "glad": 20841, "ðŁĺĬðŁĺĬ": 20842, "correction": 20843, "geon": 20844, "amd": 20845, "orship": 20846, "beast": 20847, "chment": 20848, "ìŀ": 20849, "nico": 20850, "wknd": 20851, "wels": 20852, "cushion": 20853, "belie": 20854, "voc": 20855, "idiots": 20856, "underneath": 20857, "puma": 20858, "cornell": 20859, "enation": 20860, "lul": 20861, "swach": 20862, "abig": 20863, "urer": 20864, "mie": 20865, "formerly": 20866, "caf": 20867, "ernal": 20868, "chorus": 20869, "julius": 20870, "senator": 20871, "âľį": 20872, "whir": 20873, "salvador": 20874, "phd": 20875, "unified": 20876, "booster": 20877, "graphical": 20878, "wrec": 20879, "sonny": 20880, "miz": 20881, "derers": 20882, "sall": 20883, "vens": 20884, "tuscany": 20885, "wid": 20886, "yong": 20887, "kurds": 20888, "waz": 20889, "trolls": 20890, "macro": 20891, "caturday": 20892, "pressing": 20893, "sasha": 20894, "centennial": 20895, "gusts": 20896, "emc": 20897, "before": 20898, "denise": 20899, "cust": 20900, "ðŁĵ¢": 20901, "looo": 20902, "basel": 20903, "england": 20904, "yolo": 20905, "ardu": 20906, "manifesto": 20907, "doha": 20908, "ìľ": 20909, "knives": 20910, "bournemouth": 20911, "bibl": 20912, "barb": 20913, "alicia": 20914, "Ø©": 20915, "comer": 20916, "cyclone": 20917, "git": 20918, "anews": 20919, "characteri": 20920, "ventura": 20921, "intra": 20922, "sfgiants": 20923, "hut": 20924, "bea": 20925, "darwin": 20926, "eller": 20927, "alv": 20928, "reese": 20929, "bly": 20930, "karan": 20931, "conclusion": 20932, "manny": 20933, "flakes": 20934, "uniteblue": 20935, "nadu": 20936, "copp": 20937, "edges": 20938, "lancashire": 20939, "ials": 20940, "otta": 20941, "philippe": 20942, "lent": 20943, "chee": 20944, "mentors": 20945, "festival": 20946, "anism": 20947, "complimentary": 20948, "rj": 20949, "pug": 20950, "dine": 20951, "wei": 20952, "cliffs": 20953, "sarmy": 20954, "tiveness": 20955, "treasury": 20956, "iland": 20957, "aftermath": 20958, "rabbi": 20959, "oun": 20960, "bouquet": 20961, "heritage": 20962, "zion": 20963, "surrender": 20964, "shenan": 20965, "inks": 20966, "karl": 20967, "ghty": 20968, "policing": 20969, "examination": 20970, "cey": 20971, "persu": 20972, "measurement": 20973, "hydrogen": 20974, "luhan": 20975, "âłĢâłĢâłĢâłĢ": 20976, "wari": 20977, "оÐ": 20978, "jy": 20979, "fowler": 20980, "mish": 20981, "alfre": 20982, "âĺij": 20983, "bbnaija": 20984, "catalogue": 20985, "recognised": 20986, "saver": 20987, "huskies": 20988, "colin": 20989, "mundo": 20990, "siva": 20991, "png": 20992, "discounted": 20993, "manutd": 20994, "fresno": 20995, "devin": 20996, "preliminary": 20997, "trophies": 20998, "plastics": 20999, "dug": 21000, "procu": 21001, "indigo": 21002, "gard": 21003, "dylan": 21004, "pitches": 21005, "groundbreaking": 21006, "inson": 21007, "blac": 21008, "anthology": 21009, "fh": 21010, "explic": 21011, "rard": 21012, "admiral": 21013, "sochi": 21014, "lashes": 21015, "splendid": 21016, "envy": 21017, "adv": 21018, "sexy": 21019, "festivities": 21020, "sticking": 21021, "bib": 21022, "thrill": 21023, "opp": 21024, "ariel": 21025, "botanical": 21026, "endurance": 21027, "females": 21028, "bricks": 21029, "vatican": 21030, "blackpool": 21031, "bermu": 21032, "brough": 21033, "roller": 21034, "bid": 21035, "suede": 21036, "slovenia": 21037, "mming": 21038, "mlb": 21039, "medalist": 21040, "dians": 21041, "rehabilitation": 21042, "neon": 21043, "sgo": 21044, "lithu": 21045, "ramos": 21046, "zed": 21047, "pianist": 21048, "intensive": 21049, "broadband": 21050, "study": 21051, "petersburg": 21052, "luca": 21053, "ahhhh": 21054, "physician": 21055, "dillon": 21056, "telecom": 21057, "grief": 21058, "mun": 21059, "acro": 21060, "sided": 21061, "sly": 21062, "blows": 21063, "classiccars": 21064, "trium": 21065, "argy": 21066, "?:": 21067, "hri": 21068, "marshmal": 21069, "âĢĵ": 21070, "topping": 21071, "warsaw": 21072, "transc": 21073, "preservation": 21074, "bav": 21075, "refriger": 21076, "experiments": 21077, "äº": 21078, "glit": 21079, "sliga": 21080, "gage": 21081, "factor": 21082, "flavours": 21083, "brony": 21084, "spo": 21085, "cookbook": 21086, "carriage": 21087, "away": 21088, "nyfw": 21089, "onian": 21090, "wg": 21091, "simpsons": 21092, "rolex": 21093, "ðŁı¿": 21094, "crosby": 21095, "ãħ¤": 21096, "credi": 21097, "syndic": 21098, "pubs": 21099, "alife": 21100, "poorly": 21101, "maced": 21102, "ðŁĺŀ": 21103, "behindthe": 21104, "wenger": 21105, "nats": 21106, "ðŁİŁ": 21107, "rubbish": 21108, "procedures": 21109, "typhoon": 21110, "ophobia": 21111, "erdo": 21112, "fuel": 21113, "viera": 21114, "bumps": 21115, "millennium": 21116, "newzealand": 21117, "lectures": 21118, "iton": 21119, "milky": 21120, "responded": 21121, "ê°": 21122, "landscape": 21123, "..@": 21124, "bother": 21125, "âĸ¶": 21126, "zhang": 21127, "huawei": 21128, "tuition": 21129, "sworn": 21130, "inu": 21131, "yor": 21132, "paolo": 21133, "auditions": 21134, "abil": 21135, "malaysian": 21136, "hops": 21137, "feathers": 21138, "mple": 21139, "auts": 21140, "ão": 21141, "bounty": 21142, "iche": 21143, "ìĺ": 21144, "shq": 21145, "pinot": 21146, "gears": 21147, "disappear": 21148, "videogames": 21149, "tna": 21150, "alzheimer": 21151, "ðŁĮŀ": 21152, "aji": 21153, "underwear": 21154, "switching": 21155, "signage": 21156, "oscar": 21157, "econ": 21158, "drow": 21159, "clint": 21160, "plated": 21161, "gundy": 21162, "emblem": 21163, "hoes": 21164, "icist": 21165, "nelly": 21166, "junior": 21167, "roadshow": 21168, "minerals": 21169, "atle": 21170, "alexandria": 21171, "acclaimed": 21172, "vell": 21173, "shiva": 21174, "adhe": 21175, "enne": 21176, "amnesty": 21177, "hounds": 21178, "councillor": 21179, "ðŁĴ¦": 21180, "aesthe": 21181, "partnering": 21182, "influenced": 21183, "magno": 21184, "flare": 21185, "extinction": 21186, "civilian": 21187, "majesty": 21188, "vail": 21189, "lawmakers": 21190, "racks": 21191, "mcc": 21192, "orian": 21193, "spices": 21194, "errors": 21195, "mayer": 21196, "coca": 21197, "pai": 21198, "sooooo": 21199, "retiring": 21200, "bathro": 21201, "ðŁĻĮðŁĻĮ": 21202, "âĸª": 21203, "suf": 21204, "endorsement": 21205, "building": 21206, "brooch": 21207, "palla": 21208, "arvind": 21209, "agent": 21210, "karate": 21211, "rhi": 21212, "ctv": 21213, "taine": 21214, "umm": 21215, "bax": 21216, "reigns": 21217, "uniof": 21218, "enterprises": 21219, "adele": 21220, "flake": 21221, "attire": 21222, "bruce": 21223, "bahamas": 21224, "gravy": 21225, "sain": 21226, "cheek": 21227, "trivi": 21228, "lov": 21229, "een": 21230, "bblo": 21231, "ladygaga": 21232, "itta": 21233, ".\"-": 21234, "dustin": 21235, "observatory": 21236, "eighth": 21237, "bloomberg": 21238, "khs": 21239, "fcc": 21240, "gist": 21241, "commemorate": 21242, "veer": 21243, "sexuality": 21244, "edc": 21245, "nicole": 21246, "vacancy": 21247, "user": 21248, "sona": 21249, ":'(": 21250, "diploma": 21251, "tend": 21252, "upgrades": 21253, "ÅŁ": 21254, "jurassic": 21255, "cardiac": 21256, "drs": 21257, "widespread": 21258, "Ãł": 21259, "dailies": 21260, "vendor": 21261, "simplicity": 21262, "wider": 21263, "lenses": 21264, "supplements": 21265, "depos": 21266, "observed": 21267, "vines": 21268, "partially": 21269, "renewal": 21270, "collaborate": 21271, "alig": 21272, "finity": 21273, "phu": 21274, "zzy": 21275, "petit": 21276, "ðŁĵħ": 21277, "zin": 21278, "igu": 21279, "smack": 21280, "fallon": 21281, "ðŁĵ£": 21282, "backwards": 21283, "component": 21284, "oso": 21285, "compatible": 21286, "binding": 21287, "zurich": 21288, "thome": 21289, "wounds": 21290, "lyric": 21291, "freshmen": 21292, "sneaky": 21293, "fibro": 21294, "diet": 21295, "employer": 21296, "insect": 21297, "hated": 21298, "scher": 21299, "razor": 21300, "nsw": 21301, "booker": 21302, "californi": 21303, "avfc": 21304, "°": 21305, "pretending": 21306, "pepsi": 21307, "alis": 21308, "untitled": 21309, "kart": 21310, "grandparents": 21311, "ethe": 21312, "ock": 21313, "luxemb": 21314, "visuals": 21315, "smallbusiness": 21316, "abdullah": 21317, "minho": 21318, "subaru": 21319, "hra": 21320, "revealing": 21321, "heartbreaking": 21322, "clarity": 21323, "amg": 21324, "slr": 21325, "****": 21326, "âŀĸ": 21327, "record": 21328, "iciary": 21329, "minded": 21330, "yeh": 21331, "excessive": 21332, "knuck": 21333, "icecream": 21334, "truth": 21335, "evic": 21336, "tastic": 21337, "antarc": 21338, "rendering": 21339, ",,": 21340, "mitt": 21341, "lorenzo": 21342, "stpatrick": 21343, "boundary": 21344, "zig": 21345, "vocab": 21346, "osaka": 21347, "furn": 21348, "tun": 21349, "gul": 21350, "sounding": 21351, "blogger": 21352, "utterly": 21353, "gaf": 21354, "advancing": 21355, "lcd": 21356, "margin": 21357, "lifelong": 21358, "solstice": 21359, "shra": 21360, "waits": 21361, "plear": 21362, "breach": 21363, "enligh": 21364, "ader": 21365, "ittle": 21366, "cation": 21367, "hoon": 21368, "studied": 21369, "?????": 21370, "kash": 21371, "evangeli": 21372, "psl": 21373, "weights": 21374, "metals": 21375, "tyres": 21376, "turno": 21377, "wie": 21378, "carb": 21379, "gale": 21380, "seal": 21381, "sunite": 21382, "amic": 21383, "patterson": 21384, "án": 21385, "euph": 21386, "upstairs": 21387, "qualifiers": 21388, "khalifa": 21389, "applemusic": 21390, "ìĨĮëħ": 21391, "vaughan": 21392, "alter": 21393, "cruiser": 21394, "mua": 21395, "tana": 21396, "katrina": 21397, "idols": 21398, "spoiled": 21399, "secretly": 21400, "fibre": 21401, "partnered": 21402, "umes": 21403, "giov": 21404, "comet": 21405, "screenshotsaturday": 21406, "keller": 21407, "filtr": 21408, "fet": 21409, "conway": 21410, "peu": 21411, "badminton": 21412, "gid": 21413, "mound": 21414, "donkey": 21415, "buff": 21416, "leather": 21417, "largely": 21418, "broch": 21419, "intments": 21420, "amuse": 21421, "rk": 21422, "stove": 21423, "impacted": 21424, "cont": 21425, "cracks": 21426, "prisoner": 21427, "bari": 21428, "contractor": 21429, "orioles": 21430, "dominate": 21431, "polar": 21432, "amelia": 21433, "drc": 21434, "ðŁijĮðŁijĮ": 21435, "vist": 21436, "suarez": 21437, "injection": 21438, "blooms": 21439, "ðŁļ¨ðŁļ¨": 21440, "stiff": 21441, "paypal": 21442, "snowing": 21443, "thursdays": 21444, "goose": 21445, "wedge": 21446, "educated": 21447, "weakness": 21448, "decker": 21449, "abudha": 21450, "breezy": 21451, "ÛĮ": 21452, "hopeful": 21453, "obi": 21454, "raider": 21455, "gham": 21456, "deu": 21457, "seve": 21458, "partly": 21459, "fut": 21460, "infused": 21461, "merri": 21462, "thane": 21463, "sometime": 21464, "hue": 21465, "mein": 21466, "credit": 21467, "sliding": 21468, "rande": 21469, "cherry": 21470, "deadpool": 21471, "shol": 21472, "aram": 21473, "underwood": 21474, "skye": 21475, "disturbing": 21476, "mnt": 21477, "polished": 21478, "guardians": 21479, "hadn": 21480, "picasso": 21481, "arius": 21482, "akshay": 21483, "irri": 21484, "jh": 21485, "happen": 21486, "lakh": 21487, "dalton": 21488, "atthe": 21489, "swell": 21490, "marsha": 21491, "reh": 21492, "cours": 21493, "jkt": 21494, "topus": 21495, "service": 21496, "rink": 21497, "hackers": 21498, "donovan": 21499, "horo": 21500, "tcm": 21501, "mayhem": 21502, "chase": 21503, "devops": 21504, "kensing": 21505, "scup": 21506, "shere": 21507, "qualification": 21508, "clive": 21509, "tong": 21510, "nancy": 21511, "maris": 21512, "derdale": 21513, "berman": 21514, "cinderella": 21515, "jolly": 21516, "cic": 21517, "loot": 21518, "collectibles": 21519, "homicide": 21520, "gge": 21521, "epidemic": 21522, "suites": 21523, "muddy": 21524, "gimme": 21525, "erec": 21526, "-*": 21527, "talla": 21528, "lisle": 21529, "embroide": 21530, "ðŁĩ©ðŁĩª": 21531, "verizon": 21532, "vector": 21533, "beanie": 21534, "artisan": 21535, "gain": 21536, "flores": 21537, "vigil": 21538, "uso": 21539, "ðŁĻıðŁı½": 21540, "grinding": 21541, "gher": 21542, "airports": 21543, "responsive": 21544, "shaft": 21545, "cancel": 21546, "ceremonies": 21547, "eme": 21548, "atari": 21549, "brushes": 21550, "eager": 21551, "bohemi": 21552, "childrens": 21553, "yankee": 21554, "maa": 21555, "suspense": 21556, "moran": 21557, "macar": 21558, "sunflower": 21559, "crew": 21560, "void": 21561, "kear": 21562, "fashioned": 21563, "jennings": 21564, "sundayfunday": 21565, "submissions": 21566, "mead": 21567, "herman": 21568, "wai": 21569, "critically": 21570, "leum": 21571, "baekhyun": 21572, "forcing": 21573, "cobra": 21574, "ãģ®": 21575, "acquire": 21576, "alk": 21577, "geology": 21578, "primar": 21579, "importantly": 21580, "irez": 21581, "bundesliga": 21582, "curiosity": 21583, "sena": 21584, "strict": 21585, "consoli": 21586, "winters": 21587, "venom": 21588, "cheltenham": 21589, "ðŁįº": 21590, "cena": 21591, "tat": 21592, "bain": 21593, "glover": 21594, "undercover": 21595, "asses": 21596, "carn": 21597, "memorialday": 21598, "ameli": 21599, "irene": 21600, "chon": 21601, "synthesis": 21602, "speedy": 21603, "mitsubi": 21604, "slayer": 21605, "composite": 21606, "understands": 21607, "pew": 21608, "interrup": 21609, "henri": 21610, "morrow": 21611, "anom": 21612, "thofjuly": 21613, "glee": 21614, "three": 21615, "ðŁĺ®": 21616, "andhi": 21617, "chatt": 21618, "renewables": 21619, "yes": 21620, "transfers": 21621, "!!!!!!!!": 21622, "babu": 21623, "duter": 21624, "loops": 21625, "peers": 21626, "oilers": 21627, "paulo": 21628, "ication": 21629, "hmu": 21630, "wara": 21631, "mercer": 21632, "homeland": 21633, "fuji": 21634, "aley": 21635, "yearbook": 21636, "rem": 21637, "reen": 21638, "absur": 21639, "bois": 21640, "]:": 21641, "caesar": 21642, "shotgun": 21643, "kurdish": 21644, "oren": 21645, "rae": 21646, "ancies": 21647, "typic": 21648, "fh": 21649, "default": 21650, "replic": 21651, "luk": 21652, "transactions": 21653, "rys": 21654, "infantry": 21655, "ðŁį¾": 21656, "chow": 21657, "chickens": 21658, "bagh": 21659, "wyatt": 21660, "aye": 21661, "ggi": 21662, "brews": 21663, "editions": 21664, "mira": 21665, "commencement": 21666, "presu": 21667, "periscope": 21668, "ichi": 21669, "guatemala": 21670, "zambia": 21671, "paints": 21672, "witches": 21673, "wani": 21674, "undere": 21675, "croy": 21676, "vows": 21677, "usmc": 21678, "hearted": 21679, "theatres": 21680, "shuffle": 21681, "level": 21682, "multic": 21683, "squeeze": 21684, "fern": 21685, "appet": 21686, "postal": 21687, "malt": 21688, "onboard": 21689, "ldnt": 21690, "coo": 21691, "ssc": 21692, "kac": 21693, "ðŁĺĩ": 21694, "scrap": 21695, "marcos": 21696, "dealers": 21697, "annu": 21698, "miller": 21699, "cove": 21700, "ulary": 21701, "vladimir": 21702, "beef": 21703, "thur": 21704, "pickled": 21705, "sesame": 21706, "bengaluru": 21707, "mott": 21708, "kathleen": 21709, "hist": 21710, "notor": 21711, "drank": 21712, "duchess": 21713, "snowfall": 21714, "eff": 21715, "tiny": 21716, "jn": 21717, "syour": 21718, "specialists": 21719, "scotus": 21720, "baylor": 21721, "everest": 21722, "malibu": 21723, "prem": 21724, "harmful": 21725, "lali": 21726, "bates": 21727, "gye": 21728, "differenti": 21729, "andra": 21730, "geometry": 21731, "elover": 21732, "blackout": 21733, "====": 21734, "kota": 21735, "interact": 21736, "asian": 21737, "layo": 21738, "samurai": 21739, "fidel": 21740, "exhausted": 21741, "gladi": 21742, "pdt": 21743, "spheric": 21744, "antiqu": 21745, "guitar": 21746, "sturi": 21747, "hopper": 21748, "angle": 21749, "fills": 21750, "slap": 21751, "mith": 21752, "rodney": 21753, "ongi": 21754, "insom": 21755, "preventing": 21756, "cassidy": 21757, "apho": 21758, "oregon": 21759, "loin": 21760, "hammond": 21761, "contributing": 21762, "fn": 21763, "garri": 21764, "orion": 21765, "compelling": 21766, "escaping": 21767, "aiming": 21768, "plumb": 21769, "bistro": 21770, "beasts": 21771, "concerning": 21772, "boe": 21773, "dopp": 21774, "shoplocal": 21775, "stumbled": 21776, "âĤ¹": 21777, "nazis": 21778, "âĢįâĻĤï¸ı": 21779, "gesture": 21780, "warts": 21781, "usopen": 21782, "higgins": 21783, "charli": 21784, "hangs": 21785, "bombers": 21786, "°:": 21787, "feeds": 21788, "cch": 21789, "stil": 21790, "nicola": 21791, "ðŁĵº": 21792, "clamation": 21793, "tropic": 21794, "afro": 21795, "ouk": 21796, "expenses": 21797, "derrick": 21798, "aline": 21799, "faw": 21800, "regard": 21801, "imer": 21802, "satin": 21803, "thium": 21804, "ryder": 21805, "pearl": 21806, "tess": 21807, "mmmmm": 21808, "senses": 21809, "ðŁĩ¹": 21810, "positive": 21811, "exhaust": 21812, "occur": 21813, "norris": 21814, "lilly": 21815, "isles": 21816, "directing": 21817, "yofficial": 21818, "countless": 21819, "samar": 21820, "onstage": 21821, "flock": 21822, "mirrors": 21823, "archer": 21824, "moi": 21825, "kd": 21826, "viv": 21827, "inos": 21828, "sikh": 21829, "lei": 21830, "sensory": 21831, "brits": 21832, "knox": 21833, "chestnut": 21834, "opy": 21835, "coliseum": 21836, "zaf": 21837, "divin": 21838, "adapter": 21839, ":)))": 21840, "temple": 21841, "kun": 21842, "helmets": 21843, "tdf": 21844, "guide": 21845, "mold": 21846, "oids": 21847, "luther": 21848, "heis": 21849, "monastery": 21850, "spree": 21851, "klu": 21852, "britney": 21853, "jaguars": 21854, "greats": 21855, "ccc": 21856, "kyrie": 21857, "machinery": 21858, "cricket": 21859, "rero": 21860, "abo": 21861, "aspiring": 21862, "semifinals": 21863, "aless": 21864, "signatures": 21865, "vard": 21866, "meth": 21867, "herbal": 21868, "holden": 21869, "kingdom": 21870, "apor": 21871, "reggie": 21872, "oreo": 21873, "palestinians": 21874, "emmys": 21875, "sectional": 21876, "roi": 21877, "neymar": 21878, "quel": 21879, "cull": 21880, "lka": 21881, "hazel": 21882, "estimate": 21883, "ulties": 21884, "gow": 21885, "bea": 21886, "purchases": 21887, "belts": 21888, "protects": 21889, "mé": 21890, "guessing": 21891, "bbo": 21892, "claudia": 21893, "fracking": 21894, "jonny": 21895, "elk": 21896, "celtic": 21897, "almighty": 21898, "raje": 21899, "courtyard": 21900, "igi": 21901, "canes": 21902, "ðŁĴªðŁı»": 21903, "bankrup": 21904, "lethal": 21905, "âľĮï¸ı": 21906, "graphicdesign": 21907, "vader": 21908, "pencils": 21909, "roughly": 21910, "dante": 21911, "mfg": 21912, "constell": 21913, "camel": 21914, "jb": 21915, "blossoms": 21916, "ento": 21917, "balochistan": 21918, "cinemato": 21919, "illard": 21920, "jersey": 21921, "consent": 21922, "dented": 21923, "contempl": 21924, "scher": 21925, "holi": 21926, "lough": 21927, "stour": 21928, "ayo": 21929, "beginners": 21930, "curb": 21931, "vhs": 21932, "ajax": 21933, "duff": 21934, "aveng": 21935, "domest": 21936, "committing": 21937, "aired": 21938, "chap": 21939, "hedgehog": 21940, "disappointing": 21941, "freelance": 21942, "inland": 21943, "charms": 21944, "ðŁĺįâĿ¤ï¸ı": 21945, "aish": 21946, "mx": 21947, "buckle": 21948, "tidal": 21949, "permit": 21950, "boating": 21951, "racha": 21952, "kendrick": 21953, "bello": 21954, "bhi": 21955, "plea": 21956, "estimates": 21957, "lb": 21958, "apologies": 21959, "jaya": 21960, "bbl": 21961, "astoni": 21962, "interstate": 21963, "maintaining": 21964, "elbow": 21965, "mup": 21966, "epit": 21967, "ðŁĺ¡": 21968, "violations": 21969, "defend": 21970, "beh": 21971, "slc": 21972, "amir": 21973, "puri": 21974, "tium": 21975, "fifa": 21976, "blurry": 21977, "scrim": 21978, "ðŁĻıðŁı¾": 21979, "maple": 21980, "relatives": 21981, "âĺĿ": 21982, "choc": 21983, "connor": 21984, "⾨⾨": 21985, "whisp": 21986, "listings": 21987, "maze": 21988, "thanking": 21989, "ridd": 21990, "grassroots": 21991, "shifting": 21992, "desperately": 21993, "gorilla": 21994, "deni": 21995, "jules": 21996, "strath": 21997, "gley": 21998, "jain": 21999, "buick": 22000, "tanner": 22001, "ðŁĴĿ": 22002, "gae": 22003, "prim": 22004, "itors": 22005, "nano": 22006, "separation": 22007, "armenia": 22008, "bordeaux": 22009, "ðŁħ": 22010, "pjnet": 22011, "burial": 22012, "ebon": 22013, "gloss": 22014, "renew": 22015, "grier": 22016, "speeds": 22017, "comicbooks": 22018, "symboli": 22019, "purposes": 22020, "ãħłãħł": 22021, "spatial": 22022, "notable": 22023, "cion": 22024, "nps": 22025, "hoffman": 22026, "norman": 22027, "rtg": 22028, "dusty": 22029, "situated": 22030, "tran": 22031, "kfc": 22032, "emen": 22033, "nickel": 22034, "hastings": 22035, "settling": 22036, "grit": 22037, "lena": 22038, "waw": 22039, "arts": 22040, "gum": 22041, "caregi": 22042, "lewis": 22043, "sapphire": 22044, "remember": 22045, "embedded": 22046, "tlc": 22047, "blat": 22048, "sergeant": 22049, "elsa": 22050, "bootcamp": 22051, "bowman": 22052, "photographic": 22053, "pillars": 22054, "directioners": 22055, "classified": 22056, "nois": 22057, "veer": 22058, "barrels": 22059, "whoop": 22060, "ðŁĺ±ðŁĺ±": 22061, "female": 22062, "petroleum": 22063, "media": 22064, "efc": 22065, "pokémon": 22066, "à¤ķ": 22067, "enthusiastic": 22068, "varun": 22069, "profiles": 22070, "pediatric": 22071, "accidents": 22072, "conrad": 22073, "jang": 22074, "jojo": 22075, "acor": 22076, "observer": 22077, "lf": 22078, "livestock": 22079, "forgi": 22080, "fos": 22081, "elm": 22082, "anand": 22083, "goe": 22084, "cere": 22085, "avoiding": 22086, "grit": 22087, "oman": 22088, "thankfully": 22089, "scattered": 22090, "nicky": 22091, "cylinder": 22092, "cheesy": 22093, "diver": 22094, "mahesh": 22095, "caves": 22096, "earliest": 22097, "quinte": 22098, "subjects": 22099, "bend": 22100, "gulf": 22101, "vocalist": 22102, "glue": 22103, "patches": 22104, "unstopp": 22105, "snyder": 22106, "demonstrating": 22107, "pio": 22108, "horns": 22109, "wickets": 22110, "andthe": 22111, "rama": 22112, "yoon": 22113, "straight": 22114, "bedtime": 22115, "orang": 22116, "bullets": 22117, "saurus": 22118, "miners": 22119, "incidents": 22120, "!...": 22121, "ðŁİ¸": 22122, "agers": 22123, "handles": 22124, "states": 22125, "inity": 22126, "dons": 22127, "incredible": 22128, "eminem": 22129, "aviv": 22130, "rudy": 22131, "mozart": 22132, "folklore": 22133, "appliances": 22134, "mtl": 22135, "frey": 22136, "dias": 22137, "hua": 22138, "pageant": 22139, "strive": 22140, "imprison": 22141, "bullish": 22142, "rana": 22143, "alerts": 22144, "bbmas": 22145, "hyper": 22146, "derbyshire": 22147, "recre": 22148, "redd": 22149, "deborah": 22150, "cosmos": 22151, "lawson": 22152, "melanie": 22153, "psycho": 22154, "hoor": 22155, "doodles": 22156, "sniper": 22157, "shady": 22158, "mantle": 22159, "canadian": 22160, "newyear": 22161, "interactions": 22162, "separated": 22163, "cords": 22164, "spirituality": 22165, "apu": 22166, "ito": 22167, "pct": 22168, "pelosi": 22169, "rebellion": 22170, "seiz": 22171, "worcester": 22172, "sectors": 22173, "uli": 22174, "santa": 22175, "е": 22176, "ðŁĩªðŁĩ¸": 22177, "biased": 22178, "classical": 22179, "gamma": 22180, "deeplear": 22181, "emerge": 22182, "backer": 22183, "surance": 22184, "handcrafted": 22185, "ðŁİ¥": 22186, "francis": 22187, "millan": 22188, "ici": 22189, "crown": 22190, "wow": 22191, "striped": 22192, "unfair": 22193, "relaxation": 22194, "³ï¸ı": 22195, "embracing": 22196, "shealth": 22197, "paleo": 22198, "martini": 22199, "distillery": 22200, "wrink": 22201, "ork": 22202, "nath": 22203, "hayley": 22204, "courthouse": 22205, "siber": 22206, "sadi": 22207, "quietly": 22208, "melt": 22209, "msm": 22210, "meh": 22211, "smartphones": 22212, "relent": 22213, "pping": 22214, "warwick": 22215, "cologne": 22216, "glia": 22217, "cotton": 22218, "prog": 22219, "lone": 22220, "ipsw": 22221, "starters": 22222, "expands": 22223, "ump": 22224, "sued": 22225, "skipper": 22226, "infections": 22227, "ingle": 22228, "á": 22229, "clerk": 22230, "demonstrate": 22231, "acar": 22232, "ðŁĺĤðŁĺĤðŁĺĤ": 22233, "tibet": 22234, "buns": 22235, "alom": 22236, "demolition": 22237, "ssia": 22238, "gst": 22239, "[]": 22240, "soar": 22241, "âĺĢ": 22242, "ðŁĺª": 22243, "ðŁĵĬ": 22244, "deepest": 22245, "beyond": 22246, "aret": 22247, "attends": 22248, "activated": 22249, "dimit": 22250, "âļªï¸ı": 22251, "highlighted": 22252, "magazines": 22253, "rumor": 22254, "azza": 22255, "stephens": 22256, "dolph": 22257, "shockey": 22258, "mats": 22259, "weav": 22260, "melan": 22261, "servers": 22262, "traum": 22263, "kush": 22264, "æĹ": 22265, "babys": 22266, "paz": 22267, "aal": 22268, "lause": 22269, "breakers": 22270, "canterbury": 22271, "ulture": 22272, "miri": 22273, "euros": 22274, "taneous": 22275, "impressions": 22276, "dutch": 22277, "ild": 22278, "ghi": 22279, "purdue": 22280, "adequate": 22281, "lp": 22282, "syner": 22283, "angler": 22284, "durable": 22285, "galore": 22286, "rown": 22287, "mgmt": 22288, "ðŁĵĮ": 22289, "lucia": 22290, "âĺijï¸ı": 22291, "zayn": 22292, "borrow": 22293, ".(": 22294, "northumber": 22295, "crush": 22296, "enga": 22297, "sush": 22298, "extravag": 22299, "tout": 22300, "mahal": 22301, "alistic": 22302, "thermo": 22303, "galleries": 22304, "esse": 22305, "chibi": 22306, "attractions": 22307, "lexington": 22308, "legislature": 22309, "documented": 22310, "residen": 22311, "brownies": 22312, "wf": 22313, "stool": 22314, "planets": 22315, "shoppers": 22316, "conductor": 22317, "msp": 22318, "tricky": 22319, "fruity": 22320, "endra": 22321, "feelthe": 22322, "whipped": 22323, "hairstyle": 22324, "refer": 22325, "ook": 22326, "octopus": 22327, "audiences": 22328, "kumar": 22329, "afterno": 22330, "optim": 22331, "cfl": 22332, "nip": 22333, "geni": 22334, "alphabet": 22335, "annab": 22336, "lamin": 22337, "accepts": 22338, "lng": 22339, "ðŁĺ«": 22340, "tine": 22341, "acom": 22342, "cheerleaders": 22343, "tk": 22344, "gron": 22345, "vg": 22346, "kung": 22347, "jax": 22348, "dhabi": 22349, "rss": 22350, "mackenzie": 22351, "beirut": 22352, "cleanup": 22353, "gypsy": 22354, "stell": 22355, "burger": 22356, "hurricanes": 22357, "education": 22358, "stina": 22359, "âĻ¡âĻ¡": 22360, "unfortunate": 22361, "jeremi": 22362, "badger": 22363, "aters": 22364, ":âĢ¦": 22365, "terra": 22366, "sublime": 22367, "stud": 22368, "ymca": 22369, "mru": 22370, "duterte": 22371, "brennan": 22372, "bulb": 22373, "melo": 22374, "ylon": 22375, "hacker": 22376, "cred": 22377, "gud": 22378, "asan": 22379, "padilla": 22380, "embroidered": 22381, "vietnamese": 22382, "pioneers": 22383, "projection": 22384, "reboot": 22385, "idc": 22386, "aney": 22387, "primer": 22388, "suffers": 22389, "winding": 22390, "pon": 22391, "stoday": 22392, "morn": 22393, "uch": 22394, "allin": 22395, "adidas": 22396, "elizabeth": 22397, "tuck": 22398, "ography": 22399, "ðŁļĢ": 22400, "beg": 22401, "osborne": 22402, "ghetto": 22403, "rh": 22404, "cnn": 22405, "irma": 22406, "makin": 22407, "cables": 22408, "murders": 22409, "ocks": 22410, "insta": 22411, "alas": 22412, "sik": 22413, "cuff": 22414, "lare": 22415, "foodies": 22416, "ovic": 22417, "atom": 22418, "geometric": 22419, "empathy": 22420, "ี": 22421, "centenary": 22422, "newspapers": 22423, "administrative": 22424, "ðŁİĬ": 22425, "stive": 22426, "contractors": 22427, "lett": 22428, "tasmania": 22429, "awesomeness": 22430, "density": 22431, "veen": 22432, "princeton": 22433, "frequently": 22434, "reject": 22435, "ghi": 22436, "modular": 22437, "ceramics": 22438, "shag": 22439, "kiwi": 22440, "canvas": 22441, "sweatshirt": 22442, "anj": 22443, "timm": 22444, "napoli": 22445, "iler": 22446, "appeals": 22447, "hamilton": 22448, "mayo": 22449, "weave": 22450, "arranged": 22451, "wharf": 22452, "occupy": 22453, "bvb": 22454, "asaki": 22455, "otter": 22456, "norm": 22457, "vies": 22458, "detox": 22459, "tional": 22460, "derek": 22461, "idad": 22462, "admissions": 22463, "constituency": 22464, "upper": 22465, "woot": 22466, "alloy": 22467, "seve": 22468, "lub": 22469, "uncomfortable": 22470, "edwin": 22471, "abre": 22472, "dwight": 22473, "arche": 22474, "virtually": 22475, "spol": 22476, "prie": 22477, "aii": 22478, "err": 22479, "switch": 22480, "barack": 22481, "seok": 22482, "coul": 22483, "wnt": 22484, "poul": 22485, "olive": 22486, "caffeine": 22487, "cardiff": 22488, "notorious": 22489, "demp": 22490, "excess": 22491, "barr": 22492, "tford": 22493, "ajay": 22494, "bumped": 22495, "mythology": 22496, "shelley": 22497, "falcon": 22498, "shakespeare": 22499, "mustangs": 22500, "noted": 22501, "bone": 22502, "civilization": 22503, "syd": 22504, "parsons": 22505, "unofficial": 22506, "hyped": 22507, "spends": 22508, "opposed": 22509, "vings": 22510, "spacex": 22511, "notification": 22512, "deciding": 22513, "biotech": 22514, "outsi": 22515, "salah": 22516, "!.": 22517, "fed": 22518, "ssy": 22519, "cms": 22520, "badgers": 22521, "cro": 22522, "elaine": 22523, "nba": 22524, "dyour": 22525, "nant": 22526, "honeymoon": 22527, "climbed": 22528, "conomy": 22529, "atha": 22530, "mell": 22531, "nebula": 22532, "naturephotography": 22533, "julie": 22534, "bmx": 22535, "invested": 22536, "mono": 22537, "lieutenant": 22538, "watkins": 22539, "technician": 22540, "ose": 22541, "kae": 22542, "ìĽ": 22543, "mcqueen": 22544, "preach": 22545, "traveller": 22546, "flexibility": 22547, "zebra": 22548, "retailer": 22549, "pant": 22550, "bender": 22551, "brandt": 22552, "squid": 22553, "warrant": 22554, "verified": 22555, "cass": 22556, "piercing": 22557, "honours": 22558, "tying": 22559, "morris": 22560, "kissed": 22561, "oprah": 22562, "panoramic": 22563, "mei": 22564, "splatoon": 22565, "wichita": 22566, "arias": 22567, "galli": 22568, "indyref": 22569, "goodtimes": 22570, "atheist": 22571, "confession": 22572, "owski": 22573, "repping": 22574, "additions": 22575, "mechanism": 22576, "zim": 22577, "jans": 22578, "suf": 22579, "chopped": 22580, "beginnings": 22581, "vitamins": 22582, "ãħ¤ãħ¤": 22583, "orth": 22584, "poles": 22585, "rub": 22586, "antarctica": 22587, "indiefilm": 22588, "webcam": 22589, "ketch": 22590, "brett": 22591, "clement": 22592, "heron": 22593, "defeating": 22594, "hydro": 22595, "bucket": 22596, "wandering": 22597, "sidney": 22598, "futureof": 22599, "binge": 22600, "onies": 22601, "knockout": 22602, "administrator": 22603, "synthe": 22604, "lent": 22605, "jani": 22606, "barley": 22607, "premierleague": 22608, "nerds": 22609, "crm": 22610, "bras": 22611, "botany": 22612, "evolved": 22613, "rotter": 22614, "rowed": 22615, "tumor": 22616, "wealthy": 22617, "ÂŃ": 22618, "monarch": 22619, "lished": 22620, "dahl": 22621, "ðŁİĥ": 22622, "buch": 22623, "kenyan": 22624, "ا": 22625, "redness": 22626, "assembled": 22627, "semit": 22628, "hudder": 22629, "shrop": 22630, "rani": 22631, "learning": 22632, "mory": 22633, "itia": 22634, "geographic": 22635, "worldof": 22636, "fb": 22637, "phosp": 22638, "boogie": 22639, "amped": 22640, "?...": 22641, "chew": 22642, "dwarf": 22643, "arus": 22644, "ssen": 22645, "rusty": 22646, "recruits": 22647, "hk": 22648, "garde": 22649, "applause": 22650, "volumes": 22651, "involves": 22652, "tac": 22653, "handbag": 22654, "translate": 22655, "ffel": 22656, "seym": 22657, "aquatic": 22658, "transfer": 22659, "zodi": 22660, "andr": 22661, "academia": 22662, "crater": 22663, "tez": 22664, "arse": 22665, "adapt": 22666, "coloni": 22667, "snowman": 22668, "mali": 22669, "hangin": 22670, "dischar": 22671, "oysters": 22672, "phoe": 22673, "colonel": 22674, "wba": 22675, "hispanic": 22676, "thriving": 22677, "shy": 22678, "agles": 22679, "salesforce": 22680, "creme": 22681, "soles": 22682, "lafayette": 22683, "âī": 22684, "teria": 22685, "acha": 22686, "sperson": 22687, "gogo": 22688, "carly": 22689, "theore": 22690, "amore": 22691, "vox": 22692, "aft": 22693, "ãĤ¹": 22694, "staple": 22695, "muffin": 22696, "diagram": 22697, "inox": 22698, "sustained": 22699, "avent": 22700, "meta": 22701, "arbitr": 22702, "decay": 22703, "adole": 22704, "н": 22705, "ecol": 22706, "pho": 22707, "nk": 22708, "ocu": 22709, "granny": 22710, "ça": 22711, "luxembour": 22712, "stadt": 22713, "alberto": 22714, "levit": 22715, "amas": 22716, "dx": 22717, "orphan": 22718, "cobb": 22719, "asc": 22720, "logy": 22721, "immense": 22722, "chants": 22723, "offline": 22724, "pent": 22725, "brex": 22726, "winger": 22727, "plane": 22728, "iel": 22729, "nichols": 22730, "cathy": 22731, "naruto": 22732, "lowed": 22733, "///": 22734, "ignorance": 22735, "catastro": 22736, "youts": 22737, "schen": 22738, "build": 22739, "hazi": 22740, "sine": 22741, "criticalrole": 22742, "dug": 22743, "detect": 22744, "logs": 22745, "enamel": 22746, "stpatricksday": 22747, "eddie": 22748, "copa": 22749, "cigarettes": 22750, "hoff": 22751, "kaya": 22752, "lagoon": 22753, "rapha": 22754, "airborne": 22755, "choose": 22756, "puertor": 22757, "kev": 22758, "guiding": 22759, "frosty": 22760, "borough": 22761, "mira": 22762, "ðŁİĬ": 22763, "cadet": 22764, "anush": 22765, "yogi": 22766, "eger": 22767, "fling": 22768, "slope": 22769, "ninth": 22770, "weston": 22771, "footwear": 22772, "fn": 22773, "mayweather": 22774, "aam": 22775, "plain": 22776, "staircase": 22777, "witnesses": 22778, "workouts": 22779, "robust": 22780, "dexter": 22781, "cohort": 22782, "ðŁļĹ": 22783, "spell": 22784, "haze": 22785, "oom": 22786, "organising": 22787, "wildfire": 22788, "contacts": 22789, "avon": 22790, "mino": 22791, "updating": 22792, "ðŁį»": 22793, "lithium": 22794, "ingual": 22795, "kis": 22796, "auga": 22797, "locom": 22798, "deduc": 22799, "uda": 22800, "thak": 22801, "boyle": 22802, "mper": 22803, "hottie": 22804, "erik": 22805, "revised": 22806, "isla": 22807, "travelphotography": 22808, "ooza": 22809, "enqui": 22810, "conferences": 22811, "clover": 22812, "groom": 22813, "curves": 22814, "liveon": 22815, "perf": 22816, "displaced": 22817, "bolog": 22818, "xxxx": 22819, "ðŁĺ©ðŁĺ©": 22820, "teal": 22821, "vessels": 22822, "rainforest": 22823, "calci": 22824, "panther": 22825, "giraffe": 22826, "tasted": 22827, "imagery": 22828, "padres": 22829, "daytime": 22830, "bass": 22831, "ripe": 22832, "opioid": 22833, "nue": 22834, "vinyl": 22835, "inventor": 22836, "sens": 22837, "processor": 22838, "mut": 22839, "gadgets": 22840, "biblical": 22841, "shannon": 22842, "jacqueline": 22843, "cary": 22844, "theresistance": 22845, "alien": 22846, "nvi": 22847, "cosy": 22848, "bihar": 22849, "foley": 22850, "rend": 22851, "mugs": 22852, "faken": 22853, "clone": 22854, "niallo": 22855, "grabbed": 22856, "chihu": 22857, "powerhouse": 22858, "ntt": 22859, "cherokee": 22860, "sponge": 22861, "implementing": 22862, "rhine": 22863, "leone": 22864, "ðŁįĢ": 22865, "prettiest": 22866, "infrared": 22867, "improv": 22868, "switched": 22869, "tubes": 22870, "contr": 22871, "blk": 22872, "projected": 22873, "beaver": 22874, "yot": 22875, "bbcradio": 22876, "thigh": 22877, "persecu": 22878, "apologize": 22879, "wack": 22880, "poster": 22881, "oliver": 22882, "aza": 22883, "loud": 22884, "(?)": 22885, "fthe": 22886, "womenshi": 22887, "sparrow": 22888, "blush": 22889, "usable": 22890, "scales": 22891, "itative": 22892, "peuge": 22893, "needing": 22894, "leggings": 22895, "glamorous": 22896, "matur": 22897, "cz": 22898, "watt": 22899, "dab": 22900, "tamar": 22901, "etsym": 22902, "bauer": 22903, "heartfelt": 22904, "hn": 22905, "elsewhere": 22906, "birch": 22907, "alumini": 22908, "huck": 22909, "eme": 22910, "jl": 22911, "trafford": 22912, "dz": 22913, "portions": 22914, "anasta": 22915, "arthritis": 22916, "espn": 22917, "bergen": 22918, "violation": 22919, "yoshi": 22920, "cz": 22921, "northumberland": 22922, "closures": 22923, "ðŁĩ¯ðŁĩ": 22924, "smiley": 22925, "rw": 22926, "telugu": 22927, "intensi": 22928, "gregg": 22929, "vega": 22930, "dungeon": 22931, "southbound": 22932, "bail": 22933, "dominican": 22934, "semifinal": 22935, "chapters": 22936, "hitch": 22937, "vanity": 22938, "transiti": 22939, "recommends": 22940, "satisf": 22941, "barca": 22942, "queens": 22943, "((": 22944, "destruc": 22945, "strait": 22946, "ravi": 22947, "desserts": 22948, "intru": 22949, "haram": 22950, "kos": 22951, "foe": 22952, "fatty": 22953, "paisley": 22954, "magnitude": 22955, "dridge": 22956, "comey": 22957, "schemes": 22958, "visionary": 22959, "ourt": 22960, "downloaded": 22961, "ðŁĻĮðŁı½": 22962, "gdpr": 22963, "lani": 22964, "pwc": 22965, "guad": 22966, "nicest": 22967, "stakeholders": 22968, "referred": 22969, "georgetown": 22970, "arvindkejriwal": 22971, "schneider": 22972, "indoors": 22973, "allstar": 22974, "stranded": 22975, "gender": 22976, "zepp": 22977, "masses": 22978, "ðŁIJ±": 22979, "patiently": 22980, "bldg": 22981, "zab": 22982, "wearab": 22983, "vivid": 22984, "heck": 22985, "della": 22986, "symb": 22987, "jeopar": 22988, "lager": 22989, "àª": 22990, "combines": 22991, "nec": 22992, "bray": 22993, "flop": 22994, "txwx": 22995, "joys": 22996, "pont": 22997, "profound": 22998, "surround": 22999, "madhu": 23000, "mable": 23001, "ayr": 23002, "teas": 23003, "nsa": 23004, "openly": 23005, "ernest": 23006, "ãĥ©": 23007, "topo": 23008, "gna": 23009, "antioxid": 23010, "tian": 23011, "etr": 23012, "cello": 23013, "mathi": 23014, "generosity": 23015, "biting": 23016, "manic": 23017, "kelsey": 23018, "cheeks": 23019, "tender": 23020, "wth": 23021, "pronoun": 23022, "ultimately": 23023, "gusta": 23024, "arianag": 23025, "gerry": 23026, "bleed": 23027, "reddy": 23028, "mich": 23029, "mitsubishi": 23030, "operated": 23031, "sexually": 23032, "mau": 23033, "cllr": 23034, "vids": 23035, "coc": 23036, "melted": 23037, "ðŁĮĪ": 23038, "qld": 23039, "itech": 23040, "instrumental": 23041, "endgame": 23042, "ðŁĵĸ": 23043, "energi": 23044, "brownie": 23045, "tamil": 23046, "atin": 23047, "dominated": 23048, "praises": 23049, "fireplace": 23050, "sensational": 23051, "mena": 23052, "karti": 23053, "unprece": 23054, "rupt": 23055, "oriental": 23056, "mccor": 23057, "tournaments": 23058, "scenter": 23059, "reeves": 23060, "prescription": 23061, "same": 23062, "frau": 23063, "truffle": 23064, "embo": 23065, "romans": 23066, "blasts": 23067, "technological": 23068, "prat": 23069, "bsb": 23070, "yar": 23071, "trendy": 23072, "acl": 23073, "alad": 23074, "ðŁįģ": 23075, "ohh": 23076, "bankrupt": 23077, "thoven": 23078, "regards": 23079, "iser": 23080, "warwick": 23081, "vineyards": 23082, "realm": 23083, "niallofficial": 23084, "dota": 23085, "gemini": 23086, "todo": 23087, "vable": 23088, "¨¨": 23089, "lau": 23090, "wreath": 23091, "juve": 23092, "natasha": 23093, "lever": 23094, "lori": 23095, "horser": 23096, "cctv": 23097, "airbnb": 23098, "esanders": 23099, "sinclair": 23100, "emabiggest": 23101, "highschool": 23102, "contest": 23103, "optimistic": 23104, "tte": 23105, "ðŁĴķðŁĴķ": 23106, "ssd": 23107, "yee": 23108, "helena": 23109, "consen": 23110, "ricks": 23111, "jesse": 23112, "anic": 23113, "ðŁİ¯": 23114, "reacts": 23115, "robe": 23116, "independence": 23117, "voltage": 23118, "mington": 23119, "sant": 23120, "à¸Ļà¸": 23121, "----------------": 23122, "sentinel": 23123, "kett": 23124, "rehearsing": 23125, "aaaaaaaa": 23126, "softhe": 23127, "stirling": 23128, "search": 23129, "wigan": 23130, "standout": 23131, "snail": 23132, "pentagon": 23133, "Äģ": 23134, "chlor": 23135, "crust": 23136, "netany": 23137, "chemist": 23138, "disappeared": 23139, "ricardo": 23140, "spiders": 23141, "bose": 23142, "warren": 23143, "messing": 23144, "banners": 23145, "guel": 23146, "parach": 23147, "maid": 23148, "counted": 23149, "epile": 23150, "bonfire": 23151, "speechless": 23152, "setter": 23153, "measured": 23154, "rejects": 23155, "nikki": 23156, "lester": 23157, "forensic": 23158, "fabrics": 23159, "aloha": 23160, "preserved": 23161, "watford": 23162, "detailing": 23163, "darth": 23164, "bou": 23165, "carly": 23166, "...'": 23167, "tailgate": 23168, "notifications": 23169, "å¤": 23170, "passive": 23171, "trousers": 23172, "baloch": 23173, "rother": 23174, "typically": 23175, "Ã¥": 23176, "spit": 23177, "wiz": 23178, "sicily": 23179, "technically": 23180, "expose": 23181, "stage": 23182, "hubb": 23183, "cream": 23184, "caps": 23185, "poke": 23186, "sleek": 23187, "june": 23188, "temporarily": 23189, "dez": 23190, "awakens": 23191, "lame": 23192, "_-": 23193, "jiha": 23194, "tuesdays": 23195, "advised": 23196, "advisors": 23197, "existed": 23198, "disagree": 23199, "newsroom": 23200, "losers": 23201, "worldtour": 23202, "drying": 23203, "aldi": 23204, "harness": 23205, "footprint": 23206, "hobbit": 23207, "pmln": 23208, "iro": 23209, "quered": 23210, "assess": 23211, "gaze": 23212, "sab": 23213, "thian": 23214, "íĬ": 23215, "tif": 23216, "observe": 23217, "evil": 23218, "drawer": 23219, "sweep": 23220, "cory": 23221, "cody": 23222, "kyoto": 23223, "callum": 23224, "ninj": 23225, "laurent": 23226, "bei": 23227, "sketching": 23228, "customized": 23229, "dur": 23230, "regrets": 23231, "knoxville": 23232, "ìķĦ": 23233, "messaging": 23234, "gracie": 23235, "abundance": 23236, "bidding": 23237, "brewed": 23238, "flouri": 23239, "therapeutic": 23240, "altitude": 23241, "hogs": 23242, "burner": 23243, "electro": 23244, "wonderfully": 23245, "heater": 23246, "postpon": 23247, "livery": 23248, "rall": 23249, "adas": 23250, "aac": 23251, "saul": 23252, "brooklyn": 23253, "playhouse": 23254, "âĻ¥âĻ¥âĻ¥": 23255, "charitable": 23256, "iny": 23257, "zah": 23258, "competitions": 23259, "beav": 23260, "plugged": 23261, "ois": 23262, "doom": 23263, "astronom": 23264, "specialized": 23265, "maxi": 23266, "taps": 23267, "cellular": 23268, "depressed": 23269, "folklorethursday": 23270, "crib": 23271, "emul": 23272, "ë°©": 23273, "figh": 23274, "ruz": 23275, "carlisle": 23276, "spear": 23277, "sidewalk": 23278, "dei": 23279, "dependent": 23280, "laces": 23281, "nhs": 23282, "ðŁĮĻ": 23283, "realizing": 23284, "network": 23285, "riche": 23286, "regin": 23287, "refresh": 23288, "stral": 23289, "pathology": 23290, "plaid": 23291, "psychedelic": 23292, "hind": 23293, "uka": 23294, "algorithm": 23295, "linking": 23296, "progressi": 23297, "fey": 23298, "dade": 23299, "hydrated": 23300, "bant": 23301, "famed": 23302, "cotsw": 23303, "boise": 23304, "asc": 23305, "racing": 23306, "javier": 23307, "wwen": 23308, "marlins": 23309, "poop": 23310, "swept": 23311, "tonights": 23312, "wef": 23313, "anime": 23314, "slovak": 23315, "âŀĸâŀĸ": 23316, "claus": 23317, "lemme": 23318, "clippers": 23319, "rels": 23320, "arianagrande": 23321, "rte": 23322, "kot": 23323, "thalapathy": 23324, "hungarian": 23325, "zuma": 23326, "yvon": 23327, "isu": 23328, "journeys": 23329, "clinics": 23330, "bebe": 23331, "wwf": 23332, "nws": 23333, "superheroes": 23334, "erit": 23335, "sleague": 23336, "identification": 23337, "motto": 23338, "bai": 23339, "sourced": 23340, "iller": 23341, "api": 23342, "prise": 23343, "unprecedented": 23344, "damas": 23345, "tunisia": 23346, "drain": 23347, "underestim": 23348, "ether": 23349, "quarterly": 23350, "rewarding": 23351, "alham": 23352, "wolverine": 23353, "cabine": 23354, "hypno": 23355, "nadine": 23356, "havana": 23357, "dae": 23358, "ðŁĵĪ": 23359, "dron": 23360, "readings": 23361, "bati": 23362, "pico": 23363, "merci": 23364, "itian": 23365, "walkers": 23366, "elope": 23367, "mikey": 23368, "godzilla": 23369, "burlington": 23370, "abuja": 23371, "socialism": 23372, "atility": 23373, "shell": 23374, "harrypotter": 23375, "gno": 23376, "abur": 23377, "releg": 23378, "felici": 23379, "rogen": 23380, "neuroscience": 23381, "instin": 23382, "atham": 23383, "vouchers": 23384, "jarre": 23385, "fuse": 23386, "defici": 23387, "monterey": 23388, "deport": 23389, "midday": 23390, "ppard": 23391, "freed": 23392, "ameter": 23393, "wilt": 23394, "ningham": 23395, "pratt": 23396, "liberty": 23397, "slogan": 23398, "oto": 23399, "pri": 23400, "coated": 23401, "cpd": 23402, "nett": 23403, "illas": 23404, "malawi": 23405, "evolve": 23406, "accessibility": 23407, "ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥": 23408, "ornament": 23409, "bp": 23410, "elis": 23411, "sonline": 23412, "chiro": 23413, "flick": 23414, "ibm": 23415, "arak": 23416, "enables": 23417, "garland": 23418, "sane": 23419, "cuties": 23420, "trip": 23421, "rotterdam": 23422, "nys": 23423, "lamps": 23424, "lucas": 23425, "bog": 23426, "rails": 23427, "travelled": 23428, "hicks": 23429, "enu": 23430, "sabha": 23431, "scrub": 23432, "hier": 23433, "hartford": 23434, "foo": 23435, "fernandez": 23436, "trevor": 23437, "mattress": 23438, "appointments": 23439, "alej": 23440, "fei": 23441, "ologist": 23442, "safar": 23443, "octa": 23444, "src": 23445, "shaun": 23446, "ambient": 23447, "dric": 23448, "biker": 23449, "shee": 23450, "mustache": 23451, "hta": 23452, "boone": 23453, "herty": 23454, "cardio": 23455, "brakes": 23456, "recital": 23457, "consists": 23458, "overwhelmed": 23459, "caul": 23460, "robbins": 23461, "imit": 23462, "alth": 23463, "url": 23464, "bibli": 23465, "onne": 23466, "blacklivesmatter": 23467, "difficulties": 23468, "telang": 23469, "taller": 23470, "ðŁĵĨ": 23471, "debating": 23472, "burrito": 23473, "movember": 23474, "strengthening": 23475, "boe": 23476, "testam": 23477, "miracles": 23478, "baseball": 23479, "renee": 23480, "ðŁijīðŁı»": 23481, "alfa": 23482, "âĺĺ": 23483, "unstoppable": 23484, "ecs": 23485, "gmo": 23486, "giftideas": 23487, "pathway": 23488, "fencing": 23489, "ðŁİ¤": 23490, "bham": 23491, "ras": 23492, "sko": 23493, "dled": 23494, "thelast": 23495, "magnum": 23496, "binary": 23497, "wilde": 23498, "wilder": 23499, "whati": 23500, "barbecue": 23501, "hism": 23502, "canoe": 23503, "kurdi": 23504, "elive": 23505, "advantages": 23506, "madame": 23507, "bier": 23508, "missing": 23509, "entertain": 23510, "airforce": 23511, "yama": 23512, "cis": 23513, "hashtags": 23514, "jis": 23515, "veil": 23516, "dreamy": 23517, "tense": 23518, "mayward": 23519, "chateau": 23520, "huntington": 23521, "âļĵ": 23522, "vall": 23523, "upon": 23524, "blouse": 23525, "dunes": 23526, "ðŁĺ´": 23527, "fertility": 23528, "mole": 23529, "currencies": 23530, "stu": 23531, "berlin": 23532, "toasted": 23533, "divas": 23534, "walt": 23535, "lark": 23536, "pora": 23537, "hitter": 23538, "umer": 23539, "chilled": 23540, "balancing": 23541, "fais": 23542, "yin": 23543, "ortiz": 23544, "eastenders": 23545, "hate": 23546, "ural": 23547, "april": 23548, "timel": 23549, "à±": 23550, "pero": 23551, "stocked": 23552, "respects": 23553, "tht": 23554, "bestfriends": 23555, "givingtuesday": 23556, "bead": 23557, "invent": 23558, "imi": 23559, "naples": 23560, "combining": 23561, "tokens": 23562, "thirst": 23563, "masc": 23564, "parrot": 23565, "spu": 23566, "denton": 23567, "*-*": 23568, "tres": 23569, "suburban": 23570, "width": 23571, "sive": 23572, "contender": 23573, "sirius": 23574, "lok": 23575, "troopers": 23576, "outrage": 23577, "turbo": 23578, "fragile": 23579, "messed": 23580, "doh": 23581, "discord": 23582, "netanyahu": 23583, "resign": 23584, "forgiveness": 23585, "mohan": 23586, "munch": 23587, "camou": 23588, "identifying": 23589, "enabling": 23590, "hotter": 23591, "thornton": 23592, "jaipur": 23593, "arya": 23594, "ðŁı»âĢįâĻĢï¸ı": 23595, "mustaf": 23596, "majors": 23597, "oke": 23598, "duffy": 23599, "rohing": 23600, "tilt": 23601, "ðŁĩ®ðŁĩ³": 23602, "rockstar": 23603, "sheep": 23604, "hendrix": 23605, "rav": 23606, "invention": 23607, "dou": 23608, "laguna": 23609, "grumpy": 23610, "swis": 23611, "impe": 23612, ")'": 23613, "youths": 23614, "bunker": 23615, "stache": 23616, "oppose": 23617, "indies": 23618, "accelerate": 23619, "mlp": 23620, "eden": 23621, "wann": 23622, "kail": 23623, "akshaykumar": 23624, "supt": 23625, "polym": 23626, "middleton": 23627, "extraordin": 23628, "wilson": 23629, "australian": 23630, "aluminium": 23631, "wayne": 23632, "alumnus": 23633, "matics": 23634, "grim": 23635, "ernie": 23636, "oppa": 23637, "competitors": 23638, "randall": 23639, "hence": 23640, "declares": 23641, "preaching": 23642, "shahe": 23643, "cane": 23644, "sustainable": 23645, "staples": 23646, "ledge": 23647, "adena": 23648, "doctoral": 23649, "burgundy": 23650, "decorate": 23651, "rendered": 23652, "risen": 23653, "prank": 23654, "dior": 23655, "beethoven": 23656, "floor": 23657, "accom": 23658, "tot": 23659, "hodg": 23660, "tourism": 23661, "sayin": 23662, "objective": 23663, "markers": 23664, "premiership": 23665, "enabled": 23666, "camoufla": 23667, "giant": 23668, "Ñģ": 23669, "smokey": 23670, "ricket": 23671, "pang": 23672, "depending": 23673, "sation": 23674, "evolving": 23675, "intercep": 23676, "census": 23677, "tofthe": 23678, "reen": 23679, "mendoza": 23680, "trumpet": 23681, "marketers": 23682, "anit": 23683, "ðŁĻĬ": 23684, "northwestern": 23685, "vla": 23686, "fotogra": 23687, "blackandwhite": 23688, "chewan": 23689, "wig": 23690, "troom": 23691, "gingerbread": 23692, "kn": 23693, "romero": 23694, "nfc": 23695, "orchi": 23696, "funko": 23697, "source": 23698, "fs": 23699, "raped": 23700, "ost": 23701, "tarot": 23702, "annually": 23703, "ðŁĺ¬": 23704, "rill": 23705, "delav": 23706, "..!!": 23707, "ses": 23708, "cann": 23709, "medicare": 23710, "phel": 23711, "apex": 23712, "guardian": 23713, "remained": 23714, "rpm": 23715, "añ": 23716, "storymonth": 23717, "instagood": 23718, "neighbour": 23719, "ping": 23720, "semite": 23721, "mystic": 23722, "ascot": 23723, "mater": 23724, "handful": 23725, "dangers": 23726, "tid": 23727, "anaheim": 23728, "opoly": 23729, "shallow": 23730, "namibia": 23731, "toria": 23732, "procurement": 23733, "bigbang": 23734, "announcements": 23735, "prosecutor": 23736, "bengals": 23737, "salle": 23738, "enroll": 23739, "gastro": 23740, "suggestion": 23741, "bak": 23742, "haul": 23743, "buddhism": 23744, "berniesanders": 23745, "flute": 23746, "fatigue": 23747, "cynthia": 23748, "choi": 23749, "irwin": 23750, "gua": 23751, "strous": 23752, "hp": 23753, "bap": 23754, "satisfying": 23755, "playa": 23756, "ðŁİ¼": 23757, "instap": 23758, "alice": 23759, "tp": 23760, "irrigation": 23761, "ðŁĩ¬ðŁĩ§": 23762, "intric": 23763, "clues": 23764, "plex": 23765, "sax": 23766, "hepat": 23767, "dumped": 23768, "significance": 23769, "byu": 23770, "medication": 23771, "prov": 23772, "toughest": 23773, "cornish": 23774, "âŀľ": 23775, "kelley": 23776, "uv": 23777, "sizz": 23778, "sibling": 23779, "mest": 23780, "distor": 23781, "diplomatic": 23782, "auntie": 23783, "bhat": 23784, "sonic": 23785, "brenda": 23786, "pumpkins": 23787, "roch": 23788, "blackburn": 23789, "urged": 23790, "shia": 23791, "arrangements": 23792, "flood": 23793, "saunders": 23794, "lecturer": 23795, "nouri": 23796, "populations": 23797, "diplomacy": 23798, "consistently": 23799, "ðŁ¤Ļ": 23800, "tmund": 23801, "cauliflower": 23802, "lily": 23803, "vocabulary": 23804, "varieties": 23805, "cooker": 23806, "uptown": 23807, "quent": 23808, "mosa": 23809, "reinde": 23810, "velocity": 23811, "spruce": 23812, "socialmedi": 23813, "iber": 23814, "voluntary": 23815, "processed": 23816, "baltic": 23817, "yang": 23818, "lebanese": 23819, "dp": 23820, "dolly": 23821, "arrangement": 23822, "yuri": 23823, "cranberry": 23824, "kalyan": 23825, "elevation": 23826, "cliff": 23827, "pushes": 23828, "ìĬ¤": 23829, "silic": 23830, "cowx": 23831, "eternity": 23832, "slaves": 23833, "vinegar": 23834, "gloucester": 23835, "contained": 23836, "breakingnews": 23837, "against": 23838, "renovated": 23839, "normandy": 23840, "heroin": 23841, "ysm": 23842, "mods": 23843, "greek": 23844, "undi": 23845, "trench": 23846, "vh": 23847, "encourages": 23848, "headache": 23849, "grange": 23850, ":'": 23851, "evergreen": 23852, "ÙĬ": 23853, "reckon": 23854, "abused": 23855, "thru": 23856, "choice": 23857, "tidy": 23858, "colder": 23859, "schoice": 23860, "hain": 23861, "brum": 23862, "liars": 23863, "breit": 23864, "yorker": 23865, "shack": 23866, "heidi": 23867, "michaels": 23868, "scopic": 23869, "fascist": 23870, "playful": 23871, "cac": 23872, "yasss": 23873, "shad": 23874, "..?": 23875, "quen": 23876, "ramirez": 23877, "clifton": 23878, "prs": 23879, "bestfan": 23880, "âģł": 23881, "generating": 23882, "headset": 23883, "disappointment": 23884, "abstract": 23885, "boiled": 23886, "parenthood": 23887, "azerbaijan": 23888, "exhibiting": 23889, "bombay": 23890, "olivier": 23891, "koso": 23892, "unlea": 23893, "maternity": 23894, "izer": 23895, "sives": 23896, "rhu": 23897, "coll": 23898, "saskatchewan": 23899, "freakin": 23900, "dek": 23901, "nag": 23902, "stabili": 23903, "ðŁįķ": 23904, "organizer": 23905, "bosses": 23906, "aru": 23907, "uva": 23908, "atable": 23909, "taun": 23910, "afterwards": 23911, "fertili": 23912, "verge": 23913, "azi": 23914, "morph": 23915, "à¹ģà¸": 23916, "jerk": 23917, "cosmetic": 23918, "kow": 23919, "strust": 23920, "apache": 23921, "postcards": 23922, "formul": 23923, "ìĭ": 23924, "spinal": 23925, "jackpot": 23926, "electri": 23927, "ÃŃ": 23928, "loy": 23929, "grader": 23930, "diablo": 23931, "ardi": 23932, "hesit": 23933, "fw": 23934, "archery": 23935, "pash": 23936, "theories": 23937, "repeal": 23938, "relive": 23939, "percy": 23940, "âĺĨ": 23941, "imin": 23942, "synchron": 23943, "shampoo": 23944, "coupons": 23945, "oto": 23946, "lai": 23947, "thought": 23948, "luxembourg": 23949, "mov": 23950, "ðŁĺ¥": 23951, "gemma": 23952, "seated": 23953, "mga": 23954, "stratford": 23955, "uncertainty": 23956, "shifts": 23957, "esto": 23958, "fool": 23959, "firearms": 23960, "corrie": 23961, "kiki": 23962, "apparent": 23963, "pills": 23964, "olympia": 23965, "fid": 23966, "elevated": 23967, "decks": 23968, "ignoring": 23969, "avalan": 23970, "rov": 23971, "whistle": 23972, "ptsd": 23973, "militants": 23974, "robotic": 23975, "pacers": 23976, "quilt": 23977, "bankruptcy": 23978, "lich": 23979, "percussion": 23980, "celebrity": 23981, "als": 23982, "(;": 23983, "sut": 23984, "pokemongo": 23985, "hg": 23986, "offs": 23987, "gibraltar": 23988, "screams": 23989, "billie": 23990, "genome": 23991, "marin": 23992, "beams": 23993, "archbishop": 23994, "emin": 23995, "bedrooms": 23996, "gated": 23997, "olly": 23998, "warranty": 23999, "atown": 24000, "cuddles": 24001, "gunna": 24002, "kic": 24003, "vive": 24004, "cymru": 24005, "narrow": 24006, "prob": 24007, "leo": 24008, "references": 24009, "manufactured": 24010, "chopper": 24011, "brunswick": 24012, "semis": 24013, "donia": 24014, "rye": 24015, "mano": 24016, "hurting": 24017, "?#": 24018, "holli": 24019, "investigations": 24020, "cels": 24021, "ðŁĵŀ": 24022, "lester": 24023, "temples": 24024, "storey": 24025, "mcmahon": 24026, "toilets": 24027, "woof": 24028, "ï¸İ": 24029, "leverage": 24030, "atom": 24031, "nightmares": 24032, "victorious": 24033, "haunting": 24034, "customer": 24035, "agi": 24036, "yoongi": 24037, "monty": 24038, "veronica": 24039, "wur": 24040, "intimid": 24041, "blankets": 24042, "volution": 24043, "jm": 24044, "âĺİ": 24045, "amon": 24046, "judith": 24047, "ðŁĺİðŁĺİ": 24048, "distracted": 24049, "drip": 24050, "hurricane": 24051, "andes": 24052, "revelation": 24053, "troop": 24054, "ableg": 24055, "collin": 24056, "tibetan": 24057, "worrying": 24058, "internationally": 24059, "eater": 24060, "cameroon": 24061, "brador": 24062, "yuk": 24063, "ðŁĴĹðŁĴĹ": 24064, "trak": 24065, "slopes": 24066, "cier": 24067, "nea": 24068, "oler": 24069, "taka": 24070, "albion": 24071, "volcanic": 24072, "amn": 24073, "afi": 24074, "obstac": 24075, "facetime": 24076, "gering": 24077, "npr": 24078, "metallica": 24079, "organic": 24080, "ðŁĴ¡": 24081, "kidd": 24082, "dances": 24083, "pembro": 24084, "washer": 24085, "mits": 24086, "omer": 24087, "emotionally": 24088, "tango": 24089, "ipo": 24090, "docks": 24091, "scanning": 24092, "specs": 24093, "thom": 24094, "theology": 24095, "emergen": 24096, "omi": 24097, "gpa": 24098, "selections": 24099, "unnecessary": 24100, "image": 24101, "ters": 24102, "induced": 24103, "gigan": 24104, "rentals": 24105, "supplied": 24106, "mfa": 24107, "shankar": 24108, "later": 24109, "pajam": 24110, "clave": 24111, "Ùģ": 24112, "mahin": 24113, "carlson": 24114, "avian": 24115, "anova": 24116, "katie": 24117, "ajith": 24118, "designated": 24119, "chocolates": 24120, "investigators": 24121, "glazed": 24122, "princess": 24123, "erry": 24124, "ragn": 24125, "ourable": 24126, "hru": 24127, "sundance": 24128, "peugeot": 24129, "steampunk": 24130, "ghlin": 24131, "grease": 24132, "hires": 24133, "zap": 24134, "perce": 24135, "jill": 24136, "tome": 24137, "hehehe": 24138, "joyful": 24139, "maestro": 24140, "nished": 24141, "genealo": 24142, "vich": 24143, "pits": 24144, "foxes": 24145, "goodman": 24146, "emerson": 24147, "lobes": 24148, "converse": 24149, "oats": 24150, "thomson": 24151, "rahim": 24152, "malware": 24153, "ahi": 24154, "mankind": 24155, "resin": 24156, "img": 24157, "swood": 24158, "kinder": 24159, "scroll": 24160, "ara": 24161, "sakura": 24162, "robbed": 24163, "xion": 24164, "nya": 24165, "cism": 24166, "cedar": 24167, "bein": 24168, "mourning": 24169, "torto": 24170, "heathrow": 24171, "donegal": 24172, "barb": 24173, "hydration": 24174, "kor": 24175, "elimination": 24176, "supdates": 24177, "hills": 24178, "appeti": 24179, "starred": 24180, "kom": 24181, "gwen": 24182, "ddd": 24183, "cray": 24184, "scanner": 24185, "personalised": 24186, "serenity": 24187, "redesign": 24188, "metaph": 24189, "boxed": 24190, "judgment": 24191, "nose": 24192, "ë¹": 24193, "erad": 24194, "acne": 24195, "suppliers": 24196, "energetic": 24197, "vom": 24198, "asap": 24199, "ðŁĶ¸": 24200, "irvine": 24201, "hatch": 24202, "lass": 24203, "adren": 24204, "waffles": 24205, "accurately": 24206, "icio": 24207, "ittle": 24208, "seun": 24209, "occupy": 24210, "webcam": 24211, "thenew": 24212, "entes": 24213, "gai": 24214, "jw": 24215, "accountable": 24216, "visor": 24217, "irrit": 24218, "licensing": 24219, "huddersfield": 24220, "genie": 24221, "ðŁİ¾": 24222, "atmospheric": 24223, "tensions": 24224, "spartan": 24225, "clifford": 24226, "olan": 24227, "northbound": 24228, "ameen": 24229, "censor": 24230, "uel": 24231, "stery": 24232, "$$": 24233, "farrell": 24234, "hyster": 24235, "clt": 24236, "sedan": 24237, "replied": 24238, "describing": 24239, "microwave": 24240, "slab": 24241, "prosp": 24242, "assisting": 24243, "rubio": 24244, "ethan": 24245, "hhhhh": 24246, "guay": 24247, "zman": 24248, "raise": 24249, "rolling": 24250, "oe": 24251, "nile": 24252, "ambrose": 24253, "scarborough": 24254, "heroic": 24255, "cooks": 24256, "mort": 24257, "chopra": 24258, "ðŁĮ·": 24259, "tob": 24260, "shaving": 24261, "stacey": 24262, "dorm": 24263, "motorsports": 24264, "wiki": 24265, "folds": 24266, "spiced": 24267, "stressful": 24268, "literal": 24269, "fudge": 24270, "peggy": 24271, "waite": 24272, "tresses": 24273, "sesh": 24274, "pric": 24275, "ðŁİħ": 24276, "fright": 24277, "rva": 24278, "mumbai": 24279, "pom": 24280, "ttv": 24281, "cellar": 24282, "tome": 24283, "android": 24284, "doris": 24285, "tsunami": 24286, "tinder": 24287, "oec": 24288, "mwc": 24289, "dortmund": 24290, "nothin": 24291, "liti": 24292, "sou": 24293, "believein": 24294, "atu": 24295, "knocks": 24296, "magni": 24297, "sssss": 24298, "rohit": 24299, "inews": 24300, "angi": 24301, "mandy": 24302, "kettle": 24303, "intermediate": 24304, "avant": 24305, "curl": 24306, "endorsed": 24307, "orio": 24308, "urt": 24309, "consideration": 24310, "wires": 24311, "shelters": 24312, "bino": 24313, "vikram": 24314, "implemented": 24315, "lydia": 24316, "buk": 24317, "parody": 24318, "cnews": 24319, "undergraduate": 24320, "canucks": 24321, "sami": 24322, "politically": 24323, "rotten": 24324, "ghz": 24325, "textiles": 24326, "overload": 24327, "moderni": 24328, "recreational": 24329, "flir": 24330, "baton": 24331, "typography": 24332, "ovation": 24333, "intriguing": 24334, "pilgrimage": 24335, "alge": 24336, "adays": 24337, "tcmparty": 24338, "spelled": 24339, "curls": 24340, "booze": 24341, "stem": 24342, "annes": 24343, "irls": 24344, "sponge": 24345, "shopper": 24346, "signation": 24347, "brass": 24348, "mistress": 24349, "leah": 24350, "beginner": 24351, "lauderdale": 24352, "august": 24353, "preschool": 24354, "taping": 24355, "taipei": 24356, "executives": 24357, "bd": 24358, "rhetor": 24359, "escor": 24360, "immuno": 24361, "deeplearning": 24362, "statues": 24363, "itus": 24364, "manuscript": 24365, "lyric": 24366, "corvette": 24367, "molly": 24368, "lage": 24369, "dep": 24370, "cnbc": 24371, "lest": 24372, "jessi": 24373, "fife": 24374, "griffith": 24375, "opposing": 24376, "rang": 24377, "drills": 24378, "respectful": 24379, "pity": 24380, "dell": 24381, "harding": 24382, "playboy": 24383, "bloke": 24384, "shutout": 24385, "kili": 24386, "osp": 24387, "seattle": 24388, "bcpoli": 24389, "mises": 24390, "journals": 24391, "teaming": 24392, "esther": 24393, "freddy": 24394, "Ķï¸ı": 24395, "metrics": 24396, "notre": 24397, "garry": 24398, "forty": 24399, "navigate": 24400, "periods": 24401, "benedic": 24402, "jid": 24403, "daw": 24404, "ancestors": 24405, "restoring": 24406, "cong": 24407, "allergy": 24408, "titanium": 24409, "cence": 24410, "leaning": 24411, "abbas": 24412, "vast": 24413, "ucf": 24414, "roofing": 24415, "eman": 24416, "severely": 24417, "vogue": 24418, "veau": 24419, "inbound": 24420, "dz": 24421, "taneously": 24422, "stretching": 24423, "manchester": 24424, "dryer": 24425, "davis": 24426, "kanth": 24427, "thegame": 24428, "itted": 24429, "retain": 24430, "elles": 24431, "congestion": 24432, "fraternity": 24433, "ollie": 24434, "loki": 24435, "freely": 24436, "choo": 24437, "pony": 24438, "scep": 24439, "tably": 24440, "balt": 24441, "rockn": 24442, "dime": 24443, "logging": 24444, "ðŁį·": 24445, "adu": 24446, "havoc": 24447, "waterford": 24448, "charis": 24449, "sweetie": 24450, "running": 24451, "nerd": 24452, "erdogan": 24453, "zara": 24454, "weighing": 24455, "fifty": 24456, "precise": 24457, "lowell": 24458, "kurdistan": 24459, "ryo": 24460, "orth": 24461, "synth": 24462, "liners": 24463, "phenomenon": 24464, "artillery": 24465, "illegally": 24466, "construct": 24467, "nostalgic": 24468, "garth": 24469, "alta": 24470, "shelton": 24471, "asean": 24472, "wander": 24473, "durban": 24474, "diversi": 24475, "bono": 24476, "clon": 24477, "leman": 24478, "shun": 24479, "obstacles": 24480, "appetite": 24481, "feeder": 24482, "respiratory": 24483, "dixie": 24484, "formula": 24485, "anto": 24486, "sober": 24487, "extinct": 24488, "auc": 24489, "ingles": 24490, "legitimate": 24491, ";;": 24492, "minnie": 24493, "ipswich": 24494, "dramatically": 24495, "ðŁijıðŁı¼": 24496, "ingham": 24497, "military": 24498, "monet": 24499, "usnavy": 24500, "fork": 24501, "dunno": 24502, "player": 24503, "qotd": 24504, "stoo": 24505, "exor": 24506, "ethiopian": 24507, "filmfest": 24508, "pered": 24509, "cate": 24510, "saudi": 24511, "inner": 24512, "sincere": 24513, "tionality": 24514, "alee": 24515, "deeds": 24516, "cooperative": 24517, "ironic": 24518, "crocod": 24519, "brary": 24520, "postseason": 24521, "camper": 24522, "canary": 24523, "ein": 24524, "extensions": 24525, "nbd": 24526, "sherwood": 24527, "spokane": 24528, "hump": 24529, "jitsu": 24530, "ê¹": 24531, "daryl": 24532, "psi": 24533, "stabbed": 24534, "offerings": 24535, "expects": 24536, "caval": 24537, "bodybuilding": 24538, "framing": 24539, "fca": 24540, "yearly": 24541, "bombed": 24542, "skil": 24543, "researching": 24544, "judiciary": 24545, "greeted": 24546, "tudor": 24547, "milo": 24548, "innovate": 24549, "ðŁĺĽ": 24550, "rhs": 24551, "ruby": 24552, "contributor": 24553, "famer": 24554, "socially": 24555, "mlin": 24556, "fiery": 24557, "utter": 24558, "beaut": 24559, "itos": 24560, "devoted": 24561, "rainbow": 24562, "barney": 24563, "peren": 24564, "arjun": 24565, "rna": 24566, "gabby": 24567, "uti": 24568, "hannity": 24569, "pickle": 24570, "serv": 24571, "quakes": 24572, "ppe": 24573, "fem": 24574, "whitec": 24575, "jn": 24576, "victories": 24577, "ðŁ§¡": 24578, "golfer": 24579, "congratulates": 24580, "resulting": 24581, "mechanic": 24582, "urve": 24583, "centered": 24584, "kiev": 24585, "ans": 24586, "incub": 24587, "<<": 24588, "cmo": 24589, "bestfanarmy": 24590, "daph": 24591, "enham": 24592, "oncology": 24593, "kush": 24594, "txt": 24595, "oriented": 24596, "fashionable": 24597, "csr": 24598, "sahara": 24599, "rack": 24600, "pdp": 24601, "hanson": 24602, "à¸ĩ": 24603, "tiers": 24604, "rar": 24605, "panam": 24606, "insky": 24607, "sahi": 24608, "testament": 24609, "asthma": 24610, "inher": 24611, "fisheries": 24612, "order": 24613, "howe": 24614, "gallon": 24615, "epis": 24616, "suzanne": 24617, "drowning": 24618, "panelists": 24619, "ðŁĺ²": 24620, "ë¦": 24621, "alach": 24622, "commemorative": 24623, "attribu": 24624, "ðŁij»": 24625, "moo": 24626, "visional": 24627, "weeksary": 24628, "gust": 24629, "akin": 24630, "pointe": 24631, "eee": 24632, "dispar": 24633, "nipp": 24634, "dental": 24635, "stall": 24636, "pian": 24637, "bore": 24638, "ulster": 24639, "tick": 24640, "irr": 24641, "taehyung": 24642, "microphone": 24643, "bermuda": 24644, "gaard": 24645, "eler": 24646, "plumbing": 24647, "hugely": 24648, "âļ«ï¸ı": 24649, "raceway": 24650, "cambridge": 24651, "marcel": 24652, "burnley": 24653, "toast": 24654, "hollywood": 24655, "fasting": 24656, "mered": 24657, "hibition": 24658, "capped": 24659, "beneficial": 24660, "owning": 24661, "contamin": 24662, "arabian": 24663, "toon": 24664, "capac": 24665, "hulu": 24666, "smir": 24667, "nutrients": 24668, "sein": 24669, "graphs": 24670, "conditional": 24671, "ðŁijħ": 24672, "orac": 24673, "playin": 24674, "northe": 24675, "tornad": 24676, "marian": 24677, "jumbo": 24678, "lexi": 24679, "incredibleindia": 24680, "roadto": 24681, "ukone": 24682, "confusing": 24683, "sph": 24684, "shank": 24685, "pied": 24686, "mqm": 24687, "positively": 24688, "sherry": 24689, "pathways": 24690, "considers": 24691, "tofu": 24692, "arguments": 24693, "resilient": 24694, "chett": 24695, "withdra": 24696, "tero": 24697, "atedly": 24698, "swana": 24699, "heb": 24700, "flight": 24701, "harley": 24702, "decrease": 24703, "kindle": 24704, "bookshop": 24705, "³ï¸ı": 24706, "martyrs": 24707, "smur": 24708, "mccl": 24709, "concerto": 24710, "stime": 24711, "rejoice": 24712, "applau": 24713, "clement": 24714, "merkel": 24715, "jaime": 24716, "immortal": 24717, "isleof": 24718, "marco": 24719, "youtuber": 24720, "stalking": 24721, "metoo": 24722, "stack": 24723, "spouse": 24724, "ust": 24725, "luv": 24726, "âļ¾ï¸ı": 24727, "equestrian": 24728, "eving": 24729, "flin": 24730, "nickname": 24731, "thebig": 24732, "asar": 24733, "stacks": 24734, "walker": 24735, "bora": 24736, "kidnapped": 24737, "hurling": 24738, "humbold": 24739, "recalls": 24740, "copper": 24741, "annis": 24742, "seo": 24743, "merger": 24744, "muir": 24745, "addy": 24746, "ðŁĴªðŁĴª": 24747, "bex": 24748, "cracy": 24749, "conan": 24750, "congratulation": 24751, "midst": 24752, "âĻ¬": 24753, "forbi": 24754, "optic": 24755, "crate": 24756, "crocodile": 24757, "madagas": 24758, "securing": 24759, "aston": 24760, "ogue": 24761, "savior": 24762, "salisbury": 24763, "loveit": 24764, "fujifilm": 24765, "castles": 24766, "asst": 24767, "arrows": 24768, "spacious": 24769, "trs": 24770, "polyvore": 24771, "progression": 24772, "mri": 24773, "nelson": 24774, "bim": 24775, "indicator": 24776, "oda": 24777, "pepe": 24778, "resignation": 24779, "gut": 24780, "sneaker": 24781, "logically": 24782, "azy": 24783, "arella": 24784, "tearing": 24785, "joshi": 24786, "ssionism": 24787, "qpr": 24788, "mariah": 24789, "px": 24790, "bleed": 24791, "mian": 24792, "medley": 24793, "weiss": 24794, "kerry": 24795, "gatory": 24796, "atal": 24797, "madison": 24798, "avenger": 24799, "naby": 24800, "pland": 24801, "giles": 24802, "freshwater": 24803, "dington": 24804, "taj": 24805, "demonstrates": 24806, "ntv": 24807, "bulbs": 24808, "sundaymorning": 24809, "peake": 24810, "souvenir": 24811, "wah": 24812, "tonnes": 24813, "mkt": 24814, "complexity": 24815, "conden": 24816, "rossi": 24817, "bing": 24818, "yds": 24819, "suk": 24820, "ngo": 24821, "midland": 24822, "oly": 24823, "lifeis": 24824, "ripple": 24825, "moreno": 24826, "dders": 24827, "tus": 24828, "áĥ": 24829, "boul": 24830, "xa": 24831, "holdings": 24832, "wny": 24833, "shadowhunters": 24834, "kei": 24835, "aspire": 24836, "mous": 24837, "owen": 24838, "soak": 24839, "skirts": 24840, "mountaine": 24841, "storming": 24842, "chrome": 24843, "riots": 24844, "sarato": 24845, "amaze": 24846, "lessness": 24847, "navar": 24848, "criteria": 24849, "rafa": 24850, "indulge": 24851, "ayer": 24852, "porto": 24853, "namo": 24854, "................": 24855, "yields": 24856, "valle": 24857, "jh": 24858, "macron": 24859, "sains": 24860, "durant": 24861, "trailers": 24862, "wot": 24863, "confederate": 24864, "shrin": 24865, "idol": 24866, "formally": 24867, "tene": 24868, "motorcycles": 24869, "thang": 24870, "node": 24871, "banger": 24872, "daly": 24873, "pats": 24874, "enrollment": 24875, "auctions": 24876, "atal": 24877, "arbor": 24878, "logos": 24879, "dearest": 24880, "transaction": 24881, "domingo": 24882, "flea": 24883, "sermon": 24884, "deck": 24885, "sincere": 24886, "questioning": 24887, "julio": 24888, "wasp": 24889, "pretz": 24890, "armenian": 24891, "kham": 24892, "inflammation": 24893, "picturesque": 24894, "accidental": 24895, "filmmakers": 24896, "ðŁĺļ": 24897, "ðŁĴį": 24898, "casey": 24899, "sob": 24900, "yeezy": 24901, "goodwill": 24902, "paragra": 24903, "ssly": 24904, "feather": 24905, "dyed": 24906, "assassination": 24907, "nade": 24908, "bcs": 24909, "applies": 24910, "feminine": 24911, "feu": 24912, "extent": 24913, "deputies": 24914, "lack": 24915, "psychic": 24916, "goi": 24917, "killings": 24918, "pseu": 24919, "ðŁ¤ª": 24920, "unc": 24921, "marl": 24922, "tane": 24923, "mckenna": 24924, "surfer": 24925, "influences": 24926, "freeway": 24927, "hackney": 24928, "malaria": 24929, "eland": 24930, "teau": 24931, "remastered": 24932, "ر": 24933, "razor": 24934, "ggy": 24935, "corro": 24936, "laksh": 24937, "flair": 24938, "honesty": 24939, "hooray": 24940, "depp": 24941, "amc": 24942, "wednesdays": 24943, "qa": 24944, "edits": 24945, "-$": 24946, "sevilla": 24947, "doubled": 24948, "humanities": 24949, "ccot": 24950, "somos": 24951, "rine": 24952, "afa": 24953, "sioux": 24954, "reconstruction": 24955, "welding": 24956, "threads": 24957, "amish": 24958, "encouragement": 24959, "poder": 24960, "bock": 24961, "balm": 24962, "ptions": 24963, "standup": 24964, "accomplishments": 24965, "guarding": 24966, "conviction": 24967, "acion": 24968, "napoleon": 24969, "depicting": 24970, "attack": 24971, "sui": 24972, "wearable": 24973, "âĸªï¸ı": 24974, "potter": 24975, "escort": 24976, "vise": 24977, "tots": 24978, "boon": 24979, "eventprofs": 24980, "angular": 24981, "womenshistorymonth": 24982, "barrow": 24983, "schi": 24984, "accomp": 24985, "tik": 24986, "lend": 24987, "kensington": 24988, "wolfe": 24989, "stacked": 24990, "crashing": 24991, "exhibit": 24992, "winged": 24993, "sabrina": 24994, "masa": 24995, "kms": 24996, "always": 24997, "ett": 24998, "plasma": 24999, "counseling": 25000, "pickles": 25001, "nfldraft": 25002, "mrs": 25003, "inevitable": 25004, "courageous": 25005, "stafford": 25006, "writerslife": 25007, "hos": 25008, "ej": 25009, "ghyun": 25010, "trademark": 25011, "adrian": 25012, "influencer": 25013, "coronation": 25014, "raging": 25015, "explored": 25016, "usaf": 25017, "exception": 25018, "eux": 25019, "tanker": 25020, "swami": 25021, "packet": 25022, "ðŁij¨âĢį": 25023, "fen": 25024, "sheen": 25025, "aero": 25026, "jl": 25027, "regal": 25028, "nwt": 25029, "auster": 25030, "mehta": 25031, "charge": 25032, "aste": 25033, "bate": 25034, "infeld": 25035, "racecourse": 25036, "collapsed": 25037, "fleece": 25038, "zil": 25039, "allie": 25040, "alternatives": 25041, "georges": 25042, "ðŁĵį": 25043, "quirky": 25044, "fcb": 25045, "natgeo": 25046, "philanthropy": 25047, "brai": 25048, "everyday": 25049, "ðŁIJ°": 25050, "achers": 25051, "jaan": 25052, "fines": 25053, "qi": 25054, "fisherman": 25055, "distinct": 25056, "grimes": 25057, "nationalist": 25058, "commence": 25059, "rown": 25060, "âĢ³": 25061, "zing": 25062, "fter": 25063, "hrw": 25064, "baroque": 25065, "blender": 25066, "kitty": 25067, "hooks": 25068, "cited": 25069, "wanda": 25070, "consensus": 25071, "reindeer": 25072, "anand": 25073, "supply": 25074, "meds": 25075, "vn": 25076, "olph": 25077, "ratchet": 25078, "sheldon": 25079, "securities": 25080, "ë°©íĥ": 25081, "crom": 25082, "mosquito": 25083, "jeric": 25084, "immac": 25085, "dimensions": 25086, "â¤": 25087, "dissi": 25088, "spongebob": 25089, "damien": 25090, "stevenson": 25091, "joanne": 25092, "delish": 25093, "yikes": 25094, "thanx": 25095, "surveys": 25096, "postponed": 25097, "alcoholic": 25098, "alised": 25099, "ðŁĻıðŁı»": 25100, "doch": 25101, "sentim": 25102, "meredith": 25103, "compares": 25104, "bago": 25105, "happydays": 25106, "moss": 25107, "ãħĭ": 25108, "nec": 25109, "gnment": 25110, "frustrated": 25111, "combin": 25112, "riv": 25113, "eclec": 25114, "collo": 25115, "compliment": 25116, "actorslife": 25117, "ctto": 25118, "nicar": 25119, "ophon": 25120, "aparthe": 25121, "mant": 25122, "jade": 25123, "trolley": 25124, "optimization": 25125, "eyeon": 25126, "ecological": 25127, "quist": 25128, "ephe": 25129, "à¥ĩ": 25130, "cinco": 25131, "appoints": 25132, "oldschool": 25133, "cpr": 25134, "behavioral": 25135, "minaj": 25136, ":-(": 25137, "tagging": 25138, "eval": 25139, "joaqu": 25140, "ðŁĺ«": 25141, "hak": 25142, "deme": 25143, "jamaican": 25144, "sos": 25145, "hyatt": 25146, "handbook": 25147, "librarian": 25148, "hannibal": 25149, "pumping": 25150, "chom": 25151, "fman": 25152, "gai": 25153, "hull": 25154, "responders": 25155, "greenville": 25156, "nus": 25157, "vaugh": 25158, "ðŁİīðŁİī": 25159, "taxi": 25160, "goldberg": 25161, "mantra": 25162, "tease": 25163, "forbidden": 25164, "methodist": 25165, "ativity": 25166, "****": 25167, "ect": 25168, "mcgr": 25169, "Ħëĭ": 25170, "seb": 25171, "amidst": 25172, "disappear": 25173, "thyro": 25174, "philips": 25175, "erina": 25176, "vicious": 25177, "streamer": 25178, "millionaire": 25179, "map": 25180, "strick": 25181, "hackathon": 25182, "gha": 25183, "edic": 25184, "mika": 25185, "peck": 25186, "illi": 25187, "antoine": 25188, "arca": 25189, "optic": 25190, "maure": 25191, "ðŁĩ¦ðŁĩº": 25192, "clashes": 25193, "manly": 25194, "âĺģ": 25195, "alvar": 25196, "andres": 25197, "mei": 25198, "elm": 25199, "wwww": 25200, "altered": 25201, "lte": 25202, "ê¹Ģ": 25203, "mojo": 25204, "forrest": 25205, "thalai": 25206, "nont": 25207, "speeches": 25208, "acknowledge": 25209, "ignite": 25210, "xfactor": 25211, "ðŁ¥Ĥ": 25212, "meadow": 25213, "disrupt": 25214, "debuted": 25215, "scrimmage": 25216, "pharmaceutical": 25217, "fidd": 25218, "foundations": 25219, "philosopher": 25220, "etal": 25221, "publishers": 25222, "boys": 25223, "cke": 25224, "rugged": 25225, "optimism": 25226, "rebe": 25227, "philharmon": 25228, "narcis": 25229, "rallies": 25230, "luis": 25231, "goblue": 25232, "folded": 25233, "unacceptable": 25234, "optimal": 25235, "lisa": 25236, "polaro": 25237, "+.": 25238, "enza": 25239, "âĿ£ï¸ı": 25240, "monopoly": 25241, "graceful": 25242, "dairy": 25243, "dua": 25244, "difficulty": 25245, "judgement": 25246, "osi": 25247, "mersey": 25248, "flux": 25249, "newfound": 25250, "terns": 25251, "dimensional": 25252, "invic": 25253, "alba": 25254, "amit": 25255, "abudhabi": 25256, "algeria": 25257, "automobile": 25258, "thead": 25259, "lotion": 25260, "accelerator": 25261, "vacant": 25262, "ition": 25263, "luf": 25264, "alic": 25265, "pll": 25266, "blazing": 25267, "baz": 25268, "sene": 25269, "ðŁij¼": 25270, "villains": 25271, "directory": 25272, "eisen": 25273, "tock": 25274, "brochure": 25275, "ripp": 25276, "hbd": 25277, "zaynmalik": 25278, "niche": 25279, "lolol": 25280, "certificates": 25281, "morse": 25282, "facup": 25283, "xham": 25284, "unwanted": 25285, "imports": 25286, "carnegie": 25287, "fansign": 25288, "mou": 25289, "ralph": 25290, "destroyer": 25291, "swing": 25292, "trekking": 25293, "ciliation": 25294, "pitbull": 25295, "gaps": 25296, "howell": 25297, "definitive": 25298, "mcle": 25299, "fps": 25300, "etz": 25301, "bolly": 25302, "lynn": 25303, "gano": 25304, "ature": 25305, "fursuit": 25306, "coil": 25307, "nav": 25308, "butts": 25309, "trojans": 25310, "eure": 25311, "enko": 25312, "schumer": 25313, "horrific": 25314, "installment": 25315, "brb": 25316, "suburbs": 25317, "abel": 25318, "vir": 25319, "desh": 25320, "cunningham": 25321, "ðŁIJ»": 25322, "spann": 25323, "schwe": 25324, "kemp": 25325, "tru": 25326, "stealth": 25327, "ques": 25328, "lew": 25329, "delights": 25330, "koch": 25331, "humili": 25332, "criti": 25333, "ilt": 25334, "spells": 25335, "miley": 25336, "caric": 25337, "ðŁį´": 25338, "lcfc": 25339, "substitute": 25340, "oung": 25341, "?!!": 25342, "affir": 25343, "predictable": 25344, "classof": 25345, "err": 25346, "cypress": 25347, "chandra": 25348, "ageing": 25349, "____": 25350, "therland": 25351, "doncaster": 25352, "elin": 25353, "yoshi": 25354, "sailors": 25355, "harris": 25356, "joanna": 25357, "nigerians": 25358, "hers": 25359, "plague": 25360, "procra": 25361, "kno": 25362, "canton": 25363, "busines": 25364, "unh": 25365, "prakash": 25366, "cin": 25367, "bowen": 25368, "coating": 25369, "mals": 25370, "begging": 25371, "smithson": 25372, "pontiac": 25373, "spies": 25374, "damian": 25375, "pline": 25376, "undant": 25377, "alta": 25378, "oness": 25379, "shameless": 25380, "daq": 25381, "bbm": 25382, "wales": 25383, "stampede": 25384, "serum": 25385, "ÙĨ": 25386, "catalyst": 25387, "xn": 25388, "absc": 25389, "freezer": 25390, "chun": 25391, "arios": 25392, "mccre": 25393, "forehead": 25394, "hears": 25395, "damascus": 25396, "tacoma": 25397, "arduino": 25398, "encounters": 25399, "stanton": 25400, "lgb": 25401, "abas": 25402, "\"..": 25403, "kete": 25404, "dracula": 25405, "elem": 25406, "gne": 25407, "zeppelin": 25408, "labrador": 25409, "pulp": 25410, "optional": 25411, "orn": 25412, "russians": 25413, "sanitation": 25414, "hilary": 25415, "etsymntt": 25416, "penalties": 25417, "aust": 25418, "igans": 25419, "olympian": 25420, "medicaid": 25421, "versace": 25422, "vape": 25423, "restra": 25424, "peep": 25425, "sexiest": 25426, "stalls": 25427, "dile": 25428, "thea": 25429, "punjabi": 25430, "puppy": 25431, "tuesdaymotivation": 25432, "ðŁĵļ": 25433, "theflash": 25434, "rocket": 25435, "modest": 25436, "chihuahu": 25437, "onna": 25438, "ksa": 25439, "hurdles": 25440, "cave": 25441, "failures": 25442, "split": 25443, "boho": 25444, "gurl": 25445, "disappoint": 25446, "howard": 25447, "nugget": 25448, "franz": 25449, "stalert": 25450, "kazakh": 25451, "forgetting": 25452, "schri": 25453, "agate": 25454, "amat": 25455, "everett": 25456, "duet": 25457, "veterinary": 25458, "julian": 25459, "chills": 25460, "brave": 25461, "ghostbusters": 25462, "lando": 25463, "greets": 25464, "profitable": 25465, "dé": 25466, "tir": 25467, "zee": 25468, "omen": 25469, "pdx": 25470, "grayson": 25471, "hari": 25472, "fixes": 25473, "stabbing": 25474, "swimmer": 25475, "symbols": 25476, "compliments": 25477, "pose": 25478, "functioning": 25479, "thnx": 25480, "gir": 25481, "corporations": 25482, "barlow": 25483, "loe": 25484, "offseason": 25485, "distinctive": 25486, "marvelous": 25487, "nikon": 25488, "enrique": 25489, "kyu": 25490, "jaws": 25491, "amoto": 25492, "lombar": 25493, "travelblogger": 25494, "fah": 25495, "ourism": 25496, "tristan": 25497, "soe": 25498, "cease": 25499, "ðŁıħ": 25500, "zac": 25501, "mckenzie": 25502, "taxpayers": 25503, "swimsuit": 25504, "blo": 25505, "lesley": 25506, "kansas": 25507, "wks": 25508, "kiel": 25509, "provoking": 25510, "myles": 25511, "string": 25512, "kangaroo": 25513, "galactic": 25514, "fifth": 25515, "ske": 25516, "weir": 25517, "llis": 25518, "matory": 25519, "ðŁĩ¿": 25520, "unci": 25521, "reproductive": 25522, "rooting": 25523, "tides": 25524, "gadget": 25525, "..........": 25526, "alexander": 25527, "bowler": 25528, "screw": 25529, "apolog": 25530, "erika": 25531, "walters": 25532, "shetty": 25533, "lane": 25534, "banter": 25535, "asant": 25536, "meso": 25537, "vain": 25538, "\"\"\"": 25539, "usi": 25540, "ferdin": 25541, "accomplish": 25542, "mansfield": 25543, "bombar": 25544, "collaborating": 25545, "clap": 25546, "iture": 25547, "sda": 25548, "smoky": 25549, "nak": 25550, "imperson": 25551, "carla": 25552, "comra": 25553, "burgl": 25554, "loco": 25555, "ties": 25556, "inhi": 25557, "tracey": 25558, "seis": 25559, "disser": 25560, "rrrr": 25561, "dray": 25562, "protect": 25563, "corona": 25564, "hunger": 25565, "cken": 25566, "celi": 25567, "troubled": 25568, "predators": 25569, "fictional": 25570, "shaved": 25571, "richest": 25572, "metaboli": 25573, "fulham": 25574, "grooming": 25575, "monochrome": 25576, "wasting": 25577, "asco": 25578, "aste": 25579, "tista": 25580, "remedies": 25581, "ungsoo": 25582, "southend": 25583, "permanently": 25584, "bumble": 25585, "procrastin": 25586, "identical": 25587, "practically": 25588, "mascul": 25589, "suke": 25590, "assured": 25591, "valerie": 25592, "deviant": 25593, "grizzlies": 25594, "thier": 25595, "pura": 25596, "nepal": 25597, "notts": 25598, "bilateral": 25599, "spoil": 25600, "carmel": 25601, "cinematic": 25602, "phl": 25603, "nifty": 25604, "mao": 25605, "hypocri": 25606, "laser": 25607, "pantry": 25608, "mathematical": 25609, "elisa": 25610, "coordination": 25611, "belmont": 25612, "ait": 25613, "radiant": 25614, "boiler": 25615, "mang": 25616, "fag": 25617, "crc": 25618, "hams": 25619, "brin": 25620, "â¬ĩï¸ı": 25621, "familia": 25622, "âĿ£": 25623, "saber": 25624, "rupert": 25625, "ggan": 25626, "ritz": 25627, "mich": 25628, "salford": 25629, "levi": 25630, "gral": 25631, "ðŁĴ¤": 25632, "nino": 25633, "ced": 25634, "businessman": 25635, "ultr": 25636, "simply": 25637, "compression": 25638, "pains": 25639, "halt": 25640, "ë°©íĥĦ": 25641, "landscaping": 25642, "nf": 25643, "crooked": 25644, "erd": 25645, "ittin": 25646, "ddleston": 25647, "surpassed": 25648, "inoa": 25649, "dag": 25650, "blen": 25651, "extending": 25652, "ating": 25653, "algae": 25654, "baller": 25655, "umar": 25656, "snooker": 25657, "collu": 25658, "flown": 25659, "thub": 25660, "ridiculously": 25661, "kish": 25662, "ople": 25663, "dire": 25664, "asser": 25665, "aristo": 25666, "sciss": 25667, "hating": 25668, "trouble": 25669, "sylvia": 25670, "succul": 25671, "plots": 25672, "sincerely": 25673, "aler": 25674, "laureate": 25675, "brack": 25676, "attn": 25677, "rifles": 25678, "meto": 25679, "collectible": 25680, "cuomo": 25681, "contestant": 25682, "consistency": 25683, "antz": 25684, "ranges": 25685, "abigail": 25686, "deb": 25687, "minister": 25688, "growers": 25689, "anoo": 25690, "hoover": 25691, "dreamer": 25692, "nucle": 25693, "research": 25694, "miy": 25695, "shahid": 25696, "mav": 25697, "dhoni": 25698, "cini": 25699, "doj": 25700, "hindus": 25701, "partying": 25702, "dali": 25703, "alonso": 25704, "informal": 25705, "clarkson": 25706, "itton": 25707, "kian": 25708, "cityo": 25709, "mori": 25710, "lasted": 25711, "aspen": 25712, "library": 25713, "suspici": 25714, "quat": 25715, "denial": 25716, "folder": 25717, "chori": 25718, "sweeping": 25719, "enix": 25720, "ðŁįĤ": 25721, "ØŃ": 25722, "nascar": 25723, "handmadehour": 25724, "moul": 25725, "heatwave": 25726, "emer": 25727, "examine": 25728, "ibn": 25729, "grind": 25730, "pov": 25731, "tionist": 25732, "mbo": 25733, "sheila": 25734, "integrate": 25735, "omes": 25736, "takeaway": 25737, "cerv": 25738, "connie": 25739, "ticket": 25740, "celed": 25741, "bien": 25742, "visually": 25743, "madagascar": 25744, "sorry": 25745, "gui": 25746, "parkrun": 25747, "traits": 25748, "labe": 25749, "poisoning": 25750, "à¥Ģ": 25751, "viable": 25752, "bohemian": 25753, "dentistry": 25754, "bados": 25755, "sprouts": 25756, "masked": 25757, "teddy": 25758, "ðŁĺ·": 25759, "saf": 25760, "saas": 25761, "jiang": 25762, "tight": 25763, "speaker": 25764, "withdrawal": 25765, "bcn": 25766, "assigned": 25767, "classrooms": 25768, "fleming": 25769, "ðŁĴ«": 25770, "supergirl": 25771, "totals": 25772, "tabletop": 25773, "ebooks": 25774, "horizontal": 25775, "craz": 25776, "flush": 25777, "jard": 25778, "cdc": 25779, "erson": 25780, "ãħł": 25781, "greenwood": 25782, "nih": 25783, "cox": 25784, "ada": 25785, "litre": 25786, "going": 25787, "vicky": 25788, "curved": 25789, "louie": 25790, "grains": 25791, "hye": 25792, "longe": 25793, "remedy": 25794, "trainee": 25795, "sanjay": 25796, "superstars": 25797, "maser": 25798, "manu": 25799, "sage": 25800, "whl": 25801, "ðŁĺĤðŁĺŃ": 25802, "ðŁijįðŁı»": 25803, "msd": 25804, "enz": 25805, "rabhu": 25806, "joo": 25807, "ghu": 25808, "acer": 25809, "epo": 25810, "resurrection": 25811, "justicefor": 25812, "blended": 25813, "moda": 25814, "avalanche": 25815, "francesco": 25816, "respective": 25817, "gs": 25818, "yeast": 25819, "welch": 25820, "devotion": 25821, "getin": 25822, "atheism": 25823, "amic": 25824, "carolyn": 25825, "loc": 25826, "ldnont": 25827, "avec": 25828, "usda": 25829, "legged": 25830, "bravery": 25831, "blower": 25832, "cowboy": 25833, "heh": 25834, "stible": 25835, "buffal": 25836, "channel": 25837, "runchat": 25838, "âĺķï¸ı": 25839, "ideology": 25840, "bestseller": 25841, "yoo": 25842, "peanu": 25843, "bonne": 25844, "felic": 25845, "edison": 25846, "fractu": 25847, "narendra": 25848, "ppets": 25849, "seymour": 25850, "riviera": 25851, "hector": 25852, "necessarily": 25853, "bianca": 25854, "societies": 25855, "thebest": 25856, "wg": 25857, "sentences": 25858, "wink": 25859, "vaccines": 25860, "palooza": 25861, "jamming": 25862, "asf": 25863, "mpus": 25864, "agreements": 25865, "eck": 25866, "bac": 25867, "honore": 25868, "compul": 25869, "wildcat": 25870, "imposed": 25871, "yoga": 25872, "hudson": 25873, "canceled": 25874, "lich": 25875, "fuzzy": 25876, "esque": 25877, "chuk": 25878, "wvu": 25879, "sek": 25880, "flipping": 25881, "rhon": 25882, "wished": 25883, "wha": 25884, "capability": 25885, "lenovo": 25886, "ìĨĮëħĦëĭ": 25887, "vivo": 25888, "tvd": 25889, "nora": 25890, "silk": 25891, "pasadena": 25892, "yosemite": 25893, "valuation": 25894, "clocks": 25895, "uber": 25896, "mrc": 25897, "darkest": 25898, "aubre": 25899, "sso": 25900, "belly": 25901, "wrestlers": 25902, "killin": 25903, "louder": 25904, "buckley": 25905, "geel": 25906, "adon": 25907, "uns": 25908, "appealing": 25909, "ðŁij¯": 25910, "semitism": 25911, "listens": 25912, "fitz": 25913, "ãĥ³ãĥ": 25914, "nylon": 25915, "arty": 25916, "seemingly": 25917, "hala": 25918, "suited": 25919, "ety": 25920, "sheds": 25921, "muffins": 25922, "apric": 25923, "uments": 25924, "uta": 25925, "jammu": 25926, "chelseafc": 25927, "starz": 25928, "yoko": 25929, "root": 25930, "cleansing": 25931, "diar": 25932, "pioneering": 25933, "iheartradio": 25934, "digiti": 25935, "findyour": 25936, "cano": 25937, "ðŁĴİ": 25938, "zol": 25939, "spacecraft": 25940, "sixers": 25941, "moisturi": 25942, "bile": 25943, "tists": 25944, "horton": 25945, "ranging": 25946, "columbi": 25947, "meteoro": 25948, "sentiment": 25949, "epl": 25950, "footh": 25951, "textbook": 25952, "drainage": 25953, "rly": 25954, "scue": 25955, "imrankhan": 25956, "ðŁĴ¸": 25957, "margarita": 25958, "eddy": 25959, "predicts": 25960, "gamergate": 25961, "advise": 25962, "growthhacking": 25963, "loveyou": 25964, "ugand": 25965, "vf": 25966, "benghazi": 25967, "slater": 25968, "newor": 25969, "chel": 25970, "independenceday": 25971, "pnp": 25972, "cullen": 25973, "hoodies": 25974, "numbered": 25975, "britt": 25976, "tsa": 25977, "kltu": 25978, "sages": 25979, "momo": 25980, "oneplus": 25981, "coll": 25982, "guts": 25983, "wta": 25984, "mesmeri": 25985, "enhancing": 25986, "chiroprac": 25987, "jis": 25988, "teenagers": 25989, "mone": 25990, "constellation": 25991, "sweepstakes": 25992, "eze": 25993, "slovakia": 25994, "laye": 25995, "pearce": 25996, "waver": 25997, "pogba": 25998, "kron": 25999, "surgeons": 26000, "marx": 26001, "tid": 26002, "gga": 26003, "descend": 26004, "pours": 26005, "uprising": 26006, "walla": 26007, "sabbath": 26008, "bachelore": 26009, "mackin": 26010, "kam": 26011, "peterborough": 26012, "hora": 26013, "ðŁĮŁðŁĮŁ": 26014, "thinkbig": 26015, "rj": 26016, "hydrau": 26017, "spal": 26018, "universit": 26019, "ðŁıī": 26020, "mailonline": 26021, "leagueof": 26022, "tenants": 26023, "wally": 26024, "lance": 26025, "heavens": 26026, "ddr": 26027, "bolts": 26028, "amir": 26029, "iphone": 26030, "cigar": 26031, "endu": 26032, "rei": 26033, "elabor": 26034, "ringing": 26035, "johnson": 26036, "characteristics": 26037, "saloon": 26038, "algorithms": 26039, "talkin": 26040, "mtn": 26041, "dive": 26042, "regionals": 26043, "ffice": 26044, "hati": 26045, "deviantart": 26046, "sotto": 26047, "shiro": 26048, "lama": 26049, "kwe": 26050, "faded": 26051, "porting": 26052, "tummy": 26053, "estates": 26054, "buenos": 26055, "ðŁ¦ģ": 26056, "believer": 26057, "penetr": 26058, "darn": 26059, "spite": 26060, "canopy": 26061, "fashioni": 26062, "tilla": 26063, "petals": 26064, "elijah": 26065, "brawl": 26066, "martyr": 26067, "ë°©íĥĦìĨĮëħĦëĭ": 26068, "midtown": 26069, "erich": 26070, "dapper": 26071, "smtown": 26072, "megam": 26073, "www": 26074, "lele": 26075, "ons": 26076, "catfish": 26077, "firth": 26078, "fossilfriday": 26079, "ballpark": 26080, "thaw": 26081, "potent": 26082, "illie": 26083, "creep": 26084, "carp": 26085, "soap": 26086, "gundam": 26087, "infec": 26088, "yyyyy": 26089, "न": 26090, "zag": 26091, "ritt": 26092, "calculator": 26093, "boca": 26094, "oko": 26095, "toad": 26096, "threaten": 26097, "refined": 26098, "olympic": 26099, "accomplishment": 26100, "bacterial": 26101, "aji": 26102, "tatum": 26103, "feliz": 26104, "sheed": 26105, "jat": 26106, "thic": 26107, "jamal": 26108, "ðĿĺ": 26109, "lina": 26110, "ðŁIJ¯": 26111, "joking": 26112, "yotpo": 26113, "pinch": 26114, "akron": 26115, "herb": 26116, "motivation": 26117, "lia": 26118, "hostage": 26119, "creek": 26120, "gamble": 26121, "russell": 26122, "patti": 26123, "fotos": 26124, "cpc": 26125, "broken": 26126, "backthe": 26127, "clays": 26128, "umm": 26129, "stockton": 26130, "maternal": 26131, "ür": 26132, "lakel": 26133, "century": 26134, "bek": 26135, "infected": 26136, "ม": 26137, "smackdown": 26138, "manned": 26139, "tahoe": 26140, "smes": 26141, "basa": 26142, "sula": 26143, "augusta": 26144, ".*": 26145, "rohingya": 26146, "greed": 26147, "counselor": 26148, "silhouette": 26149, "gravit": 26150, "clause": 26151, "'-": 26152, "bobc": 26153, "occasions": 26154, "nowadays": 26155, "dictat": 26156, "beard": 26157, "nally": 26158, "brightest": 26159, "kabul": 26160, "incindia": 26161, "dhanush": 26162, "archaeological": 26163, "cheape": 26164, "mizzou": 26165, "dhi": 26166, "ovski": 26167, "baxter": 26168, "assemble": 26169, "â": 26170, "gigi": 26171, "acam": 26172, "wisely": 26173, "hazard": 26174, "northampton": 26175, "âľĪï¸ı": 26176, "meth": 26177, "blasting": 26178, "reunite": 26179, "mulus": 26180, "alizes": 26181, "tread": 26182, "mila": 26183, "edward": 26184, "kova": 26185, "pesto": 26186, "ðŁij¶": 26187, "vitz": 26188, "hydraulic": 26189, "refurbished": 26190, "motel": 26191, "isabella": 26192, "homme": 26193, "severance": 26194, "uphol": 26195, "miserable": 26196, "fari": 26197, "latter": 26198, "efer": 26199, "crackers": 26200, "esl": 26201, "acio": 26202, "yyj": 26203, "inan": 26204, "ecb": 26205, "zind": 26206, "panas": 26207, "trucking": 26208, "reed": 26209, "shaker": 26210, "burgess": 26211, "empire": 26212, "agnes": 26213, "nington": 26214, "artworks": 26215, "frs": 26216, "tile": 26217, "biome": 26218, "eun": 26219, "chong": 26220, "americana": 26221, "godfather": 26222, "goblin": 26223, "ishi": 26224, "!).": 26225, "tempted": 26226, "genomics": 26227, "mandate": 26228, "cky": 26229, "ðŁĴĻðŁĴĽ": 26230, "somali": 26231, "brandy": 26232, "inven": 26233, "spokesperson": 26234, "pcb": 26235, "yuan": 26236, "hg": 26237, "faz": 26238, "starwars": 26239, "rowan": 26240, "bluegrass": 26241, "dong": 26242, "dday": 26243, "trinidad": 26244, "erton": 26245, "banning": 26246, "retention": 26247, "cured": 26248, "toberfest": 26249, "reset": 26250, "weis": 26251, "detached": 26252, "behindthescenes": 26253, "immunity": 26254, "pha": 26255, "bray": 26256, "ðŁij½": 26257, "rancho": 26258, "ramsay": 26259, "estonia": 26260, "ndtv": 26261, "].": 26262, "cabaret": 26263, "taro": 26264, "dv": 26265, "showcases": 26266, "plum": 26267, "ðŁij¸": 26268, "sonoma": 26269, "prepa": 26270, "memorab": 26271, "estu": 26272, "driveway": 26273, "ules": 26274, "magnus": 26275, "xr": 26276, "nnn": 26277, "muchas": 26278, "enge": 26279, "streamed": 26280, "forestry": 26281, "audiobook": 26282, "troy": 26283, "reckless": 26284, "kilom": 26285, "ruler": 26286, "rak": 26287, "procession": 26288, "ions": 26289, "poole": 26290, "noctur": 26291, "whs": 26292, "farmhouse": 26293, "pera": 26294, "parme": 26295, "hypocrisy": 26296, "sics": 26297, "vant": 26298, "cask": 26299, "holistic": 26300, "aust": 26301, "п": 26302, "indo": 26303, "ðŁij©âĢį": 26304, "diso": 26305, "dispatch": 26306, "olsen": 26307, "makeit": 26308, "ennis": 26309, "centre": 26310, "arrange": 26311, "ðŁĮ¼": 26312, "salted": 26313, "easiest": 26314, "fate": 26315, "regatta": 26316, "mozz": 26317, "acan": 26318, "sini": 26319, "gically": 26320, "chops": 26321, "chicken": 26322, "workin": 26323, "hagg": 26324, "involve": 26325, "weeds": 26326, "bookday": 26327, "wakeup": 26328, "kyr": 26329, "michelin": 26330, "fuss": 26331, "rejuven": 26332, "vacancies": 26333, "incarcer": 26334, "mst": 26335, "scents": 26336, "sovereign": 26337, "kicker": 26338, "à§": 26339, "bod": 26340, "âĢĶ>": 26341, "sah": 26342, "mobil": 26343, "shropshire": 26344, "ophone": 26345, "dresser": 26346, "missuni": 26347, "hepburn": 26348, "imo": 26349, "foliage": 26350, "diagnostic": 26351, "assan": 26352, "cycling": 26353, "guilt": 26354, "csa": 26355, "puertorico": 26356, "winelover": 26357, "wakefield": 26358, "doggy": 26359, "khe": 26360, "papp": 26361, "cog": 26362, "allot": 26363, "cuck": 26364, "poetic": 26365, "mio": 26366, "revit": 26367, "magician": 26368, "ç¥": 26369, "antenna": 26370, "westwood": 26371, "mberg": 26372, "luxe": 26373, "oatmeal": 26374, "ج": 26375, "teat": 26376, "ffee": 26377, "searches": 26378, "lly": 26379, "pluto": 26380, "elon": 26381, "lettering": 26382, "innocence": 26383, "fai": 26384, "annon": 26385, "telangana": 26386, "mait": 26387, "neural": 26388, "canni": 26389, "aroma": 26390, "astor": 26391, "fex": 26392, "cocac": 26393, "monetary": 26394, "fent": 26395, "unsure": 26396, "'@": 26397, "indirec": 26398, "tehran": 26399, "isolation": 26400, "libs": 26401, "makeup": 26402, "mercedes": 26403, "ffy": 26404, "hetero": 26405, "deo": 26406, "scom": 26407, "cursed": 26408, "veteransday": 26409, "frankenstein": 26410, "shrews": 26411, "deco": 26412, "geese": 26413, "leftover": 26414, "hadid": 26415, "variable": 26416, "academics": 26417, "carolin": 26418, "undergoing": 26419, "variation": 26420, "nah": 26421, "ssier": 26422, "gamersunite": 26423, "pursuing": 26424, "emerged": 26425, "llers": 26426, "controlling": 26427, "roaring": 26428, "meteor": 26429, "volt": 26430, "dawgs": 26431, "beaver": 26432, "islife": 26433, "bathrooms": 26434, "acional": 26435, "prevent": 26436, "lakedistrict": 26437, "inals": 26438, "yani": 26439, "grabbing": 26440, "sacks": 26441, "lez": 26442, "sway": 26443, "kool": 26444, "times": 26445, "klopp": 26446, "lade": 26447, "concord": 26448, "resulted": 26449, "revive": 26450, "reconciliation": 26451, "oland": 26452, "azz": 26453, "giro": 26454, "mandarin": 26455, "deen": 26456, "nutritional": 26457, "iscoming": 26458, "vani": 26459, "awwww": 26460, "derived": 26461, "loveyour": 26462, "stopthe": 26463, "shouting": 26464, "novak": 26465, "ðŁĻĮðŁı¾": 26466, "loaf": 26467, "displaying": 26468, "sundaywith": 26469, "maguire": 26470, "cheri": 26471, "ðŁıŁ": 26472, "rematch": 26473, "quic": 26474, "Ú©": 26475, "yin": 26476, "ðŁĺ¹": 26477, "ilive": 26478, "zip": 26479, "ourke": 26480, "downloads": 26481, "swat": 26482, "mississ": 26483, "carers": 26484, "tment": 26485, "property": 26486, "hahahahahaha": 26487, "gibbs": 26488, "surrey": 26489, "arise": 26490, "ticism": 26491, "stia": 26492, "irling": 26493, "frog": 26494, "cose": 26495, "bassist": 26496, "foreig": 26497, "leau": 26498, "pillows": 26499, "holla": 26500, "elie": 26501, "disclosure": 26502, "peanuts": 26503, "intech": 26504, "wwc": 26505, "plunge": 26506, "triumph": 26507, "cori": 26508, "slippers": 26509, "ðŁĻıðŁĻı": 26510, "neutrality": 26511, "mare": 26512, "hairy": 26513, "gangster": 26514, "humming": 26515, "custard": 26516, "merlin": 26517, "alea": 26518, "sby": 26519, "damp": 26520, "mohan": 26521, "verbal": 26522, "jst": 26523, "gutted": 26524, "bjor": 26525, "unfinished": 26526, "ðŁĩ¯ðŁĩµ": 26527, "unhappy": 26528, "âļ«ï¸ı": 26529, "bypass": 26530, "atsu": 26531, "fischer": 26532, "sav": 26533, "africans": 26534, "reuse": 26535, "midway": 26536, "demolished": 26537, "gerrard": 26538, "hercules": 26539, "ÄŁ": 26540, "medicines": 26541, "clicking": 26542, "surround": 26543, "joong": 26544, "waving": 26545, "tribes": 26546, "wetlands": 26547, "officiel": 26548, "arguing": 26549, "lle": 26550, "dova": 26551, "suzy": 26552, "clubhouse": 26553, "negro": 26554, "obtain": 26555, "gao": 26556, "glance": 26557, "assist": 26558, "chos": 26559, "ãĤ¢": 26560, "âĺķ": 26561, "adrid": 26562, "occurs": 26563, "stans": 26564, "pardon": 26565, "liveli": 26566, "employed": 26567, "revisit": 26568, "ffxiv": 26569, "bble": 26570, "nearing": 26571, "miner": 26572, "ðŁĺ¹": 26573, "giovanni": 26574, "upto": 26575, "marvell": 26576, "marse": 26577, "towels": 26578, "cbn": 26579, "engineered": 26580, "yelling": 26581, "spartan": 26582, "sians": 26583, "ðŁĻĮðŁı¼": 26584, "sev": 26585, "coyote": 26586, "stadi": 26587, "tcm": 26588, "appen": 26589, "shenanigans": 26590, "openaccess": 26591, "soaked": 26592, "masqu": 26593, "levine": 26594, "strokes": 26595, "lk": 26596, "apartheid": 26597, "hiphop": 26598, "chardon": 26599, "maymay": 26600, "haasan": 26601, "stripped": 26602, "fro": 26603, "scription": 26604, "fton": 26605, "hf": 26606, "prisons": 26607, "marshal": 26608, "ķãĤ": 26609, "ancho": 26610, "compromise": 26611, "classification": 26612, "buzzfeed": 26613, "bbloggers": 26614, "deserving": 26615, ")/": 26616, "sway": 26617, "obo": 26618, "campers": 26619, "podernfamily": 26620, "poured": 26621, "brie": 26622, "squirrels": 26623, "seize": 26624, ":#": 26625, "lek": 26626, "timb": 26627, "stacy": 26628, "nasdaq": 26629, "repeatedly": 26630, "brat": 26631, "mighty": 26632, "competitor": 26633, "mahone": 26634, "desi": 26635, "oke": 26636, "bmw": 26637, "shie": 26638, "fcb": 26639, "cheapest": 26640, "minimalist": 26641, "paramount": 26642, "nate": 26643, "haras": 26644, "insanity": 26645, "lateral": 26646, "mentality": 26647, "mozam": 26648, "tapped": 26649, "yadav": 26650, "usp": 26651, "bway": 26652, "theod": 26653, "bilt": 26654, "raids": 26655, "empress": 26656, "adapted": 26657, "patron": 26658, "nutshell": 26659, "agra": 26660, "beaded": 26661, "sundaywithmarsha": 26662, "viking": 26663, "proceed": 26664, "maintained": 26665, "thinkbigsundaywithmarsha": 26666, "snes": 26667, "musica": 26668, "tower": 26669, "chab": 26670, "bok": 26671, "smt": 26672, "insult": 26673, "harvesting": 26674, "window": 26675, "ruther": 26676, "beige": 26677, "decal": 26678, "indicate": 26679, "mailing": 26680, "rift": 26681, "pole": 26682, "anderson": 26683, "choral": 26684, "spride": 26685, "lili": 26686, "evelyn": 26687, "imrankhanpti": 26688, "....\"": 26689, "kered": 26690, "undp": 26691, "waterfalls": 26692, "sears": 26693, "lemans": 26694, "worldseries": 26695, "riel": 26696, "anie": 26697, "appar": 26698, "scorers": 26699, "lamp": 26700, "athan": 26701, "physicians": 26702, "quinoa": 26703, "refusing": 26704, "vuitton": 26705, "unleash": 26706, "sla": 26707, "pati": 26708, "shouts": 26709, "intentions": 26710, "foamed": 26711, "european": 26712, "neighborhoods": 26713, "meer": 26714, "manson": 26715, "duh": 26716, "brat": 26717, "cones": 26718, "bowl": 26719, "kazakhstan": 26720, "ि": 26721, "inappropriate": 26722, "delhi": 26723, "ketchup": 26724, "fulton": 26725, "sys": 26726, "consult": 26727, "garfield": 26728, "togo": 26729, "fml": 26730, "fled": 26731, "bds": 26732, "facilitate": 26733, "reebok": 26734, "selfie": 26735, "elevate": 26736, "activate": 26737, "bible": 26738, "cawx": 26739, "bys": 26740, "camille": 26741, "syou": 26742, "skool": 26743, "hert": 26744, "wbc": 26745, "pledges": 26746, "recorder": 26747, "posh": 26748, "acre": 26749, "soaking": 26750, "matil": 26751, "vsco": 26752, "shootings": 26753, "plar": 26754, "econ": 26755, "ðŁĻĮðŁı»": 26756, "rashid": 26757, "ubi": 26758, "ðŁ¤¤": 26759, "swinging": 26760, "wipe": 26761, "raptor": 26762, "msu": 26763, "musicvideo": 26764, "durham": 26765, "attic": 26766, "aparty": 26767, "fetus": 26768, "activation": 26769, "aaz": 26770, "motivate": 26771, "ðŁĴķðŁĴķðŁĴķ": 26772, "jal": 26773, "म": 26774, "agon": 26775, "scheer": 26776, "stalker": 26777, "foster": 26778, "azzo": 26779, "telegram": 26780, "vigor": 26781, "slaugh": 26782, "screenshots": 26783, "entrepreneu": 26784, "kristin": 26785, "intention": 26786, "chilli": 26787, "fraction": 26788, "dona": 26789, "gea": 26790, "tcu": 26791, "site": 26792, "lak": 26793, "emil": 26794, "dnt": 26795, "boro": 26796, "wilkinson": 26797, "recu": 26798, "atoday": 26799, "tanya": 26800, "blanco": 26801, "cdn": 26802, "brilliantly": 26803, "gcc": 26804, "acc": 26805, "evacuated": 26806, "therine": 26807, "denny": 26808, "caitlin": 26809, "shepard": 26810, "pouch": 26811, "handheld": 26812, "southeastern": 26813, "haa": 26814, "ô": 26815, "resolutions": 26816, "ledger": 26817, "srin": 26818, "rar": 26819, "shattered": 26820, "chimney": 26821, "imwith": 26822, "meteor": 26823, "handled": 26824, "rake": 26825, "townsend": 26826, "enhan": 26827, "shipy": 26828, "duct": 26829, "twx": 26830, "inflammatory": 26831, "warhammer": 26832, "theatrical": 26833, "gros": 26834, "skar": 26835, "scotty": 26836, "niel": 26837, "tito": 26838, "tini": 26839, "connection": 26840, "_.": 26841, "goldenglobes": 26842, "shaq": 26843, "ðŁı³ï¸ı": 26844, "hallway": 26845, "fronts": 26846, "effectiveness": 26847, "glaston": 26848, "dhs": 26849, "expi": 26850, "toh": 26851, "cpl": 26852, "scs": 26853, "reo": 26854, "hag": 26855, "resemblance": 26856, "horan": 26857, "abusive": 26858, "quer": 26859, "virtue": 26860, "cholester": 26861, "aq": 26862, "shane": 26863, "mce": 26864, "carriers": 26865, "distress": 26866, "rewind": 26867, "¡": 26868, "voodoo": 26869, "intact": 26870, "anno": 26871, "ðŁĺ¤": 26872, "piled": 26873, "adia": 26874, "ãĥ³": 26875, "enow": 26876, "digs": 26877, "lightly": 26878, "goofy": 26879, "turbine": 26880, "governors": 26881, "conte": 26882, "reopen": 26883, "pah": 26884, "ive": 26885, "crafting": 26886, "sweeps": 26887, "jodi": 26888, "ande": 26889, "zucker": 26890, "kawaii": 26891, "oko": 26892, "vai": 26893, "outline": 26894, "kristi": 26895, "tsn": 26896, "inspo": 26897, "quint": 26898, "filthy": 26899, "lynne": 26900, "listeners": 26901, "departing": 26902, "ord": 26903, "tweed": 26904, ",&": 26905, "alek": 26906, "selfish": 26907, "norther": 26908, "recognizes": 26909, "ips": 26910, "bes": 26911, "aed": 26912, "wills": 26913, "peat": 26914, "surroundings": 26915, "monuments": 26916, "aisle": 26917, "becker": 26918, "lav": 26919, "quantity": 26920, "vah": 26921, "helicopters": 26922, "tucked": 26923, "alvarez": 26924, "shape": 26925, "obey": 26926, "additi": 26927, "roadside": 26928, "mite": 26929, "blers": 26930, "epage": 26931, "jau": 26932, "ignorant": 26933, "bins": 26934, "lulu": 26935, "xo": 26936, "cfo": 26937, "eeeee": 26938, "apprenticeship": 26939, "sheffiel": 26940, "toi": 26941, "hok": 26942, "fakenews": 26943, "deploy": 26944, "aidan": 26945, "huskers": 26946, "ãĢİ": 26947, "westbrook": 26948, "mister": 26949, "configur": 26950, "carr": 26951, "fica": 26952, "proceedings": 26953, "haw": 26954, "steak": 26955, "murderer": 26956, "payday": 26957, "ajo": 26958, "pvc": 26959, "donates": 26960, "biaf": 26961, "nomnom": 26962, "beit": 26963, "kali": 26964, "xrp": 26965, "ahmedabad": 26966, "semic": 26967, "chey": 26968, "xtra": 26969, "antwer": 26970, "headlining": 26971, "squares": 26972, "rounded": 26973, "fluore": 26974, "bold": 26975, "disasters": 26976, "amoo": 26977, "generic": 26978, "cranes": 26979, "briefly": 26980, "gig": 26981, "austerity": 26982, "anticipation": 26983, "forti": 26984, "treasurer": 26985, "canny": 26986, "cecil": 26987, "detected": 26988, "checklist": 26989, "ว": 26990, "pamela": 26991, "barbados": 26992, "anfield": 26993, "hearty": 26994, "txlege": 26995, "perenni": 26996, "arrog": 26997, "ingram": 26998, "âĹı": 26999, "tyne": 27000, "spoon": 27001, "ration": 27002, "amba": 27003, "mbe": 27004, "camel": 27005, "hhs": 27006, "yorkshire": 27007, "reflective": 27008, "freaks": 27009, "tok": 27010, "judo": 27011, "particles": 27012, "dubs": 27013, "banjo": 27014, "accreditation": 27015, "proverbs": 27016, "overdose": 27017, "integral": 27018, "guang": 27019, "mcs": 27020, "supercar": 27021, "afb": 27022, "alvin": 27023, "ails": 27024, "xtre": 27025, "staging": 27026, "twent": 27027, "rabbits": 27028, "maro": 27029, "instem": 27030, "doll": 27031, "cray": 27032, "santana": 27033, "bleach": 27034, "minions": 27035, "cheap": 27036, "mant": 27037, "divers": 27038, "catalonia": 27039, "lois": 27040, "matri": 27041, "cougar": 27042, "kayak": 27043, "egre": 27044, "pso": 27045, "aia": 27046, "å®": 27047, "charlton": 27048, "tracked": 27049, "scari": 27050, "pett": 27051, "fwd": 27052, "xin": 27053, "gravel": 27054, "bric": 27055, "biggboss": 27056, "arden": 27057, "hugging": 27058, "palms": 27059, "stv": 27060, "limb": 27061, "themovie": 27062, "handicap": 27063, "rime": 27064, "zai": 27065, "stub": 27066, "india": 27067, "lithuania": 27068, "rhyth": 27069, "pita": 27070, "macedonia": 27071, "highered": 27072, "bridget": 27073, "schwarz": 27074, "skelet": 27075, "hikes": 27076, "antarctic": 27077, "cps": 27078, "mashup": 27079, "а": 27080, "nell": 27081, "chandra": 27082, "heir": 27083, "anus": 27084, "sheridan": 27085, "mimi": 27086, "museu": 27087, "becca": 27088, "anir": 27089, "barrie": 27090, "diocese": 27091, "comparable": 27092, "ðŁı³ï¸ıâĢį": 27093, "yukon": 27094, "mep": 27095, "hormon": 27096, "meric": 27097, "alf": 27098, "conquered": 27099, "christchurch": 27100, "ðŁĴĻðŁĴĻ": 27101, "hazardous": 27102, "pooh": 27103, "conting": 27104, "retrospective": 27105, "parame": 27106, "nair": 27107, "consor": 27108, "hotra": 27109, "astonishing": 27110, "caterpillar": 27111, "uman": 27112, "tism": 27113, "tvs": 27114, "servic": 27115, "croydon": 27116, "morales": 27117, "cg": 27118, "cum": 27119, "teur": 27120, "scanada": 27121, "sall": 27122, "magnolia": 27123, "elise": 27124, "thour": 27125, "ி": 27126, "agomez": 27127, "phelps": 27128, "ë°©íĥĦìĨĮëħĦëĭ¨": 27129, "whos": 27130, "weaving": 27131, "sisd": 27132, "proposes": 27133, "crows": 27134, "presale": 27135, "economies": 27136, "bernardo": 27137, "shahid": 27138, "airshow": 27139, "mccann": 27140, "horticul": 27141, "nrl": 27142, "duel": 27143, "mongolia": 27144, "toulou": 27145, "requirement": 27146, "structured": 27147, "edi": 27148, "olives": 27149, "hea": 27150, "cuter": 27151, "к": 27152, "enthusiast": 27153, "harriet": 27154, "dominion": 27155, "submer": 27156, "ðŁįĥ": 27157, "saab": 27158, "nesburg": 27159, "moff": 27160, "defended": 27161, "burt": 27162, "rewarded": 27163, "goldman": 27164, "optics": 27165, "khalid": 27166, "households": 27167, "buckets": 27168, "cecil": 27169, "chess": 27170, "substantial": 27171, "efl": 27172, "operation": 27173, "evaluate": 27174, "stn": 27175, "recession": 27176, "lll": 27177, "tomas": 27178, "truths": 27179, "akbar": 27180, "swords": 27181, "pact": 27182, "embarrass": 27183, "hao": 27184, "ayurve": 27185, "scripture": 27186, "nycc": 27187, "opt": 27188, "diameter": 27189, "scented": 27190, "organizers": 27191, "relat": 27192, "hae": 27193, "dreamers": 27194, "dese": 27195, "ðŁĮ»": 27196, "restricted": 27197, "nale": 27198, "rhp": 27199, "dolan": 27200, "munster": 27201, "haired": 27202, "consultants": 27203, "joints": 27204, "humil": 27205, "dill": 27206, "relentless": 27207, "té": 27208, "afil": 27209, "utilities": 27210, "japanese": 27211, "condemn": 27212, "petite": 27213, "collide": 27214, "qf": 27215, "peaches": 27216, "courier": 27217, "lore": 27218, "âĺİï¸ı": 27219, "reliability": 27220, "chuk": 27221, "ðŁĻĥ": 27222, "stures": 27223, "gether": 27224, "hostel": 27225, "bier": 27226, "-_-": 27227, "âĩ": 27228, "eze": 27229, "tailo": 27230, "dient": 27231, "bluff": 27232, "chuffed": 27233, "pilip": 27234, "monarch": 27235, "eem": 27236, "buchan": 27237, "bick": 27238, "opau": 27239, "kups": 27240, "ย": 27241, "pistons": 27242, "spins": 27243, "mand": 27244, "cest": 27245, "burne": 27246, "vile": 27247, "cherries": 27248, "beckett": 27249, "needles": 27250, "panch": 27251, "ëĤ": 27252, "hahah": 27253, "troubles": 27254, "insists": 27255, "doyou": 27256, "gmc": 27257, "mortar": 27258, "delegate": 27259, "inn": 27260, "ganda": 27261, "sinatra": 27262, "त": 27263, "speeding": 27264, "pupil": 27265, "premises": 27266, "alignment": 27267, "pikach": 27268, "asus": 27269, "jalan": 27270, "ص": 27271, "limestone": 27272, "folkl": 27273, "parmesan": 27274, "ceil": 27275, "moy": 27276, "shawnmendes": 27277, "acup": 27278, "hust": 27279, "otes": 27280, "medina": 27281, "madi": 27282, "gtav": 27283, "censorship": 27284, "arg": 27285, "sweeney": 27286, "sykes": 27287, "colo": 27288, "footsteps": 27289, "canned": 27290, "advance": 27291, "gtaonline": 27292, "healthyliving": 27293, "ðŁį¾": 27294, "aig": 27295, "pality": 27296, "ocs": 27297, "hebrew": 27298, "imminent": 27299, "berkshire": 27300, "jeremiah": 27301, "outgoing": 27302, "baker": 27303, "entrata": 27304, "maids": 27305, "groves": 27306, "boc": 27307, "adel": 27308, "mfw": 27309, "conscience": 27310, "armys": 27311, "nutella": 27312, "contestalert": 27313, "novelist": 27314, "lah": 27315, "banker": 27316, "marquez": 27317, "ðŁı¡": 27318, "toff": 27319, "outage": 27320, "grp": 27321, "ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ": 27322, "muscle": 27323, "dudley": 27324, "nvidia": 27325, "midi": 27326, "muni": 27327, "essays": 27328, "datac": 27329, "carter": 27330, "ร": 27331, "tans": 27332, "ives": 27333, "publications": 27334, "aler": 27335, "okwx": 27336, "ilu": 27337, "cutt": 27338, "harp": 27339, "outlaw": 27340, "lutheran": 27341, "brill": 27342, "bolic": 27343, "dowell": 27344, "greenland": 27345, "besties": 27346, "pathi": 27347, "payton": 27348, "guest": 27349, "harden": 27350, "ðŁ¤©": 27351, "anned": 27352, "evacuation": 27353, "poised": 27354, "mcder": 27355, "bhan": 27356, "oi": 27357, "envelope": 27358, "cid": 27359, "cavi": 27360, "tapas": 27361, "bookreview": 27362, "greyhound": 27363, "âĻª": 27364, "feud": 27365, "lungs": 27366, "forte": 27367, "raider": 27368, "ffer": 27369, "onix": 27370, "depend": 27371, "ynwa": 27372, "relating": 27373, "devs": 27374, "ðŁĴIJ": 27375, "acquires": 27376, "dha": 27377, "jyo": 27378, "privati": 27379, "canine": 27380, "kb": 27381, "crab": 27382, "sardin": 27383, "imagining": 27384, "kj": 27385, "empor": 27386, "downhill": 27387, "nez": 27388, "taeyeon": 27389, "nickimin": 27390, "gbp": 27391, "àµ": 27392, "wap": 27393, "secco": 27394, "mashed": 27395, "ðŁĴ¥ðŁĴ¥": 27396, "augustine": 27397, "dissol": 27398, "dictator": 27399, "âĵ": 27400, "viper": 27401, "edfringe": 27402, "vaux": 27403, "hardwork": 27404, "booklet": 27405, "nox": 27406, "chiff": 27407, "ðŁĴ¨": 27408, "observations": 27409, "xboxone": 27410, "usher": 27411, "keer": 27412, "lup": 27413, "dallas": 27414, "calgary": 27415, "madra": 27416, "dious": 27417, "kbs": 27418, "woodward": 27419, "heroine": 27420, "lumber": 27421, "seaworld": 27422, "ows": 27423, "mcke": 27424, "maverick": 27425, "gula": 27426, "crossroads": 27427, "fang": 27428, "sade": 27429, "nikol": 27430, "cheetah": 27431, "mec": 27432, "ppg": 27433, "erick": 27434, "ðŁİµ": 27435, "toxic": 27436, "bjj": 27437, "viola": 27438, "spire": 27439, "chino": 27440, "travis": 27441, "institutional": 27442, "haas": 27443, "lowry": 27444, "wac": 27445, "eae": 27446, "humid": 27447, "mpton": 27448, "ruck": 27449, "jew": 27450, "cine": 27451, "zimmer": 27452, "sef": 27453, "bharat": 27454, "frees": 27455, "aamir": 27456, "ðŁĴħ": 27457, "zinc": 27458, "wane": 27459, "multiplayer": 27460, "royalwedding": 27461, "eel": 27462, "precipit": 27463, "query": 27464, "kimberly": 27465, "isabel": 27466, "fulfill": 27467, "igan": 27468, "vaul": 27469, "pane": 27470, "scy": 27471, "digit": 27472, "gunn": 27473, "utah": 27474, "dogday": 27475, "fion": 27476, "xiaomi": 27477, "dac": 27478, "elast": 27479, "chavez": 27480, "roblo": 27481, "gine": 27482, "tenth": 27483, "abh": 27484, "keto": 27485, "hurdle": 27486, "nadia": 27487, "memorabilia": 27488, "habs": 27489, "quan": 27490, "hw": 27491, "hvac": 27492, "pixar": 27493, "eccle": 27494, "kramer": 27495, "accuses": 27496, "ðŁĴļðŁĴļ": 27497, "perse": 27498, "meantime": 27499, "wahl": 27500, "atletico": 27501, "âĢ¢âĢ¢âĢ¢âĢ¢": 27502, "ottoman": 27503, "novo": 27504, "kus": 27505, "connected": 27506, "trusts": 27507, "dmv": 27508, "spencer": 27509, "rahulg": 27510, "dove": 27511, "stokes": 27512, "bologna": 27513, "enthusiasts": 27514, "ê": 27515, "rockstargames": 27516, "tedcruz": 27517, "duras": 27518, "sacked": 27519, "latex": 27520, "immersive": 27521, "cert": 27522, "lucin": 27523, "principals": 27524, "fares": 27525, "sails": 27526, "farn": 27527, "ament": 27528, "saffron": 27529, "quentin": 27530, "checkpoint": 27531, "ferris": 27532, "excur": 27533, "ðŁijīðŁı¼": 27534, "bailey": 27535, "seh": 27536, "terre": 27537, "madam": 27538, "sband": 27539, "wanderers": 27540, "cumberbatch": 27541, "yyc": 27542, "digitally": 27543, "blackandwhitephotography": 27544, "rollin": 27545, "moroccan": 27546, "ðŁĮħ": 27547, "dinner": 27548, "dwell": 27549, "toom": 27550, "mye": 27551, "ezra": 27552, "cpfc": 27553, "warhol": 27554, "meer": 27555, "jonah": 27556, "noaa": 27557, "sgate": 27558, "soon": 27559, "secular": 27560, "gating": 27561, "tio": 27562, "driver": 27563, "sissy": 27564, "assange": 27565, "tath": 27566, "edmund": 27567, "bobcats": 27568, "raji": 27569, "postage": 27570, "studs": 27571, "mgm": 27572, "kato": 27573, "edinburgh": 27574, "meetthe": 27575, "shirt": 27576, "faa": 27577, "mensfashion": 27578, "spreads": 27579, "wim": 27580, "carts": 27581, "phoebe": 27582, "jars": 27583, "botswana": 27584, "ÙĤ": 27585, "edwar": 27586, "skar": 27587, "rive": 27588, "gusty": 27589, "ctv": 27590, "ferdinand": 27591, "sutherland": 27592, "nickiminaj": 27593, "kv": 27594, "sius": 27595, "beech": 27596, "rez": 27597, "desires": 27598, "onial": 27599, "campo": 27600, "quarry": 27601, "lorraine": 27602, "gilmore": 27603, "iggy": 27604, "µï¸ı": 27605, "hopping": 27606, "aviz": 27607, "ðŁĮº": 27608, "unisex": 27609, "dedicate": 27610, "attitudes": 27611, "steer": 27612, "junkie": 27613, "railway": 27614, "yb": 27615, "whisper": 27616, "keyan": 27617, "kus": 27618, "jug": 27619, "dix": 27620, "ains": 27621, "summon": 27622, "ovich": 27623, "syed": 27624, "herald": 27625, "maison": 27626, "meded": 27627, "wildflower": 27628, "mainland": 27629, "risky": 27630, "rukh": 27631, "overlooked": 27632, "kic": 27633, "destroys": 27634, "naman": 27635, "kip": 27636, "zano": 27637, "championsleague": 27638, "bandit": 27639, "quincy": 27640, "smile": 27641, "calvin": 27642, "openings": 27643, "tapp": 27644, "olulu": 27645, "spectro": 27646, "accredited": 27647, "apk": 27648, "praised": 27649, "barnett": 27650, "pollen": 27651, "premiered": 27652, "selenagomez": 27653, "toured": 27654, "screenings": 27655, "uuu": 27656, "miso": 27657, "ense": 27658, "adamlambert": 27659, "guelph": 27660, "haryana": 27661, "hutto": 27662, "lear": 27663, "ltc": 27664, "poached": 27665, "brexit": 27666, "æĿ": 27667, "ttc": 27668, "pavement": 27669, "mongers": 27670, "roe": 27671, "aders": 27672, "lington": 27673, "participant": 27674, "cared": 27675, "gail": 27676, "yates": 27677, "lantic": 27678, "dashboard": 27679, "joo": 27680, "felipe": 27681, "ssionist": 27682, "bum": 27683, "send": 27684, "aeri": 27685, "thugs": 27686, "lucifer": 27687, "ahe": 27688, "detector": 27689, "filly": 27690, "gasoline": 27691, "hamper": 27692, "humpday": 27693, "theta": 27694, "theband": 27695, "forecasts": 27696, "ohhh": 27697, "lobb": 27698, "holl": 27699, "cpu": 27700, "azu": 27701, "adar": 27702, "hailey": 27703, "bub": 27704, "cart": 27705, "quoted": 27706, "anarchy": 27707, "pancre": 27708, "twitart": 27709, "alden": 27710, "stash": 27711, "theless": 27712, "orni": 27713, "beliebers": 27714, "mormon": 27715, "particle": 27716, "aviation": 27717, "â¬Ĩ": 27718, "webcamtoy": 27719, "saddened": 27720, "cruis": 27721, "hamlet": 27722, "nct": 27723, "rollins": 27724, "marquee": 27725, "sawyer": 27726, "reliance": 27727, "aura": 27728, "diec": 27729, "soothing": 27730, "signings": 27731, "akis": 27732, "ó": 27733, "atkins": 27734, "aerop": 27735, "ðŁĮ¿": 27736, "yab": 27737, "shari": 27738, "connol": 27739, "dubbed": 27740, "manufacture": 27741, "convincing": 27742, "feelthebern": 27743, "rau": 27744, "pulit": 27745, "onec": 27746, "gemstone": 27747, "urging": 27748, "bagu": 27749, "gah": 27750, "acids": 27751, "fianc": 27752, "zodiac": 27753, "snoop": 27754, "herrera": 27755, "initiated": 27756, "venge": 27757, "professors": 27758, "prodi": 27759, "stronger": 27760, "emission": 27761, "bba": 27762, "halle": 27763, "tapp": 27764, "hawan": 27765, "whim": 27766, "competed": 27767, "myrtle": 27768, "irport": 27769, "coldplay": 27770, "ache": 27771, "skep": 27772, "mson": 27773, "ssic": 27774, "calligraphy": 27775, "swimmers": 27776, "mey": 27777, "ppc": 27778, "thrift": 27779, "poc": 27780, "replaces": 27781, "commuter": 27782, "âģ¦âģ¦@": 27783, "goers": 27784, "logue": 27785, "paradig": 27786, "baskets": 27787, "sensitivity": 27788, "johan": 27789, "atlantis": 27790, "&&": 27791, "suitcase": 27792, "anxious": 27793, "lh": 27794, "stri": 27795, "galloway": 27796, "stread": 27797, "warden": 27798, "grounded": 27799, "fficiency": 27800, "lifeat": 27801, "relic": 27802, "disguise": 27803, "islanders": 27804, "fcofficial": 27805, "classicalmusic": 27806, "bmc": 27807, "enfield": 27808, "bique": 27809, "oakley": 27810, "batman": 27811, "slaying": 27812, "nerves": 27813, "multit": 27814, "calcium": 27815, "projector": 27816, "scottsdale": 27817, "antino": 27818, "grips": 27819, "kimmel": 27820, "desmond": 27821, "protestors": 27822, "hiatus": 27823, "metabolism": 27824, "concluded": 27825, "presser": 27826, "tipping": 27827, "slide": 27828, "eto": 27829, "hunting": 27830, "ausopen": 27831, "rik": 27832, "ppery": 27833, "innovators": 27834, "pitchers": 27835, "agger": 27836, "fungi": 27837, "zad": 27838, "prolific": 27839, "rocknroll": 27840, "blames": 27841, "ctar": 27842, "stamford": 27843, "qad": 27844, "mozzarella": 27845, "insanely": 27846, "denver": 27847, "phouse": 27848, "nomad": 27849, "ï¿": 27850, "sris": 27851, "produ": 27852, "henley": 27853, "pagan": 27854, "amtrak": 27855, "rubi": 27856, "incl": 27857, "tutor": 27858, "scotia": 27859, "woes": 27860, "singapo": 27861, "funnel": 27862, "turnbull": 27863, "knowledge": 27864, "grimm": 27865, "realmadrid": 27866, "weare": 27867, "missiles": 27868, "consol": 27869, "emojis": 27870, "sneak": 27871, "smiths": 27872, "ruiz": 27873, "brou": 27874, "iel": 27875, "haver": 27876, "ðŁĮļ": 27877, "kingof": 27878, "basilica": 27879, "circulation": 27880, "printers": 27881, "tapping": 27882, "ridley": 27883, "dragged": 27884, "haj": 27885, "writer": 27886, "fundamentals": 27887, "personalities": 27888, "metre": 27889, "stereotypes": 27890, "burle": 27891, "bestof": 27892, "nffc": 27893, "hath": 27894, "ministries": 27895, "aali": 27896, "tracing": 27897, "paved": 27898, "łï¸ı": 27899, "gic": 27900, "inspire": 27901, "tug": 27902, "hare": 27903, "repeated": 27904, "expon": 27905, "lolli": 27906, "rhode": 27907, "precin": 27908, "installations": 27909, "instagram": 27910, "azar": 27911, "ies": 27912, "solely": 27913, "dukes": 27914, "missionary": 27915, "vanguard": 27916, "fursuitfriday": 27917, "ond": 27918, "polari": 27919, "mast": 27920, "haran": 27921, "josé": 27922, "jacked": 27923, "ecoun": 27924, "alities": 27925, "neph": 27926, "ravel": 27927, "moderated": 27928, "scow": 27929, "sfb": 27930, "uruguay": 27931, "aso": 27932, "nig": 27933, "audu": 27934, "pints": 27935, "latina": 27936, "benz": 27937, "mitting": 27938, "charted": 27939, "matology": 27940, "citro": 27941, "biopic": 27942, "ðŁijŃ": 27943, "djokovic": 27944, "foxy": 27945, "aguil": 27946, "soto": 27947, "anada": 27948, "sinking": 27949, "scrap": 27950, "hairs": 27951, "bethany": 27952, "factfriday": 27953, "ðŁIJIJ": 27954, "unleashed": 27955, ")(": 27956, "contradic": 27957, "ramon": 27958, "coastline": 27959, "yong": 27960, "snsd": 27961, "ligan": 27962, "pome": 27963, "mitage": 27964, "gett": 27965, "wati": 27966, "risk": 27967, "soaring": 27968, "brush": 27969, "fpl": 27970, "avan": 27971, "åĨ": 27972, "larson": 27973, "shear": 27974, "multil": 27975, "blur": 27976, "multimedia": 27977, "chunky": 27978, "pari": 27979, "nani": 27980, "weird": 27981, "cholesterol": 27982, "charles": 27983, "dreamed": 27984, "tanning": 27985, "puzzles": 27986, "fram": 27987, "handball": 27988, "chag": 27989, "belize": 27990, "alu": 27991, "bangs": 27992, "ÑĦ": 27993, "detectives": 27994, "mcg": 27995, "ishq": 27996, "bothered": 27997, "safc": 27998, "mping": 27999, "teneri": 28000, "gays": 28001, "sailor": 28002, "angi": 28003, "multicul": 28004, "guessed": 28005, "rosé": 28006, "highways": 28007, "broom": 28008, "chattanoo": 28009, "-'": 28010, "seeker": 28011, "oned": 28012, "atf": 28013, "luc": 28014, "><": 28015, "bari": 28016, "percep": 28017, "jewelry": 28018, "asph": 28019, "sorrow": 28020, "sling": 28021, "mammoth": 28022, "jackie": 28023, "ë§": 28024, "wiltshire": 28025, "sao": 28026, "cancell": 28027, "impaired": 28028, "torial": 28029, "breed": 28030, "guyen": 28031, "judice": 28032, "title": 28033, "prospective": 28034, "applicants": 28035, "ðŁįĬ": 28036, "episcop": 28037, "eid": 28038, "byo": 28039, "stockings": 28040, "ðŁĴĥðŁĴĥ": 28041, "llp": 28042, "snag": 28043, "keepit": 28044, "lough": 28045, "olson": 28046, "maturity": 28047, "!!!\"": 28048, "copter": 28049, "isha": 28050, "bli": 28051, "wilmington": 28052, "tryouts": 28053, "thai": 28054, "ðŁ¥³": 28055, "pebble": 28056, "kraft": 28057, "fp": 28058, "º": 28059, "ssively": 28060, "livin": 28061, "contestants": 28062, "textures": 28063, "joan": 28064, "hdr": 28065, "filmfestival": 28066, "provence": 28067, "wido": 28068, "opend": 28069, "csi": 28070, "stown": 28071, "croati": 28072, "adjust": 28073, "hostile": 28074, "analysts": 28075, "ilan": 28076, "cuppa": 28077, "brum": 28078, "newfoundland": 28079, "goodwin": 28080, "mett": 28081, "mallorca": 28082, "plugs": 28083, "buk": 28084, "bbhutto": 28085, "wrestle": 28086, "saire": 28087, "shopped": 28088, "forza": 28089, "lehead": 28090, "vivo": 28091, "bast": 28092, "roxy": 28093, "regis": 28094, "hardworking": 28095, "honolulu": 28096, "despair": 28097, "youngsters": 28098, "nig": 28099, "impromp": 28100, "rolltide": 28101, "deemed": 28102, "treason": 28103, "rushed": 28104, "forged": 28105, "fff": 28106, "pikachu": 28107, "briggs": 28108, "doit": 28109, "accent": 28110, "laus": 28111, "glaze": 28112, "competent": 28113, "aho": 28114, "photog": 28115, "midfield": 28116, "lego": 28117, "harvard": 28118, "minorities": 28119, "reilly": 28120, "sliced": 28121, "onceupon": 28122, "initially": 28123, "financially": 28124, "landscapephotography": 28125, "hardro": 28126, "quo": 28127, "mmers": 28128, "parkinson": 28129, "smugg": 28130, "readiness": 28131, "brutally": 28132, "gloucester": 28133, "mped": 28134, "bbhuttozardari": 28135, "murder": 28136, "yed": 28137, "dataviz": 28138, "srt": 28139, "downing": 28140, "bians": 28141, "mü": 28142, "fleck": 28143, "flipped": 28144, "sly": 28145, "brilliance": 28146, "rim": 28147, "kum": 28148, "bubba": 28149, "koi": 28150, "knitted": 28151, "sorg": 28152, "mais": 28153, "ðŁĮ²": 28154, "tiss": 28155, "sustain": 28156, "sensu": 28157, "akhan": 28158, "ziest": 28159, "examines": 28160, "chardonnay": 28161, "username": 28162, "shortlist": 28163, "rebs": 28164, "ono": 28165, "daring": 28166, "hardwood": 28167, "cheque": 28168, "righteous": 28169, "lightening": 28170, "dirk": 28171, "shradd": 28172, "dura": 28173, "downstairs": 28174, "shal": 28175, "amigos": 28176, "ruff": 28177, "slaw": 28178, "ries": 28179, "rednation": 28180, "manus": 28181, "ðŁĩ§ðŁĩ·": 28182, "distinction": 28183, "ubun": 28184, "duran": 28185, "migra": 28186, "thians": 28187, "laver": 28188, "domestic": 28189, "kx": 28190, "jazzy": 28191, "justify": 28192, "belonging": 28193, "insulation": 28194, "colorstv": 28195, "drunken": 28196, "channeling": 28197, "quand": 28198, "xiii": 28199, "enlighten": 28200, "kano": 28201, "fatima": 28202, "teenchoice": 28203, "terrified": 28204, "pba": 28205, "asley": 28206, "metmuseum": 28207, "dune": 28208, "packer": 28209, "kio": 28210, "ðŁĴľðŁĴľ": 28211, "boiler": 28212, "fascism": 28213, "armored": 28214, "backgrounds": 28215, "inmates": 28216, "embarrassed": 28217, "defines": 28218, "thd": 28219, "wego": 28220, "silicone": 28221, "loon": 28222, "elding": 28223, "borrowed": 28224, "hemp": 28225, "aksh": 28226, "kawasaki": 28227, "bry": 28228, "deaf": 28229, "killer": 28230, "disposal": 28231, "ðŁĩ°": 28232, "glastonbury": 28233, "uncovered": 28234, "oxide": 28235, "poff": 28236, "dant": 28237, "kj": 28238, "kuro": 28239, "drizzle": 28240, "peoples": 28241, "fee": 28242, "propri": 28243, "ddlovato": 28244, "piggy": 28245, "otis": 28246, "allergies": 28247, "ubis": 28248, "penguin": 28249, "sera": 28250, "viz": 28251, "prosperous": 28252, "icides": 28253, "tornadoes": 28254, "senegal": 28255, "webcast": 28256, "stored": 28257, "enchanted": 28258, "bbcone": 28259, "bayarea": 28260, "entrepreneurial": 28261, "rednationrising": 28262, "experimenting": 28263, "angan": 28264, "lotto": 28265, "theyre": 28266, "pore": 28267, "erp": 28268, "serene": 28269, "eastwood": 28270, "brokers": 28271, "barge": 28272, "stallion": 28273, "timberlake": 28274, "tailored": 28275, "dystop": 28276, "bate": 28277, "lators": 28278, "dixit": 28279, "branson": 28280, "dynamo": 28281, "kylie": 28282, "shameful": 28283, "btwn": 28284, "springtime": 28285, "mixture": 28286, "sounded": 28287, "luton": 28288, "dades": 28289, "mala": 28290, "opra": 28291, "enic": 28292, "rahulgandhi": 28293, "sewer": 28294, "~~~~": 28295, "kyu": 28296, "northeastern": 28297, "caer": 28298, "bcu": 28299, "nirvana": 28300, "kitchens": 28301, "ousy": 28302, "alm": 28303, "riverdale": 28304, "hidden": 28305, "flint": 28306, "spd": 28307, "patrons": 28308, "katyperry": 28309, "augh": 28310, "exhibitions": 28311, "smc": 28312, "shuts": 28313, "atore": 28314, "dain": 28315, "something": 28316, "berth": 28317, "bog": 28318, "porter": 28319, "gento": 28320, "concussion": 28321, "anglic": 28322, "rowe": 28323, "grilling": 28324, "scarlett": 28325, "mastering": 28326, "mornin": 28327, "commented": 28328, "sime": 28329, "sizing": 28330, "christy": 28331, "ceos": 28332, "stm": 28333, "atry": 28334, "tariffs": 28335, "vacation": 28336, "prejudice": 28337, "psu": 28338, "parental": 28339, "farage": 28340, "cana": 28341, "capcom": 28342, "kosovo": 28343, "youre": 28344, "menstru": 28345, "stalin": 28346, "grapefruit": 28347, "bran": 28348, "chesa": 28349, "daven": 28350, "excel": 28351, "!!)": 28352, "à¹Į": 28353, "distributor": 28354, "cea": 28355, "bridesma": 28356, "millennial": 28357, "wain": 28358, "observing": 28359, "misery": 28360, "planetary": 28361, "exposing": 28362, "braised": 28363, "compton": 28364, "dongha": 28365, "ql": 28366, "springsteen": 28367, "thul": 28368, "sylve": 28369, "cabo": 28370, "palad": 28371, "nielsen": 28372, "gazing": 28373, "baja": 28374, "roud": 28375, "orchids": 28376, "johannesburg": 28377, "seman": 28378, "dji": 28379, "operative": 28380, "affection": 28381, "eclectic": 28382, "atc": 28383, "mutant": 28384, "awx": 28385, "nice": 28386, "melbourne": 28387, "indulg": 28388, "tulip": 28389, "diaspora": 28390, "welp": 28391, "biggie": 28392, "mississauga": 28393, "retriever": 28394, "oran": 28395, "tammy": 28396, "cta": 28397, "hippo": 28398, "seasoned": 28399, "germans": 28400, "engv": 28401, "marvellous": 28402, "imf": 28403, "relays": 28404, "montan": 28405, "mauriti": 28406, "meister": 28407, "assurance": 28408, "reigning": 28409, "sufficient": 28410, "hane": 28411, "nothing": 28412, "posse": 28413, "navy": 28414, "inlove": 28415, "brighton": 28416, "enqu": 28417, "chung": 28418, "sweaty": 28419, "esc": 28420, "caled": 28421, "mans": 28422, "nicaragua": 28423, "slices": 28424, "mocha": 28425, "washingtonpost": 28426, "bbn": 28427, "damned": 28428, "growing": 28429, "enburg": 28430, "loan": 28431, "mes": 28432, "whoops": 28433, "believers": 28434, "spiel": 28435, "vodaf": 28436, "lat": 28437, "sled": 28438, "cricketer": 28439, "browne": 28440, "golfers": 28441, "barra": 28442, "watchers": 28443, "luigi": 28444, "swamy": 28445, "moms": 28446, "pitched": 28447, "santor": 28448, "crs": 28449, "sire": 28450, "scamp": 28451, "bode": 28452, "stewar": 28453, "jonny": 28454, "entity": 28455, "pacqui": 28456, "mindful": 28457, "minindia": 28458, "bearded": 28459, "tempt": 28460, "scorpion": 28461, "eaton": 28462, "authorized": 28463, "arto": 28464, "svp": 28465, "opathy": 28466, "cchini": 28467, "housemusic": 28468, "disneyworld": 28469, "âĢĶ@": 28470, "propose": 28471, "diy": 28472, "expense": 28473, "teng": 28474, "puppets": 28475, "smel": 28476, "daca": 28477, "perry": 28478, "finn": 28479, "boosting": 28480, "leftovers": 28481, "cougs": 28482, "satellites": 28483, "many": 28484, "aze": 28485, "gong": 28486, "fie": 28487, "methodo": 28488, "ferries": 28489, "ðŁ¤ĶðŁ¤Ķ": 28490, "explorers": 28491, "loader": 28492, "attracted": 28493, "ilton": 28494, "goddamn": 28495, "piazza": 28496, "doctr": 28497, "saving": 28498, "paragraph": 28499, "visualization": 28500, "mayors": 28501, "workflow": 28502, "ackles": 28503, "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ": 28504, "स": 28505, "twerk": 28506, "clut": 28507, "lover": 28508, "teases": 28509, "sian": 28510, "ote": 28511, "deterior": 28512, "accord": 28513, "lfw": 28514, "swarovski": 28515, "natal": 28516, "traps": 28517, "kina": 28518, "analyze": 28519, "layered": 28520, "beverages": 28521, "unit": 28522, "ransom": 28523, "peshaw": 28524, "destined": 28525, "astrology": 28526, "sipping": 28527, "mileycyrus": 28528, "camino": 28529, "marshmallow": 28530, "bliss": 28531, "outback": 28532, "faq": 28533, "intoler": 28534, "humility": 28535, "poppin": 28536, "halloween": 28537, "montene": 28538, "ophy": 28539, "nun": 28540, "tattooed": 28541, "aas": 28542, "ðŁĮ³": 28543, "daley": 28544, "quality": 28545, "dusa": 28546, "fishermen": 28547, "swif": 28548, "terrac": 28549, "stau": 28550, "lein": 28551, "trolling": 28552, "shipment": 28553, "gardener": 28554, "marchmadness": 28555, "headband": 28556, "grt": 28557, "burnett": 28558, "wand": 28559, "!!!!!!!!!": 28560, "ghe": 28561, "dux": 28562, "hud": 28563, "warner": 28564, "ðŁĩ¦": 28565, "exile": 28566, "rescue": 28567, "rata": 28568, "dhan": 28569, "ducati": 28570, "drown": 28571, "blends": 28572, "spie": 28573, "alligator": 28574, "simultaneously": 28575, "brooke": 28576, "uke": 28577, "khar": 28578, "communion": 28579, "rika": 28580, "fordfc": 28581, "chinatown": 28582, "yourown": 28583, "mey": 28584, "canal": 28585, "systematic": 28586, "depri": 28587, "oxford": 28588, "anil": 28589, "wut": 28590, "equation": 28591, "bez": 28592, "fleur": 28593, "thegood": 28594, "langley": 28595, "adity": 28596, "edith": 28597, "alfie": 28598, "оÑĤ": 28599, "encry": 28600, "brill": 28601, "exemp": 28602, "cesar": 28603, "mbling": 28604, "abri": 28605, "scicom": 28606, "jing": 28607, "schooling": 28608, "mika": 28609, "mechanisms": 28610, "impromptu": 28611, "rhea": 28612, "moore": 28613, "crimea": 28614, "besto": 28615, "wright": 28616, "elders": 28617, "rods": 28618, "kamal": 28619, "folklore": 28620, "beet": 28621, "minion": 28622, "relieve": 28623, "thro": 28624, "teamusa": 28625, "pascal": 28626, "madewith": 28627, "bolivia": 28628, "itti": 28629, "freebies": 28630, "desired": 28631, "bestselling": 28632, "liness": 28633, "laden": 28634, "keane": 28635, "mists": 28636, "hippie": 28637, "attachment": 28638, "@/": 28639, "sew": 28640, "flanagan": 28641, "âĿĹï¸ı": 28642, "supremac": 28643, "stlcards": 28644, "sias": 28645, "qu": 28646, "rhys": 28647, "steep": 28648, "valleys": 28649, "vw": 28650, "paving": 28651, "dispat": 28652, "alison": 28653, "porte": 28654, "idu": 28655, "newsc": 28656, "socket": 28657, "mos": 28658, "costar": 28659, "revo": 28660, "proteins": 28661, "stanleycup": 28662, "mcal": 28663, "earring": 28664, "secs": 28665, "mclean": 28666, "capric": 28667, "nickelo": 28668, "aden": 28669, "vc": 28670, "shouse": 28671, "adaptive": 28672, "maximize": 28673, "entertainer": 28674, "prose": 28675, "griffi": 28676, "sixteen": 28677, "lamar": 28678, "mirage": 28679, "saudiarabia": 28680, "aweather": 28681, "rust": 28682, "infiltr": 28683, "fashionweek": 28684, "ðŁĺĬðŁĺĬðŁĺĬ": 28685, "selective": 28686, "bubble": 28687, "aden": 28688, "fennel": 28689, "decisive": 28690, "mta": 28691, "mocking": 28692, "mbles": 28693, "stamp": 28694, "mule": 28695, "bernardo": 28696, "grin": 28697, "pott": 28698, "jingle": 28699, "vettel": 28700, "colombian": 28701, "camo": 28702, "motivationmonday": 28703, "bahan": 28704, "ply": 28705, "dhary": 28706, "kami": 28707, "xmen": 28708, "sleeper": 28709, "gara": 28710, "mysti": 28711, "confidential": 28712, "conflicts": 28713, "pneu": 28714, "ces": 28715, "insurtech": 28716, "cleanse": 28717, "merely": 28718, "vais": 28719, "tux": 28720, "thegreat": 28721, "sharon": 28722, "maj": 28723, "hola": 28724, "ecosystems": 28725, "ajay": 28726, "aaj": 28727, "hush": 28728, "harmon": 28729, "backtoschool": 28730, "wikileaks": 28731, "reflected": 28732, "ðŁĺĵ": 28733, "commemorating": 28734, "acet": 28735, "buckingham": 28736, "messiah": 28737, "tuous": 28738, "hornet": 28739, "tobe": 28740, "dq": 28741, "heine": 28742, "mig": 28743, "plate": 28744, "nicholson": 28745, "spie": 28746, "cumberland": 28747, "normal": 28748, "phobia": 28749, "happyhalloween": 28750, "cityfc": 28751, "mcel": 28752, "gillian": 28753, "keto": 28754, "lude": 28755, "demise": 28756, "suga": 28757, "strate": 28758, "mcgrath": 28759, "visitscotland": 28760, "fooled": 28761, "cbr": 28762, "gcse": 28763, "colori": 28764, "potd": 28765, "missuniverse": 28766, "finances": 28767, "mapoli": 28768, "forks": 28769, "Ø´": 28770, "cannon": 28771, "medicinal": 28772, "ðŁĹĵ": 28773, "kho": 28774, "wreck": 28775, "panto": 28776, "bagel": 28777, "gull": 28778, "syndicate": 28779, "icy": 28780, "prc": 28781, "kien": 28782, "zika": 28783, "tish": 28784, "peta": 28785, "cco": 28786, "liza": 28787, "chut": 28788, "extraction": 28789, "elg": 28790, "gli": 28791, "fueled": 28792, "posit": 28793, "respectively": 28794, "leicester": 28795, "brink": 28796, "vulnerability": 28797, "imported": 28798, "esha": 28799, "ðŁ¦ħ": 28800, "rural": 28801, "rell": 28802, "gaming": 28803, "atlantic": 28804, "abandon": 28805, "noah": 28806, "resolved": 28807, "prostate": 28808, "allergic": 28809, "psd": 28810, "âĺ¹": 28811, "dungeon": 28812, "fangirl": 28813, "illuminated": 28814, "mhs": 28815, "whitesox": 28816, "dently": 28817, "cko": 28818, "endorse": 28819, "overly": 28820, "dazzling": 28821, "prioriti": 28822, "nightlife": 28823, "util": 28824, "behave": 28825, "flamen": 28826, "eastbound": 28827, "ðŁĴŁ": 28828, "iloveyou": 28829, "govuk": 28830, "mozambique": 28831, "allegi": 28832, "dri": 28833, "testimonial": 28834, "aths": 28835, "ì§Ģ": 28836, "mmy": 28837, "shabby": 28838, "prosecco": 28839, "friendships": 28840, "calam": 28841, "damages": 28842, "offset": 28843, "jurassic": 28844, "juno": 28845, "arrell": 28846, "ðŁĴ©": 28847, "interventions": 28848, "daredevil": 28849, "carver": 28850, "runaway": 28851, "rane": 28852, "trustees": 28853, "haute": 28854, "depths": 28855, "ðŁİŃ": 28856, "mein": 28857, "sacrifices": 28858, "concier": 28859, "nesting": 28860, "izzy": 28861, "metam": 28862, "ilovemy": 28863, "urine": 28864, "dulu": 28865, "malhotra": 28866, "veins": 28867, "nightly": 28868, "coat": 28869, "andi": 28870, "hewitt": 28871, "lonel": 28872, "cible": 28873, "write": 28874, "jennie": 28875, "santac": 28876, "ĸï¸ı": 28877, "strato": 28878, "singapore": 28879, "soprano": 28880, "kristen": 28881, "cheerful": 28882, "fleetwood": 28883, "fairi": 28884, "meli": 28885, "wast": 28886, "turnt": 28887, "sforsale": 28888, "scrolling": 28889, "angelina": 28890, "rendition": 28891, "jericho": 28892, "nicky": 28893, "orb": 28894, "flavo": 28895, "patriot": 28896, "asheville": 28897, "sickness": 28898, "refund": 28899, "aggression": 28900, "bpl": 28901, "ãĥĥ": 28902, "elusive": 28903, "thistory": 28904, "hanger": 28905, "buffs": 28906, "villas": 28907, "atkinson": 28908, "sph": 28909, "jait": 28910, "declined": 28911, "wok": 28912, "supremacy": 28913, "ootball": 28914, "eyang": 28915, "ðŁİĵ": 28916, "sford": 28917, "athi": 28918, "consume": 28919, "roadster": 28920, "eso": 28921, "upro": 28922, "recipe": 28923, "auf": 28924, "uci": 28925, "aron": 28926, "oooh": 28927, "csgo": 28928, "reich": 28929, "mcd": 28930, "minute": 28931, "ladies": 28932, "punk": 28933, "rutgers": 28934, "meek": 28935, "arizon": 28936, "taj": 28937, "landlord": 28938, "degra": 28939, "autumn": 28940, "lynx": 28941, "usf": 28942, "bhi": 28943, "fairytale": 28944, "donghae": 28945, "betsy": 28946, "exploded": 28947, "chennai": 28948, "opa": 28949, "protag": 28950, "brant": 28951, "ðŁĵ°:": 28952, "gf": 28953, "palli": 28954, "ðŁı¼âĢįâĻĢï¸ı": 28955, "sut": 28956, "illini": 28957, "columnist": 28958, "shirtless": 28959, "decentr": 28960, "searched": 28961, "ecor": 28962, "buggy": 28963, "sack": 28964, "ðŁĺĤðŁĺŃ": 28965, "det": 28966, "theri": 28967, "ornaments": 28968, "bringback": 28969, "tov": 28970, "quarterfinals": 28971, "iche": 28972, "constra": 28973, "gier": 28974, "buchanan": 28975, "vix": 28976, "kayaking": 28977, "mustread": 28978, "swallow": 28979, "melb": 28980, "scaf": 28981, "opal": 28982, "mayoral": 28983, "harat": 28984, "ðŁ¦ĭ": 28985, "schedules": 28986, "idf": 28987, "hague": 28988, "roz": 28989, "aah": 28990, "dmc": 28991, "duplic": 28992, "cache": 28993, "orphan": 28994, "fracture": 28995, "recon": 28996, "chav": 28997, "bunnies": 28998, "alain": 28999, "mustafa": 29000, "ðŁİĻ": 29001, "vacations": 29002, "dynamite": 29003, "texted": 29004, "broadcaster": 29005, "ðŁĴ£": 29006, "steamed": 29007, "rocker": 29008, "dietary": 29009, "luxurytravel": 29010, "inaugurated": 29011, "sawards": 29012, "vaughn": 29013, "lincolnshire": 29014, "clicked": 29015, "kraja": 29016, "fanc": 29017, "removes": 29018, "layoffs": 29019, "mcfar": 29020, "breeds": 29021, "winnie": 29022, "jonghyun": 29023, "incentive": 29024, "variations": 29025, "patton": 29026, "aturday": 29027, "persistent": 29028, "prun": 29029, "piers": 29030, "dales": 29031, "æĸ": 29032, "breastfeeding": 29033, "rance": 29034, "tawa": 29035, "Ĥâĸ": 29036, "murdoch": 29037, "captive": 29038, "thistle": 29039, "nica": 29040, "commodity": 29041, "couldnt": 29042, "boardwalk": 29043, "gracious": 29044, "practitioners": 29045, "ngc": 29046, "scrum": 29047, "nero": 29048, "camouflage": 29049, "colon": 29050, "hei": 29051, "physicist": 29052, "saturdaymorning": 29053, "tener": 29054, "siwon": 29055, "columns": 29056, "brune": 29057, "yvr": 29058, "bair": 29059, "retires": 29060, "halam": 29061, "caber": 29062, "shazam": 29063, "minu": 29064, "cascade": 29065, "milkshake": 29066, "grid": 29067, "dren": 29068, "vincent": 29069, "sodium": 29070, "platter": 29071, "cheerleader": 29072, "chenko": 29073, "yak": 29074, "eliminated": 29075, "typo": 29076, "yman": 29077, "rethink": 29078, "âĿĹ": 29079, "tsville": 29080, "bernardokath": 29081, "extr": 29082, "ðŁĺģðŁĺģðŁĺģ": 29083, "tao": 29084, "reper": 29085, "moths": 29086, "empowered": 29087, "citing": 29088, "transported": 29089, "monks": 29090, "sanat": 29091, "clears": 29092, "bachelorette": 29093, "campbell": 29094, "rachael": 29095, "harle": 29096, "handler": 29097, "climbs": 29098, "interference": 29099, "release": 29100, "shand": 29101, "rbs": 29102, "hrh": 29103, "ãģª": 29104, "valle": 29105, "ré": 29106, "slime": 29107, "wakes": 29108, "chubby": 29109, "sloan": 29110, "elves": 29111, "athen": 29112, "attorneys": 29113, "microscope": 29114, "stoner": 29115, "scaling": 29116, "obe": 29117, "cout": 29118, "seman": 29119, "midweek": 29120, "balsam": 29121, "ðŁĺįâĿ¤": 29122, "tiful": 29123, "vish": 29124, "lotta": 29125, "ripping": 29126, "remn": 29127, "tire": 29128, "leap": 29129, "havent": 29130, "laby": 29131, "himach": 29132, "whispers": 29133, "wein": 29134, "ðŁİ¸": 29135, "wildflowers": 29136, "sele": 29137, "ucc": 29138, "liability": 29139, "azine": 29140, "swings": 29141, "kya": 29142, "tair": 29143, "remain": 29144, "edo": 29145, "flops": 29146, "pocket": 29147, "grandad": 29148, "examiner": 29149, "gris": 29150, "ffect": 29151, "ðŁijĬðŁı»": 29152, "studded": 29153, "heartbeat": 29154, "deacon": 29155, "firmly": 29156, "infectious": 29157, "stef": 29158, "outlines": 29159, "leasing": 29160, "claws": 29161, "sense": 29162, "tabs": 29163, "hoot": 29164, "mosul": 29165, "spawn": 29166, "coa": 29167, "hogwarts": 29168, "vein": 29169, "albania": 29170, "manuel": 29171, "bino": 29172, "vauxhall": 29173, "scotland": 29174, "gobucks": 29175, "matty": 29176, "physio": 29177, "torino": 29178, "constable": 29179, "investigated": 29180, "slower": 29181, "mistaken": 29182, "bayer": 29183, "wildfires": 29184, "voic": 29185, "xon": 29186, "timeto": 29187, "chassis": 29188, "barric": 29189, "pion": 29190, "baldhead": 29191, "wook": 29192, "registr": 29193, "drafts": 29194, "bhs": 29195, "ligue": 29196, "lick": 29197, "staffordshire": 29198, "bafta": 29199, "darry": 29200, "jeanne": 29201, "vending": 29202, "corp": 29203, "âĽ³ï¸ı": 29204, "kiddos": 29205, "fenway": 29206, "cao": 29207, "westbound": 29208, "ðŁĺĻ": 29209, "dvr": 29210, "quicker": 29211, "blah": 29212, "goodie": 29213, "ðŁĴĭðŁĴĭ": 29214, "vox": 29215, "esper": 29216, "facade": 29217, "correlation": 29218, "redbull": 29219, "roup": 29220, "declining": 29221, "chive": 29222, "mcgee": 29223, "turo": 29224, "inder": 29225, "feller": 29226, "fug": 29227, "ilysm": 29228, "mardi": 29229, "peshawar": 29230, "kieran": 29231, "inema": 29232, "meatballs": 29233, "peck": 29234, "depressing": 29235, "sensing": 29236, "giz": 29237, "ddington": 29238, "springwatch": 29239, "roaming": 29240, "yellowstone": 29241, "horseshoe": 29242, "amman": 29243, "weekday": 29244, "olor": 29245, "ðŁ¥°": 29246, "boosts": 29247, "sprint": 29248, "scarves": 29249, "jee": 29250, "beetro": 29251, "clan": 29252, "allthe": 29253, "ìĦ¸ë": 29254, "enlightenment": 29255, "adobe": 29256, "regeneration": 29257, "?@": 29258, "contag": 29259, "yachts": 29260, "tou": 29261, "mora": 29262, "envoy": 29263, "rani": 29264, "goli": 29265, "dhanushkraja": 29266, "woodworking": 29267, "strengths": 29268, "sedi": 29269, "discs": 29270, "arina": 29271, "scon": 29272, "lite": 29273, "another": 29274, "ðŁ¥Ĭ": 29275, "yemen": 29276, "guern": 29277, "savvy": 29278, "loyed": 29279, "biomed": 29280, "heartbreak": 29281, "comrades": 29282, "millie": 29283, "patch": 29284, "unf": 29285, "jarvis": 29286, "blaming": 29287, "commemoration": 29288, "gey": 29289, "å¥": 29290, "cardiovascular": 29291, "aligned": 29292, "document": 29293, ".?": 29294, "aesthetics": 29295, "emu": 29296, "theirs": 29297, "leh": 29298, "psic": 29299, "sif": 29300, "plateau": 29301, "expend": 29302, "dominating": 29303, "robes": 29304, "mauritius": 29305, "exceptionally": 29306, "homer": 29307, "discoveries": 29308, "braun": 29309, "tennant": 29310, "insulin": 29311, "ðŁİ®": 29312, "carbs": 29313, "teas": 29314, "?!\"": 29315, "zie": 29316, "francois": 29317, "browsing": 29318, "thol": 29319, "clarence": 29320, "helper": 29321, "obtained": 29322, "cassie": 29323, "lees": 29324, "!,": 29325, "pomegran": 29326, "hubs": 29327, "prestige": 29328, "][": 29329, "macher": 29330, "bottled": 29331, "punch": 29332, "pipe": 29333, "och": 29334, "gallons": 29335, "deliveries": 29336, "ura": 29337, "unday": 29338, "monde": 29339, "depicts": 29340, "regency": 29341, "outrageous": 29342, "khaled": 29343, "caro": 29344, "hearti": 29345, "zag": 29346, "developmental": 29347, "overcoming": 29348, "statistical": 29349, "flavored": 29350, "fords": 29351, "creatives": 29352, "laurence": 29353, "dias": 29354, "sunscreen": 29355, "inked": 29356, "preacher": 29357, "nul": 29358, "impacting": 29359, "autistic": 29360, "âļĶï¸ı": 29361, "oss": 29362, "pelicans": 29363, "celeste": 29364, "vb": 29365, "rump": 29366, "mcgra": 29367, "fairfax": 29368, "humor": 29369, "bbcnews": 29370, "rowling": 29371, "calder": 29372, "seamless": 29373, "agne": 29374, "pti": 29375, "mixed": 29376, "tshirts": 29377, "merci": 29378, "btob": 29379, "womeninstem": 29380, "genealogy": 29381, "preven": 29382, "lour": 29383, "cradle": 29384, "giuse": 29385, "о": 29386, "chrono": 29387, "fairness": 29388, "chocolate": 29389, "tory": 29390, "asda": 29391, "prescott": 29392, "stretched": 29393, "alman": 29394, "uil": 29395, "recharge": 29396, "intre": 29397, "obst": 29398, "hospital": 29399, "hayward": 29400, "tenerife": 29401, "friedman": 29402, "vaping": 29403, "confessions": 29404, "yeah": 29405, "balli": 29406, "lucknow": 29407, "corpse": 29408, "sculptor": 29409, "ampton": 29410, "tpp": 29411, "indicates": 29412, "surplus": 29413, "truman": 29414, "ðĿĻ": 29415, "sinha": 29416, "invo": 29417, "sovereign": 29418, "kev": 29419, "establishing": 29420, "engraved": 29421, "assuming": 29422, "ðŁıģ": 29423, "souza": 29424, "fabi": 29425, "toned": 29426, "ounge": 29427, "deloit": 29428, "downey": 29429, "noble": 29430, "omor": 29431, "cartridge": 29432, "ðŁıIJ": 29433, "uhur": 29434, "holloway": 29435, "successes": 29436, "rsa": 29437, "âĦ¢": 29438, "mazz": 29439, "twd": 29440, "discourse": 29441, ".<": 29442, "yat": 29443, "satisfy": 29444, "compri": 29445, "ह": 29446, "graphite": 29447, "dissertation": 29448, "arter": 29449, "íĶ": 29450, "bally": 29451, "zombi": 29452, "lyons": 29453, "aic": 29454, "ubc": 29455, "prada": 29456, "eil": 29457, "dax": 29458, "clai": 29459, "granddaughter": 29460, "extravaganza": 29461, "challenge": 29462, "ðŁ¤ŀ": 29463, "pover": 29464, "primarily": 29465, "daddy": 29466, "mana": 29467, "bikers": 29468, "inquiries": 29469, "daun": 29470, "feline": 29471, "generative": 29472, "hef": 29473, "benefiting": 29474, "lindsey": 29475, "polka": 29476, "demonstrated": 29477, "alle": 29478, "randy": 29479, "osu": 29480, "lowkey": 29481, "weirdest": 29482, "redbull": 29483, "oury": 29484, "nous": 29485, "woodstock": 29486, "credenti": 29487, "nicer": 29488, "gado": 29489, "alyss": 29490, "aph": 29491, "preparedness": 29492, "stationary": 29493, "incorporated": 29494, "dyer": 29495, "saratoga": 29496, "celesti": 29497, ":\"": 29498, "antibiotics": 29499, "orgs": 29500, "indefin": 29501, "apron": 29502, "иÐ": 29503, "fifteen": 29504, "nof": 29505, "ðŁĶĿ": 29506, "phx": 29507, "tega": 29508, "mz": 29509, "organizational": 29510, "onair": 29511, "bandung": 29512, "pleasures": 29513, "mori": 29514, "secretari": 29515, "raccoon": 29516, "cashi": 29517, "pilates": 29518, "kon": 29519, "geoffrey": 29520, "lao": 29521, "kamp": 29522, "departments": 29523, "backpacking": 29524, "anam": 29525, "ë": 29526, "crackdown": 29527, "aunty": 29528, "ondo": 29529, "lizzie": 29530, "phers": 29531, "cun": 29532, "ðŁĩ±": 29533, "kpop": 29534, "put": 29535, "intentional": 29536, "connolly": 29537, "barclays": 29538, "hsfb": 29539, "swindon": 29540, "uku": 29541, "sally": 29542, "aint": 29543, "âľħ": 29544, "penang": 29545, "uplifting": 29546, "epilepsy": 29547, "interro": 29548, "bungal": 29549, "goku": 29550, "blueberries": 29551, "द": 29552, "ussia": 29553, "silky": 29554, "moured": 29555, "istic": 29556, "briefs": 29557, "meats": 29558, "gob": 29559, "chaser": 29560, "statewide": 29561, "prasad": 29562, "glitch": 29563, "arin": 29564, "banff": 29565, "member": 29566, "ðŁĺŃâĿ¤ï¸ı": 29567, "loving": 29568, "halla": 29569, "ม": 29570, "smokers": 29571, "yaku": 29572, "scicomm": 29573, "physio": 29574, "swol": 29575, "lemons": 29576, "gelato": 29577, "chool": 29578, "capitals": 29579, "kistan": 29580, "tights": 29581, "spikes": 29582, "travellers": 29583, "iklan": 29584, "commissioning": 29585, "arine": 29586, "emabiggestfans": 29587, "emphasis": 29588, "frontline": 29589, "paddock": 29590, "destructive": 29591, "baha": 29592, "linger": 29593, "jewish": 29594, "shetland": 29595, "mcgin": 29596, "monkey": 29597, "koz": 29598, "sone": 29599, "rajini": 29600, "teh": 29601, "yen": 29602, "cvs": 29603, "masquer": 29604, "girly": 29605, "wesle": 29606, "wasnt": 29607, "brody": 29608, "terminator": 29609, "gille": 29610, "maggi": 29611, "birdie": 29612, "jeopardy": 29613, "cubic": 29614, "vmware": 29615, "intricate": 29616, "anup": 29617, "topia": 29618, "easton": 29619, "sabres": 29620, "investigates": 29621, "busting": 29622, "bilingual": 29623, "valentino": 29624, "informat": 29625, "ferre": 29626, "adventur": 29627, "hydrate": 29628, "forsy": 29629, "aziz": 29630, "santo": 29631, "ede": 29632, "whistler": 29633, "continuously": 29634, "dham": 29635, "unused": 29636, "jihad": 29637, "addictive": 29638, "vidy": 29639, "dob": 29640, "ido": 29641, "fied": 29642, "niversary": 29643, "none": 29644, "fuer": 29645, "ðŁĺįðŁĺĺ": 29646, "covenant": 29647, "printable": 29648, "immaculate": 29649, "oem": 29650, "clt": 29651, "servants": 29652, "consumed": 29653, "unreleased": 29654, "scum": 29655, "packaged": 29656, "mere": 29657, "ìĦ¸ë¸": 29658, "toby": 29659, "taf": 29660, "spoons": 29661, "meal": 29662, "fball": 29663, "fairfield": 29664, "janet": 29665, "silverstone": 29666, "dartmouth": 29667, "followme": 29668, "voyager": 29669, "kombat": 29670, "anniver": 29671, "enew": 29672, "magdal": 29673, "hove": 29674, "sath": 29675, "grizzly": 29676, "cardi": 29677, "gartner": 29678, "sandy": 29679, "kanye": 29680, "posture": 29681, "poign": 29682, "impulse": 29683, "radiology": 29684, "horizons": 29685, "siam": 29686, "aishwar": 29687, "==>": 29688, "noche": 29689, "tris": 29690, "elyn": 29691, "comme": 29692, "dui": 29693, "cec": 29694, "councillors": 29695, "cuddling": 29696, "creeping": 29697, "locke": 29698, "manages": 29699, "transferred": 29700, "necks": 29701, "dier": 29702, "dano": 29703, "vick": 29704, "lunches": 29705, "dhe": 29706, "ensures": 29707, "criss": 29708, "ulster": 29709, "bannon": 29710, "contenders": 29711, "spam": 29712, "sweetness": 29713, "medal": 29714, "honduras": 29715, "arctic": 29716, "ultrasound": 29717, "infr": 29718, "discovers": 29719, "eiffel": 29720, "casters": 29721, "ruben": 29722, "dust": 29723, "aweed": 29724, "atrium": 29725, "lestwe": 29726, "seared": 29727, "ðŁĵº:": 29728, "tyne": 29729, "exchanges": 29730, "littlemix": 29731, "lle": 29732, "astronauts": 29733, "hershey": 29734, "workday": 29735, "knob": 29736, "sov": 29737, "resigns": 29738, "todayshow": 29739, "derman": 29740, "anth": 29741, "afc": 29742, "taster": 29743, "swoo": 29744, "saeed": 29745, "pering": 29746, "narrowly": 29747, "rnli": 29748, "bestbuy": 29749, "panasonic": 29750, "obstacle": 29751, "farmers": 29752, "ðŁİĻ": 29753, "pawan": 29754, "kiest": 29755, "angers": 29756, "absurd": 29757, "ohmy": 29758, "sino": 29759, "pistachi": 29760, "spice": 29761, "giuli": 29762, "primetime": 29763, "kow": 29764, "kens": 29765, "exagger": 29766, "!?!": 29767, "uba": 29768, "middles": 29769, "judd": 29770, "ejec": 29771, "slammed": 29772, "pensions": 29773, "ofa": 29774, "recreate": 29775, "bhp": 29776, "xxl": 29777, "liverpool": 29778, "thresh": 29779, "purity": 29780, "nieu": 29781, "holics": 29782, "wrath": 29783, "rado": 29784, "glio": 29785, "amma": 29786, "dilemma": 29787, "cru": 29788, "letsgo": 29789, "....@": 29790, "âĿĵ": 29791, "suggesting": 29792, "trumps": 29793, "horus": 29794, "fv": 29795, "icom": 29796, "referring": 29797, "predictive": 29798, "tarts": 29799, "gette": 29800, "sock": 29801, "glossy": 29802, "pinky": 29803, "alec": 29804, "thyme": 29805, "oura": 29806, "theroad": 29807, "petr": 29808, "cram": 29809, "pfi": 29810, "dvn": 29811, "meier": 29812, "incentives": 29813, "tunnels": 29814, "mobil": 29815, "recap": 29816, "extras": 29817, "upright": 29818, "revamp": 29819, "perseverance": 29820, ",-": 29821, "otp": 29822, "mirror": 29823, "arwx": 29824, "gerry": 29825, "maher": 29826, "gor": 29827, "homepage": 29828, "amis": 29829, "agra": 29830, "madele": 29831, "bestfriend": 29832, "siriusxm": 29833, "bundles": 29834, "admiring": 29835, "tdsb": 29836, "ðŁįģ": 29837, "chas": 29838, "slowing": 29839, "roh": 29840, "wallpapers": 29841, "âĢ¦/": 29842, "tekken": 29843, "gangs": 29844, "tala": 29845, "lindsay": 29846, "shoul": 29847, "linebacker": 29848, "toolkit": 29849, "uranium": 29850, "calyp": 29851, "abrams": 29852, "matthi": 29853, "ðŁı¿": 29854, "honourable": 29855, "dayo": 29856, "versail": 29857, "tank": 29858, "stc": 29859, "fritz": 29860, "splend": 29861, "patag": 29862, "annoyed": 29863, "onday": 29864, "devastated": 29865, "chattanooga": 29866, "nationalism": 29867, "massey": 29868, "jenn": 29869, "tailor": 29870, "devgn": 29871, "organs": 29872, "zucchini": 29873, "onfox": 29874, "satire": 29875, "wexford": 29876, "disgrace": 29877, "noto": 29878, "volta": 29879, "âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı": 29880, "à¶": 29881, "homeowners": 29882, "pointer": 29883, "mcr": 29884, "austen": 29885, "daysto": 29886, "moons": 29887, "palma": 29888, "grazing": 29889, "eso": 29890, "influencers": 29891, "shahidkapoor": 29892, "compliant": 29893, "measurements": 29894, "develops": 29895, "yd": 29896, "parl": 29897, "pvt": 29898, "randolph": 29899, "tortured": 29900, "gerald": 29901, "elias": 29902, "deepikap": 29903, "warmup": 29904, "hickory": 29905, "gap": 29906, "coffin": 29907, "amour": 29908, "reneg": 29909, "mounting": 29910, "sevens": 29911, "igle": 29912, "hier": 29913, "decad": 29914, "tright": 29915, "escapes": 29916, "werner": 29917, "tfl": 29918, "fulfilled": 29919, "niger": 29920, "sourdough": 29921, "reaper": 29922, "chooses": 29923, "spinner": 29924, "weeknd": 29925, "filtered": 29926, "shuk": 29927, "kati": 29928, "oldham": 29929, "opensource": 29930, "khanna": 29931, "atelier": 29932, "connec": 29933, "ophobic": 29934, "glas": 29935, "complications": 29936, "arson": 29937, "councils": 29938, "smol": 29939, "assy": 29940, "lurking": 29941, "lingui": 29942, "hanks": 29943, "ein": 29944, "Ùħ": 29945, "rugs": 29946, "nguyen": 29947, "nouveau": 29948, "menace": 29949, "lev": 29950, "aladdin": 29951, "ruining": 29952, "roundabout": 29953, "km": 29954, "conor": 29955, "shoops": 29956, "mayday": 29957, "traumatic": 29958, "prabhas": 29959, "kaiser": 29960, "kita": 29961, "router": 29962, "pedro": 29963, "retar": 29964, "stunner": 29965, "spanish": 29966, "disturbed": 29967, "academy": 29968, "elearning": 29969, "witty": 29970, "seng": 29971, "feral": 29972, "avy": 29973, "stab": 29974, "keaton": 29975, "urdu": 29976, "koto": 29977, "hui": 29978, "cooke": 29979, "arian": 29980, "thepersonal": 29981, "uma": 29982, "seap": 29983, "asting": 29984, "rhetoric": 29985, "handwriting": 29986, "municipality": 29987, "consortium": 29988, "ðŁIJŁ": 29989, "glasgow": 29990, "raya": 29991, "eliza": 29992, "polymer": 29993, "broth": 29994, "practi": 29995, "correspondent": 29996, "addicts": 29997, "gayle": 29998, "ailing": 29999, "ofe": 30000, "pli": 30001, "heartw": 30002, "stitch": 30003, "sightings": 30004, "priests": 30005, "samo": 30006, "sloth": 30007, "goodwood": 30008, "rocco": 30009, "sabc": 30010, "summit": 30011, "lace": 30012, "presley": 30013, "itten": 30014, "cincy": 30015, "thepersonalnetwork": 30016, "sweek": 30017, "pegas": 30018, "afcon": 30019, "registry": 30020, "cim": 30021, "leth": 30022, "dicap": 30023, "candice": 30024, "fluent": 30025, "smack": 30026, "pedestri": 30027, "aloud": 30028, "carac": 30029, "priyankach": 30030, "pgh": 30031, "irons": 30032, "dolce": 30033, "latvia": 30034, "deceased": 30035, "therock": 30036, "clap": 30037, "cene": 30038, "foam": 30039, "morrissey": 30040, "gret": 30041, "essentially": 30042, "comcast": 30043, "beagle": 30044, "argues": 30045, "inged": 30046, "-âĢ¦": 30047, "sag": 30048, "hasan": 30049, "ðŁĻĨ": 30050, "ðŁį°": 30051, "nhra": 30052, "kannada": 30053, "indicators": 30054, "oner": 30055, "brixton": 30056, "atas": 30057, "screenplay": 30058, "sorority": 30059, "shaheed": 30060, "heem": 30061, "classmates": 30062, "tainment": 30063, "esi": 30064, "breastcancer": 30065, "zuckerberg": 30066, "auror": 30067, "encia": 30068, "refers": 30069, "kaeper": 30070, "vortex": 30071, "compart": 30072, "lymph": 30073, "photographing": 30074, "steff": 30075, "restling": 30076, "parsley": 30077, "momento": 30078, "thman": 30079, "lacking": 30080, "dutt": 30081, "oculus": 30082, "fino": 30083, "frenzy": 30084, "rasc": 30085, "dern": 30086, "dismissed": 30087, "nook": 30088, "metgala": 30089, "shill": 30090, "raphael": 30091, "mavericks": 30092, "exhibits": 30093, "eagerly": 30094, "cpa": 30095, "amenities": 30096, ".âłĢ": 30097, "exodus": 30098, "ernst": 30099, "lita": 30100, "dealt": 30101, "womensmarch": 30102, "iain": 30103, "scoreboard": 30104, "campeones": 30105, "cen": 30106, "tiki": 30107, "garrison": 30108, "fidelity": 30109, "brag": 30110, "roadmap": 30111, "psychop": 30112, "loe": 30113, "bleu": 30114, "ðŁijĬðŁı¼": 30115, "sauvi": 30116, "springer": 30117, "temptation": 30118, "rudolph": 30119, "acura": 30120, "wicz": 30121, "parachute": 30122, "strol": 30123, "lenny": 30124, "zik": 30125, "doms": 30126, "nbaf": 30127, "alpac": 30128, "vivian": 30129, "rove": 30130, "preet": 30131, "perpetu": 30132, "snake": 30133, "airsoft": 30134, "inflatable": 30135, "princes": 30136, "atie": 30137, "ffey": 30138, "patient": 30139, "mire": 30140, "chelle": 30141, "slack": 30142, "groovy": 30143, "#:": 30144, "uploading": 30145, "!!!!!!!!!!!!!!!!": 30146, "siemens": 30147, "provision": 30148, "vfx": 30149, "needy": 30150, "fats": 30151, "topoli": 30152, "bhutto": 30153, "sathletics": 30154, "alums": 30155, "twinning": 30156, "southwestern": 30157, "adopting": 30158, "lastnight": 30159, "manne": 30160, "laga": 30161, "twell": 30162, "acia": 30163, "----": 30164, "eyewear": 30165, "hurley": 30166, "flee": 30167, "sach": 30168, "pecker": 30169, "costly": 30170, "isk": 30171, "crates": 30172, "policy": 30173, "erosion": 30174, "ingo": 30175, "werk": 30176, "ðŁIJį": 30177, "tortoise": 30178, "therapies": 30179, "internet": 30180, "chihuahua": 30181, "rips": 30182, "frei": 30183, "edor": 30184, "taiji": 30185, "tfc": 30186, "dod": 30187, "dempsey": 30188, "christin": 30189, "cheng": 30190, "hips": 30191, "graeme": 30192, "compassionate": 30193, "cavaliers": 30194, "historic": 30195, "soulful": 30196, "criminal": 30197, "jac": 30198, "vinci": 30199, "expired": 30200, "surat": 30201, "turismo": 30202, "kona": 30203, "seaweed": 30204, "berts": 30205, "leica": 30206, "expressing": 30207, "aal": 30208, "wort": 30209, "breakfast": 30210, "herring": 30211, "amused": 30212, "rhubarb": 30213, "martian": 30214, "cosplayer": 30215, "yash": 30216, "strial": 30217, "raul": 30218, "referral": 30219, "dwts": 30220, "jw": 30221, "adler": 30222, "curtains": 30223, "gur": 30224, "valence": 30225, "tyrone": 30226, "swfc": 30227, "coached": 30228, "reborn": 30229, "diabetic": 30230, "choke": 30231, "norfolk": 30232, "investigative": 30233, "ðŁĴ¯ðŁĴ¯": 30234, "zid": 30235, "vmas": 30236, "phie": 30237, "objectives": 30238, "âľĭ": 30239, "overdue": 30240, "divers": 30241, "matsu": 30242, "ðŁİŁï¸ı": 30243, "casualties": 30244, "ว": 30245, "alk": 30246, "standardi": 30247, "realist": 30248, "artifacts": 30249, "pandor": 30250, "kex": 30251, "invin": 30252, "(!)": 30253, "iney": 30254, "paraly": 30255, "mrt": 30256, "faye": 30257, "thevoice": 30258, "onga": 30259, "deed": 30260, "skinner": 30261, "azwx": 30262, "specimen": 30263, "priyankachopra": 30264, "nuevo": 30265, "barkley": 30266, "toulouse": 30267, "resumes": 30268, "footballers": 30269, "citi": 30270, "fetch": 30271, "ère": 30272, "lestweforget": 30273, "ðŁĻĭ": 30274, "chunk": 30275, "drifting": 30276, "manipulation": 30277, "equals": 30278, "putt": 30279, "kyungsoo": 30280, "âĿ¤ï¸ı#": 30281, "elastic": 30282, "parano": 30283, "foy": 30284, "doping": 30285, "cincy": 30286, "ssler": 30287, "interrupted": 30288, "alay": 30289, "adores": 30290, "amethy": 30291, "convoy": 30292, "ãĢı": 30293, "Ĭãģ": 30294, "blacklist": 30295, "generals": 30296, "sachin": 30297, "brushed": 30298, "ounces": 30299, "nonstop": 30300, "illiams": 30301, "btsarmy": 30302, "uav": 30303, "ruff": 30304, "burma": 30305, "bik": 30306, "defence": 30307, "schultz": 30308, "boasts": 30309, "loneliness": 30310, "gore": 30311, "transforms": 30312, "alumna": 30313, "@@": 30314, "rappers": 30315, "nehru": 30316, "caro": 30317, "himalayan": 30318, "wearables": 30319, "geh": 30320, "peppermint": 30321, "redevelopment": 30322, "flamingo": 30323, "cosby": 30324, "bigbaldhead": 30325, "agri": 30326, "barefoot": 30327, "scopes": 30328, "regram": 30329, "ghana": 30330, "ðŁİ«": 30331, "iheart": 30332, "sadie": 30333, "carrie": 30334, "microbial": 30335, "kuala": 30336, "skater": 30337, "querque": 30338, "âĻ©": 30339, "genres": 30340, "reasoning": 30341, "chased": 30342, "aso": 30343, "slipped": 30344, "encan": 30345, "vamos": 30346, "kers": 30347, "adverse": 30348, "moil": 30349, "commodities": 30350, "withyou": 30351, "silent": 30352, "hype": 30353, "ande": 30354, "amination": 30355, "whispe": 30356, "litz": 30357, "âļ½ï¸ıâļ½ï¸ı": 30358, "riff": 30359, "ppy": 30360, "lambs": 30361, "ganesh": 30362, "absent": 30363, "regulator": 30364, "marseille": 30365, "enroll": 30366, "parcel": 30367, "wap": 30368, "byrd": 30369, "ðŁĩŃ": 30370, "tuber": 30371, "countrymusic": 30372, "parl": 30373, "controllers": 30374, "responsibilities": 30375, "wey": 30376, "chate": 30377, "montenegro": 30378, "chico": 30379, "milan": 30380, "lms": 30381, "trainees": 30382, "appropriately": 30383, "uncertain": 30384, "poppies": 30385, "edsheeran": 30386, "nutritious": 30387, "garo": 30388, "deutsch": 30389, "awesome": 30390, "ãĥ¼": 30391, "comfortably": 30392, "landmarks": 30393, "eti": 30394, "reusable": 30395, "danielle": 30396, "rosal": 30397, "coles": 30398, "justic": 30399, "ccs": 30400, "fanny": 30401, "nim": 30402, "mcu": 30403, "clinch": 30404, "atene": 30405, "merge": 30406, "imdb": 30407, "anglo": 30408, "uccino": 30409, "panini": 30410, "annot": 30411, "burberry": 30412, "feature": 30413, "predicting": 30414, "fashionista": 30415, "sask": 30416, "imaginary": 30417, "mmo": 30418, "southsudan": 30419, "spear": 30420, "hubble": 30421, "jointhe": 30422, "coyotes": 30423, "sligo": 30424, "kodak": 30425, "sitcom": 30426, "polaroid": 30427, "rooted": 30428, "corrup": 30429, "ðŁĻĮðŁĻĮ": 30430, "brisban": 30431, "atz": 30432, "ahl": 30433, "remy": 30434, "talent": 30435, "avalon": 30436, "rada": 30437, "pauline": 30438, "locomotive": 30439, "goons": 30440, "nemo": 30441, "maserati": 30442, "icu": 30443, "stutt": 30444, "historically": 30445, "smb": 30446, "presby": 30447, "avoid": 30448, "sooners": 30449, "rhinestone": 30450, "wad": 30451, "rising": 30452, "trot": 30453, "modes": 30454, "regent": 30455, "optimize": 30456, "reece": 30457, "smu": 30458, "verti": 30459, "newyorkcity": 30460, "cortez": 30461, "rac": 30462, "incase": 30463, "sinc": 30464, "fielding": 30465, "etta": 30466, "tiffany": 30467, "almonds": 30468, "saddle": 30469, "krat": 30470, "matter": 30471, "glow": 30472, "starving": 30473, "glo": 30474, "crappy": 30475, "slur": 30476, "std": 30477, "monitors": 30478, "receipt": 30479, "maymayentrata": 30480, "mcil": 30481, "unis": 30482, "rainbows": 30483, "caldwell": 30484, "pacquiao": 30485, "jop": 30486, "afe": 30487, "hook": 30488, "essen": 30489, "wizard": 30490, "median": 30491, "flaws": 30492, "coms": 30493, "âĿĦ": 30494, "ingh": 30495, "haynes": 30496, "antonio": 30497, "templates": 30498, "outer": 30499, "naw": 30500, "cardigan": 30501, "belgrade": 30502, "ðŁĴī": 30503, "homo": 30504, "aise": 30505, "ropes": 30506, "nove": 30507, "whatyou": 30508, "trigge": 30509, "conception": 30510, "adukone": 30511, "nadi": 30512, "friars": 30513, "swer": 30514, "adjusted": 30515, "hotline": 30516, "sanity": 30517, "kaur": 30518, "downloading": 30519, "cgi": 30520, "tenor": 30521, "ethnic": 30522, "appalach": 30523, "ุ": 30524, "pag": 30525, "golds": 30526, "onset": 30527, "investigator": 30528, "cartel": 30529, "peacefully": 30530, "jarrett": 30531, "catalan": 30532, "polio": 30533, "num": 30534, "frustration": 30535, "dharma": 30536, "mylife": 30537, "âľĮðŁı»": 30538, "aberdeen": 30539, "musa": 30540, "binder": 30541, "sparkly": 30542, "fleeing": 30543, "instinct": 30544, "coping": 30545, "dominance": 30546, "illers": 30547, "era": 30548, "uconn": 30549, "looms": 30550, "livingston": 30551, "gali": 30552, "hes": 30553, "cma": 30554, "bela": 30555, "seley": 30556, "monk": 30557, "lach": 30558, "marx": 30559, "´": 30560, "merica": 30561, "womanin": 30562, "essex": 30563, "raina": 30564, "jimi": 30565, "neptune": 30566, "zack": 30567, "chinese": 30568, "martins": 30569, "chandelier": 30570, "hern": 30571, "withus": 30572, "earl": 30573, "asphalt": 30574, "modules": 30575, "stp": 30576, "ulla": 30577, "psychiatric": 30578, "mileage": 30579, "captivating": 30580, "sider": 30581, "mento": 30582, "mort": 30583, "trance": 30584, "talbot": 30585, "abby": 30586, "ìĥ": 30587, "âľĮðŁı¼": 30588, "jak": 30589, "dawn": 30590, "turnup": 30591, "screwed": 30592, "feds": 30593, "blueprint": 30594, "ðŁĴĸðŁĴĸ": 30595, "harsh": 30596, "eros": 30597, "insomnia": 30598, "bankers": 30599, "taemin": 30600, "misconduct": 30601, "humber": 30602, "gidi": 30603, "eduardo": 30604, "cona": 30605, "muscular": 30606, "consuming": 30607, "rash": 30608, "donnie": 30609, "dipped": 30610, "collie": 30611, "samuel": 30612, "meltdown": 30613, "ðŁĺįðŁĺįðŁĺį": 30614, "mez": 30615, "examining": 30616, "schwartz": 30617, "pristine": 30618, "ðŁIJĿ": 30619, "veit": 30620, "fulfilling": 30621, "anesthe": 30622, "guesses": 30623, "draft": 30624, "somme": 30625, "solid": 30626, "pational": 30627, "hoped": 30628, "evolutionary": 30629, "aller": 30630, "entertained": 30631, "slips": 30632, "ludwig": 30633, "concludes": 30634, "sensible": 30635, "bonnet": 30636, "craze": 30637, "tras": 30638, "hazards": 30639, "constantine": 30640, "edics": 30641, "startrek": 30642, "toc": 30643, "occupational": 30644, "incheon": 30645, "deepikapadukone": 30646, "pizzas": 30647, "newcomer": 30648, "depart": 30649, "oppression": 30650, "ebony": 30651, "fossils": 30652, "trojan": 30653, "elen": 30654, "steaks": 30655, "khou": 30656, "positioning": 30657, "ugby": 30658, "redcross": 30659, "akh": 30660, "dolce": 30661, "usmnt": 30662, "ppen": 30663, "dilig": 30664, "mavs": 30665, "caller": 30666, "costello": 30667, "âĽĦ": 30668, "dyn": 30669, "things": 30670, "rhinos": 30671, "axi": 30672, "sarkar": 30673, "convocation": 30674, "atters": 30675, "ssss": 30676, "fungus": 30677, "eugen": 30678, "russo": 30679, "squat": 30680, "wsb": 30681, "elion": 30682, "williamsburg": 30683, "soff": 30684, "deficiency": 30685, "bearer": 30686, "okin": 30687, "keystone": 30688, "twain": 30689, "calming": 30690, "breakable": 30691, "wares": 30692, "horseracing": 30693, "combs": 30694, "bunting": 30695, "uit": 30696, "tland": 30697, "ðŁĴĻðŁĴĻðŁĴĻ": 30698, "gastron": 30699, "sabot": 30700, "ickers": 30701, "commissioners": 30702, "senate": 30703, "iiot": 30704, "athena": 30705, "nitrogen": 30706, "antony": 30707, "erotic": 30708, "dialo": 30709, "missou": 30710, "hypocr": 30711, "âľĪ": 30712, "kaepernick": 30713, "canv": 30714, "droo": 30715, "cleveland": 30716, "osh": 30717, "monsta": 30718, "stefano": 30719, "^)": 30720, "shul": 30721, "poison": 30722, "hae": 30723, "commercials": 30724, "maul": 30725, "nitro": 30726, "coworker": 30727, "aloe": 30728, "vapor": 30729, "tents": 30730, "russian": 30731, "quid": 30732, "questionable": 30733, "midget": 30734, "poker": 30735, "girlfriends": 30736, "sinthe": 30737, "eritrea": 30738, "tenure": 30739, "deposits": 30740, "buckeyes": 30741, "spotter": 30742, "theodore": 30743, "trinity": 30744, "joaquin": 30745, "ucci": 30746, "followthe": 30747, "cafc": 30748, "mpa": 30749, "ðŁIJ»": 30750, "plotting": 30751, "domino": 30752, "taek": 30753, "sionally": 30754, "dicaprio": 30755, "pap": 30756, "carmel": 30757, "iger": 30758, "btcc": 30759, "bethle": 30760, "wwwbigbaldhead": 30761, "foodie": 30762, "baghdad": 30763, "masonry": 30764, "offended": 30765, "à·": 30766, "à¸ģ": 30767, "scro": 30768, "verses": 30769, "orient": 30770, "arches": 30771, "piyu": 30772, "knowyour": 30773, "gree": 30774, "takers": 30775, "guard": 30776, "dishon": 30777, "bucketlist": 30778, "bhafc": 30779, "wardly": 30780, "ðŁİīðŁİĬ": 30781, "leighton": 30782, "pew": 30783, "stray": 30784, "assaulted": 30785, "inhal": 30786, "lyfe": 30787, "amarketing": 30788, "lx": 30789, "katz": 30790, "ubuntu": 30791, "meo": 30792, "cartoonist": 30793, "turnover": 30794, "miz": 30795, "dislike": 30796, "mullen": 30797, "mof": 30798, "bland": 30799, "hides": 30800, "emerges": 30801, "chorizo": 30802, "trustee": 30803, "mahog": 30804, "lansing": 30805, "paralympic": 30806, "faint": 30807, "fauna": 30808, "chal": 30809, "snar": 30810, "cath": 30811, "benton": 30812, "castillo": 30813, "slippery": 30814, "apricot": 30815, "oecd": 30816, "baro": 30817, "lz": 30818, "heming": 30819, "clowns": 30820, "coworkers": 30821, "peruvian": 30822, "commuters": 30823, "yell": 30824, "ðŁļ´": 30825, "undering": 30826, "vj": 30827, "ttp": 30828, "flipk": 30829, "wana": 30830, "socent": 30831, "ĤâĸĤâĸ": 30832, "à¤Ĥ": 30833, "oosa": 30834, "jagger": 30835, "dism": 30836, "eless": 30837, "dham": 30838, "calif": 30839, "aofficial": 30840, "eclip": 30841, "harrogate": 30842, "grapp": 30843, "comrade": 30844, "ntr": 30845, "concentrate": 30846, "thighs": 30847, "bitcoin": 30848, "belarus": 30849, "ëĵ": 30850, "enduring": 30851, "nowwatching": 30852, "industrial": 30853, "pip": 30854, "aron": 30855, "arat": 30856, "®": 30857, "whitby": 30858, "ooooooo": 30859, "saree": 30860, "ticals": 30861, "misleading": 30862, "yoon": 30863, "years": 30864, "sleigh": 30865, "romanian": 30866, "scissors": 30867, "vampires": 30868, "acup": 30869, "abba": 30870, "thweeksary": 30871, "centri": 30872, "flye": 30873, "uo": 30874, "cbi": 30875, "buena": 30876, "sind": 30877, "marino": 30878, "burr": 30879, "rebuilding": 30880, "ल": 30881, "anniversaire": 30882, "acca": 30883, "ðŁĴĢðŁĴĢ": 30884, "getting": 30885, "tulips": 30886, "wolfpack": 30887, "âľįï¸ı": 30888, "morethan": 30889, "takin": 30890, "ðŁ¤ĺðŁı»": 30891, "ube": 30892, "monic": 30893, "doubts": 30894, "mower": 30895, "cobalt": 30896, "donne": 30897, "speculation": 30898, "arguably": 30899, "kaku": 30900, "https": 30901, "prosecution": 30902, "dinah": 30903, "stamatic": 30904, "disclosed": 30905, "beverly": 30906, "flwx": 30907, "crabs": 30908, "extraordinaire": 30909, "warmest": 30910, "imperi": 30911, "ologists": 30912, "traces": 30913, "parc": 30914, "lakeside": 30915, "amr": 30916, "teri": 30917, "hourly": 30918, "domination": 30919, "arrow": 30920, "shrewsbury": 30921, "ancestry": 30922, "wrangler": 30923, "triggered": 30924, "pensac": 30925, "rooster": 30926, "survives": 30927, "aon": 30928, "boko": 30929, "valor": 30930, "loveis": 30931, "lag": 30932, "pey": 30933, "focal": 30934, "outlaws": 30935, "blanc": 30936, "articho": 30937, "wits": 30938, "marshall": 30939, "diego": 30940, "supportsmall": 30941, "uca": 30942, "sah": 30943, "jeet": 30944, "synago": 30945, "governing": 30946, "ðŁĴ¬": 30947, "salads": 30948, "create": 30949, "miriam": 30950, "censored": 30951, "amide": 30952, "nou": 30953, "zeta": 30954, "allegiance": 30955, "*)": 30956, "blm": 30957, "rican": 30958, "pastors": 30959, "olympus": 30960, "bloc": 30961, "whirl": 30962, "starry": 30963, "prone": 30964, "yk": 30965, "pne": 30966, "congratulating": 30967, "bev": 30968, "sober": 30969, "loveisland": 30970, "sair": 30971, "aning": 30972, "tutorials": 30973, "qe": 30974, "lund": 30975, "inist": 30976, "clever": 30977, "taxpayer": 30978, "aliz": 30979, "wrench": 30980, "ddling": 30981, "capri": 30982, "hpa": 30983, "ðŁı»âĢįâĻĤï¸ı": 30984, "naj": 30985, "oj": 30986, "futuristic": 30987, "jellyfish": 30988, "ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥": 30989, "celery": 30990, "plank": 30991, "fila": 30992, "neme": 30993, "unhealthy": 30994, "lections": 30995, "ðŁ§¡": 30996, "ritchie": 30997, "nws": 30998, "mikha": 30999, "wonderwoman": 31000, "âĢİ": 31001, "hipstamatic": 31002, "kag": 31003, "ðŁĴľðŁĴľðŁĴľ": 31004, "poultry": 31005, "mow": 31006, "words": 31007, "loff": 31008, "ðŁ¤£ðŁ¤£": 31009, "relatable": 31010, "remixes": 31011, "kenyatta": 31012, "kem": 31013, "resigned": 31014, "fod": 31015, "straigh": 31016, "jlo": 31017, "hutch": 31018, "boxers": 31019, "colleen": 31020, "mags": 31021, "instructional": 31022, "kol": 31023, "attracts": 31024, "prag": 31025, "accountant": 31026, "goggles": 31027, "bru": 31028, "thole": 31029, "marrow": 31030, "leuke": 31031, "octo": 31032, "ponds": 31033, "bubbly": 31034, "heist": 31035, "ìĹij": 31036, "imp": 31037, "ahar": 31038, "haunt": 31039, "hallmark": 31040, "psych": 31041, "kkkkkkkk": 31042, "columb": 31043, "jumpsuit": 31044, "costco": 31045, "sidelines": 31046, "aggies": 31047, "overturned": 31048, "nib": 31049, "keychain": 31050, "fuk": 31051, "faf": 31052, "miam": 31053, "assistants": 31054, "cycled": 31055, "rider": 31056, "dammit": 31057, "redwings": 31058, "mages": 31059, "kins": 31060, "ìĤ": 31061, "hod": 31062, "sont": 31063, "caroline": 31064, "\"'": 31065, "cule": 31066, "braid": 31067, "felony": 31068, "arities": 31069, "rutherford": 31070, "depiction": 31071, "isabelle": 31072, "roach": 31073, "kday": 31074, "fifthharmony": 31075, "emy": 31076, "ligam": 31077, "barista": 31078, "albuquerque": 31079, "gross": 31080, "ðŁįº": 31081, "ooks": 31082, "ðŁij¼": 31083, "duncan": 31084, "tryin": 31085, "jags": 31086, "gould": 31087, "litho": 31088, "âģ£": 31089, "аÐ": 31090, "sammy": 31091, "tung": 31092, "casser": 31093, "apolo": 31094, "aaaaa": 31095, "mang": 31096, "asics": 31097, "shen": 31098, "pye": 31099, "turbul": 31100, "ssp": 31101, "saintsfc": 31102, "onlin": 31103, "nanny": 31104, "hester": 31105, "doz": 31106, "à¸Ķ": 31107, "thread": 31108, "rents": 31109, "khand": 31110, "ðŁĴªðŁı½": 31111, "unconditional": 31112, "robson": 31113, "carre": 31114, "phon": 31115, "sacrificed": 31116, "£": 31117, "autos": 31118, "parker": 31119, "oca": 31120, "login": 31121, "keegan": 31122, "hardcover": 31123, "doughnuts": 31124, "ðŁĮİ": 31125, "spitfire": 31126, "refreshments": 31127, "saskatoon": 31128, "commodore": 31129, "jf": 31130, "rubber": 31131, "halamadrid": 31132, "childcare": 31133, "strada": 31134, "iom": 31135, "rik": 31136, "dakar": 31137, "thermom": 31138, "cropped": 31139, "garu": 31140, "alik": 31141, "veni": 31142, "ift": 31143, "sika": 31144, "rituals": 31145, "zul": 31146, "ech": 31147, "©": 31148, "sudan": 31149, "lland": 31150, "ime": 31151, "docker": 31152, "ì¤": 31153, "feared": 31154, "fao": 31155, "walter": 31156, "nog": 31157, "mutuals": 31158, "lh": 31159, "align": 31160, "monia": 31161, "conceptart": 31162, "ðŁĻıðŁı¼": 31163, "scoe": 31164, "competence": 31165, "swine": 31166, "lyme": 31167, "launch": 31168, "greener": 31169, "abstractart": 31170, "inquis": 31171, "granada": 31172, "gaelic": 31173, "fluff": 31174, "dbacks": 31175, "graveyard": 31176, "babe": 31177, "academic": 31178, "adventurous": 31179, "johann": 31180, "~!": 31181, "bibi": 31182, "|#": 31183, "plings": 31184, "getty": 31185, "asb": 31186, "âĿ¤ï¸ı@": 31187, "staff": 31188, "religions": 31189, "bangor": 31190, "worldbookday": 31191, "megh": 31192, "devin": 31193, "ashore": 31194, "meridian": 31195, "github": 31196, "quiz": 31197, "allstars": 31198, "bestest": 31199, "irresi": 31200, "acker": 31201, "dote": 31202, "warrington": 31203, "polly": 31204, "neworleans": 31205, "crou": 31206, "wigs": 31207, "chey": 31208, "smithsonian": 31209, "lasag": 31210, "detour": 31211, "boris": 31212, "straps": 31213, "mariah": 31214, "intentionally": 31215, "koh": 31216, "ðŁį¸": 31217, "ssian": 31218, "marissa": 31219, "coral": 31220, "episcopal": 31221, "casualty": 31222, "tomo": 31223, "supplychain": 31224, "samp": 31225, "ongo": 31226, "roo": 31227, "caviar": 31228, "pfw": 31229, "claudio": 31230, "buffalo": 31231, "sations": 31232, "matty": 31233, "snapback": 31234, "lds": 31235, "alarms": 31236, "matte": 31237, "âĺĶï¸ı": 31238, "conditioner": 31239, "dors": 31240, "hex": 31241, "fizz": 31242, "astri": 31243, "sussex": 31244, "security": 31245, "qaeda": 31246, "allstar": 31247, "cocacola": 31248, "asone": 31249, "clicks": 31250, "scans": 31251, "mute": 31252, "heavier": 31253, "ðŁİ§": 31254, "âĺŀ": 31255, "lvl": 31256, "bookboost": 31257, "youtube": 31258, "flashes": 31259, "fjor": 31260, "csu": 31261, "explode": 31262, "dodge": 31263, "cairn": 31264, "gonzales": 31265, "thill": 31266, "pelle": 31267, "hartley": 31268, "renewable": 31269, "retin": 31270, "estre": 31271, "costarica": 31272, "shipyard": 31273, "ncfc": 31274, "priya": 31275, "aghan": 31276, "anath": 31277, "plugin": 31278, "corey": 31279, "rebound": 31280, "oru": 31281, "katrin": 31282, "hormone": 31283, "gim": 31284, "mahindra": 31285, "ssus": 31286, "parkland": 31287, "harper": 31288, "fantastic": 31289, "inferno": 31290, "epilo": 31291, "wrestling": 31292, "fect": 31293, "cit": 31294, "acoun": 31295, "tossed": 31296, "monumental": 31297, "chartered": 31298, "bust": 31299, "petra": 31300, "âĮļ": 31301, "wildflowerhour": 31302, "sweaters": 31303, "*.": 31304, "bler": 31305, "atech": 31306, "gowan": 31307, "demographic": 31308, "bral": 31309, "suicide": 31310, "renovations": 31311, "vuel": 31312, "sinister": 31313, "armani": 31314, "misogy": 31315, "pharrell": 31316, "naps": 31317, "uniting": 31318, "crusaders": 31319, "corgi": 31320, "insured": 31321, "thani": 31322, "noor": 31323, "gq": 31324, "dada": 31325, "bicycles": 31326, "snuggle": 31327, "schan": 31328, "tenberg": 31329, "ssal": 31330, "femme": 31331, "boil": 31332, "½ï¸ı": 31333, "reap": 31334, "occurring": 31335, "hussein": 31336, "divid": 31337, "stoke": 31338, "shalom": 31339, "naia": 31340, "olic": 31341, "frustrating": 31342, "Ùĩ": 31343, "igs": 31344, "grover": 31345, "scenarios": 31346, "nds": 31347, "brutality": 31348, "medalli": 31349, "buon": 31350, "sass": 31351, "skateboarding": 31352, "onyx": 31353, "lorry": 31354, "nyu": 31355, "gautam": 31356, "mmings": 31357, "gug": 31358, "endi": 31359, "lothian": 31360, "commando": 31361, "chalk": 31362, "phora": 31363, "assessing": 31364, "tigh": 31365, "crunchy": 31366, "aday": 31367, "isl": 31368, "ciara": 31369, "pilgrims": 31370, "kamal": 31371, "pto": 31372, "britanni": 31373, "tani": 31374, "smc": 31375, "lure": 31376, "appstore": 31377, "aby": 31378, "golfing": 31379, "clc": 31380, "fau": 31381, "anas": 31382, "shutting": 31383, "regulated": 31384, "carnage": 31385, "scowboys": 31386, "allenge": 31387, "cma": 31388, "humboldt": 31389, "relle": 31390, "kumb": 31391, "heri": 31392, "refinery": 31393, "soundcheck": 31394, "dwayne": 31395, "bosnia": 31396, "isp": 31397, "thealth": 31398, "anniv": 31399, "relevance": 31400, "mya": 31401, "baggage": 31402, "dread": 31403, "sbc": 31404, "thed": 31405, "buh": 31406, "hijab": 31407, "loid": 31408, "kew": 31409, "cte": 31410, "respect": 31411, "lovelies": 31412, "cubes": 31413, "celebrate": 31414, "dirt": 31415, "savers": 31416, "_,": 31417, "garment": 31418, "pulitzer": 31419, "masjid": 31420, "beatport": 31421, "alarts": 31422, "encryption": 31423, "sner": 31424, "pleads": 31425, "foundry": 31426, "symmetry": 31427, "rumi": 31428, "birthplace": 31429, "scallops": 31430, "supple": 31431, "pivotal": 31432, "tati": 31433, "node": 31434, "sod": 31435, "proxim": 31436, "trics": 31437, "coldest": 31438, "brent": 31439, "mandu": 31440, "clair": 31441, "each": 31442, "andalu": 31443, "hiddleston": 31444, "ðŁIJº": 31445, "melts": 31446, "vance": 31447, "pinn": 31448, "sements": 31449, "screened": 31450, "sachs": 31451, "obl": 31452, "icha": 31453, "âĺĺï¸ı": 31454, "schoolers": 31455, "healed": 31456, "logged": 31457, "ðŁ¤ĺðŁı¼": 31458, "icus": 31459, "boredom": 31460, "bish": 31461, "bffs": 31462, "talking": 31463, "suresh": 31464, "hookem": 31465, "deon": 31466, "defl": 31467, "eileen": 31468, "ðŁįķ": 31469, "womenintech": 31470, "risotto": 31471, "ranger": 31472, "advertise": 31473, "à¸ģà¸": 31474, "telly": 31475, "lago": 31476, "dartmoor": 31477, "dong": 31478, "skates": 31479, "logo": 31480, "unner": 31481, "mailbox": 31482, "masala": 31483, "looooo": 31484, "amethyst": 31485, "chewing": 31486, "cbb": 31487, "australians": 31488, "rcmp": 31489, "gameart": 31490, "#...": 31491, "korn": 31492, "extremism": 31493, "fruitful": 31494, "ancient": 31495, "pubg": 31496, "polite": 31497, "whit": 31498, "murals": 31499, "mgr": 31500, "lineman": 31501, "davao": 31502, "stems": 31503, "tennis": 31504, "avage": 31505, "tupac": 31506, "gigantic": 31507, "hsbc": 31508, "autobiography": 31509, "upthe": 31510, "ีà¹Ī": 31511, "regal": 31512, "figuring": 31513, "kul": 31514, "missy": 31515, "hoop": 31516, "gras": 31517, "forums": 31518, "backlash": 31519, "abducted": 31520, "pnw": 31521, "minic": 31522, "butt": 31523, "bottoms": 31524, "aton": 31525, "veng": 31526, "ðŁĮı": 31527, "delaney": 31528, "prabhu": 31529, "fanclub": 31530, "overhaul": 31531, "healthye": 31532, "syno": 31533, "aaf": 31534, "renamed": 31535, "kimi": 31536, "uncle": 31537, "mancity": 31538, "seu": 31539, "quanti": 31540, "esteem": 31541, "umin": 31542, "enzo": 31543, "melvin": 31544, "undergo": 31545, "jhar": 31546, "farah": 31547, "coasters": 31548, "humphrey": 31549, "mhz": 31550, "childrens": 31551, "^.": 31552, "dhi": 31553, "disruptive": 31554, "integrating": 31555, "rnb": 31556, "oversized": 31557, "aide": 31558, "neau": 31559, "documentation": 31560, "ðŁijĢðŁijĢ": 31561, "palo": 31562, "hearth": 31563, "riyad": 31564, "punctu": 31565, "abcnews": 31566, "secures": 31567, "boyband": 31568, "birch": 31569, "juco": 31570, "traff": 31571, "legislators": 31572, "baya": 31573, "ãĤ¯": 31574, "noises": 31575, "collects": 31576, "swarm": 31577, "kner": 31578, "bishops": 31579, "sturgeon": 31580, "snapping": 31581, "mol": 31582, "freaky": 31583, "chairperson": 31584, "trop": 31585, "lynch": 31586, "carcin": 31587, "artsy": 31588, "esto": 31589, "chai": 31590, "flur": 31591, "invali": 31592, "sausages": 31593, "imel": 31594, "jor": 31595, "funfact": 31596, "witter": 31597, "punished": 31598, "acons": 31599, "hya": 31600, "reversi": 31601, "emc": 31602, "diffu": 31603, "zx": 31604, "spaw": 31605, "clad": 31606, "dmit": 31607, "holland": 31608, "fresco": 31609, "payroll": 31610, "abundant": 31611, "stuffing": 31612, "moro": 31613, "cny": 31614, "boycott": 31615, "wendy": 31616, "eleven": 31617, "provoc": 31618, "pilot": 31619, "trx": 31620, "bead": 31621, "climateaction": 31622, "rion": 31623, "assie": 31624, "ìĸ": 31625, "osm": 31626, "islamic": 31627, "hoar": 31628, "goodreads": 31629, "alici": 31630, "afternoons": 31631, "spokesman": 31632, "jolie": 31633, "itas": 31634, "mascara": 31635, "âĻ©âĻ«": 31636, "prevail": 31637, "beetroot": 31638, "lujah": 31639, "kli": 31640, "dodger": 31641, "»": 31642, "rule": 31643, "ln": 31644, "scream": 31645, "hobart": 31646, "colbert": 31647, "rtc": 31648, "erm": 31649, "patro": 31650, "quoting": 31651, "slive": 31652, "quest": 31653, "nonfiction": 31654, "seminary": 31655, "prosecutors": 31656, "vest": 31657, "expressway": 31658, "gge": 31659, "nautical": 31660, "etf": 31661, "ðŁİīðŁİĬ": 31662, "duration": 31663, "chaired": 31664, "thefilm": 31665, "fabio": 31666, "sheh": 31667, "cano": 31668, "ðŁĴªðŁı»": 31669, "withdraw": 31670, "!:)": 31671, "corpus": 31672, "phenom": 31673, "yelp": 31674, "lawn": 31675, "entom": 31676, "snapper": 31677, "butte": 31678, "pinball": 31679, "proxy": 31680, "libre": 31681, "allevi": 31682, "nada": 31683, "gabriel": 31684, "fowl": 31685, "eureka": 31686, "daphne": 31687, "tunes": 31688, "punched": 31689, "whore": 31690, "jog": 31691, "rential": 31692, "manners": 31693, "ope": 31694, "whufc": 31695, "guth": 31696, "revolt": 31697, "sneaker": 31698, "philharmonic": 31699, "hoste": 31700, "sovereignty": 31701, "ðŁĻıðŁĻıðŁĻı": 31702, "fishing": 31703, "sciart": 31704, "feta": 31705, "ipp": 31706, "dumping": 31707, "kelown": 31708, "giri": 31709, "digits": 31710, "salu": 31711, "sanjay": 31712, "tweeters": 31713, "spas": 31714, "colchester": 31715, "scab": 31716, "madd": 31717, "à¹Ħà¸": 31718, "Äĩ": 31719, "geddon": 31720, "marchfor": 31721, "dop": 31722, "maureen": 31723, "unplugged": 31724, "dido": 31725, "fashionblogger": 31726, "upa": 31727, "mexic": 31728, "tary": 31729, "polye": 31730, "jameson": 31731, "vt": 31732, "grinder": 31733, "maddy": 31734, "consultancy": 31735, "¬ë": 31736, "leagueoflegends": 31737, "accents": 31738, "umni": 31739, "janeiro": 31740, "tuss": 31741, "hens": 31742, "amplifier": 31743, "toshi": 31744, "prettier": 31745, "prevents": 31746, "newtown": 31747, "redwood": 31748, "vantage": 31749, "ballard": 31750, "artof": 31751, "ashe": 31752, "asion": 31753, "lacey": 31754, "apat": 31755, "grove": 31756, "à¸Ħ": 31757, "rwand": 31758, "realtors": 31759, "traitor": 31760, "bedding": 31761, "ör": 31762, "zion": 31763, "flashing": 31764, "campan": 31765, "boomer": 31766, "secretariat": 31767, "abol": 31768, "litigation": 31769, "contamination": 31770, "sedly": 31771, "shredded": 31772, "infor": 31773, "doherty": 31774, "benchmark": 31775, "roche": 31776, "skateboard": 31777, "shovel": 31778, "izz": 31779, "topper": 31780, "oster": 31781, "labyrin": 31782, "autum": 31783, "kong": 31784, "hummus": 31785, "viz": 31786, "technews": 31787, "klaus": 31788, "amusing": 31789, "socialmediamarketing": 31790, "ides": 31791, "castell": 31792, "stee": 31793, "underestimate": 31794, "calab": 31795, "paign": 31796, "billing": 31797, "unanimously": 31798, "gmb": 31799, "flyfishing": 31800, "hathaway": 31801, "commercial": 31802, "colouring": 31803, "skulls": 31804, "pivot": 31805, "tep": 31806, "tbc": 31807, "motorway": 31808, "xpress": 31809, "constructive": 31810, "puk": 31811, "underlying": 31812, "kirsten": 31813, "maniac": 31814, "chao": 31815, "sema": 31816, "chiffon": 31817, "ðŁijĮðŁı»": 31818, "verona": 31819, "komo": 31820, "standoff": 31821, "wiped": 31822, "cated": 31823, "blair": 31824, "workin": 31825, "msc": 31826, "bethlehem": 31827, "swipe": 31828, "unexpec": 31829, "pees": 31830, "petri": 31831, "origami": 31832, "ðŁijħ": 31833, "mexico": 31834, "flavor": 31835, "rudd": 31836, "cannabis": 31837, "maru": 31838, "riddle": 31839, "worshi": 31840, "silon": 31841, "schat": 31842, "apse": 31843, "tanger": 31844, "bious": 31845, "eer": 31846, "questioned": 31847, "ozar": 31848, "dank": 31849, "anglesey": 31850, "charan": 31851, "baku": 31852, "competen": 31853, "repri": 31854, "batter": 31855, "saxon": 31856, "calves": 31857, "lengths": 31858, "$$$": 31859, "âŀ¡ï¸ı": 31860, "immersion": 31861, "gaunt": 31862, "carry": 31863, "cyto": 31864, "banda": 31865, "shutt": 31866, "experience": 31867, "elgin": 31868, "mousse": 31869, "taz": 31870, "êµ": 31871, "incorrect": 31872, "enz": 31873, "bham": 31874, "moron": 31875, "sover": 31876, "arun": 31877, "tipped": 31878, "lable": 31879, "dearly": 31880, "bautista": 31881, "íĻ": 31882, "mortal": 31883, "woop": 31884, "dtla": 31885, "shocks": 31886, "davos": 31887, "ðŁĵĿ": 31888, "swimwear": 31889, "herman": 31890, "ðŁijĩðŁijĩ": 31891, "zir": 31892, "neglected": 31893, "graced": 31894, "campuses": 31895, "avs": 31896, "arora": 31897, "swachhb": 31898, "livepd": 31899, "accra": 31900, "enquiries": 31901, "shooters": 31902, "kurt": 31903, "vancouver": 31904, "bradley": 31905, "garda": 31906, "gü": 31907, "olla": 31908, "attracting": 31909, "upton": 31910, "newin": 31911, "lumia": 31912, "furnace": 31913, "evers": 31914, "eon": 31915, "swa": 31916, "rookies": 31917, "aoc": 31918, "vss": 31919, "brisket": 31920, "torch": 31921, "yoda": 31922, "heartland": 31923, "taco": 31924, "phony": 31925, "foodbank": 31926, "abbey": 31927, "babylon": 31928, "uy": 31929, "greate": 31930, "expresses": 31931, "dandy": 31932, "scapes": 31933, "survivor": 31934, "rond": 31935, "eci": 31936, "havin": 31937, "abel": 31938, "childish": 31939, "torque": 31940, "wavy": 31941, "urself": 31942, "kanyewest": 31943, "yearof": 31944, "alestine": 31945, "obrien": 31946, "alfon": 31947, "skag": 31948, "korean": 31949, "anchorage": 31950, "valeri": 31951, "dew": 31952, "ðŁİ¨": 31953, "landslide": 31954, "carole": 31955, "christen": 31956, "gophers": 31957, "afi": 31958, "priyanka": 31959, "qq": 31960, "powerof": 31961, "itte": 31962, "pcso": 31963, "twol": 31964, "pry": 31965, "intellectu": 31966, "guerrero": 31967, "piles": 31968, "wishlist": 31969, "wren": 31970, "timetable": 31971, "ëı": 31972, "prodigy": 31973, "gibbons": 31974, "./": 31975, "neur": 31976, "anzac": 31977, "murray": 31978, "viest": 31979, "plaster": 31980, "lair": 31981, "artgallery": 31982, "intercontinental": 31983, "gbr": 31984, "bellator": 31985, "namjoon": 31986, "mammals": 31987, "amel": 31988, "yaw": 31989, "sarasota": 31990, "camar": 31991, "budding": 31992, "summari": 31993, "acosta": 31994, "lash": 31995, "eyou": 31996, "postgraduate": 31997, "instructors": 31998, "tig": 31999, "constant": 32000, "werewolf": 32001, "icos": 32002, "clas": 32003, "glenn": 32004, "budge": 32005, "ðŁĻĤ": 32006, "erta": 32007, "stains": 32008, "persecution": 32009, "cumbri": 32010, "och": 32011, "synergy": 32012, "huang": 32013, "scandin": 32014, "midterms": 32015, "commentator": 32016, "regarded": 32017, "perpetual": 32018, "boiling": 32019, "alp": 32020, "lange": 32021, "schle": 32022, "faceli": 32023, "tweeta": 32024, "ridden": 32025, "oktoberfest": 32026, "charlottesville": 32027, "iklan": 32028, "jou": 32029, "chatham": 32030, "bsc": 32031, "ðŁį¦": 32032, "strauss": 32033, "mellow": 32034, "xxxx": 32035, "happyhour": 32036, "reactor": 32037, "wwer": 32038, "distraction": 32039, "atorial": 32040, "ðŁĴªðŁı¼": 32041, "twinpeaks": 32042, "fayette": 32043, "aor": 32044, "kok": 32045, "broom": 32046, "syfy": 32047, "ouse": 32048, "amag": 32049, "Ø·": 32050, "ubisoft": 32051, "lulu": 32052, "hallmark": 32053, "stuart": 32054, "itya": 32055, "sideline": 32056, "vengeance": 32057, "relu": 32058, "sexism": 32059, "bouncing": 32060, "unites": 32061, "gustav": 32062, "tessa": 32063, "stump": 32064, "proclamation": 32065, "imax": 32066, "dividend": 32067, "colby": 32068, "ðŁįİ": 32069, "playwright": 32070, "unsafe": 32071, "cosmo": 32072, "ðŁĩ²ðŁĩ½": 32073, "cupboard": 32074, "constituents": 32075, "anglia": 32076, "rampage": 32077, "ðŁĺįðŁĺįðŁĺįðŁĺįðŁĺį": 32078, "thanked": 32079, "takeaways": 32080, "shroff": 32081, "debat": 32082, "khur": 32083, "conducts": 32084, "formats": 32085, "à©": 32086, "portage": 32087, "graphers": 32088, "uten": 32089, "prem": 32090, "moines": 32091, "condemns": 32092, "sous": 32093, "lps": 32094, "fcs": 32095, "dealership": 32096, "leukemia": 32097, "bureau": 32098, "skid": 32099, "guardiola": 32100, "caster": 32101, "third": 32102, "avoided": 32103, "encyclo": 32104, "csr": 32105, "vixx": 32106, "analyzing": 32107, "shear": 32108, "duluth": 32109, "shapiro": 32110, "chanting": 32111, "stresses": 32112, "asbe": 32113, "militia": 32114, "ãĥª": 32115, "collin": 32116, "arsene": 32117, "suresh": 32118, "teachings": 32119, "yixing": 32120, "shill": 32121, "nudes": 32122, "svu": 32123, "clearwater": 32124, "warped": 32125, "prolife": 32126, "artistson": 32127, "itu": 32128, "versailles": 32129, "galaxy": 32130, "axel": 32131, "springst": 32132, "cala": 32133, "huhu": 32134, "scu": 32135, "commitments": 32136, "exeter": 32137, "poignant": 32138, "motion": 32139, "conservatory": 32140, "rowdy": 32141, "recalled": 32142, "musk": 32143, "embelli": 32144, "sothe": 32145, "âĺĢ": 32146, "stopper": 32147, "schild": 32148, "tope": 32149, "elmo": 32150, "ziel": 32151, "jom": 32152, "barnsley": 32153, "snowden": 32154, "ontour": 32155, "journey": 32156, "hillsborough": 32157, "parole": 32158, "wts": 32159, "moving": 32160, "agility": 32161, "tivo": 32162, "ffers": 32163, "kindleunlimited": 32164, "gwen": 32165, "annan": 32166, "ahmad": 32167, "textured": 32168, "hepatitis": 32169, "dram": 32170, "insiders": 32171, "tissues": 32172, "ãĥĦ": 32173, "fcbarcelona": 32174, "cratic": 32175, "naacp": 32176, "pecan": 32177, "fgm": 32178, "customize": 32179, "concert": 32180, "gsm": 32181, "peg": 32182, "pone": 32183, "justintrudeau": 32184, "supercars": 32185, "happyholidays": 32186, "bular": 32187, "adox": 32188, "laptops": 32189, "digitalhealth": 32190, "destination": 32191, "gradually": 32192, "áĥ¦": 32193, "poppy": 32194, "ssl": 32195, "inhibit": 32196, "starlight": 32197, "offro": 32198, "gloomy": 32199, "xper": 32200, "halder": 32201, "implants": 32202, "leto": 32203, "hassel": 32204, "aas": 32205, "untold": 32206, "enci": 32207, "liberia": 32208, "oran": 32209, "contests": 32210, "ilah": 32211, "smag": 32212, "scout": 32213, "marianne": 32214, "cryo": 32215, "scheduling": 32216, "los": 32217, "kane": 32218, "stuttgart": 32219, "nese": 32220, "lawrence": 32221, "dain": 32222, "photom": 32223, "carou": 32224, "ร": 32225, "gwy": 32226, "nationaldogday": 32227, "roasting": 32228, "bandcamp": 32229, "kentucky": 32230, "stretches": 32231, "kerel": 32232, "cashe": 32233, "ãĤ¸": 32234, "stax": 32235, "transi": 32236, "doggie": 32237, "atric": 32238, "halle": 32239, "civic": 32240, "browning": 32241, "leinster": 32242, "catday": 32243, "highland": 32244, "joyous": 32245, "incumb": 32246, "orlando": 32247, "romo": 32248, "colton": 32249, "delta": 32250, "carab": 32251, "rotc": 32252, "asteroid": 32253, "goosebumps": 32254, "mology": 32255, "yoko": 32256, "ands": 32257, "tomorrows": 32258, "redcarpet": 32259, "smp": 32260, "casio": 32261, "ðŁ¤£ðŁ¤£ðŁ¤£": 32262, "seau": 32263, "rejection": 32264, "rotating": 32265, "bipartisan": 32266, "thun": 32267, "mati": 32268, "boni": 32269, "oll": 32270, "energye": 32271, "doit": 32272, "lj": 32273, "motherhood": 32274, "louise": 32275, "necklaces": 32276, "elite": 32277, "nix": 32278, "lcs": 32279, "env": 32280, "glu": 32281, "lesh": 32282, "crank": 32283, "susie": 32284, "mclau": 32285, "sotu": 32286, "crowley": 32287, "ratri": 32288, "used": 32289, "breton": 32290, "alfredo": 32291, "yeo": 32292, "travelpics": 32293, "tipp": 32294, "ellison": 32295, "saxophone": 32296, "mered": 32297, "heughan": 32298, "taine": 32299, "fes": 32300, "viro": 32301, "supposedly": 32302, "ias": 32303, "digestive": 32304, "yle": 32305, "lizzy": 32306, "wildlifephotography": 32307, "brianna": 32308, "westfield": 32309, "rained": 32310, "amher": 32311, "ðŁĺĦðŁĺĦ": 32312, "distribute": 32313, "bottom": 32314, "preserving": 32315, "oiland": 32316, "crafty": 32317, "descen": 32318, "colling": 32319, "shakespearesunday": 32320, "rwc": 32321, "angled": 32322, "cian": 32323, "tations": 32324, "montage": 32325, "meyers": 32326, "francesca": 32327, "ðŁĮ·": 32328, "wiggins": 32329, "sanford": 32330, "volunteer": 32331, "carra": 32332, "bark": 32333, "varied": 32334, "plin": 32335, "amu": 32336, "kapil": 32337, "rockers": 32338, "quind": 32339, "brane": 32340, "inmate": 32341, "ental": 32342, "improvis": 32343, "michigan": 32344, "retweeting": 32345, "progressing": 32346, "mercedesbenz": 32347, "smoker": 32348, "physiology": 32349, "dorado": 32350, "wattpad": 32351, "hwa": 32352, "srbachchan": 32353, "wga": 32354, "volatility": 32355, "hire": 32356, "acap": 32357, "wnba": 32358, "heinz": 32359, "stitches": 32360, "kidnapping": 32361, "burys": 32362, "limb": 32363, "fitters": 32364, "thumbnail": 32365, "tone": 32366, "mirand": 32367, "desirable": 32368, "addison": 32369, "taran": 32370, "tamilnadu": 32371, "spectator": 32372, "sociology": 32373, "amitshah": 32374, "remotely": 32375, "âĻ¦": 32376, "hamid": 32377, "rds": 32378, "glee": 32379, "smoothly": 32380, "schro": 32381, "erc": 32382, "laliga": 32383, "heals": 32384, "usf": 32385, "nishi": 32386, "dhu": 32387, "unil": 32388, "hle": 32389, "tromb": 32390, "bhutan": 32391, "pilipinas": 32392, "seung": 32393, "whitman": 32394, "tey": 32395, "mince": 32396, "snowboarding": 32397, "reau": 32398, "kker": 32399, "avo": 32400, "zachary": 32401, "ranveer": 32402, "tik": 32403, "govern": 32404, "qual": 32405, "becky": 32406, "anthropology": 32407, "atten": 32408, "groceries": 32409, "debit": 32410, "warp": 32411, "silicon": 32412, "hawaii": 32413, "ðŁĴħ": 32414, "pomegranate": 32415, "peer": 32416, "oranges": 32417, "peopleschoice": 32418, "endure": 32419, "ðŁĴĽðŁĴĽ": 32420, "ãĤ¹ãĥ": 32421, "acial": 32422, "ahaha": 32423, "stuk": 32424, "imperial": 32425, "blond": 32426, "powder": 32427, "knots": 32428, "vince": 32429, "woodlands": 32430, "dena": 32431, "watchin": 32432, "matcha": 32433, "mahat": 32434, "galaxies": 32435, "middlesbrough": 32436, "kö": 32437, "stree": 32438, "rescues": 32439, "waldo": 32440, "leroy": 32441, "despic": 32442, "realities": 32443, "tmnt": 32444, "haq": 32445, "uno": 32446, "pec": 32447, "bollywood": 32448, "blinds": 32449, "designthinking": 32450, "hems": 32451, "andhra": 32452, "absen": 32453, "fans": 32454, "stech": 32455, "shirehour": 32456, "blaine": 32457, "shakti": 32458, "purely": 32459, "ðŁıı": 32460, "trafal": 32461, "keynes": 32462, "grate": 32463, "tobias": 32464, "spontaneous": 32465, "saturated": 32466, "cavalry": 32467, "prisc": 32468, "ðŁĺij": 32469, "wht": 32470, "passi": 32471, "~~~": 32472, "virat": 32473, "pattinson": 32474, "lao": 32475, "weirdo": 32476, "sympathy": 32477, "juda": 32478, "occasionally": 32479, "credited": 32480, "statu": 32481, "esco": 32482, "hilly": 32483, "escape": 32484, "discharge": 32485, "seer": 32486, "maynard": 32487, "sudbury": 32488, "zlat": 32489, "oral": 32490, "weer": 32491, "encountered": 32492, "smelling": 32493, "oversight": 32494, "ê¸": 32495, "thatcher": 32496, "mackay": 32497, "youcan": 32498, "freep": 32499, "freedoms": 32500, "prophecy": 32501, "hoe": 32502, "ishqba": 32503, "drake": 32504, "quits": 32505, "pelled": 32506, "turk": 32507, "ovi": 32508, "wesleyan": 32509, "newmusic": 32510, "legg": 32511, "cheng": 32512, "hilli": 32513, "ayy": 32514, "panties": 32515, "adversity": 32516, "adjac": 32517, "vaccination": 32518, "juke": 32519, "gac": 32520, "exceed": 32521, "timesof": 32522, "staining": 32523, "epcot": 32524, "vital": 32525, "upward": 32526, "bethesda": 32527, "apark": 32528, "mahi": 32529, "campfire": 32530, "enchanting": 32531, "rhapso": 32532, "hz": 32533, "naver": 32534, "fax": 32535, "validation": 32536, "acad": 32537, "nyr": 32538, "asym": 32539, "coordinated": 32540, "departed": 32541, "allery": 32542, "varies": 32543, "sprite": 32544, "chaplin": 32545, "ssoccer": 32546, "swat": 32547, "bret": 32548, "reluct": 32549, "tunesapp": 32550, "superstar": 32551, "reminiscing": 32552, "oco": 32553, "homegrown": 32554, "doughnut": 32555, "uncanny": 32556, "lapd": 32557, "thyroid": 32558, "!âĿ¤ï¸ı": 32559, "botanic": 32560, "bres": 32561, "spade": 32562, "iste": 32563, "echoes": 32564, "dulil": 32565, "bursting": 32566, "quiero": 32567, "ðŁijİ": 32568, "loyola": 32569, "amusement": 32570, "hails": 32571, "sleepy": 32572, "burglary": 32573, "âľı": 32574, "rogue": 32575, "cotland": 32576, "moors": 32577, "lower": 32578, "wicked": 32579, "ðŁĶĬ": 32580, "competiti": 32581, "argentine": 32582, "yvonne": 32583, "kartikeyan": 32584, "iliary": 32585, "gatsby": 32586, "precinct": 32587, "sixty": 32588, "naji": 32589, "cams": 32590, "practitioner": 32591, "ðŁĺ³ðŁĺ³": 32592, "pune": 32593, "negli": 32594, "julien": 32595, "invaded": 32596, "calibr": 32597, "clam": 32598, "dubai": 32599, "muk": 32600, "lantic": 32601, "product": 32602, "fedex": 32603, "ï¸ı:": 32604, "eura": 32605, "darius": 32606, "sling": 32607, "virtualreality": 32608, "homestead": 32609, "ðŁı³ï¸ıâĢįðŁĮĪ": 32610, "paced": 32611, "inha": 32612, "pulmon": 32613, "lazy": 32614, "premiering": 32615, "mastered": 32616, "inhe": 32617, "congregation": 32618, "bajo": 32619, "sporting": 32620, "newjersey": 32621, "horny": 32622, "lmaoo": 32623, "lengthy": 32624, "dut": 32625, "yogh": 32626, "swearing": 32627, "philosophical": 32628, "papua": 32629, "inski": 32630, "knowles": 32631, "dyke": 32632, "âĢ²": 32633, "token": 32634, "mcguire": 32635, "riot": 32636, "probability": 32637, "mccon": 32638, "gros": 32639, "sumat": 32640, "cite": 32641, "daa": 32642, "onda": 32643, "maddow": 32644, "chew": 32645, "boardgames": 32646, "sparked": 32647, "reclaimed": 32648, "adhd": 32649, "nyse": 32650, "imwithher": 32651, "equinox": 32652, "booths": 32653, "balsamic": 32654, "hazy": 32655, "dorchester": 32656, "agos": 32657, "seaw": 32658, "moderator": 32659, "seriea": 32660, "andersen": 32661, "pilgrim": 32662, "âŃIJâŃIJ": 32663, "itchen": 32664, "halli": 32665, "xton": 32666, "nathaniel": 32667, "munition": 32668, "celestial": 32669, "gaf": 32670, "zoom": 32671, "markle": 32672, "penthouse": 32673, "cale": 32674, "sfa": 32675, "barking": 32676, "tucket": 32677, "emery": 32678, "calorie": 32679, "lique": 32680, "adar": 32681, "mcnam": 32682, "tortilla": 32683, "woodpecker": 32684, "motown": 32685, "badger": 32686, "ayrshire": 32687, "scramble": 32688, "dday": 32689, "craziest": 32690, "perrie": 32691, "choco": 32692, "caste": 32693, "iot": 32694, "wrecked": 32695, "selecting": 32696, "ussr": 32697, "graft": 32698, "punt": 32699, "labou": 32700, "irst": 32701, "baek": 32702, "ÛĮ": 32703, "suki": 32704, "queu": 32705, "achat": 32706, "tester": 32707, "augmented": 32708, "wcvb": 32709, "sinks": 32710, "ðŁĵ»": 32711, "rake": 32712, "interne": 32713, "because": 32714, "bellevue": 32715, "unearth": 32716, "lighten": 32717, "ðŁĺ£": 32718, "turnaround": 32719, "labeled": 32720, "unemployed": 32721, "twitterkurds": 32722, "leia": 32723, "hye": 32724, "greater": 32725, "ðŁIJİ": 32726, "timed": 32727, "ired": 32728, "ett": 32729, "limitations": 32730, "cabe": 32731, "sout": 32732, "beech": 32733, "annihil": 32734, "retrac": 32735, "yoona": 32736, "anger": 32737, "dennis": 32738, "supplying": 32739, "diz": 32740, "\"(": 32741, "scur": 32742, "gunman": 32743, "suho": 32744, "sauvignon": 32745, "ล": 32746, "wiley": 32747, "landon": 32748, "choreography": 32749, "prehistoric": 32750, "ðŁıĥ": 32751, "vargas": 32752, "assessments": 32753, "pinnacle": 32754, "dii": 32755, "chamberlain": 32756, "ìĪ": 32757, "vp": 32758, "presenters": 32759, "deutsche": 32760, "sunshine": 32761, "salutes": 32762, "rone": 32763, "busiest": 32764, "-.-": 32765, "motorists": 32766, "hemisphere": 32767, "alwx": 32768, "psp": 32769, "owa": 32770, "denying": 32771, "choc": 32772, "gutier": 32773, "hanuk": 32774, "muskete": 32775, "jaitley": 32776, "sewage": 32777, "tame": 32778, "thinkers": 32779, "shim": 32780, "sequo": 32781, "papar": 32782, "middleeast": 32783, "kwa": 32784, "keg": 32785, "patagonia": 32786, "noy": 32787, "barça": 32788, "takeoff": 32789, "hea": 32790, "à¬": 32791, "nsc": 32792, "gdc": 32793, "ðŁijĪ": 32794, "moustache": 32795, "melania": 32796, "thra": 32797, "â¬Ĩï¸ı": 32798, "pierced": 32799, "zeus": 32800, "fonts": 32801, "bera": 32802, "itiner": 32803, "qatar": 32804, "contrary": 32805, "ireland": 32806, "ify": 32807, "oulos": 32808, "communal": 32809, "fins": 32810, "unpaid": 32811, "paa": 32812, "ðŁijĩðŁı»": 32813, "rios": 32814, "oup": 32815, "filler": 32816, "cafeteria": 32817, "à¸Ń": 32818, "kasi": 32819, "caliber": 32820, "zulu": 32821, "vsco": 32822, "tsford": 32823, "dragonfly": 32824, "smokin": 32825, "pist": 32826, "psychologist": 32827, "diplomat": 32828, "webs": 32829, "buccane": 32830, "ா": 32831, "motivational": 32832, "dune": 32833, "bae": 32834, "cfs": 32835, "without": 32836, "eron": 32837, "iac": 32838, "atee": 32839, "pension": 32840, "frazier": 32841, "ensis": 32842, "skis": 32843, "parting": 32844, "gery": 32845, "territories": 32846, "nachos": 32847, "enight": 32848, "everlasting": 32849, "msdhoni": 32850, "tele": 32851, "spun": 32852, "podi": 32853, "sabah": 32854, "environmentally": 32855, "cease": 32856, "beaumont": 32857, "marta": 32858, "kelvin": 32859, "hoff": 32860, "sunil": 32861, "nda": 32862, "cob": 32863, "shale": 32864, "reedus": 32865, "unboxing": 32866, "ubio": 32867, "reopened": 32868, "nall": 32869, "capsules": 32870, "marr": 32871, "himalayas": 32872, "sweeter": 32873, "jaz": 32874, "fmr": 32875, "tweeter": 32876, "dhaka": 32877, "nau": 32878, "demi": 32879, "dfs": 32880, "taurus": 32881, "fading": 32882, "itutes": 32883, "cip": 32884, "overflow": 32885, "jeffrey": 32886, "donny": 32887, "cartunesapp": 32888, "ðŁįij": 32889, "prefecture": 32890, "danced": 32891, "cpt": 32892, "pleasing": 32893, "italk": 32894, "earthquakes": 32895, "ulation": 32896, "hio": 32897, "ãĢĭ": 32898, "antan": 32899, "nutrient": 32900, "deere": 32901, "selects": 32902, "enrichment": 32903, "riti": 32904, "trampol": 32905, "blamed": 32906, "jia": 32907, "contributors": 32908, "chesapeake": 32909, "pigeons": 32910, "tribunal": 32911, "maduro": 32912, "wsu": 32913, "ilove": 32914, "efficiently": 32915, "darcy": 32916, "warms": 32917, "arra": 32918, "ecu": 32919, "hower": 32920, "struggled": 32921, "rajinikanth": 32922, "ðŁĺ¢ðŁĺ¢": 32923, "housing": 32924, "strat": 32925, "elix": 32926, "dispro": 32927, "raffic": 32928, "thierry": 32929, "nasty": 32930, "cfb": 32931, "staffing": 32932, "alma": 32933, "backers": 32934, "henson": 32935, "skywalker": 32936, "realestate": 32937, "roos": 32938, "nessy": 32939, "chance": 32940, "cairns": 32941, "cci": 32942, "pedal": 32943, "lyft": 32944, "crossword": 32945, "waiter": 32946, "onlyin": 32947, "kruger": 32948, "kir": 32949, "alejandro": 32950, "cartier": 32951, "carrera": 32952, "repaired": 32953, "ouat": 32954, "unclear": 32955, "unbreakable": 32956, "todayin": 32957, "queries": 32958, "jody": 32959, "genital": 32960, "winner": 32961, "tol": 32962, "kelowna": 32963, "fascinated": 32964, "ãĥ¬": 32965, "srisri": 32966, "squared": 32967, "sprung": 32968, "negotiate": 32969, "privately": 32970, "aven": 32971, ">>>>>": 32972, "gical": 32973, "gavin": 32974, "chesterfield": 32975, "zumba": 32976, "orr": 32977, "natalia": 32978, "impeachment": 32979, "mnl": 32980, "carat": 32981, "critique": 32982, "credible": 32983, "tracy": 32984, "tani": 32985, "musik": 32986, "jigsaw": 32987, "gambia": 32988, "tolkien": 32989, "feu": 32990, "asper": 32991, "savory": 32992, "foxx": 32993, "fitt": 32994, "marlon": 32995, "lrt": 32996, "vell": 32997, "pbr": 32998, "imprisoned": 32999, "iom": 33000, "chul": 33001, "windshield": 33002, "kaye": 33003, "baa": 33004, "chord": 33005, "sart": 33006, "algon": 33007, "ministerial": 33008, "natgeo": 33009, "lazio": 33010, "norms": 33011, "ðŁijįðŁijį": 33012, "licking": 33013, "futbol": 33014, "unsung": 33015, "dallascowboys": 33016, "shred": 33017, "disturb": 33018, "devine": 33019, "beards": 33020, "chf": 33021, "bday": 33022, "rosso": 33023, "igor": 33024, "ayi": 33025, "siren": 33026, "kair": 33027, "stiles": 33028, "rof": 33029, "magnets": 33030, "uncover": 33031, "mouse": 33032, "banging": 33033, "sighted": 33034, "speople": 33035, "impact": 33036, "rowland": 33037, "kira": 33038, "environment": 33039, "lovethe": 33040, "psis": 33041, "mishra": 33042, "glendale": 33043, "cajun": 33044, "oche": 33045, "deception": 33046, "sexist": 33047, "straws": 33048, "sga": 33049, "buffer": 33050, "apostle": 33051, "spl": 33052, "popup": 33053, "ðŁļĹ": 33054, "rg": 33055, "uper": 33056, "ballin": 33057, "idy": 33058, "occasional": 33059, "nationalpark": 33060, "ðŁıĬ": 33061, "uan": 33062, "innovation": 33063, "ห": 33064, "teaparty": 33065, "rette": 33066, "counterfe": 33067, "bha": 33068, "recs": 33069, "igen": 33070, "ðŁĮIJ": 33071, "hummingbird": 33072, "cur": 33073, "haven": 33074, "lazar": 33075, "pueblo": 33076, "::": 33077, "zionist": 33078, "opath": 33079, "inverness": 33080, "promoter": 33081, "cartoon": 33082, "cabinets": 33083, "mahogany": 33084, "surveying": 33085, "rational": 33086, "feeling": 33087, "testify": 33088, "sow": 33089, "ocon": 33090, "ย": 33091, "neel": 33092, "maris": 33093, "solitary": 33094, "chemo": 33095, "radcliffe": 33096, "simons": 33097, "rosary": 33098, "newer": 33099, "jodie": 33100, "retali": 33101, "prawn": 33102, "paddy": 33103, "henge": 33104, "kala": 33105, "implant": 33106, "aty": 33107, "brentwood": 33108, "paradox": 33109, "enez": 33110, "redesigned": 33111, "pour": 33112, "wyd": 33113, "alde": 33114, "à¯ģ": 33115, "sold": 33116, "biomedical": 33117, "à¹Ĥ": 33118, "tttt": 33119, "matteo": 33120, "yser": 33121, "newton": 33122, "debun": 33123, "nerdy": 33124, "lool": 33125, "woon": 33126, "elisabeth": 33127, "ecc": 33128, "whi": 33129, "acho": 33130, "salvage": 33131, "salaries": 33132, "quity": 33133, "navigating": 33134, "ophthal": 33135, "consoles": 33136, "rebuilt": 33137, "opec": 33138, "asters": 33139, "shored": 33140, "setlist": 33141, "kathryn": 33142, "rhymes": 33143, "revisiting": 33144, "ashish": 33145, "lift": 33146, "repost": 33147, "soleil": 33148, "âı±": 33149, "wealth": 33150, "saat": 33151, "wec": 33152, "kingjames": 33153, "flipkart": 33154, "fieldwork": 33155, "segu": 33156, "modal": 33157, "bub": 33158, "arers": 33159, "ðŁįĴ": 33160, "clooney": 33161, "paddington": 33162, "necessity": 33163, "guthrie": 33164, "pente": 33165, "limo": 33166, "josie": 33167, "artin": 33168, "enc": 33169, "lhs": 33170, "betrayal": 33171, "infographics": 33172, "ier": 33173, "moa": 33174, "hearings": 33175, "bonjour": 33176, "symbolic": 33177, "agro": 33178, "wedges": 33179, "kristina": 33180, "wildflower": 33181, "athletic": 33182, "photography": 33183, "pesh": 33184, "cahill": 33185, "chilean": 33186, "goul": 33187, "fioren": 33188, "ðŁij¶": 33189, "zil": 33190, "skim": 33191, "badoo": 33192, "delia": 33193, "treble": 33194, "ncc": 33195, "ðŁĩ¦ðŁĩ": 33196, "ahouse": 33197, "bullock": 33198, "solitude": 33199, "اÙĨ": 33200, "cancers": 33201, "futureofwork": 33202, "hutch": 33203, "watershed": 33204, "warmongers": 33205, "spilled": 33206, "colombo": 33207, "moth": 33208, "associations": 33209, "weighed": 33210, "globalgoals": 33211, "notjust": 33212, "christi": 33213, "torg": 33214, "sweating": 33215, "maneu": 33216, "clusters": 33217, "âĢ¼ï¸ıâĢ¼ï¸ı": 33218, "taped": 33219, "uly": 33220, "trusting": 33221, "yusuf": 33222, "tein": 33223, "rab": 33224, ",,,,": 33225, "sinai": 33226, "audible": 33227, "explicit": 33228, "crowns": 33229, "schiz": 33230, "atleast": 33231, "ðŁĹ£": 33232, "debra": 33233, "jesuit": 33234, "enegger": 33235, "zhen": 33236, "onesie": 33237, "iit": 33238, "ssf": 33239, "gurgaon": 33240, "chakra": 33241, "bearcats": 33242, "kran": 33243, "kawa": 33244, "requesting": 33245, "hanover": 33246, "gend": 33247, "soros": 33248, "mercy": 33249, "lovely": 33250, "doomed": 33251, "timmy": 33252, "kuz": 33253, "ull": 33254, "abram": 33255, "saison": 33256, "ãĥ«": 33257, "cleaners": 33258, "remo": 33259, "circuits": 33260, "barred": 33261, "oth": 33262, "moist": 33263, "madeleine": 33264, "gallo": 33265, "uj": 33266, "permits": 33267, "heaviest": 33268, "carols": 33269, "azte": 33270, "giorgio": 33271, "floats": 33272, "declaring": 33273, "usrc": 33274, "minat": 33275, "crafts": 33276, "prima": 33277, "conveni": 33278, "nickelodeon": 33279, "dancing": 33280, "ceremonial": 33281, "blogg": 33282, "twp": 33283, "anglican": 33284, "shek": 33285, "knick": 33286, "(((": 33287, "hubbard": 33288, "harvey": 33289, "hitman": 33290, "feng": 33291, "wesome": 33292, "forza": 33293, "sword": 33294, "opus": 33295, "brom": 33296, "gibility": 33297, "zal": 33298, "munch": 33299, "dancehall": 33300, "greedy": 33301, "hdmi": 33302, "rebirth": 33303, "ðŁĺĭðŁĺĭ": 33304, "sworld": 33305, "figurine": 33306, "compost": 33307, "kf": 33308, "engraving": 33309, "giorno": 33310, "stana": 33311, "kman": 33312, "hamster": 33313, "composers": 33314, "aje": 33315, "functionality": 33316, "polk": 33317, "isons": 33318, "airplanes": 33319, "tese": 33320, "horrors": 33321, "muscat": 33322, "given": 33323, "spence": 33324, "ðŁĩ¸ðŁĩ": 33325, "eliot": 33326, "achilles": 33327, "freck": 33328, "cryptocurrencies": 33329, "souther": 33330, "halo": 33331, "borneo": 33332, "politic": 33333, "hahahahah": 33334, "upstate": 33335, "siena": 33336, "obscure": 33337, "hausen": 33338, "lloyd": 33339, "happyfriday": 33340, "motorbike": 33341, "bona": 33342, "americas": 33343, "hols": 33344, "-(": 33345, "sporty": 33346, "unaware": 33347, "revenues": 33348, "christopher": 33349, "banksy": 33350, "avan": 33351, "evapor": 33352, "compress": 33353, "eyeliner": 33354, "todos": 33355, "buffy": 33356, "renewableenergy": 33357, "lyrical": 33358, "archan": 33359, "rapist": 33360, "fairtrade": 33361, "lmaooo": 33362, "beatz": 33363, "proactive": 33364, "lapse": 33365, "irical": 33366, "reversal": 33367, "pode": 33368, "mcintyre": 33369, "macau": 33370, "ãĥķãĤ": 33371, "nashgrier": 33372, "fsa": 33373, "gall": 33374, "çĶŁ": 33375, "perpetr": 33376, "ilya": 33377, "configuration": 33378, "%;": 33379, "strange": 33380, "raci": 33381, "à¸ĩ": 33382, "pickups": 33383, "kovsky": 33384, "mammal": 33385, "wps": 33386, "gable": 33387, "comparative": 33388, "zh": 33389, "saveour": 33390, "davey": 33391, "onetsy": 33392, "mussels": 33393, "miser": 33394, "cristina": 33395, "electron": 33396, "crave": 33397, "loren": 33398, "precipitation": 33399, "mz": 33400, "ðŁį«": 33401, "vincen": 33402, "snowboard": 33403, "noida": 33404, "ahn": 33405, "marinated": 33406, "gtr": 33407, "townhall": 33408, "minis": 33409, "bethel": 33410, "advan": 33411, "sura": 33412, "shiel": 33413, "furry": 33414, "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ": 33415, "lynd": 33416, "soil": 33417, "scence": 33418, "seneca": 33419, "sharjah": 33420, "dickens": 33421, "credentials": 33422, "avar": 33423, "perk": 33424, "requiring": 33425, "prefer": 33426, "jian": 33427, "deca": 33428, "rach": 33429, "ingfor": 33430, "dele": 33431, "beep": 33432, "ðŁĴ»": 33433, "cisely": 33434, "huddle": 33435, "greensboro": 33436, "hawking": 33437, "hoax": 33438, "hangar": 33439, "çľ": 33440, "miso": 33441, "lovin": 33442, "greta": 33443, "abad": 33444, "logie": 33445, "atan": 33446, "snowflake": 33447, "mahesh": 33448, "fearthe": 33449, "alkal": 33450, "bobblehead": 33451, "bahn": 33452, "judged": 33453, "futu": 33454, "felix": 33455, "ðŁįĵ": 33456, "pike": 33457, "deriv": 33458, "notices": 33459, "auer": 33460, "dissuper": 33461, "orda": 33462, "wipes": 33463, "amino": 33464, "strikers": 33465, "footb": 33466, "dramas": 33467, "punching": 33468, "scoreless": 33469, "hemingway": 33470, "bih": 33471, "ballad": 33472, "chatter": 33473, "ammo": 33474, "klein": 33475, "fabrication": 33476, "karim": 33477, "zend": 33478, "histo": 33479, "volta": 33480, "rocky": 33481, "marketer": 33482, "xtreme": 33483, "sequencing": 33484, "paradigm": 33485, "cleats": 33486, "booming": 33487, "âģłâģł": 33488, "blockade": 33489, "prompts": 33490, "yoghurt": 33491, "purpose": 33492, "nur": 33493, "regulate": 33494, "noisy": 33495, "ingrid": 33496, "birdwatching": 33497, "bartender": 33498, "Ùĥ": 33499, "wordof": 33500, "chaotic": 33501, "shorty": 33502, "eldest": 33503, "zapp": 33504, "onceuponatime": 33505, "flyo": 33506, "ritos": 33507, "mikequind": 33508, "ðŁIJ´": 33509, "registering": 33510, ".]": 33511, "adol": 33512, "gggg": 33513, "purge": 33514, "kidlit": 33515, "arbor": 33516, "valves": 33517, "synagogue": 33518, "oth": 33519, "unanimous": 33520, "verification": 33521, "darrell": 33522, "ãģĦ": 33523, "vanderbilt": 33524, "tapestry": 33525, "prosper": 33526, "diddy": 33527, "drafting": 33528, "decep": 33529, "marquis": 33530, "stint": 33531, "michaeljackson": 33532, "peeled": 33533, "menus": 33534, "bbb": 33535, "scare": 33536, "email": 33537, "wrigley": 33538, "itis": 33539, "fell": 33540, "somethin": 33541, "barra": 33542, "edgar": 33543, "dipping": 33544, "puddle": 33545, "slade": 33546, "learner": 33547, "jalen": 33548, "ðŁ§IJ": 33549, "thedaily": 33550, "mikequindazzi": 33551, "jux": 33552, "iqbal": 33553, "mckinney": 33554, "raiser": 33555, "efan": 33556, "drone": 33557, "cato": 33558, "picket": 33559, "crowe": 33560, "latt": 33561, "uko": 33562, "giuseppe": 33563, "hini": 33564, "synthesi": 33565, "pontifex": 33566, "songwriting": 33567, "tod": 33568, "switches": 33569, "dinners": 33570, "hq": 33571, "gabrielle": 33572, "pensacola": 33573, "circle": 33574, "exposes": 33575, "evs": 33576, "riyadh": 33577, "promen": 33578, "ock": 33579, "saj": 33580, "citation": 33581, "brewco": 33582, "josi": 33583, "epaper": 33584, "drif": 33585, "pointless": 33586, "tangled": 33587, "cripp": 33588, "lineups": 33589, "fairies": 33590, "daze": 33591, "mourn": 33592, "bladder": 33593, "salz": 33594, "burundi": 33595, "bookmark": 33596, "thepeople": 33597, "subsequ": 33598, "principal": 33599, "sker": 33600, "courtney": 33601, "aoki": 33602, "racers": 33603, "adm": 33604, "moma": 33605, "criticalrole": 33606, "houn": 33607, "shedding": 33608, "saka": 33609, "aceous": 33610, "mckay": 33611, "husbands": 33612, "½": 33613, "meda": 33614, "accusations": 33615, "rosel": 33616, "ncis": 33617, "witnessing": 33618, "orama": 33619, "gods": 33620, "hilton": 33621, "elman": 33622, "ÃŃn": 33623, "megap": 33624, "craven": 33625, "announcer": 33626, "criteri": 33627, "sheffieldissuper": 33628, "militant": 33629, "consul": 33630, "hooded": 33631, "abyss": 33632, "bx": 33633, "madam": 33634, "locu": 33635, "maryam": 33636, "manicure": 33637, "gratis": 33638, "actresses": 33639, "rosario": 33640, "thisdayin": 33641, "kingly": 33642, "gnome": 33643, "celine": 33644, "rous": 33645, "heel": 33646, "lilac": 33647, "vishal": 33648, "abh": 33649, "thorns": 33650, "sls": 33651, "neal": 33652, "constructing": 33653, "beren": 33654, "slang": 33655, "mains": 33656, "farra": 33657, "sarko": 33658, "paige": 33659, "guiller": 33660, "lala": 33661, "iceberg": 33662, "noun": 33663, "planners": 33664, "ummm": 33665, "ouses": 33666, "illary": 33667, "maan": 33668, "boxing": 33669, "zipper": 33670, "srinagar": 33671, "miguel": 33672, "ostr": 33673, "mpo": 33674, "responsibly": 33675, "lanterns": 33676, "appliance": 33677, "xb": 33678, "grenade": 33679, "neglect": 33680, "dysle": 33681, "hammock": 33682, "nectar": 33683, "witcher": 33684, "rgv": 33685, "dience": 33686, "serbian": 33687, "seeded": 33688, "cruz": 33689, "bish": 33690, "sphe": 33691, "eq": 33692, "skyrim": 33693, "algebra": 33694, "philately": 33695, "bungalow": 33696, "geoff": 33697, "yves": 33698, "demanded": 33699, "considerations": 33700, "thevamp": 33701, "pawankalyan": 33702, "coded": 33703, "gritty": 33704, "eruption": 33705, "seinfeld": 33706, "unidenti": 33707, "ëĭĪ": 33708, "worm": 33709, "acus": 33710, "seung": 33711, "dung": 33712, "roland": 33713, "sud": 33714, "divisions": 33715, "ablanc": 33716, "shortest": 33717, "jf": 33718, "poun": 33719, "plantbased": 33720, "beto": 33721, "tougher": 33722, "mco": 33723, "donet": 33724, "markus": 33725, "vfl": 33726, "ðŁıł": 33727, "opening": 33728, "coward": 33729, "cabernet": 33730, "oxi": 33731, "burlesque": 33732, "sandra": 33733, "sumo": 33734, "consist": 33735, "thot": 33736, "cayman": 33737, "motorola": 33738, "gutierrez": 33739, "dslr": 33740, "yw": 33741, "nobel": 33742, "novice": 33743, "momsdemand": 33744, "grunge": 33745, "spor": 33746, "dcc": 33747, "presses": 33748, "slist": 33749, "allotment": 33750, "vocational": 33751, "ftc": 33752, "puja": 33753, "loven": 33754, "uttarak": 33755, "tandem": 33756, "shep": 33757, "comedians": 33758, "anatom": 33759, "cantwait": 33760, "healthyeating": 33761, "westside": 33762, "margins": 33763, "chiang": 33764, "asbestos": 33765, "stupidity": 33766, "problematic": 33767, "fitbit": 33768, ":$": 33769, "ceilings": 33770, "shua": 33771, "protections": 33772, "biotic": 33773, "bengali": 33774, "rests": 33775, "biennale": 33776, "timo": 33777, "culmin": 33778, "eminent": 33779, "affection": 33780, "unbelievably": 33781, "individually": 33782, "canvassing": 33783, "whitt": 33784, "novasco": 33785, "chinson": 33786, "hpe": 33787, "gow": 33788, "gloucestershire": 33789, "pao": 33790, "threshold": 33791, "chevron": 33792, "sine": 33793, "wether": 33794, "ppie": 33795, "aquino": 33796, "antwerp": 33797, "âĸ¬": 33798, "poon": 33799, "instaf": 33800, "equine": 33801, "cinematography": 33802, "nbafinals": 33803, "valiant": 33804, "kilkenny": 33805, "terence": 33806, "systemic": 33807, "srl": 33808, "pound": 33809, "madeira": 33810, "plough": 33811, "trecht": 33812, "mated": 33813, "mpd": 33814, "ransomware": 33815, "phin": 33816, "liqui": 33817, "bbce": 33818, "boomer": 33819, "istandwith": 33820, "conju": 33821, "rte": 33822, "nara": 33823, "foolish": 33824, "dashing": 33825, "viernes": 33826, "brite": 33827, "dau": 33828, "juniper": 33829, "aida": 33830, "younow": 33831, "razer": 33832, "dei": 33833, "repeating": 33834, "comforting": 33835, "adjacent": 33836, "eto": 33837, "casted": 33838, "chatur": 33839, "muer": 33840, "synth": 33841, "sanitary": 33842, "macle": 33843, "independent": 33844, "lawful": 33845, "eerie": 33846, "hor": 33847, "ðŁĴŃ": 33848, "amrit": 33849, "velo": 33850, "stationery": 33851, "muf": 33852, "maymay": 33853, "contemplating": 33854, "elaborate": 33855, "gregor": 33856, "dries": 33857, "accol": 33858, "à¸ļ": 33859, "schwarzenegger": 33860, "illnesses": 33861, "daybreak": 33862, "followback": 33863, "collusion": 33864, "electronic": 33865, "jovi": 33866, "hiroshima": 33867, "taw": 33868, "homec": 33869, "micah": 33870, "quitting": 33871, "frosting": 33872, "benfica": 33873, "heli": 33874, "sical": 33875, "piccad": 33876, "corporate": 33877, "mentorship": 33878, "youare": 33879, "singer": 33880, "shiva": 33881, "rune": 33882, "inger": 33883, "rium": 33884, "playable": 33885, "doop": 33886, "willow": 33887, "terre": 33888, "nip": 33889, "atd": 33890, "warbler": 33891, "professionally": 33892, "erase": 33893, "proceed": 33894, "pedestrians": 33895, "mischief": 33896, "bending": 33897, "alaskan": 33898, "ckett": 33899, "mop": 33900, "ddles": 33901, "shutter": 33902, "geared": 33903, "ateneo": 33904, "madeline": 33905, "gations": 33906, "osha": 33907, "derick": 33908, "swild": 33909, "angry": 33910, "patents": 33911, "hunk": 33912, "decreased": 33913, "fry": 33914, "ðŁĴĸðŁĴĸðŁĴĸ": 33915, "salon": 33916, "quantities": 33917, "dario": 33918, "nigel": 33919, "kuma": 33920, "jenn": 33921, "happye": 33922, "xxx": 33923, "rexperience": 33924, "pros": 33925, "ausch": 33926, "relessly": 33927, "hamburger": 33928, "fukushima": 33929, "erne": 33930, "statec": 33931, "rend": 33932, "mayfield": 33933, "jone": 33934, "lefty": 33935, "bernstein": 33936, "smil": 33937, "generates": 33938, "forestation": 33939, "bandits": 33940, "tayo": 33941, "rca": 33942, "acci": 33943, "rodrigo": 33944, "knapp": 33945, "elovers": 33946, "vegetation": 33947, "ural": 33948, "left": 33949, "ħï¸ı": 33950, "worldre": 33951, "suri": 33952, "embark": 33953, "wson": 33954, "bayou": 33955, "muller": 33956, "movers": 33957, "ðŁķº": 33958, "presbyter": 33959, "lf": 33960, "cree": 33961, "batb": 33962, "salam": 33963, "demonstrations": 33964, "anec": 33965, "npc": 33966, "itics": 33967, "tography": 33968, "reinst": 33969, "thurst": 33970, "tale": 33971, "offences": 33972, "smartcity": 33973, "brotha": 33974, "oftheyear": 33975, "invaluable": 33976, "earn": 33977, "ðŁijıðŁı½": 33978, "kremlin": 33979, "grady": 33980, "townfc": 33981, "guernsey": 33982, "maha": 33983, "contagious": 33984, "drex": 33985, "been": 33986, "(£": 33987, "nativity": 33988, "ktm": 33989, "somerhalder": 33990, "compounds": 33991, "íķĺ": 33992, "\"âĢ¦": 33993, "afg": 33994, "ottnews": 33995, "hound": 33996, "firefly": 33997, "cilan": 33998, "donetsk": 33999, "volunteered": 34000, "akira": 34001, "èª": 34002, "singul": 34003, "sth": 34004, "drowned": 34005, "mando": 34006, "heir": 34007, "ðŁİīðŁİĪ": 34008, "taxis": 34009, "yuki": 34010, "veld": 34011, "kans": 34012, "elk": 34013, "rants": 34014, "hashtag": 34015, "teng": 34016, "rog": 34017, "aat": 34018, "grub": 34019, "eber": 34020, "inindia": 34021, "colossus": 34022, "signi": 34023, "soever": 34024, "milestones": 34025, "dero": 34026, "differential": 34027, "phuket": 34028, "mastermind": 34029, "angh": 34030, "melani": 34031, "broker": 34032, "actorvijay": 34033, "stunned": 34034, "continuity": 34035, "affl": 34036, "vocal": 34037, "perennial": 34038, "fiancé": 34039, "incomplete": 34040, "hunts": 34041, "reissue": 34042, "dominates": 34043, "turmeric": 34044, "roam": 34045, "rion": 34046, "bagged": 34047, "nassau": 34048, "fut": 34049, "xox": 34050, "nationaltrust": 34051, "joye": 34052, "sano": 34053, "hearthstone": 34054, "disrespect": 34055, "lees": 34056, "hse": 34057, "siberian": 34058, "offee": 34059, "restock": 34060, "wolfgang": 34061, "regan": 34062, "plano": 34063, "unwind": 34064, "repar": 34065, "mille": 34066, "],": 34067, "skull": 34068, "fatally": 34069, "conceptual": 34070, "ðŁĮ²": 34071, "fé": 34072, "berto": 34073, "bms": 34074, "ua": 34075, "magna": 34076, "notredame": 34077, "lete": 34078, "laundering": 34079, "heartwarming": 34080, "buffett": 34081, "goat": 34082, "peabo": 34083, "windmill": 34084, "vac": 34085, "continually": 34086, "azalea": 34087, "membrane": 34088, "cancels": 34089, "makeyourown": 34090, "athered": 34091, "pto": 34092, "torpe": 34093, "ðŁĺł": 34094, "ðŁĴ§": 34095, "scares": 34096, "leaking": 34097, "zet": 34098, "pixels": 34099, "aci": 34100, "khil": 34101, "marathi": 34102, "ðŁĻıðŁı½": 34103, "ula": 34104, "tamu": 34105, "chandigarh": 34106, "zagre": 34107, "aab": 34108, "pronounced": 34109, "aubrey": 34110, "sander": 34111, "punta": 34112, "harlow": 34113, "icelan": 34114, "celebratory": 34115, "sot": 34116, "unciation": 34117, "struly": 34118, "mcdowell": 34119, "deepika": 34120, "reminders": 34121, "mystical": 34122, "ctc": 34123, "chatted": 34124, "sica": 34125, "bargains": 34126, "chhat": 34127, "rubin": 34128, "mnet": 34129, "oilandgas": 34130, "pelican": 34131, "oat": 34132, "morality": 34133, "kour": 34134, "ih": 34135, "nuclear": 34136, "gcu": 34137, "richer": 34138, "venezia": 34139, "mma": 34140, "leith": 34141, "accompany": 34142, "richmond": 34143, "sportsnet": 34144, "baahu": 34145, "smuggling": 34146, "mmi": 34147, "ðŁĩ®ðŁĩª": 34148, "twists": 34149, "sahib": 34150, ".....": 34151, "ambitions": 34152, "illo": 34153, "historical": 34154, "forec": 34155, "showbiz": 34156, "ponies": 34157, "chasers": 34158, "remodel": 34159, "willing": 34160, "princesses": 34161, "ample": 34162, "cushions": 34163, "acles": 34164, "lotr": 34165, "dach": 34166, "anthe": 34167, "incorporate": 34168, "newbury": 34169, "kiri": 34170, "friedrich": 34171, "abv": 34172, "ballers": 34173, "albert": 34174, "ðŁijŃ": 34175, "leti": 34176, "nanop": 34177, "cide": 34178, "analo": 34179, "nsf": 34180, "))))": 34181, "griffiths": 34182, "valenci": 34183, "roano": 34184, "funrun": 34185, "babysitting": 34186, "caday": 34187, "entre": 34188, "uck": 34189, "slug": 34190, "tical": 34191, "thesims": 34192, "roar": 34193, "carney": 34194, "gam": 34195, "stowe": 34196, "fid": 34197, "bunny": 34198, "shamrock": 34199, "pecu": 34200, "molina": 34201, "gocougs": 34202, "contributes": 34203, "transformation": 34204, "moy": 34205, "vaj": 34206, "severy": 34207, "antioxidants": 34208, "thirteen": 34209, "sightseeing": 34210, "lj": 34211, "reversible": 34212, "oddly": 34213, "hookah": 34214, "nouvel": 34215, "halal": 34216, "fei": 34217, "stables": 34218, "mult": 34219, "hopped": 34220, "braids": 34221, "interchange": 34222, "ghanaian": 34223, "wwww": 34224, "ethno": 34225, "conjunction": 34226, "agov": 34227, "yeti": 34228, "earthand": 34229, "tsp": 34230, "conserve": 34231, "heirloom": 34232, "metaphor": 34233, "woof": 34234, "torio": 34235, "selfless": 34236, "nwa": 34237, "emilia": 34238, "ylene": 34239, "yxe": 34240, "giar": 34241, "moderating": 34242, "probz": 34243, "bfi": 34244, "neer": 34245, "dummy": 34246, "hanukkah": 34247, "webber": 34248, "kv": 34249, "eyebrow": 34250, "dagger": 34251, "sump": 34252, "rages": 34253, "orkney": 34254, "tbo": 34255, "halsey": 34256, "assignments": 34257, "tronic": 34258, "scrib": 34259, "coon": 34260, "anwar": 34261, "#âĢİ": 34262, "jalape": 34263, "florida": 34264, "quaid": 34265, "hawkeyes": 34266, "âĻ¡âĻ¡": 34267, "streetcar": 34268, "rog": 34269, "datlantic": 34270, "granola": 34271, "unchanged": 34272, "expectation": 34273, "Ùĩ": 34274, "marlin": 34275, "gummy": 34276, "ðŁĻıðŁı¾": 34277, "awarenessmonth": 34278, "oilpainting": 34279, "muth": 34280, "perch": 34281, "junto": 34282, "villagers": 34283, "morg": 34284, "cheated": 34285, "webcomic": 34286, "thefuture": 34287, "dps": 34288, "lakings": 34289, "mentioning": 34290, "voor": 34291, "identities": 34292, "accord": 34293, "mcgu": 34294, "lpga": 34295, "rumour": 34296, "massively": 34297, "mpls": 34298, "healy": 34299, "date": 34300, "spoli": 34301, "revisited": 34302, "ont": 34303, "aland": 34304, "scrutiny": 34305, "lakeland": 34306, "blending": 34307, "": 34308, "ankara": 34309, "jamiedor": 34310, "metabolic": 34311, "fences": 34312, "anny": 34313, "åħ": 34314, "semicon": 34315, "oott": 34316, "spaceship": 34317, "wacky": 34318, "leta": 34319, "apac": 34320, "shee": 34321, "inherit": 34322, "dores": 34323, "ðŁĩ¨ðŁĩ¦": 34324, "gente": 34325, "twick": 34326, "rims": 34327, "galve": 34328, "deville": 34329, "kingfisher": 34330, "scorpio": 34331, "owl": 34332, "alar": 34333, "varian": 34334, "ðŁĹĵ": 34335, "venetian": 34336, "stardust": 34337, "thenorth": 34338, "qing": 34339, "harrington": 34340, "consulate": 34341, "spectacle": 34342, "hobbs": 34343, "turks": 34344, "greer": 34345, "mating": 34346, "ðŁİĢ": 34347, "ðŁĮĢ": 34348, "directs": 34349, "íĭ": 34350, "pompeo": 34351, "voiced": 34352, "laos": 34353, "tzu": 34354, "prome": 34355, "prism": 34356, "merc": 34357, "fortunately": 34358, "bcfc": 34359, "mcdonnell": 34360, "notsorry": 34361, "smiled": 34362, "tba": 34363, "forwar": 34364, "midterm": 34365, "darby": 34366, "weinstein": 34367, "upgrading": 34368, "wolff": 34369, "bronco": 34370, "cabello": 34371, "ðŁ¥ĩ": 34372, "fiable": 34373, "sharpe": 34374, "battered": 34375, "sato": 34376, "mythical": 34377, "instapic": 34378, "prepped": 34379, "enium": 34380, "espo": 34381, "diaper": 34382, "explanations": 34383, "whopping": 34384, "ragnar": 34385, "peel": 34386, "antibiotic": 34387, "lacks": 34388, "harrison": 34389, "lism": 34390, "aul": 34391, "quail": 34392, "martina": 34393, "sentencing": 34394, "scams": 34395, "didi": 34396, "tronics": 34397, "ãħłãħł": 34398, "goff": 34399, "zain": 34400, "paramore": 34401, "chained": 34402, "clinton": 34403, "liff": 34404, "cottages": 34405, "emon": 34406, "reverend": 34407, "consumer": 34408, "cean": 34409, "tany": 34410, "lumpur": 34411, "ebay": 34412, "stool": 34413, "ðŁĺ»ðŁĺ»": 34414, "tapro": 34415, "hath": 34416, "modernart": 34417, "justine": 34418, "proverb": 34419, "appy": 34420, "trax": 34421, "manifest": 34422, "ambu": 34423, "naik": 34424, "pepp": 34425, "rsd": 34426, "merchants": 34427, "kitchener": 34428, "shifted": 34429, "lizz": 34430, "âĺħâĺħâĺħâĺħ": 34431, "âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ": 34432, "utopia": 34433, "tomo": 34434, "outed": 34435, "comers": 34436, "chiropractic": 34437, "bookclub": 34438, "cindy": 34439, "prohibition": 34440, "seuss": 34441, "민": 34442, "thinkin": 34443, "rrrr": 34444, "gofund": 34445, "tack": 34446, "omb": 34447, "catastrophic": 34448, "lingu": 34449, "guildford": 34450, "botd": 34451, "à¥ĭ": 34452, "planter": 34453, "^^": 34454, "wink": 34455, "kathmandu": 34456, "stoppers": 34457, "smoothies": 34458, "reefs": 34459, "hind": 34460, "bellamy": 34461, "Ħë": 34462, "wastewater": 34463, "voor": 34464, "natl": 34465, "!]": 34466, "reel": 34467, "yap": 34468, "scooby": 34469, "workspace": 34470, "corinthians": 34471, "blun": 34472, "obligation": 34473, "gbbo": 34474, "dyson": 34475, "cravings": 34476, "ellington": 34477, "dapl": 34478, "wrexham": 34479, "earthandclouds": 34480, "ukrunchat": 34481, "positioned": 34482, "kalb": 34483, "foursquare": 34484, "jock": 34485, "impending": 34486, "evening": 34487, "athy": 34488, "proclaimed": 34489, "cites": 34490, "annapolis": 34491, "sani": 34492, "marth": 34493, "irl": 34494, "accommo": 34495, "kaa": 34496, "fina": 34497, "yaa": 34498, "disper": 34499, "ecar": 34500, "bhak": 34501, "willy": 34502, "ðŁĺĢðŁĺĢ": 34503, "mcdermott": 34504, "moj": 34505, "generational": 34506, "usaid": 34507, "training": 34508, "lonely": 34509, "lores": 34510, "impecc": 34511, "âĢIJ": 34512, "beavers": 34513, "maki": 34514, "heb": 34515, "aapl": 34516, "åı": 34517, "wolverhampton": 34518, "leaderboard": 34519, "meu": 34520, "cfa": 34521, "eastern": 34522, "hur": 34523, "civilwar": 34524, "ourage": 34525, "horned": 34526, "lehigh": 34527, "awards": 34528, "evident": 34529, "gigab": 34530, "rous": 34531, "madel": 34532, "robyn": 34533, "urgently": 34534, "kors": 34535, "enas": 34536, "heisman": 34537, "bambam": 34538, "fabian": 34539, "fom": 34540, "evaluating": 34541, "assembly": 34542, "outsourcing": 34543, "huntsville": 34544, "ðŁĶª": 34545, "justified": 34546, "cashier": 34547, "spaper": 34548, "buckeye": 34549, "analytical": 34550, "illuminati": 34551, "autho": 34552, "oj": 34553, "shade": 34554, "geelong": 34555, "whey": 34556, "heaton": 34557, "terribly": 34558, "elek": 34559, "uncharted": 34560, "sdlive": 34561, "motocross": 34562, "hermes": 34563, "darshan": 34564, "darlington": 34565, "cashmere": 34566, "gripping": 34567, "cilantro": 34568, "punish": 34569, "...:": 34570, "ðŁĴĦ": 34571, "instance": 34572, "deri": 34573, "lobal": 34574, "mukher": 34575, "spar": 34576, "thinker": 34577, "fremont": 34578, "compiled": 34579, "colorado": 34580, "vigne": 34581, "smd": 34582, "whead": 34583, "village": 34584, "leek": 34585, "formulae": 34586, "tares": 34587, "persistence": 34588, "??????": 34589, "pedago": 34590, "hez": 34591, "alzheimers": 34592, "vulture": 34593, "offence": 34594, "isgreat": 34595, "suffra": 34596, "kickin": 34597, "hmmmm": 34598, "broadway": 34599, "ï¸ı@": 34600, "arti": 34601, "allison": 34602, "endorses": 34603, "ryu": 34604, "lollipop": 34605, "soybean": 34606, "kendall": 34607, "cera": 34608, "invade": 34609, "(ðŁĵ·:": 34610, "converter": 34611, "carpets": 34612, "hobo": 34613, "frit": 34614, "peac": 34615, "esqu": 34616, "ernan": 34617, "ouf": 34618, "anil": 34619, "differ": 34620, "ching": 34621, "brecht": 34622, "spg": 34623, "davenport": 34624, "strava": 34625, "severn": 34626, "ngos": 34627, "storians": 34628, "fete": 34629, "paramedic": 34630, "jhb": 34631, "alamo": 34632, "sneaking": 34633, "goldcoast": 34634, "roofs": 34635, "isil": 34636, "depicted": 34637, "projections": 34638, "numb": 34639, "oss": 34640, "epi": 34641, "glucose": 34642, "zidane": 34643, "infiniti": 34644, "íĺĦ": 34645, "ransom": 34646, "tonics": 34647, "falk": 34648, "gler": 34649, "outw": 34650, "ress": 34651, "weekly": 34652, "theon": 34653, "nole": 34654, "ðŁĩªðŁĩº": 34655, "volley": 34656, "summar": 34657, "negativity": 34658, "samson": 34659, "yew": 34660, "ausvotes": 34661, "jul": 34662, "judy": 34663, "fart": 34664, "prayed": 34665, "palate": 34666, "multicultural": 34667, "doubleheader": 34668, "cyclones": 34669, "pierre": 34670, "ãģ¨": 34671, "âĺłï¸ı": 34672, "rtw": 34673, "converting": 34674, "wirral": 34675, "lari": 34676, "irrelevant": 34677, "austinmahone": 34678, "anche": 34679, "yaan": 34680, "sdf": 34681, "$.": 34682, "exploding": 34683, "ultimate": 34684, "profici": 34685, "gofundme": 34686, "cellence": 34687, "epstein": 34688, "bullied": 34689, "septic": 34690, "த": 34691, "lumber": 34692, "cuff": 34693, "vscocam": 34694, "plor": 34695, "ล": 34696, "seok": 34697, "roto": 34698, "venezuelan": 34699, "sorta": 34700, "spirited": 34701, "danielpadilla": 34702, "teamsisd": 34703, "radioactive": 34704, "icelandic": 34705, "ðŁĴ¤": 34706, "vere": 34707, "accommodate": 34708, "shipp": 34709, "otter": 34710, "olina": 34711, "ego": 34712, "sula": 34713, "sanantonio": 34714, "deas": 34715, "similarities": 34716, "âļ¾": 34717, "yom": 34718, "broward": 34719, "å°": 34720, "cancun": 34721, "verify": 34722, "onte": 34723, "candlelight": 34724, "ìłķ": 34725, "infants": 34726, "azam": 34727, "ðŁĺ°": 34728, "leven": 34729, "unstable": 34730, "bloomington": 34731, "xford": 34732, "contour": 34733, "yp": 34734, "innovator": 34735, "histories": 34736, "poy": 34737, "lololol": 34738, "expires": 34739, "catalo": 34740, "billboards": 34741, "anab": 34742, "elic": 34743, "novascotia": 34744, "faire": 34745, "ìĿ´": 34746, "rockwell": 34747, "grille": 34748, "aztec": 34749, "johor": 34750, "urstruly": 34751, "firen": 34752, "dunlop": 34753, "idle": 34754, "portman": 34755, "joes": 34756, "txhsfb": 34757, "holm": 34758, "chamele": 34759, "underworld": 34760, "loss": 34761, "tiem": 34762, "therapists": 34763, "pasture": 34764, "paste": 34765, "ingnow": 34766, "vulcan": 34767, "ragon": 34768, "larkin": 34769, "oshi": 34770, "hoco": 34771, "childhood": 34772, "umbrel": 34773, "successor": 34774, "kathy": 34775, "izen": 34776, "°ï¸ı": 34777, "shareholders": 34778, "olga": 34779, "aib": 34780, "heap": 34781, "flaming": 34782, "rou": 34783, "airtel": 34784, "ratt": 34785, "zane": 34786, "vow": 34787, "thorough": 34788, "snag": 34789, "parth": 34790, "unconscious": 34791, "vey": 34792, "newrelease": 34793, "ghee": 34794, "croatian": 34795, "facilitating": 34796, "swanson": 34797, "astoria": 34798, "tology": 34799, "mastery": 34800, "ðŁ¤ij": 34801, "bilbao": 34802, "troupe": 34803, "theori": 34804, "cheyenne": 34805, "rott": 34806, "shoreline": 34807, "grasso": 34808, "masterchef": 34809, "+)": 34810, "vix": 34811, "ellenshow": 34812, "asg": 34813, "anak": 34814, "kuya": 34815, "safarilive": 34816, "debuting": 34817, "blum": 34818, "listener": 34819, "vins": 34820, "bookshelf": 34821, "smartcities": 34822, "makeyourownlane": 34823, ";;": 34824, "ðŁIJ¯": 34825, "rizz": 34826, "onward": 34827, "bulldog": 34828, "bearish": 34829, "viruses": 34830, "frigh": 34831, "linden": 34832, "weiser": 34833, "snt": 34834, "gona": 34835, "dresden": 34836, "flanders": 34837, "cuk": 34838, "wheeling": 34839, "bau": 34840, "atuesday": 34841, "surfers": 34842, "swift": 34843, "mccall": 34844, "arbitration": 34845, "awd": 34846, "monc": 34847, "bine": 34848, "atx": 34849, "refr": 34850, "miro": 34851, "posey": 34852, "nare": 34853, "ritter": 34854, "âģ¦": 34855, "playbook": 34856, "blowout": 34857, "sportsmanship": 34858, "soooooo": 34859, "malayalam": 34860, "grims": 34861, "burbank": 34862, "infinity": 34863, "sargent": 34864, "oitnb": 34865, "josephine": 34866, "skipping": 34867, "parkin": 34868, "excursion": 34869, "seminars": 34870, "johar": 34871, "partridge": 34872, "postgame": 34873, "llll": 34874, "blanche": 34875, "tempting": 34876, "mna": 34877, "luka": 34878, "isers": 34879, "toffee": 34880, "barron": 34881, "hemmings": 34882, "sae": 34883, "gohawks": 34884, "cupid": 34885, "limbs": 34886, "conse": 34887, "uncommon": 34888, "zada": 34889, "headshot": 34890, "soils": 34891, "pioneer": 34892, "mamma": 34893, "semitic": 34894, "pandey": 34895, "jamiedornan": 34896, "splits": 34897, "vela": 34898, "soni": 34899, "raff": 34900, "tmobile": 34901, "âŀĸ": 34902, "prawns": 34903, "liter": 34904, "enjoyment": 34905, "eggplant": 34906, "tub": 34907, "cultural": 34908, "usic": 34909, "suspicion": 34910, "sycam": 34911, "summed": 34912, "madu": 34913, "hock": 34914, "upwards": 34915, "eyeing": 34916, "rive": 34917, "assassins": 34918, "âĤ¬": 34919, "outfy": 34920, "chives": 34921, "tner": 34922, "lais": 34923, "porridge": 34924, "saddest": 34925, "wcc": 34926, "vicki": 34927, "snails": 34928, "bizitalk": 34929, "millan": 34930, "ðŁĮį": 34931, "samoa": 34932, "jing": 34933, "mikey": 34934, "guj": 34935, "chelms": 34936, "eligibility": 34937, "armada": 34938, "throp": 34939, "surgeries": 34940, "ãĤ¿": 34941, "mohawk": 34942, "exits": 34943, "mem": 34944, "islington": 34945, "cme": 34946, "landfill": 34947, "kaitlyn": 34948, "ðŁİ¼": 34949, "combinations": 34950, "tomorrowland": 34951, "verb": 34952, "cora": 34953, "precisely": 34954, "naom": 34955, "ðŁĨķ": 34956, "shrink": 34957, "softly": 34958, "mercede": 34959, "mandel": 34960, "poodle": 34961, "ballerina": 34962, "soph": 34963, "juxta": 34964, "yat": 34965, "aryan": 34966, "hesitate": 34967, "lowered": 34968, "gular": 34969, "dungeonsand": 34970, "ronan": 34971, "myri": 34972, "spf": 34973, "menopau": 34974, "grasp": 34975, "pathi": 34976, "feasi": 34977, "flaw": 34978, "shistory": 34979, "steward": 34980, "ggle": 34981, "fayre": 34982, "clique": 34983, "credibility": 34984, "yog": 34985, "section": 34986, "musko": 34987, "seville": 34988, "nott": 34989, "calm": 34990, "mateo": 34991, "indicted": 34992, "fiba": 34993, "byl": 34994, "lino": 34995, "ukin": 34996, "!!#": 34997, "enigma": 34998, "sirius": 34999, "busc": 35000, "ðŁįĬ": 35001, "mackerel": 35002, "psalms": 35003, "aat": 35004, "tomorrowspaper": 35005, "ðŁĺĸ": 35006, "pfc": 35007, "...........": 35008, "shrek": 35009, "mullet": 35010, "osh": 35011, "dangerously": 35012, "immensely": 35013, "amur": 35014, "ðŁįĤ": 35015, "propor": 35016, "sya": 35017, "londonmarathon": 35018, "above": 35019, "obligatory": 35020, "prov": 35021, "racha": 35022, "alexis": 35023, "primary": 35024, "shh": 35025, "ethernet": 35026, "dstv": 35027, "cougar": 35028, "unlucky": 35029, "nil": 35030, "steakhouse": 35031, "mela": 35032, "fcbayern": 35033, "causeway": 35034, "catherine": 35035, "fluorescent": 35036, "nxt": 35037, "tokyo": 35038, "ausp": 35039, "relegation": 35040, "quizz": 35041, "shoreditch": 35042, "proudtobe": 35043, "promos": 35044, "interacting": 35045, "homebrew": 35046, "daesh": 35047, "wpg": 35048, "steadily": 35049, "provinces": 35050, "ballots": 35051, "iah": 35052, "alto": 35053, "<<<": 35054, "youu": 35055, "riley": 35056, "preference": 35057, "traverse": 35058, "incense": 35059, "ammunition": 35060, "hodges": 35061, "#@": 35062, "hailstate": 35063, "tartan": 35064, "witchcraft": 35065, "ventilation": 35066, "libertarian": 35067, "!âĢ¦": 35068, "owes": 35069, "%!": 35070, "ongchang": 35071, "brushing": 35072, "leic": 35073, "fiber": 35074, "underattack": 35075, "download": 35076, "expir": 35077, "hyo": 35078, "pompey": 35079, "mcbride": 35080, "yag": 35081, "stree": 35082, "combat": 35083, "tending": 35084, "aira": 35085, "guggen": 35086, "abra": 35087, "inna": 35088, "flips": 35089, "awal": 35090, "mach": 35091, "dollar": 35092, "inspirations": 35093, "zum": 35094, "odu": 35095, "itty": 35096, "videogame": 35097, "aquaman": 35098, "haru": 35099, "belfast": 35100, "jeb": 35101, "butch": 35102, "usgs": 35103, "calculus": 35104, "goyal": 35105, "morgen": 35106, "xfinity": 35107, "standup": 35108, "contracep": 35109, "sabre": 35110, "nabe": 35111, "insecure": 35112, "generously": 35113, "epitome": 35114, "lw": 35115, "tca": 35116, "narratives": 35117, "donnell": 35118, "pandas": 35119, "bergh": 35120, "tut": 35121, "keral": 35122, "felicity": 35123, "brampton": 35124, "quintet": 35125, "nomore": 35126, "ðŁĶij": 35127, "loi": 35128, "alhamdulil": 35129, "ðŁĶ¥ðŁĶĹ": 35130, "stoner": 35131, "shawl": 35132, "clinical": 35133, "brendan": 35134, "gone": 35135, "flawed": 35136, "trippy": 35137, "jg": 35138, "allocation": 35139, "poaching": 35140, "vevo": 35141, "mocks": 35142, "leftist": 35143, "bonuses": 35144, "condemned": 35145, "ability": 35146, "stating": 35147, "microbiome": 35148, "biologist": 35149, "foryou": 35150, "wahlberg": 35151, "ssor": 35152, "iftar": 35153, "wul": 35154, "ÑĦоÑĤ": 35155, "pomer": 35156, "meme": 35157, "verte": 35158, "trell": 35159, "trait": 35160, "inlet": 35161, "hormones": 35162, "deliberately": 35163, "villar": 35164, "battleship": 35165, "pbl": 35166, "twenti": 35167, "hokies": 35168, "dalail": 35169, "saya": 35170, "mayfair": 35171, "hans": 35172, "diets": 35173, "⾨⾨": 35174, "odin": 35175, "hotspur": 35176, "papi": 35177, "kana": 35178, "kamp": 35179, "finna": 35180, "flotus": 35181, "tians": 35182, "unicorns": 35183, "tribeca": 35184, "changers": 35185, "foreground": 35186, "outa": 35187, "invaders": 35188, "gettys": 35189, "tomorrowspaperstoday": 35190, "macmillan": 35191, "handwritten": 35192, "wfp": 35193, "ude": 35194, "stateof": 35195, "based": 35196, "âĺģï¸ı": 35197, "casm": 35198, "psyched": 35199, "historians": 35200, "fold": 35201, "dda": 35202, "aggrav": 35203, "pans": 35204, "greenway": 35205, "ausv": 35206, "ðŁĺ¶": 35207, "shraddha": 35208, "index": 35209, "besti": 35210, "zimmer": 35211, "tness": 35212, "eyeshadow": 35213, "otte": 35214, "gots": 35215, "distributing": 35216, "promin": 35217, "yol": 35218, "acea": 35219, "tramrahim": 35220, "hooper": 35221, "supreme": 35222, "jammin": 35223, "intuitive": 35224, "qualifications": 35225, "slim": 35226, "siddi": 35227, "jayne": 35228, "tripping": 35229, "gtx": 35230, "puns": 35231, "emanuel": 35232, "omg": 35233, "midsummer": 35234, "into": 35235, "succulent": 35236, "rien": 35237, "newmexico": 35238, "oor": 35239, "hooking": 35240, "inf": 35241, "ðŁ¤Ŀ": 35242, "flirting": 35243, "nahi": 35244, "gfriend": 35245, "tps": 35246, "helix": 35247, "zs": 35248, "onie": 35249, "ctf": 35250, "kris": 35251, "irresistible": 35252, "flap": 35253, "ðŁijıðŁı»ðŁijıðŁı»": 35254, "uswnt": 35255, "rud": 35256, "ramps": 35257, "pinoy": 35258, "otw": 35259, "lolz": 35260, "lowering": 35261, "favorite": 35262, "tmc": 35263, "phrases": 35264, "hermi": 35265, "averaging": 35266, "embr": 35267, "beno": 35268, "estuary": 35269, "sleeve": 35270, "ribbons": 35271, "tash": 35272, "ู": 35273, "xf": 35274, "awgs": 35275, "sunited": 35276, "breweries": 35277, "anirud": 35278, "punches": 35279, "oldie": 35280, "ipads": 35281, "wifey": 35282, "landlords": 35283, "dji": 35284, "gunner": 35285, "íķ´": 35286, "texan": 35287, "exop": 35288, "cassandra": 35289, "soff": 35290, "ðŁļ«": 35291, "ighton": 35292, "bakers": 35293, "awarenessweek": 35294, "vall": 35295, "earp": 35296, "btsbbmas": 35297, "apologizes": 35298, "âļĵï¸ı": 35299, "wasps": 35300, "statesman": 35301, "snatch": 35302, "watchdog": 35303, "rafi": 35304, "afterparty": 35305, "spike": 35306, "jer": 35307, "periph": 35308, "rnc": 35309, "mull": 35310, "leen": 35311, "shies": 35312, "lieu": 35313, "urstrulymahesh": 35314, "merton": 35315, "desai": 35316, "shif": 35317, "ðŁĮ±": 35318, "pedic": 35319, "gosling": 35320, "arranging": 35321, "wwg": 35322, "geny": 35323, "youuu": 35324, "netflix": 35325, "ettes": 35326, "kwi": 35327, "bernardino": 35328, "amiga": 35329, "ب": 35330, "kashmiri": 35331, "tings": 35332, "emeritus": 35333, "decat": 35334, "abdomin": 35335, "dci": 35336, "phases": 35337, "djan": 35338, "beam": 35339, "opry": 35340, "ished": 35341, "theellenshow": 35342, "thest": 35343, "habitats": 35344, "toons": 35345, "mclaughlin": 35346, "ripper": 35347, "microbiology": 35348, "talaga": 35349, "clueless": 35350, "ssu": 35351, "croche": 35352, "bromance": 35353, "longevity": 35354, "zagreb": 35355, "prevented": 35356, "trave": 35357, "spoilt": 35358, "darryl": 35359, "migraine": 35360, "alcat": 35361, "dddd": 35362, "viv": 35363, "serpent": 35364, "mattel": 35365, "jama": 35366, "conquest": 35367, "îĦ": 35368, "samsung": 35369, "presbyterian": 35370, "ketch": 35371, "firefox": 35372, "motif": 35373, "lec": 35374, "chopping": 35375, "cherno": 35376, "jann": 35377, "ðŁIJ°": 35378, "prolon": 35379, "wakeup": 35380, "convergence": 35381, "merseyside": 35382, "heartbroken": 35383, "looming": 35384, "hallucin": 35385, "maize": 35386, "communism": 35387, "moh": 35388, "twitterstorians": 35389, "sergey": 35390, "reseller": 35391, "favorable": 35392, "edgy": 35393, "reiter": 35394, "malaga": 35395, "liveme": 35396, "kahn": 35397, "pulsion": 35398, "bigg": 35399, "kimkardashian": 35400, "atio": 35401, "tyranny": 35402, "ruption": 35403, "qant": 35404, "proven": 35405, "byz": 35406, "pushaw": 35407, "kristin": 35408, "eer": 35409, "tardis": 35410, "riz": 35411, "awaken": 35412, "miko": 35413, "undocumented": 35414, "pathfinder": 35415, "indirect": 35416, "resembles": 35417, "hler": 35418, "concealed": 35419, "scandal": 35420, "reim": 35421, "dnb": 35422, "critters": 35423, "attendant": 35424, "apprenticeships": 35425, "aau": 35426, "screamed": 35427, "lsu": 35428, "fah": 35429, "harbour": 35430, "edd": 35431, "batsman": 35432, "liss": 35433, "misha": 35434, "spaniel": 35435, "itf": 35436, "advancement": 35437, "fac": 35438, "closeup": 35439, "cecilia": 35440, "medic": 35441, "narcissi": 35442, "lavish": 35443, "giac": 35444, "mays": 35445, "leit": 35446, "winewednesday": 35447, "pushaward": 35448, "letto": 35449, "currents": 35450, "bugatti": 35451, "outine": 35452, "wj": 35453, "undo": 35454, "lerosis": 35455, "devotional": 35456, "ðŁij«": 35457, "onna": 35458, "faisal": 35459, "sauna": 35460, "himachal": 35461, "amii": 35462, "à®®": 35463, "dizzy": 35464, "screenwriting": 35465, "phx": 35466, "spn": 35467, "icki": 35468, "agirl": 35469, "fishes": 35470, "wbz": 35471, "pim": 35472, "boar": 35473, "acid": 35474, "!..": 35475, "rockefeller": 35476, "nga": 35477, "drastically": 35478, "simplify": 35479, "drumming": 35480, "autumnal": 35481, "gurmee": 35482, "lorde": 35483, "joann": 35484, "giveup": 35485, "bour": 35486, "amura": 35487, "derland": 35488, "simpler": 35489, "watson": 35490, "trident": 35491, "concordia": 35492, "bellum": 35493, "brek": 35494, "dumplings": 35495, "vion": 35496, "dungeonsanddragons": 35497, "spri": 35498, "ascension": 35499, "wildatlantic": 35500, "ust": 35501, "robins": 35502, "legion": 35503, "insist": 35504, "jaro": 35505, "guess": 35506, "sob": 35507, "bighit": 35508, "poolside": 35509, "negotiating": 35510, "mcgill": 35511, "bild": 35512, "technicians": 35513, "mitigation": 35514, "ajaydevgn": 35515, "bto": 35516, "anten": 35517, "cosmopolitan": 35518, "ðŁĺĬðŁĺĬðŁĺĬðŁĺĬ": 35519, "patrioti": 35520, "temper": 35521, "promenade": 35522, "navajo": 35523, "namm": 35524, "wrinkles": 35525, "dcfc": 35526, "leach": 35527, "brunette": 35528, "rf": 35529, "coutinho": 35530, "alti": 35531, "traditionally": 35532, "optome": 35533, "naz": 35534, "accordingly": 35535, "recard": 35536, "deets": 35537, "swell": 35538, "posure": 35539, "whitening": 35540, "stranger": 35541, "illion": 35542, "hereford": 35543, "uwu": 35544, "robber": 35545, "cotswolds": 35546, "clen": 35547, "gorge": 35548, "namaste": 35549, "relish": 35550, "griff": 35551, "adrenaline": 35552, "blasio": 35553, "vale": 35554, "ê²": 35555, "tolerate": 35556, "railminindia": 35557, "jensen": 35558, "hoven": 35559, "ellu": 35560, "obsole": 35561, "eisenhower": 35562, "unidentified": 35563, "thanniversary": 35564, "bodyguard": 35565, "د": 35566, "idge": 35567, "schal": 35568, "stockport": 35569, "sni": 35570, "retaining": 35571, "popo": 35572, "pixie": 35573, "olithic": 35574, "kier": 35575, "hajj": 35576, "saz": 35577, "corbin": 35578, "!!!!!!!!!!": 35579, "vit": 35580, "megat": 35581, "deh": 35582, "circuit": 35583, "affleck": 35584, "theoretical": 35585, "hopeless": 35586, "uab": 35587, "slump": 35588, "bice": 35589, "jammed": 35590, "letstalk": 35591, "cani": 35592, "sideways": 35593, "labyrinth": 35594, "refs": 35595, "hahn": 35596, "jared": 35597, "ðŁį¹": 35598, "jambo": 35599, "phyl": 35600, "enhancement": 35601, "ctr": 35602, "fullest": 35603, "seye": 35604, "doba": 35605, "choic": 35606, "yos": 35607, "cbj": 35608, "andré": 35609, "rewatch": 35610, "prima": 35611, "doctrine": 35612, "forgets": 35613, "uhm": 35614, "around": 35615, "ule": 35616, "artlovers": 35617, "shiraz": 35618, "harth": 35619, "extor": 35620, "Å¡": 35621, "unexpectedly": 35622, "elius": 35623, "yx": 35624, "emmy": 35625, "seac": 35626, "ðŁijĩðŁijĩðŁijĩ": 35627, "corrected": 35628, "combu": 35629, "womanc": 35630, "cough": 35631, "whatson": 35632, "publishes": 35633, "diversity": 35634, "backbone": 35635, "lockdown": 35636, "mesmerizing": 35637, "norte": 35638, "mab": 35639, "designer": 35640, "íģ": 35641, "ragh": 35642, "molecules": 35643, "getoutside": 35644, "thebeatles": 35645, "semiconduc": 35646, "nacho": 35647, "lunes": 35648, "hammers": 35649, "sultan": 35650, "oon": 35651, "feren": 35652, "attach": 35653, "arqu": 35654, "uttarakhand": 35655, "sash": 35656, ";-": 35657, "tread": 35658, "iko": 35659, "arthur": 35660, "scandinavian": 35661, "ration": 35662, "gael": 35663, "chargeable": 35664, "fishy": 35665, "vma": 35666, "handbags": 35667, "chara": 35668, "ayne": 35669, "defam": 35670, "settlers": 35671, "qadri": 35672, "palais": 35673, "inwx": 35674, "apocalyptic": 35675, "pooja": 35676, "aes": 35677, "atories": 35678, "proofing": 35679, "nlp": 35680, "tsla": 35681, "vina": 35682, "lido": 35683, "deephouse": 35684, "informatics": 35685, "vv": 35686, "ppings": 35687, "diss": 35688, "ï": 35689, "uhuru": 35690, "stony": 35691, "betrayed": 35692, "baff": 35693, "myra": 35694, "aspen": 35695, "allowance": 35696, "tamara": 35697, "cif": 35698, "corbett": 35699, "serge": 35700, "digo": 35701, "ambigu": 35702, "painters": 35703, "pcr": 35704, "pca": 35705, "noms": 35706, "loft": 35707, "vee": 35708, "opendata": 35709, "ðŁIJ±": 35710, "alexandre": 35711, "identifies": 35712, "fantasyfootball": 35713, "reproduction": 35714, "bromley": 35715, "wareagle": 35716, "mmer": 35717, "pss": 35718, "cues": 35719, "ayat": 35720, "hutchinson": 35721, "sarac": 35722, "jackman": 35723, "irah": 35724, "apink": 35725, "cols": 35726, "aussies": 35727, "execs": 35728, "dayton": 35729, "ðŁĻĨ": 35730, "imv": 35731, "haram": 35732, "chuckle": 35733, "authenticity": 35734, "ardo": 35735, "incubator": 35736, "ส": 35737, "photoshopped": 35738, "embraced": 35739, "fightfor": 35740, "gorman": 35741, "zzzz": 35742, "scholastic": 35743, "crisps": 35744, "teapo": 35745, "midnight": 35746, "gaine": 35747, "collier": 35748, "sate": 35749, "dette": 35750, "åŃ": 35751, "imagine": 35752, "iff": 35753, "twili": 35754, "ification": 35755, "teatro": 35756, "norma": 35757, "esur": 35758, "emergencies": 35759, "riseup": 35760, "ringer": 35761, "hassle": 35762, "caitlyn": 35763, "tranquil": 35764, "versa": 35765, "seb": 35766, "overlook": 35767, "gini": 35768, "bogo": 35769, "sere": 35770, "mayne": 35771, "henrik": 35772, "contaminated": 35773, "rhapsody": 35774, "proportion": 35775, "wildatlanticway": 35776, "âģ©.": 35777, "organisers": 35778, "trane": 35779, "standard": 35780, "sperm": 35781, "launcher": 35782, "ricci": 35783, "herts": 35784, "paperwork": 35785, "showcased": 35786, "meryl": 35787, "pena": 35788, "pimp": 35789, "disastrous": 35790, "^.^": 35791, "phara": 35792, "xis": 35793, "frontal": 35794, "swirl": 35795, "spills": 35796, "swagger": 35797, "smartwatch": 35798, "sizzling": 35799, "saviour": 35800, "catar": 35801, "bbcr": 35802, "refurbishment": 35803, "dris": 35804, "citroen": 35805, "absorb": 35806, "patriotism": 35807, "illeg": 35808, "chromo": 35809, "freshers": 35810, "rus": 35811, "limiting": 35812, "efish": 35813, "downed": 35814, "mandir": 35815, "hazelnut": 35816, "pall": 35817, "macon": 35818, "disappearing": 35819, "qualifies": 35820, "boon": 35821, "barracks": 35822, "amine": 35823, "gendere": 35824, "ðŁļĺ": 35825, "jes": 35826, "ãĥŃ": 35827, "quito": 35828, "middleweight": 35829, "schau": 35830, "quadru": 35831, "aciones": 35832, "limitless": 35833, "ðŁijĮðŁı½": 35834, "chman": 35835, "arav": 35836, "regulators": 35837, "itup": 35838, "battersea": 35839, "milford": 35840, "gz": 35841, "ticking": 35842, "ghou": 35843, "crushes": 35844, "tutu": 35845, "dreadful": 35846, "famine": 35847, "forchange": 35848, "dalailama": 35849, "ðŁĴį": 35850, "whitaker": 35851, "hashmi": 35852, "hus": 35853, "vod": 35854, "bette": 35855, "aaah": 35856, "isoo": 35857, "ðŁ¥Ī": 35858, "haar": 35859, "laine": 35860, "bv": 35861, "allday": 35862, "sprout": 35863, "indiegames": 35864, "freebie": 35865, "greeks": 35866, "butler": 35867, "illin": 35868, "haal": 35869, "wareness": 35870, "sima": 35871, "publichealth": 35872, "gama": 35873, "waa": 35874, "oung": 35875, "goooo": 35876, "okinawa": 35877, "offenders": 35878, "impose": 35879, "hoc": 35880, "youngster": 35881, "storyteller": 35882, "scap": 35883, "fighter": 35884, "+,": 35885, "whites": 35886, "musicmonday": 35887, "reza": 35888, "goducks": 35889, "bria": 35890, "mium": 35891, "casper": 35892, "crumbs": 35893, "aad": 35894, "martialarts": 35895, "chp": 35896, "rigged": 35897, "tng": 35898, "harvested": 35899, "sak": 35900, "dojo": 35901, "millwall": 35902, "bnw": 35903, "ocd": 35904, "historyof": 35905, "tmr": 35906, "sirens": 35907, "fanci": 35908, "caregivers": 35909, "vira": 35910, "soni": 35911, "recurring": 35912, "acknowledged": 35913, "ðŁıŁ": 35914, "ophile": 35915, "bucky": 35916, "stressing": 35917, "rook": 35918, "digger": 35919, "vival": 35920, "sando": 35921, "fleet": 35922, "siers": 35923, "selcaday": 35924, "refreshed": 35925, "antifa": 35926, "aque": 35927, "polo": 35928, "disappearance": 35929, "demb": 35930, "âĮļï¸ı": 35931, "rented": 35932, "berger": 35933, "gmb": 35934, "cula": 35935, "ssal": 35936, "goody": 35937, "uhh": 35938, "marcelo": 35939, "wanna": 35940, "software": 35941, "shopsmall": 35942, "turtle": 35943, "tomas": 35944, "frisco": 35945, "ðŁĺįðŁĴķ": 35946, "jimenez": 35947, "csu": 35948, "dayz": 35949, "ando": 35950, "wynne": 35951, "choreographer": 35952, "cervical": 35953, "trailblazers": 35954, "edg": 35955, "zendaya": 35956, "travelblog": 35957, "els": 35958, "wholesome": 35959, "cog": 35960, "labout": 35961, "arney": 35962, "delle": 35963, "suisse": 35964, "masi": 35965, "inese": 35966, "ombe": 35967, "fiddle": 35968, "reclaim": 35969, "pau": 35970, "watcher": 35971, "slain": 35972, "berty": 35973, "optimum": 35974, "elites": 35975, "minis": 35976, "turkey": 35977, "patrols": 35978, "gerard": 35979, "aureli": 35980, "wildly": 35981, "waltz": 35982, "brgy": 35983, "wob": 35984, "crest": 35985, "+++": 35986, "vez": 35987, "frosted": 35988, "davido": 35989, "thex": 35990, "paramedics": 35991, "pinto": 35992, "hank": 35993, "dupont": 35994, "urg": 35995, "fostering": 35996, "micropoetry": 35997, "spectre": 35998, "---->": 35999, "neuro": 36000, "frida": 36001, "musical": 36002, "galveston": 36003, "effic": 36004, "scape": 36005, "palazzo": 36006, "thall": 36007, "provisional": 36008, "pjs": 36009, "aure": 36010, "ðŁĶľ": 36011, "mamamoo": 36012, "kitties": 36013, "cree": 36014, "wak": 36015, "loool": 36016, "lupus": 36017, "cnblue": 36018, "ú": 36019, "ðŁİ¬": 36020, "raced": 36021, "trose": 36022, "omas": 36023, "stride": 36024, "coors": 36025, "⤵ï¸ı": 36026, "incomparable": 36027, "cyril": 36028, "broader": 36029, "areclipse": 36030, "ðŁįĶ": 36031, "interval": 36032, "tiru": 36033, "coworking": 36034, "waco": 36035, "aham": 36036, "abee": 36037, "flourish": 36038, "thetimes": 36039, "olini": 36040, "kickboxing": 36041, "lucer": 36042, "atla": 36043, "asun": 36044, "casserole": 36045, "miaw": 36046, "lobbying": 36047, "janice": 36048, "cirque": 36049, "reflex": 36050, "leary": 36051, "sanatomy": 36052, "tempest": 36053, "semb": 36054, "murdering": 36055, "usav": 36056, "robo": 36057, "onet": 36058, "pcc": 36059, "natives": 36060, "lifeof": 36061, "saha": 36062, "ruthless": 36063, "relates": 36064, "appetizer": 36065, "pyeongchang": 36066, "nord": 36067, "eru": 36068, "athing": 36069, "ugly": 36070, "plying": 36071, "brance": 36072, "organise": 36073, "kendra": 36074, "dato": 36075, "cheeses": 36076, "parma": 36077, "burnout": 36078, "astra": 36079, "pretoria": 36080, "adjustment": 36081, "uku": 36082, "slo": 36083, "liken": 36084, "favors": 36085, "clive": 36086, "beets": 36087, "snowdonia": 36088, "gotv": 36089, "syn": 36090, "openhouse": 36091, "pani": 36092, "portrayed": 36093, "slated": 36094, "mecca": 36095, "renal": 36096, "supportsmallstreamers": 36097, "staffs": 36098, "dao": 36099, "biker": 36100, "viktor": 36101, "titus": 36102, "admired": 36103, "ðŁĵ±": 36104, "hurrican": 36105, "heats": 36106, "glory": 36107, "photogenic": 36108, "meri": 36109, "depor": 36110, "burnham": 36111, "orangu": 36112, "djing": 36113, "impressionism": 36114, "ignition": 36115, "cai": 36116, "wynn": 36117, "depe": 36118, "coveted": 36119, "collagen": 36120, "saus": 36121, "ornam": 36122, "administrators": 36123, "sson": 36124, "nhpolitics": 36125, "hahahahahahahaha": 36126, "aspirations": 36127, "rgb": 36128, "swollen": 36129, "sowe": 36130, "scr": 36131, "divergent": 36132, "houghton": 36133, "hanoi": 36134, "dory": 36135, "niki": 36136, "landry": 36137, "bcci": 36138, "ðŁijĮðŁijĮ": 36139, "ismail": 36140, "tripod": 36141, "herd": 36142, "bhatt": 36143, "dressage": 36144, "tabby": 36145, "inguish": 36146, "huron": 36147, "à³į": 36148, "Ãł": 36149, "todas": 36150, "evangelical": 36151, "chords": 36152, "stjohn": 36153, "sloppy": 36154, "martyr": 36155, "facebook": 36156, "alight": 36157, "sensei": 36158, "kathniel": 36159, "rites": 36160, "zione": 36161, "uo": 36162, "revelations": 36163, "weightlifting": 36164, "pano": 36165, "ncwx": 36166, "acton": 36167, "à®ķ": 36168, "ز": 36169, "soma": 36170, "à¸Ĺ": 36171, "respecting": 36172, "marche": 36173, "foreman": 36174, "betty": 36175, "kik": 36176, "shibu": 36177, "poon": 36178, "argyle": 36179, "kswx": 36180, "etz": 36181, "marbella": 36182, "brackets": 36183, "standby": 36184, "fireside": 36185, "defiance": 36186, "vex": 36187, "britannia": 36188, "inhabit": 36189, "appoint": 36190, "piyush": 36191, "leash": 36192, "sciento": 36193, "flask": 36194, "senna": 36195, ">:": 36196, "atroc": 36197, "sanderson": 36198, "idlib": 36199, "dhanush": 36200, "ðŁĺĻ": 36201, "enthr": 36202, "hitch": 36203, "dedly": 36204, "alley": 36205, "dork": 36206, "mondo": 36207, "cuddly": 36208, "missin": 36209, "yesss": 36210, "nighting": 36211, "jpn": 36212, "wary": 36213, "umpire": 36214, "maz": 36215, "ê³": 36216, "babs": 36217, "ĭãģ": 36218, "stanford": 36219, "possessed": 36220, "exceeded": 36221, "ðŁĶ¶": 36222, "wallart": 36223, "trap": 36224, "jil": 36225, "hibis": 36226, "spying": 36227, "scribe": 36228, "khalil": 36229, "translator": 36230, "lumb": 36231, "dized": 36232, "chc": 36233, "supervision": 36234, "shutter": 36235, "jag": 36236, "_*": 36237, "yesterdays": 36238, "msf": 36239, "hihi": 36240, "gonzaga": 36241, "gillespie": 36242, "vivek": 36243, "ecstatic": 36244, "thismorning": 36245, "chus": 36246, "edes": 36247, "stoned": 36248, "bees": 36249, "ðŁĩ¹ðŁĩ": 36250, "turin": 36251, "hover": 36252, "atrics": 36253, "stern": 36254, "samheughan": 36255, "autism": 36256, "miya": 36257, "eyewitness": 36258, "writings": 36259, "traveltips": 36260, "chutney": 36261, "pxrtg": 36262, "kenyans": 36263, "mystic": 36264, "krit": 36265, "/$": 36266, "redhead": 36267, "worldly": 36268, "amus": 36269, "opla": 36270, "leve": 36271, "gabbana": 36272, "seen": 36273, "oclock": 36274, "ganga": 36275, "keenan": 36276, "scent": 36277, "oldies": 36278, "gogreen": 36279, "cornerstone": 36280, "comply": 36281, "concours": 36282, "ðŁİ¶ðŁİ¶": 36283, "haan": 36284, "confis": 36285, "awson": 36286, "cleop": 36287, "îĢ": 36288, "suzu": 36289, "sauté": 36290, "algar": 36291, "subscriber": 36292, "esteemed": 36293, "ãĤ¤ãĥ": 36294, "worthwhile": 36295, "melrose": 36296, "flock": 36297, "brightly": 36298, "violinist": 36299, "pere": 36300, "slipping": 36301, "andco": 36302, "sigh": 36303, "havan": 36304, "culo": 36305, "msa": 36306, "fibrosis": 36307, "matilda": 36308, "rafting": 36309, "award": 36310, "ëª": 36311, "mmmm": 36312, "geaux": 36313, "steiner": 36314, "sinn": 36315, "helpers": 36316, "beetles": 36317, "aimee": 36318, "taiwan": 36319, "pistachio": 36320, "macbeth": 36321, "mzan": 36322, "descendants": 36323, "onsale": 36324, "inr": 36325, "ilm": 36326, "grouse": 36327, "saig": 36328, "mow": 36329, "bigre": 36330, "adjustments": 36331, "tula": 36332, "mathew": 36333, "translates": 36334, "muh": 36335, "bollah": 36336, "ðŁĴĽðŁĴĻ": 36337, "amores": 36338, "abouts": 36339, "bombshell": 36340, "blaster": 36341, "xavi": 36342, "sns": 36343, "kroger": 36344, "gather": 36345, "eradic": 36346, "daft": 36347, "chemo": 36348, "benches": 36349, "ðŁĩ©ðŁĩ": 36350, "utv": 36351, "oura": 36352, "nko": 36353, "gatorade": 36354, "biafra": 36355, "okstate": 36356, "imdanielpadilla": 36357, "domains": 36358, "openingday": 36359, "kiddo": 36360, "doi": 36361, "rice": 36362, "daycare": 36363, "macmillan": 36364, "bathurst": 36365, "cheerleading": 36366, "ðŁ¦ģ": 36367, "cashback": 36368, "kwon": 36369, "hobbies": 36370, "exempl": 36371, "riesling": 36372, "âļª": 36373, "agles": 36374, "nys": 36375, "everything": 36376, "navis": 36377, "addi": 36378, "magnesium": 36379, "facelift": 36380, "arkham": 36381, "grandes": 36382, "extremist": 36383, "donat": 36384, "vitality": 36385, "pumpkin": 36386, "betta": 36387, "sltd": 36388, "artisan": 36389, "liby": 36390, "peaked": 36391, "ahhhhh": 36392, "maryam": 36393, "assim": 36394, "unsc": 36395, "mente": 36396, "alaya": 36397, "lowers": 36398, "aras": 36399, "griev": 36400, "leip": 36401, "grati": 36402, "crises": 36403, "sprints": 36404, "execute": 36405, "wto": 36406, "msd": 36407, "magical": 36408, "reviewer": 36409, "sparkles": 36410, "jukebox": 36411, "ðŁĺĤâĿ¤ï¸ı": 36412, "payback": 36413, "licenses": 36414, "dunkin": 36415, "belt": 36416, "lakewood": 36417, "hateful": 36418, "budgets": 36419, "revamped": 36420, "pherson": 36421, "kyiv": 36422, "wentworth": 36423, "rosen": 36424, "cruise": 36425, "giggle": 36426, "defstar": 36427, "assassinscre": 36428, "ymouth": 36429, "winkle": 36430, "wfc": 36431, "bandwagon": 36432, "bkk": 36433, "wiring": 36434, "kearney": 36435, "southside": 36436, "petit": 36437, "!ðŁĺį": 36438, "nordic": 36439, "mirza": 36440, "mugabe": 36441, "vl": 36442, "scones": 36443, "ktv": 36444, "sandal": 36445, "duc": 36446, "malls": 36447, "ðŁĴŀðŁĴŀ": 36448, "itc": 36449, "alay": 36450, "impair": 36451, "unrest": 36452, "floss": 36453, "cé": 36454, "abou": 36455, "varying": 36456, "museo": 36457, "server": 36458, "diya": 36459, "hibiscus": 36460, "eroy": 36461, "merritt": 36462, "findom": 36463, "fpp": 36464, "unusually": 36465, "gott": 36466, "contingent": 36467, "aliaa": 36468, "ballon": 36469, "jol": 36470, "hiked": 36471, "zyme": 36472, "ayr": 36473, "agn": 36474, "gaz": 36475, "periodic": 36476, "sparty": 36477, "practising": 36478, "linton": 36479, "talis": 36480, "cypri": 36481, "womaninbiz": 36482, "radiodisney": 36483, "ðŁĮ¼": 36484, "jumpers": 36485, "endocr": 36486, "ðŁļ¨ðŁļ¨": 36487, "andon": 36488, "sharapo": 36489, "mier": 36490, "masonic": 36491, "factories": 36492, "vien": 36493, "bbers": 36494, "ìĽIJ": 36495, "hold": 36496, "kebab": 36497, "beak": 36498, "approached": 36499, "acmilan": 36500, "munro": 36501, "kosher": 36502, "excellency": 36503, "negotiation": 36504, "waltdisneyworld": 36505, "crouch": 36506, "teasing": 36507, "suppression": 36508, "enya": 36509, "bce": 36510, "transformationtuesday": 36511, "callie": 36512, "viswas": 36513, "pgat": 36514, "icted": 36515, "endings": 36516, "escu": 36517, "recruited": 36518, "itfc": 36519, "collaborations": 36520, "gino": 36521, "snuck": 36522, "auschwitz": 36523, "ifc": 36524, "xii": 36525, "kesha": 36526, "gervais": 36527, "cloak": 36528, "xl": 36529, "saad": 36530, "probation": 36531, "precau": 36532, "macin": 36533, "anastasi": 36534, "lek": 36535, "eazy": 36536, "daysofcode": 36537, "mariahcarey": 36538, "yog": 36539, "stitched": 36540, "boyfriends": 36541, "shar": 36542, "phile": 36543, "agu": 36544, "twinkle": 36545, "phishing": 36546, "weekender": 36547, "icton": 36548, "gurmeetramrahim": 36549, "alton": 36550, "leness": 36551, "allan": 36552, "penultimate": 36553, "krystal": 36554, "gou": 36555, "lande": 36556, "dismant": 36557, "abusing": 36558, "norse": 36559, "paterson": 36560, "edmun": 36561, "apan": 36562, "xiumin": 36563, "skel": 36564, "catwalk": 36565, "react": 36566, "walled": 36567, "tangle": 36568, "bryn": 36569, "veto": 36570, "supermoon": 36571, "casablanc": 36572, "appreciates": 36573, "skid": 36574, "both": 36575, "catalina": 36576, "eleague": 36577, "cybermonday": 36578, "cautious": 36579, "ðŁ¤ĵ": 36580, "novo": 36581, "hampton": 36582, "haye": 36583, "josef": 36584, "varan": 36585, "lobos": 36586, "roanoke": 36587, "orphans": 36588, "ttin": 36589, "squads": 36590, "ishqbaaaz": 36591, "blackpanther": 36592, "etu": 36593, "ksh": 36594, "crumble": 36595, "cessna": 36596, "relieved": 36597, "scully": 36598, "pollinators": 36599, "explorecanada": 36600, "kies": 36601, "kamloops": 36602, "kiran": 36603, "primal": 36604, "settlements": 36605, "hotspot": 36606, "brainstorming": 36607, "cedric": 36608, "biennial": 36609, "shant": 36610, "âĻ¡âĻ¡âĻ¡": 36611, "doon": 36612, "hearn": 36613, "walkway": 36614, "fem": 36615, "veal": 36616, "deportation": 36617, "toxins": 36618, "eliminating": 36619, "descending": 36620, "bythe": 36621, "blasphe": 36622, "hasta": 36623, "complement": 36624, "ascent": 36625, "riga": 36626, "provost": 36627, "âĸª": 36628, "weeping": 36629, "antisemitism": 36630, "employee": 36631, "unearthed": 36632, "pino": 36633, "natalie": 36634, "blad": 36635, "angola": 36636, "lockheed": 36637, "inian": 36638, "agr": 36639, "nister": 36640, "impala": 36641, "mke": 36642, "fanatic": 36643, "âĺħâĺħ": 36644, "ðŁij¸": 36645, "luch": 36646, "simplified": 36647, "gallery": 36648, "economic": 36649, "cyborg": 36650, "coni": 36651, "selma": 36652, "inception": 36653, "koala": 36654, "dvds": 36655, "crested": 36656, "mmor": 36657, "visible": 36658, "nsd": 36659, "ðŁĻĮðŁı½": 36660, "wunder": 36661, "refrigerator": 36662, "reopening": 36663, "eera": 36664, "carousel": 36665, "asp": 36666, "ballistic": 36667, "victory": 36668, "motive": 36669, "trey": 36670, "sharapova": 36671, "sii": 36672, "monter": 36673, "intend": 36674, "westchester": 36675, "spe": 36676, "cymb": 36677, "vidal": 36678, "llama": 36679, "univ": 36680, "finer": 36681, "craftsmanship": 36682, "jazzfest": 36683, "bch": 36684, "aggio": 36685, "ncc": 36686, "lambda": 36687, "tranquility": 36688, "cisco": 36689, "baden": 36690, "sobbing": 36691, "ofi": 36692, "gota": 36693, "rumored": 36694, "warmed": 36695, "orean": 36696, "acton": 36697, "marci": 36698, "ghani": 36699, "âľĵ": 36700, "assorted": 36701, "pembroke": 36702, "penelope": 36703, "daf": 36704, "atty": 36705, "aimo": 36706, "pretzel": 36707, "carnival": 36708, "thanos": 36709, "kochi": 36710, "mersal": 36711, "hamradio": 36712, "artwit": 36713, "casc": 36714, "guerrilla": 36715, "kushner": 36716, "kapp": 36717, "alise": 36718, "toddlers": 36719, "stewardship": 36720, "otti": 36721, "terri": 36722, "tempe": 36723, "restless": 36724, "vito": 36725, "zayed": 36726, "rspb": 36727, "pion": 36728, "hippo": 36729, "hawthorne": 36730, "inas": 36731, "amily": 36732, "nutcracker": 36733, "lop": 36734, "dali": 36735, "tropic": 36736, "ðŁ¤ł": 36737, "ulo": 36738, "jaredle": 36739, "pyrene": 36740, "paleo": 36741, "usair": 36742, "mould": 36743, "itated": 36744, "genetically": 36745, "biomass": 36746, "ðŁĩ³ðŁĩ±": 36747, "dodd": 36748, "practiced": 36749, "monarchs": 36750, "unmanned": 36751, "mbuhari": 36752, "amal": 36753, "photogra": 36754, "kool": 36755, "brendon": 36756, "juices": 36757, "cure": 36758, "worldbank": 36759, "pointers": 36760, "ðŁĴĿ": 36761, "turf": 36762, "leds": 36763, "borussia": 36764, "baptism": 36765, "warwickshire": 36766, "mounts": 36767, "gayo": 36768, "begg": 36769, "copied": 36770, "asians": 36771, "kg": 36772, "modernist": 36773, "gid": 36774, "frontman": 36775, "concentrated": 36776, "yt": 36777, "scavenger": 36778, "ironically": 36779, "adic": 36780, "psn": 36781, "ðŁ¥ī": 36782, "culturally": 36783, "yuv": 36784, "macarthur": 36785, "fertilizer": 36786, "bewithyou": 36787, "rigor": 36788, "minors": 36789, "zoning": 36790, "âĸł": 36791, "rir": 36792, "adolescent": 36793, "vinny": 36794, "reng": 36795, "sandstone": 36796, "guet": 36797, "westh": 36798, "pledged": 36799, "laced": 36800, "spide": 36801, "vai": 36802, "tycoon": 36803, "seizure": 36804, "dup": 36805, "appalachian": 36806, "rok": 36807, "catholics": 36808, "seychel": 36809, "possess": 36810, "lager": 36811, "jodi": 36812, "champ": 36813, "stras": 36814, "dina": 36815, "centuri": 36816, "calder": 36817, "bluray": 36818, "ðŁĩ¨ðŁĩ³": 36819, "modo": 36820, "annette": 36821, "youtubers": 36822, "chaps": 36823, "angling": 36824, "labeling": 36825, "aqui": 36826, "pkwy": 36827, "lyle": 36828, "bisexual": 36829, "litur": 36830, "dugout": 36831, "libby": 36832, "greysanatomy": 36833, "substances": 36834, "augustus": 36835, "rallying": 36836, "fidel": 36837, "ingue": 36838, "人": 36839, "hallmarkchannel": 36840, "toothbrush": 36841, "má": 36842, "adirond": 36843, "aggi": 36844, "ðŁĵį:": 36845, "crusade": 36846, "taxation": 36847, "kz": 36848, "iver": 36849, "doubling": 36850, "roomie": 36851, "wab": 36852, "enrolled": 36853, "azon": 36854, "aju": 36855, "grandchildren": 36856, "asdf": 36857, "ðŁ¥º": 36858, "matic": 36859, "oughton": 36860, "utilize": 36861, "ðŁĴ£": 36862, "ponder": 36863, "raisin": 36864, "dysfunction": 36865, "cobain": 36866, "butternut": 36867, "eman": 36868, "sured": 36869, "drian": 36870, "andfriends": 36871, "withthe": 36872, "onomy": 36873, "heineken": 36874, "bridal": 36875, "leadership": 36876, "pyramids": 36877, "deutschland": 36878, "jocel": 36879, "bowel": 36880, "yqr": 36881, "horsepower": 36882, "beacon": 36883, "ingeni": 36884, "gradient": 36885, "fermented": 36886, "moom": 36887, "thingy": 36888, "potassi": 36889, "wristband": 36890, "bord": 36891, "bodied": 36892, "ðŁĺŃðŁĺį": 36893, "mapp": 36894, "kau": 36895, "cyberpunk": 36896, "phish": 36897, "looking": 36898, "coates": 36899, "apur": 36900, "amie": 36901, "uklabour": 36902, "atin": 36903, "gla": 36904, "adoptable": 36905, "shelby": 36906, "villi": 36907, "riya": 36908, "mingly": 36909, "climber": 36910, "bumblebee": 36911, "ðŁĺ¸": 36912, "csd": 36913, "âĿ¥": 36914, "hospitalized": 36915, "cki": 36916, "hater": 36917, "chr": 36918, "retina": 36919, "ita": 36920, "fanbase": 36921, "beatrice": 36922, "gwyne": 36923, "goss": 36924, "fos": 36925, "favorited": 36926, "swachhbharat": 36927, "malade": 36928, "monmouth": 36929, "\"[": 36930, "sivan": 36931, "shhh": 36932, "commanding": 36933, "sainsburys": 36934, "weed": 36935, "gman": 36936, "ssw": 36937, "reptile": 36938, "ivy": 36939, "tropics": 36940, "rollers": 36941, "overcast": 36942, "exposition": 36943, "masquerade": 36944, "mancrush": 36945, "waist": 36946, "sprinter": 36947, "sleet": 36948, "levin": 36949, "jpg": 36950, "_(": 36951, "opel": 36952, "exploit": 36953, "apa": 36954, "powe": 36955, "wrecking": 36956, "jongin": 36957, "orb": 36958, "erick": 36959, "bosco": 36960, "praising": 36961, "bertr": 36962, "towing": 36963, "insecurity": 36964, "kut": 36965, "restocked": 36966, "rrp": 36967, "prescribed": 36968, "trafalgar": 36969, "pert": 36970, "gases": 36971, "apprais": 36972, "ghar": 36973, "musicals": 36974, "âĸ¬âĸ¬": 36975, "mcfad": 36976, "agony": 36977, "condition": 36978, "equip": 36979, "shik": 36980, "atravel": 36981, "ðŁĩ¿ðŁĩ¦": 36982, "keh": 36983, "abduction": 36984, "peoria": 36985, "wilkins": 36986, "gms": 36987, "asd": 36988, "evi": 36989, "ðŁĴĹðŁĴĹðŁĴĹ": 36990, "uz": 36991, "moc": 36992, "hallelujah": 36993, "guadalu": 36994, "louvre": 36995, "drawing": 36996, "gove": 36997, "phant": 36998, "frie": 36999, "webdev": 37000, "programmer": 37001, "zable": 37002, "gamescom": 37003, "clarify": 37004, "lith": 37005, "kinky": 37006, "âĿ£": 37007, "labourdoorstep": 37008, "sonata": 37009, "juris": 37010, "maiden": 37011, "viadu": 37012, "bucharest": 37013, "conditioned": 37014, "capitalist": 37015, "ude": 37016, "psb": 37017, "spca": 37018, "lulla": 37019, "foothills": 37020, "kayo": 37021, "bond": 37022, "womb": 37023, "rounder": 37024, "cesar": 37025, "bursts": 37026, "apra": 37027, "swoon": 37028, "sabrin": 37029, "fragrant": 37030, "clearer": 37031, "kubrick": 37032, "climax": 37033, "journo": 37034, "agle": 37035, "ðŁı½âĢįâĻĢï¸ı": 37036, "pooch": 37037, "hale": 37038, "solit": 37039, "salmon": 37040, "organisms": 37041, "bronson": 37042, "arten": 37043, "hodgson": 37044, "alove": 37045, "venture": 37046, "bbi": 37047, "aea": 37048, "ðŁIJ¢": 37049, "ldn": 37050, "dnr": 37051, "ozone": 37052, "ellas": 37053, "manny": 37054, "azzur": 37055, "unbeat": 37056, "truffles": 37057, "thong": 37058, "mañ": 37059, "lasers": 37060, "leye": 37061, "gettysburg": 37062, "backpacks": 37063, "oris": 37064, "maison": 37065, "crawling": 37066, "labra": 37067, "cling": 37068, "dragging": 37069, "steal": 37070, "doubt": 37071, "devan": 37072, "ckers": 37073, "agentsof": 37074, "photobomb": 37075, "elonmusk": 37076, "aboy": 37077, "distances": 37078, "storyline": 37079, "spi": 37080, "northan": 37081, "europeans": 37082, "whale": 37083, "serpent": 37084, "ðŁļ²": 37085, "fior": 37086, "trit": 37087, "oxo": 37088, "awarding": 37089, "classmate": 37090, "sufc": 37091, "smartest": 37092, "riches": 37093, "prk": 37094, "bigfoot": 37095, "armb": 37096, "bipolar": 37097, "dwelling": 37098, "omars": 37099, "kwan": 37100, "grime": 37101, "meng": 37102, "frederick": 37103, "navarro": 37104, "sorrynotsorry": 37105, "jaredleto": 37106, "pave": 37107, "slack": 37108, "barnsley": 37109, "attar": 37110, "eviction": 37111, "accumulation": 37112, "oir": 37113, "catchy": 37114, "welter": 37115, "vikas": 37116, "hassee": 37117, "nikita": 37118, "moyes": 37119, "mathews": 37120, "shiv": 37121, "gatwick": 37122, "profiling": 37123, "companions": 37124, "marrake": 37125, "antics": 37126, "ðŁĻĮðŁĻĮðŁĻĮ": 37127, "sese": 37128, "boi": 37129, "bartlett": 37130, "poisonous": 37131, "abuses": 37132, "ymm": 37133, "kampala": 37134, "guggenheim": 37135, "imvkohli": 37136, "dolom": 37137, "bree": 37138, "throttle": 37139, "gareth": 37140, "fitzpatrick": 37141, "unya": 37142, "parad": 37143, "margot": 37144, "jnr": 37145, "wea": 37146, "potassium": 37147, "pnc": 37148, "disguised": 37149, "crash": 37150, "renergy": 37151, "illic": 37152, "coupled": 37153, "niels": 37154, "ciones": 37155, "æĹ¥": 37156, "iment": 37157, "despicable": 37158, "dye": 37159, "whatcha": 37160, "connections": 37161, "paralympics": 37162, "gauntlet": 37163, "waitrose": 37164, "suicidal": 37165, "starship": 37166, "vapor": 37167, "stou": 37168, "lawmaker": 37169, "cooled": 37170, "simo": 37171, "theno": 37172, "offroad": 37173, "jaden": 37174, "basque": 37175, "vicky": 37176, "lukaku": 37177, "centro": 37178, "trish": 37179, "strategist": 37180, "medications": 37181, "horst": 37182, "bfc": 37183, "grail": 37184, "sharply": 37185, "aditya": 37186, "tomb": 37187, "kaufman": 37188, "tripad": 37189, "samba": 37190, "pastoral": 37191, "britney": 37192, "sagan": 37193, "hillside": 37194, "masons": 37195, "sara": 37196, "zone": 37197, "xu": 37198, "totes": 37199, "robbie": 37200, "appen": 37201, "montag": 37202, "dero": 37203, "shortfilm": 37204, "charismatic": 37205, "tators": 37206, "kiba": 37207, "andri": 37208, "alarming": 37209, "splitting": 37210, "icar": 37211, "thug": 37212, "scariest": 37213, "sylvester": 37214, "anan": 37215, "utrecht": 37216, "adifference": 37217, "meade": 37218, "buster": 37219, "airstrikes": 37220, "cuffs": 37221, "accountants": 37222, "ðŁĺ¡ðŁĺ¡": 37223, "newt": 37224, "bott": 37225, "issuing": 37226, "clancy": 37227, "wwenetwork": 37228, "kyuhyun": 37229, "resemble": 37230, "pajamas": 37231, "sink": 37232, "kinney": 37233, "sulph": 37234, "ork": 37235, "lies": 37236, "lagh": 37237, "orton": 37238, "rahul": 37239, "dsc": 37240, "wewill": 37241, "ream": 37242, "colloqui": 37243, "sharia": 37244, "hectic": 37245, "sarcasm": 37246, "lander": 37247, "tmz": 37248, "endorf": 37249, "roz": 37250, "hammered": 37251, "fris": 37252, "wadi": 37253, "popefrancis": 37254, "heit": 37255, "flashlight": 37256, "unborn": 37257, "opes": 37258, "holiness": 37259, "ðŁIJ¦": 37260, "nacht": 37261, "imsa": 37262, "gracing": 37263, "bjp": 37264, "verts": 37265, "csc": 37266, "homeowner": 37267, "aque": 37268, "bigotry": 37269, "annie": 37270, "bagh": 37271, "âĿ¤ï¸ıðŁĺį": 37272, "cari": 37273, "thomp": 37274, "disposable": 37275, "cardiology": 37276, "patented": 37277, "hhhhhh": 37278, "ldr": 37279, "stephenson": 37280, "crores": 37281, "fanning": 37282, "climat": 37283, "ðŁijįðŁijįðŁijį": 37284, "ðŁijįðŁı¼": 37285, "aeron": 37286, "piccadilly": 37287, "bankrupt": 37288, "silvia": 37289, "employ": 37290, "donny": 37291, "commenting": 37292, "screenwriter": 37293, "iota": 37294, "cean": 37295, "ancers": 37296, "tuan": 37297, "streetwear": 37298, "य": 37299, "skine": 37300, "espa": 37301, "asif": 37302, "osce": 37303, "sheppard": 37304, "morecam": 37305, "bottle": 37306, "ders": 37307, "oracle": 37308, "googleplay": 37309, "averaged": 37310, "edmonton": 37311, "stephan": 37312, "sisterhood": 37313, "crusted": 37314, "staggering": 37315, "methodology": 37316, "congresswoman": 37317, "cabo": 37318, "triggers": 37319, "milky": 37320, "glide": 37321, "toothpaste": 37322, "roommates": 37323, "nuff": 37324, "guam": 37325, "sprinkles": 37326, "alternative": 37327, "watfordfc": 37328, "uoft": 37329, "haley": 37330, "contacted": 37331, "bundy": 37332, "prostitu": 37333, "ghar": 37334, "preston": 37335, "onsite": 37336, "hilar": 37337, "gts": 37338, "catt": 37339, "hampstead": 37340, "??!": 37341, "ðŁĩ§ðŁĩ": 37342, "bbcqt": 37343, "alessandro": 37344, "resist": 37345, "maidan": 37346, "tko": 37347, "shading": 37348, "pinup": 37349, "gallo": 37350, "sinu": 37351, "atec": 37352, "funk": 37353, "aclu": 37354, "strides": 37355, "rhyme": 37356, "wetland": 37357, "bbcspringwatch": 37358, "tins": 37359, "wildcard": 37360, "stour": 37361, "flamenco": 37362, "paula": 37363, "ontology": 37364, "gangsta": 37365, "amade": 37366, "ãĤ«": 37367, "tbs": 37368, "skeletal": 37369, "runner": 37370, "jardin": 37371, "harrier": 37372, "hunted": 37373, "zhen": 37374, "believeinfilm": 37375, "demean": 37376, "auditi": 37377, "restart": 37378, "chondri": 37379, "âĿ¤ï¸ıðŁĴĻ": 37380, "mclaren": 37381, "gab": 37382, "shum": 37383, "ausa": 37384, "lewisham": 37385, "ypg": 37386, "kjv": 37387, "furnished": 37388, "doro": 37389, "bonded": 37390, "morty": 37391, "latitude": 37392, "_)": 37393, "lova": 37394, "waterways": 37395, "vinai": 37396, "shorth": 37397, "drunk": 37398, "cay": 37399, "ayana": 37400, "kaplan": 37401, "cappuccino": 37402, "spro": 37403, "lifeboat": 37404, "hasbro": 37405, "spolice": 37406, "toron": 37407, "doing": 37408, "damn": 37409, "shree": 37410, "fountains": 37411, "entation": 37412, "maru": 37413, "boarder": 37414, "topless": 37415, "jada": 37416, "channing": 37417, "ulls": 37418, "enclosure": 37419, "gibson": 37420, "fractured": 37421, "britton": 37422, "ö": 37423, "tous": 37424, "porth": 37425, "draf": 37426, "trailing": 37427, "margate": 37428, "elife": 37429, "downward": 37430, "linn": 37431, "glades": 37432, "girlpower": 37433, "akrish": 37434, "uki": 37435, "ronda": 37436, "tsc": 37437, "appreciationday": 37438, "vising": 37439, "loom": 37440, "ðŁį³": 37441, "mexican": 37442, "argos": 37443, "yya": 37444, "jadine": 37445, "southport": 37446, "dend": 37447, "sista": 37448, "redeem": 37449, "meng": 37450, "braxton": 37451, "antioxidant": 37452, "skey": 37453, "mpg": 37454, "finding": 37455, "vibration": 37456, "ceu": 37457, "khart": 37458, "dimini": 37459, "cline": 37460, "shelly": 37461, "hines": 37462, "īï¸ı": 37463, "topical": 37464, "nover": 37465, "maxx": 37466, "primitive": 37467, "illustrate": 37468, "bounds": 37469, "trenton": 37470, "jointly": 37471, "breeders": 37472, "uchi": 37473, "wakeupamerica": 37474, "bada": 37475, "ðŁĹ£ï¸ı": 37476, "guacam": 37477, "spheres": 37478, "peregr": 37479, "youthful": 37480, "lolo": 37481, "birmin": 37482, "tly": 37483, "jeremycorbyn": 37484, "defects": 37485, "cosm": 37486, "arent": 37487, "vaa": 37488, "bagels": 37489, "mediac": 37490, "coriander": 37491, "icago": 37492, "ghaz": 37493, "abbas": 37494, "remodel": 37495, "structuring": 37496, "pum": 37497, "outlaw": 37498, "adani": 37499, "rbc": 37500, "gulls": 37501, "nli": 37502, "confuse": 37503, "ðŁijĩðŁı¼": 37504, "vila": 37505, "mcnamara": 37506, "corrections": 37507, "mughal": 37508, "seri": 37509, "regain": 37510, "ssb": 37511, "leave": 37512, "hahahah": 37513, "grande": 37514, "distressed": 37515, "rechargeable": 37516, "hoa": 37517, "housed": 37518, "stil": 37519, "attributed": 37520, "opathic": 37521, "dips": 37522, "prit": 37523, "headphone": 37524, "conclude": 37525, "pilo": 37526, "het": 37527, "utsa": 37528, "nitin": 37529, "jem": 37530, "snippet": 37531, "tutoring": 37532, "oper": 37533, "sunk": 37534, "ensla": 37535, "chau": 37536, "acorn": 37537, "quintess": 37538, "rankin": 37539, "affiliated": 37540, "ourlives": 37541, "clint": 37542, "seater": 37543, "isaac": 37544, "bashing": 37545, "smear": 37546, "nurse": 37547, "doodling": 37548, "\";": 37549, "saku": 37550, "atrocities": 37551, "imam": 37552, "gfs": 37553, "violating": 37554, "commend": 37555, "bradshaw": 37556, "erville": 37557, "billed": 37558, "bbe": 37559, "thulhu": 37560, "iphones": 37561, "moose": 37562, "dios": 37563, "rew": 37564, "methane": 37565, "strangely": 37566, "whisky": 37567, "tightly": 37568, "spielberg": 37569, "radius": 37570, "noticing": 37571, "wif": 37572, "ignati": 37573, "ifa": 37574, "apis": 37575, "wali": 37576, "haitian": 37577, "bushes": 37578, "yz": 37579, "vl": 37580, "exited": 37581, "assel": 37582, "truec": 37583, "domen": 37584, "asher": 37585, "inking": 37586, "newyearseve": 37587, "hendricks": 37588, "bati": 37589, "ìĿ´ì": 37590, "richter": 37591, "monsanto": 37592, "conline": 37593, "agreat": 37594, "ðŁ¤¯": 37595, "masterpieces": 37596, "arn": 37597, "roughs": 37598, "cleve": 37599, "sev": 37600, "fashions": 37601, "toya": 37602, "shail": 37603, "copeland": 37604, "aquari": 37605, "decals": 37606, "areyou": 37607, "yaya": 37608, "astr": 37609, "font": 37610, "mlm": 37611, "arca": 37612, "ppor": 37613, "pollock": 37614, "xperia": 37615, "conservation": 37616, "chainsaw": 37617, "aggie": 37618, "?!?!?": 37619, "sile": 37620, "shon": 37621, "ìĹIJ": 37622, "notebooks": 37623, "marquette": 37624, "deus": 37625, "bbled": 37626, "spicer": 37627, "mccabe": 37628, "norwich": 37629, "modification": 37630, "boosted": 37631, "strum": 37632, "salesman": 37633, "bangle": 37634, "nissan": 37635, "hezbollah": 37636, "breasts": 37637, "aaf": 37638, "anthus": 37639, "sker": 37640, "owed": 37641, "heros": 37642, "gifs": 37643, "fosters": 37644, "eaters": 37645, "dues": 37646, "_/": 37647, "lymphoma": 37648, "sfam": 37649, "megal": 37650, "afridi": 37651, "agic": 37652, "pamp": 37653, "jealousy": 37654, "ðŁijĮðŁı¼": 37655, "calculate": 37656, "napping": 37657, "gale": 37658, "ðŁ¦Ħ": 37659, "lubbock": 37660, "assumed": 37661, "renting": 37662, "íĥľ": 37663, "suburb": 37664, "ãĤ·": 37665, "technic": 37666, "ucla": 37667, "infront": 37668, "garnet": 37669, "steroids": 37670, "striving": 37671, "howar": 37672, "mover": 37673, "leton": 37674, "bulldo": 37675, "isin": 37676, "ciao": 37677, "snz": 37678, "forefront": 37679, "dams": 37680, "midwife": 37681, "mawards": 37682, "clapton": 37683, "wein": 37684, "subsidies": 37685, "sproud": 37686, "rotherham": 37687, "phantom": 37688, "arach": 37689, "spiel": 37690, "racket": 37691, "selamat": 37692, "noon": 37693, "lbc": 37694, "entially": 37695, "ðŁĴ¸": 37696, "silve": 37697, "moud": 37698, "kinetic": 37699, "yasi": 37700, "ðŁİ©": 37701, "ool": 37702, "miku": 37703, "iza": 37704, "fera": 37705, "floren": 37706, "barbershop": 37707, "groot": 37708, "zest": 37709, "nears": 37710, "stanis": 37711, "zand": 37712, "policeman": 37713, "jurisdic": 37714, "formations": 37715, "apparatus": 37716, "spd": 37717, "artifact": 37718, "tosc": 37719, "motivating": 37720, "womancrush": 37721, "redro": 37722, "diagnostics": 37723, "raza": 37724, "outfitters": 37725, "elxn": 37726, "dodgy": 37727, "ryn": 37728, "shd": 37729, "orthodon": 37730, "olde": 37731, "jayanti": 37732, "balances": 37733, "quickest": 37734, "canton": 37735, "fridayreads": 37736, "!*": 37737, "naa": 37738, "aak": 37739, "ðŁĶ·": 37740, "behaviors": 37741, "raspberries": 37742, "ä»": 37743, "political": 37744, "camil": 37745, "åľ": 37746, "dik": 37747, "astounding": 37748, "liebe": 37749, "novelty": 37750, "turmoil": 37751, "sully": 37752, "springbreak": 37753, "honouring": 37754, "ccg": 37755, "ðŁıĴ": 37756, "mylittle": 37757, "kyc": 37758, "proms": 37759, "ðŁķĬ": 37760, "è": 37761, "bige": 37762, "avril": 37763, "ðŁĩµðŁĩ°": 37764, "marion": 37765, "asants": 37766, "surya": 37767, "octag": 37768, "lufthan": 37769, "acron": 37770, "fayetteville": 37771, "tique": 37772, "loves": 37773, "enca": 37774, "dekalb": 37775, "taver": 37776, "devote": 37777, "auxiliary": 37778, "johannes": 37779, "treadmill": 37780, "ayan": 37781, "qur": 37782, "donaldson": 37783, "cheryl": 37784, "\"....": 37785, "sven": 37786, "kirsty": 37787, "gunners": 37788, "radish": 37789, "oahu": 37790, "vsky": 37791, "ible": 37792, "concourse": 37793, "bps": 37794, "eloqu": 37795, "ashford": 37796, "tebow": 37797, "roblox": 37798, "mada": 37799, "driving": 37800, "thday": 37801, "sproject": 37802, "mms": 37803, "banded": 37804, ".!!": 37805, "librarians": 37806, "flannel": 37807, "intolerance": 37808, "heral": 37809, "çµ": 37810, "nemesis": 37811, "lista": 37812, "tarak": 37813, "crypt": 37814, "starplus": 37815, "vishnu": 37816, "scale": 37817, "cris": 37818, "%),": 37819, "jillian": 37820, "reggae": 37821, "pegasus": 37822, "olin": 37823, "ipment": 37824, "manic": 37825, "lfc": 37826, "goddard": 37827, "iteam": 37828, "parlour": 37829, "anchors": 37830, "leeminho": 37831, "tallahassee": 37832, "antit": 37833, "dho": 37834, "kidney": 37835, "yash": 37836, "battled": 37837, "azad": 37838, "garis": 37839, "faulkner": 37840, "sniff": 37841, "paparazzi": 37842, "edm": 37843, "phyllis": 37844, "contested": 37845, "aaay": 37846, "seca": 37847, "kton": 37848, "velve": 37849, "rainier": 37850, "forum": 37851, "tampab": 37852, "hosp": 37853, "tractors": 37854, "oxfordshire": 37855, "notion": 37856, "guangzhou": 37857, "ðŁĺ¯": 37858, "refill": 37859, "wednesdaymotivation": 37860, "slider": 37861, "mukherjee": 37862, "pratt": 37863, "fontaine": 37864, "alphon": 37865, "afar": 37866, "tsi": 37867, "pesticides": 37868, "fiends": 37869, "mocking": 37870, "braw": 37871, "transat": 37872, "doses": 37873, "cores": 37874, "homophobia": 37875, "documenting": 37876, "zlatan": 37877, "condoms": 37878, "sé": 37879, "sunset": 37880, "kunst": 37881, "tonga": 37882, "ส": 37883, "vation": 37884, "spray": 37885, "chowder": 37886, "raps": 37887, "palladium": 37888, "norwood": 37889, "musichistory": 37890, "hooker": 37891, "sisi": 37892, "osprey": 37893, "phys": 37894, "conceded": 37895, "bobcat": 37896, "armad": 37897, "zeit": 37898, "ÙĦ": 37899, "ðŁĺģðŁĺģ": 37900, "meridi": 37901, "ðŁĩ·ðŁĩº": 37902, "cornwall": 37903, "!),": 37904, "touchdowns": 37905, "zeit": 37906, "chalet": 37907, "mmm": 37908, "alche": 37909, "gorilla": 37910, "foss": 37911, "atiku": 37912, "luminous": 37913, "ivanka": 37914, "beek": 37915, "stares": 37916, "swiss": 37917, "âĿ¤âĿ¤âĿ¤âĿ¤": 37918, "scrubs": 37919, "meath": 37920, "gustav": 37921, "jogging": 37922, "confetti": 37923, "asos": 37924, "ersfc": 37925, "breitbart": 37926, "applicable": 37927, "authored": 37928, "yaho": 37929, "hin": 37930, "displacement": 37931, "jv": 37932, "ðŁĮ¹ðŁĮ¹": 37933, "otc": 37934, "nonprofits": 37935, "diecast": 37936, "gusto": 37937, "intestin": 37938, "cages": 37939, "meen": 37940, "lukas": 37941, "mooney": 37942, "ðŁĺ·": 37943, "veryday": 37944, "torah": 37945, "ission": 37946, "wac": 37947, "leveraging": 37948, "ishable": 37949, "cuse": 37950, "lewood": 37951, "mayan": 37952, "turntable": 37953, "juice": 37954, "trusty": 37955, "tup": 37956, "etiquette": 37957, "supervisors": 37958, "stun": 37959, "guzman": 37960, "conferen": 37961, "rico": 37962, "feast": 37963, "backward": 37964, "polaris": 37965, "miche": 37966, "jog": 37967, "hing": 37968, "fieldhouse": 37969, "veling": 37970, "shocker": 37971, "escence": 37972, "ा": 37973, "vibe": 37974, "anastasia": 37975, "marched": 37976, "killing": 37977, "Ķë": 37978, "fett": 37979, "exoplan": 37980, "...(": 37981, "snowday": 37982, "loh": 37983, "irani": 37984, "lakhs": 37985, "dela": 37986, "pocaly": 37987, "boomers": 37988, "dictatorship": 37989, "acer": 37990, "turkeys": 37991, "quarterfinal": 37992, "musketeers": 37993, "ðŁĴĽðŁĴļ": 37994, "sfx": 37995, "museumweek": 37996, "scala": 37997, "risis": 37998, "(ðŁĵ·": 37999, "ãĢĤ": 38000, "zies": 38001, "boeh": 38002, "hues": 38003, "lusci": 38004, "dola": 38005, "impeachtrump": 38006, "rood": 38007, "doncaster": 38008, "torre": 38009, "heroes": 38010, "foyer": 38011, "tari": 38012, "blurred": 38013, "kew": 38014, "frankly": 38015, "droid": 38016, "apal": 38017, "м": 38018, "yaf": 38019, "bret": 38020, "paragu": 38021, "cacao": 38022, "ðŁĻĮðŁı¾": 38023, "rue": 38024, "headaches": 38025, "shawty": 38026, "charley": 38027, "paler": 38028, "gowns": 38029, "correctional": 38030, "ðŁĺ©ðŁĺ©": 38031, "breakingbad": 38032, "oling": 38033, "dap": 38034, "endeavour": 38035, "citadel": 38036, "trad": 38037, "incumbent": 38038, "meditate": 38039, "footed": 38040, "ðŁĴµ": 38041, "shabbat": 38042, "dayofthe": 38043, "willem": 38044, "galway": 38045, "tored": 38046, "marriage": 38047, "fillion": 38048, "sleeveless": 38049, "auditor": 38050, "jinyoung": 38051, "invincible": 38052, "kaduna": 38053, "aand": 38054, "volcanoes": 38055, "moneti": 38056, "indiegogo": 38057, "buccaneers": 38058, "ðŁijīðŁı½": 38059, "ãĢĤ": 38060, "layton": 38061, "cuckoo": 38062, "humber": 38063, "buzzer": 38064, "Ïī": 38065, "tore": 38066, "strains": 38067, "stom": 38068, "paine": 38069, "swe": 38070, "duff": 38071, "zou": 38072, "simi": 38073, "lipp": 38074, "urn": 38075, "seagu": 38076, "ðŁĶ®": 38077, "sundae": 38078, "hic": 38079, "ðŁĺ¨": 38080, "bullpen": 38081, "uper": 38082, "flyover": 38083, "aldridge": 38084, "globes": 38085, "alies": 38086, "kenzie": 38087, "gees": 38088, "ycle": 38089, "splin": 38090, "magenta": 38091, "jha": 38092, "balu": 38093, "ghorn": 38094, "tipper": 38095, "wicker": 38096, "tasteof": 38097, "conclave": 38098, "chale": 38099, "invasi": 38100, "cater": 38101, "dioxide": 38102, "megab": 38103, "winn": 38104, "atp": 38105, "transformative": 38106, "nestled": 38107, "hig": 38108, "bridging": 38109, "lilies": 38110, "cheered": 38111, "baddest": 38112, "scrolls": 38113, "realis": 38114, "diplo": 38115, "ðŁĶ«": 38116, "concession": 38117, "preferences": 38118, "explodes": 38119, "ergon": 38120, "introductory": 38121, "ineau": 38122, "chaf": 38123, "somes": 38124, "landrover": 38125, "spiration": 38126, "sexy": 38127, "scorecard": 38128, "illustrates": 38129, "soulmate": 38130, "wien": 38131, "interdisciplinary": 38132, "forecasting": 38133, "entities": 38134, "glued": 38135, "enlar": 38136, "curt": 38137, "perceptions": 38138, "bootleg": 38139, "mire": 38140, "ashok": 38141, "vaz": 38142, "horne": 38143, "calle": 38144, "aculture": 38145, "theroy": 38146, "nighttime": 38147, "ocal": 38148, "characterdesign": 38149, "armist": 38150, "ðŁĺıðŁĺı": 38151, "yahoo": 38152, "aceae": 38153, "tose": 38154, "evento": 38155, "sout": 38156, "nayanth": 38157, "whom": 38158, "vare": 38159, "rigging": 38160, "genus": 38161, "hive": 38162, "commands": 38163, "stie": 38164, "daya": 38165, "ethanol": 38166, "enf": 38167, "hifi": 38168, "fluence": 38169, "clemson": 38170, "reinvent": 38171, "thermometer": 38172, "humorous": 38173, "emerging": 38174, "ación": 38175, "ðŁĺĺðŁĺį": 38176, "sity": 38177, "hawke": 38178, "accompanying": 38179, "tility": 38180, "ðŁĺª": 38181, "recess": 38182, "protagonist": 38183, "lery": 38184, "dundal": 38185, "intl": 38186, "brittany": 38187, "qbs": 38188, "offthe": 38189, "marriages": 38190, "howto": 38191, "violated": 38192, "adelaide": 38193, "witt": 38194, "lancer": 38195, "pakv": 38196, "hume": 38197, "stade": 38198, "bragging": 38199, "outright": 38200, "adc": 38201, "superst": 38202, "realtime": 38203, "cures": 38204, "gardeners": 38205, "erock": 38206, "dalejr": 38207, "vero": 38208, "bartol": 38209, "moti": 38210, "mcfly": 38211, "vpn": 38212, "stink": 38213, "overrated": 38214, "guerra": 38215, "etis": 38216, "athome": 38217, "twdfamily": 38218, "thab": 38219, "tnx": 38220, "rafael": 38221, "familytravel": 38222, "xley": 38223, "satanic": 38224, "equations": 38225, "rudy": 38226, "waldorf": 38227, "stani": 38228, "tube": 38229, "measles": 38230, "zimmerman": 38231, "obligations": 38232, "iously": 38233, "bowser": 38234, "transformer": 38235, "shoppe": 38236, "shaken": 38237, "ghouse": 38238, "tod": 38239, "ketball": 38240, "shareholder": 38241, "marca": 38242, "kpmg": 38243, "akan": 38244, "givenchy": 38245, "coastal": 38246, "auth": 38247, "rollercoaster": 38248, "marches": 38249, "coordinate": 38250, "cinema": 38251, "apprentices": 38252, "parlor": 38253, "mito": 38254, "menon": 38255, "considerable": 38256, "barre": 38257, "gloss": 38258, "enhances": 38259, "jazeera": 38260, "falmouth": 38261, "thrash": 38262, "staten": 38263, "kzn": 38264, "engel": 38265, "samanthap": 38266, "floppy": 38267, "salom": 38268, "ðŁıĨðŁıĨ": 38269, "wack": 38270, "deliberate": 38271, "oscill": 38272, "heritag": 38273, "dusted": 38274, "ornithology": 38275, "paddle": 38276, "ferns": 38277, "barun": 38278, "clans": 38279, "anticipate": 38280, "aay": 38281, "matically": 38282, "éĩ": 38283, "tumble": 38284, "postman": 38285, "unicef": 38286, "trotter": 38287, "opd": 38288, "leaflet": 38289, "geist": 38290, "ceasefire": 38291, "screws": 38292, "creation": 38293, "walnuts": 38294, "longhorns": 38295, "understatement": 38296, "abb": 38297, "proximity": 38298, "nax": 38299, "unity": 38300, "turnpike": 38301, "ordained": 38302, "dubstep": 38303, "chakra": 38304, "mech": 38305, "loveher": 38306, "lookalike": 38307, "donnein": 38308, "viron": 38309, "ÙĪ": 38310, "bangers": 38311, "variants": 38312, "outdated": 38313, "inta": 38314, "cristo": 38315, "spelt": 38316, "foodand": 38317, "fon": 38318, "stefani": 38319, "marginal": 38320, "hutton": 38321, "tiara": 38322, "telford": 38323, "quen": 38324, "fairgrounds": 38325, "quetta": 38326, "mikhail": 38327, "healer": 38328, "vball": 38329, "tyre": 38330, "undergrad": 38331, "glend": 38332, "homers": 38333, "scribed": 38334, "maintains": 38335, "poche": 38336, "missal": 38337, "marko": 38338, "uas": 38339, "án": 38340, "shp": 38341, "convey": 38342, "padre": 38343, "saba": 38344, "puglia": 38345, "madhuri": 38346, "paxton": 38347, "chaplain": 38348, "nago": 38349, "casi": 38350, "...!!!": 38351, "flirt": 38352, "saleh": 38353, "kare": 38354, "dire": 38355, "stamped": 38356, "extreme": 38357, "ðŁĺĥðŁĺĥ": 38358, "hoppy": 38359, "guadalupe": 38360, "advantaged": 38361, "euchar": 38362, "plow": 38363, "unn": 38364, "macqu": 38365, "portland": 38366, "clash": 38367, "pes": 38368, "loubout": 38369, "yp": 38370, "keeping": 38371, "arcadia": 38372, "frankie": 38373, "fiu": 38374, "deth": 38375, "encyclopedia": 38376, "size": 38377, "invests": 38378, "ðŁį©": 38379, "geological": 38380, "franç": 38381, "confront": 38382, "ðŁĺ¥": 38383, "dys": 38384, "afm": 38385, "texan": 38386, "graphene": 38387, "repostapp": 38388, "acf": 38389, "ursula": 38390, "gaza": 38391, "ddled": 38392, "fum": 38393, "wsbtv": 38394, "mbe": 38395, "frontiers": 38396, "chronograph": 38397, "kes": 38398, "interfaith": 38399, "taboo": 38400, "sparta": 38401, "wondo": 38402, "florist": 38403, "embraces": 38404, "caw": 38405, "noel": 38406, "archers": 38407, "ðŁIJ·": 38408, "romano": 38409, "banan": 38410, "shakers": 38411, "melodies": 38412, "geothermal": 38413, "sephora": 38414, "ìļ°": 38415, "од": 38416, "proc": 38417, "handshake": 38418, "pande": 38419, "populated": 38420, "slowdown": 38421, "hortons": 38422, "registrations": 38423, "undeni": 38424, "lants": 38425, "passover": 38426, "thakur": 38427, "lief": 38428, "adhesive": 38429, "petal": 38430, "microscopy": 38431, "memphis": 38432, "confirming": 38433, "airdrop": 38434, "mesmer": 38435, "perceived": 38436, "mingle": 38437, "lifeline": 38438, "ghj": 38439, "worcestershire": 38440, "passions": 38441, "acher": 38442, "ellar": 38443, "aho": 38444, "firenze": 38445, "barang": 38446, "letterman": 38447, "hatfield": 38448, "lucha": 38449, "jeter": 38450, "eshop": 38451, "williams": 38452, "horoscope": 38453, "prede": 38454, "eastbourne": 38455, "durga": 38456, "diversion": 38457, "altrin": 38458, "seismic": 38459, "premiosm": 38460, "narco": 38461, "tir": 38462, "orig": 38463, "orm": 38464, "landfall": 38465, "cious": 38466, "lindo": 38467, "maxine": 38468, "xico": 38469, "tray": 38470, "oswald": 38471, "cba": 38472, "ricotta": 38473, "ncr": 38474, "marau": 38475, "า": 38476, "gladiator": 38477, "chery": 38478, "lung": 38479, "ume": 38480, "popsic": 38481, "longing": 38482, "canals": 38483, "taya": 38484, "decentralized": 38485, "shopp": 38486, "pressures": 38487, "maharaj": 38488, "etihad": 38489, "walgreens": 38490, "succession": 38491, "signaling": 38492, "lig": 38493, "staffer": 38494, "northkorea": 38495, "defying": 38496, "asma": 38497, "deg": 38498, "perimeter": 38499, "oakville": 38500, "msk": 38501, "baltimore": 38502, "receip": 38503, "deple": 38504, "ðŁĺŃðŁĺĤ": 38505, "jamboree": 38506, ">.<": 38507, "rspb": 38508, "punisher": 38509, "considerably": 38510, "intothe": 38511, "parisian": 38512, "accelerated": 38513, "polyester": 38514, "lowes": 38515, "frying": 38516, "sautéed": 38517, "mouths": 38518, "seychelles": 38519, "rax": 38520, "godis": 38521, "dakota": 38522, "housewives": 38523, "theme": 38524, "matinee": 38525, "blackbird": 38526, "yesung": 38527, "prefers": 38528, "pellegr": 38529, "inated": 38530, "trunks": 38531, "strongertogether": 38532, "repet": 38533, "repairing": 38534, "pedals": 38535, "tolerant": 38536, "herr": 38537, "dunne": 38538, "indication": 38539, "decatur": 38540, "btv": 38541, "exhibitors": 38542, "ikon": 38543, "fridaymotivation": 38544, "bragg": 38545, "livetweet": 38546, "alves": 38547, "womensart": 38548, "foreigners": 38549, "wallets": 38550, "mindy": 38551, "laney": 38552, "bbin": 38553, "tvmiaw": 38554, "lifter": 38555, "target": 38556, "tame": 38557, "drou": 38558, "astrophotography": 38559, "mpc": 38560, "gpu": 38561, "nordstrom": 38562, "friction": 38563, "runoff": 38564, "lovable": 38565, "spnfamily": 38566, "extingui": 38567, "bloody": 38568, "schel": 38569, "artistry": 38570, "swish": 38571, "scarce": 38572, "phils": 38573, "maxim": 38574, "possum": 38575, "compromised": 38576, "styli": 38577, "scfc": 38578, "issa": 38579, "birmingham": 38580, "sketched": 38581, "angelica": 38582, "ordinance": 38583, "jets": 38584, "conquer": 38585, "ðŁĺIJ": 38586, "onlineshopping": 38587, "sori": 38588, "reasonably": 38589, "nuestro": 38590, "arturo": 38591, "chl": 38592, "benefici": 38593, "sphoto": 38594, "welt": 38595, "nikk": 38596, "ðŁ¤ŀ": 38597, "danao": 38598, "formid": 38599, "asse": 38600, "afirst": 38601, "âľĤ": 38602, "gillette": 38603, "assor": 38604, "anonym": 38605, "selca": 38606, "femi": 38607, "bearable": 38608, "yand": 38609, "armory": 38610, "crepe": 38611, "celticfc": 38612, "bravo": 38613, "inexpensive": 38614, "delec": 38615, "gecko": 38616, "newmarket": 38617, "snowflakes": 38618, "kabir": 38619, "contra": 38620, "canning": 38621, "morpho": 38622, "garwal": 38623, "ðŁĴĥðŁı»": 38624, "fighting": 38625, "mutation": 38626, "woody": 38627, "jugg": 38628, "graces": 38629, "premiosmtvmiaw": 38630, "kennedy": 38631, "gup": 38632, "sae": 38633, "opha": 38634, "offspring": 38635, "finisher": 38636, "betts": 38637, "spanning": 38638, "marj": 38639, "hone": 38640, "shing": 38641, "continents": 38642, "samanthaprabhu": 38643, "unrelated": 38644, "lacy": 38645, "explosions": 38646, "benjamin": 38647, "sophie": 38648, "noting": 38649, "microsoft": 38650, "assen": 38651, "ahoy": 38652, "iker": 38653, "hofer": 38654, "moe": 38655, "ahmadi": 38656, "yann": 38657, "anak": 38658, "mahi": 38659, "beu": 38660, "ahah": 38661, "creeper": 38662, "baahubali": 38663, "amat": 38664, "priory": 38665, "hawkeye": 38666, "deloitte": 38667, "skoda": 38668, "printmaking": 38669, "assembling": 38670, "miraculous": 38671, "noch": 38672, "swo": 38673, "lega": 38674, "operates": 38675, "borderlands": 38676, "elie": 38677, "strongh": 38678, "reptiles": 38679, "pirate": 38680, "unfold": 38681, "¯": 38682, "qualcomm": 38683, "unpredictable": 38684, "otr": 38685, "rosewood": 38686, "directional": 38687, "counselors": 38688, "cornell": 38689, "liberated": 38690, "jad": 38691, "irregular": 38692, "bulgarian": 38693, "highness": 38694, "vodafone": 38695, "swild": 38696, "minimize": 38697, "grazie": 38698, "à¹ĩ": 38699, "rstats": 38700, "streep": 38701, "ometric": 38702, "humble": 38703, "lump": 38704, "lille": 38705, "bü": 38706, "homedepot": 38707, "tripadvisor": 38708, "kiwan": 38709, "avia": 38710, "erz": 38711, "exico": 38712, "duf": 38713, "blumen": 38714, "mizing": 38715, "arma": 38716, "inim": 38717, "constan": 38718, "sora": 38719, "jual": 38720, "aun": 38721, "twell": 38722, "trenches": 38723, "hera": 38724, "rk": 38725, "poplar": 38726, "recipeoftheday": 38727, "llan": 38728, "bhuban": 38729, "shortages": 38730, "ingdon": 38731, "bridgewater": 38732, "ðŁIJĺ": 38733, "fortnite": 38734, "camden": 38735, "uncture": 38736, "prow": 38737, "colonies": 38738, "tks": 38739, "ngo": 38740, "bhm": 38741, "livepd": 38742, "splace": 38743, "slike": 38744, "happyeaster": 38745, "terrence": 38746, "revolver": 38747, "jed": 38748, "yyyy": 38749, "officeof": 38750, "mts": 38751, "existential": 38752, "rourke": 38753, "explorebc": 38754, "ssed": 38755, "priest": 38756, "vixen": 38757, "siding": 38758, "kpa": 38759, "ahar": 38760, "juic": 38761, "obstruc": 38762, "forensics": 38763, "ukmfg": 38764, "cancellation": 38765, "weary": 38766, "abq": 38767, "elec": 38768, "prized": 38769, "debts": 38770, "mezz": 38771, "salvatore": 38772, "mdc": 38773, "grette": 38774, "cgc": 38775, "thon": 38776, "snowstorm": 38777, "tsch": 38778, "cookery": 38779, "å¹": 38780, "waxing": 38781, "nacional": 38782, "murs": 38783, "rave": 38784, "capes": 38785, "germain": 38786, "dripping": 38787, "submitting": 38788, "omelette": 38789, "iteration": 38790, "ajes": 38791, "shimmer": 38792, "fueling": 38793, "ðŁĩ§ðŁĩª": 38794, "lipo": 38795, "bobble": 38796, "unfollow": 38797, "islamist": 38798, "hiber": 38799, "cats": 38800, "agentsofshield": 38801, "sensi": 38802, "_____": 38803, "steria": 38804, "instal": 38805, "auspicious": 38806, "harrow": 38807, "overland": 38808, "feminists": 38809, "instant": 38810, "chariot": 38811, "blindness": 38812, "sped": 38813, "scarec": 38814, "nuit": 38815, "miniatures": 38816, "hoseok": 38817, "glock": 38818, "fifaworldcup": 38819, "ete": 38820, "dism": 38821, "weiner": 38822, "exfoli": 38823, "earts": 38824, "à¸Ķ": 38825, "myart": 38826, "manil": 38827, "issant": 38828, "forma": 38829, "incu": 38830, "buffalob": 38831, "intim": 38832, "mccul": 38833, "anjali": 38834, "popo": 38835, "undoub": 38836, "hila": 38837, "fungal": 38838, "thankful": 38839, "futur": 38840, "endish": 38841, "rends": 38842, "thar": 38843, "sheff": 38844, "ringo": 38845, "nicholls": 38846, "iowa": 38847, "potom": 38848, "clams": 38849, "ãģĦ": 38850, "aconf": 38851, "stadiums": 38852, "dimp": 38853, "dik": 38854, "residences": 38855, "dov": 38856, "caricature": 38857, "seagull": 38858, "klm": 38859, "confess": 38860, "slapped": 38861, "celeb": 38862, "turbines": 38863, "ppv": 38864, "nurture": 38865, "elab": 38866, ".....#": 38867, "tuff": 38868, "depress": 38869, "alfar": 38870, "amiibo": 38871, "dispon": 38872, "ewing": 38873, "queer": 38874, "friends": 38875, "forre": 38876, "âĺ¼": 38877, "swt": 38878, "aquarius": 38879, "headliner": 38880, "curd": 38881, "figs": 38882, "otters": 38883, "lovefl": 38884, "kareem": 38885, "govegan": 38886, "friyay": 38887, "consolation": 38888, "atri": 38889, "ì§Ħ": 38890, "âĺĿï¸ı": 38891, "polyne": 38892, "gued": 38893, "oya": 38894, "laus": 38895, "intestinal": 38896, "camilla": 38897, "scalp": 38898, "pir": 38899, "leeds": 38900, "horrifying": 38901, "boretum": 38902, "dandelion": 38903, "ferrer": 38904, "ellic": 38905, "asx": 38906, "soren": 38907, "reloaded": 38908, "aleague": 38909, "navigator": 38910, "inette": 38911, "addams": 38912, "alchemist": 38913, "akshay": 38914, "dystopian": 38915, "awec": 38916, "naya": 38917, "alisa": 38918, "ailed": 38919, "agor": 38920, "aviator": 38921, "alizer": 38922, "smobile": 38923, "findyourpark": 38924, "copying": 38925, "toddy": 38926, "shti": 38927, "monger": 38928, "calhoun": 38929, "napkin": 38930, "breakup": 38931, "yatra": 38932, "sethu": 38933, "richi": 38934, "erasmus": 38935, "ferry": 38936, "amore": 38937, "practise": 38938, "bobo": 38939, "powerpoint": 38940, "oose": 38941, "liffe": 38942, "china": 38943, "shka": 38944, "fadnavis": 38945, "duane": 38946, "waron": 38947, "false": 38948, "ðŁļĤ": 38949, "washes": 38950, "discip": 38951, "========": 38952, "gk": 38953, "abb": 38954, "stubborn": 38955, "medieval": 38956, "pci": 38957, "ðŁįª": 38958, "marilyn": 38959, "hyo": 38960, "mandi": 38961, "cri": 38962, "predecess": 38963, "continuation": 38964, "omusic": 38965, "slat": 38966, "whal": 38967, "mallory": 38968, "bonn": 38969, "shenzhen": 38970, "cai": 38971, "âĺĥ": 38972, "safest": 38973, "forwards": 38974, "drawers": 38975, "blasted": 38976, "slee": 38977, "morphe": 38978, "mbta": 38979, "dumbass": 38980, "ÑĦоÑĤо": 38981, "alhamdulillah": 38982, "eclub": 38983, "albeit": 38984, "healey": 38985, "ayurveda": 38986, "advertised": 38987, "crocs": 38988, "ittles": 38989, "bryson": 38990, "bei": 38991, "njpw": 38992, "honoree": 38993, "fused": 38994, "ðŁĶĺ": 38995, "multin": 38996, "naga": 38997, "departs": 38998, "kop": 38999, "kino": 39000, "jharkhand": 39001, "edna": 39002, "axle": 39003, "milton": 39004, "supremacist": 39005, "marrakech": 39006, "dominic": 39007, "transcript": 39008, "][#": 39009, ":).": 39010, "woc": 39011, "surrounds": 39012, "ogil": 39013, "leaflets": 39014, "cowell": 39015, "whew": 39016, "trude": 39017, "prolifer": 39018, "succes": 39019, "sportsman": 39020, "condom": 39021, "poche": 39022, "kup": 39023, "imprisonment": 39024, "{}": 39025, "scrambled": 39026, "åĽ": 39027, "kaine": 39028, "cellphone": 39029, "metamor": 39030, "coni": 39031, "remnants": 39032, "eez": 39033, "downpour": 39034, "afternoon": 39035, "exercising": 39036, "berser": 39037, "architecture": 39038, "wicklow": 39039, "mns": 39040, "isp": 39041, "boc": 39042, "niss": 39043, "mnwild": 39044, "stumble": 39045, "rsi": 39046, "luffy": 39047, "silen": 39048, "ddad": 39049, "bullies": 39050, "hawker": 39051, "bbcc": 39052, "scuba": 39053, "epp": 39054, "quets": 39055, "foraging": 39056, "pallet": 39057, "hadi": 39058, "cinematographer": 39059, "catchers": 39060, "toaster": 39061, "khi": 39062, "litecoin": 39063, "kidlit": 39064, "amherst": 39065, "mauricio": 39066, "ipad": 39067, "marmalade": 39068, "fey": 39069, "donnelly": 39070, "gto": 39071, "estas": 39072, "cerebral": 39073, "antgrasso": 39074, "zzled": 39075, "virgil": 39076, "swapped": 39077, "ðŁĺħðŁĺħ": 39078, "nodapl": 39079, "greatest": 39080, "nhlbruins": 39081, "fraser": 39082, "bmo": 39083, "anew": 39084, ".âĿ¤ï¸ı": 39085, "segregation": 39086, "remarkably": 39087, "mccormick": 39088, "logger": 39089, "eras": 39090, "contracting": 39091, "âłĢâłĢ": 39092, "yorks": 39093, "ukulele": 39094, "touchscreen": 39095, "decked": 39096, "benn": 39097, "southwark": 39098, "ravin": 39099, "numis": 39100, "ðŁ¤Ļ": 39101, "rut": 39102, "greco": 39103, "ethic": 39104, "redneck": 39105, "arr": 39106, "tcs": 39107, "ihri": 39108, "ðŁĩ«ðŁĩ·": 39109, "lk": 39110, "inherited": 39111, "zyk": 39112, "viaduct": 39113, "martyred": 39114, "higu": 39115, "ssn": 39116, "bein": 39117, "streetstyle": 39118, "fergie": 39119, "bankof": 39120, "æĹ¥": 39121, "stakeholder": 39122, "exemplary": 39123, "cress": 39124, "essa": 39125, "erotica": 39126, "intrepid": 39127, "gomes": 39128, "braun": 39129, "bethany": 39130, "bangtan": 39131, "pulmonary": 39132, "milling": 39133, "doctorate": 39134, "trumprussia": 39135, "र": 39136, "sani": 39137, "blatt": 39138, "plau": 39139, "deprived": 39140, "tle": 39141, "fully": 39142, "bourn": 39143, "stak": 39144, "lufthansa": 39145, "kiosk": 39146, "faroo": 39147, "defy": 39148, "badan": 39149, "ðŁĺĺâĿ¤ï¸ı": 39150, "ritz": 39151, "trisha": 39152, "rands": 39153, "middlesex": 39154, "arabs": 39155, "proj": 39156, "sportscenter": 39157, "repeats": 39158, "ivf": 39159, "bleedblue": 39160, "assure": 39161, "obs": 39162, "territorial": 39163, "elen": 39164, "beverley": 39165, "annah": 39166, "âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı": 39167, "zl": 39168, "forgood": 39169, "sciencefiction": 39170, "glau": 39171, "sonya": 39172, "prith": 39173, "stweets": 39174, "mixers": 39175, "mario": 39176, "antelope": 39177, "writingcommunity": 39178, "wentz": 39179, "denham": 39180, "bedi": 39181, "sfo": 39182, "harleydavidson": 39183, "lookbook": 39184, "immunotherapy": 39185, "orphe": 39186, "esville": 39187, "edged": 39188, "task": 39189, "sbball": 39190, "corrosion": 39191, "kilometers": 39192, "costing": 39193, "playback": 39194, "keke": 39195, "divisi": 39196, "uter": 39197, "relocation": 39198, "yelled": 39199, "peng": 39200, "upbeat": 39201, "serve": 39202, "âļł": 39203, "halen": 39204, "stirring": 39205, "rehman": 39206, "env": 39207, "schumacher": 39208, "fragment": 39209, "alkaline": 39210, "sbk": 39211, "resili": 39212, "sharepoint": 39213, "rollover": 39214, "trash": 39215, "counterpart": 39216, "âĻ«": 39217, "obitu": 39218, "à½": 39219, "ãĤ¹": 39220, "mulberry": 39221, "ðŁİĨ": 39222, "autonomy": 39223, "spraying": 39224, "natl": 39225, "loveyou": 39226, "franki": 39227, "nuk": 39228, "escar": 39229, "canteen": 39230, "alibaba": 39231, "deplor": 39232, "molecule": 39233, "pud": 39234, "fortnight": 39235, "blondie": 39236, "sphin": 39237, "portrayal": 39238, "tache": 39239, "bute": 39240, "consisting": 39241, "freepalestine": 39242, "csp": 39243, "immort": 39244, "dns": 39245, "ðŁĴ¥ðŁĴ¥": 39246, "tourde": 39247, "cooking": 39248, "archival": 39249, "gathers": 39250, "bitt": 39251, "banc": 39252, "premature": 39253, "snowball": 39254, "poetryday": 39255, "loudly": 39256, "fugitive": 39257, "eday": 39258, "emra": 39259, "ðŁĩ¸ðŁĩª": 39260, "scien": 39261, "nodejs": 39262, "jurgen": 39263, "jeong": 39264, "bandana": 39265, "unis": 39266, "foxsports": 39267, "vandy": 39268, "provisions": 39269, "weep": 39270, "tuk": 39271, "iko": 39272, "houn": 39273, "ziggy": 39274, "zr": 39275, "fillet": 39276, "bata": 39277, "tink": 39278, "cone": 39279, "wewant": 39280, "kilo": 39281, "horace": 39282, "slt": 39283, "sct": 39284, "staytuned": 39285, "victoria": 39286, "umbria": 39287, "attacker": 39288, "inghamshire": 39289, "frightening": 39290, "noir": 39291, "frat": 39292, "contempt": 39293, "liaison": 39294, "hoi": 39295, "brink": 39296, "trill": 39297, "niagar": 39298, "kickass": 39299, "dundas": 39300, "notmy": 39301, "rhode": 39302, "bumble": 39303, "noxi": 39304, "fag": 39305, "spectators": 39306, "mancrushmonday": 39307, "jinping": 39308, "distract": 39309, "daisy": 39310, "walden": 39311, "portrait": 39312, "arthistory": 39313, "voltron": 39314, "evel": 39315, "isc": 39316, "acm": 39317, "rite": 39318, "nao": 39319, "deported": 39320, "sweats": 39321, "rufus": 39322, "lobo": 39323, "laborday": 39324, "gamo": 39325, "ihrithik": 39326, "blit": 39327, "abdominal": 39328, "ãħ¤ãħ¤ãħ¤ãħ¤": 39329, "iit": 39330, "eq": 39331, "busy": 39332, "alluarjun": 39333, "undisclosed": 39334, "deton": 39335, "procreate": 39336, "kil": 39337, "ðŁİĤðŁİĤ": 39338, "mitchell": 39339, "kii": 39340, "inheritance": 39341, "alp": 39342, "joburg": 39343, "patrolling": 39344, "compulsory": 39345, "unsigned": 39346, "niam": 39347, "lga": 39348, "eshopsuk": 39349, "trilli": 39350, "maw": 39351, "appreciating": 39352, "rockab": 39353, "mañana": 39354, "antal": 39355, "malvern": 39356, "royo": 39357, "grandprix": 39358, "sutton": 39359, "goftheday": 39360, "digi": 39361, "ãħĭãħĭãħĭãħĭ": 39362, "tles": 39363, "varanasi": 39364, "erected": 39365, "disciples": 39366, "contact": 39367, "ðŁĺµ": 39368, "lid": 39369, "â¬ĩ": 39370, "scentre": 39371, "radiator": 39372, "ingtips": 39373, "transitions": 39374, "thursdaymotivation": 39375, "chemical": 39376, "separati": 39377, "salis": 39378, "mim": 39379, "geographical": 39380, "bookfest": 39381, "/.": 39382, "âľĭ": 39383, "vae": 39384, "currie": 39385, "aggarwal": 39386, "acceleration": 39387, "theses": 39388, "lgm": 39389, "umass": 39390, "proportions": 39391, "nata": 39392, "anians": 39393, "kuch": 39394, "beacons": 39395, "apr": 39396, "@#": 39397, "ðŁĴªðŁı¾": 39398, "nuke": 39399, "sheraton": 39400, "kio": 39401, "makati": 39402, "politico": 39403, "morale": 39404, "ìĻ": 39405, "economically": 39406, "ggly": 39407, "ssen": 39408, "pastries": 39409, "internships": 39410, "vicente": 39411, "fantaken": 39412, "avengers": 39413, "accuse": 39414, "sleepover": 39415, "indicated": 39416, "thedream": 39417, "sterone": 39418, "renders": 39419, "frost": 39420, "oui": 39421, "gregg": 39422, "dore": 39423, "⾨⾨⾨": 39424, "pugs": 39425, "saty": 39426, "numb": 39427, "hemsworth": 39428, "tami": 39429, "lassic": 39430, "schiff": 39431, "iglesias": 39432, "agawa": 39433, "]\"": 39434, "reshi": 39435, "gamestop": 39436, "divorced": 39437, "theater": 39438, "claudi": 39439, "unconventional": 39440, "prophets": 39441, "acin": 39442, "twelf": 39443, "towering": 39444, "tml": 39445, "sclerosis": 39446, "kwan": 39447, "gets": 39448, "disturb": 39449, "naira": 39450, "energ": 39451, "piracy": 39452, "pruitt": 39453, "notified": 39454, "henna": 39455, "bram": 39456, "groundwater": 39457, "bls": 39458, "optimis": 39459, "$)": 39460, "lucie": 39461, "bizhour": 39462, "fangirling": 39463, "grills": 39464, "orl": 39465, "verse": 39466, "cina": 39467, "lawless": 39468, "artistsontwitter": 39469, "televised": 39470, "marshmallows": 39471, "radiohead": 39472, "barr": 39473, "mfc": 39474, "brevi": 39475, "mmorpg": 39476, "gaya": 39477, "âĸ«": 39478, "subtitles": 39479, "jt": 39480, "disneyland": 39481, "tobago": 39482, "nhm": 39483, "groove": 39484, "fiawec": 39485, "\"/": 39486, "bao": 39487, "scrabble": 39488, "omni": 39489, "ffl": 39490, "umc": 39491, "simba": 39492, "alier": 39493, "terrell": 39494, "plume": 39495, "midi": 39496, "dignit": 39497, "coc": 39498, "brut": 39499, "adata": 39500, "alchemy": 39501, "dsm": 39502, "ðŁĺĨðŁĺĨ": 39503, "wintry": 39504, "spares": 39505, "cuer": 39506, "conclusions": 39507, "toys": 39508, "odor": 39509, "flann": 39510, "garvey": 39511, "scriptions": 39512, "inspections": 39513, "catap": 39514, "anglo": 39515, "stlouis": 39516, "heimer": 39517, "atay": 39518, "trich": 39519, "enyc": 39520, "childs": 39521, "ventil": 39522, "montp": 39523, "guillermo": 39524, "circulare": 39525, "zell": 39526, "modeled": 39527, "craftsman": 39528, "alina": 39529, "stimulation": 39530, "cashew": 39531, "judas": 39532, "bestof": 39533, "toire": 39534, "suspends": 39535, "scollege": 39536, "realising": 39537, "bytes": 39538, "bloods": 39539, "assi": 39540, "ðŁĴ¿": 39541, "ohs": 39542, "ðŁįĭ": 39543, "scallop": 39544, "व": 39545, "gifting": 39546, "camogie": 39547, "wilkes": 39548, "ozzy": 39549, "ðŁ¤¤": 39550, "veronic": 39551, "savoy": 39552, "demetri": 39553, "babygirl": 39554, "ðŁĺįðŁĺŃ": 39555, "sox": 39556, "clyde": 39557, "inductee": 39558, "countdown": 39559, "selfcare": 39560, "à¤ľ": 39561, "vika": 39562, "torre": 39563, "phdchat": 39564, "pears": 39565, "awh": 39566, "suffrage": 39567, "lesn": 39568, "admiration": 39569, "mpp": 39570, "sharkweek": 39571, "schulz": 39572, "santorini": 39573, "clover": 39574, "(*": 39575, "strasbourg": 39576, "exiting": 39577, "soyu": 39578, "fingerprint": 39579, "chea": 39580, "ãĢľ": 39581, "vindic": 39582, "songwriters": 39583, "soa": 39584, "prouder": 39585, "nama": 39586, "=))": 39587, "simplest": 39588, "deliciously": 39589, "gilles": 39590, "uq": 39591, "mnwx": 39592, "epp": 39593, "shun": 39594, "kennel": 39595, "fallon": 39596, "ðŁIJ£": 39597, "sind": 39598, "tragically": 39599, "outes": 39600, "modernism": 39601, "coke": 39602, "gyn": 39603, "spion": 39604, "âĺ¹ï¸ı": 39605, "leam": 39606, "compressor": 39607, "apologise": 39608, "twentyon": 39609, "fanatics": 39610, "âĻ»": 39611, "scotsman": 39612, "sawa": 39613, "kou": 39614, "aser": 39615, "à¸ļ": 39616, "welterweight": 39617, "phenom": 39618, "twickenham": 39619, "stria": 39620, "pout": 39621, "kaz": 39622, "giam": 39623, "cdp": 39624, "hoy": 39625, "employ": 39626, "redmond": 39627, "à¸Ħà¸": 39628, "smere": 39629, "trancefamily": 39630, "protocols": 39631, "piece": 39632, "luiz": 39633, "iteracy": 39634, "carls": 39635, "unitedstates": 39636, "harmed": 39637, "phdlife": 39638, "chaw": 39639, "footprints": 39640, "lé": 39641, "choker": 39642, "zana": 39643, "slipper": 39644, "ericsson": 39645, "insulting": 39646, "artichoke": 39647, "advising": 39648, "acquisitions": 39649, "opor": 39650, "mutations": 39651, "rear": 39652, "à¥ģ": 39653, "podcast": 39654, "wither": 39655, "kung": 39656, "íĺ¸": 39657, "winslow": 39658, "diapers": 39659, "ðŁĵ¸@": 39660, "ecker": 39661, "collar": 39662, "huey": 39663, "giro": 39664, "monogram": 39665, "kasich": 39666, "siveness": 39667, "malaysi": 39668, "aromatic": 39669, "gres": 39670, "galileo": 39671, "uji": 39672, "robb": 39673, "drm": 39674, "nonetheless": 39675, "asa": 39676, ":>": 39677, "loa": 39678, "lnp": 39679, "atwork": 39680, "agt": 39681, "lakshmi": 39682, "pipelines": 39683, "idal": 39684, "strel": 39685, "reall": 39686, "chainz": 39687, "stonewall": 39688, "sansk": 39689, "ðŁı´": 39690, "piedmont": 39691, "hostess": 39692, "ciu": 39693, "té": 39694, "analyses": 39695, "wilhelm": 39696, "scotty": 39697, "rwby": 39698, "mosquit": 39699, "usemb": 39700, "quins": 39701, "ðŁijİ": 39702, "tucker": 39703, "sconf": 39704, "specifications": 39705, "psychiatry": 39706, "brookes": 39707, "sils": 39708, "olaf": 39709, "deto": 39710, "codi": 39711, "clip": 39712, "filth": 39713, "womancrushwednesday": 39714, "goto": 39715, "angerous": 39716, "beale": 39717, "wtc": 39718, "panelist": 39719, "nex": 39720, "larsen": 39721, "emilio": 39722, "tableau": 39723, "hitters": 39724, "conceived": 39725, "americani": 39726, "ortega": 39727, "mardi": 39728, "Ñĥ": 39729, "paintball": 39730, "thirsty": 39731, "newyorker": 39732, "etisation": 39733, "goss": 39734, "weaker": 39735, "ugh": 39736, "troll": 39737, "harga": 39738, "dual": 39739, "ghtning": 39740, "atine": 39741, "ðŁĺİðŁĺİðŁĺİ": 39742, "cookout": 39743, "pyrenees": 39744, "poss": 39745, "authentication": 39746, "sportswear": 39747, "yunho": 39748, "kiro": 39749, "archipel": 39750, "shenko": 39751, "render": 39752, "novation": 39753, "divinity": 39754, "ðŁij£": 39755, "sufi": 39756, "humbling": 39757, "geopol": 39758, "devotees": 39759, "waitress": 39760, "trough": 39761, "pyro": 39762, "iba": 39763, "bling": 39764, "graf": 39765, "epilots": 39766, "btr": 39767, "oftball": 39768, "basking": 39769, "dominos": 39770, "soom": 39771, "rath": 39772, "sheryl": 39773, "quel": 39774, "astronomical": 39775, "weld": 39776, "tracklist": 39777, "signee": 39778, "sleepless": 39779, "comman": 39780, "chron": 39781, "summon": 39782, "puremichigan": 39783, "crispr": 39784, "slip": 39785, "lagi": 39786, "raq": 39787, "umu": 39788, "thalap": 39789, "charmed": 39790, "scrump": 39791, "quadcopter": 39792, "skip": 39793, "petersen": 39794, "muni": 39795, "ðŁĮ¾": 39796, "monaghan": 39797, "trays": 39798, "icked": 39799, "canadaday": 39800, "tegr": 39801, "�": 39802, "hotness": 39803, "heavymetal": 39804, "abar": 39805, "gopdebate": 39806, "azul": 39807, "spiderman": 39808, "sunflowers": 39809, "ľë": 39810, "webcomics": 39811, "bard": 39812, "в": 39813, "nicholas": 39814, "slush": 39815, "raman": 39816, "markham": 39817, "fficial": 39818, "ffler": 39819, "íĬ¸": 39820, "pless": 39821, "anushka": 39822, "toto": 39823, "skaters": 39824, "prowrestling": 39825, "competes": 39826, "ayala": 39827, "mystery": 39828, "thrills": 39829, "mpg": 39830, "independently": 39831, "yul": 39832, "imperative": 39833, "formidable": 39834, "tireless": 39835, "stacking": 39836, "tongues": 39837, "maltese": 39838, "potts": 39839, "matti": 39840, "charting": 39841, "chillout": 39842, "supernova": 39843, "omeo": 39844, "skysports": 39845, "nutty": 39846, "ðŁĹĵï¸ı": 39847, "rohan": 39848, "inspired": 39849, "concierge": 39850, "serra": 39851, "makk": 39852, "galat": 39853, "chipp": 39854, "yev": 39855, "ì£": 39856, "reimbur": 39857, "opul": 39858, "kimberley": 39859, "ieee": 39860, "bremen": 39861, "chitec": 39862, "orin": 39863, "naku": 39864, "bonkers": 39865, "footy": 39866, "emergence": 39867, "ðŁĨĺ": 39868, "stip": 39869, "sergei": 39870, "zoey": 39871, "aime": 39872, "would": 39873, "dyes": 39874, "destiny": 39875, "vinaigrette": 39876, "drier": 39877, "circulareconomy": 39878, "anarchi": 39879, "ssr": 39880, "schel": 39881, "ciner": 39882, "groom": 39883, "determining": 39884, "garmin": 39885, "calais": 39886, "incarceration": 39887, "bukit": 39888, "noi": 39889, "chelmsford": 39890, "mckinley": 39891, "chipped": 39892, "belonged": 39893, "tumors": 39894, "stroud": 39895, "mii": 39896, "influenza": 39897, "wwenxt": 39898, "tundra": 39899, "telecommunications": 39900, "catsofinstagram": 39901, "tages": 39902, "beatty": 39903, "odu": 39904, "mlkday": 39905, "ooper": 39906, "dangle": 39907, "akley": 39908, "crumb": 39909, "antigua": 39910, "timbers": 39911, "rouhani": 39912, "ðŁĴªðŁĴªðŁĴª": 39913, "hafi": 39914, "...!!": 39915, "wcs": 39916, "coop": 39917, "snc": 39918, "litres": 39919, "ãĢĬ": 39920, "haz": 39921, "coz": 39922, "kant": 39923, "greenfield": 39924, "curti": 39925, "yale": 39926, "flyeagles": 39927, "whatsoever": 39928, "worthing": 39929, "roulette": 39930, "flyeaglesfly": 39931, "unda": 39932, "ainted": 39933, "standing": 39934, "luscious": 39935, "hpc": 39936, "efficacy": 39937, "ashland": 39938, "meghan": 39939, "kywx": 39940, "npr": 39941, "bathtub": 39942, "acos": 39943, "hani": 39944, "marcor": 39945, "mantis": 39946, "daisi": 39947, "boba": 39948, "abbie": 39949, "mutil": 39950, "vial": 39951, "spyder": 39952, "poz": 39953, "gti": 39954, "elfie": 39955, "nightw": 39956, "metroid": 39957, "antoni": 39958, "maddie": 39959, "dhry": 39960, "darlings": 39961, "tends": 39962, "taekwondo": 39963, "atlanta": 39964, "meow": 39965, "chloe": 39966, "ãĥİ": 39967, "ymes": 39968, "siberia": 39969, "kcon": 39970, "gues": 39971, "mariner": 39972, "facil": 39973, "azzle": 39974, "[...": 39975, "hannover": 39976, "bavaria": 39977, "virgo": 39978, "teuk": 39979, "usps": 39980, ")#": 39981, "walla": 39982, "sampson": 39983, "needless": 39984, "verbally": 39985, "hayley": 39986, "bowled": 39987, "pius": 39988, "lampard": 39989, "hamstring": 39990, "volvo": 39991, "roadsafety": 39992, "choking": 39993, "sorbet": 39994, "ahem": 39995, "healthyfood": 39996, "braided": 39997, "horticulture": 39998, "crative": 39999, "cheek": 40000, "addo": 40001, "theforce": 40002, "koko": 40003, "schizoph": 40004, "jie": 40005, "wada": 40006, "twentyonepilots": 40007, "hbcu": 40008, "proton": 40009, "pauls": 40010, "louisa": 40011, "latam": 40012, "kyrgy": 40013, "compac": 40014, "sdk": 40015, "sapi": 40016, "???": 40017, "liberalism": 40018, "epsilon": 40019, "aiden": 40020, "wusa": 40021, "sprayed": 40022, "basketball": 40023, "kimono": 40024, "bluewave": 40025, "alias": 40026, "ë§Ī": 40027, "mugshot": 40028, "cec": 40029, "dogre": 40030, "adora": 40031, "ðŁĵ·@": 40032, "krakow": 40033, "intrigued": 40034, "exhausting": 40035, "astronomer": 40036, "venison": 40037, "ladybug": 40038, "civ": 40039, "brae": 40040, "usm": 40041, "bribe": 40042, "acupuncture": 40043, "pembroke": 40044, "keating": 40045, "chie": 40046, "yad": 40047, "tsi": 40048, "smi": 40049, "seeding": 40050, "gateshead": 40051, "lisboa": 40052, "gyp": 40053, "canvass": 40054, "ðŁĶ´âļªï¸ı": 40055, "opi": 40056, "nir": 40057, "societal": 40058, "lyte": 40059, "aties": 40060, "csm": 40061, "artery": 40062, "alin": 40063, "akapoor": 40064, "abstracts": 40065, "âĢ¦âĢ¦": 40066, "teenwolf": 40067, "newe": 40068, "travelgram": 40069, "sentimental": 40070, "perched": 40071, "handel": 40072, "hoek": 40073, "fay": 40074, "coordinating": 40075, "animate": 40076, "manian": 40077, "effort": 40078, "jerky": 40079, "fck": 40080, "adrienne": 40081, "mably": 40082, "trading": 40083, "myel": 40084, "spiro": 40085, "sola": 40086, "storing": 40087, "overdrive": 40088, "mondaymorning": 40089, "dreamteam": 40090, "pulse": 40091, "bondi": 40092, "bernie": 40093, "pgatour": 40094, "tripoli": 40095, "sonam": 40096, "platt": 40097, "âļ¡": 40098, "agroup": 40099, "îIJĴ": 40100, "invading": 40101, "vcu": 40102, "kell": 40103, "ños": 40104, "undead": 40105, "podcasting": 40106, "mercedesam": 40107, "manafort": 40108, "cortex": 40109, "queso": 40110, "impeccable": 40111, "palmer": 40112, "wildoz": 40113, "sportsc": 40114, "guacamole": 40115, "dispenser": 40116, "categori": 40117, "stunts": 40118, "peril": 40119, "invitations": 40120, "dunedin": 40121, "xie": 40122, "achieves": 40123, "safer": 40124, "preds": 40125, "phan": 40126, "knuckles": 40127, "kak": 40128, "ignores": 40129, "lovemyjob": 40130, "aruba": 40131, "oundation": 40132, "datacenter": 40133, "covert": 40134, "gring": 40135, "couple": 40136, "ار": 40137, "voli": 40138, "mccle": 40139, "artisans": 40140, "ludo": 40141, "kalam": 40142, "aroma": 40143, "undertaker": 40144, "hula": 40145, "wizkid": 40146, "gumb": 40147, "godfrey": 40148, "bakersfield": 40149, "kern": 40150, "engineer": 40151, "carve": 40152, "palin": 40153, "guarantees": 40154, "pebbles": 40155, "bays": 40156, "zieg": 40157, "fink": 40158, "â¬ĩï¸ıâ¬ĩï¸ı": 40159, "downpours": 40160, "rochelle": 40161, "raspberry": 40162, "ðŁĺ®": 40163, "graphies": 40164, "stomp": 40165, "cafes": 40166, "arized": 40167, "uttar": 40168, "calvary": 40169, "drie": 40170, "crusader": 40171, "busan": 40172, "tuxedo": 40173, "siu": 40174, "seamus": 40175, "cultured": 40176, "blanchard": 40177, "townhouse": 40178, "gered": 40179, "buttermilk": 40180, "fluctu": 40181, "rogerfederer": 40182, "heli": 40183, "ðŁ¦ĥ": 40184, "uous": 40185, "ramesh": 40186, "muppets": 40187, "emailmarketing": 40188, "yess": 40189, "brice": 40190, "rizio": 40191, "pelo": 40192, "donneinarte": 40193, "urable": 40194, "investin": 40195, "bumping": 40196, "rajiv": 40197, "sava": 40198, "thrower": 40199, "forex": 40200, "ohhhh": 40201, "thrust": 40202, "pullman": 40203, "rfid": 40204, "sepsis": 40205, "leed": 40206, "fright": 40207, "rounding": 40208, "neb": 40209, "phins": 40210, "aisha": 40211, "utilizing": 40212, "squats": 40213, "goldsmith": 40214, "jic": 40215, "boks": 40216, "vaus": 40217, "ipo": 40218, "exclusion": 40219, "tariff": 40220, "pokes": 40221, "minal": 40222, "lands": 40223, "enforce": 40224, "washingtondc": 40225, "orchar": 40226, "gx": 40227, "marys": 40228, "eyour": 40229, "aussie": 40230, "bakers": 40231, "unpopular": 40232, "latinos": 40233, "large": 40234, "putnam": 40235, "bolo": 40236, "wade": 40237, "pelo": 40238, "dizz": 40239, "obstruction": 40240, "flappy": 40241, "wearethe": 40242, "dependence": 40243, "pajama": 40244, "ete": 40245, "yann": 40246, "ewan": 40247, "discla": 40248, "aay": 40249, "karina": 40250, "eic": 40251, "antrim": 40252, "wsoc": 40253, "negatively": 40254, "kaido": 40255, "fotografia": 40256, "dhru": 40257, "colossal": 40258, "mcleod": 40259, "kwang": 40260, "manipu": 40261, "exhilar": 40262, "usatoday": 40263, "summerslam": 40264, "coles": 40265, "taproom": 40266, "unbeatable": 40267, "dema": 40268, "ticks": 40269, "kling": 40270, "fils": 40271, "campaigners": 40272, "à¸ķ": 40273, "brewster": 40274, "audubon": 40275, "quay": 40276, "chs": 40277, "kigali": 40278, "dler": 40279, "strengthens": 40280, "somal": 40281, "signingday": 40282, "golds": 40283, "pigment": 40284, "orchestral": 40285, "gq": 40286, "linkin": 40287, "ðŁıĩ": 40288, "taw": 40289, "algarve": 40290, "hov": 40291, "earle": 40292, "goldfish": 40293, "amig": 40294, "exer": 40295, "benin": 40296, "druid": 40297, "ðŁIJ¸": 40298, "shem": 40299, "quattro": 40300, "mercen": 40301, "mente": 40302, "incorporating": 40303, "bonanza": 40304, "statefair": 40305, "ende": 40306, "conceptions": 40307, "ees": 40308, "âĻ¥ï¸ıâĻ¥ï¸ı": 40309, "dson": 40310, "firearm": 40311, "orbital": 40312, "weh": 40313, "multip": 40314, "fob": 40315, "requiem": 40316, "plight": 40317, "thouse": 40318, "said": 40319, "ocre": 40320, "remembrance": 40321, "nold": 40322, "chipping": 40323, "bev": 40324, "ert": 40325, "cathy": 40326, "sym": 40327, "riggs": 40328, "mley": 40329, "dialogues": 40330, "slender": 40331, "howl": 40332, "gauteng": 40333, "wdw": 40334, "tobi": 40335, "smokes": 40336, "implo": 40337, "bpm": 40338, "adn": 40339, "mombasa": 40340, "capsul": 40341, "bloomfield": 40342, "articul": 40343, "cleo": 40344, "googled": 40345, "fluffy": 40346, "lard": 40347, "enzyme": 40348, "vesti": 40349, "ibrahi": 40350, "flame": 40351, "emea": 40352, "outages": 40353, "dispropor": 40354, "bleak": 40355, "ansel": 40356, "icker": 40357, "stlouis": 40358, "stockmarket": 40359, "goodfriday": 40360, "sault": 40361, "stalled": 40362, "prom": 40363, "epsom": 40364, "bé": 40365, "these": 40366, "sauces": 40367, "mew": 40368, "litfest": 40369, "pred": 40370, "reu": 40371, "karak": 40372, "sienna": 40373, "ellin": 40374, "biotechnology": 40375, "ï¸ıâĥ£-": 40376, "tactic": 40377, "sain": 40378, "pork": 40379, "monza": 40380, "kaj": 40381, "lush": 40382, "compartment": 40383, "changing": 40384, "shraddhakapoor": 40385, "foal": 40386, "artem": 40387, "cuando": 40388, "canola": 40389, "oriente": 40390, "messe": 40391, "dited": 40392, "brc": 40393, "boxer": 40394, "bbctwo": 40395, "sst": 40396, "mentday": 40397, "eming": 40398, "dewey": 40399, "kofi": 40400, "âŀĸâŀĸâŀĸâŀĸ": 40401, "realization": 40402, "smol": 40403, "twood": 40404, "sanje": 40405, "flagstaff": 40406, "berwick": 40407, "corset": 40408, "canary": 40409, "whistleblower": 40410, "etched": 40411, "composing": 40412, "squeezed": 40413, "bower": 40414, "autodesk": 40415, "neh": 40416, "mathieu": 40417, "baja": 40418, "ÅĤ": 40419, "hydra": 40420, "daim": 40421, "ameri": 40422, "insisted": 40423, "merlot": 40424, "garros": 40425, "heartnews": 40426, "gainesville": 40427, "cutler": 40428, "bode": 40429, "ðŁĺīðŁĺī": 40430, "lewes": 40431, "scountry": 40432, "gsa": 40433, "usu": 40434, "ccm": 40435, "godawgs": 40436, "pharaoh": 40437, "crae": 40438, "morley": 40439, "hypnoti": 40440, "fades": 40441, "neurons": 40442, "fuzz": 40443, "ingco": 40444, "highlanders": 40445, "stark": 40446, "vigne": 40447, "packets": 40448, "amarillo": 40449, "reuben": 40450, "insults": 40451, "basic": 40452, "vector": 40453, "nme": 40454, "acruz": 40455, "tros": 40456, "transmitter": 40457, "ðŁĺŀ": 40458, "interpret": 40459, "ðŁĺ²": 40460, "prequel": 40461, "mcgowan": 40462, "dissemin": 40463, "ðŁĴĺðŁĴĺ": 40464, "masculinity": 40465, "indiegamedev": 40466, "alive": 40467, "tet": 40468, "petal": 40469, "emailed": 40470, "armed": 40471, "koo": 40472, "heer": 40473, "baird": 40474, "superjunior": 40475, "metropolis": 40476, "delavin": 40477, "declines": 40478, "stitutes": 40479, "Ûģ": 40480, "ptbo": 40481, "glan": 40482, "chores": 40483, "ealing": 40484, "chrissy": 40485, "stemc": 40486, "vian": 40487, "assassinated": 40488, "pronounce": 40489, "illegals": 40490, "discovery": 40491, "cavill": 40492, "frifotos": 40493, "fal": 40494, "soi": 40495, "sabotage": 40496, "tint": 40497, "pdc": 40498, "ðŁİīðŁİĪ": 40499, "ãĤĬãģ": 40500, "jio": 40501, "endeavor": 40502, "insig": 40503, "committees": 40504, "shearer": 40505, "metz": 40506, "marrying": 40507, "hdd": 40508, "gby": 40509, "fret": 40510, "trish": 40511, "pul": 40512, "scripted": 40513, "saki": 40514, "lw": 40515, "keye": 40516, "shimi": 40517, "nanaimo": 40518, "cah": 40519, "ë": 40520, "tempered": 40521, "ician": 40522, "dugg": 40523, "dishwasher": 40524, "airfield": 40525, "srugby": 40526, "grinch": 40527, "yst": 40528, "rms": 40529, "mahatma": 40530, "lankan": 40531, "discar": 40532, "digestion": 40533, "nodes": 40534, "lls": 40535, "omic": 40536, "gutter": 40537, "tisgarh": 40538, "federico": 40539, "electionday": 40540, "bohe": 40541, "mastercard": 40542, "fireball": 40543, "âľĶï¸ı": 40544, "oyster": 40545, "pong": 40546, "dok": 40547, "enroute": 40548, "mvc": 40549, "beatthe": 40550, "alistair": 40551, "shub": 40552, "shaming": 40553, "chernobyl": 40554, "ghibli": 40555, "thes": 40556, "pinion": 40557, "dbs": 40558, "salts": 40559, "iction": 40560, "epiph": 40561, "ncpol": 40562, "inconvenience": 40563, "whitley": 40564, "inspecting": 40565, "woodley": 40566, "wiener": 40567, "skillet": 40568, "noles": 40569, "mca": 40570, "hina": 40571, "asha": 40572, "willingness": 40573, "wellness": 40574, "tamed": 40575, "showtime": 40576, "disadvantaged": 40577, "bernat": 40578, "usn": 40579, "missionaries": 40580, "counselling": 40581, "arrogant": 40582, "quantitative": 40583, "legalization": 40584, "hodge": 40585, "energyefficiency": 40586, "camerondallas": 40587, "possessions": 40588, "pbb": 40589, "harrisburg": 40590, "vg": 40591, "hinduism": 40592, "happythanksgiving": 40593, "fib": 40594, "reacting": 40595, "tweetapicture": 40596, "politi": 40597, "muppet": 40598, "hurrah": 40599, "pace": 40600, "coastguard": 40601, "guarded": 40602, "asam": 40603, "parry": 40604, "forevery": 40605, "xq": 40606, "oomf": 40607, "keanu": 40608, "jind": 40609, "rist": 40610, "customerservice": 40611, "sacred": 40612, "ðŁĺº": 40613, "toner": 40614, "occurrence": 40615, "matu": 40616, "valdez": 40617, "redd": 40618, "isak": 40619, "powerrangers": 40620, "peasant": 40621, "rajini": 40622, "abraham": 40623, "emil": 40624, "cardo": 40625, "tril": 40626, "hairstyles": 40627, "obsolete": 40628, "sampler": 40629, "directive": 40630, "delavinkisses": 40631, "verton": 40632, "glos": 40633, "spay": 40634, "palermo": 40635, "comets": 40636, "manziel": 40637, "chicagof": 40638, "skipped": 40639, "pictorial": 40640, "hant": 40641, "bmi": 40642, "aol": 40643, "reopens": 40644, "paddling": 40645, "devos": 40646, "fraud": 40647, "baseline": 40648, "queues": 40649, "spired": 40650, "snare": 40651, "euve": 40652, "descriptions": 40653, "daisies": 40654, "caching": 40655, "galleria": 40656, "trimmed": 40657, "stino": 40658, "recycla": 40659, "icular": 40660, "birken": 40661, "rawlings": 40662, "flix": 40663, "chicas": 40664, "bgt": 40665, "likeli": 40666, "argyll": 40667, "thelove": 40668, "gaston": 40669, "blanca": 40670, "hak": 40671, "fone": 40672, "sailormoon": 40673, "haci": 40674, "imac": 40675, "flyn": 40676, "decan": 40677, "belles": 40678, "apic": 40679, "zog": 40680, "taunton": 40681, "constance": 40682, "lasagna": 40683, "kernel": 40684, "inka": 40685, "harbor": 40686, "collectively": 40687, "calculated": 40688, "aville": 40689, "shilpa": 40690, "purdu": 40691, "gimm": 40692, "funer": 40693, "aest": 40694, "pembrokeshire": 40695, "nightingale": 40696, "nunes": 40697, "hypertension": 40698, "hubert": 40699, "sliders": 40700, "infertility": 40701, "commended": 40702, "transatlantic": 40703, "metrical": 40704, "!!@": 40705, "ÅŁ": 40706, "ssg": 40707, "bacca": 40708, "inverted": 40709, "funfactfriday": 40710, "itans": 40711, "album": 40712, "acquainted": 40713, "rier": 40714, "whelan": 40715, "sarab": 40716, "mue": 40717, "snooze": 40718, "piff": 40719, "agreeing": 40720, "spitting": 40721, "jermaine": 40722, "nye": 40723, "âľıï¸ı": 40724, "ambush": 40725, "zeph": 40726, "congreg": 40727, "university": 40728, "sapp": 40729, "wannabe": 40730, "patrice": 40731, "ibd": 40732, "doglo": 40733, "fridges": 40734, "sund": 40735, "kingston": 40736, "argon": 40737, "kamen": 40738, "hardrock": 40739, "dsley": 40740, "dolores": 40741, "ì°": 40742, "otaku": 40743, "piping": 40744, "behaving": 40745, "âŃIJï¸ıâŃIJï¸ıâŃIJï¸ı": 40746, "bluebird": 40747, "ansari": 40748, "teapot": 40749, "firework": 40750, "crop": 40751, "logans": 40752, "typed": 40753, "thickness": 40754, "igers": 40755, "cfp": 40756, "dysfunctional": 40757, "contrasting": 40758, "etty": 40759, "astonmartin": 40760, "txst": 40761, "dragrace": 40762, "attributes": 40763, "marathon": 40764, "manuscripts": 40765, "johnstone": 40766, "ðŁĺ±ðŁĺ±": 40767, "boer": 40768, "ayu": 40769, "arugula": 40770, "poorest": 40771, "condu": 40772, "assumption": 40773, "anagh": 40774, "noh": 40775, "delavin": 40776, "sitter": 40777, "gö": 40778, "morow": 40779, "kickstart": 40780, "comi": 40781, "glacial": 40782, "ghead": 40783, "bain": 40784, "kershaw": 40785, "endof": 40786, "freud": 40787, "omat": 40788, "iaf": 40789, "hug": 40790, "signup": 40791, "eachother": 40792, "definite": 40793, "tubing": 40794, "shakira": 40795, "ðŁijıðŁı½": 40796, "uuuu": 40797, "swin": 40798, "shambles": 40799, "olas": 40800, "skell": 40801, "britain": 40802, "knw": 40803, "clutter": 40804, "omy": 40805, "jens": 40806, "hanged": 40807, "cityscape": 40808, "scraps": 40809, "unlocking": 40810, "deadliest": 40811, "erno": 40812, "breastcancer": 40813, "ait": 40814, "inspect": 40815, "furi": 40816, "ðŁĴĮ": 40817, "kud": 40818, "jule": 40819, "orah": 40820, "mids": 40821, "mdt": 40822, "burgring": 40823, "rattle": 40824, "pusa": 40825, "stalk": 40826, "cleans": 40827, "issance": 40828, "zek": 40829, "worthit": 40830, "nameis": 40831, "muskoka": 40832, "councilman": 40833, "urbanart": 40834, "barrac": 40835, "unsolved": 40836, "tul": 40837, "gita": 40838, "whiteboard": 40839, "soybeans": 40840, "ement": 40841, "conti": 40842, "saturdaymotivation": 40843, "conveniently": 40844, "docking": 40845, "tado": 40846, "âı©": 40847, "spino": 40848, "puppylove": 40849, "pof": 40850, "fabricated": 40851, "robbers": 40852, "adopts": 40853, "tified": 40854, "kkr": 40855, "indulgence": 40856, "noticeable": 40857, "macquarie": 40858, "chapel": 40859, "sensual": 40860, "kiko": 40861, "melanoma": 40862, "loretta": 40863, "liance": 40864, "aben": 40865, "splus": 40866, "gaal": 40867, "acele": 40868, "libdems": 40869, "comparisons": 40870, "ðŁĮµ": 40871, "rhythms": 40872, "mery": 40873, "encapsul": 40874, "napier": 40875, "ðŁijĮðŁijĮðŁijĮ": 40876, "ðŁijIJ": 40877, "platz": 40878, "fresno": 40879, "reformed": 40880, "ranbir": 40881, "elit": 40882, "thebest": 40883, "bhushan": 40884, "vinnie": 40885, "improvised": 40886, "sittin": 40887, "recreated": 40888, "eba": 40889, "ecker": 40890, "acrob": 40891, "ponte": 40892, "cord": 40893, "giddy": 40894, "eurusd": 40895, "fever": 40896, "intuition": 40897, "gari": 40898, "dummies": 40899, "budweiser": 40900, "amendments": 40901, "tetra": 40902, "schnit": 40903, "ayas": 40904, "marys": 40905, "cist": 40906, "kani": 40907, "kermit": 40908, "ðŁĺ±ðŁĺ±ðŁĺ±": 40909, "tinker": 40910, "strolling": 40911, "divisional": 40912, "nigeri": 40913, "ominous": 40914, "menstrual": 40915, "karab": 40916, "khy": 40917, "bwfc": 40918, "panhandle": 40919, "lilli": 40920, "weller": 40921, "strapped": 40922, "sonthe": 40923, "transferring": 40924, "ethereal": 40925, "sneaks": 40926, "rudol": 40927, "gables": 40928, "jacking": 40929, "cincode": 40930, "fortune": 40931, "canadiens": 40932, "confor": 40933, "abnormal": 40934, "franklin": 40935, "tita": 40936, "mula": 40937, "persist": 40938, "cuties": 40939, "kiel": 40940, "ðŁĩ±ðŁĩ": 40941, "hermann": 40942, "awk": 40943, "fiasco": 40944, "koto": 40945, "weta": 40946, "hiker": 40947, "buddy": 40948, "preventive": 40949, "mcgraw": 40950, "gameboy": 40951, "forsyth": 40952, "topshop": 40953, "siob": 40954, "sadh": 40955, "intram": 40956, "followart": 40957, "soaps": 40958, "dragonball": 40959, "oux": 40960, "morrison": 40961, "à¹ĥ": 40962, "lubric": 40963, "adulthood": 40964, "morrisons": 40965, "âļłï¸ı": 40966, "hermo": 40967, "taka": 40968, "stallone": 40969, "misuse": 40970, "teamgb": 40971, "ragha": 40972, "confined": 40973, "aty": 40974, "homophobic": 40975, "nwo": 40976, "skynews": 40977, "hoya": 40978, "acrosse": 40979, "wiiu": 40980, "purée": 40981, "jeddah": 40982, "ðŁ¤§": 40983, "advisers": 40984, "phine": 40985, "anis": 40986, "scrumptious": 40987, "ë°ķ": 40988, "cke": 40989, "viny": 40990, "term": 40991, "sdc": 40992, "odo": 40993, "homeschool": 40994, "vasc": 40995, "leopards": 40996, "deborah": 40997, "illicit": 40998, "curran": 40999, "asroma": 41000, "naught": 41001, "marig": 41002, "brandi": 41003, "emp": 41004, "ðŁĺįðŁijĮ": 41005, "îĮ": 41006, "suspend": 41007, "luz": 41008, "initiation": 41009, "schaft": 41010, "jensenackles": 41011, "crawler": 41012, "postdoc": 41013, "desks": 41014, "trailblazer": 41015, "denomin": 41016, "trix": 41017, "noise": 41018, "poet": 41019, "±ï¸ı": 41020, "smug": 41021, "volatile": 41022, "proofs": 41023, "pharmacist": 41024, "sardinia": 41025, "mashable": 41026, "kimchi": 41027, "coed": 41028, "schalke": 41029, "doodled": 41030, "csw": 41031, "shur": 41032, "rox": 41033, "dok": 41034, "chrisbrown": 41035, "mathematician": 41036, "abound": 41037, "angelic": 41038, "rockford": 41039, "dole": 41040, "yorkers": 41041, "msn": 41042, "gman": 41043, "xavier": 41044, "borrowing": 41045, "markings": 41046, "longhorn": 41047, "kja": 41048, "diverted": 41049, "mmit": 41050, "euphoria": 41051, "ayyy": 41052, "tea": 41053, "pah": 41054, "cki": 41055, "uncut": 41056, "liven": 41057, "kyung": 41058, "fanart": 41059, "mering": 41060, "redding": 41061, "amovie": 41062, "gridi": 41063, "cthulhu": 41064, "scholarly": 41065, "judah": 41066, "thbewithyou": 41067, "eucalyp": 41068, "ðŁIJķ": 41069, "hertfordshire": 41070, "courtroom": 41071, "byu": 41072, "auctioned": 41073, "please": 41074, "marcia": 41075, "ê°ĵ": 41076, "succeeded": 41077, "elas": 41078, "arvind": 41079, "tlot": 41080, "saigon": 41081, "rett": 41082, "rakesh": 41083, "fdny": 41084, "asen": 41085, "sebring": 41086, "gladiators": 41087, "youknow": 41088, "vlad": 41089, "gola": 41090, "parap": 41091, "ÑĢи": 41092, "sabcnews": 41093, "oneteam": 41094, "ohl": 41095, "sune": 41096, "rij": 41097, "cdc": 41098, "stargate": 41099, "rundown": 41100, "plato": 41101, "phc": 41102, "chatter": 41103, "raviol": 41104, "mnf": 41105, "mandala": 41106, "liet": 41107, "à¸ķ": 41108, "maria": 41109, "hungover": 41110, "consolidation": 41111, "ferrell": 41112, "traditional": 41113, "iloveart": 41114, "galap": 41115, "ðŁıĮ": 41116, "quezon": 41117, "españa": 41118, "ðŁĩ¨ðŁĩŃ": 41119, "hobby": 41120, "steamboat": 41121, "malign": 41122, "guillau": 41123, "prohi": 41124, "itsme": 41125, "íĥĢ": 41126, "inscription": 41127, "alz": 41128, "marian": 41129, "kade": 41130, "mmon": 41131, "adjusting": 41132, "nests": 41133, "internally": 41134, "cir": 41135, "vikram": 41136, "malala": 41137, "kph": 41138, "felicia": 41139, "thereal": 41140, "captivity": 41141, "atis": 41142, "marcorubio": 41143, "kaleido": 41144, "chev": 41145, "manoj": 41146, "lemore": 41147, "gentri": 41148, "vips": 41149, "trope": 41150, "\"âĢĶ": 41151, "pairings": 41152, "malnutrition": 41153, "fray": 41154, "designation": 41155, "brunomars": 41156, "aze": 41157, "torrential": 41158, "panzer": 41159, "gail": 41160, "underthe": 41161, "theological": 41162, "schizophre": 41163, "dazzle": 41164, "frederic": 41165, "mopar": 41166, "adilla": 41167, "soggy": 41168, "raun": 41169, "mediocre": 41170, "colorec": 41171, "ife": 41172, "pinst": 41173, "bluef": 41174, "²": 41175, "worldwater": 41176, "giroud": 41177, "clarinet": 41178, "adolf": 41179, "tarantino": 41180, "receipts": 41181, "assump": 41182, "ðŁijŁ": 41183, "coffees": 41184, "âľĬðŁı¾": 41185, "duplex": 41186, "sof": 41187, "rx": 41188, "lino": 41189, "timberwolves": 41190, "pandit": 41191, "motm": 41192, "ega": 41193, "ayama": 41194, "achs": 41195, "outsider": 41196, "llen": 41197, "coer": 41198, "tilly": 41199, "cheeseburger": 41200, "mads": 41201, "pledis": 41202, "empty": 41203, "nationalparks": 41204, "aziz": 41205, "pmi": 41206, "junkies": 41207, "fener": 41208, "sqn": 41209, "ès": 41210, "generation": 41211, "cleopatra": 41212, "bhubanes": 41213, "mosques": 41214, "tyfree": 41215, "poppins": 41216, "twc": 41217, "orwell": 41218, "nage": 41219, "kawhi": 41220, "hollow": 41221, "dalai": 41222, "¨¨¨¨": 41223, "ouro": 41224, "mhealth": 41225, "gion": 41226, "azo": 41227, "visas": 41228, "renegade": 41229, "reic": 41230, "wsop": 41231, "ðŁĴļðŁĴĽ": 41232, "echel": 41233, "toxicity": 41234, "mün": 41235, "bunk": 41236, "stimulating": 41237, "asthour": 41238, "\\'": 41239, "eph": 41240, "endemic": 41241, "cnbc": 41242, "shrinking": 41243, "peabody": 41244, "michelangelo": 41245, "canyon": 41246, "wale": 41247, "sumi": 41248, "siders": 41249, "inuit": 41250, "?.": 41251, "professionalism": 41252, "dracing": 41253, "platoon": 41254, "pons": 41255, "outbound": 41256, "mapleleafs": 41257, "desol": 41258, "cency": 41259, "athan": 41260, "verma": 41261, "rubbing": 41262, "okan": 41263, "ðŁijł": 41264, "mullins": 41265, "authentic": 41266, "Åį": 41267, "almanac": 41268, "gaia": 41269, "bbq": 41270, "onimo": 41271, "keh": 41272, "tya": 41273, "touts": 41274, "yav": 41275, "reposit": 41276, ",.": 41277, "wight": 41278, "seeyou": 41279, "callof": 41280, "donesia": 41281, "bargaining": 41282, "granth": 41283, "sdsu": 41284, "amphitheater": 41285, "psu": 41286, "rewatching": 41287, "winetasting": 41288, "peakdistrict": 41289, "detecting": 41290, "thurman": 41291, "phee": 41292, "èªķ": 41293, "umich": 41294, "rer": 41295, "sculpted": 41296, "gole": 41297, "namesake": 41298, "ðŁĶģ": 41299, "servicing": 41300, "baugh": 41301, "pugh": 41302, "pencil": 41303, "darth": 41304, "munchkin": 41305, "atorium": 41306, "teners": 41307, "suny": 41308, "rollingstones": 41309, "maging": 41310, "starrer": 41311, "idris": 41312, "feinstein": 41313, "agron": 41314, "âĺºï¸ıâĺºï¸ı": 41315, "supervised": 41316, "chameleon": 41317, "aggregate": 41318, "successive": 41319, "mogul": 41320, "instyle": 41321, "poldark": 41322, "custome": 41323, "ohiostate": 41324, "haya": 41325, "cides": 41326, "brokerage": 41327, "angelou": 41328, "fifawwc": 41329, "deforestation": 41330, "alton": 41331, "pamph": 41332, "hugged": 41333, "hobo": 41334, "changeable": 41335, "kuber": 41336, "burroughs": 41337, "demonetisation": 41338, "capecod": 41339, "versatility": 41340, "orice": 41341, "leila": 41342, "womeninscience": 41343, "tua": 41344, "hedges": 41345, "embarrassment": 41346, "alife": 41347, "soars": 41348, "nighter": 41349, "hymn": 41350, "gipp": 41351, "chasu": 41352, "techs": 41353, "niall": 41354, "killa": 41355, "hika": 41356, "camels": 41357, "value": 41358, "¢": 41359, "scoops": 41360, "mahmoud": 41361, "clusive": 41362, "adriana": 41363, "paco": 41364, "ozil": 41365, "unas": 41366, "translations": 41367, "whisperer": 41368, "sbi": 41369, "buxton": 41370, "biotics": 41371, "indiffe": 41372, "kenney": 41373, "klar": 41374, "etching": 41375, "barrabest": 41376, "instability": 41377, "seine": 41378, "votel": 41379, "blogged": 41380, "whiskey": 41381, "myspace": 41382, "tant": 41383, "landia": 41384, "giveback": 41385, "illus": 41386, "awak": 41387, "acab": 41388, "fbloggers": 41389, "cloudcomputing": 41390, "blatant": 41391, "syrians": 41392, "bandra": 41393, "styn": 41394, "anem": 41395, "keted": 41396, "karthik": 41397, "barunsob": 41398, "pinot": 41399, "gubernat": 41400, "gaye": 41401, "artiste": 41402, "ified": 41403, "conventions": 41404, "huan": 41405, "geniuses": 41406, "eeeeee": 41407, "folly": 41408, "somerville": 41409, "pridemonth": 41410, "ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸": 41411, "chemotherapy": 41412, "pauls": 41413, "bakar": 41414, "ìĦ¸ë¸IJ": 41415, "taiwanese": 41416, "follo": 41417, "css": 41418, "reign": 41419, "nnnn": 41420, "flaun": 41421, "catastrophe": 41422, "ities": 41423, "fragments": 41424, "extremists": 41425, "ymoun": 41426, "carmen": 41427, "ezekiel": 41428, "connecting": 41429, "seh": 41430, "manta": 41431, "remodeling": 41432, "weymouth": 41433, "atoms": 41434, "cem": 41435, "newell": 41436, "lumi": 41437, "theopen": 41438, "moc": 41439, "miliband": 41440, "gland": 41441, "zshq": 41442, "maggie": 41443, "maniacs": 41444, "msp": 41445, "ady": 41446, "creams": 41447, "leanne": 41448, "esta": 41449, "pyg": 41450, "affinity": 41451, "prayer": 41452, "dunbar": 41453, "lightroom": 41454, "acadi": 41455, "wynonna": 41456, "romantic": 41457, "statedept": 41458, "sickle": 41459, "whos": 41460, "lamo": 41461, "etour": 41462, "finity": 41463, "shrub": 41464, "sharpen": 41465, "pundit": 41466, "edon": 41467, "afore": 41468, "mars": 41469, "jeffery": 41470, "terps": 41471, "medallist": 41472, "katharine": 41473, "accusing": 41474, "taz": 41475, "royd": 41476, "fromhome": 41477, "confrontation": 41478, "allegh": 41479, "ðŁijīðŁijī": 41480, "refresher": 41481, "ranveer": 41482, "neverland": 41483, "jojo": 41484, "lucrative": 41485, "enam": 41486, "caver": 41487, "paedi": 41488, "manjaro": 41489, "fluids": 41490, "thessal": 41491, "oppressed": 41492, "muss": 41493, "johanna": 41494, "Ø®": 41495, "cng": 41496, "buildthe": 41497, "settles": 41498, "sith": 41499, "fuego": 41500, "clamp": 41501, "arag": 41502, "payer": 41503, "tedx": 41504, "mandy": 41505, "interstellar": 41506, "frc": 41507, "chand": 41508, "bcc": 41509, "molo": 41510, "lentil": 41511, "johansson": 41512, "grimsby": 41513, "naturelovers": 41514, "ðŁļ¨ðŁļ¨ðŁļ¨": 41515, "shinde": 41516, "xin": 41517, "internationaldayof": 41518, "transitional": 41519, "sata": 41520, "caddy": 41521, "wod": 41522, "ifu": 41523, "hays": 41524, "hollyo": 41525, "jang": 41526, "irc": 41527, "coim": 41528, "gradable": 41529, "\"\"": 41530, "ðŁį´": 41531, "া": 41532, "ael": 41533, "nyo": 41534, "westlake": 41535, "timeout": 41536, "sofi": 41537, "phenomena": 41538, "cultivation": 41539, "agno": 41540, "unarmed": 41541, "sot": 41542, "conj": 41543, "geno": 41544, "royalnavy": 41545, "nutrition": 41546, "fairmont": 41547, "tirelessly": 41548, "sng": 41549, "rety": 41550, "mica": 41551, "lucent": 41552, "sloane": 41553, "drool": 41554, "rizal": 41555, "odell": 41556, "criticized": 41557, ".'\"": 41558, "laze": 41559, "deserted": 41560, "coder": 41561, "pras": 41562, "lillian": 41563, "itinerary": 41564, "davy": 41565, "anap": 41566, "whipping": 41567, "hoboken": 41568, "kareena": 41569, "羣": 41570, "vius": 41571, "tern": 41572, "nantucket": 41573, "misunderstood": 41574, "bulaga": 41575, "stant": 41576, "chinook": 41577, "zam": 41578, "relies": 41579, "dss": 41580, "edmond": 41581, "sketchy": 41582, "mell": 41583, "fex": 41584, "rector": 41585, "distill": 41586, "daydream": 41587, "winemaker": 41588, "ripley": 41589, "billionaires": 41590, "helene": 41591, "atif": 41592, "culprit": 41593, "bertrand": 41594, "wouldnt": 41595, "mapped": 41596, "vak": 41597, "gladly": 41598, "parliament": 41599, "kidlitart": 41600, "wareness": 41601, "goliath": 41602, "âĨĵ": 41603, "viewpoint": 41604, "tatted": 41605, "fuls": 41606, "dorsey": 41607, "anglers": 41608, "lids": 41609, "kiya": 41610, "bowles": 41611, "beh": 41612, "bite": 41613, "compatibility": 41614, "ancestral": 41615, "prox": 41616, "behaved": 41617, "gubernatorial": 41618, "chfield": 41619, "saban": 41620, "zh": 41621, "teeny": 41622, "shibuya": 41623, "holliday": 41624, "pancy": 41625, "âĿĦï¸ıâĿĦï¸ı": 41626, "seungri": 41627, "?,": 41628, "ðŁĩ¦ðŁĩ·": 41629, "imitation": 41630, "impactful": 41631, "anyi": 41632, "genevie": 41633, "años": 41634, "bateman": 41635, "glider": 41636, "afar": 41637, "rasheed": 41638, "effortless": 41639, "shwar": 41640, "dachsh": 41641, "erun": 41642, "atos": 41643, "kini": 41644, "chd": 41645, "khaki": 41646, "klin": 41647, "felicidades": 41648, "belo": 41649, "asl": 41650, "toppers": 41651, "finley": 41652, "stacey": 41653, "rigorous": 41654, "karting": 41655, "leppard": 41656, "carmichael": 41657, "beret": 41658, "cse": 41659, "akhi": 41660, "meringue": 41661, "aban": 41662, "hake": 41663, "geri": 41664, "erjee": 41665, "resto": 41666, "commanders": 41667, "prit": 41668, "flor": 41669, "adven": 41670, "extermin": 41671, "remainder": 41672, "åIJ": 41673, "esg": 41674, "martino": 41675, "lullaby": 41676, "|@": 41677, "mign": 41678, "instore": 41679, "bigbang": 41680, "cordi": 41681, "cauley": 41682, "antebellum": 41683, "dgate": 41684, "crock": 41685, "spandex": 41686, "scaffolding": 41687, "oreos": 41688, "ê°ĵìĦ¸ë¸IJ": 41689, "pomona": 41690, "mauro": 41691, "universi": 41692, "remi": 41693, "afootball": 41694, "tant": 41695, "smalls": 41696, "neh": 41697, "worldo": 41698, "tropical": 41699, "morph": 41700, "javelin": 41701, "glar": 41702, "arquitec": 41703, "reminiscent": 41704, "tubs": 41705, "spidey": 41706, "makeu": 41707, "sylla": 41708, "progressives": 41709, "blot": 41710, "shorten": 41711, "keepin": 41712, "chak": 41713, "angst": 41714, "superfood": 41715, "decadent": 41716, "stony": 41717, "neurological": 41718, "arboretum": 41719, "annak": 41720, "fema": 41721, "percu": 41722, "disrespectful": 41723, "smallbiz": 41724, "lox": 41725, "coom": 41726, "csc": 41727, "bsbi": 41728, "prevalence": 41729, "himss": 41730, "espan": 41731, "moga": 41732, "frampton": 41733, "skymap": 41734, "masse": 41735, "leviathan": 41736, "().": 41737, "nocturnal": 41738, "carameli": 41739, "angor": 41740, "amnesia": 41741, "outsiders": 41742, "shealth": 41743, "rhino": 41744, "antag": 41745, "agio": 41746, "ðŁĴ°ðŁĴ°": 41747, "takeme": 41748, "kabaddi": 41749, "csi": 41750, "msh": 41751, "cochrane": 41752, "thessaloni": 41753, "sila": 41754, "haus": 41755, "dusting": 41756, "obese": 41757, "macklemore": 41758, "manish": 41759, "lenin": 41760, "mdc": 41761, "grown": 41762, "sheffield": 41763, "srs": 41764, "kele": 41765, "carson": 41766, "chum": 41767, "dahlia": 41768, "cantore": 41769, "oppo": 41770, "howling": 41771, "cybercrime": 41772, "surrealism": 41773, "scran": 41774, "faiz": 41775, "thren": 41776, "racists": 41777, "rout": 41778, "pknot": 41779, "semana": 41780, "sini": 41781, "mccull": 41782, "machi": 41783, "alfonso": 41784, "yb": 41785, "sardar": 41786, "kendrick": 41787, "deng": 41788, "recipro": 41789, "onf": 41790, "doomsday": 41791, "bribery": 41792, "customiz": 41793, "artis": 41794, "cpi": 41795, "ðŁĻĪðŁĻĪ": 41796, "slava": 41797, "lette": 41798, "ens": 41799, "âĿ¤ï¸ıðŁĺĺ": 41800, "crayon": 41801, "adan": 41802, "trc": 41803, "migrate": 41804, "simpson": 41805, "rowers": 41806, "kingsley": 41807, "farmersmarket": 41808, "sheehan": 41809, "nephe": 41810, "bornon": 41811, "carton": 41812, "mickey": 41813, "allure": 41814, "ulu": 41815, "slipknot": 41816, "hebdo": 41817, "guido": 41818, "dogcelebration": 41819, "onlinemarketing": 41820, "accelerating": 41821, ")..": 41822, "originated": 41823, "macaroni": 41824, "edtech": 41825, "outfield": 41826, "mitz": 41827, "discus": 41828, "advertiser": 41829, "manor": 41830, "hashi": 41831, "descrip": 41832, "capita": 41833, "fulbright": 41834, "receptor": 41835, "conn": 41836, "coney": 41837, "spionage": 41838, "rattle": 41839, "prest": 41840, "uli": 41841, "blogpost": 41842, "ackeray": 41843, ")âĢ¦": 41844, "redvelvet": 41845, "matth": 41846, "inspiring": 41847, "bsd": 41848, "kerri": 41849, "pocon": 41850, "millar": 41851, "repur": 41852, "accenture": 41853, "ä¹": 41854, "rambo": 41855, "ragnarok": 41856, "deleting": 41857, "britishmuseum": 41858, "patory": 41859, "leipzig": 41860, "florian": 41861, "scifi": 41862, "iners": 41863, "brate": 41864, "yoy": 41865, "melissa": 41866, "aber": 41867, "masa": 41868, "pote": 41869, "mosquitoes": 41870, "transplant": 41871, "rpa": 41872, ";))": 41873, "bastille": 41874, "ylan": 41875, "joyeux": 41876, "melodic": 41877, "captions": 41878, "atrist": 41879, "rochdale": 41880, "gotti": 41881, "pewdie": 41882, "cutiesaturday": 41883, "whois": 41884, "aquaculture": 41885, "tiva": 41886, "spel": 41887, "hess": 41888, "haji": 41889, "freddie": 41890, "coper": 41891, "brando": 41892, "vk": 41893, "photobook": 41894, "*,": 41895, "mydayin": 41896, "michaela": 41897, "brunei": 41898, "srini": 41899, "inte": 41900, "ı": 41901, "deol": 41902, "dfc": 41903, "separately": 41904, "bund": 41905, "vests": 41906, "toc": 41907, "meck": 41908, "reinforced": 41909, "constraints": 41910, "carroll": 41911, "sqft": 41912, "rever": 41913, "camper": 41914, "birdman": 41915, "inaction": 41916, "generators": 41917, "triumphant": 41918, "pests": 41919, "ovo": 41920, "gypt": 41921, "alamo": 41922, "scaled": 41923, "sureshpp": 41924, "sdn": 41925, "ismo": 41926, "gios": 41927, ")@": 41928, "justiceleague": 41929, "restaurant": 41930, "gabi": 41931, "dengue": 41932, "nextgen": 41933, "exempli": 41934, "apex": 41935, "inspirational": 41936, "downside": 41937, "kidz": 41938, "upl": 41939, "etna": 41940, "alvaro": 41941, "feldman": 41942, "barnet": 41943, "mha": 41944, "esch": 41945, "blooded": 41946, ">>>>>>>>": 41947, "kani": 41948, "hofficial": 41949, "casablanca": 41950, "birds": 41951, "tyga": 41952, "swamp": 41953, "oday": 41954, "newcastle": 41955, "nbap": 41956, "cision": 41957, "chools": 41958, "aflo": 41959, "nep": 41960, "monton": 41961, "akb": 41962, "supermodel": 41963, "downtime": 41964, "thos": 41965, "scwx": 41966, "snoopy": 41967, "aggreg": 41968, "yoke": 41969, "norcal": 41970, "wett": 41971, "prolonged": 41972, "metast": 41973, "beater": 41974, "fta": 41975, "tlap": 41976, "disgusted": 41977, "yh": 41978, "voiceover": 41979, "itchy": 41980, "ipc": 41981, "ðŁİ¾": 41982, "pheasant": 41983, "straits": 41984, "rampant": 41985, "jg": 41986, "fertil": 41987, "assures": 41988, "fortunes": 41989, "salinas": 41990, "lizards": 41991, "kettle": 41992, "ibs": 41993, "cynthi": 41994, "heg": 41995, "mccr": 41996, "socceroos": 41997, "happenings": 41998, "corden": 41999, "ðŁĺĤðŁijĮ": 42000, "tches": 42001, "egret": 42002, "wolverines": 42003, "congratulated": 42004, "hogg": 42005, "bottling": 42006, "wri": 42007, "ferri": 42008, "bosch": 42009, "afire": 42010, "ogden": 42011, "sjo": 42012, "jdm": 42013, "svt": 42014, "contex": 42015, "tollywood": 42016, "mink": 42017, "mese": 42018, "supersonic": 42019, "opoulos": 42020, "å¸": 42021, "âĶģ": 42022, "knuckle": 42023, "guise": 42024, "gami": 42025, "chucky": 42026, "zinger": 42027, "radial": 42028, "complained": 42029, "boda": 42030, "fetal": 42031, "disciplines": 42032, "corro": 42033, "ðŁĩ®ðŁĩ¹": 42034, "opted": 42035, "filtration": 42036, "adnan": 42037, "emcee": 42038, "mistre": 42039, "insomni": 42040, "fergus": 42041, "trajec": 42042, "ondon": 42043, "medtech": 42044, "tangerine": 42045, "madras": 42046, "grue": 42047, "cabs": 42048, "zhu": 42049, "sureshpprabhu": 42050, "insulated": 42051, "dayswild": 42052, "ppm": 42053, "bandai": 42054, "vday": 42055, "sff": 42056, "squid": 42057, "lothing": 42058, "notdead": 42059, "expressive": 42060, "cull": 42061, "alastair": 42062, "xu": 42063, "upfront": 42064, "fishers": 42065, "enes": 42066, "umd": 42067, "dismissal": 42068, "stier": 42069, "sels": 42070, "lust": 42071, "reactive": 42072, "protester": 42073, "eyelashes": 42074, "alim": 42075, "goode": 42076, "greeng": 42077, "dair": 42078, "compen": 42079, "anushka": 42080, "prototyping": 42081, "mapu": 42082, "bearings": 42083, "ðŁIJŁ": 42084, "forme": 42085, "bsbibotany": 42086, "timothy": 42087, "outskirts": 42088, "ambed": 42089, "aretha": 42090, "wendell": 42091, "streaks": 42092, "nim": 42093, "kpk": 42094, "snee": 42095, "fitter": 42096, "quota": 42097, "pate": 42098, "winning": 42099, "ðŁįŃ": 42100, "shopping": 42101, "mainst": 42102, "culver": 42103, "stevie": 42104, "mcfadden": 42105, "counterparts": 42106, "grenfell": 42107, "folsom": 42108, "dorset": 42109, "techcrunch": 42110, "â¬ħï¸ı": 42111, "tiptuesday": 42112, "usl": 42113, "trex": 42114, "georgie": 42115, "ranveerofficial": 42116, "licks": 42117, "sewn": 42118, "kf": 42119, "'âĢ¦": 42120, "japs": 42121, "pate": 42122, "orthop": 42123, "festa": 42124, "stras": 42125, "montal": 42126, "hammersmith": 42127, "foremost": 42128, "widows": 42129, "madre": 42130, "itez": 42131, "mitochondri": 42132, "ligans": 42133, "zona": 42134, "caribou": 42135, "mss": 42136, "andrei": 42137, "weatherchannel": 42138, "ghc": 42139, ":...": 42140, "taft": 42141, "aweather": 42142, "alisation": 42143, "brutal": 42144, "blissful": 42145, "nikola": 42146, "malicious": 42147, "qm": 42148, "mpgvip": 42149, "brodie": 42150, "blitz": 42151, "applaud": 42152, "dribb": 42153, "vague": 42154, "doggo": 42155, "translating": 42156, "interpreted": 42157, "hatched": 42158, "getyour": 42159, "beneficiaries": 42160, "sparring": 42161, "caesars": 42162, "awilliams": 42163, "lahat": 42164, "broke": 42165, "timp": 42166, "virtues": 42167, "relying": 42168, "pietro": 42169, "ktn": 42170, "icists": 42171, "pablo": 42172, "loui": 42173, "aag": 42174, "pnpp": 42175, "chast": 42176, "pulses": 42177, "finish": 42178, "usairforce": 42179, "typewriter": 42180, "thompson": 42181, "dogs": 42182, "utto": 42183, "ãģį": 42184, "sandal": 42185, "newly": 42186, "doge": 42187, "zw": 42188, "wankers": 42189, "negr": 42190, "mucha": 42191, "determines": 42192, "blackfish": 42193, "skunk": 42194, "mups": 42195, "instrument": 42196, "phyto": 42197, "daystogo": 42198, "skinned": 42199, "haider": 42200, "conten": 42201, "ðŁIJ¾ðŁIJ¾": 42202, "weiler": 42203, "undoubtedly": 42204, "chairing": 42205, "wallis": 42206, "shard": 42207, "zindabad": 42208, "adult": 42209, "absorption": 42210, "presto": 42211, "deploying": 42212, "drummond": 42213, "battlefront": 42214, "seagulls": 42215, "howdy": 42216, "judaism": 42217, "desde": 42218, "partition": 42219, "âľĿ": 42220, "nology": 42221, "nationalbestfriend": 42222, "lesnar": 42223, "filmfare": 42224, "coasts": 42225, "christensen": 42226, "acan": 42227, "mbu": 42228, "copped": 42229, "rubble": 42230, "swc": 42231, "funnier": 42232, "farther": 42233, "whereas": 42234, "nanotechnology": 42235, "withstand": 42236, "pillow": 42237, "bowers": 42238, "tope": 42239, "itly": 42240, "confit": 42241, "makar": 42242, "comforts": 42243, "bosh": 42244, "clipper": 42245, "balla": 42246, "stik": 42247, "milb": 42248, "safeguard": 42249, "musique": 42250, "easport": 42251, "yaz": 42252, "padded": 42253, "bader": 42254, "foreign": 42255, "chopin": 42256, "archive": 42257, "oka": 42258, "transporting": 42259, "tmltalk": 42260, "ajit": 42261, "consequence": 42262, "scroo": 42263, "ffo": 42264, "collaborated": 42265, "pugchat": 42266, "yemi": 42267, "javed": 42268, "auburn": 42269, "oof": 42270, "maw": 42271, "saucer": 42272, "mitigate": 42273, "iles": 42274, "evangelist": 42275, "terie": 42276, "recl": 42277, "indictment": 42278, "cata": 42279, "brightness": 42280, "maythe": 42281, "whimsical": 42282, "unlv": 42283, "keyword": 42284, "cumin": 42285, "medway": 42286, "westworld": 42287, "traw": 42288, "imposing": 42289, "formity": 42290, "coulter": 42291, "abz": 42292, "nypd": 42293, "grassi": 42294, "kelsey": 42295, "qldpol": 42296, "clockwork": 42297, "fdr": 42298, "dianne": 42299, "âĺij": 42300, "adh": 42301, "pann": 42302, "bravely": 42303, "aege": 42304, "unlawful": 42305, "verdi": 42306, "pocalypse": 42307, "pharo": 42308, "karla": 42309, "resonance": 42310, "mastiff": 42311, "ladak": 42312, "buu": 42313, "mailed": 42314, "hii": 42315, "crawley": 42316, "torrent": 42317, "machado": 42318, "libyan": 42319, "effortlessly": 42320, "falsely": 42321, "qvist": 42322, "keef": 42323, "crafthour": 42324, "cherished": 42325, "valkyrie": 42326, "sari": 42327, "kalamaz": 42328, "behe": 42329, "ðŁĮĻ": 42330, "thim": 42331, "roddy": 42332, "coltrane": 42333, "butchers": 42334, "achim": 42335, "wkend": 42336, "awkward": 42337, "cabrera": 42338, ":))))": 42339, "franc": 42340, "declan": 42341, "condos": 42342, "aja": 42343, "pandoramusic": 42344, "charter": 42345, "phill": 42346, "montrose": 42347, "hatchback": 42348, "handicapp": 42349, "greaves": 42350, "eucalyptus": 42351, "utmost": 42352, "tson": 42353, "burton": 42354, "midwives": 42355, "incur": 42356, "ðŁĺį#": 42357, "mood": 42358, "compressed": 42359, "toma": 42360, "mustang": 42361, "mog": 42362, "asana": 42363, "testic": 42364, "shotel": 42365, "insol": 42366, "corsair": 42367, "nhq": 42368, "benny": 42369, "smma": 42370, "kapur": 42371, "incon": 42372, "jonas": 42373, "energies": 42374, "donal": 42375, "asad": 42376, "sez": 42377, "npa": 42378, "archived": 42379, "stimulate": 42380, "dop": 42381, "hyd": 42382, "grieving": 42383, "ãĥĪ": 42384, "rona": 42385, "whyte": 42386, "treehouse": 42387, "ssell": 42388, "sandro": 42389, "kobo": 42390, "thermost": 42391, "seclu": 42392, "hiya": 42393, "geez": 42394, "mamas": 42395, "priscilla": 42396, "flavoured": 42397, "fass": 42398, "wold": 42399, "makerspace": 42400, "cosplay": 42401, "ptv": 42402, "happyvalentinesday": 42403, "sequoia": 42404, "lovecraft": 42405, "guan": 42406, "dtm": 42407, "cii": 42408, "yokohama": 42409, "posthum": 42410, "req": 42411, "ðŁĶµâļªï¸ı": 42412, "galatasar": 42413, "dolby": 42414, "hamptons": 42415, "disturbance": 42416, "stonehenge": 42417, "okc": 42418, "disrupting": 42419, "monthsary": 42420, "jungle": 42421, "headlights": 42422, "dustin": 42423, "microsof": 42424, "happymothersday": 42425, "koko": 42426, "grazi": 42427, "testo": 42428, "naidu": 42429, "malay": 42430, "arial": 42431, "rumb": 42432, "aboo": 42433, "harman": 42434, "trape": 42435, "spoils": 42436, "jeho": 42437, "godly": 42438, "lockscreen": 42439, "zun": 42440, "pious": 42441, "magento": 42442, "lenders": 42443, "probable": 42444, "corporal": 42445, "mour": 42446, "awal": 42447, "sua": 42448, "callme": 42449, "tonne": 42450, "govin": 42451, "devastation": 42452, "xj": 42453, "gearbox": 42454, "warlock": 42455, "perme": 42456, "itate": 42457, "gazaunderattack": 42458, "duval": 42459, "parasite": 42460, "clemente": 42461, "leth": 42462, "iva": 42463, "frozen": 42464, "tholes": 42465, "tobin": 42466, "cairn": 42467, "sill": 42468, "luckiest": 42469, "converts": 42470, "stale": 42471, "pancra": 42472, "europale": 42473, "wisdom": 42474, "schur": 42475, "ì¶": 42476, "vertigo": 42477, "bij": 42478, "ubc": 42479, "nure": 42480, "righteousness": 42481, "mtc": 42482, "factory": 42483, "verst": 42484, "reversed": 42485, "huri": 42486, "heechul": 42487, "faber": 42488, "arr": 42489, "ulous": 42490, "venom": 42491, "phat": 42492, "greenery": 42493, "brady": 42494, "æ": 42495, ":((": 42496, "nevergiveup": 42497, "disha": 42498, "mota": 42499, "healthcare": 42500, "dunham": 42501, "dexpo": 42502, "denzel": 42503, "bbins": 42504, "fics": 42505, "wham": 42506, "mcg": 42507, "elian": 42508, "wata": 42509, "stralia": 42510, "tellu": 42511, "pesky": 42512, "spinoff": 42513, "armoured": 42514, "reacted": 42515, "dofficial": 42516, "tedu": 42517, "sagar": 42518, "morally": 42519, "paralleled": 42520, "fios": 42521, "downer": 42522, "daugh": 42523, "redo": 42524, "worldcup": 42525, "tariq": 42526, "barne": 42527, "glaciers": 42528, "occult": 42529, "barbarian": 42530, "hermosa": 42531, "!!!)": 42532, "yur": 42533, "internation": 42534, "pss": 42535, "situ": 42536, "pint": 42537, "americanair": 42538, "swam": 42539, "doppler": 42540, "ðŁĴĻðŁĴľ": 42541, "cincodemayo": 42542, "levan": 42543, "hellenic": 42544, "mcne": 42545, "judi": 42546, "yuh": 42547, "stx": 42548, "quare": 42549, "ðŁĺĤ.": 42550, "stig": 42551, "gels": 42552, "motley": 42553, "hardwork": 42554, "eurozone": 42555, "ead": 42556, "ç¥Ń": 42557, "seabir": 42558, "cius": 42559, "laid": 42560, "alpaca": 42561, "presumably": 42562, "pewdiepie": 42563, "booted": 42564, "amari": 42565, "tamine": 42566, "solace": 42567, "barrow": 42568, "academies": 42569, "xian": 42570, "omination": 42571, "dungeons": 42572, "bma": 42573, "deity": 42574, "aik": 42575, "stabil": 42576, "hira": 42577, "affectionate": 42578, "vingne": 42579, "newport": 42580, "ãħĭãħĭ": 42581, "thirds": 42582, "retains": 42583, "aromatherapy": 42584, "skier": 42585, "nima": 42586, "dope": 42587, "cringe": 42588, "condomin": 42589, "toor": 42590, "animator": 42591, "saraj": 42592, "seascape": 42593, "minimalism": 42594, "lakeshore": 42595, "callaway": 42596, "bergman": 42597, "à¤Ĺ": 42598, "whispering": 42599, "stupid": 42600, "rightful": 42601, "requis": 42602, "irn": 42603, "seva": 42604, "utpol": 42605, "tuberculo": 42606, "squish": 42607, "debut": 42608, "governmental": 42609, "christine": 42610, "allman": 42611, "weapon": 42612, "sito": 42613, "buri": 42614, "lolita": 42615, "leafy": 42616, "fuch": 42617, "tinted": 42618, "mcken": 42619, "ahahaha": 42620, "ðŁĩµðŁĩ¹": 42621, "repeal": 42622, "negan": 42623, "ðŁķĬ": 42624, "tailgating": 42625, "gameinsight": 42626, "ðŁıŁï¸ı": 42627, "yakuza": 42628, "zt": 42629, "tiring": 42630, "proposing": 42631, "bowlers": 42632, "traitors": 42633, "akshi": 42634, "clergy": 42635, "cito": 42636, "upsets": 42637, "tuscal": 42638, "symphonic": 42639, "silently": 42640, "shuff": 42641, "blackwell": 42642, "ðŁĺĤ)": 42643, "kobe": 42644, "roberto": 42645, "ridg": 42646, "dcu": 42647, "merino": 42648, "ftp": 42649, "eastside": 42650, ".~": 42651, "nbl": 42652, "mnleg": 42653, "tsfor": 42654, "fraudul": 42655, "capping": 42656, "inmy": 42657, "gymnast": 42658, "stones": 42659, "ssin": 42660, "tweaks": 42661, "shaggy": 42662, "oakland": 42663, "demsin": 42664, "sangria": 42665, "mmva": 42666, "hennessy": 42667, "downton": 42668, "rightly": 42669, "init": 42670, "agave": 42671, "oblast": 42672, "northeast": 42673, "friendship": 42674, "dala": 42675, "trophy": 42676, "ðŁij½": 42677, "magin": 42678, "margaritas": 42679, "ê·": 42680, "wwfc": 42681, "fash": 42682, "dike": 42683, "cud": 42684, "chart": 42685, "ðŁij®": 42686, "refugees": 42687, "joplin": 42688, "ncs": 42689, "impy": 42690, "firmware": 42691, "pascu": 42692, "flamin": 42693, "healthtech": 42694, "bellletstalk": 42695, "waka": 42696, "olls": 42697, "lago": 42698, "cowan": 42699, "bombardier": 42700, "shome": 42701, "ðŁĻħ": 42702, "mcmaster": 42703, "nave": 42704, "wells": 42705, "uta": 42706, "tellers": 42707, "misfits": 42708, "kapil": 42709, "faceoff": 42710, "affirm": 42711, "apro": 42712, "whitepaper": 42713, "superyacht": 42714, "specimens": 42715, "allocated": 42716, "...,": 42717, "-__": 42718, "kaw": 42719, "dachshund": 42720, "djoker": 42721, "swork": 42722, "quiere": 42723, "orum": 42724, "ðŁIJł": 42725, "somm": 42726, "cmt": 42727, "inghour": 42728, "skinny": 42729, "lgbti": 42730, "giggles": 42731, "breakaway": 42732, "researched": 42733, "parity": 42734, "myal": 42735, "msl": 42736, "retained": 42737, "sivity": 42738, "makeinindia": 42739, "solves": 42740, "defamation": 42741, "waltham": 42742, "sriracha": 42743, "roadway": 42744, "conceptu": 42745, "alin": 42746, "iwant": 42747, "åĪ": 42748, "delft": 42749, "tenderloin": 42750, "gains": 42751, "faults": 42752, "swire": 42753, "stellen": 42754, "pollo": 42755, "dyne": 42756, "bornonthisday": 42757, "asdfghj": 42758, "sql": 42759, "salim": 42760, "advises": 42761, "voip": 42762, "ìĹijìĨ": 42763, "untouched": 42764, "sheil": 42765, "ontario": 42766, "uphill": 42767, "sobre": 42768, "deshi": 42769, "novella": 42770, "dutton": 42771, "crawfish": 42772, "اÙĨ": 42773, "maa": 42774, "twine": 42775, "kalin": 42776, "ðŁĩµðŁĩŃ": 42777, "yess": 42778, "brooks": 42779, "hoosiers": 42780, "tonka": 42781, "umbrellas": 42782, "ayers": 42783, "ateam": 42784, "acquiring": 42785, "suction": 42786, "än": 42787, "wies": 42788, "tarians": 42789, "socio": 42790, "mattb": 42791, "shepherds": 42792, "oso": 42793, "charitytuesday": 42794, "slogans": 42795, "ninjas": 42796, "albat": 42797, "byte": 42798, "bashir": 42799, "trampoline": 42800, "mydayinla": 42801, "ija": 42802, "basel": 42803, "rory": 42804, "goldie": 42805, "firec": 42806, "unnoticed": 42807, "peculiar": 42808, "scha": 42809, "kerson": 42810, "mourns": 42811, "liquidity": 42812, "quipment": 42813, "hibs": 42814, "ars": 42815, "aeronau": 42816, "slideshow": 42817, "slabs": 42818, "deliciousness": 42819, "skitchen": 42820, "htafc": 42821, "fullerton": 42822, "creighton": 42823, "aerob": 42824, "procrastination": 42825, "azores": 42826, "whitehall": 42827, "ussoccer": 42828, "mediation": 42829, "djokernole": 42830, "andme": 42831, "umen": 42832, "noxious": 42833, "joss": 42834, "ilife": 42835, "annivers": 42836, "sudanese": 42837, "etres": 42838, "undermine": 42839, "wholefoods": 42840, "disobe": 42841, "kori": 42842, "adele": 42843, "eliz": 42844, "canti": 42845, "alon": 42846, "gymnasium": 42847, "sarkodie": 42848, "meteorologist": 42849, "ylde": 42850, "steen": 42851, "stampcollecting": 42852, "nasal": 42853, "lott": 42854, "franks": 42855, "exol": 42856, "acki": 42857, "goodyear": 42858, "animalrights": 42859, "yles": 42860, "violets": 42861, "mmes": 42862, "sthel": 42863, "rapping": 42864, "tuscan": 42865, "waiver": 42866, "turner": 42867, "eatlocal": 42868, "northeasthour": 42869, "animations": 42870, "tommorow": 42871, "tsh": 42872, "ffame": 42873, "brae": 42874, "petron": 42875, "glamour": 42876, "bryn": 42877, "dcs": 42878, "bales": 42879, "ðŁĶ¶": 42880, "brov": 42881, "brev": 42882, "bons": 42883, "physique": 42884, "carne": 42885, "xe": 42886, "elixir": 42887, "volved": 42888, "loma": 42889, "ìľł": 42890, "æĺ": 42891, "vanu": 42892, "rigs": 42893, "balance": 42894, "vares": 42895, "bonita": 42896, "sprinkle": 42897, "perfecto": 42898, "dion": 42899, "leak": 42900, "calcutta": 42901, "oba": 42902, "dma": 42903, "cmon": 42904, "tuner": 42905, "pneumonia": 42906, "bogus": 42907, "apologe": 42908, "clough": 42909, "borne": 42910, "))))": 42911, "revived": 42912, "ovarian": 42913, "nerf": 42914, "clegg": 42915, "fanfest": 42916, "chou": 42917, "realizes": 42918, "mcn": 42919, "ligu": 42920, "legalize": 42921, "justsaying": 42922, "forster": 42923, "bosni": 42924, "khi": 42925, "indom": 42926, "heidel": 42927, "encryp": 42928, "siss": 42929, "eddi": 42930, "marbles": 42931, "brisbane": 42932, "ying": 42933, "prepaid": 42934, "walsall": 42935, "cooperate": 42936, "orchestr": 42937, "marisa": 42938, "howie": 42939, "chewy": 42940, "brenner": 42941, "andromeda": 42942, "egan": 42943, "stocki": 42944, "cavendish": 42945, "agan": 42946, "bano": 42947, "deir": 42948, "gog": 42949, "blk": 42950, "rethinking": 42951, "chig": 42952, "rheu": 42953, "snip": 42954, "peng": 42955, "seminole": 42956, "mswx": 42957, "annex": 42958, "lynda": 42959, "lewishamilton": 42960, "cumul": 42961, "tbl": 42962, "dolphin": 42963, "aguero": 42964, "............": 42965, "prelude": 42966, "atour": 42967, "granger": 42968, "tooting": 42969, "rotun": 42970, "disar": 42971, "homeitems": 42972, "dares": 42973, "********": 42974, "ðŁijĨ": 42975, "compreh": 42976, "jinx": 42977, "aswell": 42978, "irie": 42979, "circulating": 42980, "ðŁIJ¥": 42981, "overboard": 42982, "cultivate": 42983, "rhett": 42984, "orienteering": 42985, "cak": 42986, "balkans": 42987, "sitt": 42988, "jasmin": 42989, "britneyspears": 42990, "rotor": 42991, "sealing": 42992, "gbc": 42993, "occi": 42994, "fas": 42995, "emancip": 42996, "comer": 42997, "wartime": 42998, "tickle": 42999, "sonny": 43000, "paces": 43001, "logg": 43002, "atrix": 43003, "srp": 43004, "gwin": 43005, "dobbs": 43006, "uzbe": 43007, "thewanted": 43008, "drush": 43009, "extru": 43010, "micky": 43011, "honorees": 43012, "darwin": 43013, "redux": 43014, "mmj": 43015, "rami": 43016, "jalapeño": 43017, "ioc": 43018, "dover": 43019, "juju": 43020, "whitney": 43021, "seng": 43022, "enly": 43023, "auch": 43024, "archipelago": 43025, "vigilant": 43026, "mangal": 43027, "wildest": 43028, "paranoid": 43029, "hali": 43030, "bbly": 43031, "sanctioned": 43032, "realms": 43033, "conco": 43034, "uddin": 43035, "csk": 43036, "playtime": 43037, "libra": 43038, "savag": 43039, "octane": 43040, "rectan": 43041, "return": 43042, "parrish": 43043, "morrha": 43044, "ccp": 43045, "cmu": 43046, "sailed": 43047, "sevent": 43048, "rosie": 43049, "piling": 43050, "hew": 43051, "boarded": 43052, "segments": 43053, "nephro": 43054, "(.": 43055, "crats": 43056, "bakes": 43057, "ðŁį¸": 43058, "backtothe": 43059, "sibling": 43060, "kirkland": 43061, "keo": 43062, "guwa": 43063, "breads": 43064, "ðŁĺľðŁĺľ": 43065, "tq": 43066, "harassed": 43067, "gau": 43068, "wilbur": 43069, "jisoo": 43070, "eper": 43071, "lisam": 43072, "trippin": 43073, "shino": 43074, "rukh": 43075, "beastmode": 43076, "choa": 43077, "instaweather": 43078, "richland": 43079, "gari": 43080, "fez": 43081, "cowboysnation": 43082, "fursuit": 43083, "krun": 43084, "aen": 43085, "sycamore": 43086, "segun": 43087, "entennial": 43088, "dih": 43089, "oax": 43090, "demsinphilly": 43091, "ðŁĻĢ": 43092, "snhl": 43093, "pennies": 43094, "passwords": 43095, "makin": 43096, "tye": 43097, "deng": 43098, "knigh": 43099, "jeeplife": 43100, "helpline": 43101, "afor": 43102, "zzzz": 43103, "steamy": 43104, "picker": 43105, "iterate": 43106, "happeningnow": 43107, "kib": 43108, "bloomberg": 43109, "martyrdom": 43110, "bully": 43111, "assortment": 43112, "ahora": 43113, "zoe": 43114, "noi": 43115, "illustri": 43116, "agarwal": 43117, "psc": 43118, "electronica": 43119, "recruiter": 43120, "gardiner": 43121, "radha": 43122, "nafta": 43123, "dotnet": 43124, "piero": 43125, "georg": 43126, "bels": 43127, "ðŁĺĤðŁĺį": 43128, "tuberculosis": 43129, "runnin": 43130, "moris": 43131, "hauling": 43132, "evoc": 43133, "brethren": 43134, "shair": 43135, "frameworks": 43136, "astu": 43137, "rigid": 43138, "kuma": 43139, "kreme": 43140, "jinnah": 43141, "insurers": 43142, "nyu": 43143, "fere": 43144, "nollywood": 43145, "goodvibes": 43146, "-...": 43147, "toile": 43148, "skril": 43149, "instaweatherpro": 43150, "czech": 43151, "pavel": 43152, "onepiece": 43153, "nikeplus": 43154, "filet": 43155, "cavity": 43156, "ðŁı½âĢįâĻĤï¸ı": 43157, "ðŁİ£": 43158, "drastic": 43159, "dailys": 43160, "siamese": 43161, "rebu": 43162, "osteo": 43163, "lark": 43164, "fre": 43165, "shelling": 43166, "pé": 43167, "gladys": 43168, "ðŁıĢðŁıĢ": 43169, "gustave": 43170, "submerged": 43171, "grandstand": 43172, "attu": 43173, "wont": 43174, "fpv": 43175, "bley": 43176, "joni": 43177, "angames": 43178, "weighted": 43179, "alou": 43180, "श": 43181, "lesbians": 43182, "fj": 43183, "annies": 43184, "aml": 43185, "doria": 43186, "davin": 43187, "beta": 43188, "canc": 43189, "madewithunity": 43190, "haj": 43191, "badlands": 43192, "mul": 43193, "bluec": 43194, "pawn": 43195, "covington": 43196, "neurology": 43197, "httweets": 43198, "dyslexia": 43199, "thelove": 43200, "neat": 43201, "forklift": 43202, "automate": 43203, "uneven": 43204, "montess": 43205, "hein": 43206, "hag": 43207, "relics": 43208, "competitiveness": 43209, "canelo": 43210, "martens": 43211, "bulletproof": 43212, "skittles": 43213, "gya": 43214, "primo": 43215, "americafirst": 43216, "wooo": 43217, "abortions": 43218, "??!!": 43219, "mache": 43220, "lders": 43221, "rlly": 43222, "prelims": 43223, "direct": 43224, "course": 43225, "swain": 43226, "supercell": 43227, "eccentric": 43228, "stingray": 43229, "plets": 43230, "wilcox": 43231, "westin": 43232, "okanagan": 43233, "kiran": 43234, "carbo": 43235, "bombings": 43236, "rarest": 43237, "boh": 43238, "gawd": 43239, "digg": 43240, "moana": 43241, "entirety": 43242, "enclosed": 43243, "dodgeball": 43244, "parton": 43245, "milkyway": 43246, "atr": 43247, "thoroughbred": 43248, "really": 43249, "qantas": 43250, "epiphany": 43251, "inee": 43252, "aerosmith": 43253, "spieth": 43254, "arthro": 43255, "ellini": 43256, "dubu": 43257, "braving": 43258, "âļ½âļ½": 43259, "restructuring": 43260, "illuminate": 43261, "equili": 43262, "mpi": 43263, "ashton": 43264, "ponytail": 43265, "mascots": 43266, "flattering": 43267, "crum": 43268, "asta": 43269, "à®°": 43270, "strangerthings": 43271, "barnab": 43272, "رÙĬ": 43273, "makeshift": 43274, "gotcha": 43275, "willam": 43276, "choirs": 43277, "kilometres": 43278, "ghosh": 43279, "euthan": 43280, "dolly": 43281, "unning": 43282, "thear": 43283, "crewe": 43284, "wsw": 43285, "jace": 43286, "dismiss": 43287, "kean": 43288, "hota": 43289, "khat": 43290, "~>": 43291, "thiru": 43292, "rendez": 43293, "hartman": 43294, "teessi": 43295, "casca": 43296, "zah": 43297, "hydrange": 43298, "fod": 43299, "awp": 43300, "mzansi": 43301, "thicker": 43302, "nagoya": 43303, "neva": 43304, "stique": 43305, "castel": 43306, "damian": 43307, "thereby": 43308, "jiang": 43309, "alek": 43310, "musicislife": 43311, "raq": 43312, "callahan": 43313, "gouache": 43314, "somaliland": 43315, "seanhannity": 43316, "raheem": 43317, "lose": 43318, "elove": 43319, "wharton": 43320, "rectangular": 43321, "illustrating": 43322, "harne": 43323, "autisma": 43324, "scrapped": 43325, "elland": 43326, "decree": 43327, "nagpur": 43328, "kipp": 43329, "sore": 43330, "nmd": 43331, "maas": 43332, "guna": 43333, "gartner": 43334, "belli": 43335, "thenight": 43336, "jeon": 43337, "genderequality": 43338, "giver": 43339, "ael": 43340, "garments": 43341, "neu": 43342, "mardigras": 43343, "marsden": 43344, "rower": 43345, "polluted": 43346, "cameraman": 43347, "vinod": 43348, "beasley": 43349, "croc": 43350, "jiu": 43351, "hollyoaks": 43352, "anesthesia": 43353, "alles": 43354, "steward": 43355, "latimes": 43356, "ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸": 43357, "tician": 43358, "goria": 43359, "comedic": 43360, "ðŁ¤ĶðŁ¤ĶðŁ¤Ķ": 43361, "naive": 43362, "slions": 43363, "łĪ": 43364, "burglar": 43365, "ðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃ": 43366, "yorkshi": 43367, "señ": 43368, "fanboy": 43369, "laurel": 43370, "incidence": 43371, "potomac": 43372, "roberta": 43373, "presiden": 43374, "pryor": 43375, "osbourne": 43376, "wku": 43377, "teme": 43378, "palae": 43379, "ðŁ¥º": 43380, "reboun": 43381, "itude": 43382, "reddish": 43383, "khand": 43384, "colonialism": 43385, "northcarolina": 43386, "ðĿĴ": 43387, "mannequin": 43388, "ladybird": 43389, "tasty": 43390, "knowledgeable": 43391, "gshore": 43392, "ðŁĮĮ": 43393, "ன": 43394, "quaker": 43395, "salzburg": 43396, "medalists": 43397, "chyna": 43398, "bridesmaid": 43399, "maori": 43400, "rop": 43401, "outraged": 43402, "inadequate": 43403, "truckers": 43404, "alana": 43405, "ìĿ¼": 43406, "rix": 43407, "oooooooo": 43408, "commandments": 43409, "lambeth": 43410, "aaj": 43411, "ecofriendly": 43412, "blaz": 43413, "morecambe": 43414, "bouncy": 43415, "roux": 43416, "raided": 43417, "mized": 43418, "shc": 43419, "gawx": 43420, "laboratories": 43421, "rubs": 43422, "restroom": 43423, "consultations": 43424, "cajun": 43425, "virgini": 43426, "soir": 43427, "revue": 43428, "plein": 43429, "wager": 43430, "ç¹": 43431, "wedo": 43432, "growingup": 43433, "!ðŁĺĬ": 43434, "faceted": 43435, "sinners": 43436, "hovering": 43437, "tiene": 43438, "seasoning": 43439, "anja": 43440, "leggo": 43441, "ilis": 43442, "flax": 43443, "devo": 43444, "ashram": 43445, "matisse": 43446, "keri": 43447, "gower": 43448, "botox": 43449, "marshes": 43450, "unhcr": 43451, "tsm": 43452, "optimus": 43453, "duni": 43454, "stuffs": 43455, "sok": 43456, "orderly": 43457, "nbad": 43458, "islamophobia": 43459, "ravioli": 43460, "faber": 43461, "creds": 43462, "wonka": 43463, "infusion": 43464, "overweight": 43465, "dailynews": 43466, "assimil": 43467, "acollege": 43468, "medallion": 43469, "kilimanjaro": 43470, "stiff": 43471, "thames": 43472, "sunken": 43473, "thard": 43474, "mydubai": 43475, "hilariously": 43476, "hannel": 43477, "plumber": 43478, "fairview": 43479, "separating": 43480, "rascal": 43481, "quien": 43482, "necessities": 43483, "confederation": 43484, "llll": 43485, ":]": 43486, "weaknesses": 43487, "bronco": 43488, "raffles": 43489, "elot": 43490, "ãĤ¸ãĥ": 43491, "adventcalendar": 43492, "ðŁİ¹": 43493, "stravel": 43494, "tunic": 43495, "ksu": 43496, "impeach": 43497, "espionage": 43498, "!-": 43499, "diment": 43500, "currant": 43501, "biode": 43502, "commuting": 43503, "byron": 43504, "ðŁĴĵðŁĴĵ": 43505, "shaded": 43506, "truro": 43507, "crayons": 43508, "arne": 43509, "hsc": 43510, "freaked": 43511, "dramati": 43512, "fleek": 43513, "ucd": 43514, "marlborough": 43515, "^-": 43516, "crossings": 43517, "malo": 43518, "blackops": 43519, "binance": 43520, "choked": 43521, "cheney": 43522, "plo": 43523, "gestures": 43524, "valedic": 43525, "ryanair": 43526, "remington": 43527, "vcs": 43528, "mckee": 43529, "ecz": 43530, "begs": 43531, "nailart": 43532, "mayorof": 43533, "happyfathersday": 43534, "wart": 43535, "petitions": 43536, "ningly": 43537, "cleanenergy": 43538, "brox": 43539, "slalom": 43540, "existent": 43541, "abay": 43542, "ugliest": 43543, "tomp": 43544, "stoma": 43545, "selby": 43546, "goalscorer": 43547, "benji": 43548, "overwhelmingly": 43549, "lans": 43550, "semiconductor": 43551, "southkorea": 43552, "rescheduled": 43553, "skyl": 43554, "enlisted": 43555, "dowski": 43556, "sidel": 43557, "rosenberg": 43558, "nasser": 43559, "whitehead": 43560, "prius": 43561, "harare": 43562, "enn": 43563, "ryder": 43564, "íĤ": 43565, "mong": 43566, "clasico": 43567, "transporter": 43568, "potty": 43569, "isme": 43570, "*****": 43571, "vice": 43572, "skit": 43573, "odessa": 43574, "lmp": 43575, "hern": 43576, "racially": 43577, "pinoy": 43578, "paraguay": 43579, "obituary": 43580, "goes": 43581, "bucha": 43582, "sidewalks": 43583, "angular": 43584, "unconstitutional": 43585, "transitioning": 43586, "ibu": 43587, "guys": 43588, "unpacking": 43589, "oooooo": 43590, "blackgirl": 43591, "bergs": 43592, "¯": 43593, "wordoftheday": 43594, "trumptrain": 43595, "thunderbolt": 43596, "msi": 43597, "fascists": 43598, "ब": 43599, "tsk": 43600, "collapses": 43601, "rajesh": 43602, "loveislove": 43603, "migrating": 43604, "setback": 43605, "ðŁĺĬâĿ¤ï¸ı": 43606, "tels": 43607, "safetyfirst": 43608, "narrated": 43609, "jaejoong": 43610, "unanswered": 43611, "liqueur": 43612, "ennes": 43613, "dalgo": 43614, "billings": 43615, "saltwater": 43616, "mermaids": 43617, "longs": 43618, "clapham": 43619, "wearec": 43620, "piccollage": 43621, "nach": 43622, "hace": 43623, "poisoned": 43624, "loth": 43625, "agna": 43626, "adelrey": 43627, "guardia": 43628, "polishing": 43629, "peacekeeping": 43630, "dall": 43631, "pisa": 43632, "lapland": 43633, "processors": 43634, "deandre": 43635, "sobs": 43636, "ponce": 43637, "drains": 43638, "cbe": 43639, "ðŁİ¥:": 43640, "splash": 43641, "meatball": 43642, "fontana": 43643, "worcestershirehour": 43644, "nev": 43645, "brisk": 43646, "bint": 43647, "acr": 43648, "pox": 43649, "cayenne": 43650, "skrillex": 43651, "jfc": 43652, "hahahahahahaha": 43653, "glas": 43654, "engul": 43655, "temporal": 43656, "onized": 43657, "concre": 43658, "compose": 43659, "vibrations": 43660, "planters": 43661, "fert": 43662, "criticalrolefanart": 43663, "tbli": 43664, "schallenge": 43665, "huckabee": 43666, "municipal": 43667, "iambic": 43668, "radios": 43669, "nevis": 43670, "durability": 43671, "mccla": 43672, "horseback": 43673, "institutes": 43674, "fulfill": 43675, "attach": 43676, "ateur": 43677, "akan": 43678, "resisting": 43679, "illumination": 43680, "handle": 43681, "haircare": 43682, "oment": 43683, "macleod": 43684, "kaiser": 43685, "gno": 43686, "beardown": 43687, "lyf": 43688, "glomer": 43689, "distortion": 43690, "zm": 43691, "sank": 43692, "roosters": 43693, "isnow": 43694, "asports": 43695, "agen": 43696, "woken": 43697, "stgeorge": 43698, "romper": 43699, "myle": 43700, "economists": 43701, "ruto": 43702, "twill": 43703, "healthand": 43704, "dito": 43705, "wsl": 43706, "tairp": 43707, "prakash": 43708, "micheal": 43709, "hts": 43710, "wrights": 43711, "katsu": 43712, "fiorentina": 43713, "defenseman": 43714, "ditch": 43715, "varsity": 43716, "texanscheer": 43717, "baham": 43718, "scanned": 43719, "weil": 43720, "seductive": 43721, "ðŁijįðŁı½": 43722, "fue": 43723, "erwin": 43724, "davison": 43725, "terran": 43726, "moods": 43727, "woolf": 43728, "resource": 43729, "@.": 43730, "cush": 43731, "ðŁį°": 43732, "regression": 43733, "curled": 43734, "lazer": 43735, "joanne": 43736, "abbott": 43737, "moz": 43738, "downers": 43739, "mmmmmm": 43740, "valentina": 43741, "khair": 43742, "dreamt": 43743, "crook": 43744, "chek": 43745, "steaming": 43746, "nephews": 43747, "cleric": 43748, "asober": 43749, "indefinitely": 43750, "wye": 43751, "usnews": 43752, "joyce": 43753, "flushing": 43754, "wynonnaearp": 43755, "rondo": 43756, "kiss": 43757, "hotdog": 43758, "barns": 43759, "saxophon": 43760, "farley": 43761, "gasp": 43762, "decreasing": 43763, "alway": 43764, "pex": 43765, "lsd": 43766, "shift": 43767, "poutine": 43768, "razz": 43769, "rescuing": 43770, "niko": 43771, "hoch": 43772, "ccl": 43773, "uaap": 43774, "nts": 43775, "mcar": 43776, "ilwx": 43777, "conquering": 43778, "kettering": 43779, "sturdy": 43780, "delaying": 43781, "stok": 43782, "vanished": 43783, "cathar": 43784, "bingham": 43785, "inv": 43786, "ichiro": 43787, "hemo": 43788, "budgeting": 43789, "[...]": 43790, "bess": 43791, "sebastian": 43792, "slowed": 43793, "ðĿij": 43794, "muslim": 43795, "stuns": 43796, "actonclimate": 43797, "vea": 43798, "seton": 43799, "rosetta": 43800, "ount": 43801, "hardin": 43802, "fluid": 43803, "caw": 43804, "ðŁ¥Ĥ": 43805, "yacht": 43806, "unl": 43807, "sphy": 43808, "provocative": 43809, "oric": 43810, "isback": 43811, "___": 43812, "nicolas": 43813, "gyan": 43814, "loose": 43815, "flin": 43816, "rebate": 43817, ":::": 43818, "!\"@": 43819, "comicon": 43820, "sheff": 43821, "downstream": 43822, "chichester": 43823, "beachlife": 43824, "momlife": 43825, "diabete": 43826, "arra": 43827, "vane": 43828, "oku": 43829, "yeo": 43830, "mango": 43831, "tryout": 43832, "appell": 43833, "heirs": 43834, "arjuna": 43835, "ddu": 43836, "naveen": 43837, "movic": 43838, "socialists": 43839, "sback": 43840, "criterion": 43841, "soyuz": 43842, "kher": 43843, "daz": 43844, "yolanda": 43845, "wineoclock": 43846, "reina": 43847, "onew": 43848, "leonard": 43849, "endez": 43850, "ubs": 43851, "supportlocal": 43852, "facilitated": 43853, "caramelized": 43854, "bpa": 43855, "vuelta": 43856, "mytho": 43857, "mami": 43858, "speare": 43859, "nbaplayoffs": 43860, "fevre": 43861, "nickjonas": 43862, "imprint": 43863, "cso": 43864, "craigslist": 43865, "lasalle": 43866, "gideon": 43867, "hadoop": 43868, "disregard": 43869, "wud": 43870, "tuc": 43871, "magee": 43872, "acoustics": 43873, "taa": 43874, "quie": 43875, "pola": 43876, "crt": 43877, "dwyer": 43878, "dissec": 43879, "capitol": 43880, "mention": 43881, "knoll": 43882, "heigh": 43883, "finders": 43884, "placements": 43885, "lse": 43886, "indira": 43887, "guri": 43888, "madhuridixit": 43889, "kingdoms": 43890, "iambicpent": 43891, "georgina": 43892, "jeky": 43893, "conflicting": 43894, "bayan": 43895, "agatha": 43896, "uphold": 43897, "dron": 43898, "vicar": 43899, "expat": 43900, "peripheral": 43901, "pessi": 43902, "faf": 43903, "ancestor": 43904, "?..": 43905, "widget": 43906, "punc": 43907, "commenced": 43908, "beavs": 43909, "airwaves": 43910, "addis": 43911, "poa": 43912, "desses": 43913, "coden": 43914, "vue": 43915, "rupee": 43916, "karin": 43917, "spock": 43918, "msy": 43919, "ะ": 43920, "prick": 43921, "fillmore": 43922, "tification": 43923, "thingsto": 43924, "sarde": 43925, "emile": 43926, "pereira": 43927, "nad": 43928, "brightening": 43929, "arresting": 43930, "woking": 43931, "uscg": 43932, "spill": 43933, "raspberrypi": 43934, "hugo": 43935, "itec": 43936, "isma": 43937, "cufflinks": 43938, "optimized": 43939, "occ": 43940, "miwx": 43941, "enka": 43942, "elited": 43943, "affordable": 43944, "sakh": 43945, "coronado": 43946, "hoh": 43947, "atul": 43948, "aioli": 43949, "jimcantore": 43950, "accounted": 43951, "vinay": 43952, "hermit": 43953, "grooves": 43954, "ranch": 43955, "rilla": 43956, "wetter": 43957, "outof": 43958, "veterin": 43959, "nikov": 43960, "kian": 43961, "fairbanks": 43962, "ramapho": 43963, "niti": 43964, "kko": 43965, "rusty": 43966, "nestle": 43967, "tvxq": 43968, "shaheer": 43969, "âĿ¤âĿ¤âĿ¤âĿ¤": 43970, "pennant": 43971, "gemstones": 43972, "demdebate": 43973, "ðŁIJĬ": 43974, "autonews": 43975, "supportindiefilm": 43976, "macho": 43977, "vex": 43978, "newsat": 43979, "neti": 43980, "concessions": 43981, "candied": 43982, "yofthe": 43983, "macau": 43984, "dends": 43985, "cricketers": 43986, "saniti": 43987, "mariano": 43988, "ghat": 43989, "artoftheday": 43990, "¡ľ": 43991, "egos": 43992, "genoa": 43993, "chatbots": 43994, "brier": 43995, "allabout": 43996, "monty": 43997, "spied": 43998, "rtr": 43999, "comfort": 44000, "snippets": 44001, "realtime": 44002, "grain": 44003, "examined": 44004, "enlightening": 44005, "ttu": 44006, "godbless": 44007, "releasethe": 44008, "singular": 44009, "kians": 44010, "haka": 44011, "sorren": 44012, "defect": 44013, "marg": 44014, "equities": 44015, "dorian": 44016, "suka": 44017, "perl": 44018, "aishwarya": 44019, "pullover": 44020, "precision": 44021, "fairway": 44022, "neve": 44023, "riveting": 44024, "villanova": 44025, "encom": 44026, "ako": 44027, "passionately": 44028, "europaleague": 44029, "siempre": 44030, "xvi": 44031, "enlightened": 44032, "cfr": 44033, "âĺħâĺħâĺħâĺħ": 44034, "wasteland": 44035, "isf": 44036, "newcomers": 44037, "emergency": 44038, "amphitheatre": 44039, "-.": 44040, "textbooks": 44041, "figurative": 44042, "tremb": 44043, "pesc": 44044, "abhin": 44045, "abbot": 44046, "acacia": 44047, "hards": 44048, "porsche": 44049, "kauai": 44050, "elisa": 44051, "carrick": 44052, "abou": 44053, "ellier": 44054, "bech": 44055, "neutron": 44056, "galapagos": 44057, "ruben": 44058, "innis": 44059, "howto": 44060, "nuns": 44061, "sabine": 44062, "iac": 44063, "clinched": 44064, "notori": 44065, "fives": 44066, "cairngor": 44067, "peri": 44068, "grc": 44069, "ðŁĴ¯ðŁĴ¯": 44070, "malm": 44071, "twelfth": 44072, "diff": 44073, "routines": 44074, "martyn": 44075, "linden": 44076, "synthesizer": 44077, "number": 44078, "gamecube": 44079, "falkirk": 44080, "byzantine": 44081, "queuing": 44082, "grill": 44083, "scalable": 44084, "charred": 44085, "routing": 44086, "herbali": 44087, "grizz": 44088, "ðŁĺŃðŁĺŃðŁĺŃ": 44089, "toll": 44090, "terminals": 44091, "lpc": 44092, "abd": 44093, "warmups": 44094, "removable": 44095, "¯\\": 44096, "vigo": 44097, "papaya": 44098, "neve": 44099, "lovingly": 44100, "jokers": 44101, "ibles": 44102, "ssett": 44103, "potenti": 44104, "pele": 44105, "gigi": 44106, "sadiq": 44107, "legacy": 44108, "sono": 44109, "rupees": 44110, "retarded": 44111, "elee": 44112, "parr": 44113, "fiance": 44114, "eyre": 44115, "sayers": 44116, "pendants": 44117, "maknae": 44118, "albans": 44119, "adapting": 44120, "pff": 44121, "puberty": 44122, "jiu": 44123, "ingrad": 44124, "hypocrite": 44125, "diplomats": 44126, "physical": 44127, "robby": 44128, "bonsai": 44129, "ãģ·": 44130, "fatt": 44131, "catalunya": 44132, "âľĸï¸ı": 44133, "roma": 44134, "moreland": 44135, "soe": 44136, "conversions": 44137, "stlblues": 44138, "sholm": 44139, "grassy": 44140, "prado": 44141, "onu": 44142, "assaulting": 44143, ">_": 44144, "settes": 44145, "disgraceful": 44146, "aphra": 44147, "âļ½ï¸ıâļ½ï¸ı": 44148, "प": 44149, "kiln": 44150, "goaltender": 44151, "sru": 44152, "philanthropist": 44153, "bals": 44154, "thn": 44155, "studen": 44156, "sandoval": 44157, "dogrescue": 44158, "elions": 44159, "assessed": 44160, "largo": 44161, "hectares": 44162, "shrm": 44163, "saif": 44164, "cleavage": 44165, "noches": 44166, "nene": 44167, "fatalities": 44168, "curing": 44169, "cleanser": 44170, "ales": 44171, "pvp": 44172, "southbank": 44173, "pizzeria": 44174, "marshals": 44175, "knife": 44176, "andover": 44177, "tblightning": 44178, "srsly": 44179, "oute": 44180, "digimon": 44181, "timesofindia": 44182, "promethe": 44183, "lebo": 44184, "fsu": 44185, "witz": 44186, "revere": 44187, "manas": 44188, "mamba": 44189, "chica": 44190, "guan": 44191, "exhibitor": 44192, "csrracing": 44193, "dere": 44194, "xxxxx": 44195, "gusta": 44196, "storytime": 44197, "stoney": 44198, "organics": 44199, "andu": 44200, "seam": 44201, "minogue": 44202, "anushkasharma": 44203, "aba": 44204, "ðŁİĻï¸ı": 44205, "ugandan": 44206, "chromatic": 44207, "assn": 44208, "documentaries": 44209, "sht": 44210, "rupaul": 44211, "loyd": 44212, "kats": 44213, "eus": 44214, "itech": 44215, "medusa": 44216, "panty": 44217, "kellogg": 44218, "etto": 44219, "tallade": 44220, "shaa": 44221, "dost": 44222, "pms": 44223, "mariana": 44224, "jester": 44225, "crooks": 44226, "ðŁĶ¬": 44227, "mindanao": 44228, "indhoven": 44229, "ðŁ¤ª": 44230, "lexi": 44231, "tvn": 44232, "janis": 44233, "cote": 44234, "ãģĨ": 44235, "serrano": 44236, "iwm": 44237, "ðŁIJ¬": 44238, "kke": 44239, "distributors": 44240, "capu": 44241, "counterfeit": 44242, "campsite": 44243, "aggie": 44244, "ðŁĺ¼": 44245, "chhattisgarh": 44246, "~@": 44247, "stateu": 44248, "sandi": 44249, "preventable": 44250, "cls": 44251, "canne": 44252, "mmc": 44253, "iver": 44254, "saharan": 44255, "palis": 44256, "nightout": 44257, "dos": 44258, "apia": 44259, "abscbn": 44260, "managerial": 44261, "arose": 44262, "mowx": 44263, "arosa": 44264, "ðŁĮ³": 44265, "underdog": 44266, "remover": 44267, "astronomers": 44268, "lentils": 44269, "suscep": 44270, "smoother": 44271, "pendleton": 44272, "faucet": 44273, "emory": 44274, "dalmati": 44275, "afcb": 44276, "ticus": 44277, "exempt": 44278, "enrol": 44279, "dheim": 44280, "ðŁIJº": 44281, "restriction": 44282, "starfish": 44283, "stow": 44284, "snorkel": 44285, "thunderbirds": 44286, "shead": 44287, "homosexual": 44288, "dyn": 44289, "asli": 44290, "andretti": 44291, "douche": 44292, "domo": 44293, "tarmac": 44294, "slumber": 44295, "pronto": 44296, "firstdayof": 44297, "miniature": 44298, "mariachi": 44299, "argus": 44300, "recommending": 44301, "mobiles": 44302, "ince": 44303, "illustrious": 44304, "orc": 44305, "adverts": 44306, "grits": 44307, "weasel": 44308, "pagoda": 44309, "overpass": 44310, "greys": 44311, "maximus": 44312, "armagh": 44313, "woodland": 44314, "sunni": 44315, "ðŁĴī": 44316, "ëĿ": 44317, "tione": 44318, "socio": 44319, "hos": 44320, "ðŁ¤ĹðŁ¤Ĺ": 44321, "windsor": 44322, "subsequent": 44323, "munchies": 44324, "idh": 44325, "excluding": 44326, "emi": 44327, "cuth": 44328, "zai": 44329, "weekdays": 44330, "lawsuits": 44331, "barnard": 44332, "ت": 44333, "petting": 44334, "netes": 44335, "mulligan": 44336, "pharmacists": 44337, "raquel": 44338, "eton": 44339, "cranston": 44340, "gilded": 44341, "cleary": 44342, "ceph": 44343, "raa": 44344, "pamper": 44345, "lombardi": 44346, "asin": 44347, "sherry": 44348, "prod": 44349, "forte": 44350, "arianism": 44351, "buffalobills": 44352, "æľ¬": 44353, "ðŁĶ¥#": 44354, "uuu": 44355, "justices": 44356, "carina": 44357, "natin": 44358, "maslow": 44359, "drooling": 44360, "cognac": 44361, "camber": 44362, "elong": 44363, "rdr": 44364, "inen": 44365, "convictions": 44366, "amuse": 44367, "trock": 44368, "harmless": 44369, "visitation": 44370, "genomic": 44371, "bland": 44372, "benoit": 44373, "chimp": 44374, "tuscaloosa": 44375, "greasy": 44376, "xpo": 44377, "gilt": 44378, "seq": 44379, "permitted": 44380, "christmaseve": 44381, "books": 44382, "mue": 44383, "oldschool": 44384, "humanright": 44385, "beati": 44386, "ðŁĶĿ": 44387, "shat": 44388, "sculpting": 44389, "hwan": 44390, "fernandes": 44391, "sciutto": 44392, "fuentes": 44393, "endeavors": 44394, "maidstone": 44395, "unparalleled": 44396, "shouted": 44397, "queenof": 44398, "merc": 44399, "bandic": 44400, "veda": 44401, "selangor": 44402, "pile": 44403, "jahan": 44404, "intimidating": 44405, "disappears": 44406, "clich": 44407, "zaha": 44408, "wurst": 44409, "hiv": 44410, "fodils": 44411, "cordless": 44412, "aaaaaa": 44413, "hydra": 44414, "belinda": 44415, "eels": 44416, "buf": 44417, "sustaining": 44418, "rugbyleague": 44419, "noc": 44420, "brigitte": 44421, "(ðŁĵ¸:": 44422, "trombone": 44423, "soothe": 44424, "smog": 44425, "adp": 44426, "stable": 44427, "ingley": 44428, "diagnose": 44429, "msg": 44430, "wess": 44431, "ticketing": 44432, "onee": 44433, "nswpol": 44434, "eup": 44435, "autopsy": 44436, "adityanath": 44437, "sundown": 44438, "riverfront": 44439, "siya": 44440, "pis": 44441, "hierarchy": 44442, "durango": 44443, "dijk": 44444, "renshaw": 44445, "heaps": 44446, "epidemi": 44447, "davidbowie": 44448, "internetof": 44449, "ddi": 44450, "nationality": 44451, "mbar": 44452, "airy": 44453, "winder": 44454, "walia": 44455, "elliott": 44456, "cx": 44457, "bavarian": 44458, "platt": 44459, "antw": 44460, "wiwx": 44461, "softer": 44462, "neha": 44463, "heller": 44464, "thand": 44465, "daniela": 44466, "boast": 44467, "degradation": 44468, "ðŁĴ¦ðŁĴ¦": 44469, "transforming": 44470, "mane": 44471, "avut": 44472, "ðŁĺĪðŁĺĪ": 44473, "voter": 44474, "thee": 44475, "tate": 44476, "puff": 44477, "indoor": 44478, "soproud": 44479, "boyce": 44480, "borisjohnson": 44481, "waitin": 44482, "immunology": 44483, "ðŁıĨðŁıĨðŁıĨ": 44484, "âĿĮ": 44485, "streetfood": 44486, "lizasober": 44487, "cavalier": 44488, "celia": 44489, "needle": 44490, "motoring": 44491, "gato": 44492, ",)": 44493, "rade": 44494, "harvest": 44495, "tms": 44496, "jarpad": 44497, "oney": 44498, "airmen": 44499, "vre": 44500, "impairment": 44501, "abhishek": 44502, "snoop": 44503, "lant": 44504, "famously": 44505, "blou": 44506, "sze": 44507, "gander": 44508, "untouch": 44509, "tuf": 44510, "deejay": 44511, "collateral": 44512, "bind": 44513, "ðŁļ©": 44514, "pinning": 44515, "icn": 44516, "';": 44517, "theeconomist": 44518, "ultram": 44519, "worldwaterday": 44520, "tipoff": 44521, "thei": 44522, "feeders": 44523, "campaign": 44524, "scumb": 44525, "dayweekend": 44526, "yom": 44527, "pedic": 44528, "hough": 44529, "psv": 44530, "plin": 44531, "onde": 44532, "bostonmarathon": 44533, "azzy": 44534, "*_*": 44535, "conley": 44536, "thiago": 44537, "hooo": 44538, "galerie": 44539, "lucid": 44540, "jett": 44541, "glitz": 44542, "finalfantasy": 44543, "achievers": 44544, "yung": 44545, "peregrine": 44546, "ophi": 44547, "dames": 44548, "biomar": 44549, "âĺĢï¸ıâĺĢï¸ı": 44550, "skc": 44551, "lics": 44552, "flank": 44553, "arrahman": 44554, "hoof": 44555, "upholstery": 44556, "tats": 44557, "woz": 44558, "¿": 44559, "snoring": 44560, "raer": 44561, "lju": 44562, "apd": 44563, "plating": 44564, "kanu": 44565, "imation": 44566, "fragrances": 44567, "mra": 44568, "moray": 44569, "mott": 44570, "immuni": 44571, "hearties": 44572, "bhopal": 44573, "timers": 44574, "gata": 44575, "colorway": 44576, "carnation": 44577, "winget": 44578, "sighs": 44579, "sville": 44580, "optimist": 44581, "chateau": 44582, "olympians": 44583, "cio": 44584, "singersongwriter": 44585, "nyo": 44586, "fibers": 44587, "burch": 44588, "agro": 44589, "milne": 44590, "igbo": 44591, "cramer": 44592, "ationals": 44593, "danube": 44594, "padma": 44595, "normani": 44596, "enforced": 44597, "breck": 44598, "boehner": 44599, "arden": 44600, "surrendered": 44601, "prosthetic": 44602, "oma": 44603, "hailed": 44604, "calculations": 44605, "wfa": 44606, "bib": 44607, "fcblive": 44608, "fonda": 44609, "westcoast": 44610, "quests": 44611, "friendly": 44612, "towie": 44613, "fitch": 44614, "balot": 44615, "stardom": 44616, "scratching": 44617, "hosa": 44618, "thika": 44619, "oven": 44620, "stroke": 44621, "outpost": 44622, "pharmaceuticals": 44623, "hikari": 44624, "muy": 44625, "afd": 44626, "fallontonight": 44627, "squat": 44628, "oru": 44629, "drained": 44630, "chocolat": 44631, "민": 44632, "worths": 44633, "rib": 44634, "muj": 44635, "thats": 44636, "residente": 44637, "itel": 44638, "boost": 44639, "migos": 44640, "mulled": 44641, "laa": 44642, "etsyshop": 44643, "donkeys": 44644, "mek": 44645, "ptc": 44646, "flinders": 44647, "ehs": 44648, "rohit": 44649, "muir": 44650, "gad": 44651, "compositions": 44652, "åĨĻ": 44653, "combustion": 44654, "ikh": 44655, "yemeni": 44656, "waved": 44657, "garci": 44658, "akos": 44659, "oods": 44660, "fusion": 44661, "seque": 44662, "slan": 44663, "plur": 44664, "kicchasu": 44665, "shenando": 44666, "sams": 44667, "worlden": 44668, "horowitz": 44669, "withme": 44670, "microbes": 44671, "kki": 44672, "ðŁĴĶðŁĴĶ": 44673, "wsu": 44674, "patchwork": 44675, "freer": 44676, "yaki": 44677, "theart": 44678, "symbolism": 44679, "miler": 44680, "btn": 44681, "mabu": 44682, "sidekick": 44683, "motivates": 44684, "sagitt": 44685, "naturals": 44686, "serviced": 44687, "psori": 44688, "paola": 44689, "quig": 44690, "ibadan": 44691, "giggs": 44692, "ë³": 44693, "scientology": 44694, "sioux": 44695, "salamat": 44696, "dres": 44697, "cadbury": 44698, "dhawan": 44699, "ción": 44700, "_'": 44701, "swapping": 44702, "mariska": 44703, "jamesbond": 44704, "explosives": 44705, "ayles": 44706, "afer": 44707, "sagu": 44708, "censor": 44709, "toma": 44710, "jefferson": 44711, "ringed": 44712, "partist": 44713, "irresponsible": 44714, "aguilar": 44715, "vacay": 44716, "equitable": 44717, "altrincham": 44718, "acur": 44719, "manish": 44720, "germin": 44721, "schooled": 44722, "putter": 44723, "edad": 44724, "naval": 44725, "toasty": 44726, "solareclipse": 44727, "dishu": 44728, "coyne": 44729, "acco": 44730, "muck": 44731, "maran": 44732, "elos": 44733, "lender": 44734, "croix": 44735, "worthless": 44736, "haber": 44737, "gunmen": 44738, "ðŁįĵ": 44739, "zenith": 44740, "tenders": 44741, "hurst": 44742, "holtz": 44743, "italians": 44744, "carlow": 44745, "ucd": 44746, "characteristic": 44747, "bung": 44748, "avl": 44749, "uth": 44750, "sasia": 44751, "rsl": 44752, "redman": 44753, "neighboring": 44754, "greenpeace": 44755, "stips": 44756, "followparty": 44757, "ygk": 44758, "enos": 44759, "omnibus": 44760, "naissance": 44761, "chrissy": 44762, "secure": 44763, "callback": 44764, "jihoon": 44765, "memory": 44766, "blocker": 44767, "lanta": 44768, "daffodils": 44769, "bilt": 44770, "fferty": 44771, "faust": 44772, "iec": 44773, "nipples": 44774, "sog": 44775, "mnd": 44776, "jaguar": 44777, "boldly": 44778, "abpoli": 44779, "proposition": 44780, "gunsense": 44781, "evansville": 44782, "cutters": 44783, "wego": 44784, "doun": 44785, "dox": 44786, "stallions": 44787, "kaj": 44788, "shippers": 44789, "jawa": 44790, "volo": 44791, "leven": 44792, "paprika": 44793, "kovich": 44794, "jordi": 44795, "inductees": 44796, "appalling": 44797, "dialysis": 44798, "alleviate": 44799, "âĢĶâĢĶ": 44800, "pieter": 44801, "midwi": 44802, "qtr": 44803, "juliette": 44804, "intermission": 44805, "hawks": 44806, "actment": 44807, "oneill": 44808, "klin": 44809, "vamps": 44810, "famous": 44811, "could": 44812, "automobi": 44813, "daan": 44814, "westend": 44815, "ellip": 44816, "nhc": 44817, "melanch": 44818, "webseries": 44819, "tongue": 44820, "snatched": 44821, "smyth": 44822, "tangible": 44823, "sli": 44824, "easing": 44825, "barstool": 44826, "overlay": 44827, "affordability": 44828, "tinged": 44829, "teras": 44830, "ayush": 44831, "wannaone": 44832, "rhine": 44833, "dana": 44834, "shana": 44835, "kendal": 44836, "fertile": 44837, "wir": 44838, "repleni": 44839, "larvae": 44840, "isro": 44841, "convos": 44842, "abbrevi": 44843, "ucc": 44844, "hungry": 44845, "burrows": 44846, "ager": 44847, "navi": 44848, "matin": 44849, "duper": 44850, "cern": 44851, "madon": 44852, "ķï¸ı": 44853, "éģ": 44854, "tups": 44855, "hyatt": 44856, "shep": 44857, "fridaynight": 44858, "wiser": 44859, "heidi": 44860, "hatton": 44861, "pgh": 44862, "fountain": 44863, "wristbands": 44864, "ahmadiyya": 44865, "aerial": 44866, "subscribed": 44867, "solos": 44868, "mace": 44869, "slayed": 44870, "forfe": 44871, "dulce": 44872, "christmass": 44873, "arunjaitley": 44874, "violate": 44875, "obstru": 44876, "nieces": 44877, "wvu": 44878, "idyl": 44879, "faze": 44880, "preserves": 44881, "infringe": 44882, "premiers": 44883, "intervals": 44884, "agency": 44885, "(©": 44886, "standalone": 44887, "dimes": 44888, "boer": 44889, "parameters": 44890, "getit": 44891, "ðŁĺĺðŁĺĺðŁĺĺðŁĺĺ": 44892, "tulane": 44893, "forgiven": 44894, "scoll": 44895, "mbps": 44896, "smashbros": 44897, "robbi": 44898, "primavera": 44899, "alist": 44900, "ghostly": 44901, "ayat": 44902, "yeats": 44903, "impressionist": 44904, "earphones": 44905, "caulfield": 44906, "waikiki": 44907, "salute": 44908, "scou": 44909, "muay": 44910, "louisvuitton": 44911, "bakhta": 44912, "adog": 44913, "inventions": 44914, "hurd": 44915, "foreclo": 44916, "streamline": 44917, "thalaivar": 44918, "chsnews": 44919, "willard": 44920, "tsn": 44921, "europarl": 44922, "crusher": 44923, "mysore": 44924, "grower": 44925, "raping": 44926, "patti": 44927, "gden": 44928, "smw": 44929, "mufti": 44930, "kidman": 44931, "abr": 44932, "sounders": 44933, "skeptical": 44934, "ðŁĶİ": 44935, "sundar": 44936, "ime": 44937, "ferg": 44938, "featherweight": 44939, "arlington": 44940, "pasqu": 44941, "agazine": 44942, "wearable": 44943, "natic": 44944, "mcclure": 44945, "intermitt": 44946, "horde": 44947, "sixties": 44948, "carte": 44949, "bhav": 44950, "zeal": 44951, "experiential": 44952, "adorned": 44953, "sommer": 44954, "enote": 44955, "hypothesis": 44956, "stinky": 44957, "proto": 44958, "deadlines": 44959, "vogel": 44960, "musings": 44961, "moncton": 44962, "guter": 44963, "fle": 44964, "acion": 44965, "voiceof": 44966, "tasha": 44967, "inhabitants": 44968, "typeface": 44969, "sba": 44970, "btsx": 44971, "ðŁĶĴ": 44972, "worx": 44973, "uhc": 44974, "joko": 44975, "cellars": 44976, "goro": 44977, "continuum": 44978, "...&": 44979, "weathercee": 44980, "hap": 44981, "srk": 44982, "risers": 44983, "lonelyplanet": 44984, "unnamed": 44985, "coeur": 44986, "ðŁįĮ": 44987, "theworld": 44988, "ilike": 44989, "fasten": 44990, "amigo": 44991, "riba": 44992, "ramaphosa": 44993, "staffers": 44994, "hadley": 44995, "??\"": 44996, "fiore": 44997, "salut": 44998, "huff": 44999, "bezos": 45000, "Ñĭ": 45001, "rader": 45002, "kamala": 45003, "inline": 45004, "fillers": 45005, "umatic": 45006, "allin": 45007, "shatter": 45008, "rein": 45009, "oku": 45010, "chases": 45011, "flagged": 45012, "babymetal": 45013, "waterstones": 45014, "tsb": 45015, "cutout": 45016, "ophel": 45017, "aama": 45018, "rockabilly": 45019, "stolic": 45020, "jetblue": 45021, "ichick": 45022, "downton": 45023, "uzbekistan": 45024, "patna": 45025, "laq": 45026, "grange": 45027, ")_/": 45028, "subsidi": 45029, "scp": 45030, "newscast": 45031, "itsa": 45032, "tweetyour": 45033, "emor": 45034, "archaeologists": 45035, "unification": 45036, "porta": 45037, "qx": 45038, "protectors": 45039, "prohib": 45040, "charisma": 45041, "cartag": 45042, "renfre": 45043, "sculpt": 45044, "guwahati": 45045, "dema": 45046, "boop": 45047, "unfpa": 45048, "dexter": 45049, "layla": 45050, "alleges": 45051, "soups": 45052, "neveragain": 45053, "lys": 45054, "calc": 45055, "baroness": 45056, "visualize": 45057, "gerber": 45058, "absorbed": 45059, "iers": 45060, "ahan": 45061, "fontein": 45062, "detectors": 45063, "verstappen": 45064, "svc": 45065, "formulated": 45066, "acdc": 45067, "lix": 45068, "incompetent": 45069, "bhk": 45070, "lourdes": 45071, "waterhouse": 45072, "snowed": 45073, "appreciative": 45074, "sigma": 45075, "lizasoberano": 45076, "penned": 45077, "paycheck": 45078, "tallinn": 45079, "fancafe": 45080, "parisi": 45081, "avalley": 45082, "vig": 45083, "rufc": 45084, "hardship": 45085, "socute": 45086, "poise": 45087, "ì¹": 45088, "rothschild": 45089, "kly": 45090, "????????": 45091, "lhp": 45092, "ilay": 45093, "fhs": 45094, "amad": 45095, "ideals": 45096, "bradbury": 45097, "balboa": 45098, "nicot": 45099, "kidnap": 45100, "wolve": 45101, "tasmanian": 45102, "opt": 45103, "matthias": 45104, "ãĥ³ãĤ": 45105, "supermarkets": 45106, "mylittlepony": 45107, "melee": 45108, "lister": 45109, "groun": 45110, "fedora": 45111, "kindness": 45112, "enen": 45113, "brahms": 45114, "¯\\_(": 45115, "roswell": 45116, "marlene": 45117, "icu": 45118, "reformation": 45119, "orail": 45120, "hebrides": 45121, "disparities": 45122, "terracotta": 45123, "swallows": 45124, "reid": 45125, "influencing": 45126, "fluor": 45127, "dene": 45128, "tumour": 45129, "blondes": 45130, "thunderbird": 45131, "sheva": 45132, "mogadishu": 45133, "kab": 45134, "creeps": 45135, "iving": 45136, "eneed": 45137, "annoy": 45138, "âĶĢ": 45139, "intrigue": 45140, "enquiry": 45141, "araj": 45142, "tural": 45143, "kubernetes": 45144, "endlessly": 45145, "dividends": 45146, "tora": 45147, "tish": 45148, "commemorates": 45149, "unra": 45150, "trib": 45151, "ponty": 45152, "nem": 45153, "dissent": 45154, "brewingco": 45155, "ðŁĺ½": 45156, "normali": 45157, "biof": 45158, "(...": 45159, "chillen": 45160, "주": 45161, "mellon": 45162, "avis": 45163, "mccormack": 45164, "ingra": 45165, "enriched": 45166, "customerexperience": 45167, "testosterone": 45168, "snug": 45169, "setti": 45170, "geronimo": 45171, "inquirer": 45172, "breaches": 45173, "verything": 45174, "blooming": 45175, "mura": 45176, "dispos": 45177, "bide": 45178, "deva": 45179, "shadesof": 45180, "intrin": 45181, "shev": 45182, "sven": 45183, "nayanthara": 45184, "ganesha": 45185, "cws": 45186, "berta": 45187, "labelled": 45188, "useum": 45189, "nicknamed": 45190, "mahan": 45191, "caruso": 45192, "apur": 45193, "ðŁijĨ": 45194, "wq": 45195, "orphanage": 45196, "discarded": 45197, "magnu": 45198, "lue": 45199, "jeon": 45200, "bridgeport": 45201, "pacing": 45202, "mercury": 45203, "(ðŁĵ¸": 45204, "marxist": 45205, "amphibious": 45206, "transplantation": 45207, "stitching": 45208, "thenburg": 45209, "gradual": 45210, "ãĤĮ": 45211, "roft": 45212, "mails": 45213, "inec": 45214, "guyana": 45215, "doppelg": 45216, "vero": 45217, "rewrite": 45218, "headless": 45219, "harbaugh": 45220, "gateway": 45221, "carsforsale": 45222, "swi": 45223, "stis": 45224, "macht": 45225, "unde": 45226, "surabaya": 45227, "stapleton": 45228, "nurturing": 45229, "milner": 45230, "yao": 45231, "lmaoooo": 45232, "kosh": 45233, "arsenal": 45234, "kame": 45235, "erry": 45236, "arroyo": 45237, "dismisses": 45238, "rubbed": 45239, "rcb": 45240, "lewd": 45241, "dilu": 45242, "andor": 45243, "vide": 45244, "urin": 45245, "intersec": 45246, "haar": 45247, "alb": 45248, "yearswith": 45249, "appleton": 45250, "éal": 45251, "ullivan": 45252, "succu": 45253, "monterrey": 45254, "dmx": 45255, "artemis": 45256, "ronnie": 45257, "farmland": 45258, "sfootball": 45259, "grotto": 45260, "anthi": 45261, "ãĢģ": 45262, "à®Ł": 45263, "vidya": 45264, "jimmyfallon": 45265, "àµį": 45266, "tzer": 45267, "gravitational": 45268, "wthr": 45269, "uhhh": 45270, "ehr": 45271, "tinker": 45272, "tijuana": 45273, "scranton": 45274, "ramcharan": 45275, "barclay": 45276, "revan": 45277, "msi": 45278, "kap": 45279, "wrs": 45280, "wethenorth": 45281, "toral": 45282, "satu": 45283, "grom": 45284, "facep": 45285, "erickson": 45286, "zyn": 45287, "sedge": 45288, "oodle": 45289, "spursofficial": 45290, "dsp": 45291, "sicilian": 45292, "solihull": 45293, "receivers": 45294, "ladakh": 45295, "hendrick": 45296, "theri": 45297, "presiding": 45298, "mcguinness": 45299, "litters": 45300, "gunnar": 45301, "ghoul": 45302, "wib": 45303, "ntv": 45304, "karo": 45305, "frock": 45306, "blau": 45307, "amplify": 45308, "allis": 45309, "ullah": 45310, "memoirs": 45311, "khloe": 45312, "interceptions": 45313, "petday": 45314, "looney": 45315, "confin": 45316, "chay": 45317, "piyushgoyal": 45318, "frequencies": 45319, "utz": 45320, "eventual": 45321, "warmly": 45322, "oblivion": 45323, "anka": 45324, "tait": 45325, "âĿ¤ï¸ı.": 45326, "directorial": 45327, "rulers": 45328, "princes": 45329, "muck": 45330, "sturridge": 45331, "deuce": 45332, "abridged": 45333, "baguette": 45334, "uncles": 45335, "pendu": 45336, "minding": 45337, "forrester": 45338, "avila": 45339, "waller": 45340, "wallstreet": 45341, "mentor": 45342, "hino": 45343, "highway": 45344, "cromwell": 45345, "fanartfriday": 45346, "mbi": 45347, "coyle": 45348, "ahi": 45349, "trove": 45350, "spiegel": 45351, "paytm": 45352, "mcintosh": 45353, "jansen": 45354, "niti": 45355, "nashville": 45356, "leno": 45357, "leicestershire": 45358, "legos": 45359, "dict": 45360, "ðŁĵ½": 45361, "spad": 45362, "beverlyhills": 45363, "syrah": 45364, "separates": 45365, "zain": 45366, "unfit": 45367, "drags": 45368, "tania": 45369, "overflowing": 45370, "hrithik": 45371, "hawthorn": 45372, "zani": 45373, "macfar": 45374, "fide": 45375, "totem": 45376, "peds": 45377, "fundamentally": 45378, "calico": 45379, "sinner": 45380, "jä": 45381, "hilde": 45382, "dsd": 45383, "tenay": 45384, "tahit": 45385, "milf": 45386, "lieb": 45387, "informing": 45388, "uplift": 45389, "rael": 45390, "mortgages": 45391, "lect": 45392, "iiii": 45393, "guillaume": 45394, "composites": 45395, "oldsmobile": 45396, "lend": 45397, "garth": 45398, "commish": 45399, "baptized": 45400, "scorpions": 45401, "rucker": 45402, "bringbackour": 45403, "alliance": 45404, "thalapathy": 45405, "tali": 45406, "spans": 45407, "eridge": 45408, "witherspoon": 45409, "linda": 45410, "skylar": 45411, "korn": 45412, "homs": 45413, "Äį": 45414, "silenced": 45415, "caffe": 45416, "arty": 45417, "distinguish": 45418, "towed": 45419, "pung": 45420, "jessica": 45421, "earnest": 45422, "beaufort": 45423, "tama": 45424, "studyabroad": 45425, "sikhs": 45426, "newbie": 45427, "navratri": 45428, "marble": 45429, "lounging": 45430, "litter": 45431, "dalit": 45432, "sosa": 45433, "izes": 45434, "grade": 45435, "compromising": 45436, "triton": 45437, "detta": 45438, "vj": 45439, "chauffe": 45440, "spectral": 45441, "powered": 45442, "montessori": 45443, "articulate": 45444, "halton": 45445, "alco": 45446, "yey": 45447, "mntwins": 45448, "acounty": 45449, "ðŁijıðŁı¾": 45450, "âīĪ": 45451, "madmen": 45452, "kala": 45453, "grum": 45454, "chik": 45455, "atis": 45456, "sume": 45457, "akhtar": 45458, "jobsearch": 45459, "highlighter": 45460, "boath": 45461, "âĦ¹": 45462, "tarzan": 45463, "lambo": 45464, "âĽĦï¸ı": 45465, "oxfam": 45466, "dumpster": 45467, "pretzels": 45468, "macos": 45469, "inclined": 45470, "factual": 45471, "advertisers": 45472, "shui": 45473, "puree": 45474, "mlpfi": 45475, "antidote": 45476, "capo": 45477, "pastr": 45478, "mercado": 45479, "button": 45480, "armin": 45481, "agg": 45482, "lolla": 45483, "horribly": 45484, "errands": 45485, "christophe": 45486, "timesnow": 45487, "mondaymotiv": 45488, "liss": 45489, "scandals": 45490, "mci": 45491, "disproportion": 45492, "âĺİ": 45493, "surpass": 45494, "samaritan": 45495, "sotho": 45496, "purest": 45497, "flatt": 45498, "triviatuesday": 45499, "delectable": 45500, "leopold": 45501, "hermione": 45502, "choudhary": 45503, "enrich": 45504, "¡¡": 45505, "subsidiary": 45506, "inequalities": 45507, "bachelor": 45508, "autoimmune": 45509, "lakota": 45510, "ihop": 45511, "adjec": 45512, "thesimpsons": 45513, "shes": 45514, "sek": 45515, "gretchen": 45516, "upstream": 45517, "hinakhan": 45518, "copernic": 45519, "xtina": 45520, "lug": 45521, "toughness": 45522, "ead": 45523, "clipped": 45524, "bius": 45525, "slv": 45526, "fahren": 45527, "deepak": 45528, "cau": 45529, "xan": 45530, "immature": 45531, "digni": 45532, "bobs": 45533, "shredding": 45534, "buttery": 45535, "accommodations": 45536, "deven": 45537, "chunks": 45538, "superleague": 45539, "skybet": 45540, "kildare": 45541, "jeet": 45542, "ëį": 45543, "cek": 45544, "wrecks": 45545, "propane": 45546, "ohl": 45547, "tbd": 45548, "quoi": 45549, "trumpp": 45550, "mimo": 45551, "reluctant": 45552, "verne": 45553, "oic": 45554, "magh": 45555, "arnau": 45556, "sever": 45557, "lidge": 45558, "stairway": 45559, "kicchasudeep": 45560, "ðŁĶº": 45561, "machining": 45562, "aamaadmi": 45563, "oti": 45564, "cda": 45565, "alit": 45566, "pany": 45567, "installs": 45568, "acct": 45569, "eshop": 45570, "diem": 45571, "hardwell": 45572, "fulfillment": 45573, "scafe": 45574, "quack": 45575, "extracts": 45576, "sweetened": 45577, "fighton": 45578, "fdi": 45579, "dinger": 45580, "waltham": 45581, "usur": 45582, "referees": 45583, "seokjin": 45584, "grann": 45585, "afrin": 45586, "thn": 45587, "schaf": 45588, "parcels": 45589, "betis": 45590, "amarine": 45591, "noman": 45592, "khtar": 45593, "moritz": 45594, "coupling": 45595, "barons": 45596, "ðŁIJ¸": 45597, "ø": 45598, "slp": 45599, "sadler": 45600, "xander": 45601, "triad": 45602, "mcmillan": 45603, "khz": 45604, "dividing": 45605, "ìĹijìĨĮ": 45606, "daryl": 45607, "zedd": 45608, "leys": 45609, "plaques": 45610, "fluori": 45611, "tipperary": 45612, "onnell": 45613, "didier": 45614, "langford": 45615, "imc": 45616, "thesun": 45617, "birdies": 45618, "archa": 45619, "yessss": 45620, "tdi": 45621, "daria": 45622, "candace": 45623, "altam": 45624, "palaces": 45625, "chit": 45626, "santam": 45627, "eventful": 45628, "bookof": 45629, "adb": 45630, "monstax": 45631, "creole": 45632, "coel": 45633, "âĸ½": 45634, "wearen": 45635, "stennis": 45636, "sheath": 45637, "atism": 45638, "groningen": 45639, "mlpfim": 45640, "lepre": 45641, "wrongly": 45642, "rspca": 45643, "rendezvous": 45644, "acknowledging": 45645, "pelvic": 45646, "solicitor": 45647, "slays": 45648, "nuestra": 45649, "lod": 45650, "islander": 45651, "feroci": 45652, "fashionshow": 45653, "rass": 45654, "dgeon": 45655, "adolescents": 45656, "smashes": 45657, "negligence": 45658, "grateful": 45659, "vedere": 45660, "swoop": 45661, "ingl": 45662, "apolice": 45663, "vandalism": 45664, "gann": 45665, "joao": 45666, "disupdates": 45667, "zimbabwe": 45668, "underage": 45669, "radiance": 45670, "wof": 45671, "bourgeo": 45672, "plas": 45673, "crani": 45674, "ghue": 45675, "wreckem": 45676, "warrants": 45677, "reform": 45678, "jimmie": 45679, "atwood": 45680, "ysl": 45681, "neilhimself": 45682, "lbj": 45683, "iman": 45684, "tanto": 45685, "noisse": 45686, "verbs": 45687, "equipo": 45688, "altogether": 45689, "mament": 45690, "lice": 45691, "douglass": 45692, "tierney": 45693, "primed": 45694, "jhal": 45695, "furnitu": 45696, "brazili": 45697, "vill": 45698, "pastels": 45699, "nison": 45700, "uff": 45701, "paralysis": 45702, "jaye": 45703, "impo": 45704, "ðŁijģ": 45705, "strategically": 45706, "pakistanis": 45707, "wassup": 45708, "superbike": 45709, "thanku": 45710, "truelove": 45711, "shaikh": 45712, "israelis": 45713, "vip": 45714, "tog": 45715, "lien": 45716, "laker": 45717, "greyhounds": 45718, "culars": 45719, "bianchi": 45720, "balotelli": 45721, "arran": 45722, "loos": 45723, "strates": 45724, "hebron": 45725, "arvo": 45726, "sunderland": 45727, "theal": 45728, "tombstone": 45729, "sandman": 45730, "cpac": 45731, "thanksgiving": 45732, "lovehim": 45733, "latino": 45734, "anin": 45735, "akaif": 45736, "ĭãĤ": 45737, "torquay": 45738, "diest": 45739, "allianz": 45740, "ðŁĺķ": 45741, "golfclub": 45742, "cllr": 45743, "walcott": 45744, "schnau": 45745, "prompted": 45746, "nominating": 45747, "lennox": 45748, "valet": 45749, "monro": 45750, "mayward": 45751, "eph": 45752, "ðŁĶĶ": 45753, "interoper": 45754, "rda": 45755, "reflex": 45756, "armchair": 45757, "ê°ķ": 45758, "stripper": 45759, "porti": 45760, "pharm": 45761, "hamza": 45762, "nireland": 45763, "neue": 45764, "hpv": 45765, "portfoli": 45766, "sunburn": 45767, "frisbee": 45768, "beal": 45769, "baptiste": 45770, "xh": 45771, "tym": 45772, "prati": 45773, "overs": 45774, "hazrat": 45775, "desert": 45776, "derry": 45777, "usky": 45778, "emmett": 45779, "acharya": 45780, ")_/¯": 45781, "shud": 45782, "maya": 45783, "hamill": 45784, "raim": 45785, "nrc": 45786, "fittings": 45787, "curvy": 45788, "ðŁıĩ": 45789, "sterling": 45790, "à¥Ģ": 45791, "walkin": 45792, "shortcuts": 45793, "milly": 45794, "astur": 45795, "alphabe": 45796, "pli": 45797, "pez": 45798, "missyou": 45799, "radford": 45800, "mlg": 45801, "taeyang": 45802, "notjustlakes": 45803, "dumps": 45804, "serendip": 45805, "leur": 45806, "raving": 45807, "ester": 45808, "depriv": 45809, "abscbn": 45810, "ðŁijĩðŁı»": 45811, "scarcity": 45812, "ocr": 45813, "meanings": 45814, "capt": 45815, "dahl": 45816, "fermentation": 45817, "brioche": 45818, "towin": 45819, "outlander": 45820, "massimo": 45821, "encro": 45822, "ðŁ¥³": 45823, "built": 45824, "potam": 45825, "kiri": 45826, "tmw": 45827, "monitored": 45828, "kites": 45829, "peoplesvote": 45830, "grayson": 45831, "íģ¬": 45832, "afrika": 45833, "adies": 45834, "ivote": 45835, "gyne": 45836, "gannon": 45837, "dix": 45838, "cmc": 45839, "oural": 45840, "foxandfriends": 45841, "beli": 45842, "igne": 45843, "glan": 45844, "katrinakaif": 45845, "copolitics": 45846, "qualitative": 45847, "psi": 45848, "lucci": 45849, "discoura": 45850, "âĺ®": 45851, "kelli": 45852, "gautam": 45853, "caracas": 45854, "realest": 45855, "pula": 45856, "inus": 45857, "hilltop": 45858, "makeaw": 45859, "attenborough": 45860, "twy": 45861, "rarity": 45862, "peckham": 45863, "mahon": 45864, "cornelius": 45865, "clinicians": 45866, "tonline": 45867, "tbi": 45868, "paradise": 45869, "kasi": 45870, "inevit": 45871, "freshness": 45872, "collingwood": 45873, "lunatic": 45874, "defense": 45875, "copd": 45876, "infra": 45877, "wainwright": 45878, "sainsbury": 45879, "alabam": 45880, "tema": 45881, "laco": 45882, "checker": 45883, "relegated": 45884, "trent": 45885, "stalks": 45886, "huffpost": 45887, "bhubaneswar": 45888, "astral": 45889, "shareyour": 45890, "primrose": 45891, "hime": 45892, "catan": 45893, "endment": 45894, "endow": 45895, "clemens": 45896, "maloney": 45897, "hilary": 45898, "gametime": 45899, "denise": 45900, "collaborators": 45901, "bwo": 45902, "radicals": 45903, "guetta": 45904, "icion": 45905, "aua": 45906, "snapmatic": 45907, "satchel": 45908, "excavation": 45909, "baseman": 45910, "são": 45911, "gnation": 45912, "feld": 45913, "survey": 45914, "shahzad": 45915, "mast": 45916, "anirudhofficial": 45917, "trucker": 45918, "otago": 45919, "geograph": 45920, "ethel": 45921, "âļ¡ï¸ıâļ¡ï¸ı": 45922, "sver": 45923, "mutt": 45924, "internetofthings": 45925, "anchored": 45926, "whouse": 45927, "bangla": 45928, "balmain": 45929, "ç¹ĭãģ": 45930, "breakfa": 45931, "áĢ": 45932, "twister": 45933, "tetris": 45934, "cav": 45935, "stags": 45936, "gz": 45937, "aub": 45938, "stormed": 45939, "helens": 45940, "yarmouth": 45941, "stasy": 45942, "gustavo": 45943, "cosc": 45944, "vinson": 45945, "upp": 45946, "scricket": 45947, "assumptions": 45948, "appe": 45949, "nuh": 45950, "uer": 45951, "premise": 45952, "naga": 45953, "eamon": 45954, "coronary": 45955, "naf": 45956, "northside": 45957, "elmer": 45958, "rotar": 45959, "outlining": 45960, "elf": 45961, "resurg": 45962, "katelyn": 45963, "incan": 45964, "hysteria": 45965, "cee": 45966, "ambani": 45967, "prolly": 45968, "ĮãĤĬãģ": 45969, "axes": 45970, "sanjose": 45971, "rembrandt": 45972, "magpie": 45973, "evenly": 45974, "scorsese": 45975, "quaint": 45976, "fg": 45977, "bbuk": 45978, "indianfootball": 45979, "weareall": 45980, "spdwy": 45981, "pisces": 45982, "ecg": 45983, "âĺħâĺħâĺħâĺħâĺħ": 45984, "preorders": 45985, ":|": 45986, "nipple": 45987, "salazar": 45988, "jume": 45989, "jailbreak": 45990, "minn": 45991, "bassett": 45992, "zetta": 45993, "jeffree": 45994, "adjun": 45995, "ticon": 45996, "sandiego": 45997, "drinklocal": 45998, "cholera": 45999, "solicitors": 46000, "obo": 46001, "compost": 46002, "nian": 46003, "wra": 46004, "treach": 46005, "icic": 46006, "professional": 46007, "delve": 46008, "legate": 46009, "historia": 46010, "croissant": 46011, "connoisse": 46012, "namo": 46013, "palliative": 46014, "chemtrails": 46015, "iority": 46016, "globalwarming": 46017, "comicart": 46018, "behavioural": 46019, "rested": 46020, "lias": 46021, "climates": 46022, "ŁãģĦ": 46023, "rutland": 46024, "nourish": 46025, "menopause": 46026, "hotties": 46027, "dementi": 46028, "vespa": 46029, "melville": 46030, "analogue": 46031, "tzman": 46032, "strung": 46033, "imperfect": 46034, "glare": 46035, "circling": 46036, "rosberg": 46037, "reco": 46038, "ocity": 46039, "loire": 46040, "embe": 46041, "dossier": 46042, "neel": 46043, "nando": 46044, "mea": 46045, "galvani": 46046, "finesse": 46047, "agp": 46048, "berkeley": 46049, "asim": 46050, "âĺºâĺº": 46051, "quilted": 46052, "ishere": 46053, "unmatched": 46054, "potion": 46055, "forz": 46056, "atre": 46057, "selfies": 46058, "juliana": 46059, "ðŁļ¶": 46060, "âĸº": 46061, "melton": 46062, "âłĢâłĢâłĢâłĢâłĢâłĢâłĢâłĢ": 46063, "spinrilla": 46064, "purcell": 46065, "edp": 46066, "atleti": 46067, "tonyawards": 46068, "raja": 46069, "progno": 46070, "molten": 46071, "stuff": 46072, "pally": 46073, "nobelprize": 46074, "âĻ»ï¸ı": 46075, "spiritual": 46076, "speake": 46077, "sasha": 46078, "brium": 46079, "truss": 46080, "criticize": 46081, "assassinscreed": 46082, "yoruba": 46083, "ulo": 46084, "fireman": 46085, "workinprogress": 46086, "efcc": 46087, "flares": 46088, "robot": 46089, "hikers": 46090, "cll": 46091, "shadowing": 46092, "patsy": 46093, "lehman": 46094, "cns": 46095, "å±": 46096, "guadal": 46097, "à±į": 46098, "rape": 46099, "rhonda": 46100, "parallels": 46101, "sonja": 46102, "language": 46103, "landings": 46104, "zola": 46105, "cramps": 46106, "burning": 46107, "appraisal": 46108, "jolla": 46109, "hamm": 46110, "kasa": 46111, "gully": 46112, "fgo": 46113, "ulysses": 46114, "ribe": 46115, "ðŁĴĦ": 46116, "ibu": 46117, "etienne": 46118, "briar": 46119, "finely": 46120, "combating": 46121, "yql": 46122, "gotham": 46123, "wechat": 46124, "topaz": 46125, "primaries": 46126, "lse": 46127, "izz": 46128, "hele": 46129, "disponible": 46130, "cystic": 46131, "belichick": 46132, "thrush": 46133, "kansascity": 46134, "geom": 46135, "solidi": 46136, "redbubble": 46137, "bystand": 46138, "cambridgeshire": 46139, "parfait": 46140, "astle": 46141, "owo": 46142, "indore": 46143, "stomping": 46144, "smelly": 46145, "ðŁ¤ĸ": 46146, "locomo": 46147, "admitting": 46148, "holme": 46149, "clockwise": 46150, "minsk": 46151, "mcco": 46152, "forget": 46153, "evp": 46154, "camra": 46155, "abella": 46156, "yotes": 46157, "universityof": 46158, "méxico": 46159, "silverado": 46160, "ricket": 46161, "crombie": 46162, "puj": 46163, "eradicate": 46164, "delight": 46165, "ygo": 46166, "glamping": 46167, "vica": 46168, "duggan": 46169, "counters": 46170, "cfd": 46171, "scour": 46172, "reactjs": 46173, "puram": 46174, "parasites": 46175, "inki": 46176, "villen": 46177, "stella": 46178, "limbo": 46179, "angas": 46180, "kcr": 46181, "ðŁĴļðŁĴļðŁĴļ": 46182, "vapori": 46183, "mumford": 46184, "oligar": 46185, "à¼": 46186, "aloo": 46187, "booties": 46188, "adr": 46189, "kelli": 46190, "drummers": 46191, "avici": 46192, "natureuk": 46193, "ronal": 46194, "intrac": 46195, "unsplash": 46196, "leche": 46197, "goma": 46198, "eline": 46199, "enviro": 46200, "bionic": 46201, "bueno": 46202, "mik": 46203, "avin": 46204, "starling": 46205, "empowers": 46206, "cakeday": 46207, "boycot": 46208, "ðŁĴļðŁĴļ": 46209, "ðŁĮ¸ðŁĮ¸": 46210, "vach": 46211, "mci": 46212, "fractures": 46213, "geri": 46214, "sking": 46215, "excluded": 46216, "luce": 46217, "jave": 46218, "iggy": 46219, "eviden": 46220, "akistan": 46221, "awn": 46222, "morals": 46223, "lucifer": 46224, "haban": 46225, "tumbling": 46226, "sundaymotivation": 46227, "mosley": 46228, "captainamerica": 46229, "schicago": 46230, "theone": 46231, "motd": 46232, "dts": 46233, "ðŁIJ¼": 46234, "repell": 46235, "iii": 46236, "locust": 46237, "geospatial": 46238, "mersey": 46239, "immerse": 46240, "descend": 46241, "bernade": 46242, "js": 46243, "boatsales": 46244, "winder": 46245, "crank": 46246, "singleton": 46247, "candidacy": 46248, "bena": 46249, "ðŁı»âĢį": 46250, "highlander": 46251, "olt": 46252, "kprs": 46253, "healthylifestyle": 46254, "fourteen": 46255, "endthe": 46256, "ithaca": 46257, "circulated": 46258, "rans": 46259, "prevalent": 46260, "havas": 46261, "splendor": 46262, "rooster": 46263, "kalamazoo": 46264, "jewellers": 46265, "ennedy": 46266, "rousey": 46267, "esy": 46268, "cannons": 46269, "ornamental": 46270, "////": 46271, "rendon": 46272, "winne": 46273, "molding": 46274, "eidmubarak": 46275, "countess": 46276, "simona": 46277, "hawa": 46278, "foes": 46279, "duster": 46280, "sbu": 46281, "portray": 46282, "marries": 46283, "goodday": 46284, "choco": 46285, "achiever": 46286, "ðŁĺ¹ðŁĺ¹": 46287, "preneur": 46288, "tramp": 46289, "tomi": 46290, "nbat": 46291, "gardenchat": 46292, "farrakhan": 46293, "everglades": 46294, "abru": 46295, "sousa": 46296, "sece": 46297, "homeswee": 46298, "terrestrial": 46299, "barit": 46300, "sridevi": 46301, "olu": 46302, "melinda": 46303, "frick": 46304, "candies": 46305, "ðŁĺŃðŁĴķ": 46306, "qureshi": 46307, "familyfun": 46308, "exorcist": 46309, "cardinal": 46310, "nyt": 46311, "diesel": 46312, "cumulus": 46313, "capricorn": 46314, "siology": 46315, "lorna": 46316, "dougie": 46317, "andie": 46318, "supersport": 46319, "cfl": 46320, "пÑĢи": 46321, "sayang": 46322, "peek": 46323, "à¸Ĭ": 46324, "lobe": 46325, "jem": 46326, "inglis": 46327, "ggled": 46328, "csn": 46329, "amnesty": 46330, "chups": 46331, "baes": 46332, "sauer": 46333, "ðŁıIJ": 46334, "mongolian": 46335, "enet": 46336, "backstreet": 46337, "drilled": 46338, "accessing": 46339, "ceo": 46340, "bse": 46341, "aiken": 46342, "purr": 46343, "worsen": 46344, "wheres": 46345, "wark": 46346, "testifying": 46347, "buri": 46348, "blast": 46349, "awg": 46350, "ðŁĵĭ": 46351, "redefining": 46352, "hearing": 46353, "uci": 46354, "cmp": 46355, "boni": 46356, "tailoring": 46357, "taji": 46358, "nocchi": 46359, "emt": 46360, "stephenking": 46361, "neet": 46362, "complains": 46363, "campaigner": 46364, "luciano": 46365, "twilight": 46366, "tiesto": 46367, "passports": 46368, "floyd": 46369, "cathedr": 46370, "naked": 46371, "caregiver": 46372, "bcoz": 46373, "adecides": 46374, "kuri": 46375, "lyk": 46376, "braries": 46377, "drenched": 46378, "disclose": 46379, "ðŁĴªðŁı½": 46380, "leblanc": 46381, "jetty": 46382, "garty": 46383, "chipmun": 46384, "bsu": 46385, "rhythmic": 46386, "icz": 46387, "frid": 46388, "annex": 46389, "amex": 46390, "soloist": 46391, "lancers": 46392, "arrowhead": 46393, "specification": 46394, "simulated": 46395, "nais": 46396, "inverte": 46397, "bowing": 46398, "worship": 46399, "fz": 46400, "aboss": 46401, "shaq": 46402, "ì¶ķ": 46403, "challengers": 46404, "anarch": 46405, "aamaadmiparty": 46406, "ãħĭãħĭãħĭ": 46407, "suffolk": 46408, "socorro": 46409, "snell": 46410, "cladding": 46411, "absorbing": 46412, "shawa": 46413, "participates": 46414, "ðŁįĶ": 46415, "bookstores": 46416, "baku": 46417, "seaport": 46418, "kojima": 46419, "gaby": 46420, "packard": 46421, "electrician": 46422, "letit": 46423, "mowing": 46424, "fawad": 46425, "youngjae": 46426, "hotmail": 46427, "mening": 46428, "urie": 46429, "intimacy": 46430, "conti": 46431, ":\")": 46432, "lifeisgood": 46433, "inciner": 46434, "idri": 46435, "craziness": 46436, "journos": 46437, "franchi": 46438, "bottlen": 46439, "alda": 46440, "ffes": 46441, "kx": 46442, "southwe": 46443, "aira": 46444, "clayton": 46445, "scoti": 46446, "fj": 46447, "briga": 46448, "ðŁ¤ĺðŁı»": 46449, "demonstrators": 46450, "yz": 46451, "stork": 46452, "naq": 46453, "cascades": 46454, "travelchat": 46455, "plata": 46456, "padma": 46457, "franci": 46458, "attain": 46459, "batgirl": 46460, "lombard": 46461, "hoos": 46462, "ddos": 46463, "neonatal": 46464, "disclaimer": 46465, "rss": 46466, "rant": 46467, "disen": 46468, "texaste": 46469, "socal": 46470, "fractal": 46471, "camry": 46472, "strife": 46473, "snacking": 46474, "muh": 46475, "santander": 46476, "morons": 46477, "graf": 46478, "parades": 46479, "huston": 46480, "drupal": 46481, "miento": 46482, "kirstel": 46483, "hyde": 46484, "vomit": 46485, "fortified": 46486, "sphinx": 46487, "dav": 46488, "biryani": 46489, "winnings": 46490, "sbaseball": 46491, "merged": 46492, "lovelondon": 46493, "lingering": 46494, "dreambig": 46495, "carleton": 46496, "livelihood": 46497, "django": 46498, "astrid": 46499, "grids": 46500, "downe": 46501, "bruised": 46502, "sne": 46503, "scarecrow": 46504, "helium": 46505, "fnc": 46506, "biggs": 46507, "anter": 46508, "restorative": 46509, "empires": 46510, "abdel": 46511, "lifestyle": 46512, "kiwanis": 46513, "colloquium": 46514, "meen": 46515, "prick": 46516, "antique": 46517, "zeb": 46518, "mimic": 46519, "edmonds": 46520, "ðŁijĬðŁijĬ": 46521, "qing": 46522, "ppel": 46523, "mcgill": 46524, "interpreting": 46525, "âŀķ": 46526, "rashad": 46527, "doka": 46528, "narrator": 46529, "electromagnetic": 46530, "ashby": 46531, "saura": 46532, "irandeal": 46533, "âģīï¸ı": 46534, "krishnan": 46535, "indi": 46536, "ffen": 46537, "brea": 46538, "osman": 46539, "multinational": 46540, "chippe": 46541, "recruiters": 46542, "ausbiz": 46543, "pounding": 46544, "regen": 46545, "cursor": 46546, "refusal": 46547, "macs": 46548, "inak": 46549, "axial": 46550, "waifu": 46551, "upcycled": 46552, "hindustan": 46553, "cassini": 46554, "carlyle": 46555, "scratches": 46556, "reef": 46557, "manatee": 46558, "eatery": 46559, "ðŁĵ¢": 46560, "uncondition": 46561, "senpai": 46562, "onther": 46563, "comicbook": 46564, "prosciutto": 46565, "demar": 46566, "mise": 46567, "mage": 46568, "freec": 46569, "ayesha": 46570, "alder": 46571, "androidgames": 46572, "leyton": 46573, "hock": 46574, "doorway": 46575, "chicagofire": 46576, "aaliyah": 46577, "swelling": 46578, "bix": 46579, ".ðŁĺĤ": 46580, "evankirstel": 46581, "torpedo": 46582, "konstant": 46583, "genevieve": 46584, "maia": 46585, "hauser": 46586, "dotorg": 46587, "hideous": 46588, "fik": 46589, "spraw": 46590, "eek": 46591, "zappa": 46592, "wandered": 46593, "''": 46594, "rajan": 46595, "bambi": 46596, "($)": 46597, "widening": 46598, "toolbox": 46599, "sair": 46600, "illuminating": 46601, "prays": 46602, "outpatient": 46603, "iw": 46604, "dayo": 46605, "lob": 46606, "swfl": 46607, "shades": 46608, "gums": 46609, "cookin": 46610, "kodi": 46611, "griffin": 46612, "traumati": 46613, "stea": 46614, "slaughtered": 46615, "godbless": 46616, "airtime": 46617, "pseudo": 46618, "bsa": 46619, "hauled": 46620, "arif": 46621, "à¸Ńà¸ĩ": 46622, "lel": 46623, "wcpo": 46624, "militi": 46625, "charters": 46626, "worlda": 46627, "ruk": 46628, "kgs": 46629, "digitalindia": 46630, "isable": 46631, "idyllic": 46632, "espino": 46633, "marietta": 46634, "ebo": 46635, "teamcanada": 46636, "abour": 46637, "wilton": 46638, "rockstars": 46639, "favored": 46640, "physic": 46641, "wrinkle": 46642, "tbr": 46643, "dprint": 46644, "ballarat": 46645, "adal": 46646, "zey": 46647, "ðŁĺįðŁĶ¥": 46648, "tomlin": 46649, "mtr": 46650, "palsy": 46651, "fenerbah": 46652, "tighten": 46653, "philia": 46654, "ironing": 46655, "ryu": 46656, "bant": 46657, "enquire": 46658, "cair": 46659, "aburger": 46660, "trun": 46661, "greenberg": 46662, "chauhan": 46663, "irina": 46664, "shani": 46665, "trendsetter": 46666, "prett": 46667, "zafar": 46668, "alove": 46669, "vici": 46670, "panic": 46671, "noo": 46672, "lustre": 46673, "disrupted": 46674, "ballis": 46675, "sonsof": 46676, "monsi": 46677, "instac": 46678, "akest": 46679, "ëĭ¤": 46680, "kwame": 46681, "horrormovies": 46682, "district": 46683, "saucy": 46684, "mban": 46685, "armies": 46686, "withdrawn": 46687, "medics": 46688, "loftus": 46689, "eroom": 46690, "bekind": 46691, "arns": 46692, "allon": 46693, "unison": 46694, "davids": 46695, "crat": 46696, "nicotine": 46697, "soor": 46698, "smx": 46699, "onco": 46700, "cosplaying": 46701, "zombies": 46702, "harms": 46703, "eger": 46704, "rosy": 46705, "moonshine": 46706, "fein": 46707, "cett": 46708, "dubrov": 46709, "regents": 46710, "benitez": 46711, "ðŁijıðŁı¼ðŁijıðŁı¼": 46712, "stec": 46713, "malia": 46714, "prioritize": 46715, "iceland": 46716, "ftse": 46717, "vamo": 46718, "lamont": 46719, "homosexuality": 46720, "brees": 46721, "regui": 46722, "cbp": 46723, "tej": 46724, "skysports": 46725, "detergent": 46726, "shasta": 46727, "derel": 46728, "conservancy": 46729, "colorized": 46730, "accolades": 46731, "viso": 46732, "showyour": 46733, "nanow": 46734, "biceps": 46735, "usability": 46736, "bim": 46737, "dailysketch": 46738, "pearljam": 46739, "strangest": 46740, "megadeth": 46741, "broadcasts": 46742, "barren": 46743, "arton": 46744, "chriss": 46745, "configu": 46746, "lures": 46747, "isthe": 46748, "eul": 46749, "railwayana": 46750, "globalhealth": 46751, "gianni": 46752, "uaap": 46753, "slum": 46754, "consciously": 46755, "abre": 46756, "nup": 46757, "budget": 46758, "vada": 46759, "esch": 46760, "realness": 46761, "erased": 46762, "thunt": 46763, "bez": 46764, "armistice": 46765, "ðŁij¹": 46766, "shrun": 46767, "oled": 46768, "driverless": 46769, "ðŁ¤·ðŁı»âĢįâĻĢï¸ı": 46770, "wondr": 46771, "skan": 46772, "salaam": 46773, "motherland": 46774, "hwang": 46775, "geno": 46776, "gangnam": 46777, "twright": 46778, "endorsing": 46779, "enic": 46780, "adoration": 46781, "paused": 46782, "patricks": 46783, "docked": 46784, "platte": 46785, "ffxv": 46786, "ethnicity": 46787, "autoshow": 46788, "sideshow": 46789, "afterlife": 46790, "relocated": 46791, "orphaned": 46792, "foodnetwork": 46793, "dareto": 46794, "andra": 46795, "slaps": 46796, "vlive": 46797, "swims": 46798, "reimagined": 46799, "mistle": 46800, "revise": 46801, "reality": 46802, "bharti": 46803, "ðŁĴĻðŁĴĽ": 46804, "latest": 46805, "proudest": 46806, "grasses": 46807, "lanyard": 46808, "freshest": 46809, "carcinoma": 46810, "anomaly": 46811, "ziegler": 46812, "sumner": 46813, "lyrix": 46814, "gorg": 46815, "isd": 46816, "avel": 46817, "swildlife": 46818, "mesqu": 46819, "johncena": 46820, "euroleague": 46821, "saber": 46822, "masterful": 46823, "yarra": 46824, "cognition": 46825, "jacobson": 46826, "abolic": 46827, "sirloin": 46828, "shukla": 46829, "mojito": 46830, "supere": 46831, "stweet": 46832, "mez": 46833, "esa": 46834, "rudolf": 46835, "gura": 46836, "whereyou": 46837, "ttm": 46838, "wins": 46839, "trustworthy": 46840, "nyk": 46841, "braden": 46842, "tabletop": 46843, "goodfood": 46844, "eson": 46845, "bek": 46846, "linguistic": 46847, "grays": 46848, "chath": 46849, "hcs": 46850, "moni": 46851, "deans": 46852, "cussions": 46853, "chell": 46854, "slows": 46855, "hemi": 46856, "dapp": 46857, "sharpie": 46858, "boosters": 46859, "aos": 46860, "strack": 46861, "sedona": 46862, "mueller": 46863, "hardwick": 46864, "ornate": 46865, "thora": 46866, "salud": 46867, "otwol": 46868, "chum": 46869, "miho": 46870, "forage": 46871, "thelittle": 46872, "tearful": 46873, "oneself": 46874, "mindy": 46875, "smg": 46876, "gmbh": 46877, "emerald": 46878, "ðŁĶ´âļªï¸ı": 46879, "tutti": 46880, "receptions": 46881, "revising": 46882, "ibrox": 46883, "topeka": 46884, "salami": 46885, "expanse": 46886, "ibooks": 46887, "dobson": 46888, "clio": 46889, "ats": 46890, "ðŁļĮ": 46891, "moha": 46892, "isance": 46893, "shutters": 46894, "moot": 46895, "janine": 46896, "marvelcomics": 46897, "jordani": 46898, "poser": 46899, "kenneth": 46900, "hyung": 46901, "deja": 46902, "aseball": 46903, "speciality": 46904, "euston": 46905, "classiccar": 46906, "hadith": 46907, "ðŁIJī": 46908, "chasing": 46909, "izo": 46910, "grosven": 46911, "aglia": 46912, "thisdayinhistory": 46913, "trow": 46914, "omile": 46915, "huar": 46916, "byn": 46917, "saline": 46918, "divine": 46919, "demonic": 46920, "tyran": 46921, "handover": 46922, "revitalization": 46923, "paella": 46924, "cryptic": 46925, "sedg": 46926, "mend": 46927, "dunkirk": 46928, "bred": 46929, "wald": 46930, "sportscar": 46931, "aard": 46932, "wheaton": 46933, "daener": 46934, "klan": 46935, "brt": 46936, "bakhtawar": 46937, "spires": 46938, "schubert": 46939, "roti": 46940, "polish": 46941, "ose": 46942, "agame": 46943, "wondercon": 46944, "protestant": 46945, "bosa": 46946, "ðŁĺŁ": 46947, "dü": 46948, "joyride": 46949, "gertrude": 46950, "âĿĿ": 46951, "gila": 46952, "vh": 46953, "twa": 46954, "trav": 46955, "swallowed": 46956, "starve": 46957, "lain": 46958, "entren": 46959, "reiki": 46960, "sukh": 46961, "craic": 46962, "azu": 46963, "webpage": 46964, "keefe": 46965, "hypothe": 46966, "hirsch": 46967, "helle": 46968, "campground": 46969, "wamy": 46970, "travi": 46971, "shahi": 46972, "sandeep": 46973, "rui": 46974, "hanuman": 46975, "dwp": 46976, "repository": 46977, "noor": 46978, "noff": 46979, "unreal": 46980, "pell": 46981, "blackhistory": 46982, "harvick": 46983, "mascar": 46984, "payee": 46985, "pasha": 46986, "gastronomy": 46987, "dÃŃ": 46988, "aig": 46989, "rosenthal": 46990, "openday": 46991, "embellished": 46992, "ttip": 46993, "sunbathing": 46994, "gopack": 46995, "endome": 46996, "ï¸ı#": 46997, "invalid": 46998, "finalfour": 46999, "stfu": 47000, "squishy": 47001, "rasta": 47002, "mosch": 47003, "jamesc": 47004, "dietrich": 47005, "sela": 47006, "melb": 47007, "elvi": 47008, "tdp": 47009, "suni": 47010, "slit": 47011, "jha": 47012, "biza": 47013, "spiked": 47014, "lli": 47015, "lillard": 47016, "vampi": 47017, "synopsis": 47018, "azhar": 47019, "kendricklamar": 47020, "ĮãĤĬãģŁãģĦ": 47021, "heartless": 47022, "countryfile": 47023, "airplay": 47024, "arrogance": 47025, "pree": 47026, "virtuoso": 47027, "ãħłãħłãħłãħł": 47028, "raju": 47029, "lebu": 47030, "forward": 47031, "tug": 47032, "dros": 47033, "mondaymotivaton": 47034, "concepcion": 47035, "thelo": 47036, "padi": 47037, "looool": 47038, "ÑĢод": 47039, "itss": 47040, "ethical": 47041, "enduro": 47042, "__:": 47043, "expenditure": 47044, "monste": 47045, "masking": 47046, "terriers": 47047, "ibis": 47048, "ember": 47049, "cumple": 47050, "punctuation": 47051, "piper": 47052, "irvin": 47053, "adee": 47054, "yyyyyy": 47055, "flashbacks": 47056, "celsius": 47057, "donnie": 47058, "bogota": 47059, "benevol": 47060, "thescript": 47061, "shilpa": 47062, "prose": 47063, "findia": 47064, "zeke": 47065, "neko": 47066, "doves": 47067, "blueslyrix": 47068, "frosh": 47069, "soweto": 47070, "mplo": 47071, "alai": 47072, "sabi": 47073, "raqqa": 47074, "wftv": 47075, "stroller": 47076, "iansomerhalder": 47077, "ðŁĶª": 47078, "anon": 47079, "moseley": 47080, "!?!?": 47081, "staking": 47082, "moly": 47083, "cartri": 47084, "csg": 47085, "astor": 47086, "transcend": 47087, "maer": 47088, "deux": 47089, "cowgirl": 47090, "sask": 47091, "punter": 47092, "maken": 47093, "oates": 47094, "lovett": 47095, "growler": 47096, "sagin": 47097, "vn": 47098, "ssible": 47099, "officeofrg": 47100, "ymc": 47101, "sabar": 47102, "faulty": 47103, "apha": 47104, "akon": 47105, "ðŁij«": 47106, "snowdon": 47107, "aew": 47108, "raisethe": 47109, "ðĿĵ": 47110, "gruesome": 47111, "clementine": 47112, "sping": 47113, "lata": 47114, "worldenviron": 47115, "mimic": 47116, "canaria": 47117, "bakhtawarbz": 47118, "aoa": 47119, "fala": 47120, "ãĤŃ": 47121, "aviva": 47122, "youuuu": 47123, "thigh": 47124, "ladders": 47125, "gumbo": 47126, "tzky": 47127, "fuzz": 47128, "plasticpollution": 47129, "estate": 47130, "strengthened": 47131, "kant": 47132, "drin": 47133, "calvert": 47134, "transformational": 47135, "frightened": 47136, "maclean": 47137, "elitedangerous": 47138, "earthy": 47139, "tson": 47140, "toda": 47141, "jnu": 47142, "..,": 47143, "michal": 47144, "iban": 47145, "jeong": 47146, "isreal": 47147, "simcoe": 47148, "exclusives": 47149, "bluebells": 47150, "bene": 47151, "teu": 47152, "pilsner": 47153, "penske": 47154, "atheists": 47155, "mpu": 47156, "cartagena": 47157, "ðŁĴĹðŁĴĹ": 47158, "millionaires": 47159, "kkkk": 47160, "itar": 47161, "subscriptions": 47162, "remote": 47163, "mafi": 47164, "hinton": 47165, "wcc": 47166, "hok": 47167, "dsb": 47168, "ableton": 47169, "seventy": 47170, "punks": 47171, "eindhoven": 47172, "shone": 47173, "mcfarlane": 47174, "limpopo": 47175, "emphasi": 47176, "ü": 47177, "sinfo": 47178, "petre": 47179, "mangrove": 47180, "chino": 47181, "bertie": 47182, "playlists": 47183, "pushawards": 47184, "paf": 47185, "debbie": 47186, "cdo": 47187, "rino": 47188, "ðŁı¾âĢįâĻĤï¸ı": 47189, "folke": 47190, "bonnar": 47191, "thine": 47192, "slan": 47193, "halter": 47194, "evie": 47195, "awsome": 47196, "vultures": 47197, "sparky": 47198, "seizures": 47199, "âľĶ": 47200, "ramone": 47201, "ineffe": 47202, "aln": 47203, "proctor": 47204, "astra": 47205, "thevoice": 47206, "grote": 47207, "scion": 47208, "deadline": 47209, "amaya": 47210, "tainted": 47211, "patterned": 47212, "exceeding": 47213, "crossfit": 47214, "kaylee": 47215, "dropbox": 47216, "rushes": 47217, "tackled": 47218, "moby": 47219, "retrogamer": 47220, "ncbd": 47221, "benefitting": 47222, "shaykh": 47223, "guildhall": 47224, "gentry": 47225, "dreamcast": 47226, "dreaded": 47227, "bundled": 47228, "thaw": 47229, "revolving": 47230, "npt": 47231, "kyliejenner": 47232, "imaginative": 47233, "roni": 47234, "overcame": 47235, "familytime": 47236, "dsburg": 47237, "carnaval": 47238, "relationship": 47239, "recognizable": 47240, "coroner": 47241, "hole": 47242, "fanfic": 47243, "emirates": 47244, "burritos": 47245, "analyse": 47246, "thinner": 47247, "nees": 47248, "gallipoli": 47249, "blr": 47250, "catwoman": 47251, "-->>": 47252, "ault": 47253, "adaily": 47254, "naughty": 47255, "ilio": 47256, "solitaire": 47257, "mtvbr": 47258, "jocelyn": 47259, "arunach": 47260, "repent": 47261, "southgate": 47262, "hyacin": 47263, "essential": 47264, "fenton": 47265, "andum": 47266, "itor": 47267, "gopal": 47268, "slinger": 47269, "posei": 47270, "awil": 47271, "wielding": 47272, "raila": 47273, "elias": 47274, "asto": 47275, "ä": 47276, "tendency": 47277, "strata": 47278, "kert": 47279, "<-": 47280, "imacele": 47281, "daes": 47282, "stimulus": 47283, "hanley": 47284, "fitnes": 47285, "ecstasy": 47286, "limous": 47287, "hailing": 47288, "ðŁ¤Ń": 47289, "chiswick": 47290, "taries": 47291, "slav": 47292, "puli": 47293, "modernization": 47294, "blackmail": 47295, "bingham": 47296, "hfx": 47297, "++": 47298, "ðŁĩ®ðŁĩ³": 47299, "niv": 47300, "wea": 47301, "professor": 47302, "koff": 47303, "bolster": 47304, "suave": 47305, "sequences": 47306, "pepperoni": 47307, "notte": 47308, "dren": 47309, "ãģ¨ç¹ĭãģ": 47310, "hsv": 47311, "oga": 47312, "aptly": 47313, "zad": 47314, "excelsi": 47315, "rinka": 47316, "moldova": 47317, "minn": 47318, "mabel": 47319, "conferencing": 47320, "basing": 47321, "ofer": 47322, "obsi": 47323, "hamillhimself": 47324, "careless": 47325, "briefed": 47326, "inherent": 47327, "parish": 47328, "dubnation": 47329, "townsville": 47330, "sarawak": 47331, "geeky": 47332, "doncasterisgreat": 47333, "wasabi": 47334, "gup": 47335, "pheno": 47336, "drainthe": 47337, "carrieunderwood": 47338, "bleeds": 47339, "bbcworld": 47340, "anew": 47341, "altaf": 47342, "dulwich": 47343, "aniston": 47344, "wti": 47345, "sumatra": 47346, "grafton": 47347, "bln": 47348, "mester": 47349, "bodega": 47350, "rego": 47351, "esq": 47352, "anjo": 47353, "sumptuous": 47354, "maisie": 47355, "�": 47356, "wilt": 47357, "jakob": 47358, "elvis": 47359, "sepul": 47360, "muster": 47361, "airpollution": 47362, "presidente": 47363, "happymonday": 47364, "extensively": 47365, "flondon": 47366, "tls": 47367, "playing": 47368, "peed": 47369, "dinho": 47370, "vardy": 47371, "pika": 47372, "niro": 47373, "aucus": 47374, "ðŁį¦": 47375, "null": 47376, "elondon": 47377, "juventus": 47378, "imagines": 47379, "disab": 47380, "lito": 47381, "dura": 47382, "workplaces": 47383, "promote": 47384, "mccaf": 47385, "woodwork": 47386, "wawx": 47387, "ப": 47388, "ttino": 47389, "shari": 47390, "semper": 47391, "bettertogether": 47392, "ðŁijĬðŁı»": 47393, "zebra": 47394, "pondering": 47395, "enchil": 47396, "hom": 47397, "cosmic": 47398, "tanz": 47399, "mocked": 47400, "eccc": 47401, "athed": 47402, "abolish": 47403, "propeller": 47404, "parisagreement": 47405, "assemblies": 47406, "industry": 47407, "fraudulent": 47408, "pesa": 47409, "changmin": 47410, "axx": 47411, "ðŁĴµ": 47412, "irrational": 47413, "cusa": 47414, "ramadhan": 47415, "octavia": 47416, "onelove": 47417, "jacki": 47418, "barak": 47419, "taxider": 47420, "serious": 47421, "nathanfillion": 47422, "mcen": 47423, "chk": 47424, "popart": 47425, "gravity": 47426, "coppola": 47427, "readingfc": 47428, "illusions": 47429, "jig": 47430, "wwx": 47431, "resh": 47432, "exporting": 47433, "buzzard": 47434, "âĻ¤": 47435, "pcm": 47436, "lanapar": 47437, "kos": 47438, "aromas": 47439, "antalya": 47440, "wwdc": 47441, "vena": 47442, "phila": 47443, "ballin": 47444, "ðŁijĦ": 47445, "quinta": 47446, "mao": 47447, "fery": 47448, "eighty": 47449, "sentiments": 47450, "safeguarding": 47451, "rwa": 47452, "puffs": 47453, "lucille": 47454, "decath": 47455, "slu": 47456, "nugent": 47457, "deter": 47458, "brazil": 47459, "zeiss": 47460, "superbowl": 47461, "subsidy": 47462, "altern": 47463, "hidalgo": 47464, "enzymes": 47465, "ä½": 47466, "tagne": 47467, "hairdresser": 47468, "adrien": 47469, "walkout": 47470, "opposes": 47471, "cantina": 47472, "bedside": 47473, "afan": 47474, "ðŁĶĹ": 47475, "prophetic": 47476, "danes": 47477, "unsuccessful": 47478, "supercharged": 47479, "pkk": 47480, "exemption": 47481, "hartle": 47482, "secular": 47483, "clipping": 47484, "brs": 47485, "unitedway": 47486, "cnet": 47487, "patchy": 47488, "hagan": 47489, "een": 47490, "âļľ": 47491, "vara": 47492, "sympathi": 47493, "nevertrump": 47494, "affirmation": 47495, "omf": 47496, "nycfc": 47497, "maja": 47498, "surro": 47499, "keerth": 47500, "upscale": 47501, "sandalwood": 47502, "monarchy": 47503, "knobs": 47504, "åĭ": 47505, "potholes": 47506, "hungergames": 47507, "terraces": 47508, "nasir": 47509, "counsell": 47510, "welcometo": 47511, "waq": 47512, "seaman": 47513, "mita": 47514, "stunningly": 47515, "ontheroad": 47516, "inability": 47517, ")!!": 47518, "bongo": 47519, "antv": 47520, "sput": 47521, "worldenvironmentday": 47522, "resusc": 47523, "ytd": 47524, "fim": 47525, "eunhyuk": 47526, "sachin": 47527, "roseanne": 47528, "clermont": 47529, "apec": 47530, "amina": 47531, "vening": 47532, "nantes": 47533, "almost": 47534, "sinus": 47535, "exas": 47536, "tyl": 47537, "tien": 47538, "plead": 47539, "lancs": 47540, "burnaby": 47541, "rek": 47542, "joom": 47543, "observers": 47544, "discography": 47545, "clg": 47546, "âĻ¦": 47547, "snack": 47548, "rti": 47549, "oily": 47550, "crystalli": 47551, "brute": 47552, "webdevelopment": 47553, "toppings": 47554, "laf": 47555, "anis": 47556, "adder": 47557, "reliving": 47558, "carlin": 47559, "battleof": 47560, "weg": 47561, "syrian": 47562, "pont": 47563, "ndc": 47564, "laghate": 47565, "yuma": 47566, "spp": 47567, "piti": 47568, "robbing": 47569, "marting": 47570, "reykja": 47571, "rajput": 47572, "ncds": 47573, "kiewicz": 47574, "âĢ¢âĢ¢": 47575, "vampire": 47576, "substantially": 47577, "opioids": 47578, "nepali": 47579, "kline": 47580, "aroo": 47581, "understand": 47582, "litt": 47583, "uit": 47584, "thrombo": 47585, "saries": 47586, "quot": 47587, "balling": 47588, "ttr": 47589, "sgh": 47590, "philipp": 47591, "brant": 47592, "acl": 47593, "mello": 47594, "whittaker": 47595, ".;": 47596, "defiant": 47597, "bgc": 47598, "replying": 47599, "mirren": 47600, "metamorpho": 47601, "schwab": 47602, "bulge": 47603, "utilized": 47604, "pickering": 47605, "pardon": 47606, "dsa": 47607, "à¸Ī": 47608, "dooley": 47609, "cumulative": 47610, "л": 47611, "urgency": 47612, "emir": 47613, "+/-": 47614, "¦Ī": 47615, "otas": 47616, "âı³": 47617, "stationed": 47618, "grapevine": 47619, "arac": 47620, "karanjohar": 47621, "fancy": 47622, "saul": 47623, "coogs": 47624, "lgbtq": 47625, "اÙħ": 47626, "javi": 47627, "ummer": 47628, "pll": 47629, "denis": 47630, "daipur": 47631, "puffin": 47632, "lewisham": 47633, "fandom": 47634, "cope": 47635, "vesmatter": 47636, "sve": 47637, "helpless": 47638, "deodor": 47639, "ostrich": 47640, "kazan": 47641, "fridaythe": 47642, "condor": 47643, "vx": 47644, "sophomores": 47645, "robles": 47646, "cutt": 47647, "climbers": 47648, "리": 47649, "sleg": 47650, "snf": 47651, "macys": 47652, "hydrating": 47653, "groupe": 47654, "poyn": 47655, "moulin": 47656, "hgtv": 47657, "lmfaooo": 47658, "sulphur": 47659, "asdfghjkl": 47660, "annabelle": 47661, "humpback": 47662, "braved": 47663, "viswasam": 47664, "multipurpose": 47665, "humidi": 47666, "escorted": 47667, "barbican": 47668, "fad": 47669, "corsa": 47670, "ðŁ¤«": 47671, "pippa": 47672, "hereto": 47673, "cany": 47674, "sergi": 47675, "orcas": 47676, "ovie": 47677, "edou": 47678, "sany": 47679, "globalization": 47680, "mancini": 47681, "foodtruck": 47682, "fis": 47683, "defibrill": 47684, "schre": 47685, "smafia": 47686, "lovewins": 47687, "laut": 47688, "kaka": 47689, "hollande": 47690, "gameon": 47691, "resurgence": 47692, "outside": 47693, "olympiad": 47694, "intan": 47695, "abstraction": 47696, "rapid": 47697, "palom": 47698, "calle": 47699, "jasmin": 47700, "attackers": 47701, "swagg": 47702, "mitra": 47703, "kylo": 47704, "ல": 47705, "hermitage": 47706, "gordo": 47707, "eira": 47708, "sosfam": 47709, "rollout": 47710, "excite": 47711, "synod": 47712, "merrill": 47713, "cals": 47714, "assa": 47715, "livelihoods": 47716, "juve": 47717, "theblack": 47718, "gopackgo": 47719, "antlers": 47720, "albanian": 47721, "woolly": 47722, "quiche": 47723, "purification": 47724, "areth": 47725, "smarthome": 47726, "nek": 47727, "allblacks": 47728, "mexicans": 47729, "ism": 47730, "germs": 47731, "complexion": 47732, "marck": 47733, "ushi": 47734, "ðŁIJIJ": 47735, "charl": 47736, "castic": 47737, "tillerson": 47738, "giuliani": 47739, "biodegradable": 47740, "malbec": 47741, "bois": 47742, "jubil": 47743, "imes": 47744, "rame": 47745, "genetic": 47746, "espnu": 47747, "chley": 47748, "soho": 47749, "gopher": 47750, "gsc": 47751, "buuren": 47752, "cube": 47753, "bridesmaids": 47754, "webinars": 47755, "toe": 47756, "manipur": 47757, "violently": 47758, "noticias": 47759, "exchanging": 47760, "chiev": 47761, "replaceable": 47762, "muaythai": 47763, "buss": 47764, "spil": 47765, "instalment": 47766, "divya": 47767, "caitlin": 47768, "olim": 47769, "filtering": 47770, "whirlwind": 47771, "stared": 47772, "priorit": 47773, "pram": 47774, "pompeii": 47775, "monologue": 47776, "kite": 47777, "buka": 47778, "âĢ¦..": 47779, "vaccine": 47780, "brero": 47781, "wozni": 47782, "solent": 47783, "referr": 47784, "myrt": 47785, "gridiron": 47786, "galatasaray": 47787, "froze": 47788, "claremont": 47789, "ðŁ¥ĥ": 47790, "victorias": 47791, "sseldorf": 47792, "pastures": 47793, "netneutrality": 47794, "chor": 47795, "ðŁijģ": 47796, "ಿ": 47797, "weho": 47798, "symptom": 47799, "josel": 47800, "inous": 47801, "dragoncon": 47802, "powerball": 47803, "pte": 47804, "fourthofjuly": 47805, "ecla": 47806, "earbuds": 47807, "whereabouts": 47808, "saltlife": 47809, "deprivation": 47810, "chter": 47811, "wiggle": 47812, "system": 47813, "psst": 47814, "chaz": 47815, "dany": 47816, "rimo": 47817, "oaxaca": 47818, "lanaparrilla": 47819, "barcelon": 47820, "melancholy": 47821, "wayback": 47822, "hotro": 47823, "nsi": 47824, "lilly": 47825, "kuro": 47826, "jahan": 47827, "intellect": 47828, "boardgame": 47829, "ðŁıĬ": 47830, "sneakpeek": 47831, "kprc": 47832, "jails": 47833, "candel": 47834, "zanzi": 47835, "mortimer": 47836, "starch": 47837, "rags": 47838, "pfa": 47839, "longlive": 47840, "kart": 47841, "girona": 47842, "crocker": 47843, "christoph": 47844, "precautions": 47845, "warship": 47846, "perm": 47847, "parent": 47848, "vangogh": 47849, "gifford": 47850, "allegheny": 47851, "rayn": 47852, "utm": 47853, "stencil": 47854, "recalling": 47855, "penney": 47856, "zazzle": 47857, "ìĥĿ": 47858, "hinds": 47859, "arenas": 47860, "nuev": 47861, "lawler": 47862, "guin": 47863, "dothis": 47864, "ðŁijķ": 47865, "ì¶ķíķĺ": 47866, "weg": 47867, "tib": 47868, "ridin": 47869, "complexes": 47870, "turbulent": 47871, "pesos": 47872, "demarcus": 47873, "vallarta": 47874, "samsun": 47875, "kisses": 47876, "heinrich": 47877, "deportes": 47878, "wilms": 47879, "urd": 47880, "thenext": 47881, "inkigayo": 47882, "howi": 47883, "firsts": 47884, "carriage": 47885, "cleanliness": 47886, "maswar": 47887, "isch": 47888, "axel": 47889, "sizzle": 47890, "roadhouse": 47891, "frans": 47892, "entourage": 47893, "cobble": 47894, "booth": 47895, "benedict": 47896, "talon": 47897, "fcu": 47898, "yearofthe": 47899, "rayon": 47900, "raidernation": 47901, "foyle": 47902, "koval": 47903, "pianos": 47904, "lpg": 47905, "burmese": 47906, "manure": 47907, "geocaching": 47908, "coscino": 47909, "bnp": 47910, "ferra": 47911, "strophy": 47912, "marais": 47913, "cees": 47914, "legendof": 47915, "katniss": 47916, "enoch": 47917, "aved": 47918, "youknow": 47919, "dprk": 47920, "ðŁĺ¢ðŁĺ¢": 47921, "spun": 47922, "prost": 47923, "sorrows": 47924, "centred": 47925, "kea": 47926, "galicia": 47927, "?ðŁ¤Ķ": 47928, "ÑĢода": 47929, "bouchard": 47930, "ðŁĴĻðŁĴľ": 47931, "yui": 47932, "seedlings": 47933, "jonah": 47934, "recovers": 47935, "nyrd": 47936, "boardroom": 47937, "suma": 47938, "myjaps": 47939, "tung": 47940, "shai": 47941, "irgc": 47942, "elio": 47943, "wagons": 47944, "kashi": 47945, "policemen": 47946, "johnnie": 47947, "alecoscino": 47948, "shopify": 47949, "dotted": 47950, "detri": 47951, "vaw": 47952, "tofficial": 47953, "inyour": 47954, "chalmers": 47955, "traced": 47956, "novi": 47957, "byes": 47958, "ariel": 47959, "nippon": 47960, "lapel": 47961, "griez": 47962, "bgs": 47963, "fooling": 47964, "dita": 47965, "vijaysethu": 47966, "nmwx": 47967, "asot": 47968, "kranti": 47969, "helm": 47970, "vedi": 47971, "sickest": 47972, "mochi": 47973, "kabo": 47974, "shrubs": 47975, "hered": 47976, "bsp": 47977, "sqm": 47978, "hamr": 47979, "dulkar": 47980, "antha": 47981, "nrf": 47982, "avoidance": 47983, "aten": 47984, "publix": 47985, "bearers": 47986, "nasi": 47987, "hap": 47988, "hells": 47989, "ðŁĸ¥": 47990, "ื": 47991, "thelastjedi": 47992, "ohwx": 47993, "ðŁį«": 47994, "wahoo": 47995, "therese": 47996, "recaps": 47997, "ssnhq": 47998, "birdphotography": 47999, "vay": 48000, "petti": 48001, "paulo": 48002, "belvedere": 48003, "(*": 48004, "grl": 48005, "duvet": 48006, "cpec": 48007, "sait": 48008, "porsch": 48009, "measurable": 48010, "aviators": 48011, "fremantle": 48012, "breen": 48013, "onom": 48014, "meand": 48015, "lifesaving": 48016, "euref": 48017, "endon": 48018, "embaras": 48019, "airasia": 48020, "elis": 48021, "dunkin": 48022, "starmagic": 48023, "sill": 48024, "portobello": 48025, "kiefer": 48026, "exe": 48027, "muted": 48028, "ãģ¦": 48029, "wethepeople": 48030, "logia": 48031, "liberal": 48032, "theforceawakens": 48033, "mined": 48034, "haunts": 48035, "freckles": 48036, "caretaker": 48037, "sindia": 48038, "âķIJ": 48039, "devlin": 48040, "liston": 48041, "directioner": 48042, "ohn": 48043, "figaro": 48044, "emmanuel": 48045, "dubois": 48046, "clones": 48047, "bruise": 48048, "ðŁİĪðŁİī": 48049, "disinfe": 48050, "dermatology": 48051, "asr": 48052, "swatch": 48053, "discomfort": 48054, "tamanna": 48055, "piday": 48056, "macken": 48057, "katic": 48058, "delusional": 48059, "shawnee": 48060, "gud": 48061, "albino": 48062, "pali": 48063, "dingh": 48064, "cucumbers": 48065, "coffey": 48066, "anticipating": 48067, "treasured": 48068, "websummit": 48069, "sheltered": 48070, "savor": 48071, "pedagogy": 48072, "mgs": 48073, "shma": 48074, "sbu": 48075, "denali": 48076, "campos": 48077, "bubblegum": 48078, "oir": 48079, "leaps": 48080, "yler": 48081, "rone": 48082, "sanskrit": 48083, "mint": 48084, "meatless": 48085, "futurist": 48086, "dude": 48087, "avel": 48088, "protested": 48089, "squire": 48090, "zaki": 48091, "szn": 48092, "harcourt": 48093, "cyclone": 48094, "bourdain": 48095, "gatherings": 48096, "dant": 48097, "adventurer": 48098, "paragon": 48099, "altman": 48100, "dding": 48101, "banerjee": 48102, "snorkeling": 48103, "motherwell": 48104, "missy": 48105, "ender": 48106, "glows": 48107, "kiwis": 48108, "chickpea": 48109, "poro": 48110, "efron": 48111, "appt": 48112, "uy": 48113, "specified": 48114, "gabby": 48115, "estrada": 48116, "combos": 48117, "bourbon": 48118, "vini": 48119, "varun": 48120, "stephani": 48121, "keywords": 48122, "carvings": 48123, "amitabh": 48124, "wrought": 48125, "twal": 48126, "reels": 48127, "clubbing": 48128, "ubiquit": 48129, "crit": 48130, "ambedkar": 48131, "æĻ": 48132, "pruning": 48133, "vaccinated": 48134, "boeing": 48135, "sks": 48136, "loona": 48137, "hypnosis": 48138, "edelman": 48139, "phol": 48140, "hew": 48141, "colosse": 48142, "mckinsey": 48143, "uon": 48144, "tote": 48145, "sacrificing": 48146, "oxi": 48147, "nang": 48148, "emu": 48149, "пÑĢиÑĢода": 48150, "mth": 48151, "kerswednesday": 48152, "argued": 48153, "timelapse": 48154, "risking": 48155, "regulating": 48156, "nigh": 48157, "likelihood": 48158, "cubic": 48159, "auction": 48160, "reinfor": 48161, "pistor": 48162, "noses": 48163, "yel": 48164, "snuggles": 48165, "pei": 48166, "jeanette": 48167, "taku": 48168, "rith": 48169, "guyz": 48170, "à¸ŀ": 48171, "yte": 48172, "verted": 48173, "paysoff": 48174, "jauregui": 48175, "hooligans": 48176, "procedural": 48177, "mib": 48178, "hardy": 48179, "eleng": 48180, "checkers": 48181, "alline": 48182, "themet": 48183, "proudof": 48184, "keerthyofficial": 48185, "collaborator": 48186, "niu": 48187, "inflicted": 48188, "advani": 48189, "retwee": 48190, "memoriam": 48191, "ficial": 48192, "tighter": 48193, "salem": 48194, "reviewers": 48195, "brics": 48196, "bendigo": 48197, "amell": 48198, "turkish": 48199, "sushmaswar": 48200, "paulson": 48201, "palawan": 48202, "mollie": 48203, "stitcher": 48204, "sburgh": 48205, "iru": 48206, "haydn": 48207, "eners": 48208, "aroa": 48209, "uzzi": 48210, "sarajevo": 48211, "hela": 48212, "apollo": 48213, "ninety": 48214, "vaca": 48215, "spon": 48216, "ventu": 48217, "jelena": 48218, "heifer": 48219, "avoids": 48220, "spine": 48221, "prize": 48222, "marist": 48223, "recreating": 48224, "mede": 48225, "wooden": 48226, "findlay": 48227, "rofl": 48228, "ndi": 48229, "comprehend": 48230, "yugo": 48231, "yü": 48232, "towork": 48233, "ufos": 48234, "sonar": 48235, "piston": 48236, "recording": 48237, "tentative": 48238, "artforsale": 48239, "pellets": 48240, "fredo": 48241, "ÙĪر": 48242, "muses": 48243, "customization": 48244, "profound": 48245, "isner": 48246, "ideally": 48247, "siam": 48248, "plankton": 48249, "cmdr": 48250, "manger": 48251, "franken": 48252, "customizable": 48253, "म": 48254, "walkaway": 48255, "swivel": 48256, "vastly": 48257, "noton": 48258, "lexa": 48259, "exmoor": 48260, "zas": 48261, "tante": 48262, "reductions": 48263, "lolly": 48264, "hipsters": 48265, "benefited": 48266, "ë²": 48267, "wwwww": 48268, "masculine": 48269, "fiji": 48270, "drey": 48271, "phill": 48272, "aneous": 48273, "nicol": 48274, "mendez": 48275, "disappro": 48276, "chner": 48277, "throughs": 48278, "shenmue": 48279, "eastman": 48280, "ðŁIJİ": 48281, "yuck": 48282, "undertale": 48283, "reys": 48284, "gobeavs": 48285, "engen": 48286, "cna": 48287, "merr": 48288, "birk": 48289, "ãģ¨ç¹ĭãģĮãĤĬãģŁãģĦ": 48290, "âĥ£@": 48291, "ynna": 48292, "steed": 48293, "offender": 48294, "atum": 48295, "vanishing": 48296, "presidenti": 48297, "lovethem": 48298, "gnocchi": 48299, "friggin": 48300, "peril": 48301, "madhya": 48302, "agne": 48303, "deejay": 48304, "marnock": 48305, "mtb": 48306, "foldable": 48307, "@___": 48308, "standre": 48309, "bronx": 48310, "bowski": 48311, "finite": 48312, "crockett": 48313, "bsf": 48314, "getit": 48315, "serenawilliams": 48316, "miro": 48317, "ignatius": 48318, "slay": 48319, "rinse": 48320, "fondue": 48321, "seldom": 48322, "smore": 48323, "gani": 48324, "dyce": 48325, "dmitry": 48326, "crumb": 48327, "latepost": 48328, "primark": 48329, "ohana": 48330, "florals": 48331, "doa": 48332, "remembranceday": 48333, "dds": 48334, "azione": 48335, "toonami": 48336, "airport": 48337, "æĿ±": 48338, "thad": 48339, "fist": 48340, "dinesh": 48341, "drwho": 48342, "adwords": 48343, "admirer": 48344, "proje": 48345, "kyrgyz": 48346, "à«": 48347, "manifestation": 48348, "lewan": 48349, "jic": 48350, "thibau": 48351, "leased": 48352, "vanity": 48353, "nourished": 48354, "nevertheless": 48355, "augmente": 48356, "fuelled": 48357, "chead": 48358, "wilshere": 48359, "rudi": 48360, "pz": 48361, "myco": 48362, "morro": 48363, "herbalife": 48364, "hardrock": 48365, "deman": 48366, "dreality": 48367, "spades": 48368, "cevic": 48369, "bhai": 48370, "baron": 48371, "ultimatefan": 48372, "hounews": 48373, "tobi": 48374, "strut": 48375, "keel": 48376, "affiliation": 48377, "themasters": 48378, "smal": 48379, "hue": 48380, "esteban": 48381, "conv": 48382, "omnic": 48383, "databases": 48384, "cov": 48385, "terti": 48386, "stg": 48387, "snoopdogg": 48388, "metabol": 48389, "lethbridge": 48390, "ðŁı»âĢįâĻĢï¸ı": 48391, "yearling": 48392, "residentevil": 48393, "nwsl": 48394, "iyaki": 48395, "griezmann": 48396, "cous": 48397, "ðŁĵĿ:": 48398, "torian": 48399, "sami": 48400, "ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥": 48401, "gare": 48402, "alliances": 48403, "whitfield": 48404, "wether": 48405, "refining": 48406, "coyi": 48407, "kraken": 48408, "ðŁĺĺâĿ¤": 48409, "singularity": 48410, "lili": 48411, "hns": 48412, "boldand": 48413, "wawrinka": 48414, "misogyny": 48415, "lovers": 48416, "cq": 48417, "bdg": 48418, "adona": 48419, "garter": 48420, "womenof": 48421, "scd": 48422, "recognising": 48423, "muna": 48424, "strou": 48425, "signalling": 48426, "laredo": 48427, "hellboy": 48428, "aleksand": 48429, "unavailable": 48430, "pediatric": 48431, "asin": 48432, "meria": 48433, "rishi": 48434, "futurism": 48435, "wye": 48436, "polarized": 48437, "ewe": 48438, "propel": 48439, "informs": 48440, "crease": 48441, "~\"": 48442, "artiston": 48443, "likefor": 48444, "heidelberg": 48445, "erra": 48446, "lifein": 48447, "lenny": 48448, "interrupt": 48449, "coherent": 48450, "caz": 48451, "vickers": 48452, "leveled": 48453, "fbs": 48454, "cabins": 48455, "bummed": 48456, "apostles": 48457, "weh": 48458, "tendon": 48459, "souvenirs": 48460, "infuri": 48461, "pierce": 48462, "asset": 48463, "mlas": 48464, "goth": 48465, "diggin": 48466, "annas": 48467, "ylor": 48468, "thwaite": 48469, "swel": 48470, "panera": 48471, "murderers": 48472, "crooked": 48473, "bsgo": 48474, "acu": 48475, "aon": 48476, "rean": 48477, "oneof": 48478, "kohl": 48479, "bloodh": 48480, "pesticide": 48481, "lostdog": 48482, "flexing": 48483, "ëĤĺ": 48484, "supra": 48485, "eternally": 48486, "ðŁļĻ": 48487, "paolo": 48488, "olan": 48489, "momo": 48490, "iselle": 48491, "captainmarvel": 48492, "slou": 48493, "mistakenly": 48494, "akhilesh": 48495, "mert": 48496, "ilinan": 48497, "buon": 48498, "balkan": 48499, "mirro": 48500, "millen": 48501, "derail": 48502, "damon": 48503, "titi": 48504, "bios": 48505, "redon": 48506, "picard": 48507, "parte": 48508, "ðŁ¤Ł": 48509, "غ": 48510, "sonics": 48511, "firsth": 48512, "ddc": 48513, "vegans": 48514, "turban": 48515, "nigan": 48516, "lottie": 48517, "lyndon": 48518, "starbuck": 48519, "pinkfloyd": 48520, "lifestyles": 48521, "amara": 48522, "ashe": 48523, "rsc": 48524, "vala": 48525, "smer": 48526, "cwgc": 48527, "client": 48528, "buenas": 48529, "jagan": 48530, "coops": 48531, "ðŁijijðŁijij": 48532, "specializes": 48533, "snagged": 48534, "glar": 48535, "bennet": 48536, "wildlifewednesday": 48537, "bowden": 48538, "pik": 48539, "artin": 48540, "emporium": 48541, "arl": 48542, "reba": 48543, "passer": 48544, "disappoints": 48545, "additive": 48546, "âľĬðŁı½": 48547, "bayer": 48548, "missoula": 48549, "haskell": 48550, "commences": 48551, "nix": 48552, "neman": 48553, "exploited": 48554, "plasticsurgery": 48555, "ccd": 48556, "asocial": 48557, "vot": 48558, "siegel": 48559, "froome": 48560, "kapam": 48561, "fara": 48562, "eha": 48563, "probes": 48564, "mwf": 48565, "meeting": 48566, "pbb": 48567, "akins": 48568, "mistletoe": 48569, "kingdomhearts": 48570, "forkids": 48571, "ecr": 48572, "bale": 48573, "escorts": 48574, "adidasoriginals": 48575, "kwa": 48576, "kts": 48577, "halloffame": 48578, "ðŁĺį.": 48579, "wags": 48580, "potted": 48581, "owing": 48582, "honeycomb": 48583, "hefty": 48584, "urology": 48585, "merle": 48586, "bpd": 48587, "stripping": 48588, "reich": 48589, "kstate": 48590, "guay": 48591, "yonge": 48592, "shakti": 48593, "gloom": 48594, "batt": 48595, "sonom": 48596, "nery": 48597, "elba": 48598, "blanks": 48599, "helle": 48600, "triplets": 48601, "bombay": 48602, "akarta": 48603, "abia": 48604, "transmitted": 48605, "rolf": 48606, "jais": 48607, "angularjs": 48608, "fierc": 48609, "mss": 48610, "trace": 48611, "à¥ĩ": 48612, "tombs": 48613, "oldman": 48614, "kombucha": 48615, "fol": 48616, "ehealth": 48617, "cereals": 48618, "arelli": 48619, "inari": 48620, "ðŁĴ©": 48621, "wol": 48622, "liberties": 48623, "fawn": 48624, "affirm": 48625, "nunavut": 48626, "hysterical": 48627, "kdrama": 48628, "artes": 48629, "âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢": 48630, "valentin": 48631, "manslaughter": 48632, "gales": 48633, "eoin": 48634, "energized": 48635, "dels": 48636, "withdraws": 48637, "stles": 48638, "sarcastic": 48639, "ramesh": 48640, "incredibles": 48641, "lockhart": 48642, "yawn": 48643, "ultimatefanlive": 48644, "oooooooooooooooo": 48645, "muen": 48646, "gurudev": 48647, "teer": 48648, "peeling": 48649, "newsnow": 48650, "linguistics": 48651, "directv": 48652, "agend": 48653, "unilever": 48654, "ruger": 48655, "handedly": 48656, "erose": 48657, "limel": 48658, "thec": 48659, "royalties": 48660, "finishers": 48661, "nrg": 48662, "mgt": 48663, "fidget": 48664, "comps": 48665, "bacon": 48666, "aggressively": 48667, "abit": 48668, "châ": 48669, "tarde": 48670, "slugger": 48671, "qanda": 48672, "greening": 48673, "dats": 48674, "enslaved": 48675, "spector": 48676, "oye": 48677, "freef": 48678, "bhand": 48679, "stopbrexit": 48680, "misconceptions": 48681, "cava": 48682, "ðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺį": 48683, "multitasking": 48684, "housel": 48685, "ferreira": 48686, "centime": 48687, "ankles": 48688, "jodh": 48689, "helly": 48690, "frome": 48691, "outtuesday": 48692, "narnia": 48693, "balaji": 48694, "lbloggers": 48695, "jyoti": 48696, "ðŁįĩ": 48697, "lancia": 48698, "capri": 48699, "yap": 48700, "natash": 48701, "downfall": 48702, ".\"âĢĶ": 48703, "î": 48704, "ligament": 48705, "coatings": 48706, "aided": 48707, "hiko": 48708, "falling": 48709, "encrypted": 48710, "yegfood": 48711, "infringement": 48712, "cudi": 48713, "cep": 48714, "ðŁĺįðŁĺĤ": 48715, "trad": 48716, "superrugby": 48717, "edwin": 48718, "whiche": 48719, "vimeo": 48720, "layne": 48721, "invigor": 48722, "hehe": 48723, "dubrovnik": 48724, "bieber": 48725, "utr": 48726, "shaman": 48727, "opers": 48728, "hamill": 48729, "enig": 48730, "dif": 48731, "arum": 48732, "scrapbook": 48733, "minh": 48734, "divergence": 48735, "mckinnon": 48736, "lifetime": 48737, "guterres": 48738, "wille": 48739, "pleas": 48740, "patty": 48741, "micron": 48742, "kz": 48743, "domaine": 48744, "rusher": 48745, "mds": 48746, "chesney": 48747, "screwdriver": 48748, "âģ©,": 48749, "sledge": 48750, "hauer": 48751, "chana": 48752, "stamina": 48753, "sprinkler": 48754, "pln": 48755, "heff": 48756, "bolton": 48757, "omon": 48758, "carrington": 48759, "accordion": 48760, "jorge": 48761, "interception": 48762, "inputs": 48763, "gull": 48764, "transcription": 48765, "vanuatu": 48766, "itical": 48767, "ethos": 48768, "tich": 48769, "spacey": 48770, "peeking": 48771, "umi": 48772, "hager": 48773, "psychotic": 48774, "illian": 48775, "illia": 48776, "bonnaroo": 48777, "anese": 48778, "puc": 48779, "laghateparth": 48780, "enhall": 48781, "economical": 48782, "dredge": 48783, "%-": 48784, "uwe": 48785, "tubular": 48786, "scouncil": 48787, "peasants": 48788, "fler": 48789, "tumbler": 48790, "hep": 48791, "fordham": 48792, "rowley": 48793, "initials": 48794, "evasion": 48795, "ernation": 48796, "plugins": 48797, "cochran": 48798, "cattle": 48799, "acidity": 48800, "ðŁİĬðŁİī": 48801, "regrann": 48802, "jumpman": 48803, "eface": 48804, "xma": 48805, "patriarchy": 48806, "escobar": 48807, "cristian": 48808, "tipton": 48809, "nueva": 48810, "hackney": 48811, "backseat": 48812, "killarney": 48813, "aidan": 48814, "stadion": 48815, "simultaneous": 48816, "idaho": 48817, "aje": 48818, "uth": 48819, "figure": 48820, "clos": 48821, "burk": 48822, "voluntar": 48823, "recite": 48824, "macfarlane": 48825, "curfew": 48826, "boudo": 48827, "wgn": 48828, "stix": 48829, "slap": 48830, "scratched": 48831, "phillip": 48832, "journe": 48833, "expelled": 48834, "waz": 48835, "uke": 48836, "tatiana": 48837, "oue": 48838, "hopp": 48839, "dimitri": 48840, "ðŁĵ£": 48841, "matologist": 48842, "electrifying": 48843, "bluffs": 48844, "billsmafia": 48845, "azcardinals": 48846, "yaa": 48847, "xmas": 48848, "shara": 48849, "rith": 48850, "gills": 48851, "dres": 48852, "barton": 48853, "authorization": 48854, "imperialism": 48855, "homeof": 48856, "todo": 48857, "footpath": 48858, "bandwidth": 48859, "visitspain": 48860, "mohsin": 48861, "erupted": 48862, "miki": 48863, "insignia": 48864, "mikel": 48865, "ssh": 48866, "gera": 48867, "bankholiday": 48868, "awan": 48869, "tweak": 48870, "starcraft": 48871, "eal": 48872, "construction": 48873, "skeletons": 48874, "leep": 48875, "inem": 48876, "barclay": 48877, "shipwreck": 48878, "monsieur": 48879, "yoh": 48880, "ront": 48881, "formative": 48882, "sero": 48883, "lep": 48884, "horseman": 48885, "hoosier": 48886, "hazmat": 48887, "cylinders": 48888, "centi": 48889, "ðŁĴ¥ðŁĴ¥ðŁĴ¥": 48890, "reem": 48891, "naire": 48892, "musically": 48893, "grasshopper": 48894, "estonian": 48895, "terminology": 48896, "romain": 48897, "bloggerrt": 48898, "toxin": 48899, "stance": 48900, "cultivated": 48901, "anast": 48902, "ðŁIJį": 48903, "shimano": 48904, "gopher": 48905, "enei": 48906, "recyclable": 48907, "gamification": 48908, "fightfor": 48909, "cq": 48910, "avocados": 48911, "keys": 48912, "elike": 48913, "glycer": 48914, "shakur": 48915, "mobilization": 48916, "galley": 48917, "explain": 48918, "exchanged": 48919, "peth": 48920, "obedience": 48921, "illage": 48922, "ennis": 48923, "ãĥŀ": 48924, "wiv": 48925, "wallabies": 48926, "maar": 48927, "igers": 48928, "fintech": 48929, "finalized": 48930, "woj": 48931, "meaningless": 48932, "infield": 48933, "onnaise": 48934, "eet": 48935, "bronte": 48936, "passages": 48937, "ðŁij§": 48938, "strickland": 48939, "northernlights": 48940, "lomond": 48941, "htc": 48942, "wray": 48943, "shifter": 48944, "dialog": 48945, "ðŁįį": 48946, ">>>>>>": 48947, "teatime": 48948, "stech": 48949, "sichuan": 48950, "quill": 48951, "franca": 48952, "complementary": 48953, "barrington": 48954, "marcus": 48955, "malam": 48956, "goooo": 48957, "forsa": 48958, "electra": 48959, "afs": 48960, "âĹĨ": 48961, "trife": 48962, "snazzy": 48963, "folia": 48964, "andolan": 48965, "afterdark": 48966, "woodson": 48967, "strade": 48968, "littlest": 48969, "ogun": 48970, "conwy": 48971, "cowards": 48972, "ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ": 48973, "íĬ¸": 48974, "seul": 48975, "murphy": 48976, "dunks": 48977, "kapilshar": 48978, "joachim": 48979, "womack": 48980, "equality": 48981, "averages": 48982, "aine": 48983, "ðŁ¦Ī": 48984, "tacular": 48985, "disability": 48986, "uked": 48987, "midcentury": 48988, "barthol": 48989, "teasers": 48990, "tabern": 48991, "njcaa": 48992, "spout": 48993, "opi": 48994, "kubball": 48995, "blom": 48996, "soar": 48997, "populism": 48998, "methyl": 48999, "ðŁijĬðŁı¼": 49000, "ospre": 49001, "aloils": 49002, "ðŁĵĸ": 49003, "ðŁĮļ": 49004, "xer": 49005, "spilling": 49006, "publica": 49007, "cardam": 49008, "adish": 49009, "sacha": 49010, "pkg": 49011, "buda": 49012, "lyricist": 49013, "ibc": 49014, "grump": 49015, "hover": 49016, "halep": 49017, "antibody": 49018, "anemone": 49019, "âĻ¥âĻ¥âĻ¥âĻ¥": 49020, "mcl": 49021, "lithograph": 49022, "ccu": 49023, "sfest": 49024, "pathic": 49025, "callister": 49026, "ottawa": 49027, "gunsn": 49028, "rutger": 49029, "halibut": 49030, "envision": 49031, "differentiate": 49032, "ðŁļĢðŁļĢ": 49033, "piran": 49034, "latel": 49035, "ucn": 49036, "troubad": 49037, "raine": 49038, "fiercely": 49039, "learnenglish": 49040, "lease": 49041, "wexmondays": 49042, "emit": 49043, "drayton": 49044, "burrell": 49045, "scubadiving": 49046, "holler": 49047, "dru": 49048, "clocked": 49049, "wral": 49050, "apro": 49051, "translucent": 49052, "wbo": 49053, "patriarch": 49054, "moja": 49055, "lannister": 49056, "fishery": 49057, "nederland": 49058, "mildly": 49059, "mirai": 49060, "mako": 49061, "jap": 49062, "ðŁĺ©ðŁĺ©ðŁĺ©": 49063, "prostatec": 49064, "panna": 49065, "arama": 49066, "undertaking": 49067, "tompkins": 49068, "neop": 49069, "solids": 49070, "savoury": 49071, "eames": 49072, "cutlery": 49073, "woodbridge": 49074, "steamer": 49075, "rizzo": 49076, "wildcat": 49077, "ratna": 49078, "laminated": 49079, "kineni": 49080, "jalap": 49081, "aides": 49082, "acknowledges": 49083, "?!?!?!": 49084, "!ðŁİī": 49085, "wafc": 49086, "maggio": 49087, "haves": 49088, "darje": 49089, "ofi": 49090, "gril": 49091, "vasi": 49092, "brux": 49093, "mohd": 49094, "fakespeare": 49095, "arnold": 49096, "rmb": 49097, "forbe": 49098, "walleye": 49099, "rodi": 49100, "therapeutics": 49101, "strategi": 49102, "obste": 49103, "mudder": 49104, "downloadable": 49105, "ddings": 49106, "dca": 49107, "asiangames": 49108, "campeon": 49109, "appropriation": 49110, "thcentury": 49111, "ramatta": 49112, "draped": 49113, "bullion": 49114, "muc": 49115, "onex": 49116, "segreg": 49117, "ophelia": 49118, "bodily": 49119, "âĿ¤ðŁĺį": 49120, "wizar": 49121, "teased": 49122, "ademy": 49123, "toid": 49124, "sura": 49125, "lazarus": 49126, "snickers": 49127, "mase": 49128, "loh": 49129, "bowed": 49130, "biblio": 49131, "xchange": 49132, "harlan": 49133, "ghoshal": 49134, "flavorful": 49135, "bhagat": 49136, "allez": 49137, "whichever": 49138, "tenstein": 49139, "discer": 49140, "organiser": 49141, "mtg": 49142, "dreamliner": 49143, "tse": 49144, "hokkaido": 49145, "mok": 49146, "indulgent": 49147, "hickman": 49148, "blinded": 49149, "alyn": 49150, "aaaah": 49151, "spool": 49152, "loughborough": 49153, "interpret": 49154, "etv": 49155, "aristotle": 49156, "optimizing": 49157, "avicii": 49158, "madurai": 49159, "juli": 49160, "nawaz": 49161, "matchups": 49162, "abide": 49163, "painting": 49164, "welling": 49165, "veli": 49166, "octagon": 49167, "inscribed": 49168, "poking": 49169, "placer": 49170, "lifecycle": 49171, "kilig": 49172, "gsp": 49173, "elives": 49174, "clements": 49175, "nasheed": 49176, "mesut": 49177, "incarcerated": 49178, "distilled": 49179, "walang": 49180, "delicacy": 49181, "delgado": 49182, "chez": 49183, "chita": 49184, "adero": 49185, "tux": 49186, "patil": 49187, "odo": 49188, "abhcosmetics": 49189, "tvc": 49190, "pbc": 49191, "inaccurate": 49192, "hardworkpaysoff": 49193, "baller": 49194, "quotation": 49195, "merchandising": 49196, "gastri": 49197, "defenses": 49198, "drogba": 49199, "bexhill": 49200, "bankno": 49201, "winona": 49202, "sieg": 49203, "pgs": 49204, "hahahha": 49205, "aguchi": 49206, "subram": 49207, "miracle": 49208, "desch": 49209, "libre": 49210, "bacher": 49211, "entine": 49212, "bbcradi": 49213, "loudest": 49214, "rps": 49215, "pierc": 49216, "fryer": 49217, "stormtrooper": 49218, "rafaelnadal": 49219, "pasco": 49220, "exhaustion": 49221, "epiconetsy": 49222, "rctid": 49223, "kellie": 49224, "gaines": 49225, "dbz": 49226, "smriti": 49227, "sbridge": 49228, "limited": 49229, "claw": 49230, "technical": 49231, "biographical": 49232, "adored": 49233, "ะ": 49234, "exclude": 49235, "acadia": 49236, "keyboards": 49237, "furman": 49238, "soca": 49239, "suru": 49240, "nips": 49241, "swaps": 49242, "serverless": 49243, "rune": 49244, "puffy": 49245, "northampton": 49246, "nishings": 49247, "hender": 49248, "cartridges": 49249, "gunshot": 49250, "ðŁĵ¹": 49251, "filament": 49252, "respondents": 49253, "peyton": 49254, "mountaineer": 49255, "merging": 49256, "lifespan": 49257, "intimidation": 49258, "pafc": 49259, "nlwx": 49260, "expansive": 49261, "purr": 49262, "fck": 49263, "cae": 49264, "atti": 49265, "telethon": 49266, "sohn": 49267, "mendel": 49268, "lopes": 49269, "dori": 49270, "unbroken": 49271, "tered": 49272, "tastings": 49273, "inactive": 49274, "disintegr": 49275, "tassel": 49276, "sharethe": 49277, "piano": 49278, "islay": 49279, "airspace": 49280, "zawa": 49281, "ricciardo": 49282, "mington": 49283, "fresher": 49284, "curry": 49285, "revs": 49286, "pharoah": 49287, "hmv": 49288, "exhilarating": 49289, "whoo": 49290, "linkin": 49291, "krispy": 49292, "competency": 49293, "stewards": 49294, "nebu": 49295, "katsu": 49296, "admins": 49297, "bazar": 49298, "asar": 49299, "givingback": 49300, "ssummit": 49301, "songz": 49302, "linus": 49303, "rajkumar": 49304, "farmington": 49305, "fantasia": 49306, "ðŁĺ´ðŁĺ´": 49307, "sobri": 49308, "lisse": 49309, "barrymore": 49310, "prism": 49311, "blob": 49312, "senew": 49313, "monoxide": 49314, "expire": 49315, "eighteen": 49316, "dipper": 49317, "xiao": 49318, "kilt": 49319, "hinch": 49320, "bbcsport": 49321, "bamboo": 49322, "pter": 49323, "exal": 49324, "ðŁ¦ĭ": 49325, "hamlin": 49326, "expeditions": 49327, "stargazing": 49328, "foodsecurity": 49329, "wylie": 49330, "ulf": 49331, "stingly": 49332, "onstorm": 49333, "loeb": 49334, "broome": 49335, "bnha": 49336, "pancreatic": 49337, "elive": 49338, "!!!!!!!!!!!": 49339, "therapper": 49340, "orthopedic": 49341, "avengersendgame": 49342, "antitrust": 49343, "ìļ°": 49344, "gote": 49345, "omd": 49346, "offside": 49347, "gyllen": 49348, "wineries": 49349, "whitewater": 49350, "adl": 49351, "lupita": 49352, "exceeds": 49353, "consisted": 49354, "chewbacca": 49355, "ashleigh": 49356, "nhljets": 49357, "issan": 49358, "shld": 49359, "hayat": 49360, "cranberries": 49361, "ðŁ¤ĺðŁı½": 49362, "rockthe": 49363, "springtraining": 49364, "fallout": 49365, "dairyfree": 49366, "waj": 49367, "undecided": 49368, "sown": 49369, "rcn": 49370, "northwales": 49371, "httr": 49372, "fumble": 49373, "dits": 49374, "compelled": 49375, "populist": 49376, "minted": 49377, "blanchett": 49378, ".''": 49379, "propulsion": 49380, "milla": 49381, "auberg": 49382, "hertz": 49383, "hta": 49384, "udaipur": 49385, "serendipity": 49386, "aztecs": 49387, "alsace": 49388, "ðŁIJij": 49389, "lun": 49390, "shoes": 49391, "charli": 49392, "garza": 49393, "ðŁĴŁ": 49394, "probiotics": 49395, "foxtv": 49396, "olis": 49397, "miff": 49398, "localized": 49399, "diffuser": 49400, "sigue": 49401, "funko": 49402, "rendous": 49403, "ðŁĴij": 49404, "jekyll": 49405, "<|startoftext|>": 49406, "<|endoftext|>": 49407} \ No newline at end of file diff --git a/annotator/color/__init__.py b/annotator/color/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..65799a2a83efd18dc556600c99d43292845aa6f2 --- /dev/null +++ b/annotator/color/__init__.py @@ -0,0 +1,20 @@ +import cv2 + +def cv2_resize_shortest_edge(image, size): + h, w = image.shape[:2] + if h < w: + new_h = size + new_w = int(round(w / h * size)) + else: + new_w = size + new_h = int(round(h / w * size)) + resized_image = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_AREA) + return resized_image + +def apply_color(img, res=512): + img = cv2_resize_shortest_edge(img, res) + h, w = img.shape[:2] + + input_img_color = cv2.resize(img, (w//64, h//64), interpolation=cv2.INTER_CUBIC) + input_img_color = cv2.resize(input_img_color, (w, h), interpolation=cv2.INTER_NEAREST) + return input_img_color \ No newline at end of file diff --git a/annotator/hed/__init__.py b/annotator/hed/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..39d8591a1a38fa535456e8693bdfe5a371e37fac --- /dev/null +++ b/annotator/hed/__init__.py @@ -0,0 +1,82 @@ +# This is an improved version and model of HED edge detection with Apache License, Version 2.0. +# Please use this implementation in your products +# This implementation may produce slightly different results from Saining Xie's official implementations, +# but it generates smoother edges and is more suitable for ControlNet as well as other image-to-image translations. +# Different from official models and other implementations, this is an RGB-input model (rather than BGR) +# and in this way it works better for gradio's RGB protocol + +import os +import cv2 +import torch +import numpy as np + +from einops import rearrange +from annotator.util import annotator_ckpts_path, safe_step + + +class DoubleConvBlock(torch.nn.Module): + def __init__(self, input_channel, output_channel, layer_number): + super().__init__() + self.convs = torch.nn.Sequential() + self.convs.append(torch.nn.Conv2d(in_channels=input_channel, out_channels=output_channel, kernel_size=(3, 3), stride=(1, 1), padding=1)) + for i in range(1, layer_number): + self.convs.append(torch.nn.Conv2d(in_channels=output_channel, out_channels=output_channel, kernel_size=(3, 3), stride=(1, 1), padding=1)) + self.projection = torch.nn.Conv2d(in_channels=output_channel, out_channels=1, kernel_size=(1, 1), stride=(1, 1), padding=0) + + def __call__(self, x, down_sampling=False): + h = x + if down_sampling: + h = torch.nn.functional.max_pool2d(h, kernel_size=(2, 2), stride=(2, 2)) + for conv in self.convs: + h = conv(h) + h = torch.nn.functional.relu(h) + return h, self.projection(h) + + +class ControlNetHED_Apache2(torch.nn.Module): + def __init__(self): + super().__init__() + self.norm = torch.nn.Parameter(torch.zeros(size=(1, 3, 1, 1))) + self.block1 = DoubleConvBlock(input_channel=3, output_channel=64, layer_number=2) + self.block2 = DoubleConvBlock(input_channel=64, output_channel=128, layer_number=2) + self.block3 = DoubleConvBlock(input_channel=128, output_channel=256, layer_number=3) + self.block4 = DoubleConvBlock(input_channel=256, output_channel=512, layer_number=3) + self.block5 = DoubleConvBlock(input_channel=512, output_channel=512, layer_number=3) + + def __call__(self, x): + h = x - self.norm + h, projection1 = self.block1(h) + h, projection2 = self.block2(h, down_sampling=True) + h, projection3 = self.block3(h, down_sampling=True) + h, projection4 = self.block4(h, down_sampling=True) + h, projection5 = self.block5(h, down_sampling=True) + return projection1, projection2, projection3, projection4, projection5 + + +class HEDdetector: + def __init__(self): + remote_model_path = "https://huggingface.co./lllyasviel/Annotators/resolve/main/ControlNetHED.pth" + modelpath = os.path.join(annotator_ckpts_path, "ControlNetHED.pth") + if not os.path.exists(modelpath): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(remote_model_path, model_dir=annotator_ckpts_path) +# self.netNetwork = ControlNetHED_Apache2().float().cuda().eval() + self.netNetwork = ControlNetHED_Apache2().float().cpu().eval() + self.netNetwork.load_state_dict(torch.load(modelpath, map_location=torch.device('cpu'))) + + def __call__(self, input_image, safe=False): + assert input_image.ndim == 3 + H, W, C = input_image.shape + with torch.no_grad(): +# image_hed = torch.from_numpy(input_image.copy()).float().cuda() + image_hed = torch.from_numpy(input_image.copy()).float().cpu() + image_hed = rearrange(image_hed, 'h w c -> 1 c h w') + edges = self.netNetwork(image_hed) + edges = [e.detach().cpu().numpy().astype(np.float32)[0, 0] for e in edges] + edges = [cv2.resize(e, (W, H), interpolation=cv2.INTER_LINEAR) for e in edges] + edges = np.stack(edges, axis=2) + edge = 1 / (1 + np.exp(-np.mean(edges, axis=2).astype(np.float64))) + if safe: + edge = safe_step(edge) + edge = (edge * 255.0).clip(0, 255).astype(np.uint8) + return edge diff --git a/annotator/keypose/__init__.py b/annotator/keypose/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..aa3dfa2e1589f22471411b3180ccaf870f147d73 --- /dev/null +++ b/annotator/keypose/__init__.py @@ -0,0 +1,212 @@ +import numpy as np +import cv2 +import torch + +import os +from modules import devices +from annotator.annotator_path import models_path + +import mmcv +from mmdet.apis import inference_detector, init_detector +from mmpose.apis import inference_top_down_pose_model +from mmpose.apis import init_pose_model, process_mmdet_results, vis_pose_result + + +def preprocessing(image, device): + # Resize + scale = 640 / max(image.shape[:2]) + image = cv2.resize(image, dsize=None, fx=scale, fy=scale) + raw_image = image.astype(np.uint8) + + # Subtract mean values + image = image.astype(np.float32) + image -= np.array( + [ + float(104.008), + float(116.669), + float(122.675), + ] + ) + + # Convert to torch.Tensor and add "batch" axis + image = torch.from_numpy(image.transpose(2, 0, 1)).float().unsqueeze(0) + image = image.to(device) + + return image, raw_image + + +def imshow_keypoints(img, + pose_result, + skeleton=None, + kpt_score_thr=0.1, + pose_kpt_color=None, + pose_link_color=None, + radius=4, + thickness=1): + """Draw keypoints and links on an image. + Args: + img (ndarry): The image to draw poses on. + pose_result (list[kpts]): The poses to draw. Each element kpts is + a set of K keypoints as an Kx3 numpy.ndarray, where each + keypoint is represented as x, y, score. + kpt_score_thr (float, optional): Minimum score of keypoints + to be shown. Default: 0.3. + pose_kpt_color (np.array[Nx3]`): Color of N keypoints. If None, + the keypoint will not be drawn. + pose_link_color (np.array[Mx3]): Color of M links. If None, the + links will not be drawn. + thickness (int): Thickness of lines. + """ + + img_h, img_w, _ = img.shape + img = np.zeros(img.shape) + + for idx, kpts in enumerate(pose_result): + if idx > 1: + continue + kpts = kpts['keypoints'] + # print(kpts) + kpts = np.array(kpts, copy=False) + + # draw each point on image + if pose_kpt_color is not None: + assert len(pose_kpt_color) == len(kpts) + + for kid, kpt in enumerate(kpts): + x_coord, y_coord, kpt_score = int(kpt[0]), int(kpt[1]), kpt[2] + + if kpt_score < kpt_score_thr or pose_kpt_color[kid] is None: + # skip the point that should not be drawn + continue + + color = tuple(int(c) for c in pose_kpt_color[kid]) + cv2.circle(img, (int(x_coord), int(y_coord)), + radius, color, -1) + + # draw links + if skeleton is not None and pose_link_color is not None: + assert len(pose_link_color) == len(skeleton) + + for sk_id, sk in enumerate(skeleton): + pos1 = (int(kpts[sk[0], 0]), int(kpts[sk[0], 1])) + pos2 = (int(kpts[sk[1], 0]), int(kpts[sk[1], 1])) + + if (pos1[0] <= 0 or pos1[0] >= img_w or pos1[1] <= 0 or pos1[1] >= img_h or pos2[0] <= 0 + or pos2[0] >= img_w or pos2[1] <= 0 or pos2[1] >= img_h or kpts[sk[0], 2] < kpt_score_thr + or kpts[sk[1], 2] < kpt_score_thr or pose_link_color[sk_id] is None): + # skip the link that should not be drawn + continue + color = tuple(int(c) for c in pose_link_color[sk_id]) + cv2.line(img, pos1, pos2, color, thickness=thickness) + + return img + + +human_det, pose_model = None, None +det_model_path = "https://download.openmmlab.com/mmdetection/v2.0/faster_rcnn/faster_rcnn_r50_fpn_1x_coco/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth" +pose_model_path = "https://download.openmmlab.com/mmpose/top_down/hrnet/hrnet_w48_coco_256x192-b9e0b3ab_20200708.pth" + +modeldir = os.path.join(models_path, "keypose") +old_modeldir = os.path.dirname(os.path.realpath(__file__)) + +det_config = 'faster_rcnn_r50_fpn_coco.py' +pose_config = 'hrnet_w48_coco_256x192.py' + +det_checkpoint = 'faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth' +pose_checkpoint = 'hrnet_w48_coco_256x192-b9e0b3ab_20200708.pth' +det_cat_id = 1 +bbox_thr = 0.2 + +skeleton = [ + [15, 13], [13, 11], [16, 14], [14, 12], [11, 12], [5, 11], [6, 12], [5, 6], [5, 7], [6, 8], + [7, 9], [8, 10], + [1, 2], [0, 1], [0, 2], [1, 3], [2, 4], [3, 5], [4, 6] +] + +pose_kpt_color = [ + [51, 153, 255], [51, 153, 255], [51, 153, 255], [51, 153, 255], [51, 153, 255], + [0, 255, 0], + [255, 128, 0], [0, 255, 0], [255, 128, 0], [0, 255, 0], [255, 128, 0], [0, 255, 0], + [255, 128, 0], + [0, 255, 0], [255, 128, 0], [0, 255, 0], [255, 128, 0] +] + +pose_link_color = [ + [0, 255, 0], [0, 255, 0], [255, 128, 0], [255, 128, 0], + [51, 153, 255], [51, 153, 255], [51, 153, 255], [51, 153, 255], [0, 255, 0], + [255, 128, 0], + [0, 255, 0], [255, 128, 0], [51, 153, 255], [51, 153, 255], [51, 153, 255], + [51, 153, 255], + [51, 153, 255], [51, 153, 255], [51, 153, 255] +] + +def find_download_model(checkpoint, remote_path): + modelpath = os.path.join(modeldir, checkpoint) + old_modelpath = os.path.join(old_modeldir, checkpoint) + + if os.path.exists(old_modelpath): + modelpath = old_modelpath + elif not os.path.exists(modelpath): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(remote_path, model_dir=modeldir) + + return modelpath + +def apply_keypose(input_image): + global human_det, pose_model + if netNetwork is None: + det_model_local = find_download_model(det_checkpoint, det_model_path) + hrnet_model_local = find_download_model(pose_checkpoint, pose_model_path) + det_config_mmcv = mmcv.Config.fromfile(det_config) + pose_config_mmcv = mmcv.Config.fromfile(pose_config) + human_det = init_detector(det_config_mmcv, det_model_local, device=devices.get_device_for("controlnet")) + pose_model = init_pose_model(pose_config_mmcv, hrnet_model_local, device=devices.get_device_for("controlnet")) + + assert input_image.ndim == 3 + input_image = input_image.copy() + with torch.no_grad(): + image = torch.from_numpy(input_image).float().to(devices.get_device_for("controlnet")) + image = image / 255.0 + mmdet_results = inference_detector(human_det, image) + + # keep the person class bounding boxes. + person_results = process_mmdet_results(mmdet_results, det_cat_id) + + return_heatmap = False + dataset = pose_model.cfg.data['test']['type'] + + # e.g. use ('backbone', ) to return backbone feature + output_layer_names = None + pose_results, _ = inference_top_down_pose_model( + pose_model, + image, + person_results, + bbox_thr=bbox_thr, + format='xyxy', + dataset=dataset, + dataset_info=None, + return_heatmap=return_heatmap, + outputs=output_layer_names + ) + + im_keypose_out = imshow_keypoints( + image, + pose_results, + skeleton=skeleton, + pose_kpt_color=pose_kpt_color, + pose_link_color=pose_link_color, + radius=2, + thickness=2 + ) + im_keypose_out = im_keypose_out.astype(np.uint8) + + # image_hed = rearrange(image_hed, 'h w c -> 1 c h w') + # edge = netNetwork(image_hed)[0] + # edge = (edge.cpu().numpy() * 255.0).clip(0, 255).astype(np.uint8) + return im_keypose_out + + +def unload_hed_model(): + global netNetwork + if netNetwork is not None: + netNetwork.cpu() diff --git a/annotator/keypose/faster_rcnn_r50_fpn_coco.py b/annotator/keypose/faster_rcnn_r50_fpn_coco.py new file mode 100644 index 0000000000000000000000000000000000000000..a9ad9528b22163ae7ce1390375b69227fd6eafd9 --- /dev/null +++ b/annotator/keypose/faster_rcnn_r50_fpn_coco.py @@ -0,0 +1,182 @@ +checkpoint_config = dict(interval=1) +# yapf:disable +log_config = dict( + interval=50, + hooks=[ + dict(type='TextLoggerHook'), + # dict(type='TensorboardLoggerHook') + ]) +# yapf:enable +dist_params = dict(backend='nccl') +log_level = 'INFO' +load_from = None +resume_from = None +workflow = [('train', 1)] +# optimizer +optimizer = dict(type='SGD', lr=0.02, momentum=0.9, weight_decay=0.0001) +optimizer_config = dict(grad_clip=None) +# learning policy +lr_config = dict( + policy='step', + warmup='linear', + warmup_iters=500, + warmup_ratio=0.001, + step=[8, 11]) +total_epochs = 12 + +model = dict( + type='FasterRCNN', + pretrained='torchvision://resnet50', + backbone=dict( + type='ResNet', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + frozen_stages=1, + norm_cfg=dict(type='BN', requires_grad=True), + norm_eval=True, + style='pytorch'), + neck=dict( + type='FPN', + in_channels=[256, 512, 1024, 2048], + out_channels=256, + num_outs=5), + rpn_head=dict( + type='RPNHead', + in_channels=256, + feat_channels=256, + anchor_generator=dict( + type='AnchorGenerator', + scales=[8], + ratios=[0.5, 1.0, 2.0], + strides=[4, 8, 16, 32, 64]), + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[.0, .0, .0, .0], + target_stds=[1.0, 1.0, 1.0, 1.0]), + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0)), + roi_head=dict( + type='StandardRoIHead', + bbox_roi_extractor=dict( + type='SingleRoIExtractor', + roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0), + out_channels=256, + featmap_strides=[4, 8, 16, 32]), + bbox_head=dict( + type='Shared2FCBBoxHead', + in_channels=256, + fc_out_channels=1024, + roi_feat_size=7, + num_classes=80, + bbox_coder=dict( + type='DeltaXYWHBBoxCoder', + target_means=[0., 0., 0., 0.], + target_stds=[0.1, 0.1, 0.2, 0.2]), + reg_class_agnostic=False, + loss_cls=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), + loss_bbox=dict(type='L1Loss', loss_weight=1.0))), + # model training and testing settings + train_cfg=dict( + rpn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.7, + neg_iou_thr=0.3, + min_pos_iou=0.3, + match_low_quality=True, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=256, + pos_fraction=0.5, + neg_pos_ub=-1, + add_gt_as_proposals=False), + allowed_border=-1, + pos_weight=-1, + debug=False), + rpn_proposal=dict( + nms_pre=2000, + max_per_img=1000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=dict( + assigner=dict( + type='MaxIoUAssigner', + pos_iou_thr=0.5, + neg_iou_thr=0.5, + min_pos_iou=0.5, + match_low_quality=False, + ignore_iof_thr=-1), + sampler=dict( + type='RandomSampler', + num=512, + pos_fraction=0.25, + neg_pos_ub=-1, + add_gt_as_proposals=True), + pos_weight=-1, + debug=False)), + test_cfg=dict( + rpn=dict( + nms_pre=1000, + max_per_img=1000, + nms=dict(type='nms', iou_threshold=0.7), + min_bbox_size=0), + rcnn=dict( + score_thr=0.05, + nms=dict(type='nms', iou_threshold=0.5), + max_per_img=100) + # soft-nms is also supported for rcnn testing + # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05) + )) + +dataset_type = 'CocoDataset' +data_root = 'data/coco' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', with_bbox=True), + dict(type='Resize', img_scale=(1333, 800), keep_ratio=True), + dict(type='RandomFlip', flip_ratio=0.5), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(1333, 800), + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=2, + workers_per_gpu=2, + train=dict( + type=dataset_type, + ann_file=f'{data_root}/annotations/instances_train2017.json', + img_prefix=f'{data_root}/train2017/', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + ann_file=f'{data_root}/annotations/instances_val2017.json', + img_prefix=f'{data_root}/val2017/', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + ann_file=f'{data_root}/annotations/instances_val2017.json', + img_prefix=f'{data_root}/val2017/', + pipeline=test_pipeline)) +evaluation = dict(interval=1, metric='bbox') diff --git a/annotator/keypose/hrnet_w48_coco_256x192.py b/annotator/keypose/hrnet_w48_coco_256x192.py new file mode 100644 index 0000000000000000000000000000000000000000..9755e6773cd3a8c0d2ac684c612d716cfd44b0ca --- /dev/null +++ b/annotator/keypose/hrnet_w48_coco_256x192.py @@ -0,0 +1,169 @@ +# _base_ = [ +# '../../../../_base_/default_runtime.py', +# '../../../../_base_/datasets/coco.py' +# ] +evaluation = dict(interval=10, metric='mAP', save_best='AP') + +optimizer = dict( + type='Adam', + lr=5e-4, +) +optimizer_config = dict(grad_clip=None) +# learning policy +lr_config = dict( + policy='step', + warmup='linear', + warmup_iters=500, + warmup_ratio=0.001, + step=[170, 200]) +total_epochs = 210 +channel_cfg = dict( + num_output_channels=17, + dataset_joints=17, + dataset_channel=[ + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + ], + inference_channel=[ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 + ]) + +# model settings +model = dict( + type='TopDown', + pretrained='https://download.openmmlab.com/mmpose/' + 'pretrain_models/hrnet_w48-8ef0771d.pth', + backbone=dict( + type='HRNet', + in_channels=3, + extra=dict( + stage1=dict( + num_modules=1, + num_branches=1, + block='BOTTLENECK', + num_blocks=(4, ), + num_channels=(64, )), + stage2=dict( + num_modules=1, + num_branches=2, + block='BASIC', + num_blocks=(4, 4), + num_channels=(48, 96)), + stage3=dict( + num_modules=4, + num_branches=3, + block='BASIC', + num_blocks=(4, 4, 4), + num_channels=(48, 96, 192)), + stage4=dict( + num_modules=3, + num_branches=4, + block='BASIC', + num_blocks=(4, 4, 4, 4), + num_channels=(48, 96, 192, 384))), + ), + keypoint_head=dict( + type='TopdownHeatmapSimpleHead', + in_channels=48, + out_channels=channel_cfg['num_output_channels'], + num_deconv_layers=0, + extra=dict(final_conv_kernel=1, ), + loss_keypoint=dict(type='JointsMSELoss', use_target_weight=True)), + train_cfg=dict(), + test_cfg=dict( + flip_test=True, + post_process='default', + shift_heatmap=True, + modulate_kernel=11)) + +data_cfg = dict( + image_size=[192, 256], + heatmap_size=[48, 64], + num_output_channels=channel_cfg['num_output_channels'], + num_joints=channel_cfg['dataset_joints'], + dataset_channel=channel_cfg['dataset_channel'], + inference_channel=channel_cfg['inference_channel'], + soft_nms=False, + nms_thr=1.0, + oks_thr=0.9, + vis_thr=0.2, + use_gt_bbox=False, + det_bbox_thr=0.0, + bbox_file='data/coco/person_detection_results/' + 'COCO_val2017_detections_AP_H_56_person.json', +) + +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='TopDownGetBboxCenterScale', padding=1.25), + dict(type='TopDownRandomShiftBboxCenter', shift_factor=0.16, prob=0.3), + dict(type='TopDownRandomFlip', flip_prob=0.5), + dict( + type='TopDownHalfBodyTransform', + num_joints_half_body=8, + prob_half_body=0.3), + dict( + type='TopDownGetRandomScaleRotation', rot_factor=40, scale_factor=0.5), + dict(type='TopDownAffine'), + dict(type='ToTensor'), + dict( + type='NormalizeTensor', + mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]), + dict(type='TopDownGenerateTarget', sigma=2), + dict( + type='Collect', + keys=['img', 'target', 'target_weight'], + meta_keys=[ + 'image_file', 'joints_3d', 'joints_3d_visible', 'center', 'scale', + 'rotation', 'bbox_score', 'flip_pairs' + ]), +] + +val_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='TopDownGetBboxCenterScale', padding=1.25), + dict(type='TopDownAffine'), + dict(type='ToTensor'), + dict( + type='NormalizeTensor', + mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]), + dict( + type='Collect', + keys=['img'], + meta_keys=[ + 'image_file', 'center', 'scale', 'rotation', 'bbox_score', + 'flip_pairs' + ]), +] + +test_pipeline = val_pipeline + +data_root = 'data/coco' +data = dict( + samples_per_gpu=32, + workers_per_gpu=2, + val_dataloader=dict(samples_per_gpu=32), + test_dataloader=dict(samples_per_gpu=32), + train=dict( + type='TopDownCocoDataset', + ann_file=f'{data_root}/annotations/person_keypoints_train2017.json', + img_prefix=f'{data_root}/train2017/', + data_cfg=data_cfg, + pipeline=train_pipeline, + dataset_info={{_base_.dataset_info}}), + val=dict( + type='TopDownCocoDataset', + ann_file=f'{data_root}/annotations/person_keypoints_val2017.json', + img_prefix=f'{data_root}/val2017/', + data_cfg=data_cfg, + pipeline=val_pipeline, + dataset_info={{_base_.dataset_info}}), + test=dict( + type='TopDownCocoDataset', + ann_file=f'{data_root}/annotations/person_keypoints_val2017.json', + img_prefix=f'{data_root}/val2017/', + data_cfg=data_cfg, + pipeline=test_pipeline, + dataset_info={{_base_.dataset_info}}), +) diff --git a/annotator/lama/__init__.py b/annotator/lama/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a7784a3837d8454fe8991d7f7a4341331d8b1f0d --- /dev/null +++ b/annotator/lama/__init__.py @@ -0,0 +1,58 @@ +# https://github.com/advimman/lama + +import yaml +import torch +from omegaconf import OmegaConf +import numpy as np + +from einops import rearrange +import os +from modules import devices +from annotator.annotator_path import models_path +from annotator.lama.saicinpainting.training.trainers import load_checkpoint + + +class LamaInpainting: + model_dir = os.path.join(models_path, "lama") + + def __init__(self): + self.model = None + self.device = devices.get_device_for("controlnet") + + def load_model(self): + remote_model_path = "https://huggingface.co./lllyasviel/Annotators/resolve/main/ControlNetLama.pth" + modelpath = os.path.join(self.model_dir, "ControlNetLama.pth") + if not os.path.exists(modelpath): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(remote_model_path, model_dir=self.model_dir) + config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config.yaml') + cfg = yaml.safe_load(open(config_path, 'rt')) + cfg = OmegaConf.create(cfg) + cfg.training_model.predict_only = True + cfg.visualizer.kind = 'noop' + self.model = load_checkpoint(cfg, os.path.abspath(modelpath), strict=False, map_location='cpu') + self.model = self.model.to(self.device) + self.model.eval() + + def unload_model(self): + if self.model is not None: + self.model.cpu() + + def __call__(self, input_image): + if self.model is None: + self.load_model() + self.model.to(self.device) + color = np.ascontiguousarray(input_image[:, :, 0:3]).astype(np.float32) / 255.0 + mask = np.ascontiguousarray(input_image[:, :, 3:4]).astype(np.float32) / 255.0 + with torch.no_grad(): + color = torch.from_numpy(color).float().to(self.device) + mask = torch.from_numpy(mask).float().to(self.device) + mask = (mask > 0.5).float() + color = color * (1 - mask) + image_feed = torch.cat([color, mask], dim=2) + image_feed = rearrange(image_feed, 'h w c -> 1 c h w') + result = self.model(image_feed)[0] + result = rearrange(result, 'c h w -> h w c') + result = result * mask + color * (1 - mask) + result *= 255.0 + return result.detach().cpu().numpy().clip(0, 255).astype(np.uint8) diff --git a/annotator/lama/config.yaml b/annotator/lama/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55fd91b5bcacd654e3045a2331e9c186818e6edc --- /dev/null +++ b/annotator/lama/config.yaml @@ -0,0 +1,157 @@ +run_title: b18_ffc075_batch8x15 +training_model: + kind: default + visualize_each_iters: 1000 + concat_mask: true + store_discr_outputs_for_vis: true +losses: + l1: + weight_missing: 0 + weight_known: 10 + perceptual: + weight: 0 + adversarial: + kind: r1 + weight: 10 + gp_coef: 0.001 + mask_as_fake_target: true + allow_scale_mask: true + feature_matching: + weight: 100 + resnet_pl: + weight: 30 + weights_path: ${env:TORCH_HOME} + +optimizers: + generator: + kind: adam + lr: 0.001 + discriminator: + kind: adam + lr: 0.0001 +visualizer: + key_order: + - image + - predicted_image + - discr_output_fake + - discr_output_real + - inpainted + rescale_keys: + - discr_output_fake + - discr_output_real + kind: directory + outdir: /group-volume/User-Driven-Content-Generation/r.suvorov/inpainting/experiments/r.suvorov_2021-04-30_14-41-12_train_simple_pix2pix2_gap_sdpl_novgg_large_b18_ffc075_batch8x15/samples +location: + data_root_dir: /group-volume/User-Driven-Content-Generation/datasets/inpainting_data_root_large + out_root_dir: /group-volume/User-Driven-Content-Generation/${env:USER}/inpainting/experiments + tb_dir: /group-volume/User-Driven-Content-Generation/${env:USER}/inpainting/tb_logs +data: + batch_size: 15 + val_batch_size: 2 + num_workers: 3 + train: + indir: ${location.data_root_dir}/train + out_size: 256 + mask_gen_kwargs: + irregular_proba: 1 + irregular_kwargs: + max_angle: 4 + max_len: 200 + max_width: 100 + max_times: 5 + min_times: 1 + box_proba: 1 + box_kwargs: + margin: 10 + bbox_min_size: 30 + bbox_max_size: 150 + max_times: 3 + min_times: 1 + segm_proba: 0 + segm_kwargs: + confidence_threshold: 0.5 + max_object_area: 0.5 + min_mask_area: 0.07 + downsample_levels: 6 + num_variants_per_mask: 1 + rigidness_mode: 1 + max_foreground_coverage: 0.3 + max_foreground_intersection: 0.7 + max_mask_intersection: 0.1 + max_hidden_area: 0.1 + max_scale_change: 0.25 + horizontal_flip: true + max_vertical_shift: 0.2 + position_shuffle: true + transform_variant: distortions + dataloader_kwargs: + batch_size: ${data.batch_size} + shuffle: true + num_workers: ${data.num_workers} + val: + indir: ${location.data_root_dir}/val + img_suffix: .png + dataloader_kwargs: + batch_size: ${data.val_batch_size} + shuffle: false + num_workers: ${data.num_workers} + visual_test: + indir: ${location.data_root_dir}/korean_test + img_suffix: _input.png + pad_out_to_modulo: 32 + dataloader_kwargs: + batch_size: 1 + shuffle: false + num_workers: ${data.num_workers} +generator: + kind: ffc_resnet + input_nc: 4 + output_nc: 3 + ngf: 64 + n_downsampling: 3 + n_blocks: 18 + add_out_act: sigmoid + init_conv_kwargs: + ratio_gin: 0 + ratio_gout: 0 + enable_lfu: false + downsample_conv_kwargs: + ratio_gin: ${generator.init_conv_kwargs.ratio_gout} + ratio_gout: ${generator.downsample_conv_kwargs.ratio_gin} + enable_lfu: false + resnet_conv_kwargs: + ratio_gin: 0.75 + ratio_gout: ${generator.resnet_conv_kwargs.ratio_gin} + enable_lfu: false +discriminator: + kind: pix2pixhd_nlayer + input_nc: 3 + ndf: 64 + n_layers: 4 +evaluator: + kind: default + inpainted_key: inpainted + integral_kind: ssim_fid100_f1 +trainer: + kwargs: + gpus: -1 + accelerator: ddp + max_epochs: 200 + gradient_clip_val: 1 + log_gpu_memory: None + limit_train_batches: 25000 + val_check_interval: ${trainer.kwargs.limit_train_batches} + log_every_n_steps: 1000 + precision: 32 + terminate_on_nan: false + check_val_every_n_epoch: 1 + num_sanity_val_steps: 8 + limit_val_batches: 1000 + replace_sampler_ddp: false + checkpoint_kwargs: + verbose: true + save_top_k: 5 + save_last: true + period: 1 + monitor: val_ssim_fid100_f1_total_mean + mode: max diff --git a/annotator/lama/saicinpainting/__init__.py b/annotator/lama/saicinpainting/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/annotator/lama/saicinpainting/training/__init__.py b/annotator/lama/saicinpainting/training/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/annotator/lama/saicinpainting/training/data/__init__.py b/annotator/lama/saicinpainting/training/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/annotator/lama/saicinpainting/training/data/masks.py b/annotator/lama/saicinpainting/training/data/masks.py new file mode 100644 index 0000000000000000000000000000000000000000..27cb9050fa67c40d7d8d492a7088a621ad1ba2ce --- /dev/null +++ b/annotator/lama/saicinpainting/training/data/masks.py @@ -0,0 +1,332 @@ +import math +import random +import hashlib +import logging +from enum import Enum + +import cv2 +import numpy as np + +# from annotator.lama.saicinpainting.evaluation.masks.mask import SegmentationMask +from annotator.lama.saicinpainting.utils import LinearRamp + +LOGGER = logging.getLogger(__name__) + + +class DrawMethod(Enum): + LINE = 'line' + CIRCLE = 'circle' + SQUARE = 'square' + + +def make_random_irregular_mask(shape, max_angle=4, max_len=60, max_width=20, min_times=0, max_times=10, + draw_method=DrawMethod.LINE): + draw_method = DrawMethod(draw_method) + + height, width = shape + mask = np.zeros((height, width), np.float32) + times = np.random.randint(min_times, max_times + 1) + for i in range(times): + start_x = np.random.randint(width) + start_y = np.random.randint(height) + for j in range(1 + np.random.randint(5)): + angle = 0.01 + np.random.randint(max_angle) + if i % 2 == 0: + angle = 2 * 3.1415926 - angle + length = 10 + np.random.randint(max_len) + brush_w = 5 + np.random.randint(max_width) + end_x = np.clip((start_x + length * np.sin(angle)).astype(np.int32), 0, width) + end_y = np.clip((start_y + length * np.cos(angle)).astype(np.int32), 0, height) + if draw_method == DrawMethod.LINE: + cv2.line(mask, (start_x, start_y), (end_x, end_y), 1.0, brush_w) + elif draw_method == DrawMethod.CIRCLE: + cv2.circle(mask, (start_x, start_y), radius=brush_w, color=1., thickness=-1) + elif draw_method == DrawMethod.SQUARE: + radius = brush_w // 2 + mask[start_y - radius:start_y + radius, start_x - radius:start_x + radius] = 1 + start_x, start_y = end_x, end_y + return mask[None, ...] + + +class RandomIrregularMaskGenerator: + def __init__(self, max_angle=4, max_len=60, max_width=20, min_times=0, max_times=10, ramp_kwargs=None, + draw_method=DrawMethod.LINE): + self.max_angle = max_angle + self.max_len = max_len + self.max_width = max_width + self.min_times = min_times + self.max_times = max_times + self.draw_method = draw_method + self.ramp = LinearRamp(**ramp_kwargs) if ramp_kwargs is not None else None + + def __call__(self, img, iter_i=None, raw_image=None): + coef = self.ramp(iter_i) if (self.ramp is not None) and (iter_i is not None) else 1 + cur_max_len = int(max(1, self.max_len * coef)) + cur_max_width = int(max(1, self.max_width * coef)) + cur_max_times = int(self.min_times + 1 + (self.max_times - self.min_times) * coef) + return make_random_irregular_mask(img.shape[1:], max_angle=self.max_angle, max_len=cur_max_len, + max_width=cur_max_width, min_times=self.min_times, max_times=cur_max_times, + draw_method=self.draw_method) + + +def make_random_rectangle_mask(shape, margin=10, bbox_min_size=30, bbox_max_size=100, min_times=0, max_times=3): + height, width = shape + mask = np.zeros((height, width), np.float32) + bbox_max_size = min(bbox_max_size, height - margin * 2, width - margin * 2) + times = np.random.randint(min_times, max_times + 1) + for i in range(times): + box_width = np.random.randint(bbox_min_size, bbox_max_size) + box_height = np.random.randint(bbox_min_size, bbox_max_size) + start_x = np.random.randint(margin, width - margin - box_width + 1) + start_y = np.random.randint(margin, height - margin - box_height + 1) + mask[start_y:start_y + box_height, start_x:start_x + box_width] = 1 + return mask[None, ...] + + +class RandomRectangleMaskGenerator: + def __init__(self, margin=10, bbox_min_size=30, bbox_max_size=100, min_times=0, max_times=3, ramp_kwargs=None): + self.margin = margin + self.bbox_min_size = bbox_min_size + self.bbox_max_size = bbox_max_size + self.min_times = min_times + self.max_times = max_times + self.ramp = LinearRamp(**ramp_kwargs) if ramp_kwargs is not None else None + + def __call__(self, img, iter_i=None, raw_image=None): + coef = self.ramp(iter_i) if (self.ramp is not None) and (iter_i is not None) else 1 + cur_bbox_max_size = int(self.bbox_min_size + 1 + (self.bbox_max_size - self.bbox_min_size) * coef) + cur_max_times = int(self.min_times + (self.max_times - self.min_times) * coef) + return make_random_rectangle_mask(img.shape[1:], margin=self.margin, bbox_min_size=self.bbox_min_size, + bbox_max_size=cur_bbox_max_size, min_times=self.min_times, + max_times=cur_max_times) + + +class RandomSegmentationMaskGenerator: + def __init__(self, **kwargs): + self.impl = None # will be instantiated in first call (effectively in subprocess) + self.kwargs = kwargs + + def __call__(self, img, iter_i=None, raw_image=None): + if self.impl is None: + self.impl = SegmentationMask(**self.kwargs) + + masks = self.impl.get_masks(np.transpose(img, (1, 2, 0))) + masks = [m for m in masks if len(np.unique(m)) > 1] + return np.random.choice(masks) + + +def make_random_superres_mask(shape, min_step=2, max_step=4, min_width=1, max_width=3): + height, width = shape + mask = np.zeros((height, width), np.float32) + step_x = np.random.randint(min_step, max_step + 1) + width_x = np.random.randint(min_width, min(step_x, max_width + 1)) + offset_x = np.random.randint(0, step_x) + + step_y = np.random.randint(min_step, max_step + 1) + width_y = np.random.randint(min_width, min(step_y, max_width + 1)) + offset_y = np.random.randint(0, step_y) + + for dy in range(width_y): + mask[offset_y + dy::step_y] = 1 + for dx in range(width_x): + mask[:, offset_x + dx::step_x] = 1 + return mask[None, ...] + + +class RandomSuperresMaskGenerator: + def __init__(self, **kwargs): + self.kwargs = kwargs + + def __call__(self, img, iter_i=None): + return make_random_superres_mask(img.shape[1:], **self.kwargs) + + +class DumbAreaMaskGenerator: + min_ratio = 0.1 + max_ratio = 0.35 + default_ratio = 0.225 + + def __init__(self, is_training): + #Parameters: + # is_training(bool): If true - random rectangular mask, if false - central square mask + self.is_training = is_training + + def _random_vector(self, dimension): + if self.is_training: + lower_limit = math.sqrt(self.min_ratio) + upper_limit = math.sqrt(self.max_ratio) + mask_side = round((random.random() * (upper_limit - lower_limit) + lower_limit) * dimension) + u = random.randint(0, dimension-mask_side-1) + v = u+mask_side + else: + margin = (math.sqrt(self.default_ratio) / 2) * dimension + u = round(dimension/2 - margin) + v = round(dimension/2 + margin) + return u, v + + def __call__(self, img, iter_i=None, raw_image=None): + c, height, width = img.shape + mask = np.zeros((height, width), np.float32) + x1, x2 = self._random_vector(width) + y1, y2 = self._random_vector(height) + mask[x1:x2, y1:y2] = 1 + return mask[None, ...] + + +class OutpaintingMaskGenerator: + def __init__(self, min_padding_percent:float=0.04, max_padding_percent:int=0.25, left_padding_prob:float=0.5, top_padding_prob:float=0.5, + right_padding_prob:float=0.5, bottom_padding_prob:float=0.5, is_fixed_randomness:bool=False): + """ + is_fixed_randomness - get identical paddings for the same image if args are the same + """ + self.min_padding_percent = min_padding_percent + self.max_padding_percent = max_padding_percent + self.probs = [left_padding_prob, top_padding_prob, right_padding_prob, bottom_padding_prob] + self.is_fixed_randomness = is_fixed_randomness + + assert self.min_padding_percent <= self.max_padding_percent + assert self.max_padding_percent > 0 + assert len([x for x in [self.min_padding_percent, self.max_padding_percent] if (x>=0 and x<=1)]) == 2, f"Padding percentage should be in [0,1]" + assert sum(self.probs) > 0, f"At least one of the padding probs should be greater than 0 - {self.probs}" + assert len([x for x in self.probs if (x >= 0) and (x <= 1)]) == 4, f"At least one of padding probs is not in [0,1] - {self.probs}" + if len([x for x in self.probs if x > 0]) == 1: + LOGGER.warning(f"Only one padding prob is greater than zero - {self.probs}. That means that the outpainting masks will be always on the same side") + + def apply_padding(self, mask, coord): + mask[int(coord[0][0]*self.img_h):int(coord[1][0]*self.img_h), + int(coord[0][1]*self.img_w):int(coord[1][1]*self.img_w)] = 1 + return mask + + def get_padding(self, size): + n1 = int(self.min_padding_percent*size) + n2 = int(self.max_padding_percent*size) + return self.rnd.randint(n1, n2) / size + + @staticmethod + def _img2rs(img): + arr = np.ascontiguousarray(img.astype(np.uint8)) + str_hash = hashlib.sha1(arr).hexdigest() + res = hash(str_hash)%(2**32) + return res + + def __call__(self, img, iter_i=None, raw_image=None): + c, self.img_h, self.img_w = img.shape + mask = np.zeros((self.img_h, self.img_w), np.float32) + at_least_one_mask_applied = False + + if self.is_fixed_randomness: + assert raw_image is not None, f"Cant calculate hash on raw_image=None" + rs = self._img2rs(raw_image) + self.rnd = np.random.RandomState(rs) + else: + self.rnd = np.random + + coords = [[ + (0,0), + (1,self.get_padding(size=self.img_h)) + ], + [ + (0,0), + (self.get_padding(size=self.img_w),1) + ], + [ + (0,1-self.get_padding(size=self.img_h)), + (1,1) + ], + [ + (1-self.get_padding(size=self.img_w),0), + (1,1) + ]] + + for pp, coord in zip(self.probs, coords): + if self.rnd.random() < pp: + at_least_one_mask_applied = True + mask = self.apply_padding(mask=mask, coord=coord) + + if not at_least_one_mask_applied: + idx = self.rnd.choice(range(len(coords)), p=np.array(self.probs)/sum(self.probs)) + mask = self.apply_padding(mask=mask, coord=coords[idx]) + return mask[None, ...] + + +class MixedMaskGenerator: + def __init__(self, irregular_proba=1/3, irregular_kwargs=None, + box_proba=1/3, box_kwargs=None, + segm_proba=1/3, segm_kwargs=None, + squares_proba=0, squares_kwargs=None, + superres_proba=0, superres_kwargs=None, + outpainting_proba=0, outpainting_kwargs=None, + invert_proba=0): + self.probas = [] + self.gens = [] + + if irregular_proba > 0: + self.probas.append(irregular_proba) + if irregular_kwargs is None: + irregular_kwargs = {} + else: + irregular_kwargs = dict(irregular_kwargs) + irregular_kwargs['draw_method'] = DrawMethod.LINE + self.gens.append(RandomIrregularMaskGenerator(**irregular_kwargs)) + + if box_proba > 0: + self.probas.append(box_proba) + if box_kwargs is None: + box_kwargs = {} + self.gens.append(RandomRectangleMaskGenerator(**box_kwargs)) + + if segm_proba > 0: + self.probas.append(segm_proba) + if segm_kwargs is None: + segm_kwargs = {} + self.gens.append(RandomSegmentationMaskGenerator(**segm_kwargs)) + + if squares_proba > 0: + self.probas.append(squares_proba) + if squares_kwargs is None: + squares_kwargs = {} + else: + squares_kwargs = dict(squares_kwargs) + squares_kwargs['draw_method'] = DrawMethod.SQUARE + self.gens.append(RandomIrregularMaskGenerator(**squares_kwargs)) + + if superres_proba > 0: + self.probas.append(superres_proba) + if superres_kwargs is None: + superres_kwargs = {} + self.gens.append(RandomSuperresMaskGenerator(**superres_kwargs)) + + if outpainting_proba > 0: + self.probas.append(outpainting_proba) + if outpainting_kwargs is None: + outpainting_kwargs = {} + self.gens.append(OutpaintingMaskGenerator(**outpainting_kwargs)) + + self.probas = np.array(self.probas, dtype='float32') + self.probas /= self.probas.sum() + self.invert_proba = invert_proba + + def __call__(self, img, iter_i=None, raw_image=None): + kind = np.random.choice(len(self.probas), p=self.probas) + gen = self.gens[kind] + result = gen(img, iter_i=iter_i, raw_image=raw_image) + if self.invert_proba > 0 and random.random() < self.invert_proba: + result = 1 - result + return result + + +def get_mask_generator(kind, kwargs): + if kind is None: + kind = "mixed" + if kwargs is None: + kwargs = {} + + if kind == "mixed": + cl = MixedMaskGenerator + elif kind == "outpainting": + cl = OutpaintingMaskGenerator + elif kind == "dumb": + cl = DumbAreaMaskGenerator + else: + raise NotImplementedError(f"No such generator kind = {kind}") + return cl(**kwargs) diff --git a/annotator/lama/saicinpainting/training/losses/__init__.py b/annotator/lama/saicinpainting/training/losses/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/annotator/lama/saicinpainting/training/losses/adversarial.py b/annotator/lama/saicinpainting/training/losses/adversarial.py new file mode 100644 index 0000000000000000000000000000000000000000..d6db2967ce5074d94ed3b4c51fc743ff2f7831b1 --- /dev/null +++ b/annotator/lama/saicinpainting/training/losses/adversarial.py @@ -0,0 +1,177 @@ +from typing import Tuple, Dict, Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class BaseAdversarialLoss: + def pre_generator_step(self, real_batch: torch.Tensor, fake_batch: torch.Tensor, + generator: nn.Module, discriminator: nn.Module): + """ + Prepare for generator step + :param real_batch: Tensor, a batch of real samples + :param fake_batch: Tensor, a batch of samples produced by generator + :param generator: + :param discriminator: + :return: None + """ + + def pre_discriminator_step(self, real_batch: torch.Tensor, fake_batch: torch.Tensor, + generator: nn.Module, discriminator: nn.Module): + """ + Prepare for discriminator step + :param real_batch: Tensor, a batch of real samples + :param fake_batch: Tensor, a batch of samples produced by generator + :param generator: + :param discriminator: + :return: None + """ + + def generator_loss(self, real_batch: torch.Tensor, fake_batch: torch.Tensor, + discr_real_pred: torch.Tensor, discr_fake_pred: torch.Tensor, + mask: Optional[torch.Tensor] = None) \ + -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Calculate generator loss + :param real_batch: Tensor, a batch of real samples + :param fake_batch: Tensor, a batch of samples produced by generator + :param discr_real_pred: Tensor, discriminator output for real_batch + :param discr_fake_pred: Tensor, discriminator output for fake_batch + :param mask: Tensor, actual mask, which was at input of generator when making fake_batch + :return: total generator loss along with some values that might be interesting to log + """ + raise NotImplemented() + + def discriminator_loss(self, real_batch: torch.Tensor, fake_batch: torch.Tensor, + discr_real_pred: torch.Tensor, discr_fake_pred: torch.Tensor, + mask: Optional[torch.Tensor] = None) \ + -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + """ + Calculate discriminator loss and call .backward() on it + :param real_batch: Tensor, a batch of real samples + :param fake_batch: Tensor, a batch of samples produced by generator + :param discr_real_pred: Tensor, discriminator output for real_batch + :param discr_fake_pred: Tensor, discriminator output for fake_batch + :param mask: Tensor, actual mask, which was at input of generator when making fake_batch + :return: total discriminator loss along with some values that might be interesting to log + """ + raise NotImplemented() + + def interpolate_mask(self, mask, shape): + assert mask is not None + assert self.allow_scale_mask or shape == mask.shape[-2:] + if shape != mask.shape[-2:] and self.allow_scale_mask: + if self.mask_scale_mode == 'maxpool': + mask = F.adaptive_max_pool2d(mask, shape) + else: + mask = F.interpolate(mask, size=shape, mode=self.mask_scale_mode) + return mask + +def make_r1_gp(discr_real_pred, real_batch): + if torch.is_grad_enabled(): + grad_real = torch.autograd.grad(outputs=discr_real_pred.sum(), inputs=real_batch, create_graph=True)[0] + grad_penalty = (grad_real.view(grad_real.shape[0], -1).norm(2, dim=1) ** 2).mean() + else: + grad_penalty = 0 + real_batch.requires_grad = False + + return grad_penalty + +class NonSaturatingWithR1(BaseAdversarialLoss): + def __init__(self, gp_coef=5, weight=1, mask_as_fake_target=False, allow_scale_mask=False, + mask_scale_mode='nearest', extra_mask_weight_for_gen=0, + use_unmasked_for_gen=True, use_unmasked_for_discr=True): + self.gp_coef = gp_coef + self.weight = weight + # use for discr => use for gen; + # otherwise we teach only the discr to pay attention to very small difference + assert use_unmasked_for_gen or (not use_unmasked_for_discr) + # mask as target => use unmasked for discr: + # if we don't care about unmasked regions at all + # then it doesn't matter if the value of mask_as_fake_target is true or false + assert use_unmasked_for_discr or (not mask_as_fake_target) + self.use_unmasked_for_gen = use_unmasked_for_gen + self.use_unmasked_for_discr = use_unmasked_for_discr + self.mask_as_fake_target = mask_as_fake_target + self.allow_scale_mask = allow_scale_mask + self.mask_scale_mode = mask_scale_mode + self.extra_mask_weight_for_gen = extra_mask_weight_for_gen + + def generator_loss(self, real_batch: torch.Tensor, fake_batch: torch.Tensor, + discr_real_pred: torch.Tensor, discr_fake_pred: torch.Tensor, + mask=None) \ + -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + fake_loss = F.softplus(-discr_fake_pred) + if (self.mask_as_fake_target and self.extra_mask_weight_for_gen > 0) or \ + not self.use_unmasked_for_gen: # == if masked region should be treated differently + mask = self.interpolate_mask(mask, discr_fake_pred.shape[-2:]) + if not self.use_unmasked_for_gen: + fake_loss = fake_loss * mask + else: + pixel_weights = 1 + mask * self.extra_mask_weight_for_gen + fake_loss = fake_loss * pixel_weights + + return fake_loss.mean() * self.weight, dict() + + def pre_discriminator_step(self, real_batch: torch.Tensor, fake_batch: torch.Tensor, + generator: nn.Module, discriminator: nn.Module): + real_batch.requires_grad = True + + def discriminator_loss(self, real_batch: torch.Tensor, fake_batch: torch.Tensor, + discr_real_pred: torch.Tensor, discr_fake_pred: torch.Tensor, + mask=None) \ + -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + + real_loss = F.softplus(-discr_real_pred) + grad_penalty = make_r1_gp(discr_real_pred, real_batch) * self.gp_coef + fake_loss = F.softplus(discr_fake_pred) + + if not self.use_unmasked_for_discr or self.mask_as_fake_target: + # == if masked region should be treated differently + mask = self.interpolate_mask(mask, discr_fake_pred.shape[-2:]) + # use_unmasked_for_discr=False only makes sense for fakes; + # for reals there is no difference beetween two regions + fake_loss = fake_loss * mask + if self.mask_as_fake_target: + fake_loss = fake_loss + (1 - mask) * F.softplus(-discr_fake_pred) + + sum_discr_loss = real_loss + grad_penalty + fake_loss + metrics = dict(discr_real_out=discr_real_pred.mean(), + discr_fake_out=discr_fake_pred.mean(), + discr_real_gp=grad_penalty) + return sum_discr_loss.mean(), metrics + +class BCELoss(BaseAdversarialLoss): + def __init__(self, weight): + self.weight = weight + self.bce_loss = nn.BCEWithLogitsLoss() + + def generator_loss(self, discr_fake_pred: torch.Tensor) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + real_mask_gt = torch.zeros(discr_fake_pred.shape).to(discr_fake_pred.device) + fake_loss = self.bce_loss(discr_fake_pred, real_mask_gt) * self.weight + return fake_loss, dict() + + def pre_discriminator_step(self, real_batch: torch.Tensor, fake_batch: torch.Tensor, + generator: nn.Module, discriminator: nn.Module): + real_batch.requires_grad = True + + def discriminator_loss(self, + mask: torch.Tensor, + discr_real_pred: torch.Tensor, + discr_fake_pred: torch.Tensor) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + + real_mask_gt = torch.zeros(discr_real_pred.shape).to(discr_real_pred.device) + sum_discr_loss = (self.bce_loss(discr_real_pred, real_mask_gt) + self.bce_loss(discr_fake_pred, mask)) / 2 + metrics = dict(discr_real_out=discr_real_pred.mean(), + discr_fake_out=discr_fake_pred.mean(), + discr_real_gp=0) + return sum_discr_loss, metrics + + +def make_discrim_loss(kind, **kwargs): + if kind == 'r1': + return NonSaturatingWithR1(**kwargs) + elif kind == 'bce': + return BCELoss(**kwargs) + raise ValueError(f'Unknown adversarial loss kind {kind}') diff --git a/annotator/lama/saicinpainting/training/losses/constants.py b/annotator/lama/saicinpainting/training/losses/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..ae3e5e151342232be8e2c2a77fe6fd5798dc2a8c --- /dev/null +++ b/annotator/lama/saicinpainting/training/losses/constants.py @@ -0,0 +1,152 @@ +weights = {"ade20k": + [6.34517766497462, + 9.328358208955224, + 11.389521640091116, + 16.10305958132045, + 20.833333333333332, + 22.22222222222222, + 25.125628140703515, + 43.29004329004329, + 50.5050505050505, + 54.6448087431694, + 55.24861878453038, + 60.24096385542168, + 62.5, + 66.2251655629139, + 84.74576271186442, + 90.90909090909092, + 91.74311926605505, + 96.15384615384616, + 96.15384615384616, + 97.08737864077669, + 102.04081632653062, + 135.13513513513513, + 149.2537313432836, + 153.84615384615384, + 163.93442622950818, + 166.66666666666666, + 188.67924528301887, + 192.30769230769232, + 217.3913043478261, + 227.27272727272725, + 227.27272727272725, + 227.27272727272725, + 303.03030303030306, + 322.5806451612903, + 333.3333333333333, + 370.3703703703703, + 384.61538461538464, + 416.6666666666667, + 416.6666666666667, + 434.7826086956522, + 434.7826086956522, + 454.5454545454545, + 454.5454545454545, + 500.0, + 526.3157894736842, + 526.3157894736842, + 555.5555555555555, + 555.5555555555555, + 555.5555555555555, + 555.5555555555555, + 555.5555555555555, + 555.5555555555555, + 555.5555555555555, + 588.2352941176471, + 588.2352941176471, + 588.2352941176471, + 588.2352941176471, + 588.2352941176471, + 666.6666666666666, + 666.6666666666666, + 666.6666666666666, + 666.6666666666666, + 714.2857142857143, + 714.2857142857143, + 714.2857142857143, + 714.2857142857143, + 714.2857142857143, + 769.2307692307693, + 769.2307692307693, + 769.2307692307693, + 833.3333333333334, + 833.3333333333334, + 833.3333333333334, + 833.3333333333334, + 909.090909090909, + 1000.0, + 1111.111111111111, + 1111.111111111111, + 1111.111111111111, + 1111.111111111111, + 1111.111111111111, + 1250.0, + 1250.0, + 1250.0, + 1250.0, + 1250.0, + 1428.5714285714287, + 1428.5714285714287, + 1428.5714285714287, + 1428.5714285714287, + 1428.5714285714287, + 1428.5714285714287, + 1428.5714285714287, + 1666.6666666666667, + 1666.6666666666667, + 1666.6666666666667, + 1666.6666666666667, + 1666.6666666666667, + 1666.6666666666667, + 1666.6666666666667, + 1666.6666666666667, + 1666.6666666666667, + 1666.6666666666667, + 1666.6666666666667, + 2000.0, + 2000.0, + 2000.0, + 2000.0, + 2000.0, + 2000.0, + 2000.0, + 2000.0, + 2000.0, + 2000.0, + 2000.0, + 2000.0, + 2000.0, + 2000.0, + 2000.0, + 2000.0, + 2000.0, + 2500.0, + 2500.0, + 2500.0, + 2500.0, + 2500.0, + 2500.0, + 2500.0, + 2500.0, + 2500.0, + 2500.0, + 2500.0, + 2500.0, + 2500.0, + 3333.3333333333335, + 3333.3333333333335, + 3333.3333333333335, + 3333.3333333333335, + 3333.3333333333335, + 3333.3333333333335, + 3333.3333333333335, + 3333.3333333333335, + 3333.3333333333335, + 3333.3333333333335, + 3333.3333333333335, + 3333.3333333333335, + 3333.3333333333335, + 5000.0, + 5000.0, + 5000.0] +} \ No newline at end of file diff --git a/annotator/lama/saicinpainting/training/losses/distance_weighting.py b/annotator/lama/saicinpainting/training/losses/distance_weighting.py new file mode 100644 index 0000000000000000000000000000000000000000..90ce05bee5f633662057b3347d8791e1b4d115a0 --- /dev/null +++ b/annotator/lama/saicinpainting/training/losses/distance_weighting.py @@ -0,0 +1,126 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision + +from annotator.lama.saicinpainting.training.losses.perceptual import IMAGENET_STD, IMAGENET_MEAN + + +def dummy_distance_weighter(real_img, pred_img, mask): + return mask + + +def get_gauss_kernel(kernel_size, width_factor=1): + coords = torch.stack(torch.meshgrid(torch.arange(kernel_size), + torch.arange(kernel_size)), + dim=0).float() + diff = torch.exp(-((coords - kernel_size // 2) ** 2).sum(0) / kernel_size / width_factor) + diff /= diff.sum() + return diff + + +class BlurMask(nn.Module): + def __init__(self, kernel_size=5, width_factor=1): + super().__init__() + self.filter = nn.Conv2d(1, 1, kernel_size, padding=kernel_size // 2, padding_mode='replicate', bias=False) + self.filter.weight.data.copy_(get_gauss_kernel(kernel_size, width_factor=width_factor)) + + def forward(self, real_img, pred_img, mask): + with torch.no_grad(): + result = self.filter(mask) * mask + return result + + +class EmulatedEDTMask(nn.Module): + def __init__(self, dilate_kernel_size=5, blur_kernel_size=5, width_factor=1): + super().__init__() + self.dilate_filter = nn.Conv2d(1, 1, dilate_kernel_size, padding=dilate_kernel_size// 2, padding_mode='replicate', + bias=False) + self.dilate_filter.weight.data.copy_(torch.ones(1, 1, dilate_kernel_size, dilate_kernel_size, dtype=torch.float)) + self.blur_filter = nn.Conv2d(1, 1, blur_kernel_size, padding=blur_kernel_size // 2, padding_mode='replicate', bias=False) + self.blur_filter.weight.data.copy_(get_gauss_kernel(blur_kernel_size, width_factor=width_factor)) + + def forward(self, real_img, pred_img, mask): + with torch.no_grad(): + known_mask = 1 - mask + dilated_known_mask = (self.dilate_filter(known_mask) > 1).float() + result = self.blur_filter(1 - dilated_known_mask) * mask + return result + + +class PropagatePerceptualSim(nn.Module): + def __init__(self, level=2, max_iters=10, temperature=500, erode_mask_size=3): + super().__init__() + vgg = torchvision.models.vgg19(pretrained=True).features + vgg_avg_pooling = [] + + for weights in vgg.parameters(): + weights.requires_grad = False + + cur_level_i = 0 + for module in vgg.modules(): + if module.__class__.__name__ == 'Sequential': + continue + elif module.__class__.__name__ == 'MaxPool2d': + vgg_avg_pooling.append(nn.AvgPool2d(kernel_size=2, stride=2, padding=0)) + else: + vgg_avg_pooling.append(module) + if module.__class__.__name__ == 'ReLU': + cur_level_i += 1 + if cur_level_i == level: + break + + self.features = nn.Sequential(*vgg_avg_pooling) + + self.max_iters = max_iters + self.temperature = temperature + self.do_erode = erode_mask_size > 0 + if self.do_erode: + self.erode_mask = nn.Conv2d(1, 1, erode_mask_size, padding=erode_mask_size // 2, bias=False) + self.erode_mask.weight.data.fill_(1) + + def forward(self, real_img, pred_img, mask): + with torch.no_grad(): + real_img = (real_img - IMAGENET_MEAN.to(real_img)) / IMAGENET_STD.to(real_img) + real_feats = self.features(real_img) + + vertical_sim = torch.exp(-(real_feats[:, :, 1:] - real_feats[:, :, :-1]).pow(2).sum(1, keepdim=True) + / self.temperature) + horizontal_sim = torch.exp(-(real_feats[:, :, :, 1:] - real_feats[:, :, :, :-1]).pow(2).sum(1, keepdim=True) + / self.temperature) + + mask_scaled = F.interpolate(mask, size=real_feats.shape[-2:], mode='bilinear', align_corners=False) + if self.do_erode: + mask_scaled = (self.erode_mask(mask_scaled) > 1).float() + + cur_knowness = 1 - mask_scaled + + for iter_i in range(self.max_iters): + new_top_knowness = F.pad(cur_knowness[:, :, :-1] * vertical_sim, (0, 0, 1, 0), mode='replicate') + new_bottom_knowness = F.pad(cur_knowness[:, :, 1:] * vertical_sim, (0, 0, 0, 1), mode='replicate') + + new_left_knowness = F.pad(cur_knowness[:, :, :, :-1] * horizontal_sim, (1, 0, 0, 0), mode='replicate') + new_right_knowness = F.pad(cur_knowness[:, :, :, 1:] * horizontal_sim, (0, 1, 0, 0), mode='replicate') + + new_knowness = torch.stack([new_top_knowness, new_bottom_knowness, + new_left_knowness, new_right_knowness], + dim=0).max(0).values + + cur_knowness = torch.max(cur_knowness, new_knowness) + + cur_knowness = F.interpolate(cur_knowness, size=mask.shape[-2:], mode='bilinear') + result = torch.min(mask, 1 - cur_knowness) + + return result + + +def make_mask_distance_weighter(kind='none', **kwargs): + if kind == 'none': + return dummy_distance_weighter + if kind == 'blur': + return BlurMask(**kwargs) + if kind == 'edt': + return EmulatedEDTMask(**kwargs) + if kind == 'pps': + return PropagatePerceptualSim(**kwargs) + raise ValueError(f'Unknown mask distance weighter kind {kind}') diff --git a/annotator/lama/saicinpainting/training/losses/feature_matching.py b/annotator/lama/saicinpainting/training/losses/feature_matching.py new file mode 100644 index 0000000000000000000000000000000000000000..c019895c9178817837d1a6773367b178a861dc61 --- /dev/null +++ b/annotator/lama/saicinpainting/training/losses/feature_matching.py @@ -0,0 +1,33 @@ +from typing import List + +import torch +import torch.nn.functional as F + + +def masked_l2_loss(pred, target, mask, weight_known, weight_missing): + per_pixel_l2 = F.mse_loss(pred, target, reduction='none') + pixel_weights = mask * weight_missing + (1 - mask) * weight_known + return (pixel_weights * per_pixel_l2).mean() + + +def masked_l1_loss(pred, target, mask, weight_known, weight_missing): + per_pixel_l1 = F.l1_loss(pred, target, reduction='none') + pixel_weights = mask * weight_missing + (1 - mask) * weight_known + return (pixel_weights * per_pixel_l1).mean() + + +def feature_matching_loss(fake_features: List[torch.Tensor], target_features: List[torch.Tensor], mask=None): + if mask is None: + res = torch.stack([F.mse_loss(fake_feat, target_feat) + for fake_feat, target_feat in zip(fake_features, target_features)]).mean() + else: + res = 0 + norm = 0 + for fake_feat, target_feat in zip(fake_features, target_features): + cur_mask = F.interpolate(mask, size=fake_feat.shape[-2:], mode='bilinear', align_corners=False) + error_weights = 1 - cur_mask + cur_val = ((fake_feat - target_feat).pow(2) * error_weights).mean() + res = res + cur_val + norm += 1 + res = res / norm + return res diff --git a/annotator/lama/saicinpainting/training/losses/perceptual.py b/annotator/lama/saicinpainting/training/losses/perceptual.py new file mode 100644 index 0000000000000000000000000000000000000000..5d8b0b309b2b8ba95172cb16af440033a4aeafae --- /dev/null +++ b/annotator/lama/saicinpainting/training/losses/perceptual.py @@ -0,0 +1,113 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision + +# from models.ade20k import ModelBuilder +from annotator.lama.saicinpainting.utils import check_and_warn_input_range + + +IMAGENET_MEAN = torch.FloatTensor([0.485, 0.456, 0.406])[None, :, None, None] +IMAGENET_STD = torch.FloatTensor([0.229, 0.224, 0.225])[None, :, None, None] + + +class PerceptualLoss(nn.Module): + def __init__(self, normalize_inputs=True): + super(PerceptualLoss, self).__init__() + + self.normalize_inputs = normalize_inputs + self.mean_ = IMAGENET_MEAN + self.std_ = IMAGENET_STD + + vgg = torchvision.models.vgg19(pretrained=True).features + vgg_avg_pooling = [] + + for weights in vgg.parameters(): + weights.requires_grad = False + + for module in vgg.modules(): + if module.__class__.__name__ == 'Sequential': + continue + elif module.__class__.__name__ == 'MaxPool2d': + vgg_avg_pooling.append(nn.AvgPool2d(kernel_size=2, stride=2, padding=0)) + else: + vgg_avg_pooling.append(module) + + self.vgg = nn.Sequential(*vgg_avg_pooling) + + def do_normalize_inputs(self, x): + return (x - self.mean_.to(x.device)) / self.std_.to(x.device) + + def partial_losses(self, input, target, mask=None): + check_and_warn_input_range(target, 0, 1, 'PerceptualLoss target in partial_losses') + + # we expect input and target to be in [0, 1] range + losses = [] + + if self.normalize_inputs: + features_input = self.do_normalize_inputs(input) + features_target = self.do_normalize_inputs(target) + else: + features_input = input + features_target = target + + for layer in self.vgg[:30]: + + features_input = layer(features_input) + features_target = layer(features_target) + + if layer.__class__.__name__ == 'ReLU': + loss = F.mse_loss(features_input, features_target, reduction='none') + + if mask is not None: + cur_mask = F.interpolate(mask, size=features_input.shape[-2:], + mode='bilinear', align_corners=False) + loss = loss * (1 - cur_mask) + + loss = loss.mean(dim=tuple(range(1, len(loss.shape)))) + losses.append(loss) + + return losses + + def forward(self, input, target, mask=None): + losses = self.partial_losses(input, target, mask=mask) + return torch.stack(losses).sum(dim=0) + + def get_global_features(self, input): + check_and_warn_input_range(input, 0, 1, 'PerceptualLoss input in get_global_features') + + if self.normalize_inputs: + features_input = self.do_normalize_inputs(input) + else: + features_input = input + + features_input = self.vgg(features_input) + return features_input + + +class ResNetPL(nn.Module): + def __init__(self, weight=1, + weights_path=None, arch_encoder='resnet50dilated', segmentation=True): + super().__init__() + self.impl = ModelBuilder.get_encoder(weights_path=weights_path, + arch_encoder=arch_encoder, + arch_decoder='ppm_deepsup', + fc_dim=2048, + segmentation=segmentation) + self.impl.eval() + for w in self.impl.parameters(): + w.requires_grad_(False) + + self.weight = weight + + def forward(self, pred, target): + pred = (pred - IMAGENET_MEAN.to(pred)) / IMAGENET_STD.to(pred) + target = (target - IMAGENET_MEAN.to(target)) / IMAGENET_STD.to(target) + + pred_feats = self.impl(pred, return_feature_maps=True) + target_feats = self.impl(target, return_feature_maps=True) + + result = torch.stack([F.mse_loss(cur_pred, cur_target) + for cur_pred, cur_target + in zip(pred_feats, target_feats)]).sum() * self.weight + return result diff --git a/annotator/lama/saicinpainting/training/losses/segmentation.py b/annotator/lama/saicinpainting/training/losses/segmentation.py new file mode 100644 index 0000000000000000000000000000000000000000..3d4a9f94eaae84722db584277dbbf9bc41ede357 --- /dev/null +++ b/annotator/lama/saicinpainting/training/losses/segmentation.py @@ -0,0 +1,43 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .constants import weights as constant_weights + + +class CrossEntropy2d(nn.Module): + def __init__(self, reduction="mean", ignore_label=255, weights=None, *args, **kwargs): + """ + weight (Tensor, optional): a manual rescaling weight given to each class. + If given, has to be a Tensor of size "nclasses" + """ + super(CrossEntropy2d, self).__init__() + self.reduction = reduction + self.ignore_label = ignore_label + self.weights = weights + if self.weights is not None: + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + self.weights = torch.FloatTensor(constant_weights[weights]).to(device) + + def forward(self, predict, target): + """ + Args: + predict:(n, c, h, w) + target:(n, 1, h, w) + """ + target = target.long() + assert not target.requires_grad + assert predict.dim() == 4, "{0}".format(predict.size()) + assert target.dim() == 4, "{0}".format(target.size()) + assert predict.size(0) == target.size(0), "{0} vs {1} ".format(predict.size(0), target.size(0)) + assert target.size(1) == 1, "{0}".format(target.size(1)) + assert predict.size(2) == target.size(2), "{0} vs {1} ".format(predict.size(2), target.size(2)) + assert predict.size(3) == target.size(3), "{0} vs {1} ".format(predict.size(3), target.size(3)) + target = target.squeeze(1) + n, c, h, w = predict.size() + target_mask = (target >= 0) * (target != self.ignore_label) + target = target[target_mask] + predict = predict.transpose(1, 2).transpose(2, 3).contiguous() + predict = predict[target_mask.view(n, h, w, 1).repeat(1, 1, 1, c)].view(-1, c) + loss = F.cross_entropy(predict, target, weight=self.weights, reduction=self.reduction) + return loss diff --git a/annotator/lama/saicinpainting/training/losses/style_loss.py b/annotator/lama/saicinpainting/training/losses/style_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..0bb42d7fbc5d17a47bec7365889868505f5fdfb5 --- /dev/null +++ b/annotator/lama/saicinpainting/training/losses/style_loss.py @@ -0,0 +1,155 @@ +import torch +import torch.nn as nn +import torchvision.models as models + + +class PerceptualLoss(nn.Module): + r""" + Perceptual loss, VGG-based + https://arxiv.org/abs/1603.08155 + https://github.com/dxyang/StyleTransfer/blob/master/utils.py + """ + + def __init__(self, weights=[1.0, 1.0, 1.0, 1.0, 1.0]): + super(PerceptualLoss, self).__init__() + self.add_module('vgg', VGG19()) + self.criterion = torch.nn.L1Loss() + self.weights = weights + + def __call__(self, x, y): + # Compute features + x_vgg, y_vgg = self.vgg(x), self.vgg(y) + + content_loss = 0.0 + content_loss += self.weights[0] * self.criterion(x_vgg['relu1_1'], y_vgg['relu1_1']) + content_loss += self.weights[1] * self.criterion(x_vgg['relu2_1'], y_vgg['relu2_1']) + content_loss += self.weights[2] * self.criterion(x_vgg['relu3_1'], y_vgg['relu3_1']) + content_loss += self.weights[3] * self.criterion(x_vgg['relu4_1'], y_vgg['relu4_1']) + content_loss += self.weights[4] * self.criterion(x_vgg['relu5_1'], y_vgg['relu5_1']) + + + return content_loss + + +class VGG19(torch.nn.Module): + def __init__(self): + super(VGG19, self).__init__() + features = models.vgg19(pretrained=True).features + self.relu1_1 = torch.nn.Sequential() + self.relu1_2 = torch.nn.Sequential() + + self.relu2_1 = torch.nn.Sequential() + self.relu2_2 = torch.nn.Sequential() + + self.relu3_1 = torch.nn.Sequential() + self.relu3_2 = torch.nn.Sequential() + self.relu3_3 = torch.nn.Sequential() + self.relu3_4 = torch.nn.Sequential() + + self.relu4_1 = torch.nn.Sequential() + self.relu4_2 = torch.nn.Sequential() + self.relu4_3 = torch.nn.Sequential() + self.relu4_4 = torch.nn.Sequential() + + self.relu5_1 = torch.nn.Sequential() + self.relu5_2 = torch.nn.Sequential() + self.relu5_3 = torch.nn.Sequential() + self.relu5_4 = torch.nn.Sequential() + + for x in range(2): + self.relu1_1.add_module(str(x), features[x]) + + for x in range(2, 4): + self.relu1_2.add_module(str(x), features[x]) + + for x in range(4, 7): + self.relu2_1.add_module(str(x), features[x]) + + for x in range(7, 9): + self.relu2_2.add_module(str(x), features[x]) + + for x in range(9, 12): + self.relu3_1.add_module(str(x), features[x]) + + for x in range(12, 14): + self.relu3_2.add_module(str(x), features[x]) + + for x in range(14, 16): + self.relu3_2.add_module(str(x), features[x]) + + for x in range(16, 18): + self.relu3_4.add_module(str(x), features[x]) + + for x in range(18, 21): + self.relu4_1.add_module(str(x), features[x]) + + for x in range(21, 23): + self.relu4_2.add_module(str(x), features[x]) + + for x in range(23, 25): + self.relu4_3.add_module(str(x), features[x]) + + for x in range(25, 27): + self.relu4_4.add_module(str(x), features[x]) + + for x in range(27, 30): + self.relu5_1.add_module(str(x), features[x]) + + for x in range(30, 32): + self.relu5_2.add_module(str(x), features[x]) + + for x in range(32, 34): + self.relu5_3.add_module(str(x), features[x]) + + for x in range(34, 36): + self.relu5_4.add_module(str(x), features[x]) + + # don't need the gradients, just want the features + for param in self.parameters(): + param.requires_grad = False + + def forward(self, x): + relu1_1 = self.relu1_1(x) + relu1_2 = self.relu1_2(relu1_1) + + relu2_1 = self.relu2_1(relu1_2) + relu2_2 = self.relu2_2(relu2_1) + + relu3_1 = self.relu3_1(relu2_2) + relu3_2 = self.relu3_2(relu3_1) + relu3_3 = self.relu3_3(relu3_2) + relu3_4 = self.relu3_4(relu3_3) + + relu4_1 = self.relu4_1(relu3_4) + relu4_2 = self.relu4_2(relu4_1) + relu4_3 = self.relu4_3(relu4_2) + relu4_4 = self.relu4_4(relu4_3) + + relu5_1 = self.relu5_1(relu4_4) + relu5_2 = self.relu5_2(relu5_1) + relu5_3 = self.relu5_3(relu5_2) + relu5_4 = self.relu5_4(relu5_3) + + out = { + 'relu1_1': relu1_1, + 'relu1_2': relu1_2, + + 'relu2_1': relu2_1, + 'relu2_2': relu2_2, + + 'relu3_1': relu3_1, + 'relu3_2': relu3_2, + 'relu3_3': relu3_3, + 'relu3_4': relu3_4, + + 'relu4_1': relu4_1, + 'relu4_2': relu4_2, + 'relu4_3': relu4_3, + 'relu4_4': relu4_4, + + 'relu5_1': relu5_1, + 'relu5_2': relu5_2, + 'relu5_3': relu5_3, + 'relu5_4': relu5_4, + } + return out diff --git a/annotator/lama/saicinpainting/training/modules/__init__.py b/annotator/lama/saicinpainting/training/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c5c56ad9965ec95f3ae28c35c2ab42456eb06066 --- /dev/null +++ b/annotator/lama/saicinpainting/training/modules/__init__.py @@ -0,0 +1,31 @@ +import logging + +from annotator.lama.saicinpainting.training.modules.ffc import FFCResNetGenerator +from annotator.lama.saicinpainting.training.modules.pix2pixhd import GlobalGenerator, MultiDilatedGlobalGenerator, \ + NLayerDiscriminator, MultidilatedNLayerDiscriminator + +def make_generator(config, kind, **kwargs): + logging.info(f'Make generator {kind}') + + if kind == 'pix2pixhd_multidilated': + return MultiDilatedGlobalGenerator(**kwargs) + + if kind == 'pix2pixhd_global': + return GlobalGenerator(**kwargs) + + if kind == 'ffc_resnet': + return FFCResNetGenerator(**kwargs) + + raise ValueError(f'Unknown generator kind {kind}') + + +def make_discriminator(kind, **kwargs): + logging.info(f'Make discriminator {kind}') + + if kind == 'pix2pixhd_nlayer_multidilated': + return MultidilatedNLayerDiscriminator(**kwargs) + + if kind == 'pix2pixhd_nlayer': + return NLayerDiscriminator(**kwargs) + + raise ValueError(f'Unknown discriminator kind {kind}') diff --git a/annotator/lama/saicinpainting/training/modules/base.py b/annotator/lama/saicinpainting/training/modules/base.py new file mode 100644 index 0000000000000000000000000000000000000000..58c513987601d6a442ca8f066f82f1af46e28939 --- /dev/null +++ b/annotator/lama/saicinpainting/training/modules/base.py @@ -0,0 +1,80 @@ +import abc +from typing import Tuple, List + +import torch +import torch.nn as nn + +from annotator.lama.saicinpainting.training.modules.depthwise_sep_conv import DepthWiseSeperableConv +from annotator.lama.saicinpainting.training.modules.multidilated_conv import MultidilatedConv + + +class BaseDiscriminator(nn.Module): + @abc.abstractmethod + def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, List[torch.Tensor]]: + """ + Predict scores and get intermediate activations. Useful for feature matching loss + :return tuple (scores, list of intermediate activations) + """ + raise NotImplemented() + + +def get_conv_block_ctor(kind='default'): + if not isinstance(kind, str): + return kind + if kind == 'default': + return nn.Conv2d + if kind == 'depthwise': + return DepthWiseSeperableConv + if kind == 'multidilated': + return MultidilatedConv + raise ValueError(f'Unknown convolutional block kind {kind}') + + +def get_norm_layer(kind='bn'): + if not isinstance(kind, str): + return kind + if kind == 'bn': + return nn.BatchNorm2d + if kind == 'in': + return nn.InstanceNorm2d + raise ValueError(f'Unknown norm block kind {kind}') + + +def get_activation(kind='tanh'): + if kind == 'tanh': + return nn.Tanh() + if kind == 'sigmoid': + return nn.Sigmoid() + if kind is False: + return nn.Identity() + raise ValueError(f'Unknown activation kind {kind}') + + +class SimpleMultiStepGenerator(nn.Module): + def __init__(self, steps: List[nn.Module]): + super().__init__() + self.steps = nn.ModuleList(steps) + + def forward(self, x): + cur_in = x + outs = [] + for step in self.steps: + cur_out = step(cur_in) + outs.append(cur_out) + cur_in = torch.cat((cur_in, cur_out), dim=1) + return torch.cat(outs[::-1], dim=1) + +def deconv_factory(kind, ngf, mult, norm_layer, activation, max_features): + if kind == 'convtranspose': + return [nn.ConvTranspose2d(min(max_features, ngf * mult), + min(max_features, int(ngf * mult / 2)), + kernel_size=3, stride=2, padding=1, output_padding=1), + norm_layer(min(max_features, int(ngf * mult / 2))), activation] + elif kind == 'bilinear': + return [nn.Upsample(scale_factor=2, mode='bilinear'), + DepthWiseSeperableConv(min(max_features, ngf * mult), + min(max_features, int(ngf * mult / 2)), + kernel_size=3, stride=1, padding=1), + norm_layer(min(max_features, int(ngf * mult / 2))), activation] + else: + raise Exception(f"Invalid deconv kind: {kind}") \ No newline at end of file diff --git a/annotator/lama/saicinpainting/training/modules/depthwise_sep_conv.py b/annotator/lama/saicinpainting/training/modules/depthwise_sep_conv.py new file mode 100644 index 0000000000000000000000000000000000000000..83dd15c3df1d9f40baf0091a373fa224532c9ddd --- /dev/null +++ b/annotator/lama/saicinpainting/training/modules/depthwise_sep_conv.py @@ -0,0 +1,17 @@ +import torch +import torch.nn as nn + +class DepthWiseSeperableConv(nn.Module): + def __init__(self, in_dim, out_dim, *args, **kwargs): + super().__init__() + if 'groups' in kwargs: + # ignoring groups for Depthwise Sep Conv + del kwargs['groups'] + + self.depthwise = nn.Conv2d(in_dim, in_dim, *args, groups=in_dim, **kwargs) + self.pointwise = nn.Conv2d(in_dim, out_dim, kernel_size=1) + + def forward(self, x): + out = self.depthwise(x) + out = self.pointwise(out) + return out \ No newline at end of file diff --git a/annotator/lama/saicinpainting/training/modules/fake_fakes.py b/annotator/lama/saicinpainting/training/modules/fake_fakes.py new file mode 100644 index 0000000000000000000000000000000000000000..45c4ad559cef2730b771a709197e00ae1c87683c --- /dev/null +++ b/annotator/lama/saicinpainting/training/modules/fake_fakes.py @@ -0,0 +1,47 @@ +import torch +from kornia import SamplePadding +from kornia.augmentation import RandomAffine, CenterCrop + + +class FakeFakesGenerator: + def __init__(self, aug_proba=0.5, img_aug_degree=30, img_aug_translate=0.2): + self.grad_aug = RandomAffine(degrees=360, + translate=0.2, + padding_mode=SamplePadding.REFLECTION, + keepdim=False, + p=1) + self.img_aug = RandomAffine(degrees=img_aug_degree, + translate=img_aug_translate, + padding_mode=SamplePadding.REFLECTION, + keepdim=True, + p=1) + self.aug_proba = aug_proba + + def __call__(self, input_images, masks): + blend_masks = self._fill_masks_with_gradient(masks) + blend_target = self._make_blend_target(input_images) + result = input_images * (1 - blend_masks) + blend_target * blend_masks + return result, blend_masks + + def _make_blend_target(self, input_images): + batch_size = input_images.shape[0] + permuted = input_images[torch.randperm(batch_size)] + augmented = self.img_aug(input_images) + is_aug = (torch.rand(batch_size, device=input_images.device)[:, None, None, None] < self.aug_proba).float() + result = augmented * is_aug + permuted * (1 - is_aug) + return result + + def _fill_masks_with_gradient(self, masks): + batch_size, _, height, width = masks.shape + grad = torch.linspace(0, 1, steps=width * 2, device=masks.device, dtype=masks.dtype) \ + .view(1, 1, 1, -1).expand(batch_size, 1, height * 2, width * 2) + grad = self.grad_aug(grad) + grad = CenterCrop((height, width))(grad) + grad *= masks + + grad_for_min = grad + (1 - masks) * 10 + grad -= grad_for_min.view(batch_size, -1).min(-1).values[:, None, None, None] + grad /= grad.view(batch_size, -1).max(-1).values[:, None, None, None] + 1e-6 + grad.clamp_(min=0, max=1) + + return grad diff --git a/annotator/lama/saicinpainting/training/modules/ffc.py b/annotator/lama/saicinpainting/training/modules/ffc.py new file mode 100644 index 0000000000000000000000000000000000000000..e67ff9c832463e5518d6ccea2c6f27531ed778d4 --- /dev/null +++ b/annotator/lama/saicinpainting/training/modules/ffc.py @@ -0,0 +1,485 @@ +# Fast Fourier Convolution NeurIPS 2020 +# original implementation https://github.com/pkumivision/FFC/blob/main/model_zoo/ffc.py +# paper https://proceedings.neurips.cc/paper/2020/file/2fd5d41ec6cfab47e32164d5624269b1-Paper.pdf + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from annotator.lama.saicinpainting.training.modules.base import get_activation, BaseDiscriminator +from annotator.lama.saicinpainting.training.modules.spatial_transform import LearnableSpatialTransformWrapper +from annotator.lama.saicinpainting.training.modules.squeeze_excitation import SELayer +from annotator.lama.saicinpainting.utils import get_shape + + +class FFCSE_block(nn.Module): + + def __init__(self, channels, ratio_g): + super(FFCSE_block, self).__init__() + in_cg = int(channels * ratio_g) + in_cl = channels - in_cg + r = 16 + + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.conv1 = nn.Conv2d(channels, channels // r, + kernel_size=1, bias=True) + self.relu1 = nn.ReLU(inplace=True) + self.conv_a2l = None if in_cl == 0 else nn.Conv2d( + channels // r, in_cl, kernel_size=1, bias=True) + self.conv_a2g = None if in_cg == 0 else nn.Conv2d( + channels // r, in_cg, kernel_size=1, bias=True) + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + x = x if type(x) is tuple else (x, 0) + id_l, id_g = x + + x = id_l if type(id_g) is int else torch.cat([id_l, id_g], dim=1) + x = self.avgpool(x) + x = self.relu1(self.conv1(x)) + + x_l = 0 if self.conv_a2l is None else id_l * \ + self.sigmoid(self.conv_a2l(x)) + x_g = 0 if self.conv_a2g is None else id_g * \ + self.sigmoid(self.conv_a2g(x)) + return x_l, x_g + + +class FourierUnit(nn.Module): + + def __init__(self, in_channels, out_channels, groups=1, spatial_scale_factor=None, spatial_scale_mode='bilinear', + spectral_pos_encoding=False, use_se=False, se_kwargs=None, ffc3d=False, fft_norm='ortho'): + # bn_layer not used + super(FourierUnit, self).__init__() + self.groups = groups + + self.conv_layer = torch.nn.Conv2d(in_channels=in_channels * 2 + (2 if spectral_pos_encoding else 0), + out_channels=out_channels * 2, + kernel_size=1, stride=1, padding=0, groups=self.groups, bias=False) + self.bn = torch.nn.BatchNorm2d(out_channels * 2) + self.relu = torch.nn.ReLU(inplace=True) + + # squeeze and excitation block + self.use_se = use_se + if use_se: + if se_kwargs is None: + se_kwargs = {} + self.se = SELayer(self.conv_layer.in_channels, **se_kwargs) + + self.spatial_scale_factor = spatial_scale_factor + self.spatial_scale_mode = spatial_scale_mode + self.spectral_pos_encoding = spectral_pos_encoding + self.ffc3d = ffc3d + self.fft_norm = fft_norm + + def forward(self, x): + batch = x.shape[0] + + if self.spatial_scale_factor is not None: + orig_size = x.shape[-2:] + x = F.interpolate(x, scale_factor=self.spatial_scale_factor, mode=self.spatial_scale_mode, align_corners=False) + + r_size = x.size() + # (batch, c, h, w/2+1, 2) + fft_dim = (-3, -2, -1) if self.ffc3d else (-2, -1) + ffted = torch.fft.rfftn(x, dim=fft_dim, norm=self.fft_norm) + ffted = torch.stack((ffted.real, ffted.imag), dim=-1) + ffted = ffted.permute(0, 1, 4, 2, 3).contiguous() # (batch, c, 2, h, w/2+1) + ffted = ffted.view((batch, -1,) + ffted.size()[3:]) + + if self.spectral_pos_encoding: + height, width = ffted.shape[-2:] + coords_vert = torch.linspace(0, 1, height)[None, None, :, None].expand(batch, 1, height, width).to(ffted) + coords_hor = torch.linspace(0, 1, width)[None, None, None, :].expand(batch, 1, height, width).to(ffted) + ffted = torch.cat((coords_vert, coords_hor, ffted), dim=1) + + if self.use_se: + ffted = self.se(ffted) + + ffted = self.conv_layer(ffted) # (batch, c*2, h, w/2+1) + ffted = self.relu(self.bn(ffted)) + + ffted = ffted.view((batch, -1, 2,) + ffted.size()[2:]).permute( + 0, 1, 3, 4, 2).contiguous() # (batch,c, t, h, w/2+1, 2) + ffted = torch.complex(ffted[..., 0], ffted[..., 1]) + + ifft_shape_slice = x.shape[-3:] if self.ffc3d else x.shape[-2:] + output = torch.fft.irfftn(ffted, s=ifft_shape_slice, dim=fft_dim, norm=self.fft_norm) + + if self.spatial_scale_factor is not None: + output = F.interpolate(output, size=orig_size, mode=self.spatial_scale_mode, align_corners=False) + + return output + + +class SeparableFourierUnit(nn.Module): + + def __init__(self, in_channels, out_channels, groups=1, kernel_size=3): + # bn_layer not used + super(SeparableFourierUnit, self).__init__() + self.groups = groups + row_out_channels = out_channels // 2 + col_out_channels = out_channels - row_out_channels + self.row_conv = torch.nn.Conv2d(in_channels=in_channels * 2, + out_channels=row_out_channels * 2, + kernel_size=(kernel_size, 1), # kernel size is always like this, but the data will be transposed + stride=1, padding=(kernel_size // 2, 0), + padding_mode='reflect', + groups=self.groups, bias=False) + self.col_conv = torch.nn.Conv2d(in_channels=in_channels * 2, + out_channels=col_out_channels * 2, + kernel_size=(kernel_size, 1), # kernel size is always like this, but the data will be transposed + stride=1, padding=(kernel_size // 2, 0), + padding_mode='reflect', + groups=self.groups, bias=False) + self.row_bn = torch.nn.BatchNorm2d(row_out_channels * 2) + self.col_bn = torch.nn.BatchNorm2d(col_out_channels * 2) + self.relu = torch.nn.ReLU(inplace=True) + + def process_branch(self, x, conv, bn): + batch = x.shape[0] + + r_size = x.size() + # (batch, c, h, w/2+1, 2) + ffted = torch.fft.rfft(x, norm="ortho") + ffted = torch.stack((ffted.real, ffted.imag), dim=-1) + ffted = ffted.permute(0, 1, 4, 2, 3).contiguous() # (batch, c, 2, h, w/2+1) + ffted = ffted.view((batch, -1,) + ffted.size()[3:]) + + ffted = self.relu(bn(conv(ffted))) + + ffted = ffted.view((batch, -1, 2,) + ffted.size()[2:]).permute( + 0, 1, 3, 4, 2).contiguous() # (batch,c, t, h, w/2+1, 2) + ffted = torch.complex(ffted[..., 0], ffted[..., 1]) + + output = torch.fft.irfft(ffted, s=x.shape[-1:], norm="ortho") + return output + + + def forward(self, x): + rowwise = self.process_branch(x, self.row_conv, self.row_bn) + colwise = self.process_branch(x.permute(0, 1, 3, 2), self.col_conv, self.col_bn).permute(0, 1, 3, 2) + out = torch.cat((rowwise, colwise), dim=1) + return out + + +class SpectralTransform(nn.Module): + + def __init__(self, in_channels, out_channels, stride=1, groups=1, enable_lfu=True, separable_fu=False, **fu_kwargs): + # bn_layer not used + super(SpectralTransform, self).__init__() + self.enable_lfu = enable_lfu + if stride == 2: + self.downsample = nn.AvgPool2d(kernel_size=(2, 2), stride=2) + else: + self.downsample = nn.Identity() + + self.stride = stride + self.conv1 = nn.Sequential( + nn.Conv2d(in_channels, out_channels // + 2, kernel_size=1, groups=groups, bias=False), + nn.BatchNorm2d(out_channels // 2), + nn.ReLU(inplace=True) + ) + fu_class = SeparableFourierUnit if separable_fu else FourierUnit + self.fu = fu_class( + out_channels // 2, out_channels // 2, groups, **fu_kwargs) + if self.enable_lfu: + self.lfu = fu_class( + out_channels // 2, out_channels // 2, groups) + self.conv2 = torch.nn.Conv2d( + out_channels // 2, out_channels, kernel_size=1, groups=groups, bias=False) + + def forward(self, x): + + x = self.downsample(x) + x = self.conv1(x) + output = self.fu(x) + + if self.enable_lfu: + n, c, h, w = x.shape + split_no = 2 + split_s = h // split_no + xs = torch.cat(torch.split( + x[:, :c // 4], split_s, dim=-2), dim=1).contiguous() + xs = torch.cat(torch.split(xs, split_s, dim=-1), + dim=1).contiguous() + xs = self.lfu(xs) + xs = xs.repeat(1, 1, split_no, split_no).contiguous() + else: + xs = 0 + + output = self.conv2(x + output + xs) + + return output + + +class FFC(nn.Module): + + def __init__(self, in_channels, out_channels, kernel_size, + ratio_gin, ratio_gout, stride=1, padding=0, + dilation=1, groups=1, bias=False, enable_lfu=True, + padding_type='reflect', gated=False, **spectral_kwargs): + super(FFC, self).__init__() + + assert stride == 1 or stride == 2, "Stride should be 1 or 2." + self.stride = stride + + in_cg = int(in_channels * ratio_gin) + in_cl = in_channels - in_cg + out_cg = int(out_channels * ratio_gout) + out_cl = out_channels - out_cg + #groups_g = 1 if groups == 1 else int(groups * ratio_gout) + #groups_l = 1 if groups == 1 else groups - groups_g + + self.ratio_gin = ratio_gin + self.ratio_gout = ratio_gout + self.global_in_num = in_cg + + module = nn.Identity if in_cl == 0 or out_cl == 0 else nn.Conv2d + self.convl2l = module(in_cl, out_cl, kernel_size, + stride, padding, dilation, groups, bias, padding_mode=padding_type) + module = nn.Identity if in_cl == 0 or out_cg == 0 else nn.Conv2d + self.convl2g = module(in_cl, out_cg, kernel_size, + stride, padding, dilation, groups, bias, padding_mode=padding_type) + module = nn.Identity if in_cg == 0 or out_cl == 0 else nn.Conv2d + self.convg2l = module(in_cg, out_cl, kernel_size, + stride, padding, dilation, groups, bias, padding_mode=padding_type) + module = nn.Identity if in_cg == 0 or out_cg == 0 else SpectralTransform + self.convg2g = module( + in_cg, out_cg, stride, 1 if groups == 1 else groups // 2, enable_lfu, **spectral_kwargs) + + self.gated = gated + module = nn.Identity if in_cg == 0 or out_cl == 0 or not self.gated else nn.Conv2d + self.gate = module(in_channels, 2, 1) + + def forward(self, x): + x_l, x_g = x if type(x) is tuple else (x, 0) + out_xl, out_xg = 0, 0 + + if self.gated: + total_input_parts = [x_l] + if torch.is_tensor(x_g): + total_input_parts.append(x_g) + total_input = torch.cat(total_input_parts, dim=1) + + gates = torch.sigmoid(self.gate(total_input)) + g2l_gate, l2g_gate = gates.chunk(2, dim=1) + else: + g2l_gate, l2g_gate = 1, 1 + + if self.ratio_gout != 1: + out_xl = self.convl2l(x_l) + self.convg2l(x_g) * g2l_gate + if self.ratio_gout != 0: + out_xg = self.convl2g(x_l) * l2g_gate + self.convg2g(x_g) + + return out_xl, out_xg + + +class FFC_BN_ACT(nn.Module): + + def __init__(self, in_channels, out_channels, + kernel_size, ratio_gin, ratio_gout, + stride=1, padding=0, dilation=1, groups=1, bias=False, + norm_layer=nn.BatchNorm2d, activation_layer=nn.Identity, + padding_type='reflect', + enable_lfu=True, **kwargs): + super(FFC_BN_ACT, self).__init__() + self.ffc = FFC(in_channels, out_channels, kernel_size, + ratio_gin, ratio_gout, stride, padding, dilation, + groups, bias, enable_lfu, padding_type=padding_type, **kwargs) + lnorm = nn.Identity if ratio_gout == 1 else norm_layer + gnorm = nn.Identity if ratio_gout == 0 else norm_layer + global_channels = int(out_channels * ratio_gout) + self.bn_l = lnorm(out_channels - global_channels) + self.bn_g = gnorm(global_channels) + + lact = nn.Identity if ratio_gout == 1 else activation_layer + gact = nn.Identity if ratio_gout == 0 else activation_layer + self.act_l = lact(inplace=True) + self.act_g = gact(inplace=True) + + def forward(self, x): + x_l, x_g = self.ffc(x) + x_l = self.act_l(self.bn_l(x_l)) + x_g = self.act_g(self.bn_g(x_g)) + return x_l, x_g + + +class FFCResnetBlock(nn.Module): + def __init__(self, dim, padding_type, norm_layer, activation_layer=nn.ReLU, dilation=1, + spatial_transform_kwargs=None, inline=False, **conv_kwargs): + super().__init__() + self.conv1 = FFC_BN_ACT(dim, dim, kernel_size=3, padding=dilation, dilation=dilation, + norm_layer=norm_layer, + activation_layer=activation_layer, + padding_type=padding_type, + **conv_kwargs) + self.conv2 = FFC_BN_ACT(dim, dim, kernel_size=3, padding=dilation, dilation=dilation, + norm_layer=norm_layer, + activation_layer=activation_layer, + padding_type=padding_type, + **conv_kwargs) + if spatial_transform_kwargs is not None: + self.conv1 = LearnableSpatialTransformWrapper(self.conv1, **spatial_transform_kwargs) + self.conv2 = LearnableSpatialTransformWrapper(self.conv2, **spatial_transform_kwargs) + self.inline = inline + + def forward(self, x): + if self.inline: + x_l, x_g = x[:, :-self.conv1.ffc.global_in_num], x[:, -self.conv1.ffc.global_in_num:] + else: + x_l, x_g = x if type(x) is tuple else (x, 0) + + id_l, id_g = x_l, x_g + + x_l, x_g = self.conv1((x_l, x_g)) + x_l, x_g = self.conv2((x_l, x_g)) + + x_l, x_g = id_l + x_l, id_g + x_g + out = x_l, x_g + if self.inline: + out = torch.cat(out, dim=1) + return out + + +class ConcatTupleLayer(nn.Module): + def forward(self, x): + assert isinstance(x, tuple) + x_l, x_g = x + assert torch.is_tensor(x_l) or torch.is_tensor(x_g) + if not torch.is_tensor(x_g): + return x_l + return torch.cat(x, dim=1) + + +class FFCResNetGenerator(nn.Module): + def __init__(self, input_nc, output_nc, ngf=64, n_downsampling=3, n_blocks=9, norm_layer=nn.BatchNorm2d, + padding_type='reflect', activation_layer=nn.ReLU, + up_norm_layer=nn.BatchNorm2d, up_activation=nn.ReLU(True), + init_conv_kwargs={}, downsample_conv_kwargs={}, resnet_conv_kwargs={}, + spatial_transform_layers=None, spatial_transform_kwargs={}, + add_out_act=True, max_features=1024, out_ffc=False, out_ffc_kwargs={}): + assert (n_blocks >= 0) + super().__init__() + + model = [nn.ReflectionPad2d(3), + FFC_BN_ACT(input_nc, ngf, kernel_size=7, padding=0, norm_layer=norm_layer, + activation_layer=activation_layer, **init_conv_kwargs)] + + ### downsample + for i in range(n_downsampling): + mult = 2 ** i + if i == n_downsampling - 1: + cur_conv_kwargs = dict(downsample_conv_kwargs) + cur_conv_kwargs['ratio_gout'] = resnet_conv_kwargs.get('ratio_gin', 0) + else: + cur_conv_kwargs = downsample_conv_kwargs + model += [FFC_BN_ACT(min(max_features, ngf * mult), + min(max_features, ngf * mult * 2), + kernel_size=3, stride=2, padding=1, + norm_layer=norm_layer, + activation_layer=activation_layer, + **cur_conv_kwargs)] + + mult = 2 ** n_downsampling + feats_num_bottleneck = min(max_features, ngf * mult) + + ### resnet blocks + for i in range(n_blocks): + cur_resblock = FFCResnetBlock(feats_num_bottleneck, padding_type=padding_type, activation_layer=activation_layer, + norm_layer=norm_layer, **resnet_conv_kwargs) + if spatial_transform_layers is not None and i in spatial_transform_layers: + cur_resblock = LearnableSpatialTransformWrapper(cur_resblock, **spatial_transform_kwargs) + model += [cur_resblock] + + model += [ConcatTupleLayer()] + + ### upsample + for i in range(n_downsampling): + mult = 2 ** (n_downsampling - i) + model += [nn.ConvTranspose2d(min(max_features, ngf * mult), + min(max_features, int(ngf * mult / 2)), + kernel_size=3, stride=2, padding=1, output_padding=1), + up_norm_layer(min(max_features, int(ngf * mult / 2))), + up_activation] + + if out_ffc: + model += [FFCResnetBlock(ngf, padding_type=padding_type, activation_layer=activation_layer, + norm_layer=norm_layer, inline=True, **out_ffc_kwargs)] + + model += [nn.ReflectionPad2d(3), + nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0)] + if add_out_act: + model.append(get_activation('tanh' if add_out_act is True else add_out_act)) + self.model = nn.Sequential(*model) + + def forward(self, input): + return self.model(input) + + +class FFCNLayerDiscriminator(BaseDiscriminator): + def __init__(self, input_nc, ndf=64, n_layers=3, norm_layer=nn.BatchNorm2d, max_features=512, + init_conv_kwargs={}, conv_kwargs={}): + super().__init__() + self.n_layers = n_layers + + def _act_ctor(inplace=True): + return nn.LeakyReLU(negative_slope=0.2, inplace=inplace) + + kw = 3 + padw = int(np.ceil((kw-1.0)/2)) + sequence = [[FFC_BN_ACT(input_nc, ndf, kernel_size=kw, padding=padw, norm_layer=norm_layer, + activation_layer=_act_ctor, **init_conv_kwargs)]] + + nf = ndf + for n in range(1, n_layers): + nf_prev = nf + nf = min(nf * 2, max_features) + + cur_model = [ + FFC_BN_ACT(nf_prev, nf, + kernel_size=kw, stride=2, padding=padw, + norm_layer=norm_layer, + activation_layer=_act_ctor, + **conv_kwargs) + ] + sequence.append(cur_model) + + nf_prev = nf + nf = min(nf * 2, 512) + + cur_model = [ + FFC_BN_ACT(nf_prev, nf, + kernel_size=kw, stride=1, padding=padw, + norm_layer=norm_layer, + activation_layer=lambda *args, **kwargs: nn.LeakyReLU(*args, negative_slope=0.2, **kwargs), + **conv_kwargs), + ConcatTupleLayer() + ] + sequence.append(cur_model) + + sequence += [[nn.Conv2d(nf, 1, kernel_size=kw, stride=1, padding=padw)]] + + for n in range(len(sequence)): + setattr(self, 'model'+str(n), nn.Sequential(*sequence[n])) + + def get_all_activations(self, x): + res = [x] + for n in range(self.n_layers + 2): + model = getattr(self, 'model' + str(n)) + res.append(model(res[-1])) + return res[1:] + + def forward(self, x): + act = self.get_all_activations(x) + feats = [] + for out in act[:-1]: + if isinstance(out, tuple): + if torch.is_tensor(out[1]): + out = torch.cat(out, dim=1) + else: + out = out[0] + feats.append(out) + return act[-1], feats diff --git a/annotator/lama/saicinpainting/training/modules/multidilated_conv.py b/annotator/lama/saicinpainting/training/modules/multidilated_conv.py new file mode 100644 index 0000000000000000000000000000000000000000..c57d0b457d4b30aeeffcd8cba138a502ba7affc5 --- /dev/null +++ b/annotator/lama/saicinpainting/training/modules/multidilated_conv.py @@ -0,0 +1,98 @@ +import torch +import torch.nn as nn +import random +from annotator.lama.saicinpainting.training.modules.depthwise_sep_conv import DepthWiseSeperableConv + +class MultidilatedConv(nn.Module): + def __init__(self, in_dim, out_dim, kernel_size, dilation_num=3, comb_mode='sum', equal_dim=True, + shared_weights=False, padding=1, min_dilation=1, shuffle_in_channels=False, use_depthwise=False, **kwargs): + super().__init__() + convs = [] + self.equal_dim = equal_dim + assert comb_mode in ('cat_out', 'sum', 'cat_in', 'cat_both'), comb_mode + if comb_mode in ('cat_out', 'cat_both'): + self.cat_out = True + if equal_dim: + assert out_dim % dilation_num == 0 + out_dims = [out_dim // dilation_num] * dilation_num + self.index = sum([[i + j * (out_dims[0]) for j in range(dilation_num)] for i in range(out_dims[0])], []) + else: + out_dims = [out_dim // 2 ** (i + 1) for i in range(dilation_num - 1)] + out_dims.append(out_dim - sum(out_dims)) + index = [] + starts = [0] + out_dims[:-1] + lengths = [out_dims[i] // out_dims[-1] for i in range(dilation_num)] + for i in range(out_dims[-1]): + for j in range(dilation_num): + index += list(range(starts[j], starts[j] + lengths[j])) + starts[j] += lengths[j] + self.index = index + assert(len(index) == out_dim) + self.out_dims = out_dims + else: + self.cat_out = False + self.out_dims = [out_dim] * dilation_num + + if comb_mode in ('cat_in', 'cat_both'): + if equal_dim: + assert in_dim % dilation_num == 0 + in_dims = [in_dim // dilation_num] * dilation_num + else: + in_dims = [in_dim // 2 ** (i + 1) for i in range(dilation_num - 1)] + in_dims.append(in_dim - sum(in_dims)) + self.in_dims = in_dims + self.cat_in = True + else: + self.cat_in = False + self.in_dims = [in_dim] * dilation_num + + conv_type = DepthWiseSeperableConv if use_depthwise else nn.Conv2d + dilation = min_dilation + for i in range(dilation_num): + if isinstance(padding, int): + cur_padding = padding * dilation + else: + cur_padding = padding[i] + convs.append(conv_type( + self.in_dims[i], self.out_dims[i], kernel_size, padding=cur_padding, dilation=dilation, **kwargs + )) + if i > 0 and shared_weights: + convs[-1].weight = convs[0].weight + convs[-1].bias = convs[0].bias + dilation *= 2 + self.convs = nn.ModuleList(convs) + + self.shuffle_in_channels = shuffle_in_channels + if self.shuffle_in_channels: + # shuffle list as shuffling of tensors is nondeterministic + in_channels_permute = list(range(in_dim)) + random.shuffle(in_channels_permute) + # save as buffer so it is saved and loaded with checkpoint + self.register_buffer('in_channels_permute', torch.tensor(in_channels_permute)) + + def forward(self, x): + if self.shuffle_in_channels: + x = x[:, self.in_channels_permute] + + outs = [] + if self.cat_in: + if self.equal_dim: + x = x.chunk(len(self.convs), dim=1) + else: + new_x = [] + start = 0 + for dim in self.in_dims: + new_x.append(x[:, start:start+dim]) + start += dim + x = new_x + for i, conv in enumerate(self.convs): + if self.cat_in: + input = x[i] + else: + input = x + outs.append(conv(input)) + if self.cat_out: + out = torch.cat(outs, dim=1)[:, self.index] + else: + out = sum(outs) + return out diff --git a/annotator/lama/saicinpainting/training/modules/multiscale.py b/annotator/lama/saicinpainting/training/modules/multiscale.py new file mode 100644 index 0000000000000000000000000000000000000000..3f41252f3c7509ee58b939215baef328cfbe48c8 --- /dev/null +++ b/annotator/lama/saicinpainting/training/modules/multiscale.py @@ -0,0 +1,244 @@ +from typing import List, Tuple, Union, Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from annotator.lama.saicinpainting.training.modules.base import get_conv_block_ctor, get_activation +from annotator.lama.saicinpainting.training.modules.pix2pixhd import ResnetBlock + + +class ResNetHead(nn.Module): + def __init__(self, input_nc, ngf=64, n_downsampling=3, n_blocks=9, norm_layer=nn.BatchNorm2d, + padding_type='reflect', conv_kind='default', activation=nn.ReLU(True)): + assert (n_blocks >= 0) + super(ResNetHead, self).__init__() + + conv_layer = get_conv_block_ctor(conv_kind) + + model = [nn.ReflectionPad2d(3), + conv_layer(input_nc, ngf, kernel_size=7, padding=0), + norm_layer(ngf), + activation] + + ### downsample + for i in range(n_downsampling): + mult = 2 ** i + model += [conv_layer(ngf * mult, ngf * mult * 2, kernel_size=3, stride=2, padding=1), + norm_layer(ngf * mult * 2), + activation] + + mult = 2 ** n_downsampling + + ### resnet blocks + for i in range(n_blocks): + model += [ResnetBlock(ngf * mult, padding_type=padding_type, activation=activation, norm_layer=norm_layer, + conv_kind=conv_kind)] + + self.model = nn.Sequential(*model) + + def forward(self, input): + return self.model(input) + + +class ResNetTail(nn.Module): + def __init__(self, output_nc, ngf=64, n_downsampling=3, n_blocks=9, norm_layer=nn.BatchNorm2d, + padding_type='reflect', conv_kind='default', activation=nn.ReLU(True), + up_norm_layer=nn.BatchNorm2d, up_activation=nn.ReLU(True), add_out_act=False, out_extra_layers_n=0, + add_in_proj=None): + assert (n_blocks >= 0) + super(ResNetTail, self).__init__() + + mult = 2 ** n_downsampling + + model = [] + + if add_in_proj is not None: + model.append(nn.Conv2d(add_in_proj, ngf * mult, kernel_size=1)) + + ### resnet blocks + for i in range(n_blocks): + model += [ResnetBlock(ngf * mult, padding_type=padding_type, activation=activation, norm_layer=norm_layer, + conv_kind=conv_kind)] + + ### upsample + for i in range(n_downsampling): + mult = 2 ** (n_downsampling - i) + model += [nn.ConvTranspose2d(ngf * mult, int(ngf * mult / 2), kernel_size=3, stride=2, padding=1, + output_padding=1), + up_norm_layer(int(ngf * mult / 2)), + up_activation] + self.model = nn.Sequential(*model) + + out_layers = [] + for _ in range(out_extra_layers_n): + out_layers += [nn.Conv2d(ngf, ngf, kernel_size=1, padding=0), + up_norm_layer(ngf), + up_activation] + out_layers += [nn.ReflectionPad2d(3), + nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0)] + + if add_out_act: + out_layers.append(get_activation('tanh' if add_out_act is True else add_out_act)) + + self.out_proj = nn.Sequential(*out_layers) + + def forward(self, input, return_last_act=False): + features = self.model(input) + out = self.out_proj(features) + if return_last_act: + return out, features + else: + return out + + +class MultiscaleResNet(nn.Module): + def __init__(self, input_nc, output_nc, ngf=64, n_downsampling=2, n_blocks_head=2, n_blocks_tail=6, n_scales=3, + norm_layer=nn.BatchNorm2d, padding_type='reflect', conv_kind='default', activation=nn.ReLU(True), + up_norm_layer=nn.BatchNorm2d, up_activation=nn.ReLU(True), add_out_act=False, out_extra_layers_n=0, + out_cumulative=False, return_only_hr=False): + super().__init__() + + self.heads = nn.ModuleList([ResNetHead(input_nc, ngf=ngf, n_downsampling=n_downsampling, + n_blocks=n_blocks_head, norm_layer=norm_layer, padding_type=padding_type, + conv_kind=conv_kind, activation=activation) + for i in range(n_scales)]) + tail_in_feats = ngf * (2 ** n_downsampling) + ngf + self.tails = nn.ModuleList([ResNetTail(output_nc, + ngf=ngf, n_downsampling=n_downsampling, + n_blocks=n_blocks_tail, norm_layer=norm_layer, padding_type=padding_type, + conv_kind=conv_kind, activation=activation, up_norm_layer=up_norm_layer, + up_activation=up_activation, add_out_act=add_out_act, + out_extra_layers_n=out_extra_layers_n, + add_in_proj=None if (i == n_scales - 1) else tail_in_feats) + for i in range(n_scales)]) + + self.out_cumulative = out_cumulative + self.return_only_hr = return_only_hr + + @property + def num_scales(self): + return len(self.heads) + + def forward(self, ms_inputs: List[torch.Tensor], smallest_scales_num: Optional[int] = None) \ + -> Union[torch.Tensor, List[torch.Tensor]]: + """ + :param ms_inputs: List of inputs of different resolutions from HR to LR + :param smallest_scales_num: int or None, number of smallest scales to take at input + :return: Depending on return_only_hr: + True: Only the most HR output + False: List of outputs of different resolutions from HR to LR + """ + if smallest_scales_num is None: + assert len(self.heads) == len(ms_inputs), (len(self.heads), len(ms_inputs), smallest_scales_num) + smallest_scales_num = len(self.heads) + else: + assert smallest_scales_num == len(ms_inputs) <= len(self.heads), (len(self.heads), len(ms_inputs), smallest_scales_num) + + cur_heads = self.heads[-smallest_scales_num:] + ms_features = [cur_head(cur_inp) for cur_head, cur_inp in zip(cur_heads, ms_inputs)] + + all_outputs = [] + prev_tail_features = None + for i in range(len(ms_features)): + scale_i = -i - 1 + + cur_tail_input = ms_features[-i - 1] + if prev_tail_features is not None: + if prev_tail_features.shape != cur_tail_input.shape: + prev_tail_features = F.interpolate(prev_tail_features, size=cur_tail_input.shape[2:], + mode='bilinear', align_corners=False) + cur_tail_input = torch.cat((cur_tail_input, prev_tail_features), dim=1) + + cur_out, cur_tail_feats = self.tails[scale_i](cur_tail_input, return_last_act=True) + + prev_tail_features = cur_tail_feats + all_outputs.append(cur_out) + + if self.out_cumulative: + all_outputs_cum = [all_outputs[0]] + for i in range(1, len(ms_features)): + cur_out = all_outputs[i] + cur_out_cum = cur_out + F.interpolate(all_outputs_cum[-1], size=cur_out.shape[2:], + mode='bilinear', align_corners=False) + all_outputs_cum.append(cur_out_cum) + all_outputs = all_outputs_cum + + if self.return_only_hr: + return all_outputs[-1] + else: + return all_outputs[::-1] + + +class MultiscaleDiscriminatorSimple(nn.Module): + def __init__(self, ms_impl): + super().__init__() + self.ms_impl = nn.ModuleList(ms_impl) + + @property + def num_scales(self): + return len(self.ms_impl) + + def forward(self, ms_inputs: List[torch.Tensor], smallest_scales_num: Optional[int] = None) \ + -> List[Tuple[torch.Tensor, List[torch.Tensor]]]: + """ + :param ms_inputs: List of inputs of different resolutions from HR to LR + :param smallest_scales_num: int or None, number of smallest scales to take at input + :return: List of pairs (prediction, features) for different resolutions from HR to LR + """ + if smallest_scales_num is None: + assert len(self.ms_impl) == len(ms_inputs), (len(self.ms_impl), len(ms_inputs), smallest_scales_num) + smallest_scales_num = len(self.heads) + else: + assert smallest_scales_num == len(ms_inputs) <= len(self.ms_impl), \ + (len(self.ms_impl), len(ms_inputs), smallest_scales_num) + + return [cur_discr(cur_input) for cur_discr, cur_input in zip(self.ms_impl[-smallest_scales_num:], ms_inputs)] + + +class SingleToMultiScaleInputMixin: + def forward(self, x: torch.Tensor) -> List: + orig_height, orig_width = x.shape[2:] + factors = [2 ** i for i in range(self.num_scales)] + ms_inputs = [F.interpolate(x, size=(orig_height // f, orig_width // f), mode='bilinear', align_corners=False) + for f in factors] + return super().forward(ms_inputs) + + +class GeneratorMultiToSingleOutputMixin: + def forward(self, x): + return super().forward(x)[0] + + +class DiscriminatorMultiToSingleOutputMixin: + def forward(self, x): + out_feat_tuples = super().forward(x) + return out_feat_tuples[0][0], [f for _, flist in out_feat_tuples for f in flist] + + +class DiscriminatorMultiToSingleOutputStackedMixin: + def __init__(self, *args, return_feats_only_levels=None, **kwargs): + super().__init__(*args, **kwargs) + self.return_feats_only_levels = return_feats_only_levels + + def forward(self, x): + out_feat_tuples = super().forward(x) + outs = [out for out, _ in out_feat_tuples] + scaled_outs = [outs[0]] + [F.interpolate(cur_out, size=outs[0].shape[-2:], + mode='bilinear', align_corners=False) + for cur_out in outs[1:]] + out = torch.cat(scaled_outs, dim=1) + if self.return_feats_only_levels is not None: + feat_lists = [out_feat_tuples[i][1] for i in self.return_feats_only_levels] + else: + feat_lists = [flist for _, flist in out_feat_tuples] + feats = [f for flist in feat_lists for f in flist] + return out, feats + + +class MultiscaleDiscrSingleInput(SingleToMultiScaleInputMixin, DiscriminatorMultiToSingleOutputStackedMixin, MultiscaleDiscriminatorSimple): + pass + + +class MultiscaleResNetSingle(GeneratorMultiToSingleOutputMixin, SingleToMultiScaleInputMixin, MultiscaleResNet): + pass diff --git a/annotator/lama/saicinpainting/training/modules/pix2pixhd.py b/annotator/lama/saicinpainting/training/modules/pix2pixhd.py new file mode 100644 index 0000000000000000000000000000000000000000..2e4fcfcff083f9ce4d3c7880ff0f74f8f745a251 --- /dev/null +++ b/annotator/lama/saicinpainting/training/modules/pix2pixhd.py @@ -0,0 +1,669 @@ +# original: https://github.com/NVIDIA/pix2pixHD/blob/master/models/networks.py +import collections +from functools import partial +import functools +import logging +from collections import defaultdict + +import numpy as np +import torch.nn as nn + +from annotator.lama.saicinpainting.training.modules.base import BaseDiscriminator, deconv_factory, get_conv_block_ctor, get_norm_layer, get_activation +from annotator.lama.saicinpainting.training.modules.ffc import FFCResnetBlock +from annotator.lama.saicinpainting.training.modules.multidilated_conv import MultidilatedConv + +class DotDict(defaultdict): + # https://stackoverflow.com/questions/2352181/how-to-use-a-dot-to-access-members-of-dictionary + """dot.notation access to dictionary attributes""" + __getattr__ = defaultdict.get + __setattr__ = defaultdict.__setitem__ + __delattr__ = defaultdict.__delitem__ + +class Identity(nn.Module): + def __init__(self): + super().__init__() + + def forward(self, x): + return x + + +class ResnetBlock(nn.Module): + def __init__(self, dim, padding_type, norm_layer, activation=nn.ReLU(True), use_dropout=False, conv_kind='default', + dilation=1, in_dim=None, groups=1, second_dilation=None): + super(ResnetBlock, self).__init__() + self.in_dim = in_dim + self.dim = dim + if second_dilation is None: + second_dilation = dilation + self.conv_block = self.build_conv_block(dim, padding_type, norm_layer, activation, use_dropout, + conv_kind=conv_kind, dilation=dilation, in_dim=in_dim, groups=groups, + second_dilation=second_dilation) + + if self.in_dim is not None: + self.input_conv = nn.Conv2d(in_dim, dim, 1) + + self.out_channnels = dim + + def build_conv_block(self, dim, padding_type, norm_layer, activation, use_dropout, conv_kind='default', + dilation=1, in_dim=None, groups=1, second_dilation=1): + conv_layer = get_conv_block_ctor(conv_kind) + + conv_block = [] + p = 0 + if padding_type == 'reflect': + conv_block += [nn.ReflectionPad2d(dilation)] + elif padding_type == 'replicate': + conv_block += [nn.ReplicationPad2d(dilation)] + elif padding_type == 'zero': + p = dilation + else: + raise NotImplementedError('padding [%s] is not implemented' % padding_type) + + if in_dim is None: + in_dim = dim + + conv_block += [conv_layer(in_dim, dim, kernel_size=3, padding=p, dilation=dilation), + norm_layer(dim), + activation] + if use_dropout: + conv_block += [nn.Dropout(0.5)] + + p = 0 + if padding_type == 'reflect': + conv_block += [nn.ReflectionPad2d(second_dilation)] + elif padding_type == 'replicate': + conv_block += [nn.ReplicationPad2d(second_dilation)] + elif padding_type == 'zero': + p = second_dilation + else: + raise NotImplementedError('padding [%s] is not implemented' % padding_type) + conv_block += [conv_layer(dim, dim, kernel_size=3, padding=p, dilation=second_dilation, groups=groups), + norm_layer(dim)] + + return nn.Sequential(*conv_block) + + def forward(self, x): + x_before = x + if self.in_dim is not None: + x = self.input_conv(x) + out = x + self.conv_block(x_before) + return out + +class ResnetBlock5x5(nn.Module): + def __init__(self, dim, padding_type, norm_layer, activation=nn.ReLU(True), use_dropout=False, conv_kind='default', + dilation=1, in_dim=None, groups=1, second_dilation=None): + super(ResnetBlock5x5, self).__init__() + self.in_dim = in_dim + self.dim = dim + if second_dilation is None: + second_dilation = dilation + self.conv_block = self.build_conv_block(dim, padding_type, norm_layer, activation, use_dropout, + conv_kind=conv_kind, dilation=dilation, in_dim=in_dim, groups=groups, + second_dilation=second_dilation) + + if self.in_dim is not None: + self.input_conv = nn.Conv2d(in_dim, dim, 1) + + self.out_channnels = dim + + def build_conv_block(self, dim, padding_type, norm_layer, activation, use_dropout, conv_kind='default', + dilation=1, in_dim=None, groups=1, second_dilation=1): + conv_layer = get_conv_block_ctor(conv_kind) + + conv_block = [] + p = 0 + if padding_type == 'reflect': + conv_block += [nn.ReflectionPad2d(dilation * 2)] + elif padding_type == 'replicate': + conv_block += [nn.ReplicationPad2d(dilation * 2)] + elif padding_type == 'zero': + p = dilation * 2 + else: + raise NotImplementedError('padding [%s] is not implemented' % padding_type) + + if in_dim is None: + in_dim = dim + + conv_block += [conv_layer(in_dim, dim, kernel_size=5, padding=p, dilation=dilation), + norm_layer(dim), + activation] + if use_dropout: + conv_block += [nn.Dropout(0.5)] + + p = 0 + if padding_type == 'reflect': + conv_block += [nn.ReflectionPad2d(second_dilation * 2)] + elif padding_type == 'replicate': + conv_block += [nn.ReplicationPad2d(second_dilation * 2)] + elif padding_type == 'zero': + p = second_dilation * 2 + else: + raise NotImplementedError('padding [%s] is not implemented' % padding_type) + conv_block += [conv_layer(dim, dim, kernel_size=5, padding=p, dilation=second_dilation, groups=groups), + norm_layer(dim)] + + return nn.Sequential(*conv_block) + + def forward(self, x): + x_before = x + if self.in_dim is not None: + x = self.input_conv(x) + out = x + self.conv_block(x_before) + return out + + +class MultidilatedResnetBlock(nn.Module): + def __init__(self, dim, padding_type, conv_layer, norm_layer, activation=nn.ReLU(True), use_dropout=False): + super().__init__() + self.conv_block = self.build_conv_block(dim, padding_type, conv_layer, norm_layer, activation, use_dropout) + + def build_conv_block(self, dim, padding_type, conv_layer, norm_layer, activation, use_dropout, dilation=1): + conv_block = [] + conv_block += [conv_layer(dim, dim, kernel_size=3, padding_mode=padding_type), + norm_layer(dim), + activation] + if use_dropout: + conv_block += [nn.Dropout(0.5)] + + conv_block += [conv_layer(dim, dim, kernel_size=3, padding_mode=padding_type), + norm_layer(dim)] + + return nn.Sequential(*conv_block) + + def forward(self, x): + out = x + self.conv_block(x) + return out + + +class MultiDilatedGlobalGenerator(nn.Module): + def __init__(self, input_nc, output_nc, ngf=64, n_downsampling=3, + n_blocks=3, norm_layer=nn.BatchNorm2d, + padding_type='reflect', conv_kind='default', + deconv_kind='convtranspose', activation=nn.ReLU(True), + up_norm_layer=nn.BatchNorm2d, affine=None, up_activation=nn.ReLU(True), + add_out_act=True, max_features=1024, multidilation_kwargs={}, + ffc_positions=None, ffc_kwargs={}): + assert (n_blocks >= 0) + super().__init__() + + conv_layer = get_conv_block_ctor(conv_kind) + resnet_conv_layer = functools.partial(get_conv_block_ctor('multidilated'), **multidilation_kwargs) + norm_layer = get_norm_layer(norm_layer) + if affine is not None: + norm_layer = partial(norm_layer, affine=affine) + up_norm_layer = get_norm_layer(up_norm_layer) + if affine is not None: + up_norm_layer = partial(up_norm_layer, affine=affine) + + model = [nn.ReflectionPad2d(3), + conv_layer(input_nc, ngf, kernel_size=7, padding=0), + norm_layer(ngf), + activation] + + identity = Identity() + ### downsample + for i in range(n_downsampling): + mult = 2 ** i + + model += [conv_layer(min(max_features, ngf * mult), + min(max_features, ngf * mult * 2), + kernel_size=3, stride=2, padding=1), + norm_layer(min(max_features, ngf * mult * 2)), + activation] + + mult = 2 ** n_downsampling + feats_num_bottleneck = min(max_features, ngf * mult) + + ### resnet blocks + for i in range(n_blocks): + if ffc_positions is not None and i in ffc_positions: + model += [FFCResnetBlock(feats_num_bottleneck, padding_type, norm_layer, activation_layer=nn.ReLU, + inline=True, **ffc_kwargs)] + model += [MultidilatedResnetBlock(feats_num_bottleneck, padding_type=padding_type, + conv_layer=resnet_conv_layer, activation=activation, + norm_layer=norm_layer)] + + ### upsample + for i in range(n_downsampling): + mult = 2 ** (n_downsampling - i) + model += deconv_factory(deconv_kind, ngf, mult, up_norm_layer, up_activation, max_features) + model += [nn.ReflectionPad2d(3), + nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0)] + if add_out_act: + model.append(get_activation('tanh' if add_out_act is True else add_out_act)) + self.model = nn.Sequential(*model) + + def forward(self, input): + return self.model(input) + +class ConfigGlobalGenerator(nn.Module): + def __init__(self, input_nc, output_nc, ngf=64, n_downsampling=3, + n_blocks=3, norm_layer=nn.BatchNorm2d, + padding_type='reflect', conv_kind='default', + deconv_kind='convtranspose', activation=nn.ReLU(True), + up_norm_layer=nn.BatchNorm2d, affine=None, up_activation=nn.ReLU(True), + add_out_act=True, max_features=1024, + manual_block_spec=[], + resnet_block_kind='multidilatedresnetblock', + resnet_conv_kind='multidilated', + resnet_dilation=1, + multidilation_kwargs={}): + assert (n_blocks >= 0) + super().__init__() + + conv_layer = get_conv_block_ctor(conv_kind) + resnet_conv_layer = functools.partial(get_conv_block_ctor(resnet_conv_kind), **multidilation_kwargs) + norm_layer = get_norm_layer(norm_layer) + if affine is not None: + norm_layer = partial(norm_layer, affine=affine) + up_norm_layer = get_norm_layer(up_norm_layer) + if affine is not None: + up_norm_layer = partial(up_norm_layer, affine=affine) + + model = [nn.ReflectionPad2d(3), + conv_layer(input_nc, ngf, kernel_size=7, padding=0), + norm_layer(ngf), + activation] + + identity = Identity() + + ### downsample + for i in range(n_downsampling): + mult = 2 ** i + model += [conv_layer(min(max_features, ngf * mult), + min(max_features, ngf * mult * 2), + kernel_size=3, stride=2, padding=1), + norm_layer(min(max_features, ngf * mult * 2)), + activation] + + mult = 2 ** n_downsampling + feats_num_bottleneck = min(max_features, ngf * mult) + + if len(manual_block_spec) == 0: + manual_block_spec = [ + DotDict(lambda : None, { + 'n_blocks': n_blocks, + 'use_default': True}) + ] + + ### resnet blocks + for block_spec in manual_block_spec: + def make_and_add_blocks(model, block_spec): + block_spec = DotDict(lambda : None, block_spec) + if not block_spec.use_default: + resnet_conv_layer = functools.partial(get_conv_block_ctor(block_spec.resnet_conv_kind), **block_spec.multidilation_kwargs) + resnet_conv_kind = block_spec.resnet_conv_kind + resnet_block_kind = block_spec.resnet_block_kind + if block_spec.resnet_dilation is not None: + resnet_dilation = block_spec.resnet_dilation + for i in range(block_spec.n_blocks): + if resnet_block_kind == "multidilatedresnetblock": + model += [MultidilatedResnetBlock(feats_num_bottleneck, padding_type=padding_type, + conv_layer=resnet_conv_layer, activation=activation, + norm_layer=norm_layer)] + if resnet_block_kind == "resnetblock": + model += [ResnetBlock(ngf * mult, padding_type=padding_type, activation=activation, norm_layer=norm_layer, + conv_kind=resnet_conv_kind)] + if resnet_block_kind == "resnetblock5x5": + model += [ResnetBlock5x5(ngf * mult, padding_type=padding_type, activation=activation, norm_layer=norm_layer, + conv_kind=resnet_conv_kind)] + if resnet_block_kind == "resnetblockdwdil": + model += [ResnetBlock(ngf * mult, padding_type=padding_type, activation=activation, norm_layer=norm_layer, + conv_kind=resnet_conv_kind, dilation=resnet_dilation, second_dilation=resnet_dilation)] + make_and_add_blocks(model, block_spec) + + ### upsample + for i in range(n_downsampling): + mult = 2 ** (n_downsampling - i) + model += deconv_factory(deconv_kind, ngf, mult, up_norm_layer, up_activation, max_features) + model += [nn.ReflectionPad2d(3), + nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0)] + if add_out_act: + model.append(get_activation('tanh' if add_out_act is True else add_out_act)) + self.model = nn.Sequential(*model) + + def forward(self, input): + return self.model(input) + + +def make_dil_blocks(dilated_blocks_n, dilation_block_kind, dilated_block_kwargs): + blocks = [] + for i in range(dilated_blocks_n): + if dilation_block_kind == 'simple': + blocks.append(ResnetBlock(**dilated_block_kwargs, dilation=2 ** (i + 1))) + elif dilation_block_kind == 'multi': + blocks.append(MultidilatedResnetBlock(**dilated_block_kwargs)) + else: + raise ValueError(f'dilation_block_kind could not be "{dilation_block_kind}"') + return blocks + + +class GlobalGenerator(nn.Module): + def __init__(self, input_nc, output_nc, ngf=64, n_downsampling=3, n_blocks=9, norm_layer=nn.BatchNorm2d, + padding_type='reflect', conv_kind='default', activation=nn.ReLU(True), + up_norm_layer=nn.BatchNorm2d, affine=None, + up_activation=nn.ReLU(True), dilated_blocks_n=0, dilated_blocks_n_start=0, + dilated_blocks_n_middle=0, + add_out_act=True, + max_features=1024, is_resblock_depthwise=False, + ffc_positions=None, ffc_kwargs={}, dilation=1, second_dilation=None, + dilation_block_kind='simple', multidilation_kwargs={}): + assert (n_blocks >= 0) + super().__init__() + + conv_layer = get_conv_block_ctor(conv_kind) + norm_layer = get_norm_layer(norm_layer) + if affine is not None: + norm_layer = partial(norm_layer, affine=affine) + up_norm_layer = get_norm_layer(up_norm_layer) + if affine is not None: + up_norm_layer = partial(up_norm_layer, affine=affine) + + if ffc_positions is not None: + ffc_positions = collections.Counter(ffc_positions) + + model = [nn.ReflectionPad2d(3), + conv_layer(input_nc, ngf, kernel_size=7, padding=0), + norm_layer(ngf), + activation] + + identity = Identity() + ### downsample + for i in range(n_downsampling): + mult = 2 ** i + + model += [conv_layer(min(max_features, ngf * mult), + min(max_features, ngf * mult * 2), + kernel_size=3, stride=2, padding=1), + norm_layer(min(max_features, ngf * mult * 2)), + activation] + + mult = 2 ** n_downsampling + feats_num_bottleneck = min(max_features, ngf * mult) + + dilated_block_kwargs = dict(dim=feats_num_bottleneck, padding_type=padding_type, + activation=activation, norm_layer=norm_layer) + if dilation_block_kind == 'simple': + dilated_block_kwargs['conv_kind'] = conv_kind + elif dilation_block_kind == 'multi': + dilated_block_kwargs['conv_layer'] = functools.partial( + get_conv_block_ctor('multidilated'), **multidilation_kwargs) + + # dilated blocks at the start of the bottleneck sausage + if dilated_blocks_n_start is not None and dilated_blocks_n_start > 0: + model += make_dil_blocks(dilated_blocks_n_start, dilation_block_kind, dilated_block_kwargs) + + # resnet blocks + for i in range(n_blocks): + # dilated blocks at the middle of the bottleneck sausage + if i == n_blocks // 2 and dilated_blocks_n_middle is not None and dilated_blocks_n_middle > 0: + model += make_dil_blocks(dilated_blocks_n_middle, dilation_block_kind, dilated_block_kwargs) + + if ffc_positions is not None and i in ffc_positions: + for _ in range(ffc_positions[i]): # same position can occur more than once + model += [FFCResnetBlock(feats_num_bottleneck, padding_type, norm_layer, activation_layer=nn.ReLU, + inline=True, **ffc_kwargs)] + + if is_resblock_depthwise: + resblock_groups = feats_num_bottleneck + else: + resblock_groups = 1 + + model += [ResnetBlock(feats_num_bottleneck, padding_type=padding_type, activation=activation, + norm_layer=norm_layer, conv_kind=conv_kind, groups=resblock_groups, + dilation=dilation, second_dilation=second_dilation)] + + + # dilated blocks at the end of the bottleneck sausage + if dilated_blocks_n is not None and dilated_blocks_n > 0: + model += make_dil_blocks(dilated_blocks_n, dilation_block_kind, dilated_block_kwargs) + + # upsample + for i in range(n_downsampling): + mult = 2 ** (n_downsampling - i) + model += [nn.ConvTranspose2d(min(max_features, ngf * mult), + min(max_features, int(ngf * mult / 2)), + kernel_size=3, stride=2, padding=1, output_padding=1), + up_norm_layer(min(max_features, int(ngf * mult / 2))), + up_activation] + model += [nn.ReflectionPad2d(3), + nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0)] + if add_out_act: + model.append(get_activation('tanh' if add_out_act is True else add_out_act)) + self.model = nn.Sequential(*model) + + def forward(self, input): + return self.model(input) + + +class GlobalGeneratorGated(GlobalGenerator): + def __init__(self, *args, **kwargs): + real_kwargs=dict( + conv_kind='gated_bn_relu', + activation=nn.Identity(), + norm_layer=nn.Identity + ) + real_kwargs.update(kwargs) + super().__init__(*args, **real_kwargs) + + +class GlobalGeneratorFromSuperChannels(nn.Module): + def __init__(self, input_nc, output_nc, n_downsampling, n_blocks, super_channels, norm_layer="bn", padding_type='reflect', add_out_act=True): + super().__init__() + self.n_downsampling = n_downsampling + norm_layer = get_norm_layer(norm_layer) + if type(norm_layer) == functools.partial: + use_bias = (norm_layer.func == nn.InstanceNorm2d) + else: + use_bias = (norm_layer == nn.InstanceNorm2d) + + channels = self.convert_super_channels(super_channels) + self.channels = channels + + model = [nn.ReflectionPad2d(3), + nn.Conv2d(input_nc, channels[0], kernel_size=7, padding=0, bias=use_bias), + norm_layer(channels[0]), + nn.ReLU(True)] + + for i in range(n_downsampling): # add downsampling layers + mult = 2 ** i + model += [nn.Conv2d(channels[0+i], channels[1+i], kernel_size=3, stride=2, padding=1, bias=use_bias), + norm_layer(channels[1+i]), + nn.ReLU(True)] + + mult = 2 ** n_downsampling + + n_blocks1 = n_blocks // 3 + n_blocks2 = n_blocks1 + n_blocks3 = n_blocks - n_blocks1 - n_blocks2 + + for i in range(n_blocks1): + c = n_downsampling + dim = channels[c] + model += [ResnetBlock(dim, padding_type=padding_type, norm_layer=norm_layer)] + + for i in range(n_blocks2): + c = n_downsampling+1 + dim = channels[c] + kwargs = {} + if i == 0: + kwargs = {"in_dim": channels[c-1]} + model += [ResnetBlock(dim, padding_type=padding_type, norm_layer=norm_layer, **kwargs)] + + for i in range(n_blocks3): + c = n_downsampling+2 + dim = channels[c] + kwargs = {} + if i == 0: + kwargs = {"in_dim": channels[c-1]} + model += [ResnetBlock(dim, padding_type=padding_type, norm_layer=norm_layer, **kwargs)] + + for i in range(n_downsampling): # add upsampling layers + mult = 2 ** (n_downsampling - i) + model += [nn.ConvTranspose2d(channels[n_downsampling+3+i], + channels[n_downsampling+3+i+1], + kernel_size=3, stride=2, + padding=1, output_padding=1, + bias=use_bias), + norm_layer(channels[n_downsampling+3+i+1]), + nn.ReLU(True)] + model += [nn.ReflectionPad2d(3)] + model += [nn.Conv2d(channels[2*n_downsampling+3], output_nc, kernel_size=7, padding=0)] + + if add_out_act: + model.append(get_activation('tanh' if add_out_act is True else add_out_act)) + self.model = nn.Sequential(*model) + + def convert_super_channels(self, super_channels): + n_downsampling = self.n_downsampling + result = [] + cnt = 0 + + if n_downsampling == 2: + N1 = 10 + elif n_downsampling == 3: + N1 = 13 + else: + raise NotImplementedError + + for i in range(0, N1): + if i in [1,4,7,10]: + channel = super_channels[cnt] * (2 ** cnt) + config = {'channel': channel} + result.append(channel) + logging.info(f"Downsample channels {result[-1]}") + cnt += 1 + + for i in range(3): + for counter, j in enumerate(range(N1 + i * 3, N1 + 3 + i * 3)): + if len(super_channels) == 6: + channel = super_channels[3] * 4 + else: + channel = super_channels[i + 3] * 4 + config = {'channel': channel} + if counter == 0: + result.append(channel) + logging.info(f"Bottleneck channels {result[-1]}") + cnt = 2 + + for i in range(N1+9, N1+21): + if i in [22, 25,28]: + cnt -= 1 + if len(super_channels) == 6: + channel = super_channels[5 - cnt] * (2 ** cnt) + else: + channel = super_channels[7 - cnt] * (2 ** cnt) + result.append(int(channel)) + logging.info(f"Upsample channels {result[-1]}") + return result + + def forward(self, input): + return self.model(input) + + +# Defines the PatchGAN discriminator with the specified arguments. +class NLayerDiscriminator(BaseDiscriminator): + def __init__(self, input_nc, ndf=64, n_layers=3, norm_layer=nn.BatchNorm2d,): + super().__init__() + self.n_layers = n_layers + + kw = 4 + padw = int(np.ceil((kw-1.0)/2)) + sequence = [[nn.Conv2d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), + nn.LeakyReLU(0.2, True)]] + + nf = ndf + for n in range(1, n_layers): + nf_prev = nf + nf = min(nf * 2, 512) + + cur_model = [] + cur_model += [ + nn.Conv2d(nf_prev, nf, kernel_size=kw, stride=2, padding=padw), + norm_layer(nf), + nn.LeakyReLU(0.2, True) + ] + sequence.append(cur_model) + + nf_prev = nf + nf = min(nf * 2, 512) + + cur_model = [] + cur_model += [ + nn.Conv2d(nf_prev, nf, kernel_size=kw, stride=1, padding=padw), + norm_layer(nf), + nn.LeakyReLU(0.2, True) + ] + sequence.append(cur_model) + + sequence += [[nn.Conv2d(nf, 1, kernel_size=kw, stride=1, padding=padw)]] + + for n in range(len(sequence)): + setattr(self, 'model'+str(n), nn.Sequential(*sequence[n])) + + def get_all_activations(self, x): + res = [x] + for n in range(self.n_layers + 2): + model = getattr(self, 'model' + str(n)) + res.append(model(res[-1])) + return res[1:] + + def forward(self, x): + act = self.get_all_activations(x) + return act[-1], act[:-1] + + +class MultidilatedNLayerDiscriminator(BaseDiscriminator): + def __init__(self, input_nc, ndf=64, n_layers=3, norm_layer=nn.BatchNorm2d, multidilation_kwargs={}): + super().__init__() + self.n_layers = n_layers + + kw = 4 + padw = int(np.ceil((kw-1.0)/2)) + sequence = [[nn.Conv2d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), + nn.LeakyReLU(0.2, True)]] + + nf = ndf + for n in range(1, n_layers): + nf_prev = nf + nf = min(nf * 2, 512) + + cur_model = [] + cur_model += [ + MultidilatedConv(nf_prev, nf, kernel_size=kw, stride=2, padding=[2, 3], **multidilation_kwargs), + norm_layer(nf), + nn.LeakyReLU(0.2, True) + ] + sequence.append(cur_model) + + nf_prev = nf + nf = min(nf * 2, 512) + + cur_model = [] + cur_model += [ + nn.Conv2d(nf_prev, nf, kernel_size=kw, stride=1, padding=padw), + norm_layer(nf), + nn.LeakyReLU(0.2, True) + ] + sequence.append(cur_model) + + sequence += [[nn.Conv2d(nf, 1, kernel_size=kw, stride=1, padding=padw)]] + + for n in range(len(sequence)): + setattr(self, 'model'+str(n), nn.Sequential(*sequence[n])) + + def get_all_activations(self, x): + res = [x] + for n in range(self.n_layers + 2): + model = getattr(self, 'model' + str(n)) + res.append(model(res[-1])) + return res[1:] + + def forward(self, x): + act = self.get_all_activations(x) + return act[-1], act[:-1] + + +class NLayerDiscriminatorAsGen(NLayerDiscriminator): + def forward(self, x): + return super().forward(x)[0] diff --git a/annotator/lama/saicinpainting/training/modules/spatial_transform.py b/annotator/lama/saicinpainting/training/modules/spatial_transform.py new file mode 100644 index 0000000000000000000000000000000000000000..2de024ba08c549605a08b64d096f1f0db7b7722a --- /dev/null +++ b/annotator/lama/saicinpainting/training/modules/spatial_transform.py @@ -0,0 +1,49 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from kornia.geometry.transform import rotate + + +class LearnableSpatialTransformWrapper(nn.Module): + def __init__(self, impl, pad_coef=0.5, angle_init_range=80, train_angle=True): + super().__init__() + self.impl = impl + self.angle = torch.rand(1) * angle_init_range + if train_angle: + self.angle = nn.Parameter(self.angle, requires_grad=True) + self.pad_coef = pad_coef + + def forward(self, x): + if torch.is_tensor(x): + return self.inverse_transform(self.impl(self.transform(x)), x) + elif isinstance(x, tuple): + x_trans = tuple(self.transform(elem) for elem in x) + y_trans = self.impl(x_trans) + return tuple(self.inverse_transform(elem, orig_x) for elem, orig_x in zip(y_trans, x)) + else: + raise ValueError(f'Unexpected input type {type(x)}') + + def transform(self, x): + height, width = x.shape[2:] + pad_h, pad_w = int(height * self.pad_coef), int(width * self.pad_coef) + x_padded = F.pad(x, [pad_w, pad_w, pad_h, pad_h], mode='reflect') + x_padded_rotated = rotate(x_padded, angle=self.angle.to(x_padded)) + return x_padded_rotated + + def inverse_transform(self, y_padded_rotated, orig_x): + height, width = orig_x.shape[2:] + pad_h, pad_w = int(height * self.pad_coef), int(width * self.pad_coef) + + y_padded = rotate(y_padded_rotated, angle=-self.angle.to(y_padded_rotated)) + y_height, y_width = y_padded.shape[2:] + y = y_padded[:, :, pad_h : y_height - pad_h, pad_w : y_width - pad_w] + return y + + +if __name__ == '__main__': + layer = LearnableSpatialTransformWrapper(nn.Identity()) + x = torch.arange(2* 3 * 15 * 15).view(2, 3, 15, 15).float() + y = layer(x) + assert x.shape == y.shape + assert torch.allclose(x[:, :, 1:, 1:][:, :, :-1, :-1], y[:, :, 1:, 1:][:, :, :-1, :-1]) + print('all ok') diff --git a/annotator/lama/saicinpainting/training/modules/squeeze_excitation.py b/annotator/lama/saicinpainting/training/modules/squeeze_excitation.py new file mode 100644 index 0000000000000000000000000000000000000000..d1d902bb30c071acbc0fa919a134c80fed86bd6c --- /dev/null +++ b/annotator/lama/saicinpainting/training/modules/squeeze_excitation.py @@ -0,0 +1,20 @@ +import torch.nn as nn + + +class SELayer(nn.Module): + def __init__(self, channel, reduction=16): + super(SELayer, self).__init__() + self.avg_pool = nn.AdaptiveAvgPool2d(1) + self.fc = nn.Sequential( + nn.Linear(channel, channel // reduction, bias=False), + nn.ReLU(inplace=True), + nn.Linear(channel // reduction, channel, bias=False), + nn.Sigmoid() + ) + + def forward(self, x): + b, c, _, _ = x.size() + y = self.avg_pool(x).view(b, c) + y = self.fc(y).view(b, c, 1, 1) + res = x * y.expand_as(x) + return res diff --git a/annotator/lama/saicinpainting/training/trainers/__init__.py b/annotator/lama/saicinpainting/training/trainers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8307cd31c2139db0ce581637403b3a95dc8cae59 --- /dev/null +++ b/annotator/lama/saicinpainting/training/trainers/__init__.py @@ -0,0 +1,29 @@ +import logging +import torch +from annotator.lama.saicinpainting.training.trainers.default import DefaultInpaintingTrainingModule + + +def get_training_model_class(kind): + if kind == 'default': + return DefaultInpaintingTrainingModule + + raise ValueError(f'Unknown trainer module {kind}') + + +def make_training_model(config): + kind = config.training_model.kind + kwargs = dict(config.training_model) + kwargs.pop('kind') + kwargs['use_ddp'] = config.trainer.kwargs.get('accelerator', None) == 'ddp' + + logging.info(f'Make training model {kind}') + + cls = get_training_model_class(kind) + return cls(config, **kwargs) + + +def load_checkpoint(train_config, path, map_location='cuda', strict=True): + model = make_training_model(train_config).generator + state = torch.load(path, map_location=map_location) + model.load_state_dict(state, strict=strict) + return model diff --git a/annotator/lama/saicinpainting/training/trainers/base.py b/annotator/lama/saicinpainting/training/trainers/base.py new file mode 100644 index 0000000000000000000000000000000000000000..372dd879a22ff6c3929abf23bb59d6b8b66256b7 --- /dev/null +++ b/annotator/lama/saicinpainting/training/trainers/base.py @@ -0,0 +1,293 @@ +import copy +import logging +from typing import Dict, Tuple + +import pandas as pd +import pytorch_lightning as ptl +import torch +import torch.nn as nn +import torch.nn.functional as F +# from torch.utils.data import DistributedSampler + +# from annotator.lama.saicinpainting.evaluation import make_evaluator +# from annotator.lama.saicinpainting.training.data.datasets import make_default_train_dataloader, make_default_val_dataloader +# from annotator.lama.saicinpainting.training.losses.adversarial import make_discrim_loss +# from annotator.lama.saicinpainting.training.losses.perceptual import PerceptualLoss, ResNetPL +from annotator.lama.saicinpainting.training.modules import make_generator #, make_discriminator +# from annotator.lama.saicinpainting.training.visualizers import make_visualizer +from annotator.lama.saicinpainting.utils import add_prefix_to_keys, average_dicts, set_requires_grad, flatten_dict, \ + get_has_ddp_rank + +LOGGER = logging.getLogger(__name__) + + +def make_optimizer(parameters, kind='adamw', **kwargs): + if kind == 'adam': + optimizer_class = torch.optim.Adam + elif kind == 'adamw': + optimizer_class = torch.optim.AdamW + else: + raise ValueError(f'Unknown optimizer kind {kind}') + return optimizer_class(parameters, **kwargs) + + +def update_running_average(result: nn.Module, new_iterate_model: nn.Module, decay=0.999): + with torch.no_grad(): + res_params = dict(result.named_parameters()) + new_params = dict(new_iterate_model.named_parameters()) + + for k in res_params.keys(): + res_params[k].data.mul_(decay).add_(new_params[k].data, alpha=1 - decay) + + +def make_multiscale_noise(base_tensor, scales=6, scale_mode='bilinear'): + batch_size, _, height, width = base_tensor.shape + cur_height, cur_width = height, width + result = [] + align_corners = False if scale_mode in ('bilinear', 'bicubic') else None + for _ in range(scales): + cur_sample = torch.randn(batch_size, 1, cur_height, cur_width, device=base_tensor.device) + cur_sample_scaled = F.interpolate(cur_sample, size=(height, width), mode=scale_mode, align_corners=align_corners) + result.append(cur_sample_scaled) + cur_height //= 2 + cur_width //= 2 + return torch.cat(result, dim=1) + + +class BaseInpaintingTrainingModule(ptl.LightningModule): + def __init__(self, config, use_ddp, *args, predict_only=False, visualize_each_iters=100, + average_generator=False, generator_avg_beta=0.999, average_generator_start_step=30000, + average_generator_period=10, store_discr_outputs_for_vis=False, + **kwargs): + super().__init__(*args, **kwargs) + LOGGER.info('BaseInpaintingTrainingModule init called') + + self.config = config + + self.generator = make_generator(config, **self.config.generator) + self.use_ddp = use_ddp + + if not get_has_ddp_rank(): + LOGGER.info(f'Generator\n{self.generator}') + + # if not predict_only: + # self.save_hyperparameters(self.config) + # self.discriminator = make_discriminator(**self.config.discriminator) + # self.adversarial_loss = make_discrim_loss(**self.config.losses.adversarial) + # self.visualizer = make_visualizer(**self.config.visualizer) + # self.val_evaluator = make_evaluator(**self.config.evaluator) + # self.test_evaluator = make_evaluator(**self.config.evaluator) + # + # if not get_has_ddp_rank(): + # LOGGER.info(f'Discriminator\n{self.discriminator}') + # + # extra_val = self.config.data.get('extra_val', ()) + # if extra_val: + # self.extra_val_titles = list(extra_val) + # self.extra_evaluators = nn.ModuleDict({k: make_evaluator(**self.config.evaluator) + # for k in extra_val}) + # else: + # self.extra_evaluators = {} + # + # self.average_generator = average_generator + # self.generator_avg_beta = generator_avg_beta + # self.average_generator_start_step = average_generator_start_step + # self.average_generator_period = average_generator_period + # self.generator_average = None + # self.last_generator_averaging_step = -1 + # self.store_discr_outputs_for_vis = store_discr_outputs_for_vis + # + # if self.config.losses.get("l1", {"weight_known": 0})['weight_known'] > 0: + # self.loss_l1 = nn.L1Loss(reduction='none') + # + # if self.config.losses.get("mse", {"weight": 0})['weight'] > 0: + # self.loss_mse = nn.MSELoss(reduction='none') + # + # if self.config.losses.perceptual.weight > 0: + # self.loss_pl = PerceptualLoss() + # + # # if self.config.losses.get("resnet_pl", {"weight": 0})['weight'] > 0: + # # self.loss_resnet_pl = ResNetPL(**self.config.losses.resnet_pl) + # # else: + # # self.loss_resnet_pl = None + # + # self.loss_resnet_pl = None + + self.visualize_each_iters = visualize_each_iters + LOGGER.info('BaseInpaintingTrainingModule init done') + + def configure_optimizers(self): + discriminator_params = list(self.discriminator.parameters()) + return [ + dict(optimizer=make_optimizer(self.generator.parameters(), **self.config.optimizers.generator)), + dict(optimizer=make_optimizer(discriminator_params, **self.config.optimizers.discriminator)), + ] + + def train_dataloader(self): + kwargs = dict(self.config.data.train) + if self.use_ddp: + kwargs['ddp_kwargs'] = dict(num_replicas=self.trainer.num_nodes * self.trainer.num_processes, + rank=self.trainer.global_rank, + shuffle=True) + dataloader = make_default_train_dataloader(**self.config.data.train) + return dataloader + + def val_dataloader(self): + res = [make_default_val_dataloader(**self.config.data.val)] + + if self.config.data.visual_test is not None: + res = res + [make_default_val_dataloader(**self.config.data.visual_test)] + else: + res = res + res + + extra_val = self.config.data.get('extra_val', ()) + if extra_val: + res += [make_default_val_dataloader(**extra_val[k]) for k in self.extra_val_titles] + + return res + + def training_step(self, batch, batch_idx, optimizer_idx=None): + self._is_training_step = True + return self._do_step(batch, batch_idx, mode='train', optimizer_idx=optimizer_idx) + + def validation_step(self, batch, batch_idx, dataloader_idx): + extra_val_key = None + if dataloader_idx == 0: + mode = 'val' + elif dataloader_idx == 1: + mode = 'test' + else: + mode = 'extra_val' + extra_val_key = self.extra_val_titles[dataloader_idx - 2] + self._is_training_step = False + return self._do_step(batch, batch_idx, mode=mode, extra_val_key=extra_val_key) + + def training_step_end(self, batch_parts_outputs): + if self.training and self.average_generator \ + and self.global_step >= self.average_generator_start_step \ + and self.global_step >= self.last_generator_averaging_step + self.average_generator_period: + if self.generator_average is None: + self.generator_average = copy.deepcopy(self.generator) + else: + update_running_average(self.generator_average, self.generator, decay=self.generator_avg_beta) + self.last_generator_averaging_step = self.global_step + + full_loss = (batch_parts_outputs['loss'].mean() + if torch.is_tensor(batch_parts_outputs['loss']) # loss is not tensor when no discriminator used + else torch.tensor(batch_parts_outputs['loss']).float().requires_grad_(True)) + log_info = {k: v.mean() for k, v in batch_parts_outputs['log_info'].items()} + self.log_dict(log_info, on_step=True, on_epoch=False) + return full_loss + + def validation_epoch_end(self, outputs): + outputs = [step_out for out_group in outputs for step_out in out_group] + averaged_logs = average_dicts(step_out['log_info'] for step_out in outputs) + self.log_dict({k: v.mean() for k, v in averaged_logs.items()}) + + pd.set_option('display.max_columns', 500) + pd.set_option('display.width', 1000) + + # standard validation + val_evaluator_states = [s['val_evaluator_state'] for s in outputs if 'val_evaluator_state' in s] + val_evaluator_res = self.val_evaluator.evaluation_end(states=val_evaluator_states) + val_evaluator_res_df = pd.DataFrame(val_evaluator_res).stack(1).unstack(0) + val_evaluator_res_df.dropna(axis=1, how='all', inplace=True) + LOGGER.info(f'Validation metrics after epoch #{self.current_epoch}, ' + f'total {self.global_step} iterations:\n{val_evaluator_res_df}') + + for k, v in flatten_dict(val_evaluator_res).items(): + self.log(f'val_{k}', v) + + # standard visual test + test_evaluator_states = [s['test_evaluator_state'] for s in outputs + if 'test_evaluator_state' in s] + test_evaluator_res = self.test_evaluator.evaluation_end(states=test_evaluator_states) + test_evaluator_res_df = pd.DataFrame(test_evaluator_res).stack(1).unstack(0) + test_evaluator_res_df.dropna(axis=1, how='all', inplace=True) + LOGGER.info(f'Test metrics after epoch #{self.current_epoch}, ' + f'total {self.global_step} iterations:\n{test_evaluator_res_df}') + + for k, v in flatten_dict(test_evaluator_res).items(): + self.log(f'test_{k}', v) + + # extra validations + if self.extra_evaluators: + for cur_eval_title, cur_evaluator in self.extra_evaluators.items(): + cur_state_key = f'extra_val_{cur_eval_title}_evaluator_state' + cur_states = [s[cur_state_key] for s in outputs if cur_state_key in s] + cur_evaluator_res = cur_evaluator.evaluation_end(states=cur_states) + cur_evaluator_res_df = pd.DataFrame(cur_evaluator_res).stack(1).unstack(0) + cur_evaluator_res_df.dropna(axis=1, how='all', inplace=True) + LOGGER.info(f'Extra val {cur_eval_title} metrics after epoch #{self.current_epoch}, ' + f'total {self.global_step} iterations:\n{cur_evaluator_res_df}') + for k, v in flatten_dict(cur_evaluator_res).items(): + self.log(f'extra_val_{cur_eval_title}_{k}', v) + + def _do_step(self, batch, batch_idx, mode='train', optimizer_idx=None, extra_val_key=None): + if optimizer_idx == 0: # step for generator + set_requires_grad(self.generator, True) + set_requires_grad(self.discriminator, False) + elif optimizer_idx == 1: # step for discriminator + set_requires_grad(self.generator, False) + set_requires_grad(self.discriminator, True) + + batch = self(batch) + + total_loss = 0 + metrics = {} + + if optimizer_idx is None or optimizer_idx == 0: # step for generator + total_loss, metrics = self.generator_loss(batch) + + elif optimizer_idx is None or optimizer_idx == 1: # step for discriminator + if self.config.losses.adversarial.weight > 0: + total_loss, metrics = self.discriminator_loss(batch) + + if self.get_ddp_rank() in (None, 0) and (batch_idx % self.visualize_each_iters == 0 or mode == 'test'): + if self.config.losses.adversarial.weight > 0: + if self.store_discr_outputs_for_vis: + with torch.no_grad(): + self.store_discr_outputs(batch) + vis_suffix = f'_{mode}' + if mode == 'extra_val': + vis_suffix += f'_{extra_val_key}' + self.visualizer(self.current_epoch, batch_idx, batch, suffix=vis_suffix) + + metrics_prefix = f'{mode}_' + if mode == 'extra_val': + metrics_prefix += f'{extra_val_key}_' + result = dict(loss=total_loss, log_info=add_prefix_to_keys(metrics, metrics_prefix)) + if mode == 'val': + result['val_evaluator_state'] = self.val_evaluator.process_batch(batch) + elif mode == 'test': + result['test_evaluator_state'] = self.test_evaluator.process_batch(batch) + elif mode == 'extra_val': + result[f'extra_val_{extra_val_key}_evaluator_state'] = self.extra_evaluators[extra_val_key].process_batch(batch) + + return result + + def get_current_generator(self, no_average=False): + if not no_average and not self.training and self.average_generator and self.generator_average is not None: + return self.generator_average + return self.generator + + def forward(self, batch: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]: + """Pass data through generator and obtain at leas 'predicted_image' and 'inpainted' keys""" + raise NotImplementedError() + + def generator_loss(self, batch) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + raise NotImplementedError() + + def discriminator_loss(self, batch) -> Tuple[torch.Tensor, Dict[str, torch.Tensor]]: + raise NotImplementedError() + + def store_discr_outputs(self, batch): + out_size = batch['image'].shape[2:] + discr_real_out, _ = self.discriminator(batch['image']) + discr_fake_out, _ = self.discriminator(batch['predicted_image']) + batch['discr_output_real'] = F.interpolate(discr_real_out, size=out_size, mode='nearest') + batch['discr_output_fake'] = F.interpolate(discr_fake_out, size=out_size, mode='nearest') + batch['discr_output_diff'] = batch['discr_output_real'] - batch['discr_output_fake'] + + def get_ddp_rank(self): + return self.trainer.global_rank if (self.trainer.num_nodes * self.trainer.num_processes) > 1 else None diff --git a/annotator/lama/saicinpainting/training/trainers/default.py b/annotator/lama/saicinpainting/training/trainers/default.py new file mode 100644 index 0000000000000000000000000000000000000000..29cd10ec376d5fe3ebcd957d807d2d3f83b6ec59 --- /dev/null +++ b/annotator/lama/saicinpainting/training/trainers/default.py @@ -0,0 +1,175 @@ +import logging + +import torch +import torch.nn.functional as F +from omegaconf import OmegaConf + +# from annotator.lama.saicinpainting.training.data.datasets import make_constant_area_crop_params +from annotator.lama.saicinpainting.training.losses.distance_weighting import make_mask_distance_weighter +from annotator.lama.saicinpainting.training.losses.feature_matching import feature_matching_loss, masked_l1_loss +# from annotator.lama.saicinpainting.training.modules.fake_fakes import FakeFakesGenerator +from annotator.lama.saicinpainting.training.trainers.base import BaseInpaintingTrainingModule, make_multiscale_noise +from annotator.lama.saicinpainting.utils import add_prefix_to_keys, get_ramp + +LOGGER = logging.getLogger(__name__) + + +def make_constant_area_crop_batch(batch, **kwargs): + crop_y, crop_x, crop_height, crop_width = make_constant_area_crop_params(img_height=batch['image'].shape[2], + img_width=batch['image'].shape[3], + **kwargs) + batch['image'] = batch['image'][:, :, crop_y : crop_y + crop_height, crop_x : crop_x + crop_width] + batch['mask'] = batch['mask'][:, :, crop_y: crop_y + crop_height, crop_x: crop_x + crop_width] + return batch + + +class DefaultInpaintingTrainingModule(BaseInpaintingTrainingModule): + def __init__(self, *args, concat_mask=True, rescale_scheduler_kwargs=None, image_to_discriminator='predicted_image', + add_noise_kwargs=None, noise_fill_hole=False, const_area_crop_kwargs=None, + distance_weighter_kwargs=None, distance_weighted_mask_for_discr=False, + fake_fakes_proba=0, fake_fakes_generator_kwargs=None, + **kwargs): + super().__init__(*args, **kwargs) + self.concat_mask = concat_mask + self.rescale_size_getter = get_ramp(**rescale_scheduler_kwargs) if rescale_scheduler_kwargs is not None else None + self.image_to_discriminator = image_to_discriminator + self.add_noise_kwargs = add_noise_kwargs + self.noise_fill_hole = noise_fill_hole + self.const_area_crop_kwargs = const_area_crop_kwargs + self.refine_mask_for_losses = make_mask_distance_weighter(**distance_weighter_kwargs) \ + if distance_weighter_kwargs is not None else None + self.distance_weighted_mask_for_discr = distance_weighted_mask_for_discr + + self.fake_fakes_proba = fake_fakes_proba + if self.fake_fakes_proba > 1e-3: + self.fake_fakes_gen = FakeFakesGenerator(**(fake_fakes_generator_kwargs or {})) + + def forward(self, batch): + if self.training and self.rescale_size_getter is not None: + cur_size = self.rescale_size_getter(self.global_step) + batch['image'] = F.interpolate(batch['image'], size=cur_size, mode='bilinear', align_corners=False) + batch['mask'] = F.interpolate(batch['mask'], size=cur_size, mode='nearest') + + if self.training and self.const_area_crop_kwargs is not None: + batch = make_constant_area_crop_batch(batch, **self.const_area_crop_kwargs) + + img = batch['image'] + mask = batch['mask'] + + masked_img = img * (1 - mask) + + if self.add_noise_kwargs is not None: + noise = make_multiscale_noise(masked_img, **self.add_noise_kwargs) + if self.noise_fill_hole: + masked_img = masked_img + mask * noise[:, :masked_img.shape[1]] + masked_img = torch.cat([masked_img, noise], dim=1) + + if self.concat_mask: + masked_img = torch.cat([masked_img, mask], dim=1) + + batch['predicted_image'] = self.generator(masked_img) + batch['inpainted'] = mask * batch['predicted_image'] + (1 - mask) * batch['image'] + + if self.fake_fakes_proba > 1e-3: + if self.training and torch.rand(1).item() < self.fake_fakes_proba: + batch['fake_fakes'], batch['fake_fakes_masks'] = self.fake_fakes_gen(img, mask) + batch['use_fake_fakes'] = True + else: + batch['fake_fakes'] = torch.zeros_like(img) + batch['fake_fakes_masks'] = torch.zeros_like(mask) + batch['use_fake_fakes'] = False + + batch['mask_for_losses'] = self.refine_mask_for_losses(img, batch['predicted_image'], mask) \ + if self.refine_mask_for_losses is not None and self.training \ + else mask + + return batch + + def generator_loss(self, batch): + img = batch['image'] + predicted_img = batch[self.image_to_discriminator] + original_mask = batch['mask'] + supervised_mask = batch['mask_for_losses'] + + # L1 + l1_value = masked_l1_loss(predicted_img, img, supervised_mask, + self.config.losses.l1.weight_known, + self.config.losses.l1.weight_missing) + + total_loss = l1_value + metrics = dict(gen_l1=l1_value) + + # vgg-based perceptual loss + if self.config.losses.perceptual.weight > 0: + pl_value = self.loss_pl(predicted_img, img, mask=supervised_mask).sum() * self.config.losses.perceptual.weight + total_loss = total_loss + pl_value + metrics['gen_pl'] = pl_value + + # discriminator + # adversarial_loss calls backward by itself + mask_for_discr = supervised_mask if self.distance_weighted_mask_for_discr else original_mask + self.adversarial_loss.pre_generator_step(real_batch=img, fake_batch=predicted_img, + generator=self.generator, discriminator=self.discriminator) + discr_real_pred, discr_real_features = self.discriminator(img) + discr_fake_pred, discr_fake_features = self.discriminator(predicted_img) + adv_gen_loss, adv_metrics = self.adversarial_loss.generator_loss(real_batch=img, + fake_batch=predicted_img, + discr_real_pred=discr_real_pred, + discr_fake_pred=discr_fake_pred, + mask=mask_for_discr) + total_loss = total_loss + adv_gen_loss + metrics['gen_adv'] = adv_gen_loss + metrics.update(add_prefix_to_keys(adv_metrics, 'adv_')) + + # feature matching + if self.config.losses.feature_matching.weight > 0: + need_mask_in_fm = OmegaConf.to_container(self.config.losses.feature_matching).get('pass_mask', False) + mask_for_fm = supervised_mask if need_mask_in_fm else None + fm_value = feature_matching_loss(discr_fake_features, discr_real_features, + mask=mask_for_fm) * self.config.losses.feature_matching.weight + total_loss = total_loss + fm_value + metrics['gen_fm'] = fm_value + + if self.loss_resnet_pl is not None: + resnet_pl_value = self.loss_resnet_pl(predicted_img, img) + total_loss = total_loss + resnet_pl_value + metrics['gen_resnet_pl'] = resnet_pl_value + + return total_loss, metrics + + def discriminator_loss(self, batch): + total_loss = 0 + metrics = {} + + predicted_img = batch[self.image_to_discriminator].detach() + self.adversarial_loss.pre_discriminator_step(real_batch=batch['image'], fake_batch=predicted_img, + generator=self.generator, discriminator=self.discriminator) + discr_real_pred, discr_real_features = self.discriminator(batch['image']) + discr_fake_pred, discr_fake_features = self.discriminator(predicted_img) + adv_discr_loss, adv_metrics = self.adversarial_loss.discriminator_loss(real_batch=batch['image'], + fake_batch=predicted_img, + discr_real_pred=discr_real_pred, + discr_fake_pred=discr_fake_pred, + mask=batch['mask']) + total_loss = total_loss + adv_discr_loss + metrics['discr_adv'] = adv_discr_loss + metrics.update(add_prefix_to_keys(adv_metrics, 'adv_')) + + + if batch.get('use_fake_fakes', False): + fake_fakes = batch['fake_fakes'] + self.adversarial_loss.pre_discriminator_step(real_batch=batch['image'], fake_batch=fake_fakes, + generator=self.generator, discriminator=self.discriminator) + discr_fake_fakes_pred, _ = self.discriminator(fake_fakes) + fake_fakes_adv_discr_loss, fake_fakes_adv_metrics = self.adversarial_loss.discriminator_loss( + real_batch=batch['image'], + fake_batch=fake_fakes, + discr_real_pred=discr_real_pred, + discr_fake_pred=discr_fake_fakes_pred, + mask=batch['mask'] + ) + total_loss = total_loss + fake_fakes_adv_discr_loss + metrics['discr_adv_fake_fakes'] = fake_fakes_adv_discr_loss + metrics.update(add_prefix_to_keys(fake_fakes_adv_metrics, 'adv_')) + + return total_loss, metrics diff --git a/annotator/lama/saicinpainting/training/visualizers/__init__.py b/annotator/lama/saicinpainting/training/visualizers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d280fd8d48428c249c40c341ecc3c36f34524c99 --- /dev/null +++ b/annotator/lama/saicinpainting/training/visualizers/__init__.py @@ -0,0 +1,15 @@ +import logging + +from annotator.lama.saicinpainting.training.visualizers.directory import DirectoryVisualizer +from annotator.lama.saicinpainting.training.visualizers.noop import NoopVisualizer + + +def make_visualizer(kind, **kwargs): + logging.info(f'Make visualizer {kind}') + + if kind == 'directory': + return DirectoryVisualizer(**kwargs) + if kind == 'noop': + return NoopVisualizer() + + raise ValueError(f'Unknown visualizer kind {kind}') diff --git a/annotator/lama/saicinpainting/training/visualizers/base.py b/annotator/lama/saicinpainting/training/visualizers/base.py new file mode 100644 index 0000000000000000000000000000000000000000..675f01682ddf5e31b6cc341735378c6f3b242e49 --- /dev/null +++ b/annotator/lama/saicinpainting/training/visualizers/base.py @@ -0,0 +1,73 @@ +import abc +from typing import Dict, List + +import numpy as np +import torch +from skimage import color +from skimage.segmentation import mark_boundaries + +from . import colors + +COLORS, _ = colors.generate_colors(151) # 151 - max classes for semantic segmentation + + +class BaseVisualizer: + @abc.abstractmethod + def __call__(self, epoch_i, batch_i, batch, suffix='', rank=None): + """ + Take a batch, make an image from it and visualize + """ + raise NotImplementedError() + + +def visualize_mask_and_images(images_dict: Dict[str, np.ndarray], keys: List[str], + last_without_mask=True, rescale_keys=None, mask_only_first=None, + black_mask=False) -> np.ndarray: + mask = images_dict['mask'] > 0.5 + result = [] + for i, k in enumerate(keys): + img = images_dict[k] + img = np.transpose(img, (1, 2, 0)) + + if rescale_keys is not None and k in rescale_keys: + img = img - img.min() + img /= img.max() + 1e-5 + if len(img.shape) == 2: + img = np.expand_dims(img, 2) + + if img.shape[2] == 1: + img = np.repeat(img, 3, axis=2) + elif (img.shape[2] > 3): + img_classes = img.argmax(2) + img = color.label2rgb(img_classes, colors=COLORS) + + if mask_only_first: + need_mark_boundaries = i == 0 + else: + need_mark_boundaries = i < len(keys) - 1 or not last_without_mask + + if need_mark_boundaries: + if black_mask: + img = img * (1 - mask[0][..., None]) + img = mark_boundaries(img, + mask[0], + color=(1., 0., 0.), + outline_color=(1., 1., 1.), + mode='thick') + result.append(img) + return np.concatenate(result, axis=1) + + +def visualize_mask_and_images_batch(batch: Dict[str, torch.Tensor], keys: List[str], max_items=10, + last_without_mask=True, rescale_keys=None) -> np.ndarray: + batch = {k: tens.detach().cpu().numpy() for k, tens in batch.items() + if k in keys or k == 'mask'} + + batch_size = next(iter(batch.values())).shape[0] + items_to_vis = min(batch_size, max_items) + result = [] + for i in range(items_to_vis): + cur_dct = {k: tens[i] for k, tens in batch.items()} + result.append(visualize_mask_and_images(cur_dct, keys, last_without_mask=last_without_mask, + rescale_keys=rescale_keys)) + return np.concatenate(result, axis=0) diff --git a/annotator/lama/saicinpainting/training/visualizers/colors.py b/annotator/lama/saicinpainting/training/visualizers/colors.py new file mode 100644 index 0000000000000000000000000000000000000000..9e9e39182c58cb06a1c5e97a7e6c497cc3388ebe --- /dev/null +++ b/annotator/lama/saicinpainting/training/visualizers/colors.py @@ -0,0 +1,76 @@ +import random +import colorsys + +import numpy as np +import matplotlib +matplotlib.use('agg') +import matplotlib.pyplot as plt +from matplotlib.colors import LinearSegmentedColormap + + +def generate_colors(nlabels, type='bright', first_color_black=False, last_color_black=True, verbose=False): + # https://stackoverflow.com/questions/14720331/how-to-generate-random-colors-in-matplotlib + """ + Creates a random colormap to be used together with matplotlib. Useful for segmentation tasks + :param nlabels: Number of labels (size of colormap) + :param type: 'bright' for strong colors, 'soft' for pastel colors + :param first_color_black: Option to use first color as black, True or False + :param last_color_black: Option to use last color as black, True or False + :param verbose: Prints the number of labels and shows the colormap. True or False + :return: colormap for matplotlib + """ + if type not in ('bright', 'soft'): + print ('Please choose "bright" or "soft" for type') + return + + if verbose: + print('Number of labels: ' + str(nlabels)) + + # Generate color map for bright colors, based on hsv + if type == 'bright': + randHSVcolors = [(np.random.uniform(low=0.0, high=1), + np.random.uniform(low=0.2, high=1), + np.random.uniform(low=0.9, high=1)) for i in range(nlabels)] + + # Convert HSV list to RGB + randRGBcolors = [] + for HSVcolor in randHSVcolors: + randRGBcolors.append(colorsys.hsv_to_rgb(HSVcolor[0], HSVcolor[1], HSVcolor[2])) + + if first_color_black: + randRGBcolors[0] = [0, 0, 0] + + if last_color_black: + randRGBcolors[-1] = [0, 0, 0] + + random_colormap = LinearSegmentedColormap.from_list('new_map', randRGBcolors, N=nlabels) + + # Generate soft pastel colors, by limiting the RGB spectrum + if type == 'soft': + low = 0.6 + high = 0.95 + randRGBcolors = [(np.random.uniform(low=low, high=high), + np.random.uniform(low=low, high=high), + np.random.uniform(low=low, high=high)) for i in range(nlabels)] + + if first_color_black: + randRGBcolors[0] = [0, 0, 0] + + if last_color_black: + randRGBcolors[-1] = [0, 0, 0] + random_colormap = LinearSegmentedColormap.from_list('new_map', randRGBcolors, N=nlabels) + + # Display colorbar + if verbose: + from matplotlib import colors, colorbar + from matplotlib import pyplot as plt + fig, ax = plt.subplots(1, 1, figsize=(15, 0.5)) + + bounds = np.linspace(0, nlabels, nlabels + 1) + norm = colors.BoundaryNorm(bounds, nlabels) + + cb = colorbar.ColorbarBase(ax, cmap=random_colormap, norm=norm, spacing='proportional', ticks=None, + boundaries=bounds, format='%1i', orientation=u'horizontal') + + return randRGBcolors, random_colormap + diff --git a/annotator/lama/saicinpainting/training/visualizers/directory.py b/annotator/lama/saicinpainting/training/visualizers/directory.py new file mode 100644 index 0000000000000000000000000000000000000000..a0a3b5eb93c0738784bf24083bdd54d50e4782f6 --- /dev/null +++ b/annotator/lama/saicinpainting/training/visualizers/directory.py @@ -0,0 +1,36 @@ +import os + +import cv2 +import numpy as np + +from annotator.lama.saicinpainting.training.visualizers.base import BaseVisualizer, visualize_mask_and_images_batch +from annotator.lama.saicinpainting.utils import check_and_warn_input_range + + +class DirectoryVisualizer(BaseVisualizer): + DEFAULT_KEY_ORDER = 'image predicted_image inpainted'.split(' ') + + def __init__(self, outdir, key_order=DEFAULT_KEY_ORDER, max_items_in_batch=10, + last_without_mask=True, rescale_keys=None): + self.outdir = outdir + os.makedirs(self.outdir, exist_ok=True) + self.key_order = key_order + self.max_items_in_batch = max_items_in_batch + self.last_without_mask = last_without_mask + self.rescale_keys = rescale_keys + + def __call__(self, epoch_i, batch_i, batch, suffix='', rank=None): + check_and_warn_input_range(batch['image'], 0, 1, 'DirectoryVisualizer target image') + vis_img = visualize_mask_and_images_batch(batch, self.key_order, max_items=self.max_items_in_batch, + last_without_mask=self.last_without_mask, + rescale_keys=self.rescale_keys) + + vis_img = np.clip(vis_img * 255, 0, 255).astype('uint8') + + curoutdir = os.path.join(self.outdir, f'epoch{epoch_i:04d}{suffix}') + os.makedirs(curoutdir, exist_ok=True) + rank_suffix = f'_r{rank}' if rank is not None else '' + out_fname = os.path.join(curoutdir, f'batch{batch_i:07d}{rank_suffix}.jpg') + + vis_img = cv2.cvtColor(vis_img, cv2.COLOR_RGB2BGR) + cv2.imwrite(out_fname, vis_img) diff --git a/annotator/lama/saicinpainting/training/visualizers/noop.py b/annotator/lama/saicinpainting/training/visualizers/noop.py new file mode 100644 index 0000000000000000000000000000000000000000..4479597baf33a817686a4f679b4576f83b6e5c31 --- /dev/null +++ b/annotator/lama/saicinpainting/training/visualizers/noop.py @@ -0,0 +1,9 @@ +from annotator.lama.saicinpainting.training.visualizers.base import BaseVisualizer + + +class NoopVisualizer(BaseVisualizer): + def __init__(self, *args, **kwargs): + pass + + def __call__(self, epoch_i, batch_i, batch, suffix='', rank=None): + pass diff --git a/annotator/lama/saicinpainting/utils.py b/annotator/lama/saicinpainting/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f36f5130d4c105b63689642da5321ce2e1863a9f --- /dev/null +++ b/annotator/lama/saicinpainting/utils.py @@ -0,0 +1,174 @@ +import bisect +import functools +import logging +import numbers +import os +import signal +import sys +import traceback +import warnings + +import torch +from pytorch_lightning import seed_everything + +LOGGER = logging.getLogger(__name__) + + +def check_and_warn_input_range(tensor, min_value, max_value, name): + actual_min = tensor.min() + actual_max = tensor.max() + if actual_min < min_value or actual_max > max_value: + warnings.warn(f"{name} must be in {min_value}..{max_value} range, but it ranges {actual_min}..{actual_max}") + + +def sum_dict_with_prefix(target, cur_dict, prefix, default=0): + for k, v in cur_dict.items(): + target_key = prefix + k + target[target_key] = target.get(target_key, default) + v + + +def average_dicts(dict_list): + result = {} + norm = 1e-3 + for dct in dict_list: + sum_dict_with_prefix(result, dct, '') + norm += 1 + for k in list(result): + result[k] /= norm + return result + + +def add_prefix_to_keys(dct, prefix): + return {prefix + k: v for k, v in dct.items()} + + +def set_requires_grad(module, value): + for param in module.parameters(): + param.requires_grad = value + + +def flatten_dict(dct): + result = {} + for k, v in dct.items(): + if isinstance(k, tuple): + k = '_'.join(k) + if isinstance(v, dict): + for sub_k, sub_v in flatten_dict(v).items(): + result[f'{k}_{sub_k}'] = sub_v + else: + result[k] = v + return result + + +class LinearRamp: + def __init__(self, start_value=0, end_value=1, start_iter=-1, end_iter=0): + self.start_value = start_value + self.end_value = end_value + self.start_iter = start_iter + self.end_iter = end_iter + + def __call__(self, i): + if i < self.start_iter: + return self.start_value + if i >= self.end_iter: + return self.end_value + part = (i - self.start_iter) / (self.end_iter - self.start_iter) + return self.start_value * (1 - part) + self.end_value * part + + +class LadderRamp: + def __init__(self, start_iters, values): + self.start_iters = start_iters + self.values = values + assert len(values) == len(start_iters) + 1, (len(values), len(start_iters)) + + def __call__(self, i): + segment_i = bisect.bisect_right(self.start_iters, i) + return self.values[segment_i] + + +def get_ramp(kind='ladder', **kwargs): + if kind == 'linear': + return LinearRamp(**kwargs) + if kind == 'ladder': + return LadderRamp(**kwargs) + raise ValueError(f'Unexpected ramp kind: {kind}') + + +def print_traceback_handler(sig, frame): + LOGGER.warning(f'Received signal {sig}') + bt = ''.join(traceback.format_stack()) + LOGGER.warning(f'Requested stack trace:\n{bt}') + + +def register_debug_signal_handlers(sig=None, handler=print_traceback_handler): + LOGGER.warning(f'Setting signal {sig} handler {handler}') + signal.signal(sig, handler) + + +def handle_deterministic_config(config): + seed = dict(config).get('seed', None) + if seed is None: + return False + + seed_everything(seed) + return True + + +def get_shape(t): + if torch.is_tensor(t): + return tuple(t.shape) + elif isinstance(t, dict): + return {n: get_shape(q) for n, q in t.items()} + elif isinstance(t, (list, tuple)): + return [get_shape(q) for q in t] + elif isinstance(t, numbers.Number): + return type(t) + else: + raise ValueError('unexpected type {}'.format(type(t))) + + +def get_has_ddp_rank(): + master_port = os.environ.get('MASTER_PORT', None) + node_rank = os.environ.get('NODE_RANK', None) + local_rank = os.environ.get('LOCAL_RANK', None) + world_size = os.environ.get('WORLD_SIZE', None) + has_rank = master_port is not None or node_rank is not None or local_rank is not None or world_size is not None + return has_rank + + +def handle_ddp_subprocess(): + def main_decorator(main_func): + @functools.wraps(main_func) + def new_main(*args, **kwargs): + # Trainer sets MASTER_PORT, NODE_RANK, LOCAL_RANK, WORLD_SIZE + parent_cwd = os.environ.get('TRAINING_PARENT_WORK_DIR', None) + has_parent = parent_cwd is not None + has_rank = get_has_ddp_rank() + assert has_parent == has_rank, f'Inconsistent state: has_parent={has_parent}, has_rank={has_rank}' + + if has_parent: + # we are in the worker + sys.argv.extend([ + f'hydra.run.dir={parent_cwd}', + # 'hydra/hydra_logging=disabled', + # 'hydra/job_logging=disabled' + ]) + # do nothing if this is a top-level process + # TRAINING_PARENT_WORK_DIR is set in handle_ddp_parent_process after hydra initialization + + main_func(*args, **kwargs) + return new_main + return main_decorator + + +def handle_ddp_parent_process(): + parent_cwd = os.environ.get('TRAINING_PARENT_WORK_DIR', None) + has_parent = parent_cwd is not None + has_rank = get_has_ddp_rank() + assert has_parent == has_rank, f'Inconsistent state: has_parent={has_parent}, has_rank={has_rank}' + + if parent_cwd is None: + os.environ['TRAINING_PARENT_WORK_DIR'] = os.getcwd() + + return has_parent diff --git a/annotator/leres/__init__.py b/annotator/leres/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9b11e44a954b68a634326d097bcb54b8876524b4 --- /dev/null +++ b/annotator/leres/__init__.py @@ -0,0 +1,113 @@ +import cv2 +import numpy as np +import torch +import os +from modules import devices, shared +from annotator.annotator_path import models_path +from torchvision.transforms import transforms + +# AdelaiDepth/LeReS imports +from .leres.depthmap import estimateleres, estimateboost +from .leres.multi_depth_model_woauxi import RelDepthModel +from .leres.net_tools import strip_prefix_if_present + +# pix2pix/merge net imports +from .pix2pix.options.test_options import TestOptions +from .pix2pix.models.pix2pix4depth_model import Pix2Pix4DepthModel + +base_model_path = os.path.join(models_path, "leres") +old_modeldir = os.path.dirname(os.path.realpath(__file__)) + +remote_model_path_leres = "https://huggingface.co./lllyasviel/Annotators/resolve/main/res101.pth" +remote_model_path_pix2pix = "https://huggingface.co./lllyasviel/Annotators/resolve/main/latest_net_G.pth" + +model = None +pix2pixmodel = None + +def unload_leres_model(): + global model, pix2pixmodel + if model is not None: + model = model.cpu() + if pix2pixmodel is not None: + pix2pixmodel = pix2pixmodel.unload_network('G') + + +def apply_leres(input_image, thr_a, thr_b, boost=False): + global model, pix2pixmodel + if model is None: + model_path = os.path.join(base_model_path, "res101.pth") + old_model_path = os.path.join(old_modeldir, "res101.pth") + + if os.path.exists(old_model_path): + model_path = old_model_path + elif not os.path.exists(model_path): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(remote_model_path_leres, model_dir=base_model_path) + + if torch.cuda.is_available(): + checkpoint = torch.load(model_path) + else: + checkpoint = torch.load(model_path, map_location=torch.device('cpu')) + + model = RelDepthModel(backbone='resnext101') + model.load_state_dict(strip_prefix_if_present(checkpoint['depth_model'], "module."), strict=True) + del checkpoint + + if boost and pix2pixmodel is None: + pix2pixmodel_path = os.path.join(base_model_path, "latest_net_G.pth") + if not os.path.exists(pix2pixmodel_path): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(remote_model_path_pix2pix, model_dir=base_model_path) + + opt = TestOptions().parse() + if not torch.cuda.is_available(): + opt.gpu_ids = [] # cpu mode + pix2pixmodel = Pix2Pix4DepthModel(opt) + pix2pixmodel.save_dir = base_model_path + pix2pixmodel.load_networks('latest') + pix2pixmodel.eval() + + if devices.get_device_for("controlnet").type != 'mps': + model = model.to(devices.get_device_for("controlnet")) + + assert input_image.ndim == 3 + height, width, dim = input_image.shape + + with torch.no_grad(): + + if boost: + depth = estimateboost(input_image, model, 0, pix2pixmodel, max(width, height)) + else: + depth = estimateleres(input_image, model, width, height) + + numbytes=2 + depth_min = depth.min() + depth_max = depth.max() + max_val = (2**(8*numbytes))-1 + + # check output before normalizing and mapping to 16 bit + if depth_max - depth_min > np.finfo("float").eps: + out = max_val * (depth - depth_min) / (depth_max - depth_min) + else: + out = np.zeros(depth.shape) + + # single channel, 16 bit image + depth_image = out.astype("uint16") + + # convert to uint8 + depth_image = cv2.convertScaleAbs(depth_image, alpha=(255.0/65535.0)) + + # remove near + if thr_a != 0: + thr_a = ((thr_a/100)*255) + depth_image = cv2.threshold(depth_image, thr_a, 255, cv2.THRESH_TOZERO)[1] + + # invert image + depth_image = cv2.bitwise_not(depth_image) + + # remove bg + if thr_b != 0: + thr_b = ((thr_b/100)*255) + depth_image = cv2.threshold(depth_image, thr_b, 255, cv2.THRESH_TOZERO)[1] + + return depth_image diff --git a/annotator/leres/leres/LICENSE b/annotator/leres/leres/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..e0f1d07d98d4e85e684734d058dfe2515d215405 --- /dev/null +++ b/annotator/leres/leres/LICENSE @@ -0,0 +1,23 @@ +https://github.com/thygate/stable-diffusion-webui-depthmap-script + +MIT License + +Copyright (c) 2023 Bob Thiry + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/annotator/leres/leres/Resnet.py b/annotator/leres/leres/Resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..f12c9975c1aa05401269be3ca3dbaa56bde55581 --- /dev/null +++ b/annotator/leres/leres/Resnet.py @@ -0,0 +1,199 @@ +import torch.nn as nn +import torch.nn as NN + +__all__ = ['ResNet', 'resnet18', 'resnet34', 'resnet50', 'resnet101', + 'resnet152'] + + +model_urls = { + 'resnet18': 'https://download.pytorch.org/models/resnet18-5c106cde.pth', + 'resnet34': 'https://download.pytorch.org/models/resnet34-333f7ec4.pth', + 'resnet50': 'https://download.pytorch.org/models/resnet50-19c8e357.pth', + 'resnet101': 'https://download.pytorch.org/models/resnet101-5d3b4d8f.pth', + 'resnet152': 'https://download.pytorch.org/models/resnet152-b121ed2d.pth', +} + + +def conv3x3(in_planes, out_planes, stride=1): + """3x3 convolution with padding""" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=1, bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(BasicBlock, self).__init__() + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = NN.BatchNorm2d(planes) #NN.BatchNorm2d + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = NN.BatchNorm2d(planes) #NN.BatchNorm2d + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) + self.bn1 = NN.BatchNorm2d(planes) #NN.BatchNorm2d + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, + padding=1, bias=False) + self.bn2 = NN.BatchNorm2d(planes) #NN.BatchNorm2d + self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1, bias=False) + self.bn3 = NN.BatchNorm2d(planes * self.expansion) #NN.BatchNorm2d + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__(self, block, layers, num_classes=1000): + self.inplanes = 64 + super(ResNet, self).__init__() + self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, + bias=False) + self.bn1 = NN.BatchNorm2d(64) #NN.BatchNorm2d + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2) + #self.avgpool = nn.AvgPool2d(7, stride=1) + #self.fc = nn.Linear(512 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, nn.BatchNorm2d): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + def _make_layer(self, block, planes, blocks, stride=1): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d(self.inplanes, planes * block.expansion, + kernel_size=1, stride=stride, bias=False), + NN.BatchNorm2d(planes * block.expansion), #NN.BatchNorm2d + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append(block(self.inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + features = [] + + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + features.append(x) + x = self.layer2(x) + features.append(x) + x = self.layer3(x) + features.append(x) + x = self.layer4(x) + features.append(x) + + return features + + +def resnet18(pretrained=True, **kwargs): + """Constructs a ResNet-18 model. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs) + return model + + +def resnet34(pretrained=True, **kwargs): + """Constructs a ResNet-34 model. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(BasicBlock, [3, 4, 6, 3], **kwargs) + return model + + +def resnet50(pretrained=True, **kwargs): + """Constructs a ResNet-50 model. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(Bottleneck, [3, 4, 6, 3], **kwargs) + + return model + + +def resnet101(pretrained=True, **kwargs): + """Constructs a ResNet-101 model. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(Bottleneck, [3, 4, 23, 3], **kwargs) + + return model + + +def resnet152(pretrained=True, **kwargs): + """Constructs a ResNet-152 model. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + model = ResNet(Bottleneck, [3, 8, 36, 3], **kwargs) + return model diff --git a/annotator/leres/leres/Resnext_torch.py b/annotator/leres/leres/Resnext_torch.py new file mode 100644 index 0000000000000000000000000000000000000000..9af54fcc3e5b363935ef60c8aaf269110c0d6611 --- /dev/null +++ b/annotator/leres/leres/Resnext_torch.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python +# coding: utf-8 +import torch.nn as nn + +try: + from urllib import urlretrieve +except ImportError: + from urllib.request import urlretrieve + +__all__ = ['resnext101_32x8d'] + + +model_urls = { + 'resnext50_32x4d': 'https://download.pytorch.org/models/resnext50_32x4d-7cdf4587.pth', + 'resnext101_32x8d': 'https://download.pytorch.org/models/resnext101_32x8d-8ba56ff5.pth', +} + + +def conv3x3(in_planes, out_planes, stride=1, groups=1, dilation=1): + """3x3 convolution with padding""" + return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, + padding=dilation, groups=groups, bias=False, dilation=dilation) + + +def conv1x1(in_planes, out_planes, stride=1): + """1x1 convolution""" + return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, + base_width=64, dilation=1, norm_layer=None): + super(BasicBlock, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + if groups != 1 or base_width != 64: + raise ValueError('BasicBlock only supports groups=1 and base_width=64') + if dilation > 1: + raise NotImplementedError("Dilation > 1 not supported in BasicBlock") + # Both self.conv1 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv3x3(inplanes, planes, stride) + self.bn1 = norm_layer(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = norm_layer(planes) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2) + # while original implementation places the stride at the first 1x1 convolution(self.conv1) + # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385. + # This variant is also known as ResNet V1.5 and improves accuracy according to + # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch. + + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None, groups=1, + base_width=64, dilation=1, norm_layer=None): + super(Bottleneck, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + width = int(planes * (base_width / 64.)) * groups + # Both self.conv2 and self.downsample layers downsample the input when stride != 1 + self.conv1 = conv1x1(inplanes, width) + self.bn1 = norm_layer(width) + self.conv2 = conv3x3(width, width, stride, groups, dilation) + self.bn2 = norm_layer(width) + self.conv3 = conv1x1(width, planes * self.expansion) + self.bn3 = norm_layer(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + def forward(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu(out) + + return out + + +class ResNet(nn.Module): + + def __init__(self, block, layers, num_classes=1000, zero_init_residual=False, + groups=1, width_per_group=64, replace_stride_with_dilation=None, + norm_layer=None): + super(ResNet, self).__init__() + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + self.inplanes = 64 + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False] + if len(replace_stride_with_dilation) != 3: + raise ValueError("replace_stride_with_dilation should be None " + "or a 3-element tuple, got {}".format(replace_stride_with_dilation)) + self.groups = groups + self.base_width = width_per_group + self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, + bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + self.layer1 = self._make_layer(block, 64, layers[0]) + self.layer2 = self._make_layer(block, 128, layers[1], stride=2, + dilate=replace_stride_with_dilation[0]) + self.layer3 = self._make_layer(block, 256, layers[2], stride=2, + dilate=replace_stride_with_dilation[1]) + self.layer4 = self._make_layer(block, 512, layers[3], stride=2, + dilate=replace_stride_with_dilation[2]) + #self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + #self.fc = nn.Linear(512 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, 0) + + def _make_layer(self, block, planes, blocks, stride=1, dilate=False): + norm_layer = self._norm_layer + downsample = None + previous_dilation = self.dilation + if dilate: + self.dilation *= stride + stride = 1 + if stride != 1 or self.inplanes != planes * block.expansion: + downsample = nn.Sequential( + conv1x1(self.inplanes, planes * block.expansion, stride), + norm_layer(planes * block.expansion), + ) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample, self.groups, + self.base_width, previous_dilation, norm_layer)) + self.inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append(block(self.inplanes, planes, groups=self.groups, + base_width=self.base_width, dilation=self.dilation, + norm_layer=norm_layer)) + + return nn.Sequential(*layers) + + def _forward_impl(self, x): + # See note [TorchScript super()] + features = [] + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + + x = self.layer1(x) + features.append(x) + + x = self.layer2(x) + features.append(x) + + x = self.layer3(x) + features.append(x) + + x = self.layer4(x) + features.append(x) + + #x = self.avgpool(x) + #x = torch.flatten(x, 1) + #x = self.fc(x) + + return features + + def forward(self, x): + return self._forward_impl(x) + + + +def resnext101_32x8d(pretrained=True, **kwargs): + """Constructs a ResNet-152 model. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + """ + kwargs['groups'] = 32 + kwargs['width_per_group'] = 8 + + model = ResNet(Bottleneck, [3, 4, 23, 3], **kwargs) + return model + diff --git a/annotator/leres/leres/depthmap.py b/annotator/leres/leres/depthmap.py new file mode 100644 index 0000000000000000000000000000000000000000..ebceecbe28ec248f6f96bb65b1c53bdbaf393ecc --- /dev/null +++ b/annotator/leres/leres/depthmap.py @@ -0,0 +1,546 @@ +# Author: thygate +# https://github.com/thygate/stable-diffusion-webui-depthmap-script + +from modules import devices +from modules.shared import opts +from torchvision.transforms import transforms +from operator import getitem + +import torch, gc +import cv2 +import numpy as np +import skimage.measure + +whole_size_threshold = 1600 # R_max from the paper +pix2pixsize = 1024 + +def scale_torch(img): + """ + Scale the image and output it in torch.tensor. + :param img: input rgb is in shape [H, W, C], input depth/disp is in shape [H, W] + :param scale: the scale factor. float + :return: img. [C, H, W] + """ + if len(img.shape) == 2: + img = img[np.newaxis, :, :] + if img.shape[2] == 3: + transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406) , (0.229, 0.224, 0.225) )]) + img = transform(img.astype(np.float32)) + else: + img = img.astype(np.float32) + img = torch.from_numpy(img) + return img + +def estimateleres(img, model, w, h): + # leres transform input + rgb_c = img[:, :, ::-1].copy() + A_resize = cv2.resize(rgb_c, (w, h)) + img_torch = scale_torch(A_resize)[None, :, :, :] + + # compute + with torch.no_grad(): + img_torch = img_torch.to(devices.get_device_for("controlnet")) + prediction = model.depth_model(img_torch) + + prediction = prediction.squeeze().cpu().numpy() + prediction = cv2.resize(prediction, (img.shape[1], img.shape[0]), interpolation=cv2.INTER_CUBIC) + + return prediction + +def generatemask(size): + # Generates a Guassian mask + mask = np.zeros(size, dtype=np.float32) + sigma = int(size[0]/16) + k_size = int(2 * np.ceil(2 * int(size[0]/16)) + 1) + mask[int(0.15*size[0]):size[0] - int(0.15*size[0]), int(0.15*size[1]): size[1] - int(0.15*size[1])] = 1 + mask = cv2.GaussianBlur(mask, (int(k_size), int(k_size)), sigma) + mask = (mask - mask.min()) / (mask.max() - mask.min()) + mask = mask.astype(np.float32) + return mask + +def resizewithpool(img, size): + i_size = img.shape[0] + n = int(np.floor(i_size/size)) + + out = skimage.measure.block_reduce(img, (n, n), np.max) + return out + +def rgb2gray(rgb): + # Converts rgb to gray + return np.dot(rgb[..., :3], [0.2989, 0.5870, 0.1140]) + +def calculateprocessingres(img, basesize, confidence=0.1, scale_threshold=3, whole_size_threshold=3000): + # Returns the R_x resolution described in section 5 of the main paper. + + # Parameters: + # img :input rgb image + # basesize : size the dilation kernel which is equal to receptive field of the network. + # confidence: value of x in R_x; allowed percentage of pixels that are not getting any contextual cue. + # scale_threshold: maximum allowed upscaling on the input image ; it has been set to 3. + # whole_size_threshold: maximum allowed resolution. (R_max from section 6 of the main paper) + + # Returns: + # outputsize_scale*speed_scale :The computed R_x resolution + # patch_scale: K parameter from section 6 of the paper + + # speed scale parameter is to process every image in a smaller size to accelerate the R_x resolution search + speed_scale = 32 + image_dim = int(min(img.shape[0:2])) + + gray = rgb2gray(img) + grad = np.abs(cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=3)) + np.abs(cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=3)) + grad = cv2.resize(grad, (image_dim, image_dim), cv2.INTER_AREA) + + # thresholding the gradient map to generate the edge-map as a proxy of the contextual cues + m = grad.min() + M = grad.max() + middle = m + (0.4 * (M - m)) + grad[grad < middle] = 0 + grad[grad >= middle] = 1 + + # dilation kernel with size of the receptive field + kernel = np.ones((int(basesize/speed_scale), int(basesize/speed_scale)), float) + # dilation kernel with size of the a quarter of receptive field used to compute k + # as described in section 6 of main paper + kernel2 = np.ones((int(basesize / (4*speed_scale)), int(basesize / (4*speed_scale))), float) + + # Output resolution limit set by the whole_size_threshold and scale_threshold. + threshold = min(whole_size_threshold, scale_threshold * max(img.shape[:2])) + + outputsize_scale = basesize / speed_scale + for p_size in range(int(basesize/speed_scale), int(threshold/speed_scale), int(basesize / (2*speed_scale))): + grad_resized = resizewithpool(grad, p_size) + grad_resized = cv2.resize(grad_resized, (p_size, p_size), cv2.INTER_NEAREST) + grad_resized[grad_resized >= 0.5] = 1 + grad_resized[grad_resized < 0.5] = 0 + + dilated = cv2.dilate(grad_resized, kernel, iterations=1) + meanvalue = (1-dilated).mean() + if meanvalue > confidence: + break + else: + outputsize_scale = p_size + + grad_region = cv2.dilate(grad_resized, kernel2, iterations=1) + patch_scale = grad_region.mean() + + return int(outputsize_scale*speed_scale), patch_scale + +# Generate a double-input depth estimation +def doubleestimate(img, size1, size2, pix2pixsize, model, net_type, pix2pixmodel): + # Generate the low resolution estimation + estimate1 = singleestimate(img, size1, model, net_type) + # Resize to the inference size of merge network. + estimate1 = cv2.resize(estimate1, (pix2pixsize, pix2pixsize), interpolation=cv2.INTER_CUBIC) + + # Generate the high resolution estimation + estimate2 = singleestimate(img, size2, model, net_type) + # Resize to the inference size of merge network. + estimate2 = cv2.resize(estimate2, (pix2pixsize, pix2pixsize), interpolation=cv2.INTER_CUBIC) + + # Inference on the merge model + pix2pixmodel.set_input(estimate1, estimate2) + pix2pixmodel.test() + visuals = pix2pixmodel.get_current_visuals() + prediction_mapped = visuals['fake_B'] + prediction_mapped = (prediction_mapped+1)/2 + prediction_mapped = (prediction_mapped - torch.min(prediction_mapped)) / ( + torch.max(prediction_mapped) - torch.min(prediction_mapped)) + prediction_mapped = prediction_mapped.squeeze().cpu().numpy() + + return prediction_mapped + +# Generate a single-input depth estimation +def singleestimate(img, msize, model, net_type): + # if net_type == 0: + return estimateleres(img, model, msize, msize) + # else: + # return estimatemidasBoost(img, model, msize, msize) + +def applyGridpatch(blsize, stride, img, box): + # Extract a simple grid patch. + counter1 = 0 + patch_bound_list = {} + for k in range(blsize, img.shape[1] - blsize, stride): + for j in range(blsize, img.shape[0] - blsize, stride): + patch_bound_list[str(counter1)] = {} + patchbounds = [j - blsize, k - blsize, j - blsize + 2 * blsize, k - blsize + 2 * blsize] + patch_bound = [box[0] + patchbounds[1], box[1] + patchbounds[0], patchbounds[3] - patchbounds[1], + patchbounds[2] - patchbounds[0]] + patch_bound_list[str(counter1)]['rect'] = patch_bound + patch_bound_list[str(counter1)]['size'] = patch_bound[2] + counter1 = counter1 + 1 + return patch_bound_list + +# Generating local patches to perform the local refinement described in section 6 of the main paper. +def generatepatchs(img, base_size): + + # Compute the gradients as a proxy of the contextual cues. + img_gray = rgb2gray(img) + whole_grad = np.abs(cv2.Sobel(img_gray, cv2.CV_64F, 0, 1, ksize=3)) +\ + np.abs(cv2.Sobel(img_gray, cv2.CV_64F, 1, 0, ksize=3)) + + threshold = whole_grad[whole_grad > 0].mean() + whole_grad[whole_grad < threshold] = 0 + + # We use the integral image to speed-up the evaluation of the amount of gradients for each patch. + gf = whole_grad.sum()/len(whole_grad.reshape(-1)) + grad_integral_image = cv2.integral(whole_grad) + + # Variables are selected such that the initial patch size would be the receptive field size + # and the stride is set to 1/3 of the receptive field size. + blsize = int(round(base_size/2)) + stride = int(round(blsize*0.75)) + + # Get initial Grid + patch_bound_list = applyGridpatch(blsize, stride, img, [0, 0, 0, 0]) + + # Refine initial Grid of patches by discarding the flat (in terms of gradients of the rgb image) ones. Refine + # each patch size to ensure that there will be enough depth cues for the network to generate a consistent depth map. + print("Selecting patches ...") + patch_bound_list = adaptiveselection(grad_integral_image, patch_bound_list, gf) + + # Sort the patch list to make sure the merging operation will be done with the correct order: starting from biggest + # patch + patchset = sorted(patch_bound_list.items(), key=lambda x: getitem(x[1], 'size'), reverse=True) + return patchset + +def getGF_fromintegral(integralimage, rect): + # Computes the gradient density of a given patch from the gradient integral image. + x1 = rect[1] + x2 = rect[1]+rect[3] + y1 = rect[0] + y2 = rect[0]+rect[2] + value = integralimage[x2, y2]-integralimage[x1, y2]-integralimage[x2, y1]+integralimage[x1, y1] + return value + +# Adaptively select patches +def adaptiveselection(integral_grad, patch_bound_list, gf): + patchlist = {} + count = 0 + height, width = integral_grad.shape + + search_step = int(32/factor) + + # Go through all patches + for c in range(len(patch_bound_list)): + # Get patch + bbox = patch_bound_list[str(c)]['rect'] + + # Compute the amount of gradients present in the patch from the integral image. + cgf = getGF_fromintegral(integral_grad, bbox)/(bbox[2]*bbox[3]) + + # Check if patching is beneficial by comparing the gradient density of the patch to + # the gradient density of the whole image + if cgf >= gf: + bbox_test = bbox.copy() + patchlist[str(count)] = {} + + # Enlarge each patch until the gradient density of the patch is equal + # to the whole image gradient density + while True: + + bbox_test[0] = bbox_test[0] - int(search_step/2) + bbox_test[1] = bbox_test[1] - int(search_step/2) + + bbox_test[2] = bbox_test[2] + search_step + bbox_test[3] = bbox_test[3] + search_step + + # Check if we are still within the image + if bbox_test[0] < 0 or bbox_test[1] < 0 or bbox_test[1] + bbox_test[3] >= height \ + or bbox_test[0] + bbox_test[2] >= width: + break + + # Compare gradient density + cgf = getGF_fromintegral(integral_grad, bbox_test)/(bbox_test[2]*bbox_test[3]) + if cgf < gf: + break + bbox = bbox_test.copy() + + # Add patch to selected patches + patchlist[str(count)]['rect'] = bbox + patchlist[str(count)]['size'] = bbox[2] + count = count + 1 + + # Return selected patches + return patchlist + +def impatch(image, rect): + # Extract the given patch pixels from a given image. + w1 = rect[0] + h1 = rect[1] + w2 = w1 + rect[2] + h2 = h1 + rect[3] + image_patch = image[h1:h2, w1:w2] + return image_patch + +class ImageandPatchs: + def __init__(self, root_dir, name, patchsinfo, rgb_image, scale=1): + self.root_dir = root_dir + self.patchsinfo = patchsinfo + self.name = name + self.patchs = patchsinfo + self.scale = scale + + self.rgb_image = cv2.resize(rgb_image, (round(rgb_image.shape[1]*scale), round(rgb_image.shape[0]*scale)), + interpolation=cv2.INTER_CUBIC) + + self.do_have_estimate = False + self.estimation_updated_image = None + self.estimation_base_image = None + + def __len__(self): + return len(self.patchs) + + def set_base_estimate(self, est): + self.estimation_base_image = est + if self.estimation_updated_image is not None: + self.do_have_estimate = True + + def set_updated_estimate(self, est): + self.estimation_updated_image = est + if self.estimation_base_image is not None: + self.do_have_estimate = True + + def __getitem__(self, index): + patch_id = int(self.patchs[index][0]) + rect = np.array(self.patchs[index][1]['rect']) + msize = self.patchs[index][1]['size'] + + ## applying scale to rect: + rect = np.round(rect * self.scale) + rect = rect.astype('int') + msize = round(msize * self.scale) + + patch_rgb = impatch(self.rgb_image, rect) + if self.do_have_estimate: + patch_whole_estimate_base = impatch(self.estimation_base_image, rect) + patch_whole_estimate_updated = impatch(self.estimation_updated_image, rect) + return {'patch_rgb': patch_rgb, 'patch_whole_estimate_base': patch_whole_estimate_base, + 'patch_whole_estimate_updated': patch_whole_estimate_updated, 'rect': rect, + 'size': msize, 'id': patch_id} + else: + return {'patch_rgb': patch_rgb, 'rect': rect, 'size': msize, 'id': patch_id} + + def print_options(self, opt): + """Print and save options + + It will print both current options and default values(if different). + It will save options into a text file / [checkpoints_dir] / opt.txt + """ + message = '' + message += '----------------- Options ---------------\n' + for k, v in sorted(vars(opt).items()): + comment = '' + default = self.parser.get_default(k) + if v != default: + comment = '\t[default: %s]' % str(default) + message += '{:>25}: {:<30}{}\n'.format(str(k), str(v), comment) + message += '----------------- End -------------------' + print(message) + + # save to the disk + """ + expr_dir = os.path.join(opt.checkpoints_dir, opt.name) + util.mkdirs(expr_dir) + file_name = os.path.join(expr_dir, '{}_opt.txt'.format(opt.phase)) + with open(file_name, 'wt') as opt_file: + opt_file.write(message) + opt_file.write('\n') + """ + + def parse(self): + """Parse our options, create checkpoints directory suffix, and set up gpu device.""" + opt = self.gather_options() + opt.isTrain = self.isTrain # train or test + + # process opt.suffix + if opt.suffix: + suffix = ('_' + opt.suffix.format(**vars(opt))) if opt.suffix != '' else '' + opt.name = opt.name + suffix + + #self.print_options(opt) + + # set gpu ids + str_ids = opt.gpu_ids.split(',') + opt.gpu_ids = [] + for str_id in str_ids: + id = int(str_id) + if id >= 0: + opt.gpu_ids.append(id) + #if len(opt.gpu_ids) > 0: + # torch.cuda.set_device(opt.gpu_ids[0]) + + self.opt = opt + return self.opt + + +def estimateboost(img, model, model_type, pix2pixmodel, max_res=512): + global whole_size_threshold + + # get settings + if hasattr(opts, 'depthmap_script_boost_rmax'): + whole_size_threshold = opts.depthmap_script_boost_rmax + + if model_type == 0: #leres + net_receptive_field_size = 448 + patch_netsize = 2 * net_receptive_field_size + elif model_type == 1: #dpt_beit_large_512 + net_receptive_field_size = 512 + patch_netsize = 2 * net_receptive_field_size + else: #other midas + net_receptive_field_size = 384 + patch_netsize = 2 * net_receptive_field_size + + gc.collect() + devices.torch_gc() + + # Generate mask used to smoothly blend the local pathc estimations to the base estimate. + # It is arbitrarily large to avoid artifacts during rescaling for each crop. + mask_org = generatemask((3000, 3000)) + mask = mask_org.copy() + + # Value x of R_x defined in the section 5 of the main paper. + r_threshold_value = 0.2 + #if R0: + # r_threshold_value = 0 + + input_resolution = img.shape + scale_threshold = 3 # Allows up-scaling with a scale up to 3 + + # Find the best input resolution R-x. The resolution search described in section 5-double estimation of the main paper and section B of the + # supplementary material. + whole_image_optimal_size, patch_scale = calculateprocessingres(img, net_receptive_field_size, r_threshold_value, scale_threshold, whole_size_threshold) + + # print('wholeImage being processed in :', whole_image_optimal_size) + + # Generate the base estimate using the double estimation. + whole_estimate = doubleestimate(img, net_receptive_field_size, whole_image_optimal_size, pix2pixsize, model, model_type, pix2pixmodel) + + # Compute the multiplier described in section 6 of the main paper to make sure our initial patch can select + # small high-density regions of the image. + global factor + factor = max(min(1, 4 * patch_scale * whole_image_optimal_size / whole_size_threshold), 0.2) + # print('Adjust factor is:', 1/factor) + + # Check if Local boosting is beneficial. + if max_res < whole_image_optimal_size: + # print("No Local boosting. Specified Max Res is smaller than R20, Returning doubleestimate result") + return cv2.resize(whole_estimate, (input_resolution[1], input_resolution[0]), interpolation=cv2.INTER_CUBIC) + + # Compute the default target resolution. + if img.shape[0] > img.shape[1]: + a = 2 * whole_image_optimal_size + b = round(2 * whole_image_optimal_size * img.shape[1] / img.shape[0]) + else: + a = round(2 * whole_image_optimal_size * img.shape[0] / img.shape[1]) + b = 2 * whole_image_optimal_size + b = int(round(b / factor)) + a = int(round(a / factor)) + + """ + # recompute a, b and saturate to max res. + if max(a,b) > max_res: + print('Default Res is higher than max-res: Reducing final resolution') + if img.shape[0] > img.shape[1]: + a = max_res + b = round(max_res * img.shape[1] / img.shape[0]) + else: + a = round(max_res * img.shape[0] / img.shape[1]) + b = max_res + b = int(b) + a = int(a) + """ + + img = cv2.resize(img, (b, a), interpolation=cv2.INTER_CUBIC) + + # Extract selected patches for local refinement + base_size = net_receptive_field_size * 2 + patchset = generatepatchs(img, base_size) + + # print('Target resolution: ', img.shape) + + # Computing a scale in case user prompted to generate the results as the same resolution of the input. + # Notice that our method output resolution is independent of the input resolution and this parameter will only + # enable a scaling operation during the local patch merge implementation to generate results with the same resolution + # as the input. + """ + if output_resolution == 1: + mergein_scale = input_resolution[0] / img.shape[0] + print('Dynamicly change merged-in resolution; scale:', mergein_scale) + else: + mergein_scale = 1 + """ + # always rescale to input res for now + mergein_scale = input_resolution[0] / img.shape[0] + + imageandpatchs = ImageandPatchs('', '', patchset, img, mergein_scale) + whole_estimate_resized = cv2.resize(whole_estimate, (round(img.shape[1]*mergein_scale), + round(img.shape[0]*mergein_scale)), interpolation=cv2.INTER_CUBIC) + imageandpatchs.set_base_estimate(whole_estimate_resized.copy()) + imageandpatchs.set_updated_estimate(whole_estimate_resized.copy()) + + print('Resulting depthmap resolution will be :', whole_estimate_resized.shape[:2]) + print('Patches to process: '+str(len(imageandpatchs))) + + # Enumerate through all patches, generate their estimations and refining the base estimate. + for patch_ind in range(len(imageandpatchs)): + + # Get patch information + patch = imageandpatchs[patch_ind] # patch object + patch_rgb = patch['patch_rgb'] # rgb patch + patch_whole_estimate_base = patch['patch_whole_estimate_base'] # corresponding patch from base + rect = patch['rect'] # patch size and location + patch_id = patch['id'] # patch ID + org_size = patch_whole_estimate_base.shape # the original size from the unscaled input + print('\t Processing patch', patch_ind, '/', len(imageandpatchs)-1, '|', rect) + + # We apply double estimation for patches. The high resolution value is fixed to twice the receptive + # field size of the network for patches to accelerate the process. + patch_estimation = doubleestimate(patch_rgb, net_receptive_field_size, patch_netsize, pix2pixsize, model, model_type, pix2pixmodel) + patch_estimation = cv2.resize(patch_estimation, (pix2pixsize, pix2pixsize), interpolation=cv2.INTER_CUBIC) + patch_whole_estimate_base = cv2.resize(patch_whole_estimate_base, (pix2pixsize, pix2pixsize), interpolation=cv2.INTER_CUBIC) + + # Merging the patch estimation into the base estimate using our merge network: + # We feed the patch estimation and the same region from the updated base estimate to the merge network + # to generate the target estimate for the corresponding region. + pix2pixmodel.set_input(patch_whole_estimate_base, patch_estimation) + + # Run merging network + pix2pixmodel.test() + visuals = pix2pixmodel.get_current_visuals() + + prediction_mapped = visuals['fake_B'] + prediction_mapped = (prediction_mapped+1)/2 + prediction_mapped = prediction_mapped.squeeze().cpu().numpy() + + mapped = prediction_mapped + + # We use a simple linear polynomial to make sure the result of the merge network would match the values of + # base estimate + p_coef = np.polyfit(mapped.reshape(-1), patch_whole_estimate_base.reshape(-1), deg=1) + merged = np.polyval(p_coef, mapped.reshape(-1)).reshape(mapped.shape) + + merged = cv2.resize(merged, (org_size[1],org_size[0]), interpolation=cv2.INTER_CUBIC) + + # Get patch size and location + w1 = rect[0] + h1 = rect[1] + w2 = w1 + rect[2] + h2 = h1 + rect[3] + + # To speed up the implementation, we only generate the Gaussian mask once with a sufficiently large size + # and resize it to our needed size while merging the patches. + if mask.shape != org_size: + mask = cv2.resize(mask_org, (org_size[1],org_size[0]), interpolation=cv2.INTER_LINEAR) + + tobemergedto = imageandpatchs.estimation_updated_image + + # Update the whole estimation: + # We use a simple Gaussian mask to blend the merged patch region with the base estimate to ensure seamless + # blending at the boundaries of the patch region. + tobemergedto[h1:h2, w1:w2] = np.multiply(tobemergedto[h1:h2, w1:w2], 1 - mask) + np.multiply(merged, mask) + imageandpatchs.set_updated_estimate(tobemergedto) + + # output + return cv2.resize(imageandpatchs.estimation_updated_image, (input_resolution[1], input_resolution[0]), interpolation=cv2.INTER_CUBIC) diff --git a/annotator/leres/leres/multi_depth_model_woauxi.py b/annotator/leres/leres/multi_depth_model_woauxi.py new file mode 100644 index 0000000000000000000000000000000000000000..822ab0893267042446c2a24ed35b4ea053c9914a --- /dev/null +++ b/annotator/leres/leres/multi_depth_model_woauxi.py @@ -0,0 +1,34 @@ +from . import network_auxi as network +from .net_tools import get_func +import torch +import torch.nn as nn +from modules import devices + +class RelDepthModel(nn.Module): + def __init__(self, backbone='resnet50'): + super(RelDepthModel, self).__init__() + if backbone == 'resnet50': + encoder = 'resnet50_stride32' + elif backbone == 'resnext101': + encoder = 'resnext101_stride32x8d' + self.depth_model = DepthModel(encoder) + + def inference(self, rgb): + with torch.no_grad(): + input = rgb.to(self.depth_model.device) + depth = self.depth_model(input) + #pred_depth_out = depth - depth.min() + 0.01 + return depth #pred_depth_out + + +class DepthModel(nn.Module): + def __init__(self, encoder): + super(DepthModel, self).__init__() + backbone = network.__name__.split('.')[-1] + '.' + encoder + self.encoder_modules = get_func(backbone)() + self.decoder_modules = network.Decoder() + + def forward(self, x): + lateral_out = self.encoder_modules(x) + out_logit = self.decoder_modules(lateral_out) + return out_logit \ No newline at end of file diff --git a/annotator/leres/leres/net_tools.py b/annotator/leres/leres/net_tools.py new file mode 100644 index 0000000000000000000000000000000000000000..745ba5a0ef19adb869525e6b252db86780b8126e --- /dev/null +++ b/annotator/leres/leres/net_tools.py @@ -0,0 +1,54 @@ +import importlib +import torch +import os +from collections import OrderedDict + + +def get_func(func_name): + """Helper to return a function object by name. func_name must identify a + function in this module or the path to a function relative to the base + 'modeling' module. + """ + if func_name == '': + return None + try: + parts = func_name.split('.') + # Refers to a function in this module + if len(parts) == 1: + return globals()[parts[0]] + # Otherwise, assume we're referencing a module under modeling + module_name = 'annotator.leres.leres.' + '.'.join(parts[:-1]) + module = importlib.import_module(module_name) + return getattr(module, parts[-1]) + except Exception: + print('Failed to f1ind function: %s', func_name) + raise + +def load_ckpt(args, depth_model, shift_model, focal_model): + """ + Load checkpoint. + """ + if os.path.isfile(args.load_ckpt): + print("loading checkpoint %s" % args.load_ckpt) + checkpoint = torch.load(args.load_ckpt) + if shift_model is not None: + shift_model.load_state_dict(strip_prefix_if_present(checkpoint['shift_model'], 'module.'), + strict=True) + if focal_model is not None: + focal_model.load_state_dict(strip_prefix_if_present(checkpoint['focal_model'], 'module.'), + strict=True) + depth_model.load_state_dict(strip_prefix_if_present(checkpoint['depth_model'], "module."), + strict=True) + del checkpoint + if torch.cuda.is_available(): + torch.cuda.empty_cache() + + +def strip_prefix_if_present(state_dict, prefix): + keys = sorted(state_dict.keys()) + if not all(key.startswith(prefix) for key in keys): + return state_dict + stripped_state_dict = OrderedDict() + for key, value in state_dict.items(): + stripped_state_dict[key.replace(prefix, "")] = value + return stripped_state_dict \ No newline at end of file diff --git a/annotator/leres/leres/network_auxi.py b/annotator/leres/leres/network_auxi.py new file mode 100644 index 0000000000000000000000000000000000000000..1bd87011a5339aca632d1a10b217c8737bdc794f --- /dev/null +++ b/annotator/leres/leres/network_auxi.py @@ -0,0 +1,417 @@ +import torch +import torch.nn as nn +import torch.nn.init as init + +from . import Resnet, Resnext_torch + + +def resnet50_stride32(): + return DepthNet(backbone='resnet', depth=50, upfactors=[2, 2, 2, 2]) + +def resnext101_stride32x8d(): + return DepthNet(backbone='resnext101_32x8d', depth=101, upfactors=[2, 2, 2, 2]) + + +class Decoder(nn.Module): + def __init__(self): + super(Decoder, self).__init__() + self.inchannels = [256, 512, 1024, 2048] + self.midchannels = [256, 256, 256, 512] + self.upfactors = [2,2,2,2] + self.outchannels = 1 + + self.conv = FTB(inchannels=self.inchannels[3], midchannels=self.midchannels[3]) + self.conv1 = nn.Conv2d(in_channels=self.midchannels[3], out_channels=self.midchannels[2], kernel_size=3, padding=1, stride=1, bias=True) + self.upsample = nn.Upsample(scale_factor=self.upfactors[3], mode='bilinear', align_corners=True) + + self.ffm2 = FFM(inchannels=self.inchannels[2], midchannels=self.midchannels[2], outchannels = self.midchannels[2], upfactor=self.upfactors[2]) + self.ffm1 = FFM(inchannels=self.inchannels[1], midchannels=self.midchannels[1], outchannels = self.midchannels[1], upfactor=self.upfactors[1]) + self.ffm0 = FFM(inchannels=self.inchannels[0], midchannels=self.midchannels[0], outchannels = self.midchannels[0], upfactor=self.upfactors[0]) + + self.outconv = AO(inchannels=self.midchannels[0], outchannels=self.outchannels, upfactor=2) + self._init_params() + + def _init_params(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + init.normal_(m.weight, std=0.01) + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.ConvTranspose2d): + init.normal_(m.weight, std=0.01) + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): #NN.BatchNorm2d + init.constant_(m.weight, 1) + init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + init.normal_(m.weight, std=0.01) + if m.bias is not None: + init.constant_(m.bias, 0) + + def forward(self, features): + x_32x = self.conv(features[3]) # 1/32 + x_32 = self.conv1(x_32x) + x_16 = self.upsample(x_32) # 1/16 + + x_8 = self.ffm2(features[2], x_16) # 1/8 + x_4 = self.ffm1(features[1], x_8) # 1/4 + x_2 = self.ffm0(features[0], x_4) # 1/2 + #----------------------------------------- + x = self.outconv(x_2) # original size + return x + +class DepthNet(nn.Module): + __factory = { + 18: Resnet.resnet18, + 34: Resnet.resnet34, + 50: Resnet.resnet50, + 101: Resnet.resnet101, + 152: Resnet.resnet152 + } + def __init__(self, + backbone='resnet', + depth=50, + upfactors=[2, 2, 2, 2]): + super(DepthNet, self).__init__() + self.backbone = backbone + self.depth = depth + self.pretrained = False + self.inchannels = [256, 512, 1024, 2048] + self.midchannels = [256, 256, 256, 512] + self.upfactors = upfactors + self.outchannels = 1 + + # Build model + if self.backbone == 'resnet': + if self.depth not in DepthNet.__factory: + raise KeyError("Unsupported depth:", self.depth) + self.encoder = DepthNet.__factory[depth](pretrained=self.pretrained) + elif self.backbone == 'resnext101_32x8d': + self.encoder = Resnext_torch.resnext101_32x8d(pretrained=self.pretrained) + else: + self.encoder = Resnext_torch.resnext101(pretrained=self.pretrained) + + def forward(self, x): + x = self.encoder(x) # 1/32, 1/16, 1/8, 1/4 + return x + + +class FTB(nn.Module): + def __init__(self, inchannels, midchannels=512): + super(FTB, self).__init__() + self.in1 = inchannels + self.mid = midchannels + self.conv1 = nn.Conv2d(in_channels=self.in1, out_channels=self.mid, kernel_size=3, padding=1, stride=1, + bias=True) + # NN.BatchNorm2d + self.conv_branch = nn.Sequential(nn.ReLU(inplace=True), \ + nn.Conv2d(in_channels=self.mid, out_channels=self.mid, kernel_size=3, + padding=1, stride=1, bias=True), \ + nn.BatchNorm2d(num_features=self.mid), \ + nn.ReLU(inplace=True), \ + nn.Conv2d(in_channels=self.mid, out_channels=self.mid, kernel_size=3, + padding=1, stride=1, bias=True)) + self.relu = nn.ReLU(inplace=True) + + self.init_params() + + def forward(self, x): + x = self.conv1(x) + x = x + self.conv_branch(x) + x = self.relu(x) + + return x + + def init_params(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + init.normal_(m.weight, std=0.01) + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.ConvTranspose2d): + # init.kaiming_normal_(m.weight, mode='fan_out') + init.normal_(m.weight, std=0.01) + # init.xavier_normal_(m.weight) + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): # NN.BatchNorm2d + init.constant_(m.weight, 1) + init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + init.normal_(m.weight, std=0.01) + if m.bias is not None: + init.constant_(m.bias, 0) + + +class ATA(nn.Module): + def __init__(self, inchannels, reduction=8): + super(ATA, self).__init__() + self.inchannels = inchannels + self.avg_pool = nn.AdaptiveAvgPool2d(1) + self.fc = nn.Sequential(nn.Linear(self.inchannels * 2, self.inchannels // reduction), + nn.ReLU(inplace=True), + nn.Linear(self.inchannels // reduction, self.inchannels), + nn.Sigmoid()) + self.init_params() + + def forward(self, low_x, high_x): + n, c, _, _ = low_x.size() + x = torch.cat([low_x, high_x], 1) + x = self.avg_pool(x) + x = x.view(n, -1) + x = self.fc(x).view(n, c, 1, 1) + x = low_x * x + high_x + + return x + + def init_params(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + # init.kaiming_normal_(m.weight, mode='fan_out') + # init.normal(m.weight, std=0.01) + init.xavier_normal_(m.weight) + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.ConvTranspose2d): + # init.kaiming_normal_(m.weight, mode='fan_out') + # init.normal_(m.weight, std=0.01) + init.xavier_normal_(m.weight) + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): # NN.BatchNorm2d + init.constant_(m.weight, 1) + init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + init.normal_(m.weight, std=0.01) + if m.bias is not None: + init.constant_(m.bias, 0) + + +class FFM(nn.Module): + def __init__(self, inchannels, midchannels, outchannels, upfactor=2): + super(FFM, self).__init__() + self.inchannels = inchannels + self.midchannels = midchannels + self.outchannels = outchannels + self.upfactor = upfactor + + self.ftb1 = FTB(inchannels=self.inchannels, midchannels=self.midchannels) + # self.ata = ATA(inchannels = self.midchannels) + self.ftb2 = FTB(inchannels=self.midchannels, midchannels=self.outchannels) + + self.upsample = nn.Upsample(scale_factor=self.upfactor, mode='bilinear', align_corners=True) + + self.init_params() + + def forward(self, low_x, high_x): + x = self.ftb1(low_x) + x = x + high_x + x = self.ftb2(x) + x = self.upsample(x) + + return x + + def init_params(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + # init.kaiming_normal_(m.weight, mode='fan_out') + init.normal_(m.weight, std=0.01) + # init.xavier_normal_(m.weight) + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.ConvTranspose2d): + # init.kaiming_normal_(m.weight, mode='fan_out') + init.normal_(m.weight, std=0.01) + # init.xavier_normal_(m.weight) + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): # NN.Batchnorm2d + init.constant_(m.weight, 1) + init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + init.normal_(m.weight, std=0.01) + if m.bias is not None: + init.constant_(m.bias, 0) + + +class AO(nn.Module): + # Adaptive output module + def __init__(self, inchannels, outchannels, upfactor=2): + super(AO, self).__init__() + self.inchannels = inchannels + self.outchannels = outchannels + self.upfactor = upfactor + + self.adapt_conv = nn.Sequential( + nn.Conv2d(in_channels=self.inchannels, out_channels=self.inchannels // 2, kernel_size=3, padding=1, + stride=1, bias=True), \ + nn.BatchNorm2d(num_features=self.inchannels // 2), \ + nn.ReLU(inplace=True), \ + nn.Conv2d(in_channels=self.inchannels // 2, out_channels=self.outchannels, kernel_size=3, padding=1, + stride=1, bias=True), \ + nn.Upsample(scale_factor=self.upfactor, mode='bilinear', align_corners=True)) + + self.init_params() + + def forward(self, x): + x = self.adapt_conv(x) + return x + + def init_params(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + # init.kaiming_normal_(m.weight, mode='fan_out') + init.normal_(m.weight, std=0.01) + # init.xavier_normal_(m.weight) + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.ConvTranspose2d): + # init.kaiming_normal_(m.weight, mode='fan_out') + init.normal_(m.weight, std=0.01) + # init.xavier_normal_(m.weight) + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): # NN.Batchnorm2d + init.constant_(m.weight, 1) + init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + init.normal_(m.weight, std=0.01) + if m.bias is not None: + init.constant_(m.bias, 0) + + + +# ============================================================================================================== + + +class ResidualConv(nn.Module): + def __init__(self, inchannels): + super(ResidualConv, self).__init__() + # NN.BatchNorm2d + self.conv = nn.Sequential( + # nn.BatchNorm2d(num_features=inchannels), + nn.ReLU(inplace=False), + # nn.Conv2d(in_channels=inchannels, out_channels=inchannels, kernel_size=3, padding=1, stride=1, groups=inchannels,bias=True), + # nn.Conv2d(in_channels=inchannels, out_channels=inchannels, kernel_size=1, padding=0, stride=1, groups=1,bias=True) + nn.Conv2d(in_channels=inchannels, out_channels=inchannels / 2, kernel_size=3, padding=1, stride=1, + bias=False), + nn.BatchNorm2d(num_features=inchannels / 2), + nn.ReLU(inplace=False), + nn.Conv2d(in_channels=inchannels / 2, out_channels=inchannels, kernel_size=3, padding=1, stride=1, + bias=False) + ) + self.init_params() + + def forward(self, x): + x = self.conv(x) + x + return x + + def init_params(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + # init.kaiming_normal_(m.weight, mode='fan_out') + init.normal_(m.weight, std=0.01) + # init.xavier_normal_(m.weight) + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.ConvTranspose2d): + # init.kaiming_normal_(m.weight, mode='fan_out') + init.normal_(m.weight, std=0.01) + # init.xavier_normal_(m.weight) + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): # NN.BatchNorm2d + init.constant_(m.weight, 1) + init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + init.normal_(m.weight, std=0.01) + if m.bias is not None: + init.constant_(m.bias, 0) + + +class FeatureFusion(nn.Module): + def __init__(self, inchannels, outchannels): + super(FeatureFusion, self).__init__() + self.conv = ResidualConv(inchannels=inchannels) + # NN.BatchNorm2d + self.up = nn.Sequential(ResidualConv(inchannels=inchannels), + nn.ConvTranspose2d(in_channels=inchannels, out_channels=outchannels, kernel_size=3, + stride=2, padding=1, output_padding=1), + nn.BatchNorm2d(num_features=outchannels), + nn.ReLU(inplace=True)) + + def forward(self, lowfeat, highfeat): + return self.up(highfeat + self.conv(lowfeat)) + + def init_params(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + # init.kaiming_normal_(m.weight, mode='fan_out') + init.normal_(m.weight, std=0.01) + # init.xavier_normal_(m.weight) + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.ConvTranspose2d): + # init.kaiming_normal_(m.weight, mode='fan_out') + init.normal_(m.weight, std=0.01) + # init.xavier_normal_(m.weight) + if m.bias is not None: + init.constant_(m.bias, 0) + elif isinstance(m, nn.BatchNorm2d): # NN.BatchNorm2d + init.constant_(m.weight, 1) + init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + init.normal_(m.weight, std=0.01) + if m.bias is not None: + init.constant_(m.bias, 0) + + +class SenceUnderstand(nn.Module): + def __init__(self, channels): + super(SenceUnderstand, self).__init__() + self.channels = channels + self.conv1 = nn.Sequential(nn.Conv2d(in_channels=512, out_channels=512, kernel_size=3, padding=1), + nn.ReLU(inplace=True)) + self.pool = nn.AdaptiveAvgPool2d(8) + self.fc = nn.Sequential(nn.Linear(512 * 8 * 8, self.channels), + nn.ReLU(inplace=True)) + self.conv2 = nn.Sequential( + nn.Conv2d(in_channels=self.channels, out_channels=self.channels, kernel_size=1, padding=0), + nn.ReLU(inplace=True)) + self.initial_params() + + def forward(self, x): + n, c, h, w = x.size() + x = self.conv1(x) + x = self.pool(x) + x = x.view(n, -1) + x = self.fc(x) + x = x.view(n, self.channels, 1, 1) + x = self.conv2(x) + x = x.repeat(1, 1, h, w) + return x + + def initial_params(self, dev=0.01): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + # print torch.sum(m.weight) + m.weight.data.normal_(0, dev) + if m.bias is not None: + m.bias.data.fill_(0) + elif isinstance(m, nn.ConvTranspose2d): + # print torch.sum(m.weight) + m.weight.data.normal_(0, dev) + if m.bias is not None: + m.bias.data.fill_(0) + elif isinstance(m, nn.Linear): + m.weight.data.normal_(0, dev) + + +if __name__ == '__main__': + net = DepthNet(depth=50, pretrained=True) + print(net) + inputs = torch.ones(4,3,128,128) + out = net(inputs) + print(out.size()) + diff --git a/annotator/leres/pix2pix/LICENSE b/annotator/leres/pix2pix/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..38b1a24fd389a138b930dcf1ee606ef97a0186c8 --- /dev/null +++ b/annotator/leres/pix2pix/LICENSE @@ -0,0 +1,19 @@ +https://github.com/compphoto/BoostingMonocularDepth + +Copyright 2021, Seyed Mahdi Hosseini Miangoleh, Sebastian Dille, Computational Photography Laboratory. All rights reserved. + +This software is for academic use only. A redistribution of this +software, with or without modifications, has to be for academic +use only, while giving the appropriate credit to the original +authors of the software. The methods implemented as a part of +this software may be covered under patents or patent applications. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ''AS IS'' AND ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file diff --git a/annotator/leres/pix2pix/models/__init__.py b/annotator/leres/pix2pix/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f96e5c7f032f2154c6bb433b68fc968d0a19b5a8 --- /dev/null +++ b/annotator/leres/pix2pix/models/__init__.py @@ -0,0 +1,67 @@ +"""This package contains modules related to objective functions, optimizations, and network architectures. + +To add a custom model class called 'dummy', you need to add a file called 'dummy_model.py' and define a subclass DummyModel inherited from BaseModel. +You need to implement the following five functions: + -- <__init__>: initialize the class; first call BaseModel.__init__(self, opt). + -- : unpack data from dataset and apply preprocessing. + -- : produce intermediate results. + -- : calculate loss, gradients, and update network weights. + -- : (optionally) add model-specific options and set default options. + +In the function <__init__>, you need to define four lists: + -- self.loss_names (str list): specify the training losses that you want to plot and save. + -- self.model_names (str list): define networks used in our training. + -- self.visual_names (str list): specify the images that you want to display and save. + -- self.optimizers (optimizer list): define and initialize optimizers. You can define one optimizer for each network. If two networks are updated at the same time, you can use itertools.chain to group them. See cycle_gan_model.py for an usage. + +Now you can use the model class by specifying flag '--model dummy'. +See our template model class 'template_model.py' for more details. +""" + +import importlib +from .base_model import BaseModel + + +def find_model_using_name(model_name): + """Import the module "models/[model_name]_model.py". + + In the file, the class called DatasetNameModel() will + be instantiated. It has to be a subclass of BaseModel, + and it is case-insensitive. + """ + model_filename = "annotator.leres.pix2pix.models." + model_name + "_model" + modellib = importlib.import_module(model_filename) + model = None + target_model_name = model_name.replace('_', '') + 'model' + for name, cls in modellib.__dict__.items(): + if name.lower() == target_model_name.lower() \ + and issubclass(cls, BaseModel): + model = cls + + if model is None: + print("In %s.py, there should be a subclass of BaseModel with class name that matches %s in lowercase." % (model_filename, target_model_name)) + exit(0) + + return model + + +def get_option_setter(model_name): + """Return the static method of the model class.""" + model_class = find_model_using_name(model_name) + return model_class.modify_commandline_options + + +def create_model(opt): + """Create a model given the option. + + This function warps the class CustomDatasetDataLoader. + This is the main interface between this package and 'train.py'/'test.py' + + Example: + >>> from models import create_model + >>> model = create_model(opt) + """ + model = find_model_using_name(opt.model) + instance = model(opt) + print("model [%s] was created" % type(instance).__name__) + return instance diff --git a/annotator/leres/pix2pix/models/base_model.py b/annotator/leres/pix2pix/models/base_model.py new file mode 100644 index 0000000000000000000000000000000000000000..a90c5f832404bc44ef247b42a72988a37fc834cb --- /dev/null +++ b/annotator/leres/pix2pix/models/base_model.py @@ -0,0 +1,241 @@ +import os +import torch, gc +from modules import devices +from collections import OrderedDict +from abc import ABC, abstractmethod +from . import networks + + +class BaseModel(ABC): + """This class is an abstract base class (ABC) for models. + To create a subclass, you need to implement the following five functions: + -- <__init__>: initialize the class; first call BaseModel.__init__(self, opt). + -- : unpack data from dataset and apply preprocessing. + -- : produce intermediate results. + -- : calculate losses, gradients, and update network weights. + -- : (optionally) add model-specific options and set default options. + """ + + def __init__(self, opt): + """Initialize the BaseModel class. + + Parameters: + opt (Option class)-- stores all the experiment flags; needs to be a subclass of BaseOptions + + When creating your custom class, you need to implement your own initialization. + In this function, you should first call + Then, you need to define four lists: + -- self.loss_names (str list): specify the training losses that you want to plot and save. + -- self.model_names (str list): define networks used in our training. + -- self.visual_names (str list): specify the images that you want to display and save. + -- self.optimizers (optimizer list): define and initialize optimizers. You can define one optimizer for each network. If two networks are updated at the same time, you can use itertools.chain to group them. See cycle_gan_model.py for an example. + """ + self.opt = opt + self.gpu_ids = opt.gpu_ids + self.isTrain = opt.isTrain + self.device = torch.device('cuda:{}'.format(self.gpu_ids[0])) if self.gpu_ids else torch.device('cpu') # get device name: CPU or GPU + self.save_dir = os.path.join(opt.checkpoints_dir, opt.name) # save all the checkpoints to save_dir + if opt.preprocess != 'scale_width': # with [scale_width], input images might have different sizes, which hurts the performance of cudnn.benchmark. + torch.backends.cudnn.benchmark = True + self.loss_names = [] + self.model_names = [] + self.visual_names = [] + self.optimizers = [] + self.image_paths = [] + self.metric = 0 # used for learning rate policy 'plateau' + + @staticmethod + def modify_commandline_options(parser, is_train): + """Add new model-specific options, and rewrite default values for existing options. + + Parameters: + parser -- original option parser + is_train (bool) -- whether training phase or test phase. You can use this flag to add training-specific or test-specific options. + + Returns: + the modified parser. + """ + return parser + + @abstractmethod + def set_input(self, input): + """Unpack input data from the dataloader and perform necessary pre-processing steps. + + Parameters: + input (dict): includes the data itself and its metadata information. + """ + pass + + @abstractmethod + def forward(self): + """Run forward pass; called by both functions and .""" + pass + + @abstractmethod + def optimize_parameters(self): + """Calculate losses, gradients, and update network weights; called in every training iteration""" + pass + + def setup(self, opt): + """Load and print networks; create schedulers + + Parameters: + opt (Option class) -- stores all the experiment flags; needs to be a subclass of BaseOptions + """ + if self.isTrain: + self.schedulers = [networks.get_scheduler(optimizer, opt) for optimizer in self.optimizers] + if not self.isTrain or opt.continue_train: + load_suffix = 'iter_%d' % opt.load_iter if opt.load_iter > 0 else opt.epoch + self.load_networks(load_suffix) + self.print_networks(opt.verbose) + + def eval(self): + """Make models eval mode during test time""" + for name in self.model_names: + if isinstance(name, str): + net = getattr(self, 'net' + name) + net.eval() + + def test(self): + """Forward function used in test time. + + This function wraps function in no_grad() so we don't save intermediate steps for backprop + It also calls to produce additional visualization results + """ + with torch.no_grad(): + self.forward() + self.compute_visuals() + + def compute_visuals(self): + """Calculate additional output images for visdom and HTML visualization""" + pass + + def get_image_paths(self): + """ Return image paths that are used to load current data""" + return self.image_paths + + def update_learning_rate(self): + """Update learning rates for all the networks; called at the end of every epoch""" + old_lr = self.optimizers[0].param_groups[0]['lr'] + for scheduler in self.schedulers: + if self.opt.lr_policy == 'plateau': + scheduler.step(self.metric) + else: + scheduler.step() + + lr = self.optimizers[0].param_groups[0]['lr'] + print('learning rate %.7f -> %.7f' % (old_lr, lr)) + + def get_current_visuals(self): + """Return visualization images. train.py will display these images with visdom, and save the images to a HTML""" + visual_ret = OrderedDict() + for name in self.visual_names: + if isinstance(name, str): + visual_ret[name] = getattr(self, name) + return visual_ret + + def get_current_losses(self): + """Return traning losses / errors. train.py will print out these errors on console, and save them to a file""" + errors_ret = OrderedDict() + for name in self.loss_names: + if isinstance(name, str): + errors_ret[name] = float(getattr(self, 'loss_' + name)) # float(...) works for both scalar tensor and float number + return errors_ret + + def save_networks(self, epoch): + """Save all the networks to the disk. + + Parameters: + epoch (int) -- current epoch; used in the file name '%s_net_%s.pth' % (epoch, name) + """ + for name in self.model_names: + if isinstance(name, str): + save_filename = '%s_net_%s.pth' % (epoch, name) + save_path = os.path.join(self.save_dir, save_filename) + net = getattr(self, 'net' + name) + + if len(self.gpu_ids) > 0 and torch.cuda.is_available(): + torch.save(net.module.cpu().state_dict(), save_path) + net.cuda(self.gpu_ids[0]) + else: + torch.save(net.cpu().state_dict(), save_path) + + def unload_network(self, name): + """Unload network and gc. + """ + if isinstance(name, str): + net = getattr(self, 'net' + name) + del net + gc.collect() + devices.torch_gc() + return None + + def __patch_instance_norm_state_dict(self, state_dict, module, keys, i=0): + """Fix InstanceNorm checkpoints incompatibility (prior to 0.4)""" + key = keys[i] + if i + 1 == len(keys): # at the end, pointing to a parameter/buffer + if module.__class__.__name__.startswith('InstanceNorm') and \ + (key == 'running_mean' or key == 'running_var'): + if getattr(module, key) is None: + state_dict.pop('.'.join(keys)) + if module.__class__.__name__.startswith('InstanceNorm') and \ + (key == 'num_batches_tracked'): + state_dict.pop('.'.join(keys)) + else: + self.__patch_instance_norm_state_dict(state_dict, getattr(module, key), keys, i + 1) + + def load_networks(self, epoch): + """Load all the networks from the disk. + + Parameters: + epoch (int) -- current epoch; used in the file name '%s_net_%s.pth' % (epoch, name) + """ + for name in self.model_names: + if isinstance(name, str): + load_filename = '%s_net_%s.pth' % (epoch, name) + load_path = os.path.join(self.save_dir, load_filename) + net = getattr(self, 'net' + name) + if isinstance(net, torch.nn.DataParallel): + net = net.module + # print('Loading depth boost model from %s' % load_path) + # if you are using PyTorch newer than 0.4 (e.g., built from + # GitHub source), you can remove str() on self.device + state_dict = torch.load(load_path, map_location=str(self.device)) + if hasattr(state_dict, '_metadata'): + del state_dict._metadata + + # patch InstanceNorm checkpoints prior to 0.4 + for key in list(state_dict.keys()): # need to copy keys here because we mutate in loop + self.__patch_instance_norm_state_dict(state_dict, net, key.split('.')) + net.load_state_dict(state_dict) + + def print_networks(self, verbose): + """Print the total number of parameters in the network and (if verbose) network architecture + + Parameters: + verbose (bool) -- if verbose: print the network architecture + """ + print('---------- Networks initialized -------------') + for name in self.model_names: + if isinstance(name, str): + net = getattr(self, 'net' + name) + num_params = 0 + for param in net.parameters(): + num_params += param.numel() + if verbose: + print(net) + print('[Network %s] Total number of parameters : %.3f M' % (name, num_params / 1e6)) + print('-----------------------------------------------') + + def set_requires_grad(self, nets, requires_grad=False): + """Set requies_grad=Fasle for all the networks to avoid unnecessary computations + Parameters: + nets (network list) -- a list of networks + requires_grad (bool) -- whether the networks require gradients or not + """ + if not isinstance(nets, list): + nets = [nets] + for net in nets: + if net is not None: + for param in net.parameters(): + param.requires_grad = requires_grad diff --git a/annotator/leres/pix2pix/models/base_model_hg.py b/annotator/leres/pix2pix/models/base_model_hg.py new file mode 100644 index 0000000000000000000000000000000000000000..1709accdf0b048b3793dfd1f58d1b06c35f7b907 --- /dev/null +++ b/annotator/leres/pix2pix/models/base_model_hg.py @@ -0,0 +1,58 @@ +import os +import torch + +class BaseModelHG(): + def name(self): + return 'BaseModel' + + def initialize(self, opt): + self.opt = opt + self.gpu_ids = opt.gpu_ids + self.isTrain = opt.isTrain + self.Tensor = torch.cuda.FloatTensor if self.gpu_ids else torch.Tensor + self.save_dir = os.path.join(opt.checkpoints_dir, opt.name) + + def set_input(self, input): + self.input = input + + def forward(self): + pass + + # used in test time, no backprop + def test(self): + pass + + def get_image_paths(self): + pass + + def optimize_parameters(self): + pass + + def get_current_visuals(self): + return self.input + + def get_current_errors(self): + return {} + + def save(self, label): + pass + + # helper saving function that can be used by subclasses + def save_network(self, network, network_label, epoch_label, gpu_ids): + save_filename = '_%s_net_%s.pth' % (epoch_label, network_label) + save_path = os.path.join(self.save_dir, save_filename) + torch.save(network.cpu().state_dict(), save_path) + if len(gpu_ids) and torch.cuda.is_available(): + network.cuda(device_id=gpu_ids[0]) + + # helper loading function that can be used by subclasses + def load_network(self, network, network_label, epoch_label): + save_filename = '%s_net_%s.pth' % (epoch_label, network_label) + save_path = os.path.join(self.save_dir, save_filename) + print(save_path) + model = torch.load(save_path) + return model + # network.load_state_dict(torch.load(save_path)) + + def update_learning_rate(): + pass diff --git a/annotator/leres/pix2pix/models/networks.py b/annotator/leres/pix2pix/models/networks.py new file mode 100644 index 0000000000000000000000000000000000000000..0cf912b2973721a02deefd042af621e732bad59f --- /dev/null +++ b/annotator/leres/pix2pix/models/networks.py @@ -0,0 +1,623 @@ +import torch +import torch.nn as nn +from torch.nn import init +import functools +from torch.optim import lr_scheduler + + +############################################################################### +# Helper Functions +############################################################################### + + +class Identity(nn.Module): + def forward(self, x): + return x + + +def get_norm_layer(norm_type='instance'): + """Return a normalization layer + + Parameters: + norm_type (str) -- the name of the normalization layer: batch | instance | none + + For BatchNorm, we use learnable affine parameters and track running statistics (mean/stddev). + For InstanceNorm, we do not use learnable affine parameters. We do not track running statistics. + """ + if norm_type == 'batch': + norm_layer = functools.partial(nn.BatchNorm2d, affine=True, track_running_stats=True) + elif norm_type == 'instance': + norm_layer = functools.partial(nn.InstanceNorm2d, affine=False, track_running_stats=False) + elif norm_type == 'none': + def norm_layer(x): return Identity() + else: + raise NotImplementedError('normalization layer [%s] is not found' % norm_type) + return norm_layer + + +def get_scheduler(optimizer, opt): + """Return a learning rate scheduler + + Parameters: + optimizer -- the optimizer of the network + opt (option class) -- stores all the experiment flags; needs to be a subclass of BaseOptions.  + opt.lr_policy is the name of learning rate policy: linear | step | plateau | cosine + + For 'linear', we keep the same learning rate for the first epochs + and linearly decay the rate to zero over the next epochs. + For other schedulers (step, plateau, and cosine), we use the default PyTorch schedulers. + See https://pytorch.org/docs/stable/optim.html for more details. + """ + if opt.lr_policy == 'linear': + def lambda_rule(epoch): + lr_l = 1.0 - max(0, epoch + opt.epoch_count - opt.n_epochs) / float(opt.n_epochs_decay + 1) + return lr_l + scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda_rule) + elif opt.lr_policy == 'step': + scheduler = lr_scheduler.StepLR(optimizer, step_size=opt.lr_decay_iters, gamma=0.1) + elif opt.lr_policy == 'plateau': + scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', factor=0.2, threshold=0.01, patience=5) + elif opt.lr_policy == 'cosine': + scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=opt.n_epochs, eta_min=0) + else: + return NotImplementedError('learning rate policy [%s] is not implemented', opt.lr_policy) + return scheduler + + +def init_weights(net, init_type='normal', init_gain=0.02): + """Initialize network weights. + + Parameters: + net (network) -- network to be initialized + init_type (str) -- the name of an initialization method: normal | xavier | kaiming | orthogonal + init_gain (float) -- scaling factor for normal, xavier and orthogonal. + + We use 'normal' in the original pix2pix and CycleGAN paper. But xavier and kaiming might + work better for some applications. Feel free to try yourself. + """ + def init_func(m): # define the initialization function + classname = m.__class__.__name__ + if hasattr(m, 'weight') and (classname.find('Conv') != -1 or classname.find('Linear') != -1): + if init_type == 'normal': + init.normal_(m.weight.data, 0.0, init_gain) + elif init_type == 'xavier': + init.xavier_normal_(m.weight.data, gain=init_gain) + elif init_type == 'kaiming': + init.kaiming_normal_(m.weight.data, a=0, mode='fan_in') + elif init_type == 'orthogonal': + init.orthogonal_(m.weight.data, gain=init_gain) + else: + raise NotImplementedError('initialization method [%s] is not implemented' % init_type) + if hasattr(m, 'bias') and m.bias is not None: + init.constant_(m.bias.data, 0.0) + elif classname.find('BatchNorm2d') != -1: # BatchNorm Layer's weight is not a matrix; only normal distribution applies. + init.normal_(m.weight.data, 1.0, init_gain) + init.constant_(m.bias.data, 0.0) + + # print('initialize network with %s' % init_type) + net.apply(init_func) # apply the initialization function + + +def init_net(net, init_type='normal', init_gain=0.02, gpu_ids=[]): + """Initialize a network: 1. register CPU/GPU device (with multi-GPU support); 2. initialize the network weights + Parameters: + net (network) -- the network to be initialized + init_type (str) -- the name of an initialization method: normal | xavier | kaiming | orthogonal + gain (float) -- scaling factor for normal, xavier and orthogonal. + gpu_ids (int list) -- which GPUs the network runs on: e.g., 0,1,2 + + Return an initialized network. + """ + if len(gpu_ids) > 0: + assert(torch.cuda.is_available()) + net.to(gpu_ids[0]) + net = torch.nn.DataParallel(net, gpu_ids) # multi-GPUs + init_weights(net, init_type, init_gain=init_gain) + return net + + +def define_G(input_nc, output_nc, ngf, netG, norm='batch', use_dropout=False, init_type='normal', init_gain=0.02, gpu_ids=[]): + """Create a generator + + Parameters: + input_nc (int) -- the number of channels in input images + output_nc (int) -- the number of channels in output images + ngf (int) -- the number of filters in the last conv layer + netG (str) -- the architecture's name: resnet_9blocks | resnet_6blocks | unet_256 | unet_128 + norm (str) -- the name of normalization layers used in the network: batch | instance | none + use_dropout (bool) -- if use dropout layers. + init_type (str) -- the name of our initialization method. + init_gain (float) -- scaling factor for normal, xavier and orthogonal. + gpu_ids (int list) -- which GPUs the network runs on: e.g., 0,1,2 + + Returns a generator + + Our current implementation provides two types of generators: + U-Net: [unet_128] (for 128x128 input images) and [unet_256] (for 256x256 input images) + The original U-Net paper: https://arxiv.org/abs/1505.04597 + + Resnet-based generator: [resnet_6blocks] (with 6 Resnet blocks) and [resnet_9blocks] (with 9 Resnet blocks) + Resnet-based generator consists of several Resnet blocks between a few downsampling/upsampling operations. + We adapt Torch code from Justin Johnson's neural style transfer project (https://github.com/jcjohnson/fast-neural-style). + + + The generator has been initialized by . It uses RELU for non-linearity. + """ + net = None + norm_layer = get_norm_layer(norm_type=norm) + + if netG == 'resnet_9blocks': + net = ResnetGenerator(input_nc, output_nc, ngf, norm_layer=norm_layer, use_dropout=use_dropout, n_blocks=9) + elif netG == 'resnet_6blocks': + net = ResnetGenerator(input_nc, output_nc, ngf, norm_layer=norm_layer, use_dropout=use_dropout, n_blocks=6) + elif netG == 'resnet_12blocks': + net = ResnetGenerator(input_nc, output_nc, ngf, norm_layer=norm_layer, use_dropout=use_dropout, n_blocks=12) + elif netG == 'unet_128': + net = UnetGenerator(input_nc, output_nc, 7, ngf, norm_layer=norm_layer, use_dropout=use_dropout) + elif netG == 'unet_256': + net = UnetGenerator(input_nc, output_nc, 8, ngf, norm_layer=norm_layer, use_dropout=use_dropout) + elif netG == 'unet_672': + net = UnetGenerator(input_nc, output_nc, 5, ngf, norm_layer=norm_layer, use_dropout=use_dropout) + elif netG == 'unet_960': + net = UnetGenerator(input_nc, output_nc, 6, ngf, norm_layer=norm_layer, use_dropout=use_dropout) + elif netG == 'unet_1024': + net = UnetGenerator(input_nc, output_nc, 10, ngf, norm_layer=norm_layer, use_dropout=use_dropout) + else: + raise NotImplementedError('Generator model name [%s] is not recognized' % netG) + return init_net(net, init_type, init_gain, gpu_ids) + + +def define_D(input_nc, ndf, netD, n_layers_D=3, norm='batch', init_type='normal', init_gain=0.02, gpu_ids=[]): + """Create a discriminator + + Parameters: + input_nc (int) -- the number of channels in input images + ndf (int) -- the number of filters in the first conv layer + netD (str) -- the architecture's name: basic | n_layers | pixel + n_layers_D (int) -- the number of conv layers in the discriminator; effective when netD=='n_layers' + norm (str) -- the type of normalization layers used in the network. + init_type (str) -- the name of the initialization method. + init_gain (float) -- scaling factor for normal, xavier and orthogonal. + gpu_ids (int list) -- which GPUs the network runs on: e.g., 0,1,2 + + Returns a discriminator + + Our current implementation provides three types of discriminators: + [basic]: 'PatchGAN' classifier described in the original pix2pix paper. + It can classify whether 70×70 overlapping patches are real or fake. + Such a patch-level discriminator architecture has fewer parameters + than a full-image discriminator and can work on arbitrarily-sized images + in a fully convolutional fashion. + + [n_layers]: With this mode, you can specify the number of conv layers in the discriminator + with the parameter (default=3 as used in [basic] (PatchGAN).) + + [pixel]: 1x1 PixelGAN discriminator can classify whether a pixel is real or not. + It encourages greater color diversity but has no effect on spatial statistics. + + The discriminator has been initialized by . It uses Leakly RELU for non-linearity. + """ + net = None + norm_layer = get_norm_layer(norm_type=norm) + + if netD == 'basic': # default PatchGAN classifier + net = NLayerDiscriminator(input_nc, ndf, n_layers=3, norm_layer=norm_layer) + elif netD == 'n_layers': # more options + net = NLayerDiscriminator(input_nc, ndf, n_layers_D, norm_layer=norm_layer) + elif netD == 'pixel': # classify if each pixel is real or fake + net = PixelDiscriminator(input_nc, ndf, norm_layer=norm_layer) + else: + raise NotImplementedError('Discriminator model name [%s] is not recognized' % netD) + return init_net(net, init_type, init_gain, gpu_ids) + + +############################################################################## +# Classes +############################################################################## +class GANLoss(nn.Module): + """Define different GAN objectives. + + The GANLoss class abstracts away the need to create the target label tensor + that has the same size as the input. + """ + + def __init__(self, gan_mode, target_real_label=1.0, target_fake_label=0.0): + """ Initialize the GANLoss class. + + Parameters: + gan_mode (str) - - the type of GAN objective. It currently supports vanilla, lsgan, and wgangp. + target_real_label (bool) - - label for a real image + target_fake_label (bool) - - label of a fake image + + Note: Do not use sigmoid as the last layer of Discriminator. + LSGAN needs no sigmoid. vanilla GANs will handle it with BCEWithLogitsLoss. + """ + super(GANLoss, self).__init__() + self.register_buffer('real_label', torch.tensor(target_real_label)) + self.register_buffer('fake_label', torch.tensor(target_fake_label)) + self.gan_mode = gan_mode + if gan_mode == 'lsgan': + self.loss = nn.MSELoss() + elif gan_mode == 'vanilla': + self.loss = nn.BCEWithLogitsLoss() + elif gan_mode in ['wgangp']: + self.loss = None + else: + raise NotImplementedError('gan mode %s not implemented' % gan_mode) + + def get_target_tensor(self, prediction, target_is_real): + """Create label tensors with the same size as the input. + + Parameters: + prediction (tensor) - - tpyically the prediction from a discriminator + target_is_real (bool) - - if the ground truth label is for real images or fake images + + Returns: + A label tensor filled with ground truth label, and with the size of the input + """ + + if target_is_real: + target_tensor = self.real_label + else: + target_tensor = self.fake_label + return target_tensor.expand_as(prediction) + + def __call__(self, prediction, target_is_real): + """Calculate loss given Discriminator's output and grount truth labels. + + Parameters: + prediction (tensor) - - tpyically the prediction output from a discriminator + target_is_real (bool) - - if the ground truth label is for real images or fake images + + Returns: + the calculated loss. + """ + if self.gan_mode in ['lsgan', 'vanilla']: + target_tensor = self.get_target_tensor(prediction, target_is_real) + loss = self.loss(prediction, target_tensor) + elif self.gan_mode == 'wgangp': + if target_is_real: + loss = -prediction.mean() + else: + loss = prediction.mean() + return loss + + +def cal_gradient_penalty(netD, real_data, fake_data, device, type='mixed', constant=1.0, lambda_gp=10.0): + """Calculate the gradient penalty loss, used in WGAN-GP paper https://arxiv.org/abs/1704.00028 + + Arguments: + netD (network) -- discriminator network + real_data (tensor array) -- real images + fake_data (tensor array) -- generated images from the generator + device (str) -- GPU / CPU: from torch.device('cuda:{}'.format(self.gpu_ids[0])) if self.gpu_ids else torch.device('cpu') + type (str) -- if we mix real and fake data or not [real | fake | mixed]. + constant (float) -- the constant used in formula ( ||gradient||_2 - constant)^2 + lambda_gp (float) -- weight for this loss + + Returns the gradient penalty loss + """ + if lambda_gp > 0.0: + if type == 'real': # either use real images, fake images, or a linear interpolation of two. + interpolatesv = real_data + elif type == 'fake': + interpolatesv = fake_data + elif type == 'mixed': + alpha = torch.rand(real_data.shape[0], 1, device=device) + alpha = alpha.expand(real_data.shape[0], real_data.nelement() // real_data.shape[0]).contiguous().view(*real_data.shape) + interpolatesv = alpha * real_data + ((1 - alpha) * fake_data) + else: + raise NotImplementedError('{} not implemented'.format(type)) + interpolatesv.requires_grad_(True) + disc_interpolates = netD(interpolatesv) + gradients = torch.autograd.grad(outputs=disc_interpolates, inputs=interpolatesv, + grad_outputs=torch.ones(disc_interpolates.size()).to(device), + create_graph=True, retain_graph=True, only_inputs=True) + gradients = gradients[0].view(real_data.size(0), -1) # flat the data + gradient_penalty = (((gradients + 1e-16).norm(2, dim=1) - constant) ** 2).mean() * lambda_gp # added eps + return gradient_penalty, gradients + else: + return 0.0, None + + +class ResnetGenerator(nn.Module): + """Resnet-based generator that consists of Resnet blocks between a few downsampling/upsampling operations. + + We adapt Torch code and idea from Justin Johnson's neural style transfer project(https://github.com/jcjohnson/fast-neural-style) + """ + + def __init__(self, input_nc, output_nc, ngf=64, norm_layer=nn.BatchNorm2d, use_dropout=False, n_blocks=6, padding_type='reflect'): + """Construct a Resnet-based generator + + Parameters: + input_nc (int) -- the number of channels in input images + output_nc (int) -- the number of channels in output images + ngf (int) -- the number of filters in the last conv layer + norm_layer -- normalization layer + use_dropout (bool) -- if use dropout layers + n_blocks (int) -- the number of ResNet blocks + padding_type (str) -- the name of padding layer in conv layers: reflect | replicate | zero + """ + assert(n_blocks >= 0) + super(ResnetGenerator, self).__init__() + if type(norm_layer) == functools.partial: + use_bias = norm_layer.func == nn.InstanceNorm2d + else: + use_bias = norm_layer == nn.InstanceNorm2d + + model = [nn.ReflectionPad2d(3), + nn.Conv2d(input_nc, ngf, kernel_size=7, padding=0, bias=use_bias), + norm_layer(ngf), + nn.ReLU(True)] + + n_downsampling = 2 + for i in range(n_downsampling): # add downsampling layers + mult = 2 ** i + model += [nn.Conv2d(ngf * mult, ngf * mult * 2, kernel_size=3, stride=2, padding=1, bias=use_bias), + norm_layer(ngf * mult * 2), + nn.ReLU(True)] + + mult = 2 ** n_downsampling + for i in range(n_blocks): # add ResNet blocks + + model += [ResnetBlock(ngf * mult, padding_type=padding_type, norm_layer=norm_layer, use_dropout=use_dropout, use_bias=use_bias)] + + for i in range(n_downsampling): # add upsampling layers + mult = 2 ** (n_downsampling - i) + model += [nn.ConvTranspose2d(ngf * mult, int(ngf * mult / 2), + kernel_size=3, stride=2, + padding=1, output_padding=1, + bias=use_bias), + norm_layer(int(ngf * mult / 2)), + nn.ReLU(True)] + model += [nn.ReflectionPad2d(3)] + model += [nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0)] + model += [nn.Tanh()] + + self.model = nn.Sequential(*model) + + def forward(self, input): + """Standard forward""" + return self.model(input) + + +class ResnetBlock(nn.Module): + """Define a Resnet block""" + + def __init__(self, dim, padding_type, norm_layer, use_dropout, use_bias): + """Initialize the Resnet block + + A resnet block is a conv block with skip connections + We construct a conv block with build_conv_block function, + and implement skip connections in function. + Original Resnet paper: https://arxiv.org/pdf/1512.03385.pdf + """ + super(ResnetBlock, self).__init__() + self.conv_block = self.build_conv_block(dim, padding_type, norm_layer, use_dropout, use_bias) + + def build_conv_block(self, dim, padding_type, norm_layer, use_dropout, use_bias): + """Construct a convolutional block. + + Parameters: + dim (int) -- the number of channels in the conv layer. + padding_type (str) -- the name of padding layer: reflect | replicate | zero + norm_layer -- normalization layer + use_dropout (bool) -- if use dropout layers. + use_bias (bool) -- if the conv layer uses bias or not + + Returns a conv block (with a conv layer, a normalization layer, and a non-linearity layer (ReLU)) + """ + conv_block = [] + p = 0 + if padding_type == 'reflect': + conv_block += [nn.ReflectionPad2d(1)] + elif padding_type == 'replicate': + conv_block += [nn.ReplicationPad2d(1)] + elif padding_type == 'zero': + p = 1 + else: + raise NotImplementedError('padding [%s] is not implemented' % padding_type) + + conv_block += [nn.Conv2d(dim, dim, kernel_size=3, padding=p, bias=use_bias), norm_layer(dim), nn.ReLU(True)] + if use_dropout: + conv_block += [nn.Dropout(0.5)] + + p = 0 + if padding_type == 'reflect': + conv_block += [nn.ReflectionPad2d(1)] + elif padding_type == 'replicate': + conv_block += [nn.ReplicationPad2d(1)] + elif padding_type == 'zero': + p = 1 + else: + raise NotImplementedError('padding [%s] is not implemented' % padding_type) + conv_block += [nn.Conv2d(dim, dim, kernel_size=3, padding=p, bias=use_bias), norm_layer(dim)] + + return nn.Sequential(*conv_block) + + def forward(self, x): + """Forward function (with skip connections)""" + out = x + self.conv_block(x) # add skip connections + return out + + +class UnetGenerator(nn.Module): + """Create a Unet-based generator""" + + def __init__(self, input_nc, output_nc, num_downs, ngf=64, norm_layer=nn.BatchNorm2d, use_dropout=False): + """Construct a Unet generator + Parameters: + input_nc (int) -- the number of channels in input images + output_nc (int) -- the number of channels in output images + num_downs (int) -- the number of downsamplings in UNet. For example, # if |num_downs| == 7, + image of size 128x128 will become of size 1x1 # at the bottleneck + ngf (int) -- the number of filters in the last conv layer + norm_layer -- normalization layer + + We construct the U-Net from the innermost layer to the outermost layer. + It is a recursive process. + """ + super(UnetGenerator, self).__init__() + # construct unet structure + unet_block = UnetSkipConnectionBlock(ngf * 8, ngf * 8, input_nc=None, submodule=None, norm_layer=norm_layer, innermost=True) # add the innermost layer + for i in range(num_downs - 5): # add intermediate layers with ngf * 8 filters + unet_block = UnetSkipConnectionBlock(ngf * 8, ngf * 8, input_nc=None, submodule=unet_block, norm_layer=norm_layer, use_dropout=use_dropout) + # gradually reduce the number of filters from ngf * 8 to ngf + unet_block = UnetSkipConnectionBlock(ngf * 4, ngf * 8, input_nc=None, submodule=unet_block, norm_layer=norm_layer) + unet_block = UnetSkipConnectionBlock(ngf * 2, ngf * 4, input_nc=None, submodule=unet_block, norm_layer=norm_layer) + unet_block = UnetSkipConnectionBlock(ngf, ngf * 2, input_nc=None, submodule=unet_block, norm_layer=norm_layer) + self.model = UnetSkipConnectionBlock(output_nc, ngf, input_nc=input_nc, submodule=unet_block, outermost=True, norm_layer=norm_layer) # add the outermost layer + + def forward(self, input): + """Standard forward""" + return self.model(input) + + +class UnetSkipConnectionBlock(nn.Module): + """Defines the Unet submodule with skip connection. + X -------------------identity---------------------- + |-- downsampling -- |submodule| -- upsampling --| + """ + + def __init__(self, outer_nc, inner_nc, input_nc=None, + submodule=None, outermost=False, innermost=False, norm_layer=nn.BatchNorm2d, use_dropout=False): + """Construct a Unet submodule with skip connections. + + Parameters: + outer_nc (int) -- the number of filters in the outer conv layer + inner_nc (int) -- the number of filters in the inner conv layer + input_nc (int) -- the number of channels in input images/features + submodule (UnetSkipConnectionBlock) -- previously defined submodules + outermost (bool) -- if this module is the outermost module + innermost (bool) -- if this module is the innermost module + norm_layer -- normalization layer + use_dropout (bool) -- if use dropout layers. + """ + super(UnetSkipConnectionBlock, self).__init__() + self.outermost = outermost + if type(norm_layer) == functools.partial: + use_bias = norm_layer.func == nn.InstanceNorm2d + else: + use_bias = norm_layer == nn.InstanceNorm2d + if input_nc is None: + input_nc = outer_nc + downconv = nn.Conv2d(input_nc, inner_nc, kernel_size=4, + stride=2, padding=1, bias=use_bias) + downrelu = nn.LeakyReLU(0.2, True) + downnorm = norm_layer(inner_nc) + uprelu = nn.ReLU(True) + upnorm = norm_layer(outer_nc) + + if outermost: + upconv = nn.ConvTranspose2d(inner_nc * 2, outer_nc, + kernel_size=4, stride=2, + padding=1) + down = [downconv] + up = [uprelu, upconv, nn.Tanh()] + model = down + [submodule] + up + elif innermost: + upconv = nn.ConvTranspose2d(inner_nc, outer_nc, + kernel_size=4, stride=2, + padding=1, bias=use_bias) + down = [downrelu, downconv] + up = [uprelu, upconv, upnorm] + model = down + up + else: + upconv = nn.ConvTranspose2d(inner_nc * 2, outer_nc, + kernel_size=4, stride=2, + padding=1, bias=use_bias) + down = [downrelu, downconv, downnorm] + up = [uprelu, upconv, upnorm] + + if use_dropout: + model = down + [submodule] + up + [nn.Dropout(0.5)] + else: + model = down + [submodule] + up + + self.model = nn.Sequential(*model) + + def forward(self, x): + if self.outermost: + return self.model(x) + else: # add skip connections + return torch.cat([x, self.model(x)], 1) + + +class NLayerDiscriminator(nn.Module): + """Defines a PatchGAN discriminator""" + + def __init__(self, input_nc, ndf=64, n_layers=3, norm_layer=nn.BatchNorm2d): + """Construct a PatchGAN discriminator + + Parameters: + input_nc (int) -- the number of channels in input images + ndf (int) -- the number of filters in the last conv layer + n_layers (int) -- the number of conv layers in the discriminator + norm_layer -- normalization layer + """ + super(NLayerDiscriminator, self).__init__() + if type(norm_layer) == functools.partial: # no need to use bias as BatchNorm2d has affine parameters + use_bias = norm_layer.func == nn.InstanceNorm2d + else: + use_bias = norm_layer == nn.InstanceNorm2d + + kw = 4 + padw = 1 + sequence = [nn.Conv2d(input_nc, ndf, kernel_size=kw, stride=2, padding=padw), nn.LeakyReLU(0.2, True)] + nf_mult = 1 + nf_mult_prev = 1 + for n in range(1, n_layers): # gradually increase the number of filters + nf_mult_prev = nf_mult + nf_mult = min(2 ** n, 8) + sequence += [ + nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=kw, stride=2, padding=padw, bias=use_bias), + norm_layer(ndf * nf_mult), + nn.LeakyReLU(0.2, True) + ] + + nf_mult_prev = nf_mult + nf_mult = min(2 ** n_layers, 8) + sequence += [ + nn.Conv2d(ndf * nf_mult_prev, ndf * nf_mult, kernel_size=kw, stride=1, padding=padw, bias=use_bias), + norm_layer(ndf * nf_mult), + nn.LeakyReLU(0.2, True) + ] + + sequence += [nn.Conv2d(ndf * nf_mult, 1, kernel_size=kw, stride=1, padding=padw)] # output 1 channel prediction map + self.model = nn.Sequential(*sequence) + + def forward(self, input): + """Standard forward.""" + return self.model(input) + + +class PixelDiscriminator(nn.Module): + """Defines a 1x1 PatchGAN discriminator (pixelGAN)""" + + def __init__(self, input_nc, ndf=64, norm_layer=nn.BatchNorm2d): + """Construct a 1x1 PatchGAN discriminator + + Parameters: + input_nc (int) -- the number of channels in input images + ndf (int) -- the number of filters in the last conv layer + norm_layer -- normalization layer + """ + super(PixelDiscriminator, self).__init__() + if type(norm_layer) == functools.partial: # no need to use bias as BatchNorm2d has affine parameters + use_bias = norm_layer.func == nn.InstanceNorm2d + else: + use_bias = norm_layer == nn.InstanceNorm2d + + self.net = [ + nn.Conv2d(input_nc, ndf, kernel_size=1, stride=1, padding=0), + nn.LeakyReLU(0.2, True), + nn.Conv2d(ndf, ndf * 2, kernel_size=1, stride=1, padding=0, bias=use_bias), + norm_layer(ndf * 2), + nn.LeakyReLU(0.2, True), + nn.Conv2d(ndf * 2, 1, kernel_size=1, stride=1, padding=0, bias=use_bias)] + + self.net = nn.Sequential(*self.net) + + def forward(self, input): + """Standard forward.""" + return self.net(input) diff --git a/annotator/leres/pix2pix/models/pix2pix4depth_model.py b/annotator/leres/pix2pix/models/pix2pix4depth_model.py new file mode 100644 index 0000000000000000000000000000000000000000..89e89652feb96314973a050c5a2477b474630abb --- /dev/null +++ b/annotator/leres/pix2pix/models/pix2pix4depth_model.py @@ -0,0 +1,155 @@ +import torch +from .base_model import BaseModel +from . import networks + + +class Pix2Pix4DepthModel(BaseModel): + """ This class implements the pix2pix model, for learning a mapping from input images to output images given paired data. + + The model training requires '--dataset_mode aligned' dataset. + By default, it uses a '--netG unet256' U-Net generator, + a '--netD basic' discriminator (PatchGAN), + and a '--gan_mode' vanilla GAN loss (the cross-entropy objective used in the orignal GAN paper). + + pix2pix paper: https://arxiv.org/pdf/1611.07004.pdf + """ + @staticmethod + def modify_commandline_options(parser, is_train=True): + """Add new dataset-specific options, and rewrite default values for existing options. + + Parameters: + parser -- original option parser + is_train (bool) -- whether training phase or test phase. You can use this flag to add training-specific or test-specific options. + + Returns: + the modified parser. + + For pix2pix, we do not use image buffer + The training objective is: GAN Loss + lambda_L1 * ||G(A)-B||_1 + By default, we use vanilla GAN loss, UNet with batchnorm, and aligned datasets. + """ + # changing the default values to match the pix2pix paper (https://phillipi.github.io/pix2pix/) + parser.set_defaults(input_nc=2,output_nc=1,norm='none', netG='unet_1024', dataset_mode='depthmerge') + if is_train: + parser.set_defaults(pool_size=0, gan_mode='vanilla',) + parser.add_argument('--lambda_L1', type=float, default=1000, help='weight for L1 loss') + return parser + + def __init__(self, opt): + """Initialize the pix2pix class. + + Parameters: + opt (Option class)-- stores all the experiment flags; needs to be a subclass of BaseOptions + """ + BaseModel.__init__(self, opt) + # specify the training losses you want to print out. The training/test scripts will call + + self.loss_names = ['G_GAN', 'G_L1', 'D_real', 'D_fake'] + # self.loss_names = ['G_L1'] + + # specify the images you want to save/display. The training/test scripts will call + if self.isTrain: + self.visual_names = ['outer','inner', 'fake_B', 'real_B'] + else: + self.visual_names = ['fake_B'] + + # specify the models you want to save to the disk. The training/test scripts will call and + if self.isTrain: + self.model_names = ['G','D'] + else: # during test time, only load G + self.model_names = ['G'] + + # define networks (both generator and discriminator) + self.netG = networks.define_G(opt.input_nc, opt.output_nc, 64, 'unet_1024', 'none', + False, 'normal', 0.02, self.gpu_ids) + + if self.isTrain: # define a discriminator; conditional GANs need to take both input and output images; Therefore, #channels for D is input_nc + output_nc + self.netD = networks.define_D(opt.input_nc + opt.output_nc, opt.ndf, opt.netD, + opt.n_layers_D, opt.norm, opt.init_type, opt.init_gain, self.gpu_ids) + + if self.isTrain: + # define loss functions + self.criterionGAN = networks.GANLoss(opt.gan_mode).to(self.device) + self.criterionL1 = torch.nn.L1Loss() + # initialize optimizers; schedulers will be automatically created by function . + self.optimizer_G = torch.optim.Adam(self.netG.parameters(), lr=1e-4, betas=(opt.beta1, 0.999)) + self.optimizer_D = torch.optim.Adam(self.netD.parameters(), lr=2e-06, betas=(opt.beta1, 0.999)) + self.optimizers.append(self.optimizer_G) + self.optimizers.append(self.optimizer_D) + + def set_input_train(self, input): + self.outer = input['data_outer'].to(self.device) + self.outer = torch.nn.functional.interpolate(self.outer,(1024,1024),mode='bilinear',align_corners=False) + + self.inner = input['data_inner'].to(self.device) + self.inner = torch.nn.functional.interpolate(self.inner,(1024,1024),mode='bilinear',align_corners=False) + + self.image_paths = input['image_path'] + + if self.isTrain: + self.gtfake = input['data_gtfake'].to(self.device) + self.gtfake = torch.nn.functional.interpolate(self.gtfake, (1024, 1024), mode='bilinear', align_corners=False) + self.real_B = self.gtfake + + self.real_A = torch.cat((self.outer, self.inner), 1) + + def set_input(self, outer, inner): + inner = torch.from_numpy(inner).unsqueeze(0).unsqueeze(0) + outer = torch.from_numpy(outer).unsqueeze(0).unsqueeze(0) + + inner = (inner - torch.min(inner))/(torch.max(inner)-torch.min(inner)) + outer = (outer - torch.min(outer))/(torch.max(outer)-torch.min(outer)) + + inner = self.normalize(inner) + outer = self.normalize(outer) + + self.real_A = torch.cat((outer, inner), 1).to(self.device) + + + def normalize(self, input): + input = input * 2 + input = input - 1 + return input + + def forward(self): + """Run forward pass; called by both functions and .""" + self.fake_B = self.netG(self.real_A) # G(A) + + def backward_D(self): + """Calculate GAN loss for the discriminator""" + # Fake; stop backprop to the generator by detaching fake_B + fake_AB = torch.cat((self.real_A, self.fake_B), 1) # we use conditional GANs; we need to feed both input and output to the discriminator + pred_fake = self.netD(fake_AB.detach()) + self.loss_D_fake = self.criterionGAN(pred_fake, False) + # Real + real_AB = torch.cat((self.real_A, self.real_B), 1) + pred_real = self.netD(real_AB) + self.loss_D_real = self.criterionGAN(pred_real, True) + # combine loss and calculate gradients + self.loss_D = (self.loss_D_fake + self.loss_D_real) * 0.5 + self.loss_D.backward() + + def backward_G(self): + """Calculate GAN and L1 loss for the generator""" + # First, G(A) should fake the discriminator + fake_AB = torch.cat((self.real_A, self.fake_B), 1) + pred_fake = self.netD(fake_AB) + self.loss_G_GAN = self.criterionGAN(pred_fake, True) + # Second, G(A) = B + self.loss_G_L1 = self.criterionL1(self.fake_B, self.real_B) * self.opt.lambda_L1 + # combine loss and calculate gradients + self.loss_G = self.loss_G_L1 + self.loss_G_GAN + self.loss_G.backward() + + def optimize_parameters(self): + self.forward() # compute fake images: G(A) + # update D + self.set_requires_grad(self.netD, True) # enable backprop for D + self.optimizer_D.zero_grad() # set D's gradients to zero + self.backward_D() # calculate gradients for D + self.optimizer_D.step() # update D's weights + # update G + self.set_requires_grad(self.netD, False) # D requires no gradients when optimizing G + self.optimizer_G.zero_grad() # set G's gradients to zero + self.backward_G() # calculate graidents for G + self.optimizer_G.step() # udpate G's weights \ No newline at end of file diff --git a/annotator/leres/pix2pix/options/__init__.py b/annotator/leres/pix2pix/options/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e7eedebe54aa70169fd25951b3034d819e396c90 --- /dev/null +++ b/annotator/leres/pix2pix/options/__init__.py @@ -0,0 +1 @@ +"""This package options includes option modules: training options, test options, and basic options (used in both training and test).""" diff --git a/annotator/leres/pix2pix/options/base_options.py b/annotator/leres/pix2pix/options/base_options.py new file mode 100644 index 0000000000000000000000000000000000000000..533a1e88a7e8494223f6994e6861c93667754f83 --- /dev/null +++ b/annotator/leres/pix2pix/options/base_options.py @@ -0,0 +1,156 @@ +import argparse +import os +from ...pix2pix.util import util +# import torch +from ...pix2pix import models +# import pix2pix.data +import numpy as np + +class BaseOptions(): + """This class defines options used during both training and test time. + + It also implements several helper functions such as parsing, printing, and saving the options. + It also gathers additional options defined in functions in both dataset class and model class. + """ + + def __init__(self): + """Reset the class; indicates the class hasn't been initailized""" + self.initialized = False + + def initialize(self, parser): + """Define the common options that are used in both training and test.""" + # basic parameters + parser.add_argument('--dataroot', help='path to images (should have subfolders trainA, trainB, valA, valB, etc)') + parser.add_argument('--name', type=str, default='void', help='mahdi_unet_new, scaled_unet') + parser.add_argument('--gpu_ids', type=str, default='0', help='gpu ids: e.g. 0 0,1,2, 0,2. use -1 for CPU') + parser.add_argument('--checkpoints_dir', type=str, default='./pix2pix/checkpoints', help='models are saved here') + # model parameters + parser.add_argument('--model', type=str, default='cycle_gan', help='chooses which model to use. [cycle_gan | pix2pix | test | colorization]') + parser.add_argument('--input_nc', type=int, default=2, help='# of input image channels: 3 for RGB and 1 for grayscale') + parser.add_argument('--output_nc', type=int, default=1, help='# of output image channels: 3 for RGB and 1 for grayscale') + parser.add_argument('--ngf', type=int, default=64, help='# of gen filters in the last conv layer') + parser.add_argument('--ndf', type=int, default=64, help='# of discrim filters in the first conv layer') + parser.add_argument('--netD', type=str, default='basic', help='specify discriminator architecture [basic | n_layers | pixel]. The basic model is a 70x70 PatchGAN. n_layers allows you to specify the layers in the discriminator') + parser.add_argument('--netG', type=str, default='resnet_9blocks', help='specify generator architecture [resnet_9blocks | resnet_6blocks | unet_256 | unet_128]') + parser.add_argument('--n_layers_D', type=int, default=3, help='only used if netD==n_layers') + parser.add_argument('--norm', type=str, default='instance', help='instance normalization or batch normalization [instance | batch | none]') + parser.add_argument('--init_type', type=str, default='normal', help='network initialization [normal | xavier | kaiming | orthogonal]') + parser.add_argument('--init_gain', type=float, default=0.02, help='scaling factor for normal, xavier and orthogonal.') + parser.add_argument('--no_dropout', action='store_true', help='no dropout for the generator') + # dataset parameters + parser.add_argument('--dataset_mode', type=str, default='unaligned', help='chooses how datasets are loaded. [unaligned | aligned | single | colorization]') + parser.add_argument('--direction', type=str, default='AtoB', help='AtoB or BtoA') + parser.add_argument('--serial_batches', action='store_true', help='if true, takes images in order to make batches, otherwise takes them randomly') + parser.add_argument('--num_threads', default=4, type=int, help='# threads for loading data') + parser.add_argument('--batch_size', type=int, default=1, help='input batch size') + parser.add_argument('--load_size', type=int, default=672, help='scale images to this size') + parser.add_argument('--crop_size', type=int, default=672, help='then crop to this size') + parser.add_argument('--max_dataset_size', type=int, default=10000, help='Maximum number of samples allowed per dataset. If the dataset directory contains more than max_dataset_size, only a subset is loaded.') + parser.add_argument('--preprocess', type=str, default='resize_and_crop', help='scaling and cropping of images at load time [resize_and_crop | crop | scale_width | scale_width_and_crop | none]') + parser.add_argument('--no_flip', action='store_true', help='if specified, do not flip the images for data augmentation') + parser.add_argument('--display_winsize', type=int, default=256, help='display window size for both visdom and HTML') + # additional parameters + parser.add_argument('--epoch', type=str, default='latest', help='which epoch to load? set to latest to use latest cached model') + parser.add_argument('--load_iter', type=int, default='0', help='which iteration to load? if load_iter > 0, the code will load models by iter_[load_iter]; otherwise, the code will load models by [epoch]') + parser.add_argument('--verbose', action='store_true', help='if specified, print more debugging information') + parser.add_argument('--suffix', default='', type=str, help='customized suffix: opt.name = opt.name + suffix: e.g., {model}_{netG}_size{load_size}') + + parser.add_argument('--data_dir', type=str, required=False, + help='input files directory images can be .png .jpg .tiff') + parser.add_argument('--output_dir', type=str, required=False, + help='result dir. result depth will be png. vides are JMPG as avi') + parser.add_argument('--savecrops', type=int, required=False) + parser.add_argument('--savewholeest', type=int, required=False) + parser.add_argument('--output_resolution', type=int, required=False, + help='0 for no restriction 1 for resize to input size') + parser.add_argument('--net_receptive_field_size', type=int, required=False) + parser.add_argument('--pix2pixsize', type=int, required=False) + parser.add_argument('--generatevideo', type=int, required=False) + parser.add_argument('--depthNet', type=int, required=False, help='0: midas 1:strurturedRL') + parser.add_argument('--R0', action='store_true') + parser.add_argument('--R20', action='store_true') + parser.add_argument('--Final', action='store_true') + parser.add_argument('--colorize_results', action='store_true') + parser.add_argument('--max_res', type=float, default=np.inf) + + self.initialized = True + return parser + + def gather_options(self): + """Initialize our parser with basic options(only once). + Add additional model-specific and dataset-specific options. + These options are defined in the function + in model and dataset classes. + """ + if not self.initialized: # check if it has been initialized + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser = self.initialize(parser) + + # get the basic options + opt, _ = parser.parse_known_args() + + # modify model-related parser options + model_name = opt.model + model_option_setter = models.get_option_setter(model_name) + parser = model_option_setter(parser, self.isTrain) + opt, _ = parser.parse_known_args() # parse again with new defaults + + # modify dataset-related parser options + # dataset_name = opt.dataset_mode + # dataset_option_setter = pix2pix.data.get_option_setter(dataset_name) + # parser = dataset_option_setter(parser, self.isTrain) + + # save and return the parser + self.parser = parser + #return parser.parse_args() #EVIL + return opt + + def print_options(self, opt): + """Print and save options + + It will print both current options and default values(if different). + It will save options into a text file / [checkpoints_dir] / opt.txt + """ + message = '' + message += '----------------- Options ---------------\n' + for k, v in sorted(vars(opt).items()): + comment = '' + default = self.parser.get_default(k) + if v != default: + comment = '\t[default: %s]' % str(default) + message += '{:>25}: {:<30}{}\n'.format(str(k), str(v), comment) + message += '----------------- End -------------------' + print(message) + + # save to the disk + expr_dir = os.path.join(opt.checkpoints_dir, opt.name) + util.mkdirs(expr_dir) + file_name = os.path.join(expr_dir, '{}_opt.txt'.format(opt.phase)) + with open(file_name, 'wt') as opt_file: + opt_file.write(message) + opt_file.write('\n') + + def parse(self): + """Parse our options, create checkpoints directory suffix, and set up gpu device.""" + opt = self.gather_options() + opt.isTrain = self.isTrain # train or test + + # process opt.suffix + if opt.suffix: + suffix = ('_' + opt.suffix.format(**vars(opt))) if opt.suffix != '' else '' + opt.name = opt.name + suffix + + #self.print_options(opt) + + # set gpu ids + str_ids = opt.gpu_ids.split(',') + opt.gpu_ids = [] + for str_id in str_ids: + id = int(str_id) + if id >= 0: + opt.gpu_ids.append(id) + #if len(opt.gpu_ids) > 0: + # torch.cuda.set_device(opt.gpu_ids[0]) + + self.opt = opt + return self.opt diff --git a/annotator/leres/pix2pix/options/test_options.py b/annotator/leres/pix2pix/options/test_options.py new file mode 100644 index 0000000000000000000000000000000000000000..a3424b5e3b66d6813f74c8cecad691d7488d121c --- /dev/null +++ b/annotator/leres/pix2pix/options/test_options.py @@ -0,0 +1,22 @@ +from .base_options import BaseOptions + + +class TestOptions(BaseOptions): + """This class includes test options. + + It also includes shared options defined in BaseOptions. + """ + + def initialize(self, parser): + parser = BaseOptions.initialize(self, parser) # define shared options + parser.add_argument('--aspect_ratio', type=float, default=1.0, help='aspect ratio of result images') + parser.add_argument('--phase', type=str, default='test', help='train, val, test, etc') + # Dropout and Batchnorm has different behavioir during training and test. + parser.add_argument('--eval', action='store_true', help='use eval mode during test time.') + parser.add_argument('--num_test', type=int, default=50, help='how many test images to run') + # rewrite devalue values + parser.set_defaults(model='pix2pix4depth') + # To avoid cropping, the load_size should be the same as crop_size + parser.set_defaults(load_size=parser.get_default('crop_size')) + self.isTrain = False + return parser diff --git a/annotator/leres/pix2pix/util/__init__.py b/annotator/leres/pix2pix/util/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ae36f63d8859ec0c60dcbfe67c4ac324e751ddf7 --- /dev/null +++ b/annotator/leres/pix2pix/util/__init__.py @@ -0,0 +1 @@ +"""This package includes a miscellaneous collection of useful helper functions.""" diff --git a/annotator/leres/pix2pix/util/get_data.py b/annotator/leres/pix2pix/util/get_data.py new file mode 100644 index 0000000000000000000000000000000000000000..97edc3ce3c3ab6d6080dca34e73a5fb77bb715fb --- /dev/null +++ b/annotator/leres/pix2pix/util/get_data.py @@ -0,0 +1,110 @@ +from __future__ import print_function +import os +import tarfile +import requests +from warnings import warn +from zipfile import ZipFile +from bs4 import BeautifulSoup +from os.path import abspath, isdir, join, basename + + +class GetData(object): + """A Python script for downloading CycleGAN or pix2pix datasets. + + Parameters: + technique (str) -- One of: 'cyclegan' or 'pix2pix'. + verbose (bool) -- If True, print additional information. + + Examples: + >>> from util.get_data import GetData + >>> gd = GetData(technique='cyclegan') + >>> new_data_path = gd.get(save_path='./datasets') # options will be displayed. + + Alternatively, You can use bash scripts: 'scripts/download_pix2pix_model.sh' + and 'scripts/download_cyclegan_model.sh'. + """ + + def __init__(self, technique='cyclegan', verbose=True): + url_dict = { + 'pix2pix': 'http://efrosgans.eecs.berkeley.edu/pix2pix/datasets/', + 'cyclegan': 'https://people.eecs.berkeley.edu/~taesung_park/CycleGAN/datasets' + } + self.url = url_dict.get(technique.lower()) + self._verbose = verbose + + def _print(self, text): + if self._verbose: + print(text) + + @staticmethod + def _get_options(r): + soup = BeautifulSoup(r.text, 'lxml') + options = [h.text for h in soup.find_all('a', href=True) + if h.text.endswith(('.zip', 'tar.gz'))] + return options + + def _present_options(self): + r = requests.get(self.url) + options = self._get_options(r) + print('Options:\n') + for i, o in enumerate(options): + print("{0}: {1}".format(i, o)) + choice = input("\nPlease enter the number of the " + "dataset above you wish to download:") + return options[int(choice)] + + def _download_data(self, dataset_url, save_path): + if not isdir(save_path): + os.makedirs(save_path) + + base = basename(dataset_url) + temp_save_path = join(save_path, base) + + with open(temp_save_path, "wb") as f: + r = requests.get(dataset_url) + f.write(r.content) + + if base.endswith('.tar.gz'): + obj = tarfile.open(temp_save_path) + elif base.endswith('.zip'): + obj = ZipFile(temp_save_path, 'r') + else: + raise ValueError("Unknown File Type: {0}.".format(base)) + + self._print("Unpacking Data...") + obj.extractall(save_path) + obj.close() + os.remove(temp_save_path) + + def get(self, save_path, dataset=None): + """ + + Download a dataset. + + Parameters: + save_path (str) -- A directory to save the data to. + dataset (str) -- (optional). A specific dataset to download. + Note: this must include the file extension. + If None, options will be presented for you + to choose from. + + Returns: + save_path_full (str) -- the absolute path to the downloaded data. + + """ + if dataset is None: + selected_dataset = self._present_options() + else: + selected_dataset = dataset + + save_path_full = join(save_path, selected_dataset.split('.')[0]) + + if isdir(save_path_full): + warn("\n'{0}' already exists. Voiding Download.".format( + save_path_full)) + else: + self._print('Downloading Data...') + url = "{0}/{1}".format(self.url, selected_dataset) + self._download_data(url, save_path=save_path) + + return abspath(save_path_full) diff --git a/annotator/leres/pix2pix/util/guidedfilter.py b/annotator/leres/pix2pix/util/guidedfilter.py new file mode 100644 index 0000000000000000000000000000000000000000..d377ff12e078a5f156e9246b63573dae71825fad --- /dev/null +++ b/annotator/leres/pix2pix/util/guidedfilter.py @@ -0,0 +1,47 @@ +import numpy as np + +class GuidedFilter(): + def __init__(self, source, reference, r=64, eps= 0.05**2): + self.source = source; + self.reference = reference; + self.r = r + self.eps = eps + + self.smooth = self.guidedfilter(self.source,self.reference,self.r,self.eps) + + def boxfilter(self,img, r): + (rows, cols) = img.shape + imDst = np.zeros_like(img) + + imCum = np.cumsum(img, 0) + imDst[0 : r+1, :] = imCum[r : 2*r+1, :] + imDst[r+1 : rows-r, :] = imCum[2*r+1 : rows, :] - imCum[0 : rows-2*r-1, :] + imDst[rows-r: rows, :] = np.tile(imCum[rows-1, :], [r, 1]) - imCum[rows-2*r-1 : rows-r-1, :] + + imCum = np.cumsum(imDst, 1) + imDst[:, 0 : r+1] = imCum[:, r : 2*r+1] + imDst[:, r+1 : cols-r] = imCum[:, 2*r+1 : cols] - imCum[:, 0 : cols-2*r-1] + imDst[:, cols-r: cols] = np.tile(imCum[:, cols-1], [r, 1]).T - imCum[:, cols-2*r-1 : cols-r-1] + + return imDst + + def guidedfilter(self,I, p, r, eps): + (rows, cols) = I.shape + N = self.boxfilter(np.ones([rows, cols]), r) + + meanI = self.boxfilter(I, r) / N + meanP = self.boxfilter(p, r) / N + meanIp = self.boxfilter(I * p, r) / N + covIp = meanIp - meanI * meanP + + meanII = self.boxfilter(I * I, r) / N + varI = meanII - meanI * meanI + + a = covIp / (varI + eps) + b = meanP - a * meanI + + meanA = self.boxfilter(a, r) / N + meanB = self.boxfilter(b, r) / N + + q = meanA * I + meanB + return q \ No newline at end of file diff --git a/annotator/leres/pix2pix/util/html.py b/annotator/leres/pix2pix/util/html.py new file mode 100644 index 0000000000000000000000000000000000000000..cc3262a1eafda34842e4dbad47bb6ba72f0c5a68 --- /dev/null +++ b/annotator/leres/pix2pix/util/html.py @@ -0,0 +1,86 @@ +import dominate +from dominate.tags import meta, h3, table, tr, td, p, a, img, br +import os + + +class HTML: + """This HTML class allows us to save images and write texts into a single HTML file. + + It consists of functions such as (add a text header to the HTML file), + (add a row of images to the HTML file), and (save the HTML to the disk). + It is based on Python library 'dominate', a Python library for creating and manipulating HTML documents using a DOM API. + """ + + def __init__(self, web_dir, title, refresh=0): + """Initialize the HTML classes + + Parameters: + web_dir (str) -- a directory that stores the webpage. HTML file will be created at /index.html; images will be saved at 0: + with self.doc.head: + meta(http_equiv="refresh", content=str(refresh)) + + def get_image_dir(self): + """Return the directory that stores images""" + return self.img_dir + + def add_header(self, text): + """Insert a header to the HTML file + + Parameters: + text (str) -- the header text + """ + with self.doc: + h3(text) + + def add_images(self, ims, txts, links, width=400): + """add images to the HTML file + + Parameters: + ims (str list) -- a list of image paths + txts (str list) -- a list of image names shown on the website + links (str list) -- a list of hyperref links; when you click an image, it will redirect you to a new page + """ + self.t = table(border=1, style="table-layout: fixed;") # Insert a table + self.doc.add(self.t) + with self.t: + with tr(): + for im, txt, link in zip(ims, txts, links): + with td(style="word-wrap: break-word;", halign="center", valign="top"): + with p(): + with a(href=os.path.join('images', link)): + img(style="width:%dpx" % width, src=os.path.join('images', im)) + br() + p(txt) + + def save(self): + """save the current content to the HMTL file""" + html_file = '%s/index.html' % self.web_dir + f = open(html_file, 'wt') + f.write(self.doc.render()) + f.close() + + +if __name__ == '__main__': # we show an example usage here. + html = HTML('web/', 'test_html') + html.add_header('hello world') + + ims, txts, links = [], [], [] + for n in range(4): + ims.append('image_%d.png' % n) + txts.append('text_%d' % n) + links.append('image_%d.png' % n) + html.add_images(ims, txts, links) + html.save() diff --git a/annotator/leres/pix2pix/util/image_pool.py b/annotator/leres/pix2pix/util/image_pool.py new file mode 100644 index 0000000000000000000000000000000000000000..6d086f882bc3d1b90c529fce6cddaaa75f2005d7 --- /dev/null +++ b/annotator/leres/pix2pix/util/image_pool.py @@ -0,0 +1,54 @@ +import random +import torch + + +class ImagePool(): + """This class implements an image buffer that stores previously generated images. + + This buffer enables us to update discriminators using a history of generated images + rather than the ones produced by the latest generators. + """ + + def __init__(self, pool_size): + """Initialize the ImagePool class + + Parameters: + pool_size (int) -- the size of image buffer, if pool_size=0, no buffer will be created + """ + self.pool_size = pool_size + if self.pool_size > 0: # create an empty pool + self.num_imgs = 0 + self.images = [] + + def query(self, images): + """Return an image from the pool. + + Parameters: + images: the latest generated images from the generator + + Returns images from the buffer. + + By 50/100, the buffer will return input images. + By 50/100, the buffer will return images previously stored in the buffer, + and insert the current images to the buffer. + """ + if self.pool_size == 0: # if the buffer size is 0, do nothing + return images + return_images = [] + for image in images: + image = torch.unsqueeze(image.data, 0) + if self.num_imgs < self.pool_size: # if the buffer is not full; keep inserting current images to the buffer + self.num_imgs = self.num_imgs + 1 + self.images.append(image) + return_images.append(image) + else: + p = random.uniform(0, 1) + if p > 0.5: # by 50% chance, the buffer will return a previously stored image, and insert the current image into the buffer + random_id = random.randint(0, self.pool_size - 1) # randint is inclusive + tmp = self.images[random_id].clone() + self.images[random_id] = image + return_images.append(tmp) + else: # by another 50% chance, the buffer will return the current image + return_images.append(image) + return_images = torch.cat(return_images, 0) # collect all the images and return + return return_images diff --git a/annotator/leres/pix2pix/util/util.py b/annotator/leres/pix2pix/util/util.py new file mode 100644 index 0000000000000000000000000000000000000000..8a7aceaa00681cb76675df7866bf8db58c8d2caf --- /dev/null +++ b/annotator/leres/pix2pix/util/util.py @@ -0,0 +1,105 @@ +"""This module contains simple helper functions """ +from __future__ import print_function +import torch +import numpy as np +from PIL import Image +import os + + +def tensor2im(input_image, imtype=np.uint16): + """"Converts a Tensor array into a numpy image array. + + Parameters: + input_image (tensor) -- the input image tensor array + imtype (type) -- the desired type of the converted numpy array + """ + if not isinstance(input_image, np.ndarray): + if isinstance(input_image, torch.Tensor): # get the data from a variable + image_tensor = input_image.data + else: + return input_image + image_numpy = torch.squeeze(image_tensor).cpu().numpy() # convert it into a numpy array + image_numpy = (image_numpy + 1) / 2.0 * (2**16-1) # + else: # if it is a numpy array, do nothing + image_numpy = input_image + return image_numpy.astype(imtype) + + +def diagnose_network(net, name='network'): + """Calculate and print the mean of average absolute(gradients) + + Parameters: + net (torch network) -- Torch network + name (str) -- the name of the network + """ + mean = 0.0 + count = 0 + for param in net.parameters(): + if param.grad is not None: + mean += torch.mean(torch.abs(param.grad.data)) + count += 1 + if count > 0: + mean = mean / count + print(name) + print(mean) + + +def save_image(image_numpy, image_path, aspect_ratio=1.0): + """Save a numpy image to the disk + + Parameters: + image_numpy (numpy array) -- input numpy array + image_path (str) -- the path of the image + """ + image_pil = Image.fromarray(image_numpy) + + image_pil = image_pil.convert('I;16') + + # image_pil = Image.fromarray(image_numpy) + # h, w, _ = image_numpy.shape + # + # if aspect_ratio > 1.0: + # image_pil = image_pil.resize((h, int(w * aspect_ratio)), Image.BICUBIC) + # if aspect_ratio < 1.0: + # image_pil = image_pil.resize((int(h / aspect_ratio), w), Image.BICUBIC) + + image_pil.save(image_path) + + +def print_numpy(x, val=True, shp=False): + """Print the mean, min, max, median, std, and size of a numpy array + + Parameters: + val (bool) -- if print the values of the numpy array + shp (bool) -- if print the shape of the numpy array + """ + x = x.astype(np.float64) + if shp: + print('shape,', x.shape) + if val: + x = x.flatten() + print('mean = %3.3f, min = %3.3f, max = %3.3f, median = %3.3f, std=%3.3f' % ( + np.mean(x), np.min(x), np.max(x), np.median(x), np.std(x))) + + +def mkdirs(paths): + """create empty directories if they don't exist + + Parameters: + paths (str list) -- a list of directory paths + """ + if isinstance(paths, list) and not isinstance(paths, str): + for path in paths: + mkdir(path) + else: + mkdir(paths) + + +def mkdir(path): + """create a single empty directory if it didn't exist + + Parameters: + path (str) -- a single directory path + """ + if not os.path.exists(path): + os.makedirs(path) diff --git a/annotator/leres/pix2pix/util/visualizer.py b/annotator/leres/pix2pix/util/visualizer.py new file mode 100644 index 0000000000000000000000000000000000000000..810a0513ab997103ace77b665c9a17f223b173c9 --- /dev/null +++ b/annotator/leres/pix2pix/util/visualizer.py @@ -0,0 +1,166 @@ +import numpy as np +import os +import sys +import ntpath +import time +from . import util, html +from subprocess import Popen, PIPE +import torch + + +if sys.version_info[0] == 2: + VisdomExceptionBase = Exception +else: + VisdomExceptionBase = ConnectionError + + +def save_images(webpage, visuals, image_path, aspect_ratio=1.0, width=256): + """Save images to the disk. + + Parameters: + webpage (the HTML class) -- the HTML webpage class that stores these imaegs (see html.py for more details) + visuals (OrderedDict) -- an ordered dictionary that stores (name, images (either tensor or numpy) ) pairs + image_path (str) -- the string is used to create image paths + aspect_ratio (float) -- the aspect ratio of saved images + width (int) -- the images will be resized to width x width + + This function will save images stored in 'visuals' to the HTML file specified by 'webpage'. + """ + image_dir = webpage.get_image_dir() + short_path = ntpath.basename(image_path[0]) + name = os.path.splitext(short_path)[0] + + webpage.add_header(name) + ims, txts, links = [], [], [] + + for label, im_data in visuals.items(): + im = util.tensor2im(im_data) + image_name = '%s_%s.png' % (name, label) + save_path = os.path.join(image_dir, image_name) + util.save_image(im, save_path, aspect_ratio=aspect_ratio) + ims.append(image_name) + txts.append(label) + links.append(image_name) + webpage.add_images(ims, txts, links, width=width) + + +class Visualizer(): + """This class includes several functions that can display/save images and print/save logging information. + + It uses a Python library 'visdom' for display, and a Python library 'dominate' (wrapped in 'HTML') for creating HTML files with images. + """ + + def __init__(self, opt): + """Initialize the Visualizer class + + Parameters: + opt -- stores all the experiment flags; needs to be a subclass of BaseOptions + Step 1: Cache the training/test options + Step 2: connect to a visdom server + Step 3: create an HTML object for saveing HTML filters + Step 4: create a logging file to store training losses + """ + self.opt = opt # cache the option + self.display_id = opt.display_id + self.use_html = opt.isTrain and not opt.no_html + self.win_size = opt.display_winsize + self.name = opt.name + self.port = opt.display_port + self.saved = False + + if self.use_html: # create an HTML object at /web/; images will be saved under /web/images/ + self.web_dir = os.path.join(opt.checkpoints_dir, opt.name, 'web') + self.img_dir = os.path.join(self.web_dir, 'images') + print('create web directory %s...' % self.web_dir) + util.mkdirs([self.web_dir, self.img_dir]) + # create a logging file to store training losses + self.log_name = os.path.join(opt.checkpoints_dir, opt.name, 'loss_log.txt') + with open(self.log_name, "a") as log_file: + now = time.strftime("%c") + log_file.write('================ Training Loss (%s) ================\n' % now) + + def reset(self): + """Reset the self.saved status""" + self.saved = False + + def create_visdom_connections(self): + """If the program could not connect to Visdom server, this function will start a new server at port < self.port > """ + cmd = sys.executable + ' -m visdom.server -p %d &>/dev/null &' % self.port + print('\n\nCould not connect to Visdom server. \n Trying to start a server....') + print('Command: %s' % cmd) + Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE) + + def display_current_results(self, visuals, epoch, save_result): + """Display current results on visdom; save current results to an HTML file. + + Parameters: + visuals (OrderedDict) - - dictionary of images to display or save + epoch (int) - - the current epoch + save_result (bool) - - if save the current results to an HTML file + """ + if self.use_html and (save_result or not self.saved): # save images to an HTML file if they haven't been saved. + self.saved = True + # save images to the disk + for label, image in visuals.items(): + image_numpy = util.tensor2im(image) + img_path = os.path.join(self.img_dir, 'epoch%.3d_%s.png' % (epoch, label)) + util.save_image(image_numpy, img_path) + + # update website + webpage = html.HTML(self.web_dir, 'Experiment name = %s' % self.name, refresh=1) + for n in range(epoch, 0, -1): + webpage.add_header('epoch [%d]' % n) + ims, txts, links = [], [], [] + + for label, image_numpy in visuals.items(): + # image_numpy = util.tensor2im(image) + img_path = 'epoch%.3d_%s.png' % (n, label) + ims.append(img_path) + txts.append(label) + links.append(img_path) + webpage.add_images(ims, txts, links, width=self.win_size) + webpage.save() + + # def plot_current_losses(self, epoch, counter_ratio, losses): + # """display the current losses on visdom display: dictionary of error labels and values + # + # Parameters: + # epoch (int) -- current epoch + # counter_ratio (float) -- progress (percentage) in the current epoch, between 0 to 1 + # losses (OrderedDict) -- training losses stored in the format of (name, float) pairs + # """ + # if not hasattr(self, 'plot_data'): + # self.plot_data = {'X': [], 'Y': [], 'legend': list(losses.keys())} + # self.plot_data['X'].append(epoch + counter_ratio) + # self.plot_data['Y'].append([losses[k] for k in self.plot_data['legend']]) + # try: + # self.vis.line( + # X=np.stack([np.array(self.plot_data['X'])] * len(self.plot_data['legend']), 1), + # Y=np.array(self.plot_data['Y']), + # opts={ + # 'title': self.name + ' loss over time', + # 'legend': self.plot_data['legend'], + # 'xlabel': 'epoch', + # 'ylabel': 'loss'}, + # win=self.display_id) + # except VisdomExceptionBase: + # self.create_visdom_connections() + + # losses: same format as |losses| of plot_current_losses + def print_current_losses(self, epoch, iters, losses, t_comp, t_data): + """print current losses on console; also save the losses to the disk + + Parameters: + epoch (int) -- current epoch + iters (int) -- current training iteration during this epoch (reset to 0 at the end of every epoch) + losses (OrderedDict) -- training losses stored in the format of (name, float) pairs + t_comp (float) -- computational time per data point (normalized by batch_size) + t_data (float) -- data loading time per data point (normalized by batch_size) + """ + message = '(epoch: %d, iters: %d, time: %.3f, data: %.3f) ' % (epoch, iters, t_comp, t_data) + for k, v in losses.items(): + message += '%s: %.3f ' % (k, v) + + print(message) # print the message + with open(self.log_name, "a") as log_file: + log_file.write('%s\n' % message) # save the message diff --git a/annotator/lineart/LICENSE b/annotator/lineart/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..16a9d56a3d4c15e4f34ac5426459c58487b01520 --- /dev/null +++ b/annotator/lineart/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Caroline Chan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/annotator/lineart/__init__.py b/annotator/lineart/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5533846f66acdbc33da39804d402941c1fb45150 --- /dev/null +++ b/annotator/lineart/__init__.py @@ -0,0 +1,127 @@ +# From https://github.com/carolineec/informative-drawings +# MIT License + +import os +import cv2 +import torch +import numpy as np + +import torch.nn as nn +from einops import rearrange +from annotator.util import annotator_ckpts_path + + +norm_layer = nn.InstanceNorm2d + + +class ResidualBlock(nn.Module): + def __init__(self, in_features): + super(ResidualBlock, self).__init__() + + conv_block = [ nn.ReflectionPad2d(1), + nn.Conv2d(in_features, in_features, 3), + norm_layer(in_features), + nn.ReLU(inplace=True), + nn.ReflectionPad2d(1), + nn.Conv2d(in_features, in_features, 3), + norm_layer(in_features) + ] + + self.conv_block = nn.Sequential(*conv_block) + + def forward(self, x): + return x + self.conv_block(x) + + +class Generator(nn.Module): + def __init__(self, input_nc, output_nc, n_residual_blocks=9, sigmoid=True): + super(Generator, self).__init__() + + # Initial convolution block + model0 = [ nn.ReflectionPad2d(3), + nn.Conv2d(input_nc, 64, 7), + norm_layer(64), + nn.ReLU(inplace=True) ] + self.model0 = nn.Sequential(*model0) + + # Downsampling + model1 = [] + in_features = 64 + out_features = in_features*2 + for _ in range(2): + model1 += [ nn.Conv2d(in_features, out_features, 3, stride=2, padding=1), + norm_layer(out_features), + nn.ReLU(inplace=True) ] + in_features = out_features + out_features = in_features*2 + self.model1 = nn.Sequential(*model1) + + model2 = [] + # Residual blocks + for _ in range(n_residual_blocks): + model2 += [ResidualBlock(in_features)] + self.model2 = nn.Sequential(*model2) + + # Upsampling + model3 = [] + out_features = in_features//2 + for _ in range(2): + model3 += [ nn.ConvTranspose2d(in_features, out_features, 3, stride=2, padding=1, output_padding=1), + norm_layer(out_features), + nn.ReLU(inplace=True) ] + in_features = out_features + out_features = in_features//2 + self.model3 = nn.Sequential(*model3) + + # Output layer + model4 = [ nn.ReflectionPad2d(3), + nn.Conv2d(64, output_nc, 7)] + if sigmoid: + model4 += [nn.Sigmoid()] + + self.model4 = nn.Sequential(*model4) + + def forward(self, x, cond=None): + out = self.model0(x) + out = self.model1(out) + out = self.model2(out) + out = self.model3(out) + out = self.model4(out) + + return out + + +class LineartDetector: + def __init__(self): + self.model = self.load_model('sk_model.pth') + self.model_coarse = self.load_model('sk_model2.pth') + map_location=torch.device('cpu') + + def load_model(self, name): + remote_model_path = "https://huggingface.co./lllyasviel/Annotators/resolve/main/" + name + modelpath = os.path.join(annotator_ckpts_path, name) + if not os.path.exists(modelpath): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(remote_model_path, model_dir=annotator_ckpts_path) + model = Generator(3, 1, 3) + model.load_state_dict(torch.load(modelpath, map_location=torch.device('cpu'))) + model.eval() +# model = model.cuda() + model = model.cpu() + return model + + def __call__(self, input_image, coarse): + model = self.model_coarse if coarse else self.model + assert input_image.ndim == 3 + image = input_image + with torch.no_grad(): +# image = torch.from_numpy(image).float().cuda() + image = torch.from_numpy(image).float().cpu() + image = image / 255.0 + image = rearrange(image, 'h w c -> 1 c h w') + line = model(image)[0][0] + + line = line.cpu().numpy() + line = (line * 255.0).clip(0, 255).astype(np.uint8) + + return line diff --git a/annotator/lineart_anime/LICENSE b/annotator/lineart_anime/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..16a9d56a3d4c15e4f34ac5426459c58487b01520 --- /dev/null +++ b/annotator/lineart_anime/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Caroline Chan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/annotator/lineart_anime/__init__.py b/annotator/lineart_anime/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9d583ef409f29cd307535790b50faa64a94ed011 --- /dev/null +++ b/annotator/lineart_anime/__init__.py @@ -0,0 +1,153 @@ +# Anime2sketch +# https://github.com/Mukosame/Anime2Sketch + +import numpy as np +import torch +import torch.nn as nn +import functools + +import os +import cv2 +from einops import rearrange +from annotator.util import annotator_ckpts_path + + +class UnetGenerator(nn.Module): + """Create a Unet-based generator""" + + def __init__(self, input_nc, output_nc, num_downs, ngf=64, norm_layer=nn.BatchNorm2d, use_dropout=False): + """Construct a Unet generator + Parameters: + input_nc (int) -- the number of channels in input images + output_nc (int) -- the number of channels in output images + num_downs (int) -- the number of downsamplings in UNet. For example, # if |num_downs| == 7, + image of size 128x128 will become of size 1x1 # at the bottleneck + ngf (int) -- the number of filters in the last conv layer + norm_layer -- normalization layer + We construct the U-Net from the innermost layer to the outermost layer. + It is a recursive process. + """ + super(UnetGenerator, self).__init__() + # construct unet structure + unet_block = UnetSkipConnectionBlock(ngf * 8, ngf * 8, input_nc=None, submodule=None, norm_layer=norm_layer, innermost=True) # add the innermost layer + for _ in range(num_downs - 5): # add intermediate layers with ngf * 8 filters + unet_block = UnetSkipConnectionBlock(ngf * 8, ngf * 8, input_nc=None, submodule=unet_block, norm_layer=norm_layer, use_dropout=use_dropout) + # gradually reduce the number of filters from ngf * 8 to ngf + unet_block = UnetSkipConnectionBlock(ngf * 4, ngf * 8, input_nc=None, submodule=unet_block, norm_layer=norm_layer) + unet_block = UnetSkipConnectionBlock(ngf * 2, ngf * 4, input_nc=None, submodule=unet_block, norm_layer=norm_layer) + unet_block = UnetSkipConnectionBlock(ngf, ngf * 2, input_nc=None, submodule=unet_block, norm_layer=norm_layer) + self.model = UnetSkipConnectionBlock(output_nc, ngf, input_nc=input_nc, submodule=unet_block, outermost=True, norm_layer=norm_layer) # add the outermost layer + + def forward(self, input): + """Standard forward""" + return self.model(input) + + +class UnetSkipConnectionBlock(nn.Module): + """Defines the Unet submodule with skip connection. + X -------------------identity---------------------- + |-- downsampling -- |submodule| -- upsampling --| + """ + + def __init__(self, outer_nc, inner_nc, input_nc=None, + submodule=None, outermost=False, innermost=False, norm_layer=nn.BatchNorm2d, use_dropout=False): + """Construct a Unet submodule with skip connections. + Parameters: + outer_nc (int) -- the number of filters in the outer conv layer + inner_nc (int) -- the number of filters in the inner conv layer + input_nc (int) -- the number of channels in input images/features + submodule (UnetSkipConnectionBlock) -- previously defined submodules + outermost (bool) -- if this module is the outermost module + innermost (bool) -- if this module is the innermost module + norm_layer -- normalization layer + use_dropout (bool) -- if use dropout layers. + """ + super(UnetSkipConnectionBlock, self).__init__() + self.outermost = outermost + if type(norm_layer) == functools.partial: + use_bias = norm_layer.func == nn.InstanceNorm2d + else: + use_bias = norm_layer == nn.InstanceNorm2d + if input_nc is None: + input_nc = outer_nc + downconv = nn.Conv2d(input_nc, inner_nc, kernel_size=4, + stride=2, padding=1, bias=use_bias) + downrelu = nn.LeakyReLU(0.2, True) + downnorm = norm_layer(inner_nc) + uprelu = nn.ReLU(True) + upnorm = norm_layer(outer_nc) + + if outermost: + upconv = nn.ConvTranspose2d(inner_nc * 2, outer_nc, + kernel_size=4, stride=2, + padding=1) + down = [downconv] + up = [uprelu, upconv, nn.Tanh()] + model = down + [submodule] + up + elif innermost: + upconv = nn.ConvTranspose2d(inner_nc, outer_nc, + kernel_size=4, stride=2, + padding=1, bias=use_bias) + down = [downrelu, downconv] + up = [uprelu, upconv, upnorm] + model = down + up + else: + upconv = nn.ConvTranspose2d(inner_nc * 2, outer_nc, + kernel_size=4, stride=2, + padding=1, bias=use_bias) + down = [downrelu, downconv, downnorm] + up = [uprelu, upconv, upnorm] + + if use_dropout: + model = down + [submodule] + up + [nn.Dropout(0.5)] + else: + model = down + [submodule] + up + + self.model = nn.Sequential(*model) + + def forward(self, x): + if self.outermost: + return self.model(x) + else: # add skip connections + return torch.cat([x, self.model(x)], 1) + + +class LineartAnimeDetector: + def __init__(self): + remote_model_path = "https://huggingface.co./lllyasviel/Annotators/resolve/main/netG.pth" + modelpath = os.path.join(annotator_ckpts_path, "netG.pth") + if not os.path.exists(modelpath): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(remote_model_path, model_dir=annotator_ckpts_path) + norm_layer = functools.partial(nn.InstanceNorm2d, affine=False, track_running_stats=False) + net = UnetGenerator(3, 1, 8, 64, norm_layer=norm_layer, use_dropout=False) +# ckpt = torch.load(modelpath) + ckpt = torch.load(modelpath, map_location=torch.device('cpu')) + for key in list(ckpt.keys()): + if 'module.' in key: + ckpt[key.replace('module.', '')] = ckpt[key] + del ckpt[key] + net.load_state_dict(ckpt) +# net = net.cuda() + net = net.cpu() + net.eval() + self.model = net + + def __call__(self, input_image): + H, W, C = input_image.shape + Hn = 256 * int(np.ceil(float(H) / 256.0)) + Wn = 256 * int(np.ceil(float(W) / 256.0)) + img = cv2.resize(input_image, (Wn, Hn), interpolation=cv2.INTER_CUBIC) + with torch.no_grad(): +# image_feed = torch.from_numpy(img).float().cuda() + image_feed = torch.from_numpy(img).float().cpu() + image_feed = image_feed / 127.5 - 1.0 + image_feed = rearrange(image_feed, 'h w c -> 1 c h w') + + line = self.model(image_feed)[0, 0] * 127.5 + 127.5 + line = line.cpu().numpy() + + line = cv2.resize(line, (W, H), interpolation=cv2.INTER_CUBIC) + line = line.clip(0, 255).astype(np.uint8) + return line + diff --git a/annotator/manga_line/LICENSE b/annotator/manga_line/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..9bad05450ca061904f97acebe04ff7183cfbdc1a --- /dev/null +++ b/annotator/manga_line/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Miaomiao Li + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/annotator/manga_line/__init__.py b/annotator/manga_line/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c797fd22c563a775162c8fd646dfa50077e4ac16 --- /dev/null +++ b/annotator/manga_line/__init__.py @@ -0,0 +1,248 @@ +import os +import torch +import torch.nn as nn +from PIL import Image +import fnmatch +import cv2 + +import sys + +import numpy as np +from einops import rearrange +from modules import devices +from annotator.annotator_path import models_path + + +class _bn_relu_conv(nn.Module): + def __init__(self, in_filters, nb_filters, fw, fh, subsample=1): + super(_bn_relu_conv, self).__init__() + self.model = nn.Sequential( + nn.BatchNorm2d(in_filters, eps=1e-3), + nn.LeakyReLU(0.2), + nn.Conv2d(in_filters, nb_filters, (fw, fh), stride=subsample, padding=(fw//2, fh//2), padding_mode='zeros') + ) + + def forward(self, x): + return self.model(x) + + # the following are for debugs + print("****", np.max(x.cpu().numpy()), np.min(x.cpu().numpy()), np.mean(x.cpu().numpy()), np.std(x.cpu().numpy()), x.shape) + for i,layer in enumerate(self.model): + if i != 2: + x = layer(x) + else: + x = layer(x) + #x = nn.functional.pad(x, (1, 1, 1, 1), mode='constant', value=0) + print("____", np.max(x.cpu().numpy()), np.min(x.cpu().numpy()), np.mean(x.cpu().numpy()), np.std(x.cpu().numpy()), x.shape) + print(x[0]) + return x + +class _u_bn_relu_conv(nn.Module): + def __init__(self, in_filters, nb_filters, fw, fh, subsample=1): + super(_u_bn_relu_conv, self).__init__() + self.model = nn.Sequential( + nn.BatchNorm2d(in_filters, eps=1e-3), + nn.LeakyReLU(0.2), + nn.Conv2d(in_filters, nb_filters, (fw, fh), stride=subsample, padding=(fw//2, fh//2)), + nn.Upsample(scale_factor=2, mode='nearest') + ) + + def forward(self, x): + return self.model(x) + + + +class _shortcut(nn.Module): + def __init__(self, in_filters, nb_filters, subsample=1): + super(_shortcut, self).__init__() + self.process = False + self.model = None + if in_filters != nb_filters or subsample != 1: + self.process = True + self.model = nn.Sequential( + nn.Conv2d(in_filters, nb_filters, (1, 1), stride=subsample) + ) + + def forward(self, x, y): + #print(x.size(), y.size(), self.process) + if self.process: + y0 = self.model(x) + #print("merge+", torch.max(y0+y), torch.min(y0+y),torch.mean(y0+y), torch.std(y0+y), y0.shape) + return y0 + y + else: + #print("merge", torch.max(x+y), torch.min(x+y),torch.mean(x+y), torch.std(x+y), y.shape) + return x + y + +class _u_shortcut(nn.Module): + def __init__(self, in_filters, nb_filters, subsample): + super(_u_shortcut, self).__init__() + self.process = False + self.model = None + if in_filters != nb_filters: + self.process = True + self.model = nn.Sequential( + nn.Conv2d(in_filters, nb_filters, (1, 1), stride=subsample, padding_mode='zeros'), + nn.Upsample(scale_factor=2, mode='nearest') + ) + + def forward(self, x, y): + if self.process: + return self.model(x) + y + else: + return x + y + + +class basic_block(nn.Module): + def __init__(self, in_filters, nb_filters, init_subsample=1): + super(basic_block, self).__init__() + self.conv1 = _bn_relu_conv(in_filters, nb_filters, 3, 3, subsample=init_subsample) + self.residual = _bn_relu_conv(nb_filters, nb_filters, 3, 3) + self.shortcut = _shortcut(in_filters, nb_filters, subsample=init_subsample) + + def forward(self, x): + x1 = self.conv1(x) + x2 = self.residual(x1) + return self.shortcut(x, x2) + +class _u_basic_block(nn.Module): + def __init__(self, in_filters, nb_filters, init_subsample=1): + super(_u_basic_block, self).__init__() + self.conv1 = _u_bn_relu_conv(in_filters, nb_filters, 3, 3, subsample=init_subsample) + self.residual = _bn_relu_conv(nb_filters, nb_filters, 3, 3) + self.shortcut = _u_shortcut(in_filters, nb_filters, subsample=init_subsample) + + def forward(self, x): + y = self.residual(self.conv1(x)) + return self.shortcut(x, y) + + +class _residual_block(nn.Module): + def __init__(self, in_filters, nb_filters, repetitions, is_first_layer=False): + super(_residual_block, self).__init__() + layers = [] + for i in range(repetitions): + init_subsample = 1 + if i == repetitions - 1 and not is_first_layer: + init_subsample = 2 + if i == 0: + l = basic_block(in_filters=in_filters, nb_filters=nb_filters, init_subsample=init_subsample) + else: + l = basic_block(in_filters=nb_filters, nb_filters=nb_filters, init_subsample=init_subsample) + layers.append(l) + + self.model = nn.Sequential(*layers) + + def forward(self, x): + return self.model(x) + + +class _upsampling_residual_block(nn.Module): + def __init__(self, in_filters, nb_filters, repetitions): + super(_upsampling_residual_block, self).__init__() + layers = [] + for i in range(repetitions): + l = None + if i == 0: + l = _u_basic_block(in_filters=in_filters, nb_filters=nb_filters)#(input) + else: + l = basic_block(in_filters=nb_filters, nb_filters=nb_filters)#(input) + layers.append(l) + + self.model = nn.Sequential(*layers) + + def forward(self, x): + return self.model(x) + + +class res_skip(nn.Module): + + def __init__(self): + super(res_skip, self).__init__() + self.block0 = _residual_block(in_filters=1, nb_filters=24, repetitions=2, is_first_layer=True)#(input) + self.block1 = _residual_block(in_filters=24, nb_filters=48, repetitions=3)#(block0) + self.block2 = _residual_block(in_filters=48, nb_filters=96, repetitions=5)#(block1) + self.block3 = _residual_block(in_filters=96, nb_filters=192, repetitions=7)#(block2) + self.block4 = _residual_block(in_filters=192, nb_filters=384, repetitions=12)#(block3) + + self.block5 = _upsampling_residual_block(in_filters=384, nb_filters=192, repetitions=7)#(block4) + self.res1 = _shortcut(in_filters=192, nb_filters=192)#(block3, block5, subsample=(1,1)) + + self.block6 = _upsampling_residual_block(in_filters=192, nb_filters=96, repetitions=5)#(res1) + self.res2 = _shortcut(in_filters=96, nb_filters=96)#(block2, block6, subsample=(1,1)) + + self.block7 = _upsampling_residual_block(in_filters=96, nb_filters=48, repetitions=3)#(res2) + self.res3 = _shortcut(in_filters=48, nb_filters=48)#(block1, block7, subsample=(1,1)) + + self.block8 = _upsampling_residual_block(in_filters=48, nb_filters=24, repetitions=2)#(res3) + self.res4 = _shortcut(in_filters=24, nb_filters=24)#(block0,block8, subsample=(1,1)) + + self.block9 = _residual_block(in_filters=24, nb_filters=16, repetitions=2, is_first_layer=True)#(res4) + self.conv15 = _bn_relu_conv(in_filters=16, nb_filters=1, fh=1, fw=1, subsample=1)#(block7) + + def forward(self, x): + x0 = self.block0(x) + x1 = self.block1(x0) + x2 = self.block2(x1) + x3 = self.block3(x2) + x4 = self.block4(x3) + + x5 = self.block5(x4) + res1 = self.res1(x3, x5) + + x6 = self.block6(res1) + res2 = self.res2(x2, x6) + + x7 = self.block7(res2) + res3 = self.res3(x1, x7) + + x8 = self.block8(res3) + res4 = self.res4(x0, x8) + + x9 = self.block9(res4) + y = self.conv15(x9) + + return y + + +class MangaLineExtration: + model_dir = os.path.join(models_path, "manga_line") + + def __init__(self): + self.model = None + self.device = devices.get_device_for("controlnet") + + def load_model(self): + remote_model_path = "https://huggingface.co./lllyasviel/Annotators/resolve/main/erika.pth" + modelpath = os.path.join(self.model_dir, "erika.pth") + if not os.path.exists(modelpath): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(remote_model_path, model_dir=self.model_dir) + #norm_layer = functools.partial(nn.InstanceNorm2d, affine=False, track_running_stats=False) + net = res_skip() + ckpt = torch.load(modelpath) + for key in list(ckpt.keys()): + if 'module.' in key: + ckpt[key.replace('module.', '')] = ckpt[key] + del ckpt[key] + net.load_state_dict(ckpt) + net.eval() + self.model = net.to(self.device) + + def unload_model(self): + if self.model is not None: + self.model.cpu() + + def __call__(self, input_image): + if self.model is None: + self.load_model() + self.model.to(self.device) + img = cv2.cvtColor(input_image, cv2.COLOR_RGB2GRAY) + img = np.ascontiguousarray(img.copy()).copy() + with torch.no_grad(): + image_feed = torch.from_numpy(img).float().to(self.device) + image_feed = rearrange(image_feed, 'h w -> 1 1 h w') + line = self.model(image_feed) + line = 255 - line.cpu().numpy()[0, 0] + return line.clip(0, 255).astype(np.uint8) + + diff --git a/annotator/mediapipe_face/__init__.py b/annotator/mediapipe_face/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f74edfb187e4e39583ed92bfe69ea29c42a34ddc --- /dev/null +++ b/annotator/mediapipe_face/__init__.py @@ -0,0 +1,5 @@ +from .mediapipe_face_common import generate_annotation + + +def apply_mediapipe_face(image, max_faces: int = 1, min_confidence: float = 0.5): + return generate_annotation(image, max_faces, min_confidence) diff --git a/annotator/mediapipe_face/mediapipe_face_common.py b/annotator/mediapipe_face/mediapipe_face_common.py new file mode 100644 index 0000000000000000000000000000000000000000..0f7d3701dc40eee88977f17a877fa800d0ae328d --- /dev/null +++ b/annotator/mediapipe_face/mediapipe_face_common.py @@ -0,0 +1,155 @@ +from typing import Mapping + +import mediapipe as mp +import numpy + + +mp_drawing = mp.solutions.drawing_utils +mp_drawing_styles = mp.solutions.drawing_styles +mp_face_detection = mp.solutions.face_detection # Only for counting faces. +mp_face_mesh = mp.solutions.face_mesh +mp_face_connections = mp.solutions.face_mesh_connections.FACEMESH_TESSELATION +mp_hand_connections = mp.solutions.hands_connections.HAND_CONNECTIONS +mp_body_connections = mp.solutions.pose_connections.POSE_CONNECTIONS + +DrawingSpec = mp.solutions.drawing_styles.DrawingSpec +PoseLandmark = mp.solutions.drawing_styles.PoseLandmark + +min_face_size_pixels: int = 64 +f_thick = 2 +f_rad = 1 +right_iris_draw = DrawingSpec(color=(10, 200, 250), thickness=f_thick, circle_radius=f_rad) +right_eye_draw = DrawingSpec(color=(10, 200, 180), thickness=f_thick, circle_radius=f_rad) +right_eyebrow_draw = DrawingSpec(color=(10, 220, 180), thickness=f_thick, circle_radius=f_rad) +left_iris_draw = DrawingSpec(color=(250, 200, 10), thickness=f_thick, circle_radius=f_rad) +left_eye_draw = DrawingSpec(color=(180, 200, 10), thickness=f_thick, circle_radius=f_rad) +left_eyebrow_draw = DrawingSpec(color=(180, 220, 10), thickness=f_thick, circle_radius=f_rad) +mouth_draw = DrawingSpec(color=(10, 180, 10), thickness=f_thick, circle_radius=f_rad) +head_draw = DrawingSpec(color=(10, 200, 10), thickness=f_thick, circle_radius=f_rad) + +# mp_face_mesh.FACEMESH_CONTOURS has all the items we care about. +face_connection_spec = {} +for edge in mp_face_mesh.FACEMESH_FACE_OVAL: + face_connection_spec[edge] = head_draw +for edge in mp_face_mesh.FACEMESH_LEFT_EYE: + face_connection_spec[edge] = left_eye_draw +for edge in mp_face_mesh.FACEMESH_LEFT_EYEBROW: + face_connection_spec[edge] = left_eyebrow_draw +# for edge in mp_face_mesh.FACEMESH_LEFT_IRIS: +# face_connection_spec[edge] = left_iris_draw +for edge in mp_face_mesh.FACEMESH_RIGHT_EYE: + face_connection_spec[edge] = right_eye_draw +for edge in mp_face_mesh.FACEMESH_RIGHT_EYEBROW: + face_connection_spec[edge] = right_eyebrow_draw +# for edge in mp_face_mesh.FACEMESH_RIGHT_IRIS: +# face_connection_spec[edge] = right_iris_draw +for edge in mp_face_mesh.FACEMESH_LIPS: + face_connection_spec[edge] = mouth_draw +iris_landmark_spec = {468: right_iris_draw, 473: left_iris_draw} + + +def draw_pupils(image, landmark_list, drawing_spec, halfwidth: int = 2): + """We have a custom function to draw the pupils because the mp.draw_landmarks method requires a parameter for all + landmarks. Until our PR is merged into mediapipe, we need this separate method.""" + if len(image.shape) != 3: + raise ValueError("Input image must be H,W,C.") + image_rows, image_cols, image_channels = image.shape + if image_channels != 3: # BGR channels + raise ValueError('Input image must contain three channel bgr data.') + for idx, landmark in enumerate(landmark_list.landmark): + if ( + (landmark.HasField('visibility') and landmark.visibility < 0.9) or + (landmark.HasField('presence') and landmark.presence < 0.5) + ): + continue + if landmark.x >= 1.0 or landmark.x < 0 or landmark.y >= 1.0 or landmark.y < 0: + continue + image_x = int(image_cols*landmark.x) + image_y = int(image_rows*landmark.y) + draw_color = None + if isinstance(drawing_spec, Mapping): + if drawing_spec.get(idx) is None: + continue + else: + draw_color = drawing_spec[idx].color + elif isinstance(drawing_spec, DrawingSpec): + draw_color = drawing_spec.color + image[image_y-halfwidth:image_y+halfwidth, image_x-halfwidth:image_x+halfwidth, :] = draw_color + + +def reverse_channels(image): + """Given a numpy array in RGB form, convert to BGR. Will also convert from BGR to RGB.""" + # im[:,:,::-1] is a neat hack to convert BGR to RGB by reversing the indexing order. + # im[:,:,::[2,1,0]] would also work but makes a copy of the data. + return image[:, :, ::-1] + + +def generate_annotation( + img_rgb, + max_faces: int, + min_confidence: float +): + """ + Find up to 'max_faces' inside the provided input image. + If min_face_size_pixels is provided and nonzero it will be used to filter faces that occupy less than this many + pixels in the image. + """ + with mp_face_mesh.FaceMesh( + static_image_mode=True, + max_num_faces=max_faces, + refine_landmarks=True, + min_detection_confidence=min_confidence, + ) as facemesh: + img_height, img_width, img_channels = img_rgb.shape + assert(img_channels == 3) + + results = facemesh.process(img_rgb).multi_face_landmarks + + if results is None: + print("No faces detected in controlnet image for Mediapipe face annotator.") + return numpy.zeros_like(img_rgb) + + # Filter faces that are too small + filtered_landmarks = [] + for lm in results: + landmarks = lm.landmark + face_rect = [ + landmarks[0].x, + landmarks[0].y, + landmarks[0].x, + landmarks[0].y, + ] # Left, up, right, down. + for i in range(len(landmarks)): + face_rect[0] = min(face_rect[0], landmarks[i].x) + face_rect[1] = min(face_rect[1], landmarks[i].y) + face_rect[2] = max(face_rect[2], landmarks[i].x) + face_rect[3] = max(face_rect[3], landmarks[i].y) + if min_face_size_pixels > 0: + face_width = abs(face_rect[2] - face_rect[0]) + face_height = abs(face_rect[3] - face_rect[1]) + face_width_pixels = face_width * img_width + face_height_pixels = face_height * img_height + face_size = min(face_width_pixels, face_height_pixels) + if face_size >= min_face_size_pixels: + filtered_landmarks.append(lm) + else: + filtered_landmarks.append(lm) + + # Annotations are drawn in BGR for some reason, but we don't need to flip a zero-filled image at the start. + empty = numpy.zeros_like(img_rgb) + + # Draw detected faces: + for face_landmarks in filtered_landmarks: + mp_drawing.draw_landmarks( + empty, + face_landmarks, + connections=face_connection_spec.keys(), + landmark_drawing_spec=None, + connection_drawing_spec=face_connection_spec + ) + draw_pupils(empty, face_landmarks, iris_landmark_spec, 2) + + # Flip BGR back to RGB. + empty = reverse_channels(empty).copy() + + return empty diff --git a/annotator/midas/LICENSE b/annotator/midas/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..277b5c11be103f028a8d10985139f1da10c2f08e --- /dev/null +++ b/annotator/midas/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Intel ISL (Intel Intelligent Systems Lab) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/annotator/midas/__init__.py b/annotator/midas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5030f45e6aa5eea53eace8545adc7569b5e8c2ed --- /dev/null +++ b/annotator/midas/__init__.py @@ -0,0 +1,33 @@ +# Midas Depth Estimation +# From https://github.com/isl-org/MiDaS +# MIT LICENSE + +import cv2 +import numpy as np +import torch + +from einops import rearrange +from .api import MiDaSInference + + +class MidasDetector: + def __init__(self): +# self.model = MiDaSInference(model_type="dpt_hybrid").cuda() + self.model = MiDaSInference(model_type="dpt_hybrid").cpu() + + def __call__(self, input_image): + assert input_image.ndim == 3 + image_depth = input_image + with torch.no_grad(): +# image_depth = torch.from_numpy(image_depth).float().cuda() + image_depth = torch.from_numpy(image_depth).float().cpu() + image_depth = image_depth / 127.5 - 1.0 + image_depth = rearrange(image_depth, 'h w c -> 1 c h w') + depth = self.model(image_depth)[0] + + depth -= torch.min(depth) + depth /= torch.max(depth) + depth = depth.cpu().numpy() + depth_image = (depth * 255.0).clip(0, 255).astype(np.uint8) + + return depth_image diff --git a/annotator/midas/api.py b/annotator/midas/api.py new file mode 100644 index 0000000000000000000000000000000000000000..0ddcf47aba1db4d533c7540fa984f4757f4fbd80 --- /dev/null +++ b/annotator/midas/api.py @@ -0,0 +1,169 @@ +# based on https://github.com/isl-org/MiDaS + +import cv2 +import os +import torch +import torch.nn as nn +from torchvision.transforms import Compose + +from .midas.dpt_depth import DPTDepthModel +from .midas.midas_net import MidasNet +from .midas.midas_net_custom import MidasNet_small +from .midas.transforms import Resize, NormalizeImage, PrepareForNet +from annotator.util import annotator_ckpts_path + + +ISL_PATHS = { + "dpt_large": os.path.join(annotator_ckpts_path, "dpt_large-midas-2f21e586.pt"), + "dpt_hybrid": os.path.join(annotator_ckpts_path, "dpt_hybrid-midas-501f0c75.pt"), + "midas_v21": "", + "midas_v21_small": "", +} + +remote_model_path = "https://huggingface.co./lllyasviel/Annotators/resolve/main/dpt_hybrid-midas-501f0c75.pt" + + +def disabled_train(self, mode=True): + """Overwrite model.train with this function to make sure train/eval mode + does not change anymore.""" + return self + + +def load_midas_transform(model_type): + # https://github.com/isl-org/MiDaS/blob/master/run.py + # load transform only + if model_type == "dpt_large": # DPT-Large + net_w, net_h = 384, 384 + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_hybrid": # DPT-Hybrid + net_w, net_h = 384, 384 + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "midas_v21": + net_w, net_h = 384, 384 + resize_mode = "upper_bound" + normalization = NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + + elif model_type == "midas_v21_small": + net_w, net_h = 256, 256 + resize_mode = "upper_bound" + normalization = NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + + else: + assert False, f"model_type '{model_type}' not implemented, use: --model_type large" + + transform = Compose( + [ + Resize( + net_w, + net_h, + resize_target=None, + keep_aspect_ratio=True, + ensure_multiple_of=32, + resize_method=resize_mode, + image_interpolation_method=cv2.INTER_CUBIC, + ), + normalization, + PrepareForNet(), + ] + ) + + return transform + + +def load_model(model_type): + # https://github.com/isl-org/MiDaS/blob/master/run.py + # load network + model_path = ISL_PATHS[model_type] + if model_type == "dpt_large": # DPT-Large + model = DPTDepthModel( + path=model_path, + backbone="vitl16_384", + non_negative=True, + ) + net_w, net_h = 384, 384 + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_hybrid": # DPT-Hybrid + if not os.path.exists(model_path): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(remote_model_path, model_dir=annotator_ckpts_path) + + model = DPTDepthModel( + path=model_path, + backbone="vitb_rn50_384", + non_negative=True, + ) + net_w, net_h = 384, 384 + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "midas_v21": + model = MidasNet(model_path, non_negative=True) + net_w, net_h = 384, 384 + resize_mode = "upper_bound" + normalization = NormalizeImage( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] + ) + + elif model_type == "midas_v21_small": + model = MidasNet_small(model_path, features=64, backbone="efficientnet_lite3", exportable=True, + non_negative=True, blocks={'expand': True}) + net_w, net_h = 256, 256 + resize_mode = "upper_bound" + normalization = NormalizeImage( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] + ) + + else: + print(f"model_type '{model_type}' not implemented, use: --model_type large") + assert False + + transform = Compose( + [ + Resize( + net_w, + net_h, + resize_target=None, + keep_aspect_ratio=True, + ensure_multiple_of=32, + resize_method=resize_mode, + image_interpolation_method=cv2.INTER_CUBIC, + ), + normalization, + PrepareForNet(), + ] + ) + + return model.eval(), transform + + +class MiDaSInference(nn.Module): + MODEL_TYPES_TORCH_HUB = [ + "DPT_Large", + "DPT_Hybrid", + "MiDaS_small" + ] + MODEL_TYPES_ISL = [ + "dpt_large", + "dpt_hybrid", + "midas_v21", + "midas_v21_small", + ] + + def __init__(self, model_type): + super().__init__() + assert (model_type in self.MODEL_TYPES_ISL) + model, _ = load_model(model_type) + self.model = model + self.model.train = disabled_train + + def forward(self, x): + with torch.no_grad(): + prediction = self.model(x) + return prediction + diff --git a/annotator/midas/midas/__init__.py b/annotator/midas/midas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/annotator/midas/midas/base_model.py b/annotator/midas/midas/base_model.py new file mode 100644 index 0000000000000000000000000000000000000000..5cf430239b47ec5ec07531263f26f5c24a2311cd --- /dev/null +++ b/annotator/midas/midas/base_model.py @@ -0,0 +1,16 @@ +import torch + + +class BaseModel(torch.nn.Module): + def load(self, path): + """Load model from file. + + Args: + path (str): file path + """ + parameters = torch.load(path, map_location=torch.device('cpu')) + + if "optimizer" in parameters: + parameters = parameters["model"] + + self.load_state_dict(parameters) diff --git a/annotator/midas/midas/blocks.py b/annotator/midas/midas/blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..2145d18fa98060a618536d9a64fe6589e9be4f78 --- /dev/null +++ b/annotator/midas/midas/blocks.py @@ -0,0 +1,342 @@ +import torch +import torch.nn as nn + +from .vit import ( + _make_pretrained_vitb_rn50_384, + _make_pretrained_vitl16_384, + _make_pretrained_vitb16_384, + forward_vit, +) + +def _make_encoder(backbone, features, use_pretrained, groups=1, expand=False, exportable=True, hooks=None, use_vit_only=False, use_readout="ignore",): + if backbone == "vitl16_384": + pretrained = _make_pretrained_vitl16_384( + use_pretrained, hooks=hooks, use_readout=use_readout + ) + scratch = _make_scratch( + [256, 512, 1024, 1024], features, groups=groups, expand=expand + ) # ViT-L/16 - 85.0% Top1 (backbone) + elif backbone == "vitb_rn50_384": + pretrained = _make_pretrained_vitb_rn50_384( + use_pretrained, + hooks=hooks, + use_vit_only=use_vit_only, + use_readout=use_readout, + ) + scratch = _make_scratch( + [256, 512, 768, 768], features, groups=groups, expand=expand + ) # ViT-H/16 - 85.0% Top1 (backbone) + elif backbone == "vitb16_384": + pretrained = _make_pretrained_vitb16_384( + use_pretrained, hooks=hooks, use_readout=use_readout + ) + scratch = _make_scratch( + [96, 192, 384, 768], features, groups=groups, expand=expand + ) # ViT-B/16 - 84.6% Top1 (backbone) + elif backbone == "resnext101_wsl": + pretrained = _make_pretrained_resnext101_wsl(use_pretrained) + scratch = _make_scratch([256, 512, 1024, 2048], features, groups=groups, expand=expand) # efficientnet_lite3 + elif backbone == "efficientnet_lite3": + pretrained = _make_pretrained_efficientnet_lite3(use_pretrained, exportable=exportable) + scratch = _make_scratch([32, 48, 136, 384], features, groups=groups, expand=expand) # efficientnet_lite3 + else: + print(f"Backbone '{backbone}' not implemented") + assert False + + return pretrained, scratch + + +def _make_scratch(in_shape, out_shape, groups=1, expand=False): + scratch = nn.Module() + + out_shape1 = out_shape + out_shape2 = out_shape + out_shape3 = out_shape + out_shape4 = out_shape + if expand==True: + out_shape1 = out_shape + out_shape2 = out_shape*2 + out_shape3 = out_shape*4 + out_shape4 = out_shape*8 + + scratch.layer1_rn = nn.Conv2d( + in_shape[0], out_shape1, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + scratch.layer2_rn = nn.Conv2d( + in_shape[1], out_shape2, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + scratch.layer3_rn = nn.Conv2d( + in_shape[2], out_shape3, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + scratch.layer4_rn = nn.Conv2d( + in_shape[3], out_shape4, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + + return scratch + + +def _make_pretrained_efficientnet_lite3(use_pretrained, exportable=False): + efficientnet = torch.hub.load( + "rwightman/gen-efficientnet-pytorch", + "tf_efficientnet_lite3", + pretrained=use_pretrained, + exportable=exportable + ) + return _make_efficientnet_backbone(efficientnet) + + +def _make_efficientnet_backbone(effnet): + pretrained = nn.Module() + + pretrained.layer1 = nn.Sequential( + effnet.conv_stem, effnet.bn1, effnet.act1, *effnet.blocks[0:2] + ) + pretrained.layer2 = nn.Sequential(*effnet.blocks[2:3]) + pretrained.layer3 = nn.Sequential(*effnet.blocks[3:5]) + pretrained.layer4 = nn.Sequential(*effnet.blocks[5:9]) + + return pretrained + + +def _make_resnet_backbone(resnet): + pretrained = nn.Module() + pretrained.layer1 = nn.Sequential( + resnet.conv1, resnet.bn1, resnet.relu, resnet.maxpool, resnet.layer1 + ) + + pretrained.layer2 = resnet.layer2 + pretrained.layer3 = resnet.layer3 + pretrained.layer4 = resnet.layer4 + + return pretrained + + +def _make_pretrained_resnext101_wsl(use_pretrained): + resnet = torch.hub.load("facebookresearch/WSL-Images", "resnext101_32x8d_wsl") + return _make_resnet_backbone(resnet) + + + +class Interpolate(nn.Module): + """Interpolation module. + """ + + def __init__(self, scale_factor, mode, align_corners=False): + """Init. + + Args: + scale_factor (float): scaling + mode (str): interpolation mode + """ + super(Interpolate, self).__init__() + + self.interp = nn.functional.interpolate + self.scale_factor = scale_factor + self.mode = mode + self.align_corners = align_corners + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: interpolated data + """ + + x = self.interp( + x, scale_factor=self.scale_factor, mode=self.mode, align_corners=self.align_corners + ) + + return x + + +class ResidualConvUnit(nn.Module): + """Residual convolution module. + """ + + def __init__(self, features): + """Init. + + Args: + features (int): number of features + """ + super().__init__() + + self.conv1 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True + ) + + self.conv2 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True + ) + + self.relu = nn.ReLU(inplace=True) + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: output + """ + out = self.relu(x) + out = self.conv1(out) + out = self.relu(out) + out = self.conv2(out) + + return out + x + + +class FeatureFusionBlock(nn.Module): + """Feature fusion block. + """ + + def __init__(self, features): + """Init. + + Args: + features (int): number of features + """ + super(FeatureFusionBlock, self).__init__() + + self.resConfUnit1 = ResidualConvUnit(features) + self.resConfUnit2 = ResidualConvUnit(features) + + def forward(self, *xs): + """Forward pass. + + Returns: + tensor: output + """ + output = xs[0] + + if len(xs) == 2: + output += self.resConfUnit1(xs[1]) + + output = self.resConfUnit2(output) + + output = nn.functional.interpolate( + output, scale_factor=2, mode="bilinear", align_corners=True + ) + + return output + + + + +class ResidualConvUnit_custom(nn.Module): + """Residual convolution module. + """ + + def __init__(self, features, activation, bn): + """Init. + + Args: + features (int): number of features + """ + super().__init__() + + self.bn = bn + + self.groups=1 + + self.conv1 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups + ) + + self.conv2 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups + ) + + if self.bn==True: + self.bn1 = nn.BatchNorm2d(features) + self.bn2 = nn.BatchNorm2d(features) + + self.activation = activation + + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: output + """ + + out = self.activation(x) + out = self.conv1(out) + if self.bn==True: + out = self.bn1(out) + + out = self.activation(out) + out = self.conv2(out) + if self.bn==True: + out = self.bn2(out) + + if self.groups > 1: + out = self.conv_merge(out) + + return self.skip_add.add(out, x) + + # return out + x + + +class FeatureFusionBlock_custom(nn.Module): + """Feature fusion block. + """ + + def __init__(self, features, activation, deconv=False, bn=False, expand=False, align_corners=True): + """Init. + + Args: + features (int): number of features + """ + super(FeatureFusionBlock_custom, self).__init__() + + self.deconv = deconv + self.align_corners = align_corners + + self.groups=1 + + self.expand = expand + out_features = features + if self.expand==True: + out_features = features//2 + + self.out_conv = nn.Conv2d(features, out_features, kernel_size=1, stride=1, padding=0, bias=True, groups=1) + + self.resConfUnit1 = ResidualConvUnit_custom(features, activation, bn) + self.resConfUnit2 = ResidualConvUnit_custom(features, activation, bn) + + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, *xs): + """Forward pass. + + Returns: + tensor: output + """ + output = xs[0] + + if len(xs) == 2: + res = self.resConfUnit1(xs[1]) + output = self.skip_add.add(output, res) + # output += res + + output = self.resConfUnit2(output) + + output = nn.functional.interpolate( + output, scale_factor=2, mode="bilinear", align_corners=self.align_corners + ) + + output = self.out_conv(output) + + return output + diff --git a/annotator/midas/midas/dpt_depth.py b/annotator/midas/midas/dpt_depth.py new file mode 100644 index 0000000000000000000000000000000000000000..4e9aab5d2767dffea39da5b3f30e2798688216f1 --- /dev/null +++ b/annotator/midas/midas/dpt_depth.py @@ -0,0 +1,109 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .base_model import BaseModel +from .blocks import ( + FeatureFusionBlock, + FeatureFusionBlock_custom, + Interpolate, + _make_encoder, + forward_vit, +) + + +def _make_fusion_block(features, use_bn): + return FeatureFusionBlock_custom( + features, + nn.ReLU(False), + deconv=False, + bn=use_bn, + expand=False, + align_corners=True, + ) + + +class DPT(BaseModel): + def __init__( + self, + head, + features=256, + backbone="vitb_rn50_384", + readout="project", + channels_last=False, + use_bn=False, + ): + + super(DPT, self).__init__() + + self.channels_last = channels_last + + hooks = { + "vitb_rn50_384": [0, 1, 8, 11], + "vitb16_384": [2, 5, 8, 11], + "vitl16_384": [5, 11, 17, 23], + } + + # Instantiate backbone and reassemble blocks + self.pretrained, self.scratch = _make_encoder( + backbone, + features, + False, # Set to true of you want to train from scratch, uses ImageNet weights + groups=1, + expand=False, + exportable=False, + hooks=hooks[backbone], + use_readout=readout, + ) + + self.scratch.refinenet1 = _make_fusion_block(features, use_bn) + self.scratch.refinenet2 = _make_fusion_block(features, use_bn) + self.scratch.refinenet3 = _make_fusion_block(features, use_bn) + self.scratch.refinenet4 = _make_fusion_block(features, use_bn) + + self.scratch.output_conv = head + + + def forward(self, x): + if self.channels_last == True: + x.contiguous(memory_format=torch.channels_last) + + layer_1, layer_2, layer_3, layer_4 = forward_vit(self.pretrained, x) + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + layer_4_rn = self.scratch.layer4_rn(layer_4) + + path_4 = self.scratch.refinenet4(layer_4_rn) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn) + path_2 = self.scratch.refinenet2(path_3, layer_2_rn) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + out = self.scratch.output_conv(path_1) + + return out + + +class DPTDepthModel(DPT): + def __init__(self, path=None, non_negative=True, **kwargs): + features = kwargs["features"] if "features" in kwargs else 256 + + head = nn.Sequential( + nn.Conv2d(features, features // 2, kernel_size=3, stride=1, padding=1), + Interpolate(scale_factor=2, mode="bilinear", align_corners=True), + nn.Conv2d(features // 2, 32, kernel_size=3, stride=1, padding=1), + nn.ReLU(True), + nn.Conv2d(32, 1, kernel_size=1, stride=1, padding=0), + nn.ReLU(True) if non_negative else nn.Identity(), + nn.Identity(), + ) + + super().__init__(head, **kwargs) + + if path is not None: + self.load(path) + + def forward(self, x): + return super().forward(x).squeeze(dim=1) + diff --git a/annotator/midas/midas/midas_net.py b/annotator/midas/midas/midas_net.py new file mode 100644 index 0000000000000000000000000000000000000000..8a954977800b0a0f48807e80fa63041910e33c1f --- /dev/null +++ b/annotator/midas/midas/midas_net.py @@ -0,0 +1,76 @@ +"""MidashNet: Network for monocular depth estimation trained by mixing several datasets. +This file contains code that is adapted from +https://github.com/thomasjpfan/pytorch_refinenet/blob/master/pytorch_refinenet/refinenet/refinenet_4cascade.py +""" +import torch +import torch.nn as nn + +from .base_model import BaseModel +from .blocks import FeatureFusionBlock, Interpolate, _make_encoder + + +class MidasNet(BaseModel): + """Network for monocular depth estimation. + """ + + def __init__(self, path=None, features=256, non_negative=True): + """Init. + + Args: + path (str, optional): Path to saved model. Defaults to None. + features (int, optional): Number of features. Defaults to 256. + backbone (str, optional): Backbone network for encoder. Defaults to resnet50 + """ + print("Loading weights: ", path) + + super(MidasNet, self).__init__() + + use_pretrained = False if path is None else True + + self.pretrained, self.scratch = _make_encoder(backbone="resnext101_wsl", features=features, use_pretrained=use_pretrained) + + self.scratch.refinenet4 = FeatureFusionBlock(features) + self.scratch.refinenet3 = FeatureFusionBlock(features) + self.scratch.refinenet2 = FeatureFusionBlock(features) + self.scratch.refinenet1 = FeatureFusionBlock(features) + + self.scratch.output_conv = nn.Sequential( + nn.Conv2d(features, 128, kernel_size=3, stride=1, padding=1), + Interpolate(scale_factor=2, mode="bilinear"), + nn.Conv2d(128, 32, kernel_size=3, stride=1, padding=1), + nn.ReLU(True), + nn.Conv2d(32, 1, kernel_size=1, stride=1, padding=0), + nn.ReLU(True) if non_negative else nn.Identity(), + ) + + if path: + self.load(path) + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input data (image) + + Returns: + tensor: depth + """ + + layer_1 = self.pretrained.layer1(x) + layer_2 = self.pretrained.layer2(layer_1) + layer_3 = self.pretrained.layer3(layer_2) + layer_4 = self.pretrained.layer4(layer_3) + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + layer_4_rn = self.scratch.layer4_rn(layer_4) + + path_4 = self.scratch.refinenet4(layer_4_rn) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn) + path_2 = self.scratch.refinenet2(path_3, layer_2_rn) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + out = self.scratch.output_conv(path_1) + + return torch.squeeze(out, dim=1) diff --git a/annotator/midas/midas/midas_net_custom.py b/annotator/midas/midas/midas_net_custom.py new file mode 100644 index 0000000000000000000000000000000000000000..50e4acb5e53d5fabefe3dde16ab49c33c2b7797c --- /dev/null +++ b/annotator/midas/midas/midas_net_custom.py @@ -0,0 +1,128 @@ +"""MidashNet: Network for monocular depth estimation trained by mixing several datasets. +This file contains code that is adapted from +https://github.com/thomasjpfan/pytorch_refinenet/blob/master/pytorch_refinenet/refinenet/refinenet_4cascade.py +""" +import torch +import torch.nn as nn + +from .base_model import BaseModel +from .blocks import FeatureFusionBlock, FeatureFusionBlock_custom, Interpolate, _make_encoder + + +class MidasNet_small(BaseModel): + """Network for monocular depth estimation. + """ + + def __init__(self, path=None, features=64, backbone="efficientnet_lite3", non_negative=True, exportable=True, channels_last=False, align_corners=True, + blocks={'expand': True}): + """Init. + + Args: + path (str, optional): Path to saved model. Defaults to None. + features (int, optional): Number of features. Defaults to 256. + backbone (str, optional): Backbone network for encoder. Defaults to resnet50 + """ + print("Loading weights: ", path) + + super(MidasNet_small, self).__init__() + + use_pretrained = False if path else True + + self.channels_last = channels_last + self.blocks = blocks + self.backbone = backbone + + self.groups = 1 + + features1=features + features2=features + features3=features + features4=features + self.expand = False + if "expand" in self.blocks and self.blocks['expand'] == True: + self.expand = True + features1=features + features2=features*2 + features3=features*4 + features4=features*8 + + self.pretrained, self.scratch = _make_encoder(self.backbone, features, use_pretrained, groups=self.groups, expand=self.expand, exportable=exportable) + + self.scratch.activation = nn.ReLU(False) + + self.scratch.refinenet4 = FeatureFusionBlock_custom(features4, self.scratch.activation, deconv=False, bn=False, expand=self.expand, align_corners=align_corners) + self.scratch.refinenet3 = FeatureFusionBlock_custom(features3, self.scratch.activation, deconv=False, bn=False, expand=self.expand, align_corners=align_corners) + self.scratch.refinenet2 = FeatureFusionBlock_custom(features2, self.scratch.activation, deconv=False, bn=False, expand=self.expand, align_corners=align_corners) + self.scratch.refinenet1 = FeatureFusionBlock_custom(features1, self.scratch.activation, deconv=False, bn=False, align_corners=align_corners) + + + self.scratch.output_conv = nn.Sequential( + nn.Conv2d(features, features//2, kernel_size=3, stride=1, padding=1, groups=self.groups), + Interpolate(scale_factor=2, mode="bilinear"), + nn.Conv2d(features//2, 32, kernel_size=3, stride=1, padding=1), + self.scratch.activation, + nn.Conv2d(32, 1, kernel_size=1, stride=1, padding=0), + nn.ReLU(True) if non_negative else nn.Identity(), + nn.Identity(), + ) + + if path: + self.load(path) + + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input data (image) + + Returns: + tensor: depth + """ + if self.channels_last==True: + print("self.channels_last = ", self.channels_last) + x.contiguous(memory_format=torch.channels_last) + + + layer_1 = self.pretrained.layer1(x) + layer_2 = self.pretrained.layer2(layer_1) + layer_3 = self.pretrained.layer3(layer_2) + layer_4 = self.pretrained.layer4(layer_3) + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + layer_4_rn = self.scratch.layer4_rn(layer_4) + + + path_4 = self.scratch.refinenet4(layer_4_rn) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn) + path_2 = self.scratch.refinenet2(path_3, layer_2_rn) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + out = self.scratch.output_conv(path_1) + + return torch.squeeze(out, dim=1) + + + +def fuse_model(m): + prev_previous_type = nn.Identity() + prev_previous_name = '' + previous_type = nn.Identity() + previous_name = '' + for name, module in m.named_modules(): + if prev_previous_type == nn.Conv2d and previous_type == nn.BatchNorm2d and type(module) == nn.ReLU: + # print("FUSED ", prev_previous_name, previous_name, name) + torch.quantization.fuse_modules(m, [prev_previous_name, previous_name, name], inplace=True) + elif prev_previous_type == nn.Conv2d and previous_type == nn.BatchNorm2d: + # print("FUSED ", prev_previous_name, previous_name) + torch.quantization.fuse_modules(m, [prev_previous_name, previous_name], inplace=True) + # elif previous_type == nn.Conv2d and type(module) == nn.ReLU: + # print("FUSED ", previous_name, name) + # torch.quantization.fuse_modules(m, [previous_name, name], inplace=True) + + prev_previous_type = previous_type + prev_previous_name = previous_name + previous_type = type(module) + previous_name = name \ No newline at end of file diff --git a/annotator/midas/midas/transforms.py b/annotator/midas/midas/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..350cbc11662633ad7f8968eb10be2e7de6e384e9 --- /dev/null +++ b/annotator/midas/midas/transforms.py @@ -0,0 +1,234 @@ +import numpy as np +import cv2 +import math + + +def apply_min_size(sample, size, image_interpolation_method=cv2.INTER_AREA): + """Rezise the sample to ensure the given size. Keeps aspect ratio. + + Args: + sample (dict): sample + size (tuple): image size + + Returns: + tuple: new size + """ + shape = list(sample["disparity"].shape) + + if shape[0] >= size[0] and shape[1] >= size[1]: + return sample + + scale = [0, 0] + scale[0] = size[0] / shape[0] + scale[1] = size[1] / shape[1] + + scale = max(scale) + + shape[0] = math.ceil(scale * shape[0]) + shape[1] = math.ceil(scale * shape[1]) + + # resize + sample["image"] = cv2.resize( + sample["image"], tuple(shape[::-1]), interpolation=image_interpolation_method + ) + + sample["disparity"] = cv2.resize( + sample["disparity"], tuple(shape[::-1]), interpolation=cv2.INTER_NEAREST + ) + sample["mask"] = cv2.resize( + sample["mask"].astype(np.float32), + tuple(shape[::-1]), + interpolation=cv2.INTER_NEAREST, + ) + sample["mask"] = sample["mask"].astype(bool) + + return tuple(shape) + + +class Resize(object): + """Resize sample to given size (width, height). + """ + + def __init__( + self, + width, + height, + resize_target=True, + keep_aspect_ratio=False, + ensure_multiple_of=1, + resize_method="lower_bound", + image_interpolation_method=cv2.INTER_AREA, + ): + """Init. + + Args: + width (int): desired output width + height (int): desired output height + resize_target (bool, optional): + True: Resize the full sample (image, mask, target). + False: Resize image only. + Defaults to True. + keep_aspect_ratio (bool, optional): + True: Keep the aspect ratio of the input sample. + Output sample might not have the given width and height, and + resize behaviour depends on the parameter 'resize_method'. + Defaults to False. + ensure_multiple_of (int, optional): + Output width and height is constrained to be multiple of this parameter. + Defaults to 1. + resize_method (str, optional): + "lower_bound": Output will be at least as large as the given size. + "upper_bound": Output will be at max as large as the given size. (Output size might be smaller than given size.) + "minimal": Scale as least as possible. (Output size might be smaller than given size.) + Defaults to "lower_bound". + """ + self.__width = width + self.__height = height + + self.__resize_target = resize_target + self.__keep_aspect_ratio = keep_aspect_ratio + self.__multiple_of = ensure_multiple_of + self.__resize_method = resize_method + self.__image_interpolation_method = image_interpolation_method + + def constrain_to_multiple_of(self, x, min_val=0, max_val=None): + y = (np.round(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if max_val is not None and y > max_val: + y = (np.floor(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if y < min_val: + y = (np.ceil(x / self.__multiple_of) * self.__multiple_of).astype(int) + + return y + + def get_size(self, width, height): + # determine new height and width + scale_height = self.__height / height + scale_width = self.__width / width + + if self.__keep_aspect_ratio: + if self.__resize_method == "lower_bound": + # scale such that output size is lower bound + if scale_width > scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "upper_bound": + # scale such that output size is upper bound + if scale_width < scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "minimal": + # scale as least as possbile + if abs(1 - scale_width) < abs(1 - scale_height): + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + else: + raise ValueError( + f"resize_method {self.__resize_method} not implemented" + ) + + if self.__resize_method == "lower_bound": + new_height = self.constrain_to_multiple_of( + scale_height * height, min_val=self.__height + ) + new_width = self.constrain_to_multiple_of( + scale_width * width, min_val=self.__width + ) + elif self.__resize_method == "upper_bound": + new_height = self.constrain_to_multiple_of( + scale_height * height, max_val=self.__height + ) + new_width = self.constrain_to_multiple_of( + scale_width * width, max_val=self.__width + ) + elif self.__resize_method == "minimal": + new_height = self.constrain_to_multiple_of(scale_height * height) + new_width = self.constrain_to_multiple_of(scale_width * width) + else: + raise ValueError(f"resize_method {self.__resize_method} not implemented") + + return (new_width, new_height) + + def __call__(self, sample): + width, height = self.get_size( + sample["image"].shape[1], sample["image"].shape[0] + ) + + # resize sample + sample["image"] = cv2.resize( + sample["image"], + (width, height), + interpolation=self.__image_interpolation_method, + ) + + if self.__resize_target: + if "disparity" in sample: + sample["disparity"] = cv2.resize( + sample["disparity"], + (width, height), + interpolation=cv2.INTER_NEAREST, + ) + + if "depth" in sample: + sample["depth"] = cv2.resize( + sample["depth"], (width, height), interpolation=cv2.INTER_NEAREST + ) + + sample["mask"] = cv2.resize( + sample["mask"].astype(np.float32), + (width, height), + interpolation=cv2.INTER_NEAREST, + ) + sample["mask"] = sample["mask"].astype(bool) + + return sample + + +class NormalizeImage(object): + """Normlize image by given mean and std. + """ + + def __init__(self, mean, std): + self.__mean = mean + self.__std = std + + def __call__(self, sample): + sample["image"] = (sample["image"] - self.__mean) / self.__std + + return sample + + +class PrepareForNet(object): + """Prepare sample for usage as network input. + """ + + def __init__(self): + pass + + def __call__(self, sample): + image = np.transpose(sample["image"], (2, 0, 1)) + sample["image"] = np.ascontiguousarray(image).astype(np.float32) + + if "mask" in sample: + sample["mask"] = sample["mask"].astype(np.float32) + sample["mask"] = np.ascontiguousarray(sample["mask"]) + + if "disparity" in sample: + disparity = sample["disparity"].astype(np.float32) + sample["disparity"] = np.ascontiguousarray(disparity) + + if "depth" in sample: + depth = sample["depth"].astype(np.float32) + sample["depth"] = np.ascontiguousarray(depth) + + return sample diff --git a/annotator/midas/midas/vit.py b/annotator/midas/midas/vit.py new file mode 100644 index 0000000000000000000000000000000000000000..ea46b1be88b261b0dec04f3da0256f5f66f88a74 --- /dev/null +++ b/annotator/midas/midas/vit.py @@ -0,0 +1,491 @@ +import torch +import torch.nn as nn +import timm +import types +import math +import torch.nn.functional as F + + +class Slice(nn.Module): + def __init__(self, start_index=1): + super(Slice, self).__init__() + self.start_index = start_index + + def forward(self, x): + return x[:, self.start_index :] + + +class AddReadout(nn.Module): + def __init__(self, start_index=1): + super(AddReadout, self).__init__() + self.start_index = start_index + + def forward(self, x): + if self.start_index == 2: + readout = (x[:, 0] + x[:, 1]) / 2 + else: + readout = x[:, 0] + return x[:, self.start_index :] + readout.unsqueeze(1) + + +class ProjectReadout(nn.Module): + def __init__(self, in_features, start_index=1): + super(ProjectReadout, self).__init__() + self.start_index = start_index + + self.project = nn.Sequential(nn.Linear(2 * in_features, in_features), nn.GELU()) + + def forward(self, x): + readout = x[:, 0].unsqueeze(1).expand_as(x[:, self.start_index :]) + features = torch.cat((x[:, self.start_index :], readout), -1) + + return self.project(features) + + +class Transpose(nn.Module): + def __init__(self, dim0, dim1): + super(Transpose, self).__init__() + self.dim0 = dim0 + self.dim1 = dim1 + + def forward(self, x): + x = x.transpose(self.dim0, self.dim1) + return x + + +def forward_vit(pretrained, x): + b, c, h, w = x.shape + + glob = pretrained.model.forward_flex(x) + + layer_1 = pretrained.activations["1"] + layer_2 = pretrained.activations["2"] + layer_3 = pretrained.activations["3"] + layer_4 = pretrained.activations["4"] + + layer_1 = pretrained.act_postprocess1[0:2](layer_1) + layer_2 = pretrained.act_postprocess2[0:2](layer_2) + layer_3 = pretrained.act_postprocess3[0:2](layer_3) + layer_4 = pretrained.act_postprocess4[0:2](layer_4) + + unflatten = nn.Sequential( + nn.Unflatten( + 2, + torch.Size( + [ + h // pretrained.model.patch_size[1], + w // pretrained.model.patch_size[0], + ] + ), + ) + ) + + if layer_1.ndim == 3: + layer_1 = unflatten(layer_1) + if layer_2.ndim == 3: + layer_2 = unflatten(layer_2) + if layer_3.ndim == 3: + layer_3 = unflatten(layer_3) + if layer_4.ndim == 3: + layer_4 = unflatten(layer_4) + + layer_1 = pretrained.act_postprocess1[3 : len(pretrained.act_postprocess1)](layer_1) + layer_2 = pretrained.act_postprocess2[3 : len(pretrained.act_postprocess2)](layer_2) + layer_3 = pretrained.act_postprocess3[3 : len(pretrained.act_postprocess3)](layer_3) + layer_4 = pretrained.act_postprocess4[3 : len(pretrained.act_postprocess4)](layer_4) + + return layer_1, layer_2, layer_3, layer_4 + + +def _resize_pos_embed(self, posemb, gs_h, gs_w): + posemb_tok, posemb_grid = ( + posemb[:, : self.start_index], + posemb[0, self.start_index :], + ) + + gs_old = int(math.sqrt(len(posemb_grid))) + + posemb_grid = posemb_grid.reshape(1, gs_old, gs_old, -1).permute(0, 3, 1, 2) + posemb_grid = F.interpolate(posemb_grid, size=(gs_h, gs_w), mode="bilinear") + posemb_grid = posemb_grid.permute(0, 2, 3, 1).reshape(1, gs_h * gs_w, -1) + + posemb = torch.cat([posemb_tok, posemb_grid], dim=1) + + return posemb + + +def forward_flex(self, x): + b, c, h, w = x.shape + + pos_embed = self._resize_pos_embed( + self.pos_embed, h // self.patch_size[1], w // self.patch_size[0] + ) + + B = x.shape[0] + + if hasattr(self.patch_embed, "backbone"): + x = self.patch_embed.backbone(x) + if isinstance(x, (list, tuple)): + x = x[-1] # last feature if backbone outputs list/tuple of features + + x = self.patch_embed.proj(x).flatten(2).transpose(1, 2) + + if getattr(self, "dist_token", None) is not None: + cls_tokens = self.cls_token.expand( + B, -1, -1 + ) # stole cls_tokens impl from Phil Wang, thanks + dist_token = self.dist_token.expand(B, -1, -1) + x = torch.cat((cls_tokens, dist_token, x), dim=1) + else: + cls_tokens = self.cls_token.expand( + B, -1, -1 + ) # stole cls_tokens impl from Phil Wang, thanks + x = torch.cat((cls_tokens, x), dim=1) + + x = x + pos_embed + x = self.pos_drop(x) + + for blk in self.blocks: + x = blk(x) + + x = self.norm(x) + + return x + + +activations = {} + + +def get_activation(name): + def hook(model, input, output): + activations[name] = output + + return hook + + +def get_readout_oper(vit_features, features, use_readout, start_index=1): + if use_readout == "ignore": + readout_oper = [Slice(start_index)] * len(features) + elif use_readout == "add": + readout_oper = [AddReadout(start_index)] * len(features) + elif use_readout == "project": + readout_oper = [ + ProjectReadout(vit_features, start_index) for out_feat in features + ] + else: + assert ( + False + ), "wrong operation for readout token, use_readout can be 'ignore', 'add', or 'project'" + + return readout_oper + + +def _make_vit_b16_backbone( + model, + features=[96, 192, 384, 768], + size=[384, 384], + hooks=[2, 5, 8, 11], + vit_features=768, + use_readout="ignore", + start_index=1, +): + pretrained = nn.Module() + + pretrained.model = model + pretrained.model.blocks[hooks[0]].register_forward_hook(get_activation("1")) + pretrained.model.blocks[hooks[1]].register_forward_hook(get_activation("2")) + pretrained.model.blocks[hooks[2]].register_forward_hook(get_activation("3")) + pretrained.model.blocks[hooks[3]].register_forward_hook(get_activation("4")) + + pretrained.activations = activations + + readout_oper = get_readout_oper(vit_features, features, use_readout, start_index) + + # 32, 48, 136, 384 + pretrained.act_postprocess1 = nn.Sequential( + readout_oper[0], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[0], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[0], + out_channels=features[0], + kernel_size=4, + stride=4, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess2 = nn.Sequential( + readout_oper[1], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[1], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[1], + out_channels=features[1], + kernel_size=2, + stride=2, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess3 = nn.Sequential( + readout_oper[2], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[2], + kernel_size=1, + stride=1, + padding=0, + ), + ) + + pretrained.act_postprocess4 = nn.Sequential( + readout_oper[3], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[3], + kernel_size=1, + stride=1, + padding=0, + ), + nn.Conv2d( + in_channels=features[3], + out_channels=features[3], + kernel_size=3, + stride=2, + padding=1, + ), + ) + + pretrained.model.start_index = start_index + pretrained.model.patch_size = [16, 16] + + # We inject this function into the VisionTransformer instances so that + # we can use it with interpolated position embeddings without modifying the library source. + pretrained.model.forward_flex = types.MethodType(forward_flex, pretrained.model) + pretrained.model._resize_pos_embed = types.MethodType( + _resize_pos_embed, pretrained.model + ) + + return pretrained + + +def _make_pretrained_vitl16_384(pretrained, use_readout="ignore", hooks=None): + model = timm.create_model("vit_large_patch16_384", pretrained=pretrained) + + hooks = [5, 11, 17, 23] if hooks == None else hooks + return _make_vit_b16_backbone( + model, + features=[256, 512, 1024, 1024], + hooks=hooks, + vit_features=1024, + use_readout=use_readout, + ) + + +def _make_pretrained_vitb16_384(pretrained, use_readout="ignore", hooks=None): + model = timm.create_model("vit_base_patch16_384", pretrained=pretrained) + + hooks = [2, 5, 8, 11] if hooks == None else hooks + return _make_vit_b16_backbone( + model, features=[96, 192, 384, 768], hooks=hooks, use_readout=use_readout + ) + + +def _make_pretrained_deitb16_384(pretrained, use_readout="ignore", hooks=None): + model = timm.create_model("vit_deit_base_patch16_384", pretrained=pretrained) + + hooks = [2, 5, 8, 11] if hooks == None else hooks + return _make_vit_b16_backbone( + model, features=[96, 192, 384, 768], hooks=hooks, use_readout=use_readout + ) + + +def _make_pretrained_deitb16_distil_384(pretrained, use_readout="ignore", hooks=None): + model = timm.create_model( + "vit_deit_base_distilled_patch16_384", pretrained=pretrained + ) + + hooks = [2, 5, 8, 11] if hooks == None else hooks + return _make_vit_b16_backbone( + model, + features=[96, 192, 384, 768], + hooks=hooks, + use_readout=use_readout, + start_index=2, + ) + + +def _make_vit_b_rn50_backbone( + model, + features=[256, 512, 768, 768], + size=[384, 384], + hooks=[0, 1, 8, 11], + vit_features=768, + use_vit_only=False, + use_readout="ignore", + start_index=1, +): + pretrained = nn.Module() + + pretrained.model = model + + if use_vit_only == True: + pretrained.model.blocks[hooks[0]].register_forward_hook(get_activation("1")) + pretrained.model.blocks[hooks[1]].register_forward_hook(get_activation("2")) + else: + pretrained.model.patch_embed.backbone.stages[0].register_forward_hook( + get_activation("1") + ) + pretrained.model.patch_embed.backbone.stages[1].register_forward_hook( + get_activation("2") + ) + + pretrained.model.blocks[hooks[2]].register_forward_hook(get_activation("3")) + pretrained.model.blocks[hooks[3]].register_forward_hook(get_activation("4")) + + pretrained.activations = activations + + readout_oper = get_readout_oper(vit_features, features, use_readout, start_index) + + if use_vit_only == True: + pretrained.act_postprocess1 = nn.Sequential( + readout_oper[0], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[0], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[0], + out_channels=features[0], + kernel_size=4, + stride=4, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess2 = nn.Sequential( + readout_oper[1], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[1], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[1], + out_channels=features[1], + kernel_size=2, + stride=2, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + else: + pretrained.act_postprocess1 = nn.Sequential( + nn.Identity(), nn.Identity(), nn.Identity() + ) + pretrained.act_postprocess2 = nn.Sequential( + nn.Identity(), nn.Identity(), nn.Identity() + ) + + pretrained.act_postprocess3 = nn.Sequential( + readout_oper[2], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[2], + kernel_size=1, + stride=1, + padding=0, + ), + ) + + pretrained.act_postprocess4 = nn.Sequential( + readout_oper[3], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[3], + kernel_size=1, + stride=1, + padding=0, + ), + nn.Conv2d( + in_channels=features[3], + out_channels=features[3], + kernel_size=3, + stride=2, + padding=1, + ), + ) + + pretrained.model.start_index = start_index + pretrained.model.patch_size = [16, 16] + + # We inject this function into the VisionTransformer instances so that + # we can use it with interpolated position embeddings without modifying the library source. + pretrained.model.forward_flex = types.MethodType(forward_flex, pretrained.model) + + # We inject this function into the VisionTransformer instances so that + # we can use it with interpolated position embeddings without modifying the library source. + pretrained.model._resize_pos_embed = types.MethodType( + _resize_pos_embed, pretrained.model + ) + + return pretrained + + +def _make_pretrained_vitb_rn50_384( + pretrained, use_readout="ignore", hooks=None, use_vit_only=False +): + model = timm.create_model("vit_base_resnet50_384", pretrained=pretrained) + + hooks = [0, 1, 8, 11] if hooks == None else hooks + return _make_vit_b_rn50_backbone( + model, + features=[256, 512, 768, 768], + size=[384, 384], + hooks=hooks, + use_vit_only=use_vit_only, + use_readout=use_readout, + ) diff --git a/annotator/midas/utils.py b/annotator/midas/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9a9d3b5b66370fa98da9e067ba53ead848ea9a59 --- /dev/null +++ b/annotator/midas/utils.py @@ -0,0 +1,189 @@ +"""Utils for monoDepth.""" +import sys +import re +import numpy as np +import cv2 +import torch + + +def read_pfm(path): + """Read pfm file. + + Args: + path (str): path to file + + Returns: + tuple: (data, scale) + """ + with open(path, "rb") as file: + + color = None + width = None + height = None + scale = None + endian = None + + header = file.readline().rstrip() + if header.decode("ascii") == "PF": + color = True + elif header.decode("ascii") == "Pf": + color = False + else: + raise Exception("Not a PFM file: " + path) + + dim_match = re.match(r"^(\d+)\s(\d+)\s$", file.readline().decode("ascii")) + if dim_match: + width, height = list(map(int, dim_match.groups())) + else: + raise Exception("Malformed PFM header.") + + scale = float(file.readline().decode("ascii").rstrip()) + if scale < 0: + # little-endian + endian = "<" + scale = -scale + else: + # big-endian + endian = ">" + + data = np.fromfile(file, endian + "f") + shape = (height, width, 3) if color else (height, width) + + data = np.reshape(data, shape) + data = np.flipud(data) + + return data, scale + + +def write_pfm(path, image, scale=1): + """Write pfm file. + + Args: + path (str): pathto file + image (array): data + scale (int, optional): Scale. Defaults to 1. + """ + + with open(path, "wb") as file: + color = None + + if image.dtype.name != "float32": + raise Exception("Image dtype must be float32.") + + image = np.flipud(image) + + if len(image.shape) == 3 and image.shape[2] == 3: # color image + color = True + elif ( + len(image.shape) == 2 or len(image.shape) == 3 and image.shape[2] == 1 + ): # greyscale + color = False + else: + raise Exception("Image must have H x W x 3, H x W x 1 or H x W dimensions.") + + file.write("PF\n" if color else "Pf\n".encode()) + file.write("%d %d\n".encode() % (image.shape[1], image.shape[0])) + + endian = image.dtype.byteorder + + if endian == "<" or endian == "=" and sys.byteorder == "little": + scale = -scale + + file.write("%f\n".encode() % scale) + + image.tofile(file) + + +def read_image(path): + """Read image and output RGB image (0-1). + + Args: + path (str): path to file + + Returns: + array: RGB image (0-1) + """ + img = cv2.imread(path) + + if img.ndim == 2: + img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) + + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) / 255.0 + + return img + + +def resize_image(img): + """Resize image and make it fit for network. + + Args: + img (array): image + + Returns: + tensor: data ready for network + """ + height_orig = img.shape[0] + width_orig = img.shape[1] + + if width_orig > height_orig: + scale = width_orig / 384 + else: + scale = height_orig / 384 + + height = (np.ceil(height_orig / scale / 32) * 32).astype(int) + width = (np.ceil(width_orig / scale / 32) * 32).astype(int) + + img_resized = cv2.resize(img, (width, height), interpolation=cv2.INTER_AREA) + + img_resized = ( + torch.from_numpy(np.transpose(img_resized, (2, 0, 1))).contiguous().float() + ) + img_resized = img_resized.unsqueeze(0) + + return img_resized + + +def resize_depth(depth, width, height): + """Resize depth map and bring to CPU (numpy). + + Args: + depth (tensor): depth + width (int): image width + height (int): image height + + Returns: + array: processed depth + """ + depth = torch.squeeze(depth[0, :, :, :]).to("cpu") + + depth_resized = cv2.resize( + depth.numpy(), (width, height), interpolation=cv2.INTER_CUBIC + ) + + return depth_resized + +def write_depth(path, depth, bits=1): + """Write depth map to pfm and png file. + + Args: + path (str): filepath without extension + depth (array): depth + """ + write_pfm(path + ".pfm", depth.astype(np.float32)) + + depth_min = depth.min() + depth_max = depth.max() + + max_val = (2**(8*bits))-1 + + if depth_max - depth_min > np.finfo("float").eps: + out = max_val * (depth - depth_min) / (depth_max - depth_min) + else: + out = np.zeros(depth.shape, dtype=depth.type) + + if bits == 1: + cv2.imwrite(path + ".png", out.astype("uint8")) + elif bits == 2: + cv2.imwrite(path + ".png", out.astype("uint16")) + + return diff --git a/annotator/mlsd/LICENSE b/annotator/mlsd/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..d855c6db44b4e873eedd750d34fa2eaf22e22363 --- /dev/null +++ b/annotator/mlsd/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2021-present NAVER Corp. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/annotator/mlsd/__init__.py b/annotator/mlsd/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d01b379230ae3a35e229557c98d2917a53d8e581 --- /dev/null +++ b/annotator/mlsd/__init__.py @@ -0,0 +1,45 @@ +# MLSD Line Detection +# From https://github.com/navervision/mlsd +# Apache-2.0 license + +import cv2 +import numpy as np +import torch +import os + +from einops import rearrange +from .models.mbv2_mlsd_tiny import MobileV2_MLSD_Tiny +from .models.mbv2_mlsd_large import MobileV2_MLSD_Large +from .utils import pred_lines + +from annotator.util import annotator_ckpts_path + + +remote_model_path = "https://huggingface.co./lllyasviel/Annotators/resolve/main/mlsd_large_512_fp32.pth" + + +class MLSDdetector: + def __init__(self): + model_path = os.path.join(annotator_ckpts_path, "mlsd_large_512_fp32.pth") + if not os.path.exists(model_path): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(remote_model_path, model_dir=annotator_ckpts_path) + model = MobileV2_MLSD_Large() +# model.load_state_dict(torch.load(model_path), strict=True) +# self.model = model.cuda().eval() + model.load_state_dict(torch.load(model_path, map_location=torch.device('cpu')), strict=True) + self.model = model.cpu().eval() + + def __call__(self, input_image, thr_v, thr_d): + assert input_image.ndim == 3 + img = input_image + img_output = np.zeros_like(img) + try: + with torch.no_grad(): + lines = pred_lines(img, self.model, [img.shape[0], img.shape[1]], thr_v, thr_d) + for line in lines: + x_start, y_start, x_end, y_end = [int(val) for val in line] + cv2.line(img_output, (x_start, y_start), (x_end, y_end), [255, 255, 255], 1) + except Exception as e: + pass + return img_output[:, :, 0] diff --git a/annotator/mlsd/models/mbv2_mlsd_large.py b/annotator/mlsd/models/mbv2_mlsd_large.py new file mode 100644 index 0000000000000000000000000000000000000000..5b9799e7573ca41549b3c3b13ac47b906b369603 --- /dev/null +++ b/annotator/mlsd/models/mbv2_mlsd_large.py @@ -0,0 +1,292 @@ +import os +import sys +import torch +import torch.nn as nn +import torch.utils.model_zoo as model_zoo +from torch.nn import functional as F + + +class BlockTypeA(nn.Module): + def __init__(self, in_c1, in_c2, out_c1, out_c2, upscale = True): + super(BlockTypeA, self).__init__() + self.conv1 = nn.Sequential( + nn.Conv2d(in_c2, out_c2, kernel_size=1), + nn.BatchNorm2d(out_c2), + nn.ReLU(inplace=True) + ) + self.conv2 = nn.Sequential( + nn.Conv2d(in_c1, out_c1, kernel_size=1), + nn.BatchNorm2d(out_c1), + nn.ReLU(inplace=True) + ) + self.upscale = upscale + + def forward(self, a, b): + b = self.conv1(b) + a = self.conv2(a) + if self.upscale: + b = F.interpolate(b, scale_factor=2.0, mode='bilinear', align_corners=True) + return torch.cat((a, b), dim=1) + + +class BlockTypeB(nn.Module): + def __init__(self, in_c, out_c): + super(BlockTypeB, self).__init__() + self.conv1 = nn.Sequential( + nn.Conv2d(in_c, in_c, kernel_size=3, padding=1), + nn.BatchNorm2d(in_c), + nn.ReLU() + ) + self.conv2 = nn.Sequential( + nn.Conv2d(in_c, out_c, kernel_size=3, padding=1), + nn.BatchNorm2d(out_c), + nn.ReLU() + ) + + def forward(self, x): + x = self.conv1(x) + x + x = self.conv2(x) + return x + +class BlockTypeC(nn.Module): + def __init__(self, in_c, out_c): + super(BlockTypeC, self).__init__() + self.conv1 = nn.Sequential( + nn.Conv2d(in_c, in_c, kernel_size=3, padding=5, dilation=5), + nn.BatchNorm2d(in_c), + nn.ReLU() + ) + self.conv2 = nn.Sequential( + nn.Conv2d(in_c, in_c, kernel_size=3, padding=1), + nn.BatchNorm2d(in_c), + nn.ReLU() + ) + self.conv3 = nn.Conv2d(in_c, out_c, kernel_size=1) + + def forward(self, x): + x = self.conv1(x) + x = self.conv2(x) + x = self.conv3(x) + return x + +def _make_divisible(v, divisor, min_value=None): + """ + This function is taken from the original tf repo. + It ensures that all layers have a channel number that is divisible by 8 + It can be seen here: + https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py + :param v: + :param divisor: + :param min_value: + :return: + """ + if min_value is None: + min_value = divisor + new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_v < 0.9 * v: + new_v += divisor + return new_v + + +class ConvBNReLU(nn.Sequential): + def __init__(self, in_planes, out_planes, kernel_size=3, stride=1, groups=1): + self.channel_pad = out_planes - in_planes + self.stride = stride + #padding = (kernel_size - 1) // 2 + + # TFLite uses slightly different padding than PyTorch + if stride == 2: + padding = 0 + else: + padding = (kernel_size - 1) // 2 + + super(ConvBNReLU, self).__init__( + nn.Conv2d(in_planes, out_planes, kernel_size, stride, padding, groups=groups, bias=False), + nn.BatchNorm2d(out_planes), + nn.ReLU6(inplace=True) + ) + self.max_pool = nn.MaxPool2d(kernel_size=stride, stride=stride) + + + def forward(self, x): + # TFLite uses different padding + if self.stride == 2: + x = F.pad(x, (0, 1, 0, 1), "constant", 0) + #print(x.shape) + + for module in self: + if not isinstance(module, nn.MaxPool2d): + x = module(x) + return x + + +class InvertedResidual(nn.Module): + def __init__(self, inp, oup, stride, expand_ratio): + super(InvertedResidual, self).__init__() + self.stride = stride + assert stride in [1, 2] + + hidden_dim = int(round(inp * expand_ratio)) + self.use_res_connect = self.stride == 1 and inp == oup + + layers = [] + if expand_ratio != 1: + # pw + layers.append(ConvBNReLU(inp, hidden_dim, kernel_size=1)) + layers.extend([ + # dw + ConvBNReLU(hidden_dim, hidden_dim, stride=stride, groups=hidden_dim), + # pw-linear + nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + ]) + self.conv = nn.Sequential(*layers) + + def forward(self, x): + if self.use_res_connect: + return x + self.conv(x) + else: + return self.conv(x) + + +class MobileNetV2(nn.Module): + def __init__(self, pretrained=True): + """ + MobileNet V2 main class + Args: + num_classes (int): Number of classes + width_mult (float): Width multiplier - adjusts number of channels in each layer by this amount + inverted_residual_setting: Network structure + round_nearest (int): Round the number of channels in each layer to be a multiple of this number + Set to 1 to turn off rounding + block: Module specifying inverted residual building block for mobilenet + """ + super(MobileNetV2, self).__init__() + + block = InvertedResidual + input_channel = 32 + last_channel = 1280 + width_mult = 1.0 + round_nearest = 8 + + inverted_residual_setting = [ + # t, c, n, s + [1, 16, 1, 1], + [6, 24, 2, 2], + [6, 32, 3, 2], + [6, 64, 4, 2], + [6, 96, 3, 1], + #[6, 160, 3, 2], + #[6, 320, 1, 1], + ] + + # only check the first element, assuming user knows t,c,n,s are required + if len(inverted_residual_setting) == 0 or len(inverted_residual_setting[0]) != 4: + raise ValueError("inverted_residual_setting should be non-empty " + "or a 4-element list, got {}".format(inverted_residual_setting)) + + # building first layer + input_channel = _make_divisible(input_channel * width_mult, round_nearest) + self.last_channel = _make_divisible(last_channel * max(1.0, width_mult), round_nearest) + features = [ConvBNReLU(4, input_channel, stride=2)] + # building inverted residual blocks + for t, c, n, s in inverted_residual_setting: + output_channel = _make_divisible(c * width_mult, round_nearest) + for i in range(n): + stride = s if i == 0 else 1 + features.append(block(input_channel, output_channel, stride, expand_ratio=t)) + input_channel = output_channel + + self.features = nn.Sequential(*features) + self.fpn_selected = [1, 3, 6, 10, 13] + # weight initialization + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out') + if m.bias is not None: + nn.init.zeros_(m.bias) + elif isinstance(m, nn.BatchNorm2d): + nn.init.ones_(m.weight) + nn.init.zeros_(m.bias) + elif isinstance(m, nn.Linear): + nn.init.normal_(m.weight, 0, 0.01) + nn.init.zeros_(m.bias) + if pretrained: + self._load_pretrained_model() + + def _forward_impl(self, x): + # This exists since TorchScript doesn't support inheritance, so the superclass method + # (this one) needs to have a name other than `forward` that can be accessed in a subclass + fpn_features = [] + for i, f in enumerate(self.features): + if i > self.fpn_selected[-1]: + break + x = f(x) + if i in self.fpn_selected: + fpn_features.append(x) + + c1, c2, c3, c4, c5 = fpn_features + return c1, c2, c3, c4, c5 + + + def forward(self, x): + return self._forward_impl(x) + + def _load_pretrained_model(self): + pretrain_dict = model_zoo.load_url('https://download.pytorch.org/models/mobilenet_v2-b0353104.pth') + model_dict = {} + state_dict = self.state_dict() + for k, v in pretrain_dict.items(): + if k in state_dict: + model_dict[k] = v + state_dict.update(model_dict) + self.load_state_dict(state_dict) + + +class MobileV2_MLSD_Large(nn.Module): + def __init__(self): + super(MobileV2_MLSD_Large, self).__init__() + + self.backbone = MobileNetV2(pretrained=False) + ## A, B + self.block15 = BlockTypeA(in_c1= 64, in_c2= 96, + out_c1= 64, out_c2=64, + upscale=False) + self.block16 = BlockTypeB(128, 64) + + ## A, B + self.block17 = BlockTypeA(in_c1 = 32, in_c2 = 64, + out_c1= 64, out_c2= 64) + self.block18 = BlockTypeB(128, 64) + + ## A, B + self.block19 = BlockTypeA(in_c1=24, in_c2=64, + out_c1=64, out_c2=64) + self.block20 = BlockTypeB(128, 64) + + ## A, B, C + self.block21 = BlockTypeA(in_c1=16, in_c2=64, + out_c1=64, out_c2=64) + self.block22 = BlockTypeB(128, 64) + + self.block23 = BlockTypeC(64, 16) + + def forward(self, x): + c1, c2, c3, c4, c5 = self.backbone(x) + + x = self.block15(c4, c5) + x = self.block16(x) + + x = self.block17(c3, x) + x = self.block18(x) + + x = self.block19(c2, x) + x = self.block20(x) + + x = self.block21(c1, x) + x = self.block22(x) + x = self.block23(x) + x = x[:, 7:, :, :] + + return x \ No newline at end of file diff --git a/annotator/mlsd/models/mbv2_mlsd_tiny.py b/annotator/mlsd/models/mbv2_mlsd_tiny.py new file mode 100644 index 0000000000000000000000000000000000000000..e3ed633f2cc23ea1829a627fdb879ab39f641f83 --- /dev/null +++ b/annotator/mlsd/models/mbv2_mlsd_tiny.py @@ -0,0 +1,275 @@ +import os +import sys +import torch +import torch.nn as nn +import torch.utils.model_zoo as model_zoo +from torch.nn import functional as F + + +class BlockTypeA(nn.Module): + def __init__(self, in_c1, in_c2, out_c1, out_c2, upscale = True): + super(BlockTypeA, self).__init__() + self.conv1 = nn.Sequential( + nn.Conv2d(in_c2, out_c2, kernel_size=1), + nn.BatchNorm2d(out_c2), + nn.ReLU(inplace=True) + ) + self.conv2 = nn.Sequential( + nn.Conv2d(in_c1, out_c1, kernel_size=1), + nn.BatchNorm2d(out_c1), + nn.ReLU(inplace=True) + ) + self.upscale = upscale + + def forward(self, a, b): + b = self.conv1(b) + a = self.conv2(a) + b = F.interpolate(b, scale_factor=2.0, mode='bilinear', align_corners=True) + return torch.cat((a, b), dim=1) + + +class BlockTypeB(nn.Module): + def __init__(self, in_c, out_c): + super(BlockTypeB, self).__init__() + self.conv1 = nn.Sequential( + nn.Conv2d(in_c, in_c, kernel_size=3, padding=1), + nn.BatchNorm2d(in_c), + nn.ReLU() + ) + self.conv2 = nn.Sequential( + nn.Conv2d(in_c, out_c, kernel_size=3, padding=1), + nn.BatchNorm2d(out_c), + nn.ReLU() + ) + + def forward(self, x): + x = self.conv1(x) + x + x = self.conv2(x) + return x + +class BlockTypeC(nn.Module): + def __init__(self, in_c, out_c): + super(BlockTypeC, self).__init__() + self.conv1 = nn.Sequential( + nn.Conv2d(in_c, in_c, kernel_size=3, padding=5, dilation=5), + nn.BatchNorm2d(in_c), + nn.ReLU() + ) + self.conv2 = nn.Sequential( + nn.Conv2d(in_c, in_c, kernel_size=3, padding=1), + nn.BatchNorm2d(in_c), + nn.ReLU() + ) + self.conv3 = nn.Conv2d(in_c, out_c, kernel_size=1) + + def forward(self, x): + x = self.conv1(x) + x = self.conv2(x) + x = self.conv3(x) + return x + +def _make_divisible(v, divisor, min_value=None): + """ + This function is taken from the original tf repo. + It ensures that all layers have a channel number that is divisible by 8 + It can be seen here: + https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py + :param v: + :param divisor: + :param min_value: + :return: + """ + if min_value is None: + min_value = divisor + new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than 10%. + if new_v < 0.9 * v: + new_v += divisor + return new_v + + +class ConvBNReLU(nn.Sequential): + def __init__(self, in_planes, out_planes, kernel_size=3, stride=1, groups=1): + self.channel_pad = out_planes - in_planes + self.stride = stride + #padding = (kernel_size - 1) // 2 + + # TFLite uses slightly different padding than PyTorch + if stride == 2: + padding = 0 + else: + padding = (kernel_size - 1) // 2 + + super(ConvBNReLU, self).__init__( + nn.Conv2d(in_planes, out_planes, kernel_size, stride, padding, groups=groups, bias=False), + nn.BatchNorm2d(out_planes), + nn.ReLU6(inplace=True) + ) + self.max_pool = nn.MaxPool2d(kernel_size=stride, stride=stride) + + + def forward(self, x): + # TFLite uses different padding + if self.stride == 2: + x = F.pad(x, (0, 1, 0, 1), "constant", 0) + #print(x.shape) + + for module in self: + if not isinstance(module, nn.MaxPool2d): + x = module(x) + return x + + +class InvertedResidual(nn.Module): + def __init__(self, inp, oup, stride, expand_ratio): + super(InvertedResidual, self).__init__() + self.stride = stride + assert stride in [1, 2] + + hidden_dim = int(round(inp * expand_ratio)) + self.use_res_connect = self.stride == 1 and inp == oup + + layers = [] + if expand_ratio != 1: + # pw + layers.append(ConvBNReLU(inp, hidden_dim, kernel_size=1)) + layers.extend([ + # dw + ConvBNReLU(hidden_dim, hidden_dim, stride=stride, groups=hidden_dim), + # pw-linear + nn.Conv2d(hidden_dim, oup, 1, 1, 0, bias=False), + nn.BatchNorm2d(oup), + ]) + self.conv = nn.Sequential(*layers) + + def forward(self, x): + if self.use_res_connect: + return x + self.conv(x) + else: + return self.conv(x) + + +class MobileNetV2(nn.Module): + def __init__(self, pretrained=True): + """ + MobileNet V2 main class + Args: + num_classes (int): Number of classes + width_mult (float): Width multiplier - adjusts number of channels in each layer by this amount + inverted_residual_setting: Network structure + round_nearest (int): Round the number of channels in each layer to be a multiple of this number + Set to 1 to turn off rounding + block: Module specifying inverted residual building block for mobilenet + """ + super(MobileNetV2, self).__init__() + + block = InvertedResidual + input_channel = 32 + last_channel = 1280 + width_mult = 1.0 + round_nearest = 8 + + inverted_residual_setting = [ + # t, c, n, s + [1, 16, 1, 1], + [6, 24, 2, 2], + [6, 32, 3, 2], + [6, 64, 4, 2], + #[6, 96, 3, 1], + #[6, 160, 3, 2], + #[6, 320, 1, 1], + ] + + # only check the first element, assuming user knows t,c,n,s are required + if len(inverted_residual_setting) == 0 or len(inverted_residual_setting[0]) != 4: + raise ValueError("inverted_residual_setting should be non-empty " + "or a 4-element list, got {}".format(inverted_residual_setting)) + + # building first layer + input_channel = _make_divisible(input_channel * width_mult, round_nearest) + self.last_channel = _make_divisible(last_channel * max(1.0, width_mult), round_nearest) + features = [ConvBNReLU(4, input_channel, stride=2)] + # building inverted residual blocks + for t, c, n, s in inverted_residual_setting: + output_channel = _make_divisible(c * width_mult, round_nearest) + for i in range(n): + stride = s if i == 0 else 1 + features.append(block(input_channel, output_channel, stride, expand_ratio=t)) + input_channel = output_channel + self.features = nn.Sequential(*features) + + self.fpn_selected = [3, 6, 10] + # weight initialization + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out') + if m.bias is not None: + nn.init.zeros_(m.bias) + elif isinstance(m, nn.BatchNorm2d): + nn.init.ones_(m.weight) + nn.init.zeros_(m.bias) + elif isinstance(m, nn.Linear): + nn.init.normal_(m.weight, 0, 0.01) + nn.init.zeros_(m.bias) + + #if pretrained: + # self._load_pretrained_model() + + def _forward_impl(self, x): + # This exists since TorchScript doesn't support inheritance, so the superclass method + # (this one) needs to have a name other than `forward` that can be accessed in a subclass + fpn_features = [] + for i, f in enumerate(self.features): + if i > self.fpn_selected[-1]: + break + x = f(x) + if i in self.fpn_selected: + fpn_features.append(x) + + c2, c3, c4 = fpn_features + return c2, c3, c4 + + + def forward(self, x): + return self._forward_impl(x) + + def _load_pretrained_model(self): + pretrain_dict = model_zoo.load_url('https://download.pytorch.org/models/mobilenet_v2-b0353104.pth') + model_dict = {} + state_dict = self.state_dict() + for k, v in pretrain_dict.items(): + if k in state_dict: + model_dict[k] = v + state_dict.update(model_dict) + self.load_state_dict(state_dict) + + +class MobileV2_MLSD_Tiny(nn.Module): + def __init__(self): + super(MobileV2_MLSD_Tiny, self).__init__() + + self.backbone = MobileNetV2(pretrained=True) + + self.block12 = BlockTypeA(in_c1= 32, in_c2= 64, + out_c1= 64, out_c2=64) + self.block13 = BlockTypeB(128, 64) + + self.block14 = BlockTypeA(in_c1 = 24, in_c2 = 64, + out_c1= 32, out_c2= 32) + self.block15 = BlockTypeB(64, 64) + + self.block16 = BlockTypeC(64, 16) + + def forward(self, x): + c2, c3, c4 = self.backbone(x) + + x = self.block12(c3, c4) + x = self.block13(x) + x = self.block14(c2, x) + x = self.block15(x) + x = self.block16(x) + x = x[:, 7:, :, :] + #print(x.shape) + x = F.interpolate(x, scale_factor=2.0, mode='bilinear', align_corners=True) + + return x \ No newline at end of file diff --git a/annotator/mlsd/utils.py b/annotator/mlsd/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e24e7fbb028b34d5871bb7a5d96c68f35774bbd0 --- /dev/null +++ b/annotator/mlsd/utils.py @@ -0,0 +1,582 @@ +''' +modified by lihaoweicv +pytorch version +''' + +''' +M-LSD +Copyright 2021-present NAVER Corp. +Apache License v2.0 +''' + +import os +import numpy as np +import cv2 +import torch +from torch.nn import functional as F + + +def deccode_output_score_and_ptss(tpMap, topk_n = 200, ksize = 5): + ''' + tpMap: + center: tpMap[1, 0, :, :] + displacement: tpMap[1, 1:5, :, :] + ''' + b, c, h, w = tpMap.shape + assert b==1, 'only support bsize==1' + displacement = tpMap[:, 1:5, :, :][0] + center = tpMap[:, 0, :, :] + heat = torch.sigmoid(center) + hmax = F.max_pool2d( heat, (ksize, ksize), stride=1, padding=(ksize-1)//2) + keep = (hmax == heat).float() + heat = heat * keep + heat = heat.reshape(-1, ) + + scores, indices = torch.topk(heat, topk_n, dim=-1, largest=True) + yy = torch.floor_divide(indices, w).unsqueeze(-1) + xx = torch.fmod(indices, w).unsqueeze(-1) + ptss = torch.cat((yy, xx),dim=-1) + + ptss = ptss.detach().cpu().numpy() + scores = scores.detach().cpu().numpy() + displacement = displacement.detach().cpu().numpy() + displacement = displacement.transpose((1,2,0)) + return ptss, scores, displacement + + +def pred_lines(image, model, + input_shape=[512, 512], + score_thr=0.10, + dist_thr=20.0): + h, w, _ = image.shape + h_ratio, w_ratio = [h / input_shape[0], w / input_shape[1]] + + resized_image = np.concatenate([cv2.resize(image, (input_shape[1], input_shape[0]), interpolation=cv2.INTER_AREA), + np.ones([input_shape[0], input_shape[1], 1])], axis=-1) + + resized_image = resized_image.transpose((2,0,1)) + batch_image = np.expand_dims(resized_image, axis=0).astype('float32') + batch_image = (batch_image / 127.5) - 1.0 + +# batch_image = torch.from_numpy(batch_image).float().cuda() + batch_image = torch.from_numpy(batch_image).float().cpu() + outputs = model(batch_image) + pts, pts_score, vmap = deccode_output_score_and_ptss(outputs, 200, 3) + start = vmap[:, :, :2] + end = vmap[:, :, 2:] + dist_map = np.sqrt(np.sum((start - end) ** 2, axis=-1)) + + segments_list = [] + for center, score in zip(pts, pts_score): + y, x = center + distance = dist_map[y, x] + if score > score_thr and distance > dist_thr: + disp_x_start, disp_y_start, disp_x_end, disp_y_end = vmap[y, x, :] + x_start = x + disp_x_start + y_start = y + disp_y_start + x_end = x + disp_x_end + y_end = y + disp_y_end + segments_list.append([x_start, y_start, x_end, y_end]) + + lines = 2 * np.array(segments_list) # 256 > 512 + lines[:, 0] = lines[:, 0] * w_ratio + lines[:, 1] = lines[:, 1] * h_ratio + lines[:, 2] = lines[:, 2] * w_ratio + lines[:, 3] = lines[:, 3] * h_ratio + + return lines + + +def pred_squares(image, + model, + input_shape=[512, 512], + params={'score': 0.06, + 'outside_ratio': 0.28, + 'inside_ratio': 0.45, + 'w_overlap': 0.0, + 'w_degree': 1.95, + 'w_length': 0.0, + 'w_area': 1.86, + 'w_center': 0.14}): + ''' + shape = [height, width] + ''' + h, w, _ = image.shape + original_shape = [h, w] + + resized_image = np.concatenate([cv2.resize(image, (input_shape[0], input_shape[1]), interpolation=cv2.INTER_AREA), + np.ones([input_shape[0], input_shape[1], 1])], axis=-1) + resized_image = resized_image.transpose((2, 0, 1)) + batch_image = np.expand_dims(resized_image, axis=0).astype('float32') + batch_image = (batch_image / 127.5) - 1.0 + +# batch_image = torch.from_numpy(batch_image).float().cuda() + batch_image = torch.from_numpy(batch_image).float().cpu() + outputs = model(batch_image) + + pts, pts_score, vmap = deccode_output_score_and_ptss(outputs, 200, 3) + start = vmap[:, :, :2] # (x, y) + end = vmap[:, :, 2:] # (x, y) + dist_map = np.sqrt(np.sum((start - end) ** 2, axis=-1)) + + junc_list = [] + segments_list = [] + for junc, score in zip(pts, pts_score): + y, x = junc + distance = dist_map[y, x] + if score > params['score'] and distance > 20.0: + junc_list.append([x, y]) + disp_x_start, disp_y_start, disp_x_end, disp_y_end = vmap[y, x, :] + d_arrow = 1.0 + x_start = x + d_arrow * disp_x_start + y_start = y + d_arrow * disp_y_start + x_end = x + d_arrow * disp_x_end + y_end = y + d_arrow * disp_y_end + segments_list.append([x_start, y_start, x_end, y_end]) + + segments = np.array(segments_list) + + ####### post processing for squares + # 1. get unique lines + point = np.array([[0, 0]]) + point = point[0] + start = segments[:, :2] + end = segments[:, 2:] + diff = start - end + a = diff[:, 1] + b = -diff[:, 0] + c = a * start[:, 0] + b * start[:, 1] + + d = np.abs(a * point[0] + b * point[1] - c) / np.sqrt(a ** 2 + b ** 2 + 1e-10) + theta = np.arctan2(diff[:, 0], diff[:, 1]) * 180 / np.pi + theta[theta < 0.0] += 180 + hough = np.concatenate([d[:, None], theta[:, None]], axis=-1) + + d_quant = 1 + theta_quant = 2 + hough[:, 0] //= d_quant + hough[:, 1] //= theta_quant + _, indices, counts = np.unique(hough, axis=0, return_index=True, return_counts=True) + + acc_map = np.zeros([512 // d_quant + 1, 360 // theta_quant + 1], dtype='float32') + idx_map = np.zeros([512 // d_quant + 1, 360 // theta_quant + 1], dtype='int32') - 1 + yx_indices = hough[indices, :].astype('int32') + acc_map[yx_indices[:, 0], yx_indices[:, 1]] = counts + idx_map[yx_indices[:, 0], yx_indices[:, 1]] = indices + + acc_map_np = acc_map + # acc_map = acc_map[None, :, :, None] + # + # ### fast suppression using tensorflow op + # acc_map = tf.constant(acc_map, dtype=tf.float32) + # max_acc_map = tf.keras.layers.MaxPool2D(pool_size=(5, 5), strides=1, padding='same')(acc_map) + # acc_map = acc_map * tf.cast(tf.math.equal(acc_map, max_acc_map), tf.float32) + # flatten_acc_map = tf.reshape(acc_map, [1, -1]) + # topk_values, topk_indices = tf.math.top_k(flatten_acc_map, k=len(pts)) + # _, h, w, _ = acc_map.shape + # y = tf.expand_dims(topk_indices // w, axis=-1) + # x = tf.expand_dims(topk_indices % w, axis=-1) + # yx = tf.concat([y, x], axis=-1) + + ### fast suppression using pytorch op + acc_map = torch.from_numpy(acc_map_np).unsqueeze(0).unsqueeze(0) + _,_, h, w = acc_map.shape + max_acc_map = F.max_pool2d(acc_map,kernel_size=5, stride=1, padding=2) + acc_map = acc_map * ( (acc_map == max_acc_map).float() ) + flatten_acc_map = acc_map.reshape([-1, ]) + + scores, indices = torch.topk(flatten_acc_map, len(pts), dim=-1, largest=True) + yy = torch.div(indices, w, rounding_mode='floor').unsqueeze(-1) + xx = torch.fmod(indices, w).unsqueeze(-1) + yx = torch.cat((yy, xx), dim=-1) + + yx = yx.detach().cpu().numpy() + + topk_values = scores.detach().cpu().numpy() + indices = idx_map[yx[:, 0], yx[:, 1]] + basis = 5 // 2 + + merged_segments = [] + for yx_pt, max_indice, value in zip(yx, indices, topk_values): + y, x = yx_pt + if max_indice == -1 or value == 0: + continue + segment_list = [] + for y_offset in range(-basis, basis + 1): + for x_offset in range(-basis, basis + 1): + indice = idx_map[y + y_offset, x + x_offset] + cnt = int(acc_map_np[y + y_offset, x + x_offset]) + if indice != -1: + segment_list.append(segments[indice]) + if cnt > 1: + check_cnt = 1 + current_hough = hough[indice] + for new_indice, new_hough in enumerate(hough): + if (current_hough == new_hough).all() and indice != new_indice: + segment_list.append(segments[new_indice]) + check_cnt += 1 + if check_cnt == cnt: + break + group_segments = np.array(segment_list).reshape([-1, 2]) + sorted_group_segments = np.sort(group_segments, axis=0) + x_min, y_min = sorted_group_segments[0, :] + x_max, y_max = sorted_group_segments[-1, :] + + deg = theta[max_indice] + if deg >= 90: + merged_segments.append([x_min, y_max, x_max, y_min]) + else: + merged_segments.append([x_min, y_min, x_max, y_max]) + + # 2. get intersections + new_segments = np.array(merged_segments) # (x1, y1, x2, y2) + start = new_segments[:, :2] # (x1, y1) + end = new_segments[:, 2:] # (x2, y2) + new_centers = (start + end) / 2.0 + diff = start - end + dist_segments = np.sqrt(np.sum(diff ** 2, axis=-1)) + + # ax + by = c + a = diff[:, 1] + b = -diff[:, 0] + c = a * start[:, 0] + b * start[:, 1] + pre_det = a[:, None] * b[None, :] + det = pre_det - np.transpose(pre_det) + + pre_inter_y = a[:, None] * c[None, :] + inter_y = (pre_inter_y - np.transpose(pre_inter_y)) / (det + 1e-10) + pre_inter_x = c[:, None] * b[None, :] + inter_x = (pre_inter_x - np.transpose(pre_inter_x)) / (det + 1e-10) + inter_pts = np.concatenate([inter_x[:, :, None], inter_y[:, :, None]], axis=-1).astype('int32') + + # 3. get corner information + # 3.1 get distance + ''' + dist_segments: + | dist(0), dist(1), dist(2), ...| + dist_inter_to_segment1: + | dist(inter,0), dist(inter,0), dist(inter,0), ... | + | dist(inter,1), dist(inter,1), dist(inter,1), ... | + ... + dist_inter_to_semgnet2: + | dist(inter,0), dist(inter,1), dist(inter,2), ... | + | dist(inter,0), dist(inter,1), dist(inter,2), ... | + ... + ''' + + dist_inter_to_segment1_start = np.sqrt( + np.sum(((inter_pts - start[:, None, :]) ** 2), axis=-1, keepdims=True)) # [n_batch, n_batch, 1] + dist_inter_to_segment1_end = np.sqrt( + np.sum(((inter_pts - end[:, None, :]) ** 2), axis=-1, keepdims=True)) # [n_batch, n_batch, 1] + dist_inter_to_segment2_start = np.sqrt( + np.sum(((inter_pts - start[None, :, :]) ** 2), axis=-1, keepdims=True)) # [n_batch, n_batch, 1] + dist_inter_to_segment2_end = np.sqrt( + np.sum(((inter_pts - end[None, :, :]) ** 2), axis=-1, keepdims=True)) # [n_batch, n_batch, 1] + + # sort ascending + dist_inter_to_segment1 = np.sort( + np.concatenate([dist_inter_to_segment1_start, dist_inter_to_segment1_end], axis=-1), + axis=-1) # [n_batch, n_batch, 2] + dist_inter_to_segment2 = np.sort( + np.concatenate([dist_inter_to_segment2_start, dist_inter_to_segment2_end], axis=-1), + axis=-1) # [n_batch, n_batch, 2] + + # 3.2 get degree + inter_to_start = new_centers[:, None, :] - inter_pts + deg_inter_to_start = np.arctan2(inter_to_start[:, :, 1], inter_to_start[:, :, 0]) * 180 / np.pi + deg_inter_to_start[deg_inter_to_start < 0.0] += 360 + inter_to_end = new_centers[None, :, :] - inter_pts + deg_inter_to_end = np.arctan2(inter_to_end[:, :, 1], inter_to_end[:, :, 0]) * 180 / np.pi + deg_inter_to_end[deg_inter_to_end < 0.0] += 360 + + ''' + B -- G + | | + C -- R + B : blue / G: green / C: cyan / R: red + + 0 -- 1 + | | + 3 -- 2 + ''' + # rename variables + deg1_map, deg2_map = deg_inter_to_start, deg_inter_to_end + # sort deg ascending + deg_sort = np.sort(np.concatenate([deg1_map[:, :, None], deg2_map[:, :, None]], axis=-1), axis=-1) + + deg_diff_map = np.abs(deg1_map - deg2_map) + # we only consider the smallest degree of intersect + deg_diff_map[deg_diff_map > 180] = 360 - deg_diff_map[deg_diff_map > 180] + + # define available degree range + deg_range = [60, 120] + + corner_dict = {corner_info: [] for corner_info in range(4)} + inter_points = [] + for i in range(inter_pts.shape[0]): + for j in range(i + 1, inter_pts.shape[1]): + # i, j > line index, always i < j + x, y = inter_pts[i, j, :] + deg1, deg2 = deg_sort[i, j, :] + deg_diff = deg_diff_map[i, j] + + check_degree = deg_diff > deg_range[0] and deg_diff < deg_range[1] + + outside_ratio = params['outside_ratio'] # over ratio >>> drop it! + inside_ratio = params['inside_ratio'] # over ratio >>> drop it! + check_distance = ((dist_inter_to_segment1[i, j, 1] >= dist_segments[i] and \ + dist_inter_to_segment1[i, j, 0] <= dist_segments[i] * outside_ratio) or \ + (dist_inter_to_segment1[i, j, 1] <= dist_segments[i] and \ + dist_inter_to_segment1[i, j, 0] <= dist_segments[i] * inside_ratio)) and \ + ((dist_inter_to_segment2[i, j, 1] >= dist_segments[j] and \ + dist_inter_to_segment2[i, j, 0] <= dist_segments[j] * outside_ratio) or \ + (dist_inter_to_segment2[i, j, 1] <= dist_segments[j] and \ + dist_inter_to_segment2[i, j, 0] <= dist_segments[j] * inside_ratio)) + + if check_degree and check_distance: + corner_info = None + + if (deg1 >= 0 and deg1 <= 45 and deg2 >= 45 and deg2 <= 120) or \ + (deg2 >= 315 and deg1 >= 45 and deg1 <= 120): + corner_info, color_info = 0, 'blue' + elif (deg1 >= 45 and deg1 <= 125 and deg2 >= 125 and deg2 <= 225): + corner_info, color_info = 1, 'green' + elif (deg1 >= 125 and deg1 <= 225 and deg2 >= 225 and deg2 <= 315): + corner_info, color_info = 2, 'black' + elif (deg1 >= 0 and deg1 <= 45 and deg2 >= 225 and deg2 <= 315) or \ + (deg2 >= 315 and deg1 >= 225 and deg1 <= 315): + corner_info, color_info = 3, 'cyan' + else: + corner_info, color_info = 4, 'red' # we don't use it + continue + + corner_dict[corner_info].append([x, y, i, j]) + inter_points.append([x, y]) + + square_list = [] + connect_list = [] + segments_list = [] + for corner0 in corner_dict[0]: + for corner1 in corner_dict[1]: + connect01 = False + for corner0_line in corner0[2:]: + if corner0_line in corner1[2:]: + connect01 = True + break + if connect01: + for corner2 in corner_dict[2]: + connect12 = False + for corner1_line in corner1[2:]: + if corner1_line in corner2[2:]: + connect12 = True + break + if connect12: + for corner3 in corner_dict[3]: + connect23 = False + for corner2_line in corner2[2:]: + if corner2_line in corner3[2:]: + connect23 = True + break + if connect23: + for corner3_line in corner3[2:]: + if corner3_line in corner0[2:]: + # SQUARE!!! + ''' + 0 -- 1 + | | + 3 -- 2 + square_list: + order: 0 > 1 > 2 > 3 + | x0, y0, x1, y1, x2, y2, x3, y3 | + | x0, y0, x1, y1, x2, y2, x3, y3 | + ... + connect_list: + order: 01 > 12 > 23 > 30 + | line_idx01, line_idx12, line_idx23, line_idx30 | + | line_idx01, line_idx12, line_idx23, line_idx30 | + ... + segments_list: + order: 0 > 1 > 2 > 3 + | line_idx0_i, line_idx0_j, line_idx1_i, line_idx1_j, line_idx2_i, line_idx2_j, line_idx3_i, line_idx3_j | + | line_idx0_i, line_idx0_j, line_idx1_i, line_idx1_j, line_idx2_i, line_idx2_j, line_idx3_i, line_idx3_j | + ... + ''' + square_list.append(corner0[:2] + corner1[:2] + corner2[:2] + corner3[:2]) + connect_list.append([corner0_line, corner1_line, corner2_line, corner3_line]) + segments_list.append(corner0[2:] + corner1[2:] + corner2[2:] + corner3[2:]) + + def check_outside_inside(segments_info, connect_idx): + # return 'outside or inside', min distance, cover_param, peri_param + if connect_idx == segments_info[0]: + check_dist_mat = dist_inter_to_segment1 + else: + check_dist_mat = dist_inter_to_segment2 + + i, j = segments_info + min_dist, max_dist = check_dist_mat[i, j, :] + connect_dist = dist_segments[connect_idx] + if max_dist > connect_dist: + return 'outside', min_dist, 0, 1 + else: + return 'inside', min_dist, -1, -1 + + top_square = None + + try: + map_size = input_shape[0] / 2 + squares = np.array(square_list).reshape([-1, 4, 2]) + score_array = [] + connect_array = np.array(connect_list) + segments_array = np.array(segments_list).reshape([-1, 4, 2]) + + # get degree of corners: + squares_rollup = np.roll(squares, 1, axis=1) + squares_rolldown = np.roll(squares, -1, axis=1) + vec1 = squares_rollup - squares + normalized_vec1 = vec1 / (np.linalg.norm(vec1, axis=-1, keepdims=True) + 1e-10) + vec2 = squares_rolldown - squares + normalized_vec2 = vec2 / (np.linalg.norm(vec2, axis=-1, keepdims=True) + 1e-10) + inner_products = np.sum(normalized_vec1 * normalized_vec2, axis=-1) # [n_squares, 4] + squares_degree = np.arccos(inner_products) * 180 / np.pi # [n_squares, 4] + + # get square score + overlap_scores = [] + degree_scores = [] + length_scores = [] + + for connects, segments, square, degree in zip(connect_array, segments_array, squares, squares_degree): + ''' + 0 -- 1 + | | + 3 -- 2 + + # segments: [4, 2] + # connects: [4] + ''' + + ###################################### OVERLAP SCORES + cover = 0 + perimeter = 0 + # check 0 > 1 > 2 > 3 + square_length = [] + + for start_idx in range(4): + end_idx = (start_idx + 1) % 4 + + connect_idx = connects[start_idx] # segment idx of segment01 + start_segments = segments[start_idx] + end_segments = segments[end_idx] + + start_point = square[start_idx] + end_point = square[end_idx] + + # check whether outside or inside + start_position, start_min, start_cover_param, start_peri_param = check_outside_inside(start_segments, + connect_idx) + end_position, end_min, end_cover_param, end_peri_param = check_outside_inside(end_segments, connect_idx) + + cover += dist_segments[connect_idx] + start_cover_param * start_min + end_cover_param * end_min + perimeter += dist_segments[connect_idx] + start_peri_param * start_min + end_peri_param * end_min + + square_length.append( + dist_segments[connect_idx] + start_peri_param * start_min + end_peri_param * end_min) + + overlap_scores.append(cover / perimeter) + ###################################### + ###################################### DEGREE SCORES + ''' + deg0 vs deg2 + deg1 vs deg3 + ''' + deg0, deg1, deg2, deg3 = degree + deg_ratio1 = deg0 / deg2 + if deg_ratio1 > 1.0: + deg_ratio1 = 1 / deg_ratio1 + deg_ratio2 = deg1 / deg3 + if deg_ratio2 > 1.0: + deg_ratio2 = 1 / deg_ratio2 + degree_scores.append((deg_ratio1 + deg_ratio2) / 2) + ###################################### + ###################################### LENGTH SCORES + ''' + len0 vs len2 + len1 vs len3 + ''' + len0, len1, len2, len3 = square_length + len_ratio1 = len0 / len2 if len2 > len0 else len2 / len0 + len_ratio2 = len1 / len3 if len3 > len1 else len3 / len1 + length_scores.append((len_ratio1 + len_ratio2) / 2) + + ###################################### + + overlap_scores = np.array(overlap_scores) + overlap_scores /= np.max(overlap_scores) + + degree_scores = np.array(degree_scores) + # degree_scores /= np.max(degree_scores) + + length_scores = np.array(length_scores) + + ###################################### AREA SCORES + area_scores = np.reshape(squares, [-1, 4, 2]) + area_x = area_scores[:, :, 0] + area_y = area_scores[:, :, 1] + correction = area_x[:, -1] * area_y[:, 0] - area_y[:, -1] * area_x[:, 0] + area_scores = np.sum(area_x[:, :-1] * area_y[:, 1:], axis=-1) - np.sum(area_y[:, :-1] * area_x[:, 1:], axis=-1) + area_scores = 0.5 * np.abs(area_scores + correction) + area_scores /= (map_size * map_size) # np.max(area_scores) + ###################################### + + ###################################### CENTER SCORES + centers = np.array([[256 // 2, 256 // 2]], dtype='float32') # [1, 2] + # squares: [n, 4, 2] + square_centers = np.mean(squares, axis=1) # [n, 2] + center2center = np.sqrt(np.sum((centers - square_centers) ** 2)) + center_scores = center2center / (map_size / np.sqrt(2.0)) + + ''' + score_w = [overlap, degree, area, center, length] + ''' + score_w = [0.0, 1.0, 10.0, 0.5, 1.0] + score_array = params['w_overlap'] * overlap_scores \ + + params['w_degree'] * degree_scores \ + + params['w_area'] * area_scores \ + - params['w_center'] * center_scores \ + + params['w_length'] * length_scores + + best_square = [] + + sorted_idx = np.argsort(score_array)[::-1] + score_array = score_array[sorted_idx] + squares = squares[sorted_idx] + + except Exception as e: + pass + + '''return list + merged_lines, squares, scores + ''' + + try: + new_segments[:, 0] = new_segments[:, 0] * 2 / input_shape[1] * original_shape[1] + new_segments[:, 1] = new_segments[:, 1] * 2 / input_shape[0] * original_shape[0] + new_segments[:, 2] = new_segments[:, 2] * 2 / input_shape[1] * original_shape[1] + new_segments[:, 3] = new_segments[:, 3] * 2 / input_shape[0] * original_shape[0] + except: + new_segments = [] + + try: + squares[:, :, 0] = squares[:, :, 0] * 2 / input_shape[1] * original_shape[1] + squares[:, :, 1] = squares[:, :, 1] * 2 / input_shape[0] * original_shape[0] + except: + squares = [] + score_array = [] + + try: + inter_points = np.array(inter_points) + inter_points[:, 0] = inter_points[:, 0] * 2 / input_shape[1] * original_shape[1] + inter_points[:, 1] = inter_points[:, 1] * 2 / input_shape[0] * original_shape[0] + except: + inter_points = [] + + return new_segments, squares, score_array, inter_points diff --git a/annotator/mmpkg/mmcv/__init__.py b/annotator/mmpkg/mmcv/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..210a2989138380559f23045b568d0fbbeb918c03 --- /dev/null +++ b/annotator/mmpkg/mmcv/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# flake8: noqa +from .arraymisc import * +from .fileio import * +from .image import * +from .utils import * +from .version import * +from .video import * +from .visualization import * + +# The following modules are not imported to this level, so mmcv may be used +# without PyTorch. +# - runner +# - parallel +# - op diff --git a/annotator/mmpkg/mmcv/arraymisc/__init__.py b/annotator/mmpkg/mmcv/arraymisc/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4b4700d6139ae3d604ff6e542468cce4200c020c --- /dev/null +++ b/annotator/mmpkg/mmcv/arraymisc/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .quantization import dequantize, quantize + +__all__ = ['quantize', 'dequantize'] diff --git a/annotator/mmpkg/mmcv/arraymisc/quantization.py b/annotator/mmpkg/mmcv/arraymisc/quantization.py new file mode 100644 index 0000000000000000000000000000000000000000..8e47a3545780cf071a1ef8195efb0b7b662c8186 --- /dev/null +++ b/annotator/mmpkg/mmcv/arraymisc/quantization.py @@ -0,0 +1,55 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numpy as np + + +def quantize(arr, min_val, max_val, levels, dtype=np.int64): + """Quantize an array of (-inf, inf) to [0, levels-1]. + + Args: + arr (ndarray): Input array. + min_val (scalar): Minimum value to be clipped. + max_val (scalar): Maximum value to be clipped. + levels (int): Quantization levels. + dtype (np.type): The type of the quantized array. + + Returns: + tuple: Quantized array. + """ + if not (isinstance(levels, int) and levels > 1): + raise ValueError( + f'levels must be a positive integer, but got {levels}') + if min_val >= max_val: + raise ValueError( + f'min_val ({min_val}) must be smaller than max_val ({max_val})') + + arr = np.clip(arr, min_val, max_val) - min_val + quantized_arr = np.minimum( + np.floor(levels * arr / (max_val - min_val)).astype(dtype), levels - 1) + + return quantized_arr + + +def dequantize(arr, min_val, max_val, levels, dtype=np.float64): + """Dequantize an array. + + Args: + arr (ndarray): Input array. + min_val (scalar): Minimum value to be clipped. + max_val (scalar): Maximum value to be clipped. + levels (int): Quantization levels. + dtype (np.type): The type of the dequantized array. + + Returns: + tuple: Dequantized array. + """ + if not (isinstance(levels, int) and levels > 1): + raise ValueError( + f'levels must be a positive integer, but got {levels}') + if min_val >= max_val: + raise ValueError( + f'min_val ({min_val}) must be smaller than max_val ({max_val})') + + dequantized_arr = (arr + 0.5).astype(dtype) * (max_val - + min_val) / levels + min_val + + return dequantized_arr diff --git a/annotator/mmpkg/mmcv/cnn/__init__.py b/annotator/mmpkg/mmcv/cnn/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7246c897430f0cc7ce12719ad8608824fc734446 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/__init__.py @@ -0,0 +1,41 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .alexnet import AlexNet +# yapf: disable +from .bricks import (ACTIVATION_LAYERS, CONV_LAYERS, NORM_LAYERS, + PADDING_LAYERS, PLUGIN_LAYERS, UPSAMPLE_LAYERS, + ContextBlock, Conv2d, Conv3d, ConvAWS2d, ConvModule, + ConvTranspose2d, ConvTranspose3d, ConvWS2d, + DepthwiseSeparableConvModule, GeneralizedAttention, + HSigmoid, HSwish, Linear, MaxPool2d, MaxPool3d, + NonLocal1d, NonLocal2d, NonLocal3d, Scale, Swish, + build_activation_layer, build_conv_layer, + build_norm_layer, build_padding_layer, build_plugin_layer, + build_upsample_layer, conv_ws_2d, is_norm) +from .builder import MODELS, build_model_from_cfg +# yapf: enable +from .resnet import ResNet, make_res_layer +from .utils import (INITIALIZERS, Caffe2XavierInit, ConstantInit, KaimingInit, + NormalInit, PretrainedInit, TruncNormalInit, UniformInit, + XavierInit, bias_init_with_prob, caffe2_xavier_init, + constant_init, fuse_conv_bn, get_model_complexity_info, + initialize, kaiming_init, normal_init, trunc_normal_init, + uniform_init, xavier_init) +from .vgg import VGG, make_vgg_layer + +__all__ = [ + 'AlexNet', 'VGG', 'make_vgg_layer', 'ResNet', 'make_res_layer', + 'constant_init', 'xavier_init', 'normal_init', 'trunc_normal_init', + 'uniform_init', 'kaiming_init', 'caffe2_xavier_init', + 'bias_init_with_prob', 'ConvModule', 'build_activation_layer', + 'build_conv_layer', 'build_norm_layer', 'build_padding_layer', + 'build_upsample_layer', 'build_plugin_layer', 'is_norm', 'NonLocal1d', + 'NonLocal2d', 'NonLocal3d', 'ContextBlock', 'HSigmoid', 'Swish', 'HSwish', + 'GeneralizedAttention', 'ACTIVATION_LAYERS', 'CONV_LAYERS', 'NORM_LAYERS', + 'PADDING_LAYERS', 'UPSAMPLE_LAYERS', 'PLUGIN_LAYERS', 'Scale', + 'get_model_complexity_info', 'conv_ws_2d', 'ConvAWS2d', 'ConvWS2d', + 'fuse_conv_bn', 'DepthwiseSeparableConvModule', 'Linear', 'Conv2d', + 'ConvTranspose2d', 'MaxPool2d', 'ConvTranspose3d', 'MaxPool3d', 'Conv3d', + 'initialize', 'INITIALIZERS', 'ConstantInit', 'XavierInit', 'NormalInit', + 'TruncNormalInit', 'UniformInit', 'KaimingInit', 'PretrainedInit', + 'Caffe2XavierInit', 'MODELS', 'build_model_from_cfg' +] diff --git a/annotator/mmpkg/mmcv/cnn/alexnet.py b/annotator/mmpkg/mmcv/cnn/alexnet.py new file mode 100644 index 0000000000000000000000000000000000000000..89e36b8c7851f895d9ae7f07149f0e707456aab0 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/alexnet.py @@ -0,0 +1,61 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import logging + +import torch.nn as nn + + +class AlexNet(nn.Module): + """AlexNet backbone. + + Args: + num_classes (int): number of classes for classification. + """ + + def __init__(self, num_classes=-1): + super(AlexNet, self).__init__() + self.num_classes = num_classes + self.features = nn.Sequential( + nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=3, stride=2), + nn.Conv2d(64, 192, kernel_size=5, padding=2), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=3, stride=2), + nn.Conv2d(192, 384, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.Conv2d(384, 256, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.Conv2d(256, 256, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=3, stride=2), + ) + if self.num_classes > 0: + self.classifier = nn.Sequential( + nn.Dropout(), + nn.Linear(256 * 6 * 6, 4096), + nn.ReLU(inplace=True), + nn.Dropout(), + nn.Linear(4096, 4096), + nn.ReLU(inplace=True), + nn.Linear(4096, num_classes), + ) + + def init_weights(self, pretrained=None): + if isinstance(pretrained, str): + logger = logging.getLogger() + from ..runner import load_checkpoint + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + # use default initializer + pass + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + + x = self.features(x) + if self.num_classes > 0: + x = x.view(x.size(0), 256 * 6 * 6) + x = self.classifier(x) + + return x diff --git a/annotator/mmpkg/mmcv/cnn/bricks/__init__.py b/annotator/mmpkg/mmcv/cnn/bricks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0f33124ed23fc6f27119a37bcb5ab004d3572be0 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/__init__.py @@ -0,0 +1,35 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .activation import build_activation_layer +from .context_block import ContextBlock +from .conv import build_conv_layer +from .conv2d_adaptive_padding import Conv2dAdaptivePadding +from .conv_module import ConvModule +from .conv_ws import ConvAWS2d, ConvWS2d, conv_ws_2d +from .depthwise_separable_conv_module import DepthwiseSeparableConvModule +from .drop import Dropout, DropPath +from .generalized_attention import GeneralizedAttention +from .hsigmoid import HSigmoid +from .hswish import HSwish +from .non_local import NonLocal1d, NonLocal2d, NonLocal3d +from .norm import build_norm_layer, is_norm +from .padding import build_padding_layer +from .plugin import build_plugin_layer +from .registry import (ACTIVATION_LAYERS, CONV_LAYERS, NORM_LAYERS, + PADDING_LAYERS, PLUGIN_LAYERS, UPSAMPLE_LAYERS) +from .scale import Scale +from .swish import Swish +from .upsample import build_upsample_layer +from .wrappers import (Conv2d, Conv3d, ConvTranspose2d, ConvTranspose3d, + Linear, MaxPool2d, MaxPool3d) + +__all__ = [ + 'ConvModule', 'build_activation_layer', 'build_conv_layer', + 'build_norm_layer', 'build_padding_layer', 'build_upsample_layer', + 'build_plugin_layer', 'is_norm', 'HSigmoid', 'HSwish', 'NonLocal1d', + 'NonLocal2d', 'NonLocal3d', 'ContextBlock', 'GeneralizedAttention', + 'ACTIVATION_LAYERS', 'CONV_LAYERS', 'NORM_LAYERS', 'PADDING_LAYERS', + 'UPSAMPLE_LAYERS', 'PLUGIN_LAYERS', 'Scale', 'ConvAWS2d', 'ConvWS2d', + 'conv_ws_2d', 'DepthwiseSeparableConvModule', 'Swish', 'Linear', + 'Conv2dAdaptivePadding', 'Conv2d', 'ConvTranspose2d', 'MaxPool2d', + 'ConvTranspose3d', 'MaxPool3d', 'Conv3d', 'Dropout', 'DropPath' +] diff --git a/annotator/mmpkg/mmcv/cnn/bricks/activation.py b/annotator/mmpkg/mmcv/cnn/bricks/activation.py new file mode 100644 index 0000000000000000000000000000000000000000..a8951058c8e77eda02c130f3401c9680702e231c --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/activation.py @@ -0,0 +1,92 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from annotator.mmpkg.mmcv.utils import TORCH_VERSION, build_from_cfg, digit_version +from .registry import ACTIVATION_LAYERS + +for module in [ + nn.ReLU, nn.LeakyReLU, nn.PReLU, nn.RReLU, nn.ReLU6, nn.ELU, + nn.Sigmoid, nn.Tanh +]: + ACTIVATION_LAYERS.register_module(module=module) + + +@ACTIVATION_LAYERS.register_module(name='Clip') +@ACTIVATION_LAYERS.register_module() +class Clamp(nn.Module): + """Clamp activation layer. + + This activation function is to clamp the feature map value within + :math:`[min, max]`. More details can be found in ``torch.clamp()``. + + Args: + min (Number | optional): Lower-bound of the range to be clamped to. + Default to -1. + max (Number | optional): Upper-bound of the range to be clamped to. + Default to 1. + """ + + def __init__(self, min=-1., max=1.): + super(Clamp, self).__init__() + self.min = min + self.max = max + + def forward(self, x): + """Forward function. + + Args: + x (torch.Tensor): The input tensor. + + Returns: + torch.Tensor: Clamped tensor. + """ + return torch.clamp(x, min=self.min, max=self.max) + + +class GELU(nn.Module): + r"""Applies the Gaussian Error Linear Units function: + + .. math:: + \text{GELU}(x) = x * \Phi(x) + where :math:`\Phi(x)` is the Cumulative Distribution Function for + Gaussian Distribution. + + Shape: + - Input: :math:`(N, *)` where `*` means, any number of additional + dimensions + - Output: :math:`(N, *)`, same shape as the input + + .. image:: scripts/activation_images/GELU.png + + Examples:: + + >>> m = nn.GELU() + >>> input = torch.randn(2) + >>> output = m(input) + """ + + def forward(self, input): + return F.gelu(input) + + +if (TORCH_VERSION == 'parrots' + or digit_version(TORCH_VERSION) < digit_version('1.4')): + ACTIVATION_LAYERS.register_module(module=GELU) +else: + ACTIVATION_LAYERS.register_module(module=nn.GELU) + + +def build_activation_layer(cfg): + """Build activation layer. + + Args: + cfg (dict): The activation layer config, which should contain: + - type (str): Layer type. + - layer args: Args needed to instantiate an activation layer. + + Returns: + nn.Module: Created activation layer. + """ + return build_from_cfg(cfg, ACTIVATION_LAYERS) diff --git a/annotator/mmpkg/mmcv/cnn/bricks/context_block.py b/annotator/mmpkg/mmcv/cnn/bricks/context_block.py new file mode 100644 index 0000000000000000000000000000000000000000..d60fdb904c749ce3b251510dff3cc63cea70d42e --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/context_block.py @@ -0,0 +1,125 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch import nn + +from ..utils import constant_init, kaiming_init +from .registry import PLUGIN_LAYERS + + +def last_zero_init(m): + if isinstance(m, nn.Sequential): + constant_init(m[-1], val=0) + else: + constant_init(m, val=0) + + +@PLUGIN_LAYERS.register_module() +class ContextBlock(nn.Module): + """ContextBlock module in GCNet. + + See 'GCNet: Non-local Networks Meet Squeeze-Excitation Networks and Beyond' + (https://arxiv.org/abs/1904.11492) for details. + + Args: + in_channels (int): Channels of the input feature map. + ratio (float): Ratio of channels of transform bottleneck + pooling_type (str): Pooling method for context modeling. + Options are 'att' and 'avg', stand for attention pooling and + average pooling respectively. Default: 'att'. + fusion_types (Sequence[str]): Fusion method for feature fusion, + Options are 'channels_add', 'channel_mul', stand for channelwise + addition and multiplication respectively. Default: ('channel_add',) + """ + + _abbr_ = 'context_block' + + def __init__(self, + in_channels, + ratio, + pooling_type='att', + fusion_types=('channel_add', )): + super(ContextBlock, self).__init__() + assert pooling_type in ['avg', 'att'] + assert isinstance(fusion_types, (list, tuple)) + valid_fusion_types = ['channel_add', 'channel_mul'] + assert all([f in valid_fusion_types for f in fusion_types]) + assert len(fusion_types) > 0, 'at least one fusion should be used' + self.in_channels = in_channels + self.ratio = ratio + self.planes = int(in_channels * ratio) + self.pooling_type = pooling_type + self.fusion_types = fusion_types + if pooling_type == 'att': + self.conv_mask = nn.Conv2d(in_channels, 1, kernel_size=1) + self.softmax = nn.Softmax(dim=2) + else: + self.avg_pool = nn.AdaptiveAvgPool2d(1) + if 'channel_add' in fusion_types: + self.channel_add_conv = nn.Sequential( + nn.Conv2d(self.in_channels, self.planes, kernel_size=1), + nn.LayerNorm([self.planes, 1, 1]), + nn.ReLU(inplace=True), # yapf: disable + nn.Conv2d(self.planes, self.in_channels, kernel_size=1)) + else: + self.channel_add_conv = None + if 'channel_mul' in fusion_types: + self.channel_mul_conv = nn.Sequential( + nn.Conv2d(self.in_channels, self.planes, kernel_size=1), + nn.LayerNorm([self.planes, 1, 1]), + nn.ReLU(inplace=True), # yapf: disable + nn.Conv2d(self.planes, self.in_channels, kernel_size=1)) + else: + self.channel_mul_conv = None + self.reset_parameters() + + def reset_parameters(self): + if self.pooling_type == 'att': + kaiming_init(self.conv_mask, mode='fan_in') + self.conv_mask.inited = True + + if self.channel_add_conv is not None: + last_zero_init(self.channel_add_conv) + if self.channel_mul_conv is not None: + last_zero_init(self.channel_mul_conv) + + def spatial_pool(self, x): + batch, channel, height, width = x.size() + if self.pooling_type == 'att': + input_x = x + # [N, C, H * W] + input_x = input_x.view(batch, channel, height * width) + # [N, 1, C, H * W] + input_x = input_x.unsqueeze(1) + # [N, 1, H, W] + context_mask = self.conv_mask(x) + # [N, 1, H * W] + context_mask = context_mask.view(batch, 1, height * width) + # [N, 1, H * W] + context_mask = self.softmax(context_mask) + # [N, 1, H * W, 1] + context_mask = context_mask.unsqueeze(-1) + # [N, 1, C, 1] + context = torch.matmul(input_x, context_mask) + # [N, C, 1, 1] + context = context.view(batch, channel, 1, 1) + else: + # [N, C, 1, 1] + context = self.avg_pool(x) + + return context + + def forward(self, x): + # [N, C, 1, 1] + context = self.spatial_pool(x) + + out = x + if self.channel_mul_conv is not None: + # [N, C, 1, 1] + channel_mul_term = torch.sigmoid(self.channel_mul_conv(context)) + out = out * channel_mul_term + if self.channel_add_conv is not None: + # [N, C, 1, 1] + channel_add_term = self.channel_add_conv(context) + out = out + channel_add_term + + return out diff --git a/annotator/mmpkg/mmcv/cnn/bricks/conv.py b/annotator/mmpkg/mmcv/cnn/bricks/conv.py new file mode 100644 index 0000000000000000000000000000000000000000..cf54491997a48ac3e7fadc4183ab7bf3e831024c --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/conv.py @@ -0,0 +1,44 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from torch import nn + +from .registry import CONV_LAYERS + +CONV_LAYERS.register_module('Conv1d', module=nn.Conv1d) +CONV_LAYERS.register_module('Conv2d', module=nn.Conv2d) +CONV_LAYERS.register_module('Conv3d', module=nn.Conv3d) +CONV_LAYERS.register_module('Conv', module=nn.Conv2d) + + +def build_conv_layer(cfg, *args, **kwargs): + """Build convolution layer. + + Args: + cfg (None or dict): The conv layer config, which should contain: + - type (str): Layer type. + - layer args: Args needed to instantiate an conv layer. + args (argument list): Arguments passed to the `__init__` + method of the corresponding conv layer. + kwargs (keyword arguments): Keyword arguments passed to the `__init__` + method of the corresponding conv layer. + + Returns: + nn.Module: Created conv layer. + """ + if cfg is None: + cfg_ = dict(type='Conv2d') + else: + if not isinstance(cfg, dict): + raise TypeError('cfg must be a dict') + if 'type' not in cfg: + raise KeyError('the cfg dict must contain the key "type"') + cfg_ = cfg.copy() + + layer_type = cfg_.pop('type') + if layer_type not in CONV_LAYERS: + raise KeyError(f'Unrecognized norm type {layer_type}') + else: + conv_layer = CONV_LAYERS.get(layer_type) + + layer = conv_layer(*args, **kwargs, **cfg_) + + return layer diff --git a/annotator/mmpkg/mmcv/cnn/bricks/conv2d_adaptive_padding.py b/annotator/mmpkg/mmcv/cnn/bricks/conv2d_adaptive_padding.py new file mode 100644 index 0000000000000000000000000000000000000000..b45e758ac6cf8dfb0382d072fe09125bc7e9b888 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/conv2d_adaptive_padding.py @@ -0,0 +1,62 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import math + +from torch import nn +from torch.nn import functional as F + +from .registry import CONV_LAYERS + + +@CONV_LAYERS.register_module() +class Conv2dAdaptivePadding(nn.Conv2d): + """Implementation of 2D convolution in tensorflow with `padding` as "same", + which applies padding to input (if needed) so that input image gets fully + covered by filter and stride you specified. For stride 1, this will ensure + that output image size is same as input. For stride of 2, output dimensions + will be half, for example. + + Args: + in_channels (int): Number of channels in the input image + out_channels (int): Number of channels produced by the convolution + kernel_size (int or tuple): Size of the convolving kernel + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of + the input. Default: 0 + dilation (int or tuple, optional): Spacing between kernel elements. + Default: 1 + groups (int, optional): Number of blocked connections from input + channels to output channels. Default: 1 + bias (bool, optional): If ``True``, adds a learnable bias to the + output. Default: ``True`` + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + bias=True): + super().__init__(in_channels, out_channels, kernel_size, stride, 0, + dilation, groups, bias) + + def forward(self, x): + img_h, img_w = x.size()[-2:] + kernel_h, kernel_w = self.weight.size()[-2:] + stride_h, stride_w = self.stride + output_h = math.ceil(img_h / stride_h) + output_w = math.ceil(img_w / stride_w) + pad_h = ( + max((output_h - 1) * self.stride[0] + + (kernel_h - 1) * self.dilation[0] + 1 - img_h, 0)) + pad_w = ( + max((output_w - 1) * self.stride[1] + + (kernel_w - 1) * self.dilation[1] + 1 - img_w, 0)) + if pad_h > 0 or pad_w > 0: + x = F.pad(x, [ + pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2 + ]) + return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, + self.dilation, self.groups) diff --git a/annotator/mmpkg/mmcv/cnn/bricks/conv_module.py b/annotator/mmpkg/mmcv/cnn/bricks/conv_module.py new file mode 100644 index 0000000000000000000000000000000000000000..43cab72624ccc04b2f7877383588a4bbacf9117a --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/conv_module.py @@ -0,0 +1,206 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import warnings + +import torch.nn as nn + +from annotator.mmpkg.mmcv.utils import _BatchNorm, _InstanceNorm +from ..utils import constant_init, kaiming_init +from .activation import build_activation_layer +from .conv import build_conv_layer +from .norm import build_norm_layer +from .padding import build_padding_layer +from .registry import PLUGIN_LAYERS + + +@PLUGIN_LAYERS.register_module() +class ConvModule(nn.Module): + """A conv block that bundles conv/norm/activation layers. + + This block simplifies the usage of convolution layers, which are commonly + used with a norm layer (e.g., BatchNorm) and activation layer (e.g., ReLU). + It is based upon three build methods: `build_conv_layer()`, + `build_norm_layer()` and `build_activation_layer()`. + + Besides, we add some additional features in this module. + 1. Automatically set `bias` of the conv layer. + 2. Spectral norm is supported. + 3. More padding modes are supported. Before PyTorch 1.5, nn.Conv2d only + supports zero and circular padding, and we add "reflect" padding mode. + + Args: + in_channels (int): Number of channels in the input feature map. + Same as that in ``nn._ConvNd``. + out_channels (int): Number of channels produced by the convolution. + Same as that in ``nn._ConvNd``. + kernel_size (int | tuple[int]): Size of the convolving kernel. + Same as that in ``nn._ConvNd``. + stride (int | tuple[int]): Stride of the convolution. + Same as that in ``nn._ConvNd``. + padding (int | tuple[int]): Zero-padding added to both sides of + the input. Same as that in ``nn._ConvNd``. + dilation (int | tuple[int]): Spacing between kernel elements. + Same as that in ``nn._ConvNd``. + groups (int): Number of blocked connections from input channels to + output channels. Same as that in ``nn._ConvNd``. + bias (bool | str): If specified as `auto`, it will be decided by the + norm_cfg. Bias will be set as True if `norm_cfg` is None, otherwise + False. Default: "auto". + conv_cfg (dict): Config dict for convolution layer. Default: None, + which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. Default: None. + act_cfg (dict): Config dict for activation layer. + Default: dict(type='ReLU'). + inplace (bool): Whether to use inplace mode for activation. + Default: True. + with_spectral_norm (bool): Whether use spectral norm in conv module. + Default: False. + padding_mode (str): If the `padding_mode` has not been supported by + current `Conv2d` in PyTorch, we will use our own padding layer + instead. Currently, we support ['zeros', 'circular'] with official + implementation and ['reflect'] with our own implementation. + Default: 'zeros'. + order (tuple[str]): The order of conv/norm/activation layers. It is a + sequence of "conv", "norm" and "act". Common examples are + ("conv", "norm", "act") and ("act", "conv", "norm"). + Default: ('conv', 'norm', 'act'). + """ + + _abbr_ = 'conv_block' + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + bias='auto', + conv_cfg=None, + norm_cfg=None, + act_cfg=dict(type='ReLU'), + inplace=True, + with_spectral_norm=False, + padding_mode='zeros', + order=('conv', 'norm', 'act')): + super(ConvModule, self).__init__() + assert conv_cfg is None or isinstance(conv_cfg, dict) + assert norm_cfg is None or isinstance(norm_cfg, dict) + assert act_cfg is None or isinstance(act_cfg, dict) + official_padding_mode = ['zeros', 'circular'] + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.inplace = inplace + self.with_spectral_norm = with_spectral_norm + self.with_explicit_padding = padding_mode not in official_padding_mode + self.order = order + assert isinstance(self.order, tuple) and len(self.order) == 3 + assert set(order) == set(['conv', 'norm', 'act']) + + self.with_norm = norm_cfg is not None + self.with_activation = act_cfg is not None + # if the conv layer is before a norm layer, bias is unnecessary. + if bias == 'auto': + bias = not self.with_norm + self.with_bias = bias + + if self.with_explicit_padding: + pad_cfg = dict(type=padding_mode) + self.padding_layer = build_padding_layer(pad_cfg, padding) + + # reset padding to 0 for conv module + conv_padding = 0 if self.with_explicit_padding else padding + # build convolution layer + self.conv = build_conv_layer( + conv_cfg, + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=conv_padding, + dilation=dilation, + groups=groups, + bias=bias) + # export the attributes of self.conv to a higher level for convenience + self.in_channels = self.conv.in_channels + self.out_channels = self.conv.out_channels + self.kernel_size = self.conv.kernel_size + self.stride = self.conv.stride + self.padding = padding + self.dilation = self.conv.dilation + self.transposed = self.conv.transposed + self.output_padding = self.conv.output_padding + self.groups = self.conv.groups + + if self.with_spectral_norm: + self.conv = nn.utils.spectral_norm(self.conv) + + # build normalization layers + if self.with_norm: + # norm layer is after conv layer + if order.index('norm') > order.index('conv'): + norm_channels = out_channels + else: + norm_channels = in_channels + self.norm_name, norm = build_norm_layer(norm_cfg, norm_channels) + self.add_module(self.norm_name, norm) + if self.with_bias: + if isinstance(norm, (_BatchNorm, _InstanceNorm)): + warnings.warn( + 'Unnecessary conv bias before batch/instance norm') + else: + self.norm_name = None + + # build activation layer + if self.with_activation: + act_cfg_ = act_cfg.copy() + # nn.Tanh has no 'inplace' argument + if act_cfg_['type'] not in [ + 'Tanh', 'PReLU', 'Sigmoid', 'HSigmoid', 'Swish' + ]: + act_cfg_.setdefault('inplace', inplace) + self.activate = build_activation_layer(act_cfg_) + + # Use msra init by default + self.init_weights() + + @property + def norm(self): + if self.norm_name: + return getattr(self, self.norm_name) + else: + return None + + def init_weights(self): + # 1. It is mainly for customized conv layers with their own + # initialization manners by calling their own ``init_weights()``, + # and we do not want ConvModule to override the initialization. + # 2. For customized conv layers without their own initialization + # manners (that is, they don't have their own ``init_weights()``) + # and PyTorch's conv layers, they will be initialized by + # this method with default ``kaiming_init``. + # Note: For PyTorch's conv layers, they will be overwritten by our + # initialization implementation using default ``kaiming_init``. + if not hasattr(self.conv, 'init_weights'): + if self.with_activation and self.act_cfg['type'] == 'LeakyReLU': + nonlinearity = 'leaky_relu' + a = self.act_cfg.get('negative_slope', 0.01) + else: + nonlinearity = 'relu' + a = 0 + kaiming_init(self.conv, a=a, nonlinearity=nonlinearity) + if self.with_norm: + constant_init(self.norm, 1, bias=0) + + def forward(self, x, activate=True, norm=True): + for layer in self.order: + if layer == 'conv': + if self.with_explicit_padding: + x = self.padding_layer(x) + x = self.conv(x) + elif layer == 'norm' and norm and self.with_norm: + x = self.norm(x) + elif layer == 'act' and activate and self.with_activation: + x = self.activate(x) + return x diff --git a/annotator/mmpkg/mmcv/cnn/bricks/conv_ws.py b/annotator/mmpkg/mmcv/cnn/bricks/conv_ws.py new file mode 100644 index 0000000000000000000000000000000000000000..a3941e27874993418b3b5708d5a7485f175ff9c8 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/conv_ws.py @@ -0,0 +1,148 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .registry import CONV_LAYERS + + +def conv_ws_2d(input, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1, + eps=1e-5): + c_in = weight.size(0) + weight_flat = weight.view(c_in, -1) + mean = weight_flat.mean(dim=1, keepdim=True).view(c_in, 1, 1, 1) + std = weight_flat.std(dim=1, keepdim=True).view(c_in, 1, 1, 1) + weight = (weight - mean) / (std + eps) + return F.conv2d(input, weight, bias, stride, padding, dilation, groups) + + +@CONV_LAYERS.register_module('ConvWS') +class ConvWS2d(nn.Conv2d): + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + bias=True, + eps=1e-5): + super(ConvWS2d, self).__init__( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + bias=bias) + self.eps = eps + + def forward(self, x): + return conv_ws_2d(x, self.weight, self.bias, self.stride, self.padding, + self.dilation, self.groups, self.eps) + + +@CONV_LAYERS.register_module(name='ConvAWS') +class ConvAWS2d(nn.Conv2d): + """AWS (Adaptive Weight Standardization) + + This is a variant of Weight Standardization + (https://arxiv.org/pdf/1903.10520.pdf) + It is used in DetectoRS to avoid NaN + (https://arxiv.org/pdf/2006.02334.pdf) + + Args: + in_channels (int): Number of channels in the input image + out_channels (int): Number of channels produced by the convolution + kernel_size (int or tuple): Size of the conv kernel + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of + the input. Default: 0 + dilation (int or tuple, optional): Spacing between kernel elements. + Default: 1 + groups (int, optional): Number of blocked connections from input + channels to output channels. Default: 1 + bias (bool, optional): If set True, adds a learnable bias to the + output. Default: True + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + bias=True): + super().__init__( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + bias=bias) + self.register_buffer('weight_gamma', + torch.ones(self.out_channels, 1, 1, 1)) + self.register_buffer('weight_beta', + torch.zeros(self.out_channels, 1, 1, 1)) + + def _get_weight(self, weight): + weight_flat = weight.view(weight.size(0), -1) + mean = weight_flat.mean(dim=1).view(-1, 1, 1, 1) + std = torch.sqrt(weight_flat.var(dim=1) + 1e-5).view(-1, 1, 1, 1) + weight = (weight - mean) / std + weight = self.weight_gamma * weight + self.weight_beta + return weight + + def forward(self, x): + weight = self._get_weight(self.weight) + return F.conv2d(x, weight, self.bias, self.stride, self.padding, + self.dilation, self.groups) + + def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs): + """Override default load function. + + AWS overrides the function _load_from_state_dict to recover + weight_gamma and weight_beta if they are missing. If weight_gamma and + weight_beta are found in the checkpoint, this function will return + after super()._load_from_state_dict. Otherwise, it will compute the + mean and std of the pretrained weights and store them in weight_beta + and weight_gamma. + """ + + self.weight_gamma.data.fill_(-1) + local_missing_keys = [] + super()._load_from_state_dict(state_dict, prefix, local_metadata, + strict, local_missing_keys, + unexpected_keys, error_msgs) + if self.weight_gamma.data.mean() > 0: + for k in local_missing_keys: + missing_keys.append(k) + return + weight = self.weight.data + weight_flat = weight.view(weight.size(0), -1) + mean = weight_flat.mean(dim=1).view(-1, 1, 1, 1) + std = torch.sqrt(weight_flat.var(dim=1) + 1e-5).view(-1, 1, 1, 1) + self.weight_beta.data.copy_(mean) + self.weight_gamma.data.copy_(std) + missing_gamma_beta = [ + k for k in local_missing_keys + if k.endswith('weight_gamma') or k.endswith('weight_beta') + ] + for k in missing_gamma_beta: + local_missing_keys.remove(k) + for k in local_missing_keys: + missing_keys.append(k) diff --git a/annotator/mmpkg/mmcv/cnn/bricks/depthwise_separable_conv_module.py b/annotator/mmpkg/mmcv/cnn/bricks/depthwise_separable_conv_module.py new file mode 100644 index 0000000000000000000000000000000000000000..722d5d8d71f75486e2db3008907c4eadfca41d63 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/depthwise_separable_conv_module.py @@ -0,0 +1,96 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch.nn as nn + +from .conv_module import ConvModule + + +class DepthwiseSeparableConvModule(nn.Module): + """Depthwise separable convolution module. + + See https://arxiv.org/pdf/1704.04861.pdf for details. + + This module can replace a ConvModule with the conv block replaced by two + conv block: depthwise conv block and pointwise conv block. The depthwise + conv block contains depthwise-conv/norm/activation layers. The pointwise + conv block contains pointwise-conv/norm/activation layers. It should be + noted that there will be norm/activation layer in the depthwise conv block + if `norm_cfg` and `act_cfg` are specified. + + Args: + in_channels (int): Number of channels in the input feature map. + Same as that in ``nn._ConvNd``. + out_channels (int): Number of channels produced by the convolution. + Same as that in ``nn._ConvNd``. + kernel_size (int | tuple[int]): Size of the convolving kernel. + Same as that in ``nn._ConvNd``. + stride (int | tuple[int]): Stride of the convolution. + Same as that in ``nn._ConvNd``. Default: 1. + padding (int | tuple[int]): Zero-padding added to both sides of + the input. Same as that in ``nn._ConvNd``. Default: 0. + dilation (int | tuple[int]): Spacing between kernel elements. + Same as that in ``nn._ConvNd``. Default: 1. + norm_cfg (dict): Default norm config for both depthwise ConvModule and + pointwise ConvModule. Default: None. + act_cfg (dict): Default activation config for both depthwise ConvModule + and pointwise ConvModule. Default: dict(type='ReLU'). + dw_norm_cfg (dict): Norm config of depthwise ConvModule. If it is + 'default', it will be the same as `norm_cfg`. Default: 'default'. + dw_act_cfg (dict): Activation config of depthwise ConvModule. If it is + 'default', it will be the same as `act_cfg`. Default: 'default'. + pw_norm_cfg (dict): Norm config of pointwise ConvModule. If it is + 'default', it will be the same as `norm_cfg`. Default: 'default'. + pw_act_cfg (dict): Activation config of pointwise ConvModule. If it is + 'default', it will be the same as `act_cfg`. Default: 'default'. + kwargs (optional): Other shared arguments for depthwise and pointwise + ConvModule. See ConvModule for ref. + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + norm_cfg=None, + act_cfg=dict(type='ReLU'), + dw_norm_cfg='default', + dw_act_cfg='default', + pw_norm_cfg='default', + pw_act_cfg='default', + **kwargs): + super(DepthwiseSeparableConvModule, self).__init__() + assert 'groups' not in kwargs, 'groups should not be specified' + + # if norm/activation config of depthwise/pointwise ConvModule is not + # specified, use default config. + dw_norm_cfg = dw_norm_cfg if dw_norm_cfg != 'default' else norm_cfg + dw_act_cfg = dw_act_cfg if dw_act_cfg != 'default' else act_cfg + pw_norm_cfg = pw_norm_cfg if pw_norm_cfg != 'default' else norm_cfg + pw_act_cfg = pw_act_cfg if pw_act_cfg != 'default' else act_cfg + + # depthwise convolution + self.depthwise_conv = ConvModule( + in_channels, + in_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=in_channels, + norm_cfg=dw_norm_cfg, + act_cfg=dw_act_cfg, + **kwargs) + + self.pointwise_conv = ConvModule( + in_channels, + out_channels, + 1, + norm_cfg=pw_norm_cfg, + act_cfg=pw_act_cfg, + **kwargs) + + def forward(self, x): + x = self.depthwise_conv(x) + x = self.pointwise_conv(x) + return x diff --git a/annotator/mmpkg/mmcv/cnn/bricks/drop.py b/annotator/mmpkg/mmcv/cnn/bricks/drop.py new file mode 100644 index 0000000000000000000000000000000000000000..465ed38339fe64dde8cdc959451b1236a3a55b95 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/drop.py @@ -0,0 +1,65 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn + +from annotator.mmpkg.mmcv import build_from_cfg +from .registry import DROPOUT_LAYERS + + +def drop_path(x, drop_prob=0., training=False): + """Drop paths (Stochastic Depth) per sample (when applied in main path of + residual blocks). + + We follow the implementation + https://github.com/rwightman/pytorch-image-models/blob/a2727c1bf78ba0d7b5727f5f95e37fb7f8866b1f/timm/models/layers/drop.py # noqa: E501 + """ + if drop_prob == 0. or not training: + return x + keep_prob = 1 - drop_prob + # handle tensors with different dimensions, not just 4D tensors. + shape = (x.shape[0], ) + (1, ) * (x.ndim - 1) + random_tensor = keep_prob + torch.rand( + shape, dtype=x.dtype, device=x.device) + output = x.div(keep_prob) * random_tensor.floor() + return output + + +@DROPOUT_LAYERS.register_module() +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of + residual blocks). + + We follow the implementation + https://github.com/rwightman/pytorch-image-models/blob/a2727c1bf78ba0d7b5727f5f95e37fb7f8866b1f/timm/models/layers/drop.py # noqa: E501 + + Args: + drop_prob (float): Probability of the path to be zeroed. Default: 0.1 + """ + + def __init__(self, drop_prob=0.1): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) + + +@DROPOUT_LAYERS.register_module() +class Dropout(nn.Dropout): + """A wrapper for ``torch.nn.Dropout``, We rename the ``p`` of + ``torch.nn.Dropout`` to ``drop_prob`` so as to be consistent with + ``DropPath`` + + Args: + drop_prob (float): Probability of the elements to be + zeroed. Default: 0.5. + inplace (bool): Do the operation inplace or not. Default: False. + """ + + def __init__(self, drop_prob=0.5, inplace=False): + super().__init__(p=drop_prob, inplace=inplace) + + +def build_dropout(cfg, default_args=None): + """Builder for drop out layers.""" + return build_from_cfg(cfg, DROPOUT_LAYERS, default_args) diff --git a/annotator/mmpkg/mmcv/cnn/bricks/generalized_attention.py b/annotator/mmpkg/mmcv/cnn/bricks/generalized_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..988d9adf2f289ef223bd1c680a5ae1d3387f0269 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/generalized_attention.py @@ -0,0 +1,412 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import math + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..utils import kaiming_init +from .registry import PLUGIN_LAYERS + + +@PLUGIN_LAYERS.register_module() +class GeneralizedAttention(nn.Module): + """GeneralizedAttention module. + + See 'An Empirical Study of Spatial Attention Mechanisms in Deep Networks' + (https://arxiv.org/abs/1711.07971) for details. + + Args: + in_channels (int): Channels of the input feature map. + spatial_range (int): The spatial range. -1 indicates no spatial range + constraint. Default: -1. + num_heads (int): The head number of empirical_attention module. + Default: 9. + position_embedding_dim (int): The position embedding dimension. + Default: -1. + position_magnitude (int): A multiplier acting on coord difference. + Default: 1. + kv_stride (int): The feature stride acting on key/value feature map. + Default: 2. + q_stride (int): The feature stride acting on query feature map. + Default: 1. + attention_type (str): A binary indicator string for indicating which + items in generalized empirical_attention module are used. + Default: '1111'. + + - '1000' indicates 'query and key content' (appr - appr) item, + - '0100' indicates 'query content and relative position' + (appr - position) item, + - '0010' indicates 'key content only' (bias - appr) item, + - '0001' indicates 'relative position only' (bias - position) item. + """ + + _abbr_ = 'gen_attention_block' + + def __init__(self, + in_channels, + spatial_range=-1, + num_heads=9, + position_embedding_dim=-1, + position_magnitude=1, + kv_stride=2, + q_stride=1, + attention_type='1111'): + + super(GeneralizedAttention, self).__init__() + + # hard range means local range for non-local operation + self.position_embedding_dim = ( + position_embedding_dim + if position_embedding_dim > 0 else in_channels) + + self.position_magnitude = position_magnitude + self.num_heads = num_heads + self.in_channels = in_channels + self.spatial_range = spatial_range + self.kv_stride = kv_stride + self.q_stride = q_stride + self.attention_type = [bool(int(_)) for _ in attention_type] + self.qk_embed_dim = in_channels // num_heads + out_c = self.qk_embed_dim * num_heads + + if self.attention_type[0] or self.attention_type[1]: + self.query_conv = nn.Conv2d( + in_channels=in_channels, + out_channels=out_c, + kernel_size=1, + bias=False) + self.query_conv.kaiming_init = True + + if self.attention_type[0] or self.attention_type[2]: + self.key_conv = nn.Conv2d( + in_channels=in_channels, + out_channels=out_c, + kernel_size=1, + bias=False) + self.key_conv.kaiming_init = True + + self.v_dim = in_channels // num_heads + self.value_conv = nn.Conv2d( + in_channels=in_channels, + out_channels=self.v_dim * num_heads, + kernel_size=1, + bias=False) + self.value_conv.kaiming_init = True + + if self.attention_type[1] or self.attention_type[3]: + self.appr_geom_fc_x = nn.Linear( + self.position_embedding_dim // 2, out_c, bias=False) + self.appr_geom_fc_x.kaiming_init = True + + self.appr_geom_fc_y = nn.Linear( + self.position_embedding_dim // 2, out_c, bias=False) + self.appr_geom_fc_y.kaiming_init = True + + if self.attention_type[2]: + stdv = 1.0 / math.sqrt(self.qk_embed_dim * 2) + appr_bias_value = -2 * stdv * torch.rand(out_c) + stdv + self.appr_bias = nn.Parameter(appr_bias_value) + + if self.attention_type[3]: + stdv = 1.0 / math.sqrt(self.qk_embed_dim * 2) + geom_bias_value = -2 * stdv * torch.rand(out_c) + stdv + self.geom_bias = nn.Parameter(geom_bias_value) + + self.proj_conv = nn.Conv2d( + in_channels=self.v_dim * num_heads, + out_channels=in_channels, + kernel_size=1, + bias=True) + self.proj_conv.kaiming_init = True + self.gamma = nn.Parameter(torch.zeros(1)) + + if self.spatial_range >= 0: + # only works when non local is after 3*3 conv + if in_channels == 256: + max_len = 84 + elif in_channels == 512: + max_len = 42 + + max_len_kv = int((max_len - 1.0) / self.kv_stride + 1) + local_constraint_map = np.ones( + (max_len, max_len, max_len_kv, max_len_kv), dtype=np.int) + for iy in range(max_len): + for ix in range(max_len): + local_constraint_map[ + iy, ix, + max((iy - self.spatial_range) // + self.kv_stride, 0):min((iy + self.spatial_range + + 1) // self.kv_stride + + 1, max_len), + max((ix - self.spatial_range) // + self.kv_stride, 0):min((ix + self.spatial_range + + 1) // self.kv_stride + + 1, max_len)] = 0 + + self.local_constraint_map = nn.Parameter( + torch.from_numpy(local_constraint_map).byte(), + requires_grad=False) + + if self.q_stride > 1: + self.q_downsample = nn.AvgPool2d( + kernel_size=1, stride=self.q_stride) + else: + self.q_downsample = None + + if self.kv_stride > 1: + self.kv_downsample = nn.AvgPool2d( + kernel_size=1, stride=self.kv_stride) + else: + self.kv_downsample = None + + self.init_weights() + + def get_position_embedding(self, + h, + w, + h_kv, + w_kv, + q_stride, + kv_stride, + device, + dtype, + feat_dim, + wave_length=1000): + # the default type of Tensor is float32, leading to type mismatch + # in fp16 mode. Cast it to support fp16 mode. + h_idxs = torch.linspace(0, h - 1, h).to(device=device, dtype=dtype) + h_idxs = h_idxs.view((h, 1)) * q_stride + + w_idxs = torch.linspace(0, w - 1, w).to(device=device, dtype=dtype) + w_idxs = w_idxs.view((w, 1)) * q_stride + + h_kv_idxs = torch.linspace(0, h_kv - 1, h_kv).to( + device=device, dtype=dtype) + h_kv_idxs = h_kv_idxs.view((h_kv, 1)) * kv_stride + + w_kv_idxs = torch.linspace(0, w_kv - 1, w_kv).to( + device=device, dtype=dtype) + w_kv_idxs = w_kv_idxs.view((w_kv, 1)) * kv_stride + + # (h, h_kv, 1) + h_diff = h_idxs.unsqueeze(1) - h_kv_idxs.unsqueeze(0) + h_diff *= self.position_magnitude + + # (w, w_kv, 1) + w_diff = w_idxs.unsqueeze(1) - w_kv_idxs.unsqueeze(0) + w_diff *= self.position_magnitude + + feat_range = torch.arange(0, feat_dim / 4).to( + device=device, dtype=dtype) + + dim_mat = torch.Tensor([wave_length]).to(device=device, dtype=dtype) + dim_mat = dim_mat**((4. / feat_dim) * feat_range) + dim_mat = dim_mat.view((1, 1, -1)) + + embedding_x = torch.cat( + ((w_diff / dim_mat).sin(), (w_diff / dim_mat).cos()), dim=2) + + embedding_y = torch.cat( + ((h_diff / dim_mat).sin(), (h_diff / dim_mat).cos()), dim=2) + + return embedding_x, embedding_y + + def forward(self, x_input): + num_heads = self.num_heads + + # use empirical_attention + if self.q_downsample is not None: + x_q = self.q_downsample(x_input) + else: + x_q = x_input + n, _, h, w = x_q.shape + + if self.kv_downsample is not None: + x_kv = self.kv_downsample(x_input) + else: + x_kv = x_input + _, _, h_kv, w_kv = x_kv.shape + + if self.attention_type[0] or self.attention_type[1]: + proj_query = self.query_conv(x_q).view( + (n, num_heads, self.qk_embed_dim, h * w)) + proj_query = proj_query.permute(0, 1, 3, 2) + + if self.attention_type[0] or self.attention_type[2]: + proj_key = self.key_conv(x_kv).view( + (n, num_heads, self.qk_embed_dim, h_kv * w_kv)) + + if self.attention_type[1] or self.attention_type[3]: + position_embed_x, position_embed_y = self.get_position_embedding( + h, w, h_kv, w_kv, self.q_stride, self.kv_stride, + x_input.device, x_input.dtype, self.position_embedding_dim) + # (n, num_heads, w, w_kv, dim) + position_feat_x = self.appr_geom_fc_x(position_embed_x).\ + view(1, w, w_kv, num_heads, self.qk_embed_dim).\ + permute(0, 3, 1, 2, 4).\ + repeat(n, 1, 1, 1, 1) + + # (n, num_heads, h, h_kv, dim) + position_feat_y = self.appr_geom_fc_y(position_embed_y).\ + view(1, h, h_kv, num_heads, self.qk_embed_dim).\ + permute(0, 3, 1, 2, 4).\ + repeat(n, 1, 1, 1, 1) + + position_feat_x /= math.sqrt(2) + position_feat_y /= math.sqrt(2) + + # accelerate for saliency only + if (np.sum(self.attention_type) == 1) and self.attention_type[2]: + appr_bias = self.appr_bias.\ + view(1, num_heads, 1, self.qk_embed_dim).\ + repeat(n, 1, 1, 1) + + energy = torch.matmul(appr_bias, proj_key).\ + view(n, num_heads, 1, h_kv * w_kv) + + h = 1 + w = 1 + else: + # (n, num_heads, h*w, h_kv*w_kv), query before key, 540mb for + if not self.attention_type[0]: + energy = torch.zeros( + n, + num_heads, + h, + w, + h_kv, + w_kv, + dtype=x_input.dtype, + device=x_input.device) + + # attention_type[0]: appr - appr + # attention_type[1]: appr - position + # attention_type[2]: bias - appr + # attention_type[3]: bias - position + if self.attention_type[0] or self.attention_type[2]: + if self.attention_type[0] and self.attention_type[2]: + appr_bias = self.appr_bias.\ + view(1, num_heads, 1, self.qk_embed_dim) + energy = torch.matmul(proj_query + appr_bias, proj_key).\ + view(n, num_heads, h, w, h_kv, w_kv) + + elif self.attention_type[0]: + energy = torch.matmul(proj_query, proj_key).\ + view(n, num_heads, h, w, h_kv, w_kv) + + elif self.attention_type[2]: + appr_bias = self.appr_bias.\ + view(1, num_heads, 1, self.qk_embed_dim).\ + repeat(n, 1, 1, 1) + + energy += torch.matmul(appr_bias, proj_key).\ + view(n, num_heads, 1, 1, h_kv, w_kv) + + if self.attention_type[1] or self.attention_type[3]: + if self.attention_type[1] and self.attention_type[3]: + geom_bias = self.geom_bias.\ + view(1, num_heads, 1, self.qk_embed_dim) + + proj_query_reshape = (proj_query + geom_bias).\ + view(n, num_heads, h, w, self.qk_embed_dim) + + energy_x = torch.matmul( + proj_query_reshape.permute(0, 1, 3, 2, 4), + position_feat_x.permute(0, 1, 2, 4, 3)) + energy_x = energy_x.\ + permute(0, 1, 3, 2, 4).unsqueeze(4) + + energy_y = torch.matmul( + proj_query_reshape, + position_feat_y.permute(0, 1, 2, 4, 3)) + energy_y = energy_y.unsqueeze(5) + + energy += energy_x + energy_y + + elif self.attention_type[1]: + proj_query_reshape = proj_query.\ + view(n, num_heads, h, w, self.qk_embed_dim) + proj_query_reshape = proj_query_reshape.\ + permute(0, 1, 3, 2, 4) + position_feat_x_reshape = position_feat_x.\ + permute(0, 1, 2, 4, 3) + position_feat_y_reshape = position_feat_y.\ + permute(0, 1, 2, 4, 3) + + energy_x = torch.matmul(proj_query_reshape, + position_feat_x_reshape) + energy_x = energy_x.permute(0, 1, 3, 2, 4).unsqueeze(4) + + energy_y = torch.matmul(proj_query_reshape, + position_feat_y_reshape) + energy_y = energy_y.unsqueeze(5) + + energy += energy_x + energy_y + + elif self.attention_type[3]: + geom_bias = self.geom_bias.\ + view(1, num_heads, self.qk_embed_dim, 1).\ + repeat(n, 1, 1, 1) + + position_feat_x_reshape = position_feat_x.\ + view(n, num_heads, w*w_kv, self.qk_embed_dim) + + position_feat_y_reshape = position_feat_y.\ + view(n, num_heads, h * h_kv, self.qk_embed_dim) + + energy_x = torch.matmul(position_feat_x_reshape, geom_bias) + energy_x = energy_x.view(n, num_heads, 1, w, 1, w_kv) + + energy_y = torch.matmul(position_feat_y_reshape, geom_bias) + energy_y = energy_y.view(n, num_heads, h, 1, h_kv, 1) + + energy += energy_x + energy_y + + energy = energy.view(n, num_heads, h * w, h_kv * w_kv) + + if self.spatial_range >= 0: + cur_local_constraint_map = \ + self.local_constraint_map[:h, :w, :h_kv, :w_kv].\ + contiguous().\ + view(1, 1, h*w, h_kv*w_kv) + + energy = energy.masked_fill_(cur_local_constraint_map, + float('-inf')) + + attention = F.softmax(energy, 3) + + proj_value = self.value_conv(x_kv) + proj_value_reshape = proj_value.\ + view((n, num_heads, self.v_dim, h_kv * w_kv)).\ + permute(0, 1, 3, 2) + + out = torch.matmul(attention, proj_value_reshape).\ + permute(0, 1, 3, 2).\ + contiguous().\ + view(n, self.v_dim * self.num_heads, h, w) + + out = self.proj_conv(out) + + # output is downsampled, upsample back to input size + if self.q_downsample is not None: + out = F.interpolate( + out, + size=x_input.shape[2:], + mode='bilinear', + align_corners=False) + + out = self.gamma * out + x_input + return out + + def init_weights(self): + for m in self.modules(): + if hasattr(m, 'kaiming_init') and m.kaiming_init: + kaiming_init( + m, + mode='fan_in', + nonlinearity='leaky_relu', + bias=0, + distribution='uniform', + a=1) diff --git a/annotator/mmpkg/mmcv/cnn/bricks/hsigmoid.py b/annotator/mmpkg/mmcv/cnn/bricks/hsigmoid.py new file mode 100644 index 0000000000000000000000000000000000000000..30b1a3d6580cf0360710426fbea1f05acdf07b4b --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/hsigmoid.py @@ -0,0 +1,34 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch.nn as nn + +from .registry import ACTIVATION_LAYERS + + +@ACTIVATION_LAYERS.register_module() +class HSigmoid(nn.Module): + """Hard Sigmoid Module. Apply the hard sigmoid function: + Hsigmoid(x) = min(max((x + bias) / divisor, min_value), max_value) + Default: Hsigmoid(x) = min(max((x + 1) / 2, 0), 1) + + Args: + bias (float): Bias of the input feature map. Default: 1.0. + divisor (float): Divisor of the input feature map. Default: 2.0. + min_value (float): Lower bound value. Default: 0.0. + max_value (float): Upper bound value. Default: 1.0. + + Returns: + Tensor: The output tensor. + """ + + def __init__(self, bias=1.0, divisor=2.0, min_value=0.0, max_value=1.0): + super(HSigmoid, self).__init__() + self.bias = bias + self.divisor = divisor + assert self.divisor != 0 + self.min_value = min_value + self.max_value = max_value + + def forward(self, x): + x = (x + self.bias) / self.divisor + + return x.clamp_(self.min_value, self.max_value) diff --git a/annotator/mmpkg/mmcv/cnn/bricks/hswish.py b/annotator/mmpkg/mmcv/cnn/bricks/hswish.py new file mode 100644 index 0000000000000000000000000000000000000000..7e0c090ff037c99ee6c5c84c4592e87beae02208 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/hswish.py @@ -0,0 +1,29 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch.nn as nn + +from .registry import ACTIVATION_LAYERS + + +@ACTIVATION_LAYERS.register_module() +class HSwish(nn.Module): + """Hard Swish Module. + + This module applies the hard swish function: + + .. math:: + Hswish(x) = x * ReLU6(x + 3) / 6 + + Args: + inplace (bool): can optionally do the operation in-place. + Default: False. + + Returns: + Tensor: The output tensor. + """ + + def __init__(self, inplace=False): + super(HSwish, self).__init__() + self.act = nn.ReLU6(inplace) + + def forward(self, x): + return x * self.act(x + 3) / 6 diff --git a/annotator/mmpkg/mmcv/cnn/bricks/non_local.py b/annotator/mmpkg/mmcv/cnn/bricks/non_local.py new file mode 100644 index 0000000000000000000000000000000000000000..92d00155ef275c1201ea66bba30470a1785cc5d7 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/non_local.py @@ -0,0 +1,306 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from abc import ABCMeta + +import torch +import torch.nn as nn + +from ..utils import constant_init, normal_init +from .conv_module import ConvModule +from .registry import PLUGIN_LAYERS + + +class _NonLocalNd(nn.Module, metaclass=ABCMeta): + """Basic Non-local module. + + This module is proposed in + "Non-local Neural Networks" + Paper reference: https://arxiv.org/abs/1711.07971 + Code reference: https://github.com/AlexHex7/Non-local_pytorch + + Args: + in_channels (int): Channels of the input feature map. + reduction (int): Channel reduction ratio. Default: 2. + use_scale (bool): Whether to scale pairwise_weight by + `1/sqrt(inter_channels)` when the mode is `embedded_gaussian`. + Default: True. + conv_cfg (None | dict): The config dict for convolution layers. + If not specified, it will use `nn.Conv2d` for convolution layers. + Default: None. + norm_cfg (None | dict): The config dict for normalization layers. + Default: None. (This parameter is only applicable to conv_out.) + mode (str): Options are `gaussian`, `concatenation`, + `embedded_gaussian` and `dot_product`. Default: embedded_gaussian. + """ + + def __init__(self, + in_channels, + reduction=2, + use_scale=True, + conv_cfg=None, + norm_cfg=None, + mode='embedded_gaussian', + **kwargs): + super(_NonLocalNd, self).__init__() + self.in_channels = in_channels + self.reduction = reduction + self.use_scale = use_scale + self.inter_channels = max(in_channels // reduction, 1) + self.mode = mode + + if mode not in [ + 'gaussian', 'embedded_gaussian', 'dot_product', 'concatenation' + ]: + raise ValueError("Mode should be in 'gaussian', 'concatenation', " + f"'embedded_gaussian' or 'dot_product', but got " + f'{mode} instead.') + + # g, theta, phi are defaulted as `nn.ConvNd`. + # Here we use ConvModule for potential usage. + self.g = ConvModule( + self.in_channels, + self.inter_channels, + kernel_size=1, + conv_cfg=conv_cfg, + act_cfg=None) + self.conv_out = ConvModule( + self.inter_channels, + self.in_channels, + kernel_size=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=None) + + if self.mode != 'gaussian': + self.theta = ConvModule( + self.in_channels, + self.inter_channels, + kernel_size=1, + conv_cfg=conv_cfg, + act_cfg=None) + self.phi = ConvModule( + self.in_channels, + self.inter_channels, + kernel_size=1, + conv_cfg=conv_cfg, + act_cfg=None) + + if self.mode == 'concatenation': + self.concat_project = ConvModule( + self.inter_channels * 2, + 1, + kernel_size=1, + stride=1, + padding=0, + bias=False, + act_cfg=dict(type='ReLU')) + + self.init_weights(**kwargs) + + def init_weights(self, std=0.01, zeros_init=True): + if self.mode != 'gaussian': + for m in [self.g, self.theta, self.phi]: + normal_init(m.conv, std=std) + else: + normal_init(self.g.conv, std=std) + if zeros_init: + if self.conv_out.norm_cfg is None: + constant_init(self.conv_out.conv, 0) + else: + constant_init(self.conv_out.norm, 0) + else: + if self.conv_out.norm_cfg is None: + normal_init(self.conv_out.conv, std=std) + else: + normal_init(self.conv_out.norm, std=std) + + def gaussian(self, theta_x, phi_x): + # NonLocal1d pairwise_weight: [N, H, H] + # NonLocal2d pairwise_weight: [N, HxW, HxW] + # NonLocal3d pairwise_weight: [N, TxHxW, TxHxW] + pairwise_weight = torch.matmul(theta_x, phi_x) + pairwise_weight = pairwise_weight.softmax(dim=-1) + return pairwise_weight + + def embedded_gaussian(self, theta_x, phi_x): + # NonLocal1d pairwise_weight: [N, H, H] + # NonLocal2d pairwise_weight: [N, HxW, HxW] + # NonLocal3d pairwise_weight: [N, TxHxW, TxHxW] + pairwise_weight = torch.matmul(theta_x, phi_x) + if self.use_scale: + # theta_x.shape[-1] is `self.inter_channels` + pairwise_weight /= theta_x.shape[-1]**0.5 + pairwise_weight = pairwise_weight.softmax(dim=-1) + return pairwise_weight + + def dot_product(self, theta_x, phi_x): + # NonLocal1d pairwise_weight: [N, H, H] + # NonLocal2d pairwise_weight: [N, HxW, HxW] + # NonLocal3d pairwise_weight: [N, TxHxW, TxHxW] + pairwise_weight = torch.matmul(theta_x, phi_x) + pairwise_weight /= pairwise_weight.shape[-1] + return pairwise_weight + + def concatenation(self, theta_x, phi_x): + # NonLocal1d pairwise_weight: [N, H, H] + # NonLocal2d pairwise_weight: [N, HxW, HxW] + # NonLocal3d pairwise_weight: [N, TxHxW, TxHxW] + h = theta_x.size(2) + w = phi_x.size(3) + theta_x = theta_x.repeat(1, 1, 1, w) + phi_x = phi_x.repeat(1, 1, h, 1) + + concat_feature = torch.cat([theta_x, phi_x], dim=1) + pairwise_weight = self.concat_project(concat_feature) + n, _, h, w = pairwise_weight.size() + pairwise_weight = pairwise_weight.view(n, h, w) + pairwise_weight /= pairwise_weight.shape[-1] + + return pairwise_weight + + def forward(self, x): + # Assume `reduction = 1`, then `inter_channels = C` + # or `inter_channels = C` when `mode="gaussian"` + + # NonLocal1d x: [N, C, H] + # NonLocal2d x: [N, C, H, W] + # NonLocal3d x: [N, C, T, H, W] + n = x.size(0) + + # NonLocal1d g_x: [N, H, C] + # NonLocal2d g_x: [N, HxW, C] + # NonLocal3d g_x: [N, TxHxW, C] + g_x = self.g(x).view(n, self.inter_channels, -1) + g_x = g_x.permute(0, 2, 1) + + # NonLocal1d theta_x: [N, H, C], phi_x: [N, C, H] + # NonLocal2d theta_x: [N, HxW, C], phi_x: [N, C, HxW] + # NonLocal3d theta_x: [N, TxHxW, C], phi_x: [N, C, TxHxW] + if self.mode == 'gaussian': + theta_x = x.view(n, self.in_channels, -1) + theta_x = theta_x.permute(0, 2, 1) + if self.sub_sample: + phi_x = self.phi(x).view(n, self.in_channels, -1) + else: + phi_x = x.view(n, self.in_channels, -1) + elif self.mode == 'concatenation': + theta_x = self.theta(x).view(n, self.inter_channels, -1, 1) + phi_x = self.phi(x).view(n, self.inter_channels, 1, -1) + else: + theta_x = self.theta(x).view(n, self.inter_channels, -1) + theta_x = theta_x.permute(0, 2, 1) + phi_x = self.phi(x).view(n, self.inter_channels, -1) + + pairwise_func = getattr(self, self.mode) + # NonLocal1d pairwise_weight: [N, H, H] + # NonLocal2d pairwise_weight: [N, HxW, HxW] + # NonLocal3d pairwise_weight: [N, TxHxW, TxHxW] + pairwise_weight = pairwise_func(theta_x, phi_x) + + # NonLocal1d y: [N, H, C] + # NonLocal2d y: [N, HxW, C] + # NonLocal3d y: [N, TxHxW, C] + y = torch.matmul(pairwise_weight, g_x) + # NonLocal1d y: [N, C, H] + # NonLocal2d y: [N, C, H, W] + # NonLocal3d y: [N, C, T, H, W] + y = y.permute(0, 2, 1).contiguous().reshape(n, self.inter_channels, + *x.size()[2:]) + + output = x + self.conv_out(y) + + return output + + +class NonLocal1d(_NonLocalNd): + """1D Non-local module. + + Args: + in_channels (int): Same as `NonLocalND`. + sub_sample (bool): Whether to apply max pooling after pairwise + function (Note that the `sub_sample` is applied on spatial only). + Default: False. + conv_cfg (None | dict): Same as `NonLocalND`. + Default: dict(type='Conv1d'). + """ + + def __init__(self, + in_channels, + sub_sample=False, + conv_cfg=dict(type='Conv1d'), + **kwargs): + super(NonLocal1d, self).__init__( + in_channels, conv_cfg=conv_cfg, **kwargs) + + self.sub_sample = sub_sample + + if sub_sample: + max_pool_layer = nn.MaxPool1d(kernel_size=2) + self.g = nn.Sequential(self.g, max_pool_layer) + if self.mode != 'gaussian': + self.phi = nn.Sequential(self.phi, max_pool_layer) + else: + self.phi = max_pool_layer + + +@PLUGIN_LAYERS.register_module() +class NonLocal2d(_NonLocalNd): + """2D Non-local module. + + Args: + in_channels (int): Same as `NonLocalND`. + sub_sample (bool): Whether to apply max pooling after pairwise + function (Note that the `sub_sample` is applied on spatial only). + Default: False. + conv_cfg (None | dict): Same as `NonLocalND`. + Default: dict(type='Conv2d'). + """ + + _abbr_ = 'nonlocal_block' + + def __init__(self, + in_channels, + sub_sample=False, + conv_cfg=dict(type='Conv2d'), + **kwargs): + super(NonLocal2d, self).__init__( + in_channels, conv_cfg=conv_cfg, **kwargs) + + self.sub_sample = sub_sample + + if sub_sample: + max_pool_layer = nn.MaxPool2d(kernel_size=(2, 2)) + self.g = nn.Sequential(self.g, max_pool_layer) + if self.mode != 'gaussian': + self.phi = nn.Sequential(self.phi, max_pool_layer) + else: + self.phi = max_pool_layer + + +class NonLocal3d(_NonLocalNd): + """3D Non-local module. + + Args: + in_channels (int): Same as `NonLocalND`. + sub_sample (bool): Whether to apply max pooling after pairwise + function (Note that the `sub_sample` is applied on spatial only). + Default: False. + conv_cfg (None | dict): Same as `NonLocalND`. + Default: dict(type='Conv3d'). + """ + + def __init__(self, + in_channels, + sub_sample=False, + conv_cfg=dict(type='Conv3d'), + **kwargs): + super(NonLocal3d, self).__init__( + in_channels, conv_cfg=conv_cfg, **kwargs) + self.sub_sample = sub_sample + + if sub_sample: + max_pool_layer = nn.MaxPool3d(kernel_size=(1, 2, 2)) + self.g = nn.Sequential(self.g, max_pool_layer) + if self.mode != 'gaussian': + self.phi = nn.Sequential(self.phi, max_pool_layer) + else: + self.phi = max_pool_layer diff --git a/annotator/mmpkg/mmcv/cnn/bricks/norm.py b/annotator/mmpkg/mmcv/cnn/bricks/norm.py new file mode 100644 index 0000000000000000000000000000000000000000..31f4e49b24080485fc1d85b3e8ff810dc1383c95 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/norm.py @@ -0,0 +1,144 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import inspect + +import torch.nn as nn + +from annotator.mmpkg.mmcv.utils import is_tuple_of +from annotator.mmpkg.mmcv.utils.parrots_wrapper import SyncBatchNorm, _BatchNorm, _InstanceNorm +from .registry import NORM_LAYERS + +NORM_LAYERS.register_module('BN', module=nn.BatchNorm2d) +NORM_LAYERS.register_module('BN1d', module=nn.BatchNorm1d) +NORM_LAYERS.register_module('BN2d', module=nn.BatchNorm2d) +NORM_LAYERS.register_module('BN3d', module=nn.BatchNorm3d) +NORM_LAYERS.register_module('SyncBN', module=SyncBatchNorm) +NORM_LAYERS.register_module('GN', module=nn.GroupNorm) +NORM_LAYERS.register_module('LN', module=nn.LayerNorm) +NORM_LAYERS.register_module('IN', module=nn.InstanceNorm2d) +NORM_LAYERS.register_module('IN1d', module=nn.InstanceNorm1d) +NORM_LAYERS.register_module('IN2d', module=nn.InstanceNorm2d) +NORM_LAYERS.register_module('IN3d', module=nn.InstanceNorm3d) + + +def infer_abbr(class_type): + """Infer abbreviation from the class name. + + When we build a norm layer with `build_norm_layer()`, we want to preserve + the norm type in variable names, e.g, self.bn1, self.gn. This method will + infer the abbreviation to map class types to abbreviations. + + Rule 1: If the class has the property "_abbr_", return the property. + Rule 2: If the parent class is _BatchNorm, GroupNorm, LayerNorm or + InstanceNorm, the abbreviation of this layer will be "bn", "gn", "ln" and + "in" respectively. + Rule 3: If the class name contains "batch", "group", "layer" or "instance", + the abbreviation of this layer will be "bn", "gn", "ln" and "in" + respectively. + Rule 4: Otherwise, the abbreviation falls back to "norm". + + Args: + class_type (type): The norm layer type. + + Returns: + str: The inferred abbreviation. + """ + if not inspect.isclass(class_type): + raise TypeError( + f'class_type must be a type, but got {type(class_type)}') + if hasattr(class_type, '_abbr_'): + return class_type._abbr_ + if issubclass(class_type, _InstanceNorm): # IN is a subclass of BN + return 'in' + elif issubclass(class_type, _BatchNorm): + return 'bn' + elif issubclass(class_type, nn.GroupNorm): + return 'gn' + elif issubclass(class_type, nn.LayerNorm): + return 'ln' + else: + class_name = class_type.__name__.lower() + if 'batch' in class_name: + return 'bn' + elif 'group' in class_name: + return 'gn' + elif 'layer' in class_name: + return 'ln' + elif 'instance' in class_name: + return 'in' + else: + return 'norm_layer' + + +def build_norm_layer(cfg, num_features, postfix=''): + """Build normalization layer. + + Args: + cfg (dict): The norm layer config, which should contain: + + - type (str): Layer type. + - layer args: Args needed to instantiate a norm layer. + - requires_grad (bool, optional): Whether stop gradient updates. + num_features (int): Number of input channels. + postfix (int | str): The postfix to be appended into norm abbreviation + to create named layer. + + Returns: + (str, nn.Module): The first element is the layer name consisting of + abbreviation and postfix, e.g., bn1, gn. The second element is the + created norm layer. + """ + if not isinstance(cfg, dict): + raise TypeError('cfg must be a dict') + if 'type' not in cfg: + raise KeyError('the cfg dict must contain the key "type"') + cfg_ = cfg.copy() + + layer_type = cfg_.pop('type') + if layer_type not in NORM_LAYERS: + raise KeyError(f'Unrecognized norm type {layer_type}') + + norm_layer = NORM_LAYERS.get(layer_type) + abbr = infer_abbr(norm_layer) + + assert isinstance(postfix, (int, str)) + name = abbr + str(postfix) + + requires_grad = cfg_.pop('requires_grad', True) + cfg_.setdefault('eps', 1e-5) + if layer_type != 'GN': + layer = norm_layer(num_features, **cfg_) + if layer_type == 'SyncBN' and hasattr(layer, '_specify_ddp_gpu_num'): + layer._specify_ddp_gpu_num(1) + else: + assert 'num_groups' in cfg_ + layer = norm_layer(num_channels=num_features, **cfg_) + + for param in layer.parameters(): + param.requires_grad = requires_grad + + return name, layer + + +def is_norm(layer, exclude=None): + """Check if a layer is a normalization layer. + + Args: + layer (nn.Module): The layer to be checked. + exclude (type | tuple[type]): Types to be excluded. + + Returns: + bool: Whether the layer is a norm layer. + """ + if exclude is not None: + if not isinstance(exclude, tuple): + exclude = (exclude, ) + if not is_tuple_of(exclude, type): + raise TypeError( + f'"exclude" must be either None or type or a tuple of types, ' + f'but got {type(exclude)}: {exclude}') + + if exclude and isinstance(layer, exclude): + return False + + all_norm_bases = (_BatchNorm, _InstanceNorm, nn.GroupNorm, nn.LayerNorm) + return isinstance(layer, all_norm_bases) diff --git a/annotator/mmpkg/mmcv/cnn/bricks/padding.py b/annotator/mmpkg/mmcv/cnn/bricks/padding.py new file mode 100644 index 0000000000000000000000000000000000000000..e4ac6b28a1789bd551c613a7d3e7b622433ac7ec --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/padding.py @@ -0,0 +1,36 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch.nn as nn + +from .registry import PADDING_LAYERS + +PADDING_LAYERS.register_module('zero', module=nn.ZeroPad2d) +PADDING_LAYERS.register_module('reflect', module=nn.ReflectionPad2d) +PADDING_LAYERS.register_module('replicate', module=nn.ReplicationPad2d) + + +def build_padding_layer(cfg, *args, **kwargs): + """Build padding layer. + + Args: + cfg (None or dict): The padding layer config, which should contain: + - type (str): Layer type. + - layer args: Args needed to instantiate a padding layer. + + Returns: + nn.Module: Created padding layer. + """ + if not isinstance(cfg, dict): + raise TypeError('cfg must be a dict') + if 'type' not in cfg: + raise KeyError('the cfg dict must contain the key "type"') + + cfg_ = cfg.copy() + padding_type = cfg_.pop('type') + if padding_type not in PADDING_LAYERS: + raise KeyError(f'Unrecognized padding type {padding_type}.') + else: + padding_layer = PADDING_LAYERS.get(padding_type) + + layer = padding_layer(*args, **kwargs, **cfg_) + + return layer diff --git a/annotator/mmpkg/mmcv/cnn/bricks/plugin.py b/annotator/mmpkg/mmcv/cnn/bricks/plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..07c010d4053174dd41107aa654ea67e82b46a25c --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/plugin.py @@ -0,0 +1,88 @@ +import inspect +import platform + +from .registry import PLUGIN_LAYERS + +if platform.system() == 'Windows': + import regex as re +else: + import re + + +def infer_abbr(class_type): + """Infer abbreviation from the class name. + + This method will infer the abbreviation to map class types to + abbreviations. + + Rule 1: If the class has the property "abbr", return the property. + Rule 2: Otherwise, the abbreviation falls back to snake case of class + name, e.g. the abbreviation of ``FancyBlock`` will be ``fancy_block``. + + Args: + class_type (type): The norm layer type. + + Returns: + str: The inferred abbreviation. + """ + + def camel2snack(word): + """Convert camel case word into snack case. + + Modified from `inflection lib + `_. + + Example:: + + >>> camel2snack("FancyBlock") + 'fancy_block' + """ + + word = re.sub(r'([A-Z]+)([A-Z][a-z])', r'\1_\2', word) + word = re.sub(r'([a-z\d])([A-Z])', r'\1_\2', word) + word = word.replace('-', '_') + return word.lower() + + if not inspect.isclass(class_type): + raise TypeError( + f'class_type must be a type, but got {type(class_type)}') + if hasattr(class_type, '_abbr_'): + return class_type._abbr_ + else: + return camel2snack(class_type.__name__) + + +def build_plugin_layer(cfg, postfix='', **kwargs): + """Build plugin layer. + + Args: + cfg (None or dict): cfg should contain: + type (str): identify plugin layer type. + layer args: args needed to instantiate a plugin layer. + postfix (int, str): appended into norm abbreviation to + create named layer. Default: ''. + + Returns: + tuple[str, nn.Module]: + name (str): abbreviation + postfix + layer (nn.Module): created plugin layer + """ + if not isinstance(cfg, dict): + raise TypeError('cfg must be a dict') + if 'type' not in cfg: + raise KeyError('the cfg dict must contain the key "type"') + cfg_ = cfg.copy() + + layer_type = cfg_.pop('type') + if layer_type not in PLUGIN_LAYERS: + raise KeyError(f'Unrecognized plugin type {layer_type}') + + plugin_layer = PLUGIN_LAYERS.get(layer_type) + abbr = infer_abbr(plugin_layer) + + assert isinstance(postfix, (int, str)) + name = abbr + str(postfix) + + layer = plugin_layer(**kwargs, **cfg_) + + return name, layer diff --git a/annotator/mmpkg/mmcv/cnn/bricks/registry.py b/annotator/mmpkg/mmcv/cnn/bricks/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..4f374cca4961c06babf328bb7407723a14026c47 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/registry.py @@ -0,0 +1,16 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from annotator.mmpkg.mmcv.utils import Registry + +CONV_LAYERS = Registry('conv layer') +NORM_LAYERS = Registry('norm layer') +ACTIVATION_LAYERS = Registry('activation layer') +PADDING_LAYERS = Registry('padding layer') +UPSAMPLE_LAYERS = Registry('upsample layer') +PLUGIN_LAYERS = Registry('plugin layer') + +DROPOUT_LAYERS = Registry('drop out layers') +POSITIONAL_ENCODING = Registry('position encoding') +ATTENTION = Registry('attention') +FEEDFORWARD_NETWORK = Registry('feed-forward Network') +TRANSFORMER_LAYER = Registry('transformerLayer') +TRANSFORMER_LAYER_SEQUENCE = Registry('transformer-layers sequence') diff --git a/annotator/mmpkg/mmcv/cnn/bricks/scale.py b/annotator/mmpkg/mmcv/cnn/bricks/scale.py new file mode 100644 index 0000000000000000000000000000000000000000..c905fffcc8bf998d18d94f927591963c428025e2 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/scale.py @@ -0,0 +1,21 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn + + +class Scale(nn.Module): + """A learnable scale parameter. + + This layer scales the input by a learnable factor. It multiplies a + learnable scale parameter of shape (1,) with input of any shape. + + Args: + scale (float): Initial value of scale factor. Default: 1.0 + """ + + def __init__(self, scale=1.0): + super(Scale, self).__init__() + self.scale = nn.Parameter(torch.tensor(scale, dtype=torch.float)) + + def forward(self, x): + return x * self.scale diff --git a/annotator/mmpkg/mmcv/cnn/bricks/swish.py b/annotator/mmpkg/mmcv/cnn/bricks/swish.py new file mode 100644 index 0000000000000000000000000000000000000000..e2ca8ed7b749413f011ae54aac0cab27e6f0b51f --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/swish.py @@ -0,0 +1,25 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn + +from .registry import ACTIVATION_LAYERS + + +@ACTIVATION_LAYERS.register_module() +class Swish(nn.Module): + """Swish Module. + + This module applies the swish function: + + .. math:: + Swish(x) = x * Sigmoid(x) + + Returns: + Tensor: The output tensor. + """ + + def __init__(self): + super(Swish, self).__init__() + + def forward(self, x): + return x * torch.sigmoid(x) diff --git a/annotator/mmpkg/mmcv/cnn/bricks/transformer.py b/annotator/mmpkg/mmcv/cnn/bricks/transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..e16707142b645144b676059ffa992fc4306ef778 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/transformer.py @@ -0,0 +1,595 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy +import warnings + +import torch +import torch.nn as nn + +from annotator.mmpkg.mmcv import ConfigDict, deprecated_api_warning +from annotator.mmpkg.mmcv.cnn import Linear, build_activation_layer, build_norm_layer +from annotator.mmpkg.mmcv.runner.base_module import BaseModule, ModuleList, Sequential +from annotator.mmpkg.mmcv.utils import build_from_cfg +from .drop import build_dropout +from .registry import (ATTENTION, FEEDFORWARD_NETWORK, POSITIONAL_ENCODING, + TRANSFORMER_LAYER, TRANSFORMER_LAYER_SEQUENCE) + +# Avoid BC-breaking of importing MultiScaleDeformableAttention from this file +try: + from annotator.mmpkg.mmcv.ops.multi_scale_deform_attn import MultiScaleDeformableAttention # noqa F401 + warnings.warn( + ImportWarning( + '``MultiScaleDeformableAttention`` has been moved to ' + '``mmcv.ops.multi_scale_deform_attn``, please change original path ' # noqa E501 + '``from annotator.mmpkg.mmcv.cnn.bricks.transformer import MultiScaleDeformableAttention`` ' # noqa E501 + 'to ``from annotator.mmpkg.mmcv.ops.multi_scale_deform_attn import MultiScaleDeformableAttention`` ' # noqa E501 + )) + +except ImportError: + warnings.warn('Fail to import ``MultiScaleDeformableAttention`` from ' + '``mmcv.ops.multi_scale_deform_attn``, ' + 'You should install ``mmcv-full`` if you need this module. ') + + +def build_positional_encoding(cfg, default_args=None): + """Builder for Position Encoding.""" + return build_from_cfg(cfg, POSITIONAL_ENCODING, default_args) + + +def build_attention(cfg, default_args=None): + """Builder for attention.""" + return build_from_cfg(cfg, ATTENTION, default_args) + + +def build_feedforward_network(cfg, default_args=None): + """Builder for feed-forward network (FFN).""" + return build_from_cfg(cfg, FEEDFORWARD_NETWORK, default_args) + + +def build_transformer_layer(cfg, default_args=None): + """Builder for transformer layer.""" + return build_from_cfg(cfg, TRANSFORMER_LAYER, default_args) + + +def build_transformer_layer_sequence(cfg, default_args=None): + """Builder for transformer encoder and transformer decoder.""" + return build_from_cfg(cfg, TRANSFORMER_LAYER_SEQUENCE, default_args) + + +@ATTENTION.register_module() +class MultiheadAttention(BaseModule): + """A wrapper for ``torch.nn.MultiheadAttention``. + + This module implements MultiheadAttention with identity connection, + and positional encoding is also passed as input. + + Args: + embed_dims (int): The embedding dimension. + num_heads (int): Parallel attention heads. + attn_drop (float): A Dropout layer on attn_output_weights. + Default: 0.0. + proj_drop (float): A Dropout layer after `nn.MultiheadAttention`. + Default: 0.0. + dropout_layer (obj:`ConfigDict`): The dropout_layer used + when adding the shortcut. + init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. + Default: None. + batch_first (bool): When it is True, Key, Query and Value are shape of + (batch, n, embed_dim), otherwise (n, batch, embed_dim). + Default to False. + """ + + def __init__(self, + embed_dims, + num_heads, + attn_drop=0., + proj_drop=0., + dropout_layer=dict(type='Dropout', drop_prob=0.), + init_cfg=None, + batch_first=False, + **kwargs): + super(MultiheadAttention, self).__init__(init_cfg) + if 'dropout' in kwargs: + warnings.warn('The arguments `dropout` in MultiheadAttention ' + 'has been deprecated, now you can separately ' + 'set `attn_drop`(float), proj_drop(float), ' + 'and `dropout_layer`(dict) ') + attn_drop = kwargs['dropout'] + dropout_layer['drop_prob'] = kwargs.pop('dropout') + + self.embed_dims = embed_dims + self.num_heads = num_heads + self.batch_first = batch_first + + self.attn = nn.MultiheadAttention(embed_dims, num_heads, attn_drop, + **kwargs) + + self.proj_drop = nn.Dropout(proj_drop) + self.dropout_layer = build_dropout( + dropout_layer) if dropout_layer else nn.Identity() + + @deprecated_api_warning({'residual': 'identity'}, + cls_name='MultiheadAttention') + def forward(self, + query, + key=None, + value=None, + identity=None, + query_pos=None, + key_pos=None, + attn_mask=None, + key_padding_mask=None, + **kwargs): + """Forward function for `MultiheadAttention`. + + **kwargs allow passing a more general data flow when combining + with other operations in `transformerlayer`. + + Args: + query (Tensor): The input query with shape [num_queries, bs, + embed_dims] if self.batch_first is False, else + [bs, num_queries embed_dims]. + key (Tensor): The key tensor with shape [num_keys, bs, + embed_dims] if self.batch_first is False, else + [bs, num_keys, embed_dims] . + If None, the ``query`` will be used. Defaults to None. + value (Tensor): The value tensor with same shape as `key`. + Same in `nn.MultiheadAttention.forward`. Defaults to None. + If None, the `key` will be used. + identity (Tensor): This tensor, with the same shape as x, + will be used for the identity link. + If None, `x` will be used. Defaults to None. + query_pos (Tensor): The positional encoding for query, with + the same shape as `x`. If not None, it will + be added to `x` before forward function. Defaults to None. + key_pos (Tensor): The positional encoding for `key`, with the + same shape as `key`. Defaults to None. If not None, it will + be added to `key` before forward function. If None, and + `query_pos` has the same shape as `key`, then `query_pos` + will be used for `key_pos`. Defaults to None. + attn_mask (Tensor): ByteTensor mask with shape [num_queries, + num_keys]. Same in `nn.MultiheadAttention.forward`. + Defaults to None. + key_padding_mask (Tensor): ByteTensor with shape [bs, num_keys]. + Defaults to None. + + Returns: + Tensor: forwarded results with shape + [num_queries, bs, embed_dims] + if self.batch_first is False, else + [bs, num_queries embed_dims]. + """ + + if key is None: + key = query + if value is None: + value = key + if identity is None: + identity = query + if key_pos is None: + if query_pos is not None: + # use query_pos if key_pos is not available + if query_pos.shape == key.shape: + key_pos = query_pos + else: + warnings.warn(f'position encoding of key is' + f'missing in {self.__class__.__name__}.') + if query_pos is not None: + query = query + query_pos + if key_pos is not None: + key = key + key_pos + + # Because the dataflow('key', 'query', 'value') of + # ``torch.nn.MultiheadAttention`` is (num_query, batch, + # embed_dims), We should adjust the shape of dataflow from + # batch_first (batch, num_query, embed_dims) to num_query_first + # (num_query ,batch, embed_dims), and recover ``attn_output`` + # from num_query_first to batch_first. + if self.batch_first: + query = query.transpose(0, 1) + key = key.transpose(0, 1) + value = value.transpose(0, 1) + + out = self.attn( + query=query, + key=key, + value=value, + attn_mask=attn_mask, + key_padding_mask=key_padding_mask)[0] + + if self.batch_first: + out = out.transpose(0, 1) + + return identity + self.dropout_layer(self.proj_drop(out)) + + +@FEEDFORWARD_NETWORK.register_module() +class FFN(BaseModule): + """Implements feed-forward networks (FFNs) with identity connection. + + Args: + embed_dims (int): The feature dimension. Same as + `MultiheadAttention`. Defaults: 256. + feedforward_channels (int): The hidden dimension of FFNs. + Defaults: 1024. + num_fcs (int, optional): The number of fully-connected layers in + FFNs. Default: 2. + act_cfg (dict, optional): The activation config for FFNs. + Default: dict(type='ReLU') + ffn_drop (float, optional): Probability of an element to be + zeroed in FFN. Default 0.0. + add_identity (bool, optional): Whether to add the + identity connection. Default: `True`. + dropout_layer (obj:`ConfigDict`): The dropout_layer used + when adding the shortcut. + init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. + Default: None. + """ + + @deprecated_api_warning( + { + 'dropout': 'ffn_drop', + 'add_residual': 'add_identity' + }, + cls_name='FFN') + def __init__(self, + embed_dims=256, + feedforward_channels=1024, + num_fcs=2, + act_cfg=dict(type='ReLU', inplace=True), + ffn_drop=0., + dropout_layer=None, + add_identity=True, + init_cfg=None, + **kwargs): + super(FFN, self).__init__(init_cfg) + assert num_fcs >= 2, 'num_fcs should be no less ' \ + f'than 2. got {num_fcs}.' + self.embed_dims = embed_dims + self.feedforward_channels = feedforward_channels + self.num_fcs = num_fcs + self.act_cfg = act_cfg + self.activate = build_activation_layer(act_cfg) + + layers = [] + in_channels = embed_dims + for _ in range(num_fcs - 1): + layers.append( + Sequential( + Linear(in_channels, feedforward_channels), self.activate, + nn.Dropout(ffn_drop))) + in_channels = feedforward_channels + layers.append(Linear(feedforward_channels, embed_dims)) + layers.append(nn.Dropout(ffn_drop)) + self.layers = Sequential(*layers) + self.dropout_layer = build_dropout( + dropout_layer) if dropout_layer else torch.nn.Identity() + self.add_identity = add_identity + + @deprecated_api_warning({'residual': 'identity'}, cls_name='FFN') + def forward(self, x, identity=None): + """Forward function for `FFN`. + + The function would add x to the output tensor if residue is None. + """ + out = self.layers(x) + if not self.add_identity: + return self.dropout_layer(out) + if identity is None: + identity = x + return identity + self.dropout_layer(out) + + +@TRANSFORMER_LAYER.register_module() +class BaseTransformerLayer(BaseModule): + """Base `TransformerLayer` for vision transformer. + + It can be built from `mmcv.ConfigDict` and support more flexible + customization, for example, using any number of `FFN or LN ` and + use different kinds of `attention` by specifying a list of `ConfigDict` + named `attn_cfgs`. It is worth mentioning that it supports `prenorm` + when you specifying `norm` as the first element of `operation_order`. + More details about the `prenorm`: `On Layer Normalization in the + Transformer Architecture `_ . + + Args: + attn_cfgs (list[`mmcv.ConfigDict`] | obj:`mmcv.ConfigDict` | None )): + Configs for `self_attention` or `cross_attention` modules, + The order of the configs in the list should be consistent with + corresponding attentions in operation_order. + If it is a dict, all of the attention modules in operation_order + will be built with this config. Default: None. + ffn_cfgs (list[`mmcv.ConfigDict`] | obj:`mmcv.ConfigDict` | None )): + Configs for FFN, The order of the configs in the list should be + consistent with corresponding ffn in operation_order. + If it is a dict, all of the attention modules in operation_order + will be built with this config. + operation_order (tuple[str]): The execution order of operation + in transformer. Such as ('self_attn', 'norm', 'ffn', 'norm'). + Support `prenorm` when you specifying first element as `norm`. + Default:None. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='LN'). + init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. + Default: None. + batch_first (bool): Key, Query and Value are shape + of (batch, n, embed_dim) + or (n, batch, embed_dim). Default to False. + """ + + def __init__(self, + attn_cfgs=None, + ffn_cfgs=dict( + type='FFN', + embed_dims=256, + feedforward_channels=1024, + num_fcs=2, + ffn_drop=0., + act_cfg=dict(type='ReLU', inplace=True), + ), + operation_order=None, + norm_cfg=dict(type='LN'), + init_cfg=None, + batch_first=False, + **kwargs): + + deprecated_args = dict( + feedforward_channels='feedforward_channels', + ffn_dropout='ffn_drop', + ffn_num_fcs='num_fcs') + for ori_name, new_name in deprecated_args.items(): + if ori_name in kwargs: + warnings.warn( + f'The arguments `{ori_name}` in BaseTransformerLayer ' + f'has been deprecated, now you should set `{new_name}` ' + f'and other FFN related arguments ' + f'to a dict named `ffn_cfgs`. ') + ffn_cfgs[new_name] = kwargs[ori_name] + + super(BaseTransformerLayer, self).__init__(init_cfg) + + self.batch_first = batch_first + + assert set(operation_order) & set( + ['self_attn', 'norm', 'ffn', 'cross_attn']) == \ + set(operation_order), f'The operation_order of' \ + f' {self.__class__.__name__} should ' \ + f'contains all four operation type ' \ + f"{['self_attn', 'norm', 'ffn', 'cross_attn']}" + + num_attn = operation_order.count('self_attn') + operation_order.count( + 'cross_attn') + if isinstance(attn_cfgs, dict): + attn_cfgs = [copy.deepcopy(attn_cfgs) for _ in range(num_attn)] + else: + assert num_attn == len(attn_cfgs), f'The length ' \ + f'of attn_cfg {num_attn} is ' \ + f'not consistent with the number of attention' \ + f'in operation_order {operation_order}.' + + self.num_attn = num_attn + self.operation_order = operation_order + self.norm_cfg = norm_cfg + self.pre_norm = operation_order[0] == 'norm' + self.attentions = ModuleList() + + index = 0 + for operation_name in operation_order: + if operation_name in ['self_attn', 'cross_attn']: + if 'batch_first' in attn_cfgs[index]: + assert self.batch_first == attn_cfgs[index]['batch_first'] + else: + attn_cfgs[index]['batch_first'] = self.batch_first + attention = build_attention(attn_cfgs[index]) + # Some custom attentions used as `self_attn` + # or `cross_attn` can have different behavior. + attention.operation_name = operation_name + self.attentions.append(attention) + index += 1 + + self.embed_dims = self.attentions[0].embed_dims + + self.ffns = ModuleList() + num_ffns = operation_order.count('ffn') + if isinstance(ffn_cfgs, dict): + ffn_cfgs = ConfigDict(ffn_cfgs) + if isinstance(ffn_cfgs, dict): + ffn_cfgs = [copy.deepcopy(ffn_cfgs) for _ in range(num_ffns)] + assert len(ffn_cfgs) == num_ffns + for ffn_index in range(num_ffns): + if 'embed_dims' not in ffn_cfgs[ffn_index]: + ffn_cfgs['embed_dims'] = self.embed_dims + else: + assert ffn_cfgs[ffn_index]['embed_dims'] == self.embed_dims + self.ffns.append( + build_feedforward_network(ffn_cfgs[ffn_index], + dict(type='FFN'))) + + self.norms = ModuleList() + num_norms = operation_order.count('norm') + for _ in range(num_norms): + self.norms.append(build_norm_layer(norm_cfg, self.embed_dims)[1]) + + def forward(self, + query, + key=None, + value=None, + query_pos=None, + key_pos=None, + attn_masks=None, + query_key_padding_mask=None, + key_padding_mask=None, + **kwargs): + """Forward function for `TransformerDecoderLayer`. + + **kwargs contains some specific arguments of attentions. + + Args: + query (Tensor): The input query with shape + [num_queries, bs, embed_dims] if + self.batch_first is False, else + [bs, num_queries embed_dims]. + key (Tensor): The key tensor with shape [num_keys, bs, + embed_dims] if self.batch_first is False, else + [bs, num_keys, embed_dims] . + value (Tensor): The value tensor with same shape as `key`. + query_pos (Tensor): The positional encoding for `query`. + Default: None. + key_pos (Tensor): The positional encoding for `key`. + Default: None. + attn_masks (List[Tensor] | None): 2D Tensor used in + calculation of corresponding attention. The length of + it should equal to the number of `attention` in + `operation_order`. Default: None. + query_key_padding_mask (Tensor): ByteTensor for `query`, with + shape [bs, num_queries]. Only used in `self_attn` layer. + Defaults to None. + key_padding_mask (Tensor): ByteTensor for `query`, with + shape [bs, num_keys]. Default: None. + + Returns: + Tensor: forwarded results with shape [num_queries, bs, embed_dims]. + """ + + norm_index = 0 + attn_index = 0 + ffn_index = 0 + identity = query + if attn_masks is None: + attn_masks = [None for _ in range(self.num_attn)] + elif isinstance(attn_masks, torch.Tensor): + attn_masks = [ + copy.deepcopy(attn_masks) for _ in range(self.num_attn) + ] + warnings.warn(f'Use same attn_mask in all attentions in ' + f'{self.__class__.__name__} ') + else: + assert len(attn_masks) == self.num_attn, f'The length of ' \ + f'attn_masks {len(attn_masks)} must be equal ' \ + f'to the number of attention in ' \ + f'operation_order {self.num_attn}' + + for layer in self.operation_order: + if layer == 'self_attn': + temp_key = temp_value = query + query = self.attentions[attn_index]( + query, + temp_key, + temp_value, + identity if self.pre_norm else None, + query_pos=query_pos, + key_pos=query_pos, + attn_mask=attn_masks[attn_index], + key_padding_mask=query_key_padding_mask, + **kwargs) + attn_index += 1 + identity = query + + elif layer == 'norm': + query = self.norms[norm_index](query) + norm_index += 1 + + elif layer == 'cross_attn': + query = self.attentions[attn_index]( + query, + key, + value, + identity if self.pre_norm else None, + query_pos=query_pos, + key_pos=key_pos, + attn_mask=attn_masks[attn_index], + key_padding_mask=key_padding_mask, + **kwargs) + attn_index += 1 + identity = query + + elif layer == 'ffn': + query = self.ffns[ffn_index]( + query, identity if self.pre_norm else None) + ffn_index += 1 + + return query + + +@TRANSFORMER_LAYER_SEQUENCE.register_module() +class TransformerLayerSequence(BaseModule): + """Base class for TransformerEncoder and TransformerDecoder in vision + transformer. + + As base-class of Encoder and Decoder in vision transformer. + Support customization such as specifying different kind + of `transformer_layer` in `transformer_coder`. + + Args: + transformerlayer (list[obj:`mmcv.ConfigDict`] | + obj:`mmcv.ConfigDict`): Config of transformerlayer + in TransformerCoder. If it is obj:`mmcv.ConfigDict`, + it would be repeated `num_layer` times to a + list[`mmcv.ConfigDict`]. Default: None. + num_layers (int): The number of `TransformerLayer`. Default: None. + init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. + Default: None. + """ + + def __init__(self, transformerlayers=None, num_layers=None, init_cfg=None): + super(TransformerLayerSequence, self).__init__(init_cfg) + if isinstance(transformerlayers, dict): + transformerlayers = [ + copy.deepcopy(transformerlayers) for _ in range(num_layers) + ] + else: + assert isinstance(transformerlayers, list) and \ + len(transformerlayers) == num_layers + self.num_layers = num_layers + self.layers = ModuleList() + for i in range(num_layers): + self.layers.append(build_transformer_layer(transformerlayers[i])) + self.embed_dims = self.layers[0].embed_dims + self.pre_norm = self.layers[0].pre_norm + + def forward(self, + query, + key, + value, + query_pos=None, + key_pos=None, + attn_masks=None, + query_key_padding_mask=None, + key_padding_mask=None, + **kwargs): + """Forward function for `TransformerCoder`. + + Args: + query (Tensor): Input query with shape + `(num_queries, bs, embed_dims)`. + key (Tensor): The key tensor with shape + `(num_keys, bs, embed_dims)`. + value (Tensor): The value tensor with shape + `(num_keys, bs, embed_dims)`. + query_pos (Tensor): The positional encoding for `query`. + Default: None. + key_pos (Tensor): The positional encoding for `key`. + Default: None. + attn_masks (List[Tensor], optional): Each element is 2D Tensor + which is used in calculation of corresponding attention in + operation_order. Default: None. + query_key_padding_mask (Tensor): ByteTensor for `query`, with + shape [bs, num_queries]. Only used in self-attention + Default: None. + key_padding_mask (Tensor): ByteTensor for `query`, with + shape [bs, num_keys]. Default: None. + + Returns: + Tensor: results with shape [num_queries, bs, embed_dims]. + """ + for layer in self.layers: + query = layer( + query, + key, + value, + query_pos=query_pos, + key_pos=key_pos, + attn_masks=attn_masks, + query_key_padding_mask=query_key_padding_mask, + key_padding_mask=key_padding_mask, + **kwargs) + return query diff --git a/annotator/mmpkg/mmcv/cnn/bricks/upsample.py b/annotator/mmpkg/mmcv/cnn/bricks/upsample.py new file mode 100644 index 0000000000000000000000000000000000000000..a1a353767d0ce8518f0d7289bed10dba0178ed12 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/upsample.py @@ -0,0 +1,84 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch.nn as nn +import torch.nn.functional as F + +from ..utils import xavier_init +from .registry import UPSAMPLE_LAYERS + +UPSAMPLE_LAYERS.register_module('nearest', module=nn.Upsample) +UPSAMPLE_LAYERS.register_module('bilinear', module=nn.Upsample) + + +@UPSAMPLE_LAYERS.register_module(name='pixel_shuffle') +class PixelShufflePack(nn.Module): + """Pixel Shuffle upsample layer. + + This module packs `F.pixel_shuffle()` and a nn.Conv2d module together to + achieve a simple upsampling with pixel shuffle. + + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + scale_factor (int): Upsample ratio. + upsample_kernel (int): Kernel size of the conv layer to expand the + channels. + """ + + def __init__(self, in_channels, out_channels, scale_factor, + upsample_kernel): + super(PixelShufflePack, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.scale_factor = scale_factor + self.upsample_kernel = upsample_kernel + self.upsample_conv = nn.Conv2d( + self.in_channels, + self.out_channels * scale_factor * scale_factor, + self.upsample_kernel, + padding=(self.upsample_kernel - 1) // 2) + self.init_weights() + + def init_weights(self): + xavier_init(self.upsample_conv, distribution='uniform') + + def forward(self, x): + x = self.upsample_conv(x) + x = F.pixel_shuffle(x, self.scale_factor) + return x + + +def build_upsample_layer(cfg, *args, **kwargs): + """Build upsample layer. + + Args: + cfg (dict): The upsample layer config, which should contain: + + - type (str): Layer type. + - scale_factor (int): Upsample ratio, which is not applicable to + deconv. + - layer args: Args needed to instantiate a upsample layer. + args (argument list): Arguments passed to the ``__init__`` + method of the corresponding conv layer. + kwargs (keyword arguments): Keyword arguments passed to the + ``__init__`` method of the corresponding conv layer. + + Returns: + nn.Module: Created upsample layer. + """ + if not isinstance(cfg, dict): + raise TypeError(f'cfg must be a dict, but got {type(cfg)}') + if 'type' not in cfg: + raise KeyError( + f'the cfg dict must contain the key "type", but got {cfg}') + cfg_ = cfg.copy() + + layer_type = cfg_.pop('type') + if layer_type not in UPSAMPLE_LAYERS: + raise KeyError(f'Unrecognized upsample type {layer_type}') + else: + upsample = UPSAMPLE_LAYERS.get(layer_type) + + if upsample is nn.Upsample: + cfg_['mode'] = layer_type + layer = upsample(*args, **kwargs, **cfg_) + return layer diff --git a/annotator/mmpkg/mmcv/cnn/bricks/wrappers.py b/annotator/mmpkg/mmcv/cnn/bricks/wrappers.py new file mode 100644 index 0000000000000000000000000000000000000000..8aebf67bf52355a513f21756ee74fe510902d075 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/bricks/wrappers.py @@ -0,0 +1,180 @@ +# Copyright (c) OpenMMLab. All rights reserved. +r"""Modified from https://github.com/facebookresearch/detectron2/blob/master/detectron2/layers/wrappers.py # noqa: E501 + +Wrap some nn modules to support empty tensor input. Currently, these wrappers +are mainly used in mask heads like fcn_mask_head and maskiou_heads since mask +heads are trained on only positive RoIs. +""" +import math + +import torch +import torch.nn as nn +from torch.nn.modules.utils import _pair, _triple + +from .registry import CONV_LAYERS, UPSAMPLE_LAYERS + +if torch.__version__ == 'parrots': + TORCH_VERSION = torch.__version__ +else: + # torch.__version__ could be 1.3.1+cu92, we only need the first two + # for comparison + TORCH_VERSION = tuple(int(x) for x in torch.__version__.split('.')[:2]) + + +def obsolete_torch_version(torch_version, version_threshold): + return torch_version == 'parrots' or torch_version <= version_threshold + + +class NewEmptyTensorOp(torch.autograd.Function): + + @staticmethod + def forward(ctx, x, new_shape): + ctx.shape = x.shape + return x.new_empty(new_shape) + + @staticmethod + def backward(ctx, grad): + shape = ctx.shape + return NewEmptyTensorOp.apply(grad, shape), None + + +@CONV_LAYERS.register_module('Conv', force=True) +class Conv2d(nn.Conv2d): + + def forward(self, x): + if x.numel() == 0 and obsolete_torch_version(TORCH_VERSION, (1, 4)): + out_shape = [x.shape[0], self.out_channels] + for i, k, p, s, d in zip(x.shape[-2:], self.kernel_size, + self.padding, self.stride, self.dilation): + o = (i + 2 * p - (d * (k - 1) + 1)) // s + 1 + out_shape.append(o) + empty = NewEmptyTensorOp.apply(x, out_shape) + if self.training: + # produce dummy gradient to avoid DDP warning. + dummy = sum(x.view(-1)[0] for x in self.parameters()) * 0.0 + return empty + dummy + else: + return empty + + return super().forward(x) + + +@CONV_LAYERS.register_module('Conv3d', force=True) +class Conv3d(nn.Conv3d): + + def forward(self, x): + if x.numel() == 0 and obsolete_torch_version(TORCH_VERSION, (1, 4)): + out_shape = [x.shape[0], self.out_channels] + for i, k, p, s, d in zip(x.shape[-3:], self.kernel_size, + self.padding, self.stride, self.dilation): + o = (i + 2 * p - (d * (k - 1) + 1)) // s + 1 + out_shape.append(o) + empty = NewEmptyTensorOp.apply(x, out_shape) + if self.training: + # produce dummy gradient to avoid DDP warning. + dummy = sum(x.view(-1)[0] for x in self.parameters()) * 0.0 + return empty + dummy + else: + return empty + + return super().forward(x) + + +@CONV_LAYERS.register_module() +@CONV_LAYERS.register_module('deconv') +@UPSAMPLE_LAYERS.register_module('deconv', force=True) +class ConvTranspose2d(nn.ConvTranspose2d): + + def forward(self, x): + if x.numel() == 0 and obsolete_torch_version(TORCH_VERSION, (1, 4)): + out_shape = [x.shape[0], self.out_channels] + for i, k, p, s, d, op in zip(x.shape[-2:], self.kernel_size, + self.padding, self.stride, + self.dilation, self.output_padding): + out_shape.append((i - 1) * s - 2 * p + (d * (k - 1) + 1) + op) + empty = NewEmptyTensorOp.apply(x, out_shape) + if self.training: + # produce dummy gradient to avoid DDP warning. + dummy = sum(x.view(-1)[0] for x in self.parameters()) * 0.0 + return empty + dummy + else: + return empty + + return super().forward(x) + + +@CONV_LAYERS.register_module() +@CONV_LAYERS.register_module('deconv3d') +@UPSAMPLE_LAYERS.register_module('deconv3d', force=True) +class ConvTranspose3d(nn.ConvTranspose3d): + + def forward(self, x): + if x.numel() == 0 and obsolete_torch_version(TORCH_VERSION, (1, 4)): + out_shape = [x.shape[0], self.out_channels] + for i, k, p, s, d, op in zip(x.shape[-3:], self.kernel_size, + self.padding, self.stride, + self.dilation, self.output_padding): + out_shape.append((i - 1) * s - 2 * p + (d * (k - 1) + 1) + op) + empty = NewEmptyTensorOp.apply(x, out_shape) + if self.training: + # produce dummy gradient to avoid DDP warning. + dummy = sum(x.view(-1)[0] for x in self.parameters()) * 0.0 + return empty + dummy + else: + return empty + + return super().forward(x) + + +class MaxPool2d(nn.MaxPool2d): + + def forward(self, x): + # PyTorch 1.9 does not support empty tensor inference yet + if x.numel() == 0 and obsolete_torch_version(TORCH_VERSION, (1, 9)): + out_shape = list(x.shape[:2]) + for i, k, p, s, d in zip(x.shape[-2:], _pair(self.kernel_size), + _pair(self.padding), _pair(self.stride), + _pair(self.dilation)): + o = (i + 2 * p - (d * (k - 1) + 1)) / s + 1 + o = math.ceil(o) if self.ceil_mode else math.floor(o) + out_shape.append(o) + empty = NewEmptyTensorOp.apply(x, out_shape) + return empty + + return super().forward(x) + + +class MaxPool3d(nn.MaxPool3d): + + def forward(self, x): + # PyTorch 1.9 does not support empty tensor inference yet + if x.numel() == 0 and obsolete_torch_version(TORCH_VERSION, (1, 9)): + out_shape = list(x.shape[:2]) + for i, k, p, s, d in zip(x.shape[-3:], _triple(self.kernel_size), + _triple(self.padding), + _triple(self.stride), + _triple(self.dilation)): + o = (i + 2 * p - (d * (k - 1) + 1)) / s + 1 + o = math.ceil(o) if self.ceil_mode else math.floor(o) + out_shape.append(o) + empty = NewEmptyTensorOp.apply(x, out_shape) + return empty + + return super().forward(x) + + +class Linear(torch.nn.Linear): + + def forward(self, x): + # empty tensor forward of Linear layer is supported in Pytorch 1.6 + if x.numel() == 0 and obsolete_torch_version(TORCH_VERSION, (1, 5)): + out_shape = [x.shape[0], self.out_features] + empty = NewEmptyTensorOp.apply(x, out_shape) + if self.training: + # produce dummy gradient to avoid DDP warning. + dummy = sum(x.view(-1)[0] for x in self.parameters()) * 0.0 + return empty + dummy + else: + return empty + + return super().forward(x) diff --git a/annotator/mmpkg/mmcv/cnn/builder.py b/annotator/mmpkg/mmcv/cnn/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..7567316c566bd3aca6d8f65a84b00e9e890948a7 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/builder.py @@ -0,0 +1,30 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from ..runner import Sequential +from ..utils import Registry, build_from_cfg + + +def build_model_from_cfg(cfg, registry, default_args=None): + """Build a PyTorch model from config dict(s). Different from + ``build_from_cfg``, if cfg is a list, a ``nn.Sequential`` will be built. + + Args: + cfg (dict, list[dict]): The config of modules, is is either a config + dict or a list of config dicts. If cfg is a list, a + the built modules will be wrapped with ``nn.Sequential``. + registry (:obj:`Registry`): A registry the module belongs to. + default_args (dict, optional): Default arguments to build the module. + Defaults to None. + + Returns: + nn.Module: A built nn module. + """ + if isinstance(cfg, list): + modules = [ + build_from_cfg(cfg_, registry, default_args) for cfg_ in cfg + ] + return Sequential(*modules) + else: + return build_from_cfg(cfg, registry, default_args) + + +MODELS = Registry('model', build_func=build_model_from_cfg) diff --git a/annotator/mmpkg/mmcv/cnn/resnet.py b/annotator/mmpkg/mmcv/cnn/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..1cb3ac057ee2d52c46fc94685b5d4e698aad8d5f --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/resnet.py @@ -0,0 +1,316 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import logging + +import torch.nn as nn +import torch.utils.checkpoint as cp + +from .utils import constant_init, kaiming_init + + +def conv3x3(in_planes, out_planes, stride=1, dilation=1): + """3x3 convolution with padding.""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=dilation, + dilation=dilation, + bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, + inplanes, + planes, + stride=1, + dilation=1, + downsample=None, + style='pytorch', + with_cp=False): + super(BasicBlock, self).__init__() + assert style in ['pytorch', 'caffe'] + self.conv1 = conv3x3(inplanes, planes, stride, dilation) + self.bn1 = nn.BatchNorm2d(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = nn.BatchNorm2d(planes) + self.downsample = downsample + self.stride = stride + self.dilation = dilation + assert not with_cp + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, + inplanes, + planes, + stride=1, + dilation=1, + downsample=None, + style='pytorch', + with_cp=False): + """Bottleneck block. + + If style is "pytorch", the stride-two layer is the 3x3 conv layer, if + it is "caffe", the stride-two layer is the first 1x1 conv layer. + """ + super(Bottleneck, self).__init__() + assert style in ['pytorch', 'caffe'] + if style == 'pytorch': + conv1_stride = 1 + conv2_stride = stride + else: + conv1_stride = stride + conv2_stride = 1 + self.conv1 = nn.Conv2d( + inplanes, planes, kernel_size=1, stride=conv1_stride, bias=False) + self.conv2 = nn.Conv2d( + planes, + planes, + kernel_size=3, + stride=conv2_stride, + padding=dilation, + dilation=dilation, + bias=False) + + self.bn1 = nn.BatchNorm2d(planes) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d( + planes, planes * self.expansion, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + self.dilation = dilation + self.with_cp = with_cp + + def forward(self, x): + + def _inner_forward(x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + out = self.relu(out) + + return out + + +def make_res_layer(block, + inplanes, + planes, + blocks, + stride=1, + dilation=1, + style='pytorch', + with_cp=False): + downsample = None + if stride != 1 or inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d( + inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + nn.BatchNorm2d(planes * block.expansion), + ) + + layers = [] + layers.append( + block( + inplanes, + planes, + stride, + dilation, + downsample, + style=style, + with_cp=with_cp)) + inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block(inplanes, planes, 1, dilation, style=style, with_cp=with_cp)) + + return nn.Sequential(*layers) + + +class ResNet(nn.Module): + """ResNet backbone. + + Args: + depth (int): Depth of resnet, from {18, 34, 50, 101, 152}. + num_stages (int): Resnet stages, normally 4. + strides (Sequence[int]): Strides of the first block of each stage. + dilations (Sequence[int]): Dilation of each stage. + out_indices (Sequence[int]): Output from which stages. + style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two + layer is the 3x3 conv layer, otherwise the stride-two layer is + the first 1x1 conv layer. + frozen_stages (int): Stages to be frozen (all param fixed). -1 means + not freezing any parameters. + bn_eval (bool): Whether to set BN layers as eval mode, namely, freeze + running stats (mean and var). + bn_frozen (bool): Whether to freeze weight and bias of BN layers. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. + """ + + arch_settings = { + 18: (BasicBlock, (2, 2, 2, 2)), + 34: (BasicBlock, (3, 4, 6, 3)), + 50: (Bottleneck, (3, 4, 6, 3)), + 101: (Bottleneck, (3, 4, 23, 3)), + 152: (Bottleneck, (3, 8, 36, 3)) + } + + def __init__(self, + depth, + num_stages=4, + strides=(1, 2, 2, 2), + dilations=(1, 1, 1, 1), + out_indices=(0, 1, 2, 3), + style='pytorch', + frozen_stages=-1, + bn_eval=True, + bn_frozen=False, + with_cp=False): + super(ResNet, self).__init__() + if depth not in self.arch_settings: + raise KeyError(f'invalid depth {depth} for resnet') + assert num_stages >= 1 and num_stages <= 4 + block, stage_blocks = self.arch_settings[depth] + stage_blocks = stage_blocks[:num_stages] + assert len(strides) == len(dilations) == num_stages + assert max(out_indices) < num_stages + + self.out_indices = out_indices + self.style = style + self.frozen_stages = frozen_stages + self.bn_eval = bn_eval + self.bn_frozen = bn_frozen + self.with_cp = with_cp + + self.inplanes = 64 + self.conv1 = nn.Conv2d( + 3, 64, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + self.res_layers = [] + for i, num_blocks in enumerate(stage_blocks): + stride = strides[i] + dilation = dilations[i] + planes = 64 * 2**i + res_layer = make_res_layer( + block, + self.inplanes, + planes, + num_blocks, + stride=stride, + dilation=dilation, + style=self.style, + with_cp=with_cp) + self.inplanes = planes * block.expansion + layer_name = f'layer{i + 1}' + self.add_module(layer_name, res_layer) + self.res_layers.append(layer_name) + + self.feat_dim = block.expansion * 64 * 2**(len(stage_blocks) - 1) + + def init_weights(self, pretrained=None): + if isinstance(pretrained, str): + logger = logging.getLogger() + from ..runner import load_checkpoint + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, nn.BatchNorm2d): + constant_init(m, 1) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + outs = [] + for i, layer_name in enumerate(self.res_layers): + res_layer = getattr(self, layer_name) + x = res_layer(x) + if i in self.out_indices: + outs.append(x) + if len(outs) == 1: + return outs[0] + else: + return tuple(outs) + + def train(self, mode=True): + super(ResNet, self).train(mode) + if self.bn_eval: + for m in self.modules(): + if isinstance(m, nn.BatchNorm2d): + m.eval() + if self.bn_frozen: + for params in m.parameters(): + params.requires_grad = False + if mode and self.frozen_stages >= 0: + for param in self.conv1.parameters(): + param.requires_grad = False + for param in self.bn1.parameters(): + param.requires_grad = False + self.bn1.eval() + self.bn1.weight.requires_grad = False + self.bn1.bias.requires_grad = False + for i in range(1, self.frozen_stages + 1): + mod = getattr(self, f'layer{i}') + mod.eval() + for param in mod.parameters(): + param.requires_grad = False diff --git a/annotator/mmpkg/mmcv/cnn/utils/__init__.py b/annotator/mmpkg/mmcv/cnn/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a263e31c1e3977712827ca229bbc04910b4e928e --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/utils/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .flops_counter import get_model_complexity_info +from .fuse_conv_bn import fuse_conv_bn +from .sync_bn import revert_sync_batchnorm +from .weight_init import (INITIALIZERS, Caffe2XavierInit, ConstantInit, + KaimingInit, NormalInit, PretrainedInit, + TruncNormalInit, UniformInit, XavierInit, + bias_init_with_prob, caffe2_xavier_init, + constant_init, initialize, kaiming_init, normal_init, + trunc_normal_init, uniform_init, xavier_init) + +__all__ = [ + 'get_model_complexity_info', 'bias_init_with_prob', 'caffe2_xavier_init', + 'constant_init', 'kaiming_init', 'normal_init', 'trunc_normal_init', + 'uniform_init', 'xavier_init', 'fuse_conv_bn', 'initialize', + 'INITIALIZERS', 'ConstantInit', 'XavierInit', 'NormalInit', + 'TruncNormalInit', 'UniformInit', 'KaimingInit', 'PretrainedInit', + 'Caffe2XavierInit', 'revert_sync_batchnorm' +] diff --git a/annotator/mmpkg/mmcv/cnn/utils/flops_counter.py b/annotator/mmpkg/mmcv/cnn/utils/flops_counter.py new file mode 100644 index 0000000000000000000000000000000000000000..104240bfa524af727782ceb781147c5815529ee6 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/utils/flops_counter.py @@ -0,0 +1,599 @@ +# Modified from flops-counter.pytorch by Vladislav Sovrasov +# original repo: https://github.com/sovrasov/flops-counter.pytorch + +# MIT License + +# Copyright (c) 2018 Vladislav Sovrasov + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import sys +from functools import partial + +import numpy as np +import torch +import torch.nn as nn + +import annotator.mmpkg.mmcv as mmcv + + +def get_model_complexity_info(model, + input_shape, + print_per_layer_stat=True, + as_strings=True, + input_constructor=None, + flush=False, + ost=sys.stdout): + """Get complexity information of a model. + + This method can calculate FLOPs and parameter counts of a model with + corresponding input shape. It can also print complexity information for + each layer in a model. + + Supported layers are listed as below: + - Convolutions: ``nn.Conv1d``, ``nn.Conv2d``, ``nn.Conv3d``. + - Activations: ``nn.ReLU``, ``nn.PReLU``, ``nn.ELU``, ``nn.LeakyReLU``, + ``nn.ReLU6``. + - Poolings: ``nn.MaxPool1d``, ``nn.MaxPool2d``, ``nn.MaxPool3d``, + ``nn.AvgPool1d``, ``nn.AvgPool2d``, ``nn.AvgPool3d``, + ``nn.AdaptiveMaxPool1d``, ``nn.AdaptiveMaxPool2d``, + ``nn.AdaptiveMaxPool3d``, ``nn.AdaptiveAvgPool1d``, + ``nn.AdaptiveAvgPool2d``, ``nn.AdaptiveAvgPool3d``. + - BatchNorms: ``nn.BatchNorm1d``, ``nn.BatchNorm2d``, + ``nn.BatchNorm3d``, ``nn.GroupNorm``, ``nn.InstanceNorm1d``, + ``InstanceNorm2d``, ``InstanceNorm3d``, ``nn.LayerNorm``. + - Linear: ``nn.Linear``. + - Deconvolution: ``nn.ConvTranspose2d``. + - Upsample: ``nn.Upsample``. + + Args: + model (nn.Module): The model for complexity calculation. + input_shape (tuple): Input shape used for calculation. + print_per_layer_stat (bool): Whether to print complexity information + for each layer in a model. Default: True. + as_strings (bool): Output FLOPs and params counts in a string form. + Default: True. + input_constructor (None | callable): If specified, it takes a callable + method that generates input. otherwise, it will generate a random + tensor with input shape to calculate FLOPs. Default: None. + flush (bool): same as that in :func:`print`. Default: False. + ost (stream): same as ``file`` param in :func:`print`. + Default: sys.stdout. + + Returns: + tuple[float | str]: If ``as_strings`` is set to True, it will return + FLOPs and parameter counts in a string format. otherwise, it will + return those in a float number format. + """ + assert type(input_shape) is tuple + assert len(input_shape) >= 1 + assert isinstance(model, nn.Module) + flops_model = add_flops_counting_methods(model) + flops_model.eval() + flops_model.start_flops_count() + if input_constructor: + input = input_constructor(input_shape) + _ = flops_model(**input) + else: + try: + batch = torch.ones(()).new_empty( + (1, *input_shape), + dtype=next(flops_model.parameters()).dtype, + device=next(flops_model.parameters()).device) + except StopIteration: + # Avoid StopIteration for models which have no parameters, + # like `nn.Relu()`, `nn.AvgPool2d`, etc. + batch = torch.ones(()).new_empty((1, *input_shape)) + + _ = flops_model(batch) + + flops_count, params_count = flops_model.compute_average_flops_cost() + if print_per_layer_stat: + print_model_with_flops( + flops_model, flops_count, params_count, ost=ost, flush=flush) + flops_model.stop_flops_count() + + if as_strings: + return flops_to_string(flops_count), params_to_string(params_count) + + return flops_count, params_count + + +def flops_to_string(flops, units='GFLOPs', precision=2): + """Convert FLOPs number into a string. + + Note that Here we take a multiply-add counts as one FLOP. + + Args: + flops (float): FLOPs number to be converted. + units (str | None): Converted FLOPs units. Options are None, 'GFLOPs', + 'MFLOPs', 'KFLOPs', 'FLOPs'. If set to None, it will automatically + choose the most suitable unit for FLOPs. Default: 'GFLOPs'. + precision (int): Digit number after the decimal point. Default: 2. + + Returns: + str: The converted FLOPs number with units. + + Examples: + >>> flops_to_string(1e9) + '1.0 GFLOPs' + >>> flops_to_string(2e5, 'MFLOPs') + '0.2 MFLOPs' + >>> flops_to_string(3e-9, None) + '3e-09 FLOPs' + """ + if units is None: + if flops // 10**9 > 0: + return str(round(flops / 10.**9, precision)) + ' GFLOPs' + elif flops // 10**6 > 0: + return str(round(flops / 10.**6, precision)) + ' MFLOPs' + elif flops // 10**3 > 0: + return str(round(flops / 10.**3, precision)) + ' KFLOPs' + else: + return str(flops) + ' FLOPs' + else: + if units == 'GFLOPs': + return str(round(flops / 10.**9, precision)) + ' ' + units + elif units == 'MFLOPs': + return str(round(flops / 10.**6, precision)) + ' ' + units + elif units == 'KFLOPs': + return str(round(flops / 10.**3, precision)) + ' ' + units + else: + return str(flops) + ' FLOPs' + + +def params_to_string(num_params, units=None, precision=2): + """Convert parameter number into a string. + + Args: + num_params (float): Parameter number to be converted. + units (str | None): Converted FLOPs units. Options are None, 'M', + 'K' and ''. If set to None, it will automatically choose the most + suitable unit for Parameter number. Default: None. + precision (int): Digit number after the decimal point. Default: 2. + + Returns: + str: The converted parameter number with units. + + Examples: + >>> params_to_string(1e9) + '1000.0 M' + >>> params_to_string(2e5) + '200.0 k' + >>> params_to_string(3e-9) + '3e-09' + """ + if units is None: + if num_params // 10**6 > 0: + return str(round(num_params / 10**6, precision)) + ' M' + elif num_params // 10**3: + return str(round(num_params / 10**3, precision)) + ' k' + else: + return str(num_params) + else: + if units == 'M': + return str(round(num_params / 10.**6, precision)) + ' ' + units + elif units == 'K': + return str(round(num_params / 10.**3, precision)) + ' ' + units + else: + return str(num_params) + + +def print_model_with_flops(model, + total_flops, + total_params, + units='GFLOPs', + precision=3, + ost=sys.stdout, + flush=False): + """Print a model with FLOPs for each layer. + + Args: + model (nn.Module): The model to be printed. + total_flops (float): Total FLOPs of the model. + total_params (float): Total parameter counts of the model. + units (str | None): Converted FLOPs units. Default: 'GFLOPs'. + precision (int): Digit number after the decimal point. Default: 3. + ost (stream): same as `file` param in :func:`print`. + Default: sys.stdout. + flush (bool): same as that in :func:`print`. Default: False. + + Example: + >>> class ExampleModel(nn.Module): + + >>> def __init__(self): + >>> super().__init__() + >>> self.conv1 = nn.Conv2d(3, 8, 3) + >>> self.conv2 = nn.Conv2d(8, 256, 3) + >>> self.conv3 = nn.Conv2d(256, 8, 3) + >>> self.avg_pool = nn.AdaptiveAvgPool2d((1, 1)) + >>> self.flatten = nn.Flatten() + >>> self.fc = nn.Linear(8, 1) + + >>> def forward(self, x): + >>> x = self.conv1(x) + >>> x = self.conv2(x) + >>> x = self.conv3(x) + >>> x = self.avg_pool(x) + >>> x = self.flatten(x) + >>> x = self.fc(x) + >>> return x + + >>> model = ExampleModel() + >>> x = (3, 16, 16) + to print the complexity information state for each layer, you can use + >>> get_model_complexity_info(model, x) + or directly use + >>> print_model_with_flops(model, 4579784.0, 37361) + ExampleModel( + 0.037 M, 100.000% Params, 0.005 GFLOPs, 100.000% FLOPs, + (conv1): Conv2d(0.0 M, 0.600% Params, 0.0 GFLOPs, 0.959% FLOPs, 3, 8, kernel_size=(3, 3), stride=(1, 1)) # noqa: E501 + (conv2): Conv2d(0.019 M, 50.020% Params, 0.003 GFLOPs, 58.760% FLOPs, 8, 256, kernel_size=(3, 3), stride=(1, 1)) + (conv3): Conv2d(0.018 M, 49.356% Params, 0.002 GFLOPs, 40.264% FLOPs, 256, 8, kernel_size=(3, 3), stride=(1, 1)) + (avg_pool): AdaptiveAvgPool2d(0.0 M, 0.000% Params, 0.0 GFLOPs, 0.017% FLOPs, output_size=(1, 1)) + (flatten): Flatten(0.0 M, 0.000% Params, 0.0 GFLOPs, 0.000% FLOPs, ) + (fc): Linear(0.0 M, 0.024% Params, 0.0 GFLOPs, 0.000% FLOPs, in_features=8, out_features=1, bias=True) + ) + """ + + def accumulate_params(self): + if is_supported_instance(self): + return self.__params__ + else: + sum = 0 + for m in self.children(): + sum += m.accumulate_params() + return sum + + def accumulate_flops(self): + if is_supported_instance(self): + return self.__flops__ / model.__batch_counter__ + else: + sum = 0 + for m in self.children(): + sum += m.accumulate_flops() + return sum + + def flops_repr(self): + accumulated_num_params = self.accumulate_params() + accumulated_flops_cost = self.accumulate_flops() + return ', '.join([ + params_to_string( + accumulated_num_params, units='M', precision=precision), + '{:.3%} Params'.format(accumulated_num_params / total_params), + flops_to_string( + accumulated_flops_cost, units=units, precision=precision), + '{:.3%} FLOPs'.format(accumulated_flops_cost / total_flops), + self.original_extra_repr() + ]) + + def add_extra_repr(m): + m.accumulate_flops = accumulate_flops.__get__(m) + m.accumulate_params = accumulate_params.__get__(m) + flops_extra_repr = flops_repr.__get__(m) + if m.extra_repr != flops_extra_repr: + m.original_extra_repr = m.extra_repr + m.extra_repr = flops_extra_repr + assert m.extra_repr != m.original_extra_repr + + def del_extra_repr(m): + if hasattr(m, 'original_extra_repr'): + m.extra_repr = m.original_extra_repr + del m.original_extra_repr + if hasattr(m, 'accumulate_flops'): + del m.accumulate_flops + + model.apply(add_extra_repr) + print(model, file=ost, flush=flush) + model.apply(del_extra_repr) + + +def get_model_parameters_number(model): + """Calculate parameter number of a model. + + Args: + model (nn.module): The model for parameter number calculation. + + Returns: + float: Parameter number of the model. + """ + num_params = sum(p.numel() for p in model.parameters() if p.requires_grad) + return num_params + + +def add_flops_counting_methods(net_main_module): + # adding additional methods to the existing module object, + # this is done this way so that each function has access to self object + net_main_module.start_flops_count = start_flops_count.__get__( + net_main_module) + net_main_module.stop_flops_count = stop_flops_count.__get__( + net_main_module) + net_main_module.reset_flops_count = reset_flops_count.__get__( + net_main_module) + net_main_module.compute_average_flops_cost = compute_average_flops_cost.__get__( # noqa: E501 + net_main_module) + + net_main_module.reset_flops_count() + + return net_main_module + + +def compute_average_flops_cost(self): + """Compute average FLOPs cost. + + A method to compute average FLOPs cost, which will be available after + `add_flops_counting_methods()` is called on a desired net object. + + Returns: + float: Current mean flops consumption per image. + """ + batches_count = self.__batch_counter__ + flops_sum = 0 + for module in self.modules(): + if is_supported_instance(module): + flops_sum += module.__flops__ + params_sum = get_model_parameters_number(self) + return flops_sum / batches_count, params_sum + + +def start_flops_count(self): + """Activate the computation of mean flops consumption per image. + + A method to activate the computation of mean flops consumption per image. + which will be available after ``add_flops_counting_methods()`` is called on + a desired net object. It should be called before running the network. + """ + add_batch_counter_hook_function(self) + + def add_flops_counter_hook_function(module): + if is_supported_instance(module): + if hasattr(module, '__flops_handle__'): + return + + else: + handle = module.register_forward_hook( + get_modules_mapping()[type(module)]) + + module.__flops_handle__ = handle + + self.apply(partial(add_flops_counter_hook_function)) + + +def stop_flops_count(self): + """Stop computing the mean flops consumption per image. + + A method to stop computing the mean flops consumption per image, which will + be available after ``add_flops_counting_methods()`` is called on a desired + net object. It can be called to pause the computation whenever. + """ + remove_batch_counter_hook_function(self) + self.apply(remove_flops_counter_hook_function) + + +def reset_flops_count(self): + """Reset statistics computed so far. + + A method to Reset computed statistics, which will be available after + `add_flops_counting_methods()` is called on a desired net object. + """ + add_batch_counter_variables_or_reset(self) + self.apply(add_flops_counter_variable_or_reset) + + +# ---- Internal functions +def empty_flops_counter_hook(module, input, output): + module.__flops__ += 0 + + +def upsample_flops_counter_hook(module, input, output): + output_size = output[0] + batch_size = output_size.shape[0] + output_elements_count = batch_size + for val in output_size.shape[1:]: + output_elements_count *= val + module.__flops__ += int(output_elements_count) + + +def relu_flops_counter_hook(module, input, output): + active_elements_count = output.numel() + module.__flops__ += int(active_elements_count) + + +def linear_flops_counter_hook(module, input, output): + input = input[0] + output_last_dim = output.shape[ + -1] # pytorch checks dimensions, so here we don't care much + module.__flops__ += int(np.prod(input.shape) * output_last_dim) + + +def pool_flops_counter_hook(module, input, output): + input = input[0] + module.__flops__ += int(np.prod(input.shape)) + + +def norm_flops_counter_hook(module, input, output): + input = input[0] + + batch_flops = np.prod(input.shape) + if (getattr(module, 'affine', False) + or getattr(module, 'elementwise_affine', False)): + batch_flops *= 2 + module.__flops__ += int(batch_flops) + + +def deconv_flops_counter_hook(conv_module, input, output): + # Can have multiple inputs, getting the first one + input = input[0] + + batch_size = input.shape[0] + input_height, input_width = input.shape[2:] + + kernel_height, kernel_width = conv_module.kernel_size + in_channels = conv_module.in_channels + out_channels = conv_module.out_channels + groups = conv_module.groups + + filters_per_channel = out_channels // groups + conv_per_position_flops = ( + kernel_height * kernel_width * in_channels * filters_per_channel) + + active_elements_count = batch_size * input_height * input_width + overall_conv_flops = conv_per_position_flops * active_elements_count + bias_flops = 0 + if conv_module.bias is not None: + output_height, output_width = output.shape[2:] + bias_flops = out_channels * batch_size * output_height * output_height + overall_flops = overall_conv_flops + bias_flops + + conv_module.__flops__ += int(overall_flops) + + +def conv_flops_counter_hook(conv_module, input, output): + # Can have multiple inputs, getting the first one + input = input[0] + + batch_size = input.shape[0] + output_dims = list(output.shape[2:]) + + kernel_dims = list(conv_module.kernel_size) + in_channels = conv_module.in_channels + out_channels = conv_module.out_channels + groups = conv_module.groups + + filters_per_channel = out_channels // groups + conv_per_position_flops = int( + np.prod(kernel_dims)) * in_channels * filters_per_channel + + active_elements_count = batch_size * int(np.prod(output_dims)) + + overall_conv_flops = conv_per_position_flops * active_elements_count + + bias_flops = 0 + + if conv_module.bias is not None: + + bias_flops = out_channels * active_elements_count + + overall_flops = overall_conv_flops + bias_flops + + conv_module.__flops__ += int(overall_flops) + + +def batch_counter_hook(module, input, output): + batch_size = 1 + if len(input) > 0: + # Can have multiple inputs, getting the first one + input = input[0] + batch_size = len(input) + else: + pass + print('Warning! No positional inputs found for a module, ' + 'assuming batch size is 1.') + module.__batch_counter__ += batch_size + + +def add_batch_counter_variables_or_reset(module): + + module.__batch_counter__ = 0 + + +def add_batch_counter_hook_function(module): + if hasattr(module, '__batch_counter_handle__'): + return + + handle = module.register_forward_hook(batch_counter_hook) + module.__batch_counter_handle__ = handle + + +def remove_batch_counter_hook_function(module): + if hasattr(module, '__batch_counter_handle__'): + module.__batch_counter_handle__.remove() + del module.__batch_counter_handle__ + + +def add_flops_counter_variable_or_reset(module): + if is_supported_instance(module): + if hasattr(module, '__flops__') or hasattr(module, '__params__'): + print('Warning: variables __flops__ or __params__ are already ' + 'defined for the module' + type(module).__name__ + + ' ptflops can affect your code!') + module.__flops__ = 0 + module.__params__ = get_model_parameters_number(module) + + +def is_supported_instance(module): + if type(module) in get_modules_mapping(): + return True + return False + + +def remove_flops_counter_hook_function(module): + if is_supported_instance(module): + if hasattr(module, '__flops_handle__'): + module.__flops_handle__.remove() + del module.__flops_handle__ + + +def get_modules_mapping(): + return { + # convolutions + nn.Conv1d: conv_flops_counter_hook, + nn.Conv2d: conv_flops_counter_hook, + mmcv.cnn.bricks.Conv2d: conv_flops_counter_hook, + nn.Conv3d: conv_flops_counter_hook, + mmcv.cnn.bricks.Conv3d: conv_flops_counter_hook, + # activations + nn.ReLU: relu_flops_counter_hook, + nn.PReLU: relu_flops_counter_hook, + nn.ELU: relu_flops_counter_hook, + nn.LeakyReLU: relu_flops_counter_hook, + nn.ReLU6: relu_flops_counter_hook, + # poolings + nn.MaxPool1d: pool_flops_counter_hook, + nn.AvgPool1d: pool_flops_counter_hook, + nn.AvgPool2d: pool_flops_counter_hook, + nn.MaxPool2d: pool_flops_counter_hook, + mmcv.cnn.bricks.MaxPool2d: pool_flops_counter_hook, + nn.MaxPool3d: pool_flops_counter_hook, + mmcv.cnn.bricks.MaxPool3d: pool_flops_counter_hook, + nn.AvgPool3d: pool_flops_counter_hook, + nn.AdaptiveMaxPool1d: pool_flops_counter_hook, + nn.AdaptiveAvgPool1d: pool_flops_counter_hook, + nn.AdaptiveMaxPool2d: pool_flops_counter_hook, + nn.AdaptiveAvgPool2d: pool_flops_counter_hook, + nn.AdaptiveMaxPool3d: pool_flops_counter_hook, + nn.AdaptiveAvgPool3d: pool_flops_counter_hook, + # normalizations + nn.BatchNorm1d: norm_flops_counter_hook, + nn.BatchNorm2d: norm_flops_counter_hook, + nn.BatchNorm3d: norm_flops_counter_hook, + nn.GroupNorm: norm_flops_counter_hook, + nn.InstanceNorm1d: norm_flops_counter_hook, + nn.InstanceNorm2d: norm_flops_counter_hook, + nn.InstanceNorm3d: norm_flops_counter_hook, + nn.LayerNorm: norm_flops_counter_hook, + # FC + nn.Linear: linear_flops_counter_hook, + mmcv.cnn.bricks.Linear: linear_flops_counter_hook, + # Upscale + nn.Upsample: upsample_flops_counter_hook, + # Deconvolution + nn.ConvTranspose2d: deconv_flops_counter_hook, + mmcv.cnn.bricks.ConvTranspose2d: deconv_flops_counter_hook, + } diff --git a/annotator/mmpkg/mmcv/cnn/utils/fuse_conv_bn.py b/annotator/mmpkg/mmcv/cnn/utils/fuse_conv_bn.py new file mode 100644 index 0000000000000000000000000000000000000000..cb7076f80bf37f7931185bf0293ffcc1ce19c8ef --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/utils/fuse_conv_bn.py @@ -0,0 +1,59 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn + + +def _fuse_conv_bn(conv, bn): + """Fuse conv and bn into one module. + + Args: + conv (nn.Module): Conv to be fused. + bn (nn.Module): BN to be fused. + + Returns: + nn.Module: Fused module. + """ + conv_w = conv.weight + conv_b = conv.bias if conv.bias is not None else torch.zeros_like( + bn.running_mean) + + factor = bn.weight / torch.sqrt(bn.running_var + bn.eps) + conv.weight = nn.Parameter(conv_w * + factor.reshape([conv.out_channels, 1, 1, 1])) + conv.bias = nn.Parameter((conv_b - bn.running_mean) * factor + bn.bias) + return conv + + +def fuse_conv_bn(module): + """Recursively fuse conv and bn in a module. + + During inference, the functionary of batch norm layers is turned off + but only the mean and var alone channels are used, which exposes the + chance to fuse it with the preceding conv layers to save computations and + simplify network structures. + + Args: + module (nn.Module): Module to be fused. + + Returns: + nn.Module: Fused module. + """ + last_conv = None + last_conv_name = None + + for name, child in module.named_children(): + if isinstance(child, + (nn.modules.batchnorm._BatchNorm, nn.SyncBatchNorm)): + if last_conv is None: # only fuse BN that is after Conv + continue + fused_conv = _fuse_conv_bn(last_conv, child) + module._modules[last_conv_name] = fused_conv + # To reduce changes, set BN as Identity instead of deleting it. + module._modules[name] = nn.Identity() + last_conv = None + elif isinstance(child, nn.Conv2d): + last_conv = child + last_conv_name = name + else: + fuse_conv_bn(child) + return module diff --git a/annotator/mmpkg/mmcv/cnn/utils/sync_bn.py b/annotator/mmpkg/mmcv/cnn/utils/sync_bn.py new file mode 100644 index 0000000000000000000000000000000000000000..c0dbcb1b167ea0df690c0f47fe0217a3454b5d59 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/utils/sync_bn.py @@ -0,0 +1,59 @@ +import torch + +import annotator.mmpkg.mmcv as mmcv + + +class _BatchNormXd(torch.nn.modules.batchnorm._BatchNorm): + """A general BatchNorm layer without input dimension check. + + Reproduced from @kapily's work: + (https://github.com/pytorch/pytorch/issues/41081#issuecomment-783961547) + The only difference between BatchNorm1d, BatchNorm2d, BatchNorm3d, etc + is `_check_input_dim` that is designed for tensor sanity checks. + The check has been bypassed in this class for the convenience of converting + SyncBatchNorm. + """ + + def _check_input_dim(self, input): + return + + +def revert_sync_batchnorm(module): + """Helper function to convert all `SyncBatchNorm` (SyncBN) and + `mmcv.ops.sync_bn.SyncBatchNorm`(MMSyncBN) layers in the model to + `BatchNormXd` layers. + + Adapted from @kapily's work: + (https://github.com/pytorch/pytorch/issues/41081#issuecomment-783961547) + + Args: + module (nn.Module): The module containing `SyncBatchNorm` layers. + + Returns: + module_output: The converted module with `BatchNormXd` layers. + """ + module_output = module + module_checklist = [torch.nn.modules.batchnorm.SyncBatchNorm] + if hasattr(mmcv, 'ops'): + module_checklist.append(mmcv.ops.SyncBatchNorm) + if isinstance(module, tuple(module_checklist)): + module_output = _BatchNormXd(module.num_features, module.eps, + module.momentum, module.affine, + module.track_running_stats) + if module.affine: + # no_grad() may not be needed here but + # just to be consistent with `convert_sync_batchnorm()` + with torch.no_grad(): + module_output.weight = module.weight + module_output.bias = module.bias + module_output.running_mean = module.running_mean + module_output.running_var = module.running_var + module_output.num_batches_tracked = module.num_batches_tracked + module_output.training = module.training + # qconfig exists in quantized models + if hasattr(module, 'qconfig'): + module_output.qconfig = module.qconfig + for name, child in module.named_children(): + module_output.add_module(name, revert_sync_batchnorm(child)) + del module + return module_output diff --git a/annotator/mmpkg/mmcv/cnn/utils/weight_init.py b/annotator/mmpkg/mmcv/cnn/utils/weight_init.py new file mode 100644 index 0000000000000000000000000000000000000000..096d0ddcccbec84675f0771cb546d0fa003417e7 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/utils/weight_init.py @@ -0,0 +1,684 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy +import math +import warnings + +import numpy as np +import torch +import torch.nn as nn +from torch import Tensor + +from annotator.mmpkg.mmcv.utils import Registry, build_from_cfg, get_logger, print_log + +INITIALIZERS = Registry('initializer') + + +def update_init_info(module, init_info): + """Update the `_params_init_info` in the module if the value of parameters + are changed. + + Args: + module (obj:`nn.Module`): The module of PyTorch with a user-defined + attribute `_params_init_info` which records the initialization + information. + init_info (str): The string that describes the initialization. + """ + assert hasattr( + module, + '_params_init_info'), f'Can not find `_params_init_info` in {module}' + for name, param in module.named_parameters(): + + assert param in module._params_init_info, ( + f'Find a new :obj:`Parameter` ' + f'named `{name}` during executing the ' + f'`init_weights` of ' + f'`{module.__class__.__name__}`. ' + f'Please do not add or ' + f'replace parameters during executing ' + f'the `init_weights`. ') + + # The parameter has been changed during executing the + # `init_weights` of module + mean_value = param.data.mean() + if module._params_init_info[param]['tmp_mean_value'] != mean_value: + module._params_init_info[param]['init_info'] = init_info + module._params_init_info[param]['tmp_mean_value'] = mean_value + + +def constant_init(module, val, bias=0): + if hasattr(module, 'weight') and module.weight is not None: + nn.init.constant_(module.weight, val) + if hasattr(module, 'bias') and module.bias is not None: + nn.init.constant_(module.bias, bias) + + +def xavier_init(module, gain=1, bias=0, distribution='normal'): + assert distribution in ['uniform', 'normal'] + if hasattr(module, 'weight') and module.weight is not None: + if distribution == 'uniform': + nn.init.xavier_uniform_(module.weight, gain=gain) + else: + nn.init.xavier_normal_(module.weight, gain=gain) + if hasattr(module, 'bias') and module.bias is not None: + nn.init.constant_(module.bias, bias) + + +def normal_init(module, mean=0, std=1, bias=0): + if hasattr(module, 'weight') and module.weight is not None: + nn.init.normal_(module.weight, mean, std) + if hasattr(module, 'bias') and module.bias is not None: + nn.init.constant_(module.bias, bias) + + +def trunc_normal_init(module: nn.Module, + mean: float = 0, + std: float = 1, + a: float = -2, + b: float = 2, + bias: float = 0) -> None: + if hasattr(module, 'weight') and module.weight is not None: + trunc_normal_(module.weight, mean, std, a, b) # type: ignore + if hasattr(module, 'bias') and module.bias is not None: + nn.init.constant_(module.bias, bias) # type: ignore + + +def uniform_init(module, a=0, b=1, bias=0): + if hasattr(module, 'weight') and module.weight is not None: + nn.init.uniform_(module.weight, a, b) + if hasattr(module, 'bias') and module.bias is not None: + nn.init.constant_(module.bias, bias) + + +def kaiming_init(module, + a=0, + mode='fan_out', + nonlinearity='relu', + bias=0, + distribution='normal'): + assert distribution in ['uniform', 'normal'] + if hasattr(module, 'weight') and module.weight is not None: + if distribution == 'uniform': + nn.init.kaiming_uniform_( + module.weight, a=a, mode=mode, nonlinearity=nonlinearity) + else: + nn.init.kaiming_normal_( + module.weight, a=a, mode=mode, nonlinearity=nonlinearity) + if hasattr(module, 'bias') and module.bias is not None: + nn.init.constant_(module.bias, bias) + + +def caffe2_xavier_init(module, bias=0): + # `XavierFill` in Caffe2 corresponds to `kaiming_uniform_` in PyTorch + # Acknowledgment to FAIR's internal code + kaiming_init( + module, + a=1, + mode='fan_in', + nonlinearity='leaky_relu', + bias=bias, + distribution='uniform') + + +def bias_init_with_prob(prior_prob): + """initialize conv/fc bias value according to a given probability value.""" + bias_init = float(-np.log((1 - prior_prob) / prior_prob)) + return bias_init + + +def _get_bases_name(m): + return [b.__name__ for b in m.__class__.__bases__] + + +class BaseInit(object): + + def __init__(self, *, bias=0, bias_prob=None, layer=None): + self.wholemodule = False + if not isinstance(bias, (int, float)): + raise TypeError(f'bias must be a number, but got a {type(bias)}') + + if bias_prob is not None: + if not isinstance(bias_prob, float): + raise TypeError(f'bias_prob type must be float, \ + but got {type(bias_prob)}') + + if layer is not None: + if not isinstance(layer, (str, list)): + raise TypeError(f'layer must be a str or a list of str, \ + but got a {type(layer)}') + else: + layer = [] + + if bias_prob is not None: + self.bias = bias_init_with_prob(bias_prob) + else: + self.bias = bias + self.layer = [layer] if isinstance(layer, str) else layer + + def _get_init_info(self): + info = f'{self.__class__.__name__}, bias={self.bias}' + return info + + +@INITIALIZERS.register_module(name='Constant') +class ConstantInit(BaseInit): + """Initialize module parameters with constant values. + + Args: + val (int | float): the value to fill the weights in the module with + bias (int | float): the value to fill the bias. Defaults to 0. + bias_prob (float, optional): the probability for bias initialization. + Defaults to None. + layer (str | list[str], optional): the layer will be initialized. + Defaults to None. + """ + + def __init__(self, val, **kwargs): + super().__init__(**kwargs) + self.val = val + + def __call__(self, module): + + def init(m): + if self.wholemodule: + constant_init(m, self.val, self.bias) + else: + layername = m.__class__.__name__ + basesname = _get_bases_name(m) + if len(set(self.layer) & set([layername] + basesname)): + constant_init(m, self.val, self.bias) + + module.apply(init) + if hasattr(module, '_params_init_info'): + update_init_info(module, init_info=self._get_init_info()) + + def _get_init_info(self): + info = f'{self.__class__.__name__}: val={self.val}, bias={self.bias}' + return info + + +@INITIALIZERS.register_module(name='Xavier') +class XavierInit(BaseInit): + r"""Initialize module parameters with values according to the method + described in `Understanding the difficulty of training deep feedforward + neural networks - Glorot, X. & Bengio, Y. (2010). + `_ + + Args: + gain (int | float): an optional scaling factor. Defaults to 1. + bias (int | float): the value to fill the bias. Defaults to 0. + bias_prob (float, optional): the probability for bias initialization. + Defaults to None. + distribution (str): distribution either be ``'normal'`` + or ``'uniform'``. Defaults to ``'normal'``. + layer (str | list[str], optional): the layer will be initialized. + Defaults to None. + """ + + def __init__(self, gain=1, distribution='normal', **kwargs): + super().__init__(**kwargs) + self.gain = gain + self.distribution = distribution + + def __call__(self, module): + + def init(m): + if self.wholemodule: + xavier_init(m, self.gain, self.bias, self.distribution) + else: + layername = m.__class__.__name__ + basesname = _get_bases_name(m) + if len(set(self.layer) & set([layername] + basesname)): + xavier_init(m, self.gain, self.bias, self.distribution) + + module.apply(init) + if hasattr(module, '_params_init_info'): + update_init_info(module, init_info=self._get_init_info()) + + def _get_init_info(self): + info = f'{self.__class__.__name__}: gain={self.gain}, ' \ + f'distribution={self.distribution}, bias={self.bias}' + return info + + +@INITIALIZERS.register_module(name='Normal') +class NormalInit(BaseInit): + r"""Initialize module parameters with the values drawn from the normal + distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)`. + + Args: + mean (int | float):the mean of the normal distribution. Defaults to 0. + std (int | float): the standard deviation of the normal distribution. + Defaults to 1. + bias (int | float): the value to fill the bias. Defaults to 0. + bias_prob (float, optional): the probability for bias initialization. + Defaults to None. + layer (str | list[str], optional): the layer will be initialized. + Defaults to None. + + """ + + def __init__(self, mean=0, std=1, **kwargs): + super().__init__(**kwargs) + self.mean = mean + self.std = std + + def __call__(self, module): + + def init(m): + if self.wholemodule: + normal_init(m, self.mean, self.std, self.bias) + else: + layername = m.__class__.__name__ + basesname = _get_bases_name(m) + if len(set(self.layer) & set([layername] + basesname)): + normal_init(m, self.mean, self.std, self.bias) + + module.apply(init) + if hasattr(module, '_params_init_info'): + update_init_info(module, init_info=self._get_init_info()) + + def _get_init_info(self): + info = f'{self.__class__.__name__}: mean={self.mean},' \ + f' std={self.std}, bias={self.bias}' + return info + + +@INITIALIZERS.register_module(name='TruncNormal') +class TruncNormalInit(BaseInit): + r"""Initialize module parameters with the values drawn from the normal + distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)` with values + outside :math:`[a, b]`. + + Args: + mean (float): the mean of the normal distribution. Defaults to 0. + std (float): the standard deviation of the normal distribution. + Defaults to 1. + a (float): The minimum cutoff value. + b ( float): The maximum cutoff value. + bias (float): the value to fill the bias. Defaults to 0. + bias_prob (float, optional): the probability for bias initialization. + Defaults to None. + layer (str | list[str], optional): the layer will be initialized. + Defaults to None. + + """ + + def __init__(self, + mean: float = 0, + std: float = 1, + a: float = -2, + b: float = 2, + **kwargs) -> None: + super().__init__(**kwargs) + self.mean = mean + self.std = std + self.a = a + self.b = b + + def __call__(self, module: nn.Module) -> None: + + def init(m): + if self.wholemodule: + trunc_normal_init(m, self.mean, self.std, self.a, self.b, + self.bias) + else: + layername = m.__class__.__name__ + basesname = _get_bases_name(m) + if len(set(self.layer) & set([layername] + basesname)): + trunc_normal_init(m, self.mean, self.std, self.a, self.b, + self.bias) + + module.apply(init) + if hasattr(module, '_params_init_info'): + update_init_info(module, init_info=self._get_init_info()) + + def _get_init_info(self): + info = f'{self.__class__.__name__}: a={self.a}, b={self.b},' \ + f' mean={self.mean}, std={self.std}, bias={self.bias}' + return info + + +@INITIALIZERS.register_module(name='Uniform') +class UniformInit(BaseInit): + r"""Initialize module parameters with values drawn from the uniform + distribution :math:`\mathcal{U}(a, b)`. + + Args: + a (int | float): the lower bound of the uniform distribution. + Defaults to 0. + b (int | float): the upper bound of the uniform distribution. + Defaults to 1. + bias (int | float): the value to fill the bias. Defaults to 0. + bias_prob (float, optional): the probability for bias initialization. + Defaults to None. + layer (str | list[str], optional): the layer will be initialized. + Defaults to None. + """ + + def __init__(self, a=0, b=1, **kwargs): + super().__init__(**kwargs) + self.a = a + self.b = b + + def __call__(self, module): + + def init(m): + if self.wholemodule: + uniform_init(m, self.a, self.b, self.bias) + else: + layername = m.__class__.__name__ + basesname = _get_bases_name(m) + if len(set(self.layer) & set([layername] + basesname)): + uniform_init(m, self.a, self.b, self.bias) + + module.apply(init) + if hasattr(module, '_params_init_info'): + update_init_info(module, init_info=self._get_init_info()) + + def _get_init_info(self): + info = f'{self.__class__.__name__}: a={self.a},' \ + f' b={self.b}, bias={self.bias}' + return info + + +@INITIALIZERS.register_module(name='Kaiming') +class KaimingInit(BaseInit): + r"""Initialize module parameters with the values according to the method + described in `Delving deep into rectifiers: Surpassing human-level + performance on ImageNet classification - He, K. et al. (2015). + `_ + + Args: + a (int | float): the negative slope of the rectifier used after this + layer (only used with ``'leaky_relu'``). Defaults to 0. + mode (str): either ``'fan_in'`` or ``'fan_out'``. Choosing + ``'fan_in'`` preserves the magnitude of the variance of the weights + in the forward pass. Choosing ``'fan_out'`` preserves the + magnitudes in the backwards pass. Defaults to ``'fan_out'``. + nonlinearity (str): the non-linear function (`nn.functional` name), + recommended to use only with ``'relu'`` or ``'leaky_relu'`` . + Defaults to 'relu'. + bias (int | float): the value to fill the bias. Defaults to 0. + bias_prob (float, optional): the probability for bias initialization. + Defaults to None. + distribution (str): distribution either be ``'normal'`` or + ``'uniform'``. Defaults to ``'normal'``. + layer (str | list[str], optional): the layer will be initialized. + Defaults to None. + """ + + def __init__(self, + a=0, + mode='fan_out', + nonlinearity='relu', + distribution='normal', + **kwargs): + super().__init__(**kwargs) + self.a = a + self.mode = mode + self.nonlinearity = nonlinearity + self.distribution = distribution + + def __call__(self, module): + + def init(m): + if self.wholemodule: + kaiming_init(m, self.a, self.mode, self.nonlinearity, + self.bias, self.distribution) + else: + layername = m.__class__.__name__ + basesname = _get_bases_name(m) + if len(set(self.layer) & set([layername] + basesname)): + kaiming_init(m, self.a, self.mode, self.nonlinearity, + self.bias, self.distribution) + + module.apply(init) + if hasattr(module, '_params_init_info'): + update_init_info(module, init_info=self._get_init_info()) + + def _get_init_info(self): + info = f'{self.__class__.__name__}: a={self.a}, mode={self.mode}, ' \ + f'nonlinearity={self.nonlinearity}, ' \ + f'distribution ={self.distribution}, bias={self.bias}' + return info + + +@INITIALIZERS.register_module(name='Caffe2Xavier') +class Caffe2XavierInit(KaimingInit): + # `XavierFill` in Caffe2 corresponds to `kaiming_uniform_` in PyTorch + # Acknowledgment to FAIR's internal code + def __init__(self, **kwargs): + super().__init__( + a=1, + mode='fan_in', + nonlinearity='leaky_relu', + distribution='uniform', + **kwargs) + + def __call__(self, module): + super().__call__(module) + + +@INITIALIZERS.register_module(name='Pretrained') +class PretrainedInit(object): + """Initialize module by loading a pretrained model. + + Args: + checkpoint (str): the checkpoint file of the pretrained model should + be load. + prefix (str, optional): the prefix of a sub-module in the pretrained + model. it is for loading a part of the pretrained model to + initialize. For example, if we would like to only load the + backbone of a detector model, we can set ``prefix='backbone.'``. + Defaults to None. + map_location (str): map tensors into proper locations. + """ + + def __init__(self, checkpoint, prefix=None, map_location=None): + self.checkpoint = checkpoint + self.prefix = prefix + self.map_location = map_location + + def __call__(self, module): + from annotator.mmpkg.mmcv.runner import (_load_checkpoint_with_prefix, load_checkpoint, + load_state_dict) + logger = get_logger('mmcv') + if self.prefix is None: + print_log(f'load model from: {self.checkpoint}', logger=logger) + load_checkpoint( + module, + self.checkpoint, + map_location=self.map_location, + strict=False, + logger=logger) + else: + print_log( + f'load {self.prefix} in model from: {self.checkpoint}', + logger=logger) + state_dict = _load_checkpoint_with_prefix( + self.prefix, self.checkpoint, map_location=self.map_location) + load_state_dict(module, state_dict, strict=False, logger=logger) + + if hasattr(module, '_params_init_info'): + update_init_info(module, init_info=self._get_init_info()) + + def _get_init_info(self): + info = f'{self.__class__.__name__}: load from {self.checkpoint}' + return info + + +def _initialize(module, cfg, wholemodule=False): + func = build_from_cfg(cfg, INITIALIZERS) + # wholemodule flag is for override mode, there is no layer key in override + # and initializer will give init values for the whole module with the name + # in override. + func.wholemodule = wholemodule + func(module) + + +def _initialize_override(module, override, cfg): + if not isinstance(override, (dict, list)): + raise TypeError(f'override must be a dict or a list of dict, \ + but got {type(override)}') + + override = [override] if isinstance(override, dict) else override + + for override_ in override: + + cp_override = copy.deepcopy(override_) + name = cp_override.pop('name', None) + if name is None: + raise ValueError('`override` must contain the key "name",' + f'but got {cp_override}') + # if override only has name key, it means use args in init_cfg + if not cp_override: + cp_override.update(cfg) + # if override has name key and other args except type key, it will + # raise error + elif 'type' not in cp_override.keys(): + raise ValueError( + f'`override` need "type" key, but got {cp_override}') + + if hasattr(module, name): + _initialize(getattr(module, name), cp_override, wholemodule=True) + else: + raise RuntimeError(f'module did not have attribute {name}, ' + f'but init_cfg is {cp_override}.') + + +def initialize(module, init_cfg): + """Initialize a module. + + Args: + module (``torch.nn.Module``): the module will be initialized. + init_cfg (dict | list[dict]): initialization configuration dict to + define initializer. OpenMMLab has implemented 6 initializers + including ``Constant``, ``Xavier``, ``Normal``, ``Uniform``, + ``Kaiming``, and ``Pretrained``. + Example: + >>> module = nn.Linear(2, 3, bias=True) + >>> init_cfg = dict(type='Constant', layer='Linear', val =1 , bias =2) + >>> initialize(module, init_cfg) + + >>> module = nn.Sequential(nn.Conv1d(3, 1, 3), nn.Linear(1,2)) + >>> # define key ``'layer'`` for initializing layer with different + >>> # configuration + >>> init_cfg = [dict(type='Constant', layer='Conv1d', val=1), + dict(type='Constant', layer='Linear', val=2)] + >>> initialize(module, init_cfg) + + >>> # define key``'override'`` to initialize some specific part in + >>> # module + >>> class FooNet(nn.Module): + >>> def __init__(self): + >>> super().__init__() + >>> self.feat = nn.Conv2d(3, 16, 3) + >>> self.reg = nn.Conv2d(16, 10, 3) + >>> self.cls = nn.Conv2d(16, 5, 3) + >>> model = FooNet() + >>> init_cfg = dict(type='Constant', val=1, bias=2, layer='Conv2d', + >>> override=dict(type='Constant', name='reg', val=3, bias=4)) + >>> initialize(model, init_cfg) + + >>> model = ResNet(depth=50) + >>> # Initialize weights with the pretrained model. + >>> init_cfg = dict(type='Pretrained', + checkpoint='torchvision://resnet50') + >>> initialize(model, init_cfg) + + >>> # Initialize weights of a sub-module with the specific part of + >>> # a pretrained model by using "prefix". + >>> url = 'http://download.openmmlab.com/mmdetection/v2.0/retinanet/'\ + >>> 'retinanet_r50_fpn_1x_coco/'\ + >>> 'retinanet_r50_fpn_1x_coco_20200130-c2398f9e.pth' + >>> init_cfg = dict(type='Pretrained', + checkpoint=url, prefix='backbone.') + """ + if not isinstance(init_cfg, (dict, list)): + raise TypeError(f'init_cfg must be a dict or a list of dict, \ + but got {type(init_cfg)}') + + if isinstance(init_cfg, dict): + init_cfg = [init_cfg] + + for cfg in init_cfg: + # should deeply copy the original config because cfg may be used by + # other modules, e.g., one init_cfg shared by multiple bottleneck + # blocks, the expected cfg will be changed after pop and will change + # the initialization behavior of other modules + cp_cfg = copy.deepcopy(cfg) + override = cp_cfg.pop('override', None) + _initialize(module, cp_cfg) + + if override is not None: + cp_cfg.pop('layer', None) + _initialize_override(module, override, cp_cfg) + else: + # All attributes in module have same initialization. + pass + + +def _no_grad_trunc_normal_(tensor: Tensor, mean: float, std: float, a: float, + b: float) -> Tensor: + # Method based on + # https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf + # Modified from + # https://github.com/pytorch/pytorch/blob/master/torch/nn/init.py + def norm_cdf(x): + # Computes standard normal cumulative distribution function + return (1. + math.erf(x / math.sqrt(2.))) / 2. + + if (mean < a - 2 * std) or (mean > b + 2 * std): + warnings.warn( + 'mean is more than 2 std from [a, b] in nn.init.trunc_normal_. ' + 'The distribution of values may be incorrect.', + stacklevel=2) + + with torch.no_grad(): + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + lower = norm_cdf((a - mean) / std) + upper = norm_cdf((b - mean) / std) + + # Uniformly fill tensor with values from [lower, upper], then translate + # to [2lower-1, 2upper-1]. + tensor.uniform_(2 * lower - 1, 2 * upper - 1) + + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() + + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.)) + tensor.add_(mean) + + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor + + +def trunc_normal_(tensor: Tensor, + mean: float = 0., + std: float = 1., + a: float = -2., + b: float = 2.) -> Tensor: + r"""Fills the input Tensor with values drawn from a truncated + normal distribution. The values are effectively drawn from the + normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)` + with values outside :math:`[a, b]` redrawn until they are within + the bounds. The method used for generating the random values works + best when :math:`a \leq \text{mean} \leq b`. + + Modified from + https://github.com/pytorch/pytorch/blob/master/torch/nn/init.py + + Args: + tensor (``torch.Tensor``): an n-dimensional `torch.Tensor`. + mean (float): the mean of the normal distribution. + std (float): the standard deviation of the normal distribution. + a (float): the minimum cutoff value. + b (float): the maximum cutoff value. + """ + return _no_grad_trunc_normal_(tensor, mean, std, a, b) diff --git a/annotator/mmpkg/mmcv/cnn/vgg.py b/annotator/mmpkg/mmcv/cnn/vgg.py new file mode 100644 index 0000000000000000000000000000000000000000..8778b649561a45a9652b1a15a26c2d171e58f3e1 --- /dev/null +++ b/annotator/mmpkg/mmcv/cnn/vgg.py @@ -0,0 +1,175 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import logging + +import torch.nn as nn + +from .utils import constant_init, kaiming_init, normal_init + + +def conv3x3(in_planes, out_planes, dilation=1): + """3x3 convolution with padding.""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + padding=dilation, + dilation=dilation) + + +def make_vgg_layer(inplanes, + planes, + num_blocks, + dilation=1, + with_bn=False, + ceil_mode=False): + layers = [] + for _ in range(num_blocks): + layers.append(conv3x3(inplanes, planes, dilation)) + if with_bn: + layers.append(nn.BatchNorm2d(planes)) + layers.append(nn.ReLU(inplace=True)) + inplanes = planes + layers.append(nn.MaxPool2d(kernel_size=2, stride=2, ceil_mode=ceil_mode)) + + return layers + + +class VGG(nn.Module): + """VGG backbone. + + Args: + depth (int): Depth of vgg, from {11, 13, 16, 19}. + with_bn (bool): Use BatchNorm or not. + num_classes (int): number of classes for classification. + num_stages (int): VGG stages, normally 5. + dilations (Sequence[int]): Dilation of each stage. + out_indices (Sequence[int]): Output from which stages. + frozen_stages (int): Stages to be frozen (all param fixed). -1 means + not freezing any parameters. + bn_eval (bool): Whether to set BN layers as eval mode, namely, freeze + running stats (mean and var). + bn_frozen (bool): Whether to freeze weight and bias of BN layers. + """ + + arch_settings = { + 11: (1, 1, 2, 2, 2), + 13: (2, 2, 2, 2, 2), + 16: (2, 2, 3, 3, 3), + 19: (2, 2, 4, 4, 4) + } + + def __init__(self, + depth, + with_bn=False, + num_classes=-1, + num_stages=5, + dilations=(1, 1, 1, 1, 1), + out_indices=(0, 1, 2, 3, 4), + frozen_stages=-1, + bn_eval=True, + bn_frozen=False, + ceil_mode=False, + with_last_pool=True): + super(VGG, self).__init__() + if depth not in self.arch_settings: + raise KeyError(f'invalid depth {depth} for vgg') + assert num_stages >= 1 and num_stages <= 5 + stage_blocks = self.arch_settings[depth] + self.stage_blocks = stage_blocks[:num_stages] + assert len(dilations) == num_stages + assert max(out_indices) <= num_stages + + self.num_classes = num_classes + self.out_indices = out_indices + self.frozen_stages = frozen_stages + self.bn_eval = bn_eval + self.bn_frozen = bn_frozen + + self.inplanes = 3 + start_idx = 0 + vgg_layers = [] + self.range_sub_modules = [] + for i, num_blocks in enumerate(self.stage_blocks): + num_modules = num_blocks * (2 + with_bn) + 1 + end_idx = start_idx + num_modules + dilation = dilations[i] + planes = 64 * 2**i if i < 4 else 512 + vgg_layer = make_vgg_layer( + self.inplanes, + planes, + num_blocks, + dilation=dilation, + with_bn=with_bn, + ceil_mode=ceil_mode) + vgg_layers.extend(vgg_layer) + self.inplanes = planes + self.range_sub_modules.append([start_idx, end_idx]) + start_idx = end_idx + if not with_last_pool: + vgg_layers.pop(-1) + self.range_sub_modules[-1][1] -= 1 + self.module_name = 'features' + self.add_module(self.module_name, nn.Sequential(*vgg_layers)) + + if self.num_classes > 0: + self.classifier = nn.Sequential( + nn.Linear(512 * 7 * 7, 4096), + nn.ReLU(True), + nn.Dropout(), + nn.Linear(4096, 4096), + nn.ReLU(True), + nn.Dropout(), + nn.Linear(4096, num_classes), + ) + + def init_weights(self, pretrained=None): + if isinstance(pretrained, str): + logger = logging.getLogger() + from ..runner import load_checkpoint + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, nn.BatchNorm2d): + constant_init(m, 1) + elif isinstance(m, nn.Linear): + normal_init(m, std=0.01) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + outs = [] + vgg_layers = getattr(self, self.module_name) + for i in range(len(self.stage_blocks)): + for j in range(*self.range_sub_modules[i]): + vgg_layer = vgg_layers[j] + x = vgg_layer(x) + if i in self.out_indices: + outs.append(x) + if self.num_classes > 0: + x = x.view(x.size(0), -1) + x = self.classifier(x) + outs.append(x) + if len(outs) == 1: + return outs[0] + else: + return tuple(outs) + + def train(self, mode=True): + super(VGG, self).train(mode) + if self.bn_eval: + for m in self.modules(): + if isinstance(m, nn.BatchNorm2d): + m.eval() + if self.bn_frozen: + for params in m.parameters(): + params.requires_grad = False + vgg_layers = getattr(self, self.module_name) + if mode and self.frozen_stages >= 0: + for i in range(self.frozen_stages): + for j in range(*self.range_sub_modules[i]): + mod = vgg_layers[j] + mod.eval() + for param in mod.parameters(): + param.requires_grad = False diff --git a/annotator/mmpkg/mmcv/engine/__init__.py b/annotator/mmpkg/mmcv/engine/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3193b7f664e19ce2458d81c836597fa22e4bb082 --- /dev/null +++ b/annotator/mmpkg/mmcv/engine/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .test import (collect_results_cpu, collect_results_gpu, multi_gpu_test, + single_gpu_test) + +__all__ = [ + 'collect_results_cpu', 'collect_results_gpu', 'multi_gpu_test', + 'single_gpu_test' +] diff --git a/annotator/mmpkg/mmcv/engine/test.py b/annotator/mmpkg/mmcv/engine/test.py new file mode 100644 index 0000000000000000000000000000000000000000..ad5f55c4b181f7ad7bf17ed9003496f7377bbd3e --- /dev/null +++ b/annotator/mmpkg/mmcv/engine/test.py @@ -0,0 +1,202 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os.path as osp +import pickle +import shutil +import tempfile +import time + +import torch +import torch.distributed as dist + +import annotator.mmpkg.mmcv as mmcv +from annotator.mmpkg.mmcv.runner import get_dist_info + + +def single_gpu_test(model, data_loader): + """Test model with a single gpu. + + This method tests model with a single gpu and displays test progress bar. + + Args: + model (nn.Module): Model to be tested. + data_loader (nn.Dataloader): Pytorch data loader. + + Returns: + list: The prediction results. + """ + model.eval() + results = [] + dataset = data_loader.dataset + prog_bar = mmcv.ProgressBar(len(dataset)) + for data in data_loader: + with torch.no_grad(): + result = model(return_loss=False, **data) + results.extend(result) + + # Assume result has the same length of batch_size + # refer to https://github.com/open-mmlab/mmcv/issues/985 + batch_size = len(result) + for _ in range(batch_size): + prog_bar.update() + return results + + +def multi_gpu_test(model, data_loader, tmpdir=None, gpu_collect=False): + """Test model with multiple gpus. + + This method tests model with multiple gpus and collects the results + under two different modes: gpu and cpu modes. By setting + ``gpu_collect=True``, it encodes results to gpu tensors and use gpu + communication for results collection. On cpu mode it saves the results on + different gpus to ``tmpdir`` and collects them by the rank 0 worker. + + Args: + model (nn.Module): Model to be tested. + data_loader (nn.Dataloader): Pytorch data loader. + tmpdir (str): Path of directory to save the temporary results from + different gpus under cpu mode. + gpu_collect (bool): Option to use either gpu or cpu to collect results. + + Returns: + list: The prediction results. + """ + model.eval() + results = [] + dataset = data_loader.dataset + rank, world_size = get_dist_info() + if rank == 0: + prog_bar = mmcv.ProgressBar(len(dataset)) + time.sleep(2) # This line can prevent deadlock problem in some cases. + for i, data in enumerate(data_loader): + with torch.no_grad(): + result = model(return_loss=False, **data) + results.extend(result) + + if rank == 0: + batch_size = len(result) + batch_size_all = batch_size * world_size + if batch_size_all + prog_bar.completed > len(dataset): + batch_size_all = len(dataset) - prog_bar.completed + for _ in range(batch_size_all): + prog_bar.update() + + # collect results from all ranks + if gpu_collect: + results = collect_results_gpu(results, len(dataset)) + else: + results = collect_results_cpu(results, len(dataset), tmpdir) + return results + + +def collect_results_cpu(result_part, size, tmpdir=None): + """Collect results under cpu mode. + + On cpu mode, this function will save the results on different gpus to + ``tmpdir`` and collect them by the rank 0 worker. + + Args: + result_part (list): Result list containing result parts + to be collected. + size (int): Size of the results, commonly equal to length of + the results. + tmpdir (str | None): temporal directory for collected results to + store. If set to None, it will create a random temporal directory + for it. + + Returns: + list: The collected results. + """ + rank, world_size = get_dist_info() + # create a tmp dir if it is not specified + if tmpdir is None: + MAX_LEN = 512 + # 32 is whitespace + dir_tensor = torch.full((MAX_LEN, ), + 32, + dtype=torch.uint8, + device='cuda') + if rank == 0: + mmcv.mkdir_or_exist('.dist_test') + tmpdir = tempfile.mkdtemp(dir='.dist_test') + tmpdir = torch.tensor( + bytearray(tmpdir.encode()), dtype=torch.uint8, device='cuda') + dir_tensor[:len(tmpdir)] = tmpdir + dist.broadcast(dir_tensor, 0) + tmpdir = dir_tensor.cpu().numpy().tobytes().decode().rstrip() + else: + mmcv.mkdir_or_exist(tmpdir) + # dump the part result to the dir + mmcv.dump(result_part, osp.join(tmpdir, f'part_{rank}.pkl')) + dist.barrier() + # collect all parts + if rank != 0: + return None + else: + # load results of all parts from tmp dir + part_list = [] + for i in range(world_size): + part_file = osp.join(tmpdir, f'part_{i}.pkl') + part_result = mmcv.load(part_file) + # When data is severely insufficient, an empty part_result + # on a certain gpu could makes the overall outputs empty. + if part_result: + part_list.append(part_result) + # sort the results + ordered_results = [] + for res in zip(*part_list): + ordered_results.extend(list(res)) + # the dataloader may pad some samples + ordered_results = ordered_results[:size] + # remove tmp dir + shutil.rmtree(tmpdir) + return ordered_results + + +def collect_results_gpu(result_part, size): + """Collect results under gpu mode. + + On gpu mode, this function will encode results to gpu tensors and use gpu + communication for results collection. + + Args: + result_part (list): Result list containing result parts + to be collected. + size (int): Size of the results, commonly equal to length of + the results. + + Returns: + list: The collected results. + """ + rank, world_size = get_dist_info() + # dump result part to tensor with pickle + part_tensor = torch.tensor( + bytearray(pickle.dumps(result_part)), dtype=torch.uint8, device='cuda') + # gather all result part tensor shape + shape_tensor = torch.tensor(part_tensor.shape, device='cuda') + shape_list = [shape_tensor.clone() for _ in range(world_size)] + dist.all_gather(shape_list, shape_tensor) + # padding result part tensor to max length + shape_max = torch.tensor(shape_list).max() + part_send = torch.zeros(shape_max, dtype=torch.uint8, device='cuda') + part_send[:shape_tensor[0]] = part_tensor + part_recv_list = [ + part_tensor.new_zeros(shape_max) for _ in range(world_size) + ] + # gather all result part + dist.all_gather(part_recv_list, part_send) + + if rank == 0: + part_list = [] + for recv, shape in zip(part_recv_list, shape_list): + part_result = pickle.loads(recv[:shape[0]].cpu().numpy().tobytes()) + # When data is severely insufficient, an empty part_result + # on a certain gpu could makes the overall outputs empty. + if part_result: + part_list.append(part_result) + # sort the results + ordered_results = [] + for res in zip(*part_list): + ordered_results.extend(list(res)) + # the dataloader may pad some samples + ordered_results = ordered_results[:size] + return ordered_results diff --git a/annotator/mmpkg/mmcv/fileio/__init__.py b/annotator/mmpkg/mmcv/fileio/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2051b85f7e59bff7bdbaa131849ce8cd31f059a4 --- /dev/null +++ b/annotator/mmpkg/mmcv/fileio/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .file_client import BaseStorageBackend, FileClient +from .handlers import BaseFileHandler, JsonHandler, PickleHandler, YamlHandler +from .io import dump, load, register_handler +from .parse import dict_from_file, list_from_file + +__all__ = [ + 'BaseStorageBackend', 'FileClient', 'load', 'dump', 'register_handler', + 'BaseFileHandler', 'JsonHandler', 'PickleHandler', 'YamlHandler', + 'list_from_file', 'dict_from_file' +] diff --git a/annotator/mmpkg/mmcv/fileio/file_client.py b/annotator/mmpkg/mmcv/fileio/file_client.py new file mode 100644 index 0000000000000000000000000000000000000000..1ed2bf5f41a29000f9a080066497d8f3674fae15 --- /dev/null +++ b/annotator/mmpkg/mmcv/fileio/file_client.py @@ -0,0 +1,1148 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import inspect +import os +import os.path as osp +import re +import tempfile +import warnings +from abc import ABCMeta, abstractmethod +from contextlib import contextmanager +from pathlib import Path +from typing import Iterable, Iterator, Optional, Tuple, Union +from urllib.request import urlopen + +import annotator.mmpkg.mmcv as mmcv +from annotator.mmpkg.mmcv.utils.misc import has_method +from annotator.mmpkg.mmcv.utils.path import is_filepath + + +class BaseStorageBackend(metaclass=ABCMeta): + """Abstract class of storage backends. + + All backends need to implement two apis: ``get()`` and ``get_text()``. + ``get()`` reads the file as a byte stream and ``get_text()`` reads the file + as texts. + """ + + # a flag to indicate whether the backend can create a symlink for a file + _allow_symlink = False + + @property + def name(self): + return self.__class__.__name__ + + @property + def allow_symlink(self): + return self._allow_symlink + + @abstractmethod + def get(self, filepath): + pass + + @abstractmethod + def get_text(self, filepath): + pass + + +class CephBackend(BaseStorageBackend): + """Ceph storage backend (for internal use). + + Args: + path_mapping (dict|None): path mapping dict from local path to Petrel + path. When ``path_mapping={'src': 'dst'}``, ``src`` in ``filepath`` + will be replaced by ``dst``. Default: None. + + .. warning:: + :class:`mmcv.fileio.file_client.CephBackend` will be deprecated, + please use :class:`mmcv.fileio.file_client.PetrelBackend` instead. + """ + + def __init__(self, path_mapping=None): + try: + import ceph + except ImportError: + raise ImportError('Please install ceph to enable CephBackend.') + + warnings.warn( + 'CephBackend will be deprecated, please use PetrelBackend instead') + self._client = ceph.S3Client() + assert isinstance(path_mapping, dict) or path_mapping is None + self.path_mapping = path_mapping + + def get(self, filepath): + filepath = str(filepath) + if self.path_mapping is not None: + for k, v in self.path_mapping.items(): + filepath = filepath.replace(k, v) + value = self._client.Get(filepath) + value_buf = memoryview(value) + return value_buf + + def get_text(self, filepath, encoding=None): + raise NotImplementedError + + +class PetrelBackend(BaseStorageBackend): + """Petrel storage backend (for internal use). + + PetrelBackend supports reading and writing data to multiple clusters. + If the file path contains the cluster name, PetrelBackend will read data + from specified cluster or write data to it. Otherwise, PetrelBackend will + access the default cluster. + + Args: + path_mapping (dict, optional): Path mapping dict from local path to + Petrel path. When ``path_mapping={'src': 'dst'}``, ``src`` in + ``filepath`` will be replaced by ``dst``. Default: None. + enable_mc (bool, optional): Whether to enable memcached support. + Default: True. + + Examples: + >>> filepath1 = 's3://path/of/file' + >>> filepath2 = 'cluster-name:s3://path/of/file' + >>> client = PetrelBackend() + >>> client.get(filepath1) # get data from default cluster + >>> client.get(filepath2) # get data from 'cluster-name' cluster + """ + + def __init__(self, + path_mapping: Optional[dict] = None, + enable_mc: bool = True): + try: + from petrel_client import client + except ImportError: + raise ImportError('Please install petrel_client to enable ' + 'PetrelBackend.') + + self._client = client.Client(enable_mc=enable_mc) + assert isinstance(path_mapping, dict) or path_mapping is None + self.path_mapping = path_mapping + + def _map_path(self, filepath: Union[str, Path]) -> str: + """Map ``filepath`` to a string path whose prefix will be replaced by + :attr:`self.path_mapping`. + + Args: + filepath (str): Path to be mapped. + """ + filepath = str(filepath) + if self.path_mapping is not None: + for k, v in self.path_mapping.items(): + filepath = filepath.replace(k, v) + return filepath + + def _format_path(self, filepath: str) -> str: + """Convert a ``filepath`` to standard format of petrel oss. + + If the ``filepath`` is concatenated by ``os.path.join``, in a Windows + environment, the ``filepath`` will be the format of + 's3://bucket_name\\image.jpg'. By invoking :meth:`_format_path`, the + above ``filepath`` will be converted to 's3://bucket_name/image.jpg'. + + Args: + filepath (str): Path to be formatted. + """ + return re.sub(r'\\+', '/', filepath) + + def get(self, filepath: Union[str, Path]) -> memoryview: + """Read data from a given ``filepath`` with 'rb' mode. + + Args: + filepath (str or Path): Path to read data. + + Returns: + memoryview: A memory view of expected bytes object to avoid + copying. The memoryview object can be converted to bytes by + ``value_buf.tobytes()``. + """ + filepath = self._map_path(filepath) + filepath = self._format_path(filepath) + value = self._client.Get(filepath) + value_buf = memoryview(value) + return value_buf + + def get_text(self, + filepath: Union[str, Path], + encoding: str = 'utf-8') -> str: + """Read data from a given ``filepath`` with 'r' mode. + + Args: + filepath (str or Path): Path to read data. + encoding (str): The encoding format used to open the ``filepath``. + Default: 'utf-8'. + + Returns: + str: Expected text reading from ``filepath``. + """ + return str(self.get(filepath), encoding=encoding) + + def put(self, obj: bytes, filepath: Union[str, Path]) -> None: + """Save data to a given ``filepath``. + + Args: + obj (bytes): Data to be saved. + filepath (str or Path): Path to write data. + """ + filepath = self._map_path(filepath) + filepath = self._format_path(filepath) + self._client.put(filepath, obj) + + def put_text(self, + obj: str, + filepath: Union[str, Path], + encoding: str = 'utf-8') -> None: + """Save data to a given ``filepath``. + + Args: + obj (str): Data to be written. + filepath (str or Path): Path to write data. + encoding (str): The encoding format used to encode the ``obj``. + Default: 'utf-8'. + """ + self.put(bytes(obj, encoding=encoding), filepath) + + def remove(self, filepath: Union[str, Path]) -> None: + """Remove a file. + + Args: + filepath (str or Path): Path to be removed. + """ + if not has_method(self._client, 'delete'): + raise NotImplementedError( + ('Current version of Petrel Python SDK has not supported ' + 'the `delete` method, please use a higher version or dev' + ' branch instead.')) + + filepath = self._map_path(filepath) + filepath = self._format_path(filepath) + self._client.delete(filepath) + + def exists(self, filepath: Union[str, Path]) -> bool: + """Check whether a file path exists. + + Args: + filepath (str or Path): Path to be checked whether exists. + + Returns: + bool: Return ``True`` if ``filepath`` exists, ``False`` otherwise. + """ + if not (has_method(self._client, 'contains') + and has_method(self._client, 'isdir')): + raise NotImplementedError( + ('Current version of Petrel Python SDK has not supported ' + 'the `contains` and `isdir` methods, please use a higher' + 'version or dev branch instead.')) + + filepath = self._map_path(filepath) + filepath = self._format_path(filepath) + return self._client.contains(filepath) or self._client.isdir(filepath) + + def isdir(self, filepath: Union[str, Path]) -> bool: + """Check whether a file path is a directory. + + Args: + filepath (str or Path): Path to be checked whether it is a + directory. + + Returns: + bool: Return ``True`` if ``filepath`` points to a directory, + ``False`` otherwise. + """ + if not has_method(self._client, 'isdir'): + raise NotImplementedError( + ('Current version of Petrel Python SDK has not supported ' + 'the `isdir` method, please use a higher version or dev' + ' branch instead.')) + + filepath = self._map_path(filepath) + filepath = self._format_path(filepath) + return self._client.isdir(filepath) + + def isfile(self, filepath: Union[str, Path]) -> bool: + """Check whether a file path is a file. + + Args: + filepath (str or Path): Path to be checked whether it is a file. + + Returns: + bool: Return ``True`` if ``filepath`` points to a file, ``False`` + otherwise. + """ + if not has_method(self._client, 'contains'): + raise NotImplementedError( + ('Current version of Petrel Python SDK has not supported ' + 'the `contains` method, please use a higher version or ' + 'dev branch instead.')) + + filepath = self._map_path(filepath) + filepath = self._format_path(filepath) + return self._client.contains(filepath) + + def join_path(self, filepath: Union[str, Path], + *filepaths: Union[str, Path]) -> str: + """Concatenate all file paths. + + Args: + filepath (str or Path): Path to be concatenated. + + Returns: + str: The result after concatenation. + """ + filepath = self._format_path(self._map_path(filepath)) + if filepath.endswith('/'): + filepath = filepath[:-1] + formatted_paths = [filepath] + for path in filepaths: + formatted_paths.append(self._format_path(self._map_path(path))) + return '/'.join(formatted_paths) + + @contextmanager + def get_local_path(self, filepath: Union[str, Path]) -> Iterable[str]: + """Download a file from ``filepath`` and return a temporary path. + + ``get_local_path`` is decorated by :meth:`contxtlib.contextmanager`. It + can be called with ``with`` statement, and when exists from the + ``with`` statement, the temporary path will be released. + + Args: + filepath (str | Path): Download a file from ``filepath``. + + Examples: + >>> client = PetrelBackend() + >>> # After existing from the ``with`` clause, + >>> # the path will be removed + >>> with client.get_local_path('s3://path/of/your/file') as path: + ... # do something here + + Yields: + Iterable[str]: Only yield one temporary path. + """ + filepath = self._map_path(filepath) + filepath = self._format_path(filepath) + assert self.isfile(filepath) + try: + f = tempfile.NamedTemporaryFile(delete=False) + f.write(self.get(filepath)) + f.close() + yield f.name + finally: + os.remove(f.name) + + def list_dir_or_file(self, + dir_path: Union[str, Path], + list_dir: bool = True, + list_file: bool = True, + suffix: Optional[Union[str, Tuple[str]]] = None, + recursive: bool = False) -> Iterator[str]: + """Scan a directory to find the interested directories or files in + arbitrary order. + + Note: + Petrel has no concept of directories but it simulates the directory + hierarchy in the filesystem through public prefixes. In addition, + if the returned path ends with '/', it means the path is a public + prefix which is a logical directory. + + Note: + :meth:`list_dir_or_file` returns the path relative to ``dir_path``. + In addition, the returned path of directory will not contains the + suffix '/' which is consistent with other backends. + + Args: + dir_path (str | Path): Path of the directory. + list_dir (bool): List the directories. Default: True. + list_file (bool): List the path of files. Default: True. + suffix (str or tuple[str], optional): File suffix + that we are interested in. Default: None. + recursive (bool): If set to True, recursively scan the + directory. Default: False. + + Yields: + Iterable[str]: A relative path to ``dir_path``. + """ + if not has_method(self._client, 'list'): + raise NotImplementedError( + ('Current version of Petrel Python SDK has not supported ' + 'the `list` method, please use a higher version or dev' + ' branch instead.')) + + dir_path = self._map_path(dir_path) + dir_path = self._format_path(dir_path) + if list_dir and suffix is not None: + raise TypeError( + '`list_dir` should be False when `suffix` is not None') + + if (suffix is not None) and not isinstance(suffix, (str, tuple)): + raise TypeError('`suffix` must be a string or tuple of strings') + + # Petrel's simulated directory hierarchy assumes that directory paths + # should end with `/` + if not dir_path.endswith('/'): + dir_path += '/' + + root = dir_path + + def _list_dir_or_file(dir_path, list_dir, list_file, suffix, + recursive): + for path in self._client.list(dir_path): + # the `self.isdir` is not used here to determine whether path + # is a directory, because `self.isdir` relies on + # `self._client.list` + if path.endswith('/'): # a directory path + next_dir_path = self.join_path(dir_path, path) + if list_dir: + # get the relative path and exclude the last + # character '/' + rel_dir = next_dir_path[len(root):-1] + yield rel_dir + if recursive: + yield from _list_dir_or_file(next_dir_path, list_dir, + list_file, suffix, + recursive) + else: # a file path + absolute_path = self.join_path(dir_path, path) + rel_path = absolute_path[len(root):] + if (suffix is None + or rel_path.endswith(suffix)) and list_file: + yield rel_path + + return _list_dir_or_file(dir_path, list_dir, list_file, suffix, + recursive) + + +class MemcachedBackend(BaseStorageBackend): + """Memcached storage backend. + + Attributes: + server_list_cfg (str): Config file for memcached server list. + client_cfg (str): Config file for memcached client. + sys_path (str | None): Additional path to be appended to `sys.path`. + Default: None. + """ + + def __init__(self, server_list_cfg, client_cfg, sys_path=None): + if sys_path is not None: + import sys + sys.path.append(sys_path) + try: + import mc + except ImportError: + raise ImportError( + 'Please install memcached to enable MemcachedBackend.') + + self.server_list_cfg = server_list_cfg + self.client_cfg = client_cfg + self._client = mc.MemcachedClient.GetInstance(self.server_list_cfg, + self.client_cfg) + # mc.pyvector servers as a point which points to a memory cache + self._mc_buffer = mc.pyvector() + + def get(self, filepath): + filepath = str(filepath) + import mc + self._client.Get(filepath, self._mc_buffer) + value_buf = mc.ConvertBuffer(self._mc_buffer) + return value_buf + + def get_text(self, filepath, encoding=None): + raise NotImplementedError + + +class LmdbBackend(BaseStorageBackend): + """Lmdb storage backend. + + Args: + db_path (str): Lmdb database path. + readonly (bool, optional): Lmdb environment parameter. If True, + disallow any write operations. Default: True. + lock (bool, optional): Lmdb environment parameter. If False, when + concurrent access occurs, do not lock the database. Default: False. + readahead (bool, optional): Lmdb environment parameter. If False, + disable the OS filesystem readahead mechanism, which may improve + random read performance when a database is larger than RAM. + Default: False. + + Attributes: + db_path (str): Lmdb database path. + """ + + def __init__(self, + db_path, + readonly=True, + lock=False, + readahead=False, + **kwargs): + try: + import lmdb + except ImportError: + raise ImportError('Please install lmdb to enable LmdbBackend.') + + self.db_path = str(db_path) + self._client = lmdb.open( + self.db_path, + readonly=readonly, + lock=lock, + readahead=readahead, + **kwargs) + + def get(self, filepath): + """Get values according to the filepath. + + Args: + filepath (str | obj:`Path`): Here, filepath is the lmdb key. + """ + filepath = str(filepath) + with self._client.begin(write=False) as txn: + value_buf = txn.get(filepath.encode('ascii')) + return value_buf + + def get_text(self, filepath, encoding=None): + raise NotImplementedError + + +class HardDiskBackend(BaseStorageBackend): + """Raw hard disks storage backend.""" + + _allow_symlink = True + + def get(self, filepath: Union[str, Path]) -> bytes: + """Read data from a given ``filepath`` with 'rb' mode. + + Args: + filepath (str or Path): Path to read data. + + Returns: + bytes: Expected bytes object. + """ + with open(filepath, 'rb') as f: + value_buf = f.read() + return value_buf + + def get_text(self, + filepath: Union[str, Path], + encoding: str = 'utf-8') -> str: + """Read data from a given ``filepath`` with 'r' mode. + + Args: + filepath (str or Path): Path to read data. + encoding (str): The encoding format used to open the ``filepath``. + Default: 'utf-8'. + + Returns: + str: Expected text reading from ``filepath``. + """ + with open(filepath, 'r', encoding=encoding) as f: + value_buf = f.read() + return value_buf + + def put(self, obj: bytes, filepath: Union[str, Path]) -> None: + """Write data to a given ``filepath`` with 'wb' mode. + + Note: + ``put`` will create a directory if the directory of ``filepath`` + does not exist. + + Args: + obj (bytes): Data to be written. + filepath (str or Path): Path to write data. + """ + mmcv.mkdir_or_exist(osp.dirname(filepath)) + with open(filepath, 'wb') as f: + f.write(obj) + + def put_text(self, + obj: str, + filepath: Union[str, Path], + encoding: str = 'utf-8') -> None: + """Write data to a given ``filepath`` with 'w' mode. + + Note: + ``put_text`` will create a directory if the directory of + ``filepath`` does not exist. + + Args: + obj (str): Data to be written. + filepath (str or Path): Path to write data. + encoding (str): The encoding format used to open the ``filepath``. + Default: 'utf-8'. + """ + mmcv.mkdir_or_exist(osp.dirname(filepath)) + with open(filepath, 'w', encoding=encoding) as f: + f.write(obj) + + def remove(self, filepath: Union[str, Path]) -> None: + """Remove a file. + + Args: + filepath (str or Path): Path to be removed. + """ + os.remove(filepath) + + def exists(self, filepath: Union[str, Path]) -> bool: + """Check whether a file path exists. + + Args: + filepath (str or Path): Path to be checked whether exists. + + Returns: + bool: Return ``True`` if ``filepath`` exists, ``False`` otherwise. + """ + return osp.exists(filepath) + + def isdir(self, filepath: Union[str, Path]) -> bool: + """Check whether a file path is a directory. + + Args: + filepath (str or Path): Path to be checked whether it is a + directory. + + Returns: + bool: Return ``True`` if ``filepath`` points to a directory, + ``False`` otherwise. + """ + return osp.isdir(filepath) + + def isfile(self, filepath: Union[str, Path]) -> bool: + """Check whether a file path is a file. + + Args: + filepath (str or Path): Path to be checked whether it is a file. + + Returns: + bool: Return ``True`` if ``filepath`` points to a file, ``False`` + otherwise. + """ + return osp.isfile(filepath) + + def join_path(self, filepath: Union[str, Path], + *filepaths: Union[str, Path]) -> str: + """Concatenate all file paths. + + Join one or more filepath components intelligently. The return value + is the concatenation of filepath and any members of *filepaths. + + Args: + filepath (str or Path): Path to be concatenated. + + Returns: + str: The result of concatenation. + """ + return osp.join(filepath, *filepaths) + + @contextmanager + def get_local_path( + self, filepath: Union[str, Path]) -> Iterable[Union[str, Path]]: + """Only for unified API and do nothing.""" + yield filepath + + def list_dir_or_file(self, + dir_path: Union[str, Path], + list_dir: bool = True, + list_file: bool = True, + suffix: Optional[Union[str, Tuple[str]]] = None, + recursive: bool = False) -> Iterator[str]: + """Scan a directory to find the interested directories or files in + arbitrary order. + + Note: + :meth:`list_dir_or_file` returns the path relative to ``dir_path``. + + Args: + dir_path (str | Path): Path of the directory. + list_dir (bool): List the directories. Default: True. + list_file (bool): List the path of files. Default: True. + suffix (str or tuple[str], optional): File suffix + that we are interested in. Default: None. + recursive (bool): If set to True, recursively scan the + directory. Default: False. + + Yields: + Iterable[str]: A relative path to ``dir_path``. + """ + if list_dir and suffix is not None: + raise TypeError('`suffix` should be None when `list_dir` is True') + + if (suffix is not None) and not isinstance(suffix, (str, tuple)): + raise TypeError('`suffix` must be a string or tuple of strings') + + root = dir_path + + def _list_dir_or_file(dir_path, list_dir, list_file, suffix, + recursive): + for entry in os.scandir(dir_path): + if not entry.name.startswith('.') and entry.is_file(): + rel_path = osp.relpath(entry.path, root) + if (suffix is None + or rel_path.endswith(suffix)) and list_file: + yield rel_path + elif osp.isdir(entry.path): + if list_dir: + rel_dir = osp.relpath(entry.path, root) + yield rel_dir + if recursive: + yield from _list_dir_or_file(entry.path, list_dir, + list_file, suffix, + recursive) + + return _list_dir_or_file(dir_path, list_dir, list_file, suffix, + recursive) + + +class HTTPBackend(BaseStorageBackend): + """HTTP and HTTPS storage bachend.""" + + def get(self, filepath): + value_buf = urlopen(filepath).read() + return value_buf + + def get_text(self, filepath, encoding='utf-8'): + value_buf = urlopen(filepath).read() + return value_buf.decode(encoding) + + @contextmanager + def get_local_path(self, filepath: str) -> Iterable[str]: + """Download a file from ``filepath``. + + ``get_local_path`` is decorated by :meth:`contxtlib.contextmanager`. It + can be called with ``with`` statement, and when exists from the + ``with`` statement, the temporary path will be released. + + Args: + filepath (str): Download a file from ``filepath``. + + Examples: + >>> client = HTTPBackend() + >>> # After existing from the ``with`` clause, + >>> # the path will be removed + >>> with client.get_local_path('http://path/of/your/file') as path: + ... # do something here + """ + try: + f = tempfile.NamedTemporaryFile(delete=False) + f.write(self.get(filepath)) + f.close() + yield f.name + finally: + os.remove(f.name) + + +class FileClient: + """A general file client to access files in different backends. + + The client loads a file or text in a specified backend from its path + and returns it as a binary or text file. There are two ways to choose a + backend, the name of backend and the prefix of path. Although both of them + can be used to choose a storage backend, ``backend`` has a higher priority + that is if they are all set, the storage backend will be chosen by the + backend argument. If they are all `None`, the disk backend will be chosen. + Note that It can also register other backend accessor with a given name, + prefixes, and backend class. In addition, We use the singleton pattern to + avoid repeated object creation. If the arguments are the same, the same + object will be returned. + + Args: + backend (str, optional): The storage backend type. Options are "disk", + "ceph", "memcached", "lmdb", "http" and "petrel". Default: None. + prefix (str, optional): The prefix of the registered storage backend. + Options are "s3", "http", "https". Default: None. + + Examples: + >>> # only set backend + >>> file_client = FileClient(backend='petrel') + >>> # only set prefix + >>> file_client = FileClient(prefix='s3') + >>> # set both backend and prefix but use backend to choose client + >>> file_client = FileClient(backend='petrel', prefix='s3') + >>> # if the arguments are the same, the same object is returned + >>> file_client1 = FileClient(backend='petrel') + >>> file_client1 is file_client + True + + Attributes: + client (:obj:`BaseStorageBackend`): The backend object. + """ + + _backends = { + 'disk': HardDiskBackend, + 'ceph': CephBackend, + 'memcached': MemcachedBackend, + 'lmdb': LmdbBackend, + 'petrel': PetrelBackend, + 'http': HTTPBackend, + } + # This collection is used to record the overridden backends, and when a + # backend appears in the collection, the singleton pattern is disabled for + # that backend, because if the singleton pattern is used, then the object + # returned will be the backend before overwriting + _overridden_backends = set() + _prefix_to_backends = { + 's3': PetrelBackend, + 'http': HTTPBackend, + 'https': HTTPBackend, + } + _overridden_prefixes = set() + + _instances = {} + + def __new__(cls, backend=None, prefix=None, **kwargs): + if backend is None and prefix is None: + backend = 'disk' + if backend is not None and backend not in cls._backends: + raise ValueError( + f'Backend {backend} is not supported. Currently supported ones' + f' are {list(cls._backends.keys())}') + if prefix is not None and prefix not in cls._prefix_to_backends: + raise ValueError( + f'prefix {prefix} is not supported. Currently supported ones ' + f'are {list(cls._prefix_to_backends.keys())}') + + # concatenate the arguments to a unique key for determining whether + # objects with the same arguments were created + arg_key = f'{backend}:{prefix}' + for key, value in kwargs.items(): + arg_key += f':{key}:{value}' + + # if a backend was overridden, it will create a new object + if (arg_key in cls._instances + and backend not in cls._overridden_backends + and prefix not in cls._overridden_prefixes): + _instance = cls._instances[arg_key] + else: + # create a new object and put it to _instance + _instance = super().__new__(cls) + if backend is not None: + _instance.client = cls._backends[backend](**kwargs) + else: + _instance.client = cls._prefix_to_backends[prefix](**kwargs) + + cls._instances[arg_key] = _instance + + return _instance + + @property + def name(self): + return self.client.name + + @property + def allow_symlink(self): + return self.client.allow_symlink + + @staticmethod + def parse_uri_prefix(uri: Union[str, Path]) -> Optional[str]: + """Parse the prefix of a uri. + + Args: + uri (str | Path): Uri to be parsed that contains the file prefix. + + Examples: + >>> FileClient.parse_uri_prefix('s3://path/of/your/file') + 's3' + + Returns: + str | None: Return the prefix of uri if the uri contains '://' + else ``None``. + """ + assert is_filepath(uri) + uri = str(uri) + if '://' not in uri: + return None + else: + prefix, _ = uri.split('://') + # In the case of PetrelBackend, the prefix may contains the cluster + # name like clusterName:s3 + if ':' in prefix: + _, prefix = prefix.split(':') + return prefix + + @classmethod + def infer_client(cls, + file_client_args: Optional[dict] = None, + uri: Optional[Union[str, Path]] = None) -> 'FileClient': + """Infer a suitable file client based on the URI and arguments. + + Args: + file_client_args (dict, optional): Arguments to instantiate a + FileClient. Default: None. + uri (str | Path, optional): Uri to be parsed that contains the file + prefix. Default: None. + + Examples: + >>> uri = 's3://path/of/your/file' + >>> file_client = FileClient.infer_client(uri=uri) + >>> file_client_args = {'backend': 'petrel'} + >>> file_client = FileClient.infer_client(file_client_args) + + Returns: + FileClient: Instantiated FileClient object. + """ + assert file_client_args is not None or uri is not None + if file_client_args is None: + file_prefix = cls.parse_uri_prefix(uri) # type: ignore + return cls(prefix=file_prefix) + else: + return cls(**file_client_args) + + @classmethod + def _register_backend(cls, name, backend, force=False, prefixes=None): + if not isinstance(name, str): + raise TypeError('the backend name should be a string, ' + f'but got {type(name)}') + if not inspect.isclass(backend): + raise TypeError( + f'backend should be a class but got {type(backend)}') + if not issubclass(backend, BaseStorageBackend): + raise TypeError( + f'backend {backend} is not a subclass of BaseStorageBackend') + if not force and name in cls._backends: + raise KeyError( + f'{name} is already registered as a storage backend, ' + 'add "force=True" if you want to override it') + + if name in cls._backends and force: + cls._overridden_backends.add(name) + cls._backends[name] = backend + + if prefixes is not None: + if isinstance(prefixes, str): + prefixes = [prefixes] + else: + assert isinstance(prefixes, (list, tuple)) + for prefix in prefixes: + if prefix not in cls._prefix_to_backends: + cls._prefix_to_backends[prefix] = backend + elif (prefix in cls._prefix_to_backends) and force: + cls._overridden_prefixes.add(prefix) + cls._prefix_to_backends[prefix] = backend + else: + raise KeyError( + f'{prefix} is already registered as a storage backend,' + ' add "force=True" if you want to override it') + + @classmethod + def register_backend(cls, name, backend=None, force=False, prefixes=None): + """Register a backend to FileClient. + + This method can be used as a normal class method or a decorator. + + .. code-block:: python + + class NewBackend(BaseStorageBackend): + + def get(self, filepath): + return filepath + + def get_text(self, filepath): + return filepath + + FileClient.register_backend('new', NewBackend) + + or + + .. code-block:: python + + @FileClient.register_backend('new') + class NewBackend(BaseStorageBackend): + + def get(self, filepath): + return filepath + + def get_text(self, filepath): + return filepath + + Args: + name (str): The name of the registered backend. + backend (class, optional): The backend class to be registered, + which must be a subclass of :class:`BaseStorageBackend`. + When this method is used as a decorator, backend is None. + Defaults to None. + force (bool, optional): Whether to override the backend if the name + has already been registered. Defaults to False. + prefixes (str or list[str] or tuple[str], optional): The prefixes + of the registered storage backend. Default: None. + `New in version 1.3.15.` + """ + if backend is not None: + cls._register_backend( + name, backend, force=force, prefixes=prefixes) + return + + def _register(backend_cls): + cls._register_backend( + name, backend_cls, force=force, prefixes=prefixes) + return backend_cls + + return _register + + def get(self, filepath: Union[str, Path]) -> Union[bytes, memoryview]: + """Read data from a given ``filepath`` with 'rb' mode. + + Note: + There are two types of return values for ``get``, one is ``bytes`` + and the other is ``memoryview``. The advantage of using memoryview + is that you can avoid copying, and if you want to convert it to + ``bytes``, you can use ``.tobytes()``. + + Args: + filepath (str or Path): Path to read data. + + Returns: + bytes | memoryview: Expected bytes object or a memory view of the + bytes object. + """ + return self.client.get(filepath) + + def get_text(self, filepath: Union[str, Path], encoding='utf-8') -> str: + """Read data from a given ``filepath`` with 'r' mode. + + Args: + filepath (str or Path): Path to read data. + encoding (str): The encoding format used to open the ``filepath``. + Default: 'utf-8'. + + Returns: + str: Expected text reading from ``filepath``. + """ + return self.client.get_text(filepath, encoding) + + def put(self, obj: bytes, filepath: Union[str, Path]) -> None: + """Write data to a given ``filepath`` with 'wb' mode. + + Note: + ``put`` should create a directory if the directory of ``filepath`` + does not exist. + + Args: + obj (bytes): Data to be written. + filepath (str or Path): Path to write data. + """ + self.client.put(obj, filepath) + + def put_text(self, obj: str, filepath: Union[str, Path]) -> None: + """Write data to a given ``filepath`` with 'w' mode. + + Note: + ``put_text`` should create a directory if the directory of + ``filepath`` does not exist. + + Args: + obj (str): Data to be written. + filepath (str or Path): Path to write data. + encoding (str, optional): The encoding format used to open the + `filepath`. Default: 'utf-8'. + """ + self.client.put_text(obj, filepath) + + def remove(self, filepath: Union[str, Path]) -> None: + """Remove a file. + + Args: + filepath (str, Path): Path to be removed. + """ + self.client.remove(filepath) + + def exists(self, filepath: Union[str, Path]) -> bool: + """Check whether a file path exists. + + Args: + filepath (str or Path): Path to be checked whether exists. + + Returns: + bool: Return ``True`` if ``filepath`` exists, ``False`` otherwise. + """ + return self.client.exists(filepath) + + def isdir(self, filepath: Union[str, Path]) -> bool: + """Check whether a file path is a directory. + + Args: + filepath (str or Path): Path to be checked whether it is a + directory. + + Returns: + bool: Return ``True`` if ``filepath`` points to a directory, + ``False`` otherwise. + """ + return self.client.isdir(filepath) + + def isfile(self, filepath: Union[str, Path]) -> bool: + """Check whether a file path is a file. + + Args: + filepath (str or Path): Path to be checked whether it is a file. + + Returns: + bool: Return ``True`` if ``filepath`` points to a file, ``False`` + otherwise. + """ + return self.client.isfile(filepath) + + def join_path(self, filepath: Union[str, Path], + *filepaths: Union[str, Path]) -> str: + """Concatenate all file paths. + + Join one or more filepath components intelligently. The return value + is the concatenation of filepath and any members of *filepaths. + + Args: + filepath (str or Path): Path to be concatenated. + + Returns: + str: The result of concatenation. + """ + return self.client.join_path(filepath, *filepaths) + + @contextmanager + def get_local_path(self, filepath: Union[str, Path]) -> Iterable[str]: + """Download data from ``filepath`` and write the data to local path. + + ``get_local_path`` is decorated by :meth:`contxtlib.contextmanager`. It + can be called with ``with`` statement, and when exists from the + ``with`` statement, the temporary path will be released. + + Note: + If the ``filepath`` is a local path, just return itself. + + .. warning:: + ``get_local_path`` is an experimental interface that may change in + the future. + + Args: + filepath (str or Path): Path to be read data. + + Examples: + >>> file_client = FileClient(prefix='s3') + >>> with file_client.get_local_path('s3://bucket/abc.jpg') as path: + ... # do something here + + Yields: + Iterable[str]: Only yield one path. + """ + with self.client.get_local_path(str(filepath)) as local_path: + yield local_path + + def list_dir_or_file(self, + dir_path: Union[str, Path], + list_dir: bool = True, + list_file: bool = True, + suffix: Optional[Union[str, Tuple[str]]] = None, + recursive: bool = False) -> Iterator[str]: + """Scan a directory to find the interested directories or files in + arbitrary order. + + Note: + :meth:`list_dir_or_file` returns the path relative to ``dir_path``. + + Args: + dir_path (str | Path): Path of the directory. + list_dir (bool): List the directories. Default: True. + list_file (bool): List the path of files. Default: True. + suffix (str or tuple[str], optional): File suffix + that we are interested in. Default: None. + recursive (bool): If set to True, recursively scan the + directory. Default: False. + + Yields: + Iterable[str]: A relative path to ``dir_path``. + """ + yield from self.client.list_dir_or_file(dir_path, list_dir, list_file, + suffix, recursive) diff --git a/annotator/mmpkg/mmcv/fileio/handlers/__init__.py b/annotator/mmpkg/mmcv/fileio/handlers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..aa24d91972837b8756b225f4879bac20436eb72a --- /dev/null +++ b/annotator/mmpkg/mmcv/fileio/handlers/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .base import BaseFileHandler +from .json_handler import JsonHandler +from .pickle_handler import PickleHandler +from .yaml_handler import YamlHandler + +__all__ = ['BaseFileHandler', 'JsonHandler', 'PickleHandler', 'YamlHandler'] diff --git a/annotator/mmpkg/mmcv/fileio/handlers/base.py b/annotator/mmpkg/mmcv/fileio/handlers/base.py new file mode 100644 index 0000000000000000000000000000000000000000..288878bc57282fbb2f12b32290152ca8e9d3cab0 --- /dev/null +++ b/annotator/mmpkg/mmcv/fileio/handlers/base.py @@ -0,0 +1,30 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from abc import ABCMeta, abstractmethod + + +class BaseFileHandler(metaclass=ABCMeta): + # `str_like` is a flag to indicate whether the type of file object is + # str-like object or bytes-like object. Pickle only processes bytes-like + # objects but json only processes str-like object. If it is str-like + # object, `StringIO` will be used to process the buffer. + str_like = True + + @abstractmethod + def load_from_fileobj(self, file, **kwargs): + pass + + @abstractmethod + def dump_to_fileobj(self, obj, file, **kwargs): + pass + + @abstractmethod + def dump_to_str(self, obj, **kwargs): + pass + + def load_from_path(self, filepath, mode='r', **kwargs): + with open(filepath, mode) as f: + return self.load_from_fileobj(f, **kwargs) + + def dump_to_path(self, obj, filepath, mode='w', **kwargs): + with open(filepath, mode) as f: + self.dump_to_fileobj(obj, f, **kwargs) diff --git a/annotator/mmpkg/mmcv/fileio/handlers/json_handler.py b/annotator/mmpkg/mmcv/fileio/handlers/json_handler.py new file mode 100644 index 0000000000000000000000000000000000000000..18d4f15f74139d20adff18b20be5529c592a66b6 --- /dev/null +++ b/annotator/mmpkg/mmcv/fileio/handlers/json_handler.py @@ -0,0 +1,36 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import json + +import numpy as np + +from .base import BaseFileHandler + + +def set_default(obj): + """Set default json values for non-serializable values. + + It helps convert ``set``, ``range`` and ``np.ndarray`` data types to list. + It also converts ``np.generic`` (including ``np.int32``, ``np.float32``, + etc.) into plain numbers of plain python built-in types. + """ + if isinstance(obj, (set, range)): + return list(obj) + elif isinstance(obj, np.ndarray): + return obj.tolist() + elif isinstance(obj, np.generic): + return obj.item() + raise TypeError(f'{type(obj)} is unsupported for json dump') + + +class JsonHandler(BaseFileHandler): + + def load_from_fileobj(self, file): + return json.load(file) + + def dump_to_fileobj(self, obj, file, **kwargs): + kwargs.setdefault('default', set_default) + json.dump(obj, file, **kwargs) + + def dump_to_str(self, obj, **kwargs): + kwargs.setdefault('default', set_default) + return json.dumps(obj, **kwargs) diff --git a/annotator/mmpkg/mmcv/fileio/handlers/pickle_handler.py b/annotator/mmpkg/mmcv/fileio/handlers/pickle_handler.py new file mode 100644 index 0000000000000000000000000000000000000000..b37c79bed4ef9fd8913715e62dbe3fc5cafdc3aa --- /dev/null +++ b/annotator/mmpkg/mmcv/fileio/handlers/pickle_handler.py @@ -0,0 +1,28 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import pickle + +from .base import BaseFileHandler + + +class PickleHandler(BaseFileHandler): + + str_like = False + + def load_from_fileobj(self, file, **kwargs): + return pickle.load(file, **kwargs) + + def load_from_path(self, filepath, **kwargs): + return super(PickleHandler, self).load_from_path( + filepath, mode='rb', **kwargs) + + def dump_to_str(self, obj, **kwargs): + kwargs.setdefault('protocol', 2) + return pickle.dumps(obj, **kwargs) + + def dump_to_fileobj(self, obj, file, **kwargs): + kwargs.setdefault('protocol', 2) + pickle.dump(obj, file, **kwargs) + + def dump_to_path(self, obj, filepath, **kwargs): + super(PickleHandler, self).dump_to_path( + obj, filepath, mode='wb', **kwargs) diff --git a/annotator/mmpkg/mmcv/fileio/handlers/yaml_handler.py b/annotator/mmpkg/mmcv/fileio/handlers/yaml_handler.py new file mode 100644 index 0000000000000000000000000000000000000000..c5aa2eea1e8c76f8baf753d1c8c959dee665e543 --- /dev/null +++ b/annotator/mmpkg/mmcv/fileio/handlers/yaml_handler.py @@ -0,0 +1,24 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import yaml + +try: + from yaml import CLoader as Loader, CDumper as Dumper +except ImportError: + from yaml import Loader, Dumper + +from .base import BaseFileHandler # isort:skip + + +class YamlHandler(BaseFileHandler): + + def load_from_fileobj(self, file, **kwargs): + kwargs.setdefault('Loader', Loader) + return yaml.load(file, **kwargs) + + def dump_to_fileobj(self, obj, file, **kwargs): + kwargs.setdefault('Dumper', Dumper) + yaml.dump(obj, file, **kwargs) + + def dump_to_str(self, obj, **kwargs): + kwargs.setdefault('Dumper', Dumper) + return yaml.dump(obj, **kwargs) diff --git a/annotator/mmpkg/mmcv/fileio/io.py b/annotator/mmpkg/mmcv/fileio/io.py new file mode 100644 index 0000000000000000000000000000000000000000..aaefde58aa3ea5b58f86249ce7e1c40c186eb8dd --- /dev/null +++ b/annotator/mmpkg/mmcv/fileio/io.py @@ -0,0 +1,151 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from io import BytesIO, StringIO +from pathlib import Path + +from ..utils import is_list_of, is_str +from .file_client import FileClient +from .handlers import BaseFileHandler, JsonHandler, PickleHandler, YamlHandler + +file_handlers = { + 'json': JsonHandler(), + 'yaml': YamlHandler(), + 'yml': YamlHandler(), + 'pickle': PickleHandler(), + 'pkl': PickleHandler() +} + + +def load(file, file_format=None, file_client_args=None, **kwargs): + """Load data from json/yaml/pickle files. + + This method provides a unified api for loading data from serialized files. + + Note: + In v1.3.16 and later, ``load`` supports loading data from serialized + files those can be storaged in different backends. + + Args: + file (str or :obj:`Path` or file-like object): Filename or a file-like + object. + file_format (str, optional): If not specified, the file format will be + inferred from the file extension, otherwise use the specified one. + Currently supported formats include "json", "yaml/yml" and + "pickle/pkl". + file_client_args (dict, optional): Arguments to instantiate a + FileClient. See :class:`mmcv.fileio.FileClient` for details. + Default: None. + + Examples: + >>> load('/path/of/your/file') # file is storaged in disk + >>> load('https://path/of/your/file') # file is storaged in Internet + >>> load('s3://path/of/your/file') # file is storaged in petrel + + Returns: + The content from the file. + """ + if isinstance(file, Path): + file = str(file) + if file_format is None and is_str(file): + file_format = file.split('.')[-1] + if file_format not in file_handlers: + raise TypeError(f'Unsupported format: {file_format}') + + handler = file_handlers[file_format] + if is_str(file): + file_client = FileClient.infer_client(file_client_args, file) + if handler.str_like: + with StringIO(file_client.get_text(file)) as f: + obj = handler.load_from_fileobj(f, **kwargs) + else: + with BytesIO(file_client.get(file)) as f: + obj = handler.load_from_fileobj(f, **kwargs) + elif hasattr(file, 'read'): + obj = handler.load_from_fileobj(file, **kwargs) + else: + raise TypeError('"file" must be a filepath str or a file-object') + return obj + + +def dump(obj, file=None, file_format=None, file_client_args=None, **kwargs): + """Dump data to json/yaml/pickle strings or files. + + This method provides a unified api for dumping data as strings or to files, + and also supports custom arguments for each file format. + + Note: + In v1.3.16 and later, ``dump`` supports dumping data as strings or to + files which is saved to different backends. + + Args: + obj (any): The python object to be dumped. + file (str or :obj:`Path` or file-like object, optional): If not + specified, then the object is dumped to a str, otherwise to a file + specified by the filename or file-like object. + file_format (str, optional): Same as :func:`load`. + file_client_args (dict, optional): Arguments to instantiate a + FileClient. See :class:`mmcv.fileio.FileClient` for details. + Default: None. + + Examples: + >>> dump('hello world', '/path/of/your/file') # disk + >>> dump('hello world', 's3://path/of/your/file') # ceph or petrel + + Returns: + bool: True for success, False otherwise. + """ + if isinstance(file, Path): + file = str(file) + if file_format is None: + if is_str(file): + file_format = file.split('.')[-1] + elif file is None: + raise ValueError( + 'file_format must be specified since file is None') + if file_format not in file_handlers: + raise TypeError(f'Unsupported format: {file_format}') + + handler = file_handlers[file_format] + if file is None: + return handler.dump_to_str(obj, **kwargs) + elif is_str(file): + file_client = FileClient.infer_client(file_client_args, file) + if handler.str_like: + with StringIO() as f: + handler.dump_to_fileobj(obj, f, **kwargs) + file_client.put_text(f.getvalue(), file) + else: + with BytesIO() as f: + handler.dump_to_fileobj(obj, f, **kwargs) + file_client.put(f.getvalue(), file) + elif hasattr(file, 'write'): + handler.dump_to_fileobj(obj, file, **kwargs) + else: + raise TypeError('"file" must be a filename str or a file-object') + + +def _register_handler(handler, file_formats): + """Register a handler for some file extensions. + + Args: + handler (:obj:`BaseFileHandler`): Handler to be registered. + file_formats (str or list[str]): File formats to be handled by this + handler. + """ + if not isinstance(handler, BaseFileHandler): + raise TypeError( + f'handler must be a child of BaseFileHandler, not {type(handler)}') + if isinstance(file_formats, str): + file_formats = [file_formats] + if not is_list_of(file_formats, str): + raise TypeError('file_formats must be a str or a list of str') + for ext in file_formats: + file_handlers[ext] = handler + + +def register_handler(file_formats, **kwargs): + + def wrap(cls): + _register_handler(cls(**kwargs), file_formats) + return cls + + return wrap diff --git a/annotator/mmpkg/mmcv/fileio/parse.py b/annotator/mmpkg/mmcv/fileio/parse.py new file mode 100644 index 0000000000000000000000000000000000000000..f60f0d611b8d75692221d0edd7dc993b0a6445c9 --- /dev/null +++ b/annotator/mmpkg/mmcv/fileio/parse.py @@ -0,0 +1,97 @@ +# Copyright (c) OpenMMLab. All rights reserved. + +from io import StringIO + +from .file_client import FileClient + + +def list_from_file(filename, + prefix='', + offset=0, + max_num=0, + encoding='utf-8', + file_client_args=None): + """Load a text file and parse the content as a list of strings. + + Note: + In v1.3.16 and later, ``list_from_file`` supports loading a text file + which can be storaged in different backends and parsing the content as + a list for strings. + + Args: + filename (str): Filename. + prefix (str): The prefix to be inserted to the beginning of each item. + offset (int): The offset of lines. + max_num (int): The maximum number of lines to be read, + zeros and negatives mean no limitation. + encoding (str): Encoding used to open the file. Default utf-8. + file_client_args (dict, optional): Arguments to instantiate a + FileClient. See :class:`mmcv.fileio.FileClient` for details. + Default: None. + + Examples: + >>> list_from_file('/path/of/your/file') # disk + ['hello', 'world'] + >>> list_from_file('s3://path/of/your/file') # ceph or petrel + ['hello', 'world'] + + Returns: + list[str]: A list of strings. + """ + cnt = 0 + item_list = [] + file_client = FileClient.infer_client(file_client_args, filename) + with StringIO(file_client.get_text(filename, encoding)) as f: + for _ in range(offset): + f.readline() + for line in f: + if 0 < max_num <= cnt: + break + item_list.append(prefix + line.rstrip('\n\r')) + cnt += 1 + return item_list + + +def dict_from_file(filename, + key_type=str, + encoding='utf-8', + file_client_args=None): + """Load a text file and parse the content as a dict. + + Each line of the text file will be two or more columns split by + whitespaces or tabs. The first column will be parsed as dict keys, and + the following columns will be parsed as dict values. + + Note: + In v1.3.16 and later, ``dict_from_file`` supports loading a text file + which can be storaged in different backends and parsing the content as + a dict. + + Args: + filename(str): Filename. + key_type(type): Type of the dict keys. str is user by default and + type conversion will be performed if specified. + encoding (str): Encoding used to open the file. Default utf-8. + file_client_args (dict, optional): Arguments to instantiate a + FileClient. See :class:`mmcv.fileio.FileClient` for details. + Default: None. + + Examples: + >>> dict_from_file('/path/of/your/file') # disk + {'key1': 'value1', 'key2': 'value2'} + >>> dict_from_file('s3://path/of/your/file') # ceph or petrel + {'key1': 'value1', 'key2': 'value2'} + + Returns: + dict: The parsed contents. + """ + mapping = {} + file_client = FileClient.infer_client(file_client_args, filename) + with StringIO(file_client.get_text(filename, encoding)) as f: + for line in f: + items = line.rstrip('\n').split() + assert len(items) >= 2 + key = key_type(items[0]) + val = items[1:] if len(items) > 2 else items[1] + mapping[key] = val + return mapping diff --git a/annotator/mmpkg/mmcv/image/__init__.py b/annotator/mmpkg/mmcv/image/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d0051d609d3de4e7562e3fe638335c66617c4d91 --- /dev/null +++ b/annotator/mmpkg/mmcv/image/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .colorspace import (bgr2gray, bgr2hls, bgr2hsv, bgr2rgb, bgr2ycbcr, + gray2bgr, gray2rgb, hls2bgr, hsv2bgr, imconvert, + rgb2bgr, rgb2gray, rgb2ycbcr, ycbcr2bgr, ycbcr2rgb) +from .geometric import (cutout, imcrop, imflip, imflip_, impad, + impad_to_multiple, imrescale, imresize, imresize_like, + imresize_to_multiple, imrotate, imshear, imtranslate, + rescale_size) +from .io import imfrombytes, imread, imwrite, supported_backends, use_backend +from .misc import tensor2imgs +from .photometric import (adjust_brightness, adjust_color, adjust_contrast, + adjust_lighting, adjust_sharpness, auto_contrast, + clahe, imdenormalize, imequalize, iminvert, + imnormalize, imnormalize_, lut_transform, posterize, + solarize) + +__all__ = [ + 'bgr2gray', 'bgr2hls', 'bgr2hsv', 'bgr2rgb', 'gray2bgr', 'gray2rgb', + 'hls2bgr', 'hsv2bgr', 'imconvert', 'rgb2bgr', 'rgb2gray', 'imrescale', + 'imresize', 'imresize_like', 'imresize_to_multiple', 'rescale_size', + 'imcrop', 'imflip', 'imflip_', 'impad', 'impad_to_multiple', 'imrotate', + 'imfrombytes', 'imread', 'imwrite', 'supported_backends', 'use_backend', + 'imdenormalize', 'imnormalize', 'imnormalize_', 'iminvert', 'posterize', + 'solarize', 'rgb2ycbcr', 'bgr2ycbcr', 'ycbcr2rgb', 'ycbcr2bgr', + 'tensor2imgs', 'imshear', 'imtranslate', 'adjust_color', 'imequalize', + 'adjust_brightness', 'adjust_contrast', 'lut_transform', 'clahe', + 'adjust_sharpness', 'auto_contrast', 'cutout', 'adjust_lighting' +] diff --git a/annotator/mmpkg/mmcv/image/colorspace.py b/annotator/mmpkg/mmcv/image/colorspace.py new file mode 100644 index 0000000000000000000000000000000000000000..814533952fdfda23d67cb6a3073692d8c1156add --- /dev/null +++ b/annotator/mmpkg/mmcv/image/colorspace.py @@ -0,0 +1,306 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import cv2 +import numpy as np + + +def imconvert(img, src, dst): + """Convert an image from the src colorspace to dst colorspace. + + Args: + img (ndarray): The input image. + src (str): The source colorspace, e.g., 'rgb', 'hsv'. + dst (str): The destination colorspace, e.g., 'rgb', 'hsv'. + + Returns: + ndarray: The converted image. + """ + code = getattr(cv2, f'COLOR_{src.upper()}2{dst.upper()}') + out_img = cv2.cvtColor(img, code) + return out_img + + +def bgr2gray(img, keepdim=False): + """Convert a BGR image to grayscale image. + + Args: + img (ndarray): The input image. + keepdim (bool): If False (by default), then return the grayscale image + with 2 dims, otherwise 3 dims. + + Returns: + ndarray: The converted grayscale image. + """ + out_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + if keepdim: + out_img = out_img[..., None] + return out_img + + +def rgb2gray(img, keepdim=False): + """Convert a RGB image to grayscale image. + + Args: + img (ndarray): The input image. + keepdim (bool): If False (by default), then return the grayscale image + with 2 dims, otherwise 3 dims. + + Returns: + ndarray: The converted grayscale image. + """ + out_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) + if keepdim: + out_img = out_img[..., None] + return out_img + + +def gray2bgr(img): + """Convert a grayscale image to BGR image. + + Args: + img (ndarray): The input image. + + Returns: + ndarray: The converted BGR image. + """ + img = img[..., None] if img.ndim == 2 else img + out_img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) + return out_img + + +def gray2rgb(img): + """Convert a grayscale image to RGB image. + + Args: + img (ndarray): The input image. + + Returns: + ndarray: The converted RGB image. + """ + img = img[..., None] if img.ndim == 2 else img + out_img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB) + return out_img + + +def _convert_input_type_range(img): + """Convert the type and range of the input image. + + It converts the input image to np.float32 type and range of [0, 1]. + It is mainly used for pre-processing the input image in colorspace + conversion functions such as rgb2ycbcr and ycbcr2rgb. + + Args: + img (ndarray): The input image. It accepts: + 1. np.uint8 type with range [0, 255]; + 2. np.float32 type with range [0, 1]. + + Returns: + (ndarray): The converted image with type of np.float32 and range of + [0, 1]. + """ + img_type = img.dtype + img = img.astype(np.float32) + if img_type == np.float32: + pass + elif img_type == np.uint8: + img /= 255. + else: + raise TypeError('The img type should be np.float32 or np.uint8, ' + f'but got {img_type}') + return img + + +def _convert_output_type_range(img, dst_type): + """Convert the type and range of the image according to dst_type. + + It converts the image to desired type and range. If `dst_type` is np.uint8, + images will be converted to np.uint8 type with range [0, 255]. If + `dst_type` is np.float32, it converts the image to np.float32 type with + range [0, 1]. + It is mainly used for post-processing images in colorspace conversion + functions such as rgb2ycbcr and ycbcr2rgb. + + Args: + img (ndarray): The image to be converted with np.float32 type and + range [0, 255]. + dst_type (np.uint8 | np.float32): If dst_type is np.uint8, it + converts the image to np.uint8 type with range [0, 255]. If + dst_type is np.float32, it converts the image to np.float32 type + with range [0, 1]. + + Returns: + (ndarray): The converted image with desired type and range. + """ + if dst_type not in (np.uint8, np.float32): + raise TypeError('The dst_type should be np.float32 or np.uint8, ' + f'but got {dst_type}') + if dst_type == np.uint8: + img = img.round() + else: + img /= 255. + return img.astype(dst_type) + + +def rgb2ycbcr(img, y_only=False): + """Convert a RGB image to YCbCr image. + + This function produces the same results as Matlab's `rgb2ycbcr` function. + It implements the ITU-R BT.601 conversion for standard-definition + television. See more details in + https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion. + + It differs from a similar function in cv2.cvtColor: `RGB <-> YCrCb`. + In OpenCV, it implements a JPEG conversion. See more details in + https://en.wikipedia.org/wiki/YCbCr#JPEG_conversion. + + Args: + img (ndarray): The input image. It accepts: + 1. np.uint8 type with range [0, 255]; + 2. np.float32 type with range [0, 1]. + y_only (bool): Whether to only return Y channel. Default: False. + + Returns: + ndarray: The converted YCbCr image. The output image has the same type + and range as input image. + """ + img_type = img.dtype + img = _convert_input_type_range(img) + if y_only: + out_img = np.dot(img, [65.481, 128.553, 24.966]) + 16.0 + else: + out_img = np.matmul( + img, [[65.481, -37.797, 112.0], [128.553, -74.203, -93.786], + [24.966, 112.0, -18.214]]) + [16, 128, 128] + out_img = _convert_output_type_range(out_img, img_type) + return out_img + + +def bgr2ycbcr(img, y_only=False): + """Convert a BGR image to YCbCr image. + + The bgr version of rgb2ycbcr. + It implements the ITU-R BT.601 conversion for standard-definition + television. See more details in + https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion. + + It differs from a similar function in cv2.cvtColor: `BGR <-> YCrCb`. + In OpenCV, it implements a JPEG conversion. See more details in + https://en.wikipedia.org/wiki/YCbCr#JPEG_conversion. + + Args: + img (ndarray): The input image. It accepts: + 1. np.uint8 type with range [0, 255]; + 2. np.float32 type with range [0, 1]. + y_only (bool): Whether to only return Y channel. Default: False. + + Returns: + ndarray: The converted YCbCr image. The output image has the same type + and range as input image. + """ + img_type = img.dtype + img = _convert_input_type_range(img) + if y_only: + out_img = np.dot(img, [24.966, 128.553, 65.481]) + 16.0 + else: + out_img = np.matmul( + img, [[24.966, 112.0, -18.214], [128.553, -74.203, -93.786], + [65.481, -37.797, 112.0]]) + [16, 128, 128] + out_img = _convert_output_type_range(out_img, img_type) + return out_img + + +def ycbcr2rgb(img): + """Convert a YCbCr image to RGB image. + + This function produces the same results as Matlab's ycbcr2rgb function. + It implements the ITU-R BT.601 conversion for standard-definition + television. See more details in + https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion. + + It differs from a similar function in cv2.cvtColor: `YCrCb <-> RGB`. + In OpenCV, it implements a JPEG conversion. See more details in + https://en.wikipedia.org/wiki/YCbCr#JPEG_conversion. + + Args: + img (ndarray): The input image. It accepts: + 1. np.uint8 type with range [0, 255]; + 2. np.float32 type with range [0, 1]. + + Returns: + ndarray: The converted RGB image. The output image has the same type + and range as input image. + """ + img_type = img.dtype + img = _convert_input_type_range(img) * 255 + out_img = np.matmul(img, [[0.00456621, 0.00456621, 0.00456621], + [0, -0.00153632, 0.00791071], + [0.00625893, -0.00318811, 0]]) * 255.0 + [ + -222.921, 135.576, -276.836 + ] + out_img = _convert_output_type_range(out_img, img_type) + return out_img + + +def ycbcr2bgr(img): + """Convert a YCbCr image to BGR image. + + The bgr version of ycbcr2rgb. + It implements the ITU-R BT.601 conversion for standard-definition + television. See more details in + https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion. + + It differs from a similar function in cv2.cvtColor: `YCrCb <-> BGR`. + In OpenCV, it implements a JPEG conversion. See more details in + https://en.wikipedia.org/wiki/YCbCr#JPEG_conversion. + + Args: + img (ndarray): The input image. It accepts: + 1. np.uint8 type with range [0, 255]; + 2. np.float32 type with range [0, 1]. + + Returns: + ndarray: The converted BGR image. The output image has the same type + and range as input image. + """ + img_type = img.dtype + img = _convert_input_type_range(img) * 255 + out_img = np.matmul(img, [[0.00456621, 0.00456621, 0.00456621], + [0.00791071, -0.00153632, 0], + [0, -0.00318811, 0.00625893]]) * 255.0 + [ + -276.836, 135.576, -222.921 + ] + out_img = _convert_output_type_range(out_img, img_type) + return out_img + + +def convert_color_factory(src, dst): + + code = getattr(cv2, f'COLOR_{src.upper()}2{dst.upper()}') + + def convert_color(img): + out_img = cv2.cvtColor(img, code) + return out_img + + convert_color.__doc__ = f"""Convert a {src.upper()} image to {dst.upper()} + image. + + Args: + img (ndarray or str): The input image. + + Returns: + ndarray: The converted {dst.upper()} image. + """ + + return convert_color + + +bgr2rgb = convert_color_factory('bgr', 'rgb') + +rgb2bgr = convert_color_factory('rgb', 'bgr') + +bgr2hsv = convert_color_factory('bgr', 'hsv') + +hsv2bgr = convert_color_factory('hsv', 'bgr') + +bgr2hls = convert_color_factory('bgr', 'hls') + +hls2bgr = convert_color_factory('hls', 'bgr') diff --git a/annotator/mmpkg/mmcv/image/geometric.py b/annotator/mmpkg/mmcv/image/geometric.py new file mode 100644 index 0000000000000000000000000000000000000000..cf97c201cb4e43796c911919d03fb26a07ed817d --- /dev/null +++ b/annotator/mmpkg/mmcv/image/geometric.py @@ -0,0 +1,728 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numbers + +import cv2 +import numpy as np + +from ..utils import to_2tuple +from .io import imread_backend + +try: + from PIL import Image +except ImportError: + Image = None + + +def _scale_size(size, scale): + """Rescale a size by a ratio. + + Args: + size (tuple[int]): (w, h). + scale (float | tuple(float)): Scaling factor. + + Returns: + tuple[int]: scaled size. + """ + if isinstance(scale, (float, int)): + scale = (scale, scale) + w, h = size + return int(w * float(scale[0]) + 0.5), int(h * float(scale[1]) + 0.5) + + +cv2_interp_codes = { + 'nearest': cv2.INTER_NEAREST, + 'bilinear': cv2.INTER_LINEAR, + 'bicubic': cv2.INTER_CUBIC, + 'area': cv2.INTER_AREA, + 'lanczos': cv2.INTER_LANCZOS4 +} + +if Image is not None: + pillow_interp_codes = { + 'nearest': Image.NEAREST, + 'bilinear': Image.BILINEAR, + 'bicubic': Image.BICUBIC, + 'box': Image.BOX, + 'lanczos': Image.LANCZOS, + 'hamming': Image.HAMMING + } + + +def imresize(img, + size, + return_scale=False, + interpolation='bilinear', + out=None, + backend=None): + """Resize image to a given size. + + Args: + img (ndarray): The input image. + size (tuple[int]): Target size (w, h). + return_scale (bool): Whether to return `w_scale` and `h_scale`. + interpolation (str): Interpolation method, accepted values are + "nearest", "bilinear", "bicubic", "area", "lanczos" for 'cv2' + backend, "nearest", "bilinear" for 'pillow' backend. + out (ndarray): The output destination. + backend (str | None): The image resize backend type. Options are `cv2`, + `pillow`, `None`. If backend is None, the global imread_backend + specified by ``mmcv.use_backend()`` will be used. Default: None. + + Returns: + tuple | ndarray: (`resized_img`, `w_scale`, `h_scale`) or + `resized_img`. + """ + h, w = img.shape[:2] + if backend is None: + backend = imread_backend + if backend not in ['cv2', 'pillow']: + raise ValueError(f'backend: {backend} is not supported for resize.' + f"Supported backends are 'cv2', 'pillow'") + + if backend == 'pillow': + assert img.dtype == np.uint8, 'Pillow backend only support uint8 type' + pil_image = Image.fromarray(img) + pil_image = pil_image.resize(size, pillow_interp_codes[interpolation]) + resized_img = np.array(pil_image) + else: + resized_img = cv2.resize( + img, size, dst=out, interpolation=cv2_interp_codes[interpolation]) + if not return_scale: + return resized_img + else: + w_scale = size[0] / w + h_scale = size[1] / h + return resized_img, w_scale, h_scale + + +def imresize_to_multiple(img, + divisor, + size=None, + scale_factor=None, + keep_ratio=False, + return_scale=False, + interpolation='bilinear', + out=None, + backend=None): + """Resize image according to a given size or scale factor and then rounds + up the the resized or rescaled image size to the nearest value that can be + divided by the divisor. + + Args: + img (ndarray): The input image. + divisor (int | tuple): Resized image size will be a multiple of + divisor. If divisor is a tuple, divisor should be + (w_divisor, h_divisor). + size (None | int | tuple[int]): Target size (w, h). Default: None. + scale_factor (None | float | tuple[float]): Multiplier for spatial + size. Should match input size if it is a tuple and the 2D style is + (w_scale_factor, h_scale_factor). Default: None. + keep_ratio (bool): Whether to keep the aspect ratio when resizing the + image. Default: False. + return_scale (bool): Whether to return `w_scale` and `h_scale`. + interpolation (str): Interpolation method, accepted values are + "nearest", "bilinear", "bicubic", "area", "lanczos" for 'cv2' + backend, "nearest", "bilinear" for 'pillow' backend. + out (ndarray): The output destination. + backend (str | None): The image resize backend type. Options are `cv2`, + `pillow`, `None`. If backend is None, the global imread_backend + specified by ``mmcv.use_backend()`` will be used. Default: None. + + Returns: + tuple | ndarray: (`resized_img`, `w_scale`, `h_scale`) or + `resized_img`. + """ + h, w = img.shape[:2] + if size is not None and scale_factor is not None: + raise ValueError('only one of size or scale_factor should be defined') + elif size is None and scale_factor is None: + raise ValueError('one of size or scale_factor should be defined') + elif size is not None: + size = to_2tuple(size) + if keep_ratio: + size = rescale_size((w, h), size, return_scale=False) + else: + size = _scale_size((w, h), scale_factor) + + divisor = to_2tuple(divisor) + size = tuple([int(np.ceil(s / d)) * d for s, d in zip(size, divisor)]) + resized_img, w_scale, h_scale = imresize( + img, + size, + return_scale=True, + interpolation=interpolation, + out=out, + backend=backend) + if return_scale: + return resized_img, w_scale, h_scale + else: + return resized_img + + +def imresize_like(img, + dst_img, + return_scale=False, + interpolation='bilinear', + backend=None): + """Resize image to the same size of a given image. + + Args: + img (ndarray): The input image. + dst_img (ndarray): The target image. + return_scale (bool): Whether to return `w_scale` and `h_scale`. + interpolation (str): Same as :func:`resize`. + backend (str | None): Same as :func:`resize`. + + Returns: + tuple or ndarray: (`resized_img`, `w_scale`, `h_scale`) or + `resized_img`. + """ + h, w = dst_img.shape[:2] + return imresize(img, (w, h), return_scale, interpolation, backend=backend) + + +def rescale_size(old_size, scale, return_scale=False): + """Calculate the new size to be rescaled to. + + Args: + old_size (tuple[int]): The old size (w, h) of image. + scale (float | tuple[int]): The scaling factor or maximum size. + If it is a float number, then the image will be rescaled by this + factor, else if it is a tuple of 2 integers, then the image will + be rescaled as large as possible within the scale. + return_scale (bool): Whether to return the scaling factor besides the + rescaled image size. + + Returns: + tuple[int]: The new rescaled image size. + """ + w, h = old_size + if isinstance(scale, (float, int)): + if scale <= 0: + raise ValueError(f'Invalid scale {scale}, must be positive.') + scale_factor = scale + elif isinstance(scale, tuple): + max_long_edge = max(scale) + max_short_edge = min(scale) + scale_factor = min(max_long_edge / max(h, w), + max_short_edge / min(h, w)) + else: + raise TypeError( + f'Scale must be a number or tuple of int, but got {type(scale)}') + + new_size = _scale_size((w, h), scale_factor) + + if return_scale: + return new_size, scale_factor + else: + return new_size + + +def imrescale(img, + scale, + return_scale=False, + interpolation='bilinear', + backend=None): + """Resize image while keeping the aspect ratio. + + Args: + img (ndarray): The input image. + scale (float | tuple[int]): The scaling factor or maximum size. + If it is a float number, then the image will be rescaled by this + factor, else if it is a tuple of 2 integers, then the image will + be rescaled as large as possible within the scale. + return_scale (bool): Whether to return the scaling factor besides the + rescaled image. + interpolation (str): Same as :func:`resize`. + backend (str | None): Same as :func:`resize`. + + Returns: + ndarray: The rescaled image. + """ + h, w = img.shape[:2] + new_size, scale_factor = rescale_size((w, h), scale, return_scale=True) + rescaled_img = imresize( + img, new_size, interpolation=interpolation, backend=backend) + if return_scale: + return rescaled_img, scale_factor + else: + return rescaled_img + + +def imflip(img, direction='horizontal'): + """Flip an image horizontally or vertically. + + Args: + img (ndarray): Image to be flipped. + direction (str): The flip direction, either "horizontal" or + "vertical" or "diagonal". + + Returns: + ndarray: The flipped image. + """ + assert direction in ['horizontal', 'vertical', 'diagonal'] + if direction == 'horizontal': + return np.flip(img, axis=1) + elif direction == 'vertical': + return np.flip(img, axis=0) + else: + return np.flip(img, axis=(0, 1)) + + +def imflip_(img, direction='horizontal'): + """Inplace flip an image horizontally or vertically. + + Args: + img (ndarray): Image to be flipped. + direction (str): The flip direction, either "horizontal" or + "vertical" or "diagonal". + + Returns: + ndarray: The flipped image (inplace). + """ + assert direction in ['horizontal', 'vertical', 'diagonal'] + if direction == 'horizontal': + return cv2.flip(img, 1, img) + elif direction == 'vertical': + return cv2.flip(img, 0, img) + else: + return cv2.flip(img, -1, img) + + +def imrotate(img, + angle, + center=None, + scale=1.0, + border_value=0, + interpolation='bilinear', + auto_bound=False): + """Rotate an image. + + Args: + img (ndarray): Image to be rotated. + angle (float): Rotation angle in degrees, positive values mean + clockwise rotation. + center (tuple[float], optional): Center point (w, h) of the rotation in + the source image. If not specified, the center of the image will be + used. + scale (float): Isotropic scale factor. + border_value (int): Border value. + interpolation (str): Same as :func:`resize`. + auto_bound (bool): Whether to adjust the image size to cover the whole + rotated image. + + Returns: + ndarray: The rotated image. + """ + if center is not None and auto_bound: + raise ValueError('`auto_bound` conflicts with `center`') + h, w = img.shape[:2] + if center is None: + center = ((w - 1) * 0.5, (h - 1) * 0.5) + assert isinstance(center, tuple) + + matrix = cv2.getRotationMatrix2D(center, -angle, scale) + if auto_bound: + cos = np.abs(matrix[0, 0]) + sin = np.abs(matrix[0, 1]) + new_w = h * sin + w * cos + new_h = h * cos + w * sin + matrix[0, 2] += (new_w - w) * 0.5 + matrix[1, 2] += (new_h - h) * 0.5 + w = int(np.round(new_w)) + h = int(np.round(new_h)) + rotated = cv2.warpAffine( + img, + matrix, (w, h), + flags=cv2_interp_codes[interpolation], + borderValue=border_value) + return rotated + + +def bbox_clip(bboxes, img_shape): + """Clip bboxes to fit the image shape. + + Args: + bboxes (ndarray): Shape (..., 4*k) + img_shape (tuple[int]): (height, width) of the image. + + Returns: + ndarray: Clipped bboxes. + """ + assert bboxes.shape[-1] % 4 == 0 + cmin = np.empty(bboxes.shape[-1], dtype=bboxes.dtype) + cmin[0::2] = img_shape[1] - 1 + cmin[1::2] = img_shape[0] - 1 + clipped_bboxes = np.maximum(np.minimum(bboxes, cmin), 0) + return clipped_bboxes + + +def bbox_scaling(bboxes, scale, clip_shape=None): + """Scaling bboxes w.r.t the box center. + + Args: + bboxes (ndarray): Shape(..., 4). + scale (float): Scaling factor. + clip_shape (tuple[int], optional): If specified, bboxes that exceed the + boundary will be clipped according to the given shape (h, w). + + Returns: + ndarray: Scaled bboxes. + """ + if float(scale) == 1.0: + scaled_bboxes = bboxes.copy() + else: + w = bboxes[..., 2] - bboxes[..., 0] + 1 + h = bboxes[..., 3] - bboxes[..., 1] + 1 + dw = (w * (scale - 1)) * 0.5 + dh = (h * (scale - 1)) * 0.5 + scaled_bboxes = bboxes + np.stack((-dw, -dh, dw, dh), axis=-1) + if clip_shape is not None: + return bbox_clip(scaled_bboxes, clip_shape) + else: + return scaled_bboxes + + +def imcrop(img, bboxes, scale=1.0, pad_fill=None): + """Crop image patches. + + 3 steps: scale the bboxes -> clip bboxes -> crop and pad. + + Args: + img (ndarray): Image to be cropped. + bboxes (ndarray): Shape (k, 4) or (4, ), location of cropped bboxes. + scale (float, optional): Scale ratio of bboxes, the default value + 1.0 means no padding. + pad_fill (Number | list[Number]): Value to be filled for padding. + Default: None, which means no padding. + + Returns: + list[ndarray] | ndarray: The cropped image patches. + """ + chn = 1 if img.ndim == 2 else img.shape[2] + if pad_fill is not None: + if isinstance(pad_fill, (int, float)): + pad_fill = [pad_fill for _ in range(chn)] + assert len(pad_fill) == chn + + _bboxes = bboxes[None, ...] if bboxes.ndim == 1 else bboxes + scaled_bboxes = bbox_scaling(_bboxes, scale).astype(np.int32) + clipped_bbox = bbox_clip(scaled_bboxes, img.shape) + + patches = [] + for i in range(clipped_bbox.shape[0]): + x1, y1, x2, y2 = tuple(clipped_bbox[i, :]) + if pad_fill is None: + patch = img[y1:y2 + 1, x1:x2 + 1, ...] + else: + _x1, _y1, _x2, _y2 = tuple(scaled_bboxes[i, :]) + if chn == 1: + patch_shape = (_y2 - _y1 + 1, _x2 - _x1 + 1) + else: + patch_shape = (_y2 - _y1 + 1, _x2 - _x1 + 1, chn) + patch = np.array( + pad_fill, dtype=img.dtype) * np.ones( + patch_shape, dtype=img.dtype) + x_start = 0 if _x1 >= 0 else -_x1 + y_start = 0 if _y1 >= 0 else -_y1 + w = x2 - x1 + 1 + h = y2 - y1 + 1 + patch[y_start:y_start + h, x_start:x_start + w, + ...] = img[y1:y1 + h, x1:x1 + w, ...] + patches.append(patch) + + if bboxes.ndim == 1: + return patches[0] + else: + return patches + + +def impad(img, + *, + shape=None, + padding=None, + pad_val=0, + padding_mode='constant'): + """Pad the given image to a certain shape or pad on all sides with + specified padding mode and padding value. + + Args: + img (ndarray): Image to be padded. + shape (tuple[int]): Expected padding shape (h, w). Default: None. + padding (int or tuple[int]): Padding on each border. If a single int is + provided this is used to pad all borders. If tuple of length 2 is + provided this is the padding on left/right and top/bottom + respectively. If a tuple of length 4 is provided this is the + padding for the left, top, right and bottom borders respectively. + Default: None. Note that `shape` and `padding` can not be both + set. + pad_val (Number | Sequence[Number]): Values to be filled in padding + areas when padding_mode is 'constant'. Default: 0. + padding_mode (str): Type of padding. Should be: constant, edge, + reflect or symmetric. Default: constant. + + - constant: pads with a constant value, this value is specified + with pad_val. + - edge: pads with the last value at the edge of the image. + - reflect: pads with reflection of image without repeating the + last value on the edge. For example, padding [1, 2, 3, 4] + with 2 elements on both sides in reflect mode will result + in [3, 2, 1, 2, 3, 4, 3, 2]. + - symmetric: pads with reflection of image repeating the last + value on the edge. For example, padding [1, 2, 3, 4] with + 2 elements on both sides in symmetric mode will result in + [2, 1, 1, 2, 3, 4, 4, 3] + + Returns: + ndarray: The padded image. + """ + + assert (shape is not None) ^ (padding is not None) + if shape is not None: + padding = (0, 0, shape[1] - img.shape[1], shape[0] - img.shape[0]) + + # check pad_val + if isinstance(pad_val, tuple): + assert len(pad_val) == img.shape[-1] + elif not isinstance(pad_val, numbers.Number): + raise TypeError('pad_val must be a int or a tuple. ' + f'But received {type(pad_val)}') + + # check padding + if isinstance(padding, tuple) and len(padding) in [2, 4]: + if len(padding) == 2: + padding = (padding[0], padding[1], padding[0], padding[1]) + elif isinstance(padding, numbers.Number): + padding = (padding, padding, padding, padding) + else: + raise ValueError('Padding must be a int or a 2, or 4 element tuple.' + f'But received {padding}') + + # check padding mode + assert padding_mode in ['constant', 'edge', 'reflect', 'symmetric'] + + border_type = { + 'constant': cv2.BORDER_CONSTANT, + 'edge': cv2.BORDER_REPLICATE, + 'reflect': cv2.BORDER_REFLECT_101, + 'symmetric': cv2.BORDER_REFLECT + } + img = cv2.copyMakeBorder( + img, + padding[1], + padding[3], + padding[0], + padding[2], + border_type[padding_mode], + value=pad_val) + + return img + + +def impad_to_multiple(img, divisor, pad_val=0): + """Pad an image to ensure each edge to be multiple to some number. + + Args: + img (ndarray): Image to be padded. + divisor (int): Padded image edges will be multiple to divisor. + pad_val (Number | Sequence[Number]): Same as :func:`impad`. + + Returns: + ndarray: The padded image. + """ + pad_h = int(np.ceil(img.shape[0] / divisor)) * divisor + pad_w = int(np.ceil(img.shape[1] / divisor)) * divisor + return impad(img, shape=(pad_h, pad_w), pad_val=pad_val) + + +def cutout(img, shape, pad_val=0): + """Randomly cut out a rectangle from the original img. + + Args: + img (ndarray): Image to be cutout. + shape (int | tuple[int]): Expected cutout shape (h, w). If given as a + int, the value will be used for both h and w. + pad_val (int | float | tuple[int | float]): Values to be filled in the + cut area. Defaults to 0. + + Returns: + ndarray: The cutout image. + """ + + channels = 1 if img.ndim == 2 else img.shape[2] + if isinstance(shape, int): + cut_h, cut_w = shape, shape + else: + assert isinstance(shape, tuple) and len(shape) == 2, \ + f'shape must be a int or a tuple with length 2, but got type ' \ + f'{type(shape)} instead.' + cut_h, cut_w = shape + if isinstance(pad_val, (int, float)): + pad_val = tuple([pad_val] * channels) + elif isinstance(pad_val, tuple): + assert len(pad_val) == channels, \ + 'Expected the num of elements in tuple equals the channels' \ + 'of input image. Found {} vs {}'.format( + len(pad_val), channels) + else: + raise TypeError(f'Invalid type {type(pad_val)} for `pad_val`') + + img_h, img_w = img.shape[:2] + y0 = np.random.uniform(img_h) + x0 = np.random.uniform(img_w) + + y1 = int(max(0, y0 - cut_h / 2.)) + x1 = int(max(0, x0 - cut_w / 2.)) + y2 = min(img_h, y1 + cut_h) + x2 = min(img_w, x1 + cut_w) + + if img.ndim == 2: + patch_shape = (y2 - y1, x2 - x1) + else: + patch_shape = (y2 - y1, x2 - x1, channels) + + img_cutout = img.copy() + patch = np.array( + pad_val, dtype=img.dtype) * np.ones( + patch_shape, dtype=img.dtype) + img_cutout[y1:y2, x1:x2, ...] = patch + + return img_cutout + + +def _get_shear_matrix(magnitude, direction='horizontal'): + """Generate the shear matrix for transformation. + + Args: + magnitude (int | float): The magnitude used for shear. + direction (str): The flip direction, either "horizontal" + or "vertical". + + Returns: + ndarray: The shear matrix with dtype float32. + """ + if direction == 'horizontal': + shear_matrix = np.float32([[1, magnitude, 0], [0, 1, 0]]) + elif direction == 'vertical': + shear_matrix = np.float32([[1, 0, 0], [magnitude, 1, 0]]) + return shear_matrix + + +def imshear(img, + magnitude, + direction='horizontal', + border_value=0, + interpolation='bilinear'): + """Shear an image. + + Args: + img (ndarray): Image to be sheared with format (h, w) + or (h, w, c). + magnitude (int | float): The magnitude used for shear. + direction (str): The flip direction, either "horizontal" + or "vertical". + border_value (int | tuple[int]): Value used in case of a + constant border. + interpolation (str): Same as :func:`resize`. + + Returns: + ndarray: The sheared image. + """ + assert direction in ['horizontal', + 'vertical'], f'Invalid direction: {direction}' + height, width = img.shape[:2] + if img.ndim == 2: + channels = 1 + elif img.ndim == 3: + channels = img.shape[-1] + if isinstance(border_value, int): + border_value = tuple([border_value] * channels) + elif isinstance(border_value, tuple): + assert len(border_value) == channels, \ + 'Expected the num of elements in tuple equals the channels' \ + 'of input image. Found {} vs {}'.format( + len(border_value), channels) + else: + raise ValueError( + f'Invalid type {type(border_value)} for `border_value`') + shear_matrix = _get_shear_matrix(magnitude, direction) + sheared = cv2.warpAffine( + img, + shear_matrix, + (width, height), + # Note case when the number elements in `border_value` + # greater than 3 (e.g. shearing masks whose channels large + # than 3) will raise TypeError in `cv2.warpAffine`. + # Here simply slice the first 3 values in `border_value`. + borderValue=border_value[:3], + flags=cv2_interp_codes[interpolation]) + return sheared + + +def _get_translate_matrix(offset, direction='horizontal'): + """Generate the translate matrix. + + Args: + offset (int | float): The offset used for translate. + direction (str): The translate direction, either + "horizontal" or "vertical". + + Returns: + ndarray: The translate matrix with dtype float32. + """ + if direction == 'horizontal': + translate_matrix = np.float32([[1, 0, offset], [0, 1, 0]]) + elif direction == 'vertical': + translate_matrix = np.float32([[1, 0, 0], [0, 1, offset]]) + return translate_matrix + + +def imtranslate(img, + offset, + direction='horizontal', + border_value=0, + interpolation='bilinear'): + """Translate an image. + + Args: + img (ndarray): Image to be translated with format + (h, w) or (h, w, c). + offset (int | float): The offset used for translate. + direction (str): The translate direction, either "horizontal" + or "vertical". + border_value (int | tuple[int]): Value used in case of a + constant border. + interpolation (str): Same as :func:`resize`. + + Returns: + ndarray: The translated image. + """ + assert direction in ['horizontal', + 'vertical'], f'Invalid direction: {direction}' + height, width = img.shape[:2] + if img.ndim == 2: + channels = 1 + elif img.ndim == 3: + channels = img.shape[-1] + if isinstance(border_value, int): + border_value = tuple([border_value] * channels) + elif isinstance(border_value, tuple): + assert len(border_value) == channels, \ + 'Expected the num of elements in tuple equals the channels' \ + 'of input image. Found {} vs {}'.format( + len(border_value), channels) + else: + raise ValueError( + f'Invalid type {type(border_value)} for `border_value`.') + translate_matrix = _get_translate_matrix(offset, direction) + translated = cv2.warpAffine( + img, + translate_matrix, + (width, height), + # Note case when the number elements in `border_value` + # greater than 3 (e.g. translating masks whose channels + # large than 3) will raise TypeError in `cv2.warpAffine`. + # Here simply slice the first 3 values in `border_value`. + borderValue=border_value[:3], + flags=cv2_interp_codes[interpolation]) + return translated diff --git a/annotator/mmpkg/mmcv/image/io.py b/annotator/mmpkg/mmcv/image/io.py new file mode 100644 index 0000000000000000000000000000000000000000..4e8f1877978840aede93774d86643b129751db13 --- /dev/null +++ b/annotator/mmpkg/mmcv/image/io.py @@ -0,0 +1,258 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import io +import os.path as osp +from pathlib import Path + +import cv2 +import numpy as np +from cv2 import (IMREAD_COLOR, IMREAD_GRAYSCALE, IMREAD_IGNORE_ORIENTATION, + IMREAD_UNCHANGED) + +from annotator.mmpkg.mmcv.utils import check_file_exist, is_str, mkdir_or_exist + +try: + from turbojpeg import TJCS_RGB, TJPF_BGR, TJPF_GRAY, TurboJPEG +except ImportError: + TJCS_RGB = TJPF_GRAY = TJPF_BGR = TurboJPEG = None + +try: + from PIL import Image, ImageOps +except ImportError: + Image = None + +try: + import tifffile +except ImportError: + tifffile = None + +jpeg = None +supported_backends = ['cv2', 'turbojpeg', 'pillow', 'tifffile'] + +imread_flags = { + 'color': IMREAD_COLOR, + 'grayscale': IMREAD_GRAYSCALE, + 'unchanged': IMREAD_UNCHANGED, + 'color_ignore_orientation': IMREAD_IGNORE_ORIENTATION | IMREAD_COLOR, + 'grayscale_ignore_orientation': + IMREAD_IGNORE_ORIENTATION | IMREAD_GRAYSCALE +} + +imread_backend = 'cv2' + + +def use_backend(backend): + """Select a backend for image decoding. + + Args: + backend (str): The image decoding backend type. Options are `cv2`, + `pillow`, `turbojpeg` (see https://github.com/lilohuang/PyTurboJPEG) + and `tifffile`. `turbojpeg` is faster but it only supports `.jpeg` + file format. + """ + assert backend in supported_backends + global imread_backend + imread_backend = backend + if imread_backend == 'turbojpeg': + if TurboJPEG is None: + raise ImportError('`PyTurboJPEG` is not installed') + global jpeg + if jpeg is None: + jpeg = TurboJPEG() + elif imread_backend == 'pillow': + if Image is None: + raise ImportError('`Pillow` is not installed') + elif imread_backend == 'tifffile': + if tifffile is None: + raise ImportError('`tifffile` is not installed') + + +def _jpegflag(flag='color', channel_order='bgr'): + channel_order = channel_order.lower() + if channel_order not in ['rgb', 'bgr']: + raise ValueError('channel order must be either "rgb" or "bgr"') + + if flag == 'color': + if channel_order == 'bgr': + return TJPF_BGR + elif channel_order == 'rgb': + return TJCS_RGB + elif flag == 'grayscale': + return TJPF_GRAY + else: + raise ValueError('flag must be "color" or "grayscale"') + + +def _pillow2array(img, flag='color', channel_order='bgr'): + """Convert a pillow image to numpy array. + + Args: + img (:obj:`PIL.Image.Image`): The image loaded using PIL + flag (str): Flags specifying the color type of a loaded image, + candidates are 'color', 'grayscale' and 'unchanged'. + Default to 'color'. + channel_order (str): The channel order of the output image array, + candidates are 'bgr' and 'rgb'. Default to 'bgr'. + + Returns: + np.ndarray: The converted numpy array + """ + channel_order = channel_order.lower() + if channel_order not in ['rgb', 'bgr']: + raise ValueError('channel order must be either "rgb" or "bgr"') + + if flag == 'unchanged': + array = np.array(img) + if array.ndim >= 3 and array.shape[2] >= 3: # color image + array[:, :, :3] = array[:, :, (2, 1, 0)] # RGB to BGR + else: + # Handle exif orientation tag + if flag in ['color', 'grayscale']: + img = ImageOps.exif_transpose(img) + # If the image mode is not 'RGB', convert it to 'RGB' first. + if img.mode != 'RGB': + if img.mode != 'LA': + # Most formats except 'LA' can be directly converted to RGB + img = img.convert('RGB') + else: + # When the mode is 'LA', the default conversion will fill in + # the canvas with black, which sometimes shadows black objects + # in the foreground. + # + # Therefore, a random color (124, 117, 104) is used for canvas + img_rgba = img.convert('RGBA') + img = Image.new('RGB', img_rgba.size, (124, 117, 104)) + img.paste(img_rgba, mask=img_rgba.split()[3]) # 3 is alpha + if flag in ['color', 'color_ignore_orientation']: + array = np.array(img) + if channel_order != 'rgb': + array = array[:, :, ::-1] # RGB to BGR + elif flag in ['grayscale', 'grayscale_ignore_orientation']: + img = img.convert('L') + array = np.array(img) + else: + raise ValueError( + 'flag must be "color", "grayscale", "unchanged", ' + f'"color_ignore_orientation" or "grayscale_ignore_orientation"' + f' but got {flag}') + return array + + +def imread(img_or_path, flag='color', channel_order='bgr', backend=None): + """Read an image. + + Args: + img_or_path (ndarray or str or Path): Either a numpy array or str or + pathlib.Path. If it is a numpy array (loaded image), then + it will be returned as is. + flag (str): Flags specifying the color type of a loaded image, + candidates are `color`, `grayscale`, `unchanged`, + `color_ignore_orientation` and `grayscale_ignore_orientation`. + By default, `cv2` and `pillow` backend would rotate the image + according to its EXIF info unless called with `unchanged` or + `*_ignore_orientation` flags. `turbojpeg` and `tifffile` backend + always ignore image's EXIF info regardless of the flag. + The `turbojpeg` backend only supports `color` and `grayscale`. + channel_order (str): Order of channel, candidates are `bgr` and `rgb`. + backend (str | None): The image decoding backend type. Options are + `cv2`, `pillow`, `turbojpeg`, `tifffile`, `None`. + If backend is None, the global imread_backend specified by + ``mmcv.use_backend()`` will be used. Default: None. + + Returns: + ndarray: Loaded image array. + """ + + if backend is None: + backend = imread_backend + if backend not in supported_backends: + raise ValueError(f'backend: {backend} is not supported. Supported ' + "backends are 'cv2', 'turbojpeg', 'pillow'") + if isinstance(img_or_path, Path): + img_or_path = str(img_or_path) + + if isinstance(img_or_path, np.ndarray): + return img_or_path + elif is_str(img_or_path): + check_file_exist(img_or_path, + f'img file does not exist: {img_or_path}') + if backend == 'turbojpeg': + with open(img_or_path, 'rb') as in_file: + img = jpeg.decode(in_file.read(), + _jpegflag(flag, channel_order)) + if img.shape[-1] == 1: + img = img[:, :, 0] + return img + elif backend == 'pillow': + img = Image.open(img_or_path) + img = _pillow2array(img, flag, channel_order) + return img + elif backend == 'tifffile': + img = tifffile.imread(img_or_path) + return img + else: + flag = imread_flags[flag] if is_str(flag) else flag + img = cv2.imread(img_or_path, flag) + if flag == IMREAD_COLOR and channel_order == 'rgb': + cv2.cvtColor(img, cv2.COLOR_BGR2RGB, img) + return img + else: + raise TypeError('"img" must be a numpy array or a str or ' + 'a pathlib.Path object') + + +def imfrombytes(content, flag='color', channel_order='bgr', backend=None): + """Read an image from bytes. + + Args: + content (bytes): Image bytes got from files or other streams. + flag (str): Same as :func:`imread`. + backend (str | None): The image decoding backend type. Options are + `cv2`, `pillow`, `turbojpeg`, `None`. If backend is None, the + global imread_backend specified by ``mmcv.use_backend()`` will be + used. Default: None. + + Returns: + ndarray: Loaded image array. + """ + + if backend is None: + backend = imread_backend + if backend not in supported_backends: + raise ValueError(f'backend: {backend} is not supported. Supported ' + "backends are 'cv2', 'turbojpeg', 'pillow'") + if backend == 'turbojpeg': + img = jpeg.decode(content, _jpegflag(flag, channel_order)) + if img.shape[-1] == 1: + img = img[:, :, 0] + return img + elif backend == 'pillow': + buff = io.BytesIO(content) + img = Image.open(buff) + img = _pillow2array(img, flag, channel_order) + return img + else: + img_np = np.frombuffer(content, np.uint8) + flag = imread_flags[flag] if is_str(flag) else flag + img = cv2.imdecode(img_np, flag) + if flag == IMREAD_COLOR and channel_order == 'rgb': + cv2.cvtColor(img, cv2.COLOR_BGR2RGB, img) + return img + + +def imwrite(img, file_path, params=None, auto_mkdir=True): + """Write image to file. + + Args: + img (ndarray): Image array to be written. + file_path (str): Image file path. + params (None or list): Same as opencv :func:`imwrite` interface. + auto_mkdir (bool): If the parent folder of `file_path` does not exist, + whether to create it automatically. + + Returns: + bool: Successful or not. + """ + if auto_mkdir: + dir_name = osp.abspath(osp.dirname(file_path)) + mkdir_or_exist(dir_name) + return cv2.imwrite(file_path, img, params) diff --git a/annotator/mmpkg/mmcv/image/misc.py b/annotator/mmpkg/mmcv/image/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..cd60e66131719ca0627569598809366b9c1ac64d --- /dev/null +++ b/annotator/mmpkg/mmcv/image/misc.py @@ -0,0 +1,44 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numpy as np + +import annotator.mmpkg.mmcv as mmcv + +try: + import torch +except ImportError: + torch = None + + +def tensor2imgs(tensor, mean=(0, 0, 0), std=(1, 1, 1), to_rgb=True): + """Convert tensor to 3-channel images. + + Args: + tensor (torch.Tensor): Tensor that contains multiple images, shape ( + N, C, H, W). + mean (tuple[float], optional): Mean of images. Defaults to (0, 0, 0). + std (tuple[float], optional): Standard deviation of images. + Defaults to (1, 1, 1). + to_rgb (bool, optional): Whether the tensor was converted to RGB + format in the first place. If so, convert it back to BGR. + Defaults to True. + + Returns: + list[np.ndarray]: A list that contains multiple images. + """ + + if torch is None: + raise RuntimeError('pytorch is not installed') + assert torch.is_tensor(tensor) and tensor.ndim == 4 + assert len(mean) == 3 + assert len(std) == 3 + + num_imgs = tensor.size(0) + mean = np.array(mean, dtype=np.float32) + std = np.array(std, dtype=np.float32) + imgs = [] + for img_id in range(num_imgs): + img = tensor[img_id, ...].cpu().numpy().transpose(1, 2, 0) + img = mmcv.imdenormalize( + img, mean, std, to_bgr=to_rgb).astype(np.uint8) + imgs.append(np.ascontiguousarray(img)) + return imgs diff --git a/annotator/mmpkg/mmcv/image/photometric.py b/annotator/mmpkg/mmcv/image/photometric.py new file mode 100644 index 0000000000000000000000000000000000000000..5085d012019c0cbf56f66f421a378278c1a058ae --- /dev/null +++ b/annotator/mmpkg/mmcv/image/photometric.py @@ -0,0 +1,428 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import cv2 +import numpy as np + +from ..utils import is_tuple_of +from .colorspace import bgr2gray, gray2bgr + + +def imnormalize(img, mean, std, to_rgb=True): + """Normalize an image with mean and std. + + Args: + img (ndarray): Image to be normalized. + mean (ndarray): The mean to be used for normalize. + std (ndarray): The std to be used for normalize. + to_rgb (bool): Whether to convert to rgb. + + Returns: + ndarray: The normalized image. + """ + img = img.copy().astype(np.float32) + return imnormalize_(img, mean, std, to_rgb) + + +def imnormalize_(img, mean, std, to_rgb=True): + """Inplace normalize an image with mean and std. + + Args: + img (ndarray): Image to be normalized. + mean (ndarray): The mean to be used for normalize. + std (ndarray): The std to be used for normalize. + to_rgb (bool): Whether to convert to rgb. + + Returns: + ndarray: The normalized image. + """ + # cv2 inplace normalization does not accept uint8 + assert img.dtype != np.uint8 + mean = np.float64(mean.reshape(1, -1)) + stdinv = 1 / np.float64(std.reshape(1, -1)) + if to_rgb: + cv2.cvtColor(img, cv2.COLOR_BGR2RGB, img) # inplace + cv2.subtract(img, mean, img) # inplace + cv2.multiply(img, stdinv, img) # inplace + return img + + +def imdenormalize(img, mean, std, to_bgr=True): + assert img.dtype != np.uint8 + mean = mean.reshape(1, -1).astype(np.float64) + std = std.reshape(1, -1).astype(np.float64) + img = cv2.multiply(img, std) # make a copy + cv2.add(img, mean, img) # inplace + if to_bgr: + cv2.cvtColor(img, cv2.COLOR_RGB2BGR, img) # inplace + return img + + +def iminvert(img): + """Invert (negate) an image. + + Args: + img (ndarray): Image to be inverted. + + Returns: + ndarray: The inverted image. + """ + return np.full_like(img, 255) - img + + +def solarize(img, thr=128): + """Solarize an image (invert all pixel values above a threshold) + + Args: + img (ndarray): Image to be solarized. + thr (int): Threshold for solarizing (0 - 255). + + Returns: + ndarray: The solarized image. + """ + img = np.where(img < thr, img, 255 - img) + return img + + +def posterize(img, bits): + """Posterize an image (reduce the number of bits for each color channel) + + Args: + img (ndarray): Image to be posterized. + bits (int): Number of bits (1 to 8) to use for posterizing. + + Returns: + ndarray: The posterized image. + """ + shift = 8 - bits + img = np.left_shift(np.right_shift(img, shift), shift) + return img + + +def adjust_color(img, alpha=1, beta=None, gamma=0): + r"""It blends the source image and its gray image: + + .. math:: + output = img * alpha + gray\_img * beta + gamma + + Args: + img (ndarray): The input source image. + alpha (int | float): Weight for the source image. Default 1. + beta (int | float): Weight for the converted gray image. + If None, it's assigned the value (1 - `alpha`). + gamma (int | float): Scalar added to each sum. + Same as :func:`cv2.addWeighted`. Default 0. + + Returns: + ndarray: Colored image which has the same size and dtype as input. + """ + gray_img = bgr2gray(img) + gray_img = np.tile(gray_img[..., None], [1, 1, 3]) + if beta is None: + beta = 1 - alpha + colored_img = cv2.addWeighted(img, alpha, gray_img, beta, gamma) + if not colored_img.dtype == np.uint8: + # Note when the dtype of `img` is not the default `np.uint8` + # (e.g. np.float32), the value in `colored_img` got from cv2 + # is not guaranteed to be in range [0, 255], so here clip + # is needed. + colored_img = np.clip(colored_img, 0, 255) + return colored_img + + +def imequalize(img): + """Equalize the image histogram. + + This function applies a non-linear mapping to the input image, + in order to create a uniform distribution of grayscale values + in the output image. + + Args: + img (ndarray): Image to be equalized. + + Returns: + ndarray: The equalized image. + """ + + def _scale_channel(im, c): + """Scale the data in the corresponding channel.""" + im = im[:, :, c] + # Compute the histogram of the image channel. + histo = np.histogram(im, 256, (0, 255))[0] + # For computing the step, filter out the nonzeros. + nonzero_histo = histo[histo > 0] + step = (np.sum(nonzero_histo) - nonzero_histo[-1]) // 255 + if not step: + lut = np.array(range(256)) + else: + # Compute the cumulative sum, shifted by step // 2 + # and then normalized by step. + lut = (np.cumsum(histo) + (step // 2)) // step + # Shift lut, prepending with 0. + lut = np.concatenate([[0], lut[:-1]], 0) + # handle potential integer overflow + lut[lut > 255] = 255 + # If step is zero, return the original image. + # Otherwise, index from lut. + return np.where(np.equal(step, 0), im, lut[im]) + + # Scales each channel independently and then stacks + # the result. + s1 = _scale_channel(img, 0) + s2 = _scale_channel(img, 1) + s3 = _scale_channel(img, 2) + equalized_img = np.stack([s1, s2, s3], axis=-1) + return equalized_img.astype(img.dtype) + + +def adjust_brightness(img, factor=1.): + """Adjust image brightness. + + This function controls the brightness of an image. An + enhancement factor of 0.0 gives a black image. + A factor of 1.0 gives the original image. This function + blends the source image and the degenerated black image: + + .. math:: + output = img * factor + degenerated * (1 - factor) + + Args: + img (ndarray): Image to be brightened. + factor (float): A value controls the enhancement. + Factor 1.0 returns the original image, lower + factors mean less color (brightness, contrast, + etc), and higher values more. Default 1. + + Returns: + ndarray: The brightened image. + """ + degenerated = np.zeros_like(img) + # Note manually convert the dtype to np.float32, to + # achieve as close results as PIL.ImageEnhance.Brightness. + # Set beta=1-factor, and gamma=0 + brightened_img = cv2.addWeighted( + img.astype(np.float32), factor, degenerated.astype(np.float32), + 1 - factor, 0) + brightened_img = np.clip(brightened_img, 0, 255) + return brightened_img.astype(img.dtype) + + +def adjust_contrast(img, factor=1.): + """Adjust image contrast. + + This function controls the contrast of an image. An + enhancement factor of 0.0 gives a solid grey + image. A factor of 1.0 gives the original image. It + blends the source image and the degenerated mean image: + + .. math:: + output = img * factor + degenerated * (1 - factor) + + Args: + img (ndarray): Image to be contrasted. BGR order. + factor (float): Same as :func:`mmcv.adjust_brightness`. + + Returns: + ndarray: The contrasted image. + """ + gray_img = bgr2gray(img) + hist = np.histogram(gray_img, 256, (0, 255))[0] + mean = round(np.sum(gray_img) / np.sum(hist)) + degenerated = (np.ones_like(img[..., 0]) * mean).astype(img.dtype) + degenerated = gray2bgr(degenerated) + contrasted_img = cv2.addWeighted( + img.astype(np.float32), factor, degenerated.astype(np.float32), + 1 - factor, 0) + contrasted_img = np.clip(contrasted_img, 0, 255) + return contrasted_img.astype(img.dtype) + + +def auto_contrast(img, cutoff=0): + """Auto adjust image contrast. + + This function maximize (normalize) image contrast by first removing cutoff + percent of the lightest and darkest pixels from the histogram and remapping + the image so that the darkest pixel becomes black (0), and the lightest + becomes white (255). + + Args: + img (ndarray): Image to be contrasted. BGR order. + cutoff (int | float | tuple): The cutoff percent of the lightest and + darkest pixels to be removed. If given as tuple, it shall be + (low, high). Otherwise, the single value will be used for both. + Defaults to 0. + + Returns: + ndarray: The contrasted image. + """ + + def _auto_contrast_channel(im, c, cutoff): + im = im[:, :, c] + # Compute the histogram of the image channel. + histo = np.histogram(im, 256, (0, 255))[0] + # Remove cut-off percent pixels from histo + histo_sum = np.cumsum(histo) + cut_low = histo_sum[-1] * cutoff[0] // 100 + cut_high = histo_sum[-1] - histo_sum[-1] * cutoff[1] // 100 + histo_sum = np.clip(histo_sum, cut_low, cut_high) - cut_low + histo = np.concatenate([[histo_sum[0]], np.diff(histo_sum)], 0) + + # Compute mapping + low, high = np.nonzero(histo)[0][0], np.nonzero(histo)[0][-1] + # If all the values have been cut off, return the origin img + if low >= high: + return im + scale = 255.0 / (high - low) + offset = -low * scale + lut = np.array(range(256)) + lut = lut * scale + offset + lut = np.clip(lut, 0, 255) + return lut[im] + + if isinstance(cutoff, (int, float)): + cutoff = (cutoff, cutoff) + else: + assert isinstance(cutoff, tuple), 'cutoff must be of type int, ' \ + f'float or tuple, but got {type(cutoff)} instead.' + # Auto adjusts contrast for each channel independently and then stacks + # the result. + s1 = _auto_contrast_channel(img, 0, cutoff) + s2 = _auto_contrast_channel(img, 1, cutoff) + s3 = _auto_contrast_channel(img, 2, cutoff) + contrasted_img = np.stack([s1, s2, s3], axis=-1) + return contrasted_img.astype(img.dtype) + + +def adjust_sharpness(img, factor=1., kernel=None): + """Adjust image sharpness. + + This function controls the sharpness of an image. An + enhancement factor of 0.0 gives a blurred image. A + factor of 1.0 gives the original image. And a factor + of 2.0 gives a sharpened image. It blends the source + image and the degenerated mean image: + + .. math:: + output = img * factor + degenerated * (1 - factor) + + Args: + img (ndarray): Image to be sharpened. BGR order. + factor (float): Same as :func:`mmcv.adjust_brightness`. + kernel (np.ndarray, optional): Filter kernel to be applied on the img + to obtain the degenerated img. Defaults to None. + + Note: + No value sanity check is enforced on the kernel set by users. So with + an inappropriate kernel, the ``adjust_sharpness`` may fail to perform + the function its name indicates but end up performing whatever + transform determined by the kernel. + + Returns: + ndarray: The sharpened image. + """ + + if kernel is None: + # adopted from PIL.ImageFilter.SMOOTH + kernel = np.array([[1., 1., 1.], [1., 5., 1.], [1., 1., 1.]]) / 13 + assert isinstance(kernel, np.ndarray), \ + f'kernel must be of type np.ndarray, but got {type(kernel)} instead.' + assert kernel.ndim == 2, \ + f'kernel must have a dimension of 2, but got {kernel.ndim} instead.' + + degenerated = cv2.filter2D(img, -1, kernel) + sharpened_img = cv2.addWeighted( + img.astype(np.float32), factor, degenerated.astype(np.float32), + 1 - factor, 0) + sharpened_img = np.clip(sharpened_img, 0, 255) + return sharpened_img.astype(img.dtype) + + +def adjust_lighting(img, eigval, eigvec, alphastd=0.1, to_rgb=True): + """AlexNet-style PCA jitter. + + This data augmentation is proposed in `ImageNet Classification with Deep + Convolutional Neural Networks + `_. + + Args: + img (ndarray): Image to be adjusted lighting. BGR order. + eigval (ndarray): the eigenvalue of the convariance matrix of pixel + values, respectively. + eigvec (ndarray): the eigenvector of the convariance matrix of pixel + values, respectively. + alphastd (float): The standard deviation for distribution of alpha. + Defaults to 0.1 + to_rgb (bool): Whether to convert img to rgb. + + Returns: + ndarray: The adjusted image. + """ + assert isinstance(eigval, np.ndarray) and isinstance(eigvec, np.ndarray), \ + f'eigval and eigvec should both be of type np.ndarray, got ' \ + f'{type(eigval)} and {type(eigvec)} instead.' + + assert eigval.ndim == 1 and eigvec.ndim == 2 + assert eigvec.shape == (3, eigval.shape[0]) + n_eigval = eigval.shape[0] + assert isinstance(alphastd, float), 'alphastd should be of type float, ' \ + f'got {type(alphastd)} instead.' + + img = img.copy().astype(np.float32) + if to_rgb: + cv2.cvtColor(img, cv2.COLOR_BGR2RGB, img) # inplace + + alpha = np.random.normal(0, alphastd, n_eigval) + alter = eigvec \ + * np.broadcast_to(alpha.reshape(1, n_eigval), (3, n_eigval)) \ + * np.broadcast_to(eigval.reshape(1, n_eigval), (3, n_eigval)) + alter = np.broadcast_to(alter.sum(axis=1).reshape(1, 1, 3), img.shape) + img_adjusted = img + alter + return img_adjusted + + +def lut_transform(img, lut_table): + """Transform array by look-up table. + + The function lut_transform fills the output array with values from the + look-up table. Indices of the entries are taken from the input array. + + Args: + img (ndarray): Image to be transformed. + lut_table (ndarray): look-up table of 256 elements; in case of + multi-channel input array, the table should either have a single + channel (in this case the same table is used for all channels) or + the same number of channels as in the input array. + + Returns: + ndarray: The transformed image. + """ + assert isinstance(img, np.ndarray) + assert 0 <= np.min(img) and np.max(img) <= 255 + assert isinstance(lut_table, np.ndarray) + assert lut_table.shape == (256, ) + + return cv2.LUT(np.array(img, dtype=np.uint8), lut_table) + + +def clahe(img, clip_limit=40.0, tile_grid_size=(8, 8)): + """Use CLAHE method to process the image. + + See `ZUIDERVELD,K. Contrast Limited Adaptive Histogram Equalization[J]. + Graphics Gems, 1994:474-485.` for more information. + + Args: + img (ndarray): Image to be processed. + clip_limit (float): Threshold for contrast limiting. Default: 40.0. + tile_grid_size (tuple[int]): Size of grid for histogram equalization. + Input image will be divided into equally sized rectangular tiles. + It defines the number of tiles in row and column. Default: (8, 8). + + Returns: + ndarray: The processed image. + """ + assert isinstance(img, np.ndarray) + assert img.ndim == 2 + assert isinstance(clip_limit, (float, int)) + assert is_tuple_of(tile_grid_size, int) + assert len(tile_grid_size) == 2 + + clahe = cv2.createCLAHE(clip_limit, tile_grid_size) + return clahe.apply(np.array(img, dtype=np.uint8)) diff --git a/annotator/mmpkg/mmcv/model_zoo/deprecated.json b/annotator/mmpkg/mmcv/model_zoo/deprecated.json new file mode 100644 index 0000000000000000000000000000000000000000..25cf6f28caecc22a77e3136fefa6b8dfc0e6cb5b --- /dev/null +++ b/annotator/mmpkg/mmcv/model_zoo/deprecated.json @@ -0,0 +1,6 @@ +{ + "resnet50_caffe": "detectron/resnet50_caffe", + "resnet50_caffe_bgr": "detectron2/resnet50_caffe_bgr", + "resnet101_caffe": "detectron/resnet101_caffe", + "resnet101_caffe_bgr": "detectron2/resnet101_caffe_bgr" +} diff --git a/annotator/mmpkg/mmcv/model_zoo/mmcls.json b/annotator/mmpkg/mmcv/model_zoo/mmcls.json new file mode 100644 index 0000000000000000000000000000000000000000..bdb311d9fe6d9f317290feedc9e37236c6cf6e8f --- /dev/null +++ b/annotator/mmpkg/mmcv/model_zoo/mmcls.json @@ -0,0 +1,31 @@ +{ + "vgg11": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg11_batch256_imagenet_20210208-4271cd6c.pth", + "vgg13": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg13_batch256_imagenet_20210208-4d1d6080.pth", + "vgg16": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg16_batch256_imagenet_20210208-db26f1a5.pth", + "vgg19": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg19_batch256_imagenet_20210208-e6920e4a.pth", + "vgg11_bn": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg11_bn_batch256_imagenet_20210207-f244902c.pth", + "vgg13_bn": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg13_bn_batch256_imagenet_20210207-1a8b7864.pth", + "vgg16_bn": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg16_bn_batch256_imagenet_20210208-7e55cd29.pth", + "vgg19_bn": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg19_bn_batch256_imagenet_20210208-da620c4f.pth", + "resnet18": "https://download.openmmlab.com/mmclassification/v0/resnet/resnet18_batch256_imagenet_20200708-34ab8f90.pth", + "resnet34": "https://download.openmmlab.com/mmclassification/v0/resnet/resnet34_batch256_imagenet_20200708-32ffb4f7.pth", + "resnet50": "https://download.openmmlab.com/mmclassification/v0/resnet/resnet50_batch256_imagenet_20200708-cfb998bf.pth", + "resnet101": "https://download.openmmlab.com/mmclassification/v0/resnet/resnet101_batch256_imagenet_20200708-753f3608.pth", + "resnet152": "https://download.openmmlab.com/mmclassification/v0/resnet/resnet152_batch256_imagenet_20200708-ec25b1f9.pth", + "resnet50_v1d": "https://download.openmmlab.com/mmclassification/v0/resnet/resnetv1d50_batch256_imagenet_20200708-1ad0ce94.pth", + "resnet101_v1d": "https://download.openmmlab.com/mmclassification/v0/resnet/resnetv1d101_batch256_imagenet_20200708-9cb302ef.pth", + "resnet152_v1d": "https://download.openmmlab.com/mmclassification/v0/resnet/resnetv1d152_batch256_imagenet_20200708-e79cb6a2.pth", + "resnext50_32x4d": "https://download.openmmlab.com/mmclassification/v0/resnext/resnext50_32x4d_b32x8_imagenet_20210429-56066e27.pth", + "resnext101_32x4d": "https://download.openmmlab.com/mmclassification/v0/resnext/resnext101_32x4d_b32x8_imagenet_20210506-e0fa3dd5.pth", + "resnext101_32x8d": "https://download.openmmlab.com/mmclassification/v0/resnext/resnext101_32x8d_b32x8_imagenet_20210506-23a247d5.pth", + "resnext152_32x4d": "https://download.openmmlab.com/mmclassification/v0/resnext/resnext152_32x4d_b32x8_imagenet_20210524-927787be.pth", + "se-resnet50": "https://download.openmmlab.com/mmclassification/v0/se-resnet/se-resnet50_batch256_imagenet_20200804-ae206104.pth", + "se-resnet101": "https://download.openmmlab.com/mmclassification/v0/se-resnet/se-resnet101_batch256_imagenet_20200804-ba5b51d4.pth", + "resnest50": "https://download.openmmlab.com/mmclassification/v0/resnest/resnest50_imagenet_converted-1ebf0afe.pth", + "resnest101": "https://download.openmmlab.com/mmclassification/v0/resnest/resnest101_imagenet_converted-032caa52.pth", + "resnest200": "https://download.openmmlab.com/mmclassification/v0/resnest/resnest200_imagenet_converted-581a60f2.pth", + "resnest269": "https://download.openmmlab.com/mmclassification/v0/resnest/resnest269_imagenet_converted-59930960.pth", + "shufflenet_v1": "https://download.openmmlab.com/mmclassification/v0/shufflenet_v1/shufflenet_v1_batch1024_imagenet_20200804-5d6cec73.pth", + "shufflenet_v2": "https://download.openmmlab.com/mmclassification/v0/shufflenet_v2/shufflenet_v2_batch1024_imagenet_20200812-5bf4721e.pth", + "mobilenet_v2": "https://download.openmmlab.com/mmclassification/v0/mobilenet_v2/mobilenet_v2_batch256_imagenet_20200708-3b2dc3af.pth" +} diff --git a/annotator/mmpkg/mmcv/model_zoo/open_mmlab.json b/annotator/mmpkg/mmcv/model_zoo/open_mmlab.json new file mode 100644 index 0000000000000000000000000000000000000000..8311db4feef92faa0841c697d75efbee8430c3a0 --- /dev/null +++ b/annotator/mmpkg/mmcv/model_zoo/open_mmlab.json @@ -0,0 +1,50 @@ +{ + "vgg16_caffe": "https://download.openmmlab.com/pretrain/third_party/vgg16_caffe-292e1171.pth", + "detectron/resnet50_caffe": "https://download.openmmlab.com/pretrain/third_party/resnet50_caffe-788b5fa3.pth", + "detectron2/resnet50_caffe": "https://download.openmmlab.com/pretrain/third_party/resnet50_msra-5891d200.pth", + "detectron/resnet101_caffe": "https://download.openmmlab.com/pretrain/third_party/resnet101_caffe-3ad79236.pth", + "detectron2/resnet101_caffe": "https://download.openmmlab.com/pretrain/third_party/resnet101_msra-6cc46731.pth", + "detectron2/resnext101_32x8d": "https://download.openmmlab.com/pretrain/third_party/resnext101_32x8d-1516f1aa.pth", + "resnext50_32x4d": "https://download.openmmlab.com/pretrain/third_party/resnext50-32x4d-0ab1a123.pth", + "resnext101_32x4d": "https://download.openmmlab.com/pretrain/third_party/resnext101_32x4d-a5af3160.pth", + "resnext101_64x4d": "https://download.openmmlab.com/pretrain/third_party/resnext101_64x4d-ee2c6f71.pth", + "contrib/resnet50_gn": "https://download.openmmlab.com/pretrain/third_party/resnet50_gn_thangvubk-ad1730dd.pth", + "detectron/resnet50_gn": "https://download.openmmlab.com/pretrain/third_party/resnet50_gn-9186a21c.pth", + "detectron/resnet101_gn": "https://download.openmmlab.com/pretrain/third_party/resnet101_gn-cac0ab98.pth", + "jhu/resnet50_gn_ws": "https://download.openmmlab.com/pretrain/third_party/resnet50_gn_ws-15beedd8.pth", + "jhu/resnet101_gn_ws": "https://download.openmmlab.com/pretrain/third_party/resnet101_gn_ws-3e3c308c.pth", + "jhu/resnext50_32x4d_gn_ws": "https://download.openmmlab.com/pretrain/third_party/resnext50_32x4d_gn_ws-0d87ac85.pth", + "jhu/resnext101_32x4d_gn_ws": "https://download.openmmlab.com/pretrain/third_party/resnext101_32x4d_gn_ws-34ac1a9e.pth", + "jhu/resnext50_32x4d_gn": "https://download.openmmlab.com/pretrain/third_party/resnext50_32x4d_gn-c7e8b754.pth", + "jhu/resnext101_32x4d_gn": "https://download.openmmlab.com/pretrain/third_party/resnext101_32x4d_gn-ac3bb84e.pth", + "msra/hrnetv2_w18_small": "https://download.openmmlab.com/pretrain/third_party/hrnetv2_w18_small-b5a04e21.pth", + "msra/hrnetv2_w18": "https://download.openmmlab.com/pretrain/third_party/hrnetv2_w18-00eb2006.pth", + "msra/hrnetv2_w32": "https://download.openmmlab.com/pretrain/third_party/hrnetv2_w32-dc9eeb4f.pth", + "msra/hrnetv2_w40": "https://download.openmmlab.com/pretrain/third_party/hrnetv2_w40-ed0b031c.pth", + "msra/hrnetv2_w48": "https://download.openmmlab.com/pretrain/third_party/hrnetv2_w48-d2186c55.pth", + "bninception_caffe": "https://download.openmmlab.com/pretrain/third_party/bn_inception_caffe-ed2e8665.pth", + "kin400/i3d_r50_f32s2_k400": "https://download.openmmlab.com/pretrain/third_party/i3d_r50_f32s2_k400-2c57e077.pth", + "kin400/nl3d_r50_f32s2_k400": "https://download.openmmlab.com/pretrain/third_party/nl3d_r50_f32s2_k400-fa7e7caa.pth", + "res2net101_v1d_26w_4s": "https://download.openmmlab.com/pretrain/third_party/res2net101_v1d_26w_4s_mmdetv2-f0a600f9.pth", + "regnetx_400mf": "https://download.openmmlab.com/pretrain/third_party/regnetx_400mf-a5b10d96.pth", + "regnetx_800mf": "https://download.openmmlab.com/pretrain/third_party/regnetx_800mf-1f4be4c7.pth", + "regnetx_1.6gf": "https://download.openmmlab.com/pretrain/third_party/regnetx_1.6gf-5791c176.pth", + "regnetx_3.2gf": "https://download.openmmlab.com/pretrain/third_party/regnetx_3.2gf-c2599b0f.pth", + "regnetx_4.0gf": "https://download.openmmlab.com/pretrain/third_party/regnetx_4.0gf-a88f671e.pth", + "regnetx_6.4gf": "https://download.openmmlab.com/pretrain/third_party/regnetx_6.4gf-006af45d.pth", + "regnetx_8.0gf": "https://download.openmmlab.com/pretrain/third_party/regnetx_8.0gf-3c68abe7.pth", + "regnetx_12gf": "https://download.openmmlab.com/pretrain/third_party/regnetx_12gf-4c2a3350.pth", + "resnet18_v1c": "https://download.openmmlab.com/pretrain/third_party/resnet18_v1c-b5776b93.pth", + "resnet50_v1c": "https://download.openmmlab.com/pretrain/third_party/resnet50_v1c-2cccc1ad.pth", + "resnet101_v1c": "https://download.openmmlab.com/pretrain/third_party/resnet101_v1c-e67eebb6.pth", + "mmedit/vgg16": "https://download.openmmlab.com/mmediting/third_party/vgg_state_dict.pth", + "mmedit/res34_en_nomixup": "https://download.openmmlab.com/mmediting/third_party/model_best_resnet34_En_nomixup.pth", + "mmedit/mobilenet_v2": "https://download.openmmlab.com/mmediting/third_party/mobilenet_v2.pth", + "contrib/mobilenet_v3_large": "https://download.openmmlab.com/pretrain/third_party/mobilenet_v3_large-bc2c3fd3.pth", + "contrib/mobilenet_v3_small": "https://download.openmmlab.com/pretrain/third_party/mobilenet_v3_small-47085aa1.pth", + "resnest50": "https://download.openmmlab.com/pretrain/third_party/resnest50_d2-7497a55b.pth", + "resnest101": "https://download.openmmlab.com/pretrain/third_party/resnest101_d2-f3b931b2.pth", + "resnest200": "https://download.openmmlab.com/pretrain/third_party/resnest200_d2-ca88e41f.pth", + "darknet53": "https://download.openmmlab.com/pretrain/third_party/darknet53-a628ea1b.pth", + "mmdet/mobilenet_v2": "https://download.openmmlab.com/mmdetection/v2.0/third_party/mobilenet_v2_batch256_imagenet-ff34753d.pth" +} diff --git a/annotator/mmpkg/mmcv/ops/__init__.py b/annotator/mmpkg/mmcv/ops/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..999e090a458ee148ceca0649f1e3806a40e909bd --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/__init__.py @@ -0,0 +1,81 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .assign_score_withk import assign_score_withk +from .ball_query import ball_query +from .bbox import bbox_overlaps +from .border_align import BorderAlign, border_align +from .box_iou_rotated import box_iou_rotated +from .carafe import CARAFE, CARAFENaive, CARAFEPack, carafe, carafe_naive +from .cc_attention import CrissCrossAttention +from .contour_expand import contour_expand +from .corner_pool import CornerPool +from .correlation import Correlation +from .deform_conv import DeformConv2d, DeformConv2dPack, deform_conv2d +from .deform_roi_pool import (DeformRoIPool, DeformRoIPoolPack, + ModulatedDeformRoIPoolPack, deform_roi_pool) +from .deprecated_wrappers import Conv2d_deprecated as Conv2d +from .deprecated_wrappers import ConvTranspose2d_deprecated as ConvTranspose2d +from .deprecated_wrappers import Linear_deprecated as Linear +from .deprecated_wrappers import MaxPool2d_deprecated as MaxPool2d +from .focal_loss import (SigmoidFocalLoss, SoftmaxFocalLoss, + sigmoid_focal_loss, softmax_focal_loss) +from .furthest_point_sample import (furthest_point_sample, + furthest_point_sample_with_dist) +from .fused_bias_leakyrelu import FusedBiasLeakyReLU, fused_bias_leakyrelu +from .gather_points import gather_points +from .group_points import GroupAll, QueryAndGroup, grouping_operation +from .info import (get_compiler_version, get_compiling_cuda_version, + get_onnxruntime_op_path) +from .iou3d import boxes_iou_bev, nms_bev, nms_normal_bev +from .knn import knn +from .masked_conv import MaskedConv2d, masked_conv2d +from .modulated_deform_conv import (ModulatedDeformConv2d, + ModulatedDeformConv2dPack, + modulated_deform_conv2d) +from .multi_scale_deform_attn import MultiScaleDeformableAttention +from .nms import batched_nms, nms, nms_match, nms_rotated, soft_nms +from .pixel_group import pixel_group +from .point_sample import (SimpleRoIAlign, point_sample, + rel_roi_point_to_rel_img_point) +from .points_in_boxes import (points_in_boxes_all, points_in_boxes_cpu, + points_in_boxes_part) +from .points_sampler import PointsSampler +from .psa_mask import PSAMask +from .roi_align import RoIAlign, roi_align +from .roi_align_rotated import RoIAlignRotated, roi_align_rotated +from .roi_pool import RoIPool, roi_pool +from .roiaware_pool3d import RoIAwarePool3d +from .roipoint_pool3d import RoIPointPool3d +from .saconv import SAConv2d +from .scatter_points import DynamicScatter, dynamic_scatter +from .sync_bn import SyncBatchNorm +from .three_interpolate import three_interpolate +from .three_nn import three_nn +from .tin_shift import TINShift, tin_shift +from .upfirdn2d import upfirdn2d +from .voxelize import Voxelization, voxelization + +__all__ = [ + 'bbox_overlaps', 'CARAFE', 'CARAFENaive', 'CARAFEPack', 'carafe', + 'carafe_naive', 'CornerPool', 'DeformConv2d', 'DeformConv2dPack', + 'deform_conv2d', 'DeformRoIPool', 'DeformRoIPoolPack', + 'ModulatedDeformRoIPoolPack', 'deform_roi_pool', 'SigmoidFocalLoss', + 'SoftmaxFocalLoss', 'sigmoid_focal_loss', 'softmax_focal_loss', + 'get_compiler_version', 'get_compiling_cuda_version', + 'get_onnxruntime_op_path', 'MaskedConv2d', 'masked_conv2d', + 'ModulatedDeformConv2d', 'ModulatedDeformConv2dPack', + 'modulated_deform_conv2d', 'batched_nms', 'nms', 'soft_nms', 'nms_match', + 'RoIAlign', 'roi_align', 'RoIPool', 'roi_pool', 'SyncBatchNorm', 'Conv2d', + 'ConvTranspose2d', 'Linear', 'MaxPool2d', 'CrissCrossAttention', 'PSAMask', + 'point_sample', 'rel_roi_point_to_rel_img_point', 'SimpleRoIAlign', + 'SAConv2d', 'TINShift', 'tin_shift', 'assign_score_withk', + 'box_iou_rotated', 'RoIPointPool3d', 'nms_rotated', 'knn', 'ball_query', + 'upfirdn2d', 'FusedBiasLeakyReLU', 'fused_bias_leakyrelu', + 'RoIAlignRotated', 'roi_align_rotated', 'pixel_group', 'QueryAndGroup', + 'GroupAll', 'grouping_operation', 'contour_expand', 'three_nn', + 'three_interpolate', 'MultiScaleDeformableAttention', 'BorderAlign', + 'border_align', 'gather_points', 'furthest_point_sample', + 'furthest_point_sample_with_dist', 'PointsSampler', 'Correlation', + 'boxes_iou_bev', 'nms_bev', 'nms_normal_bev', 'Voxelization', + 'voxelization', 'dynamic_scatter', 'DynamicScatter', 'RoIAwarePool3d', + 'points_in_boxes_part', 'points_in_boxes_cpu', 'points_in_boxes_all' +] diff --git a/annotator/mmpkg/mmcv/ops/assign_score_withk.py b/annotator/mmpkg/mmcv/ops/assign_score_withk.py new file mode 100644 index 0000000000000000000000000000000000000000..4906adaa2cffd1b46912fbe7d4f87ef2f9fa0012 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/assign_score_withk.py @@ -0,0 +1,123 @@ +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['assign_score_withk_forward', 'assign_score_withk_backward']) + + +class AssignScoreWithK(Function): + r"""Perform weighted sum to generate output features according to scores. + Modified from `PAConv `_. + + This is a memory-efficient CUDA implementation of assign_scores operation, + which first transform all point features with weight bank, then assemble + neighbor features with ``knn_idx`` and perform weighted sum of ``scores``. + + See the `paper `_ appendix Sec. D for + more detailed descriptions. + + Note: + This implementation assumes using ``neighbor`` kernel input, which is + (point_features - center_features, point_features). + See https://github.com/CVMI-Lab/PAConv/blob/main/scene_seg/model/ + pointnet2/paconv.py#L128 for more details. + """ + + @staticmethod + def forward(ctx, + scores, + point_features, + center_features, + knn_idx, + aggregate='sum'): + """ + Args: + scores (torch.Tensor): (B, npoint, K, M), predicted scores to + aggregate weight matrices in the weight bank. + ``npoint`` is the number of sampled centers. + ``K`` is the number of queried neighbors. + ``M`` is the number of weight matrices in the weight bank. + point_features (torch.Tensor): (B, N, M, out_dim) + Pre-computed point features to be aggregated. + center_features (torch.Tensor): (B, N, M, out_dim) + Pre-computed center features to be aggregated. + knn_idx (torch.Tensor): (B, npoint, K), index of sampled kNN. + We assume the first idx in each row is the idx of the center. + aggregate (str, optional): Aggregation method. + Can be 'sum', 'avg' or 'max'. Defaults: 'sum'. + + Returns: + torch.Tensor: (B, out_dim, npoint, K), the aggregated features. + """ + agg = {'sum': 0, 'avg': 1, 'max': 2} + + B, N, M, out_dim = point_features.size() + _, npoint, K, _ = scores.size() + + output = point_features.new_zeros((B, out_dim, npoint, K)) + ext_module.assign_score_withk_forward( + point_features.contiguous(), + center_features.contiguous(), + scores.contiguous(), + knn_idx.contiguous(), + output, + B=B, + N0=N, + N1=npoint, + M=M, + K=K, + O=out_dim, + aggregate=agg[aggregate]) + + ctx.save_for_backward(output, point_features, center_features, scores, + knn_idx) + ctx.agg = agg[aggregate] + + return output + + @staticmethod + def backward(ctx, grad_out): + """ + Args: + grad_out (torch.Tensor): (B, out_dim, npoint, K) + + Returns: + grad_scores (torch.Tensor): (B, npoint, K, M) + grad_point_features (torch.Tensor): (B, N, M, out_dim) + grad_center_features (torch.Tensor): (B, N, M, out_dim) + """ + _, point_features, center_features, scores, knn_idx = ctx.saved_tensors + + agg = ctx.agg + + B, N, M, out_dim = point_features.size() + _, npoint, K, _ = scores.size() + + grad_point_features = point_features.new_zeros(point_features.shape) + grad_center_features = center_features.new_zeros(center_features.shape) + grad_scores = scores.new_zeros(scores.shape) + + ext_module.assign_score_withk_backward( + grad_out.contiguous(), + point_features.contiguous(), + center_features.contiguous(), + scores.contiguous(), + knn_idx.contiguous(), + grad_point_features, + grad_center_features, + grad_scores, + B=B, + N0=N, + N1=npoint, + M=M, + K=K, + O=out_dim, + aggregate=agg) + + return grad_scores, grad_point_features, \ + grad_center_features, None, None + + +assign_score_withk = AssignScoreWithK.apply diff --git a/annotator/mmpkg/mmcv/ops/ball_query.py b/annotator/mmpkg/mmcv/ops/ball_query.py new file mode 100644 index 0000000000000000000000000000000000000000..d0466847c6e5c1239e359a0397568413ebc1504a --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/ball_query.py @@ -0,0 +1,55 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', ['ball_query_forward']) + + +class BallQuery(Function): + """Find nearby points in spherical space.""" + + @staticmethod + def forward(ctx, min_radius: float, max_radius: float, sample_num: int, + xyz: torch.Tensor, center_xyz: torch.Tensor) -> torch.Tensor: + """ + Args: + min_radius (float): minimum radius of the balls. + max_radius (float): maximum radius of the balls. + sample_num (int): maximum number of features in the balls. + xyz (Tensor): (B, N, 3) xyz coordinates of the features. + center_xyz (Tensor): (B, npoint, 3) centers of the ball query. + + Returns: + Tensor: (B, npoint, nsample) tensor with the indices of + the features that form the query balls. + """ + assert center_xyz.is_contiguous() + assert xyz.is_contiguous() + assert min_radius < max_radius + + B, N, _ = xyz.size() + npoint = center_xyz.size(1) + idx = xyz.new_zeros(B, npoint, sample_num, dtype=torch.int) + + ext_module.ball_query_forward( + center_xyz, + xyz, + idx, + b=B, + n=N, + m=npoint, + min_radius=min_radius, + max_radius=max_radius, + nsample=sample_num) + if torch.__version__ != 'parrots': + ctx.mark_non_differentiable(idx) + return idx + + @staticmethod + def backward(ctx, a=None): + return None, None, None, None + + +ball_query = BallQuery.apply diff --git a/annotator/mmpkg/mmcv/ops/bbox.py b/annotator/mmpkg/mmcv/ops/bbox.py new file mode 100644 index 0000000000000000000000000000000000000000..0c4d58b6c91f652933974f519acd3403a833e906 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/bbox.py @@ -0,0 +1,72 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', ['bbox_overlaps']) + + +def bbox_overlaps(bboxes1, bboxes2, mode='iou', aligned=False, offset=0): + """Calculate overlap between two set of bboxes. + + If ``aligned`` is ``False``, then calculate the ious between each bbox + of bboxes1 and bboxes2, otherwise the ious between each aligned pair of + bboxes1 and bboxes2. + + Args: + bboxes1 (Tensor): shape (m, 4) in format or empty. + bboxes2 (Tensor): shape (n, 4) in format or empty. + If aligned is ``True``, then m and n must be equal. + mode (str): "iou" (intersection over union) or iof (intersection over + foreground). + + Returns: + ious(Tensor): shape (m, n) if aligned == False else shape (m, 1) + + Example: + >>> bboxes1 = torch.FloatTensor([ + >>> [0, 0, 10, 10], + >>> [10, 10, 20, 20], + >>> [32, 32, 38, 42], + >>> ]) + >>> bboxes2 = torch.FloatTensor([ + >>> [0, 0, 10, 20], + >>> [0, 10, 10, 19], + >>> [10, 10, 20, 20], + >>> ]) + >>> bbox_overlaps(bboxes1, bboxes2) + tensor([[0.5000, 0.0000, 0.0000], + [0.0000, 0.0000, 1.0000], + [0.0000, 0.0000, 0.0000]]) + + Example: + >>> empty = torch.FloatTensor([]) + >>> nonempty = torch.FloatTensor([ + >>> [0, 0, 10, 9], + >>> ]) + >>> assert tuple(bbox_overlaps(empty, nonempty).shape) == (0, 1) + >>> assert tuple(bbox_overlaps(nonempty, empty).shape) == (1, 0) + >>> assert tuple(bbox_overlaps(empty, empty).shape) == (0, 0) + """ + + mode_dict = {'iou': 0, 'iof': 1} + assert mode in mode_dict.keys() + mode_flag = mode_dict[mode] + # Either the boxes are empty or the length of boxes' last dimension is 4 + assert (bboxes1.size(-1) == 4 or bboxes1.size(0) == 0) + assert (bboxes2.size(-1) == 4 or bboxes2.size(0) == 0) + assert offset == 1 or offset == 0 + + rows = bboxes1.size(0) + cols = bboxes2.size(0) + if aligned: + assert rows == cols + + if rows * cols == 0: + return bboxes1.new(rows, 1) if aligned else bboxes1.new(rows, cols) + + if aligned: + ious = bboxes1.new_zeros(rows) + else: + ious = bboxes1.new_zeros((rows, cols)) + ext_module.bbox_overlaps( + bboxes1, bboxes2, ious, mode=mode_flag, aligned=aligned, offset=offset) + return ious diff --git a/annotator/mmpkg/mmcv/ops/border_align.py b/annotator/mmpkg/mmcv/ops/border_align.py new file mode 100644 index 0000000000000000000000000000000000000000..ff305be328e9b0a15e1bbb5e6b41beb940f55c81 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/border_align.py @@ -0,0 +1,109 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# modified from +# https://github.com/Megvii-BaseDetection/cvpods/blob/master/cvpods/layers/border_align.py + +import torch +import torch.nn as nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['border_align_forward', 'border_align_backward']) + + +class BorderAlignFunction(Function): + + @staticmethod + def symbolic(g, input, boxes, pool_size): + return g.op( + 'mmcv::MMCVBorderAlign', input, boxes, pool_size_i=pool_size) + + @staticmethod + def forward(ctx, input, boxes, pool_size): + ctx.pool_size = pool_size + ctx.input_shape = input.size() + + assert boxes.ndim == 3, 'boxes must be with shape [B, H*W, 4]' + assert boxes.size(2) == 4, \ + 'the last dimension of boxes must be (x1, y1, x2, y2)' + assert input.size(1) % 4 == 0, \ + 'the channel for input feature must be divisible by factor 4' + + # [B, C//4, H*W, 4] + output_shape = (input.size(0), input.size(1) // 4, boxes.size(1), 4) + output = input.new_zeros(output_shape) + # `argmax_idx` only used for backward + argmax_idx = input.new_zeros(output_shape).to(torch.int) + + ext_module.border_align_forward( + input, boxes, output, argmax_idx, pool_size=ctx.pool_size) + + ctx.save_for_backward(boxes, argmax_idx) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + boxes, argmax_idx = ctx.saved_tensors + grad_input = grad_output.new_zeros(ctx.input_shape) + # complex head architecture may cause grad_output uncontiguous + grad_output = grad_output.contiguous() + ext_module.border_align_backward( + grad_output, + boxes, + argmax_idx, + grad_input, + pool_size=ctx.pool_size) + return grad_input, None, None + + +border_align = BorderAlignFunction.apply + + +class BorderAlign(nn.Module): + r"""Border align pooling layer. + + Applies border_align over the input feature based on predicted bboxes. + The details were described in the paper + `BorderDet: Border Feature for Dense Object Detection + `_. + + For each border line (e.g. top, left, bottom or right) of each box, + border_align does the following: + 1. uniformly samples `pool_size`+1 positions on this line, involving \ + the start and end points. + 2. the corresponding features on these points are computed by \ + bilinear interpolation. + 3. max pooling over all the `pool_size`+1 positions are used for \ + computing pooled feature. + + Args: + pool_size (int): number of positions sampled over the boxes' borders + (e.g. top, bottom, left, right). + + """ + + def __init__(self, pool_size): + super(BorderAlign, self).__init__() + self.pool_size = pool_size + + def forward(self, input, boxes): + """ + Args: + input: Features with shape [N,4C,H,W]. Channels ranged in [0,C), + [C,2C), [2C,3C), [3C,4C) represent the top, left, bottom, + right features respectively. + boxes: Boxes with shape [N,H*W,4]. Coordinate format (x1,y1,x2,y2). + + Returns: + Tensor: Pooled features with shape [N,C,H*W,4]. The order is + (top,left,bottom,right) for the last dimension. + """ + return border_align(input, boxes, self.pool_size) + + def __repr__(self): + s = self.__class__.__name__ + s += f'(pool_size={self.pool_size})' + return s diff --git a/annotator/mmpkg/mmcv/ops/box_iou_rotated.py b/annotator/mmpkg/mmcv/ops/box_iou_rotated.py new file mode 100644 index 0000000000000000000000000000000000000000..2d78015e9c2a9e7a52859b4e18f84a9aa63481a0 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/box_iou_rotated.py @@ -0,0 +1,45 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', ['box_iou_rotated']) + + +def box_iou_rotated(bboxes1, bboxes2, mode='iou', aligned=False): + """Return intersection-over-union (Jaccard index) of boxes. + + Both sets of boxes are expected to be in + (x_center, y_center, width, height, angle) format. + + If ``aligned`` is ``False``, then calculate the ious between each bbox + of bboxes1 and bboxes2, otherwise the ious between each aligned pair of + bboxes1 and bboxes2. + + Arguments: + boxes1 (Tensor): rotated bboxes 1. \ + It has shape (N, 5), indicating (x, y, w, h, theta) for each row. + Note that theta is in radian. + boxes2 (Tensor): rotated bboxes 2. \ + It has shape (M, 5), indicating (x, y, w, h, theta) for each row. + Note that theta is in radian. + mode (str): "iou" (intersection over union) or iof (intersection over + foreground). + + Returns: + ious(Tensor): shape (N, M) if aligned == False else shape (N,) + """ + assert mode in ['iou', 'iof'] + mode_dict = {'iou': 0, 'iof': 1} + mode_flag = mode_dict[mode] + rows = bboxes1.size(0) + cols = bboxes2.size(0) + if aligned: + ious = bboxes1.new_zeros(rows) + else: + ious = bboxes1.new_zeros((rows * cols)) + bboxes1 = bboxes1.contiguous() + bboxes2 = bboxes2.contiguous() + ext_module.box_iou_rotated( + bboxes1, bboxes2, ious, mode_flag=mode_flag, aligned=aligned) + if not aligned: + ious = ious.view(rows, cols) + return ious diff --git a/annotator/mmpkg/mmcv/ops/carafe.py b/annotator/mmpkg/mmcv/ops/carafe.py new file mode 100644 index 0000000000000000000000000000000000000000..5154cb3abfccfbbe0a1b2daa67018dbf80aaf6d2 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/carafe.py @@ -0,0 +1,287 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Function +from torch.nn.modules.module import Module + +from ..cnn import UPSAMPLE_LAYERS, normal_init, xavier_init +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', [ + 'carafe_naive_forward', 'carafe_naive_backward', 'carafe_forward', + 'carafe_backward' +]) + + +class CARAFENaiveFunction(Function): + + @staticmethod + def symbolic(g, features, masks, kernel_size, group_size, scale_factor): + return g.op( + 'mmcv::MMCVCARAFENaive', + features, + masks, + kernel_size_i=kernel_size, + group_size_i=group_size, + scale_factor_f=scale_factor) + + @staticmethod + def forward(ctx, features, masks, kernel_size, group_size, scale_factor): + assert scale_factor >= 1 + assert masks.size(1) == kernel_size * kernel_size * group_size + assert masks.size(-1) == features.size(-1) * scale_factor + assert masks.size(-2) == features.size(-2) * scale_factor + assert features.size(1) % group_size == 0 + assert (kernel_size - 1) % 2 == 0 and kernel_size >= 1 + ctx.kernel_size = kernel_size + ctx.group_size = group_size + ctx.scale_factor = scale_factor + ctx.feature_size = features.size() + ctx.mask_size = masks.size() + + n, c, h, w = features.size() + output = features.new_zeros((n, c, h * scale_factor, w * scale_factor)) + ext_module.carafe_naive_forward( + features, + masks, + output, + kernel_size=kernel_size, + group_size=group_size, + scale_factor=scale_factor) + + if features.requires_grad or masks.requires_grad: + ctx.save_for_backward(features, masks) + return output + + @staticmethod + def backward(ctx, grad_output): + assert grad_output.is_cuda + + features, masks = ctx.saved_tensors + kernel_size = ctx.kernel_size + group_size = ctx.group_size + scale_factor = ctx.scale_factor + + grad_input = torch.zeros_like(features) + grad_masks = torch.zeros_like(masks) + ext_module.carafe_naive_backward( + grad_output.contiguous(), + features, + masks, + grad_input, + grad_masks, + kernel_size=kernel_size, + group_size=group_size, + scale_factor=scale_factor) + + return grad_input, grad_masks, None, None, None + + +carafe_naive = CARAFENaiveFunction.apply + + +class CARAFENaive(Module): + + def __init__(self, kernel_size, group_size, scale_factor): + super(CARAFENaive, self).__init__() + + assert isinstance(kernel_size, int) and isinstance( + group_size, int) and isinstance(scale_factor, int) + self.kernel_size = kernel_size + self.group_size = group_size + self.scale_factor = scale_factor + + def forward(self, features, masks): + return carafe_naive(features, masks, self.kernel_size, self.group_size, + self.scale_factor) + + +class CARAFEFunction(Function): + + @staticmethod + def symbolic(g, features, masks, kernel_size, group_size, scale_factor): + return g.op( + 'mmcv::MMCVCARAFE', + features, + masks, + kernel_size_i=kernel_size, + group_size_i=group_size, + scale_factor_f=scale_factor) + + @staticmethod + def forward(ctx, features, masks, kernel_size, group_size, scale_factor): + assert scale_factor >= 1 + assert masks.size(1) == kernel_size * kernel_size * group_size + assert masks.size(-1) == features.size(-1) * scale_factor + assert masks.size(-2) == features.size(-2) * scale_factor + assert features.size(1) % group_size == 0 + assert (kernel_size - 1) % 2 == 0 and kernel_size >= 1 + ctx.kernel_size = kernel_size + ctx.group_size = group_size + ctx.scale_factor = scale_factor + ctx.feature_size = features.size() + ctx.mask_size = masks.size() + + n, c, h, w = features.size() + output = features.new_zeros((n, c, h * scale_factor, w * scale_factor)) + routput = features.new_zeros(output.size(), requires_grad=False) + rfeatures = features.new_zeros(features.size(), requires_grad=False) + rmasks = masks.new_zeros(masks.size(), requires_grad=False) + ext_module.carafe_forward( + features, + masks, + rfeatures, + routput, + rmasks, + output, + kernel_size=kernel_size, + group_size=group_size, + scale_factor=scale_factor) + + if features.requires_grad or masks.requires_grad: + ctx.save_for_backward(features, masks, rfeatures) + return output + + @staticmethod + def backward(ctx, grad_output): + assert grad_output.is_cuda + + features, masks, rfeatures = ctx.saved_tensors + kernel_size = ctx.kernel_size + group_size = ctx.group_size + scale_factor = ctx.scale_factor + + rgrad_output = torch.zeros_like(grad_output, requires_grad=False) + rgrad_input_hs = torch.zeros_like(grad_output, requires_grad=False) + rgrad_input = torch.zeros_like(features, requires_grad=False) + rgrad_masks = torch.zeros_like(masks, requires_grad=False) + grad_input = torch.zeros_like(features, requires_grad=False) + grad_masks = torch.zeros_like(masks, requires_grad=False) + ext_module.carafe_backward( + grad_output.contiguous(), + rfeatures, + masks, + rgrad_output, + rgrad_input_hs, + rgrad_input, + rgrad_masks, + grad_input, + grad_masks, + kernel_size=kernel_size, + group_size=group_size, + scale_factor=scale_factor) + return grad_input, grad_masks, None, None, None + + +carafe = CARAFEFunction.apply + + +class CARAFE(Module): + """ CARAFE: Content-Aware ReAssembly of FEatures + + Please refer to https://arxiv.org/abs/1905.02188 for more details. + + Args: + kernel_size (int): reassemble kernel size + group_size (int): reassemble group size + scale_factor (int): upsample ratio + + Returns: + upsampled feature map + """ + + def __init__(self, kernel_size, group_size, scale_factor): + super(CARAFE, self).__init__() + + assert isinstance(kernel_size, int) and isinstance( + group_size, int) and isinstance(scale_factor, int) + self.kernel_size = kernel_size + self.group_size = group_size + self.scale_factor = scale_factor + + def forward(self, features, masks): + return carafe(features, masks, self.kernel_size, self.group_size, + self.scale_factor) + + +@UPSAMPLE_LAYERS.register_module(name='carafe') +class CARAFEPack(nn.Module): + """A unified package of CARAFE upsampler that contains: 1) channel + compressor 2) content encoder 3) CARAFE op. + + Official implementation of ICCV 2019 paper + CARAFE: Content-Aware ReAssembly of FEatures + Please refer to https://arxiv.org/abs/1905.02188 for more details. + + Args: + channels (int): input feature channels + scale_factor (int): upsample ratio + up_kernel (int): kernel size of CARAFE op + up_group (int): group size of CARAFE op + encoder_kernel (int): kernel size of content encoder + encoder_dilation (int): dilation of content encoder + compressed_channels (int): output channels of channels compressor + + Returns: + upsampled feature map + """ + + def __init__(self, + channels, + scale_factor, + up_kernel=5, + up_group=1, + encoder_kernel=3, + encoder_dilation=1, + compressed_channels=64): + super(CARAFEPack, self).__init__() + self.channels = channels + self.scale_factor = scale_factor + self.up_kernel = up_kernel + self.up_group = up_group + self.encoder_kernel = encoder_kernel + self.encoder_dilation = encoder_dilation + self.compressed_channels = compressed_channels + self.channel_compressor = nn.Conv2d(channels, self.compressed_channels, + 1) + self.content_encoder = nn.Conv2d( + self.compressed_channels, + self.up_kernel * self.up_kernel * self.up_group * + self.scale_factor * self.scale_factor, + self.encoder_kernel, + padding=int((self.encoder_kernel - 1) * self.encoder_dilation / 2), + dilation=self.encoder_dilation, + groups=1) + self.init_weights() + + def init_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + xavier_init(m, distribution='uniform') + normal_init(self.content_encoder, std=0.001) + + def kernel_normalizer(self, mask): + mask = F.pixel_shuffle(mask, self.scale_factor) + n, mask_c, h, w = mask.size() + # use float division explicitly, + # to void inconsistency while exporting to onnx + mask_channel = int(mask_c / float(self.up_kernel**2)) + mask = mask.view(n, mask_channel, -1, h, w) + + mask = F.softmax(mask, dim=2, dtype=mask.dtype) + mask = mask.view(n, mask_c, h, w).contiguous() + + return mask + + def feature_reassemble(self, x, mask): + x = carafe(x, mask, self.up_kernel, self.up_group, self.scale_factor) + return x + + def forward(self, x): + compressed_x = self.channel_compressor(x) + mask = self.content_encoder(compressed_x) + mask = self.kernel_normalizer(mask) + + x = self.feature_reassemble(x, mask) + return x diff --git a/annotator/mmpkg/mmcv/ops/cc_attention.py b/annotator/mmpkg/mmcv/ops/cc_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..8982f467185b5d839832baa2e51722613a8b87a2 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/cc_attention.py @@ -0,0 +1,83 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from annotator.mmpkg.mmcv.cnn import PLUGIN_LAYERS, Scale + + +def NEG_INF_DIAG(n, device): + """Returns a diagonal matrix of size [n, n]. + + The diagonal are all "-inf". This is for avoiding calculating the + overlapped element in the Criss-Cross twice. + """ + return torch.diag(torch.tensor(float('-inf')).to(device).repeat(n), 0) + + +@PLUGIN_LAYERS.register_module() +class CrissCrossAttention(nn.Module): + """Criss-Cross Attention Module. + + .. note:: + Before v1.3.13, we use a CUDA op. Since v1.3.13, we switch + to a pure PyTorch and equivalent implementation. For more + details, please refer to https://github.com/open-mmlab/mmcv/pull/1201. + + Speed comparison for one forward pass + + - Input size: [2,512,97,97] + - Device: 1 NVIDIA GeForce RTX 2080 Ti + + +-----------------------+---------------+------------+---------------+ + | |PyTorch version|CUDA version|Relative speed | + +=======================+===============+============+===============+ + |with torch.no_grad() |0.00554402 s |0.0299619 s |5.4x | + +-----------------------+---------------+------------+---------------+ + |no with torch.no_grad()|0.00562803 s |0.0301349 s |5.4x | + +-----------------------+---------------+------------+---------------+ + + Args: + in_channels (int): Channels of the input feature map. + """ + + def __init__(self, in_channels): + super().__init__() + self.query_conv = nn.Conv2d(in_channels, in_channels // 8, 1) + self.key_conv = nn.Conv2d(in_channels, in_channels // 8, 1) + self.value_conv = nn.Conv2d(in_channels, in_channels, 1) + self.gamma = Scale(0.) + self.in_channels = in_channels + + def forward(self, x): + """forward function of Criss-Cross Attention. + + Args: + x (Tensor): Input feature. \ + shape (batch_size, in_channels, height, width) + Returns: + Tensor: Output of the layer, with shape of \ + (batch_size, in_channels, height, width) + """ + B, C, H, W = x.size() + query = self.query_conv(x) + key = self.key_conv(x) + value = self.value_conv(x) + energy_H = torch.einsum('bchw,bciw->bwhi', query, key) + NEG_INF_DIAG( + H, query.device) + energy_H = energy_H.transpose(1, 2) + energy_W = torch.einsum('bchw,bchj->bhwj', query, key) + attn = F.softmax( + torch.cat([energy_H, energy_W], dim=-1), dim=-1) # [B,H,W,(H+W)] + out = torch.einsum('bciw,bhwi->bchw', value, attn[..., :H]) + out += torch.einsum('bchj,bhwj->bchw', value, attn[..., H:]) + + out = self.gamma(out) + x + out = out.contiguous() + + return out + + def __repr__(self): + s = self.__class__.__name__ + s += f'(in_channels={self.in_channels})' + return s diff --git a/annotator/mmpkg/mmcv/ops/contour_expand.py b/annotator/mmpkg/mmcv/ops/contour_expand.py new file mode 100644 index 0000000000000000000000000000000000000000..ea1111e1768b5f27e118bf7dbc0d9c70a7afd6d7 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/contour_expand.py @@ -0,0 +1,49 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numpy as np +import torch + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', ['contour_expand']) + + +def contour_expand(kernel_mask, internal_kernel_label, min_kernel_area, + kernel_num): + """Expand kernel contours so that foreground pixels are assigned into + instances. + + Arguments: + kernel_mask (np.array or Tensor): The instance kernel mask with + size hxw. + internal_kernel_label (np.array or Tensor): The instance internal + kernel label with size hxw. + min_kernel_area (int): The minimum kernel area. + kernel_num (int): The instance kernel number. + + Returns: + label (list): The instance index map with size hxw. + """ + assert isinstance(kernel_mask, (torch.Tensor, np.ndarray)) + assert isinstance(internal_kernel_label, (torch.Tensor, np.ndarray)) + assert isinstance(min_kernel_area, int) + assert isinstance(kernel_num, int) + + if isinstance(kernel_mask, np.ndarray): + kernel_mask = torch.from_numpy(kernel_mask) + if isinstance(internal_kernel_label, np.ndarray): + internal_kernel_label = torch.from_numpy(internal_kernel_label) + + if torch.__version__ == 'parrots': + if kernel_mask.shape[0] == 0 or internal_kernel_label.shape[0] == 0: + label = [] + else: + label = ext_module.contour_expand( + kernel_mask, + internal_kernel_label, + min_kernel_area=min_kernel_area, + kernel_num=kernel_num) + label = label.tolist() + else: + label = ext_module.contour_expand(kernel_mask, internal_kernel_label, + min_kernel_area, kernel_num) + return label diff --git a/annotator/mmpkg/mmcv/ops/corner_pool.py b/annotator/mmpkg/mmcv/ops/corner_pool.py new file mode 100644 index 0000000000000000000000000000000000000000..a33d798b43d405e4c86bee4cd6389be21ca9c637 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/corner_pool.py @@ -0,0 +1,161 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch import nn +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', [ + 'top_pool_forward', 'top_pool_backward', 'bottom_pool_forward', + 'bottom_pool_backward', 'left_pool_forward', 'left_pool_backward', + 'right_pool_forward', 'right_pool_backward' +]) + +_mode_dict = {'top': 0, 'bottom': 1, 'left': 2, 'right': 3} + + +class TopPoolFunction(Function): + + @staticmethod + def symbolic(g, input): + output = g.op( + 'mmcv::MMCVCornerPool', input, mode_i=int(_mode_dict['top'])) + return output + + @staticmethod + def forward(ctx, input): + output = ext_module.top_pool_forward(input) + ctx.save_for_backward(input) + return output + + @staticmethod + def backward(ctx, grad_output): + input, = ctx.saved_tensors + output = ext_module.top_pool_backward(input, grad_output) + return output + + +class BottomPoolFunction(Function): + + @staticmethod + def symbolic(g, input): + output = g.op( + 'mmcv::MMCVCornerPool', input, mode_i=int(_mode_dict['bottom'])) + return output + + @staticmethod + def forward(ctx, input): + output = ext_module.bottom_pool_forward(input) + ctx.save_for_backward(input) + return output + + @staticmethod + def backward(ctx, grad_output): + input, = ctx.saved_tensors + output = ext_module.bottom_pool_backward(input, grad_output) + return output + + +class LeftPoolFunction(Function): + + @staticmethod + def symbolic(g, input): + output = g.op( + 'mmcv::MMCVCornerPool', input, mode_i=int(_mode_dict['left'])) + return output + + @staticmethod + def forward(ctx, input): + output = ext_module.left_pool_forward(input) + ctx.save_for_backward(input) + return output + + @staticmethod + def backward(ctx, grad_output): + input, = ctx.saved_tensors + output = ext_module.left_pool_backward(input, grad_output) + return output + + +class RightPoolFunction(Function): + + @staticmethod + def symbolic(g, input): + output = g.op( + 'mmcv::MMCVCornerPool', input, mode_i=int(_mode_dict['right'])) + return output + + @staticmethod + def forward(ctx, input): + output = ext_module.right_pool_forward(input) + ctx.save_for_backward(input) + return output + + @staticmethod + def backward(ctx, grad_output): + input, = ctx.saved_tensors + output = ext_module.right_pool_backward(input, grad_output) + return output + + +class CornerPool(nn.Module): + """Corner Pooling. + + Corner Pooling is a new type of pooling layer that helps a + convolutional network better localize corners of bounding boxes. + + Please refer to https://arxiv.org/abs/1808.01244 for more details. + Code is modified from https://github.com/princeton-vl/CornerNet-Lite. + + Args: + mode(str): Pooling orientation for the pooling layer + + - 'bottom': Bottom Pooling + - 'left': Left Pooling + - 'right': Right Pooling + - 'top': Top Pooling + + Returns: + Feature map after pooling. + """ + + pool_functions = { + 'bottom': BottomPoolFunction, + 'left': LeftPoolFunction, + 'right': RightPoolFunction, + 'top': TopPoolFunction, + } + + cummax_dim_flip = { + 'bottom': (2, False), + 'left': (3, True), + 'right': (3, False), + 'top': (2, True), + } + + def __init__(self, mode): + super(CornerPool, self).__init__() + assert mode in self.pool_functions + self.mode = mode + self.corner_pool = self.pool_functions[mode] + + def forward(self, x): + if torch.__version__ != 'parrots' and torch.__version__ >= '1.5.0': + if torch.onnx.is_in_onnx_export(): + assert torch.__version__ >= '1.7.0', \ + 'When `cummax` serves as an intermediate component whose '\ + 'outputs is used as inputs for another modules, it\'s '\ + 'expected that pytorch version must be >= 1.7.0, '\ + 'otherwise Error appears like: `RuntimeError: tuple '\ + 'appears in op that does not forward tuples, unsupported '\ + 'kind: prim::PythonOp`.' + + dim, flip = self.cummax_dim_flip[self.mode] + if flip: + x = x.flip(dim) + pool_tensor, _ = torch.cummax(x, dim=dim) + if flip: + pool_tensor = pool_tensor.flip(dim) + return pool_tensor + else: + return self.corner_pool.apply(x) diff --git a/annotator/mmpkg/mmcv/ops/correlation.py b/annotator/mmpkg/mmcv/ops/correlation.py new file mode 100644 index 0000000000000000000000000000000000000000..3d0b79c301b29915dfaf4d2b1846c59be73127d3 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/correlation.py @@ -0,0 +1,196 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch import Tensor, nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.utils import _pair + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['correlation_forward', 'correlation_backward']) + + +class CorrelationFunction(Function): + + @staticmethod + def forward(ctx, + input1, + input2, + kernel_size=1, + max_displacement=1, + stride=1, + padding=1, + dilation=1, + dilation_patch=1): + + ctx.save_for_backward(input1, input2) + + kH, kW = ctx.kernel_size = _pair(kernel_size) + patch_size = max_displacement * 2 + 1 + ctx.patch_size = patch_size + dH, dW = ctx.stride = _pair(stride) + padH, padW = ctx.padding = _pair(padding) + dilationH, dilationW = ctx.dilation = _pair(dilation) + dilation_patchH, dilation_patchW = ctx.dilation_patch = _pair( + dilation_patch) + + output_size = CorrelationFunction._output_size(ctx, input1) + + output = input1.new_zeros(output_size) + + ext_module.correlation_forward( + input1, + input2, + output, + kH=kH, + kW=kW, + patchH=patch_size, + patchW=patch_size, + padH=padH, + padW=padW, + dilationH=dilationH, + dilationW=dilationW, + dilation_patchH=dilation_patchH, + dilation_patchW=dilation_patchW, + dH=dH, + dW=dW) + + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + input1, input2 = ctx.saved_tensors + + kH, kW = ctx.kernel_size + patch_size = ctx.patch_size + padH, padW = ctx.padding + dilationH, dilationW = ctx.dilation + dilation_patchH, dilation_patchW = ctx.dilation_patch + dH, dW = ctx.stride + grad_input1 = torch.zeros_like(input1) + grad_input2 = torch.zeros_like(input2) + + ext_module.correlation_backward( + grad_output, + input1, + input2, + grad_input1, + grad_input2, + kH=kH, + kW=kW, + patchH=patch_size, + patchW=patch_size, + padH=padH, + padW=padW, + dilationH=dilationH, + dilationW=dilationW, + dilation_patchH=dilation_patchH, + dilation_patchW=dilation_patchW, + dH=dH, + dW=dW) + return grad_input1, grad_input2, None, None, None, None, None, None + + @staticmethod + def _output_size(ctx, input1): + iH, iW = input1.size(2), input1.size(3) + batch_size = input1.size(0) + kH, kW = ctx.kernel_size + patch_size = ctx.patch_size + dH, dW = ctx.stride + padH, padW = ctx.padding + dilationH, dilationW = ctx.dilation + dilatedKH = (kH - 1) * dilationH + 1 + dilatedKW = (kW - 1) * dilationW + 1 + + oH = int((iH + 2 * padH - dilatedKH) / dH + 1) + oW = int((iW + 2 * padW - dilatedKW) / dW + 1) + + output_size = (batch_size, patch_size, patch_size, oH, oW) + return output_size + + +class Correlation(nn.Module): + r"""Correlation operator + + This correlation operator works for optical flow correlation computation. + + There are two batched tensors with shape :math:`(N, C, H, W)`, + and the correlation output's shape is :math:`(N, max\_displacement \times + 2 + 1, max\_displacement * 2 + 1, H_{out}, W_{out})` + + where + + .. math:: + H_{out} = \left\lfloor\frac{H_{in} + 2 \times padding - + dilation \times (kernel\_size - 1) - 1} + {stride} + 1\right\rfloor + + .. math:: + W_{out} = \left\lfloor\frac{W_{in} + 2 \times padding - dilation + \times (kernel\_size - 1) - 1} + {stride} + 1\right\rfloor + + the correlation item :math:`(N_i, dy, dx)` is formed by taking the sliding + window convolution between input1 and shifted input2, + + .. math:: + Corr(N_i, dx, dy) = + \sum_{c=0}^{C-1} + input1(N_i, c) \star + \mathcal{S}(input2(N_i, c), dy, dx) + + where :math:`\star` is the valid 2d sliding window convolution operator, + and :math:`\mathcal{S}` means shifting the input features (auto-complete + zero marginal), and :math:`dx, dy` are shifting distance, :math:`dx, dy \in + [-max\_displacement \times dilation\_patch, max\_displacement \times + dilation\_patch]`. + + Args: + kernel_size (int): The size of sliding window i.e. local neighborhood + representing the center points and involved in correlation + computation. Defaults to 1. + max_displacement (int): The radius for computing correlation volume, + but the actual working space can be dilated by dilation_patch. + Defaults to 1. + stride (int): The stride of the sliding blocks in the input spatial + dimensions. Defaults to 1. + padding (int): Zero padding added to all four sides of the input1. + Defaults to 0. + dilation (int): The spacing of local neighborhood that will involved + in correlation. Defaults to 1. + dilation_patch (int): The spacing between position need to compute + correlation. Defaults to 1. + """ + + def __init__(self, + kernel_size: int = 1, + max_displacement: int = 1, + stride: int = 1, + padding: int = 0, + dilation: int = 1, + dilation_patch: int = 1) -> None: + super().__init__() + self.kernel_size = kernel_size + self.max_displacement = max_displacement + self.stride = stride + self.padding = padding + self.dilation = dilation + self.dilation_patch = dilation_patch + + def forward(self, input1: Tensor, input2: Tensor) -> Tensor: + return CorrelationFunction.apply(input1, input2, self.kernel_size, + self.max_displacement, self.stride, + self.padding, self.dilation, + self.dilation_patch) + + def __repr__(self) -> str: + s = self.__class__.__name__ + s += f'(kernel_size={self.kernel_size}, ' + s += f'max_displacement={self.max_displacement}, ' + s += f'stride={self.stride}, ' + s += f'padding={self.padding}, ' + s += f'dilation={self.dilation}, ' + s += f'dilation_patch={self.dilation_patch})' + return s diff --git a/annotator/mmpkg/mmcv/ops/deform_conv.py b/annotator/mmpkg/mmcv/ops/deform_conv.py new file mode 100644 index 0000000000000000000000000000000000000000..3de3aae1e7b2258360aef3ad9eb3a351f080f10f --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/deform_conv.py @@ -0,0 +1,405 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from typing import Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch import Tensor +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.utils import _pair, _single + +from annotator.mmpkg.mmcv.utils import deprecated_api_warning +from ..cnn import CONV_LAYERS +from ..utils import ext_loader, print_log + +ext_module = ext_loader.load_ext('_ext', [ + 'deform_conv_forward', 'deform_conv_backward_input', + 'deform_conv_backward_parameters' +]) + + +class DeformConv2dFunction(Function): + + @staticmethod + def symbolic(g, + input, + offset, + weight, + stride, + padding, + dilation, + groups, + deform_groups, + bias=False, + im2col_step=32): + return g.op( + 'mmcv::MMCVDeformConv2d', + input, + offset, + weight, + stride_i=stride, + padding_i=padding, + dilation_i=dilation, + groups_i=groups, + deform_groups_i=deform_groups, + bias_i=bias, + im2col_step_i=im2col_step) + + @staticmethod + def forward(ctx, + input, + offset, + weight, + stride=1, + padding=0, + dilation=1, + groups=1, + deform_groups=1, + bias=False, + im2col_step=32): + if input is not None and input.dim() != 4: + raise ValueError( + f'Expected 4D tensor as input, got {input.dim()}D tensor \ + instead.') + assert bias is False, 'Only support bias is False.' + ctx.stride = _pair(stride) + ctx.padding = _pair(padding) + ctx.dilation = _pair(dilation) + ctx.groups = groups + ctx.deform_groups = deform_groups + ctx.im2col_step = im2col_step + + # When pytorch version >= 1.6.0, amp is adopted for fp16 mode; + # amp won't cast the type of model (float32), but "offset" is cast + # to float16 by nn.Conv2d automatically, leading to the type + # mismatch with input (when it is float32) or weight. + # The flag for whether to use fp16 or amp is the type of "offset", + # we cast weight and input to temporarily support fp16 and amp + # whatever the pytorch version is. + input = input.type_as(offset) + weight = weight.type_as(input) + ctx.save_for_backward(input, offset, weight) + + output = input.new_empty( + DeformConv2dFunction._output_size(ctx, input, weight)) + + ctx.bufs_ = [input.new_empty(0), input.new_empty(0)] # columns, ones + + cur_im2col_step = min(ctx.im2col_step, input.size(0)) + assert (input.size(0) % + cur_im2col_step) == 0, 'im2col step must divide batchsize' + ext_module.deform_conv_forward( + input, + weight, + offset, + output, + ctx.bufs_[0], + ctx.bufs_[1], + kW=weight.size(3), + kH=weight.size(2), + dW=ctx.stride[1], + dH=ctx.stride[0], + padW=ctx.padding[1], + padH=ctx.padding[0], + dilationW=ctx.dilation[1], + dilationH=ctx.dilation[0], + group=ctx.groups, + deformable_group=ctx.deform_groups, + im2col_step=cur_im2col_step) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + input, offset, weight = ctx.saved_tensors + + grad_input = grad_offset = grad_weight = None + + cur_im2col_step = min(ctx.im2col_step, input.size(0)) + assert (input.size(0) % cur_im2col_step + ) == 0, 'batch size must be divisible by im2col_step' + + grad_output = grad_output.contiguous() + if ctx.needs_input_grad[0] or ctx.needs_input_grad[1]: + grad_input = torch.zeros_like(input) + grad_offset = torch.zeros_like(offset) + ext_module.deform_conv_backward_input( + input, + offset, + grad_output, + grad_input, + grad_offset, + weight, + ctx.bufs_[0], + kW=weight.size(3), + kH=weight.size(2), + dW=ctx.stride[1], + dH=ctx.stride[0], + padW=ctx.padding[1], + padH=ctx.padding[0], + dilationW=ctx.dilation[1], + dilationH=ctx.dilation[0], + group=ctx.groups, + deformable_group=ctx.deform_groups, + im2col_step=cur_im2col_step) + + if ctx.needs_input_grad[2]: + grad_weight = torch.zeros_like(weight) + ext_module.deform_conv_backward_parameters( + input, + offset, + grad_output, + grad_weight, + ctx.bufs_[0], + ctx.bufs_[1], + kW=weight.size(3), + kH=weight.size(2), + dW=ctx.stride[1], + dH=ctx.stride[0], + padW=ctx.padding[1], + padH=ctx.padding[0], + dilationW=ctx.dilation[1], + dilationH=ctx.dilation[0], + group=ctx.groups, + deformable_group=ctx.deform_groups, + scale=1, + im2col_step=cur_im2col_step) + + return grad_input, grad_offset, grad_weight, \ + None, None, None, None, None, None, None + + @staticmethod + def _output_size(ctx, input, weight): + channels = weight.size(0) + output_size = (input.size(0), channels) + for d in range(input.dim() - 2): + in_size = input.size(d + 2) + pad = ctx.padding[d] + kernel = ctx.dilation[d] * (weight.size(d + 2) - 1) + 1 + stride_ = ctx.stride[d] + output_size += ((in_size + (2 * pad) - kernel) // stride_ + 1, ) + if not all(map(lambda s: s > 0, output_size)): + raise ValueError( + 'convolution input is too small (output would be ' + + 'x'.join(map(str, output_size)) + ')') + return output_size + + +deform_conv2d = DeformConv2dFunction.apply + + +class DeformConv2d(nn.Module): + r"""Deformable 2D convolution. + + Applies a deformable 2D convolution over an input signal composed of + several input planes. DeformConv2d was described in the paper + `Deformable Convolutional Networks + `_ + + Note: + The argument ``im2col_step`` was added in version 1.3.17, which means + number of samples processed by the ``im2col_cuda_kernel`` per call. + It enables users to define ``batch_size`` and ``im2col_step`` more + flexibly and solved `issue mmcv#1440 + `_. + + Args: + in_channels (int): Number of channels in the input image. + out_channels (int): Number of channels produced by the convolution. + kernel_size(int, tuple): Size of the convolving kernel. + stride(int, tuple): Stride of the convolution. Default: 1. + padding (int or tuple): Zero-padding added to both sides of the input. + Default: 0. + dilation (int or tuple): Spacing between kernel elements. Default: 1. + groups (int): Number of blocked connections from input. + channels to output channels. Default: 1. + deform_groups (int): Number of deformable group partitions. + bias (bool): If True, adds a learnable bias to the output. + Default: False. + im2col_step (int): Number of samples processed by im2col_cuda_kernel + per call. It will work when ``batch_size`` > ``im2col_step``, but + ``batch_size`` must be divisible by ``im2col_step``. Default: 32. + `New in version 1.3.17.` + """ + + @deprecated_api_warning({'deformable_groups': 'deform_groups'}, + cls_name='DeformConv2d') + def __init__(self, + in_channels: int, + out_channels: int, + kernel_size: Union[int, Tuple[int, ...]], + stride: Union[int, Tuple[int, ...]] = 1, + padding: Union[int, Tuple[int, ...]] = 0, + dilation: Union[int, Tuple[int, ...]] = 1, + groups: int = 1, + deform_groups: int = 1, + bias: bool = False, + im2col_step: int = 32) -> None: + super(DeformConv2d, self).__init__() + + assert not bias, \ + f'bias={bias} is not supported in DeformConv2d.' + assert in_channels % groups == 0, \ + f'in_channels {in_channels} cannot be divisible by groups {groups}' + assert out_channels % groups == 0, \ + f'out_channels {out_channels} cannot be divisible by groups \ + {groups}' + + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = _pair(stride) + self.padding = _pair(padding) + self.dilation = _pair(dilation) + self.groups = groups + self.deform_groups = deform_groups + self.im2col_step = im2col_step + # enable compatibility with nn.Conv2d + self.transposed = False + self.output_padding = _single(0) + + # only weight, no bias + self.weight = nn.Parameter( + torch.Tensor(out_channels, in_channels // self.groups, + *self.kernel_size)) + + self.reset_parameters() + + def reset_parameters(self): + # switch the initialization of `self.weight` to the standard kaiming + # method described in `Delving deep into rectifiers: Surpassing + # human-level performance on ImageNet classification` - He, K. et al. + # (2015), using a uniform distribution + nn.init.kaiming_uniform_(self.weight, nonlinearity='relu') + + def forward(self, x: Tensor, offset: Tensor) -> Tensor: + """Deformable Convolutional forward function. + + Args: + x (Tensor): Input feature, shape (B, C_in, H_in, W_in) + offset (Tensor): Offset for deformable convolution, shape + (B, deform_groups*kernel_size[0]*kernel_size[1]*2, + H_out, W_out), H_out, W_out are equal to the output's. + + An offset is like `[y0, x0, y1, x1, y2, x2, ..., y8, x8]`. + The spatial arrangement is like: + + .. code:: text + + (x0, y0) (x1, y1) (x2, y2) + (x3, y3) (x4, y4) (x5, y5) + (x6, y6) (x7, y7) (x8, y8) + + Returns: + Tensor: Output of the layer. + """ + # To fix an assert error in deform_conv_cuda.cpp:128 + # input image is smaller than kernel + input_pad = (x.size(2) < self.kernel_size[0]) or (x.size(3) < + self.kernel_size[1]) + if input_pad: + pad_h = max(self.kernel_size[0] - x.size(2), 0) + pad_w = max(self.kernel_size[1] - x.size(3), 0) + x = F.pad(x, (0, pad_w, 0, pad_h), 'constant', 0).contiguous() + offset = F.pad(offset, (0, pad_w, 0, pad_h), 'constant', 0) + offset = offset.contiguous() + out = deform_conv2d(x, offset, self.weight, self.stride, self.padding, + self.dilation, self.groups, self.deform_groups, + False, self.im2col_step) + if input_pad: + out = out[:, :, :out.size(2) - pad_h, :out.size(3) - + pad_w].contiguous() + return out + + def __repr__(self): + s = self.__class__.__name__ + s += f'(in_channels={self.in_channels},\n' + s += f'out_channels={self.out_channels},\n' + s += f'kernel_size={self.kernel_size},\n' + s += f'stride={self.stride},\n' + s += f'padding={self.padding},\n' + s += f'dilation={self.dilation},\n' + s += f'groups={self.groups},\n' + s += f'deform_groups={self.deform_groups},\n' + # bias is not supported in DeformConv2d. + s += 'bias=False)' + return s + + +@CONV_LAYERS.register_module('DCN') +class DeformConv2dPack(DeformConv2d): + """A Deformable Conv Encapsulation that acts as normal Conv layers. + + The offset tensor is like `[y0, x0, y1, x1, y2, x2, ..., y8, x8]`. + The spatial arrangement is like: + + .. code:: text + + (x0, y0) (x1, y1) (x2, y2) + (x3, y3) (x4, y4) (x5, y5) + (x6, y6) (x7, y7) (x8, y8) + + Args: + in_channels (int): Same as nn.Conv2d. + out_channels (int): Same as nn.Conv2d. + kernel_size (int or tuple[int]): Same as nn.Conv2d. + stride (int or tuple[int]): Same as nn.Conv2d. + padding (int or tuple[int]): Same as nn.Conv2d. + dilation (int or tuple[int]): Same as nn.Conv2d. + groups (int): Same as nn.Conv2d. + bias (bool or str): If specified as `auto`, it will be decided by the + norm_cfg. Bias will be set as True if norm_cfg is None, otherwise + False. + """ + + _version = 2 + + def __init__(self, *args, **kwargs): + super(DeformConv2dPack, self).__init__(*args, **kwargs) + self.conv_offset = nn.Conv2d( + self.in_channels, + self.deform_groups * 2 * self.kernel_size[0] * self.kernel_size[1], + kernel_size=self.kernel_size, + stride=_pair(self.stride), + padding=_pair(self.padding), + dilation=_pair(self.dilation), + bias=True) + self.init_offset() + + def init_offset(self): + self.conv_offset.weight.data.zero_() + self.conv_offset.bias.data.zero_() + + def forward(self, x): + offset = self.conv_offset(x) + return deform_conv2d(x, offset, self.weight, self.stride, self.padding, + self.dilation, self.groups, self.deform_groups, + False, self.im2col_step) + + def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs): + version = local_metadata.get('version', None) + + if version is None or version < 2: + # the key is different in early versions + # In version < 2, DeformConvPack loads previous benchmark models. + if (prefix + 'conv_offset.weight' not in state_dict + and prefix[:-1] + '_offset.weight' in state_dict): + state_dict[prefix + 'conv_offset.weight'] = state_dict.pop( + prefix[:-1] + '_offset.weight') + if (prefix + 'conv_offset.bias' not in state_dict + and prefix[:-1] + '_offset.bias' in state_dict): + state_dict[prefix + + 'conv_offset.bias'] = state_dict.pop(prefix[:-1] + + '_offset.bias') + + if version is not None and version > 1: + print_log( + f'DeformConv2dPack {prefix.rstrip(".")} is upgraded to ' + 'version 2.', + logger='root') + + super()._load_from_state_dict(state_dict, prefix, local_metadata, + strict, missing_keys, unexpected_keys, + error_msgs) diff --git a/annotator/mmpkg/mmcv/ops/deform_roi_pool.py b/annotator/mmpkg/mmcv/ops/deform_roi_pool.py new file mode 100644 index 0000000000000000000000000000000000000000..cc245ba91fee252226ba22e76bb94a35db9a629b --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/deform_roi_pool.py @@ -0,0 +1,204 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from torch import nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.utils import _pair + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['deform_roi_pool_forward', 'deform_roi_pool_backward']) + + +class DeformRoIPoolFunction(Function): + + @staticmethod + def symbolic(g, input, rois, offset, output_size, spatial_scale, + sampling_ratio, gamma): + return g.op( + 'mmcv::MMCVDeformRoIPool', + input, + rois, + offset, + pooled_height_i=output_size[0], + pooled_width_i=output_size[1], + spatial_scale_f=spatial_scale, + sampling_ratio_f=sampling_ratio, + gamma_f=gamma) + + @staticmethod + def forward(ctx, + input, + rois, + offset, + output_size, + spatial_scale=1.0, + sampling_ratio=0, + gamma=0.1): + if offset is None: + offset = input.new_zeros(0) + ctx.output_size = _pair(output_size) + ctx.spatial_scale = float(spatial_scale) + ctx.sampling_ratio = int(sampling_ratio) + ctx.gamma = float(gamma) + + assert rois.size(1) == 5, 'RoI must be (idx, x1, y1, x2, y2)!' + + output_shape = (rois.size(0), input.size(1), ctx.output_size[0], + ctx.output_size[1]) + output = input.new_zeros(output_shape) + + ext_module.deform_roi_pool_forward( + input, + rois, + offset, + output, + pooled_height=ctx.output_size[0], + pooled_width=ctx.output_size[1], + spatial_scale=ctx.spatial_scale, + sampling_ratio=ctx.sampling_ratio, + gamma=ctx.gamma) + + ctx.save_for_backward(input, rois, offset) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + input, rois, offset = ctx.saved_tensors + grad_input = grad_output.new_zeros(input.shape) + grad_offset = grad_output.new_zeros(offset.shape) + + ext_module.deform_roi_pool_backward( + grad_output, + input, + rois, + offset, + grad_input, + grad_offset, + pooled_height=ctx.output_size[0], + pooled_width=ctx.output_size[1], + spatial_scale=ctx.spatial_scale, + sampling_ratio=ctx.sampling_ratio, + gamma=ctx.gamma) + if grad_offset.numel() == 0: + grad_offset = None + return grad_input, None, grad_offset, None, None, None, None + + +deform_roi_pool = DeformRoIPoolFunction.apply + + +class DeformRoIPool(nn.Module): + + def __init__(self, + output_size, + spatial_scale=1.0, + sampling_ratio=0, + gamma=0.1): + super(DeformRoIPool, self).__init__() + self.output_size = _pair(output_size) + self.spatial_scale = float(spatial_scale) + self.sampling_ratio = int(sampling_ratio) + self.gamma = float(gamma) + + def forward(self, input, rois, offset=None): + return deform_roi_pool(input, rois, offset, self.output_size, + self.spatial_scale, self.sampling_ratio, + self.gamma) + + +class DeformRoIPoolPack(DeformRoIPool): + + def __init__(self, + output_size, + output_channels, + deform_fc_channels=1024, + spatial_scale=1.0, + sampling_ratio=0, + gamma=0.1): + super(DeformRoIPoolPack, self).__init__(output_size, spatial_scale, + sampling_ratio, gamma) + + self.output_channels = output_channels + self.deform_fc_channels = deform_fc_channels + + self.offset_fc = nn.Sequential( + nn.Linear( + self.output_size[0] * self.output_size[1] * + self.output_channels, self.deform_fc_channels), + nn.ReLU(inplace=True), + nn.Linear(self.deform_fc_channels, self.deform_fc_channels), + nn.ReLU(inplace=True), + nn.Linear(self.deform_fc_channels, + self.output_size[0] * self.output_size[1] * 2)) + self.offset_fc[-1].weight.data.zero_() + self.offset_fc[-1].bias.data.zero_() + + def forward(self, input, rois): + assert input.size(1) == self.output_channels + x = deform_roi_pool(input, rois, None, self.output_size, + self.spatial_scale, self.sampling_ratio, + self.gamma) + rois_num = rois.size(0) + offset = self.offset_fc(x.view(rois_num, -1)) + offset = offset.view(rois_num, 2, self.output_size[0], + self.output_size[1]) + return deform_roi_pool(input, rois, offset, self.output_size, + self.spatial_scale, self.sampling_ratio, + self.gamma) + + +class ModulatedDeformRoIPoolPack(DeformRoIPool): + + def __init__(self, + output_size, + output_channels, + deform_fc_channels=1024, + spatial_scale=1.0, + sampling_ratio=0, + gamma=0.1): + super(ModulatedDeformRoIPoolPack, + self).__init__(output_size, spatial_scale, sampling_ratio, gamma) + + self.output_channels = output_channels + self.deform_fc_channels = deform_fc_channels + + self.offset_fc = nn.Sequential( + nn.Linear( + self.output_size[0] * self.output_size[1] * + self.output_channels, self.deform_fc_channels), + nn.ReLU(inplace=True), + nn.Linear(self.deform_fc_channels, self.deform_fc_channels), + nn.ReLU(inplace=True), + nn.Linear(self.deform_fc_channels, + self.output_size[0] * self.output_size[1] * 2)) + self.offset_fc[-1].weight.data.zero_() + self.offset_fc[-1].bias.data.zero_() + + self.mask_fc = nn.Sequential( + nn.Linear( + self.output_size[0] * self.output_size[1] * + self.output_channels, self.deform_fc_channels), + nn.ReLU(inplace=True), + nn.Linear(self.deform_fc_channels, + self.output_size[0] * self.output_size[1] * 1), + nn.Sigmoid()) + self.mask_fc[2].weight.data.zero_() + self.mask_fc[2].bias.data.zero_() + + def forward(self, input, rois): + assert input.size(1) == self.output_channels + x = deform_roi_pool(input, rois, None, self.output_size, + self.spatial_scale, self.sampling_ratio, + self.gamma) + rois_num = rois.size(0) + offset = self.offset_fc(x.view(rois_num, -1)) + offset = offset.view(rois_num, 2, self.output_size[0], + self.output_size[1]) + mask = self.mask_fc(x.view(rois_num, -1)) + mask = mask.view(rois_num, 1, self.output_size[0], self.output_size[1]) + d = deform_roi_pool(input, rois, offset, self.output_size, + self.spatial_scale, self.sampling_ratio, + self.gamma) + return d * mask diff --git a/annotator/mmpkg/mmcv/ops/deprecated_wrappers.py b/annotator/mmpkg/mmcv/ops/deprecated_wrappers.py new file mode 100644 index 0000000000000000000000000000000000000000..a2e593df9ee57637038683d7a1efaa347b2b69e7 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/deprecated_wrappers.py @@ -0,0 +1,43 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# This file is for backward compatibility. +# Module wrappers for empty tensor have been moved to mmcv.cnn.bricks. +import warnings + +from ..cnn.bricks.wrappers import Conv2d, ConvTranspose2d, Linear, MaxPool2d + + +class Conv2d_deprecated(Conv2d): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + warnings.warn( + 'Importing Conv2d wrapper from "mmcv.ops" will be deprecated in' + ' the future. Please import them from "mmcv.cnn" instead') + + +class ConvTranspose2d_deprecated(ConvTranspose2d): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + warnings.warn( + 'Importing ConvTranspose2d wrapper from "mmcv.ops" will be ' + 'deprecated in the future. Please import them from "mmcv.cnn" ' + 'instead') + + +class MaxPool2d_deprecated(MaxPool2d): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + warnings.warn( + 'Importing MaxPool2d wrapper from "mmcv.ops" will be deprecated in' + ' the future. Please import them from "mmcv.cnn" instead') + + +class Linear_deprecated(Linear): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + warnings.warn( + 'Importing Linear wrapper from "mmcv.ops" will be deprecated in' + ' the future. Please import them from "mmcv.cnn" instead') diff --git a/annotator/mmpkg/mmcv/ops/focal_loss.py b/annotator/mmpkg/mmcv/ops/focal_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..763bc93bd2575c49ca8ccf20996bbd92d1e0d1a4 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/focal_loss.py @@ -0,0 +1,212 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', [ + 'sigmoid_focal_loss_forward', 'sigmoid_focal_loss_backward', + 'softmax_focal_loss_forward', 'softmax_focal_loss_backward' +]) + + +class SigmoidFocalLossFunction(Function): + + @staticmethod + def symbolic(g, input, target, gamma, alpha, weight, reduction): + return g.op( + 'mmcv::MMCVSigmoidFocalLoss', + input, + target, + gamma_f=gamma, + alpha_f=alpha, + weight_f=weight, + reduction_s=reduction) + + @staticmethod + def forward(ctx, + input, + target, + gamma=2.0, + alpha=0.25, + weight=None, + reduction='mean'): + + assert isinstance(target, (torch.LongTensor, torch.cuda.LongTensor)) + assert input.dim() == 2 + assert target.dim() == 1 + assert input.size(0) == target.size(0) + if weight is None: + weight = input.new_empty(0) + else: + assert weight.dim() == 1 + assert input.size(1) == weight.size(0) + ctx.reduction_dict = {'none': 0, 'mean': 1, 'sum': 2} + assert reduction in ctx.reduction_dict.keys() + + ctx.gamma = float(gamma) + ctx.alpha = float(alpha) + ctx.reduction = ctx.reduction_dict[reduction] + + output = input.new_zeros(input.size()) + + ext_module.sigmoid_focal_loss_forward( + input, target, weight, output, gamma=ctx.gamma, alpha=ctx.alpha) + if ctx.reduction == ctx.reduction_dict['mean']: + output = output.sum() / input.size(0) + elif ctx.reduction == ctx.reduction_dict['sum']: + output = output.sum() + ctx.save_for_backward(input, target, weight) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + input, target, weight = ctx.saved_tensors + + grad_input = input.new_zeros(input.size()) + + ext_module.sigmoid_focal_loss_backward( + input, + target, + weight, + grad_input, + gamma=ctx.gamma, + alpha=ctx.alpha) + + grad_input *= grad_output + if ctx.reduction == ctx.reduction_dict['mean']: + grad_input /= input.size(0) + return grad_input, None, None, None, None, None + + +sigmoid_focal_loss = SigmoidFocalLossFunction.apply + + +class SigmoidFocalLoss(nn.Module): + + def __init__(self, gamma, alpha, weight=None, reduction='mean'): + super(SigmoidFocalLoss, self).__init__() + self.gamma = gamma + self.alpha = alpha + self.register_buffer('weight', weight) + self.reduction = reduction + + def forward(self, input, target): + return sigmoid_focal_loss(input, target, self.gamma, self.alpha, + self.weight, self.reduction) + + def __repr__(self): + s = self.__class__.__name__ + s += f'(gamma={self.gamma}, ' + s += f'alpha={self.alpha}, ' + s += f'reduction={self.reduction})' + return s + + +class SoftmaxFocalLossFunction(Function): + + @staticmethod + def symbolic(g, input, target, gamma, alpha, weight, reduction): + return g.op( + 'mmcv::MMCVSoftmaxFocalLoss', + input, + target, + gamma_f=gamma, + alpha_f=alpha, + weight_f=weight, + reduction_s=reduction) + + @staticmethod + def forward(ctx, + input, + target, + gamma=2.0, + alpha=0.25, + weight=None, + reduction='mean'): + + assert isinstance(target, (torch.LongTensor, torch.cuda.LongTensor)) + assert input.dim() == 2 + assert target.dim() == 1 + assert input.size(0) == target.size(0) + if weight is None: + weight = input.new_empty(0) + else: + assert weight.dim() == 1 + assert input.size(1) == weight.size(0) + ctx.reduction_dict = {'none': 0, 'mean': 1, 'sum': 2} + assert reduction in ctx.reduction_dict.keys() + + ctx.gamma = float(gamma) + ctx.alpha = float(alpha) + ctx.reduction = ctx.reduction_dict[reduction] + + channel_stats, _ = torch.max(input, dim=1) + input_softmax = input - channel_stats.unsqueeze(1).expand_as(input) + input_softmax.exp_() + + channel_stats = input_softmax.sum(dim=1) + input_softmax /= channel_stats.unsqueeze(1).expand_as(input) + + output = input.new_zeros(input.size(0)) + ext_module.softmax_focal_loss_forward( + input_softmax, + target, + weight, + output, + gamma=ctx.gamma, + alpha=ctx.alpha) + + if ctx.reduction == ctx.reduction_dict['mean']: + output = output.sum() / input.size(0) + elif ctx.reduction == ctx.reduction_dict['sum']: + output = output.sum() + ctx.save_for_backward(input_softmax, target, weight) + return output + + @staticmethod + def backward(ctx, grad_output): + input_softmax, target, weight = ctx.saved_tensors + buff = input_softmax.new_zeros(input_softmax.size(0)) + grad_input = input_softmax.new_zeros(input_softmax.size()) + + ext_module.softmax_focal_loss_backward( + input_softmax, + target, + weight, + buff, + grad_input, + gamma=ctx.gamma, + alpha=ctx.alpha) + + grad_input *= grad_output + if ctx.reduction == ctx.reduction_dict['mean']: + grad_input /= input_softmax.size(0) + return grad_input, None, None, None, None, None + + +softmax_focal_loss = SoftmaxFocalLossFunction.apply + + +class SoftmaxFocalLoss(nn.Module): + + def __init__(self, gamma, alpha, weight=None, reduction='mean'): + super(SoftmaxFocalLoss, self).__init__() + self.gamma = gamma + self.alpha = alpha + self.register_buffer('weight', weight) + self.reduction = reduction + + def forward(self, input, target): + return softmax_focal_loss(input, target, self.gamma, self.alpha, + self.weight, self.reduction) + + def __repr__(self): + s = self.__class__.__name__ + s += f'(gamma={self.gamma}, ' + s += f'alpha={self.alpha}, ' + s += f'reduction={self.reduction})' + return s diff --git a/annotator/mmpkg/mmcv/ops/furthest_point_sample.py b/annotator/mmpkg/mmcv/ops/furthest_point_sample.py new file mode 100644 index 0000000000000000000000000000000000000000..374b7a878f1972c183941af28ba1df216ac1a60f --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/furthest_point_sample.py @@ -0,0 +1,83 @@ +import torch +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', [ + 'furthest_point_sampling_forward', + 'furthest_point_sampling_with_dist_forward' +]) + + +class FurthestPointSampling(Function): + """Uses iterative furthest point sampling to select a set of features whose + corresponding points have the furthest distance.""" + + @staticmethod + def forward(ctx, points_xyz: torch.Tensor, + num_points: int) -> torch.Tensor: + """ + Args: + points_xyz (Tensor): (B, N, 3) where N > num_points. + num_points (int): Number of points in the sampled set. + + Returns: + Tensor: (B, num_points) indices of the sampled points. + """ + assert points_xyz.is_contiguous() + + B, N = points_xyz.size()[:2] + output = torch.cuda.IntTensor(B, num_points) + temp = torch.cuda.FloatTensor(B, N).fill_(1e10) + + ext_module.furthest_point_sampling_forward( + points_xyz, + temp, + output, + b=B, + n=N, + m=num_points, + ) + if torch.__version__ != 'parrots': + ctx.mark_non_differentiable(output) + return output + + @staticmethod + def backward(xyz, a=None): + return None, None + + +class FurthestPointSamplingWithDist(Function): + """Uses iterative furthest point sampling to select a set of features whose + corresponding points have the furthest distance.""" + + @staticmethod + def forward(ctx, points_dist: torch.Tensor, + num_points: int) -> torch.Tensor: + """ + Args: + points_dist (Tensor): (B, N, N) Distance between each point pair. + num_points (int): Number of points in the sampled set. + + Returns: + Tensor: (B, num_points) indices of the sampled points. + """ + assert points_dist.is_contiguous() + + B, N, _ = points_dist.size() + output = points_dist.new_zeros([B, num_points], dtype=torch.int32) + temp = points_dist.new_zeros([B, N]).fill_(1e10) + + ext_module.furthest_point_sampling_with_dist_forward( + points_dist, temp, output, b=B, n=N, m=num_points) + if torch.__version__ != 'parrots': + ctx.mark_non_differentiable(output) + return output + + @staticmethod + def backward(xyz, a=None): + return None, None + + +furthest_point_sample = FurthestPointSampling.apply +furthest_point_sample_with_dist = FurthestPointSamplingWithDist.apply diff --git a/annotator/mmpkg/mmcv/ops/fused_bias_leakyrelu.py b/annotator/mmpkg/mmcv/ops/fused_bias_leakyrelu.py new file mode 100644 index 0000000000000000000000000000000000000000..6d12508469c6c8fa1884debece44c58d158cb6fa --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/fused_bias_leakyrelu.py @@ -0,0 +1,268 @@ +# modified from https://github.com/rosinality/stylegan2-pytorch/blob/master/op/fused_act.py # noqa:E501 + +# Copyright (c) 2021, NVIDIA Corporation. All rights reserved. +# NVIDIA Source Code License for StyleGAN2 with Adaptive Discriminator +# Augmentation (ADA) +# ======================================================================= + +# 1. Definitions + +# "Licensor" means any person or entity that distributes its Work. + +# "Software" means the original work of authorship made available under +# this License. + +# "Work" means the Software and any additions to or derivative works of +# the Software that are made available under this License. + +# The terms "reproduce," "reproduction," "derivative works," and +# "distribution" have the meaning as provided under U.S. copyright law; +# provided, however, that for the purposes of this License, derivative +# works shall not include works that remain separable from, or merely +# link (or bind by name) to the interfaces of, the Work. + +# Works, including the Software, are "made available" under this License +# by including in or with the Work either (a) a copyright notice +# referencing the applicability of this License to the Work, or (b) a +# copy of this License. + +# 2. License Grants + +# 2.1 Copyright Grant. Subject to the terms and conditions of this +# License, each Licensor grants to you a perpetual, worldwide, +# non-exclusive, royalty-free, copyright license to reproduce, +# prepare derivative works of, publicly display, publicly perform, +# sublicense and distribute its Work and any resulting derivative +# works in any form. + +# 3. Limitations + +# 3.1 Redistribution. You may reproduce or distribute the Work only +# if (a) you do so under this License, (b) you include a complete +# copy of this License with your distribution, and (c) you retain +# without modification any copyright, patent, trademark, or +# attribution notices that are present in the Work. + +# 3.2 Derivative Works. You may specify that additional or different +# terms apply to the use, reproduction, and distribution of your +# derivative works of the Work ("Your Terms") only if (a) Your Terms +# provide that the use limitation in Section 3.3 applies to your +# derivative works, and (b) you identify the specific derivative +# works that are subject to Your Terms. Notwithstanding Your Terms, +# this License (including the redistribution requirements in Section +# 3.1) will continue to apply to the Work itself. + +# 3.3 Use Limitation. The Work and any derivative works thereof only +# may be used or intended for use non-commercially. Notwithstanding +# the foregoing, NVIDIA and its affiliates may use the Work and any +# derivative works commercially. As used herein, "non-commercially" +# means for research or evaluation purposes only. + +# 3.4 Patent Claims. If you bring or threaten to bring a patent claim +# against any Licensor (including any claim, cross-claim or +# counterclaim in a lawsuit) to enforce any patents that you allege +# are infringed by any Work, then your rights under this License from +# such Licensor (including the grant in Section 2.1) will terminate +# immediately. + +# 3.5 Trademarks. This License does not grant any rights to use any +# Licensor’s or its affiliates’ names, logos, or trademarks, except +# as necessary to reproduce the notices described in this License. + +# 3.6 Termination. If you violate any term of this License, then your +# rights under this License (including the grant in Section 2.1) will +# terminate immediately. + +# 4. Disclaimer of Warranty. + +# THE WORK IS PROVIDED "AS IS" WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WARRANTIES OR CONDITIONS OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR +# NON-INFRINGEMENT. YOU BEAR THE RISK OF UNDERTAKING ANY ACTIVITIES UNDER +# THIS LICENSE. + +# 5. Limitation of Liability. + +# EXCEPT AS PROHIBITED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL +# THEORY, WHETHER IN TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE +# SHALL ANY LICENSOR BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, +# INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF +# OR RELATED TO THIS LICENSE, THE USE OR INABILITY TO USE THE WORK +# (INCLUDING BUT NOT LIMITED TO LOSS OF GOODWILL, BUSINESS INTERRUPTION, +# LOST PROFITS OR DATA, COMPUTER FAILURE OR MALFUNCTION, OR ANY OTHER +# COMMERCIAL DAMAGES OR LOSSES), EVEN IF THE LICENSOR HAS BEEN ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGES. + +# ======================================================================= + +import torch +import torch.nn.functional as F +from torch import nn +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', ['fused_bias_leakyrelu']) + + +class FusedBiasLeakyReLUFunctionBackward(Function): + """Calculate second order deviation. + + This function is to compute the second order deviation for the fused leaky + relu operation. + """ + + @staticmethod + def forward(ctx, grad_output, out, negative_slope, scale): + ctx.save_for_backward(out) + ctx.negative_slope = negative_slope + ctx.scale = scale + + empty = grad_output.new_empty(0) + + grad_input = ext_module.fused_bias_leakyrelu( + grad_output, + empty, + out, + act=3, + grad=1, + alpha=negative_slope, + scale=scale) + + dim = [0] + + if grad_input.ndim > 2: + dim += list(range(2, grad_input.ndim)) + + grad_bias = grad_input.sum(dim).detach() + + return grad_input, grad_bias + + @staticmethod + def backward(ctx, gradgrad_input, gradgrad_bias): + out, = ctx.saved_tensors + + # The second order deviation, in fact, contains two parts, while the + # the first part is zero. Thus, we direct consider the second part + # which is similar with the first order deviation in implementation. + gradgrad_out = ext_module.fused_bias_leakyrelu( + gradgrad_input, + gradgrad_bias.to(out.dtype), + out, + act=3, + grad=1, + alpha=ctx.negative_slope, + scale=ctx.scale) + + return gradgrad_out, None, None, None + + +class FusedBiasLeakyReLUFunction(Function): + + @staticmethod + def forward(ctx, input, bias, negative_slope, scale): + empty = input.new_empty(0) + + out = ext_module.fused_bias_leakyrelu( + input, + bias, + empty, + act=3, + grad=0, + alpha=negative_slope, + scale=scale) + ctx.save_for_backward(out) + ctx.negative_slope = negative_slope + ctx.scale = scale + + return out + + @staticmethod + def backward(ctx, grad_output): + out, = ctx.saved_tensors + + grad_input, grad_bias = FusedBiasLeakyReLUFunctionBackward.apply( + grad_output, out, ctx.negative_slope, ctx.scale) + + return grad_input, grad_bias, None, None + + +class FusedBiasLeakyReLU(nn.Module): + """Fused bias leaky ReLU. + + This function is introduced in the StyleGAN2: + http://arxiv.org/abs/1912.04958 + + The bias term comes from the convolution operation. In addition, to keep + the variance of the feature map or gradients unchanged, they also adopt a + scale similarly with Kaiming initialization. However, since the + :math:`1+{alpha}^2` : is too small, we can just ignore it. Therefore, the + final scale is just :math:`\sqrt{2}`:. Of course, you may change it with # noqa: W605, E501 + your own scale. + + TODO: Implement the CPU version. + + Args: + channel (int): The channel number of the feature map. + negative_slope (float, optional): Same as nn.LeakyRelu. + Defaults to 0.2. + scale (float, optional): A scalar to adjust the variance of the feature + map. Defaults to 2**0.5. + """ + + def __init__(self, num_channels, negative_slope=0.2, scale=2**0.5): + super(FusedBiasLeakyReLU, self).__init__() + + self.bias = nn.Parameter(torch.zeros(num_channels)) + self.negative_slope = negative_slope + self.scale = scale + + def forward(self, input): + return fused_bias_leakyrelu(input, self.bias, self.negative_slope, + self.scale) + + +def fused_bias_leakyrelu(input, bias, negative_slope=0.2, scale=2**0.5): + """Fused bias leaky ReLU function. + + This function is introduced in the StyleGAN2: + http://arxiv.org/abs/1912.04958 + + The bias term comes from the convolution operation. In addition, to keep + the variance of the feature map or gradients unchanged, they also adopt a + scale similarly with Kaiming initialization. However, since the + :math:`1+{alpha}^2` : is too small, we can just ignore it. Therefore, the + final scale is just :math:`\sqrt{2}`:. Of course, you may change it with # noqa: W605, E501 + your own scale. + + Args: + input (torch.Tensor): Input feature map. + bias (nn.Parameter): The bias from convolution operation. + negative_slope (float, optional): Same as nn.LeakyRelu. + Defaults to 0.2. + scale (float, optional): A scalar to adjust the variance of the feature + map. Defaults to 2**0.5. + + Returns: + torch.Tensor: Feature map after non-linear activation. + """ + + if not input.is_cuda: + return bias_leakyrelu_ref(input, bias, negative_slope, scale) + + return FusedBiasLeakyReLUFunction.apply(input, bias.to(input.dtype), + negative_slope, scale) + + +def bias_leakyrelu_ref(x, bias, negative_slope=0.2, scale=2**0.5): + + if bias is not None: + assert bias.ndim == 1 + assert bias.shape[0] == x.shape[1] + x = x + bias.reshape([-1 if i == 1 else 1 for i in range(x.ndim)]) + + x = F.leaky_relu(x, negative_slope) + if scale != 1: + x = x * scale + + return x diff --git a/annotator/mmpkg/mmcv/ops/gather_points.py b/annotator/mmpkg/mmcv/ops/gather_points.py new file mode 100644 index 0000000000000000000000000000000000000000..f52f1677d8ea0facafc56a3672d37adb44677ff3 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/gather_points.py @@ -0,0 +1,57 @@ +import torch +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['gather_points_forward', 'gather_points_backward']) + + +class GatherPoints(Function): + """Gather points with given index.""" + + @staticmethod + def forward(ctx, features: torch.Tensor, + indices: torch.Tensor) -> torch.Tensor: + """ + Args: + features (Tensor): (B, C, N) features to gather. + indices (Tensor): (B, M) where M is the number of points. + + Returns: + Tensor: (B, C, M) where M is the number of points. + """ + assert features.is_contiguous() + assert indices.is_contiguous() + + B, npoint = indices.size() + _, C, N = features.size() + output = torch.cuda.FloatTensor(B, C, npoint) + + ext_module.gather_points_forward( + features, indices, output, b=B, c=C, n=N, npoints=npoint) + + ctx.for_backwards = (indices, C, N) + if torch.__version__ != 'parrots': + ctx.mark_non_differentiable(indices) + return output + + @staticmethod + def backward(ctx, grad_out): + idx, C, N = ctx.for_backwards + B, npoint = idx.size() + + grad_features = torch.cuda.FloatTensor(B, C, N).zero_() + grad_out_data = grad_out.data.contiguous() + ext_module.gather_points_backward( + grad_out_data, + idx, + grad_features.data, + b=B, + c=C, + n=N, + npoints=npoint) + return grad_features, None + + +gather_points = GatherPoints.apply diff --git a/annotator/mmpkg/mmcv/ops/group_points.py b/annotator/mmpkg/mmcv/ops/group_points.py new file mode 100644 index 0000000000000000000000000000000000000000..6c3ec9d758ebe4e1c2205882af4be154008253a5 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/group_points.py @@ -0,0 +1,224 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from typing import Tuple + +import torch +from torch import nn as nn +from torch.autograd import Function + +from ..utils import ext_loader +from .ball_query import ball_query +from .knn import knn + +ext_module = ext_loader.load_ext( + '_ext', ['group_points_forward', 'group_points_backward']) + + +class QueryAndGroup(nn.Module): + """Groups points with a ball query of radius. + + Args: + max_radius (float): The maximum radius of the balls. + If None is given, we will use kNN sampling instead of ball query. + sample_num (int): Maximum number of features to gather in the ball. + min_radius (float, optional): The minimum radius of the balls. + Default: 0. + use_xyz (bool, optional): Whether to use xyz. + Default: True. + return_grouped_xyz (bool, optional): Whether to return grouped xyz. + Default: False. + normalize_xyz (bool, optional): Whether to normalize xyz. + Default: False. + uniform_sample (bool, optional): Whether to sample uniformly. + Default: False + return_unique_cnt (bool, optional): Whether to return the count of + unique samples. Default: False. + return_grouped_idx (bool, optional): Whether to return grouped idx. + Default: False. + """ + + def __init__(self, + max_radius, + sample_num, + min_radius=0, + use_xyz=True, + return_grouped_xyz=False, + normalize_xyz=False, + uniform_sample=False, + return_unique_cnt=False, + return_grouped_idx=False): + super().__init__() + self.max_radius = max_radius + self.min_radius = min_radius + self.sample_num = sample_num + self.use_xyz = use_xyz + self.return_grouped_xyz = return_grouped_xyz + self.normalize_xyz = normalize_xyz + self.uniform_sample = uniform_sample + self.return_unique_cnt = return_unique_cnt + self.return_grouped_idx = return_grouped_idx + if self.return_unique_cnt: + assert self.uniform_sample, \ + 'uniform_sample should be True when ' \ + 'returning the count of unique samples' + if self.max_radius is None: + assert not self.normalize_xyz, \ + 'can not normalize grouped xyz when max_radius is None' + + def forward(self, points_xyz, center_xyz, features=None): + """ + Args: + points_xyz (Tensor): (B, N, 3) xyz coordinates of the features. + center_xyz (Tensor): (B, npoint, 3) coordinates of the centriods. + features (Tensor): (B, C, N) Descriptors of the features. + + Returns: + Tensor: (B, 3 + C, npoint, sample_num) Grouped feature. + """ + # if self.max_radius is None, we will perform kNN instead of ball query + # idx is of shape [B, npoint, sample_num] + if self.max_radius is None: + idx = knn(self.sample_num, points_xyz, center_xyz, False) + idx = idx.transpose(1, 2).contiguous() + else: + idx = ball_query(self.min_radius, self.max_radius, self.sample_num, + points_xyz, center_xyz) + + if self.uniform_sample: + unique_cnt = torch.zeros((idx.shape[0], idx.shape[1])) + for i_batch in range(idx.shape[0]): + for i_region in range(idx.shape[1]): + unique_ind = torch.unique(idx[i_batch, i_region, :]) + num_unique = unique_ind.shape[0] + unique_cnt[i_batch, i_region] = num_unique + sample_ind = torch.randint( + 0, + num_unique, (self.sample_num - num_unique, ), + dtype=torch.long) + all_ind = torch.cat((unique_ind, unique_ind[sample_ind])) + idx[i_batch, i_region, :] = all_ind + + xyz_trans = points_xyz.transpose(1, 2).contiguous() + # (B, 3, npoint, sample_num) + grouped_xyz = grouping_operation(xyz_trans, idx) + grouped_xyz_diff = grouped_xyz - \ + center_xyz.transpose(1, 2).unsqueeze(-1) # relative offsets + if self.normalize_xyz: + grouped_xyz_diff /= self.max_radius + + if features is not None: + grouped_features = grouping_operation(features, idx) + if self.use_xyz: + # (B, C + 3, npoint, sample_num) + new_features = torch.cat([grouped_xyz_diff, grouped_features], + dim=1) + else: + new_features = grouped_features + else: + assert (self.use_xyz + ), 'Cannot have not features and not use xyz as a feature!' + new_features = grouped_xyz_diff + + ret = [new_features] + if self.return_grouped_xyz: + ret.append(grouped_xyz) + if self.return_unique_cnt: + ret.append(unique_cnt) + if self.return_grouped_idx: + ret.append(idx) + if len(ret) == 1: + return ret[0] + else: + return tuple(ret) + + +class GroupAll(nn.Module): + """Group xyz with feature. + + Args: + use_xyz (bool): Whether to use xyz. + """ + + def __init__(self, use_xyz: bool = True): + super().__init__() + self.use_xyz = use_xyz + + def forward(self, + xyz: torch.Tensor, + new_xyz: torch.Tensor, + features: torch.Tensor = None): + """ + Args: + xyz (Tensor): (B, N, 3) xyz coordinates of the features. + new_xyz (Tensor): new xyz coordinates of the features. + features (Tensor): (B, C, N) features to group. + + Returns: + Tensor: (B, C + 3, 1, N) Grouped feature. + """ + grouped_xyz = xyz.transpose(1, 2).unsqueeze(2) + if features is not None: + grouped_features = features.unsqueeze(2) + if self.use_xyz: + # (B, 3 + C, 1, N) + new_features = torch.cat([grouped_xyz, grouped_features], + dim=1) + else: + new_features = grouped_features + else: + new_features = grouped_xyz + + return new_features + + +class GroupingOperation(Function): + """Group feature with given index.""" + + @staticmethod + def forward(ctx, features: torch.Tensor, + indices: torch.Tensor) -> torch.Tensor: + """ + Args: + features (Tensor): (B, C, N) tensor of features to group. + indices (Tensor): (B, npoint, nsample) the indices of + features to group with. + + Returns: + Tensor: (B, C, npoint, nsample) Grouped features. + """ + features = features.contiguous() + indices = indices.contiguous() + + B, nfeatures, nsample = indices.size() + _, C, N = features.size() + output = torch.cuda.FloatTensor(B, C, nfeatures, nsample) + + ext_module.group_points_forward(B, C, N, nfeatures, nsample, features, + indices, output) + + ctx.for_backwards = (indices, N) + return output + + @staticmethod + def backward(ctx, + grad_out: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Args: + grad_out (Tensor): (B, C, npoint, nsample) tensor of the gradients + of the output from forward. + + Returns: + Tensor: (B, C, N) gradient of the features. + """ + idx, N = ctx.for_backwards + + B, C, npoint, nsample = grad_out.size() + grad_features = torch.cuda.FloatTensor(B, C, N).zero_() + + grad_out_data = grad_out.data.contiguous() + ext_module.group_points_backward(B, C, N, npoint, nsample, + grad_out_data, idx, + grad_features.data) + return grad_features, None + + +grouping_operation = GroupingOperation.apply diff --git a/annotator/mmpkg/mmcv/ops/info.py b/annotator/mmpkg/mmcv/ops/info.py new file mode 100644 index 0000000000000000000000000000000000000000..29f2e5598ae2bb5866ccd15a7d3b4de33c0cd14d --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/info.py @@ -0,0 +1,36 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import glob +import os + +import torch + +if torch.__version__ == 'parrots': + import parrots + + def get_compiler_version(): + return 'GCC ' + parrots.version.compiler + + def get_compiling_cuda_version(): + return parrots.version.cuda +else: + from ..utils import ext_loader + ext_module = ext_loader.load_ext( + '_ext', ['get_compiler_version', 'get_compiling_cuda_version']) + + def get_compiler_version(): + return ext_module.get_compiler_version() + + def get_compiling_cuda_version(): + return ext_module.get_compiling_cuda_version() + + +def get_onnxruntime_op_path(): + wildcard = os.path.join( + os.path.abspath(os.path.dirname(os.path.dirname(__file__))), + '_ext_ort.*.so') + + paths = glob.glob(wildcard) + if len(paths) > 0: + return paths[0] + else: + return '' diff --git a/annotator/mmpkg/mmcv/ops/iou3d.py b/annotator/mmpkg/mmcv/ops/iou3d.py new file mode 100644 index 0000000000000000000000000000000000000000..6fc71979190323f44c09f8b7e1761cf49cd2d76b --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/iou3d.py @@ -0,0 +1,85 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', [ + 'iou3d_boxes_iou_bev_forward', 'iou3d_nms_forward', + 'iou3d_nms_normal_forward' +]) + + +def boxes_iou_bev(boxes_a, boxes_b): + """Calculate boxes IoU in the Bird's Eye View. + + Args: + boxes_a (torch.Tensor): Input boxes a with shape (M, 5). + boxes_b (torch.Tensor): Input boxes b with shape (N, 5). + + Returns: + ans_iou (torch.Tensor): IoU result with shape (M, N). + """ + ans_iou = boxes_a.new_zeros( + torch.Size((boxes_a.shape[0], boxes_b.shape[0]))) + + ext_module.iou3d_boxes_iou_bev_forward(boxes_a.contiguous(), + boxes_b.contiguous(), ans_iou) + + return ans_iou + + +def nms_bev(boxes, scores, thresh, pre_max_size=None, post_max_size=None): + """NMS function GPU implementation (for BEV boxes). The overlap of two + boxes for IoU calculation is defined as the exact overlapping area of the + two boxes. In this function, one can also set ``pre_max_size`` and + ``post_max_size``. + + Args: + boxes (torch.Tensor): Input boxes with the shape of [N, 5] + ([x1, y1, x2, y2, ry]). + scores (torch.Tensor): Scores of boxes with the shape of [N]. + thresh (float): Overlap threshold of NMS. + pre_max_size (int, optional): Max size of boxes before NMS. + Default: None. + post_max_size (int, optional): Max size of boxes after NMS. + Default: None. + + Returns: + torch.Tensor: Indexes after NMS. + """ + assert boxes.size(1) == 5, 'Input boxes shape should be [N, 5]' + order = scores.sort(0, descending=True)[1] + + if pre_max_size is not None: + order = order[:pre_max_size] + boxes = boxes[order].contiguous() + + keep = torch.zeros(boxes.size(0), dtype=torch.long) + num_out = ext_module.iou3d_nms_forward(boxes, keep, thresh) + keep = order[keep[:num_out].cuda(boxes.device)].contiguous() + if post_max_size is not None: + keep = keep[:post_max_size] + return keep + + +def nms_normal_bev(boxes, scores, thresh): + """Normal NMS function GPU implementation (for BEV boxes). The overlap of + two boxes for IoU calculation is defined as the exact overlapping area of + the two boxes WITH their yaw angle set to 0. + + Args: + boxes (torch.Tensor): Input boxes with shape (N, 5). + scores (torch.Tensor): Scores of predicted boxes with shape (N). + thresh (float): Overlap threshold of NMS. + + Returns: + torch.Tensor: Remaining indices with scores in descending order. + """ + assert boxes.shape[1] == 5, 'Input boxes shape should be [N, 5]' + order = scores.sort(0, descending=True)[1] + + boxes = boxes[order].contiguous() + + keep = torch.zeros(boxes.size(0), dtype=torch.long) + num_out = ext_module.iou3d_nms_normal_forward(boxes, keep, thresh) + return order[keep[:num_out].cuda(boxes.device)].contiguous() diff --git a/annotator/mmpkg/mmcv/ops/knn.py b/annotator/mmpkg/mmcv/ops/knn.py new file mode 100644 index 0000000000000000000000000000000000000000..f335785036669fc19239825b0aae6dde3f73bf92 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/knn.py @@ -0,0 +1,77 @@ +import torch +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', ['knn_forward']) + + +class KNN(Function): + r"""KNN (CUDA) based on heap data structure. + Modified from `PAConv `_. + + Find k-nearest points. + """ + + @staticmethod + def forward(ctx, + k: int, + xyz: torch.Tensor, + center_xyz: torch.Tensor = None, + transposed: bool = False) -> torch.Tensor: + """ + Args: + k (int): number of nearest neighbors. + xyz (Tensor): (B, N, 3) if transposed == False, else (B, 3, N). + xyz coordinates of the features. + center_xyz (Tensor, optional): (B, npoint, 3) if transposed == + False, else (B, 3, npoint). centers of the knn query. + Default: None. + transposed (bool, optional): whether the input tensors are + transposed. Should not explicitly use this keyword when + calling knn (=KNN.apply), just add the fourth param. + Default: False. + + Returns: + Tensor: (B, k, npoint) tensor with the indices of + the features that form k-nearest neighbours. + """ + assert (k > 0) & (k < 100), 'k should be in range(0, 100)' + + if center_xyz is None: + center_xyz = xyz + + if transposed: + xyz = xyz.transpose(2, 1).contiguous() + center_xyz = center_xyz.transpose(2, 1).contiguous() + + assert xyz.is_contiguous() # [B, N, 3] + assert center_xyz.is_contiguous() # [B, npoint, 3] + + center_xyz_device = center_xyz.get_device() + assert center_xyz_device == xyz.get_device(), \ + 'center_xyz and xyz should be put on the same device' + if torch.cuda.current_device() != center_xyz_device: + torch.cuda.set_device(center_xyz_device) + + B, npoint, _ = center_xyz.shape + N = xyz.shape[1] + + idx = center_xyz.new_zeros((B, npoint, k)).int() + dist2 = center_xyz.new_zeros((B, npoint, k)).float() + + ext_module.knn_forward( + xyz, center_xyz, idx, dist2, b=B, n=N, m=npoint, nsample=k) + # idx shape to [B, k, npoint] + idx = idx.transpose(2, 1).contiguous() + if torch.__version__ != 'parrots': + ctx.mark_non_differentiable(idx) + return idx + + @staticmethod + def backward(ctx, a=None): + return None, None, None + + +knn = KNN.apply diff --git a/annotator/mmpkg/mmcv/ops/masked_conv.py b/annotator/mmpkg/mmcv/ops/masked_conv.py new file mode 100644 index 0000000000000000000000000000000000000000..cd514cc204c1d571ea5dc7e74b038c0f477a008b --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/masked_conv.py @@ -0,0 +1,111 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import math + +import torch +import torch.nn as nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.utils import _pair + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['masked_im2col_forward', 'masked_col2im_forward']) + + +class MaskedConv2dFunction(Function): + + @staticmethod + def symbolic(g, features, mask, weight, bias, padding, stride): + return g.op( + 'mmcv::MMCVMaskedConv2d', + features, + mask, + weight, + bias, + padding_i=padding, + stride_i=stride) + + @staticmethod + def forward(ctx, features, mask, weight, bias, padding=0, stride=1): + assert mask.dim() == 3 and mask.size(0) == 1 + assert features.dim() == 4 and features.size(0) == 1 + assert features.size()[2:] == mask.size()[1:] + pad_h, pad_w = _pair(padding) + stride_h, stride_w = _pair(stride) + if stride_h != 1 or stride_w != 1: + raise ValueError( + 'Stride could not only be 1 in masked_conv2d currently.') + out_channel, in_channel, kernel_h, kernel_w = weight.size() + + batch_size = features.size(0) + out_h = int( + math.floor((features.size(2) + 2 * pad_h - + (kernel_h - 1) - 1) / stride_h + 1)) + out_w = int( + math.floor((features.size(3) + 2 * pad_w - + (kernel_h - 1) - 1) / stride_w + 1)) + mask_inds = torch.nonzero(mask[0] > 0, as_tuple=False) + output = features.new_zeros(batch_size, out_channel, out_h, out_w) + if mask_inds.numel() > 0: + mask_h_idx = mask_inds[:, 0].contiguous() + mask_w_idx = mask_inds[:, 1].contiguous() + data_col = features.new_zeros(in_channel * kernel_h * kernel_w, + mask_inds.size(0)) + ext_module.masked_im2col_forward( + features, + mask_h_idx, + mask_w_idx, + data_col, + kernel_h=kernel_h, + kernel_w=kernel_w, + pad_h=pad_h, + pad_w=pad_w) + + masked_output = torch.addmm(1, bias[:, None], 1, + weight.view(out_channel, -1), data_col) + ext_module.masked_col2im_forward( + masked_output, + mask_h_idx, + mask_w_idx, + output, + height=out_h, + width=out_w, + channels=out_channel) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + return (None, ) * 5 + + +masked_conv2d = MaskedConv2dFunction.apply + + +class MaskedConv2d(nn.Conv2d): + """A MaskedConv2d which inherits the official Conv2d. + + The masked forward doesn't implement the backward function and only + supports the stride parameter to be 1 currently. + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + bias=True): + super(MaskedConv2d, + self).__init__(in_channels, out_channels, kernel_size, stride, + padding, dilation, groups, bias) + + def forward(self, input, mask=None): + if mask is None: # fallback to the normal Conv2d + return super(MaskedConv2d, self).forward(input) + else: + return masked_conv2d(input, mask, self.weight, self.bias, + self.padding) diff --git a/annotator/mmpkg/mmcv/ops/merge_cells.py b/annotator/mmpkg/mmcv/ops/merge_cells.py new file mode 100644 index 0000000000000000000000000000000000000000..48ca8cc0a8aca8432835bd760c0403a3c35b34cf --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/merge_cells.py @@ -0,0 +1,149 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from abc import abstractmethod + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..cnn import ConvModule + + +class BaseMergeCell(nn.Module): + """The basic class for cells used in NAS-FPN and NAS-FCOS. + + BaseMergeCell takes 2 inputs. After applying convolution + on them, they are resized to the target size. Then, + they go through binary_op, which depends on the type of cell. + If with_out_conv is True, the result of output will go through + another convolution layer. + + Args: + in_channels (int): number of input channels in out_conv layer. + out_channels (int): number of output channels in out_conv layer. + with_out_conv (bool): Whether to use out_conv layer + out_conv_cfg (dict): Config dict for convolution layer, which should + contain "groups", "kernel_size", "padding", "bias" to build + out_conv layer. + out_norm_cfg (dict): Config dict for normalization layer in out_conv. + out_conv_order (tuple): The order of conv/norm/activation layers in + out_conv. + with_input1_conv (bool): Whether to use convolution on input1. + with_input2_conv (bool): Whether to use convolution on input2. + input_conv_cfg (dict): Config dict for building input1_conv layer and + input2_conv layer, which is expected to contain the type of + convolution. + Default: None, which means using conv2d. + input_norm_cfg (dict): Config dict for normalization layer in + input1_conv and input2_conv layer. Default: None. + upsample_mode (str): Interpolation method used to resize the output + of input1_conv and input2_conv to target size. Currently, we + support ['nearest', 'bilinear']. Default: 'nearest'. + """ + + def __init__(self, + fused_channels=256, + out_channels=256, + with_out_conv=True, + out_conv_cfg=dict( + groups=1, kernel_size=3, padding=1, bias=True), + out_norm_cfg=None, + out_conv_order=('act', 'conv', 'norm'), + with_input1_conv=False, + with_input2_conv=False, + input_conv_cfg=None, + input_norm_cfg=None, + upsample_mode='nearest'): + super(BaseMergeCell, self).__init__() + assert upsample_mode in ['nearest', 'bilinear'] + self.with_out_conv = with_out_conv + self.with_input1_conv = with_input1_conv + self.with_input2_conv = with_input2_conv + self.upsample_mode = upsample_mode + + if self.with_out_conv: + self.out_conv = ConvModule( + fused_channels, + out_channels, + **out_conv_cfg, + norm_cfg=out_norm_cfg, + order=out_conv_order) + + self.input1_conv = self._build_input_conv( + out_channels, input_conv_cfg, + input_norm_cfg) if with_input1_conv else nn.Sequential() + self.input2_conv = self._build_input_conv( + out_channels, input_conv_cfg, + input_norm_cfg) if with_input2_conv else nn.Sequential() + + def _build_input_conv(self, channel, conv_cfg, norm_cfg): + return ConvModule( + channel, + channel, + 3, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + bias=True) + + @abstractmethod + def _binary_op(self, x1, x2): + pass + + def _resize(self, x, size): + if x.shape[-2:] == size: + return x + elif x.shape[-2:] < size: + return F.interpolate(x, size=size, mode=self.upsample_mode) + else: + assert x.shape[-2] % size[-2] == 0 and x.shape[-1] % size[-1] == 0 + kernel_size = x.shape[-1] // size[-1] + x = F.max_pool2d(x, kernel_size=kernel_size, stride=kernel_size) + return x + + def forward(self, x1, x2, out_size=None): + assert x1.shape[:2] == x2.shape[:2] + assert out_size is None or len(out_size) == 2 + if out_size is None: # resize to larger one + out_size = max(x1.size()[2:], x2.size()[2:]) + + x1 = self.input1_conv(x1) + x2 = self.input2_conv(x2) + + x1 = self._resize(x1, out_size) + x2 = self._resize(x2, out_size) + + x = self._binary_op(x1, x2) + if self.with_out_conv: + x = self.out_conv(x) + return x + + +class SumCell(BaseMergeCell): + + def __init__(self, in_channels, out_channels, **kwargs): + super(SumCell, self).__init__(in_channels, out_channels, **kwargs) + + def _binary_op(self, x1, x2): + return x1 + x2 + + +class ConcatCell(BaseMergeCell): + + def __init__(self, in_channels, out_channels, **kwargs): + super(ConcatCell, self).__init__(in_channels * 2, out_channels, + **kwargs) + + def _binary_op(self, x1, x2): + ret = torch.cat([x1, x2], dim=1) + return ret + + +class GlobalPoolingCell(BaseMergeCell): + + def __init__(self, in_channels=None, out_channels=None, **kwargs): + super().__init__(in_channels, out_channels, **kwargs) + self.global_pool = nn.AdaptiveAvgPool2d((1, 1)) + + def _binary_op(self, x1, x2): + x2_att = self.global_pool(x2).sigmoid() + return x2 + x2_att * x1 diff --git a/annotator/mmpkg/mmcv/ops/modulated_deform_conv.py b/annotator/mmpkg/mmcv/ops/modulated_deform_conv.py new file mode 100644 index 0000000000000000000000000000000000000000..f97278361d5262b1a87432dc5e3eb842b39ceb10 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/modulated_deform_conv.py @@ -0,0 +1,282 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import math + +import torch +import torch.nn as nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.utils import _pair, _single + +from annotator.mmpkg.mmcv.utils import deprecated_api_warning +from ..cnn import CONV_LAYERS +from ..utils import ext_loader, print_log + +ext_module = ext_loader.load_ext( + '_ext', + ['modulated_deform_conv_forward', 'modulated_deform_conv_backward']) + + +class ModulatedDeformConv2dFunction(Function): + + @staticmethod + def symbolic(g, input, offset, mask, weight, bias, stride, padding, + dilation, groups, deform_groups): + input_tensors = [input, offset, mask, weight] + if bias is not None: + input_tensors.append(bias) + return g.op( + 'mmcv::MMCVModulatedDeformConv2d', + *input_tensors, + stride_i=stride, + padding_i=padding, + dilation_i=dilation, + groups_i=groups, + deform_groups_i=deform_groups) + + @staticmethod + def forward(ctx, + input, + offset, + mask, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1, + deform_groups=1): + if input is not None and input.dim() != 4: + raise ValueError( + f'Expected 4D tensor as input, got {input.dim()}D tensor \ + instead.') + ctx.stride = _pair(stride) + ctx.padding = _pair(padding) + ctx.dilation = _pair(dilation) + ctx.groups = groups + ctx.deform_groups = deform_groups + ctx.with_bias = bias is not None + if not ctx.with_bias: + bias = input.new_empty(0) # fake tensor + # When pytorch version >= 1.6.0, amp is adopted for fp16 mode; + # amp won't cast the type of model (float32), but "offset" is cast + # to float16 by nn.Conv2d automatically, leading to the type + # mismatch with input (when it is float32) or weight. + # The flag for whether to use fp16 or amp is the type of "offset", + # we cast weight and input to temporarily support fp16 and amp + # whatever the pytorch version is. + input = input.type_as(offset) + weight = weight.type_as(input) + ctx.save_for_backward(input, offset, mask, weight, bias) + output = input.new_empty( + ModulatedDeformConv2dFunction._output_size(ctx, input, weight)) + ctx._bufs = [input.new_empty(0), input.new_empty(0)] + ext_module.modulated_deform_conv_forward( + input, + weight, + bias, + ctx._bufs[0], + offset, + mask, + output, + ctx._bufs[1], + kernel_h=weight.size(2), + kernel_w=weight.size(3), + stride_h=ctx.stride[0], + stride_w=ctx.stride[1], + pad_h=ctx.padding[0], + pad_w=ctx.padding[1], + dilation_h=ctx.dilation[0], + dilation_w=ctx.dilation[1], + group=ctx.groups, + deformable_group=ctx.deform_groups, + with_bias=ctx.with_bias) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + input, offset, mask, weight, bias = ctx.saved_tensors + grad_input = torch.zeros_like(input) + grad_offset = torch.zeros_like(offset) + grad_mask = torch.zeros_like(mask) + grad_weight = torch.zeros_like(weight) + grad_bias = torch.zeros_like(bias) + grad_output = grad_output.contiguous() + ext_module.modulated_deform_conv_backward( + input, + weight, + bias, + ctx._bufs[0], + offset, + mask, + ctx._bufs[1], + grad_input, + grad_weight, + grad_bias, + grad_offset, + grad_mask, + grad_output, + kernel_h=weight.size(2), + kernel_w=weight.size(3), + stride_h=ctx.stride[0], + stride_w=ctx.stride[1], + pad_h=ctx.padding[0], + pad_w=ctx.padding[1], + dilation_h=ctx.dilation[0], + dilation_w=ctx.dilation[1], + group=ctx.groups, + deformable_group=ctx.deform_groups, + with_bias=ctx.with_bias) + if not ctx.with_bias: + grad_bias = None + + return (grad_input, grad_offset, grad_mask, grad_weight, grad_bias, + None, None, None, None, None) + + @staticmethod + def _output_size(ctx, input, weight): + channels = weight.size(0) + output_size = (input.size(0), channels) + for d in range(input.dim() - 2): + in_size = input.size(d + 2) + pad = ctx.padding[d] + kernel = ctx.dilation[d] * (weight.size(d + 2) - 1) + 1 + stride_ = ctx.stride[d] + output_size += ((in_size + (2 * pad) - kernel) // stride_ + 1, ) + if not all(map(lambda s: s > 0, output_size)): + raise ValueError( + 'convolution input is too small (output would be ' + + 'x'.join(map(str, output_size)) + ')') + return output_size + + +modulated_deform_conv2d = ModulatedDeformConv2dFunction.apply + + +class ModulatedDeformConv2d(nn.Module): + + @deprecated_api_warning({'deformable_groups': 'deform_groups'}, + cls_name='ModulatedDeformConv2d') + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + deform_groups=1, + bias=True): + super(ModulatedDeformConv2d, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = _pair(stride) + self.padding = _pair(padding) + self.dilation = _pair(dilation) + self.groups = groups + self.deform_groups = deform_groups + # enable compatibility with nn.Conv2d + self.transposed = False + self.output_padding = _single(0) + + self.weight = nn.Parameter( + torch.Tensor(out_channels, in_channels // groups, + *self.kernel_size)) + if bias: + self.bias = nn.Parameter(torch.Tensor(out_channels)) + else: + self.register_parameter('bias', None) + self.init_weights() + + def init_weights(self): + n = self.in_channels + for k in self.kernel_size: + n *= k + stdv = 1. / math.sqrt(n) + self.weight.data.uniform_(-stdv, stdv) + if self.bias is not None: + self.bias.data.zero_() + + def forward(self, x, offset, mask): + return modulated_deform_conv2d(x, offset, mask, self.weight, self.bias, + self.stride, self.padding, + self.dilation, self.groups, + self.deform_groups) + + +@CONV_LAYERS.register_module('DCNv2') +class ModulatedDeformConv2dPack(ModulatedDeformConv2d): + """A ModulatedDeformable Conv Encapsulation that acts as normal Conv + layers. + + Args: + in_channels (int): Same as nn.Conv2d. + out_channels (int): Same as nn.Conv2d. + kernel_size (int or tuple[int]): Same as nn.Conv2d. + stride (int): Same as nn.Conv2d, while tuple is not supported. + padding (int): Same as nn.Conv2d, while tuple is not supported. + dilation (int): Same as nn.Conv2d, while tuple is not supported. + groups (int): Same as nn.Conv2d. + bias (bool or str): If specified as `auto`, it will be decided by the + norm_cfg. Bias will be set as True if norm_cfg is None, otherwise + False. + """ + + _version = 2 + + def __init__(self, *args, **kwargs): + super(ModulatedDeformConv2dPack, self).__init__(*args, **kwargs) + self.conv_offset = nn.Conv2d( + self.in_channels, + self.deform_groups * 3 * self.kernel_size[0] * self.kernel_size[1], + kernel_size=self.kernel_size, + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + bias=True) + self.init_weights() + + def init_weights(self): + super(ModulatedDeformConv2dPack, self).init_weights() + if hasattr(self, 'conv_offset'): + self.conv_offset.weight.data.zero_() + self.conv_offset.bias.data.zero_() + + def forward(self, x): + out = self.conv_offset(x) + o1, o2, mask = torch.chunk(out, 3, dim=1) + offset = torch.cat((o1, o2), dim=1) + mask = torch.sigmoid(mask) + return modulated_deform_conv2d(x, offset, mask, self.weight, self.bias, + self.stride, self.padding, + self.dilation, self.groups, + self.deform_groups) + + def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs): + version = local_metadata.get('version', None) + + if version is None or version < 2: + # the key is different in early versions + # In version < 2, ModulatedDeformConvPack + # loads previous benchmark models. + if (prefix + 'conv_offset.weight' not in state_dict + and prefix[:-1] + '_offset.weight' in state_dict): + state_dict[prefix + 'conv_offset.weight'] = state_dict.pop( + prefix[:-1] + '_offset.weight') + if (prefix + 'conv_offset.bias' not in state_dict + and prefix[:-1] + '_offset.bias' in state_dict): + state_dict[prefix + + 'conv_offset.bias'] = state_dict.pop(prefix[:-1] + + '_offset.bias') + + if version is not None and version > 1: + print_log( + f'ModulatedDeformConvPack {prefix.rstrip(".")} is upgraded to ' + 'version 2.', + logger='root') + + super()._load_from_state_dict(state_dict, prefix, local_metadata, + strict, missing_keys, unexpected_keys, + error_msgs) diff --git a/annotator/mmpkg/mmcv/ops/multi_scale_deform_attn.py b/annotator/mmpkg/mmcv/ops/multi_scale_deform_attn.py new file mode 100644 index 0000000000000000000000000000000000000000..fe755eaa931565aab77ecc387990328c01447343 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/multi_scale_deform_attn.py @@ -0,0 +1,358 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import math +import warnings + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd.function import Function, once_differentiable + +from annotator.mmpkg.mmcv import deprecated_api_warning +from annotator.mmpkg.mmcv.cnn import constant_init, xavier_init +from annotator.mmpkg.mmcv.cnn.bricks.registry import ATTENTION +from annotator.mmpkg.mmcv.runner import BaseModule +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['ms_deform_attn_backward', 'ms_deform_attn_forward']) + + +class MultiScaleDeformableAttnFunction(Function): + + @staticmethod + def forward(ctx, value, value_spatial_shapes, value_level_start_index, + sampling_locations, attention_weights, im2col_step): + """GPU version of multi-scale deformable attention. + + Args: + value (Tensor): The value has shape + (bs, num_keys, mum_heads, embed_dims//num_heads) + value_spatial_shapes (Tensor): Spatial shape of + each feature map, has shape (num_levels, 2), + last dimension 2 represent (h, w) + sampling_locations (Tensor): The location of sampling points, + has shape + (bs ,num_queries, num_heads, num_levels, num_points, 2), + the last dimension 2 represent (x, y). + attention_weights (Tensor): The weight of sampling points used + when calculate the attention, has shape + (bs ,num_queries, num_heads, num_levels, num_points), + im2col_step (Tensor): The step used in image to column. + + Returns: + Tensor: has shape (bs, num_queries, embed_dims) + """ + + ctx.im2col_step = im2col_step + output = ext_module.ms_deform_attn_forward( + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + im2col_step=ctx.im2col_step) + ctx.save_for_backward(value, value_spatial_shapes, + value_level_start_index, sampling_locations, + attention_weights) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + """GPU version of backward function. + + Args: + grad_output (Tensor): Gradient + of output tensor of forward. + + Returns: + Tuple[Tensor]: Gradient + of input tensors in forward. + """ + value, value_spatial_shapes, value_level_start_index,\ + sampling_locations, attention_weights = ctx.saved_tensors + grad_value = torch.zeros_like(value) + grad_sampling_loc = torch.zeros_like(sampling_locations) + grad_attn_weight = torch.zeros_like(attention_weights) + + ext_module.ms_deform_attn_backward( + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + grad_output.contiguous(), + grad_value, + grad_sampling_loc, + grad_attn_weight, + im2col_step=ctx.im2col_step) + + return grad_value, None, None, \ + grad_sampling_loc, grad_attn_weight, None + + +def multi_scale_deformable_attn_pytorch(value, value_spatial_shapes, + sampling_locations, attention_weights): + """CPU version of multi-scale deformable attention. + + Args: + value (Tensor): The value has shape + (bs, num_keys, mum_heads, embed_dims//num_heads) + value_spatial_shapes (Tensor): Spatial shape of + each feature map, has shape (num_levels, 2), + last dimension 2 represent (h, w) + sampling_locations (Tensor): The location of sampling points, + has shape + (bs ,num_queries, num_heads, num_levels, num_points, 2), + the last dimension 2 represent (x, y). + attention_weights (Tensor): The weight of sampling points used + when calculate the attention, has shape + (bs ,num_queries, num_heads, num_levels, num_points), + + Returns: + Tensor: has shape (bs, num_queries, embed_dims) + """ + + bs, _, num_heads, embed_dims = value.shape + _, num_queries, num_heads, num_levels, num_points, _ =\ + sampling_locations.shape + value_list = value.split([H_ * W_ for H_, W_ in value_spatial_shapes], + dim=1) + sampling_grids = 2 * sampling_locations - 1 + sampling_value_list = [] + for level, (H_, W_) in enumerate(value_spatial_shapes): + # bs, H_*W_, num_heads, embed_dims -> + # bs, H_*W_, num_heads*embed_dims -> + # bs, num_heads*embed_dims, H_*W_ -> + # bs*num_heads, embed_dims, H_, W_ + value_l_ = value_list[level].flatten(2).transpose(1, 2).reshape( + bs * num_heads, embed_dims, H_, W_) + # bs, num_queries, num_heads, num_points, 2 -> + # bs, num_heads, num_queries, num_points, 2 -> + # bs*num_heads, num_queries, num_points, 2 + sampling_grid_l_ = sampling_grids[:, :, :, + level].transpose(1, 2).flatten(0, 1) + # bs*num_heads, embed_dims, num_queries, num_points + sampling_value_l_ = F.grid_sample( + value_l_, + sampling_grid_l_, + mode='bilinear', + padding_mode='zeros', + align_corners=False) + sampling_value_list.append(sampling_value_l_) + # (bs, num_queries, num_heads, num_levels, num_points) -> + # (bs, num_heads, num_queries, num_levels, num_points) -> + # (bs, num_heads, 1, num_queries, num_levels*num_points) + attention_weights = attention_weights.transpose(1, 2).reshape( + bs * num_heads, 1, num_queries, num_levels * num_points) + output = (torch.stack(sampling_value_list, dim=-2).flatten(-2) * + attention_weights).sum(-1).view(bs, num_heads * embed_dims, + num_queries) + return output.transpose(1, 2).contiguous() + + +@ATTENTION.register_module() +class MultiScaleDeformableAttention(BaseModule): + """An attention module used in Deformable-Detr. + + `Deformable DETR: Deformable Transformers for End-to-End Object Detection. + `_. + + Args: + embed_dims (int): The embedding dimension of Attention. + Default: 256. + num_heads (int): Parallel attention heads. Default: 64. + num_levels (int): The number of feature map used in + Attention. Default: 4. + num_points (int): The number of sampling points for + each query in each head. Default: 4. + im2col_step (int): The step used in image_to_column. + Default: 64. + dropout (float): A Dropout layer on `inp_identity`. + Default: 0.1. + batch_first (bool): Key, Query and Value are shape of + (batch, n, embed_dim) + or (n, batch, embed_dim). Default to False. + norm_cfg (dict): Config dict for normalization layer. + Default: None. + init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. + Default: None. + """ + + def __init__(self, + embed_dims=256, + num_heads=8, + num_levels=4, + num_points=4, + im2col_step=64, + dropout=0.1, + batch_first=False, + norm_cfg=None, + init_cfg=None): + super().__init__(init_cfg) + if embed_dims % num_heads != 0: + raise ValueError(f'embed_dims must be divisible by num_heads, ' + f'but got {embed_dims} and {num_heads}') + dim_per_head = embed_dims // num_heads + self.norm_cfg = norm_cfg + self.dropout = nn.Dropout(dropout) + self.batch_first = batch_first + + # you'd better set dim_per_head to a power of 2 + # which is more efficient in the CUDA implementation + def _is_power_of_2(n): + if (not isinstance(n, int)) or (n < 0): + raise ValueError( + 'invalid input for _is_power_of_2: {} (type: {})'.format( + n, type(n))) + return (n & (n - 1) == 0) and n != 0 + + if not _is_power_of_2(dim_per_head): + warnings.warn( + "You'd better set embed_dims in " + 'MultiScaleDeformAttention to make ' + 'the dimension of each attention head a power of 2 ' + 'which is more efficient in our CUDA implementation.') + + self.im2col_step = im2col_step + self.embed_dims = embed_dims + self.num_levels = num_levels + self.num_heads = num_heads + self.num_points = num_points + self.sampling_offsets = nn.Linear( + embed_dims, num_heads * num_levels * num_points * 2) + self.attention_weights = nn.Linear(embed_dims, + num_heads * num_levels * num_points) + self.value_proj = nn.Linear(embed_dims, embed_dims) + self.output_proj = nn.Linear(embed_dims, embed_dims) + self.init_weights() + + def init_weights(self): + """Default initialization for Parameters of Module.""" + constant_init(self.sampling_offsets, 0.) + thetas = torch.arange( + self.num_heads, + dtype=torch.float32) * (2.0 * math.pi / self.num_heads) + grid_init = torch.stack([thetas.cos(), thetas.sin()], -1) + grid_init = (grid_init / + grid_init.abs().max(-1, keepdim=True)[0]).view( + self.num_heads, 1, 1, + 2).repeat(1, self.num_levels, self.num_points, 1) + for i in range(self.num_points): + grid_init[:, :, i, :] *= i + 1 + + self.sampling_offsets.bias.data = grid_init.view(-1) + constant_init(self.attention_weights, val=0., bias=0.) + xavier_init(self.value_proj, distribution='uniform', bias=0.) + xavier_init(self.output_proj, distribution='uniform', bias=0.) + self._is_init = True + + @deprecated_api_warning({'residual': 'identity'}, + cls_name='MultiScaleDeformableAttention') + def forward(self, + query, + key=None, + value=None, + identity=None, + query_pos=None, + key_padding_mask=None, + reference_points=None, + spatial_shapes=None, + level_start_index=None, + **kwargs): + """Forward Function of MultiScaleDeformAttention. + + Args: + query (Tensor): Query of Transformer with shape + (num_query, bs, embed_dims). + key (Tensor): The key tensor with shape + `(num_key, bs, embed_dims)`. + value (Tensor): The value tensor with shape + `(num_key, bs, embed_dims)`. + identity (Tensor): The tensor used for addition, with the + same shape as `query`. Default None. If None, + `query` will be used. + query_pos (Tensor): The positional encoding for `query`. + Default: None. + key_pos (Tensor): The positional encoding for `key`. Default + None. + reference_points (Tensor): The normalized reference + points with shape (bs, num_query, num_levels, 2), + all elements is range in [0, 1], top-left (0,0), + bottom-right (1, 1), including padding area. + or (N, Length_{query}, num_levels, 4), add + additional two dimensions is (w, h) to + form reference boxes. + key_padding_mask (Tensor): ByteTensor for `query`, with + shape [bs, num_key]. + spatial_shapes (Tensor): Spatial shape of features in + different levels. With shape (num_levels, 2), + last dimension represents (h, w). + level_start_index (Tensor): The start index of each level. + A tensor has shape ``(num_levels, )`` and can be represented + as [0, h_0*w_0, h_0*w_0+h_1*w_1, ...]. + + Returns: + Tensor: forwarded results with shape [num_query, bs, embed_dims]. + """ + + if value is None: + value = query + + if identity is None: + identity = query + if query_pos is not None: + query = query + query_pos + if not self.batch_first: + # change to (bs, num_query ,embed_dims) + query = query.permute(1, 0, 2) + value = value.permute(1, 0, 2) + + bs, num_query, _ = query.shape + bs, num_value, _ = value.shape + assert (spatial_shapes[:, 0] * spatial_shapes[:, 1]).sum() == num_value + + value = self.value_proj(value) + if key_padding_mask is not None: + value = value.masked_fill(key_padding_mask[..., None], 0.0) + value = value.view(bs, num_value, self.num_heads, -1) + sampling_offsets = self.sampling_offsets(query).view( + bs, num_query, self.num_heads, self.num_levels, self.num_points, 2) + attention_weights = self.attention_weights(query).view( + bs, num_query, self.num_heads, self.num_levels * self.num_points) + attention_weights = attention_weights.softmax(-1) + + attention_weights = attention_weights.view(bs, num_query, + self.num_heads, + self.num_levels, + self.num_points) + if reference_points.shape[-1] == 2: + offset_normalizer = torch.stack( + [spatial_shapes[..., 1], spatial_shapes[..., 0]], -1) + sampling_locations = reference_points[:, :, None, :, None, :] \ + + sampling_offsets \ + / offset_normalizer[None, None, None, :, None, :] + elif reference_points.shape[-1] == 4: + sampling_locations = reference_points[:, :, None, :, None, :2] \ + + sampling_offsets / self.num_points \ + * reference_points[:, :, None, :, None, 2:] \ + * 0.5 + else: + raise ValueError( + f'Last dim of reference_points must be' + f' 2 or 4, but get {reference_points.shape[-1]} instead.') + if torch.cuda.is_available() and value.is_cuda: + output = MultiScaleDeformableAttnFunction.apply( + value, spatial_shapes, level_start_index, sampling_locations, + attention_weights, self.im2col_step) + else: + output = multi_scale_deformable_attn_pytorch( + value, spatial_shapes, sampling_locations, attention_weights) + + output = self.output_proj(output) + + if not self.batch_first: + # (num_query, bs ,embed_dims) + output = output.permute(1, 0, 2) + + return self.dropout(output) + identity diff --git a/annotator/mmpkg/mmcv/ops/nms.py b/annotator/mmpkg/mmcv/ops/nms.py new file mode 100644 index 0000000000000000000000000000000000000000..908ac66645eef29fb55fce82497eb9f6af1a2667 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/nms.py @@ -0,0 +1,417 @@ +import os + +import numpy as np +import torch + +from annotator.mmpkg.mmcv.utils import deprecated_api_warning +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['nms', 'softnms', 'nms_match', 'nms_rotated']) + + +# This function is modified from: https://github.com/pytorch/vision/ +class NMSop(torch.autograd.Function): + + @staticmethod + def forward(ctx, bboxes, scores, iou_threshold, offset, score_threshold, + max_num): + is_filtering_by_score = score_threshold > 0 + if is_filtering_by_score: + valid_mask = scores > score_threshold + bboxes, scores = bboxes[valid_mask], scores[valid_mask] + valid_inds = torch.nonzero( + valid_mask, as_tuple=False).squeeze(dim=1) + + inds = ext_module.nms( + bboxes, scores, iou_threshold=float(iou_threshold), offset=offset) + + if max_num > 0: + inds = inds[:max_num] + if is_filtering_by_score: + inds = valid_inds[inds] + return inds + + @staticmethod + def symbolic(g, bboxes, scores, iou_threshold, offset, score_threshold, + max_num): + from ..onnx import is_custom_op_loaded + has_custom_op = is_custom_op_loaded() + # TensorRT nms plugin is aligned with original nms in ONNXRuntime + is_trt_backend = os.environ.get('ONNX_BACKEND') == 'MMCVTensorRT' + if has_custom_op and (not is_trt_backend): + return g.op( + 'mmcv::NonMaxSuppression', + bboxes, + scores, + iou_threshold_f=float(iou_threshold), + offset_i=int(offset)) + else: + from torch.onnx.symbolic_opset9 import select, squeeze, unsqueeze + from ..onnx.onnx_utils.symbolic_helper import _size_helper + + boxes = unsqueeze(g, bboxes, 0) + scores = unsqueeze(g, unsqueeze(g, scores, 0), 0) + + if max_num > 0: + max_num = g.op( + 'Constant', + value_t=torch.tensor(max_num, dtype=torch.long)) + else: + dim = g.op('Constant', value_t=torch.tensor(0)) + max_num = _size_helper(g, bboxes, dim) + max_output_per_class = max_num + iou_threshold = g.op( + 'Constant', + value_t=torch.tensor([iou_threshold], dtype=torch.float)) + score_threshold = g.op( + 'Constant', + value_t=torch.tensor([score_threshold], dtype=torch.float)) + nms_out = g.op('NonMaxSuppression', boxes, scores, + max_output_per_class, iou_threshold, + score_threshold) + return squeeze( + g, + select( + g, nms_out, 1, + g.op( + 'Constant', + value_t=torch.tensor([2], dtype=torch.long))), 1) + + +class SoftNMSop(torch.autograd.Function): + + @staticmethod + def forward(ctx, boxes, scores, iou_threshold, sigma, min_score, method, + offset): + dets = boxes.new_empty((boxes.size(0), 5), device='cpu') + inds = ext_module.softnms( + boxes.cpu(), + scores.cpu(), + dets.cpu(), + iou_threshold=float(iou_threshold), + sigma=float(sigma), + min_score=float(min_score), + method=int(method), + offset=int(offset)) + return dets, inds + + @staticmethod + def symbolic(g, boxes, scores, iou_threshold, sigma, min_score, method, + offset): + from packaging import version + assert version.parse(torch.__version__) >= version.parse('1.7.0') + nms_out = g.op( + 'mmcv::SoftNonMaxSuppression', + boxes, + scores, + iou_threshold_f=float(iou_threshold), + sigma_f=float(sigma), + min_score_f=float(min_score), + method_i=int(method), + offset_i=int(offset), + outputs=2) + return nms_out + + +@deprecated_api_warning({'iou_thr': 'iou_threshold'}) +def nms(boxes, scores, iou_threshold, offset=0, score_threshold=0, max_num=-1): + """Dispatch to either CPU or GPU NMS implementations. + + The input can be either torch tensor or numpy array. GPU NMS will be used + if the input is gpu tensor, otherwise CPU NMS + will be used. The returned type will always be the same as inputs. + + Arguments: + boxes (torch.Tensor or np.ndarray): boxes in shape (N, 4). + scores (torch.Tensor or np.ndarray): scores in shape (N, ). + iou_threshold (float): IoU threshold for NMS. + offset (int, 0 or 1): boxes' width or height is (x2 - x1 + offset). + score_threshold (float): score threshold for NMS. + max_num (int): maximum number of boxes after NMS. + + Returns: + tuple: kept dets(boxes and scores) and indice, which is always the \ + same data type as the input. + + Example: + >>> boxes = np.array([[49.1, 32.4, 51.0, 35.9], + >>> [49.3, 32.9, 51.0, 35.3], + >>> [49.2, 31.8, 51.0, 35.4], + >>> [35.1, 11.5, 39.1, 15.7], + >>> [35.6, 11.8, 39.3, 14.2], + >>> [35.3, 11.5, 39.9, 14.5], + >>> [35.2, 11.7, 39.7, 15.7]], dtype=np.float32) + >>> scores = np.array([0.9, 0.9, 0.5, 0.5, 0.5, 0.4, 0.3],\ + dtype=np.float32) + >>> iou_threshold = 0.6 + >>> dets, inds = nms(boxes, scores, iou_threshold) + >>> assert len(inds) == len(dets) == 3 + """ + assert isinstance(boxes, (torch.Tensor, np.ndarray)) + assert isinstance(scores, (torch.Tensor, np.ndarray)) + is_numpy = False + if isinstance(boxes, np.ndarray): + is_numpy = True + boxes = torch.from_numpy(boxes) + if isinstance(scores, np.ndarray): + scores = torch.from_numpy(scores) + assert boxes.size(1) == 4 + assert boxes.size(0) == scores.size(0) + assert offset in (0, 1) + + if torch.__version__ == 'parrots': + indata_list = [boxes, scores] + indata_dict = { + 'iou_threshold': float(iou_threshold), + 'offset': int(offset) + } + inds = ext_module.nms(*indata_list, **indata_dict) + else: + inds = NMSop.apply(boxes, scores, iou_threshold, offset, + score_threshold, max_num) + dets = torch.cat((boxes[inds], scores[inds].reshape(-1, 1)), dim=1) + if is_numpy: + dets = dets.cpu().numpy() + inds = inds.cpu().numpy() + return dets, inds + + +@deprecated_api_warning({'iou_thr': 'iou_threshold'}) +def soft_nms(boxes, + scores, + iou_threshold=0.3, + sigma=0.5, + min_score=1e-3, + method='linear', + offset=0): + """Dispatch to only CPU Soft NMS implementations. + + The input can be either a torch tensor or numpy array. + The returned type will always be the same as inputs. + + Arguments: + boxes (torch.Tensor or np.ndarray): boxes in shape (N, 4). + scores (torch.Tensor or np.ndarray): scores in shape (N, ). + iou_threshold (float): IoU threshold for NMS. + sigma (float): hyperparameter for gaussian method + min_score (float): score filter threshold + method (str): either 'linear' or 'gaussian' + offset (int, 0 or 1): boxes' width or height is (x2 - x1 + offset). + + Returns: + tuple: kept dets(boxes and scores) and indice, which is always the \ + same data type as the input. + + Example: + >>> boxes = np.array([[4., 3., 5., 3.], + >>> [4., 3., 5., 4.], + >>> [3., 1., 3., 1.], + >>> [3., 1., 3., 1.], + >>> [3., 1., 3., 1.], + >>> [3., 1., 3., 1.]], dtype=np.float32) + >>> scores = np.array([0.9, 0.9, 0.5, 0.5, 0.4, 0.0], dtype=np.float32) + >>> iou_threshold = 0.6 + >>> dets, inds = soft_nms(boxes, scores, iou_threshold, sigma=0.5) + >>> assert len(inds) == len(dets) == 5 + """ + + assert isinstance(boxes, (torch.Tensor, np.ndarray)) + assert isinstance(scores, (torch.Tensor, np.ndarray)) + is_numpy = False + if isinstance(boxes, np.ndarray): + is_numpy = True + boxes = torch.from_numpy(boxes) + if isinstance(scores, np.ndarray): + scores = torch.from_numpy(scores) + assert boxes.size(1) == 4 + assert boxes.size(0) == scores.size(0) + assert offset in (0, 1) + method_dict = {'naive': 0, 'linear': 1, 'gaussian': 2} + assert method in method_dict.keys() + + if torch.__version__ == 'parrots': + dets = boxes.new_empty((boxes.size(0), 5), device='cpu') + indata_list = [boxes.cpu(), scores.cpu(), dets.cpu()] + indata_dict = { + 'iou_threshold': float(iou_threshold), + 'sigma': float(sigma), + 'min_score': min_score, + 'method': method_dict[method], + 'offset': int(offset) + } + inds = ext_module.softnms(*indata_list, **indata_dict) + else: + dets, inds = SoftNMSop.apply(boxes.cpu(), scores.cpu(), + float(iou_threshold), float(sigma), + float(min_score), method_dict[method], + int(offset)) + + dets = dets[:inds.size(0)] + + if is_numpy: + dets = dets.cpu().numpy() + inds = inds.cpu().numpy() + return dets, inds + else: + return dets.to(device=boxes.device), inds.to(device=boxes.device) + + +def batched_nms(boxes, scores, idxs, nms_cfg, class_agnostic=False): + """Performs non-maximum suppression in a batched fashion. + + Modified from https://github.com/pytorch/vision/blob + /505cd6957711af790211896d32b40291bea1bc21/torchvision/ops/boxes.py#L39. + In order to perform NMS independently per class, we add an offset to all + the boxes. The offset is dependent only on the class idx, and is large + enough so that boxes from different classes do not overlap. + + Arguments: + boxes (torch.Tensor): boxes in shape (N, 4). + scores (torch.Tensor): scores in shape (N, ). + idxs (torch.Tensor): each index value correspond to a bbox cluster, + and NMS will not be applied between elements of different idxs, + shape (N, ). + nms_cfg (dict): specify nms type and other parameters like iou_thr. + Possible keys includes the following. + + - iou_thr (float): IoU threshold used for NMS. + - split_thr (float): threshold number of boxes. In some cases the + number of boxes is large (e.g., 200k). To avoid OOM during + training, the users could set `split_thr` to a small value. + If the number of boxes is greater than the threshold, it will + perform NMS on each group of boxes separately and sequentially. + Defaults to 10000. + class_agnostic (bool): if true, nms is class agnostic, + i.e. IoU thresholding happens over all boxes, + regardless of the predicted class. + + Returns: + tuple: kept dets and indice. + """ + nms_cfg_ = nms_cfg.copy() + class_agnostic = nms_cfg_.pop('class_agnostic', class_agnostic) + if class_agnostic: + boxes_for_nms = boxes + else: + max_coordinate = boxes.max() + offsets = idxs.to(boxes) * (max_coordinate + torch.tensor(1).to(boxes)) + boxes_for_nms = boxes + offsets[:, None] + + nms_type = nms_cfg_.pop('type', 'nms') + nms_op = eval(nms_type) + + split_thr = nms_cfg_.pop('split_thr', 10000) + # Won't split to multiple nms nodes when exporting to onnx + if boxes_for_nms.shape[0] < split_thr or torch.onnx.is_in_onnx_export(): + dets, keep = nms_op(boxes_for_nms, scores, **nms_cfg_) + boxes = boxes[keep] + # -1 indexing works abnormal in TensorRT + # This assumes `dets` has 5 dimensions where + # the last dimension is score. + # TODO: more elegant way to handle the dimension issue. + # Some type of nms would reweight the score, such as SoftNMS + scores = dets[:, 4] + else: + max_num = nms_cfg_.pop('max_num', -1) + total_mask = scores.new_zeros(scores.size(), dtype=torch.bool) + # Some type of nms would reweight the score, such as SoftNMS + scores_after_nms = scores.new_zeros(scores.size()) + for id in torch.unique(idxs): + mask = (idxs == id).nonzero(as_tuple=False).view(-1) + dets, keep = nms_op(boxes_for_nms[mask], scores[mask], **nms_cfg_) + total_mask[mask[keep]] = True + scores_after_nms[mask[keep]] = dets[:, -1] + keep = total_mask.nonzero(as_tuple=False).view(-1) + + scores, inds = scores_after_nms[keep].sort(descending=True) + keep = keep[inds] + boxes = boxes[keep] + + if max_num > 0: + keep = keep[:max_num] + boxes = boxes[:max_num] + scores = scores[:max_num] + + return torch.cat([boxes, scores[:, None]], -1), keep + + +def nms_match(dets, iou_threshold): + """Matched dets into different groups by NMS. + + NMS match is Similar to NMS but when a bbox is suppressed, nms match will + record the indice of suppressed bbox and form a group with the indice of + kept bbox. In each group, indice is sorted as score order. + + Arguments: + dets (torch.Tensor | np.ndarray): Det boxes with scores, shape (N, 5). + iou_thr (float): IoU thresh for NMS. + + Returns: + List[torch.Tensor | np.ndarray]: The outer list corresponds different + matched group, the inner Tensor corresponds the indices for a group + in score order. + """ + if dets.shape[0] == 0: + matched = [] + else: + assert dets.shape[-1] == 5, 'inputs dets.shape should be (N, 5), ' \ + f'but get {dets.shape}' + if isinstance(dets, torch.Tensor): + dets_t = dets.detach().cpu() + else: + dets_t = torch.from_numpy(dets) + indata_list = [dets_t] + indata_dict = {'iou_threshold': float(iou_threshold)} + matched = ext_module.nms_match(*indata_list, **indata_dict) + if torch.__version__ == 'parrots': + matched = matched.tolist() + + if isinstance(dets, torch.Tensor): + return [dets.new_tensor(m, dtype=torch.long) for m in matched] + else: + return [np.array(m, dtype=np.int) for m in matched] + + +def nms_rotated(dets, scores, iou_threshold, labels=None): + """Performs non-maximum suppression (NMS) on the rotated boxes according to + their intersection-over-union (IoU). + + Rotated NMS iteratively removes lower scoring rotated boxes which have an + IoU greater than iou_threshold with another (higher scoring) rotated box. + + Args: + boxes (Tensor): Rotated boxes in shape (N, 5). They are expected to \ + be in (x_ctr, y_ctr, width, height, angle_radian) format. + scores (Tensor): scores in shape (N, ). + iou_threshold (float): IoU thresh for NMS. + labels (Tensor): boxes' label in shape (N,). + + Returns: + tuple: kept dets(boxes and scores) and indice, which is always the \ + same data type as the input. + """ + if dets.shape[0] == 0: + return dets, None + multi_label = labels is not None + if multi_label: + dets_wl = torch.cat((dets, labels.unsqueeze(1)), 1) + else: + dets_wl = dets + _, order = scores.sort(0, descending=True) + dets_sorted = dets_wl.index_select(0, order) + + if torch.__version__ == 'parrots': + keep_inds = ext_module.nms_rotated( + dets_wl, + scores, + order, + dets_sorted, + iou_threshold=iou_threshold, + multi_label=multi_label) + else: + keep_inds = ext_module.nms_rotated(dets_wl, scores, order, dets_sorted, + iou_threshold, multi_label) + dets = torch.cat((dets[keep_inds], scores[keep_inds].reshape(-1, 1)), + dim=1) + return dets, keep_inds diff --git a/annotator/mmpkg/mmcv/ops/pixel_group.py b/annotator/mmpkg/mmcv/ops/pixel_group.py new file mode 100644 index 0000000000000000000000000000000000000000..2143c75f835a467c802fc3c37ecd3ac0f85bcda4 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/pixel_group.py @@ -0,0 +1,75 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numpy as np +import torch + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', ['pixel_group']) + + +def pixel_group(score, mask, embedding, kernel_label, kernel_contour, + kernel_region_num, distance_threshold): + """Group pixels into text instances, which is widely used text detection + methods. + + Arguments: + score (np.array or Tensor): The foreground score with size hxw. + mask (np.array or Tensor): The foreground mask with size hxw. + embedding (np.array or Tensor): The embedding with size hxwxc to + distinguish instances. + kernel_label (np.array or Tensor): The instance kernel index with + size hxw. + kernel_contour (np.array or Tensor): The kernel contour with size hxw. + kernel_region_num (int): The instance kernel region number. + distance_threshold (float): The embedding distance threshold between + kernel and pixel in one instance. + + Returns: + pixel_assignment (List[List[float]]): The instance coordinate list. + Each element consists of averaged confidence, pixel number, and + coordinates (x_i, y_i for all pixels) in order. + """ + assert isinstance(score, (torch.Tensor, np.ndarray)) + assert isinstance(mask, (torch.Tensor, np.ndarray)) + assert isinstance(embedding, (torch.Tensor, np.ndarray)) + assert isinstance(kernel_label, (torch.Tensor, np.ndarray)) + assert isinstance(kernel_contour, (torch.Tensor, np.ndarray)) + assert isinstance(kernel_region_num, int) + assert isinstance(distance_threshold, float) + + if isinstance(score, np.ndarray): + score = torch.from_numpy(score) + if isinstance(mask, np.ndarray): + mask = torch.from_numpy(mask) + if isinstance(embedding, np.ndarray): + embedding = torch.from_numpy(embedding) + if isinstance(kernel_label, np.ndarray): + kernel_label = torch.from_numpy(kernel_label) + if isinstance(kernel_contour, np.ndarray): + kernel_contour = torch.from_numpy(kernel_contour) + + if torch.__version__ == 'parrots': + label = ext_module.pixel_group( + score, + mask, + embedding, + kernel_label, + kernel_contour, + kernel_region_num=kernel_region_num, + distance_threshold=distance_threshold) + label = label.tolist() + label = label[0] + list_index = kernel_region_num + pixel_assignment = [] + for x in range(kernel_region_num): + pixel_assignment.append( + np.array( + label[list_index:list_index + int(label[x])], + dtype=np.float)) + list_index = list_index + int(label[x]) + else: + pixel_assignment = ext_module.pixel_group(score, mask, embedding, + kernel_label, kernel_contour, + kernel_region_num, + distance_threshold) + return pixel_assignment diff --git a/annotator/mmpkg/mmcv/ops/point_sample.py b/annotator/mmpkg/mmcv/ops/point_sample.py new file mode 100644 index 0000000000000000000000000000000000000000..08b1617805fa84e1c8afc61f3263b4b86bd2a136 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/point_sample.py @@ -0,0 +1,336 @@ +# Modified from https://github.com/facebookresearch/detectron2/tree/master/projects/PointRend # noqa + +from os import path as osp + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.modules.utils import _pair +from torch.onnx.operators import shape_as_tensor + + +def bilinear_grid_sample(im, grid, align_corners=False): + """Given an input and a flow-field grid, computes the output using input + values and pixel locations from grid. Supported only bilinear interpolation + method to sample the input pixels. + + Args: + im (torch.Tensor): Input feature map, shape (N, C, H, W) + grid (torch.Tensor): Point coordinates, shape (N, Hg, Wg, 2) + align_corners {bool}: If set to True, the extrema (-1 and 1) are + considered as referring to the center points of the input’s + corner pixels. If set to False, they are instead considered as + referring to the corner points of the input’s corner pixels, + making the sampling more resolution agnostic. + Returns: + torch.Tensor: A tensor with sampled points, shape (N, C, Hg, Wg) + """ + n, c, h, w = im.shape + gn, gh, gw, _ = grid.shape + assert n == gn + + x = grid[:, :, :, 0] + y = grid[:, :, :, 1] + + if align_corners: + x = ((x + 1) / 2) * (w - 1) + y = ((y + 1) / 2) * (h - 1) + else: + x = ((x + 1) * w - 1) / 2 + y = ((y + 1) * h - 1) / 2 + + x = x.view(n, -1) + y = y.view(n, -1) + + x0 = torch.floor(x).long() + y0 = torch.floor(y).long() + x1 = x0 + 1 + y1 = y0 + 1 + + wa = ((x1 - x) * (y1 - y)).unsqueeze(1) + wb = ((x1 - x) * (y - y0)).unsqueeze(1) + wc = ((x - x0) * (y1 - y)).unsqueeze(1) + wd = ((x - x0) * (y - y0)).unsqueeze(1) + + # Apply default for grid_sample function zero padding + im_padded = F.pad(im, pad=[1, 1, 1, 1], mode='constant', value=0) + padded_h = h + 2 + padded_w = w + 2 + # save points positions after padding + x0, x1, y0, y1 = x0 + 1, x1 + 1, y0 + 1, y1 + 1 + + # Clip coordinates to padded image size + x0 = torch.where(x0 < 0, torch.tensor(0), x0) + x0 = torch.where(x0 > padded_w - 1, torch.tensor(padded_w - 1), x0) + x1 = torch.where(x1 < 0, torch.tensor(0), x1) + x1 = torch.where(x1 > padded_w - 1, torch.tensor(padded_w - 1), x1) + y0 = torch.where(y0 < 0, torch.tensor(0), y0) + y0 = torch.where(y0 > padded_h - 1, torch.tensor(padded_h - 1), y0) + y1 = torch.where(y1 < 0, torch.tensor(0), y1) + y1 = torch.where(y1 > padded_h - 1, torch.tensor(padded_h - 1), y1) + + im_padded = im_padded.view(n, c, -1) + + x0_y0 = (x0 + y0 * padded_w).unsqueeze(1).expand(-1, c, -1) + x0_y1 = (x0 + y1 * padded_w).unsqueeze(1).expand(-1, c, -1) + x1_y0 = (x1 + y0 * padded_w).unsqueeze(1).expand(-1, c, -1) + x1_y1 = (x1 + y1 * padded_w).unsqueeze(1).expand(-1, c, -1) + + Ia = torch.gather(im_padded, 2, x0_y0) + Ib = torch.gather(im_padded, 2, x0_y1) + Ic = torch.gather(im_padded, 2, x1_y0) + Id = torch.gather(im_padded, 2, x1_y1) + + return (Ia * wa + Ib * wb + Ic * wc + Id * wd).reshape(n, c, gh, gw) + + +def is_in_onnx_export_without_custom_ops(): + from annotator.mmpkg.mmcv.ops import get_onnxruntime_op_path + ort_custom_op_path = get_onnxruntime_op_path() + return torch.onnx.is_in_onnx_export( + ) and not osp.exists(ort_custom_op_path) + + +def normalize(grid): + """Normalize input grid from [-1, 1] to [0, 1] + Args: + grid (Tensor): The grid to be normalize, range [-1, 1]. + Returns: + Tensor: Normalized grid, range [0, 1]. + """ + + return (grid + 1.0) / 2.0 + + +def denormalize(grid): + """Denormalize input grid from range [0, 1] to [-1, 1] + Args: + grid (Tensor): The grid to be denormalize, range [0, 1]. + Returns: + Tensor: Denormalized grid, range [-1, 1]. + """ + + return grid * 2.0 - 1.0 + + +def generate_grid(num_grid, size, device): + """Generate regular square grid of points in [0, 1] x [0, 1] coordinate + space. + + Args: + num_grid (int): The number of grids to sample, one for each region. + size (tuple(int, int)): The side size of the regular grid. + device (torch.device): Desired device of returned tensor. + + Returns: + (torch.Tensor): A tensor of shape (num_grid, size[0]*size[1], 2) that + contains coordinates for the regular grids. + """ + + affine_trans = torch.tensor([[[1., 0., 0.], [0., 1., 0.]]], device=device) + grid = F.affine_grid( + affine_trans, torch.Size((1, 1, *size)), align_corners=False) + grid = normalize(grid) + return grid.view(1, -1, 2).expand(num_grid, -1, -1) + + +def rel_roi_point_to_abs_img_point(rois, rel_roi_points): + """Convert roi based relative point coordinates to image based absolute + point coordinates. + + Args: + rois (Tensor): RoIs or BBoxes, shape (N, 4) or (N, 5) + rel_roi_points (Tensor): Point coordinates inside RoI, relative to + RoI, location, range (0, 1), shape (N, P, 2) + Returns: + Tensor: Image based absolute point coordinates, shape (N, P, 2) + """ + + with torch.no_grad(): + assert rel_roi_points.size(0) == rois.size(0) + assert rois.dim() == 2 + assert rel_roi_points.dim() == 3 + assert rel_roi_points.size(2) == 2 + # remove batch idx + if rois.size(1) == 5: + rois = rois[:, 1:] + abs_img_points = rel_roi_points.clone() + # To avoid an error during exporting to onnx use independent + # variables instead inplace computation + xs = abs_img_points[:, :, 0] * (rois[:, None, 2] - rois[:, None, 0]) + ys = abs_img_points[:, :, 1] * (rois[:, None, 3] - rois[:, None, 1]) + xs += rois[:, None, 0] + ys += rois[:, None, 1] + abs_img_points = torch.stack([xs, ys], dim=2) + return abs_img_points + + +def get_shape_from_feature_map(x): + """Get spatial resolution of input feature map considering exporting to + onnx mode. + + Args: + x (torch.Tensor): Input tensor, shape (N, C, H, W) + Returns: + torch.Tensor: Spatial resolution (width, height), shape (1, 1, 2) + """ + if torch.onnx.is_in_onnx_export(): + img_shape = shape_as_tensor(x)[2:].flip(0).view(1, 1, 2).to( + x.device).float() + else: + img_shape = torch.tensor(x.shape[2:]).flip(0).view(1, 1, 2).to( + x.device).float() + return img_shape + + +def abs_img_point_to_rel_img_point(abs_img_points, img, spatial_scale=1.): + """Convert image based absolute point coordinates to image based relative + coordinates for sampling. + + Args: + abs_img_points (Tensor): Image based absolute point coordinates, + shape (N, P, 2) + img (tuple/Tensor): (height, width) of image or feature map. + spatial_scale (float): Scale points by this factor. Default: 1. + + Returns: + Tensor: Image based relative point coordinates for sampling, + shape (N, P, 2) + """ + + assert (isinstance(img, tuple) and len(img) == 2) or \ + (isinstance(img, torch.Tensor) and len(img.shape) == 4) + + if isinstance(img, tuple): + h, w = img + scale = torch.tensor([w, h], + dtype=torch.float, + device=abs_img_points.device) + scale = scale.view(1, 1, 2) + else: + scale = get_shape_from_feature_map(img) + + return abs_img_points / scale * spatial_scale + + +def rel_roi_point_to_rel_img_point(rois, + rel_roi_points, + img, + spatial_scale=1.): + """Convert roi based relative point coordinates to image based absolute + point coordinates. + + Args: + rois (Tensor): RoIs or BBoxes, shape (N, 4) or (N, 5) + rel_roi_points (Tensor): Point coordinates inside RoI, relative to + RoI, location, range (0, 1), shape (N, P, 2) + img (tuple/Tensor): (height, width) of image or feature map. + spatial_scale (float): Scale points by this factor. Default: 1. + + Returns: + Tensor: Image based relative point coordinates for sampling, + shape (N, P, 2) + """ + + abs_img_point = rel_roi_point_to_abs_img_point(rois, rel_roi_points) + rel_img_point = abs_img_point_to_rel_img_point(abs_img_point, img, + spatial_scale) + + return rel_img_point + + +def point_sample(input, points, align_corners=False, **kwargs): + """A wrapper around :func:`grid_sample` to support 3D point_coords tensors + Unlike :func:`torch.nn.functional.grid_sample` it assumes point_coords to + lie inside ``[0, 1] x [0, 1]`` square. + + Args: + input (Tensor): Feature map, shape (N, C, H, W). + points (Tensor): Image based absolute point coordinates (normalized), + range [0, 1] x [0, 1], shape (N, P, 2) or (N, Hgrid, Wgrid, 2). + align_corners (bool): Whether align_corners. Default: False + + Returns: + Tensor: Features of `point` on `input`, shape (N, C, P) or + (N, C, Hgrid, Wgrid). + """ + + add_dim = False + if points.dim() == 3: + add_dim = True + points = points.unsqueeze(2) + if is_in_onnx_export_without_custom_ops(): + # If custom ops for onnx runtime not compiled use python + # implementation of grid_sample function to make onnx graph + # with supported nodes + output = bilinear_grid_sample( + input, denormalize(points), align_corners=align_corners) + else: + output = F.grid_sample( + input, denormalize(points), align_corners=align_corners, **kwargs) + if add_dim: + output = output.squeeze(3) + return output + + +class SimpleRoIAlign(nn.Module): + + def __init__(self, output_size, spatial_scale, aligned=True): + """Simple RoI align in PointRend, faster than standard RoIAlign. + + Args: + output_size (tuple[int]): h, w + spatial_scale (float): scale the input boxes by this number + aligned (bool): if False, use the legacy implementation in + MMDetection, align_corners=True will be used in F.grid_sample. + If True, align the results more perfectly. + """ + + super(SimpleRoIAlign, self).__init__() + self.output_size = _pair(output_size) + self.spatial_scale = float(spatial_scale) + # to be consistent with other RoI ops + self.use_torchvision = False + self.aligned = aligned + + def forward(self, features, rois): + num_imgs = features.size(0) + num_rois = rois.size(0) + rel_roi_points = generate_grid( + num_rois, self.output_size, device=rois.device) + + if torch.onnx.is_in_onnx_export(): + rel_img_points = rel_roi_point_to_rel_img_point( + rois, rel_roi_points, features, self.spatial_scale) + rel_img_points = rel_img_points.reshape(num_imgs, -1, + *rel_img_points.shape[1:]) + point_feats = point_sample( + features, rel_img_points, align_corners=not self.aligned) + point_feats = point_feats.transpose(1, 2) + else: + point_feats = [] + for batch_ind in range(num_imgs): + # unravel batch dim + feat = features[batch_ind].unsqueeze(0) + inds = (rois[:, 0].long() == batch_ind) + if inds.any(): + rel_img_points = rel_roi_point_to_rel_img_point( + rois[inds], rel_roi_points[inds], feat, + self.spatial_scale).unsqueeze(0) + point_feat = point_sample( + feat, rel_img_points, align_corners=not self.aligned) + point_feat = point_feat.squeeze(0).transpose(0, 1) + point_feats.append(point_feat) + + point_feats = torch.cat(point_feats, dim=0) + + channels = features.size(1) + roi_feats = point_feats.reshape(num_rois, channels, *self.output_size) + + return roi_feats + + def __repr__(self): + format_str = self.__class__.__name__ + format_str += '(output_size={}, spatial_scale={}'.format( + self.output_size, self.spatial_scale) + return format_str diff --git a/annotator/mmpkg/mmcv/ops/points_in_boxes.py b/annotator/mmpkg/mmcv/ops/points_in_boxes.py new file mode 100644 index 0000000000000000000000000000000000000000..4003173a53052161dbcd687a2fa1d755642fdab8 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/points_in_boxes.py @@ -0,0 +1,133 @@ +import torch + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', [ + 'points_in_boxes_part_forward', 'points_in_boxes_cpu_forward', + 'points_in_boxes_all_forward' +]) + + +def points_in_boxes_part(points, boxes): + """Find the box in which each point is (CUDA). + + Args: + points (torch.Tensor): [B, M, 3], [x, y, z] in LiDAR/DEPTH coordinate + boxes (torch.Tensor): [B, T, 7], + num_valid_boxes <= T, [x, y, z, x_size, y_size, z_size, rz] in + LiDAR/DEPTH coordinate, (x, y, z) is the bottom center + + Returns: + box_idxs_of_pts (torch.Tensor): (B, M), default background = -1 + """ + assert points.shape[0] == boxes.shape[0], \ + 'Points and boxes should have the same batch size, ' \ + f'but got {points.shape[0]} and {boxes.shape[0]}' + assert boxes.shape[2] == 7, \ + 'boxes dimension should be 7, ' \ + f'but got unexpected shape {boxes.shape[2]}' + assert points.shape[2] == 3, \ + 'points dimension should be 3, ' \ + f'but got unexpected shape {points.shape[2]}' + batch_size, num_points, _ = points.shape + + box_idxs_of_pts = points.new_zeros((batch_size, num_points), + dtype=torch.int).fill_(-1) + + # If manually put the tensor 'points' or 'boxes' on a device + # which is not the current device, some temporary variables + # will be created on the current device in the cuda op, + # and the output will be incorrect. + # Therefore, we force the current device to be the same + # as the device of the tensors if it was not. + # Please refer to https://github.com/open-mmlab/mmdetection3d/issues/305 + # for the incorrect output before the fix. + points_device = points.get_device() + assert points_device == boxes.get_device(), \ + 'Points and boxes should be put on the same device' + if torch.cuda.current_device() != points_device: + torch.cuda.set_device(points_device) + + ext_module.points_in_boxes_part_forward(boxes.contiguous(), + points.contiguous(), + box_idxs_of_pts) + + return box_idxs_of_pts + + +def points_in_boxes_cpu(points, boxes): + """Find all boxes in which each point is (CPU). The CPU version of + :meth:`points_in_boxes_all`. + + Args: + points (torch.Tensor): [B, M, 3], [x, y, z] in + LiDAR/DEPTH coordinate + boxes (torch.Tensor): [B, T, 7], + num_valid_boxes <= T, [x, y, z, x_size, y_size, z_size, rz], + (x, y, z) is the bottom center. + + Returns: + box_idxs_of_pts (torch.Tensor): (B, M, T), default background = 0. + """ + assert points.shape[0] == boxes.shape[0], \ + 'Points and boxes should have the same batch size, ' \ + f'but got {points.shape[0]} and {boxes.shape[0]}' + assert boxes.shape[2] == 7, \ + 'boxes dimension should be 7, ' \ + f'but got unexpected shape {boxes.shape[2]}' + assert points.shape[2] == 3, \ + 'points dimension should be 3, ' \ + f'but got unexpected shape {points.shape[2]}' + batch_size, num_points, _ = points.shape + num_boxes = boxes.shape[1] + + point_indices = points.new_zeros((batch_size, num_boxes, num_points), + dtype=torch.int) + for b in range(batch_size): + ext_module.points_in_boxes_cpu_forward(boxes[b].float().contiguous(), + points[b].float().contiguous(), + point_indices[b]) + point_indices = point_indices.transpose(1, 2) + + return point_indices + + +def points_in_boxes_all(points, boxes): + """Find all boxes in which each point is (CUDA). + + Args: + points (torch.Tensor): [B, M, 3], [x, y, z] in LiDAR/DEPTH coordinate + boxes (torch.Tensor): [B, T, 7], + num_valid_boxes <= T, [x, y, z, x_size, y_size, z_size, rz], + (x, y, z) is the bottom center. + + Returns: + box_idxs_of_pts (torch.Tensor): (B, M, T), default background = 0. + """ + assert boxes.shape[0] == points.shape[0], \ + 'Points and boxes should have the same batch size, ' \ + f'but got {boxes.shape[0]} and {boxes.shape[0]}' + assert boxes.shape[2] == 7, \ + 'boxes dimension should be 7, ' \ + f'but got unexpected shape {boxes.shape[2]}' + assert points.shape[2] == 3, \ + 'points dimension should be 3, ' \ + f'but got unexpected shape {points.shape[2]}' + batch_size, num_points, _ = points.shape + num_boxes = boxes.shape[1] + + box_idxs_of_pts = points.new_zeros((batch_size, num_points, num_boxes), + dtype=torch.int).fill_(0) + + # Same reason as line 25-32 + points_device = points.get_device() + assert points_device == boxes.get_device(), \ + 'Points and boxes should be put on the same device' + if torch.cuda.current_device() != points_device: + torch.cuda.set_device(points_device) + + ext_module.points_in_boxes_all_forward(boxes.contiguous(), + points.contiguous(), + box_idxs_of_pts) + + return box_idxs_of_pts diff --git a/annotator/mmpkg/mmcv/ops/points_sampler.py b/annotator/mmpkg/mmcv/ops/points_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..ae1a24f939dd0e2934765326363ea51c2f2b4cca --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/points_sampler.py @@ -0,0 +1,177 @@ +from typing import List + +import torch +from torch import nn as nn + +from annotator.mmpkg.mmcv.runner import force_fp32 +from .furthest_point_sample import (furthest_point_sample, + furthest_point_sample_with_dist) + + +def calc_square_dist(point_feat_a, point_feat_b, norm=True): + """Calculating square distance between a and b. + + Args: + point_feat_a (Tensor): (B, N, C) Feature vector of each point. + point_feat_b (Tensor): (B, M, C) Feature vector of each point. + norm (Bool, optional): Whether to normalize the distance. + Default: True. + + Returns: + Tensor: (B, N, M) Distance between each pair points. + """ + num_channel = point_feat_a.shape[-1] + # [bs, n, 1] + a_square = torch.sum(point_feat_a.unsqueeze(dim=2).pow(2), dim=-1) + # [bs, 1, m] + b_square = torch.sum(point_feat_b.unsqueeze(dim=1).pow(2), dim=-1) + + corr_matrix = torch.matmul(point_feat_a, point_feat_b.transpose(1, 2)) + + dist = a_square + b_square - 2 * corr_matrix + if norm: + dist = torch.sqrt(dist) / num_channel + return dist + + +def get_sampler_cls(sampler_type): + """Get the type and mode of points sampler. + + Args: + sampler_type (str): The type of points sampler. + The valid value are "D-FPS", "F-FPS", or "FS". + + Returns: + class: Points sampler type. + """ + sampler_mappings = { + 'D-FPS': DFPSSampler, + 'F-FPS': FFPSSampler, + 'FS': FSSampler, + } + try: + return sampler_mappings[sampler_type] + except KeyError: + raise KeyError( + f'Supported `sampler_type` are {sampler_mappings.keys()}, but got \ + {sampler_type}') + + +class PointsSampler(nn.Module): + """Points sampling. + + Args: + num_point (list[int]): Number of sample points. + fps_mod_list (list[str], optional): Type of FPS method, valid mod + ['F-FPS', 'D-FPS', 'FS'], Default: ['D-FPS']. + F-FPS: using feature distances for FPS. + D-FPS: using Euclidean distances of points for FPS. + FS: using F-FPS and D-FPS simultaneously. + fps_sample_range_list (list[int], optional): + Range of points to apply FPS. Default: [-1]. + """ + + def __init__(self, + num_point: List[int], + fps_mod_list: List[str] = ['D-FPS'], + fps_sample_range_list: List[int] = [-1]): + super().__init__() + # FPS would be applied to different fps_mod in the list, + # so the length of the num_point should be equal to + # fps_mod_list and fps_sample_range_list. + assert len(num_point) == len(fps_mod_list) == len( + fps_sample_range_list) + self.num_point = num_point + self.fps_sample_range_list = fps_sample_range_list + self.samplers = nn.ModuleList() + for fps_mod in fps_mod_list: + self.samplers.append(get_sampler_cls(fps_mod)()) + self.fp16_enabled = False + + @force_fp32() + def forward(self, points_xyz, features): + """ + Args: + points_xyz (Tensor): (B, N, 3) xyz coordinates of the features. + features (Tensor): (B, C, N) Descriptors of the features. + + Returns: + Tensor: (B, npoint, sample_num) Indices of sampled points. + """ + indices = [] + last_fps_end_index = 0 + + for fps_sample_range, sampler, npoint in zip( + self.fps_sample_range_list, self.samplers, self.num_point): + assert fps_sample_range < points_xyz.shape[1] + + if fps_sample_range == -1: + sample_points_xyz = points_xyz[:, last_fps_end_index:] + if features is not None: + sample_features = features[:, :, last_fps_end_index:] + else: + sample_features = None + else: + sample_points_xyz = \ + points_xyz[:, last_fps_end_index:fps_sample_range] + if features is not None: + sample_features = features[:, :, last_fps_end_index: + fps_sample_range] + else: + sample_features = None + + fps_idx = sampler(sample_points_xyz.contiguous(), sample_features, + npoint) + + indices.append(fps_idx + last_fps_end_index) + last_fps_end_index += fps_sample_range + indices = torch.cat(indices, dim=1) + + return indices + + +class DFPSSampler(nn.Module): + """Using Euclidean distances of points for FPS.""" + + def __init__(self): + super().__init__() + + def forward(self, points, features, npoint): + """Sampling points with D-FPS.""" + fps_idx = furthest_point_sample(points.contiguous(), npoint) + return fps_idx + + +class FFPSSampler(nn.Module): + """Using feature distances for FPS.""" + + def __init__(self): + super().__init__() + + def forward(self, points, features, npoint): + """Sampling points with F-FPS.""" + assert features is not None, \ + 'feature input to FFPS_Sampler should not be None' + features_for_fps = torch.cat([points, features.transpose(1, 2)], dim=2) + features_dist = calc_square_dist( + features_for_fps, features_for_fps, norm=False) + fps_idx = furthest_point_sample_with_dist(features_dist, npoint) + return fps_idx + + +class FSSampler(nn.Module): + """Using F-FPS and D-FPS simultaneously.""" + + def __init__(self): + super().__init__() + + def forward(self, points, features, npoint): + """Sampling points with FS_Sampling.""" + assert features is not None, \ + 'feature input to FS_Sampler should not be None' + ffps_sampler = FFPSSampler() + dfps_sampler = DFPSSampler() + fps_idx_ffps = ffps_sampler(points, features, npoint) + fps_idx_dfps = dfps_sampler(points, features, npoint) + fps_idx = torch.cat([fps_idx_ffps, fps_idx_dfps], dim=1) + return fps_idx diff --git a/annotator/mmpkg/mmcv/ops/psa_mask.py b/annotator/mmpkg/mmcv/ops/psa_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..cdf14e62b50e8d4dd6856c94333c703bcc4c9ab6 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/psa_mask.py @@ -0,0 +1,92 @@ +# Modified from https://github.com/hszhao/semseg/blob/master/lib/psa +from torch import nn +from torch.autograd import Function +from torch.nn.modules.utils import _pair + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', + ['psamask_forward', 'psamask_backward']) + + +class PSAMaskFunction(Function): + + @staticmethod + def symbolic(g, input, psa_type, mask_size): + return g.op( + 'mmcv::MMCVPSAMask', + input, + psa_type_i=psa_type, + mask_size_i=mask_size) + + @staticmethod + def forward(ctx, input, psa_type, mask_size): + ctx.psa_type = psa_type + ctx.mask_size = _pair(mask_size) + ctx.save_for_backward(input) + + h_mask, w_mask = ctx.mask_size + batch_size, channels, h_feature, w_feature = input.size() + assert channels == h_mask * w_mask + output = input.new_zeros( + (batch_size, h_feature * w_feature, h_feature, w_feature)) + + ext_module.psamask_forward( + input, + output, + psa_type=psa_type, + num_=batch_size, + h_feature=h_feature, + w_feature=w_feature, + h_mask=h_mask, + w_mask=w_mask, + half_h_mask=(h_mask - 1) // 2, + half_w_mask=(w_mask - 1) // 2) + return output + + @staticmethod + def backward(ctx, grad_output): + input = ctx.saved_tensors[0] + psa_type = ctx.psa_type + h_mask, w_mask = ctx.mask_size + batch_size, channels, h_feature, w_feature = input.size() + grad_input = grad_output.new_zeros( + (batch_size, channels, h_feature, w_feature)) + ext_module.psamask_backward( + grad_output, + grad_input, + psa_type=psa_type, + num_=batch_size, + h_feature=h_feature, + w_feature=w_feature, + h_mask=h_mask, + w_mask=w_mask, + half_h_mask=(h_mask - 1) // 2, + half_w_mask=(w_mask - 1) // 2) + return grad_input, None, None, None + + +psa_mask = PSAMaskFunction.apply + + +class PSAMask(nn.Module): + + def __init__(self, psa_type, mask_size=None): + super(PSAMask, self).__init__() + assert psa_type in ['collect', 'distribute'] + if psa_type == 'collect': + psa_type_enum = 0 + else: + psa_type_enum = 1 + self.psa_type_enum = psa_type_enum + self.mask_size = mask_size + self.psa_type = psa_type + + def forward(self, input): + return psa_mask(input, self.psa_type_enum, self.mask_size) + + def __repr__(self): + s = self.__class__.__name__ + s += f'(psa_type={self.psa_type}, ' + s += f'mask_size={self.mask_size})' + return s diff --git a/annotator/mmpkg/mmcv/ops/roi_align.py b/annotator/mmpkg/mmcv/ops/roi_align.py new file mode 100644 index 0000000000000000000000000000000000000000..0755aefc66e67233ceae0f4b77948301c443e9fb --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/roi_align.py @@ -0,0 +1,223 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.utils import _pair + +from ..utils import deprecated_api_warning, ext_loader + +ext_module = ext_loader.load_ext('_ext', + ['roi_align_forward', 'roi_align_backward']) + + +class RoIAlignFunction(Function): + + @staticmethod + def symbolic(g, input, rois, output_size, spatial_scale, sampling_ratio, + pool_mode, aligned): + from ..onnx import is_custom_op_loaded + has_custom_op = is_custom_op_loaded() + if has_custom_op: + return g.op( + 'mmcv::MMCVRoiAlign', + input, + rois, + output_height_i=output_size[0], + output_width_i=output_size[1], + spatial_scale_f=spatial_scale, + sampling_ratio_i=sampling_ratio, + mode_s=pool_mode, + aligned_i=aligned) + else: + from torch.onnx.symbolic_opset9 import sub, squeeze + from torch.onnx.symbolic_helper import _slice_helper + from torch.onnx import TensorProtoDataType + # batch_indices = rois[:, 0].long() + batch_indices = _slice_helper( + g, rois, axes=[1], starts=[0], ends=[1]) + batch_indices = squeeze(g, batch_indices, 1) + batch_indices = g.op( + 'Cast', batch_indices, to_i=TensorProtoDataType.INT64) + # rois = rois[:, 1:] + rois = _slice_helper(g, rois, axes=[1], starts=[1], ends=[5]) + if aligned: + # rois -= 0.5/spatial_scale + aligned_offset = g.op( + 'Constant', + value_t=torch.tensor([0.5 / spatial_scale], + dtype=torch.float32)) + rois = sub(g, rois, aligned_offset) + # roi align + return g.op( + 'RoiAlign', + input, + rois, + batch_indices, + output_height_i=output_size[0], + output_width_i=output_size[1], + spatial_scale_f=spatial_scale, + sampling_ratio_i=max(0, sampling_ratio), + mode_s=pool_mode) + + @staticmethod + def forward(ctx, + input, + rois, + output_size, + spatial_scale=1.0, + sampling_ratio=0, + pool_mode='avg', + aligned=True): + ctx.output_size = _pair(output_size) + ctx.spatial_scale = spatial_scale + ctx.sampling_ratio = sampling_ratio + assert pool_mode in ('max', 'avg') + ctx.pool_mode = 0 if pool_mode == 'max' else 1 + ctx.aligned = aligned + ctx.input_shape = input.size() + + assert rois.size(1) == 5, 'RoI must be (idx, x1, y1, x2, y2)!' + + output_shape = (rois.size(0), input.size(1), ctx.output_size[0], + ctx.output_size[1]) + output = input.new_zeros(output_shape) + if ctx.pool_mode == 0: + argmax_y = input.new_zeros(output_shape) + argmax_x = input.new_zeros(output_shape) + else: + argmax_y = input.new_zeros(0) + argmax_x = input.new_zeros(0) + + ext_module.roi_align_forward( + input, + rois, + output, + argmax_y, + argmax_x, + aligned_height=ctx.output_size[0], + aligned_width=ctx.output_size[1], + spatial_scale=ctx.spatial_scale, + sampling_ratio=ctx.sampling_ratio, + pool_mode=ctx.pool_mode, + aligned=ctx.aligned) + + ctx.save_for_backward(rois, argmax_y, argmax_x) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + rois, argmax_y, argmax_x = ctx.saved_tensors + grad_input = grad_output.new_zeros(ctx.input_shape) + # complex head architecture may cause grad_output uncontiguous. + grad_output = grad_output.contiguous() + ext_module.roi_align_backward( + grad_output, + rois, + argmax_y, + argmax_x, + grad_input, + aligned_height=ctx.output_size[0], + aligned_width=ctx.output_size[1], + spatial_scale=ctx.spatial_scale, + sampling_ratio=ctx.sampling_ratio, + pool_mode=ctx.pool_mode, + aligned=ctx.aligned) + return grad_input, None, None, None, None, None, None + + +roi_align = RoIAlignFunction.apply + + +class RoIAlign(nn.Module): + """RoI align pooling layer. + + Args: + output_size (tuple): h, w + spatial_scale (float): scale the input boxes by this number + sampling_ratio (int): number of inputs samples to take for each + output sample. 0 to take samples densely for current models. + pool_mode (str, 'avg' or 'max'): pooling mode in each bin. + aligned (bool): if False, use the legacy implementation in + MMDetection. If True, align the results more perfectly. + use_torchvision (bool): whether to use roi_align from torchvision. + + Note: + The implementation of RoIAlign when aligned=True is modified from + https://github.com/facebookresearch/detectron2/ + + The meaning of aligned=True: + + Given a continuous coordinate c, its two neighboring pixel + indices (in our pixel model) are computed by floor(c - 0.5) and + ceil(c - 0.5). For example, c=1.3 has pixel neighbors with discrete + indices [0] and [1] (which are sampled from the underlying signal + at continuous coordinates 0.5 and 1.5). But the original roi_align + (aligned=False) does not subtract the 0.5 when computing + neighboring pixel indices and therefore it uses pixels with a + slightly incorrect alignment (relative to our pixel model) when + performing bilinear interpolation. + + With `aligned=True`, + we first appropriately scale the ROI and then shift it by -0.5 + prior to calling roi_align. This produces the correct neighbors; + + The difference does not make a difference to the model's + performance if ROIAlign is used together with conv layers. + """ + + @deprecated_api_warning( + { + 'out_size': 'output_size', + 'sample_num': 'sampling_ratio' + }, + cls_name='RoIAlign') + def __init__(self, + output_size, + spatial_scale=1.0, + sampling_ratio=0, + pool_mode='avg', + aligned=True, + use_torchvision=False): + super(RoIAlign, self).__init__() + + self.output_size = _pair(output_size) + self.spatial_scale = float(spatial_scale) + self.sampling_ratio = int(sampling_ratio) + self.pool_mode = pool_mode + self.aligned = aligned + self.use_torchvision = use_torchvision + + def forward(self, input, rois): + """ + Args: + input: NCHW images + rois: Bx5 boxes. First column is the index into N.\ + The other 4 columns are xyxy. + """ + if self.use_torchvision: + from torchvision.ops import roi_align as tv_roi_align + if 'aligned' in tv_roi_align.__code__.co_varnames: + return tv_roi_align(input, rois, self.output_size, + self.spatial_scale, self.sampling_ratio, + self.aligned) + else: + if self.aligned: + rois -= rois.new_tensor([0.] + + [0.5 / self.spatial_scale] * 4) + return tv_roi_align(input, rois, self.output_size, + self.spatial_scale, self.sampling_ratio) + else: + return roi_align(input, rois, self.output_size, self.spatial_scale, + self.sampling_ratio, self.pool_mode, self.aligned) + + def __repr__(self): + s = self.__class__.__name__ + s += f'(output_size={self.output_size}, ' + s += f'spatial_scale={self.spatial_scale}, ' + s += f'sampling_ratio={self.sampling_ratio}, ' + s += f'pool_mode={self.pool_mode}, ' + s += f'aligned={self.aligned}, ' + s += f'use_torchvision={self.use_torchvision})' + return s diff --git a/annotator/mmpkg/mmcv/ops/roi_align_rotated.py b/annotator/mmpkg/mmcv/ops/roi_align_rotated.py new file mode 100644 index 0000000000000000000000000000000000000000..0ce4961a3555d4da8bc3e32f1f7d5ad50036587d --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/roi_align_rotated.py @@ -0,0 +1,177 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch.nn as nn +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['roi_align_rotated_forward', 'roi_align_rotated_backward']) + + +class RoIAlignRotatedFunction(Function): + + @staticmethod + def symbolic(g, features, rois, out_size, spatial_scale, sample_num, + aligned, clockwise): + if isinstance(out_size, int): + out_h = out_size + out_w = out_size + elif isinstance(out_size, tuple): + assert len(out_size) == 2 + assert isinstance(out_size[0], int) + assert isinstance(out_size[1], int) + out_h, out_w = out_size + else: + raise TypeError( + '"out_size" must be an integer or tuple of integers') + return g.op( + 'mmcv::MMCVRoIAlignRotated', + features, + rois, + output_height_i=out_h, + output_width_i=out_h, + spatial_scale_f=spatial_scale, + sampling_ratio_i=sample_num, + aligned_i=aligned, + clockwise_i=clockwise) + + @staticmethod + def forward(ctx, + features, + rois, + out_size, + spatial_scale, + sample_num=0, + aligned=True, + clockwise=False): + if isinstance(out_size, int): + out_h = out_size + out_w = out_size + elif isinstance(out_size, tuple): + assert len(out_size) == 2 + assert isinstance(out_size[0], int) + assert isinstance(out_size[1], int) + out_h, out_w = out_size + else: + raise TypeError( + '"out_size" must be an integer or tuple of integers') + ctx.spatial_scale = spatial_scale + ctx.sample_num = sample_num + ctx.aligned = aligned + ctx.clockwise = clockwise + ctx.save_for_backward(rois) + ctx.feature_size = features.size() + + batch_size, num_channels, data_height, data_width = features.size() + num_rois = rois.size(0) + + output = features.new_zeros(num_rois, num_channels, out_h, out_w) + ext_module.roi_align_rotated_forward( + features, + rois, + output, + pooled_height=out_h, + pooled_width=out_w, + spatial_scale=spatial_scale, + sample_num=sample_num, + aligned=aligned, + clockwise=clockwise) + return output + + @staticmethod + def backward(ctx, grad_output): + feature_size = ctx.feature_size + spatial_scale = ctx.spatial_scale + aligned = ctx.aligned + clockwise = ctx.clockwise + sample_num = ctx.sample_num + rois = ctx.saved_tensors[0] + assert feature_size is not None + batch_size, num_channels, data_height, data_width = feature_size + + out_w = grad_output.size(3) + out_h = grad_output.size(2) + + grad_input = grad_rois = None + + if ctx.needs_input_grad[0]: + grad_input = rois.new_zeros(batch_size, num_channels, data_height, + data_width) + ext_module.roi_align_rotated_backward( + grad_output.contiguous(), + rois, + grad_input, + pooled_height=out_h, + pooled_width=out_w, + spatial_scale=spatial_scale, + sample_num=sample_num, + aligned=aligned, + clockwise=clockwise) + return grad_input, grad_rois, None, None, None, None, None + + +roi_align_rotated = RoIAlignRotatedFunction.apply + + +class RoIAlignRotated(nn.Module): + """RoI align pooling layer for rotated proposals. + + It accepts a feature map of shape (N, C, H, W) and rois with shape + (n, 6) with each roi decoded as (batch_index, center_x, center_y, + w, h, angle). The angle is in radian. + + Args: + out_size (tuple): h, w + spatial_scale (float): scale the input boxes by this number + sample_num (int): number of inputs samples to take for each + output sample. 0 to take samples densely for current models. + aligned (bool): if False, use the legacy implementation in + MMDetection. If True, align the results more perfectly. + Default: True. + clockwise (bool): If True, the angle in each proposal follows a + clockwise fashion in image space, otherwise, the angle is + counterclockwise. Default: False. + + Note: + The implementation of RoIAlign when aligned=True is modified from + https://github.com/facebookresearch/detectron2/ + + The meaning of aligned=True: + + Given a continuous coordinate c, its two neighboring pixel + indices (in our pixel model) are computed by floor(c - 0.5) and + ceil(c - 0.5). For example, c=1.3 has pixel neighbors with discrete + indices [0] and [1] (which are sampled from the underlying signal + at continuous coordinates 0.5 and 1.5). But the original roi_align + (aligned=False) does not subtract the 0.5 when computing + neighboring pixel indices and therefore it uses pixels with a + slightly incorrect alignment (relative to our pixel model) when + performing bilinear interpolation. + + With `aligned=True`, + we first appropriately scale the ROI and then shift it by -0.5 + prior to calling roi_align. This produces the correct neighbors; + + The difference does not make a difference to the model's + performance if ROIAlign is used together with conv layers. + """ + + def __init__(self, + out_size, + spatial_scale, + sample_num=0, + aligned=True, + clockwise=False): + super(RoIAlignRotated, self).__init__() + + self.out_size = out_size + self.spatial_scale = float(spatial_scale) + self.sample_num = int(sample_num) + self.aligned = aligned + self.clockwise = clockwise + + def forward(self, features, rois): + return RoIAlignRotatedFunction.apply(features, rois, self.out_size, + self.spatial_scale, + self.sample_num, self.aligned, + self.clockwise) diff --git a/annotator/mmpkg/mmcv/ops/roi_pool.py b/annotator/mmpkg/mmcv/ops/roi_pool.py new file mode 100644 index 0000000000000000000000000000000000000000..d339d8f2941eabc1cbe181a9c6c5ab5ff4ff4e5f --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/roi_pool.py @@ -0,0 +1,86 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.utils import _pair + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', + ['roi_pool_forward', 'roi_pool_backward']) + + +class RoIPoolFunction(Function): + + @staticmethod + def symbolic(g, input, rois, output_size, spatial_scale): + return g.op( + 'MaxRoiPool', + input, + rois, + pooled_shape_i=output_size, + spatial_scale_f=spatial_scale) + + @staticmethod + def forward(ctx, input, rois, output_size, spatial_scale=1.0): + ctx.output_size = _pair(output_size) + ctx.spatial_scale = spatial_scale + ctx.input_shape = input.size() + + assert rois.size(1) == 5, 'RoI must be (idx, x1, y1, x2, y2)!' + + output_shape = (rois.size(0), input.size(1), ctx.output_size[0], + ctx.output_size[1]) + output = input.new_zeros(output_shape) + argmax = input.new_zeros(output_shape, dtype=torch.int) + + ext_module.roi_pool_forward( + input, + rois, + output, + argmax, + pooled_height=ctx.output_size[0], + pooled_width=ctx.output_size[1], + spatial_scale=ctx.spatial_scale) + + ctx.save_for_backward(rois, argmax) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + rois, argmax = ctx.saved_tensors + grad_input = grad_output.new_zeros(ctx.input_shape) + + ext_module.roi_pool_backward( + grad_output, + rois, + argmax, + grad_input, + pooled_height=ctx.output_size[0], + pooled_width=ctx.output_size[1], + spatial_scale=ctx.spatial_scale) + + return grad_input, None, None, None + + +roi_pool = RoIPoolFunction.apply + + +class RoIPool(nn.Module): + + def __init__(self, output_size, spatial_scale=1.0): + super(RoIPool, self).__init__() + + self.output_size = _pair(output_size) + self.spatial_scale = float(spatial_scale) + + def forward(self, input, rois): + return roi_pool(input, rois, self.output_size, self.spatial_scale) + + def __repr__(self): + s = self.__class__.__name__ + s += f'(output_size={self.output_size}, ' + s += f'spatial_scale={self.spatial_scale})' + return s diff --git a/annotator/mmpkg/mmcv/ops/roiaware_pool3d.py b/annotator/mmpkg/mmcv/ops/roiaware_pool3d.py new file mode 100644 index 0000000000000000000000000000000000000000..8191920ca50b388ef58f577dc986da101662ac53 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/roiaware_pool3d.py @@ -0,0 +1,114 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch import nn as nn +from torch.autograd import Function + +import annotator.mmpkg.mmcv as mmcv +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['roiaware_pool3d_forward', 'roiaware_pool3d_backward']) + + +class RoIAwarePool3d(nn.Module): + """Encode the geometry-specific features of each 3D proposal. + + Please refer to `PartA2 `_ for more + details. + + Args: + out_size (int or tuple): The size of output features. n or + [n1, n2, n3]. + max_pts_per_voxel (int, optional): The maximum number of points per + voxel. Default: 128. + mode (str, optional): Pooling method of RoIAware, 'max' or 'avg'. + Default: 'max'. + """ + + def __init__(self, out_size, max_pts_per_voxel=128, mode='max'): + super().__init__() + + self.out_size = out_size + self.max_pts_per_voxel = max_pts_per_voxel + assert mode in ['max', 'avg'] + pool_mapping = {'max': 0, 'avg': 1} + self.mode = pool_mapping[mode] + + def forward(self, rois, pts, pts_feature): + """ + Args: + rois (torch.Tensor): [N, 7], in LiDAR coordinate, + (x, y, z) is the bottom center of rois. + pts (torch.Tensor): [npoints, 3], coordinates of input points. + pts_feature (torch.Tensor): [npoints, C], features of input points. + + Returns: + pooled_features (torch.Tensor): [N, out_x, out_y, out_z, C] + """ + + return RoIAwarePool3dFunction.apply(rois, pts, pts_feature, + self.out_size, + self.max_pts_per_voxel, self.mode) + + +class RoIAwarePool3dFunction(Function): + + @staticmethod + def forward(ctx, rois, pts, pts_feature, out_size, max_pts_per_voxel, + mode): + """ + Args: + rois (torch.Tensor): [N, 7], in LiDAR coordinate, + (x, y, z) is the bottom center of rois. + pts (torch.Tensor): [npoints, 3], coordinates of input points. + pts_feature (torch.Tensor): [npoints, C], features of input points. + out_size (int or tuple): The size of output features. n or + [n1, n2, n3]. + max_pts_per_voxel (int): The maximum number of points per voxel. + Default: 128. + mode (int): Pooling method of RoIAware, 0 (max pool) or 1 (average + pool). + + Returns: + pooled_features (torch.Tensor): [N, out_x, out_y, out_z, C], output + pooled features. + """ + + if isinstance(out_size, int): + out_x = out_y = out_z = out_size + else: + assert len(out_size) == 3 + assert mmcv.is_tuple_of(out_size, int) + out_x, out_y, out_z = out_size + + num_rois = rois.shape[0] + num_channels = pts_feature.shape[-1] + num_pts = pts.shape[0] + + pooled_features = pts_feature.new_zeros( + (num_rois, out_x, out_y, out_z, num_channels)) + argmax = pts_feature.new_zeros( + (num_rois, out_x, out_y, out_z, num_channels), dtype=torch.int) + pts_idx_of_voxels = pts_feature.new_zeros( + (num_rois, out_x, out_y, out_z, max_pts_per_voxel), + dtype=torch.int) + + ext_module.roiaware_pool3d_forward(rois, pts, pts_feature, argmax, + pts_idx_of_voxels, pooled_features, + mode) + + ctx.roiaware_pool3d_for_backward = (pts_idx_of_voxels, argmax, mode, + num_pts, num_channels) + return pooled_features + + @staticmethod + def backward(ctx, grad_out): + ret = ctx.roiaware_pool3d_for_backward + pts_idx_of_voxels, argmax, mode, num_pts, num_channels = ret + + grad_in = grad_out.new_zeros((num_pts, num_channels)) + ext_module.roiaware_pool3d_backward(pts_idx_of_voxels, argmax, + grad_out.contiguous(), grad_in, + mode) + + return None, None, grad_in, None, None, None diff --git a/annotator/mmpkg/mmcv/ops/roipoint_pool3d.py b/annotator/mmpkg/mmcv/ops/roipoint_pool3d.py new file mode 100644 index 0000000000000000000000000000000000000000..0a21412c0728431c04b84245bc2e3109eea9aefc --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/roipoint_pool3d.py @@ -0,0 +1,77 @@ +from torch import nn as nn +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', ['roipoint_pool3d_forward']) + + +class RoIPointPool3d(nn.Module): + """Encode the geometry-specific features of each 3D proposal. + + Please refer to `Paper of PartA2 `_ + for more details. + + Args: + num_sampled_points (int, optional): Number of samples in each roi. + Default: 512. + """ + + def __init__(self, num_sampled_points=512): + super().__init__() + self.num_sampled_points = num_sampled_points + + def forward(self, points, point_features, boxes3d): + """ + Args: + points (torch.Tensor): Input points whose shape is (B, N, C). + point_features (torch.Tensor): Features of input points whose shape + is (B, N, C). + boxes3d (B, M, 7), Input bounding boxes whose shape is (B, M, 7). + + Returns: + pooled_features (torch.Tensor): The output pooled features whose + shape is (B, M, 512, 3 + C). + pooled_empty_flag (torch.Tensor): Empty flag whose shape is (B, M). + """ + return RoIPointPool3dFunction.apply(points, point_features, boxes3d, + self.num_sampled_points) + + +class RoIPointPool3dFunction(Function): + + @staticmethod + def forward(ctx, points, point_features, boxes3d, num_sampled_points=512): + """ + Args: + points (torch.Tensor): Input points whose shape is (B, N, C). + point_features (torch.Tensor): Features of input points whose shape + is (B, N, C). + boxes3d (B, M, 7), Input bounding boxes whose shape is (B, M, 7). + num_sampled_points (int, optional): The num of sampled points. + Default: 512. + + Returns: + pooled_features (torch.Tensor): The output pooled features whose + shape is (B, M, 512, 3 + C). + pooled_empty_flag (torch.Tensor): Empty flag whose shape is (B, M). + """ + assert len(points.shape) == 3 and points.shape[2] == 3 + batch_size, boxes_num, feature_len = points.shape[0], boxes3d.shape[ + 1], point_features.shape[2] + pooled_boxes3d = boxes3d.view(batch_size, -1, 7) + pooled_features = point_features.new_zeros( + (batch_size, boxes_num, num_sampled_points, 3 + feature_len)) + pooled_empty_flag = point_features.new_zeros( + (batch_size, boxes_num)).int() + + ext_module.roipoint_pool3d_forward(points.contiguous(), + pooled_boxes3d.contiguous(), + point_features.contiguous(), + pooled_features, pooled_empty_flag) + + return pooled_features, pooled_empty_flag + + @staticmethod + def backward(ctx, grad_out): + raise NotImplementedError diff --git a/annotator/mmpkg/mmcv/ops/saconv.py b/annotator/mmpkg/mmcv/ops/saconv.py new file mode 100644 index 0000000000000000000000000000000000000000..9d7be88c428ea2b9af2c32c60a86dddd13988ce8 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/saconv.py @@ -0,0 +1,145 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from annotator.mmpkg.mmcv.cnn import CONV_LAYERS, ConvAWS2d, constant_init +from annotator.mmpkg.mmcv.ops.deform_conv import deform_conv2d +from annotator.mmpkg.mmcv.utils import TORCH_VERSION, digit_version + + +@CONV_LAYERS.register_module(name='SAC') +class SAConv2d(ConvAWS2d): + """SAC (Switchable Atrous Convolution) + + This is an implementation of SAC in DetectoRS + (https://arxiv.org/pdf/2006.02334.pdf). + + Args: + in_channels (int): Number of channels in the input image + out_channels (int): Number of channels produced by the convolution + kernel_size (int or tuple): Size of the convolving kernel + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of + the input. Default: 0 + padding_mode (string, optional): ``'zeros'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. Default: ``'zeros'`` + dilation (int or tuple, optional): Spacing between kernel elements. + Default: 1 + groups (int, optional): Number of blocked connections from input + channels to output channels. Default: 1 + bias (bool, optional): If ``True``, adds a learnable bias to the + output. Default: ``True`` + use_deform: If ``True``, replace convolution with deformable + convolution. Default: ``False``. + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + bias=True, + use_deform=False): + super().__init__( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + bias=bias) + self.use_deform = use_deform + self.switch = nn.Conv2d( + self.in_channels, 1, kernel_size=1, stride=stride, bias=True) + self.weight_diff = nn.Parameter(torch.Tensor(self.weight.size())) + self.pre_context = nn.Conv2d( + self.in_channels, self.in_channels, kernel_size=1, bias=True) + self.post_context = nn.Conv2d( + self.out_channels, self.out_channels, kernel_size=1, bias=True) + if self.use_deform: + self.offset_s = nn.Conv2d( + self.in_channels, + 18, + kernel_size=3, + padding=1, + stride=stride, + bias=True) + self.offset_l = nn.Conv2d( + self.in_channels, + 18, + kernel_size=3, + padding=1, + stride=stride, + bias=True) + self.init_weights() + + def init_weights(self): + constant_init(self.switch, 0, bias=1) + self.weight_diff.data.zero_() + constant_init(self.pre_context, 0) + constant_init(self.post_context, 0) + if self.use_deform: + constant_init(self.offset_s, 0) + constant_init(self.offset_l, 0) + + def forward(self, x): + # pre-context + avg_x = F.adaptive_avg_pool2d(x, output_size=1) + avg_x = self.pre_context(avg_x) + avg_x = avg_x.expand_as(x) + x = x + avg_x + # switch + avg_x = F.pad(x, pad=(2, 2, 2, 2), mode='reflect') + avg_x = F.avg_pool2d(avg_x, kernel_size=5, stride=1, padding=0) + switch = self.switch(avg_x) + # sac + weight = self._get_weight(self.weight) + zero_bias = torch.zeros( + self.out_channels, device=weight.device, dtype=weight.dtype) + + if self.use_deform: + offset = self.offset_s(avg_x) + out_s = deform_conv2d(x, offset, weight, self.stride, self.padding, + self.dilation, self.groups, 1) + else: + if (TORCH_VERSION == 'parrots' + or digit_version(TORCH_VERSION) < digit_version('1.5.0')): + out_s = super().conv2d_forward(x, weight) + elif digit_version(TORCH_VERSION) >= digit_version('1.8.0'): + # bias is a required argument of _conv_forward in torch 1.8.0 + out_s = super()._conv_forward(x, weight, zero_bias) + else: + out_s = super()._conv_forward(x, weight) + ori_p = self.padding + ori_d = self.dilation + self.padding = tuple(3 * p for p in self.padding) + self.dilation = tuple(3 * d for d in self.dilation) + weight = weight + self.weight_diff + if self.use_deform: + offset = self.offset_l(avg_x) + out_l = deform_conv2d(x, offset, weight, self.stride, self.padding, + self.dilation, self.groups, 1) + else: + if (TORCH_VERSION == 'parrots' + or digit_version(TORCH_VERSION) < digit_version('1.5.0')): + out_l = super().conv2d_forward(x, weight) + elif digit_version(TORCH_VERSION) >= digit_version('1.8.0'): + # bias is a required argument of _conv_forward in torch 1.8.0 + out_l = super()._conv_forward(x, weight, zero_bias) + else: + out_l = super()._conv_forward(x, weight) + + out = switch * out_s + (1 - switch) * out_l + self.padding = ori_p + self.dilation = ori_d + # post-context + avg_x = F.adaptive_avg_pool2d(out, output_size=1) + avg_x = self.post_context(avg_x) + avg_x = avg_x.expand_as(out) + out = out + avg_x + return out diff --git a/annotator/mmpkg/mmcv/ops/scatter_points.py b/annotator/mmpkg/mmcv/ops/scatter_points.py new file mode 100644 index 0000000000000000000000000000000000000000..2b8aa4169e9f6ca4a6f845ce17d6d1e4db416bb8 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/scatter_points.py @@ -0,0 +1,135 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch import nn +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', + ['dynamic_point_to_voxel_forward', 'dynamic_point_to_voxel_backward']) + + +class _DynamicScatter(Function): + + @staticmethod + def forward(ctx, feats, coors, reduce_type='max'): + """convert kitti points(N, >=3) to voxels. + + Args: + feats (torch.Tensor): [N, C]. Points features to be reduced + into voxels. + coors (torch.Tensor): [N, ndim]. Corresponding voxel coordinates + (specifically multi-dim voxel index) of each points. + reduce_type (str, optional): Reduce op. support 'max', 'sum' and + 'mean'. Default: 'max'. + + Returns: + voxel_feats (torch.Tensor): [M, C]. Reduced features, input + features that shares the same voxel coordinates are reduced to + one row. + voxel_coors (torch.Tensor): [M, ndim]. Voxel coordinates. + """ + results = ext_module.dynamic_point_to_voxel_forward( + feats, coors, reduce_type) + (voxel_feats, voxel_coors, point2voxel_map, + voxel_points_count) = results + ctx.reduce_type = reduce_type + ctx.save_for_backward(feats, voxel_feats, point2voxel_map, + voxel_points_count) + ctx.mark_non_differentiable(voxel_coors) + return voxel_feats, voxel_coors + + @staticmethod + def backward(ctx, grad_voxel_feats, grad_voxel_coors=None): + (feats, voxel_feats, point2voxel_map, + voxel_points_count) = ctx.saved_tensors + grad_feats = torch.zeros_like(feats) + # TODO: whether to use index put or use cuda_backward + # To use index put, need point to voxel index + ext_module.dynamic_point_to_voxel_backward( + grad_feats, grad_voxel_feats.contiguous(), feats, voxel_feats, + point2voxel_map, voxel_points_count, ctx.reduce_type) + return grad_feats, None, None + + +dynamic_scatter = _DynamicScatter.apply + + +class DynamicScatter(nn.Module): + """Scatters points into voxels, used in the voxel encoder with dynamic + voxelization. + + Note: + The CPU and GPU implementation get the same output, but have numerical + difference after summation and division (e.g., 5e-7). + + Args: + voxel_size (list): list [x, y, z] size of three dimension. + point_cloud_range (list): The coordinate range of points, [x_min, + y_min, z_min, x_max, y_max, z_max]. + average_points (bool): whether to use avg pooling to scatter points + into voxel. + """ + + def __init__(self, voxel_size, point_cloud_range, average_points: bool): + super().__init__() + + self.voxel_size = voxel_size + self.point_cloud_range = point_cloud_range + self.average_points = average_points + + def forward_single(self, points, coors): + """Scatters points into voxels. + + Args: + points (torch.Tensor): Points to be reduced into voxels. + coors (torch.Tensor): Corresponding voxel coordinates (specifically + multi-dim voxel index) of each points. + + Returns: + voxel_feats (torch.Tensor): Reduced features, input features that + shares the same voxel coordinates are reduced to one row. + voxel_coors (torch.Tensor): Voxel coordinates. + """ + reduce = 'mean' if self.average_points else 'max' + return dynamic_scatter(points.contiguous(), coors.contiguous(), reduce) + + def forward(self, points, coors): + """Scatters points/features into voxels. + + Args: + points (torch.Tensor): Points to be reduced into voxels. + coors (torch.Tensor): Corresponding voxel coordinates (specifically + multi-dim voxel index) of each points. + + Returns: + voxel_feats (torch.Tensor): Reduced features, input features that + shares the same voxel coordinates are reduced to one row. + voxel_coors (torch.Tensor): Voxel coordinates. + """ + if coors.size(-1) == 3: + return self.forward_single(points, coors) + else: + batch_size = coors[-1, 0] + 1 + voxels, voxel_coors = [], [] + for i in range(batch_size): + inds = torch.where(coors[:, 0] == i) + voxel, voxel_coor = self.forward_single( + points[inds], coors[inds][:, 1:]) + coor_pad = nn.functional.pad( + voxel_coor, (1, 0), mode='constant', value=i) + voxel_coors.append(coor_pad) + voxels.append(voxel) + features = torch.cat(voxels, dim=0) + feature_coors = torch.cat(voxel_coors, dim=0) + + return features, feature_coors + + def __repr__(self): + s = self.__class__.__name__ + '(' + s += 'voxel_size=' + str(self.voxel_size) + s += ', point_cloud_range=' + str(self.point_cloud_range) + s += ', average_points=' + str(self.average_points) + s += ')' + return s diff --git a/annotator/mmpkg/mmcv/ops/sync_bn.py b/annotator/mmpkg/mmcv/ops/sync_bn.py new file mode 100644 index 0000000000000000000000000000000000000000..46db9200f9eafbad662a04e71f60a099a3178346 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/sync_bn.py @@ -0,0 +1,279 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.module import Module +from torch.nn.parameter import Parameter + +from annotator.mmpkg.mmcv.cnn import NORM_LAYERS +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', [ + 'sync_bn_forward_mean', 'sync_bn_forward_var', 'sync_bn_forward_output', + 'sync_bn_backward_param', 'sync_bn_backward_data' +]) + + +class SyncBatchNormFunction(Function): + + @staticmethod + def symbolic(g, input, running_mean, running_var, weight, bias, momentum, + eps, group, group_size, stats_mode): + return g.op( + 'mmcv::MMCVSyncBatchNorm', + input, + running_mean, + running_var, + weight, + bias, + momentum_f=momentum, + eps_f=eps, + group_i=group, + group_size_i=group_size, + stats_mode=stats_mode) + + @staticmethod + def forward(self, input, running_mean, running_var, weight, bias, momentum, + eps, group, group_size, stats_mode): + self.momentum = momentum + self.eps = eps + self.group = group + self.group_size = group_size + self.stats_mode = stats_mode + + assert isinstance( + input, (torch.HalfTensor, torch.FloatTensor, + torch.cuda.HalfTensor, torch.cuda.FloatTensor)), \ + f'only support Half or Float Tensor, but {input.type()}' + output = torch.zeros_like(input) + input3d = input.flatten(start_dim=2) + output3d = output.view_as(input3d) + num_channels = input3d.size(1) + + # ensure mean/var/norm/std are initialized as zeros + # ``torch.empty()`` does not guarantee that + mean = torch.zeros( + num_channels, dtype=torch.float, device=input3d.device) + var = torch.zeros( + num_channels, dtype=torch.float, device=input3d.device) + norm = torch.zeros_like( + input3d, dtype=torch.float, device=input3d.device) + std = torch.zeros( + num_channels, dtype=torch.float, device=input3d.device) + + batch_size = input3d.size(0) + if batch_size > 0: + ext_module.sync_bn_forward_mean(input3d, mean) + batch_flag = torch.ones([1], device=mean.device, dtype=mean.dtype) + else: + # skip updating mean and leave it as zeros when the input is empty + batch_flag = torch.zeros([1], device=mean.device, dtype=mean.dtype) + + # synchronize mean and the batch flag + vec = torch.cat([mean, batch_flag]) + if self.stats_mode == 'N': + vec *= batch_size + if self.group_size > 1: + dist.all_reduce(vec, group=self.group) + total_batch = vec[-1].detach() + mean = vec[:num_channels] + + if self.stats_mode == 'default': + mean = mean / self.group_size + elif self.stats_mode == 'N': + mean = mean / total_batch.clamp(min=1) + else: + raise NotImplementedError + + # leave var as zeros when the input is empty + if batch_size > 0: + ext_module.sync_bn_forward_var(input3d, mean, var) + + if self.stats_mode == 'N': + var *= batch_size + if self.group_size > 1: + dist.all_reduce(var, group=self.group) + + if self.stats_mode == 'default': + var /= self.group_size + elif self.stats_mode == 'N': + var /= total_batch.clamp(min=1) + else: + raise NotImplementedError + + # if the total batch size over all the ranks is zero, + # we should not update the statistics in the current batch + update_flag = total_batch.clamp(max=1) + momentum = update_flag * self.momentum + ext_module.sync_bn_forward_output( + input3d, + mean, + var, + weight, + bias, + running_mean, + running_var, + norm, + std, + output3d, + eps=self.eps, + momentum=momentum, + group_size=self.group_size) + self.save_for_backward(norm, std, weight) + return output + + @staticmethod + @once_differentiable + def backward(self, grad_output): + norm, std, weight = self.saved_tensors + grad_weight = torch.zeros_like(weight) + grad_bias = torch.zeros_like(weight) + grad_input = torch.zeros_like(grad_output) + grad_output3d = grad_output.flatten(start_dim=2) + grad_input3d = grad_input.view_as(grad_output3d) + + batch_size = grad_input3d.size(0) + if batch_size > 0: + ext_module.sync_bn_backward_param(grad_output3d, norm, grad_weight, + grad_bias) + + # all reduce + if self.group_size > 1: + dist.all_reduce(grad_weight, group=self.group) + dist.all_reduce(grad_bias, group=self.group) + grad_weight /= self.group_size + grad_bias /= self.group_size + + if batch_size > 0: + ext_module.sync_bn_backward_data(grad_output3d, weight, + grad_weight, grad_bias, norm, std, + grad_input3d) + + return grad_input, None, None, grad_weight, grad_bias, \ + None, None, None, None, None + + +@NORM_LAYERS.register_module(name='MMSyncBN') +class SyncBatchNorm(Module): + """Synchronized Batch Normalization. + + Args: + num_features (int): number of features/chennels in input tensor + eps (float, optional): a value added to the denominator for numerical + stability. Defaults to 1e-5. + momentum (float, optional): the value used for the running_mean and + running_var computation. Defaults to 0.1. + affine (bool, optional): whether to use learnable affine parameters. + Defaults to True. + track_running_stats (bool, optional): whether to track the running + mean and variance during training. When set to False, this + module does not track such statistics, and initializes statistics + buffers ``running_mean`` and ``running_var`` as ``None``. When + these buffers are ``None``, this module always uses batch + statistics in both training and eval modes. Defaults to True. + group (int, optional): synchronization of stats happen within + each process group individually. By default it is synchronization + across the whole world. Defaults to None. + stats_mode (str, optional): The statistical mode. Available options + includes ``'default'`` and ``'N'``. Defaults to 'default'. + When ``stats_mode=='default'``, it computes the overall statistics + using those from each worker with equal weight, i.e., the + statistics are synchronized and simply divied by ``group``. This + mode will produce inaccurate statistics when empty tensors occur. + When ``stats_mode=='N'``, it compute the overall statistics using + the total number of batches in each worker ignoring the number of + group, i.e., the statistics are synchronized and then divied by + the total batch ``N``. This mode is beneficial when empty tensors + occur during training, as it average the total mean by the real + number of batch. + """ + + def __init__(self, + num_features, + eps=1e-5, + momentum=0.1, + affine=True, + track_running_stats=True, + group=None, + stats_mode='default'): + super(SyncBatchNorm, self).__init__() + self.num_features = num_features + self.eps = eps + self.momentum = momentum + self.affine = affine + self.track_running_stats = track_running_stats + group = dist.group.WORLD if group is None else group + self.group = group + self.group_size = dist.get_world_size(group) + assert stats_mode in ['default', 'N'], \ + f'"stats_mode" only accepts "default" and "N", got "{stats_mode}"' + self.stats_mode = stats_mode + if self.affine: + self.weight = Parameter(torch.Tensor(num_features)) + self.bias = Parameter(torch.Tensor(num_features)) + else: + self.register_parameter('weight', None) + self.register_parameter('bias', None) + if self.track_running_stats: + self.register_buffer('running_mean', torch.zeros(num_features)) + self.register_buffer('running_var', torch.ones(num_features)) + self.register_buffer('num_batches_tracked', + torch.tensor(0, dtype=torch.long)) + else: + self.register_buffer('running_mean', None) + self.register_buffer('running_var', None) + self.register_buffer('num_batches_tracked', None) + self.reset_parameters() + + def reset_running_stats(self): + if self.track_running_stats: + self.running_mean.zero_() + self.running_var.fill_(1) + self.num_batches_tracked.zero_() + + def reset_parameters(self): + self.reset_running_stats() + if self.affine: + self.weight.data.uniform_() # pytorch use ones_() + self.bias.data.zero_() + + def forward(self, input): + if input.dim() < 2: + raise ValueError( + f'expected at least 2D input, got {input.dim()}D input') + if self.momentum is None: + exponential_average_factor = 0.0 + else: + exponential_average_factor = self.momentum + + if self.training and self.track_running_stats: + if self.num_batches_tracked is not None: + self.num_batches_tracked += 1 + if self.momentum is None: # use cumulative moving average + exponential_average_factor = 1.0 / float( + self.num_batches_tracked) + else: # use exponential moving average + exponential_average_factor = self.momentum + + if self.training or not self.track_running_stats: + return SyncBatchNormFunction.apply( + input, self.running_mean, self.running_var, self.weight, + self.bias, exponential_average_factor, self.eps, self.group, + self.group_size, self.stats_mode) + else: + return F.batch_norm(input, self.running_mean, self.running_var, + self.weight, self.bias, False, + exponential_average_factor, self.eps) + + def __repr__(self): + s = self.__class__.__name__ + s += f'({self.num_features}, ' + s += f'eps={self.eps}, ' + s += f'momentum={self.momentum}, ' + s += f'affine={self.affine}, ' + s += f'track_running_stats={self.track_running_stats}, ' + s += f'group_size={self.group_size},' + s += f'stats_mode={self.stats_mode})' + return s diff --git a/annotator/mmpkg/mmcv/ops/three_interpolate.py b/annotator/mmpkg/mmcv/ops/three_interpolate.py new file mode 100644 index 0000000000000000000000000000000000000000..203f47f05d58087e034fb3cd8cd6a09233947b4a --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/three_interpolate.py @@ -0,0 +1,68 @@ +from typing import Tuple + +import torch +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['three_interpolate_forward', 'three_interpolate_backward']) + + +class ThreeInterpolate(Function): + """Performs weighted linear interpolation on 3 features. + + Please refer to `Paper of PointNet++ `_ + for more details. + """ + + @staticmethod + def forward(ctx, features: torch.Tensor, indices: torch.Tensor, + weight: torch.Tensor) -> torch.Tensor: + """ + Args: + features (Tensor): (B, C, M) Features descriptors to be + interpolated + indices (Tensor): (B, n, 3) index three nearest neighbors + of the target features in features + weight (Tensor): (B, n, 3) weights of interpolation + + Returns: + Tensor: (B, C, N) tensor of the interpolated features + """ + assert features.is_contiguous() + assert indices.is_contiguous() + assert weight.is_contiguous() + + B, c, m = features.size() + n = indices.size(1) + ctx.three_interpolate_for_backward = (indices, weight, m) + output = torch.cuda.FloatTensor(B, c, n) + + ext_module.three_interpolate_forward( + features, indices, weight, output, b=B, c=c, m=m, n=n) + return output + + @staticmethod + def backward( + ctx, grad_out: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Args: + grad_out (Tensor): (B, C, N) tensor with gradients of outputs + + Returns: + Tensor: (B, C, M) tensor with gradients of features + """ + idx, weight, m = ctx.three_interpolate_for_backward + B, c, n = grad_out.size() + + grad_features = torch.cuda.FloatTensor(B, c, m).zero_() + grad_out_data = grad_out.data.contiguous() + + ext_module.three_interpolate_backward( + grad_out_data, idx, weight, grad_features.data, b=B, c=c, n=n, m=m) + return grad_features, None, None + + +three_interpolate = ThreeInterpolate.apply diff --git a/annotator/mmpkg/mmcv/ops/three_nn.py b/annotator/mmpkg/mmcv/ops/three_nn.py new file mode 100644 index 0000000000000000000000000000000000000000..2b01047a129989cd5545a0a86f23a487f4a13ce1 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/three_nn.py @@ -0,0 +1,51 @@ +from typing import Tuple + +import torch +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', ['three_nn_forward']) + + +class ThreeNN(Function): + """Find the top-3 nearest neighbors of the target set from the source set. + + Please refer to `Paper of PointNet++ `_ + for more details. + """ + + @staticmethod + def forward(ctx, target: torch.Tensor, + source: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Args: + target (Tensor): shape (B, N, 3), points set that needs to + find the nearest neighbors. + source (Tensor): shape (B, M, 3), points set that is used + to find the nearest neighbors of points in target set. + + Returns: + Tensor: shape (B, N, 3), L2 distance of each point in target + set to their corresponding nearest neighbors. + """ + target = target.contiguous() + source = source.contiguous() + + B, N, _ = target.size() + m = source.size(1) + dist2 = torch.cuda.FloatTensor(B, N, 3) + idx = torch.cuda.IntTensor(B, N, 3) + + ext_module.three_nn_forward(target, source, dist2, idx, b=B, n=N, m=m) + if torch.__version__ != 'parrots': + ctx.mark_non_differentiable(idx) + + return torch.sqrt(dist2), idx + + @staticmethod + def backward(ctx, a=None, b=None): + return None, None + + +three_nn = ThreeNN.apply diff --git a/annotator/mmpkg/mmcv/ops/tin_shift.py b/annotator/mmpkg/mmcv/ops/tin_shift.py new file mode 100644 index 0000000000000000000000000000000000000000..472c9fcfe45a124e819b7ed5653e585f94a8811e --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/tin_shift.py @@ -0,0 +1,68 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# Code reference from "Temporal Interlacing Network" +# https://github.com/deepcs233/TIN/blob/master/cuda_shift/rtc_wrap.py +# Hao Shao, Shengju Qian, Yu Liu +# shaoh19@mails.tsinghua.edu.cn, sjqian@cse.cuhk.edu.hk, yuliu@ee.cuhk.edu.hk + +import torch +import torch.nn as nn +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', + ['tin_shift_forward', 'tin_shift_backward']) + + +class TINShiftFunction(Function): + + @staticmethod + def forward(ctx, input, shift): + C = input.size(2) + num_segments = shift.size(1) + if C // num_segments <= 0 or C % num_segments != 0: + raise ValueError('C should be a multiple of num_segments, ' + f'but got C={C} and num_segments={num_segments}.') + + ctx.save_for_backward(shift) + + out = torch.zeros_like(input) + ext_module.tin_shift_forward(input, shift, out) + + return out + + @staticmethod + def backward(ctx, grad_output): + + shift = ctx.saved_tensors[0] + data_grad_input = grad_output.new(*grad_output.size()).zero_() + shift_grad_input = shift.new(*shift.size()).zero_() + ext_module.tin_shift_backward(grad_output, shift, data_grad_input) + + return data_grad_input, shift_grad_input + + +tin_shift = TINShiftFunction.apply + + +class TINShift(nn.Module): + """Temporal Interlace Shift. + + Temporal Interlace shift is a differentiable temporal-wise frame shifting + which is proposed in "Temporal Interlacing Network" + + Please refer to https://arxiv.org/abs/2001.06499 for more details. + Code is modified from https://github.com/mit-han-lab/temporal-shift-module + """ + + def forward(self, input, shift): + """Perform temporal interlace shift. + + Args: + input (Tensor): Feature map with shape [N, num_segments, C, H * W]. + shift (Tensor): Shift tensor with shape [N, num_segments]. + + Returns: + Feature map after temporal interlace shift. + """ + return tin_shift(input, shift) diff --git a/annotator/mmpkg/mmcv/ops/upfirdn2d.py b/annotator/mmpkg/mmcv/ops/upfirdn2d.py new file mode 100644 index 0000000000000000000000000000000000000000..751db20a344e1164748d8d4d8b2f775247925eab --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/upfirdn2d.py @@ -0,0 +1,330 @@ +# modified from https://github.com/rosinality/stylegan2-pytorch/blob/master/op/upfirdn2d.py # noqa:E501 + +# Copyright (c) 2021, NVIDIA Corporation. All rights reserved. +# NVIDIA Source Code License for StyleGAN2 with Adaptive Discriminator +# Augmentation (ADA) +# ======================================================================= + +# 1. Definitions + +# "Licensor" means any person or entity that distributes its Work. + +# "Software" means the original work of authorship made available under +# this License. + +# "Work" means the Software and any additions to or derivative works of +# the Software that are made available under this License. + +# The terms "reproduce," "reproduction," "derivative works," and +# "distribution" have the meaning as provided under U.S. copyright law; +# provided, however, that for the purposes of this License, derivative +# works shall not include works that remain separable from, or merely +# link (or bind by name) to the interfaces of, the Work. + +# Works, including the Software, are "made available" under this License +# by including in or with the Work either (a) a copyright notice +# referencing the applicability of this License to the Work, or (b) a +# copy of this License. + +# 2. License Grants + +# 2.1 Copyright Grant. Subject to the terms and conditions of this +# License, each Licensor grants to you a perpetual, worldwide, +# non-exclusive, royalty-free, copyright license to reproduce, +# prepare derivative works of, publicly display, publicly perform, +# sublicense and distribute its Work and any resulting derivative +# works in any form. + +# 3. Limitations + +# 3.1 Redistribution. You may reproduce or distribute the Work only +# if (a) you do so under this License, (b) you include a complete +# copy of this License with your distribution, and (c) you retain +# without modification any copyright, patent, trademark, or +# attribution notices that are present in the Work. + +# 3.2 Derivative Works. You may specify that additional or different +# terms apply to the use, reproduction, and distribution of your +# derivative works of the Work ("Your Terms") only if (a) Your Terms +# provide that the use limitation in Section 3.3 applies to your +# derivative works, and (b) you identify the specific derivative +# works that are subject to Your Terms. Notwithstanding Your Terms, +# this License (including the redistribution requirements in Section +# 3.1) will continue to apply to the Work itself. + +# 3.3 Use Limitation. The Work and any derivative works thereof only +# may be used or intended for use non-commercially. Notwithstanding +# the foregoing, NVIDIA and its affiliates may use the Work and any +# derivative works commercially. As used herein, "non-commercially" +# means for research or evaluation purposes only. + +# 3.4 Patent Claims. If you bring or threaten to bring a patent claim +# against any Licensor (including any claim, cross-claim or +# counterclaim in a lawsuit) to enforce any patents that you allege +# are infringed by any Work, then your rights under this License from +# such Licensor (including the grant in Section 2.1) will terminate +# immediately. + +# 3.5 Trademarks. This License does not grant any rights to use any +# Licensor’s or its affiliates’ names, logos, or trademarks, except +# as necessary to reproduce the notices described in this License. + +# 3.6 Termination. If you violate any term of this License, then your +# rights under this License (including the grant in Section 2.1) will +# terminate immediately. + +# 4. Disclaimer of Warranty. + +# THE WORK IS PROVIDED "AS IS" WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WARRANTIES OR CONDITIONS OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR +# NON-INFRINGEMENT. YOU BEAR THE RISK OF UNDERTAKING ANY ACTIVITIES UNDER +# THIS LICENSE. + +# 5. Limitation of Liability. + +# EXCEPT AS PROHIBITED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL +# THEORY, WHETHER IN TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE +# SHALL ANY LICENSOR BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, +# INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF +# OR RELATED TO THIS LICENSE, THE USE OR INABILITY TO USE THE WORK +# (INCLUDING BUT NOT LIMITED TO LOSS OF GOODWILL, BUSINESS INTERRUPTION, +# LOST PROFITS OR DATA, COMPUTER FAILURE OR MALFUNCTION, OR ANY OTHER +# COMMERCIAL DAMAGES OR LOSSES), EVEN IF THE LICENSOR HAS BEEN ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGES. + +# ======================================================================= + +import torch +from torch.autograd import Function +from torch.nn import functional as F + +from annotator.mmpkg.mmcv.utils import to_2tuple +from ..utils import ext_loader + +upfirdn2d_ext = ext_loader.load_ext('_ext', ['upfirdn2d']) + + +class UpFirDn2dBackward(Function): + + @staticmethod + def forward(ctx, grad_output, kernel, grad_kernel, up, down, pad, g_pad, + in_size, out_size): + + up_x, up_y = up + down_x, down_y = down + g_pad_x0, g_pad_x1, g_pad_y0, g_pad_y1 = g_pad + + grad_output = grad_output.reshape(-1, out_size[0], out_size[1], 1) + + grad_input = upfirdn2d_ext.upfirdn2d( + grad_output, + grad_kernel, + up_x=down_x, + up_y=down_y, + down_x=up_x, + down_y=up_y, + pad_x0=g_pad_x0, + pad_x1=g_pad_x1, + pad_y0=g_pad_y0, + pad_y1=g_pad_y1) + grad_input = grad_input.view(in_size[0], in_size[1], in_size[2], + in_size[3]) + + ctx.save_for_backward(kernel) + + pad_x0, pad_x1, pad_y0, pad_y1 = pad + + ctx.up_x = up_x + ctx.up_y = up_y + ctx.down_x = down_x + ctx.down_y = down_y + ctx.pad_x0 = pad_x0 + ctx.pad_x1 = pad_x1 + ctx.pad_y0 = pad_y0 + ctx.pad_y1 = pad_y1 + ctx.in_size = in_size + ctx.out_size = out_size + + return grad_input + + @staticmethod + def backward(ctx, gradgrad_input): + kernel, = ctx.saved_tensors + + gradgrad_input = gradgrad_input.reshape(-1, ctx.in_size[2], + ctx.in_size[3], 1) + + gradgrad_out = upfirdn2d_ext.upfirdn2d( + gradgrad_input, + kernel, + up_x=ctx.up_x, + up_y=ctx.up_y, + down_x=ctx.down_x, + down_y=ctx.down_y, + pad_x0=ctx.pad_x0, + pad_x1=ctx.pad_x1, + pad_y0=ctx.pad_y0, + pad_y1=ctx.pad_y1) + # gradgrad_out = gradgrad_out.view(ctx.in_size[0], ctx.out_size[0], + # ctx.out_size[1], ctx.in_size[3]) + gradgrad_out = gradgrad_out.view(ctx.in_size[0], ctx.in_size[1], + ctx.out_size[0], ctx.out_size[1]) + + return gradgrad_out, None, None, None, None, None, None, None, None + + +class UpFirDn2d(Function): + + @staticmethod + def forward(ctx, input, kernel, up, down, pad): + up_x, up_y = up + down_x, down_y = down + pad_x0, pad_x1, pad_y0, pad_y1 = pad + + kernel_h, kernel_w = kernel.shape + batch, channel, in_h, in_w = input.shape + ctx.in_size = input.shape + + input = input.reshape(-1, in_h, in_w, 1) + + ctx.save_for_backward(kernel, torch.flip(kernel, [0, 1])) + + out_h = (in_h * up_y + pad_y0 + pad_y1 - kernel_h) // down_y + 1 + out_w = (in_w * up_x + pad_x0 + pad_x1 - kernel_w) // down_x + 1 + ctx.out_size = (out_h, out_w) + + ctx.up = (up_x, up_y) + ctx.down = (down_x, down_y) + ctx.pad = (pad_x0, pad_x1, pad_y0, pad_y1) + + g_pad_x0 = kernel_w - pad_x0 - 1 + g_pad_y0 = kernel_h - pad_y0 - 1 + g_pad_x1 = in_w * up_x - out_w * down_x + pad_x0 - up_x + 1 + g_pad_y1 = in_h * up_y - out_h * down_y + pad_y0 - up_y + 1 + + ctx.g_pad = (g_pad_x0, g_pad_x1, g_pad_y0, g_pad_y1) + + out = upfirdn2d_ext.upfirdn2d( + input, + kernel, + up_x=up_x, + up_y=up_y, + down_x=down_x, + down_y=down_y, + pad_x0=pad_x0, + pad_x1=pad_x1, + pad_y0=pad_y0, + pad_y1=pad_y1) + # out = out.view(major, out_h, out_w, minor) + out = out.view(-1, channel, out_h, out_w) + + return out + + @staticmethod + def backward(ctx, grad_output): + kernel, grad_kernel = ctx.saved_tensors + + grad_input = UpFirDn2dBackward.apply( + grad_output, + kernel, + grad_kernel, + ctx.up, + ctx.down, + ctx.pad, + ctx.g_pad, + ctx.in_size, + ctx.out_size, + ) + + return grad_input, None, None, None, None + + +def upfirdn2d(input, kernel, up=1, down=1, pad=(0, 0)): + """UpFRIDn for 2d features. + + UpFIRDn is short for upsample, apply FIR filter and downsample. More + details can be found in: + https://www.mathworks.com/help/signal/ref/upfirdn.html + + Args: + input (Tensor): Tensor with shape of (n, c, h, w). + kernel (Tensor): Filter kernel. + up (int | tuple[int], optional): Upsampling factor. If given a number, + we will use this factor for the both height and width side. + Defaults to 1. + down (int | tuple[int], optional): Downsampling factor. If given a + number, we will use this factor for the both height and width side. + Defaults to 1. + pad (tuple[int], optional): Padding for tensors, (x_pad, y_pad) or + (x_pad_0, x_pad_1, y_pad_0, y_pad_1). Defaults to (0, 0). + + Returns: + Tensor: Tensor after UpFIRDn. + """ + if input.device.type == 'cpu': + if len(pad) == 2: + pad = (pad[0], pad[1], pad[0], pad[1]) + + up = to_2tuple(up) + + down = to_2tuple(down) + + out = upfirdn2d_native(input, kernel, up[0], up[1], down[0], down[1], + pad[0], pad[1], pad[2], pad[3]) + else: + _up = to_2tuple(up) + + _down = to_2tuple(down) + + if len(pad) == 4: + _pad = pad + elif len(pad) == 2: + _pad = (pad[0], pad[1], pad[0], pad[1]) + + out = UpFirDn2d.apply(input, kernel, _up, _down, _pad) + + return out + + +def upfirdn2d_native(input, kernel, up_x, up_y, down_x, down_y, pad_x0, pad_x1, + pad_y0, pad_y1): + _, channel, in_h, in_w = input.shape + input = input.reshape(-1, in_h, in_w, 1) + + _, in_h, in_w, minor = input.shape + kernel_h, kernel_w = kernel.shape + + out = input.view(-1, in_h, 1, in_w, 1, minor) + out = F.pad(out, [0, 0, 0, up_x - 1, 0, 0, 0, up_y - 1]) + out = out.view(-1, in_h * up_y, in_w * up_x, minor) + + out = F.pad( + out, + [0, 0, + max(pad_x0, 0), + max(pad_x1, 0), + max(pad_y0, 0), + max(pad_y1, 0)]) + out = out[:, + max(-pad_y0, 0):out.shape[1] - max(-pad_y1, 0), + max(-pad_x0, 0):out.shape[2] - max(-pad_x1, 0), :, ] + + out = out.permute(0, 3, 1, 2) + out = out.reshape( + [-1, 1, in_h * up_y + pad_y0 + pad_y1, in_w * up_x + pad_x0 + pad_x1]) + w = torch.flip(kernel, [0, 1]).view(1, 1, kernel_h, kernel_w) + out = F.conv2d(out, w) + out = out.reshape( + -1, + minor, + in_h * up_y + pad_y0 + pad_y1 - kernel_h + 1, + in_w * up_x + pad_x0 + pad_x1 - kernel_w + 1, + ) + out = out.permute(0, 2, 3, 1) + out = out[:, ::down_y, ::down_x, :] + + out_h = (in_h * up_y + pad_y0 + pad_y1 - kernel_h) // down_y + 1 + out_w = (in_w * up_x + pad_x0 + pad_x1 - kernel_w) // down_x + 1 + + return out.view(-1, channel, out_h, out_w) diff --git a/annotator/mmpkg/mmcv/ops/voxelize.py b/annotator/mmpkg/mmcv/ops/voxelize.py new file mode 100644 index 0000000000000000000000000000000000000000..ca3226a4fbcbfe58490fa2ea8e1c16b531214121 --- /dev/null +++ b/annotator/mmpkg/mmcv/ops/voxelize.py @@ -0,0 +1,132 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch import nn +from torch.autograd import Function +from torch.nn.modules.utils import _pair + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['dynamic_voxelize_forward', 'hard_voxelize_forward']) + + +class _Voxelization(Function): + + @staticmethod + def forward(ctx, + points, + voxel_size, + coors_range, + max_points=35, + max_voxels=20000): + """Convert kitti points(N, >=3) to voxels. + + Args: + points (torch.Tensor): [N, ndim]. Points[:, :3] contain xyz points + and points[:, 3:] contain other information like reflectivity. + voxel_size (tuple or float): The size of voxel with the shape of + [3]. + coors_range (tuple or float): The coordinate range of voxel with + the shape of [6]. + max_points (int, optional): maximum points contained in a voxel. if + max_points=-1, it means using dynamic_voxelize. Default: 35. + max_voxels (int, optional): maximum voxels this function create. + for second, 20000 is a good choice. Users should shuffle points + before call this function because max_voxels may drop points. + Default: 20000. + + Returns: + voxels_out (torch.Tensor): Output voxels with the shape of [M, + max_points, ndim]. Only contain points and returned when + max_points != -1. + coors_out (torch.Tensor): Output coordinates with the shape of + [M, 3]. + num_points_per_voxel_out (torch.Tensor): Num points per voxel with + the shape of [M]. Only returned when max_points != -1. + """ + if max_points == -1 or max_voxels == -1: + coors = points.new_zeros(size=(points.size(0), 3), dtype=torch.int) + ext_module.dynamic_voxelize_forward(points, coors, voxel_size, + coors_range, 3) + return coors + else: + voxels = points.new_zeros( + size=(max_voxels, max_points, points.size(1))) + coors = points.new_zeros(size=(max_voxels, 3), dtype=torch.int) + num_points_per_voxel = points.new_zeros( + size=(max_voxels, ), dtype=torch.int) + voxel_num = ext_module.hard_voxelize_forward( + points, voxels, coors, num_points_per_voxel, voxel_size, + coors_range, max_points, max_voxels, 3) + # select the valid voxels + voxels_out = voxels[:voxel_num] + coors_out = coors[:voxel_num] + num_points_per_voxel_out = num_points_per_voxel[:voxel_num] + return voxels_out, coors_out, num_points_per_voxel_out + + +voxelization = _Voxelization.apply + + +class Voxelization(nn.Module): + """Convert kitti points(N, >=3) to voxels. + + Please refer to `PVCNN `_ for more + details. + + Args: + voxel_size (tuple or float): The size of voxel with the shape of [3]. + point_cloud_range (tuple or float): The coordinate range of voxel with + the shape of [6]. + max_num_points (int): maximum points contained in a voxel. if + max_points=-1, it means using dynamic_voxelize. + max_voxels (int, optional): maximum voxels this function create. + for second, 20000 is a good choice. Users should shuffle points + before call this function because max_voxels may drop points. + Default: 20000. + """ + + def __init__(self, + voxel_size, + point_cloud_range, + max_num_points, + max_voxels=20000): + super().__init__() + + self.voxel_size = voxel_size + self.point_cloud_range = point_cloud_range + self.max_num_points = max_num_points + if isinstance(max_voxels, tuple): + self.max_voxels = max_voxels + else: + self.max_voxels = _pair(max_voxels) + + point_cloud_range = torch.tensor( + point_cloud_range, dtype=torch.float32) + voxel_size = torch.tensor(voxel_size, dtype=torch.float32) + grid_size = (point_cloud_range[3:] - + point_cloud_range[:3]) / voxel_size + grid_size = torch.round(grid_size).long() + input_feat_shape = grid_size[:2] + self.grid_size = grid_size + # the origin shape is as [x-len, y-len, z-len] + # [w, h, d] -> [d, h, w] + self.pcd_shape = [*input_feat_shape, 1][::-1] + + def forward(self, input): + if self.training: + max_voxels = self.max_voxels[0] + else: + max_voxels = self.max_voxels[1] + + return voxelization(input, self.voxel_size, self.point_cloud_range, + self.max_num_points, max_voxels) + + def __repr__(self): + s = self.__class__.__name__ + '(' + s += 'voxel_size=' + str(self.voxel_size) + s += ', point_cloud_range=' + str(self.point_cloud_range) + s += ', max_num_points=' + str(self.max_num_points) + s += ', max_voxels=' + str(self.max_voxels) + s += ')' + return s diff --git a/annotator/mmpkg/mmcv/parallel/__init__.py b/annotator/mmpkg/mmcv/parallel/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2ed2c17ad357742e423beeaf4d35db03fe9af469 --- /dev/null +++ b/annotator/mmpkg/mmcv/parallel/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .collate import collate +from .data_container import DataContainer +from .data_parallel import MMDataParallel +from .distributed import MMDistributedDataParallel +from .registry import MODULE_WRAPPERS +from .scatter_gather import scatter, scatter_kwargs +from .utils import is_module_wrapper + +__all__ = [ + 'collate', 'DataContainer', 'MMDataParallel', 'MMDistributedDataParallel', + 'scatter', 'scatter_kwargs', 'is_module_wrapper', 'MODULE_WRAPPERS' +] diff --git a/annotator/mmpkg/mmcv/parallel/_functions.py b/annotator/mmpkg/mmcv/parallel/_functions.py new file mode 100644 index 0000000000000000000000000000000000000000..9b5a8a44483ab991411d07122b22a1d027e4be8e --- /dev/null +++ b/annotator/mmpkg/mmcv/parallel/_functions.py @@ -0,0 +1,79 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch.nn.parallel._functions import _get_stream + + +def scatter(input, devices, streams=None): + """Scatters tensor across multiple GPUs.""" + if streams is None: + streams = [None] * len(devices) + + if isinstance(input, list): + chunk_size = (len(input) - 1) // len(devices) + 1 + outputs = [ + scatter(input[i], [devices[i // chunk_size]], + [streams[i // chunk_size]]) for i in range(len(input)) + ] + return outputs + elif isinstance(input, torch.Tensor): + output = input.contiguous() + # TODO: copy to a pinned buffer first (if copying from CPU) + stream = streams[0] if output.numel() > 0 else None + if devices != [-1]: + with torch.cuda.device(devices[0]), torch.cuda.stream(stream): + output = output.cuda(devices[0], non_blocking=True) + else: + # unsqueeze the first dimension thus the tensor's shape is the + # same as those scattered with GPU. + output = output.unsqueeze(0) + return output + else: + raise Exception(f'Unknown type {type(input)}.') + + +def synchronize_stream(output, devices, streams): + if isinstance(output, list): + chunk_size = len(output) // len(devices) + for i in range(len(devices)): + for j in range(chunk_size): + synchronize_stream(output[i * chunk_size + j], [devices[i]], + [streams[i]]) + elif isinstance(output, torch.Tensor): + if output.numel() != 0: + with torch.cuda.device(devices[0]): + main_stream = torch.cuda.current_stream() + main_stream.wait_stream(streams[0]) + output.record_stream(main_stream) + else: + raise Exception(f'Unknown type {type(output)}.') + + +def get_input_device(input): + if isinstance(input, list): + for item in input: + input_device = get_input_device(item) + if input_device != -1: + return input_device + return -1 + elif isinstance(input, torch.Tensor): + return input.get_device() if input.is_cuda else -1 + else: + raise Exception(f'Unknown type {type(input)}.') + + +class Scatter: + + @staticmethod + def forward(target_gpus, input): + input_device = get_input_device(input) + streams = None + if input_device == -1 and target_gpus != [-1]: + # Perform CPU to GPU copies in a background stream + streams = [_get_stream(device) for device in target_gpus] + + outputs = scatter(input, target_gpus, streams) + # Synchronize with the copy stream + if streams is not None: + synchronize_stream(outputs, target_gpus, streams) + + return tuple(outputs) diff --git a/annotator/mmpkg/mmcv/parallel/collate.py b/annotator/mmpkg/mmcv/parallel/collate.py new file mode 100644 index 0000000000000000000000000000000000000000..ad749197df21b0d74297548be5f66a696adebf7f --- /dev/null +++ b/annotator/mmpkg/mmcv/parallel/collate.py @@ -0,0 +1,84 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from collections.abc import Mapping, Sequence + +import torch +import torch.nn.functional as F +from torch.utils.data.dataloader import default_collate + +from .data_container import DataContainer + + +def collate(batch, samples_per_gpu=1): + """Puts each data field into a tensor/DataContainer with outer dimension + batch size. + + Extend default_collate to add support for + :type:`~mmcv.parallel.DataContainer`. There are 3 cases. + + 1. cpu_only = True, e.g., meta data + 2. cpu_only = False, stack = True, e.g., images tensors + 3. cpu_only = False, stack = False, e.g., gt bboxes + """ + + if not isinstance(batch, Sequence): + raise TypeError(f'{batch.dtype} is not supported.') + + if isinstance(batch[0], DataContainer): + stacked = [] + if batch[0].cpu_only: + for i in range(0, len(batch), samples_per_gpu): + stacked.append( + [sample.data for sample in batch[i:i + samples_per_gpu]]) + return DataContainer( + stacked, batch[0].stack, batch[0].padding_value, cpu_only=True) + elif batch[0].stack: + for i in range(0, len(batch), samples_per_gpu): + assert isinstance(batch[i].data, torch.Tensor) + + if batch[i].pad_dims is not None: + ndim = batch[i].dim() + assert ndim > batch[i].pad_dims + max_shape = [0 for _ in range(batch[i].pad_dims)] + for dim in range(1, batch[i].pad_dims + 1): + max_shape[dim - 1] = batch[i].size(-dim) + for sample in batch[i:i + samples_per_gpu]: + for dim in range(0, ndim - batch[i].pad_dims): + assert batch[i].size(dim) == sample.size(dim) + for dim in range(1, batch[i].pad_dims + 1): + max_shape[dim - 1] = max(max_shape[dim - 1], + sample.size(-dim)) + padded_samples = [] + for sample in batch[i:i + samples_per_gpu]: + pad = [0 for _ in range(batch[i].pad_dims * 2)] + for dim in range(1, batch[i].pad_dims + 1): + pad[2 * dim - + 1] = max_shape[dim - 1] - sample.size(-dim) + padded_samples.append( + F.pad( + sample.data, pad, value=sample.padding_value)) + stacked.append(default_collate(padded_samples)) + elif batch[i].pad_dims is None: + stacked.append( + default_collate([ + sample.data + for sample in batch[i:i + samples_per_gpu] + ])) + else: + raise ValueError( + 'pad_dims should be either None or integers (1-3)') + + else: + for i in range(0, len(batch), samples_per_gpu): + stacked.append( + [sample.data for sample in batch[i:i + samples_per_gpu]]) + return DataContainer(stacked, batch[0].stack, batch[0].padding_value) + elif isinstance(batch[0], Sequence): + transposed = zip(*batch) + return [collate(samples, samples_per_gpu) for samples in transposed] + elif isinstance(batch[0], Mapping): + return { + key: collate([d[key] for d in batch], samples_per_gpu) + for key in batch[0] + } + else: + return default_collate(batch) diff --git a/annotator/mmpkg/mmcv/parallel/data_container.py b/annotator/mmpkg/mmcv/parallel/data_container.py new file mode 100644 index 0000000000000000000000000000000000000000..cedb0d32a51a1f575a622b38de2cee3ab4757821 --- /dev/null +++ b/annotator/mmpkg/mmcv/parallel/data_container.py @@ -0,0 +1,89 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import functools + +import torch + + +def assert_tensor_type(func): + + @functools.wraps(func) + def wrapper(*args, **kwargs): + if not isinstance(args[0].data, torch.Tensor): + raise AttributeError( + f'{args[0].__class__.__name__} has no attribute ' + f'{func.__name__} for type {args[0].datatype}') + return func(*args, **kwargs) + + return wrapper + + +class DataContainer: + """A container for any type of objects. + + Typically tensors will be stacked in the collate function and sliced along + some dimension in the scatter function. This behavior has some limitations. + 1. All tensors have to be the same size. + 2. Types are limited (numpy array or Tensor). + + We design `DataContainer` and `MMDataParallel` to overcome these + limitations. The behavior can be either of the following. + + - copy to GPU, pad all tensors to the same size and stack them + - copy to GPU without stacking + - leave the objects as is and pass it to the model + - pad_dims specifies the number of last few dimensions to do padding + """ + + def __init__(self, + data, + stack=False, + padding_value=0, + cpu_only=False, + pad_dims=2): + self._data = data + self._cpu_only = cpu_only + self._stack = stack + self._padding_value = padding_value + assert pad_dims in [None, 1, 2, 3] + self._pad_dims = pad_dims + + def __repr__(self): + return f'{self.__class__.__name__}({repr(self.data)})' + + def __len__(self): + return len(self._data) + + @property + def data(self): + return self._data + + @property + def datatype(self): + if isinstance(self.data, torch.Tensor): + return self.data.type() + else: + return type(self.data) + + @property + def cpu_only(self): + return self._cpu_only + + @property + def stack(self): + return self._stack + + @property + def padding_value(self): + return self._padding_value + + @property + def pad_dims(self): + return self._pad_dims + + @assert_tensor_type + def size(self, *args, **kwargs): + return self.data.size(*args, **kwargs) + + @assert_tensor_type + def dim(self): + return self.data.dim() diff --git a/annotator/mmpkg/mmcv/parallel/data_parallel.py b/annotator/mmpkg/mmcv/parallel/data_parallel.py new file mode 100644 index 0000000000000000000000000000000000000000..79b5f69b654cf647dc7ae9174223781ab5c607d2 --- /dev/null +++ b/annotator/mmpkg/mmcv/parallel/data_parallel.py @@ -0,0 +1,89 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from itertools import chain + +from torch.nn.parallel import DataParallel + +from .scatter_gather import scatter_kwargs + + +class MMDataParallel(DataParallel): + """The DataParallel module that supports DataContainer. + + MMDataParallel has two main differences with PyTorch DataParallel: + + - It supports a custom type :class:`DataContainer` which allows more + flexible control of input data during both GPU and CPU inference. + - It implement two more APIs ``train_step()`` and ``val_step()``. + + Args: + module (:class:`nn.Module`): Module to be encapsulated. + device_ids (list[int]): Device IDS of modules to be scattered to. + Defaults to None when GPU is not available. + output_device (str | int): Device ID for output. Defaults to None. + dim (int): Dimension used to scatter the data. Defaults to 0. + """ + + def __init__(self, *args, dim=0, **kwargs): + super(MMDataParallel, self).__init__(*args, dim=dim, **kwargs) + self.dim = dim + + def forward(self, *inputs, **kwargs): + """Override the original forward function. + + The main difference lies in the CPU inference where the data in + :class:`DataContainers` will still be gathered. + """ + if not self.device_ids: + # We add the following line thus the module could gather and + # convert data containers as those in GPU inference + inputs, kwargs = self.scatter(inputs, kwargs, [-1]) + return self.module(*inputs[0], **kwargs[0]) + else: + return super().forward(*inputs, **kwargs) + + def scatter(self, inputs, kwargs, device_ids): + return scatter_kwargs(inputs, kwargs, device_ids, dim=self.dim) + + def train_step(self, *inputs, **kwargs): + if not self.device_ids: + # We add the following line thus the module could gather and + # convert data containers as those in GPU inference + inputs, kwargs = self.scatter(inputs, kwargs, [-1]) + return self.module.train_step(*inputs[0], **kwargs[0]) + + assert len(self.device_ids) == 1, \ + ('MMDataParallel only supports single GPU training, if you need to' + ' train with multiple GPUs, please use MMDistributedDataParallel' + 'instead.') + + for t in chain(self.module.parameters(), self.module.buffers()): + if t.device != self.src_device_obj: + raise RuntimeError( + 'module must have its parameters and buffers ' + f'on device {self.src_device_obj} (device_ids[0]) but ' + f'found one of them on device: {t.device}') + + inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids) + return self.module.train_step(*inputs[0], **kwargs[0]) + + def val_step(self, *inputs, **kwargs): + if not self.device_ids: + # We add the following line thus the module could gather and + # convert data containers as those in GPU inference + inputs, kwargs = self.scatter(inputs, kwargs, [-1]) + return self.module.val_step(*inputs[0], **kwargs[0]) + + assert len(self.device_ids) == 1, \ + ('MMDataParallel only supports single GPU training, if you need to' + ' train with multiple GPUs, please use MMDistributedDataParallel' + ' instead.') + + for t in chain(self.module.parameters(), self.module.buffers()): + if t.device != self.src_device_obj: + raise RuntimeError( + 'module must have its parameters and buffers ' + f'on device {self.src_device_obj} (device_ids[0]) but ' + f'found one of them on device: {t.device}') + + inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids) + return self.module.val_step(*inputs[0], **kwargs[0]) diff --git a/annotator/mmpkg/mmcv/parallel/distributed.py b/annotator/mmpkg/mmcv/parallel/distributed.py new file mode 100644 index 0000000000000000000000000000000000000000..929c7a451a7443d715ab0cceef530c53eff44cb9 --- /dev/null +++ b/annotator/mmpkg/mmcv/parallel/distributed.py @@ -0,0 +1,112 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch.nn.parallel.distributed import (DistributedDataParallel, + _find_tensors) + +from annotator.mmpkg.mmcv import print_log +from annotator.mmpkg.mmcv.utils import TORCH_VERSION, digit_version +from .scatter_gather import scatter_kwargs + + +class MMDistributedDataParallel(DistributedDataParallel): + """The DDP module that supports DataContainer. + + MMDDP has two main differences with PyTorch DDP: + + - It supports a custom type :class:`DataContainer` which allows more + flexible control of input data. + - It implement two APIs ``train_step()`` and ``val_step()``. + """ + + def to_kwargs(self, inputs, kwargs, device_id): + # Use `self.to_kwargs` instead of `self.scatter` in pytorch1.8 + # to move all tensors to device_id + return scatter_kwargs(inputs, kwargs, [device_id], dim=self.dim) + + def scatter(self, inputs, kwargs, device_ids): + return scatter_kwargs(inputs, kwargs, device_ids, dim=self.dim) + + def train_step(self, *inputs, **kwargs): + """train_step() API for module wrapped by DistributedDataParallel. + + This method is basically the same as + ``DistributedDataParallel.forward()``, while replacing + ``self.module.forward()`` with ``self.module.train_step()``. + It is compatible with PyTorch 1.1 - 1.5. + """ + + # In PyTorch >= 1.7, ``reducer._rebuild_buckets()`` is moved from the + # end of backward to the beginning of forward. + if ('parrots' not in TORCH_VERSION + and digit_version(TORCH_VERSION) >= digit_version('1.7') + and self.reducer._rebuild_buckets()): + print_log( + 'Reducer buckets have been rebuilt in this iteration.', + logger='mmcv') + + if getattr(self, 'require_forward_param_sync', True): + self._sync_params() + if self.device_ids: + inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids) + if len(self.device_ids) == 1: + output = self.module.train_step(*inputs[0], **kwargs[0]) + else: + outputs = self.parallel_apply( + self._module_copies[:len(inputs)], inputs, kwargs) + output = self.gather(outputs, self.output_device) + else: + output = self.module.train_step(*inputs, **kwargs) + + if torch.is_grad_enabled() and getattr( + self, 'require_backward_grad_sync', True): + if self.find_unused_parameters: + self.reducer.prepare_for_backward(list(_find_tensors(output))) + else: + self.reducer.prepare_for_backward([]) + else: + if ('parrots' not in TORCH_VERSION + and digit_version(TORCH_VERSION) > digit_version('1.2')): + self.require_forward_param_sync = False + return output + + def val_step(self, *inputs, **kwargs): + """val_step() API for module wrapped by DistributedDataParallel. + + This method is basically the same as + ``DistributedDataParallel.forward()``, while replacing + ``self.module.forward()`` with ``self.module.val_step()``. + It is compatible with PyTorch 1.1 - 1.5. + """ + # In PyTorch >= 1.7, ``reducer._rebuild_buckets()`` is moved from the + # end of backward to the beginning of forward. + if ('parrots' not in TORCH_VERSION + and digit_version(TORCH_VERSION) >= digit_version('1.7') + and self.reducer._rebuild_buckets()): + print_log( + 'Reducer buckets have been rebuilt in this iteration.', + logger='mmcv') + + if getattr(self, 'require_forward_param_sync', True): + self._sync_params() + if self.device_ids: + inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids) + if len(self.device_ids) == 1: + output = self.module.val_step(*inputs[0], **kwargs[0]) + else: + outputs = self.parallel_apply( + self._module_copies[:len(inputs)], inputs, kwargs) + output = self.gather(outputs, self.output_device) + else: + output = self.module.val_step(*inputs, **kwargs) + + if torch.is_grad_enabled() and getattr( + self, 'require_backward_grad_sync', True): + if self.find_unused_parameters: + self.reducer.prepare_for_backward(list(_find_tensors(output))) + else: + self.reducer.prepare_for_backward([]) + else: + if ('parrots' not in TORCH_VERSION + and digit_version(TORCH_VERSION) > digit_version('1.2')): + self.require_forward_param_sync = False + return output diff --git a/annotator/mmpkg/mmcv/parallel/distributed_deprecated.py b/annotator/mmpkg/mmcv/parallel/distributed_deprecated.py new file mode 100644 index 0000000000000000000000000000000000000000..be60a37041fc6a76deae1851dde30448eaff054f --- /dev/null +++ b/annotator/mmpkg/mmcv/parallel/distributed_deprecated.py @@ -0,0 +1,70 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.distributed as dist +import torch.nn as nn +from torch._utils import (_flatten_dense_tensors, _take_tensors, + _unflatten_dense_tensors) + +from annotator.mmpkg.mmcv.utils import TORCH_VERSION, digit_version +from .registry import MODULE_WRAPPERS +from .scatter_gather import scatter_kwargs + + +@MODULE_WRAPPERS.register_module() +class MMDistributedDataParallel(nn.Module): + + def __init__(self, + module, + dim=0, + broadcast_buffers=True, + bucket_cap_mb=25): + super(MMDistributedDataParallel, self).__init__() + self.module = module + self.dim = dim + self.broadcast_buffers = broadcast_buffers + + self.broadcast_bucket_size = bucket_cap_mb * 1024 * 1024 + self._sync_params() + + def _dist_broadcast_coalesced(self, tensors, buffer_size): + for tensors in _take_tensors(tensors, buffer_size): + flat_tensors = _flatten_dense_tensors(tensors) + dist.broadcast(flat_tensors, 0) + for tensor, synced in zip( + tensors, _unflatten_dense_tensors(flat_tensors, tensors)): + tensor.copy_(synced) + + def _sync_params(self): + module_states = list(self.module.state_dict().values()) + if len(module_states) > 0: + self._dist_broadcast_coalesced(module_states, + self.broadcast_bucket_size) + if self.broadcast_buffers: + if (TORCH_VERSION != 'parrots' + and digit_version(TORCH_VERSION) < digit_version('1.0')): + buffers = [b.data for b in self.module._all_buffers()] + else: + buffers = [b.data for b in self.module.buffers()] + if len(buffers) > 0: + self._dist_broadcast_coalesced(buffers, + self.broadcast_bucket_size) + + def scatter(self, inputs, kwargs, device_ids): + return scatter_kwargs(inputs, kwargs, device_ids, dim=self.dim) + + def forward(self, *inputs, **kwargs): + inputs, kwargs = self.scatter(inputs, kwargs, + [torch.cuda.current_device()]) + return self.module(*inputs[0], **kwargs[0]) + + def train_step(self, *inputs, **kwargs): + inputs, kwargs = self.scatter(inputs, kwargs, + [torch.cuda.current_device()]) + output = self.module.train_step(*inputs[0], **kwargs[0]) + return output + + def val_step(self, *inputs, **kwargs): + inputs, kwargs = self.scatter(inputs, kwargs, + [torch.cuda.current_device()]) + output = self.module.val_step(*inputs[0], **kwargs[0]) + return output diff --git a/annotator/mmpkg/mmcv/parallel/registry.py b/annotator/mmpkg/mmcv/parallel/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..6ce151e5f890691e8b583e5d50b492801bae82bd --- /dev/null +++ b/annotator/mmpkg/mmcv/parallel/registry.py @@ -0,0 +1,8 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from torch.nn.parallel import DataParallel, DistributedDataParallel + +from annotator.mmpkg.mmcv.utils import Registry + +MODULE_WRAPPERS = Registry('module wrapper') +MODULE_WRAPPERS.register_module(module=DataParallel) +MODULE_WRAPPERS.register_module(module=DistributedDataParallel) diff --git a/annotator/mmpkg/mmcv/parallel/scatter_gather.py b/annotator/mmpkg/mmcv/parallel/scatter_gather.py new file mode 100644 index 0000000000000000000000000000000000000000..900ff88566f8f14830590459dc4fd16d4b382e47 --- /dev/null +++ b/annotator/mmpkg/mmcv/parallel/scatter_gather.py @@ -0,0 +1,59 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch.nn.parallel._functions import Scatter as OrigScatter + +from ._functions import Scatter +from .data_container import DataContainer + + +def scatter(inputs, target_gpus, dim=0): + """Scatter inputs to target gpus. + + The only difference from original :func:`scatter` is to add support for + :type:`~mmcv.parallel.DataContainer`. + """ + + def scatter_map(obj): + if isinstance(obj, torch.Tensor): + if target_gpus != [-1]: + return OrigScatter.apply(target_gpus, None, dim, obj) + else: + # for CPU inference we use self-implemented scatter + return Scatter.forward(target_gpus, obj) + if isinstance(obj, DataContainer): + if obj.cpu_only: + return obj.data + else: + return Scatter.forward(target_gpus, obj.data) + if isinstance(obj, tuple) and len(obj) > 0: + return list(zip(*map(scatter_map, obj))) + if isinstance(obj, list) and len(obj) > 0: + out = list(map(list, zip(*map(scatter_map, obj)))) + return out + if isinstance(obj, dict) and len(obj) > 0: + out = list(map(type(obj), zip(*map(scatter_map, obj.items())))) + return out + return [obj for targets in target_gpus] + + # After scatter_map is called, a scatter_map cell will exist. This cell + # has a reference to the actual function scatter_map, which has references + # to a closure that has a reference to the scatter_map cell (because the + # fn is recursive). To avoid this reference cycle, we set the function to + # None, clearing the cell + try: + return scatter_map(inputs) + finally: + scatter_map = None + + +def scatter_kwargs(inputs, kwargs, target_gpus, dim=0): + """Scatter with support for kwargs dictionary.""" + inputs = scatter(inputs, target_gpus, dim) if inputs else [] + kwargs = scatter(kwargs, target_gpus, dim) if kwargs else [] + if len(inputs) < len(kwargs): + inputs.extend([() for _ in range(len(kwargs) - len(inputs))]) + elif len(kwargs) < len(inputs): + kwargs.extend([{} for _ in range(len(inputs) - len(kwargs))]) + inputs = tuple(inputs) + kwargs = tuple(kwargs) + return inputs, kwargs diff --git a/annotator/mmpkg/mmcv/parallel/utils.py b/annotator/mmpkg/mmcv/parallel/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0f5712cb42c38a2e8563bf563efb6681383cab9b --- /dev/null +++ b/annotator/mmpkg/mmcv/parallel/utils.py @@ -0,0 +1,20 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .registry import MODULE_WRAPPERS + + +def is_module_wrapper(module): + """Check if a module is a module wrapper. + + The following 3 modules in MMCV (and their subclasses) are regarded as + module wrappers: DataParallel, DistributedDataParallel, + MMDistributedDataParallel (the deprecated version). You may add you own + module wrapper by registering it to mmcv.parallel.MODULE_WRAPPERS. + + Args: + module (nn.Module): The module to be checked. + + Returns: + bool: True if the input module is a module wrapper. + """ + module_wrappers = tuple(MODULE_WRAPPERS.module_dict.values()) + return isinstance(module, module_wrappers) diff --git a/annotator/mmpkg/mmcv/runner/__init__.py b/annotator/mmpkg/mmcv/runner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..52e4b48d383a84a055dcd7f6236f6e8e58eab924 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/__init__.py @@ -0,0 +1,47 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .base_module import BaseModule, ModuleList, Sequential +from .base_runner import BaseRunner +from .builder import RUNNERS, build_runner +from .checkpoint import (CheckpointLoader, _load_checkpoint, + _load_checkpoint_with_prefix, load_checkpoint, + load_state_dict, save_checkpoint, weights_to_cpu) +from .default_constructor import DefaultRunnerConstructor +from .dist_utils import (allreduce_grads, allreduce_params, get_dist_info, + init_dist, master_only) +from .epoch_based_runner import EpochBasedRunner, Runner +from .fp16_utils import LossScaler, auto_fp16, force_fp32, wrap_fp16_model +from .hooks import (HOOKS, CheckpointHook, ClosureHook, DistEvalHook, + DistSamplerSeedHook, DvcliveLoggerHook, EMAHook, EvalHook, + Fp16OptimizerHook, GradientCumulativeFp16OptimizerHook, + GradientCumulativeOptimizerHook, Hook, IterTimerHook, + LoggerHook, LrUpdaterHook, MlflowLoggerHook, + NeptuneLoggerHook, OptimizerHook, PaviLoggerHook, + SyncBuffersHook, TensorboardLoggerHook, TextLoggerHook, + WandbLoggerHook) +from .iter_based_runner import IterBasedRunner, IterLoader +from .log_buffer import LogBuffer +from .optimizer import (OPTIMIZER_BUILDERS, OPTIMIZERS, + DefaultOptimizerConstructor, build_optimizer, + build_optimizer_constructor) +from .priority import Priority, get_priority +from .utils import get_host_info, get_time_str, obj_from_dict, set_random_seed + +__all__ = [ + 'BaseRunner', 'Runner', 'EpochBasedRunner', 'IterBasedRunner', 'LogBuffer', + 'HOOKS', 'Hook', 'CheckpointHook', 'ClosureHook', 'LrUpdaterHook', + 'OptimizerHook', 'IterTimerHook', 'DistSamplerSeedHook', 'LoggerHook', + 'PaviLoggerHook', 'TextLoggerHook', 'TensorboardLoggerHook', + 'NeptuneLoggerHook', 'WandbLoggerHook', 'MlflowLoggerHook', + 'DvcliveLoggerHook', '_load_checkpoint', 'load_state_dict', + 'load_checkpoint', 'weights_to_cpu', 'save_checkpoint', 'Priority', + 'get_priority', 'get_host_info', 'get_time_str', 'obj_from_dict', + 'init_dist', 'get_dist_info', 'master_only', 'OPTIMIZER_BUILDERS', + 'OPTIMIZERS', 'DefaultOptimizerConstructor', 'build_optimizer', + 'build_optimizer_constructor', 'IterLoader', 'set_random_seed', + 'auto_fp16', 'force_fp32', 'wrap_fp16_model', 'Fp16OptimizerHook', + 'SyncBuffersHook', 'EMAHook', 'build_runner', 'RUNNERS', 'allreduce_grads', + 'allreduce_params', 'LossScaler', 'CheckpointLoader', 'BaseModule', + '_load_checkpoint_with_prefix', 'EvalHook', 'DistEvalHook', 'Sequential', + 'ModuleList', 'GradientCumulativeOptimizerHook', + 'GradientCumulativeFp16OptimizerHook', 'DefaultRunnerConstructor' +] diff --git a/annotator/mmpkg/mmcv/runner/base_module.py b/annotator/mmpkg/mmcv/runner/base_module.py new file mode 100644 index 0000000000000000000000000000000000000000..72e1164dfc442056cdc386050177f011b4e9900f --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/base_module.py @@ -0,0 +1,195 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy +import warnings +from abc import ABCMeta +from collections import defaultdict +from logging import FileHandler + +import torch.nn as nn + +from annotator.mmpkg.mmcv.runner.dist_utils import master_only +from annotator.mmpkg.mmcv.utils.logging import get_logger, logger_initialized, print_log + + +class BaseModule(nn.Module, metaclass=ABCMeta): + """Base module for all modules in openmmlab. + + ``BaseModule`` is a wrapper of ``torch.nn.Module`` with additional + functionality of parameter initialization. Compared with + ``torch.nn.Module``, ``BaseModule`` mainly adds three attributes. + + - ``init_cfg``: the config to control the initialization. + - ``init_weights``: The function of parameter + initialization and recording initialization + information. + - ``_params_init_info``: Used to track the parameter + initialization information. This attribute only + exists during executing the ``init_weights``. + + Args: + init_cfg (dict, optional): Initialization config dict. + """ + + def __init__(self, init_cfg=None): + """Initialize BaseModule, inherited from `torch.nn.Module`""" + + # NOTE init_cfg can be defined in different levels, but init_cfg + # in low levels has a higher priority. + + super(BaseModule, self).__init__() + # define default value of init_cfg instead of hard code + # in init_weights() function + self._is_init = False + + self.init_cfg = copy.deepcopy(init_cfg) + + # Backward compatibility in derived classes + # if pretrained is not None: + # warnings.warn('DeprecationWarning: pretrained is a deprecated \ + # key, please consider using init_cfg') + # self.init_cfg = dict(type='Pretrained', checkpoint=pretrained) + + @property + def is_init(self): + return self._is_init + + def init_weights(self): + """Initialize the weights.""" + + is_top_level_module = False + # check if it is top-level module + if not hasattr(self, '_params_init_info'): + # The `_params_init_info` is used to record the initialization + # information of the parameters + # the key should be the obj:`nn.Parameter` of model and the value + # should be a dict containing + # - init_info (str): The string that describes the initialization. + # - tmp_mean_value (FloatTensor): The mean of the parameter, + # which indicates whether the parameter has been modified. + # this attribute would be deleted after all parameters + # is initialized. + self._params_init_info = defaultdict(dict) + is_top_level_module = True + + # Initialize the `_params_init_info`, + # When detecting the `tmp_mean_value` of + # the corresponding parameter is changed, update related + # initialization information + for name, param in self.named_parameters(): + self._params_init_info[param][ + 'init_info'] = f'The value is the same before and ' \ + f'after calling `init_weights` ' \ + f'of {self.__class__.__name__} ' + self._params_init_info[param][ + 'tmp_mean_value'] = param.data.mean() + + # pass `params_init_info` to all submodules + # All submodules share the same `params_init_info`, + # so it will be updated when parameters are + # modified at any level of the model. + for sub_module in self.modules(): + sub_module._params_init_info = self._params_init_info + + # Get the initialized logger, if not exist, + # create a logger named `mmcv` + logger_names = list(logger_initialized.keys()) + logger_name = logger_names[0] if logger_names else 'mmcv' + + from ..cnn import initialize + from ..cnn.utils.weight_init import update_init_info + module_name = self.__class__.__name__ + if not self._is_init: + if self.init_cfg: + print_log( + f'initialize {module_name} with init_cfg {self.init_cfg}', + logger=logger_name) + initialize(self, self.init_cfg) + if isinstance(self.init_cfg, dict): + # prevent the parameters of + # the pre-trained model + # from being overwritten by + # the `init_weights` + if self.init_cfg['type'] == 'Pretrained': + return + + for m in self.children(): + if hasattr(m, 'init_weights'): + m.init_weights() + # users may overload the `init_weights` + update_init_info( + m, + init_info=f'Initialized by ' + f'user-defined `init_weights`' + f' in {m.__class__.__name__} ') + + self._is_init = True + else: + warnings.warn(f'init_weights of {self.__class__.__name__} has ' + f'been called more than once.') + + if is_top_level_module: + self._dump_init_info(logger_name) + + for sub_module in self.modules(): + del sub_module._params_init_info + + @master_only + def _dump_init_info(self, logger_name): + """Dump the initialization information to a file named + `initialization.log.json` in workdir. + + Args: + logger_name (str): The name of logger. + """ + + logger = get_logger(logger_name) + + with_file_handler = False + # dump the information to the logger file if there is a `FileHandler` + for handler in logger.handlers: + if isinstance(handler, FileHandler): + handler.stream.write( + 'Name of parameter - Initialization information\n') + for name, param in self.named_parameters(): + handler.stream.write( + f'\n{name} - {param.shape}: ' + f"\n{self._params_init_info[param]['init_info']} \n") + handler.stream.flush() + with_file_handler = True + if not with_file_handler: + for name, param in self.named_parameters(): + print_log( + f'\n{name} - {param.shape}: ' + f"\n{self._params_init_info[param]['init_info']} \n ", + logger=logger_name) + + def __repr__(self): + s = super().__repr__() + if self.init_cfg: + s += f'\ninit_cfg={self.init_cfg}' + return s + + +class Sequential(BaseModule, nn.Sequential): + """Sequential module in openmmlab. + + Args: + init_cfg (dict, optional): Initialization config dict. + """ + + def __init__(self, *args, init_cfg=None): + BaseModule.__init__(self, init_cfg) + nn.Sequential.__init__(self, *args) + + +class ModuleList(BaseModule, nn.ModuleList): + """ModuleList in openmmlab. + + Args: + modules (iterable, optional): an iterable of modules to add. + init_cfg (dict, optional): Initialization config dict. + """ + + def __init__(self, modules=None, init_cfg=None): + BaseModule.__init__(self, init_cfg) + nn.ModuleList.__init__(self, modules) diff --git a/annotator/mmpkg/mmcv/runner/base_runner.py b/annotator/mmpkg/mmcv/runner/base_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..a75a7d5db9f281fda10008636b24e2b98d9336a0 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/base_runner.py @@ -0,0 +1,542 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy +import logging +import os.path as osp +import warnings +from abc import ABCMeta, abstractmethod + +import torch +from torch.optim import Optimizer + +import annotator.mmpkg.mmcv as mmcv +from ..parallel import is_module_wrapper +from .checkpoint import load_checkpoint +from .dist_utils import get_dist_info +from .hooks import HOOKS, Hook +from .log_buffer import LogBuffer +from .priority import Priority, get_priority +from .utils import get_time_str + + +class BaseRunner(metaclass=ABCMeta): + """The base class of Runner, a training helper for PyTorch. + + All subclasses should implement the following APIs: + + - ``run()`` + - ``train()`` + - ``val()`` + - ``save_checkpoint()`` + + Args: + model (:obj:`torch.nn.Module`): The model to be run. + batch_processor (callable): A callable method that process a data + batch. The interface of this method should be + `batch_processor(model, data, train_mode) -> dict` + optimizer (dict or :obj:`torch.optim.Optimizer`): It can be either an + optimizer (in most cases) or a dict of optimizers (in models that + requires more than one optimizer, e.g., GAN). + work_dir (str, optional): The working directory to save checkpoints + and logs. Defaults to None. + logger (:obj:`logging.Logger`): Logger used during training. + Defaults to None. (The default value is just for backward + compatibility) + meta (dict | None): A dict records some import information such as + environment info and seed, which will be logged in logger hook. + Defaults to None. + max_epochs (int, optional): Total training epochs. + max_iters (int, optional): Total training iterations. + """ + + def __init__(self, + model, + batch_processor=None, + optimizer=None, + work_dir=None, + logger=None, + meta=None, + max_iters=None, + max_epochs=None): + if batch_processor is not None: + if not callable(batch_processor): + raise TypeError('batch_processor must be callable, ' + f'but got {type(batch_processor)}') + warnings.warn('batch_processor is deprecated, please implement ' + 'train_step() and val_step() in the model instead.') + # raise an error is `batch_processor` is not None and + # `model.train_step()` exists. + if is_module_wrapper(model): + _model = model.module + else: + _model = model + if hasattr(_model, 'train_step') or hasattr(_model, 'val_step'): + raise RuntimeError( + 'batch_processor and model.train_step()/model.val_step() ' + 'cannot be both available.') + else: + assert hasattr(model, 'train_step') + + # check the type of `optimizer` + if isinstance(optimizer, dict): + for name, optim in optimizer.items(): + if not isinstance(optim, Optimizer): + raise TypeError( + f'optimizer must be a dict of torch.optim.Optimizers, ' + f'but optimizer["{name}"] is a {type(optim)}') + elif not isinstance(optimizer, Optimizer) and optimizer is not None: + raise TypeError( + f'optimizer must be a torch.optim.Optimizer object ' + f'or dict or None, but got {type(optimizer)}') + + # check the type of `logger` + if not isinstance(logger, logging.Logger): + raise TypeError(f'logger must be a logging.Logger object, ' + f'but got {type(logger)}') + + # check the type of `meta` + if meta is not None and not isinstance(meta, dict): + raise TypeError( + f'meta must be a dict or None, but got {type(meta)}') + + self.model = model + self.batch_processor = batch_processor + self.optimizer = optimizer + self.logger = logger + self.meta = meta + # create work_dir + if mmcv.is_str(work_dir): + self.work_dir = osp.abspath(work_dir) + mmcv.mkdir_or_exist(self.work_dir) + elif work_dir is None: + self.work_dir = None + else: + raise TypeError('"work_dir" must be a str or None') + + # get model name from the model class + if hasattr(self.model, 'module'): + self._model_name = self.model.module.__class__.__name__ + else: + self._model_name = self.model.__class__.__name__ + + self._rank, self._world_size = get_dist_info() + self.timestamp = get_time_str() + self.mode = None + self._hooks = [] + self._epoch = 0 + self._iter = 0 + self._inner_iter = 0 + + if max_epochs is not None and max_iters is not None: + raise ValueError( + 'Only one of `max_epochs` or `max_iters` can be set.') + + self._max_epochs = max_epochs + self._max_iters = max_iters + # TODO: Redesign LogBuffer, it is not flexible and elegant enough + self.log_buffer = LogBuffer() + + @property + def model_name(self): + """str: Name of the model, usually the module class name.""" + return self._model_name + + @property + def rank(self): + """int: Rank of current process. (distributed training)""" + return self._rank + + @property + def world_size(self): + """int: Number of processes participating in the job. + (distributed training)""" + return self._world_size + + @property + def hooks(self): + """list[:obj:`Hook`]: A list of registered hooks.""" + return self._hooks + + @property + def epoch(self): + """int: Current epoch.""" + return self._epoch + + @property + def iter(self): + """int: Current iteration.""" + return self._iter + + @property + def inner_iter(self): + """int: Iteration in an epoch.""" + return self._inner_iter + + @property + def max_epochs(self): + """int: Maximum training epochs.""" + return self._max_epochs + + @property + def max_iters(self): + """int: Maximum training iterations.""" + return self._max_iters + + @abstractmethod + def train(self): + pass + + @abstractmethod + def val(self): + pass + + @abstractmethod + def run(self, data_loaders, workflow, **kwargs): + pass + + @abstractmethod + def save_checkpoint(self, + out_dir, + filename_tmpl, + save_optimizer=True, + meta=None, + create_symlink=True): + pass + + def current_lr(self): + """Get current learning rates. + + Returns: + list[float] | dict[str, list[float]]: Current learning rates of all + param groups. If the runner has a dict of optimizers, this + method will return a dict. + """ + if isinstance(self.optimizer, torch.optim.Optimizer): + lr = [group['lr'] for group in self.optimizer.param_groups] + elif isinstance(self.optimizer, dict): + lr = dict() + for name, optim in self.optimizer.items(): + lr[name] = [group['lr'] for group in optim.param_groups] + else: + raise RuntimeError( + 'lr is not applicable because optimizer does not exist.') + return lr + + def current_momentum(self): + """Get current momentums. + + Returns: + list[float] | dict[str, list[float]]: Current momentums of all + param groups. If the runner has a dict of optimizers, this + method will return a dict. + """ + + def _get_momentum(optimizer): + momentums = [] + for group in optimizer.param_groups: + if 'momentum' in group.keys(): + momentums.append(group['momentum']) + elif 'betas' in group.keys(): + momentums.append(group['betas'][0]) + else: + momentums.append(0) + return momentums + + if self.optimizer is None: + raise RuntimeError( + 'momentum is not applicable because optimizer does not exist.') + elif isinstance(self.optimizer, torch.optim.Optimizer): + momentums = _get_momentum(self.optimizer) + elif isinstance(self.optimizer, dict): + momentums = dict() + for name, optim in self.optimizer.items(): + momentums[name] = _get_momentum(optim) + return momentums + + def register_hook(self, hook, priority='NORMAL'): + """Register a hook into the hook list. + + The hook will be inserted into a priority queue, with the specified + priority (See :class:`Priority` for details of priorities). + For hooks with the same priority, they will be triggered in the same + order as they are registered. + + Args: + hook (:obj:`Hook`): The hook to be registered. + priority (int or str or :obj:`Priority`): Hook priority. + Lower value means higher priority. + """ + assert isinstance(hook, Hook) + if hasattr(hook, 'priority'): + raise ValueError('"priority" is a reserved attribute for hooks') + priority = get_priority(priority) + hook.priority = priority + # insert the hook to a sorted list + inserted = False + for i in range(len(self._hooks) - 1, -1, -1): + if priority >= self._hooks[i].priority: + self._hooks.insert(i + 1, hook) + inserted = True + break + if not inserted: + self._hooks.insert(0, hook) + + def register_hook_from_cfg(self, hook_cfg): + """Register a hook from its cfg. + + Args: + hook_cfg (dict): Hook config. It should have at least keys 'type' + and 'priority' indicating its type and priority. + + Notes: + The specific hook class to register should not use 'type' and + 'priority' arguments during initialization. + """ + hook_cfg = hook_cfg.copy() + priority = hook_cfg.pop('priority', 'NORMAL') + hook = mmcv.build_from_cfg(hook_cfg, HOOKS) + self.register_hook(hook, priority=priority) + + def call_hook(self, fn_name): + """Call all hooks. + + Args: + fn_name (str): The function name in each hook to be called, such as + "before_train_epoch". + """ + for hook in self._hooks: + getattr(hook, fn_name)(self) + + def get_hook_info(self): + # Get hooks info in each stage + stage_hook_map = {stage: [] for stage in Hook.stages} + for hook in self.hooks: + try: + priority = Priority(hook.priority).name + except ValueError: + priority = hook.priority + classname = hook.__class__.__name__ + hook_info = f'({priority:<12}) {classname:<35}' + for trigger_stage in hook.get_triggered_stages(): + stage_hook_map[trigger_stage].append(hook_info) + + stage_hook_infos = [] + for stage in Hook.stages: + hook_infos = stage_hook_map[stage] + if len(hook_infos) > 0: + info = f'{stage}:\n' + info += '\n'.join(hook_infos) + info += '\n -------------------- ' + stage_hook_infos.append(info) + return '\n'.join(stage_hook_infos) + + def load_checkpoint(self, + filename, + map_location='cpu', + strict=False, + revise_keys=[(r'^module.', '')]): + return load_checkpoint( + self.model, + filename, + map_location, + strict, + self.logger, + revise_keys=revise_keys) + + def resume(self, + checkpoint, + resume_optimizer=True, + map_location='default'): + if map_location == 'default': + if torch.cuda.is_available(): + device_id = torch.cuda.current_device() + checkpoint = self.load_checkpoint( + checkpoint, + map_location=lambda storage, loc: storage.cuda(device_id)) + else: + checkpoint = self.load_checkpoint(checkpoint) + else: + checkpoint = self.load_checkpoint( + checkpoint, map_location=map_location) + + self._epoch = checkpoint['meta']['epoch'] + self._iter = checkpoint['meta']['iter'] + if self.meta is None: + self.meta = {} + self.meta.setdefault('hook_msgs', {}) + # load `last_ckpt`, `best_score`, `best_ckpt`, etc. for hook messages + self.meta['hook_msgs'].update(checkpoint['meta'].get('hook_msgs', {})) + + # Re-calculate the number of iterations when resuming + # models with different number of GPUs + if 'config' in checkpoint['meta']: + config = mmcv.Config.fromstring( + checkpoint['meta']['config'], file_format='.py') + previous_gpu_ids = config.get('gpu_ids', None) + if previous_gpu_ids and len(previous_gpu_ids) > 0 and len( + previous_gpu_ids) != self.world_size: + self._iter = int(self._iter * len(previous_gpu_ids) / + self.world_size) + self.logger.info('the iteration number is changed due to ' + 'change of GPU number') + + # resume meta information meta + self.meta = checkpoint['meta'] + + if 'optimizer' in checkpoint and resume_optimizer: + if isinstance(self.optimizer, Optimizer): + self.optimizer.load_state_dict(checkpoint['optimizer']) + elif isinstance(self.optimizer, dict): + for k in self.optimizer.keys(): + self.optimizer[k].load_state_dict( + checkpoint['optimizer'][k]) + else: + raise TypeError( + 'Optimizer should be dict or torch.optim.Optimizer ' + f'but got {type(self.optimizer)}') + + self.logger.info('resumed epoch %d, iter %d', self.epoch, self.iter) + + def register_lr_hook(self, lr_config): + if lr_config is None: + return + elif isinstance(lr_config, dict): + assert 'policy' in lr_config + policy_type = lr_config.pop('policy') + # If the type of policy is all in lower case, e.g., 'cyclic', + # then its first letter will be capitalized, e.g., to be 'Cyclic'. + # This is for the convenient usage of Lr updater. + # Since this is not applicable for ` + # CosineAnnealingLrUpdater`, + # the string will not be changed if it contains capital letters. + if policy_type == policy_type.lower(): + policy_type = policy_type.title() + hook_type = policy_type + 'LrUpdaterHook' + lr_config['type'] = hook_type + hook = mmcv.build_from_cfg(lr_config, HOOKS) + else: + hook = lr_config + self.register_hook(hook, priority='VERY_HIGH') + + def register_momentum_hook(self, momentum_config): + if momentum_config is None: + return + if isinstance(momentum_config, dict): + assert 'policy' in momentum_config + policy_type = momentum_config.pop('policy') + # If the type of policy is all in lower case, e.g., 'cyclic', + # then its first letter will be capitalized, e.g., to be 'Cyclic'. + # This is for the convenient usage of momentum updater. + # Since this is not applicable for + # `CosineAnnealingMomentumUpdater`, + # the string will not be changed if it contains capital letters. + if policy_type == policy_type.lower(): + policy_type = policy_type.title() + hook_type = policy_type + 'MomentumUpdaterHook' + momentum_config['type'] = hook_type + hook = mmcv.build_from_cfg(momentum_config, HOOKS) + else: + hook = momentum_config + self.register_hook(hook, priority='HIGH') + + def register_optimizer_hook(self, optimizer_config): + if optimizer_config is None: + return + if isinstance(optimizer_config, dict): + optimizer_config.setdefault('type', 'OptimizerHook') + hook = mmcv.build_from_cfg(optimizer_config, HOOKS) + else: + hook = optimizer_config + self.register_hook(hook, priority='ABOVE_NORMAL') + + def register_checkpoint_hook(self, checkpoint_config): + if checkpoint_config is None: + return + if isinstance(checkpoint_config, dict): + checkpoint_config.setdefault('type', 'CheckpointHook') + hook = mmcv.build_from_cfg(checkpoint_config, HOOKS) + else: + hook = checkpoint_config + self.register_hook(hook, priority='NORMAL') + + def register_logger_hooks(self, log_config): + if log_config is None: + return + log_interval = log_config['interval'] + for info in log_config['hooks']: + logger_hook = mmcv.build_from_cfg( + info, HOOKS, default_args=dict(interval=log_interval)) + self.register_hook(logger_hook, priority='VERY_LOW') + + def register_timer_hook(self, timer_config): + if timer_config is None: + return + if isinstance(timer_config, dict): + timer_config_ = copy.deepcopy(timer_config) + hook = mmcv.build_from_cfg(timer_config_, HOOKS) + else: + hook = timer_config + self.register_hook(hook, priority='LOW') + + def register_custom_hooks(self, custom_config): + if custom_config is None: + return + + if not isinstance(custom_config, list): + custom_config = [custom_config] + + for item in custom_config: + if isinstance(item, dict): + self.register_hook_from_cfg(item) + else: + self.register_hook(item, priority='NORMAL') + + def register_profiler_hook(self, profiler_config): + if profiler_config is None: + return + if isinstance(profiler_config, dict): + profiler_config.setdefault('type', 'ProfilerHook') + hook = mmcv.build_from_cfg(profiler_config, HOOKS) + else: + hook = profiler_config + self.register_hook(hook) + + def register_training_hooks(self, + lr_config, + optimizer_config=None, + checkpoint_config=None, + log_config=None, + momentum_config=None, + timer_config=dict(type='IterTimerHook'), + custom_hooks_config=None): + """Register default and custom hooks for training. + + Default and custom hooks include: + + +----------------------+-------------------------+ + | Hooks | Priority | + +======================+=========================+ + | LrUpdaterHook | VERY_HIGH (10) | + +----------------------+-------------------------+ + | MomentumUpdaterHook | HIGH (30) | + +----------------------+-------------------------+ + | OptimizerStepperHook | ABOVE_NORMAL (40) | + +----------------------+-------------------------+ + | CheckpointSaverHook | NORMAL (50) | + +----------------------+-------------------------+ + | IterTimerHook | LOW (70) | + +----------------------+-------------------------+ + | LoggerHook(s) | VERY_LOW (90) | + +----------------------+-------------------------+ + | CustomHook(s) | defaults to NORMAL (50) | + +----------------------+-------------------------+ + + If custom hooks have same priority with default hooks, custom hooks + will be triggered after default hooks. + """ + self.register_lr_hook(lr_config) + self.register_momentum_hook(momentum_config) + self.register_optimizer_hook(optimizer_config) + self.register_checkpoint_hook(checkpoint_config) + self.register_timer_hook(timer_config) + self.register_logger_hooks(log_config) + self.register_custom_hooks(custom_hooks_config) diff --git a/annotator/mmpkg/mmcv/runner/builder.py b/annotator/mmpkg/mmcv/runner/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..77c96ba0b2f30ead9da23f293c5dc84dd3e4a74f --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/builder.py @@ -0,0 +1,24 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy + +from ..utils import Registry + +RUNNERS = Registry('runner') +RUNNER_BUILDERS = Registry('runner builder') + + +def build_runner_constructor(cfg): + return RUNNER_BUILDERS.build(cfg) + + +def build_runner(cfg, default_args=None): + runner_cfg = copy.deepcopy(cfg) + constructor_type = runner_cfg.pop('constructor', + 'DefaultRunnerConstructor') + runner_constructor = build_runner_constructor( + dict( + type=constructor_type, + runner_cfg=runner_cfg, + default_args=default_args)) + runner = runner_constructor() + return runner diff --git a/annotator/mmpkg/mmcv/runner/checkpoint.py b/annotator/mmpkg/mmcv/runner/checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..d690be1dfe70b1b82eaac8fe4db7022b35d5426c --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/checkpoint.py @@ -0,0 +1,707 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import io +import os +import os.path as osp +import pkgutil +import re +import time +import warnings +from collections import OrderedDict +from importlib import import_module +from tempfile import TemporaryDirectory + +import torch +import torchvision +from torch.optim import Optimizer +from torch.utils import model_zoo + +import annotator.mmpkg.mmcv as mmcv +from ..fileio import FileClient +from ..fileio import load as load_file +from ..parallel import is_module_wrapper +from ..utils import mkdir_or_exist +from .dist_utils import get_dist_info + +ENV_MMCV_HOME = 'MMCV_HOME' +ENV_XDG_CACHE_HOME = 'XDG_CACHE_HOME' +DEFAULT_CACHE_DIR = '~/.cache' + + +def _get_mmcv_home(): + mmcv_home = os.path.expanduser( + os.getenv( + ENV_MMCV_HOME, + os.path.join( + os.getenv(ENV_XDG_CACHE_HOME, DEFAULT_CACHE_DIR), 'mmcv'))) + + mkdir_or_exist(mmcv_home) + return mmcv_home + + +def load_state_dict(module, state_dict, strict=False, logger=None): + """Load state_dict to a module. + + This method is modified from :meth:`torch.nn.Module.load_state_dict`. + Default value for ``strict`` is set to ``False`` and the message for + param mismatch will be shown even if strict is False. + + Args: + module (Module): Module that receives the state_dict. + state_dict (OrderedDict): Weights. + strict (bool): whether to strictly enforce that the keys + in :attr:`state_dict` match the keys returned by this module's + :meth:`~torch.nn.Module.state_dict` function. Default: ``False``. + logger (:obj:`logging.Logger`, optional): Logger to log the error + message. If not specified, print function will be used. + """ + unexpected_keys = [] + all_missing_keys = [] + err_msg = [] + + metadata = getattr(state_dict, '_metadata', None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + # use _load_from_state_dict to enable checkpoint version control + def load(module, prefix=''): + # recursively check parallel module in case that the model has a + # complicated structure, e.g., nn.Module(nn.Module(DDP)) + if is_module_wrapper(module): + module = module.module + local_metadata = {} if metadata is None else metadata.get( + prefix[:-1], {}) + module._load_from_state_dict(state_dict, prefix, local_metadata, True, + all_missing_keys, unexpected_keys, + err_msg) + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + '.') + + load(module) + load = None # break load->load reference cycle + + # ignore "num_batches_tracked" of BN layers + missing_keys = [ + key for key in all_missing_keys if 'num_batches_tracked' not in key + ] + + if unexpected_keys: + err_msg.append('unexpected key in source ' + f'state_dict: {", ".join(unexpected_keys)}\n') + if missing_keys: + err_msg.append( + f'missing keys in source state_dict: {", ".join(missing_keys)}\n') + + rank, _ = get_dist_info() + if len(err_msg) > 0 and rank == 0: + err_msg.insert( + 0, 'The model and loaded state dict do not match exactly\n') + err_msg = '\n'.join(err_msg) + if strict: + raise RuntimeError(err_msg) + elif logger is not None: + logger.warning(err_msg) + else: + print(err_msg) + + +def get_torchvision_models(): + model_urls = dict() + for _, name, ispkg in pkgutil.walk_packages(torchvision.models.__path__): + if ispkg: + continue + _zoo = import_module(f'torchvision.models.{name}') + if hasattr(_zoo, 'model_urls'): + _urls = getattr(_zoo, 'model_urls') + model_urls.update(_urls) + return model_urls + + +def get_external_models(): + mmcv_home = _get_mmcv_home() + default_json_path = osp.join(mmcv.__path__[0], 'model_zoo/open_mmlab.json') + default_urls = load_file(default_json_path) + assert isinstance(default_urls, dict) + external_json_path = osp.join(mmcv_home, 'open_mmlab.json') + if osp.exists(external_json_path): + external_urls = load_file(external_json_path) + assert isinstance(external_urls, dict) + default_urls.update(external_urls) + + return default_urls + + +def get_mmcls_models(): + mmcls_json_path = osp.join(mmcv.__path__[0], 'model_zoo/mmcls.json') + mmcls_urls = load_file(mmcls_json_path) + + return mmcls_urls + + +def get_deprecated_model_names(): + deprecate_json_path = osp.join(mmcv.__path__[0], + 'model_zoo/deprecated.json') + deprecate_urls = load_file(deprecate_json_path) + assert isinstance(deprecate_urls, dict) + + return deprecate_urls + + +def _process_mmcls_checkpoint(checkpoint): + state_dict = checkpoint['state_dict'] + new_state_dict = OrderedDict() + for k, v in state_dict.items(): + if k.startswith('backbone.'): + new_state_dict[k[9:]] = v + new_checkpoint = dict(state_dict=new_state_dict) + + return new_checkpoint + + +class CheckpointLoader: + """A general checkpoint loader to manage all schemes.""" + + _schemes = {} + + @classmethod + def _register_scheme(cls, prefixes, loader, force=False): + if isinstance(prefixes, str): + prefixes = [prefixes] + else: + assert isinstance(prefixes, (list, tuple)) + for prefix in prefixes: + if (prefix not in cls._schemes) or force: + cls._schemes[prefix] = loader + else: + raise KeyError( + f'{prefix} is already registered as a loader backend, ' + 'add "force=True" if you want to override it') + # sort, longer prefixes take priority + cls._schemes = OrderedDict( + sorted(cls._schemes.items(), key=lambda t: t[0], reverse=True)) + + @classmethod + def register_scheme(cls, prefixes, loader=None, force=False): + """Register a loader to CheckpointLoader. + + This method can be used as a normal class method or a decorator. + + Args: + prefixes (str or list[str] or tuple[str]): + The prefix of the registered loader. + loader (function, optional): The loader function to be registered. + When this method is used as a decorator, loader is None. + Defaults to None. + force (bool, optional): Whether to override the loader + if the prefix has already been registered. Defaults to False. + """ + + if loader is not None: + cls._register_scheme(prefixes, loader, force=force) + return + + def _register(loader_cls): + cls._register_scheme(prefixes, loader_cls, force=force) + return loader_cls + + return _register + + @classmethod + def _get_checkpoint_loader(cls, path): + """Finds a loader that supports the given path. Falls back to the local + loader if no other loader is found. + + Args: + path (str): checkpoint path + + Returns: + loader (function): checkpoint loader + """ + + for p in cls._schemes: + if path.startswith(p): + return cls._schemes[p] + + @classmethod + def load_checkpoint(cls, filename, map_location=None, logger=None): + """load checkpoint through URL scheme path. + + Args: + filename (str): checkpoint file name with given prefix + map_location (str, optional): Same as :func:`torch.load`. + Default: None + logger (:mod:`logging.Logger`, optional): The logger for message. + Default: None + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + + checkpoint_loader = cls._get_checkpoint_loader(filename) + class_name = checkpoint_loader.__name__ + mmcv.print_log( + f'load checkpoint from {class_name[10:]} path: {filename}', logger) + return checkpoint_loader(filename, map_location) + + +@CheckpointLoader.register_scheme(prefixes='') +def load_from_local(filename, map_location): + """load checkpoint by local file path. + + Args: + filename (str): local checkpoint file path + map_location (str, optional): Same as :func:`torch.load`. + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + + if not osp.isfile(filename): + raise IOError(f'{filename} is not a checkpoint file') + checkpoint = torch.load(filename, map_location=map_location) + return checkpoint + + +@CheckpointLoader.register_scheme(prefixes=('http://', 'https://')) +def load_from_http(filename, map_location=None, model_dir=None): + """load checkpoint through HTTP or HTTPS scheme path. In distributed + setting, this function only download checkpoint at local rank 0. + + Args: + filename (str): checkpoint file path with modelzoo or + torchvision prefix + map_location (str, optional): Same as :func:`torch.load`. + model_dir (string, optional): directory in which to save the object, + Default: None + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + rank, world_size = get_dist_info() + rank = int(os.environ.get('LOCAL_RANK', rank)) + if rank == 0: + checkpoint = model_zoo.load_url( + filename, model_dir=model_dir, map_location=map_location) + if world_size > 1: + torch.distributed.barrier() + if rank > 0: + checkpoint = model_zoo.load_url( + filename, model_dir=model_dir, map_location=map_location) + return checkpoint + + +@CheckpointLoader.register_scheme(prefixes='pavi://') +def load_from_pavi(filename, map_location=None): + """load checkpoint through the file path prefixed with pavi. In distributed + setting, this function download ckpt at all ranks to different temporary + directories. + + Args: + filename (str): checkpoint file path with pavi prefix + map_location (str, optional): Same as :func:`torch.load`. + Default: None + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + assert filename.startswith('pavi://'), \ + f'Expected filename startswith `pavi://`, but get {filename}' + model_path = filename[7:] + + try: + from pavi import modelcloud + except ImportError: + raise ImportError( + 'Please install pavi to load checkpoint from modelcloud.') + + model = modelcloud.get(model_path) + with TemporaryDirectory() as tmp_dir: + downloaded_file = osp.join(tmp_dir, model.name) + model.download(downloaded_file) + checkpoint = torch.load(downloaded_file, map_location=map_location) + return checkpoint + + +@CheckpointLoader.register_scheme(prefixes='s3://') +def load_from_ceph(filename, map_location=None, backend='petrel'): + """load checkpoint through the file path prefixed with s3. In distributed + setting, this function download ckpt at all ranks to different temporary + directories. + + Args: + filename (str): checkpoint file path with s3 prefix + map_location (str, optional): Same as :func:`torch.load`. + backend (str, optional): The storage backend type. Options are 'ceph', + 'petrel'. Default: 'petrel'. + + .. warning:: + :class:`mmcv.fileio.file_client.CephBackend` will be deprecated, + please use :class:`mmcv.fileio.file_client.PetrelBackend` instead. + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + allowed_backends = ['ceph', 'petrel'] + if backend not in allowed_backends: + raise ValueError(f'Load from Backend {backend} is not supported.') + + if backend == 'ceph': + warnings.warn( + 'CephBackend will be deprecated, please use PetrelBackend instead') + + # CephClient and PetrelBackend have the same prefix 's3://' and the latter + # will be chosen as default. If PetrelBackend can not be instantiated + # successfully, the CephClient will be chosen. + try: + file_client = FileClient(backend=backend) + except ImportError: + allowed_backends.remove(backend) + file_client = FileClient(backend=allowed_backends[0]) + + with io.BytesIO(file_client.get(filename)) as buffer: + checkpoint = torch.load(buffer, map_location=map_location) + return checkpoint + + +@CheckpointLoader.register_scheme(prefixes=('modelzoo://', 'torchvision://')) +def load_from_torchvision(filename, map_location=None): + """load checkpoint through the file path prefixed with modelzoo or + torchvision. + + Args: + filename (str): checkpoint file path with modelzoo or + torchvision prefix + map_location (str, optional): Same as :func:`torch.load`. + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + model_urls = get_torchvision_models() + if filename.startswith('modelzoo://'): + warnings.warn('The URL scheme of "modelzoo://" is deprecated, please ' + 'use "torchvision://" instead') + model_name = filename[11:] + else: + model_name = filename[14:] + return load_from_http(model_urls[model_name], map_location=map_location) + + +@CheckpointLoader.register_scheme(prefixes=('open-mmlab://', 'openmmlab://')) +def load_from_openmmlab(filename, map_location=None): + """load checkpoint through the file path prefixed with open-mmlab or + openmmlab. + + Args: + filename (str): checkpoint file path with open-mmlab or + openmmlab prefix + map_location (str, optional): Same as :func:`torch.load`. + Default: None + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + + model_urls = get_external_models() + prefix_str = 'open-mmlab://' + if filename.startswith(prefix_str): + model_name = filename[13:] + else: + model_name = filename[12:] + prefix_str = 'openmmlab://' + + deprecated_urls = get_deprecated_model_names() + if model_name in deprecated_urls: + warnings.warn(f'{prefix_str}{model_name} is deprecated in favor ' + f'of {prefix_str}{deprecated_urls[model_name]}') + model_name = deprecated_urls[model_name] + model_url = model_urls[model_name] + # check if is url + if model_url.startswith(('http://', 'https://')): + checkpoint = load_from_http(model_url, map_location=map_location) + else: + filename = osp.join(_get_mmcv_home(), model_url) + if not osp.isfile(filename): + raise IOError(f'{filename} is not a checkpoint file') + checkpoint = torch.load(filename, map_location=map_location) + return checkpoint + + +@CheckpointLoader.register_scheme(prefixes='mmcls://') +def load_from_mmcls(filename, map_location=None): + """load checkpoint through the file path prefixed with mmcls. + + Args: + filename (str): checkpoint file path with mmcls prefix + map_location (str, optional): Same as :func:`torch.load`. + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + + model_urls = get_mmcls_models() + model_name = filename[8:] + checkpoint = load_from_http( + model_urls[model_name], map_location=map_location) + checkpoint = _process_mmcls_checkpoint(checkpoint) + return checkpoint + + +def _load_checkpoint(filename, map_location=None, logger=None): + """Load checkpoint from somewhere (modelzoo, file, url). + + Args: + filename (str): Accept local filepath, URL, ``torchvision://xxx``, + ``open-mmlab://xxx``. Please refer to ``docs/model_zoo.md`` for + details. + map_location (str, optional): Same as :func:`torch.load`. + Default: None. + logger (:mod:`logging.Logger`, optional): The logger for error message. + Default: None + + Returns: + dict or OrderedDict: The loaded checkpoint. It can be either an + OrderedDict storing model weights or a dict containing other + information, which depends on the checkpoint. + """ + return CheckpointLoader.load_checkpoint(filename, map_location, logger) + + +def _load_checkpoint_with_prefix(prefix, filename, map_location=None): + """Load partial pretrained model with specific prefix. + + Args: + prefix (str): The prefix of sub-module. + filename (str): Accept local filepath, URL, ``torchvision://xxx``, + ``open-mmlab://xxx``. Please refer to ``docs/model_zoo.md`` for + details. + map_location (str | None): Same as :func:`torch.load`. Default: None. + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + + checkpoint = _load_checkpoint(filename, map_location=map_location) + + if 'state_dict' in checkpoint: + state_dict = checkpoint['state_dict'] + else: + state_dict = checkpoint + if not prefix.endswith('.'): + prefix += '.' + prefix_len = len(prefix) + + state_dict = { + k[prefix_len:]: v + for k, v in state_dict.items() if k.startswith(prefix) + } + + assert state_dict, f'{prefix} is not in the pretrained model' + return state_dict + + +def load_checkpoint(model, + filename, + map_location=None, + strict=False, + logger=None, + revise_keys=[(r'^module\.', '')]): + """Load checkpoint from a file or URI. + + Args: + model (Module): Module to load checkpoint. + filename (str): Accept local filepath, URL, ``torchvision://xxx``, + ``open-mmlab://xxx``. Please refer to ``docs/model_zoo.md`` for + details. + map_location (str): Same as :func:`torch.load`. + strict (bool): Whether to allow different params for the model and + checkpoint. + logger (:mod:`logging.Logger` or None): The logger for error message. + revise_keys (list): A list of customized keywords to modify the + state_dict in checkpoint. Each item is a (pattern, replacement) + pair of the regular expression operations. Default: strip + the prefix 'module.' by [(r'^module\\.', '')]. + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + checkpoint = _load_checkpoint(filename, map_location, logger) + # OrderedDict is a subclass of dict + if not isinstance(checkpoint, dict): + raise RuntimeError( + f'No state_dict found in checkpoint file {filename}') + # get state_dict from checkpoint + if 'state_dict' in checkpoint: + state_dict = checkpoint['state_dict'] + else: + state_dict = checkpoint + + # strip prefix of state_dict + metadata = getattr(state_dict, '_metadata', OrderedDict()) + for p, r in revise_keys: + state_dict = OrderedDict( + {re.sub(p, r, k): v + for k, v in state_dict.items()}) + # Keep metadata in state_dict + state_dict._metadata = metadata + + # load state_dict + load_state_dict(model, state_dict, strict, logger) + return checkpoint + + +def weights_to_cpu(state_dict): + """Copy a model state_dict to cpu. + + Args: + state_dict (OrderedDict): Model weights on GPU. + + Returns: + OrderedDict: Model weights on GPU. + """ + state_dict_cpu = OrderedDict() + for key, val in state_dict.items(): + state_dict_cpu[key] = val.cpu() + # Keep metadata in state_dict + state_dict_cpu._metadata = getattr(state_dict, '_metadata', OrderedDict()) + return state_dict_cpu + + +def _save_to_state_dict(module, destination, prefix, keep_vars): + """Saves module state to `destination` dictionary. + + This method is modified from :meth:`torch.nn.Module._save_to_state_dict`. + + Args: + module (nn.Module): The module to generate state_dict. + destination (dict): A dict where state will be stored. + prefix (str): The prefix for parameters and buffers used in this + module. + """ + for name, param in module._parameters.items(): + if param is not None: + destination[prefix + name] = param if keep_vars else param.detach() + for name, buf in module._buffers.items(): + # remove check of _non_persistent_buffers_set to allow nn.BatchNorm2d + if buf is not None: + destination[prefix + name] = buf if keep_vars else buf.detach() + + +def get_state_dict(module, destination=None, prefix='', keep_vars=False): + """Returns a dictionary containing a whole state of the module. + + Both parameters and persistent buffers (e.g. running averages) are + included. Keys are corresponding parameter and buffer names. + + This method is modified from :meth:`torch.nn.Module.state_dict` to + recursively check parallel module in case that the model has a complicated + structure, e.g., nn.Module(nn.Module(DDP)). + + Args: + module (nn.Module): The module to generate state_dict. + destination (OrderedDict): Returned dict for the state of the + module. + prefix (str): Prefix of the key. + keep_vars (bool): Whether to keep the variable property of the + parameters. Default: False. + + Returns: + dict: A dictionary containing a whole state of the module. + """ + # recursively check parallel module in case that the model has a + # complicated structure, e.g., nn.Module(nn.Module(DDP)) + if is_module_wrapper(module): + module = module.module + + # below is the same as torch.nn.Module.state_dict() + if destination is None: + destination = OrderedDict() + destination._metadata = OrderedDict() + destination._metadata[prefix[:-1]] = local_metadata = dict( + version=module._version) + _save_to_state_dict(module, destination, prefix, keep_vars) + for name, child in module._modules.items(): + if child is not None: + get_state_dict( + child, destination, prefix + name + '.', keep_vars=keep_vars) + for hook in module._state_dict_hooks.values(): + hook_result = hook(module, destination, prefix, local_metadata) + if hook_result is not None: + destination = hook_result + return destination + + +def save_checkpoint(model, + filename, + optimizer=None, + meta=None, + file_client_args=None): + """Save checkpoint to file. + + The checkpoint will have 3 fields: ``meta``, ``state_dict`` and + ``optimizer``. By default ``meta`` will contain version and time info. + + Args: + model (Module): Module whose params are to be saved. + filename (str): Checkpoint filename. + optimizer (:obj:`Optimizer`, optional): Optimizer to be saved. + meta (dict, optional): Metadata to be saved in checkpoint. + file_client_args (dict, optional): Arguments to instantiate a + FileClient. See :class:`mmcv.fileio.FileClient` for details. + Default: None. + `New in version 1.3.16.` + """ + if meta is None: + meta = {} + elif not isinstance(meta, dict): + raise TypeError(f'meta must be a dict or None, but got {type(meta)}') + meta.update(mmcv_version=mmcv.__version__, time=time.asctime()) + + if is_module_wrapper(model): + model = model.module + + if hasattr(model, 'CLASSES') and model.CLASSES is not None: + # save class name to the meta + meta.update(CLASSES=model.CLASSES) + + checkpoint = { + 'meta': meta, + 'state_dict': weights_to_cpu(get_state_dict(model)) + } + # save optimizer state dict in the checkpoint + if isinstance(optimizer, Optimizer): + checkpoint['optimizer'] = optimizer.state_dict() + elif isinstance(optimizer, dict): + checkpoint['optimizer'] = {} + for name, optim in optimizer.items(): + checkpoint['optimizer'][name] = optim.state_dict() + + if filename.startswith('pavi://'): + if file_client_args is not None: + raise ValueError( + 'file_client_args should be "None" if filename starts with' + f'"pavi://", but got {file_client_args}') + try: + from pavi import modelcloud + from pavi import exception + except ImportError: + raise ImportError( + 'Please install pavi to load checkpoint from modelcloud.') + model_path = filename[7:] + root = modelcloud.Folder() + model_dir, model_name = osp.split(model_path) + try: + model = modelcloud.get(model_dir) + except exception.NodeNotFoundError: + model = root.create_training_model(model_dir) + with TemporaryDirectory() as tmp_dir: + checkpoint_file = osp.join(tmp_dir, model_name) + with open(checkpoint_file, 'wb') as f: + torch.save(checkpoint, f) + f.flush() + model.create_file(checkpoint_file, name=model_name) + else: + file_client = FileClient.infer_client(file_client_args, filename) + with io.BytesIO() as f: + torch.save(checkpoint, f) + file_client.put(f.getvalue(), filename) diff --git a/annotator/mmpkg/mmcv/runner/default_constructor.py b/annotator/mmpkg/mmcv/runner/default_constructor.py new file mode 100644 index 0000000000000000000000000000000000000000..bdd7803289d6d70240977fa243d7f4432ccde8f8 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/default_constructor.py @@ -0,0 +1,44 @@ +from .builder import RUNNER_BUILDERS, RUNNERS + + +@RUNNER_BUILDERS.register_module() +class DefaultRunnerConstructor: + """Default constructor for runners. + + Custom existing `Runner` like `EpocBasedRunner` though `RunnerConstructor`. + For example, We can inject some new properties and functions for `Runner`. + + Example: + >>> from annotator.mmpkg.mmcv.runner import RUNNER_BUILDERS, build_runner + >>> # Define a new RunnerReconstructor + >>> @RUNNER_BUILDERS.register_module() + >>> class MyRunnerConstructor: + ... def __init__(self, runner_cfg, default_args=None): + ... if not isinstance(runner_cfg, dict): + ... raise TypeError('runner_cfg should be a dict', + ... f'but got {type(runner_cfg)}') + ... self.runner_cfg = runner_cfg + ... self.default_args = default_args + ... + ... def __call__(self): + ... runner = RUNNERS.build(self.runner_cfg, + ... default_args=self.default_args) + ... # Add new properties for existing runner + ... runner.my_name = 'my_runner' + ... runner.my_function = lambda self: print(self.my_name) + ... ... + >>> # build your runner + >>> runner_cfg = dict(type='EpochBasedRunner', max_epochs=40, + ... constructor='MyRunnerConstructor') + >>> runner = build_runner(runner_cfg) + """ + + def __init__(self, runner_cfg, default_args=None): + if not isinstance(runner_cfg, dict): + raise TypeError('runner_cfg should be a dict', + f'but got {type(runner_cfg)}') + self.runner_cfg = runner_cfg + self.default_args = default_args + + def __call__(self): + return RUNNERS.build(self.runner_cfg, default_args=self.default_args) diff --git a/annotator/mmpkg/mmcv/runner/dist_utils.py b/annotator/mmpkg/mmcv/runner/dist_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d3a1ef3fda5ceeb31bf15a73779da1b1903ab0fe --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/dist_utils.py @@ -0,0 +1,164 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import functools +import os +import subprocess +from collections import OrderedDict + +import torch +import torch.multiprocessing as mp +from torch import distributed as dist +from torch._utils import (_flatten_dense_tensors, _take_tensors, + _unflatten_dense_tensors) + + +def init_dist(launcher, backend='nccl', **kwargs): + if mp.get_start_method(allow_none=True) is None: + mp.set_start_method('spawn') + if launcher == 'pytorch': + _init_dist_pytorch(backend, **kwargs) + elif launcher == 'mpi': + _init_dist_mpi(backend, **kwargs) + elif launcher == 'slurm': + _init_dist_slurm(backend, **kwargs) + else: + raise ValueError(f'Invalid launcher type: {launcher}') + + +def _init_dist_pytorch(backend, **kwargs): + # TODO: use local_rank instead of rank % num_gpus + rank = int(os.environ['RANK']) + num_gpus = torch.cuda.device_count() + torch.cuda.set_device(rank % num_gpus) + dist.init_process_group(backend=backend, **kwargs) + + +def _init_dist_mpi(backend, **kwargs): + # TODO: use local_rank instead of rank % num_gpus + rank = int(os.environ['OMPI_COMM_WORLD_RANK']) + num_gpus = torch.cuda.device_count() + torch.cuda.set_device(rank % num_gpus) + dist.init_process_group(backend=backend, **kwargs) + + +def _init_dist_slurm(backend, port=None): + """Initialize slurm distributed training environment. + + If argument ``port`` is not specified, then the master port will be system + environment variable ``MASTER_PORT``. If ``MASTER_PORT`` is not in system + environment variable, then a default port ``29500`` will be used. + + Args: + backend (str): Backend of torch.distributed. + port (int, optional): Master port. Defaults to None. + """ + proc_id = int(os.environ['SLURM_PROCID']) + ntasks = int(os.environ['SLURM_NTASKS']) + node_list = os.environ['SLURM_NODELIST'] + num_gpus = torch.cuda.device_count() + torch.cuda.set_device(proc_id % num_gpus) + addr = subprocess.getoutput( + f'scontrol show hostname {node_list} | head -n1') + # specify master port + if port is not None: + os.environ['MASTER_PORT'] = str(port) + elif 'MASTER_PORT' in os.environ: + pass # use MASTER_PORT in the environment variable + else: + # 29500 is torch.distributed default port + os.environ['MASTER_PORT'] = '29500' + # use MASTER_ADDR in the environment variable if it already exists + if 'MASTER_ADDR' not in os.environ: + os.environ['MASTER_ADDR'] = addr + os.environ['WORLD_SIZE'] = str(ntasks) + os.environ['LOCAL_RANK'] = str(proc_id % num_gpus) + os.environ['RANK'] = str(proc_id) + dist.init_process_group(backend=backend) + + +def get_dist_info(): + if dist.is_available() and dist.is_initialized(): + rank = dist.get_rank() + world_size = dist.get_world_size() + else: + rank = 0 + world_size = 1 + return rank, world_size + + +def master_only(func): + + @functools.wraps(func) + def wrapper(*args, **kwargs): + rank, _ = get_dist_info() + if rank == 0: + return func(*args, **kwargs) + + return wrapper + + +def allreduce_params(params, coalesce=True, bucket_size_mb=-1): + """Allreduce parameters. + + Args: + params (list[torch.Parameters]): List of parameters or buffers of a + model. + coalesce (bool, optional): Whether allreduce parameters as a whole. + Defaults to True. + bucket_size_mb (int, optional): Size of bucket, the unit is MB. + Defaults to -1. + """ + _, world_size = get_dist_info() + if world_size == 1: + return + params = [param.data for param in params] + if coalesce: + _allreduce_coalesced(params, world_size, bucket_size_mb) + else: + for tensor in params: + dist.all_reduce(tensor.div_(world_size)) + + +def allreduce_grads(params, coalesce=True, bucket_size_mb=-1): + """Allreduce gradients. + + Args: + params (list[torch.Parameters]): List of parameters of a model + coalesce (bool, optional): Whether allreduce parameters as a whole. + Defaults to True. + bucket_size_mb (int, optional): Size of bucket, the unit is MB. + Defaults to -1. + """ + grads = [ + param.grad.data for param in params + if param.requires_grad and param.grad is not None + ] + _, world_size = get_dist_info() + if world_size == 1: + return + if coalesce: + _allreduce_coalesced(grads, world_size, bucket_size_mb) + else: + for tensor in grads: + dist.all_reduce(tensor.div_(world_size)) + + +def _allreduce_coalesced(tensors, world_size, bucket_size_mb=-1): + if bucket_size_mb > 0: + bucket_size_bytes = bucket_size_mb * 1024 * 1024 + buckets = _take_tensors(tensors, bucket_size_bytes) + else: + buckets = OrderedDict() + for tensor in tensors: + tp = tensor.type() + if tp not in buckets: + buckets[tp] = [] + buckets[tp].append(tensor) + buckets = buckets.values() + + for bucket in buckets: + flat_tensors = _flatten_dense_tensors(bucket) + dist.all_reduce(flat_tensors) + flat_tensors.div_(world_size) + for tensor, synced in zip( + bucket, _unflatten_dense_tensors(flat_tensors, bucket)): + tensor.copy_(synced) diff --git a/annotator/mmpkg/mmcv/runner/epoch_based_runner.py b/annotator/mmpkg/mmcv/runner/epoch_based_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..d4df071e1740baa4aea2951590ac929b3715daa2 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/epoch_based_runner.py @@ -0,0 +1,187 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os.path as osp +import platform +import shutil +import time +import warnings + +import torch + +import annotator.mmpkg.mmcv as mmcv +from .base_runner import BaseRunner +from .builder import RUNNERS +from .checkpoint import save_checkpoint +from .utils import get_host_info + + +@RUNNERS.register_module() +class EpochBasedRunner(BaseRunner): + """Epoch-based Runner. + + This runner train models epoch by epoch. + """ + + def run_iter(self, data_batch, train_mode, **kwargs): + if self.batch_processor is not None: + outputs = self.batch_processor( + self.model, data_batch, train_mode=train_mode, **kwargs) + elif train_mode: + outputs = self.model.train_step(data_batch, self.optimizer, + **kwargs) + else: + outputs = self.model.val_step(data_batch, self.optimizer, **kwargs) + if not isinstance(outputs, dict): + raise TypeError('"batch_processor()" or "model.train_step()"' + 'and "model.val_step()" must return a dict') + if 'log_vars' in outputs: + self.log_buffer.update(outputs['log_vars'], outputs['num_samples']) + self.outputs = outputs + + def train(self, data_loader, **kwargs): + self.model.train() + self.mode = 'train' + self.data_loader = data_loader + self._max_iters = self._max_epochs * len(self.data_loader) + self.call_hook('before_train_epoch') + time.sleep(2) # Prevent possible deadlock during epoch transition + for i, data_batch in enumerate(self.data_loader): + self._inner_iter = i + self.call_hook('before_train_iter') + self.run_iter(data_batch, train_mode=True, **kwargs) + self.call_hook('after_train_iter') + self._iter += 1 + + self.call_hook('after_train_epoch') + self._epoch += 1 + + @torch.no_grad() + def val(self, data_loader, **kwargs): + self.model.eval() + self.mode = 'val' + self.data_loader = data_loader + self.call_hook('before_val_epoch') + time.sleep(2) # Prevent possible deadlock during epoch transition + for i, data_batch in enumerate(self.data_loader): + self._inner_iter = i + self.call_hook('before_val_iter') + self.run_iter(data_batch, train_mode=False) + self.call_hook('after_val_iter') + + self.call_hook('after_val_epoch') + + def run(self, data_loaders, workflow, max_epochs=None, **kwargs): + """Start running. + + Args: + data_loaders (list[:obj:`DataLoader`]): Dataloaders for training + and validation. + workflow (list[tuple]): A list of (phase, epochs) to specify the + running order and epochs. E.g, [('train', 2), ('val', 1)] means + running 2 epochs for training and 1 epoch for validation, + iteratively. + """ + assert isinstance(data_loaders, list) + assert mmcv.is_list_of(workflow, tuple) + assert len(data_loaders) == len(workflow) + if max_epochs is not None: + warnings.warn( + 'setting max_epochs in run is deprecated, ' + 'please set max_epochs in runner_config', DeprecationWarning) + self._max_epochs = max_epochs + + assert self._max_epochs is not None, ( + 'max_epochs must be specified during instantiation') + + for i, flow in enumerate(workflow): + mode, epochs = flow + if mode == 'train': + self._max_iters = self._max_epochs * len(data_loaders[i]) + break + + work_dir = self.work_dir if self.work_dir is not None else 'NONE' + self.logger.info('Start running, host: %s, work_dir: %s', + get_host_info(), work_dir) + self.logger.info('Hooks will be executed in the following order:\n%s', + self.get_hook_info()) + self.logger.info('workflow: %s, max: %d epochs', workflow, + self._max_epochs) + self.call_hook('before_run') + + while self.epoch < self._max_epochs: + for i, flow in enumerate(workflow): + mode, epochs = flow + if isinstance(mode, str): # self.train() + if not hasattr(self, mode): + raise ValueError( + f'runner has no method named "{mode}" to run an ' + 'epoch') + epoch_runner = getattr(self, mode) + else: + raise TypeError( + 'mode in workflow must be a str, but got {}'.format( + type(mode))) + + for _ in range(epochs): + if mode == 'train' and self.epoch >= self._max_epochs: + break + epoch_runner(data_loaders[i], **kwargs) + + time.sleep(1) # wait for some hooks like loggers to finish + self.call_hook('after_run') + + def save_checkpoint(self, + out_dir, + filename_tmpl='epoch_{}.pth', + save_optimizer=True, + meta=None, + create_symlink=True): + """Save the checkpoint. + + Args: + out_dir (str): The directory that checkpoints are saved. + filename_tmpl (str, optional): The checkpoint filename template, + which contains a placeholder for the epoch number. + Defaults to 'epoch_{}.pth'. + save_optimizer (bool, optional): Whether to save the optimizer to + the checkpoint. Defaults to True. + meta (dict, optional): The meta information to be saved in the + checkpoint. Defaults to None. + create_symlink (bool, optional): Whether to create a symlink + "latest.pth" to point to the latest checkpoint. + Defaults to True. + """ + if meta is None: + meta = {} + elif not isinstance(meta, dict): + raise TypeError( + f'meta should be a dict or None, but got {type(meta)}') + if self.meta is not None: + meta.update(self.meta) + # Note: meta.update(self.meta) should be done before + # meta.update(epoch=self.epoch + 1, iter=self.iter) otherwise + # there will be problems with resumed checkpoints. + # More details in https://github.com/open-mmlab/mmcv/pull/1108 + meta.update(epoch=self.epoch + 1, iter=self.iter) + + filename = filename_tmpl.format(self.epoch + 1) + filepath = osp.join(out_dir, filename) + optimizer = self.optimizer if save_optimizer else None + save_checkpoint(self.model, filepath, optimizer=optimizer, meta=meta) + # in some environments, `os.symlink` is not supported, you may need to + # set `create_symlink` to False + if create_symlink: + dst_file = osp.join(out_dir, 'latest.pth') + if platform.system() != 'Windows': + mmcv.symlink(filename, dst_file) + else: + shutil.copy(filepath, dst_file) + + +@RUNNERS.register_module() +class Runner(EpochBasedRunner): + """Deprecated name of EpochBasedRunner.""" + + def __init__(self, *args, **kwargs): + warnings.warn( + 'Runner was deprecated, please use EpochBasedRunner instead') + super().__init__(*args, **kwargs) diff --git a/annotator/mmpkg/mmcv/runner/fp16_utils.py b/annotator/mmpkg/mmcv/runner/fp16_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f6b54886519fd2808360b1632e5bebf6563eced2 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/fp16_utils.py @@ -0,0 +1,410 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import functools +import warnings +from collections import abc +from inspect import getfullargspec + +import numpy as np +import torch +import torch.nn as nn + +from annotator.mmpkg.mmcv.utils import TORCH_VERSION, digit_version +from .dist_utils import allreduce_grads as _allreduce_grads + +try: + # If PyTorch version >= 1.6.0, torch.cuda.amp.autocast would be imported + # and used; otherwise, auto fp16 will adopt mmcv's implementation. + # Note that when PyTorch >= 1.6.0, we still cast tensor types to fp16 + # manually, so the behavior may not be consistent with real amp. + from torch.cuda.amp import autocast +except ImportError: + pass + + +def cast_tensor_type(inputs, src_type, dst_type): + """Recursively convert Tensor in inputs from src_type to dst_type. + + Args: + inputs: Inputs that to be casted. + src_type (torch.dtype): Source type.. + dst_type (torch.dtype): Destination type. + + Returns: + The same type with inputs, but all contained Tensors have been cast. + """ + if isinstance(inputs, nn.Module): + return inputs + elif isinstance(inputs, torch.Tensor): + return inputs.to(dst_type) + elif isinstance(inputs, str): + return inputs + elif isinstance(inputs, np.ndarray): + return inputs + elif isinstance(inputs, abc.Mapping): + return type(inputs)({ + k: cast_tensor_type(v, src_type, dst_type) + for k, v in inputs.items() + }) + elif isinstance(inputs, abc.Iterable): + return type(inputs)( + cast_tensor_type(item, src_type, dst_type) for item in inputs) + else: + return inputs + + +def auto_fp16(apply_to=None, out_fp32=False): + """Decorator to enable fp16 training automatically. + + This decorator is useful when you write custom modules and want to support + mixed precision training. If inputs arguments are fp32 tensors, they will + be converted to fp16 automatically. Arguments other than fp32 tensors are + ignored. If you are using PyTorch >= 1.6, torch.cuda.amp is used as the + backend, otherwise, original mmcv implementation will be adopted. + + Args: + apply_to (Iterable, optional): The argument names to be converted. + `None` indicates all arguments. + out_fp32 (bool): Whether to convert the output back to fp32. + + Example: + + >>> import torch.nn as nn + >>> class MyModule1(nn.Module): + >>> + >>> # Convert x and y to fp16 + >>> @auto_fp16() + >>> def forward(self, x, y): + >>> pass + + >>> import torch.nn as nn + >>> class MyModule2(nn.Module): + >>> + >>> # convert pred to fp16 + >>> @auto_fp16(apply_to=('pred', )) + >>> def do_something(self, pred, others): + >>> pass + """ + + def auto_fp16_wrapper(old_func): + + @functools.wraps(old_func) + def new_func(*args, **kwargs): + # check if the module has set the attribute `fp16_enabled`, if not, + # just fallback to the original method. + if not isinstance(args[0], torch.nn.Module): + raise TypeError('@auto_fp16 can only be used to decorate the ' + 'method of nn.Module') + if not (hasattr(args[0], 'fp16_enabled') and args[0].fp16_enabled): + return old_func(*args, **kwargs) + + # get the arg spec of the decorated method + args_info = getfullargspec(old_func) + # get the argument names to be casted + args_to_cast = args_info.args if apply_to is None else apply_to + # convert the args that need to be processed + new_args = [] + # NOTE: default args are not taken into consideration + if args: + arg_names = args_info.args[:len(args)] + for i, arg_name in enumerate(arg_names): + if arg_name in args_to_cast: + new_args.append( + cast_tensor_type(args[i], torch.float, torch.half)) + else: + new_args.append(args[i]) + # convert the kwargs that need to be processed + new_kwargs = {} + if kwargs: + for arg_name, arg_value in kwargs.items(): + if arg_name in args_to_cast: + new_kwargs[arg_name] = cast_tensor_type( + arg_value, torch.float, torch.half) + else: + new_kwargs[arg_name] = arg_value + # apply converted arguments to the decorated method + if (TORCH_VERSION != 'parrots' and + digit_version(TORCH_VERSION) >= digit_version('1.6.0')): + with autocast(enabled=True): + output = old_func(*new_args, **new_kwargs) + else: + output = old_func(*new_args, **new_kwargs) + # cast the results back to fp32 if necessary + if out_fp32: + output = cast_tensor_type(output, torch.half, torch.float) + return output + + return new_func + + return auto_fp16_wrapper + + +def force_fp32(apply_to=None, out_fp16=False): + """Decorator to convert input arguments to fp32 in force. + + This decorator is useful when you write custom modules and want to support + mixed precision training. If there are some inputs that must be processed + in fp32 mode, then this decorator can handle it. If inputs arguments are + fp16 tensors, they will be converted to fp32 automatically. Arguments other + than fp16 tensors are ignored. If you are using PyTorch >= 1.6, + torch.cuda.amp is used as the backend, otherwise, original mmcv + implementation will be adopted. + + Args: + apply_to (Iterable, optional): The argument names to be converted. + `None` indicates all arguments. + out_fp16 (bool): Whether to convert the output back to fp16. + + Example: + + >>> import torch.nn as nn + >>> class MyModule1(nn.Module): + >>> + >>> # Convert x and y to fp32 + >>> @force_fp32() + >>> def loss(self, x, y): + >>> pass + + >>> import torch.nn as nn + >>> class MyModule2(nn.Module): + >>> + >>> # convert pred to fp32 + >>> @force_fp32(apply_to=('pred', )) + >>> def post_process(self, pred, others): + >>> pass + """ + + def force_fp32_wrapper(old_func): + + @functools.wraps(old_func) + def new_func(*args, **kwargs): + # check if the module has set the attribute `fp16_enabled`, if not, + # just fallback to the original method. + if not isinstance(args[0], torch.nn.Module): + raise TypeError('@force_fp32 can only be used to decorate the ' + 'method of nn.Module') + if not (hasattr(args[0], 'fp16_enabled') and args[0].fp16_enabled): + return old_func(*args, **kwargs) + # get the arg spec of the decorated method + args_info = getfullargspec(old_func) + # get the argument names to be casted + args_to_cast = args_info.args if apply_to is None else apply_to + # convert the args that need to be processed + new_args = [] + if args: + arg_names = args_info.args[:len(args)] + for i, arg_name in enumerate(arg_names): + if arg_name in args_to_cast: + new_args.append( + cast_tensor_type(args[i], torch.half, torch.float)) + else: + new_args.append(args[i]) + # convert the kwargs that need to be processed + new_kwargs = dict() + if kwargs: + for arg_name, arg_value in kwargs.items(): + if arg_name in args_to_cast: + new_kwargs[arg_name] = cast_tensor_type( + arg_value, torch.half, torch.float) + else: + new_kwargs[arg_name] = arg_value + # apply converted arguments to the decorated method + if (TORCH_VERSION != 'parrots' and + digit_version(TORCH_VERSION) >= digit_version('1.6.0')): + with autocast(enabled=False): + output = old_func(*new_args, **new_kwargs) + else: + output = old_func(*new_args, **new_kwargs) + # cast the results back to fp32 if necessary + if out_fp16: + output = cast_tensor_type(output, torch.float, torch.half) + return output + + return new_func + + return force_fp32_wrapper + + +def allreduce_grads(params, coalesce=True, bucket_size_mb=-1): + warnings.warning( + '"mmcv.runner.fp16_utils.allreduce_grads" is deprecated, and will be ' + 'removed in v2.8. Please switch to "mmcv.runner.allreduce_grads') + _allreduce_grads(params, coalesce=coalesce, bucket_size_mb=bucket_size_mb) + + +def wrap_fp16_model(model): + """Wrap the FP32 model to FP16. + + If you are using PyTorch >= 1.6, torch.cuda.amp is used as the + backend, otherwise, original mmcv implementation will be adopted. + + For PyTorch >= 1.6, this function will + 1. Set fp16 flag inside the model to True. + + Otherwise: + 1. Convert FP32 model to FP16. + 2. Remain some necessary layers to be FP32, e.g., normalization layers. + 3. Set `fp16_enabled` flag inside the model to True. + + Args: + model (nn.Module): Model in FP32. + """ + if (TORCH_VERSION == 'parrots' + or digit_version(TORCH_VERSION) < digit_version('1.6.0')): + # convert model to fp16 + model.half() + # patch the normalization layers to make it work in fp32 mode + patch_norm_fp32(model) + # set `fp16_enabled` flag + for m in model.modules(): + if hasattr(m, 'fp16_enabled'): + m.fp16_enabled = True + + +def patch_norm_fp32(module): + """Recursively convert normalization layers from FP16 to FP32. + + Args: + module (nn.Module): The modules to be converted in FP16. + + Returns: + nn.Module: The converted module, the normalization layers have been + converted to FP32. + """ + if isinstance(module, (nn.modules.batchnorm._BatchNorm, nn.GroupNorm)): + module.float() + if isinstance(module, nn.GroupNorm) or torch.__version__ < '1.3': + module.forward = patch_forward_method(module.forward, torch.half, + torch.float) + for child in module.children(): + patch_norm_fp32(child) + return module + + +def patch_forward_method(func, src_type, dst_type, convert_output=True): + """Patch the forward method of a module. + + Args: + func (callable): The original forward method. + src_type (torch.dtype): Type of input arguments to be converted from. + dst_type (torch.dtype): Type of input arguments to be converted to. + convert_output (bool): Whether to convert the output back to src_type. + + Returns: + callable: The patched forward method. + """ + + def new_forward(*args, **kwargs): + output = func(*cast_tensor_type(args, src_type, dst_type), + **cast_tensor_type(kwargs, src_type, dst_type)) + if convert_output: + output = cast_tensor_type(output, dst_type, src_type) + return output + + return new_forward + + +class LossScaler: + """Class that manages loss scaling in mixed precision training which + supports both dynamic or static mode. + + The implementation refers to + https://github.com/NVIDIA/apex/blob/master/apex/fp16_utils/loss_scaler.py. + Indirectly, by supplying ``mode='dynamic'`` for dynamic loss scaling. + It's important to understand how :class:`LossScaler` operates. + Loss scaling is designed to combat the problem of underflowing + gradients encountered at long times when training fp16 networks. + Dynamic loss scaling begins by attempting a very high loss + scale. Ironically, this may result in OVERflowing gradients. + If overflowing gradients are encountered, :class:`FP16_Optimizer` then + skips the update step for this particular iteration/minibatch, + and :class:`LossScaler` adjusts the loss scale to a lower value. + If a certain number of iterations occur without overflowing gradients + detected,:class:`LossScaler` increases the loss scale once more. + In this way :class:`LossScaler` attempts to "ride the edge" of always + using the highest loss scale possible without incurring overflow. + + Args: + init_scale (float): Initial loss scale value, default: 2**32. + scale_factor (float): Factor used when adjusting the loss scale. + Default: 2. + mode (str): Loss scaling mode. 'dynamic' or 'static' + scale_window (int): Number of consecutive iterations without an + overflow to wait before increasing the loss scale. Default: 1000. + """ + + def __init__(self, + init_scale=2**32, + mode='dynamic', + scale_factor=2., + scale_window=1000): + self.cur_scale = init_scale + self.cur_iter = 0 + assert mode in ('dynamic', + 'static'), 'mode can only be dynamic or static' + self.mode = mode + self.last_overflow_iter = -1 + self.scale_factor = scale_factor + self.scale_window = scale_window + + def has_overflow(self, params): + """Check if params contain overflow.""" + if self.mode != 'dynamic': + return False + for p in params: + if p.grad is not None and LossScaler._has_inf_or_nan(p.grad.data): + return True + return False + + def _has_inf_or_nan(x): + """Check if params contain NaN.""" + try: + cpu_sum = float(x.float().sum()) + except RuntimeError as instance: + if 'value cannot be converted' not in instance.args[0]: + raise + return True + else: + if cpu_sum == float('inf') or cpu_sum == -float('inf') \ + or cpu_sum != cpu_sum: + return True + return False + + def update_scale(self, overflow): + """update the current loss scale value when overflow happens.""" + if self.mode != 'dynamic': + return + if overflow: + self.cur_scale = max(self.cur_scale / self.scale_factor, 1) + self.last_overflow_iter = self.cur_iter + else: + if (self.cur_iter - self.last_overflow_iter) % \ + self.scale_window == 0: + self.cur_scale *= self.scale_factor + self.cur_iter += 1 + + def state_dict(self): + """Returns the state of the scaler as a :class:`dict`.""" + return dict( + cur_scale=self.cur_scale, + cur_iter=self.cur_iter, + mode=self.mode, + last_overflow_iter=self.last_overflow_iter, + scale_factor=self.scale_factor, + scale_window=self.scale_window) + + def load_state_dict(self, state_dict): + """Loads the loss_scaler state dict. + + Args: + state_dict (dict): scaler state. + """ + self.cur_scale = state_dict['cur_scale'] + self.cur_iter = state_dict['cur_iter'] + self.mode = state_dict['mode'] + self.last_overflow_iter = state_dict['last_overflow_iter'] + self.scale_factor = state_dict['scale_factor'] + self.scale_window = state_dict['scale_window'] + + @property + def loss_scale(self): + return self.cur_scale diff --git a/annotator/mmpkg/mmcv/runner/hooks/__init__.py b/annotator/mmpkg/mmcv/runner/hooks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..915af28cefab14a14c1188ed861161080fd138a3 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/__init__.py @@ -0,0 +1,29 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .checkpoint import CheckpointHook +from .closure import ClosureHook +from .ema import EMAHook +from .evaluation import DistEvalHook, EvalHook +from .hook import HOOKS, Hook +from .iter_timer import IterTimerHook +from .logger import (DvcliveLoggerHook, LoggerHook, MlflowLoggerHook, + NeptuneLoggerHook, PaviLoggerHook, TensorboardLoggerHook, + TextLoggerHook, WandbLoggerHook) +from .lr_updater import LrUpdaterHook +from .memory import EmptyCacheHook +from .momentum_updater import MomentumUpdaterHook +from .optimizer import (Fp16OptimizerHook, GradientCumulativeFp16OptimizerHook, + GradientCumulativeOptimizerHook, OptimizerHook) +from .profiler import ProfilerHook +from .sampler_seed import DistSamplerSeedHook +from .sync_buffer import SyncBuffersHook + +__all__ = [ + 'HOOKS', 'Hook', 'CheckpointHook', 'ClosureHook', 'LrUpdaterHook', + 'OptimizerHook', 'Fp16OptimizerHook', 'IterTimerHook', + 'DistSamplerSeedHook', 'EmptyCacheHook', 'LoggerHook', 'MlflowLoggerHook', + 'PaviLoggerHook', 'TextLoggerHook', 'TensorboardLoggerHook', + 'NeptuneLoggerHook', 'WandbLoggerHook', 'DvcliveLoggerHook', + 'MomentumUpdaterHook', 'SyncBuffersHook', 'EMAHook', 'EvalHook', + 'DistEvalHook', 'ProfilerHook', 'GradientCumulativeOptimizerHook', + 'GradientCumulativeFp16OptimizerHook' +] diff --git a/annotator/mmpkg/mmcv/runner/hooks/checkpoint.py b/annotator/mmpkg/mmcv/runner/hooks/checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..877aa8b84ac48bea0a06f9d0733d74f88be2ecfc --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/checkpoint.py @@ -0,0 +1,167 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os.path as osp +import warnings + +from annotator.mmpkg.mmcv.fileio import FileClient +from ..dist_utils import allreduce_params, master_only +from .hook import HOOKS, Hook + + +@HOOKS.register_module() +class CheckpointHook(Hook): + """Save checkpoints periodically. + + Args: + interval (int): The saving period. If ``by_epoch=True``, interval + indicates epochs, otherwise it indicates iterations. + Default: -1, which means "never". + by_epoch (bool): Saving checkpoints by epoch or by iteration. + Default: True. + save_optimizer (bool): Whether to save optimizer state_dict in the + checkpoint. It is usually used for resuming experiments. + Default: True. + out_dir (str, optional): The root directory to save checkpoints. If not + specified, ``runner.work_dir`` will be used by default. If + specified, the ``out_dir`` will be the concatenation of ``out_dir`` + and the last level directory of ``runner.work_dir``. + `Changed in version 1.3.16.` + max_keep_ckpts (int, optional): The maximum checkpoints to keep. + In some cases we want only the latest few checkpoints and would + like to delete old ones to save the disk space. + Default: -1, which means unlimited. + save_last (bool, optional): Whether to force the last checkpoint to be + saved regardless of interval. Default: True. + sync_buffer (bool, optional): Whether to synchronize buffers in + different gpus. Default: False. + file_client_args (dict, optional): Arguments to instantiate a + FileClient. See :class:`mmcv.fileio.FileClient` for details. + Default: None. + `New in version 1.3.16.` + + .. warning:: + Before v1.3.16, the ``out_dir`` argument indicates the path where the + checkpoint is stored. However, since v1.3.16, ``out_dir`` indicates the + root directory and the final path to save checkpoint is the + concatenation of ``out_dir`` and the last level directory of + ``runner.work_dir``. Suppose the value of ``out_dir`` is "/path/of/A" + and the value of ``runner.work_dir`` is "/path/of/B", then the final + path will be "/path/of/A/B". + """ + + def __init__(self, + interval=-1, + by_epoch=True, + save_optimizer=True, + out_dir=None, + max_keep_ckpts=-1, + save_last=True, + sync_buffer=False, + file_client_args=None, + **kwargs): + self.interval = interval + self.by_epoch = by_epoch + self.save_optimizer = save_optimizer + self.out_dir = out_dir + self.max_keep_ckpts = max_keep_ckpts + self.save_last = save_last + self.args = kwargs + self.sync_buffer = sync_buffer + self.file_client_args = file_client_args + + def before_run(self, runner): + if not self.out_dir: + self.out_dir = runner.work_dir + + self.file_client = FileClient.infer_client(self.file_client_args, + self.out_dir) + + # if `self.out_dir` is not equal to `runner.work_dir`, it means that + # `self.out_dir` is set so the final `self.out_dir` is the + # concatenation of `self.out_dir` and the last level directory of + # `runner.work_dir` + if self.out_dir != runner.work_dir: + basename = osp.basename(runner.work_dir.rstrip(osp.sep)) + self.out_dir = self.file_client.join_path(self.out_dir, basename) + + runner.logger.info((f'Checkpoints will be saved to {self.out_dir} by ' + f'{self.file_client.name}.')) + + # disable the create_symlink option because some file backends do not + # allow to create a symlink + if 'create_symlink' in self.args: + if self.args[ + 'create_symlink'] and not self.file_client.allow_symlink: + self.args['create_symlink'] = False + warnings.warn( + ('create_symlink is set as True by the user but is changed' + 'to be False because creating symbolic link is not ' + f'allowed in {self.file_client.name}')) + else: + self.args['create_symlink'] = self.file_client.allow_symlink + + def after_train_epoch(self, runner): + if not self.by_epoch: + return + + # save checkpoint for following cases: + # 1. every ``self.interval`` epochs + # 2. reach the last epoch of training + if self.every_n_epochs( + runner, self.interval) or (self.save_last + and self.is_last_epoch(runner)): + runner.logger.info( + f'Saving checkpoint at {runner.epoch + 1} epochs') + if self.sync_buffer: + allreduce_params(runner.model.buffers()) + self._save_checkpoint(runner) + + @master_only + def _save_checkpoint(self, runner): + """Save the current checkpoint and delete unwanted checkpoint.""" + runner.save_checkpoint( + self.out_dir, save_optimizer=self.save_optimizer, **self.args) + if runner.meta is not None: + if self.by_epoch: + cur_ckpt_filename = self.args.get( + 'filename_tmpl', 'epoch_{}.pth').format(runner.epoch + 1) + else: + cur_ckpt_filename = self.args.get( + 'filename_tmpl', 'iter_{}.pth').format(runner.iter + 1) + runner.meta.setdefault('hook_msgs', dict()) + runner.meta['hook_msgs']['last_ckpt'] = self.file_client.join_path( + self.out_dir, cur_ckpt_filename) + # remove other checkpoints + if self.max_keep_ckpts > 0: + if self.by_epoch: + name = 'epoch_{}.pth' + current_ckpt = runner.epoch + 1 + else: + name = 'iter_{}.pth' + current_ckpt = runner.iter + 1 + redundant_ckpts = range( + current_ckpt - self.max_keep_ckpts * self.interval, 0, + -self.interval) + filename_tmpl = self.args.get('filename_tmpl', name) + for _step in redundant_ckpts: + ckpt_path = self.file_client.join_path( + self.out_dir, filename_tmpl.format(_step)) + if self.file_client.isfile(ckpt_path): + self.file_client.remove(ckpt_path) + else: + break + + def after_train_iter(self, runner): + if self.by_epoch: + return + + # save checkpoint for following cases: + # 1. every ``self.interval`` iterations + # 2. reach the last iteration of training + if self.every_n_iters( + runner, self.interval) or (self.save_last + and self.is_last_iter(runner)): + runner.logger.info( + f'Saving checkpoint at {runner.iter + 1} iterations') + if self.sync_buffer: + allreduce_params(runner.model.buffers()) + self._save_checkpoint(runner) diff --git a/annotator/mmpkg/mmcv/runner/hooks/closure.py b/annotator/mmpkg/mmcv/runner/hooks/closure.py new file mode 100644 index 0000000000000000000000000000000000000000..b955f81f425be4ac3e6bb3f4aac653887989e872 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/closure.py @@ -0,0 +1,11 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .hook import HOOKS, Hook + + +@HOOKS.register_module() +class ClosureHook(Hook): + + def __init__(self, fn_name, fn): + assert hasattr(self, fn_name) + assert callable(fn) + setattr(self, fn_name, fn) diff --git a/annotator/mmpkg/mmcv/runner/hooks/ema.py b/annotator/mmpkg/mmcv/runner/hooks/ema.py new file mode 100644 index 0000000000000000000000000000000000000000..15c7e68088f019802a59e7ae41cc1fe0c7f28f96 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/ema.py @@ -0,0 +1,89 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from ...parallel import is_module_wrapper +from ..hooks.hook import HOOKS, Hook + + +@HOOKS.register_module() +class EMAHook(Hook): + r"""Exponential Moving Average Hook. + + Use Exponential Moving Average on all parameters of model in training + process. All parameters have a ema backup, which update by the formula + as below. EMAHook takes priority over EvalHook and CheckpointSaverHook. + + .. math:: + + \text{Xema\_{t+1}} = (1 - \text{momentum}) \times + \text{Xema\_{t}} + \text{momentum} \times X_t + + Args: + momentum (float): The momentum used for updating ema parameter. + Defaults to 0.0002. + interval (int): Update ema parameter every interval iteration. + Defaults to 1. + warm_up (int): During first warm_up steps, we may use smaller momentum + to update ema parameters more slowly. Defaults to 100. + resume_from (str): The checkpoint path. Defaults to None. + """ + + def __init__(self, + momentum=0.0002, + interval=1, + warm_up=100, + resume_from=None): + assert isinstance(interval, int) and interval > 0 + self.warm_up = warm_up + self.interval = interval + assert momentum > 0 and momentum < 1 + self.momentum = momentum**interval + self.checkpoint = resume_from + + def before_run(self, runner): + """To resume model with it's ema parameters more friendly. + + Register ema parameter as ``named_buffer`` to model + """ + model = runner.model + if is_module_wrapper(model): + model = model.module + self.param_ema_buffer = {} + self.model_parameters = dict(model.named_parameters(recurse=True)) + for name, value in self.model_parameters.items(): + # "." is not allowed in module's buffer name + buffer_name = f"ema_{name.replace('.', '_')}" + self.param_ema_buffer[name] = buffer_name + model.register_buffer(buffer_name, value.data.clone()) + self.model_buffers = dict(model.named_buffers(recurse=True)) + if self.checkpoint is not None: + runner.resume(self.checkpoint) + + def after_train_iter(self, runner): + """Update ema parameter every self.interval iterations.""" + curr_step = runner.iter + # We warm up the momentum considering the instability at beginning + momentum = min(self.momentum, + (1 + curr_step) / (self.warm_up + curr_step)) + if curr_step % self.interval != 0: + return + for name, parameter in self.model_parameters.items(): + buffer_name = self.param_ema_buffer[name] + buffer_parameter = self.model_buffers[buffer_name] + buffer_parameter.mul_(1 - momentum).add_(momentum, parameter.data) + + def after_train_epoch(self, runner): + """We load parameter values from ema backup to model before the + EvalHook.""" + self._swap_ema_parameters() + + def before_train_epoch(self, runner): + """We recover model's parameter from ema backup after last epoch's + EvalHook.""" + self._swap_ema_parameters() + + def _swap_ema_parameters(self): + """Swap the parameter of model with parameter in ema_buffer.""" + for name, value in self.model_parameters.items(): + temp = value.data.clone() + ema_buffer = self.model_buffers[self.param_ema_buffer[name]] + value.data.copy_(ema_buffer.data) + ema_buffer.data.copy_(temp) diff --git a/annotator/mmpkg/mmcv/runner/hooks/evaluation.py b/annotator/mmpkg/mmcv/runner/hooks/evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..a1dbdfd593bae505a70534226b79791baec6453e --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/evaluation.py @@ -0,0 +1,509 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os.path as osp +import warnings +from math import inf + +import torch.distributed as dist +from torch.nn.modules.batchnorm import _BatchNorm +from torch.utils.data import DataLoader + +from annotator.mmpkg.mmcv.fileio import FileClient +from annotator.mmpkg.mmcv.utils import is_seq_of +from .hook import Hook +from .logger import LoggerHook + + +class EvalHook(Hook): + """Non-Distributed evaluation hook. + + This hook will regularly perform evaluation in a given interval when + performing in non-distributed environment. + + Args: + dataloader (DataLoader): A PyTorch dataloader, whose dataset has + implemented ``evaluate`` function. + start (int | None, optional): Evaluation starting epoch. It enables + evaluation before the training starts if ``start`` <= the resuming + epoch. If None, whether to evaluate is merely decided by + ``interval``. Default: None. + interval (int): Evaluation interval. Default: 1. + by_epoch (bool): Determine perform evaluation by epoch or by iteration. + If set to True, it will perform by epoch. Otherwise, by iteration. + Default: True. + save_best (str, optional): If a metric is specified, it would measure + the best checkpoint during evaluation. The information about best + checkpoint would be saved in ``runner.meta['hook_msgs']`` to keep + best score value and best checkpoint path, which will be also + loaded when resume checkpoint. Options are the evaluation metrics + on the test dataset. e.g., ``bbox_mAP``, ``segm_mAP`` for bbox + detection and instance segmentation. ``AR@100`` for proposal + recall. If ``save_best`` is ``auto``, the first key of the returned + ``OrderedDict`` result will be used. Default: None. + rule (str | None, optional): Comparison rule for best score. If set to + None, it will infer a reasonable rule. Keys such as 'acc', 'top' + .etc will be inferred by 'greater' rule. Keys contain 'loss' will + be inferred by 'less' rule. Options are 'greater', 'less', None. + Default: None. + test_fn (callable, optional): test a model with samples from a + dataloader, and return the test results. If ``None``, the default + test function ``mmcv.engine.single_gpu_test`` will be used. + (default: ``None``) + greater_keys (List[str] | None, optional): Metric keys that will be + inferred by 'greater' comparison rule. If ``None``, + _default_greater_keys will be used. (default: ``None``) + less_keys (List[str] | None, optional): Metric keys that will be + inferred by 'less' comparison rule. If ``None``, _default_less_keys + will be used. (default: ``None``) + out_dir (str, optional): The root directory to save checkpoints. If not + specified, `runner.work_dir` will be used by default. If specified, + the `out_dir` will be the concatenation of `out_dir` and the last + level directory of `runner.work_dir`. + `New in version 1.3.16.` + file_client_args (dict): Arguments to instantiate a FileClient. + See :class:`mmcv.fileio.FileClient` for details. Default: None. + `New in version 1.3.16.` + **eval_kwargs: Evaluation arguments fed into the evaluate function of + the dataset. + + Notes: + If new arguments are added for EvalHook, tools/test.py, + tools/eval_metric.py may be affected. + """ + + # Since the key for determine greater or less is related to the downstream + # tasks, downstream repos may need to overwrite the following inner + # variable accordingly. + + rule_map = {'greater': lambda x, y: x > y, 'less': lambda x, y: x < y} + init_value_map = {'greater': -inf, 'less': inf} + _default_greater_keys = [ + 'acc', 'top', 'AR@', 'auc', 'precision', 'mAP', 'mDice', 'mIoU', + 'mAcc', 'aAcc' + ] + _default_less_keys = ['loss'] + + def __init__(self, + dataloader, + start=None, + interval=1, + by_epoch=True, + save_best=None, + rule=None, + test_fn=None, + greater_keys=None, + less_keys=None, + out_dir=None, + file_client_args=None, + **eval_kwargs): + if not isinstance(dataloader, DataLoader): + raise TypeError(f'dataloader must be a pytorch DataLoader, ' + f'but got {type(dataloader)}') + + if interval <= 0: + raise ValueError(f'interval must be a positive number, ' + f'but got {interval}') + + assert isinstance(by_epoch, bool), '``by_epoch`` should be a boolean' + + if start is not None and start < 0: + raise ValueError(f'The evaluation start epoch {start} is smaller ' + f'than 0') + + self.dataloader = dataloader + self.interval = interval + self.start = start + self.by_epoch = by_epoch + + assert isinstance(save_best, str) or save_best is None, \ + '""save_best"" should be a str or None ' \ + f'rather than {type(save_best)}' + self.save_best = save_best + self.eval_kwargs = eval_kwargs + self.initial_flag = True + + if test_fn is None: + from annotator.mmpkg.mmcv.engine import single_gpu_test + self.test_fn = single_gpu_test + else: + self.test_fn = test_fn + + if greater_keys is None: + self.greater_keys = self._default_greater_keys + else: + if not isinstance(greater_keys, (list, tuple)): + greater_keys = (greater_keys, ) + assert is_seq_of(greater_keys, str) + self.greater_keys = greater_keys + + if less_keys is None: + self.less_keys = self._default_less_keys + else: + if not isinstance(less_keys, (list, tuple)): + less_keys = (less_keys, ) + assert is_seq_of(less_keys, str) + self.less_keys = less_keys + + if self.save_best is not None: + self.best_ckpt_path = None + self._init_rule(rule, self.save_best) + + self.out_dir = out_dir + self.file_client_args = file_client_args + + def _init_rule(self, rule, key_indicator): + """Initialize rule, key_indicator, comparison_func, and best score. + + Here is the rule to determine which rule is used for key indicator + when the rule is not specific (note that the key indicator matching + is case-insensitive): + 1. If the key indicator is in ``self.greater_keys``, the rule will be + specified as 'greater'. + 2. Or if the key indicator is in ``self.less_keys``, the rule will be + specified as 'less'. + 3. Or if the key indicator is equal to the substring in any one item + in ``self.greater_keys``, the rule will be specified as 'greater'. + 4. Or if the key indicator is equal to the substring in any one item + in ``self.less_keys``, the rule will be specified as 'less'. + + Args: + rule (str | None): Comparison rule for best score. + key_indicator (str | None): Key indicator to determine the + comparison rule. + """ + if rule not in self.rule_map and rule is not None: + raise KeyError(f'rule must be greater, less or None, ' + f'but got {rule}.') + + if rule is None: + if key_indicator != 'auto': + # `_lc` here means we use the lower case of keys for + # case-insensitive matching + key_indicator_lc = key_indicator.lower() + greater_keys = [key.lower() for key in self.greater_keys] + less_keys = [key.lower() for key in self.less_keys] + + if key_indicator_lc in greater_keys: + rule = 'greater' + elif key_indicator_lc in less_keys: + rule = 'less' + elif any(key in key_indicator_lc for key in greater_keys): + rule = 'greater' + elif any(key in key_indicator_lc for key in less_keys): + rule = 'less' + else: + raise ValueError(f'Cannot infer the rule for key ' + f'{key_indicator}, thus a specific rule ' + f'must be specified.') + self.rule = rule + self.key_indicator = key_indicator + if self.rule is not None: + self.compare_func = self.rule_map[self.rule] + + def before_run(self, runner): + if not self.out_dir: + self.out_dir = runner.work_dir + + self.file_client = FileClient.infer_client(self.file_client_args, + self.out_dir) + + # if `self.out_dir` is not equal to `runner.work_dir`, it means that + # `self.out_dir` is set so the final `self.out_dir` is the + # concatenation of `self.out_dir` and the last level directory of + # `runner.work_dir` + if self.out_dir != runner.work_dir: + basename = osp.basename(runner.work_dir.rstrip(osp.sep)) + self.out_dir = self.file_client.join_path(self.out_dir, basename) + runner.logger.info( + (f'The best checkpoint will be saved to {self.out_dir} by ' + f'{self.file_client.name}')) + + if self.save_best is not None: + if runner.meta is None: + warnings.warn('runner.meta is None. Creating an empty one.') + runner.meta = dict() + runner.meta.setdefault('hook_msgs', dict()) + self.best_ckpt_path = runner.meta['hook_msgs'].get( + 'best_ckpt', None) + + def before_train_iter(self, runner): + """Evaluate the model only at the start of training by iteration.""" + if self.by_epoch or not self.initial_flag: + return + if self.start is not None and runner.iter >= self.start: + self.after_train_iter(runner) + self.initial_flag = False + + def before_train_epoch(self, runner): + """Evaluate the model only at the start of training by epoch.""" + if not (self.by_epoch and self.initial_flag): + return + if self.start is not None and runner.epoch >= self.start: + self.after_train_epoch(runner) + self.initial_flag = False + + def after_train_iter(self, runner): + """Called after every training iter to evaluate the results.""" + if not self.by_epoch and self._should_evaluate(runner): + # Because the priority of EvalHook is higher than LoggerHook, the + # training log and the evaluating log are mixed. Therefore, + # we need to dump the training log and clear it before evaluating + # log is generated. In addition, this problem will only appear in + # `IterBasedRunner` whose `self.by_epoch` is False, because + # `EpochBasedRunner` whose `self.by_epoch` is True calls + # `_do_evaluate` in `after_train_epoch` stage, and at this stage + # the training log has been printed, so it will not cause any + # problem. more details at + # https://github.com/open-mmlab/mmsegmentation/issues/694 + for hook in runner._hooks: + if isinstance(hook, LoggerHook): + hook.after_train_iter(runner) + runner.log_buffer.clear() + + self._do_evaluate(runner) + + def after_train_epoch(self, runner): + """Called after every training epoch to evaluate the results.""" + if self.by_epoch and self._should_evaluate(runner): + self._do_evaluate(runner) + + def _do_evaluate(self, runner): + """perform evaluation and save ckpt.""" + results = self.test_fn(runner.model, self.dataloader) + runner.log_buffer.output['eval_iter_num'] = len(self.dataloader) + key_score = self.evaluate(runner, results) + # the key_score may be `None` so it needs to skip the action to save + # the best checkpoint + if self.save_best and key_score: + self._save_ckpt(runner, key_score) + + def _should_evaluate(self, runner): + """Judge whether to perform evaluation. + + Here is the rule to judge whether to perform evaluation: + 1. It will not perform evaluation during the epoch/iteration interval, + which is determined by ``self.interval``. + 2. It will not perform evaluation if the start time is larger than + current time. + 3. It will not perform evaluation when current time is larger than + the start time but during epoch/iteration interval. + + Returns: + bool: The flag indicating whether to perform evaluation. + """ + if self.by_epoch: + current = runner.epoch + check_time = self.every_n_epochs + else: + current = runner.iter + check_time = self.every_n_iters + + if self.start is None: + if not check_time(runner, self.interval): + # No evaluation during the interval. + return False + elif (current + 1) < self.start: + # No evaluation if start is larger than the current time. + return False + else: + # Evaluation only at epochs/iters 3, 5, 7... + # if start==3 and interval==2 + if (current + 1 - self.start) % self.interval: + return False + return True + + def _save_ckpt(self, runner, key_score): + """Save the best checkpoint. + + It will compare the score according to the compare function, write + related information (best score, best checkpoint path) and save the + best checkpoint into ``work_dir``. + """ + if self.by_epoch: + current = f'epoch_{runner.epoch + 1}' + cur_type, cur_time = 'epoch', runner.epoch + 1 + else: + current = f'iter_{runner.iter + 1}' + cur_type, cur_time = 'iter', runner.iter + 1 + + best_score = runner.meta['hook_msgs'].get( + 'best_score', self.init_value_map[self.rule]) + if self.compare_func(key_score, best_score): + best_score = key_score + runner.meta['hook_msgs']['best_score'] = best_score + + if self.best_ckpt_path and self.file_client.isfile( + self.best_ckpt_path): + self.file_client.remove(self.best_ckpt_path) + runner.logger.info( + (f'The previous best checkpoint {self.best_ckpt_path} was ' + 'removed')) + + best_ckpt_name = f'best_{self.key_indicator}_{current}.pth' + self.best_ckpt_path = self.file_client.join_path( + self.out_dir, best_ckpt_name) + runner.meta['hook_msgs']['best_ckpt'] = self.best_ckpt_path + + runner.save_checkpoint( + self.out_dir, best_ckpt_name, create_symlink=False) + runner.logger.info( + f'Now best checkpoint is saved as {best_ckpt_name}.') + runner.logger.info( + f'Best {self.key_indicator} is {best_score:0.4f} ' + f'at {cur_time} {cur_type}.') + + def evaluate(self, runner, results): + """Evaluate the results. + + Args: + runner (:obj:`mmcv.Runner`): The underlined training runner. + results (list): Output results. + """ + eval_res = self.dataloader.dataset.evaluate( + results, logger=runner.logger, **self.eval_kwargs) + + for name, val in eval_res.items(): + runner.log_buffer.output[name] = val + runner.log_buffer.ready = True + + if self.save_best is not None: + # If the performance of model is pool, the `eval_res` may be an + # empty dict and it will raise exception when `self.save_best` is + # not None. More details at + # https://github.com/open-mmlab/mmdetection/issues/6265. + if not eval_res: + warnings.warn( + 'Since `eval_res` is an empty dict, the behavior to save ' + 'the best checkpoint will be skipped in this evaluation.') + return None + + if self.key_indicator == 'auto': + # infer from eval_results + self._init_rule(self.rule, list(eval_res.keys())[0]) + return eval_res[self.key_indicator] + + return None + + +class DistEvalHook(EvalHook): + """Distributed evaluation hook. + + This hook will regularly perform evaluation in a given interval when + performing in distributed environment. + + Args: + dataloader (DataLoader): A PyTorch dataloader, whose dataset has + implemented ``evaluate`` function. + start (int | None, optional): Evaluation starting epoch. It enables + evaluation before the training starts if ``start`` <= the resuming + epoch. If None, whether to evaluate is merely decided by + ``interval``. Default: None. + interval (int): Evaluation interval. Default: 1. + by_epoch (bool): Determine perform evaluation by epoch or by iteration. + If set to True, it will perform by epoch. Otherwise, by iteration. + default: True. + save_best (str, optional): If a metric is specified, it would measure + the best checkpoint during evaluation. The information about best + checkpoint would be saved in ``runner.meta['hook_msgs']`` to keep + best score value and best checkpoint path, which will be also + loaded when resume checkpoint. Options are the evaluation metrics + on the test dataset. e.g., ``bbox_mAP``, ``segm_mAP`` for bbox + detection and instance segmentation. ``AR@100`` for proposal + recall. If ``save_best`` is ``auto``, the first key of the returned + ``OrderedDict`` result will be used. Default: None. + rule (str | None, optional): Comparison rule for best score. If set to + None, it will infer a reasonable rule. Keys such as 'acc', 'top' + .etc will be inferred by 'greater' rule. Keys contain 'loss' will + be inferred by 'less' rule. Options are 'greater', 'less', None. + Default: None. + test_fn (callable, optional): test a model with samples from a + dataloader in a multi-gpu manner, and return the test results. If + ``None``, the default test function ``mmcv.engine.multi_gpu_test`` + will be used. (default: ``None``) + tmpdir (str | None): Temporary directory to save the results of all + processes. Default: None. + gpu_collect (bool): Whether to use gpu or cpu to collect results. + Default: False. + broadcast_bn_buffer (bool): Whether to broadcast the + buffer(running_mean and running_var) of rank 0 to other rank + before evaluation. Default: True. + out_dir (str, optional): The root directory to save checkpoints. If not + specified, `runner.work_dir` will be used by default. If specified, + the `out_dir` will be the concatenation of `out_dir` and the last + level directory of `runner.work_dir`. + file_client_args (dict): Arguments to instantiate a FileClient. + See :class:`mmcv.fileio.FileClient` for details. Default: None. + **eval_kwargs: Evaluation arguments fed into the evaluate function of + the dataset. + """ + + def __init__(self, + dataloader, + start=None, + interval=1, + by_epoch=True, + save_best=None, + rule=None, + test_fn=None, + greater_keys=None, + less_keys=None, + broadcast_bn_buffer=True, + tmpdir=None, + gpu_collect=False, + out_dir=None, + file_client_args=None, + **eval_kwargs): + + if test_fn is None: + from annotator.mmpkg.mmcv.engine import multi_gpu_test + test_fn = multi_gpu_test + + super().__init__( + dataloader, + start=start, + interval=interval, + by_epoch=by_epoch, + save_best=save_best, + rule=rule, + test_fn=test_fn, + greater_keys=greater_keys, + less_keys=less_keys, + out_dir=out_dir, + file_client_args=file_client_args, + **eval_kwargs) + + self.broadcast_bn_buffer = broadcast_bn_buffer + self.tmpdir = tmpdir + self.gpu_collect = gpu_collect + + def _do_evaluate(self, runner): + """perform evaluation and save ckpt.""" + # Synchronization of BatchNorm's buffer (running_mean + # and running_var) is not supported in the DDP of pytorch, + # which may cause the inconsistent performance of models in + # different ranks, so we broadcast BatchNorm's buffers + # of rank 0 to other ranks to avoid this. + if self.broadcast_bn_buffer: + model = runner.model + for name, module in model.named_modules(): + if isinstance(module, + _BatchNorm) and module.track_running_stats: + dist.broadcast(module.running_var, 0) + dist.broadcast(module.running_mean, 0) + + tmpdir = self.tmpdir + if tmpdir is None: + tmpdir = osp.join(runner.work_dir, '.eval_hook') + + results = self.test_fn( + runner.model, + self.dataloader, + tmpdir=tmpdir, + gpu_collect=self.gpu_collect) + if runner.rank == 0: + print('\n') + runner.log_buffer.output['eval_iter_num'] = len(self.dataloader) + key_score = self.evaluate(runner, results) + # the key_score may be `None` so it needs to skip the action to + # save the best checkpoint + if self.save_best and key_score: + self._save_ckpt(runner, key_score) diff --git a/annotator/mmpkg/mmcv/runner/hooks/hook.py b/annotator/mmpkg/mmcv/runner/hooks/hook.py new file mode 100644 index 0000000000000000000000000000000000000000..bd31f985fee739ccb7ac62eefc6cef9f0c0d65d0 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/hook.py @@ -0,0 +1,92 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from annotator.mmpkg.mmcv.utils import Registry, is_method_overridden + +HOOKS = Registry('hook') + + +class Hook: + stages = ('before_run', 'before_train_epoch', 'before_train_iter', + 'after_train_iter', 'after_train_epoch', 'before_val_epoch', + 'before_val_iter', 'after_val_iter', 'after_val_epoch', + 'after_run') + + def before_run(self, runner): + pass + + def after_run(self, runner): + pass + + def before_epoch(self, runner): + pass + + def after_epoch(self, runner): + pass + + def before_iter(self, runner): + pass + + def after_iter(self, runner): + pass + + def before_train_epoch(self, runner): + self.before_epoch(runner) + + def before_val_epoch(self, runner): + self.before_epoch(runner) + + def after_train_epoch(self, runner): + self.after_epoch(runner) + + def after_val_epoch(self, runner): + self.after_epoch(runner) + + def before_train_iter(self, runner): + self.before_iter(runner) + + def before_val_iter(self, runner): + self.before_iter(runner) + + def after_train_iter(self, runner): + self.after_iter(runner) + + def after_val_iter(self, runner): + self.after_iter(runner) + + def every_n_epochs(self, runner, n): + return (runner.epoch + 1) % n == 0 if n > 0 else False + + def every_n_inner_iters(self, runner, n): + return (runner.inner_iter + 1) % n == 0 if n > 0 else False + + def every_n_iters(self, runner, n): + return (runner.iter + 1) % n == 0 if n > 0 else False + + def end_of_epoch(self, runner): + return runner.inner_iter + 1 == len(runner.data_loader) + + def is_last_epoch(self, runner): + return runner.epoch + 1 == runner._max_epochs + + def is_last_iter(self, runner): + return runner.iter + 1 == runner._max_iters + + def get_triggered_stages(self): + trigger_stages = set() + for stage in Hook.stages: + if is_method_overridden(stage, Hook, self): + trigger_stages.add(stage) + + # some methods will be triggered in multi stages + # use this dict to map method to stages. + method_stages_map = { + 'before_epoch': ['before_train_epoch', 'before_val_epoch'], + 'after_epoch': ['after_train_epoch', 'after_val_epoch'], + 'before_iter': ['before_train_iter', 'before_val_iter'], + 'after_iter': ['after_train_iter', 'after_val_iter'], + } + + for method, map_stages in method_stages_map.items(): + if is_method_overridden(method, Hook, self): + trigger_stages.update(map_stages) + + return [stage for stage in Hook.stages if stage in trigger_stages] diff --git a/annotator/mmpkg/mmcv/runner/hooks/iter_timer.py b/annotator/mmpkg/mmcv/runner/hooks/iter_timer.py new file mode 100644 index 0000000000000000000000000000000000000000..cfd5002fe85ffc6992155ac01003878064a1d9be --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/iter_timer.py @@ -0,0 +1,18 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import time + +from .hook import HOOKS, Hook + + +@HOOKS.register_module() +class IterTimerHook(Hook): + + def before_epoch(self, runner): + self.t = time.time() + + def before_iter(self, runner): + runner.log_buffer.update({'data_time': time.time() - self.t}) + + def after_iter(self, runner): + runner.log_buffer.update({'time': time.time() - self.t}) + self.t = time.time() diff --git a/annotator/mmpkg/mmcv/runner/hooks/logger/__init__.py b/annotator/mmpkg/mmcv/runner/hooks/logger/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a0b6b345640a895368ac8a647afef6f24333d90e --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/logger/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .base import LoggerHook +from .dvclive import DvcliveLoggerHook +from .mlflow import MlflowLoggerHook +from .neptune import NeptuneLoggerHook +from .pavi import PaviLoggerHook +from .tensorboard import TensorboardLoggerHook +from .text import TextLoggerHook +from .wandb import WandbLoggerHook + +__all__ = [ + 'LoggerHook', 'MlflowLoggerHook', 'PaviLoggerHook', + 'TensorboardLoggerHook', 'TextLoggerHook', 'WandbLoggerHook', + 'NeptuneLoggerHook', 'DvcliveLoggerHook' +] diff --git a/annotator/mmpkg/mmcv/runner/hooks/logger/base.py b/annotator/mmpkg/mmcv/runner/hooks/logger/base.py new file mode 100644 index 0000000000000000000000000000000000000000..f845256729458ced821762a1b8ef881e17ff9955 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/logger/base.py @@ -0,0 +1,166 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numbers +from abc import ABCMeta, abstractmethod + +import numpy as np +import torch + +from ..hook import Hook + + +class LoggerHook(Hook): + """Base class for logger hooks. + + Args: + interval (int): Logging interval (every k iterations). + ignore_last (bool): Ignore the log of last iterations in each epoch + if less than `interval`. + reset_flag (bool): Whether to clear the output buffer after logging. + by_epoch (bool): Whether EpochBasedRunner is used. + """ + + __metaclass__ = ABCMeta + + def __init__(self, + interval=10, + ignore_last=True, + reset_flag=False, + by_epoch=True): + self.interval = interval + self.ignore_last = ignore_last + self.reset_flag = reset_flag + self.by_epoch = by_epoch + + @abstractmethod + def log(self, runner): + pass + + @staticmethod + def is_scalar(val, include_np=True, include_torch=True): + """Tell the input variable is a scalar or not. + + Args: + val: Input variable. + include_np (bool): Whether include 0-d np.ndarray as a scalar. + include_torch (bool): Whether include 0-d torch.Tensor as a scalar. + + Returns: + bool: True or False. + """ + if isinstance(val, numbers.Number): + return True + elif include_np and isinstance(val, np.ndarray) and val.ndim == 0: + return True + elif include_torch and isinstance(val, torch.Tensor) and len(val) == 1: + return True + else: + return False + + def get_mode(self, runner): + if runner.mode == 'train': + if 'time' in runner.log_buffer.output: + mode = 'train' + else: + mode = 'val' + elif runner.mode == 'val': + mode = 'val' + else: + raise ValueError(f"runner mode should be 'train' or 'val', " + f'but got {runner.mode}') + return mode + + def get_epoch(self, runner): + if runner.mode == 'train': + epoch = runner.epoch + 1 + elif runner.mode == 'val': + # normal val mode + # runner.epoch += 1 has been done before val workflow + epoch = runner.epoch + else: + raise ValueError(f"runner mode should be 'train' or 'val', " + f'but got {runner.mode}') + return epoch + + def get_iter(self, runner, inner_iter=False): + """Get the current training iteration step.""" + if self.by_epoch and inner_iter: + current_iter = runner.inner_iter + 1 + else: + current_iter = runner.iter + 1 + return current_iter + + def get_lr_tags(self, runner): + tags = {} + lrs = runner.current_lr() + if isinstance(lrs, dict): + for name, value in lrs.items(): + tags[f'learning_rate/{name}'] = value[0] + else: + tags['learning_rate'] = lrs[0] + return tags + + def get_momentum_tags(self, runner): + tags = {} + momentums = runner.current_momentum() + if isinstance(momentums, dict): + for name, value in momentums.items(): + tags[f'momentum/{name}'] = value[0] + else: + tags['momentum'] = momentums[0] + return tags + + def get_loggable_tags(self, + runner, + allow_scalar=True, + allow_text=False, + add_mode=True, + tags_to_skip=('time', 'data_time')): + tags = {} + for var, val in runner.log_buffer.output.items(): + if var in tags_to_skip: + continue + if self.is_scalar(val) and not allow_scalar: + continue + if isinstance(val, str) and not allow_text: + continue + if add_mode: + var = f'{self.get_mode(runner)}/{var}' + tags[var] = val + tags.update(self.get_lr_tags(runner)) + tags.update(self.get_momentum_tags(runner)) + return tags + + def before_run(self, runner): + for hook in runner.hooks[::-1]: + if isinstance(hook, LoggerHook): + hook.reset_flag = True + break + + def before_epoch(self, runner): + runner.log_buffer.clear() # clear logs of last epoch + + def after_train_iter(self, runner): + if self.by_epoch and self.every_n_inner_iters(runner, self.interval): + runner.log_buffer.average(self.interval) + elif not self.by_epoch and self.every_n_iters(runner, self.interval): + runner.log_buffer.average(self.interval) + elif self.end_of_epoch(runner) and not self.ignore_last: + # not precise but more stable + runner.log_buffer.average(self.interval) + + if runner.log_buffer.ready: + self.log(runner) + if self.reset_flag: + runner.log_buffer.clear_output() + + def after_train_epoch(self, runner): + if runner.log_buffer.ready: + self.log(runner) + if self.reset_flag: + runner.log_buffer.clear_output() + + def after_val_epoch(self, runner): + runner.log_buffer.average() + self.log(runner) + if self.reset_flag: + runner.log_buffer.clear_output() diff --git a/annotator/mmpkg/mmcv/runner/hooks/logger/dvclive.py b/annotator/mmpkg/mmcv/runner/hooks/logger/dvclive.py new file mode 100644 index 0000000000000000000000000000000000000000..687cdc58c0336c92b1e4f9a410ba67ebaab2bc7a --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/logger/dvclive.py @@ -0,0 +1,58 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from ...dist_utils import master_only +from ..hook import HOOKS +from .base import LoggerHook + + +@HOOKS.register_module() +class DvcliveLoggerHook(LoggerHook): + """Class to log metrics with dvclive. + + It requires `dvclive`_ to be installed. + + Args: + path (str): Directory where dvclive will write TSV log files. + interval (int): Logging interval (every k iterations). + Default 10. + ignore_last (bool): Ignore the log of last iterations in each epoch + if less than `interval`. + Default: True. + reset_flag (bool): Whether to clear the output buffer after logging. + Default: True. + by_epoch (bool): Whether EpochBasedRunner is used. + Default: True. + + .. _dvclive: + https://dvc.org/doc/dvclive + """ + + def __init__(self, + path, + interval=10, + ignore_last=True, + reset_flag=True, + by_epoch=True): + + super(DvcliveLoggerHook, self).__init__(interval, ignore_last, + reset_flag, by_epoch) + self.path = path + self.import_dvclive() + + def import_dvclive(self): + try: + import dvclive + except ImportError: + raise ImportError( + 'Please run "pip install dvclive" to install dvclive') + self.dvclive = dvclive + + @master_only + def before_run(self, runner): + self.dvclive.init(self.path) + + @master_only + def log(self, runner): + tags = self.get_loggable_tags(runner) + if tags: + for k, v in tags.items(): + self.dvclive.log(k, v, step=self.get_iter(runner)) diff --git a/annotator/mmpkg/mmcv/runner/hooks/logger/mlflow.py b/annotator/mmpkg/mmcv/runner/hooks/logger/mlflow.py new file mode 100644 index 0000000000000000000000000000000000000000..f9a72592be47b534ce22573775fd5a7e8e86d72d --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/logger/mlflow.py @@ -0,0 +1,78 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from ...dist_utils import master_only +from ..hook import HOOKS +from .base import LoggerHook + + +@HOOKS.register_module() +class MlflowLoggerHook(LoggerHook): + + def __init__(self, + exp_name=None, + tags=None, + log_model=True, + interval=10, + ignore_last=True, + reset_flag=False, + by_epoch=True): + """Class to log metrics and (optionally) a trained model to MLflow. + + It requires `MLflow`_ to be installed. + + Args: + exp_name (str, optional): Name of the experiment to be used. + Default None. + If not None, set the active experiment. + If experiment does not exist, an experiment with provided name + will be created. + tags (dict of str: str, optional): Tags for the current run. + Default None. + If not None, set tags for the current run. + log_model (bool, optional): Whether to log an MLflow artifact. + Default True. + If True, log runner.model as an MLflow artifact + for the current run. + interval (int): Logging interval (every k iterations). + ignore_last (bool): Ignore the log of last iterations in each epoch + if less than `interval`. + reset_flag (bool): Whether to clear the output buffer after logging + by_epoch (bool): Whether EpochBasedRunner is used. + + .. _MLflow: + https://www.mlflow.org/docs/latest/index.html + """ + super(MlflowLoggerHook, self).__init__(interval, ignore_last, + reset_flag, by_epoch) + self.import_mlflow() + self.exp_name = exp_name + self.tags = tags + self.log_model = log_model + + def import_mlflow(self): + try: + import mlflow + import mlflow.pytorch as mlflow_pytorch + except ImportError: + raise ImportError( + 'Please run "pip install mlflow" to install mlflow') + self.mlflow = mlflow + self.mlflow_pytorch = mlflow_pytorch + + @master_only + def before_run(self, runner): + super(MlflowLoggerHook, self).before_run(runner) + if self.exp_name is not None: + self.mlflow.set_experiment(self.exp_name) + if self.tags is not None: + self.mlflow.set_tags(self.tags) + + @master_only + def log(self, runner): + tags = self.get_loggable_tags(runner) + if tags: + self.mlflow.log_metrics(tags, step=self.get_iter(runner)) + + @master_only + def after_run(self, runner): + if self.log_model: + self.mlflow_pytorch.log_model(runner.model, 'models') diff --git a/annotator/mmpkg/mmcv/runner/hooks/logger/neptune.py b/annotator/mmpkg/mmcv/runner/hooks/logger/neptune.py new file mode 100644 index 0000000000000000000000000000000000000000..7a38772b0c93a8608f32c6357b8616e77c139dc9 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/logger/neptune.py @@ -0,0 +1,82 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from ...dist_utils import master_only +from ..hook import HOOKS +from .base import LoggerHook + + +@HOOKS.register_module() +class NeptuneLoggerHook(LoggerHook): + """Class to log metrics to NeptuneAI. + + It requires `neptune-client` to be installed. + + Args: + init_kwargs (dict): a dict contains the initialization keys as below: + - project (str): Name of a project in a form of + namespace/project_name. If None, the value of + NEPTUNE_PROJECT environment variable will be taken. + - api_token (str): User’s API token. + If None, the value of NEPTUNE_API_TOKEN environment + variable will be taken. Note: It is strongly recommended + to use NEPTUNE_API_TOKEN environment variable rather than + placing your API token in plain text in your source code. + - name (str, optional, default is 'Untitled'): Editable name of + the run. Name is displayed in the run's Details and in + Runs table as a column. + Check https://docs.neptune.ai/api-reference/neptune#init for + more init arguments. + interval (int): Logging interval (every k iterations). + ignore_last (bool): Ignore the log of last iterations in each epoch + if less than `interval`. + reset_flag (bool): Whether to clear the output buffer after logging + by_epoch (bool): Whether EpochBasedRunner is used. + + .. _NeptuneAI: + https://docs.neptune.ai/you-should-know/logging-metadata + """ + + def __init__(self, + init_kwargs=None, + interval=10, + ignore_last=True, + reset_flag=True, + with_step=True, + by_epoch=True): + + super(NeptuneLoggerHook, self).__init__(interval, ignore_last, + reset_flag, by_epoch) + self.import_neptune() + self.init_kwargs = init_kwargs + self.with_step = with_step + + def import_neptune(self): + try: + import neptune.new as neptune + except ImportError: + raise ImportError( + 'Please run "pip install neptune-client" to install neptune') + self.neptune = neptune + self.run = None + + @master_only + def before_run(self, runner): + if self.init_kwargs: + self.run = self.neptune.init(**self.init_kwargs) + else: + self.run = self.neptune.init() + + @master_only + def log(self, runner): + tags = self.get_loggable_tags(runner) + if tags: + for tag_name, tag_value in tags.items(): + if self.with_step: + self.run[tag_name].log( + tag_value, step=self.get_iter(runner)) + else: + tags['global_step'] = self.get_iter(runner) + self.run[tag_name].log(tags) + + @master_only + def after_run(self, runner): + self.run.stop() diff --git a/annotator/mmpkg/mmcv/runner/hooks/logger/pavi.py b/annotator/mmpkg/mmcv/runner/hooks/logger/pavi.py new file mode 100644 index 0000000000000000000000000000000000000000..5d1c4286920361e6b80f135b8d60b250f98f507a --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/logger/pavi.py @@ -0,0 +1,117 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import json +import os +import os.path as osp + +import torch +import yaml + +import annotator.mmpkg.mmcv as mmcv +from ....parallel.utils import is_module_wrapper +from ...dist_utils import master_only +from ..hook import HOOKS +from .base import LoggerHook + + +@HOOKS.register_module() +class PaviLoggerHook(LoggerHook): + + def __init__(self, + init_kwargs=None, + add_graph=False, + add_last_ckpt=False, + interval=10, + ignore_last=True, + reset_flag=False, + by_epoch=True, + img_key='img_info'): + super(PaviLoggerHook, self).__init__(interval, ignore_last, reset_flag, + by_epoch) + self.init_kwargs = init_kwargs + self.add_graph = add_graph + self.add_last_ckpt = add_last_ckpt + self.img_key = img_key + + @master_only + def before_run(self, runner): + super(PaviLoggerHook, self).before_run(runner) + try: + from pavi import SummaryWriter + except ImportError: + raise ImportError('Please run "pip install pavi" to install pavi.') + + self.run_name = runner.work_dir.split('/')[-1] + + if not self.init_kwargs: + self.init_kwargs = dict() + self.init_kwargs['name'] = self.run_name + self.init_kwargs['model'] = runner._model_name + if runner.meta is not None: + if 'config_dict' in runner.meta: + config_dict = runner.meta['config_dict'] + assert isinstance( + config_dict, + dict), ('meta["config_dict"] has to be of a dict, ' + f'but got {type(config_dict)}') + elif 'config_file' in runner.meta: + config_file = runner.meta['config_file'] + config_dict = dict(mmcv.Config.fromfile(config_file)) + else: + config_dict = None + if config_dict is not None: + # 'max_.*iter' is parsed in pavi sdk as the maximum iterations + # to properly set up the progress bar. + config_dict = config_dict.copy() + config_dict.setdefault('max_iter', runner.max_iters) + # non-serializable values are first converted in + # mmcv.dump to json + config_dict = json.loads( + mmcv.dump(config_dict, file_format='json')) + session_text = yaml.dump(config_dict) + self.init_kwargs['session_text'] = session_text + self.writer = SummaryWriter(**self.init_kwargs) + + def get_step(self, runner): + """Get the total training step/epoch.""" + if self.get_mode(runner) == 'val' and self.by_epoch: + return self.get_epoch(runner) + else: + return self.get_iter(runner) + + @master_only + def log(self, runner): + tags = self.get_loggable_tags(runner, add_mode=False) + if tags: + self.writer.add_scalars( + self.get_mode(runner), tags, self.get_step(runner)) + + @master_only + def after_run(self, runner): + if self.add_last_ckpt: + ckpt_path = osp.join(runner.work_dir, 'latest.pth') + if osp.islink(ckpt_path): + ckpt_path = osp.join(runner.work_dir, os.readlink(ckpt_path)) + + if osp.isfile(ckpt_path): + # runner.epoch += 1 has been done before `after_run`. + iteration = runner.epoch if self.by_epoch else runner.iter + return self.writer.add_snapshot_file( + tag=self.run_name, + snapshot_file_path=ckpt_path, + iteration=iteration) + + # flush the buffer and send a task ending signal to Pavi + self.writer.close() + + @master_only + def before_epoch(self, runner): + if runner.epoch == 0 and self.add_graph: + if is_module_wrapper(runner.model): + _model = runner.model.module + else: + _model = runner.model + device = next(_model.parameters()).device + data = next(iter(runner.data_loader)) + image = data[self.img_key][0:1].to(device) + with torch.no_grad(): + self.writer.add_graph(_model, image) diff --git a/annotator/mmpkg/mmcv/runner/hooks/logger/tensorboard.py b/annotator/mmpkg/mmcv/runner/hooks/logger/tensorboard.py new file mode 100644 index 0000000000000000000000000000000000000000..7c480a560e90f5b06abb4afaf9597aaf7c1eaa82 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/logger/tensorboard.py @@ -0,0 +1,57 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os.path as osp + +from annotator.mmpkg.mmcv.utils import TORCH_VERSION, digit_version +from ...dist_utils import master_only +from ..hook import HOOKS +from .base import LoggerHook + + +@HOOKS.register_module() +class TensorboardLoggerHook(LoggerHook): + + def __init__(self, + log_dir=None, + interval=10, + ignore_last=True, + reset_flag=False, + by_epoch=True): + super(TensorboardLoggerHook, self).__init__(interval, ignore_last, + reset_flag, by_epoch) + self.log_dir = log_dir + + @master_only + def before_run(self, runner): + super(TensorboardLoggerHook, self).before_run(runner) + if (TORCH_VERSION == 'parrots' + or digit_version(TORCH_VERSION) < digit_version('1.1')): + try: + from tensorboardX import SummaryWriter + except ImportError: + raise ImportError('Please install tensorboardX to use ' + 'TensorboardLoggerHook.') + else: + try: + from torch.utils.tensorboard import SummaryWriter + except ImportError: + raise ImportError( + 'Please run "pip install future tensorboard" to install ' + 'the dependencies to use torch.utils.tensorboard ' + '(applicable to PyTorch 1.1 or higher)') + + if self.log_dir is None: + self.log_dir = osp.join(runner.work_dir, 'tf_logs') + self.writer = SummaryWriter(self.log_dir) + + @master_only + def log(self, runner): + tags = self.get_loggable_tags(runner, allow_text=True) + for tag, val in tags.items(): + if isinstance(val, str): + self.writer.add_text(tag, val, self.get_iter(runner)) + else: + self.writer.add_scalar(tag, val, self.get_iter(runner)) + + @master_only + def after_run(self, runner): + self.writer.close() diff --git a/annotator/mmpkg/mmcv/runner/hooks/logger/text.py b/annotator/mmpkg/mmcv/runner/hooks/logger/text.py new file mode 100644 index 0000000000000000000000000000000000000000..0b30577469d5f70e544e1ce73816326e38dadb20 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/logger/text.py @@ -0,0 +1,256 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import datetime +import os +import os.path as osp +from collections import OrderedDict + +import torch +import torch.distributed as dist + +import annotator.mmpkg.mmcv as mmcv +from annotator.mmpkg.mmcv.fileio.file_client import FileClient +from annotator.mmpkg.mmcv.utils import is_tuple_of, scandir +from ..hook import HOOKS +from .base import LoggerHook + + +@HOOKS.register_module() +class TextLoggerHook(LoggerHook): + """Logger hook in text. + + In this logger hook, the information will be printed on terminal and + saved in json file. + + Args: + by_epoch (bool, optional): Whether EpochBasedRunner is used. + Default: True. + interval (int, optional): Logging interval (every k iterations). + Default: 10. + ignore_last (bool, optional): Ignore the log of last iterations in each + epoch if less than :attr:`interval`. Default: True. + reset_flag (bool, optional): Whether to clear the output buffer after + logging. Default: False. + interval_exp_name (int, optional): Logging interval for experiment + name. This feature is to help users conveniently get the experiment + information from screen or log file. Default: 1000. + out_dir (str, optional): Logs are saved in ``runner.work_dir`` default. + If ``out_dir`` is specified, logs will be copied to a new directory + which is the concatenation of ``out_dir`` and the last level + directory of ``runner.work_dir``. Default: None. + `New in version 1.3.16.` + out_suffix (str or tuple[str], optional): Those filenames ending with + ``out_suffix`` will be copied to ``out_dir``. + Default: ('.log.json', '.log', '.py'). + `New in version 1.3.16.` + keep_local (bool, optional): Whether to keep local log when + :attr:`out_dir` is specified. If False, the local log will be + removed. Default: True. + `New in version 1.3.16.` + file_client_args (dict, optional): Arguments to instantiate a + FileClient. See :class:`mmcv.fileio.FileClient` for details. + Default: None. + `New in version 1.3.16.` + """ + + def __init__(self, + by_epoch=True, + interval=10, + ignore_last=True, + reset_flag=False, + interval_exp_name=1000, + out_dir=None, + out_suffix=('.log.json', '.log', '.py'), + keep_local=True, + file_client_args=None): + super(TextLoggerHook, self).__init__(interval, ignore_last, reset_flag, + by_epoch) + self.by_epoch = by_epoch + self.time_sec_tot = 0 + self.interval_exp_name = interval_exp_name + + if out_dir is None and file_client_args is not None: + raise ValueError( + 'file_client_args should be "None" when `out_dir` is not' + 'specified.') + self.out_dir = out_dir + + if not (out_dir is None or isinstance(out_dir, str) + or is_tuple_of(out_dir, str)): + raise TypeError('out_dir should be "None" or string or tuple of ' + 'string, but got {out_dir}') + self.out_suffix = out_suffix + + self.keep_local = keep_local + self.file_client_args = file_client_args + if self.out_dir is not None: + self.file_client = FileClient.infer_client(file_client_args, + self.out_dir) + + def before_run(self, runner): + super(TextLoggerHook, self).before_run(runner) + + if self.out_dir is not None: + self.file_client = FileClient.infer_client(self.file_client_args, + self.out_dir) + # The final `self.out_dir` is the concatenation of `self.out_dir` + # and the last level directory of `runner.work_dir` + basename = osp.basename(runner.work_dir.rstrip(osp.sep)) + self.out_dir = self.file_client.join_path(self.out_dir, basename) + runner.logger.info( + (f'Text logs will be saved to {self.out_dir} by ' + f'{self.file_client.name} after the training process.')) + + self.start_iter = runner.iter + self.json_log_path = osp.join(runner.work_dir, + f'{runner.timestamp}.log.json') + if runner.meta is not None: + self._dump_log(runner.meta, runner) + + def _get_max_memory(self, runner): + device = getattr(runner.model, 'output_device', None) + mem = torch.cuda.max_memory_allocated(device=device) + mem_mb = torch.tensor([mem / (1024 * 1024)], + dtype=torch.int, + device=device) + if runner.world_size > 1: + dist.reduce(mem_mb, 0, op=dist.ReduceOp.MAX) + return mem_mb.item() + + def _log_info(self, log_dict, runner): + # print exp name for users to distinguish experiments + # at every ``interval_exp_name`` iterations and the end of each epoch + if runner.meta is not None and 'exp_name' in runner.meta: + if (self.every_n_iters(runner, self.interval_exp_name)) or ( + self.by_epoch and self.end_of_epoch(runner)): + exp_info = f'Exp name: {runner.meta["exp_name"]}' + runner.logger.info(exp_info) + + if log_dict['mode'] == 'train': + if isinstance(log_dict['lr'], dict): + lr_str = [] + for k, val in log_dict['lr'].items(): + lr_str.append(f'lr_{k}: {val:.3e}') + lr_str = ' '.join(lr_str) + else: + lr_str = f'lr: {log_dict["lr"]:.3e}' + + # by epoch: Epoch [4][100/1000] + # by iter: Iter [100/100000] + if self.by_epoch: + log_str = f'Epoch [{log_dict["epoch"]}]' \ + f'[{log_dict["iter"]}/{len(runner.data_loader)}]\t' + else: + log_str = f'Iter [{log_dict["iter"]}/{runner.max_iters}]\t' + log_str += f'{lr_str}, ' + + if 'time' in log_dict.keys(): + self.time_sec_tot += (log_dict['time'] * self.interval) + time_sec_avg = self.time_sec_tot / ( + runner.iter - self.start_iter + 1) + eta_sec = time_sec_avg * (runner.max_iters - runner.iter - 1) + eta_str = str(datetime.timedelta(seconds=int(eta_sec))) + log_str += f'eta: {eta_str}, ' + log_str += f'time: {log_dict["time"]:.3f}, ' \ + f'data_time: {log_dict["data_time"]:.3f}, ' + # statistic memory + if torch.cuda.is_available(): + log_str += f'memory: {log_dict["memory"]}, ' + else: + # val/test time + # here 1000 is the length of the val dataloader + # by epoch: Epoch[val] [4][1000] + # by iter: Iter[val] [1000] + if self.by_epoch: + log_str = f'Epoch({log_dict["mode"]}) ' \ + f'[{log_dict["epoch"]}][{log_dict["iter"]}]\t' + else: + log_str = f'Iter({log_dict["mode"]}) [{log_dict["iter"]}]\t' + + log_items = [] + for name, val in log_dict.items(): + # TODO: resolve this hack + # these items have been in log_str + if name in [ + 'mode', 'Epoch', 'iter', 'lr', 'time', 'data_time', + 'memory', 'epoch' + ]: + continue + if isinstance(val, float): + val = f'{val:.4f}' + log_items.append(f'{name}: {val}') + log_str += ', '.join(log_items) + + runner.logger.info(log_str) + + def _dump_log(self, log_dict, runner): + # dump log in json format + json_log = OrderedDict() + for k, v in log_dict.items(): + json_log[k] = self._round_float(v) + # only append log at last line + if runner.rank == 0: + with open(self.json_log_path, 'a+') as f: + mmcv.dump(json_log, f, file_format='json') + f.write('\n') + + def _round_float(self, items): + if isinstance(items, list): + return [self._round_float(item) for item in items] + elif isinstance(items, float): + return round(items, 5) + else: + return items + + def log(self, runner): + if 'eval_iter_num' in runner.log_buffer.output: + # this doesn't modify runner.iter and is regardless of by_epoch + cur_iter = runner.log_buffer.output.pop('eval_iter_num') + else: + cur_iter = self.get_iter(runner, inner_iter=True) + + log_dict = OrderedDict( + mode=self.get_mode(runner), + epoch=self.get_epoch(runner), + iter=cur_iter) + + # only record lr of the first param group + cur_lr = runner.current_lr() + if isinstance(cur_lr, list): + log_dict['lr'] = cur_lr[0] + else: + assert isinstance(cur_lr, dict) + log_dict['lr'] = {} + for k, lr_ in cur_lr.items(): + assert isinstance(lr_, list) + log_dict['lr'].update({k: lr_[0]}) + + if 'time' in runner.log_buffer.output: + # statistic memory + if torch.cuda.is_available(): + log_dict['memory'] = self._get_max_memory(runner) + + log_dict = dict(log_dict, **runner.log_buffer.output) + + self._log_info(log_dict, runner) + self._dump_log(log_dict, runner) + return log_dict + + def after_run(self, runner): + # copy or upload logs to self.out_dir + if self.out_dir is not None: + for filename in scandir(runner.work_dir, self.out_suffix, True): + local_filepath = osp.join(runner.work_dir, filename) + out_filepath = self.file_client.join_path( + self.out_dir, filename) + with open(local_filepath, 'r') as f: + self.file_client.put_text(f.read(), out_filepath) + + runner.logger.info( + (f'The file {local_filepath} has been uploaded to ' + f'{out_filepath}.')) + + if not self.keep_local: + os.remove(local_filepath) + runner.logger.info( + (f'{local_filepath} was removed due to the ' + '`self.keep_local=False`')) diff --git a/annotator/mmpkg/mmcv/runner/hooks/logger/wandb.py b/annotator/mmpkg/mmcv/runner/hooks/logger/wandb.py new file mode 100644 index 0000000000000000000000000000000000000000..9f6808462eb79ab2b04806a5d9f0d3dd079b5ea9 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/logger/wandb.py @@ -0,0 +1,56 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from ...dist_utils import master_only +from ..hook import HOOKS +from .base import LoggerHook + + +@HOOKS.register_module() +class WandbLoggerHook(LoggerHook): + + def __init__(self, + init_kwargs=None, + interval=10, + ignore_last=True, + reset_flag=False, + commit=True, + by_epoch=True, + with_step=True): + super(WandbLoggerHook, self).__init__(interval, ignore_last, + reset_flag, by_epoch) + self.import_wandb() + self.init_kwargs = init_kwargs + self.commit = commit + self.with_step = with_step + + def import_wandb(self): + try: + import wandb + except ImportError: + raise ImportError( + 'Please run "pip install wandb" to install wandb') + self.wandb = wandb + + @master_only + def before_run(self, runner): + super(WandbLoggerHook, self).before_run(runner) + if self.wandb is None: + self.import_wandb() + if self.init_kwargs: + self.wandb.init(**self.init_kwargs) + else: + self.wandb.init() + + @master_only + def log(self, runner): + tags = self.get_loggable_tags(runner) + if tags: + if self.with_step: + self.wandb.log( + tags, step=self.get_iter(runner), commit=self.commit) + else: + tags['global_step'] = self.get_iter(runner) + self.wandb.log(tags, commit=self.commit) + + @master_only + def after_run(self, runner): + self.wandb.join() diff --git a/annotator/mmpkg/mmcv/runner/hooks/lr_updater.py b/annotator/mmpkg/mmcv/runner/hooks/lr_updater.py new file mode 100644 index 0000000000000000000000000000000000000000..b9851d2ca3c4e60b95ad734c19a2484b9ca7c708 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/lr_updater.py @@ -0,0 +1,670 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numbers +from math import cos, pi + +import annotator.mmpkg.mmcv as mmcv +from .hook import HOOKS, Hook + + +class LrUpdaterHook(Hook): + """LR Scheduler in MMCV. + + Args: + by_epoch (bool): LR changes epoch by epoch + warmup (string): Type of warmup used. It can be None(use no warmup), + 'constant', 'linear' or 'exp' + warmup_iters (int): The number of iterations or epochs that warmup + lasts + warmup_ratio (float): LR used at the beginning of warmup equals to + warmup_ratio * initial_lr + warmup_by_epoch (bool): When warmup_by_epoch == True, warmup_iters + means the number of epochs that warmup lasts, otherwise means the + number of iteration that warmup lasts + """ + + def __init__(self, + by_epoch=True, + warmup=None, + warmup_iters=0, + warmup_ratio=0.1, + warmup_by_epoch=False): + # validate the "warmup" argument + if warmup is not None: + if warmup not in ['constant', 'linear', 'exp']: + raise ValueError( + f'"{warmup}" is not a supported type for warming up, valid' + ' types are "constant" and "linear"') + if warmup is not None: + assert warmup_iters > 0, \ + '"warmup_iters" must be a positive integer' + assert 0 < warmup_ratio <= 1.0, \ + '"warmup_ratio" must be in range (0,1]' + + self.by_epoch = by_epoch + self.warmup = warmup + self.warmup_iters = warmup_iters + self.warmup_ratio = warmup_ratio + self.warmup_by_epoch = warmup_by_epoch + + if self.warmup_by_epoch: + self.warmup_epochs = self.warmup_iters + self.warmup_iters = None + else: + self.warmup_epochs = None + + self.base_lr = [] # initial lr for all param groups + self.regular_lr = [] # expected lr if no warming up is performed + + def _set_lr(self, runner, lr_groups): + if isinstance(runner.optimizer, dict): + for k, optim in runner.optimizer.items(): + for param_group, lr in zip(optim.param_groups, lr_groups[k]): + param_group['lr'] = lr + else: + for param_group, lr in zip(runner.optimizer.param_groups, + lr_groups): + param_group['lr'] = lr + + def get_lr(self, runner, base_lr): + raise NotImplementedError + + def get_regular_lr(self, runner): + if isinstance(runner.optimizer, dict): + lr_groups = {} + for k in runner.optimizer.keys(): + _lr_group = [ + self.get_lr(runner, _base_lr) + for _base_lr in self.base_lr[k] + ] + lr_groups.update({k: _lr_group}) + + return lr_groups + else: + return [self.get_lr(runner, _base_lr) for _base_lr in self.base_lr] + + def get_warmup_lr(self, cur_iters): + + def _get_warmup_lr(cur_iters, regular_lr): + if self.warmup == 'constant': + warmup_lr = [_lr * self.warmup_ratio for _lr in regular_lr] + elif self.warmup == 'linear': + k = (1 - cur_iters / self.warmup_iters) * (1 - + self.warmup_ratio) + warmup_lr = [_lr * (1 - k) for _lr in regular_lr] + elif self.warmup == 'exp': + k = self.warmup_ratio**(1 - cur_iters / self.warmup_iters) + warmup_lr = [_lr * k for _lr in regular_lr] + return warmup_lr + + if isinstance(self.regular_lr, dict): + lr_groups = {} + for key, regular_lr in self.regular_lr.items(): + lr_groups[key] = _get_warmup_lr(cur_iters, regular_lr) + return lr_groups + else: + return _get_warmup_lr(cur_iters, self.regular_lr) + + def before_run(self, runner): + # NOTE: when resuming from a checkpoint, if 'initial_lr' is not saved, + # it will be set according to the optimizer params + if isinstance(runner.optimizer, dict): + self.base_lr = {} + for k, optim in runner.optimizer.items(): + for group in optim.param_groups: + group.setdefault('initial_lr', group['lr']) + _base_lr = [ + group['initial_lr'] for group in optim.param_groups + ] + self.base_lr.update({k: _base_lr}) + else: + for group in runner.optimizer.param_groups: + group.setdefault('initial_lr', group['lr']) + self.base_lr = [ + group['initial_lr'] for group in runner.optimizer.param_groups + ] + + def before_train_epoch(self, runner): + if self.warmup_iters is None: + epoch_len = len(runner.data_loader) + self.warmup_iters = self.warmup_epochs * epoch_len + + if not self.by_epoch: + return + + self.regular_lr = self.get_regular_lr(runner) + self._set_lr(runner, self.regular_lr) + + def before_train_iter(self, runner): + cur_iter = runner.iter + if not self.by_epoch: + self.regular_lr = self.get_regular_lr(runner) + if self.warmup is None or cur_iter >= self.warmup_iters: + self._set_lr(runner, self.regular_lr) + else: + warmup_lr = self.get_warmup_lr(cur_iter) + self._set_lr(runner, warmup_lr) + elif self.by_epoch: + if self.warmup is None or cur_iter > self.warmup_iters: + return + elif cur_iter == self.warmup_iters: + self._set_lr(runner, self.regular_lr) + else: + warmup_lr = self.get_warmup_lr(cur_iter) + self._set_lr(runner, warmup_lr) + + +@HOOKS.register_module() +class FixedLrUpdaterHook(LrUpdaterHook): + + def __init__(self, **kwargs): + super(FixedLrUpdaterHook, self).__init__(**kwargs) + + def get_lr(self, runner, base_lr): + return base_lr + + +@HOOKS.register_module() +class StepLrUpdaterHook(LrUpdaterHook): + """Step LR scheduler with min_lr clipping. + + Args: + step (int | list[int]): Step to decay the LR. If an int value is given, + regard it as the decay interval. If a list is given, decay LR at + these steps. + gamma (float, optional): Decay LR ratio. Default: 0.1. + min_lr (float, optional): Minimum LR value to keep. If LR after decay + is lower than `min_lr`, it will be clipped to this value. If None + is given, we don't perform lr clipping. Default: None. + """ + + def __init__(self, step, gamma=0.1, min_lr=None, **kwargs): + if isinstance(step, list): + assert mmcv.is_list_of(step, int) + assert all([s > 0 for s in step]) + elif isinstance(step, int): + assert step > 0 + else: + raise TypeError('"step" must be a list or integer') + self.step = step + self.gamma = gamma + self.min_lr = min_lr + super(StepLrUpdaterHook, self).__init__(**kwargs) + + def get_lr(self, runner, base_lr): + progress = runner.epoch if self.by_epoch else runner.iter + + # calculate exponential term + if isinstance(self.step, int): + exp = progress // self.step + else: + exp = len(self.step) + for i, s in enumerate(self.step): + if progress < s: + exp = i + break + + lr = base_lr * (self.gamma**exp) + if self.min_lr is not None: + # clip to a minimum value + lr = max(lr, self.min_lr) + return lr + + +@HOOKS.register_module() +class ExpLrUpdaterHook(LrUpdaterHook): + + def __init__(self, gamma, **kwargs): + self.gamma = gamma + super(ExpLrUpdaterHook, self).__init__(**kwargs) + + def get_lr(self, runner, base_lr): + progress = runner.epoch if self.by_epoch else runner.iter + return base_lr * self.gamma**progress + + +@HOOKS.register_module() +class PolyLrUpdaterHook(LrUpdaterHook): + + def __init__(self, power=1., min_lr=0., **kwargs): + self.power = power + self.min_lr = min_lr + super(PolyLrUpdaterHook, self).__init__(**kwargs) + + def get_lr(self, runner, base_lr): + if self.by_epoch: + progress = runner.epoch + max_progress = runner.max_epochs + else: + progress = runner.iter + max_progress = runner.max_iters + coeff = (1 - progress / max_progress)**self.power + return (base_lr - self.min_lr) * coeff + self.min_lr + + +@HOOKS.register_module() +class InvLrUpdaterHook(LrUpdaterHook): + + def __init__(self, gamma, power=1., **kwargs): + self.gamma = gamma + self.power = power + super(InvLrUpdaterHook, self).__init__(**kwargs) + + def get_lr(self, runner, base_lr): + progress = runner.epoch if self.by_epoch else runner.iter + return base_lr * (1 + self.gamma * progress)**(-self.power) + + +@HOOKS.register_module() +class CosineAnnealingLrUpdaterHook(LrUpdaterHook): + + def __init__(self, min_lr=None, min_lr_ratio=None, **kwargs): + assert (min_lr is None) ^ (min_lr_ratio is None) + self.min_lr = min_lr + self.min_lr_ratio = min_lr_ratio + super(CosineAnnealingLrUpdaterHook, self).__init__(**kwargs) + + def get_lr(self, runner, base_lr): + if self.by_epoch: + progress = runner.epoch + max_progress = runner.max_epochs + else: + progress = runner.iter + max_progress = runner.max_iters + + if self.min_lr_ratio is not None: + target_lr = base_lr * self.min_lr_ratio + else: + target_lr = self.min_lr + return annealing_cos(base_lr, target_lr, progress / max_progress) + + +@HOOKS.register_module() +class FlatCosineAnnealingLrUpdaterHook(LrUpdaterHook): + """Flat + Cosine lr schedule. + + Modified from https://github.com/fastai/fastai/blob/master/fastai/callback/schedule.py#L128 # noqa: E501 + + Args: + start_percent (float): When to start annealing the learning rate + after the percentage of the total training steps. + The value should be in range [0, 1). + Default: 0.75 + min_lr (float, optional): The minimum lr. Default: None. + min_lr_ratio (float, optional): The ratio of minimum lr to the base lr. + Either `min_lr` or `min_lr_ratio` should be specified. + Default: None. + """ + + def __init__(self, + start_percent=0.75, + min_lr=None, + min_lr_ratio=None, + **kwargs): + assert (min_lr is None) ^ (min_lr_ratio is None) + if start_percent < 0 or start_percent > 1 or not isinstance( + start_percent, float): + raise ValueError( + 'expected float between 0 and 1 start_percent, but ' + f'got {start_percent}') + self.start_percent = start_percent + self.min_lr = min_lr + self.min_lr_ratio = min_lr_ratio + super(FlatCosineAnnealingLrUpdaterHook, self).__init__(**kwargs) + + def get_lr(self, runner, base_lr): + if self.by_epoch: + start = round(runner.max_epochs * self.start_percent) + progress = runner.epoch - start + max_progress = runner.max_epochs - start + else: + start = round(runner.max_iters * self.start_percent) + progress = runner.iter - start + max_progress = runner.max_iters - start + + if self.min_lr_ratio is not None: + target_lr = base_lr * self.min_lr_ratio + else: + target_lr = self.min_lr + + if progress < 0: + return base_lr + else: + return annealing_cos(base_lr, target_lr, progress / max_progress) + + +@HOOKS.register_module() +class CosineRestartLrUpdaterHook(LrUpdaterHook): + """Cosine annealing with restarts learning rate scheme. + + Args: + periods (list[int]): Periods for each cosine anneling cycle. + restart_weights (list[float], optional): Restart weights at each + restart iteration. Default: [1]. + min_lr (float, optional): The minimum lr. Default: None. + min_lr_ratio (float, optional): The ratio of minimum lr to the base lr. + Either `min_lr` or `min_lr_ratio` should be specified. + Default: None. + """ + + def __init__(self, + periods, + restart_weights=[1], + min_lr=None, + min_lr_ratio=None, + **kwargs): + assert (min_lr is None) ^ (min_lr_ratio is None) + self.periods = periods + self.min_lr = min_lr + self.min_lr_ratio = min_lr_ratio + self.restart_weights = restart_weights + assert (len(self.periods) == len(self.restart_weights) + ), 'periods and restart_weights should have the same length.' + super(CosineRestartLrUpdaterHook, self).__init__(**kwargs) + + self.cumulative_periods = [ + sum(self.periods[0:i + 1]) for i in range(0, len(self.periods)) + ] + + def get_lr(self, runner, base_lr): + if self.by_epoch: + progress = runner.epoch + else: + progress = runner.iter + + if self.min_lr_ratio is not None: + target_lr = base_lr * self.min_lr_ratio + else: + target_lr = self.min_lr + + idx = get_position_from_periods(progress, self.cumulative_periods) + current_weight = self.restart_weights[idx] + nearest_restart = 0 if idx == 0 else self.cumulative_periods[idx - 1] + current_periods = self.periods[idx] + + alpha = min((progress - nearest_restart) / current_periods, 1) + return annealing_cos(base_lr, target_lr, alpha, current_weight) + + +def get_position_from_periods(iteration, cumulative_periods): + """Get the position from a period list. + + It will return the index of the right-closest number in the period list. + For example, the cumulative_periods = [100, 200, 300, 400], + if iteration == 50, return 0; + if iteration == 210, return 2; + if iteration == 300, return 3. + + Args: + iteration (int): Current iteration. + cumulative_periods (list[int]): Cumulative period list. + + Returns: + int: The position of the right-closest number in the period list. + """ + for i, period in enumerate(cumulative_periods): + if iteration < period: + return i + raise ValueError(f'Current iteration {iteration} exceeds ' + f'cumulative_periods {cumulative_periods}') + + +@HOOKS.register_module() +class CyclicLrUpdaterHook(LrUpdaterHook): + """Cyclic LR Scheduler. + + Implement the cyclical learning rate policy (CLR) described in + https://arxiv.org/pdf/1506.01186.pdf + + Different from the original paper, we use cosine annealing rather than + triangular policy inside a cycle. This improves the performance in the + 3D detection area. + + Args: + by_epoch (bool): Whether to update LR by epoch. + target_ratio (tuple[float]): Relative ratio of the highest LR and the + lowest LR to the initial LR. + cyclic_times (int): Number of cycles during training + step_ratio_up (float): The ratio of the increasing process of LR in + the total cycle. + anneal_strategy (str): {'cos', 'linear'} + Specifies the annealing strategy: 'cos' for cosine annealing, + 'linear' for linear annealing. Default: 'cos'. + """ + + def __init__(self, + by_epoch=False, + target_ratio=(10, 1e-4), + cyclic_times=1, + step_ratio_up=0.4, + anneal_strategy='cos', + **kwargs): + if isinstance(target_ratio, float): + target_ratio = (target_ratio, target_ratio / 1e5) + elif isinstance(target_ratio, tuple): + target_ratio = (target_ratio[0], target_ratio[0] / 1e5) \ + if len(target_ratio) == 1 else target_ratio + else: + raise ValueError('target_ratio should be either float ' + f'or tuple, got {type(target_ratio)}') + + assert len(target_ratio) == 2, \ + '"target_ratio" must be list or tuple of two floats' + assert 0 <= step_ratio_up < 1.0, \ + '"step_ratio_up" must be in range [0,1)' + + self.target_ratio = target_ratio + self.cyclic_times = cyclic_times + self.step_ratio_up = step_ratio_up + self.lr_phases = [] # init lr_phases + # validate anneal_strategy + if anneal_strategy not in ['cos', 'linear']: + raise ValueError('anneal_strategy must be one of "cos" or ' + f'"linear", instead got {anneal_strategy}') + elif anneal_strategy == 'cos': + self.anneal_func = annealing_cos + elif anneal_strategy == 'linear': + self.anneal_func = annealing_linear + + assert not by_epoch, \ + 'currently only support "by_epoch" = False' + super(CyclicLrUpdaterHook, self).__init__(by_epoch, **kwargs) + + def before_run(self, runner): + super(CyclicLrUpdaterHook, self).before_run(runner) + # initiate lr_phases + # total lr_phases are separated as up and down + max_iter_per_phase = runner.max_iters // self.cyclic_times + iter_up_phase = int(self.step_ratio_up * max_iter_per_phase) + self.lr_phases.append( + [0, iter_up_phase, max_iter_per_phase, 1, self.target_ratio[0]]) + self.lr_phases.append([ + iter_up_phase, max_iter_per_phase, max_iter_per_phase, + self.target_ratio[0], self.target_ratio[1] + ]) + + def get_lr(self, runner, base_lr): + curr_iter = runner.iter + for (start_iter, end_iter, max_iter_per_phase, start_ratio, + end_ratio) in self.lr_phases: + curr_iter %= max_iter_per_phase + if start_iter <= curr_iter < end_iter: + progress = curr_iter - start_iter + return self.anneal_func(base_lr * start_ratio, + base_lr * end_ratio, + progress / (end_iter - start_iter)) + + +@HOOKS.register_module() +class OneCycleLrUpdaterHook(LrUpdaterHook): + """One Cycle LR Scheduler. + + The 1cycle learning rate policy changes the learning rate after every + batch. The one cycle learning rate policy is described in + https://arxiv.org/pdf/1708.07120.pdf + + Args: + max_lr (float or list): Upper learning rate boundaries in the cycle + for each parameter group. + total_steps (int, optional): The total number of steps in the cycle. + Note that if a value is not provided here, it will be the max_iter + of runner. Default: None. + pct_start (float): The percentage of the cycle (in number of steps) + spent increasing the learning rate. + Default: 0.3 + anneal_strategy (str): {'cos', 'linear'} + Specifies the annealing strategy: 'cos' for cosine annealing, + 'linear' for linear annealing. + Default: 'cos' + div_factor (float): Determines the initial learning rate via + initial_lr = max_lr/div_factor + Default: 25 + final_div_factor (float): Determines the minimum learning rate via + min_lr = initial_lr/final_div_factor + Default: 1e4 + three_phase (bool): If three_phase is True, use a third phase of the + schedule to annihilate the learning rate according to + final_div_factor instead of modifying the second phase (the first + two phases will be symmetrical about the step indicated by + pct_start). + Default: False + """ + + def __init__(self, + max_lr, + total_steps=None, + pct_start=0.3, + anneal_strategy='cos', + div_factor=25, + final_div_factor=1e4, + three_phase=False, + **kwargs): + # validate by_epoch, currently only support by_epoch = False + if 'by_epoch' not in kwargs: + kwargs['by_epoch'] = False + else: + assert not kwargs['by_epoch'], \ + 'currently only support "by_epoch" = False' + if not isinstance(max_lr, (numbers.Number, list, dict)): + raise ValueError('the type of max_lr must be the one of list or ' + f'dict, but got {type(max_lr)}') + self._max_lr = max_lr + if total_steps is not None: + if not isinstance(total_steps, int): + raise ValueError('the type of total_steps must be int, but' + f'got {type(total_steps)}') + self.total_steps = total_steps + # validate pct_start + if pct_start < 0 or pct_start > 1 or not isinstance(pct_start, float): + raise ValueError('expected float between 0 and 1 pct_start, but ' + f'got {pct_start}') + self.pct_start = pct_start + # validate anneal_strategy + if anneal_strategy not in ['cos', 'linear']: + raise ValueError('anneal_strategy must be one of "cos" or ' + f'"linear", instead got {anneal_strategy}') + elif anneal_strategy == 'cos': + self.anneal_func = annealing_cos + elif anneal_strategy == 'linear': + self.anneal_func = annealing_linear + self.div_factor = div_factor + self.final_div_factor = final_div_factor + self.three_phase = three_phase + self.lr_phases = [] # init lr_phases + super(OneCycleLrUpdaterHook, self).__init__(**kwargs) + + def before_run(self, runner): + if hasattr(self, 'total_steps'): + total_steps = self.total_steps + else: + total_steps = runner.max_iters + if total_steps < runner.max_iters: + raise ValueError( + 'The total steps must be greater than or equal to max ' + f'iterations {runner.max_iters} of runner, but total steps ' + f'is {total_steps}.') + + if isinstance(runner.optimizer, dict): + self.base_lr = {} + for k, optim in runner.optimizer.items(): + _max_lr = format_param(k, optim, self._max_lr) + self.base_lr[k] = [lr / self.div_factor for lr in _max_lr] + for group, lr in zip(optim.param_groups, self.base_lr[k]): + group.setdefault('initial_lr', lr) + else: + k = type(runner.optimizer).__name__ + _max_lr = format_param(k, runner.optimizer, self._max_lr) + self.base_lr = [lr / self.div_factor for lr in _max_lr] + for group, lr in zip(runner.optimizer.param_groups, self.base_lr): + group.setdefault('initial_lr', lr) + + if self.three_phase: + self.lr_phases.append( + [float(self.pct_start * total_steps) - 1, 1, self.div_factor]) + self.lr_phases.append([ + float(2 * self.pct_start * total_steps) - 2, self.div_factor, 1 + ]) + self.lr_phases.append( + [total_steps - 1, 1, 1 / self.final_div_factor]) + else: + self.lr_phases.append( + [float(self.pct_start * total_steps) - 1, 1, self.div_factor]) + self.lr_phases.append( + [total_steps - 1, self.div_factor, 1 / self.final_div_factor]) + + def get_lr(self, runner, base_lr): + curr_iter = runner.iter + start_iter = 0 + for i, (end_iter, start_lr, end_lr) in enumerate(self.lr_phases): + if curr_iter <= end_iter: + pct = (curr_iter - start_iter) / (end_iter - start_iter) + lr = self.anneal_func(base_lr * start_lr, base_lr * end_lr, + pct) + break + start_iter = end_iter + return lr + + +def annealing_cos(start, end, factor, weight=1): + """Calculate annealing cos learning rate. + + Cosine anneal from `weight * start + (1 - weight) * end` to `end` as + percentage goes from 0.0 to 1.0. + + Args: + start (float): The starting learning rate of the cosine annealing. + end (float): The ending learing rate of the cosine annealing. + factor (float): The coefficient of `pi` when calculating the current + percentage. Range from 0.0 to 1.0. + weight (float, optional): The combination factor of `start` and `end` + when calculating the actual starting learning rate. Default to 1. + """ + cos_out = cos(pi * factor) + 1 + return end + 0.5 * weight * (start - end) * cos_out + + +def annealing_linear(start, end, factor): + """Calculate annealing linear learning rate. + + Linear anneal from `start` to `end` as percentage goes from 0.0 to 1.0. + + Args: + start (float): The starting learning rate of the linear annealing. + end (float): The ending learing rate of the linear annealing. + factor (float): The coefficient of `pi` when calculating the current + percentage. Range from 0.0 to 1.0. + """ + return start + (end - start) * factor + + +def format_param(name, optim, param): + if isinstance(param, numbers.Number): + return [param] * len(optim.param_groups) + elif isinstance(param, (list, tuple)): # multi param groups + if len(param) != len(optim.param_groups): + raise ValueError(f'expected {len(optim.param_groups)} ' + f'values for {name}, got {len(param)}') + return param + else: # multi optimizers + if name not in param: + raise KeyError(f'{name} is not found in {param.keys()}') + return param[name] diff --git a/annotator/mmpkg/mmcv/runner/hooks/memory.py b/annotator/mmpkg/mmcv/runner/hooks/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..70cf9a838fb314e3bd3c07aadbc00921a81e83ed --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/memory.py @@ -0,0 +1,25 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch + +from .hook import HOOKS, Hook + + +@HOOKS.register_module() +class EmptyCacheHook(Hook): + + def __init__(self, before_epoch=False, after_epoch=True, after_iter=False): + self._before_epoch = before_epoch + self._after_epoch = after_epoch + self._after_iter = after_iter + + def after_iter(self, runner): + if self._after_iter: + torch.cuda.empty_cache() + + def before_epoch(self, runner): + if self._before_epoch: + torch.cuda.empty_cache() + + def after_epoch(self, runner): + if self._after_epoch: + torch.cuda.empty_cache() diff --git a/annotator/mmpkg/mmcv/runner/hooks/momentum_updater.py b/annotator/mmpkg/mmcv/runner/hooks/momentum_updater.py new file mode 100644 index 0000000000000000000000000000000000000000..cdc70246280c2318f51034bb6b66eade7b478b79 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/momentum_updater.py @@ -0,0 +1,493 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import annotator.mmpkg.mmcv as mmcv +from .hook import HOOKS, Hook +from .lr_updater import annealing_cos, annealing_linear, format_param + + +class MomentumUpdaterHook(Hook): + + def __init__(self, + by_epoch=True, + warmup=None, + warmup_iters=0, + warmup_ratio=0.9): + # validate the "warmup" argument + if warmup is not None: + if warmup not in ['constant', 'linear', 'exp']: + raise ValueError( + f'"{warmup}" is not a supported type for warming up, valid' + ' types are "constant" and "linear"') + if warmup is not None: + assert warmup_iters > 0, \ + '"warmup_iters" must be a positive integer' + assert 0 < warmup_ratio <= 1.0, \ + '"warmup_momentum" must be in range (0,1]' + + self.by_epoch = by_epoch + self.warmup = warmup + self.warmup_iters = warmup_iters + self.warmup_ratio = warmup_ratio + + self.base_momentum = [] # initial momentum for all param groups + self.regular_momentum = [ + ] # expected momentum if no warming up is performed + + def _set_momentum(self, runner, momentum_groups): + if isinstance(runner.optimizer, dict): + for k, optim in runner.optimizer.items(): + for param_group, mom in zip(optim.param_groups, + momentum_groups[k]): + if 'momentum' in param_group.keys(): + param_group['momentum'] = mom + elif 'betas' in param_group.keys(): + param_group['betas'] = (mom, param_group['betas'][1]) + else: + for param_group, mom in zip(runner.optimizer.param_groups, + momentum_groups): + if 'momentum' in param_group.keys(): + param_group['momentum'] = mom + elif 'betas' in param_group.keys(): + param_group['betas'] = (mom, param_group['betas'][1]) + + def get_momentum(self, runner, base_momentum): + raise NotImplementedError + + def get_regular_momentum(self, runner): + if isinstance(runner.optimizer, dict): + momentum_groups = {} + for k in runner.optimizer.keys(): + _momentum_group = [ + self.get_momentum(runner, _base_momentum) + for _base_momentum in self.base_momentum[k] + ] + momentum_groups.update({k: _momentum_group}) + return momentum_groups + else: + return [ + self.get_momentum(runner, _base_momentum) + for _base_momentum in self.base_momentum + ] + + def get_warmup_momentum(self, cur_iters): + + def _get_warmup_momentum(cur_iters, regular_momentum): + if self.warmup == 'constant': + warmup_momentum = [ + _momentum / self.warmup_ratio + for _momentum in self.regular_momentum + ] + elif self.warmup == 'linear': + k = (1 - cur_iters / self.warmup_iters) * (1 - + self.warmup_ratio) + warmup_momentum = [ + _momentum / (1 - k) for _momentum in self.regular_mom + ] + elif self.warmup == 'exp': + k = self.warmup_ratio**(1 - cur_iters / self.warmup_iters) + warmup_momentum = [ + _momentum / k for _momentum in self.regular_mom + ] + return warmup_momentum + + if isinstance(self.regular_momentum, dict): + momentum_groups = {} + for key, regular_momentum in self.regular_momentum.items(): + momentum_groups[key] = _get_warmup_momentum( + cur_iters, regular_momentum) + return momentum_groups + else: + return _get_warmup_momentum(cur_iters, self.regular_momentum) + + def before_run(self, runner): + # NOTE: when resuming from a checkpoint, + # if 'initial_momentum' is not saved, + # it will be set according to the optimizer params + if isinstance(runner.optimizer, dict): + self.base_momentum = {} + for k, optim in runner.optimizer.items(): + for group in optim.param_groups: + if 'momentum' in group.keys(): + group.setdefault('initial_momentum', group['momentum']) + else: + group.setdefault('initial_momentum', group['betas'][0]) + _base_momentum = [ + group['initial_momentum'] for group in optim.param_groups + ] + self.base_momentum.update({k: _base_momentum}) + else: + for group in runner.optimizer.param_groups: + if 'momentum' in group.keys(): + group.setdefault('initial_momentum', group['momentum']) + else: + group.setdefault('initial_momentum', group['betas'][0]) + self.base_momentum = [ + group['initial_momentum'] + for group in runner.optimizer.param_groups + ] + + def before_train_epoch(self, runner): + if not self.by_epoch: + return + self.regular_mom = self.get_regular_momentum(runner) + self._set_momentum(runner, self.regular_mom) + + def before_train_iter(self, runner): + cur_iter = runner.iter + if not self.by_epoch: + self.regular_mom = self.get_regular_momentum(runner) + if self.warmup is None or cur_iter >= self.warmup_iters: + self._set_momentum(runner, self.regular_mom) + else: + warmup_momentum = self.get_warmup_momentum(cur_iter) + self._set_momentum(runner, warmup_momentum) + elif self.by_epoch: + if self.warmup is None or cur_iter > self.warmup_iters: + return + elif cur_iter == self.warmup_iters: + self._set_momentum(runner, self.regular_mom) + else: + warmup_momentum = self.get_warmup_momentum(cur_iter) + self._set_momentum(runner, warmup_momentum) + + +@HOOKS.register_module() +class StepMomentumUpdaterHook(MomentumUpdaterHook): + """Step momentum scheduler with min value clipping. + + Args: + step (int | list[int]): Step to decay the momentum. If an int value is + given, regard it as the decay interval. If a list is given, decay + momentum at these steps. + gamma (float, optional): Decay momentum ratio. Default: 0.5. + min_momentum (float, optional): Minimum momentum value to keep. If + momentum after decay is lower than this value, it will be clipped + accordingly. If None is given, we don't perform lr clipping. + Default: None. + """ + + def __init__(self, step, gamma=0.5, min_momentum=None, **kwargs): + if isinstance(step, list): + assert mmcv.is_list_of(step, int) + assert all([s > 0 for s in step]) + elif isinstance(step, int): + assert step > 0 + else: + raise TypeError('"step" must be a list or integer') + self.step = step + self.gamma = gamma + self.min_momentum = min_momentum + super(StepMomentumUpdaterHook, self).__init__(**kwargs) + + def get_momentum(self, runner, base_momentum): + progress = runner.epoch if self.by_epoch else runner.iter + + # calculate exponential term + if isinstance(self.step, int): + exp = progress // self.step + else: + exp = len(self.step) + for i, s in enumerate(self.step): + if progress < s: + exp = i + break + + momentum = base_momentum * (self.gamma**exp) + if self.min_momentum is not None: + # clip to a minimum value + momentum = max(momentum, self.min_momentum) + return momentum + + +@HOOKS.register_module() +class CosineAnnealingMomentumUpdaterHook(MomentumUpdaterHook): + + def __init__(self, min_momentum=None, min_momentum_ratio=None, **kwargs): + assert (min_momentum is None) ^ (min_momentum_ratio is None) + self.min_momentum = min_momentum + self.min_momentum_ratio = min_momentum_ratio + super(CosineAnnealingMomentumUpdaterHook, self).__init__(**kwargs) + + def get_momentum(self, runner, base_momentum): + if self.by_epoch: + progress = runner.epoch + max_progress = runner.max_epochs + else: + progress = runner.iter + max_progress = runner.max_iters + if self.min_momentum_ratio is not None: + target_momentum = base_momentum * self.min_momentum_ratio + else: + target_momentum = self.min_momentum + return annealing_cos(base_momentum, target_momentum, + progress / max_progress) + + +@HOOKS.register_module() +class CyclicMomentumUpdaterHook(MomentumUpdaterHook): + """Cyclic momentum Scheduler. + + Implement the cyclical momentum scheduler policy described in + https://arxiv.org/pdf/1708.07120.pdf + + This momentum scheduler usually used together with the CyclicLRUpdater + to improve the performance in the 3D detection area. + + Attributes: + target_ratio (tuple[float]): Relative ratio of the lowest momentum and + the highest momentum to the initial momentum. + cyclic_times (int): Number of cycles during training + step_ratio_up (float): The ratio of the increasing process of momentum + in the total cycle. + by_epoch (bool): Whether to update momentum by epoch. + """ + + def __init__(self, + by_epoch=False, + target_ratio=(0.85 / 0.95, 1), + cyclic_times=1, + step_ratio_up=0.4, + **kwargs): + if isinstance(target_ratio, float): + target_ratio = (target_ratio, target_ratio / 1e5) + elif isinstance(target_ratio, tuple): + target_ratio = (target_ratio[0], target_ratio[0] / 1e5) \ + if len(target_ratio) == 1 else target_ratio + else: + raise ValueError('target_ratio should be either float ' + f'or tuple, got {type(target_ratio)}') + + assert len(target_ratio) == 2, \ + '"target_ratio" must be list or tuple of two floats' + assert 0 <= step_ratio_up < 1.0, \ + '"step_ratio_up" must be in range [0,1)' + + self.target_ratio = target_ratio + self.cyclic_times = cyclic_times + self.step_ratio_up = step_ratio_up + self.momentum_phases = [] # init momentum_phases + # currently only support by_epoch=False + assert not by_epoch, \ + 'currently only support "by_epoch" = False' + super(CyclicMomentumUpdaterHook, self).__init__(by_epoch, **kwargs) + + def before_run(self, runner): + super(CyclicMomentumUpdaterHook, self).before_run(runner) + # initiate momentum_phases + # total momentum_phases are separated as up and down + max_iter_per_phase = runner.max_iters // self.cyclic_times + iter_up_phase = int(self.step_ratio_up * max_iter_per_phase) + self.momentum_phases.append( + [0, iter_up_phase, max_iter_per_phase, 1, self.target_ratio[0]]) + self.momentum_phases.append([ + iter_up_phase, max_iter_per_phase, max_iter_per_phase, + self.target_ratio[0], self.target_ratio[1] + ]) + + def get_momentum(self, runner, base_momentum): + curr_iter = runner.iter + for (start_iter, end_iter, max_iter_per_phase, start_ratio, + end_ratio) in self.momentum_phases: + curr_iter %= max_iter_per_phase + if start_iter <= curr_iter < end_iter: + progress = curr_iter - start_iter + return annealing_cos(base_momentum * start_ratio, + base_momentum * end_ratio, + progress / (end_iter - start_iter)) + + +@HOOKS.register_module() +class OneCycleMomentumUpdaterHook(MomentumUpdaterHook): + """OneCycle momentum Scheduler. + + This momentum scheduler usually used together with the OneCycleLrUpdater + to improve the performance. + + Args: + base_momentum (float or list): Lower momentum boundaries in the cycle + for each parameter group. Note that momentum is cycled inversely + to learning rate; at the peak of a cycle, momentum is + 'base_momentum' and learning rate is 'max_lr'. + Default: 0.85 + max_momentum (float or list): Upper momentum boundaries in the cycle + for each parameter group. Functionally, + it defines the cycle amplitude (max_momentum - base_momentum). + Note that momentum is cycled inversely + to learning rate; at the start of a cycle, momentum is + 'max_momentum' and learning rate is 'base_lr' + Default: 0.95 + pct_start (float): The percentage of the cycle (in number of steps) + spent increasing the learning rate. + Default: 0.3 + anneal_strategy (str): {'cos', 'linear'} + Specifies the annealing strategy: 'cos' for cosine annealing, + 'linear' for linear annealing. + Default: 'cos' + three_phase (bool): If three_phase is True, use a third phase of the + schedule to annihilate the learning rate according to + final_div_factor instead of modifying the second phase (the first + two phases will be symmetrical about the step indicated by + pct_start). + Default: False + """ + + def __init__(self, + base_momentum=0.85, + max_momentum=0.95, + pct_start=0.3, + anneal_strategy='cos', + three_phase=False, + **kwargs): + # validate by_epoch, currently only support by_epoch=False + if 'by_epoch' not in kwargs: + kwargs['by_epoch'] = False + else: + assert not kwargs['by_epoch'], \ + 'currently only support "by_epoch" = False' + if not isinstance(base_momentum, (float, list, dict)): + raise ValueError('base_momentum must be the type among of float,' + 'list or dict.') + self._base_momentum = base_momentum + if not isinstance(max_momentum, (float, list, dict)): + raise ValueError('max_momentum must be the type among of float,' + 'list or dict.') + self._max_momentum = max_momentum + # validate pct_start + if pct_start < 0 or pct_start > 1 or not isinstance(pct_start, float): + raise ValueError('Expected float between 0 and 1 pct_start, but ' + f'got {pct_start}') + self.pct_start = pct_start + # validate anneal_strategy + if anneal_strategy not in ['cos', 'linear']: + raise ValueError('anneal_strategy must by one of "cos" or ' + f'"linear", instead got {anneal_strategy}') + elif anneal_strategy == 'cos': + self.anneal_func = annealing_cos + elif anneal_strategy == 'linear': + self.anneal_func = annealing_linear + self.three_phase = three_phase + self.momentum_phases = [] # init momentum_phases + super(OneCycleMomentumUpdaterHook, self).__init__(**kwargs) + + def before_run(self, runner): + if isinstance(runner.optimizer, dict): + for k, optim in runner.optimizer.items(): + if ('momentum' not in optim.defaults + and 'betas' not in optim.defaults): + raise ValueError('optimizer must support momentum with' + 'option enabled') + self.use_beta1 = 'betas' in optim.defaults + _base_momentum = format_param(k, optim, self._base_momentum) + _max_momentum = format_param(k, optim, self._max_momentum) + for group, b_momentum, m_momentum in zip( + optim.param_groups, _base_momentum, _max_momentum): + if self.use_beta1: + _, beta2 = group['betas'] + group['betas'] = (m_momentum, beta2) + else: + group['momentum'] = m_momentum + group['base_momentum'] = b_momentum + group['max_momentum'] = m_momentum + else: + optim = runner.optimizer + if ('momentum' not in optim.defaults + and 'betas' not in optim.defaults): + raise ValueError('optimizer must support momentum with' + 'option enabled') + self.use_beta1 = 'betas' in optim.defaults + k = type(optim).__name__ + _base_momentum = format_param(k, optim, self._base_momentum) + _max_momentum = format_param(k, optim, self._max_momentum) + for group, b_momentum, m_momentum in zip(optim.param_groups, + _base_momentum, + _max_momentum): + if self.use_beta1: + _, beta2 = group['betas'] + group['betas'] = (m_momentum, beta2) + else: + group['momentum'] = m_momentum + group['base_momentum'] = b_momentum + group['max_momentum'] = m_momentum + + if self.three_phase: + self.momentum_phases.append({ + 'end_iter': + float(self.pct_start * runner.max_iters) - 1, + 'start_momentum': + 'max_momentum', + 'end_momentum': + 'base_momentum' + }) + self.momentum_phases.append({ + 'end_iter': + float(2 * self.pct_start * runner.max_iters) - 2, + 'start_momentum': + 'base_momentum', + 'end_momentum': + 'max_momentum' + }) + self.momentum_phases.append({ + 'end_iter': runner.max_iters - 1, + 'start_momentum': 'max_momentum', + 'end_momentum': 'max_momentum' + }) + else: + self.momentum_phases.append({ + 'end_iter': + float(self.pct_start * runner.max_iters) - 1, + 'start_momentum': + 'max_momentum', + 'end_momentum': + 'base_momentum' + }) + self.momentum_phases.append({ + 'end_iter': runner.max_iters - 1, + 'start_momentum': 'base_momentum', + 'end_momentum': 'max_momentum' + }) + + def _set_momentum(self, runner, momentum_groups): + if isinstance(runner.optimizer, dict): + for k, optim in runner.optimizer.items(): + for param_group, mom in zip(optim.param_groups, + momentum_groups[k]): + if 'momentum' in param_group.keys(): + param_group['momentum'] = mom + elif 'betas' in param_group.keys(): + param_group['betas'] = (mom, param_group['betas'][1]) + else: + for param_group, mom in zip(runner.optimizer.param_groups, + momentum_groups): + if 'momentum' in param_group.keys(): + param_group['momentum'] = mom + elif 'betas' in param_group.keys(): + param_group['betas'] = (mom, param_group['betas'][1]) + + def get_momentum(self, runner, param_group): + curr_iter = runner.iter + start_iter = 0 + for i, phase in enumerate(self.momentum_phases): + end_iter = phase['end_iter'] + if curr_iter <= end_iter or i == len(self.momentum_phases) - 1: + pct = (curr_iter - start_iter) / (end_iter - start_iter) + momentum = self.anneal_func( + param_group[phase['start_momentum']], + param_group[phase['end_momentum']], pct) + break + start_iter = end_iter + return momentum + + def get_regular_momentum(self, runner): + if isinstance(runner.optimizer, dict): + momentum_groups = {} + for k, optim in runner.optimizer.items(): + _momentum_group = [ + self.get_momentum(runner, param_group) + for param_group in optim.param_groups + ] + momentum_groups.update({k: _momentum_group}) + return momentum_groups + else: + momentum_groups = [] + for param_group in runner.optimizer.param_groups: + momentum_groups.append(self.get_momentum(runner, param_group)) + return momentum_groups diff --git a/annotator/mmpkg/mmcv/runner/hooks/optimizer.py b/annotator/mmpkg/mmcv/runner/hooks/optimizer.py new file mode 100644 index 0000000000000000000000000000000000000000..580a183639a5d95c04ecae9c619afb795a169e9e --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/optimizer.py @@ -0,0 +1,508 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy +from collections import defaultdict +from itertools import chain + +from torch.nn.utils import clip_grad + +from annotator.mmpkg.mmcv.utils import TORCH_VERSION, _BatchNorm, digit_version +from ..dist_utils import allreduce_grads +from ..fp16_utils import LossScaler, wrap_fp16_model +from .hook import HOOKS, Hook + +try: + # If PyTorch version >= 1.6.0, torch.cuda.amp.GradScaler would be imported + # and used; otherwise, auto fp16 will adopt mmcv's implementation. + from torch.cuda.amp import GradScaler +except ImportError: + pass + + +@HOOKS.register_module() +class OptimizerHook(Hook): + + def __init__(self, grad_clip=None): + self.grad_clip = grad_clip + + def clip_grads(self, params): + params = list( + filter(lambda p: p.requires_grad and p.grad is not None, params)) + if len(params) > 0: + return clip_grad.clip_grad_norm_(params, **self.grad_clip) + + def after_train_iter(self, runner): + runner.optimizer.zero_grad() + runner.outputs['loss'].backward() + if self.grad_clip is not None: + grad_norm = self.clip_grads(runner.model.parameters()) + if grad_norm is not None: + # Add grad norm to the logger + runner.log_buffer.update({'grad_norm': float(grad_norm)}, + runner.outputs['num_samples']) + runner.optimizer.step() + + +@HOOKS.register_module() +class GradientCumulativeOptimizerHook(OptimizerHook): + """Optimizer Hook implements multi-iters gradient cumulating. + + Args: + cumulative_iters (int, optional): Num of gradient cumulative iters. + The optimizer will step every `cumulative_iters` iters. + Defaults to 1. + + Examples: + >>> # Use cumulative_iters to simulate a large batch size + >>> # It is helpful when the hardware cannot handle a large batch size. + >>> loader = DataLoader(data, batch_size=64) + >>> optim_hook = GradientCumulativeOptimizerHook(cumulative_iters=4) + >>> # almost equals to + >>> loader = DataLoader(data, batch_size=256) + >>> optim_hook = OptimizerHook() + """ + + def __init__(self, cumulative_iters=1, **kwargs): + super(GradientCumulativeOptimizerHook, self).__init__(**kwargs) + + assert isinstance(cumulative_iters, int) and cumulative_iters > 0, \ + f'cumulative_iters only accepts positive int, but got ' \ + f'{type(cumulative_iters)} instead.' + + self.cumulative_iters = cumulative_iters + self.divisible_iters = 0 + self.remainder_iters = 0 + self.initialized = False + + def has_batch_norm(self, module): + if isinstance(module, _BatchNorm): + return True + for m in module.children(): + if self.has_batch_norm(m): + return True + return False + + def _init(self, runner): + if runner.iter % self.cumulative_iters != 0: + runner.logger.warning( + 'Resume iter number is not divisible by cumulative_iters in ' + 'GradientCumulativeOptimizerHook, which means the gradient of ' + 'some iters is lost and the result may be influenced slightly.' + ) + + if self.has_batch_norm(runner.model) and self.cumulative_iters > 1: + runner.logger.warning( + 'GradientCumulativeOptimizerHook may slightly decrease ' + 'performance if the model has BatchNorm layers.') + + residual_iters = runner.max_iters - runner.iter + + self.divisible_iters = ( + residual_iters // self.cumulative_iters * self.cumulative_iters) + self.remainder_iters = residual_iters - self.divisible_iters + + self.initialized = True + + def after_train_iter(self, runner): + if not self.initialized: + self._init(runner) + + if runner.iter < self.divisible_iters: + loss_factor = self.cumulative_iters + else: + loss_factor = self.remainder_iters + loss = runner.outputs['loss'] + loss = loss / loss_factor + loss.backward() + + if (self.every_n_iters(runner, self.cumulative_iters) + or self.is_last_iter(runner)): + + if self.grad_clip is not None: + grad_norm = self.clip_grads(runner.model.parameters()) + if grad_norm is not None: + # Add grad norm to the logger + runner.log_buffer.update({'grad_norm': float(grad_norm)}, + runner.outputs['num_samples']) + runner.optimizer.step() + runner.optimizer.zero_grad() + + +if (TORCH_VERSION != 'parrots' + and digit_version(TORCH_VERSION) >= digit_version('1.6.0')): + + @HOOKS.register_module() + class Fp16OptimizerHook(OptimizerHook): + """FP16 optimizer hook (using PyTorch's implementation). + + If you are using PyTorch >= 1.6, torch.cuda.amp is used as the backend, + to take care of the optimization procedure. + + Args: + loss_scale (float | str | dict): Scale factor configuration. + If loss_scale is a float, static loss scaling will be used with + the specified scale. If loss_scale is a string, it must be + 'dynamic', then dynamic loss scaling will be used. + It can also be a dict containing arguments of GradScalar. + Defaults to 512. For Pytorch >= 1.6, mmcv uses official + implementation of GradScaler. If you use a dict version of + loss_scale to create GradScaler, please refer to: + https://pytorch.org/docs/stable/amp.html#torch.cuda.amp.GradScaler + for the parameters. + + Examples: + >>> loss_scale = dict( + ... init_scale=65536.0, + ... growth_factor=2.0, + ... backoff_factor=0.5, + ... growth_interval=2000 + ... ) + >>> optimizer_hook = Fp16OptimizerHook(loss_scale=loss_scale) + """ + + def __init__(self, + grad_clip=None, + coalesce=True, + bucket_size_mb=-1, + loss_scale=512., + distributed=True): + self.grad_clip = grad_clip + self.coalesce = coalesce + self.bucket_size_mb = bucket_size_mb + self.distributed = distributed + self._scale_update_param = None + if loss_scale == 'dynamic': + self.loss_scaler = GradScaler() + elif isinstance(loss_scale, float): + self._scale_update_param = loss_scale + self.loss_scaler = GradScaler(init_scale=loss_scale) + elif isinstance(loss_scale, dict): + self.loss_scaler = GradScaler(**loss_scale) + else: + raise ValueError('loss_scale must be of type float, dict, or ' + f'"dynamic", got {loss_scale}') + + def before_run(self, runner): + """Preparing steps before Mixed Precision Training.""" + # wrap model mode to fp16 + wrap_fp16_model(runner.model) + # resume from state dict + if 'fp16' in runner.meta and 'loss_scaler' in runner.meta['fp16']: + scaler_state_dict = runner.meta['fp16']['loss_scaler'] + self.loss_scaler.load_state_dict(scaler_state_dict) + + def copy_grads_to_fp32(self, fp16_net, fp32_weights): + """Copy gradients from fp16 model to fp32 weight copy.""" + for fp32_param, fp16_param in zip(fp32_weights, + fp16_net.parameters()): + if fp16_param.grad is not None: + if fp32_param.grad is None: + fp32_param.grad = fp32_param.data.new( + fp32_param.size()) + fp32_param.grad.copy_(fp16_param.grad) + + def copy_params_to_fp16(self, fp16_net, fp32_weights): + """Copy updated params from fp32 weight copy to fp16 model.""" + for fp16_param, fp32_param in zip(fp16_net.parameters(), + fp32_weights): + fp16_param.data.copy_(fp32_param.data) + + def after_train_iter(self, runner): + """Backward optimization steps for Mixed Precision Training. For + dynamic loss scaling, please refer to + https://pytorch.org/docs/stable/amp.html#torch.cuda.amp.GradScaler. + + 1. Scale the loss by a scale factor. + 2. Backward the loss to obtain the gradients. + 3. Unscale the optimizer’s gradient tensors. + 4. Call optimizer.step() and update scale factor. + 5. Save loss_scaler state_dict for resume purpose. + """ + # clear grads of last iteration + runner.model.zero_grad() + runner.optimizer.zero_grad() + + self.loss_scaler.scale(runner.outputs['loss']).backward() + self.loss_scaler.unscale_(runner.optimizer) + # grad clip + if self.grad_clip is not None: + grad_norm = self.clip_grads(runner.model.parameters()) + if grad_norm is not None: + # Add grad norm to the logger + runner.log_buffer.update({'grad_norm': float(grad_norm)}, + runner.outputs['num_samples']) + # backward and update scaler + self.loss_scaler.step(runner.optimizer) + self.loss_scaler.update(self._scale_update_param) + + # save state_dict of loss_scaler + runner.meta.setdefault( + 'fp16', {})['loss_scaler'] = self.loss_scaler.state_dict() + + @HOOKS.register_module() + class GradientCumulativeFp16OptimizerHook(GradientCumulativeOptimizerHook, + Fp16OptimizerHook): + """Fp16 optimizer Hook (using PyTorch's implementation) implements + multi-iters gradient cumulating. + + If you are using PyTorch >= 1.6, torch.cuda.amp is used as the backend, + to take care of the optimization procedure. + """ + + def __init__(self, *args, **kwargs): + super(GradientCumulativeFp16OptimizerHook, + self).__init__(*args, **kwargs) + + def after_train_iter(self, runner): + if not self.initialized: + self._init(runner) + + if runner.iter < self.divisible_iters: + loss_factor = self.cumulative_iters + else: + loss_factor = self.remainder_iters + loss = runner.outputs['loss'] + loss = loss / loss_factor + + self.loss_scaler.scale(loss).backward() + + if (self.every_n_iters(runner, self.cumulative_iters) + or self.is_last_iter(runner)): + + # copy fp16 grads in the model to fp32 params in the optimizer + self.loss_scaler.unscale_(runner.optimizer) + + if self.grad_clip is not None: + grad_norm = self.clip_grads(runner.model.parameters()) + if grad_norm is not None: + # Add grad norm to the logger + runner.log_buffer.update( + {'grad_norm': float(grad_norm)}, + runner.outputs['num_samples']) + + # backward and update scaler + self.loss_scaler.step(runner.optimizer) + self.loss_scaler.update(self._scale_update_param) + + # save state_dict of loss_scaler + runner.meta.setdefault( + 'fp16', {})['loss_scaler'] = self.loss_scaler.state_dict() + + # clear grads + runner.model.zero_grad() + runner.optimizer.zero_grad() + +else: + + @HOOKS.register_module() + class Fp16OptimizerHook(OptimizerHook): + """FP16 optimizer hook (mmcv's implementation). + + The steps of fp16 optimizer is as follows. + 1. Scale the loss value. + 2. BP in the fp16 model. + 2. Copy gradients from fp16 model to fp32 weights. + 3. Update fp32 weights. + 4. Copy updated parameters from fp32 weights to fp16 model. + + Refer to https://arxiv.org/abs/1710.03740 for more details. + + Args: + loss_scale (float | str | dict): Scale factor configuration. + If loss_scale is a float, static loss scaling will be used with + the specified scale. If loss_scale is a string, it must be + 'dynamic', then dynamic loss scaling will be used. + It can also be a dict containing arguments of LossScaler. + Defaults to 512. + """ + + def __init__(self, + grad_clip=None, + coalesce=True, + bucket_size_mb=-1, + loss_scale=512., + distributed=True): + self.grad_clip = grad_clip + self.coalesce = coalesce + self.bucket_size_mb = bucket_size_mb + self.distributed = distributed + if loss_scale == 'dynamic': + self.loss_scaler = LossScaler(mode='dynamic') + elif isinstance(loss_scale, float): + self.loss_scaler = LossScaler( + init_scale=loss_scale, mode='static') + elif isinstance(loss_scale, dict): + self.loss_scaler = LossScaler(**loss_scale) + else: + raise ValueError('loss_scale must be of type float, dict, or ' + f'"dynamic", got {loss_scale}') + + def before_run(self, runner): + """Preparing steps before Mixed Precision Training. + + 1. Make a master copy of fp32 weights for optimization. + 2. Convert the main model from fp32 to fp16. + """ + # keep a copy of fp32 weights + old_groups = runner.optimizer.param_groups + runner.optimizer.param_groups = copy.deepcopy( + runner.optimizer.param_groups) + state = defaultdict(dict) + p_map = { + old_p: p + for old_p, p in zip( + chain(*(g['params'] for g in old_groups)), + chain(*(g['params'] + for g in runner.optimizer.param_groups))) + } + for k, v in runner.optimizer.state.items(): + state[p_map[k]] = v + runner.optimizer.state = state + # convert model to fp16 + wrap_fp16_model(runner.model) + # resume from state dict + if 'fp16' in runner.meta and 'loss_scaler' in runner.meta['fp16']: + scaler_state_dict = runner.meta['fp16']['loss_scaler'] + self.loss_scaler.load_state_dict(scaler_state_dict) + + def copy_grads_to_fp32(self, fp16_net, fp32_weights): + """Copy gradients from fp16 model to fp32 weight copy.""" + for fp32_param, fp16_param in zip(fp32_weights, + fp16_net.parameters()): + if fp16_param.grad is not None: + if fp32_param.grad is None: + fp32_param.grad = fp32_param.data.new( + fp32_param.size()) + fp32_param.grad.copy_(fp16_param.grad) + + def copy_params_to_fp16(self, fp16_net, fp32_weights): + """Copy updated params from fp32 weight copy to fp16 model.""" + for fp16_param, fp32_param in zip(fp16_net.parameters(), + fp32_weights): + fp16_param.data.copy_(fp32_param.data) + + def after_train_iter(self, runner): + """Backward optimization steps for Mixed Precision Training. For + dynamic loss scaling, please refer `loss_scalar.py` + + 1. Scale the loss by a scale factor. + 2. Backward the loss to obtain the gradients (fp16). + 3. Copy gradients from the model to the fp32 weight copy. + 4. Scale the gradients back and update the fp32 weight copy. + 5. Copy back the params from fp32 weight copy to the fp16 model. + 6. Save loss_scaler state_dict for resume purpose. + """ + # clear grads of last iteration + runner.model.zero_grad() + runner.optimizer.zero_grad() + # scale the loss value + scaled_loss = runner.outputs['loss'] * self.loss_scaler.loss_scale + scaled_loss.backward() + # copy fp16 grads in the model to fp32 params in the optimizer + + fp32_weights = [] + for param_group in runner.optimizer.param_groups: + fp32_weights += param_group['params'] + self.copy_grads_to_fp32(runner.model, fp32_weights) + # allreduce grads + if self.distributed: + allreduce_grads(fp32_weights, self.coalesce, + self.bucket_size_mb) + + has_overflow = self.loss_scaler.has_overflow(fp32_weights) + # if has overflow, skip this iteration + if not has_overflow: + # scale the gradients back + for param in fp32_weights: + if param.grad is not None: + param.grad.div_(self.loss_scaler.loss_scale) + if self.grad_clip is not None: + grad_norm = self.clip_grads(fp32_weights) + if grad_norm is not None: + # Add grad norm to the logger + runner.log_buffer.update( + {'grad_norm': float(grad_norm)}, + runner.outputs['num_samples']) + # update fp32 params + runner.optimizer.step() + # copy fp32 params to the fp16 model + self.copy_params_to_fp16(runner.model, fp32_weights) + self.loss_scaler.update_scale(has_overflow) + if has_overflow: + runner.logger.warning('Check overflow, downscale loss scale ' + f'to {self.loss_scaler.cur_scale}') + + # save state_dict of loss_scaler + runner.meta.setdefault( + 'fp16', {})['loss_scaler'] = self.loss_scaler.state_dict() + + @HOOKS.register_module() + class GradientCumulativeFp16OptimizerHook(GradientCumulativeOptimizerHook, + Fp16OptimizerHook): + """Fp16 optimizer Hook (using mmcv implementation) implements multi- + iters gradient cumulating.""" + + def __init__(self, *args, **kwargs): + super(GradientCumulativeFp16OptimizerHook, + self).__init__(*args, **kwargs) + + def after_train_iter(self, runner): + if not self.initialized: + self._init(runner) + + if runner.iter < self.divisible_iters: + loss_factor = self.cumulative_iters + else: + loss_factor = self.remainder_iters + + loss = runner.outputs['loss'] + loss = loss / loss_factor + + # scale the loss value + scaled_loss = loss * self.loss_scaler.loss_scale + scaled_loss.backward() + + if (self.every_n_iters(runner, self.cumulative_iters) + or self.is_last_iter(runner)): + + # copy fp16 grads in the model to fp32 params in the optimizer + fp32_weights = [] + for param_group in runner.optimizer.param_groups: + fp32_weights += param_group['params'] + self.copy_grads_to_fp32(runner.model, fp32_weights) + # allreduce grads + if self.distributed: + allreduce_grads(fp32_weights, self.coalesce, + self.bucket_size_mb) + + has_overflow = self.loss_scaler.has_overflow(fp32_weights) + # if has overflow, skip this iteration + if not has_overflow: + # scale the gradients back + for param in fp32_weights: + if param.grad is not None: + param.grad.div_(self.loss_scaler.loss_scale) + if self.grad_clip is not None: + grad_norm = self.clip_grads(fp32_weights) + if grad_norm is not None: + # Add grad norm to the logger + runner.log_buffer.update( + {'grad_norm': float(grad_norm)}, + runner.outputs['num_samples']) + # update fp32 params + runner.optimizer.step() + # copy fp32 params to the fp16 model + self.copy_params_to_fp16(runner.model, fp32_weights) + else: + runner.logger.warning( + 'Check overflow, downscale loss scale ' + f'to {self.loss_scaler.cur_scale}') + + self.loss_scaler.update_scale(has_overflow) + + # save state_dict of loss_scaler + runner.meta.setdefault( + 'fp16', {})['loss_scaler'] = self.loss_scaler.state_dict() + + # clear grads + runner.model.zero_grad() + runner.optimizer.zero_grad() diff --git a/annotator/mmpkg/mmcv/runner/hooks/profiler.py b/annotator/mmpkg/mmcv/runner/hooks/profiler.py new file mode 100644 index 0000000000000000000000000000000000000000..b70236997eec59c2209ef351ae38863b4112d0ec --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/profiler.py @@ -0,0 +1,180 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import warnings +from typing import Callable, List, Optional, Union + +import torch + +from ..dist_utils import master_only +from .hook import HOOKS, Hook + + +@HOOKS.register_module() +class ProfilerHook(Hook): + """Profiler to analyze performance during training. + + PyTorch Profiler is a tool that allows the collection of the performance + metrics during the training. More details on Profiler can be found at + https://pytorch.org/docs/1.8.1/profiler.html#torch.profiler.profile + + Args: + by_epoch (bool): Profile performance by epoch or by iteration. + Default: True. + profile_iters (int): Number of iterations for profiling. + If ``by_epoch=True``, profile_iters indicates that they are the + first profile_iters epochs at the beginning of the + training, otherwise it indicates the first profile_iters + iterations. Default: 1. + activities (list[str]): List of activity groups (CPU, CUDA) to use in + profiling. Default: ['cpu', 'cuda']. + schedule (dict, optional): Config of generating the callable schedule. + if schedule is None, profiler will not add step markers into the + trace and table view. Default: None. + on_trace_ready (callable, dict): Either a handler or a dict of generate + handler. Default: None. + record_shapes (bool): Save information about operator's input shapes. + Default: False. + profile_memory (bool): Track tensor memory allocation/deallocation. + Default: False. + with_stack (bool): Record source information (file and line number) + for the ops. Default: False. + with_flops (bool): Use formula to estimate the FLOPS of specific + operators (matrix multiplication and 2D convolution). + Default: False. + json_trace_path (str, optional): Exports the collected trace in Chrome + JSON format. Default: None. + + Example: + >>> runner = ... # instantiate a Runner + >>> # tensorboard trace + >>> trace_config = dict(type='tb_trace', dir_name='work_dir') + >>> profiler_config = dict(on_trace_ready=trace_config) + >>> runner.register_profiler_hook(profiler_config) + >>> runner.run(data_loaders=[trainloader], workflow=[('train', 1)]) + """ + + def __init__(self, + by_epoch: bool = True, + profile_iters: int = 1, + activities: List[str] = ['cpu', 'cuda'], + schedule: Optional[dict] = None, + on_trace_ready: Optional[Union[Callable, dict]] = None, + record_shapes: bool = False, + profile_memory: bool = False, + with_stack: bool = False, + with_flops: bool = False, + json_trace_path: Optional[str] = None) -> None: + try: + from torch import profiler # torch version >= 1.8.1 + except ImportError: + raise ImportError('profiler is the new feature of torch1.8.1, ' + f'but your version is {torch.__version__}') + + assert isinstance(by_epoch, bool), '``by_epoch`` should be a boolean.' + self.by_epoch = by_epoch + + if profile_iters < 1: + raise ValueError('profile_iters should be greater than 0, but got ' + f'{profile_iters}') + self.profile_iters = profile_iters + + if not isinstance(activities, list): + raise ValueError( + f'activities should be list, but got {type(activities)}') + self.activities = [] + for activity in activities: + activity = activity.lower() + if activity == 'cpu': + self.activities.append(profiler.ProfilerActivity.CPU) + elif activity == 'cuda': + self.activities.append(profiler.ProfilerActivity.CUDA) + else: + raise ValueError( + f'activity should be "cpu" or "cuda", but got {activity}') + + if schedule is not None: + self.schedule = profiler.schedule(**schedule) + else: + self.schedule = None + + self.on_trace_ready = on_trace_ready + self.record_shapes = record_shapes + self.profile_memory = profile_memory + self.with_stack = with_stack + self.with_flops = with_flops + self.json_trace_path = json_trace_path + + @master_only + def before_run(self, runner): + if self.by_epoch and runner.max_epochs < self.profile_iters: + raise ValueError('self.profile_iters should not be greater than ' + f'{runner.max_epochs}') + + if not self.by_epoch and runner.max_iters < self.profile_iters: + raise ValueError('self.profile_iters should not be greater than ' + f'{runner.max_iters}') + + if callable(self.on_trace_ready): # handler + _on_trace_ready = self.on_trace_ready + elif isinstance(self.on_trace_ready, dict): # config of handler + trace_cfg = self.on_trace_ready.copy() + trace_type = trace_cfg.pop('type') # log_trace handler + if trace_type == 'log_trace': + + def _log_handler(prof): + print(prof.key_averages().table(**trace_cfg)) + + _on_trace_ready = _log_handler + elif trace_type == 'tb_trace': # tensorboard_trace handler + try: + import torch_tb_profiler # noqa: F401 + except ImportError: + raise ImportError('please run "pip install ' + 'torch-tb-profiler" to install ' + 'torch_tb_profiler') + _on_trace_ready = torch.profiler.tensorboard_trace_handler( + **trace_cfg) + else: + raise ValueError('trace_type should be "log_trace" or ' + f'"tb_trace", but got {trace_type}') + elif self.on_trace_ready is None: + _on_trace_ready = None # type: ignore + else: + raise ValueError('on_trace_ready should be handler, dict or None, ' + f'but got {type(self.on_trace_ready)}') + + if runner.max_epochs > 1: + warnings.warn(f'profiler will profile {runner.max_epochs} epochs ' + 'instead of 1 epoch. Since profiler will slow down ' + 'the training, it is recommended to train 1 epoch ' + 'with ProfilerHook and adjust your setting according' + ' to the profiler summary. During normal training ' + '(epoch > 1), you may disable the ProfilerHook.') + + self.profiler = torch.profiler.profile( + activities=self.activities, + schedule=self.schedule, + on_trace_ready=_on_trace_ready, + record_shapes=self.record_shapes, + profile_memory=self.profile_memory, + with_stack=self.with_stack, + with_flops=self.with_flops) + + self.profiler.__enter__() + runner.logger.info('profiler is profiling...') + + @master_only + def after_train_epoch(self, runner): + if self.by_epoch and runner.epoch == self.profile_iters - 1: + runner.logger.info('profiler may take a few minutes...') + self.profiler.__exit__(None, None, None) + if self.json_trace_path is not None: + self.profiler.export_chrome_trace(self.json_trace_path) + + @master_only + def after_train_iter(self, runner): + self.profiler.step() + if not self.by_epoch and runner.iter == self.profile_iters - 1: + runner.logger.info('profiler may take a few minutes...') + self.profiler.__exit__(None, None, None) + if self.json_trace_path is not None: + self.profiler.export_chrome_trace(self.json_trace_path) diff --git a/annotator/mmpkg/mmcv/runner/hooks/sampler_seed.py b/annotator/mmpkg/mmcv/runner/hooks/sampler_seed.py new file mode 100644 index 0000000000000000000000000000000000000000..ee0dc6bdd8df5775857028aaed5444c0f59caf80 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/sampler_seed.py @@ -0,0 +1,20 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .hook import HOOKS, Hook + + +@HOOKS.register_module() +class DistSamplerSeedHook(Hook): + """Data-loading sampler for distributed training. + + When distributed training, it is only useful in conjunction with + :obj:`EpochBasedRunner`, while :obj:`IterBasedRunner` achieves the same + purpose with :obj:`IterLoader`. + """ + + def before_epoch(self, runner): + if hasattr(runner.data_loader.sampler, 'set_epoch'): + # in case the data loader uses `SequentialSampler` in Pytorch + runner.data_loader.sampler.set_epoch(runner.epoch) + elif hasattr(runner.data_loader.batch_sampler.sampler, 'set_epoch'): + # batch sampler in pytorch warps the sampler as its attributes. + runner.data_loader.batch_sampler.sampler.set_epoch(runner.epoch) diff --git a/annotator/mmpkg/mmcv/runner/hooks/sync_buffer.py b/annotator/mmpkg/mmcv/runner/hooks/sync_buffer.py new file mode 100644 index 0000000000000000000000000000000000000000..6376b7ff894280cb2782243b25e8973650591577 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/hooks/sync_buffer.py @@ -0,0 +1,22 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from ..dist_utils import allreduce_params +from .hook import HOOKS, Hook + + +@HOOKS.register_module() +class SyncBuffersHook(Hook): + """Synchronize model buffers such as running_mean and running_var in BN at + the end of each epoch. + + Args: + distributed (bool): Whether distributed training is used. It is + effective only for distributed training. Defaults to True. + """ + + def __init__(self, distributed=True): + self.distributed = distributed + + def after_epoch(self, runner): + """All-reduce model buffers at the end of each epoch.""" + if self.distributed: + allreduce_params(runner.model.buffers()) diff --git a/annotator/mmpkg/mmcv/runner/iter_based_runner.py b/annotator/mmpkg/mmcv/runner/iter_based_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..e93849ba8a0960d958c76151d5bdd406e4b795a4 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/iter_based_runner.py @@ -0,0 +1,273 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os.path as osp +import platform +import shutil +import time +import warnings + +import torch +from torch.optim import Optimizer + +import annotator.mmpkg.mmcv as mmcv +from .base_runner import BaseRunner +from .builder import RUNNERS +from .checkpoint import save_checkpoint +from .hooks import IterTimerHook +from .utils import get_host_info + + +class IterLoader: + + def __init__(self, dataloader): + self._dataloader = dataloader + self.iter_loader = iter(self._dataloader) + self._epoch = 0 + + @property + def epoch(self): + return self._epoch + + def __next__(self): + try: + data = next(self.iter_loader) + except StopIteration: + self._epoch += 1 + if hasattr(self._dataloader.sampler, 'set_epoch'): + self._dataloader.sampler.set_epoch(self._epoch) + time.sleep(2) # Prevent possible deadlock during epoch transition + self.iter_loader = iter(self._dataloader) + data = next(self.iter_loader) + + return data + + def __len__(self): + return len(self._dataloader) + + +@RUNNERS.register_module() +class IterBasedRunner(BaseRunner): + """Iteration-based Runner. + + This runner train models iteration by iteration. + """ + + def train(self, data_loader, **kwargs): + self.model.train() + self.mode = 'train' + self.data_loader = data_loader + self._epoch = data_loader.epoch + data_batch = next(data_loader) + self.call_hook('before_train_iter') + outputs = self.model.train_step(data_batch, self.optimizer, **kwargs) + if not isinstance(outputs, dict): + raise TypeError('model.train_step() must return a dict') + if 'log_vars' in outputs: + self.log_buffer.update(outputs['log_vars'], outputs['num_samples']) + self.outputs = outputs + self.call_hook('after_train_iter') + self._inner_iter += 1 + self._iter += 1 + + @torch.no_grad() + def val(self, data_loader, **kwargs): + self.model.eval() + self.mode = 'val' + self.data_loader = data_loader + data_batch = next(data_loader) + self.call_hook('before_val_iter') + outputs = self.model.val_step(data_batch, **kwargs) + if not isinstance(outputs, dict): + raise TypeError('model.val_step() must return a dict') + if 'log_vars' in outputs: + self.log_buffer.update(outputs['log_vars'], outputs['num_samples']) + self.outputs = outputs + self.call_hook('after_val_iter') + self._inner_iter += 1 + + def run(self, data_loaders, workflow, max_iters=None, **kwargs): + """Start running. + + Args: + data_loaders (list[:obj:`DataLoader`]): Dataloaders for training + and validation. + workflow (list[tuple]): A list of (phase, iters) to specify the + running order and iterations. E.g, [('train', 10000), + ('val', 1000)] means running 10000 iterations for training and + 1000 iterations for validation, iteratively. + """ + assert isinstance(data_loaders, list) + assert mmcv.is_list_of(workflow, tuple) + assert len(data_loaders) == len(workflow) + if max_iters is not None: + warnings.warn( + 'setting max_iters in run is deprecated, ' + 'please set max_iters in runner_config', DeprecationWarning) + self._max_iters = max_iters + assert self._max_iters is not None, ( + 'max_iters must be specified during instantiation') + + work_dir = self.work_dir if self.work_dir is not None else 'NONE' + self.logger.info('Start running, host: %s, work_dir: %s', + get_host_info(), work_dir) + self.logger.info('Hooks will be executed in the following order:\n%s', + self.get_hook_info()) + self.logger.info('workflow: %s, max: %d iters', workflow, + self._max_iters) + self.call_hook('before_run') + + iter_loaders = [IterLoader(x) for x in data_loaders] + + self.call_hook('before_epoch') + + while self.iter < self._max_iters: + for i, flow in enumerate(workflow): + self._inner_iter = 0 + mode, iters = flow + if not isinstance(mode, str) or not hasattr(self, mode): + raise ValueError( + 'runner has no method named "{}" to run a workflow'. + format(mode)) + iter_runner = getattr(self, mode) + for _ in range(iters): + if mode == 'train' and self.iter >= self._max_iters: + break + iter_runner(iter_loaders[i], **kwargs) + + time.sleep(1) # wait for some hooks like loggers to finish + self.call_hook('after_epoch') + self.call_hook('after_run') + + def resume(self, + checkpoint, + resume_optimizer=True, + map_location='default'): + """Resume model from checkpoint. + + Args: + checkpoint (str): Checkpoint to resume from. + resume_optimizer (bool, optional): Whether resume the optimizer(s) + if the checkpoint file includes optimizer(s). Default to True. + map_location (str, optional): Same as :func:`torch.load`. + Default to 'default'. + """ + if map_location == 'default': + device_id = torch.cuda.current_device() + checkpoint = self.load_checkpoint( + checkpoint, + map_location=lambda storage, loc: storage.cuda(device_id)) + else: + checkpoint = self.load_checkpoint( + checkpoint, map_location=map_location) + + self._epoch = checkpoint['meta']['epoch'] + self._iter = checkpoint['meta']['iter'] + self._inner_iter = checkpoint['meta']['iter'] + if 'optimizer' in checkpoint and resume_optimizer: + if isinstance(self.optimizer, Optimizer): + self.optimizer.load_state_dict(checkpoint['optimizer']) + elif isinstance(self.optimizer, dict): + for k in self.optimizer.keys(): + self.optimizer[k].load_state_dict( + checkpoint['optimizer'][k]) + else: + raise TypeError( + 'Optimizer should be dict or torch.optim.Optimizer ' + f'but got {type(self.optimizer)}') + + self.logger.info(f'resumed from epoch: {self.epoch}, iter {self.iter}') + + def save_checkpoint(self, + out_dir, + filename_tmpl='iter_{}.pth', + meta=None, + save_optimizer=True, + create_symlink=True): + """Save checkpoint to file. + + Args: + out_dir (str): Directory to save checkpoint files. + filename_tmpl (str, optional): Checkpoint file template. + Defaults to 'iter_{}.pth'. + meta (dict, optional): Metadata to be saved in checkpoint. + Defaults to None. + save_optimizer (bool, optional): Whether save optimizer. + Defaults to True. + create_symlink (bool, optional): Whether create symlink to the + latest checkpoint file. Defaults to True. + """ + if meta is None: + meta = {} + elif not isinstance(meta, dict): + raise TypeError( + f'meta should be a dict or None, but got {type(meta)}') + if self.meta is not None: + meta.update(self.meta) + # Note: meta.update(self.meta) should be done before + # meta.update(epoch=self.epoch + 1, iter=self.iter) otherwise + # there will be problems with resumed checkpoints. + # More details in https://github.com/open-mmlab/mmcv/pull/1108 + meta.update(epoch=self.epoch + 1, iter=self.iter) + + filename = filename_tmpl.format(self.iter + 1) + filepath = osp.join(out_dir, filename) + optimizer = self.optimizer if save_optimizer else None + save_checkpoint(self.model, filepath, optimizer=optimizer, meta=meta) + # in some environments, `os.symlink` is not supported, you may need to + # set `create_symlink` to False + if create_symlink: + dst_file = osp.join(out_dir, 'latest.pth') + if platform.system() != 'Windows': + mmcv.symlink(filename, dst_file) + else: + shutil.copy(filepath, dst_file) + + def register_training_hooks(self, + lr_config, + optimizer_config=None, + checkpoint_config=None, + log_config=None, + momentum_config=None, + custom_hooks_config=None): + """Register default hooks for iter-based training. + + Checkpoint hook, optimizer stepper hook and logger hooks will be set to + `by_epoch=False` by default. + + Default hooks include: + + +----------------------+-------------------------+ + | Hooks | Priority | + +======================+=========================+ + | LrUpdaterHook | VERY_HIGH (10) | + +----------------------+-------------------------+ + | MomentumUpdaterHook | HIGH (30) | + +----------------------+-------------------------+ + | OptimizerStepperHook | ABOVE_NORMAL (40) | + +----------------------+-------------------------+ + | CheckpointSaverHook | NORMAL (50) | + +----------------------+-------------------------+ + | IterTimerHook | LOW (70) | + +----------------------+-------------------------+ + | LoggerHook(s) | VERY_LOW (90) | + +----------------------+-------------------------+ + | CustomHook(s) | defaults to NORMAL (50) | + +----------------------+-------------------------+ + + If custom hooks have same priority with default hooks, custom hooks + will be triggered after default hooks. + """ + if checkpoint_config is not None: + checkpoint_config.setdefault('by_epoch', False) + if lr_config is not None: + lr_config.setdefault('by_epoch', False) + if log_config is not None: + for info in log_config['hooks']: + info.setdefault('by_epoch', False) + super(IterBasedRunner, self).register_training_hooks( + lr_config=lr_config, + momentum_config=momentum_config, + optimizer_config=optimizer_config, + checkpoint_config=checkpoint_config, + log_config=log_config, + timer_config=IterTimerHook(), + custom_hooks_config=custom_hooks_config) diff --git a/annotator/mmpkg/mmcv/runner/log_buffer.py b/annotator/mmpkg/mmcv/runner/log_buffer.py new file mode 100644 index 0000000000000000000000000000000000000000..d949e2941c5400088c7cd8a1dc893d8b233ae785 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/log_buffer.py @@ -0,0 +1,41 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from collections import OrderedDict + +import numpy as np + + +class LogBuffer: + + def __init__(self): + self.val_history = OrderedDict() + self.n_history = OrderedDict() + self.output = OrderedDict() + self.ready = False + + def clear(self): + self.val_history.clear() + self.n_history.clear() + self.clear_output() + + def clear_output(self): + self.output.clear() + self.ready = False + + def update(self, vars, count=1): + assert isinstance(vars, dict) + for key, var in vars.items(): + if key not in self.val_history: + self.val_history[key] = [] + self.n_history[key] = [] + self.val_history[key].append(var) + self.n_history[key].append(count) + + def average(self, n=0): + """Average latest n values or all values.""" + assert n >= 0 + for key in self.val_history: + values = np.array(self.val_history[key][-n:]) + nums = np.array(self.n_history[key][-n:]) + avg = np.sum(values * nums) / np.sum(nums) + self.output[key] = avg + self.ready = True diff --git a/annotator/mmpkg/mmcv/runner/optimizer/__init__.py b/annotator/mmpkg/mmcv/runner/optimizer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..53c34d0470992cbc374f29681fdd00dc0e57968d --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/optimizer/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .builder import (OPTIMIZER_BUILDERS, OPTIMIZERS, build_optimizer, + build_optimizer_constructor) +from .default_constructor import DefaultOptimizerConstructor + +__all__ = [ + 'OPTIMIZER_BUILDERS', 'OPTIMIZERS', 'DefaultOptimizerConstructor', + 'build_optimizer', 'build_optimizer_constructor' +] diff --git a/annotator/mmpkg/mmcv/runner/optimizer/builder.py b/annotator/mmpkg/mmcv/runner/optimizer/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..f9234eed8f1f186d9d8dfda34562157ee39bdb3a --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/optimizer/builder.py @@ -0,0 +1,44 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy +import inspect + +import torch + +from ...utils import Registry, build_from_cfg + +OPTIMIZERS = Registry('optimizer') +OPTIMIZER_BUILDERS = Registry('optimizer builder') + + +def register_torch_optimizers(): + torch_optimizers = [] + for module_name in dir(torch.optim): + if module_name.startswith('__'): + continue + _optim = getattr(torch.optim, module_name) + if inspect.isclass(_optim) and issubclass(_optim, + torch.optim.Optimizer): + OPTIMIZERS.register_module()(_optim) + torch_optimizers.append(module_name) + return torch_optimizers + + +TORCH_OPTIMIZERS = register_torch_optimizers() + + +def build_optimizer_constructor(cfg): + return build_from_cfg(cfg, OPTIMIZER_BUILDERS) + + +def build_optimizer(model, cfg): + optimizer_cfg = copy.deepcopy(cfg) + constructor_type = optimizer_cfg.pop('constructor', + 'DefaultOptimizerConstructor') + paramwise_cfg = optimizer_cfg.pop('paramwise_cfg', None) + optim_constructor = build_optimizer_constructor( + dict( + type=constructor_type, + optimizer_cfg=optimizer_cfg, + paramwise_cfg=paramwise_cfg)) + optimizer = optim_constructor(model) + return optimizer diff --git a/annotator/mmpkg/mmcv/runner/optimizer/default_constructor.py b/annotator/mmpkg/mmcv/runner/optimizer/default_constructor.py new file mode 100644 index 0000000000000000000000000000000000000000..de2ae39cb6378cc17c098f5324f5d5c321879b91 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/optimizer/default_constructor.py @@ -0,0 +1,249 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import warnings + +import torch +from torch.nn import GroupNorm, LayerNorm + +from annotator.mmpkg.mmcv.utils import _BatchNorm, _InstanceNorm, build_from_cfg, is_list_of +from annotator.mmpkg.mmcv.utils.ext_loader import check_ops_exist +from .builder import OPTIMIZER_BUILDERS, OPTIMIZERS + + +@OPTIMIZER_BUILDERS.register_module() +class DefaultOptimizerConstructor: + """Default constructor for optimizers. + + By default each parameter share the same optimizer settings, and we + provide an argument ``paramwise_cfg`` to specify parameter-wise settings. + It is a dict and may contain the following fields: + + - ``custom_keys`` (dict): Specified parameters-wise settings by keys. If + one of the keys in ``custom_keys`` is a substring of the name of one + parameter, then the setting of the parameter will be specified by + ``custom_keys[key]`` and other setting like ``bias_lr_mult`` etc. will + be ignored. It should be noted that the aforementioned ``key`` is the + longest key that is a substring of the name of the parameter. If there + are multiple matched keys with the same length, then the key with lower + alphabet order will be chosen. + ``custom_keys[key]`` should be a dict and may contain fields ``lr_mult`` + and ``decay_mult``. See Example 2 below. + - ``bias_lr_mult`` (float): It will be multiplied to the learning + rate for all bias parameters (except for those in normalization + layers and offset layers of DCN). + - ``bias_decay_mult`` (float): It will be multiplied to the weight + decay for all bias parameters (except for those in + normalization layers, depthwise conv layers, offset layers of DCN). + - ``norm_decay_mult`` (float): It will be multiplied to the weight + decay for all weight and bias parameters of normalization + layers. + - ``dwconv_decay_mult`` (float): It will be multiplied to the weight + decay for all weight and bias parameters of depthwise conv + layers. + - ``dcn_offset_lr_mult`` (float): It will be multiplied to the learning + rate for parameters of offset layer in the deformable convs + of a model. + - ``bypass_duplicate`` (bool): If true, the duplicate parameters + would not be added into optimizer. Default: False. + + Note: + 1. If the option ``dcn_offset_lr_mult`` is used, the constructor will + override the effect of ``bias_lr_mult`` in the bias of offset + layer. So be careful when using both ``bias_lr_mult`` and + ``dcn_offset_lr_mult``. If you wish to apply both of them to the + offset layer in deformable convs, set ``dcn_offset_lr_mult`` + to the original ``dcn_offset_lr_mult`` * ``bias_lr_mult``. + 2. If the option ``dcn_offset_lr_mult`` is used, the constructor will + apply it to all the DCN layers in the model. So be careful when + the model contains multiple DCN layers in places other than + backbone. + + Args: + model (:obj:`nn.Module`): The model with parameters to be optimized. + optimizer_cfg (dict): The config dict of the optimizer. + Positional fields are + + - `type`: class name of the optimizer. + + Optional fields are + + - any arguments of the corresponding optimizer type, e.g., + lr, weight_decay, momentum, etc. + paramwise_cfg (dict, optional): Parameter-wise options. + + Example 1: + >>> model = torch.nn.modules.Conv1d(1, 1, 1) + >>> optimizer_cfg = dict(type='SGD', lr=0.01, momentum=0.9, + >>> weight_decay=0.0001) + >>> paramwise_cfg = dict(norm_decay_mult=0.) + >>> optim_builder = DefaultOptimizerConstructor( + >>> optimizer_cfg, paramwise_cfg) + >>> optimizer = optim_builder(model) + + Example 2: + >>> # assume model have attribute model.backbone and model.cls_head + >>> optimizer_cfg = dict(type='SGD', lr=0.01, weight_decay=0.95) + >>> paramwise_cfg = dict(custom_keys={ + '.backbone': dict(lr_mult=0.1, decay_mult=0.9)}) + >>> optim_builder = DefaultOptimizerConstructor( + >>> optimizer_cfg, paramwise_cfg) + >>> optimizer = optim_builder(model) + >>> # Then the `lr` and `weight_decay` for model.backbone is + >>> # (0.01 * 0.1, 0.95 * 0.9). `lr` and `weight_decay` for + >>> # model.cls_head is (0.01, 0.95). + """ + + def __init__(self, optimizer_cfg, paramwise_cfg=None): + if not isinstance(optimizer_cfg, dict): + raise TypeError('optimizer_cfg should be a dict', + f'but got {type(optimizer_cfg)}') + self.optimizer_cfg = optimizer_cfg + self.paramwise_cfg = {} if paramwise_cfg is None else paramwise_cfg + self.base_lr = optimizer_cfg.get('lr', None) + self.base_wd = optimizer_cfg.get('weight_decay', None) + self._validate_cfg() + + def _validate_cfg(self): + if not isinstance(self.paramwise_cfg, dict): + raise TypeError('paramwise_cfg should be None or a dict, ' + f'but got {type(self.paramwise_cfg)}') + + if 'custom_keys' in self.paramwise_cfg: + if not isinstance(self.paramwise_cfg['custom_keys'], dict): + raise TypeError( + 'If specified, custom_keys must be a dict, ' + f'but got {type(self.paramwise_cfg["custom_keys"])}') + if self.base_wd is None: + for key in self.paramwise_cfg['custom_keys']: + if 'decay_mult' in self.paramwise_cfg['custom_keys'][key]: + raise ValueError('base_wd should not be None') + + # get base lr and weight decay + # weight_decay must be explicitly specified if mult is specified + if ('bias_decay_mult' in self.paramwise_cfg + or 'norm_decay_mult' in self.paramwise_cfg + or 'dwconv_decay_mult' in self.paramwise_cfg): + if self.base_wd is None: + raise ValueError('base_wd should not be None') + + def _is_in(self, param_group, param_group_list): + assert is_list_of(param_group_list, dict) + param = set(param_group['params']) + param_set = set() + for group in param_group_list: + param_set.update(set(group['params'])) + + return not param.isdisjoint(param_set) + + def add_params(self, params, module, prefix='', is_dcn_module=None): + """Add all parameters of module to the params list. + + The parameters of the given module will be added to the list of param + groups, with specific rules defined by paramwise_cfg. + + Args: + params (list[dict]): A list of param groups, it will be modified + in place. + module (nn.Module): The module to be added. + prefix (str): The prefix of the module + is_dcn_module (int|float|None): If the current module is a + submodule of DCN, `is_dcn_module` will be passed to + control conv_offset layer's learning rate. Defaults to None. + """ + # get param-wise options + custom_keys = self.paramwise_cfg.get('custom_keys', {}) + # first sort with alphabet order and then sort with reversed len of str + sorted_keys = sorted(sorted(custom_keys.keys()), key=len, reverse=True) + + bias_lr_mult = self.paramwise_cfg.get('bias_lr_mult', 1.) + bias_decay_mult = self.paramwise_cfg.get('bias_decay_mult', 1.) + norm_decay_mult = self.paramwise_cfg.get('norm_decay_mult', 1.) + dwconv_decay_mult = self.paramwise_cfg.get('dwconv_decay_mult', 1.) + bypass_duplicate = self.paramwise_cfg.get('bypass_duplicate', False) + dcn_offset_lr_mult = self.paramwise_cfg.get('dcn_offset_lr_mult', 1.) + + # special rules for norm layers and depth-wise conv layers + is_norm = isinstance(module, + (_BatchNorm, _InstanceNorm, GroupNorm, LayerNorm)) + is_dwconv = ( + isinstance(module, torch.nn.Conv2d) + and module.in_channels == module.groups) + + for name, param in module.named_parameters(recurse=False): + param_group = {'params': [param]} + if not param.requires_grad: + params.append(param_group) + continue + if bypass_duplicate and self._is_in(param_group, params): + warnings.warn(f'{prefix} is duplicate. It is skipped since ' + f'bypass_duplicate={bypass_duplicate}') + continue + # if the parameter match one of the custom keys, ignore other rules + is_custom = False + for key in sorted_keys: + if key in f'{prefix}.{name}': + is_custom = True + lr_mult = custom_keys[key].get('lr_mult', 1.) + param_group['lr'] = self.base_lr * lr_mult + if self.base_wd is not None: + decay_mult = custom_keys[key].get('decay_mult', 1.) + param_group['weight_decay'] = self.base_wd * decay_mult + break + + if not is_custom: + # bias_lr_mult affects all bias parameters + # except for norm.bias dcn.conv_offset.bias + if name == 'bias' and not (is_norm or is_dcn_module): + param_group['lr'] = self.base_lr * bias_lr_mult + + if (prefix.find('conv_offset') != -1 and is_dcn_module + and isinstance(module, torch.nn.Conv2d)): + # deal with both dcn_offset's bias & weight + param_group['lr'] = self.base_lr * dcn_offset_lr_mult + + # apply weight decay policies + if self.base_wd is not None: + # norm decay + if is_norm: + param_group[ + 'weight_decay'] = self.base_wd * norm_decay_mult + # depth-wise conv + elif is_dwconv: + param_group[ + 'weight_decay'] = self.base_wd * dwconv_decay_mult + # bias lr and decay + elif name == 'bias' and not is_dcn_module: + # TODO: current bias_decay_mult will have affect on DCN + param_group[ + 'weight_decay'] = self.base_wd * bias_decay_mult + params.append(param_group) + + if check_ops_exist(): + from annotator.mmpkg.mmcv.ops import DeformConv2d, ModulatedDeformConv2d + is_dcn_module = isinstance(module, + (DeformConv2d, ModulatedDeformConv2d)) + else: + is_dcn_module = False + for child_name, child_mod in module.named_children(): + child_prefix = f'{prefix}.{child_name}' if prefix else child_name + self.add_params( + params, + child_mod, + prefix=child_prefix, + is_dcn_module=is_dcn_module) + + def __call__(self, model): + if hasattr(model, 'module'): + model = model.module + + optimizer_cfg = self.optimizer_cfg.copy() + # if no paramwise option is specified, just use the global setting + if not self.paramwise_cfg: + optimizer_cfg['params'] = model.parameters() + return build_from_cfg(optimizer_cfg, OPTIMIZERS) + + # set param-wise lr and weight decay recursively + params = [] + self.add_params(params, model) + optimizer_cfg['params'] = params + + return build_from_cfg(optimizer_cfg, OPTIMIZERS) diff --git a/annotator/mmpkg/mmcv/runner/priority.py b/annotator/mmpkg/mmcv/runner/priority.py new file mode 100644 index 0000000000000000000000000000000000000000..64cc4e3a05f8d5b89ab6eb32461e6e80f1d62e67 --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/priority.py @@ -0,0 +1,60 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from enum import Enum + + +class Priority(Enum): + """Hook priority levels. + + +--------------+------------+ + | Level | Value | + +==============+============+ + | HIGHEST | 0 | + +--------------+------------+ + | VERY_HIGH | 10 | + +--------------+------------+ + | HIGH | 30 | + +--------------+------------+ + | ABOVE_NORMAL | 40 | + +--------------+------------+ + | NORMAL | 50 | + +--------------+------------+ + | BELOW_NORMAL | 60 | + +--------------+------------+ + | LOW | 70 | + +--------------+------------+ + | VERY_LOW | 90 | + +--------------+------------+ + | LOWEST | 100 | + +--------------+------------+ + """ + + HIGHEST = 0 + VERY_HIGH = 10 + HIGH = 30 + ABOVE_NORMAL = 40 + NORMAL = 50 + BELOW_NORMAL = 60 + LOW = 70 + VERY_LOW = 90 + LOWEST = 100 + + +def get_priority(priority): + """Get priority value. + + Args: + priority (int or str or :obj:`Priority`): Priority. + + Returns: + int: The priority value. + """ + if isinstance(priority, int): + if priority < 0 or priority > 100: + raise ValueError('priority must be between 0 and 100') + return priority + elif isinstance(priority, Priority): + return priority.value + elif isinstance(priority, str): + return Priority[priority.upper()].value + else: + raise TypeError('priority must be an integer or Priority enum value') diff --git a/annotator/mmpkg/mmcv/runner/utils.py b/annotator/mmpkg/mmcv/runner/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..11bbc523e9a009119531c5eb903a93fe40cc5bca --- /dev/null +++ b/annotator/mmpkg/mmcv/runner/utils.py @@ -0,0 +1,93 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os +import random +import sys +import time +import warnings +from getpass import getuser +from socket import gethostname + +import numpy as np +import torch + +import annotator.mmpkg.mmcv as mmcv + + +def get_host_info(): + """Get hostname and username. + + Return empty string if exception raised, e.g. ``getpass.getuser()`` will + lead to error in docker container + """ + host = '' + try: + host = f'{getuser()}@{gethostname()}' + except Exception as e: + warnings.warn(f'Host or user not found: {str(e)}') + finally: + return host + + +def get_time_str(): + return time.strftime('%Y%m%d_%H%M%S', time.localtime()) + + +def obj_from_dict(info, parent=None, default_args=None): + """Initialize an object from dict. + + The dict must contain the key "type", which indicates the object type, it + can be either a string or type, such as "list" or ``list``. Remaining + fields are treated as the arguments for constructing the object. + + Args: + info (dict): Object types and arguments. + parent (:class:`module`): Module which may containing expected object + classes. + default_args (dict, optional): Default arguments for initializing the + object. + + Returns: + any type: Object built from the dict. + """ + assert isinstance(info, dict) and 'type' in info + assert isinstance(default_args, dict) or default_args is None + args = info.copy() + obj_type = args.pop('type') + if mmcv.is_str(obj_type): + if parent is not None: + obj_type = getattr(parent, obj_type) + else: + obj_type = sys.modules[obj_type] + elif not isinstance(obj_type, type): + raise TypeError('type must be a str or valid type, but ' + f'got {type(obj_type)}') + if default_args is not None: + for name, value in default_args.items(): + args.setdefault(name, value) + return obj_type(**args) + + +def set_random_seed(seed, deterministic=False, use_rank_shift=False): + """Set random seed. + + Args: + seed (int): Seed to be used. + deterministic (bool): Whether to set the deterministic option for + CUDNN backend, i.e., set `torch.backends.cudnn.deterministic` + to True and `torch.backends.cudnn.benchmark` to False. + Default: False. + rank_shift (bool): Whether to add rank number to the random seed to + have different random seed in different threads. Default: False. + """ + if use_rank_shift: + rank, _ = mmcv.runner.get_dist_info() + seed += rank + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + os.environ['PYTHONHASHSEED'] = str(seed) + if deterministic: + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False diff --git a/annotator/mmpkg/mmcv/utils/__init__.py b/annotator/mmpkg/mmcv/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..378a0068432a371af364de9d73785901c0f83383 --- /dev/null +++ b/annotator/mmpkg/mmcv/utils/__init__.py @@ -0,0 +1,69 @@ +# flake8: noqa +# Copyright (c) OpenMMLab. All rights reserved. +from .config import Config, ConfigDict, DictAction +from .misc import (check_prerequisites, concat_list, deprecated_api_warning, + has_method, import_modules_from_strings, is_list_of, + is_method_overridden, is_seq_of, is_str, is_tuple_of, + iter_cast, list_cast, requires_executable, requires_package, + slice_list, to_1tuple, to_2tuple, to_3tuple, to_4tuple, + to_ntuple, tuple_cast) +from .path import (check_file_exist, fopen, is_filepath, mkdir_or_exist, + scandir, symlink) +from .progressbar import (ProgressBar, track_iter_progress, + track_parallel_progress, track_progress) +from .testing import (assert_attrs_equal, assert_dict_contains_subset, + assert_dict_has_keys, assert_is_norm_layer, + assert_keys_equal, assert_params_all_zeros, + check_python_script) +from .timer import Timer, TimerError, check_time +from .version_utils import digit_version, get_git_hash + +try: + import torch +except ImportError: + __all__ = [ + 'Config', 'ConfigDict', 'DictAction', 'is_str', 'iter_cast', + 'list_cast', 'tuple_cast', 'is_seq_of', 'is_list_of', 'is_tuple_of', + 'slice_list', 'concat_list', 'check_prerequisites', 'requires_package', + 'requires_executable', 'is_filepath', 'fopen', 'check_file_exist', + 'mkdir_or_exist', 'symlink', 'scandir', 'ProgressBar', + 'track_progress', 'track_iter_progress', 'track_parallel_progress', + 'Timer', 'TimerError', 'check_time', 'deprecated_api_warning', + 'digit_version', 'get_git_hash', 'import_modules_from_strings', + 'assert_dict_contains_subset', 'assert_attrs_equal', + 'assert_dict_has_keys', 'assert_keys_equal', 'check_python_script', + 'to_1tuple', 'to_2tuple', 'to_3tuple', 'to_4tuple', 'to_ntuple', + 'is_method_overridden', 'has_method' + ] +else: + from .env import collect_env + from .logging import get_logger, print_log + from .parrots_jit import jit, skip_no_elena + from .parrots_wrapper import ( + TORCH_VERSION, BuildExtension, CppExtension, CUDAExtension, DataLoader, + PoolDataLoader, SyncBatchNorm, _AdaptiveAvgPoolNd, _AdaptiveMaxPoolNd, + _AvgPoolNd, _BatchNorm, _ConvNd, _ConvTransposeMixin, _InstanceNorm, + _MaxPoolNd, get_build_config, is_rocm_pytorch, _get_cuda_home) + from .registry import Registry, build_from_cfg + from .trace import is_jit_tracing + __all__ = [ + 'Config', 'ConfigDict', 'DictAction', 'collect_env', 'get_logger', + 'print_log', 'is_str', 'iter_cast', 'list_cast', 'tuple_cast', + 'is_seq_of', 'is_list_of', 'is_tuple_of', 'slice_list', 'concat_list', + 'check_prerequisites', 'requires_package', 'requires_executable', + 'is_filepath', 'fopen', 'check_file_exist', 'mkdir_or_exist', + 'symlink', 'scandir', 'ProgressBar', 'track_progress', + 'track_iter_progress', 'track_parallel_progress', 'Registry', + 'build_from_cfg', 'Timer', 'TimerError', 'check_time', 'SyncBatchNorm', + '_AdaptiveAvgPoolNd', '_AdaptiveMaxPoolNd', '_AvgPoolNd', '_BatchNorm', + '_ConvNd', '_ConvTransposeMixin', '_InstanceNorm', '_MaxPoolNd', + 'get_build_config', 'BuildExtension', 'CppExtension', 'CUDAExtension', + 'DataLoader', 'PoolDataLoader', 'TORCH_VERSION', + 'deprecated_api_warning', 'digit_version', 'get_git_hash', + 'import_modules_from_strings', 'jit', 'skip_no_elena', + 'assert_dict_contains_subset', 'assert_attrs_equal', + 'assert_dict_has_keys', 'assert_keys_equal', 'assert_is_norm_layer', + 'assert_params_all_zeros', 'check_python_script', + 'is_method_overridden', 'is_jit_tracing', 'is_rocm_pytorch', + '_get_cuda_home', 'has_method' + ] diff --git a/annotator/mmpkg/mmcv/utils/config.py b/annotator/mmpkg/mmcv/utils/config.py new file mode 100644 index 0000000000000000000000000000000000000000..e2f7551f95cbf5d8ffa225bba7325632b5e7f01b --- /dev/null +++ b/annotator/mmpkg/mmcv/utils/config.py @@ -0,0 +1,688 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import ast +import copy +import os +import os.path as osp +import platform +import shutil +import sys +import tempfile +import uuid +import warnings +from argparse import Action, ArgumentParser +from collections import abc +from importlib import import_module + +from addict import Dict +from yapf.yapflib.yapf_api import FormatCode + +from .misc import import_modules_from_strings +from .path import check_file_exist + +if platform.system() == 'Windows': + import regex as re +else: + import re + +BASE_KEY = '_base_' +DELETE_KEY = '_delete_' +DEPRECATION_KEY = '_deprecation_' +RESERVED_KEYS = ['filename', 'text', 'pretty_text'] + + +class ConfigDict(Dict): + + def __missing__(self, name): + raise KeyError(name) + + def __getattr__(self, name): + try: + value = super(ConfigDict, self).__getattr__(name) + except KeyError: + ex = AttributeError(f"'{self.__class__.__name__}' object has no " + f"attribute '{name}'") + except Exception as e: + ex = e + else: + return value + raise ex + + +def add_args(parser, cfg, prefix=''): + for k, v in cfg.items(): + if isinstance(v, str): + parser.add_argument('--' + prefix + k) + elif isinstance(v, int): + parser.add_argument('--' + prefix + k, type=int) + elif isinstance(v, float): + parser.add_argument('--' + prefix + k, type=float) + elif isinstance(v, bool): + parser.add_argument('--' + prefix + k, action='store_true') + elif isinstance(v, dict): + add_args(parser, v, prefix + k + '.') + elif isinstance(v, abc.Iterable): + parser.add_argument('--' + prefix + k, type=type(v[0]), nargs='+') + else: + print(f'cannot parse key {prefix + k} of type {type(v)}') + return parser + + +class Config: + """A facility for config and config files. + + It supports common file formats as configs: python/json/yaml. The interface + is the same as a dict object and also allows access config values as + attributes. + + Example: + >>> cfg = Config(dict(a=1, b=dict(b1=[0, 1]))) + >>> cfg.a + 1 + >>> cfg.b + {'b1': [0, 1]} + >>> cfg.b.b1 + [0, 1] + >>> cfg = Config.fromfile('tests/data/config/a.py') + >>> cfg.filename + "/home/kchen/projects/mmcv/tests/data/config/a.py" + >>> cfg.item4 + 'test' + >>> cfg + "Config [path: /home/kchen/projects/mmcv/tests/data/config/a.py]: " + "{'item1': [1, 2], 'item2': {'a': 0}, 'item3': True, 'item4': 'test'}" + """ + + @staticmethod + def _validate_py_syntax(filename): + with open(filename, 'r', encoding='utf-8') as f: + # Setting encoding explicitly to resolve coding issue on windows + content = f.read() + try: + ast.parse(content) + except SyntaxError as e: + raise SyntaxError('There are syntax errors in config ' + f'file {filename}: {e}') + + @staticmethod + def _substitute_predefined_vars(filename, temp_config_name): + file_dirname = osp.dirname(filename) + file_basename = osp.basename(filename) + file_basename_no_extension = osp.splitext(file_basename)[0] + file_extname = osp.splitext(filename)[1] + support_templates = dict( + fileDirname=file_dirname, + fileBasename=file_basename, + fileBasenameNoExtension=file_basename_no_extension, + fileExtname=file_extname) + with open(filename, 'r', encoding='utf-8') as f: + # Setting encoding explicitly to resolve coding issue on windows + config_file = f.read() + for key, value in support_templates.items(): + regexp = r'\{\{\s*' + str(key) + r'\s*\}\}' + value = value.replace('\\', '/') + config_file = re.sub(regexp, value, config_file) + with open(temp_config_name, 'w', encoding='utf-8') as tmp_config_file: + tmp_config_file.write(config_file) + + @staticmethod + def _pre_substitute_base_vars(filename, temp_config_name): + """Substitute base variable placehoders to string, so that parsing + would work.""" + with open(filename, 'r', encoding='utf-8') as f: + # Setting encoding explicitly to resolve coding issue on windows + config_file = f.read() + base_var_dict = {} + regexp = r'\{\{\s*' + BASE_KEY + r'\.([\w\.]+)\s*\}\}' + base_vars = set(re.findall(regexp, config_file)) + for base_var in base_vars: + randstr = f'_{base_var}_{uuid.uuid4().hex.lower()[:6]}' + base_var_dict[randstr] = base_var + regexp = r'\{\{\s*' + BASE_KEY + r'\.' + base_var + r'\s*\}\}' + config_file = re.sub(regexp, f'"{randstr}"', config_file) + with open(temp_config_name, 'w', encoding='utf-8') as tmp_config_file: + tmp_config_file.write(config_file) + return base_var_dict + + @staticmethod + def _substitute_base_vars(cfg, base_var_dict, base_cfg): + """Substitute variable strings to their actual values.""" + cfg = copy.deepcopy(cfg) + + if isinstance(cfg, dict): + for k, v in cfg.items(): + if isinstance(v, str) and v in base_var_dict: + new_v = base_cfg + for new_k in base_var_dict[v].split('.'): + new_v = new_v[new_k] + cfg[k] = new_v + elif isinstance(v, (list, tuple, dict)): + cfg[k] = Config._substitute_base_vars( + v, base_var_dict, base_cfg) + elif isinstance(cfg, tuple): + cfg = tuple( + Config._substitute_base_vars(c, base_var_dict, base_cfg) + for c in cfg) + elif isinstance(cfg, list): + cfg = [ + Config._substitute_base_vars(c, base_var_dict, base_cfg) + for c in cfg + ] + elif isinstance(cfg, str) and cfg in base_var_dict: + new_v = base_cfg + for new_k in base_var_dict[cfg].split('.'): + new_v = new_v[new_k] + cfg = new_v + + return cfg + + @staticmethod + def _file2dict(filename, use_predefined_variables=True): + filename = osp.abspath(osp.expanduser(filename)) + check_file_exist(filename) + fileExtname = osp.splitext(filename)[1] + if fileExtname not in ['.py', '.json', '.yaml', '.yml']: + raise IOError('Only py/yml/yaml/json type are supported now!') + + with tempfile.TemporaryDirectory() as temp_config_dir: + temp_config_file = tempfile.NamedTemporaryFile( + dir=temp_config_dir, suffix=fileExtname) + if platform.system() == 'Windows': + temp_config_file.close() + temp_config_name = osp.basename(temp_config_file.name) + # Substitute predefined variables + if use_predefined_variables: + Config._substitute_predefined_vars(filename, + temp_config_file.name) + else: + shutil.copyfile(filename, temp_config_file.name) + # Substitute base variables from placeholders to strings + base_var_dict = Config._pre_substitute_base_vars( + temp_config_file.name, temp_config_file.name) + + if filename.endswith('.py'): + temp_module_name = osp.splitext(temp_config_name)[0] + sys.path.insert(0, temp_config_dir) + Config._validate_py_syntax(filename) + mod = import_module(temp_module_name) + sys.path.pop(0) + cfg_dict = { + name: value + for name, value in mod.__dict__.items() + if not name.startswith('__') + } + # delete imported module + del sys.modules[temp_module_name] + elif filename.endswith(('.yml', '.yaml', '.json')): + import annotator.mmpkg.mmcv as mmcv + cfg_dict = mmcv.load(temp_config_file.name) + # close temp file + temp_config_file.close() + + # check deprecation information + if DEPRECATION_KEY in cfg_dict: + deprecation_info = cfg_dict.pop(DEPRECATION_KEY) + warning_msg = f'The config file {filename} will be deprecated ' \ + 'in the future.' + if 'expected' in deprecation_info: + warning_msg += f' Please use {deprecation_info["expected"]} ' \ + 'instead.' + if 'reference' in deprecation_info: + warning_msg += ' More information can be found at ' \ + f'{deprecation_info["reference"]}' + warnings.warn(warning_msg) + + cfg_text = filename + '\n' + with open(filename, 'r', encoding='utf-8') as f: + # Setting encoding explicitly to resolve coding issue on windows + cfg_text += f.read() + + if BASE_KEY in cfg_dict: + cfg_dir = osp.dirname(filename) + base_filename = cfg_dict.pop(BASE_KEY) + base_filename = base_filename if isinstance( + base_filename, list) else [base_filename] + + cfg_dict_list = list() + cfg_text_list = list() + for f in base_filename: + _cfg_dict, _cfg_text = Config._file2dict(osp.join(cfg_dir, f)) + cfg_dict_list.append(_cfg_dict) + cfg_text_list.append(_cfg_text) + + base_cfg_dict = dict() + for c in cfg_dict_list: + duplicate_keys = base_cfg_dict.keys() & c.keys() + if len(duplicate_keys) > 0: + raise KeyError('Duplicate key is not allowed among bases. ' + f'Duplicate keys: {duplicate_keys}') + base_cfg_dict.update(c) + + # Substitute base variables from strings to their actual values + cfg_dict = Config._substitute_base_vars(cfg_dict, base_var_dict, + base_cfg_dict) + + base_cfg_dict = Config._merge_a_into_b(cfg_dict, base_cfg_dict) + cfg_dict = base_cfg_dict + + # merge cfg_text + cfg_text_list.append(cfg_text) + cfg_text = '\n'.join(cfg_text_list) + + return cfg_dict, cfg_text + + @staticmethod + def _merge_a_into_b(a, b, allow_list_keys=False): + """merge dict ``a`` into dict ``b`` (non-inplace). + + Values in ``a`` will overwrite ``b``. ``b`` is copied first to avoid + in-place modifications. + + Args: + a (dict): The source dict to be merged into ``b``. + b (dict): The origin dict to be fetch keys from ``a``. + allow_list_keys (bool): If True, int string keys (e.g. '0', '1') + are allowed in source ``a`` and will replace the element of the + corresponding index in b if b is a list. Default: False. + + Returns: + dict: The modified dict of ``b`` using ``a``. + + Examples: + # Normally merge a into b. + >>> Config._merge_a_into_b( + ... dict(obj=dict(a=2)), dict(obj=dict(a=1))) + {'obj': {'a': 2}} + + # Delete b first and merge a into b. + >>> Config._merge_a_into_b( + ... dict(obj=dict(_delete_=True, a=2)), dict(obj=dict(a=1))) + {'obj': {'a': 2}} + + # b is a list + >>> Config._merge_a_into_b( + ... {'0': dict(a=2)}, [dict(a=1), dict(b=2)], True) + [{'a': 2}, {'b': 2}] + """ + b = b.copy() + for k, v in a.items(): + if allow_list_keys and k.isdigit() and isinstance(b, list): + k = int(k) + if len(b) <= k: + raise KeyError(f'Index {k} exceeds the length of list {b}') + b[k] = Config._merge_a_into_b(v, b[k], allow_list_keys) + elif isinstance(v, + dict) and k in b and not v.pop(DELETE_KEY, False): + allowed_types = (dict, list) if allow_list_keys else dict + if not isinstance(b[k], allowed_types): + raise TypeError( + f'{k}={v} in child config cannot inherit from base ' + f'because {k} is a dict in the child config but is of ' + f'type {type(b[k])} in base config. You may set ' + f'`{DELETE_KEY}=True` to ignore the base config') + b[k] = Config._merge_a_into_b(v, b[k], allow_list_keys) + else: + b[k] = v + return b + + @staticmethod + def fromfile(filename, + use_predefined_variables=True, + import_custom_modules=True): + cfg_dict, cfg_text = Config._file2dict(filename, + use_predefined_variables) + if import_custom_modules and cfg_dict.get('custom_imports', None): + import_modules_from_strings(**cfg_dict['custom_imports']) + return Config(cfg_dict, cfg_text=cfg_text, filename=filename) + + @staticmethod + def fromstring(cfg_str, file_format): + """Generate config from config str. + + Args: + cfg_str (str): Config str. + file_format (str): Config file format corresponding to the + config str. Only py/yml/yaml/json type are supported now! + + Returns: + obj:`Config`: Config obj. + """ + if file_format not in ['.py', '.json', '.yaml', '.yml']: + raise IOError('Only py/yml/yaml/json type are supported now!') + if file_format != '.py' and 'dict(' in cfg_str: + # check if users specify a wrong suffix for python + warnings.warn( + 'Please check "file_format", the file format may be .py') + with tempfile.NamedTemporaryFile( + 'w', encoding='utf-8', suffix=file_format, + delete=False) as temp_file: + temp_file.write(cfg_str) + # on windows, previous implementation cause error + # see PR 1077 for details + cfg = Config.fromfile(temp_file.name) + os.remove(temp_file.name) + return cfg + + @staticmethod + def auto_argparser(description=None): + """Generate argparser from config file automatically (experimental)""" + partial_parser = ArgumentParser(description=description) + partial_parser.add_argument('config', help='config file path') + cfg_file = partial_parser.parse_known_args()[0].config + cfg = Config.fromfile(cfg_file) + parser = ArgumentParser(description=description) + parser.add_argument('config', help='config file path') + add_args(parser, cfg) + return parser, cfg + + def __init__(self, cfg_dict=None, cfg_text=None, filename=None): + if cfg_dict is None: + cfg_dict = dict() + elif not isinstance(cfg_dict, dict): + raise TypeError('cfg_dict must be a dict, but ' + f'got {type(cfg_dict)}') + for key in cfg_dict: + if key in RESERVED_KEYS: + raise KeyError(f'{key} is reserved for config file') + + super(Config, self).__setattr__('_cfg_dict', ConfigDict(cfg_dict)) + super(Config, self).__setattr__('_filename', filename) + if cfg_text: + text = cfg_text + elif filename: + with open(filename, 'r') as f: + text = f.read() + else: + text = '' + super(Config, self).__setattr__('_text', text) + + @property + def filename(self): + return self._filename + + @property + def text(self): + return self._text + + @property + def pretty_text(self): + + indent = 4 + + def _indent(s_, num_spaces): + s = s_.split('\n') + if len(s) == 1: + return s_ + first = s.pop(0) + s = [(num_spaces * ' ') + line for line in s] + s = '\n'.join(s) + s = first + '\n' + s + return s + + def _format_basic_types(k, v, use_mapping=False): + if isinstance(v, str): + v_str = f"'{v}'" + else: + v_str = str(v) + + if use_mapping: + k_str = f"'{k}'" if isinstance(k, str) else str(k) + attr_str = f'{k_str}: {v_str}' + else: + attr_str = f'{str(k)}={v_str}' + attr_str = _indent(attr_str, indent) + + return attr_str + + def _format_list(k, v, use_mapping=False): + # check if all items in the list are dict + if all(isinstance(_, dict) for _ in v): + v_str = '[\n' + v_str += '\n'.join( + f'dict({_indent(_format_dict(v_), indent)}),' + for v_ in v).rstrip(',') + if use_mapping: + k_str = f"'{k}'" if isinstance(k, str) else str(k) + attr_str = f'{k_str}: {v_str}' + else: + attr_str = f'{str(k)}={v_str}' + attr_str = _indent(attr_str, indent) + ']' + else: + attr_str = _format_basic_types(k, v, use_mapping) + return attr_str + + def _contain_invalid_identifier(dict_str): + contain_invalid_identifier = False + for key_name in dict_str: + contain_invalid_identifier |= \ + (not str(key_name).isidentifier()) + return contain_invalid_identifier + + def _format_dict(input_dict, outest_level=False): + r = '' + s = [] + + use_mapping = _contain_invalid_identifier(input_dict) + if use_mapping: + r += '{' + for idx, (k, v) in enumerate(input_dict.items()): + is_last = idx >= len(input_dict) - 1 + end = '' if outest_level or is_last else ',' + if isinstance(v, dict): + v_str = '\n' + _format_dict(v) + if use_mapping: + k_str = f"'{k}'" if isinstance(k, str) else str(k) + attr_str = f'{k_str}: dict({v_str}' + else: + attr_str = f'{str(k)}=dict({v_str}' + attr_str = _indent(attr_str, indent) + ')' + end + elif isinstance(v, list): + attr_str = _format_list(k, v, use_mapping) + end + else: + attr_str = _format_basic_types(k, v, use_mapping) + end + + s.append(attr_str) + r += '\n'.join(s) + if use_mapping: + r += '}' + return r + + cfg_dict = self._cfg_dict.to_dict() + text = _format_dict(cfg_dict, outest_level=True) + # copied from setup.cfg + yapf_style = dict( + based_on_style='pep8', + blank_line_before_nested_class_or_def=True, + split_before_expression_after_opening_paren=True) + text, _ = FormatCode(text, style_config=yapf_style, verify=True) + + return text + + def __repr__(self): + return f'Config (path: {self.filename}): {self._cfg_dict.__repr__()}' + + def __len__(self): + return len(self._cfg_dict) + + def __getattr__(self, name): + return getattr(self._cfg_dict, name) + + def __getitem__(self, name): + return self._cfg_dict.__getitem__(name) + + def __setattr__(self, name, value): + if isinstance(value, dict): + value = ConfigDict(value) + self._cfg_dict.__setattr__(name, value) + + def __setitem__(self, name, value): + if isinstance(value, dict): + value = ConfigDict(value) + self._cfg_dict.__setitem__(name, value) + + def __iter__(self): + return iter(self._cfg_dict) + + def __getstate__(self): + return (self._cfg_dict, self._filename, self._text) + + def __setstate__(self, state): + _cfg_dict, _filename, _text = state + super(Config, self).__setattr__('_cfg_dict', _cfg_dict) + super(Config, self).__setattr__('_filename', _filename) + super(Config, self).__setattr__('_text', _text) + + def dump(self, file=None): + cfg_dict = super(Config, self).__getattribute__('_cfg_dict').to_dict() + if self.filename.endswith('.py'): + if file is None: + return self.pretty_text + else: + with open(file, 'w', encoding='utf-8') as f: + f.write(self.pretty_text) + else: + import annotator.mmpkg.mmcv as mmcv + if file is None: + file_format = self.filename.split('.')[-1] + return mmcv.dump(cfg_dict, file_format=file_format) + else: + mmcv.dump(cfg_dict, file) + + def merge_from_dict(self, options, allow_list_keys=True): + """Merge list into cfg_dict. + + Merge the dict parsed by MultipleKVAction into this cfg. + + Examples: + >>> options = {'model.backbone.depth': 50, + ... 'model.backbone.with_cp':True} + >>> cfg = Config(dict(model=dict(backbone=dict(type='ResNet')))) + >>> cfg.merge_from_dict(options) + >>> cfg_dict = super(Config, self).__getattribute__('_cfg_dict') + >>> assert cfg_dict == dict( + ... model=dict(backbone=dict(depth=50, with_cp=True))) + + # Merge list element + >>> cfg = Config(dict(pipeline=[ + ... dict(type='LoadImage'), dict(type='LoadAnnotations')])) + >>> options = dict(pipeline={'0': dict(type='SelfLoadImage')}) + >>> cfg.merge_from_dict(options, allow_list_keys=True) + >>> cfg_dict = super(Config, self).__getattribute__('_cfg_dict') + >>> assert cfg_dict == dict(pipeline=[ + ... dict(type='SelfLoadImage'), dict(type='LoadAnnotations')]) + + Args: + options (dict): dict of configs to merge from. + allow_list_keys (bool): If True, int string keys (e.g. '0', '1') + are allowed in ``options`` and will replace the element of the + corresponding index in the config if the config is a list. + Default: True. + """ + option_cfg_dict = {} + for full_key, v in options.items(): + d = option_cfg_dict + key_list = full_key.split('.') + for subkey in key_list[:-1]: + d.setdefault(subkey, ConfigDict()) + d = d[subkey] + subkey = key_list[-1] + d[subkey] = v + + cfg_dict = super(Config, self).__getattribute__('_cfg_dict') + super(Config, self).__setattr__( + '_cfg_dict', + Config._merge_a_into_b( + option_cfg_dict, cfg_dict, allow_list_keys=allow_list_keys)) + + +class DictAction(Action): + """ + argparse action to split an argument into KEY=VALUE form + on the first = and append to a dictionary. List options can + be passed as comma separated values, i.e 'KEY=V1,V2,V3', or with explicit + brackets, i.e. 'KEY=[V1,V2,V3]'. It also support nested brackets to build + list/tuple values. e.g. 'KEY=[(V1,V2),(V3,V4)]' + """ + + @staticmethod + def _parse_int_float_bool(val): + try: + return int(val) + except ValueError: + pass + try: + return float(val) + except ValueError: + pass + if val.lower() in ['true', 'false']: + return True if val.lower() == 'true' else False + return val + + @staticmethod + def _parse_iterable(val): + """Parse iterable values in the string. + + All elements inside '()' or '[]' are treated as iterable values. + + Args: + val (str): Value string. + + Returns: + list | tuple: The expanded list or tuple from the string. + + Examples: + >>> DictAction._parse_iterable('1,2,3') + [1, 2, 3] + >>> DictAction._parse_iterable('[a, b, c]') + ['a', 'b', 'c'] + >>> DictAction._parse_iterable('[(1, 2, 3), [a, b], c]') + [(1, 2, 3), ['a', 'b'], 'c'] + """ + + def find_next_comma(string): + """Find the position of next comma in the string. + + If no ',' is found in the string, return the string length. All + chars inside '()' and '[]' are treated as one element and thus ',' + inside these brackets are ignored. + """ + assert (string.count('(') == string.count(')')) and ( + string.count('[') == string.count(']')), \ + f'Imbalanced brackets exist in {string}' + end = len(string) + for idx, char in enumerate(string): + pre = string[:idx] + # The string before this ',' is balanced + if ((char == ',') and (pre.count('(') == pre.count(')')) + and (pre.count('[') == pre.count(']'))): + end = idx + break + return end + + # Strip ' and " characters and replace whitespace. + val = val.strip('\'\"').replace(' ', '') + is_tuple = False + if val.startswith('(') and val.endswith(')'): + is_tuple = True + val = val[1:-1] + elif val.startswith('[') and val.endswith(']'): + val = val[1:-1] + elif ',' not in val: + # val is a single value + return DictAction._parse_int_float_bool(val) + + values = [] + while len(val) > 0: + comma_idx = find_next_comma(val) + element = DictAction._parse_iterable(val[:comma_idx]) + values.append(element) + val = val[comma_idx + 1:] + if is_tuple: + values = tuple(values) + return values + + def __call__(self, parser, namespace, values, option_string=None): + options = {} + for kv in values: + key, val = kv.split('=', maxsplit=1) + options[key] = self._parse_iterable(val) + setattr(namespace, self.dest, options) diff --git a/annotator/mmpkg/mmcv/utils/env.py b/annotator/mmpkg/mmcv/utils/env.py new file mode 100644 index 0000000000000000000000000000000000000000..a0c6e64a63f8a3ed813b749c134823a0ef69964c --- /dev/null +++ b/annotator/mmpkg/mmcv/utils/env.py @@ -0,0 +1,95 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""This file holding some environment constant for sharing by other files.""" + +import os.path as osp +import subprocess +import sys +from collections import defaultdict + +import cv2 +import torch + +import annotator.mmpkg.mmcv as mmcv +from .parrots_wrapper import get_build_config + + +def collect_env(): + """Collect the information of the running environments. + + Returns: + dict: The environment information. The following fields are contained. + + - sys.platform: The variable of ``sys.platform``. + - Python: Python version. + - CUDA available: Bool, indicating if CUDA is available. + - GPU devices: Device type of each GPU. + - CUDA_HOME (optional): The env var ``CUDA_HOME``. + - NVCC (optional): NVCC version. + - GCC: GCC version, "n/a" if GCC is not installed. + - PyTorch: PyTorch version. + - PyTorch compiling details: The output of \ + ``torch.__config__.show()``. + - TorchVision (optional): TorchVision version. + - OpenCV: OpenCV version. + - MMCV: MMCV version. + - MMCV Compiler: The GCC version for compiling MMCV ops. + - MMCV CUDA Compiler: The CUDA version for compiling MMCV ops. + """ + env_info = {} + env_info['sys.platform'] = sys.platform + env_info['Python'] = sys.version.replace('\n', '') + + cuda_available = torch.cuda.is_available() + env_info['CUDA available'] = cuda_available + + if cuda_available: + devices = defaultdict(list) + for k in range(torch.cuda.device_count()): + devices[torch.cuda.get_device_name(k)].append(str(k)) + for name, device_ids in devices.items(): + env_info['GPU ' + ','.join(device_ids)] = name + + from annotator.mmpkg.mmcv.utils.parrots_wrapper import _get_cuda_home + CUDA_HOME = _get_cuda_home() + env_info['CUDA_HOME'] = CUDA_HOME + + if CUDA_HOME is not None and osp.isdir(CUDA_HOME): + try: + nvcc = osp.join(CUDA_HOME, 'bin/nvcc') + nvcc = subprocess.check_output( + f'"{nvcc}" -V | tail -n1', shell=True) + nvcc = nvcc.decode('utf-8').strip() + except subprocess.SubprocessError: + nvcc = 'Not Available' + env_info['NVCC'] = nvcc + + try: + gcc = subprocess.check_output('gcc --version | head -n1', shell=True) + gcc = gcc.decode('utf-8').strip() + env_info['GCC'] = gcc + except subprocess.CalledProcessError: # gcc is unavailable + env_info['GCC'] = 'n/a' + + env_info['PyTorch'] = torch.__version__ + env_info['PyTorch compiling details'] = get_build_config() + + try: + import torchvision + env_info['TorchVision'] = torchvision.__version__ + except ModuleNotFoundError: + pass + + env_info['OpenCV'] = cv2.__version__ + + env_info['MMCV'] = mmcv.__version__ + + try: + from annotator.mmpkg.mmcv.ops import get_compiler_version, get_compiling_cuda_version + except ModuleNotFoundError: + env_info['MMCV Compiler'] = 'n/a' + env_info['MMCV CUDA Compiler'] = 'n/a' + else: + env_info['MMCV Compiler'] = get_compiler_version() + env_info['MMCV CUDA Compiler'] = get_compiling_cuda_version() + + return env_info diff --git a/annotator/mmpkg/mmcv/utils/ext_loader.py b/annotator/mmpkg/mmcv/utils/ext_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..08132d2c1b9a1c28880e4bab4d4fa1ba39d9d083 --- /dev/null +++ b/annotator/mmpkg/mmcv/utils/ext_loader.py @@ -0,0 +1,71 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import importlib +import os +import pkgutil +import warnings +from collections import namedtuple + +import torch + +if torch.__version__ != 'parrots': + + def load_ext(name, funcs): + ext = importlib.import_module('mmcv.' + name) + for fun in funcs: + assert hasattr(ext, fun), f'{fun} miss in module {name}' + return ext +else: + from parrots import extension + from parrots.base import ParrotsException + + has_return_value_ops = [ + 'nms', + 'softnms', + 'nms_match', + 'nms_rotated', + 'top_pool_forward', + 'top_pool_backward', + 'bottom_pool_forward', + 'bottom_pool_backward', + 'left_pool_forward', + 'left_pool_backward', + 'right_pool_forward', + 'right_pool_backward', + 'fused_bias_leakyrelu', + 'upfirdn2d', + 'ms_deform_attn_forward', + 'pixel_group', + 'contour_expand', + ] + + def get_fake_func(name, e): + + def fake_func(*args, **kwargs): + warnings.warn(f'{name} is not supported in parrots now') + raise e + + return fake_func + + def load_ext(name, funcs): + ExtModule = namedtuple('ExtModule', funcs) + ext_list = [] + lib_root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + for fun in funcs: + try: + ext_fun = extension.load(fun, name, lib_dir=lib_root) + except ParrotsException as e: + if 'No element registered' not in e.message: + warnings.warn(e.message) + ext_fun = get_fake_func(fun, e) + ext_list.append(ext_fun) + else: + if fun in has_return_value_ops: + ext_list.append(ext_fun.op) + else: + ext_list.append(ext_fun.op_) + return ExtModule(*ext_list) + + +def check_ops_exist(): + ext_loader = pkgutil.find_loader('mmcv._ext') + return ext_loader is not None diff --git a/annotator/mmpkg/mmcv/utils/logging.py b/annotator/mmpkg/mmcv/utils/logging.py new file mode 100644 index 0000000000000000000000000000000000000000..4aa0e04bb9b3ab2a4bfbc4def50404ccbac2c6e6 --- /dev/null +++ b/annotator/mmpkg/mmcv/utils/logging.py @@ -0,0 +1,110 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import logging + +import torch.distributed as dist + +logger_initialized = {} + + +def get_logger(name, log_file=None, log_level=logging.INFO, file_mode='w'): + """Initialize and get a logger by name. + + If the logger has not been initialized, this method will initialize the + logger by adding one or two handlers, otherwise the initialized logger will + be directly returned. During initialization, a StreamHandler will always be + added. If `log_file` is specified and the process rank is 0, a FileHandler + will also be added. + + Args: + name (str): Logger name. + log_file (str | None): The log filename. If specified, a FileHandler + will be added to the logger. + log_level (int): The logger level. Note that only the process of + rank 0 is affected, and other processes will set the level to + "Error" thus be silent most of the time. + file_mode (str): The file mode used in opening log file. + Defaults to 'w'. + + Returns: + logging.Logger: The expected logger. + """ + logger = logging.getLogger(name) + if name in logger_initialized: + return logger + # handle hierarchical names + # e.g., logger "a" is initialized, then logger "a.b" will skip the + # initialization since it is a child of "a". + for logger_name in logger_initialized: + if name.startswith(logger_name): + return logger + + # handle duplicate logs to the console + # Starting in 1.8.0, PyTorch DDP attaches a StreamHandler (NOTSET) + # to the root logger. As logger.propagate is True by default, this root + # level handler causes logging messages from rank>0 processes to + # unexpectedly show up on the console, creating much unwanted clutter. + # To fix this issue, we set the root logger's StreamHandler, if any, to log + # at the ERROR level. + for handler in logger.root.handlers: + if type(handler) is logging.StreamHandler: + handler.setLevel(logging.ERROR) + + stream_handler = logging.StreamHandler() + handlers = [stream_handler] + + if dist.is_available() and dist.is_initialized(): + rank = dist.get_rank() + else: + rank = 0 + + # only rank 0 will add a FileHandler + if rank == 0 and log_file is not None: + # Here, the default behaviour of the official logger is 'a'. Thus, we + # provide an interface to change the file mode to the default + # behaviour. + file_handler = logging.FileHandler(log_file, file_mode) + handlers.append(file_handler) + + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + for handler in handlers: + handler.setFormatter(formatter) + handler.setLevel(log_level) + logger.addHandler(handler) + + if rank == 0: + logger.setLevel(log_level) + else: + logger.setLevel(logging.ERROR) + + logger_initialized[name] = True + + return logger + + +def print_log(msg, logger=None, level=logging.INFO): + """Print a log message. + + Args: + msg (str): The message to be logged. + logger (logging.Logger | str | None): The logger to be used. + Some special loggers are: + - "silent": no message will be printed. + - other str: the logger obtained with `get_root_logger(logger)`. + - None: The `print()` method will be used to print log messages. + level (int): Logging level. Only available when `logger` is a Logger + object or "root". + """ + if logger is None: + print(msg) + elif isinstance(logger, logging.Logger): + logger.log(level, msg) + elif logger == 'silent': + pass + elif isinstance(logger, str): + _logger = get_logger(logger) + _logger.log(level, msg) + else: + raise TypeError( + 'logger should be either a logging.Logger object, str, ' + f'"silent" or None, but got {type(logger)}') diff --git a/annotator/mmpkg/mmcv/utils/misc.py b/annotator/mmpkg/mmcv/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..2c58d0d7fee9fe3d4519270ad8c1e998d0d8a18c --- /dev/null +++ b/annotator/mmpkg/mmcv/utils/misc.py @@ -0,0 +1,377 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import collections.abc +import functools +import itertools +import subprocess +import warnings +from collections import abc +from importlib import import_module +from inspect import getfullargspec +from itertools import repeat + + +# From PyTorch internals +def _ntuple(n): + + def parse(x): + if isinstance(x, collections.abc.Iterable): + return x + return tuple(repeat(x, n)) + + return parse + + +to_1tuple = _ntuple(1) +to_2tuple = _ntuple(2) +to_3tuple = _ntuple(3) +to_4tuple = _ntuple(4) +to_ntuple = _ntuple + + +def is_str(x): + """Whether the input is an string instance. + + Note: This method is deprecated since python 2 is no longer supported. + """ + return isinstance(x, str) + + +def import_modules_from_strings(imports, allow_failed_imports=False): + """Import modules from the given list of strings. + + Args: + imports (list | str | None): The given module names to be imported. + allow_failed_imports (bool): If True, the failed imports will return + None. Otherwise, an ImportError is raise. Default: False. + + Returns: + list[module] | module | None: The imported modules. + + Examples: + >>> osp, sys = import_modules_from_strings( + ... ['os.path', 'sys']) + >>> import os.path as osp_ + >>> import sys as sys_ + >>> assert osp == osp_ + >>> assert sys == sys_ + """ + if not imports: + return + single_import = False + if isinstance(imports, str): + single_import = True + imports = [imports] + if not isinstance(imports, list): + raise TypeError( + f'custom_imports must be a list but got type {type(imports)}') + imported = [] + for imp in imports: + if not isinstance(imp, str): + raise TypeError( + f'{imp} is of type {type(imp)} and cannot be imported.') + try: + imported_tmp = import_module(imp) + except ImportError: + if allow_failed_imports: + warnings.warn(f'{imp} failed to import and is ignored.', + UserWarning) + imported_tmp = None + else: + raise ImportError + imported.append(imported_tmp) + if single_import: + imported = imported[0] + return imported + + +def iter_cast(inputs, dst_type, return_type=None): + """Cast elements of an iterable object into some type. + + Args: + inputs (Iterable): The input object. + dst_type (type): Destination type. + return_type (type, optional): If specified, the output object will be + converted to this type, otherwise an iterator. + + Returns: + iterator or specified type: The converted object. + """ + if not isinstance(inputs, abc.Iterable): + raise TypeError('inputs must be an iterable object') + if not isinstance(dst_type, type): + raise TypeError('"dst_type" must be a valid type') + + out_iterable = map(dst_type, inputs) + + if return_type is None: + return out_iterable + else: + return return_type(out_iterable) + + +def list_cast(inputs, dst_type): + """Cast elements of an iterable object into a list of some type. + + A partial method of :func:`iter_cast`. + """ + return iter_cast(inputs, dst_type, return_type=list) + + +def tuple_cast(inputs, dst_type): + """Cast elements of an iterable object into a tuple of some type. + + A partial method of :func:`iter_cast`. + """ + return iter_cast(inputs, dst_type, return_type=tuple) + + +def is_seq_of(seq, expected_type, seq_type=None): + """Check whether it is a sequence of some type. + + Args: + seq (Sequence): The sequence to be checked. + expected_type (type): Expected type of sequence items. + seq_type (type, optional): Expected sequence type. + + Returns: + bool: Whether the sequence is valid. + """ + if seq_type is None: + exp_seq_type = abc.Sequence + else: + assert isinstance(seq_type, type) + exp_seq_type = seq_type + if not isinstance(seq, exp_seq_type): + return False + for item in seq: + if not isinstance(item, expected_type): + return False + return True + + +def is_list_of(seq, expected_type): + """Check whether it is a list of some type. + + A partial method of :func:`is_seq_of`. + """ + return is_seq_of(seq, expected_type, seq_type=list) + + +def is_tuple_of(seq, expected_type): + """Check whether it is a tuple of some type. + + A partial method of :func:`is_seq_of`. + """ + return is_seq_of(seq, expected_type, seq_type=tuple) + + +def slice_list(in_list, lens): + """Slice a list into several sub lists by a list of given length. + + Args: + in_list (list): The list to be sliced. + lens(int or list): The expected length of each out list. + + Returns: + list: A list of sliced list. + """ + if isinstance(lens, int): + assert len(in_list) % lens == 0 + lens = [lens] * int(len(in_list) / lens) + if not isinstance(lens, list): + raise TypeError('"indices" must be an integer or a list of integers') + elif sum(lens) != len(in_list): + raise ValueError('sum of lens and list length does not ' + f'match: {sum(lens)} != {len(in_list)}') + out_list = [] + idx = 0 + for i in range(len(lens)): + out_list.append(in_list[idx:idx + lens[i]]) + idx += lens[i] + return out_list + + +def concat_list(in_list): + """Concatenate a list of list into a single list. + + Args: + in_list (list): The list of list to be merged. + + Returns: + list: The concatenated flat list. + """ + return list(itertools.chain(*in_list)) + + +def check_prerequisites( + prerequisites, + checker, + msg_tmpl='Prerequisites "{}" are required in method "{}" but not ' + 'found, please install them first.'): # yapf: disable + """A decorator factory to check if prerequisites are satisfied. + + Args: + prerequisites (str of list[str]): Prerequisites to be checked. + checker (callable): The checker method that returns True if a + prerequisite is meet, False otherwise. + msg_tmpl (str): The message template with two variables. + + Returns: + decorator: A specific decorator. + """ + + def wrap(func): + + @functools.wraps(func) + def wrapped_func(*args, **kwargs): + requirements = [prerequisites] if isinstance( + prerequisites, str) else prerequisites + missing = [] + for item in requirements: + if not checker(item): + missing.append(item) + if missing: + print(msg_tmpl.format(', '.join(missing), func.__name__)) + raise RuntimeError('Prerequisites not meet.') + else: + return func(*args, **kwargs) + + return wrapped_func + + return wrap + + +def _check_py_package(package): + try: + import_module(package) + except ImportError: + return False + else: + return True + + +def _check_executable(cmd): + if subprocess.call(f'which {cmd}', shell=True) != 0: + return False + else: + return True + + +def requires_package(prerequisites): + """A decorator to check if some python packages are installed. + + Example: + >>> @requires_package('numpy') + >>> func(arg1, args): + >>> return numpy.zeros(1) + array([0.]) + >>> @requires_package(['numpy', 'non_package']) + >>> func(arg1, args): + >>> return numpy.zeros(1) + ImportError + """ + return check_prerequisites(prerequisites, checker=_check_py_package) + + +def requires_executable(prerequisites): + """A decorator to check if some executable files are installed. + + Example: + >>> @requires_executable('ffmpeg') + >>> func(arg1, args): + >>> print(1) + 1 + """ + return check_prerequisites(prerequisites, checker=_check_executable) + + +def deprecated_api_warning(name_dict, cls_name=None): + """A decorator to check if some arguments are deprecate and try to replace + deprecate src_arg_name to dst_arg_name. + + Args: + name_dict(dict): + key (str): Deprecate argument names. + val (str): Expected argument names. + + Returns: + func: New function. + """ + + def api_warning_wrapper(old_func): + + @functools.wraps(old_func) + def new_func(*args, **kwargs): + # get the arg spec of the decorated method + args_info = getfullargspec(old_func) + # get name of the function + func_name = old_func.__name__ + if cls_name is not None: + func_name = f'{cls_name}.{func_name}' + if args: + arg_names = args_info.args[:len(args)] + for src_arg_name, dst_arg_name in name_dict.items(): + if src_arg_name in arg_names: + warnings.warn( + f'"{src_arg_name}" is deprecated in ' + f'`{func_name}`, please use "{dst_arg_name}" ' + 'instead') + arg_names[arg_names.index(src_arg_name)] = dst_arg_name + if kwargs: + for src_arg_name, dst_arg_name in name_dict.items(): + if src_arg_name in kwargs: + + assert dst_arg_name not in kwargs, ( + f'The expected behavior is to replace ' + f'the deprecated key `{src_arg_name}` to ' + f'new key `{dst_arg_name}`, but got them ' + f'in the arguments at the same time, which ' + f'is confusing. `{src_arg_name} will be ' + f'deprecated in the future, please ' + f'use `{dst_arg_name}` instead.') + + warnings.warn( + f'"{src_arg_name}" is deprecated in ' + f'`{func_name}`, please use "{dst_arg_name}" ' + 'instead') + kwargs[dst_arg_name] = kwargs.pop(src_arg_name) + + # apply converted arguments to the decorated method + output = old_func(*args, **kwargs) + return output + + return new_func + + return api_warning_wrapper + + +def is_method_overridden(method, base_class, derived_class): + """Check if a method of base class is overridden in derived class. + + Args: + method (str): the method name to check. + base_class (type): the class of the base class. + derived_class (type | Any): the class or instance of the derived class. + """ + assert isinstance(base_class, type), \ + "base_class doesn't accept instance, Please pass class instead." + + if not isinstance(derived_class, type): + derived_class = derived_class.__class__ + + base_method = getattr(base_class, method) + derived_method = getattr(derived_class, method) + return derived_method != base_method + + +def has_method(obj: object, method: str) -> bool: + """Check whether the object has a method. + + Args: + method (str): The method name to check. + obj (object): The object to check. + + Returns: + bool: True if the object has the method else False. + """ + return hasattr(obj, method) and callable(getattr(obj, method)) diff --git a/annotator/mmpkg/mmcv/utils/parrots_jit.py b/annotator/mmpkg/mmcv/utils/parrots_jit.py new file mode 100644 index 0000000000000000000000000000000000000000..61873f6dbb9b10ed972c90aa8faa321e3cb3249e --- /dev/null +++ b/annotator/mmpkg/mmcv/utils/parrots_jit.py @@ -0,0 +1,41 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os + +from .parrots_wrapper import TORCH_VERSION + +parrots_jit_option = os.getenv('PARROTS_JIT_OPTION') + +if TORCH_VERSION == 'parrots' and parrots_jit_option == 'ON': + from parrots.jit import pat as jit +else: + + def jit(func=None, + check_input=None, + full_shape=True, + derivate=False, + coderize=False, + optimize=False): + + def wrapper(func): + + def wrapper_inner(*args, **kargs): + return func(*args, **kargs) + + return wrapper_inner + + if func is None: + return wrapper + else: + return func + + +if TORCH_VERSION == 'parrots': + from parrots.utils.tester import skip_no_elena +else: + + def skip_no_elena(func): + + def wrapper(*args, **kargs): + return func(*args, **kargs) + + return wrapper diff --git a/annotator/mmpkg/mmcv/utils/parrots_wrapper.py b/annotator/mmpkg/mmcv/utils/parrots_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..93c97640d4b9ed088ca82cfe03e6efebfcfa9dbf --- /dev/null +++ b/annotator/mmpkg/mmcv/utils/parrots_wrapper.py @@ -0,0 +1,107 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from functools import partial + +import torch + +TORCH_VERSION = torch.__version__ + + +def is_rocm_pytorch() -> bool: + is_rocm = False + if TORCH_VERSION != 'parrots': + try: + from torch.utils.cpp_extension import ROCM_HOME + is_rocm = True if ((torch.version.hip is not None) and + (ROCM_HOME is not None)) else False + except ImportError: + pass + return is_rocm + + +def _get_cuda_home(): + if TORCH_VERSION == 'parrots': + from parrots.utils.build_extension import CUDA_HOME + else: + if is_rocm_pytorch(): + from torch.utils.cpp_extension import ROCM_HOME + CUDA_HOME = ROCM_HOME + else: + from torch.utils.cpp_extension import CUDA_HOME + return CUDA_HOME + + +def get_build_config(): + if TORCH_VERSION == 'parrots': + from parrots.config import get_build_info + return get_build_info() + else: + return torch.__config__.show() + + +def _get_conv(): + if TORCH_VERSION == 'parrots': + from parrots.nn.modules.conv import _ConvNd, _ConvTransposeMixin + else: + from torch.nn.modules.conv import _ConvNd, _ConvTransposeMixin + return _ConvNd, _ConvTransposeMixin + + +def _get_dataloader(): + if TORCH_VERSION == 'parrots': + from torch.utils.data import DataLoader, PoolDataLoader + else: + from torch.utils.data import DataLoader + PoolDataLoader = DataLoader + return DataLoader, PoolDataLoader + + +def _get_extension(): + if TORCH_VERSION == 'parrots': + from parrots.utils.build_extension import BuildExtension, Extension + CppExtension = partial(Extension, cuda=False) + CUDAExtension = partial(Extension, cuda=True) + else: + from torch.utils.cpp_extension import (BuildExtension, CppExtension, + CUDAExtension) + return BuildExtension, CppExtension, CUDAExtension + + +def _get_pool(): + if TORCH_VERSION == 'parrots': + from parrots.nn.modules.pool import (_AdaptiveAvgPoolNd, + _AdaptiveMaxPoolNd, _AvgPoolNd, + _MaxPoolNd) + else: + from torch.nn.modules.pooling import (_AdaptiveAvgPoolNd, + _AdaptiveMaxPoolNd, _AvgPoolNd, + _MaxPoolNd) + return _AdaptiveAvgPoolNd, _AdaptiveMaxPoolNd, _AvgPoolNd, _MaxPoolNd + + +def _get_norm(): + if TORCH_VERSION == 'parrots': + from parrots.nn.modules.batchnorm import _BatchNorm, _InstanceNorm + SyncBatchNorm_ = torch.nn.SyncBatchNorm2d + else: + from torch.nn.modules.instancenorm import _InstanceNorm + from torch.nn.modules.batchnorm import _BatchNorm + SyncBatchNorm_ = torch.nn.SyncBatchNorm + return _BatchNorm, _InstanceNorm, SyncBatchNorm_ + + +_ConvNd, _ConvTransposeMixin = _get_conv() +DataLoader, PoolDataLoader = _get_dataloader() +BuildExtension, CppExtension, CUDAExtension = _get_extension() +_BatchNorm, _InstanceNorm, SyncBatchNorm_ = _get_norm() +_AdaptiveAvgPoolNd, _AdaptiveMaxPoolNd, _AvgPoolNd, _MaxPoolNd = _get_pool() + + +class SyncBatchNorm(SyncBatchNorm_): + + def _check_input_dim(self, input): + if TORCH_VERSION == 'parrots': + if input.dim() < 2: + raise ValueError( + f'expected at least 2D input (got {input.dim()}D input)') + else: + super()._check_input_dim(input) diff --git a/annotator/mmpkg/mmcv/utils/path.py b/annotator/mmpkg/mmcv/utils/path.py new file mode 100644 index 0000000000000000000000000000000000000000..7dab4b3041413b1432b0f434b8b14783097d33c6 --- /dev/null +++ b/annotator/mmpkg/mmcv/utils/path.py @@ -0,0 +1,101 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os +import os.path as osp +from pathlib import Path + +from .misc import is_str + + +def is_filepath(x): + return is_str(x) or isinstance(x, Path) + + +def fopen(filepath, *args, **kwargs): + if is_str(filepath): + return open(filepath, *args, **kwargs) + elif isinstance(filepath, Path): + return filepath.open(*args, **kwargs) + raise ValueError('`filepath` should be a string or a Path') + + +def check_file_exist(filename, msg_tmpl='file "{}" does not exist'): + if not osp.isfile(filename): + raise FileNotFoundError(msg_tmpl.format(filename)) + + +def mkdir_or_exist(dir_name, mode=0o777): + if dir_name == '': + return + dir_name = osp.expanduser(dir_name) + os.makedirs(dir_name, mode=mode, exist_ok=True) + + +def symlink(src, dst, overwrite=True, **kwargs): + if os.path.lexists(dst) and overwrite: + os.remove(dst) + os.symlink(src, dst, **kwargs) + + +def scandir(dir_path, suffix=None, recursive=False, case_sensitive=True): + """Scan a directory to find the interested files. + + Args: + dir_path (str | obj:`Path`): Path of the directory. + suffix (str | tuple(str), optional): File suffix that we are + interested in. Default: None. + recursive (bool, optional): If set to True, recursively scan the + directory. Default: False. + case_sensitive (bool, optional) : If set to False, ignore the case of + suffix. Default: True. + + Returns: + A generator for all the interested files with relative paths. + """ + if isinstance(dir_path, (str, Path)): + dir_path = str(dir_path) + else: + raise TypeError('"dir_path" must be a string or Path object') + + if (suffix is not None) and not isinstance(suffix, (str, tuple)): + raise TypeError('"suffix" must be a string or tuple of strings') + + if suffix is not None and not case_sensitive: + suffix = suffix.lower() if isinstance(suffix, str) else tuple( + item.lower() for item in suffix) + + root = dir_path + + def _scandir(dir_path, suffix, recursive, case_sensitive): + for entry in os.scandir(dir_path): + if not entry.name.startswith('.') and entry.is_file(): + rel_path = osp.relpath(entry.path, root) + _rel_path = rel_path if case_sensitive else rel_path.lower() + if suffix is None or _rel_path.endswith(suffix): + yield rel_path + elif recursive and os.path.isdir(entry.path): + # scan recursively if entry.path is a directory + yield from _scandir(entry.path, suffix, recursive, + case_sensitive) + + return _scandir(dir_path, suffix, recursive, case_sensitive) + + +def find_vcs_root(path, markers=('.git', )): + """Finds the root directory (including itself) of specified markers. + + Args: + path (str): Path of directory or file. + markers (list[str], optional): List of file or directory names. + + Returns: + The directory contained one of the markers or None if not found. + """ + if osp.isfile(path): + path = osp.dirname(path) + + prev, cur = None, osp.abspath(osp.expanduser(path)) + while cur != prev: + if any(osp.exists(osp.join(cur, marker)) for marker in markers): + return cur + prev, cur = cur, osp.split(cur)[0] + return None diff --git a/annotator/mmpkg/mmcv/utils/progressbar.py b/annotator/mmpkg/mmcv/utils/progressbar.py new file mode 100644 index 0000000000000000000000000000000000000000..0062f670dd94fa9da559ab26ef85517dcf5211c7 --- /dev/null +++ b/annotator/mmpkg/mmcv/utils/progressbar.py @@ -0,0 +1,208 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import sys +from collections.abc import Iterable +from multiprocessing import Pool +from shutil import get_terminal_size + +from .timer import Timer + + +class ProgressBar: + """A progress bar which can print the progress.""" + + def __init__(self, task_num=0, bar_width=50, start=True, file=sys.stdout): + self.task_num = task_num + self.bar_width = bar_width + self.completed = 0 + self.file = file + if start: + self.start() + + @property + def terminal_width(self): + width, _ = get_terminal_size() + return width + + def start(self): + if self.task_num > 0: + self.file.write(f'[{" " * self.bar_width}] 0/{self.task_num}, ' + 'elapsed: 0s, ETA:') + else: + self.file.write('completed: 0, elapsed: 0s') + self.file.flush() + self.timer = Timer() + + def update(self, num_tasks=1): + assert num_tasks > 0 + self.completed += num_tasks + elapsed = self.timer.since_start() + if elapsed > 0: + fps = self.completed / elapsed + else: + fps = float('inf') + if self.task_num > 0: + percentage = self.completed / float(self.task_num) + eta = int(elapsed * (1 - percentage) / percentage + 0.5) + msg = f'\r[{{}}] {self.completed}/{self.task_num}, ' \ + f'{fps:.1f} task/s, elapsed: {int(elapsed + 0.5)}s, ' \ + f'ETA: {eta:5}s' + + bar_width = min(self.bar_width, + int(self.terminal_width - len(msg)) + 2, + int(self.terminal_width * 0.6)) + bar_width = max(2, bar_width) + mark_width = int(bar_width * percentage) + bar_chars = '>' * mark_width + ' ' * (bar_width - mark_width) + self.file.write(msg.format(bar_chars)) + else: + self.file.write( + f'completed: {self.completed}, elapsed: {int(elapsed + 0.5)}s,' + f' {fps:.1f} tasks/s') + self.file.flush() + + +def track_progress(func, tasks, bar_width=50, file=sys.stdout, **kwargs): + """Track the progress of tasks execution with a progress bar. + + Tasks are done with a simple for-loop. + + Args: + func (callable): The function to be applied to each task. + tasks (list or tuple[Iterable, int]): A list of tasks or + (tasks, total num). + bar_width (int): Width of progress bar. + + Returns: + list: The task results. + """ + if isinstance(tasks, tuple): + assert len(tasks) == 2 + assert isinstance(tasks[0], Iterable) + assert isinstance(tasks[1], int) + task_num = tasks[1] + tasks = tasks[0] + elif isinstance(tasks, Iterable): + task_num = len(tasks) + else: + raise TypeError( + '"tasks" must be an iterable object or a (iterator, int) tuple') + prog_bar = ProgressBar(task_num, bar_width, file=file) + results = [] + for task in tasks: + results.append(func(task, **kwargs)) + prog_bar.update() + prog_bar.file.write('\n') + return results + + +def init_pool(process_num, initializer=None, initargs=None): + if initializer is None: + return Pool(process_num) + elif initargs is None: + return Pool(process_num, initializer) + else: + if not isinstance(initargs, tuple): + raise TypeError('"initargs" must be a tuple') + return Pool(process_num, initializer, initargs) + + +def track_parallel_progress(func, + tasks, + nproc, + initializer=None, + initargs=None, + bar_width=50, + chunksize=1, + skip_first=False, + keep_order=True, + file=sys.stdout): + """Track the progress of parallel task execution with a progress bar. + + The built-in :mod:`multiprocessing` module is used for process pools and + tasks are done with :func:`Pool.map` or :func:`Pool.imap_unordered`. + + Args: + func (callable): The function to be applied to each task. + tasks (list or tuple[Iterable, int]): A list of tasks or + (tasks, total num). + nproc (int): Process (worker) number. + initializer (None or callable): Refer to :class:`multiprocessing.Pool` + for details. + initargs (None or tuple): Refer to :class:`multiprocessing.Pool` for + details. + chunksize (int): Refer to :class:`multiprocessing.Pool` for details. + bar_width (int): Width of progress bar. + skip_first (bool): Whether to skip the first sample for each worker + when estimating fps, since the initialization step may takes + longer. + keep_order (bool): If True, :func:`Pool.imap` is used, otherwise + :func:`Pool.imap_unordered` is used. + + Returns: + list: The task results. + """ + if isinstance(tasks, tuple): + assert len(tasks) == 2 + assert isinstance(tasks[0], Iterable) + assert isinstance(tasks[1], int) + task_num = tasks[1] + tasks = tasks[0] + elif isinstance(tasks, Iterable): + task_num = len(tasks) + else: + raise TypeError( + '"tasks" must be an iterable object or a (iterator, int) tuple') + pool = init_pool(nproc, initializer, initargs) + start = not skip_first + task_num -= nproc * chunksize * int(skip_first) + prog_bar = ProgressBar(task_num, bar_width, start, file=file) + results = [] + if keep_order: + gen = pool.imap(func, tasks, chunksize) + else: + gen = pool.imap_unordered(func, tasks, chunksize) + for result in gen: + results.append(result) + if skip_first: + if len(results) < nproc * chunksize: + continue + elif len(results) == nproc * chunksize: + prog_bar.start() + continue + prog_bar.update() + prog_bar.file.write('\n') + pool.close() + pool.join() + return results + + +def track_iter_progress(tasks, bar_width=50, file=sys.stdout): + """Track the progress of tasks iteration or enumeration with a progress + bar. + + Tasks are yielded with a simple for-loop. + + Args: + tasks (list or tuple[Iterable, int]): A list of tasks or + (tasks, total num). + bar_width (int): Width of progress bar. + + Yields: + list: The task results. + """ + if isinstance(tasks, tuple): + assert len(tasks) == 2 + assert isinstance(tasks[0], Iterable) + assert isinstance(tasks[1], int) + task_num = tasks[1] + tasks = tasks[0] + elif isinstance(tasks, Iterable): + task_num = len(tasks) + else: + raise TypeError( + '"tasks" must be an iterable object or a (iterator, int) tuple') + prog_bar = ProgressBar(task_num, bar_width, file=file) + for task in tasks: + yield task + prog_bar.update() + prog_bar.file.write('\n') diff --git a/annotator/mmpkg/mmcv/utils/registry.py b/annotator/mmpkg/mmcv/utils/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..fa9df39bc9f3d8d568361e7250ab35468f2b74e0 --- /dev/null +++ b/annotator/mmpkg/mmcv/utils/registry.py @@ -0,0 +1,315 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import inspect +import warnings +from functools import partial + +from .misc import is_seq_of + + +def build_from_cfg(cfg, registry, default_args=None): + """Build a module from config dict. + + Args: + cfg (dict): Config dict. It should at least contain the key "type". + registry (:obj:`Registry`): The registry to search the type from. + default_args (dict, optional): Default initialization arguments. + + Returns: + object: The constructed object. + """ + if not isinstance(cfg, dict): + raise TypeError(f'cfg must be a dict, but got {type(cfg)}') + if 'type' not in cfg: + if default_args is None or 'type' not in default_args: + raise KeyError( + '`cfg` or `default_args` must contain the key "type", ' + f'but got {cfg}\n{default_args}') + if not isinstance(registry, Registry): + raise TypeError('registry must be an mmcv.Registry object, ' + f'but got {type(registry)}') + if not (isinstance(default_args, dict) or default_args is None): + raise TypeError('default_args must be a dict or None, ' + f'but got {type(default_args)}') + + args = cfg.copy() + + if default_args is not None: + for name, value in default_args.items(): + args.setdefault(name, value) + + obj_type = args.pop('type') + if isinstance(obj_type, str): + obj_cls = registry.get(obj_type) + if obj_cls is None: + raise KeyError( + f'{obj_type} is not in the {registry.name} registry') + elif inspect.isclass(obj_type): + obj_cls = obj_type + else: + raise TypeError( + f'type must be a str or valid type, but got {type(obj_type)}') + try: + return obj_cls(**args) + except Exception as e: + # Normal TypeError does not print class name. + raise type(e)(f'{obj_cls.__name__}: {e}') + + +class Registry: + """A registry to map strings to classes. + + Registered object could be built from registry. + Example: + >>> MODELS = Registry('models') + >>> @MODELS.register_module() + >>> class ResNet: + >>> pass + >>> resnet = MODELS.build(dict(type='ResNet')) + + Please refer to + https://mmcv.readthedocs.io/en/latest/understand_mmcv/registry.html for + advanced usage. + + Args: + name (str): Registry name. + build_func(func, optional): Build function to construct instance from + Registry, func:`build_from_cfg` is used if neither ``parent`` or + ``build_func`` is specified. If ``parent`` is specified and + ``build_func`` is not given, ``build_func`` will be inherited + from ``parent``. Default: None. + parent (Registry, optional): Parent registry. The class registered in + children registry could be built from parent. Default: None. + scope (str, optional): The scope of registry. It is the key to search + for children registry. If not specified, scope will be the name of + the package where class is defined, e.g. mmdet, mmcls, mmseg. + Default: None. + """ + + def __init__(self, name, build_func=None, parent=None, scope=None): + self._name = name + self._module_dict = dict() + self._children = dict() + self._scope = self.infer_scope() if scope is None else scope + + # self.build_func will be set with the following priority: + # 1. build_func + # 2. parent.build_func + # 3. build_from_cfg + if build_func is None: + if parent is not None: + self.build_func = parent.build_func + else: + self.build_func = build_from_cfg + else: + self.build_func = build_func + if parent is not None: + assert isinstance(parent, Registry) + parent._add_children(self) + self.parent = parent + else: + self.parent = None + + def __len__(self): + return len(self._module_dict) + + def __contains__(self, key): + return self.get(key) is not None + + def __repr__(self): + format_str = self.__class__.__name__ + \ + f'(name={self._name}, ' \ + f'items={self._module_dict})' + return format_str + + @staticmethod + def infer_scope(): + """Infer the scope of registry. + + The name of the package where registry is defined will be returned. + + Example: + # in mmdet/models/backbone/resnet.py + >>> MODELS = Registry('models') + >>> @MODELS.register_module() + >>> class ResNet: + >>> pass + The scope of ``ResNet`` will be ``mmdet``. + + + Returns: + scope (str): The inferred scope name. + """ + # inspect.stack() trace where this function is called, the index-2 + # indicates the frame where `infer_scope()` is called + filename = inspect.getmodule(inspect.stack()[2][0]).__name__ + split_filename = filename.split('.') + return split_filename[0] + + @staticmethod + def split_scope_key(key): + """Split scope and key. + + The first scope will be split from key. + + Examples: + >>> Registry.split_scope_key('mmdet.ResNet') + 'mmdet', 'ResNet' + >>> Registry.split_scope_key('ResNet') + None, 'ResNet' + + Return: + scope (str, None): The first scope. + key (str): The remaining key. + """ + split_index = key.find('.') + if split_index != -1: + return key[:split_index], key[split_index + 1:] + else: + return None, key + + @property + def name(self): + return self._name + + @property + def scope(self): + return self._scope + + @property + def module_dict(self): + return self._module_dict + + @property + def children(self): + return self._children + + def get(self, key): + """Get the registry record. + + Args: + key (str): The class name in string format. + + Returns: + class: The corresponding class. + """ + scope, real_key = self.split_scope_key(key) + if scope is None or scope == self._scope: + # get from self + if real_key in self._module_dict: + return self._module_dict[real_key] + else: + # get from self._children + if scope in self._children: + return self._children[scope].get(real_key) + else: + # goto root + parent = self.parent + while parent.parent is not None: + parent = parent.parent + return parent.get(key) + + def build(self, *args, **kwargs): + return self.build_func(*args, **kwargs, registry=self) + + def _add_children(self, registry): + """Add children for a registry. + + The ``registry`` will be added as children based on its scope. + The parent registry could build objects from children registry. + + Example: + >>> models = Registry('models') + >>> mmdet_models = Registry('models', parent=models) + >>> @mmdet_models.register_module() + >>> class ResNet: + >>> pass + >>> resnet = models.build(dict(type='mmdet.ResNet')) + """ + + assert isinstance(registry, Registry) + assert registry.scope is not None + assert registry.scope not in self.children, \ + f'scope {registry.scope} exists in {self.name} registry' + self.children[registry.scope] = registry + + def _register_module(self, module_class, module_name=None, force=False): + if not inspect.isclass(module_class): + raise TypeError('module must be a class, ' + f'but got {type(module_class)}') + + if module_name is None: + module_name = module_class.__name__ + if isinstance(module_name, str): + module_name = [module_name] + for name in module_name: + if not force and name in self._module_dict: + raise KeyError(f'{name} is already registered ' + f'in {self.name}') + self._module_dict[name] = module_class + + def deprecated_register_module(self, cls=None, force=False): + warnings.warn( + 'The old API of register_module(module, force=False) ' + 'is deprecated and will be removed, please use the new API ' + 'register_module(name=None, force=False, module=None) instead.') + if cls is None: + return partial(self.deprecated_register_module, force=force) + self._register_module(cls, force=force) + return cls + + def register_module(self, name=None, force=False, module=None): + """Register a module. + + A record will be added to `self._module_dict`, whose key is the class + name or the specified name, and value is the class itself. + It can be used as a decorator or a normal function. + + Example: + >>> backbones = Registry('backbone') + >>> @backbones.register_module() + >>> class ResNet: + >>> pass + + >>> backbones = Registry('backbone') + >>> @backbones.register_module(name='mnet') + >>> class MobileNet: + >>> pass + + >>> backbones = Registry('backbone') + >>> class ResNet: + >>> pass + >>> backbones.register_module(ResNet) + + Args: + name (str | None): The module name to be registered. If not + specified, the class name will be used. + force (bool, optional): Whether to override an existing class with + the same name. Default: False. + module (type): Module class to be registered. + """ + if not isinstance(force, bool): + raise TypeError(f'force must be a boolean, but got {type(force)}') + # NOTE: This is a walkaround to be compatible with the old api, + # while it may introduce unexpected bugs. + if isinstance(name, type): + return self.deprecated_register_module(name, force=force) + + # raise the error ahead of time + if not (name is None or isinstance(name, str) or is_seq_of(name, str)): + raise TypeError( + 'name must be either of None, an instance of str or a sequence' + f' of str, but got {type(name)}') + + # use it as a normal method: x.register_module(module=SomeClass) + if module is not None: + self._register_module( + module_class=module, module_name=name, force=force) + return module + + # use it as a decorator: @x.register_module() + def _register(cls): + self._register_module( + module_class=cls, module_name=name, force=force) + return cls + + return _register diff --git a/annotator/mmpkg/mmcv/utils/testing.py b/annotator/mmpkg/mmcv/utils/testing.py new file mode 100644 index 0000000000000000000000000000000000000000..a27f936da8ec14bac18562ede0a79d476d82f797 --- /dev/null +++ b/annotator/mmpkg/mmcv/utils/testing.py @@ -0,0 +1,140 @@ +# Copyright (c) Open-MMLab. +import sys +from collections.abc import Iterable +from runpy import run_path +from shlex import split +from typing import Any, Dict, List +from unittest.mock import patch + + +def check_python_script(cmd): + """Run the python cmd script with `__main__`. The difference between + `os.system` is that, this function exectues code in the current process, so + that it can be tracked by coverage tools. Currently it supports two forms: + + - ./tests/data/scripts/hello.py zz + - python tests/data/scripts/hello.py zz + """ + args = split(cmd) + if args[0] == 'python': + args = args[1:] + with patch.object(sys, 'argv', args): + run_path(args[0], run_name='__main__') + + +def _any(judge_result): + """Since built-in ``any`` works only when the element of iterable is not + iterable, implement the function.""" + if not isinstance(judge_result, Iterable): + return judge_result + + try: + for element in judge_result: + if _any(element): + return True + except TypeError: + # Maybe encounter the case: torch.tensor(True) | torch.tensor(False) + if judge_result: + return True + return False + + +def assert_dict_contains_subset(dict_obj: Dict[Any, Any], + expected_subset: Dict[Any, Any]) -> bool: + """Check if the dict_obj contains the expected_subset. + + Args: + dict_obj (Dict[Any, Any]): Dict object to be checked. + expected_subset (Dict[Any, Any]): Subset expected to be contained in + dict_obj. + + Returns: + bool: Whether the dict_obj contains the expected_subset. + """ + + for key, value in expected_subset.items(): + if key not in dict_obj.keys() or _any(dict_obj[key] != value): + return False + return True + + +def assert_attrs_equal(obj: Any, expected_attrs: Dict[str, Any]) -> bool: + """Check if attribute of class object is correct. + + Args: + obj (object): Class object to be checked. + expected_attrs (Dict[str, Any]): Dict of the expected attrs. + + Returns: + bool: Whether the attribute of class object is correct. + """ + for attr, value in expected_attrs.items(): + if not hasattr(obj, attr) or _any(getattr(obj, attr) != value): + return False + return True + + +def assert_dict_has_keys(obj: Dict[str, Any], + expected_keys: List[str]) -> bool: + """Check if the obj has all the expected_keys. + + Args: + obj (Dict[str, Any]): Object to be checked. + expected_keys (List[str]): Keys expected to contained in the keys of + the obj. + + Returns: + bool: Whether the obj has the expected keys. + """ + return set(expected_keys).issubset(set(obj.keys())) + + +def assert_keys_equal(result_keys: List[str], target_keys: List[str]) -> bool: + """Check if target_keys is equal to result_keys. + + Args: + result_keys (List[str]): Result keys to be checked. + target_keys (List[str]): Target keys to be checked. + + Returns: + bool: Whether target_keys is equal to result_keys. + """ + return set(result_keys) == set(target_keys) + + +def assert_is_norm_layer(module) -> bool: + """Check if the module is a norm layer. + + Args: + module (nn.Module): The module to be checked. + + Returns: + bool: Whether the module is a norm layer. + """ + from .parrots_wrapper import _BatchNorm, _InstanceNorm + from torch.nn import GroupNorm, LayerNorm + norm_layer_candidates = (_BatchNorm, _InstanceNorm, GroupNorm, LayerNorm) + return isinstance(module, norm_layer_candidates) + + +def assert_params_all_zeros(module) -> bool: + """Check if the parameters of the module is all zeros. + + Args: + module (nn.Module): The module to be checked. + + Returns: + bool: Whether the parameters of the module is all zeros. + """ + weight_data = module.weight.data + is_weight_zero = weight_data.allclose( + weight_data.new_zeros(weight_data.size())) + + if hasattr(module, 'bias') and module.bias is not None: + bias_data = module.bias.data + is_bias_zero = bias_data.allclose( + bias_data.new_zeros(bias_data.size())) + else: + is_bias_zero = True + + return is_weight_zero and is_bias_zero diff --git a/annotator/mmpkg/mmcv/utils/timer.py b/annotator/mmpkg/mmcv/utils/timer.py new file mode 100644 index 0000000000000000000000000000000000000000..0435c1250ebb63e0d881d7022979a76b2dcc7298 --- /dev/null +++ b/annotator/mmpkg/mmcv/utils/timer.py @@ -0,0 +1,118 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from time import time + + +class TimerError(Exception): + + def __init__(self, message): + self.message = message + super(TimerError, self).__init__(message) + + +class Timer: + """A flexible Timer class. + + :Example: + + >>> import time + >>> import annotator.mmpkg.mmcv as mmcv + >>> with mmcv.Timer(): + >>> # simulate a code block that will run for 1s + >>> time.sleep(1) + 1.000 + >>> with mmcv.Timer(print_tmpl='it takes {:.1f} seconds'): + >>> # simulate a code block that will run for 1s + >>> time.sleep(1) + it takes 1.0 seconds + >>> timer = mmcv.Timer() + >>> time.sleep(0.5) + >>> print(timer.since_start()) + 0.500 + >>> time.sleep(0.5) + >>> print(timer.since_last_check()) + 0.500 + >>> print(timer.since_start()) + 1.000 + """ + + def __init__(self, start=True, print_tmpl=None): + self._is_running = False + self.print_tmpl = print_tmpl if print_tmpl else '{:.3f}' + if start: + self.start() + + @property + def is_running(self): + """bool: indicate whether the timer is running""" + return self._is_running + + def __enter__(self): + self.start() + return self + + def __exit__(self, type, value, traceback): + print(self.print_tmpl.format(self.since_last_check())) + self._is_running = False + + def start(self): + """Start the timer.""" + if not self._is_running: + self._t_start = time() + self._is_running = True + self._t_last = time() + + def since_start(self): + """Total time since the timer is started. + + Returns (float): Time in seconds. + """ + if not self._is_running: + raise TimerError('timer is not running') + self._t_last = time() + return self._t_last - self._t_start + + def since_last_check(self): + """Time since the last checking. + + Either :func:`since_start` or :func:`since_last_check` is a checking + operation. + + Returns (float): Time in seconds. + """ + if not self._is_running: + raise TimerError('timer is not running') + dur = time() - self._t_last + self._t_last = time() + return dur + + +_g_timers = {} # global timers + + +def check_time(timer_id): + """Add check points in a single line. + + This method is suitable for running a task on a list of items. A timer will + be registered when the method is called for the first time. + + :Example: + + >>> import time + >>> import annotator.mmpkg.mmcv as mmcv + >>> for i in range(1, 6): + >>> # simulate a code block + >>> time.sleep(i) + >>> mmcv.check_time('task1') + 2.000 + 3.000 + 4.000 + 5.000 + + Args: + timer_id (str): Timer identifier. + """ + if timer_id not in _g_timers: + _g_timers[timer_id] = Timer() + return 0 + else: + return _g_timers[timer_id].since_last_check() diff --git a/annotator/mmpkg/mmcv/utils/trace.py b/annotator/mmpkg/mmcv/utils/trace.py new file mode 100644 index 0000000000000000000000000000000000000000..51f6e3cab4ac7bbdf561583d7463a5f2897960e7 --- /dev/null +++ b/annotator/mmpkg/mmcv/utils/trace.py @@ -0,0 +1,23 @@ +import warnings + +import torch + +from annotator.mmpkg.mmcv.utils import digit_version + + +def is_jit_tracing() -> bool: + if (torch.__version__ != 'parrots' + and digit_version(torch.__version__) >= digit_version('1.6.0')): + on_trace = torch.jit.is_tracing() + # In PyTorch 1.6, torch.jit.is_tracing has a bug. + # Refers to https://github.com/pytorch/pytorch/issues/42448 + if isinstance(on_trace, bool): + return on_trace + else: + return torch._C._is_tracing() + else: + warnings.warn( + 'torch.jit.is_tracing is only supported after v1.6.0. ' + 'Therefore is_tracing returns False automatically. Please ' + 'set on_trace manually if you are using trace.', UserWarning) + return False diff --git a/annotator/mmpkg/mmcv/utils/version_utils.py b/annotator/mmpkg/mmcv/utils/version_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..963c45a2e8a86a88413ab6c18c22481fb9831985 --- /dev/null +++ b/annotator/mmpkg/mmcv/utils/version_utils.py @@ -0,0 +1,90 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os +import subprocess +import warnings + +from packaging.version import parse + + +def digit_version(version_str: str, length: int = 4): + """Convert a version string into a tuple of integers. + + This method is usually used for comparing two versions. For pre-release + versions: alpha < beta < rc. + + Args: + version_str (str): The version string. + length (int): The maximum number of version levels. Default: 4. + + Returns: + tuple[int]: The version info in digits (integers). + """ + assert 'parrots' not in version_str + version = parse(version_str) + assert version.release, f'failed to parse version {version_str}' + release = list(version.release) + release = release[:length] + if len(release) < length: + release = release + [0] * (length - len(release)) + if version.is_prerelease: + mapping = {'a': -3, 'b': -2, 'rc': -1} + val = -4 + # version.pre can be None + if version.pre: + if version.pre[0] not in mapping: + warnings.warn(f'unknown prerelease version {version.pre[0]}, ' + 'version checking may go wrong') + else: + val = mapping[version.pre[0]] + release.extend([val, version.pre[-1]]) + else: + release.extend([val, 0]) + + elif version.is_postrelease: + release.extend([1, version.post]) + else: + release.extend([0, 0]) + return tuple(release) + + +def _minimal_ext_cmd(cmd): + # construct minimal environment + env = {} + for k in ['SYSTEMROOT', 'PATH', 'HOME']: + v = os.environ.get(k) + if v is not None: + env[k] = v + # LANGUAGE is used on win32 + env['LANGUAGE'] = 'C' + env['LANG'] = 'C' + env['LC_ALL'] = 'C' + out = subprocess.Popen( + cmd, stdout=subprocess.PIPE, env=env).communicate()[0] + return out + + +def get_git_hash(fallback='unknown', digits=None): + """Get the git hash of the current repo. + + Args: + fallback (str, optional): The fallback string when git hash is + unavailable. Defaults to 'unknown'. + digits (int, optional): kept digits of the hash. Defaults to None, + meaning all digits are kept. + + Returns: + str: Git commit hash. + """ + + if digits is not None and not isinstance(digits, int): + raise TypeError('digits must be None or an integer') + + try: + out = _minimal_ext_cmd(['git', 'rev-parse', 'HEAD']) + sha = out.strip().decode('ascii') + if digits is not None: + sha = sha[:digits] + except OSError: + sha = fallback + + return sha diff --git a/annotator/mmpkg/mmcv/version.py b/annotator/mmpkg/mmcv/version.py new file mode 100644 index 0000000000000000000000000000000000000000..1cce4e50bd692d4002e3cac3c545a3fb2efe95d0 --- /dev/null +++ b/annotator/mmpkg/mmcv/version.py @@ -0,0 +1,35 @@ +# Copyright (c) OpenMMLab. All rights reserved. +__version__ = '1.3.17' + + +def parse_version_info(version_str: str, length: int = 4) -> tuple: + """Parse a version string into a tuple. + + Args: + version_str (str): The version string. + length (int): The maximum number of version levels. Default: 4. + + Returns: + tuple[int | str]: The version info, e.g., "1.3.0" is parsed into + (1, 3, 0, 0, 0, 0), and "2.0.0rc1" is parsed into + (2, 0, 0, 0, 'rc', 1) (when length is set to 4). + """ + from packaging.version import parse + version = parse(version_str) + assert version.release, f'failed to parse version {version_str}' + release = list(version.release) + release = release[:length] + if len(release) < length: + release = release + [0] * (length - len(release)) + if version.is_prerelease: + release.extend(list(version.pre)) + elif version.is_postrelease: + release.extend(list(version.post)) + else: + release.extend([0, 0]) + return tuple(release) + + +version_info = tuple(int(x) for x in __version__.split('.')[:3]) + +__all__ = ['__version__', 'version_info', 'parse_version_info'] diff --git a/annotator/mmpkg/mmcv/video/__init__.py b/annotator/mmpkg/mmcv/video/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..73199b01dec52820dc6ca0139903536344d5a1eb --- /dev/null +++ b/annotator/mmpkg/mmcv/video/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .io import Cache, VideoReader, frames2video +from .optflow import (dequantize_flow, flow_from_bytes, flow_warp, flowread, + flowwrite, quantize_flow, sparse_flow_from_bytes) +from .processing import concat_video, convert_video, cut_video, resize_video + +__all__ = [ + 'Cache', 'VideoReader', 'frames2video', 'convert_video', 'resize_video', + 'cut_video', 'concat_video', 'flowread', 'flowwrite', 'quantize_flow', + 'dequantize_flow', 'flow_warp', 'flow_from_bytes', 'sparse_flow_from_bytes' +] diff --git a/annotator/mmpkg/mmcv/video/io.py b/annotator/mmpkg/mmcv/video/io.py new file mode 100644 index 0000000000000000000000000000000000000000..06ae9b8ae4404ec7822fd49c01c183a0be0cbf35 --- /dev/null +++ b/annotator/mmpkg/mmcv/video/io.py @@ -0,0 +1,318 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os.path as osp +from collections import OrderedDict + +import cv2 +from cv2 import (CAP_PROP_FOURCC, CAP_PROP_FPS, CAP_PROP_FRAME_COUNT, + CAP_PROP_FRAME_HEIGHT, CAP_PROP_FRAME_WIDTH, + CAP_PROP_POS_FRAMES, VideoWriter_fourcc) + +from annotator.mmpkg.mmcv.utils import (check_file_exist, mkdir_or_exist, scandir, + track_progress) + + +class Cache: + + def __init__(self, capacity): + self._cache = OrderedDict() + self._capacity = int(capacity) + if capacity <= 0: + raise ValueError('capacity must be a positive integer') + + @property + def capacity(self): + return self._capacity + + @property + def size(self): + return len(self._cache) + + def put(self, key, val): + if key in self._cache: + return + if len(self._cache) >= self.capacity: + self._cache.popitem(last=False) + self._cache[key] = val + + def get(self, key, default=None): + val = self._cache[key] if key in self._cache else default + return val + + +class VideoReader: + """Video class with similar usage to a list object. + + This video warpper class provides convenient apis to access frames. + There exists an issue of OpenCV's VideoCapture class that jumping to a + certain frame may be inaccurate. It is fixed in this class by checking + the position after jumping each time. + Cache is used when decoding videos. So if the same frame is visited for + the second time, there is no need to decode again if it is stored in the + cache. + + :Example: + + >>> import annotator.mmpkg.mmcv as mmcv + >>> v = mmcv.VideoReader('sample.mp4') + >>> len(v) # get the total frame number with `len()` + 120 + >>> for img in v: # v is iterable + >>> mmcv.imshow(img) + >>> v[5] # get the 6th frame + """ + + def __init__(self, filename, cache_capacity=10): + # Check whether the video path is a url + if not filename.startswith(('https://', 'http://')): + check_file_exist(filename, 'Video file not found: ' + filename) + self._vcap = cv2.VideoCapture(filename) + assert cache_capacity > 0 + self._cache = Cache(cache_capacity) + self._position = 0 + # get basic info + self._width = int(self._vcap.get(CAP_PROP_FRAME_WIDTH)) + self._height = int(self._vcap.get(CAP_PROP_FRAME_HEIGHT)) + self._fps = self._vcap.get(CAP_PROP_FPS) + self._frame_cnt = int(self._vcap.get(CAP_PROP_FRAME_COUNT)) + self._fourcc = self._vcap.get(CAP_PROP_FOURCC) + + @property + def vcap(self): + """:obj:`cv2.VideoCapture`: The raw VideoCapture object.""" + return self._vcap + + @property + def opened(self): + """bool: Indicate whether the video is opened.""" + return self._vcap.isOpened() + + @property + def width(self): + """int: Width of video frames.""" + return self._width + + @property + def height(self): + """int: Height of video frames.""" + return self._height + + @property + def resolution(self): + """tuple: Video resolution (width, height).""" + return (self._width, self._height) + + @property + def fps(self): + """float: FPS of the video.""" + return self._fps + + @property + def frame_cnt(self): + """int: Total frames of the video.""" + return self._frame_cnt + + @property + def fourcc(self): + """str: "Four character code" of the video.""" + return self._fourcc + + @property + def position(self): + """int: Current cursor position, indicating frame decoded.""" + return self._position + + def _get_real_position(self): + return int(round(self._vcap.get(CAP_PROP_POS_FRAMES))) + + def _set_real_position(self, frame_id): + self._vcap.set(CAP_PROP_POS_FRAMES, frame_id) + pos = self._get_real_position() + for _ in range(frame_id - pos): + self._vcap.read() + self._position = frame_id + + def read(self): + """Read the next frame. + + If the next frame have been decoded before and in the cache, then + return it directly, otherwise decode, cache and return it. + + Returns: + ndarray or None: Return the frame if successful, otherwise None. + """ + # pos = self._position + if self._cache: + img = self._cache.get(self._position) + if img is not None: + ret = True + else: + if self._position != self._get_real_position(): + self._set_real_position(self._position) + ret, img = self._vcap.read() + if ret: + self._cache.put(self._position, img) + else: + ret, img = self._vcap.read() + if ret: + self._position += 1 + return img + + def get_frame(self, frame_id): + """Get frame by index. + + Args: + frame_id (int): Index of the expected frame, 0-based. + + Returns: + ndarray or None: Return the frame if successful, otherwise None. + """ + if frame_id < 0 or frame_id >= self._frame_cnt: + raise IndexError( + f'"frame_id" must be between 0 and {self._frame_cnt - 1}') + if frame_id == self._position: + return self.read() + if self._cache: + img = self._cache.get(frame_id) + if img is not None: + self._position = frame_id + 1 + return img + self._set_real_position(frame_id) + ret, img = self._vcap.read() + if ret: + if self._cache: + self._cache.put(self._position, img) + self._position += 1 + return img + + def current_frame(self): + """Get the current frame (frame that is just visited). + + Returns: + ndarray or None: If the video is fresh, return None, otherwise + return the frame. + """ + if self._position == 0: + return None + return self._cache.get(self._position - 1) + + def cvt2frames(self, + frame_dir, + file_start=0, + filename_tmpl='{:06d}.jpg', + start=0, + max_num=0, + show_progress=True): + """Convert a video to frame images. + + Args: + frame_dir (str): Output directory to store all the frame images. + file_start (int): Filenames will start from the specified number. + filename_tmpl (str): Filename template with the index as the + placeholder. + start (int): The starting frame index. + max_num (int): Maximum number of frames to be written. + show_progress (bool): Whether to show a progress bar. + """ + mkdir_or_exist(frame_dir) + if max_num == 0: + task_num = self.frame_cnt - start + else: + task_num = min(self.frame_cnt - start, max_num) + if task_num <= 0: + raise ValueError('start must be less than total frame number') + if start > 0: + self._set_real_position(start) + + def write_frame(file_idx): + img = self.read() + if img is None: + return + filename = osp.join(frame_dir, filename_tmpl.format(file_idx)) + cv2.imwrite(filename, img) + + if show_progress: + track_progress(write_frame, range(file_start, + file_start + task_num)) + else: + for i in range(task_num): + write_frame(file_start + i) + + def __len__(self): + return self.frame_cnt + + def __getitem__(self, index): + if isinstance(index, slice): + return [ + self.get_frame(i) + for i in range(*index.indices(self.frame_cnt)) + ] + # support negative indexing + if index < 0: + index += self.frame_cnt + if index < 0: + raise IndexError('index out of range') + return self.get_frame(index) + + def __iter__(self): + self._set_real_position(0) + return self + + def __next__(self): + img = self.read() + if img is not None: + return img + else: + raise StopIteration + + next = __next__ + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self._vcap.release() + + +def frames2video(frame_dir, + video_file, + fps=30, + fourcc='XVID', + filename_tmpl='{:06d}.jpg', + start=0, + end=0, + show_progress=True): + """Read the frame images from a directory and join them as a video. + + Args: + frame_dir (str): The directory containing video frames. + video_file (str): Output filename. + fps (float): FPS of the output video. + fourcc (str): Fourcc of the output video, this should be compatible + with the output file type. + filename_tmpl (str): Filename template with the index as the variable. + start (int): Starting frame index. + end (int): Ending frame index. + show_progress (bool): Whether to show a progress bar. + """ + if end == 0: + ext = filename_tmpl.split('.')[-1] + end = len([name for name in scandir(frame_dir, ext)]) + first_file = osp.join(frame_dir, filename_tmpl.format(start)) + check_file_exist(first_file, 'The start frame not found: ' + first_file) + img = cv2.imread(first_file) + height, width = img.shape[:2] + resolution = (width, height) + vwriter = cv2.VideoWriter(video_file, VideoWriter_fourcc(*fourcc), fps, + resolution) + + def write_frame(file_idx): + filename = osp.join(frame_dir, filename_tmpl.format(file_idx)) + img = cv2.imread(filename) + vwriter.write(img) + + if show_progress: + track_progress(write_frame, range(start, end)) + else: + for i in range(start, end): + write_frame(i) + vwriter.release() diff --git a/annotator/mmpkg/mmcv/video/optflow.py b/annotator/mmpkg/mmcv/video/optflow.py new file mode 100644 index 0000000000000000000000000000000000000000..7bd78970dce8faf30bce0d5f2ec278b994fdd623 --- /dev/null +++ b/annotator/mmpkg/mmcv/video/optflow.py @@ -0,0 +1,254 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import warnings + +import cv2 +import numpy as np + +from annotator.mmpkg.mmcv.arraymisc import dequantize, quantize +from annotator.mmpkg.mmcv.image import imread, imwrite +from annotator.mmpkg.mmcv.utils import is_str + + +def flowread(flow_or_path, quantize=False, concat_axis=0, *args, **kwargs): + """Read an optical flow map. + + Args: + flow_or_path (ndarray or str): A flow map or filepath. + quantize (bool): whether to read quantized pair, if set to True, + remaining args will be passed to :func:`dequantize_flow`. + concat_axis (int): The axis that dx and dy are concatenated, + can be either 0 or 1. Ignored if quantize is False. + + Returns: + ndarray: Optical flow represented as a (h, w, 2) numpy array + """ + if isinstance(flow_or_path, np.ndarray): + if (flow_or_path.ndim != 3) or (flow_or_path.shape[-1] != 2): + raise ValueError(f'Invalid flow with shape {flow_or_path.shape}') + return flow_or_path + elif not is_str(flow_or_path): + raise TypeError(f'"flow_or_path" must be a filename or numpy array, ' + f'not {type(flow_or_path)}') + + if not quantize: + with open(flow_or_path, 'rb') as f: + try: + header = f.read(4).decode('utf-8') + except Exception: + raise IOError(f'Invalid flow file: {flow_or_path}') + else: + if header != 'PIEH': + raise IOError(f'Invalid flow file: {flow_or_path}, ' + 'header does not contain PIEH') + + w = np.fromfile(f, np.int32, 1).squeeze() + h = np.fromfile(f, np.int32, 1).squeeze() + flow = np.fromfile(f, np.float32, w * h * 2).reshape((h, w, 2)) + else: + assert concat_axis in [0, 1] + cat_flow = imread(flow_or_path, flag='unchanged') + if cat_flow.ndim != 2: + raise IOError( + f'{flow_or_path} is not a valid quantized flow file, ' + f'its dimension is {cat_flow.ndim}.') + assert cat_flow.shape[concat_axis] % 2 == 0 + dx, dy = np.split(cat_flow, 2, axis=concat_axis) + flow = dequantize_flow(dx, dy, *args, **kwargs) + + return flow.astype(np.float32) + + +def flowwrite(flow, filename, quantize=False, concat_axis=0, *args, **kwargs): + """Write optical flow to file. + + If the flow is not quantized, it will be saved as a .flo file losslessly, + otherwise a jpeg image which is lossy but of much smaller size. (dx and dy + will be concatenated horizontally into a single image if quantize is True.) + + Args: + flow (ndarray): (h, w, 2) array of optical flow. + filename (str): Output filepath. + quantize (bool): Whether to quantize the flow and save it to 2 jpeg + images. If set to True, remaining args will be passed to + :func:`quantize_flow`. + concat_axis (int): The axis that dx and dy are concatenated, + can be either 0 or 1. Ignored if quantize is False. + """ + if not quantize: + with open(filename, 'wb') as f: + f.write('PIEH'.encode('utf-8')) + np.array([flow.shape[1], flow.shape[0]], dtype=np.int32).tofile(f) + flow = flow.astype(np.float32) + flow.tofile(f) + f.flush() + else: + assert concat_axis in [0, 1] + dx, dy = quantize_flow(flow, *args, **kwargs) + dxdy = np.concatenate((dx, dy), axis=concat_axis) + imwrite(dxdy, filename) + + +def quantize_flow(flow, max_val=0.02, norm=True): + """Quantize flow to [0, 255]. + + After this step, the size of flow will be much smaller, and can be + dumped as jpeg images. + + Args: + flow (ndarray): (h, w, 2) array of optical flow. + max_val (float): Maximum value of flow, values beyond + [-max_val, max_val] will be truncated. + norm (bool): Whether to divide flow values by image width/height. + + Returns: + tuple[ndarray]: Quantized dx and dy. + """ + h, w, _ = flow.shape + dx = flow[..., 0] + dy = flow[..., 1] + if norm: + dx = dx / w # avoid inplace operations + dy = dy / h + # use 255 levels instead of 256 to make sure 0 is 0 after dequantization. + flow_comps = [ + quantize(d, -max_val, max_val, 255, np.uint8) for d in [dx, dy] + ] + return tuple(flow_comps) + + +def dequantize_flow(dx, dy, max_val=0.02, denorm=True): + """Recover from quantized flow. + + Args: + dx (ndarray): Quantized dx. + dy (ndarray): Quantized dy. + max_val (float): Maximum value used when quantizing. + denorm (bool): Whether to multiply flow values with width/height. + + Returns: + ndarray: Dequantized flow. + """ + assert dx.shape == dy.shape + assert dx.ndim == 2 or (dx.ndim == 3 and dx.shape[-1] == 1) + + dx, dy = [dequantize(d, -max_val, max_val, 255) for d in [dx, dy]] + + if denorm: + dx *= dx.shape[1] + dy *= dx.shape[0] + flow = np.dstack((dx, dy)) + return flow + + +def flow_warp(img, flow, filling_value=0, interpolate_mode='nearest'): + """Use flow to warp img. + + Args: + img (ndarray, float or uint8): Image to be warped. + flow (ndarray, float): Optical Flow. + filling_value (int): The missing pixels will be set with filling_value. + interpolate_mode (str): bilinear -> Bilinear Interpolation; + nearest -> Nearest Neighbor. + + Returns: + ndarray: Warped image with the same shape of img + """ + warnings.warn('This function is just for prototyping and cannot ' + 'guarantee the computational efficiency.') + assert flow.ndim == 3, 'Flow must be in 3D arrays.' + height = flow.shape[0] + width = flow.shape[1] + channels = img.shape[2] + + output = np.ones( + (height, width, channels), dtype=img.dtype) * filling_value + + grid = np.indices((height, width)).swapaxes(0, 1).swapaxes(1, 2) + dx = grid[:, :, 0] + flow[:, :, 1] + dy = grid[:, :, 1] + flow[:, :, 0] + sx = np.floor(dx).astype(int) + sy = np.floor(dy).astype(int) + valid = (sx >= 0) & (sx < height - 1) & (sy >= 0) & (sy < width - 1) + + if interpolate_mode == 'nearest': + output[valid, :] = img[dx[valid].round().astype(int), + dy[valid].round().astype(int), :] + elif interpolate_mode == 'bilinear': + # dirty walkround for integer positions + eps_ = 1e-6 + dx, dy = dx + eps_, dy + eps_ + left_top_ = img[np.floor(dx[valid]).astype(int), + np.floor(dy[valid]).astype(int), :] * ( + np.ceil(dx[valid]) - dx[valid])[:, None] * ( + np.ceil(dy[valid]) - dy[valid])[:, None] + left_down_ = img[np.ceil(dx[valid]).astype(int), + np.floor(dy[valid]).astype(int), :] * ( + dx[valid] - np.floor(dx[valid]))[:, None] * ( + np.ceil(dy[valid]) - dy[valid])[:, None] + right_top_ = img[np.floor(dx[valid]).astype(int), + np.ceil(dy[valid]).astype(int), :] * ( + np.ceil(dx[valid]) - dx[valid])[:, None] * ( + dy[valid] - np.floor(dy[valid]))[:, None] + right_down_ = img[np.ceil(dx[valid]).astype(int), + np.ceil(dy[valid]).astype(int), :] * ( + dx[valid] - np.floor(dx[valid]))[:, None] * ( + dy[valid] - np.floor(dy[valid]))[:, None] + output[valid, :] = left_top_ + left_down_ + right_top_ + right_down_ + else: + raise NotImplementedError( + 'We only support interpolation modes of nearest and bilinear, ' + f'but got {interpolate_mode}.') + return output.astype(img.dtype) + + +def flow_from_bytes(content): + """Read dense optical flow from bytes. + + .. note:: + This load optical flow function works for FlyingChairs, FlyingThings3D, + Sintel, FlyingChairsOcc datasets, but cannot load the data from + ChairsSDHom. + + Args: + content (bytes): Optical flow bytes got from files or other streams. + + Returns: + ndarray: Loaded optical flow with the shape (H, W, 2). + """ + + # header in first 4 bytes + header = content[:4] + if header.decode('utf-8') != 'PIEH': + raise Exception('Flow file header does not contain PIEH') + # width in second 4 bytes + width = np.frombuffer(content[4:], np.int32, 1).squeeze() + # height in third 4 bytes + height = np.frombuffer(content[8:], np.int32, 1).squeeze() + # after first 12 bytes, all bytes are flow + flow = np.frombuffer(content[12:], np.float32, width * height * 2).reshape( + (height, width, 2)) + + return flow + + +def sparse_flow_from_bytes(content): + """Read the optical flow in KITTI datasets from bytes. + + This function is modified from RAFT load the `KITTI datasets + `_. + + Args: + content (bytes): Optical flow bytes got from files or other streams. + + Returns: + Tuple(ndarray, ndarray): Loaded optical flow with the shape (H, W, 2) + and flow valid mask with the shape (H, W). + """ # nopa + + content = np.frombuffer(content, np.uint8) + flow = cv2.imdecode(content, cv2.IMREAD_ANYDEPTH | cv2.IMREAD_COLOR) + flow = flow[:, :, ::-1].astype(np.float32) + # flow shape (H, W, 2) valid shape (H, W) + flow, valid = flow[:, :, :2], flow[:, :, 2] + flow = (flow - 2**15) / 64.0 + return flow, valid diff --git a/annotator/mmpkg/mmcv/video/processing.py b/annotator/mmpkg/mmcv/video/processing.py new file mode 100644 index 0000000000000000000000000000000000000000..2b93a59215d56b6e5ba05f48bca3527772f0c744 --- /dev/null +++ b/annotator/mmpkg/mmcv/video/processing.py @@ -0,0 +1,160 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os +import os.path as osp +import subprocess +import tempfile + +from annotator.mmpkg.mmcv.utils import requires_executable + + +@requires_executable('ffmpeg') +def convert_video(in_file, + out_file, + print_cmd=False, + pre_options='', + **kwargs): + """Convert a video with ffmpeg. + + This provides a general api to ffmpeg, the executed command is:: + + `ffmpeg -y -i ` + + Options(kwargs) are mapped to ffmpeg commands with the following rules: + + - key=val: "-key val" + - key=True: "-key" + - key=False: "" + + Args: + in_file (str): Input video filename. + out_file (str): Output video filename. + pre_options (str): Options appears before "-i ". + print_cmd (bool): Whether to print the final ffmpeg command. + """ + options = [] + for k, v in kwargs.items(): + if isinstance(v, bool): + if v: + options.append(f'-{k}') + elif k == 'log_level': + assert v in [ + 'quiet', 'panic', 'fatal', 'error', 'warning', 'info', + 'verbose', 'debug', 'trace' + ] + options.append(f'-loglevel {v}') + else: + options.append(f'-{k} {v}') + cmd = f'ffmpeg -y {pre_options} -i {in_file} {" ".join(options)} ' \ + f'{out_file}' + if print_cmd: + print(cmd) + subprocess.call(cmd, shell=True) + + +@requires_executable('ffmpeg') +def resize_video(in_file, + out_file, + size=None, + ratio=None, + keep_ar=False, + log_level='info', + print_cmd=False): + """Resize a video. + + Args: + in_file (str): Input video filename. + out_file (str): Output video filename. + size (tuple): Expected size (w, h), eg, (320, 240) or (320, -1). + ratio (tuple or float): Expected resize ratio, (2, 0.5) means + (w*2, h*0.5). + keep_ar (bool): Whether to keep original aspect ratio. + log_level (str): Logging level of ffmpeg. + print_cmd (bool): Whether to print the final ffmpeg command. + """ + if size is None and ratio is None: + raise ValueError('expected size or ratio must be specified') + if size is not None and ratio is not None: + raise ValueError('size and ratio cannot be specified at the same time') + options = {'log_level': log_level} + if size: + if not keep_ar: + options['vf'] = f'scale={size[0]}:{size[1]}' + else: + options['vf'] = f'scale=w={size[0]}:h={size[1]}:' \ + 'force_original_aspect_ratio=decrease' + else: + if not isinstance(ratio, tuple): + ratio = (ratio, ratio) + options['vf'] = f'scale="trunc(iw*{ratio[0]}):trunc(ih*{ratio[1]})"' + convert_video(in_file, out_file, print_cmd, **options) + + +@requires_executable('ffmpeg') +def cut_video(in_file, + out_file, + start=None, + end=None, + vcodec=None, + acodec=None, + log_level='info', + print_cmd=False): + """Cut a clip from a video. + + Args: + in_file (str): Input video filename. + out_file (str): Output video filename. + start (None or float): Start time (in seconds). + end (None or float): End time (in seconds). + vcodec (None or str): Output video codec, None for unchanged. + acodec (None or str): Output audio codec, None for unchanged. + log_level (str): Logging level of ffmpeg. + print_cmd (bool): Whether to print the final ffmpeg command. + """ + options = {'log_level': log_level} + if vcodec is None: + options['vcodec'] = 'copy' + if acodec is None: + options['acodec'] = 'copy' + if start: + options['ss'] = start + else: + start = 0 + if end: + options['t'] = end - start + convert_video(in_file, out_file, print_cmd, **options) + + +@requires_executable('ffmpeg') +def concat_video(video_list, + out_file, + vcodec=None, + acodec=None, + log_level='info', + print_cmd=False): + """Concatenate multiple videos into a single one. + + Args: + video_list (list): A list of video filenames + out_file (str): Output video filename + vcodec (None or str): Output video codec, None for unchanged + acodec (None or str): Output audio codec, None for unchanged + log_level (str): Logging level of ffmpeg. + print_cmd (bool): Whether to print the final ffmpeg command. + """ + tmp_filehandler, tmp_filename = tempfile.mkstemp(suffix='.txt', text=True) + with open(tmp_filename, 'w') as f: + for filename in video_list: + f.write(f'file {osp.abspath(filename)}\n') + options = {'log_level': log_level} + if vcodec is None: + options['vcodec'] = 'copy' + if acodec is None: + options['acodec'] = 'copy' + convert_video( + tmp_filename, + out_file, + print_cmd, + pre_options='-f concat -safe 0', + **options) + os.close(tmp_filehandler) + os.remove(tmp_filename) diff --git a/annotator/mmpkg/mmcv/visualization/__init__.py b/annotator/mmpkg/mmcv/visualization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..835df136bdcf69348281d22914d41aa84cdf92b1 --- /dev/null +++ b/annotator/mmpkg/mmcv/visualization/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .color import Color, color_val +from .image import imshow, imshow_bboxes, imshow_det_bboxes +from .optflow import flow2rgb, flowshow, make_color_wheel + +__all__ = [ + 'Color', 'color_val', 'imshow', 'imshow_bboxes', 'imshow_det_bboxes', + 'flowshow', 'flow2rgb', 'make_color_wheel' +] diff --git a/annotator/mmpkg/mmcv/visualization/color.py b/annotator/mmpkg/mmcv/visualization/color.py new file mode 100644 index 0000000000000000000000000000000000000000..48379a283e48570f226426510270de8e15323c8d --- /dev/null +++ b/annotator/mmpkg/mmcv/visualization/color.py @@ -0,0 +1,51 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from enum import Enum + +import numpy as np + +from annotator.mmpkg.mmcv.utils import is_str + + +class Color(Enum): + """An enum that defines common colors. + + Contains red, green, blue, cyan, yellow, magenta, white and black. + """ + red = (0, 0, 255) + green = (0, 255, 0) + blue = (255, 0, 0) + cyan = (255, 255, 0) + yellow = (0, 255, 255) + magenta = (255, 0, 255) + white = (255, 255, 255) + black = (0, 0, 0) + + +def color_val(color): + """Convert various input to color tuples. + + Args: + color (:obj:`Color`/str/tuple/int/ndarray): Color inputs + + Returns: + tuple[int]: A tuple of 3 integers indicating BGR channels. + """ + if is_str(color): + return Color[color].value + elif isinstance(color, Color): + return color.value + elif isinstance(color, tuple): + assert len(color) == 3 + for channel in color: + assert 0 <= channel <= 255 + return color + elif isinstance(color, int): + assert 0 <= color <= 255 + return color, color, color + elif isinstance(color, np.ndarray): + assert color.ndim == 1 and color.size == 3 + assert np.all((color >= 0) & (color <= 255)) + color = color.astype(np.uint8) + return tuple(color) + else: + raise TypeError(f'Invalid type for color: {type(color)}') diff --git a/annotator/mmpkg/mmcv/visualization/image.py b/annotator/mmpkg/mmcv/visualization/image.py new file mode 100644 index 0000000000000000000000000000000000000000..378de2104f6554389fcb2e6a3904283345fd74b0 --- /dev/null +++ b/annotator/mmpkg/mmcv/visualization/image.py @@ -0,0 +1,152 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import cv2 +import numpy as np + +from annotator.mmpkg.mmcv.image import imread, imwrite +from .color import color_val + + +def imshow(img, win_name='', wait_time=0): + """Show an image. + + Args: + img (str or ndarray): The image to be displayed. + win_name (str): The window name. + wait_time (int): Value of waitKey param. + """ + cv2.imshow(win_name, imread(img)) + if wait_time == 0: # prevent from hanging if windows was closed + while True: + ret = cv2.waitKey(1) + + closed = cv2.getWindowProperty(win_name, cv2.WND_PROP_VISIBLE) < 1 + # if user closed window or if some key pressed + if closed or ret != -1: + break + else: + ret = cv2.waitKey(wait_time) + + +def imshow_bboxes(img, + bboxes, + colors='green', + top_k=-1, + thickness=1, + show=True, + win_name='', + wait_time=0, + out_file=None): + """Draw bboxes on an image. + + Args: + img (str or ndarray): The image to be displayed. + bboxes (list or ndarray): A list of ndarray of shape (k, 4). + colors (list[str or tuple or Color]): A list of colors. + top_k (int): Plot the first k bboxes only if set positive. + thickness (int): Thickness of lines. + show (bool): Whether to show the image. + win_name (str): The window name. + wait_time (int): Value of waitKey param. + out_file (str, optional): The filename to write the image. + + Returns: + ndarray: The image with bboxes drawn on it. + """ + img = imread(img) + img = np.ascontiguousarray(img) + + if isinstance(bboxes, np.ndarray): + bboxes = [bboxes] + if not isinstance(colors, list): + colors = [colors for _ in range(len(bboxes))] + colors = [color_val(c) for c in colors] + assert len(bboxes) == len(colors) + + for i, _bboxes in enumerate(bboxes): + _bboxes = _bboxes.astype(np.int32) + if top_k <= 0: + _top_k = _bboxes.shape[0] + else: + _top_k = min(top_k, _bboxes.shape[0]) + for j in range(_top_k): + left_top = (_bboxes[j, 0], _bboxes[j, 1]) + right_bottom = (_bboxes[j, 2], _bboxes[j, 3]) + cv2.rectangle( + img, left_top, right_bottom, colors[i], thickness=thickness) + + if show: + imshow(img, win_name, wait_time) + if out_file is not None: + imwrite(img, out_file) + return img + + +def imshow_det_bboxes(img, + bboxes, + labels, + class_names=None, + score_thr=0, + bbox_color='green', + text_color='green', + thickness=1, + font_scale=0.5, + show=True, + win_name='', + wait_time=0, + out_file=None): + """Draw bboxes and class labels (with scores) on an image. + + Args: + img (str or ndarray): The image to be displayed. + bboxes (ndarray): Bounding boxes (with scores), shaped (n, 4) or + (n, 5). + labels (ndarray): Labels of bboxes. + class_names (list[str]): Names of each classes. + score_thr (float): Minimum score of bboxes to be shown. + bbox_color (str or tuple or :obj:`Color`): Color of bbox lines. + text_color (str or tuple or :obj:`Color`): Color of texts. + thickness (int): Thickness of lines. + font_scale (float): Font scales of texts. + show (bool): Whether to show the image. + win_name (str): The window name. + wait_time (int): Value of waitKey param. + out_file (str or None): The filename to write the image. + + Returns: + ndarray: The image with bboxes drawn on it. + """ + assert bboxes.ndim == 2 + assert labels.ndim == 1 + assert bboxes.shape[0] == labels.shape[0] + assert bboxes.shape[1] == 4 or bboxes.shape[1] == 5 + img = imread(img) + img = np.ascontiguousarray(img) + + if score_thr > 0: + assert bboxes.shape[1] == 5 + scores = bboxes[:, -1] + inds = scores > score_thr + bboxes = bboxes[inds, :] + labels = labels[inds] + + bbox_color = color_val(bbox_color) + text_color = color_val(text_color) + + for bbox, label in zip(bboxes, labels): + bbox_int = bbox.astype(np.int32) + left_top = (bbox_int[0], bbox_int[1]) + right_bottom = (bbox_int[2], bbox_int[3]) + cv2.rectangle( + img, left_top, right_bottom, bbox_color, thickness=thickness) + label_text = class_names[ + label] if class_names is not None else f'cls {label}' + if len(bbox) > 4: + label_text += f'|{bbox[-1]:.02f}' + cv2.putText(img, label_text, (bbox_int[0], bbox_int[1] - 2), + cv2.FONT_HERSHEY_COMPLEX, font_scale, text_color) + + if show: + imshow(img, win_name, wait_time) + if out_file is not None: + imwrite(img, out_file) + return img diff --git a/annotator/mmpkg/mmcv/visualization/optflow.py b/annotator/mmpkg/mmcv/visualization/optflow.py new file mode 100644 index 0000000000000000000000000000000000000000..b4c3ce980f9f6c74c85fe714aca1623a08ae7a8d --- /dev/null +++ b/annotator/mmpkg/mmcv/visualization/optflow.py @@ -0,0 +1,112 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from __future__ import division + +import numpy as np + +from annotator.mmpkg.mmcv.image import rgb2bgr +from annotator.mmpkg.mmcv.video import flowread +from .image import imshow + + +def flowshow(flow, win_name='', wait_time=0): + """Show optical flow. + + Args: + flow (ndarray or str): The optical flow to be displayed. + win_name (str): The window name. + wait_time (int): Value of waitKey param. + """ + flow = flowread(flow) + flow_img = flow2rgb(flow) + imshow(rgb2bgr(flow_img), win_name, wait_time) + + +def flow2rgb(flow, color_wheel=None, unknown_thr=1e6): + """Convert flow map to RGB image. + + Args: + flow (ndarray): Array of optical flow. + color_wheel (ndarray or None): Color wheel used to map flow field to + RGB colorspace. Default color wheel will be used if not specified. + unknown_thr (str): Values above this threshold will be marked as + unknown and thus ignored. + + Returns: + ndarray: RGB image that can be visualized. + """ + assert flow.ndim == 3 and flow.shape[-1] == 2 + if color_wheel is None: + color_wheel = make_color_wheel() + assert color_wheel.ndim == 2 and color_wheel.shape[1] == 3 + num_bins = color_wheel.shape[0] + + dx = flow[:, :, 0].copy() + dy = flow[:, :, 1].copy() + + ignore_inds = ( + np.isnan(dx) | np.isnan(dy) | (np.abs(dx) > unknown_thr) | + (np.abs(dy) > unknown_thr)) + dx[ignore_inds] = 0 + dy[ignore_inds] = 0 + + rad = np.sqrt(dx**2 + dy**2) + if np.any(rad > np.finfo(float).eps): + max_rad = np.max(rad) + dx /= max_rad + dy /= max_rad + + rad = np.sqrt(dx**2 + dy**2) + angle = np.arctan2(-dy, -dx) / np.pi + + bin_real = (angle + 1) / 2 * (num_bins - 1) + bin_left = np.floor(bin_real).astype(int) + bin_right = (bin_left + 1) % num_bins + w = (bin_real - bin_left.astype(np.float32))[..., None] + flow_img = (1 - + w) * color_wheel[bin_left, :] + w * color_wheel[bin_right, :] + small_ind = rad <= 1 + flow_img[small_ind] = 1 - rad[small_ind, None] * (1 - flow_img[small_ind]) + flow_img[np.logical_not(small_ind)] *= 0.75 + + flow_img[ignore_inds, :] = 0 + + return flow_img + + +def make_color_wheel(bins=None): + """Build a color wheel. + + Args: + bins(list or tuple, optional): Specify the number of bins for each + color range, corresponding to six ranges: red -> yellow, + yellow -> green, green -> cyan, cyan -> blue, blue -> magenta, + magenta -> red. [15, 6, 4, 11, 13, 6] is used for default + (see Middlebury). + + Returns: + ndarray: Color wheel of shape (total_bins, 3). + """ + if bins is None: + bins = [15, 6, 4, 11, 13, 6] + assert len(bins) == 6 + + RY, YG, GC, CB, BM, MR = tuple(bins) + + ry = [1, np.arange(RY) / RY, 0] + yg = [1 - np.arange(YG) / YG, 1, 0] + gc = [0, 1, np.arange(GC) / GC] + cb = [0, 1 - np.arange(CB) / CB, 1] + bm = [np.arange(BM) / BM, 0, 1] + mr = [1, 0, 1 - np.arange(MR) / MR] + + num_bins = RY + YG + GC + CB + BM + MR + + color_wheel = np.zeros((3, num_bins), dtype=np.float32) + + col = 0 + for i, color in enumerate([ry, yg, gc, cb, bm, mr]): + for j in range(3): + color_wheel[j, col:col + bins[i]] = color[j] + col += bins[i] + + return color_wheel.T diff --git a/annotator/mmpkg/mmseg/apis/__init__.py b/annotator/mmpkg/mmseg/apis/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..170724be38de42daf2bc1a1910e181d68818f165 --- /dev/null +++ b/annotator/mmpkg/mmseg/apis/__init__.py @@ -0,0 +1,9 @@ +from .inference import inference_segmentor, init_segmentor, show_result_pyplot +from .test import multi_gpu_test, single_gpu_test +from .train import get_root_logger, set_random_seed, train_segmentor + +__all__ = [ + 'get_root_logger', 'set_random_seed', 'train_segmentor', 'init_segmentor', + 'inference_segmentor', 'multi_gpu_test', 'single_gpu_test', + 'show_result_pyplot' +] diff --git a/annotator/mmpkg/mmseg/apis/inference.py b/annotator/mmpkg/mmseg/apis/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..515e459ff6e66e955624fedaf32d2076be750563 --- /dev/null +++ b/annotator/mmpkg/mmseg/apis/inference.py @@ -0,0 +1,138 @@ +import matplotlib.pyplot as plt +import annotator.mmpkg.mmcv as mmcv +import torch +from annotator.mmpkg.mmcv.parallel import collate, scatter +from annotator.mmpkg.mmcv.runner import load_checkpoint + +from annotator.mmpkg.mmseg.datasets.pipelines import Compose +from annotator.mmpkg.mmseg.models import build_segmentor +from modules import devices + + +def init_segmentor(config, checkpoint=None, device=devices.get_device_for("controlnet")): + """Initialize a segmentor from config file. + + Args: + config (str or :obj:`mmcv.Config`): Config file path or the config + object. + checkpoint (str, optional): Checkpoint path. If left as None, the model + will not load any weights. + device (str, optional) CPU/CUDA device option. Default 'cuda:0'. + Use 'cpu' for loading model on CPU. + Returns: + nn.Module: The constructed segmentor. + """ + if isinstance(config, str): + config = mmcv.Config.fromfile(config) + elif not isinstance(config, mmcv.Config): + raise TypeError('config must be a filename or Config object, ' + 'but got {}'.format(type(config))) + config.model.pretrained = None + config.model.train_cfg = None + model = build_segmentor(config.model, test_cfg=config.get('test_cfg')) + if checkpoint is not None: + checkpoint = load_checkpoint(model, checkpoint, map_location='cpu') + model.CLASSES = checkpoint['meta']['CLASSES'] + model.PALETTE = checkpoint['meta']['PALETTE'] + model.cfg = config # save the config in the model for convenience + model.to(device) + model.eval() + return model + + +class LoadImage: + """A simple pipeline to load image.""" + + def __call__(self, results): + """Call function to load images into results. + + Args: + results (dict): A result dict contains the file name + of the image to be read. + + Returns: + dict: ``results`` will be returned containing loaded image. + """ + + if isinstance(results['img'], str): + results['filename'] = results['img'] + results['ori_filename'] = results['img'] + else: + results['filename'] = None + results['ori_filename'] = None + img = mmcv.imread(results['img']) + results['img'] = img + results['img_shape'] = img.shape + results['ori_shape'] = img.shape + return results + + +def inference_segmentor(model, img): + """Inference image(s) with the segmentor. + + Args: + model (nn.Module): The loaded segmentor. + imgs (str/ndarray or list[str/ndarray]): Either image files or loaded + images. + + Returns: + (list[Tensor]): The segmentation result. + """ + cfg = model.cfg + device = next(model.parameters()).device # model device + # build the data pipeline + test_pipeline = [LoadImage()] + cfg.data.test.pipeline[1:] + test_pipeline = Compose(test_pipeline) + # prepare data + data = dict(img=img) + data = test_pipeline(data) + data = collate([data], samples_per_gpu=1) + if next(model.parameters()).is_cuda: + # scatter to specified GPU + data = scatter(data, [device])[0] + else: + data['img'][0] = data['img'][0].to(devices.get_device_for("controlnet")) + data['img_metas'] = [i.data[0] for i in data['img_metas']] + + # forward the model + with torch.no_grad(): + result = model(return_loss=False, rescale=True, **data) + return result + + +def show_result_pyplot(model, + img, + result, + palette=None, + fig_size=(15, 10), + opacity=0.5, + title='', + block=True): + """Visualize the segmentation results on the image. + + Args: + model (nn.Module): The loaded segmentor. + img (str or np.ndarray): Image filename or loaded image. + result (list): The segmentation result. + palette (list[list[int]]] | None): The palette of segmentation + map. If None is given, random palette will be generated. + Default: None + fig_size (tuple): Figure size of the pyplot figure. + opacity(float): Opacity of painted segmentation map. + Default 0.5. + Must be in (0, 1] range. + title (str): The title of pyplot figure. + Default is ''. + block (bool): Whether to block the pyplot figure. + Default is True. + """ + if hasattr(model, 'module'): + model = model.module + img = model.show_result( + img, result, palette=palette, show=False, opacity=opacity) + # plt.figure(figsize=fig_size) + # plt.imshow(mmcv.bgr2rgb(img)) + # plt.title(title) + # plt.tight_layout() + # plt.show(block=block) + return mmcv.bgr2rgb(img) diff --git a/annotator/mmpkg/mmseg/apis/test.py b/annotator/mmpkg/mmseg/apis/test.py new file mode 100644 index 0000000000000000000000000000000000000000..f9954e6a3709afdbf6a2027b213afcad644c47d7 --- /dev/null +++ b/annotator/mmpkg/mmseg/apis/test.py @@ -0,0 +1,238 @@ +import os.path as osp +import pickle +import shutil +import tempfile + +import annotator.mmpkg.mmcv as mmcv +import numpy as np +import torch +import torch.distributed as dist +from annotator.mmpkg.mmcv.image import tensor2imgs +from annotator.mmpkg.mmcv.runner import get_dist_info + + +def np2tmp(array, temp_file_name=None): + """Save ndarray to local numpy file. + + Args: + array (ndarray): Ndarray to save. + temp_file_name (str): Numpy file name. If 'temp_file_name=None', this + function will generate a file name with tempfile.NamedTemporaryFile + to save ndarray. Default: None. + + Returns: + str: The numpy file name. + """ + + if temp_file_name is None: + temp_file_name = tempfile.NamedTemporaryFile( + suffix='.npy', delete=False).name + np.save(temp_file_name, array) + return temp_file_name + + +def single_gpu_test(model, + data_loader, + show=False, + out_dir=None, + efficient_test=False, + opacity=0.5): + """Test with single GPU. + + Args: + model (nn.Module): Model to be tested. + data_loader (utils.data.Dataloader): Pytorch data loader. + show (bool): Whether show results during inference. Default: False. + out_dir (str, optional): If specified, the results will be dumped into + the directory to save output results. + efficient_test (bool): Whether save the results as local numpy files to + save CPU memory during evaluation. Default: False. + opacity(float): Opacity of painted segmentation map. + Default 0.5. + Must be in (0, 1] range. + Returns: + list: The prediction results. + """ + + model.eval() + results = [] + dataset = data_loader.dataset + prog_bar = mmcv.ProgressBar(len(dataset)) + for i, data in enumerate(data_loader): + with torch.no_grad(): + result = model(return_loss=False, **data) + + if show or out_dir: + img_tensor = data['img'][0] + img_metas = data['img_metas'][0].data[0] + imgs = tensor2imgs(img_tensor, **img_metas[0]['img_norm_cfg']) + assert len(imgs) == len(img_metas) + + for img, img_meta in zip(imgs, img_metas): + h, w, _ = img_meta['img_shape'] + img_show = img[:h, :w, :] + + ori_h, ori_w = img_meta['ori_shape'][:-1] + img_show = mmcv.imresize(img_show, (ori_w, ori_h)) + + if out_dir: + out_file = osp.join(out_dir, img_meta['ori_filename']) + else: + out_file = None + + model.module.show_result( + img_show, + result, + palette=dataset.PALETTE, + show=show, + out_file=out_file, + opacity=opacity) + + if isinstance(result, list): + if efficient_test: + result = [np2tmp(_) for _ in result] + results.extend(result) + else: + if efficient_test: + result = np2tmp(result) + results.append(result) + + batch_size = len(result) + for _ in range(batch_size): + prog_bar.update() + return results + + +def multi_gpu_test(model, + data_loader, + tmpdir=None, + gpu_collect=False, + efficient_test=False): + """Test model with multiple gpus. + + This method tests model with multiple gpus and collects the results + under two different modes: gpu and cpu modes. By setting 'gpu_collect=True' + it encodes results to gpu tensors and use gpu communication for results + collection. On cpu mode it saves the results on different gpus to 'tmpdir' + and collects them by the rank 0 worker. + + Args: + model (nn.Module): Model to be tested. + data_loader (utils.data.Dataloader): Pytorch data loader. + tmpdir (str): Path of directory to save the temporary results from + different gpus under cpu mode. + gpu_collect (bool): Option to use either gpu or cpu to collect results. + efficient_test (bool): Whether save the results as local numpy files to + save CPU memory during evaluation. Default: False. + + Returns: + list: The prediction results. + """ + + model.eval() + results = [] + dataset = data_loader.dataset + rank, world_size = get_dist_info() + if rank == 0: + prog_bar = mmcv.ProgressBar(len(dataset)) + for i, data in enumerate(data_loader): + with torch.no_grad(): + result = model(return_loss=False, rescale=True, **data) + + if isinstance(result, list): + if efficient_test: + result = [np2tmp(_) for _ in result] + results.extend(result) + else: + if efficient_test: + result = np2tmp(result) + results.append(result) + + if rank == 0: + batch_size = data['img'][0].size(0) + for _ in range(batch_size * world_size): + prog_bar.update() + + # collect results from all ranks + if gpu_collect: + results = collect_results_gpu(results, len(dataset)) + else: + results = collect_results_cpu(results, len(dataset), tmpdir) + return results + + +def collect_results_cpu(result_part, size, tmpdir=None): + """Collect results with CPU.""" + rank, world_size = get_dist_info() + # create a tmp dir if it is not specified + if tmpdir is None: + MAX_LEN = 512 + # 32 is whitespace + dir_tensor = torch.full((MAX_LEN, ), + 32, + dtype=torch.uint8, + device='cuda') + if rank == 0: + tmpdir = tempfile.mkdtemp() + tmpdir = torch.tensor( + bytearray(tmpdir.encode()), dtype=torch.uint8, device='cuda') + dir_tensor[:len(tmpdir)] = tmpdir + dist.broadcast(dir_tensor, 0) + tmpdir = dir_tensor.cpu().numpy().tobytes().decode().rstrip() + else: + mmcv.mkdir_or_exist(tmpdir) + # dump the part result to the dir + mmcv.dump(result_part, osp.join(tmpdir, 'part_{}.pkl'.format(rank))) + dist.barrier() + # collect all parts + if rank != 0: + return None + else: + # load results of all parts from tmp dir + part_list = [] + for i in range(world_size): + part_file = osp.join(tmpdir, 'part_{}.pkl'.format(i)) + part_list.append(mmcv.load(part_file)) + # sort the results + ordered_results = [] + for res in zip(*part_list): + ordered_results.extend(list(res)) + # the dataloader may pad some samples + ordered_results = ordered_results[:size] + # remove tmp dir + shutil.rmtree(tmpdir) + return ordered_results + + +def collect_results_gpu(result_part, size): + """Collect results with GPU.""" + rank, world_size = get_dist_info() + # dump result part to tensor with pickle + part_tensor = torch.tensor( + bytearray(pickle.dumps(result_part)), dtype=torch.uint8, device='cuda') + # gather all result part tensor shape + shape_tensor = torch.tensor(part_tensor.shape, device='cuda') + shape_list = [shape_tensor.clone() for _ in range(world_size)] + dist.all_gather(shape_list, shape_tensor) + # padding result part tensor to max length + shape_max = torch.tensor(shape_list).max() + part_send = torch.zeros(shape_max, dtype=torch.uint8, device='cuda') + part_send[:shape_tensor[0]] = part_tensor + part_recv_list = [ + part_tensor.new_zeros(shape_max) for _ in range(world_size) + ] + # gather all result part + dist.all_gather(part_recv_list, part_send) + + if rank == 0: + part_list = [] + for recv, shape in zip(part_recv_list, shape_list): + part_list.append( + pickle.loads(recv[:shape[0]].cpu().numpy().tobytes())) + # sort the results + ordered_results = [] + for res in zip(*part_list): + ordered_results.extend(list(res)) + # the dataloader may pad some samples + ordered_results = ordered_results[:size] + return ordered_results diff --git a/annotator/mmpkg/mmseg/apis/train.py b/annotator/mmpkg/mmseg/apis/train.py new file mode 100644 index 0000000000000000000000000000000000000000..f0a87d65c72e4581c96b41aebf879905510c9d22 --- /dev/null +++ b/annotator/mmpkg/mmseg/apis/train.py @@ -0,0 +1,116 @@ +import random +import warnings + +import numpy as np +import torch +from annotator.mmpkg.mmcv.parallel import MMDataParallel, MMDistributedDataParallel +from annotator.mmpkg.mmcv.runner import build_optimizer, build_runner + +from annotator.mmpkg.mmseg.core import DistEvalHook, EvalHook +from annotator.mmpkg.mmseg.datasets import build_dataloader, build_dataset +from annotator.mmpkg.mmseg.utils import get_root_logger + + +def set_random_seed(seed, deterministic=False): + """Set random seed. + + Args: + seed (int): Seed to be used. + deterministic (bool): Whether to set the deterministic option for + CUDNN backend, i.e., set `torch.backends.cudnn.deterministic` + to True and `torch.backends.cudnn.benchmark` to False. + Default: False. + """ + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + if deterministic: + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + +def train_segmentor(model, + dataset, + cfg, + distributed=False, + validate=False, + timestamp=None, + meta=None): + """Launch segmentor training.""" + logger = get_root_logger(cfg.log_level) + + # prepare data loaders + dataset = dataset if isinstance(dataset, (list, tuple)) else [dataset] + data_loaders = [ + build_dataloader( + ds, + cfg.data.samples_per_gpu, + cfg.data.workers_per_gpu, + # cfg.gpus will be ignored if distributed + len(cfg.gpu_ids), + dist=distributed, + seed=cfg.seed, + drop_last=True) for ds in dataset + ] + + # put model on gpus + if distributed: + find_unused_parameters = cfg.get('find_unused_parameters', False) + # Sets the `find_unused_parameters` parameter in + # torch.nn.parallel.DistributedDataParallel + model = MMDistributedDataParallel( + model.cuda(), + device_ids=[torch.cuda.current_device()], + broadcast_buffers=False, + find_unused_parameters=find_unused_parameters) + else: + model = MMDataParallel( + model.cuda(cfg.gpu_ids[0]), device_ids=cfg.gpu_ids) + + # build runner + optimizer = build_optimizer(model, cfg.optimizer) + + if cfg.get('runner') is None: + cfg.runner = {'type': 'IterBasedRunner', 'max_iters': cfg.total_iters} + warnings.warn( + 'config is now expected to have a `runner` section, ' + 'please set `runner` in your config.', UserWarning) + + runner = build_runner( + cfg.runner, + default_args=dict( + model=model, + batch_processor=None, + optimizer=optimizer, + work_dir=cfg.work_dir, + logger=logger, + meta=meta)) + + # register hooks + runner.register_training_hooks(cfg.lr_config, cfg.optimizer_config, + cfg.checkpoint_config, cfg.log_config, + cfg.get('momentum_config', None)) + + # an ugly walkaround to make the .log and .log.json filenames the same + runner.timestamp = timestamp + + # register eval hooks + if validate: + val_dataset = build_dataset(cfg.data.val, dict(test_mode=True)) + val_dataloader = build_dataloader( + val_dataset, + samples_per_gpu=1, + workers_per_gpu=cfg.data.workers_per_gpu, + dist=distributed, + shuffle=False) + eval_cfg = cfg.get('evaluation', {}) + eval_cfg['by_epoch'] = cfg.runner['type'] != 'IterBasedRunner' + eval_hook = DistEvalHook if distributed else EvalHook + runner.register_hook(eval_hook(val_dataloader, **eval_cfg), priority='LOW') + + if cfg.resume_from: + runner.resume(cfg.resume_from) + elif cfg.load_from: + runner.load_checkpoint(cfg.load_from) + runner.run(data_loaders, cfg.workflow) diff --git a/annotator/mmpkg/mmseg/core/__init__.py b/annotator/mmpkg/mmseg/core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..965605587211b7bf0bd6bc3acdbb33dd49cab023 --- /dev/null +++ b/annotator/mmpkg/mmseg/core/__init__.py @@ -0,0 +1,3 @@ +from .evaluation import * # noqa: F401, F403 +from .seg import * # noqa: F401, F403 +from .utils import * # noqa: F401, F403 diff --git a/annotator/mmpkg/mmseg/core/evaluation/__init__.py b/annotator/mmpkg/mmseg/core/evaluation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f7cc4b23413a0639e9de00eeb0bf600632d2c6cd --- /dev/null +++ b/annotator/mmpkg/mmseg/core/evaluation/__init__.py @@ -0,0 +1,8 @@ +from .class_names import get_classes, get_palette +from .eval_hooks import DistEvalHook, EvalHook +from .metrics import eval_metrics, mean_dice, mean_fscore, mean_iou + +__all__ = [ + 'EvalHook', 'DistEvalHook', 'mean_dice', 'mean_iou', 'mean_fscore', + 'eval_metrics', 'get_classes', 'get_palette' +] diff --git a/annotator/mmpkg/mmseg/core/evaluation/class_names.py b/annotator/mmpkg/mmseg/core/evaluation/class_names.py new file mode 100644 index 0000000000000000000000000000000000000000..532c5fd78946ede66d747ec8e7b72dbb66471aac --- /dev/null +++ b/annotator/mmpkg/mmseg/core/evaluation/class_names.py @@ -0,0 +1,152 @@ +import annotator.mmpkg.mmcv as mmcv + + +def cityscapes_classes(): + """Cityscapes class names for external use.""" + return [ + 'road', 'sidewalk', 'building', 'wall', 'fence', 'pole', + 'traffic light', 'traffic sign', 'vegetation', 'terrain', 'sky', + 'person', 'rider', 'car', 'truck', 'bus', 'train', 'motorcycle', + 'bicycle' + ] + + +def ade_classes(): + """ADE20K class names for external use.""" + return [ + 'wall', 'building', 'sky', 'floor', 'tree', 'ceiling', 'road', 'bed ', + 'windowpane', 'grass', 'cabinet', 'sidewalk', 'person', 'earth', + 'door', 'table', 'mountain', 'plant', 'curtain', 'chair', 'car', + 'water', 'painting', 'sofa', 'shelf', 'house', 'sea', 'mirror', 'rug', + 'field', 'armchair', 'seat', 'fence', 'desk', 'rock', 'wardrobe', + 'lamp', 'bathtub', 'railing', 'cushion', 'base', 'box', 'column', + 'signboard', 'chest of drawers', 'counter', 'sand', 'sink', + 'skyscraper', 'fireplace', 'refrigerator', 'grandstand', 'path', + 'stairs', 'runway', 'case', 'pool table', 'pillow', 'screen door', + 'stairway', 'river', 'bridge', 'bookcase', 'blind', 'coffee table', + 'toilet', 'flower', 'book', 'hill', 'bench', 'countertop', 'stove', + 'palm', 'kitchen island', 'computer', 'swivel chair', 'boat', 'bar', + 'arcade machine', 'hovel', 'bus', 'towel', 'light', 'truck', 'tower', + 'chandelier', 'awning', 'streetlight', 'booth', 'television receiver', + 'airplane', 'dirt track', 'apparel', 'pole', 'land', 'bannister', + 'escalator', 'ottoman', 'bottle', 'buffet', 'poster', 'stage', 'van', + 'ship', 'fountain', 'conveyer belt', 'canopy', 'washer', 'plaything', + 'swimming pool', 'stool', 'barrel', 'basket', 'waterfall', 'tent', + 'bag', 'minibike', 'cradle', 'oven', 'ball', 'food', 'step', 'tank', + 'trade name', 'microwave', 'pot', 'animal', 'bicycle', 'lake', + 'dishwasher', 'screen', 'blanket', 'sculpture', 'hood', 'sconce', + 'vase', 'traffic light', 'tray', 'ashcan', 'fan', 'pier', 'crt screen', + 'plate', 'monitor', 'bulletin board', 'shower', 'radiator', 'glass', + 'clock', 'flag' + ] + + +def voc_classes(): + """Pascal VOC class names for external use.""" + return [ + 'background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', + 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', + 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', + 'tvmonitor' + ] + + +def cityscapes_palette(): + """Cityscapes palette for external use.""" + return [[128, 64, 128], [244, 35, 232], [70, 70, 70], [102, 102, 156], + [190, 153, 153], [153, 153, 153], [250, 170, 30], [220, 220, 0], + [107, 142, 35], [152, 251, 152], [70, 130, 180], [220, 20, 60], + [255, 0, 0], [0, 0, 142], [0, 0, 70], [0, 60, 100], [0, 80, 100], + [0, 0, 230], [119, 11, 32]] + + +def ade_palette(): + """ADE20K palette for external use.""" + return [[120, 120, 120], [180, 120, 120], [6, 230, 230], [80, 50, 50], + [4, 200, 3], [120, 120, 80], [140, 140, 140], [204, 5, 255], + [230, 230, 230], [4, 250, 7], [224, 5, 255], [235, 255, 7], + [150, 5, 61], [120, 120, 70], [8, 255, 51], [255, 6, 82], + [143, 255, 140], [204, 255, 4], [255, 51, 7], [204, 70, 3], + [0, 102, 200], [61, 230, 250], [255, 6, 51], [11, 102, 255], + [255, 7, 71], [255, 9, 224], [9, 7, 230], [220, 220, 220], + [255, 9, 92], [112, 9, 255], [8, 255, 214], [7, 255, 224], + [255, 184, 6], [10, 255, 71], [255, 41, 10], [7, 255, 255], + [224, 255, 8], [102, 8, 255], [255, 61, 6], [255, 194, 7], + [255, 122, 8], [0, 255, 20], [255, 8, 41], [255, 5, 153], + [6, 51, 255], [235, 12, 255], [160, 150, 20], [0, 163, 255], + [140, 140, 140], [250, 10, 15], [20, 255, 0], [31, 255, 0], + [255, 31, 0], [255, 224, 0], [153, 255, 0], [0, 0, 255], + [255, 71, 0], [0, 235, 255], [0, 173, 255], [31, 0, 255], + [11, 200, 200], [255, 82, 0], [0, 255, 245], [0, 61, 255], + [0, 255, 112], [0, 255, 133], [255, 0, 0], [255, 163, 0], + [255, 102, 0], [194, 255, 0], [0, 143, 255], [51, 255, 0], + [0, 82, 255], [0, 255, 41], [0, 255, 173], [10, 0, 255], + [173, 255, 0], [0, 255, 153], [255, 92, 0], [255, 0, 255], + [255, 0, 245], [255, 0, 102], [255, 173, 0], [255, 0, 20], + [255, 184, 184], [0, 31, 255], [0, 255, 61], [0, 71, 255], + [255, 0, 204], [0, 255, 194], [0, 255, 82], [0, 10, 255], + [0, 112, 255], [51, 0, 255], [0, 194, 255], [0, 122, 255], + [0, 255, 163], [255, 153, 0], [0, 255, 10], [255, 112, 0], + [143, 255, 0], [82, 0, 255], [163, 255, 0], [255, 235, 0], + [8, 184, 170], [133, 0, 255], [0, 255, 92], [184, 0, 255], + [255, 0, 31], [0, 184, 255], [0, 214, 255], [255, 0, 112], + [92, 255, 0], [0, 224, 255], [112, 224, 255], [70, 184, 160], + [163, 0, 255], [153, 0, 255], [71, 255, 0], [255, 0, 163], + [255, 204, 0], [255, 0, 143], [0, 255, 235], [133, 255, 0], + [255, 0, 235], [245, 0, 255], [255, 0, 122], [255, 245, 0], + [10, 190, 212], [214, 255, 0], [0, 204, 255], [20, 0, 255], + [255, 255, 0], [0, 153, 255], [0, 41, 255], [0, 255, 204], + [41, 0, 255], [41, 255, 0], [173, 0, 255], [0, 245, 255], + [71, 0, 255], [122, 0, 255], [0, 255, 184], [0, 92, 255], + [184, 255, 0], [0, 133, 255], [255, 214, 0], [25, 194, 194], + [102, 255, 0], [92, 0, 255]] + + +def voc_palette(): + """Pascal VOC palette for external use.""" + return [[0, 0, 0], [128, 0, 0], [0, 128, 0], [128, 128, 0], [0, 0, 128], + [128, 0, 128], [0, 128, 128], [128, 128, 128], [64, 0, 0], + [192, 0, 0], [64, 128, 0], [192, 128, 0], [64, 0, 128], + [192, 0, 128], [64, 128, 128], [192, 128, 128], [0, 64, 0], + [128, 64, 0], [0, 192, 0], [128, 192, 0], [0, 64, 128]] + + +dataset_aliases = { + 'cityscapes': ['cityscapes'], + 'ade': ['ade', 'ade20k'], + 'voc': ['voc', 'pascal_voc', 'voc12', 'voc12aug'] +} + + +def get_classes(dataset): + """Get class names of a dataset.""" + alias2name = {} + for name, aliases in dataset_aliases.items(): + for alias in aliases: + alias2name[alias] = name + + if mmcv.is_str(dataset): + if dataset in alias2name: + labels = eval(alias2name[dataset] + '_classes()') + else: + raise ValueError(f'Unrecognized dataset: {dataset}') + else: + raise TypeError(f'dataset must a str, but got {type(dataset)}') + return labels + + +def get_palette(dataset): + """Get class palette (RGB) of a dataset.""" + alias2name = {} + for name, aliases in dataset_aliases.items(): + for alias in aliases: + alias2name[alias] = name + + if mmcv.is_str(dataset): + if dataset in alias2name: + labels = eval(alias2name[dataset] + '_palette()') + else: + raise ValueError(f'Unrecognized dataset: {dataset}') + else: + raise TypeError(f'dataset must a str, but got {type(dataset)}') + return labels diff --git a/annotator/mmpkg/mmseg/core/evaluation/eval_hooks.py b/annotator/mmpkg/mmseg/core/evaluation/eval_hooks.py new file mode 100644 index 0000000000000000000000000000000000000000..408e9670f61d1b118477562b341adc644c52799a --- /dev/null +++ b/annotator/mmpkg/mmseg/core/evaluation/eval_hooks.py @@ -0,0 +1,109 @@ +import os.path as osp + +from annotator.mmpkg.mmcv.runner import DistEvalHook as _DistEvalHook +from annotator.mmpkg.mmcv.runner import EvalHook as _EvalHook + + +class EvalHook(_EvalHook): + """Single GPU EvalHook, with efficient test support. + + Args: + by_epoch (bool): Determine perform evaluation by epoch or by iteration. + If set to True, it will perform by epoch. Otherwise, by iteration. + Default: False. + efficient_test (bool): Whether save the results as local numpy files to + save CPU memory during evaluation. Default: False. + Returns: + list: The prediction results. + """ + + greater_keys = ['mIoU', 'mAcc', 'aAcc'] + + def __init__(self, *args, by_epoch=False, efficient_test=False, **kwargs): + super().__init__(*args, by_epoch=by_epoch, **kwargs) + self.efficient_test = efficient_test + + def after_train_iter(self, runner): + """After train epoch hook. + + Override default ``single_gpu_test``. + """ + if self.by_epoch or not self.every_n_iters(runner, self.interval): + return + from annotator.mmpkg.mmseg.apis import single_gpu_test + runner.log_buffer.clear() + results = single_gpu_test( + runner.model, + self.dataloader, + show=False, + efficient_test=self.efficient_test) + self.evaluate(runner, results) + + def after_train_epoch(self, runner): + """After train epoch hook. + + Override default ``single_gpu_test``. + """ + if not self.by_epoch or not self.every_n_epochs(runner, self.interval): + return + from annotator.mmpkg.mmseg.apis import single_gpu_test + runner.log_buffer.clear() + results = single_gpu_test(runner.model, self.dataloader, show=False) + self.evaluate(runner, results) + + +class DistEvalHook(_DistEvalHook): + """Distributed EvalHook, with efficient test support. + + Args: + by_epoch (bool): Determine perform evaluation by epoch or by iteration. + If set to True, it will perform by epoch. Otherwise, by iteration. + Default: False. + efficient_test (bool): Whether save the results as local numpy files to + save CPU memory during evaluation. Default: False. + Returns: + list: The prediction results. + """ + + greater_keys = ['mIoU', 'mAcc', 'aAcc'] + + def __init__(self, *args, by_epoch=False, efficient_test=False, **kwargs): + super().__init__(*args, by_epoch=by_epoch, **kwargs) + self.efficient_test = efficient_test + + def after_train_iter(self, runner): + """After train epoch hook. + + Override default ``multi_gpu_test``. + """ + if self.by_epoch or not self.every_n_iters(runner, self.interval): + return + from annotator.mmpkg.mmseg.apis import multi_gpu_test + runner.log_buffer.clear() + results = multi_gpu_test( + runner.model, + self.dataloader, + tmpdir=osp.join(runner.work_dir, '.eval_hook'), + gpu_collect=self.gpu_collect, + efficient_test=self.efficient_test) + if runner.rank == 0: + print('\n') + self.evaluate(runner, results) + + def after_train_epoch(self, runner): + """After train epoch hook. + + Override default ``multi_gpu_test``. + """ + if not self.by_epoch or not self.every_n_epochs(runner, self.interval): + return + from annotator.mmpkg.mmseg.apis import multi_gpu_test + runner.log_buffer.clear() + results = multi_gpu_test( + runner.model, + self.dataloader, + tmpdir=osp.join(runner.work_dir, '.eval_hook'), + gpu_collect=self.gpu_collect) + if runner.rank == 0: + print('\n') + self.evaluate(runner, results) diff --git a/annotator/mmpkg/mmseg/core/evaluation/metrics.py b/annotator/mmpkg/mmseg/core/evaluation/metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..8ede737624a0ba6e6365639f7019ac2527052cfd --- /dev/null +++ b/annotator/mmpkg/mmseg/core/evaluation/metrics.py @@ -0,0 +1,326 @@ +from collections import OrderedDict + +import annotator.mmpkg.mmcv as mmcv +import numpy as np +import torch + + +def f_score(precision, recall, beta=1): + """calcuate the f-score value. + + Args: + precision (float | torch.Tensor): The precision value. + recall (float | torch.Tensor): The recall value. + beta (int): Determines the weight of recall in the combined score. + Default: False. + + Returns: + [torch.tensor]: The f-score value. + """ + score = (1 + beta**2) * (precision * recall) / ( + (beta**2 * precision) + recall) + return score + + +def intersect_and_union(pred_label, + label, + num_classes, + ignore_index, + label_map=dict(), + reduce_zero_label=False): + """Calculate intersection and Union. + + Args: + pred_label (ndarray | str): Prediction segmentation map + or predict result filename. + label (ndarray | str): Ground truth segmentation map + or label filename. + num_classes (int): Number of categories. + ignore_index (int): Index that will be ignored in evaluation. + label_map (dict): Mapping old labels to new labels. The parameter will + work only when label is str. Default: dict(). + reduce_zero_label (bool): Whether ignore zero label. The parameter will + work only when label is str. Default: False. + + Returns: + torch.Tensor: The intersection of prediction and ground truth + histogram on all classes. + torch.Tensor: The union of prediction and ground truth histogram on + all classes. + torch.Tensor: The prediction histogram on all classes. + torch.Tensor: The ground truth histogram on all classes. + """ + + if isinstance(pred_label, str): + pred_label = torch.from_numpy(np.load(pred_label)) + else: + pred_label = torch.from_numpy((pred_label)) + + if isinstance(label, str): + label = torch.from_numpy( + mmcv.imread(label, flag='unchanged', backend='pillow')) + else: + label = torch.from_numpy(label) + + if label_map is not None: + for old_id, new_id in label_map.items(): + label[label == old_id] = new_id + if reduce_zero_label: + label[label == 0] = 255 + label = label - 1 + label[label == 254] = 255 + + mask = (label != ignore_index) + pred_label = pred_label[mask] + label = label[mask] + + intersect = pred_label[pred_label == label] + area_intersect = torch.histc( + intersect.float(), bins=(num_classes), min=0, max=num_classes - 1) + area_pred_label = torch.histc( + pred_label.float(), bins=(num_classes), min=0, max=num_classes - 1) + area_label = torch.histc( + label.float(), bins=(num_classes), min=0, max=num_classes - 1) + area_union = area_pred_label + area_label - area_intersect + return area_intersect, area_union, area_pred_label, area_label + + +def total_intersect_and_union(results, + gt_seg_maps, + num_classes, + ignore_index, + label_map=dict(), + reduce_zero_label=False): + """Calculate Total Intersection and Union. + + Args: + results (list[ndarray] | list[str]): List of prediction segmentation + maps or list of prediction result filenames. + gt_seg_maps (list[ndarray] | list[str]): list of ground truth + segmentation maps or list of label filenames. + num_classes (int): Number of categories. + ignore_index (int): Index that will be ignored in evaluation. + label_map (dict): Mapping old labels to new labels. Default: dict(). + reduce_zero_label (bool): Whether ignore zero label. Default: False. + + Returns: + ndarray: The intersection of prediction and ground truth histogram + on all classes. + ndarray: The union of prediction and ground truth histogram on all + classes. + ndarray: The prediction histogram on all classes. + ndarray: The ground truth histogram on all classes. + """ + num_imgs = len(results) + assert len(gt_seg_maps) == num_imgs + total_area_intersect = torch.zeros((num_classes, ), dtype=torch.float64) + total_area_union = torch.zeros((num_classes, ), dtype=torch.float64) + total_area_pred_label = torch.zeros((num_classes, ), dtype=torch.float64) + total_area_label = torch.zeros((num_classes, ), dtype=torch.float64) + for i in range(num_imgs): + area_intersect, area_union, area_pred_label, area_label = \ + intersect_and_union( + results[i], gt_seg_maps[i], num_classes, ignore_index, + label_map, reduce_zero_label) + total_area_intersect += area_intersect + total_area_union += area_union + total_area_pred_label += area_pred_label + total_area_label += area_label + return total_area_intersect, total_area_union, total_area_pred_label, \ + total_area_label + + +def mean_iou(results, + gt_seg_maps, + num_classes, + ignore_index, + nan_to_num=None, + label_map=dict(), + reduce_zero_label=False): + """Calculate Mean Intersection and Union (mIoU) + + Args: + results (list[ndarray] | list[str]): List of prediction segmentation + maps or list of prediction result filenames. + gt_seg_maps (list[ndarray] | list[str]): list of ground truth + segmentation maps or list of label filenames. + num_classes (int): Number of categories. + ignore_index (int): Index that will be ignored in evaluation. + nan_to_num (int, optional): If specified, NaN values will be replaced + by the numbers defined by the user. Default: None. + label_map (dict): Mapping old labels to new labels. Default: dict(). + reduce_zero_label (bool): Whether ignore zero label. Default: False. + + Returns: + dict[str, float | ndarray]: + float: Overall accuracy on all images. + ndarray: Per category accuracy, shape (num_classes, ). + ndarray: Per category IoU, shape (num_classes, ). + """ + iou_result = eval_metrics( + results=results, + gt_seg_maps=gt_seg_maps, + num_classes=num_classes, + ignore_index=ignore_index, + metrics=['mIoU'], + nan_to_num=nan_to_num, + label_map=label_map, + reduce_zero_label=reduce_zero_label) + return iou_result + + +def mean_dice(results, + gt_seg_maps, + num_classes, + ignore_index, + nan_to_num=None, + label_map=dict(), + reduce_zero_label=False): + """Calculate Mean Dice (mDice) + + Args: + results (list[ndarray] | list[str]): List of prediction segmentation + maps or list of prediction result filenames. + gt_seg_maps (list[ndarray] | list[str]): list of ground truth + segmentation maps or list of label filenames. + num_classes (int): Number of categories. + ignore_index (int): Index that will be ignored in evaluation. + nan_to_num (int, optional): If specified, NaN values will be replaced + by the numbers defined by the user. Default: None. + label_map (dict): Mapping old labels to new labels. Default: dict(). + reduce_zero_label (bool): Whether ignore zero label. Default: False. + + Returns: + dict[str, float | ndarray]: Default metrics. + float: Overall accuracy on all images. + ndarray: Per category accuracy, shape (num_classes, ). + ndarray: Per category dice, shape (num_classes, ). + """ + + dice_result = eval_metrics( + results=results, + gt_seg_maps=gt_seg_maps, + num_classes=num_classes, + ignore_index=ignore_index, + metrics=['mDice'], + nan_to_num=nan_to_num, + label_map=label_map, + reduce_zero_label=reduce_zero_label) + return dice_result + + +def mean_fscore(results, + gt_seg_maps, + num_classes, + ignore_index, + nan_to_num=None, + label_map=dict(), + reduce_zero_label=False, + beta=1): + """Calculate Mean Intersection and Union (mIoU) + + Args: + results (list[ndarray] | list[str]): List of prediction segmentation + maps or list of prediction result filenames. + gt_seg_maps (list[ndarray] | list[str]): list of ground truth + segmentation maps or list of label filenames. + num_classes (int): Number of categories. + ignore_index (int): Index that will be ignored in evaluation. + nan_to_num (int, optional): If specified, NaN values will be replaced + by the numbers defined by the user. Default: None. + label_map (dict): Mapping old labels to new labels. Default: dict(). + reduce_zero_label (bool): Whether ignore zero label. Default: False. + beta (int): Determines the weight of recall in the combined score. + Default: False. + + + Returns: + dict[str, float | ndarray]: Default metrics. + float: Overall accuracy on all images. + ndarray: Per category recall, shape (num_classes, ). + ndarray: Per category precision, shape (num_classes, ). + ndarray: Per category f-score, shape (num_classes, ). + """ + fscore_result = eval_metrics( + results=results, + gt_seg_maps=gt_seg_maps, + num_classes=num_classes, + ignore_index=ignore_index, + metrics=['mFscore'], + nan_to_num=nan_to_num, + label_map=label_map, + reduce_zero_label=reduce_zero_label, + beta=beta) + return fscore_result + + +def eval_metrics(results, + gt_seg_maps, + num_classes, + ignore_index, + metrics=['mIoU'], + nan_to_num=None, + label_map=dict(), + reduce_zero_label=False, + beta=1): + """Calculate evaluation metrics + Args: + results (list[ndarray] | list[str]): List of prediction segmentation + maps or list of prediction result filenames. + gt_seg_maps (list[ndarray] | list[str]): list of ground truth + segmentation maps or list of label filenames. + num_classes (int): Number of categories. + ignore_index (int): Index that will be ignored in evaluation. + metrics (list[str] | str): Metrics to be evaluated, 'mIoU' and 'mDice'. + nan_to_num (int, optional): If specified, NaN values will be replaced + by the numbers defined by the user. Default: None. + label_map (dict): Mapping old labels to new labels. Default: dict(). + reduce_zero_label (bool): Whether ignore zero label. Default: False. + Returns: + float: Overall accuracy on all images. + ndarray: Per category accuracy, shape (num_classes, ). + ndarray: Per category evaluation metrics, shape (num_classes, ). + """ + if isinstance(metrics, str): + metrics = [metrics] + allowed_metrics = ['mIoU', 'mDice', 'mFscore'] + if not set(metrics).issubset(set(allowed_metrics)): + raise KeyError('metrics {} is not supported'.format(metrics)) + + total_area_intersect, total_area_union, total_area_pred_label, \ + total_area_label = total_intersect_and_union( + results, gt_seg_maps, num_classes, ignore_index, label_map, + reduce_zero_label) + all_acc = total_area_intersect.sum() / total_area_label.sum() + ret_metrics = OrderedDict({'aAcc': all_acc}) + for metric in metrics: + if metric == 'mIoU': + iou = total_area_intersect / total_area_union + acc = total_area_intersect / total_area_label + ret_metrics['IoU'] = iou + ret_metrics['Acc'] = acc + elif metric == 'mDice': + dice = 2 * total_area_intersect / ( + total_area_pred_label + total_area_label) + acc = total_area_intersect / total_area_label + ret_metrics['Dice'] = dice + ret_metrics['Acc'] = acc + elif metric == 'mFscore': + precision = total_area_intersect / total_area_pred_label + recall = total_area_intersect / total_area_label + f_value = torch.tensor( + [f_score(x[0], x[1], beta) for x in zip(precision, recall)]) + ret_metrics['Fscore'] = f_value + ret_metrics['Precision'] = precision + ret_metrics['Recall'] = recall + + ret_metrics = { + metric: value.numpy() + for metric, value in ret_metrics.items() + } + if nan_to_num is not None: + ret_metrics = OrderedDict({ + metric: np.nan_to_num(metric_value, nan=nan_to_num) + for metric, metric_value in ret_metrics.items() + }) + return ret_metrics diff --git a/annotator/mmpkg/mmseg/core/seg/__init__.py b/annotator/mmpkg/mmseg/core/seg/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..93bc129b685e4a3efca2cc891729981b2865900d --- /dev/null +++ b/annotator/mmpkg/mmseg/core/seg/__init__.py @@ -0,0 +1,4 @@ +from .builder import build_pixel_sampler +from .sampler import BasePixelSampler, OHEMPixelSampler + +__all__ = ['build_pixel_sampler', 'BasePixelSampler', 'OHEMPixelSampler'] diff --git a/annotator/mmpkg/mmseg/core/seg/builder.py b/annotator/mmpkg/mmseg/core/seg/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..f8fff6375622282f85b3acf15af1a7d27fb9c426 --- /dev/null +++ b/annotator/mmpkg/mmseg/core/seg/builder.py @@ -0,0 +1,8 @@ +from annotator.mmpkg.mmcv.utils import Registry, build_from_cfg + +PIXEL_SAMPLERS = Registry('pixel sampler') + + +def build_pixel_sampler(cfg, **default_args): + """Build pixel sampler for segmentation map.""" + return build_from_cfg(cfg, PIXEL_SAMPLERS, default_args) diff --git a/annotator/mmpkg/mmseg/core/seg/sampler/__init__.py b/annotator/mmpkg/mmseg/core/seg/sampler/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..332b242c03d1c5e80d4577df442a9a037b1816e1 --- /dev/null +++ b/annotator/mmpkg/mmseg/core/seg/sampler/__init__.py @@ -0,0 +1,4 @@ +from .base_pixel_sampler import BasePixelSampler +from .ohem_pixel_sampler import OHEMPixelSampler + +__all__ = ['BasePixelSampler', 'OHEMPixelSampler'] diff --git a/annotator/mmpkg/mmseg/core/seg/sampler/base_pixel_sampler.py b/annotator/mmpkg/mmseg/core/seg/sampler/base_pixel_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..b75b1566c9f18169cee51d4b55d75e0357b69c57 --- /dev/null +++ b/annotator/mmpkg/mmseg/core/seg/sampler/base_pixel_sampler.py @@ -0,0 +1,12 @@ +from abc import ABCMeta, abstractmethod + + +class BasePixelSampler(metaclass=ABCMeta): + """Base class of pixel sampler.""" + + def __init__(self, **kwargs): + pass + + @abstractmethod + def sample(self, seg_logit, seg_label): + """Placeholder for sample function.""" diff --git a/annotator/mmpkg/mmseg/core/seg/sampler/ohem_pixel_sampler.py b/annotator/mmpkg/mmseg/core/seg/sampler/ohem_pixel_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..88bb10d44026ba9f21756eaea9e550841cd59b9f --- /dev/null +++ b/annotator/mmpkg/mmseg/core/seg/sampler/ohem_pixel_sampler.py @@ -0,0 +1,76 @@ +import torch +import torch.nn.functional as F + +from ..builder import PIXEL_SAMPLERS +from .base_pixel_sampler import BasePixelSampler + + +@PIXEL_SAMPLERS.register_module() +class OHEMPixelSampler(BasePixelSampler): + """Online Hard Example Mining Sampler for segmentation. + + Args: + context (nn.Module): The context of sampler, subclass of + :obj:`BaseDecodeHead`. + thresh (float, optional): The threshold for hard example selection. + Below which, are prediction with low confidence. If not + specified, the hard examples will be pixels of top ``min_kept`` + loss. Default: None. + min_kept (int, optional): The minimum number of predictions to keep. + Default: 100000. + """ + + def __init__(self, context, thresh=None, min_kept=100000): + super(OHEMPixelSampler, self).__init__() + self.context = context + assert min_kept > 1 + self.thresh = thresh + self.min_kept = min_kept + + def sample(self, seg_logit, seg_label): + """Sample pixels that have high loss or with low prediction confidence. + + Args: + seg_logit (torch.Tensor): segmentation logits, shape (N, C, H, W) + seg_label (torch.Tensor): segmentation label, shape (N, 1, H, W) + + Returns: + torch.Tensor: segmentation weight, shape (N, H, W) + """ + with torch.no_grad(): + assert seg_logit.shape[2:] == seg_label.shape[2:] + assert seg_label.shape[1] == 1 + seg_label = seg_label.squeeze(1).long() + batch_kept = self.min_kept * seg_label.size(0) + valid_mask = seg_label != self.context.ignore_index + seg_weight = seg_logit.new_zeros(size=seg_label.size()) + valid_seg_weight = seg_weight[valid_mask] + if self.thresh is not None: + seg_prob = F.softmax(seg_logit, dim=1) + + tmp_seg_label = seg_label.clone().unsqueeze(1) + tmp_seg_label[tmp_seg_label == self.context.ignore_index] = 0 + seg_prob = seg_prob.gather(1, tmp_seg_label).squeeze(1) + sort_prob, sort_indices = seg_prob[valid_mask].sort() + + if sort_prob.numel() > 0: + min_threshold = sort_prob[min(batch_kept, + sort_prob.numel() - 1)] + else: + min_threshold = 0.0 + threshold = max(min_threshold, self.thresh) + valid_seg_weight[seg_prob[valid_mask] < threshold] = 1. + else: + losses = self.context.loss_decode( + seg_logit, + seg_label, + weight=None, + ignore_index=self.context.ignore_index, + reduction_override='none') + # faster than topk according to https://github.com/pytorch/pytorch/issues/22812 # noqa + _, sort_indices = losses[valid_mask].sort(descending=True) + valid_seg_weight[sort_indices[:batch_kept]] = 1. + + seg_weight[valid_mask] = valid_seg_weight + + return seg_weight diff --git a/annotator/mmpkg/mmseg/core/utils/__init__.py b/annotator/mmpkg/mmseg/core/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f2678b321c295bcceaef945111ac3524be19d6e4 --- /dev/null +++ b/annotator/mmpkg/mmseg/core/utils/__init__.py @@ -0,0 +1,3 @@ +from .misc import add_prefix + +__all__ = ['add_prefix'] diff --git a/annotator/mmpkg/mmseg/core/utils/misc.py b/annotator/mmpkg/mmseg/core/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..eb862a82bd47c8624db3dd5c6fb6ad8a03b62466 --- /dev/null +++ b/annotator/mmpkg/mmseg/core/utils/misc.py @@ -0,0 +1,17 @@ +def add_prefix(inputs, prefix): + """Add prefix for dict. + + Args: + inputs (dict): The input dict with str keys. + prefix (str): The prefix to add. + + Returns: + + dict: The dict with keys updated with ``prefix``. + """ + + outputs = dict() + for name, value in inputs.items(): + outputs[f'{prefix}.{name}'] = value + + return outputs diff --git a/annotator/mmpkg/mmseg/datasets/__init__.py b/annotator/mmpkg/mmseg/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ebeaef4a28ef655e43578552a8aef6b77f13a636 --- /dev/null +++ b/annotator/mmpkg/mmseg/datasets/__init__.py @@ -0,0 +1,19 @@ +from .ade import ADE20KDataset +from .builder import DATASETS, PIPELINES, build_dataloader, build_dataset +from .chase_db1 import ChaseDB1Dataset +from .cityscapes import CityscapesDataset +from .custom import CustomDataset +from .dataset_wrappers import ConcatDataset, RepeatDataset +from .drive import DRIVEDataset +from .hrf import HRFDataset +from .pascal_context import PascalContextDataset, PascalContextDataset59 +from .stare import STAREDataset +from .voc import PascalVOCDataset + +__all__ = [ + 'CustomDataset', 'build_dataloader', 'ConcatDataset', 'RepeatDataset', + 'DATASETS', 'build_dataset', 'PIPELINES', 'CityscapesDataset', + 'PascalVOCDataset', 'ADE20KDataset', 'PascalContextDataset', + 'PascalContextDataset59', 'ChaseDB1Dataset', 'DRIVEDataset', 'HRFDataset', + 'STAREDataset' +] diff --git a/annotator/mmpkg/mmseg/datasets/ade.py b/annotator/mmpkg/mmseg/datasets/ade.py new file mode 100644 index 0000000000000000000000000000000000000000..5913e43775ed4920b6934c855eb5a37c54218ebf --- /dev/null +++ b/annotator/mmpkg/mmseg/datasets/ade.py @@ -0,0 +1,84 @@ +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class ADE20KDataset(CustomDataset): + """ADE20K dataset. + + In segmentation map annotation for ADE20K, 0 stands for background, which + is not included in 150 categories. ``reduce_zero_label`` is fixed to True. + The ``img_suffix`` is fixed to '.jpg' and ``seg_map_suffix`` is fixed to + '.png'. + """ + CLASSES = ( + 'wall', 'building', 'sky', 'floor', 'tree', 'ceiling', 'road', 'bed ', + 'windowpane', 'grass', 'cabinet', 'sidewalk', 'person', 'earth', + 'door', 'table', 'mountain', 'plant', 'curtain', 'chair', 'car', + 'water', 'painting', 'sofa', 'shelf', 'house', 'sea', 'mirror', 'rug', + 'field', 'armchair', 'seat', 'fence', 'desk', 'rock', 'wardrobe', + 'lamp', 'bathtub', 'railing', 'cushion', 'base', 'box', 'column', + 'signboard', 'chest of drawers', 'counter', 'sand', 'sink', + 'skyscraper', 'fireplace', 'refrigerator', 'grandstand', 'path', + 'stairs', 'runway', 'case', 'pool table', 'pillow', 'screen door', + 'stairway', 'river', 'bridge', 'bookcase', 'blind', 'coffee table', + 'toilet', 'flower', 'book', 'hill', 'bench', 'countertop', 'stove', + 'palm', 'kitchen island', 'computer', 'swivel chair', 'boat', 'bar', + 'arcade machine', 'hovel', 'bus', 'towel', 'light', 'truck', 'tower', + 'chandelier', 'awning', 'streetlight', 'booth', 'television receiver', + 'airplane', 'dirt track', 'apparel', 'pole', 'land', 'bannister', + 'escalator', 'ottoman', 'bottle', 'buffet', 'poster', 'stage', 'van', + 'ship', 'fountain', 'conveyer belt', 'canopy', 'washer', 'plaything', + 'swimming pool', 'stool', 'barrel', 'basket', 'waterfall', 'tent', + 'bag', 'minibike', 'cradle', 'oven', 'ball', 'food', 'step', 'tank', + 'trade name', 'microwave', 'pot', 'animal', 'bicycle', 'lake', + 'dishwasher', 'screen', 'blanket', 'sculpture', 'hood', 'sconce', + 'vase', 'traffic light', 'tray', 'ashcan', 'fan', 'pier', 'crt screen', + 'plate', 'monitor', 'bulletin board', 'shower', 'radiator', 'glass', + 'clock', 'flag') + + PALETTE = [[120, 120, 120], [180, 120, 120], [6, 230, 230], [80, 50, 50], + [4, 200, 3], [120, 120, 80], [140, 140, 140], [204, 5, 255], + [230, 230, 230], [4, 250, 7], [224, 5, 255], [235, 255, 7], + [150, 5, 61], [120, 120, 70], [8, 255, 51], [255, 6, 82], + [143, 255, 140], [204, 255, 4], [255, 51, 7], [204, 70, 3], + [0, 102, 200], [61, 230, 250], [255, 6, 51], [11, 102, 255], + [255, 7, 71], [255, 9, 224], [9, 7, 230], [220, 220, 220], + [255, 9, 92], [112, 9, 255], [8, 255, 214], [7, 255, 224], + [255, 184, 6], [10, 255, 71], [255, 41, 10], [7, 255, 255], + [224, 255, 8], [102, 8, 255], [255, 61, 6], [255, 194, 7], + [255, 122, 8], [0, 255, 20], [255, 8, 41], [255, 5, 153], + [6, 51, 255], [235, 12, 255], [160, 150, 20], [0, 163, 255], + [140, 140, 140], [250, 10, 15], [20, 255, 0], [31, 255, 0], + [255, 31, 0], [255, 224, 0], [153, 255, 0], [0, 0, 255], + [255, 71, 0], [0, 235, 255], [0, 173, 255], [31, 0, 255], + [11, 200, 200], [255, 82, 0], [0, 255, 245], [0, 61, 255], + [0, 255, 112], [0, 255, 133], [255, 0, 0], [255, 163, 0], + [255, 102, 0], [194, 255, 0], [0, 143, 255], [51, 255, 0], + [0, 82, 255], [0, 255, 41], [0, 255, 173], [10, 0, 255], + [173, 255, 0], [0, 255, 153], [255, 92, 0], [255, 0, 255], + [255, 0, 245], [255, 0, 102], [255, 173, 0], [255, 0, 20], + [255, 184, 184], [0, 31, 255], [0, 255, 61], [0, 71, 255], + [255, 0, 204], [0, 255, 194], [0, 255, 82], [0, 10, 255], + [0, 112, 255], [51, 0, 255], [0, 194, 255], [0, 122, 255], + [0, 255, 163], [255, 153, 0], [0, 255, 10], [255, 112, 0], + [143, 255, 0], [82, 0, 255], [163, 255, 0], [255, 235, 0], + [8, 184, 170], [133, 0, 255], [0, 255, 92], [184, 0, 255], + [255, 0, 31], [0, 184, 255], [0, 214, 255], [255, 0, 112], + [92, 255, 0], [0, 224, 255], [112, 224, 255], [70, 184, 160], + [163, 0, 255], [153, 0, 255], [71, 255, 0], [255, 0, 163], + [255, 204, 0], [255, 0, 143], [0, 255, 235], [133, 255, 0], + [255, 0, 235], [245, 0, 255], [255, 0, 122], [255, 245, 0], + [10, 190, 212], [214, 255, 0], [0, 204, 255], [20, 0, 255], + [255, 255, 0], [0, 153, 255], [0, 41, 255], [0, 255, 204], + [41, 0, 255], [41, 255, 0], [173, 0, 255], [0, 245, 255], + [71, 0, 255], [122, 0, 255], [0, 255, 184], [0, 92, 255], + [184, 255, 0], [0, 133, 255], [255, 214, 0], [25, 194, 194], + [102, 255, 0], [92, 0, 255]] + + def __init__(self, **kwargs): + super(ADE20KDataset, self).__init__( + img_suffix='.jpg', + seg_map_suffix='.png', + reduce_zero_label=True, + **kwargs) diff --git a/annotator/mmpkg/mmseg/datasets/builder.py b/annotator/mmpkg/mmseg/datasets/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..6cf8b4d9d32d4464905507cd54a84eb534f38bb6 --- /dev/null +++ b/annotator/mmpkg/mmseg/datasets/builder.py @@ -0,0 +1,169 @@ +import copy +import platform +import random +from functools import partial + +import numpy as np +from annotator.mmpkg.mmcv.parallel import collate +from annotator.mmpkg.mmcv.runner import get_dist_info +from annotator.mmpkg.mmcv.utils import Registry, build_from_cfg +from annotator.mmpkg.mmcv.utils.parrots_wrapper import DataLoader, PoolDataLoader +from torch.utils.data import DistributedSampler + +if platform.system() != 'Windows': + # https://github.com/pytorch/pytorch/issues/973 + import resource + rlimit = resource.getrlimit(resource.RLIMIT_NOFILE) + hard_limit = rlimit[1] + soft_limit = min(4096, hard_limit) + resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit)) + +DATASETS = Registry('dataset') +PIPELINES = Registry('pipeline') + + +def _concat_dataset(cfg, default_args=None): + """Build :obj:`ConcatDataset by.""" + from .dataset_wrappers import ConcatDataset + img_dir = cfg['img_dir'] + ann_dir = cfg.get('ann_dir', None) + split = cfg.get('split', None) + num_img_dir = len(img_dir) if isinstance(img_dir, (list, tuple)) else 1 + if ann_dir is not None: + num_ann_dir = len(ann_dir) if isinstance(ann_dir, (list, tuple)) else 1 + else: + num_ann_dir = 0 + if split is not None: + num_split = len(split) if isinstance(split, (list, tuple)) else 1 + else: + num_split = 0 + if num_img_dir > 1: + assert num_img_dir == num_ann_dir or num_ann_dir == 0 + assert num_img_dir == num_split or num_split == 0 + else: + assert num_split == num_ann_dir or num_ann_dir <= 1 + num_dset = max(num_split, num_img_dir) + + datasets = [] + for i in range(num_dset): + data_cfg = copy.deepcopy(cfg) + if isinstance(img_dir, (list, tuple)): + data_cfg['img_dir'] = img_dir[i] + if isinstance(ann_dir, (list, tuple)): + data_cfg['ann_dir'] = ann_dir[i] + if isinstance(split, (list, tuple)): + data_cfg['split'] = split[i] + datasets.append(build_dataset(data_cfg, default_args)) + + return ConcatDataset(datasets) + + +def build_dataset(cfg, default_args=None): + """Build datasets.""" + from .dataset_wrappers import ConcatDataset, RepeatDataset + if isinstance(cfg, (list, tuple)): + dataset = ConcatDataset([build_dataset(c, default_args) for c in cfg]) + elif cfg['type'] == 'RepeatDataset': + dataset = RepeatDataset( + build_dataset(cfg['dataset'], default_args), cfg['times']) + elif isinstance(cfg.get('img_dir'), (list, tuple)) or isinstance( + cfg.get('split', None), (list, tuple)): + dataset = _concat_dataset(cfg, default_args) + else: + dataset = build_from_cfg(cfg, DATASETS, default_args) + + return dataset + + +def build_dataloader(dataset, + samples_per_gpu, + workers_per_gpu, + num_gpus=1, + dist=True, + shuffle=True, + seed=None, + drop_last=False, + pin_memory=True, + dataloader_type='PoolDataLoader', + **kwargs): + """Build PyTorch DataLoader. + + In distributed training, each GPU/process has a dataloader. + In non-distributed training, there is only one dataloader for all GPUs. + + Args: + dataset (Dataset): A PyTorch dataset. + samples_per_gpu (int): Number of training samples on each GPU, i.e., + batch size of each GPU. + workers_per_gpu (int): How many subprocesses to use for data loading + for each GPU. + num_gpus (int): Number of GPUs. Only used in non-distributed training. + dist (bool): Distributed training/test or not. Default: True. + shuffle (bool): Whether to shuffle the data at every epoch. + Default: True. + seed (int | None): Seed to be used. Default: None. + drop_last (bool): Whether to drop the last incomplete batch in epoch. + Default: False + pin_memory (bool): Whether to use pin_memory in DataLoader. + Default: True + dataloader_type (str): Type of dataloader. Default: 'PoolDataLoader' + kwargs: any keyword argument to be used to initialize DataLoader + + Returns: + DataLoader: A PyTorch dataloader. + """ + rank, world_size = get_dist_info() + if dist: + sampler = DistributedSampler( + dataset, world_size, rank, shuffle=shuffle) + shuffle = False + batch_size = samples_per_gpu + num_workers = workers_per_gpu + else: + sampler = None + batch_size = num_gpus * samples_per_gpu + num_workers = num_gpus * workers_per_gpu + + init_fn = partial( + worker_init_fn, num_workers=num_workers, rank=rank, + seed=seed) if seed is not None else None + + assert dataloader_type in ( + 'DataLoader', + 'PoolDataLoader'), f'unsupported dataloader {dataloader_type}' + + if dataloader_type == 'PoolDataLoader': + dataloader = PoolDataLoader + elif dataloader_type == 'DataLoader': + dataloader = DataLoader + + data_loader = dataloader( + dataset, + batch_size=batch_size, + sampler=sampler, + num_workers=num_workers, + collate_fn=partial(collate, samples_per_gpu=samples_per_gpu), + pin_memory=pin_memory, + shuffle=shuffle, + worker_init_fn=init_fn, + drop_last=drop_last, + **kwargs) + + return data_loader + + +def worker_init_fn(worker_id, num_workers, rank, seed): + """Worker init func for dataloader. + + The seed of each worker equals to num_worker * rank + worker_id + user_seed + + Args: + worker_id (int): Worker id. + num_workers (int): Number of workers. + rank (int): The rank of current process. + seed (int): The random seed to use. + """ + + worker_seed = num_workers * rank + worker_id + seed + np.random.seed(worker_seed) + random.seed(worker_seed) diff --git a/annotator/mmpkg/mmseg/datasets/chase_db1.py b/annotator/mmpkg/mmseg/datasets/chase_db1.py new file mode 100644 index 0000000000000000000000000000000000000000..8bc29bea14704a4407f83474610cbc3bef32c708 --- /dev/null +++ b/annotator/mmpkg/mmseg/datasets/chase_db1.py @@ -0,0 +1,27 @@ +import os.path as osp + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class ChaseDB1Dataset(CustomDataset): + """Chase_db1 dataset. + + In segmentation map annotation for Chase_db1, 0 stands for background, + which is included in 2 categories. ``reduce_zero_label`` is fixed to False. + The ``img_suffix`` is fixed to '.png' and ``seg_map_suffix`` is fixed to + '_1stHO.png'. + """ + + CLASSES = ('background', 'vessel') + + PALETTE = [[120, 120, 120], [6, 230, 230]] + + def __init__(self, **kwargs): + super(ChaseDB1Dataset, self).__init__( + img_suffix='.png', + seg_map_suffix='_1stHO.png', + reduce_zero_label=False, + **kwargs) + assert osp.exists(self.img_dir) diff --git a/annotator/mmpkg/mmseg/datasets/cityscapes.py b/annotator/mmpkg/mmseg/datasets/cityscapes.py new file mode 100644 index 0000000000000000000000000000000000000000..38f80e8043d25178cf5dac18911241c74be4e3ac --- /dev/null +++ b/annotator/mmpkg/mmseg/datasets/cityscapes.py @@ -0,0 +1,217 @@ +import os.path as osp +import tempfile + +import annotator.mmpkg.mmcv as mmcv +import numpy as np +from annotator.mmpkg.mmcv.utils import print_log +from PIL import Image + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class CityscapesDataset(CustomDataset): + """Cityscapes dataset. + + The ``img_suffix`` is fixed to '_leftImg8bit.png' and ``seg_map_suffix`` is + fixed to '_gtFine_labelTrainIds.png' for Cityscapes dataset. + """ + + CLASSES = ('road', 'sidewalk', 'building', 'wall', 'fence', 'pole', + 'traffic light', 'traffic sign', 'vegetation', 'terrain', 'sky', + 'person', 'rider', 'car', 'truck', 'bus', 'train', 'motorcycle', + 'bicycle') + + PALETTE = [[128, 64, 128], [244, 35, 232], [70, 70, 70], [102, 102, 156], + [190, 153, 153], [153, 153, 153], [250, 170, 30], [220, 220, 0], + [107, 142, 35], [152, 251, 152], [70, 130, 180], [220, 20, 60], + [255, 0, 0], [0, 0, 142], [0, 0, 70], [0, 60, 100], + [0, 80, 100], [0, 0, 230], [119, 11, 32]] + + def __init__(self, **kwargs): + super(CityscapesDataset, self).__init__( + img_suffix='_leftImg8bit.png', + seg_map_suffix='_gtFine_labelTrainIds.png', + **kwargs) + + @staticmethod + def _convert_to_label_id(result): + """Convert trainId to id for cityscapes.""" + if isinstance(result, str): + result = np.load(result) + import cityscapesscripts.helpers.labels as CSLabels + result_copy = result.copy() + for trainId, label in CSLabels.trainId2label.items(): + result_copy[result == trainId] = label.id + + return result_copy + + def results2img(self, results, imgfile_prefix, to_label_id): + """Write the segmentation results to images. + + Args: + results (list[list | tuple | ndarray]): Testing results of the + dataset. + imgfile_prefix (str): The filename prefix of the png files. + If the prefix is "somepath/xxx", + the png files will be named "somepath/xxx.png". + to_label_id (bool): whether convert output to label_id for + submission + + Returns: + list[str: str]: result txt files which contains corresponding + semantic segmentation images. + """ + mmcv.mkdir_or_exist(imgfile_prefix) + result_files = [] + prog_bar = mmcv.ProgressBar(len(self)) + for idx in range(len(self)): + result = results[idx] + if to_label_id: + result = self._convert_to_label_id(result) + filename = self.img_infos[idx]['filename'] + basename = osp.splitext(osp.basename(filename))[0] + + png_filename = osp.join(imgfile_prefix, f'{basename}.png') + + output = Image.fromarray(result.astype(np.uint8)).convert('P') + import cityscapesscripts.helpers.labels as CSLabels + palette = np.zeros((len(CSLabels.id2label), 3), dtype=np.uint8) + for label_id, label in CSLabels.id2label.items(): + palette[label_id] = label.color + + output.putpalette(palette) + output.save(png_filename) + result_files.append(png_filename) + prog_bar.update() + + return result_files + + def format_results(self, results, imgfile_prefix=None, to_label_id=True): + """Format the results into dir (standard format for Cityscapes + evaluation). + + Args: + results (list): Testing results of the dataset. + imgfile_prefix (str | None): The prefix of images files. It + includes the file path and the prefix of filename, e.g., + "a/b/prefix". If not specified, a temp file will be created. + Default: None. + to_label_id (bool): whether convert output to label_id for + submission. Default: False + + Returns: + tuple: (result_files, tmp_dir), result_files is a list containing + the image paths, tmp_dir is the temporal directory created + for saving json/png files when img_prefix is not specified. + """ + + assert isinstance(results, list), 'results must be a list' + assert len(results) == len(self), ( + 'The length of results is not equal to the dataset len: ' + f'{len(results)} != {len(self)}') + + if imgfile_prefix is None: + tmp_dir = tempfile.TemporaryDirectory() + imgfile_prefix = tmp_dir.name + else: + tmp_dir = None + result_files = self.results2img(results, imgfile_prefix, to_label_id) + + return result_files, tmp_dir + + def evaluate(self, + results, + metric='mIoU', + logger=None, + imgfile_prefix=None, + efficient_test=False): + """Evaluation in Cityscapes/default protocol. + + Args: + results (list): Testing results of the dataset. + metric (str | list[str]): Metrics to be evaluated. + logger (logging.Logger | None | str): Logger used for printing + related information during evaluation. Default: None. + imgfile_prefix (str | None): The prefix of output image file, + for cityscapes evaluation only. It includes the file path and + the prefix of filename, e.g., "a/b/prefix". + If results are evaluated with cityscapes protocol, it would be + the prefix of output png files. The output files would be + png images under folder "a/b/prefix/xxx.png", where "xxx" is + the image name of cityscapes. If not specified, a temp file + will be created for evaluation. + Default: None. + + Returns: + dict[str, float]: Cityscapes/default metrics. + """ + + eval_results = dict() + metrics = metric.copy() if isinstance(metric, list) else [metric] + if 'cityscapes' in metrics: + eval_results.update( + self._evaluate_cityscapes(results, logger, imgfile_prefix)) + metrics.remove('cityscapes') + if len(metrics) > 0: + eval_results.update( + super(CityscapesDataset, + self).evaluate(results, metrics, logger, efficient_test)) + + return eval_results + + def _evaluate_cityscapes(self, results, logger, imgfile_prefix): + """Evaluation in Cityscapes protocol. + + Args: + results (list): Testing results of the dataset. + logger (logging.Logger | str | None): Logger used for printing + related information during evaluation. Default: None. + imgfile_prefix (str | None): The prefix of output image file + + Returns: + dict[str: float]: Cityscapes evaluation results. + """ + try: + import cityscapesscripts.evaluation.evalPixelLevelSemanticLabeling as CSEval # noqa + except ImportError: + raise ImportError('Please run "pip install cityscapesscripts" to ' + 'install cityscapesscripts first.') + msg = 'Evaluating in Cityscapes style' + if logger is None: + msg = '\n' + msg + print_log(msg, logger=logger) + + result_files, tmp_dir = self.format_results(results, imgfile_prefix) + + if tmp_dir is None: + result_dir = imgfile_prefix + else: + result_dir = tmp_dir.name + + eval_results = dict() + print_log(f'Evaluating results under {result_dir} ...', logger=logger) + + CSEval.args.evalInstLevelScore = True + CSEval.args.predictionPath = osp.abspath(result_dir) + CSEval.args.evalPixelAccuracy = True + CSEval.args.JSONOutput = False + + seg_map_list = [] + pred_list = [] + + # when evaluating with official cityscapesscripts, + # **_gtFine_labelIds.png is used + for seg_map in mmcv.scandir( + self.ann_dir, 'gtFine_labelIds.png', recursive=True): + seg_map_list.append(osp.join(self.ann_dir, seg_map)) + pred_list.append(CSEval.getPrediction(CSEval.args, seg_map)) + + eval_results.update( + CSEval.evaluateImgLists(pred_list, seg_map_list, CSEval.args)) + + if tmp_dir is not None: + tmp_dir.cleanup() + + return eval_results diff --git a/annotator/mmpkg/mmseg/datasets/custom.py b/annotator/mmpkg/mmseg/datasets/custom.py new file mode 100644 index 0000000000000000000000000000000000000000..3a626976c7fa88c3d1c1e871ef621422acc1be83 --- /dev/null +++ b/annotator/mmpkg/mmseg/datasets/custom.py @@ -0,0 +1,403 @@ +import os +import os.path as osp +from collections import OrderedDict +from functools import reduce + +import annotator.mmpkg.mmcv as mmcv +import numpy as np +from annotator.mmpkg.mmcv.utils import print_log +from torch.utils.data import Dataset + +from annotator.mmpkg.mmseg.core import eval_metrics +from annotator.mmpkg.mmseg.utils import get_root_logger +from .builder import DATASETS +from .pipelines import Compose + + +@DATASETS.register_module() +class CustomDataset(Dataset): + """Custom dataset for semantic segmentation. An example of file structure + is as followed. + + .. code-block:: none + + ├── data + │ ├── my_dataset + │ │ ├── img_dir + │ │ │ ├── train + │ │ │ │ ├── xxx{img_suffix} + │ │ │ │ ├── yyy{img_suffix} + │ │ │ │ ├── zzz{img_suffix} + │ │ │ ├── val + │ │ ├── ann_dir + │ │ │ ├── train + │ │ │ │ ├── xxx{seg_map_suffix} + │ │ │ │ ├── yyy{seg_map_suffix} + │ │ │ │ ├── zzz{seg_map_suffix} + │ │ │ ├── val + + The img/gt_semantic_seg pair of CustomDataset should be of the same + except suffix. A valid img/gt_semantic_seg filename pair should be like + ``xxx{img_suffix}`` and ``xxx{seg_map_suffix}`` (extension is also included + in the suffix). If split is given, then ``xxx`` is specified in txt file. + Otherwise, all files in ``img_dir/``and ``ann_dir`` will be loaded. + Please refer to ``docs/tutorials/new_dataset.md`` for more details. + + + Args: + pipeline (list[dict]): Processing pipeline + img_dir (str): Path to image directory + img_suffix (str): Suffix of images. Default: '.jpg' + ann_dir (str, optional): Path to annotation directory. Default: None + seg_map_suffix (str): Suffix of segmentation maps. Default: '.png' + split (str, optional): Split txt file. If split is specified, only + file with suffix in the splits will be loaded. Otherwise, all + images in img_dir/ann_dir will be loaded. Default: None + data_root (str, optional): Data root for img_dir/ann_dir. Default: + None. + test_mode (bool): If test_mode=True, gt wouldn't be loaded. + ignore_index (int): The label index to be ignored. Default: 255 + reduce_zero_label (bool): Whether to mark label zero as ignored. + Default: False + classes (str | Sequence[str], optional): Specify classes to load. + If is None, ``cls.CLASSES`` will be used. Default: None. + palette (Sequence[Sequence[int]]] | np.ndarray | None): + The palette of segmentation map. If None is given, and + self.PALETTE is None, random palette will be generated. + Default: None + """ + + CLASSES = None + + PALETTE = None + + def __init__(self, + pipeline, + img_dir, + img_suffix='.jpg', + ann_dir=None, + seg_map_suffix='.png', + split=None, + data_root=None, + test_mode=False, + ignore_index=255, + reduce_zero_label=False, + classes=None, + palette=None): + self.pipeline = Compose(pipeline) + self.img_dir = img_dir + self.img_suffix = img_suffix + self.ann_dir = ann_dir + self.seg_map_suffix = seg_map_suffix + self.split = split + self.data_root = data_root + self.test_mode = test_mode + self.ignore_index = ignore_index + self.reduce_zero_label = reduce_zero_label + self.label_map = None + self.CLASSES, self.PALETTE = self.get_classes_and_palette( + classes, palette) + + # join paths if data_root is specified + if self.data_root is not None: + if not osp.isabs(self.img_dir): + self.img_dir = osp.join(self.data_root, self.img_dir) + if not (self.ann_dir is None or osp.isabs(self.ann_dir)): + self.ann_dir = osp.join(self.data_root, self.ann_dir) + if not (self.split is None or osp.isabs(self.split)): + self.split = osp.join(self.data_root, self.split) + + # load annotations + self.img_infos = self.load_annotations(self.img_dir, self.img_suffix, + self.ann_dir, + self.seg_map_suffix, self.split) + + def __len__(self): + """Total number of samples of data.""" + return len(self.img_infos) + + def load_annotations(self, img_dir, img_suffix, ann_dir, seg_map_suffix, + split): + """Load annotation from directory. + + Args: + img_dir (str): Path to image directory + img_suffix (str): Suffix of images. + ann_dir (str|None): Path to annotation directory. + seg_map_suffix (str|None): Suffix of segmentation maps. + split (str|None): Split txt file. If split is specified, only file + with suffix in the splits will be loaded. Otherwise, all images + in img_dir/ann_dir will be loaded. Default: None + + Returns: + list[dict]: All image info of dataset. + """ + + img_infos = [] + if split is not None: + with open(split) as f: + for line in f: + img_name = line.strip() + img_info = dict(filename=img_name + img_suffix) + if ann_dir is not None: + seg_map = img_name + seg_map_suffix + img_info['ann'] = dict(seg_map=seg_map) + img_infos.append(img_info) + else: + for img in mmcv.scandir(img_dir, img_suffix, recursive=True): + img_info = dict(filename=img) + if ann_dir is not None: + seg_map = img.replace(img_suffix, seg_map_suffix) + img_info['ann'] = dict(seg_map=seg_map) + img_infos.append(img_info) + + print_log(f'Loaded {len(img_infos)} images', logger=get_root_logger()) + return img_infos + + def get_ann_info(self, idx): + """Get annotation by index. + + Args: + idx (int): Index of data. + + Returns: + dict: Annotation info of specified index. + """ + + return self.img_infos[idx]['ann'] + + def pre_pipeline(self, results): + """Prepare results dict for pipeline.""" + results['seg_fields'] = [] + results['img_prefix'] = self.img_dir + results['seg_prefix'] = self.ann_dir + if self.custom_classes: + results['label_map'] = self.label_map + + def __getitem__(self, idx): + """Get training/test data after pipeline. + + Args: + idx (int): Index of data. + + Returns: + dict: Training/test data (with annotation if `test_mode` is set + False). + """ + + if self.test_mode: + return self.prepare_test_img(idx) + else: + return self.prepare_train_img(idx) + + def prepare_train_img(self, idx): + """Get training data and annotations after pipeline. + + Args: + idx (int): Index of data. + + Returns: + dict: Training data and annotation after pipeline with new keys + introduced by pipeline. + """ + + img_info = self.img_infos[idx] + ann_info = self.get_ann_info(idx) + results = dict(img_info=img_info, ann_info=ann_info) + self.pre_pipeline(results) + return self.pipeline(results) + + def prepare_test_img(self, idx): + """Get testing data after pipeline. + + Args: + idx (int): Index of data. + + Returns: + dict: Testing data after pipeline with new keys introduced by + pipeline. + """ + + img_info = self.img_infos[idx] + results = dict(img_info=img_info) + self.pre_pipeline(results) + return self.pipeline(results) + + def format_results(self, results, **kwargs): + """Place holder to format result to dataset specific output.""" + + def get_gt_seg_maps(self, efficient_test=False): + """Get ground truth segmentation maps for evaluation.""" + gt_seg_maps = [] + for img_info in self.img_infos: + seg_map = osp.join(self.ann_dir, img_info['ann']['seg_map']) + if efficient_test: + gt_seg_map = seg_map + else: + gt_seg_map = mmcv.imread( + seg_map, flag='unchanged', backend='pillow') + gt_seg_maps.append(gt_seg_map) + return gt_seg_maps + + def get_classes_and_palette(self, classes=None, palette=None): + """Get class names of current dataset. + + Args: + classes (Sequence[str] | str | None): If classes is None, use + default CLASSES defined by builtin dataset. If classes is a + string, take it as a file name. The file contains the name of + classes where each line contains one class name. If classes is + a tuple or list, override the CLASSES defined by the dataset. + palette (Sequence[Sequence[int]]] | np.ndarray | None): + The palette of segmentation map. If None is given, random + palette will be generated. Default: None + """ + if classes is None: + self.custom_classes = False + return self.CLASSES, self.PALETTE + + self.custom_classes = True + if isinstance(classes, str): + # take it as a file path + class_names = mmcv.list_from_file(classes) + elif isinstance(classes, (tuple, list)): + class_names = classes + else: + raise ValueError(f'Unsupported type {type(classes)} of classes.') + + if self.CLASSES: + if not set(classes).issubset(self.CLASSES): + raise ValueError('classes is not a subset of CLASSES.') + + # dictionary, its keys are the old label ids and its values + # are the new label ids. + # used for changing pixel labels in load_annotations. + self.label_map = {} + for i, c in enumerate(self.CLASSES): + if c not in class_names: + self.label_map[i] = -1 + else: + self.label_map[i] = classes.index(c) + + palette = self.get_palette_for_custom_classes(class_names, palette) + + return class_names, palette + + def get_palette_for_custom_classes(self, class_names, palette=None): + + if self.label_map is not None: + # return subset of palette + palette = [] + for old_id, new_id in sorted( + self.label_map.items(), key=lambda x: x[1]): + if new_id != -1: + palette.append(self.PALETTE[old_id]) + palette = type(self.PALETTE)(palette) + + elif palette is None: + if self.PALETTE is None: + palette = np.random.randint(0, 255, size=(len(class_names), 3)) + else: + palette = self.PALETTE + + return palette + + def evaluate(self, + results, + metric='mIoU', + logger=None, + efficient_test=False, + **kwargs): + """Evaluate the dataset. + + Args: + results (list): Testing results of the dataset. + metric (str | list[str]): Metrics to be evaluated. 'mIoU', + 'mDice' and 'mFscore' are supported. + logger (logging.Logger | None | str): Logger used for printing + related information during evaluation. Default: None. + + Returns: + dict[str, float]: Default metrics. + """ + + if isinstance(metric, str): + metric = [metric] + allowed_metrics = ['mIoU', 'mDice', 'mFscore'] + if not set(metric).issubset(set(allowed_metrics)): + raise KeyError('metric {} is not supported'.format(metric)) + eval_results = {} + gt_seg_maps = self.get_gt_seg_maps(efficient_test) + if self.CLASSES is None: + num_classes = len( + reduce(np.union1d, [np.unique(_) for _ in gt_seg_maps])) + else: + num_classes = len(self.CLASSES) + ret_metrics = eval_metrics( + results, + gt_seg_maps, + num_classes, + self.ignore_index, + metric, + label_map=self.label_map, + reduce_zero_label=self.reduce_zero_label) + + if self.CLASSES is None: + class_names = tuple(range(num_classes)) + else: + class_names = self.CLASSES + + # summary table + ret_metrics_summary = OrderedDict({ + ret_metric: np.round(np.nanmean(ret_metric_value) * 100, 2) + for ret_metric, ret_metric_value in ret_metrics.items() + }) + + # each class table + ret_metrics.pop('aAcc', None) + ret_metrics_class = OrderedDict({ + ret_metric: np.round(ret_metric_value * 100, 2) + for ret_metric, ret_metric_value in ret_metrics.items() + }) + ret_metrics_class.update({'Class': class_names}) + ret_metrics_class.move_to_end('Class', last=False) + + try: + from prettytable import PrettyTable + # for logger + class_table_data = PrettyTable() + for key, val in ret_metrics_class.items(): + class_table_data.add_column(key, val) + + summary_table_data = PrettyTable() + for key, val in ret_metrics_summary.items(): + if key == 'aAcc': + summary_table_data.add_column(key, [val]) + else: + summary_table_data.add_column('m' + key, [val]) + + print_log('per class results:', logger) + print_log('\n' + class_table_data.get_string(), logger=logger) + print_log('Summary:', logger) + print_log('\n' + summary_table_data.get_string(), logger=logger) + except ImportError: # prettytable is not installed + pass + + # each metric dict + for key, value in ret_metrics_summary.items(): + if key == 'aAcc': + eval_results[key] = value / 100.0 + else: + eval_results['m' + key] = value / 100.0 + + ret_metrics_class.pop('Class', None) + for key, value in ret_metrics_class.items(): + eval_results.update({ + key + '.' + str(name): value[idx] / 100.0 + for idx, name in enumerate(class_names) + }) + + if mmcv.is_list_of(results, str): + for file_name in results: + os.remove(file_name) + return eval_results diff --git a/annotator/mmpkg/mmseg/datasets/dataset_wrappers.py b/annotator/mmpkg/mmseg/datasets/dataset_wrappers.py new file mode 100644 index 0000000000000000000000000000000000000000..d6a5e957ec3b44465432617cf6e8f0b86a8a5efa --- /dev/null +++ b/annotator/mmpkg/mmseg/datasets/dataset_wrappers.py @@ -0,0 +1,50 @@ +from torch.utils.data.dataset import ConcatDataset as _ConcatDataset + +from .builder import DATASETS + + +@DATASETS.register_module() +class ConcatDataset(_ConcatDataset): + """A wrapper of concatenated dataset. + + Same as :obj:`torch.utils.data.dataset.ConcatDataset`, but + concat the group flag for image aspect ratio. + + Args: + datasets (list[:obj:`Dataset`]): A list of datasets. + """ + + def __init__(self, datasets): + super(ConcatDataset, self).__init__(datasets) + self.CLASSES = datasets[0].CLASSES + self.PALETTE = datasets[0].PALETTE + + +@DATASETS.register_module() +class RepeatDataset(object): + """A wrapper of repeated dataset. + + The length of repeated dataset will be `times` larger than the original + dataset. This is useful when the data loading time is long but the dataset + is small. Using RepeatDataset can reduce the data loading time between + epochs. + + Args: + dataset (:obj:`Dataset`): The dataset to be repeated. + times (int): Repeat times. + """ + + def __init__(self, dataset, times): + self.dataset = dataset + self.times = times + self.CLASSES = dataset.CLASSES + self.PALETTE = dataset.PALETTE + self._ori_len = len(self.dataset) + + def __getitem__(self, idx): + """Get item from original dataset.""" + return self.dataset[idx % self._ori_len] + + def __len__(self): + """The length is multiplied by ``times``""" + return self.times * self._ori_len diff --git a/annotator/mmpkg/mmseg/datasets/drive.py b/annotator/mmpkg/mmseg/datasets/drive.py new file mode 100644 index 0000000000000000000000000000000000000000..3cbfda8ae74bdf26c5aef197ff2866a7c7ad0cfd --- /dev/null +++ b/annotator/mmpkg/mmseg/datasets/drive.py @@ -0,0 +1,27 @@ +import os.path as osp + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class DRIVEDataset(CustomDataset): + """DRIVE dataset. + + In segmentation map annotation for DRIVE, 0 stands for background, which is + included in 2 categories. ``reduce_zero_label`` is fixed to False. The + ``img_suffix`` is fixed to '.png' and ``seg_map_suffix`` is fixed to + '_manual1.png'. + """ + + CLASSES = ('background', 'vessel') + + PALETTE = [[120, 120, 120], [6, 230, 230]] + + def __init__(self, **kwargs): + super(DRIVEDataset, self).__init__( + img_suffix='.png', + seg_map_suffix='_manual1.png', + reduce_zero_label=False, + **kwargs) + assert osp.exists(self.img_dir) diff --git a/annotator/mmpkg/mmseg/datasets/hrf.py b/annotator/mmpkg/mmseg/datasets/hrf.py new file mode 100644 index 0000000000000000000000000000000000000000..923203b51377f9344277fc561803d7a78bd2c684 --- /dev/null +++ b/annotator/mmpkg/mmseg/datasets/hrf.py @@ -0,0 +1,27 @@ +import os.path as osp + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class HRFDataset(CustomDataset): + """HRF dataset. + + In segmentation map annotation for HRF, 0 stands for background, which is + included in 2 categories. ``reduce_zero_label`` is fixed to False. The + ``img_suffix`` is fixed to '.png' and ``seg_map_suffix`` is fixed to + '.png'. + """ + + CLASSES = ('background', 'vessel') + + PALETTE = [[120, 120, 120], [6, 230, 230]] + + def __init__(self, **kwargs): + super(HRFDataset, self).__init__( + img_suffix='.png', + seg_map_suffix='.png', + reduce_zero_label=False, + **kwargs) + assert osp.exists(self.img_dir) diff --git a/annotator/mmpkg/mmseg/datasets/pascal_context.py b/annotator/mmpkg/mmseg/datasets/pascal_context.py new file mode 100644 index 0000000000000000000000000000000000000000..541a63c66a13fb16fd52921e755715ad8d078fdd --- /dev/null +++ b/annotator/mmpkg/mmseg/datasets/pascal_context.py @@ -0,0 +1,103 @@ +import os.path as osp + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class PascalContextDataset(CustomDataset): + """PascalContext dataset. + + In segmentation map annotation for PascalContext, 0 stands for background, + which is included in 60 categories. ``reduce_zero_label`` is fixed to + False. The ``img_suffix`` is fixed to '.jpg' and ``seg_map_suffix`` is + fixed to '.png'. + + Args: + split (str): Split txt file for PascalContext. + """ + + CLASSES = ('background', 'aeroplane', 'bag', 'bed', 'bedclothes', 'bench', + 'bicycle', 'bird', 'boat', 'book', 'bottle', 'building', 'bus', + 'cabinet', 'car', 'cat', 'ceiling', 'chair', 'cloth', + 'computer', 'cow', 'cup', 'curtain', 'dog', 'door', 'fence', + 'floor', 'flower', 'food', 'grass', 'ground', 'horse', + 'keyboard', 'light', 'motorbike', 'mountain', 'mouse', 'person', + 'plate', 'platform', 'pottedplant', 'road', 'rock', 'sheep', + 'shelves', 'sidewalk', 'sign', 'sky', 'snow', 'sofa', 'table', + 'track', 'train', 'tree', 'truck', 'tvmonitor', 'wall', 'water', + 'window', 'wood') + + PALETTE = [[120, 120, 120], [180, 120, 120], [6, 230, 230], [80, 50, 50], + [4, 200, 3], [120, 120, 80], [140, 140, 140], [204, 5, 255], + [230, 230, 230], [4, 250, 7], [224, 5, 255], [235, 255, 7], + [150, 5, 61], [120, 120, 70], [8, 255, 51], [255, 6, 82], + [143, 255, 140], [204, 255, 4], [255, 51, 7], [204, 70, 3], + [0, 102, 200], [61, 230, 250], [255, 6, 51], [11, 102, 255], + [255, 7, 71], [255, 9, 224], [9, 7, 230], [220, 220, 220], + [255, 9, 92], [112, 9, 255], [8, 255, 214], [7, 255, 224], + [255, 184, 6], [10, 255, 71], [255, 41, 10], [7, 255, 255], + [224, 255, 8], [102, 8, 255], [255, 61, 6], [255, 194, 7], + [255, 122, 8], [0, 255, 20], [255, 8, 41], [255, 5, 153], + [6, 51, 255], [235, 12, 255], [160, 150, 20], [0, 163, 255], + [140, 140, 140], [250, 10, 15], [20, 255, 0], [31, 255, 0], + [255, 31, 0], [255, 224, 0], [153, 255, 0], [0, 0, 255], + [255, 71, 0], [0, 235, 255], [0, 173, 255], [31, 0, 255]] + + def __init__(self, split, **kwargs): + super(PascalContextDataset, self).__init__( + img_suffix='.jpg', + seg_map_suffix='.png', + split=split, + reduce_zero_label=False, + **kwargs) + assert osp.exists(self.img_dir) and self.split is not None + + +@DATASETS.register_module() +class PascalContextDataset59(CustomDataset): + """PascalContext dataset. + + In segmentation map annotation for PascalContext, 0 stands for background, + which is included in 60 categories. ``reduce_zero_label`` is fixed to + False. The ``img_suffix`` is fixed to '.jpg' and ``seg_map_suffix`` is + fixed to '.png'. + + Args: + split (str): Split txt file for PascalContext. + """ + + CLASSES = ('aeroplane', 'bag', 'bed', 'bedclothes', 'bench', 'bicycle', + 'bird', 'boat', 'book', 'bottle', 'building', 'bus', 'cabinet', + 'car', 'cat', 'ceiling', 'chair', 'cloth', 'computer', 'cow', + 'cup', 'curtain', 'dog', 'door', 'fence', 'floor', 'flower', + 'food', 'grass', 'ground', 'horse', 'keyboard', 'light', + 'motorbike', 'mountain', 'mouse', 'person', 'plate', 'platform', + 'pottedplant', 'road', 'rock', 'sheep', 'shelves', 'sidewalk', + 'sign', 'sky', 'snow', 'sofa', 'table', 'track', 'train', + 'tree', 'truck', 'tvmonitor', 'wall', 'water', 'window', 'wood') + + PALETTE = [[180, 120, 120], [6, 230, 230], [80, 50, 50], [4, 200, 3], + [120, 120, 80], [140, 140, 140], [204, 5, 255], [230, 230, 230], + [4, 250, 7], [224, 5, 255], [235, 255, 7], [150, 5, 61], + [120, 120, 70], [8, 255, 51], [255, 6, 82], [143, 255, 140], + [204, 255, 4], [255, 51, 7], [204, 70, 3], [0, 102, 200], + [61, 230, 250], [255, 6, 51], [11, 102, 255], [255, 7, 71], + [255, 9, 224], [9, 7, 230], [220, 220, 220], [255, 9, 92], + [112, 9, 255], [8, 255, 214], [7, 255, 224], [255, 184, 6], + [10, 255, 71], [255, 41, 10], [7, 255, 255], [224, 255, 8], + [102, 8, 255], [255, 61, 6], [255, 194, 7], [255, 122, 8], + [0, 255, 20], [255, 8, 41], [255, 5, 153], [6, 51, 255], + [235, 12, 255], [160, 150, 20], [0, 163, 255], [140, 140, 140], + [250, 10, 15], [20, 255, 0], [31, 255, 0], [255, 31, 0], + [255, 224, 0], [153, 255, 0], [0, 0, 255], [255, 71, 0], + [0, 235, 255], [0, 173, 255], [31, 0, 255]] + + def __init__(self, split, **kwargs): + super(PascalContextDataset59, self).__init__( + img_suffix='.jpg', + seg_map_suffix='.png', + split=split, + reduce_zero_label=True, + **kwargs) + assert osp.exists(self.img_dir) and self.split is not None diff --git a/annotator/mmpkg/mmseg/datasets/pipelines/__init__.py b/annotator/mmpkg/mmseg/datasets/pipelines/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b9046b07bb4ddea7a707a392b42e72db7c9df67 --- /dev/null +++ b/annotator/mmpkg/mmseg/datasets/pipelines/__init__.py @@ -0,0 +1,16 @@ +from .compose import Compose +from .formating import (Collect, ImageToTensor, ToDataContainer, ToTensor, + Transpose, to_tensor) +from .loading import LoadAnnotations, LoadImageFromFile +from .test_time_aug import MultiScaleFlipAug +from .transforms import (CLAHE, AdjustGamma, Normalize, Pad, + PhotoMetricDistortion, RandomCrop, RandomFlip, + RandomRotate, Rerange, Resize, RGB2Gray, SegRescale) + +__all__ = [ + 'Compose', 'to_tensor', 'ToTensor', 'ImageToTensor', 'ToDataContainer', + 'Transpose', 'Collect', 'LoadAnnotations', 'LoadImageFromFile', + 'MultiScaleFlipAug', 'Resize', 'RandomFlip', 'Pad', 'RandomCrop', + 'Normalize', 'SegRescale', 'PhotoMetricDistortion', 'RandomRotate', + 'AdjustGamma', 'CLAHE', 'Rerange', 'RGB2Gray' +] diff --git a/annotator/mmpkg/mmseg/datasets/pipelines/compose.py b/annotator/mmpkg/mmseg/datasets/pipelines/compose.py new file mode 100644 index 0000000000000000000000000000000000000000..1683e533237ce6420e4a53e477513853d6b33b3e --- /dev/null +++ b/annotator/mmpkg/mmseg/datasets/pipelines/compose.py @@ -0,0 +1,51 @@ +import collections + +from annotator.mmpkg.mmcv.utils import build_from_cfg + +from ..builder import PIPELINES + + +@PIPELINES.register_module() +class Compose(object): + """Compose multiple transforms sequentially. + + Args: + transforms (Sequence[dict | callable]): Sequence of transform object or + config dict to be composed. + """ + + def __init__(self, transforms): + assert isinstance(transforms, collections.abc.Sequence) + self.transforms = [] + for transform in transforms: + if isinstance(transform, dict): + transform = build_from_cfg(transform, PIPELINES) + self.transforms.append(transform) + elif callable(transform): + self.transforms.append(transform) + else: + raise TypeError('transform must be callable or a dict') + + def __call__(self, data): + """Call function to apply transforms sequentially. + + Args: + data (dict): A result dict contains the data to transform. + + Returns: + dict: Transformed data. + """ + + for t in self.transforms: + data = t(data) + if data is None: + return None + return data + + def __repr__(self): + format_string = self.__class__.__name__ + '(' + for t in self.transforms: + format_string += '\n' + format_string += f' {t}' + format_string += '\n)' + return format_string diff --git a/annotator/mmpkg/mmseg/datasets/pipelines/formating.py b/annotator/mmpkg/mmseg/datasets/pipelines/formating.py new file mode 100644 index 0000000000000000000000000000000000000000..82e2e08ff819506bb7a7693be189017d473e677f --- /dev/null +++ b/annotator/mmpkg/mmseg/datasets/pipelines/formating.py @@ -0,0 +1,288 @@ +from collections.abc import Sequence + +import annotator.mmpkg.mmcv as mmcv +import numpy as np +import torch +from annotator.mmpkg.mmcv.parallel import DataContainer as DC + +from ..builder import PIPELINES + + +def to_tensor(data): + """Convert objects of various python types to :obj:`torch.Tensor`. + + Supported types are: :class:`numpy.ndarray`, :class:`torch.Tensor`, + :class:`Sequence`, :class:`int` and :class:`float`. + + Args: + data (torch.Tensor | numpy.ndarray | Sequence | int | float): Data to + be converted. + """ + + if isinstance(data, torch.Tensor): + return data + elif isinstance(data, np.ndarray): + return torch.from_numpy(data) + elif isinstance(data, Sequence) and not mmcv.is_str(data): + return torch.tensor(data) + elif isinstance(data, int): + return torch.LongTensor([data]) + elif isinstance(data, float): + return torch.FloatTensor([data]) + else: + raise TypeError(f'type {type(data)} cannot be converted to tensor.') + + +@PIPELINES.register_module() +class ToTensor(object): + """Convert some results to :obj:`torch.Tensor` by given keys. + + Args: + keys (Sequence[str]): Keys that need to be converted to Tensor. + """ + + def __init__(self, keys): + self.keys = keys + + def __call__(self, results): + """Call function to convert data in results to :obj:`torch.Tensor`. + + Args: + results (dict): Result dict contains the data to convert. + + Returns: + dict: The result dict contains the data converted + to :obj:`torch.Tensor`. + """ + + for key in self.keys: + results[key] = to_tensor(results[key]) + return results + + def __repr__(self): + return self.__class__.__name__ + f'(keys={self.keys})' + + +@PIPELINES.register_module() +class ImageToTensor(object): + """Convert image to :obj:`torch.Tensor` by given keys. + + The dimension order of input image is (H, W, C). The pipeline will convert + it to (C, H, W). If only 2 dimension (H, W) is given, the output would be + (1, H, W). + + Args: + keys (Sequence[str]): Key of images to be converted to Tensor. + """ + + def __init__(self, keys): + self.keys = keys + + def __call__(self, results): + """Call function to convert image in results to :obj:`torch.Tensor` and + transpose the channel order. + + Args: + results (dict): Result dict contains the image data to convert. + + Returns: + dict: The result dict contains the image converted + to :obj:`torch.Tensor` and transposed to (C, H, W) order. + """ + + for key in self.keys: + img = results[key] + if len(img.shape) < 3: + img = np.expand_dims(img, -1) + results[key] = to_tensor(img.transpose(2, 0, 1)) + return results + + def __repr__(self): + return self.__class__.__name__ + f'(keys={self.keys})' + + +@PIPELINES.register_module() +class Transpose(object): + """Transpose some results by given keys. + + Args: + keys (Sequence[str]): Keys of results to be transposed. + order (Sequence[int]): Order of transpose. + """ + + def __init__(self, keys, order): + self.keys = keys + self.order = order + + def __call__(self, results): + """Call function to convert image in results to :obj:`torch.Tensor` and + transpose the channel order. + + Args: + results (dict): Result dict contains the image data to convert. + + Returns: + dict: The result dict contains the image converted + to :obj:`torch.Tensor` and transposed to (C, H, W) order. + """ + + for key in self.keys: + results[key] = results[key].transpose(self.order) + return results + + def __repr__(self): + return self.__class__.__name__ + \ + f'(keys={self.keys}, order={self.order})' + + +@PIPELINES.register_module() +class ToDataContainer(object): + """Convert results to :obj:`mmcv.DataContainer` by given fields. + + Args: + fields (Sequence[dict]): Each field is a dict like + ``dict(key='xxx', **kwargs)``. The ``key`` in result will + be converted to :obj:`mmcv.DataContainer` with ``**kwargs``. + Default: ``(dict(key='img', stack=True), + dict(key='gt_semantic_seg'))``. + """ + + def __init__(self, + fields=(dict(key='img', + stack=True), dict(key='gt_semantic_seg'))): + self.fields = fields + + def __call__(self, results): + """Call function to convert data in results to + :obj:`mmcv.DataContainer`. + + Args: + results (dict): Result dict contains the data to convert. + + Returns: + dict: The result dict contains the data converted to + :obj:`mmcv.DataContainer`. + """ + + for field in self.fields: + field = field.copy() + key = field.pop('key') + results[key] = DC(results[key], **field) + return results + + def __repr__(self): + return self.__class__.__name__ + f'(fields={self.fields})' + + +@PIPELINES.register_module() +class DefaultFormatBundle(object): + """Default formatting bundle. + + It simplifies the pipeline of formatting common fields, including "img" + and "gt_semantic_seg". These fields are formatted as follows. + + - img: (1)transpose, (2)to tensor, (3)to DataContainer (stack=True) + - gt_semantic_seg: (1)unsqueeze dim-0 (2)to tensor, + (3)to DataContainer (stack=True) + """ + + def __call__(self, results): + """Call function to transform and format common fields in results. + + Args: + results (dict): Result dict contains the data to convert. + + Returns: + dict: The result dict contains the data that is formatted with + default bundle. + """ + + if 'img' in results: + img = results['img'] + if len(img.shape) < 3: + img = np.expand_dims(img, -1) + img = np.ascontiguousarray(img.transpose(2, 0, 1)) + results['img'] = DC(to_tensor(img), stack=True) + if 'gt_semantic_seg' in results: + # convert to long + results['gt_semantic_seg'] = DC( + to_tensor(results['gt_semantic_seg'][None, + ...].astype(np.int64)), + stack=True) + return results + + def __repr__(self): + return self.__class__.__name__ + + +@PIPELINES.register_module() +class Collect(object): + """Collect data from the loader relevant to the specific task. + + This is usually the last stage of the data loader pipeline. Typically keys + is set to some subset of "img", "gt_semantic_seg". + + The "img_meta" item is always populated. The contents of the "img_meta" + dictionary depends on "meta_keys". By default this includes: + + - "img_shape": shape of the image input to the network as a tuple + (h, w, c). Note that images may be zero padded on the bottom/right + if the batch tensor is larger than this shape. + + - "scale_factor": a float indicating the preprocessing scale + + - "flip": a boolean indicating if image flip transform was used + + - "filename": path to the image file + + - "ori_shape": original shape of the image as a tuple (h, w, c) + + - "pad_shape": image shape after padding + + - "img_norm_cfg": a dict of normalization information: + - mean - per channel mean subtraction + - std - per channel std divisor + - to_rgb - bool indicating if bgr was converted to rgb + + Args: + keys (Sequence[str]): Keys of results to be collected in ``data``. + meta_keys (Sequence[str], optional): Meta keys to be converted to + ``mmcv.DataContainer`` and collected in ``data[img_metas]``. + Default: ``('filename', 'ori_filename', 'ori_shape', 'img_shape', + 'pad_shape', 'scale_factor', 'flip', 'flip_direction', + 'img_norm_cfg')`` + """ + + def __init__(self, + keys, + meta_keys=('filename', 'ori_filename', 'ori_shape', + 'img_shape', 'pad_shape', 'scale_factor', 'flip', + 'flip_direction', 'img_norm_cfg')): + self.keys = keys + self.meta_keys = meta_keys + + def __call__(self, results): + """Call function to collect keys in results. The keys in ``meta_keys`` + will be converted to :obj:mmcv.DataContainer. + + Args: + results (dict): Result dict contains the data to collect. + + Returns: + dict: The result dict contains the following keys + - keys in``self.keys`` + - ``img_metas`` + """ + + data = {} + img_meta = {} + for key in self.meta_keys: + img_meta[key] = results[key] + data['img_metas'] = DC(img_meta, cpu_only=True) + for key in self.keys: + data[key] = results[key] + return data + + def __repr__(self): + return self.__class__.__name__ + \ + f'(keys={self.keys}, meta_keys={self.meta_keys})' diff --git a/annotator/mmpkg/mmseg/datasets/pipelines/loading.py b/annotator/mmpkg/mmseg/datasets/pipelines/loading.py new file mode 100644 index 0000000000000000000000000000000000000000..3ad8c2cb67cb1d2b593217fb1fb2e0ca5834c24f --- /dev/null +++ b/annotator/mmpkg/mmseg/datasets/pipelines/loading.py @@ -0,0 +1,153 @@ +import os.path as osp + +import annotator.mmpkg.mmcv as mmcv +import numpy as np + +from ..builder import PIPELINES + + +@PIPELINES.register_module() +class LoadImageFromFile(object): + """Load an image from file. + + Required keys are "img_prefix" and "img_info" (a dict that must contain the + key "filename"). Added or updated keys are "filename", "img", "img_shape", + "ori_shape" (same as `img_shape`), "pad_shape" (same as `img_shape`), + "scale_factor" (1.0) and "img_norm_cfg" (means=0 and stds=1). + + Args: + to_float32 (bool): Whether to convert the loaded image to a float32 + numpy array. If set to False, the loaded image is an uint8 array. + Defaults to False. + color_type (str): The flag argument for :func:`mmcv.imfrombytes`. + Defaults to 'color'. + file_client_args (dict): Arguments to instantiate a FileClient. + See :class:`mmcv.fileio.FileClient` for details. + Defaults to ``dict(backend='disk')``. + imdecode_backend (str): Backend for :func:`mmcv.imdecode`. Default: + 'cv2' + """ + + def __init__(self, + to_float32=False, + color_type='color', + file_client_args=dict(backend='disk'), + imdecode_backend='cv2'): + self.to_float32 = to_float32 + self.color_type = color_type + self.file_client_args = file_client_args.copy() + self.file_client = None + self.imdecode_backend = imdecode_backend + + def __call__(self, results): + """Call functions to load image and get image meta information. + + Args: + results (dict): Result dict from :obj:`mmseg.CustomDataset`. + + Returns: + dict: The dict contains loaded image and meta information. + """ + + if self.file_client is None: + self.file_client = mmcv.FileClient(**self.file_client_args) + + if results.get('img_prefix') is not None: + filename = osp.join(results['img_prefix'], + results['img_info']['filename']) + else: + filename = results['img_info']['filename'] + img_bytes = self.file_client.get(filename) + img = mmcv.imfrombytes( + img_bytes, flag=self.color_type, backend=self.imdecode_backend) + if self.to_float32: + img = img.astype(np.float32) + + results['filename'] = filename + results['ori_filename'] = results['img_info']['filename'] + results['img'] = img + results['img_shape'] = img.shape + results['ori_shape'] = img.shape + # Set initial values for default meta_keys + results['pad_shape'] = img.shape + results['scale_factor'] = 1.0 + num_channels = 1 if len(img.shape) < 3 else img.shape[2] + results['img_norm_cfg'] = dict( + mean=np.zeros(num_channels, dtype=np.float32), + std=np.ones(num_channels, dtype=np.float32), + to_rgb=False) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(to_float32={self.to_float32},' + repr_str += f"color_type='{self.color_type}'," + repr_str += f"imdecode_backend='{self.imdecode_backend}')" + return repr_str + + +@PIPELINES.register_module() +class LoadAnnotations(object): + """Load annotations for semantic segmentation. + + Args: + reduce_zero_label (bool): Whether reduce all label value by 1. + Usually used for datasets where 0 is background label. + Default: False. + file_client_args (dict): Arguments to instantiate a FileClient. + See :class:`mmcv.fileio.FileClient` for details. + Defaults to ``dict(backend='disk')``. + imdecode_backend (str): Backend for :func:`mmcv.imdecode`. Default: + 'pillow' + """ + + def __init__(self, + reduce_zero_label=False, + file_client_args=dict(backend='disk'), + imdecode_backend='pillow'): + self.reduce_zero_label = reduce_zero_label + self.file_client_args = file_client_args.copy() + self.file_client = None + self.imdecode_backend = imdecode_backend + + def __call__(self, results): + """Call function to load multiple types annotations. + + Args: + results (dict): Result dict from :obj:`mmseg.CustomDataset`. + + Returns: + dict: The dict contains loaded semantic segmentation annotations. + """ + + if self.file_client is None: + self.file_client = mmcv.FileClient(**self.file_client_args) + + if results.get('seg_prefix', None) is not None: + filename = osp.join(results['seg_prefix'], + results['ann_info']['seg_map']) + else: + filename = results['ann_info']['seg_map'] + img_bytes = self.file_client.get(filename) + gt_semantic_seg = mmcv.imfrombytes( + img_bytes, flag='unchanged', + backend=self.imdecode_backend).squeeze().astype(np.uint8) + # modify if custom classes + if results.get('label_map', None) is not None: + for old_id, new_id in results['label_map'].items(): + gt_semantic_seg[gt_semantic_seg == old_id] = new_id + # reduce zero_label + if self.reduce_zero_label: + # avoid using underflow conversion + gt_semantic_seg[gt_semantic_seg == 0] = 255 + gt_semantic_seg = gt_semantic_seg - 1 + gt_semantic_seg[gt_semantic_seg == 254] = 255 + results['gt_semantic_seg'] = gt_semantic_seg + results['seg_fields'].append('gt_semantic_seg') + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(reduce_zero_label={self.reduce_zero_label},' + repr_str += f"imdecode_backend='{self.imdecode_backend}')" + return repr_str diff --git a/annotator/mmpkg/mmseg/datasets/pipelines/test_time_aug.py b/annotator/mmpkg/mmseg/datasets/pipelines/test_time_aug.py new file mode 100644 index 0000000000000000000000000000000000000000..fb781d928ed71aceb1abcaef44d3889c00d2261e --- /dev/null +++ b/annotator/mmpkg/mmseg/datasets/pipelines/test_time_aug.py @@ -0,0 +1,133 @@ +import warnings + +import annotator.mmpkg.mmcv as mmcv + +from ..builder import PIPELINES +from .compose import Compose + + +@PIPELINES.register_module() +class MultiScaleFlipAug(object): + """Test-time augmentation with multiple scales and flipping. + + An example configuration is as followed: + + .. code-block:: + + img_scale=(2048, 1024), + img_ratios=[0.5, 1.0], + flip=True, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ] + + After MultiScaleFLipAug with above configuration, the results are wrapped + into lists of the same length as followed: + + .. code-block:: + + dict( + img=[...], + img_shape=[...], + scale=[(1024, 512), (1024, 512), (2048, 1024), (2048, 1024)] + flip=[False, True, False, True] + ... + ) + + Args: + transforms (list[dict]): Transforms to apply in each augmentation. + img_scale (None | tuple | list[tuple]): Images scales for resizing. + img_ratios (float | list[float]): Image ratios for resizing + flip (bool): Whether apply flip augmentation. Default: False. + flip_direction (str | list[str]): Flip augmentation directions, + options are "horizontal" and "vertical". If flip_direction is list, + multiple flip augmentations will be applied. + It has no effect when flip == False. Default: "horizontal". + """ + + def __init__(self, + transforms, + img_scale, + img_ratios=None, + flip=False, + flip_direction='horizontal'): + self.transforms = Compose(transforms) + if img_ratios is not None: + img_ratios = img_ratios if isinstance(img_ratios, + list) else [img_ratios] + assert mmcv.is_list_of(img_ratios, float) + if img_scale is None: + # mode 1: given img_scale=None and a range of image ratio + self.img_scale = None + assert mmcv.is_list_of(img_ratios, float) + elif isinstance(img_scale, tuple) and mmcv.is_list_of( + img_ratios, float): + assert len(img_scale) == 2 + # mode 2: given a scale and a range of image ratio + self.img_scale = [(int(img_scale[0] * ratio), + int(img_scale[1] * ratio)) + for ratio in img_ratios] + else: + # mode 3: given multiple scales + self.img_scale = img_scale if isinstance(img_scale, + list) else [img_scale] + assert mmcv.is_list_of(self.img_scale, tuple) or self.img_scale is None + self.flip = flip + self.img_ratios = img_ratios + self.flip_direction = flip_direction if isinstance( + flip_direction, list) else [flip_direction] + assert mmcv.is_list_of(self.flip_direction, str) + if not self.flip and self.flip_direction != ['horizontal']: + warnings.warn( + 'flip_direction has no effect when flip is set to False') + if (self.flip + and not any([t['type'] == 'RandomFlip' for t in transforms])): + warnings.warn( + 'flip has no effect when RandomFlip is not in transforms') + + def __call__(self, results): + """Call function to apply test time augment transforms on results. + + Args: + results (dict): Result dict contains the data to transform. + + Returns: + dict[str: list]: The augmented data, where each value is wrapped + into a list. + """ + + aug_data = [] + if self.img_scale is None and mmcv.is_list_of(self.img_ratios, float): + h, w = results['img'].shape[:2] + img_scale = [(int(w * ratio), int(h * ratio)) + for ratio in self.img_ratios] + else: + img_scale = self.img_scale + flip_aug = [False, True] if self.flip else [False] + for scale in img_scale: + for flip in flip_aug: + for direction in self.flip_direction: + _results = results.copy() + _results['scale'] = scale + _results['flip'] = flip + _results['flip_direction'] = direction + data = self.transforms(_results) + aug_data.append(data) + # list of dict to dict of list + aug_data_dict = {key: [] for key in aug_data[0]} + for data in aug_data: + for key, val in data.items(): + aug_data_dict[key].append(val) + return aug_data_dict + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(transforms={self.transforms}, ' + repr_str += f'img_scale={self.img_scale}, flip={self.flip})' + repr_str += f'flip_direction={self.flip_direction}' + return repr_str diff --git a/annotator/mmpkg/mmseg/datasets/pipelines/transforms.py b/annotator/mmpkg/mmseg/datasets/pipelines/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..842763db97685dd9280424204d62ee65993fdd5a --- /dev/null +++ b/annotator/mmpkg/mmseg/datasets/pipelines/transforms.py @@ -0,0 +1,889 @@ +import annotator.mmpkg.mmcv as mmcv +import numpy as np +from annotator.mmpkg.mmcv.utils import deprecated_api_warning, is_tuple_of +from numpy import random + +from ..builder import PIPELINES + + +@PIPELINES.register_module() +class Resize(object): + """Resize images & seg. + + This transform resizes the input image to some scale. If the input dict + contains the key "scale", then the scale in the input dict is used, + otherwise the specified scale in the init method is used. + + ``img_scale`` can be None, a tuple (single-scale) or a list of tuple + (multi-scale). There are 4 multiscale modes: + + - ``ratio_range is not None``: + 1. When img_scale is None, img_scale is the shape of image in results + (img_scale = results['img'].shape[:2]) and the image is resized based + on the original size. (mode 1) + 2. When img_scale is a tuple (single-scale), randomly sample a ratio from + the ratio range and multiply it with the image scale. (mode 2) + + - ``ratio_range is None and multiscale_mode == "range"``: randomly sample a + scale from the a range. (mode 3) + + - ``ratio_range is None and multiscale_mode == "value"``: randomly sample a + scale from multiple scales. (mode 4) + + Args: + img_scale (tuple or list[tuple]): Images scales for resizing. + multiscale_mode (str): Either "range" or "value". + ratio_range (tuple[float]): (min_ratio, max_ratio) + keep_ratio (bool): Whether to keep the aspect ratio when resizing the + image. + """ + + def __init__(self, + img_scale=None, + multiscale_mode='range', + ratio_range=None, + keep_ratio=True): + if img_scale is None: + self.img_scale = None + else: + if isinstance(img_scale, list): + self.img_scale = img_scale + else: + self.img_scale = [img_scale] + assert mmcv.is_list_of(self.img_scale, tuple) + + if ratio_range is not None: + # mode 1: given img_scale=None and a range of image ratio + # mode 2: given a scale and a range of image ratio + assert self.img_scale is None or len(self.img_scale) == 1 + else: + # mode 3 and 4: given multiple scales or a range of scales + assert multiscale_mode in ['value', 'range'] + + self.multiscale_mode = multiscale_mode + self.ratio_range = ratio_range + self.keep_ratio = keep_ratio + + @staticmethod + def random_select(img_scales): + """Randomly select an img_scale from given candidates. + + Args: + img_scales (list[tuple]): Images scales for selection. + + Returns: + (tuple, int): Returns a tuple ``(img_scale, scale_dix)``, + where ``img_scale`` is the selected image scale and + ``scale_idx`` is the selected index in the given candidates. + """ + + assert mmcv.is_list_of(img_scales, tuple) + scale_idx = np.random.randint(len(img_scales)) + img_scale = img_scales[scale_idx] + return img_scale, scale_idx + + @staticmethod + def random_sample(img_scales): + """Randomly sample an img_scale when ``multiscale_mode=='range'``. + + Args: + img_scales (list[tuple]): Images scale range for sampling. + There must be two tuples in img_scales, which specify the lower + and upper bound of image scales. + + Returns: + (tuple, None): Returns a tuple ``(img_scale, None)``, where + ``img_scale`` is sampled scale and None is just a placeholder + to be consistent with :func:`random_select`. + """ + + assert mmcv.is_list_of(img_scales, tuple) and len(img_scales) == 2 + img_scale_long = [max(s) for s in img_scales] + img_scale_short = [min(s) for s in img_scales] + long_edge = np.random.randint( + min(img_scale_long), + max(img_scale_long) + 1) + short_edge = np.random.randint( + min(img_scale_short), + max(img_scale_short) + 1) + img_scale = (long_edge, short_edge) + return img_scale, None + + @staticmethod + def random_sample_ratio(img_scale, ratio_range): + """Randomly sample an img_scale when ``ratio_range`` is specified. + + A ratio will be randomly sampled from the range specified by + ``ratio_range``. Then it would be multiplied with ``img_scale`` to + generate sampled scale. + + Args: + img_scale (tuple): Images scale base to multiply with ratio. + ratio_range (tuple[float]): The minimum and maximum ratio to scale + the ``img_scale``. + + Returns: + (tuple, None): Returns a tuple ``(scale, None)``, where + ``scale`` is sampled ratio multiplied with ``img_scale`` and + None is just a placeholder to be consistent with + :func:`random_select`. + """ + + assert isinstance(img_scale, tuple) and len(img_scale) == 2 + min_ratio, max_ratio = ratio_range + assert min_ratio <= max_ratio + ratio = np.random.random_sample() * (max_ratio - min_ratio) + min_ratio + scale = int(img_scale[0] * ratio), int(img_scale[1] * ratio) + return scale, None + + def _random_scale(self, results): + """Randomly sample an img_scale according to ``ratio_range`` and + ``multiscale_mode``. + + If ``ratio_range`` is specified, a ratio will be sampled and be + multiplied with ``img_scale``. + If multiple scales are specified by ``img_scale``, a scale will be + sampled according to ``multiscale_mode``. + Otherwise, single scale will be used. + + Args: + results (dict): Result dict from :obj:`dataset`. + + Returns: + dict: Two new keys 'scale` and 'scale_idx` are added into + ``results``, which would be used by subsequent pipelines. + """ + + if self.ratio_range is not None: + if self.img_scale is None: + h, w = results['img'].shape[:2] + scale, scale_idx = self.random_sample_ratio((w, h), + self.ratio_range) + else: + scale, scale_idx = self.random_sample_ratio( + self.img_scale[0], self.ratio_range) + elif len(self.img_scale) == 1: + scale, scale_idx = self.img_scale[0], 0 + elif self.multiscale_mode == 'range': + scale, scale_idx = self.random_sample(self.img_scale) + elif self.multiscale_mode == 'value': + scale, scale_idx = self.random_select(self.img_scale) + else: + raise NotImplementedError + + results['scale'] = scale + results['scale_idx'] = scale_idx + + def _resize_img(self, results): + """Resize images with ``results['scale']``.""" + if self.keep_ratio: + img, scale_factor = mmcv.imrescale( + results['img'], results['scale'], return_scale=True) + # the w_scale and h_scale has minor difference + # a real fix should be done in the mmcv.imrescale in the future + new_h, new_w = img.shape[:2] + h, w = results['img'].shape[:2] + w_scale = new_w / w + h_scale = new_h / h + else: + img, w_scale, h_scale = mmcv.imresize( + results['img'], results['scale'], return_scale=True) + scale_factor = np.array([w_scale, h_scale, w_scale, h_scale], + dtype=np.float32) + results['img'] = img + results['img_shape'] = img.shape + results['pad_shape'] = img.shape # in case that there is no padding + results['scale_factor'] = scale_factor + results['keep_ratio'] = self.keep_ratio + + def _resize_seg(self, results): + """Resize semantic segmentation map with ``results['scale']``.""" + for key in results.get('seg_fields', []): + if self.keep_ratio: + gt_seg = mmcv.imrescale( + results[key], results['scale'], interpolation='nearest') + else: + gt_seg = mmcv.imresize( + results[key], results['scale'], interpolation='nearest') + results[key] = gt_seg + + def __call__(self, results): + """Call function to resize images, bounding boxes, masks, semantic + segmentation map. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Resized results, 'img_shape', 'pad_shape', 'scale_factor', + 'keep_ratio' keys are added into result dict. + """ + + if 'scale' not in results: + self._random_scale(results) + self._resize_img(results) + self._resize_seg(results) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += (f'(img_scale={self.img_scale}, ' + f'multiscale_mode={self.multiscale_mode}, ' + f'ratio_range={self.ratio_range}, ' + f'keep_ratio={self.keep_ratio})') + return repr_str + + +@PIPELINES.register_module() +class RandomFlip(object): + """Flip the image & seg. + + If the input dict contains the key "flip", then the flag will be used, + otherwise it will be randomly decided by a ratio specified in the init + method. + + Args: + prob (float, optional): The flipping probability. Default: None. + direction(str, optional): The flipping direction. Options are + 'horizontal' and 'vertical'. Default: 'horizontal'. + """ + + @deprecated_api_warning({'flip_ratio': 'prob'}, cls_name='RandomFlip') + def __init__(self, prob=None, direction='horizontal'): + self.prob = prob + self.direction = direction + if prob is not None: + assert prob >= 0 and prob <= 1 + assert direction in ['horizontal', 'vertical'] + + def __call__(self, results): + """Call function to flip bounding boxes, masks, semantic segmentation + maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Flipped results, 'flip', 'flip_direction' keys are added into + result dict. + """ + + if 'flip' not in results: + flip = True if np.random.rand() < self.prob else False + results['flip'] = flip + if 'flip_direction' not in results: + results['flip_direction'] = self.direction + if results['flip']: + # flip image + results['img'] = mmcv.imflip( + results['img'], direction=results['flip_direction']) + + # flip segs + for key in results.get('seg_fields', []): + # use copy() to make numpy stride positive + results[key] = mmcv.imflip( + results[key], direction=results['flip_direction']).copy() + return results + + def __repr__(self): + return self.__class__.__name__ + f'(prob={self.prob})' + + +@PIPELINES.register_module() +class Pad(object): + """Pad the image & mask. + + There are two padding modes: (1) pad to a fixed size and (2) pad to the + minimum size that is divisible by some number. + Added keys are "pad_shape", "pad_fixed_size", "pad_size_divisor", + + Args: + size (tuple, optional): Fixed padding size. + size_divisor (int, optional): The divisor of padded size. + pad_val (float, optional): Padding value. Default: 0. + seg_pad_val (float, optional): Padding value of segmentation map. + Default: 255. + """ + + def __init__(self, + size=None, + size_divisor=None, + pad_val=0, + seg_pad_val=255): + self.size = size + self.size_divisor = size_divisor + self.pad_val = pad_val + self.seg_pad_val = seg_pad_val + # only one of size and size_divisor should be valid + assert size is not None or size_divisor is not None + assert size is None or size_divisor is None + + def _pad_img(self, results): + """Pad images according to ``self.size``.""" + if self.size is not None: + padded_img = mmcv.impad( + results['img'], shape=self.size, pad_val=self.pad_val) + elif self.size_divisor is not None: + padded_img = mmcv.impad_to_multiple( + results['img'], self.size_divisor, pad_val=self.pad_val) + results['img'] = padded_img + results['pad_shape'] = padded_img.shape + results['pad_fixed_size'] = self.size + results['pad_size_divisor'] = self.size_divisor + + def _pad_seg(self, results): + """Pad masks according to ``results['pad_shape']``.""" + for key in results.get('seg_fields', []): + results[key] = mmcv.impad( + results[key], + shape=results['pad_shape'][:2], + pad_val=self.seg_pad_val) + + def __call__(self, results): + """Call function to pad images, masks, semantic segmentation maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Updated result dict. + """ + + self._pad_img(results) + self._pad_seg(results) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(size={self.size}, size_divisor={self.size_divisor}, ' \ + f'pad_val={self.pad_val})' + return repr_str + + +@PIPELINES.register_module() +class Normalize(object): + """Normalize the image. + + Added key is "img_norm_cfg". + + Args: + mean (sequence): Mean values of 3 channels. + std (sequence): Std values of 3 channels. + to_rgb (bool): Whether to convert the image from BGR to RGB, + default is true. + """ + + def __init__(self, mean, std, to_rgb=True): + self.mean = np.array(mean, dtype=np.float32) + self.std = np.array(std, dtype=np.float32) + self.to_rgb = to_rgb + + def __call__(self, results): + """Call function to normalize images. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Normalized results, 'img_norm_cfg' key is added into + result dict. + """ + + results['img'] = mmcv.imnormalize(results['img'], self.mean, self.std, + self.to_rgb) + results['img_norm_cfg'] = dict( + mean=self.mean, std=self.std, to_rgb=self.to_rgb) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(mean={self.mean}, std={self.std}, to_rgb=' \ + f'{self.to_rgb})' + return repr_str + + +@PIPELINES.register_module() +class Rerange(object): + """Rerange the image pixel value. + + Args: + min_value (float or int): Minimum value of the reranged image. + Default: 0. + max_value (float or int): Maximum value of the reranged image. + Default: 255. + """ + + def __init__(self, min_value=0, max_value=255): + assert isinstance(min_value, float) or isinstance(min_value, int) + assert isinstance(max_value, float) or isinstance(max_value, int) + assert min_value < max_value + self.min_value = min_value + self.max_value = max_value + + def __call__(self, results): + """Call function to rerange images. + + Args: + results (dict): Result dict from loading pipeline. + Returns: + dict: Reranged results. + """ + + img = results['img'] + img_min_value = np.min(img) + img_max_value = np.max(img) + + assert img_min_value < img_max_value + # rerange to [0, 1] + img = (img - img_min_value) / (img_max_value - img_min_value) + # rerange to [min_value, max_value] + img = img * (self.max_value - self.min_value) + self.min_value + results['img'] = img + + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(min_value={self.min_value}, max_value={self.max_value})' + return repr_str + + +@PIPELINES.register_module() +class CLAHE(object): + """Use CLAHE method to process the image. + + See `ZUIDERVELD,K. Contrast Limited Adaptive Histogram Equalization[J]. + Graphics Gems, 1994:474-485.` for more information. + + Args: + clip_limit (float): Threshold for contrast limiting. Default: 40.0. + tile_grid_size (tuple[int]): Size of grid for histogram equalization. + Input image will be divided into equally sized rectangular tiles. + It defines the number of tiles in row and column. Default: (8, 8). + """ + + def __init__(self, clip_limit=40.0, tile_grid_size=(8, 8)): + assert isinstance(clip_limit, (float, int)) + self.clip_limit = clip_limit + assert is_tuple_of(tile_grid_size, int) + assert len(tile_grid_size) == 2 + self.tile_grid_size = tile_grid_size + + def __call__(self, results): + """Call function to Use CLAHE method process images. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Processed results. + """ + + for i in range(results['img'].shape[2]): + results['img'][:, :, i] = mmcv.clahe( + np.array(results['img'][:, :, i], dtype=np.uint8), + self.clip_limit, self.tile_grid_size) + + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(clip_limit={self.clip_limit}, '\ + f'tile_grid_size={self.tile_grid_size})' + return repr_str + + +@PIPELINES.register_module() +class RandomCrop(object): + """Random crop the image & seg. + + Args: + crop_size (tuple): Expected size after cropping, (h, w). + cat_max_ratio (float): The maximum ratio that single category could + occupy. + """ + + def __init__(self, crop_size, cat_max_ratio=1., ignore_index=255): + assert crop_size[0] > 0 and crop_size[1] > 0 + self.crop_size = crop_size + self.cat_max_ratio = cat_max_ratio + self.ignore_index = ignore_index + + def get_crop_bbox(self, img): + """Randomly get a crop bounding box.""" + margin_h = max(img.shape[0] - self.crop_size[0], 0) + margin_w = max(img.shape[1] - self.crop_size[1], 0) + offset_h = np.random.randint(0, margin_h + 1) + offset_w = np.random.randint(0, margin_w + 1) + crop_y1, crop_y2 = offset_h, offset_h + self.crop_size[0] + crop_x1, crop_x2 = offset_w, offset_w + self.crop_size[1] + + return crop_y1, crop_y2, crop_x1, crop_x2 + + def crop(self, img, crop_bbox): + """Crop from ``img``""" + crop_y1, crop_y2, crop_x1, crop_x2 = crop_bbox + img = img[crop_y1:crop_y2, crop_x1:crop_x2, ...] + return img + + def __call__(self, results): + """Call function to randomly crop images, semantic segmentation maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Randomly cropped results, 'img_shape' key in result dict is + updated according to crop size. + """ + + img = results['img'] + crop_bbox = self.get_crop_bbox(img) + if self.cat_max_ratio < 1.: + # Repeat 10 times + for _ in range(10): + seg_temp = self.crop(results['gt_semantic_seg'], crop_bbox) + labels, cnt = np.unique(seg_temp, return_counts=True) + cnt = cnt[labels != self.ignore_index] + if len(cnt) > 1 and np.max(cnt) / np.sum( + cnt) < self.cat_max_ratio: + break + crop_bbox = self.get_crop_bbox(img) + + # crop the image + img = self.crop(img, crop_bbox) + img_shape = img.shape + results['img'] = img + results['img_shape'] = img_shape + + # crop semantic seg + for key in results.get('seg_fields', []): + results[key] = self.crop(results[key], crop_bbox) + + return results + + def __repr__(self): + return self.__class__.__name__ + f'(crop_size={self.crop_size})' + + +@PIPELINES.register_module() +class RandomRotate(object): + """Rotate the image & seg. + + Args: + prob (float): The rotation probability. + degree (float, tuple[float]): Range of degrees to select from. If + degree is a number instead of tuple like (min, max), + the range of degree will be (``-degree``, ``+degree``) + pad_val (float, optional): Padding value of image. Default: 0. + seg_pad_val (float, optional): Padding value of segmentation map. + Default: 255. + center (tuple[float], optional): Center point (w, h) of the rotation in + the source image. If not specified, the center of the image will be + used. Default: None. + auto_bound (bool): Whether to adjust the image size to cover the whole + rotated image. Default: False + """ + + def __init__(self, + prob, + degree, + pad_val=0, + seg_pad_val=255, + center=None, + auto_bound=False): + self.prob = prob + assert prob >= 0 and prob <= 1 + if isinstance(degree, (float, int)): + assert degree > 0, f'degree {degree} should be positive' + self.degree = (-degree, degree) + else: + self.degree = degree + assert len(self.degree) == 2, f'degree {self.degree} should be a ' \ + f'tuple of (min, max)' + self.pal_val = pad_val + self.seg_pad_val = seg_pad_val + self.center = center + self.auto_bound = auto_bound + + def __call__(self, results): + """Call function to rotate image, semantic segmentation maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Rotated results. + """ + + rotate = True if np.random.rand() < self.prob else False + degree = np.random.uniform(min(*self.degree), max(*self.degree)) + if rotate: + # rotate image + results['img'] = mmcv.imrotate( + results['img'], + angle=degree, + border_value=self.pal_val, + center=self.center, + auto_bound=self.auto_bound) + + # rotate segs + for key in results.get('seg_fields', []): + results[key] = mmcv.imrotate( + results[key], + angle=degree, + border_value=self.seg_pad_val, + center=self.center, + auto_bound=self.auto_bound, + interpolation='nearest') + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(prob={self.prob}, ' \ + f'degree={self.degree}, ' \ + f'pad_val={self.pal_val}, ' \ + f'seg_pad_val={self.seg_pad_val}, ' \ + f'center={self.center}, ' \ + f'auto_bound={self.auto_bound})' + return repr_str + + +@PIPELINES.register_module() +class RGB2Gray(object): + """Convert RGB image to grayscale image. + + This transform calculate the weighted mean of input image channels with + ``weights`` and then expand the channels to ``out_channels``. When + ``out_channels`` is None, the number of output channels is the same as + input channels. + + Args: + out_channels (int): Expected number of output channels after + transforming. Default: None. + weights (tuple[float]): The weights to calculate the weighted mean. + Default: (0.299, 0.587, 0.114). + """ + + def __init__(self, out_channels=None, weights=(0.299, 0.587, 0.114)): + assert out_channels is None or out_channels > 0 + self.out_channels = out_channels + assert isinstance(weights, tuple) + for item in weights: + assert isinstance(item, (float, int)) + self.weights = weights + + def __call__(self, results): + """Call function to convert RGB image to grayscale image. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Result dict with grayscale image. + """ + img = results['img'] + assert len(img.shape) == 3 + assert img.shape[2] == len(self.weights) + weights = np.array(self.weights).reshape((1, 1, -1)) + img = (img * weights).sum(2, keepdims=True) + if self.out_channels is None: + img = img.repeat(weights.shape[2], axis=2) + else: + img = img.repeat(self.out_channels, axis=2) + + results['img'] = img + results['img_shape'] = img.shape + + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(out_channels={self.out_channels}, ' \ + f'weights={self.weights})' + return repr_str + + +@PIPELINES.register_module() +class AdjustGamma(object): + """Using gamma correction to process the image. + + Args: + gamma (float or int): Gamma value used in gamma correction. + Default: 1.0. + """ + + def __init__(self, gamma=1.0): + assert isinstance(gamma, float) or isinstance(gamma, int) + assert gamma > 0 + self.gamma = gamma + inv_gamma = 1.0 / gamma + self.table = np.array([(i / 255.0)**inv_gamma * 255 + for i in np.arange(256)]).astype('uint8') + + def __call__(self, results): + """Call function to process the image with gamma correction. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Processed results. + """ + + results['img'] = mmcv.lut_transform( + np.array(results['img'], dtype=np.uint8), self.table) + + return results + + def __repr__(self): + return self.__class__.__name__ + f'(gamma={self.gamma})' + + +@PIPELINES.register_module() +class SegRescale(object): + """Rescale semantic segmentation maps. + + Args: + scale_factor (float): The scale factor of the final output. + """ + + def __init__(self, scale_factor=1): + self.scale_factor = scale_factor + + def __call__(self, results): + """Call function to scale the semantic segmentation map. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Result dict with semantic segmentation map scaled. + """ + for key in results.get('seg_fields', []): + if self.scale_factor != 1: + results[key] = mmcv.imrescale( + results[key], self.scale_factor, interpolation='nearest') + return results + + def __repr__(self): + return self.__class__.__name__ + f'(scale_factor={self.scale_factor})' + + +@PIPELINES.register_module() +class PhotoMetricDistortion(object): + """Apply photometric distortion to image sequentially, every transformation + is applied with a probability of 0.5. The position of random contrast is in + second or second to last. + + 1. random brightness + 2. random contrast (mode 0) + 3. convert color from BGR to HSV + 4. random saturation + 5. random hue + 6. convert color from HSV to BGR + 7. random contrast (mode 1) + + Args: + brightness_delta (int): delta of brightness. + contrast_range (tuple): range of contrast. + saturation_range (tuple): range of saturation. + hue_delta (int): delta of hue. + """ + + def __init__(self, + brightness_delta=32, + contrast_range=(0.5, 1.5), + saturation_range=(0.5, 1.5), + hue_delta=18): + self.brightness_delta = brightness_delta + self.contrast_lower, self.contrast_upper = contrast_range + self.saturation_lower, self.saturation_upper = saturation_range + self.hue_delta = hue_delta + + def convert(self, img, alpha=1, beta=0): + """Multiple with alpha and add beat with clip.""" + img = img.astype(np.float32) * alpha + beta + img = np.clip(img, 0, 255) + return img.astype(np.uint8) + + def brightness(self, img): + """Brightness distortion.""" + if random.randint(2): + return self.convert( + img, + beta=random.uniform(-self.brightness_delta, + self.brightness_delta)) + return img + + def contrast(self, img): + """Contrast distortion.""" + if random.randint(2): + return self.convert( + img, + alpha=random.uniform(self.contrast_lower, self.contrast_upper)) + return img + + def saturation(self, img): + """Saturation distortion.""" + if random.randint(2): + img = mmcv.bgr2hsv(img) + img[:, :, 1] = self.convert( + img[:, :, 1], + alpha=random.uniform(self.saturation_lower, + self.saturation_upper)) + img = mmcv.hsv2bgr(img) + return img + + def hue(self, img): + """Hue distortion.""" + if random.randint(2): + img = mmcv.bgr2hsv(img) + img[:, :, + 0] = (img[:, :, 0].astype(int) + + random.randint(-self.hue_delta, self.hue_delta)) % 180 + img = mmcv.hsv2bgr(img) + return img + + def __call__(self, results): + """Call function to perform photometric distortion on images. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Result dict with images distorted. + """ + + img = results['img'] + # random brightness + img = self.brightness(img) + + # mode == 0 --> do random contrast first + # mode == 1 --> do random contrast last + mode = random.randint(2) + if mode == 1: + img = self.contrast(img) + + # random saturation + img = self.saturation(img) + + # random hue + img = self.hue(img) + + # random contrast + if mode == 0: + img = self.contrast(img) + + results['img'] = img + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += (f'(brightness_delta={self.brightness_delta}, ' + f'contrast_range=({self.contrast_lower}, ' + f'{self.contrast_upper}), ' + f'saturation_range=({self.saturation_lower}, ' + f'{self.saturation_upper}), ' + f'hue_delta={self.hue_delta})') + return repr_str diff --git a/annotator/mmpkg/mmseg/datasets/stare.py b/annotator/mmpkg/mmseg/datasets/stare.py new file mode 100644 index 0000000000000000000000000000000000000000..cbd14e0920e7f6a73baff1432e5a32ccfdb0dfae --- /dev/null +++ b/annotator/mmpkg/mmseg/datasets/stare.py @@ -0,0 +1,27 @@ +import os.path as osp + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class STAREDataset(CustomDataset): + """STARE dataset. + + In segmentation map annotation for STARE, 0 stands for background, which is + included in 2 categories. ``reduce_zero_label`` is fixed to False. The + ``img_suffix`` is fixed to '.png' and ``seg_map_suffix`` is fixed to + '.ah.png'. + """ + + CLASSES = ('background', 'vessel') + + PALETTE = [[120, 120, 120], [6, 230, 230]] + + def __init__(self, **kwargs): + super(STAREDataset, self).__init__( + img_suffix='.png', + seg_map_suffix='.ah.png', + reduce_zero_label=False, + **kwargs) + assert osp.exists(self.img_dir) diff --git a/annotator/mmpkg/mmseg/datasets/voc.py b/annotator/mmpkg/mmseg/datasets/voc.py new file mode 100644 index 0000000000000000000000000000000000000000..a8855203b14ee0dc4da9099a2945d4aedcffbcd6 --- /dev/null +++ b/annotator/mmpkg/mmseg/datasets/voc.py @@ -0,0 +1,29 @@ +import os.path as osp + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class PascalVOCDataset(CustomDataset): + """Pascal VOC dataset. + + Args: + split (str): Split txt file for Pascal VOC. + """ + + CLASSES = ('background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', + 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', + 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', + 'train', 'tvmonitor') + + PALETTE = [[0, 0, 0], [128, 0, 0], [0, 128, 0], [128, 128, 0], [0, 0, 128], + [128, 0, 128], [0, 128, 128], [128, 128, 128], [64, 0, 0], + [192, 0, 0], [64, 128, 0], [192, 128, 0], [64, 0, 128], + [192, 0, 128], [64, 128, 128], [192, 128, 128], [0, 64, 0], + [128, 64, 0], [0, 192, 0], [128, 192, 0], [0, 64, 128]] + + def __init__(self, split, **kwargs): + super(PascalVOCDataset, self).__init__( + img_suffix='.jpg', seg_map_suffix='.png', split=split, **kwargs) + assert osp.exists(self.img_dir) and self.split is not None diff --git a/annotator/mmpkg/mmseg/models/__init__.py b/annotator/mmpkg/mmseg/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3cf93f8bec9cf0cef0a3bd76ca3ca92eb188f535 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/__init__.py @@ -0,0 +1,12 @@ +from .backbones import * # noqa: F401,F403 +from .builder import (BACKBONES, HEADS, LOSSES, SEGMENTORS, build_backbone, + build_head, build_loss, build_segmentor) +from .decode_heads import * # noqa: F401,F403 +from .losses import * # noqa: F401,F403 +from .necks import * # noqa: F401,F403 +from .segmentors import * # noqa: F401,F403 + +__all__ = [ + 'BACKBONES', 'HEADS', 'LOSSES', 'SEGMENTORS', 'build_backbone', + 'build_head', 'build_loss', 'build_segmentor' +] diff --git a/annotator/mmpkg/mmseg/models/backbones/__init__.py b/annotator/mmpkg/mmseg/models/backbones/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1116c00a17c8bd9ed7f18743baee22b3b7d3f8d --- /dev/null +++ b/annotator/mmpkg/mmseg/models/backbones/__init__.py @@ -0,0 +1,16 @@ +from .cgnet import CGNet +# from .fast_scnn import FastSCNN +from .hrnet import HRNet +from .mobilenet_v2 import MobileNetV2 +from .mobilenet_v3 import MobileNetV3 +from .resnest import ResNeSt +from .resnet import ResNet, ResNetV1c, ResNetV1d +from .resnext import ResNeXt +from .unet import UNet +from .vit import VisionTransformer + +__all__ = [ + 'ResNet', 'ResNetV1c', 'ResNetV1d', 'ResNeXt', 'HRNet', + 'ResNeSt', 'MobileNetV2', 'UNet', 'CGNet', 'MobileNetV3', + 'VisionTransformer' +] diff --git a/annotator/mmpkg/mmseg/models/backbones/cgnet.py b/annotator/mmpkg/mmseg/models/backbones/cgnet.py new file mode 100644 index 0000000000000000000000000000000000000000..45c235e2e7fcef21e933ecb3ff88a37fa953abe6 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/backbones/cgnet.py @@ -0,0 +1,367 @@ +import torch +import torch.nn as nn +import torch.utils.checkpoint as cp +from annotator.mmpkg.mmcv.cnn import (ConvModule, build_conv_layer, build_norm_layer, + constant_init, kaiming_init) +from annotator.mmpkg.mmcv.runner import load_checkpoint +from annotator.mmpkg.mmcv.utils.parrots_wrapper import _BatchNorm + +from annotator.mmpkg.mmseg.utils import get_root_logger +from ..builder import BACKBONES + + +class GlobalContextExtractor(nn.Module): + """Global Context Extractor for CGNet. + + This class is employed to refine the joint feature of both local feature + and surrounding context. + + Args: + channel (int): Number of input feature channels. + reduction (int): Reductions for global context extractor. Default: 16. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + """ + + def __init__(self, channel, reduction=16, with_cp=False): + super(GlobalContextExtractor, self).__init__() + self.channel = channel + self.reduction = reduction + assert reduction >= 1 and channel >= reduction + self.with_cp = with_cp + self.avg_pool = nn.AdaptiveAvgPool2d(1) + self.fc = nn.Sequential( + nn.Linear(channel, channel // reduction), nn.ReLU(inplace=True), + nn.Linear(channel // reduction, channel), nn.Sigmoid()) + + def forward(self, x): + + def _inner_forward(x): + num_batch, num_channel = x.size()[:2] + y = self.avg_pool(x).view(num_batch, num_channel) + y = self.fc(y).view(num_batch, num_channel, 1, 1) + return x * y + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + return out + + +class ContextGuidedBlock(nn.Module): + """Context Guided Block for CGNet. + + This class consists of four components: local feature extractor, + surrounding feature extractor, joint feature extractor and global + context extractor. + + Args: + in_channels (int): Number of input feature channels. + out_channels (int): Number of output feature channels. + dilation (int): Dilation rate for surrounding context extractor. + Default: 2. + reduction (int): Reduction for global context extractor. Default: 16. + skip_connect (bool): Add input to output or not. Default: True. + downsample (bool): Downsample the input to 1/2 or not. Default: False. + conv_cfg (dict): Config dict for convolution layer. + Default: None, which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN', requires_grad=True). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='PReLU'). + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + """ + + def __init__(self, + in_channels, + out_channels, + dilation=2, + reduction=16, + skip_connect=True, + downsample=False, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + act_cfg=dict(type='PReLU'), + with_cp=False): + super(ContextGuidedBlock, self).__init__() + self.with_cp = with_cp + self.downsample = downsample + + channels = out_channels if downsample else out_channels // 2 + if 'type' in act_cfg and act_cfg['type'] == 'PReLU': + act_cfg['num_parameters'] = channels + kernel_size = 3 if downsample else 1 + stride = 2 if downsample else 1 + padding = (kernel_size - 1) // 2 + + self.conv1x1 = ConvModule( + in_channels, + channels, + kernel_size, + stride, + padding, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + + self.f_loc = build_conv_layer( + conv_cfg, + channels, + channels, + kernel_size=3, + padding=1, + groups=channels, + bias=False) + self.f_sur = build_conv_layer( + conv_cfg, + channels, + channels, + kernel_size=3, + padding=dilation, + groups=channels, + dilation=dilation, + bias=False) + + self.bn = build_norm_layer(norm_cfg, 2 * channels)[1] + self.activate = nn.PReLU(2 * channels) + + if downsample: + self.bottleneck = build_conv_layer( + conv_cfg, + 2 * channels, + out_channels, + kernel_size=1, + bias=False) + + self.skip_connect = skip_connect and not downsample + self.f_glo = GlobalContextExtractor(out_channels, reduction, with_cp) + + def forward(self, x): + + def _inner_forward(x): + out = self.conv1x1(x) + loc = self.f_loc(out) + sur = self.f_sur(out) + + joi_feat = torch.cat([loc, sur], 1) # the joint feature + joi_feat = self.bn(joi_feat) + joi_feat = self.activate(joi_feat) + if self.downsample: + joi_feat = self.bottleneck(joi_feat) # channel = out_channels + # f_glo is employed to refine the joint feature + out = self.f_glo(joi_feat) + + if self.skip_connect: + return x + out + else: + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + return out + + +class InputInjection(nn.Module): + """Downsampling module for CGNet.""" + + def __init__(self, num_downsampling): + super(InputInjection, self).__init__() + self.pool = nn.ModuleList() + for i in range(num_downsampling): + self.pool.append(nn.AvgPool2d(3, stride=2, padding=1)) + + def forward(self, x): + for pool in self.pool: + x = pool(x) + return x + + +@BACKBONES.register_module() +class CGNet(nn.Module): + """CGNet backbone. + + A Light-weight Context Guided Network for Semantic Segmentation + arXiv: https://arxiv.org/abs/1811.08201 + + Args: + in_channels (int): Number of input image channels. Normally 3. + num_channels (tuple[int]): Numbers of feature channels at each stages. + Default: (32, 64, 128). + num_blocks (tuple[int]): Numbers of CG blocks at stage 1 and stage 2. + Default: (3, 21). + dilations (tuple[int]): Dilation rate for surrounding context + extractors at stage 1 and stage 2. Default: (2, 4). + reductions (tuple[int]): Reductions for global context extractors at + stage 1 and stage 2. Default: (8, 16). + conv_cfg (dict): Config dict for convolution layer. + Default: None, which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN', requires_grad=True). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='PReLU'). + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. Default: False. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + """ + + def __init__(self, + in_channels=3, + num_channels=(32, 64, 128), + num_blocks=(3, 21), + dilations=(2, 4), + reductions=(8, 16), + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + act_cfg=dict(type='PReLU'), + norm_eval=False, + with_cp=False): + + super(CGNet, self).__init__() + self.in_channels = in_channels + self.num_channels = num_channels + assert isinstance(self.num_channels, tuple) and len( + self.num_channels) == 3 + self.num_blocks = num_blocks + assert isinstance(self.num_blocks, tuple) and len(self.num_blocks) == 2 + self.dilations = dilations + assert isinstance(self.dilations, tuple) and len(self.dilations) == 2 + self.reductions = reductions + assert isinstance(self.reductions, tuple) and len(self.reductions) == 2 + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + if 'type' in self.act_cfg and self.act_cfg['type'] == 'PReLU': + self.act_cfg['num_parameters'] = num_channels[0] + self.norm_eval = norm_eval + self.with_cp = with_cp + + cur_channels = in_channels + self.stem = nn.ModuleList() + for i in range(3): + self.stem.append( + ConvModule( + cur_channels, + num_channels[0], + 3, + 2 if i == 0 else 1, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + cur_channels = num_channels[0] + + self.inject_2x = InputInjection(1) # down-sample for Input, factor=2 + self.inject_4x = InputInjection(2) # down-sample for Input, factor=4 + + cur_channels += in_channels + self.norm_prelu_0 = nn.Sequential( + build_norm_layer(norm_cfg, cur_channels)[1], + nn.PReLU(cur_channels)) + + # stage 1 + self.level1 = nn.ModuleList() + for i in range(num_blocks[0]): + self.level1.append( + ContextGuidedBlock( + cur_channels if i == 0 else num_channels[1], + num_channels[1], + dilations[0], + reductions[0], + downsample=(i == 0), + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + with_cp=with_cp)) # CG block + + cur_channels = 2 * num_channels[1] + in_channels + self.norm_prelu_1 = nn.Sequential( + build_norm_layer(norm_cfg, cur_channels)[1], + nn.PReLU(cur_channels)) + + # stage 2 + self.level2 = nn.ModuleList() + for i in range(num_blocks[1]): + self.level2.append( + ContextGuidedBlock( + cur_channels if i == 0 else num_channels[2], + num_channels[2], + dilations[1], + reductions[1], + downsample=(i == 0), + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + with_cp=with_cp)) # CG block + + cur_channels = 2 * num_channels[2] + self.norm_prelu_2 = nn.Sequential( + build_norm_layer(norm_cfg, cur_channels)[1], + nn.PReLU(cur_channels)) + + def forward(self, x): + output = [] + + # stage 0 + inp_2x = self.inject_2x(x) + inp_4x = self.inject_4x(x) + for layer in self.stem: + x = layer(x) + x = self.norm_prelu_0(torch.cat([x, inp_2x], 1)) + output.append(x) + + # stage 1 + for i, layer in enumerate(self.level1): + x = layer(x) + if i == 0: + down1 = x + x = self.norm_prelu_1(torch.cat([x, down1, inp_4x], 1)) + output.append(x) + + # stage 2 + for i, layer in enumerate(self.level2): + x = layer(x) + if i == 0: + down2 = x + x = self.norm_prelu_2(torch.cat([down2, x], 1)) + output.append(x) + + return output + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, (nn.Conv2d, nn.Linear)): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + elif isinstance(m, nn.PReLU): + constant_init(m, 0) + else: + raise TypeError('pretrained must be a str or None') + + def train(self, mode=True): + """Convert the model into training mode will keeping the normalization + layer freezed.""" + super(CGNet, self).train(mode) + if mode and self.norm_eval: + for m in self.modules(): + # trick: eval have effect on BatchNorm only + if isinstance(m, _BatchNorm): + m.eval() diff --git a/annotator/mmpkg/mmseg/models/backbones/fast_scnn.py b/annotator/mmpkg/mmseg/models/backbones/fast_scnn.py new file mode 100644 index 0000000000000000000000000000000000000000..417114417ebc830ea11ae7216aa12d8f7a79e5cb --- /dev/null +++ b/annotator/mmpkg/mmseg/models/backbones/fast_scnn.py @@ -0,0 +1,375 @@ +import torch +import torch.nn as nn +from annotator.mmpkg.mmcv.cnn import (ConvModule, DepthwiseSeparableConvModule, constant_init, + kaiming_init) +from torch.nn.modules.batchnorm import _BatchNorm + +from annotator.mmpkg.mmseg.models.decode_heads.psp_head import PPM +from annotator.mmpkg.mmseg.ops import resize +from ..builder import BACKBONES +from ..utils.inverted_residual import InvertedResidual + + +class LearningToDownsample(nn.Module): + """Learning to downsample module. + + Args: + in_channels (int): Number of input channels. + dw_channels (tuple[int]): Number of output channels of the first and + the second depthwise conv (dwconv) layers. + out_channels (int): Number of output channels of the whole + 'learning to downsample' module. + conv_cfg (dict | None): Config of conv layers. Default: None + norm_cfg (dict | None): Config of norm layers. Default: + dict(type='BN') + act_cfg (dict): Config of activation layers. Default: + dict(type='ReLU') + """ + + def __init__(self, + in_channels, + dw_channels, + out_channels, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU')): + super(LearningToDownsample, self).__init__() + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + dw_channels1 = dw_channels[0] + dw_channels2 = dw_channels[1] + + self.conv = ConvModule( + in_channels, + dw_channels1, + 3, + stride=2, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.dsconv1 = DepthwiseSeparableConvModule( + dw_channels1, + dw_channels2, + kernel_size=3, + stride=2, + padding=1, + norm_cfg=self.norm_cfg) + self.dsconv2 = DepthwiseSeparableConvModule( + dw_channels2, + out_channels, + kernel_size=3, + stride=2, + padding=1, + norm_cfg=self.norm_cfg) + + def forward(self, x): + x = self.conv(x) + x = self.dsconv1(x) + x = self.dsconv2(x) + return x + + +class GlobalFeatureExtractor(nn.Module): + """Global feature extractor module. + + Args: + in_channels (int): Number of input channels of the GFE module. + Default: 64 + block_channels (tuple[int]): Tuple of ints. Each int specifies the + number of output channels of each Inverted Residual module. + Default: (64, 96, 128) + out_channels(int): Number of output channels of the GFE module. + Default: 128 + expand_ratio (int): Adjusts number of channels of the hidden layer + in InvertedResidual by this amount. + Default: 6 + num_blocks (tuple[int]): Tuple of ints. Each int specifies the + number of times each Inverted Residual module is repeated. + The repeated Inverted Residual modules are called a 'group'. + Default: (3, 3, 3) + strides (tuple[int]): Tuple of ints. Each int specifies + the downsampling factor of each 'group'. + Default: (2, 2, 1) + pool_scales (tuple[int]): Tuple of ints. Each int specifies + the parameter required in 'global average pooling' within PPM. + Default: (1, 2, 3, 6) + conv_cfg (dict | None): Config of conv layers. Default: None + norm_cfg (dict | None): Config of norm layers. Default: + dict(type='BN') + act_cfg (dict): Config of activation layers. Default: + dict(type='ReLU') + align_corners (bool): align_corners argument of F.interpolate. + Default: False + """ + + def __init__(self, + in_channels=64, + block_channels=(64, 96, 128), + out_channels=128, + expand_ratio=6, + num_blocks=(3, 3, 3), + strides=(2, 2, 1), + pool_scales=(1, 2, 3, 6), + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + align_corners=False): + super(GlobalFeatureExtractor, self).__init__() + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + assert len(block_channels) == len(num_blocks) == 3 + self.bottleneck1 = self._make_layer(in_channels, block_channels[0], + num_blocks[0], strides[0], + expand_ratio) + self.bottleneck2 = self._make_layer(block_channels[0], + block_channels[1], num_blocks[1], + strides[1], expand_ratio) + self.bottleneck3 = self._make_layer(block_channels[1], + block_channels[2], num_blocks[2], + strides[2], expand_ratio) + self.ppm = PPM( + pool_scales, + block_channels[2], + block_channels[2] // 4, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + align_corners=align_corners) + self.out = ConvModule( + block_channels[2] * 2, + out_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def _make_layer(self, + in_channels, + out_channels, + blocks, + stride=1, + expand_ratio=6): + layers = [ + InvertedResidual( + in_channels, + out_channels, + stride, + expand_ratio, + norm_cfg=self.norm_cfg) + ] + for i in range(1, blocks): + layers.append( + InvertedResidual( + out_channels, + out_channels, + 1, + expand_ratio, + norm_cfg=self.norm_cfg)) + return nn.Sequential(*layers) + + def forward(self, x): + x = self.bottleneck1(x) + x = self.bottleneck2(x) + x = self.bottleneck3(x) + x = torch.cat([x, *self.ppm(x)], dim=1) + x = self.out(x) + return x + + +class FeatureFusionModule(nn.Module): + """Feature fusion module. + + Args: + higher_in_channels (int): Number of input channels of the + higher-resolution branch. + lower_in_channels (int): Number of input channels of the + lower-resolution branch. + out_channels (int): Number of output channels. + conv_cfg (dict | None): Config of conv layers. Default: None + norm_cfg (dict | None): Config of norm layers. Default: + dict(type='BN') + act_cfg (dict): Config of activation layers. Default: + dict(type='ReLU') + align_corners (bool): align_corners argument of F.interpolate. + Default: False + """ + + def __init__(self, + higher_in_channels, + lower_in_channels, + out_channels, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + align_corners=False): + super(FeatureFusionModule, self).__init__() + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.align_corners = align_corners + self.dwconv = ConvModule( + lower_in_channels, + out_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.conv_lower_res = ConvModule( + out_channels, + out_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=None) + self.conv_higher_res = ConvModule( + higher_in_channels, + out_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=None) + self.relu = nn.ReLU(True) + + def forward(self, higher_res_feature, lower_res_feature): + lower_res_feature = resize( + lower_res_feature, + size=higher_res_feature.size()[2:], + mode='bilinear', + align_corners=self.align_corners) + lower_res_feature = self.dwconv(lower_res_feature) + lower_res_feature = self.conv_lower_res(lower_res_feature) + + higher_res_feature = self.conv_higher_res(higher_res_feature) + out = higher_res_feature + lower_res_feature + return self.relu(out) + + +@BACKBONES.register_module() +class FastSCNN(nn.Module): + """Fast-SCNN Backbone. + + Args: + in_channels (int): Number of input image channels. Default: 3. + downsample_dw_channels (tuple[int]): Number of output channels after + the first conv layer & the second conv layer in + Learning-To-Downsample (LTD) module. + Default: (32, 48). + global_in_channels (int): Number of input channels of + Global Feature Extractor(GFE). + Equal to number of output channels of LTD. + Default: 64. + global_block_channels (tuple[int]): Tuple of integers that describe + the output channels for each of the MobileNet-v2 bottleneck + residual blocks in GFE. + Default: (64, 96, 128). + global_block_strides (tuple[int]): Tuple of integers + that describe the strides (downsampling factors) for each of the + MobileNet-v2 bottleneck residual blocks in GFE. + Default: (2, 2, 1). + global_out_channels (int): Number of output channels of GFE. + Default: 128. + higher_in_channels (int): Number of input channels of the higher + resolution branch in FFM. + Equal to global_in_channels. + Default: 64. + lower_in_channels (int): Number of input channels of the lower + resolution branch in FFM. + Equal to global_out_channels. + Default: 128. + fusion_out_channels (int): Number of output channels of FFM. + Default: 128. + out_indices (tuple): Tuple of indices of list + [higher_res_features, lower_res_features, fusion_output]. + Often set to (0,1,2) to enable aux. heads. + Default: (0, 1, 2). + conv_cfg (dict | None): Config of conv layers. Default: None + norm_cfg (dict | None): Config of norm layers. Default: + dict(type='BN') + act_cfg (dict): Config of activation layers. Default: + dict(type='ReLU') + align_corners (bool): align_corners argument of F.interpolate. + Default: False + """ + + def __init__(self, + in_channels=3, + downsample_dw_channels=(32, 48), + global_in_channels=64, + global_block_channels=(64, 96, 128), + global_block_strides=(2, 2, 1), + global_out_channels=128, + higher_in_channels=64, + lower_in_channels=128, + fusion_out_channels=128, + out_indices=(0, 1, 2), + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + align_corners=False): + + super(FastSCNN, self).__init__() + if global_in_channels != higher_in_channels: + raise AssertionError('Global Input Channels must be the same \ + with Higher Input Channels!') + elif global_out_channels != lower_in_channels: + raise AssertionError('Global Output Channels must be the same \ + with Lower Input Channels!') + + self.in_channels = in_channels + self.downsample_dw_channels1 = downsample_dw_channels[0] + self.downsample_dw_channels2 = downsample_dw_channels[1] + self.global_in_channels = global_in_channels + self.global_block_channels = global_block_channels + self.global_block_strides = global_block_strides + self.global_out_channels = global_out_channels + self.higher_in_channels = higher_in_channels + self.lower_in_channels = lower_in_channels + self.fusion_out_channels = fusion_out_channels + self.out_indices = out_indices + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.align_corners = align_corners + self.learning_to_downsample = LearningToDownsample( + in_channels, + downsample_dw_channels, + global_in_channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.global_feature_extractor = GlobalFeatureExtractor( + global_in_channels, + global_block_channels, + global_out_channels, + strides=self.global_block_strides, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + align_corners=self.align_corners) + self.feature_fusion = FeatureFusionModule( + higher_in_channels, + lower_in_channels, + fusion_out_channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + align_corners=self.align_corners) + + def init_weights(self, pretrained=None): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + + def forward(self, x): + higher_res_features = self.learning_to_downsample(x) + lower_res_features = self.global_feature_extractor(higher_res_features) + fusion_output = self.feature_fusion(higher_res_features, + lower_res_features) + + outs = [higher_res_features, lower_res_features, fusion_output] + outs = [outs[i] for i in self.out_indices] + return tuple(outs) diff --git a/annotator/mmpkg/mmseg/models/backbones/hrnet.py b/annotator/mmpkg/mmseg/models/backbones/hrnet.py new file mode 100644 index 0000000000000000000000000000000000000000..8d77fd6eadeec25a6b84619f0d7efa7c577b0464 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/backbones/hrnet.py @@ -0,0 +1,555 @@ +import torch.nn as nn +from annotator.mmpkg.mmcv.cnn import (build_conv_layer, build_norm_layer, constant_init, + kaiming_init) +from annotator.mmpkg.mmcv.runner import load_checkpoint +from annotator.mmpkg.mmcv.utils.parrots_wrapper import _BatchNorm + +from annotator.mmpkg.mmseg.ops import Upsample, resize +from annotator.mmpkg.mmseg.utils import get_root_logger +from ..builder import BACKBONES +from .resnet import BasicBlock, Bottleneck + + +class HRModule(nn.Module): + """High-Resolution Module for HRNet. + + In this module, every branch has 4 BasicBlocks/Bottlenecks. Fusion/Exchange + is in this module. + """ + + def __init__(self, + num_branches, + blocks, + num_blocks, + in_channels, + num_channels, + multiscale_output=True, + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True)): + super(HRModule, self).__init__() + self._check_branches(num_branches, num_blocks, in_channels, + num_channels) + + self.in_channels = in_channels + self.num_branches = num_branches + + self.multiscale_output = multiscale_output + self.norm_cfg = norm_cfg + self.conv_cfg = conv_cfg + self.with_cp = with_cp + self.branches = self._make_branches(num_branches, blocks, num_blocks, + num_channels) + self.fuse_layers = self._make_fuse_layers() + self.relu = nn.ReLU(inplace=False) + + def _check_branches(self, num_branches, num_blocks, in_channels, + num_channels): + """Check branches configuration.""" + if num_branches != len(num_blocks): + error_msg = f'NUM_BRANCHES({num_branches}) <> NUM_BLOCKS(' \ + f'{len(num_blocks)})' + raise ValueError(error_msg) + + if num_branches != len(num_channels): + error_msg = f'NUM_BRANCHES({num_branches}) <> NUM_CHANNELS(' \ + f'{len(num_channels)})' + raise ValueError(error_msg) + + if num_branches != len(in_channels): + error_msg = f'NUM_BRANCHES({num_branches}) <> NUM_INCHANNELS(' \ + f'{len(in_channels)})' + raise ValueError(error_msg) + + def _make_one_branch(self, + branch_index, + block, + num_blocks, + num_channels, + stride=1): + """Build one branch.""" + downsample = None + if stride != 1 or \ + self.in_channels[branch_index] != \ + num_channels[branch_index] * block.expansion: + downsample = nn.Sequential( + build_conv_layer( + self.conv_cfg, + self.in_channels[branch_index], + num_channels[branch_index] * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + build_norm_layer(self.norm_cfg, num_channels[branch_index] * + block.expansion)[1]) + + layers = [] + layers.append( + block( + self.in_channels[branch_index], + num_channels[branch_index], + stride, + downsample=downsample, + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + self.in_channels[branch_index] = \ + num_channels[branch_index] * block.expansion + for i in range(1, num_blocks[branch_index]): + layers.append( + block( + self.in_channels[branch_index], + num_channels[branch_index], + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + + return nn.Sequential(*layers) + + def _make_branches(self, num_branches, block, num_blocks, num_channels): + """Build multiple branch.""" + branches = [] + + for i in range(num_branches): + branches.append( + self._make_one_branch(i, block, num_blocks, num_channels)) + + return nn.ModuleList(branches) + + def _make_fuse_layers(self): + """Build fuse layer.""" + if self.num_branches == 1: + return None + + num_branches = self.num_branches + in_channels = self.in_channels + fuse_layers = [] + num_out_branches = num_branches if self.multiscale_output else 1 + for i in range(num_out_branches): + fuse_layer = [] + for j in range(num_branches): + if j > i: + fuse_layer.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels[j], + in_channels[i], + kernel_size=1, + stride=1, + padding=0, + bias=False), + build_norm_layer(self.norm_cfg, in_channels[i])[1], + # we set align_corners=False for HRNet + Upsample( + scale_factor=2**(j - i), + mode='bilinear', + align_corners=False))) + elif j == i: + fuse_layer.append(None) + else: + conv_downsamples = [] + for k in range(i - j): + if k == i - j - 1: + conv_downsamples.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels[j], + in_channels[i], + kernel_size=3, + stride=2, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, + in_channels[i])[1])) + else: + conv_downsamples.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels[j], + in_channels[j], + kernel_size=3, + stride=2, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, + in_channels[j])[1], + nn.ReLU(inplace=False))) + fuse_layer.append(nn.Sequential(*conv_downsamples)) + fuse_layers.append(nn.ModuleList(fuse_layer)) + + return nn.ModuleList(fuse_layers) + + def forward(self, x): + """Forward function.""" + if self.num_branches == 1: + return [self.branches[0](x[0])] + + for i in range(self.num_branches): + x[i] = self.branches[i](x[i]) + + x_fuse = [] + for i in range(len(self.fuse_layers)): + y = 0 + for j in range(self.num_branches): + if i == j: + y += x[j] + elif j > i: + y = y + resize( + self.fuse_layers[i][j](x[j]), + size=x[i].shape[2:], + mode='bilinear', + align_corners=False) + else: + y += self.fuse_layers[i][j](x[j]) + x_fuse.append(self.relu(y)) + return x_fuse + + +@BACKBONES.register_module() +class HRNet(nn.Module): + """HRNet backbone. + + High-Resolution Representations for Labeling Pixels and Regions + arXiv: https://arxiv.org/abs/1904.04514 + + Args: + extra (dict): detailed configuration for each stage of HRNet. + in_channels (int): Number of input image channels. Normally 3. + conv_cfg (dict): dictionary to construct and config conv layer. + norm_cfg (dict): dictionary to construct and config norm layer. + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. + zero_init_residual (bool): whether to use zero init for last norm layer + in resblocks to let them behave as identity. + + Example: + >>> from annotator.mmpkg.mmseg.models import HRNet + >>> import torch + >>> extra = dict( + >>> stage1=dict( + >>> num_modules=1, + >>> num_branches=1, + >>> block='BOTTLENECK', + >>> num_blocks=(4, ), + >>> num_channels=(64, )), + >>> stage2=dict( + >>> num_modules=1, + >>> num_branches=2, + >>> block='BASIC', + >>> num_blocks=(4, 4), + >>> num_channels=(32, 64)), + >>> stage3=dict( + >>> num_modules=4, + >>> num_branches=3, + >>> block='BASIC', + >>> num_blocks=(4, 4, 4), + >>> num_channels=(32, 64, 128)), + >>> stage4=dict( + >>> num_modules=3, + >>> num_branches=4, + >>> block='BASIC', + >>> num_blocks=(4, 4, 4, 4), + >>> num_channels=(32, 64, 128, 256))) + >>> self = HRNet(extra, in_channels=1) + >>> self.eval() + >>> inputs = torch.rand(1, 1, 32, 32) + >>> level_outputs = self.forward(inputs) + >>> for level_out in level_outputs: + ... print(tuple(level_out.shape)) + (1, 32, 8, 8) + (1, 64, 4, 4) + (1, 128, 2, 2) + (1, 256, 1, 1) + """ + + blocks_dict = {'BASIC': BasicBlock, 'BOTTLENECK': Bottleneck} + + def __init__(self, + extra, + in_channels=3, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + norm_eval=False, + with_cp=False, + zero_init_residual=False): + super(HRNet, self).__init__() + self.extra = extra + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.norm_eval = norm_eval + self.with_cp = with_cp + self.zero_init_residual = zero_init_residual + + # stem net + self.norm1_name, norm1 = build_norm_layer(self.norm_cfg, 64, postfix=1) + self.norm2_name, norm2 = build_norm_layer(self.norm_cfg, 64, postfix=2) + + self.conv1 = build_conv_layer( + self.conv_cfg, + in_channels, + 64, + kernel_size=3, + stride=2, + padding=1, + bias=False) + + self.add_module(self.norm1_name, norm1) + self.conv2 = build_conv_layer( + self.conv_cfg, + 64, + 64, + kernel_size=3, + stride=2, + padding=1, + bias=False) + + self.add_module(self.norm2_name, norm2) + self.relu = nn.ReLU(inplace=True) + + # stage 1 + self.stage1_cfg = self.extra['stage1'] + num_channels = self.stage1_cfg['num_channels'][0] + block_type = self.stage1_cfg['block'] + num_blocks = self.stage1_cfg['num_blocks'][0] + + block = self.blocks_dict[block_type] + stage1_out_channels = num_channels * block.expansion + self.layer1 = self._make_layer(block, 64, num_channels, num_blocks) + + # stage 2 + self.stage2_cfg = self.extra['stage2'] + num_channels = self.stage2_cfg['num_channels'] + block_type = self.stage2_cfg['block'] + + block = self.blocks_dict[block_type] + num_channels = [channel * block.expansion for channel in num_channels] + self.transition1 = self._make_transition_layer([stage1_out_channels], + num_channels) + self.stage2, pre_stage_channels = self._make_stage( + self.stage2_cfg, num_channels) + + # stage 3 + self.stage3_cfg = self.extra['stage3'] + num_channels = self.stage3_cfg['num_channels'] + block_type = self.stage3_cfg['block'] + + block = self.blocks_dict[block_type] + num_channels = [channel * block.expansion for channel in num_channels] + self.transition2 = self._make_transition_layer(pre_stage_channels, + num_channels) + self.stage3, pre_stage_channels = self._make_stage( + self.stage3_cfg, num_channels) + + # stage 4 + self.stage4_cfg = self.extra['stage4'] + num_channels = self.stage4_cfg['num_channels'] + block_type = self.stage4_cfg['block'] + + block = self.blocks_dict[block_type] + num_channels = [channel * block.expansion for channel in num_channels] + self.transition3 = self._make_transition_layer(pre_stage_channels, + num_channels) + self.stage4, pre_stage_channels = self._make_stage( + self.stage4_cfg, num_channels) + + @property + def norm1(self): + """nn.Module: the normalization layer named "norm1" """ + return getattr(self, self.norm1_name) + + @property + def norm2(self): + """nn.Module: the normalization layer named "norm2" """ + return getattr(self, self.norm2_name) + + def _make_transition_layer(self, num_channels_pre_layer, + num_channels_cur_layer): + """Make transition layer.""" + num_branches_cur = len(num_channels_cur_layer) + num_branches_pre = len(num_channels_pre_layer) + + transition_layers = [] + for i in range(num_branches_cur): + if i < num_branches_pre: + if num_channels_cur_layer[i] != num_channels_pre_layer[i]: + transition_layers.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + num_channels_pre_layer[i], + num_channels_cur_layer[i], + kernel_size=3, + stride=1, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, + num_channels_cur_layer[i])[1], + nn.ReLU(inplace=True))) + else: + transition_layers.append(None) + else: + conv_downsamples = [] + for j in range(i + 1 - num_branches_pre): + in_channels = num_channels_pre_layer[-1] + out_channels = num_channels_cur_layer[i] \ + if j == i - num_branches_pre else in_channels + conv_downsamples.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels, + out_channels, + kernel_size=3, + stride=2, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, out_channels)[1], + nn.ReLU(inplace=True))) + transition_layers.append(nn.Sequential(*conv_downsamples)) + + return nn.ModuleList(transition_layers) + + def _make_layer(self, block, inplanes, planes, blocks, stride=1): + """Make each layer.""" + downsample = None + if stride != 1 or inplanes != planes * block.expansion: + downsample = nn.Sequential( + build_conv_layer( + self.conv_cfg, + inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + build_norm_layer(self.norm_cfg, planes * block.expansion)[1]) + + layers = [] + layers.append( + block( + inplanes, + planes, + stride, + downsample=downsample, + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append( + block( + inplanes, + planes, + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + + return nn.Sequential(*layers) + + def _make_stage(self, layer_config, in_channels, multiscale_output=True): + """Make each stage.""" + num_modules = layer_config['num_modules'] + num_branches = layer_config['num_branches'] + num_blocks = layer_config['num_blocks'] + num_channels = layer_config['num_channels'] + block = self.blocks_dict[layer_config['block']] + + hr_modules = [] + for i in range(num_modules): + # multi_scale_output is only used for the last module + if not multiscale_output and i == num_modules - 1: + reset_multiscale_output = False + else: + reset_multiscale_output = True + + hr_modules.append( + HRModule( + num_branches, + block, + num_blocks, + in_channels, + num_channels, + reset_multiscale_output, + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + + return nn.Sequential(*hr_modules), in_channels + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + + if self.zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + constant_init(m.norm3, 0) + elif isinstance(m, BasicBlock): + constant_init(m.norm2, 0) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + """Forward function.""" + + x = self.conv1(x) + x = self.norm1(x) + x = self.relu(x) + x = self.conv2(x) + x = self.norm2(x) + x = self.relu(x) + x = self.layer1(x) + + x_list = [] + for i in range(self.stage2_cfg['num_branches']): + if self.transition1[i] is not None: + x_list.append(self.transition1[i](x)) + else: + x_list.append(x) + y_list = self.stage2(x_list) + + x_list = [] + for i in range(self.stage3_cfg['num_branches']): + if self.transition2[i] is not None: + x_list.append(self.transition2[i](y_list[-1])) + else: + x_list.append(y_list[i]) + y_list = self.stage3(x_list) + + x_list = [] + for i in range(self.stage4_cfg['num_branches']): + if self.transition3[i] is not None: + x_list.append(self.transition3[i](y_list[-1])) + else: + x_list.append(y_list[i]) + y_list = self.stage4(x_list) + + return y_list + + def train(self, mode=True): + """Convert the model into training mode will keeping the normalization + layer freezed.""" + super(HRNet, self).train(mode) + if mode and self.norm_eval: + for m in self.modules(): + # trick: eval have effect on BatchNorm only + if isinstance(m, _BatchNorm): + m.eval() diff --git a/annotator/mmpkg/mmseg/models/backbones/mobilenet_v2.py b/annotator/mmpkg/mmseg/models/backbones/mobilenet_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..7b5b6cd6d04c9da04669550d7f1fd24381460bf3 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/backbones/mobilenet_v2.py @@ -0,0 +1,180 @@ +import logging + +import torch.nn as nn +from annotator.mmpkg.mmcv.cnn import ConvModule, constant_init, kaiming_init +from annotator.mmpkg.mmcv.runner import load_checkpoint +from torch.nn.modules.batchnorm import _BatchNorm + +from ..builder import BACKBONES +from ..utils import InvertedResidual, make_divisible + + +@BACKBONES.register_module() +class MobileNetV2(nn.Module): + """MobileNetV2 backbone. + + Args: + widen_factor (float): Width multiplier, multiply number of + channels in each layer by this amount. Default: 1.0. + strides (Sequence[int], optional): Strides of the first block of each + layer. If not specified, default config in ``arch_setting`` will + be used. + dilations (Sequence[int]): Dilation of each layer. + out_indices (None or Sequence[int]): Output from which stages. + Default: (7, ). + frozen_stages (int): Stages to be frozen (all param fixed). + Default: -1, which means not freezing any parameters. + conv_cfg (dict): Config dict for convolution layer. + Default: None, which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='ReLU6'). + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. Default: False. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + """ + + # Parameters to build layers. 3 parameters are needed to construct a + # layer, from left to right: expand_ratio, channel, num_blocks. + arch_settings = [[1, 16, 1], [6, 24, 2], [6, 32, 3], [6, 64, 4], + [6, 96, 3], [6, 160, 3], [6, 320, 1]] + + def __init__(self, + widen_factor=1., + strides=(1, 2, 2, 2, 1, 2, 1), + dilations=(1, 1, 1, 1, 1, 1, 1), + out_indices=(1, 2, 4, 6), + frozen_stages=-1, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU6'), + norm_eval=False, + with_cp=False): + super(MobileNetV2, self).__init__() + self.widen_factor = widen_factor + self.strides = strides + self.dilations = dilations + assert len(strides) == len(dilations) == len(self.arch_settings) + self.out_indices = out_indices + for index in out_indices: + if index not in range(0, 7): + raise ValueError('the item in out_indices must in ' + f'range(0, 8). But received {index}') + + if frozen_stages not in range(-1, 7): + raise ValueError('frozen_stages must be in range(-1, 7). ' + f'But received {frozen_stages}') + self.out_indices = out_indices + self.frozen_stages = frozen_stages + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.norm_eval = norm_eval + self.with_cp = with_cp + + self.in_channels = make_divisible(32 * widen_factor, 8) + + self.conv1 = ConvModule( + in_channels=3, + out_channels=self.in_channels, + kernel_size=3, + stride=2, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + self.layers = [] + + for i, layer_cfg in enumerate(self.arch_settings): + expand_ratio, channel, num_blocks = layer_cfg + stride = self.strides[i] + dilation = self.dilations[i] + out_channels = make_divisible(channel * widen_factor, 8) + inverted_res_layer = self.make_layer( + out_channels=out_channels, + num_blocks=num_blocks, + stride=stride, + dilation=dilation, + expand_ratio=expand_ratio) + layer_name = f'layer{i + 1}' + self.add_module(layer_name, inverted_res_layer) + self.layers.append(layer_name) + + def make_layer(self, out_channels, num_blocks, stride, dilation, + expand_ratio): + """Stack InvertedResidual blocks to build a layer for MobileNetV2. + + Args: + out_channels (int): out_channels of block. + num_blocks (int): Number of blocks. + stride (int): Stride of the first block. + dilation (int): Dilation of the first block. + expand_ratio (int): Expand the number of channels of the + hidden layer in InvertedResidual by this ratio. + """ + layers = [] + for i in range(num_blocks): + layers.append( + InvertedResidual( + self.in_channels, + out_channels, + stride if i == 0 else 1, + expand_ratio=expand_ratio, + dilation=dilation if i == 0 else 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + with_cp=self.with_cp)) + self.in_channels = out_channels + + return nn.Sequential(*layers) + + def init_weights(self, pretrained=None): + if isinstance(pretrained, str): + logger = logging.getLogger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + x = self.conv1(x) + + outs = [] + for i, layer_name in enumerate(self.layers): + layer = getattr(self, layer_name) + x = layer(x) + if i in self.out_indices: + outs.append(x) + + if len(outs) == 1: + return outs[0] + else: + return tuple(outs) + + def _freeze_stages(self): + if self.frozen_stages >= 0: + for param in self.conv1.parameters(): + param.requires_grad = False + for i in range(1, self.frozen_stages + 1): + layer = getattr(self, f'layer{i}') + layer.eval() + for param in layer.parameters(): + param.requires_grad = False + + def train(self, mode=True): + super(MobileNetV2, self).train(mode) + self._freeze_stages() + if mode and self.norm_eval: + for m in self.modules(): + if isinstance(m, _BatchNorm): + m.eval() diff --git a/annotator/mmpkg/mmseg/models/backbones/mobilenet_v3.py b/annotator/mmpkg/mmseg/models/backbones/mobilenet_v3.py new file mode 100644 index 0000000000000000000000000000000000000000..e3c22bdd22356a600454f14c2ed12e7ef72c8ca1 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/backbones/mobilenet_v3.py @@ -0,0 +1,255 @@ +import logging + +import annotator.mmpkg.mmcv as mmcv +import torch.nn as nn +from annotator.mmpkg.mmcv.cnn import ConvModule, constant_init, kaiming_init +from annotator.mmpkg.mmcv.cnn.bricks import Conv2dAdaptivePadding +from annotator.mmpkg.mmcv.runner import load_checkpoint +from torch.nn.modules.batchnorm import _BatchNorm + +from ..builder import BACKBONES +from ..utils import InvertedResidualV3 as InvertedResidual + + +@BACKBONES.register_module() +class MobileNetV3(nn.Module): + """MobileNetV3 backbone. + + This backbone is the improved implementation of `Searching for MobileNetV3 + `_. + + Args: + arch (str): Architecture of mobilnetv3, from {'small', 'large'}. + Default: 'small'. + conv_cfg (dict): Config dict for convolution layer. + Default: None, which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN'). + out_indices (tuple[int]): Output from which layer. + Default: (0, 1, 12). + frozen_stages (int): Stages to be frozen (all param fixed). + Default: -1, which means not freezing any parameters. + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. Default: False. + with_cp (bool): Use checkpoint or not. Using checkpoint will save + some memory while slowing down the training speed. + Default: False. + """ + # Parameters to build each block: + # [kernel size, mid channels, out channels, with_se, act type, stride] + arch_settings = { + 'small': [[3, 16, 16, True, 'ReLU', 2], # block0 layer1 os=4 + [3, 72, 24, False, 'ReLU', 2], # block1 layer2 os=8 + [3, 88, 24, False, 'ReLU', 1], + [5, 96, 40, True, 'HSwish', 2], # block2 layer4 os=16 + [5, 240, 40, True, 'HSwish', 1], + [5, 240, 40, True, 'HSwish', 1], + [5, 120, 48, True, 'HSwish', 1], # block3 layer7 os=16 + [5, 144, 48, True, 'HSwish', 1], + [5, 288, 96, True, 'HSwish', 2], # block4 layer9 os=32 + [5, 576, 96, True, 'HSwish', 1], + [5, 576, 96, True, 'HSwish', 1]], + 'large': [[3, 16, 16, False, 'ReLU', 1], # block0 layer1 os=2 + [3, 64, 24, False, 'ReLU', 2], # block1 layer2 os=4 + [3, 72, 24, False, 'ReLU', 1], + [5, 72, 40, True, 'ReLU', 2], # block2 layer4 os=8 + [5, 120, 40, True, 'ReLU', 1], + [5, 120, 40, True, 'ReLU', 1], + [3, 240, 80, False, 'HSwish', 2], # block3 layer7 os=16 + [3, 200, 80, False, 'HSwish', 1], + [3, 184, 80, False, 'HSwish', 1], + [3, 184, 80, False, 'HSwish', 1], + [3, 480, 112, True, 'HSwish', 1], # block4 layer11 os=16 + [3, 672, 112, True, 'HSwish', 1], + [5, 672, 160, True, 'HSwish', 2], # block5 layer13 os=32 + [5, 960, 160, True, 'HSwish', 1], + [5, 960, 160, True, 'HSwish', 1]] + } # yapf: disable + + def __init__(self, + arch='small', + conv_cfg=None, + norm_cfg=dict(type='BN'), + out_indices=(0, 1, 12), + frozen_stages=-1, + reduction_factor=1, + norm_eval=False, + with_cp=False): + super(MobileNetV3, self).__init__() + assert arch in self.arch_settings + assert isinstance(reduction_factor, int) and reduction_factor > 0 + assert mmcv.is_tuple_of(out_indices, int) + for index in out_indices: + if index not in range(0, len(self.arch_settings[arch]) + 2): + raise ValueError( + 'the item in out_indices must in ' + f'range(0, {len(self.arch_settings[arch])+2}). ' + f'But received {index}') + + if frozen_stages not in range(-1, len(self.arch_settings[arch]) + 2): + raise ValueError('frozen_stages must be in range(-1, ' + f'{len(self.arch_settings[arch])+2}). ' + f'But received {frozen_stages}') + self.arch = arch + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.out_indices = out_indices + self.frozen_stages = frozen_stages + self.reduction_factor = reduction_factor + self.norm_eval = norm_eval + self.with_cp = with_cp + self.layers = self._make_layer() + + def _make_layer(self): + layers = [] + + # build the first layer (layer0) + in_channels = 16 + layer = ConvModule( + in_channels=3, + out_channels=in_channels, + kernel_size=3, + stride=2, + padding=1, + conv_cfg=dict(type='Conv2dAdaptivePadding'), + norm_cfg=self.norm_cfg, + act_cfg=dict(type='HSwish')) + self.add_module('layer0', layer) + layers.append('layer0') + + layer_setting = self.arch_settings[self.arch] + for i, params in enumerate(layer_setting): + (kernel_size, mid_channels, out_channels, with_se, act, + stride) = params + + if self.arch == 'large' and i >= 12 or self.arch == 'small' and \ + i >= 8: + mid_channels = mid_channels // self.reduction_factor + out_channels = out_channels // self.reduction_factor + + if with_se: + se_cfg = dict( + channels=mid_channels, + ratio=4, + act_cfg=(dict(type='ReLU'), + dict(type='HSigmoid', bias=3.0, divisor=6.0))) + else: + se_cfg = None + + layer = InvertedResidual( + in_channels=in_channels, + out_channels=out_channels, + mid_channels=mid_channels, + kernel_size=kernel_size, + stride=stride, + se_cfg=se_cfg, + with_expand_conv=(in_channels != mid_channels), + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=dict(type=act), + with_cp=self.with_cp) + in_channels = out_channels + layer_name = 'layer{}'.format(i + 1) + self.add_module(layer_name, layer) + layers.append(layer_name) + + # build the last layer + # block5 layer12 os=32 for small model + # block6 layer16 os=32 for large model + layer = ConvModule( + in_channels=in_channels, + out_channels=576 if self.arch == 'small' else 960, + kernel_size=1, + stride=1, + dilation=4, + padding=0, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=dict(type='HSwish')) + layer_name = 'layer{}'.format(len(layer_setting) + 1) + self.add_module(layer_name, layer) + layers.append(layer_name) + + # next, convert backbone MobileNetV3 to a semantic segmentation version + if self.arch == 'small': + self.layer4.depthwise_conv.conv.stride = (1, 1) + self.layer9.depthwise_conv.conv.stride = (1, 1) + for i in range(4, len(layers)): + layer = getattr(self, layers[i]) + if isinstance(layer, InvertedResidual): + modified_module = layer.depthwise_conv.conv + else: + modified_module = layer.conv + + if i < 9: + modified_module.dilation = (2, 2) + pad = 2 + else: + modified_module.dilation = (4, 4) + pad = 4 + + if not isinstance(modified_module, Conv2dAdaptivePadding): + # Adjust padding + pad *= (modified_module.kernel_size[0] - 1) // 2 + modified_module.padding = (pad, pad) + else: + self.layer7.depthwise_conv.conv.stride = (1, 1) + self.layer13.depthwise_conv.conv.stride = (1, 1) + for i in range(7, len(layers)): + layer = getattr(self, layers[i]) + if isinstance(layer, InvertedResidual): + modified_module = layer.depthwise_conv.conv + else: + modified_module = layer.conv + + if i < 13: + modified_module.dilation = (2, 2) + pad = 2 + else: + modified_module.dilation = (4, 4) + pad = 4 + + if not isinstance(modified_module, Conv2dAdaptivePadding): + # Adjust padding + pad *= (modified_module.kernel_size[0] - 1) // 2 + modified_module.padding = (pad, pad) + + return layers + + def init_weights(self, pretrained=None): + if isinstance(pretrained, str): + logger = logging.getLogger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, nn.BatchNorm2d): + constant_init(m, 1) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + outs = [] + for i, layer_name in enumerate(self.layers): + layer = getattr(self, layer_name) + x = layer(x) + if i in self.out_indices: + outs.append(x) + return outs + + def _freeze_stages(self): + for i in range(self.frozen_stages + 1): + layer = getattr(self, f'layer{i}') + layer.eval() + for param in layer.parameters(): + param.requires_grad = False + + def train(self, mode=True): + super(MobileNetV3, self).train(mode) + self._freeze_stages() + if mode and self.norm_eval: + for m in self.modules(): + if isinstance(m, _BatchNorm): + m.eval() diff --git a/annotator/mmpkg/mmseg/models/backbones/resnest.py b/annotator/mmpkg/mmseg/models/backbones/resnest.py new file mode 100644 index 0000000000000000000000000000000000000000..076ef62195bac2a9660261446b5756c3880dfdf2 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/backbones/resnest.py @@ -0,0 +1,314 @@ +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as cp +from annotator.mmpkg.mmcv.cnn import build_conv_layer, build_norm_layer + +from ..builder import BACKBONES +from ..utils import ResLayer +from .resnet import Bottleneck as _Bottleneck +from .resnet import ResNetV1d + + +class RSoftmax(nn.Module): + """Radix Softmax module in ``SplitAttentionConv2d``. + + Args: + radix (int): Radix of input. + groups (int): Groups of input. + """ + + def __init__(self, radix, groups): + super().__init__() + self.radix = radix + self.groups = groups + + def forward(self, x): + batch = x.size(0) + if self.radix > 1: + x = x.view(batch, self.groups, self.radix, -1).transpose(1, 2) + x = F.softmax(x, dim=1) + x = x.reshape(batch, -1) + else: + x = torch.sigmoid(x) + return x + + +class SplitAttentionConv2d(nn.Module): + """Split-Attention Conv2d in ResNeSt. + + Args: + in_channels (int): Same as nn.Conv2d. + out_channels (int): Same as nn.Conv2d. + kernel_size (int | tuple[int]): Same as nn.Conv2d. + stride (int | tuple[int]): Same as nn.Conv2d. + padding (int | tuple[int]): Same as nn.Conv2d. + dilation (int | tuple[int]): Same as nn.Conv2d. + groups (int): Same as nn.Conv2d. + radix (int): Radix of SpltAtConv2d. Default: 2 + reduction_factor (int): Reduction factor of inter_channels. Default: 4. + conv_cfg (dict): Config dict for convolution layer. Default: None, + which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. Default: None. + dcn (dict): Config dict for DCN. Default: None. + """ + + def __init__(self, + in_channels, + channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + radix=2, + reduction_factor=4, + conv_cfg=None, + norm_cfg=dict(type='BN'), + dcn=None): + super(SplitAttentionConv2d, self).__init__() + inter_channels = max(in_channels * radix // reduction_factor, 32) + self.radix = radix + self.groups = groups + self.channels = channels + self.with_dcn = dcn is not None + self.dcn = dcn + fallback_on_stride = False + if self.with_dcn: + fallback_on_stride = self.dcn.pop('fallback_on_stride', False) + if self.with_dcn and not fallback_on_stride: + assert conv_cfg is None, 'conv_cfg must be None for DCN' + conv_cfg = dcn + self.conv = build_conv_layer( + conv_cfg, + in_channels, + channels * radix, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups * radix, + bias=False) + self.norm0_name, norm0 = build_norm_layer( + norm_cfg, channels * radix, postfix=0) + self.add_module(self.norm0_name, norm0) + self.relu = nn.ReLU(inplace=True) + self.fc1 = build_conv_layer( + None, channels, inter_channels, 1, groups=self.groups) + self.norm1_name, norm1 = build_norm_layer( + norm_cfg, inter_channels, postfix=1) + self.add_module(self.norm1_name, norm1) + self.fc2 = build_conv_layer( + None, inter_channels, channels * radix, 1, groups=self.groups) + self.rsoftmax = RSoftmax(radix, groups) + + @property + def norm0(self): + """nn.Module: the normalization layer named "norm0" """ + return getattr(self, self.norm0_name) + + @property + def norm1(self): + """nn.Module: the normalization layer named "norm1" """ + return getattr(self, self.norm1_name) + + def forward(self, x): + x = self.conv(x) + x = self.norm0(x) + x = self.relu(x) + + batch, rchannel = x.shape[:2] + batch = x.size(0) + if self.radix > 1: + splits = x.view(batch, self.radix, -1, *x.shape[2:]) + gap = splits.sum(dim=1) + else: + gap = x + gap = F.adaptive_avg_pool2d(gap, 1) + gap = self.fc1(gap) + + gap = self.norm1(gap) + gap = self.relu(gap) + + atten = self.fc2(gap) + atten = self.rsoftmax(atten).view(batch, -1, 1, 1) + + if self.radix > 1: + attens = atten.view(batch, self.radix, -1, *atten.shape[2:]) + out = torch.sum(attens * splits, dim=1) + else: + out = atten * x + return out.contiguous() + + +class Bottleneck(_Bottleneck): + """Bottleneck block for ResNeSt. + + Args: + inplane (int): Input planes of this block. + planes (int): Middle planes of this block. + groups (int): Groups of conv2. + width_per_group (int): Width per group of conv2. 64x4d indicates + ``groups=64, width_per_group=4`` and 32x8d indicates + ``groups=32, width_per_group=8``. + radix (int): Radix of SpltAtConv2d. Default: 2 + reduction_factor (int): Reduction factor of inter_channels in + SplitAttentionConv2d. Default: 4. + avg_down_stride (bool): Whether to use average pool for stride in + Bottleneck. Default: True. + kwargs (dict): Key word arguments for base class. + """ + expansion = 4 + + def __init__(self, + inplanes, + planes, + groups=1, + base_width=4, + base_channels=64, + radix=2, + reduction_factor=4, + avg_down_stride=True, + **kwargs): + """Bottleneck block for ResNeSt.""" + super(Bottleneck, self).__init__(inplanes, planes, **kwargs) + + if groups == 1: + width = self.planes + else: + width = math.floor(self.planes * + (base_width / base_channels)) * groups + + self.avg_down_stride = avg_down_stride and self.conv2_stride > 1 + + self.norm1_name, norm1 = build_norm_layer( + self.norm_cfg, width, postfix=1) + self.norm3_name, norm3 = build_norm_layer( + self.norm_cfg, self.planes * self.expansion, postfix=3) + + self.conv1 = build_conv_layer( + self.conv_cfg, + self.inplanes, + width, + kernel_size=1, + stride=self.conv1_stride, + bias=False) + self.add_module(self.norm1_name, norm1) + self.with_modulated_dcn = False + self.conv2 = SplitAttentionConv2d( + width, + width, + kernel_size=3, + stride=1 if self.avg_down_stride else self.conv2_stride, + padding=self.dilation, + dilation=self.dilation, + groups=groups, + radix=radix, + reduction_factor=reduction_factor, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + dcn=self.dcn) + delattr(self, self.norm2_name) + + if self.avg_down_stride: + self.avd_layer = nn.AvgPool2d(3, self.conv2_stride, padding=1) + + self.conv3 = build_conv_layer( + self.conv_cfg, + width, + self.planes * self.expansion, + kernel_size=1, + bias=False) + self.add_module(self.norm3_name, norm3) + + def forward(self, x): + + def _inner_forward(x): + identity = x + + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv1_plugin_names) + + out = self.conv2(out) + + if self.avg_down_stride: + out = self.avd_layer(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv2_plugin_names) + + out = self.conv3(out) + out = self.norm3(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv3_plugin_names) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + out = self.relu(out) + + return out + + +@BACKBONES.register_module() +class ResNeSt(ResNetV1d): + """ResNeSt backbone. + + Args: + groups (int): Number of groups of Bottleneck. Default: 1 + base_width (int): Base width of Bottleneck. Default: 4 + radix (int): Radix of SpltAtConv2d. Default: 2 + reduction_factor (int): Reduction factor of inter_channels in + SplitAttentionConv2d. Default: 4. + avg_down_stride (bool): Whether to use average pool for stride in + Bottleneck. Default: True. + kwargs (dict): Keyword arguments for ResNet. + """ + + arch_settings = { + 50: (Bottleneck, (3, 4, 6, 3)), + 101: (Bottleneck, (3, 4, 23, 3)), + 152: (Bottleneck, (3, 8, 36, 3)), + 200: (Bottleneck, (3, 24, 36, 3)) + } + + def __init__(self, + groups=1, + base_width=4, + radix=2, + reduction_factor=4, + avg_down_stride=True, + **kwargs): + self.groups = groups + self.base_width = base_width + self.radix = radix + self.reduction_factor = reduction_factor + self.avg_down_stride = avg_down_stride + super(ResNeSt, self).__init__(**kwargs) + + def make_res_layer(self, **kwargs): + """Pack all blocks in a stage into a ``ResLayer``.""" + return ResLayer( + groups=self.groups, + base_width=self.base_width, + base_channels=self.base_channels, + radix=self.radix, + reduction_factor=self.reduction_factor, + avg_down_stride=self.avg_down_stride, + **kwargs) diff --git a/annotator/mmpkg/mmseg/models/backbones/resnet.py b/annotator/mmpkg/mmseg/models/backbones/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..b3304dc5238110adcf21fa4c0a4e230158894fea --- /dev/null +++ b/annotator/mmpkg/mmseg/models/backbones/resnet.py @@ -0,0 +1,688 @@ +import torch.nn as nn +import torch.utils.checkpoint as cp +from annotator.mmpkg.mmcv.cnn import (build_conv_layer, build_norm_layer, build_plugin_layer, + constant_init, kaiming_init) +from annotator.mmpkg.mmcv.runner import load_checkpoint +from annotator.mmpkg.mmcv.utils.parrots_wrapper import _BatchNorm + +from annotator.mmpkg.mmseg.utils import get_root_logger +from ..builder import BACKBONES +from ..utils import ResLayer + + +class BasicBlock(nn.Module): + """Basic block for ResNet.""" + + expansion = 1 + + def __init__(self, + inplanes, + planes, + stride=1, + dilation=1, + downsample=None, + style='pytorch', + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + dcn=None, + plugins=None): + super(BasicBlock, self).__init__() + assert dcn is None, 'Not implemented yet.' + assert plugins is None, 'Not implemented yet.' + + self.norm1_name, norm1 = build_norm_layer(norm_cfg, planes, postfix=1) + self.norm2_name, norm2 = build_norm_layer(norm_cfg, planes, postfix=2) + + self.conv1 = build_conv_layer( + conv_cfg, + inplanes, + planes, + 3, + stride=stride, + padding=dilation, + dilation=dilation, + bias=False) + self.add_module(self.norm1_name, norm1) + self.conv2 = build_conv_layer( + conv_cfg, planes, planes, 3, padding=1, bias=False) + self.add_module(self.norm2_name, norm2) + + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + self.dilation = dilation + self.with_cp = with_cp + + @property + def norm1(self): + """nn.Module: normalization layer after the first convolution layer""" + return getattr(self, self.norm1_name) + + @property + def norm2(self): + """nn.Module: normalization layer after the second convolution layer""" + return getattr(self, self.norm2_name) + + def forward(self, x): + """Forward function.""" + + def _inner_forward(x): + identity = x + + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.norm2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + """Bottleneck block for ResNet. + + If style is "pytorch", the stride-two layer is the 3x3 conv layer, if it is + "caffe", the stride-two layer is the first 1x1 conv layer. + """ + + expansion = 4 + + def __init__(self, + inplanes, + planes, + stride=1, + dilation=1, + downsample=None, + style='pytorch', + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + dcn=None, + plugins=None): + super(Bottleneck, self).__init__() + assert style in ['pytorch', 'caffe'] + assert dcn is None or isinstance(dcn, dict) + assert plugins is None or isinstance(plugins, list) + if plugins is not None: + allowed_position = ['after_conv1', 'after_conv2', 'after_conv3'] + assert all(p['position'] in allowed_position for p in plugins) + + self.inplanes = inplanes + self.planes = planes + self.stride = stride + self.dilation = dilation + self.style = style + self.with_cp = with_cp + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.dcn = dcn + self.with_dcn = dcn is not None + self.plugins = plugins + self.with_plugins = plugins is not None + + if self.with_plugins: + # collect plugins for conv1/conv2/conv3 + self.after_conv1_plugins = [ + plugin['cfg'] for plugin in plugins + if plugin['position'] == 'after_conv1' + ] + self.after_conv2_plugins = [ + plugin['cfg'] for plugin in plugins + if plugin['position'] == 'after_conv2' + ] + self.after_conv3_plugins = [ + plugin['cfg'] for plugin in plugins + if plugin['position'] == 'after_conv3' + ] + + if self.style == 'pytorch': + self.conv1_stride = 1 + self.conv2_stride = stride + else: + self.conv1_stride = stride + self.conv2_stride = 1 + + self.norm1_name, norm1 = build_norm_layer(norm_cfg, planes, postfix=1) + self.norm2_name, norm2 = build_norm_layer(norm_cfg, planes, postfix=2) + self.norm3_name, norm3 = build_norm_layer( + norm_cfg, planes * self.expansion, postfix=3) + + self.conv1 = build_conv_layer( + conv_cfg, + inplanes, + planes, + kernel_size=1, + stride=self.conv1_stride, + bias=False) + self.add_module(self.norm1_name, norm1) + fallback_on_stride = False + if self.with_dcn: + fallback_on_stride = dcn.pop('fallback_on_stride', False) + if not self.with_dcn or fallback_on_stride: + self.conv2 = build_conv_layer( + conv_cfg, + planes, + planes, + kernel_size=3, + stride=self.conv2_stride, + padding=dilation, + dilation=dilation, + bias=False) + else: + assert self.conv_cfg is None, 'conv_cfg must be None for DCN' + self.conv2 = build_conv_layer( + dcn, + planes, + planes, + kernel_size=3, + stride=self.conv2_stride, + padding=dilation, + dilation=dilation, + bias=False) + + self.add_module(self.norm2_name, norm2) + self.conv3 = build_conv_layer( + conv_cfg, + planes, + planes * self.expansion, + kernel_size=1, + bias=False) + self.add_module(self.norm3_name, norm3) + + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + + if self.with_plugins: + self.after_conv1_plugin_names = self.make_block_plugins( + planes, self.after_conv1_plugins) + self.after_conv2_plugin_names = self.make_block_plugins( + planes, self.after_conv2_plugins) + self.after_conv3_plugin_names = self.make_block_plugins( + planes * self.expansion, self.after_conv3_plugins) + + def make_block_plugins(self, in_channels, plugins): + """make plugins for block. + + Args: + in_channels (int): Input channels of plugin. + plugins (list[dict]): List of plugins cfg to build. + + Returns: + list[str]: List of the names of plugin. + """ + assert isinstance(plugins, list) + plugin_names = [] + for plugin in plugins: + plugin = plugin.copy() + name, layer = build_plugin_layer( + plugin, + in_channels=in_channels, + postfix=plugin.pop('postfix', '')) + assert not hasattr(self, name), f'duplicate plugin {name}' + self.add_module(name, layer) + plugin_names.append(name) + return plugin_names + + def forward_plugin(self, x, plugin_names): + """Forward function for plugins.""" + out = x + for name in plugin_names: + out = getattr(self, name)(x) + return out + + @property + def norm1(self): + """nn.Module: normalization layer after the first convolution layer""" + return getattr(self, self.norm1_name) + + @property + def norm2(self): + """nn.Module: normalization layer after the second convolution layer""" + return getattr(self, self.norm2_name) + + @property + def norm3(self): + """nn.Module: normalization layer after the third convolution layer""" + return getattr(self, self.norm3_name) + + def forward(self, x): + """Forward function.""" + + def _inner_forward(x): + identity = x + + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv1_plugin_names) + + out = self.conv2(out) + out = self.norm2(out) + out = self.relu(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv2_plugin_names) + + out = self.conv3(out) + out = self.norm3(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv3_plugin_names) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + out = self.relu(out) + + return out + + +@BACKBONES.register_module() +class ResNet(nn.Module): + """ResNet backbone. + + Args: + depth (int): Depth of resnet, from {18, 34, 50, 101, 152}. + in_channels (int): Number of input image channels. Default" 3. + stem_channels (int): Number of stem channels. Default: 64. + base_channels (int): Number of base channels of res layer. Default: 64. + num_stages (int): Resnet stages, normally 4. + strides (Sequence[int]): Strides of the first block of each stage. + dilations (Sequence[int]): Dilation of each stage. + out_indices (Sequence[int]): Output from which stages. + style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two + layer is the 3x3 conv layer, otherwise the stride-two layer is + the first 1x1 conv layer. + deep_stem (bool): Replace 7x7 conv in input stem with 3 3x3 conv + avg_down (bool): Use AvgPool instead of stride conv when + downsampling in the bottleneck. + frozen_stages (int): Stages to be frozen (stop grad and set eval mode). + -1 means not freezing any parameters. + norm_cfg (dict): Dictionary to construct and config norm layer. + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. + plugins (list[dict]): List of plugins for stages, each dict contains: + + - cfg (dict, required): Cfg dict to build plugin. + + - position (str, required): Position inside block to insert plugin, + options: 'after_conv1', 'after_conv2', 'after_conv3'. + + - stages (tuple[bool], optional): Stages to apply plugin, length + should be same as 'num_stages' + multi_grid (Sequence[int]|None): Multi grid dilation rates of last + stage. Default: None + contract_dilation (bool): Whether contract first dilation of each layer + Default: False + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. + zero_init_residual (bool): Whether to use zero init for last norm layer + in resblocks to let them behave as identity. + + Example: + >>> from annotator.mmpkg.mmseg.models import ResNet + >>> import torch + >>> self = ResNet(depth=18) + >>> self.eval() + >>> inputs = torch.rand(1, 3, 32, 32) + >>> level_outputs = self.forward(inputs) + >>> for level_out in level_outputs: + ... print(tuple(level_out.shape)) + (1, 64, 8, 8) + (1, 128, 4, 4) + (1, 256, 2, 2) + (1, 512, 1, 1) + """ + + arch_settings = { + 18: (BasicBlock, (2, 2, 2, 2)), + 34: (BasicBlock, (3, 4, 6, 3)), + 50: (Bottleneck, (3, 4, 6, 3)), + 101: (Bottleneck, (3, 4, 23, 3)), + 152: (Bottleneck, (3, 8, 36, 3)) + } + + def __init__(self, + depth, + in_channels=3, + stem_channels=64, + base_channels=64, + num_stages=4, + strides=(1, 2, 2, 2), + dilations=(1, 1, 1, 1), + out_indices=(0, 1, 2, 3), + style='pytorch', + deep_stem=False, + avg_down=False, + frozen_stages=-1, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + norm_eval=False, + dcn=None, + stage_with_dcn=(False, False, False, False), + plugins=None, + multi_grid=None, + contract_dilation=False, + with_cp=False, + zero_init_residual=True): + super(ResNet, self).__init__() + if depth not in self.arch_settings: + raise KeyError(f'invalid depth {depth} for resnet') + self.depth = depth + self.stem_channels = stem_channels + self.base_channels = base_channels + self.num_stages = num_stages + assert num_stages >= 1 and num_stages <= 4 + self.strides = strides + self.dilations = dilations + assert len(strides) == len(dilations) == num_stages + self.out_indices = out_indices + assert max(out_indices) < num_stages + self.style = style + self.deep_stem = deep_stem + self.avg_down = avg_down + self.frozen_stages = frozen_stages + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.with_cp = with_cp + self.norm_eval = norm_eval + self.dcn = dcn + self.stage_with_dcn = stage_with_dcn + if dcn is not None: + assert len(stage_with_dcn) == num_stages + self.plugins = plugins + self.multi_grid = multi_grid + self.contract_dilation = contract_dilation + self.zero_init_residual = zero_init_residual + self.block, stage_blocks = self.arch_settings[depth] + self.stage_blocks = stage_blocks[:num_stages] + self.inplanes = stem_channels + + self._make_stem_layer(in_channels, stem_channels) + + self.res_layers = [] + for i, num_blocks in enumerate(self.stage_blocks): + stride = strides[i] + dilation = dilations[i] + dcn = self.dcn if self.stage_with_dcn[i] else None + if plugins is not None: + stage_plugins = self.make_stage_plugins(plugins, i) + else: + stage_plugins = None + # multi grid is applied to last layer only + stage_multi_grid = multi_grid if i == len( + self.stage_blocks) - 1 else None + planes = base_channels * 2**i + res_layer = self.make_res_layer( + block=self.block, + inplanes=self.inplanes, + planes=planes, + num_blocks=num_blocks, + stride=stride, + dilation=dilation, + style=self.style, + avg_down=self.avg_down, + with_cp=with_cp, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + dcn=dcn, + plugins=stage_plugins, + multi_grid=stage_multi_grid, + contract_dilation=contract_dilation) + self.inplanes = planes * self.block.expansion + layer_name = f'layer{i+1}' + self.add_module(layer_name, res_layer) + self.res_layers.append(layer_name) + + self._freeze_stages() + + self.feat_dim = self.block.expansion * base_channels * 2**( + len(self.stage_blocks) - 1) + + def make_stage_plugins(self, plugins, stage_idx): + """make plugins for ResNet 'stage_idx'th stage . + + Currently we support to insert 'context_block', + 'empirical_attention_block', 'nonlocal_block' into the backbone like + ResNet/ResNeXt. They could be inserted after conv1/conv2/conv3 of + Bottleneck. + + An example of plugins format could be : + >>> plugins=[ + ... dict(cfg=dict(type='xxx', arg1='xxx'), + ... stages=(False, True, True, True), + ... position='after_conv2'), + ... dict(cfg=dict(type='yyy'), + ... stages=(True, True, True, True), + ... position='after_conv3'), + ... dict(cfg=dict(type='zzz', postfix='1'), + ... stages=(True, True, True, True), + ... position='after_conv3'), + ... dict(cfg=dict(type='zzz', postfix='2'), + ... stages=(True, True, True, True), + ... position='after_conv3') + ... ] + >>> self = ResNet(depth=18) + >>> stage_plugins = self.make_stage_plugins(plugins, 0) + >>> assert len(stage_plugins) == 3 + + Suppose 'stage_idx=0', the structure of blocks in the stage would be: + conv1-> conv2->conv3->yyy->zzz1->zzz2 + Suppose 'stage_idx=1', the structure of blocks in the stage would be: + conv1-> conv2->xxx->conv3->yyy->zzz1->zzz2 + + If stages is missing, the plugin would be applied to all stages. + + Args: + plugins (list[dict]): List of plugins cfg to build. The postfix is + required if multiple same type plugins are inserted. + stage_idx (int): Index of stage to build + + Returns: + list[dict]: Plugins for current stage + """ + stage_plugins = [] + for plugin in plugins: + plugin = plugin.copy() + stages = plugin.pop('stages', None) + assert stages is None or len(stages) == self.num_stages + # whether to insert plugin into current stage + if stages is None or stages[stage_idx]: + stage_plugins.append(plugin) + + return stage_plugins + + def make_res_layer(self, **kwargs): + """Pack all blocks in a stage into a ``ResLayer``.""" + return ResLayer(**kwargs) + + @property + def norm1(self): + """nn.Module: the normalization layer named "norm1" """ + return getattr(self, self.norm1_name) + + def _make_stem_layer(self, in_channels, stem_channels): + """Make stem layer for ResNet.""" + if self.deep_stem: + self.stem = nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels, + stem_channels // 2, + kernel_size=3, + stride=2, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, stem_channels // 2)[1], + nn.ReLU(inplace=True), + build_conv_layer( + self.conv_cfg, + stem_channels // 2, + stem_channels // 2, + kernel_size=3, + stride=1, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, stem_channels // 2)[1], + nn.ReLU(inplace=True), + build_conv_layer( + self.conv_cfg, + stem_channels // 2, + stem_channels, + kernel_size=3, + stride=1, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, stem_channels)[1], + nn.ReLU(inplace=True)) + else: + self.conv1 = build_conv_layer( + self.conv_cfg, + in_channels, + stem_channels, + kernel_size=7, + stride=2, + padding=3, + bias=False) + self.norm1_name, norm1 = build_norm_layer( + self.norm_cfg, stem_channels, postfix=1) + self.add_module(self.norm1_name, norm1) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + def _freeze_stages(self): + """Freeze stages param and norm stats.""" + if self.frozen_stages >= 0: + if self.deep_stem: + self.stem.eval() + for param in self.stem.parameters(): + param.requires_grad = False + else: + self.norm1.eval() + for m in [self.conv1, self.norm1]: + for param in m.parameters(): + param.requires_grad = False + + for i in range(1, self.frozen_stages + 1): + m = getattr(self, f'layer{i}') + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + + if self.dcn is not None: + for m in self.modules(): + if isinstance(m, Bottleneck) and hasattr( + m, 'conv2_offset'): + constant_init(m.conv2_offset, 0) + + if self.zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + constant_init(m.norm3, 0) + elif isinstance(m, BasicBlock): + constant_init(m.norm2, 0) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + """Forward function.""" + if self.deep_stem: + x = self.stem(x) + else: + x = self.conv1(x) + x = self.norm1(x) + x = self.relu(x) + x = self.maxpool(x) + outs = [] + for i, layer_name in enumerate(self.res_layers): + res_layer = getattr(self, layer_name) + x = res_layer(x) + if i in self.out_indices: + outs.append(x) + return tuple(outs) + + def train(self, mode=True): + """Convert the model into training mode while keep normalization layer + freezed.""" + super(ResNet, self).train(mode) + self._freeze_stages() + if mode and self.norm_eval: + for m in self.modules(): + # trick: eval have effect on BatchNorm only + if isinstance(m, _BatchNorm): + m.eval() + + +@BACKBONES.register_module() +class ResNetV1c(ResNet): + """ResNetV1c variant described in [1]_. + + Compared with default ResNet(ResNetV1b), ResNetV1c replaces the 7x7 conv + in the input stem with three 3x3 convs. + + References: + .. [1] https://arxiv.org/pdf/1812.01187.pdf + """ + + def __init__(self, **kwargs): + super(ResNetV1c, self).__init__( + deep_stem=True, avg_down=False, **kwargs) + + +@BACKBONES.register_module() +class ResNetV1d(ResNet): + """ResNetV1d variant described in [1]_. + + Compared with default ResNet(ResNetV1b), ResNetV1d replaces the 7x7 conv in + the input stem with three 3x3 convs. And in the downsampling block, a 2x2 + avg_pool with stride 2 is added before conv, whose stride is changed to 1. + """ + + def __init__(self, **kwargs): + super(ResNetV1d, self).__init__( + deep_stem=True, avg_down=True, **kwargs) diff --git a/annotator/mmpkg/mmseg/models/backbones/resnext.py b/annotator/mmpkg/mmseg/models/backbones/resnext.py new file mode 100644 index 0000000000000000000000000000000000000000..be0194da1714e8431309a9dd8a42afebdbc1baf5 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/backbones/resnext.py @@ -0,0 +1,145 @@ +import math + +from annotator.mmpkg.mmcv.cnn import build_conv_layer, build_norm_layer + +from ..builder import BACKBONES +from ..utils import ResLayer +from .resnet import Bottleneck as _Bottleneck +from .resnet import ResNet + + +class Bottleneck(_Bottleneck): + """Bottleneck block for ResNeXt. + + If style is "pytorch", the stride-two layer is the 3x3 conv layer, if it is + "caffe", the stride-two layer is the first 1x1 conv layer. + """ + + def __init__(self, + inplanes, + planes, + groups=1, + base_width=4, + base_channels=64, + **kwargs): + super(Bottleneck, self).__init__(inplanes, planes, **kwargs) + + if groups == 1: + width = self.planes + else: + width = math.floor(self.planes * + (base_width / base_channels)) * groups + + self.norm1_name, norm1 = build_norm_layer( + self.norm_cfg, width, postfix=1) + self.norm2_name, norm2 = build_norm_layer( + self.norm_cfg, width, postfix=2) + self.norm3_name, norm3 = build_norm_layer( + self.norm_cfg, self.planes * self.expansion, postfix=3) + + self.conv1 = build_conv_layer( + self.conv_cfg, + self.inplanes, + width, + kernel_size=1, + stride=self.conv1_stride, + bias=False) + self.add_module(self.norm1_name, norm1) + fallback_on_stride = False + self.with_modulated_dcn = False + if self.with_dcn: + fallback_on_stride = self.dcn.pop('fallback_on_stride', False) + if not self.with_dcn or fallback_on_stride: + self.conv2 = build_conv_layer( + self.conv_cfg, + width, + width, + kernel_size=3, + stride=self.conv2_stride, + padding=self.dilation, + dilation=self.dilation, + groups=groups, + bias=False) + else: + assert self.conv_cfg is None, 'conv_cfg must be None for DCN' + self.conv2 = build_conv_layer( + self.dcn, + width, + width, + kernel_size=3, + stride=self.conv2_stride, + padding=self.dilation, + dilation=self.dilation, + groups=groups, + bias=False) + + self.add_module(self.norm2_name, norm2) + self.conv3 = build_conv_layer( + self.conv_cfg, + width, + self.planes * self.expansion, + kernel_size=1, + bias=False) + self.add_module(self.norm3_name, norm3) + + +@BACKBONES.register_module() +class ResNeXt(ResNet): + """ResNeXt backbone. + + Args: + depth (int): Depth of resnet, from {18, 34, 50, 101, 152}. + in_channels (int): Number of input image channels. Normally 3. + num_stages (int): Resnet stages, normally 4. + groups (int): Group of resnext. + base_width (int): Base width of resnext. + strides (Sequence[int]): Strides of the first block of each stage. + dilations (Sequence[int]): Dilation of each stage. + out_indices (Sequence[int]): Output from which stages. + style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two + layer is the 3x3 conv layer, otherwise the stride-two layer is + the first 1x1 conv layer. + frozen_stages (int): Stages to be frozen (all param fixed). -1 means + not freezing any parameters. + norm_cfg (dict): dictionary to construct and config norm layer. + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. + zero_init_residual (bool): whether to use zero init for last norm layer + in resblocks to let them behave as identity. + + Example: + >>> from annotator.mmpkg.mmseg.models import ResNeXt + >>> import torch + >>> self = ResNeXt(depth=50) + >>> self.eval() + >>> inputs = torch.rand(1, 3, 32, 32) + >>> level_outputs = self.forward(inputs) + >>> for level_out in level_outputs: + ... print(tuple(level_out.shape)) + (1, 256, 8, 8) + (1, 512, 4, 4) + (1, 1024, 2, 2) + (1, 2048, 1, 1) + """ + + arch_settings = { + 50: (Bottleneck, (3, 4, 6, 3)), + 101: (Bottleneck, (3, 4, 23, 3)), + 152: (Bottleneck, (3, 8, 36, 3)) + } + + def __init__(self, groups=1, base_width=4, **kwargs): + self.groups = groups + self.base_width = base_width + super(ResNeXt, self).__init__(**kwargs) + + def make_res_layer(self, **kwargs): + """Pack all blocks in a stage into a ``ResLayer``""" + return ResLayer( + groups=self.groups, + base_width=self.base_width, + base_channels=self.base_channels, + **kwargs) diff --git a/annotator/mmpkg/mmseg/models/backbones/unet.py b/annotator/mmpkg/mmseg/models/backbones/unet.py new file mode 100644 index 0000000000000000000000000000000000000000..3d19902ba273af02f8c9ce60f6632634633c1101 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/backbones/unet.py @@ -0,0 +1,429 @@ +import torch.nn as nn +import torch.utils.checkpoint as cp +from annotator.mmpkg.mmcv.cnn import (UPSAMPLE_LAYERS, ConvModule, build_activation_layer, + build_norm_layer, constant_init, kaiming_init) +from annotator.mmpkg.mmcv.runner import load_checkpoint +from annotator.mmpkg.mmcv.utils.parrots_wrapper import _BatchNorm + +from annotator.mmpkg.mmseg.utils import get_root_logger +from ..builder import BACKBONES +from ..utils import UpConvBlock + + +class BasicConvBlock(nn.Module): + """Basic convolutional block for UNet. + + This module consists of several plain convolutional layers. + + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + num_convs (int): Number of convolutional layers. Default: 2. + stride (int): Whether use stride convolution to downsample + the input feature map. If stride=2, it only uses stride convolution + in the first convolutional layer to downsample the input feature + map. Options are 1 or 2. Default: 1. + dilation (int): Whether use dilated convolution to expand the + receptive field. Set dilation rate of each convolutional layer and + the dilation rate of the first convolutional layer is always 1. + Default: 1. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + conv_cfg (dict | None): Config dict for convolution layer. + Default: None. + norm_cfg (dict | None): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict | None): Config dict for activation layer in ConvModule. + Default: dict(type='ReLU'). + dcn (bool): Use deformable convolution in convolutional layer or not. + Default: None. + plugins (dict): plugins for convolutional layers. Default: None. + """ + + def __init__(self, + in_channels, + out_channels, + num_convs=2, + stride=1, + dilation=1, + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + dcn=None, + plugins=None): + super(BasicConvBlock, self).__init__() + assert dcn is None, 'Not implemented yet.' + assert plugins is None, 'Not implemented yet.' + + self.with_cp = with_cp + convs = [] + for i in range(num_convs): + convs.append( + ConvModule( + in_channels=in_channels if i == 0 else out_channels, + out_channels=out_channels, + kernel_size=3, + stride=stride if i == 0 else 1, + dilation=1 if i == 0 else dilation, + padding=1 if i == 0 else dilation, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + + self.convs = nn.Sequential(*convs) + + def forward(self, x): + """Forward function.""" + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(self.convs, x) + else: + out = self.convs(x) + return out + + +@UPSAMPLE_LAYERS.register_module() +class DeconvModule(nn.Module): + """Deconvolution upsample module in decoder for UNet (2X upsample). + + This module uses deconvolution to upsample feature map in the decoder + of UNet. + + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + norm_cfg (dict | None): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict | None): Config dict for activation layer in ConvModule. + Default: dict(type='ReLU'). + kernel_size (int): Kernel size of the convolutional layer. Default: 4. + """ + + def __init__(self, + in_channels, + out_channels, + with_cp=False, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + *, + kernel_size=4, + scale_factor=2): + super(DeconvModule, self).__init__() + + assert (kernel_size - scale_factor >= 0) and\ + (kernel_size - scale_factor) % 2 == 0,\ + f'kernel_size should be greater than or equal to scale_factor '\ + f'and (kernel_size - scale_factor) should be even numbers, '\ + f'while the kernel size is {kernel_size} and scale_factor is '\ + f'{scale_factor}.' + + stride = scale_factor + padding = (kernel_size - scale_factor) // 2 + self.with_cp = with_cp + deconv = nn.ConvTranspose2d( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding) + + norm_name, norm = build_norm_layer(norm_cfg, out_channels) + activate = build_activation_layer(act_cfg) + self.deconv_upsamping = nn.Sequential(deconv, norm, activate) + + def forward(self, x): + """Forward function.""" + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(self.deconv_upsamping, x) + else: + out = self.deconv_upsamping(x) + return out + + +@UPSAMPLE_LAYERS.register_module() +class InterpConv(nn.Module): + """Interpolation upsample module in decoder for UNet. + + This module uses interpolation to upsample feature map in the decoder + of UNet. It consists of one interpolation upsample layer and one + convolutional layer. It can be one interpolation upsample layer followed + by one convolutional layer (conv_first=False) or one convolutional layer + followed by one interpolation upsample layer (conv_first=True). + + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + norm_cfg (dict | None): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict | None): Config dict for activation layer in ConvModule. + Default: dict(type='ReLU'). + conv_cfg (dict | None): Config dict for convolution layer. + Default: None. + conv_first (bool): Whether convolutional layer or interpolation + upsample layer first. Default: False. It means interpolation + upsample layer followed by one convolutional layer. + kernel_size (int): Kernel size of the convolutional layer. Default: 1. + stride (int): Stride of the convolutional layer. Default: 1. + padding (int): Padding of the convolutional layer. Default: 1. + upsample_cfg (dict): Interpolation config of the upsample layer. + Default: dict( + scale_factor=2, mode='bilinear', align_corners=False). + """ + + def __init__(self, + in_channels, + out_channels, + with_cp=False, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + *, + conv_cfg=None, + conv_first=False, + kernel_size=1, + stride=1, + padding=0, + upsample_cfg=dict( + scale_factor=2, mode='bilinear', align_corners=False)): + super(InterpConv, self).__init__() + + self.with_cp = with_cp + conv = ConvModule( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + upsample = nn.Upsample(**upsample_cfg) + if conv_first: + self.interp_upsample = nn.Sequential(conv, upsample) + else: + self.interp_upsample = nn.Sequential(upsample, conv) + + def forward(self, x): + """Forward function.""" + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(self.interp_upsample, x) + else: + out = self.interp_upsample(x) + return out + + +@BACKBONES.register_module() +class UNet(nn.Module): + """UNet backbone. + U-Net: Convolutional Networks for Biomedical Image Segmentation. + https://arxiv.org/pdf/1505.04597.pdf + + Args: + in_channels (int): Number of input image channels. Default" 3. + base_channels (int): Number of base channels of each stage. + The output channels of the first stage. Default: 64. + num_stages (int): Number of stages in encoder, normally 5. Default: 5. + strides (Sequence[int 1 | 2]): Strides of each stage in encoder. + len(strides) is equal to num_stages. Normally the stride of the + first stage in encoder is 1. If strides[i]=2, it uses stride + convolution to downsample in the correspondence encoder stage. + Default: (1, 1, 1, 1, 1). + enc_num_convs (Sequence[int]): Number of convolutional layers in the + convolution block of the correspondence encoder stage. + Default: (2, 2, 2, 2, 2). + dec_num_convs (Sequence[int]): Number of convolutional layers in the + convolution block of the correspondence decoder stage. + Default: (2, 2, 2, 2). + downsamples (Sequence[int]): Whether use MaxPool to downsample the + feature map after the first stage of encoder + (stages: [1, num_stages)). If the correspondence encoder stage use + stride convolution (strides[i]=2), it will never use MaxPool to + downsample, even downsamples[i-1]=True. + Default: (True, True, True, True). + enc_dilations (Sequence[int]): Dilation rate of each stage in encoder. + Default: (1, 1, 1, 1, 1). + dec_dilations (Sequence[int]): Dilation rate of each stage in decoder. + Default: (1, 1, 1, 1). + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + conv_cfg (dict | None): Config dict for convolution layer. + Default: None. + norm_cfg (dict | None): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict | None): Config dict for activation layer in ConvModule. + Default: dict(type='ReLU'). + upsample_cfg (dict): The upsample config of the upsample module in + decoder. Default: dict(type='InterpConv'). + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. Default: False. + dcn (bool): Use deformable convolution in convolutional layer or not. + Default: None. + plugins (dict): plugins for convolutional layers. Default: None. + + Notice: + The input image size should be divisible by the whole downsample rate + of the encoder. More detail of the whole downsample rate can be found + in UNet._check_input_divisible. + + """ + + def __init__(self, + in_channels=3, + base_channels=64, + num_stages=5, + strides=(1, 1, 1, 1, 1), + enc_num_convs=(2, 2, 2, 2, 2), + dec_num_convs=(2, 2, 2, 2), + downsamples=(True, True, True, True), + enc_dilations=(1, 1, 1, 1, 1), + dec_dilations=(1, 1, 1, 1), + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + upsample_cfg=dict(type='InterpConv'), + norm_eval=False, + dcn=None, + plugins=None): + super(UNet, self).__init__() + assert dcn is None, 'Not implemented yet.' + assert plugins is None, 'Not implemented yet.' + assert len(strides) == num_stages, \ + 'The length of strides should be equal to num_stages, '\ + f'while the strides is {strides}, the length of '\ + f'strides is {len(strides)}, and the num_stages is '\ + f'{num_stages}.' + assert len(enc_num_convs) == num_stages, \ + 'The length of enc_num_convs should be equal to num_stages, '\ + f'while the enc_num_convs is {enc_num_convs}, the length of '\ + f'enc_num_convs is {len(enc_num_convs)}, and the num_stages is '\ + f'{num_stages}.' + assert len(dec_num_convs) == (num_stages-1), \ + 'The length of dec_num_convs should be equal to (num_stages-1), '\ + f'while the dec_num_convs is {dec_num_convs}, the length of '\ + f'dec_num_convs is {len(dec_num_convs)}, and the num_stages is '\ + f'{num_stages}.' + assert len(downsamples) == (num_stages-1), \ + 'The length of downsamples should be equal to (num_stages-1), '\ + f'while the downsamples is {downsamples}, the length of '\ + f'downsamples is {len(downsamples)}, and the num_stages is '\ + f'{num_stages}.' + assert len(enc_dilations) == num_stages, \ + 'The length of enc_dilations should be equal to num_stages, '\ + f'while the enc_dilations is {enc_dilations}, the length of '\ + f'enc_dilations is {len(enc_dilations)}, and the num_stages is '\ + f'{num_stages}.' + assert len(dec_dilations) == (num_stages-1), \ + 'The length of dec_dilations should be equal to (num_stages-1), '\ + f'while the dec_dilations is {dec_dilations}, the length of '\ + f'dec_dilations is {len(dec_dilations)}, and the num_stages is '\ + f'{num_stages}.' + self.num_stages = num_stages + self.strides = strides + self.downsamples = downsamples + self.norm_eval = norm_eval + self.base_channels = base_channels + + self.encoder = nn.ModuleList() + self.decoder = nn.ModuleList() + + for i in range(num_stages): + enc_conv_block = [] + if i != 0: + if strides[i] == 1 and downsamples[i - 1]: + enc_conv_block.append(nn.MaxPool2d(kernel_size=2)) + upsample = (strides[i] != 1 or downsamples[i - 1]) + self.decoder.append( + UpConvBlock( + conv_block=BasicConvBlock, + in_channels=base_channels * 2**i, + skip_channels=base_channels * 2**(i - 1), + out_channels=base_channels * 2**(i - 1), + num_convs=dec_num_convs[i - 1], + stride=1, + dilation=dec_dilations[i - 1], + with_cp=with_cp, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + upsample_cfg=upsample_cfg if upsample else None, + dcn=None, + plugins=None)) + + enc_conv_block.append( + BasicConvBlock( + in_channels=in_channels, + out_channels=base_channels * 2**i, + num_convs=enc_num_convs[i], + stride=strides[i], + dilation=enc_dilations[i], + with_cp=with_cp, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + dcn=None, + plugins=None)) + self.encoder.append((nn.Sequential(*enc_conv_block))) + in_channels = base_channels * 2**i + + def forward(self, x): + self._check_input_divisible(x) + enc_outs = [] + for enc in self.encoder: + x = enc(x) + enc_outs.append(x) + dec_outs = [x] + for i in reversed(range(len(self.decoder))): + x = self.decoder[i](enc_outs[i], x) + dec_outs.append(x) + + return dec_outs + + def train(self, mode=True): + """Convert the model into training mode while keep normalization layer + freezed.""" + super(UNet, self).train(mode) + if mode and self.norm_eval: + for m in self.modules(): + # trick: eval have effect on BatchNorm only + if isinstance(m, _BatchNorm): + m.eval() + + def _check_input_divisible(self, x): + h, w = x.shape[-2:] + whole_downsample_rate = 1 + for i in range(1, self.num_stages): + if self.strides[i] == 2 or self.downsamples[i - 1]: + whole_downsample_rate *= 2 + assert (h % whole_downsample_rate == 0) \ + and (w % whole_downsample_rate == 0),\ + f'The input image size {(h, w)} should be divisible by the whole '\ + f'downsample rate {whole_downsample_rate}, when num_stages is '\ + f'{self.num_stages}, strides is {self.strides}, and downsamples '\ + f'is {self.downsamples}.' + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + else: + raise TypeError('pretrained must be a str or None') diff --git a/annotator/mmpkg/mmseg/models/backbones/vit.py b/annotator/mmpkg/mmseg/models/backbones/vit.py new file mode 100644 index 0000000000000000000000000000000000000000..ab1a393741b21c8185f4204946b751b1913ef98c --- /dev/null +++ b/annotator/mmpkg/mmseg/models/backbones/vit.py @@ -0,0 +1,459 @@ +"""Modified from https://github.com/rwightman/pytorch-image- +models/blob/master/timm/models/vision_transformer.py.""" + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as cp +from annotator.mmpkg.mmcv.cnn import (Conv2d, Linear, build_activation_layer, build_norm_layer, + constant_init, kaiming_init, normal_init) +from annotator.mmpkg.mmcv.runner import _load_checkpoint +from annotator.mmpkg.mmcv.utils.parrots_wrapper import _BatchNorm + +from annotator.mmpkg.mmseg.utils import get_root_logger +from ..builder import BACKBONES +from ..utils import DropPath, trunc_normal_ + + +class Mlp(nn.Module): + """MLP layer for Encoder block. + + Args: + in_features(int): Input dimension for the first fully + connected layer. + hidden_features(int): Output dimension for the first fully + connected layer. + out_features(int): Output dementsion for the second fully + connected layer. + act_cfg(dict): Config dict for activation layer. + Default: dict(type='GELU'). + drop(float): Drop rate for the dropout layer. Dropout rate has + to be between 0 and 1. Default: 0. + """ + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_cfg=dict(type='GELU'), + drop=0.): + super(Mlp, self).__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = Linear(in_features, hidden_features) + self.act = build_activation_layer(act_cfg) + self.fc2 = Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class Attention(nn.Module): + """Attention layer for Encoder block. + + Args: + dim (int): Dimension for the input vector. + num_heads (int): Number of parallel attention heads. + qkv_bias (bool): Enable bias for qkv if True. Default: False. + qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. + attn_drop (float): Drop rate for attention output weights. + Default: 0. + proj_drop (float): Drop rate for output weights. Default: 0. + """ + + def __init__(self, + dim, + num_heads=8, + qkv_bias=False, + qk_scale=None, + attn_drop=0., + proj_drop=0.): + super(Attention, self).__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x): + b, n, c = x.shape + qkv = self.qkv(x).reshape(b, n, 3, self.num_heads, + c // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(b, n, c) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class Block(nn.Module): + """Implements encoder block with residual connection. + + Args: + dim (int): The feature dimension. + num_heads (int): Number of parallel attention heads. + mlp_ratio (int): Ratio of mlp hidden dim to embedding dim. + qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. + drop (float): Drop rate for mlp output weights. Default: 0. + attn_drop (float): Drop rate for attention output weights. + Default: 0. + proj_drop (float): Drop rate for attn layer output weights. + Default: 0. + drop_path (float): Drop rate for paths of model. + Default: 0. + act_cfg (dict): Config dict for activation layer. + Default: dict(type='GELU'). + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='LN', requires_grad=True). + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + """ + + def __init__(self, + dim, + num_heads, + mlp_ratio=4, + qkv_bias=False, + qk_scale=None, + drop=0., + attn_drop=0., + proj_drop=0., + drop_path=0., + act_cfg=dict(type='GELU'), + norm_cfg=dict(type='LN', eps=1e-6), + with_cp=False): + super(Block, self).__init__() + self.with_cp = with_cp + _, self.norm1 = build_norm_layer(norm_cfg, dim) + self.attn = Attention(dim, num_heads, qkv_bias, qk_scale, attn_drop, + proj_drop) + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + _, self.norm2 = build_norm_layer(norm_cfg, dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_cfg=act_cfg, + drop=drop) + + def forward(self, x): + + def _inner_forward(x): + out = x + self.drop_path(self.attn(self.norm1(x))) + out = out + self.drop_path(self.mlp(self.norm2(out))) + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + return out + + +class PatchEmbed(nn.Module): + """Image to Patch Embedding. + + Args: + img_size (int | tuple): Input image size. + default: 224. + patch_size (int): Width and height for a patch. + default: 16. + in_channels (int): Input channels for images. Default: 3. + embed_dim (int): The embedding dimension. Default: 768. + """ + + def __init__(self, + img_size=224, + patch_size=16, + in_channels=3, + embed_dim=768): + super(PatchEmbed, self).__init__() + if isinstance(img_size, int): + self.img_size = (img_size, img_size) + elif isinstance(img_size, tuple): + self.img_size = img_size + else: + raise TypeError('img_size must be type of int or tuple') + h, w = self.img_size + self.patch_size = (patch_size, patch_size) + self.num_patches = (h // patch_size) * (w // patch_size) + self.proj = Conv2d( + in_channels, embed_dim, kernel_size=patch_size, stride=patch_size) + + def forward(self, x): + return self.proj(x).flatten(2).transpose(1, 2) + + +@BACKBONES.register_module() +class VisionTransformer(nn.Module): + """Vision transformer backbone. + + A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for + Image Recognition at Scale` - https://arxiv.org/abs/2010.11929 + + Args: + img_size (tuple): input image size. Default: (224, 224). + patch_size (int, tuple): patch size. Default: 16. + in_channels (int): number of input channels. Default: 3. + embed_dim (int): embedding dimension. Default: 768. + depth (int): depth of transformer. Default: 12. + num_heads (int): number of attention heads. Default: 12. + mlp_ratio (int): ratio of mlp hidden dim to embedding dim. + Default: 4. + out_indices (list | tuple | int): Output from which stages. + Default: -1. + qkv_bias (bool): enable bias for qkv if True. Default: True. + qk_scale (float): override default qk scale of head_dim ** -0.5 if set. + drop_rate (float): dropout rate. Default: 0. + attn_drop_rate (float): attention dropout rate. Default: 0. + drop_path_rate (float): Rate of DropPath. Default: 0. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='LN', eps=1e-6, requires_grad=True). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='GELU'). + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. Default: False. + final_norm (bool): Whether to add a additional layer to normalize + final feature map. Default: False. + interpolate_mode (str): Select the interpolate mode for position + embeding vector resize. Default: bicubic. + with_cls_token (bool): If concatenating class token into image tokens + as transformer input. Default: True. + with_cp (bool): Use checkpoint or not. Using checkpoint + will save some memory while slowing down the training speed. + Default: False. + """ + + def __init__(self, + img_size=(224, 224), + patch_size=16, + in_channels=3, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4, + out_indices=11, + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0., + norm_cfg=dict(type='LN', eps=1e-6, requires_grad=True), + act_cfg=dict(type='GELU'), + norm_eval=False, + final_norm=False, + with_cls_token=True, + interpolate_mode='bicubic', + with_cp=False): + super(VisionTransformer, self).__init__() + self.img_size = img_size + self.patch_size = patch_size + self.features = self.embed_dim = embed_dim + self.patch_embed = PatchEmbed( + img_size=img_size, + patch_size=patch_size, + in_channels=in_channels, + embed_dim=embed_dim) + + self.with_cls_token = with_cls_token + self.cls_token = nn.Parameter(torch.zeros(1, 1, self.embed_dim)) + self.pos_embed = nn.Parameter( + torch.zeros(1, self.patch_embed.num_patches + 1, embed_dim)) + self.pos_drop = nn.Dropout(p=drop_rate) + + if isinstance(out_indices, int): + self.out_indices = [out_indices] + elif isinstance(out_indices, list) or isinstance(out_indices, tuple): + self.out_indices = out_indices + else: + raise TypeError('out_indices must be type of int, list or tuple') + + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth) + ] # stochastic depth decay rule + self.blocks = nn.ModuleList([ + Block( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=dpr[i], + attn_drop=attn_drop_rate, + act_cfg=act_cfg, + norm_cfg=norm_cfg, + with_cp=with_cp) for i in range(depth) + ]) + + self.interpolate_mode = interpolate_mode + self.final_norm = final_norm + if final_norm: + _, self.norm = build_norm_layer(norm_cfg, embed_dim) + + self.norm_eval = norm_eval + self.with_cp = with_cp + + def init_weights(self, pretrained=None): + if isinstance(pretrained, str): + logger = get_root_logger() + checkpoint = _load_checkpoint(pretrained, logger=logger) + if 'state_dict' in checkpoint: + state_dict = checkpoint['state_dict'] + else: + state_dict = checkpoint + + if 'pos_embed' in state_dict.keys(): + if self.pos_embed.shape != state_dict['pos_embed'].shape: + logger.info(msg=f'Resize the pos_embed shape from \ +{state_dict["pos_embed"].shape} to {self.pos_embed.shape}') + h, w = self.img_size + pos_size = int( + math.sqrt(state_dict['pos_embed'].shape[1] - 1)) + state_dict['pos_embed'] = self.resize_pos_embed( + state_dict['pos_embed'], (h, w), (pos_size, pos_size), + self.patch_size, self.interpolate_mode) + + self.load_state_dict(state_dict, False) + + elif pretrained is None: + # We only implement the 'jax_impl' initialization implemented at + # https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py#L353 # noqa: E501 + trunc_normal_(self.pos_embed, std=.02) + trunc_normal_(self.cls_token, std=.02) + for n, m in self.named_modules(): + if isinstance(m, Linear): + trunc_normal_(m.weight, std=.02) + if m.bias is not None: + if 'mlp' in n: + normal_init(m.bias, std=1e-6) + else: + constant_init(m.bias, 0) + elif isinstance(m, Conv2d): + kaiming_init(m.weight, mode='fan_in') + if m.bias is not None: + constant_init(m.bias, 0) + elif isinstance(m, (_BatchNorm, nn.GroupNorm, nn.LayerNorm)): + constant_init(m.bias, 0) + constant_init(m.weight, 1.0) + else: + raise TypeError('pretrained must be a str or None') + + def _pos_embeding(self, img, patched_img, pos_embed): + """Positiong embeding method. + + Resize the pos_embed, if the input image size doesn't match + the training size. + Args: + img (torch.Tensor): The inference image tensor, the shape + must be [B, C, H, W]. + patched_img (torch.Tensor): The patched image, it should be + shape of [B, L1, C]. + pos_embed (torch.Tensor): The pos_embed weighs, it should be + shape of [B, L2, c]. + Return: + torch.Tensor: The pos encoded image feature. + """ + assert patched_img.ndim == 3 and pos_embed.ndim == 3, \ + 'the shapes of patched_img and pos_embed must be [B, L, C]' + x_len, pos_len = patched_img.shape[1], pos_embed.shape[1] + if x_len != pos_len: + if pos_len == (self.img_size[0] // self.patch_size) * ( + self.img_size[1] // self.patch_size) + 1: + pos_h = self.img_size[0] // self.patch_size + pos_w = self.img_size[1] // self.patch_size + else: + raise ValueError( + 'Unexpected shape of pos_embed, got {}.'.format( + pos_embed.shape)) + pos_embed = self.resize_pos_embed(pos_embed, img.shape[2:], + (pos_h, pos_w), self.patch_size, + self.interpolate_mode) + return self.pos_drop(patched_img + pos_embed) + + @staticmethod + def resize_pos_embed(pos_embed, input_shpae, pos_shape, patch_size, mode): + """Resize pos_embed weights. + + Resize pos_embed using bicubic interpolate method. + Args: + pos_embed (torch.Tensor): pos_embed weights. + input_shpae (tuple): Tuple for (input_h, intput_w). + pos_shape (tuple): Tuple for (pos_h, pos_w). + patch_size (int): Patch size. + Return: + torch.Tensor: The resized pos_embed of shape [B, L_new, C] + """ + assert pos_embed.ndim == 3, 'shape of pos_embed must be [B, L, C]' + input_h, input_w = input_shpae + pos_h, pos_w = pos_shape + cls_token_weight = pos_embed[:, 0] + pos_embed_weight = pos_embed[:, (-1 * pos_h * pos_w):] + pos_embed_weight = pos_embed_weight.reshape( + 1, pos_h, pos_w, pos_embed.shape[2]).permute(0, 3, 1, 2) + pos_embed_weight = F.interpolate( + pos_embed_weight, + size=[input_h // patch_size, input_w // patch_size], + align_corners=False, + mode=mode) + cls_token_weight = cls_token_weight.unsqueeze(1) + pos_embed_weight = torch.flatten(pos_embed_weight, 2).transpose(1, 2) + pos_embed = torch.cat((cls_token_weight, pos_embed_weight), dim=1) + return pos_embed + + def forward(self, inputs): + B = inputs.shape[0] + + x = self.patch_embed(inputs) + + cls_tokens = self.cls_token.expand(B, -1, -1) + x = torch.cat((cls_tokens, x), dim=1) + x = self._pos_embeding(inputs, x, self.pos_embed) + + if not self.with_cls_token: + # Remove class token for transformer input + x = x[:, 1:] + + outs = [] + for i, blk in enumerate(self.blocks): + x = blk(x) + if i == len(self.blocks) - 1: + if self.final_norm: + x = self.norm(x) + if i in self.out_indices: + if self.with_cls_token: + # Remove class token and reshape token for decoder head + out = x[:, 1:] + else: + out = x + B, _, C = out.shape + out = out.reshape(B, inputs.shape[2] // self.patch_size, + inputs.shape[3] // self.patch_size, + C).permute(0, 3, 1, 2) + outs.append(out) + + return tuple(outs) + + def train(self, mode=True): + super(VisionTransformer, self).train(mode) + if mode and self.norm_eval: + for m in self.modules(): + if isinstance(m, nn.LayerNorm): + m.eval() diff --git a/annotator/mmpkg/mmseg/models/builder.py b/annotator/mmpkg/mmseg/models/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..fd29ff66d523b854c739b580137db6f4155fc550 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/builder.py @@ -0,0 +1,46 @@ +import warnings + +from annotator.mmpkg.mmcv.cnn import MODELS as MMCV_MODELS +from annotator.mmpkg.mmcv.utils import Registry + +MODELS = Registry('models', parent=MMCV_MODELS) + +BACKBONES = MODELS +NECKS = MODELS +HEADS = MODELS +LOSSES = MODELS +SEGMENTORS = MODELS + + +def build_backbone(cfg): + """Build backbone.""" + return BACKBONES.build(cfg) + + +def build_neck(cfg): + """Build neck.""" + return NECKS.build(cfg) + + +def build_head(cfg): + """Build head.""" + return HEADS.build(cfg) + + +def build_loss(cfg): + """Build loss.""" + return LOSSES.build(cfg) + + +def build_segmentor(cfg, train_cfg=None, test_cfg=None): + """Build segmentor.""" + if train_cfg is not None or test_cfg is not None: + warnings.warn( + 'train_cfg and test_cfg is deprecated, ' + 'please specify them in model', UserWarning) + assert cfg.get('train_cfg') is None or train_cfg is None, \ + 'train_cfg specified in both outer field and model field ' + assert cfg.get('test_cfg') is None or test_cfg is None, \ + 'test_cfg specified in both outer field and model field ' + return SEGMENTORS.build( + cfg, default_args=dict(train_cfg=train_cfg, test_cfg=test_cfg)) diff --git a/annotator/mmpkg/mmseg/models/decode_heads/__init__.py b/annotator/mmpkg/mmseg/models/decode_heads/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ac66d3cfe0ea04af45c0f3594bf135841c3812e3 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/__init__.py @@ -0,0 +1,28 @@ +from .ann_head import ANNHead +from .apc_head import APCHead +from .aspp_head import ASPPHead +from .cc_head import CCHead +from .da_head import DAHead +from .dm_head import DMHead +from .dnl_head import DNLHead +from .ema_head import EMAHead +from .enc_head import EncHead +from .fcn_head import FCNHead +from .fpn_head import FPNHead +from .gc_head import GCHead +from .lraspp_head import LRASPPHead +from .nl_head import NLHead +from .ocr_head import OCRHead +# from .point_head import PointHead +from .psa_head import PSAHead +from .psp_head import PSPHead +from .sep_aspp_head import DepthwiseSeparableASPPHead +from .sep_fcn_head import DepthwiseSeparableFCNHead +from .uper_head import UPerHead + +__all__ = [ + 'FCNHead', 'PSPHead', 'ASPPHead', 'PSAHead', 'NLHead', 'GCHead', 'CCHead', + 'UPerHead', 'DepthwiseSeparableASPPHead', 'ANNHead', 'DAHead', 'OCRHead', + 'EncHead', 'DepthwiseSeparableFCNHead', 'FPNHead', 'EMAHead', 'DNLHead', + 'APCHead', 'DMHead', 'LRASPPHead' +] diff --git a/annotator/mmpkg/mmseg/models/decode_heads/ann_head.py b/annotator/mmpkg/mmseg/models/decode_heads/ann_head.py new file mode 100644 index 0000000000000000000000000000000000000000..958c88e0ca4b9acdaf146b836462b9a101b2cdad --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/ann_head.py @@ -0,0 +1,245 @@ +import torch +import torch.nn as nn +from annotator.mmpkg.mmcv.cnn import ConvModule + +from ..builder import HEADS +from ..utils import SelfAttentionBlock as _SelfAttentionBlock +from .decode_head import BaseDecodeHead + + +class PPMConcat(nn.ModuleList): + """Pyramid Pooling Module that only concat the features of each layer. + + Args: + pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module. + """ + + def __init__(self, pool_scales=(1, 3, 6, 8)): + super(PPMConcat, self).__init__( + [nn.AdaptiveAvgPool2d(pool_scale) for pool_scale in pool_scales]) + + def forward(self, feats): + """Forward function.""" + ppm_outs = [] + for ppm in self: + ppm_out = ppm(feats) + ppm_outs.append(ppm_out.view(*feats.shape[:2], -1)) + concat_outs = torch.cat(ppm_outs, dim=2) + return concat_outs + + +class SelfAttentionBlock(_SelfAttentionBlock): + """Make a ANN used SelfAttentionBlock. + + Args: + low_in_channels (int): Input channels of lower level feature, + which is the key feature for self-attention. + high_in_channels (int): Input channels of higher level feature, + which is the query feature for self-attention. + channels (int): Output channels of key/query transform. + out_channels (int): Output channels. + share_key_query (bool): Whether share projection weight between key + and query projection. + query_scale (int): The scale of query feature map. + key_pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module of key feature. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict|None): Config of activation layers. + """ + + def __init__(self, low_in_channels, high_in_channels, channels, + out_channels, share_key_query, query_scale, key_pool_scales, + conv_cfg, norm_cfg, act_cfg): + key_psp = PPMConcat(key_pool_scales) + if query_scale > 1: + query_downsample = nn.MaxPool2d(kernel_size=query_scale) + else: + query_downsample = None + super(SelfAttentionBlock, self).__init__( + key_in_channels=low_in_channels, + query_in_channels=high_in_channels, + channels=channels, + out_channels=out_channels, + share_key_query=share_key_query, + query_downsample=query_downsample, + key_downsample=key_psp, + key_query_num_convs=1, + key_query_norm=True, + value_out_num_convs=1, + value_out_norm=False, + matmul_norm=True, + with_out=True, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + + +class AFNB(nn.Module): + """Asymmetric Fusion Non-local Block(AFNB) + + Args: + low_in_channels (int): Input channels of lower level feature, + which is the key feature for self-attention. + high_in_channels (int): Input channels of higher level feature, + which is the query feature for self-attention. + channels (int): Output channels of key/query transform. + out_channels (int): Output channels. + and query projection. + query_scales (tuple[int]): The scales of query feature map. + Default: (1,) + key_pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module of key feature. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict|None): Config of activation layers. + """ + + def __init__(self, low_in_channels, high_in_channels, channels, + out_channels, query_scales, key_pool_scales, conv_cfg, + norm_cfg, act_cfg): + super(AFNB, self).__init__() + self.stages = nn.ModuleList() + for query_scale in query_scales: + self.stages.append( + SelfAttentionBlock( + low_in_channels=low_in_channels, + high_in_channels=high_in_channels, + channels=channels, + out_channels=out_channels, + share_key_query=False, + query_scale=query_scale, + key_pool_scales=key_pool_scales, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + self.bottleneck = ConvModule( + out_channels + high_in_channels, + out_channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=None) + + def forward(self, low_feats, high_feats): + """Forward function.""" + priors = [stage(high_feats, low_feats) for stage in self.stages] + context = torch.stack(priors, dim=0).sum(dim=0) + output = self.bottleneck(torch.cat([context, high_feats], 1)) + return output + + +class APNB(nn.Module): + """Asymmetric Pyramid Non-local Block (APNB) + + Args: + in_channels (int): Input channels of key/query feature, + which is the key feature for self-attention. + channels (int): Output channels of key/query transform. + out_channels (int): Output channels. + query_scales (tuple[int]): The scales of query feature map. + Default: (1,) + key_pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module of key feature. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict|None): Config of activation layers. + """ + + def __init__(self, in_channels, channels, out_channels, query_scales, + key_pool_scales, conv_cfg, norm_cfg, act_cfg): + super(APNB, self).__init__() + self.stages = nn.ModuleList() + for query_scale in query_scales: + self.stages.append( + SelfAttentionBlock( + low_in_channels=in_channels, + high_in_channels=in_channels, + channels=channels, + out_channels=out_channels, + share_key_query=True, + query_scale=query_scale, + key_pool_scales=key_pool_scales, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + self.bottleneck = ConvModule( + 2 * in_channels, + out_channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + + def forward(self, feats): + """Forward function.""" + priors = [stage(feats, feats) for stage in self.stages] + context = torch.stack(priors, dim=0).sum(dim=0) + output = self.bottleneck(torch.cat([context, feats], 1)) + return output + + +@HEADS.register_module() +class ANNHead(BaseDecodeHead): + """Asymmetric Non-local Neural Networks for Semantic Segmentation. + + This head is the implementation of `ANNNet + `_. + + Args: + project_channels (int): Projection channels for Nonlocal. + query_scales (tuple[int]): The scales of query feature map. + Default: (1,) + key_pool_scales (tuple[int]): The pooling scales of key feature map. + Default: (1, 3, 6, 8). + """ + + def __init__(self, + project_channels, + query_scales=(1, ), + key_pool_scales=(1, 3, 6, 8), + **kwargs): + super(ANNHead, self).__init__( + input_transform='multiple_select', **kwargs) + assert len(self.in_channels) == 2 + low_in_channels, high_in_channels = self.in_channels + self.project_channels = project_channels + self.fusion = AFNB( + low_in_channels=low_in_channels, + high_in_channels=high_in_channels, + out_channels=high_in_channels, + channels=project_channels, + query_scales=query_scales, + key_pool_scales=key_pool_scales, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.bottleneck = ConvModule( + high_in_channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.context = APNB( + in_channels=self.channels, + out_channels=self.channels, + channels=project_channels, + query_scales=query_scales, + key_pool_scales=key_pool_scales, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + low_feats, high_feats = self._transform_inputs(inputs) + output = self.fusion(low_feats, high_feats) + output = self.dropout(output) + output = self.bottleneck(output) + output = self.context(output) + output = self.cls_seg(output) + + return output diff --git a/annotator/mmpkg/mmseg/models/decode_heads/apc_head.py b/annotator/mmpkg/mmseg/models/decode_heads/apc_head.py new file mode 100644 index 0000000000000000000000000000000000000000..4f363dba391c3eb6fb5a4d61c145fd4976a5717d --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/apc_head.py @@ -0,0 +1,158 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from annotator.mmpkg.mmcv.cnn import ConvModule + +from annotator.mmpkg.mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +class ACM(nn.Module): + """Adaptive Context Module used in APCNet. + + Args: + pool_scale (int): Pooling scale used in Adaptive Context + Module to extract region features. + fusion (bool): Add one conv to fuse residual feature. + in_channels (int): Input channels. + channels (int): Channels after modules, before conv_seg. + conv_cfg (dict | None): Config of conv layers. + norm_cfg (dict | None): Config of norm layers. + act_cfg (dict): Config of activation layers. + """ + + def __init__(self, pool_scale, fusion, in_channels, channels, conv_cfg, + norm_cfg, act_cfg): + super(ACM, self).__init__() + self.pool_scale = pool_scale + self.fusion = fusion + self.in_channels = in_channels + self.channels = channels + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.pooled_redu_conv = ConvModule( + self.in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + self.input_redu_conv = ConvModule( + self.in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + self.global_info = ConvModule( + self.channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + self.gla = nn.Conv2d(self.channels, self.pool_scale**2, 1, 1, 0) + + self.residual_conv = ConvModule( + self.channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + if self.fusion: + self.fusion_conv = ConvModule( + self.channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, x): + """Forward function.""" + pooled_x = F.adaptive_avg_pool2d(x, self.pool_scale) + # [batch_size, channels, h, w] + x = self.input_redu_conv(x) + # [batch_size, channels, pool_scale, pool_scale] + pooled_x = self.pooled_redu_conv(pooled_x) + batch_size = x.size(0) + # [batch_size, pool_scale * pool_scale, channels] + pooled_x = pooled_x.view(batch_size, self.channels, + -1).permute(0, 2, 1).contiguous() + # [batch_size, h * w, pool_scale * pool_scale] + affinity_matrix = self.gla(x + resize( + self.global_info(F.adaptive_avg_pool2d(x, 1)), size=x.shape[2:]) + ).permute(0, 2, 3, 1).reshape( + batch_size, -1, self.pool_scale**2) + affinity_matrix = F.sigmoid(affinity_matrix) + # [batch_size, h * w, channels] + z_out = torch.matmul(affinity_matrix, pooled_x) + # [batch_size, channels, h * w] + z_out = z_out.permute(0, 2, 1).contiguous() + # [batch_size, channels, h, w] + z_out = z_out.view(batch_size, self.channels, x.size(2), x.size(3)) + z_out = self.residual_conv(z_out) + z_out = F.relu(z_out + x) + if self.fusion: + z_out = self.fusion_conv(z_out) + + return z_out + + +@HEADS.register_module() +class APCHead(BaseDecodeHead): + """Adaptive Pyramid Context Network for Semantic Segmentation. + + This head is the implementation of + `APCNet `_. + + Args: + pool_scales (tuple[int]): Pooling scales used in Adaptive Context + Module. Default: (1, 2, 3, 6). + fusion (bool): Add one conv to fuse residual feature. + """ + + def __init__(self, pool_scales=(1, 2, 3, 6), fusion=True, **kwargs): + super(APCHead, self).__init__(**kwargs) + assert isinstance(pool_scales, (list, tuple)) + self.pool_scales = pool_scales + self.fusion = fusion + acm_modules = [] + for pool_scale in self.pool_scales: + acm_modules.append( + ACM(pool_scale, + self.fusion, + self.in_channels, + self.channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + self.acm_modules = nn.ModuleList(acm_modules) + self.bottleneck = ConvModule( + self.in_channels + len(pool_scales) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + acm_outs = [x] + for acm_module in self.acm_modules: + acm_outs.append(acm_module(x)) + acm_outs = torch.cat(acm_outs, dim=1) + output = self.bottleneck(acm_outs) + output = self.cls_seg(output) + return output diff --git a/annotator/mmpkg/mmseg/models/decode_heads/aspp_head.py b/annotator/mmpkg/mmseg/models/decode_heads/aspp_head.py new file mode 100644 index 0000000000000000000000000000000000000000..3c0aadb2b097a604d96ba1c99c05663b7884b6e0 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/aspp_head.py @@ -0,0 +1,107 @@ +import torch +import torch.nn as nn +from annotator.mmpkg.mmcv.cnn import ConvModule + +from annotator.mmpkg.mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +class ASPPModule(nn.ModuleList): + """Atrous Spatial Pyramid Pooling (ASPP) Module. + + Args: + dilations (tuple[int]): Dilation rate of each layer. + in_channels (int): Input channels. + channels (int): Channels after modules, before conv_seg. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict): Config of activation layers. + """ + + def __init__(self, dilations, in_channels, channels, conv_cfg, norm_cfg, + act_cfg): + super(ASPPModule, self).__init__() + self.dilations = dilations + self.in_channels = in_channels + self.channels = channels + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + for dilation in dilations: + self.append( + ConvModule( + self.in_channels, + self.channels, + 1 if dilation == 1 else 3, + dilation=dilation, + padding=0 if dilation == 1 else dilation, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + + def forward(self, x): + """Forward function.""" + aspp_outs = [] + for aspp_module in self: + aspp_outs.append(aspp_module(x)) + + return aspp_outs + + +@HEADS.register_module() +class ASPPHead(BaseDecodeHead): + """Rethinking Atrous Convolution for Semantic Image Segmentation. + + This head is the implementation of `DeepLabV3 + `_. + + Args: + dilations (tuple[int]): Dilation rates for ASPP module. + Default: (1, 6, 12, 18). + """ + + def __init__(self, dilations=(1, 6, 12, 18), **kwargs): + super(ASPPHead, self).__init__(**kwargs) + assert isinstance(dilations, (list, tuple)) + self.dilations = dilations + self.image_pool = nn.Sequential( + nn.AdaptiveAvgPool2d(1), + ConvModule( + self.in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + self.aspp_modules = ASPPModule( + dilations, + self.in_channels, + self.channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.bottleneck = ConvModule( + (len(dilations) + 1) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + aspp_outs = [ + resize( + self.image_pool(x), + size=x.size()[2:], + mode='bilinear', + align_corners=self.align_corners) + ] + aspp_outs.extend(self.aspp_modules(x)) + aspp_outs = torch.cat(aspp_outs, dim=1) + output = self.bottleneck(aspp_outs) + output = self.cls_seg(output) + return output diff --git a/annotator/mmpkg/mmseg/models/decode_heads/cascade_decode_head.py b/annotator/mmpkg/mmseg/models/decode_heads/cascade_decode_head.py new file mode 100644 index 0000000000000000000000000000000000000000..d02122ca0e68743b1bf7a893afae96042f23838c --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/cascade_decode_head.py @@ -0,0 +1,57 @@ +from abc import ABCMeta, abstractmethod + +from .decode_head import BaseDecodeHead + + +class BaseCascadeDecodeHead(BaseDecodeHead, metaclass=ABCMeta): + """Base class for cascade decode head used in + :class:`CascadeEncoderDecoder.""" + + def __init__(self, *args, **kwargs): + super(BaseCascadeDecodeHead, self).__init__(*args, **kwargs) + + @abstractmethod + def forward(self, inputs, prev_output): + """Placeholder of forward function.""" + pass + + def forward_train(self, inputs, prev_output, img_metas, gt_semantic_seg, + train_cfg): + """Forward function for training. + Args: + inputs (list[Tensor]): List of multi-level img features. + prev_output (Tensor): The output of previous decode head. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + gt_semantic_seg (Tensor): Semantic segmentation masks + used if the architecture supports semantic segmentation task. + train_cfg (dict): The training config. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + seg_logits = self.forward(inputs, prev_output) + losses = self.losses(seg_logits, gt_semantic_seg) + + return losses + + def forward_test(self, inputs, prev_output, img_metas, test_cfg): + """Forward function for testing. + + Args: + inputs (list[Tensor]): List of multi-level img features. + prev_output (Tensor): The output of previous decode head. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + test_cfg (dict): The testing config. + + Returns: + Tensor: Output segmentation map. + """ + return self.forward(inputs, prev_output) diff --git a/annotator/mmpkg/mmseg/models/decode_heads/cc_head.py b/annotator/mmpkg/mmseg/models/decode_heads/cc_head.py new file mode 100644 index 0000000000000000000000000000000000000000..1f4f5b052445a4071952aa04274274da7d897c2c --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/cc_head.py @@ -0,0 +1,45 @@ +import torch + +from ..builder import HEADS +from .fcn_head import FCNHead + +try: + try: + from mmcv.ops import CrissCrossAttention + except ImportError: + from annotator.mmpkg.mmcv.ops import CrissCrossAttention +except ModuleNotFoundError: + CrissCrossAttention = None + + +@HEADS.register_module() +class CCHead(FCNHead): + """CCNet: Criss-Cross Attention for Semantic Segmentation. + + This head is the implementation of `CCNet + `_. + + Args: + recurrence (int): Number of recurrence of Criss Cross Attention + module. Default: 2. + """ + + def __init__(self, recurrence=2, **kwargs): + if CrissCrossAttention is None: + raise RuntimeError('Please install mmcv-full for ' + 'CrissCrossAttention ops') + super(CCHead, self).__init__(num_convs=2, **kwargs) + self.recurrence = recurrence + self.cca = CrissCrossAttention(self.channels) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + output = self.convs[0](x) + for _ in range(self.recurrence): + output = self.cca(output) + output = self.convs[1](output) + if self.concat_input: + output = self.conv_cat(torch.cat([x, output], dim=1)) + output = self.cls_seg(output) + return output diff --git a/annotator/mmpkg/mmseg/models/decode_heads/da_head.py b/annotator/mmpkg/mmseg/models/decode_heads/da_head.py new file mode 100644 index 0000000000000000000000000000000000000000..b0b7616501c04cc0faf92accac9d3fdb6807f9e1 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/da_head.py @@ -0,0 +1,178 @@ +import torch +import torch.nn.functional as F +from annotator.mmpkg.mmcv.cnn import ConvModule, Scale +from torch import nn + +from annotator.mmpkg.mmseg.core import add_prefix +from ..builder import HEADS +from ..utils import SelfAttentionBlock as _SelfAttentionBlock +from .decode_head import BaseDecodeHead + + +class PAM(_SelfAttentionBlock): + """Position Attention Module (PAM) + + Args: + in_channels (int): Input channels of key/query feature. + channels (int): Output channels of key/query transform. + """ + + def __init__(self, in_channels, channels): + super(PAM, self).__init__( + key_in_channels=in_channels, + query_in_channels=in_channels, + channels=channels, + out_channels=in_channels, + share_key_query=False, + query_downsample=None, + key_downsample=None, + key_query_num_convs=1, + key_query_norm=False, + value_out_num_convs=1, + value_out_norm=False, + matmul_norm=False, + with_out=False, + conv_cfg=None, + norm_cfg=None, + act_cfg=None) + + self.gamma = Scale(0) + + def forward(self, x): + """Forward function.""" + out = super(PAM, self).forward(x, x) + + out = self.gamma(out) + x + return out + + +class CAM(nn.Module): + """Channel Attention Module (CAM)""" + + def __init__(self): + super(CAM, self).__init__() + self.gamma = Scale(0) + + def forward(self, x): + """Forward function.""" + batch_size, channels, height, width = x.size() + proj_query = x.view(batch_size, channels, -1) + proj_key = x.view(batch_size, channels, -1).permute(0, 2, 1) + energy = torch.bmm(proj_query, proj_key) + energy_new = torch.max( + energy, -1, keepdim=True)[0].expand_as(energy) - energy + attention = F.softmax(energy_new, dim=-1) + proj_value = x.view(batch_size, channels, -1) + + out = torch.bmm(attention, proj_value) + out = out.view(batch_size, channels, height, width) + + out = self.gamma(out) + x + return out + + +@HEADS.register_module() +class DAHead(BaseDecodeHead): + """Dual Attention Network for Scene Segmentation. + + This head is the implementation of `DANet + `_. + + Args: + pam_channels (int): The channels of Position Attention Module(PAM). + """ + + def __init__(self, pam_channels, **kwargs): + super(DAHead, self).__init__(**kwargs) + self.pam_channels = pam_channels + self.pam_in_conv = ConvModule( + self.in_channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.pam = PAM(self.channels, pam_channels) + self.pam_out_conv = ConvModule( + self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.pam_conv_seg = nn.Conv2d( + self.channels, self.num_classes, kernel_size=1) + + self.cam_in_conv = ConvModule( + self.in_channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.cam = CAM() + self.cam_out_conv = ConvModule( + self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.cam_conv_seg = nn.Conv2d( + self.channels, self.num_classes, kernel_size=1) + + def pam_cls_seg(self, feat): + """PAM feature classification.""" + if self.dropout is not None: + feat = self.dropout(feat) + output = self.pam_conv_seg(feat) + return output + + def cam_cls_seg(self, feat): + """CAM feature classification.""" + if self.dropout is not None: + feat = self.dropout(feat) + output = self.cam_conv_seg(feat) + return output + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + pam_feat = self.pam_in_conv(x) + pam_feat = self.pam(pam_feat) + pam_feat = self.pam_out_conv(pam_feat) + pam_out = self.pam_cls_seg(pam_feat) + + cam_feat = self.cam_in_conv(x) + cam_feat = self.cam(cam_feat) + cam_feat = self.cam_out_conv(cam_feat) + cam_out = self.cam_cls_seg(cam_feat) + + feat_sum = pam_feat + cam_feat + pam_cam_out = self.cls_seg(feat_sum) + + return pam_cam_out, pam_out, cam_out + + def forward_test(self, inputs, img_metas, test_cfg): + """Forward function for testing, only ``pam_cam`` is used.""" + return self.forward(inputs)[0] + + def losses(self, seg_logit, seg_label): + """Compute ``pam_cam``, ``pam``, ``cam`` loss.""" + pam_cam_seg_logit, pam_seg_logit, cam_seg_logit = seg_logit + loss = dict() + loss.update( + add_prefix( + super(DAHead, self).losses(pam_cam_seg_logit, seg_label), + 'pam_cam')) + loss.update( + add_prefix( + super(DAHead, self).losses(pam_seg_logit, seg_label), 'pam')) + loss.update( + add_prefix( + super(DAHead, self).losses(cam_seg_logit, seg_label), 'cam')) + return loss diff --git a/annotator/mmpkg/mmseg/models/decode_heads/decode_head.py b/annotator/mmpkg/mmseg/models/decode_heads/decode_head.py new file mode 100644 index 0000000000000000000000000000000000000000..a74c89f2ef1274ffe947995722576ab2c78eaec1 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/decode_head.py @@ -0,0 +1,234 @@ +from abc import ABCMeta, abstractmethod + +import torch +import torch.nn as nn +from annotator.mmpkg.mmcv.cnn import normal_init +from annotator.mmpkg.mmcv.runner import auto_fp16, force_fp32 + +from annotator.mmpkg.mmseg.core import build_pixel_sampler +from annotator.mmpkg.mmseg.ops import resize +from ..builder import build_loss +from ..losses import accuracy + + +class BaseDecodeHead(nn.Module, metaclass=ABCMeta): + """Base class for BaseDecodeHead. + + Args: + in_channels (int|Sequence[int]): Input channels. + channels (int): Channels after modules, before conv_seg. + num_classes (int): Number of classes. + dropout_ratio (float): Ratio of dropout layer. Default: 0.1. + conv_cfg (dict|None): Config of conv layers. Default: None. + norm_cfg (dict|None): Config of norm layers. Default: None. + act_cfg (dict): Config of activation layers. + Default: dict(type='ReLU') + in_index (int|Sequence[int]): Input feature index. Default: -1 + input_transform (str|None): Transformation type of input features. + Options: 'resize_concat', 'multiple_select', None. + 'resize_concat': Multiple feature maps will be resize to the + same size as first one and than concat together. + Usually used in FCN head of HRNet. + 'multiple_select': Multiple feature maps will be bundle into + a list and passed into decode head. + None: Only one select feature map is allowed. + Default: None. + loss_decode (dict): Config of decode loss. + Default: dict(type='CrossEntropyLoss'). + ignore_index (int | None): The label index to be ignored. When using + masked BCE loss, ignore_index should be set to None. Default: 255 + sampler (dict|None): The config of segmentation map sampler. + Default: None. + align_corners (bool): align_corners argument of F.interpolate. + Default: False. + """ + + def __init__(self, + in_channels, + channels, + *, + num_classes, + dropout_ratio=0.1, + conv_cfg=None, + norm_cfg=None, + act_cfg=dict(type='ReLU'), + in_index=-1, + input_transform=None, + loss_decode=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0), + ignore_index=255, + sampler=None, + align_corners=False): + super(BaseDecodeHead, self).__init__() + self._init_inputs(in_channels, in_index, input_transform) + self.channels = channels + self.num_classes = num_classes + self.dropout_ratio = dropout_ratio + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.in_index = in_index + self.loss_decode = build_loss(loss_decode) + self.ignore_index = ignore_index + self.align_corners = align_corners + if sampler is not None: + self.sampler = build_pixel_sampler(sampler, context=self) + else: + self.sampler = None + + self.conv_seg = nn.Conv2d(channels, num_classes, kernel_size=1) + if dropout_ratio > 0: + self.dropout = nn.Dropout2d(dropout_ratio) + else: + self.dropout = None + self.fp16_enabled = False + + def extra_repr(self): + """Extra repr.""" + s = f'input_transform={self.input_transform}, ' \ + f'ignore_index={self.ignore_index}, ' \ + f'align_corners={self.align_corners}' + return s + + def _init_inputs(self, in_channels, in_index, input_transform): + """Check and initialize input transforms. + + The in_channels, in_index and input_transform must match. + Specifically, when input_transform is None, only single feature map + will be selected. So in_channels and in_index must be of type int. + When input_transform + + Args: + in_channels (int|Sequence[int]): Input channels. + in_index (int|Sequence[int]): Input feature index. + input_transform (str|None): Transformation type of input features. + Options: 'resize_concat', 'multiple_select', None. + 'resize_concat': Multiple feature maps will be resize to the + same size as first one and than concat together. + Usually used in FCN head of HRNet. + 'multiple_select': Multiple feature maps will be bundle into + a list and passed into decode head. + None: Only one select feature map is allowed. + """ + + if input_transform is not None: + assert input_transform in ['resize_concat', 'multiple_select'] + self.input_transform = input_transform + self.in_index = in_index + if input_transform is not None: + assert isinstance(in_channels, (list, tuple)) + assert isinstance(in_index, (list, tuple)) + assert len(in_channels) == len(in_index) + if input_transform == 'resize_concat': + self.in_channels = sum(in_channels) + else: + self.in_channels = in_channels + else: + assert isinstance(in_channels, int) + assert isinstance(in_index, int) + self.in_channels = in_channels + + def init_weights(self): + """Initialize weights of classification layer.""" + normal_init(self.conv_seg, mean=0, std=0.01) + + def _transform_inputs(self, inputs): + """Transform inputs for decoder. + + Args: + inputs (list[Tensor]): List of multi-level img features. + + Returns: + Tensor: The transformed inputs + """ + + if self.input_transform == 'resize_concat': + inputs = [inputs[i] for i in self.in_index] + upsampled_inputs = [ + resize( + input=x, + size=inputs[0].shape[2:], + mode='bilinear', + align_corners=self.align_corners) for x in inputs + ] + inputs = torch.cat(upsampled_inputs, dim=1) + elif self.input_transform == 'multiple_select': + inputs = [inputs[i] for i in self.in_index] + else: + inputs = inputs[self.in_index] + + return inputs + + @auto_fp16() + @abstractmethod + def forward(self, inputs): + """Placeholder of forward function.""" + pass + + def forward_train(self, inputs, img_metas, gt_semantic_seg, train_cfg): + """Forward function for training. + Args: + inputs (list[Tensor]): List of multi-level img features. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + gt_semantic_seg (Tensor): Semantic segmentation masks + used if the architecture supports semantic segmentation task. + train_cfg (dict): The training config. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + seg_logits = self.forward(inputs) + losses = self.losses(seg_logits, gt_semantic_seg) + return losses + + def forward_test(self, inputs, img_metas, test_cfg): + """Forward function for testing. + + Args: + inputs (list[Tensor]): List of multi-level img features. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + test_cfg (dict): The testing config. + + Returns: + Tensor: Output segmentation map. + """ + return self.forward(inputs) + + def cls_seg(self, feat): + """Classify each pixel.""" + if self.dropout is not None: + feat = self.dropout(feat) + output = self.conv_seg(feat) + return output + + @force_fp32(apply_to=('seg_logit', )) + def losses(self, seg_logit, seg_label): + """Compute segmentation loss.""" + loss = dict() + seg_logit = resize( + input=seg_logit, + size=seg_label.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + if self.sampler is not None: + seg_weight = self.sampler.sample(seg_logit, seg_label) + else: + seg_weight = None + seg_label = seg_label.squeeze(1) + loss['loss_seg'] = self.loss_decode( + seg_logit, + seg_label, + weight=seg_weight, + ignore_index=self.ignore_index) + loss['acc_seg'] = accuracy(seg_logit, seg_label) + return loss diff --git a/annotator/mmpkg/mmseg/models/decode_heads/dm_head.py b/annotator/mmpkg/mmseg/models/decode_heads/dm_head.py new file mode 100644 index 0000000000000000000000000000000000000000..de6d0f6390d96c1eef4242cdc9aed91ec7714c6a --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/dm_head.py @@ -0,0 +1,140 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from annotator.mmpkg.mmcv.cnn import ConvModule, build_activation_layer, build_norm_layer + +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +class DCM(nn.Module): + """Dynamic Convolutional Module used in DMNet. + + Args: + filter_size (int): The filter size of generated convolution kernel + used in Dynamic Convolutional Module. + fusion (bool): Add one conv to fuse DCM output feature. + in_channels (int): Input channels. + channels (int): Channels after modules, before conv_seg. + conv_cfg (dict | None): Config of conv layers. + norm_cfg (dict | None): Config of norm layers. + act_cfg (dict): Config of activation layers. + """ + + def __init__(self, filter_size, fusion, in_channels, channels, conv_cfg, + norm_cfg, act_cfg): + super(DCM, self).__init__() + self.filter_size = filter_size + self.fusion = fusion + self.in_channels = in_channels + self.channels = channels + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.filter_gen_conv = nn.Conv2d(self.in_channels, self.channels, 1, 1, + 0) + + self.input_redu_conv = ConvModule( + self.in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + if self.norm_cfg is not None: + self.norm = build_norm_layer(self.norm_cfg, self.channels)[1] + else: + self.norm = None + self.activate = build_activation_layer(self.act_cfg) + + if self.fusion: + self.fusion_conv = ConvModule( + self.channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, x): + """Forward function.""" + generated_filter = self.filter_gen_conv( + F.adaptive_avg_pool2d(x, self.filter_size)) + x = self.input_redu_conv(x) + b, c, h, w = x.shape + # [1, b * c, h, w], c = self.channels + x = x.view(1, b * c, h, w) + # [b * c, 1, filter_size, filter_size] + generated_filter = generated_filter.view(b * c, 1, self.filter_size, + self.filter_size) + pad = (self.filter_size - 1) // 2 + if (self.filter_size - 1) % 2 == 0: + p2d = (pad, pad, pad, pad) + else: + p2d = (pad + 1, pad, pad + 1, pad) + x = F.pad(input=x, pad=p2d, mode='constant', value=0) + # [1, b * c, h, w] + output = F.conv2d(input=x, weight=generated_filter, groups=b * c) + # [b, c, h, w] + output = output.view(b, c, h, w) + if self.norm is not None: + output = self.norm(output) + output = self.activate(output) + + if self.fusion: + output = self.fusion_conv(output) + + return output + + +@HEADS.register_module() +class DMHead(BaseDecodeHead): + """Dynamic Multi-scale Filters for Semantic Segmentation. + + This head is the implementation of + `DMNet `_. + + Args: + filter_sizes (tuple[int]): The size of generated convolutional filters + used in Dynamic Convolutional Module. Default: (1, 3, 5, 7). + fusion (bool): Add one conv to fuse DCM output feature. + """ + + def __init__(self, filter_sizes=(1, 3, 5, 7), fusion=False, **kwargs): + super(DMHead, self).__init__(**kwargs) + assert isinstance(filter_sizes, (list, tuple)) + self.filter_sizes = filter_sizes + self.fusion = fusion + dcm_modules = [] + for filter_size in self.filter_sizes: + dcm_modules.append( + DCM(filter_size, + self.fusion, + self.in_channels, + self.channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + self.dcm_modules = nn.ModuleList(dcm_modules) + self.bottleneck = ConvModule( + self.in_channels + len(filter_sizes) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + dcm_outs = [x] + for dcm_module in self.dcm_modules: + dcm_outs.append(dcm_module(x)) + dcm_outs = torch.cat(dcm_outs, dim=1) + output = self.bottleneck(dcm_outs) + output = self.cls_seg(output) + return output diff --git a/annotator/mmpkg/mmseg/models/decode_heads/dnl_head.py b/annotator/mmpkg/mmseg/models/decode_heads/dnl_head.py new file mode 100644 index 0000000000000000000000000000000000000000..b3bb1de1499ad043cc51b2269b4d970d07c16076 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/dnl_head.py @@ -0,0 +1,131 @@ +import torch +from annotator.mmpkg.mmcv.cnn import NonLocal2d +from torch import nn + +from ..builder import HEADS +from .fcn_head import FCNHead + + +class DisentangledNonLocal2d(NonLocal2d): + """Disentangled Non-Local Blocks. + + Args: + temperature (float): Temperature to adjust attention. Default: 0.05 + """ + + def __init__(self, *arg, temperature, **kwargs): + super().__init__(*arg, **kwargs) + self.temperature = temperature + self.conv_mask = nn.Conv2d(self.in_channels, 1, kernel_size=1) + + def embedded_gaussian(self, theta_x, phi_x): + """Embedded gaussian with temperature.""" + + # NonLocal2d pairwise_weight: [N, HxW, HxW] + pairwise_weight = torch.matmul(theta_x, phi_x) + if self.use_scale: + # theta_x.shape[-1] is `self.inter_channels` + pairwise_weight /= theta_x.shape[-1]**0.5 + pairwise_weight /= self.temperature + pairwise_weight = pairwise_weight.softmax(dim=-1) + return pairwise_weight + + def forward(self, x): + # x: [N, C, H, W] + n = x.size(0) + + # g_x: [N, HxW, C] + g_x = self.g(x).view(n, self.inter_channels, -1) + g_x = g_x.permute(0, 2, 1) + + # theta_x: [N, HxW, C], phi_x: [N, C, HxW] + if self.mode == 'gaussian': + theta_x = x.view(n, self.in_channels, -1) + theta_x = theta_x.permute(0, 2, 1) + if self.sub_sample: + phi_x = self.phi(x).view(n, self.in_channels, -1) + else: + phi_x = x.view(n, self.in_channels, -1) + elif self.mode == 'concatenation': + theta_x = self.theta(x).view(n, self.inter_channels, -1, 1) + phi_x = self.phi(x).view(n, self.inter_channels, 1, -1) + else: + theta_x = self.theta(x).view(n, self.inter_channels, -1) + theta_x = theta_x.permute(0, 2, 1) + phi_x = self.phi(x).view(n, self.inter_channels, -1) + + # subtract mean + theta_x -= theta_x.mean(dim=-2, keepdim=True) + phi_x -= phi_x.mean(dim=-1, keepdim=True) + + pairwise_func = getattr(self, self.mode) + # pairwise_weight: [N, HxW, HxW] + pairwise_weight = pairwise_func(theta_x, phi_x) + + # y: [N, HxW, C] + y = torch.matmul(pairwise_weight, g_x) + # y: [N, C, H, W] + y = y.permute(0, 2, 1).contiguous().reshape(n, self.inter_channels, + *x.size()[2:]) + + # unary_mask: [N, 1, HxW] + unary_mask = self.conv_mask(x) + unary_mask = unary_mask.view(n, 1, -1) + unary_mask = unary_mask.softmax(dim=-1) + # unary_x: [N, 1, C] + unary_x = torch.matmul(unary_mask, g_x) + # unary_x: [N, C, 1, 1] + unary_x = unary_x.permute(0, 2, 1).contiguous().reshape( + n, self.inter_channels, 1, 1) + + output = x + self.conv_out(y + unary_x) + + return output + + +@HEADS.register_module() +class DNLHead(FCNHead): + """Disentangled Non-Local Neural Networks. + + This head is the implementation of `DNLNet + `_. + + Args: + reduction (int): Reduction factor of projection transform. Default: 2. + use_scale (bool): Whether to scale pairwise_weight by + sqrt(1/inter_channels). Default: False. + mode (str): The nonlocal mode. Options are 'embedded_gaussian', + 'dot_product'. Default: 'embedded_gaussian.'. + temperature (float): Temperature to adjust attention. Default: 0.05 + """ + + def __init__(self, + reduction=2, + use_scale=True, + mode='embedded_gaussian', + temperature=0.05, + **kwargs): + super(DNLHead, self).__init__(num_convs=2, **kwargs) + self.reduction = reduction + self.use_scale = use_scale + self.mode = mode + self.temperature = temperature + self.dnl_block = DisentangledNonLocal2d( + in_channels=self.channels, + reduction=self.reduction, + use_scale=self.use_scale, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + mode=self.mode, + temperature=self.temperature) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + output = self.convs[0](x) + output = self.dnl_block(output) + output = self.convs[1](output) + if self.concat_input: + output = self.conv_cat(torch.cat([x, output], dim=1)) + output = self.cls_seg(output) + return output diff --git a/annotator/mmpkg/mmseg/models/decode_heads/ema_head.py b/annotator/mmpkg/mmseg/models/decode_heads/ema_head.py new file mode 100644 index 0000000000000000000000000000000000000000..aaebae7b25579cabcd3967da765568a282869a49 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/ema_head.py @@ -0,0 +1,168 @@ +import math + +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +from annotator.mmpkg.mmcv.cnn import ConvModule + +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +def reduce_mean(tensor): + """Reduce mean when distributed training.""" + if not (dist.is_available() and dist.is_initialized()): + return tensor + tensor = tensor.clone() + dist.all_reduce(tensor.div_(dist.get_world_size()), op=dist.ReduceOp.SUM) + return tensor + + +class EMAModule(nn.Module): + """Expectation Maximization Attention Module used in EMANet. + + Args: + channels (int): Channels of the whole module. + num_bases (int): Number of bases. + num_stages (int): Number of the EM iterations. + """ + + def __init__(self, channels, num_bases, num_stages, momentum): + super(EMAModule, self).__init__() + assert num_stages >= 1, 'num_stages must be at least 1!' + self.num_bases = num_bases + self.num_stages = num_stages + self.momentum = momentum + + bases = torch.zeros(1, channels, self.num_bases) + bases.normal_(0, math.sqrt(2. / self.num_bases)) + # [1, channels, num_bases] + bases = F.normalize(bases, dim=1, p=2) + self.register_buffer('bases', bases) + + def forward(self, feats): + """Forward function.""" + batch_size, channels, height, width = feats.size() + # [batch_size, channels, height*width] + feats = feats.view(batch_size, channels, height * width) + # [batch_size, channels, num_bases] + bases = self.bases.repeat(batch_size, 1, 1) + + with torch.no_grad(): + for i in range(self.num_stages): + # [batch_size, height*width, num_bases] + attention = torch.einsum('bcn,bck->bnk', feats, bases) + attention = F.softmax(attention, dim=2) + # l1 norm + attention_normed = F.normalize(attention, dim=1, p=1) + # [batch_size, channels, num_bases] + bases = torch.einsum('bcn,bnk->bck', feats, attention_normed) + # l2 norm + bases = F.normalize(bases, dim=1, p=2) + + feats_recon = torch.einsum('bck,bnk->bcn', bases, attention) + feats_recon = feats_recon.view(batch_size, channels, height, width) + + if self.training: + bases = bases.mean(dim=0, keepdim=True) + bases = reduce_mean(bases) + # l2 norm + bases = F.normalize(bases, dim=1, p=2) + self.bases = (1 - + self.momentum) * self.bases + self.momentum * bases + + return feats_recon + + +@HEADS.register_module() +class EMAHead(BaseDecodeHead): + """Expectation Maximization Attention Networks for Semantic Segmentation. + + This head is the implementation of `EMANet + `_. + + Args: + ema_channels (int): EMA module channels + num_bases (int): Number of bases. + num_stages (int): Number of the EM iterations. + concat_input (bool): Whether concat the input and output of convs + before classification layer. Default: True + momentum (float): Momentum to update the base. Default: 0.1. + """ + + def __init__(self, + ema_channels, + num_bases, + num_stages, + concat_input=True, + momentum=0.1, + **kwargs): + super(EMAHead, self).__init__(**kwargs) + self.ema_channels = ema_channels + self.num_bases = num_bases + self.num_stages = num_stages + self.concat_input = concat_input + self.momentum = momentum + self.ema_module = EMAModule(self.ema_channels, self.num_bases, + self.num_stages, self.momentum) + + self.ema_in_conv = ConvModule( + self.in_channels, + self.ema_channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + # project (0, inf) -> (-inf, inf) + self.ema_mid_conv = ConvModule( + self.ema_channels, + self.ema_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=None, + act_cfg=None) + for param in self.ema_mid_conv.parameters(): + param.requires_grad = False + + self.ema_out_conv = ConvModule( + self.ema_channels, + self.ema_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=None) + self.bottleneck = ConvModule( + self.ema_channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + if self.concat_input: + self.conv_cat = ConvModule( + self.in_channels + self.channels, + self.channels, + kernel_size=3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + feats = self.ema_in_conv(x) + identity = feats + feats = self.ema_mid_conv(feats) + recon = self.ema_module(feats) + recon = F.relu(recon, inplace=True) + recon = self.ema_out_conv(recon) + output = F.relu(identity + recon, inplace=True) + output = self.bottleneck(output) + if self.concat_input: + output = self.conv_cat(torch.cat([x, output], dim=1)) + output = self.cls_seg(output) + return output diff --git a/annotator/mmpkg/mmseg/models/decode_heads/enc_head.py b/annotator/mmpkg/mmseg/models/decode_heads/enc_head.py new file mode 100644 index 0000000000000000000000000000000000000000..4c2a22a90b26f3264f63234694f0f290a7891ea2 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/enc_head.py @@ -0,0 +1,187 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from annotator.mmpkg.mmcv.cnn import ConvModule, build_norm_layer + +from annotator.mmpkg.mmseg.ops import Encoding, resize +from ..builder import HEADS, build_loss +from .decode_head import BaseDecodeHead + + +class EncModule(nn.Module): + """Encoding Module used in EncNet. + + Args: + in_channels (int): Input channels. + num_codes (int): Number of code words. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict): Config of activation layers. + """ + + def __init__(self, in_channels, num_codes, conv_cfg, norm_cfg, act_cfg): + super(EncModule, self).__init__() + self.encoding_project = ConvModule( + in_channels, + in_channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + # TODO: resolve this hack + # change to 1d + if norm_cfg is not None: + encoding_norm_cfg = norm_cfg.copy() + if encoding_norm_cfg['type'] in ['BN', 'IN']: + encoding_norm_cfg['type'] += '1d' + else: + encoding_norm_cfg['type'] = encoding_norm_cfg['type'].replace( + '2d', '1d') + else: + # fallback to BN1d + encoding_norm_cfg = dict(type='BN1d') + self.encoding = nn.Sequential( + Encoding(channels=in_channels, num_codes=num_codes), + build_norm_layer(encoding_norm_cfg, num_codes)[1], + nn.ReLU(inplace=True)) + self.fc = nn.Sequential( + nn.Linear(in_channels, in_channels), nn.Sigmoid()) + + def forward(self, x): + """Forward function.""" + encoding_projection = self.encoding_project(x) + encoding_feat = self.encoding(encoding_projection).mean(dim=1) + batch_size, channels, _, _ = x.size() + gamma = self.fc(encoding_feat) + y = gamma.view(batch_size, channels, 1, 1) + output = F.relu_(x + x * y) + return encoding_feat, output + + +@HEADS.register_module() +class EncHead(BaseDecodeHead): + """Context Encoding for Semantic Segmentation. + + This head is the implementation of `EncNet + `_. + + Args: + num_codes (int): Number of code words. Default: 32. + use_se_loss (bool): Whether use Semantic Encoding Loss (SE-loss) to + regularize the training. Default: True. + add_lateral (bool): Whether use lateral connection to fuse features. + Default: False. + loss_se_decode (dict): Config of decode loss. + Default: dict(type='CrossEntropyLoss', use_sigmoid=True). + """ + + def __init__(self, + num_codes=32, + use_se_loss=True, + add_lateral=False, + loss_se_decode=dict( + type='CrossEntropyLoss', + use_sigmoid=True, + loss_weight=0.2), + **kwargs): + super(EncHead, self).__init__( + input_transform='multiple_select', **kwargs) + self.use_se_loss = use_se_loss + self.add_lateral = add_lateral + self.num_codes = num_codes + self.bottleneck = ConvModule( + self.in_channels[-1], + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + if add_lateral: + self.lateral_convs = nn.ModuleList() + for in_channels in self.in_channels[:-1]: # skip the last one + self.lateral_convs.append( + ConvModule( + in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + self.fusion = ConvModule( + len(self.in_channels) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.enc_module = EncModule( + self.channels, + num_codes=num_codes, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + if self.use_se_loss: + self.loss_se_decode = build_loss(loss_se_decode) + self.se_layer = nn.Linear(self.channels, self.num_classes) + + def forward(self, inputs): + """Forward function.""" + inputs = self._transform_inputs(inputs) + feat = self.bottleneck(inputs[-1]) + if self.add_lateral: + laterals = [ + resize( + lateral_conv(inputs[i]), + size=feat.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + feat = self.fusion(torch.cat([feat, *laterals], 1)) + encode_feat, output = self.enc_module(feat) + output = self.cls_seg(output) + if self.use_se_loss: + se_output = self.se_layer(encode_feat) + return output, se_output + else: + return output + + def forward_test(self, inputs, img_metas, test_cfg): + """Forward function for testing, ignore se_loss.""" + if self.use_se_loss: + return self.forward(inputs)[0] + else: + return self.forward(inputs) + + @staticmethod + def _convert_to_onehot_labels(seg_label, num_classes): + """Convert segmentation label to onehot. + + Args: + seg_label (Tensor): Segmentation label of shape (N, H, W). + num_classes (int): Number of classes. + + Returns: + Tensor: Onehot labels of shape (N, num_classes). + """ + + batch_size = seg_label.size(0) + onehot_labels = seg_label.new_zeros((batch_size, num_classes)) + for i in range(batch_size): + hist = seg_label[i].float().histc( + bins=num_classes, min=0, max=num_classes - 1) + onehot_labels[i] = hist > 0 + return onehot_labels + + def losses(self, seg_logit, seg_label): + """Compute segmentation and semantic encoding loss.""" + seg_logit, se_seg_logit = seg_logit + loss = dict() + loss.update(super(EncHead, self).losses(seg_logit, seg_label)) + se_loss = self.loss_se_decode( + se_seg_logit, + self._convert_to_onehot_labels(seg_label, self.num_classes)) + loss['loss_se'] = se_loss + return loss diff --git a/annotator/mmpkg/mmseg/models/decode_heads/fcn_head.py b/annotator/mmpkg/mmseg/models/decode_heads/fcn_head.py new file mode 100644 index 0000000000000000000000000000000000000000..c4583c57246e8e3b1d15d240b943d046afa5cba5 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/fcn_head.py @@ -0,0 +1,81 @@ +import torch +import torch.nn as nn +from annotator.mmpkg.mmcv.cnn import ConvModule + +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +@HEADS.register_module() +class FCNHead(BaseDecodeHead): + """Fully Convolution Networks for Semantic Segmentation. + + This head is implemented of `FCNNet `_. + + Args: + num_convs (int): Number of convs in the head. Default: 2. + kernel_size (int): The kernel size for convs in the head. Default: 3. + concat_input (bool): Whether concat the input and output of convs + before classification layer. + dilation (int): The dilation rate for convs in the head. Default: 1. + """ + + def __init__(self, + num_convs=2, + kernel_size=3, + concat_input=True, + dilation=1, + **kwargs): + assert num_convs >= 0 and dilation > 0 and isinstance(dilation, int) + self.num_convs = num_convs + self.concat_input = concat_input + self.kernel_size = kernel_size + super(FCNHead, self).__init__(**kwargs) + if num_convs == 0: + assert self.in_channels == self.channels + + conv_padding = (kernel_size // 2) * dilation + convs = [] + convs.append( + ConvModule( + self.in_channels, + self.channels, + kernel_size=kernel_size, + padding=conv_padding, + dilation=dilation, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + for i in range(num_convs - 1): + convs.append( + ConvModule( + self.channels, + self.channels, + kernel_size=kernel_size, + padding=conv_padding, + dilation=dilation, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + if num_convs == 0: + self.convs = nn.Identity() + else: + self.convs = nn.Sequential(*convs) + if self.concat_input: + self.conv_cat = ConvModule( + self.in_channels + self.channels, + self.channels, + kernel_size=kernel_size, + padding=kernel_size // 2, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + output = self.convs(x) + if self.concat_input: + output = self.conv_cat(torch.cat([x, output], dim=1)) + output = self.cls_seg(output) + return output diff --git a/annotator/mmpkg/mmseg/models/decode_heads/fpn_head.py b/annotator/mmpkg/mmseg/models/decode_heads/fpn_head.py new file mode 100644 index 0000000000000000000000000000000000000000..1a9ba39eebc406bfa422dc98eeaa32a800008a83 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/fpn_head.py @@ -0,0 +1,68 @@ +import numpy as np +import torch.nn as nn +from annotator.mmpkg.mmcv.cnn import ConvModule + +from annotator.mmpkg.mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +@HEADS.register_module() +class FPNHead(BaseDecodeHead): + """Panoptic Feature Pyramid Networks. + + This head is the implementation of `Semantic FPN + `_. + + Args: + feature_strides (tuple[int]): The strides for input feature maps. + stack_lateral. All strides suppose to be power of 2. The first + one is of largest resolution. + """ + + def __init__(self, feature_strides, **kwargs): + super(FPNHead, self).__init__( + input_transform='multiple_select', **kwargs) + assert len(feature_strides) == len(self.in_channels) + assert min(feature_strides) == feature_strides[0] + self.feature_strides = feature_strides + + self.scale_heads = nn.ModuleList() + for i in range(len(feature_strides)): + head_length = max( + 1, + int(np.log2(feature_strides[i]) - np.log2(feature_strides[0]))) + scale_head = [] + for k in range(head_length): + scale_head.append( + ConvModule( + self.in_channels[i] if k == 0 else self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + if feature_strides[i] != feature_strides[0]: + scale_head.append( + nn.Upsample( + scale_factor=2, + mode='bilinear', + align_corners=self.align_corners)) + self.scale_heads.append(nn.Sequential(*scale_head)) + + def forward(self, inputs): + + x = self._transform_inputs(inputs) + + output = self.scale_heads[0](x[0]) + for i in range(1, len(self.feature_strides)): + # non inplace + output = output + resize( + self.scale_heads[i](x[i]), + size=output.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + + output = self.cls_seg(output) + return output diff --git a/annotator/mmpkg/mmseg/models/decode_heads/gc_head.py b/annotator/mmpkg/mmseg/models/decode_heads/gc_head.py new file mode 100644 index 0000000000000000000000000000000000000000..6342811f67e4affac7886c8fc745a28abcc32c55 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/gc_head.py @@ -0,0 +1,47 @@ +import torch +from annotator.mmpkg.mmcv.cnn import ContextBlock + +from ..builder import HEADS +from .fcn_head import FCNHead + + +@HEADS.register_module() +class GCHead(FCNHead): + """GCNet: Non-local Networks Meet Squeeze-Excitation Networks and Beyond. + + This head is the implementation of `GCNet + `_. + + Args: + ratio (float): Multiplier of channels ratio. Default: 1/4. + pooling_type (str): The pooling type of context aggregation. + Options are 'att', 'avg'. Default: 'avg'. + fusion_types (tuple[str]): The fusion type for feature fusion. + Options are 'channel_add', 'channel_mul'. Default: ('channel_add',) + """ + + def __init__(self, + ratio=1 / 4., + pooling_type='att', + fusion_types=('channel_add', ), + **kwargs): + super(GCHead, self).__init__(num_convs=2, **kwargs) + self.ratio = ratio + self.pooling_type = pooling_type + self.fusion_types = fusion_types + self.gc_block = ContextBlock( + in_channels=self.channels, + ratio=self.ratio, + pooling_type=self.pooling_type, + fusion_types=self.fusion_types) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + output = self.convs[0](x) + output = self.gc_block(output) + output = self.convs[1](output) + if self.concat_input: + output = self.conv_cat(torch.cat([x, output], dim=1)) + output = self.cls_seg(output) + return output diff --git a/annotator/mmpkg/mmseg/models/decode_heads/lraspp_head.py b/annotator/mmpkg/mmseg/models/decode_heads/lraspp_head.py new file mode 100644 index 0000000000000000000000000000000000000000..b29d80e77d05cc0c12118e335e266a73bda99ed0 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/lraspp_head.py @@ -0,0 +1,90 @@ +import torch +import torch.nn as nn +from annotator.mmpkg.mmcv import is_tuple_of +from annotator.mmpkg.mmcv.cnn import ConvModule + +from annotator.mmpkg.mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +@HEADS.register_module() +class LRASPPHead(BaseDecodeHead): + """Lite R-ASPP (LRASPP) head is proposed in Searching for MobileNetV3. + + This head is the improved implementation of `Searching for MobileNetV3 + `_. + + Args: + branch_channels (tuple[int]): The number of output channels in every + each branch. Default: (32, 64). + """ + + def __init__(self, branch_channels=(32, 64), **kwargs): + super(LRASPPHead, self).__init__(**kwargs) + if self.input_transform != 'multiple_select': + raise ValueError('in Lite R-ASPP (LRASPP) head, input_transform ' + f'must be \'multiple_select\'. But received ' + f'\'{self.input_transform}\'') + assert is_tuple_of(branch_channels, int) + assert len(branch_channels) == len(self.in_channels) - 1 + self.branch_channels = branch_channels + + self.convs = nn.Sequential() + self.conv_ups = nn.Sequential() + for i in range(len(branch_channels)): + self.convs.add_module( + f'conv{i}', + nn.Conv2d( + self.in_channels[i], branch_channels[i], 1, bias=False)) + self.conv_ups.add_module( + f'conv_up{i}', + ConvModule( + self.channels + branch_channels[i], + self.channels, + 1, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + bias=False)) + + self.conv_up_input = nn.Conv2d(self.channels, self.channels, 1) + + self.aspp_conv = ConvModule( + self.in_channels[-1], + self.channels, + 1, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + bias=False) + self.image_pool = nn.Sequential( + nn.AvgPool2d(kernel_size=49, stride=(16, 20)), + ConvModule( + self.in_channels[2], + self.channels, + 1, + act_cfg=dict(type='Sigmoid'), + bias=False)) + + def forward(self, inputs): + """Forward function.""" + inputs = self._transform_inputs(inputs) + + x = inputs[-1] + + x = self.aspp_conv(x) * resize( + self.image_pool(x), + size=x.size()[2:], + mode='bilinear', + align_corners=self.align_corners) + x = self.conv_up_input(x) + + for i in range(len(self.branch_channels) - 1, -1, -1): + x = resize( + x, + size=inputs[i].size()[2:], + mode='bilinear', + align_corners=self.align_corners) + x = torch.cat([x, self.convs[i](inputs[i])], 1) + x = self.conv_ups[i](x) + + return self.cls_seg(x) diff --git a/annotator/mmpkg/mmseg/models/decode_heads/nl_head.py b/annotator/mmpkg/mmseg/models/decode_heads/nl_head.py new file mode 100644 index 0000000000000000000000000000000000000000..5990df1b8b0d57cfa772ec1b6b6be20a8f667ce7 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/nl_head.py @@ -0,0 +1,49 @@ +import torch +from annotator.mmpkg.mmcv.cnn import NonLocal2d + +from ..builder import HEADS +from .fcn_head import FCNHead + + +@HEADS.register_module() +class NLHead(FCNHead): + """Non-local Neural Networks. + + This head is the implementation of `NLNet + `_. + + Args: + reduction (int): Reduction factor of projection transform. Default: 2. + use_scale (bool): Whether to scale pairwise_weight by + sqrt(1/inter_channels). Default: True. + mode (str): The nonlocal mode. Options are 'embedded_gaussian', + 'dot_product'. Default: 'embedded_gaussian.'. + """ + + def __init__(self, + reduction=2, + use_scale=True, + mode='embedded_gaussian', + **kwargs): + super(NLHead, self).__init__(num_convs=2, **kwargs) + self.reduction = reduction + self.use_scale = use_scale + self.mode = mode + self.nl_block = NonLocal2d( + in_channels=self.channels, + reduction=self.reduction, + use_scale=self.use_scale, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + mode=self.mode) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + output = self.convs[0](x) + output = self.nl_block(output) + output = self.convs[1](output) + if self.concat_input: + output = self.conv_cat(torch.cat([x, output], dim=1)) + output = self.cls_seg(output) + return output diff --git a/annotator/mmpkg/mmseg/models/decode_heads/ocr_head.py b/annotator/mmpkg/mmseg/models/decode_heads/ocr_head.py new file mode 100644 index 0000000000000000000000000000000000000000..c46d10e5baff54e182af0426a1ecfea9ca190a9f --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/ocr_head.py @@ -0,0 +1,127 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from annotator.mmpkg.mmcv.cnn import ConvModule + +from annotator.mmpkg.mmseg.ops import resize +from ..builder import HEADS +from ..utils import SelfAttentionBlock as _SelfAttentionBlock +from .cascade_decode_head import BaseCascadeDecodeHead + + +class SpatialGatherModule(nn.Module): + """Aggregate the context features according to the initial predicted + probability distribution. + + Employ the soft-weighted method to aggregate the context. + """ + + def __init__(self, scale): + super(SpatialGatherModule, self).__init__() + self.scale = scale + + def forward(self, feats, probs): + """Forward function.""" + batch_size, num_classes, height, width = probs.size() + channels = feats.size(1) + probs = probs.view(batch_size, num_classes, -1) + feats = feats.view(batch_size, channels, -1) + # [batch_size, height*width, num_classes] + feats = feats.permute(0, 2, 1) + # [batch_size, channels, height*width] + probs = F.softmax(self.scale * probs, dim=2) + # [batch_size, channels, num_classes] + ocr_context = torch.matmul(probs, feats) + ocr_context = ocr_context.permute(0, 2, 1).contiguous().unsqueeze(3) + return ocr_context + + +class ObjectAttentionBlock(_SelfAttentionBlock): + """Make a OCR used SelfAttentionBlock.""" + + def __init__(self, in_channels, channels, scale, conv_cfg, norm_cfg, + act_cfg): + if scale > 1: + query_downsample = nn.MaxPool2d(kernel_size=scale) + else: + query_downsample = None + super(ObjectAttentionBlock, self).__init__( + key_in_channels=in_channels, + query_in_channels=in_channels, + channels=channels, + out_channels=in_channels, + share_key_query=False, + query_downsample=query_downsample, + key_downsample=None, + key_query_num_convs=2, + key_query_norm=True, + value_out_num_convs=1, + value_out_norm=True, + matmul_norm=True, + with_out=True, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + self.bottleneck = ConvModule( + in_channels * 2, + in_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, query_feats, key_feats): + """Forward function.""" + context = super(ObjectAttentionBlock, + self).forward(query_feats, key_feats) + output = self.bottleneck(torch.cat([context, query_feats], dim=1)) + if self.query_downsample is not None: + output = resize(query_feats) + + return output + + +@HEADS.register_module() +class OCRHead(BaseCascadeDecodeHead): + """Object-Contextual Representations for Semantic Segmentation. + + This head is the implementation of `OCRNet + `_. + + Args: + ocr_channels (int): The intermediate channels of OCR block. + scale (int): The scale of probability map in SpatialGatherModule in + Default: 1. + """ + + def __init__(self, ocr_channels, scale=1, **kwargs): + super(OCRHead, self).__init__(**kwargs) + self.ocr_channels = ocr_channels + self.scale = scale + self.object_context_block = ObjectAttentionBlock( + self.channels, + self.ocr_channels, + self.scale, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.spatial_gather_module = SpatialGatherModule(self.scale) + + self.bottleneck = ConvModule( + self.in_channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs, prev_output): + """Forward function.""" + x = self._transform_inputs(inputs) + feats = self.bottleneck(x) + context = self.spatial_gather_module(feats, prev_output) + object_context = self.object_context_block(feats, context) + output = self.cls_seg(object_context) + + return output diff --git a/annotator/mmpkg/mmseg/models/decode_heads/point_head.py b/annotator/mmpkg/mmseg/models/decode_heads/point_head.py new file mode 100644 index 0000000000000000000000000000000000000000..c6782763e30386d99115977ebe5a4d9291bae8d9 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/point_head.py @@ -0,0 +1,354 @@ +# Modified from https://github.com/facebookresearch/detectron2/tree/master/projects/PointRend/point_head/point_head.py # noqa + +import torch +import torch.nn as nn + +try: + from mmcv.cnn import ConvModule, normal_init + from mmcv.ops import point_sample +except ImportError: + from annotator.mmpkg.mmcv.cnn import ConvModule, normal_init + from annotator.mmpkg.mmcv.ops import point_sample + +from annotator.mmpkg.mmseg.models.builder import HEADS +from annotator.mmpkg.mmseg.ops import resize +from ..losses import accuracy +from .cascade_decode_head import BaseCascadeDecodeHead + + +def calculate_uncertainty(seg_logits): + """Estimate uncertainty based on seg logits. + + For each location of the prediction ``seg_logits`` we estimate + uncertainty as the difference between top first and top second + predicted logits. + + Args: + seg_logits (Tensor): Semantic segmentation logits, + shape (batch_size, num_classes, height, width). + + Returns: + scores (Tensor): T uncertainty scores with the most uncertain + locations having the highest uncertainty score, shape ( + batch_size, 1, height, width) + """ + top2_scores = torch.topk(seg_logits, k=2, dim=1)[0] + return (top2_scores[:, 1] - top2_scores[:, 0]).unsqueeze(1) + + +@HEADS.register_module() +class PointHead(BaseCascadeDecodeHead): + """A mask point head use in PointRend. + + ``PointHead`` use shared multi-layer perceptron (equivalent to + nn.Conv1d) to predict the logit of input points. The fine-grained feature + and coarse feature will be concatenate together for predication. + + Args: + num_fcs (int): Number of fc layers in the head. Default: 3. + in_channels (int): Number of input channels. Default: 256. + fc_channels (int): Number of fc channels. Default: 256. + num_classes (int): Number of classes for logits. Default: 80. + class_agnostic (bool): Whether use class agnostic classification. + If so, the output channels of logits will be 1. Default: False. + coarse_pred_each_layer (bool): Whether concatenate coarse feature with + the output of each fc layer. Default: True. + conv_cfg (dict|None): Dictionary to construct and config conv layer. + Default: dict(type='Conv1d')) + norm_cfg (dict|None): Dictionary to construct and config norm layer. + Default: None. + loss_point (dict): Dictionary to construct and config loss layer of + point head. Default: dict(type='CrossEntropyLoss', use_mask=True, + loss_weight=1.0). + """ + + def __init__(self, + num_fcs=3, + coarse_pred_each_layer=True, + conv_cfg=dict(type='Conv1d'), + norm_cfg=None, + act_cfg=dict(type='ReLU', inplace=False), + **kwargs): + super(PointHead, self).__init__( + input_transform='multiple_select', + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + **kwargs) + + self.num_fcs = num_fcs + self.coarse_pred_each_layer = coarse_pred_each_layer + + fc_in_channels = sum(self.in_channels) + self.num_classes + fc_channels = self.channels + self.fcs = nn.ModuleList() + for k in range(num_fcs): + fc = ConvModule( + fc_in_channels, + fc_channels, + kernel_size=1, + stride=1, + padding=0, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + self.fcs.append(fc) + fc_in_channels = fc_channels + fc_in_channels += self.num_classes if self.coarse_pred_each_layer \ + else 0 + self.fc_seg = nn.Conv1d( + fc_in_channels, + self.num_classes, + kernel_size=1, + stride=1, + padding=0) + if self.dropout_ratio > 0: + self.dropout = nn.Dropout(self.dropout_ratio) + delattr(self, 'conv_seg') + + def init_weights(self): + """Initialize weights of classification layer.""" + normal_init(self.fc_seg, std=0.001) + + def cls_seg(self, feat): + """Classify each pixel with fc.""" + if self.dropout is not None: + feat = self.dropout(feat) + output = self.fc_seg(feat) + return output + + def forward(self, fine_grained_point_feats, coarse_point_feats): + x = torch.cat([fine_grained_point_feats, coarse_point_feats], dim=1) + for fc in self.fcs: + x = fc(x) + if self.coarse_pred_each_layer: + x = torch.cat((x, coarse_point_feats), dim=1) + return self.cls_seg(x) + + def _get_fine_grained_point_feats(self, x, points): + """Sample from fine grained features. + + Args: + x (list[Tensor]): Feature pyramid from by neck or backbone. + points (Tensor): Point coordinates, shape (batch_size, + num_points, 2). + + Returns: + fine_grained_feats (Tensor): Sampled fine grained feature, + shape (batch_size, sum(channels of x), num_points). + """ + + fine_grained_feats_list = [ + point_sample(_, points, align_corners=self.align_corners) + for _ in x + ] + if len(fine_grained_feats_list) > 1: + fine_grained_feats = torch.cat(fine_grained_feats_list, dim=1) + else: + fine_grained_feats = fine_grained_feats_list[0] + + return fine_grained_feats + + def _get_coarse_point_feats(self, prev_output, points): + """Sample from fine grained features. + + Args: + prev_output (list[Tensor]): Prediction of previous decode head. + points (Tensor): Point coordinates, shape (batch_size, + num_points, 2). + + Returns: + coarse_feats (Tensor): Sampled coarse feature, shape (batch_size, + num_classes, num_points). + """ + + coarse_feats = point_sample( + prev_output, points, align_corners=self.align_corners) + + return coarse_feats + + def forward_train(self, inputs, prev_output, img_metas, gt_semantic_seg, + train_cfg): + """Forward function for training. + Args: + inputs (list[Tensor]): List of multi-level img features. + prev_output (Tensor): The output of previous decode head. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + gt_semantic_seg (Tensor): Semantic segmentation masks + used if the architecture supports semantic segmentation task. + train_cfg (dict): The training config. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + x = self._transform_inputs(inputs) + with torch.no_grad(): + points = self.get_points_train( + prev_output, calculate_uncertainty, cfg=train_cfg) + fine_grained_point_feats = self._get_fine_grained_point_feats( + x, points) + coarse_point_feats = self._get_coarse_point_feats(prev_output, points) + point_logits = self.forward(fine_grained_point_feats, + coarse_point_feats) + point_label = point_sample( + gt_semantic_seg.float(), + points, + mode='nearest', + align_corners=self.align_corners) + point_label = point_label.squeeze(1).long() + + losses = self.losses(point_logits, point_label) + + return losses + + def forward_test(self, inputs, prev_output, img_metas, test_cfg): + """Forward function for testing. + + Args: + inputs (list[Tensor]): List of multi-level img features. + prev_output (Tensor): The output of previous decode head. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + test_cfg (dict): The testing config. + + Returns: + Tensor: Output segmentation map. + """ + + x = self._transform_inputs(inputs) + refined_seg_logits = prev_output.clone() + for _ in range(test_cfg.subdivision_steps): + refined_seg_logits = resize( + refined_seg_logits, + scale_factor=test_cfg.scale_factor, + mode='bilinear', + align_corners=self.align_corners) + batch_size, channels, height, width = refined_seg_logits.shape + point_indices, points = self.get_points_test( + refined_seg_logits, calculate_uncertainty, cfg=test_cfg) + fine_grained_point_feats = self._get_fine_grained_point_feats( + x, points) + coarse_point_feats = self._get_coarse_point_feats( + prev_output, points) + point_logits = self.forward(fine_grained_point_feats, + coarse_point_feats) + + point_indices = point_indices.unsqueeze(1).expand(-1, channels, -1) + refined_seg_logits = refined_seg_logits.reshape( + batch_size, channels, height * width) + refined_seg_logits = refined_seg_logits.scatter_( + 2, point_indices, point_logits) + refined_seg_logits = refined_seg_logits.view( + batch_size, channels, height, width) + + return refined_seg_logits + + def losses(self, point_logits, point_label): + """Compute segmentation loss.""" + loss = dict() + loss['loss_point'] = self.loss_decode( + point_logits, point_label, ignore_index=self.ignore_index) + loss['acc_point'] = accuracy(point_logits, point_label) + return loss + + def get_points_train(self, seg_logits, uncertainty_func, cfg): + """Sample points for training. + + Sample points in [0, 1] x [0, 1] coordinate space based on their + uncertainty. The uncertainties are calculated for each point using + 'uncertainty_func' function that takes point's logit prediction as + input. + + Args: + seg_logits (Tensor): Semantic segmentation logits, shape ( + batch_size, num_classes, height, width). + uncertainty_func (func): uncertainty calculation function. + cfg (dict): Training config of point head. + + Returns: + point_coords (Tensor): A tensor of shape (batch_size, num_points, + 2) that contains the coordinates of ``num_points`` sampled + points. + """ + num_points = cfg.num_points + oversample_ratio = cfg.oversample_ratio + importance_sample_ratio = cfg.importance_sample_ratio + assert oversample_ratio >= 1 + assert 0 <= importance_sample_ratio <= 1 + batch_size = seg_logits.shape[0] + num_sampled = int(num_points * oversample_ratio) + point_coords = torch.rand( + batch_size, num_sampled, 2, device=seg_logits.device) + point_logits = point_sample(seg_logits, point_coords) + # It is crucial to calculate uncertainty based on the sampled + # prediction value for the points. Calculating uncertainties of the + # coarse predictions first and sampling them for points leads to + # incorrect results. To illustrate this: assume uncertainty func( + # logits)=-abs(logits), a sampled point between two coarse + # predictions with -1 and 1 logits has 0 logits, and therefore 0 + # uncertainty value. However, if we calculate uncertainties for the + # coarse predictions first, both will have -1 uncertainty, + # and sampled point will get -1 uncertainty. + point_uncertainties = uncertainty_func(point_logits) + num_uncertain_points = int(importance_sample_ratio * num_points) + num_random_points = num_points - num_uncertain_points + idx = torch.topk( + point_uncertainties[:, 0, :], k=num_uncertain_points, dim=1)[1] + shift = num_sampled * torch.arange( + batch_size, dtype=torch.long, device=seg_logits.device) + idx += shift[:, None] + point_coords = point_coords.view(-1, 2)[idx.view(-1), :].view( + batch_size, num_uncertain_points, 2) + if num_random_points > 0: + rand_point_coords = torch.rand( + batch_size, num_random_points, 2, device=seg_logits.device) + point_coords = torch.cat((point_coords, rand_point_coords), dim=1) + return point_coords + + def get_points_test(self, seg_logits, uncertainty_func, cfg): + """Sample points for testing. + + Find ``num_points`` most uncertain points from ``uncertainty_map``. + + Args: + seg_logits (Tensor): A tensor of shape (batch_size, num_classes, + height, width) for class-specific or class-agnostic prediction. + uncertainty_func (func): uncertainty calculation function. + cfg (dict): Testing config of point head. + + Returns: + point_indices (Tensor): A tensor of shape (batch_size, num_points) + that contains indices from [0, height x width) of the most + uncertain points. + point_coords (Tensor): A tensor of shape (batch_size, num_points, + 2) that contains [0, 1] x [0, 1] normalized coordinates of the + most uncertain points from the ``height x width`` grid . + """ + + num_points = cfg.subdivision_num_points + uncertainty_map = uncertainty_func(seg_logits) + batch_size, _, height, width = uncertainty_map.shape + h_step = 1.0 / height + w_step = 1.0 / width + + uncertainty_map = uncertainty_map.view(batch_size, height * width) + num_points = min(height * width, num_points) + point_indices = uncertainty_map.topk(num_points, dim=1)[1] + point_coords = torch.zeros( + batch_size, + num_points, + 2, + dtype=torch.float, + device=seg_logits.device) + point_coords[:, :, 0] = w_step / 2.0 + (point_indices % + width).float() * w_step + point_coords[:, :, 1] = h_step / 2.0 + (point_indices // + width).float() * h_step + return point_indices, point_coords diff --git a/annotator/mmpkg/mmseg/models/decode_heads/psa_head.py b/annotator/mmpkg/mmseg/models/decode_heads/psa_head.py new file mode 100644 index 0000000000000000000000000000000000000000..ba6fe3a8b8f8dc7c4e4d3b9bc09e9642c0b3732f --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/psa_head.py @@ -0,0 +1,199 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from annotator.mmpkg.mmcv.cnn import ConvModule + +from annotator.mmpkg.mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead + +try: + try: + from mmcv.ops import PSAMask + except ImportError: + from annotator.mmpkg.mmcv.ops import PSAMask +except ModuleNotFoundError: + PSAMask = None + + +@HEADS.register_module() +class PSAHead(BaseDecodeHead): + """Point-wise Spatial Attention Network for Scene Parsing. + + This head is the implementation of `PSANet + `_. + + Args: + mask_size (tuple[int]): The PSA mask size. It usually equals input + size. + psa_type (str): The type of psa module. Options are 'collect', + 'distribute', 'bi-direction'. Default: 'bi-direction' + compact (bool): Whether use compact map for 'collect' mode. + Default: True. + shrink_factor (int): The downsample factors of psa mask. Default: 2. + normalization_factor (float): The normalize factor of attention. + psa_softmax (bool): Whether use softmax for attention. + """ + + def __init__(self, + mask_size, + psa_type='bi-direction', + compact=False, + shrink_factor=2, + normalization_factor=1.0, + psa_softmax=True, + **kwargs): + if PSAMask is None: + raise RuntimeError('Please install mmcv-full for PSAMask ops') + super(PSAHead, self).__init__(**kwargs) + assert psa_type in ['collect', 'distribute', 'bi-direction'] + self.psa_type = psa_type + self.compact = compact + self.shrink_factor = shrink_factor + self.mask_size = mask_size + mask_h, mask_w = mask_size + self.psa_softmax = psa_softmax + if normalization_factor is None: + normalization_factor = mask_h * mask_w + self.normalization_factor = normalization_factor + + self.reduce = ConvModule( + self.in_channels, + self.channels, + kernel_size=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.attention = nn.Sequential( + ConvModule( + self.channels, + self.channels, + kernel_size=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg), + nn.Conv2d( + self.channels, mask_h * mask_w, kernel_size=1, bias=False)) + if psa_type == 'bi-direction': + self.reduce_p = ConvModule( + self.in_channels, + self.channels, + kernel_size=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.attention_p = nn.Sequential( + ConvModule( + self.channels, + self.channels, + kernel_size=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg), + nn.Conv2d( + self.channels, mask_h * mask_w, kernel_size=1, bias=False)) + self.psamask_collect = PSAMask('collect', mask_size) + self.psamask_distribute = PSAMask('distribute', mask_size) + else: + self.psamask = PSAMask(psa_type, mask_size) + self.proj = ConvModule( + self.channels * (2 if psa_type == 'bi-direction' else 1), + self.in_channels, + kernel_size=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.bottleneck = ConvModule( + self.in_channels * 2, + self.channels, + kernel_size=3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + identity = x + align_corners = self.align_corners + if self.psa_type in ['collect', 'distribute']: + out = self.reduce(x) + n, c, h, w = out.size() + if self.shrink_factor != 1: + if h % self.shrink_factor and w % self.shrink_factor: + h = (h - 1) // self.shrink_factor + 1 + w = (w - 1) // self.shrink_factor + 1 + align_corners = True + else: + h = h // self.shrink_factor + w = w // self.shrink_factor + align_corners = False + out = resize( + out, + size=(h, w), + mode='bilinear', + align_corners=align_corners) + y = self.attention(out) + if self.compact: + if self.psa_type == 'collect': + y = y.view(n, h * w, + h * w).transpose(1, 2).view(n, h * w, h, w) + else: + y = self.psamask(y) + if self.psa_softmax: + y = F.softmax(y, dim=1) + out = torch.bmm( + out.view(n, c, h * w), y.view(n, h * w, h * w)).view( + n, c, h, w) * (1.0 / self.normalization_factor) + else: + x_col = self.reduce(x) + x_dis = self.reduce_p(x) + n, c, h, w = x_col.size() + if self.shrink_factor != 1: + if h % self.shrink_factor and w % self.shrink_factor: + h = (h - 1) // self.shrink_factor + 1 + w = (w - 1) // self.shrink_factor + 1 + align_corners = True + else: + h = h // self.shrink_factor + w = w // self.shrink_factor + align_corners = False + x_col = resize( + x_col, + size=(h, w), + mode='bilinear', + align_corners=align_corners) + x_dis = resize( + x_dis, + size=(h, w), + mode='bilinear', + align_corners=align_corners) + y_col = self.attention(x_col) + y_dis = self.attention_p(x_dis) + if self.compact: + y_dis = y_dis.view(n, h * w, + h * w).transpose(1, 2).view(n, h * w, h, w) + else: + y_col = self.psamask_collect(y_col) + y_dis = self.psamask_distribute(y_dis) + if self.psa_softmax: + y_col = F.softmax(y_col, dim=1) + y_dis = F.softmax(y_dis, dim=1) + x_col = torch.bmm( + x_col.view(n, c, h * w), y_col.view(n, h * w, h * w)).view( + n, c, h, w) * (1.0 / self.normalization_factor) + x_dis = torch.bmm( + x_dis.view(n, c, h * w), y_dis.view(n, h * w, h * w)).view( + n, c, h, w) * (1.0 / self.normalization_factor) + out = torch.cat([x_col, x_dis], 1) + out = self.proj(out) + out = resize( + out, + size=identity.shape[2:], + mode='bilinear', + align_corners=align_corners) + out = self.bottleneck(torch.cat((identity, out), dim=1)) + out = self.cls_seg(out) + return out diff --git a/annotator/mmpkg/mmseg/models/decode_heads/psp_head.py b/annotator/mmpkg/mmseg/models/decode_heads/psp_head.py new file mode 100644 index 0000000000000000000000000000000000000000..2a88d807bfe11fe224305f8de745cde3aa739db0 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/psp_head.py @@ -0,0 +1,101 @@ +import torch +import torch.nn as nn +from annotator.mmpkg.mmcv.cnn import ConvModule + +from annotator.mmpkg.mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +class PPM(nn.ModuleList): + """Pooling Pyramid Module used in PSPNet. + + Args: + pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module. + in_channels (int): Input channels. + channels (int): Channels after modules, before conv_seg. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict): Config of activation layers. + align_corners (bool): align_corners argument of F.interpolate. + """ + + def __init__(self, pool_scales, in_channels, channels, conv_cfg, norm_cfg, + act_cfg, align_corners): + super(PPM, self).__init__() + self.pool_scales = pool_scales + self.align_corners = align_corners + self.in_channels = in_channels + self.channels = channels + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + for pool_scale in pool_scales: + self.append( + nn.Sequential( + nn.AdaptiveAvgPool2d(pool_scale), + ConvModule( + self.in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg))) + + def forward(self, x): + """Forward function.""" + ppm_outs = [] + for ppm in self: + ppm_out = ppm(x) + upsampled_ppm_out = resize( + ppm_out, + size=x.size()[2:], + mode='bilinear', + align_corners=self.align_corners) + ppm_outs.append(upsampled_ppm_out) + return ppm_outs + + +@HEADS.register_module() +class PSPHead(BaseDecodeHead): + """Pyramid Scene Parsing Network. + + This head is the implementation of + `PSPNet `_. + + Args: + pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module. Default: (1, 2, 3, 6). + """ + + def __init__(self, pool_scales=(1, 2, 3, 6), **kwargs): + super(PSPHead, self).__init__(**kwargs) + assert isinstance(pool_scales, (list, tuple)) + self.pool_scales = pool_scales + self.psp_modules = PPM( + self.pool_scales, + self.in_channels, + self.channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + align_corners=self.align_corners) + self.bottleneck = ConvModule( + self.in_channels + len(pool_scales) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + psp_outs = [x] + psp_outs.extend(self.psp_modules(x)) + psp_outs = torch.cat(psp_outs, dim=1) + output = self.bottleneck(psp_outs) + output = self.cls_seg(output) + return output diff --git a/annotator/mmpkg/mmseg/models/decode_heads/sep_aspp_head.py b/annotator/mmpkg/mmseg/models/decode_heads/sep_aspp_head.py new file mode 100644 index 0000000000000000000000000000000000000000..a23970699df7afd86f483316be3c8d1a34d43c18 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/sep_aspp_head.py @@ -0,0 +1,101 @@ +import torch +import torch.nn as nn +from annotator.mmpkg.mmcv.cnn import ConvModule, DepthwiseSeparableConvModule + +from annotator.mmpkg.mmseg.ops import resize +from ..builder import HEADS +from .aspp_head import ASPPHead, ASPPModule + + +class DepthwiseSeparableASPPModule(ASPPModule): + """Atrous Spatial Pyramid Pooling (ASPP) Module with depthwise separable + conv.""" + + def __init__(self, **kwargs): + super(DepthwiseSeparableASPPModule, self).__init__(**kwargs) + for i, dilation in enumerate(self.dilations): + if dilation > 1: + self[i] = DepthwiseSeparableConvModule( + self.in_channels, + self.channels, + 3, + dilation=dilation, + padding=dilation, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + +@HEADS.register_module() +class DepthwiseSeparableASPPHead(ASPPHead): + """Encoder-Decoder with Atrous Separable Convolution for Semantic Image + Segmentation. + + This head is the implementation of `DeepLabV3+ + `_. + + Args: + c1_in_channels (int): The input channels of c1 decoder. If is 0, + the no decoder will be used. + c1_channels (int): The intermediate channels of c1 decoder. + """ + + def __init__(self, c1_in_channels, c1_channels, **kwargs): + super(DepthwiseSeparableASPPHead, self).__init__(**kwargs) + assert c1_in_channels >= 0 + self.aspp_modules = DepthwiseSeparableASPPModule( + dilations=self.dilations, + in_channels=self.in_channels, + channels=self.channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + if c1_in_channels > 0: + self.c1_bottleneck = ConvModule( + c1_in_channels, + c1_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + else: + self.c1_bottleneck = None + self.sep_bottleneck = nn.Sequential( + DepthwiseSeparableConvModule( + self.channels + c1_channels, + self.channels, + 3, + padding=1, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg), + DepthwiseSeparableConvModule( + self.channels, + self.channels, + 3, + padding=1, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + aspp_outs = [ + resize( + self.image_pool(x), + size=x.size()[2:], + mode='bilinear', + align_corners=self.align_corners) + ] + aspp_outs.extend(self.aspp_modules(x)) + aspp_outs = torch.cat(aspp_outs, dim=1) + output = self.bottleneck(aspp_outs) + if self.c1_bottleneck is not None: + c1_output = self.c1_bottleneck(inputs[0]) + output = resize( + input=output, + size=c1_output.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + output = torch.cat([output, c1_output], dim=1) + output = self.sep_bottleneck(output) + output = self.cls_seg(output) + return output diff --git a/annotator/mmpkg/mmseg/models/decode_heads/sep_fcn_head.py b/annotator/mmpkg/mmseg/models/decode_heads/sep_fcn_head.py new file mode 100644 index 0000000000000000000000000000000000000000..3ea198ab8a96919dfb6974fd73b1476aa488aef2 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/sep_fcn_head.py @@ -0,0 +1,51 @@ +from annotator.mmpkg.mmcv.cnn import DepthwiseSeparableConvModule + +from ..builder import HEADS +from .fcn_head import FCNHead + + +@HEADS.register_module() +class DepthwiseSeparableFCNHead(FCNHead): + """Depthwise-Separable Fully Convolutional Network for Semantic + Segmentation. + + This head is implemented according to Fast-SCNN paper. + Args: + in_channels(int): Number of output channels of FFM. + channels(int): Number of middle-stage channels in the decode head. + concat_input(bool): Whether to concatenate original decode input into + the result of several consecutive convolution layers. + Default: True. + num_classes(int): Used to determine the dimension of + final prediction tensor. + in_index(int): Correspond with 'out_indices' in FastSCNN backbone. + norm_cfg (dict | None): Config of norm layers. + align_corners (bool): align_corners argument of F.interpolate. + Default: False. + loss_decode(dict): Config of loss type and some + relevant additional options. + """ + + def __init__(self, **kwargs): + super(DepthwiseSeparableFCNHead, self).__init__(**kwargs) + self.convs[0] = DepthwiseSeparableConvModule( + self.in_channels, + self.channels, + kernel_size=self.kernel_size, + padding=self.kernel_size // 2, + norm_cfg=self.norm_cfg) + for i in range(1, self.num_convs): + self.convs[i] = DepthwiseSeparableConvModule( + self.channels, + self.channels, + kernel_size=self.kernel_size, + padding=self.kernel_size // 2, + norm_cfg=self.norm_cfg) + + if self.concat_input: + self.conv_cat = DepthwiseSeparableConvModule( + self.in_channels + self.channels, + self.channels, + kernel_size=self.kernel_size, + padding=self.kernel_size // 2, + norm_cfg=self.norm_cfg) diff --git a/annotator/mmpkg/mmseg/models/decode_heads/uper_head.py b/annotator/mmpkg/mmseg/models/decode_heads/uper_head.py new file mode 100644 index 0000000000000000000000000000000000000000..952473578c1f5b903f5fc7f9d13a4e40ea5dec87 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/decode_heads/uper_head.py @@ -0,0 +1,126 @@ +import torch +import torch.nn as nn +from annotator.mmpkg.mmcv.cnn import ConvModule + +from annotator.mmpkg.mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead +from .psp_head import PPM + + +@HEADS.register_module() +class UPerHead(BaseDecodeHead): + """Unified Perceptual Parsing for Scene Understanding. + + This head is the implementation of `UPerNet + `_. + + Args: + pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module applied on the last feature. Default: (1, 2, 3, 6). + """ + + def __init__(self, pool_scales=(1, 2, 3, 6), **kwargs): + super(UPerHead, self).__init__( + input_transform='multiple_select', **kwargs) + # PSP Module + self.psp_modules = PPM( + pool_scales, + self.in_channels[-1], + self.channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + align_corners=self.align_corners) + self.bottleneck = ConvModule( + self.in_channels[-1] + len(pool_scales) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + # FPN Module + self.lateral_convs = nn.ModuleList() + self.fpn_convs = nn.ModuleList() + for in_channels in self.in_channels[:-1]: # skip the top layer + l_conv = ConvModule( + in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + inplace=False) + fpn_conv = ConvModule( + self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + inplace=False) + self.lateral_convs.append(l_conv) + self.fpn_convs.append(fpn_conv) + + self.fpn_bottleneck = ConvModule( + len(self.in_channels) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def psp_forward(self, inputs): + """Forward function of PSP module.""" + x = inputs[-1] + psp_outs = [x] + psp_outs.extend(self.psp_modules(x)) + psp_outs = torch.cat(psp_outs, dim=1) + output = self.bottleneck(psp_outs) + + return output + + def forward(self, inputs): + """Forward function.""" + + inputs = self._transform_inputs(inputs) + + # build laterals + laterals = [ + lateral_conv(inputs[i]) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + + laterals.append(self.psp_forward(inputs)) + + # build top-down path + used_backbone_levels = len(laterals) + for i in range(used_backbone_levels - 1, 0, -1): + prev_shape = laterals[i - 1].shape[2:] + laterals[i - 1] += resize( + laterals[i], + size=prev_shape, + mode='bilinear', + align_corners=self.align_corners) + + # build outputs + fpn_outs = [ + self.fpn_convs[i](laterals[i]) + for i in range(used_backbone_levels - 1) + ] + # append psp feature + fpn_outs.append(laterals[-1]) + + for i in range(used_backbone_levels - 1, 0, -1): + fpn_outs[i] = resize( + fpn_outs[i], + size=fpn_outs[0].shape[2:], + mode='bilinear', + align_corners=self.align_corners) + fpn_outs = torch.cat(fpn_outs, dim=1) + output = self.fpn_bottleneck(fpn_outs) + output = self.cls_seg(output) + return output diff --git a/annotator/mmpkg/mmseg/models/losses/__init__.py b/annotator/mmpkg/mmseg/models/losses/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..beca72045694273d63465bac2f27dbc6672271db --- /dev/null +++ b/annotator/mmpkg/mmseg/models/losses/__init__.py @@ -0,0 +1,12 @@ +from .accuracy import Accuracy, accuracy +from .cross_entropy_loss import (CrossEntropyLoss, binary_cross_entropy, + cross_entropy, mask_cross_entropy) +from .dice_loss import DiceLoss +from .lovasz_loss import LovaszLoss +from .utils import reduce_loss, weight_reduce_loss, weighted_loss + +__all__ = [ + 'accuracy', 'Accuracy', 'cross_entropy', 'binary_cross_entropy', + 'mask_cross_entropy', 'CrossEntropyLoss', 'reduce_loss', + 'weight_reduce_loss', 'weighted_loss', 'LovaszLoss', 'DiceLoss' +] diff --git a/annotator/mmpkg/mmseg/models/losses/accuracy.py b/annotator/mmpkg/mmseg/models/losses/accuracy.py new file mode 100644 index 0000000000000000000000000000000000000000..c0fd2e7e74a0f721c4a814c09d6e453e5956bb38 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/losses/accuracy.py @@ -0,0 +1,78 @@ +import torch.nn as nn + + +def accuracy(pred, target, topk=1, thresh=None): + """Calculate accuracy according to the prediction and target. + + Args: + pred (torch.Tensor): The model prediction, shape (N, num_class, ...) + target (torch.Tensor): The target of each prediction, shape (N, , ...) + topk (int | tuple[int], optional): If the predictions in ``topk`` + matches the target, the predictions will be regarded as + correct ones. Defaults to 1. + thresh (float, optional): If not None, predictions with scores under + this threshold are considered incorrect. Default to None. + + Returns: + float | tuple[float]: If the input ``topk`` is a single integer, + the function will return a single float as accuracy. If + ``topk`` is a tuple containing multiple integers, the + function will return a tuple containing accuracies of + each ``topk`` number. + """ + assert isinstance(topk, (int, tuple)) + if isinstance(topk, int): + topk = (topk, ) + return_single = True + else: + return_single = False + + maxk = max(topk) + if pred.size(0) == 0: + accu = [pred.new_tensor(0.) for i in range(len(topk))] + return accu[0] if return_single else accu + assert pred.ndim == target.ndim + 1 + assert pred.size(0) == target.size(0) + assert maxk <= pred.size(1), \ + f'maxk {maxk} exceeds pred dimension {pred.size(1)}' + pred_value, pred_label = pred.topk(maxk, dim=1) + # transpose to shape (maxk, N, ...) + pred_label = pred_label.transpose(0, 1) + correct = pred_label.eq(target.unsqueeze(0).expand_as(pred_label)) + if thresh is not None: + # Only prediction values larger than thresh are counted as correct + correct = correct & (pred_value > thresh).t() + res = [] + for k in topk: + correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / target.numel())) + return res[0] if return_single else res + + +class Accuracy(nn.Module): + """Accuracy calculation module.""" + + def __init__(self, topk=(1, ), thresh=None): + """Module to calculate the accuracy. + + Args: + topk (tuple, optional): The criterion used to calculate the + accuracy. Defaults to (1,). + thresh (float, optional): If not None, predictions with scores + under this threshold are considered incorrect. Default to None. + """ + super().__init__() + self.topk = topk + self.thresh = thresh + + def forward(self, pred, target): + """Forward function to calculate accuracy. + + Args: + pred (torch.Tensor): Prediction of models. + target (torch.Tensor): Target for each prediction. + + Returns: + tuple[float]: The accuracies under different topk criterions. + """ + return accuracy(pred, target, self.topk, self.thresh) diff --git a/annotator/mmpkg/mmseg/models/losses/cross_entropy_loss.py b/annotator/mmpkg/mmseg/models/losses/cross_entropy_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..42c0790c98616bb69621deed55547fc04c7392ef --- /dev/null +++ b/annotator/mmpkg/mmseg/models/losses/cross_entropy_loss.py @@ -0,0 +1,198 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..builder import LOSSES +from .utils import get_class_weight, weight_reduce_loss + + +def cross_entropy(pred, + label, + weight=None, + class_weight=None, + reduction='mean', + avg_factor=None, + ignore_index=-100): + """The wrapper function for :func:`F.cross_entropy`""" + # class_weight is a manual rescaling weight given to each class. + # If given, has to be a Tensor of size C element-wise losses + loss = F.cross_entropy( + pred, + label, + weight=class_weight, + reduction='none', + ignore_index=ignore_index) + + # apply weights and do the reduction + if weight is not None: + weight = weight.float() + loss = weight_reduce_loss( + loss, weight=weight, reduction=reduction, avg_factor=avg_factor) + + return loss + + +def _expand_onehot_labels(labels, label_weights, target_shape, ignore_index): + """Expand onehot labels to match the size of prediction.""" + bin_labels = labels.new_zeros(target_shape) + valid_mask = (labels >= 0) & (labels != ignore_index) + inds = torch.nonzero(valid_mask, as_tuple=True) + + if inds[0].numel() > 0: + if labels.dim() == 3: + bin_labels[inds[0], labels[valid_mask], inds[1], inds[2]] = 1 + else: + bin_labels[inds[0], labels[valid_mask]] = 1 + + valid_mask = valid_mask.unsqueeze(1).expand(target_shape).float() + if label_weights is None: + bin_label_weights = valid_mask + else: + bin_label_weights = label_weights.unsqueeze(1).expand(target_shape) + bin_label_weights *= valid_mask + + return bin_labels, bin_label_weights + + +def binary_cross_entropy(pred, + label, + weight=None, + reduction='mean', + avg_factor=None, + class_weight=None, + ignore_index=255): + """Calculate the binary CrossEntropy loss. + + Args: + pred (torch.Tensor): The prediction with shape (N, 1). + label (torch.Tensor): The learning label of the prediction. + weight (torch.Tensor, optional): Sample-wise loss weight. + reduction (str, optional): The method used to reduce the loss. + Options are "none", "mean" and "sum". + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + class_weight (list[float], optional): The weight for each class. + ignore_index (int | None): The label index to be ignored. Default: 255 + + Returns: + torch.Tensor: The calculated loss + """ + if pred.dim() != label.dim(): + assert (pred.dim() == 2 and label.dim() == 1) or ( + pred.dim() == 4 and label.dim() == 3), \ + 'Only pred shape [N, C], label shape [N] or pred shape [N, C, ' \ + 'H, W], label shape [N, H, W] are supported' + label, weight = _expand_onehot_labels(label, weight, pred.shape, + ignore_index) + + # weighted element-wise losses + if weight is not None: + weight = weight.float() + loss = F.binary_cross_entropy_with_logits( + pred, label.float(), pos_weight=class_weight, reduction='none') + # do the reduction for the weighted loss + loss = weight_reduce_loss( + loss, weight, reduction=reduction, avg_factor=avg_factor) + + return loss + + +def mask_cross_entropy(pred, + target, + label, + reduction='mean', + avg_factor=None, + class_weight=None, + ignore_index=None): + """Calculate the CrossEntropy loss for masks. + + Args: + pred (torch.Tensor): The prediction with shape (N, C), C is the number + of classes. + target (torch.Tensor): The learning label of the prediction. + label (torch.Tensor): ``label`` indicates the class label of the mask' + corresponding object. This will be used to select the mask in the + of the class which the object belongs to when the mask prediction + if not class-agnostic. + reduction (str, optional): The method used to reduce the loss. + Options are "none", "mean" and "sum". + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + class_weight (list[float], optional): The weight for each class. + ignore_index (None): Placeholder, to be consistent with other loss. + Default: None. + + Returns: + torch.Tensor: The calculated loss + """ + assert ignore_index is None, 'BCE loss does not support ignore_index' + # TODO: handle these two reserved arguments + assert reduction == 'mean' and avg_factor is None + num_rois = pred.size()[0] + inds = torch.arange(0, num_rois, dtype=torch.long, device=pred.device) + pred_slice = pred[inds, label].squeeze(1) + return F.binary_cross_entropy_with_logits( + pred_slice, target, weight=class_weight, reduction='mean')[None] + + +@LOSSES.register_module() +class CrossEntropyLoss(nn.Module): + """CrossEntropyLoss. + + Args: + use_sigmoid (bool, optional): Whether the prediction uses sigmoid + of softmax. Defaults to False. + use_mask (bool, optional): Whether to use mask cross entropy loss. + Defaults to False. + reduction (str, optional): . Defaults to 'mean'. + Options are "none", "mean" and "sum". + class_weight (list[float] | str, optional): Weight of each class. If in + str format, read them from a file. Defaults to None. + loss_weight (float, optional): Weight of the loss. Defaults to 1.0. + """ + + def __init__(self, + use_sigmoid=False, + use_mask=False, + reduction='mean', + class_weight=None, + loss_weight=1.0): + super(CrossEntropyLoss, self).__init__() + assert (use_sigmoid is False) or (use_mask is False) + self.use_sigmoid = use_sigmoid + self.use_mask = use_mask + self.reduction = reduction + self.loss_weight = loss_weight + self.class_weight = get_class_weight(class_weight) + + if self.use_sigmoid: + self.cls_criterion = binary_cross_entropy + elif self.use_mask: + self.cls_criterion = mask_cross_entropy + else: + self.cls_criterion = cross_entropy + + def forward(self, + cls_score, + label, + weight=None, + avg_factor=None, + reduction_override=None, + **kwargs): + """Forward function.""" + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + if self.class_weight is not None: + class_weight = cls_score.new_tensor(self.class_weight) + else: + class_weight = None + loss_cls = self.loss_weight * self.cls_criterion( + cls_score, + label, + weight, + class_weight=class_weight, + reduction=reduction, + avg_factor=avg_factor, + **kwargs) + return loss_cls diff --git a/annotator/mmpkg/mmseg/models/losses/dice_loss.py b/annotator/mmpkg/mmseg/models/losses/dice_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..27a77b962d7d8b3079c7d6cd9db52280c6fb4970 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/losses/dice_loss.py @@ -0,0 +1,119 @@ +"""Modified from https://github.com/LikeLy-Journey/SegmenTron/blob/master/ +segmentron/solver/loss.py (Apache-2.0 License)""" +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..builder import LOSSES +from .utils import get_class_weight, weighted_loss + + +@weighted_loss +def dice_loss(pred, + target, + valid_mask, + smooth=1, + exponent=2, + class_weight=None, + ignore_index=255): + assert pred.shape[0] == target.shape[0] + total_loss = 0 + num_classes = pred.shape[1] + for i in range(num_classes): + if i != ignore_index: + dice_loss = binary_dice_loss( + pred[:, i], + target[..., i], + valid_mask=valid_mask, + smooth=smooth, + exponent=exponent) + if class_weight is not None: + dice_loss *= class_weight[i] + total_loss += dice_loss + return total_loss / num_classes + + +@weighted_loss +def binary_dice_loss(pred, target, valid_mask, smooth=1, exponent=2, **kwards): + assert pred.shape[0] == target.shape[0] + pred = pred.reshape(pred.shape[0], -1) + target = target.reshape(target.shape[0], -1) + valid_mask = valid_mask.reshape(valid_mask.shape[0], -1) + + num = torch.sum(torch.mul(pred, target) * valid_mask, dim=1) * 2 + smooth + den = torch.sum(pred.pow(exponent) + target.pow(exponent), dim=1) + smooth + + return 1 - num / den + + +@LOSSES.register_module() +class DiceLoss(nn.Module): + """DiceLoss. + + This loss is proposed in `V-Net: Fully Convolutional Neural Networks for + Volumetric Medical Image Segmentation `_. + + Args: + loss_type (str, optional): Binary or multi-class loss. + Default: 'multi_class'. Options are "binary" and "multi_class". + smooth (float): A float number to smooth loss, and avoid NaN error. + Default: 1 + exponent (float): An float number to calculate denominator + value: \\sum{x^exponent} + \\sum{y^exponent}. Default: 2. + reduction (str, optional): The method used to reduce the loss. Options + are "none", "mean" and "sum". This parameter only works when + per_image is True. Default: 'mean'. + class_weight (list[float] | str, optional): Weight of each class. If in + str format, read them from a file. Defaults to None. + loss_weight (float, optional): Weight of the loss. Default to 1.0. + ignore_index (int | None): The label index to be ignored. Default: 255. + """ + + def __init__(self, + smooth=1, + exponent=2, + reduction='mean', + class_weight=None, + loss_weight=1.0, + ignore_index=255, + **kwards): + super(DiceLoss, self).__init__() + self.smooth = smooth + self.exponent = exponent + self.reduction = reduction + self.class_weight = get_class_weight(class_weight) + self.loss_weight = loss_weight + self.ignore_index = ignore_index + + def forward(self, + pred, + target, + avg_factor=None, + reduction_override=None, + **kwards): + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + if self.class_weight is not None: + class_weight = pred.new_tensor(self.class_weight) + else: + class_weight = None + + pred = F.softmax(pred, dim=1) + num_classes = pred.shape[1] + one_hot_target = F.one_hot( + torch.clamp(target.long(), 0, num_classes - 1), + num_classes=num_classes) + valid_mask = (target != self.ignore_index).long() + + loss = self.loss_weight * dice_loss( + pred, + one_hot_target, + valid_mask=valid_mask, + reduction=reduction, + avg_factor=avg_factor, + smooth=self.smooth, + exponent=self.exponent, + class_weight=class_weight, + ignore_index=self.ignore_index) + return loss diff --git a/annotator/mmpkg/mmseg/models/losses/lovasz_loss.py b/annotator/mmpkg/mmseg/models/losses/lovasz_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..50f0f70fd432316b081a0114c28df61d320b5a47 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/losses/lovasz_loss.py @@ -0,0 +1,303 @@ +"""Modified from https://github.com/bermanmaxim/LovaszSoftmax/blob/master/pytor +ch/lovasz_losses.py Lovasz-Softmax and Jaccard hinge loss in PyTorch Maxim +Berman 2018 ESAT-PSI KU Leuven (MIT License)""" + +import annotator.mmpkg.mmcv as mmcv +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..builder import LOSSES +from .utils import get_class_weight, weight_reduce_loss + + +def lovasz_grad(gt_sorted): + """Computes gradient of the Lovasz extension w.r.t sorted errors. + + See Alg. 1 in paper. + """ + p = len(gt_sorted) + gts = gt_sorted.sum() + intersection = gts - gt_sorted.float().cumsum(0) + union = gts + (1 - gt_sorted).float().cumsum(0) + jaccard = 1. - intersection / union + if p > 1: # cover 1-pixel case + jaccard[1:p] = jaccard[1:p] - jaccard[0:-1] + return jaccard + + +def flatten_binary_logits(logits, labels, ignore_index=None): + """Flattens predictions in the batch (binary case) Remove labels equal to + 'ignore_index'.""" + logits = logits.view(-1) + labels = labels.view(-1) + if ignore_index is None: + return logits, labels + valid = (labels != ignore_index) + vlogits = logits[valid] + vlabels = labels[valid] + return vlogits, vlabels + + +def flatten_probs(probs, labels, ignore_index=None): + """Flattens predictions in the batch.""" + if probs.dim() == 3: + # assumes output of a sigmoid layer + B, H, W = probs.size() + probs = probs.view(B, 1, H, W) + B, C, H, W = probs.size() + probs = probs.permute(0, 2, 3, 1).contiguous().view(-1, C) # B*H*W, C=P,C + labels = labels.view(-1) + if ignore_index is None: + return probs, labels + valid = (labels != ignore_index) + vprobs = probs[valid.nonzero().squeeze()] + vlabels = labels[valid] + return vprobs, vlabels + + +def lovasz_hinge_flat(logits, labels): + """Binary Lovasz hinge loss. + + Args: + logits (torch.Tensor): [P], logits at each prediction + (between -infty and +infty). + labels (torch.Tensor): [P], binary ground truth labels (0 or 1). + + Returns: + torch.Tensor: The calculated loss. + """ + if len(labels) == 0: + # only void pixels, the gradients should be 0 + return logits.sum() * 0. + signs = 2. * labels.float() - 1. + errors = (1. - logits * signs) + errors_sorted, perm = torch.sort(errors, dim=0, descending=True) + perm = perm.data + gt_sorted = labels[perm] + grad = lovasz_grad(gt_sorted) + loss = torch.dot(F.relu(errors_sorted), grad) + return loss + + +def lovasz_hinge(logits, + labels, + classes='present', + per_image=False, + class_weight=None, + reduction='mean', + avg_factor=None, + ignore_index=255): + """Binary Lovasz hinge loss. + + Args: + logits (torch.Tensor): [B, H, W], logits at each pixel + (between -infty and +infty). + labels (torch.Tensor): [B, H, W], binary ground truth masks (0 or 1). + classes (str | list[int], optional): Placeholder, to be consistent with + other loss. Default: None. + per_image (bool, optional): If per_image is True, compute the loss per + image instead of per batch. Default: False. + class_weight (list[float], optional): Placeholder, to be consistent + with other loss. Default: None. + reduction (str, optional): The method used to reduce the loss. Options + are "none", "mean" and "sum". This parameter only works when + per_image is True. Default: 'mean'. + avg_factor (int, optional): Average factor that is used to average + the loss. This parameter only works when per_image is True. + Default: None. + ignore_index (int | None): The label index to be ignored. Default: 255. + + Returns: + torch.Tensor: The calculated loss. + """ + if per_image: + loss = [ + lovasz_hinge_flat(*flatten_binary_logits( + logit.unsqueeze(0), label.unsqueeze(0), ignore_index)) + for logit, label in zip(logits, labels) + ] + loss = weight_reduce_loss( + torch.stack(loss), None, reduction, avg_factor) + else: + loss = lovasz_hinge_flat( + *flatten_binary_logits(logits, labels, ignore_index)) + return loss + + +def lovasz_softmax_flat(probs, labels, classes='present', class_weight=None): + """Multi-class Lovasz-Softmax loss. + + Args: + probs (torch.Tensor): [P, C], class probabilities at each prediction + (between 0 and 1). + labels (torch.Tensor): [P], ground truth labels (between 0 and C - 1). + classes (str | list[int], optional): Classes chosen to calculate loss. + 'all' for all classes, 'present' for classes present in labels, or + a list of classes to average. Default: 'present'. + class_weight (list[float], optional): The weight for each class. + Default: None. + + Returns: + torch.Tensor: The calculated loss. + """ + if probs.numel() == 0: + # only void pixels, the gradients should be 0 + return probs * 0. + C = probs.size(1) + losses = [] + class_to_sum = list(range(C)) if classes in ['all', 'present'] else classes + for c in class_to_sum: + fg = (labels == c).float() # foreground for class c + if (classes == 'present' and fg.sum() == 0): + continue + if C == 1: + if len(classes) > 1: + raise ValueError('Sigmoid output possible only with 1 class') + class_pred = probs[:, 0] + else: + class_pred = probs[:, c] + errors = (fg - class_pred).abs() + errors_sorted, perm = torch.sort(errors, 0, descending=True) + perm = perm.data + fg_sorted = fg[perm] + loss = torch.dot(errors_sorted, lovasz_grad(fg_sorted)) + if class_weight is not None: + loss *= class_weight[c] + losses.append(loss) + return torch.stack(losses).mean() + + +def lovasz_softmax(probs, + labels, + classes='present', + per_image=False, + class_weight=None, + reduction='mean', + avg_factor=None, + ignore_index=255): + """Multi-class Lovasz-Softmax loss. + + Args: + probs (torch.Tensor): [B, C, H, W], class probabilities at each + prediction (between 0 and 1). + labels (torch.Tensor): [B, H, W], ground truth labels (between 0 and + C - 1). + classes (str | list[int], optional): Classes chosen to calculate loss. + 'all' for all classes, 'present' for classes present in labels, or + a list of classes to average. Default: 'present'. + per_image (bool, optional): If per_image is True, compute the loss per + image instead of per batch. Default: False. + class_weight (list[float], optional): The weight for each class. + Default: None. + reduction (str, optional): The method used to reduce the loss. Options + are "none", "mean" and "sum". This parameter only works when + per_image is True. Default: 'mean'. + avg_factor (int, optional): Average factor that is used to average + the loss. This parameter only works when per_image is True. + Default: None. + ignore_index (int | None): The label index to be ignored. Default: 255. + + Returns: + torch.Tensor: The calculated loss. + """ + + if per_image: + loss = [ + lovasz_softmax_flat( + *flatten_probs( + prob.unsqueeze(0), label.unsqueeze(0), ignore_index), + classes=classes, + class_weight=class_weight) + for prob, label in zip(probs, labels) + ] + loss = weight_reduce_loss( + torch.stack(loss), None, reduction, avg_factor) + else: + loss = lovasz_softmax_flat( + *flatten_probs(probs, labels, ignore_index), + classes=classes, + class_weight=class_weight) + return loss + + +@LOSSES.register_module() +class LovaszLoss(nn.Module): + """LovaszLoss. + + This loss is proposed in `The Lovasz-Softmax loss: A tractable surrogate + for the optimization of the intersection-over-union measure in neural + networks `_. + + Args: + loss_type (str, optional): Binary or multi-class loss. + Default: 'multi_class'. Options are "binary" and "multi_class". + classes (str | list[int], optional): Classes chosen to calculate loss. + 'all' for all classes, 'present' for classes present in labels, or + a list of classes to average. Default: 'present'. + per_image (bool, optional): If per_image is True, compute the loss per + image instead of per batch. Default: False. + reduction (str, optional): The method used to reduce the loss. Options + are "none", "mean" and "sum". This parameter only works when + per_image is True. Default: 'mean'. + class_weight (list[float] | str, optional): Weight of each class. If in + str format, read them from a file. Defaults to None. + loss_weight (float, optional): Weight of the loss. Defaults to 1.0. + """ + + def __init__(self, + loss_type='multi_class', + classes='present', + per_image=False, + reduction='mean', + class_weight=None, + loss_weight=1.0): + super(LovaszLoss, self).__init__() + assert loss_type in ('binary', 'multi_class'), "loss_type should be \ + 'binary' or 'multi_class'." + + if loss_type == 'binary': + self.cls_criterion = lovasz_hinge + else: + self.cls_criterion = lovasz_softmax + assert classes in ('all', 'present') or mmcv.is_list_of(classes, int) + if not per_image: + assert reduction == 'none', "reduction should be 'none' when \ + per_image is False." + + self.classes = classes + self.per_image = per_image + self.reduction = reduction + self.loss_weight = loss_weight + self.class_weight = get_class_weight(class_weight) + + def forward(self, + cls_score, + label, + weight=None, + avg_factor=None, + reduction_override=None, + **kwargs): + """Forward function.""" + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + if self.class_weight is not None: + class_weight = cls_score.new_tensor(self.class_weight) + else: + class_weight = None + + # if multi-class loss, transform logits to probs + if self.cls_criterion == lovasz_softmax: + cls_score = F.softmax(cls_score, dim=1) + + loss_cls = self.loss_weight * self.cls_criterion( + cls_score, + label, + self.classes, + self.per_image, + class_weight=class_weight, + reduction=reduction, + avg_factor=avg_factor, + **kwargs) + return loss_cls diff --git a/annotator/mmpkg/mmseg/models/losses/utils.py b/annotator/mmpkg/mmseg/models/losses/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2afb477a153ba9dead71066fa66ee024482afd82 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/losses/utils.py @@ -0,0 +1,121 @@ +import functools + +import annotator.mmpkg.mmcv as mmcv +import numpy as np +import torch.nn.functional as F + + +def get_class_weight(class_weight): + """Get class weight for loss function. + + Args: + class_weight (list[float] | str | None): If class_weight is a str, + take it as a file name and read from it. + """ + if isinstance(class_weight, str): + # take it as a file path + if class_weight.endswith('.npy'): + class_weight = np.load(class_weight) + else: + # pkl, json or yaml + class_weight = mmcv.load(class_weight) + + return class_weight + + +def reduce_loss(loss, reduction): + """Reduce loss as specified. + + Args: + loss (Tensor): Elementwise loss tensor. + reduction (str): Options are "none", "mean" and "sum". + + Return: + Tensor: Reduced loss tensor. + """ + reduction_enum = F._Reduction.get_enum(reduction) + # none: 0, elementwise_mean:1, sum: 2 + if reduction_enum == 0: + return loss + elif reduction_enum == 1: + return loss.mean() + elif reduction_enum == 2: + return loss.sum() + + +def weight_reduce_loss(loss, weight=None, reduction='mean', avg_factor=None): + """Apply element-wise weight and reduce loss. + + Args: + loss (Tensor): Element-wise loss. + weight (Tensor): Element-wise weights. + reduction (str): Same as built-in losses of PyTorch. + avg_factor (float): Avarage factor when computing the mean of losses. + + Returns: + Tensor: Processed loss values. + """ + # if weight is specified, apply element-wise weight + if weight is not None: + assert weight.dim() == loss.dim() + if weight.dim() > 1: + assert weight.size(1) == 1 or weight.size(1) == loss.size(1) + loss = loss * weight + + # if avg_factor is not specified, just reduce the loss + if avg_factor is None: + loss = reduce_loss(loss, reduction) + else: + # if reduction is mean, then average the loss by avg_factor + if reduction == 'mean': + loss = loss.sum() / avg_factor + # if reduction is 'none', then do nothing, otherwise raise an error + elif reduction != 'none': + raise ValueError('avg_factor can not be used with reduction="sum"') + return loss + + +def weighted_loss(loss_func): + """Create a weighted version of a given loss function. + + To use this decorator, the loss function must have the signature like + `loss_func(pred, target, **kwargs)`. The function only needs to compute + element-wise loss without any reduction. This decorator will add weight + and reduction arguments to the function. The decorated function will have + the signature like `loss_func(pred, target, weight=None, reduction='mean', + avg_factor=None, **kwargs)`. + + :Example: + + >>> import torch + >>> @weighted_loss + >>> def l1_loss(pred, target): + >>> return (pred - target).abs() + + >>> pred = torch.Tensor([0, 2, 3]) + >>> target = torch.Tensor([1, 1, 1]) + >>> weight = torch.Tensor([1, 0, 1]) + + >>> l1_loss(pred, target) + tensor(1.3333) + >>> l1_loss(pred, target, weight) + tensor(1.) + >>> l1_loss(pred, target, reduction='none') + tensor([1., 1., 2.]) + >>> l1_loss(pred, target, weight, avg_factor=2) + tensor(1.5000) + """ + + @functools.wraps(loss_func) + def wrapper(pred, + target, + weight=None, + reduction='mean', + avg_factor=None, + **kwargs): + # get element-wise loss + loss = loss_func(pred, target, **kwargs) + loss = weight_reduce_loss(loss, weight, reduction, avg_factor) + return loss + + return wrapper diff --git a/annotator/mmpkg/mmseg/models/necks/__init__.py b/annotator/mmpkg/mmseg/models/necks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9b9d3d5b3fe80247642d962edd6fb787537d01d6 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/necks/__init__.py @@ -0,0 +1,4 @@ +from .fpn import FPN +from .multilevel_neck import MultiLevelNeck + +__all__ = ['FPN', 'MultiLevelNeck'] diff --git a/annotator/mmpkg/mmseg/models/necks/fpn.py b/annotator/mmpkg/mmseg/models/necks/fpn.py new file mode 100644 index 0000000000000000000000000000000000000000..ba47bbe1a0225587315627ac288e5ddf6497a244 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/necks/fpn.py @@ -0,0 +1,212 @@ +import torch.nn as nn +import torch.nn.functional as F +from annotator.mmpkg.mmcv.cnn import ConvModule, xavier_init + +from ..builder import NECKS + + +@NECKS.register_module() +class FPN(nn.Module): + """Feature Pyramid Network. + + This is an implementation of - Feature Pyramid Networks for Object + Detection (https://arxiv.org/abs/1612.03144) + + Args: + in_channels (List[int]): Number of input channels per scale. + out_channels (int): Number of output channels (used at each scale) + num_outs (int): Number of output scales. + start_level (int): Index of the start input backbone level used to + build the feature pyramid. Default: 0. + end_level (int): Index of the end input backbone level (exclusive) to + build the feature pyramid. Default: -1, which means the last level. + add_extra_convs (bool | str): If bool, it decides whether to add conv + layers on top of the original feature maps. Default to False. + If True, its actual mode is specified by `extra_convs_on_inputs`. + If str, it specifies the source feature map of the extra convs. + Only the following options are allowed + + - 'on_input': Last feat map of neck inputs (i.e. backbone feature). + - 'on_lateral': Last feature map after lateral convs. + - 'on_output': The last output feature map after fpn convs. + extra_convs_on_inputs (bool, deprecated): Whether to apply extra convs + on the original feature from the backbone. If True, + it is equivalent to `add_extra_convs='on_input'`. If False, it is + equivalent to set `add_extra_convs='on_output'`. Default to True. + relu_before_extra_convs (bool): Whether to apply relu before the extra + conv. Default: False. + no_norm_on_lateral (bool): Whether to apply norm on lateral. + Default: False. + conv_cfg (dict): Config dict for convolution layer. Default: None. + norm_cfg (dict): Config dict for normalization layer. Default: None. + act_cfg (str): Config dict for activation layer in ConvModule. + Default: None. + upsample_cfg (dict): Config dict for interpolate layer. + Default: `dict(mode='nearest')` + + Example: + >>> import torch + >>> in_channels = [2, 3, 5, 7] + >>> scales = [340, 170, 84, 43] + >>> inputs = [torch.rand(1, c, s, s) + ... for c, s in zip(in_channels, scales)] + >>> self = FPN(in_channels, 11, len(in_channels)).eval() + >>> outputs = self.forward(inputs) + >>> for i in range(len(outputs)): + ... print(f'outputs[{i}].shape = {outputs[i].shape}') + outputs[0].shape = torch.Size([1, 11, 340, 340]) + outputs[1].shape = torch.Size([1, 11, 170, 170]) + outputs[2].shape = torch.Size([1, 11, 84, 84]) + outputs[3].shape = torch.Size([1, 11, 43, 43]) + """ + + def __init__(self, + in_channels, + out_channels, + num_outs, + start_level=0, + end_level=-1, + add_extra_convs=False, + extra_convs_on_inputs=False, + relu_before_extra_convs=False, + no_norm_on_lateral=False, + conv_cfg=None, + norm_cfg=None, + act_cfg=None, + upsample_cfg=dict(mode='nearest')): + super(FPN, self).__init__() + assert isinstance(in_channels, list) + self.in_channels = in_channels + self.out_channels = out_channels + self.num_ins = len(in_channels) + self.num_outs = num_outs + self.relu_before_extra_convs = relu_before_extra_convs + self.no_norm_on_lateral = no_norm_on_lateral + self.fp16_enabled = False + self.upsample_cfg = upsample_cfg.copy() + + if end_level == -1: + self.backbone_end_level = self.num_ins + assert num_outs >= self.num_ins - start_level + else: + # if end_level < inputs, no extra level is allowed + self.backbone_end_level = end_level + assert end_level <= len(in_channels) + assert num_outs == end_level - start_level + self.start_level = start_level + self.end_level = end_level + self.add_extra_convs = add_extra_convs + assert isinstance(add_extra_convs, (str, bool)) + if isinstance(add_extra_convs, str): + # Extra_convs_source choices: 'on_input', 'on_lateral', 'on_output' + assert add_extra_convs in ('on_input', 'on_lateral', 'on_output') + elif add_extra_convs: # True + if extra_convs_on_inputs: + # For compatibility with previous release + # TODO: deprecate `extra_convs_on_inputs` + self.add_extra_convs = 'on_input' + else: + self.add_extra_convs = 'on_output' + + self.lateral_convs = nn.ModuleList() + self.fpn_convs = nn.ModuleList() + + for i in range(self.start_level, self.backbone_end_level): + l_conv = ConvModule( + in_channels[i], + out_channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg if not self.no_norm_on_lateral else None, + act_cfg=act_cfg, + inplace=False) + fpn_conv = ConvModule( + out_channels, + out_channels, + 3, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + inplace=False) + + self.lateral_convs.append(l_conv) + self.fpn_convs.append(fpn_conv) + + # add extra conv layers (e.g., RetinaNet) + extra_levels = num_outs - self.backbone_end_level + self.start_level + if self.add_extra_convs and extra_levels >= 1: + for i in range(extra_levels): + if i == 0 and self.add_extra_convs == 'on_input': + in_channels = self.in_channels[self.backbone_end_level - 1] + else: + in_channels = out_channels + extra_fpn_conv = ConvModule( + in_channels, + out_channels, + 3, + stride=2, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + inplace=False) + self.fpn_convs.append(extra_fpn_conv) + + # default init_weights for conv(msra) and norm in ConvModule + def init_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + xavier_init(m, distribution='uniform') + + def forward(self, inputs): + assert len(inputs) == len(self.in_channels) + + # build laterals + laterals = [ + lateral_conv(inputs[i + self.start_level]) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + + # build top-down path + used_backbone_levels = len(laterals) + for i in range(used_backbone_levels - 1, 0, -1): + # In some cases, fixing `scale factor` (e.g. 2) is preferred, but + # it cannot co-exist with `size` in `F.interpolate`. + if 'scale_factor' in self.upsample_cfg: + laterals[i - 1] += F.interpolate(laterals[i], + **self.upsample_cfg) + else: + prev_shape = laterals[i - 1].shape[2:] + laterals[i - 1] += F.interpolate( + laterals[i], size=prev_shape, **self.upsample_cfg) + + # build outputs + # part 1: from original levels + outs = [ + self.fpn_convs[i](laterals[i]) for i in range(used_backbone_levels) + ] + # part 2: add extra levels + if self.num_outs > len(outs): + # use max pool to get more levels on top of outputs + # (e.g., Faster R-CNN, Mask R-CNN) + if not self.add_extra_convs: + for i in range(self.num_outs - used_backbone_levels): + outs.append(F.max_pool2d(outs[-1], 1, stride=2)) + # add conv layers on top of original feature maps (RetinaNet) + else: + if self.add_extra_convs == 'on_input': + extra_source = inputs[self.backbone_end_level - 1] + elif self.add_extra_convs == 'on_lateral': + extra_source = laterals[-1] + elif self.add_extra_convs == 'on_output': + extra_source = outs[-1] + else: + raise NotImplementedError + outs.append(self.fpn_convs[used_backbone_levels](extra_source)) + for i in range(used_backbone_levels + 1, self.num_outs): + if self.relu_before_extra_convs: + outs.append(self.fpn_convs[i](F.relu(outs[-1]))) + else: + outs.append(self.fpn_convs[i](outs[-1])) + return tuple(outs) diff --git a/annotator/mmpkg/mmseg/models/necks/multilevel_neck.py b/annotator/mmpkg/mmseg/models/necks/multilevel_neck.py new file mode 100644 index 0000000000000000000000000000000000000000..0b86c073cd1a72354d2426846125e80f7ab20dbc --- /dev/null +++ b/annotator/mmpkg/mmseg/models/necks/multilevel_neck.py @@ -0,0 +1,70 @@ +import torch.nn as nn +import torch.nn.functional as F +from annotator.mmpkg.mmcv.cnn import ConvModule + +from ..builder import NECKS + + +@NECKS.register_module() +class MultiLevelNeck(nn.Module): + """MultiLevelNeck. + + A neck structure connect vit backbone and decoder_heads. + Args: + in_channels (List[int]): Number of input channels per scale. + out_channels (int): Number of output channels (used at each scale). + scales (List[int]): Scale factors for each input feature map. + norm_cfg (dict): Config dict for normalization layer. Default: None. + act_cfg (dict): Config dict for activation layer in ConvModule. + Default: None. + """ + + def __init__(self, + in_channels, + out_channels, + scales=[0.5, 1, 2, 4], + norm_cfg=None, + act_cfg=None): + super(MultiLevelNeck, self).__init__() + assert isinstance(in_channels, list) + self.in_channels = in_channels + self.out_channels = out_channels + self.scales = scales + self.num_outs = len(scales) + self.lateral_convs = nn.ModuleList() + self.convs = nn.ModuleList() + for in_channel in in_channels: + self.lateral_convs.append( + ConvModule( + in_channel, + out_channels, + kernel_size=1, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + for _ in range(self.num_outs): + self.convs.append( + ConvModule( + out_channels, + out_channels, + kernel_size=3, + padding=1, + stride=1, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + + def forward(self, inputs): + assert len(inputs) == len(self.in_channels) + print(inputs[0].shape) + inputs = [ + lateral_conv(inputs[i]) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + # for len(inputs) not equal to self.num_outs + if len(inputs) == 1: + inputs = [inputs[0] for _ in range(self.num_outs)] + outs = [] + for i in range(self.num_outs): + x_resize = F.interpolate( + inputs[i], scale_factor=self.scales[i], mode='bilinear') + outs.append(self.convs[i](x_resize)) + return tuple(outs) diff --git a/annotator/mmpkg/mmseg/models/segmentors/__init__.py b/annotator/mmpkg/mmseg/models/segmentors/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..dca2f09405330743c476e190896bee39c45498ea --- /dev/null +++ b/annotator/mmpkg/mmseg/models/segmentors/__init__.py @@ -0,0 +1,5 @@ +from .base import BaseSegmentor +from .cascade_encoder_decoder import CascadeEncoderDecoder +from .encoder_decoder import EncoderDecoder + +__all__ = ['BaseSegmentor', 'EncoderDecoder', 'CascadeEncoderDecoder'] diff --git a/annotator/mmpkg/mmseg/models/segmentors/base.py b/annotator/mmpkg/mmseg/models/segmentors/base.py new file mode 100644 index 0000000000000000000000000000000000000000..a12d8beb8ea40bfa234197eddb4d3ef40dbfeb6f --- /dev/null +++ b/annotator/mmpkg/mmseg/models/segmentors/base.py @@ -0,0 +1,273 @@ +import logging +import warnings +from abc import ABCMeta, abstractmethod +from collections import OrderedDict + +import annotator.mmpkg.mmcv as mmcv +import numpy as np +import torch +import torch.distributed as dist +import torch.nn as nn +from annotator.mmpkg.mmcv.runner import auto_fp16 + + +class BaseSegmentor(nn.Module): + """Base class for segmentors.""" + + __metaclass__ = ABCMeta + + def __init__(self): + super(BaseSegmentor, self).__init__() + self.fp16_enabled = False + + @property + def with_neck(self): + """bool: whether the segmentor has neck""" + return hasattr(self, 'neck') and self.neck is not None + + @property + def with_auxiliary_head(self): + """bool: whether the segmentor has auxiliary head""" + return hasattr(self, + 'auxiliary_head') and self.auxiliary_head is not None + + @property + def with_decode_head(self): + """bool: whether the segmentor has decode head""" + return hasattr(self, 'decode_head') and self.decode_head is not None + + @abstractmethod + def extract_feat(self, imgs): + """Placeholder for extract features from images.""" + pass + + @abstractmethod + def encode_decode(self, img, img_metas): + """Placeholder for encode images with backbone and decode into a + semantic segmentation map of the same size as input.""" + pass + + @abstractmethod + def forward_train(self, imgs, img_metas, **kwargs): + """Placeholder for Forward function for training.""" + pass + + @abstractmethod + def simple_test(self, img, img_meta, **kwargs): + """Placeholder for single image test.""" + pass + + @abstractmethod + def aug_test(self, imgs, img_metas, **kwargs): + """Placeholder for augmentation test.""" + pass + + def init_weights(self, pretrained=None): + """Initialize the weights in segmentor. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if pretrained is not None: + logger = logging.getLogger() + logger.info(f'load model from: {pretrained}') + + def forward_test(self, imgs, img_metas, **kwargs): + """ + Args: + imgs (List[Tensor]): the outer list indicates test-time + augmentations and inner Tensor should have a shape NxCxHxW, + which contains all images in the batch. + img_metas (List[List[dict]]): the outer list indicates test-time + augs (multiscale, flip, etc.) and the inner list indicates + images in a batch. + """ + for var, name in [(imgs, 'imgs'), (img_metas, 'img_metas')]: + if not isinstance(var, list): + raise TypeError(f'{name} must be a list, but got ' + f'{type(var)}') + + num_augs = len(imgs) + if num_augs != len(img_metas): + raise ValueError(f'num of augmentations ({len(imgs)}) != ' + f'num of image meta ({len(img_metas)})') + # all images in the same aug batch all of the same ori_shape and pad + # shape + for img_meta in img_metas: + ori_shapes = [_['ori_shape'] for _ in img_meta] + assert all(shape == ori_shapes[0] for shape in ori_shapes) + img_shapes = [_['img_shape'] for _ in img_meta] + assert all(shape == img_shapes[0] for shape in img_shapes) + pad_shapes = [_['pad_shape'] for _ in img_meta] + assert all(shape == pad_shapes[0] for shape in pad_shapes) + + if num_augs == 1: + return self.simple_test(imgs[0], img_metas[0], **kwargs) + else: + return self.aug_test(imgs, img_metas, **kwargs) + + @auto_fp16(apply_to=('img', )) + def forward(self, img, img_metas, return_loss=True, **kwargs): + """Calls either :func:`forward_train` or :func:`forward_test` depending + on whether ``return_loss`` is ``True``. + + Note this setting will change the expected inputs. When + ``return_loss=True``, img and img_meta are single-nested (i.e. Tensor + and List[dict]), and when ``resturn_loss=False``, img and img_meta + should be double nested (i.e. List[Tensor], List[List[dict]]), with + the outer list indicating test time augmentations. + """ + if return_loss: + return self.forward_train(img, img_metas, **kwargs) + else: + return self.forward_test(img, img_metas, **kwargs) + + def train_step(self, data_batch, optimizer, **kwargs): + """The iteration step during training. + + This method defines an iteration step during training, except for the + back propagation and optimizer updating, which are done in an optimizer + hook. Note that in some complicated cases or models, the whole process + including back propagation and optimizer updating is also defined in + this method, such as GAN. + + Args: + data (dict): The output of dataloader. + optimizer (:obj:`torch.optim.Optimizer` | dict): The optimizer of + runner is passed to ``train_step()``. This argument is unused + and reserved. + + Returns: + dict: It should contain at least 3 keys: ``loss``, ``log_vars``, + ``num_samples``. + ``loss`` is a tensor for back propagation, which can be a + weighted sum of multiple losses. + ``log_vars`` contains all the variables to be sent to the + logger. + ``num_samples`` indicates the batch size (when the model is + DDP, it means the batch size on each GPU), which is used for + averaging the logs. + """ + losses = self(**data_batch) + loss, log_vars = self._parse_losses(losses) + + outputs = dict( + loss=loss, + log_vars=log_vars, + num_samples=len(data_batch['img_metas'])) + + return outputs + + def val_step(self, data_batch, **kwargs): + """The iteration step during validation. + + This method shares the same signature as :func:`train_step`, but used + during val epochs. Note that the evaluation after training epochs is + not implemented with this method, but an evaluation hook. + """ + output = self(**data_batch, **kwargs) + return output + + @staticmethod + def _parse_losses(losses): + """Parse the raw outputs (losses) of the network. + + Args: + losses (dict): Raw output of the network, which usually contain + losses and other necessary information. + + Returns: + tuple[Tensor, dict]: (loss, log_vars), loss is the loss tensor + which may be a weighted sum of all losses, log_vars contains + all the variables to be sent to the logger. + """ + log_vars = OrderedDict() + for loss_name, loss_value in losses.items(): + if isinstance(loss_value, torch.Tensor): + log_vars[loss_name] = loss_value.mean() + elif isinstance(loss_value, list): + log_vars[loss_name] = sum(_loss.mean() for _loss in loss_value) + else: + raise TypeError( + f'{loss_name} is not a tensor or list of tensors') + + loss = sum(_value for _key, _value in log_vars.items() + if 'loss' in _key) + + log_vars['loss'] = loss + for loss_name, loss_value in log_vars.items(): + # reduce loss when distributed training + if dist.is_available() and dist.is_initialized(): + loss_value = loss_value.data.clone() + dist.all_reduce(loss_value.div_(dist.get_world_size())) + log_vars[loss_name] = loss_value.item() + + return loss, log_vars + + def show_result(self, + img, + result, + palette=None, + win_name='', + show=False, + wait_time=0, + out_file=None, + opacity=0.5): + """Draw `result` over `img`. + + Args: + img (str or Tensor): The image to be displayed. + result (Tensor): The semantic segmentation results to draw over + `img`. + palette (list[list[int]]] | np.ndarray | None): The palette of + segmentation map. If None is given, random palette will be + generated. Default: None + win_name (str): The window name. + wait_time (int): Value of waitKey param. + Default: 0. + show (bool): Whether to show the image. + Default: False. + out_file (str or None): The filename to write the image. + Default: None. + opacity(float): Opacity of painted segmentation map. + Default 0.5. + Must be in (0, 1] range. + Returns: + img (Tensor): Only if not `show` or `out_file` + """ + img = mmcv.imread(img) + img = img.copy() + seg = result[0] + if palette is None: + if self.PALETTE is None: + palette = np.random.randint( + 0, 255, size=(len(self.CLASSES), 3)) + else: + palette = self.PALETTE + palette = np.array(palette) + assert palette.shape[0] == len(self.CLASSES) + assert palette.shape[1] == 3 + assert len(palette.shape) == 2 + assert 0 < opacity <= 1.0 + color_seg = np.zeros((seg.shape[0], seg.shape[1], 3), dtype=np.uint8) + for label, color in enumerate(palette): + color_seg[seg == label, :] = color + # convert to BGR + color_seg = color_seg[..., ::-1] + + img = img * (1 - opacity) + color_seg * opacity + img = img.astype(np.uint8) + # if out_file specified, do not show image in window + if out_file is not None: + show = False + + if show: + mmcv.imshow(img, win_name, wait_time) + if out_file is not None: + mmcv.imwrite(img, out_file) + + if not (show or out_file): + warnings.warn('show==False and out_file is not specified, only ' + 'result image will be returned') + return img diff --git a/annotator/mmpkg/mmseg/models/segmentors/cascade_encoder_decoder.py b/annotator/mmpkg/mmseg/models/segmentors/cascade_encoder_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..74547f0fb01da9fe32c1d142768eb788b7e8673c --- /dev/null +++ b/annotator/mmpkg/mmseg/models/segmentors/cascade_encoder_decoder.py @@ -0,0 +1,98 @@ +from torch import nn + +from annotator.mmpkg.mmseg.core import add_prefix +from annotator.mmpkg.mmseg.ops import resize +from .. import builder +from ..builder import SEGMENTORS +from .encoder_decoder import EncoderDecoder + + +@SEGMENTORS.register_module() +class CascadeEncoderDecoder(EncoderDecoder): + """Cascade Encoder Decoder segmentors. + + CascadeEncoderDecoder almost the same as EncoderDecoder, while decoders of + CascadeEncoderDecoder are cascaded. The output of previous decoder_head + will be the input of next decoder_head. + """ + + def __init__(self, + num_stages, + backbone, + decode_head, + neck=None, + auxiliary_head=None, + train_cfg=None, + test_cfg=None, + pretrained=None): + self.num_stages = num_stages + super(CascadeEncoderDecoder, self).__init__( + backbone=backbone, + decode_head=decode_head, + neck=neck, + auxiliary_head=auxiliary_head, + train_cfg=train_cfg, + test_cfg=test_cfg, + pretrained=pretrained) + + def _init_decode_head(self, decode_head): + """Initialize ``decode_head``""" + assert isinstance(decode_head, list) + assert len(decode_head) == self.num_stages + self.decode_head = nn.ModuleList() + for i in range(self.num_stages): + self.decode_head.append(builder.build_head(decode_head[i])) + self.align_corners = self.decode_head[-1].align_corners + self.num_classes = self.decode_head[-1].num_classes + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone and heads. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + self.backbone.init_weights(pretrained=pretrained) + for i in range(self.num_stages): + self.decode_head[i].init_weights() + if self.with_auxiliary_head: + if isinstance(self.auxiliary_head, nn.ModuleList): + for aux_head in self.auxiliary_head: + aux_head.init_weights() + else: + self.auxiliary_head.init_weights() + + def encode_decode(self, img, img_metas): + """Encode images with backbone and decode into a semantic segmentation + map of the same size as input.""" + x = self.extract_feat(img) + out = self.decode_head[0].forward_test(x, img_metas, self.test_cfg) + for i in range(1, self.num_stages): + out = self.decode_head[i].forward_test(x, out, img_metas, + self.test_cfg) + out = resize( + input=out, + size=img.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + return out + + def _decode_head_forward_train(self, x, img_metas, gt_semantic_seg): + """Run forward function and calculate loss for decode head in + training.""" + losses = dict() + + loss_decode = self.decode_head[0].forward_train( + x, img_metas, gt_semantic_seg, self.train_cfg) + + losses.update(add_prefix(loss_decode, 'decode_0')) + + for i in range(1, self.num_stages): + # forward test again, maybe unnecessary for most methods. + prev_outputs = self.decode_head[i - 1].forward_test( + x, img_metas, self.test_cfg) + loss_decode = self.decode_head[i].forward_train( + x, prev_outputs, img_metas, gt_semantic_seg, self.train_cfg) + losses.update(add_prefix(loss_decode, f'decode_{i}')) + + return losses diff --git a/annotator/mmpkg/mmseg/models/segmentors/encoder_decoder.py b/annotator/mmpkg/mmseg/models/segmentors/encoder_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..30c25f35a15e65e45f9221a3f19ace8579f73301 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/segmentors/encoder_decoder.py @@ -0,0 +1,298 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from annotator.mmpkg.mmseg.core import add_prefix +from annotator.mmpkg.mmseg.ops import resize +from .. import builder +from ..builder import SEGMENTORS +from .base import BaseSegmentor + + +@SEGMENTORS.register_module() +class EncoderDecoder(BaseSegmentor): + """Encoder Decoder segmentors. + + EncoderDecoder typically consists of backbone, decode_head, auxiliary_head. + Note that auxiliary_head is only used for deep supervision during training, + which could be dumped during inference. + """ + + def __init__(self, + backbone, + decode_head, + neck=None, + auxiliary_head=None, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(EncoderDecoder, self).__init__() + self.backbone = builder.build_backbone(backbone) + if neck is not None: + self.neck = builder.build_neck(neck) + self._init_decode_head(decode_head) + self._init_auxiliary_head(auxiliary_head) + + self.train_cfg = train_cfg + self.test_cfg = test_cfg + + self.init_weights(pretrained=pretrained) + + assert self.with_decode_head + + def _init_decode_head(self, decode_head): + """Initialize ``decode_head``""" + self.decode_head = builder.build_head(decode_head) + self.align_corners = self.decode_head.align_corners + self.num_classes = self.decode_head.num_classes + + def _init_auxiliary_head(self, auxiliary_head): + """Initialize ``auxiliary_head``""" + if auxiliary_head is not None: + if isinstance(auxiliary_head, list): + self.auxiliary_head = nn.ModuleList() + for head_cfg in auxiliary_head: + self.auxiliary_head.append(builder.build_head(head_cfg)) + else: + self.auxiliary_head = builder.build_head(auxiliary_head) + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone and heads. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + + super(EncoderDecoder, self).init_weights(pretrained) + self.backbone.init_weights(pretrained=pretrained) + self.decode_head.init_weights() + if self.with_auxiliary_head: + if isinstance(self.auxiliary_head, nn.ModuleList): + for aux_head in self.auxiliary_head: + aux_head.init_weights() + else: + self.auxiliary_head.init_weights() + + def extract_feat(self, img): + """Extract features from images.""" + x = self.backbone(img) + if self.with_neck: + x = self.neck(x) + return x + + def encode_decode(self, img, img_metas): + """Encode images with backbone and decode into a semantic segmentation + map of the same size as input.""" + x = self.extract_feat(img) + out = self._decode_head_forward_test(x, img_metas) + out = resize( + input=out, + size=img.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + return out + + def _decode_head_forward_train(self, x, img_metas, gt_semantic_seg): + """Run forward function and calculate loss for decode head in + training.""" + losses = dict() + loss_decode = self.decode_head.forward_train(x, img_metas, + gt_semantic_seg, + self.train_cfg) + + losses.update(add_prefix(loss_decode, 'decode')) + return losses + + def _decode_head_forward_test(self, x, img_metas): + """Run forward function and calculate loss for decode head in + inference.""" + seg_logits = self.decode_head.forward_test(x, img_metas, self.test_cfg) + return seg_logits + + def _auxiliary_head_forward_train(self, x, img_metas, gt_semantic_seg): + """Run forward function and calculate loss for auxiliary head in + training.""" + losses = dict() + if isinstance(self.auxiliary_head, nn.ModuleList): + for idx, aux_head in enumerate(self.auxiliary_head): + loss_aux = aux_head.forward_train(x, img_metas, + gt_semantic_seg, + self.train_cfg) + losses.update(add_prefix(loss_aux, f'aux_{idx}')) + else: + loss_aux = self.auxiliary_head.forward_train( + x, img_metas, gt_semantic_seg, self.train_cfg) + losses.update(add_prefix(loss_aux, 'aux')) + + return losses + + def forward_dummy(self, img): + """Dummy forward function.""" + seg_logit = self.encode_decode(img, None) + + return seg_logit + + def forward_train(self, img, img_metas, gt_semantic_seg): + """Forward function for training. + + Args: + img (Tensor): Input images. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + gt_semantic_seg (Tensor): Semantic segmentation masks + used if the architecture supports semantic segmentation task. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + + x = self.extract_feat(img) + + losses = dict() + + loss_decode = self._decode_head_forward_train(x, img_metas, + gt_semantic_seg) + losses.update(loss_decode) + + if self.with_auxiliary_head: + loss_aux = self._auxiliary_head_forward_train( + x, img_metas, gt_semantic_seg) + losses.update(loss_aux) + + return losses + + # TODO refactor + def slide_inference(self, img, img_meta, rescale): + """Inference by sliding-window with overlap. + + If h_crop > h_img or w_crop > w_img, the small patch will be used to + decode without padding. + """ + + h_stride, w_stride = self.test_cfg.stride + h_crop, w_crop = self.test_cfg.crop_size + batch_size, _, h_img, w_img = img.size() + num_classes = self.num_classes + h_grids = max(h_img - h_crop + h_stride - 1, 0) // h_stride + 1 + w_grids = max(w_img - w_crop + w_stride - 1, 0) // w_stride + 1 + preds = img.new_zeros((batch_size, num_classes, h_img, w_img)) + count_mat = img.new_zeros((batch_size, 1, h_img, w_img)) + for h_idx in range(h_grids): + for w_idx in range(w_grids): + y1 = h_idx * h_stride + x1 = w_idx * w_stride + y2 = min(y1 + h_crop, h_img) + x2 = min(x1 + w_crop, w_img) + y1 = max(y2 - h_crop, 0) + x1 = max(x2 - w_crop, 0) + crop_img = img[:, :, y1:y2, x1:x2] + crop_seg_logit = self.encode_decode(crop_img, img_meta) + preds += F.pad(crop_seg_logit, + (int(x1), int(preds.shape[3] - x2), int(y1), + int(preds.shape[2] - y2))) + + count_mat[:, :, y1:y2, x1:x2] += 1 + assert (count_mat == 0).sum() == 0 + if torch.onnx.is_in_onnx_export(): + # cast count_mat to constant while exporting to ONNX + count_mat = torch.from_numpy( + count_mat.cpu().detach().numpy()).to(device=img.device) + preds = preds / count_mat + if rescale: + preds = resize( + preds, + size=img_meta[0]['ori_shape'][:2], + mode='bilinear', + align_corners=self.align_corners, + warning=False) + return preds + + def whole_inference(self, img, img_meta, rescale): + """Inference with full image.""" + + seg_logit = self.encode_decode(img, img_meta) + if rescale: + # support dynamic shape for onnx + if torch.onnx.is_in_onnx_export(): + size = img.shape[2:] + else: + size = img_meta[0]['ori_shape'][:2] + seg_logit = resize( + seg_logit, + size=size, + mode='bilinear', + align_corners=self.align_corners, + warning=False) + + return seg_logit + + def inference(self, img, img_meta, rescale): + """Inference with slide/whole style. + + Args: + img (Tensor): The input image of shape (N, 3, H, W). + img_meta (dict): Image info dict where each dict has: 'img_shape', + 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + rescale (bool): Whether rescale back to original shape. + + Returns: + Tensor: The output segmentation map. + """ + + assert self.test_cfg.mode in ['slide', 'whole'] + ori_shape = img_meta[0]['ori_shape'] + assert all(_['ori_shape'] == ori_shape for _ in img_meta) + if self.test_cfg.mode == 'slide': + seg_logit = self.slide_inference(img, img_meta, rescale) + else: + seg_logit = self.whole_inference(img, img_meta, rescale) + output = F.softmax(seg_logit, dim=1) + flip = img_meta[0]['flip'] + if flip: + flip_direction = img_meta[0]['flip_direction'] + assert flip_direction in ['horizontal', 'vertical'] + if flip_direction == 'horizontal': + output = output.flip(dims=(3, )) + elif flip_direction == 'vertical': + output = output.flip(dims=(2, )) + + return output + + def simple_test(self, img, img_meta, rescale=True): + """Simple test with single image.""" + seg_logit = self.inference(img, img_meta, rescale) + seg_pred = seg_logit.argmax(dim=1) + if torch.onnx.is_in_onnx_export(): + # our inference backend only support 4D output + seg_pred = seg_pred.unsqueeze(0) + return seg_pred + seg_pred = seg_pred.cpu().numpy() + # unravel batch dim + seg_pred = list(seg_pred) + return seg_pred + + def aug_test(self, imgs, img_metas, rescale=True): + """Test with augmentations. + + Only rescale=True is supported. + """ + # aug_test rescale all imgs back to ori_shape for now + assert rescale + # to save memory, we get augmented seg logit inplace + seg_logit = self.inference(imgs[0], img_metas[0], rescale) + for i in range(1, len(imgs)): + cur_seg_logit = self.inference(imgs[i], img_metas[i], rescale) + seg_logit += cur_seg_logit + seg_logit /= len(imgs) + seg_pred = seg_logit.argmax(dim=1) + seg_pred = seg_pred.cpu().numpy() + # unravel batch dim + seg_pred = list(seg_pred) + return seg_pred diff --git a/annotator/mmpkg/mmseg/models/utils/__init__.py b/annotator/mmpkg/mmseg/models/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3d3bdd349b9f2ae499a2fcb2ac1d2e3c77befebe --- /dev/null +++ b/annotator/mmpkg/mmseg/models/utils/__init__.py @@ -0,0 +1,13 @@ +from .drop import DropPath +from .inverted_residual import InvertedResidual, InvertedResidualV3 +from .make_divisible import make_divisible +from .res_layer import ResLayer +from .se_layer import SELayer +from .self_attention_block import SelfAttentionBlock +from .up_conv_block import UpConvBlock +from .weight_init import trunc_normal_ + +__all__ = [ + 'ResLayer', 'SelfAttentionBlock', 'make_divisible', 'InvertedResidual', + 'UpConvBlock', 'InvertedResidualV3', 'SELayer', 'DropPath', 'trunc_normal_' +] diff --git a/annotator/mmpkg/mmseg/models/utils/drop.py b/annotator/mmpkg/mmseg/models/utils/drop.py new file mode 100644 index 0000000000000000000000000000000000000000..4520b0ff407d2a95a864086bdbca0065f222aa63 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/utils/drop.py @@ -0,0 +1,31 @@ +"""Modified from https://github.com/rwightman/pytorch-image- +models/blob/master/timm/models/layers/drop.py.""" + +import torch +from torch import nn + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of + residual blocks). + + Args: + drop_prob (float): Drop rate for paths of model. Dropout rate has + to be between 0 and 1. Default: 0. + """ + + def __init__(self, drop_prob=0.): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + self.keep_prob = 1 - drop_prob + + def forward(self, x): + if self.drop_prob == 0. or not self.training: + return x + shape = (x.shape[0], ) + (1, ) * ( + x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = self.keep_prob + torch.rand( + shape, dtype=x.dtype, device=x.device) + random_tensor.floor_() # binarize + output = x.div(self.keep_prob) * random_tensor + return output diff --git a/annotator/mmpkg/mmseg/models/utils/inverted_residual.py b/annotator/mmpkg/mmseg/models/utils/inverted_residual.py new file mode 100644 index 0000000000000000000000000000000000000000..2df5ebd7c94c0a66b0d05ef9e200ddbeabfa79f6 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/utils/inverted_residual.py @@ -0,0 +1,208 @@ +from annotator.mmpkg.mmcv.cnn import ConvModule +from torch import nn +from torch.utils import checkpoint as cp + +from .se_layer import SELayer + + +class InvertedResidual(nn.Module): + """InvertedResidual block for MobileNetV2. + + Args: + in_channels (int): The input channels of the InvertedResidual block. + out_channels (int): The output channels of the InvertedResidual block. + stride (int): Stride of the middle (first) 3x3 convolution. + expand_ratio (int): Adjusts number of channels of the hidden layer + in InvertedResidual by this amount. + dilation (int): Dilation rate of depthwise conv. Default: 1 + conv_cfg (dict): Config dict for convolution layer. + Default: None, which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='ReLU6'). + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + + Returns: + Tensor: The output tensor. + """ + + def __init__(self, + in_channels, + out_channels, + stride, + expand_ratio, + dilation=1, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU6'), + with_cp=False): + super(InvertedResidual, self).__init__() + self.stride = stride + assert stride in [1, 2], f'stride must in [1, 2]. ' \ + f'But received {stride}.' + self.with_cp = with_cp + self.use_res_connect = self.stride == 1 and in_channels == out_channels + hidden_dim = int(round(in_channels * expand_ratio)) + + layers = [] + if expand_ratio != 1: + layers.append( + ConvModule( + in_channels=in_channels, + out_channels=hidden_dim, + kernel_size=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + layers.extend([ + ConvModule( + in_channels=hidden_dim, + out_channels=hidden_dim, + kernel_size=3, + stride=stride, + padding=dilation, + dilation=dilation, + groups=hidden_dim, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg), + ConvModule( + in_channels=hidden_dim, + out_channels=out_channels, + kernel_size=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=None) + ]) + self.conv = nn.Sequential(*layers) + + def forward(self, x): + + def _inner_forward(x): + if self.use_res_connect: + return x + self.conv(x) + else: + return self.conv(x) + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + return out + + +class InvertedResidualV3(nn.Module): + """Inverted Residual Block for MobileNetV3. + + Args: + in_channels (int): The input channels of this Module. + out_channels (int): The output channels of this Module. + mid_channels (int): The input channels of the depthwise convolution. + kernel_size (int): The kernel size of the depthwise convolution. + Default: 3. + stride (int): The stride of the depthwise convolution. Default: 1. + se_cfg (dict): Config dict for se layer. Default: None, which means no + se layer. + with_expand_conv (bool): Use expand conv or not. If set False, + mid_channels must be the same with in_channels. Default: True. + conv_cfg (dict): Config dict for convolution layer. Default: None, + which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='ReLU'). + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + + Returns: + Tensor: The output tensor. + """ + + def __init__(self, + in_channels, + out_channels, + mid_channels, + kernel_size=3, + stride=1, + se_cfg=None, + with_expand_conv=True, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + with_cp=False): + super(InvertedResidualV3, self).__init__() + self.with_res_shortcut = (stride == 1 and in_channels == out_channels) + assert stride in [1, 2] + self.with_cp = with_cp + self.with_se = se_cfg is not None + self.with_expand_conv = with_expand_conv + + if self.with_se: + assert isinstance(se_cfg, dict) + if not self.with_expand_conv: + assert mid_channels == in_channels + + if self.with_expand_conv: + self.expand_conv = ConvModule( + in_channels=in_channels, + out_channels=mid_channels, + kernel_size=1, + stride=1, + padding=0, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + self.depthwise_conv = ConvModule( + in_channels=mid_channels, + out_channels=mid_channels, + kernel_size=kernel_size, + stride=stride, + padding=kernel_size // 2, + groups=mid_channels, + conv_cfg=dict( + type='Conv2dAdaptivePadding') if stride == 2 else conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + + if self.with_se: + self.se = SELayer(**se_cfg) + + self.linear_conv = ConvModule( + in_channels=mid_channels, + out_channels=out_channels, + kernel_size=1, + stride=1, + padding=0, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=None) + + def forward(self, x): + + def _inner_forward(x): + out = x + + if self.with_expand_conv: + out = self.expand_conv(out) + + out = self.depthwise_conv(out) + + if self.with_se: + out = self.se(out) + + out = self.linear_conv(out) + + if self.with_res_shortcut: + return x + out + else: + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + return out diff --git a/annotator/mmpkg/mmseg/models/utils/make_divisible.py b/annotator/mmpkg/mmseg/models/utils/make_divisible.py new file mode 100644 index 0000000000000000000000000000000000000000..75ad756052529f52fe83bb95dd1f0ecfc9a13078 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/utils/make_divisible.py @@ -0,0 +1,27 @@ +def make_divisible(value, divisor, min_value=None, min_ratio=0.9): + """Make divisible function. + + This function rounds the channel number to the nearest value that can be + divisible by the divisor. It is taken from the original tf repo. It ensures + that all layers have a channel number that is divisible by divisor. It can + be seen here: https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py # noqa + + Args: + value (int): The original channel number. + divisor (int): The divisor to fully divide the channel number. + min_value (int): The minimum value of the output channel. + Default: None, means that the minimum value equal to the divisor. + min_ratio (float): The minimum ratio of the rounded channel number to + the original channel number. Default: 0.9. + + Returns: + int: The modified output channel number. + """ + + if min_value is None: + min_value = divisor + new_value = max(min_value, int(value + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than (1-min_ratio). + if new_value < min_ratio * value: + new_value += divisor + return new_value diff --git a/annotator/mmpkg/mmseg/models/utils/res_layer.py b/annotator/mmpkg/mmseg/models/utils/res_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..d41075a57356b4fd802bc4ff199e55e63678b589 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/utils/res_layer.py @@ -0,0 +1,94 @@ +from annotator.mmpkg.mmcv.cnn import build_conv_layer, build_norm_layer +from torch import nn as nn + + +class ResLayer(nn.Sequential): + """ResLayer to build ResNet style backbone. + + Args: + block (nn.Module): block used to build ResLayer. + inplanes (int): inplanes of block. + planes (int): planes of block. + num_blocks (int): number of blocks. + stride (int): stride of the first block. Default: 1 + avg_down (bool): Use AvgPool instead of stride conv when + downsampling in the bottleneck. Default: False + conv_cfg (dict): dictionary to construct and config conv layer. + Default: None + norm_cfg (dict): dictionary to construct and config norm layer. + Default: dict(type='BN') + multi_grid (int | None): Multi grid dilation rates of last + stage. Default: None + contract_dilation (bool): Whether contract first dilation of each layer + Default: False + """ + + def __init__(self, + block, + inplanes, + planes, + num_blocks, + stride=1, + dilation=1, + avg_down=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + multi_grid=None, + contract_dilation=False, + **kwargs): + self.block = block + + downsample = None + if stride != 1 or inplanes != planes * block.expansion: + downsample = [] + conv_stride = stride + if avg_down: + conv_stride = 1 + downsample.append( + nn.AvgPool2d( + kernel_size=stride, + stride=stride, + ceil_mode=True, + count_include_pad=False)) + downsample.extend([ + build_conv_layer( + conv_cfg, + inplanes, + planes * block.expansion, + kernel_size=1, + stride=conv_stride, + bias=False), + build_norm_layer(norm_cfg, planes * block.expansion)[1] + ]) + downsample = nn.Sequential(*downsample) + + layers = [] + if multi_grid is None: + if dilation > 1 and contract_dilation: + first_dilation = dilation // 2 + else: + first_dilation = dilation + else: + first_dilation = multi_grid[0] + layers.append( + block( + inplanes=inplanes, + planes=planes, + stride=stride, + dilation=first_dilation, + downsample=downsample, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + **kwargs)) + inplanes = planes * block.expansion + for i in range(1, num_blocks): + layers.append( + block( + inplanes=inplanes, + planes=planes, + stride=1, + dilation=dilation if multi_grid is None else multi_grid[i], + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + **kwargs)) + super(ResLayer, self).__init__(*layers) diff --git a/annotator/mmpkg/mmseg/models/utils/se_layer.py b/annotator/mmpkg/mmseg/models/utils/se_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..42ab005e1fe2211e9ecb651d31de128cf95cfec7 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/utils/se_layer.py @@ -0,0 +1,57 @@ +import annotator.mmpkg.mmcv as mmcv +import torch.nn as nn +from annotator.mmpkg.mmcv.cnn import ConvModule + +from .make_divisible import make_divisible + + +class SELayer(nn.Module): + """Squeeze-and-Excitation Module. + + Args: + channels (int): The input (and output) channels of the SE layer. + ratio (int): Squeeze ratio in SELayer, the intermediate channel will be + ``int(channels/ratio)``. Default: 16. + conv_cfg (None or dict): Config dict for convolution layer. + Default: None, which means using conv2d. + act_cfg (dict or Sequence[dict]): Config dict for activation layer. + If act_cfg is a dict, two activation layers will be configured + by this dict. If act_cfg is a sequence of dicts, the first + activation layer will be configured by the first dict and the + second activation layer will be configured by the second dict. + Default: (dict(type='ReLU'), dict(type='HSigmoid', bias=3.0, + divisor=6.0)). + """ + + def __init__(self, + channels, + ratio=16, + conv_cfg=None, + act_cfg=(dict(type='ReLU'), + dict(type='HSigmoid', bias=3.0, divisor=6.0))): + super(SELayer, self).__init__() + if isinstance(act_cfg, dict): + act_cfg = (act_cfg, act_cfg) + assert len(act_cfg) == 2 + assert mmcv.is_tuple_of(act_cfg, dict) + self.global_avgpool = nn.AdaptiveAvgPool2d(1) + self.conv1 = ConvModule( + in_channels=channels, + out_channels=make_divisible(channels // ratio, 8), + kernel_size=1, + stride=1, + conv_cfg=conv_cfg, + act_cfg=act_cfg[0]) + self.conv2 = ConvModule( + in_channels=make_divisible(channels // ratio, 8), + out_channels=channels, + kernel_size=1, + stride=1, + conv_cfg=conv_cfg, + act_cfg=act_cfg[1]) + + def forward(self, x): + out = self.global_avgpool(x) + out = self.conv1(out) + out = self.conv2(out) + return x * out diff --git a/annotator/mmpkg/mmseg/models/utils/self_attention_block.py b/annotator/mmpkg/mmseg/models/utils/self_attention_block.py new file mode 100644 index 0000000000000000000000000000000000000000..a342e2b29ad53916c98d0342bde8f0f6cb10197a --- /dev/null +++ b/annotator/mmpkg/mmseg/models/utils/self_attention_block.py @@ -0,0 +1,159 @@ +import torch +from annotator.mmpkg.mmcv.cnn import ConvModule, constant_init +from torch import nn as nn +from torch.nn import functional as F + + +class SelfAttentionBlock(nn.Module): + """General self-attention block/non-local block. + + Please refer to https://arxiv.org/abs/1706.03762 for details about key, + query and value. + + Args: + key_in_channels (int): Input channels of key feature. + query_in_channels (int): Input channels of query feature. + channels (int): Output channels of key/query transform. + out_channels (int): Output channels. + share_key_query (bool): Whether share projection weight between key + and query projection. + query_downsample (nn.Module): Query downsample module. + key_downsample (nn.Module): Key downsample module. + key_query_num_convs (int): Number of convs for key/query projection. + value_num_convs (int): Number of convs for value projection. + matmul_norm (bool): Whether normalize attention map with sqrt of + channels + with_out (bool): Whether use out projection. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict|None): Config of activation layers. + """ + + def __init__(self, key_in_channels, query_in_channels, channels, + out_channels, share_key_query, query_downsample, + key_downsample, key_query_num_convs, value_out_num_convs, + key_query_norm, value_out_norm, matmul_norm, with_out, + conv_cfg, norm_cfg, act_cfg): + super(SelfAttentionBlock, self).__init__() + if share_key_query: + assert key_in_channels == query_in_channels + self.key_in_channels = key_in_channels + self.query_in_channels = query_in_channels + self.out_channels = out_channels + self.channels = channels + self.share_key_query = share_key_query + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.key_project = self.build_project( + key_in_channels, + channels, + num_convs=key_query_num_convs, + use_conv_module=key_query_norm, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + if share_key_query: + self.query_project = self.key_project + else: + self.query_project = self.build_project( + query_in_channels, + channels, + num_convs=key_query_num_convs, + use_conv_module=key_query_norm, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + self.value_project = self.build_project( + key_in_channels, + channels if with_out else out_channels, + num_convs=value_out_num_convs, + use_conv_module=value_out_norm, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + if with_out: + self.out_project = self.build_project( + channels, + out_channels, + num_convs=value_out_num_convs, + use_conv_module=value_out_norm, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + else: + self.out_project = None + + self.query_downsample = query_downsample + self.key_downsample = key_downsample + self.matmul_norm = matmul_norm + + self.init_weights() + + def init_weights(self): + """Initialize weight of later layer.""" + if self.out_project is not None: + if not isinstance(self.out_project, ConvModule): + constant_init(self.out_project, 0) + + def build_project(self, in_channels, channels, num_convs, use_conv_module, + conv_cfg, norm_cfg, act_cfg): + """Build projection layer for key/query/value/out.""" + if use_conv_module: + convs = [ + ConvModule( + in_channels, + channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + ] + for _ in range(num_convs - 1): + convs.append( + ConvModule( + channels, + channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + else: + convs = [nn.Conv2d(in_channels, channels, 1)] + for _ in range(num_convs - 1): + convs.append(nn.Conv2d(channels, channels, 1)) + if len(convs) > 1: + convs = nn.Sequential(*convs) + else: + convs = convs[0] + return convs + + def forward(self, query_feats, key_feats): + """Forward function.""" + batch_size = query_feats.size(0) + query = self.query_project(query_feats) + if self.query_downsample is not None: + query = self.query_downsample(query) + query = query.reshape(*query.shape[:2], -1) + query = query.permute(0, 2, 1).contiguous() + + key = self.key_project(key_feats) + value = self.value_project(key_feats) + if self.key_downsample is not None: + key = self.key_downsample(key) + value = self.key_downsample(value) + key = key.reshape(*key.shape[:2], -1) + value = value.reshape(*value.shape[:2], -1) + value = value.permute(0, 2, 1).contiguous() + + sim_map = torch.matmul(query, key) + if self.matmul_norm: + sim_map = (self.channels**-.5) * sim_map + sim_map = F.softmax(sim_map, dim=-1) + + context = torch.matmul(sim_map, value) + context = context.permute(0, 2, 1).contiguous() + context = context.reshape(batch_size, -1, *query_feats.shape[2:]) + if self.out_project is not None: + context = self.out_project(context) + return context diff --git a/annotator/mmpkg/mmseg/models/utils/up_conv_block.py b/annotator/mmpkg/mmseg/models/utils/up_conv_block.py new file mode 100644 index 0000000000000000000000000000000000000000..86328011a9704d17e9f9d0d54994719ead5caa56 --- /dev/null +++ b/annotator/mmpkg/mmseg/models/utils/up_conv_block.py @@ -0,0 +1,101 @@ +import torch +import torch.nn as nn +from annotator.mmpkg.mmcv.cnn import ConvModule, build_upsample_layer + + +class UpConvBlock(nn.Module): + """Upsample convolution block in decoder for UNet. + + This upsample convolution block consists of one upsample module + followed by one convolution block. The upsample module expands the + high-level low-resolution feature map and the convolution block fuses + the upsampled high-level low-resolution feature map and the low-level + high-resolution feature map from encoder. + + Args: + conv_block (nn.Sequential): Sequential of convolutional layers. + in_channels (int): Number of input channels of the high-level + skip_channels (int): Number of input channels of the low-level + high-resolution feature map from encoder. + out_channels (int): Number of output channels. + num_convs (int): Number of convolutional layers in the conv_block. + Default: 2. + stride (int): Stride of convolutional layer in conv_block. Default: 1. + dilation (int): Dilation rate of convolutional layer in conv_block. + Default: 1. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + conv_cfg (dict | None): Config dict for convolution layer. + Default: None. + norm_cfg (dict | None): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict | None): Config dict for activation layer in ConvModule. + Default: dict(type='ReLU'). + upsample_cfg (dict): The upsample config of the upsample module in + decoder. Default: dict(type='InterpConv'). If the size of + high-level feature map is the same as that of skip feature map + (low-level feature map from encoder), it does not need upsample the + high-level feature map and the upsample_cfg is None. + dcn (bool): Use deformable convolution in convolutional layer or not. + Default: None. + plugins (dict): plugins for convolutional layers. Default: None. + """ + + def __init__(self, + conv_block, + in_channels, + skip_channels, + out_channels, + num_convs=2, + stride=1, + dilation=1, + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + upsample_cfg=dict(type='InterpConv'), + dcn=None, + plugins=None): + super(UpConvBlock, self).__init__() + assert dcn is None, 'Not implemented yet.' + assert plugins is None, 'Not implemented yet.' + + self.conv_block = conv_block( + in_channels=2 * skip_channels, + out_channels=out_channels, + num_convs=num_convs, + stride=stride, + dilation=dilation, + with_cp=with_cp, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + dcn=None, + plugins=None) + if upsample_cfg is not None: + self.upsample = build_upsample_layer( + cfg=upsample_cfg, + in_channels=in_channels, + out_channels=skip_channels, + with_cp=with_cp, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + else: + self.upsample = ConvModule( + in_channels, + skip_channels, + kernel_size=1, + stride=1, + padding=0, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + + def forward(self, skip, x): + """Forward function.""" + + x = self.upsample(x) + out = torch.cat([skip, x], dim=1) + out = self.conv_block(out) + + return out diff --git a/annotator/mmpkg/mmseg/models/utils/weight_init.py b/annotator/mmpkg/mmseg/models/utils/weight_init.py new file mode 100644 index 0000000000000000000000000000000000000000..38141ba3d61f64ddfc0a31574b4648cbad96d7dd --- /dev/null +++ b/annotator/mmpkg/mmseg/models/utils/weight_init.py @@ -0,0 +1,62 @@ +"""Modified from https://github.com/rwightman/pytorch-image- +models/blob/master/timm/models/layers/drop.py.""" + +import math +import warnings + +import torch + + +def _no_grad_trunc_normal_(tensor, mean, std, a, b): + """Reference: https://people.sc.fsu.edu/~jburkardt/presentations + /truncated_normal.pdf""" + + def norm_cdf(x): + # Computes standard normal cumulative distribution function + return (1. + math.erf(x / math.sqrt(2.))) / 2. + + if (mean < a - 2 * std) or (mean > b + 2 * std): + warnings.warn( + 'mean is more than 2 std from [a, b] in nn.init.trunc_normal_. ' + 'The distribution of values may be incorrect.', + stacklevel=2) + + with torch.no_grad(): + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + lower_bound = norm_cdf((a - mean) / std) + upper_bound = norm_cdf((b - mean) / std) + + # Uniformly fill tensor with values from [l, u], then translate to + # [2l-1, 2u-1]. + tensor.uniform_(2 * lower_bound - 1, 2 * upper_bound - 1) + + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() + + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.)) + tensor.add_(mean) + + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor + + +def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.): + r"""Fills the input Tensor with values drawn from a truncated + normal distribution. The values are effectively drawn from the + normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)` + with values outside :math:`[a, b]` redrawn until they are within + the bounds. The method used for generating the random values works + best when :math:`a \leq \text{mean} \leq b`. + Args: + tensor (``torch.Tensor``): an n-dimensional `torch.Tensor` + mean (float): the mean of the normal distribution + std (float): the standard deviation of the normal distribution + a (float): the minimum cutoff value + b (float): the maximum cutoff value + """ + return _no_grad_trunc_normal_(tensor, mean, std, a, b) diff --git a/annotator/mmpkg/mmseg/ops/__init__.py b/annotator/mmpkg/mmseg/ops/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bec51c75b9363a9a19e9fb5c35f4e7dbd6f7751c --- /dev/null +++ b/annotator/mmpkg/mmseg/ops/__init__.py @@ -0,0 +1,4 @@ +from .encoding import Encoding +from .wrappers import Upsample, resize + +__all__ = ['Upsample', 'resize', 'Encoding'] diff --git a/annotator/mmpkg/mmseg/ops/encoding.py b/annotator/mmpkg/mmseg/ops/encoding.py new file mode 100644 index 0000000000000000000000000000000000000000..7eb3629a6426550b8e4c537ee1ff4341893e489e --- /dev/null +++ b/annotator/mmpkg/mmseg/ops/encoding.py @@ -0,0 +1,74 @@ +import torch +from torch import nn +from torch.nn import functional as F + + +class Encoding(nn.Module): + """Encoding Layer: a learnable residual encoder. + + Input is of shape (batch_size, channels, height, width). + Output is of shape (batch_size, num_codes, channels). + + Args: + channels: dimension of the features or feature channels + num_codes: number of code words + """ + + def __init__(self, channels, num_codes): + super(Encoding, self).__init__() + # init codewords and smoothing factor + self.channels, self.num_codes = channels, num_codes + std = 1. / ((num_codes * channels)**0.5) + # [num_codes, channels] + self.codewords = nn.Parameter( + torch.empty(num_codes, channels, + dtype=torch.float).uniform_(-std, std), + requires_grad=True) + # [num_codes] + self.scale = nn.Parameter( + torch.empty(num_codes, dtype=torch.float).uniform_(-1, 0), + requires_grad=True) + + @staticmethod + def scaled_l2(x, codewords, scale): + num_codes, channels = codewords.size() + batch_size = x.size(0) + reshaped_scale = scale.view((1, 1, num_codes)) + expanded_x = x.unsqueeze(2).expand( + (batch_size, x.size(1), num_codes, channels)) + reshaped_codewords = codewords.view((1, 1, num_codes, channels)) + + scaled_l2_norm = reshaped_scale * ( + expanded_x - reshaped_codewords).pow(2).sum(dim=3) + return scaled_l2_norm + + @staticmethod + def aggregate(assignment_weights, x, codewords): + num_codes, channels = codewords.size() + reshaped_codewords = codewords.view((1, 1, num_codes, channels)) + batch_size = x.size(0) + + expanded_x = x.unsqueeze(2).expand( + (batch_size, x.size(1), num_codes, channels)) + encoded_feat = (assignment_weights.unsqueeze(3) * + (expanded_x - reshaped_codewords)).sum(dim=1) + return encoded_feat + + def forward(self, x): + assert x.dim() == 4 and x.size(1) == self.channels + # [batch_size, channels, height, width] + batch_size = x.size(0) + # [batch_size, height x width, channels] + x = x.view(batch_size, self.channels, -1).transpose(1, 2).contiguous() + # assignment_weights: [batch_size, channels, num_codes] + assignment_weights = F.softmax( + self.scaled_l2(x, self.codewords, self.scale), dim=2) + # aggregate + encoded_feat = self.aggregate(assignment_weights, x, self.codewords) + return encoded_feat + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(Nx{self.channels}xHxW =>Nx{self.num_codes}' \ + f'x{self.channels})' + return repr_str diff --git a/annotator/mmpkg/mmseg/ops/wrappers.py b/annotator/mmpkg/mmseg/ops/wrappers.py new file mode 100644 index 0000000000000000000000000000000000000000..0ed9a0cb8d7c0e0ec2748dd89c652756653cac78 --- /dev/null +++ b/annotator/mmpkg/mmseg/ops/wrappers.py @@ -0,0 +1,50 @@ +import warnings + +import torch.nn as nn +import torch.nn.functional as F + + +def resize(input, + size=None, + scale_factor=None, + mode='nearest', + align_corners=None, + warning=True): + if warning: + if size is not None and align_corners: + input_h, input_w = tuple(int(x) for x in input.shape[2:]) + output_h, output_w = tuple(int(x) for x in size) + if output_h > input_h or output_w > output_h: + if ((output_h > 1 and output_w > 1 and input_h > 1 + and input_w > 1) and (output_h - 1) % (input_h - 1) + and (output_w - 1) % (input_w - 1)): + warnings.warn( + f'When align_corners={align_corners}, ' + 'the output would more aligned if ' + f'input size {(input_h, input_w)} is `x+1` and ' + f'out size {(output_h, output_w)} is `nx+1`') + return F.interpolate(input, size, scale_factor, mode, align_corners) + + +class Upsample(nn.Module): + + def __init__(self, + size=None, + scale_factor=None, + mode='nearest', + align_corners=None): + super(Upsample, self).__init__() + self.size = size + if isinstance(scale_factor, tuple): + self.scale_factor = tuple(float(factor) for factor in scale_factor) + else: + self.scale_factor = float(scale_factor) if scale_factor else None + self.mode = mode + self.align_corners = align_corners + + def forward(self, x): + if not self.size: + size = [int(t * self.scale_factor) for t in x.shape[-2:]] + else: + size = self.size + return resize(x, size, None, self.mode, self.align_corners) diff --git a/annotator/mmpkg/mmseg/utils/__init__.py b/annotator/mmpkg/mmseg/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ac489e2dbbc0e6fa87f5088b4edcc20f8cadc1a6 --- /dev/null +++ b/annotator/mmpkg/mmseg/utils/__init__.py @@ -0,0 +1,4 @@ +from .collect_env import collect_env +from .logger import get_root_logger + +__all__ = ['get_root_logger', 'collect_env'] diff --git a/annotator/mmpkg/mmseg/utils/collect_env.py b/annotator/mmpkg/mmseg/utils/collect_env.py new file mode 100644 index 0000000000000000000000000000000000000000..015d5a6b4f3ff31859cca36584879f646b3864d4 --- /dev/null +++ b/annotator/mmpkg/mmseg/utils/collect_env.py @@ -0,0 +1,17 @@ +from annotator.mmpkg.mmcv.utils import collect_env as collect_base_env +from annotator.mmpkg.mmcv.utils import get_git_hash + +import annotator.mmpkg.mmseg as mmseg + + +def collect_env(): + """Collect the information of the running environments.""" + env_info = collect_base_env() + env_info['MMSegmentation'] = f'{mmseg.__version__}+{get_git_hash()[:7]}' + + return env_info + + +if __name__ == '__main__': + for name, val in collect_env().items(): + print('{}: {}'.format(name, val)) diff --git a/annotator/mmpkg/mmseg/utils/logger.py b/annotator/mmpkg/mmseg/utils/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..0c37733358e3e21479b41f54220bfe34b482009c --- /dev/null +++ b/annotator/mmpkg/mmseg/utils/logger.py @@ -0,0 +1,27 @@ +import logging + +from annotator.mmpkg.mmcv.utils import get_logger + + +def get_root_logger(log_file=None, log_level=logging.INFO): + """Get the root logger. + + The logger will be initialized if it has not been initialized. By default a + StreamHandler will be added. If `log_file` is specified, a FileHandler will + also be added. The name of the root logger is the top-level package name, + e.g., "mmseg". + + Args: + log_file (str | None): The log filename. If specified, a FileHandler + will be added to the root logger. + log_level (int): The root logger level. Note that only the process of + rank 0 is affected, while other processes will set the level to + "Error" and be silent most of the time. + + Returns: + logging.Logger: The root logger. + """ + + logger = get_logger(name='mmseg', log_file=log_file, log_level=log_level) + + return logger diff --git a/annotator/normalbae/LICENSE b/annotator/normalbae/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..16a9d56a3d4c15e4f34ac5426459c58487b01520 --- /dev/null +++ b/annotator/normalbae/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Caroline Chan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/annotator/normalbae/__init__.py b/annotator/normalbae/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1d8cf903ce1e2d0e9166e9a0a0626c79f475bb75 --- /dev/null +++ b/annotator/normalbae/__init__.py @@ -0,0 +1,55 @@ +# Estimating and Exploiting the Aleatoric Uncertainty in Surface Normal Estimation +# https://github.com/baegwangbin/surface_normal_uncertainty + +import os +import types +import torch +import numpy as np + +from einops import rearrange +from .models.NNET import NNET +from .utils import utils +from annotator.util import annotator_ckpts_path +import torchvision.transforms as transforms + + +class NormalBaeDetector: + def __init__(self): + remote_model_path = "https://huggingface.co./lllyasviel/Annotators/resolve/main/scannet.pt" + modelpath = os.path.join(annotator_ckpts_path, "scannet.pt") + if not os.path.exists(modelpath): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(remote_model_path, model_dir=annotator_ckpts_path) + args = types.SimpleNamespace() + args.mode = 'client' + args.architecture = 'BN' + args.pretrained = 'scannet' + args.sampling_ratio = 0.4 + args.importance_ratio = 0.7 + model = NNET(args) + model = utils.load_checkpoint(modelpath, model) + model = model.cuda() + model.eval() + self.model = model + self.norm = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + + def __call__(self, input_image): + assert input_image.ndim == 3 + image_normal = input_image + with torch.no_grad(): + image_normal = torch.from_numpy(image_normal).float().cuda() + image_normal = image_normal / 255.0 + image_normal = rearrange(image_normal, 'h w c -> 1 c h w') + image_normal = self.norm(image_normal) + + normal = self.model(image_normal) + normal = normal[0][-1][:, :3] + # d = torch.sum(normal ** 2.0, dim=1, keepdim=True) ** 0.5 + # d = torch.maximum(d, torch.ones_like(d) * 1e-5) + # normal /= d + normal = ((normal + 1) * 0.5).clip(0, 1) + + normal = rearrange(normal[0], 'c h w -> h w c').cpu().numpy() + normal_image = (normal * 255.0).clip(0, 255).astype(np.uint8) + + return normal_image diff --git a/annotator/normalbae/models/NNET.py b/annotator/normalbae/models/NNET.py new file mode 100644 index 0000000000000000000000000000000000000000..3ddbc50c3ac18aa4b7f16779fe3c0133981ecc7a --- /dev/null +++ b/annotator/normalbae/models/NNET.py @@ -0,0 +1,22 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .submodules.encoder import Encoder +from .submodules.decoder import Decoder + + +class NNET(nn.Module): + def __init__(self, args): + super(NNET, self).__init__() + self.encoder = Encoder() + self.decoder = Decoder(args) + + def get_1x_lr_params(self): # lr/10 learning rate + return self.encoder.parameters() + + def get_10x_lr_params(self): # lr learning rate + return self.decoder.parameters() + + def forward(self, img, **kwargs): + return self.decoder(self.encoder(img), **kwargs) \ No newline at end of file diff --git a/annotator/normalbae/models/baseline.py b/annotator/normalbae/models/baseline.py new file mode 100644 index 0000000000000000000000000000000000000000..602d0fbdac1acc9ede9bc1f2e10a5df78831ce9d --- /dev/null +++ b/annotator/normalbae/models/baseline.py @@ -0,0 +1,85 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .submodules.submodules import UpSampleBN, norm_normalize + + +# This is the baseline encoder-decoder we used in the ablation study +class NNET(nn.Module): + def __init__(self, args=None): + super(NNET, self).__init__() + self.encoder = Encoder() + self.decoder = Decoder(num_classes=4) + + def forward(self, x, **kwargs): + out = self.decoder(self.encoder(x), **kwargs) + + # Bilinearly upsample the output to match the input resolution + up_out = F.interpolate(out, size=[x.size(2), x.size(3)], mode='bilinear', align_corners=False) + + # L2-normalize the first three channels / ensure positive value for concentration parameters (kappa) + up_out = norm_normalize(up_out) + return up_out + + def get_1x_lr_params(self): # lr/10 learning rate + return self.encoder.parameters() + + def get_10x_lr_params(self): # lr learning rate + modules = [self.decoder] + for m in modules: + yield from m.parameters() + + +# Encoder +class Encoder(nn.Module): + def __init__(self): + super(Encoder, self).__init__() + + basemodel_name = 'tf_efficientnet_b5_ap' + basemodel = torch.hub.load('rwightman/gen-efficientnet-pytorch', basemodel_name, pretrained=True) + + # Remove last layer + basemodel.global_pool = nn.Identity() + basemodel.classifier = nn.Identity() + + self.original_model = basemodel + + def forward(self, x): + features = [x] + for k, v in self.original_model._modules.items(): + if (k == 'blocks'): + for ki, vi in v._modules.items(): + features.append(vi(features[-1])) + else: + features.append(v(features[-1])) + return features + + +# Decoder (no pixel-wise MLP, no uncertainty-guided sampling) +class Decoder(nn.Module): + def __init__(self, num_classes=4): + super(Decoder, self).__init__() + self.conv2 = nn.Conv2d(2048, 2048, kernel_size=1, stride=1, padding=0) + self.up1 = UpSampleBN(skip_input=2048 + 176, output_features=1024) + self.up2 = UpSampleBN(skip_input=1024 + 64, output_features=512) + self.up3 = UpSampleBN(skip_input=512 + 40, output_features=256) + self.up4 = UpSampleBN(skip_input=256 + 24, output_features=128) + self.conv3 = nn.Conv2d(128, num_classes, kernel_size=3, stride=1, padding=1) + + def forward(self, features): + x_block0, x_block1, x_block2, x_block3, x_block4 = features[4], features[5], features[6], features[8], features[11] + x_d0 = self.conv2(x_block4) + x_d1 = self.up1(x_d0, x_block3) + x_d2 = self.up2(x_d1, x_block2) + x_d3 = self.up3(x_d2, x_block1) + x_d4 = self.up4(x_d3, x_block0) + out = self.conv3(x_d4) + return out + + +if __name__ == '__main__': + model = Baseline() + x = torch.rand(2, 3, 480, 640) + out = model(x) + print(out.shape) diff --git a/annotator/normalbae/models/submodules/decoder.py b/annotator/normalbae/models/submodules/decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..993203d1792311f1c492091eaea3c1ac9088187f --- /dev/null +++ b/annotator/normalbae/models/submodules/decoder.py @@ -0,0 +1,202 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from .submodules import UpSampleBN, UpSampleGN, norm_normalize, sample_points + + +class Decoder(nn.Module): + def __init__(self, args): + super(Decoder, self).__init__() + + # hyper-parameter for sampling + self.sampling_ratio = args.sampling_ratio + self.importance_ratio = args.importance_ratio + + # feature-map + self.conv2 = nn.Conv2d(2048, 2048, kernel_size=1, stride=1, padding=0) + if args.architecture == 'BN': + self.up1 = UpSampleBN(skip_input=2048 + 176, output_features=1024) + self.up2 = UpSampleBN(skip_input=1024 + 64, output_features=512) + self.up3 = UpSampleBN(skip_input=512 + 40, output_features=256) + self.up4 = UpSampleBN(skip_input=256 + 24, output_features=128) + + elif args.architecture == 'GN': + self.up1 = UpSampleGN(skip_input=2048 + 176, output_features=1024) + self.up2 = UpSampleGN(skip_input=1024 + 64, output_features=512) + self.up3 = UpSampleGN(skip_input=512 + 40, output_features=256) + self.up4 = UpSampleGN(skip_input=256 + 24, output_features=128) + + else: + raise Exception('invalid architecture') + + # produces 1/8 res output + self.out_conv_res8 = nn.Conv2d(512, 4, kernel_size=3, stride=1, padding=1) + + # produces 1/4 res output + self.out_conv_res4 = nn.Sequential( + nn.Conv1d(512 + 4, 128, kernel_size=1), nn.ReLU(), + nn.Conv1d(128, 128, kernel_size=1), nn.ReLU(), + nn.Conv1d(128, 128, kernel_size=1), nn.ReLU(), + nn.Conv1d(128, 4, kernel_size=1), + ) + + # produces 1/2 res output + self.out_conv_res2 = nn.Sequential( + nn.Conv1d(256 + 4, 128, kernel_size=1), nn.ReLU(), + nn.Conv1d(128, 128, kernel_size=1), nn.ReLU(), + nn.Conv1d(128, 128, kernel_size=1), nn.ReLU(), + nn.Conv1d(128, 4, kernel_size=1), + ) + + # produces 1/1 res output + self.out_conv_res1 = nn.Sequential( + nn.Conv1d(128 + 4, 128, kernel_size=1), nn.ReLU(), + nn.Conv1d(128, 128, kernel_size=1), nn.ReLU(), + nn.Conv1d(128, 128, kernel_size=1), nn.ReLU(), + nn.Conv1d(128, 4, kernel_size=1), + ) + + def forward(self, features, gt_norm_mask=None, mode='test'): + x_block0, x_block1, x_block2, x_block3, x_block4 = features[4], features[5], features[6], features[8], features[11] + + # generate feature-map + + x_d0 = self.conv2(x_block4) # x_d0 : [2, 2048, 15, 20] 1/32 res + x_d1 = self.up1(x_d0, x_block3) # x_d1 : [2, 1024, 30, 40] 1/16 res + x_d2 = self.up2(x_d1, x_block2) # x_d2 : [2, 512, 60, 80] 1/8 res + x_d3 = self.up3(x_d2, x_block1) # x_d3: [2, 256, 120, 160] 1/4 res + x_d4 = self.up4(x_d3, x_block0) # x_d4: [2, 128, 240, 320] 1/2 res + + # 1/8 res output + out_res8 = self.out_conv_res8(x_d2) # out_res8: [2, 4, 60, 80] 1/8 res output + out_res8 = norm_normalize(out_res8) # out_res8: [2, 4, 60, 80] 1/8 res output + + ################################################################################################################ + # out_res4 + ################################################################################################################ + + if mode == 'train': + # upsampling ... out_res8: [2, 4, 60, 80] -> out_res8_res4: [2, 4, 120, 160] + out_res8_res4 = F.interpolate(out_res8, scale_factor=2, mode='bilinear', align_corners=True) + B, _, H, W = out_res8_res4.shape + + # samples: [B, 1, N, 2] + point_coords_res4, rows_int, cols_int = sample_points(out_res8_res4.detach(), gt_norm_mask, + sampling_ratio=self.sampling_ratio, + beta=self.importance_ratio) + + # output (needed for evaluation / visualization) + out_res4 = out_res8_res4 + + # grid_sample feature-map + feat_res4 = F.grid_sample(x_d2, point_coords_res4, mode='bilinear', align_corners=True) # (B, 512, 1, N) + init_pred = F.grid_sample(out_res8, point_coords_res4, mode='bilinear', align_corners=True) # (B, 4, 1, N) + feat_res4 = torch.cat([feat_res4, init_pred], dim=1) # (B, 512+4, 1, N) + + # prediction (needed to compute loss) + samples_pred_res4 = self.out_conv_res4(feat_res4[:, :, 0, :]) # (B, 4, N) + samples_pred_res4 = norm_normalize(samples_pred_res4) # (B, 4, N) - normalized + + for i in range(B): + out_res4[i, :, rows_int[i, :], cols_int[i, :]] = samples_pred_res4[i, :, :] + + else: + # grid_sample feature-map + feat_map = F.interpolate(x_d2, scale_factor=2, mode='bilinear', align_corners=True) + init_pred = F.interpolate(out_res8, scale_factor=2, mode='bilinear', align_corners=True) + feat_map = torch.cat([feat_map, init_pred], dim=1) # (B, 512+4, H, W) + B, _, H, W = feat_map.shape + + # try all pixels + out_res4 = self.out_conv_res4(feat_map.view(B, 512 + 4, -1)) # (B, 4, N) + out_res4 = norm_normalize(out_res4) # (B, 4, N) - normalized + out_res4 = out_res4.view(B, 4, H, W) + samples_pred_res4 = point_coords_res4 = None + + ################################################################################################################ + # out_res2 + ################################################################################################################ + + if mode == 'train': + + # upsampling ... out_res4: [2, 4, 120, 160] -> out_res4_res2: [2, 4, 240, 320] + out_res4_res2 = F.interpolate(out_res4, scale_factor=2, mode='bilinear', align_corners=True) + B, _, H, W = out_res4_res2.shape + + # samples: [B, 1, N, 2] + point_coords_res2, rows_int, cols_int = sample_points(out_res4_res2.detach(), gt_norm_mask, + sampling_ratio=self.sampling_ratio, + beta=self.importance_ratio) + + # output (needed for evaluation / visualization) + out_res2 = out_res4_res2 + + # grid_sample feature-map + feat_res2 = F.grid_sample(x_d3, point_coords_res2, mode='bilinear', align_corners=True) # (B, 256, 1, N) + init_pred = F.grid_sample(out_res4, point_coords_res2, mode='bilinear', align_corners=True) # (B, 4, 1, N) + feat_res2 = torch.cat([feat_res2, init_pred], dim=1) # (B, 256+4, 1, N) + + # prediction (needed to compute loss) + samples_pred_res2 = self.out_conv_res2(feat_res2[:, :, 0, :]) # (B, 4, N) + samples_pred_res2 = norm_normalize(samples_pred_res2) # (B, 4, N) - normalized + + for i in range(B): + out_res2[i, :, rows_int[i, :], cols_int[i, :]] = samples_pred_res2[i, :, :] + + else: + # grid_sample feature-map + feat_map = F.interpolate(x_d3, scale_factor=2, mode='bilinear', align_corners=True) + init_pred = F.interpolate(out_res4, scale_factor=2, mode='bilinear', align_corners=True) + feat_map = torch.cat([feat_map, init_pred], dim=1) # (B, 512+4, H, W) + B, _, H, W = feat_map.shape + + out_res2 = self.out_conv_res2(feat_map.view(B, 256 + 4, -1)) # (B, 4, N) + out_res2 = norm_normalize(out_res2) # (B, 4, N) - normalized + out_res2 = out_res2.view(B, 4, H, W) + samples_pred_res2 = point_coords_res2 = None + + ################################################################################################################ + # out_res1 + ################################################################################################################ + + if mode == 'train': + # upsampling ... out_res4: [2, 4, 120, 160] -> out_res4_res2: [2, 4, 240, 320] + out_res2_res1 = F.interpolate(out_res2, scale_factor=2, mode='bilinear', align_corners=True) + B, _, H, W = out_res2_res1.shape + + # samples: [B, 1, N, 2] + point_coords_res1, rows_int, cols_int = sample_points(out_res2_res1.detach(), gt_norm_mask, + sampling_ratio=self.sampling_ratio, + beta=self.importance_ratio) + + # output (needed for evaluation / visualization) + out_res1 = out_res2_res1 + + # grid_sample feature-map + feat_res1 = F.grid_sample(x_d4, point_coords_res1, mode='bilinear', align_corners=True) # (B, 128, 1, N) + init_pred = F.grid_sample(out_res2, point_coords_res1, mode='bilinear', align_corners=True) # (B, 4, 1, N) + feat_res1 = torch.cat([feat_res1, init_pred], dim=1) # (B, 128+4, 1, N) + + # prediction (needed to compute loss) + samples_pred_res1 = self.out_conv_res1(feat_res1[:, :, 0, :]) # (B, 4, N) + samples_pred_res1 = norm_normalize(samples_pred_res1) # (B, 4, N) - normalized + + for i in range(B): + out_res1[i, :, rows_int[i, :], cols_int[i, :]] = samples_pred_res1[i, :, :] + + else: + # grid_sample feature-map + feat_map = F.interpolate(x_d4, scale_factor=2, mode='bilinear', align_corners=True) + init_pred = F.interpolate(out_res2, scale_factor=2, mode='bilinear', align_corners=True) + feat_map = torch.cat([feat_map, init_pred], dim=1) # (B, 512+4, H, W) + B, _, H, W = feat_map.shape + + out_res1 = self.out_conv_res1(feat_map.view(B, 128 + 4, -1)) # (B, 4, N) + out_res1 = norm_normalize(out_res1) # (B, 4, N) - normalized + out_res1 = out_res1.view(B, 4, H, W) + samples_pred_res1 = point_coords_res1 = None + + return [out_res8, out_res4, out_res2, out_res1], \ + [out_res8, samples_pred_res4, samples_pred_res2, samples_pred_res1], \ + [None, point_coords_res4, point_coords_res2, point_coords_res1] + diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/BENCHMARK.md b/annotator/normalbae/models/submodules/efficientnet_repo/BENCHMARK.md new file mode 100644 index 0000000000000000000000000000000000000000..6ead7171ce5a5bbd2702f6b5c825dc9808ba5658 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/BENCHMARK.md @@ -0,0 +1,555 @@ +# Model Performance Benchmarks + +All benchmarks run as per: + +``` +python onnx_export.py --model mobilenetv3_100 ./mobilenetv3_100.onnx +python onnx_optimize.py ./mobilenetv3_100.onnx --output mobilenetv3_100-opt.onnx +python onnx_to_caffe.py ./mobilenetv3_100.onnx --c2-prefix mobilenetv3 +python onnx_to_caffe.py ./mobilenetv3_100-opt.onnx --c2-prefix mobilenetv3-opt +python caffe2_benchmark.py --c2-init ./mobilenetv3.init.pb --c2-predict ./mobilenetv3.predict.pb +python caffe2_benchmark.py --c2-init ./mobilenetv3-opt.init.pb --c2-predict ./mobilenetv3-opt.predict.pb +``` + +## EfficientNet-B0 + +### Unoptimized +``` +Main run finished. Milliseconds per iter: 49.2862. Iters per second: 20.2897 +Time per operator type: + 29.7378 ms. 60.5145%. Conv + 12.1785 ms. 24.7824%. Sigmoid + 3.62811 ms. 7.38297%. SpatialBN + 2.98444 ms. 6.07314%. Mul + 0.326902 ms. 0.665225%. AveragePool + 0.197317 ms. 0.401528%. FC + 0.0852877 ms. 0.173555%. Add + 0.0032607 ms. 0.00663532%. Squeeze + 49.1416 ms in Total +FLOP per operator type: + 0.76907 GFLOP. 95.2696%. Conv + 0.0269508 GFLOP. 3.33857%. SpatialBN + 0.00846444 GFLOP. 1.04855%. Mul + 0.002561 GFLOP. 0.317248%. FC + 0.000210112 GFLOP. 0.0260279%. Add + 0.807256 GFLOP in Total +Feature Memory Read per operator type: + 58.5253 MB. 43.0891%. Mul + 43.2015 MB. 31.807%. Conv + 27.2869 MB. 20.0899%. SpatialBN + 5.12912 MB. 3.77631%. FC + 1.6809 MB. 1.23756%. Add + 135.824 MB in Total +Feature Memory Written per operator type: + 33.8578 MB. 38.1965%. Mul + 26.9881 MB. 30.4465%. Conv + 26.9508 MB. 30.4044%. SpatialBN + 0.840448 MB. 0.948147%. Add + 0.004 MB. 0.00451258%. FC + 88.6412 MB in Total +Parameter Memory per operator type: + 15.8248 MB. 74.9391%. Conv + 5.124 MB. 24.265%. FC + 0.168064 MB. 0.795877%. SpatialBN + 0 MB. 0%. Add + 0 MB. 0%. Mul + 21.1168 MB in Total +``` +### Optimized +``` +Main run finished. Milliseconds per iter: 46.0838. Iters per second: 21.6996 +Time per operator type: + 29.776 ms. 65.002%. Conv + 12.2803 ms. 26.8084%. Sigmoid + 3.15073 ms. 6.87815%. Mul + 0.328651 ms. 0.717456%. AveragePool + 0.186237 ms. 0.406563%. FC + 0.0832429 ms. 0.181722%. Add + 0.0026184 ms. 0.00571606%. Squeeze + 45.8078 ms in Total +FLOP per operator type: + 0.76907 GFLOP. 98.5601%. Conv + 0.00846444 GFLOP. 1.08476%. Mul + 0.002561 GFLOP. 0.328205%. FC + 0.000210112 GFLOP. 0.0269269%. Add + 0.780305 GFLOP in Total +Feature Memory Read per operator type: + 58.5253 MB. 53.8803%. Mul + 43.2855 MB. 39.8501%. Conv + 5.12912 MB. 4.72204%. FC + 1.6809 MB. 1.54749%. Add + 108.621 MB in Total +Feature Memory Written per operator type: + 33.8578 MB. 54.8834%. Mul + 26.9881 MB. 43.7477%. Conv + 0.840448 MB. 1.36237%. Add + 0.004 MB. 0.00648399%. FC + 61.6904 MB in Total +Parameter Memory per operator type: + 15.8248 MB. 75.5403%. Conv + 5.124 MB. 24.4597%. FC + 0 MB. 0%. Add + 0 MB. 0%. Mul + 20.9488 MB in Total +``` + +## EfficientNet-B1 +### Optimized +``` +Main run finished. Milliseconds per iter: 71.8102. Iters per second: 13.9256 +Time per operator type: + 45.7915 ms. 66.3206%. Conv + 17.8718 ms. 25.8841%. Sigmoid + 4.44132 ms. 6.43244%. Mul + 0.51001 ms. 0.738658%. AveragePool + 0.233283 ms. 0.337868%. Add + 0.194986 ms. 0.282402%. FC + 0.00268255 ms. 0.00388519%. Squeeze + 69.0456 ms in Total +FLOP per operator type: + 1.37105 GFLOP. 98.7673%. Conv + 0.0138759 GFLOP. 0.99959%. Mul + 0.002561 GFLOP. 0.184489%. FC + 0.000674432 GFLOP. 0.0485847%. Add + 1.38816 GFLOP in Total +Feature Memory Read per operator type: + 94.624 MB. 54.0789%. Mul + 69.8255 MB. 39.9062%. Conv + 5.39546 MB. 3.08357%. Add + 5.12912 MB. 2.93136%. FC + 174.974 MB in Total +Feature Memory Written per operator type: + 55.5035 MB. 54.555%. Mul + 43.5333 MB. 42.7894%. Conv + 2.69773 MB. 2.65163%. Add + 0.004 MB. 0.00393165%. FC + 101.739 MB in Total +Parameter Memory per operator type: + 25.7479 MB. 83.4024%. Conv + 5.124 MB. 16.5976%. FC + 0 MB. 0%. Add + 0 MB. 0%. Mul + 30.8719 MB in Total +``` + +## EfficientNet-B2 +### Optimized +``` +Main run finished. Milliseconds per iter: 92.28. Iters per second: 10.8366 +Time per operator type: + 61.4627 ms. 67.5845%. Conv + 22.7458 ms. 25.0113%. Sigmoid + 5.59931 ms. 6.15701%. Mul + 0.642567 ms. 0.706568%. AveragePool + 0.272795 ms. 0.299965%. Add + 0.216178 ms. 0.237709%. FC + 0.00268895 ms. 0.00295677%. Squeeze + 90.942 ms in Total +FLOP per operator type: + 1.98431 GFLOP. 98.9343%. Conv + 0.0177039 GFLOP. 0.882686%. Mul + 0.002817 GFLOP. 0.140451%. FC + 0.000853984 GFLOP. 0.0425782%. Add + 2.00568 GFLOP in Total +Feature Memory Read per operator type: + 120.609 MB. 54.9637%. Mul + 86.3512 MB. 39.3519%. Conv + 6.83187 MB. 3.11341%. Add + 5.64163 MB. 2.571%. FC + 219.433 MB in Total +Feature Memory Written per operator type: + 70.8155 MB. 54.6573%. Mul + 55.3273 MB. 42.7031%. Conv + 3.41594 MB. 2.63651%. Add + 0.004 MB. 0.00308731%. FC + 129.563 MB in Total +Parameter Memory per operator type: + 30.4721 MB. 84.3913%. Conv + 5.636 MB. 15.6087%. FC + 0 MB. 0%. Add + 0 MB. 0%. Mul + 36.1081 MB in Total +``` + +## MixNet-M +### Optimized +``` +Main run finished. Milliseconds per iter: 63.1122. Iters per second: 15.8448 +Time per operator type: + 48.1139 ms. 75.2052%. Conv + 7.1341 ms. 11.1511%. Sigmoid + 2.63706 ms. 4.12189%. SpatialBN + 1.73186 ms. 2.70701%. Mul + 1.38707 ms. 2.16809%. Split + 1.29322 ms. 2.02139%. Concat + 1.00093 ms. 1.56452%. Relu + 0.235309 ms. 0.367803%. Add + 0.221579 ms. 0.346343%. FC + 0.219315 ms. 0.342803%. AveragePool + 0.00250145 ms. 0.00390993%. Squeeze + 63.9768 ms in Total +FLOP per operator type: + 0.675273 GFLOP. 95.5827%. Conv + 0.0221072 GFLOP. 3.12921%. SpatialBN + 0.00538445 GFLOP. 0.762152%. Mul + 0.003073 GFLOP. 0.434973%. FC + 0.000642488 GFLOP. 0.0909421%. Add + 0 GFLOP. 0%. Concat + 0 GFLOP. 0%. Relu + 0.70648 GFLOP in Total +Feature Memory Read per operator type: + 46.8424 MB. 30.502%. Conv + 36.8626 MB. 24.0036%. Mul + 22.3152 MB. 14.5309%. SpatialBN + 22.1074 MB. 14.3955%. Concat + 14.1496 MB. 9.21372%. Relu + 6.15414 MB. 4.00735%. FC + 5.1399 MB. 3.34692%. Add + 153.571 MB in Total +Feature Memory Written per operator type: + 32.7672 MB. 28.4331%. Conv + 22.1072 MB. 19.1831%. Concat + 22.1072 MB. 19.1831%. SpatialBN + 21.5378 MB. 18.689%. Mul + 14.1496 MB. 12.2781%. Relu + 2.56995 MB. 2.23003%. Add + 0.004 MB. 0.00347092%. FC + 115.243 MB in Total +Parameter Memory per operator type: + 13.7059 MB. 68.674%. Conv + 6.148 MB. 30.8049%. FC + 0.104 MB. 0.521097%. SpatialBN + 0 MB. 0%. Add + 0 MB. 0%. Concat + 0 MB. 0%. Mul + 0 MB. 0%. Relu + 19.9579 MB in Total +``` + +## TF MobileNet-V3 Large 1.0 + +### Optimized +``` +Main run finished. Milliseconds per iter: 22.0495. Iters per second: 45.3525 +Time per operator type: + 17.437 ms. 80.0087%. Conv + 1.27662 ms. 5.8577%. Add + 1.12759 ms. 5.17387%. Div + 0.701155 ms. 3.21721%. Mul + 0.562654 ms. 2.58171%. Relu + 0.431144 ms. 1.97828%. Clip + 0.156902 ms. 0.719936%. FC + 0.0996858 ms. 0.457402%. AveragePool + 0.00112455 ms. 0.00515993%. Flatten + 21.7939 ms in Total +FLOP per operator type: + 0.43062 GFLOP. 98.1484%. Conv + 0.002561 GFLOP. 0.583713%. FC + 0.00210867 GFLOP. 0.480616%. Mul + 0.00193868 GFLOP. 0.441871%. Add + 0.00151532 GFLOP. 0.345377%. Div + 0 GFLOP. 0%. Relu + 0.438743 GFLOP in Total +Feature Memory Read per operator type: + 34.7967 MB. 43.9391%. Conv + 14.496 MB. 18.3046%. Mul + 9.44828 MB. 11.9307%. Add + 9.26157 MB. 11.6949%. Relu + 6.0614 MB. 7.65395%. Div + 5.12912 MB. 6.47673%. FC + 79.193 MB in Total +Feature Memory Written per operator type: + 17.6247 MB. 35.8656%. Conv + 9.26157 MB. 18.847%. Relu + 8.43469 MB. 17.1643%. Mul + 7.75472 MB. 15.7806%. Add + 6.06128 MB. 12.3345%. Div + 0.004 MB. 0.00813985%. FC + 49.1409 MB in Total +Parameter Memory per operator type: + 16.6851 MB. 76.5052%. Conv + 5.124 MB. 23.4948%. FC + 0 MB. 0%. Add + 0 MB. 0%. Div + 0 MB. 0%. Mul + 0 MB. 0%. Relu + 21.8091 MB in Total +``` + +## MobileNet-V3 (RW) + +### Unoptimized +``` +Main run finished. Milliseconds per iter: 24.8316. Iters per second: 40.2712 +Time per operator type: + 15.9266 ms. 69.2624%. Conv + 2.36551 ms. 10.2873%. SpatialBN + 1.39102 ms. 6.04936%. Add + 1.30327 ms. 5.66773%. Div + 0.737014 ms. 3.20517%. Mul + 0.639697 ms. 2.78195%. Relu + 0.375681 ms. 1.63378%. Clip + 0.153126 ms. 0.665921%. FC + 0.0993787 ms. 0.432184%. AveragePool + 0.0032632 ms. 0.0141912%. Squeeze + 22.9946 ms in Total +FLOP per operator type: + 0.430616 GFLOP. 94.4041%. Conv + 0.0175992 GFLOP. 3.85829%. SpatialBN + 0.002561 GFLOP. 0.561449%. FC + 0.00210961 GFLOP. 0.46249%. Mul + 0.00173891 GFLOP. 0.381223%. Add + 0.00151626 GFLOP. 0.33241%. Div + 0 GFLOP. 0%. Relu + 0.456141 GFLOP in Total +Feature Memory Read per operator type: + 34.7354 MB. 36.4363%. Conv + 17.7944 MB. 18.6658%. SpatialBN + 14.5035 MB. 15.2137%. Mul + 9.25778 MB. 9.71113%. Relu + 7.84641 MB. 8.23064%. Add + 6.06516 MB. 6.36216%. Div + 5.12912 MB. 5.38029%. FC + 95.3317 MB in Total +Feature Memory Written per operator type: + 17.6246 MB. 26.7264%. Conv + 17.5992 MB. 26.6878%. SpatialBN + 9.25778 MB. 14.0387%. Relu + 8.43843 MB. 12.7962%. Mul + 6.95565 MB. 10.5477%. Add + 6.06502 MB. 9.19713%. Div + 0.004 MB. 0.00606568%. FC + 65.9447 MB in Total +Parameter Memory per operator type: + 16.6778 MB. 76.1564%. Conv + 5.124 MB. 23.3979%. FC + 0.0976 MB. 0.445674%. SpatialBN + 0 MB. 0%. Add + 0 MB. 0%. Div + 0 MB. 0%. Mul + 0 MB. 0%. Relu + 21.8994 MB in Total + +``` +### Optimized + +``` +Main run finished. Milliseconds per iter: 22.0981. Iters per second: 45.2527 +Time per operator type: + 17.146 ms. 78.8965%. Conv + 1.38453 ms. 6.37084%. Add + 1.30991 ms. 6.02749%. Div + 0.685417 ms. 3.15391%. Mul + 0.532589 ms. 2.45068%. Relu + 0.418263 ms. 1.92461%. Clip + 0.15128 ms. 0.696106%. FC + 0.102065 ms. 0.469648%. AveragePool + 0.0022143 ms. 0.010189%. Squeeze + 21.7323 ms in Total +FLOP per operator type: + 0.430616 GFLOP. 98.1927%. Conv + 0.002561 GFLOP. 0.583981%. FC + 0.00210961 GFLOP. 0.481051%. Mul + 0.00173891 GFLOP. 0.396522%. Add + 0.00151626 GFLOP. 0.34575%. Div + 0 GFLOP. 0%. Relu + 0.438542 GFLOP in Total +Feature Memory Read per operator type: + 34.7842 MB. 44.833%. Conv + 14.5035 MB. 18.6934%. Mul + 9.25778 MB. 11.9323%. Relu + 7.84641 MB. 10.1132%. Add + 6.06516 MB. 7.81733%. Div + 5.12912 MB. 6.61087%. FC + 77.5861 MB in Total +Feature Memory Written per operator type: + 17.6246 MB. 36.4556%. Conv + 9.25778 MB. 19.1492%. Relu + 8.43843 MB. 17.4544%. Mul + 6.95565 MB. 14.3874%. Add + 6.06502 MB. 12.5452%. Div + 0.004 MB. 0.00827378%. FC + 48.3455 MB in Total +Parameter Memory per operator type: + 16.6778 MB. 76.4973%. Conv + 5.124 MB. 23.5027%. FC + 0 MB. 0%. Add + 0 MB. 0%. Div + 0 MB. 0%. Mul + 0 MB. 0%. Relu + 21.8018 MB in Total + +``` + +## MnasNet-A1 + +### Unoptimized +``` +Main run finished. Milliseconds per iter: 30.0892. Iters per second: 33.2345 +Time per operator type: + 24.4656 ms. 79.0905%. Conv + 4.14958 ms. 13.4144%. SpatialBN + 1.60598 ms. 5.19169%. Relu + 0.295219 ms. 0.95436%. Mul + 0.187609 ms. 0.606486%. FC + 0.120556 ms. 0.389724%. AveragePool + 0.09036 ms. 0.292109%. Add + 0.015727 ms. 0.050841%. Sigmoid + 0.00306205 ms. 0.00989875%. Squeeze + 30.9337 ms in Total +FLOP per operator type: + 0.620598 GFLOP. 95.6434%. Conv + 0.0248873 GFLOP. 3.8355%. SpatialBN + 0.002561 GFLOP. 0.394688%. FC + 0.000597408 GFLOP. 0.0920695%. Mul + 0.000222656 GFLOP. 0.0343146%. Add + 0 GFLOP. 0%. Relu + 0.648867 GFLOP in Total +Feature Memory Read per operator type: + 35.5457 MB. 38.4109%. Conv + 25.1552 MB. 27.1829%. SpatialBN + 22.5235 MB. 24.339%. Relu + 5.12912 MB. 5.54256%. FC + 2.40586 MB. 2.59978%. Mul + 1.78125 MB. 1.92483%. Add + 92.5406 MB in Total +Feature Memory Written per operator type: + 24.9042 MB. 32.9424%. Conv + 24.8873 MB. 32.92%. SpatialBN + 22.5235 MB. 29.7932%. Relu + 2.38963 MB. 3.16092%. Mul + 0.890624 MB. 1.17809%. Add + 0.004 MB. 0.00529106%. FC + 75.5993 MB in Total +Parameter Memory per operator type: + 10.2732 MB. 66.1459%. Conv + 5.124 MB. 32.9917%. FC + 0.133952 MB. 0.86247%. SpatialBN + 0 MB. 0%. Add + 0 MB. 0%. Mul + 0 MB. 0%. Relu + 15.5312 MB in Total +``` + +### Optimized +``` +Main run finished. Milliseconds per iter: 24.2367. Iters per second: 41.2597 +Time per operator type: + 22.0547 ms. 91.1375%. Conv + 1.49096 ms. 6.16116%. Relu + 0.253417 ms. 1.0472%. Mul + 0.18506 ms. 0.76473%. FC + 0.112942 ms. 0.466717%. AveragePool + 0.086769 ms. 0.358559%. Add + 0.0127889 ms. 0.0528479%. Sigmoid + 0.0027346 ms. 0.0113003%. Squeeze + 24.1994 ms in Total +FLOP per operator type: + 0.620598 GFLOP. 99.4581%. Conv + 0.002561 GFLOP. 0.41043%. FC + 0.000597408 GFLOP. 0.0957417%. Mul + 0.000222656 GFLOP. 0.0356832%. Add + 0 GFLOP. 0%. Relu + 0.623979 GFLOP in Total +Feature Memory Read per operator type: + 35.6127 MB. 52.7968%. Conv + 22.5235 MB. 33.3917%. Relu + 5.12912 MB. 7.60406%. FC + 2.40586 MB. 3.56675%. Mul + 1.78125 MB. 2.64075%. Add + 67.4524 MB in Total +Feature Memory Written per operator type: + 24.9042 MB. 49.1092%. Conv + 22.5235 MB. 44.4145%. Relu + 2.38963 MB. 4.71216%. Mul + 0.890624 MB. 1.75624%. Add + 0.004 MB. 0.00788768%. FC + 50.712 MB in Total +Parameter Memory per operator type: + 10.2732 MB. 66.7213%. Conv + 5.124 MB. 33.2787%. FC + 0 MB. 0%. Add + 0 MB. 0%. Mul + 0 MB. 0%. Relu + 15.3972 MB in Total +``` +## MnasNet-B1 + +### Unoptimized +``` +Main run finished. Milliseconds per iter: 28.3109. Iters per second: 35.322 +Time per operator type: + 29.1121 ms. 83.3081%. Conv + 4.14959 ms. 11.8746%. SpatialBN + 1.35823 ms. 3.88675%. Relu + 0.186188 ms. 0.532802%. FC + 0.116244 ms. 0.332647%. Add + 0.018641 ms. 0.0533437%. AveragePool + 0.0040904 ms. 0.0117052%. Squeeze + 34.9451 ms in Total +FLOP per operator type: + 0.626272 GFLOP. 96.2088%. Conv + 0.0218266 GFLOP. 3.35303%. SpatialBN + 0.002561 GFLOP. 0.393424%. FC + 0.000291648 GFLOP. 0.0448034%. Add + 0 GFLOP. 0%. Relu + 0.650951 GFLOP in Total +Feature Memory Read per operator type: + 34.4354 MB. 41.3788%. Conv + 22.1299 MB. 26.5921%. SpatialBN + 19.1923 MB. 23.0622%. Relu + 5.12912 MB. 6.16333%. FC + 2.33318 MB. 2.80364%. Add + 83.2199 MB in Total +Feature Memory Written per operator type: + 21.8266 MB. 34.0955%. Conv + 21.8266 MB. 34.0955%. SpatialBN + 19.1923 MB. 29.9805%. Relu + 1.16659 MB. 1.82234%. Add + 0.004 MB. 0.00624844%. FC + 64.016 MB in Total +Parameter Memory per operator type: + 12.2576 MB. 69.9104%. Conv + 5.124 MB. 29.2245%. FC + 0.15168 MB. 0.865099%. SpatialBN + 0 MB. 0%. Add + 0 MB. 0%. Relu + 17.5332 MB in Total +``` + +### Optimized +``` +Main run finished. Milliseconds per iter: 26.6364. Iters per second: 37.5426 +Time per operator type: + 24.9888 ms. 94.0962%. Conv + 1.26147 ms. 4.75011%. Relu + 0.176234 ms. 0.663619%. FC + 0.113309 ms. 0.426672%. Add + 0.0138708 ms. 0.0522311%. AveragePool + 0.00295685 ms. 0.0111341%. Squeeze + 26.5566 ms in Total +FLOP per operator type: + 0.626272 GFLOP. 99.5466%. Conv + 0.002561 GFLOP. 0.407074%. FC + 0.000291648 GFLOP. 0.0463578%. Add + 0 GFLOP. 0%. Relu + 0.629124 GFLOP in Total +Feature Memory Read per operator type: + 34.5112 MB. 56.4224%. Conv + 19.1923 MB. 31.3775%. Relu + 5.12912 MB. 8.3856%. FC + 2.33318 MB. 3.81452%. Add + 61.1658 MB in Total +Feature Memory Written per operator type: + 21.8266 MB. 51.7346%. Conv + 19.1923 MB. 45.4908%. Relu + 1.16659 MB. 2.76513%. Add + 0.004 MB. 0.00948104%. FC + 42.1895 MB in Total +Parameter Memory per operator type: + 12.2576 MB. 70.5205%. Conv + 5.124 MB. 29.4795%. FC + 0 MB. 0%. Add + 0 MB. 0%. Relu + 17.3816 MB in Total +``` diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/LICENSE b/annotator/normalbae/models/submodules/efficientnet_repo/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..80e7d15508202f3262a50db27f5198460d7f509f --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 Ross Wightman + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/README.md b/annotator/normalbae/models/submodules/efficientnet_repo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..463368280d6a5015060eb73d20fe6512f8e04c50 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/README.md @@ -0,0 +1,323 @@ +# (Generic) EfficientNets for PyTorch + +A 'generic' implementation of EfficientNet, MixNet, MobileNetV3, etc. that covers most of the compute/parameter efficient architectures derived from the MobileNet V1/V2 block sequence, including those found via automated neural architecture search. + +All models are implemented by GenEfficientNet or MobileNetV3 classes, with string based architecture definitions to configure the block layouts (idea from [here](https://github.com/tensorflow/tpu/blob/master/models/official/mnasnet/mnasnet_models.py)) + +## What's New + +### Aug 19, 2020 +* Add updated PyTorch trained EfficientNet-B3 weights trained by myself with `timm` (82.1 top-1) +* Add PyTorch trained EfficientNet-Lite0 contributed by [@hal-314](https://github.com/hal-314) (75.5 top-1) +* Update ONNX and Caffe2 export / utility scripts to work with latest PyTorch / ONNX +* ONNX runtime based validation script added +* activations (mostly) brought in sync with `timm` equivalents + + +### April 5, 2020 +* Add some newly trained MobileNet-V2 models trained with latest h-params, rand augment. They compare quite favourably to EfficientNet-Lite + * 3.5M param MobileNet-V2 100 @ 73% + * 4.5M param MobileNet-V2 110d @ 75% + * 6.1M param MobileNet-V2 140 @ 76.5% + * 5.8M param MobileNet-V2 120d @ 77.3% + +### March 23, 2020 + * Add EfficientNet-Lite models w/ weights ported from [Tensorflow TPU](https://github.com/tensorflow/tpu/tree/master/models/official/efficientnet/lite) + * Add PyTorch trained MobileNet-V3 Large weights with 75.77% top-1 + * IMPORTANT CHANGE (if training from scratch) - weight init changed to better match Tensorflow impl, set `fix_group_fanout=False` in `initialize_weight_goog` for old behavior + +### Feb 12, 2020 + * Add EfficientNet-L2 and B0-B7 NoisyStudent weights ported from [Tensorflow TPU](https://github.com/tensorflow/tpu/tree/master/models/official/efficientnet) + * Port new EfficientNet-B8 (RandAugment) weights from TF TPU, these are different than the B8 AdvProp, different input normalization. + * Add RandAugment PyTorch trained EfficientNet-ES (EdgeTPU-Small) weights with 78.1 top-1. Trained by [Andrew Lavin](https://github.com/andravin) + +### Jan 22, 2020 + * Update weights for EfficientNet B0, B2, B3 and MixNet-XL with latest RandAugment trained weights. Trained with (https://github.com/rwightman/pytorch-image-models) + * Fix torchscript compatibility for PyTorch 1.4, add torchscript support for MixedConv2d using ModuleDict + * Test models, torchscript, onnx export with PyTorch 1.4 -- no issues + +### Nov 22, 2019 + * New top-1 high! Ported official TF EfficientNet AdvProp (https://arxiv.org/abs/1911.09665) weights and B8 model spec. Created a new set of `ap` models since they use a different + preprocessing (Inception mean/std) from the original EfficientNet base/AA/RA weights. + +### Nov 15, 2019 + * Ported official TF MobileNet-V3 float32 large/small/minimalistic weights + * Modifications to MobileNet-V3 model and components to support some additional config needed for differences between TF MobileNet-V3 and mine + +### Oct 30, 2019 + * Many of the models will now work with torch.jit.script, MixNet being the biggest exception + * Improved interface for enabling torchscript or ONNX export compatible modes (via config) + * Add JIT optimized mem-efficient Swish/Mish autograd.fn in addition to memory-efficient autgrad.fn + * Activation factory to select best version of activation by name or override one globally + * Add pretrained checkpoint load helper that handles input conv and classifier changes + +### Oct 27, 2019 + * Add CondConv EfficientNet variants ported from https://github.com/tensorflow/tpu/tree/master/models/official/efficientnet/condconv + * Add RandAug weights for TF EfficientNet B5 and B7 from https://github.com/tensorflow/tpu/tree/master/models/official/efficientnet + * Bring over MixNet-XL model and depth scaling algo from my pytorch-image-models code base + * Switch activations and global pooling to modules + * Add memory-efficient Swish/Mish impl + * Add as_sequential() method to all models and allow as an argument in entrypoint fns + * Move MobileNetV3 into own file since it has a different head + * Remove ChamNet, MobileNet V2/V1 since they will likely never be used here + +## Models + +Implemented models include: + * EfficientNet NoisyStudent (B0-B7, L2) (https://arxiv.org/abs/1911.04252) + * EfficientNet AdvProp (B0-B8) (https://arxiv.org/abs/1911.09665) + * EfficientNet (B0-B8) (https://arxiv.org/abs/1905.11946) + * EfficientNet-EdgeTPU (S, M, L) (https://ai.googleblog.com/2019/08/efficientnet-edgetpu-creating.html) + * EfficientNet-CondConv (https://arxiv.org/abs/1904.04971) + * EfficientNet-Lite (https://github.com/tensorflow/tpu/tree/master/models/official/efficientnet/lite) + * MixNet (https://arxiv.org/abs/1907.09595) + * MNASNet B1, A1 (Squeeze-Excite), and Small (https://arxiv.org/abs/1807.11626) + * MobileNet-V3 (https://arxiv.org/abs/1905.02244) + * FBNet-C (https://arxiv.org/abs/1812.03443) + * Single-Path NAS (https://arxiv.org/abs/1904.02877) + +I originally implemented and trained some these models with code [here](https://github.com/rwightman/pytorch-image-models), this repository contains just the GenEfficientNet models, validation, and associated ONNX/Caffe2 export code. + +## Pretrained + +I've managed to train several of the models to accuracies close to or above the originating papers and official impl. My training code is here: https://github.com/rwightman/pytorch-image-models + + +|Model | Prec@1 (Err) | Prec@5 (Err) | Param#(M) | MAdds(M) | Image Scaling | Resolution | Crop | +|---|---|---|---|---|---|---|---| +| efficientnet_b3 | 82.240 (17.760) | 96.116 (3.884) | 12.23 | TBD | bicubic | 320 | 1.0 | +| efficientnet_b3 | 82.076 (17.924) | 96.020 (3.980) | 12.23 | TBD | bicubic | 300 | 0.904 | +| mixnet_xl | 81.074 (18.926) | 95.282 (4.718) | 11.90 | TBD | bicubic | 256 | 1.0 | +| efficientnet_b2 | 80.612 (19.388) | 95.318 (4.682) | 9.1 | TBD | bicubic | 288 | 1.0 | +| mixnet_xl | 80.476 (19.524) | 94.936 (5.064) | 11.90 | TBD | bicubic | 224 | 0.875 | +| efficientnet_b2 | 80.288 (19.712) | 95.166 (4.834) | 9.1 | 1003 | bicubic | 260 | 0.890 | +| mixnet_l | 78.976 (21.024 | 94.184 (5.816) | 7.33 | TBD | bicubic | 224 | 0.875 | +| efficientnet_b1 | 78.692 (21.308) | 94.086 (5.914) | 7.8 | 694 | bicubic | 240 | 0.882 | +| efficientnet_es | 78.066 (21.934) | 93.926 (6.074) | 5.44 | TBD | bicubic | 224 | 0.875 | +| efficientnet_b0 | 77.698 (22.302) | 93.532 (6.468) | 5.3 | 390 | bicubic | 224 | 0.875 | +| mobilenetv2_120d | 77.294 (22.706 | 93.502 (6.498) | 5.8 | TBD | bicubic | 224 | 0.875 | +| mixnet_m | 77.256 (22.744) | 93.418 (6.582) | 5.01 | 353 | bicubic | 224 | 0.875 | +| mobilenetv2_140 | 76.524 (23.476) | 92.990 (7.010) | 6.1 | TBD | bicubic | 224 | 0.875 | +| mixnet_s | 75.988 (24.012) | 92.794 (7.206) | 4.13 | TBD | bicubic | 224 | 0.875 | +| mobilenetv3_large_100 | 75.766 (24.234) | 92.542 (7.458) | 5.5 | TBD | bicubic | 224 | 0.875 | +| mobilenetv3_rw | 75.634 (24.366) | 92.708 (7.292) | 5.5 | 219 | bicubic | 224 | 0.875 | +| efficientnet_lite0 | 75.472 (24.528) | 92.520 (7.480) | 4.65 | TBD | bicubic | 224 | 0.875 | +| mnasnet_a1 | 75.448 (24.552) | 92.604 (7.396) | 3.9 | 312 | bicubic | 224 | 0.875 | +| fbnetc_100 | 75.124 (24.876) | 92.386 (7.614) | 5.6 | 385 | bilinear | 224 | 0.875 | +| mobilenetv2_110d | 75.052 (24.948) | 92.180 (7.820) | 4.5 | TBD | bicubic | 224 | 0.875 | +| mnasnet_b1 | 74.658 (25.342) | 92.114 (7.886) | 4.4 | 315 | bicubic | 224 | 0.875 | +| spnasnet_100 | 74.084 (25.916) | 91.818 (8.182) | 4.4 | TBD | bilinear | 224 | 0.875 | +| mobilenetv2_100 | 72.978 (27.022) | 91.016 (8.984) | 3.5 | TBD | bicubic | 224 | 0.875 | + + +More pretrained models to come... + + +## Ported Weights + +The weights ported from Tensorflow checkpoints for the EfficientNet models do pretty much match accuracy in Tensorflow once a SAME convolution padding equivalent is added, and the same crop factors, image scaling, etc (see table) are used via cmd line args. + +**IMPORTANT:** +* Tensorflow ported weights for EfficientNet AdvProp (AP), EfficientNet EdgeTPU, EfficientNet-CondConv, EfficientNet-Lite, and MobileNet-V3 models use Inception style (0.5, 0.5, 0.5) for mean and std. +* Enabling the Tensorflow preprocessing pipeline with `--tf-preprocessing` at validation time will improve scores by 0.1-0.5%, very close to original TF impl. + +To run validation for tf_efficientnet_b5: +`python validate.py /path/to/imagenet/validation/ --model tf_efficientnet_b5 -b 64 --img-size 456 --crop-pct 0.934 --interpolation bicubic` + +To run validation w/ TF preprocessing for tf_efficientnet_b5: +`python validate.py /path/to/imagenet/validation/ --model tf_efficientnet_b5 -b 64 --img-size 456 --tf-preprocessing` + +To run validation for a model with Inception preprocessing, ie EfficientNet-B8 AdvProp: +`python validate.py /path/to/imagenet/validation/ --model tf_efficientnet_b8_ap -b 48 --num-gpu 2 --img-size 672 --crop-pct 0.954 --mean 0.5 --std 0.5` + +|Model | Prec@1 (Err) | Prec@5 (Err) | Param # | Image Scaling | Image Size | Crop | +|---|---|---|---|---|---|---| +| tf_efficientnet_l2_ns *tfp | 88.352 (11.648) | 98.652 (1.348) | 480 | bicubic | 800 | N/A | +| tf_efficientnet_l2_ns | TBD | TBD | 480 | bicubic | 800 | 0.961 | +| tf_efficientnet_l2_ns_475 | 88.234 (11.766) | 98.546 (1.454) | 480 | bicubic | 475 | 0.936 | +| tf_efficientnet_l2_ns_475 *tfp | 88.172 (11.828) | 98.566 (1.434) | 480 | bicubic | 475 | N/A | +| tf_efficientnet_b7_ns *tfp | 86.844 (13.156) | 98.084 (1.916) | 66.35 | bicubic | 600 | N/A | +| tf_efficientnet_b7_ns | 86.840 (13.160) | 98.094 (1.906) | 66.35 | bicubic | 600 | N/A | +| tf_efficientnet_b6_ns | 86.452 (13.548) | 97.882 (2.118) | 43.04 | bicubic | 528 | N/A | +| tf_efficientnet_b6_ns *tfp | 86.444 (13.556) | 97.880 (2.120) | 43.04 | bicubic | 528 | N/A | +| tf_efficientnet_b5_ns *tfp | 86.064 (13.936) | 97.746 (2.254) | 30.39 | bicubic | 456 | N/A | +| tf_efficientnet_b5_ns | 86.088 (13.912) | 97.752 (2.248) | 30.39 | bicubic | 456 | N/A | +| tf_efficientnet_b8_ap *tfp | 85.436 (14.564) | 97.272 (2.728) | 87.4 | bicubic | 672 | N/A | +| tf_efficientnet_b8 *tfp | 85.384 (14.616) | 97.394 (2.606) | 87.4 | bicubic | 672 | N/A | +| tf_efficientnet_b8 | 85.370 (14.630) | 97.390 (2.610) | 87.4 | bicubic | 672 | 0.954 | +| tf_efficientnet_b8_ap | 85.368 (14.632) | 97.294 (2.706) | 87.4 | bicubic | 672 | 0.954 | +| tf_efficientnet_b4_ns *tfp | 85.298 (14.702) | 97.504 (2.496) | 19.34 | bicubic | 380 | N/A | +| tf_efficientnet_b4_ns | 85.162 (14.838) | 97.470 (2.530) | 19.34 | bicubic | 380 | 0.922 | +| tf_efficientnet_b7_ap *tfp | 85.154 (14.846) | 97.244 (2.756) | 66.35 | bicubic | 600 | N/A | +| tf_efficientnet_b7_ap | 85.118 (14.882) | 97.252 (2.748) | 66.35 | bicubic | 600 | 0.949 | +| tf_efficientnet_b7 *tfp | 84.940 (15.060) | 97.214 (2.786) | 66.35 | bicubic | 600 | N/A | +| tf_efficientnet_b7 | 84.932 (15.068) | 97.208 (2.792) | 66.35 | bicubic | 600 | 0.949 | +| tf_efficientnet_b6_ap | 84.786 (15.214) | 97.138 (2.862) | 43.04 | bicubic | 528 | 0.942 | +| tf_efficientnet_b6_ap *tfp | 84.760 (15.240) | 97.124 (2.876) | 43.04 | bicubic | 528 | N/A | +| tf_efficientnet_b5_ap *tfp | 84.276 (15.724) | 96.932 (3.068) | 30.39 | bicubic | 456 | N/A | +| tf_efficientnet_b5_ap | 84.254 (15.746) | 96.976 (3.024) | 30.39 | bicubic | 456 | 0.934 | +| tf_efficientnet_b6 *tfp | 84.140 (15.860) | 96.852 (3.148) | 43.04 | bicubic | 528 | N/A | +| tf_efficientnet_b6 | 84.110 (15.890) | 96.886 (3.114) | 43.04 | bicubic | 528 | 0.942 | +| tf_efficientnet_b3_ns *tfp | 84.054 (15.946) | 96.918 (3.082) | 12.23 | bicubic | 300 | N/A | +| tf_efficientnet_b3_ns | 84.048 (15.952) | 96.910 (3.090) | 12.23 | bicubic | 300 | .904 | +| tf_efficientnet_b5 *tfp | 83.822 (16.178) | 96.756 (3.244) | 30.39 | bicubic | 456 | N/A | +| tf_efficientnet_b5 | 83.812 (16.188) | 96.748 (3.252) | 30.39 | bicubic | 456 | 0.934 | +| tf_efficientnet_b4_ap *tfp | 83.278 (16.722) | 96.376 (3.624) | 19.34 | bicubic | 380 | N/A | +| tf_efficientnet_b4_ap | 83.248 (16.752) | 96.388 (3.612) | 19.34 | bicubic | 380 | 0.922 | +| tf_efficientnet_b4 | 83.022 (16.978) | 96.300 (3.700) | 19.34 | bicubic | 380 | 0.922 | +| tf_efficientnet_b4 *tfp | 82.948 (17.052) | 96.308 (3.692) | 19.34 | bicubic | 380 | N/A | +| tf_efficientnet_b2_ns *tfp | 82.436 (17.564) | 96.268 (3.732) | 9.11 | bicubic | 260 | N/A | +| tf_efficientnet_b2_ns | 82.380 (17.620) | 96.248 (3.752) | 9.11 | bicubic | 260 | 0.89 | +| tf_efficientnet_b3_ap *tfp | 81.882 (18.118) | 95.662 (4.338) | 12.23 | bicubic | 300 | N/A | +| tf_efficientnet_b3_ap | 81.828 (18.172) | 95.624 (4.376) | 12.23 | bicubic | 300 | 0.904 | +| tf_efficientnet_b3 | 81.636 (18.364) | 95.718 (4.282) | 12.23 | bicubic | 300 | 0.904 | +| tf_efficientnet_b3 *tfp | 81.576 (18.424) | 95.662 (4.338) | 12.23 | bicubic | 300 | N/A | +| tf_efficientnet_lite4 | 81.528 (18.472) | 95.668 (4.332) | 13.00 | bilinear | 380 | 0.92 | +| tf_efficientnet_b1_ns *tfp | 81.514 (18.486) | 95.776 (4.224) | 7.79 | bicubic | 240 | N/A | +| tf_efficientnet_lite4 *tfp | 81.502 (18.498) | 95.676 (4.324) | 13.00 | bilinear | 380 | N/A | +| tf_efficientnet_b1_ns | 81.388 (18.612) | 95.738 (4.262) | 7.79 | bicubic | 240 | 0.88 | +| tf_efficientnet_el | 80.534 (19.466) | 95.190 (4.810) | 10.59 | bicubic | 300 | 0.904 | +| tf_efficientnet_el *tfp | 80.476 (19.524) | 95.200 (4.800) | 10.59 | bicubic | 300 | N/A | +| tf_efficientnet_b2_ap *tfp | 80.420 (19.580) | 95.040 (4.960) | 9.11 | bicubic | 260 | N/A | +| tf_efficientnet_b2_ap | 80.306 (19.694) | 95.028 (4.972) | 9.11 | bicubic | 260 | 0.890 | +| tf_efficientnet_b2 *tfp | 80.188 (19.812) | 94.974 (5.026) | 9.11 | bicubic | 260 | N/A | +| tf_efficientnet_b2 | 80.086 (19.914) | 94.908 (5.092) | 9.11 | bicubic | 260 | 0.890 | +| tf_efficientnet_lite3 | 79.812 (20.188) | 94.914 (5.086) | 8.20 | bilinear | 300 | 0.904 | +| tf_efficientnet_lite3 *tfp | 79.734 (20.266) | 94.838 (5.162) | 8.20 | bilinear | 300 | N/A | +| tf_efficientnet_b1_ap *tfp | 79.532 (20.468) | 94.378 (5.622) | 7.79 | bicubic | 240 | N/A | +| tf_efficientnet_cc_b1_8e *tfp | 79.464 (20.536)| 94.492 (5.508) | 39.7 | bicubic | 240 | 0.88 | +| tf_efficientnet_cc_b1_8e | 79.298 (20.702) | 94.364 (5.636) | 39.7 | bicubic | 240 | 0.88 | +| tf_efficientnet_b1_ap | 79.278 (20.722) | 94.308 (5.692) | 7.79 | bicubic | 240 | 0.88 | +| tf_efficientnet_b1 *tfp | 79.172 (20.828) | 94.450 (5.550) | 7.79 | bicubic | 240 | N/A | +| tf_efficientnet_em *tfp | 78.958 (21.042) | 94.458 (5.542) | 6.90 | bicubic | 240 | N/A | +| tf_efficientnet_b0_ns *tfp | 78.806 (21.194) | 94.496 (5.504) | 5.29 | bicubic | 224 | N/A | +| tf_mixnet_l *tfp | 78.846 (21.154) | 94.212 (5.788) | 7.33 | bilinear | 224 | N/A | +| tf_efficientnet_b1 | 78.826 (21.174) | 94.198 (5.802) | 7.79 | bicubic | 240 | 0.88 | +| tf_mixnet_l | 78.770 (21.230) | 94.004 (5.996) | 7.33 | bicubic | 224 | 0.875 | +| tf_efficientnet_em | 78.742 (21.258) | 94.332 (5.668) | 6.90 | bicubic | 240 | 0.875 | +| tf_efficientnet_b0_ns | 78.658 (21.342) | 94.376 (5.624) | 5.29 | bicubic | 224 | 0.875 | +| tf_efficientnet_cc_b0_8e *tfp | 78.314 (21.686) | 93.790 (6.210) | 24.0 | bicubic | 224 | 0.875 | +| tf_efficientnet_cc_b0_8e | 77.908 (22.092) | 93.656 (6.344) | 24.0 | bicubic | 224 | 0.875 | +| tf_efficientnet_cc_b0_4e *tfp | 77.746 (22.254) | 93.552 (6.448) | 13.3 | bicubic | 224 | 0.875 | +| tf_efficientnet_cc_b0_4e | 77.304 (22.696) | 93.332 (6.668) | 13.3 | bicubic | 224 | 0.875 | +| tf_efficientnet_es *tfp | 77.616 (22.384) | 93.750 (6.250) | 5.44 | bicubic | 224 | N/A | +| tf_efficientnet_lite2 *tfp | 77.544 (22.456) | 93.800 (6.200) | 6.09 | bilinear | 260 | N/A | +| tf_efficientnet_lite2 | 77.460 (22.540) | 93.746 (6.254) | 6.09 | bicubic | 260 | 0.89 | +| tf_efficientnet_b0_ap *tfp | 77.514 (22.486) | 93.576 (6.424) | 5.29 | bicubic | 224 | N/A | +| tf_efficientnet_es | 77.264 (22.736) | 93.600 (6.400) | 5.44 | bicubic | 224 | N/A | +| tf_efficientnet_b0 *tfp | 77.258 (22.742) | 93.478 (6.522) | 5.29 | bicubic | 224 | N/A | +| tf_efficientnet_b0_ap | 77.084 (22.916) | 93.254 (6.746) | 5.29 | bicubic | 224 | 0.875 | +| tf_mixnet_m *tfp | 77.072 (22.928) | 93.368 (6.632) | 5.01 | bilinear | 224 | N/A | +| tf_mixnet_m | 76.950 (23.050) | 93.156 (6.844) | 5.01 | bicubic | 224 | 0.875 | +| tf_efficientnet_b0 | 76.848 (23.152) | 93.228 (6.772) | 5.29 | bicubic | 224 | 0.875 | +| tf_efficientnet_lite1 *tfp | 76.764 (23.236) | 93.326 (6.674) | 5.42 | bilinear | 240 | N/A | +| tf_efficientnet_lite1 | 76.638 (23.362) | 93.232 (6.768) | 5.42 | bicubic | 240 | 0.882 | +| tf_mixnet_s *tfp | 75.800 (24.200) | 92.788 (7.212) | 4.13 | bilinear | 224 | N/A | +| tf_mobilenetv3_large_100 *tfp | 75.768 (24.232) | 92.710 (7.290) | 5.48 | bilinear | 224 | N/A | +| tf_mixnet_s | 75.648 (24.352) | 92.636 (7.364) | 4.13 | bicubic | 224 | 0.875 | +| tf_mobilenetv3_large_100 | 75.516 (24.484) | 92.600 (7.400) | 5.48 | bilinear | 224 | 0.875 | +| tf_efficientnet_lite0 *tfp | 75.074 (24.926) | 92.314 (7.686) | 4.65 | bilinear | 224 | N/A | +| tf_efficientnet_lite0 | 74.842 (25.158) | 92.170 (7.830) | 4.65 | bicubic | 224 | 0.875 | +| tf_mobilenetv3_large_075 *tfp | 73.730 (26.270) | 91.616 (8.384) | 3.99 | bilinear | 224 |N/A | +| tf_mobilenetv3_large_075 | 73.442 (26.558) | 91.352 (8.648) | 3.99 | bilinear | 224 | 0.875 | +| tf_mobilenetv3_large_minimal_100 *tfp | 72.678 (27.322) | 90.860 (9.140) | 3.92 | bilinear | 224 | N/A | +| tf_mobilenetv3_large_minimal_100 | 72.244 (27.756) | 90.636 (9.364) | 3.92 | bilinear | 224 | 0.875 | +| tf_mobilenetv3_small_100 *tfp | 67.918 (32.082) | 87.958 (12.042 | 2.54 | bilinear | 224 | N/A | +| tf_mobilenetv3_small_100 | 67.918 (32.082) | 87.662 (12.338) | 2.54 | bilinear | 224 | 0.875 | +| tf_mobilenetv3_small_075 *tfp | 66.142 (33.858) | 86.498 (13.502) | 2.04 | bilinear | 224 | N/A | +| tf_mobilenetv3_small_075 | 65.718 (34.282) | 86.136 (13.864) | 2.04 | bilinear | 224 | 0.875 | +| tf_mobilenetv3_small_minimal_100 *tfp | 63.378 (36.622) | 84.802 (15.198) | 2.04 | bilinear | 224 | N/A | +| tf_mobilenetv3_small_minimal_100 | 62.898 (37.102) | 84.230 (15.770) | 2.04 | bilinear | 224 | 0.875 | + + +*tfp models validated with `tf-preprocessing` pipeline + +Google tf and tflite weights ported from official Tensorflow repositories +* https://github.com/tensorflow/tpu/tree/master/models/official/mnasnet +* https://github.com/tensorflow/tpu/tree/master/models/official/efficientnet +* https://github.com/tensorflow/models/tree/master/research/slim/nets/mobilenet + +## Usage + +### Environment + +All development and testing has been done in Conda Python 3 environments on Linux x86-64 systems, specifically Python 3.6.x, 3.7.x, 3.8.x. + +Users have reported that a Python 3 Anaconda install in Windows works. I have not verified this myself. + +PyTorch versions 1.4, 1.5, 1.6 have been tested with this code. + +I've tried to keep the dependencies minimal, the setup is as per the PyTorch default install instructions for Conda: +``` +conda create -n torch-env +conda activate torch-env +conda install -c pytorch pytorch torchvision cudatoolkit=10.2 +``` + +### PyTorch Hub + +Models can be accessed via the PyTorch Hub API + +``` +>>> torch.hub.list('rwightman/gen-efficientnet-pytorch') +['efficientnet_b0', ...] +>>> model = torch.hub.load('rwightman/gen-efficientnet-pytorch', 'efficientnet_b0', pretrained=True) +>>> model.eval() +>>> output = model(torch.randn(1,3,224,224)) +``` + +### Pip +This package can be installed via pip. + +Install (after conda env/install): +``` +pip install geffnet +``` + +Eval use: +``` +>>> import geffnet +>>> m = geffnet.create_model('mobilenetv3_large_100', pretrained=True) +>>> m.eval() +``` + +Train use: +``` +>>> import geffnet +>>> # models can also be created by using the entrypoint directly +>>> m = geffnet.efficientnet_b2(pretrained=True, drop_rate=0.25, drop_connect_rate=0.2) +>>> m.train() +``` + +Create in a nn.Sequential container, for fast.ai, etc: +``` +>>> import geffnet +>>> m = geffnet.mixnet_l(pretrained=True, drop_rate=0.25, drop_connect_rate=0.2, as_sequential=True) +``` + +### Exporting + +Scripts are included to +* export models to ONNX (`onnx_export.py`) +* optimized ONNX graph (`onnx_optimize.py` or `onnx_validate.py` w/ `--onnx-output-opt` arg) +* validate with ONNX runtime (`onnx_validate.py`) +* convert ONNX model to Caffe2 (`onnx_to_caffe.py`) +* validate in Caffe2 (`caffe2_validate.py`) +* benchmark in Caffe2 w/ FLOPs, parameters output (`caffe2_benchmark.py`) + +As an example, to export the MobileNet-V3 pretrained model and then run an Imagenet validation: +``` +python onnx_export.py --model mobilenetv3_large_100 ./mobilenetv3_100.onnx +python onnx_validate.py /imagenet/validation/ --onnx-input ./mobilenetv3_100.onnx +``` + +These scripts were tested to be working as of PyTorch 1.6 and ONNX 1.7 w/ ONNX runtime 1.4. Caffe2 compatible +export now requires additional args mentioned in the export script (not needed in earlier versions). + +#### Export Notes +1. The TF ported weights with the 'SAME' conv padding activated cannot be exported to ONNX unless `_EXPORTABLE` flag in `config.py` is set to True. Use `config.set_exportable(True)` as in the `onnx_export.py` script. +2. TF ported models with 'SAME' padding will have the padding fixed at export time to the resolution used for export. Even though dynamic padding is supported in opset >= 11, I can't get it working. +3. ONNX optimize facility doesn't work reliably in PyTorch 1.6 / ONNX 1.7. Fortunately, the onnxruntime based inference is working very well now and includes on the fly optimization. +3. ONNX / Caffe2 export/import frequently breaks with different PyTorch and ONNX version releases. Please check their respective issue trackers before filing issues here. + + diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/caffe2_benchmark.py b/annotator/normalbae/models/submodules/efficientnet_repo/caffe2_benchmark.py new file mode 100644 index 0000000000000000000000000000000000000000..93f28a1e63d9f7287ca02997c7991fe66dd0aeb9 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/caffe2_benchmark.py @@ -0,0 +1,65 @@ +""" Caffe2 validation script + +This script runs Caffe2 benchmark on exported ONNX model. +It is a useful tool for reporting model FLOPS. + +Copyright 2020 Ross Wightman +""" +import argparse +from caffe2.python import core, workspace, model_helper +from caffe2.proto import caffe2_pb2 + + +parser = argparse.ArgumentParser(description='Caffe2 Model Benchmark') +parser.add_argument('--c2-prefix', default='', type=str, metavar='NAME', + help='caffe2 model pb name prefix') +parser.add_argument('--c2-init', default='', type=str, metavar='PATH', + help='caffe2 model init .pb') +parser.add_argument('--c2-predict', default='', type=str, metavar='PATH', + help='caffe2 model predict .pb') +parser.add_argument('-b', '--batch-size', default=1, type=int, + metavar='N', help='mini-batch size (default: 1)') +parser.add_argument('--img-size', default=224, type=int, + metavar='N', help='Input image dimension, uses model default if empty') + + +def main(): + args = parser.parse_args() + args.gpu_id = 0 + if args.c2_prefix: + args.c2_init = args.c2_prefix + '.init.pb' + args.c2_predict = args.c2_prefix + '.predict.pb' + + model = model_helper.ModelHelper(name="le_net", init_params=False) + + # Bring in the init net from init_net.pb + init_net_proto = caffe2_pb2.NetDef() + with open(args.c2_init, "rb") as f: + init_net_proto.ParseFromString(f.read()) + model.param_init_net = core.Net(init_net_proto) + + # bring in the predict net from predict_net.pb + predict_net_proto = caffe2_pb2.NetDef() + with open(args.c2_predict, "rb") as f: + predict_net_proto.ParseFromString(f.read()) + model.net = core.Net(predict_net_proto) + + # CUDA performance not impressive + #device_opts = core.DeviceOption(caffe2_pb2.PROTO_CUDA, args.gpu_id) + #model.net.RunAllOnGPU(gpu_id=args.gpu_id, use_cudnn=True) + #model.param_init_net.RunAllOnGPU(gpu_id=args.gpu_id, use_cudnn=True) + + input_blob = model.net.external_inputs[0] + model.param_init_net.GaussianFill( + [], + input_blob.GetUnscopedName(), + shape=(args.batch_size, 3, args.img_size, args.img_size), + mean=0.0, + std=1.0) + workspace.RunNetOnce(model.param_init_net) + workspace.CreateNet(model.net, overwrite=True) + workspace.BenchmarkNet(model.net.Proto().name, 5, 20, True) + + +if __name__ == '__main__': + main() diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/caffe2_validate.py b/annotator/normalbae/models/submodules/efficientnet_repo/caffe2_validate.py new file mode 100644 index 0000000000000000000000000000000000000000..7cfaab38c095663fe32e4addbdf06b57bcb53614 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/caffe2_validate.py @@ -0,0 +1,138 @@ +""" Caffe2 validation script + +This script is created to verify exported ONNX models running in Caffe2 +It utilizes the same PyTorch dataloader/processing pipeline for a +fair comparison against the originals. + +Copyright 2020 Ross Wightman +""" +import argparse +import numpy as np +from caffe2.python import core, workspace, model_helper +from caffe2.proto import caffe2_pb2 +from data import create_loader, resolve_data_config, Dataset +from utils import AverageMeter +import time + +parser = argparse.ArgumentParser(description='Caffe2 ImageNet Validation') +parser.add_argument('data', metavar='DIR', + help='path to dataset') +parser.add_argument('--c2-prefix', default='', type=str, metavar='NAME', + help='caffe2 model pb name prefix') +parser.add_argument('--c2-init', default='', type=str, metavar='PATH', + help='caffe2 model init .pb') +parser.add_argument('--c2-predict', default='', type=str, metavar='PATH', + help='caffe2 model predict .pb') +parser.add_argument('-j', '--workers', default=2, type=int, metavar='N', + help='number of data loading workers (default: 2)') +parser.add_argument('-b', '--batch-size', default=256, type=int, + metavar='N', help='mini-batch size (default: 256)') +parser.add_argument('--img-size', default=None, type=int, + metavar='N', help='Input image dimension, uses model default if empty') +parser.add_argument('--mean', type=float, nargs='+', default=None, metavar='MEAN', + help='Override mean pixel value of dataset') +parser.add_argument('--std', type=float, nargs='+', default=None, metavar='STD', + help='Override std deviation of of dataset') +parser.add_argument('--crop-pct', type=float, default=None, metavar='PCT', + help='Override default crop pct of 0.875') +parser.add_argument('--interpolation', default='', type=str, metavar='NAME', + help='Image resize interpolation type (overrides model)') +parser.add_argument('--tf-preprocessing', dest='tf_preprocessing', action='store_true', + help='use tensorflow mnasnet preporcessing') +parser.add_argument('--print-freq', '-p', default=10, type=int, + metavar='N', help='print frequency (default: 10)') + + +def main(): + args = parser.parse_args() + args.gpu_id = 0 + if args.c2_prefix: + args.c2_init = args.c2_prefix + '.init.pb' + args.c2_predict = args.c2_prefix + '.predict.pb' + + model = model_helper.ModelHelper(name="validation_net", init_params=False) + + # Bring in the init net from init_net.pb + init_net_proto = caffe2_pb2.NetDef() + with open(args.c2_init, "rb") as f: + init_net_proto.ParseFromString(f.read()) + model.param_init_net = core.Net(init_net_proto) + + # bring in the predict net from predict_net.pb + predict_net_proto = caffe2_pb2.NetDef() + with open(args.c2_predict, "rb") as f: + predict_net_proto.ParseFromString(f.read()) + model.net = core.Net(predict_net_proto) + + data_config = resolve_data_config(None, args) + loader = create_loader( + Dataset(args.data, load_bytes=args.tf_preprocessing), + input_size=data_config['input_size'], + batch_size=args.batch_size, + use_prefetcher=False, + interpolation=data_config['interpolation'], + mean=data_config['mean'], + std=data_config['std'], + num_workers=args.workers, + crop_pct=data_config['crop_pct'], + tensorflow_preprocessing=args.tf_preprocessing) + + # this is so obvious, wonderful interface + input_blob = model.net.external_inputs[0] + output_blob = model.net.external_outputs[0] + + if True: + device_opts = None + else: + # CUDA is crashing, no idea why, awesome error message, give it a try for kicks + device_opts = core.DeviceOption(caffe2_pb2.PROTO_CUDA, args.gpu_id) + model.net.RunAllOnGPU(gpu_id=args.gpu_id, use_cudnn=True) + model.param_init_net.RunAllOnGPU(gpu_id=args.gpu_id, use_cudnn=True) + + model.param_init_net.GaussianFill( + [], input_blob.GetUnscopedName(), + shape=(1,) + data_config['input_size'], mean=0.0, std=1.0) + workspace.RunNetOnce(model.param_init_net) + workspace.CreateNet(model.net, overwrite=True) + + batch_time = AverageMeter() + top1 = AverageMeter() + top5 = AverageMeter() + end = time.time() + for i, (input, target) in enumerate(loader): + # run the net and return prediction + caffe2_in = input.data.numpy() + workspace.FeedBlob(input_blob, caffe2_in, device_opts) + workspace.RunNet(model.net, num_iter=1) + output = workspace.FetchBlob(output_blob) + + # measure accuracy and record loss + prec1, prec5 = accuracy_np(output.data, target.numpy()) + top1.update(prec1.item(), input.size(0)) + top5.update(prec5.item(), input.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + print('Test: [{0}/{1}]\t' + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f}, {rate_avg:.3f}/s, {ms_avg:.3f} ms/sample) \t' + 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' + 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( + i, len(loader), batch_time=batch_time, rate_avg=input.size(0) / batch_time.avg, + ms_avg=100 * batch_time.avg / input.size(0), top1=top1, top5=top5)) + + print(' * Prec@1 {top1.avg:.3f} ({top1a:.3f}) Prec@5 {top5.avg:.3f} ({top5a:.3f})'.format( + top1=top1, top1a=100-top1.avg, top5=top5, top5a=100.-top5.avg)) + + +def accuracy_np(output, target): + max_indices = np.argsort(output, axis=1)[:, ::-1] + top5 = 100 * np.equal(max_indices[:, :5], target[:, np.newaxis]).sum(axis=1).mean() + top1 = 100 * np.equal(max_indices[:, 0], target).mean() + return top1, top5 + + +if __name__ == '__main__': + main() diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/data/__init__.py b/annotator/normalbae/models/submodules/efficientnet_repo/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..75824e6203af44be8b60c06dded81bf393f9134a --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/data/__init__.py @@ -0,0 +1,3 @@ +from .dataset import Dataset +from .transforms import * +from .loader import create_loader diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/data/dataset.py b/annotator/normalbae/models/submodules/efficientnet_repo/data/dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..8b4b07f0eb57dbe517714e170796647db8d7a6ed --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/data/dataset.py @@ -0,0 +1,91 @@ +""" Quick n simple image folder dataset + +Copyright 2020 Ross Wightman +""" +import torch.utils.data as data + +import os +import re +import torch +from PIL import Image + + +IMG_EXTENSIONS = ['.png', '.jpg', '.jpeg'] + + +def natural_key(string_): + """See http://www.codinghorror.com/blog/archives/001018.html""" + return [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', string_.lower())] + + +def find_images_and_targets(folder, types=IMG_EXTENSIONS, class_to_idx=None, leaf_name_only=True, sort=True): + if class_to_idx is None: + class_to_idx = dict() + build_class_idx = True + else: + build_class_idx = False + labels = [] + filenames = [] + for root, subdirs, files in os.walk(folder, topdown=False): + rel_path = os.path.relpath(root, folder) if (root != folder) else '' + label = os.path.basename(rel_path) if leaf_name_only else rel_path.replace(os.path.sep, '_') + if build_class_idx and not subdirs: + class_to_idx[label] = None + for f in files: + base, ext = os.path.splitext(f) + if ext.lower() in types: + filenames.append(os.path.join(root, f)) + labels.append(label) + if build_class_idx: + classes = sorted(class_to_idx.keys(), key=natural_key) + for idx, c in enumerate(classes): + class_to_idx[c] = idx + images_and_targets = zip(filenames, [class_to_idx[l] for l in labels]) + if sort: + images_and_targets = sorted(images_and_targets, key=lambda k: natural_key(k[0])) + if build_class_idx: + return images_and_targets, classes, class_to_idx + else: + return images_and_targets + + +class Dataset(data.Dataset): + + def __init__( + self, + root, + transform=None, + load_bytes=False): + + imgs, _, _ = find_images_and_targets(root) + if len(imgs) == 0: + raise(RuntimeError("Found 0 images in subfolders of: " + root + "\n" + "Supported image extensions are: " + ",".join(IMG_EXTENSIONS))) + self.root = root + self.imgs = imgs + self.transform = transform + self.load_bytes = load_bytes + + def __getitem__(self, index): + path, target = self.imgs[index] + img = open(path, 'rb').read() if self.load_bytes else Image.open(path).convert('RGB') + if self.transform is not None: + img = self.transform(img) + if target is None: + target = torch.zeros(1).long() + return img, target + + def __len__(self): + return len(self.imgs) + + def filenames(self, indices=[], basename=False): + if indices: + if basename: + return [os.path.basename(self.imgs[i][0]) for i in indices] + else: + return [self.imgs[i][0] for i in indices] + else: + if basename: + return [os.path.basename(x[0]) for x in self.imgs] + else: + return [x[0] for x in self.imgs] diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/data/loader.py b/annotator/normalbae/models/submodules/efficientnet_repo/data/loader.py new file mode 100644 index 0000000000000000000000000000000000000000..23e0f69ea23ac0a9be7752986220ed434da72ef4 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/data/loader.py @@ -0,0 +1,108 @@ +""" Fast Collate, CUDA Prefetcher + +Prefetcher and Fast Collate inspired by NVIDIA APEX example at +https://github.com/NVIDIA/apex/commit/d5e2bb4bdeedd27b1dfaf5bb2b24d6c000dee9be#diff-cf86c282ff7fba81fad27a559379d5bf + +Hacked together by / Copyright 2020 Ross Wightman +""" +import torch +import torch.utils.data +from .transforms import * + + +def fast_collate(batch): + targets = torch.tensor([b[1] for b in batch], dtype=torch.int64) + batch_size = len(targets) + tensor = torch.zeros((batch_size, *batch[0][0].shape), dtype=torch.uint8) + for i in range(batch_size): + tensor[i] += torch.from_numpy(batch[i][0]) + + return tensor, targets + + +class PrefetchLoader: + + def __init__(self, + loader, + mean=IMAGENET_DEFAULT_MEAN, + std=IMAGENET_DEFAULT_STD): + self.loader = loader + self.mean = torch.tensor([x * 255 for x in mean]).cuda().view(1, 3, 1, 1) + self.std = torch.tensor([x * 255 for x in std]).cuda().view(1, 3, 1, 1) + + def __iter__(self): + stream = torch.cuda.Stream() + first = True + + for next_input, next_target in self.loader: + with torch.cuda.stream(stream): + next_input = next_input.cuda(non_blocking=True) + next_target = next_target.cuda(non_blocking=True) + next_input = next_input.float().sub_(self.mean).div_(self.std) + + if not first: + yield input, target + else: + first = False + + torch.cuda.current_stream().wait_stream(stream) + input = next_input + target = next_target + + yield input, target + + def __len__(self): + return len(self.loader) + + @property + def sampler(self): + return self.loader.sampler + + +def create_loader( + dataset, + input_size, + batch_size, + is_training=False, + use_prefetcher=True, + interpolation='bilinear', + mean=IMAGENET_DEFAULT_MEAN, + std=IMAGENET_DEFAULT_STD, + num_workers=1, + crop_pct=None, + tensorflow_preprocessing=False +): + if isinstance(input_size, tuple): + img_size = input_size[-2:] + else: + img_size = input_size + + if tensorflow_preprocessing and use_prefetcher: + from data.tf_preprocessing import TfPreprocessTransform + transform = TfPreprocessTransform( + is_training=is_training, size=img_size, interpolation=interpolation) + else: + transform = transforms_imagenet_eval( + img_size, + interpolation=interpolation, + use_prefetcher=use_prefetcher, + mean=mean, + std=std, + crop_pct=crop_pct) + + dataset.transform = transform + + loader = torch.utils.data.DataLoader( + dataset, + batch_size=batch_size, + shuffle=False, + num_workers=num_workers, + collate_fn=fast_collate if use_prefetcher else torch.utils.data.dataloader.default_collate, + ) + if use_prefetcher: + loader = PrefetchLoader( + loader, + mean=mean, + std=std) + + return loader diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/data/tf_preprocessing.py b/annotator/normalbae/models/submodules/efficientnet_repo/data/tf_preprocessing.py new file mode 100644 index 0000000000000000000000000000000000000000..ee3adeaed6e06bcdec68e312e335b6e2fa31faec --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/data/tf_preprocessing.py @@ -0,0 +1,234 @@ +""" Tensorflow Preprocessing Adapter + +Allows use of Tensorflow preprocessing pipeline in PyTorch Transform + +Copyright of original Tensorflow code below. + +Hacked together by / Copyright 2020 Ross Wightman +""" +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import tensorflow as tf +import numpy as np + +IMAGE_SIZE = 224 +CROP_PADDING = 32 + + +def distorted_bounding_box_crop(image_bytes, + bbox, + min_object_covered=0.1, + aspect_ratio_range=(0.75, 1.33), + area_range=(0.05, 1.0), + max_attempts=100, + scope=None): + """Generates cropped_image using one of the bboxes randomly distorted. + + See `tf.image.sample_distorted_bounding_box` for more documentation. + + Args: + image_bytes: `Tensor` of binary image data. + bbox: `Tensor` of bounding boxes arranged `[1, num_boxes, coords]` + where each coordinate is [0, 1) and the coordinates are arranged + as `[ymin, xmin, ymax, xmax]`. If num_boxes is 0 then use the whole + image. + min_object_covered: An optional `float`. Defaults to `0.1`. The cropped + area of the image must contain at least this fraction of any bounding + box supplied. + aspect_ratio_range: An optional list of `float`s. The cropped area of the + image must have an aspect ratio = width / height within this range. + area_range: An optional list of `float`s. The cropped area of the image + must contain a fraction of the supplied image within in this range. + max_attempts: An optional `int`. Number of attempts at generating a cropped + region of the image of the specified constraints. After `max_attempts` + failures, return the entire image. + scope: Optional `str` for name scope. + Returns: + cropped image `Tensor` + """ + with tf.name_scope(scope, 'distorted_bounding_box_crop', [image_bytes, bbox]): + shape = tf.image.extract_jpeg_shape(image_bytes) + sample_distorted_bounding_box = tf.image.sample_distorted_bounding_box( + shape, + bounding_boxes=bbox, + min_object_covered=min_object_covered, + aspect_ratio_range=aspect_ratio_range, + area_range=area_range, + max_attempts=max_attempts, + use_image_if_no_bounding_boxes=True) + bbox_begin, bbox_size, _ = sample_distorted_bounding_box + + # Crop the image to the specified bounding box. + offset_y, offset_x, _ = tf.unstack(bbox_begin) + target_height, target_width, _ = tf.unstack(bbox_size) + crop_window = tf.stack([offset_y, offset_x, target_height, target_width]) + image = tf.image.decode_and_crop_jpeg(image_bytes, crop_window, channels=3) + + return image + + +def _at_least_x_are_equal(a, b, x): + """At least `x` of `a` and `b` `Tensors` are equal.""" + match = tf.equal(a, b) + match = tf.cast(match, tf.int32) + return tf.greater_equal(tf.reduce_sum(match), x) + + +def _decode_and_random_crop(image_bytes, image_size, resize_method): + """Make a random crop of image_size.""" + bbox = tf.constant([0.0, 0.0, 1.0, 1.0], dtype=tf.float32, shape=[1, 1, 4]) + image = distorted_bounding_box_crop( + image_bytes, + bbox, + min_object_covered=0.1, + aspect_ratio_range=(3. / 4, 4. / 3.), + area_range=(0.08, 1.0), + max_attempts=10, + scope=None) + original_shape = tf.image.extract_jpeg_shape(image_bytes) + bad = _at_least_x_are_equal(original_shape, tf.shape(image), 3) + + image = tf.cond( + bad, + lambda: _decode_and_center_crop(image_bytes, image_size), + lambda: tf.image.resize([image], [image_size, image_size], resize_method)[0]) + + return image + + +def _decode_and_center_crop(image_bytes, image_size, resize_method): + """Crops to center of image with padding then scales image_size.""" + shape = tf.image.extract_jpeg_shape(image_bytes) + image_height = shape[0] + image_width = shape[1] + + padded_center_crop_size = tf.cast( + ((image_size / (image_size + CROP_PADDING)) * + tf.cast(tf.minimum(image_height, image_width), tf.float32)), + tf.int32) + + offset_height = ((image_height - padded_center_crop_size) + 1) // 2 + offset_width = ((image_width - padded_center_crop_size) + 1) // 2 + crop_window = tf.stack([offset_height, offset_width, + padded_center_crop_size, padded_center_crop_size]) + image = tf.image.decode_and_crop_jpeg(image_bytes, crop_window, channels=3) + image = tf.image.resize([image], [image_size, image_size], resize_method)[0] + + return image + + +def _flip(image): + """Random horizontal image flip.""" + image = tf.image.random_flip_left_right(image) + return image + + +def preprocess_for_train(image_bytes, use_bfloat16, image_size=IMAGE_SIZE, interpolation='bicubic'): + """Preprocesses the given image for evaluation. + + Args: + image_bytes: `Tensor` representing an image binary of arbitrary size. + use_bfloat16: `bool` for whether to use bfloat16. + image_size: image size. + interpolation: image interpolation method + + Returns: + A preprocessed image `Tensor`. + """ + resize_method = tf.image.ResizeMethod.BICUBIC if interpolation == 'bicubic' else tf.image.ResizeMethod.BILINEAR + image = _decode_and_random_crop(image_bytes, image_size, resize_method) + image = _flip(image) + image = tf.reshape(image, [image_size, image_size, 3]) + image = tf.image.convert_image_dtype( + image, dtype=tf.bfloat16 if use_bfloat16 else tf.float32) + return image + + +def preprocess_for_eval(image_bytes, use_bfloat16, image_size=IMAGE_SIZE, interpolation='bicubic'): + """Preprocesses the given image for evaluation. + + Args: + image_bytes: `Tensor` representing an image binary of arbitrary size. + use_bfloat16: `bool` for whether to use bfloat16. + image_size: image size. + interpolation: image interpolation method + + Returns: + A preprocessed image `Tensor`. + """ + resize_method = tf.image.ResizeMethod.BICUBIC if interpolation == 'bicubic' else tf.image.ResizeMethod.BILINEAR + image = _decode_and_center_crop(image_bytes, image_size, resize_method) + image = tf.reshape(image, [image_size, image_size, 3]) + image = tf.image.convert_image_dtype( + image, dtype=tf.bfloat16 if use_bfloat16 else tf.float32) + return image + + +def preprocess_image(image_bytes, + is_training=False, + use_bfloat16=False, + image_size=IMAGE_SIZE, + interpolation='bicubic'): + """Preprocesses the given image. + + Args: + image_bytes: `Tensor` representing an image binary of arbitrary size. + is_training: `bool` for whether the preprocessing is for training. + use_bfloat16: `bool` for whether to use bfloat16. + image_size: image size. + interpolation: image interpolation method + + Returns: + A preprocessed image `Tensor` with value range of [0, 255]. + """ + if is_training: + return preprocess_for_train(image_bytes, use_bfloat16, image_size, interpolation) + else: + return preprocess_for_eval(image_bytes, use_bfloat16, image_size, interpolation) + + +class TfPreprocessTransform: + + def __init__(self, is_training=False, size=224, interpolation='bicubic'): + self.is_training = is_training + self.size = size[0] if isinstance(size, tuple) else size + self.interpolation = interpolation + self._image_bytes = None + self.process_image = self._build_tf_graph() + self.sess = None + + def _build_tf_graph(self): + with tf.device('/cpu:0'): + self._image_bytes = tf.placeholder( + shape=[], + dtype=tf.string, + ) + img = preprocess_image( + self._image_bytes, self.is_training, False, self.size, self.interpolation) + return img + + def __call__(self, image_bytes): + if self.sess is None: + self.sess = tf.Session() + img = self.sess.run(self.process_image, feed_dict={self._image_bytes: image_bytes}) + img = img.round().clip(0, 255).astype(np.uint8) + if img.ndim < 3: + img = np.expand_dims(img, axis=-1) + img = np.rollaxis(img, 2) # HWC to CHW + return img diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/data/transforms.py b/annotator/normalbae/models/submodules/efficientnet_repo/data/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..a570d484683f8ad5612bc22fb9ecae7e75c36afc --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/data/transforms.py @@ -0,0 +1,150 @@ +import torch +from torchvision import transforms +from PIL import Image +import math +import numpy as np + +DEFAULT_CROP_PCT = 0.875 + +IMAGENET_DEFAULT_MEAN = (0.485, 0.456, 0.406) +IMAGENET_DEFAULT_STD = (0.229, 0.224, 0.225) +IMAGENET_INCEPTION_MEAN = (0.5, 0.5, 0.5) +IMAGENET_INCEPTION_STD = (0.5, 0.5, 0.5) +IMAGENET_DPN_MEAN = (124 / 255, 117 / 255, 104 / 255) +IMAGENET_DPN_STD = tuple([1 / (.0167 * 255)] * 3) + + +def resolve_data_config(model, args, default_cfg={}, verbose=True): + new_config = {} + default_cfg = default_cfg + if not default_cfg and model is not None and hasattr(model, 'default_cfg'): + default_cfg = model.default_cfg + + # Resolve input/image size + # FIXME grayscale/chans arg to use different # channels? + in_chans = 3 + input_size = (in_chans, 224, 224) + if args.img_size is not None: + # FIXME support passing img_size as tuple, non-square + assert isinstance(args.img_size, int) + input_size = (in_chans, args.img_size, args.img_size) + elif 'input_size' in default_cfg: + input_size = default_cfg['input_size'] + new_config['input_size'] = input_size + + # resolve interpolation method + new_config['interpolation'] = 'bicubic' + if args.interpolation: + new_config['interpolation'] = args.interpolation + elif 'interpolation' in default_cfg: + new_config['interpolation'] = default_cfg['interpolation'] + + # resolve dataset + model mean for normalization + new_config['mean'] = IMAGENET_DEFAULT_MEAN + if args.mean is not None: + mean = tuple(args.mean) + if len(mean) == 1: + mean = tuple(list(mean) * in_chans) + else: + assert len(mean) == in_chans + new_config['mean'] = mean + elif 'mean' in default_cfg: + new_config['mean'] = default_cfg['mean'] + + # resolve dataset + model std deviation for normalization + new_config['std'] = IMAGENET_DEFAULT_STD + if args.std is not None: + std = tuple(args.std) + if len(std) == 1: + std = tuple(list(std) * in_chans) + else: + assert len(std) == in_chans + new_config['std'] = std + elif 'std' in default_cfg: + new_config['std'] = default_cfg['std'] + + # resolve default crop percentage + new_config['crop_pct'] = DEFAULT_CROP_PCT + if args.crop_pct is not None: + new_config['crop_pct'] = args.crop_pct + elif 'crop_pct' in default_cfg: + new_config['crop_pct'] = default_cfg['crop_pct'] + + if verbose: + print('Data processing configuration for current model + dataset:') + for n, v in new_config.items(): + print('\t%s: %s' % (n, str(v))) + + return new_config + + +class ToNumpy: + + def __call__(self, pil_img): + np_img = np.array(pil_img, dtype=np.uint8) + if np_img.ndim < 3: + np_img = np.expand_dims(np_img, axis=-1) + np_img = np.rollaxis(np_img, 2) # HWC to CHW + return np_img + + +class ToTensor: + + def __init__(self, dtype=torch.float32): + self.dtype = dtype + + def __call__(self, pil_img): + np_img = np.array(pil_img, dtype=np.uint8) + if np_img.ndim < 3: + np_img = np.expand_dims(np_img, axis=-1) + np_img = np.rollaxis(np_img, 2) # HWC to CHW + return torch.from_numpy(np_img).to(dtype=self.dtype) + + +def _pil_interp(method): + if method == 'bicubic': + return Image.BICUBIC + elif method == 'lanczos': + return Image.LANCZOS + elif method == 'hamming': + return Image.HAMMING + else: + # default bilinear, do we want to allow nearest? + return Image.BILINEAR + + +def transforms_imagenet_eval( + img_size=224, + crop_pct=None, + interpolation='bilinear', + use_prefetcher=False, + mean=IMAGENET_DEFAULT_MEAN, + std=IMAGENET_DEFAULT_STD): + crop_pct = crop_pct or DEFAULT_CROP_PCT + + if isinstance(img_size, tuple): + assert len(img_size) == 2 + if img_size[-1] == img_size[-2]: + # fall-back to older behaviour so Resize scales to shortest edge if target is square + scale_size = int(math.floor(img_size[0] / crop_pct)) + else: + scale_size = tuple([int(x / crop_pct) for x in img_size]) + else: + scale_size = int(math.floor(img_size / crop_pct)) + + tfl = [ + transforms.Resize(scale_size, _pil_interp(interpolation)), + transforms.CenterCrop(img_size), + ] + if use_prefetcher: + # prefetcher and collate will handle tensor conversion and norm + tfl += [ToNumpy()] + else: + tfl += [ + transforms.ToTensor(), + transforms.Normalize( + mean=torch.tensor(mean), + std=torch.tensor(std)) + ] + + return transforms.Compose(tfl) diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/__init__.py b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2e441a5838d1e972823b9668ac8d459445f6f6ce --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/__init__.py @@ -0,0 +1,5 @@ +from .gen_efficientnet import * +from .mobilenetv3 import * +from .model_factory import create_model +from .config import is_exportable, is_scriptable, set_exportable, set_scriptable +from .activations import * \ No newline at end of file diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/activations/__init__.py b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/activations/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..813421a743ffc33b8eb53ebf62dd4a03d831b654 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/activations/__init__.py @@ -0,0 +1,137 @@ +from geffnet import config +from geffnet.activations.activations_me import * +from geffnet.activations.activations_jit import * +from geffnet.activations.activations import * +import torch + +_has_silu = 'silu' in dir(torch.nn.functional) + +_ACT_FN_DEFAULT = dict( + silu=F.silu if _has_silu else swish, + swish=F.silu if _has_silu else swish, + mish=mish, + relu=F.relu, + relu6=F.relu6, + sigmoid=sigmoid, + tanh=tanh, + hard_sigmoid=hard_sigmoid, + hard_swish=hard_swish, +) + +_ACT_FN_JIT = dict( + silu=F.silu if _has_silu else swish_jit, + swish=F.silu if _has_silu else swish_jit, + mish=mish_jit, +) + +_ACT_FN_ME = dict( + silu=F.silu if _has_silu else swish_me, + swish=F.silu if _has_silu else swish_me, + mish=mish_me, + hard_swish=hard_swish_me, + hard_sigmoid_jit=hard_sigmoid_me, +) + +_ACT_LAYER_DEFAULT = dict( + silu=nn.SiLU if _has_silu else Swish, + swish=nn.SiLU if _has_silu else Swish, + mish=Mish, + relu=nn.ReLU, + relu6=nn.ReLU6, + sigmoid=Sigmoid, + tanh=Tanh, + hard_sigmoid=HardSigmoid, + hard_swish=HardSwish, +) + +_ACT_LAYER_JIT = dict( + silu=nn.SiLU if _has_silu else SwishJit, + swish=nn.SiLU if _has_silu else SwishJit, + mish=MishJit, +) + +_ACT_LAYER_ME = dict( + silu=nn.SiLU if _has_silu else SwishMe, + swish=nn.SiLU if _has_silu else SwishMe, + mish=MishMe, + hard_swish=HardSwishMe, + hard_sigmoid=HardSigmoidMe +) + +_OVERRIDE_FN = dict() +_OVERRIDE_LAYER = dict() + + +def add_override_act_fn(name, fn): + global _OVERRIDE_FN + _OVERRIDE_FN[name] = fn + + +def update_override_act_fn(overrides): + assert isinstance(overrides, dict) + global _OVERRIDE_FN + _OVERRIDE_FN.update(overrides) + + +def clear_override_act_fn(): + global _OVERRIDE_FN + _OVERRIDE_FN = dict() + + +def add_override_act_layer(name, fn): + _OVERRIDE_LAYER[name] = fn + + +def update_override_act_layer(overrides): + assert isinstance(overrides, dict) + global _OVERRIDE_LAYER + _OVERRIDE_LAYER.update(overrides) + + +def clear_override_act_layer(): + global _OVERRIDE_LAYER + _OVERRIDE_LAYER = dict() + + +def get_act_fn(name='relu'): + """ Activation Function Factory + Fetching activation fns by name with this function allows export or torch script friendly + functions to be returned dynamically based on current config. + """ + if name in _OVERRIDE_FN: + return _OVERRIDE_FN[name] + use_me = not (config.is_exportable() or config.is_scriptable() or config.is_no_jit()) + if use_me and name in _ACT_FN_ME: + # If not exporting or scripting the model, first look for a memory optimized version + # activation with custom autograd, then fallback to jit scripted, then a Python or Torch builtin + return _ACT_FN_ME[name] + if config.is_exportable() and name in ('silu', 'swish'): + # FIXME PyTorch SiLU doesn't ONNX export, this is a temp hack + return swish + use_jit = not (config.is_exportable() or config.is_no_jit()) + # NOTE: export tracing should work with jit scripted components, but I keep running into issues + if use_jit and name in _ACT_FN_JIT: # jit scripted models should be okay for export/scripting + return _ACT_FN_JIT[name] + return _ACT_FN_DEFAULT[name] + + +def get_act_layer(name='relu'): + """ Activation Layer Factory + Fetching activation layers by name with this function allows export or torch script friendly + functions to be returned dynamically based on current config. + """ + if name in _OVERRIDE_LAYER: + return _OVERRIDE_LAYER[name] + use_me = not (config.is_exportable() or config.is_scriptable() or config.is_no_jit()) + if use_me and name in _ACT_LAYER_ME: + return _ACT_LAYER_ME[name] + if config.is_exportable() and name in ('silu', 'swish'): + # FIXME PyTorch SiLU doesn't ONNX export, this is a temp hack + return Swish + use_jit = not (config.is_exportable() or config.is_no_jit()) + # NOTE: export tracing should work with jit scripted components, but I keep running into issues + if use_jit and name in _ACT_FN_JIT: # jit scripted models should be okay for export/scripting + return _ACT_LAYER_JIT[name] + return _ACT_LAYER_DEFAULT[name] + + diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/activations/activations.py b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/activations/activations.py new file mode 100644 index 0000000000000000000000000000000000000000..bdea692d1397673b2513d898c33edbcb37d94240 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/activations/activations.py @@ -0,0 +1,102 @@ +""" Activations + +A collection of activations fn and modules with a common interface so that they can +easily be swapped. All have an `inplace` arg even if not used. + +Copyright 2020 Ross Wightman +""" +from torch import nn as nn +from torch.nn import functional as F + + +def swish(x, inplace: bool = False): + """Swish - Described originally as SiLU (https://arxiv.org/abs/1702.03118v3) + and also as Swish (https://arxiv.org/abs/1710.05941). + + TODO Rename to SiLU with addition to PyTorch + """ + return x.mul_(x.sigmoid()) if inplace else x.mul(x.sigmoid()) + + +class Swish(nn.Module): + def __init__(self, inplace: bool = False): + super(Swish, self).__init__() + self.inplace = inplace + + def forward(self, x): + return swish(x, self.inplace) + + +def mish(x, inplace: bool = False): + """Mish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681 + """ + return x.mul(F.softplus(x).tanh()) + + +class Mish(nn.Module): + def __init__(self, inplace: bool = False): + super(Mish, self).__init__() + self.inplace = inplace + + def forward(self, x): + return mish(x, self.inplace) + + +def sigmoid(x, inplace: bool = False): + return x.sigmoid_() if inplace else x.sigmoid() + + +# PyTorch has this, but not with a consistent inplace argmument interface +class Sigmoid(nn.Module): + def __init__(self, inplace: bool = False): + super(Sigmoid, self).__init__() + self.inplace = inplace + + def forward(self, x): + return x.sigmoid_() if self.inplace else x.sigmoid() + + +def tanh(x, inplace: bool = False): + return x.tanh_() if inplace else x.tanh() + + +# PyTorch has this, but not with a consistent inplace argmument interface +class Tanh(nn.Module): + def __init__(self, inplace: bool = False): + super(Tanh, self).__init__() + self.inplace = inplace + + def forward(self, x): + return x.tanh_() if self.inplace else x.tanh() + + +def hard_swish(x, inplace: bool = False): + inner = F.relu6(x + 3.).div_(6.) + return x.mul_(inner) if inplace else x.mul(inner) + + +class HardSwish(nn.Module): + def __init__(self, inplace: bool = False): + super(HardSwish, self).__init__() + self.inplace = inplace + + def forward(self, x): + return hard_swish(x, self.inplace) + + +def hard_sigmoid(x, inplace: bool = False): + if inplace: + return x.add_(3.).clamp_(0., 6.).div_(6.) + else: + return F.relu6(x + 3.) / 6. + + +class HardSigmoid(nn.Module): + def __init__(self, inplace: bool = False): + super(HardSigmoid, self).__init__() + self.inplace = inplace + + def forward(self, x): + return hard_sigmoid(x, self.inplace) + + diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/activations/activations_jit.py b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/activations/activations_jit.py new file mode 100644 index 0000000000000000000000000000000000000000..7176b05e779787528a47f20d55d64d4a0f219360 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/activations/activations_jit.py @@ -0,0 +1,79 @@ +""" Activations (jit) + +A collection of jit-scripted activations fn and modules with a common interface so that they can +easily be swapped. All have an `inplace` arg even if not used. + +All jit scripted activations are lacking in-place variations on purpose, scripted kernel fusion does not +currently work across in-place op boundaries, thus performance is equal to or less than the non-scripted +versions if they contain in-place ops. + +Copyright 2020 Ross Wightman +""" + +import torch +from torch import nn as nn +from torch.nn import functional as F + +__all__ = ['swish_jit', 'SwishJit', 'mish_jit', 'MishJit', + 'hard_sigmoid_jit', 'HardSigmoidJit', 'hard_swish_jit', 'HardSwishJit'] + + +@torch.jit.script +def swish_jit(x, inplace: bool = False): + """Swish - Described originally as SiLU (https://arxiv.org/abs/1702.03118v3) + and also as Swish (https://arxiv.org/abs/1710.05941). + + TODO Rename to SiLU with addition to PyTorch + """ + return x.mul(x.sigmoid()) + + +@torch.jit.script +def mish_jit(x, _inplace: bool = False): + """Mish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681 + """ + return x.mul(F.softplus(x).tanh()) + + +class SwishJit(nn.Module): + def __init__(self, inplace: bool = False): + super(SwishJit, self).__init__() + + def forward(self, x): + return swish_jit(x) + + +class MishJit(nn.Module): + def __init__(self, inplace: bool = False): + super(MishJit, self).__init__() + + def forward(self, x): + return mish_jit(x) + + +@torch.jit.script +def hard_sigmoid_jit(x, inplace: bool = False): + # return F.relu6(x + 3.) / 6. + return (x + 3).clamp(min=0, max=6).div(6.) # clamp seems ever so slightly faster? + + +class HardSigmoidJit(nn.Module): + def __init__(self, inplace: bool = False): + super(HardSigmoidJit, self).__init__() + + def forward(self, x): + return hard_sigmoid_jit(x) + + +@torch.jit.script +def hard_swish_jit(x, inplace: bool = False): + # return x * (F.relu6(x + 3.) / 6) + return x * (x + 3).clamp(min=0, max=6).div(6.) # clamp seems ever so slightly faster? + + +class HardSwishJit(nn.Module): + def __init__(self, inplace: bool = False): + super(HardSwishJit, self).__init__() + + def forward(self, x): + return hard_swish_jit(x) diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/activations/activations_me.py b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/activations/activations_me.py new file mode 100644 index 0000000000000000000000000000000000000000..e91df5a50fdbe40bc386e2541a4fda743ad95e9a --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/activations/activations_me.py @@ -0,0 +1,174 @@ +""" Activations (memory-efficient w/ custom autograd) + +A collection of activations fn and modules with a common interface so that they can +easily be swapped. All have an `inplace` arg even if not used. + +These activations are not compatible with jit scripting or ONNX export of the model, please use either +the JIT or basic versions of the activations. + +Copyright 2020 Ross Wightman +""" + +import torch +from torch import nn as nn +from torch.nn import functional as F + + +__all__ = ['swish_me', 'SwishMe', 'mish_me', 'MishMe', + 'hard_sigmoid_me', 'HardSigmoidMe', 'hard_swish_me', 'HardSwishMe'] + + +@torch.jit.script +def swish_jit_fwd(x): + return x.mul(torch.sigmoid(x)) + + +@torch.jit.script +def swish_jit_bwd(x, grad_output): + x_sigmoid = torch.sigmoid(x) + return grad_output * (x_sigmoid * (1 + x * (1 - x_sigmoid))) + + +class SwishJitAutoFn(torch.autograd.Function): + """ torch.jit.script optimised Swish w/ memory-efficient checkpoint + Inspired by conversation btw Jeremy Howard & Adam Pazske + https://twitter.com/jeremyphoward/status/1188251041835315200 + + Swish - Described originally as SiLU (https://arxiv.org/abs/1702.03118v3) + and also as Swish (https://arxiv.org/abs/1710.05941). + + TODO Rename to SiLU with addition to PyTorch + """ + + @staticmethod + def forward(ctx, x): + ctx.save_for_backward(x) + return swish_jit_fwd(x) + + @staticmethod + def backward(ctx, grad_output): + x = ctx.saved_tensors[0] + return swish_jit_bwd(x, grad_output) + + +def swish_me(x, inplace=False): + return SwishJitAutoFn.apply(x) + + +class SwishMe(nn.Module): + def __init__(self, inplace: bool = False): + super(SwishMe, self).__init__() + + def forward(self, x): + return SwishJitAutoFn.apply(x) + + +@torch.jit.script +def mish_jit_fwd(x): + return x.mul(torch.tanh(F.softplus(x))) + + +@torch.jit.script +def mish_jit_bwd(x, grad_output): + x_sigmoid = torch.sigmoid(x) + x_tanh_sp = F.softplus(x).tanh() + return grad_output.mul(x_tanh_sp + x * x_sigmoid * (1 - x_tanh_sp * x_tanh_sp)) + + +class MishJitAutoFn(torch.autograd.Function): + """ Mish: A Self Regularized Non-Monotonic Neural Activation Function - https://arxiv.org/abs/1908.08681 + A memory efficient, jit scripted variant of Mish + """ + @staticmethod + def forward(ctx, x): + ctx.save_for_backward(x) + return mish_jit_fwd(x) + + @staticmethod + def backward(ctx, grad_output): + x = ctx.saved_tensors[0] + return mish_jit_bwd(x, grad_output) + + +def mish_me(x, inplace=False): + return MishJitAutoFn.apply(x) + + +class MishMe(nn.Module): + def __init__(self, inplace: bool = False): + super(MishMe, self).__init__() + + def forward(self, x): + return MishJitAutoFn.apply(x) + + +@torch.jit.script +def hard_sigmoid_jit_fwd(x, inplace: bool = False): + return (x + 3).clamp(min=0, max=6).div(6.) + + +@torch.jit.script +def hard_sigmoid_jit_bwd(x, grad_output): + m = torch.ones_like(x) * ((x >= -3.) & (x <= 3.)) / 6. + return grad_output * m + + +class HardSigmoidJitAutoFn(torch.autograd.Function): + @staticmethod + def forward(ctx, x): + ctx.save_for_backward(x) + return hard_sigmoid_jit_fwd(x) + + @staticmethod + def backward(ctx, grad_output): + x = ctx.saved_tensors[0] + return hard_sigmoid_jit_bwd(x, grad_output) + + +def hard_sigmoid_me(x, inplace: bool = False): + return HardSigmoidJitAutoFn.apply(x) + + +class HardSigmoidMe(nn.Module): + def __init__(self, inplace: bool = False): + super(HardSigmoidMe, self).__init__() + + def forward(self, x): + return HardSigmoidJitAutoFn.apply(x) + + +@torch.jit.script +def hard_swish_jit_fwd(x): + return x * (x + 3).clamp(min=0, max=6).div(6.) + + +@torch.jit.script +def hard_swish_jit_bwd(x, grad_output): + m = torch.ones_like(x) * (x >= 3.) + m = torch.where((x >= -3.) & (x <= 3.), x / 3. + .5, m) + return grad_output * m + + +class HardSwishJitAutoFn(torch.autograd.Function): + """A memory efficient, jit-scripted HardSwish activation""" + @staticmethod + def forward(ctx, x): + ctx.save_for_backward(x) + return hard_swish_jit_fwd(x) + + @staticmethod + def backward(ctx, grad_output): + x = ctx.saved_tensors[0] + return hard_swish_jit_bwd(x, grad_output) + + +def hard_swish_me(x, inplace=False): + return HardSwishJitAutoFn.apply(x) + + +class HardSwishMe(nn.Module): + def __init__(self, inplace: bool = False): + super(HardSwishMe, self).__init__() + + def forward(self, x): + return HardSwishJitAutoFn.apply(x) diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/config.py b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/config.py new file mode 100644 index 0000000000000000000000000000000000000000..27d5307fd9ee0246f1e35f41520f17385d23f1dd --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/config.py @@ -0,0 +1,123 @@ +""" Global layer config state +""" +from typing import Any, Optional + +__all__ = [ + 'is_exportable', 'is_scriptable', 'is_no_jit', 'layer_config_kwargs', + 'set_exportable', 'set_scriptable', 'set_no_jit', 'set_layer_config' +] + +# Set to True if prefer to have layers with no jit optimization (includes activations) +_NO_JIT = False + +# Set to True if prefer to have activation layers with no jit optimization +# NOTE not currently used as no difference between no_jit and no_activation jit as only layers obeying +# the jit flags so far are activations. This will change as more layers are updated and/or added. +_NO_ACTIVATION_JIT = False + +# Set to True if exporting a model with Same padding via ONNX +_EXPORTABLE = False + +# Set to True if wanting to use torch.jit.script on a model +_SCRIPTABLE = False + + +def is_no_jit(): + return _NO_JIT + + +class set_no_jit: + def __init__(self, mode: bool) -> None: + global _NO_JIT + self.prev = _NO_JIT + _NO_JIT = mode + + def __enter__(self) -> None: + pass + + def __exit__(self, *args: Any) -> bool: + global _NO_JIT + _NO_JIT = self.prev + return False + + +def is_exportable(): + return _EXPORTABLE + + +class set_exportable: + def __init__(self, mode: bool) -> None: + global _EXPORTABLE + self.prev = _EXPORTABLE + _EXPORTABLE = mode + + def __enter__(self) -> None: + pass + + def __exit__(self, *args: Any) -> bool: + global _EXPORTABLE + _EXPORTABLE = self.prev + return False + + +def is_scriptable(): + return _SCRIPTABLE + + +class set_scriptable: + def __init__(self, mode: bool) -> None: + global _SCRIPTABLE + self.prev = _SCRIPTABLE + _SCRIPTABLE = mode + + def __enter__(self) -> None: + pass + + def __exit__(self, *args: Any) -> bool: + global _SCRIPTABLE + _SCRIPTABLE = self.prev + return False + + +class set_layer_config: + """ Layer config context manager that allows setting all layer config flags at once. + If a flag arg is None, it will not change the current value. + """ + def __init__( + self, + scriptable: Optional[bool] = None, + exportable: Optional[bool] = None, + no_jit: Optional[bool] = None, + no_activation_jit: Optional[bool] = None): + global _SCRIPTABLE + global _EXPORTABLE + global _NO_JIT + global _NO_ACTIVATION_JIT + self.prev = _SCRIPTABLE, _EXPORTABLE, _NO_JIT, _NO_ACTIVATION_JIT + if scriptable is not None: + _SCRIPTABLE = scriptable + if exportable is not None: + _EXPORTABLE = exportable + if no_jit is not None: + _NO_JIT = no_jit + if no_activation_jit is not None: + _NO_ACTIVATION_JIT = no_activation_jit + + def __enter__(self) -> None: + pass + + def __exit__(self, *args: Any) -> bool: + global _SCRIPTABLE + global _EXPORTABLE + global _NO_JIT + global _NO_ACTIVATION_JIT + _SCRIPTABLE, _EXPORTABLE, _NO_JIT, _NO_ACTIVATION_JIT = self.prev + return False + + +def layer_config_kwargs(kwargs): + """ Consume config kwargs and return contextmgr obj """ + return set_layer_config( + scriptable=kwargs.pop('scriptable', None), + exportable=kwargs.pop('exportable', None), + no_jit=kwargs.pop('no_jit', None)) diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/conv2d_layers.py b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/conv2d_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..d8467460c4b36e54c83ce2dcd3ebe91d3432cad2 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/conv2d_layers.py @@ -0,0 +1,304 @@ +""" Conv2D w/ SAME padding, CondConv, MixedConv + +A collection of conv layers and padding helpers needed by EfficientNet, MixNet, and +MobileNetV3 models that maintain weight compatibility with original Tensorflow models. + +Copyright 2020 Ross Wightman +""" +import collections.abc +import math +from functools import partial +from itertools import repeat +from typing import Tuple, Optional + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .config import * + + +# From PyTorch internals +def _ntuple(n): + def parse(x): + if isinstance(x, collections.abc.Iterable): + return x + return tuple(repeat(x, n)) + return parse + + +_single = _ntuple(1) +_pair = _ntuple(2) +_triple = _ntuple(3) +_quadruple = _ntuple(4) + + +def _is_static_pad(kernel_size, stride=1, dilation=1, **_): + return stride == 1 and (dilation * (kernel_size - 1)) % 2 == 0 + + +def _get_padding(kernel_size, stride=1, dilation=1, **_): + padding = ((stride - 1) + dilation * (kernel_size - 1)) // 2 + return padding + + +def _calc_same_pad(i: int, k: int, s: int, d: int): + return max((-(i // -s) - 1) * s + (k - 1) * d + 1 - i, 0) + + +def _same_pad_arg(input_size, kernel_size, stride, dilation): + ih, iw = input_size + kh, kw = kernel_size + pad_h = _calc_same_pad(ih, kh, stride[0], dilation[0]) + pad_w = _calc_same_pad(iw, kw, stride[1], dilation[1]) + return [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2] + + +def _split_channels(num_chan, num_groups): + split = [num_chan // num_groups for _ in range(num_groups)] + split[0] += num_chan - sum(split) + return split + + +def conv2d_same( + x, weight: torch.Tensor, bias: Optional[torch.Tensor] = None, stride: Tuple[int, int] = (1, 1), + padding: Tuple[int, int] = (0, 0), dilation: Tuple[int, int] = (1, 1), groups: int = 1): + ih, iw = x.size()[-2:] + kh, kw = weight.size()[-2:] + pad_h = _calc_same_pad(ih, kh, stride[0], dilation[0]) + pad_w = _calc_same_pad(iw, kw, stride[1], dilation[1]) + x = F.pad(x, [pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2]) + return F.conv2d(x, weight, bias, stride, (0, 0), dilation, groups) + + +class Conv2dSame(nn.Conv2d): + """ Tensorflow like 'SAME' convolution wrapper for 2D convolutions + """ + + # pylint: disable=unused-argument + def __init__(self, in_channels, out_channels, kernel_size, stride=1, + padding=0, dilation=1, groups=1, bias=True): + super(Conv2dSame, self).__init__( + in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias) + + def forward(self, x): + return conv2d_same(x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups) + + +class Conv2dSameExport(nn.Conv2d): + """ ONNX export friendly Tensorflow like 'SAME' convolution wrapper for 2D convolutions + + NOTE: This does not currently work with torch.jit.script + """ + + # pylint: disable=unused-argument + def __init__(self, in_channels, out_channels, kernel_size, stride=1, + padding=0, dilation=1, groups=1, bias=True): + super(Conv2dSameExport, self).__init__( + in_channels, out_channels, kernel_size, stride, 0, dilation, groups, bias) + self.pad = None + self.pad_input_size = (0, 0) + + def forward(self, x): + input_size = x.size()[-2:] + if self.pad is None: + pad_arg = _same_pad_arg(input_size, self.weight.size()[-2:], self.stride, self.dilation) + self.pad = nn.ZeroPad2d(pad_arg) + self.pad_input_size = input_size + + if self.pad is not None: + x = self.pad(x) + return F.conv2d( + x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups) + + +def get_padding_value(padding, kernel_size, **kwargs): + dynamic = False + if isinstance(padding, str): + # for any string padding, the padding will be calculated for you, one of three ways + padding = padding.lower() + if padding == 'same': + # TF compatible 'SAME' padding, has a performance and GPU memory allocation impact + if _is_static_pad(kernel_size, **kwargs): + # static case, no extra overhead + padding = _get_padding(kernel_size, **kwargs) + else: + # dynamic padding + padding = 0 + dynamic = True + elif padding == 'valid': + # 'VALID' padding, same as padding=0 + padding = 0 + else: + # Default to PyTorch style 'same'-ish symmetric padding + padding = _get_padding(kernel_size, **kwargs) + return padding, dynamic + + +def create_conv2d_pad(in_chs, out_chs, kernel_size, **kwargs): + padding = kwargs.pop('padding', '') + kwargs.setdefault('bias', False) + padding, is_dynamic = get_padding_value(padding, kernel_size, **kwargs) + if is_dynamic: + if is_exportable(): + assert not is_scriptable() + return Conv2dSameExport(in_chs, out_chs, kernel_size, **kwargs) + else: + return Conv2dSame(in_chs, out_chs, kernel_size, **kwargs) + else: + return nn.Conv2d(in_chs, out_chs, kernel_size, padding=padding, **kwargs) + + +class MixedConv2d(nn.ModuleDict): + """ Mixed Grouped Convolution + Based on MDConv and GroupedConv in MixNet impl: + https://github.com/tensorflow/tpu/blob/master/models/official/mnasnet/mixnet/custom_layers.py + """ + + def __init__(self, in_channels, out_channels, kernel_size=3, + stride=1, padding='', dilation=1, depthwise=False, **kwargs): + super(MixedConv2d, self).__init__() + + kernel_size = kernel_size if isinstance(kernel_size, list) else [kernel_size] + num_groups = len(kernel_size) + in_splits = _split_channels(in_channels, num_groups) + out_splits = _split_channels(out_channels, num_groups) + self.in_channels = sum(in_splits) + self.out_channels = sum(out_splits) + for idx, (k, in_ch, out_ch) in enumerate(zip(kernel_size, in_splits, out_splits)): + conv_groups = out_ch if depthwise else 1 + self.add_module( + str(idx), + create_conv2d_pad( + in_ch, out_ch, k, stride=stride, + padding=padding, dilation=dilation, groups=conv_groups, **kwargs) + ) + self.splits = in_splits + + def forward(self, x): + x_split = torch.split(x, self.splits, 1) + x_out = [conv(x_split[i]) for i, conv in enumerate(self.values())] + x = torch.cat(x_out, 1) + return x + + +def get_condconv_initializer(initializer, num_experts, expert_shape): + def condconv_initializer(weight): + """CondConv initializer function.""" + num_params = np.prod(expert_shape) + if (len(weight.shape) != 2 or weight.shape[0] != num_experts or + weight.shape[1] != num_params): + raise (ValueError( + 'CondConv variables must have shape [num_experts, num_params]')) + for i in range(num_experts): + initializer(weight[i].view(expert_shape)) + return condconv_initializer + + +class CondConv2d(nn.Module): + """ Conditional Convolution + Inspired by: https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/condconv/condconv_layers.py + + Grouped convolution hackery for parallel execution of the per-sample kernel filters inspired by this discussion: + https://github.com/pytorch/pytorch/issues/17983 + """ + __constants__ = ['bias', 'in_channels', 'out_channels', 'dynamic_padding'] + + def __init__(self, in_channels, out_channels, kernel_size=3, + stride=1, padding='', dilation=1, groups=1, bias=False, num_experts=4): + super(CondConv2d, self).__init__() + + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = _pair(stride) + padding_val, is_padding_dynamic = get_padding_value( + padding, kernel_size, stride=stride, dilation=dilation) + self.dynamic_padding = is_padding_dynamic # if in forward to work with torchscript + self.padding = _pair(padding_val) + self.dilation = _pair(dilation) + self.groups = groups + self.num_experts = num_experts + + self.weight_shape = (self.out_channels, self.in_channels // self.groups) + self.kernel_size + weight_num_param = 1 + for wd in self.weight_shape: + weight_num_param *= wd + self.weight = torch.nn.Parameter(torch.Tensor(self.num_experts, weight_num_param)) + + if bias: + self.bias_shape = (self.out_channels,) + self.bias = torch.nn.Parameter(torch.Tensor(self.num_experts, self.out_channels)) + else: + self.register_parameter('bias', None) + + self.reset_parameters() + + def reset_parameters(self): + init_weight = get_condconv_initializer( + partial(nn.init.kaiming_uniform_, a=math.sqrt(5)), self.num_experts, self.weight_shape) + init_weight(self.weight) + if self.bias is not None: + fan_in = np.prod(self.weight_shape[1:]) + bound = 1 / math.sqrt(fan_in) + init_bias = get_condconv_initializer( + partial(nn.init.uniform_, a=-bound, b=bound), self.num_experts, self.bias_shape) + init_bias(self.bias) + + def forward(self, x, routing_weights): + B, C, H, W = x.shape + weight = torch.matmul(routing_weights, self.weight) + new_weight_shape = (B * self.out_channels, self.in_channels // self.groups) + self.kernel_size + weight = weight.view(new_weight_shape) + bias = None + if self.bias is not None: + bias = torch.matmul(routing_weights, self.bias) + bias = bias.view(B * self.out_channels) + # move batch elements with channels so each batch element can be efficiently convolved with separate kernel + x = x.view(1, B * C, H, W) + if self.dynamic_padding: + out = conv2d_same( + x, weight, bias, stride=self.stride, padding=self.padding, + dilation=self.dilation, groups=self.groups * B) + else: + out = F.conv2d( + x, weight, bias, stride=self.stride, padding=self.padding, + dilation=self.dilation, groups=self.groups * B) + out = out.permute([1, 0, 2, 3]).view(B, self.out_channels, out.shape[-2], out.shape[-1]) + + # Literal port (from TF definition) + # x = torch.split(x, 1, 0) + # weight = torch.split(weight, 1, 0) + # if self.bias is not None: + # bias = torch.matmul(routing_weights, self.bias) + # bias = torch.split(bias, 1, 0) + # else: + # bias = [None] * B + # out = [] + # for xi, wi, bi in zip(x, weight, bias): + # wi = wi.view(*self.weight_shape) + # if bi is not None: + # bi = bi.view(*self.bias_shape) + # out.append(self.conv_fn( + # xi, wi, bi, stride=self.stride, padding=self.padding, + # dilation=self.dilation, groups=self.groups)) + # out = torch.cat(out, 0) + return out + + +def select_conv2d(in_chs, out_chs, kernel_size, **kwargs): + assert 'groups' not in kwargs # only use 'depthwise' bool arg + if isinstance(kernel_size, list): + assert 'num_experts' not in kwargs # MixNet + CondConv combo not supported currently + # We're going to use only lists for defining the MixedConv2d kernel groups, + # ints, tuples, other iterables will continue to pass to normal conv and specify h, w. + m = MixedConv2d(in_chs, out_chs, kernel_size, **kwargs) + else: + depthwise = kwargs.pop('depthwise', False) + groups = out_chs if depthwise else 1 + if 'num_experts' in kwargs and kwargs['num_experts'] > 0: + m = CondConv2d(in_chs, out_chs, kernel_size, groups=groups, **kwargs) + else: + m = create_conv2d_pad(in_chs, out_chs, kernel_size, groups=groups, **kwargs) + return m diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/efficientnet_builder.py b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/efficientnet_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..95dd63d400e70d70664c5a433a2772363f865e61 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/efficientnet_builder.py @@ -0,0 +1,683 @@ +""" EfficientNet / MobileNetV3 Blocks and Builder + +Copyright 2020 Ross Wightman +""" +import re +from copy import deepcopy + +from .conv2d_layers import * +from geffnet.activations import * + +__all__ = ['get_bn_args_tf', 'resolve_bn_args', 'resolve_se_args', 'resolve_act_layer', 'make_divisible', + 'round_channels', 'drop_connect', 'SqueezeExcite', 'ConvBnAct', 'DepthwiseSeparableConv', + 'InvertedResidual', 'CondConvResidual', 'EdgeResidual', 'EfficientNetBuilder', 'decode_arch_def', + 'initialize_weight_default', 'initialize_weight_goog', 'BN_MOMENTUM_TF_DEFAULT', 'BN_EPS_TF_DEFAULT' +] + +# Defaults used for Google/Tensorflow training of mobile networks /w RMSprop as per +# papers and TF reference implementations. PT momentum equiv for TF decay is (1 - TF decay) +# NOTE: momentum varies btw .99 and .9997 depending on source +# .99 in official TF TPU impl +# .9997 (/w .999 in search space) for paper +# +# PyTorch defaults are momentum = .1, eps = 1e-5 +# +BN_MOMENTUM_TF_DEFAULT = 1 - 0.99 +BN_EPS_TF_DEFAULT = 1e-3 +_BN_ARGS_TF = dict(momentum=BN_MOMENTUM_TF_DEFAULT, eps=BN_EPS_TF_DEFAULT) + + +def get_bn_args_tf(): + return _BN_ARGS_TF.copy() + + +def resolve_bn_args(kwargs): + bn_args = get_bn_args_tf() if kwargs.pop('bn_tf', False) else {} + bn_momentum = kwargs.pop('bn_momentum', None) + if bn_momentum is not None: + bn_args['momentum'] = bn_momentum + bn_eps = kwargs.pop('bn_eps', None) + if bn_eps is not None: + bn_args['eps'] = bn_eps + return bn_args + + +_SE_ARGS_DEFAULT = dict( + gate_fn=sigmoid, + act_layer=None, # None == use containing block's activation layer + reduce_mid=False, + divisor=1) + + +def resolve_se_args(kwargs, in_chs, act_layer=None): + se_kwargs = kwargs.copy() if kwargs is not None else {} + # fill in args that aren't specified with the defaults + for k, v in _SE_ARGS_DEFAULT.items(): + se_kwargs.setdefault(k, v) + # some models, like MobilNetV3, calculate SE reduction chs from the containing block's mid_ch instead of in_ch + if not se_kwargs.pop('reduce_mid'): + se_kwargs['reduced_base_chs'] = in_chs + # act_layer override, if it remains None, the containing block's act_layer will be used + if se_kwargs['act_layer'] is None: + assert act_layer is not None + se_kwargs['act_layer'] = act_layer + return se_kwargs + + +def resolve_act_layer(kwargs, default='relu'): + act_layer = kwargs.pop('act_layer', default) + if isinstance(act_layer, str): + act_layer = get_act_layer(act_layer) + return act_layer + + +def make_divisible(v: int, divisor: int = 8, min_value: int = None): + min_value = min_value or divisor + new_v = max(min_value, int(v + divisor / 2) // divisor * divisor) + if new_v < 0.9 * v: # ensure round down does not go down by more than 10%. + new_v += divisor + return new_v + + +def round_channels(channels, multiplier=1.0, divisor=8, channel_min=None): + """Round number of filters based on depth multiplier.""" + if not multiplier: + return channels + channels *= multiplier + return make_divisible(channels, divisor, channel_min) + + +def drop_connect(inputs, training: bool = False, drop_connect_rate: float = 0.): + """Apply drop connect.""" + if not training: + return inputs + + keep_prob = 1 - drop_connect_rate + random_tensor = keep_prob + torch.rand( + (inputs.size()[0], 1, 1, 1), dtype=inputs.dtype, device=inputs.device) + random_tensor.floor_() # binarize + output = inputs.div(keep_prob) * random_tensor + return output + + +class SqueezeExcite(nn.Module): + + def __init__(self, in_chs, se_ratio=0.25, reduced_base_chs=None, act_layer=nn.ReLU, gate_fn=sigmoid, divisor=1): + super(SqueezeExcite, self).__init__() + reduced_chs = make_divisible((reduced_base_chs or in_chs) * se_ratio, divisor) + self.conv_reduce = nn.Conv2d(in_chs, reduced_chs, 1, bias=True) + self.act1 = act_layer(inplace=True) + self.conv_expand = nn.Conv2d(reduced_chs, in_chs, 1, bias=True) + self.gate_fn = gate_fn + + def forward(self, x): + x_se = x.mean((2, 3), keepdim=True) + x_se = self.conv_reduce(x_se) + x_se = self.act1(x_se) + x_se = self.conv_expand(x_se) + x = x * self.gate_fn(x_se) + return x + + +class ConvBnAct(nn.Module): + def __init__(self, in_chs, out_chs, kernel_size, + stride=1, pad_type='', act_layer=nn.ReLU, norm_layer=nn.BatchNorm2d, norm_kwargs=None): + super(ConvBnAct, self).__init__() + assert stride in [1, 2] + norm_kwargs = norm_kwargs or {} + self.conv = select_conv2d(in_chs, out_chs, kernel_size, stride=stride, padding=pad_type) + self.bn1 = norm_layer(out_chs, **norm_kwargs) + self.act1 = act_layer(inplace=True) + + def forward(self, x): + x = self.conv(x) + x = self.bn1(x) + x = self.act1(x) + return x + + +class DepthwiseSeparableConv(nn.Module): + """ DepthwiseSeparable block + Used for DS convs in MobileNet-V1 and in the place of IR blocks with an expansion + factor of 1.0. This is an alternative to having a IR with optional first pw conv. + """ + def __init__(self, in_chs, out_chs, dw_kernel_size=3, + stride=1, pad_type='', act_layer=nn.ReLU, noskip=False, + pw_kernel_size=1, pw_act=False, se_ratio=0., se_kwargs=None, + norm_layer=nn.BatchNorm2d, norm_kwargs=None, drop_connect_rate=0.): + super(DepthwiseSeparableConv, self).__init__() + assert stride in [1, 2] + norm_kwargs = norm_kwargs or {} + self.has_residual = (stride == 1 and in_chs == out_chs) and not noskip + self.drop_connect_rate = drop_connect_rate + + self.conv_dw = select_conv2d( + in_chs, in_chs, dw_kernel_size, stride=stride, padding=pad_type, depthwise=True) + self.bn1 = norm_layer(in_chs, **norm_kwargs) + self.act1 = act_layer(inplace=True) + + # Squeeze-and-excitation + if se_ratio is not None and se_ratio > 0.: + se_kwargs = resolve_se_args(se_kwargs, in_chs, act_layer) + self.se = SqueezeExcite(in_chs, se_ratio=se_ratio, **se_kwargs) + else: + self.se = nn.Identity() + + self.conv_pw = select_conv2d(in_chs, out_chs, pw_kernel_size, padding=pad_type) + self.bn2 = norm_layer(out_chs, **norm_kwargs) + self.act2 = act_layer(inplace=True) if pw_act else nn.Identity() + + def forward(self, x): + residual = x + + x = self.conv_dw(x) + x = self.bn1(x) + x = self.act1(x) + + x = self.se(x) + + x = self.conv_pw(x) + x = self.bn2(x) + x = self.act2(x) + + if self.has_residual: + if self.drop_connect_rate > 0.: + x = drop_connect(x, self.training, self.drop_connect_rate) + x += residual + return x + + +class InvertedResidual(nn.Module): + """ Inverted residual block w/ optional SE""" + + def __init__(self, in_chs, out_chs, dw_kernel_size=3, + stride=1, pad_type='', act_layer=nn.ReLU, noskip=False, + exp_ratio=1.0, exp_kernel_size=1, pw_kernel_size=1, + se_ratio=0., se_kwargs=None, norm_layer=nn.BatchNorm2d, norm_kwargs=None, + conv_kwargs=None, drop_connect_rate=0.): + super(InvertedResidual, self).__init__() + norm_kwargs = norm_kwargs or {} + conv_kwargs = conv_kwargs or {} + mid_chs: int = make_divisible(in_chs * exp_ratio) + self.has_residual = (in_chs == out_chs and stride == 1) and not noskip + self.drop_connect_rate = drop_connect_rate + + # Point-wise expansion + self.conv_pw = select_conv2d(in_chs, mid_chs, exp_kernel_size, padding=pad_type, **conv_kwargs) + self.bn1 = norm_layer(mid_chs, **norm_kwargs) + self.act1 = act_layer(inplace=True) + + # Depth-wise convolution + self.conv_dw = select_conv2d( + mid_chs, mid_chs, dw_kernel_size, stride=stride, padding=pad_type, depthwise=True, **conv_kwargs) + self.bn2 = norm_layer(mid_chs, **norm_kwargs) + self.act2 = act_layer(inplace=True) + + # Squeeze-and-excitation + if se_ratio is not None and se_ratio > 0.: + se_kwargs = resolve_se_args(se_kwargs, in_chs, act_layer) + self.se = SqueezeExcite(mid_chs, se_ratio=se_ratio, **se_kwargs) + else: + self.se = nn.Identity() # for jit.script compat + + # Point-wise linear projection + self.conv_pwl = select_conv2d(mid_chs, out_chs, pw_kernel_size, padding=pad_type, **conv_kwargs) + self.bn3 = norm_layer(out_chs, **norm_kwargs) + + def forward(self, x): + residual = x + + # Point-wise expansion + x = self.conv_pw(x) + x = self.bn1(x) + x = self.act1(x) + + # Depth-wise convolution + x = self.conv_dw(x) + x = self.bn2(x) + x = self.act2(x) + + # Squeeze-and-excitation + x = self.se(x) + + # Point-wise linear projection + x = self.conv_pwl(x) + x = self.bn3(x) + + if self.has_residual: + if self.drop_connect_rate > 0.: + x = drop_connect(x, self.training, self.drop_connect_rate) + x += residual + return x + + +class CondConvResidual(InvertedResidual): + """ Inverted residual block w/ CondConv routing""" + + def __init__(self, in_chs, out_chs, dw_kernel_size=3, + stride=1, pad_type='', act_layer=nn.ReLU, noskip=False, + exp_ratio=1.0, exp_kernel_size=1, pw_kernel_size=1, + se_ratio=0., se_kwargs=None, norm_layer=nn.BatchNorm2d, norm_kwargs=None, + num_experts=0, drop_connect_rate=0.): + + self.num_experts = num_experts + conv_kwargs = dict(num_experts=self.num_experts) + + super(CondConvResidual, self).__init__( + in_chs, out_chs, dw_kernel_size=dw_kernel_size, stride=stride, pad_type=pad_type, + act_layer=act_layer, noskip=noskip, exp_ratio=exp_ratio, exp_kernel_size=exp_kernel_size, + pw_kernel_size=pw_kernel_size, se_ratio=se_ratio, se_kwargs=se_kwargs, + norm_layer=norm_layer, norm_kwargs=norm_kwargs, conv_kwargs=conv_kwargs, + drop_connect_rate=drop_connect_rate) + + self.routing_fn = nn.Linear(in_chs, self.num_experts) + + def forward(self, x): + residual = x + + # CondConv routing + pooled_inputs = F.adaptive_avg_pool2d(x, 1).flatten(1) + routing_weights = torch.sigmoid(self.routing_fn(pooled_inputs)) + + # Point-wise expansion + x = self.conv_pw(x, routing_weights) + x = self.bn1(x) + x = self.act1(x) + + # Depth-wise convolution + x = self.conv_dw(x, routing_weights) + x = self.bn2(x) + x = self.act2(x) + + # Squeeze-and-excitation + x = self.se(x) + + # Point-wise linear projection + x = self.conv_pwl(x, routing_weights) + x = self.bn3(x) + + if self.has_residual: + if self.drop_connect_rate > 0.: + x = drop_connect(x, self.training, self.drop_connect_rate) + x += residual + return x + + +class EdgeResidual(nn.Module): + """ EdgeTPU Residual block with expansion convolution followed by pointwise-linear w/ stride""" + + def __init__(self, in_chs, out_chs, exp_kernel_size=3, exp_ratio=1.0, fake_in_chs=0, + stride=1, pad_type='', act_layer=nn.ReLU, noskip=False, pw_kernel_size=1, + se_ratio=0., se_kwargs=None, norm_layer=nn.BatchNorm2d, norm_kwargs=None, drop_connect_rate=0.): + super(EdgeResidual, self).__init__() + norm_kwargs = norm_kwargs or {} + mid_chs = make_divisible(fake_in_chs * exp_ratio) if fake_in_chs > 0 else make_divisible(in_chs * exp_ratio) + self.has_residual = (in_chs == out_chs and stride == 1) and not noskip + self.drop_connect_rate = drop_connect_rate + + # Expansion convolution + self.conv_exp = select_conv2d(in_chs, mid_chs, exp_kernel_size, padding=pad_type) + self.bn1 = norm_layer(mid_chs, **norm_kwargs) + self.act1 = act_layer(inplace=True) + + # Squeeze-and-excitation + if se_ratio is not None and se_ratio > 0.: + se_kwargs = resolve_se_args(se_kwargs, in_chs, act_layer) + self.se = SqueezeExcite(mid_chs, se_ratio=se_ratio, **se_kwargs) + else: + self.se = nn.Identity() + + # Point-wise linear projection + self.conv_pwl = select_conv2d(mid_chs, out_chs, pw_kernel_size, stride=stride, padding=pad_type) + self.bn2 = nn.BatchNorm2d(out_chs, **norm_kwargs) + + def forward(self, x): + residual = x + + # Expansion convolution + x = self.conv_exp(x) + x = self.bn1(x) + x = self.act1(x) + + # Squeeze-and-excitation + x = self.se(x) + + # Point-wise linear projection + x = self.conv_pwl(x) + x = self.bn2(x) + + if self.has_residual: + if self.drop_connect_rate > 0.: + x = drop_connect(x, self.training, self.drop_connect_rate) + x += residual + + return x + + +class EfficientNetBuilder: + """ Build Trunk Blocks for Efficient/Mobile Networks + + This ended up being somewhat of a cross between + https://github.com/tensorflow/tpu/blob/master/models/official/mnasnet/mnasnet_models.py + and + https://github.com/facebookresearch/maskrcnn-benchmark/blob/master/maskrcnn_benchmark/modeling/backbone/fbnet_builder.py + + """ + + def __init__(self, channel_multiplier=1.0, channel_divisor=8, channel_min=None, + pad_type='', act_layer=None, se_kwargs=None, + norm_layer=nn.BatchNorm2d, norm_kwargs=None, drop_connect_rate=0.): + self.channel_multiplier = channel_multiplier + self.channel_divisor = channel_divisor + self.channel_min = channel_min + self.pad_type = pad_type + self.act_layer = act_layer + self.se_kwargs = se_kwargs + self.norm_layer = norm_layer + self.norm_kwargs = norm_kwargs + self.drop_connect_rate = drop_connect_rate + + # updated during build + self.in_chs = None + self.block_idx = 0 + self.block_count = 0 + + def _round_channels(self, chs): + return round_channels(chs, self.channel_multiplier, self.channel_divisor, self.channel_min) + + def _make_block(self, ba): + bt = ba.pop('block_type') + ba['in_chs'] = self.in_chs + ba['out_chs'] = self._round_channels(ba['out_chs']) + if 'fake_in_chs' in ba and ba['fake_in_chs']: + # FIXME this is a hack to work around mismatch in origin impl input filters for EdgeTPU + ba['fake_in_chs'] = self._round_channels(ba['fake_in_chs']) + ba['norm_layer'] = self.norm_layer + ba['norm_kwargs'] = self.norm_kwargs + ba['pad_type'] = self.pad_type + # block act fn overrides the model default + ba['act_layer'] = ba['act_layer'] if ba['act_layer'] is not None else self.act_layer + assert ba['act_layer'] is not None + if bt == 'ir': + ba['drop_connect_rate'] = self.drop_connect_rate * self.block_idx / self.block_count + ba['se_kwargs'] = self.se_kwargs + if ba.get('num_experts', 0) > 0: + block = CondConvResidual(**ba) + else: + block = InvertedResidual(**ba) + elif bt == 'ds' or bt == 'dsa': + ba['drop_connect_rate'] = self.drop_connect_rate * self.block_idx / self.block_count + ba['se_kwargs'] = self.se_kwargs + block = DepthwiseSeparableConv(**ba) + elif bt == 'er': + ba['drop_connect_rate'] = self.drop_connect_rate * self.block_idx / self.block_count + ba['se_kwargs'] = self.se_kwargs + block = EdgeResidual(**ba) + elif bt == 'cn': + block = ConvBnAct(**ba) + else: + assert False, 'Uknkown block type (%s) while building model.' % bt + self.in_chs = ba['out_chs'] # update in_chs for arg of next block + return block + + def _make_stack(self, stack_args): + blocks = [] + # each stack (stage) contains a list of block arguments + for i, ba in enumerate(stack_args): + if i >= 1: + # only the first block in any stack can have a stride > 1 + ba['stride'] = 1 + block = self._make_block(ba) + blocks.append(block) + self.block_idx += 1 # incr global idx (across all stacks) + return nn.Sequential(*blocks) + + def __call__(self, in_chs, block_args): + """ Build the blocks + Args: + in_chs: Number of input-channels passed to first block + block_args: A list of lists, outer list defines stages, inner + list contains strings defining block configuration(s) + Return: + List of block stacks (each stack wrapped in nn.Sequential) + """ + self.in_chs = in_chs + self.block_count = sum([len(x) for x in block_args]) + self.block_idx = 0 + blocks = [] + # outer list of block_args defines the stacks ('stages' by some conventions) + for stack_idx, stack in enumerate(block_args): + assert isinstance(stack, list) + stack = self._make_stack(stack) + blocks.append(stack) + return blocks + + +def _parse_ksize(ss): + if ss.isdigit(): + return int(ss) + else: + return [int(k) for k in ss.split('.')] + + +def _decode_block_str(block_str): + """ Decode block definition string + + Gets a list of block arg (dicts) through a string notation of arguments. + E.g. ir_r2_k3_s2_e1_i32_o16_se0.25_noskip + + All args can exist in any order with the exception of the leading string which + is assumed to indicate the block type. + + leading string - block type ( + ir = InvertedResidual, ds = DepthwiseSep, dsa = DeptwhiseSep with pw act, cn = ConvBnAct) + r - number of repeat blocks, + k - kernel size, + s - strides (1-9), + e - expansion ratio, + c - output channels, + se - squeeze/excitation ratio + n - activation fn ('re', 'r6', 'hs', or 'sw') + Args: + block_str: a string representation of block arguments. + Returns: + A list of block args (dicts) + Raises: + ValueError: if the string def not properly specified (TODO) + """ + assert isinstance(block_str, str) + ops = block_str.split('_') + block_type = ops[0] # take the block type off the front + ops = ops[1:] + options = {} + noskip = False + for op in ops: + # string options being checked on individual basis, combine if they grow + if op == 'noskip': + noskip = True + elif op.startswith('n'): + # activation fn + key = op[0] + v = op[1:] + if v == 're': + value = get_act_layer('relu') + elif v == 'r6': + value = get_act_layer('relu6') + elif v == 'hs': + value = get_act_layer('hard_swish') + elif v == 'sw': + value = get_act_layer('swish') + else: + continue + options[key] = value + else: + # all numeric options + splits = re.split(r'(\d.*)', op) + if len(splits) >= 2: + key, value = splits[:2] + options[key] = value + + # if act_layer is None, the model default (passed to model init) will be used + act_layer = options['n'] if 'n' in options else None + exp_kernel_size = _parse_ksize(options['a']) if 'a' in options else 1 + pw_kernel_size = _parse_ksize(options['p']) if 'p' in options else 1 + fake_in_chs = int(options['fc']) if 'fc' in options else 0 # FIXME hack to deal with in_chs issue in TPU def + + num_repeat = int(options['r']) + # each type of block has different valid arguments, fill accordingly + if block_type == 'ir': + block_args = dict( + block_type=block_type, + dw_kernel_size=_parse_ksize(options['k']), + exp_kernel_size=exp_kernel_size, + pw_kernel_size=pw_kernel_size, + out_chs=int(options['c']), + exp_ratio=float(options['e']), + se_ratio=float(options['se']) if 'se' in options else None, + stride=int(options['s']), + act_layer=act_layer, + noskip=noskip, + ) + if 'cc' in options: + block_args['num_experts'] = int(options['cc']) + elif block_type == 'ds' or block_type == 'dsa': + block_args = dict( + block_type=block_type, + dw_kernel_size=_parse_ksize(options['k']), + pw_kernel_size=pw_kernel_size, + out_chs=int(options['c']), + se_ratio=float(options['se']) if 'se' in options else None, + stride=int(options['s']), + act_layer=act_layer, + pw_act=block_type == 'dsa', + noskip=block_type == 'dsa' or noskip, + ) + elif block_type == 'er': + block_args = dict( + block_type=block_type, + exp_kernel_size=_parse_ksize(options['k']), + pw_kernel_size=pw_kernel_size, + out_chs=int(options['c']), + exp_ratio=float(options['e']), + fake_in_chs=fake_in_chs, + se_ratio=float(options['se']) if 'se' in options else None, + stride=int(options['s']), + act_layer=act_layer, + noskip=noskip, + ) + elif block_type == 'cn': + block_args = dict( + block_type=block_type, + kernel_size=int(options['k']), + out_chs=int(options['c']), + stride=int(options['s']), + act_layer=act_layer, + ) + else: + assert False, 'Unknown block type (%s)' % block_type + + return block_args, num_repeat + + +def _scale_stage_depth(stack_args, repeats, depth_multiplier=1.0, depth_trunc='ceil'): + """ Per-stage depth scaling + Scales the block repeats in each stage. This depth scaling impl maintains + compatibility with the EfficientNet scaling method, while allowing sensible + scaling for other models that may have multiple block arg definitions in each stage. + """ + + # We scale the total repeat count for each stage, there may be multiple + # block arg defs per stage so we need to sum. + num_repeat = sum(repeats) + if depth_trunc == 'round': + # Truncating to int by rounding allows stages with few repeats to remain + # proportionally smaller for longer. This is a good choice when stage definitions + # include single repeat stages that we'd prefer to keep that way as long as possible + num_repeat_scaled = max(1, round(num_repeat * depth_multiplier)) + else: + # The default for EfficientNet truncates repeats to int via 'ceil'. + # Any multiplier > 1.0 will result in an increased depth for every stage. + num_repeat_scaled = int(math.ceil(num_repeat * depth_multiplier)) + + # Proportionally distribute repeat count scaling to each block definition in the stage. + # Allocation is done in reverse as it results in the first block being less likely to be scaled. + # The first block makes less sense to repeat in most of the arch definitions. + repeats_scaled = [] + for r in repeats[::-1]: + rs = max(1, round((r / num_repeat * num_repeat_scaled))) + repeats_scaled.append(rs) + num_repeat -= r + num_repeat_scaled -= rs + repeats_scaled = repeats_scaled[::-1] + + # Apply the calculated scaling to each block arg in the stage + sa_scaled = [] + for ba, rep in zip(stack_args, repeats_scaled): + sa_scaled.extend([deepcopy(ba) for _ in range(rep)]) + return sa_scaled + + +def decode_arch_def(arch_def, depth_multiplier=1.0, depth_trunc='ceil', experts_multiplier=1, fix_first_last=False): + arch_args = [] + for stack_idx, block_strings in enumerate(arch_def): + assert isinstance(block_strings, list) + stack_args = [] + repeats = [] + for block_str in block_strings: + assert isinstance(block_str, str) + ba, rep = _decode_block_str(block_str) + if ba.get('num_experts', 0) > 0 and experts_multiplier > 1: + ba['num_experts'] *= experts_multiplier + stack_args.append(ba) + repeats.append(rep) + if fix_first_last and (stack_idx == 0 or stack_idx == len(arch_def) - 1): + arch_args.append(_scale_stage_depth(stack_args, repeats, 1.0, depth_trunc)) + else: + arch_args.append(_scale_stage_depth(stack_args, repeats, depth_multiplier, depth_trunc)) + return arch_args + + +def initialize_weight_goog(m, n='', fix_group_fanout=True): + # weight init as per Tensorflow Official impl + # https://github.com/tensorflow/tpu/blob/master/models/official/mnasnet/mnasnet_model.py + if isinstance(m, CondConv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + if fix_group_fanout: + fan_out //= m.groups + init_weight_fn = get_condconv_initializer( + lambda w: w.data.normal_(0, math.sqrt(2.0 / fan_out)), m.num_experts, m.weight_shape) + init_weight_fn(m.weight) + if m.bias is not None: + m.bias.data.zero_() + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + if fix_group_fanout: + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1.0) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + fan_out = m.weight.size(0) # fan-out + fan_in = 0 + if 'routing_fn' in n: + fan_in = m.weight.size(1) + init_range = 1.0 / math.sqrt(fan_in + fan_out) + m.weight.data.uniform_(-init_range, init_range) + m.bias.data.zero_() + + +def initialize_weight_default(m, n=''): + if isinstance(m, CondConv2d): + init_fn = get_condconv_initializer(partial( + nn.init.kaiming_normal_, mode='fan_out', nonlinearity='relu'), m.num_experts, m.weight_shape) + init_fn(m.weight) + elif isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1.0) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + nn.init.kaiming_uniform_(m.weight, mode='fan_in', nonlinearity='linear') diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/gen_efficientnet.py b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/gen_efficientnet.py new file mode 100644 index 0000000000000000000000000000000000000000..cd170d4cc5bed6ca82b61539902b470d3320c691 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/gen_efficientnet.py @@ -0,0 +1,1450 @@ +""" Generic Efficient Networks + +A generic MobileNet class with building blocks to support a variety of models: + +* EfficientNet (B0-B8, L2 + Tensorflow pretrained AutoAug/RandAug/AdvProp/NoisyStudent ports) + - EfficientNet: Rethinking Model Scaling for CNNs - https://arxiv.org/abs/1905.11946 + - CondConv: Conditionally Parameterized Convolutions for Efficient Inference - https://arxiv.org/abs/1904.04971 + - Adversarial Examples Improve Image Recognition - https://arxiv.org/abs/1911.09665 + - Self-training with Noisy Student improves ImageNet classification - https://arxiv.org/abs/1911.04252 + +* EfficientNet-Lite + +* MixNet (Small, Medium, and Large) + - MixConv: Mixed Depthwise Convolutional Kernels - https://arxiv.org/abs/1907.09595 + +* MNasNet B1, A1 (SE), Small + - MnasNet: Platform-Aware Neural Architecture Search for Mobile - https://arxiv.org/abs/1807.11626 + +* FBNet-C + - FBNet: Hardware-Aware Efficient ConvNet Design via Differentiable NAS - https://arxiv.org/abs/1812.03443 + +* Single-Path NAS Pixel1 + - Single-Path NAS: Designing Hardware-Efficient ConvNets - https://arxiv.org/abs/1904.02877 + +* And likely more... + +Hacked together by / Copyright 2020 Ross Wightman +""" +import torch.nn as nn +import torch.nn.functional as F + +from .config import layer_config_kwargs, is_scriptable +from .conv2d_layers import select_conv2d +from .helpers import load_pretrained +from .efficientnet_builder import * + +__all__ = ['GenEfficientNet', 'mnasnet_050', 'mnasnet_075', 'mnasnet_100', 'mnasnet_b1', 'mnasnet_140', + 'semnasnet_050', 'semnasnet_075', 'semnasnet_100', 'mnasnet_a1', 'semnasnet_140', 'mnasnet_small', + 'mobilenetv2_100', 'mobilenetv2_140', 'mobilenetv2_110d', 'mobilenetv2_120d', + 'fbnetc_100', 'spnasnet_100', 'efficientnet_b0', 'efficientnet_b1', 'efficientnet_b2', 'efficientnet_b3', + 'efficientnet_b4', 'efficientnet_b5', 'efficientnet_b6', 'efficientnet_b7', 'efficientnet_b8', + 'efficientnet_l2', 'efficientnet_es', 'efficientnet_em', 'efficientnet_el', + 'efficientnet_cc_b0_4e', 'efficientnet_cc_b0_8e', 'efficientnet_cc_b1_8e', + 'efficientnet_lite0', 'efficientnet_lite1', 'efficientnet_lite2', 'efficientnet_lite3', 'efficientnet_lite4', + 'tf_efficientnet_b0', 'tf_efficientnet_b1', 'tf_efficientnet_b2', 'tf_efficientnet_b3', + 'tf_efficientnet_b4', 'tf_efficientnet_b5', 'tf_efficientnet_b6', 'tf_efficientnet_b7', 'tf_efficientnet_b8', + 'tf_efficientnet_b0_ap', 'tf_efficientnet_b1_ap', 'tf_efficientnet_b2_ap', 'tf_efficientnet_b3_ap', + 'tf_efficientnet_b4_ap', 'tf_efficientnet_b5_ap', 'tf_efficientnet_b6_ap', 'tf_efficientnet_b7_ap', + 'tf_efficientnet_b8_ap', 'tf_efficientnet_b0_ns', 'tf_efficientnet_b1_ns', 'tf_efficientnet_b2_ns', + 'tf_efficientnet_b3_ns', 'tf_efficientnet_b4_ns', 'tf_efficientnet_b5_ns', 'tf_efficientnet_b6_ns', + 'tf_efficientnet_b7_ns', 'tf_efficientnet_l2_ns', 'tf_efficientnet_l2_ns_475', + 'tf_efficientnet_es', 'tf_efficientnet_em', 'tf_efficientnet_el', + 'tf_efficientnet_cc_b0_4e', 'tf_efficientnet_cc_b0_8e', 'tf_efficientnet_cc_b1_8e', + 'tf_efficientnet_lite0', 'tf_efficientnet_lite1', 'tf_efficientnet_lite2', 'tf_efficientnet_lite3', + 'tf_efficientnet_lite4', + 'mixnet_s', 'mixnet_m', 'mixnet_l', 'mixnet_xl', 'tf_mixnet_s', 'tf_mixnet_m', 'tf_mixnet_l'] + + +model_urls = { + 'mnasnet_050': None, + 'mnasnet_075': None, + 'mnasnet_100': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/mnasnet_b1-74cb7081.pth', + 'mnasnet_140': None, + 'mnasnet_small': None, + + 'semnasnet_050': None, + 'semnasnet_075': None, + 'semnasnet_100': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/mnasnet_a1-d9418771.pth', + 'semnasnet_140': None, + + 'mobilenetv2_100': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/mobilenetv2_100_ra-b33bc2c4.pth', + 'mobilenetv2_110d': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/mobilenetv2_110d_ra-77090ade.pth', + 'mobilenetv2_120d': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/mobilenetv2_120d_ra-5987e2ed.pth', + 'mobilenetv2_140': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/mobilenetv2_140_ra-21a4e913.pth', + + 'fbnetc_100': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/fbnetc_100-c345b898.pth', + 'spnasnet_100': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/spnasnet_100-048bc3f4.pth', + + 'efficientnet_b0': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/efficientnet_b0_ra-3dd342df.pth', + 'efficientnet_b1': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/efficientnet_b1-533bc792.pth', + 'efficientnet_b2': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/efficientnet_b2_ra-bcdf34b7.pth', + 'efficientnet_b3': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/efficientnet_b3_ra2-cf984f9c.pth', + 'efficientnet_b4': None, + 'efficientnet_b5': None, + 'efficientnet_b6': None, + 'efficientnet_b7': None, + 'efficientnet_b8': None, + 'efficientnet_l2': None, + + 'efficientnet_es': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/efficientnet_es_ra-f111e99c.pth', + 'efficientnet_em': None, + 'efficientnet_el': None, + + 'efficientnet_cc_b0_4e': None, + 'efficientnet_cc_b0_8e': None, + 'efficientnet_cc_b1_8e': None, + + 'efficientnet_lite0': 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/efficientnet_lite0_ra-37913777.pth', + 'efficientnet_lite1': None, + 'efficientnet_lite2': None, + 'efficientnet_lite3': None, + 'efficientnet_lite4': None, + + 'tf_efficientnet_b0': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b0_aa-827b6e33.pth', + 'tf_efficientnet_b1': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b1_aa-ea7a6ee0.pth', + 'tf_efficientnet_b2': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b2_aa-60c94f97.pth', + 'tf_efficientnet_b3': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b3_aa-84b4657e.pth', + 'tf_efficientnet_b4': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b4_aa-818f208c.pth', + 'tf_efficientnet_b5': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b5_ra-9a3e5369.pth', + 'tf_efficientnet_b6': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b6_aa-80ba17e4.pth', + 'tf_efficientnet_b7': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b7_ra-6c08e654.pth', + 'tf_efficientnet_b8': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b8_ra-572d5dd9.pth', + + 'tf_efficientnet_b0_ap': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b0_ap-f262efe1.pth', + 'tf_efficientnet_b1_ap': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b1_ap-44ef0a3d.pth', + 'tf_efficientnet_b2_ap': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b2_ap-2f8e7636.pth', + 'tf_efficientnet_b3_ap': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b3_ap-aad25bdd.pth', + 'tf_efficientnet_b4_ap': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b4_ap-dedb23e6.pth', + 'tf_efficientnet_b5_ap': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b5_ap-9e82fae8.pth', + 'tf_efficientnet_b6_ap': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b6_ap-4ffb161f.pth', + 'tf_efficientnet_b7_ap': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b7_ap-ddb28fec.pth', + 'tf_efficientnet_b8_ap': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b8_ap-00e169fa.pth', + + 'tf_efficientnet_b0_ns': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b0_ns-c0e6a31c.pth', + 'tf_efficientnet_b1_ns': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b1_ns-99dd0c41.pth', + 'tf_efficientnet_b2_ns': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b2_ns-00306e48.pth', + 'tf_efficientnet_b3_ns': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b3_ns-9d44bf68.pth', + 'tf_efficientnet_b4_ns': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b4_ns-d6313a46.pth', + 'tf_efficientnet_b5_ns': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b5_ns-6f26d0cf.pth', + 'tf_efficientnet_b6_ns': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b6_ns-51548356.pth', + 'tf_efficientnet_b7_ns': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_b7_ns-1dbc32de.pth', + 'tf_efficientnet_l2_ns_475': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_l2_ns_475-bebbd00a.pth', + 'tf_efficientnet_l2_ns': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_l2_ns-df73bb44.pth', + + 'tf_efficientnet_es': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_es-ca1afbfe.pth', + 'tf_efficientnet_em': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_em-e78cfe58.pth', + 'tf_efficientnet_el': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_el-5143854e.pth', + + 'tf_efficientnet_cc_b0_4e': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_cc_b0_4e-4362b6b2.pth', + 'tf_efficientnet_cc_b0_8e': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_cc_b0_8e-66184a25.pth', + 'tf_efficientnet_cc_b1_8e': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_cc_b1_8e-f7c79ae1.pth', + + 'tf_efficientnet_lite0': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_lite0-0aa007d2.pth', + 'tf_efficientnet_lite1': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_lite1-bde8b488.pth', + 'tf_efficientnet_lite2': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_lite2-dcccb7df.pth', + 'tf_efficientnet_lite3': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_lite3-b733e338.pth', + 'tf_efficientnet_lite4': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_efficientnet_lite4-741542c3.pth', + + 'mixnet_s': 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/mixnet_s-a907afbc.pth', + 'mixnet_m': 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/mixnet_m-4647fc68.pth', + 'mixnet_l': 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/mixnet_l-5a9a2ed8.pth', + 'mixnet_xl': 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/mixnet_xl_ra-aac3c00c.pth', + + 'tf_mixnet_s': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_mixnet_s-89d3354b.pth', + 'tf_mixnet_m': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_mixnet_m-0f4d8805.pth', + 'tf_mixnet_l': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_mixnet_l-6c92e0c8.pth', +} + + +class GenEfficientNet(nn.Module): + """ Generic EfficientNets + + An implementation of mobile optimized networks that covers: + * EfficientNet (B0-B8, L2, CondConv, EdgeTPU) + * MixNet (Small, Medium, and Large, XL) + * MNASNet A1, B1, and small + * FBNet C + * Single-Path NAS Pixel1 + """ + + def __init__(self, block_args, num_classes=1000, in_chans=3, num_features=1280, stem_size=32, fix_stem=False, + channel_multiplier=1.0, channel_divisor=8, channel_min=None, + pad_type='', act_layer=nn.ReLU, drop_rate=0., drop_connect_rate=0., + se_kwargs=None, norm_layer=nn.BatchNorm2d, norm_kwargs=None, + weight_init='goog'): + super(GenEfficientNet, self).__init__() + self.drop_rate = drop_rate + + if not fix_stem: + stem_size = round_channels(stem_size, channel_multiplier, channel_divisor, channel_min) + self.conv_stem = select_conv2d(in_chans, stem_size, 3, stride=2, padding=pad_type) + self.bn1 = norm_layer(stem_size, **norm_kwargs) + self.act1 = act_layer(inplace=True) + in_chs = stem_size + + builder = EfficientNetBuilder( + channel_multiplier, channel_divisor, channel_min, + pad_type, act_layer, se_kwargs, norm_layer, norm_kwargs, drop_connect_rate) + self.blocks = nn.Sequential(*builder(in_chs, block_args)) + in_chs = builder.in_chs + + self.conv_head = select_conv2d(in_chs, num_features, 1, padding=pad_type) + self.bn2 = norm_layer(num_features, **norm_kwargs) + self.act2 = act_layer(inplace=True) + self.global_pool = nn.AdaptiveAvgPool2d(1) + self.classifier = nn.Linear(num_features, num_classes) + + for n, m in self.named_modules(): + if weight_init == 'goog': + initialize_weight_goog(m, n) + else: + initialize_weight_default(m, n) + + def features(self, x): + x = self.conv_stem(x) + x = self.bn1(x) + x = self.act1(x) + x = self.blocks(x) + x = self.conv_head(x) + x = self.bn2(x) + x = self.act2(x) + return x + + def as_sequential(self): + layers = [self.conv_stem, self.bn1, self.act1] + layers.extend(self.blocks) + layers.extend([ + self.conv_head, self.bn2, self.act2, + self.global_pool, nn.Flatten(), nn.Dropout(self.drop_rate), self.classifier]) + return nn.Sequential(*layers) + + def forward(self, x): + x = self.features(x) + x = self.global_pool(x) + x = x.flatten(1) + if self.drop_rate > 0.: + x = F.dropout(x, p=self.drop_rate, training=self.training) + return self.classifier(x) + + +def _create_model(model_kwargs, variant, pretrained=False): + as_sequential = model_kwargs.pop('as_sequential', False) + model = GenEfficientNet(**model_kwargs) + if pretrained: + load_pretrained(model, model_urls[variant]) + if as_sequential: + model = model.as_sequential() + return model + + +def _gen_mnasnet_a1(variant, channel_multiplier=1.0, pretrained=False, **kwargs): + """Creates a mnasnet-a1 model. + + Ref impl: https://github.com/tensorflow/tpu/tree/master/models/official/mnasnet + Paper: https://arxiv.org/pdf/1807.11626.pdf. + + Args: + channel_multiplier: multiplier to number of channels per layer. + """ + arch_def = [ + # stage 0, 112x112 in + ['ds_r1_k3_s1_e1_c16_noskip'], + # stage 1, 112x112 in + ['ir_r2_k3_s2_e6_c24'], + # stage 2, 56x56 in + ['ir_r3_k5_s2_e3_c40_se0.25'], + # stage 3, 28x28 in + ['ir_r4_k3_s2_e6_c80'], + # stage 4, 14x14in + ['ir_r2_k3_s1_e6_c112_se0.25'], + # stage 5, 14x14in + ['ir_r3_k5_s2_e6_c160_se0.25'], + # stage 6, 7x7 in + ['ir_r1_k3_s1_e6_c320'], + ] + with layer_config_kwargs(kwargs): + model_kwargs = dict( + block_args=decode_arch_def(arch_def), + stem_size=32, + channel_multiplier=channel_multiplier, + act_layer=resolve_act_layer(kwargs, 'relu'), + norm_kwargs=resolve_bn_args(kwargs), + **kwargs + ) + model = _create_model(model_kwargs, variant, pretrained) + return model + + +def _gen_mnasnet_b1(variant, channel_multiplier=1.0, pretrained=False, **kwargs): + """Creates a mnasnet-b1 model. + + Ref impl: https://github.com/tensorflow/tpu/tree/master/models/official/mnasnet + Paper: https://arxiv.org/pdf/1807.11626.pdf. + + Args: + channel_multiplier: multiplier to number of channels per layer. + """ + arch_def = [ + # stage 0, 112x112 in + ['ds_r1_k3_s1_c16_noskip'], + # stage 1, 112x112 in + ['ir_r3_k3_s2_e3_c24'], + # stage 2, 56x56 in + ['ir_r3_k5_s2_e3_c40'], + # stage 3, 28x28 in + ['ir_r3_k5_s2_e6_c80'], + # stage 4, 14x14in + ['ir_r2_k3_s1_e6_c96'], + # stage 5, 14x14in + ['ir_r4_k5_s2_e6_c192'], + # stage 6, 7x7 in + ['ir_r1_k3_s1_e6_c320_noskip'] + ] + with layer_config_kwargs(kwargs): + model_kwargs = dict( + block_args=decode_arch_def(arch_def), + stem_size=32, + channel_multiplier=channel_multiplier, + act_layer=resolve_act_layer(kwargs, 'relu'), + norm_kwargs=resolve_bn_args(kwargs), + **kwargs + ) + model = _create_model(model_kwargs, variant, pretrained) + return model + + +def _gen_mnasnet_small(variant, channel_multiplier=1.0, pretrained=False, **kwargs): + """Creates a mnasnet-b1 model. + + Ref impl: https://github.com/tensorflow/tpu/tree/master/models/official/mnasnet + Paper: https://arxiv.org/pdf/1807.11626.pdf. + + Args: + channel_multiplier: multiplier to number of channels per layer. + """ + arch_def = [ + ['ds_r1_k3_s1_c8'], + ['ir_r1_k3_s2_e3_c16'], + ['ir_r2_k3_s2_e6_c16'], + ['ir_r4_k5_s2_e6_c32_se0.25'], + ['ir_r3_k3_s1_e6_c32_se0.25'], + ['ir_r3_k5_s2_e6_c88_se0.25'], + ['ir_r1_k3_s1_e6_c144'] + ] + with layer_config_kwargs(kwargs): + model_kwargs = dict( + block_args=decode_arch_def(arch_def), + stem_size=8, + channel_multiplier=channel_multiplier, + act_layer=resolve_act_layer(kwargs, 'relu'), + norm_kwargs=resolve_bn_args(kwargs), + **kwargs + ) + model = _create_model(model_kwargs, variant, pretrained) + return model + + +def _gen_mobilenet_v2( + variant, channel_multiplier=1.0, depth_multiplier=1.0, fix_stem_head=False, pretrained=False, **kwargs): + """ Generate MobileNet-V2 network + Ref impl: https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet_v2.py + Paper: https://arxiv.org/abs/1801.04381 + """ + arch_def = [ + ['ds_r1_k3_s1_c16'], + ['ir_r2_k3_s2_e6_c24'], + ['ir_r3_k3_s2_e6_c32'], + ['ir_r4_k3_s2_e6_c64'], + ['ir_r3_k3_s1_e6_c96'], + ['ir_r3_k3_s2_e6_c160'], + ['ir_r1_k3_s1_e6_c320'], + ] + with layer_config_kwargs(kwargs): + model_kwargs = dict( + block_args=decode_arch_def(arch_def, depth_multiplier=depth_multiplier, fix_first_last=fix_stem_head), + num_features=1280 if fix_stem_head else round_channels(1280, channel_multiplier, 8, None), + stem_size=32, + fix_stem=fix_stem_head, + channel_multiplier=channel_multiplier, + norm_kwargs=resolve_bn_args(kwargs), + act_layer=nn.ReLU6, + **kwargs + ) + model = _create_model(model_kwargs, variant, pretrained) + return model + + +def _gen_fbnetc(variant, channel_multiplier=1.0, pretrained=False, **kwargs): + """ FBNet-C + + Paper: https://arxiv.org/abs/1812.03443 + Ref Impl: https://github.com/facebookresearch/maskrcnn-benchmark/blob/master/maskrcnn_benchmark/modeling/backbone/fbnet_modeldef.py + + NOTE: the impl above does not relate to the 'C' variant here, that was derived from paper, + it was used to confirm some building block details + """ + arch_def = [ + ['ir_r1_k3_s1_e1_c16'], + ['ir_r1_k3_s2_e6_c24', 'ir_r2_k3_s1_e1_c24'], + ['ir_r1_k5_s2_e6_c32', 'ir_r1_k5_s1_e3_c32', 'ir_r1_k5_s1_e6_c32', 'ir_r1_k3_s1_e6_c32'], + ['ir_r1_k5_s2_e6_c64', 'ir_r1_k5_s1_e3_c64', 'ir_r2_k5_s1_e6_c64'], + ['ir_r3_k5_s1_e6_c112', 'ir_r1_k5_s1_e3_c112'], + ['ir_r4_k5_s2_e6_c184'], + ['ir_r1_k3_s1_e6_c352'], + ] + with layer_config_kwargs(kwargs): + model_kwargs = dict( + block_args=decode_arch_def(arch_def), + stem_size=16, + num_features=1984, # paper suggests this, but is not 100% clear + channel_multiplier=channel_multiplier, + act_layer=resolve_act_layer(kwargs, 'relu'), + norm_kwargs=resolve_bn_args(kwargs), + **kwargs + ) + model = _create_model(model_kwargs, variant, pretrained) + return model + + +def _gen_spnasnet(variant, channel_multiplier=1.0, pretrained=False, **kwargs): + """Creates the Single-Path NAS model from search targeted for Pixel1 phone. + + Paper: https://arxiv.org/abs/1904.02877 + + Args: + channel_multiplier: multiplier to number of channels per layer. + """ + arch_def = [ + # stage 0, 112x112 in + ['ds_r1_k3_s1_c16_noskip'], + # stage 1, 112x112 in + ['ir_r3_k3_s2_e3_c24'], + # stage 2, 56x56 in + ['ir_r1_k5_s2_e6_c40', 'ir_r3_k3_s1_e3_c40'], + # stage 3, 28x28 in + ['ir_r1_k5_s2_e6_c80', 'ir_r3_k3_s1_e3_c80'], + # stage 4, 14x14in + ['ir_r1_k5_s1_e6_c96', 'ir_r3_k5_s1_e3_c96'], + # stage 5, 14x14in + ['ir_r4_k5_s2_e6_c192'], + # stage 6, 7x7 in + ['ir_r1_k3_s1_e6_c320_noskip'] + ] + with layer_config_kwargs(kwargs): + model_kwargs = dict( + block_args=decode_arch_def(arch_def), + stem_size=32, + channel_multiplier=channel_multiplier, + act_layer=resolve_act_layer(kwargs, 'relu'), + norm_kwargs=resolve_bn_args(kwargs), + **kwargs + ) + model = _create_model(model_kwargs, variant, pretrained) + return model + + +def _gen_efficientnet(variant, channel_multiplier=1.0, depth_multiplier=1.0, pretrained=False, **kwargs): + """Creates an EfficientNet model. + + Ref impl: https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/efficientnet_model.py + Paper: https://arxiv.org/abs/1905.11946 + + EfficientNet params + name: (channel_multiplier, depth_multiplier, resolution, dropout_rate) + 'efficientnet-b0': (1.0, 1.0, 224, 0.2), + 'efficientnet-b1': (1.0, 1.1, 240, 0.2), + 'efficientnet-b2': (1.1, 1.2, 260, 0.3), + 'efficientnet-b3': (1.2, 1.4, 300, 0.3), + 'efficientnet-b4': (1.4, 1.8, 380, 0.4), + 'efficientnet-b5': (1.6, 2.2, 456, 0.4), + 'efficientnet-b6': (1.8, 2.6, 528, 0.5), + 'efficientnet-b7': (2.0, 3.1, 600, 0.5), + 'efficientnet-b8': (2.2, 3.6, 672, 0.5), + + Args: + channel_multiplier: multiplier to number of channels per layer + depth_multiplier: multiplier to number of repeats per stage + + """ + arch_def = [ + ['ds_r1_k3_s1_e1_c16_se0.25'], + ['ir_r2_k3_s2_e6_c24_se0.25'], + ['ir_r2_k5_s2_e6_c40_se0.25'], + ['ir_r3_k3_s2_e6_c80_se0.25'], + ['ir_r3_k5_s1_e6_c112_se0.25'], + ['ir_r4_k5_s2_e6_c192_se0.25'], + ['ir_r1_k3_s1_e6_c320_se0.25'], + ] + with layer_config_kwargs(kwargs): + model_kwargs = dict( + block_args=decode_arch_def(arch_def, depth_multiplier), + num_features=round_channels(1280, channel_multiplier, 8, None), + stem_size=32, + channel_multiplier=channel_multiplier, + act_layer=resolve_act_layer(kwargs, 'swish'), + norm_kwargs=resolve_bn_args(kwargs), + **kwargs, + ) + model = _create_model(model_kwargs, variant, pretrained) + return model + + +def _gen_efficientnet_edge(variant, channel_multiplier=1.0, depth_multiplier=1.0, pretrained=False, **kwargs): + arch_def = [ + # NOTE `fc` is present to override a mismatch between stem channels and in chs not + # present in other models + ['er_r1_k3_s1_e4_c24_fc24_noskip'], + ['er_r2_k3_s2_e8_c32'], + ['er_r4_k3_s2_e8_c48'], + ['ir_r5_k5_s2_e8_c96'], + ['ir_r4_k5_s1_e8_c144'], + ['ir_r2_k5_s2_e8_c192'], + ] + with layer_config_kwargs(kwargs): + model_kwargs = dict( + block_args=decode_arch_def(arch_def, depth_multiplier), + num_features=round_channels(1280, channel_multiplier, 8, None), + stem_size=32, + channel_multiplier=channel_multiplier, + act_layer=resolve_act_layer(kwargs, 'relu'), + norm_kwargs=resolve_bn_args(kwargs), + **kwargs, + ) + model = _create_model(model_kwargs, variant, pretrained) + return model + + +def _gen_efficientnet_condconv( + variant, channel_multiplier=1.0, depth_multiplier=1.0, experts_multiplier=1, pretrained=False, **kwargs): + """Creates an efficientnet-condconv model.""" + arch_def = [ + ['ds_r1_k3_s1_e1_c16_se0.25'], + ['ir_r2_k3_s2_e6_c24_se0.25'], + ['ir_r2_k5_s2_e6_c40_se0.25'], + ['ir_r3_k3_s2_e6_c80_se0.25'], + ['ir_r3_k5_s1_e6_c112_se0.25_cc4'], + ['ir_r4_k5_s2_e6_c192_se0.25_cc4'], + ['ir_r1_k3_s1_e6_c320_se0.25_cc4'], + ] + with layer_config_kwargs(kwargs): + model_kwargs = dict( + block_args=decode_arch_def(arch_def, depth_multiplier, experts_multiplier=experts_multiplier), + num_features=round_channels(1280, channel_multiplier, 8, None), + stem_size=32, + channel_multiplier=channel_multiplier, + act_layer=resolve_act_layer(kwargs, 'swish'), + norm_kwargs=resolve_bn_args(kwargs), + **kwargs, + ) + model = _create_model(model_kwargs, variant, pretrained) + return model + + +def _gen_efficientnet_lite(variant, channel_multiplier=1.0, depth_multiplier=1.0, pretrained=False, **kwargs): + """Creates an EfficientNet-Lite model. + + Ref impl: https://github.com/tensorflow/tpu/tree/master/models/official/efficientnet/lite + Paper: https://arxiv.org/abs/1905.11946 + + EfficientNet params + name: (channel_multiplier, depth_multiplier, resolution, dropout_rate) + 'efficientnet-lite0': (1.0, 1.0, 224, 0.2), + 'efficientnet-lite1': (1.0, 1.1, 240, 0.2), + 'efficientnet-lite2': (1.1, 1.2, 260, 0.3), + 'efficientnet-lite3': (1.2, 1.4, 280, 0.3), + 'efficientnet-lite4': (1.4, 1.8, 300, 0.3), + + Args: + channel_multiplier: multiplier to number of channels per layer + depth_multiplier: multiplier to number of repeats per stage + """ + arch_def = [ + ['ds_r1_k3_s1_e1_c16'], + ['ir_r2_k3_s2_e6_c24'], + ['ir_r2_k5_s2_e6_c40'], + ['ir_r3_k3_s2_e6_c80'], + ['ir_r3_k5_s1_e6_c112'], + ['ir_r4_k5_s2_e6_c192'], + ['ir_r1_k3_s1_e6_c320'], + ] + with layer_config_kwargs(kwargs): + model_kwargs = dict( + block_args=decode_arch_def(arch_def, depth_multiplier, fix_first_last=True), + num_features=1280, + stem_size=32, + fix_stem=True, + channel_multiplier=channel_multiplier, + act_layer=nn.ReLU6, + norm_kwargs=resolve_bn_args(kwargs), + **kwargs, + ) + model = _create_model(model_kwargs, variant, pretrained) + return model + + +def _gen_mixnet_s(variant, channel_multiplier=1.0, pretrained=False, **kwargs): + """Creates a MixNet Small model. + + Ref impl: https://github.com/tensorflow/tpu/tree/master/models/official/mnasnet/mixnet + Paper: https://arxiv.org/abs/1907.09595 + """ + arch_def = [ + # stage 0, 112x112 in + ['ds_r1_k3_s1_e1_c16'], # relu + # stage 1, 112x112 in + ['ir_r1_k3_a1.1_p1.1_s2_e6_c24', 'ir_r1_k3_a1.1_p1.1_s1_e3_c24'], # relu + # stage 2, 56x56 in + ['ir_r1_k3.5.7_s2_e6_c40_se0.5_nsw', 'ir_r3_k3.5_a1.1_p1.1_s1_e6_c40_se0.5_nsw'], # swish + # stage 3, 28x28 in + ['ir_r1_k3.5.7_p1.1_s2_e6_c80_se0.25_nsw', 'ir_r2_k3.5_p1.1_s1_e6_c80_se0.25_nsw'], # swish + # stage 4, 14x14in + ['ir_r1_k3.5.7_a1.1_p1.1_s1_e6_c120_se0.5_nsw', 'ir_r2_k3.5.7.9_a1.1_p1.1_s1_e3_c120_se0.5_nsw'], # swish + # stage 5, 14x14in + ['ir_r1_k3.5.7.9.11_s2_e6_c200_se0.5_nsw', 'ir_r2_k3.5.7.9_p1.1_s1_e6_c200_se0.5_nsw'], # swish + # 7x7 + ] + with layer_config_kwargs(kwargs): + model_kwargs = dict( + block_args=decode_arch_def(arch_def), + num_features=1536, + stem_size=16, + channel_multiplier=channel_multiplier, + act_layer=resolve_act_layer(kwargs, 'relu'), + norm_kwargs=resolve_bn_args(kwargs), + **kwargs + ) + model = _create_model(model_kwargs, variant, pretrained) + return model + + +def _gen_mixnet_m(variant, channel_multiplier=1.0, depth_multiplier=1.0, pretrained=False, **kwargs): + """Creates a MixNet Medium-Large model. + + Ref impl: https://github.com/tensorflow/tpu/tree/master/models/official/mnasnet/mixnet + Paper: https://arxiv.org/abs/1907.09595 + """ + arch_def = [ + # stage 0, 112x112 in + ['ds_r1_k3_s1_e1_c24'], # relu + # stage 1, 112x112 in + ['ir_r1_k3.5.7_a1.1_p1.1_s2_e6_c32', 'ir_r1_k3_a1.1_p1.1_s1_e3_c32'], # relu + # stage 2, 56x56 in + ['ir_r1_k3.5.7.9_s2_e6_c40_se0.5_nsw', 'ir_r3_k3.5_a1.1_p1.1_s1_e6_c40_se0.5_nsw'], # swish + # stage 3, 28x28 in + ['ir_r1_k3.5.7_s2_e6_c80_se0.25_nsw', 'ir_r3_k3.5.7.9_a1.1_p1.1_s1_e6_c80_se0.25_nsw'], # swish + # stage 4, 14x14in + ['ir_r1_k3_s1_e6_c120_se0.5_nsw', 'ir_r3_k3.5.7.9_a1.1_p1.1_s1_e3_c120_se0.5_nsw'], # swish + # stage 5, 14x14in + ['ir_r1_k3.5.7.9_s2_e6_c200_se0.5_nsw', 'ir_r3_k3.5.7.9_p1.1_s1_e6_c200_se0.5_nsw'], # swish + # 7x7 + ] + with layer_config_kwargs(kwargs): + model_kwargs = dict( + block_args=decode_arch_def(arch_def, depth_multiplier, depth_trunc='round'), + num_features=1536, + stem_size=24, + channel_multiplier=channel_multiplier, + act_layer=resolve_act_layer(kwargs, 'relu'), + norm_kwargs=resolve_bn_args(kwargs), + **kwargs + ) + model = _create_model(model_kwargs, variant, pretrained) + return model + + +def mnasnet_050(pretrained=False, **kwargs): + """ MNASNet B1, depth multiplier of 0.5. """ + model = _gen_mnasnet_b1('mnasnet_050', 0.5, pretrained=pretrained, **kwargs) + return model + + +def mnasnet_075(pretrained=False, **kwargs): + """ MNASNet B1, depth multiplier of 0.75. """ + model = _gen_mnasnet_b1('mnasnet_075', 0.75, pretrained=pretrained, **kwargs) + return model + + +def mnasnet_100(pretrained=False, **kwargs): + """ MNASNet B1, depth multiplier of 1.0. """ + model = _gen_mnasnet_b1('mnasnet_100', 1.0, pretrained=pretrained, **kwargs) + return model + + +def mnasnet_b1(pretrained=False, **kwargs): + """ MNASNet B1, depth multiplier of 1.0. """ + return mnasnet_100(pretrained, **kwargs) + + +def mnasnet_140(pretrained=False, **kwargs): + """ MNASNet B1, depth multiplier of 1.4 """ + model = _gen_mnasnet_b1('mnasnet_140', 1.4, pretrained=pretrained, **kwargs) + return model + + +def semnasnet_050(pretrained=False, **kwargs): + """ MNASNet A1 (w/ SE), depth multiplier of 0.5 """ + model = _gen_mnasnet_a1('semnasnet_050', 0.5, pretrained=pretrained, **kwargs) + return model + + +def semnasnet_075(pretrained=False, **kwargs): + """ MNASNet A1 (w/ SE), depth multiplier of 0.75. """ + model = _gen_mnasnet_a1('semnasnet_075', 0.75, pretrained=pretrained, **kwargs) + return model + + +def semnasnet_100(pretrained=False, **kwargs): + """ MNASNet A1 (w/ SE), depth multiplier of 1.0. """ + model = _gen_mnasnet_a1('semnasnet_100', 1.0, pretrained=pretrained, **kwargs) + return model + + +def mnasnet_a1(pretrained=False, **kwargs): + """ MNASNet A1 (w/ SE), depth multiplier of 1.0. """ + return semnasnet_100(pretrained, **kwargs) + + +def semnasnet_140(pretrained=False, **kwargs): + """ MNASNet A1 (w/ SE), depth multiplier of 1.4. """ + model = _gen_mnasnet_a1('semnasnet_140', 1.4, pretrained=pretrained, **kwargs) + return model + + +def mnasnet_small(pretrained=False, **kwargs): + """ MNASNet Small, depth multiplier of 1.0. """ + model = _gen_mnasnet_small('mnasnet_small', 1.0, pretrained=pretrained, **kwargs) + return model + + +def mobilenetv2_100(pretrained=False, **kwargs): + """ MobileNet V2 w/ 1.0 channel multiplier """ + model = _gen_mobilenet_v2('mobilenetv2_100', 1.0, pretrained=pretrained, **kwargs) + return model + + +def mobilenetv2_140(pretrained=False, **kwargs): + """ MobileNet V2 w/ 1.4 channel multiplier """ + model = _gen_mobilenet_v2('mobilenetv2_140', 1.4, pretrained=pretrained, **kwargs) + return model + + +def mobilenetv2_110d(pretrained=False, **kwargs): + """ MobileNet V2 w/ 1.1 channel, 1.2 depth multipliers""" + model = _gen_mobilenet_v2( + 'mobilenetv2_110d', 1.1, depth_multiplier=1.2, fix_stem_head=True, pretrained=pretrained, **kwargs) + return model + + +def mobilenetv2_120d(pretrained=False, **kwargs): + """ MobileNet V2 w/ 1.2 channel, 1.4 depth multipliers """ + model = _gen_mobilenet_v2( + 'mobilenetv2_120d', 1.2, depth_multiplier=1.4, fix_stem_head=True, pretrained=pretrained, **kwargs) + return model + + +def fbnetc_100(pretrained=False, **kwargs): + """ FBNet-C """ + if pretrained: + # pretrained model trained with non-default BN epsilon + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + model = _gen_fbnetc('fbnetc_100', 1.0, pretrained=pretrained, **kwargs) + return model + + +def spnasnet_100(pretrained=False, **kwargs): + """ Single-Path NAS Pixel1""" + model = _gen_spnasnet('spnasnet_100', 1.0, pretrained=pretrained, **kwargs) + return model + + +def efficientnet_b0(pretrained=False, **kwargs): + """ EfficientNet-B0 """ + # NOTE for train set drop_rate=0.2, drop_connect_rate=0.2 + model = _gen_efficientnet( + 'efficientnet_b0', channel_multiplier=1.0, depth_multiplier=1.0, pretrained=pretrained, **kwargs) + return model + + +def efficientnet_b1(pretrained=False, **kwargs): + """ EfficientNet-B1 """ + # NOTE for train set drop_rate=0.2, drop_connect_rate=0.2 + model = _gen_efficientnet( + 'efficientnet_b1', channel_multiplier=1.0, depth_multiplier=1.1, pretrained=pretrained, **kwargs) + return model + + +def efficientnet_b2(pretrained=False, **kwargs): + """ EfficientNet-B2 """ + # NOTE for train set drop_rate=0.3, drop_connect_rate=0.2 + model = _gen_efficientnet( + 'efficientnet_b2', channel_multiplier=1.1, depth_multiplier=1.2, pretrained=pretrained, **kwargs) + return model + + +def efficientnet_b3(pretrained=False, **kwargs): + """ EfficientNet-B3 """ + # NOTE for train set drop_rate=0.3, drop_connect_rate=0.2 + model = _gen_efficientnet( + 'efficientnet_b3', channel_multiplier=1.2, depth_multiplier=1.4, pretrained=pretrained, **kwargs) + return model + + +def efficientnet_b4(pretrained=False, **kwargs): + """ EfficientNet-B4 """ + # NOTE for train set drop_rate=0.4, drop_connect_rate=0.2 + model = _gen_efficientnet( + 'efficientnet_b4', channel_multiplier=1.4, depth_multiplier=1.8, pretrained=pretrained, **kwargs) + return model + + +def efficientnet_b5(pretrained=False, **kwargs): + """ EfficientNet-B5 """ + # NOTE for train set drop_rate=0.4, drop_connect_rate=0.2 + model = _gen_efficientnet( + 'efficientnet_b5', channel_multiplier=1.6, depth_multiplier=2.2, pretrained=pretrained, **kwargs) + return model + + +def efficientnet_b6(pretrained=False, **kwargs): + """ EfficientNet-B6 """ + # NOTE for train set drop_rate=0.5, drop_connect_rate=0.2 + model = _gen_efficientnet( + 'efficientnet_b6', channel_multiplier=1.8, depth_multiplier=2.6, pretrained=pretrained, **kwargs) + return model + + +def efficientnet_b7(pretrained=False, **kwargs): + """ EfficientNet-B7 """ + # NOTE for train set drop_rate=0.5, drop_connect_rate=0.2 + model = _gen_efficientnet( + 'efficientnet_b7', channel_multiplier=2.0, depth_multiplier=3.1, pretrained=pretrained, **kwargs) + return model + + +def efficientnet_b8(pretrained=False, **kwargs): + """ EfficientNet-B8 """ + # NOTE for train set drop_rate=0.5, drop_connect_rate=0.2 + model = _gen_efficientnet( + 'efficientnet_b8', channel_multiplier=2.2, depth_multiplier=3.6, pretrained=pretrained, **kwargs) + return model + + +def efficientnet_l2(pretrained=False, **kwargs): + """ EfficientNet-L2. """ + # NOTE for train, drop_rate should be 0.5 + model = _gen_efficientnet( + 'efficientnet_l2', channel_multiplier=4.3, depth_multiplier=5.3, pretrained=pretrained, **kwargs) + return model + + +def efficientnet_es(pretrained=False, **kwargs): + """ EfficientNet-Edge Small. """ + model = _gen_efficientnet_edge( + 'efficientnet_es', channel_multiplier=1.0, depth_multiplier=1.0, pretrained=pretrained, **kwargs) + return model + + +def efficientnet_em(pretrained=False, **kwargs): + """ EfficientNet-Edge-Medium. """ + model = _gen_efficientnet_edge( + 'efficientnet_em', channel_multiplier=1.0, depth_multiplier=1.1, pretrained=pretrained, **kwargs) + return model + + +def efficientnet_el(pretrained=False, **kwargs): + """ EfficientNet-Edge-Large. """ + model = _gen_efficientnet_edge( + 'efficientnet_el', channel_multiplier=1.2, depth_multiplier=1.4, pretrained=pretrained, **kwargs) + return model + + +def efficientnet_cc_b0_4e(pretrained=False, **kwargs): + """ EfficientNet-CondConv-B0 w/ 8 Experts """ + # NOTE for train set drop_rate=0.25, drop_connect_rate=0.2 + model = _gen_efficientnet_condconv( + 'efficientnet_cc_b0_4e', channel_multiplier=1.0, depth_multiplier=1.0, pretrained=pretrained, **kwargs) + return model + + +def efficientnet_cc_b0_8e(pretrained=False, **kwargs): + """ EfficientNet-CondConv-B0 w/ 8 Experts """ + # NOTE for train set drop_rate=0.25, drop_connect_rate=0.2 + model = _gen_efficientnet_condconv( + 'efficientnet_cc_b0_8e', channel_multiplier=1.0, depth_multiplier=1.0, experts_multiplier=2, + pretrained=pretrained, **kwargs) + return model + + +def efficientnet_cc_b1_8e(pretrained=False, **kwargs): + """ EfficientNet-CondConv-B1 w/ 8 Experts """ + # NOTE for train set drop_rate=0.25, drop_connect_rate=0.2 + model = _gen_efficientnet_condconv( + 'efficientnet_cc_b1_8e', channel_multiplier=1.0, depth_multiplier=1.1, experts_multiplier=2, + pretrained=pretrained, **kwargs) + return model + + +def efficientnet_lite0(pretrained=False, **kwargs): + """ EfficientNet-Lite0 """ + model = _gen_efficientnet_lite( + 'efficientnet_lite0', channel_multiplier=1.0, depth_multiplier=1.0, pretrained=pretrained, **kwargs) + return model + + +def efficientnet_lite1(pretrained=False, **kwargs): + """ EfficientNet-Lite1 """ + model = _gen_efficientnet_lite( + 'efficientnet_lite1', channel_multiplier=1.0, depth_multiplier=1.1, pretrained=pretrained, **kwargs) + return model + + +def efficientnet_lite2(pretrained=False, **kwargs): + """ EfficientNet-Lite2 """ + model = _gen_efficientnet_lite( + 'efficientnet_lite2', channel_multiplier=1.1, depth_multiplier=1.2, pretrained=pretrained, **kwargs) + return model + + +def efficientnet_lite3(pretrained=False, **kwargs): + """ EfficientNet-Lite3 """ + model = _gen_efficientnet_lite( + 'efficientnet_lite3', channel_multiplier=1.2, depth_multiplier=1.4, pretrained=pretrained, **kwargs) + return model + + +def efficientnet_lite4(pretrained=False, **kwargs): + """ EfficientNet-Lite4 """ + model = _gen_efficientnet_lite( + 'efficientnet_lite4', channel_multiplier=1.4, depth_multiplier=1.8, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b0(pretrained=False, **kwargs): + """ EfficientNet-B0 AutoAug. Tensorflow compatible variant """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b0', channel_multiplier=1.0, depth_multiplier=1.0, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b1(pretrained=False, **kwargs): + """ EfficientNet-B1 AutoAug. Tensorflow compatible variant """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b1', channel_multiplier=1.0, depth_multiplier=1.1, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b2(pretrained=False, **kwargs): + """ EfficientNet-B2 AutoAug. Tensorflow compatible variant """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b2', channel_multiplier=1.1, depth_multiplier=1.2, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b3(pretrained=False, **kwargs): + """ EfficientNet-B3 AutoAug. Tensorflow compatible variant """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b3', channel_multiplier=1.2, depth_multiplier=1.4, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b4(pretrained=False, **kwargs): + """ EfficientNet-B4 AutoAug. Tensorflow compatible variant """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b4', channel_multiplier=1.4, depth_multiplier=1.8, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b5(pretrained=False, **kwargs): + """ EfficientNet-B5 RandAug. Tensorflow compatible variant """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b5', channel_multiplier=1.6, depth_multiplier=2.2, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b6(pretrained=False, **kwargs): + """ EfficientNet-B6 AutoAug. Tensorflow compatible variant """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b6', channel_multiplier=1.8, depth_multiplier=2.6, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b7(pretrained=False, **kwargs): + """ EfficientNet-B7 RandAug. Tensorflow compatible variant """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b7', channel_multiplier=2.0, depth_multiplier=3.1, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b8(pretrained=False, **kwargs): + """ EfficientNet-B8 RandAug. Tensorflow compatible variant """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b8', channel_multiplier=2.2, depth_multiplier=3.6, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b0_ap(pretrained=False, **kwargs): + """ EfficientNet-B0 AdvProp. Tensorflow compatible variant + Paper: Adversarial Examples Improve Image Recognition (https://arxiv.org/abs/1911.09665) + """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b0_ap', channel_multiplier=1.0, depth_multiplier=1.0, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b1_ap(pretrained=False, **kwargs): + """ EfficientNet-B1 AdvProp. Tensorflow compatible variant + Paper: Adversarial Examples Improve Image Recognition (https://arxiv.org/abs/1911.09665) + """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b1_ap', channel_multiplier=1.0, depth_multiplier=1.1, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b2_ap(pretrained=False, **kwargs): + """ EfficientNet-B2 AdvProp. Tensorflow compatible variant + Paper: Adversarial Examples Improve Image Recognition (https://arxiv.org/abs/1911.09665) + """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b2_ap', channel_multiplier=1.1, depth_multiplier=1.2, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b3_ap(pretrained=False, **kwargs): + """ EfficientNet-B3 AdvProp. Tensorflow compatible variant + Paper: Adversarial Examples Improve Image Recognition (https://arxiv.org/abs/1911.09665) + """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b3_ap', channel_multiplier=1.2, depth_multiplier=1.4, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b4_ap(pretrained=False, **kwargs): + """ EfficientNet-B4 AdvProp. Tensorflow compatible variant + Paper: Adversarial Examples Improve Image Recognition (https://arxiv.org/abs/1911.09665) + """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b4_ap', channel_multiplier=1.4, depth_multiplier=1.8, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b5_ap(pretrained=False, **kwargs): + """ EfficientNet-B5 AdvProp. Tensorflow compatible variant + Paper: Adversarial Examples Improve Image Recognition (https://arxiv.org/abs/1911.09665) + """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b5_ap', channel_multiplier=1.6, depth_multiplier=2.2, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b6_ap(pretrained=False, **kwargs): + """ EfficientNet-B6 AdvProp. Tensorflow compatible variant + Paper: Adversarial Examples Improve Image Recognition (https://arxiv.org/abs/1911.09665) + """ + # NOTE for train, drop_rate should be 0.5 + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b6_ap', channel_multiplier=1.8, depth_multiplier=2.6, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b7_ap(pretrained=False, **kwargs): + """ EfficientNet-B7 AdvProp. Tensorflow compatible variant + Paper: Adversarial Examples Improve Image Recognition (https://arxiv.org/abs/1911.09665) + """ + # NOTE for train, drop_rate should be 0.5 + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b7_ap', channel_multiplier=2.0, depth_multiplier=3.1, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b8_ap(pretrained=False, **kwargs): + """ EfficientNet-B8 AdvProp. Tensorflow compatible variant + Paper: Adversarial Examples Improve Image Recognition (https://arxiv.org/abs/1911.09665) + """ + # NOTE for train, drop_rate should be 0.5 + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b8_ap', channel_multiplier=2.2, depth_multiplier=3.6, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b0_ns(pretrained=False, **kwargs): + """ EfficientNet-B0 NoisyStudent. Tensorflow compatible variant + Paper: Self-training with Noisy Student improves ImageNet classification (https://arxiv.org/abs/1911.04252) + """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b0_ns', channel_multiplier=1.0, depth_multiplier=1.0, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b1_ns(pretrained=False, **kwargs): + """ EfficientNet-B1 NoisyStudent. Tensorflow compatible variant + Paper: Self-training with Noisy Student improves ImageNet classification (https://arxiv.org/abs/1911.04252) + """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b1_ns', channel_multiplier=1.0, depth_multiplier=1.1, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b2_ns(pretrained=False, **kwargs): + """ EfficientNet-B2 NoisyStudent. Tensorflow compatible variant + Paper: Self-training with Noisy Student improves ImageNet classification (https://arxiv.org/abs/1911.04252) + """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b2_ns', channel_multiplier=1.1, depth_multiplier=1.2, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b3_ns(pretrained=False, **kwargs): + """ EfficientNet-B3 NoisyStudent. Tensorflow compatible variant + Paper: Self-training with Noisy Student improves ImageNet classification (https://arxiv.org/abs/1911.04252) + """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b3_ns', channel_multiplier=1.2, depth_multiplier=1.4, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b4_ns(pretrained=False, **kwargs): + """ EfficientNet-B4 NoisyStudent. Tensorflow compatible variant + Paper: Self-training with Noisy Student improves ImageNet classification (https://arxiv.org/abs/1911.04252) + """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b4_ns', channel_multiplier=1.4, depth_multiplier=1.8, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b5_ns(pretrained=False, **kwargs): + """ EfficientNet-B5 NoisyStudent. Tensorflow compatible variant + Paper: Self-training with Noisy Student improves ImageNet classification (https://arxiv.org/abs/1911.04252) + """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b5_ns', channel_multiplier=1.6, depth_multiplier=2.2, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b6_ns(pretrained=False, **kwargs): + """ EfficientNet-B6 NoisyStudent. Tensorflow compatible variant + Paper: Self-training with Noisy Student improves ImageNet classification (https://arxiv.org/abs/1911.04252) + """ + # NOTE for train, drop_rate should be 0.5 + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b6_ns', channel_multiplier=1.8, depth_multiplier=2.6, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_b7_ns(pretrained=False, **kwargs): + """ EfficientNet-B7 NoisyStudent. Tensorflow compatible variant + Paper: Self-training with Noisy Student improves ImageNet classification (https://arxiv.org/abs/1911.04252) + """ + # NOTE for train, drop_rate should be 0.5 + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_b7_ns', channel_multiplier=2.0, depth_multiplier=3.1, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_l2_ns_475(pretrained=False, **kwargs): + """ EfficientNet-L2 NoisyStudent @ 475x475. Tensorflow compatible variant + Paper: Self-training with Noisy Student improves ImageNet classification (https://arxiv.org/abs/1911.04252) + """ + # NOTE for train, drop_rate should be 0.5 + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_l2_ns_475', channel_multiplier=4.3, depth_multiplier=5.3, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_l2_ns(pretrained=False, **kwargs): + """ EfficientNet-L2 NoisyStudent. Tensorflow compatible variant + Paper: Self-training with Noisy Student improves ImageNet classification (https://arxiv.org/abs/1911.04252) + """ + # NOTE for train, drop_rate should be 0.5 + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet( + 'tf_efficientnet_l2_ns', channel_multiplier=4.3, depth_multiplier=5.3, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_es(pretrained=False, **kwargs): + """ EfficientNet-Edge Small. Tensorflow compatible variant """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet_edge( + 'tf_efficientnet_es', channel_multiplier=1.0, depth_multiplier=1.0, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_em(pretrained=False, **kwargs): + """ EfficientNet-Edge-Medium. Tensorflow compatible variant """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet_edge( + 'tf_efficientnet_em', channel_multiplier=1.0, depth_multiplier=1.1, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_el(pretrained=False, **kwargs): + """ EfficientNet-Edge-Large. Tensorflow compatible variant """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet_edge( + 'tf_efficientnet_el', channel_multiplier=1.2, depth_multiplier=1.4, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_cc_b0_4e(pretrained=False, **kwargs): + """ EfficientNet-CondConv-B0 w/ 4 Experts """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet_condconv( + 'tf_efficientnet_cc_b0_4e', channel_multiplier=1.0, depth_multiplier=1.0, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_cc_b0_8e(pretrained=False, **kwargs): + """ EfficientNet-CondConv-B0 w/ 8 Experts """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet_condconv( + 'tf_efficientnet_cc_b0_8e', channel_multiplier=1.0, depth_multiplier=1.0, experts_multiplier=2, + pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_cc_b1_8e(pretrained=False, **kwargs): + """ EfficientNet-CondConv-B1 w/ 8 Experts """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet_condconv( + 'tf_efficientnet_cc_b1_8e', channel_multiplier=1.0, depth_multiplier=1.1, experts_multiplier=2, + pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_lite0(pretrained=False, **kwargs): + """ EfficientNet-Lite0. Tensorflow compatible variant """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet_lite( + 'tf_efficientnet_lite0', channel_multiplier=1.0, depth_multiplier=1.0, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_lite1(pretrained=False, **kwargs): + """ EfficientNet-Lite1. Tensorflow compatible variant """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet_lite( + 'tf_efficientnet_lite1', channel_multiplier=1.0, depth_multiplier=1.1, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_lite2(pretrained=False, **kwargs): + """ EfficientNet-Lite2. Tensorflow compatible variant """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet_lite( + 'tf_efficientnet_lite2', channel_multiplier=1.1, depth_multiplier=1.2, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_lite3(pretrained=False, **kwargs): + """ EfficientNet-Lite3. Tensorflow compatible variant """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet_lite( + 'tf_efficientnet_lite3', channel_multiplier=1.2, depth_multiplier=1.4, pretrained=pretrained, **kwargs) + return model + + +def tf_efficientnet_lite4(pretrained=False, **kwargs): + """ EfficientNet-Lite4. Tensorflow compatible variant """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_efficientnet_lite( + 'tf_efficientnet_lite4', channel_multiplier=1.4, depth_multiplier=1.8, pretrained=pretrained, **kwargs) + return model + + +def mixnet_s(pretrained=False, **kwargs): + """Creates a MixNet Small model. + """ + # NOTE for train set drop_rate=0.2 + model = _gen_mixnet_s( + 'mixnet_s', channel_multiplier=1.0, pretrained=pretrained, **kwargs) + return model + + +def mixnet_m(pretrained=False, **kwargs): + """Creates a MixNet Medium model. + """ + # NOTE for train set drop_rate=0.25 + model = _gen_mixnet_m( + 'mixnet_m', channel_multiplier=1.0, pretrained=pretrained, **kwargs) + return model + + +def mixnet_l(pretrained=False, **kwargs): + """Creates a MixNet Large model. + """ + # NOTE for train set drop_rate=0.25 + model = _gen_mixnet_m( + 'mixnet_l', channel_multiplier=1.3, pretrained=pretrained, **kwargs) + return model + + +def mixnet_xl(pretrained=False, **kwargs): + """Creates a MixNet Extra-Large model. + Not a paper spec, experimental def by RW w/ depth scaling. + """ + # NOTE for train set drop_rate=0.25, drop_connect_rate=0.2 + model = _gen_mixnet_m( + 'mixnet_xl', channel_multiplier=1.6, depth_multiplier=1.2, pretrained=pretrained, **kwargs) + return model + + +def mixnet_xxl(pretrained=False, **kwargs): + """Creates a MixNet Double Extra Large model. + Not a paper spec, experimental def by RW w/ depth scaling. + """ + # NOTE for train set drop_rate=0.3, drop_connect_rate=0.2 + model = _gen_mixnet_m( + 'mixnet_xxl', channel_multiplier=2.4, depth_multiplier=1.3, pretrained=pretrained, **kwargs) + return model + + +def tf_mixnet_s(pretrained=False, **kwargs): + """Creates a MixNet Small model. Tensorflow compatible variant + """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_mixnet_s( + 'tf_mixnet_s', channel_multiplier=1.0, pretrained=pretrained, **kwargs) + return model + + +def tf_mixnet_m(pretrained=False, **kwargs): + """Creates a MixNet Medium model. Tensorflow compatible variant + """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_mixnet_m( + 'tf_mixnet_m', channel_multiplier=1.0, pretrained=pretrained, **kwargs) + return model + + +def tf_mixnet_l(pretrained=False, **kwargs): + """Creates a MixNet Large model. Tensorflow compatible variant + """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_mixnet_m( + 'tf_mixnet_l', channel_multiplier=1.3, pretrained=pretrained, **kwargs) + return model diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/helpers.py b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..3f83a07d690c7ad681c777c19b1e7a5bb95da007 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/helpers.py @@ -0,0 +1,71 @@ +""" Checkpoint loading / state_dict helpers +Copyright 2020 Ross Wightman +""" +import torch +import os +from collections import OrderedDict +try: + from torch.hub import load_state_dict_from_url +except ImportError: + from torch.utils.model_zoo import load_url as load_state_dict_from_url + + +def load_checkpoint(model, checkpoint_path): + if checkpoint_path and os.path.isfile(checkpoint_path): + print("=> Loading checkpoint '{}'".format(checkpoint_path)) + checkpoint = torch.load(checkpoint_path) + if isinstance(checkpoint, dict) and 'state_dict' in checkpoint: + new_state_dict = OrderedDict() + for k, v in checkpoint['state_dict'].items(): + if k.startswith('module'): + name = k[7:] # remove `module.` + else: + name = k + new_state_dict[name] = v + model.load_state_dict(new_state_dict) + else: + model.load_state_dict(checkpoint) + print("=> Loaded checkpoint '{}'".format(checkpoint_path)) + else: + print("=> Error: No checkpoint found at '{}'".format(checkpoint_path)) + raise FileNotFoundError() + + +def load_pretrained(model, url, filter_fn=None, strict=True): + if not url: + print("=> Warning: Pretrained model URL is empty, using random initialization.") + return + + state_dict = load_state_dict_from_url(url, progress=False, map_location='cpu') + + input_conv = 'conv_stem' + classifier = 'classifier' + in_chans = getattr(model, input_conv).weight.shape[1] + num_classes = getattr(model, classifier).weight.shape[0] + + input_conv_weight = input_conv + '.weight' + pretrained_in_chans = state_dict[input_conv_weight].shape[1] + if in_chans != pretrained_in_chans: + if in_chans == 1: + print('=> Converting pretrained input conv {} from {} to 1 channel'.format( + input_conv_weight, pretrained_in_chans)) + conv1_weight = state_dict[input_conv_weight] + state_dict[input_conv_weight] = conv1_weight.sum(dim=1, keepdim=True) + else: + print('=> Discarding pretrained input conv {} since input channel count != {}'.format( + input_conv_weight, pretrained_in_chans)) + del state_dict[input_conv_weight] + strict = False + + classifier_weight = classifier + '.weight' + pretrained_num_classes = state_dict[classifier_weight].shape[0] + if num_classes != pretrained_num_classes: + print('=> Discarding pretrained classifier since num_classes != {}'.format(pretrained_num_classes)) + del state_dict[classifier_weight] + del state_dict[classifier + '.bias'] + strict = False + + if filter_fn is not None: + state_dict = filter_fn(state_dict) + + model.load_state_dict(state_dict, strict=strict) diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/mobilenetv3.py b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/mobilenetv3.py new file mode 100644 index 0000000000000000000000000000000000000000..b5966c28f7207e98ee50745b1bc8f3663c650f9d --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/mobilenetv3.py @@ -0,0 +1,364 @@ +""" MobileNet-V3 + +A PyTorch impl of MobileNet-V3, compatible with TF weights from official impl. + +Paper: Searching for MobileNetV3 - https://arxiv.org/abs/1905.02244 + +Hacked together by / Copyright 2020 Ross Wightman +""" +import torch.nn as nn +import torch.nn.functional as F + +from .activations import get_act_fn, get_act_layer, HardSwish +from .config import layer_config_kwargs +from .conv2d_layers import select_conv2d +from .helpers import load_pretrained +from .efficientnet_builder import * + +__all__ = ['mobilenetv3_rw', 'mobilenetv3_large_075', 'mobilenetv3_large_100', 'mobilenetv3_large_minimal_100', + 'mobilenetv3_small_075', 'mobilenetv3_small_100', 'mobilenetv3_small_minimal_100', + 'tf_mobilenetv3_large_075', 'tf_mobilenetv3_large_100', 'tf_mobilenetv3_large_minimal_100', + 'tf_mobilenetv3_small_075', 'tf_mobilenetv3_small_100', 'tf_mobilenetv3_small_minimal_100'] + +model_urls = { + 'mobilenetv3_rw': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/mobilenetv3_100-35495452.pth', + 'mobilenetv3_large_075': None, + 'mobilenetv3_large_100': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/mobilenetv3_large_100_ra-f55367f5.pth', + 'mobilenetv3_large_minimal_100': None, + 'mobilenetv3_small_075': None, + 'mobilenetv3_small_100': None, + 'mobilenetv3_small_minimal_100': None, + 'tf_mobilenetv3_large_075': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_mobilenetv3_large_075-150ee8b0.pth', + 'tf_mobilenetv3_large_100': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_mobilenetv3_large_100-427764d5.pth', + 'tf_mobilenetv3_large_minimal_100': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_mobilenetv3_large_minimal_100-8596ae28.pth', + 'tf_mobilenetv3_small_075': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_mobilenetv3_small_075-da427f52.pth', + 'tf_mobilenetv3_small_100': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_mobilenetv3_small_100-37f49e2b.pth', + 'tf_mobilenetv3_small_minimal_100': + 'https://github.com/rwightman/pytorch-image-models/releases/download/v0.1-weights/tf_mobilenetv3_small_minimal_100-922a7843.pth', +} + + +class MobileNetV3(nn.Module): + """ MobileNet-V3 + + A this model utilizes the MobileNet-v3 specific 'efficient head', where global pooling is done before the + head convolution without a final batch-norm layer before the classifier. + + Paper: https://arxiv.org/abs/1905.02244 + """ + + def __init__(self, block_args, num_classes=1000, in_chans=3, stem_size=16, num_features=1280, head_bias=True, + channel_multiplier=1.0, pad_type='', act_layer=HardSwish, drop_rate=0., drop_connect_rate=0., + se_kwargs=None, norm_layer=nn.BatchNorm2d, norm_kwargs=None, weight_init='goog'): + super(MobileNetV3, self).__init__() + self.drop_rate = drop_rate + + stem_size = round_channels(stem_size, channel_multiplier) + self.conv_stem = select_conv2d(in_chans, stem_size, 3, stride=2, padding=pad_type) + self.bn1 = nn.BatchNorm2d(stem_size, **norm_kwargs) + self.act1 = act_layer(inplace=True) + in_chs = stem_size + + builder = EfficientNetBuilder( + channel_multiplier, pad_type=pad_type, act_layer=act_layer, se_kwargs=se_kwargs, + norm_layer=norm_layer, norm_kwargs=norm_kwargs, drop_connect_rate=drop_connect_rate) + self.blocks = nn.Sequential(*builder(in_chs, block_args)) + in_chs = builder.in_chs + + self.global_pool = nn.AdaptiveAvgPool2d(1) + self.conv_head = select_conv2d(in_chs, num_features, 1, padding=pad_type, bias=head_bias) + self.act2 = act_layer(inplace=True) + self.classifier = nn.Linear(num_features, num_classes) + + for m in self.modules(): + if weight_init == 'goog': + initialize_weight_goog(m) + else: + initialize_weight_default(m) + + def as_sequential(self): + layers = [self.conv_stem, self.bn1, self.act1] + layers.extend(self.blocks) + layers.extend([ + self.global_pool, self.conv_head, self.act2, + nn.Flatten(), nn.Dropout(self.drop_rate), self.classifier]) + return nn.Sequential(*layers) + + def features(self, x): + x = self.conv_stem(x) + x = self.bn1(x) + x = self.act1(x) + x = self.blocks(x) + x = self.global_pool(x) + x = self.conv_head(x) + x = self.act2(x) + return x + + def forward(self, x): + x = self.features(x) + x = x.flatten(1) + if self.drop_rate > 0.: + x = F.dropout(x, p=self.drop_rate, training=self.training) + return self.classifier(x) + + +def _create_model(model_kwargs, variant, pretrained=False): + as_sequential = model_kwargs.pop('as_sequential', False) + model = MobileNetV3(**model_kwargs) + if pretrained and model_urls[variant]: + load_pretrained(model, model_urls[variant]) + if as_sequential: + model = model.as_sequential() + return model + + +def _gen_mobilenet_v3_rw(variant, channel_multiplier=1.0, pretrained=False, **kwargs): + """Creates a MobileNet-V3 model (RW variant). + + Paper: https://arxiv.org/abs/1905.02244 + + This was my first attempt at reproducing the MobileNet-V3 from paper alone. It came close to the + eventual Tensorflow reference impl but has a few differences: + 1. This model has no bias on the head convolution + 2. This model forces no residual (noskip) on the first DWS block, this is different than MnasNet + 3. This model always uses ReLU for the SE activation layer, other models in the family inherit their act layer + from their parent block + 4. This model does not enforce divisible by 8 limitation on the SE reduction channel count + + Overall the changes are fairly minor and result in a very small parameter count difference and no + top-1/5 + + Args: + channel_multiplier: multiplier to number of channels per layer. + """ + arch_def = [ + # stage 0, 112x112 in + ['ds_r1_k3_s1_e1_c16_nre_noskip'], # relu + # stage 1, 112x112 in + ['ir_r1_k3_s2_e4_c24_nre', 'ir_r1_k3_s1_e3_c24_nre'], # relu + # stage 2, 56x56 in + ['ir_r3_k5_s2_e3_c40_se0.25_nre'], # relu + # stage 3, 28x28 in + ['ir_r1_k3_s2_e6_c80', 'ir_r1_k3_s1_e2.5_c80', 'ir_r2_k3_s1_e2.3_c80'], # hard-swish + # stage 4, 14x14in + ['ir_r2_k3_s1_e6_c112_se0.25'], # hard-swish + # stage 5, 14x14in + ['ir_r3_k5_s2_e6_c160_se0.25'], # hard-swish + # stage 6, 7x7 in + ['cn_r1_k1_s1_c960'], # hard-swish + ] + with layer_config_kwargs(kwargs): + model_kwargs = dict( + block_args=decode_arch_def(arch_def), + head_bias=False, # one of my mistakes + channel_multiplier=channel_multiplier, + act_layer=resolve_act_layer(kwargs, 'hard_swish'), + se_kwargs=dict(gate_fn=get_act_fn('hard_sigmoid'), reduce_mid=True), + norm_kwargs=resolve_bn_args(kwargs), + **kwargs, + ) + model = _create_model(model_kwargs, variant, pretrained) + return model + + +def _gen_mobilenet_v3(variant, channel_multiplier=1.0, pretrained=False, **kwargs): + """Creates a MobileNet-V3 large/small/minimal models. + + Ref impl: https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet_v3.py + Paper: https://arxiv.org/abs/1905.02244 + + Args: + channel_multiplier: multiplier to number of channels per layer. + """ + if 'small' in variant: + num_features = 1024 + if 'minimal' in variant: + act_layer = 'relu' + arch_def = [ + # stage 0, 112x112 in + ['ds_r1_k3_s2_e1_c16'], + # stage 1, 56x56 in + ['ir_r1_k3_s2_e4.5_c24', 'ir_r1_k3_s1_e3.67_c24'], + # stage 2, 28x28 in + ['ir_r1_k3_s2_e4_c40', 'ir_r2_k3_s1_e6_c40'], + # stage 3, 14x14 in + ['ir_r2_k3_s1_e3_c48'], + # stage 4, 14x14in + ['ir_r3_k3_s2_e6_c96'], + # stage 6, 7x7 in + ['cn_r1_k1_s1_c576'], + ] + else: + act_layer = 'hard_swish' + arch_def = [ + # stage 0, 112x112 in + ['ds_r1_k3_s2_e1_c16_se0.25_nre'], # relu + # stage 1, 56x56 in + ['ir_r1_k3_s2_e4.5_c24_nre', 'ir_r1_k3_s1_e3.67_c24_nre'], # relu + # stage 2, 28x28 in + ['ir_r1_k5_s2_e4_c40_se0.25', 'ir_r2_k5_s1_e6_c40_se0.25'], # hard-swish + # stage 3, 14x14 in + ['ir_r2_k5_s1_e3_c48_se0.25'], # hard-swish + # stage 4, 14x14in + ['ir_r3_k5_s2_e6_c96_se0.25'], # hard-swish + # stage 6, 7x7 in + ['cn_r1_k1_s1_c576'], # hard-swish + ] + else: + num_features = 1280 + if 'minimal' in variant: + act_layer = 'relu' + arch_def = [ + # stage 0, 112x112 in + ['ds_r1_k3_s1_e1_c16'], + # stage 1, 112x112 in + ['ir_r1_k3_s2_e4_c24', 'ir_r1_k3_s1_e3_c24'], + # stage 2, 56x56 in + ['ir_r3_k3_s2_e3_c40'], + # stage 3, 28x28 in + ['ir_r1_k3_s2_e6_c80', 'ir_r1_k3_s1_e2.5_c80', 'ir_r2_k3_s1_e2.3_c80'], + # stage 4, 14x14in + ['ir_r2_k3_s1_e6_c112'], + # stage 5, 14x14in + ['ir_r3_k3_s2_e6_c160'], + # stage 6, 7x7 in + ['cn_r1_k1_s1_c960'], + ] + else: + act_layer = 'hard_swish' + arch_def = [ + # stage 0, 112x112 in + ['ds_r1_k3_s1_e1_c16_nre'], # relu + # stage 1, 112x112 in + ['ir_r1_k3_s2_e4_c24_nre', 'ir_r1_k3_s1_e3_c24_nre'], # relu + # stage 2, 56x56 in + ['ir_r3_k5_s2_e3_c40_se0.25_nre'], # relu + # stage 3, 28x28 in + ['ir_r1_k3_s2_e6_c80', 'ir_r1_k3_s1_e2.5_c80', 'ir_r2_k3_s1_e2.3_c80'], # hard-swish + # stage 4, 14x14in + ['ir_r2_k3_s1_e6_c112_se0.25'], # hard-swish + # stage 5, 14x14in + ['ir_r3_k5_s2_e6_c160_se0.25'], # hard-swish + # stage 6, 7x7 in + ['cn_r1_k1_s1_c960'], # hard-swish + ] + with layer_config_kwargs(kwargs): + model_kwargs = dict( + block_args=decode_arch_def(arch_def), + num_features=num_features, + stem_size=16, + channel_multiplier=channel_multiplier, + act_layer=resolve_act_layer(kwargs, act_layer), + se_kwargs=dict( + act_layer=get_act_layer('relu'), gate_fn=get_act_fn('hard_sigmoid'), reduce_mid=True, divisor=8), + norm_kwargs=resolve_bn_args(kwargs), + **kwargs, + ) + model = _create_model(model_kwargs, variant, pretrained) + return model + + +def mobilenetv3_rw(pretrained=False, **kwargs): + """ MobileNet-V3 RW + Attn: See note in gen function for this variant. + """ + # NOTE for train set drop_rate=0.2 + if pretrained: + # pretrained model trained with non-default BN epsilon + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + model = _gen_mobilenet_v3_rw('mobilenetv3_rw', 1.0, pretrained=pretrained, **kwargs) + return model + + +def mobilenetv3_large_075(pretrained=False, **kwargs): + """ MobileNet V3 Large 0.75""" + # NOTE for train set drop_rate=0.2 + model = _gen_mobilenet_v3('mobilenetv3_large_075', 0.75, pretrained=pretrained, **kwargs) + return model + + +def mobilenetv3_large_100(pretrained=False, **kwargs): + """ MobileNet V3 Large 1.0 """ + # NOTE for train set drop_rate=0.2 + model = _gen_mobilenet_v3('mobilenetv3_large_100', 1.0, pretrained=pretrained, **kwargs) + return model + + +def mobilenetv3_large_minimal_100(pretrained=False, **kwargs): + """ MobileNet V3 Large (Minimalistic) 1.0 """ + # NOTE for train set drop_rate=0.2 + model = _gen_mobilenet_v3('mobilenetv3_large_minimal_100', 1.0, pretrained=pretrained, **kwargs) + return model + + +def mobilenetv3_small_075(pretrained=False, **kwargs): + """ MobileNet V3 Small 0.75 """ + model = _gen_mobilenet_v3('mobilenetv3_small_075', 0.75, pretrained=pretrained, **kwargs) + return model + + +def mobilenetv3_small_100(pretrained=False, **kwargs): + """ MobileNet V3 Small 1.0 """ + model = _gen_mobilenet_v3('mobilenetv3_small_100', 1.0, pretrained=pretrained, **kwargs) + return model + + +def mobilenetv3_small_minimal_100(pretrained=False, **kwargs): + """ MobileNet V3 Small (Minimalistic) 1.0 """ + model = _gen_mobilenet_v3('mobilenetv3_small_minimal_100', 1.0, pretrained=pretrained, **kwargs) + return model + + +def tf_mobilenetv3_large_075(pretrained=False, **kwargs): + """ MobileNet V3 Large 0.75. Tensorflow compat variant. """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_mobilenet_v3('tf_mobilenetv3_large_075', 0.75, pretrained=pretrained, **kwargs) + return model + + +def tf_mobilenetv3_large_100(pretrained=False, **kwargs): + """ MobileNet V3 Large 1.0. Tensorflow compat variant. """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_mobilenet_v3('tf_mobilenetv3_large_100', 1.0, pretrained=pretrained, **kwargs) + return model + + +def tf_mobilenetv3_large_minimal_100(pretrained=False, **kwargs): + """ MobileNet V3 Large Minimalistic 1.0. Tensorflow compat variant. """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_mobilenet_v3('tf_mobilenetv3_large_minimal_100', 1.0, pretrained=pretrained, **kwargs) + return model + + +def tf_mobilenetv3_small_075(pretrained=False, **kwargs): + """ MobileNet V3 Small 0.75. Tensorflow compat variant. """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_mobilenet_v3('tf_mobilenetv3_small_075', 0.75, pretrained=pretrained, **kwargs) + return model + + +def tf_mobilenetv3_small_100(pretrained=False, **kwargs): + """ MobileNet V3 Small 1.0. Tensorflow compat variant.""" + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_mobilenet_v3('tf_mobilenetv3_small_100', 1.0, pretrained=pretrained, **kwargs) + return model + + +def tf_mobilenetv3_small_minimal_100(pretrained=False, **kwargs): + """ MobileNet V3 Small Minimalistic 1.0. Tensorflow compat variant. """ + kwargs['bn_eps'] = BN_EPS_TF_DEFAULT + kwargs['pad_type'] = 'same' + model = _gen_mobilenet_v3('tf_mobilenetv3_small_minimal_100', 1.0, pretrained=pretrained, **kwargs) + return model diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/model_factory.py b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/model_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..4d46ea8baedaf3d787826eb3bb314b4230514647 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/model_factory.py @@ -0,0 +1,27 @@ +from .config import set_layer_config +from .helpers import load_checkpoint + +from .gen_efficientnet import * +from .mobilenetv3 import * + + +def create_model( + model_name='mnasnet_100', + pretrained=None, + num_classes=1000, + in_chans=3, + checkpoint_path='', + **kwargs): + + model_kwargs = dict(num_classes=num_classes, in_chans=in_chans, pretrained=pretrained, **kwargs) + + if model_name in globals(): + create_fn = globals()[model_name] + model = create_fn(**model_kwargs) + else: + raise RuntimeError('Unknown model (%s)' % model_name) + + if checkpoint_path and not pretrained: + load_checkpoint(model, checkpoint_path) + + return model diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/version.py b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/version.py new file mode 100644 index 0000000000000000000000000000000000000000..a6221b3de7b1490c5e712e8b5fcc94c3d9d04295 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/geffnet/version.py @@ -0,0 +1 @@ +__version__ = '1.0.2' diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/hubconf.py b/annotator/normalbae/models/submodules/efficientnet_repo/hubconf.py new file mode 100644 index 0000000000000000000000000000000000000000..45b17b99bbeba34596569e6e50f6e8a2ebc45c54 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/hubconf.py @@ -0,0 +1,84 @@ +dependencies = ['torch', 'math'] + +from geffnet import efficientnet_b0 +from geffnet import efficientnet_b1 +from geffnet import efficientnet_b2 +from geffnet import efficientnet_b3 + +from geffnet import efficientnet_es + +from geffnet import efficientnet_lite0 + +from geffnet import mixnet_s +from geffnet import mixnet_m +from geffnet import mixnet_l +from geffnet import mixnet_xl + +from geffnet import mobilenetv2_100 +from geffnet import mobilenetv2_110d +from geffnet import mobilenetv2_120d +from geffnet import mobilenetv2_140 + +from geffnet import mobilenetv3_large_100 +from geffnet import mobilenetv3_rw +from geffnet import mnasnet_a1 +from geffnet import mnasnet_b1 +from geffnet import fbnetc_100 +from geffnet import spnasnet_100 + +from geffnet import tf_efficientnet_b0 +from geffnet import tf_efficientnet_b1 +from geffnet import tf_efficientnet_b2 +from geffnet import tf_efficientnet_b3 +from geffnet import tf_efficientnet_b4 +from geffnet import tf_efficientnet_b5 +from geffnet import tf_efficientnet_b6 +from geffnet import tf_efficientnet_b7 +from geffnet import tf_efficientnet_b8 + +from geffnet import tf_efficientnet_b0_ap +from geffnet import tf_efficientnet_b1_ap +from geffnet import tf_efficientnet_b2_ap +from geffnet import tf_efficientnet_b3_ap +from geffnet import tf_efficientnet_b4_ap +from geffnet import tf_efficientnet_b5_ap +from geffnet import tf_efficientnet_b6_ap +from geffnet import tf_efficientnet_b7_ap +from geffnet import tf_efficientnet_b8_ap + +from geffnet import tf_efficientnet_b0_ns +from geffnet import tf_efficientnet_b1_ns +from geffnet import tf_efficientnet_b2_ns +from geffnet import tf_efficientnet_b3_ns +from geffnet import tf_efficientnet_b4_ns +from geffnet import tf_efficientnet_b5_ns +from geffnet import tf_efficientnet_b6_ns +from geffnet import tf_efficientnet_b7_ns +from geffnet import tf_efficientnet_l2_ns_475 +from geffnet import tf_efficientnet_l2_ns + +from geffnet import tf_efficientnet_es +from geffnet import tf_efficientnet_em +from geffnet import tf_efficientnet_el + +from geffnet import tf_efficientnet_cc_b0_4e +from geffnet import tf_efficientnet_cc_b0_8e +from geffnet import tf_efficientnet_cc_b1_8e + +from geffnet import tf_efficientnet_lite0 +from geffnet import tf_efficientnet_lite1 +from geffnet import tf_efficientnet_lite2 +from geffnet import tf_efficientnet_lite3 +from geffnet import tf_efficientnet_lite4 + +from geffnet import tf_mixnet_s +from geffnet import tf_mixnet_m +from geffnet import tf_mixnet_l + +from geffnet import tf_mobilenetv3_large_075 +from geffnet import tf_mobilenetv3_large_100 +from geffnet import tf_mobilenetv3_large_minimal_100 +from geffnet import tf_mobilenetv3_small_075 +from geffnet import tf_mobilenetv3_small_100 +from geffnet import tf_mobilenetv3_small_minimal_100 + diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/onnx_export.py b/annotator/normalbae/models/submodules/efficientnet_repo/onnx_export.py new file mode 100644 index 0000000000000000000000000000000000000000..7a5162ce214830df501bdb81edb66c095122f69d --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/onnx_export.py @@ -0,0 +1,120 @@ +""" ONNX export script + +Export PyTorch models as ONNX graphs. + +This export script originally started as an adaptation of code snippets found at +https://pytorch.org/tutorials/advanced/super_resolution_with_onnxruntime.html + +The default parameters work with PyTorch 1.6 and ONNX 1.7 and produce an optimal ONNX graph +for hosting in the ONNX runtime (see onnx_validate.py). To export an ONNX model compatible +with caffe2 (see caffe2_benchmark.py and caffe2_validate.py), the --keep-init and --aten-fallback +flags are currently required. + +Older versions of PyTorch/ONNX (tested PyTorch 1.4, ONNX 1.5) do not need extra flags for +caffe2 compatibility, but they produce a model that isn't as fast running on ONNX runtime. + +Most new release of PyTorch and ONNX cause some sort of breakage in the export / usage of ONNX models. +Please do your research and search ONNX and PyTorch issue tracker before asking me. Thanks. + +Copyright 2020 Ross Wightman +""" +import argparse +import torch +import numpy as np + +import onnx +import geffnet + +parser = argparse.ArgumentParser(description='PyTorch ImageNet Validation') +parser.add_argument('output', metavar='ONNX_FILE', + help='output model filename') +parser.add_argument('--model', '-m', metavar='MODEL', default='mobilenetv3_large_100', + help='model architecture (default: mobilenetv3_large_100)') +parser.add_argument('--opset', type=int, default=10, + help='ONNX opset to use (default: 10)') +parser.add_argument('--keep-init', action='store_true', default=False, + help='Keep initializers as input. Needed for Caffe2 compatible export in newer PyTorch/ONNX.') +parser.add_argument('--aten-fallback', action='store_true', default=False, + help='Fallback to ATEN ops. Helps fix AdaptiveAvgPool issue with Caffe2 in newer PyTorch/ONNX.') +parser.add_argument('--dynamic-size', action='store_true', default=False, + help='Export model width dynamic width/height. Not recommended for "tf" models with SAME padding.') +parser.add_argument('-b', '--batch-size', default=1, type=int, + metavar='N', help='mini-batch size (default: 1)') +parser.add_argument('--img-size', default=None, type=int, + metavar='N', help='Input image dimension, uses model default if empty') +parser.add_argument('--mean', type=float, nargs='+', default=None, metavar='MEAN', + help='Override mean pixel value of dataset') +parser.add_argument('--std', type=float, nargs='+', default=None, metavar='STD', + help='Override std deviation of of dataset') +parser.add_argument('--num-classes', type=int, default=1000, + help='Number classes in dataset') +parser.add_argument('--checkpoint', default='', type=str, metavar='PATH', + help='path to checkpoint (default: none)') + + +def main(): + args = parser.parse_args() + + args.pretrained = True + if args.checkpoint: + args.pretrained = False + + print("==> Creating PyTorch {} model".format(args.model)) + # NOTE exportable=True flag disables autofn/jit scripted activations and uses Conv2dSameExport layers + # for models using SAME padding + model = geffnet.create_model( + args.model, + num_classes=args.num_classes, + in_chans=3, + pretrained=args.pretrained, + checkpoint_path=args.checkpoint, + exportable=True) + + model.eval() + + example_input = torch.randn((args.batch_size, 3, args.img_size or 224, args.img_size or 224), requires_grad=True) + + # Run model once before export trace, sets padding for models with Conv2dSameExport. This means + # that the padding for models with Conv2dSameExport (most models with tf_ prefix) is fixed for + # the input img_size specified in this script. + # Opset >= 11 should allow for dynamic padding, however I cannot get it to work due to + # issues in the tracing of the dynamic padding or errors attempting to export the model after jit + # scripting it (an approach that should work). Perhaps in a future PyTorch or ONNX versions... + model(example_input) + + print("==> Exporting model to ONNX format at '{}'".format(args.output)) + input_names = ["input0"] + output_names = ["output0"] + dynamic_axes = {'input0': {0: 'batch'}, 'output0': {0: 'batch'}} + if args.dynamic_size: + dynamic_axes['input0'][2] = 'height' + dynamic_axes['input0'][3] = 'width' + if args.aten_fallback: + export_type = torch.onnx.OperatorExportTypes.ONNX_ATEN_FALLBACK + else: + export_type = torch.onnx.OperatorExportTypes.ONNX + + torch_out = torch.onnx._export( + model, example_input, args.output, export_params=True, verbose=True, input_names=input_names, + output_names=output_names, keep_initializers_as_inputs=args.keep_init, dynamic_axes=dynamic_axes, + opset_version=args.opset, operator_export_type=export_type) + + print("==> Loading and checking exported model from '{}'".format(args.output)) + onnx_model = onnx.load(args.output) + onnx.checker.check_model(onnx_model) # assuming throw on error + print("==> Passed") + + if args.keep_init and args.aten_fallback: + import caffe2.python.onnx.backend as onnx_caffe2 + # Caffe2 loading only works properly in newer PyTorch/ONNX combos when + # keep_initializers_as_inputs and aten_fallback are set to True. + print("==> Loading model into Caffe2 backend and comparing forward pass.".format(args.output)) + caffe2_backend = onnx_caffe2.prepare(onnx_model) + B = {onnx_model.graph.input[0].name: x.data.numpy()} + c2_out = caffe2_backend.run(B)[0] + np.testing.assert_almost_equal(torch_out.data.numpy(), c2_out, decimal=5) + print("==> Passed") + + +if __name__ == '__main__': + main() diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/onnx_optimize.py b/annotator/normalbae/models/submodules/efficientnet_repo/onnx_optimize.py new file mode 100644 index 0000000000000000000000000000000000000000..ee20bbf9f0f9473370489512eb96ca0b570b5388 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/onnx_optimize.py @@ -0,0 +1,84 @@ +""" ONNX optimization script + +Run ONNX models through the optimizer to prune unneeded nodes, fuse batchnorm layers into conv, etc. + +NOTE: This isn't working consistently in recent PyTorch/ONNX combos (ie PyTorch 1.6 and ONNX 1.7), +it seems time to switch to using the onnxruntime online optimizer (can also be saved for offline). + +Copyright 2020 Ross Wightman +""" +import argparse +import warnings + +import onnx +from onnx import optimizer + + +parser = argparse.ArgumentParser(description="Optimize ONNX model") + +parser.add_argument("model", help="The ONNX model") +parser.add_argument("--output", required=True, help="The optimized model output filename") + + +def traverse_graph(graph, prefix=''): + content = [] + indent = prefix + ' ' + graphs = [] + num_nodes = 0 + for node in graph.node: + pn, gs = onnx.helper.printable_node(node, indent, subgraphs=True) + assert isinstance(gs, list) + content.append(pn) + graphs.extend(gs) + num_nodes += 1 + for g in graphs: + g_count, g_str = traverse_graph(g) + content.append('\n' + g_str) + num_nodes += g_count + return num_nodes, '\n'.join(content) + + +def main(): + args = parser.parse_args() + onnx_model = onnx.load(args.model) + num_original_nodes, original_graph_str = traverse_graph(onnx_model.graph) + + # Optimizer passes to perform + passes = [ + #'eliminate_deadend', + 'eliminate_identity', + 'eliminate_nop_dropout', + 'eliminate_nop_pad', + 'eliminate_nop_transpose', + 'eliminate_unused_initializer', + 'extract_constant_to_initializer', + 'fuse_add_bias_into_conv', + 'fuse_bn_into_conv', + 'fuse_consecutive_concats', + 'fuse_consecutive_reduce_unsqueeze', + 'fuse_consecutive_squeezes', + 'fuse_consecutive_transposes', + #'fuse_matmul_add_bias_into_gemm', + 'fuse_pad_into_conv', + #'fuse_transpose_into_gemm', + #'lift_lexical_references', + ] + + # Apply the optimization on the original serialized model + # WARNING I've had issues with optimizer in recent versions of PyTorch / ONNX causing + # 'duplicate definition of name' errors, see: https://github.com/onnx/onnx/issues/2401 + # It may be better to rely on onnxruntime optimizations, see onnx_validate.py script. + warnings.warn("I've had issues with optimizer in recent versions of PyTorch / ONNX." + "Try onnxruntime optimization if this doesn't work.") + optimized_model = optimizer.optimize(onnx_model, passes) + + num_optimized_nodes, optimzied_graph_str = traverse_graph(optimized_model.graph) + print('==> The model after optimization:\n{}\n'.format(optimzied_graph_str)) + print('==> The optimized model has {} nodes, the original had {}.'.format(num_optimized_nodes, num_original_nodes)) + + # Save the ONNX model + onnx.save(optimized_model, args.output) + + +if __name__ == "__main__": + main() diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/onnx_to_caffe.py b/annotator/normalbae/models/submodules/efficientnet_repo/onnx_to_caffe.py new file mode 100644 index 0000000000000000000000000000000000000000..44399aafababcdf6b84147a0613eb0909730db4b --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/onnx_to_caffe.py @@ -0,0 +1,27 @@ +import argparse + +import onnx +from caffe2.python.onnx.backend import Caffe2Backend + + +parser = argparse.ArgumentParser(description="Convert ONNX to Caffe2") + +parser.add_argument("model", help="The ONNX model") +parser.add_argument("--c2-prefix", required=True, + help="The output file prefix for the caffe2 model init and predict file. ") + + +def main(): + args = parser.parse_args() + onnx_model = onnx.load(args.model) + caffe2_init, caffe2_predict = Caffe2Backend.onnx_graph_to_caffe2_net(onnx_model) + caffe2_init_str = caffe2_init.SerializeToString() + with open(args.c2_prefix + '.init.pb', "wb") as f: + f.write(caffe2_init_str) + caffe2_predict_str = caffe2_predict.SerializeToString() + with open(args.c2_prefix + '.predict.pb', "wb") as f: + f.write(caffe2_predict_str) + + +if __name__ == "__main__": + main() diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/onnx_validate.py b/annotator/normalbae/models/submodules/efficientnet_repo/onnx_validate.py new file mode 100644 index 0000000000000000000000000000000000000000..ab3e4fb141b6ef660dcc5b447fd9f368a2ea19a0 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/onnx_validate.py @@ -0,0 +1,112 @@ +""" ONNX-runtime validation script + +This script was created to verify accuracy and performance of exported ONNX +models running with the onnxruntime. It utilizes the PyTorch dataloader/processing +pipeline for a fair comparison against the originals. + +Copyright 2020 Ross Wightman +""" +import argparse +import numpy as np +import onnxruntime +from data import create_loader, resolve_data_config, Dataset +from utils import AverageMeter +import time + +parser = argparse.ArgumentParser(description='Caffe2 ImageNet Validation') +parser.add_argument('data', metavar='DIR', + help='path to dataset') +parser.add_argument('--onnx-input', default='', type=str, metavar='PATH', + help='path to onnx model/weights file') +parser.add_argument('--onnx-output-opt', default='', type=str, metavar='PATH', + help='path to output optimized onnx graph') +parser.add_argument('--profile', action='store_true', default=False, + help='Enable profiler output.') +parser.add_argument('-j', '--workers', default=2, type=int, metavar='N', + help='number of data loading workers (default: 2)') +parser.add_argument('-b', '--batch-size', default=256, type=int, + metavar='N', help='mini-batch size (default: 256)') +parser.add_argument('--img-size', default=None, type=int, + metavar='N', help='Input image dimension, uses model default if empty') +parser.add_argument('--mean', type=float, nargs='+', default=None, metavar='MEAN', + help='Override mean pixel value of dataset') +parser.add_argument('--std', type=float, nargs='+', default=None, metavar='STD', + help='Override std deviation of of dataset') +parser.add_argument('--crop-pct', type=float, default=None, metavar='PCT', + help='Override default crop pct of 0.875') +parser.add_argument('--interpolation', default='', type=str, metavar='NAME', + help='Image resize interpolation type (overrides model)') +parser.add_argument('--tf-preprocessing', dest='tf_preprocessing', action='store_true', + help='use tensorflow mnasnet preporcessing') +parser.add_argument('--print-freq', '-p', default=10, type=int, + metavar='N', help='print frequency (default: 10)') + + +def main(): + args = parser.parse_args() + args.gpu_id = 0 + + # Set graph optimization level + sess_options = onnxruntime.SessionOptions() + sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL + if args.profile: + sess_options.enable_profiling = True + if args.onnx_output_opt: + sess_options.optimized_model_filepath = args.onnx_output_opt + + session = onnxruntime.InferenceSession(args.onnx_input, sess_options) + + data_config = resolve_data_config(None, args) + loader = create_loader( + Dataset(args.data, load_bytes=args.tf_preprocessing), + input_size=data_config['input_size'], + batch_size=args.batch_size, + use_prefetcher=False, + interpolation=data_config['interpolation'], + mean=data_config['mean'], + std=data_config['std'], + num_workers=args.workers, + crop_pct=data_config['crop_pct'], + tensorflow_preprocessing=args.tf_preprocessing) + + input_name = session.get_inputs()[0].name + + batch_time = AverageMeter() + top1 = AverageMeter() + top5 = AverageMeter() + end = time.time() + for i, (input, target) in enumerate(loader): + # run the net and return prediction + output = session.run([], {input_name: input.data.numpy()}) + output = output[0] + + # measure accuracy and record loss + prec1, prec5 = accuracy_np(output, target.numpy()) + top1.update(prec1.item(), input.size(0)) + top5.update(prec5.item(), input.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + print('Test: [{0}/{1}]\t' + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f}, {rate_avg:.3f}/s, {ms_avg:.3f} ms/sample) \t' + 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' + 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( + i, len(loader), batch_time=batch_time, rate_avg=input.size(0) / batch_time.avg, + ms_avg=100 * batch_time.avg / input.size(0), top1=top1, top5=top5)) + + print(' * Prec@1 {top1.avg:.3f} ({top1a:.3f}) Prec@5 {top5.avg:.3f} ({top5a:.3f})'.format( + top1=top1, top1a=100-top1.avg, top5=top5, top5a=100.-top5.avg)) + + +def accuracy_np(output, target): + max_indices = np.argsort(output, axis=1)[:, ::-1] + top5 = 100 * np.equal(max_indices[:, :5], target[:, np.newaxis]).sum(axis=1).mean() + top1 = 100 * np.equal(max_indices[:, 0], target).mean() + return top1, top5 + + +if __name__ == '__main__': + main() diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/requirements.txt b/annotator/normalbae/models/submodules/efficientnet_repo/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ac3ffc13bae15f9b11f7cbe3705760056ecd7f13 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/requirements.txt @@ -0,0 +1,2 @@ +torch>=1.2.0 +torchvision>=0.4.0 diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/setup.py b/annotator/normalbae/models/submodules/efficientnet_repo/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..023e4c30f98164595964423e3a83eefaf7ffdad6 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/setup.py @@ -0,0 +1,47 @@ +""" Setup +""" +from setuptools import setup, find_packages +from codecs import open +from os import path + +here = path.abspath(path.dirname(__file__)) + +# Get the long description from the README file +with open(path.join(here, 'README.md'), encoding='utf-8') as f: + long_description = f.read() + +exec(open('geffnet/version.py').read()) +setup( + name='geffnet', + version=__version__, + description='(Generic) EfficientNets for PyTorch', + long_description=long_description, + long_description_content_type='text/markdown', + url='https://github.com/rwightman/gen-efficientnet-pytorch', + author='Ross Wightman', + author_email='hello@rwightman.com', + classifiers=[ + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Education', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + + # Note that this is a string of words separated by whitespace, not a list. + keywords='pytorch pretrained models efficientnet mixnet mobilenetv3 mnasnet', + packages=find_packages(exclude=['data']), + install_requires=['torch >= 1.4', 'torchvision'], + python_requires='>=3.6', +) diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/utils.py b/annotator/normalbae/models/submodules/efficientnet_repo/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d327e8bd8120c5cd09ae6c15c3991ccbe27f6c1f --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/utils.py @@ -0,0 +1,52 @@ +import os + + +class AverageMeter: + """Computes and stores the average and current value""" + def __init__(self): + self.reset() + + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / self.count + + +def accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k""" + maxk = max(topk) + batch_size = target.size(0) + + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() + correct = pred.eq(target.view(1, -1).expand_as(pred)) + + res = [] + for k in topk: + correct_k = correct[:k].reshape(-1).float().sum(0) + res.append(correct_k.mul_(100.0 / batch_size)) + return res + + +def get_outdir(path, *paths, inc=False): + outdir = os.path.join(path, *paths) + if not os.path.exists(outdir): + os.makedirs(outdir) + elif inc: + count = 1 + outdir_inc = outdir + '-' + str(count) + while os.path.exists(outdir_inc): + count = count + 1 + outdir_inc = outdir + '-' + str(count) + assert count < 100 + outdir = outdir_inc + os.makedirs(outdir) + return outdir + diff --git a/annotator/normalbae/models/submodules/efficientnet_repo/validate.py b/annotator/normalbae/models/submodules/efficientnet_repo/validate.py new file mode 100644 index 0000000000000000000000000000000000000000..5fd44fbb3165ef81ef81251b6299f6aaa80bf2c2 --- /dev/null +++ b/annotator/normalbae/models/submodules/efficientnet_repo/validate.py @@ -0,0 +1,166 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import time +import torch +import torch.nn as nn +import torch.nn.parallel +from contextlib import suppress + +import geffnet +from data import Dataset, create_loader, resolve_data_config +from utils import accuracy, AverageMeter + +has_native_amp = False +try: + if getattr(torch.cuda.amp, 'autocast') is not None: + has_native_amp = True +except AttributeError: + pass + +torch.backends.cudnn.benchmark = True + +parser = argparse.ArgumentParser(description='PyTorch ImageNet Validation') +parser.add_argument('data', metavar='DIR', + help='path to dataset') +parser.add_argument('--model', '-m', metavar='MODEL', default='spnasnet1_00', + help='model architecture (default: dpn92)') +parser.add_argument('-j', '--workers', default=4, type=int, metavar='N', + help='number of data loading workers (default: 2)') +parser.add_argument('-b', '--batch-size', default=256, type=int, + metavar='N', help='mini-batch size (default: 256)') +parser.add_argument('--img-size', default=None, type=int, + metavar='N', help='Input image dimension, uses model default if empty') +parser.add_argument('--mean', type=float, nargs='+', default=None, metavar='MEAN', + help='Override mean pixel value of dataset') +parser.add_argument('--std', type=float, nargs='+', default=None, metavar='STD', + help='Override std deviation of of dataset') +parser.add_argument('--crop-pct', type=float, default=None, metavar='PCT', + help='Override default crop pct of 0.875') +parser.add_argument('--interpolation', default='', type=str, metavar='NAME', + help='Image resize interpolation type (overrides model)') +parser.add_argument('--num-classes', type=int, default=1000, + help='Number classes in dataset') +parser.add_argument('--print-freq', '-p', default=10, type=int, + metavar='N', help='print frequency (default: 10)') +parser.add_argument('--checkpoint', default='', type=str, metavar='PATH', + help='path to latest checkpoint (default: none)') +parser.add_argument('--pretrained', dest='pretrained', action='store_true', + help='use pre-trained model') +parser.add_argument('--torchscript', dest='torchscript', action='store_true', + help='convert model torchscript for inference') +parser.add_argument('--num-gpu', type=int, default=1, + help='Number of GPUS to use') +parser.add_argument('--tf-preprocessing', dest='tf_preprocessing', action='store_true', + help='use tensorflow mnasnet preporcessing') +parser.add_argument('--no-cuda', dest='no_cuda', action='store_true', + help='') +parser.add_argument('--channels-last', action='store_true', default=False, + help='Use channels_last memory layout') +parser.add_argument('--amp', action='store_true', default=False, + help='Use native Torch AMP mixed precision.') + + +def main(): + args = parser.parse_args() + + if not args.checkpoint and not args.pretrained: + args.pretrained = True + + amp_autocast = suppress # do nothing + if args.amp: + if not has_native_amp: + print("Native Torch AMP is not available (requires torch >= 1.6), using FP32.") + else: + amp_autocast = torch.cuda.amp.autocast + + # create model + model = geffnet.create_model( + args.model, + num_classes=args.num_classes, + in_chans=3, + pretrained=args.pretrained, + checkpoint_path=args.checkpoint, + scriptable=args.torchscript) + + if args.channels_last: + model = model.to(memory_format=torch.channels_last) + + if args.torchscript: + torch.jit.optimized_execution(True) + model = torch.jit.script(model) + + print('Model %s created, param count: %d' % + (args.model, sum([m.numel() for m in model.parameters()]))) + + data_config = resolve_data_config(model, args) + + criterion = nn.CrossEntropyLoss() + + if not args.no_cuda: + if args.num_gpu > 1: + model = torch.nn.DataParallel(model, device_ids=list(range(args.num_gpu))).cuda() + else: + model = model.cuda() + criterion = criterion.cuda() + + loader = create_loader( + Dataset(args.data, load_bytes=args.tf_preprocessing), + input_size=data_config['input_size'], + batch_size=args.batch_size, + use_prefetcher=not args.no_cuda, + interpolation=data_config['interpolation'], + mean=data_config['mean'], + std=data_config['std'], + num_workers=args.workers, + crop_pct=data_config['crop_pct'], + tensorflow_preprocessing=args.tf_preprocessing) + + batch_time = AverageMeter() + losses = AverageMeter() + top1 = AverageMeter() + top5 = AverageMeter() + + model.eval() + end = time.time() + with torch.no_grad(): + for i, (input, target) in enumerate(loader): + if not args.no_cuda: + target = target.cuda() + input = input.cuda() + if args.channels_last: + input = input.contiguous(memory_format=torch.channels_last) + + # compute output + with amp_autocast(): + output = model(input) + loss = criterion(output, target) + + # measure accuracy and record loss + prec1, prec5 = accuracy(output.data, target, topk=(1, 5)) + losses.update(loss.item(), input.size(0)) + top1.update(prec1.item(), input.size(0)) + top5.update(prec5.item(), input.size(0)) + + # measure elapsed time + batch_time.update(time.time() - end) + end = time.time() + + if i % args.print_freq == 0: + print('Test: [{0}/{1}]\t' + 'Time {batch_time.val:.3f} ({batch_time.avg:.3f}, {rate_avg:.3f}/s) \t' + 'Loss {loss.val:.4f} ({loss.avg:.4f})\t' + 'Prec@1 {top1.val:.3f} ({top1.avg:.3f})\t' + 'Prec@5 {top5.val:.3f} ({top5.avg:.3f})'.format( + i, len(loader), batch_time=batch_time, + rate_avg=input.size(0) / batch_time.avg, + loss=losses, top1=top1, top5=top5)) + + print(' * Prec@1 {top1.avg:.3f} ({top1a:.3f}) Prec@5 {top5.avg:.3f} ({top5a:.3f})'.format( + top1=top1, top1a=100-top1.avg, top5=top5, top5a=100.-top5.avg)) + + +if __name__ == '__main__': + main() diff --git a/annotator/normalbae/models/submodules/encoder.py b/annotator/normalbae/models/submodules/encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..7f7149ca3c0cf2b6e019105af7e645cfbb3eda11 --- /dev/null +++ b/annotator/normalbae/models/submodules/encoder.py @@ -0,0 +1,34 @@ +import os +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class Encoder(nn.Module): + def __init__(self): + super(Encoder, self).__init__() + + basemodel_name = 'tf_efficientnet_b5_ap' + print('Loading base model ()...'.format(basemodel_name), end='') + repo_path = os.path.join(os.path.dirname(__file__), 'efficientnet_repo') + basemodel = torch.hub.load(repo_path, basemodel_name, pretrained=False, source='local') + print('Done.') + + # Remove last layer + print('Removing last two layers (global_pool & classifier).') + basemodel.global_pool = nn.Identity() + basemodel.classifier = nn.Identity() + + self.original_model = basemodel + + def forward(self, x): + features = [x] + for k, v in self.original_model._modules.items(): + if (k == 'blocks'): + for ki, vi in v._modules.items(): + features.append(vi(features[-1])) + else: + features.append(v(features[-1])) + return features + + diff --git a/annotator/normalbae/models/submodules/submodules.py b/annotator/normalbae/models/submodules/submodules.py new file mode 100644 index 0000000000000000000000000000000000000000..409733351bd6ab5d191c800aff1bc05bfa4cb6f8 --- /dev/null +++ b/annotator/normalbae/models/submodules/submodules.py @@ -0,0 +1,140 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + + +######################################################################################################################## + + +# Upsample + BatchNorm +class UpSampleBN(nn.Module): + def __init__(self, skip_input, output_features): + super(UpSampleBN, self).__init__() + + self._net = nn.Sequential(nn.Conv2d(skip_input, output_features, kernel_size=3, stride=1, padding=1), + nn.BatchNorm2d(output_features), + nn.LeakyReLU(), + nn.Conv2d(output_features, output_features, kernel_size=3, stride=1, padding=1), + nn.BatchNorm2d(output_features), + nn.LeakyReLU()) + + def forward(self, x, concat_with): + up_x = F.interpolate(x, size=[concat_with.size(2), concat_with.size(3)], mode='bilinear', align_corners=True) + f = torch.cat([up_x, concat_with], dim=1) + return self._net(f) + + +# Upsample + GroupNorm + Weight Standardization +class UpSampleGN(nn.Module): + def __init__(self, skip_input, output_features): + super(UpSampleGN, self).__init__() + + self._net = nn.Sequential(Conv2d(skip_input, output_features, kernel_size=3, stride=1, padding=1), + nn.GroupNorm(8, output_features), + nn.LeakyReLU(), + Conv2d(output_features, output_features, kernel_size=3, stride=1, padding=1), + nn.GroupNorm(8, output_features), + nn.LeakyReLU()) + + def forward(self, x, concat_with): + up_x = F.interpolate(x, size=[concat_with.size(2), concat_with.size(3)], mode='bilinear', align_corners=True) + f = torch.cat([up_x, concat_with], dim=1) + return self._net(f) + + +# Conv2d with weight standardization +class Conv2d(nn.Conv2d): + def __init__(self, in_channels, out_channels, kernel_size, stride=1, + padding=0, dilation=1, groups=1, bias=True): + super(Conv2d, self).__init__(in_channels, out_channels, kernel_size, stride, + padding, dilation, groups, bias) + + def forward(self, x): + weight = self.weight + weight_mean = weight.mean(dim=1, keepdim=True).mean(dim=2, + keepdim=True).mean(dim=3, keepdim=True) + weight = weight - weight_mean + std = weight.view(weight.size(0), -1).std(dim=1).view(-1, 1, 1, 1) + 1e-5 + weight = weight / std.expand_as(weight) + return F.conv2d(x, weight, self.bias, self.stride, + self.padding, self.dilation, self.groups) + + +# normalize +def norm_normalize(norm_out): + min_kappa = 0.01 + norm_x, norm_y, norm_z, kappa = torch.split(norm_out, 1, dim=1) + norm = torch.sqrt(norm_x ** 2.0 + norm_y ** 2.0 + norm_z ** 2.0) + 1e-10 + kappa = F.elu(kappa) + 1.0 + min_kappa + final_out = torch.cat([norm_x / norm, norm_y / norm, norm_z / norm, kappa], dim=1) + return final_out + + +# uncertainty-guided sampling (only used during training) +@torch.no_grad() +def sample_points(init_normal, gt_norm_mask, sampling_ratio, beta): + device = init_normal.device + B, _, H, W = init_normal.shape + N = int(sampling_ratio * H * W) + beta = beta + + # uncertainty map + uncertainty_map = -1 * init_normal[:, 3, :, :] # B, H, W + + # gt_invalid_mask (B, H, W) + if gt_norm_mask is not None: + gt_invalid_mask = F.interpolate(gt_norm_mask.float(), size=[H, W], mode='nearest') + gt_invalid_mask = gt_invalid_mask[:, 0, :, :] < 0.5 + uncertainty_map[gt_invalid_mask] = -1e4 + + # (B, H*W) + _, idx = uncertainty_map.view(B, -1).sort(1, descending=True) + + # importance sampling + if int(beta * N) > 0: + importance = idx[:, :int(beta * N)] # B, beta*N + + # remaining + remaining = idx[:, int(beta * N):] # B, H*W - beta*N + + # coverage + num_coverage = N - int(beta * N) + + if num_coverage <= 0: + samples = importance + else: + coverage_list = [] + for i in range(B): + idx_c = torch.randperm(remaining.size()[1]) # shuffles "H*W - beta*N" + coverage_list.append(remaining[i, :][idx_c[:num_coverage]].view(1, -1)) # 1, N-beta*N + coverage = torch.cat(coverage_list, dim=0) # B, N-beta*N + samples = torch.cat((importance, coverage), dim=1) # B, N + + else: + # remaining + remaining = idx[:, :] # B, H*W + + # coverage + num_coverage = N + + coverage_list = [] + for i in range(B): + idx_c = torch.randperm(remaining.size()[1]) # shuffles "H*W - beta*N" + coverage_list.append(remaining[i, :][idx_c[:num_coverage]].view(1, -1)) # 1, N-beta*N + coverage = torch.cat(coverage_list, dim=0) # B, N-beta*N + samples = coverage + + # point coordinates + rows_int = samples // W # 0 for first row, H-1 for last row + rows_float = rows_int / float(H-1) # 0 to 1.0 + rows_float = (rows_float * 2.0) - 1.0 # -1.0 to 1.0 + + cols_int = samples % W # 0 for first column, W-1 for last column + cols_float = cols_int / float(W-1) # 0 to 1.0 + cols_float = (cols_float * 2.0) - 1.0 # -1.0 to 1.0 + + point_coords = torch.zeros(B, 1, N, 2) + point_coords[:, 0, :, 0] = cols_float # x coord + point_coords[:, 0, :, 1] = rows_float # y coord + point_coords = point_coords.to(device) + return point_coords, rows_int, cols_int \ No newline at end of file diff --git a/annotator/normalbae/utils/losses.py b/annotator/normalbae/utils/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..f47f610f94b6c14c260433bb0cb94c24820f132e --- /dev/null +++ b/annotator/normalbae/utils/losses.py @@ -0,0 +1,178 @@ +import torch +import torch.nn as nn +import numpy as np +import torch.nn.functional as F + + +# compute loss +class compute_loss(nn.Module): + def __init__(self, args): + """args.loss_fn can be one of following: + - L1 - L1 loss (no uncertainty) + - L2 - L2 loss (no uncertainty) + - AL - Angular loss (no uncertainty) + - NLL_vMF - NLL of vonMF distribution + - NLL_ours - NLL of Angular vonMF distribution + - UG_NLL_vMF - NLL of vonMF distribution (+ pixel-wise MLP + uncertainty-guided sampling) + - UG_NLL_ours - NLL of Angular vonMF distribution (+ pixel-wise MLP + uncertainty-guided sampling) + """ + super(compute_loss, self).__init__() + self.loss_type = args.loss_fn + if self.loss_type in ['L1', 'L2', 'AL', 'NLL_vMF', 'NLL_ours']: + self.loss_fn = self.forward_R + elif self.loss_type in ['UG_NLL_vMF', 'UG_NLL_ours']: + self.loss_fn = self.forward_UG + else: + raise Exception('invalid loss type') + + def forward(self, *args): + return self.loss_fn(*args) + + def forward_R(self, norm_out, gt_norm, gt_norm_mask): + pred_norm, pred_kappa = norm_out[:, 0:3, :, :], norm_out[:, 3:, :, :] + + if self.loss_type == 'L1': + l1 = torch.sum(torch.abs(gt_norm - pred_norm), dim=1, keepdim=True) + loss = torch.mean(l1[gt_norm_mask]) + + elif self.loss_type == 'L2': + l2 = torch.sum(torch.square(gt_norm - pred_norm), dim=1, keepdim=True) + loss = torch.mean(l2[gt_norm_mask]) + + elif self.loss_type == 'AL': + dot = torch.cosine_similarity(pred_norm, gt_norm, dim=1) + + valid_mask = gt_norm_mask[:, 0, :, :].float() \ + * (dot.detach() < 0.999).float() \ + * (dot.detach() > -0.999).float() + valid_mask = valid_mask > 0.0 + + al = torch.acos(dot[valid_mask]) + loss = torch.mean(al) + + elif self.loss_type == 'NLL_vMF': + dot = torch.cosine_similarity(pred_norm, gt_norm, dim=1) + + valid_mask = gt_norm_mask[:, 0, :, :].float() \ + * (dot.detach() < 0.999).float() \ + * (dot.detach() > -0.999).float() + valid_mask = valid_mask > 0.0 + + dot = dot[valid_mask] + kappa = pred_kappa[:, 0, :, :][valid_mask] + + loss_pixelwise = - torch.log(kappa) \ + - (kappa * (dot - 1)) \ + + torch.log(1 - torch.exp(- 2 * kappa)) + loss = torch.mean(loss_pixelwise) + + elif self.loss_type == 'NLL_ours': + dot = torch.cosine_similarity(pred_norm, gt_norm, dim=1) + + valid_mask = gt_norm_mask[:, 0, :, :].float() \ + * (dot.detach() < 0.999).float() \ + * (dot.detach() > -0.999).float() + valid_mask = valid_mask > 0.0 + + dot = dot[valid_mask] + kappa = pred_kappa[:, 0, :, :][valid_mask] + + loss_pixelwise = - torch.log(torch.square(kappa) + 1) \ + + kappa * torch.acos(dot) \ + + torch.log(1 + torch.exp(-kappa * np.pi)) + loss = torch.mean(loss_pixelwise) + + else: + raise Exception('invalid loss type') + + return loss + + + def forward_UG(self, pred_list, coord_list, gt_norm, gt_norm_mask): + loss = 0.0 + for (pred, coord) in zip(pred_list, coord_list): + if coord is None: + pred = F.interpolate(pred, size=[gt_norm.size(2), gt_norm.size(3)], mode='bilinear', align_corners=True) + pred_norm, pred_kappa = pred[:, 0:3, :, :], pred[:, 3:, :, :] + + if self.loss_type == 'UG_NLL_vMF': + dot = torch.cosine_similarity(pred_norm, gt_norm, dim=1) + + valid_mask = gt_norm_mask[:, 0, :, :].float() \ + * (dot.detach() < 0.999).float() \ + * (dot.detach() > -0.999).float() + valid_mask = valid_mask > 0.5 + + # mask + dot = dot[valid_mask] + kappa = pred_kappa[:, 0, :, :][valid_mask] + + loss_pixelwise = - torch.log(kappa) \ + - (kappa * (dot - 1)) \ + + torch.log(1 - torch.exp(- 2 * kappa)) + loss = loss + torch.mean(loss_pixelwise) + + elif self.loss_type == 'UG_NLL_ours': + dot = torch.cosine_similarity(pred_norm, gt_norm, dim=1) + + valid_mask = gt_norm_mask[:, 0, :, :].float() \ + * (dot.detach() < 0.999).float() \ + * (dot.detach() > -0.999).float() + valid_mask = valid_mask > 0.5 + + dot = dot[valid_mask] + kappa = pred_kappa[:, 0, :, :][valid_mask] + + loss_pixelwise = - torch.log(torch.square(kappa) + 1) \ + + kappa * torch.acos(dot) \ + + torch.log(1 + torch.exp(-kappa * np.pi)) + loss = loss + torch.mean(loss_pixelwise) + + else: + raise Exception + + else: + # coord: B, 1, N, 2 + # pred: B, 4, N + gt_norm_ = F.grid_sample(gt_norm, coord, mode='nearest', align_corners=True) # (B, 3, 1, N) + gt_norm_mask_ = F.grid_sample(gt_norm_mask.float(), coord, mode='nearest', align_corners=True) # (B, 1, 1, N) + gt_norm_ = gt_norm_[:, :, 0, :] # (B, 3, N) + gt_norm_mask_ = gt_norm_mask_[:, :, 0, :] > 0.5 # (B, 1, N) + + pred_norm, pred_kappa = pred[:, 0:3, :], pred[:, 3:, :] + + if self.loss_type == 'UG_NLL_vMF': + dot = torch.cosine_similarity(pred_norm, gt_norm_, dim=1) # (B, N) + + valid_mask = gt_norm_mask_[:, 0, :].float() \ + * (dot.detach() < 0.999).float() \ + * (dot.detach() > -0.999).float() + valid_mask = valid_mask > 0.5 + + dot = dot[valid_mask] + kappa = pred_kappa[:, 0, :][valid_mask] + + loss_pixelwise = - torch.log(kappa) \ + - (kappa * (dot - 1)) \ + + torch.log(1 - torch.exp(- 2 * kappa)) + loss = loss + torch.mean(loss_pixelwise) + + elif self.loss_type == 'UG_NLL_ours': + dot = torch.cosine_similarity(pred_norm, gt_norm_, dim=1) # (B, N) + + valid_mask = gt_norm_mask_[:, 0, :].float() \ + * (dot.detach() < 0.999).float() \ + * (dot.detach() > -0.999).float() + valid_mask = valid_mask > 0.5 + + dot = dot[valid_mask] + kappa = pred_kappa[:, 0, :][valid_mask] + + loss_pixelwise = - torch.log(torch.square(kappa) + 1) \ + + kappa * torch.acos(dot) \ + + torch.log(1 + torch.exp(-kappa * np.pi)) + loss = loss + torch.mean(loss_pixelwise) + + else: + raise Exception + return loss diff --git a/annotator/normalbae/utils/utils.py b/annotator/normalbae/utils/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ca7dde89929c1f9f696cc93e995af9a667cf86c8 --- /dev/null +++ b/annotator/normalbae/utils/utils.py @@ -0,0 +1,191 @@ +import os +import numpy as np +from PIL import Image + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt + + + +# convert arg line to args +def convert_arg_line_to_args(arg_line): + for arg in arg_line.split(): + if not arg.strip(): + continue + yield str(arg) + + +# save args +def save_args(args, filename): + with open(filename, 'w') as f: + for arg in vars(args): + f.write('{}: {}\n'.format(arg, getattr(args, arg))) + + +# concatenate images +def concat_image(image_path_list, concat_image_path): + imgs = [Image.open(i).convert("RGB").resize((640, 480), resample=Image.BILINEAR) for i in image_path_list] + imgs_list = [] + for i in range(len(imgs)): + img = imgs[i] + imgs_list.append(np.asarray(img)) + + H, W, _ = np.asarray(img).shape + imgs_list.append(255 * np.ones((H, 20, 3)).astype('uint8')) + + imgs_comb = np.hstack(imgs_list[:-1]) + imgs_comb = Image.fromarray(imgs_comb) + imgs_comb.save(concat_image_path) + + +# load model +def load_checkpoint(fpath, model): + ckpt = torch.load(fpath, map_location='cpu')['model'] + + load_dict = {} + for k, v in ckpt.items(): + if k.startswith('module.'): + k_ = k.replace('module.', '') + load_dict[k_] = v + else: + load_dict[k] = v + + model.load_state_dict(load_dict) + return model + + +# compute normal errors +def compute_normal_errors(total_normal_errors): + metrics = { + 'mean': np.average(total_normal_errors), + 'median': np.median(total_normal_errors), + 'rmse': np.sqrt(np.sum(total_normal_errors * total_normal_errors) / total_normal_errors.shape), + 'a1': 100.0 * (np.sum(total_normal_errors < 5) / total_normal_errors.shape[0]), + 'a2': 100.0 * (np.sum(total_normal_errors < 7.5) / total_normal_errors.shape[0]), + 'a3': 100.0 * (np.sum(total_normal_errors < 11.25) / total_normal_errors.shape[0]), + 'a4': 100.0 * (np.sum(total_normal_errors < 22.5) / total_normal_errors.shape[0]), + 'a5': 100.0 * (np.sum(total_normal_errors < 30) / total_normal_errors.shape[0]) + } + return metrics + + +# log normal errors +def log_normal_errors(metrics, where_to_write, first_line): + print(first_line) + print("mean median rmse 5 7.5 11.25 22.5 30") + print("%.3f %.3f %.3f %.3f %.3f %.3f %.3f %.3f" % ( + metrics['mean'], metrics['median'], metrics['rmse'], + metrics['a1'], metrics['a2'], metrics['a3'], metrics['a4'], metrics['a5'])) + + with open(where_to_write, 'a') as f: + f.write('%s\n' % first_line) + f.write("mean median rmse 5 7.5 11.25 22.5 30\n") + f.write("%.3f %.3f %.3f %.3f %.3f %.3f %.3f %.3f\n\n" % ( + metrics['mean'], metrics['median'], metrics['rmse'], + metrics['a1'], metrics['a2'], metrics['a3'], metrics['a4'], metrics['a5'])) + + +# makedir +def makedir(dirpath): + if not os.path.exists(dirpath): + os.makedirs(dirpath) + + +# makedir from list +def make_dir_from_list(dirpath_list): + for dirpath in dirpath_list: + makedir(dirpath) + + + +######################################################################################################################## +# Visualization +######################################################################################################################## + + +# unnormalize image +__imagenet_stats = {'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]} +def unnormalize(img_in): + img_out = np.zeros(img_in.shape) + for ich in range(3): + img_out[:, :, ich] = img_in[:, :, ich] * __imagenet_stats['std'][ich] + img_out[:, :, ich] += __imagenet_stats['mean'][ich] + img_out = (img_out * 255).astype(np.uint8) + return img_out + + +# kappa to exp error (only applicable to AngMF distribution) +def kappa_to_alpha(pred_kappa): + alpha = ((2 * pred_kappa) / ((pred_kappa ** 2.0) + 1)) \ + + ((np.exp(- pred_kappa * np.pi) * np.pi) / (1 + np.exp(- pred_kappa * np.pi))) + alpha = np.degrees(alpha) + return alpha + + +# normal vector to rgb values +def norm_to_rgb(norm): + # norm: (B, H, W, 3) + norm_rgb = ((norm[0, ...] + 1) * 0.5) * 255 + norm_rgb = np.clip(norm_rgb, a_min=0, a_max=255) + norm_rgb = norm_rgb.astype(np.uint8) + return norm_rgb + + +# visualize during training +def visualize(args, img, gt_norm, gt_norm_mask, norm_out_list, total_iter): + B, _, H, W = gt_norm.shape + + pred_norm_list = [] + pred_kappa_list = [] + for norm_out in norm_out_list: + norm_out = F.interpolate(norm_out, size=[gt_norm.size(2), gt_norm.size(3)], mode='nearest') + pred_norm = norm_out[:, :3, :, :] # (B, 3, H, W) + pred_norm = pred_norm.detach().cpu().permute(0, 2, 3, 1).numpy() # (B, H, W, 3) + pred_norm_list.append(pred_norm) + + pred_kappa = norm_out[:, 3:, :, :] # (B, 1, H, W) + pred_kappa = pred_kappa.detach().cpu().permute(0, 2, 3, 1).numpy() # (B, H, W, 1) + pred_kappa_list.append(pred_kappa) + + # to numpy arrays + img = img.detach().cpu().permute(0, 2, 3, 1).numpy() # (B, H, W, 3) + gt_norm = gt_norm.detach().cpu().permute(0, 2, 3, 1).numpy() # (B, H, W, 3) + gt_norm_mask = gt_norm_mask.detach().cpu().permute(0, 2, 3, 1).numpy() # (B, H, W, 1) + + # input image + target_path = '%s/%08d_img.jpg' % (args.exp_vis_dir, total_iter) + img = unnormalize(img[0, ...]) + plt.imsave(target_path, img) + + # gt norm + gt_norm_rgb = ((gt_norm[0, ...] + 1) * 0.5) * 255 + gt_norm_rgb = np.clip(gt_norm_rgb, a_min=0, a_max=255) + gt_norm_rgb = gt_norm_rgb.astype(np.uint8) + + target_path = '%s/%08d_gt_norm.jpg' % (args.exp_vis_dir, total_iter) + plt.imsave(target_path, gt_norm_rgb * gt_norm_mask[0, ...]) + + # pred_norm + for i in range(len(pred_norm_list)): + pred_norm = pred_norm_list[i] + pred_norm_rgb = norm_to_rgb(pred_norm) + target_path = '%s/%08d_pred_norm_%d.jpg' % (args.exp_vis_dir, total_iter, i) + plt.imsave(target_path, pred_norm_rgb) + + pred_kappa = pred_kappa_list[i] + pred_alpha = kappa_to_alpha(pred_kappa) + target_path = '%s/%08d_pred_alpha_%d.jpg' % (args.exp_vis_dir, total_iter, i) + plt.imsave(target_path, pred_alpha[0, :, :, 0], vmin=0, vmax=60, cmap='jet') + + # error in angles + DP = np.sum(gt_norm * pred_norm, axis=3, keepdims=True) # (B, H, W, 1) + DP = np.clip(DP, -1, 1) + E = np.degrees(np.arccos(DP)) # (B, H, W, 1) + E = E * gt_norm_mask + target_path = '%s/%08d_pred_error_%d.jpg' % (args.exp_vis_dir, total_iter, i) + plt.imsave(target_path, E[0, :, :, 0], vmin=0, vmax=60, cmap='jet') diff --git a/annotator/oneformer/LICENSE b/annotator/oneformer/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..16a9d56a3d4c15e4f34ac5426459c58487b01520 --- /dev/null +++ b/annotator/oneformer/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Caroline Chan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/annotator/oneformer/__init__.py b/annotator/oneformer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..57ff0c6581e552df750abe5bb92ed4f39a7dfa46 --- /dev/null +++ b/annotator/oneformer/__init__.py @@ -0,0 +1,34 @@ +# https://github.com/SHI-Labs/OneFormer + +import os +from annotator.util import annotator_ckpts_path +from .api import make_detectron2_model, semantic_run + + +class OneformerCOCODetector: + def __init__(self): + remote_model_path = "https://huggingface.co./lllyasviel/Annotators/resolve/main/150_16_swin_l_oneformer_coco_100ep.pth" + modelpath = os.path.join(annotator_ckpts_path, "150_16_swin_l_oneformer_coco_100ep.pth") + if not os.path.exists(modelpath): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(remote_model_path, model_dir=annotator_ckpts_path) + config = os.path.join(os.path.dirname(__file__), 'configs/coco/oneformer_swin_large_IN21k_384_bs16_100ep.yaml') + self.model, self.meta = make_detectron2_model(config, modelpath) + + def __call__(self, img): + return semantic_run(img, self.model, self.meta) + + +class OneformerADE20kDetector: + def __init__(self): + remote_model_path = "https://huggingface.co./lllyasviel/Annotators/resolve/main/250_16_swin_l_oneformer_ade20k_160k.pth" + modelpath = os.path.join(annotator_ckpts_path, "250_16_swin_l_oneformer_ade20k_160k.pth") + if not os.path.exists(modelpath): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(remote_model_path, model_dir=annotator_ckpts_path) + config = os.path.join(os.path.dirname(__file__), 'configs/ade20k/oneformer_swin_large_IN21k_384_bs16_160k.yaml') + self.model, self.meta = make_detectron2_model(config, modelpath) + + def __call__(self, img): + return semantic_run(img, self.model, self.meta) + diff --git a/annotator/oneformer/api.py b/annotator/oneformer/api.py new file mode 100644 index 0000000000000000000000000000000000000000..e96d7d6e32d7e52fae776792d810a19dfee18015 --- /dev/null +++ b/annotator/oneformer/api.py @@ -0,0 +1,43 @@ +import os +os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" + +import torch + +from annotator.oneformer.detectron2.config import get_cfg +from annotator.oneformer.detectron2.projects.deeplab import add_deeplab_config +from annotator.oneformer.detectron2.data import MetadataCatalog + +from annotator.oneformer.oneformer import ( + add_oneformer_config, + add_common_config, + add_swin_config, + add_dinat_config, +) + +from annotator.oneformer.oneformer.demo.defaults import DefaultPredictor +from annotator.oneformer.oneformer.demo.visualizer import Visualizer, ColorMode + + +def make_detectron2_model(config_path, ckpt_path): + cfg = get_cfg() + add_deeplab_config(cfg) + add_common_config(cfg) + add_swin_config(cfg) + add_oneformer_config(cfg) + add_dinat_config(cfg) + cfg.merge_from_file(config_path) + if torch.cuda.is_available(): + cfg.MODEL.DEVICE = 'cuda' + else: + cfg.MODEL.DEVICE = 'cpu' + cfg.MODEL.WEIGHTS = ckpt_path + cfg.freeze() + metadata = MetadataCatalog.get(cfg.DATASETS.TEST_PANOPTIC[0] if len(cfg.DATASETS.TEST_PANOPTIC) else "__unused") + return DefaultPredictor(cfg), metadata + + +def semantic_run(img, predictor, metadata): + predictions = predictor(img[:, :, ::-1], "semantic") # Predictor of OneFormer must use BGR image !!! + visualizer_map = Visualizer(img, is_img=False, metadata=metadata, instance_mode=ColorMode.IMAGE) + out_map = visualizer_map.draw_sem_seg(predictions["sem_seg"].argmax(dim=0).cpu(), alpha=1, is_text=False).get_image() + return out_map diff --git a/annotator/oneformer/configs/ade20k/Base-ADE20K-UnifiedSegmentation.yaml b/annotator/oneformer/configs/ade20k/Base-ADE20K-UnifiedSegmentation.yaml new file mode 100644 index 0000000000000000000000000000000000000000..31eab45b878433fc844a13dbdd54f97c936d9b89 --- /dev/null +++ b/annotator/oneformer/configs/ade20k/Base-ADE20K-UnifiedSegmentation.yaml @@ -0,0 +1,68 @@ +MODEL: + BACKBONE: + FREEZE_AT: 0 + NAME: "build_resnet_backbone" + WEIGHTS: "detectron2://ImageNetPretrained/torchvision/R-50.pkl" + PIXEL_MEAN: [123.675, 116.280, 103.530] + PIXEL_STD: [58.395, 57.120, 57.375] + RESNETS: + DEPTH: 50 + STEM_TYPE: "basic" # not used + STEM_OUT_CHANNELS: 64 + STRIDE_IN_1X1: False + OUT_FEATURES: ["res2", "res3", "res4", "res5"] + # NORM: "SyncBN" + RES5_MULTI_GRID: [1, 1, 1] # not used +DATASETS: + TRAIN: ("ade20k_panoptic_train",) + TEST_PANOPTIC: ("ade20k_panoptic_val",) + TEST_INSTANCE: ("ade20k_instance_val",) + TEST_SEMANTIC: ("ade20k_sem_seg_val",) +SOLVER: + IMS_PER_BATCH: 16 + BASE_LR: 0.0001 + MAX_ITER: 160000 + WARMUP_FACTOR: 1.0 + WARMUP_ITERS: 0 + WEIGHT_DECAY: 0.05 + OPTIMIZER: "ADAMW" + LR_SCHEDULER_NAME: "WarmupPolyLR" + BACKBONE_MULTIPLIER: 0.1 + CLIP_GRADIENTS: + ENABLED: True + CLIP_TYPE: "full_model" + CLIP_VALUE: 0.01 + NORM_TYPE: 2.0 + AMP: + ENABLED: True +INPUT: + MIN_SIZE_TRAIN: !!python/object/apply:eval ["[int(x * 0.1 * 512) for x in range(5, 21)]"] + MIN_SIZE_TRAIN_SAMPLING: "choice" + MIN_SIZE_TEST: 512 + MAX_SIZE_TRAIN: 2048 + MAX_SIZE_TEST: 2048 + CROP: + ENABLED: True + TYPE: "absolute" + SIZE: (512, 512) + SINGLE_CATEGORY_MAX_AREA: 1.0 + COLOR_AUG_SSD: True + SIZE_DIVISIBILITY: 512 # used in dataset mapper + FORMAT: "RGB" + DATASET_MAPPER_NAME: "oneformer_unified" + MAX_SEQ_LEN: 77 + TASK_SEQ_LEN: 77 + TASK_PROB: + SEMANTIC: 0.33 + INSTANCE: 0.66 +TEST: + EVAL_PERIOD: 5000 + AUG: + ENABLED: False + MIN_SIZES: [256, 384, 512, 640, 768, 896] + MAX_SIZE: 3584 + FLIP: True +DATALOADER: + FILTER_EMPTY_ANNOTATIONS: True + NUM_WORKERS: 4 +VERSION: 2 \ No newline at end of file diff --git a/annotator/oneformer/configs/ade20k/oneformer_R50_bs16_160k.yaml b/annotator/oneformer/configs/ade20k/oneformer_R50_bs16_160k.yaml new file mode 100644 index 0000000000000000000000000000000000000000..770ffc81907f8d7c7520e079b1c46060707254b8 --- /dev/null +++ b/annotator/oneformer/configs/ade20k/oneformer_R50_bs16_160k.yaml @@ -0,0 +1,58 @@ +_BASE_: Base-ADE20K-UnifiedSegmentation.yaml +MODEL: + META_ARCHITECTURE: "OneFormer" + SEM_SEG_HEAD: + NAME: "OneFormerHead" + IGNORE_VALUE: 255 + NUM_CLASSES: 150 + LOSS_WEIGHT: 1.0 + CONVS_DIM: 256 + MASK_DIM: 256 + NORM: "GN" + # pixel decoder + PIXEL_DECODER_NAME: "MSDeformAttnPixelDecoder" + IN_FEATURES: ["res2", "res3", "res4", "res5"] + DEFORMABLE_TRANSFORMER_ENCODER_IN_FEATURES: ["res3", "res4", "res5"] + COMMON_STRIDE: 4 + TRANSFORMER_ENC_LAYERS: 6 + ONE_FORMER: + TRANSFORMER_DECODER_NAME: "ContrastiveMultiScaleMaskedTransformerDecoder" + TRANSFORMER_IN_FEATURE: "multi_scale_pixel_decoder" + DEEP_SUPERVISION: True + NO_OBJECT_WEIGHT: 0.1 + CLASS_WEIGHT: 2.0 + MASK_WEIGHT: 5.0 + DICE_WEIGHT: 5.0 + CONTRASTIVE_WEIGHT: 0.5 + CONTRASTIVE_TEMPERATURE: 0.07 + HIDDEN_DIM: 256 + NUM_OBJECT_QUERIES: 150 + USE_TASK_NORM: True + NHEADS: 8 + DROPOUT: 0.1 + DIM_FEEDFORWARD: 2048 + ENC_LAYERS: 0 + PRE_NORM: False + ENFORCE_INPUT_PROJ: False + SIZE_DIVISIBILITY: 32 + CLASS_DEC_LAYERS: 2 + DEC_LAYERS: 10 # 9 decoder layers, add one for the loss on learnable query + TRAIN_NUM_POINTS: 12544 + OVERSAMPLE_RATIO: 3.0 + IMPORTANCE_SAMPLE_RATIO: 0.75 + TEXT_ENCODER: + WIDTH: 256 + CONTEXT_LENGTH: 77 + NUM_LAYERS: 6 + VOCAB_SIZE: 49408 + PROJ_NUM_LAYERS: 2 + N_CTX: 16 + TEST: + SEMANTIC_ON: True + INSTANCE_ON: True + PANOPTIC_ON: True + OVERLAP_THRESHOLD: 0.8 + OBJECT_MASK_THRESHOLD: 0.8 + TASK: "panoptic" +TEST: + DETECTIONS_PER_IMAGE: 150 diff --git a/annotator/oneformer/configs/ade20k/oneformer_swin_large_IN21k_384_bs16_160k.yaml b/annotator/oneformer/configs/ade20k/oneformer_swin_large_IN21k_384_bs16_160k.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69c44ade144e4504077c0fe04fa8bb3491a679ed --- /dev/null +++ b/annotator/oneformer/configs/ade20k/oneformer_swin_large_IN21k_384_bs16_160k.yaml @@ -0,0 +1,40 @@ +_BASE_: oneformer_R50_bs16_160k.yaml +MODEL: + BACKBONE: + NAME: "D2SwinTransformer" + SWIN: + EMBED_DIM: 192 + DEPTHS: [2, 2, 18, 2] + NUM_HEADS: [6, 12, 24, 48] + WINDOW_SIZE: 12 + APE: False + DROP_PATH_RATE: 0.3 + PATCH_NORM: True + PRETRAIN_IMG_SIZE: 384 + WEIGHTS: "swin_large_patch4_window12_384_22k.pkl" + PIXEL_MEAN: [123.675, 116.280, 103.530] + PIXEL_STD: [58.395, 57.120, 57.375] + ONE_FORMER: + NUM_OBJECT_QUERIES: 250 +INPUT: + MIN_SIZE_TRAIN: !!python/object/apply:eval ["[int(x * 0.1 * 640) for x in range(5, 21)]"] + MIN_SIZE_TRAIN_SAMPLING: "choice" + MIN_SIZE_TEST: 640 + MAX_SIZE_TRAIN: 2560 + MAX_SIZE_TEST: 2560 + CROP: + ENABLED: True + TYPE: "absolute" + SIZE: (640, 640) + SINGLE_CATEGORY_MAX_AREA: 1.0 + COLOR_AUG_SSD: True + SIZE_DIVISIBILITY: 640 # used in dataset mapper + FORMAT: "RGB" +TEST: + DETECTIONS_PER_IMAGE: 250 + EVAL_PERIOD: 5000 + AUG: + ENABLED: False + MIN_SIZES: [320, 480, 640, 800, 960, 1120] + MAX_SIZE: 4480 + FLIP: True diff --git a/annotator/oneformer/configs/coco/Base-COCO-UnifiedSegmentation.yaml b/annotator/oneformer/configs/coco/Base-COCO-UnifiedSegmentation.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ccd24f348f9bc7d60dcdc4b74d887708e57cb8a8 --- /dev/null +++ b/annotator/oneformer/configs/coco/Base-COCO-UnifiedSegmentation.yaml @@ -0,0 +1,54 @@ +MODEL: + BACKBONE: + FREEZE_AT: 0 + NAME: "build_resnet_backbone" + WEIGHTS: "detectron2://ImageNetPretrained/torchvision/R-50.pkl" + PIXEL_MEAN: [123.675, 116.280, 103.530] + PIXEL_STD: [58.395, 57.120, 57.375] + RESNETS: + DEPTH: 50 + STEM_TYPE: "basic" # not used + STEM_OUT_CHANNELS: 64 + STRIDE_IN_1X1: False + OUT_FEATURES: ["res2", "res3", "res4", "res5"] + # NORM: "SyncBN" + RES5_MULTI_GRID: [1, 1, 1] # not used +DATASETS: + TRAIN: ("coco_2017_train_panoptic_with_sem_seg",) + TEST_PANOPTIC: ("coco_2017_val_panoptic_with_sem_seg",) # to evaluate instance and semantic performance as well + TEST_INSTANCE: ("coco_2017_val",) + TEST_SEMANTIC: ("coco_2017_val_panoptic_with_sem_seg",) +SOLVER: + IMS_PER_BATCH: 16 + BASE_LR: 0.0001 + STEPS: (327778, 355092) + MAX_ITER: 368750 + WARMUP_FACTOR: 1.0 + WARMUP_ITERS: 10 + WEIGHT_DECAY: 0.05 + OPTIMIZER: "ADAMW" + BACKBONE_MULTIPLIER: 0.1 + CLIP_GRADIENTS: + ENABLED: True + CLIP_TYPE: "full_model" + CLIP_VALUE: 0.01 + NORM_TYPE: 2.0 + AMP: + ENABLED: True +INPUT: + IMAGE_SIZE: 1024 + MIN_SCALE: 0.1 + MAX_SCALE: 2.0 + FORMAT: "RGB" + DATASET_MAPPER_NAME: "coco_unified_lsj" + MAX_SEQ_LEN: 77 + TASK_SEQ_LEN: 77 + TASK_PROB: + SEMANTIC: 0.33 + INSTANCE: 0.66 +TEST: + EVAL_PERIOD: 5000 +DATALOADER: + FILTER_EMPTY_ANNOTATIONS: True + NUM_WORKERS: 4 +VERSION: 2 diff --git a/annotator/oneformer/configs/coco/oneformer_R50_bs16_50ep.yaml b/annotator/oneformer/configs/coco/oneformer_R50_bs16_50ep.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f768c8fa8b5e4fc1121e65e050053e0d8870cd73 --- /dev/null +++ b/annotator/oneformer/configs/coco/oneformer_R50_bs16_50ep.yaml @@ -0,0 +1,59 @@ +_BASE_: Base-COCO-UnifiedSegmentation.yaml +MODEL: + META_ARCHITECTURE: "OneFormer" + SEM_SEG_HEAD: + NAME: "OneFormerHead" + IGNORE_VALUE: 255 + NUM_CLASSES: 133 + LOSS_WEIGHT: 1.0 + CONVS_DIM: 256 + MASK_DIM: 256 + NORM: "GN" + # pixel decoder + PIXEL_DECODER_NAME: "MSDeformAttnPixelDecoder" + IN_FEATURES: ["res2", "res3", "res4", "res5"] + DEFORMABLE_TRANSFORMER_ENCODER_IN_FEATURES: ["res3", "res4", "res5"] + COMMON_STRIDE: 4 + TRANSFORMER_ENC_LAYERS: 6 + ONE_FORMER: + TRANSFORMER_DECODER_NAME: "ContrastiveMultiScaleMaskedTransformerDecoder" + TRANSFORMER_IN_FEATURE: "multi_scale_pixel_decoder" + DEEP_SUPERVISION: True + NO_OBJECT_WEIGHT: 0.1 + CLASS_WEIGHT: 2.0 + MASK_WEIGHT: 5.0 + DICE_WEIGHT: 5.0 + CONTRASTIVE_WEIGHT: 0.5 + CONTRASTIVE_TEMPERATURE: 0.07 + HIDDEN_DIM: 256 + NUM_OBJECT_QUERIES: 150 + USE_TASK_NORM: True + NHEADS: 8 + DROPOUT: 0.1 + DIM_FEEDFORWARD: 2048 + ENC_LAYERS: 0 + PRE_NORM: False + ENFORCE_INPUT_PROJ: False + SIZE_DIVISIBILITY: 32 + CLASS_DEC_LAYERS: 2 + DEC_LAYERS: 10 # 9 decoder layers, add one for the loss on learnable query + TRAIN_NUM_POINTS: 12544 + OVERSAMPLE_RATIO: 3.0 + IMPORTANCE_SAMPLE_RATIO: 0.75 + TEXT_ENCODER: + WIDTH: 256 + CONTEXT_LENGTH: 77 + NUM_LAYERS: 6 + VOCAB_SIZE: 49408 + PROJ_NUM_LAYERS: 2 + N_CTX: 16 + TEST: + SEMANTIC_ON: True + INSTANCE_ON: True + PANOPTIC_ON: True + DETECTION_ON: False + OVERLAP_THRESHOLD: 0.8 + OBJECT_MASK_THRESHOLD: 0.8 + TASK: "panoptic" +TEST: + DETECTIONS_PER_IMAGE: 150 diff --git a/annotator/oneformer/configs/coco/oneformer_swin_large_IN21k_384_bs16_100ep.yaml b/annotator/oneformer/configs/coco/oneformer_swin_large_IN21k_384_bs16_100ep.yaml new file mode 100644 index 0000000000000000000000000000000000000000..faae655317c52d90b9f756417f8b1a1adcbe78f2 --- /dev/null +++ b/annotator/oneformer/configs/coco/oneformer_swin_large_IN21k_384_bs16_100ep.yaml @@ -0,0 +1,25 @@ +_BASE_: oneformer_R50_bs16_50ep.yaml +MODEL: + BACKBONE: + NAME: "D2SwinTransformer" + SWIN: + EMBED_DIM: 192 + DEPTHS: [2, 2, 18, 2] + NUM_HEADS: [6, 12, 24, 48] + WINDOW_SIZE: 12 + APE: False + DROP_PATH_RATE: 0.3 + PATCH_NORM: True + PRETRAIN_IMG_SIZE: 384 + WEIGHTS: "swin_large_patch4_window12_384_22k.pkl" + PIXEL_MEAN: [123.675, 116.280, 103.530] + PIXEL_STD: [58.395, 57.120, 57.375] + ONE_FORMER: + NUM_OBJECT_QUERIES: 150 +SOLVER: + STEPS: (655556, 735184) + MAX_ITER: 737500 + AMP: + ENABLED: False +TEST: + DETECTIONS_PER_IMAGE: 150 diff --git a/annotator/oneformer/detectron2/__init__.py b/annotator/oneformer/detectron2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bdd994b49294485c27610772f97f177741f5518f --- /dev/null +++ b/annotator/oneformer/detectron2/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from .utils.env import setup_environment + +setup_environment() + + +# This line will be programatically read/write by setup.py. +# Leave them at the bottom of this file and don't touch them. +__version__ = "0.6" diff --git a/annotator/oneformer/detectron2/checkpoint/__init__.py b/annotator/oneformer/detectron2/checkpoint/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..99da0469ae7e169d8970e4b642fed3f870076860 --- /dev/null +++ b/annotator/oneformer/detectron2/checkpoint/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. +# File: + + +from . import catalog as _UNUSED # register the handler +from .detection_checkpoint import DetectionCheckpointer +from fvcore.common.checkpoint import Checkpointer, PeriodicCheckpointer + +__all__ = ["Checkpointer", "PeriodicCheckpointer", "DetectionCheckpointer"] diff --git a/annotator/oneformer/detectron2/checkpoint/c2_model_loading.py b/annotator/oneformer/detectron2/checkpoint/c2_model_loading.py new file mode 100644 index 0000000000000000000000000000000000000000..c6de2a3c830089aa7a0d27df96bb4a45fc5a7b0d --- /dev/null +++ b/annotator/oneformer/detectron2/checkpoint/c2_model_loading.py @@ -0,0 +1,412 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +import logging +import re +from typing import Dict, List +import torch +from tabulate import tabulate + + +def convert_basic_c2_names(original_keys): + """ + Apply some basic name conversion to names in C2 weights. + It only deals with typical backbone models. + + Args: + original_keys (list[str]): + Returns: + list[str]: The same number of strings matching those in original_keys. + """ + layer_keys = copy.deepcopy(original_keys) + layer_keys = [ + {"pred_b": "linear_b", "pred_w": "linear_w"}.get(k, k) for k in layer_keys + ] # some hard-coded mappings + + layer_keys = [k.replace("_", ".") for k in layer_keys] + layer_keys = [re.sub("\\.b$", ".bias", k) for k in layer_keys] + layer_keys = [re.sub("\\.w$", ".weight", k) for k in layer_keys] + # Uniform both bn and gn names to "norm" + layer_keys = [re.sub("bn\\.s$", "norm.weight", k) for k in layer_keys] + layer_keys = [re.sub("bn\\.bias$", "norm.bias", k) for k in layer_keys] + layer_keys = [re.sub("bn\\.rm", "norm.running_mean", k) for k in layer_keys] + layer_keys = [re.sub("bn\\.running.mean$", "norm.running_mean", k) for k in layer_keys] + layer_keys = [re.sub("bn\\.riv$", "norm.running_var", k) for k in layer_keys] + layer_keys = [re.sub("bn\\.running.var$", "norm.running_var", k) for k in layer_keys] + layer_keys = [re.sub("bn\\.gamma$", "norm.weight", k) for k in layer_keys] + layer_keys = [re.sub("bn\\.beta$", "norm.bias", k) for k in layer_keys] + layer_keys = [re.sub("gn\\.s$", "norm.weight", k) for k in layer_keys] + layer_keys = [re.sub("gn\\.bias$", "norm.bias", k) for k in layer_keys] + + # stem + layer_keys = [re.sub("^res\\.conv1\\.norm\\.", "conv1.norm.", k) for k in layer_keys] + # to avoid mis-matching with "conv1" in other components (e.g. detection head) + layer_keys = [re.sub("^conv1\\.", "stem.conv1.", k) for k in layer_keys] + + # layer1-4 is used by torchvision, however we follow the C2 naming strategy (res2-5) + # layer_keys = [re.sub("^res2.", "layer1.", k) for k in layer_keys] + # layer_keys = [re.sub("^res3.", "layer2.", k) for k in layer_keys] + # layer_keys = [re.sub("^res4.", "layer3.", k) for k in layer_keys] + # layer_keys = [re.sub("^res5.", "layer4.", k) for k in layer_keys] + + # blocks + layer_keys = [k.replace(".branch1.", ".shortcut.") for k in layer_keys] + layer_keys = [k.replace(".branch2a.", ".conv1.") for k in layer_keys] + layer_keys = [k.replace(".branch2b.", ".conv2.") for k in layer_keys] + layer_keys = [k.replace(".branch2c.", ".conv3.") for k in layer_keys] + + # DensePose substitutions + layer_keys = [re.sub("^body.conv.fcn", "body_conv_fcn", k) for k in layer_keys] + layer_keys = [k.replace("AnnIndex.lowres", "ann_index_lowres") for k in layer_keys] + layer_keys = [k.replace("Index.UV.lowres", "index_uv_lowres") for k in layer_keys] + layer_keys = [k.replace("U.lowres", "u_lowres") for k in layer_keys] + layer_keys = [k.replace("V.lowres", "v_lowres") for k in layer_keys] + return layer_keys + + +def convert_c2_detectron_names(weights): + """ + Map Caffe2 Detectron weight names to Detectron2 names. + + Args: + weights (dict): name -> tensor + + Returns: + dict: detectron2 names -> tensor + dict: detectron2 names -> C2 names + """ + logger = logging.getLogger(__name__) + logger.info("Renaming Caffe2 weights ......") + original_keys = sorted(weights.keys()) + layer_keys = copy.deepcopy(original_keys) + + layer_keys = convert_basic_c2_names(layer_keys) + + # -------------------------------------------------------------------------- + # RPN hidden representation conv + # -------------------------------------------------------------------------- + # FPN case + # In the C2 model, the RPN hidden layer conv is defined for FPN level 2 and then + # shared for all other levels, hence the appearance of "fpn2" + layer_keys = [ + k.replace("conv.rpn.fpn2", "proposal_generator.rpn_head.conv") for k in layer_keys + ] + # Non-FPN case + layer_keys = [k.replace("conv.rpn", "proposal_generator.rpn_head.conv") for k in layer_keys] + + # -------------------------------------------------------------------------- + # RPN box transformation conv + # -------------------------------------------------------------------------- + # FPN case (see note above about "fpn2") + layer_keys = [ + k.replace("rpn.bbox.pred.fpn2", "proposal_generator.rpn_head.anchor_deltas") + for k in layer_keys + ] + layer_keys = [ + k.replace("rpn.cls.logits.fpn2", "proposal_generator.rpn_head.objectness_logits") + for k in layer_keys + ] + # Non-FPN case + layer_keys = [ + k.replace("rpn.bbox.pred", "proposal_generator.rpn_head.anchor_deltas") for k in layer_keys + ] + layer_keys = [ + k.replace("rpn.cls.logits", "proposal_generator.rpn_head.objectness_logits") + for k in layer_keys + ] + + # -------------------------------------------------------------------------- + # Fast R-CNN box head + # -------------------------------------------------------------------------- + layer_keys = [re.sub("^bbox\\.pred", "bbox_pred", k) for k in layer_keys] + layer_keys = [re.sub("^cls\\.score", "cls_score", k) for k in layer_keys] + layer_keys = [re.sub("^fc6\\.", "box_head.fc1.", k) for k in layer_keys] + layer_keys = [re.sub("^fc7\\.", "box_head.fc2.", k) for k in layer_keys] + # 4conv1fc head tensor names: head_conv1_w, head_conv1_gn_s + layer_keys = [re.sub("^head\\.conv", "box_head.conv", k) for k in layer_keys] + + # -------------------------------------------------------------------------- + # FPN lateral and output convolutions + # -------------------------------------------------------------------------- + def fpn_map(name): + """ + Look for keys with the following patterns: + 1) Starts with "fpn.inner." + Example: "fpn.inner.res2.2.sum.lateral.weight" + Meaning: These are lateral pathway convolutions + 2) Starts with "fpn.res" + Example: "fpn.res2.2.sum.weight" + Meaning: These are FPN output convolutions + """ + splits = name.split(".") + norm = ".norm" if "norm" in splits else "" + if name.startswith("fpn.inner."): + # splits example: ['fpn', 'inner', 'res2', '2', 'sum', 'lateral', 'weight'] + stage = int(splits[2][len("res") :]) + return "fpn_lateral{}{}.{}".format(stage, norm, splits[-1]) + elif name.startswith("fpn.res"): + # splits example: ['fpn', 'res2', '2', 'sum', 'weight'] + stage = int(splits[1][len("res") :]) + return "fpn_output{}{}.{}".format(stage, norm, splits[-1]) + return name + + layer_keys = [fpn_map(k) for k in layer_keys] + + # -------------------------------------------------------------------------- + # Mask R-CNN mask head + # -------------------------------------------------------------------------- + # roi_heads.StandardROIHeads case + layer_keys = [k.replace(".[mask].fcn", "mask_head.mask_fcn") for k in layer_keys] + layer_keys = [re.sub("^\\.mask\\.fcn", "mask_head.mask_fcn", k) for k in layer_keys] + layer_keys = [k.replace("mask.fcn.logits", "mask_head.predictor") for k in layer_keys] + # roi_heads.Res5ROIHeads case + layer_keys = [k.replace("conv5.mask", "mask_head.deconv") for k in layer_keys] + + # -------------------------------------------------------------------------- + # Keypoint R-CNN head + # -------------------------------------------------------------------------- + # interestingly, the keypoint head convs have blob names that are simply "conv_fcnX" + layer_keys = [k.replace("conv.fcn", "roi_heads.keypoint_head.conv_fcn") for k in layer_keys] + layer_keys = [ + k.replace("kps.score.lowres", "roi_heads.keypoint_head.score_lowres") for k in layer_keys + ] + layer_keys = [k.replace("kps.score.", "roi_heads.keypoint_head.score.") for k in layer_keys] + + # -------------------------------------------------------------------------- + # Done with replacements + # -------------------------------------------------------------------------- + assert len(set(layer_keys)) == len(layer_keys) + assert len(original_keys) == len(layer_keys) + + new_weights = {} + new_keys_to_original_keys = {} + for orig, renamed in zip(original_keys, layer_keys): + new_keys_to_original_keys[renamed] = orig + if renamed.startswith("bbox_pred.") or renamed.startswith("mask_head.predictor."): + # remove the meaningless prediction weight for background class + new_start_idx = 4 if renamed.startswith("bbox_pred.") else 1 + new_weights[renamed] = weights[orig][new_start_idx:] + logger.info( + "Remove prediction weight for background class in {}. The shape changes from " + "{} to {}.".format( + renamed, tuple(weights[orig].shape), tuple(new_weights[renamed].shape) + ) + ) + elif renamed.startswith("cls_score."): + # move weights of bg class from original index 0 to last index + logger.info( + "Move classification weights for background class in {} from index 0 to " + "index {}.".format(renamed, weights[orig].shape[0] - 1) + ) + new_weights[renamed] = torch.cat([weights[orig][1:], weights[orig][:1]]) + else: + new_weights[renamed] = weights[orig] + + return new_weights, new_keys_to_original_keys + + +# Note the current matching is not symmetric. +# it assumes model_state_dict will have longer names. +def align_and_update_state_dicts(model_state_dict, ckpt_state_dict, c2_conversion=True): + """ + Match names between the two state-dict, and returns a new chkpt_state_dict with names + converted to match model_state_dict with heuristics. The returned dict can be later + loaded with fvcore checkpointer. + If `c2_conversion==True`, `ckpt_state_dict` is assumed to be a Caffe2 + model and will be renamed at first. + + Strategy: suppose that the models that we will create will have prefixes appended + to each of its keys, for example due to an extra level of nesting that the original + pre-trained weights from ImageNet won't contain. For example, model.state_dict() + might return backbone[0].body.res2.conv1.weight, while the pre-trained model contains + res2.conv1.weight. We thus want to match both parameters together. + For that, we look for each model weight, look among all loaded keys if there is one + that is a suffix of the current weight name, and use it if that's the case. + If multiple matches exist, take the one with longest size + of the corresponding name. For example, for the same model as before, the pretrained + weight file can contain both res2.conv1.weight, as well as conv1.weight. In this case, + we want to match backbone[0].body.conv1.weight to conv1.weight, and + backbone[0].body.res2.conv1.weight to res2.conv1.weight. + """ + model_keys = sorted(model_state_dict.keys()) + if c2_conversion: + ckpt_state_dict, original_keys = convert_c2_detectron_names(ckpt_state_dict) + # original_keys: the name in the original dict (before renaming) + else: + original_keys = {x: x for x in ckpt_state_dict.keys()} + ckpt_keys = sorted(ckpt_state_dict.keys()) + + def match(a, b): + # Matched ckpt_key should be a complete (starts with '.') suffix. + # For example, roi_heads.mesh_head.whatever_conv1 does not match conv1, + # but matches whatever_conv1 or mesh_head.whatever_conv1. + return a == b or a.endswith("." + b) + + # get a matrix of string matches, where each (i, j) entry correspond to the size of the + # ckpt_key string, if it matches + match_matrix = [len(j) if match(i, j) else 0 for i in model_keys for j in ckpt_keys] + match_matrix = torch.as_tensor(match_matrix).view(len(model_keys), len(ckpt_keys)) + # use the matched one with longest size in case of multiple matches + max_match_size, idxs = match_matrix.max(1) + # remove indices that correspond to no-match + idxs[max_match_size == 0] = -1 + + logger = logging.getLogger(__name__) + # matched_pairs (matched checkpoint key --> matched model key) + matched_keys = {} + result_state_dict = {} + for idx_model, idx_ckpt in enumerate(idxs.tolist()): + if idx_ckpt == -1: + continue + key_model = model_keys[idx_model] + key_ckpt = ckpt_keys[idx_ckpt] + value_ckpt = ckpt_state_dict[key_ckpt] + shape_in_model = model_state_dict[key_model].shape + + if shape_in_model != value_ckpt.shape: + logger.warning( + "Shape of {} in checkpoint is {}, while shape of {} in model is {}.".format( + key_ckpt, value_ckpt.shape, key_model, shape_in_model + ) + ) + logger.warning( + "{} will not be loaded. Please double check and see if this is desired.".format( + key_ckpt + ) + ) + continue + + assert key_model not in result_state_dict + result_state_dict[key_model] = value_ckpt + if key_ckpt in matched_keys: # already added to matched_keys + logger.error( + "Ambiguity found for {} in checkpoint!" + "It matches at least two keys in the model ({} and {}).".format( + key_ckpt, key_model, matched_keys[key_ckpt] + ) + ) + raise ValueError("Cannot match one checkpoint key to multiple keys in the model.") + + matched_keys[key_ckpt] = key_model + + # logging: + matched_model_keys = sorted(matched_keys.values()) + if len(matched_model_keys) == 0: + logger.warning("No weights in checkpoint matched with model.") + return ckpt_state_dict + common_prefix = _longest_common_prefix(matched_model_keys) + rev_matched_keys = {v: k for k, v in matched_keys.items()} + original_keys = {k: original_keys[rev_matched_keys[k]] for k in matched_model_keys} + + model_key_groups = _group_keys_by_module(matched_model_keys, original_keys) + table = [] + memo = set() + for key_model in matched_model_keys: + if key_model in memo: + continue + if key_model in model_key_groups: + group = model_key_groups[key_model] + memo |= set(group) + shapes = [tuple(model_state_dict[k].shape) for k in group] + table.append( + ( + _longest_common_prefix([k[len(common_prefix) :] for k in group]) + "*", + _group_str([original_keys[k] for k in group]), + " ".join([str(x).replace(" ", "") for x in shapes]), + ) + ) + else: + key_checkpoint = original_keys[key_model] + shape = str(tuple(model_state_dict[key_model].shape)) + table.append((key_model[len(common_prefix) :], key_checkpoint, shape)) + table_str = tabulate( + table, tablefmt="pipe", headers=["Names in Model", "Names in Checkpoint", "Shapes"] + ) + logger.info( + "Following weights matched with " + + (f"submodule {common_prefix[:-1]}" if common_prefix else "model") + + ":\n" + + table_str + ) + + unmatched_ckpt_keys = [k for k in ckpt_keys if k not in set(matched_keys.keys())] + for k in unmatched_ckpt_keys: + result_state_dict[k] = ckpt_state_dict[k] + return result_state_dict + + +def _group_keys_by_module(keys: List[str], original_names: Dict[str, str]): + """ + Params in the same submodule are grouped together. + + Args: + keys: names of all parameters + original_names: mapping from parameter name to their name in the checkpoint + + Returns: + dict[name -> all other names in the same group] + """ + + def _submodule_name(key): + pos = key.rfind(".") + if pos < 0: + return None + prefix = key[: pos + 1] + return prefix + + all_submodules = [_submodule_name(k) for k in keys] + all_submodules = [x for x in all_submodules if x] + all_submodules = sorted(all_submodules, key=len) + + ret = {} + for prefix in all_submodules: + group = [k for k in keys if k.startswith(prefix)] + if len(group) <= 1: + continue + original_name_lcp = _longest_common_prefix_str([original_names[k] for k in group]) + if len(original_name_lcp) == 0: + # don't group weights if original names don't share prefix + continue + + for k in group: + if k in ret: + continue + ret[k] = group + return ret + + +def _longest_common_prefix(names: List[str]) -> str: + """ + ["abc.zfg", "abc.zef"] -> "abc." + """ + names = [n.split(".") for n in names] + m1, m2 = min(names), max(names) + ret = [a for a, b in zip(m1, m2) if a == b] + ret = ".".join(ret) + "." if len(ret) else "" + return ret + + +def _longest_common_prefix_str(names: List[str]) -> str: + m1, m2 = min(names), max(names) + lcp = [] + for a, b in zip(m1, m2): + if a == b: + lcp.append(a) + else: + break + lcp = "".join(lcp) + return lcp + + +def _group_str(names: List[str]) -> str: + """ + Turn "common1", "common2", "common3" into "common{1,2,3}" + """ + lcp = _longest_common_prefix_str(names) + rest = [x[len(lcp) :] for x in names] + rest = "{" + ",".join(rest) + "}" + ret = lcp + rest + + # add some simplification for BN specifically + ret = ret.replace("bn_{beta,running_mean,running_var,gamma}", "bn_*") + ret = ret.replace("bn_beta,bn_running_mean,bn_running_var,bn_gamma", "bn_*") + return ret diff --git a/annotator/oneformer/detectron2/checkpoint/catalog.py b/annotator/oneformer/detectron2/checkpoint/catalog.py new file mode 100644 index 0000000000000000000000000000000000000000..b5641858fea4936ad10b07a4237faba78dda77ff --- /dev/null +++ b/annotator/oneformer/detectron2/checkpoint/catalog.py @@ -0,0 +1,115 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging + +from annotator.oneformer.detectron2.utils.file_io import PathHandler, PathManager + + +class ModelCatalog(object): + """ + Store mappings from names to third-party models. + """ + + S3_C2_DETECTRON_PREFIX = "https://dl.fbaipublicfiles.com/detectron" + + # MSRA models have STRIDE_IN_1X1=True. False otherwise. + # NOTE: all BN models here have fused BN into an affine layer. + # As a result, you should only load them to a model with "FrozenBN". + # Loading them to a model with regular BN or SyncBN is wrong. + # Even when loaded to FrozenBN, it is still different from affine by an epsilon, + # which should be negligible for training. + # NOTE: all models here uses PIXEL_STD=[1,1,1] + # NOTE: Most of the BN models here are no longer used. We use the + # re-converted pre-trained models under detectron2 model zoo instead. + C2_IMAGENET_MODELS = { + "MSRA/R-50": "ImageNetPretrained/MSRA/R-50.pkl", + "MSRA/R-101": "ImageNetPretrained/MSRA/R-101.pkl", + "FAIR/R-50-GN": "ImageNetPretrained/47261647/R-50-GN.pkl", + "FAIR/R-101-GN": "ImageNetPretrained/47592356/R-101-GN.pkl", + "FAIR/X-101-32x8d": "ImageNetPretrained/20171220/X-101-32x8d.pkl", + "FAIR/X-101-64x4d": "ImageNetPretrained/FBResNeXt/X-101-64x4d.pkl", + "FAIR/X-152-32x8d-IN5k": "ImageNetPretrained/25093814/X-152-32x8d-IN5k.pkl", + } + + C2_DETECTRON_PATH_FORMAT = ( + "{prefix}/{url}/output/train/{dataset}/{type}/model_final.pkl" # noqa B950 + ) + + C2_DATASET_COCO = "coco_2014_train%3Acoco_2014_valminusminival" + C2_DATASET_COCO_KEYPOINTS = "keypoints_coco_2014_train%3Akeypoints_coco_2014_valminusminival" + + # format: {model_name} -> part of the url + C2_DETECTRON_MODELS = { + "35857197/e2e_faster_rcnn_R-50-C4_1x": "35857197/12_2017_baselines/e2e_faster_rcnn_R-50-C4_1x.yaml.01_33_49.iAX0mXvW", # noqa B950 + "35857345/e2e_faster_rcnn_R-50-FPN_1x": "35857345/12_2017_baselines/e2e_faster_rcnn_R-50-FPN_1x.yaml.01_36_30.cUF7QR7I", # noqa B950 + "35857890/e2e_faster_rcnn_R-101-FPN_1x": "35857890/12_2017_baselines/e2e_faster_rcnn_R-101-FPN_1x.yaml.01_38_50.sNxI7sX7", # noqa B950 + "36761737/e2e_faster_rcnn_X-101-32x8d-FPN_1x": "36761737/12_2017_baselines/e2e_faster_rcnn_X-101-32x8d-FPN_1x.yaml.06_31_39.5MIHi1fZ", # noqa B950 + "35858791/e2e_mask_rcnn_R-50-C4_1x": "35858791/12_2017_baselines/e2e_mask_rcnn_R-50-C4_1x.yaml.01_45_57.ZgkA7hPB", # noqa B950 + "35858933/e2e_mask_rcnn_R-50-FPN_1x": "35858933/12_2017_baselines/e2e_mask_rcnn_R-50-FPN_1x.yaml.01_48_14.DzEQe4wC", # noqa B950 + "35861795/e2e_mask_rcnn_R-101-FPN_1x": "35861795/12_2017_baselines/e2e_mask_rcnn_R-101-FPN_1x.yaml.02_31_37.KqyEK4tT", # noqa B950 + "36761843/e2e_mask_rcnn_X-101-32x8d-FPN_1x": "36761843/12_2017_baselines/e2e_mask_rcnn_X-101-32x8d-FPN_1x.yaml.06_35_59.RZotkLKI", # noqa B950 + "48616381/e2e_mask_rcnn_R-50-FPN_2x_gn": "GN/48616381/04_2018_gn_baselines/e2e_mask_rcnn_R-50-FPN_2x_gn_0416.13_23_38.bTlTI97Q", # noqa B950 + "37697547/e2e_keypoint_rcnn_R-50-FPN_1x": "37697547/12_2017_baselines/e2e_keypoint_rcnn_R-50-FPN_1x.yaml.08_42_54.kdzV35ao", # noqa B950 + "35998355/rpn_R-50-C4_1x": "35998355/12_2017_baselines/rpn_R-50-C4_1x.yaml.08_00_43.njH5oD9L", # noqa B950 + "35998814/rpn_R-50-FPN_1x": "35998814/12_2017_baselines/rpn_R-50-FPN_1x.yaml.08_06_03.Axg0r179", # noqa B950 + "36225147/fast_R-50-FPN_1x": "36225147/12_2017_baselines/fast_rcnn_R-50-FPN_1x.yaml.08_39_09.L3obSdQ2", # noqa B950 + } + + @staticmethod + def get(name): + if name.startswith("Caffe2Detectron/COCO"): + return ModelCatalog._get_c2_detectron_baseline(name) + if name.startswith("ImageNetPretrained/"): + return ModelCatalog._get_c2_imagenet_pretrained(name) + raise RuntimeError("model not present in the catalog: {}".format(name)) + + @staticmethod + def _get_c2_imagenet_pretrained(name): + prefix = ModelCatalog.S3_C2_DETECTRON_PREFIX + name = name[len("ImageNetPretrained/") :] + name = ModelCatalog.C2_IMAGENET_MODELS[name] + url = "/".join([prefix, name]) + return url + + @staticmethod + def _get_c2_detectron_baseline(name): + name = name[len("Caffe2Detectron/COCO/") :] + url = ModelCatalog.C2_DETECTRON_MODELS[name] + if "keypoint_rcnn" in name: + dataset = ModelCatalog.C2_DATASET_COCO_KEYPOINTS + else: + dataset = ModelCatalog.C2_DATASET_COCO + + if "35998355/rpn_R-50-C4_1x" in name: + # this one model is somehow different from others .. + type = "rpn" + else: + type = "generalized_rcnn" + + # Detectron C2 models are stored in the structure defined in `C2_DETECTRON_PATH_FORMAT`. + url = ModelCatalog.C2_DETECTRON_PATH_FORMAT.format( + prefix=ModelCatalog.S3_C2_DETECTRON_PREFIX, url=url, type=type, dataset=dataset + ) + return url + + +class ModelCatalogHandler(PathHandler): + """ + Resolve URL like catalog://. + """ + + PREFIX = "catalog://" + + def _get_supported_prefixes(self): + return [self.PREFIX] + + def _get_local_path(self, path, **kwargs): + logger = logging.getLogger(__name__) + catalog_path = ModelCatalog.get(path[len(self.PREFIX) :]) + logger.info("Catalog entry {} points to {}".format(path, catalog_path)) + return PathManager.get_local_path(catalog_path, **kwargs) + + def _open(self, path, mode="r", **kwargs): + return PathManager.open(self._get_local_path(path), mode, **kwargs) + + +PathManager.register_handler(ModelCatalogHandler()) diff --git a/annotator/oneformer/detectron2/checkpoint/detection_checkpoint.py b/annotator/oneformer/detectron2/checkpoint/detection_checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..7d411e54bd5e004504423ba052db6f85ec511f72 --- /dev/null +++ b/annotator/oneformer/detectron2/checkpoint/detection_checkpoint.py @@ -0,0 +1,145 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import os +import pickle +from urllib.parse import parse_qs, urlparse +import torch +from fvcore.common.checkpoint import Checkpointer +from torch.nn.parallel import DistributedDataParallel + +import annotator.oneformer.detectron2.utils.comm as comm +from annotator.oneformer.detectron2.utils.file_io import PathManager + +from .c2_model_loading import align_and_update_state_dicts + + +class DetectionCheckpointer(Checkpointer): + """ + Same as :class:`Checkpointer`, but is able to: + 1. handle models in detectron & detectron2 model zoo, and apply conversions for legacy models. + 2. correctly load checkpoints that are only available on the master worker + """ + + def __init__(self, model, save_dir="", *, save_to_disk=None, **checkpointables): + is_main_process = comm.is_main_process() + super().__init__( + model, + save_dir, + save_to_disk=is_main_process if save_to_disk is None else save_to_disk, + **checkpointables, + ) + self.path_manager = PathManager + self._parsed_url_during_load = None + + def load(self, path, *args, **kwargs): + assert self._parsed_url_during_load is None + need_sync = False + logger = logging.getLogger(__name__) + logger.info("[DetectionCheckpointer] Loading from {} ...".format(path)) + + if path and isinstance(self.model, DistributedDataParallel): + path = self.path_manager.get_local_path(path) + has_file = os.path.isfile(path) + all_has_file = comm.all_gather(has_file) + if not all_has_file[0]: + raise OSError(f"File {path} not found on main worker.") + if not all(all_has_file): + logger.warning( + f"Not all workers can read checkpoint {path}. " + "Training may fail to fully resume." + ) + # TODO: broadcast the checkpoint file contents from main + # worker, and load from it instead. + need_sync = True + if not has_file: + path = None # don't load if not readable + + if path: + parsed_url = urlparse(path) + self._parsed_url_during_load = parsed_url + path = parsed_url._replace(query="").geturl() # remove query from filename + path = self.path_manager.get_local_path(path) + + self.logger.setLevel('CRITICAL') + ret = super().load(path, *args, **kwargs) + + if need_sync: + logger.info("Broadcasting model states from main worker ...") + self.model._sync_params_and_buffers() + self._parsed_url_during_load = None # reset to None + return ret + + def _load_file(self, filename): + if filename.endswith(".pkl"): + with PathManager.open(filename, "rb") as f: + data = pickle.load(f, encoding="latin1") + if "model" in data and "__author__" in data: + # file is in Detectron2 model zoo format + self.logger.info("Reading a file from '{}'".format(data["__author__"])) + return data + else: + # assume file is from Caffe2 / Detectron1 model zoo + if "blobs" in data: + # Detection models have "blobs", but ImageNet models don't + data = data["blobs"] + data = {k: v for k, v in data.items() if not k.endswith("_momentum")} + return {"model": data, "__author__": "Caffe2", "matching_heuristics": True} + elif filename.endswith(".pyth"): + # assume file is from pycls; no one else seems to use the ".pyth" extension + with PathManager.open(filename, "rb") as f: + data = torch.load(f) + assert ( + "model_state" in data + ), f"Cannot load .pyth file {filename}; pycls checkpoints must contain 'model_state'." + model_state = { + k: v + for k, v in data["model_state"].items() + if not k.endswith("num_batches_tracked") + } + return {"model": model_state, "__author__": "pycls", "matching_heuristics": True} + + loaded = self._torch_load(filename) + if "model" not in loaded: + loaded = {"model": loaded} + assert self._parsed_url_during_load is not None, "`_load_file` must be called inside `load`" + parsed_url = self._parsed_url_during_load + queries = parse_qs(parsed_url.query) + if queries.pop("matching_heuristics", "False") == ["True"]: + loaded["matching_heuristics"] = True + if len(queries) > 0: + raise ValueError( + f"Unsupported query remaining: f{queries}, orginal filename: {parsed_url.geturl()}" + ) + return loaded + + def _torch_load(self, f): + return super()._load_file(f) + + def _load_model(self, checkpoint): + if checkpoint.get("matching_heuristics", False): + self._convert_ndarray_to_tensor(checkpoint["model"]) + # convert weights by name-matching heuristics + checkpoint["model"] = align_and_update_state_dicts( + self.model.state_dict(), + checkpoint["model"], + c2_conversion=checkpoint.get("__author__", None) == "Caffe2", + ) + # for non-caffe2 models, use standard ways to load it + incompatible = super()._load_model(checkpoint) + + model_buffers = dict(self.model.named_buffers(recurse=False)) + for k in ["pixel_mean", "pixel_std"]: + # Ignore missing key message about pixel_mean/std. + # Though they may be missing in old checkpoints, they will be correctly + # initialized from config anyway. + if k in model_buffers: + try: + incompatible.missing_keys.remove(k) + except ValueError: + pass + for k in incompatible.unexpected_keys[:]: + # Ignore unexpected keys about cell anchors. They exist in old checkpoints + # but now they are non-persistent buffers and will not be in new checkpoints. + if "anchor_generator.cell_anchors" in k: + incompatible.unexpected_keys.remove(k) + return incompatible diff --git a/annotator/oneformer/detectron2/config/__init__.py b/annotator/oneformer/detectron2/config/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a78ed118685fcfd869f7a72caf6b94621530196a --- /dev/null +++ b/annotator/oneformer/detectron2/config/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .compat import downgrade_config, upgrade_config +from .config import CfgNode, get_cfg, global_cfg, set_global_cfg, configurable +from .instantiate import instantiate +from .lazy import LazyCall, LazyConfig + +__all__ = [ + "CfgNode", + "get_cfg", + "global_cfg", + "set_global_cfg", + "downgrade_config", + "upgrade_config", + "configurable", + "instantiate", + "LazyCall", + "LazyConfig", +] + + +from annotator.oneformer.detectron2.utils.env import fixup_module_metadata + +fixup_module_metadata(__name__, globals(), __all__) +del fixup_module_metadata diff --git a/annotator/oneformer/detectron2/config/compat.py b/annotator/oneformer/detectron2/config/compat.py new file mode 100644 index 0000000000000000000000000000000000000000..11a08c439bf14defd880e37a938fab8a08e68eeb --- /dev/null +++ b/annotator/oneformer/detectron2/config/compat.py @@ -0,0 +1,229 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +""" +Backward compatibility of configs. + +Instructions to bump version: ++ It's not needed to bump version if new keys are added. + It's only needed when backward-incompatible changes happen + (i.e., some existing keys disappear, or the meaning of a key changes) ++ To bump version, do the following: + 1. Increment _C.VERSION in defaults.py + 2. Add a converter in this file. + + Each ConverterVX has a function "upgrade" which in-place upgrades config from X-1 to X, + and a function "downgrade" which in-place downgrades config from X to X-1 + + In each function, VERSION is left unchanged. + + Each converter assumes that its input has the relevant keys + (i.e., the input is not a partial config). + 3. Run the tests (test_config.py) to make sure the upgrade & downgrade + functions are consistent. +""" + +import logging +from typing import List, Optional, Tuple + +from .config import CfgNode as CN +from .defaults import _C + +__all__ = ["upgrade_config", "downgrade_config"] + + +def upgrade_config(cfg: CN, to_version: Optional[int] = None) -> CN: + """ + Upgrade a config from its current version to a newer version. + + Args: + cfg (CfgNode): + to_version (int): defaults to the latest version. + """ + cfg = cfg.clone() + if to_version is None: + to_version = _C.VERSION + + assert cfg.VERSION <= to_version, "Cannot upgrade from v{} to v{}!".format( + cfg.VERSION, to_version + ) + for k in range(cfg.VERSION, to_version): + converter = globals()["ConverterV" + str(k + 1)] + converter.upgrade(cfg) + cfg.VERSION = k + 1 + return cfg + + +def downgrade_config(cfg: CN, to_version: int) -> CN: + """ + Downgrade a config from its current version to an older version. + + Args: + cfg (CfgNode): + to_version (int): + + Note: + A general downgrade of arbitrary configs is not always possible due to the + different functionalities in different versions. + The purpose of downgrade is only to recover the defaults in old versions, + allowing it to load an old partial yaml config. + Therefore, the implementation only needs to fill in the default values + in the old version when a general downgrade is not possible. + """ + cfg = cfg.clone() + assert cfg.VERSION >= to_version, "Cannot downgrade from v{} to v{}!".format( + cfg.VERSION, to_version + ) + for k in range(cfg.VERSION, to_version, -1): + converter = globals()["ConverterV" + str(k)] + converter.downgrade(cfg) + cfg.VERSION = k - 1 + return cfg + + +def guess_version(cfg: CN, filename: str) -> int: + """ + Guess the version of a partial config where the VERSION field is not specified. + Returns the version, or the latest if cannot make a guess. + + This makes it easier for users to migrate. + """ + logger = logging.getLogger(__name__) + + def _has(name: str) -> bool: + cur = cfg + for n in name.split("."): + if n not in cur: + return False + cur = cur[n] + return True + + # Most users' partial configs have "MODEL.WEIGHT", so guess on it + ret = None + if _has("MODEL.WEIGHT") or _has("TEST.AUG_ON"): + ret = 1 + + if ret is not None: + logger.warning("Config '{}' has no VERSION. Assuming it to be v{}.".format(filename, ret)) + else: + ret = _C.VERSION + logger.warning( + "Config '{}' has no VERSION. Assuming it to be compatible with latest v{}.".format( + filename, ret + ) + ) + return ret + + +def _rename(cfg: CN, old: str, new: str) -> None: + old_keys = old.split(".") + new_keys = new.split(".") + + def _set(key_seq: List[str], val: str) -> None: + cur = cfg + for k in key_seq[:-1]: + if k not in cur: + cur[k] = CN() + cur = cur[k] + cur[key_seq[-1]] = val + + def _get(key_seq: List[str]) -> CN: + cur = cfg + for k in key_seq: + cur = cur[k] + return cur + + def _del(key_seq: List[str]) -> None: + cur = cfg + for k in key_seq[:-1]: + cur = cur[k] + del cur[key_seq[-1]] + if len(cur) == 0 and len(key_seq) > 1: + _del(key_seq[:-1]) + + _set(new_keys, _get(old_keys)) + _del(old_keys) + + +class _RenameConverter: + """ + A converter that handles simple rename. + """ + + RENAME: List[Tuple[str, str]] = [] # list of tuples of (old name, new name) + + @classmethod + def upgrade(cls, cfg: CN) -> None: + for old, new in cls.RENAME: + _rename(cfg, old, new) + + @classmethod + def downgrade(cls, cfg: CN) -> None: + for old, new in cls.RENAME[::-1]: + _rename(cfg, new, old) + + +class ConverterV1(_RenameConverter): + RENAME = [("MODEL.RPN_HEAD.NAME", "MODEL.RPN.HEAD_NAME")] + + +class ConverterV2(_RenameConverter): + """ + A large bulk of rename, before public release. + """ + + RENAME = [ + ("MODEL.WEIGHT", "MODEL.WEIGHTS"), + ("MODEL.PANOPTIC_FPN.SEMANTIC_LOSS_SCALE", "MODEL.SEM_SEG_HEAD.LOSS_WEIGHT"), + ("MODEL.PANOPTIC_FPN.RPN_LOSS_SCALE", "MODEL.RPN.LOSS_WEIGHT"), + ("MODEL.PANOPTIC_FPN.INSTANCE_LOSS_SCALE", "MODEL.PANOPTIC_FPN.INSTANCE_LOSS_WEIGHT"), + ("MODEL.PANOPTIC_FPN.COMBINE_ON", "MODEL.PANOPTIC_FPN.COMBINE.ENABLED"), + ( + "MODEL.PANOPTIC_FPN.COMBINE_OVERLAP_THRESHOLD", + "MODEL.PANOPTIC_FPN.COMBINE.OVERLAP_THRESH", + ), + ( + "MODEL.PANOPTIC_FPN.COMBINE_STUFF_AREA_LIMIT", + "MODEL.PANOPTIC_FPN.COMBINE.STUFF_AREA_LIMIT", + ), + ( + "MODEL.PANOPTIC_FPN.COMBINE_INSTANCES_CONFIDENCE_THRESHOLD", + "MODEL.PANOPTIC_FPN.COMBINE.INSTANCES_CONFIDENCE_THRESH", + ), + ("MODEL.ROI_HEADS.SCORE_THRESH", "MODEL.ROI_HEADS.SCORE_THRESH_TEST"), + ("MODEL.ROI_HEADS.NMS", "MODEL.ROI_HEADS.NMS_THRESH_TEST"), + ("MODEL.RETINANET.INFERENCE_SCORE_THRESHOLD", "MODEL.RETINANET.SCORE_THRESH_TEST"), + ("MODEL.RETINANET.INFERENCE_TOPK_CANDIDATES", "MODEL.RETINANET.TOPK_CANDIDATES_TEST"), + ("MODEL.RETINANET.INFERENCE_NMS_THRESHOLD", "MODEL.RETINANET.NMS_THRESH_TEST"), + ("TEST.DETECTIONS_PER_IMG", "TEST.DETECTIONS_PER_IMAGE"), + ("TEST.AUG_ON", "TEST.AUG.ENABLED"), + ("TEST.AUG_MIN_SIZES", "TEST.AUG.MIN_SIZES"), + ("TEST.AUG_MAX_SIZE", "TEST.AUG.MAX_SIZE"), + ("TEST.AUG_FLIP", "TEST.AUG.FLIP"), + ] + + @classmethod + def upgrade(cls, cfg: CN) -> None: + super().upgrade(cfg) + + if cfg.MODEL.META_ARCHITECTURE == "RetinaNet": + _rename( + cfg, "MODEL.RETINANET.ANCHOR_ASPECT_RATIOS", "MODEL.ANCHOR_GENERATOR.ASPECT_RATIOS" + ) + _rename(cfg, "MODEL.RETINANET.ANCHOR_SIZES", "MODEL.ANCHOR_GENERATOR.SIZES") + del cfg["MODEL"]["RPN"]["ANCHOR_SIZES"] + del cfg["MODEL"]["RPN"]["ANCHOR_ASPECT_RATIOS"] + else: + _rename(cfg, "MODEL.RPN.ANCHOR_ASPECT_RATIOS", "MODEL.ANCHOR_GENERATOR.ASPECT_RATIOS") + _rename(cfg, "MODEL.RPN.ANCHOR_SIZES", "MODEL.ANCHOR_GENERATOR.SIZES") + del cfg["MODEL"]["RETINANET"]["ANCHOR_SIZES"] + del cfg["MODEL"]["RETINANET"]["ANCHOR_ASPECT_RATIOS"] + del cfg["MODEL"]["RETINANET"]["ANCHOR_STRIDES"] + + @classmethod + def downgrade(cls, cfg: CN) -> None: + super().downgrade(cfg) + + _rename(cfg, "MODEL.ANCHOR_GENERATOR.ASPECT_RATIOS", "MODEL.RPN.ANCHOR_ASPECT_RATIOS") + _rename(cfg, "MODEL.ANCHOR_GENERATOR.SIZES", "MODEL.RPN.ANCHOR_SIZES") + cfg.MODEL.RETINANET.ANCHOR_ASPECT_RATIOS = cfg.MODEL.RPN.ANCHOR_ASPECT_RATIOS + cfg.MODEL.RETINANET.ANCHOR_SIZES = cfg.MODEL.RPN.ANCHOR_SIZES + cfg.MODEL.RETINANET.ANCHOR_STRIDES = [] # this is not used anywhere in any version diff --git a/annotator/oneformer/detectron2/config/config.py b/annotator/oneformer/detectron2/config/config.py new file mode 100644 index 0000000000000000000000000000000000000000..c5b1303422481dc7adb3ee5221377770e0c01a81 --- /dev/null +++ b/annotator/oneformer/detectron2/config/config.py @@ -0,0 +1,265 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +import functools +import inspect +import logging +from fvcore.common.config import CfgNode as _CfgNode + +from annotator.oneformer.detectron2.utils.file_io import PathManager + + +class CfgNode(_CfgNode): + """ + The same as `fvcore.common.config.CfgNode`, but different in: + + 1. Use unsafe yaml loading by default. + Note that this may lead to arbitrary code execution: you must not + load a config file from untrusted sources before manually inspecting + the content of the file. + 2. Support config versioning. + When attempting to merge an old config, it will convert the old config automatically. + + .. automethod:: clone + .. automethod:: freeze + .. automethod:: defrost + .. automethod:: is_frozen + .. automethod:: load_yaml_with_base + .. automethod:: merge_from_list + .. automethod:: merge_from_other_cfg + """ + + @classmethod + def _open_cfg(cls, filename): + return PathManager.open(filename, "r") + + # Note that the default value of allow_unsafe is changed to True + def merge_from_file(self, cfg_filename: str, allow_unsafe: bool = True) -> None: + """ + Load content from the given config file and merge it into self. + + Args: + cfg_filename: config filename + allow_unsafe: allow unsafe yaml syntax + """ + assert PathManager.isfile(cfg_filename), f"Config file '{cfg_filename}' does not exist!" + loaded_cfg = self.load_yaml_with_base(cfg_filename, allow_unsafe=allow_unsafe) + loaded_cfg = type(self)(loaded_cfg) + + # defaults.py needs to import CfgNode + from .defaults import _C + + latest_ver = _C.VERSION + assert ( + latest_ver == self.VERSION + ), "CfgNode.merge_from_file is only allowed on a config object of latest version!" + + logger = logging.getLogger(__name__) + + loaded_ver = loaded_cfg.get("VERSION", None) + if loaded_ver is None: + from .compat import guess_version + + loaded_ver = guess_version(loaded_cfg, cfg_filename) + assert loaded_ver <= self.VERSION, "Cannot merge a v{} config into a v{} config.".format( + loaded_ver, self.VERSION + ) + + if loaded_ver == self.VERSION: + self.merge_from_other_cfg(loaded_cfg) + else: + # compat.py needs to import CfgNode + from .compat import upgrade_config, downgrade_config + + logger.warning( + "Loading an old v{} config file '{}' by automatically upgrading to v{}. " + "See docs/CHANGELOG.md for instructions to update your files.".format( + loaded_ver, cfg_filename, self.VERSION + ) + ) + # To convert, first obtain a full config at an old version + old_self = downgrade_config(self, to_version=loaded_ver) + old_self.merge_from_other_cfg(loaded_cfg) + new_config = upgrade_config(old_self) + self.clear() + self.update(new_config) + + def dump(self, *args, **kwargs): + """ + Returns: + str: a yaml string representation of the config + """ + # to make it show up in docs + return super().dump(*args, **kwargs) + + +global_cfg = CfgNode() + + +def get_cfg() -> CfgNode: + """ + Get a copy of the default config. + + Returns: + a detectron2 CfgNode instance. + """ + from .defaults import _C + + return _C.clone() + + +def set_global_cfg(cfg: CfgNode) -> None: + """ + Let the global config point to the given cfg. + + Assume that the given "cfg" has the key "KEY", after calling + `set_global_cfg(cfg)`, the key can be accessed by: + :: + from annotator.oneformer.detectron2.config import global_cfg + print(global_cfg.KEY) + + By using a hacky global config, you can access these configs anywhere, + without having to pass the config object or the values deep into the code. + This is a hacky feature introduced for quick prototyping / research exploration. + """ + global global_cfg + global_cfg.clear() + global_cfg.update(cfg) + + +def configurable(init_func=None, *, from_config=None): + """ + Decorate a function or a class's __init__ method so that it can be called + with a :class:`CfgNode` object using a :func:`from_config` function that translates + :class:`CfgNode` to arguments. + + Examples: + :: + # Usage 1: Decorator on __init__: + class A: + @configurable + def __init__(self, a, b=2, c=3): + pass + + @classmethod + def from_config(cls, cfg): # 'cfg' must be the first argument + # Returns kwargs to be passed to __init__ + return {"a": cfg.A, "b": cfg.B} + + a1 = A(a=1, b=2) # regular construction + a2 = A(cfg) # construct with a cfg + a3 = A(cfg, b=3, c=4) # construct with extra overwrite + + # Usage 2: Decorator on any function. Needs an extra from_config argument: + @configurable(from_config=lambda cfg: {"a: cfg.A, "b": cfg.B}) + def a_func(a, b=2, c=3): + pass + + a1 = a_func(a=1, b=2) # regular call + a2 = a_func(cfg) # call with a cfg + a3 = a_func(cfg, b=3, c=4) # call with extra overwrite + + Args: + init_func (callable): a class's ``__init__`` method in usage 1. The + class must have a ``from_config`` classmethod which takes `cfg` as + the first argument. + from_config (callable): the from_config function in usage 2. It must take `cfg` + as its first argument. + """ + + if init_func is not None: + assert ( + inspect.isfunction(init_func) + and from_config is None + and init_func.__name__ == "__init__" + ), "Incorrect use of @configurable. Check API documentation for examples." + + @functools.wraps(init_func) + def wrapped(self, *args, **kwargs): + try: + from_config_func = type(self).from_config + except AttributeError as e: + raise AttributeError( + "Class with @configurable must have a 'from_config' classmethod." + ) from e + if not inspect.ismethod(from_config_func): + raise TypeError("Class with @configurable must have a 'from_config' classmethod.") + + if _called_with_cfg(*args, **kwargs): + explicit_args = _get_args_from_config(from_config_func, *args, **kwargs) + init_func(self, **explicit_args) + else: + init_func(self, *args, **kwargs) + + return wrapped + + else: + if from_config is None: + return configurable # @configurable() is made equivalent to @configurable + assert inspect.isfunction( + from_config + ), "from_config argument of configurable must be a function!" + + def wrapper(orig_func): + @functools.wraps(orig_func) + def wrapped(*args, **kwargs): + if _called_with_cfg(*args, **kwargs): + explicit_args = _get_args_from_config(from_config, *args, **kwargs) + return orig_func(**explicit_args) + else: + return orig_func(*args, **kwargs) + + wrapped.from_config = from_config + return wrapped + + return wrapper + + +def _get_args_from_config(from_config_func, *args, **kwargs): + """ + Use `from_config` to obtain explicit arguments. + + Returns: + dict: arguments to be used for cls.__init__ + """ + signature = inspect.signature(from_config_func) + if list(signature.parameters.keys())[0] != "cfg": + if inspect.isfunction(from_config_func): + name = from_config_func.__name__ + else: + name = f"{from_config_func.__self__}.from_config" + raise TypeError(f"{name} must take 'cfg' as the first argument!") + support_var_arg = any( + param.kind in [param.VAR_POSITIONAL, param.VAR_KEYWORD] + for param in signature.parameters.values() + ) + if support_var_arg: # forward all arguments to from_config, if from_config accepts them + ret = from_config_func(*args, **kwargs) + else: + # forward supported arguments to from_config + supported_arg_names = set(signature.parameters.keys()) + extra_kwargs = {} + for name in list(kwargs.keys()): + if name not in supported_arg_names: + extra_kwargs[name] = kwargs.pop(name) + ret = from_config_func(*args, **kwargs) + # forward the other arguments to __init__ + ret.update(extra_kwargs) + return ret + + +def _called_with_cfg(*args, **kwargs): + """ + Returns: + bool: whether the arguments contain CfgNode and should be considered + forwarded to from_config. + """ + from omegaconf import DictConfig + + if len(args) and isinstance(args[0], (_CfgNode, DictConfig)): + return True + if isinstance(kwargs.pop("cfg", None), (_CfgNode, DictConfig)): + return True + # `from_config`'s first argument is forced to be "cfg". + # So the above check covers all cases. + return False diff --git a/annotator/oneformer/detectron2/config/defaults.py b/annotator/oneformer/detectron2/config/defaults.py new file mode 100644 index 0000000000000000000000000000000000000000..ffb79e763f076c9ae982c727309e19b8e0ef170f --- /dev/null +++ b/annotator/oneformer/detectron2/config/defaults.py @@ -0,0 +1,650 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .config import CfgNode as CN + +# NOTE: given the new config system +# (https://detectron2.readthedocs.io/en/latest/tutorials/lazyconfigs.html), +# we will stop adding new functionalities to default CfgNode. + +# ----------------------------------------------------------------------------- +# Convention about Training / Test specific parameters +# ----------------------------------------------------------------------------- +# Whenever an argument can be either used for training or for testing, the +# corresponding name will be post-fixed by a _TRAIN for a training parameter, +# or _TEST for a test-specific parameter. +# For example, the number of images during training will be +# IMAGES_PER_BATCH_TRAIN, while the number of images for testing will be +# IMAGES_PER_BATCH_TEST + +# ----------------------------------------------------------------------------- +# Config definition +# ----------------------------------------------------------------------------- + +_C = CN() + +# The version number, to upgrade from old configs to new ones if any +# changes happen. It's recommended to keep a VERSION in your config file. +_C.VERSION = 2 + +_C.MODEL = CN() +_C.MODEL.LOAD_PROPOSALS = False +_C.MODEL.MASK_ON = False +_C.MODEL.KEYPOINT_ON = False +_C.MODEL.DEVICE = "cuda" +_C.MODEL.META_ARCHITECTURE = "GeneralizedRCNN" + +# Path (a file path, or URL like detectron2://.., https://..) to a checkpoint file +# to be loaded to the model. You can find available models in the model zoo. +_C.MODEL.WEIGHTS = "" + +# Values to be used for image normalization (BGR order, since INPUT.FORMAT defaults to BGR). +# To train on images of different number of channels, just set different mean & std. +# Default values are the mean pixel value from ImageNet: [103.53, 116.28, 123.675] +_C.MODEL.PIXEL_MEAN = [103.530, 116.280, 123.675] +# When using pre-trained models in Detectron1 or any MSRA models, +# std has been absorbed into its conv1 weights, so the std needs to be set 1. +# Otherwise, you can use [57.375, 57.120, 58.395] (ImageNet std) +_C.MODEL.PIXEL_STD = [1.0, 1.0, 1.0] + + +# ----------------------------------------------------------------------------- +# INPUT +# ----------------------------------------------------------------------------- +_C.INPUT = CN() +# By default, {MIN,MAX}_SIZE options are used in transforms.ResizeShortestEdge. +# Please refer to ResizeShortestEdge for detailed definition. +# Size of the smallest side of the image during training +_C.INPUT.MIN_SIZE_TRAIN = (800,) +# Sample size of smallest side by choice or random selection from range give by +# INPUT.MIN_SIZE_TRAIN +_C.INPUT.MIN_SIZE_TRAIN_SAMPLING = "choice" +# Maximum size of the side of the image during training +_C.INPUT.MAX_SIZE_TRAIN = 1333 +# Size of the smallest side of the image during testing. Set to zero to disable resize in testing. +_C.INPUT.MIN_SIZE_TEST = 800 +# Maximum size of the side of the image during testing +_C.INPUT.MAX_SIZE_TEST = 1333 +# Mode for flipping images used in data augmentation during training +# choose one of ["horizontal, "vertical", "none"] +_C.INPUT.RANDOM_FLIP = "horizontal" + +# `True` if cropping is used for data augmentation during training +_C.INPUT.CROP = CN({"ENABLED": False}) +# Cropping type. See documentation of `detectron2.data.transforms.RandomCrop` for explanation. +_C.INPUT.CROP.TYPE = "relative_range" +# Size of crop in range (0, 1] if CROP.TYPE is "relative" or "relative_range" and in number of +# pixels if CROP.TYPE is "absolute" +_C.INPUT.CROP.SIZE = [0.9, 0.9] + + +# Whether the model needs RGB, YUV, HSV etc. +# Should be one of the modes defined here, as we use PIL to read the image: +# https://pillow.readthedocs.io/en/stable/handbook/concepts.html#concept-modes +# with BGR being the one exception. One can set image format to BGR, we will +# internally use RGB for conversion and flip the channels over +_C.INPUT.FORMAT = "BGR" +# The ground truth mask format that the model will use. +# Mask R-CNN supports either "polygon" or "bitmask" as ground truth. +_C.INPUT.MASK_FORMAT = "polygon" # alternative: "bitmask" + + +# ----------------------------------------------------------------------------- +# Dataset +# ----------------------------------------------------------------------------- +_C.DATASETS = CN() +# List of the dataset names for training. Must be registered in DatasetCatalog +# Samples from these datasets will be merged and used as one dataset. +_C.DATASETS.TRAIN = () +# List of the pre-computed proposal files for training, which must be consistent +# with datasets listed in DATASETS.TRAIN. +_C.DATASETS.PROPOSAL_FILES_TRAIN = () +# Number of top scoring precomputed proposals to keep for training +_C.DATASETS.PRECOMPUTED_PROPOSAL_TOPK_TRAIN = 2000 +# List of the dataset names for testing. Must be registered in DatasetCatalog +_C.DATASETS.TEST = () +# List of the pre-computed proposal files for test, which must be consistent +# with datasets listed in DATASETS.TEST. +_C.DATASETS.PROPOSAL_FILES_TEST = () +# Number of top scoring precomputed proposals to keep for test +_C.DATASETS.PRECOMPUTED_PROPOSAL_TOPK_TEST = 1000 + +# ----------------------------------------------------------------------------- +# DataLoader +# ----------------------------------------------------------------------------- +_C.DATALOADER = CN() +# Number of data loading threads +_C.DATALOADER.NUM_WORKERS = 4 +# If True, each batch should contain only images for which the aspect ratio +# is compatible. This groups portrait images together, and landscape images +# are not batched with portrait images. +_C.DATALOADER.ASPECT_RATIO_GROUPING = True +# Options: TrainingSampler, RepeatFactorTrainingSampler +_C.DATALOADER.SAMPLER_TRAIN = "TrainingSampler" +# Repeat threshold for RepeatFactorTrainingSampler +_C.DATALOADER.REPEAT_THRESHOLD = 0.0 +# Tf True, when working on datasets that have instance annotations, the +# training dataloader will filter out images without associated annotations +_C.DATALOADER.FILTER_EMPTY_ANNOTATIONS = True + +# ---------------------------------------------------------------------------- # +# Backbone options +# ---------------------------------------------------------------------------- # +_C.MODEL.BACKBONE = CN() + +_C.MODEL.BACKBONE.NAME = "build_resnet_backbone" +# Freeze the first several stages so they are not trained. +# There are 5 stages in ResNet. The first is a convolution, and the following +# stages are each group of residual blocks. +_C.MODEL.BACKBONE.FREEZE_AT = 2 + + +# ---------------------------------------------------------------------------- # +# FPN options +# ---------------------------------------------------------------------------- # +_C.MODEL.FPN = CN() +# Names of the input feature maps to be used by FPN +# They must have contiguous power of 2 strides +# e.g., ["res2", "res3", "res4", "res5"] +_C.MODEL.FPN.IN_FEATURES = [] +_C.MODEL.FPN.OUT_CHANNELS = 256 + +# Options: "" (no norm), "GN" +_C.MODEL.FPN.NORM = "" + +# Types for fusing the FPN top-down and lateral features. Can be either "sum" or "avg" +_C.MODEL.FPN.FUSE_TYPE = "sum" + + +# ---------------------------------------------------------------------------- # +# Proposal generator options +# ---------------------------------------------------------------------------- # +_C.MODEL.PROPOSAL_GENERATOR = CN() +# Current proposal generators include "RPN", "RRPN" and "PrecomputedProposals" +_C.MODEL.PROPOSAL_GENERATOR.NAME = "RPN" +# Proposal height and width both need to be greater than MIN_SIZE +# (a the scale used during training or inference) +_C.MODEL.PROPOSAL_GENERATOR.MIN_SIZE = 0 + + +# ---------------------------------------------------------------------------- # +# Anchor generator options +# ---------------------------------------------------------------------------- # +_C.MODEL.ANCHOR_GENERATOR = CN() +# The generator can be any name in the ANCHOR_GENERATOR registry +_C.MODEL.ANCHOR_GENERATOR.NAME = "DefaultAnchorGenerator" +# Anchor sizes (i.e. sqrt of area) in absolute pixels w.r.t. the network input. +# Format: list[list[float]]. SIZES[i] specifies the list of sizes to use for +# IN_FEATURES[i]; len(SIZES) must be equal to len(IN_FEATURES) or 1. +# When len(SIZES) == 1, SIZES[0] is used for all IN_FEATURES. +_C.MODEL.ANCHOR_GENERATOR.SIZES = [[32, 64, 128, 256, 512]] +# Anchor aspect ratios. For each area given in `SIZES`, anchors with different aspect +# ratios are generated by an anchor generator. +# Format: list[list[float]]. ASPECT_RATIOS[i] specifies the list of aspect ratios (H/W) +# to use for IN_FEATURES[i]; len(ASPECT_RATIOS) == len(IN_FEATURES) must be true, +# or len(ASPECT_RATIOS) == 1 is true and aspect ratio list ASPECT_RATIOS[0] is used +# for all IN_FEATURES. +_C.MODEL.ANCHOR_GENERATOR.ASPECT_RATIOS = [[0.5, 1.0, 2.0]] +# Anchor angles. +# list[list[float]], the angle in degrees, for each input feature map. +# ANGLES[i] specifies the list of angles for IN_FEATURES[i]. +_C.MODEL.ANCHOR_GENERATOR.ANGLES = [[-90, 0, 90]] +# Relative offset between the center of the first anchor and the top-left corner of the image +# Value has to be in [0, 1). Recommend to use 0.5, which means half stride. +# The value is not expected to affect model accuracy. +_C.MODEL.ANCHOR_GENERATOR.OFFSET = 0.0 + +# ---------------------------------------------------------------------------- # +# RPN options +# ---------------------------------------------------------------------------- # +_C.MODEL.RPN = CN() +_C.MODEL.RPN.HEAD_NAME = "StandardRPNHead" # used by RPN_HEAD_REGISTRY + +# Names of the input feature maps to be used by RPN +# e.g., ["p2", "p3", "p4", "p5", "p6"] for FPN +_C.MODEL.RPN.IN_FEATURES = ["res4"] +# Remove RPN anchors that go outside the image by BOUNDARY_THRESH pixels +# Set to -1 or a large value, e.g. 100000, to disable pruning anchors +_C.MODEL.RPN.BOUNDARY_THRESH = -1 +# IOU overlap ratios [BG_IOU_THRESHOLD, FG_IOU_THRESHOLD] +# Minimum overlap required between an anchor and ground-truth box for the +# (anchor, gt box) pair to be a positive example (IoU >= FG_IOU_THRESHOLD +# ==> positive RPN example: 1) +# Maximum overlap allowed between an anchor and ground-truth box for the +# (anchor, gt box) pair to be a negative examples (IoU < BG_IOU_THRESHOLD +# ==> negative RPN example: 0) +# Anchors with overlap in between (BG_IOU_THRESHOLD <= IoU < FG_IOU_THRESHOLD) +# are ignored (-1) +_C.MODEL.RPN.IOU_THRESHOLDS = [0.3, 0.7] +_C.MODEL.RPN.IOU_LABELS = [0, -1, 1] +# Number of regions per image used to train RPN +_C.MODEL.RPN.BATCH_SIZE_PER_IMAGE = 256 +# Target fraction of foreground (positive) examples per RPN minibatch +_C.MODEL.RPN.POSITIVE_FRACTION = 0.5 +# Options are: "smooth_l1", "giou", "diou", "ciou" +_C.MODEL.RPN.BBOX_REG_LOSS_TYPE = "smooth_l1" +_C.MODEL.RPN.BBOX_REG_LOSS_WEIGHT = 1.0 +# Weights on (dx, dy, dw, dh) for normalizing RPN anchor regression targets +_C.MODEL.RPN.BBOX_REG_WEIGHTS = (1.0, 1.0, 1.0, 1.0) +# The transition point from L1 to L2 loss. Set to 0.0 to make the loss simply L1. +_C.MODEL.RPN.SMOOTH_L1_BETA = 0.0 +_C.MODEL.RPN.LOSS_WEIGHT = 1.0 +# Number of top scoring RPN proposals to keep before applying NMS +# When FPN is used, this is *per FPN level* (not total) +_C.MODEL.RPN.PRE_NMS_TOPK_TRAIN = 12000 +_C.MODEL.RPN.PRE_NMS_TOPK_TEST = 6000 +# Number of top scoring RPN proposals to keep after applying NMS +# When FPN is used, this limit is applied per level and then again to the union +# of proposals from all levels +# NOTE: When FPN is used, the meaning of this config is different from Detectron1. +# It means per-batch topk in Detectron1, but per-image topk here. +# See the "find_top_rpn_proposals" function for details. +_C.MODEL.RPN.POST_NMS_TOPK_TRAIN = 2000 +_C.MODEL.RPN.POST_NMS_TOPK_TEST = 1000 +# NMS threshold used on RPN proposals +_C.MODEL.RPN.NMS_THRESH = 0.7 +# Set this to -1 to use the same number of output channels as input channels. +_C.MODEL.RPN.CONV_DIMS = [-1] + +# ---------------------------------------------------------------------------- # +# ROI HEADS options +# ---------------------------------------------------------------------------- # +_C.MODEL.ROI_HEADS = CN() +_C.MODEL.ROI_HEADS.NAME = "Res5ROIHeads" +# Number of foreground classes +_C.MODEL.ROI_HEADS.NUM_CLASSES = 80 +# Names of the input feature maps to be used by ROI heads +# Currently all heads (box, mask, ...) use the same input feature map list +# e.g., ["p2", "p3", "p4", "p5"] is commonly used for FPN +_C.MODEL.ROI_HEADS.IN_FEATURES = ["res4"] +# IOU overlap ratios [IOU_THRESHOLD] +# Overlap threshold for an RoI to be considered background (if < IOU_THRESHOLD) +# Overlap threshold for an RoI to be considered foreground (if >= IOU_THRESHOLD) +_C.MODEL.ROI_HEADS.IOU_THRESHOLDS = [0.5] +_C.MODEL.ROI_HEADS.IOU_LABELS = [0, 1] +# RoI minibatch size *per image* (number of regions of interest [ROIs]) during training +# Total number of RoIs per training minibatch = +# ROI_HEADS.BATCH_SIZE_PER_IMAGE * SOLVER.IMS_PER_BATCH +# E.g., a common configuration is: 512 * 16 = 8192 +_C.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE = 512 +# Target fraction of RoI minibatch that is labeled foreground (i.e. class > 0) +_C.MODEL.ROI_HEADS.POSITIVE_FRACTION = 0.25 + +# Only used on test mode + +# Minimum score threshold (assuming scores in a [0, 1] range); a value chosen to +# balance obtaining high recall with not having too many low precision +# detections that will slow down inference post processing steps (like NMS) +# A default threshold of 0.0 increases AP by ~0.2-0.3 but significantly slows down +# inference. +_C.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.05 +# Overlap threshold used for non-maximum suppression (suppress boxes with +# IoU >= this threshold) +_C.MODEL.ROI_HEADS.NMS_THRESH_TEST = 0.5 +# If True, augment proposals with ground-truth boxes before sampling proposals to +# train ROI heads. +_C.MODEL.ROI_HEADS.PROPOSAL_APPEND_GT = True + +# ---------------------------------------------------------------------------- # +# Box Head +# ---------------------------------------------------------------------------- # +_C.MODEL.ROI_BOX_HEAD = CN() +# C4 don't use head name option +# Options for non-C4 models: FastRCNNConvFCHead, +_C.MODEL.ROI_BOX_HEAD.NAME = "" +# Options are: "smooth_l1", "giou", "diou", "ciou" +_C.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_TYPE = "smooth_l1" +# The final scaling coefficient on the box regression loss, used to balance the magnitude of its +# gradients with other losses in the model. See also `MODEL.ROI_KEYPOINT_HEAD.LOSS_WEIGHT`. +_C.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_WEIGHT = 1.0 +# Default weights on (dx, dy, dw, dh) for normalizing bbox regression targets +# These are empirically chosen to approximately lead to unit variance targets +_C.MODEL.ROI_BOX_HEAD.BBOX_REG_WEIGHTS = (10.0, 10.0, 5.0, 5.0) +# The transition point from L1 to L2 loss. Set to 0.0 to make the loss simply L1. +_C.MODEL.ROI_BOX_HEAD.SMOOTH_L1_BETA = 0.0 +_C.MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION = 14 +_C.MODEL.ROI_BOX_HEAD.POOLER_SAMPLING_RATIO = 0 +# Type of pooling operation applied to the incoming feature map for each RoI +_C.MODEL.ROI_BOX_HEAD.POOLER_TYPE = "ROIAlignV2" + +_C.MODEL.ROI_BOX_HEAD.NUM_FC = 0 +# Hidden layer dimension for FC layers in the RoI box head +_C.MODEL.ROI_BOX_HEAD.FC_DIM = 1024 +_C.MODEL.ROI_BOX_HEAD.NUM_CONV = 0 +# Channel dimension for Conv layers in the RoI box head +_C.MODEL.ROI_BOX_HEAD.CONV_DIM = 256 +# Normalization method for the convolution layers. +# Options: "" (no norm), "GN", "SyncBN". +_C.MODEL.ROI_BOX_HEAD.NORM = "" +# Whether to use class agnostic for bbox regression +_C.MODEL.ROI_BOX_HEAD.CLS_AGNOSTIC_BBOX_REG = False +# If true, RoI heads use bounding boxes predicted by the box head rather than proposal boxes. +_C.MODEL.ROI_BOX_HEAD.TRAIN_ON_PRED_BOXES = False + +# Federated loss can be used to improve the training of LVIS +_C.MODEL.ROI_BOX_HEAD.USE_FED_LOSS = False +# Sigmoid cross entrophy is used with federated loss +_C.MODEL.ROI_BOX_HEAD.USE_SIGMOID_CE = False +# The power value applied to image_count when calcualting frequency weight +_C.MODEL.ROI_BOX_HEAD.FED_LOSS_FREQ_WEIGHT_POWER = 0.5 +# Number of classes to keep in total +_C.MODEL.ROI_BOX_HEAD.FED_LOSS_NUM_CLASSES = 50 + +# ---------------------------------------------------------------------------- # +# Cascaded Box Head +# ---------------------------------------------------------------------------- # +_C.MODEL.ROI_BOX_CASCADE_HEAD = CN() +# The number of cascade stages is implicitly defined by the length of the following two configs. +_C.MODEL.ROI_BOX_CASCADE_HEAD.BBOX_REG_WEIGHTS = ( + (10.0, 10.0, 5.0, 5.0), + (20.0, 20.0, 10.0, 10.0), + (30.0, 30.0, 15.0, 15.0), +) +_C.MODEL.ROI_BOX_CASCADE_HEAD.IOUS = (0.5, 0.6, 0.7) + + +# ---------------------------------------------------------------------------- # +# Mask Head +# ---------------------------------------------------------------------------- # +_C.MODEL.ROI_MASK_HEAD = CN() +_C.MODEL.ROI_MASK_HEAD.NAME = "MaskRCNNConvUpsampleHead" +_C.MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION = 14 +_C.MODEL.ROI_MASK_HEAD.POOLER_SAMPLING_RATIO = 0 +_C.MODEL.ROI_MASK_HEAD.NUM_CONV = 0 # The number of convs in the mask head +_C.MODEL.ROI_MASK_HEAD.CONV_DIM = 256 +# Normalization method for the convolution layers. +# Options: "" (no norm), "GN", "SyncBN". +_C.MODEL.ROI_MASK_HEAD.NORM = "" +# Whether to use class agnostic for mask prediction +_C.MODEL.ROI_MASK_HEAD.CLS_AGNOSTIC_MASK = False +# Type of pooling operation applied to the incoming feature map for each RoI +_C.MODEL.ROI_MASK_HEAD.POOLER_TYPE = "ROIAlignV2" + + +# ---------------------------------------------------------------------------- # +# Keypoint Head +# ---------------------------------------------------------------------------- # +_C.MODEL.ROI_KEYPOINT_HEAD = CN() +_C.MODEL.ROI_KEYPOINT_HEAD.NAME = "KRCNNConvDeconvUpsampleHead" +_C.MODEL.ROI_KEYPOINT_HEAD.POOLER_RESOLUTION = 14 +_C.MODEL.ROI_KEYPOINT_HEAD.POOLER_SAMPLING_RATIO = 0 +_C.MODEL.ROI_KEYPOINT_HEAD.CONV_DIMS = tuple(512 for _ in range(8)) +_C.MODEL.ROI_KEYPOINT_HEAD.NUM_KEYPOINTS = 17 # 17 is the number of keypoints in COCO. + +# Images with too few (or no) keypoints are excluded from training. +_C.MODEL.ROI_KEYPOINT_HEAD.MIN_KEYPOINTS_PER_IMAGE = 1 +# Normalize by the total number of visible keypoints in the minibatch if True. +# Otherwise, normalize by the total number of keypoints that could ever exist +# in the minibatch. +# The keypoint softmax loss is only calculated on visible keypoints. +# Since the number of visible keypoints can vary significantly between +# minibatches, this has the effect of up-weighting the importance of +# minibatches with few visible keypoints. (Imagine the extreme case of +# only one visible keypoint versus N: in the case of N, each one +# contributes 1/N to the gradient compared to the single keypoint +# determining the gradient direction). Instead, we can normalize the +# loss by the total number of keypoints, if it were the case that all +# keypoints were visible in a full minibatch. (Returning to the example, +# this means that the one visible keypoint contributes as much as each +# of the N keypoints.) +_C.MODEL.ROI_KEYPOINT_HEAD.NORMALIZE_LOSS_BY_VISIBLE_KEYPOINTS = True +# Multi-task loss weight to use for keypoints +# Recommended values: +# - use 1.0 if NORMALIZE_LOSS_BY_VISIBLE_KEYPOINTS is True +# - use 4.0 if NORMALIZE_LOSS_BY_VISIBLE_KEYPOINTS is False +_C.MODEL.ROI_KEYPOINT_HEAD.LOSS_WEIGHT = 1.0 +# Type of pooling operation applied to the incoming feature map for each RoI +_C.MODEL.ROI_KEYPOINT_HEAD.POOLER_TYPE = "ROIAlignV2" + +# ---------------------------------------------------------------------------- # +# Semantic Segmentation Head +# ---------------------------------------------------------------------------- # +_C.MODEL.SEM_SEG_HEAD = CN() +_C.MODEL.SEM_SEG_HEAD.NAME = "SemSegFPNHead" +_C.MODEL.SEM_SEG_HEAD.IN_FEATURES = ["p2", "p3", "p4", "p5"] +# Label in the semantic segmentation ground truth that is ignored, i.e., no loss is calculated for +# the correposnding pixel. +_C.MODEL.SEM_SEG_HEAD.IGNORE_VALUE = 255 +# Number of classes in the semantic segmentation head +_C.MODEL.SEM_SEG_HEAD.NUM_CLASSES = 54 +# Number of channels in the 3x3 convs inside semantic-FPN heads. +_C.MODEL.SEM_SEG_HEAD.CONVS_DIM = 128 +# Outputs from semantic-FPN heads are up-scaled to the COMMON_STRIDE stride. +_C.MODEL.SEM_SEG_HEAD.COMMON_STRIDE = 4 +# Normalization method for the convolution layers. Options: "" (no norm), "GN". +_C.MODEL.SEM_SEG_HEAD.NORM = "GN" +_C.MODEL.SEM_SEG_HEAD.LOSS_WEIGHT = 1.0 + +_C.MODEL.PANOPTIC_FPN = CN() +# Scaling of all losses from instance detection / segmentation head. +_C.MODEL.PANOPTIC_FPN.INSTANCE_LOSS_WEIGHT = 1.0 + +# options when combining instance & semantic segmentation outputs +_C.MODEL.PANOPTIC_FPN.COMBINE = CN({"ENABLED": True}) # "COMBINE.ENABLED" is deprecated & not used +_C.MODEL.PANOPTIC_FPN.COMBINE.OVERLAP_THRESH = 0.5 +_C.MODEL.PANOPTIC_FPN.COMBINE.STUFF_AREA_LIMIT = 4096 +_C.MODEL.PANOPTIC_FPN.COMBINE.INSTANCES_CONFIDENCE_THRESH = 0.5 + + +# ---------------------------------------------------------------------------- # +# RetinaNet Head +# ---------------------------------------------------------------------------- # +_C.MODEL.RETINANET = CN() + +# This is the number of foreground classes. +_C.MODEL.RETINANET.NUM_CLASSES = 80 + +_C.MODEL.RETINANET.IN_FEATURES = ["p3", "p4", "p5", "p6", "p7"] + +# Convolutions to use in the cls and bbox tower +# NOTE: this doesn't include the last conv for logits +_C.MODEL.RETINANET.NUM_CONVS = 4 + +# IoU overlap ratio [bg, fg] for labeling anchors. +# Anchors with < bg are labeled negative (0) +# Anchors with >= bg and < fg are ignored (-1) +# Anchors with >= fg are labeled positive (1) +_C.MODEL.RETINANET.IOU_THRESHOLDS = [0.4, 0.5] +_C.MODEL.RETINANET.IOU_LABELS = [0, -1, 1] + +# Prior prob for rare case (i.e. foreground) at the beginning of training. +# This is used to set the bias for the logits layer of the classifier subnet. +# This improves training stability in the case of heavy class imbalance. +_C.MODEL.RETINANET.PRIOR_PROB = 0.01 + +# Inference cls score threshold, only anchors with score > INFERENCE_TH are +# considered for inference (to improve speed) +_C.MODEL.RETINANET.SCORE_THRESH_TEST = 0.05 +# Select topk candidates before NMS +_C.MODEL.RETINANET.TOPK_CANDIDATES_TEST = 1000 +_C.MODEL.RETINANET.NMS_THRESH_TEST = 0.5 + +# Weights on (dx, dy, dw, dh) for normalizing Retinanet anchor regression targets +_C.MODEL.RETINANET.BBOX_REG_WEIGHTS = (1.0, 1.0, 1.0, 1.0) + +# Loss parameters +_C.MODEL.RETINANET.FOCAL_LOSS_GAMMA = 2.0 +_C.MODEL.RETINANET.FOCAL_LOSS_ALPHA = 0.25 +_C.MODEL.RETINANET.SMOOTH_L1_LOSS_BETA = 0.1 +# Options are: "smooth_l1", "giou", "diou", "ciou" +_C.MODEL.RETINANET.BBOX_REG_LOSS_TYPE = "smooth_l1" + +# One of BN, SyncBN, FrozenBN, GN +# Only supports GN until unshared norm is implemented +_C.MODEL.RETINANET.NORM = "" + + +# ---------------------------------------------------------------------------- # +# ResNe[X]t options (ResNets = {ResNet, ResNeXt} +# Note that parts of a resnet may be used for both the backbone and the head +# These options apply to both +# ---------------------------------------------------------------------------- # +_C.MODEL.RESNETS = CN() + +_C.MODEL.RESNETS.DEPTH = 50 +_C.MODEL.RESNETS.OUT_FEATURES = ["res4"] # res4 for C4 backbone, res2..5 for FPN backbone + +# Number of groups to use; 1 ==> ResNet; > 1 ==> ResNeXt +_C.MODEL.RESNETS.NUM_GROUPS = 1 + +# Options: FrozenBN, GN, "SyncBN", "BN" +_C.MODEL.RESNETS.NORM = "FrozenBN" + +# Baseline width of each group. +# Scaling this parameters will scale the width of all bottleneck layers. +_C.MODEL.RESNETS.WIDTH_PER_GROUP = 64 + +# Place the stride 2 conv on the 1x1 filter +# Use True only for the original MSRA ResNet; use False for C2 and Torch models +_C.MODEL.RESNETS.STRIDE_IN_1X1 = True + +# Apply dilation in stage "res5" +_C.MODEL.RESNETS.RES5_DILATION = 1 + +# Output width of res2. Scaling this parameters will scale the width of all 1x1 convs in ResNet +# For R18 and R34, this needs to be set to 64 +_C.MODEL.RESNETS.RES2_OUT_CHANNELS = 256 +_C.MODEL.RESNETS.STEM_OUT_CHANNELS = 64 + +# Apply Deformable Convolution in stages +# Specify if apply deform_conv on Res2, Res3, Res4, Res5 +_C.MODEL.RESNETS.DEFORM_ON_PER_STAGE = [False, False, False, False] +# Use True to use modulated deform_conv (DeformableV2, https://arxiv.org/abs/1811.11168); +# Use False for DeformableV1. +_C.MODEL.RESNETS.DEFORM_MODULATED = False +# Number of groups in deformable conv. +_C.MODEL.RESNETS.DEFORM_NUM_GROUPS = 1 + + +# ---------------------------------------------------------------------------- # +# Solver +# ---------------------------------------------------------------------------- # +_C.SOLVER = CN() + +# Options: WarmupMultiStepLR, WarmupCosineLR. +# See detectron2/solver/build.py for definition. +_C.SOLVER.LR_SCHEDULER_NAME = "WarmupMultiStepLR" + +_C.SOLVER.MAX_ITER = 40000 + +_C.SOLVER.BASE_LR = 0.001 +# The end lr, only used by WarmupCosineLR +_C.SOLVER.BASE_LR_END = 0.0 + +_C.SOLVER.MOMENTUM = 0.9 + +_C.SOLVER.NESTEROV = False + +_C.SOLVER.WEIGHT_DECAY = 0.0001 +# The weight decay that's applied to parameters of normalization layers +# (typically the affine transformation) +_C.SOLVER.WEIGHT_DECAY_NORM = 0.0 + +_C.SOLVER.GAMMA = 0.1 +# The iteration number to decrease learning rate by GAMMA. +_C.SOLVER.STEPS = (30000,) +# Number of decays in WarmupStepWithFixedGammaLR schedule +_C.SOLVER.NUM_DECAYS = 3 + +_C.SOLVER.WARMUP_FACTOR = 1.0 / 1000 +_C.SOLVER.WARMUP_ITERS = 1000 +_C.SOLVER.WARMUP_METHOD = "linear" +# Whether to rescale the interval for the learning schedule after warmup +_C.SOLVER.RESCALE_INTERVAL = False + +# Save a checkpoint after every this number of iterations +_C.SOLVER.CHECKPOINT_PERIOD = 5000 + +# Number of images per batch across all machines. This is also the number +# of training images per step (i.e. per iteration). If we use 16 GPUs +# and IMS_PER_BATCH = 32, each GPU will see 2 images per batch. +# May be adjusted automatically if REFERENCE_WORLD_SIZE is set. +_C.SOLVER.IMS_PER_BATCH = 16 + +# The reference number of workers (GPUs) this config is meant to train with. +# It takes no effect when set to 0. +# With a non-zero value, it will be used by DefaultTrainer to compute a desired +# per-worker batch size, and then scale the other related configs (total batch size, +# learning rate, etc) to match the per-worker batch size. +# See documentation of `DefaultTrainer.auto_scale_workers` for details: +_C.SOLVER.REFERENCE_WORLD_SIZE = 0 + +# Detectron v1 (and previous detection code) used a 2x higher LR and 0 WD for +# biases. This is not useful (at least for recent models). You should avoid +# changing these and they exist only to reproduce Detectron v1 training if +# desired. +_C.SOLVER.BIAS_LR_FACTOR = 1.0 +_C.SOLVER.WEIGHT_DECAY_BIAS = None # None means following WEIGHT_DECAY + +# Gradient clipping +_C.SOLVER.CLIP_GRADIENTS = CN({"ENABLED": False}) +# Type of gradient clipping, currently 2 values are supported: +# - "value": the absolute values of elements of each gradients are clipped +# - "norm": the norm of the gradient for each parameter is clipped thus +# affecting all elements in the parameter +_C.SOLVER.CLIP_GRADIENTS.CLIP_TYPE = "value" +# Maximum absolute value used for clipping gradients +_C.SOLVER.CLIP_GRADIENTS.CLIP_VALUE = 1.0 +# Floating point number p for L-p norm to be used with the "norm" +# gradient clipping type; for L-inf, please specify .inf +_C.SOLVER.CLIP_GRADIENTS.NORM_TYPE = 2.0 + +# Enable automatic mixed precision for training +# Note that this does not change model's inference behavior. +# To use AMP in inference, run inference under autocast() +_C.SOLVER.AMP = CN({"ENABLED": False}) + +# ---------------------------------------------------------------------------- # +# Specific test options +# ---------------------------------------------------------------------------- # +_C.TEST = CN() +# For end-to-end tests to verify the expected accuracy. +# Each item is [task, metric, value, tolerance] +# e.g.: [['bbox', 'AP', 38.5, 0.2]] +_C.TEST.EXPECTED_RESULTS = [] +# The period (in terms of steps) to evaluate the model during training. +# Set to 0 to disable. +_C.TEST.EVAL_PERIOD = 0 +# The sigmas used to calculate keypoint OKS. See http://cocodataset.org/#keypoints-eval +# When empty, it will use the defaults in COCO. +# Otherwise it should be a list[float] with the same length as ROI_KEYPOINT_HEAD.NUM_KEYPOINTS. +_C.TEST.KEYPOINT_OKS_SIGMAS = [] +# Maximum number of detections to return per image during inference (100 is +# based on the limit established for the COCO dataset). +_C.TEST.DETECTIONS_PER_IMAGE = 100 + +_C.TEST.AUG = CN({"ENABLED": False}) +_C.TEST.AUG.MIN_SIZES = (400, 500, 600, 700, 800, 900, 1000, 1100, 1200) +_C.TEST.AUG.MAX_SIZE = 4000 +_C.TEST.AUG.FLIP = True + +_C.TEST.PRECISE_BN = CN({"ENABLED": False}) +_C.TEST.PRECISE_BN.NUM_ITER = 200 + +# ---------------------------------------------------------------------------- # +# Misc options +# ---------------------------------------------------------------------------- # +# Directory where output files are written +_C.OUTPUT_DIR = "./output" +# Set seed to negative to fully randomize everything. +# Set seed to positive to use a fixed seed. Note that a fixed seed increases +# reproducibility but does not guarantee fully deterministic behavior. +# Disabling all parallelism further increases reproducibility. +_C.SEED = -1 +# Benchmark different cudnn algorithms. +# If input images have very different sizes, this option will have large overhead +# for about 10k iterations. It usually hurts total time, but can benefit for certain models. +# If input images have the same or similar sizes, benchmark is often helpful. +_C.CUDNN_BENCHMARK = False +# The period (in terms of steps) for minibatch visualization at train time. +# Set to 0 to disable. +_C.VIS_PERIOD = 0 + +# global config is for quick hack purposes. +# You can set them in command line or config files, +# and access it with: +# +# from annotator.oneformer.detectron2.config import global_cfg +# print(global_cfg.HACK) +# +# Do not commit any configs into it. +_C.GLOBAL = CN() +_C.GLOBAL.HACK = 1.0 diff --git a/annotator/oneformer/detectron2/config/instantiate.py b/annotator/oneformer/detectron2/config/instantiate.py new file mode 100644 index 0000000000000000000000000000000000000000..26d191b03f800dae5620128957d137cd4fdb1728 --- /dev/null +++ b/annotator/oneformer/detectron2/config/instantiate.py @@ -0,0 +1,88 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import collections.abc as abc +import dataclasses +import logging +from typing import Any + +from annotator.oneformer.detectron2.utils.registry import _convert_target_to_string, locate + +__all__ = ["dump_dataclass", "instantiate"] + + +def dump_dataclass(obj: Any): + """ + Dump a dataclass recursively into a dict that can be later instantiated. + + Args: + obj: a dataclass object + + Returns: + dict + """ + assert dataclasses.is_dataclass(obj) and not isinstance( + obj, type + ), "dump_dataclass() requires an instance of a dataclass." + ret = {"_target_": _convert_target_to_string(type(obj))} + for f in dataclasses.fields(obj): + v = getattr(obj, f.name) + if dataclasses.is_dataclass(v): + v = dump_dataclass(v) + if isinstance(v, (list, tuple)): + v = [dump_dataclass(x) if dataclasses.is_dataclass(x) else x for x in v] + ret[f.name] = v + return ret + + +def instantiate(cfg): + """ + Recursively instantiate objects defined in dictionaries by + "_target_" and arguments. + + Args: + cfg: a dict-like object with "_target_" that defines the caller, and + other keys that define the arguments + + Returns: + object instantiated by cfg + """ + from omegaconf import ListConfig, DictConfig, OmegaConf + + if isinstance(cfg, ListConfig): + lst = [instantiate(x) for x in cfg] + return ListConfig(lst, flags={"allow_objects": True}) + if isinstance(cfg, list): + # Specialize for list, because many classes take + # list[objects] as arguments, such as ResNet, DatasetMapper + return [instantiate(x) for x in cfg] + + # If input is a DictConfig backed by dataclasses (i.e. omegaconf's structured config), + # instantiate it to the actual dataclass. + if isinstance(cfg, DictConfig) and dataclasses.is_dataclass(cfg._metadata.object_type): + return OmegaConf.to_object(cfg) + + if isinstance(cfg, abc.Mapping) and "_target_" in cfg: + # conceptually equivalent to hydra.utils.instantiate(cfg) with _convert_=all, + # but faster: https://github.com/facebookresearch/hydra/issues/1200 + cfg = {k: instantiate(v) for k, v in cfg.items()} + cls = cfg.pop("_target_") + cls = instantiate(cls) + + if isinstance(cls, str): + cls_name = cls + cls = locate(cls_name) + assert cls is not None, cls_name + else: + try: + cls_name = cls.__module__ + "." + cls.__qualname__ + except Exception: + # target could be anything, so the above could fail + cls_name = str(cls) + assert callable(cls), f"_target_ {cls} does not define a callable object" + try: + return cls(**cfg) + except TypeError: + logger = logging.getLogger(__name__) + logger.error(f"Error when instantiating {cls_name}!") + raise + return cfg # return as-is if don't know what to do diff --git a/annotator/oneformer/detectron2/config/lazy.py b/annotator/oneformer/detectron2/config/lazy.py new file mode 100644 index 0000000000000000000000000000000000000000..72a3e5c036f9f78a2cdf3ef0975639da3299d694 --- /dev/null +++ b/annotator/oneformer/detectron2/config/lazy.py @@ -0,0 +1,435 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import ast +import builtins +import collections.abc as abc +import importlib +import inspect +import logging +import os +import uuid +from contextlib import contextmanager +from copy import deepcopy +from dataclasses import is_dataclass +from typing import List, Tuple, Union +import yaml +from omegaconf import DictConfig, ListConfig, OmegaConf, SCMode + +from annotator.oneformer.detectron2.utils.file_io import PathManager +from annotator.oneformer.detectron2.utils.registry import _convert_target_to_string + +__all__ = ["LazyCall", "LazyConfig"] + + +class LazyCall: + """ + Wrap a callable so that when it's called, the call will not be executed, + but returns a dict that describes the call. + + LazyCall object has to be called with only keyword arguments. Positional + arguments are not yet supported. + + Examples: + :: + from annotator.oneformer.detectron2.config import instantiate, LazyCall + + layer_cfg = LazyCall(nn.Conv2d)(in_channels=32, out_channels=32) + layer_cfg.out_channels = 64 # can edit it afterwards + layer = instantiate(layer_cfg) + """ + + def __init__(self, target): + if not (callable(target) or isinstance(target, (str, abc.Mapping))): + raise TypeError( + f"target of LazyCall must be a callable or defines a callable! Got {target}" + ) + self._target = target + + def __call__(self, **kwargs): + if is_dataclass(self._target): + # omegaconf object cannot hold dataclass type + # https://github.com/omry/omegaconf/issues/784 + target = _convert_target_to_string(self._target) + else: + target = self._target + kwargs["_target_"] = target + + return DictConfig(content=kwargs, flags={"allow_objects": True}) + + +def _visit_dict_config(cfg, func): + """ + Apply func recursively to all DictConfig in cfg. + """ + if isinstance(cfg, DictConfig): + func(cfg) + for v in cfg.values(): + _visit_dict_config(v, func) + elif isinstance(cfg, ListConfig): + for v in cfg: + _visit_dict_config(v, func) + + +def _validate_py_syntax(filename): + # see also https://github.com/open-mmlab/mmcv/blob/master/mmcv/utils/config.py + with PathManager.open(filename, "r") as f: + content = f.read() + try: + ast.parse(content) + except SyntaxError as e: + raise SyntaxError(f"Config file {filename} has syntax error!") from e + + +def _cast_to_config(obj): + # if given a dict, return DictConfig instead + if isinstance(obj, dict): + return DictConfig(obj, flags={"allow_objects": True}) + return obj + + +_CFG_PACKAGE_NAME = "detectron2._cfg_loader" +""" +A namespace to put all imported config into. +""" + + +def _random_package_name(filename): + # generate a random package name when loading config files + return _CFG_PACKAGE_NAME + str(uuid.uuid4())[:4] + "." + os.path.basename(filename) + + +@contextmanager +def _patch_import(): + """ + Enhance relative import statements in config files, so that they: + 1. locate files purely based on relative location, regardless of packages. + e.g. you can import file without having __init__ + 2. do not cache modules globally; modifications of module states has no side effect + 3. support other storage system through PathManager, so config files can be in the cloud + 4. imported dict are turned into omegaconf.DictConfig automatically + """ + old_import = builtins.__import__ + + def find_relative_file(original_file, relative_import_path, level): + # NOTE: "from . import x" is not handled. Because then it's unclear + # if such import should produce `x` as a python module or DictConfig. + # This can be discussed further if needed. + relative_import_err = """ +Relative import of directories is not allowed within config files. +Within a config file, relative import can only import other config files. +""".replace( + "\n", " " + ) + if not len(relative_import_path): + raise ImportError(relative_import_err) + + cur_file = os.path.dirname(original_file) + for _ in range(level - 1): + cur_file = os.path.dirname(cur_file) + cur_name = relative_import_path.lstrip(".") + for part in cur_name.split("."): + cur_file = os.path.join(cur_file, part) + if not cur_file.endswith(".py"): + cur_file += ".py" + if not PathManager.isfile(cur_file): + cur_file_no_suffix = cur_file[: -len(".py")] + if PathManager.isdir(cur_file_no_suffix): + raise ImportError(f"Cannot import from {cur_file_no_suffix}." + relative_import_err) + else: + raise ImportError( + f"Cannot import name {relative_import_path} from " + f"{original_file}: {cur_file} does not exist." + ) + return cur_file + + def new_import(name, globals=None, locals=None, fromlist=(), level=0): + if ( + # Only deal with relative imports inside config files + level != 0 + and globals is not None + and (globals.get("__package__", "") or "").startswith(_CFG_PACKAGE_NAME) + ): + cur_file = find_relative_file(globals["__file__"], name, level) + _validate_py_syntax(cur_file) + spec = importlib.machinery.ModuleSpec( + _random_package_name(cur_file), None, origin=cur_file + ) + module = importlib.util.module_from_spec(spec) + module.__file__ = cur_file + with PathManager.open(cur_file) as f: + content = f.read() + exec(compile(content, cur_file, "exec"), module.__dict__) + for name in fromlist: # turn imported dict into DictConfig automatically + val = _cast_to_config(module.__dict__[name]) + module.__dict__[name] = val + return module + return old_import(name, globals, locals, fromlist=fromlist, level=level) + + builtins.__import__ = new_import + yield new_import + builtins.__import__ = old_import + + +class LazyConfig: + """ + Provide methods to save, load, and overrides an omegaconf config object + which may contain definition of lazily-constructed objects. + """ + + @staticmethod + def load_rel(filename: str, keys: Union[None, str, Tuple[str, ...]] = None): + """ + Similar to :meth:`load()`, but load path relative to the caller's + source file. + + This has the same functionality as a relative import, except that this method + accepts filename as a string, so more characters are allowed in the filename. + """ + caller_frame = inspect.stack()[1] + caller_fname = caller_frame[0].f_code.co_filename + assert caller_fname != "", "load_rel Unable to find caller" + caller_dir = os.path.dirname(caller_fname) + filename = os.path.join(caller_dir, filename) + return LazyConfig.load(filename, keys) + + @staticmethod + def load(filename: str, keys: Union[None, str, Tuple[str, ...]] = None): + """ + Load a config file. + + Args: + filename: absolute path or relative path w.r.t. the current working directory + keys: keys to load and return. If not given, return all keys + (whose values are config objects) in a dict. + """ + has_keys = keys is not None + filename = filename.replace("/./", "/") # redundant + if os.path.splitext(filename)[1] not in [".py", ".yaml", ".yml"]: + raise ValueError(f"Config file {filename} has to be a python or yaml file.") + if filename.endswith(".py"): + _validate_py_syntax(filename) + + with _patch_import(): + # Record the filename + module_namespace = { + "__file__": filename, + "__package__": _random_package_name(filename), + } + with PathManager.open(filename) as f: + content = f.read() + # Compile first with filename to: + # 1. make filename appears in stacktrace + # 2. make load_rel able to find its parent's (possibly remote) location + exec(compile(content, filename, "exec"), module_namespace) + + ret = module_namespace + else: + with PathManager.open(filename) as f: + obj = yaml.unsafe_load(f) + ret = OmegaConf.create(obj, flags={"allow_objects": True}) + + if has_keys: + if isinstance(keys, str): + return _cast_to_config(ret[keys]) + else: + return tuple(_cast_to_config(ret[a]) for a in keys) + else: + if filename.endswith(".py"): + # when not specified, only load those that are config objects + ret = DictConfig( + { + name: _cast_to_config(value) + for name, value in ret.items() + if isinstance(value, (DictConfig, ListConfig, dict)) + and not name.startswith("_") + }, + flags={"allow_objects": True}, + ) + return ret + + @staticmethod + def save(cfg, filename: str): + """ + Save a config object to a yaml file. + Note that when the config dictionary contains complex objects (e.g. lambda), + it can't be saved to yaml. In that case we will print an error and + attempt to save to a pkl file instead. + + Args: + cfg: an omegaconf config object + filename: yaml file name to save the config file + """ + logger = logging.getLogger(__name__) + try: + cfg = deepcopy(cfg) + except Exception: + pass + else: + # if it's deep-copyable, then... + def _replace_type_by_name(x): + if "_target_" in x and callable(x._target_): + try: + x._target_ = _convert_target_to_string(x._target_) + except AttributeError: + pass + + # not necessary, but makes yaml looks nicer + _visit_dict_config(cfg, _replace_type_by_name) + + save_pkl = False + try: + dict = OmegaConf.to_container( + cfg, + # Do not resolve interpolation when saving, i.e. do not turn ${a} into + # actual values when saving. + resolve=False, + # Save structures (dataclasses) in a format that can be instantiated later. + # Without this option, the type information of the dataclass will be erased. + structured_config_mode=SCMode.INSTANTIATE, + ) + dumped = yaml.dump(dict, default_flow_style=None, allow_unicode=True, width=9999) + with PathManager.open(filename, "w") as f: + f.write(dumped) + + try: + _ = yaml.unsafe_load(dumped) # test that it is loadable + except Exception: + logger.warning( + "The config contains objects that cannot serialize to a valid yaml. " + f"{filename} is human-readable but cannot be loaded." + ) + save_pkl = True + except Exception: + logger.exception("Unable to serialize the config to yaml. Error:") + save_pkl = True + + if save_pkl: + new_filename = filename + ".pkl" + # try: + # # retry by pickle + # with PathManager.open(new_filename, "wb") as f: + # cloudpickle.dump(cfg, f) + # logger.warning(f"Config is saved using cloudpickle at {new_filename}.") + # except Exception: + # pass + + @staticmethod + def apply_overrides(cfg, overrides: List[str]): + """ + In-place override contents of cfg. + + Args: + cfg: an omegaconf config object + overrides: list of strings in the format of "a=b" to override configs. + See https://hydra.cc/docs/next/advanced/override_grammar/basic/ + for syntax. + + Returns: + the cfg object + """ + + def safe_update(cfg, key, value): + parts = key.split(".") + for idx in range(1, len(parts)): + prefix = ".".join(parts[:idx]) + v = OmegaConf.select(cfg, prefix, default=None) + if v is None: + break + if not OmegaConf.is_config(v): + raise KeyError( + f"Trying to update key {key}, but {prefix} " + f"is not a config, but has type {type(v)}." + ) + OmegaConf.update(cfg, key, value, merge=True) + + try: + from hydra.core.override_parser.overrides_parser import OverridesParser + + has_hydra = True + except ImportError: + has_hydra = False + + if has_hydra: + parser = OverridesParser.create() + overrides = parser.parse_overrides(overrides) + for o in overrides: + key = o.key_or_group + value = o.value() + if o.is_delete(): + # TODO support this + raise NotImplementedError("deletion is not yet a supported override") + safe_update(cfg, key, value) + else: + # Fallback. Does not support all the features and error checking like hydra. + for o in overrides: + key, value = o.split("=") + try: + value = eval(value, {}) + except NameError: + pass + safe_update(cfg, key, value) + return cfg + + # @staticmethod + # def to_py(cfg, prefix: str = "cfg."): + # """ + # Try to convert a config object into Python-like psuedo code. + # + # Note that perfect conversion is not always possible. So the returned + # results are mainly meant to be human-readable, and not meant to be executed. + # + # Args: + # cfg: an omegaconf config object + # prefix: root name for the resulting code (default: "cfg.") + # + # + # Returns: + # str of formatted Python code + # """ + # import black + # + # cfg = OmegaConf.to_container(cfg, resolve=True) + # + # def _to_str(obj, prefix=None, inside_call=False): + # if prefix is None: + # prefix = [] + # if isinstance(obj, abc.Mapping) and "_target_" in obj: + # # Dict representing a function call + # target = _convert_target_to_string(obj.pop("_target_")) + # args = [] + # for k, v in sorted(obj.items()): + # args.append(f"{k}={_to_str(v, inside_call=True)}") + # args = ", ".join(args) + # call = f"{target}({args})" + # return "".join(prefix) + call + # elif isinstance(obj, abc.Mapping) and not inside_call: + # # Dict that is not inside a call is a list of top-level config objects that we + # # render as one object per line with dot separated prefixes + # key_list = [] + # for k, v in sorted(obj.items()): + # if isinstance(v, abc.Mapping) and "_target_" not in v: + # key_list.append(_to_str(v, prefix=prefix + [k + "."])) + # else: + # key = "".join(prefix) + k + # key_list.append(f"{key}={_to_str(v)}") + # return "\n".join(key_list) + # elif isinstance(obj, abc.Mapping): + # # Dict that is inside a call is rendered as a regular dict + # return ( + # "{" + # + ",".join( + # f"{repr(k)}: {_to_str(v, inside_call=inside_call)}" + # for k, v in sorted(obj.items()) + # ) + # + "}" + # ) + # elif isinstance(obj, list): + # return "[" + ",".join(_to_str(x, inside_call=inside_call) for x in obj) + "]" + # else: + # return repr(obj) + # + # py_str = _to_str(cfg, prefix=[prefix]) + # try: + # return black.format_str(py_str, mode=black.Mode()) + # except black.InvalidInput: + # return py_str diff --git a/annotator/oneformer/detectron2/data/__init__.py b/annotator/oneformer/detectron2/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..259f669b78bd05815cb8d3351fd6c5fc9a1b85a1 --- /dev/null +++ b/annotator/oneformer/detectron2/data/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from . import transforms # isort:skip + +from .build import ( + build_batch_data_loader, + build_detection_test_loader, + build_detection_train_loader, + get_detection_dataset_dicts, + load_proposals_into_dataset, + print_instances_class_histogram, +) +from .catalog import DatasetCatalog, MetadataCatalog, Metadata +from .common import DatasetFromList, MapDataset, ToIterableDataset +from .dataset_mapper import DatasetMapper + +# ensure the builtin datasets are registered +from . import datasets, samplers # isort:skip + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/annotator/oneformer/detectron2/data/benchmark.py b/annotator/oneformer/detectron2/data/benchmark.py new file mode 100644 index 0000000000000000000000000000000000000000..bfd650582c83cd032b4fe76303517cdfd9a2a8b4 --- /dev/null +++ b/annotator/oneformer/detectron2/data/benchmark.py @@ -0,0 +1,225 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import numpy as np +from itertools import count +from typing import List, Tuple +import torch +import tqdm +from fvcore.common.timer import Timer + +from annotator.oneformer.detectron2.utils import comm + +from .build import build_batch_data_loader +from .common import DatasetFromList, MapDataset +from .samplers import TrainingSampler + +logger = logging.getLogger(__name__) + + +class _EmptyMapDataset(torch.utils.data.Dataset): + """ + Map anything to emptiness. + """ + + def __init__(self, dataset): + self.ds = dataset + + def __len__(self): + return len(self.ds) + + def __getitem__(self, idx): + _ = self.ds[idx] + return [0] + + +def iter_benchmark( + iterator, num_iter: int, warmup: int = 5, max_time_seconds: float = 60 +) -> Tuple[float, List[float]]: + """ + Benchmark an iterator/iterable for `num_iter` iterations with an extra + `warmup` iterations of warmup. + End early if `max_time_seconds` time is spent on iterations. + + Returns: + float: average time (seconds) per iteration + list[float]: time spent on each iteration. Sometimes useful for further analysis. + """ + num_iter, warmup = int(num_iter), int(warmup) + + iterator = iter(iterator) + for _ in range(warmup): + next(iterator) + timer = Timer() + all_times = [] + for curr_iter in tqdm.trange(num_iter): + start = timer.seconds() + if start > max_time_seconds: + num_iter = curr_iter + break + next(iterator) + all_times.append(timer.seconds() - start) + avg = timer.seconds() / num_iter + return avg, all_times + + +class DataLoaderBenchmark: + """ + Some common benchmarks that help understand perf bottleneck of a standard dataloader + made of dataset, mapper and sampler. + """ + + def __init__( + self, + dataset, + *, + mapper, + sampler=None, + total_batch_size, + num_workers=0, + max_time_seconds: int = 90, + ): + """ + Args: + max_time_seconds (int): maximum time to spent for each benchmark + other args: same as in `build.py:build_detection_train_loader` + """ + if isinstance(dataset, list): + dataset = DatasetFromList(dataset, copy=False, serialize=True) + if sampler is None: + sampler = TrainingSampler(len(dataset)) + + self.dataset = dataset + self.mapper = mapper + self.sampler = sampler + self.total_batch_size = total_batch_size + self.num_workers = num_workers + self.per_gpu_batch_size = self.total_batch_size // comm.get_world_size() + + self.max_time_seconds = max_time_seconds + + def _benchmark(self, iterator, num_iter, warmup, msg=None): + avg, all_times = iter_benchmark(iterator, num_iter, warmup, self.max_time_seconds) + if msg is not None: + self._log_time(msg, avg, all_times) + return avg, all_times + + def _log_time(self, msg, avg, all_times, distributed=False): + percentiles = [np.percentile(all_times, k, interpolation="nearest") for k in [1, 5, 95, 99]] + if not distributed: + logger.info( + f"{msg}: avg={1.0/avg:.1f} it/s, " + f"p1={percentiles[0]:.2g}s, p5={percentiles[1]:.2g}s, " + f"p95={percentiles[2]:.2g}s, p99={percentiles[3]:.2g}s." + ) + return + avg_per_gpu = comm.all_gather(avg) + percentiles_per_gpu = comm.all_gather(percentiles) + if comm.get_rank() > 0: + return + for idx, avg, percentiles in zip(count(), avg_per_gpu, percentiles_per_gpu): + logger.info( + f"GPU{idx} {msg}: avg={1.0/avg:.1f} it/s, " + f"p1={percentiles[0]:.2g}s, p5={percentiles[1]:.2g}s, " + f"p95={percentiles[2]:.2g}s, p99={percentiles[3]:.2g}s." + ) + + def benchmark_dataset(self, num_iter, warmup=5): + """ + Benchmark the speed of taking raw samples from the dataset. + """ + + def loader(): + while True: + for k in self.sampler: + yield self.dataset[k] + + self._benchmark(loader(), num_iter, warmup, "Dataset Alone") + + def benchmark_mapper(self, num_iter, warmup=5): + """ + Benchmark the speed of taking raw samples from the dataset and map + them in a single process. + """ + + def loader(): + while True: + for k in self.sampler: + yield self.mapper(self.dataset[k]) + + self._benchmark(loader(), num_iter, warmup, "Single Process Mapper (sec/sample)") + + def benchmark_workers(self, num_iter, warmup=10): + """ + Benchmark the dataloader by tuning num_workers to [0, 1, self.num_workers]. + """ + candidates = [0, 1] + if self.num_workers not in candidates: + candidates.append(self.num_workers) + + dataset = MapDataset(self.dataset, self.mapper) + for n in candidates: + loader = build_batch_data_loader( + dataset, + self.sampler, + self.total_batch_size, + num_workers=n, + ) + self._benchmark( + iter(loader), + num_iter * max(n, 1), + warmup * max(n, 1), + f"DataLoader ({n} workers, bs={self.per_gpu_batch_size})", + ) + del loader + + def benchmark_IPC(self, num_iter, warmup=10): + """ + Benchmark the dataloader where each worker outputs nothing. This + eliminates the IPC overhead compared to the regular dataloader. + + PyTorch multiprocessing's IPC only optimizes for torch tensors. + Large numpy arrays or other data structure may incur large IPC overhead. + """ + n = self.num_workers + dataset = _EmptyMapDataset(MapDataset(self.dataset, self.mapper)) + loader = build_batch_data_loader( + dataset, self.sampler, self.total_batch_size, num_workers=n + ) + self._benchmark( + iter(loader), + num_iter * max(n, 1), + warmup * max(n, 1), + f"DataLoader ({n} workers, bs={self.per_gpu_batch_size}) w/o comm", + ) + + def benchmark_distributed(self, num_iter, warmup=10): + """ + Benchmark the dataloader in each distributed worker, and log results of + all workers. This helps understand the final performance as well as + the variances among workers. + + It also prints startup time (first iter) of the dataloader. + """ + gpu = comm.get_world_size() + dataset = MapDataset(self.dataset, self.mapper) + n = self.num_workers + loader = build_batch_data_loader( + dataset, self.sampler, self.total_batch_size, num_workers=n + ) + + timer = Timer() + loader = iter(loader) + next(loader) + startup_time = timer.seconds() + logger.info("Dataloader startup time: {:.2f} seconds".format(startup_time)) + + comm.synchronize() + + avg, all_times = self._benchmark(loader, num_iter * max(n, 1), warmup * max(n, 1)) + del loader + self._log_time( + f"DataLoader ({gpu} GPUs x {n} workers, total bs={self.total_batch_size})", + avg, + all_times, + True, + ) diff --git a/annotator/oneformer/detectron2/data/build.py b/annotator/oneformer/detectron2/data/build.py new file mode 100644 index 0000000000000000000000000000000000000000..d03137a9aabfc4a056dd671d4c3d0ba6f349fe03 --- /dev/null +++ b/annotator/oneformer/detectron2/data/build.py @@ -0,0 +1,556 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import itertools +import logging +import numpy as np +import operator +import pickle +from typing import Any, Callable, Dict, List, Optional, Union +import torch +import torch.utils.data as torchdata +from tabulate import tabulate +from termcolor import colored + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.structures import BoxMode +from annotator.oneformer.detectron2.utils.comm import get_world_size +from annotator.oneformer.detectron2.utils.env import seed_all_rng +from annotator.oneformer.detectron2.utils.file_io import PathManager +from annotator.oneformer.detectron2.utils.logger import _log_api_usage, log_first_n + +from .catalog import DatasetCatalog, MetadataCatalog +from .common import AspectRatioGroupedDataset, DatasetFromList, MapDataset, ToIterableDataset +from .dataset_mapper import DatasetMapper +from .detection_utils import check_metadata_consistency +from .samplers import ( + InferenceSampler, + RandomSubsetTrainingSampler, + RepeatFactorTrainingSampler, + TrainingSampler, +) + +""" +This file contains the default logic to build a dataloader for training or testing. +""" + +__all__ = [ + "build_batch_data_loader", + "build_detection_train_loader", + "build_detection_test_loader", + "get_detection_dataset_dicts", + "load_proposals_into_dataset", + "print_instances_class_histogram", +] + + +def filter_images_with_only_crowd_annotations(dataset_dicts): + """ + Filter out images with none annotations or only crowd annotations + (i.e., images without non-crowd annotations). + A common training-time preprocessing on COCO dataset. + + Args: + dataset_dicts (list[dict]): annotations in Detectron2 Dataset format. + + Returns: + list[dict]: the same format, but filtered. + """ + num_before = len(dataset_dicts) + + def valid(anns): + for ann in anns: + if ann.get("iscrowd", 0) == 0: + return True + return False + + dataset_dicts = [x for x in dataset_dicts if valid(x["annotations"])] + num_after = len(dataset_dicts) + logger = logging.getLogger(__name__) + logger.info( + "Removed {} images with no usable annotations. {} images left.".format( + num_before - num_after, num_after + ) + ) + return dataset_dicts + + +def filter_images_with_few_keypoints(dataset_dicts, min_keypoints_per_image): + """ + Filter out images with too few number of keypoints. + + Args: + dataset_dicts (list[dict]): annotations in Detectron2 Dataset format. + + Returns: + list[dict]: the same format as dataset_dicts, but filtered. + """ + num_before = len(dataset_dicts) + + def visible_keypoints_in_image(dic): + # Each keypoints field has the format [x1, y1, v1, ...], where v is visibility + annotations = dic["annotations"] + return sum( + (np.array(ann["keypoints"][2::3]) > 0).sum() + for ann in annotations + if "keypoints" in ann + ) + + dataset_dicts = [ + x for x in dataset_dicts if visible_keypoints_in_image(x) >= min_keypoints_per_image + ] + num_after = len(dataset_dicts) + logger = logging.getLogger(__name__) + logger.info( + "Removed {} images with fewer than {} keypoints.".format( + num_before - num_after, min_keypoints_per_image + ) + ) + return dataset_dicts + + +def load_proposals_into_dataset(dataset_dicts, proposal_file): + """ + Load precomputed object proposals into the dataset. + + The proposal file should be a pickled dict with the following keys: + + - "ids": list[int] or list[str], the image ids + - "boxes": list[np.ndarray], each is an Nx4 array of boxes corresponding to the image id + - "objectness_logits": list[np.ndarray], each is an N sized array of objectness scores + corresponding to the boxes. + - "bbox_mode": the BoxMode of the boxes array. Defaults to ``BoxMode.XYXY_ABS``. + + Args: + dataset_dicts (list[dict]): annotations in Detectron2 Dataset format. + proposal_file (str): file path of pre-computed proposals, in pkl format. + + Returns: + list[dict]: the same format as dataset_dicts, but added proposal field. + """ + logger = logging.getLogger(__name__) + logger.info("Loading proposals from: {}".format(proposal_file)) + + with PathManager.open(proposal_file, "rb") as f: + proposals = pickle.load(f, encoding="latin1") + + # Rename the key names in D1 proposal files + rename_keys = {"indexes": "ids", "scores": "objectness_logits"} + for key in rename_keys: + if key in proposals: + proposals[rename_keys[key]] = proposals.pop(key) + + # Fetch the indexes of all proposals that are in the dataset + # Convert image_id to str since they could be int. + img_ids = set({str(record["image_id"]) for record in dataset_dicts}) + id_to_index = {str(id): i for i, id in enumerate(proposals["ids"]) if str(id) in img_ids} + + # Assuming default bbox_mode of precomputed proposals are 'XYXY_ABS' + bbox_mode = BoxMode(proposals["bbox_mode"]) if "bbox_mode" in proposals else BoxMode.XYXY_ABS + + for record in dataset_dicts: + # Get the index of the proposal + i = id_to_index[str(record["image_id"])] + + boxes = proposals["boxes"][i] + objectness_logits = proposals["objectness_logits"][i] + # Sort the proposals in descending order of the scores + inds = objectness_logits.argsort()[::-1] + record["proposal_boxes"] = boxes[inds] + record["proposal_objectness_logits"] = objectness_logits[inds] + record["proposal_bbox_mode"] = bbox_mode + + return dataset_dicts + + +def print_instances_class_histogram(dataset_dicts, class_names): + """ + Args: + dataset_dicts (list[dict]): list of dataset dicts. + class_names (list[str]): list of class names (zero-indexed). + """ + num_classes = len(class_names) + hist_bins = np.arange(num_classes + 1) + histogram = np.zeros((num_classes,), dtype=np.int) + for entry in dataset_dicts: + annos = entry["annotations"] + classes = np.asarray( + [x["category_id"] for x in annos if not x.get("iscrowd", 0)], dtype=np.int + ) + if len(classes): + assert classes.min() >= 0, f"Got an invalid category_id={classes.min()}" + assert ( + classes.max() < num_classes + ), f"Got an invalid category_id={classes.max()} for a dataset of {num_classes} classes" + histogram += np.histogram(classes, bins=hist_bins)[0] + + N_COLS = min(6, len(class_names) * 2) + + def short_name(x): + # make long class names shorter. useful for lvis + if len(x) > 13: + return x[:11] + ".." + return x + + data = list( + itertools.chain(*[[short_name(class_names[i]), int(v)] for i, v in enumerate(histogram)]) + ) + total_num_instances = sum(data[1::2]) + data.extend([None] * (N_COLS - (len(data) % N_COLS))) + if num_classes > 1: + data.extend(["total", total_num_instances]) + data = itertools.zip_longest(*[data[i::N_COLS] for i in range(N_COLS)]) + table = tabulate( + data, + headers=["category", "#instances"] * (N_COLS // 2), + tablefmt="pipe", + numalign="left", + stralign="center", + ) + log_first_n( + logging.INFO, + "Distribution of instances among all {} categories:\n".format(num_classes) + + colored(table, "cyan"), + key="message", + ) + + +def get_detection_dataset_dicts( + names, + filter_empty=True, + min_keypoints=0, + proposal_files=None, + check_consistency=True, +): + """ + Load and prepare dataset dicts for instance detection/segmentation and semantic segmentation. + + Args: + names (str or list[str]): a dataset name or a list of dataset names + filter_empty (bool): whether to filter out images without instance annotations + min_keypoints (int): filter out images with fewer keypoints than + `min_keypoints`. Set to 0 to do nothing. + proposal_files (list[str]): if given, a list of object proposal files + that match each dataset in `names`. + check_consistency (bool): whether to check if datasets have consistent metadata. + + Returns: + list[dict]: a list of dicts following the standard dataset dict format. + """ + if isinstance(names, str): + names = [names] + assert len(names), names + dataset_dicts = [DatasetCatalog.get(dataset_name) for dataset_name in names] + + if isinstance(dataset_dicts[0], torchdata.Dataset): + if len(dataset_dicts) > 1: + # ConcatDataset does not work for iterable style dataset. + # We could support concat for iterable as well, but it's often + # not a good idea to concat iterables anyway. + return torchdata.ConcatDataset(dataset_dicts) + return dataset_dicts[0] + + for dataset_name, dicts in zip(names, dataset_dicts): + assert len(dicts), "Dataset '{}' is empty!".format(dataset_name) + + if proposal_files is not None: + assert len(names) == len(proposal_files) + # load precomputed proposals from proposal files + dataset_dicts = [ + load_proposals_into_dataset(dataset_i_dicts, proposal_file) + for dataset_i_dicts, proposal_file in zip(dataset_dicts, proposal_files) + ] + + dataset_dicts = list(itertools.chain.from_iterable(dataset_dicts)) + + has_instances = "annotations" in dataset_dicts[0] + if filter_empty and has_instances: + dataset_dicts = filter_images_with_only_crowd_annotations(dataset_dicts) + if min_keypoints > 0 and has_instances: + dataset_dicts = filter_images_with_few_keypoints(dataset_dicts, min_keypoints) + + if check_consistency and has_instances: + try: + class_names = MetadataCatalog.get(names[0]).thing_classes + check_metadata_consistency("thing_classes", names) + print_instances_class_histogram(dataset_dicts, class_names) + except AttributeError: # class names are not available for this dataset + pass + + assert len(dataset_dicts), "No valid data found in {}.".format(",".join(names)) + return dataset_dicts + + +def build_batch_data_loader( + dataset, + sampler, + total_batch_size, + *, + aspect_ratio_grouping=False, + num_workers=0, + collate_fn=None, +): + """ + Build a batched dataloader. The main differences from `torch.utils.data.DataLoader` are: + 1. support aspect ratio grouping options + 2. use no "batch collation", because this is common for detection training + + Args: + dataset (torch.utils.data.Dataset): a pytorch map-style or iterable dataset. + sampler (torch.utils.data.sampler.Sampler or None): a sampler that produces indices. + Must be provided iff. ``dataset`` is a map-style dataset. + total_batch_size, aspect_ratio_grouping, num_workers, collate_fn: see + :func:`build_detection_train_loader`. + + Returns: + iterable[list]. Length of each list is the batch size of the current + GPU. Each element in the list comes from the dataset. + """ + world_size = get_world_size() + assert ( + total_batch_size > 0 and total_batch_size % world_size == 0 + ), "Total batch size ({}) must be divisible by the number of gpus ({}).".format( + total_batch_size, world_size + ) + batch_size = total_batch_size // world_size + + if isinstance(dataset, torchdata.IterableDataset): + assert sampler is None, "sampler must be None if dataset is IterableDataset" + else: + dataset = ToIterableDataset(dataset, sampler) + + if aspect_ratio_grouping: + data_loader = torchdata.DataLoader( + dataset, + num_workers=num_workers, + collate_fn=operator.itemgetter(0), # don't batch, but yield individual elements + worker_init_fn=worker_init_reset_seed, + ) # yield individual mapped dict + data_loader = AspectRatioGroupedDataset(data_loader, batch_size) + if collate_fn is None: + return data_loader + return MapDataset(data_loader, collate_fn) + else: + return torchdata.DataLoader( + dataset, + batch_size=batch_size, + drop_last=True, + num_workers=num_workers, + collate_fn=trivial_batch_collator if collate_fn is None else collate_fn, + worker_init_fn=worker_init_reset_seed, + ) + + +def _train_loader_from_config(cfg, mapper=None, *, dataset=None, sampler=None): + if dataset is None: + dataset = get_detection_dataset_dicts( + cfg.DATASETS.TRAIN, + filter_empty=cfg.DATALOADER.FILTER_EMPTY_ANNOTATIONS, + min_keypoints=cfg.MODEL.ROI_KEYPOINT_HEAD.MIN_KEYPOINTS_PER_IMAGE + if cfg.MODEL.KEYPOINT_ON + else 0, + proposal_files=cfg.DATASETS.PROPOSAL_FILES_TRAIN if cfg.MODEL.LOAD_PROPOSALS else None, + ) + _log_api_usage("dataset." + cfg.DATASETS.TRAIN[0]) + + if mapper is None: + mapper = DatasetMapper(cfg, True) + + if sampler is None: + sampler_name = cfg.DATALOADER.SAMPLER_TRAIN + logger = logging.getLogger(__name__) + if isinstance(dataset, torchdata.IterableDataset): + logger.info("Not using any sampler since the dataset is IterableDataset.") + sampler = None + else: + logger.info("Using training sampler {}".format(sampler_name)) + if sampler_name == "TrainingSampler": + sampler = TrainingSampler(len(dataset)) + elif sampler_name == "RepeatFactorTrainingSampler": + repeat_factors = RepeatFactorTrainingSampler.repeat_factors_from_category_frequency( + dataset, cfg.DATALOADER.REPEAT_THRESHOLD + ) + sampler = RepeatFactorTrainingSampler(repeat_factors) + elif sampler_name == "RandomSubsetTrainingSampler": + sampler = RandomSubsetTrainingSampler( + len(dataset), cfg.DATALOADER.RANDOM_SUBSET_RATIO + ) + else: + raise ValueError("Unknown training sampler: {}".format(sampler_name)) + + return { + "dataset": dataset, + "sampler": sampler, + "mapper": mapper, + "total_batch_size": cfg.SOLVER.IMS_PER_BATCH, + "aspect_ratio_grouping": cfg.DATALOADER.ASPECT_RATIO_GROUPING, + "num_workers": cfg.DATALOADER.NUM_WORKERS, + } + + +@configurable(from_config=_train_loader_from_config) +def build_detection_train_loader( + dataset, + *, + mapper, + sampler=None, + total_batch_size, + aspect_ratio_grouping=True, + num_workers=0, + collate_fn=None, +): + """ + Build a dataloader for object detection with some default features. + + Args: + dataset (list or torch.utils.data.Dataset): a list of dataset dicts, + or a pytorch dataset (either map-style or iterable). It can be obtained + by using :func:`DatasetCatalog.get` or :func:`get_detection_dataset_dicts`. + mapper (callable): a callable which takes a sample (dict) from dataset and + returns the format to be consumed by the model. + When using cfg, the default choice is ``DatasetMapper(cfg, is_train=True)``. + sampler (torch.utils.data.sampler.Sampler or None): a sampler that produces + indices to be applied on ``dataset``. + If ``dataset`` is map-style, the default sampler is a :class:`TrainingSampler`, + which coordinates an infinite random shuffle sequence across all workers. + Sampler must be None if ``dataset`` is iterable. + total_batch_size (int): total batch size across all workers. + aspect_ratio_grouping (bool): whether to group images with similar + aspect ratio for efficiency. When enabled, it requires each + element in dataset be a dict with keys "width" and "height". + num_workers (int): number of parallel data loading workers + collate_fn: a function that determines how to do batching, same as the argument of + `torch.utils.data.DataLoader`. Defaults to do no collation and return a list of + data. No collation is OK for small batch size and simple data structures. + If your batch size is large and each sample contains too many small tensors, + it's more efficient to collate them in data loader. + + Returns: + torch.utils.data.DataLoader: + a dataloader. Each output from it is a ``list[mapped_element]`` of length + ``total_batch_size / num_workers``, where ``mapped_element`` is produced + by the ``mapper``. + """ + if isinstance(dataset, list): + dataset = DatasetFromList(dataset, copy=False) + if mapper is not None: + dataset = MapDataset(dataset, mapper) + + if isinstance(dataset, torchdata.IterableDataset): + assert sampler is None, "sampler must be None if dataset is IterableDataset" + else: + if sampler is None: + sampler = TrainingSampler(len(dataset)) + assert isinstance(sampler, torchdata.Sampler), f"Expect a Sampler but got {type(sampler)}" + return build_batch_data_loader( + dataset, + sampler, + total_batch_size, + aspect_ratio_grouping=aspect_ratio_grouping, + num_workers=num_workers, + collate_fn=collate_fn, + ) + + +def _test_loader_from_config(cfg, dataset_name, mapper=None): + """ + Uses the given `dataset_name` argument (instead of the names in cfg), because the + standard practice is to evaluate each test set individually (not combining them). + """ + if isinstance(dataset_name, str): + dataset_name = [dataset_name] + + dataset = get_detection_dataset_dicts( + dataset_name, + filter_empty=False, + proposal_files=[ + cfg.DATASETS.PROPOSAL_FILES_TEST[list(cfg.DATASETS.TEST).index(x)] for x in dataset_name + ] + if cfg.MODEL.LOAD_PROPOSALS + else None, + ) + if mapper is None: + mapper = DatasetMapper(cfg, False) + return { + "dataset": dataset, + "mapper": mapper, + "num_workers": cfg.DATALOADER.NUM_WORKERS, + "sampler": InferenceSampler(len(dataset)) + if not isinstance(dataset, torchdata.IterableDataset) + else None, + } + + +@configurable(from_config=_test_loader_from_config) +def build_detection_test_loader( + dataset: Union[List[Any], torchdata.Dataset], + *, + mapper: Callable[[Dict[str, Any]], Any], + sampler: Optional[torchdata.Sampler] = None, + batch_size: int = 1, + num_workers: int = 0, + collate_fn: Optional[Callable[[List[Any]], Any]] = None, +) -> torchdata.DataLoader: + """ + Similar to `build_detection_train_loader`, with default batch size = 1, + and sampler = :class:`InferenceSampler`. This sampler coordinates all workers + to produce the exact set of all samples. + + Args: + dataset: a list of dataset dicts, + or a pytorch dataset (either map-style or iterable). They can be obtained + by using :func:`DatasetCatalog.get` or :func:`get_detection_dataset_dicts`. + mapper: a callable which takes a sample (dict) from dataset + and returns the format to be consumed by the model. + When using cfg, the default choice is ``DatasetMapper(cfg, is_train=False)``. + sampler: a sampler that produces + indices to be applied on ``dataset``. Default to :class:`InferenceSampler`, + which splits the dataset across all workers. Sampler must be None + if `dataset` is iterable. + batch_size: the batch size of the data loader to be created. + Default to 1 image per worker since this is the standard when reporting + inference time in papers. + num_workers: number of parallel data loading workers + collate_fn: same as the argument of `torch.utils.data.DataLoader`. + Defaults to do no collation and return a list of data. + + Returns: + DataLoader: a torch DataLoader, that loads the given detection + dataset, with test-time transformation and batching. + + Examples: + :: + data_loader = build_detection_test_loader( + DatasetRegistry.get("my_test"), + mapper=DatasetMapper(...)) + + # or, instantiate with a CfgNode: + data_loader = build_detection_test_loader(cfg, "my_test") + """ + if isinstance(dataset, list): + dataset = DatasetFromList(dataset, copy=False) + if mapper is not None: + dataset = MapDataset(dataset, mapper) + if isinstance(dataset, torchdata.IterableDataset): + assert sampler is None, "sampler must be None if dataset is IterableDataset" + else: + if sampler is None: + sampler = InferenceSampler(len(dataset)) + return torchdata.DataLoader( + dataset, + batch_size=batch_size, + sampler=sampler, + drop_last=False, + num_workers=num_workers, + collate_fn=trivial_batch_collator if collate_fn is None else collate_fn, + ) + + +def trivial_batch_collator(batch): + """ + A batch collator that does nothing. + """ + return batch + + +def worker_init_reset_seed(worker_id): + initial_seed = torch.initial_seed() % 2**31 + seed_all_rng(initial_seed + worker_id) diff --git a/annotator/oneformer/detectron2/data/catalog.py b/annotator/oneformer/detectron2/data/catalog.py new file mode 100644 index 0000000000000000000000000000000000000000..4f5209b5583d01258437bdc9b52a3dd716bdbbf6 --- /dev/null +++ b/annotator/oneformer/detectron2/data/catalog.py @@ -0,0 +1,236 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +import logging +import types +from collections import UserDict +from typing import List + +from annotator.oneformer.detectron2.utils.logger import log_first_n + +__all__ = ["DatasetCatalog", "MetadataCatalog", "Metadata"] + + +class _DatasetCatalog(UserDict): + """ + A global dictionary that stores information about the datasets and how to obtain them. + + It contains a mapping from strings + (which are names that identify a dataset, e.g. "coco_2014_train") + to a function which parses the dataset and returns the samples in the + format of `list[dict]`. + + The returned dicts should be in Detectron2 Dataset format (See DATASETS.md for details) + if used with the data loader functionalities in `data/build.py,data/detection_transform.py`. + + The purpose of having this catalog is to make it easy to choose + different datasets, by just using the strings in the config. + """ + + def register(self, name, func): + """ + Args: + name (str): the name that identifies a dataset, e.g. "coco_2014_train". + func (callable): a callable which takes no arguments and returns a list of dicts. + It must return the same results if called multiple times. + """ + assert callable(func), "You must register a function with `DatasetCatalog.register`!" + assert name not in self, "Dataset '{}' is already registered!".format(name) + self[name] = func + + def get(self, name): + """ + Call the registered function and return its results. + + Args: + name (str): the name that identifies a dataset, e.g. "coco_2014_train". + + Returns: + list[dict]: dataset annotations. + """ + try: + f = self[name] + except KeyError as e: + raise KeyError( + "Dataset '{}' is not registered! Available datasets are: {}".format( + name, ", ".join(list(self.keys())) + ) + ) from e + return f() + + def list(self) -> List[str]: + """ + List all registered datasets. + + Returns: + list[str] + """ + return list(self.keys()) + + def remove(self, name): + """ + Alias of ``pop``. + """ + self.pop(name) + + def __str__(self): + return "DatasetCatalog(registered datasets: {})".format(", ".join(self.keys())) + + __repr__ = __str__ + + +DatasetCatalog = _DatasetCatalog() +DatasetCatalog.__doc__ = ( + _DatasetCatalog.__doc__ + + """ + .. automethod:: detectron2.data.catalog.DatasetCatalog.register + .. automethod:: detectron2.data.catalog.DatasetCatalog.get +""" +) + + +class Metadata(types.SimpleNamespace): + """ + A class that supports simple attribute setter/getter. + It is intended for storing metadata of a dataset and make it accessible globally. + + Examples: + :: + # somewhere when you load the data: + MetadataCatalog.get("mydataset").thing_classes = ["person", "dog"] + + # somewhere when you print statistics or visualize: + classes = MetadataCatalog.get("mydataset").thing_classes + """ + + # the name of the dataset + # set default to N/A so that `self.name` in the errors will not trigger getattr again + name: str = "N/A" + + _RENAMED = { + "class_names": "thing_classes", + "dataset_id_to_contiguous_id": "thing_dataset_id_to_contiguous_id", + "stuff_class_names": "stuff_classes", + } + + def __getattr__(self, key): + if key in self._RENAMED: + log_first_n( + logging.WARNING, + "Metadata '{}' was renamed to '{}'!".format(key, self._RENAMED[key]), + n=10, + ) + return getattr(self, self._RENAMED[key]) + + # "name" exists in every metadata + if len(self.__dict__) > 1: + raise AttributeError( + "Attribute '{}' does not exist in the metadata of dataset '{}'. Available " + "keys are {}.".format(key, self.name, str(self.__dict__.keys())) + ) + else: + raise AttributeError( + f"Attribute '{key}' does not exist in the metadata of dataset '{self.name}': " + "metadata is empty." + ) + + def __setattr__(self, key, val): + if key in self._RENAMED: + log_first_n( + logging.WARNING, + "Metadata '{}' was renamed to '{}'!".format(key, self._RENAMED[key]), + n=10, + ) + setattr(self, self._RENAMED[key], val) + + # Ensure that metadata of the same name stays consistent + try: + oldval = getattr(self, key) + assert oldval == val, ( + "Attribute '{}' in the metadata of '{}' cannot be set " + "to a different value!\n{} != {}".format(key, self.name, oldval, val) + ) + except AttributeError: + super().__setattr__(key, val) + + def as_dict(self): + """ + Returns all the metadata as a dict. + Note that modifications to the returned dict will not reflect on the Metadata object. + """ + return copy.copy(self.__dict__) + + def set(self, **kwargs): + """ + Set multiple metadata with kwargs. + """ + for k, v in kwargs.items(): + setattr(self, k, v) + return self + + def get(self, key, default=None): + """ + Access an attribute and return its value if exists. + Otherwise return default. + """ + try: + return getattr(self, key) + except AttributeError: + return default + + +class _MetadataCatalog(UserDict): + """ + MetadataCatalog is a global dictionary that provides access to + :class:`Metadata` of a given dataset. + + The metadata associated with a certain name is a singleton: once created, the + metadata will stay alive and will be returned by future calls to ``get(name)``. + + It's like global variables, so don't abuse it. + It's meant for storing knowledge that's constant and shared across the execution + of the program, e.g.: the class names in COCO. + """ + + def get(self, name): + """ + Args: + name (str): name of a dataset (e.g. coco_2014_train). + + Returns: + Metadata: The :class:`Metadata` instance associated with this name, + or create an empty one if none is available. + """ + assert len(name) + r = super().get(name, None) + if r is None: + r = self[name] = Metadata(name=name) + return r + + def list(self): + """ + List all registered metadata. + + Returns: + list[str]: keys (names of datasets) of all registered metadata + """ + return list(self.keys()) + + def remove(self, name): + """ + Alias of ``pop``. + """ + self.pop(name) + + def __str__(self): + return "MetadataCatalog(registered metadata: {})".format(", ".join(self.keys())) + + __repr__ = __str__ + + +MetadataCatalog = _MetadataCatalog() +MetadataCatalog.__doc__ = ( + _MetadataCatalog.__doc__ + + """ + .. automethod:: detectron2.data.catalog.MetadataCatalog.get +""" +) diff --git a/annotator/oneformer/detectron2/data/common.py b/annotator/oneformer/detectron2/data/common.py new file mode 100644 index 0000000000000000000000000000000000000000..aa69a6a6546030aee818b195a0fbb399d5b776f6 --- /dev/null +++ b/annotator/oneformer/detectron2/data/common.py @@ -0,0 +1,301 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import contextlib +import copy +import itertools +import logging +import numpy as np +import pickle +import random +from typing import Callable, Union +import torch +import torch.utils.data as data +from torch.utils.data.sampler import Sampler + +from annotator.oneformer.detectron2.utils.serialize import PicklableWrapper + +__all__ = ["MapDataset", "DatasetFromList", "AspectRatioGroupedDataset", "ToIterableDataset"] + +logger = logging.getLogger(__name__) + + +def _shard_iterator_dataloader_worker(iterable): + # Shard the iterable if we're currently inside pytorch dataloader worker. + worker_info = data.get_worker_info() + if worker_info is None or worker_info.num_workers == 1: + # do nothing + yield from iterable + else: + yield from itertools.islice(iterable, worker_info.id, None, worker_info.num_workers) + + +class _MapIterableDataset(data.IterableDataset): + """ + Map a function over elements in an IterableDataset. + + Similar to pytorch's MapIterDataPipe, but support filtering when map_func + returns None. + + This class is not public-facing. Will be called by `MapDataset`. + """ + + def __init__(self, dataset, map_func): + self._dataset = dataset + self._map_func = PicklableWrapper(map_func) # wrap so that a lambda will work + + def __len__(self): + return len(self._dataset) + + def __iter__(self): + for x in map(self._map_func, self._dataset): + if x is not None: + yield x + + +class MapDataset(data.Dataset): + """ + Map a function over the elements in a dataset. + """ + + def __init__(self, dataset, map_func): + """ + Args: + dataset: a dataset where map function is applied. Can be either + map-style or iterable dataset. When given an iterable dataset, + the returned object will also be an iterable dataset. + map_func: a callable which maps the element in dataset. map_func can + return None to skip the data (e.g. in case of errors). + How None is handled depends on the style of `dataset`. + If `dataset` is map-style, it randomly tries other elements. + If `dataset` is iterable, it skips the data and tries the next. + """ + self._dataset = dataset + self._map_func = PicklableWrapper(map_func) # wrap so that a lambda will work + + self._rng = random.Random(42) + self._fallback_candidates = set(range(len(dataset))) + + def __new__(cls, dataset, map_func): + is_iterable = isinstance(dataset, data.IterableDataset) + if is_iterable: + return _MapIterableDataset(dataset, map_func) + else: + return super().__new__(cls) + + def __getnewargs__(self): + return self._dataset, self._map_func + + def __len__(self): + return len(self._dataset) + + def __getitem__(self, idx): + retry_count = 0 + cur_idx = int(idx) + + while True: + data = self._map_func(self._dataset[cur_idx]) + if data is not None: + self._fallback_candidates.add(cur_idx) + return data + + # _map_func fails for this idx, use a random new index from the pool + retry_count += 1 + self._fallback_candidates.discard(cur_idx) + cur_idx = self._rng.sample(self._fallback_candidates, k=1)[0] + + if retry_count >= 3: + logger = logging.getLogger(__name__) + logger.warning( + "Failed to apply `_map_func` for idx: {}, retry count: {}".format( + idx, retry_count + ) + ) + + +class _TorchSerializedList(object): + """ + A list-like object whose items are serialized and stored in a torch tensor. When + launching a process that uses TorchSerializedList with "fork" start method, + the subprocess can read the same buffer without triggering copy-on-access. When + launching a process that uses TorchSerializedList with "spawn/forkserver" start + method, the list will be pickled by a special ForkingPickler registered by PyTorch + that moves data to shared memory. In both cases, this allows parent and child + processes to share RAM for the list data, hence avoids the issue in + https://github.com/pytorch/pytorch/issues/13246. + + See also https://ppwwyyxx.com/blog/2022/Demystify-RAM-Usage-in-Multiprocess-DataLoader/ + on how it works. + """ + + def __init__(self, lst: list): + self._lst = lst + + def _serialize(data): + buffer = pickle.dumps(data, protocol=-1) + return np.frombuffer(buffer, dtype=np.uint8) + + logger.info( + "Serializing {} elements to byte tensors and concatenating them all ...".format( + len(self._lst) + ) + ) + self._lst = [_serialize(x) for x in self._lst] + self._addr = np.asarray([len(x) for x in self._lst], dtype=np.int64) + self._addr = torch.from_numpy(np.cumsum(self._addr)) + self._lst = torch.from_numpy(np.concatenate(self._lst)) + logger.info("Serialized dataset takes {:.2f} MiB".format(len(self._lst) / 1024**2)) + + def __len__(self): + return len(self._addr) + + def __getitem__(self, idx): + start_addr = 0 if idx == 0 else self._addr[idx - 1].item() + end_addr = self._addr[idx].item() + bytes = memoryview(self._lst[start_addr:end_addr].numpy()) + + # @lint-ignore PYTHONPICKLEISBAD + return pickle.loads(bytes) + + +_DEFAULT_DATASET_FROM_LIST_SERIALIZE_METHOD = _TorchSerializedList + + +@contextlib.contextmanager +def set_default_dataset_from_list_serialize_method(new): + """ + Context manager for using custom serialize function when creating DatasetFromList + """ + + global _DEFAULT_DATASET_FROM_LIST_SERIALIZE_METHOD + orig = _DEFAULT_DATASET_FROM_LIST_SERIALIZE_METHOD + _DEFAULT_DATASET_FROM_LIST_SERIALIZE_METHOD = new + yield + _DEFAULT_DATASET_FROM_LIST_SERIALIZE_METHOD = orig + + +class DatasetFromList(data.Dataset): + """ + Wrap a list to a torch Dataset. It produces elements of the list as data. + """ + + def __init__( + self, + lst: list, + copy: bool = True, + serialize: Union[bool, Callable] = True, + ): + """ + Args: + lst (list): a list which contains elements to produce. + copy (bool): whether to deepcopy the element when producing it, + so that the result can be modified in place without affecting the + source in the list. + serialize (bool or callable): whether to serialize the stroage to other + backend. If `True`, the default serialize method will be used, if given + a callable, the callable will be used as serialize method. + """ + self._lst = lst + self._copy = copy + if not isinstance(serialize, (bool, Callable)): + raise TypeError(f"Unsupported type for argument `serailzie`: {serialize}") + self._serialize = serialize is not False + + if self._serialize: + serialize_method = ( + serialize + if isinstance(serialize, Callable) + else _DEFAULT_DATASET_FROM_LIST_SERIALIZE_METHOD + ) + logger.info(f"Serializing the dataset using: {serialize_method}") + self._lst = serialize_method(self._lst) + + def __len__(self): + return len(self._lst) + + def __getitem__(self, idx): + if self._copy and not self._serialize: + return copy.deepcopy(self._lst[idx]) + else: + return self._lst[idx] + + +class ToIterableDataset(data.IterableDataset): + """ + Convert an old indices-based (also called map-style) dataset + to an iterable-style dataset. + """ + + def __init__(self, dataset: data.Dataset, sampler: Sampler, shard_sampler: bool = True): + """ + Args: + dataset: an old-style dataset with ``__getitem__`` + sampler: a cheap iterable that produces indices to be applied on ``dataset``. + shard_sampler: whether to shard the sampler based on the current pytorch data loader + worker id. When an IterableDataset is forked by pytorch's DataLoader into multiple + workers, it is responsible for sharding its data based on worker id so that workers + don't produce identical data. + + Most samplers (like our TrainingSampler) do not shard based on dataloader worker id + and this argument should be set to True. But certain samplers may be already + sharded, in that case this argument should be set to False. + """ + assert not isinstance(dataset, data.IterableDataset), dataset + assert isinstance(sampler, Sampler), sampler + self.dataset = dataset + self.sampler = sampler + self.shard_sampler = shard_sampler + + def __iter__(self): + if not self.shard_sampler: + sampler = self.sampler + else: + # With map-style dataset, `DataLoader(dataset, sampler)` runs the + # sampler in main process only. But `DataLoader(ToIterableDataset(dataset, sampler))` + # will run sampler in every of the N worker. So we should only keep 1/N of the ids on + # each worker. The assumption is that sampler is cheap to iterate so it's fine to + # discard ids in workers. + sampler = _shard_iterator_dataloader_worker(self.sampler) + for idx in sampler: + yield self.dataset[idx] + + def __len__(self): + return len(self.sampler) + + +class AspectRatioGroupedDataset(data.IterableDataset): + """ + Batch data that have similar aspect ratio together. + In this implementation, images whose aspect ratio < (or >) 1 will + be batched together. + This improves training speed because the images then need less padding + to form a batch. + + It assumes the underlying dataset produces dicts with "width" and "height" keys. + It will then produce a list of original dicts with length = batch_size, + all with similar aspect ratios. + """ + + def __init__(self, dataset, batch_size): + """ + Args: + dataset: an iterable. Each element must be a dict with keys + "width" and "height", which will be used to batch data. + batch_size (int): + """ + self.dataset = dataset + self.batch_size = batch_size + self._buckets = [[] for _ in range(2)] + # Hard-coded two aspect ratio groups: w > h and w < h. + # Can add support for more aspect ratio groups, but doesn't seem useful + + def __iter__(self): + for d in self.dataset: + w, h = d["width"], d["height"] + bucket_id = 0 if w > h else 1 + bucket = self._buckets[bucket_id] + bucket.append(d) + if len(bucket) == self.batch_size: + data = bucket[:] + # Clear bucket first, because code after yield is not + # guaranteed to execute + del bucket[:] + yield data diff --git a/annotator/oneformer/detectron2/data/dataset_mapper.py b/annotator/oneformer/detectron2/data/dataset_mapper.py new file mode 100644 index 0000000000000000000000000000000000000000..3bb6bb1057a68bfb12e55872f391065f02023ed3 --- /dev/null +++ b/annotator/oneformer/detectron2/data/dataset_mapper.py @@ -0,0 +1,191 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +import logging +import numpy as np +from typing import List, Optional, Union +import torch + +from annotator.oneformer.detectron2.config import configurable + +from . import detection_utils as utils +from . import transforms as T + +""" +This file contains the default mapping that's applied to "dataset dicts". +""" + +__all__ = ["DatasetMapper"] + + +class DatasetMapper: + """ + A callable which takes a dataset dict in Detectron2 Dataset format, + and map it into a format used by the model. + + This is the default callable to be used to map your dataset dict into training data. + You may need to follow it to implement your own one for customized logic, + such as a different way to read or transform images. + See :doc:`/tutorials/data_loading` for details. + + The callable currently does the following: + + 1. Read the image from "file_name" + 2. Applies cropping/geometric transforms to the image and annotations + 3. Prepare data and annotations to Tensor and :class:`Instances` + """ + + @configurable + def __init__( + self, + is_train: bool, + *, + augmentations: List[Union[T.Augmentation, T.Transform]], + image_format: str, + use_instance_mask: bool = False, + use_keypoint: bool = False, + instance_mask_format: str = "polygon", + keypoint_hflip_indices: Optional[np.ndarray] = None, + precomputed_proposal_topk: Optional[int] = None, + recompute_boxes: bool = False, + ): + """ + NOTE: this interface is experimental. + + Args: + is_train: whether it's used in training or inference + augmentations: a list of augmentations or deterministic transforms to apply + image_format: an image format supported by :func:`detection_utils.read_image`. + use_instance_mask: whether to process instance segmentation annotations, if available + use_keypoint: whether to process keypoint annotations if available + instance_mask_format: one of "polygon" or "bitmask". Process instance segmentation + masks into this format. + keypoint_hflip_indices: see :func:`detection_utils.create_keypoint_hflip_indices` + precomputed_proposal_topk: if given, will load pre-computed + proposals from dataset_dict and keep the top k proposals for each image. + recompute_boxes: whether to overwrite bounding box annotations + by computing tight bounding boxes from instance mask annotations. + """ + if recompute_boxes: + assert use_instance_mask, "recompute_boxes requires instance masks" + # fmt: off + self.is_train = is_train + self.augmentations = T.AugmentationList(augmentations) + self.image_format = image_format + self.use_instance_mask = use_instance_mask + self.instance_mask_format = instance_mask_format + self.use_keypoint = use_keypoint + self.keypoint_hflip_indices = keypoint_hflip_indices + self.proposal_topk = precomputed_proposal_topk + self.recompute_boxes = recompute_boxes + # fmt: on + logger = logging.getLogger(__name__) + mode = "training" if is_train else "inference" + logger.info(f"[DatasetMapper] Augmentations used in {mode}: {augmentations}") + + @classmethod + def from_config(cls, cfg, is_train: bool = True): + augs = utils.build_augmentation(cfg, is_train) + if cfg.INPUT.CROP.ENABLED and is_train: + augs.insert(0, T.RandomCrop(cfg.INPUT.CROP.TYPE, cfg.INPUT.CROP.SIZE)) + recompute_boxes = cfg.MODEL.MASK_ON + else: + recompute_boxes = False + + ret = { + "is_train": is_train, + "augmentations": augs, + "image_format": cfg.INPUT.FORMAT, + "use_instance_mask": cfg.MODEL.MASK_ON, + "instance_mask_format": cfg.INPUT.MASK_FORMAT, + "use_keypoint": cfg.MODEL.KEYPOINT_ON, + "recompute_boxes": recompute_boxes, + } + + if cfg.MODEL.KEYPOINT_ON: + ret["keypoint_hflip_indices"] = utils.create_keypoint_hflip_indices(cfg.DATASETS.TRAIN) + + if cfg.MODEL.LOAD_PROPOSALS: + ret["precomputed_proposal_topk"] = ( + cfg.DATASETS.PRECOMPUTED_PROPOSAL_TOPK_TRAIN + if is_train + else cfg.DATASETS.PRECOMPUTED_PROPOSAL_TOPK_TEST + ) + return ret + + def _transform_annotations(self, dataset_dict, transforms, image_shape): + # USER: Modify this if you want to keep them for some reason. + for anno in dataset_dict["annotations"]: + if not self.use_instance_mask: + anno.pop("segmentation", None) + if not self.use_keypoint: + anno.pop("keypoints", None) + + # USER: Implement additional transformations if you have other types of data + annos = [ + utils.transform_instance_annotations( + obj, transforms, image_shape, keypoint_hflip_indices=self.keypoint_hflip_indices + ) + for obj in dataset_dict.pop("annotations") + if obj.get("iscrowd", 0) == 0 + ] + instances = utils.annotations_to_instances( + annos, image_shape, mask_format=self.instance_mask_format + ) + + # After transforms such as cropping are applied, the bounding box may no longer + # tightly bound the object. As an example, imagine a triangle object + # [(0,0), (2,0), (0,2)] cropped by a box [(1,0),(2,2)] (XYXY format). The tight + # bounding box of the cropped triangle should be [(1,0),(2,1)], which is not equal to + # the intersection of original bounding box and the cropping box. + if self.recompute_boxes: + instances.gt_boxes = instances.gt_masks.get_bounding_boxes() + dataset_dict["instances"] = utils.filter_empty_instances(instances) + + def __call__(self, dataset_dict): + """ + Args: + dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. + + Returns: + dict: a format that builtin models in detectron2 accept + """ + dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below + # USER: Write your own image loading if it's not from a file + image = utils.read_image(dataset_dict["file_name"], format=self.image_format) + utils.check_image_size(dataset_dict, image) + + # USER: Remove if you don't do semantic/panoptic segmentation. + if "sem_seg_file_name" in dataset_dict: + sem_seg_gt = utils.read_image(dataset_dict.pop("sem_seg_file_name"), "L").squeeze(2) + else: + sem_seg_gt = None + + aug_input = T.AugInput(image, sem_seg=sem_seg_gt) + transforms = self.augmentations(aug_input) + image, sem_seg_gt = aug_input.image, aug_input.sem_seg + + image_shape = image.shape[:2] # h, w + # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, + # but not efficient on large generic data structures due to the use of pickle & mp.Queue. + # Therefore it's important to use torch.Tensor. + dataset_dict["image"] = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1))) + if sem_seg_gt is not None: + dataset_dict["sem_seg"] = torch.as_tensor(sem_seg_gt.astype("long")) + + # USER: Remove if you don't use pre-computed proposals. + # Most users would not need this feature. + if self.proposal_topk is not None: + utils.transform_proposals( + dataset_dict, image_shape, transforms, proposal_topk=self.proposal_topk + ) + + if not self.is_train: + # USER: Modify this if you want to keep them for some reason. + dataset_dict.pop("annotations", None) + dataset_dict.pop("sem_seg_file_name", None) + return dataset_dict + + if "annotations" in dataset_dict: + self._transform_annotations(dataset_dict, transforms, image_shape) + + return dataset_dict diff --git a/annotator/oneformer/detectron2/data/datasets/README.md b/annotator/oneformer/detectron2/data/datasets/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9fb3e4f7afec17137c95c78be6ef06d520ec8032 --- /dev/null +++ b/annotator/oneformer/detectron2/data/datasets/README.md @@ -0,0 +1,9 @@ + + +### Common Datasets + +The dataset implemented here do not need to load the data into the final format. +It should provide the minimal data structure needed to use the dataset, so it can be very efficient. + +For example, for an image dataset, just provide the file names and labels, but don't read the images. +Let the downstream decide how to read. diff --git a/annotator/oneformer/detectron2/data/datasets/__init__.py b/annotator/oneformer/detectron2/data/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a44bedc15e5f0e762fc4d77efd6f1b07c6ff77d0 --- /dev/null +++ b/annotator/oneformer/detectron2/data/datasets/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .coco import load_coco_json, load_sem_seg, register_coco_instances, convert_to_coco_json +from .coco_panoptic import register_coco_panoptic, register_coco_panoptic_separated +from .lvis import load_lvis_json, register_lvis_instances, get_lvis_instances_meta +from .pascal_voc import load_voc_instances, register_pascal_voc +from . import builtin as _builtin # ensure the builtin datasets are registered + + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/annotator/oneformer/detectron2/data/datasets/builtin.py b/annotator/oneformer/detectron2/data/datasets/builtin.py new file mode 100644 index 0000000000000000000000000000000000000000..39bbb1feec64f76705ba32c46f19f89f71be2ca7 --- /dev/null +++ b/annotator/oneformer/detectron2/data/datasets/builtin.py @@ -0,0 +1,259 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + + +""" +This file registers pre-defined datasets at hard-coded paths, and their metadata. + +We hard-code metadata for common datasets. This will enable: +1. Consistency check when loading the datasets +2. Use models on these standard datasets directly and run demos, + without having to download the dataset annotations + +We hard-code some paths to the dataset that's assumed to +exist in "./datasets/". + +Users SHOULD NOT use this file to create new dataset / metadata for new dataset. +To add new dataset, refer to the tutorial "docs/DATASETS.md". +""" + +import os + +from annotator.oneformer.detectron2.data import DatasetCatalog, MetadataCatalog + +from .builtin_meta import ADE20K_SEM_SEG_CATEGORIES, _get_builtin_metadata +from .cityscapes import load_cityscapes_instances, load_cityscapes_semantic +from .cityscapes_panoptic import register_all_cityscapes_panoptic +from .coco import load_sem_seg, register_coco_instances +from .coco_panoptic import register_coco_panoptic, register_coco_panoptic_separated +from .lvis import get_lvis_instances_meta, register_lvis_instances +from .pascal_voc import register_pascal_voc + +# ==== Predefined datasets and splits for COCO ========== + +_PREDEFINED_SPLITS_COCO = {} +_PREDEFINED_SPLITS_COCO["coco"] = { + "coco_2014_train": ("coco/train2014", "coco/annotations/instances_train2014.json"), + "coco_2014_val": ("coco/val2014", "coco/annotations/instances_val2014.json"), + "coco_2014_minival": ("coco/val2014", "coco/annotations/instances_minival2014.json"), + "coco_2014_valminusminival": ( + "coco/val2014", + "coco/annotations/instances_valminusminival2014.json", + ), + "coco_2017_train": ("coco/train2017", "coco/annotations/instances_train2017.json"), + "coco_2017_val": ("coco/val2017", "coco/annotations/instances_val2017.json"), + "coco_2017_test": ("coco/test2017", "coco/annotations/image_info_test2017.json"), + "coco_2017_test-dev": ("coco/test2017", "coco/annotations/image_info_test-dev2017.json"), + "coco_2017_val_100": ("coco/val2017", "coco/annotations/instances_val2017_100.json"), +} + +_PREDEFINED_SPLITS_COCO["coco_person"] = { + "keypoints_coco_2014_train": ( + "coco/train2014", + "coco/annotations/person_keypoints_train2014.json", + ), + "keypoints_coco_2014_val": ("coco/val2014", "coco/annotations/person_keypoints_val2014.json"), + "keypoints_coco_2014_minival": ( + "coco/val2014", + "coco/annotations/person_keypoints_minival2014.json", + ), + "keypoints_coco_2014_valminusminival": ( + "coco/val2014", + "coco/annotations/person_keypoints_valminusminival2014.json", + ), + "keypoints_coco_2017_train": ( + "coco/train2017", + "coco/annotations/person_keypoints_train2017.json", + ), + "keypoints_coco_2017_val": ("coco/val2017", "coco/annotations/person_keypoints_val2017.json"), + "keypoints_coco_2017_val_100": ( + "coco/val2017", + "coco/annotations/person_keypoints_val2017_100.json", + ), +} + + +_PREDEFINED_SPLITS_COCO_PANOPTIC = { + "coco_2017_train_panoptic": ( + # This is the original panoptic annotation directory + "coco/panoptic_train2017", + "coco/annotations/panoptic_train2017.json", + # This directory contains semantic annotations that are + # converted from panoptic annotations. + # It is used by PanopticFPN. + # You can use the script at detectron2/datasets/prepare_panoptic_fpn.py + # to create these directories. + "coco/panoptic_stuff_train2017", + ), + "coco_2017_val_panoptic": ( + "coco/panoptic_val2017", + "coco/annotations/panoptic_val2017.json", + "coco/panoptic_stuff_val2017", + ), + "coco_2017_val_100_panoptic": ( + "coco/panoptic_val2017_100", + "coco/annotations/panoptic_val2017_100.json", + "coco/panoptic_stuff_val2017_100", + ), +} + + +def register_all_coco(root): + for dataset_name, splits_per_dataset in _PREDEFINED_SPLITS_COCO.items(): + for key, (image_root, json_file) in splits_per_dataset.items(): + # Assume pre-defined datasets live in `./datasets`. + register_coco_instances( + key, + _get_builtin_metadata(dataset_name), + os.path.join(root, json_file) if "://" not in json_file else json_file, + os.path.join(root, image_root), + ) + + for ( + prefix, + (panoptic_root, panoptic_json, semantic_root), + ) in _PREDEFINED_SPLITS_COCO_PANOPTIC.items(): + prefix_instances = prefix[: -len("_panoptic")] + instances_meta = MetadataCatalog.get(prefix_instances) + image_root, instances_json = instances_meta.image_root, instances_meta.json_file + # The "separated" version of COCO panoptic segmentation dataset, + # e.g. used by Panoptic FPN + register_coco_panoptic_separated( + prefix, + _get_builtin_metadata("coco_panoptic_separated"), + image_root, + os.path.join(root, panoptic_root), + os.path.join(root, panoptic_json), + os.path.join(root, semantic_root), + instances_json, + ) + # The "standard" version of COCO panoptic segmentation dataset, + # e.g. used by Panoptic-DeepLab + register_coco_panoptic( + prefix, + _get_builtin_metadata("coco_panoptic_standard"), + image_root, + os.path.join(root, panoptic_root), + os.path.join(root, panoptic_json), + instances_json, + ) + + +# ==== Predefined datasets and splits for LVIS ========== + + +_PREDEFINED_SPLITS_LVIS = { + "lvis_v1": { + "lvis_v1_train": ("coco/", "lvis/lvis_v1_train.json"), + "lvis_v1_val": ("coco/", "lvis/lvis_v1_val.json"), + "lvis_v1_test_dev": ("coco/", "lvis/lvis_v1_image_info_test_dev.json"), + "lvis_v1_test_challenge": ("coco/", "lvis/lvis_v1_image_info_test_challenge.json"), + }, + "lvis_v0.5": { + "lvis_v0.5_train": ("coco/", "lvis/lvis_v0.5_train.json"), + "lvis_v0.5_val": ("coco/", "lvis/lvis_v0.5_val.json"), + "lvis_v0.5_val_rand_100": ("coco/", "lvis/lvis_v0.5_val_rand_100.json"), + "lvis_v0.5_test": ("coco/", "lvis/lvis_v0.5_image_info_test.json"), + }, + "lvis_v0.5_cocofied": { + "lvis_v0.5_train_cocofied": ("coco/", "lvis/lvis_v0.5_train_cocofied.json"), + "lvis_v0.5_val_cocofied": ("coco/", "lvis/lvis_v0.5_val_cocofied.json"), + }, +} + + +def register_all_lvis(root): + for dataset_name, splits_per_dataset in _PREDEFINED_SPLITS_LVIS.items(): + for key, (image_root, json_file) in splits_per_dataset.items(): + register_lvis_instances( + key, + get_lvis_instances_meta(dataset_name), + os.path.join(root, json_file) if "://" not in json_file else json_file, + os.path.join(root, image_root), + ) + + +# ==== Predefined splits for raw cityscapes images =========== +_RAW_CITYSCAPES_SPLITS = { + "cityscapes_fine_{task}_train": ("cityscapes/leftImg8bit/train/", "cityscapes/gtFine/train/"), + "cityscapes_fine_{task}_val": ("cityscapes/leftImg8bit/val/", "cityscapes/gtFine/val/"), + "cityscapes_fine_{task}_test": ("cityscapes/leftImg8bit/test/", "cityscapes/gtFine/test/"), +} + + +def register_all_cityscapes(root): + for key, (image_dir, gt_dir) in _RAW_CITYSCAPES_SPLITS.items(): + meta = _get_builtin_metadata("cityscapes") + image_dir = os.path.join(root, image_dir) + gt_dir = os.path.join(root, gt_dir) + + inst_key = key.format(task="instance_seg") + DatasetCatalog.register( + inst_key, + lambda x=image_dir, y=gt_dir: load_cityscapes_instances( + x, y, from_json=True, to_polygons=True + ), + ) + MetadataCatalog.get(inst_key).set( + image_dir=image_dir, gt_dir=gt_dir, evaluator_type="cityscapes_instance", **meta + ) + + sem_key = key.format(task="sem_seg") + DatasetCatalog.register( + sem_key, lambda x=image_dir, y=gt_dir: load_cityscapes_semantic(x, y) + ) + MetadataCatalog.get(sem_key).set( + image_dir=image_dir, + gt_dir=gt_dir, + evaluator_type="cityscapes_sem_seg", + ignore_label=255, + **meta, + ) + + +# ==== Predefined splits for PASCAL VOC =========== +def register_all_pascal_voc(root): + SPLITS = [ + ("voc_2007_trainval", "VOC2007", "trainval"), + ("voc_2007_train", "VOC2007", "train"), + ("voc_2007_val", "VOC2007", "val"), + ("voc_2007_test", "VOC2007", "test"), + ("voc_2012_trainval", "VOC2012", "trainval"), + ("voc_2012_train", "VOC2012", "train"), + ("voc_2012_val", "VOC2012", "val"), + ] + for name, dirname, split in SPLITS: + year = 2007 if "2007" in name else 2012 + register_pascal_voc(name, os.path.join(root, dirname), split, year) + MetadataCatalog.get(name).evaluator_type = "pascal_voc" + + +def register_all_ade20k(root): + root = os.path.join(root, "ADEChallengeData2016") + for name, dirname in [("train", "training"), ("val", "validation")]: + image_dir = os.path.join(root, "images", dirname) + gt_dir = os.path.join(root, "annotations_detectron2", dirname) + name = f"ade20k_sem_seg_{name}" + DatasetCatalog.register( + name, lambda x=image_dir, y=gt_dir: load_sem_seg(y, x, gt_ext="png", image_ext="jpg") + ) + MetadataCatalog.get(name).set( + stuff_classes=ADE20K_SEM_SEG_CATEGORIES[:], + image_root=image_dir, + sem_seg_root=gt_dir, + evaluator_type="sem_seg", + ignore_label=255, + ) + + +# True for open source; +# Internally at fb, we register them elsewhere +if __name__.endswith(".builtin"): + # Assume pre-defined datasets live in `./datasets`. + _root = os.path.expanduser(os.getenv("DETECTRON2_DATASETS", "datasets")) + register_all_coco(_root) + register_all_lvis(_root) + register_all_cityscapes(_root) + register_all_cityscapes_panoptic(_root) + register_all_pascal_voc(_root) + register_all_ade20k(_root) diff --git a/annotator/oneformer/detectron2/data/datasets/builtin_meta.py b/annotator/oneformer/detectron2/data/datasets/builtin_meta.py new file mode 100644 index 0000000000000000000000000000000000000000..63c7a1a31b31dd89b82011effee26471faccacf5 --- /dev/null +++ b/annotator/oneformer/detectron2/data/datasets/builtin_meta.py @@ -0,0 +1,350 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +""" +Note: +For your custom dataset, there is no need to hard-code metadata anywhere in the code. +For example, for COCO-format dataset, metadata will be obtained automatically +when calling `load_coco_json`. For other dataset, metadata may also be obtained in other ways +during loading. + +However, we hard-coded metadata for a few common dataset here. +The only goal is to allow users who don't have these dataset to use pre-trained models. +Users don't have to download a COCO json (which contains metadata), in order to visualize a +COCO model (with correct class names and colors). +""" + + +# All coco categories, together with their nice-looking visualization colors +# It's from https://github.com/cocodataset/panopticapi/blob/master/panoptic_coco_categories.json +COCO_CATEGORIES = [ + {"color": [220, 20, 60], "isthing": 1, "id": 1, "name": "person"}, + {"color": [119, 11, 32], "isthing": 1, "id": 2, "name": "bicycle"}, + {"color": [0, 0, 142], "isthing": 1, "id": 3, "name": "car"}, + {"color": [0, 0, 230], "isthing": 1, "id": 4, "name": "motorcycle"}, + {"color": [106, 0, 228], "isthing": 1, "id": 5, "name": "airplane"}, + {"color": [0, 60, 100], "isthing": 1, "id": 6, "name": "bus"}, + {"color": [0, 80, 100], "isthing": 1, "id": 7, "name": "train"}, + {"color": [0, 0, 70], "isthing": 1, "id": 8, "name": "truck"}, + {"color": [0, 0, 192], "isthing": 1, "id": 9, "name": "boat"}, + {"color": [250, 170, 30], "isthing": 1, "id": 10, "name": "traffic light"}, + {"color": [100, 170, 30], "isthing": 1, "id": 11, "name": "fire hydrant"}, + {"color": [220, 220, 0], "isthing": 1, "id": 13, "name": "stop sign"}, + {"color": [175, 116, 175], "isthing": 1, "id": 14, "name": "parking meter"}, + {"color": [250, 0, 30], "isthing": 1, "id": 15, "name": "bench"}, + {"color": [165, 42, 42], "isthing": 1, "id": 16, "name": "bird"}, + {"color": [255, 77, 255], "isthing": 1, "id": 17, "name": "cat"}, + {"color": [0, 226, 252], "isthing": 1, "id": 18, "name": "dog"}, + {"color": [182, 182, 255], "isthing": 1, "id": 19, "name": "horse"}, + {"color": [0, 82, 0], "isthing": 1, "id": 20, "name": "sheep"}, + {"color": [120, 166, 157], "isthing": 1, "id": 21, "name": "cow"}, + {"color": [110, 76, 0], "isthing": 1, "id": 22, "name": "elephant"}, + {"color": [174, 57, 255], "isthing": 1, "id": 23, "name": "bear"}, + {"color": [199, 100, 0], "isthing": 1, "id": 24, "name": "zebra"}, + {"color": [72, 0, 118], "isthing": 1, "id": 25, "name": "giraffe"}, + {"color": [255, 179, 240], "isthing": 1, "id": 27, "name": "backpack"}, + {"color": [0, 125, 92], "isthing": 1, "id": 28, "name": "umbrella"}, + {"color": [209, 0, 151], "isthing": 1, "id": 31, "name": "handbag"}, + {"color": [188, 208, 182], "isthing": 1, "id": 32, "name": "tie"}, + {"color": [0, 220, 176], "isthing": 1, "id": 33, "name": "suitcase"}, + {"color": [255, 99, 164], "isthing": 1, "id": 34, "name": "frisbee"}, + {"color": [92, 0, 73], "isthing": 1, "id": 35, "name": "skis"}, + {"color": [133, 129, 255], "isthing": 1, "id": 36, "name": "snowboard"}, + {"color": [78, 180, 255], "isthing": 1, "id": 37, "name": "sports ball"}, + {"color": [0, 228, 0], "isthing": 1, "id": 38, "name": "kite"}, + {"color": [174, 255, 243], "isthing": 1, "id": 39, "name": "baseball bat"}, + {"color": [45, 89, 255], "isthing": 1, "id": 40, "name": "baseball glove"}, + {"color": [134, 134, 103], "isthing": 1, "id": 41, "name": "skateboard"}, + {"color": [145, 148, 174], "isthing": 1, "id": 42, "name": "surfboard"}, + {"color": [255, 208, 186], "isthing": 1, "id": 43, "name": "tennis racket"}, + {"color": [197, 226, 255], "isthing": 1, "id": 44, "name": "bottle"}, + {"color": [171, 134, 1], "isthing": 1, "id": 46, "name": "wine glass"}, + {"color": [109, 63, 54], "isthing": 1, "id": 47, "name": "cup"}, + {"color": [207, 138, 255], "isthing": 1, "id": 48, "name": "fork"}, + {"color": [151, 0, 95], "isthing": 1, "id": 49, "name": "knife"}, + {"color": [9, 80, 61], "isthing": 1, "id": 50, "name": "spoon"}, + {"color": [84, 105, 51], "isthing": 1, "id": 51, "name": "bowl"}, + {"color": [74, 65, 105], "isthing": 1, "id": 52, "name": "banana"}, + {"color": [166, 196, 102], "isthing": 1, "id": 53, "name": "apple"}, + {"color": [208, 195, 210], "isthing": 1, "id": 54, "name": "sandwich"}, + {"color": [255, 109, 65], "isthing": 1, "id": 55, "name": "orange"}, + {"color": [0, 143, 149], "isthing": 1, "id": 56, "name": "broccoli"}, + {"color": [179, 0, 194], "isthing": 1, "id": 57, "name": "carrot"}, + {"color": [209, 99, 106], "isthing": 1, "id": 58, "name": "hot dog"}, + {"color": [5, 121, 0], "isthing": 1, "id": 59, "name": "pizza"}, + {"color": [227, 255, 205], "isthing": 1, "id": 60, "name": "donut"}, + {"color": [147, 186, 208], "isthing": 1, "id": 61, "name": "cake"}, + {"color": [153, 69, 1], "isthing": 1, "id": 62, "name": "chair"}, + {"color": [3, 95, 161], "isthing": 1, "id": 63, "name": "couch"}, + {"color": [163, 255, 0], "isthing": 1, "id": 64, "name": "potted plant"}, + {"color": [119, 0, 170], "isthing": 1, "id": 65, "name": "bed"}, + {"color": [0, 182, 199], "isthing": 1, "id": 67, "name": "dining table"}, + {"color": [0, 165, 120], "isthing": 1, "id": 70, "name": "toilet"}, + {"color": [183, 130, 88], "isthing": 1, "id": 72, "name": "tv"}, + {"color": [95, 32, 0], "isthing": 1, "id": 73, "name": "laptop"}, + {"color": [130, 114, 135], "isthing": 1, "id": 74, "name": "mouse"}, + {"color": [110, 129, 133], "isthing": 1, "id": 75, "name": "remote"}, + {"color": [166, 74, 118], "isthing": 1, "id": 76, "name": "keyboard"}, + {"color": [219, 142, 185], "isthing": 1, "id": 77, "name": "cell phone"}, + {"color": [79, 210, 114], "isthing": 1, "id": 78, "name": "microwave"}, + {"color": [178, 90, 62], "isthing": 1, "id": 79, "name": "oven"}, + {"color": [65, 70, 15], "isthing": 1, "id": 80, "name": "toaster"}, + {"color": [127, 167, 115], "isthing": 1, "id": 81, "name": "sink"}, + {"color": [59, 105, 106], "isthing": 1, "id": 82, "name": "refrigerator"}, + {"color": [142, 108, 45], "isthing": 1, "id": 84, "name": "book"}, + {"color": [196, 172, 0], "isthing": 1, "id": 85, "name": "clock"}, + {"color": [95, 54, 80], "isthing": 1, "id": 86, "name": "vase"}, + {"color": [128, 76, 255], "isthing": 1, "id": 87, "name": "scissors"}, + {"color": [201, 57, 1], "isthing": 1, "id": 88, "name": "teddy bear"}, + {"color": [246, 0, 122], "isthing": 1, "id": 89, "name": "hair drier"}, + {"color": [191, 162, 208], "isthing": 1, "id": 90, "name": "toothbrush"}, + {"color": [255, 255, 128], "isthing": 0, "id": 92, "name": "banner"}, + {"color": [147, 211, 203], "isthing": 0, "id": 93, "name": "blanket"}, + {"color": [150, 100, 100], "isthing": 0, "id": 95, "name": "bridge"}, + {"color": [168, 171, 172], "isthing": 0, "id": 100, "name": "cardboard"}, + {"color": [146, 112, 198], "isthing": 0, "id": 107, "name": "counter"}, + {"color": [210, 170, 100], "isthing": 0, "id": 109, "name": "curtain"}, + {"color": [92, 136, 89], "isthing": 0, "id": 112, "name": "door-stuff"}, + {"color": [218, 88, 184], "isthing": 0, "id": 118, "name": "floor-wood"}, + {"color": [241, 129, 0], "isthing": 0, "id": 119, "name": "flower"}, + {"color": [217, 17, 255], "isthing": 0, "id": 122, "name": "fruit"}, + {"color": [124, 74, 181], "isthing": 0, "id": 125, "name": "gravel"}, + {"color": [70, 70, 70], "isthing": 0, "id": 128, "name": "house"}, + {"color": [255, 228, 255], "isthing": 0, "id": 130, "name": "light"}, + {"color": [154, 208, 0], "isthing": 0, "id": 133, "name": "mirror-stuff"}, + {"color": [193, 0, 92], "isthing": 0, "id": 138, "name": "net"}, + {"color": [76, 91, 113], "isthing": 0, "id": 141, "name": "pillow"}, + {"color": [255, 180, 195], "isthing": 0, "id": 144, "name": "platform"}, + {"color": [106, 154, 176], "isthing": 0, "id": 145, "name": "playingfield"}, + {"color": [230, 150, 140], "isthing": 0, "id": 147, "name": "railroad"}, + {"color": [60, 143, 255], "isthing": 0, "id": 148, "name": "river"}, + {"color": [128, 64, 128], "isthing": 0, "id": 149, "name": "road"}, + {"color": [92, 82, 55], "isthing": 0, "id": 151, "name": "roof"}, + {"color": [254, 212, 124], "isthing": 0, "id": 154, "name": "sand"}, + {"color": [73, 77, 174], "isthing": 0, "id": 155, "name": "sea"}, + {"color": [255, 160, 98], "isthing": 0, "id": 156, "name": "shelf"}, + {"color": [255, 255, 255], "isthing": 0, "id": 159, "name": "snow"}, + {"color": [104, 84, 109], "isthing": 0, "id": 161, "name": "stairs"}, + {"color": [169, 164, 131], "isthing": 0, "id": 166, "name": "tent"}, + {"color": [225, 199, 255], "isthing": 0, "id": 168, "name": "towel"}, + {"color": [137, 54, 74], "isthing": 0, "id": 171, "name": "wall-brick"}, + {"color": [135, 158, 223], "isthing": 0, "id": 175, "name": "wall-stone"}, + {"color": [7, 246, 231], "isthing": 0, "id": 176, "name": "wall-tile"}, + {"color": [107, 255, 200], "isthing": 0, "id": 177, "name": "wall-wood"}, + {"color": [58, 41, 149], "isthing": 0, "id": 178, "name": "water-other"}, + {"color": [183, 121, 142], "isthing": 0, "id": 180, "name": "window-blind"}, + {"color": [255, 73, 97], "isthing": 0, "id": 181, "name": "window-other"}, + {"color": [107, 142, 35], "isthing": 0, "id": 184, "name": "tree-merged"}, + {"color": [190, 153, 153], "isthing": 0, "id": 185, "name": "fence-merged"}, + {"color": [146, 139, 141], "isthing": 0, "id": 186, "name": "ceiling-merged"}, + {"color": [70, 130, 180], "isthing": 0, "id": 187, "name": "sky-other-merged"}, + {"color": [134, 199, 156], "isthing": 0, "id": 188, "name": "cabinet-merged"}, + {"color": [209, 226, 140], "isthing": 0, "id": 189, "name": "table-merged"}, + {"color": [96, 36, 108], "isthing": 0, "id": 190, "name": "floor-other-merged"}, + {"color": [96, 96, 96], "isthing": 0, "id": 191, "name": "pavement-merged"}, + {"color": [64, 170, 64], "isthing": 0, "id": 192, "name": "mountain-merged"}, + {"color": [152, 251, 152], "isthing": 0, "id": 193, "name": "grass-merged"}, + {"color": [208, 229, 228], "isthing": 0, "id": 194, "name": "dirt-merged"}, + {"color": [206, 186, 171], "isthing": 0, "id": 195, "name": "paper-merged"}, + {"color": [152, 161, 64], "isthing": 0, "id": 196, "name": "food-other-merged"}, + {"color": [116, 112, 0], "isthing": 0, "id": 197, "name": "building-other-merged"}, + {"color": [0, 114, 143], "isthing": 0, "id": 198, "name": "rock-merged"}, + {"color": [102, 102, 156], "isthing": 0, "id": 199, "name": "wall-other-merged"}, + {"color": [250, 141, 255], "isthing": 0, "id": 200, "name": "rug-merged"}, +] + +# fmt: off +COCO_PERSON_KEYPOINT_NAMES = ( + "nose", + "left_eye", "right_eye", + "left_ear", "right_ear", + "left_shoulder", "right_shoulder", + "left_elbow", "right_elbow", + "left_wrist", "right_wrist", + "left_hip", "right_hip", + "left_knee", "right_knee", + "left_ankle", "right_ankle", +) +# fmt: on + +# Pairs of keypoints that should be exchanged under horizontal flipping +COCO_PERSON_KEYPOINT_FLIP_MAP = ( + ("left_eye", "right_eye"), + ("left_ear", "right_ear"), + ("left_shoulder", "right_shoulder"), + ("left_elbow", "right_elbow"), + ("left_wrist", "right_wrist"), + ("left_hip", "right_hip"), + ("left_knee", "right_knee"), + ("left_ankle", "right_ankle"), +) + +# rules for pairs of keypoints to draw a line between, and the line color to use. +KEYPOINT_CONNECTION_RULES = [ + # face + ("left_ear", "left_eye", (102, 204, 255)), + ("right_ear", "right_eye", (51, 153, 255)), + ("left_eye", "nose", (102, 0, 204)), + ("nose", "right_eye", (51, 102, 255)), + # upper-body + ("left_shoulder", "right_shoulder", (255, 128, 0)), + ("left_shoulder", "left_elbow", (153, 255, 204)), + ("right_shoulder", "right_elbow", (128, 229, 255)), + ("left_elbow", "left_wrist", (153, 255, 153)), + ("right_elbow", "right_wrist", (102, 255, 224)), + # lower-body + ("left_hip", "right_hip", (255, 102, 0)), + ("left_hip", "left_knee", (255, 255, 77)), + ("right_hip", "right_knee", (153, 255, 204)), + ("left_knee", "left_ankle", (191, 255, 128)), + ("right_knee", "right_ankle", (255, 195, 77)), +] + +# All Cityscapes categories, together with their nice-looking visualization colors +# It's from https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/helpers/labels.py # noqa +CITYSCAPES_CATEGORIES = [ + {"color": (128, 64, 128), "isthing": 0, "id": 7, "trainId": 0, "name": "road"}, + {"color": (244, 35, 232), "isthing": 0, "id": 8, "trainId": 1, "name": "sidewalk"}, + {"color": (70, 70, 70), "isthing": 0, "id": 11, "trainId": 2, "name": "building"}, + {"color": (102, 102, 156), "isthing": 0, "id": 12, "trainId": 3, "name": "wall"}, + {"color": (190, 153, 153), "isthing": 0, "id": 13, "trainId": 4, "name": "fence"}, + {"color": (153, 153, 153), "isthing": 0, "id": 17, "trainId": 5, "name": "pole"}, + {"color": (250, 170, 30), "isthing": 0, "id": 19, "trainId": 6, "name": "traffic light"}, + {"color": (220, 220, 0), "isthing": 0, "id": 20, "trainId": 7, "name": "traffic sign"}, + {"color": (107, 142, 35), "isthing": 0, "id": 21, "trainId": 8, "name": "vegetation"}, + {"color": (152, 251, 152), "isthing": 0, "id": 22, "trainId": 9, "name": "terrain"}, + {"color": (70, 130, 180), "isthing": 0, "id": 23, "trainId": 10, "name": "sky"}, + {"color": (220, 20, 60), "isthing": 1, "id": 24, "trainId": 11, "name": "person"}, + {"color": (255, 0, 0), "isthing": 1, "id": 25, "trainId": 12, "name": "rider"}, + {"color": (0, 0, 142), "isthing": 1, "id": 26, "trainId": 13, "name": "car"}, + {"color": (0, 0, 70), "isthing": 1, "id": 27, "trainId": 14, "name": "truck"}, + {"color": (0, 60, 100), "isthing": 1, "id": 28, "trainId": 15, "name": "bus"}, + {"color": (0, 80, 100), "isthing": 1, "id": 31, "trainId": 16, "name": "train"}, + {"color": (0, 0, 230), "isthing": 1, "id": 32, "trainId": 17, "name": "motorcycle"}, + {"color": (119, 11, 32), "isthing": 1, "id": 33, "trainId": 18, "name": "bicycle"}, +] + +# fmt: off +ADE20K_SEM_SEG_CATEGORIES = [ + "wall", "building", "sky", "floor", "tree", "ceiling", "road, route", "bed", "window ", "grass", "cabinet", "sidewalk, pavement", "person", "earth, ground", "door", "table", "mountain, mount", "plant", "curtain", "chair", "car", "water", "painting, picture", "sofa", "shelf", "house", "sea", "mirror", "rug", "field", "armchair", "seat", "fence", "desk", "rock, stone", "wardrobe, closet, press", "lamp", "tub", "rail", "cushion", "base, pedestal, stand", "box", "column, pillar", "signboard, sign", "chest of drawers, chest, bureau, dresser", "counter", "sand", "sink", "skyscraper", "fireplace", "refrigerator, icebox", "grandstand, covered stand", "path", "stairs", "runway", "case, display case, showcase, vitrine", "pool table, billiard table, snooker table", "pillow", "screen door, screen", "stairway, staircase", "river", "bridge, span", "bookcase", "blind, screen", "coffee table", "toilet, can, commode, crapper, pot, potty, stool, throne", "flower", "book", "hill", "bench", "countertop", "stove", "palm, palm tree", "kitchen island", "computer", "swivel chair", "boat", "bar", "arcade machine", "hovel, hut, hutch, shack, shanty", "bus", "towel", "light", "truck", "tower", "chandelier", "awning, sunshade, sunblind", "street lamp", "booth", "tv", "plane", "dirt track", "clothes", "pole", "land, ground, soil", "bannister, banister, balustrade, balusters, handrail", "escalator, moving staircase, moving stairway", "ottoman, pouf, pouffe, puff, hassock", "bottle", "buffet, counter, sideboard", "poster, posting, placard, notice, bill, card", "stage", "van", "ship", "fountain", "conveyer belt, conveyor belt, conveyer, conveyor, transporter", "canopy", "washer, automatic washer, washing machine", "plaything, toy", "pool", "stool", "barrel, cask", "basket, handbasket", "falls", "tent", "bag", "minibike, motorbike", "cradle", "oven", "ball", "food, solid food", "step, stair", "tank, storage tank", "trade name", "microwave", "pot", "animal", "bicycle", "lake", "dishwasher", "screen", "blanket, cover", "sculpture", "hood, exhaust hood", "sconce", "vase", "traffic light", "tray", "trash can", "fan", "pier", "crt screen", "plate", "monitor", "bulletin board", "shower", "radiator", "glass, drinking glass", "clock", "flag", # noqa +] +# After processed by `prepare_ade20k_sem_seg.py`, id 255 means ignore +# fmt: on + + +def _get_coco_instances_meta(): + thing_ids = [k["id"] for k in COCO_CATEGORIES if k["isthing"] == 1] + thing_colors = [k["color"] for k in COCO_CATEGORIES if k["isthing"] == 1] + assert len(thing_ids) == 80, len(thing_ids) + # Mapping from the incontiguous COCO category id to an id in [0, 79] + thing_dataset_id_to_contiguous_id = {k: i for i, k in enumerate(thing_ids)} + thing_classes = [k["name"] for k in COCO_CATEGORIES if k["isthing"] == 1] + ret = { + "thing_dataset_id_to_contiguous_id": thing_dataset_id_to_contiguous_id, + "thing_classes": thing_classes, + "thing_colors": thing_colors, + } + return ret + + +def _get_coco_panoptic_separated_meta(): + """ + Returns metadata for "separated" version of the panoptic segmentation dataset. + """ + stuff_ids = [k["id"] for k in COCO_CATEGORIES if k["isthing"] == 0] + assert len(stuff_ids) == 53, len(stuff_ids) + + # For semantic segmentation, this mapping maps from contiguous stuff id + # (in [0, 53], used in models) to ids in the dataset (used for processing results) + # The id 0 is mapped to an extra category "thing". + stuff_dataset_id_to_contiguous_id = {k: i + 1 for i, k in enumerate(stuff_ids)} + # When converting COCO panoptic annotations to semantic annotations + # We label the "thing" category to 0 + stuff_dataset_id_to_contiguous_id[0] = 0 + + # 54 names for COCO stuff categories (including "things") + stuff_classes = ["things"] + [ + k["name"].replace("-other", "").replace("-merged", "") + for k in COCO_CATEGORIES + if k["isthing"] == 0 + ] + + # NOTE: I randomly picked a color for things + stuff_colors = [[82, 18, 128]] + [k["color"] for k in COCO_CATEGORIES if k["isthing"] == 0] + ret = { + "stuff_dataset_id_to_contiguous_id": stuff_dataset_id_to_contiguous_id, + "stuff_classes": stuff_classes, + "stuff_colors": stuff_colors, + } + ret.update(_get_coco_instances_meta()) + return ret + + +def _get_builtin_metadata(dataset_name): + if dataset_name == "coco": + return _get_coco_instances_meta() + if dataset_name == "coco_panoptic_separated": + return _get_coco_panoptic_separated_meta() + elif dataset_name == "coco_panoptic_standard": + meta = {} + # The following metadata maps contiguous id from [0, #thing categories + + # #stuff categories) to their names and colors. We have to replica of the + # same name and color under "thing_*" and "stuff_*" because the current + # visualization function in D2 handles thing and class classes differently + # due to some heuristic used in Panoptic FPN. We keep the same naming to + # enable reusing existing visualization functions. + thing_classes = [k["name"] for k in COCO_CATEGORIES] + thing_colors = [k["color"] for k in COCO_CATEGORIES] + stuff_classes = [k["name"] for k in COCO_CATEGORIES] + stuff_colors = [k["color"] for k in COCO_CATEGORIES] + + meta["thing_classes"] = thing_classes + meta["thing_colors"] = thing_colors + meta["stuff_classes"] = stuff_classes + meta["stuff_colors"] = stuff_colors + + # Convert category id for training: + # category id: like semantic segmentation, it is the class id for each + # pixel. Since there are some classes not used in evaluation, the category + # id is not always contiguous and thus we have two set of category ids: + # - original category id: category id in the original dataset, mainly + # used for evaluation. + # - contiguous category id: [0, #classes), in order to train the linear + # softmax classifier. + thing_dataset_id_to_contiguous_id = {} + stuff_dataset_id_to_contiguous_id = {} + + for i, cat in enumerate(COCO_CATEGORIES): + if cat["isthing"]: + thing_dataset_id_to_contiguous_id[cat["id"]] = i + else: + stuff_dataset_id_to_contiguous_id[cat["id"]] = i + + meta["thing_dataset_id_to_contiguous_id"] = thing_dataset_id_to_contiguous_id + meta["stuff_dataset_id_to_contiguous_id"] = stuff_dataset_id_to_contiguous_id + + return meta + elif dataset_name == "coco_person": + return { + "thing_classes": ["person"], + "keypoint_names": COCO_PERSON_KEYPOINT_NAMES, + "keypoint_flip_map": COCO_PERSON_KEYPOINT_FLIP_MAP, + "keypoint_connection_rules": KEYPOINT_CONNECTION_RULES, + } + elif dataset_name == "cityscapes": + # fmt: off + CITYSCAPES_THING_CLASSES = [ + "person", "rider", "car", "truck", + "bus", "train", "motorcycle", "bicycle", + ] + CITYSCAPES_STUFF_CLASSES = [ + "road", "sidewalk", "building", "wall", "fence", "pole", "traffic light", + "traffic sign", "vegetation", "terrain", "sky", "person", "rider", "car", + "truck", "bus", "train", "motorcycle", "bicycle", + ] + # fmt: on + return { + "thing_classes": CITYSCAPES_THING_CLASSES, + "stuff_classes": CITYSCAPES_STUFF_CLASSES, + } + raise KeyError("No built-in metadata for dataset {}".format(dataset_name)) diff --git a/annotator/oneformer/detectron2/data/datasets/cityscapes.py b/annotator/oneformer/detectron2/data/datasets/cityscapes.py new file mode 100644 index 0000000000000000000000000000000000000000..18c3f3a8279e2511016fa61885d26dad726ffe5e --- /dev/null +++ b/annotator/oneformer/detectron2/data/datasets/cityscapes.py @@ -0,0 +1,329 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import functools +import json +import logging +import multiprocessing as mp +import numpy as np +import os +from itertools import chain +import pycocotools.mask as mask_util +from PIL import Image + +from annotator.oneformer.detectron2.structures import BoxMode +from annotator.oneformer.detectron2.utils.comm import get_world_size +from annotator.oneformer.detectron2.utils.file_io import PathManager +from annotator.oneformer.detectron2.utils.logger import setup_logger + +try: + import cv2 # noqa +except ImportError: + # OpenCV is an optional dependency at the moment + pass + + +logger = logging.getLogger(__name__) + + +def _get_cityscapes_files(image_dir, gt_dir): + files = [] + # scan through the directory + cities = PathManager.ls(image_dir) + logger.info(f"{len(cities)} cities found in '{image_dir}'.") + for city in cities: + city_img_dir = os.path.join(image_dir, city) + city_gt_dir = os.path.join(gt_dir, city) + for basename in PathManager.ls(city_img_dir): + image_file = os.path.join(city_img_dir, basename) + + suffix = "leftImg8bit.png" + assert basename.endswith(suffix), basename + basename = basename[: -len(suffix)] + + instance_file = os.path.join(city_gt_dir, basename + "gtFine_instanceIds.png") + label_file = os.path.join(city_gt_dir, basename + "gtFine_labelIds.png") + json_file = os.path.join(city_gt_dir, basename + "gtFine_polygons.json") + + files.append((image_file, instance_file, label_file, json_file)) + assert len(files), "No images found in {}".format(image_dir) + for f in files[0]: + assert PathManager.isfile(f), f + return files + + +def load_cityscapes_instances(image_dir, gt_dir, from_json=True, to_polygons=True): + """ + Args: + image_dir (str): path to the raw dataset. e.g., "~/cityscapes/leftImg8bit/train". + gt_dir (str): path to the raw annotations. e.g., "~/cityscapes/gtFine/train". + from_json (bool): whether to read annotations from the raw json file or the png files. + to_polygons (bool): whether to represent the segmentation as polygons + (COCO's format) instead of masks (cityscapes's format). + + Returns: + list[dict]: a list of dicts in Detectron2 standard format. (See + `Using Custom Datasets `_ ) + """ + if from_json: + assert to_polygons, ( + "Cityscapes's json annotations are in polygon format. " + "Converting to mask format is not supported now." + ) + files = _get_cityscapes_files(image_dir, gt_dir) + + logger.info("Preprocessing cityscapes annotations ...") + # This is still not fast: all workers will execute duplicate works and will + # take up to 10m on a 8GPU server. + pool = mp.Pool(processes=max(mp.cpu_count() // get_world_size() // 2, 4)) + + ret = pool.map( + functools.partial(_cityscapes_files_to_dict, from_json=from_json, to_polygons=to_polygons), + files, + ) + logger.info("Loaded {} images from {}".format(len(ret), image_dir)) + + # Map cityscape ids to contiguous ids + from cityscapesscripts.helpers.labels import labels + + labels = [l for l in labels if l.hasInstances and not l.ignoreInEval] + dataset_id_to_contiguous_id = {l.id: idx for idx, l in enumerate(labels)} + for dict_per_image in ret: + for anno in dict_per_image["annotations"]: + anno["category_id"] = dataset_id_to_contiguous_id[anno["category_id"]] + return ret + + +def load_cityscapes_semantic(image_dir, gt_dir): + """ + Args: + image_dir (str): path to the raw dataset. e.g., "~/cityscapes/leftImg8bit/train". + gt_dir (str): path to the raw annotations. e.g., "~/cityscapes/gtFine/train". + + Returns: + list[dict]: a list of dict, each has "file_name" and + "sem_seg_file_name". + """ + ret = [] + # gt_dir is small and contain many small files. make sense to fetch to local first + gt_dir = PathManager.get_local_path(gt_dir) + for image_file, _, label_file, json_file in _get_cityscapes_files(image_dir, gt_dir): + label_file = label_file.replace("labelIds", "labelTrainIds") + + with PathManager.open(json_file, "r") as f: + jsonobj = json.load(f) + ret.append( + { + "file_name": image_file, + "sem_seg_file_name": label_file, + "height": jsonobj["imgHeight"], + "width": jsonobj["imgWidth"], + } + ) + assert len(ret), f"No images found in {image_dir}!" + assert PathManager.isfile( + ret[0]["sem_seg_file_name"] + ), "Please generate labelTrainIds.png with cityscapesscripts/preparation/createTrainIdLabelImgs.py" # noqa + return ret + + +def _cityscapes_files_to_dict(files, from_json, to_polygons): + """ + Parse cityscapes annotation files to a instance segmentation dataset dict. + + Args: + files (tuple): consists of (image_file, instance_id_file, label_id_file, json_file) + from_json (bool): whether to read annotations from the raw json file or the png files. + to_polygons (bool): whether to represent the segmentation as polygons + (COCO's format) instead of masks (cityscapes's format). + + Returns: + A dict in Detectron2 Dataset format. + """ + from cityscapesscripts.helpers.labels import id2label, name2label + + image_file, instance_id_file, _, json_file = files + + annos = [] + + if from_json: + from shapely.geometry import MultiPolygon, Polygon + + with PathManager.open(json_file, "r") as f: + jsonobj = json.load(f) + ret = { + "file_name": image_file, + "image_id": os.path.basename(image_file), + "height": jsonobj["imgHeight"], + "width": jsonobj["imgWidth"], + } + + # `polygons_union` contains the union of all valid polygons. + polygons_union = Polygon() + + # CityscapesScripts draw the polygons in sequential order + # and each polygon *overwrites* existing ones. See + # (https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/preparation/json2instanceImg.py) # noqa + # We use reverse order, and each polygon *avoids* early ones. + # This will resolve the ploygon overlaps in the same way as CityscapesScripts. + for obj in jsonobj["objects"][::-1]: + if "deleted" in obj: # cityscapes data format specific + continue + label_name = obj["label"] + + try: + label = name2label[label_name] + except KeyError: + if label_name.endswith("group"): # crowd area + label = name2label[label_name[: -len("group")]] + else: + raise + if label.id < 0: # cityscapes data format + continue + + # Cityscapes's raw annotations uses integer coordinates + # Therefore +0.5 here + poly_coord = np.asarray(obj["polygon"], dtype="f4") + 0.5 + # CityscapesScript uses PIL.ImageDraw.polygon to rasterize + # polygons for evaluation. This function operates in integer space + # and draws each pixel whose center falls into the polygon. + # Therefore it draws a polygon which is 0.5 "fatter" in expectation. + # We therefore dilate the input polygon by 0.5 as our input. + poly = Polygon(poly_coord).buffer(0.5, resolution=4) + + if not label.hasInstances or label.ignoreInEval: + # even if we won't store the polygon it still contributes to overlaps resolution + polygons_union = polygons_union.union(poly) + continue + + # Take non-overlapping part of the polygon + poly_wo_overlaps = poly.difference(polygons_union) + if poly_wo_overlaps.is_empty: + continue + polygons_union = polygons_union.union(poly) + + anno = {} + anno["iscrowd"] = label_name.endswith("group") + anno["category_id"] = label.id + + if isinstance(poly_wo_overlaps, Polygon): + poly_list = [poly_wo_overlaps] + elif isinstance(poly_wo_overlaps, MultiPolygon): + poly_list = poly_wo_overlaps.geoms + else: + raise NotImplementedError("Unknown geometric structure {}".format(poly_wo_overlaps)) + + poly_coord = [] + for poly_el in poly_list: + # COCO API can work only with exterior boundaries now, hence we store only them. + # TODO: store both exterior and interior boundaries once other parts of the + # codebase support holes in polygons. + poly_coord.append(list(chain(*poly_el.exterior.coords))) + anno["segmentation"] = poly_coord + (xmin, ymin, xmax, ymax) = poly_wo_overlaps.bounds + + anno["bbox"] = (xmin, ymin, xmax, ymax) + anno["bbox_mode"] = BoxMode.XYXY_ABS + + annos.append(anno) + else: + # See also the official annotation parsing scripts at + # https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/evaluation/instances2dict.py # noqa + with PathManager.open(instance_id_file, "rb") as f: + inst_image = np.asarray(Image.open(f), order="F") + # ids < 24 are stuff labels (filtering them first is about 5% faster) + flattened_ids = np.unique(inst_image[inst_image >= 24]) + + ret = { + "file_name": image_file, + "image_id": os.path.basename(image_file), + "height": inst_image.shape[0], + "width": inst_image.shape[1], + } + + for instance_id in flattened_ids: + # For non-crowd annotations, instance_id // 1000 is the label_id + # Crowd annotations have <1000 instance ids + label_id = instance_id // 1000 if instance_id >= 1000 else instance_id + label = id2label[label_id] + if not label.hasInstances or label.ignoreInEval: + continue + + anno = {} + anno["iscrowd"] = instance_id < 1000 + anno["category_id"] = label.id + + mask = np.asarray(inst_image == instance_id, dtype=np.uint8, order="F") + + inds = np.nonzero(mask) + ymin, ymax = inds[0].min(), inds[0].max() + xmin, xmax = inds[1].min(), inds[1].max() + anno["bbox"] = (xmin, ymin, xmax, ymax) + if xmax <= xmin or ymax <= ymin: + continue + anno["bbox_mode"] = BoxMode.XYXY_ABS + if to_polygons: + # This conversion comes from D4809743 and D5171122, + # when Mask-RCNN was first developed. + contours = cv2.findContours(mask.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)[ + -2 + ] + polygons = [c.reshape(-1).tolist() for c in contours if len(c) >= 3] + # opencv's can produce invalid polygons + if len(polygons) == 0: + continue + anno["segmentation"] = polygons + else: + anno["segmentation"] = mask_util.encode(mask[:, :, None])[0] + annos.append(anno) + ret["annotations"] = annos + return ret + + +if __name__ == "__main__": + """ + Test the cityscapes dataset loader. + + Usage: + python -m detectron2.data.datasets.cityscapes \ + cityscapes/leftImg8bit/train cityscapes/gtFine/train + """ + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("image_dir") + parser.add_argument("gt_dir") + parser.add_argument("--type", choices=["instance", "semantic"], default="instance") + args = parser.parse_args() + from annotator.oneformer.detectron2.data.catalog import Metadata + from annotator.oneformer.detectron2.utils.visualizer import Visualizer + from cityscapesscripts.helpers.labels import labels + + logger = setup_logger(name=__name__) + + dirname = "cityscapes-data-vis" + os.makedirs(dirname, exist_ok=True) + + if args.type == "instance": + dicts = load_cityscapes_instances( + args.image_dir, args.gt_dir, from_json=True, to_polygons=True + ) + logger.info("Done loading {} samples.".format(len(dicts))) + + thing_classes = [k.name for k in labels if k.hasInstances and not k.ignoreInEval] + meta = Metadata().set(thing_classes=thing_classes) + + else: + dicts = load_cityscapes_semantic(args.image_dir, args.gt_dir) + logger.info("Done loading {} samples.".format(len(dicts))) + + stuff_classes = [k.name for k in labels if k.trainId != 255] + stuff_colors = [k.color for k in labels if k.trainId != 255] + meta = Metadata().set(stuff_classes=stuff_classes, stuff_colors=stuff_colors) + + for d in dicts: + img = np.array(Image.open(PathManager.open(d["file_name"], "rb"))) + visualizer = Visualizer(img, metadata=meta) + vis = visualizer.draw_dataset_dict(d) + # cv2.imshow("a", vis.get_image()[:, :, ::-1]) + # cv2.waitKey() + fpath = os.path.join(dirname, os.path.basename(d["file_name"])) + vis.save(fpath) diff --git a/annotator/oneformer/detectron2/data/datasets/cityscapes_panoptic.py b/annotator/oneformer/detectron2/data/datasets/cityscapes_panoptic.py new file mode 100644 index 0000000000000000000000000000000000000000..7ce9ec48f673dadf3f5b4ae0592fc82415d9f925 --- /dev/null +++ b/annotator/oneformer/detectron2/data/datasets/cityscapes_panoptic.py @@ -0,0 +1,187 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import json +import logging +import os + +from annotator.oneformer.detectron2.data import DatasetCatalog, MetadataCatalog +from annotator.oneformer.detectron2.data.datasets.builtin_meta import CITYSCAPES_CATEGORIES +from annotator.oneformer.detectron2.utils.file_io import PathManager + +""" +This file contains functions to register the Cityscapes panoptic dataset to the DatasetCatalog. +""" + + +logger = logging.getLogger(__name__) + + +def get_cityscapes_panoptic_files(image_dir, gt_dir, json_info): + files = [] + # scan through the directory + cities = PathManager.ls(image_dir) + logger.info(f"{len(cities)} cities found in '{image_dir}'.") + image_dict = {} + for city in cities: + city_img_dir = os.path.join(image_dir, city) + for basename in PathManager.ls(city_img_dir): + image_file = os.path.join(city_img_dir, basename) + + suffix = "_leftImg8bit.png" + assert basename.endswith(suffix), basename + basename = os.path.basename(basename)[: -len(suffix)] + + image_dict[basename] = image_file + + for ann in json_info["annotations"]: + image_file = image_dict.get(ann["image_id"], None) + assert image_file is not None, "No image {} found for annotation {}".format( + ann["image_id"], ann["file_name"] + ) + label_file = os.path.join(gt_dir, ann["file_name"]) + segments_info = ann["segments_info"] + + files.append((image_file, label_file, segments_info)) + + assert len(files), "No images found in {}".format(image_dir) + assert PathManager.isfile(files[0][0]), files[0][0] + assert PathManager.isfile(files[0][1]), files[0][1] + return files + + +def load_cityscapes_panoptic(image_dir, gt_dir, gt_json, meta): + """ + Args: + image_dir (str): path to the raw dataset. e.g., "~/cityscapes/leftImg8bit/train". + gt_dir (str): path to the raw annotations. e.g., + "~/cityscapes/gtFine/cityscapes_panoptic_train". + gt_json (str): path to the json file. e.g., + "~/cityscapes/gtFine/cityscapes_panoptic_train.json". + meta (dict): dictionary containing "thing_dataset_id_to_contiguous_id" + and "stuff_dataset_id_to_contiguous_id" to map category ids to + contiguous ids for training. + + Returns: + list[dict]: a list of dicts in Detectron2 standard format. (See + `Using Custom Datasets `_ ) + """ + + def _convert_category_id(segment_info, meta): + if segment_info["category_id"] in meta["thing_dataset_id_to_contiguous_id"]: + segment_info["category_id"] = meta["thing_dataset_id_to_contiguous_id"][ + segment_info["category_id"] + ] + else: + segment_info["category_id"] = meta["stuff_dataset_id_to_contiguous_id"][ + segment_info["category_id"] + ] + return segment_info + + assert os.path.exists( + gt_json + ), "Please run `python cityscapesscripts/preparation/createPanopticImgs.py` to generate label files." # noqa + with open(gt_json) as f: + json_info = json.load(f) + files = get_cityscapes_panoptic_files(image_dir, gt_dir, json_info) + ret = [] + for image_file, label_file, segments_info in files: + sem_label_file = ( + image_file.replace("leftImg8bit", "gtFine").split(".")[0] + "_labelTrainIds.png" + ) + segments_info = [_convert_category_id(x, meta) for x in segments_info] + ret.append( + { + "file_name": image_file, + "image_id": "_".join( + os.path.splitext(os.path.basename(image_file))[0].split("_")[:3] + ), + "sem_seg_file_name": sem_label_file, + "pan_seg_file_name": label_file, + "segments_info": segments_info, + } + ) + assert len(ret), f"No images found in {image_dir}!" + assert PathManager.isfile( + ret[0]["sem_seg_file_name"] + ), "Please generate labelTrainIds.png with cityscapesscripts/preparation/createTrainIdLabelImgs.py" # noqa + assert PathManager.isfile( + ret[0]["pan_seg_file_name"] + ), "Please generate panoptic annotation with python cityscapesscripts/preparation/createPanopticImgs.py" # noqa + return ret + + +_RAW_CITYSCAPES_PANOPTIC_SPLITS = { + "cityscapes_fine_panoptic_train": ( + "cityscapes/leftImg8bit/train", + "cityscapes/gtFine/cityscapes_panoptic_train", + "cityscapes/gtFine/cityscapes_panoptic_train.json", + ), + "cityscapes_fine_panoptic_val": ( + "cityscapes/leftImg8bit/val", + "cityscapes/gtFine/cityscapes_panoptic_val", + "cityscapes/gtFine/cityscapes_panoptic_val.json", + ), + # "cityscapes_fine_panoptic_test": not supported yet +} + + +def register_all_cityscapes_panoptic(root): + meta = {} + # The following metadata maps contiguous id from [0, #thing categories + + # #stuff categories) to their names and colors. We have to replica of the + # same name and color under "thing_*" and "stuff_*" because the current + # visualization function in D2 handles thing and class classes differently + # due to some heuristic used in Panoptic FPN. We keep the same naming to + # enable reusing existing visualization functions. + thing_classes = [k["name"] for k in CITYSCAPES_CATEGORIES] + thing_colors = [k["color"] for k in CITYSCAPES_CATEGORIES] + stuff_classes = [k["name"] for k in CITYSCAPES_CATEGORIES] + stuff_colors = [k["color"] for k in CITYSCAPES_CATEGORIES] + + meta["thing_classes"] = thing_classes + meta["thing_colors"] = thing_colors + meta["stuff_classes"] = stuff_classes + meta["stuff_colors"] = stuff_colors + + # There are three types of ids in cityscapes panoptic segmentation: + # (1) category id: like semantic segmentation, it is the class id for each + # pixel. Since there are some classes not used in evaluation, the category + # id is not always contiguous and thus we have two set of category ids: + # - original category id: category id in the original dataset, mainly + # used for evaluation. + # - contiguous category id: [0, #classes), in order to train the classifier + # (2) instance id: this id is used to differentiate different instances from + # the same category. For "stuff" classes, the instance id is always 0; for + # "thing" classes, the instance id starts from 1 and 0 is reserved for + # ignored instances (e.g. crowd annotation). + # (3) panoptic id: this is the compact id that encode both category and + # instance id by: category_id * 1000 + instance_id. + thing_dataset_id_to_contiguous_id = {} + stuff_dataset_id_to_contiguous_id = {} + + for k in CITYSCAPES_CATEGORIES: + if k["isthing"] == 1: + thing_dataset_id_to_contiguous_id[k["id"]] = k["trainId"] + else: + stuff_dataset_id_to_contiguous_id[k["id"]] = k["trainId"] + + meta["thing_dataset_id_to_contiguous_id"] = thing_dataset_id_to_contiguous_id + meta["stuff_dataset_id_to_contiguous_id"] = stuff_dataset_id_to_contiguous_id + + for key, (image_dir, gt_dir, gt_json) in _RAW_CITYSCAPES_PANOPTIC_SPLITS.items(): + image_dir = os.path.join(root, image_dir) + gt_dir = os.path.join(root, gt_dir) + gt_json = os.path.join(root, gt_json) + + DatasetCatalog.register( + key, lambda x=image_dir, y=gt_dir, z=gt_json: load_cityscapes_panoptic(x, y, z, meta) + ) + MetadataCatalog.get(key).set( + panoptic_root=gt_dir, + image_root=image_dir, + panoptic_json=gt_json, + gt_dir=gt_dir.replace("cityscapes_panoptic_", ""), + evaluator_type="cityscapes_panoptic_seg", + ignore_label=255, + label_divisor=1000, + **meta, + ) diff --git a/annotator/oneformer/detectron2/data/datasets/coco.py b/annotator/oneformer/detectron2/data/datasets/coco.py new file mode 100644 index 0000000000000000000000000000000000000000..b0b2956b27568e5926a5d35adf0106fba1cd96b9 --- /dev/null +++ b/annotator/oneformer/detectron2/data/datasets/coco.py @@ -0,0 +1,539 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import contextlib +import datetime +import io +import json +import logging +import numpy as np +import os +import shutil +import pycocotools.mask as mask_util +from fvcore.common.timer import Timer +from iopath.common.file_io import file_lock +from PIL import Image + +from annotator.oneformer.detectron2.structures import Boxes, BoxMode, PolygonMasks, RotatedBoxes +from annotator.oneformer.detectron2.utils.file_io import PathManager + +from .. import DatasetCatalog, MetadataCatalog + +""" +This file contains functions to parse COCO-format annotations into dicts in "Detectron2 format". +""" + + +logger = logging.getLogger(__name__) + +__all__ = ["load_coco_json", "load_sem_seg", "convert_to_coco_json", "register_coco_instances"] + + +def load_coco_json(json_file, image_root, dataset_name=None, extra_annotation_keys=None): + """ + Load a json file with COCO's instances annotation format. + Currently supports instance detection, instance segmentation, + and person keypoints annotations. + + Args: + json_file (str): full path to the json file in COCO instances annotation format. + image_root (str or path-like): the directory where the images in this json file exists. + dataset_name (str or None): the name of the dataset (e.g., coco_2017_train). + When provided, this function will also do the following: + + * Put "thing_classes" into the metadata associated with this dataset. + * Map the category ids into a contiguous range (needed by standard dataset format), + and add "thing_dataset_id_to_contiguous_id" to the metadata associated + with this dataset. + + This option should usually be provided, unless users need to load + the original json content and apply more processing manually. + extra_annotation_keys (list[str]): list of per-annotation keys that should also be + loaded into the dataset dict (besides "iscrowd", "bbox", "keypoints", + "category_id", "segmentation"). The values for these keys will be returned as-is. + For example, the densepose annotations are loaded in this way. + + Returns: + list[dict]: a list of dicts in Detectron2 standard dataset dicts format (See + `Using Custom Datasets `_ ) when `dataset_name` is not None. + If `dataset_name` is None, the returned `category_ids` may be + incontiguous and may not conform to the Detectron2 standard format. + + Notes: + 1. This function does not read the image files. + The results do not have the "image" field. + """ + from pycocotools.coco import COCO + + timer = Timer() + json_file = PathManager.get_local_path(json_file) + with contextlib.redirect_stdout(io.StringIO()): + coco_api = COCO(json_file) + if timer.seconds() > 1: + logger.info("Loading {} takes {:.2f} seconds.".format(json_file, timer.seconds())) + + id_map = None + if dataset_name is not None: + meta = MetadataCatalog.get(dataset_name) + cat_ids = sorted(coco_api.getCatIds()) + cats = coco_api.loadCats(cat_ids) + # The categories in a custom json file may not be sorted. + thing_classes = [c["name"] for c in sorted(cats, key=lambda x: x["id"])] + meta.thing_classes = thing_classes + + # In COCO, certain category ids are artificially removed, + # and by convention they are always ignored. + # We deal with COCO's id issue and translate + # the category ids to contiguous ids in [0, 80). + + # It works by looking at the "categories" field in the json, therefore + # if users' own json also have incontiguous ids, we'll + # apply this mapping as well but print a warning. + if not (min(cat_ids) == 1 and max(cat_ids) == len(cat_ids)): + if "coco" not in dataset_name: + logger.warning( + """ +Category ids in annotations are not in [1, #categories]! We'll apply a mapping for you. +""" + ) + id_map = {v: i for i, v in enumerate(cat_ids)} + meta.thing_dataset_id_to_contiguous_id = id_map + + # sort indices for reproducible results + img_ids = sorted(coco_api.imgs.keys()) + # imgs is a list of dicts, each looks something like: + # {'license': 4, + # 'url': 'http://farm6.staticflickr.com/5454/9413846304_881d5e5c3b_z.jpg', + # 'file_name': 'COCO_val2014_000000001268.jpg', + # 'height': 427, + # 'width': 640, + # 'date_captured': '2013-11-17 05:57:24', + # 'id': 1268} + imgs = coco_api.loadImgs(img_ids) + # anns is a list[list[dict]], where each dict is an annotation + # record for an object. The inner list enumerates the objects in an image + # and the outer list enumerates over images. Example of anns[0]: + # [{'segmentation': [[192.81, + # 247.09, + # ... + # 219.03, + # 249.06]], + # 'area': 1035.749, + # 'iscrowd': 0, + # 'image_id': 1268, + # 'bbox': [192.81, 224.8, 74.73, 33.43], + # 'category_id': 16, + # 'id': 42986}, + # ...] + anns = [coco_api.imgToAnns[img_id] for img_id in img_ids] + total_num_valid_anns = sum([len(x) for x in anns]) + total_num_anns = len(coco_api.anns) + if total_num_valid_anns < total_num_anns: + logger.warning( + f"{json_file} contains {total_num_anns} annotations, but only " + f"{total_num_valid_anns} of them match to images in the file." + ) + + if "minival" not in json_file: + # The popular valminusminival & minival annotations for COCO2014 contain this bug. + # However the ratio of buggy annotations there is tiny and does not affect accuracy. + # Therefore we explicitly white-list them. + ann_ids = [ann["id"] for anns_per_image in anns for ann in anns_per_image] + assert len(set(ann_ids)) == len(ann_ids), "Annotation ids in '{}' are not unique!".format( + json_file + ) + + imgs_anns = list(zip(imgs, anns)) + logger.info("Loaded {} images in COCO format from {}".format(len(imgs_anns), json_file)) + + dataset_dicts = [] + + ann_keys = ["iscrowd", "bbox", "keypoints", "category_id"] + (extra_annotation_keys or []) + + num_instances_without_valid_segmentation = 0 + + for (img_dict, anno_dict_list) in imgs_anns: + record = {} + record["file_name"] = os.path.join(image_root, img_dict["file_name"]) + record["height"] = img_dict["height"] + record["width"] = img_dict["width"] + image_id = record["image_id"] = img_dict["id"] + + objs = [] + for anno in anno_dict_list: + # Check that the image_id in this annotation is the same as + # the image_id we're looking at. + # This fails only when the data parsing logic or the annotation file is buggy. + + # The original COCO valminusminival2014 & minival2014 annotation files + # actually contains bugs that, together with certain ways of using COCO API, + # can trigger this assertion. + assert anno["image_id"] == image_id + + assert anno.get("ignore", 0) == 0, '"ignore" in COCO json file is not supported.' + + obj = {key: anno[key] for key in ann_keys if key in anno} + if "bbox" in obj and len(obj["bbox"]) == 0: + raise ValueError( + f"One annotation of image {image_id} contains empty 'bbox' value! " + "This json does not have valid COCO format." + ) + + segm = anno.get("segmentation", None) + if segm: # either list[list[float]] or dict(RLE) + if isinstance(segm, dict): + if isinstance(segm["counts"], list): + # convert to compressed RLE + segm = mask_util.frPyObjects(segm, *segm["size"]) + else: + # filter out invalid polygons (< 3 points) + segm = [poly for poly in segm if len(poly) % 2 == 0 and len(poly) >= 6] + if len(segm) == 0: + num_instances_without_valid_segmentation += 1 + continue # ignore this instance + obj["segmentation"] = segm + + keypts = anno.get("keypoints", None) + if keypts: # list[int] + for idx, v in enumerate(keypts): + if idx % 3 != 2: + # COCO's segmentation coordinates are floating points in [0, H or W], + # but keypoint coordinates are integers in [0, H-1 or W-1] + # Therefore we assume the coordinates are "pixel indices" and + # add 0.5 to convert to floating point coordinates. + keypts[idx] = v + 0.5 + obj["keypoints"] = keypts + + obj["bbox_mode"] = BoxMode.XYWH_ABS + if id_map: + annotation_category_id = obj["category_id"] + try: + obj["category_id"] = id_map[annotation_category_id] + except KeyError as e: + raise KeyError( + f"Encountered category_id={annotation_category_id} " + "but this id does not exist in 'categories' of the json file." + ) from e + objs.append(obj) + record["annotations"] = objs + dataset_dicts.append(record) + + if num_instances_without_valid_segmentation > 0: + logger.warning( + "Filtered out {} instances without valid segmentation. ".format( + num_instances_without_valid_segmentation + ) + + "There might be issues in your dataset generation process. Please " + "check https://detectron2.readthedocs.io/en/latest/tutorials/datasets.html carefully" + ) + return dataset_dicts + + +def load_sem_seg(gt_root, image_root, gt_ext="png", image_ext="jpg"): + """ + Load semantic segmentation datasets. All files under "gt_root" with "gt_ext" extension are + treated as ground truth annotations and all files under "image_root" with "image_ext" extension + as input images. Ground truth and input images are matched using file paths relative to + "gt_root" and "image_root" respectively without taking into account file extensions. + This works for COCO as well as some other datasets. + + Args: + gt_root (str): full path to ground truth semantic segmentation files. Semantic segmentation + annotations are stored as images with integer values in pixels that represent + corresponding semantic labels. + image_root (str): the directory where the input images are. + gt_ext (str): file extension for ground truth annotations. + image_ext (str): file extension for input images. + + Returns: + list[dict]: + a list of dicts in detectron2 standard format without instance-level + annotation. + + Notes: + 1. This function does not read the image and ground truth files. + The results do not have the "image" and "sem_seg" fields. + """ + + # We match input images with ground truth based on their relative filepaths (without file + # extensions) starting from 'image_root' and 'gt_root' respectively. + def file2id(folder_path, file_path): + # extract relative path starting from `folder_path` + image_id = os.path.normpath(os.path.relpath(file_path, start=folder_path)) + # remove file extension + image_id = os.path.splitext(image_id)[0] + return image_id + + input_files = sorted( + (os.path.join(image_root, f) for f in PathManager.ls(image_root) if f.endswith(image_ext)), + key=lambda file_path: file2id(image_root, file_path), + ) + gt_files = sorted( + (os.path.join(gt_root, f) for f in PathManager.ls(gt_root) if f.endswith(gt_ext)), + key=lambda file_path: file2id(gt_root, file_path), + ) + + assert len(gt_files) > 0, "No annotations found in {}.".format(gt_root) + + # Use the intersection, so that val2017_100 annotations can run smoothly with val2017 images + if len(input_files) != len(gt_files): + logger.warn( + "Directory {} and {} has {} and {} files, respectively.".format( + image_root, gt_root, len(input_files), len(gt_files) + ) + ) + input_basenames = [os.path.basename(f)[: -len(image_ext)] for f in input_files] + gt_basenames = [os.path.basename(f)[: -len(gt_ext)] for f in gt_files] + intersect = list(set(input_basenames) & set(gt_basenames)) + # sort, otherwise each worker may obtain a list[dict] in different order + intersect = sorted(intersect) + logger.warn("Will use their intersection of {} files.".format(len(intersect))) + input_files = [os.path.join(image_root, f + image_ext) for f in intersect] + gt_files = [os.path.join(gt_root, f + gt_ext) for f in intersect] + + logger.info( + "Loaded {} images with semantic segmentation from {}".format(len(input_files), image_root) + ) + + dataset_dicts = [] + for (img_path, gt_path) in zip(input_files, gt_files): + record = {} + record["file_name"] = img_path + record["sem_seg_file_name"] = gt_path + dataset_dicts.append(record) + + return dataset_dicts + + +def convert_to_coco_dict(dataset_name): + """ + Convert an instance detection/segmentation or keypoint detection dataset + in detectron2's standard format into COCO json format. + + Generic dataset description can be found here: + https://detectron2.readthedocs.io/tutorials/datasets.html#register-a-dataset + + COCO data format description can be found here: + http://cocodataset.org/#format-data + + Args: + dataset_name (str): + name of the source dataset + Must be registered in DatastCatalog and in detectron2's standard format. + Must have corresponding metadata "thing_classes" + Returns: + coco_dict: serializable dict in COCO json format + """ + + dataset_dicts = DatasetCatalog.get(dataset_name) + metadata = MetadataCatalog.get(dataset_name) + + # unmap the category mapping ids for COCO + if hasattr(metadata, "thing_dataset_id_to_contiguous_id"): + reverse_id_mapping = {v: k for k, v in metadata.thing_dataset_id_to_contiguous_id.items()} + reverse_id_mapper = lambda contiguous_id: reverse_id_mapping[contiguous_id] # noqa + else: + reverse_id_mapper = lambda contiguous_id: contiguous_id # noqa + + categories = [ + {"id": reverse_id_mapper(id), "name": name} + for id, name in enumerate(metadata.thing_classes) + ] + + logger.info("Converting dataset dicts into COCO format") + coco_images = [] + coco_annotations = [] + + for image_id, image_dict in enumerate(dataset_dicts): + coco_image = { + "id": image_dict.get("image_id", image_id), + "width": int(image_dict["width"]), + "height": int(image_dict["height"]), + "file_name": str(image_dict["file_name"]), + } + coco_images.append(coco_image) + + anns_per_image = image_dict.get("annotations", []) + for annotation in anns_per_image: + # create a new dict with only COCO fields + coco_annotation = {} + + # COCO requirement: XYWH box format for axis-align and XYWHA for rotated + bbox = annotation["bbox"] + if isinstance(bbox, np.ndarray): + if bbox.ndim != 1: + raise ValueError(f"bbox has to be 1-dimensional. Got shape={bbox.shape}.") + bbox = bbox.tolist() + if len(bbox) not in [4, 5]: + raise ValueError(f"bbox has to has length 4 or 5. Got {bbox}.") + from_bbox_mode = annotation["bbox_mode"] + to_bbox_mode = BoxMode.XYWH_ABS if len(bbox) == 4 else BoxMode.XYWHA_ABS + bbox = BoxMode.convert(bbox, from_bbox_mode, to_bbox_mode) + + # COCO requirement: instance area + if "segmentation" in annotation: + # Computing areas for instances by counting the pixels + segmentation = annotation["segmentation"] + # TODO: check segmentation type: RLE, BinaryMask or Polygon + if isinstance(segmentation, list): + polygons = PolygonMasks([segmentation]) + area = polygons.area()[0].item() + elif isinstance(segmentation, dict): # RLE + area = mask_util.area(segmentation).item() + else: + raise TypeError(f"Unknown segmentation type {type(segmentation)}!") + else: + # Computing areas using bounding boxes + if to_bbox_mode == BoxMode.XYWH_ABS: + bbox_xy = BoxMode.convert(bbox, to_bbox_mode, BoxMode.XYXY_ABS) + area = Boxes([bbox_xy]).area()[0].item() + else: + area = RotatedBoxes([bbox]).area()[0].item() + + if "keypoints" in annotation: + keypoints = annotation["keypoints"] # list[int] + for idx, v in enumerate(keypoints): + if idx % 3 != 2: + # COCO's segmentation coordinates are floating points in [0, H or W], + # but keypoint coordinates are integers in [0, H-1 or W-1] + # For COCO format consistency we substract 0.5 + # https://github.com/facebookresearch/detectron2/pull/175#issuecomment-551202163 + keypoints[idx] = v - 0.5 + if "num_keypoints" in annotation: + num_keypoints = annotation["num_keypoints"] + else: + num_keypoints = sum(kp > 0 for kp in keypoints[2::3]) + + # COCO requirement: + # linking annotations to images + # "id" field must start with 1 + coco_annotation["id"] = len(coco_annotations) + 1 + coco_annotation["image_id"] = coco_image["id"] + coco_annotation["bbox"] = [round(float(x), 3) for x in bbox] + coco_annotation["area"] = float(area) + coco_annotation["iscrowd"] = int(annotation.get("iscrowd", 0)) + coco_annotation["category_id"] = int(reverse_id_mapper(annotation["category_id"])) + + # Add optional fields + if "keypoints" in annotation: + coco_annotation["keypoints"] = keypoints + coco_annotation["num_keypoints"] = num_keypoints + + if "segmentation" in annotation: + seg = coco_annotation["segmentation"] = annotation["segmentation"] + if isinstance(seg, dict): # RLE + counts = seg["counts"] + if not isinstance(counts, str): + # make it json-serializable + seg["counts"] = counts.decode("ascii") + + coco_annotations.append(coco_annotation) + + logger.info( + "Conversion finished, " + f"#images: {len(coco_images)}, #annotations: {len(coco_annotations)}" + ) + + info = { + "date_created": str(datetime.datetime.now()), + "description": "Automatically generated COCO json file for Detectron2.", + } + coco_dict = {"info": info, "images": coco_images, "categories": categories, "licenses": None} + if len(coco_annotations) > 0: + coco_dict["annotations"] = coco_annotations + return coco_dict + + +def convert_to_coco_json(dataset_name, output_file, allow_cached=True): + """ + Converts dataset into COCO format and saves it to a json file. + dataset_name must be registered in DatasetCatalog and in detectron2's standard format. + + Args: + dataset_name: + reference from the config file to the catalogs + must be registered in DatasetCatalog and in detectron2's standard format + output_file: path of json file that will be saved to + allow_cached: if json file is already present then skip conversion + """ + + # TODO: The dataset or the conversion script *may* change, + # a checksum would be useful for validating the cached data + + PathManager.mkdirs(os.path.dirname(output_file)) + with file_lock(output_file): + if PathManager.exists(output_file) and allow_cached: + logger.warning( + f"Using previously cached COCO format annotations at '{output_file}'. " + "You need to clear the cache file if your dataset has been modified." + ) + else: + logger.info(f"Converting annotations of dataset '{dataset_name}' to COCO format ...)") + coco_dict = convert_to_coco_dict(dataset_name) + + logger.info(f"Caching COCO format annotations at '{output_file}' ...") + tmp_file = output_file + ".tmp" + with PathManager.open(tmp_file, "w") as f: + json.dump(coco_dict, f) + shutil.move(tmp_file, output_file) + + +def register_coco_instances(name, metadata, json_file, image_root): + """ + Register a dataset in COCO's json annotation format for + instance detection, instance segmentation and keypoint detection. + (i.e., Type 1 and 2 in http://cocodataset.org/#format-data. + `instances*.json` and `person_keypoints*.json` in the dataset). + + This is an example of how to register a new dataset. + You can do something similar to this function, to register new datasets. + + Args: + name (str): the name that identifies a dataset, e.g. "coco_2014_train". + metadata (dict): extra metadata associated with this dataset. You can + leave it as an empty dict. + json_file (str): path to the json instance annotation file. + image_root (str or path-like): directory which contains all the images. + """ + assert isinstance(name, str), name + assert isinstance(json_file, (str, os.PathLike)), json_file + assert isinstance(image_root, (str, os.PathLike)), image_root + # 1. register a function which returns dicts + DatasetCatalog.register(name, lambda: load_coco_json(json_file, image_root, name)) + + # 2. Optionally, add metadata about this dataset, + # since they might be useful in evaluation, visualization or logging + MetadataCatalog.get(name).set( + json_file=json_file, image_root=image_root, evaluator_type="coco", **metadata + ) + + +if __name__ == "__main__": + """ + Test the COCO json dataset loader. + + Usage: + python -m detectron2.data.datasets.coco \ + path/to/json path/to/image_root dataset_name + + "dataset_name" can be "coco_2014_minival_100", or other + pre-registered ones + """ + from annotator.oneformer.detectron2.utils.logger import setup_logger + from annotator.oneformer.detectron2.utils.visualizer import Visualizer + import annotator.oneformer.detectron2.data.datasets # noqa # add pre-defined metadata + import sys + + logger = setup_logger(name=__name__) + assert sys.argv[3] in DatasetCatalog.list() + meta = MetadataCatalog.get(sys.argv[3]) + + dicts = load_coco_json(sys.argv[1], sys.argv[2], sys.argv[3]) + logger.info("Done loading {} samples.".format(len(dicts))) + + dirname = "coco-data-vis" + os.makedirs(dirname, exist_ok=True) + for d in dicts: + img = np.array(Image.open(d["file_name"])) + visualizer = Visualizer(img, metadata=meta) + vis = visualizer.draw_dataset_dict(d) + fpath = os.path.join(dirname, os.path.basename(d["file_name"])) + vis.save(fpath) diff --git a/annotator/oneformer/detectron2/data/datasets/coco_panoptic.py b/annotator/oneformer/detectron2/data/datasets/coco_panoptic.py new file mode 100644 index 0000000000000000000000000000000000000000..a7180df512c29665222b1a90323ccfa7e7623137 --- /dev/null +++ b/annotator/oneformer/detectron2/data/datasets/coco_panoptic.py @@ -0,0 +1,228 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +import json +import os + +from annotator.oneformer.detectron2.data import DatasetCatalog, MetadataCatalog +from annotator.oneformer.detectron2.utils.file_io import PathManager + +from .coco import load_coco_json, load_sem_seg + +__all__ = ["register_coco_panoptic", "register_coco_panoptic_separated"] + + +def load_coco_panoptic_json(json_file, image_dir, gt_dir, meta): + """ + Args: + image_dir (str): path to the raw dataset. e.g., "~/coco/train2017". + gt_dir (str): path to the raw annotations. e.g., "~/coco/panoptic_train2017". + json_file (str): path to the json file. e.g., "~/coco/annotations/panoptic_train2017.json". + + Returns: + list[dict]: a list of dicts in Detectron2 standard format. (See + `Using Custom Datasets `_ ) + """ + + def _convert_category_id(segment_info, meta): + if segment_info["category_id"] in meta["thing_dataset_id_to_contiguous_id"]: + segment_info["category_id"] = meta["thing_dataset_id_to_contiguous_id"][ + segment_info["category_id"] + ] + segment_info["isthing"] = True + else: + segment_info["category_id"] = meta["stuff_dataset_id_to_contiguous_id"][ + segment_info["category_id"] + ] + segment_info["isthing"] = False + return segment_info + + with PathManager.open(json_file) as f: + json_info = json.load(f) + + ret = [] + for ann in json_info["annotations"]: + image_id = int(ann["image_id"]) + # TODO: currently we assume image and label has the same filename but + # different extension, and images have extension ".jpg" for COCO. Need + # to make image extension a user-provided argument if we extend this + # function to support other COCO-like datasets. + image_file = os.path.join(image_dir, os.path.splitext(ann["file_name"])[0] + ".jpg") + label_file = os.path.join(gt_dir, ann["file_name"]) + segments_info = [_convert_category_id(x, meta) for x in ann["segments_info"]] + ret.append( + { + "file_name": image_file, + "image_id": image_id, + "pan_seg_file_name": label_file, + "segments_info": segments_info, + } + ) + assert len(ret), f"No images found in {image_dir}!" + assert PathManager.isfile(ret[0]["file_name"]), ret[0]["file_name"] + assert PathManager.isfile(ret[0]["pan_seg_file_name"]), ret[0]["pan_seg_file_name"] + return ret + + +def register_coco_panoptic( + name, metadata, image_root, panoptic_root, panoptic_json, instances_json=None +): + """ + Register a "standard" version of COCO panoptic segmentation dataset named `name`. + The dictionaries in this registered dataset follows detectron2's standard format. + Hence it's called "standard". + + Args: + name (str): the name that identifies a dataset, + e.g. "coco_2017_train_panoptic" + metadata (dict): extra metadata associated with this dataset. + image_root (str): directory which contains all the images + panoptic_root (str): directory which contains panoptic annotation images in COCO format + panoptic_json (str): path to the json panoptic annotation file in COCO format + sem_seg_root (none): not used, to be consistent with + `register_coco_panoptic_separated`. + instances_json (str): path to the json instance annotation file + """ + panoptic_name = name + DatasetCatalog.register( + panoptic_name, + lambda: load_coco_panoptic_json(panoptic_json, image_root, panoptic_root, metadata), + ) + MetadataCatalog.get(panoptic_name).set( + panoptic_root=panoptic_root, + image_root=image_root, + panoptic_json=panoptic_json, + json_file=instances_json, + evaluator_type="coco_panoptic_seg", + ignore_label=255, + label_divisor=1000, + **metadata, + ) + + +def register_coco_panoptic_separated( + name, metadata, image_root, panoptic_root, panoptic_json, sem_seg_root, instances_json +): + """ + Register a "separated" version of COCO panoptic segmentation dataset named `name`. + The annotations in this registered dataset will contain both instance annotations and + semantic annotations, each with its own contiguous ids. Hence it's called "separated". + + It follows the setting used by the PanopticFPN paper: + + 1. The instance annotations directly come from polygons in the COCO + instances annotation task, rather than from the masks in the COCO panoptic annotations. + + The two format have small differences: + Polygons in the instance annotations may have overlaps. + The mask annotations are produced by labeling the overlapped polygons + with depth ordering. + + 2. The semantic annotations are converted from panoptic annotations, where + all "things" are assigned a semantic id of 0. + All semantic categories will therefore have ids in contiguous + range [1, #stuff_categories]. + + This function will also register a pure semantic segmentation dataset + named ``name + '_stuffonly'``. + + Args: + name (str): the name that identifies a dataset, + e.g. "coco_2017_train_panoptic" + metadata (dict): extra metadata associated with this dataset. + image_root (str): directory which contains all the images + panoptic_root (str): directory which contains panoptic annotation images + panoptic_json (str): path to the json panoptic annotation file + sem_seg_root (str): directory which contains all the ground truth segmentation annotations. + instances_json (str): path to the json instance annotation file + """ + panoptic_name = name + "_separated" + DatasetCatalog.register( + panoptic_name, + lambda: merge_to_panoptic( + load_coco_json(instances_json, image_root, panoptic_name), + load_sem_seg(sem_seg_root, image_root), + ), + ) + MetadataCatalog.get(panoptic_name).set( + panoptic_root=panoptic_root, + image_root=image_root, + panoptic_json=panoptic_json, + sem_seg_root=sem_seg_root, + json_file=instances_json, # TODO rename + evaluator_type="coco_panoptic_seg", + ignore_label=255, + **metadata, + ) + + semantic_name = name + "_stuffonly" + DatasetCatalog.register(semantic_name, lambda: load_sem_seg(sem_seg_root, image_root)) + MetadataCatalog.get(semantic_name).set( + sem_seg_root=sem_seg_root, + image_root=image_root, + evaluator_type="sem_seg", + ignore_label=255, + **metadata, + ) + + +def merge_to_panoptic(detection_dicts, sem_seg_dicts): + """ + Create dataset dicts for panoptic segmentation, by + merging two dicts using "file_name" field to match their entries. + + Args: + detection_dicts (list[dict]): lists of dicts for object detection or instance segmentation. + sem_seg_dicts (list[dict]): lists of dicts for semantic segmentation. + + Returns: + list[dict] (one per input image): Each dict contains all (key, value) pairs from dicts in + both detection_dicts and sem_seg_dicts that correspond to the same image. + The function assumes that the same key in different dicts has the same value. + """ + results = [] + sem_seg_file_to_entry = {x["file_name"]: x for x in sem_seg_dicts} + assert len(sem_seg_file_to_entry) > 0 + + for det_dict in detection_dicts: + dic = copy.copy(det_dict) + dic.update(sem_seg_file_to_entry[dic["file_name"]]) + results.append(dic) + return results + + +if __name__ == "__main__": + """ + Test the COCO panoptic dataset loader. + + Usage: + python -m detectron2.data.datasets.coco_panoptic \ + path/to/image_root path/to/panoptic_root path/to/panoptic_json dataset_name 10 + + "dataset_name" can be "coco_2017_train_panoptic", or other + pre-registered ones + """ + from annotator.oneformer.detectron2.utils.logger import setup_logger + from annotator.oneformer.detectron2.utils.visualizer import Visualizer + import annotator.oneformer.detectron2.data.datasets # noqa # add pre-defined metadata + import sys + from PIL import Image + import numpy as np + + logger = setup_logger(name=__name__) + assert sys.argv[4] in DatasetCatalog.list() + meta = MetadataCatalog.get(sys.argv[4]) + + dicts = load_coco_panoptic_json(sys.argv[3], sys.argv[1], sys.argv[2], meta.as_dict()) + logger.info("Done loading {} samples.".format(len(dicts))) + + dirname = "coco-data-vis" + os.makedirs(dirname, exist_ok=True) + num_imgs_to_vis = int(sys.argv[5]) + for i, d in enumerate(dicts): + img = np.array(Image.open(d["file_name"])) + visualizer = Visualizer(img, metadata=meta) + vis = visualizer.draw_dataset_dict(d) + fpath = os.path.join(dirname, os.path.basename(d["file_name"])) + vis.save(fpath) + if i + 1 >= num_imgs_to_vis: + break diff --git a/annotator/oneformer/detectron2/data/datasets/lvis.py b/annotator/oneformer/detectron2/data/datasets/lvis.py new file mode 100644 index 0000000000000000000000000000000000000000..6e1e6ecc657e83d6df57da342b0655177402c514 --- /dev/null +++ b/annotator/oneformer/detectron2/data/datasets/lvis.py @@ -0,0 +1,241 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import os +from fvcore.common.timer import Timer + +from annotator.oneformer.detectron2.data import DatasetCatalog, MetadataCatalog +from annotator.oneformer.detectron2.structures import BoxMode +from annotator.oneformer.detectron2.utils.file_io import PathManager + +from .builtin_meta import _get_coco_instances_meta +from .lvis_v0_5_categories import LVIS_CATEGORIES as LVIS_V0_5_CATEGORIES +from .lvis_v1_categories import LVIS_CATEGORIES as LVIS_V1_CATEGORIES +from .lvis_v1_category_image_count import LVIS_CATEGORY_IMAGE_COUNT as LVIS_V1_CATEGORY_IMAGE_COUNT + +""" +This file contains functions to parse LVIS-format annotations into dicts in the +"Detectron2 format". +""" + +logger = logging.getLogger(__name__) + +__all__ = ["load_lvis_json", "register_lvis_instances", "get_lvis_instances_meta"] + + +def register_lvis_instances(name, metadata, json_file, image_root): + """ + Register a dataset in LVIS's json annotation format for instance detection and segmentation. + + Args: + name (str): a name that identifies the dataset, e.g. "lvis_v0.5_train". + metadata (dict): extra metadata associated with this dataset. It can be an empty dict. + json_file (str): path to the json instance annotation file. + image_root (str or path-like): directory which contains all the images. + """ + DatasetCatalog.register(name, lambda: load_lvis_json(json_file, image_root, name)) + MetadataCatalog.get(name).set( + json_file=json_file, image_root=image_root, evaluator_type="lvis", **metadata + ) + + +def load_lvis_json(json_file, image_root, dataset_name=None, extra_annotation_keys=None): + """ + Load a json file in LVIS's annotation format. + + Args: + json_file (str): full path to the LVIS json annotation file. + image_root (str): the directory where the images in this json file exists. + dataset_name (str): the name of the dataset (e.g., "lvis_v0.5_train"). + If provided, this function will put "thing_classes" into the metadata + associated with this dataset. + extra_annotation_keys (list[str]): list of per-annotation keys that should also be + loaded into the dataset dict (besides "bbox", "bbox_mode", "category_id", + "segmentation"). The values for these keys will be returned as-is. + + Returns: + list[dict]: a list of dicts in Detectron2 standard format. (See + `Using Custom Datasets `_ ) + + Notes: + 1. This function does not read the image files. + The results do not have the "image" field. + """ + from lvis import LVIS + + json_file = PathManager.get_local_path(json_file) + + timer = Timer() + lvis_api = LVIS(json_file) + if timer.seconds() > 1: + logger.info("Loading {} takes {:.2f} seconds.".format(json_file, timer.seconds())) + + if dataset_name is not None: + meta = get_lvis_instances_meta(dataset_name) + MetadataCatalog.get(dataset_name).set(**meta) + + # sort indices for reproducible results + img_ids = sorted(lvis_api.imgs.keys()) + # imgs is a list of dicts, each looks something like: + # {'license': 4, + # 'url': 'http://farm6.staticflickr.com/5454/9413846304_881d5e5c3b_z.jpg', + # 'file_name': 'COCO_val2014_000000001268.jpg', + # 'height': 427, + # 'width': 640, + # 'date_captured': '2013-11-17 05:57:24', + # 'id': 1268} + imgs = lvis_api.load_imgs(img_ids) + # anns is a list[list[dict]], where each dict is an annotation + # record for an object. The inner list enumerates the objects in an image + # and the outer list enumerates over images. Example of anns[0]: + # [{'segmentation': [[192.81, + # 247.09, + # ... + # 219.03, + # 249.06]], + # 'area': 1035.749, + # 'image_id': 1268, + # 'bbox': [192.81, 224.8, 74.73, 33.43], + # 'category_id': 16, + # 'id': 42986}, + # ...] + anns = [lvis_api.img_ann_map[img_id] for img_id in img_ids] + + # Sanity check that each annotation has a unique id + ann_ids = [ann["id"] for anns_per_image in anns for ann in anns_per_image] + assert len(set(ann_ids)) == len(ann_ids), "Annotation ids in '{}' are not unique".format( + json_file + ) + + imgs_anns = list(zip(imgs, anns)) + + logger.info("Loaded {} images in the LVIS format from {}".format(len(imgs_anns), json_file)) + + if extra_annotation_keys: + logger.info( + "The following extra annotation keys will be loaded: {} ".format(extra_annotation_keys) + ) + else: + extra_annotation_keys = [] + + def get_file_name(img_root, img_dict): + # Determine the path including the split folder ("train2017", "val2017", "test2017") from + # the coco_url field. Example: + # 'coco_url': 'http://images.cocodataset.org/train2017/000000155379.jpg' + split_folder, file_name = img_dict["coco_url"].split("/")[-2:] + return os.path.join(img_root + split_folder, file_name) + + dataset_dicts = [] + + for (img_dict, anno_dict_list) in imgs_anns: + record = {} + record["file_name"] = get_file_name(image_root, img_dict) + record["height"] = img_dict["height"] + record["width"] = img_dict["width"] + record["not_exhaustive_category_ids"] = img_dict.get("not_exhaustive_category_ids", []) + record["neg_category_ids"] = img_dict.get("neg_category_ids", []) + image_id = record["image_id"] = img_dict["id"] + + objs = [] + for anno in anno_dict_list: + # Check that the image_id in this annotation is the same as + # the image_id we're looking at. + # This fails only when the data parsing logic or the annotation file is buggy. + assert anno["image_id"] == image_id + obj = {"bbox": anno["bbox"], "bbox_mode": BoxMode.XYWH_ABS} + # LVIS data loader can be used to load COCO dataset categories. In this case `meta` + # variable will have a field with COCO-specific category mapping. + if dataset_name is not None and "thing_dataset_id_to_contiguous_id" in meta: + obj["category_id"] = meta["thing_dataset_id_to_contiguous_id"][anno["category_id"]] + else: + obj["category_id"] = anno["category_id"] - 1 # Convert 1-indexed to 0-indexed + segm = anno["segmentation"] # list[list[float]] + # filter out invalid polygons (< 3 points) + valid_segm = [poly for poly in segm if len(poly) % 2 == 0 and len(poly) >= 6] + assert len(segm) == len( + valid_segm + ), "Annotation contains an invalid polygon with < 3 points" + assert len(segm) > 0 + obj["segmentation"] = segm + for extra_ann_key in extra_annotation_keys: + obj[extra_ann_key] = anno[extra_ann_key] + objs.append(obj) + record["annotations"] = objs + dataset_dicts.append(record) + + return dataset_dicts + + +def get_lvis_instances_meta(dataset_name): + """ + Load LVIS metadata. + + Args: + dataset_name (str): LVIS dataset name without the split name (e.g., "lvis_v0.5"). + + Returns: + dict: LVIS metadata with keys: thing_classes + """ + if "cocofied" in dataset_name: + return _get_coco_instances_meta() + if "v0.5" in dataset_name: + return _get_lvis_instances_meta_v0_5() + elif "v1" in dataset_name: + return _get_lvis_instances_meta_v1() + raise ValueError("No built-in metadata for dataset {}".format(dataset_name)) + + +def _get_lvis_instances_meta_v0_5(): + assert len(LVIS_V0_5_CATEGORIES) == 1230 + cat_ids = [k["id"] for k in LVIS_V0_5_CATEGORIES] + assert min(cat_ids) == 1 and max(cat_ids) == len( + cat_ids + ), "Category ids are not in [1, #categories], as expected" + # Ensure that the category list is sorted by id + lvis_categories = sorted(LVIS_V0_5_CATEGORIES, key=lambda x: x["id"]) + thing_classes = [k["synonyms"][0] for k in lvis_categories] + meta = {"thing_classes": thing_classes} + return meta + + +def _get_lvis_instances_meta_v1(): + assert len(LVIS_V1_CATEGORIES) == 1203 + cat_ids = [k["id"] for k in LVIS_V1_CATEGORIES] + assert min(cat_ids) == 1 and max(cat_ids) == len( + cat_ids + ), "Category ids are not in [1, #categories], as expected" + # Ensure that the category list is sorted by id + lvis_categories = sorted(LVIS_V1_CATEGORIES, key=lambda x: x["id"]) + thing_classes = [k["synonyms"][0] for k in lvis_categories] + meta = {"thing_classes": thing_classes, "class_image_count": LVIS_V1_CATEGORY_IMAGE_COUNT} + return meta + + +if __name__ == "__main__": + """ + Test the LVIS json dataset loader. + + Usage: + python -m detectron2.data.datasets.lvis \ + path/to/json path/to/image_root dataset_name vis_limit + """ + import sys + import numpy as np + from annotator.oneformer.detectron2.utils.logger import setup_logger + from PIL import Image + import annotator.oneformer.detectron2.data.datasets # noqa # add pre-defined metadata + from annotator.oneformer.detectron2.utils.visualizer import Visualizer + + logger = setup_logger(name=__name__) + meta = MetadataCatalog.get(sys.argv[3]) + + dicts = load_lvis_json(sys.argv[1], sys.argv[2], sys.argv[3]) + logger.info("Done loading {} samples.".format(len(dicts))) + + dirname = "lvis-data-vis" + os.makedirs(dirname, exist_ok=True) + for d in dicts[: int(sys.argv[4])]: + img = np.array(Image.open(d["file_name"])) + visualizer = Visualizer(img, metadata=meta) + vis = visualizer.draw_dataset_dict(d) + fpath = os.path.join(dirname, os.path.basename(d["file_name"])) + vis.save(fpath) diff --git a/annotator/oneformer/detectron2/data/datasets/lvis_v0_5_categories.py b/annotator/oneformer/detectron2/data/datasets/lvis_v0_5_categories.py new file mode 100644 index 0000000000000000000000000000000000000000..d3dab6198da614937b08682f4c9edf52bdf1d236 --- /dev/null +++ b/annotator/oneformer/detectron2/data/datasets/lvis_v0_5_categories.py @@ -0,0 +1,13 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# Autogen with +# with open("lvis_v0.5_val.json", "r") as f: +# a = json.load(f) +# c = a["categories"] +# for x in c: +# del x["image_count"] +# del x["instance_count"] +# LVIS_CATEGORIES = repr(c) + " # noqa" + +# fmt: off +LVIS_CATEGORIES = [{'frequency': 'r', 'id': 1, 'synset': 'acorn.n.01', 'synonyms': ['acorn'], 'def': 'nut from an oak tree', 'name': 'acorn'}, {'frequency': 'c', 'id': 2, 'synset': 'aerosol.n.02', 'synonyms': ['aerosol_can', 'spray_can'], 'def': 'a dispenser that holds a substance under pressure', 'name': 'aerosol_can'}, {'frequency': 'f', 'id': 3, 'synset': 'air_conditioner.n.01', 'synonyms': ['air_conditioner'], 'def': 'a machine that keeps air cool and dry', 'name': 'air_conditioner'}, {'frequency': 'f', 'id': 4, 'synset': 'airplane.n.01', 'synonyms': ['airplane', 'aeroplane'], 'def': 'an aircraft that has a fixed wing and is powered by propellers or jets', 'name': 'airplane'}, {'frequency': 'c', 'id': 5, 'synset': 'alarm_clock.n.01', 'synonyms': ['alarm_clock'], 'def': 'a clock that wakes a sleeper at some preset time', 'name': 'alarm_clock'}, {'frequency': 'c', 'id': 6, 'synset': 'alcohol.n.01', 'synonyms': ['alcohol', 'alcoholic_beverage'], 'def': 'a liquor or brew containing alcohol as the active agent', 'name': 'alcohol'}, {'frequency': 'r', 'id': 7, 'synset': 'alligator.n.02', 'synonyms': ['alligator', 'gator'], 'def': 'amphibious reptiles related to crocodiles but with shorter broader snouts', 'name': 'alligator'}, {'frequency': 'c', 'id': 8, 'synset': 'almond.n.02', 'synonyms': ['almond'], 'def': 'oval-shaped edible seed of the almond tree', 'name': 'almond'}, {'frequency': 'c', 'id': 9, 'synset': 'ambulance.n.01', 'synonyms': ['ambulance'], 'def': 'a vehicle that takes people to and from hospitals', 'name': 'ambulance'}, {'frequency': 'r', 'id': 10, 'synset': 'amplifier.n.01', 'synonyms': ['amplifier'], 'def': 'electronic equipment that increases strength of signals', 'name': 'amplifier'}, {'frequency': 'c', 'id': 11, 'synset': 'anklet.n.03', 'synonyms': ['anklet', 'ankle_bracelet'], 'def': 'an ornament worn around the ankle', 'name': 'anklet'}, {'frequency': 'f', 'id': 12, 'synset': 'antenna.n.01', 'synonyms': ['antenna', 'aerial', 'transmitting_aerial'], 'def': 'an electrical device that sends or receives radio or television signals', 'name': 'antenna'}, {'frequency': 'f', 'id': 13, 'synset': 'apple.n.01', 'synonyms': ['apple'], 'def': 'fruit with red or yellow or green skin and sweet to tart crisp whitish flesh', 'name': 'apple'}, {'frequency': 'r', 'id': 14, 'synset': 'apple_juice.n.01', 'synonyms': ['apple_juice'], 'def': 'the juice of apples', 'name': 'apple_juice'}, {'frequency': 'r', 'id': 15, 'synset': 'applesauce.n.01', 'synonyms': ['applesauce'], 'def': 'puree of stewed apples usually sweetened and spiced', 'name': 'applesauce'}, {'frequency': 'r', 'id': 16, 'synset': 'apricot.n.02', 'synonyms': ['apricot'], 'def': 'downy yellow to rosy-colored fruit resembling a small peach', 'name': 'apricot'}, {'frequency': 'f', 'id': 17, 'synset': 'apron.n.01', 'synonyms': ['apron'], 'def': 'a garment of cloth that is tied about the waist and worn to protect clothing', 'name': 'apron'}, {'frequency': 'c', 'id': 18, 'synset': 'aquarium.n.01', 'synonyms': ['aquarium', 'fish_tank'], 'def': 'a tank/pool/bowl filled with water for keeping live fish and underwater animals', 'name': 'aquarium'}, {'frequency': 'c', 'id': 19, 'synset': 'armband.n.02', 'synonyms': ['armband'], 'def': 'a band worn around the upper arm', 'name': 'armband'}, {'frequency': 'f', 'id': 20, 'synset': 'armchair.n.01', 'synonyms': ['armchair'], 'def': 'chair with a support on each side for arms', 'name': 'armchair'}, {'frequency': 'r', 'id': 21, 'synset': 'armoire.n.01', 'synonyms': ['armoire'], 'def': 'a large wardrobe or cabinet', 'name': 'armoire'}, {'frequency': 'r', 'id': 22, 'synset': 'armor.n.01', 'synonyms': ['armor', 'armour'], 'def': 'protective covering made of metal and used in combat', 'name': 'armor'}, {'frequency': 'c', 'id': 23, 'synset': 'artichoke.n.02', 'synonyms': ['artichoke'], 'def': 'a thistlelike flower head with edible fleshy leaves and heart', 'name': 'artichoke'}, {'frequency': 'f', 'id': 24, 'synset': 'ashcan.n.01', 'synonyms': ['trash_can', 'garbage_can', 'wastebin', 'dustbin', 'trash_barrel', 'trash_bin'], 'def': 'a bin that holds rubbish until it is collected', 'name': 'trash_can'}, {'frequency': 'c', 'id': 25, 'synset': 'ashtray.n.01', 'synonyms': ['ashtray'], 'def': "a receptacle for the ash from smokers' cigars or cigarettes", 'name': 'ashtray'}, {'frequency': 'c', 'id': 26, 'synset': 'asparagus.n.02', 'synonyms': ['asparagus'], 'def': 'edible young shoots of the asparagus plant', 'name': 'asparagus'}, {'frequency': 'c', 'id': 27, 'synset': 'atomizer.n.01', 'synonyms': ['atomizer', 'atomiser', 'spray', 'sprayer', 'nebulizer', 'nebuliser'], 'def': 'a dispenser that turns a liquid (such as perfume) into a fine mist', 'name': 'atomizer'}, {'frequency': 'c', 'id': 28, 'synset': 'avocado.n.01', 'synonyms': ['avocado'], 'def': 'a pear-shaped fruit with green or blackish skin and rich yellowish pulp enclosing a single large seed', 'name': 'avocado'}, {'frequency': 'c', 'id': 29, 'synset': 'award.n.02', 'synonyms': ['award', 'accolade'], 'def': 'a tangible symbol signifying approval or distinction', 'name': 'award'}, {'frequency': 'f', 'id': 30, 'synset': 'awning.n.01', 'synonyms': ['awning'], 'def': 'a canopy made of canvas to shelter people or things from rain or sun', 'name': 'awning'}, {'frequency': 'r', 'id': 31, 'synset': 'ax.n.01', 'synonyms': ['ax', 'axe'], 'def': 'an edge tool with a heavy bladed head mounted across a handle', 'name': 'ax'}, {'frequency': 'f', 'id': 32, 'synset': 'baby_buggy.n.01', 'synonyms': ['baby_buggy', 'baby_carriage', 'perambulator', 'pram', 'stroller'], 'def': 'a small vehicle with four wheels in which a baby or child is pushed around', 'name': 'baby_buggy'}, {'frequency': 'c', 'id': 33, 'synset': 'backboard.n.01', 'synonyms': ['basketball_backboard'], 'def': 'a raised vertical board with basket attached; used to play basketball', 'name': 'basketball_backboard'}, {'frequency': 'f', 'id': 34, 'synset': 'backpack.n.01', 'synonyms': ['backpack', 'knapsack', 'packsack', 'rucksack', 'haversack'], 'def': 'a bag carried by a strap on your back or shoulder', 'name': 'backpack'}, {'frequency': 'f', 'id': 35, 'synset': 'bag.n.04', 'synonyms': ['handbag', 'purse', 'pocketbook'], 'def': 'a container used for carrying money and small personal items or accessories', 'name': 'handbag'}, {'frequency': 'f', 'id': 36, 'synset': 'bag.n.06', 'synonyms': ['suitcase', 'baggage', 'luggage'], 'def': 'cases used to carry belongings when traveling', 'name': 'suitcase'}, {'frequency': 'c', 'id': 37, 'synset': 'bagel.n.01', 'synonyms': ['bagel', 'beigel'], 'def': 'glazed yeast-raised doughnut-shaped roll with hard crust', 'name': 'bagel'}, {'frequency': 'r', 'id': 38, 'synset': 'bagpipe.n.01', 'synonyms': ['bagpipe'], 'def': 'a tubular wind instrument; the player blows air into a bag and squeezes it out', 'name': 'bagpipe'}, {'frequency': 'r', 'id': 39, 'synset': 'baguet.n.01', 'synonyms': ['baguet', 'baguette'], 'def': 'narrow French stick loaf', 'name': 'baguet'}, {'frequency': 'r', 'id': 40, 'synset': 'bait.n.02', 'synonyms': ['bait', 'lure'], 'def': 'something used to lure fish or other animals into danger so they can be trapped or killed', 'name': 'bait'}, {'frequency': 'f', 'id': 41, 'synset': 'ball.n.06', 'synonyms': ['ball'], 'def': 'a spherical object used as a plaything', 'name': 'ball'}, {'frequency': 'r', 'id': 42, 'synset': 'ballet_skirt.n.01', 'synonyms': ['ballet_skirt', 'tutu'], 'def': 'very short skirt worn by ballerinas', 'name': 'ballet_skirt'}, {'frequency': 'f', 'id': 43, 'synset': 'balloon.n.01', 'synonyms': ['balloon'], 'def': 'large tough nonrigid bag filled with gas or heated air', 'name': 'balloon'}, {'frequency': 'c', 'id': 44, 'synset': 'bamboo.n.02', 'synonyms': ['bamboo'], 'def': 'woody tropical grass having hollow woody stems', 'name': 'bamboo'}, {'frequency': 'f', 'id': 45, 'synset': 'banana.n.02', 'synonyms': ['banana'], 'def': 'elongated crescent-shaped yellow fruit with soft sweet flesh', 'name': 'banana'}, {'frequency': 'r', 'id': 46, 'synset': 'band_aid.n.01', 'synonyms': ['Band_Aid'], 'def': 'trade name for an adhesive bandage to cover small cuts or blisters', 'name': 'Band_Aid'}, {'frequency': 'c', 'id': 47, 'synset': 'bandage.n.01', 'synonyms': ['bandage'], 'def': 'a piece of soft material that covers and protects an injured part of the body', 'name': 'bandage'}, {'frequency': 'c', 'id': 48, 'synset': 'bandanna.n.01', 'synonyms': ['bandanna', 'bandana'], 'def': 'large and brightly colored handkerchief; often used as a neckerchief', 'name': 'bandanna'}, {'frequency': 'r', 'id': 49, 'synset': 'banjo.n.01', 'synonyms': ['banjo'], 'def': 'a stringed instrument of the guitar family with a long neck and circular body', 'name': 'banjo'}, {'frequency': 'f', 'id': 50, 'synset': 'banner.n.01', 'synonyms': ['banner', 'streamer'], 'def': 'long strip of cloth or paper used for decoration or advertising', 'name': 'banner'}, {'frequency': 'r', 'id': 51, 'synset': 'barbell.n.01', 'synonyms': ['barbell'], 'def': 'a bar to which heavy discs are attached at each end; used in weightlifting', 'name': 'barbell'}, {'frequency': 'r', 'id': 52, 'synset': 'barge.n.01', 'synonyms': ['barge'], 'def': 'a flatbottom boat for carrying heavy loads (especially on canals)', 'name': 'barge'}, {'frequency': 'f', 'id': 53, 'synset': 'barrel.n.02', 'synonyms': ['barrel', 'cask'], 'def': 'a cylindrical container that holds liquids', 'name': 'barrel'}, {'frequency': 'c', 'id': 54, 'synset': 'barrette.n.01', 'synonyms': ['barrette'], 'def': "a pin for holding women's hair in place", 'name': 'barrette'}, {'frequency': 'c', 'id': 55, 'synset': 'barrow.n.03', 'synonyms': ['barrow', 'garden_cart', 'lawn_cart', 'wheelbarrow'], 'def': 'a cart for carrying small loads; has handles and one or more wheels', 'name': 'barrow'}, {'frequency': 'f', 'id': 56, 'synset': 'base.n.03', 'synonyms': ['baseball_base'], 'def': 'a place that the runner must touch before scoring', 'name': 'baseball_base'}, {'frequency': 'f', 'id': 57, 'synset': 'baseball.n.02', 'synonyms': ['baseball'], 'def': 'a ball used in playing baseball', 'name': 'baseball'}, {'frequency': 'f', 'id': 58, 'synset': 'baseball_bat.n.01', 'synonyms': ['baseball_bat'], 'def': 'an implement used in baseball by the batter', 'name': 'baseball_bat'}, {'frequency': 'f', 'id': 59, 'synset': 'baseball_cap.n.01', 'synonyms': ['baseball_cap', 'jockey_cap', 'golf_cap'], 'def': 'a cap with a bill', 'name': 'baseball_cap'}, {'frequency': 'f', 'id': 60, 'synset': 'baseball_glove.n.01', 'synonyms': ['baseball_glove', 'baseball_mitt'], 'def': 'the handwear used by fielders in playing baseball', 'name': 'baseball_glove'}, {'frequency': 'f', 'id': 61, 'synset': 'basket.n.01', 'synonyms': ['basket', 'handbasket'], 'def': 'a container that is usually woven and has handles', 'name': 'basket'}, {'frequency': 'c', 'id': 62, 'synset': 'basket.n.03', 'synonyms': ['basketball_hoop'], 'def': 'metal hoop supporting a net through which players try to throw the basketball', 'name': 'basketball_hoop'}, {'frequency': 'c', 'id': 63, 'synset': 'basketball.n.02', 'synonyms': ['basketball'], 'def': 'an inflated ball used in playing basketball', 'name': 'basketball'}, {'frequency': 'r', 'id': 64, 'synset': 'bass_horn.n.01', 'synonyms': ['bass_horn', 'sousaphone', 'tuba'], 'def': 'the lowest brass wind instrument', 'name': 'bass_horn'}, {'frequency': 'r', 'id': 65, 'synset': 'bat.n.01', 'synonyms': ['bat_(animal)'], 'def': 'nocturnal mouselike mammal with forelimbs modified to form membranous wings', 'name': 'bat_(animal)'}, {'frequency': 'f', 'id': 66, 'synset': 'bath_mat.n.01', 'synonyms': ['bath_mat'], 'def': 'a heavy towel or mat to stand on while drying yourself after a bath', 'name': 'bath_mat'}, {'frequency': 'f', 'id': 67, 'synset': 'bath_towel.n.01', 'synonyms': ['bath_towel'], 'def': 'a large towel; to dry yourself after a bath', 'name': 'bath_towel'}, {'frequency': 'c', 'id': 68, 'synset': 'bathrobe.n.01', 'synonyms': ['bathrobe'], 'def': 'a loose-fitting robe of towelling; worn after a bath or swim', 'name': 'bathrobe'}, {'frequency': 'f', 'id': 69, 'synset': 'bathtub.n.01', 'synonyms': ['bathtub', 'bathing_tub'], 'def': 'a large open container that you fill with water and use to wash the body', 'name': 'bathtub'}, {'frequency': 'r', 'id': 70, 'synset': 'batter.n.02', 'synonyms': ['batter_(food)'], 'def': 'a liquid or semiliquid mixture, as of flour, eggs, and milk, used in cooking', 'name': 'batter_(food)'}, {'frequency': 'c', 'id': 71, 'synset': 'battery.n.02', 'synonyms': ['battery'], 'def': 'a portable device that produces electricity', 'name': 'battery'}, {'frequency': 'r', 'id': 72, 'synset': 'beach_ball.n.01', 'synonyms': ['beachball'], 'def': 'large and light ball; for play at the seaside', 'name': 'beachball'}, {'frequency': 'c', 'id': 73, 'synset': 'bead.n.01', 'synonyms': ['bead'], 'def': 'a small ball with a hole through the middle used for ornamentation, jewellery, etc.', 'name': 'bead'}, {'frequency': 'r', 'id': 74, 'synset': 'beaker.n.01', 'synonyms': ['beaker'], 'def': 'a flatbottomed jar made of glass or plastic; used for chemistry', 'name': 'beaker'}, {'frequency': 'c', 'id': 75, 'synset': 'bean_curd.n.01', 'synonyms': ['bean_curd', 'tofu'], 'def': 'cheeselike food made of curdled soybean milk', 'name': 'bean_curd'}, {'frequency': 'c', 'id': 76, 'synset': 'beanbag.n.01', 'synonyms': ['beanbag'], 'def': 'a bag filled with dried beans or similar items; used in games or to sit on', 'name': 'beanbag'}, {'frequency': 'f', 'id': 77, 'synset': 'beanie.n.01', 'synonyms': ['beanie', 'beany'], 'def': 'a small skullcap; formerly worn by schoolboys and college freshmen', 'name': 'beanie'}, {'frequency': 'f', 'id': 78, 'synset': 'bear.n.01', 'synonyms': ['bear'], 'def': 'large carnivorous or omnivorous mammals with shaggy coats and claws', 'name': 'bear'}, {'frequency': 'f', 'id': 79, 'synset': 'bed.n.01', 'synonyms': ['bed'], 'def': 'a piece of furniture that provides a place to sleep', 'name': 'bed'}, {'frequency': 'c', 'id': 80, 'synset': 'bedspread.n.01', 'synonyms': ['bedspread', 'bedcover', 'bed_covering', 'counterpane', 'spread'], 'def': 'decorative cover for a bed', 'name': 'bedspread'}, {'frequency': 'f', 'id': 81, 'synset': 'beef.n.01', 'synonyms': ['cow'], 'def': 'cattle that are reared for their meat', 'name': 'cow'}, {'frequency': 'c', 'id': 82, 'synset': 'beef.n.02', 'synonyms': ['beef_(food)', 'boeuf_(food)'], 'def': 'meat from an adult domestic bovine', 'name': 'beef_(food)'}, {'frequency': 'r', 'id': 83, 'synset': 'beeper.n.01', 'synonyms': ['beeper', 'pager'], 'def': 'an device that beeps when the person carrying it is being paged', 'name': 'beeper'}, {'frequency': 'f', 'id': 84, 'synset': 'beer_bottle.n.01', 'synonyms': ['beer_bottle'], 'def': 'a bottle that holds beer', 'name': 'beer_bottle'}, {'frequency': 'c', 'id': 85, 'synset': 'beer_can.n.01', 'synonyms': ['beer_can'], 'def': 'a can that holds beer', 'name': 'beer_can'}, {'frequency': 'r', 'id': 86, 'synset': 'beetle.n.01', 'synonyms': ['beetle'], 'def': 'insect with hard wing covers', 'name': 'beetle'}, {'frequency': 'f', 'id': 87, 'synset': 'bell.n.01', 'synonyms': ['bell'], 'def': 'a hollow device made of metal that makes a ringing sound when struck', 'name': 'bell'}, {'frequency': 'f', 'id': 88, 'synset': 'bell_pepper.n.02', 'synonyms': ['bell_pepper', 'capsicum'], 'def': 'large bell-shaped sweet pepper in green or red or yellow or orange or black varieties', 'name': 'bell_pepper'}, {'frequency': 'f', 'id': 89, 'synset': 'belt.n.02', 'synonyms': ['belt'], 'def': 'a band to tie or buckle around the body (usually at the waist)', 'name': 'belt'}, {'frequency': 'f', 'id': 90, 'synset': 'belt_buckle.n.01', 'synonyms': ['belt_buckle'], 'def': 'the buckle used to fasten a belt', 'name': 'belt_buckle'}, {'frequency': 'f', 'id': 91, 'synset': 'bench.n.01', 'synonyms': ['bench'], 'def': 'a long seat for more than one person', 'name': 'bench'}, {'frequency': 'c', 'id': 92, 'synset': 'beret.n.01', 'synonyms': ['beret'], 'def': 'a cap with no brim or bill; made of soft cloth', 'name': 'beret'}, {'frequency': 'c', 'id': 93, 'synset': 'bib.n.02', 'synonyms': ['bib'], 'def': 'a napkin tied under the chin of a child while eating', 'name': 'bib'}, {'frequency': 'r', 'id': 94, 'synset': 'bible.n.01', 'synonyms': ['Bible'], 'def': 'the sacred writings of the Christian religions', 'name': 'Bible'}, {'frequency': 'f', 'id': 95, 'synset': 'bicycle.n.01', 'synonyms': ['bicycle', 'bike_(bicycle)'], 'def': 'a wheeled vehicle that has two wheels and is moved by foot pedals', 'name': 'bicycle'}, {'frequency': 'f', 'id': 96, 'synset': 'bill.n.09', 'synonyms': ['visor', 'vizor'], 'def': 'a brim that projects to the front to shade the eyes', 'name': 'visor'}, {'frequency': 'c', 'id': 97, 'synset': 'binder.n.03', 'synonyms': ['binder', 'ring-binder'], 'def': 'holds loose papers or magazines', 'name': 'binder'}, {'frequency': 'c', 'id': 98, 'synset': 'binoculars.n.01', 'synonyms': ['binoculars', 'field_glasses', 'opera_glasses'], 'def': 'an optical instrument designed for simultaneous use by both eyes', 'name': 'binoculars'}, {'frequency': 'f', 'id': 99, 'synset': 'bird.n.01', 'synonyms': ['bird'], 'def': 'animal characterized by feathers and wings', 'name': 'bird'}, {'frequency': 'r', 'id': 100, 'synset': 'bird_feeder.n.01', 'synonyms': ['birdfeeder'], 'def': 'an outdoor device that supplies food for wild birds', 'name': 'birdfeeder'}, {'frequency': 'r', 'id': 101, 'synset': 'birdbath.n.01', 'synonyms': ['birdbath'], 'def': 'an ornamental basin (usually in a garden) for birds to bathe in', 'name': 'birdbath'}, {'frequency': 'c', 'id': 102, 'synset': 'birdcage.n.01', 'synonyms': ['birdcage'], 'def': 'a cage in which a bird can be kept', 'name': 'birdcage'}, {'frequency': 'c', 'id': 103, 'synset': 'birdhouse.n.01', 'synonyms': ['birdhouse'], 'def': 'a shelter for birds', 'name': 'birdhouse'}, {'frequency': 'f', 'id': 104, 'synset': 'birthday_cake.n.01', 'synonyms': ['birthday_cake'], 'def': 'decorated cake served at a birthday party', 'name': 'birthday_cake'}, {'frequency': 'r', 'id': 105, 'synset': 'birthday_card.n.01', 'synonyms': ['birthday_card'], 'def': 'a card expressing a birthday greeting', 'name': 'birthday_card'}, {'frequency': 'r', 'id': 106, 'synset': 'biscuit.n.01', 'synonyms': ['biscuit_(bread)'], 'def': 'small round bread leavened with baking-powder or soda', 'name': 'biscuit_(bread)'}, {'frequency': 'r', 'id': 107, 'synset': 'black_flag.n.01', 'synonyms': ['pirate_flag'], 'def': 'a flag usually bearing a white skull and crossbones on a black background', 'name': 'pirate_flag'}, {'frequency': 'c', 'id': 108, 'synset': 'black_sheep.n.02', 'synonyms': ['black_sheep'], 'def': 'sheep with a black coat', 'name': 'black_sheep'}, {'frequency': 'c', 'id': 109, 'synset': 'blackboard.n.01', 'synonyms': ['blackboard', 'chalkboard'], 'def': 'sheet of slate; for writing with chalk', 'name': 'blackboard'}, {'frequency': 'f', 'id': 110, 'synset': 'blanket.n.01', 'synonyms': ['blanket'], 'def': 'bedding that keeps a person warm in bed', 'name': 'blanket'}, {'frequency': 'c', 'id': 111, 'synset': 'blazer.n.01', 'synonyms': ['blazer', 'sport_jacket', 'sport_coat', 'sports_jacket', 'sports_coat'], 'def': 'lightweight jacket; often striped in the colors of a club or school', 'name': 'blazer'}, {'frequency': 'f', 'id': 112, 'synset': 'blender.n.01', 'synonyms': ['blender', 'liquidizer', 'liquidiser'], 'def': 'an electrically powered mixer that mix or chop or liquefy foods', 'name': 'blender'}, {'frequency': 'r', 'id': 113, 'synset': 'blimp.n.02', 'synonyms': ['blimp'], 'def': 'a small nonrigid airship used for observation or as a barrage balloon', 'name': 'blimp'}, {'frequency': 'c', 'id': 114, 'synset': 'blinker.n.01', 'synonyms': ['blinker', 'flasher'], 'def': 'a light that flashes on and off; used as a signal or to send messages', 'name': 'blinker'}, {'frequency': 'c', 'id': 115, 'synset': 'blueberry.n.02', 'synonyms': ['blueberry'], 'def': 'sweet edible dark-blue berries of blueberry plants', 'name': 'blueberry'}, {'frequency': 'r', 'id': 116, 'synset': 'boar.n.02', 'synonyms': ['boar'], 'def': 'an uncastrated male hog', 'name': 'boar'}, {'frequency': 'r', 'id': 117, 'synset': 'board.n.09', 'synonyms': ['gameboard'], 'def': 'a flat portable surface (usually rectangular) designed for board games', 'name': 'gameboard'}, {'frequency': 'f', 'id': 118, 'synset': 'boat.n.01', 'synonyms': ['boat', 'ship_(boat)'], 'def': 'a vessel for travel on water', 'name': 'boat'}, {'frequency': 'c', 'id': 119, 'synset': 'bobbin.n.01', 'synonyms': ['bobbin', 'spool', 'reel'], 'def': 'a thing around which thread/tape/film or other flexible materials can be wound', 'name': 'bobbin'}, {'frequency': 'r', 'id': 120, 'synset': 'bobby_pin.n.01', 'synonyms': ['bobby_pin', 'hairgrip'], 'def': 'a flat wire hairpin used to hold bobbed hair in place', 'name': 'bobby_pin'}, {'frequency': 'c', 'id': 121, 'synset': 'boiled_egg.n.01', 'synonyms': ['boiled_egg', 'coddled_egg'], 'def': 'egg cooked briefly in the shell in gently boiling water', 'name': 'boiled_egg'}, {'frequency': 'r', 'id': 122, 'synset': 'bolo_tie.n.01', 'synonyms': ['bolo_tie', 'bolo', 'bola_tie', 'bola'], 'def': 'a cord fastened around the neck with an ornamental clasp and worn as a necktie', 'name': 'bolo_tie'}, {'frequency': 'c', 'id': 123, 'synset': 'bolt.n.03', 'synonyms': ['deadbolt'], 'def': 'the part of a lock that is engaged or withdrawn with a key', 'name': 'deadbolt'}, {'frequency': 'f', 'id': 124, 'synset': 'bolt.n.06', 'synonyms': ['bolt'], 'def': 'a screw that screws into a nut to form a fastener', 'name': 'bolt'}, {'frequency': 'r', 'id': 125, 'synset': 'bonnet.n.01', 'synonyms': ['bonnet'], 'def': 'a hat tied under the chin', 'name': 'bonnet'}, {'frequency': 'f', 'id': 126, 'synset': 'book.n.01', 'synonyms': ['book'], 'def': 'a written work or composition that has been published', 'name': 'book'}, {'frequency': 'r', 'id': 127, 'synset': 'book_bag.n.01', 'synonyms': ['book_bag'], 'def': 'a bag in which students carry their books', 'name': 'book_bag'}, {'frequency': 'c', 'id': 128, 'synset': 'bookcase.n.01', 'synonyms': ['bookcase'], 'def': 'a piece of furniture with shelves for storing books', 'name': 'bookcase'}, {'frequency': 'c', 'id': 129, 'synset': 'booklet.n.01', 'synonyms': ['booklet', 'brochure', 'leaflet', 'pamphlet'], 'def': 'a small book usually having a paper cover', 'name': 'booklet'}, {'frequency': 'r', 'id': 130, 'synset': 'bookmark.n.01', 'synonyms': ['bookmark', 'bookmarker'], 'def': 'a marker (a piece of paper or ribbon) placed between the pages of a book', 'name': 'bookmark'}, {'frequency': 'r', 'id': 131, 'synset': 'boom.n.04', 'synonyms': ['boom_microphone', 'microphone_boom'], 'def': 'a pole carrying an overhead microphone projected over a film or tv set', 'name': 'boom_microphone'}, {'frequency': 'f', 'id': 132, 'synset': 'boot.n.01', 'synonyms': ['boot'], 'def': 'footwear that covers the whole foot and lower leg', 'name': 'boot'}, {'frequency': 'f', 'id': 133, 'synset': 'bottle.n.01', 'synonyms': ['bottle'], 'def': 'a glass or plastic vessel used for storing drinks or other liquids', 'name': 'bottle'}, {'frequency': 'c', 'id': 134, 'synset': 'bottle_opener.n.01', 'synonyms': ['bottle_opener'], 'def': 'an opener for removing caps or corks from bottles', 'name': 'bottle_opener'}, {'frequency': 'c', 'id': 135, 'synset': 'bouquet.n.01', 'synonyms': ['bouquet'], 'def': 'an arrangement of flowers that is usually given as a present', 'name': 'bouquet'}, {'frequency': 'r', 'id': 136, 'synset': 'bow.n.04', 'synonyms': ['bow_(weapon)'], 'def': 'a weapon for shooting arrows', 'name': 'bow_(weapon)'}, {'frequency': 'f', 'id': 137, 'synset': 'bow.n.08', 'synonyms': ['bow_(decorative_ribbons)'], 'def': 'a decorative interlacing of ribbons', 'name': 'bow_(decorative_ribbons)'}, {'frequency': 'f', 'id': 138, 'synset': 'bow_tie.n.01', 'synonyms': ['bow-tie', 'bowtie'], 'def': "a man's tie that ties in a bow", 'name': 'bow-tie'}, {'frequency': 'f', 'id': 139, 'synset': 'bowl.n.03', 'synonyms': ['bowl'], 'def': 'a dish that is round and open at the top for serving foods', 'name': 'bowl'}, {'frequency': 'r', 'id': 140, 'synset': 'bowl.n.08', 'synonyms': ['pipe_bowl'], 'def': 'a small round container that is open at the top for holding tobacco', 'name': 'pipe_bowl'}, {'frequency': 'c', 'id': 141, 'synset': 'bowler_hat.n.01', 'synonyms': ['bowler_hat', 'bowler', 'derby_hat', 'derby', 'plug_hat'], 'def': 'a felt hat that is round and hard with a narrow brim', 'name': 'bowler_hat'}, {'frequency': 'r', 'id': 142, 'synset': 'bowling_ball.n.01', 'synonyms': ['bowling_ball'], 'def': 'a large ball with finger holes used in the sport of bowling', 'name': 'bowling_ball'}, {'frequency': 'r', 'id': 143, 'synset': 'bowling_pin.n.01', 'synonyms': ['bowling_pin'], 'def': 'a club-shaped wooden object used in bowling', 'name': 'bowling_pin'}, {'frequency': 'r', 'id': 144, 'synset': 'boxing_glove.n.01', 'synonyms': ['boxing_glove'], 'def': 'large glove coverings the fists of a fighter worn for the sport of boxing', 'name': 'boxing_glove'}, {'frequency': 'c', 'id': 145, 'synset': 'brace.n.06', 'synonyms': ['suspenders'], 'def': 'elastic straps that hold trousers up (usually used in the plural)', 'name': 'suspenders'}, {'frequency': 'f', 'id': 146, 'synset': 'bracelet.n.02', 'synonyms': ['bracelet', 'bangle'], 'def': 'jewelry worn around the wrist for decoration', 'name': 'bracelet'}, {'frequency': 'r', 'id': 147, 'synset': 'brass.n.07', 'synonyms': ['brass_plaque'], 'def': 'a memorial made of brass', 'name': 'brass_plaque'}, {'frequency': 'c', 'id': 148, 'synset': 'brassiere.n.01', 'synonyms': ['brassiere', 'bra', 'bandeau'], 'def': 'an undergarment worn by women to support their breasts', 'name': 'brassiere'}, {'frequency': 'c', 'id': 149, 'synset': 'bread-bin.n.01', 'synonyms': ['bread-bin', 'breadbox'], 'def': 'a container used to keep bread or cake in', 'name': 'bread-bin'}, {'frequency': 'r', 'id': 150, 'synset': 'breechcloth.n.01', 'synonyms': ['breechcloth', 'breechclout', 'loincloth'], 'def': 'a garment that provides covering for the loins', 'name': 'breechcloth'}, {'frequency': 'c', 'id': 151, 'synset': 'bridal_gown.n.01', 'synonyms': ['bridal_gown', 'wedding_gown', 'wedding_dress'], 'def': 'a gown worn by the bride at a wedding', 'name': 'bridal_gown'}, {'frequency': 'c', 'id': 152, 'synset': 'briefcase.n.01', 'synonyms': ['briefcase'], 'def': 'a case with a handle; for carrying papers or files or books', 'name': 'briefcase'}, {'frequency': 'c', 'id': 153, 'synset': 'bristle_brush.n.01', 'synonyms': ['bristle_brush'], 'def': 'a brush that is made with the short stiff hairs of an animal or plant', 'name': 'bristle_brush'}, {'frequency': 'f', 'id': 154, 'synset': 'broccoli.n.01', 'synonyms': ['broccoli'], 'def': 'plant with dense clusters of tight green flower buds', 'name': 'broccoli'}, {'frequency': 'r', 'id': 155, 'synset': 'brooch.n.01', 'synonyms': ['broach'], 'def': 'a decorative pin worn by women', 'name': 'broach'}, {'frequency': 'c', 'id': 156, 'synset': 'broom.n.01', 'synonyms': ['broom'], 'def': 'bundle of straws or twigs attached to a long handle; used for cleaning', 'name': 'broom'}, {'frequency': 'c', 'id': 157, 'synset': 'brownie.n.03', 'synonyms': ['brownie'], 'def': 'square or bar of very rich chocolate cake usually with nuts', 'name': 'brownie'}, {'frequency': 'c', 'id': 158, 'synset': 'brussels_sprouts.n.01', 'synonyms': ['brussels_sprouts'], 'def': 'the small edible cabbage-like buds growing along a stalk', 'name': 'brussels_sprouts'}, {'frequency': 'r', 'id': 159, 'synset': 'bubble_gum.n.01', 'synonyms': ['bubble_gum'], 'def': 'a kind of chewing gum that can be blown into bubbles', 'name': 'bubble_gum'}, {'frequency': 'f', 'id': 160, 'synset': 'bucket.n.01', 'synonyms': ['bucket', 'pail'], 'def': 'a roughly cylindrical vessel that is open at the top', 'name': 'bucket'}, {'frequency': 'r', 'id': 161, 'synset': 'buggy.n.01', 'synonyms': ['horse_buggy'], 'def': 'a small lightweight carriage; drawn by a single horse', 'name': 'horse_buggy'}, {'frequency': 'c', 'id': 162, 'synset': 'bull.n.11', 'synonyms': ['bull'], 'def': 'mature male cow', 'name': 'bull'}, {'frequency': 'r', 'id': 163, 'synset': 'bulldog.n.01', 'synonyms': ['bulldog'], 'def': 'a thickset short-haired dog with a large head and strong undershot lower jaw', 'name': 'bulldog'}, {'frequency': 'r', 'id': 164, 'synset': 'bulldozer.n.01', 'synonyms': ['bulldozer', 'dozer'], 'def': 'large powerful tractor; a large blade in front flattens areas of ground', 'name': 'bulldozer'}, {'frequency': 'c', 'id': 165, 'synset': 'bullet_train.n.01', 'synonyms': ['bullet_train'], 'def': 'a high-speed passenger train', 'name': 'bullet_train'}, {'frequency': 'c', 'id': 166, 'synset': 'bulletin_board.n.02', 'synonyms': ['bulletin_board', 'notice_board'], 'def': 'a board that hangs on a wall; displays announcements', 'name': 'bulletin_board'}, {'frequency': 'r', 'id': 167, 'synset': 'bulletproof_vest.n.01', 'synonyms': ['bulletproof_vest'], 'def': 'a vest capable of resisting the impact of a bullet', 'name': 'bulletproof_vest'}, {'frequency': 'c', 'id': 168, 'synset': 'bullhorn.n.01', 'synonyms': ['bullhorn', 'megaphone'], 'def': 'a portable loudspeaker with built-in microphone and amplifier', 'name': 'bullhorn'}, {'frequency': 'r', 'id': 169, 'synset': 'bully_beef.n.01', 'synonyms': ['corned_beef', 'corn_beef'], 'def': 'beef cured or pickled in brine', 'name': 'corned_beef'}, {'frequency': 'f', 'id': 170, 'synset': 'bun.n.01', 'synonyms': ['bun', 'roll'], 'def': 'small rounded bread either plain or sweet', 'name': 'bun'}, {'frequency': 'c', 'id': 171, 'synset': 'bunk_bed.n.01', 'synonyms': ['bunk_bed'], 'def': 'beds built one above the other', 'name': 'bunk_bed'}, {'frequency': 'f', 'id': 172, 'synset': 'buoy.n.01', 'synonyms': ['buoy'], 'def': 'a float attached by rope to the seabed to mark channels in a harbor or underwater hazards', 'name': 'buoy'}, {'frequency': 'r', 'id': 173, 'synset': 'burrito.n.01', 'synonyms': ['burrito'], 'def': 'a flour tortilla folded around a filling', 'name': 'burrito'}, {'frequency': 'f', 'id': 174, 'synset': 'bus.n.01', 'synonyms': ['bus_(vehicle)', 'autobus', 'charabanc', 'double-decker', 'motorbus', 'motorcoach'], 'def': 'a vehicle carrying many passengers; used for public transport', 'name': 'bus_(vehicle)'}, {'frequency': 'c', 'id': 175, 'synset': 'business_card.n.01', 'synonyms': ['business_card'], 'def': "a card on which are printed the person's name and business affiliation", 'name': 'business_card'}, {'frequency': 'c', 'id': 176, 'synset': 'butcher_knife.n.01', 'synonyms': ['butcher_knife'], 'def': 'a large sharp knife for cutting or trimming meat', 'name': 'butcher_knife'}, {'frequency': 'c', 'id': 177, 'synset': 'butter.n.01', 'synonyms': ['butter'], 'def': 'an edible emulsion of fat globules made by churning milk or cream; for cooking and table use', 'name': 'butter'}, {'frequency': 'c', 'id': 178, 'synset': 'butterfly.n.01', 'synonyms': ['butterfly'], 'def': 'insect typically having a slender body with knobbed antennae and broad colorful wings', 'name': 'butterfly'}, {'frequency': 'f', 'id': 179, 'synset': 'button.n.01', 'synonyms': ['button'], 'def': 'a round fastener sewn to shirts and coats etc to fit through buttonholes', 'name': 'button'}, {'frequency': 'f', 'id': 180, 'synset': 'cab.n.03', 'synonyms': ['cab_(taxi)', 'taxi', 'taxicab'], 'def': 'a car that takes passengers where they want to go in exchange for money', 'name': 'cab_(taxi)'}, {'frequency': 'r', 'id': 181, 'synset': 'cabana.n.01', 'synonyms': ['cabana'], 'def': 'a small tent used as a dressing room beside the sea or a swimming pool', 'name': 'cabana'}, {'frequency': 'r', 'id': 182, 'synset': 'cabin_car.n.01', 'synonyms': ['cabin_car', 'caboose'], 'def': 'a car on a freight train for use of the train crew; usually the last car on the train', 'name': 'cabin_car'}, {'frequency': 'f', 'id': 183, 'synset': 'cabinet.n.01', 'synonyms': ['cabinet'], 'def': 'a piece of furniture resembling a cupboard with doors and shelves and drawers', 'name': 'cabinet'}, {'frequency': 'r', 'id': 184, 'synset': 'cabinet.n.03', 'synonyms': ['locker', 'storage_locker'], 'def': 'a storage compartment for clothes and valuables; usually it has a lock', 'name': 'locker'}, {'frequency': 'f', 'id': 185, 'synset': 'cake.n.03', 'synonyms': ['cake'], 'def': 'baked goods made from or based on a mixture of flour, sugar, eggs, and fat', 'name': 'cake'}, {'frequency': 'c', 'id': 186, 'synset': 'calculator.n.02', 'synonyms': ['calculator'], 'def': 'a small machine that is used for mathematical calculations', 'name': 'calculator'}, {'frequency': 'f', 'id': 187, 'synset': 'calendar.n.02', 'synonyms': ['calendar'], 'def': 'a list or register of events (appointments/social events/court cases, etc)', 'name': 'calendar'}, {'frequency': 'c', 'id': 188, 'synset': 'calf.n.01', 'synonyms': ['calf'], 'def': 'young of domestic cattle', 'name': 'calf'}, {'frequency': 'c', 'id': 189, 'synset': 'camcorder.n.01', 'synonyms': ['camcorder'], 'def': 'a portable television camera and videocassette recorder', 'name': 'camcorder'}, {'frequency': 'c', 'id': 190, 'synset': 'camel.n.01', 'synonyms': ['camel'], 'def': 'cud-chewing mammal used as a draft or saddle animal in desert regions', 'name': 'camel'}, {'frequency': 'f', 'id': 191, 'synset': 'camera.n.01', 'synonyms': ['camera'], 'def': 'equipment for taking photographs', 'name': 'camera'}, {'frequency': 'c', 'id': 192, 'synset': 'camera_lens.n.01', 'synonyms': ['camera_lens'], 'def': 'a lens that focuses the image in a camera', 'name': 'camera_lens'}, {'frequency': 'c', 'id': 193, 'synset': 'camper.n.02', 'synonyms': ['camper_(vehicle)', 'camping_bus', 'motor_home'], 'def': 'a recreational vehicle equipped for camping out while traveling', 'name': 'camper_(vehicle)'}, {'frequency': 'f', 'id': 194, 'synset': 'can.n.01', 'synonyms': ['can', 'tin_can'], 'def': 'airtight sealed metal container for food or drink or paint etc.', 'name': 'can'}, {'frequency': 'c', 'id': 195, 'synset': 'can_opener.n.01', 'synonyms': ['can_opener', 'tin_opener'], 'def': 'a device for cutting cans open', 'name': 'can_opener'}, {'frequency': 'r', 'id': 196, 'synset': 'candelabrum.n.01', 'synonyms': ['candelabrum', 'candelabra'], 'def': 'branched candlestick; ornamental; has several lights', 'name': 'candelabrum'}, {'frequency': 'f', 'id': 197, 'synset': 'candle.n.01', 'synonyms': ['candle', 'candlestick'], 'def': 'stick of wax with a wick in the middle', 'name': 'candle'}, {'frequency': 'f', 'id': 198, 'synset': 'candlestick.n.01', 'synonyms': ['candle_holder'], 'def': 'a holder with sockets for candles', 'name': 'candle_holder'}, {'frequency': 'r', 'id': 199, 'synset': 'candy_bar.n.01', 'synonyms': ['candy_bar'], 'def': 'a candy shaped as a bar', 'name': 'candy_bar'}, {'frequency': 'c', 'id': 200, 'synset': 'candy_cane.n.01', 'synonyms': ['candy_cane'], 'def': 'a hard candy in the shape of a rod (usually with stripes)', 'name': 'candy_cane'}, {'frequency': 'c', 'id': 201, 'synset': 'cane.n.01', 'synonyms': ['walking_cane'], 'def': 'a stick that people can lean on to help them walk', 'name': 'walking_cane'}, {'frequency': 'c', 'id': 202, 'synset': 'canister.n.02', 'synonyms': ['canister', 'cannister'], 'def': 'metal container for storing dry foods such as tea or flour', 'name': 'canister'}, {'frequency': 'r', 'id': 203, 'synset': 'cannon.n.02', 'synonyms': ['cannon'], 'def': 'heavy gun fired from a tank', 'name': 'cannon'}, {'frequency': 'c', 'id': 204, 'synset': 'canoe.n.01', 'synonyms': ['canoe'], 'def': 'small and light boat; pointed at both ends; propelled with a paddle', 'name': 'canoe'}, {'frequency': 'r', 'id': 205, 'synset': 'cantaloup.n.02', 'synonyms': ['cantaloup', 'cantaloupe'], 'def': 'the fruit of a cantaloup vine; small to medium-sized melon with yellowish flesh', 'name': 'cantaloup'}, {'frequency': 'r', 'id': 206, 'synset': 'canteen.n.01', 'synonyms': ['canteen'], 'def': 'a flask for carrying water; used by soldiers or travelers', 'name': 'canteen'}, {'frequency': 'c', 'id': 207, 'synset': 'cap.n.01', 'synonyms': ['cap_(headwear)'], 'def': 'a tight-fitting headwear', 'name': 'cap_(headwear)'}, {'frequency': 'f', 'id': 208, 'synset': 'cap.n.02', 'synonyms': ['bottle_cap', 'cap_(container_lid)'], 'def': 'a top (as for a bottle)', 'name': 'bottle_cap'}, {'frequency': 'r', 'id': 209, 'synset': 'cape.n.02', 'synonyms': ['cape'], 'def': 'a sleeveless garment like a cloak but shorter', 'name': 'cape'}, {'frequency': 'c', 'id': 210, 'synset': 'cappuccino.n.01', 'synonyms': ['cappuccino', 'coffee_cappuccino'], 'def': 'equal parts of espresso and steamed milk', 'name': 'cappuccino'}, {'frequency': 'f', 'id': 211, 'synset': 'car.n.01', 'synonyms': ['car_(automobile)', 'auto_(automobile)', 'automobile'], 'def': 'a motor vehicle with four wheels', 'name': 'car_(automobile)'}, {'frequency': 'f', 'id': 212, 'synset': 'car.n.02', 'synonyms': ['railcar_(part_of_a_train)', 'railway_car_(part_of_a_train)', 'railroad_car_(part_of_a_train)'], 'def': 'a wheeled vehicle adapted to the rails of railroad', 'name': 'railcar_(part_of_a_train)'}, {'frequency': 'r', 'id': 213, 'synset': 'car.n.04', 'synonyms': ['elevator_car'], 'def': 'where passengers ride up and down', 'name': 'elevator_car'}, {'frequency': 'r', 'id': 214, 'synset': 'car_battery.n.01', 'synonyms': ['car_battery', 'automobile_battery'], 'def': 'a battery in a motor vehicle', 'name': 'car_battery'}, {'frequency': 'c', 'id': 215, 'synset': 'card.n.02', 'synonyms': ['identity_card'], 'def': 'a card certifying the identity of the bearer', 'name': 'identity_card'}, {'frequency': 'c', 'id': 216, 'synset': 'card.n.03', 'synonyms': ['card'], 'def': 'a rectangular piece of paper used to send messages (e.g. greetings or pictures)', 'name': 'card'}, {'frequency': 'r', 'id': 217, 'synset': 'cardigan.n.01', 'synonyms': ['cardigan'], 'def': 'knitted jacket that is fastened up the front with buttons or a zipper', 'name': 'cardigan'}, {'frequency': 'r', 'id': 218, 'synset': 'cargo_ship.n.01', 'synonyms': ['cargo_ship', 'cargo_vessel'], 'def': 'a ship designed to carry cargo', 'name': 'cargo_ship'}, {'frequency': 'r', 'id': 219, 'synset': 'carnation.n.01', 'synonyms': ['carnation'], 'def': 'plant with pink to purple-red spice-scented usually double flowers', 'name': 'carnation'}, {'frequency': 'c', 'id': 220, 'synset': 'carriage.n.02', 'synonyms': ['horse_carriage'], 'def': 'a vehicle with wheels drawn by one or more horses', 'name': 'horse_carriage'}, {'frequency': 'f', 'id': 221, 'synset': 'carrot.n.01', 'synonyms': ['carrot'], 'def': 'deep orange edible root of the cultivated carrot plant', 'name': 'carrot'}, {'frequency': 'c', 'id': 222, 'synset': 'carryall.n.01', 'synonyms': ['tote_bag'], 'def': 'a capacious bag or basket', 'name': 'tote_bag'}, {'frequency': 'c', 'id': 223, 'synset': 'cart.n.01', 'synonyms': ['cart'], 'def': 'a heavy open wagon usually having two wheels and drawn by an animal', 'name': 'cart'}, {'frequency': 'c', 'id': 224, 'synset': 'carton.n.02', 'synonyms': ['carton'], 'def': 'a box made of cardboard; opens by flaps on top', 'name': 'carton'}, {'frequency': 'c', 'id': 225, 'synset': 'cash_register.n.01', 'synonyms': ['cash_register', 'register_(for_cash_transactions)'], 'def': 'a cashbox with an adding machine to register transactions', 'name': 'cash_register'}, {'frequency': 'r', 'id': 226, 'synset': 'casserole.n.01', 'synonyms': ['casserole'], 'def': 'food cooked and served in a casserole', 'name': 'casserole'}, {'frequency': 'r', 'id': 227, 'synset': 'cassette.n.01', 'synonyms': ['cassette'], 'def': 'a container that holds a magnetic tape used for recording or playing sound or video', 'name': 'cassette'}, {'frequency': 'c', 'id': 228, 'synset': 'cast.n.05', 'synonyms': ['cast', 'plaster_cast', 'plaster_bandage'], 'def': 'bandage consisting of a firm covering that immobilizes broken bones while they heal', 'name': 'cast'}, {'frequency': 'f', 'id': 229, 'synset': 'cat.n.01', 'synonyms': ['cat'], 'def': 'a domestic house cat', 'name': 'cat'}, {'frequency': 'c', 'id': 230, 'synset': 'cauliflower.n.02', 'synonyms': ['cauliflower'], 'def': 'edible compact head of white undeveloped flowers', 'name': 'cauliflower'}, {'frequency': 'r', 'id': 231, 'synset': 'caviar.n.01', 'synonyms': ['caviar', 'caviare'], 'def': "salted roe of sturgeon or other large fish; usually served as an hors d'oeuvre", 'name': 'caviar'}, {'frequency': 'c', 'id': 232, 'synset': 'cayenne.n.02', 'synonyms': ['cayenne_(spice)', 'cayenne_pepper_(spice)', 'red_pepper_(spice)'], 'def': 'ground pods and seeds of pungent red peppers of the genus Capsicum', 'name': 'cayenne_(spice)'}, {'frequency': 'c', 'id': 233, 'synset': 'cd_player.n.01', 'synonyms': ['CD_player'], 'def': 'electronic equipment for playing compact discs (CDs)', 'name': 'CD_player'}, {'frequency': 'c', 'id': 234, 'synset': 'celery.n.01', 'synonyms': ['celery'], 'def': 'widely cultivated herb with aromatic leaf stalks that are eaten raw or cooked', 'name': 'celery'}, {'frequency': 'f', 'id': 235, 'synset': 'cellular_telephone.n.01', 'synonyms': ['cellular_telephone', 'cellular_phone', 'cellphone', 'mobile_phone', 'smart_phone'], 'def': 'a hand-held mobile telephone', 'name': 'cellular_telephone'}, {'frequency': 'r', 'id': 236, 'synset': 'chain_mail.n.01', 'synonyms': ['chain_mail', 'ring_mail', 'chain_armor', 'chain_armour', 'ring_armor', 'ring_armour'], 'def': '(Middle Ages) flexible armor made of interlinked metal rings', 'name': 'chain_mail'}, {'frequency': 'f', 'id': 237, 'synset': 'chair.n.01', 'synonyms': ['chair'], 'def': 'a seat for one person, with a support for the back', 'name': 'chair'}, {'frequency': 'r', 'id': 238, 'synset': 'chaise_longue.n.01', 'synonyms': ['chaise_longue', 'chaise', 'daybed'], 'def': 'a long chair; for reclining', 'name': 'chaise_longue'}, {'frequency': 'r', 'id': 239, 'synset': 'champagne.n.01', 'synonyms': ['champagne'], 'def': 'a white sparkling wine produced in Champagne or resembling that produced there', 'name': 'champagne'}, {'frequency': 'f', 'id': 240, 'synset': 'chandelier.n.01', 'synonyms': ['chandelier'], 'def': 'branched lighting fixture; often ornate; hangs from the ceiling', 'name': 'chandelier'}, {'frequency': 'r', 'id': 241, 'synset': 'chap.n.04', 'synonyms': ['chap'], 'def': 'leather leggings without a seat; worn over trousers by cowboys to protect their legs', 'name': 'chap'}, {'frequency': 'r', 'id': 242, 'synset': 'checkbook.n.01', 'synonyms': ['checkbook', 'chequebook'], 'def': 'a book issued to holders of checking accounts', 'name': 'checkbook'}, {'frequency': 'r', 'id': 243, 'synset': 'checkerboard.n.01', 'synonyms': ['checkerboard'], 'def': 'a board having 64 squares of two alternating colors', 'name': 'checkerboard'}, {'frequency': 'c', 'id': 244, 'synset': 'cherry.n.03', 'synonyms': ['cherry'], 'def': 'a red fruit with a single hard stone', 'name': 'cherry'}, {'frequency': 'r', 'id': 245, 'synset': 'chessboard.n.01', 'synonyms': ['chessboard'], 'def': 'a checkerboard used to play chess', 'name': 'chessboard'}, {'frequency': 'r', 'id': 246, 'synset': 'chest_of_drawers.n.01', 'synonyms': ['chest_of_drawers_(furniture)', 'bureau_(furniture)', 'chest_(furniture)'], 'def': 'furniture with drawers for keeping clothes', 'name': 'chest_of_drawers_(furniture)'}, {'frequency': 'c', 'id': 247, 'synset': 'chicken.n.02', 'synonyms': ['chicken_(animal)'], 'def': 'a domestic fowl bred for flesh or eggs', 'name': 'chicken_(animal)'}, {'frequency': 'c', 'id': 248, 'synset': 'chicken_wire.n.01', 'synonyms': ['chicken_wire'], 'def': 'a galvanized wire network with a hexagonal mesh; used to build fences', 'name': 'chicken_wire'}, {'frequency': 'r', 'id': 249, 'synset': 'chickpea.n.01', 'synonyms': ['chickpea', 'garbanzo'], 'def': 'the seed of the chickpea plant; usually dried', 'name': 'chickpea'}, {'frequency': 'r', 'id': 250, 'synset': 'chihuahua.n.03', 'synonyms': ['Chihuahua'], 'def': 'an old breed of tiny short-haired dog with protruding eyes from Mexico', 'name': 'Chihuahua'}, {'frequency': 'r', 'id': 251, 'synset': 'chili.n.02', 'synonyms': ['chili_(vegetable)', 'chili_pepper_(vegetable)', 'chilli_(vegetable)', 'chilly_(vegetable)', 'chile_(vegetable)'], 'def': 'very hot and finely tapering pepper of special pungency', 'name': 'chili_(vegetable)'}, {'frequency': 'r', 'id': 252, 'synset': 'chime.n.01', 'synonyms': ['chime', 'gong'], 'def': 'an instrument consisting of a set of bells that are struck with a hammer', 'name': 'chime'}, {'frequency': 'r', 'id': 253, 'synset': 'chinaware.n.01', 'synonyms': ['chinaware'], 'def': 'dishware made of high quality porcelain', 'name': 'chinaware'}, {'frequency': 'c', 'id': 254, 'synset': 'chip.n.04', 'synonyms': ['crisp_(potato_chip)', 'potato_chip'], 'def': 'a thin crisp slice of potato fried in deep fat', 'name': 'crisp_(potato_chip)'}, {'frequency': 'r', 'id': 255, 'synset': 'chip.n.06', 'synonyms': ['poker_chip'], 'def': 'a small disk-shaped counter used to represent money when gambling', 'name': 'poker_chip'}, {'frequency': 'c', 'id': 256, 'synset': 'chocolate_bar.n.01', 'synonyms': ['chocolate_bar'], 'def': 'a bar of chocolate candy', 'name': 'chocolate_bar'}, {'frequency': 'c', 'id': 257, 'synset': 'chocolate_cake.n.01', 'synonyms': ['chocolate_cake'], 'def': 'cake containing chocolate', 'name': 'chocolate_cake'}, {'frequency': 'r', 'id': 258, 'synset': 'chocolate_milk.n.01', 'synonyms': ['chocolate_milk'], 'def': 'milk flavored with chocolate syrup', 'name': 'chocolate_milk'}, {'frequency': 'r', 'id': 259, 'synset': 'chocolate_mousse.n.01', 'synonyms': ['chocolate_mousse'], 'def': 'dessert mousse made with chocolate', 'name': 'chocolate_mousse'}, {'frequency': 'f', 'id': 260, 'synset': 'choker.n.03', 'synonyms': ['choker', 'collar', 'neckband'], 'def': 'necklace that fits tightly around the neck', 'name': 'choker'}, {'frequency': 'f', 'id': 261, 'synset': 'chopping_board.n.01', 'synonyms': ['chopping_board', 'cutting_board', 'chopping_block'], 'def': 'a wooden board where meats or vegetables can be cut', 'name': 'chopping_board'}, {'frequency': 'c', 'id': 262, 'synset': 'chopstick.n.01', 'synonyms': ['chopstick'], 'def': 'one of a pair of slender sticks used as oriental tableware to eat food with', 'name': 'chopstick'}, {'frequency': 'f', 'id': 263, 'synset': 'christmas_tree.n.05', 'synonyms': ['Christmas_tree'], 'def': 'an ornamented evergreen used as a Christmas decoration', 'name': 'Christmas_tree'}, {'frequency': 'c', 'id': 264, 'synset': 'chute.n.02', 'synonyms': ['slide'], 'def': 'sloping channel through which things can descend', 'name': 'slide'}, {'frequency': 'r', 'id': 265, 'synset': 'cider.n.01', 'synonyms': ['cider', 'cyder'], 'def': 'a beverage made from juice pressed from apples', 'name': 'cider'}, {'frequency': 'r', 'id': 266, 'synset': 'cigar_box.n.01', 'synonyms': ['cigar_box'], 'def': 'a box for holding cigars', 'name': 'cigar_box'}, {'frequency': 'c', 'id': 267, 'synset': 'cigarette.n.01', 'synonyms': ['cigarette'], 'def': 'finely ground tobacco wrapped in paper; for smoking', 'name': 'cigarette'}, {'frequency': 'c', 'id': 268, 'synset': 'cigarette_case.n.01', 'synonyms': ['cigarette_case', 'cigarette_pack'], 'def': 'a small flat case for holding cigarettes', 'name': 'cigarette_case'}, {'frequency': 'f', 'id': 269, 'synset': 'cistern.n.02', 'synonyms': ['cistern', 'water_tank'], 'def': 'a tank that holds the water used to flush a toilet', 'name': 'cistern'}, {'frequency': 'r', 'id': 270, 'synset': 'clarinet.n.01', 'synonyms': ['clarinet'], 'def': 'a single-reed instrument with a straight tube', 'name': 'clarinet'}, {'frequency': 'r', 'id': 271, 'synset': 'clasp.n.01', 'synonyms': ['clasp'], 'def': 'a fastener (as a buckle or hook) that is used to hold two things together', 'name': 'clasp'}, {'frequency': 'c', 'id': 272, 'synset': 'cleansing_agent.n.01', 'synonyms': ['cleansing_agent', 'cleanser', 'cleaner'], 'def': 'a preparation used in cleaning something', 'name': 'cleansing_agent'}, {'frequency': 'r', 'id': 273, 'synset': 'clementine.n.01', 'synonyms': ['clementine'], 'def': 'a variety of mandarin orange', 'name': 'clementine'}, {'frequency': 'c', 'id': 274, 'synset': 'clip.n.03', 'synonyms': ['clip'], 'def': 'any of various small fasteners used to hold loose articles together', 'name': 'clip'}, {'frequency': 'c', 'id': 275, 'synset': 'clipboard.n.01', 'synonyms': ['clipboard'], 'def': 'a small writing board with a clip at the top for holding papers', 'name': 'clipboard'}, {'frequency': 'f', 'id': 276, 'synset': 'clock.n.01', 'synonyms': ['clock', 'timepiece', 'timekeeper'], 'def': 'a timepiece that shows the time of day', 'name': 'clock'}, {'frequency': 'f', 'id': 277, 'synset': 'clock_tower.n.01', 'synonyms': ['clock_tower'], 'def': 'a tower with a large clock visible high up on an outside face', 'name': 'clock_tower'}, {'frequency': 'c', 'id': 278, 'synset': 'clothes_hamper.n.01', 'synonyms': ['clothes_hamper', 'laundry_basket', 'clothes_basket'], 'def': 'a hamper that holds dirty clothes to be washed or wet clothes to be dried', 'name': 'clothes_hamper'}, {'frequency': 'c', 'id': 279, 'synset': 'clothespin.n.01', 'synonyms': ['clothespin', 'clothes_peg'], 'def': 'wood or plastic fastener; for holding clothes on a clothesline', 'name': 'clothespin'}, {'frequency': 'r', 'id': 280, 'synset': 'clutch_bag.n.01', 'synonyms': ['clutch_bag'], 'def': "a woman's strapless purse that is carried in the hand", 'name': 'clutch_bag'}, {'frequency': 'f', 'id': 281, 'synset': 'coaster.n.03', 'synonyms': ['coaster'], 'def': 'a covering (plate or mat) that protects the surface of a table', 'name': 'coaster'}, {'frequency': 'f', 'id': 282, 'synset': 'coat.n.01', 'synonyms': ['coat'], 'def': 'an outer garment that has sleeves and covers the body from shoulder down', 'name': 'coat'}, {'frequency': 'c', 'id': 283, 'synset': 'coat_hanger.n.01', 'synonyms': ['coat_hanger', 'clothes_hanger', 'dress_hanger'], 'def': "a hanger that is shaped like a person's shoulders", 'name': 'coat_hanger'}, {'frequency': 'r', 'id': 284, 'synset': 'coatrack.n.01', 'synonyms': ['coatrack', 'hatrack'], 'def': 'a rack with hooks for temporarily holding coats and hats', 'name': 'coatrack'}, {'frequency': 'c', 'id': 285, 'synset': 'cock.n.04', 'synonyms': ['cock', 'rooster'], 'def': 'adult male chicken', 'name': 'cock'}, {'frequency': 'c', 'id': 286, 'synset': 'coconut.n.02', 'synonyms': ['coconut', 'cocoanut'], 'def': 'large hard-shelled brown oval nut with a fibrous husk', 'name': 'coconut'}, {'frequency': 'r', 'id': 287, 'synset': 'coffee_filter.n.01', 'synonyms': ['coffee_filter'], 'def': 'filter (usually of paper) that passes the coffee and retains the coffee grounds', 'name': 'coffee_filter'}, {'frequency': 'f', 'id': 288, 'synset': 'coffee_maker.n.01', 'synonyms': ['coffee_maker', 'coffee_machine'], 'def': 'a kitchen appliance for brewing coffee automatically', 'name': 'coffee_maker'}, {'frequency': 'f', 'id': 289, 'synset': 'coffee_table.n.01', 'synonyms': ['coffee_table', 'cocktail_table'], 'def': 'low table where magazines can be placed and coffee or cocktails are served', 'name': 'coffee_table'}, {'frequency': 'c', 'id': 290, 'synset': 'coffeepot.n.01', 'synonyms': ['coffeepot'], 'def': 'tall pot in which coffee is brewed', 'name': 'coffeepot'}, {'frequency': 'r', 'id': 291, 'synset': 'coil.n.05', 'synonyms': ['coil'], 'def': 'tubing that is wound in a spiral', 'name': 'coil'}, {'frequency': 'c', 'id': 292, 'synset': 'coin.n.01', 'synonyms': ['coin'], 'def': 'a flat metal piece (usually a disc) used as money', 'name': 'coin'}, {'frequency': 'r', 'id': 293, 'synset': 'colander.n.01', 'synonyms': ['colander', 'cullender'], 'def': 'bowl-shaped strainer; used to wash or drain foods', 'name': 'colander'}, {'frequency': 'c', 'id': 294, 'synset': 'coleslaw.n.01', 'synonyms': ['coleslaw', 'slaw'], 'def': 'basically shredded cabbage', 'name': 'coleslaw'}, {'frequency': 'r', 'id': 295, 'synset': 'coloring_material.n.01', 'synonyms': ['coloring_material', 'colouring_material'], 'def': 'any material used for its color', 'name': 'coloring_material'}, {'frequency': 'r', 'id': 296, 'synset': 'combination_lock.n.01', 'synonyms': ['combination_lock'], 'def': 'lock that can be opened only by turning dials in a special sequence', 'name': 'combination_lock'}, {'frequency': 'c', 'id': 297, 'synset': 'comforter.n.04', 'synonyms': ['pacifier', 'teething_ring'], 'def': 'device used for an infant to suck or bite on', 'name': 'pacifier'}, {'frequency': 'r', 'id': 298, 'synset': 'comic_book.n.01', 'synonyms': ['comic_book'], 'def': 'a magazine devoted to comic strips', 'name': 'comic_book'}, {'frequency': 'f', 'id': 299, 'synset': 'computer_keyboard.n.01', 'synonyms': ['computer_keyboard', 'keyboard_(computer)'], 'def': 'a keyboard that is a data input device for computers', 'name': 'computer_keyboard'}, {'frequency': 'r', 'id': 300, 'synset': 'concrete_mixer.n.01', 'synonyms': ['concrete_mixer', 'cement_mixer'], 'def': 'a machine with a large revolving drum in which cement/concrete is mixed', 'name': 'concrete_mixer'}, {'frequency': 'f', 'id': 301, 'synset': 'cone.n.01', 'synonyms': ['cone', 'traffic_cone'], 'def': 'a cone-shaped object used to direct traffic', 'name': 'cone'}, {'frequency': 'f', 'id': 302, 'synset': 'control.n.09', 'synonyms': ['control', 'controller'], 'def': 'a mechanism that controls the operation of a machine', 'name': 'control'}, {'frequency': 'r', 'id': 303, 'synset': 'convertible.n.01', 'synonyms': ['convertible_(automobile)'], 'def': 'a car that has top that can be folded or removed', 'name': 'convertible_(automobile)'}, {'frequency': 'r', 'id': 304, 'synset': 'convertible.n.03', 'synonyms': ['sofa_bed'], 'def': 'a sofa that can be converted into a bed', 'name': 'sofa_bed'}, {'frequency': 'c', 'id': 305, 'synset': 'cookie.n.01', 'synonyms': ['cookie', 'cooky', 'biscuit_(cookie)'], 'def': "any of various small flat sweet cakes (`biscuit' is the British term)", 'name': 'cookie'}, {'frequency': 'r', 'id': 306, 'synset': 'cookie_jar.n.01', 'synonyms': ['cookie_jar', 'cooky_jar'], 'def': 'a jar in which cookies are kept (and sometimes money is hidden)', 'name': 'cookie_jar'}, {'frequency': 'r', 'id': 307, 'synset': 'cooking_utensil.n.01', 'synonyms': ['cooking_utensil'], 'def': 'a kitchen utensil made of material that does not melt easily; used for cooking', 'name': 'cooking_utensil'}, {'frequency': 'f', 'id': 308, 'synset': 'cooler.n.01', 'synonyms': ['cooler_(for_food)', 'ice_chest'], 'def': 'an insulated box for storing food often with ice', 'name': 'cooler_(for_food)'}, {'frequency': 'c', 'id': 309, 'synset': 'cork.n.04', 'synonyms': ['cork_(bottle_plug)', 'bottle_cork'], 'def': 'the plug in the mouth of a bottle (especially a wine bottle)', 'name': 'cork_(bottle_plug)'}, {'frequency': 'r', 'id': 310, 'synset': 'corkboard.n.01', 'synonyms': ['corkboard'], 'def': 'a sheet consisting of cork granules', 'name': 'corkboard'}, {'frequency': 'r', 'id': 311, 'synset': 'corkscrew.n.01', 'synonyms': ['corkscrew', 'bottle_screw'], 'def': 'a bottle opener that pulls corks', 'name': 'corkscrew'}, {'frequency': 'c', 'id': 312, 'synset': 'corn.n.03', 'synonyms': ['edible_corn', 'corn', 'maize'], 'def': 'ears of corn that can be prepared and served for human food', 'name': 'edible_corn'}, {'frequency': 'r', 'id': 313, 'synset': 'cornbread.n.01', 'synonyms': ['cornbread'], 'def': 'bread made primarily of cornmeal', 'name': 'cornbread'}, {'frequency': 'c', 'id': 314, 'synset': 'cornet.n.01', 'synonyms': ['cornet', 'horn', 'trumpet'], 'def': 'a brass musical instrument with a narrow tube and a flared bell and many valves', 'name': 'cornet'}, {'frequency': 'c', 'id': 315, 'synset': 'cornice.n.01', 'synonyms': ['cornice', 'valance', 'valance_board', 'pelmet'], 'def': 'a decorative framework to conceal curtain fixtures at the top of a window casing', 'name': 'cornice'}, {'frequency': 'r', 'id': 316, 'synset': 'cornmeal.n.01', 'synonyms': ['cornmeal'], 'def': 'coarsely ground corn', 'name': 'cornmeal'}, {'frequency': 'r', 'id': 317, 'synset': 'corset.n.01', 'synonyms': ['corset', 'girdle'], 'def': "a woman's close-fitting foundation garment", 'name': 'corset'}, {'frequency': 'r', 'id': 318, 'synset': 'cos.n.02', 'synonyms': ['romaine_lettuce'], 'def': 'lettuce with long dark-green leaves in a loosely packed elongated head', 'name': 'romaine_lettuce'}, {'frequency': 'c', 'id': 319, 'synset': 'costume.n.04', 'synonyms': ['costume'], 'def': 'the attire characteristic of a country or a time or a social class', 'name': 'costume'}, {'frequency': 'r', 'id': 320, 'synset': 'cougar.n.01', 'synonyms': ['cougar', 'puma', 'catamount', 'mountain_lion', 'panther'], 'def': 'large American feline resembling a lion', 'name': 'cougar'}, {'frequency': 'r', 'id': 321, 'synset': 'coverall.n.01', 'synonyms': ['coverall'], 'def': 'a loose-fitting protective garment that is worn over other clothing', 'name': 'coverall'}, {'frequency': 'r', 'id': 322, 'synset': 'cowbell.n.01', 'synonyms': ['cowbell'], 'def': 'a bell hung around the neck of cow so that the cow can be easily located', 'name': 'cowbell'}, {'frequency': 'f', 'id': 323, 'synset': 'cowboy_hat.n.01', 'synonyms': ['cowboy_hat', 'ten-gallon_hat'], 'def': 'a hat with a wide brim and a soft crown; worn by American ranch hands', 'name': 'cowboy_hat'}, {'frequency': 'r', 'id': 324, 'synset': 'crab.n.01', 'synonyms': ['crab_(animal)'], 'def': 'decapod having eyes on short stalks and a broad flattened shell and pincers', 'name': 'crab_(animal)'}, {'frequency': 'c', 'id': 325, 'synset': 'cracker.n.01', 'synonyms': ['cracker'], 'def': 'a thin crisp wafer', 'name': 'cracker'}, {'frequency': 'r', 'id': 326, 'synset': 'crape.n.01', 'synonyms': ['crape', 'crepe', 'French_pancake'], 'def': 'small very thin pancake', 'name': 'crape'}, {'frequency': 'f', 'id': 327, 'synset': 'crate.n.01', 'synonyms': ['crate'], 'def': 'a rugged box (usually made of wood); used for shipping', 'name': 'crate'}, {'frequency': 'r', 'id': 328, 'synset': 'crayon.n.01', 'synonyms': ['crayon', 'wax_crayon'], 'def': 'writing or drawing implement made of a colored stick of composition wax', 'name': 'crayon'}, {'frequency': 'r', 'id': 329, 'synset': 'cream_pitcher.n.01', 'synonyms': ['cream_pitcher'], 'def': 'a small pitcher for serving cream', 'name': 'cream_pitcher'}, {'frequency': 'r', 'id': 330, 'synset': 'credit_card.n.01', 'synonyms': ['credit_card', 'charge_card', 'debit_card'], 'def': 'a card, usually plastic, used to pay for goods and services', 'name': 'credit_card'}, {'frequency': 'c', 'id': 331, 'synset': 'crescent_roll.n.01', 'synonyms': ['crescent_roll', 'croissant'], 'def': 'very rich flaky crescent-shaped roll', 'name': 'crescent_roll'}, {'frequency': 'c', 'id': 332, 'synset': 'crib.n.01', 'synonyms': ['crib', 'cot'], 'def': 'baby bed with high sides made of slats', 'name': 'crib'}, {'frequency': 'c', 'id': 333, 'synset': 'crock.n.03', 'synonyms': ['crock_pot', 'earthenware_jar'], 'def': 'an earthen jar (made of baked clay)', 'name': 'crock_pot'}, {'frequency': 'f', 'id': 334, 'synset': 'crossbar.n.01', 'synonyms': ['crossbar'], 'def': 'a horizontal bar that goes across something', 'name': 'crossbar'}, {'frequency': 'r', 'id': 335, 'synset': 'crouton.n.01', 'synonyms': ['crouton'], 'def': 'a small piece of toasted or fried bread; served in soup or salads', 'name': 'crouton'}, {'frequency': 'r', 'id': 336, 'synset': 'crow.n.01', 'synonyms': ['crow'], 'def': 'black birds having a raucous call', 'name': 'crow'}, {'frequency': 'c', 'id': 337, 'synset': 'crown.n.04', 'synonyms': ['crown'], 'def': 'an ornamental jeweled headdress signifying sovereignty', 'name': 'crown'}, {'frequency': 'c', 'id': 338, 'synset': 'crucifix.n.01', 'synonyms': ['crucifix'], 'def': 'representation of the cross on which Jesus died', 'name': 'crucifix'}, {'frequency': 'c', 'id': 339, 'synset': 'cruise_ship.n.01', 'synonyms': ['cruise_ship', 'cruise_liner'], 'def': 'a passenger ship used commercially for pleasure cruises', 'name': 'cruise_ship'}, {'frequency': 'c', 'id': 340, 'synset': 'cruiser.n.01', 'synonyms': ['police_cruiser', 'patrol_car', 'police_car', 'squad_car'], 'def': 'a car in which policemen cruise the streets', 'name': 'police_cruiser'}, {'frequency': 'c', 'id': 341, 'synset': 'crumb.n.03', 'synonyms': ['crumb'], 'def': 'small piece of e.g. bread or cake', 'name': 'crumb'}, {'frequency': 'r', 'id': 342, 'synset': 'crutch.n.01', 'synonyms': ['crutch'], 'def': 'a wooden or metal staff that fits under the armpit and reaches to the ground', 'name': 'crutch'}, {'frequency': 'c', 'id': 343, 'synset': 'cub.n.03', 'synonyms': ['cub_(animal)'], 'def': 'the young of certain carnivorous mammals such as the bear or wolf or lion', 'name': 'cub_(animal)'}, {'frequency': 'r', 'id': 344, 'synset': 'cube.n.05', 'synonyms': ['cube', 'square_block'], 'def': 'a block in the (approximate) shape of a cube', 'name': 'cube'}, {'frequency': 'f', 'id': 345, 'synset': 'cucumber.n.02', 'synonyms': ['cucumber', 'cuke'], 'def': 'cylindrical green fruit with thin green rind and white flesh eaten as a vegetable', 'name': 'cucumber'}, {'frequency': 'c', 'id': 346, 'synset': 'cufflink.n.01', 'synonyms': ['cufflink'], 'def': 'jewelry consisting of linked buttons used to fasten the cuffs of a shirt', 'name': 'cufflink'}, {'frequency': 'f', 'id': 347, 'synset': 'cup.n.01', 'synonyms': ['cup'], 'def': 'a small open container usually used for drinking; usually has a handle', 'name': 'cup'}, {'frequency': 'c', 'id': 348, 'synset': 'cup.n.08', 'synonyms': ['trophy_cup'], 'def': 'a metal vessel with handles that is awarded as a trophy to a competition winner', 'name': 'trophy_cup'}, {'frequency': 'c', 'id': 349, 'synset': 'cupcake.n.01', 'synonyms': ['cupcake'], 'def': 'small cake baked in a muffin tin', 'name': 'cupcake'}, {'frequency': 'r', 'id': 350, 'synset': 'curler.n.01', 'synonyms': ['hair_curler', 'hair_roller', 'hair_crimper'], 'def': 'a cylindrical tube around which the hair is wound to curl it', 'name': 'hair_curler'}, {'frequency': 'r', 'id': 351, 'synset': 'curling_iron.n.01', 'synonyms': ['curling_iron'], 'def': 'a cylindrical home appliance that heats hair that has been curled around it', 'name': 'curling_iron'}, {'frequency': 'f', 'id': 352, 'synset': 'curtain.n.01', 'synonyms': ['curtain', 'drapery'], 'def': 'hanging cloth used as a blind (especially for a window)', 'name': 'curtain'}, {'frequency': 'f', 'id': 353, 'synset': 'cushion.n.03', 'synonyms': ['cushion'], 'def': 'a soft bag filled with air or padding such as feathers or foam rubber', 'name': 'cushion'}, {'frequency': 'r', 'id': 354, 'synset': 'custard.n.01', 'synonyms': ['custard'], 'def': 'sweetened mixture of milk and eggs baked or boiled or frozen', 'name': 'custard'}, {'frequency': 'c', 'id': 355, 'synset': 'cutter.n.06', 'synonyms': ['cutting_tool'], 'def': 'a cutting implement; a tool for cutting', 'name': 'cutting_tool'}, {'frequency': 'r', 'id': 356, 'synset': 'cylinder.n.04', 'synonyms': ['cylinder'], 'def': 'a cylindrical container', 'name': 'cylinder'}, {'frequency': 'r', 'id': 357, 'synset': 'cymbal.n.01', 'synonyms': ['cymbal'], 'def': 'a percussion instrument consisting of a concave brass disk', 'name': 'cymbal'}, {'frequency': 'r', 'id': 358, 'synset': 'dachshund.n.01', 'synonyms': ['dachshund', 'dachsie', 'badger_dog'], 'def': 'small long-bodied short-legged breed of dog having a short sleek coat and long drooping ears', 'name': 'dachshund'}, {'frequency': 'r', 'id': 359, 'synset': 'dagger.n.01', 'synonyms': ['dagger'], 'def': 'a short knife with a pointed blade used for piercing or stabbing', 'name': 'dagger'}, {'frequency': 'r', 'id': 360, 'synset': 'dartboard.n.01', 'synonyms': ['dartboard'], 'def': 'a circular board of wood or cork used as the target in the game of darts', 'name': 'dartboard'}, {'frequency': 'r', 'id': 361, 'synset': 'date.n.08', 'synonyms': ['date_(fruit)'], 'def': 'sweet edible fruit of the date palm with a single long woody seed', 'name': 'date_(fruit)'}, {'frequency': 'f', 'id': 362, 'synset': 'deck_chair.n.01', 'synonyms': ['deck_chair', 'beach_chair'], 'def': 'a folding chair for use outdoors; a wooden frame supports a length of canvas', 'name': 'deck_chair'}, {'frequency': 'c', 'id': 363, 'synset': 'deer.n.01', 'synonyms': ['deer', 'cervid'], 'def': "distinguished from Bovidae by the male's having solid deciduous antlers", 'name': 'deer'}, {'frequency': 'c', 'id': 364, 'synset': 'dental_floss.n.01', 'synonyms': ['dental_floss', 'floss'], 'def': 'a soft thread for cleaning the spaces between the teeth', 'name': 'dental_floss'}, {'frequency': 'f', 'id': 365, 'synset': 'desk.n.01', 'synonyms': ['desk'], 'def': 'a piece of furniture with a writing surface and usually drawers or other compartments', 'name': 'desk'}, {'frequency': 'r', 'id': 366, 'synset': 'detergent.n.01', 'synonyms': ['detergent'], 'def': 'a surface-active chemical widely used in industry and laundering', 'name': 'detergent'}, {'frequency': 'c', 'id': 367, 'synset': 'diaper.n.01', 'synonyms': ['diaper'], 'def': 'garment consisting of a folded cloth drawn up between the legs and fastened at the waist', 'name': 'diaper'}, {'frequency': 'r', 'id': 368, 'synset': 'diary.n.01', 'synonyms': ['diary', 'journal'], 'def': 'a daily written record of (usually personal) experiences and observations', 'name': 'diary'}, {'frequency': 'r', 'id': 369, 'synset': 'die.n.01', 'synonyms': ['die', 'dice'], 'def': 'a small cube with 1 to 6 spots on the six faces; used in gambling', 'name': 'die'}, {'frequency': 'r', 'id': 370, 'synset': 'dinghy.n.01', 'synonyms': ['dinghy', 'dory', 'rowboat'], 'def': 'a small boat of shallow draft with seats and oars with which it is propelled', 'name': 'dinghy'}, {'frequency': 'f', 'id': 371, 'synset': 'dining_table.n.01', 'synonyms': ['dining_table'], 'def': 'a table at which meals are served', 'name': 'dining_table'}, {'frequency': 'r', 'id': 372, 'synset': 'dinner_jacket.n.01', 'synonyms': ['tux', 'tuxedo'], 'def': 'semiformal evening dress for men', 'name': 'tux'}, {'frequency': 'c', 'id': 373, 'synset': 'dish.n.01', 'synonyms': ['dish'], 'def': 'a piece of dishware normally used as a container for holding or serving food', 'name': 'dish'}, {'frequency': 'c', 'id': 374, 'synset': 'dish.n.05', 'synonyms': ['dish_antenna'], 'def': 'directional antenna consisting of a parabolic reflector', 'name': 'dish_antenna'}, {'frequency': 'c', 'id': 375, 'synset': 'dishrag.n.01', 'synonyms': ['dishrag', 'dishcloth'], 'def': 'a cloth for washing dishes', 'name': 'dishrag'}, {'frequency': 'c', 'id': 376, 'synset': 'dishtowel.n.01', 'synonyms': ['dishtowel', 'tea_towel'], 'def': 'a towel for drying dishes', 'name': 'dishtowel'}, {'frequency': 'f', 'id': 377, 'synset': 'dishwasher.n.01', 'synonyms': ['dishwasher', 'dishwashing_machine'], 'def': 'a machine for washing dishes', 'name': 'dishwasher'}, {'frequency': 'r', 'id': 378, 'synset': 'dishwasher_detergent.n.01', 'synonyms': ['dishwasher_detergent', 'dishwashing_detergent', 'dishwashing_liquid'], 'def': 'a low-sudsing detergent designed for use in dishwashers', 'name': 'dishwasher_detergent'}, {'frequency': 'r', 'id': 379, 'synset': 'diskette.n.01', 'synonyms': ['diskette', 'floppy', 'floppy_disk'], 'def': 'a small plastic magnetic disk enclosed in a stiff envelope used to store data', 'name': 'diskette'}, {'frequency': 'c', 'id': 380, 'synset': 'dispenser.n.01', 'synonyms': ['dispenser'], 'def': 'a container so designed that the contents can be used in prescribed amounts', 'name': 'dispenser'}, {'frequency': 'c', 'id': 381, 'synset': 'dixie_cup.n.01', 'synonyms': ['Dixie_cup', 'paper_cup'], 'def': 'a disposable cup made of paper; for holding drinks', 'name': 'Dixie_cup'}, {'frequency': 'f', 'id': 382, 'synset': 'dog.n.01', 'synonyms': ['dog'], 'def': 'a common domesticated dog', 'name': 'dog'}, {'frequency': 'f', 'id': 383, 'synset': 'dog_collar.n.01', 'synonyms': ['dog_collar'], 'def': 'a collar for a dog', 'name': 'dog_collar'}, {'frequency': 'c', 'id': 384, 'synset': 'doll.n.01', 'synonyms': ['doll'], 'def': 'a toy replica of a HUMAN (NOT AN ANIMAL)', 'name': 'doll'}, {'frequency': 'r', 'id': 385, 'synset': 'dollar.n.02', 'synonyms': ['dollar', 'dollar_bill', 'one_dollar_bill'], 'def': 'a piece of paper money worth one dollar', 'name': 'dollar'}, {'frequency': 'r', 'id': 386, 'synset': 'dolphin.n.02', 'synonyms': ['dolphin'], 'def': 'any of various small toothed whales with a beaklike snout; larger than porpoises', 'name': 'dolphin'}, {'frequency': 'c', 'id': 387, 'synset': 'domestic_ass.n.01', 'synonyms': ['domestic_ass', 'donkey'], 'def': 'domestic beast of burden descended from the African wild ass; patient but stubborn', 'name': 'domestic_ass'}, {'frequency': 'r', 'id': 388, 'synset': 'domino.n.03', 'synonyms': ['eye_mask'], 'def': 'a mask covering the upper part of the face but with holes for the eyes', 'name': 'eye_mask'}, {'frequency': 'r', 'id': 389, 'synset': 'doorbell.n.01', 'synonyms': ['doorbell', 'buzzer'], 'def': 'a button at an outer door that gives a ringing or buzzing signal when pushed', 'name': 'doorbell'}, {'frequency': 'f', 'id': 390, 'synset': 'doorknob.n.01', 'synonyms': ['doorknob', 'doorhandle'], 'def': "a knob used to open a door (often called `doorhandle' in Great Britain)", 'name': 'doorknob'}, {'frequency': 'c', 'id': 391, 'synset': 'doormat.n.02', 'synonyms': ['doormat', 'welcome_mat'], 'def': 'a mat placed outside an exterior door for wiping the shoes before entering', 'name': 'doormat'}, {'frequency': 'f', 'id': 392, 'synset': 'doughnut.n.02', 'synonyms': ['doughnut', 'donut'], 'def': 'a small ring-shaped friedcake', 'name': 'doughnut'}, {'frequency': 'r', 'id': 393, 'synset': 'dove.n.01', 'synonyms': ['dove'], 'def': 'any of numerous small pigeons', 'name': 'dove'}, {'frequency': 'r', 'id': 394, 'synset': 'dragonfly.n.01', 'synonyms': ['dragonfly'], 'def': 'slender-bodied non-stinging insect having iridescent wings that are outspread at rest', 'name': 'dragonfly'}, {'frequency': 'f', 'id': 395, 'synset': 'drawer.n.01', 'synonyms': ['drawer'], 'def': 'a boxlike container in a piece of furniture; made so as to slide in and out', 'name': 'drawer'}, {'frequency': 'c', 'id': 396, 'synset': 'drawers.n.01', 'synonyms': ['underdrawers', 'boxers', 'boxershorts'], 'def': 'underpants worn by men', 'name': 'underdrawers'}, {'frequency': 'f', 'id': 397, 'synset': 'dress.n.01', 'synonyms': ['dress', 'frock'], 'def': 'a one-piece garment for a woman; has skirt and bodice', 'name': 'dress'}, {'frequency': 'c', 'id': 398, 'synset': 'dress_hat.n.01', 'synonyms': ['dress_hat', 'high_hat', 'opera_hat', 'silk_hat', 'top_hat'], 'def': "a man's hat with a tall crown; usually covered with silk or with beaver fur", 'name': 'dress_hat'}, {'frequency': 'c', 'id': 399, 'synset': 'dress_suit.n.01', 'synonyms': ['dress_suit'], 'def': 'formalwear consisting of full evening dress for men', 'name': 'dress_suit'}, {'frequency': 'c', 'id': 400, 'synset': 'dresser.n.05', 'synonyms': ['dresser'], 'def': 'a cabinet with shelves', 'name': 'dresser'}, {'frequency': 'c', 'id': 401, 'synset': 'drill.n.01', 'synonyms': ['drill'], 'def': 'a tool with a sharp rotating point for making holes in hard materials', 'name': 'drill'}, {'frequency': 'r', 'id': 402, 'synset': 'drinking_fountain.n.01', 'synonyms': ['drinking_fountain'], 'def': 'a public fountain to provide a jet of drinking water', 'name': 'drinking_fountain'}, {'frequency': 'r', 'id': 403, 'synset': 'drone.n.04', 'synonyms': ['drone'], 'def': 'an aircraft without a pilot that is operated by remote control', 'name': 'drone'}, {'frequency': 'r', 'id': 404, 'synset': 'dropper.n.01', 'synonyms': ['dropper', 'eye_dropper'], 'def': 'pipet consisting of a small tube with a vacuum bulb at one end for drawing liquid in and releasing it a drop at a time', 'name': 'dropper'}, {'frequency': 'c', 'id': 405, 'synset': 'drum.n.01', 'synonyms': ['drum_(musical_instrument)'], 'def': 'a musical percussion instrument; usually consists of a hollow cylinder with a membrane stretched across each end', 'name': 'drum_(musical_instrument)'}, {'frequency': 'r', 'id': 406, 'synset': 'drumstick.n.02', 'synonyms': ['drumstick'], 'def': 'a stick used for playing a drum', 'name': 'drumstick'}, {'frequency': 'f', 'id': 407, 'synset': 'duck.n.01', 'synonyms': ['duck'], 'def': 'small web-footed broad-billed swimming bird', 'name': 'duck'}, {'frequency': 'r', 'id': 408, 'synset': 'duckling.n.02', 'synonyms': ['duckling'], 'def': 'young duck', 'name': 'duckling'}, {'frequency': 'c', 'id': 409, 'synset': 'duct_tape.n.01', 'synonyms': ['duct_tape'], 'def': 'a wide silvery adhesive tape', 'name': 'duct_tape'}, {'frequency': 'f', 'id': 410, 'synset': 'duffel_bag.n.01', 'synonyms': ['duffel_bag', 'duffle_bag', 'duffel', 'duffle'], 'def': 'a large cylindrical bag of heavy cloth', 'name': 'duffel_bag'}, {'frequency': 'r', 'id': 411, 'synset': 'dumbbell.n.01', 'synonyms': ['dumbbell'], 'def': 'an exercising weight with two ball-like ends connected by a short handle', 'name': 'dumbbell'}, {'frequency': 'c', 'id': 412, 'synset': 'dumpster.n.01', 'synonyms': ['dumpster'], 'def': 'a container designed to receive and transport and dump waste', 'name': 'dumpster'}, {'frequency': 'r', 'id': 413, 'synset': 'dustpan.n.02', 'synonyms': ['dustpan'], 'def': 'a short-handled receptacle into which dust can be swept', 'name': 'dustpan'}, {'frequency': 'r', 'id': 414, 'synset': 'dutch_oven.n.02', 'synonyms': ['Dutch_oven'], 'def': 'iron or earthenware cooking pot; used for stews', 'name': 'Dutch_oven'}, {'frequency': 'c', 'id': 415, 'synset': 'eagle.n.01', 'synonyms': ['eagle'], 'def': 'large birds of prey noted for their broad wings and strong soaring flight', 'name': 'eagle'}, {'frequency': 'f', 'id': 416, 'synset': 'earphone.n.01', 'synonyms': ['earphone', 'earpiece', 'headphone'], 'def': 'device for listening to audio that is held over or inserted into the ear', 'name': 'earphone'}, {'frequency': 'r', 'id': 417, 'synset': 'earplug.n.01', 'synonyms': ['earplug'], 'def': 'a soft plug that is inserted into the ear canal to block sound', 'name': 'earplug'}, {'frequency': 'f', 'id': 418, 'synset': 'earring.n.01', 'synonyms': ['earring'], 'def': 'jewelry to ornament the ear', 'name': 'earring'}, {'frequency': 'c', 'id': 419, 'synset': 'easel.n.01', 'synonyms': ['easel'], 'def': "an upright tripod for displaying something (usually an artist's canvas)", 'name': 'easel'}, {'frequency': 'r', 'id': 420, 'synset': 'eclair.n.01', 'synonyms': ['eclair'], 'def': 'oblong cream puff', 'name': 'eclair'}, {'frequency': 'r', 'id': 421, 'synset': 'eel.n.01', 'synonyms': ['eel'], 'def': 'an elongate fish with fatty flesh', 'name': 'eel'}, {'frequency': 'f', 'id': 422, 'synset': 'egg.n.02', 'synonyms': ['egg', 'eggs'], 'def': 'oval reproductive body of a fowl (especially a hen) used as food', 'name': 'egg'}, {'frequency': 'r', 'id': 423, 'synset': 'egg_roll.n.01', 'synonyms': ['egg_roll', 'spring_roll'], 'def': 'minced vegetables and meat wrapped in a pancake and fried', 'name': 'egg_roll'}, {'frequency': 'c', 'id': 424, 'synset': 'egg_yolk.n.01', 'synonyms': ['egg_yolk', 'yolk_(egg)'], 'def': 'the yellow spherical part of an egg', 'name': 'egg_yolk'}, {'frequency': 'c', 'id': 425, 'synset': 'eggbeater.n.02', 'synonyms': ['eggbeater', 'eggwhisk'], 'def': 'a mixer for beating eggs or whipping cream', 'name': 'eggbeater'}, {'frequency': 'c', 'id': 426, 'synset': 'eggplant.n.01', 'synonyms': ['eggplant', 'aubergine'], 'def': 'egg-shaped vegetable having a shiny skin typically dark purple', 'name': 'eggplant'}, {'frequency': 'r', 'id': 427, 'synset': 'electric_chair.n.01', 'synonyms': ['electric_chair'], 'def': 'a chair-shaped instrument of execution by electrocution', 'name': 'electric_chair'}, {'frequency': 'f', 'id': 428, 'synset': 'electric_refrigerator.n.01', 'synonyms': ['refrigerator'], 'def': 'a refrigerator in which the coolant is pumped around by an electric motor', 'name': 'refrigerator'}, {'frequency': 'f', 'id': 429, 'synset': 'elephant.n.01', 'synonyms': ['elephant'], 'def': 'a common elephant', 'name': 'elephant'}, {'frequency': 'r', 'id': 430, 'synset': 'elk.n.01', 'synonyms': ['elk', 'moose'], 'def': 'large northern deer with enormous flattened antlers in the male', 'name': 'elk'}, {'frequency': 'c', 'id': 431, 'synset': 'envelope.n.01', 'synonyms': ['envelope'], 'def': 'a flat (usually rectangular) container for a letter, thin package, etc.', 'name': 'envelope'}, {'frequency': 'c', 'id': 432, 'synset': 'eraser.n.01', 'synonyms': ['eraser'], 'def': 'an implement used to erase something', 'name': 'eraser'}, {'frequency': 'r', 'id': 433, 'synset': 'escargot.n.01', 'synonyms': ['escargot'], 'def': 'edible snail usually served in the shell with a sauce of melted butter and garlic', 'name': 'escargot'}, {'frequency': 'r', 'id': 434, 'synset': 'eyepatch.n.01', 'synonyms': ['eyepatch'], 'def': 'a protective cloth covering for an injured eye', 'name': 'eyepatch'}, {'frequency': 'r', 'id': 435, 'synset': 'falcon.n.01', 'synonyms': ['falcon'], 'def': 'birds of prey having long pointed powerful wings adapted for swift flight', 'name': 'falcon'}, {'frequency': 'f', 'id': 436, 'synset': 'fan.n.01', 'synonyms': ['fan'], 'def': 'a device for creating a current of air by movement of a surface or surfaces', 'name': 'fan'}, {'frequency': 'f', 'id': 437, 'synset': 'faucet.n.01', 'synonyms': ['faucet', 'spigot', 'tap'], 'def': 'a regulator for controlling the flow of a liquid from a reservoir', 'name': 'faucet'}, {'frequency': 'r', 'id': 438, 'synset': 'fedora.n.01', 'synonyms': ['fedora'], 'def': 'a hat made of felt with a creased crown', 'name': 'fedora'}, {'frequency': 'r', 'id': 439, 'synset': 'ferret.n.02', 'synonyms': ['ferret'], 'def': 'domesticated albino variety of the European polecat bred for hunting rats and rabbits', 'name': 'ferret'}, {'frequency': 'c', 'id': 440, 'synset': 'ferris_wheel.n.01', 'synonyms': ['Ferris_wheel'], 'def': 'a large wheel with suspended seats that remain upright as the wheel rotates', 'name': 'Ferris_wheel'}, {'frequency': 'r', 'id': 441, 'synset': 'ferry.n.01', 'synonyms': ['ferry', 'ferryboat'], 'def': 'a boat that transports people or vehicles across a body of water and operates on a regular schedule', 'name': 'ferry'}, {'frequency': 'r', 'id': 442, 'synset': 'fig.n.04', 'synonyms': ['fig_(fruit)'], 'def': 'fleshy sweet pear-shaped yellowish or purple fruit eaten fresh or preserved or dried', 'name': 'fig_(fruit)'}, {'frequency': 'c', 'id': 443, 'synset': 'fighter.n.02', 'synonyms': ['fighter_jet', 'fighter_aircraft', 'attack_aircraft'], 'def': 'a high-speed military or naval airplane designed to destroy enemy targets', 'name': 'fighter_jet'}, {'frequency': 'f', 'id': 444, 'synset': 'figurine.n.01', 'synonyms': ['figurine'], 'def': 'a small carved or molded figure', 'name': 'figurine'}, {'frequency': 'c', 'id': 445, 'synset': 'file.n.03', 'synonyms': ['file_cabinet', 'filing_cabinet'], 'def': 'office furniture consisting of a container for keeping papers in order', 'name': 'file_cabinet'}, {'frequency': 'r', 'id': 446, 'synset': 'file.n.04', 'synonyms': ['file_(tool)'], 'def': 'a steel hand tool with small sharp teeth on some or all of its surfaces; used for smoothing wood or metal', 'name': 'file_(tool)'}, {'frequency': 'f', 'id': 447, 'synset': 'fire_alarm.n.02', 'synonyms': ['fire_alarm', 'smoke_alarm'], 'def': 'an alarm that is tripped off by fire or smoke', 'name': 'fire_alarm'}, {'frequency': 'c', 'id': 448, 'synset': 'fire_engine.n.01', 'synonyms': ['fire_engine', 'fire_truck'], 'def': 'large trucks that carry firefighters and equipment to the site of a fire', 'name': 'fire_engine'}, {'frequency': 'c', 'id': 449, 'synset': 'fire_extinguisher.n.01', 'synonyms': ['fire_extinguisher', 'extinguisher'], 'def': 'a manually operated device for extinguishing small fires', 'name': 'fire_extinguisher'}, {'frequency': 'c', 'id': 450, 'synset': 'fire_hose.n.01', 'synonyms': ['fire_hose'], 'def': 'a large hose that carries water from a fire hydrant to the site of the fire', 'name': 'fire_hose'}, {'frequency': 'f', 'id': 451, 'synset': 'fireplace.n.01', 'synonyms': ['fireplace'], 'def': 'an open recess in a wall at the base of a chimney where a fire can be built', 'name': 'fireplace'}, {'frequency': 'f', 'id': 452, 'synset': 'fireplug.n.01', 'synonyms': ['fireplug', 'fire_hydrant', 'hydrant'], 'def': 'an upright hydrant for drawing water to use in fighting a fire', 'name': 'fireplug'}, {'frequency': 'c', 'id': 453, 'synset': 'fish.n.01', 'synonyms': ['fish'], 'def': 'any of various mostly cold-blooded aquatic vertebrates usually having scales and breathing through gills', 'name': 'fish'}, {'frequency': 'r', 'id': 454, 'synset': 'fish.n.02', 'synonyms': ['fish_(food)'], 'def': 'the flesh of fish used as food', 'name': 'fish_(food)'}, {'frequency': 'r', 'id': 455, 'synset': 'fishbowl.n.02', 'synonyms': ['fishbowl', 'goldfish_bowl'], 'def': 'a transparent bowl in which small fish are kept', 'name': 'fishbowl'}, {'frequency': 'r', 'id': 456, 'synset': 'fishing_boat.n.01', 'synonyms': ['fishing_boat', 'fishing_vessel'], 'def': 'a vessel for fishing', 'name': 'fishing_boat'}, {'frequency': 'c', 'id': 457, 'synset': 'fishing_rod.n.01', 'synonyms': ['fishing_rod', 'fishing_pole'], 'def': 'a rod that is used in fishing to extend the fishing line', 'name': 'fishing_rod'}, {'frequency': 'f', 'id': 458, 'synset': 'flag.n.01', 'synonyms': ['flag'], 'def': 'emblem usually consisting of a rectangular piece of cloth of distinctive design (do not include pole)', 'name': 'flag'}, {'frequency': 'f', 'id': 459, 'synset': 'flagpole.n.02', 'synonyms': ['flagpole', 'flagstaff'], 'def': 'a tall staff or pole on which a flag is raised', 'name': 'flagpole'}, {'frequency': 'c', 'id': 460, 'synset': 'flamingo.n.01', 'synonyms': ['flamingo'], 'def': 'large pink web-footed bird with down-bent bill', 'name': 'flamingo'}, {'frequency': 'c', 'id': 461, 'synset': 'flannel.n.01', 'synonyms': ['flannel'], 'def': 'a soft light woolen fabric; used for clothing', 'name': 'flannel'}, {'frequency': 'r', 'id': 462, 'synset': 'flash.n.10', 'synonyms': ['flash', 'flashbulb'], 'def': 'a lamp for providing momentary light to take a photograph', 'name': 'flash'}, {'frequency': 'c', 'id': 463, 'synset': 'flashlight.n.01', 'synonyms': ['flashlight', 'torch'], 'def': 'a small portable battery-powered electric lamp', 'name': 'flashlight'}, {'frequency': 'r', 'id': 464, 'synset': 'fleece.n.03', 'synonyms': ['fleece'], 'def': 'a soft bulky fabric with deep pile; used chiefly for clothing', 'name': 'fleece'}, {'frequency': 'f', 'id': 465, 'synset': 'flip-flop.n.02', 'synonyms': ['flip-flop_(sandal)'], 'def': 'a backless sandal held to the foot by a thong between two toes', 'name': 'flip-flop_(sandal)'}, {'frequency': 'c', 'id': 466, 'synset': 'flipper.n.01', 'synonyms': ['flipper_(footwear)', 'fin_(footwear)'], 'def': 'a shoe to aid a person in swimming', 'name': 'flipper_(footwear)'}, {'frequency': 'f', 'id': 467, 'synset': 'flower_arrangement.n.01', 'synonyms': ['flower_arrangement', 'floral_arrangement'], 'def': 'a decorative arrangement of flowers', 'name': 'flower_arrangement'}, {'frequency': 'c', 'id': 468, 'synset': 'flute.n.02', 'synonyms': ['flute_glass', 'champagne_flute'], 'def': 'a tall narrow wineglass', 'name': 'flute_glass'}, {'frequency': 'r', 'id': 469, 'synset': 'foal.n.01', 'synonyms': ['foal'], 'def': 'a young horse', 'name': 'foal'}, {'frequency': 'c', 'id': 470, 'synset': 'folding_chair.n.01', 'synonyms': ['folding_chair'], 'def': 'a chair that can be folded flat for storage', 'name': 'folding_chair'}, {'frequency': 'c', 'id': 471, 'synset': 'food_processor.n.01', 'synonyms': ['food_processor'], 'def': 'a kitchen appliance for shredding, blending, chopping, or slicing food', 'name': 'food_processor'}, {'frequency': 'c', 'id': 472, 'synset': 'football.n.02', 'synonyms': ['football_(American)'], 'def': 'the inflated oblong ball used in playing American football', 'name': 'football_(American)'}, {'frequency': 'r', 'id': 473, 'synset': 'football_helmet.n.01', 'synonyms': ['football_helmet'], 'def': 'a padded helmet with a face mask to protect the head of football players', 'name': 'football_helmet'}, {'frequency': 'c', 'id': 474, 'synset': 'footstool.n.01', 'synonyms': ['footstool', 'footrest'], 'def': 'a low seat or a stool to rest the feet of a seated person', 'name': 'footstool'}, {'frequency': 'f', 'id': 475, 'synset': 'fork.n.01', 'synonyms': ['fork'], 'def': 'cutlery used for serving and eating food', 'name': 'fork'}, {'frequency': 'r', 'id': 476, 'synset': 'forklift.n.01', 'synonyms': ['forklift'], 'def': 'an industrial vehicle with a power operated fork in front that can be inserted under loads to lift and move them', 'name': 'forklift'}, {'frequency': 'r', 'id': 477, 'synset': 'freight_car.n.01', 'synonyms': ['freight_car'], 'def': 'a railway car that carries freight', 'name': 'freight_car'}, {'frequency': 'r', 'id': 478, 'synset': 'french_toast.n.01', 'synonyms': ['French_toast'], 'def': 'bread slice dipped in egg and milk and fried', 'name': 'French_toast'}, {'frequency': 'c', 'id': 479, 'synset': 'freshener.n.01', 'synonyms': ['freshener', 'air_freshener'], 'def': 'anything that freshens', 'name': 'freshener'}, {'frequency': 'f', 'id': 480, 'synset': 'frisbee.n.01', 'synonyms': ['frisbee'], 'def': 'a light, plastic disk propelled with a flip of the wrist for recreation or competition', 'name': 'frisbee'}, {'frequency': 'c', 'id': 481, 'synset': 'frog.n.01', 'synonyms': ['frog', 'toad', 'toad_frog'], 'def': 'a tailless stout-bodied amphibians with long hind limbs for leaping', 'name': 'frog'}, {'frequency': 'c', 'id': 482, 'synset': 'fruit_juice.n.01', 'synonyms': ['fruit_juice'], 'def': 'drink produced by squeezing or crushing fruit', 'name': 'fruit_juice'}, {'frequency': 'r', 'id': 483, 'synset': 'fruit_salad.n.01', 'synonyms': ['fruit_salad'], 'def': 'salad composed of fruits', 'name': 'fruit_salad'}, {'frequency': 'c', 'id': 484, 'synset': 'frying_pan.n.01', 'synonyms': ['frying_pan', 'frypan', 'skillet'], 'def': 'a pan used for frying foods', 'name': 'frying_pan'}, {'frequency': 'r', 'id': 485, 'synset': 'fudge.n.01', 'synonyms': ['fudge'], 'def': 'soft creamy candy', 'name': 'fudge'}, {'frequency': 'r', 'id': 486, 'synset': 'funnel.n.02', 'synonyms': ['funnel'], 'def': 'a cone-shaped utensil used to channel a substance into a container with a small mouth', 'name': 'funnel'}, {'frequency': 'c', 'id': 487, 'synset': 'futon.n.01', 'synonyms': ['futon'], 'def': 'a pad that is used for sleeping on the floor or on a raised frame', 'name': 'futon'}, {'frequency': 'r', 'id': 488, 'synset': 'gag.n.02', 'synonyms': ['gag', 'muzzle'], 'def': "restraint put into a person's mouth to prevent speaking or shouting", 'name': 'gag'}, {'frequency': 'r', 'id': 489, 'synset': 'garbage.n.03', 'synonyms': ['garbage'], 'def': 'a receptacle where waste can be discarded', 'name': 'garbage'}, {'frequency': 'c', 'id': 490, 'synset': 'garbage_truck.n.01', 'synonyms': ['garbage_truck'], 'def': 'a truck for collecting domestic refuse', 'name': 'garbage_truck'}, {'frequency': 'c', 'id': 491, 'synset': 'garden_hose.n.01', 'synonyms': ['garden_hose'], 'def': 'a hose used for watering a lawn or garden', 'name': 'garden_hose'}, {'frequency': 'c', 'id': 492, 'synset': 'gargle.n.01', 'synonyms': ['gargle', 'mouthwash'], 'def': 'a medicated solution used for gargling and rinsing the mouth', 'name': 'gargle'}, {'frequency': 'r', 'id': 493, 'synset': 'gargoyle.n.02', 'synonyms': ['gargoyle'], 'def': 'an ornament consisting of a grotesquely carved figure of a person or animal', 'name': 'gargoyle'}, {'frequency': 'c', 'id': 494, 'synset': 'garlic.n.02', 'synonyms': ['garlic', 'ail'], 'def': 'aromatic bulb used as seasoning', 'name': 'garlic'}, {'frequency': 'r', 'id': 495, 'synset': 'gasmask.n.01', 'synonyms': ['gasmask', 'respirator', 'gas_helmet'], 'def': 'a protective face mask with a filter', 'name': 'gasmask'}, {'frequency': 'r', 'id': 496, 'synset': 'gazelle.n.01', 'synonyms': ['gazelle'], 'def': 'small swift graceful antelope of Africa and Asia having lustrous eyes', 'name': 'gazelle'}, {'frequency': 'c', 'id': 497, 'synset': 'gelatin.n.02', 'synonyms': ['gelatin', 'jelly'], 'def': 'an edible jelly made with gelatin and used as a dessert or salad base or a coating for foods', 'name': 'gelatin'}, {'frequency': 'r', 'id': 498, 'synset': 'gem.n.02', 'synonyms': ['gemstone'], 'def': 'a crystalline rock that can be cut and polished for jewelry', 'name': 'gemstone'}, {'frequency': 'c', 'id': 499, 'synset': 'giant_panda.n.01', 'synonyms': ['giant_panda', 'panda', 'panda_bear'], 'def': 'large black-and-white herbivorous mammal of bamboo forests of China and Tibet', 'name': 'giant_panda'}, {'frequency': 'c', 'id': 500, 'synset': 'gift_wrap.n.01', 'synonyms': ['gift_wrap'], 'def': 'attractive wrapping paper suitable for wrapping gifts', 'name': 'gift_wrap'}, {'frequency': 'c', 'id': 501, 'synset': 'ginger.n.03', 'synonyms': ['ginger', 'gingerroot'], 'def': 'the root of the common ginger plant; used fresh as a seasoning', 'name': 'ginger'}, {'frequency': 'f', 'id': 502, 'synset': 'giraffe.n.01', 'synonyms': ['giraffe'], 'def': 'tall animal having a spotted coat and small horns and very long neck and legs', 'name': 'giraffe'}, {'frequency': 'c', 'id': 503, 'synset': 'girdle.n.02', 'synonyms': ['cincture', 'sash', 'waistband', 'waistcloth'], 'def': 'a band of material around the waist that strengthens a skirt or trousers', 'name': 'cincture'}, {'frequency': 'f', 'id': 504, 'synset': 'glass.n.02', 'synonyms': ['glass_(drink_container)', 'drinking_glass'], 'def': 'a container for holding liquids while drinking', 'name': 'glass_(drink_container)'}, {'frequency': 'c', 'id': 505, 'synset': 'globe.n.03', 'synonyms': ['globe'], 'def': 'a sphere on which a map (especially of the earth) is represented', 'name': 'globe'}, {'frequency': 'f', 'id': 506, 'synset': 'glove.n.02', 'synonyms': ['glove'], 'def': 'handwear covering the hand', 'name': 'glove'}, {'frequency': 'c', 'id': 507, 'synset': 'goat.n.01', 'synonyms': ['goat'], 'def': 'a common goat', 'name': 'goat'}, {'frequency': 'f', 'id': 508, 'synset': 'goggles.n.01', 'synonyms': ['goggles'], 'def': 'tight-fitting spectacles worn to protect the eyes', 'name': 'goggles'}, {'frequency': 'r', 'id': 509, 'synset': 'goldfish.n.01', 'synonyms': ['goldfish'], 'def': 'small golden or orange-red freshwater fishes used as pond or aquarium pets', 'name': 'goldfish'}, {'frequency': 'r', 'id': 510, 'synset': 'golf_club.n.02', 'synonyms': ['golf_club', 'golf-club'], 'def': 'golf equipment used by a golfer to hit a golf ball', 'name': 'golf_club'}, {'frequency': 'c', 'id': 511, 'synset': 'golfcart.n.01', 'synonyms': ['golfcart'], 'def': 'a small motor vehicle in which golfers can ride between shots', 'name': 'golfcart'}, {'frequency': 'r', 'id': 512, 'synset': 'gondola.n.02', 'synonyms': ['gondola_(boat)'], 'def': 'long narrow flat-bottomed boat propelled by sculling; traditionally used on canals of Venice', 'name': 'gondola_(boat)'}, {'frequency': 'c', 'id': 513, 'synset': 'goose.n.01', 'synonyms': ['goose'], 'def': 'loud, web-footed long-necked aquatic birds usually larger than ducks', 'name': 'goose'}, {'frequency': 'r', 'id': 514, 'synset': 'gorilla.n.01', 'synonyms': ['gorilla'], 'def': 'largest ape', 'name': 'gorilla'}, {'frequency': 'r', 'id': 515, 'synset': 'gourd.n.02', 'synonyms': ['gourd'], 'def': 'any of numerous inedible fruits with hard rinds', 'name': 'gourd'}, {'frequency': 'r', 'id': 516, 'synset': 'gown.n.04', 'synonyms': ['surgical_gown', 'scrubs_(surgical_clothing)'], 'def': 'protective garment worn by surgeons during operations', 'name': 'surgical_gown'}, {'frequency': 'f', 'id': 517, 'synset': 'grape.n.01', 'synonyms': ['grape'], 'def': 'any of various juicy fruit with green or purple skins; grow in clusters', 'name': 'grape'}, {'frequency': 'r', 'id': 518, 'synset': 'grasshopper.n.01', 'synonyms': ['grasshopper'], 'def': 'plant-eating insect with hind legs adapted for leaping', 'name': 'grasshopper'}, {'frequency': 'c', 'id': 519, 'synset': 'grater.n.01', 'synonyms': ['grater'], 'def': 'utensil with sharp perforations for shredding foods (as vegetables or cheese)', 'name': 'grater'}, {'frequency': 'c', 'id': 520, 'synset': 'gravestone.n.01', 'synonyms': ['gravestone', 'headstone', 'tombstone'], 'def': 'a stone that is used to mark a grave', 'name': 'gravestone'}, {'frequency': 'r', 'id': 521, 'synset': 'gravy_boat.n.01', 'synonyms': ['gravy_boat', 'gravy_holder'], 'def': 'a dish (often boat-shaped) for serving gravy or sauce', 'name': 'gravy_boat'}, {'frequency': 'c', 'id': 522, 'synset': 'green_bean.n.02', 'synonyms': ['green_bean'], 'def': 'a common bean plant cultivated for its slender green edible pods', 'name': 'green_bean'}, {'frequency': 'c', 'id': 523, 'synset': 'green_onion.n.01', 'synonyms': ['green_onion', 'spring_onion', 'scallion'], 'def': 'a young onion before the bulb has enlarged', 'name': 'green_onion'}, {'frequency': 'r', 'id': 524, 'synset': 'griddle.n.01', 'synonyms': ['griddle'], 'def': 'cooking utensil consisting of a flat heated surface on which food is cooked', 'name': 'griddle'}, {'frequency': 'r', 'id': 525, 'synset': 'grillroom.n.01', 'synonyms': ['grillroom', 'grill_(restaurant)'], 'def': 'a restaurant where food is cooked on a grill', 'name': 'grillroom'}, {'frequency': 'r', 'id': 526, 'synset': 'grinder.n.04', 'synonyms': ['grinder_(tool)'], 'def': 'a machine tool that polishes metal', 'name': 'grinder_(tool)'}, {'frequency': 'r', 'id': 527, 'synset': 'grits.n.01', 'synonyms': ['grits', 'hominy_grits'], 'def': 'coarsely ground corn boiled as a breakfast dish', 'name': 'grits'}, {'frequency': 'c', 'id': 528, 'synset': 'grizzly.n.01', 'synonyms': ['grizzly', 'grizzly_bear'], 'def': 'powerful brownish-yellow bear of the uplands of western North America', 'name': 'grizzly'}, {'frequency': 'c', 'id': 529, 'synset': 'grocery_bag.n.01', 'synonyms': ['grocery_bag'], 'def': "a sack for holding customer's groceries", 'name': 'grocery_bag'}, {'frequency': 'r', 'id': 530, 'synset': 'guacamole.n.01', 'synonyms': ['guacamole'], 'def': 'a dip made of mashed avocado mixed with chopped onions and other seasonings', 'name': 'guacamole'}, {'frequency': 'f', 'id': 531, 'synset': 'guitar.n.01', 'synonyms': ['guitar'], 'def': 'a stringed instrument usually having six strings; played by strumming or plucking', 'name': 'guitar'}, {'frequency': 'c', 'id': 532, 'synset': 'gull.n.02', 'synonyms': ['gull', 'seagull'], 'def': 'mostly white aquatic bird having long pointed wings and short legs', 'name': 'gull'}, {'frequency': 'c', 'id': 533, 'synset': 'gun.n.01', 'synonyms': ['gun'], 'def': 'a weapon that discharges a bullet at high velocity from a metal tube', 'name': 'gun'}, {'frequency': 'r', 'id': 534, 'synset': 'hair_spray.n.01', 'synonyms': ['hair_spray'], 'def': 'substance sprayed on the hair to hold it in place', 'name': 'hair_spray'}, {'frequency': 'c', 'id': 535, 'synset': 'hairbrush.n.01', 'synonyms': ['hairbrush'], 'def': "a brush used to groom a person's hair", 'name': 'hairbrush'}, {'frequency': 'c', 'id': 536, 'synset': 'hairnet.n.01', 'synonyms': ['hairnet'], 'def': 'a small net that someone wears over their hair to keep it in place', 'name': 'hairnet'}, {'frequency': 'c', 'id': 537, 'synset': 'hairpin.n.01', 'synonyms': ['hairpin'], 'def': "a double pronged pin used to hold women's hair in place", 'name': 'hairpin'}, {'frequency': 'f', 'id': 538, 'synset': 'ham.n.01', 'synonyms': ['ham', 'jambon', 'gammon'], 'def': 'meat cut from the thigh of a hog (usually smoked)', 'name': 'ham'}, {'frequency': 'c', 'id': 539, 'synset': 'hamburger.n.01', 'synonyms': ['hamburger', 'beefburger', 'burger'], 'def': 'a sandwich consisting of a patty of minced beef served on a bun', 'name': 'hamburger'}, {'frequency': 'c', 'id': 540, 'synset': 'hammer.n.02', 'synonyms': ['hammer'], 'def': 'a hand tool with a heavy head and a handle; used to deliver an impulsive force by striking', 'name': 'hammer'}, {'frequency': 'r', 'id': 541, 'synset': 'hammock.n.02', 'synonyms': ['hammock'], 'def': 'a hanging bed of canvas or rope netting (usually suspended between two trees)', 'name': 'hammock'}, {'frequency': 'r', 'id': 542, 'synset': 'hamper.n.02', 'synonyms': ['hamper'], 'def': 'a basket usually with a cover', 'name': 'hamper'}, {'frequency': 'r', 'id': 543, 'synset': 'hamster.n.01', 'synonyms': ['hamster'], 'def': 'short-tailed burrowing rodent with large cheek pouches', 'name': 'hamster'}, {'frequency': 'c', 'id': 544, 'synset': 'hand_blower.n.01', 'synonyms': ['hair_dryer'], 'def': 'a hand-held electric blower that can blow warm air onto the hair', 'name': 'hair_dryer'}, {'frequency': 'r', 'id': 545, 'synset': 'hand_glass.n.01', 'synonyms': ['hand_glass', 'hand_mirror'], 'def': 'a mirror intended to be held in the hand', 'name': 'hand_glass'}, {'frequency': 'f', 'id': 546, 'synset': 'hand_towel.n.01', 'synonyms': ['hand_towel', 'face_towel'], 'def': 'a small towel used to dry the hands or face', 'name': 'hand_towel'}, {'frequency': 'c', 'id': 547, 'synset': 'handcart.n.01', 'synonyms': ['handcart', 'pushcart', 'hand_truck'], 'def': 'wheeled vehicle that can be pushed by a person', 'name': 'handcart'}, {'frequency': 'r', 'id': 548, 'synset': 'handcuff.n.01', 'synonyms': ['handcuff'], 'def': 'shackle that consists of a metal loop that can be locked around the wrist', 'name': 'handcuff'}, {'frequency': 'c', 'id': 549, 'synset': 'handkerchief.n.01', 'synonyms': ['handkerchief'], 'def': 'a square piece of cloth used for wiping the eyes or nose or as a costume accessory', 'name': 'handkerchief'}, {'frequency': 'f', 'id': 550, 'synset': 'handle.n.01', 'synonyms': ['handle', 'grip', 'handgrip'], 'def': 'the appendage to an object that is designed to be held in order to use or move it', 'name': 'handle'}, {'frequency': 'r', 'id': 551, 'synset': 'handsaw.n.01', 'synonyms': ['handsaw', "carpenter's_saw"], 'def': 'a saw used with one hand for cutting wood', 'name': 'handsaw'}, {'frequency': 'r', 'id': 552, 'synset': 'hardback.n.01', 'synonyms': ['hardback_book', 'hardcover_book'], 'def': 'a book with cardboard or cloth or leather covers', 'name': 'hardback_book'}, {'frequency': 'r', 'id': 553, 'synset': 'harmonium.n.01', 'synonyms': ['harmonium', 'organ_(musical_instrument)', 'reed_organ_(musical_instrument)'], 'def': 'a free-reed instrument in which air is forced through the reeds by bellows', 'name': 'harmonium'}, {'frequency': 'f', 'id': 554, 'synset': 'hat.n.01', 'synonyms': ['hat'], 'def': 'headwear that protects the head from bad weather, sun, or worn for fashion', 'name': 'hat'}, {'frequency': 'r', 'id': 555, 'synset': 'hatbox.n.01', 'synonyms': ['hatbox'], 'def': 'a round piece of luggage for carrying hats', 'name': 'hatbox'}, {'frequency': 'r', 'id': 556, 'synset': 'hatch.n.03', 'synonyms': ['hatch'], 'def': 'a movable barrier covering a hatchway', 'name': 'hatch'}, {'frequency': 'c', 'id': 557, 'synset': 'head_covering.n.01', 'synonyms': ['veil'], 'def': 'a garment that covers the head and face', 'name': 'veil'}, {'frequency': 'f', 'id': 558, 'synset': 'headband.n.01', 'synonyms': ['headband'], 'def': 'a band worn around or over the head', 'name': 'headband'}, {'frequency': 'f', 'id': 559, 'synset': 'headboard.n.01', 'synonyms': ['headboard'], 'def': 'a vertical board or panel forming the head of a bedstead', 'name': 'headboard'}, {'frequency': 'f', 'id': 560, 'synset': 'headlight.n.01', 'synonyms': ['headlight', 'headlamp'], 'def': 'a powerful light with reflector; attached to the front of an automobile or locomotive', 'name': 'headlight'}, {'frequency': 'c', 'id': 561, 'synset': 'headscarf.n.01', 'synonyms': ['headscarf'], 'def': 'a kerchief worn over the head and tied under the chin', 'name': 'headscarf'}, {'frequency': 'r', 'id': 562, 'synset': 'headset.n.01', 'synonyms': ['headset'], 'def': 'receiver consisting of a pair of headphones', 'name': 'headset'}, {'frequency': 'c', 'id': 563, 'synset': 'headstall.n.01', 'synonyms': ['headstall_(for_horses)', 'headpiece_(for_horses)'], 'def': "the band that is the part of a bridle that fits around a horse's head", 'name': 'headstall_(for_horses)'}, {'frequency': 'r', 'id': 564, 'synset': 'hearing_aid.n.02', 'synonyms': ['hearing_aid'], 'def': 'an acoustic device used to direct sound to the ear of a hearing-impaired person', 'name': 'hearing_aid'}, {'frequency': 'c', 'id': 565, 'synset': 'heart.n.02', 'synonyms': ['heart'], 'def': 'a muscular organ; its contractions move the blood through the body', 'name': 'heart'}, {'frequency': 'c', 'id': 566, 'synset': 'heater.n.01', 'synonyms': ['heater', 'warmer'], 'def': 'device that heats water or supplies warmth to a room', 'name': 'heater'}, {'frequency': 'c', 'id': 567, 'synset': 'helicopter.n.01', 'synonyms': ['helicopter'], 'def': 'an aircraft without wings that obtains its lift from the rotation of overhead blades', 'name': 'helicopter'}, {'frequency': 'f', 'id': 568, 'synset': 'helmet.n.02', 'synonyms': ['helmet'], 'def': 'a protective headgear made of hard material to resist blows', 'name': 'helmet'}, {'frequency': 'r', 'id': 569, 'synset': 'heron.n.02', 'synonyms': ['heron'], 'def': 'grey or white wading bird with long neck and long legs and (usually) long bill', 'name': 'heron'}, {'frequency': 'c', 'id': 570, 'synset': 'highchair.n.01', 'synonyms': ['highchair', 'feeding_chair'], 'def': 'a chair for feeding a very young child', 'name': 'highchair'}, {'frequency': 'f', 'id': 571, 'synset': 'hinge.n.01', 'synonyms': ['hinge'], 'def': 'a joint that holds two parts together so that one can swing relative to the other', 'name': 'hinge'}, {'frequency': 'r', 'id': 572, 'synset': 'hippopotamus.n.01', 'synonyms': ['hippopotamus'], 'def': 'massive thick-skinned animal living in or around rivers of tropical Africa', 'name': 'hippopotamus'}, {'frequency': 'r', 'id': 573, 'synset': 'hockey_stick.n.01', 'synonyms': ['hockey_stick'], 'def': 'sports implement consisting of a stick used by hockey players to move the puck', 'name': 'hockey_stick'}, {'frequency': 'c', 'id': 574, 'synset': 'hog.n.03', 'synonyms': ['hog', 'pig'], 'def': 'domestic swine', 'name': 'hog'}, {'frequency': 'f', 'id': 575, 'synset': 'home_plate.n.01', 'synonyms': ['home_plate_(baseball)', 'home_base_(baseball)'], 'def': '(baseball) a rubber slab where the batter stands; it must be touched by a base runner in order to score', 'name': 'home_plate_(baseball)'}, {'frequency': 'c', 'id': 576, 'synset': 'honey.n.01', 'synonyms': ['honey'], 'def': 'a sweet yellow liquid produced by bees', 'name': 'honey'}, {'frequency': 'f', 'id': 577, 'synset': 'hood.n.06', 'synonyms': ['fume_hood', 'exhaust_hood'], 'def': 'metal covering leading to a vent that exhausts smoke or fumes', 'name': 'fume_hood'}, {'frequency': 'f', 'id': 578, 'synset': 'hook.n.05', 'synonyms': ['hook'], 'def': 'a curved or bent implement for suspending or pulling something', 'name': 'hook'}, {'frequency': 'f', 'id': 579, 'synset': 'horse.n.01', 'synonyms': ['horse'], 'def': 'a common horse', 'name': 'horse'}, {'frequency': 'f', 'id': 580, 'synset': 'hose.n.03', 'synonyms': ['hose', 'hosepipe'], 'def': 'a flexible pipe for conveying a liquid or gas', 'name': 'hose'}, {'frequency': 'r', 'id': 581, 'synset': 'hot-air_balloon.n.01', 'synonyms': ['hot-air_balloon'], 'def': 'balloon for travel through the air in a basket suspended below a large bag of heated air', 'name': 'hot-air_balloon'}, {'frequency': 'r', 'id': 582, 'synset': 'hot_plate.n.01', 'synonyms': ['hotplate'], 'def': 'a portable electric appliance for heating or cooking or keeping food warm', 'name': 'hotplate'}, {'frequency': 'c', 'id': 583, 'synset': 'hot_sauce.n.01', 'synonyms': ['hot_sauce'], 'def': 'a pungent peppery sauce', 'name': 'hot_sauce'}, {'frequency': 'r', 'id': 584, 'synset': 'hourglass.n.01', 'synonyms': ['hourglass'], 'def': 'a sandglass timer that runs for sixty minutes', 'name': 'hourglass'}, {'frequency': 'r', 'id': 585, 'synset': 'houseboat.n.01', 'synonyms': ['houseboat'], 'def': 'a barge that is designed and equipped for use as a dwelling', 'name': 'houseboat'}, {'frequency': 'r', 'id': 586, 'synset': 'hummingbird.n.01', 'synonyms': ['hummingbird'], 'def': 'tiny American bird having brilliant iridescent plumage and long slender bills', 'name': 'hummingbird'}, {'frequency': 'r', 'id': 587, 'synset': 'hummus.n.01', 'synonyms': ['hummus', 'humus', 'hommos', 'hoummos', 'humous'], 'def': 'a thick spread made from mashed chickpeas', 'name': 'hummus'}, {'frequency': 'c', 'id': 588, 'synset': 'ice_bear.n.01', 'synonyms': ['polar_bear'], 'def': 'white bear of Arctic regions', 'name': 'polar_bear'}, {'frequency': 'c', 'id': 589, 'synset': 'ice_cream.n.01', 'synonyms': ['icecream'], 'def': 'frozen dessert containing cream and sugar and flavoring', 'name': 'icecream'}, {'frequency': 'r', 'id': 590, 'synset': 'ice_lolly.n.01', 'synonyms': ['popsicle'], 'def': 'ice cream or water ice on a small wooden stick', 'name': 'popsicle'}, {'frequency': 'c', 'id': 591, 'synset': 'ice_maker.n.01', 'synonyms': ['ice_maker'], 'def': 'an appliance included in some electric refrigerators for making ice cubes', 'name': 'ice_maker'}, {'frequency': 'r', 'id': 592, 'synset': 'ice_pack.n.01', 'synonyms': ['ice_pack', 'ice_bag'], 'def': 'a waterproof bag filled with ice: applied to the body (especially the head) to cool or reduce swelling', 'name': 'ice_pack'}, {'frequency': 'r', 'id': 593, 'synset': 'ice_skate.n.01', 'synonyms': ['ice_skate'], 'def': 'skate consisting of a boot with a steel blade fitted to the sole', 'name': 'ice_skate'}, {'frequency': 'r', 'id': 594, 'synset': 'ice_tea.n.01', 'synonyms': ['ice_tea', 'iced_tea'], 'def': 'strong tea served over ice', 'name': 'ice_tea'}, {'frequency': 'c', 'id': 595, 'synset': 'igniter.n.01', 'synonyms': ['igniter', 'ignitor', 'lighter'], 'def': 'a substance or device used to start a fire', 'name': 'igniter'}, {'frequency': 'r', 'id': 596, 'synset': 'incense.n.01', 'synonyms': ['incense'], 'def': 'a substance that produces a fragrant odor when burned', 'name': 'incense'}, {'frequency': 'r', 'id': 597, 'synset': 'inhaler.n.01', 'synonyms': ['inhaler', 'inhalator'], 'def': 'a dispenser that produces a chemical vapor to be inhaled through mouth or nose', 'name': 'inhaler'}, {'frequency': 'c', 'id': 598, 'synset': 'ipod.n.01', 'synonyms': ['iPod'], 'def': 'a pocket-sized device used to play music files', 'name': 'iPod'}, {'frequency': 'c', 'id': 599, 'synset': 'iron.n.04', 'synonyms': ['iron_(for_clothing)', 'smoothing_iron_(for_clothing)'], 'def': 'home appliance consisting of a flat metal base that is heated and used to smooth cloth', 'name': 'iron_(for_clothing)'}, {'frequency': 'r', 'id': 600, 'synset': 'ironing_board.n.01', 'synonyms': ['ironing_board'], 'def': 'narrow padded board on collapsible supports; used for ironing clothes', 'name': 'ironing_board'}, {'frequency': 'f', 'id': 601, 'synset': 'jacket.n.01', 'synonyms': ['jacket'], 'def': 'a waist-length coat', 'name': 'jacket'}, {'frequency': 'r', 'id': 602, 'synset': 'jam.n.01', 'synonyms': ['jam'], 'def': 'preserve of crushed fruit', 'name': 'jam'}, {'frequency': 'f', 'id': 603, 'synset': 'jean.n.01', 'synonyms': ['jean', 'blue_jean', 'denim'], 'def': '(usually plural) close-fitting trousers of heavy denim for manual work or casual wear', 'name': 'jean'}, {'frequency': 'c', 'id': 604, 'synset': 'jeep.n.01', 'synonyms': ['jeep', 'landrover'], 'def': 'a car suitable for traveling over rough terrain', 'name': 'jeep'}, {'frequency': 'r', 'id': 605, 'synset': 'jelly_bean.n.01', 'synonyms': ['jelly_bean', 'jelly_egg'], 'def': 'sugar-glazed jellied candy', 'name': 'jelly_bean'}, {'frequency': 'f', 'id': 606, 'synset': 'jersey.n.03', 'synonyms': ['jersey', 'T-shirt', 'tee_shirt'], 'def': 'a close-fitting pullover shirt', 'name': 'jersey'}, {'frequency': 'c', 'id': 607, 'synset': 'jet.n.01', 'synonyms': ['jet_plane', 'jet-propelled_plane'], 'def': 'an airplane powered by one or more jet engines', 'name': 'jet_plane'}, {'frequency': 'c', 'id': 608, 'synset': 'jewelry.n.01', 'synonyms': ['jewelry', 'jewellery'], 'def': 'an adornment (as a bracelet or ring or necklace) made of precious metals and set with gems (or imitation gems)', 'name': 'jewelry'}, {'frequency': 'r', 'id': 609, 'synset': 'joystick.n.02', 'synonyms': ['joystick'], 'def': 'a control device for computers consisting of a vertical handle that can move freely in two directions', 'name': 'joystick'}, {'frequency': 'r', 'id': 610, 'synset': 'jump_suit.n.01', 'synonyms': ['jumpsuit'], 'def': "one-piece garment fashioned after a parachutist's uniform", 'name': 'jumpsuit'}, {'frequency': 'c', 'id': 611, 'synset': 'kayak.n.01', 'synonyms': ['kayak'], 'def': 'a small canoe consisting of a light frame made watertight with animal skins', 'name': 'kayak'}, {'frequency': 'r', 'id': 612, 'synset': 'keg.n.02', 'synonyms': ['keg'], 'def': 'small cask or barrel', 'name': 'keg'}, {'frequency': 'r', 'id': 613, 'synset': 'kennel.n.01', 'synonyms': ['kennel', 'doghouse'], 'def': 'outbuilding that serves as a shelter for a dog', 'name': 'kennel'}, {'frequency': 'c', 'id': 614, 'synset': 'kettle.n.01', 'synonyms': ['kettle', 'boiler'], 'def': 'a metal pot for stewing or boiling; usually has a lid', 'name': 'kettle'}, {'frequency': 'f', 'id': 615, 'synset': 'key.n.01', 'synonyms': ['key'], 'def': 'metal instrument used to unlock a lock', 'name': 'key'}, {'frequency': 'r', 'id': 616, 'synset': 'keycard.n.01', 'synonyms': ['keycard'], 'def': 'a plastic card used to gain access typically to a door', 'name': 'keycard'}, {'frequency': 'r', 'id': 617, 'synset': 'kilt.n.01', 'synonyms': ['kilt'], 'def': 'a knee-length pleated tartan skirt worn by men as part of the traditional dress in the Highlands of northern Scotland', 'name': 'kilt'}, {'frequency': 'c', 'id': 618, 'synset': 'kimono.n.01', 'synonyms': ['kimono'], 'def': 'a loose robe; imitated from robes originally worn by Japanese', 'name': 'kimono'}, {'frequency': 'f', 'id': 619, 'synset': 'kitchen_sink.n.01', 'synonyms': ['kitchen_sink'], 'def': 'a sink in a kitchen', 'name': 'kitchen_sink'}, {'frequency': 'c', 'id': 620, 'synset': 'kitchen_table.n.01', 'synonyms': ['kitchen_table'], 'def': 'a table in the kitchen', 'name': 'kitchen_table'}, {'frequency': 'f', 'id': 621, 'synset': 'kite.n.03', 'synonyms': ['kite'], 'def': 'plaything consisting of a light frame covered with tissue paper; flown in wind at end of a string', 'name': 'kite'}, {'frequency': 'c', 'id': 622, 'synset': 'kitten.n.01', 'synonyms': ['kitten', 'kitty'], 'def': 'young domestic cat', 'name': 'kitten'}, {'frequency': 'c', 'id': 623, 'synset': 'kiwi.n.03', 'synonyms': ['kiwi_fruit'], 'def': 'fuzzy brown egg-shaped fruit with slightly tart green flesh', 'name': 'kiwi_fruit'}, {'frequency': 'f', 'id': 624, 'synset': 'knee_pad.n.01', 'synonyms': ['knee_pad'], 'def': 'protective garment consisting of a pad worn by football or baseball or hockey players', 'name': 'knee_pad'}, {'frequency': 'f', 'id': 625, 'synset': 'knife.n.01', 'synonyms': ['knife'], 'def': 'tool with a blade and point used as a cutting instrument', 'name': 'knife'}, {'frequency': 'r', 'id': 626, 'synset': 'knight.n.02', 'synonyms': ['knight_(chess_piece)', 'horse_(chess_piece)'], 'def': 'a chess game piece shaped to resemble the head of a horse', 'name': 'knight_(chess_piece)'}, {'frequency': 'r', 'id': 627, 'synset': 'knitting_needle.n.01', 'synonyms': ['knitting_needle'], 'def': 'needle consisting of a slender rod with pointed ends; usually used in pairs', 'name': 'knitting_needle'}, {'frequency': 'f', 'id': 628, 'synset': 'knob.n.02', 'synonyms': ['knob'], 'def': 'a round handle often found on a door', 'name': 'knob'}, {'frequency': 'r', 'id': 629, 'synset': 'knocker.n.05', 'synonyms': ['knocker_(on_a_door)', 'doorknocker'], 'def': 'a device (usually metal and ornamental) attached by a hinge to a door', 'name': 'knocker_(on_a_door)'}, {'frequency': 'r', 'id': 630, 'synset': 'koala.n.01', 'synonyms': ['koala', 'koala_bear'], 'def': 'sluggish tailless Australian marsupial with grey furry ears and coat', 'name': 'koala'}, {'frequency': 'r', 'id': 631, 'synset': 'lab_coat.n.01', 'synonyms': ['lab_coat', 'laboratory_coat'], 'def': 'a light coat worn to protect clothing from substances used while working in a laboratory', 'name': 'lab_coat'}, {'frequency': 'f', 'id': 632, 'synset': 'ladder.n.01', 'synonyms': ['ladder'], 'def': 'steps consisting of two parallel members connected by rungs', 'name': 'ladder'}, {'frequency': 'c', 'id': 633, 'synset': 'ladle.n.01', 'synonyms': ['ladle'], 'def': 'a spoon-shaped vessel with a long handle frequently used to transfer liquids', 'name': 'ladle'}, {'frequency': 'r', 'id': 634, 'synset': 'ladybug.n.01', 'synonyms': ['ladybug', 'ladybeetle', 'ladybird_beetle'], 'def': 'small round bright-colored and spotted beetle, typically red and black', 'name': 'ladybug'}, {'frequency': 'c', 'id': 635, 'synset': 'lamb.n.01', 'synonyms': ['lamb_(animal)'], 'def': 'young sheep', 'name': 'lamb_(animal)'}, {'frequency': 'r', 'id': 636, 'synset': 'lamb_chop.n.01', 'synonyms': ['lamb-chop', 'lambchop'], 'def': 'chop cut from a lamb', 'name': 'lamb-chop'}, {'frequency': 'f', 'id': 637, 'synset': 'lamp.n.02', 'synonyms': ['lamp'], 'def': 'a piece of furniture holding one or more electric light bulbs', 'name': 'lamp'}, {'frequency': 'f', 'id': 638, 'synset': 'lamppost.n.01', 'synonyms': ['lamppost'], 'def': 'a metal post supporting an outdoor lamp (such as a streetlight)', 'name': 'lamppost'}, {'frequency': 'f', 'id': 639, 'synset': 'lampshade.n.01', 'synonyms': ['lampshade'], 'def': 'a protective ornamental shade used to screen a light bulb from direct view', 'name': 'lampshade'}, {'frequency': 'c', 'id': 640, 'synset': 'lantern.n.01', 'synonyms': ['lantern'], 'def': 'light in a transparent protective case', 'name': 'lantern'}, {'frequency': 'f', 'id': 641, 'synset': 'lanyard.n.02', 'synonyms': ['lanyard', 'laniard'], 'def': 'a cord worn around the neck to hold a knife or whistle, etc.', 'name': 'lanyard'}, {'frequency': 'f', 'id': 642, 'synset': 'laptop.n.01', 'synonyms': ['laptop_computer', 'notebook_computer'], 'def': 'a portable computer small enough to use in your lap', 'name': 'laptop_computer'}, {'frequency': 'r', 'id': 643, 'synset': 'lasagna.n.01', 'synonyms': ['lasagna', 'lasagne'], 'def': 'baked dish of layers of lasagna pasta with sauce and cheese and meat or vegetables', 'name': 'lasagna'}, {'frequency': 'c', 'id': 644, 'synset': 'latch.n.02', 'synonyms': ['latch'], 'def': 'a bar that can be lowered or slid into a groove to fasten a door or gate', 'name': 'latch'}, {'frequency': 'r', 'id': 645, 'synset': 'lawn_mower.n.01', 'synonyms': ['lawn_mower'], 'def': 'garden tool for mowing grass on lawns', 'name': 'lawn_mower'}, {'frequency': 'r', 'id': 646, 'synset': 'leather.n.01', 'synonyms': ['leather'], 'def': 'an animal skin made smooth and flexible by removing the hair and then tanning', 'name': 'leather'}, {'frequency': 'c', 'id': 647, 'synset': 'legging.n.01', 'synonyms': ['legging_(clothing)', 'leging_(clothing)', 'leg_covering'], 'def': 'a garment covering the leg (usually extending from the knee to the ankle)', 'name': 'legging_(clothing)'}, {'frequency': 'c', 'id': 648, 'synset': 'lego.n.01', 'synonyms': ['Lego', 'Lego_set'], 'def': "a child's plastic construction set for making models from blocks", 'name': 'Lego'}, {'frequency': 'f', 'id': 649, 'synset': 'lemon.n.01', 'synonyms': ['lemon'], 'def': 'yellow oval fruit with juicy acidic flesh', 'name': 'lemon'}, {'frequency': 'r', 'id': 650, 'synset': 'lemonade.n.01', 'synonyms': ['lemonade'], 'def': 'sweetened beverage of diluted lemon juice', 'name': 'lemonade'}, {'frequency': 'f', 'id': 651, 'synset': 'lettuce.n.02', 'synonyms': ['lettuce'], 'def': 'leafy plant commonly eaten in salad or on sandwiches', 'name': 'lettuce'}, {'frequency': 'f', 'id': 652, 'synset': 'license_plate.n.01', 'synonyms': ['license_plate', 'numberplate'], 'def': "a plate mounted on the front and back of car and bearing the car's registration number", 'name': 'license_plate'}, {'frequency': 'f', 'id': 653, 'synset': 'life_buoy.n.01', 'synonyms': ['life_buoy', 'lifesaver', 'life_belt', 'life_ring'], 'def': 'a ring-shaped life preserver used to prevent drowning (NOT a life-jacket or vest)', 'name': 'life_buoy'}, {'frequency': 'f', 'id': 654, 'synset': 'life_jacket.n.01', 'synonyms': ['life_jacket', 'life_vest'], 'def': 'life preserver consisting of a sleeveless jacket of buoyant or inflatable design', 'name': 'life_jacket'}, {'frequency': 'f', 'id': 655, 'synset': 'light_bulb.n.01', 'synonyms': ['lightbulb'], 'def': 'glass bulb or tube shaped electric device that emits light (DO NOT MARK LAMPS AS A WHOLE)', 'name': 'lightbulb'}, {'frequency': 'r', 'id': 656, 'synset': 'lightning_rod.n.02', 'synonyms': ['lightning_rod', 'lightning_conductor'], 'def': 'a metallic conductor that is attached to a high point and leads to the ground', 'name': 'lightning_rod'}, {'frequency': 'c', 'id': 657, 'synset': 'lime.n.06', 'synonyms': ['lime'], 'def': 'the green acidic fruit of any of various lime trees', 'name': 'lime'}, {'frequency': 'r', 'id': 658, 'synset': 'limousine.n.01', 'synonyms': ['limousine'], 'def': 'long luxurious car; usually driven by a chauffeur', 'name': 'limousine'}, {'frequency': 'r', 'id': 659, 'synset': 'linen.n.02', 'synonyms': ['linen_paper'], 'def': 'a high-quality paper made of linen fibers or with a linen finish', 'name': 'linen_paper'}, {'frequency': 'c', 'id': 660, 'synset': 'lion.n.01', 'synonyms': ['lion'], 'def': 'large gregarious predatory cat of Africa and India', 'name': 'lion'}, {'frequency': 'c', 'id': 661, 'synset': 'lip_balm.n.01', 'synonyms': ['lip_balm'], 'def': 'a balm applied to the lips', 'name': 'lip_balm'}, {'frequency': 'c', 'id': 662, 'synset': 'lipstick.n.01', 'synonyms': ['lipstick', 'lip_rouge'], 'def': 'makeup that is used to color the lips', 'name': 'lipstick'}, {'frequency': 'r', 'id': 663, 'synset': 'liquor.n.01', 'synonyms': ['liquor', 'spirits', 'hard_liquor', 'liqueur', 'cordial'], 'def': 'an alcoholic beverage that is distilled rather than fermented', 'name': 'liquor'}, {'frequency': 'r', 'id': 664, 'synset': 'lizard.n.01', 'synonyms': ['lizard'], 'def': 'a reptile with usually two pairs of legs and a tapering tail', 'name': 'lizard'}, {'frequency': 'r', 'id': 665, 'synset': 'loafer.n.02', 'synonyms': ['Loafer_(type_of_shoe)'], 'def': 'a low leather step-in shoe', 'name': 'Loafer_(type_of_shoe)'}, {'frequency': 'f', 'id': 666, 'synset': 'log.n.01', 'synonyms': ['log'], 'def': 'a segment of the trunk of a tree when stripped of branches', 'name': 'log'}, {'frequency': 'c', 'id': 667, 'synset': 'lollipop.n.02', 'synonyms': ['lollipop'], 'def': 'hard candy on a stick', 'name': 'lollipop'}, {'frequency': 'c', 'id': 668, 'synset': 'lotion.n.01', 'synonyms': ['lotion'], 'def': 'any of various cosmetic preparations that are applied to the skin', 'name': 'lotion'}, {'frequency': 'f', 'id': 669, 'synset': 'loudspeaker.n.01', 'synonyms': ['speaker_(stero_equipment)'], 'def': 'electronic device that produces sound often as part of a stereo system', 'name': 'speaker_(stero_equipment)'}, {'frequency': 'c', 'id': 670, 'synset': 'love_seat.n.01', 'synonyms': ['loveseat'], 'def': 'small sofa that seats two people', 'name': 'loveseat'}, {'frequency': 'r', 'id': 671, 'synset': 'machine_gun.n.01', 'synonyms': ['machine_gun'], 'def': 'a rapidly firing automatic gun', 'name': 'machine_gun'}, {'frequency': 'f', 'id': 672, 'synset': 'magazine.n.02', 'synonyms': ['magazine'], 'def': 'a paperback periodic publication', 'name': 'magazine'}, {'frequency': 'f', 'id': 673, 'synset': 'magnet.n.01', 'synonyms': ['magnet'], 'def': 'a device that attracts iron and produces a magnetic field', 'name': 'magnet'}, {'frequency': 'r', 'id': 674, 'synset': 'mail_slot.n.01', 'synonyms': ['mail_slot'], 'def': 'a slot (usually in a door) through which mail can be delivered', 'name': 'mail_slot'}, {'frequency': 'c', 'id': 675, 'synset': 'mailbox.n.01', 'synonyms': ['mailbox_(at_home)', 'letter_box_(at_home)'], 'def': 'a private box for delivery of mail', 'name': 'mailbox_(at_home)'}, {'frequency': 'r', 'id': 676, 'synset': 'mallet.n.01', 'synonyms': ['mallet'], 'def': 'a sports implement with a long handle and a hammer-like head used to hit a ball', 'name': 'mallet'}, {'frequency': 'r', 'id': 677, 'synset': 'mammoth.n.01', 'synonyms': ['mammoth'], 'def': 'any of numerous extinct elephants widely distributed in the Pleistocene', 'name': 'mammoth'}, {'frequency': 'c', 'id': 678, 'synset': 'mandarin.n.05', 'synonyms': ['mandarin_orange'], 'def': 'a somewhat flat reddish-orange loose skinned citrus of China', 'name': 'mandarin_orange'}, {'frequency': 'c', 'id': 679, 'synset': 'manger.n.01', 'synonyms': ['manger', 'trough'], 'def': 'a container (usually in a barn or stable) from which cattle or horses feed', 'name': 'manger'}, {'frequency': 'f', 'id': 680, 'synset': 'manhole.n.01', 'synonyms': ['manhole'], 'def': 'a hole (usually with a flush cover) through which a person can gain access to an underground structure', 'name': 'manhole'}, {'frequency': 'c', 'id': 681, 'synset': 'map.n.01', 'synonyms': ['map'], 'def': "a diagrammatic representation of the earth's surface (or part of it)", 'name': 'map'}, {'frequency': 'c', 'id': 682, 'synset': 'marker.n.03', 'synonyms': ['marker'], 'def': 'a writing implement for making a mark', 'name': 'marker'}, {'frequency': 'r', 'id': 683, 'synset': 'martini.n.01', 'synonyms': ['martini'], 'def': 'a cocktail made of gin (or vodka) with dry vermouth', 'name': 'martini'}, {'frequency': 'r', 'id': 684, 'synset': 'mascot.n.01', 'synonyms': ['mascot'], 'def': 'a person or animal that is adopted by a team or other group as a symbolic figure', 'name': 'mascot'}, {'frequency': 'c', 'id': 685, 'synset': 'mashed_potato.n.01', 'synonyms': ['mashed_potato'], 'def': 'potato that has been peeled and boiled and then mashed', 'name': 'mashed_potato'}, {'frequency': 'r', 'id': 686, 'synset': 'masher.n.02', 'synonyms': ['masher'], 'def': 'a kitchen utensil used for mashing (e.g. potatoes)', 'name': 'masher'}, {'frequency': 'f', 'id': 687, 'synset': 'mask.n.04', 'synonyms': ['mask', 'facemask'], 'def': 'a protective covering worn over the face', 'name': 'mask'}, {'frequency': 'f', 'id': 688, 'synset': 'mast.n.01', 'synonyms': ['mast'], 'def': 'a vertical spar for supporting sails', 'name': 'mast'}, {'frequency': 'c', 'id': 689, 'synset': 'mat.n.03', 'synonyms': ['mat_(gym_equipment)', 'gym_mat'], 'def': 'sports equipment consisting of a piece of thick padding on the floor for gymnastics', 'name': 'mat_(gym_equipment)'}, {'frequency': 'r', 'id': 690, 'synset': 'matchbox.n.01', 'synonyms': ['matchbox'], 'def': 'a box for holding matches', 'name': 'matchbox'}, {'frequency': 'f', 'id': 691, 'synset': 'mattress.n.01', 'synonyms': ['mattress'], 'def': 'a thick pad filled with resilient material used as a bed or part of a bed', 'name': 'mattress'}, {'frequency': 'c', 'id': 692, 'synset': 'measuring_cup.n.01', 'synonyms': ['measuring_cup'], 'def': 'graduated cup used to measure liquid or granular ingredients', 'name': 'measuring_cup'}, {'frequency': 'c', 'id': 693, 'synset': 'measuring_stick.n.01', 'synonyms': ['measuring_stick', 'ruler_(measuring_stick)', 'measuring_rod'], 'def': 'measuring instrument having a sequence of marks at regular intervals', 'name': 'measuring_stick'}, {'frequency': 'c', 'id': 694, 'synset': 'meatball.n.01', 'synonyms': ['meatball'], 'def': 'ground meat formed into a ball and fried or simmered in broth', 'name': 'meatball'}, {'frequency': 'c', 'id': 695, 'synset': 'medicine.n.02', 'synonyms': ['medicine'], 'def': 'something that treats or prevents or alleviates the symptoms of disease', 'name': 'medicine'}, {'frequency': 'r', 'id': 696, 'synset': 'melon.n.01', 'synonyms': ['melon'], 'def': 'fruit of the gourd family having a hard rind and sweet juicy flesh', 'name': 'melon'}, {'frequency': 'f', 'id': 697, 'synset': 'microphone.n.01', 'synonyms': ['microphone'], 'def': 'device for converting sound waves into electrical energy', 'name': 'microphone'}, {'frequency': 'r', 'id': 698, 'synset': 'microscope.n.01', 'synonyms': ['microscope'], 'def': 'magnifier of the image of small objects', 'name': 'microscope'}, {'frequency': 'f', 'id': 699, 'synset': 'microwave.n.02', 'synonyms': ['microwave_oven'], 'def': 'kitchen appliance that cooks food by passing an electromagnetic wave through it', 'name': 'microwave_oven'}, {'frequency': 'r', 'id': 700, 'synset': 'milestone.n.01', 'synonyms': ['milestone', 'milepost'], 'def': 'stone post at side of a road to show distances', 'name': 'milestone'}, {'frequency': 'c', 'id': 701, 'synset': 'milk.n.01', 'synonyms': ['milk'], 'def': 'a white nutritious liquid secreted by mammals and used as food by human beings', 'name': 'milk'}, {'frequency': 'f', 'id': 702, 'synset': 'minivan.n.01', 'synonyms': ['minivan'], 'def': 'a small box-shaped passenger van', 'name': 'minivan'}, {'frequency': 'r', 'id': 703, 'synset': 'mint.n.05', 'synonyms': ['mint_candy'], 'def': 'a candy that is flavored with a mint oil', 'name': 'mint_candy'}, {'frequency': 'f', 'id': 704, 'synset': 'mirror.n.01', 'synonyms': ['mirror'], 'def': 'polished surface that forms images by reflecting light', 'name': 'mirror'}, {'frequency': 'c', 'id': 705, 'synset': 'mitten.n.01', 'synonyms': ['mitten'], 'def': 'glove that encases the thumb separately and the other four fingers together', 'name': 'mitten'}, {'frequency': 'c', 'id': 706, 'synset': 'mixer.n.04', 'synonyms': ['mixer_(kitchen_tool)', 'stand_mixer'], 'def': 'a kitchen utensil that is used for mixing foods', 'name': 'mixer_(kitchen_tool)'}, {'frequency': 'c', 'id': 707, 'synset': 'money.n.03', 'synonyms': ['money'], 'def': 'the official currency issued by a government or national bank', 'name': 'money'}, {'frequency': 'f', 'id': 708, 'synset': 'monitor.n.04', 'synonyms': ['monitor_(computer_equipment) computer_monitor'], 'def': 'a computer monitor', 'name': 'monitor_(computer_equipment) computer_monitor'}, {'frequency': 'c', 'id': 709, 'synset': 'monkey.n.01', 'synonyms': ['monkey'], 'def': 'any of various long-tailed primates', 'name': 'monkey'}, {'frequency': 'f', 'id': 710, 'synset': 'motor.n.01', 'synonyms': ['motor'], 'def': 'machine that converts other forms of energy into mechanical energy and so imparts motion', 'name': 'motor'}, {'frequency': 'f', 'id': 711, 'synset': 'motor_scooter.n.01', 'synonyms': ['motor_scooter', 'scooter'], 'def': 'a wheeled vehicle with small wheels and a low-powered engine', 'name': 'motor_scooter'}, {'frequency': 'r', 'id': 712, 'synset': 'motor_vehicle.n.01', 'synonyms': ['motor_vehicle', 'automotive_vehicle'], 'def': 'a self-propelled wheeled vehicle that does not run on rails', 'name': 'motor_vehicle'}, {'frequency': 'r', 'id': 713, 'synset': 'motorboat.n.01', 'synonyms': ['motorboat', 'powerboat'], 'def': 'a boat propelled by an internal-combustion engine', 'name': 'motorboat'}, {'frequency': 'f', 'id': 714, 'synset': 'motorcycle.n.01', 'synonyms': ['motorcycle'], 'def': 'a motor vehicle with two wheels and a strong frame', 'name': 'motorcycle'}, {'frequency': 'f', 'id': 715, 'synset': 'mound.n.01', 'synonyms': ['mound_(baseball)', "pitcher's_mound"], 'def': '(baseball) the slight elevation on which the pitcher stands', 'name': 'mound_(baseball)'}, {'frequency': 'r', 'id': 716, 'synset': 'mouse.n.01', 'synonyms': ['mouse_(animal_rodent)'], 'def': 'a small rodent with pointed snouts and small ears on elongated bodies with slender usually hairless tails', 'name': 'mouse_(animal_rodent)'}, {'frequency': 'f', 'id': 717, 'synset': 'mouse.n.04', 'synonyms': ['mouse_(computer_equipment)', 'computer_mouse'], 'def': 'a computer input device that controls an on-screen pointer', 'name': 'mouse_(computer_equipment)'}, {'frequency': 'f', 'id': 718, 'synset': 'mousepad.n.01', 'synonyms': ['mousepad'], 'def': 'a small portable pad that provides an operating surface for a computer mouse', 'name': 'mousepad'}, {'frequency': 'c', 'id': 719, 'synset': 'muffin.n.01', 'synonyms': ['muffin'], 'def': 'a sweet quick bread baked in a cup-shaped pan', 'name': 'muffin'}, {'frequency': 'f', 'id': 720, 'synset': 'mug.n.04', 'synonyms': ['mug'], 'def': 'with handle and usually cylindrical', 'name': 'mug'}, {'frequency': 'f', 'id': 721, 'synset': 'mushroom.n.02', 'synonyms': ['mushroom'], 'def': 'a common mushroom', 'name': 'mushroom'}, {'frequency': 'r', 'id': 722, 'synset': 'music_stool.n.01', 'synonyms': ['music_stool', 'piano_stool'], 'def': 'a stool for piano players; usually adjustable in height', 'name': 'music_stool'}, {'frequency': 'r', 'id': 723, 'synset': 'musical_instrument.n.01', 'synonyms': ['musical_instrument', 'instrument_(musical)'], 'def': 'any of various devices or contrivances that can be used to produce musical tones or sounds', 'name': 'musical_instrument'}, {'frequency': 'r', 'id': 724, 'synset': 'nailfile.n.01', 'synonyms': ['nailfile'], 'def': 'a small flat file for shaping the nails', 'name': 'nailfile'}, {'frequency': 'r', 'id': 725, 'synset': 'nameplate.n.01', 'synonyms': ['nameplate'], 'def': 'a plate bearing a name', 'name': 'nameplate'}, {'frequency': 'f', 'id': 726, 'synset': 'napkin.n.01', 'synonyms': ['napkin', 'table_napkin', 'serviette'], 'def': 'a small piece of table linen or paper that is used to wipe the mouth and to cover the lap in order to protect clothing', 'name': 'napkin'}, {'frequency': 'r', 'id': 727, 'synset': 'neckerchief.n.01', 'synonyms': ['neckerchief'], 'def': 'a kerchief worn around the neck', 'name': 'neckerchief'}, {'frequency': 'f', 'id': 728, 'synset': 'necklace.n.01', 'synonyms': ['necklace'], 'def': 'jewelry consisting of a cord or chain (often bearing gems) worn about the neck as an ornament', 'name': 'necklace'}, {'frequency': 'f', 'id': 729, 'synset': 'necktie.n.01', 'synonyms': ['necktie', 'tie_(necktie)'], 'def': 'neckwear consisting of a long narrow piece of material worn under a collar and tied in knot at the front', 'name': 'necktie'}, {'frequency': 'r', 'id': 730, 'synset': 'needle.n.03', 'synonyms': ['needle'], 'def': 'a sharp pointed implement (usually metal)', 'name': 'needle'}, {'frequency': 'c', 'id': 731, 'synset': 'nest.n.01', 'synonyms': ['nest'], 'def': 'a structure in which animals lay eggs or give birth to their young', 'name': 'nest'}, {'frequency': 'r', 'id': 732, 'synset': 'newsstand.n.01', 'synonyms': ['newsstand'], 'def': 'a stall where newspapers and other periodicals are sold', 'name': 'newsstand'}, {'frequency': 'c', 'id': 733, 'synset': 'nightwear.n.01', 'synonyms': ['nightshirt', 'nightwear', 'sleepwear', 'nightclothes'], 'def': 'garments designed to be worn in bed', 'name': 'nightshirt'}, {'frequency': 'r', 'id': 734, 'synset': 'nosebag.n.01', 'synonyms': ['nosebag_(for_animals)', 'feedbag'], 'def': 'a canvas bag that is used to feed an animal (such as a horse); covers the muzzle and fastens at the top of the head', 'name': 'nosebag_(for_animals)'}, {'frequency': 'r', 'id': 735, 'synset': 'noseband.n.01', 'synonyms': ['noseband_(for_animals)', 'nosepiece_(for_animals)'], 'def': "a strap that is the part of a bridle that goes over the animal's nose", 'name': 'noseband_(for_animals)'}, {'frequency': 'f', 'id': 736, 'synset': 'notebook.n.01', 'synonyms': ['notebook'], 'def': 'a book with blank pages for recording notes or memoranda', 'name': 'notebook'}, {'frequency': 'c', 'id': 737, 'synset': 'notepad.n.01', 'synonyms': ['notepad'], 'def': 'a pad of paper for keeping notes', 'name': 'notepad'}, {'frequency': 'c', 'id': 738, 'synset': 'nut.n.03', 'synonyms': ['nut'], 'def': 'a small metal block (usually square or hexagonal) with internal screw thread to be fitted onto a bolt', 'name': 'nut'}, {'frequency': 'r', 'id': 739, 'synset': 'nutcracker.n.01', 'synonyms': ['nutcracker'], 'def': 'a hand tool used to crack nuts open', 'name': 'nutcracker'}, {'frequency': 'c', 'id': 740, 'synset': 'oar.n.01', 'synonyms': ['oar'], 'def': 'an implement used to propel or steer a boat', 'name': 'oar'}, {'frequency': 'r', 'id': 741, 'synset': 'octopus.n.01', 'synonyms': ['octopus_(food)'], 'def': 'tentacles of octopus prepared as food', 'name': 'octopus_(food)'}, {'frequency': 'r', 'id': 742, 'synset': 'octopus.n.02', 'synonyms': ['octopus_(animal)'], 'def': 'bottom-living cephalopod having a soft oval body with eight long tentacles', 'name': 'octopus_(animal)'}, {'frequency': 'c', 'id': 743, 'synset': 'oil_lamp.n.01', 'synonyms': ['oil_lamp', 'kerosene_lamp', 'kerosine_lamp'], 'def': 'a lamp that burns oil (as kerosine) for light', 'name': 'oil_lamp'}, {'frequency': 'c', 'id': 744, 'synset': 'olive_oil.n.01', 'synonyms': ['olive_oil'], 'def': 'oil from olives', 'name': 'olive_oil'}, {'frequency': 'r', 'id': 745, 'synset': 'omelet.n.01', 'synonyms': ['omelet', 'omelette'], 'def': 'beaten eggs cooked until just set; may be folded around e.g. ham or cheese or jelly', 'name': 'omelet'}, {'frequency': 'f', 'id': 746, 'synset': 'onion.n.01', 'synonyms': ['onion'], 'def': 'the bulb of an onion plant', 'name': 'onion'}, {'frequency': 'f', 'id': 747, 'synset': 'orange.n.01', 'synonyms': ['orange_(fruit)'], 'def': 'orange (FRUIT of an orange tree)', 'name': 'orange_(fruit)'}, {'frequency': 'c', 'id': 748, 'synset': 'orange_juice.n.01', 'synonyms': ['orange_juice'], 'def': 'bottled or freshly squeezed juice of oranges', 'name': 'orange_juice'}, {'frequency': 'r', 'id': 749, 'synset': 'oregano.n.01', 'synonyms': ['oregano', 'marjoram'], 'def': 'aromatic Eurasian perennial herb used in cooking and baking', 'name': 'oregano'}, {'frequency': 'c', 'id': 750, 'synset': 'ostrich.n.02', 'synonyms': ['ostrich'], 'def': 'fast-running African flightless bird with two-toed feet; largest living bird', 'name': 'ostrich'}, {'frequency': 'c', 'id': 751, 'synset': 'ottoman.n.03', 'synonyms': ['ottoman', 'pouf', 'pouffe', 'hassock'], 'def': 'thick cushion used as a seat', 'name': 'ottoman'}, {'frequency': 'c', 'id': 752, 'synset': 'overall.n.01', 'synonyms': ['overalls_(clothing)'], 'def': 'work clothing consisting of denim trousers usually with a bib and shoulder straps', 'name': 'overalls_(clothing)'}, {'frequency': 'c', 'id': 753, 'synset': 'owl.n.01', 'synonyms': ['owl'], 'def': 'nocturnal bird of prey with hawk-like beak and claws and large head with front-facing eyes', 'name': 'owl'}, {'frequency': 'c', 'id': 754, 'synset': 'packet.n.03', 'synonyms': ['packet'], 'def': 'a small package or bundle', 'name': 'packet'}, {'frequency': 'r', 'id': 755, 'synset': 'pad.n.03', 'synonyms': ['inkpad', 'inking_pad', 'stamp_pad'], 'def': 'absorbent material saturated with ink used to transfer ink evenly to a rubber stamp', 'name': 'inkpad'}, {'frequency': 'c', 'id': 756, 'synset': 'pad.n.04', 'synonyms': ['pad'], 'def': 'a flat mass of soft material used for protection, stuffing, or comfort', 'name': 'pad'}, {'frequency': 'c', 'id': 757, 'synset': 'paddle.n.04', 'synonyms': ['paddle', 'boat_paddle'], 'def': 'a short light oar used without an oarlock to propel a canoe or small boat', 'name': 'paddle'}, {'frequency': 'c', 'id': 758, 'synset': 'padlock.n.01', 'synonyms': ['padlock'], 'def': 'a detachable, portable lock', 'name': 'padlock'}, {'frequency': 'r', 'id': 759, 'synset': 'paintbox.n.01', 'synonyms': ['paintbox'], 'def': "a box containing a collection of cubes or tubes of artists' paint", 'name': 'paintbox'}, {'frequency': 'c', 'id': 760, 'synset': 'paintbrush.n.01', 'synonyms': ['paintbrush'], 'def': 'a brush used as an applicator to apply paint', 'name': 'paintbrush'}, {'frequency': 'f', 'id': 761, 'synset': 'painting.n.01', 'synonyms': ['painting'], 'def': 'graphic art consisting of an artistic composition made by applying paints to a surface', 'name': 'painting'}, {'frequency': 'c', 'id': 762, 'synset': 'pajama.n.02', 'synonyms': ['pajamas', 'pyjamas'], 'def': 'loose-fitting nightclothes worn for sleeping or lounging', 'name': 'pajamas'}, {'frequency': 'c', 'id': 763, 'synset': 'palette.n.02', 'synonyms': ['palette', 'pallet'], 'def': 'board that provides a flat surface on which artists mix paints and the range of colors used', 'name': 'palette'}, {'frequency': 'f', 'id': 764, 'synset': 'pan.n.01', 'synonyms': ['pan_(for_cooking)', 'cooking_pan'], 'def': 'cooking utensil consisting of a wide metal vessel', 'name': 'pan_(for_cooking)'}, {'frequency': 'r', 'id': 765, 'synset': 'pan.n.03', 'synonyms': ['pan_(metal_container)'], 'def': 'shallow container made of metal', 'name': 'pan_(metal_container)'}, {'frequency': 'c', 'id': 766, 'synset': 'pancake.n.01', 'synonyms': ['pancake'], 'def': 'a flat cake of thin batter fried on both sides on a griddle', 'name': 'pancake'}, {'frequency': 'r', 'id': 767, 'synset': 'pantyhose.n.01', 'synonyms': ['pantyhose'], 'def': "a woman's tights consisting of underpants and stockings", 'name': 'pantyhose'}, {'frequency': 'r', 'id': 768, 'synset': 'papaya.n.02', 'synonyms': ['papaya'], 'def': 'large oval melon-like tropical fruit with yellowish flesh', 'name': 'papaya'}, {'frequency': 'r', 'id': 769, 'synset': 'paper_clip.n.01', 'synonyms': ['paperclip'], 'def': 'a wire or plastic clip for holding sheets of paper together', 'name': 'paperclip'}, {'frequency': 'f', 'id': 770, 'synset': 'paper_plate.n.01', 'synonyms': ['paper_plate'], 'def': 'a disposable plate made of cardboard', 'name': 'paper_plate'}, {'frequency': 'f', 'id': 771, 'synset': 'paper_towel.n.01', 'synonyms': ['paper_towel'], 'def': 'a disposable towel made of absorbent paper', 'name': 'paper_towel'}, {'frequency': 'r', 'id': 772, 'synset': 'paperback_book.n.01', 'synonyms': ['paperback_book', 'paper-back_book', 'softback_book', 'soft-cover_book'], 'def': 'a book with paper covers', 'name': 'paperback_book'}, {'frequency': 'r', 'id': 773, 'synset': 'paperweight.n.01', 'synonyms': ['paperweight'], 'def': 'a weight used to hold down a stack of papers', 'name': 'paperweight'}, {'frequency': 'c', 'id': 774, 'synset': 'parachute.n.01', 'synonyms': ['parachute'], 'def': 'rescue equipment consisting of a device that fills with air and retards your fall', 'name': 'parachute'}, {'frequency': 'r', 'id': 775, 'synset': 'parakeet.n.01', 'synonyms': ['parakeet', 'parrakeet', 'parroket', 'paraquet', 'paroquet', 'parroquet'], 'def': 'any of numerous small slender long-tailed parrots', 'name': 'parakeet'}, {'frequency': 'c', 'id': 776, 'synset': 'parasail.n.01', 'synonyms': ['parasail_(sports)'], 'def': 'parachute that will lift a person up into the air when it is towed by a motorboat or a car', 'name': 'parasail_(sports)'}, {'frequency': 'r', 'id': 777, 'synset': 'parchment.n.01', 'synonyms': ['parchment'], 'def': 'a superior paper resembling sheepskin', 'name': 'parchment'}, {'frequency': 'r', 'id': 778, 'synset': 'parka.n.01', 'synonyms': ['parka', 'anorak'], 'def': "a kind of heavy jacket (`windcheater' is a British term)", 'name': 'parka'}, {'frequency': 'f', 'id': 779, 'synset': 'parking_meter.n.01', 'synonyms': ['parking_meter'], 'def': 'a coin-operated timer located next to a parking space', 'name': 'parking_meter'}, {'frequency': 'c', 'id': 780, 'synset': 'parrot.n.01', 'synonyms': ['parrot'], 'def': 'usually brightly colored tropical birds with short hooked beaks and the ability to mimic sounds', 'name': 'parrot'}, {'frequency': 'c', 'id': 781, 'synset': 'passenger_car.n.01', 'synonyms': ['passenger_car_(part_of_a_train)', 'coach_(part_of_a_train)'], 'def': 'a railcar where passengers ride', 'name': 'passenger_car_(part_of_a_train)'}, {'frequency': 'r', 'id': 782, 'synset': 'passenger_ship.n.01', 'synonyms': ['passenger_ship'], 'def': 'a ship built to carry passengers', 'name': 'passenger_ship'}, {'frequency': 'r', 'id': 783, 'synset': 'passport.n.02', 'synonyms': ['passport'], 'def': 'a document issued by a country to a citizen allowing that person to travel abroad and re-enter the home country', 'name': 'passport'}, {'frequency': 'f', 'id': 784, 'synset': 'pastry.n.02', 'synonyms': ['pastry'], 'def': 'any of various baked foods made of dough or batter', 'name': 'pastry'}, {'frequency': 'r', 'id': 785, 'synset': 'patty.n.01', 'synonyms': ['patty_(food)'], 'def': 'small flat mass of chopped food', 'name': 'patty_(food)'}, {'frequency': 'c', 'id': 786, 'synset': 'pea.n.01', 'synonyms': ['pea_(food)'], 'def': 'seed of a pea plant used for food', 'name': 'pea_(food)'}, {'frequency': 'c', 'id': 787, 'synset': 'peach.n.03', 'synonyms': ['peach'], 'def': 'downy juicy fruit with sweet yellowish or whitish flesh', 'name': 'peach'}, {'frequency': 'c', 'id': 788, 'synset': 'peanut_butter.n.01', 'synonyms': ['peanut_butter'], 'def': 'a spread made from ground peanuts', 'name': 'peanut_butter'}, {'frequency': 'c', 'id': 789, 'synset': 'pear.n.01', 'synonyms': ['pear'], 'def': 'sweet juicy gritty-textured fruit available in many varieties', 'name': 'pear'}, {'frequency': 'r', 'id': 790, 'synset': 'peeler.n.03', 'synonyms': ['peeler_(tool_for_fruit_and_vegetables)'], 'def': 'a device for peeling vegetables or fruits', 'name': 'peeler_(tool_for_fruit_and_vegetables)'}, {'frequency': 'r', 'id': 791, 'synset': 'pegboard.n.01', 'synonyms': ['pegboard'], 'def': 'a board perforated with regularly spaced holes into which pegs can be fitted', 'name': 'pegboard'}, {'frequency': 'c', 'id': 792, 'synset': 'pelican.n.01', 'synonyms': ['pelican'], 'def': 'large long-winged warm-water seabird having a large bill with a distensible pouch for fish', 'name': 'pelican'}, {'frequency': 'f', 'id': 793, 'synset': 'pen.n.01', 'synonyms': ['pen'], 'def': 'a writing implement with a point from which ink flows', 'name': 'pen'}, {'frequency': 'c', 'id': 794, 'synset': 'pencil.n.01', 'synonyms': ['pencil'], 'def': 'a thin cylindrical pointed writing implement made of wood and graphite', 'name': 'pencil'}, {'frequency': 'r', 'id': 795, 'synset': 'pencil_box.n.01', 'synonyms': ['pencil_box', 'pencil_case'], 'def': 'a box for holding pencils', 'name': 'pencil_box'}, {'frequency': 'r', 'id': 796, 'synset': 'pencil_sharpener.n.01', 'synonyms': ['pencil_sharpener'], 'def': 'a rotary implement for sharpening the point on pencils', 'name': 'pencil_sharpener'}, {'frequency': 'r', 'id': 797, 'synset': 'pendulum.n.01', 'synonyms': ['pendulum'], 'def': 'an apparatus consisting of an object mounted so that it swings freely under the influence of gravity', 'name': 'pendulum'}, {'frequency': 'c', 'id': 798, 'synset': 'penguin.n.01', 'synonyms': ['penguin'], 'def': 'short-legged flightless birds of cold southern regions having webbed feet and wings modified as flippers', 'name': 'penguin'}, {'frequency': 'r', 'id': 799, 'synset': 'pennant.n.02', 'synonyms': ['pennant'], 'def': 'a flag longer than it is wide (and often tapering)', 'name': 'pennant'}, {'frequency': 'r', 'id': 800, 'synset': 'penny.n.02', 'synonyms': ['penny_(coin)'], 'def': 'a coin worth one-hundredth of the value of the basic unit', 'name': 'penny_(coin)'}, {'frequency': 'c', 'id': 801, 'synset': 'pepper.n.03', 'synonyms': ['pepper', 'peppercorn'], 'def': 'pungent seasoning from the berry of the common pepper plant; whole or ground', 'name': 'pepper'}, {'frequency': 'c', 'id': 802, 'synset': 'pepper_mill.n.01', 'synonyms': ['pepper_mill', 'pepper_grinder'], 'def': 'a mill for grinding pepper', 'name': 'pepper_mill'}, {'frequency': 'c', 'id': 803, 'synset': 'perfume.n.02', 'synonyms': ['perfume'], 'def': 'a toiletry that emits and diffuses a fragrant odor', 'name': 'perfume'}, {'frequency': 'r', 'id': 804, 'synset': 'persimmon.n.02', 'synonyms': ['persimmon'], 'def': 'orange fruit resembling a plum; edible when fully ripe', 'name': 'persimmon'}, {'frequency': 'f', 'id': 805, 'synset': 'person.n.01', 'synonyms': ['baby', 'child', 'boy', 'girl', 'man', 'woman', 'person', 'human'], 'def': 'a human being', 'name': 'baby'}, {'frequency': 'r', 'id': 806, 'synset': 'pet.n.01', 'synonyms': ['pet'], 'def': 'a domesticated animal kept for companionship or amusement', 'name': 'pet'}, {'frequency': 'r', 'id': 807, 'synset': 'petfood.n.01', 'synonyms': ['petfood', 'pet-food'], 'def': 'food prepared for animal pets', 'name': 'petfood'}, {'frequency': 'r', 'id': 808, 'synset': 'pew.n.01', 'synonyms': ['pew_(church_bench)', 'church_bench'], 'def': 'long bench with backs; used in church by the congregation', 'name': 'pew_(church_bench)'}, {'frequency': 'r', 'id': 809, 'synset': 'phonebook.n.01', 'synonyms': ['phonebook', 'telephone_book', 'telephone_directory'], 'def': 'a directory containing an alphabetical list of telephone subscribers and their telephone numbers', 'name': 'phonebook'}, {'frequency': 'c', 'id': 810, 'synset': 'phonograph_record.n.01', 'synonyms': ['phonograph_record', 'phonograph_recording', 'record_(phonograph_recording)'], 'def': 'sound recording consisting of a typically black disk with a continuous groove', 'name': 'phonograph_record'}, {'frequency': 'c', 'id': 811, 'synset': 'piano.n.01', 'synonyms': ['piano'], 'def': 'a keyboard instrument that is played by depressing keys that cause hammers to strike tuned strings and produce sounds', 'name': 'piano'}, {'frequency': 'f', 'id': 812, 'synset': 'pickle.n.01', 'synonyms': ['pickle'], 'def': 'vegetables (especially cucumbers) preserved in brine or vinegar', 'name': 'pickle'}, {'frequency': 'f', 'id': 813, 'synset': 'pickup.n.01', 'synonyms': ['pickup_truck'], 'def': 'a light truck with an open body and low sides and a tailboard', 'name': 'pickup_truck'}, {'frequency': 'c', 'id': 814, 'synset': 'pie.n.01', 'synonyms': ['pie'], 'def': 'dish baked in pastry-lined pan often with a pastry top', 'name': 'pie'}, {'frequency': 'c', 'id': 815, 'synset': 'pigeon.n.01', 'synonyms': ['pigeon'], 'def': 'wild and domesticated birds having a heavy body and short legs', 'name': 'pigeon'}, {'frequency': 'r', 'id': 816, 'synset': 'piggy_bank.n.01', 'synonyms': ['piggy_bank', 'penny_bank'], 'def': "a child's coin bank (often shaped like a pig)", 'name': 'piggy_bank'}, {'frequency': 'f', 'id': 817, 'synset': 'pillow.n.01', 'synonyms': ['pillow'], 'def': 'a cushion to support the head of a sleeping person', 'name': 'pillow'}, {'frequency': 'r', 'id': 818, 'synset': 'pin.n.09', 'synonyms': ['pin_(non_jewelry)'], 'def': 'a small slender (often pointed) piece of wood or metal used to support or fasten or attach things', 'name': 'pin_(non_jewelry)'}, {'frequency': 'f', 'id': 819, 'synset': 'pineapple.n.02', 'synonyms': ['pineapple'], 'def': 'large sweet fleshy tropical fruit with a tuft of stiff leaves', 'name': 'pineapple'}, {'frequency': 'c', 'id': 820, 'synset': 'pinecone.n.01', 'synonyms': ['pinecone'], 'def': 'the seed-producing cone of a pine tree', 'name': 'pinecone'}, {'frequency': 'r', 'id': 821, 'synset': 'ping-pong_ball.n.01', 'synonyms': ['ping-pong_ball'], 'def': 'light hollow ball used in playing table tennis', 'name': 'ping-pong_ball'}, {'frequency': 'r', 'id': 822, 'synset': 'pinwheel.n.03', 'synonyms': ['pinwheel'], 'def': 'a toy consisting of vanes of colored paper or plastic that is pinned to a stick and spins when it is pointed into the wind', 'name': 'pinwheel'}, {'frequency': 'r', 'id': 823, 'synset': 'pipe.n.01', 'synonyms': ['tobacco_pipe'], 'def': 'a tube with a small bowl at one end; used for smoking tobacco', 'name': 'tobacco_pipe'}, {'frequency': 'f', 'id': 824, 'synset': 'pipe.n.02', 'synonyms': ['pipe', 'piping'], 'def': 'a long tube made of metal or plastic that is used to carry water or oil or gas etc.', 'name': 'pipe'}, {'frequency': 'r', 'id': 825, 'synset': 'pistol.n.01', 'synonyms': ['pistol', 'handgun'], 'def': 'a firearm that is held and fired with one hand', 'name': 'pistol'}, {'frequency': 'r', 'id': 826, 'synset': 'pita.n.01', 'synonyms': ['pita_(bread)', 'pocket_bread'], 'def': 'usually small round bread that can open into a pocket for filling', 'name': 'pita_(bread)'}, {'frequency': 'f', 'id': 827, 'synset': 'pitcher.n.02', 'synonyms': ['pitcher_(vessel_for_liquid)', 'ewer'], 'def': 'an open vessel with a handle and a spout for pouring', 'name': 'pitcher_(vessel_for_liquid)'}, {'frequency': 'r', 'id': 828, 'synset': 'pitchfork.n.01', 'synonyms': ['pitchfork'], 'def': 'a long-handled hand tool with sharp widely spaced prongs for lifting and pitching hay', 'name': 'pitchfork'}, {'frequency': 'f', 'id': 829, 'synset': 'pizza.n.01', 'synonyms': ['pizza'], 'def': 'Italian open pie made of thin bread dough spread with a spiced mixture of e.g. tomato sauce and cheese', 'name': 'pizza'}, {'frequency': 'f', 'id': 830, 'synset': 'place_mat.n.01', 'synonyms': ['place_mat'], 'def': 'a mat placed on a table for an individual place setting', 'name': 'place_mat'}, {'frequency': 'f', 'id': 831, 'synset': 'plate.n.04', 'synonyms': ['plate'], 'def': 'dish on which food is served or from which food is eaten', 'name': 'plate'}, {'frequency': 'c', 'id': 832, 'synset': 'platter.n.01', 'synonyms': ['platter'], 'def': 'a large shallow dish used for serving food', 'name': 'platter'}, {'frequency': 'r', 'id': 833, 'synset': 'playing_card.n.01', 'synonyms': ['playing_card'], 'def': 'one of a pack of cards that are used to play card games', 'name': 'playing_card'}, {'frequency': 'r', 'id': 834, 'synset': 'playpen.n.01', 'synonyms': ['playpen'], 'def': 'a portable enclosure in which babies may be left to play', 'name': 'playpen'}, {'frequency': 'c', 'id': 835, 'synset': 'pliers.n.01', 'synonyms': ['pliers', 'plyers'], 'def': 'a gripping hand tool with two hinged arms and (usually) serrated jaws', 'name': 'pliers'}, {'frequency': 'r', 'id': 836, 'synset': 'plow.n.01', 'synonyms': ['plow_(farm_equipment)', 'plough_(farm_equipment)'], 'def': 'a farm tool having one or more heavy blades to break the soil and cut a furrow prior to sowing', 'name': 'plow_(farm_equipment)'}, {'frequency': 'r', 'id': 837, 'synset': 'pocket_watch.n.01', 'synonyms': ['pocket_watch'], 'def': 'a watch that is carried in a small watch pocket', 'name': 'pocket_watch'}, {'frequency': 'c', 'id': 838, 'synset': 'pocketknife.n.01', 'synonyms': ['pocketknife'], 'def': 'a knife with a blade that folds into the handle; suitable for carrying in the pocket', 'name': 'pocketknife'}, {'frequency': 'c', 'id': 839, 'synset': 'poker.n.01', 'synonyms': ['poker_(fire_stirring_tool)', 'stove_poker', 'fire_hook'], 'def': 'fire iron consisting of a metal rod with a handle; used to stir a fire', 'name': 'poker_(fire_stirring_tool)'}, {'frequency': 'f', 'id': 840, 'synset': 'pole.n.01', 'synonyms': ['pole', 'post'], 'def': 'a long (usually round) rod of wood or metal or plastic', 'name': 'pole'}, {'frequency': 'r', 'id': 841, 'synset': 'police_van.n.01', 'synonyms': ['police_van', 'police_wagon', 'paddy_wagon', 'patrol_wagon'], 'def': 'van used by police to transport prisoners', 'name': 'police_van'}, {'frequency': 'f', 'id': 842, 'synset': 'polo_shirt.n.01', 'synonyms': ['polo_shirt', 'sport_shirt'], 'def': 'a shirt with short sleeves designed for comfort and casual wear', 'name': 'polo_shirt'}, {'frequency': 'r', 'id': 843, 'synset': 'poncho.n.01', 'synonyms': ['poncho'], 'def': 'a blanket-like cloak with a hole in the center for the head', 'name': 'poncho'}, {'frequency': 'c', 'id': 844, 'synset': 'pony.n.05', 'synonyms': ['pony'], 'def': 'any of various breeds of small gentle horses usually less than five feet high at the shoulder', 'name': 'pony'}, {'frequency': 'r', 'id': 845, 'synset': 'pool_table.n.01', 'synonyms': ['pool_table', 'billiard_table', 'snooker_table'], 'def': 'game equipment consisting of a heavy table on which pool is played', 'name': 'pool_table'}, {'frequency': 'f', 'id': 846, 'synset': 'pop.n.02', 'synonyms': ['pop_(soda)', 'soda_(pop)', 'tonic', 'soft_drink'], 'def': 'a sweet drink containing carbonated water and flavoring', 'name': 'pop_(soda)'}, {'frequency': 'r', 'id': 847, 'synset': 'portrait.n.02', 'synonyms': ['portrait', 'portrayal'], 'def': 'any likeness of a person, in any medium', 'name': 'portrait'}, {'frequency': 'c', 'id': 848, 'synset': 'postbox.n.01', 'synonyms': ['postbox_(public)', 'mailbox_(public)'], 'def': 'public box for deposit of mail', 'name': 'postbox_(public)'}, {'frequency': 'c', 'id': 849, 'synset': 'postcard.n.01', 'synonyms': ['postcard', 'postal_card', 'mailing-card'], 'def': 'a card for sending messages by post without an envelope', 'name': 'postcard'}, {'frequency': 'f', 'id': 850, 'synset': 'poster.n.01', 'synonyms': ['poster', 'placard'], 'def': 'a sign posted in a public place as an advertisement', 'name': 'poster'}, {'frequency': 'f', 'id': 851, 'synset': 'pot.n.01', 'synonyms': ['pot'], 'def': 'metal or earthenware cooking vessel that is usually round and deep; often has a handle and lid', 'name': 'pot'}, {'frequency': 'f', 'id': 852, 'synset': 'pot.n.04', 'synonyms': ['flowerpot'], 'def': 'a container in which plants are cultivated', 'name': 'flowerpot'}, {'frequency': 'f', 'id': 853, 'synset': 'potato.n.01', 'synonyms': ['potato'], 'def': 'an edible tuber native to South America', 'name': 'potato'}, {'frequency': 'c', 'id': 854, 'synset': 'potholder.n.01', 'synonyms': ['potholder'], 'def': 'an insulated pad for holding hot pots', 'name': 'potholder'}, {'frequency': 'c', 'id': 855, 'synset': 'pottery.n.01', 'synonyms': ['pottery', 'clayware'], 'def': 'ceramic ware made from clay and baked in a kiln', 'name': 'pottery'}, {'frequency': 'c', 'id': 856, 'synset': 'pouch.n.01', 'synonyms': ['pouch'], 'def': 'a small or medium size container for holding or carrying things', 'name': 'pouch'}, {'frequency': 'r', 'id': 857, 'synset': 'power_shovel.n.01', 'synonyms': ['power_shovel', 'excavator', 'digger'], 'def': 'a machine for excavating', 'name': 'power_shovel'}, {'frequency': 'c', 'id': 858, 'synset': 'prawn.n.01', 'synonyms': ['prawn', 'shrimp'], 'def': 'any of various edible decapod crustaceans', 'name': 'prawn'}, {'frequency': 'f', 'id': 859, 'synset': 'printer.n.03', 'synonyms': ['printer', 'printing_machine'], 'def': 'a machine that prints', 'name': 'printer'}, {'frequency': 'c', 'id': 860, 'synset': 'projectile.n.01', 'synonyms': ['projectile_(weapon)', 'missile'], 'def': 'a weapon that is forcibly thrown or projected at a targets', 'name': 'projectile_(weapon)'}, {'frequency': 'c', 'id': 861, 'synset': 'projector.n.02', 'synonyms': ['projector'], 'def': 'an optical instrument that projects an enlarged image onto a screen', 'name': 'projector'}, {'frequency': 'f', 'id': 862, 'synset': 'propeller.n.01', 'synonyms': ['propeller', 'propellor'], 'def': 'a mechanical device that rotates to push against air or water', 'name': 'propeller'}, {'frequency': 'r', 'id': 863, 'synset': 'prune.n.01', 'synonyms': ['prune'], 'def': 'dried plum', 'name': 'prune'}, {'frequency': 'r', 'id': 864, 'synset': 'pudding.n.01', 'synonyms': ['pudding'], 'def': 'any of various soft thick unsweetened baked dishes', 'name': 'pudding'}, {'frequency': 'r', 'id': 865, 'synset': 'puffer.n.02', 'synonyms': ['puffer_(fish)', 'pufferfish', 'blowfish', 'globefish'], 'def': 'fishes whose elongated spiny body can inflate itself with water or air to form a globe', 'name': 'puffer_(fish)'}, {'frequency': 'r', 'id': 866, 'synset': 'puffin.n.01', 'synonyms': ['puffin'], 'def': 'seabirds having short necks and brightly colored compressed bills', 'name': 'puffin'}, {'frequency': 'r', 'id': 867, 'synset': 'pug.n.01', 'synonyms': ['pug-dog'], 'def': 'small compact smooth-coated breed of Asiatic origin having a tightly curled tail and broad flat wrinkled muzzle', 'name': 'pug-dog'}, {'frequency': 'c', 'id': 868, 'synset': 'pumpkin.n.02', 'synonyms': ['pumpkin'], 'def': 'usually large pulpy deep-yellow round fruit of the squash family maturing in late summer or early autumn', 'name': 'pumpkin'}, {'frequency': 'r', 'id': 869, 'synset': 'punch.n.03', 'synonyms': ['puncher'], 'def': 'a tool for making holes or indentations', 'name': 'puncher'}, {'frequency': 'r', 'id': 870, 'synset': 'puppet.n.01', 'synonyms': ['puppet', 'marionette'], 'def': 'a small figure of a person operated from above with strings by a puppeteer', 'name': 'puppet'}, {'frequency': 'r', 'id': 871, 'synset': 'puppy.n.01', 'synonyms': ['puppy'], 'def': 'a young dog', 'name': 'puppy'}, {'frequency': 'r', 'id': 872, 'synset': 'quesadilla.n.01', 'synonyms': ['quesadilla'], 'def': 'a tortilla that is filled with cheese and heated', 'name': 'quesadilla'}, {'frequency': 'r', 'id': 873, 'synset': 'quiche.n.02', 'synonyms': ['quiche'], 'def': 'a tart filled with rich unsweetened custard; often contains other ingredients (as cheese or ham or seafood or vegetables)', 'name': 'quiche'}, {'frequency': 'f', 'id': 874, 'synset': 'quilt.n.01', 'synonyms': ['quilt', 'comforter'], 'def': 'bedding made of two layers of cloth filled with stuffing and stitched together', 'name': 'quilt'}, {'frequency': 'c', 'id': 875, 'synset': 'rabbit.n.01', 'synonyms': ['rabbit'], 'def': 'any of various burrowing animals of the family Leporidae having long ears and short tails', 'name': 'rabbit'}, {'frequency': 'r', 'id': 876, 'synset': 'racer.n.02', 'synonyms': ['race_car', 'racing_car'], 'def': 'a fast car that competes in races', 'name': 'race_car'}, {'frequency': 'c', 'id': 877, 'synset': 'racket.n.04', 'synonyms': ['racket', 'racquet'], 'def': 'a sports implement used to strike a ball in various games', 'name': 'racket'}, {'frequency': 'r', 'id': 878, 'synset': 'radar.n.01', 'synonyms': ['radar'], 'def': 'measuring instrument in which the echo of a pulse of microwave radiation is used to detect and locate distant objects', 'name': 'radar'}, {'frequency': 'c', 'id': 879, 'synset': 'radiator.n.03', 'synonyms': ['radiator'], 'def': 'a mechanism consisting of a metal honeycomb through which hot fluids circulate', 'name': 'radiator'}, {'frequency': 'c', 'id': 880, 'synset': 'radio_receiver.n.01', 'synonyms': ['radio_receiver', 'radio_set', 'radio', 'tuner_(radio)'], 'def': 'an electronic receiver that detects and demodulates and amplifies transmitted radio signals', 'name': 'radio_receiver'}, {'frequency': 'c', 'id': 881, 'synset': 'radish.n.03', 'synonyms': ['radish', 'daikon'], 'def': 'pungent edible root of any of various cultivated radish plants', 'name': 'radish'}, {'frequency': 'c', 'id': 882, 'synset': 'raft.n.01', 'synonyms': ['raft'], 'def': 'a flat float (usually made of logs or planks) that can be used for transport or as a platform for swimmers', 'name': 'raft'}, {'frequency': 'r', 'id': 883, 'synset': 'rag_doll.n.01', 'synonyms': ['rag_doll'], 'def': 'a cloth doll that is stuffed and (usually) painted', 'name': 'rag_doll'}, {'frequency': 'c', 'id': 884, 'synset': 'raincoat.n.01', 'synonyms': ['raincoat', 'waterproof_jacket'], 'def': 'a water-resistant coat', 'name': 'raincoat'}, {'frequency': 'c', 'id': 885, 'synset': 'ram.n.05', 'synonyms': ['ram_(animal)'], 'def': 'uncastrated adult male sheep', 'name': 'ram_(animal)'}, {'frequency': 'c', 'id': 886, 'synset': 'raspberry.n.02', 'synonyms': ['raspberry'], 'def': 'red or black edible aggregate berries usually smaller than the related blackberries', 'name': 'raspberry'}, {'frequency': 'r', 'id': 887, 'synset': 'rat.n.01', 'synonyms': ['rat'], 'def': 'any of various long-tailed rodents similar to but larger than a mouse', 'name': 'rat'}, {'frequency': 'c', 'id': 888, 'synset': 'razorblade.n.01', 'synonyms': ['razorblade'], 'def': 'a blade that has very sharp edge', 'name': 'razorblade'}, {'frequency': 'c', 'id': 889, 'synset': 'reamer.n.01', 'synonyms': ['reamer_(juicer)', 'juicer', 'juice_reamer'], 'def': 'a squeezer with a conical ridged center that is used for squeezing juice from citrus fruit', 'name': 'reamer_(juicer)'}, {'frequency': 'f', 'id': 890, 'synset': 'rearview_mirror.n.01', 'synonyms': ['rearview_mirror'], 'def': 'car mirror that reflects the view out of the rear window', 'name': 'rearview_mirror'}, {'frequency': 'c', 'id': 891, 'synset': 'receipt.n.02', 'synonyms': ['receipt'], 'def': 'an acknowledgment (usually tangible) that payment has been made', 'name': 'receipt'}, {'frequency': 'c', 'id': 892, 'synset': 'recliner.n.01', 'synonyms': ['recliner', 'reclining_chair', 'lounger_(chair)'], 'def': 'an armchair whose back can be lowered and foot can be raised to allow the sitter to recline in it', 'name': 'recliner'}, {'frequency': 'r', 'id': 893, 'synset': 'record_player.n.01', 'synonyms': ['record_player', 'phonograph_(record_player)', 'turntable'], 'def': 'machine in which rotating records cause a stylus to vibrate and the vibrations are amplified acoustically or electronically', 'name': 'record_player'}, {'frequency': 'r', 'id': 894, 'synset': 'red_cabbage.n.02', 'synonyms': ['red_cabbage'], 'def': 'compact head of purplish-red leaves', 'name': 'red_cabbage'}, {'frequency': 'f', 'id': 895, 'synset': 'reflector.n.01', 'synonyms': ['reflector'], 'def': 'device that reflects light, radiation, etc.', 'name': 'reflector'}, {'frequency': 'f', 'id': 896, 'synset': 'remote_control.n.01', 'synonyms': ['remote_control'], 'def': 'a device that can be used to control a machine or apparatus from a distance', 'name': 'remote_control'}, {'frequency': 'c', 'id': 897, 'synset': 'rhinoceros.n.01', 'synonyms': ['rhinoceros'], 'def': 'massive powerful herbivorous odd-toed ungulate of southeast Asia and Africa having very thick skin and one or two horns on the snout', 'name': 'rhinoceros'}, {'frequency': 'r', 'id': 898, 'synset': 'rib.n.03', 'synonyms': ['rib_(food)'], 'def': 'cut of meat including one or more ribs', 'name': 'rib_(food)'}, {'frequency': 'r', 'id': 899, 'synset': 'rifle.n.01', 'synonyms': ['rifle'], 'def': 'a shoulder firearm with a long barrel', 'name': 'rifle'}, {'frequency': 'f', 'id': 900, 'synset': 'ring.n.08', 'synonyms': ['ring'], 'def': 'jewelry consisting of a circlet of precious metal (often set with jewels) worn on the finger', 'name': 'ring'}, {'frequency': 'r', 'id': 901, 'synset': 'river_boat.n.01', 'synonyms': ['river_boat'], 'def': 'a boat used on rivers or to ply a river', 'name': 'river_boat'}, {'frequency': 'r', 'id': 902, 'synset': 'road_map.n.02', 'synonyms': ['road_map'], 'def': '(NOT A ROAD) a MAP showing roads (for automobile travel)', 'name': 'road_map'}, {'frequency': 'c', 'id': 903, 'synset': 'robe.n.01', 'synonyms': ['robe'], 'def': 'any loose flowing garment', 'name': 'robe'}, {'frequency': 'c', 'id': 904, 'synset': 'rocking_chair.n.01', 'synonyms': ['rocking_chair'], 'def': 'a chair mounted on rockers', 'name': 'rocking_chair'}, {'frequency': 'r', 'id': 905, 'synset': 'roller_skate.n.01', 'synonyms': ['roller_skate'], 'def': 'a shoe with pairs of rollers (small hard wheels) fixed to the sole', 'name': 'roller_skate'}, {'frequency': 'r', 'id': 906, 'synset': 'rollerblade.n.01', 'synonyms': ['Rollerblade'], 'def': 'an in-line variant of a roller skate', 'name': 'Rollerblade'}, {'frequency': 'c', 'id': 907, 'synset': 'rolling_pin.n.01', 'synonyms': ['rolling_pin'], 'def': 'utensil consisting of a cylinder (usually of wood) with a handle at each end; used to roll out dough', 'name': 'rolling_pin'}, {'frequency': 'r', 'id': 908, 'synset': 'root_beer.n.01', 'synonyms': ['root_beer'], 'def': 'carbonated drink containing extracts of roots and herbs', 'name': 'root_beer'}, {'frequency': 'c', 'id': 909, 'synset': 'router.n.02', 'synonyms': ['router_(computer_equipment)'], 'def': 'a device that forwards data packets between computer networks', 'name': 'router_(computer_equipment)'}, {'frequency': 'f', 'id': 910, 'synset': 'rubber_band.n.01', 'synonyms': ['rubber_band', 'elastic_band'], 'def': 'a narrow band of elastic rubber used to hold things (such as papers) together', 'name': 'rubber_band'}, {'frequency': 'c', 'id': 911, 'synset': 'runner.n.08', 'synonyms': ['runner_(carpet)'], 'def': 'a long narrow carpet', 'name': 'runner_(carpet)'}, {'frequency': 'f', 'id': 912, 'synset': 'sack.n.01', 'synonyms': ['plastic_bag', 'paper_bag'], 'def': "a bag made of paper or plastic for holding customer's purchases", 'name': 'plastic_bag'}, {'frequency': 'f', 'id': 913, 'synset': 'saddle.n.01', 'synonyms': ['saddle_(on_an_animal)'], 'def': 'a seat for the rider of a horse or camel', 'name': 'saddle_(on_an_animal)'}, {'frequency': 'f', 'id': 914, 'synset': 'saddle_blanket.n.01', 'synonyms': ['saddle_blanket', 'saddlecloth', 'horse_blanket'], 'def': 'stable gear consisting of a blanket placed under the saddle', 'name': 'saddle_blanket'}, {'frequency': 'c', 'id': 915, 'synset': 'saddlebag.n.01', 'synonyms': ['saddlebag'], 'def': 'a large bag (or pair of bags) hung over a saddle', 'name': 'saddlebag'}, {'frequency': 'r', 'id': 916, 'synset': 'safety_pin.n.01', 'synonyms': ['safety_pin'], 'def': 'a pin in the form of a clasp; has a guard so the point of the pin will not stick the user', 'name': 'safety_pin'}, {'frequency': 'c', 'id': 917, 'synset': 'sail.n.01', 'synonyms': ['sail'], 'def': 'a large piece of fabric by means of which wind is used to propel a sailing vessel', 'name': 'sail'}, {'frequency': 'c', 'id': 918, 'synset': 'salad.n.01', 'synonyms': ['salad'], 'def': 'food mixtures either arranged on a plate or tossed and served with a moist dressing; usually consisting of or including greens', 'name': 'salad'}, {'frequency': 'r', 'id': 919, 'synset': 'salad_plate.n.01', 'synonyms': ['salad_plate', 'salad_bowl'], 'def': 'a plate or bowl for individual servings of salad', 'name': 'salad_plate'}, {'frequency': 'r', 'id': 920, 'synset': 'salami.n.01', 'synonyms': ['salami'], 'def': 'highly seasoned fatty sausage of pork and beef usually dried', 'name': 'salami'}, {'frequency': 'r', 'id': 921, 'synset': 'salmon.n.01', 'synonyms': ['salmon_(fish)'], 'def': 'any of various large food and game fishes of northern waters', 'name': 'salmon_(fish)'}, {'frequency': 'r', 'id': 922, 'synset': 'salmon.n.03', 'synonyms': ['salmon_(food)'], 'def': 'flesh of any of various marine or freshwater fish of the family Salmonidae', 'name': 'salmon_(food)'}, {'frequency': 'r', 'id': 923, 'synset': 'salsa.n.01', 'synonyms': ['salsa'], 'def': 'spicy sauce of tomatoes and onions and chili peppers to accompany Mexican foods', 'name': 'salsa'}, {'frequency': 'f', 'id': 924, 'synset': 'saltshaker.n.01', 'synonyms': ['saltshaker'], 'def': 'a shaker with a perforated top for sprinkling salt', 'name': 'saltshaker'}, {'frequency': 'f', 'id': 925, 'synset': 'sandal.n.01', 'synonyms': ['sandal_(type_of_shoe)'], 'def': 'a shoe consisting of a sole fastened by straps to the foot', 'name': 'sandal_(type_of_shoe)'}, {'frequency': 'f', 'id': 926, 'synset': 'sandwich.n.01', 'synonyms': ['sandwich'], 'def': 'two (or more) slices of bread with a filling between them', 'name': 'sandwich'}, {'frequency': 'r', 'id': 927, 'synset': 'satchel.n.01', 'synonyms': ['satchel'], 'def': 'luggage consisting of a small case with a flat bottom and (usually) a shoulder strap', 'name': 'satchel'}, {'frequency': 'r', 'id': 928, 'synset': 'saucepan.n.01', 'synonyms': ['saucepan'], 'def': 'a deep pan with a handle; used for stewing or boiling', 'name': 'saucepan'}, {'frequency': 'f', 'id': 929, 'synset': 'saucer.n.02', 'synonyms': ['saucer'], 'def': 'a small shallow dish for holding a cup at the table', 'name': 'saucer'}, {'frequency': 'f', 'id': 930, 'synset': 'sausage.n.01', 'synonyms': ['sausage'], 'def': 'highly seasoned minced meat stuffed in casings', 'name': 'sausage'}, {'frequency': 'r', 'id': 931, 'synset': 'sawhorse.n.01', 'synonyms': ['sawhorse', 'sawbuck'], 'def': 'a framework for holding wood that is being sawed', 'name': 'sawhorse'}, {'frequency': 'r', 'id': 932, 'synset': 'sax.n.02', 'synonyms': ['saxophone'], 'def': "a wind instrument with a `J'-shaped form typically made of brass", 'name': 'saxophone'}, {'frequency': 'f', 'id': 933, 'synset': 'scale.n.07', 'synonyms': ['scale_(measuring_instrument)'], 'def': 'a measuring instrument for weighing; shows amount of mass', 'name': 'scale_(measuring_instrument)'}, {'frequency': 'r', 'id': 934, 'synset': 'scarecrow.n.01', 'synonyms': ['scarecrow', 'strawman'], 'def': 'an effigy in the shape of a man to frighten birds away from seeds', 'name': 'scarecrow'}, {'frequency': 'f', 'id': 935, 'synset': 'scarf.n.01', 'synonyms': ['scarf'], 'def': 'a garment worn around the head or neck or shoulders for warmth or decoration', 'name': 'scarf'}, {'frequency': 'c', 'id': 936, 'synset': 'school_bus.n.01', 'synonyms': ['school_bus'], 'def': 'a bus used to transport children to or from school', 'name': 'school_bus'}, {'frequency': 'f', 'id': 937, 'synset': 'scissors.n.01', 'synonyms': ['scissors'], 'def': 'a tool having two crossed pivoting blades with looped handles', 'name': 'scissors'}, {'frequency': 'c', 'id': 938, 'synset': 'scoreboard.n.01', 'synonyms': ['scoreboard'], 'def': 'a large board for displaying the score of a contest (and some other information)', 'name': 'scoreboard'}, {'frequency': 'c', 'id': 939, 'synset': 'scrambled_eggs.n.01', 'synonyms': ['scrambled_eggs'], 'def': 'eggs beaten and cooked to a soft firm consistency while stirring', 'name': 'scrambled_eggs'}, {'frequency': 'r', 'id': 940, 'synset': 'scraper.n.01', 'synonyms': ['scraper'], 'def': 'any of various hand tools for scraping', 'name': 'scraper'}, {'frequency': 'r', 'id': 941, 'synset': 'scratcher.n.03', 'synonyms': ['scratcher'], 'def': 'a device used for scratching', 'name': 'scratcher'}, {'frequency': 'c', 'id': 942, 'synset': 'screwdriver.n.01', 'synonyms': ['screwdriver'], 'def': 'a hand tool for driving screws; has a tip that fits into the head of a screw', 'name': 'screwdriver'}, {'frequency': 'c', 'id': 943, 'synset': 'scrub_brush.n.01', 'synonyms': ['scrubbing_brush'], 'def': 'a brush with short stiff bristles for heavy cleaning', 'name': 'scrubbing_brush'}, {'frequency': 'c', 'id': 944, 'synset': 'sculpture.n.01', 'synonyms': ['sculpture'], 'def': 'a three-dimensional work of art', 'name': 'sculpture'}, {'frequency': 'r', 'id': 945, 'synset': 'seabird.n.01', 'synonyms': ['seabird', 'seafowl'], 'def': 'a bird that frequents coastal waters and the open ocean: gulls; pelicans; gannets; cormorants; albatrosses; petrels; etc.', 'name': 'seabird'}, {'frequency': 'r', 'id': 946, 'synset': 'seahorse.n.02', 'synonyms': ['seahorse'], 'def': 'small fish with horse-like heads bent sharply downward and curled tails', 'name': 'seahorse'}, {'frequency': 'r', 'id': 947, 'synset': 'seaplane.n.01', 'synonyms': ['seaplane', 'hydroplane'], 'def': 'an airplane that can land on or take off from water', 'name': 'seaplane'}, {'frequency': 'c', 'id': 948, 'synset': 'seashell.n.01', 'synonyms': ['seashell'], 'def': 'the shell of a marine organism', 'name': 'seashell'}, {'frequency': 'r', 'id': 949, 'synset': 'seedling.n.01', 'synonyms': ['seedling'], 'def': 'young plant or tree grown from a seed', 'name': 'seedling'}, {'frequency': 'c', 'id': 950, 'synset': 'serving_dish.n.01', 'synonyms': ['serving_dish'], 'def': 'a dish used for serving food', 'name': 'serving_dish'}, {'frequency': 'r', 'id': 951, 'synset': 'sewing_machine.n.01', 'synonyms': ['sewing_machine'], 'def': 'a textile machine used as a home appliance for sewing', 'name': 'sewing_machine'}, {'frequency': 'r', 'id': 952, 'synset': 'shaker.n.03', 'synonyms': ['shaker'], 'def': 'a container in which something can be shaken', 'name': 'shaker'}, {'frequency': 'c', 'id': 953, 'synset': 'shampoo.n.01', 'synonyms': ['shampoo'], 'def': 'cleansing agent consisting of soaps or detergents used for washing the hair', 'name': 'shampoo'}, {'frequency': 'r', 'id': 954, 'synset': 'shark.n.01', 'synonyms': ['shark'], 'def': 'typically large carnivorous fishes with sharpe teeth', 'name': 'shark'}, {'frequency': 'r', 'id': 955, 'synset': 'sharpener.n.01', 'synonyms': ['sharpener'], 'def': 'any implement that is used to make something (an edge or a point) sharper', 'name': 'sharpener'}, {'frequency': 'r', 'id': 956, 'synset': 'sharpie.n.03', 'synonyms': ['Sharpie'], 'def': 'a pen with indelible ink that will write on any surface', 'name': 'Sharpie'}, {'frequency': 'r', 'id': 957, 'synset': 'shaver.n.03', 'synonyms': ['shaver_(electric)', 'electric_shaver', 'electric_razor'], 'def': 'a razor powered by an electric motor', 'name': 'shaver_(electric)'}, {'frequency': 'c', 'id': 958, 'synset': 'shaving_cream.n.01', 'synonyms': ['shaving_cream', 'shaving_soap'], 'def': 'toiletry consisting that forms a rich lather for softening the beard before shaving', 'name': 'shaving_cream'}, {'frequency': 'r', 'id': 959, 'synset': 'shawl.n.01', 'synonyms': ['shawl'], 'def': 'cloak consisting of an oblong piece of cloth used to cover the head and shoulders', 'name': 'shawl'}, {'frequency': 'r', 'id': 960, 'synset': 'shears.n.01', 'synonyms': ['shears'], 'def': 'large scissors with strong blades', 'name': 'shears'}, {'frequency': 'f', 'id': 961, 'synset': 'sheep.n.01', 'synonyms': ['sheep'], 'def': 'woolly usually horned ruminant mammal related to the goat', 'name': 'sheep'}, {'frequency': 'r', 'id': 962, 'synset': 'shepherd_dog.n.01', 'synonyms': ['shepherd_dog', 'sheepdog'], 'def': 'any of various usually long-haired breeds of dog reared to herd and guard sheep', 'name': 'shepherd_dog'}, {'frequency': 'r', 'id': 963, 'synset': 'sherbert.n.01', 'synonyms': ['sherbert', 'sherbet'], 'def': 'a frozen dessert made primarily of fruit juice and sugar', 'name': 'sherbert'}, {'frequency': 'r', 'id': 964, 'synset': 'shield.n.02', 'synonyms': ['shield'], 'def': 'armor carried on the arm to intercept blows', 'name': 'shield'}, {'frequency': 'f', 'id': 965, 'synset': 'shirt.n.01', 'synonyms': ['shirt'], 'def': 'a garment worn on the upper half of the body', 'name': 'shirt'}, {'frequency': 'f', 'id': 966, 'synset': 'shoe.n.01', 'synonyms': ['shoe', 'sneaker_(type_of_shoe)', 'tennis_shoe'], 'def': 'common footwear covering the foot', 'name': 'shoe'}, {'frequency': 'c', 'id': 967, 'synset': 'shopping_bag.n.01', 'synonyms': ['shopping_bag'], 'def': 'a bag made of plastic or strong paper (often with handles); used to transport goods after shopping', 'name': 'shopping_bag'}, {'frequency': 'c', 'id': 968, 'synset': 'shopping_cart.n.01', 'synonyms': ['shopping_cart'], 'def': 'a handcart that holds groceries or other goods while shopping', 'name': 'shopping_cart'}, {'frequency': 'f', 'id': 969, 'synset': 'short_pants.n.01', 'synonyms': ['short_pants', 'shorts_(clothing)', 'trunks_(clothing)'], 'def': 'trousers that end at or above the knee', 'name': 'short_pants'}, {'frequency': 'r', 'id': 970, 'synset': 'shot_glass.n.01', 'synonyms': ['shot_glass'], 'def': 'a small glass adequate to hold a single swallow of whiskey', 'name': 'shot_glass'}, {'frequency': 'c', 'id': 971, 'synset': 'shoulder_bag.n.01', 'synonyms': ['shoulder_bag'], 'def': 'a large handbag that can be carried by a strap looped over the shoulder', 'name': 'shoulder_bag'}, {'frequency': 'c', 'id': 972, 'synset': 'shovel.n.01', 'synonyms': ['shovel'], 'def': 'a hand tool for lifting loose material such as snow, dirt, etc.', 'name': 'shovel'}, {'frequency': 'f', 'id': 973, 'synset': 'shower.n.01', 'synonyms': ['shower_head'], 'def': 'a plumbing fixture that sprays water over you', 'name': 'shower_head'}, {'frequency': 'f', 'id': 974, 'synset': 'shower_curtain.n.01', 'synonyms': ['shower_curtain'], 'def': 'a curtain that keeps water from splashing out of the shower area', 'name': 'shower_curtain'}, {'frequency': 'r', 'id': 975, 'synset': 'shredder.n.01', 'synonyms': ['shredder_(for_paper)'], 'def': 'a device that shreds documents', 'name': 'shredder_(for_paper)'}, {'frequency': 'r', 'id': 976, 'synset': 'sieve.n.01', 'synonyms': ['sieve', 'screen_(sieve)'], 'def': 'a strainer for separating lumps from powdered material or grading particles', 'name': 'sieve'}, {'frequency': 'f', 'id': 977, 'synset': 'signboard.n.01', 'synonyms': ['signboard'], 'def': 'structure displaying a board on which advertisements can be posted', 'name': 'signboard'}, {'frequency': 'c', 'id': 978, 'synset': 'silo.n.01', 'synonyms': ['silo'], 'def': 'a cylindrical tower used for storing goods', 'name': 'silo'}, {'frequency': 'f', 'id': 979, 'synset': 'sink.n.01', 'synonyms': ['sink'], 'def': 'plumbing fixture consisting of a water basin fixed to a wall or floor and having a drainpipe', 'name': 'sink'}, {'frequency': 'f', 'id': 980, 'synset': 'skateboard.n.01', 'synonyms': ['skateboard'], 'def': 'a board with wheels that is ridden in a standing or crouching position and propelled by foot', 'name': 'skateboard'}, {'frequency': 'c', 'id': 981, 'synset': 'skewer.n.01', 'synonyms': ['skewer'], 'def': 'a long pin for holding meat in position while it is being roasted', 'name': 'skewer'}, {'frequency': 'f', 'id': 982, 'synset': 'ski.n.01', 'synonyms': ['ski'], 'def': 'sports equipment for skiing on snow', 'name': 'ski'}, {'frequency': 'f', 'id': 983, 'synset': 'ski_boot.n.01', 'synonyms': ['ski_boot'], 'def': 'a stiff boot that is fastened to a ski with a ski binding', 'name': 'ski_boot'}, {'frequency': 'f', 'id': 984, 'synset': 'ski_parka.n.01', 'synonyms': ['ski_parka', 'ski_jacket'], 'def': 'a parka to be worn while skiing', 'name': 'ski_parka'}, {'frequency': 'f', 'id': 985, 'synset': 'ski_pole.n.01', 'synonyms': ['ski_pole'], 'def': 'a pole with metal points used as an aid in skiing', 'name': 'ski_pole'}, {'frequency': 'f', 'id': 986, 'synset': 'skirt.n.02', 'synonyms': ['skirt'], 'def': 'a garment hanging from the waist; worn mainly by girls and women', 'name': 'skirt'}, {'frequency': 'c', 'id': 987, 'synset': 'sled.n.01', 'synonyms': ['sled', 'sledge', 'sleigh'], 'def': 'a vehicle or flat object for transportation over snow by sliding or pulled by dogs, etc.', 'name': 'sled'}, {'frequency': 'c', 'id': 988, 'synset': 'sleeping_bag.n.01', 'synonyms': ['sleeping_bag'], 'def': 'large padded bag designed to be slept in outdoors', 'name': 'sleeping_bag'}, {'frequency': 'r', 'id': 989, 'synset': 'sling.n.05', 'synonyms': ['sling_(bandage)', 'triangular_bandage'], 'def': 'bandage to support an injured forearm; slung over the shoulder or neck', 'name': 'sling_(bandage)'}, {'frequency': 'c', 'id': 990, 'synset': 'slipper.n.01', 'synonyms': ['slipper_(footwear)', 'carpet_slipper_(footwear)'], 'def': 'low footwear that can be slipped on and off easily; usually worn indoors', 'name': 'slipper_(footwear)'}, {'frequency': 'r', 'id': 991, 'synset': 'smoothie.n.02', 'synonyms': ['smoothie'], 'def': 'a thick smooth drink consisting of fresh fruit pureed with ice cream or yoghurt or milk', 'name': 'smoothie'}, {'frequency': 'r', 'id': 992, 'synset': 'snake.n.01', 'synonyms': ['snake', 'serpent'], 'def': 'limbless scaly elongate reptile; some are venomous', 'name': 'snake'}, {'frequency': 'f', 'id': 993, 'synset': 'snowboard.n.01', 'synonyms': ['snowboard'], 'def': 'a board that resembles a broad ski or a small surfboard; used in a standing position to slide down snow-covered slopes', 'name': 'snowboard'}, {'frequency': 'c', 'id': 994, 'synset': 'snowman.n.01', 'synonyms': ['snowman'], 'def': 'a figure of a person made of packed snow', 'name': 'snowman'}, {'frequency': 'c', 'id': 995, 'synset': 'snowmobile.n.01', 'synonyms': ['snowmobile'], 'def': 'tracked vehicle for travel on snow having skis in front', 'name': 'snowmobile'}, {'frequency': 'f', 'id': 996, 'synset': 'soap.n.01', 'synonyms': ['soap'], 'def': 'a cleansing agent made from the salts of vegetable or animal fats', 'name': 'soap'}, {'frequency': 'f', 'id': 997, 'synset': 'soccer_ball.n.01', 'synonyms': ['soccer_ball'], 'def': "an inflated ball used in playing soccer (called `football' outside of the United States)", 'name': 'soccer_ball'}, {'frequency': 'f', 'id': 998, 'synset': 'sock.n.01', 'synonyms': ['sock'], 'def': 'cloth covering for the foot; worn inside the shoe; reaches to between the ankle and the knee', 'name': 'sock'}, {'frequency': 'r', 'id': 999, 'synset': 'soda_fountain.n.02', 'synonyms': ['soda_fountain'], 'def': 'an apparatus for dispensing soda water', 'name': 'soda_fountain'}, {'frequency': 'r', 'id': 1000, 'synset': 'soda_water.n.01', 'synonyms': ['carbonated_water', 'club_soda', 'seltzer', 'sparkling_water'], 'def': 'effervescent beverage artificially charged with carbon dioxide', 'name': 'carbonated_water'}, {'frequency': 'f', 'id': 1001, 'synset': 'sofa.n.01', 'synonyms': ['sofa', 'couch', 'lounge'], 'def': 'an upholstered seat for more than one person', 'name': 'sofa'}, {'frequency': 'r', 'id': 1002, 'synset': 'softball.n.01', 'synonyms': ['softball'], 'def': 'ball used in playing softball', 'name': 'softball'}, {'frequency': 'c', 'id': 1003, 'synset': 'solar_array.n.01', 'synonyms': ['solar_array', 'solar_battery', 'solar_panel'], 'def': 'electrical device consisting of a large array of connected solar cells', 'name': 'solar_array'}, {'frequency': 'r', 'id': 1004, 'synset': 'sombrero.n.02', 'synonyms': ['sombrero'], 'def': 'a straw hat with a tall crown and broad brim; worn in American southwest and in Mexico', 'name': 'sombrero'}, {'frequency': 'c', 'id': 1005, 'synset': 'soup.n.01', 'synonyms': ['soup'], 'def': 'liquid food especially of meat or fish or vegetable stock often containing pieces of solid food', 'name': 'soup'}, {'frequency': 'r', 'id': 1006, 'synset': 'soup_bowl.n.01', 'synonyms': ['soup_bowl'], 'def': 'a bowl for serving soup', 'name': 'soup_bowl'}, {'frequency': 'c', 'id': 1007, 'synset': 'soupspoon.n.01', 'synonyms': ['soupspoon'], 'def': 'a spoon with a rounded bowl for eating soup', 'name': 'soupspoon'}, {'frequency': 'c', 'id': 1008, 'synset': 'sour_cream.n.01', 'synonyms': ['sour_cream', 'soured_cream'], 'def': 'soured light cream', 'name': 'sour_cream'}, {'frequency': 'r', 'id': 1009, 'synset': 'soya_milk.n.01', 'synonyms': ['soya_milk', 'soybean_milk', 'soymilk'], 'def': 'a milk substitute containing soybean flour and water; used in some infant formulas and in making tofu', 'name': 'soya_milk'}, {'frequency': 'r', 'id': 1010, 'synset': 'space_shuttle.n.01', 'synonyms': ['space_shuttle'], 'def': "a reusable spacecraft with wings for a controlled descent through the Earth's atmosphere", 'name': 'space_shuttle'}, {'frequency': 'r', 'id': 1011, 'synset': 'sparkler.n.02', 'synonyms': ['sparkler_(fireworks)'], 'def': 'a firework that burns slowly and throws out a shower of sparks', 'name': 'sparkler_(fireworks)'}, {'frequency': 'f', 'id': 1012, 'synset': 'spatula.n.02', 'synonyms': ['spatula'], 'def': 'a hand tool with a thin flexible blade used to mix or spread soft substances', 'name': 'spatula'}, {'frequency': 'r', 'id': 1013, 'synset': 'spear.n.01', 'synonyms': ['spear', 'lance'], 'def': 'a long pointed rod used as a tool or weapon', 'name': 'spear'}, {'frequency': 'f', 'id': 1014, 'synset': 'spectacles.n.01', 'synonyms': ['spectacles', 'specs', 'eyeglasses', 'glasses'], 'def': 'optical instrument consisting of a frame that holds a pair of lenses for correcting defective vision', 'name': 'spectacles'}, {'frequency': 'c', 'id': 1015, 'synset': 'spice_rack.n.01', 'synonyms': ['spice_rack'], 'def': 'a rack for displaying containers filled with spices', 'name': 'spice_rack'}, {'frequency': 'r', 'id': 1016, 'synset': 'spider.n.01', 'synonyms': ['spider'], 'def': 'predatory arachnid with eight legs, two poison fangs, two feelers, and usually two silk-spinning organs at the back end of the body', 'name': 'spider'}, {'frequency': 'c', 'id': 1017, 'synset': 'sponge.n.01', 'synonyms': ['sponge'], 'def': 'a porous mass usable to absorb water typically used for cleaning', 'name': 'sponge'}, {'frequency': 'f', 'id': 1018, 'synset': 'spoon.n.01', 'synonyms': ['spoon'], 'def': 'a piece of cutlery with a shallow bowl-shaped container and a handle', 'name': 'spoon'}, {'frequency': 'c', 'id': 1019, 'synset': 'sportswear.n.01', 'synonyms': ['sportswear', 'athletic_wear', 'activewear'], 'def': 'attire worn for sport or for casual wear', 'name': 'sportswear'}, {'frequency': 'c', 'id': 1020, 'synset': 'spotlight.n.02', 'synonyms': ['spotlight'], 'def': 'a lamp that produces a strong beam of light to illuminate a restricted area; used to focus attention of a stage performer', 'name': 'spotlight'}, {'frequency': 'r', 'id': 1021, 'synset': 'squirrel.n.01', 'synonyms': ['squirrel'], 'def': 'a kind of arboreal rodent having a long bushy tail', 'name': 'squirrel'}, {'frequency': 'c', 'id': 1022, 'synset': 'stapler.n.01', 'synonyms': ['stapler_(stapling_machine)'], 'def': 'a machine that inserts staples into sheets of paper in order to fasten them together', 'name': 'stapler_(stapling_machine)'}, {'frequency': 'r', 'id': 1023, 'synset': 'starfish.n.01', 'synonyms': ['starfish', 'sea_star'], 'def': 'echinoderms characterized by five arms extending from a central disk', 'name': 'starfish'}, {'frequency': 'f', 'id': 1024, 'synset': 'statue.n.01', 'synonyms': ['statue_(sculpture)'], 'def': 'a sculpture representing a human or animal', 'name': 'statue_(sculpture)'}, {'frequency': 'c', 'id': 1025, 'synset': 'steak.n.01', 'synonyms': ['steak_(food)'], 'def': 'a slice of meat cut from the fleshy part of an animal or large fish', 'name': 'steak_(food)'}, {'frequency': 'r', 'id': 1026, 'synset': 'steak_knife.n.01', 'synonyms': ['steak_knife'], 'def': 'a sharp table knife used in eating steak', 'name': 'steak_knife'}, {'frequency': 'r', 'id': 1027, 'synset': 'steamer.n.02', 'synonyms': ['steamer_(kitchen_appliance)'], 'def': 'a cooking utensil that can be used to cook food by steaming it', 'name': 'steamer_(kitchen_appliance)'}, {'frequency': 'f', 'id': 1028, 'synset': 'steering_wheel.n.01', 'synonyms': ['steering_wheel'], 'def': 'a handwheel that is used for steering', 'name': 'steering_wheel'}, {'frequency': 'r', 'id': 1029, 'synset': 'stencil.n.01', 'synonyms': ['stencil'], 'def': 'a sheet of material (metal, plastic, etc.) that has been perforated with a pattern; ink or paint can pass through the perforations to create the printed pattern on the surface below', 'name': 'stencil'}, {'frequency': 'r', 'id': 1030, 'synset': 'step_ladder.n.01', 'synonyms': ['stepladder'], 'def': 'a folding portable ladder hinged at the top', 'name': 'stepladder'}, {'frequency': 'c', 'id': 1031, 'synset': 'step_stool.n.01', 'synonyms': ['step_stool'], 'def': 'a stool that has one or two steps that fold under the seat', 'name': 'step_stool'}, {'frequency': 'c', 'id': 1032, 'synset': 'stereo.n.01', 'synonyms': ['stereo_(sound_system)'], 'def': 'electronic device for playing audio', 'name': 'stereo_(sound_system)'}, {'frequency': 'r', 'id': 1033, 'synset': 'stew.n.02', 'synonyms': ['stew'], 'def': 'food prepared by stewing especially meat or fish with vegetables', 'name': 'stew'}, {'frequency': 'r', 'id': 1034, 'synset': 'stirrer.n.02', 'synonyms': ['stirrer'], 'def': 'an implement used for stirring', 'name': 'stirrer'}, {'frequency': 'f', 'id': 1035, 'synset': 'stirrup.n.01', 'synonyms': ['stirrup'], 'def': "support consisting of metal loops into which rider's feet go", 'name': 'stirrup'}, {'frequency': 'c', 'id': 1036, 'synset': 'stocking.n.01', 'synonyms': ['stockings_(leg_wear)'], 'def': 'close-fitting hosiery to cover the foot and leg; come in matched pairs', 'name': 'stockings_(leg_wear)'}, {'frequency': 'f', 'id': 1037, 'synset': 'stool.n.01', 'synonyms': ['stool'], 'def': 'a simple seat without a back or arms', 'name': 'stool'}, {'frequency': 'f', 'id': 1038, 'synset': 'stop_sign.n.01', 'synonyms': ['stop_sign'], 'def': 'a traffic sign to notify drivers that they must come to a complete stop', 'name': 'stop_sign'}, {'frequency': 'f', 'id': 1039, 'synset': 'stoplight.n.01', 'synonyms': ['brake_light'], 'def': 'a red light on the rear of a motor vehicle that signals when the brakes are applied', 'name': 'brake_light'}, {'frequency': 'f', 'id': 1040, 'synset': 'stove.n.01', 'synonyms': ['stove', 'kitchen_stove', 'range_(kitchen_appliance)', 'kitchen_range', 'cooking_stove'], 'def': 'a kitchen appliance used for cooking food', 'name': 'stove'}, {'frequency': 'c', 'id': 1041, 'synset': 'strainer.n.01', 'synonyms': ['strainer'], 'def': 'a filter to retain larger pieces while smaller pieces and liquids pass through', 'name': 'strainer'}, {'frequency': 'f', 'id': 1042, 'synset': 'strap.n.01', 'synonyms': ['strap'], 'def': 'an elongated strip of material for binding things together or holding', 'name': 'strap'}, {'frequency': 'f', 'id': 1043, 'synset': 'straw.n.04', 'synonyms': ['straw_(for_drinking)', 'drinking_straw'], 'def': 'a thin paper or plastic tube used to suck liquids into the mouth', 'name': 'straw_(for_drinking)'}, {'frequency': 'f', 'id': 1044, 'synset': 'strawberry.n.01', 'synonyms': ['strawberry'], 'def': 'sweet fleshy red fruit', 'name': 'strawberry'}, {'frequency': 'f', 'id': 1045, 'synset': 'street_sign.n.01', 'synonyms': ['street_sign'], 'def': 'a sign visible from the street', 'name': 'street_sign'}, {'frequency': 'f', 'id': 1046, 'synset': 'streetlight.n.01', 'synonyms': ['streetlight', 'street_lamp'], 'def': 'a lamp supported on a lamppost; for illuminating a street', 'name': 'streetlight'}, {'frequency': 'r', 'id': 1047, 'synset': 'string_cheese.n.01', 'synonyms': ['string_cheese'], 'def': 'cheese formed in long strings twisted together', 'name': 'string_cheese'}, {'frequency': 'r', 'id': 1048, 'synset': 'stylus.n.02', 'synonyms': ['stylus'], 'def': 'a pointed tool for writing or drawing or engraving', 'name': 'stylus'}, {'frequency': 'r', 'id': 1049, 'synset': 'subwoofer.n.01', 'synonyms': ['subwoofer'], 'def': 'a loudspeaker that is designed to reproduce very low bass frequencies', 'name': 'subwoofer'}, {'frequency': 'r', 'id': 1050, 'synset': 'sugar_bowl.n.01', 'synonyms': ['sugar_bowl'], 'def': 'a dish in which sugar is served', 'name': 'sugar_bowl'}, {'frequency': 'r', 'id': 1051, 'synset': 'sugarcane.n.01', 'synonyms': ['sugarcane_(plant)'], 'def': 'juicy canes whose sap is a source of molasses and commercial sugar; fresh canes are sometimes chewed for the juice', 'name': 'sugarcane_(plant)'}, {'frequency': 'c', 'id': 1052, 'synset': 'suit.n.01', 'synonyms': ['suit_(clothing)'], 'def': 'a set of garments (usually including a jacket and trousers or skirt) for outerwear all of the same fabric and color', 'name': 'suit_(clothing)'}, {'frequency': 'c', 'id': 1053, 'synset': 'sunflower.n.01', 'synonyms': ['sunflower'], 'def': 'any plant of the genus Helianthus having large flower heads with dark disk florets and showy yellow rays', 'name': 'sunflower'}, {'frequency': 'f', 'id': 1054, 'synset': 'sunglasses.n.01', 'synonyms': ['sunglasses'], 'def': 'spectacles that are darkened or polarized to protect the eyes from the glare of the sun', 'name': 'sunglasses'}, {'frequency': 'c', 'id': 1055, 'synset': 'sunhat.n.01', 'synonyms': ['sunhat'], 'def': 'a hat with a broad brim that protects the face from direct exposure to the sun', 'name': 'sunhat'}, {'frequency': 'r', 'id': 1056, 'synset': 'sunscreen.n.01', 'synonyms': ['sunscreen', 'sunblock'], 'def': 'a cream spread on the skin; contains a chemical to filter out ultraviolet light and so protect from sunburn', 'name': 'sunscreen'}, {'frequency': 'f', 'id': 1057, 'synset': 'surfboard.n.01', 'synonyms': ['surfboard'], 'def': 'a narrow buoyant board for riding surf', 'name': 'surfboard'}, {'frequency': 'c', 'id': 1058, 'synset': 'sushi.n.01', 'synonyms': ['sushi'], 'def': 'rice (with raw fish) wrapped in seaweed', 'name': 'sushi'}, {'frequency': 'c', 'id': 1059, 'synset': 'swab.n.02', 'synonyms': ['mop'], 'def': 'cleaning implement consisting of absorbent material fastened to a handle; for cleaning floors', 'name': 'mop'}, {'frequency': 'c', 'id': 1060, 'synset': 'sweat_pants.n.01', 'synonyms': ['sweat_pants'], 'def': 'loose-fitting trousers with elastic cuffs; worn by athletes', 'name': 'sweat_pants'}, {'frequency': 'c', 'id': 1061, 'synset': 'sweatband.n.02', 'synonyms': ['sweatband'], 'def': 'a band of material tied around the forehead or wrist to absorb sweat', 'name': 'sweatband'}, {'frequency': 'f', 'id': 1062, 'synset': 'sweater.n.01', 'synonyms': ['sweater'], 'def': 'a crocheted or knitted garment covering the upper part of the body', 'name': 'sweater'}, {'frequency': 'f', 'id': 1063, 'synset': 'sweatshirt.n.01', 'synonyms': ['sweatshirt'], 'def': 'cotton knit pullover with long sleeves worn during athletic activity', 'name': 'sweatshirt'}, {'frequency': 'c', 'id': 1064, 'synset': 'sweet_potato.n.02', 'synonyms': ['sweet_potato'], 'def': 'the edible tuberous root of the sweet potato vine', 'name': 'sweet_potato'}, {'frequency': 'f', 'id': 1065, 'synset': 'swimsuit.n.01', 'synonyms': ['swimsuit', 'swimwear', 'bathing_suit', 'swimming_costume', 'bathing_costume', 'swimming_trunks', 'bathing_trunks'], 'def': 'garment worn for swimming', 'name': 'swimsuit'}, {'frequency': 'c', 'id': 1066, 'synset': 'sword.n.01', 'synonyms': ['sword'], 'def': 'a cutting or thrusting weapon that has a long metal blade', 'name': 'sword'}, {'frequency': 'r', 'id': 1067, 'synset': 'syringe.n.01', 'synonyms': ['syringe'], 'def': 'a medical instrument used to inject or withdraw fluids', 'name': 'syringe'}, {'frequency': 'r', 'id': 1068, 'synset': 'tabasco.n.02', 'synonyms': ['Tabasco_sauce'], 'def': 'very spicy sauce (trade name Tabasco) made from fully-aged red peppers', 'name': 'Tabasco_sauce'}, {'frequency': 'r', 'id': 1069, 'synset': 'table-tennis_table.n.01', 'synonyms': ['table-tennis_table', 'ping-pong_table'], 'def': 'a table used for playing table tennis', 'name': 'table-tennis_table'}, {'frequency': 'f', 'id': 1070, 'synset': 'table.n.02', 'synonyms': ['table'], 'def': 'a piece of furniture having a smooth flat top that is usually supported by one or more vertical legs', 'name': 'table'}, {'frequency': 'c', 'id': 1071, 'synset': 'table_lamp.n.01', 'synonyms': ['table_lamp'], 'def': 'a lamp that sits on a table', 'name': 'table_lamp'}, {'frequency': 'f', 'id': 1072, 'synset': 'tablecloth.n.01', 'synonyms': ['tablecloth'], 'def': 'a covering spread over a dining table', 'name': 'tablecloth'}, {'frequency': 'r', 'id': 1073, 'synset': 'tachometer.n.01', 'synonyms': ['tachometer'], 'def': 'measuring instrument for indicating speed of rotation', 'name': 'tachometer'}, {'frequency': 'r', 'id': 1074, 'synset': 'taco.n.02', 'synonyms': ['taco'], 'def': 'a small tortilla cupped around a filling', 'name': 'taco'}, {'frequency': 'f', 'id': 1075, 'synset': 'tag.n.02', 'synonyms': ['tag'], 'def': 'a label associated with something for the purpose of identification or information', 'name': 'tag'}, {'frequency': 'f', 'id': 1076, 'synset': 'taillight.n.01', 'synonyms': ['taillight', 'rear_light'], 'def': 'lamp (usually red) mounted at the rear of a motor vehicle', 'name': 'taillight'}, {'frequency': 'r', 'id': 1077, 'synset': 'tambourine.n.01', 'synonyms': ['tambourine'], 'def': 'a shallow drum with a single drumhead and with metallic disks in the sides', 'name': 'tambourine'}, {'frequency': 'r', 'id': 1078, 'synset': 'tank.n.01', 'synonyms': ['army_tank', 'armored_combat_vehicle', 'armoured_combat_vehicle'], 'def': 'an enclosed armored military vehicle; has a cannon and moves on caterpillar treads', 'name': 'army_tank'}, {'frequency': 'c', 'id': 1079, 'synset': 'tank.n.02', 'synonyms': ['tank_(storage_vessel)', 'storage_tank'], 'def': 'a large (usually metallic) vessel for holding gases or liquids', 'name': 'tank_(storage_vessel)'}, {'frequency': 'f', 'id': 1080, 'synset': 'tank_top.n.01', 'synonyms': ['tank_top_(clothing)'], 'def': 'a tight-fitting sleeveless shirt with wide shoulder straps and low neck and no front opening', 'name': 'tank_top_(clothing)'}, {'frequency': 'c', 'id': 1081, 'synset': 'tape.n.01', 'synonyms': ['tape_(sticky_cloth_or_paper)'], 'def': 'a long thin piece of cloth or paper as used for binding or fastening', 'name': 'tape_(sticky_cloth_or_paper)'}, {'frequency': 'c', 'id': 1082, 'synset': 'tape.n.04', 'synonyms': ['tape_measure', 'measuring_tape'], 'def': 'measuring instrument consisting of a narrow strip (cloth or metal) marked in inches or centimeters and used for measuring lengths', 'name': 'tape_measure'}, {'frequency': 'c', 'id': 1083, 'synset': 'tapestry.n.02', 'synonyms': ['tapestry'], 'def': 'a heavy textile with a woven design; used for curtains and upholstery', 'name': 'tapestry'}, {'frequency': 'f', 'id': 1084, 'synset': 'tarpaulin.n.01', 'synonyms': ['tarp'], 'def': 'waterproofed canvas', 'name': 'tarp'}, {'frequency': 'c', 'id': 1085, 'synset': 'tartan.n.01', 'synonyms': ['tartan', 'plaid'], 'def': 'a cloth having a crisscross design', 'name': 'tartan'}, {'frequency': 'c', 'id': 1086, 'synset': 'tassel.n.01', 'synonyms': ['tassel'], 'def': 'adornment consisting of a bunch of cords fastened at one end', 'name': 'tassel'}, {'frequency': 'r', 'id': 1087, 'synset': 'tea_bag.n.01', 'synonyms': ['tea_bag'], 'def': 'a measured amount of tea in a bag for an individual serving of tea', 'name': 'tea_bag'}, {'frequency': 'c', 'id': 1088, 'synset': 'teacup.n.02', 'synonyms': ['teacup'], 'def': 'a cup from which tea is drunk', 'name': 'teacup'}, {'frequency': 'c', 'id': 1089, 'synset': 'teakettle.n.01', 'synonyms': ['teakettle'], 'def': 'kettle for boiling water to make tea', 'name': 'teakettle'}, {'frequency': 'c', 'id': 1090, 'synset': 'teapot.n.01', 'synonyms': ['teapot'], 'def': 'pot for brewing tea; usually has a spout and handle', 'name': 'teapot'}, {'frequency': 'f', 'id': 1091, 'synset': 'teddy.n.01', 'synonyms': ['teddy_bear'], 'def': "plaything consisting of a child's toy bear (usually plush and stuffed with soft materials)", 'name': 'teddy_bear'}, {'frequency': 'f', 'id': 1092, 'synset': 'telephone.n.01', 'synonyms': ['telephone', 'phone', 'telephone_set'], 'def': 'electronic device for communicating by voice over long distances', 'name': 'telephone'}, {'frequency': 'c', 'id': 1093, 'synset': 'telephone_booth.n.01', 'synonyms': ['telephone_booth', 'phone_booth', 'call_box', 'telephone_box', 'telephone_kiosk'], 'def': 'booth for using a telephone', 'name': 'telephone_booth'}, {'frequency': 'f', 'id': 1094, 'synset': 'telephone_pole.n.01', 'synonyms': ['telephone_pole', 'telegraph_pole', 'telegraph_post'], 'def': 'tall pole supporting telephone wires', 'name': 'telephone_pole'}, {'frequency': 'r', 'id': 1095, 'synset': 'telephoto_lens.n.01', 'synonyms': ['telephoto_lens', 'zoom_lens'], 'def': 'a camera lens that magnifies the image', 'name': 'telephoto_lens'}, {'frequency': 'c', 'id': 1096, 'synset': 'television_camera.n.01', 'synonyms': ['television_camera', 'tv_camera'], 'def': 'television equipment for capturing and recording video', 'name': 'television_camera'}, {'frequency': 'f', 'id': 1097, 'synset': 'television_receiver.n.01', 'synonyms': ['television_set', 'tv', 'tv_set'], 'def': 'an electronic device that receives television signals and displays them on a screen', 'name': 'television_set'}, {'frequency': 'f', 'id': 1098, 'synset': 'tennis_ball.n.01', 'synonyms': ['tennis_ball'], 'def': 'ball about the size of a fist used in playing tennis', 'name': 'tennis_ball'}, {'frequency': 'f', 'id': 1099, 'synset': 'tennis_racket.n.01', 'synonyms': ['tennis_racket'], 'def': 'a racket used to play tennis', 'name': 'tennis_racket'}, {'frequency': 'r', 'id': 1100, 'synset': 'tequila.n.01', 'synonyms': ['tequila'], 'def': 'Mexican liquor made from fermented juices of an agave plant', 'name': 'tequila'}, {'frequency': 'c', 'id': 1101, 'synset': 'thermometer.n.01', 'synonyms': ['thermometer'], 'def': 'measuring instrument for measuring temperature', 'name': 'thermometer'}, {'frequency': 'c', 'id': 1102, 'synset': 'thermos.n.01', 'synonyms': ['thermos_bottle'], 'def': 'vacuum flask that preserves temperature of hot or cold drinks', 'name': 'thermos_bottle'}, {'frequency': 'c', 'id': 1103, 'synset': 'thermostat.n.01', 'synonyms': ['thermostat'], 'def': 'a regulator for automatically regulating temperature by starting or stopping the supply of heat', 'name': 'thermostat'}, {'frequency': 'r', 'id': 1104, 'synset': 'thimble.n.02', 'synonyms': ['thimble'], 'def': 'a small metal cap to protect the finger while sewing; can be used as a small container', 'name': 'thimble'}, {'frequency': 'c', 'id': 1105, 'synset': 'thread.n.01', 'synonyms': ['thread', 'yarn'], 'def': 'a fine cord of twisted fibers (of cotton or silk or wool or nylon etc.) used in sewing and weaving', 'name': 'thread'}, {'frequency': 'c', 'id': 1106, 'synset': 'thumbtack.n.01', 'synonyms': ['thumbtack', 'drawing_pin', 'pushpin'], 'def': 'a tack for attaching papers to a bulletin board or drawing board', 'name': 'thumbtack'}, {'frequency': 'c', 'id': 1107, 'synset': 'tiara.n.01', 'synonyms': ['tiara'], 'def': 'a jeweled headdress worn by women on formal occasions', 'name': 'tiara'}, {'frequency': 'c', 'id': 1108, 'synset': 'tiger.n.02', 'synonyms': ['tiger'], 'def': 'large feline of forests in most of Asia having a tawny coat with black stripes', 'name': 'tiger'}, {'frequency': 'c', 'id': 1109, 'synset': 'tights.n.01', 'synonyms': ['tights_(clothing)', 'leotards'], 'def': 'skintight knit hose covering the body from the waist to the feet worn by acrobats and dancers and as stockings by women and girls', 'name': 'tights_(clothing)'}, {'frequency': 'c', 'id': 1110, 'synset': 'timer.n.01', 'synonyms': ['timer', 'stopwatch'], 'def': 'a timepiece that measures a time interval and signals its end', 'name': 'timer'}, {'frequency': 'f', 'id': 1111, 'synset': 'tinfoil.n.01', 'synonyms': ['tinfoil'], 'def': 'foil made of tin or an alloy of tin and lead', 'name': 'tinfoil'}, {'frequency': 'r', 'id': 1112, 'synset': 'tinsel.n.01', 'synonyms': ['tinsel'], 'def': 'a showy decoration that is basically valueless', 'name': 'tinsel'}, {'frequency': 'f', 'id': 1113, 'synset': 'tissue.n.02', 'synonyms': ['tissue_paper'], 'def': 'a soft thin (usually translucent) paper', 'name': 'tissue_paper'}, {'frequency': 'c', 'id': 1114, 'synset': 'toast.n.01', 'synonyms': ['toast_(food)'], 'def': 'slice of bread that has been toasted', 'name': 'toast_(food)'}, {'frequency': 'f', 'id': 1115, 'synset': 'toaster.n.02', 'synonyms': ['toaster'], 'def': 'a kitchen appliance (usually electric) for toasting bread', 'name': 'toaster'}, {'frequency': 'c', 'id': 1116, 'synset': 'toaster_oven.n.01', 'synonyms': ['toaster_oven'], 'def': 'kitchen appliance consisting of a small electric oven for toasting or warming food', 'name': 'toaster_oven'}, {'frequency': 'f', 'id': 1117, 'synset': 'toilet.n.02', 'synonyms': ['toilet'], 'def': 'a plumbing fixture for defecation and urination', 'name': 'toilet'}, {'frequency': 'f', 'id': 1118, 'synset': 'toilet_tissue.n.01', 'synonyms': ['toilet_tissue', 'toilet_paper', 'bathroom_tissue'], 'def': 'a soft thin absorbent paper for use in toilets', 'name': 'toilet_tissue'}, {'frequency': 'f', 'id': 1119, 'synset': 'tomato.n.01', 'synonyms': ['tomato'], 'def': 'mildly acid red or yellow pulpy fruit eaten as a vegetable', 'name': 'tomato'}, {'frequency': 'c', 'id': 1120, 'synset': 'tongs.n.01', 'synonyms': ['tongs'], 'def': 'any of various devices for taking hold of objects; usually have two hinged legs with handles above and pointed hooks below', 'name': 'tongs'}, {'frequency': 'c', 'id': 1121, 'synset': 'toolbox.n.01', 'synonyms': ['toolbox'], 'def': 'a box or chest or cabinet for holding hand tools', 'name': 'toolbox'}, {'frequency': 'f', 'id': 1122, 'synset': 'toothbrush.n.01', 'synonyms': ['toothbrush'], 'def': 'small brush; has long handle; used to clean teeth', 'name': 'toothbrush'}, {'frequency': 'f', 'id': 1123, 'synset': 'toothpaste.n.01', 'synonyms': ['toothpaste'], 'def': 'a dentifrice in the form of a paste', 'name': 'toothpaste'}, {'frequency': 'c', 'id': 1124, 'synset': 'toothpick.n.01', 'synonyms': ['toothpick'], 'def': 'pick consisting of a small strip of wood or plastic; used to pick food from between the teeth', 'name': 'toothpick'}, {'frequency': 'c', 'id': 1125, 'synset': 'top.n.09', 'synonyms': ['cover'], 'def': 'covering for a hole (especially a hole in the top of a container)', 'name': 'cover'}, {'frequency': 'c', 'id': 1126, 'synset': 'tortilla.n.01', 'synonyms': ['tortilla'], 'def': 'thin unleavened pancake made from cornmeal or wheat flour', 'name': 'tortilla'}, {'frequency': 'c', 'id': 1127, 'synset': 'tow_truck.n.01', 'synonyms': ['tow_truck'], 'def': 'a truck equipped to hoist and pull wrecked cars (or to remove cars from no-parking zones)', 'name': 'tow_truck'}, {'frequency': 'f', 'id': 1128, 'synset': 'towel.n.01', 'synonyms': ['towel'], 'def': 'a rectangular piece of absorbent cloth (or paper) for drying or wiping', 'name': 'towel'}, {'frequency': 'f', 'id': 1129, 'synset': 'towel_rack.n.01', 'synonyms': ['towel_rack', 'towel_rail', 'towel_bar'], 'def': 'a rack consisting of one or more bars on which towels can be hung', 'name': 'towel_rack'}, {'frequency': 'f', 'id': 1130, 'synset': 'toy.n.03', 'synonyms': ['toy'], 'def': 'a device regarded as providing amusement', 'name': 'toy'}, {'frequency': 'c', 'id': 1131, 'synset': 'tractor.n.01', 'synonyms': ['tractor_(farm_equipment)'], 'def': 'a wheeled vehicle with large wheels; used in farming and other applications', 'name': 'tractor_(farm_equipment)'}, {'frequency': 'f', 'id': 1132, 'synset': 'traffic_light.n.01', 'synonyms': ['traffic_light'], 'def': 'a device to control vehicle traffic often consisting of three or more lights', 'name': 'traffic_light'}, {'frequency': 'r', 'id': 1133, 'synset': 'trail_bike.n.01', 'synonyms': ['dirt_bike'], 'def': 'a lightweight motorcycle equipped with rugged tires and suspension for off-road use', 'name': 'dirt_bike'}, {'frequency': 'c', 'id': 1134, 'synset': 'trailer_truck.n.01', 'synonyms': ['trailer_truck', 'tractor_trailer', 'trucking_rig', 'articulated_lorry', 'semi_truck'], 'def': 'a truck consisting of a tractor and trailer together', 'name': 'trailer_truck'}, {'frequency': 'f', 'id': 1135, 'synset': 'train.n.01', 'synonyms': ['train_(railroad_vehicle)', 'railroad_train'], 'def': 'public or private transport provided by a line of railway cars coupled together and drawn by a locomotive', 'name': 'train_(railroad_vehicle)'}, {'frequency': 'r', 'id': 1136, 'synset': 'trampoline.n.01', 'synonyms': ['trampoline'], 'def': 'gymnastic apparatus consisting of a strong canvas sheet attached with springs to a metal frame', 'name': 'trampoline'}, {'frequency': 'f', 'id': 1137, 'synset': 'tray.n.01', 'synonyms': ['tray'], 'def': 'an open receptacle for holding or displaying or serving articles or food', 'name': 'tray'}, {'frequency': 'r', 'id': 1138, 'synset': 'tree_house.n.01', 'synonyms': ['tree_house'], 'def': '(NOT A TREE) a PLAYHOUSE built in the branches of a tree', 'name': 'tree_house'}, {'frequency': 'r', 'id': 1139, 'synset': 'trench_coat.n.01', 'synonyms': ['trench_coat'], 'def': 'a military style raincoat; belted with deep pockets', 'name': 'trench_coat'}, {'frequency': 'r', 'id': 1140, 'synset': 'triangle.n.05', 'synonyms': ['triangle_(musical_instrument)'], 'def': 'a percussion instrument consisting of a metal bar bent in the shape of an open triangle', 'name': 'triangle_(musical_instrument)'}, {'frequency': 'r', 'id': 1141, 'synset': 'tricycle.n.01', 'synonyms': ['tricycle'], 'def': 'a vehicle with three wheels that is moved by foot pedals', 'name': 'tricycle'}, {'frequency': 'c', 'id': 1142, 'synset': 'tripod.n.01', 'synonyms': ['tripod'], 'def': 'a three-legged rack used for support', 'name': 'tripod'}, {'frequency': 'f', 'id': 1143, 'synset': 'trouser.n.01', 'synonyms': ['trousers', 'pants_(clothing)'], 'def': 'a garment extending from the waist to the knee or ankle, covering each leg separately', 'name': 'trousers'}, {'frequency': 'f', 'id': 1144, 'synset': 'truck.n.01', 'synonyms': ['truck'], 'def': 'an automotive vehicle suitable for hauling', 'name': 'truck'}, {'frequency': 'r', 'id': 1145, 'synset': 'truffle.n.03', 'synonyms': ['truffle_(chocolate)', 'chocolate_truffle'], 'def': 'creamy chocolate candy', 'name': 'truffle_(chocolate)'}, {'frequency': 'c', 'id': 1146, 'synset': 'trunk.n.02', 'synonyms': ['trunk'], 'def': 'luggage consisting of a large strong case used when traveling or for storage', 'name': 'trunk'}, {'frequency': 'r', 'id': 1147, 'synset': 'tub.n.02', 'synonyms': ['vat'], 'def': 'a large open vessel for holding or storing liquids', 'name': 'vat'}, {'frequency': 'c', 'id': 1148, 'synset': 'turban.n.01', 'synonyms': ['turban'], 'def': 'a traditional headdress consisting of a long scarf wrapped around the head', 'name': 'turban'}, {'frequency': 'r', 'id': 1149, 'synset': 'turkey.n.01', 'synonyms': ['turkey_(bird)'], 'def': 'large gallinaceous bird with fan-shaped tail; widely domesticated for food', 'name': 'turkey_(bird)'}, {'frequency': 'c', 'id': 1150, 'synset': 'turkey.n.04', 'synonyms': ['turkey_(food)'], 'def': 'flesh of large domesticated fowl usually roasted', 'name': 'turkey_(food)'}, {'frequency': 'r', 'id': 1151, 'synset': 'turnip.n.01', 'synonyms': ['turnip'], 'def': 'widely cultivated plant having a large fleshy edible white or yellow root', 'name': 'turnip'}, {'frequency': 'c', 'id': 1152, 'synset': 'turtle.n.02', 'synonyms': ['turtle'], 'def': 'any of various aquatic and land reptiles having a bony shell and flipper-like limbs for swimming', 'name': 'turtle'}, {'frequency': 'r', 'id': 1153, 'synset': 'turtleneck.n.01', 'synonyms': ['turtleneck_(clothing)', 'polo-neck'], 'def': 'a sweater or jersey with a high close-fitting collar', 'name': 'turtleneck_(clothing)'}, {'frequency': 'r', 'id': 1154, 'synset': 'typewriter.n.01', 'synonyms': ['typewriter'], 'def': 'hand-operated character printer for printing written messages one character at a time', 'name': 'typewriter'}, {'frequency': 'f', 'id': 1155, 'synset': 'umbrella.n.01', 'synonyms': ['umbrella'], 'def': 'a lightweight handheld collapsible canopy', 'name': 'umbrella'}, {'frequency': 'c', 'id': 1156, 'synset': 'underwear.n.01', 'synonyms': ['underwear', 'underclothes', 'underclothing', 'underpants'], 'def': 'undergarment worn next to the skin and under the outer garments', 'name': 'underwear'}, {'frequency': 'r', 'id': 1157, 'synset': 'unicycle.n.01', 'synonyms': ['unicycle'], 'def': 'a vehicle with a single wheel that is driven by pedals', 'name': 'unicycle'}, {'frequency': 'c', 'id': 1158, 'synset': 'urinal.n.01', 'synonyms': ['urinal'], 'def': 'a plumbing fixture (usually attached to the wall) used by men to urinate', 'name': 'urinal'}, {'frequency': 'r', 'id': 1159, 'synset': 'urn.n.01', 'synonyms': ['urn'], 'def': 'a large vase that usually has a pedestal or feet', 'name': 'urn'}, {'frequency': 'c', 'id': 1160, 'synset': 'vacuum.n.04', 'synonyms': ['vacuum_cleaner'], 'def': 'an electrical home appliance that cleans by suction', 'name': 'vacuum_cleaner'}, {'frequency': 'c', 'id': 1161, 'synset': 'valve.n.03', 'synonyms': ['valve'], 'def': 'control consisting of a mechanical device for controlling the flow of a fluid', 'name': 'valve'}, {'frequency': 'f', 'id': 1162, 'synset': 'vase.n.01', 'synonyms': ['vase'], 'def': 'an open jar of glass or porcelain used as an ornament or to hold flowers', 'name': 'vase'}, {'frequency': 'c', 'id': 1163, 'synset': 'vending_machine.n.01', 'synonyms': ['vending_machine'], 'def': 'a slot machine for selling goods', 'name': 'vending_machine'}, {'frequency': 'f', 'id': 1164, 'synset': 'vent.n.01', 'synonyms': ['vent', 'blowhole', 'air_vent'], 'def': 'a hole for the escape of gas or air', 'name': 'vent'}, {'frequency': 'c', 'id': 1165, 'synset': 'videotape.n.01', 'synonyms': ['videotape'], 'def': 'a video recording made on magnetic tape', 'name': 'videotape'}, {'frequency': 'r', 'id': 1166, 'synset': 'vinegar.n.01', 'synonyms': ['vinegar'], 'def': 'sour-tasting liquid produced usually by oxidation of the alcohol in wine or cider and used as a condiment or food preservative', 'name': 'vinegar'}, {'frequency': 'r', 'id': 1167, 'synset': 'violin.n.01', 'synonyms': ['violin', 'fiddle'], 'def': 'bowed stringed instrument that is the highest member of the violin family', 'name': 'violin'}, {'frequency': 'r', 'id': 1168, 'synset': 'vodka.n.01', 'synonyms': ['vodka'], 'def': 'unaged colorless liquor originating in Russia', 'name': 'vodka'}, {'frequency': 'r', 'id': 1169, 'synset': 'volleyball.n.02', 'synonyms': ['volleyball'], 'def': 'an inflated ball used in playing volleyball', 'name': 'volleyball'}, {'frequency': 'r', 'id': 1170, 'synset': 'vulture.n.01', 'synonyms': ['vulture'], 'def': 'any of various large birds of prey having naked heads and weak claws and feeding chiefly on carrion', 'name': 'vulture'}, {'frequency': 'c', 'id': 1171, 'synset': 'waffle.n.01', 'synonyms': ['waffle'], 'def': 'pancake batter baked in a waffle iron', 'name': 'waffle'}, {'frequency': 'r', 'id': 1172, 'synset': 'waffle_iron.n.01', 'synonyms': ['waffle_iron'], 'def': 'a kitchen appliance for baking waffles', 'name': 'waffle_iron'}, {'frequency': 'c', 'id': 1173, 'synset': 'wagon.n.01', 'synonyms': ['wagon'], 'def': 'any of various kinds of wheeled vehicles drawn by an animal or a tractor', 'name': 'wagon'}, {'frequency': 'c', 'id': 1174, 'synset': 'wagon_wheel.n.01', 'synonyms': ['wagon_wheel'], 'def': 'a wheel of a wagon', 'name': 'wagon_wheel'}, {'frequency': 'c', 'id': 1175, 'synset': 'walking_stick.n.01', 'synonyms': ['walking_stick'], 'def': 'a stick carried in the hand for support in walking', 'name': 'walking_stick'}, {'frequency': 'c', 'id': 1176, 'synset': 'wall_clock.n.01', 'synonyms': ['wall_clock'], 'def': 'a clock mounted on a wall', 'name': 'wall_clock'}, {'frequency': 'f', 'id': 1177, 'synset': 'wall_socket.n.01', 'synonyms': ['wall_socket', 'wall_plug', 'electric_outlet', 'electrical_outlet', 'outlet', 'electric_receptacle'], 'def': 'receptacle providing a place in a wiring system where current can be taken to run electrical devices', 'name': 'wall_socket'}, {'frequency': 'c', 'id': 1178, 'synset': 'wallet.n.01', 'synonyms': ['wallet', 'billfold'], 'def': 'a pocket-size case for holding papers and paper money', 'name': 'wallet'}, {'frequency': 'r', 'id': 1179, 'synset': 'walrus.n.01', 'synonyms': ['walrus'], 'def': 'either of two large northern marine mammals having ivory tusks and tough hide over thick blubber', 'name': 'walrus'}, {'frequency': 'r', 'id': 1180, 'synset': 'wardrobe.n.01', 'synonyms': ['wardrobe'], 'def': 'a tall piece of furniture that provides storage space for clothes; has a door and rails or hooks for hanging clothes', 'name': 'wardrobe'}, {'frequency': 'r', 'id': 1181, 'synset': 'wasabi.n.02', 'synonyms': ['wasabi'], 'def': 'the thick green root of the wasabi plant that the Japanese use in cooking and that tastes like strong horseradish', 'name': 'wasabi'}, {'frequency': 'c', 'id': 1182, 'synset': 'washer.n.03', 'synonyms': ['automatic_washer', 'washing_machine'], 'def': 'a home appliance for washing clothes and linens automatically', 'name': 'automatic_washer'}, {'frequency': 'f', 'id': 1183, 'synset': 'watch.n.01', 'synonyms': ['watch', 'wristwatch'], 'def': 'a small, portable timepiece', 'name': 'watch'}, {'frequency': 'f', 'id': 1184, 'synset': 'water_bottle.n.01', 'synonyms': ['water_bottle'], 'def': 'a bottle for holding water', 'name': 'water_bottle'}, {'frequency': 'c', 'id': 1185, 'synset': 'water_cooler.n.01', 'synonyms': ['water_cooler'], 'def': 'a device for cooling and dispensing drinking water', 'name': 'water_cooler'}, {'frequency': 'c', 'id': 1186, 'synset': 'water_faucet.n.01', 'synonyms': ['water_faucet', 'water_tap', 'tap_(water_faucet)'], 'def': 'a faucet for drawing water from a pipe or cask', 'name': 'water_faucet'}, {'frequency': 'r', 'id': 1187, 'synset': 'water_filter.n.01', 'synonyms': ['water_filter'], 'def': 'a filter to remove impurities from the water supply', 'name': 'water_filter'}, {'frequency': 'r', 'id': 1188, 'synset': 'water_heater.n.01', 'synonyms': ['water_heater', 'hot-water_heater'], 'def': 'a heater and storage tank to supply heated water', 'name': 'water_heater'}, {'frequency': 'r', 'id': 1189, 'synset': 'water_jug.n.01', 'synonyms': ['water_jug'], 'def': 'a jug that holds water', 'name': 'water_jug'}, {'frequency': 'r', 'id': 1190, 'synset': 'water_pistol.n.01', 'synonyms': ['water_gun', 'squirt_gun'], 'def': 'plaything consisting of a toy pistol that squirts water', 'name': 'water_gun'}, {'frequency': 'c', 'id': 1191, 'synset': 'water_scooter.n.01', 'synonyms': ['water_scooter', 'sea_scooter', 'jet_ski'], 'def': 'a motorboat resembling a motor scooter (NOT A SURFBOARD OR WATER SKI)', 'name': 'water_scooter'}, {'frequency': 'c', 'id': 1192, 'synset': 'water_ski.n.01', 'synonyms': ['water_ski'], 'def': 'broad ski for skimming over water towed by a speedboat (DO NOT MARK WATER)', 'name': 'water_ski'}, {'frequency': 'c', 'id': 1193, 'synset': 'water_tower.n.01', 'synonyms': ['water_tower'], 'def': 'a large reservoir for water', 'name': 'water_tower'}, {'frequency': 'c', 'id': 1194, 'synset': 'watering_can.n.01', 'synonyms': ['watering_can'], 'def': 'a container with a handle and a spout with a perforated nozzle; used to sprinkle water over plants', 'name': 'watering_can'}, {'frequency': 'c', 'id': 1195, 'synset': 'watermelon.n.02', 'synonyms': ['watermelon'], 'def': 'large oblong or roundish melon with a hard green rind and sweet watery red or occasionally yellowish pulp', 'name': 'watermelon'}, {'frequency': 'f', 'id': 1196, 'synset': 'weathervane.n.01', 'synonyms': ['weathervane', 'vane_(weathervane)', 'wind_vane'], 'def': 'mechanical device attached to an elevated structure; rotates freely to show the direction of the wind', 'name': 'weathervane'}, {'frequency': 'c', 'id': 1197, 'synset': 'webcam.n.01', 'synonyms': ['webcam'], 'def': 'a digital camera designed to take digital photographs and transmit them over the internet', 'name': 'webcam'}, {'frequency': 'c', 'id': 1198, 'synset': 'wedding_cake.n.01', 'synonyms': ['wedding_cake', 'bridecake'], 'def': 'a rich cake with two or more tiers and covered with frosting and decorations; served at a wedding reception', 'name': 'wedding_cake'}, {'frequency': 'c', 'id': 1199, 'synset': 'wedding_ring.n.01', 'synonyms': ['wedding_ring', 'wedding_band'], 'def': 'a ring given to the bride and/or groom at the wedding', 'name': 'wedding_ring'}, {'frequency': 'f', 'id': 1200, 'synset': 'wet_suit.n.01', 'synonyms': ['wet_suit'], 'def': 'a close-fitting garment made of a permeable material; worn in cold water to retain body heat', 'name': 'wet_suit'}, {'frequency': 'f', 'id': 1201, 'synset': 'wheel.n.01', 'synonyms': ['wheel'], 'def': 'a circular frame with spokes (or a solid disc) that can rotate on a shaft or axle', 'name': 'wheel'}, {'frequency': 'c', 'id': 1202, 'synset': 'wheelchair.n.01', 'synonyms': ['wheelchair'], 'def': 'a movable chair mounted on large wheels', 'name': 'wheelchair'}, {'frequency': 'c', 'id': 1203, 'synset': 'whipped_cream.n.01', 'synonyms': ['whipped_cream'], 'def': 'cream that has been beaten until light and fluffy', 'name': 'whipped_cream'}, {'frequency': 'r', 'id': 1204, 'synset': 'whiskey.n.01', 'synonyms': ['whiskey'], 'def': 'a liquor made from fermented mash of grain', 'name': 'whiskey'}, {'frequency': 'r', 'id': 1205, 'synset': 'whistle.n.03', 'synonyms': ['whistle'], 'def': 'a small wind instrument that produces a whistling sound by blowing into it', 'name': 'whistle'}, {'frequency': 'r', 'id': 1206, 'synset': 'wick.n.02', 'synonyms': ['wick'], 'def': 'a loosely woven cord in a candle or oil lamp that is lit on fire', 'name': 'wick'}, {'frequency': 'c', 'id': 1207, 'synset': 'wig.n.01', 'synonyms': ['wig'], 'def': 'hairpiece covering the head and made of real or synthetic hair', 'name': 'wig'}, {'frequency': 'c', 'id': 1208, 'synset': 'wind_chime.n.01', 'synonyms': ['wind_chime'], 'def': 'a decorative arrangement of pieces of metal or glass or pottery that hang together loosely so the wind can cause them to tinkle', 'name': 'wind_chime'}, {'frequency': 'c', 'id': 1209, 'synset': 'windmill.n.01', 'synonyms': ['windmill'], 'def': 'a mill that is powered by the wind', 'name': 'windmill'}, {'frequency': 'c', 'id': 1210, 'synset': 'window_box.n.01', 'synonyms': ['window_box_(for_plants)'], 'def': 'a container for growing plants on a windowsill', 'name': 'window_box_(for_plants)'}, {'frequency': 'f', 'id': 1211, 'synset': 'windshield_wiper.n.01', 'synonyms': ['windshield_wiper', 'windscreen_wiper', 'wiper_(for_windshield/screen)'], 'def': 'a mechanical device that cleans the windshield', 'name': 'windshield_wiper'}, {'frequency': 'c', 'id': 1212, 'synset': 'windsock.n.01', 'synonyms': ['windsock', 'air_sock', 'air-sleeve', 'wind_sleeve', 'wind_cone'], 'def': 'a truncated cloth cone mounted on a mast/pole; shows wind direction', 'name': 'windsock'}, {'frequency': 'f', 'id': 1213, 'synset': 'wine_bottle.n.01', 'synonyms': ['wine_bottle'], 'def': 'a bottle for holding wine', 'name': 'wine_bottle'}, {'frequency': 'r', 'id': 1214, 'synset': 'wine_bucket.n.01', 'synonyms': ['wine_bucket', 'wine_cooler'], 'def': 'a bucket of ice used to chill a bottle of wine', 'name': 'wine_bucket'}, {'frequency': 'f', 'id': 1215, 'synset': 'wineglass.n.01', 'synonyms': ['wineglass'], 'def': 'a glass that has a stem and in which wine is served', 'name': 'wineglass'}, {'frequency': 'r', 'id': 1216, 'synset': 'wing_chair.n.01', 'synonyms': ['wing_chair'], 'def': 'easy chair having wings on each side of a high back', 'name': 'wing_chair'}, {'frequency': 'c', 'id': 1217, 'synset': 'winker.n.02', 'synonyms': ['blinder_(for_horses)'], 'def': 'blinds that prevent a horse from seeing something on either side', 'name': 'blinder_(for_horses)'}, {'frequency': 'c', 'id': 1218, 'synset': 'wok.n.01', 'synonyms': ['wok'], 'def': 'pan with a convex bottom; used for frying in Chinese cooking', 'name': 'wok'}, {'frequency': 'r', 'id': 1219, 'synset': 'wolf.n.01', 'synonyms': ['wolf'], 'def': 'a wild carnivorous mammal of the dog family, living and hunting in packs', 'name': 'wolf'}, {'frequency': 'c', 'id': 1220, 'synset': 'wooden_spoon.n.02', 'synonyms': ['wooden_spoon'], 'def': 'a spoon made of wood', 'name': 'wooden_spoon'}, {'frequency': 'c', 'id': 1221, 'synset': 'wreath.n.01', 'synonyms': ['wreath'], 'def': 'an arrangement of flowers, leaves, or stems fastened in a ring', 'name': 'wreath'}, {'frequency': 'c', 'id': 1222, 'synset': 'wrench.n.03', 'synonyms': ['wrench', 'spanner'], 'def': 'a hand tool that is used to hold or twist a nut or bolt', 'name': 'wrench'}, {'frequency': 'c', 'id': 1223, 'synset': 'wristband.n.01', 'synonyms': ['wristband'], 'def': 'band consisting of a part of a sleeve that covers the wrist', 'name': 'wristband'}, {'frequency': 'f', 'id': 1224, 'synset': 'wristlet.n.01', 'synonyms': ['wristlet', 'wrist_band'], 'def': 'a band or bracelet worn around the wrist', 'name': 'wristlet'}, {'frequency': 'r', 'id': 1225, 'synset': 'yacht.n.01', 'synonyms': ['yacht'], 'def': 'an expensive vessel propelled by sail or power and used for cruising or racing', 'name': 'yacht'}, {'frequency': 'r', 'id': 1226, 'synset': 'yak.n.02', 'synonyms': ['yak'], 'def': 'large long-haired wild ox of Tibet often domesticated', 'name': 'yak'}, {'frequency': 'c', 'id': 1227, 'synset': 'yogurt.n.01', 'synonyms': ['yogurt', 'yoghurt', 'yoghourt'], 'def': 'a custard-like food made from curdled milk', 'name': 'yogurt'}, {'frequency': 'r', 'id': 1228, 'synset': 'yoke.n.07', 'synonyms': ['yoke_(animal_equipment)'], 'def': 'gear joining two animals at the neck; NOT egg yolk', 'name': 'yoke_(animal_equipment)'}, {'frequency': 'f', 'id': 1229, 'synset': 'zebra.n.01', 'synonyms': ['zebra'], 'def': 'any of several fleet black-and-white striped African equines', 'name': 'zebra'}, {'frequency': 'c', 'id': 1230, 'synset': 'zucchini.n.02', 'synonyms': ['zucchini', 'courgette'], 'def': 'small cucumber-shaped vegetable marrow; typically dark green', 'name': 'zucchini'}] # noqa +# fmt: on diff --git a/annotator/oneformer/detectron2/data/datasets/lvis_v1_categories.py b/annotator/oneformer/detectron2/data/datasets/lvis_v1_categories.py new file mode 100644 index 0000000000000000000000000000000000000000..7374e6968bb006f5d8c49e75d9d3b31ea3d77d05 --- /dev/null +++ b/annotator/oneformer/detectron2/data/datasets/lvis_v1_categories.py @@ -0,0 +1,16 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# Autogen with +# with open("lvis_v1_val.json", "r") as f: +# a = json.load(f) +# c = a["categories"] +# for x in c: +# del x["image_count"] +# del x["instance_count"] +# LVIS_CATEGORIES = repr(c) + " # noqa" +# with open("/tmp/lvis_categories.py", "wt") as f: +# f.write(f"LVIS_CATEGORIES = {LVIS_CATEGORIES}") +# Then paste the contents of that file below + +# fmt: off +LVIS_CATEGORIES = [{'frequency': 'c', 'synset': 'aerosol.n.02', 'synonyms': ['aerosol_can', 'spray_can'], 'id': 1, 'def': 'a dispenser that holds a substance under pressure', 'name': 'aerosol_can'}, {'frequency': 'f', 'synset': 'air_conditioner.n.01', 'synonyms': ['air_conditioner'], 'id': 2, 'def': 'a machine that keeps air cool and dry', 'name': 'air_conditioner'}, {'frequency': 'f', 'synset': 'airplane.n.01', 'synonyms': ['airplane', 'aeroplane'], 'id': 3, 'def': 'an aircraft that has a fixed wing and is powered by propellers or jets', 'name': 'airplane'}, {'frequency': 'f', 'synset': 'alarm_clock.n.01', 'synonyms': ['alarm_clock'], 'id': 4, 'def': 'a clock that wakes a sleeper at some preset time', 'name': 'alarm_clock'}, {'frequency': 'c', 'synset': 'alcohol.n.01', 'synonyms': ['alcohol', 'alcoholic_beverage'], 'id': 5, 'def': 'a liquor or brew containing alcohol as the active agent', 'name': 'alcohol'}, {'frequency': 'c', 'synset': 'alligator.n.02', 'synonyms': ['alligator', 'gator'], 'id': 6, 'def': 'amphibious reptiles related to crocodiles but with shorter broader snouts', 'name': 'alligator'}, {'frequency': 'c', 'synset': 'almond.n.02', 'synonyms': ['almond'], 'id': 7, 'def': 'oval-shaped edible seed of the almond tree', 'name': 'almond'}, {'frequency': 'c', 'synset': 'ambulance.n.01', 'synonyms': ['ambulance'], 'id': 8, 'def': 'a vehicle that takes people to and from hospitals', 'name': 'ambulance'}, {'frequency': 'c', 'synset': 'amplifier.n.01', 'synonyms': ['amplifier'], 'id': 9, 'def': 'electronic equipment that increases strength of signals', 'name': 'amplifier'}, {'frequency': 'c', 'synset': 'anklet.n.03', 'synonyms': ['anklet', 'ankle_bracelet'], 'id': 10, 'def': 'an ornament worn around the ankle', 'name': 'anklet'}, {'frequency': 'f', 'synset': 'antenna.n.01', 'synonyms': ['antenna', 'aerial', 'transmitting_aerial'], 'id': 11, 'def': 'an electrical device that sends or receives radio or television signals', 'name': 'antenna'}, {'frequency': 'f', 'synset': 'apple.n.01', 'synonyms': ['apple'], 'id': 12, 'def': 'fruit with red or yellow or green skin and sweet to tart crisp whitish flesh', 'name': 'apple'}, {'frequency': 'r', 'synset': 'applesauce.n.01', 'synonyms': ['applesauce'], 'id': 13, 'def': 'puree of stewed apples usually sweetened and spiced', 'name': 'applesauce'}, {'frequency': 'r', 'synset': 'apricot.n.02', 'synonyms': ['apricot'], 'id': 14, 'def': 'downy yellow to rosy-colored fruit resembling a small peach', 'name': 'apricot'}, {'frequency': 'f', 'synset': 'apron.n.01', 'synonyms': ['apron'], 'id': 15, 'def': 'a garment of cloth that is tied about the waist and worn to protect clothing', 'name': 'apron'}, {'frequency': 'c', 'synset': 'aquarium.n.01', 'synonyms': ['aquarium', 'fish_tank'], 'id': 16, 'def': 'a tank/pool/bowl filled with water for keeping live fish and underwater animals', 'name': 'aquarium'}, {'frequency': 'r', 'synset': 'arctic.n.02', 'synonyms': ['arctic_(type_of_shoe)', 'galosh', 'golosh', 'rubber_(type_of_shoe)', 'gumshoe'], 'id': 17, 'def': 'a waterproof overshoe that protects shoes from water or snow', 'name': 'arctic_(type_of_shoe)'}, {'frequency': 'c', 'synset': 'armband.n.02', 'synonyms': ['armband'], 'id': 18, 'def': 'a band worn around the upper arm', 'name': 'armband'}, {'frequency': 'f', 'synset': 'armchair.n.01', 'synonyms': ['armchair'], 'id': 19, 'def': 'chair with a support on each side for arms', 'name': 'armchair'}, {'frequency': 'r', 'synset': 'armoire.n.01', 'synonyms': ['armoire'], 'id': 20, 'def': 'a large wardrobe or cabinet', 'name': 'armoire'}, {'frequency': 'r', 'synset': 'armor.n.01', 'synonyms': ['armor', 'armour'], 'id': 21, 'def': 'protective covering made of metal and used in combat', 'name': 'armor'}, {'frequency': 'c', 'synset': 'artichoke.n.02', 'synonyms': ['artichoke'], 'id': 22, 'def': 'a thistlelike flower head with edible fleshy leaves and heart', 'name': 'artichoke'}, {'frequency': 'f', 'synset': 'ashcan.n.01', 'synonyms': ['trash_can', 'garbage_can', 'wastebin', 'dustbin', 'trash_barrel', 'trash_bin'], 'id': 23, 'def': 'a bin that holds rubbish until it is collected', 'name': 'trash_can'}, {'frequency': 'c', 'synset': 'ashtray.n.01', 'synonyms': ['ashtray'], 'id': 24, 'def': "a receptacle for the ash from smokers' cigars or cigarettes", 'name': 'ashtray'}, {'frequency': 'c', 'synset': 'asparagus.n.02', 'synonyms': ['asparagus'], 'id': 25, 'def': 'edible young shoots of the asparagus plant', 'name': 'asparagus'}, {'frequency': 'c', 'synset': 'atomizer.n.01', 'synonyms': ['atomizer', 'atomiser', 'spray', 'sprayer', 'nebulizer', 'nebuliser'], 'id': 26, 'def': 'a dispenser that turns a liquid (such as perfume) into a fine mist', 'name': 'atomizer'}, {'frequency': 'f', 'synset': 'avocado.n.01', 'synonyms': ['avocado'], 'id': 27, 'def': 'a pear-shaped fruit with green or blackish skin and rich yellowish pulp enclosing a single large seed', 'name': 'avocado'}, {'frequency': 'c', 'synset': 'award.n.02', 'synonyms': ['award', 'accolade'], 'id': 28, 'def': 'a tangible symbol signifying approval or distinction', 'name': 'award'}, {'frequency': 'f', 'synset': 'awning.n.01', 'synonyms': ['awning'], 'id': 29, 'def': 'a canopy made of canvas to shelter people or things from rain or sun', 'name': 'awning'}, {'frequency': 'r', 'synset': 'ax.n.01', 'synonyms': ['ax', 'axe'], 'id': 30, 'def': 'an edge tool with a heavy bladed head mounted across a handle', 'name': 'ax'}, {'frequency': 'r', 'synset': 'baboon.n.01', 'synonyms': ['baboon'], 'id': 31, 'def': 'large terrestrial monkeys having doglike muzzles', 'name': 'baboon'}, {'frequency': 'f', 'synset': 'baby_buggy.n.01', 'synonyms': ['baby_buggy', 'baby_carriage', 'perambulator', 'pram', 'stroller'], 'id': 32, 'def': 'a small vehicle with four wheels in which a baby or child is pushed around', 'name': 'baby_buggy'}, {'frequency': 'c', 'synset': 'backboard.n.01', 'synonyms': ['basketball_backboard'], 'id': 33, 'def': 'a raised vertical board with basket attached; used to play basketball', 'name': 'basketball_backboard'}, {'frequency': 'f', 'synset': 'backpack.n.01', 'synonyms': ['backpack', 'knapsack', 'packsack', 'rucksack', 'haversack'], 'id': 34, 'def': 'a bag carried by a strap on your back or shoulder', 'name': 'backpack'}, {'frequency': 'f', 'synset': 'bag.n.04', 'synonyms': ['handbag', 'purse', 'pocketbook'], 'id': 35, 'def': 'a container used for carrying money and small personal items or accessories', 'name': 'handbag'}, {'frequency': 'f', 'synset': 'bag.n.06', 'synonyms': ['suitcase', 'baggage', 'luggage'], 'id': 36, 'def': 'cases used to carry belongings when traveling', 'name': 'suitcase'}, {'frequency': 'c', 'synset': 'bagel.n.01', 'synonyms': ['bagel', 'beigel'], 'id': 37, 'def': 'glazed yeast-raised doughnut-shaped roll with hard crust', 'name': 'bagel'}, {'frequency': 'r', 'synset': 'bagpipe.n.01', 'synonyms': ['bagpipe'], 'id': 38, 'def': 'a tubular wind instrument; the player blows air into a bag and squeezes it out', 'name': 'bagpipe'}, {'frequency': 'r', 'synset': 'baguet.n.01', 'synonyms': ['baguet', 'baguette'], 'id': 39, 'def': 'narrow French stick loaf', 'name': 'baguet'}, {'frequency': 'r', 'synset': 'bait.n.02', 'synonyms': ['bait', 'lure'], 'id': 40, 'def': 'something used to lure fish or other animals into danger so they can be trapped or killed', 'name': 'bait'}, {'frequency': 'f', 'synset': 'ball.n.06', 'synonyms': ['ball'], 'id': 41, 'def': 'a spherical object used as a plaything', 'name': 'ball'}, {'frequency': 'r', 'synset': 'ballet_skirt.n.01', 'synonyms': ['ballet_skirt', 'tutu'], 'id': 42, 'def': 'very short skirt worn by ballerinas', 'name': 'ballet_skirt'}, {'frequency': 'f', 'synset': 'balloon.n.01', 'synonyms': ['balloon'], 'id': 43, 'def': 'large tough nonrigid bag filled with gas or heated air', 'name': 'balloon'}, {'frequency': 'c', 'synset': 'bamboo.n.02', 'synonyms': ['bamboo'], 'id': 44, 'def': 'woody tropical grass having hollow woody stems', 'name': 'bamboo'}, {'frequency': 'f', 'synset': 'banana.n.02', 'synonyms': ['banana'], 'id': 45, 'def': 'elongated crescent-shaped yellow fruit with soft sweet flesh', 'name': 'banana'}, {'frequency': 'c', 'synset': 'band_aid.n.01', 'synonyms': ['Band_Aid'], 'id': 46, 'def': 'trade name for an adhesive bandage to cover small cuts or blisters', 'name': 'Band_Aid'}, {'frequency': 'c', 'synset': 'bandage.n.01', 'synonyms': ['bandage'], 'id': 47, 'def': 'a piece of soft material that covers and protects an injured part of the body', 'name': 'bandage'}, {'frequency': 'f', 'synset': 'bandanna.n.01', 'synonyms': ['bandanna', 'bandana'], 'id': 48, 'def': 'large and brightly colored handkerchief; often used as a neckerchief', 'name': 'bandanna'}, {'frequency': 'r', 'synset': 'banjo.n.01', 'synonyms': ['banjo'], 'id': 49, 'def': 'a stringed instrument of the guitar family with a long neck and circular body', 'name': 'banjo'}, {'frequency': 'f', 'synset': 'banner.n.01', 'synonyms': ['banner', 'streamer'], 'id': 50, 'def': 'long strip of cloth or paper used for decoration or advertising', 'name': 'banner'}, {'frequency': 'r', 'synset': 'barbell.n.01', 'synonyms': ['barbell'], 'id': 51, 'def': 'a bar to which heavy discs are attached at each end; used in weightlifting', 'name': 'barbell'}, {'frequency': 'r', 'synset': 'barge.n.01', 'synonyms': ['barge'], 'id': 52, 'def': 'a flatbottom boat for carrying heavy loads (especially on canals)', 'name': 'barge'}, {'frequency': 'f', 'synset': 'barrel.n.02', 'synonyms': ['barrel', 'cask'], 'id': 53, 'def': 'a cylindrical container that holds liquids', 'name': 'barrel'}, {'frequency': 'c', 'synset': 'barrette.n.01', 'synonyms': ['barrette'], 'id': 54, 'def': "a pin for holding women's hair in place", 'name': 'barrette'}, {'frequency': 'c', 'synset': 'barrow.n.03', 'synonyms': ['barrow', 'garden_cart', 'lawn_cart', 'wheelbarrow'], 'id': 55, 'def': 'a cart for carrying small loads; has handles and one or more wheels', 'name': 'barrow'}, {'frequency': 'f', 'synset': 'base.n.03', 'synonyms': ['baseball_base'], 'id': 56, 'def': 'a place that the runner must touch before scoring', 'name': 'baseball_base'}, {'frequency': 'f', 'synset': 'baseball.n.02', 'synonyms': ['baseball'], 'id': 57, 'def': 'a ball used in playing baseball', 'name': 'baseball'}, {'frequency': 'f', 'synset': 'baseball_bat.n.01', 'synonyms': ['baseball_bat'], 'id': 58, 'def': 'an implement used in baseball by the batter', 'name': 'baseball_bat'}, {'frequency': 'f', 'synset': 'baseball_cap.n.01', 'synonyms': ['baseball_cap', 'jockey_cap', 'golf_cap'], 'id': 59, 'def': 'a cap with a bill', 'name': 'baseball_cap'}, {'frequency': 'f', 'synset': 'baseball_glove.n.01', 'synonyms': ['baseball_glove', 'baseball_mitt'], 'id': 60, 'def': 'the handwear used by fielders in playing baseball', 'name': 'baseball_glove'}, {'frequency': 'f', 'synset': 'basket.n.01', 'synonyms': ['basket', 'handbasket'], 'id': 61, 'def': 'a container that is usually woven and has handles', 'name': 'basket'}, {'frequency': 'c', 'synset': 'basketball.n.02', 'synonyms': ['basketball'], 'id': 62, 'def': 'an inflated ball used in playing basketball', 'name': 'basketball'}, {'frequency': 'r', 'synset': 'bass_horn.n.01', 'synonyms': ['bass_horn', 'sousaphone', 'tuba'], 'id': 63, 'def': 'the lowest brass wind instrument', 'name': 'bass_horn'}, {'frequency': 'c', 'synset': 'bat.n.01', 'synonyms': ['bat_(animal)'], 'id': 64, 'def': 'nocturnal mouselike mammal with forelimbs modified to form membranous wings', 'name': 'bat_(animal)'}, {'frequency': 'f', 'synset': 'bath_mat.n.01', 'synonyms': ['bath_mat'], 'id': 65, 'def': 'a heavy towel or mat to stand on while drying yourself after a bath', 'name': 'bath_mat'}, {'frequency': 'f', 'synset': 'bath_towel.n.01', 'synonyms': ['bath_towel'], 'id': 66, 'def': 'a large towel; to dry yourself after a bath', 'name': 'bath_towel'}, {'frequency': 'c', 'synset': 'bathrobe.n.01', 'synonyms': ['bathrobe'], 'id': 67, 'def': 'a loose-fitting robe of towelling; worn after a bath or swim', 'name': 'bathrobe'}, {'frequency': 'f', 'synset': 'bathtub.n.01', 'synonyms': ['bathtub', 'bathing_tub'], 'id': 68, 'def': 'a large open container that you fill with water and use to wash the body', 'name': 'bathtub'}, {'frequency': 'r', 'synset': 'batter.n.02', 'synonyms': ['batter_(food)'], 'id': 69, 'def': 'a liquid or semiliquid mixture, as of flour, eggs, and milk, used in cooking', 'name': 'batter_(food)'}, {'frequency': 'c', 'synset': 'battery.n.02', 'synonyms': ['battery'], 'id': 70, 'def': 'a portable device that produces electricity', 'name': 'battery'}, {'frequency': 'r', 'synset': 'beach_ball.n.01', 'synonyms': ['beachball'], 'id': 71, 'def': 'large and light ball; for play at the seaside', 'name': 'beachball'}, {'frequency': 'c', 'synset': 'bead.n.01', 'synonyms': ['bead'], 'id': 72, 'def': 'a small ball with a hole through the middle used for ornamentation, jewellery, etc.', 'name': 'bead'}, {'frequency': 'c', 'synset': 'bean_curd.n.01', 'synonyms': ['bean_curd', 'tofu'], 'id': 73, 'def': 'cheeselike food made of curdled soybean milk', 'name': 'bean_curd'}, {'frequency': 'c', 'synset': 'beanbag.n.01', 'synonyms': ['beanbag'], 'id': 74, 'def': 'a bag filled with dried beans or similar items; used in games or to sit on', 'name': 'beanbag'}, {'frequency': 'f', 'synset': 'beanie.n.01', 'synonyms': ['beanie', 'beany'], 'id': 75, 'def': 'a small skullcap; formerly worn by schoolboys and college freshmen', 'name': 'beanie'}, {'frequency': 'f', 'synset': 'bear.n.01', 'synonyms': ['bear'], 'id': 76, 'def': 'large carnivorous or omnivorous mammals with shaggy coats and claws', 'name': 'bear'}, {'frequency': 'f', 'synset': 'bed.n.01', 'synonyms': ['bed'], 'id': 77, 'def': 'a piece of furniture that provides a place to sleep', 'name': 'bed'}, {'frequency': 'r', 'synset': 'bedpan.n.01', 'synonyms': ['bedpan'], 'id': 78, 'def': 'a shallow vessel used by a bedridden patient for defecation and urination', 'name': 'bedpan'}, {'frequency': 'f', 'synset': 'bedspread.n.01', 'synonyms': ['bedspread', 'bedcover', 'bed_covering', 'counterpane', 'spread'], 'id': 79, 'def': 'decorative cover for a bed', 'name': 'bedspread'}, {'frequency': 'f', 'synset': 'beef.n.01', 'synonyms': ['cow'], 'id': 80, 'def': 'cattle/cow', 'name': 'cow'}, {'frequency': 'f', 'synset': 'beef.n.02', 'synonyms': ['beef_(food)', 'boeuf_(food)'], 'id': 81, 'def': 'meat from an adult domestic bovine', 'name': 'beef_(food)'}, {'frequency': 'r', 'synset': 'beeper.n.01', 'synonyms': ['beeper', 'pager'], 'id': 82, 'def': 'an device that beeps when the person carrying it is being paged', 'name': 'beeper'}, {'frequency': 'f', 'synset': 'beer_bottle.n.01', 'synonyms': ['beer_bottle'], 'id': 83, 'def': 'a bottle that holds beer', 'name': 'beer_bottle'}, {'frequency': 'c', 'synset': 'beer_can.n.01', 'synonyms': ['beer_can'], 'id': 84, 'def': 'a can that holds beer', 'name': 'beer_can'}, {'frequency': 'r', 'synset': 'beetle.n.01', 'synonyms': ['beetle'], 'id': 85, 'def': 'insect with hard wing covers', 'name': 'beetle'}, {'frequency': 'f', 'synset': 'bell.n.01', 'synonyms': ['bell'], 'id': 86, 'def': 'a hollow device made of metal that makes a ringing sound when struck', 'name': 'bell'}, {'frequency': 'f', 'synset': 'bell_pepper.n.02', 'synonyms': ['bell_pepper', 'capsicum'], 'id': 87, 'def': 'large bell-shaped sweet pepper in green or red or yellow or orange or black varieties', 'name': 'bell_pepper'}, {'frequency': 'f', 'synset': 'belt.n.02', 'synonyms': ['belt'], 'id': 88, 'def': 'a band to tie or buckle around the body (usually at the waist)', 'name': 'belt'}, {'frequency': 'f', 'synset': 'belt_buckle.n.01', 'synonyms': ['belt_buckle'], 'id': 89, 'def': 'the buckle used to fasten a belt', 'name': 'belt_buckle'}, {'frequency': 'f', 'synset': 'bench.n.01', 'synonyms': ['bench'], 'id': 90, 'def': 'a long seat for more than one person', 'name': 'bench'}, {'frequency': 'c', 'synset': 'beret.n.01', 'synonyms': ['beret'], 'id': 91, 'def': 'a cap with no brim or bill; made of soft cloth', 'name': 'beret'}, {'frequency': 'c', 'synset': 'bib.n.02', 'synonyms': ['bib'], 'id': 92, 'def': 'a napkin tied under the chin of a child while eating', 'name': 'bib'}, {'frequency': 'r', 'synset': 'bible.n.01', 'synonyms': ['Bible'], 'id': 93, 'def': 'the sacred writings of the Christian religions', 'name': 'Bible'}, {'frequency': 'f', 'synset': 'bicycle.n.01', 'synonyms': ['bicycle', 'bike_(bicycle)'], 'id': 94, 'def': 'a wheeled vehicle that has two wheels and is moved by foot pedals', 'name': 'bicycle'}, {'frequency': 'f', 'synset': 'bill.n.09', 'synonyms': ['visor', 'vizor'], 'id': 95, 'def': 'a brim that projects to the front to shade the eyes', 'name': 'visor'}, {'frequency': 'f', 'synset': 'billboard.n.01', 'synonyms': ['billboard'], 'id': 96, 'def': 'large outdoor signboard', 'name': 'billboard'}, {'frequency': 'c', 'synset': 'binder.n.03', 'synonyms': ['binder', 'ring-binder'], 'id': 97, 'def': 'holds loose papers or magazines', 'name': 'binder'}, {'frequency': 'c', 'synset': 'binoculars.n.01', 'synonyms': ['binoculars', 'field_glasses', 'opera_glasses'], 'id': 98, 'def': 'an optical instrument designed for simultaneous use by both eyes', 'name': 'binoculars'}, {'frequency': 'f', 'synset': 'bird.n.01', 'synonyms': ['bird'], 'id': 99, 'def': 'animal characterized by feathers and wings', 'name': 'bird'}, {'frequency': 'c', 'synset': 'bird_feeder.n.01', 'synonyms': ['birdfeeder'], 'id': 100, 'def': 'an outdoor device that supplies food for wild birds', 'name': 'birdfeeder'}, {'frequency': 'c', 'synset': 'birdbath.n.01', 'synonyms': ['birdbath'], 'id': 101, 'def': 'an ornamental basin (usually in a garden) for birds to bathe in', 'name': 'birdbath'}, {'frequency': 'c', 'synset': 'birdcage.n.01', 'synonyms': ['birdcage'], 'id': 102, 'def': 'a cage in which a bird can be kept', 'name': 'birdcage'}, {'frequency': 'c', 'synset': 'birdhouse.n.01', 'synonyms': ['birdhouse'], 'id': 103, 'def': 'a shelter for birds', 'name': 'birdhouse'}, {'frequency': 'f', 'synset': 'birthday_cake.n.01', 'synonyms': ['birthday_cake'], 'id': 104, 'def': 'decorated cake served at a birthday party', 'name': 'birthday_cake'}, {'frequency': 'r', 'synset': 'birthday_card.n.01', 'synonyms': ['birthday_card'], 'id': 105, 'def': 'a card expressing a birthday greeting', 'name': 'birthday_card'}, {'frequency': 'r', 'synset': 'black_flag.n.01', 'synonyms': ['pirate_flag'], 'id': 106, 'def': 'a flag usually bearing a white skull and crossbones on a black background', 'name': 'pirate_flag'}, {'frequency': 'c', 'synset': 'black_sheep.n.02', 'synonyms': ['black_sheep'], 'id': 107, 'def': 'sheep with a black coat', 'name': 'black_sheep'}, {'frequency': 'c', 'synset': 'blackberry.n.01', 'synonyms': ['blackberry'], 'id': 108, 'def': 'large sweet black or very dark purple edible aggregate fruit', 'name': 'blackberry'}, {'frequency': 'f', 'synset': 'blackboard.n.01', 'synonyms': ['blackboard', 'chalkboard'], 'id': 109, 'def': 'sheet of slate; for writing with chalk', 'name': 'blackboard'}, {'frequency': 'f', 'synset': 'blanket.n.01', 'synonyms': ['blanket'], 'id': 110, 'def': 'bedding that keeps a person warm in bed', 'name': 'blanket'}, {'frequency': 'c', 'synset': 'blazer.n.01', 'synonyms': ['blazer', 'sport_jacket', 'sport_coat', 'sports_jacket', 'sports_coat'], 'id': 111, 'def': 'lightweight jacket; often striped in the colors of a club or school', 'name': 'blazer'}, {'frequency': 'f', 'synset': 'blender.n.01', 'synonyms': ['blender', 'liquidizer', 'liquidiser'], 'id': 112, 'def': 'an electrically powered mixer that mix or chop or liquefy foods', 'name': 'blender'}, {'frequency': 'r', 'synset': 'blimp.n.02', 'synonyms': ['blimp'], 'id': 113, 'def': 'a small nonrigid airship used for observation or as a barrage balloon', 'name': 'blimp'}, {'frequency': 'f', 'synset': 'blinker.n.01', 'synonyms': ['blinker', 'flasher'], 'id': 114, 'def': 'a light that flashes on and off; used as a signal or to send messages', 'name': 'blinker'}, {'frequency': 'f', 'synset': 'blouse.n.01', 'synonyms': ['blouse'], 'id': 115, 'def': 'a top worn by women', 'name': 'blouse'}, {'frequency': 'f', 'synset': 'blueberry.n.02', 'synonyms': ['blueberry'], 'id': 116, 'def': 'sweet edible dark-blue berries of blueberry plants', 'name': 'blueberry'}, {'frequency': 'r', 'synset': 'board.n.09', 'synonyms': ['gameboard'], 'id': 117, 'def': 'a flat portable surface (usually rectangular) designed for board games', 'name': 'gameboard'}, {'frequency': 'f', 'synset': 'boat.n.01', 'synonyms': ['boat', 'ship_(boat)'], 'id': 118, 'def': 'a vessel for travel on water', 'name': 'boat'}, {'frequency': 'r', 'synset': 'bob.n.05', 'synonyms': ['bob', 'bobber', 'bobfloat'], 'id': 119, 'def': 'a small float usually made of cork; attached to a fishing line', 'name': 'bob'}, {'frequency': 'c', 'synset': 'bobbin.n.01', 'synonyms': ['bobbin', 'spool', 'reel'], 'id': 120, 'def': 'a thing around which thread/tape/film or other flexible materials can be wound', 'name': 'bobbin'}, {'frequency': 'c', 'synset': 'bobby_pin.n.01', 'synonyms': ['bobby_pin', 'hairgrip'], 'id': 121, 'def': 'a flat wire hairpin used to hold bobbed hair in place', 'name': 'bobby_pin'}, {'frequency': 'c', 'synset': 'boiled_egg.n.01', 'synonyms': ['boiled_egg', 'coddled_egg'], 'id': 122, 'def': 'egg cooked briefly in the shell in gently boiling water', 'name': 'boiled_egg'}, {'frequency': 'r', 'synset': 'bolo_tie.n.01', 'synonyms': ['bolo_tie', 'bolo', 'bola_tie', 'bola'], 'id': 123, 'def': 'a cord fastened around the neck with an ornamental clasp and worn as a necktie', 'name': 'bolo_tie'}, {'frequency': 'c', 'synset': 'bolt.n.03', 'synonyms': ['deadbolt'], 'id': 124, 'def': 'the part of a lock that is engaged or withdrawn with a key', 'name': 'deadbolt'}, {'frequency': 'f', 'synset': 'bolt.n.06', 'synonyms': ['bolt'], 'id': 125, 'def': 'a screw that screws into a nut to form a fastener', 'name': 'bolt'}, {'frequency': 'r', 'synset': 'bonnet.n.01', 'synonyms': ['bonnet'], 'id': 126, 'def': 'a hat tied under the chin', 'name': 'bonnet'}, {'frequency': 'f', 'synset': 'book.n.01', 'synonyms': ['book'], 'id': 127, 'def': 'a written work or composition that has been published', 'name': 'book'}, {'frequency': 'c', 'synset': 'bookcase.n.01', 'synonyms': ['bookcase'], 'id': 128, 'def': 'a piece of furniture with shelves for storing books', 'name': 'bookcase'}, {'frequency': 'c', 'synset': 'booklet.n.01', 'synonyms': ['booklet', 'brochure', 'leaflet', 'pamphlet'], 'id': 129, 'def': 'a small book usually having a paper cover', 'name': 'booklet'}, {'frequency': 'r', 'synset': 'bookmark.n.01', 'synonyms': ['bookmark', 'bookmarker'], 'id': 130, 'def': 'a marker (a piece of paper or ribbon) placed between the pages of a book', 'name': 'bookmark'}, {'frequency': 'r', 'synset': 'boom.n.04', 'synonyms': ['boom_microphone', 'microphone_boom'], 'id': 131, 'def': 'a pole carrying an overhead microphone projected over a film or tv set', 'name': 'boom_microphone'}, {'frequency': 'f', 'synset': 'boot.n.01', 'synonyms': ['boot'], 'id': 132, 'def': 'footwear that covers the whole foot and lower leg', 'name': 'boot'}, {'frequency': 'f', 'synset': 'bottle.n.01', 'synonyms': ['bottle'], 'id': 133, 'def': 'a glass or plastic vessel used for storing drinks or other liquids', 'name': 'bottle'}, {'frequency': 'c', 'synset': 'bottle_opener.n.01', 'synonyms': ['bottle_opener'], 'id': 134, 'def': 'an opener for removing caps or corks from bottles', 'name': 'bottle_opener'}, {'frequency': 'c', 'synset': 'bouquet.n.01', 'synonyms': ['bouquet'], 'id': 135, 'def': 'an arrangement of flowers that is usually given as a present', 'name': 'bouquet'}, {'frequency': 'r', 'synset': 'bow.n.04', 'synonyms': ['bow_(weapon)'], 'id': 136, 'def': 'a weapon for shooting arrows', 'name': 'bow_(weapon)'}, {'frequency': 'f', 'synset': 'bow.n.08', 'synonyms': ['bow_(decorative_ribbons)'], 'id': 137, 'def': 'a decorative interlacing of ribbons', 'name': 'bow_(decorative_ribbons)'}, {'frequency': 'f', 'synset': 'bow_tie.n.01', 'synonyms': ['bow-tie', 'bowtie'], 'id': 138, 'def': "a man's tie that ties in a bow", 'name': 'bow-tie'}, {'frequency': 'f', 'synset': 'bowl.n.03', 'synonyms': ['bowl'], 'id': 139, 'def': 'a dish that is round and open at the top for serving foods', 'name': 'bowl'}, {'frequency': 'r', 'synset': 'bowl.n.08', 'synonyms': ['pipe_bowl'], 'id': 140, 'def': 'a small round container that is open at the top for holding tobacco', 'name': 'pipe_bowl'}, {'frequency': 'c', 'synset': 'bowler_hat.n.01', 'synonyms': ['bowler_hat', 'bowler', 'derby_hat', 'derby', 'plug_hat'], 'id': 141, 'def': 'a felt hat that is round and hard with a narrow brim', 'name': 'bowler_hat'}, {'frequency': 'r', 'synset': 'bowling_ball.n.01', 'synonyms': ['bowling_ball'], 'id': 142, 'def': 'a large ball with finger holes used in the sport of bowling', 'name': 'bowling_ball'}, {'frequency': 'f', 'synset': 'box.n.01', 'synonyms': ['box'], 'id': 143, 'def': 'a (usually rectangular) container; may have a lid', 'name': 'box'}, {'frequency': 'r', 'synset': 'boxing_glove.n.01', 'synonyms': ['boxing_glove'], 'id': 144, 'def': 'large glove coverings the fists of a fighter worn for the sport of boxing', 'name': 'boxing_glove'}, {'frequency': 'c', 'synset': 'brace.n.06', 'synonyms': ['suspenders'], 'id': 145, 'def': 'elastic straps that hold trousers up (usually used in the plural)', 'name': 'suspenders'}, {'frequency': 'f', 'synset': 'bracelet.n.02', 'synonyms': ['bracelet', 'bangle'], 'id': 146, 'def': 'jewelry worn around the wrist for decoration', 'name': 'bracelet'}, {'frequency': 'r', 'synset': 'brass.n.07', 'synonyms': ['brass_plaque'], 'id': 147, 'def': 'a memorial made of brass', 'name': 'brass_plaque'}, {'frequency': 'c', 'synset': 'brassiere.n.01', 'synonyms': ['brassiere', 'bra', 'bandeau'], 'id': 148, 'def': 'an undergarment worn by women to support their breasts', 'name': 'brassiere'}, {'frequency': 'c', 'synset': 'bread-bin.n.01', 'synonyms': ['bread-bin', 'breadbox'], 'id': 149, 'def': 'a container used to keep bread or cake in', 'name': 'bread-bin'}, {'frequency': 'f', 'synset': 'bread.n.01', 'synonyms': ['bread'], 'id': 150, 'def': 'food made from dough of flour or meal and usually raised with yeast or baking powder and then baked', 'name': 'bread'}, {'frequency': 'r', 'synset': 'breechcloth.n.01', 'synonyms': ['breechcloth', 'breechclout', 'loincloth'], 'id': 151, 'def': 'a garment that provides covering for the loins', 'name': 'breechcloth'}, {'frequency': 'f', 'synset': 'bridal_gown.n.01', 'synonyms': ['bridal_gown', 'wedding_gown', 'wedding_dress'], 'id': 152, 'def': 'a gown worn by the bride at a wedding', 'name': 'bridal_gown'}, {'frequency': 'c', 'synset': 'briefcase.n.01', 'synonyms': ['briefcase'], 'id': 153, 'def': 'a case with a handle; for carrying papers or files or books', 'name': 'briefcase'}, {'frequency': 'f', 'synset': 'broccoli.n.01', 'synonyms': ['broccoli'], 'id': 154, 'def': 'plant with dense clusters of tight green flower buds', 'name': 'broccoli'}, {'frequency': 'r', 'synset': 'brooch.n.01', 'synonyms': ['broach'], 'id': 155, 'def': 'a decorative pin worn by women', 'name': 'broach'}, {'frequency': 'c', 'synset': 'broom.n.01', 'synonyms': ['broom'], 'id': 156, 'def': 'bundle of straws or twigs attached to a long handle; used for cleaning', 'name': 'broom'}, {'frequency': 'c', 'synset': 'brownie.n.03', 'synonyms': ['brownie'], 'id': 157, 'def': 'square or bar of very rich chocolate cake usually with nuts', 'name': 'brownie'}, {'frequency': 'c', 'synset': 'brussels_sprouts.n.01', 'synonyms': ['brussels_sprouts'], 'id': 158, 'def': 'the small edible cabbage-like buds growing along a stalk', 'name': 'brussels_sprouts'}, {'frequency': 'r', 'synset': 'bubble_gum.n.01', 'synonyms': ['bubble_gum'], 'id': 159, 'def': 'a kind of chewing gum that can be blown into bubbles', 'name': 'bubble_gum'}, {'frequency': 'f', 'synset': 'bucket.n.01', 'synonyms': ['bucket', 'pail'], 'id': 160, 'def': 'a roughly cylindrical vessel that is open at the top', 'name': 'bucket'}, {'frequency': 'r', 'synset': 'buggy.n.01', 'synonyms': ['horse_buggy'], 'id': 161, 'def': 'a small lightweight carriage; drawn by a single horse', 'name': 'horse_buggy'}, {'frequency': 'c', 'synset': 'bull.n.11', 'synonyms': ['horned_cow'], 'id': 162, 'def': 'a cow with horns', 'name': 'bull'}, {'frequency': 'c', 'synset': 'bulldog.n.01', 'synonyms': ['bulldog'], 'id': 163, 'def': 'a thickset short-haired dog with a large head and strong undershot lower jaw', 'name': 'bulldog'}, {'frequency': 'r', 'synset': 'bulldozer.n.01', 'synonyms': ['bulldozer', 'dozer'], 'id': 164, 'def': 'large powerful tractor; a large blade in front flattens areas of ground', 'name': 'bulldozer'}, {'frequency': 'c', 'synset': 'bullet_train.n.01', 'synonyms': ['bullet_train'], 'id': 165, 'def': 'a high-speed passenger train', 'name': 'bullet_train'}, {'frequency': 'c', 'synset': 'bulletin_board.n.02', 'synonyms': ['bulletin_board', 'notice_board'], 'id': 166, 'def': 'a board that hangs on a wall; displays announcements', 'name': 'bulletin_board'}, {'frequency': 'r', 'synset': 'bulletproof_vest.n.01', 'synonyms': ['bulletproof_vest'], 'id': 167, 'def': 'a vest capable of resisting the impact of a bullet', 'name': 'bulletproof_vest'}, {'frequency': 'c', 'synset': 'bullhorn.n.01', 'synonyms': ['bullhorn', 'megaphone'], 'id': 168, 'def': 'a portable loudspeaker with built-in microphone and amplifier', 'name': 'bullhorn'}, {'frequency': 'f', 'synset': 'bun.n.01', 'synonyms': ['bun', 'roll'], 'id': 169, 'def': 'small rounded bread either plain or sweet', 'name': 'bun'}, {'frequency': 'c', 'synset': 'bunk_bed.n.01', 'synonyms': ['bunk_bed'], 'id': 170, 'def': 'beds built one above the other', 'name': 'bunk_bed'}, {'frequency': 'f', 'synset': 'buoy.n.01', 'synonyms': ['buoy'], 'id': 171, 'def': 'a float attached by rope to the seabed to mark channels in a harbor or underwater hazards', 'name': 'buoy'}, {'frequency': 'r', 'synset': 'burrito.n.01', 'synonyms': ['burrito'], 'id': 172, 'def': 'a flour tortilla folded around a filling', 'name': 'burrito'}, {'frequency': 'f', 'synset': 'bus.n.01', 'synonyms': ['bus_(vehicle)', 'autobus', 'charabanc', 'double-decker', 'motorbus', 'motorcoach'], 'id': 173, 'def': 'a vehicle carrying many passengers; used for public transport', 'name': 'bus_(vehicle)'}, {'frequency': 'c', 'synset': 'business_card.n.01', 'synonyms': ['business_card'], 'id': 174, 'def': "a card on which are printed the person's name and business affiliation", 'name': 'business_card'}, {'frequency': 'f', 'synset': 'butter.n.01', 'synonyms': ['butter'], 'id': 175, 'def': 'an edible emulsion of fat globules made by churning milk or cream; for cooking and table use', 'name': 'butter'}, {'frequency': 'c', 'synset': 'butterfly.n.01', 'synonyms': ['butterfly'], 'id': 176, 'def': 'insect typically having a slender body with knobbed antennae and broad colorful wings', 'name': 'butterfly'}, {'frequency': 'f', 'synset': 'button.n.01', 'synonyms': ['button'], 'id': 177, 'def': 'a round fastener sewn to shirts and coats etc to fit through buttonholes', 'name': 'button'}, {'frequency': 'f', 'synset': 'cab.n.03', 'synonyms': ['cab_(taxi)', 'taxi', 'taxicab'], 'id': 178, 'def': 'a car that takes passengers where they want to go in exchange for money', 'name': 'cab_(taxi)'}, {'frequency': 'r', 'synset': 'cabana.n.01', 'synonyms': ['cabana'], 'id': 179, 'def': 'a small tent used as a dressing room beside the sea or a swimming pool', 'name': 'cabana'}, {'frequency': 'c', 'synset': 'cabin_car.n.01', 'synonyms': ['cabin_car', 'caboose'], 'id': 180, 'def': 'a car on a freight train for use of the train crew; usually the last car on the train', 'name': 'cabin_car'}, {'frequency': 'f', 'synset': 'cabinet.n.01', 'synonyms': ['cabinet'], 'id': 181, 'def': 'a piece of furniture resembling a cupboard with doors and shelves and drawers', 'name': 'cabinet'}, {'frequency': 'r', 'synset': 'cabinet.n.03', 'synonyms': ['locker', 'storage_locker'], 'id': 182, 'def': 'a storage compartment for clothes and valuables; usually it has a lock', 'name': 'locker'}, {'frequency': 'f', 'synset': 'cake.n.03', 'synonyms': ['cake'], 'id': 183, 'def': 'baked goods made from or based on a mixture of flour, sugar, eggs, and fat', 'name': 'cake'}, {'frequency': 'c', 'synset': 'calculator.n.02', 'synonyms': ['calculator'], 'id': 184, 'def': 'a small machine that is used for mathematical calculations', 'name': 'calculator'}, {'frequency': 'f', 'synset': 'calendar.n.02', 'synonyms': ['calendar'], 'id': 185, 'def': 'a list or register of events (appointments/social events/court cases, etc)', 'name': 'calendar'}, {'frequency': 'c', 'synset': 'calf.n.01', 'synonyms': ['calf'], 'id': 186, 'def': 'young of domestic cattle', 'name': 'calf'}, {'frequency': 'c', 'synset': 'camcorder.n.01', 'synonyms': ['camcorder'], 'id': 187, 'def': 'a portable television camera and videocassette recorder', 'name': 'camcorder'}, {'frequency': 'c', 'synset': 'camel.n.01', 'synonyms': ['camel'], 'id': 188, 'def': 'cud-chewing mammal used as a draft or saddle animal in desert regions', 'name': 'camel'}, {'frequency': 'f', 'synset': 'camera.n.01', 'synonyms': ['camera'], 'id': 189, 'def': 'equipment for taking photographs', 'name': 'camera'}, {'frequency': 'c', 'synset': 'camera_lens.n.01', 'synonyms': ['camera_lens'], 'id': 190, 'def': 'a lens that focuses the image in a camera', 'name': 'camera_lens'}, {'frequency': 'c', 'synset': 'camper.n.02', 'synonyms': ['camper_(vehicle)', 'camping_bus', 'motor_home'], 'id': 191, 'def': 'a recreational vehicle equipped for camping out while traveling', 'name': 'camper_(vehicle)'}, {'frequency': 'f', 'synset': 'can.n.01', 'synonyms': ['can', 'tin_can'], 'id': 192, 'def': 'airtight sealed metal container for food or drink or paint etc.', 'name': 'can'}, {'frequency': 'c', 'synset': 'can_opener.n.01', 'synonyms': ['can_opener', 'tin_opener'], 'id': 193, 'def': 'a device for cutting cans open', 'name': 'can_opener'}, {'frequency': 'f', 'synset': 'candle.n.01', 'synonyms': ['candle', 'candlestick'], 'id': 194, 'def': 'stick of wax with a wick in the middle', 'name': 'candle'}, {'frequency': 'f', 'synset': 'candlestick.n.01', 'synonyms': ['candle_holder'], 'id': 195, 'def': 'a holder with sockets for candles', 'name': 'candle_holder'}, {'frequency': 'r', 'synset': 'candy_bar.n.01', 'synonyms': ['candy_bar'], 'id': 196, 'def': 'a candy shaped as a bar', 'name': 'candy_bar'}, {'frequency': 'c', 'synset': 'candy_cane.n.01', 'synonyms': ['candy_cane'], 'id': 197, 'def': 'a hard candy in the shape of a rod (usually with stripes)', 'name': 'candy_cane'}, {'frequency': 'c', 'synset': 'cane.n.01', 'synonyms': ['walking_cane'], 'id': 198, 'def': 'a stick that people can lean on to help them walk', 'name': 'walking_cane'}, {'frequency': 'c', 'synset': 'canister.n.02', 'synonyms': ['canister', 'cannister'], 'id': 199, 'def': 'metal container for storing dry foods such as tea or flour', 'name': 'canister'}, {'frequency': 'c', 'synset': 'canoe.n.01', 'synonyms': ['canoe'], 'id': 200, 'def': 'small and light boat; pointed at both ends; propelled with a paddle', 'name': 'canoe'}, {'frequency': 'c', 'synset': 'cantaloup.n.02', 'synonyms': ['cantaloup', 'cantaloupe'], 'id': 201, 'def': 'the fruit of a cantaloup vine; small to medium-sized melon with yellowish flesh', 'name': 'cantaloup'}, {'frequency': 'r', 'synset': 'canteen.n.01', 'synonyms': ['canteen'], 'id': 202, 'def': 'a flask for carrying water; used by soldiers or travelers', 'name': 'canteen'}, {'frequency': 'f', 'synset': 'cap.n.01', 'synonyms': ['cap_(headwear)'], 'id': 203, 'def': 'a tight-fitting headwear', 'name': 'cap_(headwear)'}, {'frequency': 'f', 'synset': 'cap.n.02', 'synonyms': ['bottle_cap', 'cap_(container_lid)'], 'id': 204, 'def': 'a top (as for a bottle)', 'name': 'bottle_cap'}, {'frequency': 'c', 'synset': 'cape.n.02', 'synonyms': ['cape'], 'id': 205, 'def': 'a sleeveless garment like a cloak but shorter', 'name': 'cape'}, {'frequency': 'c', 'synset': 'cappuccino.n.01', 'synonyms': ['cappuccino', 'coffee_cappuccino'], 'id': 206, 'def': 'equal parts of espresso and steamed milk', 'name': 'cappuccino'}, {'frequency': 'f', 'synset': 'car.n.01', 'synonyms': ['car_(automobile)', 'auto_(automobile)', 'automobile'], 'id': 207, 'def': 'a motor vehicle with four wheels', 'name': 'car_(automobile)'}, {'frequency': 'f', 'synset': 'car.n.02', 'synonyms': ['railcar_(part_of_a_train)', 'railway_car_(part_of_a_train)', 'railroad_car_(part_of_a_train)'], 'id': 208, 'def': 'a wheeled vehicle adapted to the rails of railroad (mark each individual railcar separately)', 'name': 'railcar_(part_of_a_train)'}, {'frequency': 'r', 'synset': 'car.n.04', 'synonyms': ['elevator_car'], 'id': 209, 'def': 'where passengers ride up and down', 'name': 'elevator_car'}, {'frequency': 'r', 'synset': 'car_battery.n.01', 'synonyms': ['car_battery', 'automobile_battery'], 'id': 210, 'def': 'a battery in a motor vehicle', 'name': 'car_battery'}, {'frequency': 'c', 'synset': 'card.n.02', 'synonyms': ['identity_card'], 'id': 211, 'def': 'a card certifying the identity of the bearer', 'name': 'identity_card'}, {'frequency': 'c', 'synset': 'card.n.03', 'synonyms': ['card'], 'id': 212, 'def': 'a rectangular piece of paper used to send messages (e.g. greetings or pictures)', 'name': 'card'}, {'frequency': 'c', 'synset': 'cardigan.n.01', 'synonyms': ['cardigan'], 'id': 213, 'def': 'knitted jacket that is fastened up the front with buttons or a zipper', 'name': 'cardigan'}, {'frequency': 'r', 'synset': 'cargo_ship.n.01', 'synonyms': ['cargo_ship', 'cargo_vessel'], 'id': 214, 'def': 'a ship designed to carry cargo', 'name': 'cargo_ship'}, {'frequency': 'r', 'synset': 'carnation.n.01', 'synonyms': ['carnation'], 'id': 215, 'def': 'plant with pink to purple-red spice-scented usually double flowers', 'name': 'carnation'}, {'frequency': 'c', 'synset': 'carriage.n.02', 'synonyms': ['horse_carriage'], 'id': 216, 'def': 'a vehicle with wheels drawn by one or more horses', 'name': 'horse_carriage'}, {'frequency': 'f', 'synset': 'carrot.n.01', 'synonyms': ['carrot'], 'id': 217, 'def': 'deep orange edible root of the cultivated carrot plant', 'name': 'carrot'}, {'frequency': 'f', 'synset': 'carryall.n.01', 'synonyms': ['tote_bag'], 'id': 218, 'def': 'a capacious bag or basket', 'name': 'tote_bag'}, {'frequency': 'c', 'synset': 'cart.n.01', 'synonyms': ['cart'], 'id': 219, 'def': 'a heavy open wagon usually having two wheels and drawn by an animal', 'name': 'cart'}, {'frequency': 'c', 'synset': 'carton.n.02', 'synonyms': ['carton'], 'id': 220, 'def': 'a container made of cardboard for holding food or drink', 'name': 'carton'}, {'frequency': 'c', 'synset': 'cash_register.n.01', 'synonyms': ['cash_register', 'register_(for_cash_transactions)'], 'id': 221, 'def': 'a cashbox with an adding machine to register transactions', 'name': 'cash_register'}, {'frequency': 'r', 'synset': 'casserole.n.01', 'synonyms': ['casserole'], 'id': 222, 'def': 'food cooked and served in a casserole', 'name': 'casserole'}, {'frequency': 'r', 'synset': 'cassette.n.01', 'synonyms': ['cassette'], 'id': 223, 'def': 'a container that holds a magnetic tape used for recording or playing sound or video', 'name': 'cassette'}, {'frequency': 'c', 'synset': 'cast.n.05', 'synonyms': ['cast', 'plaster_cast', 'plaster_bandage'], 'id': 224, 'def': 'bandage consisting of a firm covering that immobilizes broken bones while they heal', 'name': 'cast'}, {'frequency': 'f', 'synset': 'cat.n.01', 'synonyms': ['cat'], 'id': 225, 'def': 'a domestic house cat', 'name': 'cat'}, {'frequency': 'f', 'synset': 'cauliflower.n.02', 'synonyms': ['cauliflower'], 'id': 226, 'def': 'edible compact head of white undeveloped flowers', 'name': 'cauliflower'}, {'frequency': 'c', 'synset': 'cayenne.n.02', 'synonyms': ['cayenne_(spice)', 'cayenne_pepper_(spice)', 'red_pepper_(spice)'], 'id': 227, 'def': 'ground pods and seeds of pungent red peppers of the genus Capsicum', 'name': 'cayenne_(spice)'}, {'frequency': 'c', 'synset': 'cd_player.n.01', 'synonyms': ['CD_player'], 'id': 228, 'def': 'electronic equipment for playing compact discs (CDs)', 'name': 'CD_player'}, {'frequency': 'f', 'synset': 'celery.n.01', 'synonyms': ['celery'], 'id': 229, 'def': 'widely cultivated herb with aromatic leaf stalks that are eaten raw or cooked', 'name': 'celery'}, {'frequency': 'f', 'synset': 'cellular_telephone.n.01', 'synonyms': ['cellular_telephone', 'cellular_phone', 'cellphone', 'mobile_phone', 'smart_phone'], 'id': 230, 'def': 'a hand-held mobile telephone', 'name': 'cellular_telephone'}, {'frequency': 'r', 'synset': 'chain_mail.n.01', 'synonyms': ['chain_mail', 'ring_mail', 'chain_armor', 'chain_armour', 'ring_armor', 'ring_armour'], 'id': 231, 'def': '(Middle Ages) flexible armor made of interlinked metal rings', 'name': 'chain_mail'}, {'frequency': 'f', 'synset': 'chair.n.01', 'synonyms': ['chair'], 'id': 232, 'def': 'a seat for one person, with a support for the back', 'name': 'chair'}, {'frequency': 'r', 'synset': 'chaise_longue.n.01', 'synonyms': ['chaise_longue', 'chaise', 'daybed'], 'id': 233, 'def': 'a long chair; for reclining', 'name': 'chaise_longue'}, {'frequency': 'r', 'synset': 'chalice.n.01', 'synonyms': ['chalice'], 'id': 234, 'def': 'a bowl-shaped drinking vessel; especially the Eucharistic cup', 'name': 'chalice'}, {'frequency': 'f', 'synset': 'chandelier.n.01', 'synonyms': ['chandelier'], 'id': 235, 'def': 'branched lighting fixture; often ornate; hangs from the ceiling', 'name': 'chandelier'}, {'frequency': 'r', 'synset': 'chap.n.04', 'synonyms': ['chap'], 'id': 236, 'def': 'leather leggings without a seat; worn over trousers by cowboys to protect their legs', 'name': 'chap'}, {'frequency': 'r', 'synset': 'checkbook.n.01', 'synonyms': ['checkbook', 'chequebook'], 'id': 237, 'def': 'a book issued to holders of checking accounts', 'name': 'checkbook'}, {'frequency': 'r', 'synset': 'checkerboard.n.01', 'synonyms': ['checkerboard'], 'id': 238, 'def': 'a board having 64 squares of two alternating colors', 'name': 'checkerboard'}, {'frequency': 'c', 'synset': 'cherry.n.03', 'synonyms': ['cherry'], 'id': 239, 'def': 'a red fruit with a single hard stone', 'name': 'cherry'}, {'frequency': 'r', 'synset': 'chessboard.n.01', 'synonyms': ['chessboard'], 'id': 240, 'def': 'a checkerboard used to play chess', 'name': 'chessboard'}, {'frequency': 'c', 'synset': 'chicken.n.02', 'synonyms': ['chicken_(animal)'], 'id': 241, 'def': 'a domestic fowl bred for flesh or eggs', 'name': 'chicken_(animal)'}, {'frequency': 'c', 'synset': 'chickpea.n.01', 'synonyms': ['chickpea', 'garbanzo'], 'id': 242, 'def': 'the seed of the chickpea plant; usually dried', 'name': 'chickpea'}, {'frequency': 'c', 'synset': 'chili.n.02', 'synonyms': ['chili_(vegetable)', 'chili_pepper_(vegetable)', 'chilli_(vegetable)', 'chilly_(vegetable)', 'chile_(vegetable)'], 'id': 243, 'def': 'very hot and finely tapering pepper of special pungency', 'name': 'chili_(vegetable)'}, {'frequency': 'r', 'synset': 'chime.n.01', 'synonyms': ['chime', 'gong'], 'id': 244, 'def': 'an instrument consisting of a set of bells that are struck with a hammer', 'name': 'chime'}, {'frequency': 'r', 'synset': 'chinaware.n.01', 'synonyms': ['chinaware'], 'id': 245, 'def': 'dishware made of high quality porcelain', 'name': 'chinaware'}, {'frequency': 'c', 'synset': 'chip.n.04', 'synonyms': ['crisp_(potato_chip)', 'potato_chip'], 'id': 246, 'def': 'a thin crisp slice of potato fried in deep fat', 'name': 'crisp_(potato_chip)'}, {'frequency': 'r', 'synset': 'chip.n.06', 'synonyms': ['poker_chip'], 'id': 247, 'def': 'a small disk-shaped counter used to represent money when gambling', 'name': 'poker_chip'}, {'frequency': 'c', 'synset': 'chocolate_bar.n.01', 'synonyms': ['chocolate_bar'], 'id': 248, 'def': 'a bar of chocolate candy', 'name': 'chocolate_bar'}, {'frequency': 'c', 'synset': 'chocolate_cake.n.01', 'synonyms': ['chocolate_cake'], 'id': 249, 'def': 'cake containing chocolate', 'name': 'chocolate_cake'}, {'frequency': 'r', 'synset': 'chocolate_milk.n.01', 'synonyms': ['chocolate_milk'], 'id': 250, 'def': 'milk flavored with chocolate syrup', 'name': 'chocolate_milk'}, {'frequency': 'r', 'synset': 'chocolate_mousse.n.01', 'synonyms': ['chocolate_mousse'], 'id': 251, 'def': 'dessert mousse made with chocolate', 'name': 'chocolate_mousse'}, {'frequency': 'f', 'synset': 'choker.n.03', 'synonyms': ['choker', 'collar', 'neckband'], 'id': 252, 'def': 'shirt collar, animal collar, or tight-fitting necklace', 'name': 'choker'}, {'frequency': 'f', 'synset': 'chopping_board.n.01', 'synonyms': ['chopping_board', 'cutting_board', 'chopping_block'], 'id': 253, 'def': 'a wooden board where meats or vegetables can be cut', 'name': 'chopping_board'}, {'frequency': 'f', 'synset': 'chopstick.n.01', 'synonyms': ['chopstick'], 'id': 254, 'def': 'one of a pair of slender sticks used as oriental tableware to eat food with', 'name': 'chopstick'}, {'frequency': 'f', 'synset': 'christmas_tree.n.05', 'synonyms': ['Christmas_tree'], 'id': 255, 'def': 'an ornamented evergreen used as a Christmas decoration', 'name': 'Christmas_tree'}, {'frequency': 'c', 'synset': 'chute.n.02', 'synonyms': ['slide'], 'id': 256, 'def': 'sloping channel through which things can descend', 'name': 'slide'}, {'frequency': 'r', 'synset': 'cider.n.01', 'synonyms': ['cider', 'cyder'], 'id': 257, 'def': 'a beverage made from juice pressed from apples', 'name': 'cider'}, {'frequency': 'r', 'synset': 'cigar_box.n.01', 'synonyms': ['cigar_box'], 'id': 258, 'def': 'a box for holding cigars', 'name': 'cigar_box'}, {'frequency': 'f', 'synset': 'cigarette.n.01', 'synonyms': ['cigarette'], 'id': 259, 'def': 'finely ground tobacco wrapped in paper; for smoking', 'name': 'cigarette'}, {'frequency': 'c', 'synset': 'cigarette_case.n.01', 'synonyms': ['cigarette_case', 'cigarette_pack'], 'id': 260, 'def': 'a small flat case for holding cigarettes', 'name': 'cigarette_case'}, {'frequency': 'f', 'synset': 'cistern.n.02', 'synonyms': ['cistern', 'water_tank'], 'id': 261, 'def': 'a tank that holds the water used to flush a toilet', 'name': 'cistern'}, {'frequency': 'r', 'synset': 'clarinet.n.01', 'synonyms': ['clarinet'], 'id': 262, 'def': 'a single-reed instrument with a straight tube', 'name': 'clarinet'}, {'frequency': 'c', 'synset': 'clasp.n.01', 'synonyms': ['clasp'], 'id': 263, 'def': 'a fastener (as a buckle or hook) that is used to hold two things together', 'name': 'clasp'}, {'frequency': 'c', 'synset': 'cleansing_agent.n.01', 'synonyms': ['cleansing_agent', 'cleanser', 'cleaner'], 'id': 264, 'def': 'a preparation used in cleaning something', 'name': 'cleansing_agent'}, {'frequency': 'r', 'synset': 'cleat.n.02', 'synonyms': ['cleat_(for_securing_rope)'], 'id': 265, 'def': 'a fastener (usually with two projecting horns) around which a rope can be secured', 'name': 'cleat_(for_securing_rope)'}, {'frequency': 'r', 'synset': 'clementine.n.01', 'synonyms': ['clementine'], 'id': 266, 'def': 'a variety of mandarin orange', 'name': 'clementine'}, {'frequency': 'c', 'synset': 'clip.n.03', 'synonyms': ['clip'], 'id': 267, 'def': 'any of various small fasteners used to hold loose articles together', 'name': 'clip'}, {'frequency': 'c', 'synset': 'clipboard.n.01', 'synonyms': ['clipboard'], 'id': 268, 'def': 'a small writing board with a clip at the top for holding papers', 'name': 'clipboard'}, {'frequency': 'r', 'synset': 'clipper.n.03', 'synonyms': ['clippers_(for_plants)'], 'id': 269, 'def': 'shears for cutting grass or shrubbery (often used in the plural)', 'name': 'clippers_(for_plants)'}, {'frequency': 'r', 'synset': 'cloak.n.02', 'synonyms': ['cloak'], 'id': 270, 'def': 'a loose outer garment', 'name': 'cloak'}, {'frequency': 'f', 'synset': 'clock.n.01', 'synonyms': ['clock', 'timepiece', 'timekeeper'], 'id': 271, 'def': 'a timepiece that shows the time of day', 'name': 'clock'}, {'frequency': 'f', 'synset': 'clock_tower.n.01', 'synonyms': ['clock_tower'], 'id': 272, 'def': 'a tower with a large clock visible high up on an outside face', 'name': 'clock_tower'}, {'frequency': 'c', 'synset': 'clothes_hamper.n.01', 'synonyms': ['clothes_hamper', 'laundry_basket', 'clothes_basket'], 'id': 273, 'def': 'a hamper that holds dirty clothes to be washed or wet clothes to be dried', 'name': 'clothes_hamper'}, {'frequency': 'c', 'synset': 'clothespin.n.01', 'synonyms': ['clothespin', 'clothes_peg'], 'id': 274, 'def': 'wood or plastic fastener; for holding clothes on a clothesline', 'name': 'clothespin'}, {'frequency': 'r', 'synset': 'clutch_bag.n.01', 'synonyms': ['clutch_bag'], 'id': 275, 'def': "a woman's strapless purse that is carried in the hand", 'name': 'clutch_bag'}, {'frequency': 'f', 'synset': 'coaster.n.03', 'synonyms': ['coaster'], 'id': 276, 'def': 'a covering (plate or mat) that protects the surface of a table', 'name': 'coaster'}, {'frequency': 'f', 'synset': 'coat.n.01', 'synonyms': ['coat'], 'id': 277, 'def': 'an outer garment that has sleeves and covers the body from shoulder down', 'name': 'coat'}, {'frequency': 'c', 'synset': 'coat_hanger.n.01', 'synonyms': ['coat_hanger', 'clothes_hanger', 'dress_hanger'], 'id': 278, 'def': "a hanger that is shaped like a person's shoulders", 'name': 'coat_hanger'}, {'frequency': 'c', 'synset': 'coatrack.n.01', 'synonyms': ['coatrack', 'hatrack'], 'id': 279, 'def': 'a rack with hooks for temporarily holding coats and hats', 'name': 'coatrack'}, {'frequency': 'c', 'synset': 'cock.n.04', 'synonyms': ['cock', 'rooster'], 'id': 280, 'def': 'adult male chicken', 'name': 'cock'}, {'frequency': 'r', 'synset': 'cockroach.n.01', 'synonyms': ['cockroach'], 'id': 281, 'def': 'any of numerous chiefly nocturnal insects; some are domestic pests', 'name': 'cockroach'}, {'frequency': 'r', 'synset': 'cocoa.n.01', 'synonyms': ['cocoa_(beverage)', 'hot_chocolate_(beverage)', 'drinking_chocolate'], 'id': 282, 'def': 'a beverage made from cocoa powder and milk and sugar; usually drunk hot', 'name': 'cocoa_(beverage)'}, {'frequency': 'c', 'synset': 'coconut.n.02', 'synonyms': ['coconut', 'cocoanut'], 'id': 283, 'def': 'large hard-shelled brown oval nut with a fibrous husk', 'name': 'coconut'}, {'frequency': 'f', 'synset': 'coffee_maker.n.01', 'synonyms': ['coffee_maker', 'coffee_machine'], 'id': 284, 'def': 'a kitchen appliance for brewing coffee automatically', 'name': 'coffee_maker'}, {'frequency': 'f', 'synset': 'coffee_table.n.01', 'synonyms': ['coffee_table', 'cocktail_table'], 'id': 285, 'def': 'low table where magazines can be placed and coffee or cocktails are served', 'name': 'coffee_table'}, {'frequency': 'c', 'synset': 'coffeepot.n.01', 'synonyms': ['coffeepot'], 'id': 286, 'def': 'tall pot in which coffee is brewed', 'name': 'coffeepot'}, {'frequency': 'r', 'synset': 'coil.n.05', 'synonyms': ['coil'], 'id': 287, 'def': 'tubing that is wound in a spiral', 'name': 'coil'}, {'frequency': 'c', 'synset': 'coin.n.01', 'synonyms': ['coin'], 'id': 288, 'def': 'a flat metal piece (usually a disc) used as money', 'name': 'coin'}, {'frequency': 'c', 'synset': 'colander.n.01', 'synonyms': ['colander', 'cullender'], 'id': 289, 'def': 'bowl-shaped strainer; used to wash or drain foods', 'name': 'colander'}, {'frequency': 'c', 'synset': 'coleslaw.n.01', 'synonyms': ['coleslaw', 'slaw'], 'id': 290, 'def': 'basically shredded cabbage', 'name': 'coleslaw'}, {'frequency': 'r', 'synset': 'coloring_material.n.01', 'synonyms': ['coloring_material', 'colouring_material'], 'id': 291, 'def': 'any material used for its color', 'name': 'coloring_material'}, {'frequency': 'r', 'synset': 'combination_lock.n.01', 'synonyms': ['combination_lock'], 'id': 292, 'def': 'lock that can be opened only by turning dials in a special sequence', 'name': 'combination_lock'}, {'frequency': 'c', 'synset': 'comforter.n.04', 'synonyms': ['pacifier', 'teething_ring'], 'id': 293, 'def': 'device used for an infant to suck or bite on', 'name': 'pacifier'}, {'frequency': 'r', 'synset': 'comic_book.n.01', 'synonyms': ['comic_book'], 'id': 294, 'def': 'a magazine devoted to comic strips', 'name': 'comic_book'}, {'frequency': 'r', 'synset': 'compass.n.01', 'synonyms': ['compass'], 'id': 295, 'def': 'navigational instrument for finding directions', 'name': 'compass'}, {'frequency': 'f', 'synset': 'computer_keyboard.n.01', 'synonyms': ['computer_keyboard', 'keyboard_(computer)'], 'id': 296, 'def': 'a keyboard that is a data input device for computers', 'name': 'computer_keyboard'}, {'frequency': 'f', 'synset': 'condiment.n.01', 'synonyms': ['condiment'], 'id': 297, 'def': 'a preparation (a sauce or relish or spice) to enhance flavor or enjoyment', 'name': 'condiment'}, {'frequency': 'f', 'synset': 'cone.n.01', 'synonyms': ['cone', 'traffic_cone'], 'id': 298, 'def': 'a cone-shaped object used to direct traffic', 'name': 'cone'}, {'frequency': 'f', 'synset': 'control.n.09', 'synonyms': ['control', 'controller'], 'id': 299, 'def': 'a mechanism that controls the operation of a machine', 'name': 'control'}, {'frequency': 'r', 'synset': 'convertible.n.01', 'synonyms': ['convertible_(automobile)'], 'id': 300, 'def': 'a car that has top that can be folded or removed', 'name': 'convertible_(automobile)'}, {'frequency': 'r', 'synset': 'convertible.n.03', 'synonyms': ['sofa_bed'], 'id': 301, 'def': 'a sofa that can be converted into a bed', 'name': 'sofa_bed'}, {'frequency': 'r', 'synset': 'cooker.n.01', 'synonyms': ['cooker'], 'id': 302, 'def': 'a utensil for cooking', 'name': 'cooker'}, {'frequency': 'f', 'synset': 'cookie.n.01', 'synonyms': ['cookie', 'cooky', 'biscuit_(cookie)'], 'id': 303, 'def': "any of various small flat sweet cakes (`biscuit' is the British term)", 'name': 'cookie'}, {'frequency': 'r', 'synset': 'cooking_utensil.n.01', 'synonyms': ['cooking_utensil'], 'id': 304, 'def': 'a kitchen utensil made of material that does not melt easily; used for cooking', 'name': 'cooking_utensil'}, {'frequency': 'f', 'synset': 'cooler.n.01', 'synonyms': ['cooler_(for_food)', 'ice_chest'], 'id': 305, 'def': 'an insulated box for storing food often with ice', 'name': 'cooler_(for_food)'}, {'frequency': 'f', 'synset': 'cork.n.04', 'synonyms': ['cork_(bottle_plug)', 'bottle_cork'], 'id': 306, 'def': 'the plug in the mouth of a bottle (especially a wine bottle)', 'name': 'cork_(bottle_plug)'}, {'frequency': 'r', 'synset': 'corkboard.n.01', 'synonyms': ['corkboard'], 'id': 307, 'def': 'a sheet consisting of cork granules', 'name': 'corkboard'}, {'frequency': 'c', 'synset': 'corkscrew.n.01', 'synonyms': ['corkscrew', 'bottle_screw'], 'id': 308, 'def': 'a bottle opener that pulls corks', 'name': 'corkscrew'}, {'frequency': 'f', 'synset': 'corn.n.03', 'synonyms': ['edible_corn', 'corn', 'maize'], 'id': 309, 'def': 'ears or kernels of corn that can be prepared and served for human food (only mark individual ears or kernels)', 'name': 'edible_corn'}, {'frequency': 'r', 'synset': 'cornbread.n.01', 'synonyms': ['cornbread'], 'id': 310, 'def': 'bread made primarily of cornmeal', 'name': 'cornbread'}, {'frequency': 'c', 'synset': 'cornet.n.01', 'synonyms': ['cornet', 'horn', 'trumpet'], 'id': 311, 'def': 'a brass musical instrument with a narrow tube and a flared bell and many valves', 'name': 'cornet'}, {'frequency': 'c', 'synset': 'cornice.n.01', 'synonyms': ['cornice', 'valance', 'valance_board', 'pelmet'], 'id': 312, 'def': 'a decorative framework to conceal curtain fixtures at the top of a window casing', 'name': 'cornice'}, {'frequency': 'r', 'synset': 'cornmeal.n.01', 'synonyms': ['cornmeal'], 'id': 313, 'def': 'coarsely ground corn', 'name': 'cornmeal'}, {'frequency': 'c', 'synset': 'corset.n.01', 'synonyms': ['corset', 'girdle'], 'id': 314, 'def': "a woman's close-fitting foundation garment", 'name': 'corset'}, {'frequency': 'c', 'synset': 'costume.n.04', 'synonyms': ['costume'], 'id': 315, 'def': 'the attire characteristic of a country or a time or a social class', 'name': 'costume'}, {'frequency': 'r', 'synset': 'cougar.n.01', 'synonyms': ['cougar', 'puma', 'catamount', 'mountain_lion', 'panther'], 'id': 316, 'def': 'large American feline resembling a lion', 'name': 'cougar'}, {'frequency': 'r', 'synset': 'coverall.n.01', 'synonyms': ['coverall'], 'id': 317, 'def': 'a loose-fitting protective garment that is worn over other clothing', 'name': 'coverall'}, {'frequency': 'c', 'synset': 'cowbell.n.01', 'synonyms': ['cowbell'], 'id': 318, 'def': 'a bell hung around the neck of cow so that the cow can be easily located', 'name': 'cowbell'}, {'frequency': 'f', 'synset': 'cowboy_hat.n.01', 'synonyms': ['cowboy_hat', 'ten-gallon_hat'], 'id': 319, 'def': 'a hat with a wide brim and a soft crown; worn by American ranch hands', 'name': 'cowboy_hat'}, {'frequency': 'c', 'synset': 'crab.n.01', 'synonyms': ['crab_(animal)'], 'id': 320, 'def': 'decapod having eyes on short stalks and a broad flattened shell and pincers', 'name': 'crab_(animal)'}, {'frequency': 'r', 'synset': 'crab.n.05', 'synonyms': ['crabmeat'], 'id': 321, 'def': 'the edible flesh of any of various crabs', 'name': 'crabmeat'}, {'frequency': 'c', 'synset': 'cracker.n.01', 'synonyms': ['cracker'], 'id': 322, 'def': 'a thin crisp wafer', 'name': 'cracker'}, {'frequency': 'r', 'synset': 'crape.n.01', 'synonyms': ['crape', 'crepe', 'French_pancake'], 'id': 323, 'def': 'small very thin pancake', 'name': 'crape'}, {'frequency': 'f', 'synset': 'crate.n.01', 'synonyms': ['crate'], 'id': 324, 'def': 'a rugged box (usually made of wood); used for shipping', 'name': 'crate'}, {'frequency': 'c', 'synset': 'crayon.n.01', 'synonyms': ['crayon', 'wax_crayon'], 'id': 325, 'def': 'writing or drawing implement made of a colored stick of composition wax', 'name': 'crayon'}, {'frequency': 'r', 'synset': 'cream_pitcher.n.01', 'synonyms': ['cream_pitcher'], 'id': 326, 'def': 'a small pitcher for serving cream', 'name': 'cream_pitcher'}, {'frequency': 'c', 'synset': 'crescent_roll.n.01', 'synonyms': ['crescent_roll', 'croissant'], 'id': 327, 'def': 'very rich flaky crescent-shaped roll', 'name': 'crescent_roll'}, {'frequency': 'c', 'synset': 'crib.n.01', 'synonyms': ['crib', 'cot'], 'id': 328, 'def': 'baby bed with high sides made of slats', 'name': 'crib'}, {'frequency': 'c', 'synset': 'crock.n.03', 'synonyms': ['crock_pot', 'earthenware_jar'], 'id': 329, 'def': 'an earthen jar (made of baked clay) or a modern electric crockpot', 'name': 'crock_pot'}, {'frequency': 'f', 'synset': 'crossbar.n.01', 'synonyms': ['crossbar'], 'id': 330, 'def': 'a horizontal bar that goes across something', 'name': 'crossbar'}, {'frequency': 'r', 'synset': 'crouton.n.01', 'synonyms': ['crouton'], 'id': 331, 'def': 'a small piece of toasted or fried bread; served in soup or salads', 'name': 'crouton'}, {'frequency': 'c', 'synset': 'crow.n.01', 'synonyms': ['crow'], 'id': 332, 'def': 'black birds having a raucous call', 'name': 'crow'}, {'frequency': 'r', 'synset': 'crowbar.n.01', 'synonyms': ['crowbar', 'wrecking_bar', 'pry_bar'], 'id': 333, 'def': 'a heavy iron lever with one end forged into a wedge', 'name': 'crowbar'}, {'frequency': 'c', 'synset': 'crown.n.04', 'synonyms': ['crown'], 'id': 334, 'def': 'an ornamental jeweled headdress signifying sovereignty', 'name': 'crown'}, {'frequency': 'c', 'synset': 'crucifix.n.01', 'synonyms': ['crucifix'], 'id': 335, 'def': 'representation of the cross on which Jesus died', 'name': 'crucifix'}, {'frequency': 'c', 'synset': 'cruise_ship.n.01', 'synonyms': ['cruise_ship', 'cruise_liner'], 'id': 336, 'def': 'a passenger ship used commercially for pleasure cruises', 'name': 'cruise_ship'}, {'frequency': 'c', 'synset': 'cruiser.n.01', 'synonyms': ['police_cruiser', 'patrol_car', 'police_car', 'squad_car'], 'id': 337, 'def': 'a car in which policemen cruise the streets', 'name': 'police_cruiser'}, {'frequency': 'f', 'synset': 'crumb.n.03', 'synonyms': ['crumb'], 'id': 338, 'def': 'small piece of e.g. bread or cake', 'name': 'crumb'}, {'frequency': 'c', 'synset': 'crutch.n.01', 'synonyms': ['crutch'], 'id': 339, 'def': 'a wooden or metal staff that fits under the armpit and reaches to the ground', 'name': 'crutch'}, {'frequency': 'c', 'synset': 'cub.n.03', 'synonyms': ['cub_(animal)'], 'id': 340, 'def': 'the young of certain carnivorous mammals such as the bear or wolf or lion', 'name': 'cub_(animal)'}, {'frequency': 'c', 'synset': 'cube.n.05', 'synonyms': ['cube', 'square_block'], 'id': 341, 'def': 'a block in the (approximate) shape of a cube', 'name': 'cube'}, {'frequency': 'f', 'synset': 'cucumber.n.02', 'synonyms': ['cucumber', 'cuke'], 'id': 342, 'def': 'cylindrical green fruit with thin green rind and white flesh eaten as a vegetable', 'name': 'cucumber'}, {'frequency': 'c', 'synset': 'cufflink.n.01', 'synonyms': ['cufflink'], 'id': 343, 'def': 'jewelry consisting of linked buttons used to fasten the cuffs of a shirt', 'name': 'cufflink'}, {'frequency': 'f', 'synset': 'cup.n.01', 'synonyms': ['cup'], 'id': 344, 'def': 'a small open container usually used for drinking; usually has a handle', 'name': 'cup'}, {'frequency': 'c', 'synset': 'cup.n.08', 'synonyms': ['trophy_cup'], 'id': 345, 'def': 'a metal award or cup-shaped vessel with handles that is awarded as a trophy to a competition winner', 'name': 'trophy_cup'}, {'frequency': 'f', 'synset': 'cupboard.n.01', 'synonyms': ['cupboard', 'closet'], 'id': 346, 'def': 'a small room (or recess) or cabinet used for storage space', 'name': 'cupboard'}, {'frequency': 'f', 'synset': 'cupcake.n.01', 'synonyms': ['cupcake'], 'id': 347, 'def': 'small cake baked in a muffin tin', 'name': 'cupcake'}, {'frequency': 'r', 'synset': 'curler.n.01', 'synonyms': ['hair_curler', 'hair_roller', 'hair_crimper'], 'id': 348, 'def': 'a cylindrical tube around which the hair is wound to curl it', 'name': 'hair_curler'}, {'frequency': 'r', 'synset': 'curling_iron.n.01', 'synonyms': ['curling_iron'], 'id': 349, 'def': 'a cylindrical home appliance that heats hair that has been curled around it', 'name': 'curling_iron'}, {'frequency': 'f', 'synset': 'curtain.n.01', 'synonyms': ['curtain', 'drapery'], 'id': 350, 'def': 'hanging cloth used as a blind (especially for a window)', 'name': 'curtain'}, {'frequency': 'f', 'synset': 'cushion.n.03', 'synonyms': ['cushion'], 'id': 351, 'def': 'a soft bag filled with air or padding such as feathers or foam rubber', 'name': 'cushion'}, {'frequency': 'r', 'synset': 'cylinder.n.04', 'synonyms': ['cylinder'], 'id': 352, 'def': 'a cylindrical container', 'name': 'cylinder'}, {'frequency': 'r', 'synset': 'cymbal.n.01', 'synonyms': ['cymbal'], 'id': 353, 'def': 'a percussion instrument consisting of a concave brass disk', 'name': 'cymbal'}, {'frequency': 'r', 'synset': 'dagger.n.01', 'synonyms': ['dagger'], 'id': 354, 'def': 'a short knife with a pointed blade used for piercing or stabbing', 'name': 'dagger'}, {'frequency': 'r', 'synset': 'dalmatian.n.02', 'synonyms': ['dalmatian'], 'id': 355, 'def': 'a large breed having a smooth white coat with black or brown spots', 'name': 'dalmatian'}, {'frequency': 'c', 'synset': 'dartboard.n.01', 'synonyms': ['dartboard'], 'id': 356, 'def': 'a circular board of wood or cork used as the target in the game of darts', 'name': 'dartboard'}, {'frequency': 'r', 'synset': 'date.n.08', 'synonyms': ['date_(fruit)'], 'id': 357, 'def': 'sweet edible fruit of the date palm with a single long woody seed', 'name': 'date_(fruit)'}, {'frequency': 'f', 'synset': 'deck_chair.n.01', 'synonyms': ['deck_chair', 'beach_chair'], 'id': 358, 'def': 'a folding chair for use outdoors; a wooden frame supports a length of canvas', 'name': 'deck_chair'}, {'frequency': 'c', 'synset': 'deer.n.01', 'synonyms': ['deer', 'cervid'], 'id': 359, 'def': "distinguished from Bovidae by the male's having solid deciduous antlers", 'name': 'deer'}, {'frequency': 'c', 'synset': 'dental_floss.n.01', 'synonyms': ['dental_floss', 'floss'], 'id': 360, 'def': 'a soft thread for cleaning the spaces between the teeth', 'name': 'dental_floss'}, {'frequency': 'f', 'synset': 'desk.n.01', 'synonyms': ['desk'], 'id': 361, 'def': 'a piece of furniture with a writing surface and usually drawers or other compartments', 'name': 'desk'}, {'frequency': 'r', 'synset': 'detergent.n.01', 'synonyms': ['detergent'], 'id': 362, 'def': 'a surface-active chemical widely used in industry and laundering', 'name': 'detergent'}, {'frequency': 'c', 'synset': 'diaper.n.01', 'synonyms': ['diaper'], 'id': 363, 'def': 'garment consisting of a folded cloth drawn up between the legs and fastened at the waist', 'name': 'diaper'}, {'frequency': 'r', 'synset': 'diary.n.01', 'synonyms': ['diary', 'journal'], 'id': 364, 'def': 'yearly planner book', 'name': 'diary'}, {'frequency': 'r', 'synset': 'die.n.01', 'synonyms': ['die', 'dice'], 'id': 365, 'def': 'a small cube with 1 to 6 spots on the six faces; used in gambling', 'name': 'die'}, {'frequency': 'r', 'synset': 'dinghy.n.01', 'synonyms': ['dinghy', 'dory', 'rowboat'], 'id': 366, 'def': 'a small boat of shallow draft with seats and oars with which it is propelled', 'name': 'dinghy'}, {'frequency': 'f', 'synset': 'dining_table.n.01', 'synonyms': ['dining_table'], 'id': 367, 'def': 'a table at which meals are served', 'name': 'dining_table'}, {'frequency': 'r', 'synset': 'dinner_jacket.n.01', 'synonyms': ['tux', 'tuxedo'], 'id': 368, 'def': 'semiformal evening dress for men', 'name': 'tux'}, {'frequency': 'f', 'synset': 'dish.n.01', 'synonyms': ['dish'], 'id': 369, 'def': 'a piece of dishware normally used as a container for holding or serving food', 'name': 'dish'}, {'frequency': 'c', 'synset': 'dish.n.05', 'synonyms': ['dish_antenna'], 'id': 370, 'def': 'directional antenna consisting of a parabolic reflector', 'name': 'dish_antenna'}, {'frequency': 'c', 'synset': 'dishrag.n.01', 'synonyms': ['dishrag', 'dishcloth'], 'id': 371, 'def': 'a cloth for washing dishes or cleaning in general', 'name': 'dishrag'}, {'frequency': 'f', 'synset': 'dishtowel.n.01', 'synonyms': ['dishtowel', 'tea_towel'], 'id': 372, 'def': 'a towel for drying dishes', 'name': 'dishtowel'}, {'frequency': 'f', 'synset': 'dishwasher.n.01', 'synonyms': ['dishwasher', 'dishwashing_machine'], 'id': 373, 'def': 'a machine for washing dishes', 'name': 'dishwasher'}, {'frequency': 'r', 'synset': 'dishwasher_detergent.n.01', 'synonyms': ['dishwasher_detergent', 'dishwashing_detergent', 'dishwashing_liquid', 'dishsoap'], 'id': 374, 'def': 'dishsoap or dish detergent designed for use in dishwashers', 'name': 'dishwasher_detergent'}, {'frequency': 'f', 'synset': 'dispenser.n.01', 'synonyms': ['dispenser'], 'id': 375, 'def': 'a container so designed that the contents can be used in prescribed amounts', 'name': 'dispenser'}, {'frequency': 'r', 'synset': 'diving_board.n.01', 'synonyms': ['diving_board'], 'id': 376, 'def': 'a springboard from which swimmers can dive', 'name': 'diving_board'}, {'frequency': 'f', 'synset': 'dixie_cup.n.01', 'synonyms': ['Dixie_cup', 'paper_cup'], 'id': 377, 'def': 'a disposable cup made of paper; for holding drinks', 'name': 'Dixie_cup'}, {'frequency': 'f', 'synset': 'dog.n.01', 'synonyms': ['dog'], 'id': 378, 'def': 'a common domesticated dog', 'name': 'dog'}, {'frequency': 'f', 'synset': 'dog_collar.n.01', 'synonyms': ['dog_collar'], 'id': 379, 'def': 'a collar for a dog', 'name': 'dog_collar'}, {'frequency': 'f', 'synset': 'doll.n.01', 'synonyms': ['doll'], 'id': 380, 'def': 'a toy replica of a HUMAN (NOT AN ANIMAL)', 'name': 'doll'}, {'frequency': 'r', 'synset': 'dollar.n.02', 'synonyms': ['dollar', 'dollar_bill', 'one_dollar_bill'], 'id': 381, 'def': 'a piece of paper money worth one dollar', 'name': 'dollar'}, {'frequency': 'r', 'synset': 'dollhouse.n.01', 'synonyms': ['dollhouse', "doll's_house"], 'id': 382, 'def': "a house so small that it is likened to a child's plaything", 'name': 'dollhouse'}, {'frequency': 'c', 'synset': 'dolphin.n.02', 'synonyms': ['dolphin'], 'id': 383, 'def': 'any of various small toothed whales with a beaklike snout; larger than porpoises', 'name': 'dolphin'}, {'frequency': 'c', 'synset': 'domestic_ass.n.01', 'synonyms': ['domestic_ass', 'donkey'], 'id': 384, 'def': 'domestic beast of burden descended from the African wild ass; patient but stubborn', 'name': 'domestic_ass'}, {'frequency': 'f', 'synset': 'doorknob.n.01', 'synonyms': ['doorknob', 'doorhandle'], 'id': 385, 'def': "a knob used to open a door (often called `doorhandle' in Great Britain)", 'name': 'doorknob'}, {'frequency': 'c', 'synset': 'doormat.n.02', 'synonyms': ['doormat', 'welcome_mat'], 'id': 386, 'def': 'a mat placed outside an exterior door for wiping the shoes before entering', 'name': 'doormat'}, {'frequency': 'f', 'synset': 'doughnut.n.02', 'synonyms': ['doughnut', 'donut'], 'id': 387, 'def': 'a small ring-shaped friedcake', 'name': 'doughnut'}, {'frequency': 'r', 'synset': 'dove.n.01', 'synonyms': ['dove'], 'id': 388, 'def': 'any of numerous small pigeons', 'name': 'dove'}, {'frequency': 'r', 'synset': 'dragonfly.n.01', 'synonyms': ['dragonfly'], 'id': 389, 'def': 'slender-bodied non-stinging insect having iridescent wings that are outspread at rest', 'name': 'dragonfly'}, {'frequency': 'f', 'synset': 'drawer.n.01', 'synonyms': ['drawer'], 'id': 390, 'def': 'a boxlike container in a piece of furniture; made so as to slide in and out', 'name': 'drawer'}, {'frequency': 'c', 'synset': 'drawers.n.01', 'synonyms': ['underdrawers', 'boxers', 'boxershorts'], 'id': 391, 'def': 'underpants worn by men', 'name': 'underdrawers'}, {'frequency': 'f', 'synset': 'dress.n.01', 'synonyms': ['dress', 'frock'], 'id': 392, 'def': 'a one-piece garment for a woman; has skirt and bodice', 'name': 'dress'}, {'frequency': 'c', 'synset': 'dress_hat.n.01', 'synonyms': ['dress_hat', 'high_hat', 'opera_hat', 'silk_hat', 'top_hat'], 'id': 393, 'def': "a man's hat with a tall crown; usually covered with silk or with beaver fur", 'name': 'dress_hat'}, {'frequency': 'f', 'synset': 'dress_suit.n.01', 'synonyms': ['dress_suit'], 'id': 394, 'def': 'formalwear consisting of full evening dress for men', 'name': 'dress_suit'}, {'frequency': 'f', 'synset': 'dresser.n.05', 'synonyms': ['dresser'], 'id': 395, 'def': 'a cabinet with shelves', 'name': 'dresser'}, {'frequency': 'c', 'synset': 'drill.n.01', 'synonyms': ['drill'], 'id': 396, 'def': 'a tool with a sharp rotating point for making holes in hard materials', 'name': 'drill'}, {'frequency': 'r', 'synset': 'drone.n.04', 'synonyms': ['drone'], 'id': 397, 'def': 'an aircraft without a pilot that is operated by remote control', 'name': 'drone'}, {'frequency': 'r', 'synset': 'dropper.n.01', 'synonyms': ['dropper', 'eye_dropper'], 'id': 398, 'def': 'pipet consisting of a small tube with a vacuum bulb at one end for drawing liquid in and releasing it a drop at a time', 'name': 'dropper'}, {'frequency': 'c', 'synset': 'drum.n.01', 'synonyms': ['drum_(musical_instrument)'], 'id': 399, 'def': 'a musical percussion instrument; usually consists of a hollow cylinder with a membrane stretched across each end', 'name': 'drum_(musical_instrument)'}, {'frequency': 'r', 'synset': 'drumstick.n.02', 'synonyms': ['drumstick'], 'id': 400, 'def': 'a stick used for playing a drum', 'name': 'drumstick'}, {'frequency': 'f', 'synset': 'duck.n.01', 'synonyms': ['duck'], 'id': 401, 'def': 'small web-footed broad-billed swimming bird', 'name': 'duck'}, {'frequency': 'c', 'synset': 'duckling.n.02', 'synonyms': ['duckling'], 'id': 402, 'def': 'young duck', 'name': 'duckling'}, {'frequency': 'c', 'synset': 'duct_tape.n.01', 'synonyms': ['duct_tape'], 'id': 403, 'def': 'a wide silvery adhesive tape', 'name': 'duct_tape'}, {'frequency': 'f', 'synset': 'duffel_bag.n.01', 'synonyms': ['duffel_bag', 'duffle_bag', 'duffel', 'duffle'], 'id': 404, 'def': 'a large cylindrical bag of heavy cloth (does not include suitcases)', 'name': 'duffel_bag'}, {'frequency': 'r', 'synset': 'dumbbell.n.01', 'synonyms': ['dumbbell'], 'id': 405, 'def': 'an exercising weight with two ball-like ends connected by a short handle', 'name': 'dumbbell'}, {'frequency': 'c', 'synset': 'dumpster.n.01', 'synonyms': ['dumpster'], 'id': 406, 'def': 'a container designed to receive and transport and dump waste', 'name': 'dumpster'}, {'frequency': 'r', 'synset': 'dustpan.n.02', 'synonyms': ['dustpan'], 'id': 407, 'def': 'a short-handled receptacle into which dust can be swept', 'name': 'dustpan'}, {'frequency': 'c', 'synset': 'eagle.n.01', 'synonyms': ['eagle'], 'id': 408, 'def': 'large birds of prey noted for their broad wings and strong soaring flight', 'name': 'eagle'}, {'frequency': 'f', 'synset': 'earphone.n.01', 'synonyms': ['earphone', 'earpiece', 'headphone'], 'id': 409, 'def': 'device for listening to audio that is held over or inserted into the ear', 'name': 'earphone'}, {'frequency': 'r', 'synset': 'earplug.n.01', 'synonyms': ['earplug'], 'id': 410, 'def': 'a soft plug that is inserted into the ear canal to block sound', 'name': 'earplug'}, {'frequency': 'f', 'synset': 'earring.n.01', 'synonyms': ['earring'], 'id': 411, 'def': 'jewelry to ornament the ear', 'name': 'earring'}, {'frequency': 'c', 'synset': 'easel.n.01', 'synonyms': ['easel'], 'id': 412, 'def': "an upright tripod for displaying something (usually an artist's canvas)", 'name': 'easel'}, {'frequency': 'r', 'synset': 'eclair.n.01', 'synonyms': ['eclair'], 'id': 413, 'def': 'oblong cream puff', 'name': 'eclair'}, {'frequency': 'r', 'synset': 'eel.n.01', 'synonyms': ['eel'], 'id': 414, 'def': 'an elongate fish with fatty flesh', 'name': 'eel'}, {'frequency': 'f', 'synset': 'egg.n.02', 'synonyms': ['egg', 'eggs'], 'id': 415, 'def': 'oval reproductive body of a fowl (especially a hen) used as food', 'name': 'egg'}, {'frequency': 'r', 'synset': 'egg_roll.n.01', 'synonyms': ['egg_roll', 'spring_roll'], 'id': 416, 'def': 'minced vegetables and meat wrapped in a pancake and fried', 'name': 'egg_roll'}, {'frequency': 'c', 'synset': 'egg_yolk.n.01', 'synonyms': ['egg_yolk', 'yolk_(egg)'], 'id': 417, 'def': 'the yellow spherical part of an egg', 'name': 'egg_yolk'}, {'frequency': 'c', 'synset': 'eggbeater.n.02', 'synonyms': ['eggbeater', 'eggwhisk'], 'id': 418, 'def': 'a mixer for beating eggs or whipping cream', 'name': 'eggbeater'}, {'frequency': 'c', 'synset': 'eggplant.n.01', 'synonyms': ['eggplant', 'aubergine'], 'id': 419, 'def': 'egg-shaped vegetable having a shiny skin typically dark purple', 'name': 'eggplant'}, {'frequency': 'r', 'synset': 'electric_chair.n.01', 'synonyms': ['electric_chair'], 'id': 420, 'def': 'a chair-shaped instrument of execution by electrocution', 'name': 'electric_chair'}, {'frequency': 'f', 'synset': 'electric_refrigerator.n.01', 'synonyms': ['refrigerator'], 'id': 421, 'def': 'a refrigerator in which the coolant is pumped around by an electric motor', 'name': 'refrigerator'}, {'frequency': 'f', 'synset': 'elephant.n.01', 'synonyms': ['elephant'], 'id': 422, 'def': 'a common elephant', 'name': 'elephant'}, {'frequency': 'c', 'synset': 'elk.n.01', 'synonyms': ['elk', 'moose'], 'id': 423, 'def': 'large northern deer with enormous flattened antlers in the male', 'name': 'elk'}, {'frequency': 'c', 'synset': 'envelope.n.01', 'synonyms': ['envelope'], 'id': 424, 'def': 'a flat (usually rectangular) container for a letter, thin package, etc.', 'name': 'envelope'}, {'frequency': 'c', 'synset': 'eraser.n.01', 'synonyms': ['eraser'], 'id': 425, 'def': 'an implement used to erase something', 'name': 'eraser'}, {'frequency': 'r', 'synset': 'escargot.n.01', 'synonyms': ['escargot'], 'id': 426, 'def': 'edible snail usually served in the shell with a sauce of melted butter and garlic', 'name': 'escargot'}, {'frequency': 'r', 'synset': 'eyepatch.n.01', 'synonyms': ['eyepatch'], 'id': 427, 'def': 'a protective cloth covering for an injured eye', 'name': 'eyepatch'}, {'frequency': 'r', 'synset': 'falcon.n.01', 'synonyms': ['falcon'], 'id': 428, 'def': 'birds of prey having long pointed powerful wings adapted for swift flight', 'name': 'falcon'}, {'frequency': 'f', 'synset': 'fan.n.01', 'synonyms': ['fan'], 'id': 429, 'def': 'a device for creating a current of air by movement of a surface or surfaces', 'name': 'fan'}, {'frequency': 'f', 'synset': 'faucet.n.01', 'synonyms': ['faucet', 'spigot', 'tap'], 'id': 430, 'def': 'a regulator for controlling the flow of a liquid from a reservoir', 'name': 'faucet'}, {'frequency': 'r', 'synset': 'fedora.n.01', 'synonyms': ['fedora'], 'id': 431, 'def': 'a hat made of felt with a creased crown', 'name': 'fedora'}, {'frequency': 'r', 'synset': 'ferret.n.02', 'synonyms': ['ferret'], 'id': 432, 'def': 'domesticated albino variety of the European polecat bred for hunting rats and rabbits', 'name': 'ferret'}, {'frequency': 'c', 'synset': 'ferris_wheel.n.01', 'synonyms': ['Ferris_wheel'], 'id': 433, 'def': 'a large wheel with suspended seats that remain upright as the wheel rotates', 'name': 'Ferris_wheel'}, {'frequency': 'c', 'synset': 'ferry.n.01', 'synonyms': ['ferry', 'ferryboat'], 'id': 434, 'def': 'a boat that transports people or vehicles across a body of water and operates on a regular schedule', 'name': 'ferry'}, {'frequency': 'r', 'synset': 'fig.n.04', 'synonyms': ['fig_(fruit)'], 'id': 435, 'def': 'fleshy sweet pear-shaped yellowish or purple fruit eaten fresh or preserved or dried', 'name': 'fig_(fruit)'}, {'frequency': 'c', 'synset': 'fighter.n.02', 'synonyms': ['fighter_jet', 'fighter_aircraft', 'attack_aircraft'], 'id': 436, 'def': 'a high-speed military or naval airplane designed to destroy enemy targets', 'name': 'fighter_jet'}, {'frequency': 'f', 'synset': 'figurine.n.01', 'synonyms': ['figurine'], 'id': 437, 'def': 'a small carved or molded figure', 'name': 'figurine'}, {'frequency': 'c', 'synset': 'file.n.03', 'synonyms': ['file_cabinet', 'filing_cabinet'], 'id': 438, 'def': 'office furniture consisting of a container for keeping papers in order', 'name': 'file_cabinet'}, {'frequency': 'r', 'synset': 'file.n.04', 'synonyms': ['file_(tool)'], 'id': 439, 'def': 'a steel hand tool with small sharp teeth on some or all of its surfaces; used for smoothing wood or metal', 'name': 'file_(tool)'}, {'frequency': 'f', 'synset': 'fire_alarm.n.02', 'synonyms': ['fire_alarm', 'smoke_alarm'], 'id': 440, 'def': 'an alarm that is tripped off by fire or smoke', 'name': 'fire_alarm'}, {'frequency': 'f', 'synset': 'fire_engine.n.01', 'synonyms': ['fire_engine', 'fire_truck'], 'id': 441, 'def': 'large trucks that carry firefighters and equipment to the site of a fire', 'name': 'fire_engine'}, {'frequency': 'f', 'synset': 'fire_extinguisher.n.01', 'synonyms': ['fire_extinguisher', 'extinguisher'], 'id': 442, 'def': 'a manually operated device for extinguishing small fires', 'name': 'fire_extinguisher'}, {'frequency': 'c', 'synset': 'fire_hose.n.01', 'synonyms': ['fire_hose'], 'id': 443, 'def': 'a large hose that carries water from a fire hydrant to the site of the fire', 'name': 'fire_hose'}, {'frequency': 'f', 'synset': 'fireplace.n.01', 'synonyms': ['fireplace'], 'id': 444, 'def': 'an open recess in a wall at the base of a chimney where a fire can be built', 'name': 'fireplace'}, {'frequency': 'f', 'synset': 'fireplug.n.01', 'synonyms': ['fireplug', 'fire_hydrant', 'hydrant'], 'id': 445, 'def': 'an upright hydrant for drawing water to use in fighting a fire', 'name': 'fireplug'}, {'frequency': 'r', 'synset': 'first-aid_kit.n.01', 'synonyms': ['first-aid_kit'], 'id': 446, 'def': 'kit consisting of a set of bandages and medicines for giving first aid', 'name': 'first-aid_kit'}, {'frequency': 'f', 'synset': 'fish.n.01', 'synonyms': ['fish'], 'id': 447, 'def': 'any of various mostly cold-blooded aquatic vertebrates usually having scales and breathing through gills', 'name': 'fish'}, {'frequency': 'c', 'synset': 'fish.n.02', 'synonyms': ['fish_(food)'], 'id': 448, 'def': 'the flesh of fish used as food', 'name': 'fish_(food)'}, {'frequency': 'r', 'synset': 'fishbowl.n.02', 'synonyms': ['fishbowl', 'goldfish_bowl'], 'id': 449, 'def': 'a transparent bowl in which small fish are kept', 'name': 'fishbowl'}, {'frequency': 'c', 'synset': 'fishing_rod.n.01', 'synonyms': ['fishing_rod', 'fishing_pole'], 'id': 450, 'def': 'a rod that is used in fishing to extend the fishing line', 'name': 'fishing_rod'}, {'frequency': 'f', 'synset': 'flag.n.01', 'synonyms': ['flag'], 'id': 451, 'def': 'emblem usually consisting of a rectangular piece of cloth of distinctive design (do not include pole)', 'name': 'flag'}, {'frequency': 'f', 'synset': 'flagpole.n.02', 'synonyms': ['flagpole', 'flagstaff'], 'id': 452, 'def': 'a tall staff or pole on which a flag is raised', 'name': 'flagpole'}, {'frequency': 'c', 'synset': 'flamingo.n.01', 'synonyms': ['flamingo'], 'id': 453, 'def': 'large pink web-footed bird with down-bent bill', 'name': 'flamingo'}, {'frequency': 'c', 'synset': 'flannel.n.01', 'synonyms': ['flannel'], 'id': 454, 'def': 'a soft light woolen fabric; used for clothing', 'name': 'flannel'}, {'frequency': 'c', 'synset': 'flap.n.01', 'synonyms': ['flap'], 'id': 455, 'def': 'any broad thin covering attached at one edge, such as a mud flap next to a wheel or a flap on an airplane wing', 'name': 'flap'}, {'frequency': 'r', 'synset': 'flash.n.10', 'synonyms': ['flash', 'flashbulb'], 'id': 456, 'def': 'a lamp for providing momentary light to take a photograph', 'name': 'flash'}, {'frequency': 'c', 'synset': 'flashlight.n.01', 'synonyms': ['flashlight', 'torch'], 'id': 457, 'def': 'a small portable battery-powered electric lamp', 'name': 'flashlight'}, {'frequency': 'r', 'synset': 'fleece.n.03', 'synonyms': ['fleece'], 'id': 458, 'def': 'a soft bulky fabric with deep pile; used chiefly for clothing', 'name': 'fleece'}, {'frequency': 'f', 'synset': 'flip-flop.n.02', 'synonyms': ['flip-flop_(sandal)'], 'id': 459, 'def': 'a backless sandal held to the foot by a thong between two toes', 'name': 'flip-flop_(sandal)'}, {'frequency': 'c', 'synset': 'flipper.n.01', 'synonyms': ['flipper_(footwear)', 'fin_(footwear)'], 'id': 460, 'def': 'a shoe to aid a person in swimming', 'name': 'flipper_(footwear)'}, {'frequency': 'f', 'synset': 'flower_arrangement.n.01', 'synonyms': ['flower_arrangement', 'floral_arrangement'], 'id': 461, 'def': 'a decorative arrangement of flowers', 'name': 'flower_arrangement'}, {'frequency': 'c', 'synset': 'flute.n.02', 'synonyms': ['flute_glass', 'champagne_flute'], 'id': 462, 'def': 'a tall narrow wineglass', 'name': 'flute_glass'}, {'frequency': 'c', 'synset': 'foal.n.01', 'synonyms': ['foal'], 'id': 463, 'def': 'a young horse', 'name': 'foal'}, {'frequency': 'c', 'synset': 'folding_chair.n.01', 'synonyms': ['folding_chair'], 'id': 464, 'def': 'a chair that can be folded flat for storage', 'name': 'folding_chair'}, {'frequency': 'c', 'synset': 'food_processor.n.01', 'synonyms': ['food_processor'], 'id': 465, 'def': 'a kitchen appliance for shredding, blending, chopping, or slicing food', 'name': 'food_processor'}, {'frequency': 'c', 'synset': 'football.n.02', 'synonyms': ['football_(American)'], 'id': 466, 'def': 'the inflated oblong ball used in playing American football', 'name': 'football_(American)'}, {'frequency': 'r', 'synset': 'football_helmet.n.01', 'synonyms': ['football_helmet'], 'id': 467, 'def': 'a padded helmet with a face mask to protect the head of football players', 'name': 'football_helmet'}, {'frequency': 'c', 'synset': 'footstool.n.01', 'synonyms': ['footstool', 'footrest'], 'id': 468, 'def': 'a low seat or a stool to rest the feet of a seated person', 'name': 'footstool'}, {'frequency': 'f', 'synset': 'fork.n.01', 'synonyms': ['fork'], 'id': 469, 'def': 'cutlery used for serving and eating food', 'name': 'fork'}, {'frequency': 'c', 'synset': 'forklift.n.01', 'synonyms': ['forklift'], 'id': 470, 'def': 'an industrial vehicle with a power operated fork in front that can be inserted under loads to lift and move them', 'name': 'forklift'}, {'frequency': 'c', 'synset': 'freight_car.n.01', 'synonyms': ['freight_car'], 'id': 471, 'def': 'a railway car that carries freight', 'name': 'freight_car'}, {'frequency': 'c', 'synset': 'french_toast.n.01', 'synonyms': ['French_toast'], 'id': 472, 'def': 'bread slice dipped in egg and milk and fried', 'name': 'French_toast'}, {'frequency': 'c', 'synset': 'freshener.n.01', 'synonyms': ['freshener', 'air_freshener'], 'id': 473, 'def': 'anything that freshens air by removing or covering odor', 'name': 'freshener'}, {'frequency': 'f', 'synset': 'frisbee.n.01', 'synonyms': ['frisbee'], 'id': 474, 'def': 'a light, plastic disk propelled with a flip of the wrist for recreation or competition', 'name': 'frisbee'}, {'frequency': 'c', 'synset': 'frog.n.01', 'synonyms': ['frog', 'toad', 'toad_frog'], 'id': 475, 'def': 'a tailless stout-bodied amphibians with long hind limbs for leaping', 'name': 'frog'}, {'frequency': 'c', 'synset': 'fruit_juice.n.01', 'synonyms': ['fruit_juice'], 'id': 476, 'def': 'drink produced by squeezing or crushing fruit', 'name': 'fruit_juice'}, {'frequency': 'f', 'synset': 'frying_pan.n.01', 'synonyms': ['frying_pan', 'frypan', 'skillet'], 'id': 477, 'def': 'a pan used for frying foods', 'name': 'frying_pan'}, {'frequency': 'r', 'synset': 'fudge.n.01', 'synonyms': ['fudge'], 'id': 478, 'def': 'soft creamy candy', 'name': 'fudge'}, {'frequency': 'r', 'synset': 'funnel.n.02', 'synonyms': ['funnel'], 'id': 479, 'def': 'a cone-shaped utensil used to channel a substance into a container with a small mouth', 'name': 'funnel'}, {'frequency': 'r', 'synset': 'futon.n.01', 'synonyms': ['futon'], 'id': 480, 'def': 'a pad that is used for sleeping on the floor or on a raised frame', 'name': 'futon'}, {'frequency': 'r', 'synset': 'gag.n.02', 'synonyms': ['gag', 'muzzle'], 'id': 481, 'def': "restraint put into a person's mouth to prevent speaking or shouting", 'name': 'gag'}, {'frequency': 'r', 'synset': 'garbage.n.03', 'synonyms': ['garbage'], 'id': 482, 'def': 'a receptacle where waste can be discarded', 'name': 'garbage'}, {'frequency': 'c', 'synset': 'garbage_truck.n.01', 'synonyms': ['garbage_truck'], 'id': 483, 'def': 'a truck for collecting domestic refuse', 'name': 'garbage_truck'}, {'frequency': 'c', 'synset': 'garden_hose.n.01', 'synonyms': ['garden_hose'], 'id': 484, 'def': 'a hose used for watering a lawn or garden', 'name': 'garden_hose'}, {'frequency': 'c', 'synset': 'gargle.n.01', 'synonyms': ['gargle', 'mouthwash'], 'id': 485, 'def': 'a medicated solution used for gargling and rinsing the mouth', 'name': 'gargle'}, {'frequency': 'r', 'synset': 'gargoyle.n.02', 'synonyms': ['gargoyle'], 'id': 486, 'def': 'an ornament consisting of a grotesquely carved figure of a person or animal', 'name': 'gargoyle'}, {'frequency': 'c', 'synset': 'garlic.n.02', 'synonyms': ['garlic', 'ail'], 'id': 487, 'def': 'aromatic bulb used as seasoning', 'name': 'garlic'}, {'frequency': 'r', 'synset': 'gasmask.n.01', 'synonyms': ['gasmask', 'respirator', 'gas_helmet'], 'id': 488, 'def': 'a protective face mask with a filter', 'name': 'gasmask'}, {'frequency': 'c', 'synset': 'gazelle.n.01', 'synonyms': ['gazelle'], 'id': 489, 'def': 'small swift graceful antelope of Africa and Asia having lustrous eyes', 'name': 'gazelle'}, {'frequency': 'c', 'synset': 'gelatin.n.02', 'synonyms': ['gelatin', 'jelly'], 'id': 490, 'def': 'an edible jelly made with gelatin and used as a dessert or salad base or a coating for foods', 'name': 'gelatin'}, {'frequency': 'r', 'synset': 'gem.n.02', 'synonyms': ['gemstone'], 'id': 491, 'def': 'a crystalline rock that can be cut and polished for jewelry', 'name': 'gemstone'}, {'frequency': 'r', 'synset': 'generator.n.02', 'synonyms': ['generator'], 'id': 492, 'def': 'engine that converts mechanical energy into electrical energy by electromagnetic induction', 'name': 'generator'}, {'frequency': 'c', 'synset': 'giant_panda.n.01', 'synonyms': ['giant_panda', 'panda', 'panda_bear'], 'id': 493, 'def': 'large black-and-white herbivorous mammal of bamboo forests of China and Tibet', 'name': 'giant_panda'}, {'frequency': 'c', 'synset': 'gift_wrap.n.01', 'synonyms': ['gift_wrap'], 'id': 494, 'def': 'attractive wrapping paper suitable for wrapping gifts', 'name': 'gift_wrap'}, {'frequency': 'c', 'synset': 'ginger.n.03', 'synonyms': ['ginger', 'gingerroot'], 'id': 495, 'def': 'the root of the common ginger plant; used fresh as a seasoning', 'name': 'ginger'}, {'frequency': 'f', 'synset': 'giraffe.n.01', 'synonyms': ['giraffe'], 'id': 496, 'def': 'tall animal having a spotted coat and small horns and very long neck and legs', 'name': 'giraffe'}, {'frequency': 'c', 'synset': 'girdle.n.02', 'synonyms': ['cincture', 'sash', 'waistband', 'waistcloth'], 'id': 497, 'def': 'a band of material around the waist that strengthens a skirt or trousers', 'name': 'cincture'}, {'frequency': 'f', 'synset': 'glass.n.02', 'synonyms': ['glass_(drink_container)', 'drinking_glass'], 'id': 498, 'def': 'a container for holding liquids while drinking', 'name': 'glass_(drink_container)'}, {'frequency': 'c', 'synset': 'globe.n.03', 'synonyms': ['globe'], 'id': 499, 'def': 'a sphere on which a map (especially of the earth) is represented', 'name': 'globe'}, {'frequency': 'f', 'synset': 'glove.n.02', 'synonyms': ['glove'], 'id': 500, 'def': 'handwear covering the hand', 'name': 'glove'}, {'frequency': 'c', 'synset': 'goat.n.01', 'synonyms': ['goat'], 'id': 501, 'def': 'a common goat', 'name': 'goat'}, {'frequency': 'f', 'synset': 'goggles.n.01', 'synonyms': ['goggles'], 'id': 502, 'def': 'tight-fitting spectacles worn to protect the eyes', 'name': 'goggles'}, {'frequency': 'r', 'synset': 'goldfish.n.01', 'synonyms': ['goldfish'], 'id': 503, 'def': 'small golden or orange-red freshwater fishes used as pond or aquarium pets', 'name': 'goldfish'}, {'frequency': 'c', 'synset': 'golf_club.n.02', 'synonyms': ['golf_club', 'golf-club'], 'id': 504, 'def': 'golf equipment used by a golfer to hit a golf ball', 'name': 'golf_club'}, {'frequency': 'c', 'synset': 'golfcart.n.01', 'synonyms': ['golfcart'], 'id': 505, 'def': 'a small motor vehicle in which golfers can ride between shots', 'name': 'golfcart'}, {'frequency': 'r', 'synset': 'gondola.n.02', 'synonyms': ['gondola_(boat)'], 'id': 506, 'def': 'long narrow flat-bottomed boat propelled by sculling; traditionally used on canals of Venice', 'name': 'gondola_(boat)'}, {'frequency': 'c', 'synset': 'goose.n.01', 'synonyms': ['goose'], 'id': 507, 'def': 'loud, web-footed long-necked aquatic birds usually larger than ducks', 'name': 'goose'}, {'frequency': 'r', 'synset': 'gorilla.n.01', 'synonyms': ['gorilla'], 'id': 508, 'def': 'largest ape', 'name': 'gorilla'}, {'frequency': 'r', 'synset': 'gourd.n.02', 'synonyms': ['gourd'], 'id': 509, 'def': 'any of numerous inedible fruits with hard rinds', 'name': 'gourd'}, {'frequency': 'f', 'synset': 'grape.n.01', 'synonyms': ['grape'], 'id': 510, 'def': 'any of various juicy fruit with green or purple skins; grow in clusters', 'name': 'grape'}, {'frequency': 'c', 'synset': 'grater.n.01', 'synonyms': ['grater'], 'id': 511, 'def': 'utensil with sharp perforations for shredding foods (as vegetables or cheese)', 'name': 'grater'}, {'frequency': 'c', 'synset': 'gravestone.n.01', 'synonyms': ['gravestone', 'headstone', 'tombstone'], 'id': 512, 'def': 'a stone that is used to mark a grave', 'name': 'gravestone'}, {'frequency': 'r', 'synset': 'gravy_boat.n.01', 'synonyms': ['gravy_boat', 'gravy_holder'], 'id': 513, 'def': 'a dish (often boat-shaped) for serving gravy or sauce', 'name': 'gravy_boat'}, {'frequency': 'f', 'synset': 'green_bean.n.02', 'synonyms': ['green_bean'], 'id': 514, 'def': 'a common bean plant cultivated for its slender green edible pods', 'name': 'green_bean'}, {'frequency': 'f', 'synset': 'green_onion.n.01', 'synonyms': ['green_onion', 'spring_onion', 'scallion'], 'id': 515, 'def': 'a young onion before the bulb has enlarged', 'name': 'green_onion'}, {'frequency': 'r', 'synset': 'griddle.n.01', 'synonyms': ['griddle'], 'id': 516, 'def': 'cooking utensil consisting of a flat heated surface on which food is cooked', 'name': 'griddle'}, {'frequency': 'f', 'synset': 'grill.n.02', 'synonyms': ['grill', 'grille', 'grillwork', 'radiator_grille'], 'id': 517, 'def': 'a framework of metal bars used as a partition or a grate', 'name': 'grill'}, {'frequency': 'r', 'synset': 'grits.n.01', 'synonyms': ['grits', 'hominy_grits'], 'id': 518, 'def': 'coarsely ground corn boiled as a breakfast dish', 'name': 'grits'}, {'frequency': 'c', 'synset': 'grizzly.n.01', 'synonyms': ['grizzly', 'grizzly_bear'], 'id': 519, 'def': 'powerful brownish-yellow bear of the uplands of western North America', 'name': 'grizzly'}, {'frequency': 'c', 'synset': 'grocery_bag.n.01', 'synonyms': ['grocery_bag'], 'id': 520, 'def': "a sack for holding customer's groceries", 'name': 'grocery_bag'}, {'frequency': 'f', 'synset': 'guitar.n.01', 'synonyms': ['guitar'], 'id': 521, 'def': 'a stringed instrument usually having six strings; played by strumming or plucking', 'name': 'guitar'}, {'frequency': 'c', 'synset': 'gull.n.02', 'synonyms': ['gull', 'seagull'], 'id': 522, 'def': 'mostly white aquatic bird having long pointed wings and short legs', 'name': 'gull'}, {'frequency': 'c', 'synset': 'gun.n.01', 'synonyms': ['gun'], 'id': 523, 'def': 'a weapon that discharges a bullet at high velocity from a metal tube', 'name': 'gun'}, {'frequency': 'f', 'synset': 'hairbrush.n.01', 'synonyms': ['hairbrush'], 'id': 524, 'def': "a brush used to groom a person's hair", 'name': 'hairbrush'}, {'frequency': 'c', 'synset': 'hairnet.n.01', 'synonyms': ['hairnet'], 'id': 525, 'def': 'a small net that someone wears over their hair to keep it in place', 'name': 'hairnet'}, {'frequency': 'c', 'synset': 'hairpin.n.01', 'synonyms': ['hairpin'], 'id': 526, 'def': "a double pronged pin used to hold women's hair in place", 'name': 'hairpin'}, {'frequency': 'r', 'synset': 'halter.n.03', 'synonyms': ['halter_top'], 'id': 527, 'def': "a woman's top that fastens behind the back and neck leaving the back and arms uncovered", 'name': 'halter_top'}, {'frequency': 'f', 'synset': 'ham.n.01', 'synonyms': ['ham', 'jambon', 'gammon'], 'id': 528, 'def': 'meat cut from the thigh of a hog (usually smoked)', 'name': 'ham'}, {'frequency': 'c', 'synset': 'hamburger.n.01', 'synonyms': ['hamburger', 'beefburger', 'burger'], 'id': 529, 'def': 'a sandwich consisting of a patty of minced beef served on a bun', 'name': 'hamburger'}, {'frequency': 'c', 'synset': 'hammer.n.02', 'synonyms': ['hammer'], 'id': 530, 'def': 'a hand tool with a heavy head and a handle; used to deliver an impulsive force by striking', 'name': 'hammer'}, {'frequency': 'c', 'synset': 'hammock.n.02', 'synonyms': ['hammock'], 'id': 531, 'def': 'a hanging bed of canvas or rope netting (usually suspended between two trees)', 'name': 'hammock'}, {'frequency': 'r', 'synset': 'hamper.n.02', 'synonyms': ['hamper'], 'id': 532, 'def': 'a basket usually with a cover', 'name': 'hamper'}, {'frequency': 'c', 'synset': 'hamster.n.01', 'synonyms': ['hamster'], 'id': 533, 'def': 'short-tailed burrowing rodent with large cheek pouches', 'name': 'hamster'}, {'frequency': 'f', 'synset': 'hand_blower.n.01', 'synonyms': ['hair_dryer'], 'id': 534, 'def': 'a hand-held electric blower that can blow warm air onto the hair', 'name': 'hair_dryer'}, {'frequency': 'r', 'synset': 'hand_glass.n.01', 'synonyms': ['hand_glass', 'hand_mirror'], 'id': 535, 'def': 'a mirror intended to be held in the hand', 'name': 'hand_glass'}, {'frequency': 'f', 'synset': 'hand_towel.n.01', 'synonyms': ['hand_towel', 'face_towel'], 'id': 536, 'def': 'a small towel used to dry the hands or face', 'name': 'hand_towel'}, {'frequency': 'c', 'synset': 'handcart.n.01', 'synonyms': ['handcart', 'pushcart', 'hand_truck'], 'id': 537, 'def': 'wheeled vehicle that can be pushed by a person', 'name': 'handcart'}, {'frequency': 'r', 'synset': 'handcuff.n.01', 'synonyms': ['handcuff'], 'id': 538, 'def': 'shackle that consists of a metal loop that can be locked around the wrist', 'name': 'handcuff'}, {'frequency': 'c', 'synset': 'handkerchief.n.01', 'synonyms': ['handkerchief'], 'id': 539, 'def': 'a square piece of cloth used for wiping the eyes or nose or as a costume accessory', 'name': 'handkerchief'}, {'frequency': 'f', 'synset': 'handle.n.01', 'synonyms': ['handle', 'grip', 'handgrip'], 'id': 540, 'def': 'the appendage to an object that is designed to be held in order to use or move it', 'name': 'handle'}, {'frequency': 'r', 'synset': 'handsaw.n.01', 'synonyms': ['handsaw', "carpenter's_saw"], 'id': 541, 'def': 'a saw used with one hand for cutting wood', 'name': 'handsaw'}, {'frequency': 'r', 'synset': 'hardback.n.01', 'synonyms': ['hardback_book', 'hardcover_book'], 'id': 542, 'def': 'a book with cardboard or cloth or leather covers', 'name': 'hardback_book'}, {'frequency': 'r', 'synset': 'harmonium.n.01', 'synonyms': ['harmonium', 'organ_(musical_instrument)', 'reed_organ_(musical_instrument)'], 'id': 543, 'def': 'a free-reed instrument in which air is forced through the reeds by bellows', 'name': 'harmonium'}, {'frequency': 'f', 'synset': 'hat.n.01', 'synonyms': ['hat'], 'id': 544, 'def': 'headwear that protects the head from bad weather, sun, or worn for fashion', 'name': 'hat'}, {'frequency': 'r', 'synset': 'hatbox.n.01', 'synonyms': ['hatbox'], 'id': 545, 'def': 'a round piece of luggage for carrying hats', 'name': 'hatbox'}, {'frequency': 'c', 'synset': 'head_covering.n.01', 'synonyms': ['veil'], 'id': 546, 'def': 'a garment that covers the head OR face', 'name': 'veil'}, {'frequency': 'f', 'synset': 'headband.n.01', 'synonyms': ['headband'], 'id': 547, 'def': 'a band worn around or over the head', 'name': 'headband'}, {'frequency': 'f', 'synset': 'headboard.n.01', 'synonyms': ['headboard'], 'id': 548, 'def': 'a vertical board or panel forming the head of a bedstead', 'name': 'headboard'}, {'frequency': 'f', 'synset': 'headlight.n.01', 'synonyms': ['headlight', 'headlamp'], 'id': 549, 'def': 'a powerful light with reflector; attached to the front of an automobile or locomotive', 'name': 'headlight'}, {'frequency': 'c', 'synset': 'headscarf.n.01', 'synonyms': ['headscarf'], 'id': 550, 'def': 'a kerchief worn over the head and tied under the chin', 'name': 'headscarf'}, {'frequency': 'r', 'synset': 'headset.n.01', 'synonyms': ['headset'], 'id': 551, 'def': 'receiver consisting of a pair of headphones', 'name': 'headset'}, {'frequency': 'c', 'synset': 'headstall.n.01', 'synonyms': ['headstall_(for_horses)', 'headpiece_(for_horses)'], 'id': 552, 'def': "the band that is the part of a bridle that fits around a horse's head", 'name': 'headstall_(for_horses)'}, {'frequency': 'c', 'synset': 'heart.n.02', 'synonyms': ['heart'], 'id': 553, 'def': 'a muscular organ; its contractions move the blood through the body', 'name': 'heart'}, {'frequency': 'c', 'synset': 'heater.n.01', 'synonyms': ['heater', 'warmer'], 'id': 554, 'def': 'device that heats water or supplies warmth to a room', 'name': 'heater'}, {'frequency': 'c', 'synset': 'helicopter.n.01', 'synonyms': ['helicopter'], 'id': 555, 'def': 'an aircraft without wings that obtains its lift from the rotation of overhead blades', 'name': 'helicopter'}, {'frequency': 'f', 'synset': 'helmet.n.02', 'synonyms': ['helmet'], 'id': 556, 'def': 'a protective headgear made of hard material to resist blows', 'name': 'helmet'}, {'frequency': 'r', 'synset': 'heron.n.02', 'synonyms': ['heron'], 'id': 557, 'def': 'grey or white wading bird with long neck and long legs and (usually) long bill', 'name': 'heron'}, {'frequency': 'c', 'synset': 'highchair.n.01', 'synonyms': ['highchair', 'feeding_chair'], 'id': 558, 'def': 'a chair for feeding a very young child', 'name': 'highchair'}, {'frequency': 'f', 'synset': 'hinge.n.01', 'synonyms': ['hinge'], 'id': 559, 'def': 'a joint that holds two parts together so that one can swing relative to the other', 'name': 'hinge'}, {'frequency': 'r', 'synset': 'hippopotamus.n.01', 'synonyms': ['hippopotamus'], 'id': 560, 'def': 'massive thick-skinned animal living in or around rivers of tropical Africa', 'name': 'hippopotamus'}, {'frequency': 'r', 'synset': 'hockey_stick.n.01', 'synonyms': ['hockey_stick'], 'id': 561, 'def': 'sports implement consisting of a stick used by hockey players to move the puck', 'name': 'hockey_stick'}, {'frequency': 'c', 'synset': 'hog.n.03', 'synonyms': ['hog', 'pig'], 'id': 562, 'def': 'domestic swine', 'name': 'hog'}, {'frequency': 'f', 'synset': 'home_plate.n.01', 'synonyms': ['home_plate_(baseball)', 'home_base_(baseball)'], 'id': 563, 'def': '(baseball) a rubber slab where the batter stands; it must be touched by a base runner in order to score', 'name': 'home_plate_(baseball)'}, {'frequency': 'c', 'synset': 'honey.n.01', 'synonyms': ['honey'], 'id': 564, 'def': 'a sweet yellow liquid produced by bees', 'name': 'honey'}, {'frequency': 'f', 'synset': 'hood.n.06', 'synonyms': ['fume_hood', 'exhaust_hood'], 'id': 565, 'def': 'metal covering leading to a vent that exhausts smoke or fumes', 'name': 'fume_hood'}, {'frequency': 'f', 'synset': 'hook.n.05', 'synonyms': ['hook'], 'id': 566, 'def': 'a curved or bent implement for suspending or pulling something', 'name': 'hook'}, {'frequency': 'r', 'synset': 'hookah.n.01', 'synonyms': ['hookah', 'narghile', 'nargileh', 'sheesha', 'shisha', 'water_pipe'], 'id': 567, 'def': 'a tobacco pipe with a long flexible tube connected to a container where the smoke is cooled by passing through water', 'name': 'hookah'}, {'frequency': 'r', 'synset': 'hornet.n.01', 'synonyms': ['hornet'], 'id': 568, 'def': 'large stinging wasp', 'name': 'hornet'}, {'frequency': 'f', 'synset': 'horse.n.01', 'synonyms': ['horse'], 'id': 569, 'def': 'a common horse', 'name': 'horse'}, {'frequency': 'f', 'synset': 'hose.n.03', 'synonyms': ['hose', 'hosepipe'], 'id': 570, 'def': 'a flexible pipe for conveying a liquid or gas', 'name': 'hose'}, {'frequency': 'r', 'synset': 'hot-air_balloon.n.01', 'synonyms': ['hot-air_balloon'], 'id': 571, 'def': 'balloon for travel through the air in a basket suspended below a large bag of heated air', 'name': 'hot-air_balloon'}, {'frequency': 'r', 'synset': 'hot_plate.n.01', 'synonyms': ['hotplate'], 'id': 572, 'def': 'a portable electric appliance for heating or cooking or keeping food warm', 'name': 'hotplate'}, {'frequency': 'c', 'synset': 'hot_sauce.n.01', 'synonyms': ['hot_sauce'], 'id': 573, 'def': 'a pungent peppery sauce', 'name': 'hot_sauce'}, {'frequency': 'r', 'synset': 'hourglass.n.01', 'synonyms': ['hourglass'], 'id': 574, 'def': 'a sandglass timer that runs for sixty minutes', 'name': 'hourglass'}, {'frequency': 'r', 'synset': 'houseboat.n.01', 'synonyms': ['houseboat'], 'id': 575, 'def': 'a barge that is designed and equipped for use as a dwelling', 'name': 'houseboat'}, {'frequency': 'c', 'synset': 'hummingbird.n.01', 'synonyms': ['hummingbird'], 'id': 576, 'def': 'tiny American bird having brilliant iridescent plumage and long slender bills', 'name': 'hummingbird'}, {'frequency': 'r', 'synset': 'hummus.n.01', 'synonyms': ['hummus', 'humus', 'hommos', 'hoummos', 'humous'], 'id': 577, 'def': 'a thick spread made from mashed chickpeas', 'name': 'hummus'}, {'frequency': 'f', 'synset': 'ice_bear.n.01', 'synonyms': ['polar_bear'], 'id': 578, 'def': 'white bear of Arctic regions', 'name': 'polar_bear'}, {'frequency': 'c', 'synset': 'ice_cream.n.01', 'synonyms': ['icecream'], 'id': 579, 'def': 'frozen dessert containing cream and sugar and flavoring', 'name': 'icecream'}, {'frequency': 'r', 'synset': 'ice_lolly.n.01', 'synonyms': ['popsicle'], 'id': 580, 'def': 'ice cream or water ice on a small wooden stick', 'name': 'popsicle'}, {'frequency': 'c', 'synset': 'ice_maker.n.01', 'synonyms': ['ice_maker'], 'id': 581, 'def': 'an appliance included in some electric refrigerators for making ice cubes', 'name': 'ice_maker'}, {'frequency': 'r', 'synset': 'ice_pack.n.01', 'synonyms': ['ice_pack', 'ice_bag'], 'id': 582, 'def': 'a waterproof bag filled with ice: applied to the body (especially the head) to cool or reduce swelling', 'name': 'ice_pack'}, {'frequency': 'r', 'synset': 'ice_skate.n.01', 'synonyms': ['ice_skate'], 'id': 583, 'def': 'skate consisting of a boot with a steel blade fitted to the sole', 'name': 'ice_skate'}, {'frequency': 'c', 'synset': 'igniter.n.01', 'synonyms': ['igniter', 'ignitor', 'lighter'], 'id': 584, 'def': 'a substance or device used to start a fire', 'name': 'igniter'}, {'frequency': 'r', 'synset': 'inhaler.n.01', 'synonyms': ['inhaler', 'inhalator'], 'id': 585, 'def': 'a dispenser that produces a chemical vapor to be inhaled through mouth or nose', 'name': 'inhaler'}, {'frequency': 'f', 'synset': 'ipod.n.01', 'synonyms': ['iPod'], 'id': 586, 'def': 'a pocket-sized device used to play music files', 'name': 'iPod'}, {'frequency': 'c', 'synset': 'iron.n.04', 'synonyms': ['iron_(for_clothing)', 'smoothing_iron_(for_clothing)'], 'id': 587, 'def': 'home appliance consisting of a flat metal base that is heated and used to smooth cloth', 'name': 'iron_(for_clothing)'}, {'frequency': 'c', 'synset': 'ironing_board.n.01', 'synonyms': ['ironing_board'], 'id': 588, 'def': 'narrow padded board on collapsible supports; used for ironing clothes', 'name': 'ironing_board'}, {'frequency': 'f', 'synset': 'jacket.n.01', 'synonyms': ['jacket'], 'id': 589, 'def': 'a waist-length coat', 'name': 'jacket'}, {'frequency': 'c', 'synset': 'jam.n.01', 'synonyms': ['jam'], 'id': 590, 'def': 'preserve of crushed fruit', 'name': 'jam'}, {'frequency': 'f', 'synset': 'jar.n.01', 'synonyms': ['jar'], 'id': 591, 'def': 'a vessel (usually cylindrical) with a wide mouth and without handles', 'name': 'jar'}, {'frequency': 'f', 'synset': 'jean.n.01', 'synonyms': ['jean', 'blue_jean', 'denim'], 'id': 592, 'def': '(usually plural) close-fitting trousers of heavy denim for manual work or casual wear', 'name': 'jean'}, {'frequency': 'c', 'synset': 'jeep.n.01', 'synonyms': ['jeep', 'landrover'], 'id': 593, 'def': 'a car suitable for traveling over rough terrain', 'name': 'jeep'}, {'frequency': 'r', 'synset': 'jelly_bean.n.01', 'synonyms': ['jelly_bean', 'jelly_egg'], 'id': 594, 'def': 'sugar-glazed jellied candy', 'name': 'jelly_bean'}, {'frequency': 'f', 'synset': 'jersey.n.03', 'synonyms': ['jersey', 'T-shirt', 'tee_shirt'], 'id': 595, 'def': 'a close-fitting pullover shirt', 'name': 'jersey'}, {'frequency': 'c', 'synset': 'jet.n.01', 'synonyms': ['jet_plane', 'jet-propelled_plane'], 'id': 596, 'def': 'an airplane powered by one or more jet engines', 'name': 'jet_plane'}, {'frequency': 'r', 'synset': 'jewel.n.01', 'synonyms': ['jewel', 'gem', 'precious_stone'], 'id': 597, 'def': 'a precious or semiprecious stone incorporated into a piece of jewelry', 'name': 'jewel'}, {'frequency': 'c', 'synset': 'jewelry.n.01', 'synonyms': ['jewelry', 'jewellery'], 'id': 598, 'def': 'an adornment (as a bracelet or ring or necklace) made of precious metals and set with gems (or imitation gems)', 'name': 'jewelry'}, {'frequency': 'r', 'synset': 'joystick.n.02', 'synonyms': ['joystick'], 'id': 599, 'def': 'a control device for computers consisting of a vertical handle that can move freely in two directions', 'name': 'joystick'}, {'frequency': 'c', 'synset': 'jump_suit.n.01', 'synonyms': ['jumpsuit'], 'id': 600, 'def': "one-piece garment fashioned after a parachutist's uniform", 'name': 'jumpsuit'}, {'frequency': 'c', 'synset': 'kayak.n.01', 'synonyms': ['kayak'], 'id': 601, 'def': 'a small canoe consisting of a light frame made watertight with animal skins', 'name': 'kayak'}, {'frequency': 'r', 'synset': 'keg.n.02', 'synonyms': ['keg'], 'id': 602, 'def': 'small cask or barrel', 'name': 'keg'}, {'frequency': 'r', 'synset': 'kennel.n.01', 'synonyms': ['kennel', 'doghouse'], 'id': 603, 'def': 'outbuilding that serves as a shelter for a dog', 'name': 'kennel'}, {'frequency': 'c', 'synset': 'kettle.n.01', 'synonyms': ['kettle', 'boiler'], 'id': 604, 'def': 'a metal pot for stewing or boiling; usually has a lid', 'name': 'kettle'}, {'frequency': 'f', 'synset': 'key.n.01', 'synonyms': ['key'], 'id': 605, 'def': 'metal instrument used to unlock a lock', 'name': 'key'}, {'frequency': 'r', 'synset': 'keycard.n.01', 'synonyms': ['keycard'], 'id': 606, 'def': 'a plastic card used to gain access typically to a door', 'name': 'keycard'}, {'frequency': 'c', 'synset': 'kilt.n.01', 'synonyms': ['kilt'], 'id': 607, 'def': 'a knee-length pleated tartan skirt worn by men as part of the traditional dress in the Highlands of northern Scotland', 'name': 'kilt'}, {'frequency': 'c', 'synset': 'kimono.n.01', 'synonyms': ['kimono'], 'id': 608, 'def': 'a loose robe; imitated from robes originally worn by Japanese', 'name': 'kimono'}, {'frequency': 'f', 'synset': 'kitchen_sink.n.01', 'synonyms': ['kitchen_sink'], 'id': 609, 'def': 'a sink in a kitchen', 'name': 'kitchen_sink'}, {'frequency': 'r', 'synset': 'kitchen_table.n.01', 'synonyms': ['kitchen_table'], 'id': 610, 'def': 'a table in the kitchen', 'name': 'kitchen_table'}, {'frequency': 'f', 'synset': 'kite.n.03', 'synonyms': ['kite'], 'id': 611, 'def': 'plaything consisting of a light frame covered with tissue paper; flown in wind at end of a string', 'name': 'kite'}, {'frequency': 'c', 'synset': 'kitten.n.01', 'synonyms': ['kitten', 'kitty'], 'id': 612, 'def': 'young domestic cat', 'name': 'kitten'}, {'frequency': 'c', 'synset': 'kiwi.n.03', 'synonyms': ['kiwi_fruit'], 'id': 613, 'def': 'fuzzy brown egg-shaped fruit with slightly tart green flesh', 'name': 'kiwi_fruit'}, {'frequency': 'f', 'synset': 'knee_pad.n.01', 'synonyms': ['knee_pad'], 'id': 614, 'def': 'protective garment consisting of a pad worn by football or baseball or hockey players', 'name': 'knee_pad'}, {'frequency': 'f', 'synset': 'knife.n.01', 'synonyms': ['knife'], 'id': 615, 'def': 'tool with a blade and point used as a cutting instrument', 'name': 'knife'}, {'frequency': 'r', 'synset': 'knitting_needle.n.01', 'synonyms': ['knitting_needle'], 'id': 616, 'def': 'needle consisting of a slender rod with pointed ends; usually used in pairs', 'name': 'knitting_needle'}, {'frequency': 'f', 'synset': 'knob.n.02', 'synonyms': ['knob'], 'id': 617, 'def': 'a round handle often found on a door', 'name': 'knob'}, {'frequency': 'r', 'synset': 'knocker.n.05', 'synonyms': ['knocker_(on_a_door)', 'doorknocker'], 'id': 618, 'def': 'a device (usually metal and ornamental) attached by a hinge to a door', 'name': 'knocker_(on_a_door)'}, {'frequency': 'r', 'synset': 'koala.n.01', 'synonyms': ['koala', 'koala_bear'], 'id': 619, 'def': 'sluggish tailless Australian marsupial with grey furry ears and coat', 'name': 'koala'}, {'frequency': 'r', 'synset': 'lab_coat.n.01', 'synonyms': ['lab_coat', 'laboratory_coat'], 'id': 620, 'def': 'a light coat worn to protect clothing from substances used while working in a laboratory', 'name': 'lab_coat'}, {'frequency': 'f', 'synset': 'ladder.n.01', 'synonyms': ['ladder'], 'id': 621, 'def': 'steps consisting of two parallel members connected by rungs', 'name': 'ladder'}, {'frequency': 'c', 'synset': 'ladle.n.01', 'synonyms': ['ladle'], 'id': 622, 'def': 'a spoon-shaped vessel with a long handle frequently used to transfer liquids', 'name': 'ladle'}, {'frequency': 'c', 'synset': 'ladybug.n.01', 'synonyms': ['ladybug', 'ladybeetle', 'ladybird_beetle'], 'id': 623, 'def': 'small round bright-colored and spotted beetle, typically red and black', 'name': 'ladybug'}, {'frequency': 'f', 'synset': 'lamb.n.01', 'synonyms': ['lamb_(animal)'], 'id': 624, 'def': 'young sheep', 'name': 'lamb_(animal)'}, {'frequency': 'r', 'synset': 'lamb_chop.n.01', 'synonyms': ['lamb-chop', 'lambchop'], 'id': 625, 'def': 'chop cut from a lamb', 'name': 'lamb-chop'}, {'frequency': 'f', 'synset': 'lamp.n.02', 'synonyms': ['lamp'], 'id': 626, 'def': 'a piece of furniture holding one or more electric light bulbs', 'name': 'lamp'}, {'frequency': 'f', 'synset': 'lamppost.n.01', 'synonyms': ['lamppost'], 'id': 627, 'def': 'a metal post supporting an outdoor lamp (such as a streetlight)', 'name': 'lamppost'}, {'frequency': 'f', 'synset': 'lampshade.n.01', 'synonyms': ['lampshade'], 'id': 628, 'def': 'a protective ornamental shade used to screen a light bulb from direct view', 'name': 'lampshade'}, {'frequency': 'c', 'synset': 'lantern.n.01', 'synonyms': ['lantern'], 'id': 629, 'def': 'light in a transparent protective case', 'name': 'lantern'}, {'frequency': 'f', 'synset': 'lanyard.n.02', 'synonyms': ['lanyard', 'laniard'], 'id': 630, 'def': 'a cord worn around the neck to hold a knife or whistle, etc.', 'name': 'lanyard'}, {'frequency': 'f', 'synset': 'laptop.n.01', 'synonyms': ['laptop_computer', 'notebook_computer'], 'id': 631, 'def': 'a portable computer small enough to use in your lap', 'name': 'laptop_computer'}, {'frequency': 'r', 'synset': 'lasagna.n.01', 'synonyms': ['lasagna', 'lasagne'], 'id': 632, 'def': 'baked dish of layers of lasagna pasta with sauce and cheese and meat or vegetables', 'name': 'lasagna'}, {'frequency': 'f', 'synset': 'latch.n.02', 'synonyms': ['latch'], 'id': 633, 'def': 'a bar that can be lowered or slid into a groove to fasten a door or gate', 'name': 'latch'}, {'frequency': 'r', 'synset': 'lawn_mower.n.01', 'synonyms': ['lawn_mower'], 'id': 634, 'def': 'garden tool for mowing grass on lawns', 'name': 'lawn_mower'}, {'frequency': 'r', 'synset': 'leather.n.01', 'synonyms': ['leather'], 'id': 635, 'def': 'an animal skin made smooth and flexible by removing the hair and then tanning', 'name': 'leather'}, {'frequency': 'c', 'synset': 'legging.n.01', 'synonyms': ['legging_(clothing)', 'leging_(clothing)', 'leg_covering'], 'id': 636, 'def': 'a garment covering the leg (usually extending from the knee to the ankle)', 'name': 'legging_(clothing)'}, {'frequency': 'c', 'synset': 'lego.n.01', 'synonyms': ['Lego', 'Lego_set'], 'id': 637, 'def': "a child's plastic construction set for making models from blocks", 'name': 'Lego'}, {'frequency': 'r', 'synset': 'legume.n.02', 'synonyms': ['legume'], 'id': 638, 'def': 'the fruit or seed of bean or pea plants', 'name': 'legume'}, {'frequency': 'f', 'synset': 'lemon.n.01', 'synonyms': ['lemon'], 'id': 639, 'def': 'yellow oval fruit with juicy acidic flesh', 'name': 'lemon'}, {'frequency': 'r', 'synset': 'lemonade.n.01', 'synonyms': ['lemonade'], 'id': 640, 'def': 'sweetened beverage of diluted lemon juice', 'name': 'lemonade'}, {'frequency': 'f', 'synset': 'lettuce.n.02', 'synonyms': ['lettuce'], 'id': 641, 'def': 'leafy plant commonly eaten in salad or on sandwiches', 'name': 'lettuce'}, {'frequency': 'f', 'synset': 'license_plate.n.01', 'synonyms': ['license_plate', 'numberplate'], 'id': 642, 'def': "a plate mounted on the front and back of car and bearing the car's registration number", 'name': 'license_plate'}, {'frequency': 'f', 'synset': 'life_buoy.n.01', 'synonyms': ['life_buoy', 'lifesaver', 'life_belt', 'life_ring'], 'id': 643, 'def': 'a ring-shaped life preserver used to prevent drowning (NOT a life-jacket or vest)', 'name': 'life_buoy'}, {'frequency': 'f', 'synset': 'life_jacket.n.01', 'synonyms': ['life_jacket', 'life_vest'], 'id': 644, 'def': 'life preserver consisting of a sleeveless jacket of buoyant or inflatable design', 'name': 'life_jacket'}, {'frequency': 'f', 'synset': 'light_bulb.n.01', 'synonyms': ['lightbulb'], 'id': 645, 'def': 'lightblub/source of light', 'name': 'lightbulb'}, {'frequency': 'r', 'synset': 'lightning_rod.n.02', 'synonyms': ['lightning_rod', 'lightning_conductor'], 'id': 646, 'def': 'a metallic conductor that is attached to a high point and leads to the ground', 'name': 'lightning_rod'}, {'frequency': 'f', 'synset': 'lime.n.06', 'synonyms': ['lime'], 'id': 647, 'def': 'the green acidic fruit of any of various lime trees', 'name': 'lime'}, {'frequency': 'r', 'synset': 'limousine.n.01', 'synonyms': ['limousine'], 'id': 648, 'def': 'long luxurious car; usually driven by a chauffeur', 'name': 'limousine'}, {'frequency': 'c', 'synset': 'lion.n.01', 'synonyms': ['lion'], 'id': 649, 'def': 'large gregarious predatory cat of Africa and India', 'name': 'lion'}, {'frequency': 'c', 'synset': 'lip_balm.n.01', 'synonyms': ['lip_balm'], 'id': 650, 'def': 'a balm applied to the lips', 'name': 'lip_balm'}, {'frequency': 'r', 'synset': 'liquor.n.01', 'synonyms': ['liquor', 'spirits', 'hard_liquor', 'liqueur', 'cordial'], 'id': 651, 'def': 'liquor or beer', 'name': 'liquor'}, {'frequency': 'c', 'synset': 'lizard.n.01', 'synonyms': ['lizard'], 'id': 652, 'def': 'a reptile with usually two pairs of legs and a tapering tail', 'name': 'lizard'}, {'frequency': 'f', 'synset': 'log.n.01', 'synonyms': ['log'], 'id': 653, 'def': 'a segment of the trunk of a tree when stripped of branches', 'name': 'log'}, {'frequency': 'c', 'synset': 'lollipop.n.02', 'synonyms': ['lollipop'], 'id': 654, 'def': 'hard candy on a stick', 'name': 'lollipop'}, {'frequency': 'f', 'synset': 'loudspeaker.n.01', 'synonyms': ['speaker_(stero_equipment)'], 'id': 655, 'def': 'electronic device that produces sound often as part of a stereo system', 'name': 'speaker_(stero_equipment)'}, {'frequency': 'c', 'synset': 'love_seat.n.01', 'synonyms': ['loveseat'], 'id': 656, 'def': 'small sofa that seats two people', 'name': 'loveseat'}, {'frequency': 'r', 'synset': 'machine_gun.n.01', 'synonyms': ['machine_gun'], 'id': 657, 'def': 'a rapidly firing automatic gun', 'name': 'machine_gun'}, {'frequency': 'f', 'synset': 'magazine.n.02', 'synonyms': ['magazine'], 'id': 658, 'def': 'a paperback periodic publication', 'name': 'magazine'}, {'frequency': 'f', 'synset': 'magnet.n.01', 'synonyms': ['magnet'], 'id': 659, 'def': 'a device that attracts iron and produces a magnetic field', 'name': 'magnet'}, {'frequency': 'c', 'synset': 'mail_slot.n.01', 'synonyms': ['mail_slot'], 'id': 660, 'def': 'a slot (usually in a door) through which mail can be delivered', 'name': 'mail_slot'}, {'frequency': 'f', 'synset': 'mailbox.n.01', 'synonyms': ['mailbox_(at_home)', 'letter_box_(at_home)'], 'id': 661, 'def': 'a private box for delivery of mail', 'name': 'mailbox_(at_home)'}, {'frequency': 'r', 'synset': 'mallard.n.01', 'synonyms': ['mallard'], 'id': 662, 'def': 'wild dabbling duck from which domestic ducks are descended', 'name': 'mallard'}, {'frequency': 'r', 'synset': 'mallet.n.01', 'synonyms': ['mallet'], 'id': 663, 'def': 'a sports implement with a long handle and a hammer-like head used to hit a ball', 'name': 'mallet'}, {'frequency': 'r', 'synset': 'mammoth.n.01', 'synonyms': ['mammoth'], 'id': 664, 'def': 'any of numerous extinct elephants widely distributed in the Pleistocene', 'name': 'mammoth'}, {'frequency': 'r', 'synset': 'manatee.n.01', 'synonyms': ['manatee'], 'id': 665, 'def': 'sirenian mammal of tropical coastal waters of America', 'name': 'manatee'}, {'frequency': 'c', 'synset': 'mandarin.n.05', 'synonyms': ['mandarin_orange'], 'id': 666, 'def': 'a somewhat flat reddish-orange loose skinned citrus of China', 'name': 'mandarin_orange'}, {'frequency': 'c', 'synset': 'manger.n.01', 'synonyms': ['manger', 'trough'], 'id': 667, 'def': 'a container (usually in a barn or stable) from which cattle or horses feed', 'name': 'manger'}, {'frequency': 'f', 'synset': 'manhole.n.01', 'synonyms': ['manhole'], 'id': 668, 'def': 'a hole (usually with a flush cover) through which a person can gain access to an underground structure', 'name': 'manhole'}, {'frequency': 'f', 'synset': 'map.n.01', 'synonyms': ['map'], 'id': 669, 'def': "a diagrammatic representation of the earth's surface (or part of it)", 'name': 'map'}, {'frequency': 'f', 'synset': 'marker.n.03', 'synonyms': ['marker'], 'id': 670, 'def': 'a writing implement for making a mark', 'name': 'marker'}, {'frequency': 'r', 'synset': 'martini.n.01', 'synonyms': ['martini'], 'id': 671, 'def': 'a cocktail made of gin (or vodka) with dry vermouth', 'name': 'martini'}, {'frequency': 'r', 'synset': 'mascot.n.01', 'synonyms': ['mascot'], 'id': 672, 'def': 'a person or animal that is adopted by a team or other group as a symbolic figure', 'name': 'mascot'}, {'frequency': 'c', 'synset': 'mashed_potato.n.01', 'synonyms': ['mashed_potato'], 'id': 673, 'def': 'potato that has been peeled and boiled and then mashed', 'name': 'mashed_potato'}, {'frequency': 'r', 'synset': 'masher.n.02', 'synonyms': ['masher'], 'id': 674, 'def': 'a kitchen utensil used for mashing (e.g. potatoes)', 'name': 'masher'}, {'frequency': 'f', 'synset': 'mask.n.04', 'synonyms': ['mask', 'facemask'], 'id': 675, 'def': 'a protective covering worn over the face', 'name': 'mask'}, {'frequency': 'f', 'synset': 'mast.n.01', 'synonyms': ['mast'], 'id': 676, 'def': 'a vertical spar for supporting sails', 'name': 'mast'}, {'frequency': 'c', 'synset': 'mat.n.03', 'synonyms': ['mat_(gym_equipment)', 'gym_mat'], 'id': 677, 'def': 'sports equipment consisting of a piece of thick padding on the floor for gymnastics', 'name': 'mat_(gym_equipment)'}, {'frequency': 'r', 'synset': 'matchbox.n.01', 'synonyms': ['matchbox'], 'id': 678, 'def': 'a box for holding matches', 'name': 'matchbox'}, {'frequency': 'f', 'synset': 'mattress.n.01', 'synonyms': ['mattress'], 'id': 679, 'def': 'a thick pad filled with resilient material used as a bed or part of a bed', 'name': 'mattress'}, {'frequency': 'c', 'synset': 'measuring_cup.n.01', 'synonyms': ['measuring_cup'], 'id': 680, 'def': 'graduated cup used to measure liquid or granular ingredients', 'name': 'measuring_cup'}, {'frequency': 'c', 'synset': 'measuring_stick.n.01', 'synonyms': ['measuring_stick', 'ruler_(measuring_stick)', 'measuring_rod'], 'id': 681, 'def': 'measuring instrument having a sequence of marks at regular intervals', 'name': 'measuring_stick'}, {'frequency': 'c', 'synset': 'meatball.n.01', 'synonyms': ['meatball'], 'id': 682, 'def': 'ground meat formed into a ball and fried or simmered in broth', 'name': 'meatball'}, {'frequency': 'c', 'synset': 'medicine.n.02', 'synonyms': ['medicine'], 'id': 683, 'def': 'something that treats or prevents or alleviates the symptoms of disease', 'name': 'medicine'}, {'frequency': 'c', 'synset': 'melon.n.01', 'synonyms': ['melon'], 'id': 684, 'def': 'fruit of the gourd family having a hard rind and sweet juicy flesh', 'name': 'melon'}, {'frequency': 'f', 'synset': 'microphone.n.01', 'synonyms': ['microphone'], 'id': 685, 'def': 'device for converting sound waves into electrical energy', 'name': 'microphone'}, {'frequency': 'r', 'synset': 'microscope.n.01', 'synonyms': ['microscope'], 'id': 686, 'def': 'magnifier of the image of small objects', 'name': 'microscope'}, {'frequency': 'f', 'synset': 'microwave.n.02', 'synonyms': ['microwave_oven'], 'id': 687, 'def': 'kitchen appliance that cooks food by passing an electromagnetic wave through it', 'name': 'microwave_oven'}, {'frequency': 'r', 'synset': 'milestone.n.01', 'synonyms': ['milestone', 'milepost'], 'id': 688, 'def': 'stone post at side of a road to show distances', 'name': 'milestone'}, {'frequency': 'f', 'synset': 'milk.n.01', 'synonyms': ['milk'], 'id': 689, 'def': 'a white nutritious liquid secreted by mammals and used as food by human beings', 'name': 'milk'}, {'frequency': 'r', 'synset': 'milk_can.n.01', 'synonyms': ['milk_can'], 'id': 690, 'def': 'can for transporting milk', 'name': 'milk_can'}, {'frequency': 'r', 'synset': 'milkshake.n.01', 'synonyms': ['milkshake'], 'id': 691, 'def': 'frothy drink of milk and flavoring and sometimes fruit or ice cream', 'name': 'milkshake'}, {'frequency': 'f', 'synset': 'minivan.n.01', 'synonyms': ['minivan'], 'id': 692, 'def': 'a small box-shaped passenger van', 'name': 'minivan'}, {'frequency': 'r', 'synset': 'mint.n.05', 'synonyms': ['mint_candy'], 'id': 693, 'def': 'a candy that is flavored with a mint oil', 'name': 'mint_candy'}, {'frequency': 'f', 'synset': 'mirror.n.01', 'synonyms': ['mirror'], 'id': 694, 'def': 'polished surface that forms images by reflecting light', 'name': 'mirror'}, {'frequency': 'c', 'synset': 'mitten.n.01', 'synonyms': ['mitten'], 'id': 695, 'def': 'glove that encases the thumb separately and the other four fingers together', 'name': 'mitten'}, {'frequency': 'c', 'synset': 'mixer.n.04', 'synonyms': ['mixer_(kitchen_tool)', 'stand_mixer'], 'id': 696, 'def': 'a kitchen utensil that is used for mixing foods', 'name': 'mixer_(kitchen_tool)'}, {'frequency': 'c', 'synset': 'money.n.03', 'synonyms': ['money'], 'id': 697, 'def': 'the official currency issued by a government or national bank', 'name': 'money'}, {'frequency': 'f', 'synset': 'monitor.n.04', 'synonyms': ['monitor_(computer_equipment) computer_monitor'], 'id': 698, 'def': 'a computer monitor', 'name': 'monitor_(computer_equipment) computer_monitor'}, {'frequency': 'c', 'synset': 'monkey.n.01', 'synonyms': ['monkey'], 'id': 699, 'def': 'any of various long-tailed primates', 'name': 'monkey'}, {'frequency': 'f', 'synset': 'motor.n.01', 'synonyms': ['motor'], 'id': 700, 'def': 'machine that converts other forms of energy into mechanical energy and so imparts motion', 'name': 'motor'}, {'frequency': 'f', 'synset': 'motor_scooter.n.01', 'synonyms': ['motor_scooter', 'scooter'], 'id': 701, 'def': 'a wheeled vehicle with small wheels and a low-powered engine', 'name': 'motor_scooter'}, {'frequency': 'r', 'synset': 'motor_vehicle.n.01', 'synonyms': ['motor_vehicle', 'automotive_vehicle'], 'id': 702, 'def': 'a self-propelled wheeled vehicle that does not run on rails', 'name': 'motor_vehicle'}, {'frequency': 'f', 'synset': 'motorcycle.n.01', 'synonyms': ['motorcycle'], 'id': 703, 'def': 'a motor vehicle with two wheels and a strong frame', 'name': 'motorcycle'}, {'frequency': 'f', 'synset': 'mound.n.01', 'synonyms': ['mound_(baseball)', "pitcher's_mound"], 'id': 704, 'def': '(baseball) the slight elevation on which the pitcher stands', 'name': 'mound_(baseball)'}, {'frequency': 'f', 'synset': 'mouse.n.04', 'synonyms': ['mouse_(computer_equipment)', 'computer_mouse'], 'id': 705, 'def': 'a computer input device that controls an on-screen pointer (does not include trackpads / touchpads)', 'name': 'mouse_(computer_equipment)'}, {'frequency': 'f', 'synset': 'mousepad.n.01', 'synonyms': ['mousepad'], 'id': 706, 'def': 'a small portable pad that provides an operating surface for a computer mouse', 'name': 'mousepad'}, {'frequency': 'c', 'synset': 'muffin.n.01', 'synonyms': ['muffin'], 'id': 707, 'def': 'a sweet quick bread baked in a cup-shaped pan', 'name': 'muffin'}, {'frequency': 'f', 'synset': 'mug.n.04', 'synonyms': ['mug'], 'id': 708, 'def': 'with handle and usually cylindrical', 'name': 'mug'}, {'frequency': 'f', 'synset': 'mushroom.n.02', 'synonyms': ['mushroom'], 'id': 709, 'def': 'a common mushroom', 'name': 'mushroom'}, {'frequency': 'r', 'synset': 'music_stool.n.01', 'synonyms': ['music_stool', 'piano_stool'], 'id': 710, 'def': 'a stool for piano players; usually adjustable in height', 'name': 'music_stool'}, {'frequency': 'c', 'synset': 'musical_instrument.n.01', 'synonyms': ['musical_instrument', 'instrument_(musical)'], 'id': 711, 'def': 'any of various devices or contrivances that can be used to produce musical tones or sounds', 'name': 'musical_instrument'}, {'frequency': 'r', 'synset': 'nailfile.n.01', 'synonyms': ['nailfile'], 'id': 712, 'def': 'a small flat file for shaping the nails', 'name': 'nailfile'}, {'frequency': 'f', 'synset': 'napkin.n.01', 'synonyms': ['napkin', 'table_napkin', 'serviette'], 'id': 713, 'def': 'a small piece of table linen or paper that is used to wipe the mouth and to cover the lap in order to protect clothing', 'name': 'napkin'}, {'frequency': 'r', 'synset': 'neckerchief.n.01', 'synonyms': ['neckerchief'], 'id': 714, 'def': 'a kerchief worn around the neck', 'name': 'neckerchief'}, {'frequency': 'f', 'synset': 'necklace.n.01', 'synonyms': ['necklace'], 'id': 715, 'def': 'jewelry consisting of a cord or chain (often bearing gems) worn about the neck as an ornament', 'name': 'necklace'}, {'frequency': 'f', 'synset': 'necktie.n.01', 'synonyms': ['necktie', 'tie_(necktie)'], 'id': 716, 'def': 'neckwear consisting of a long narrow piece of material worn under a collar and tied in knot at the front', 'name': 'necktie'}, {'frequency': 'c', 'synset': 'needle.n.03', 'synonyms': ['needle'], 'id': 717, 'def': 'a sharp pointed implement (usually metal)', 'name': 'needle'}, {'frequency': 'c', 'synset': 'nest.n.01', 'synonyms': ['nest'], 'id': 718, 'def': 'a structure in which animals lay eggs or give birth to their young', 'name': 'nest'}, {'frequency': 'f', 'synset': 'newspaper.n.01', 'synonyms': ['newspaper', 'paper_(newspaper)'], 'id': 719, 'def': 'a daily or weekly publication on folded sheets containing news, articles, and advertisements', 'name': 'newspaper'}, {'frequency': 'c', 'synset': 'newsstand.n.01', 'synonyms': ['newsstand'], 'id': 720, 'def': 'a stall where newspapers and other periodicals are sold', 'name': 'newsstand'}, {'frequency': 'c', 'synset': 'nightwear.n.01', 'synonyms': ['nightshirt', 'nightwear', 'sleepwear', 'nightclothes'], 'id': 721, 'def': 'garments designed to be worn in bed', 'name': 'nightshirt'}, {'frequency': 'r', 'synset': 'nosebag.n.01', 'synonyms': ['nosebag_(for_animals)', 'feedbag'], 'id': 722, 'def': 'a canvas bag that is used to feed an animal (such as a horse); covers the muzzle and fastens at the top of the head', 'name': 'nosebag_(for_animals)'}, {'frequency': 'c', 'synset': 'noseband.n.01', 'synonyms': ['noseband_(for_animals)', 'nosepiece_(for_animals)'], 'id': 723, 'def': "a strap that is the part of a bridle that goes over the animal's nose", 'name': 'noseband_(for_animals)'}, {'frequency': 'f', 'synset': 'notebook.n.01', 'synonyms': ['notebook'], 'id': 724, 'def': 'a book with blank pages for recording notes or memoranda', 'name': 'notebook'}, {'frequency': 'c', 'synset': 'notepad.n.01', 'synonyms': ['notepad'], 'id': 725, 'def': 'a pad of paper for keeping notes', 'name': 'notepad'}, {'frequency': 'f', 'synset': 'nut.n.03', 'synonyms': ['nut'], 'id': 726, 'def': 'a small metal block (usually square or hexagonal) with internal screw thread to be fitted onto a bolt', 'name': 'nut'}, {'frequency': 'r', 'synset': 'nutcracker.n.01', 'synonyms': ['nutcracker'], 'id': 727, 'def': 'a hand tool used to crack nuts open', 'name': 'nutcracker'}, {'frequency': 'f', 'synset': 'oar.n.01', 'synonyms': ['oar'], 'id': 728, 'def': 'an implement used to propel or steer a boat', 'name': 'oar'}, {'frequency': 'r', 'synset': 'octopus.n.01', 'synonyms': ['octopus_(food)'], 'id': 729, 'def': 'tentacles of octopus prepared as food', 'name': 'octopus_(food)'}, {'frequency': 'r', 'synset': 'octopus.n.02', 'synonyms': ['octopus_(animal)'], 'id': 730, 'def': 'bottom-living cephalopod having a soft oval body with eight long tentacles', 'name': 'octopus_(animal)'}, {'frequency': 'c', 'synset': 'oil_lamp.n.01', 'synonyms': ['oil_lamp', 'kerosene_lamp', 'kerosine_lamp'], 'id': 731, 'def': 'a lamp that burns oil (as kerosine) for light', 'name': 'oil_lamp'}, {'frequency': 'c', 'synset': 'olive_oil.n.01', 'synonyms': ['olive_oil'], 'id': 732, 'def': 'oil from olives', 'name': 'olive_oil'}, {'frequency': 'r', 'synset': 'omelet.n.01', 'synonyms': ['omelet', 'omelette'], 'id': 733, 'def': 'beaten eggs cooked until just set; may be folded around e.g. ham or cheese or jelly', 'name': 'omelet'}, {'frequency': 'f', 'synset': 'onion.n.01', 'synonyms': ['onion'], 'id': 734, 'def': 'the bulb of an onion plant', 'name': 'onion'}, {'frequency': 'f', 'synset': 'orange.n.01', 'synonyms': ['orange_(fruit)'], 'id': 735, 'def': 'orange (FRUIT of an orange tree)', 'name': 'orange_(fruit)'}, {'frequency': 'c', 'synset': 'orange_juice.n.01', 'synonyms': ['orange_juice'], 'id': 736, 'def': 'bottled or freshly squeezed juice of oranges', 'name': 'orange_juice'}, {'frequency': 'c', 'synset': 'ostrich.n.02', 'synonyms': ['ostrich'], 'id': 737, 'def': 'fast-running African flightless bird with two-toed feet; largest living bird', 'name': 'ostrich'}, {'frequency': 'f', 'synset': 'ottoman.n.03', 'synonyms': ['ottoman', 'pouf', 'pouffe', 'hassock'], 'id': 738, 'def': 'a thick standalone cushion used as a seat or footrest, often next to a chair', 'name': 'ottoman'}, {'frequency': 'f', 'synset': 'oven.n.01', 'synonyms': ['oven'], 'id': 739, 'def': 'kitchen appliance used for baking or roasting', 'name': 'oven'}, {'frequency': 'c', 'synset': 'overall.n.01', 'synonyms': ['overalls_(clothing)'], 'id': 740, 'def': 'work clothing consisting of denim trousers usually with a bib and shoulder straps', 'name': 'overalls_(clothing)'}, {'frequency': 'c', 'synset': 'owl.n.01', 'synonyms': ['owl'], 'id': 741, 'def': 'nocturnal bird of prey with hawk-like beak and claws and large head with front-facing eyes', 'name': 'owl'}, {'frequency': 'c', 'synset': 'packet.n.03', 'synonyms': ['packet'], 'id': 742, 'def': 'a small package or bundle', 'name': 'packet'}, {'frequency': 'r', 'synset': 'pad.n.03', 'synonyms': ['inkpad', 'inking_pad', 'stamp_pad'], 'id': 743, 'def': 'absorbent material saturated with ink used to transfer ink evenly to a rubber stamp', 'name': 'inkpad'}, {'frequency': 'c', 'synset': 'pad.n.04', 'synonyms': ['pad'], 'id': 744, 'def': 'mostly arm/knee pads labeled', 'name': 'pad'}, {'frequency': 'f', 'synset': 'paddle.n.04', 'synonyms': ['paddle', 'boat_paddle'], 'id': 745, 'def': 'a short light oar used without an oarlock to propel a canoe or small boat', 'name': 'paddle'}, {'frequency': 'c', 'synset': 'padlock.n.01', 'synonyms': ['padlock'], 'id': 746, 'def': 'a detachable, portable lock', 'name': 'padlock'}, {'frequency': 'c', 'synset': 'paintbrush.n.01', 'synonyms': ['paintbrush'], 'id': 747, 'def': 'a brush used as an applicator to apply paint', 'name': 'paintbrush'}, {'frequency': 'f', 'synset': 'painting.n.01', 'synonyms': ['painting'], 'id': 748, 'def': 'graphic art consisting of an artistic composition made by applying paints to a surface', 'name': 'painting'}, {'frequency': 'f', 'synset': 'pajama.n.02', 'synonyms': ['pajamas', 'pyjamas'], 'id': 749, 'def': 'loose-fitting nightclothes worn for sleeping or lounging', 'name': 'pajamas'}, {'frequency': 'c', 'synset': 'palette.n.02', 'synonyms': ['palette', 'pallet'], 'id': 750, 'def': 'board that provides a flat surface on which artists mix paints and the range of colors used', 'name': 'palette'}, {'frequency': 'f', 'synset': 'pan.n.01', 'synonyms': ['pan_(for_cooking)', 'cooking_pan'], 'id': 751, 'def': 'cooking utensil consisting of a wide metal vessel', 'name': 'pan_(for_cooking)'}, {'frequency': 'r', 'synset': 'pan.n.03', 'synonyms': ['pan_(metal_container)'], 'id': 752, 'def': 'shallow container made of metal', 'name': 'pan_(metal_container)'}, {'frequency': 'c', 'synset': 'pancake.n.01', 'synonyms': ['pancake'], 'id': 753, 'def': 'a flat cake of thin batter fried on both sides on a griddle', 'name': 'pancake'}, {'frequency': 'r', 'synset': 'pantyhose.n.01', 'synonyms': ['pantyhose'], 'id': 754, 'def': "a woman's tights consisting of underpants and stockings", 'name': 'pantyhose'}, {'frequency': 'r', 'synset': 'papaya.n.02', 'synonyms': ['papaya'], 'id': 755, 'def': 'large oval melon-like tropical fruit with yellowish flesh', 'name': 'papaya'}, {'frequency': 'f', 'synset': 'paper_plate.n.01', 'synonyms': ['paper_plate'], 'id': 756, 'def': 'a disposable plate made of cardboard', 'name': 'paper_plate'}, {'frequency': 'f', 'synset': 'paper_towel.n.01', 'synonyms': ['paper_towel'], 'id': 757, 'def': 'a disposable towel made of absorbent paper', 'name': 'paper_towel'}, {'frequency': 'r', 'synset': 'paperback_book.n.01', 'synonyms': ['paperback_book', 'paper-back_book', 'softback_book', 'soft-cover_book'], 'id': 758, 'def': 'a book with paper covers', 'name': 'paperback_book'}, {'frequency': 'r', 'synset': 'paperweight.n.01', 'synonyms': ['paperweight'], 'id': 759, 'def': 'a weight used to hold down a stack of papers', 'name': 'paperweight'}, {'frequency': 'c', 'synset': 'parachute.n.01', 'synonyms': ['parachute'], 'id': 760, 'def': 'rescue equipment consisting of a device that fills with air and retards your fall', 'name': 'parachute'}, {'frequency': 'c', 'synset': 'parakeet.n.01', 'synonyms': ['parakeet', 'parrakeet', 'parroket', 'paraquet', 'paroquet', 'parroquet'], 'id': 761, 'def': 'any of numerous small slender long-tailed parrots', 'name': 'parakeet'}, {'frequency': 'c', 'synset': 'parasail.n.01', 'synonyms': ['parasail_(sports)'], 'id': 762, 'def': 'parachute that will lift a person up into the air when it is towed by a motorboat or a car', 'name': 'parasail_(sports)'}, {'frequency': 'c', 'synset': 'parasol.n.01', 'synonyms': ['parasol', 'sunshade'], 'id': 763, 'def': 'a handheld collapsible source of shade', 'name': 'parasol'}, {'frequency': 'r', 'synset': 'parchment.n.01', 'synonyms': ['parchment'], 'id': 764, 'def': 'a superior paper resembling sheepskin', 'name': 'parchment'}, {'frequency': 'c', 'synset': 'parka.n.01', 'synonyms': ['parka', 'anorak'], 'id': 765, 'def': "a kind of heavy jacket (`windcheater' is a British term)", 'name': 'parka'}, {'frequency': 'f', 'synset': 'parking_meter.n.01', 'synonyms': ['parking_meter'], 'id': 766, 'def': 'a coin-operated timer located next to a parking space', 'name': 'parking_meter'}, {'frequency': 'c', 'synset': 'parrot.n.01', 'synonyms': ['parrot'], 'id': 767, 'def': 'usually brightly colored tropical birds with short hooked beaks and the ability to mimic sounds', 'name': 'parrot'}, {'frequency': 'c', 'synset': 'passenger_car.n.01', 'synonyms': ['passenger_car_(part_of_a_train)', 'coach_(part_of_a_train)'], 'id': 768, 'def': 'a railcar where passengers ride', 'name': 'passenger_car_(part_of_a_train)'}, {'frequency': 'r', 'synset': 'passenger_ship.n.01', 'synonyms': ['passenger_ship'], 'id': 769, 'def': 'a ship built to carry passengers', 'name': 'passenger_ship'}, {'frequency': 'c', 'synset': 'passport.n.02', 'synonyms': ['passport'], 'id': 770, 'def': 'a document issued by a country to a citizen allowing that person to travel abroad and re-enter the home country', 'name': 'passport'}, {'frequency': 'f', 'synset': 'pastry.n.02', 'synonyms': ['pastry'], 'id': 771, 'def': 'any of various baked foods made of dough or batter', 'name': 'pastry'}, {'frequency': 'r', 'synset': 'patty.n.01', 'synonyms': ['patty_(food)'], 'id': 772, 'def': 'small flat mass of chopped food', 'name': 'patty_(food)'}, {'frequency': 'c', 'synset': 'pea.n.01', 'synonyms': ['pea_(food)'], 'id': 773, 'def': 'seed of a pea plant used for food', 'name': 'pea_(food)'}, {'frequency': 'c', 'synset': 'peach.n.03', 'synonyms': ['peach'], 'id': 774, 'def': 'downy juicy fruit with sweet yellowish or whitish flesh', 'name': 'peach'}, {'frequency': 'c', 'synset': 'peanut_butter.n.01', 'synonyms': ['peanut_butter'], 'id': 775, 'def': 'a spread made from ground peanuts', 'name': 'peanut_butter'}, {'frequency': 'f', 'synset': 'pear.n.01', 'synonyms': ['pear'], 'id': 776, 'def': 'sweet juicy gritty-textured fruit available in many varieties', 'name': 'pear'}, {'frequency': 'c', 'synset': 'peeler.n.03', 'synonyms': ['peeler_(tool_for_fruit_and_vegetables)'], 'id': 777, 'def': 'a device for peeling vegetables or fruits', 'name': 'peeler_(tool_for_fruit_and_vegetables)'}, {'frequency': 'r', 'synset': 'peg.n.04', 'synonyms': ['wooden_leg', 'pegleg'], 'id': 778, 'def': 'a prosthesis that replaces a missing leg', 'name': 'wooden_leg'}, {'frequency': 'r', 'synset': 'pegboard.n.01', 'synonyms': ['pegboard'], 'id': 779, 'def': 'a board perforated with regularly spaced holes into which pegs can be fitted', 'name': 'pegboard'}, {'frequency': 'c', 'synset': 'pelican.n.01', 'synonyms': ['pelican'], 'id': 780, 'def': 'large long-winged warm-water seabird having a large bill with a distensible pouch for fish', 'name': 'pelican'}, {'frequency': 'f', 'synset': 'pen.n.01', 'synonyms': ['pen'], 'id': 781, 'def': 'a writing implement with a point from which ink flows', 'name': 'pen'}, {'frequency': 'f', 'synset': 'pencil.n.01', 'synonyms': ['pencil'], 'id': 782, 'def': 'a thin cylindrical pointed writing implement made of wood and graphite', 'name': 'pencil'}, {'frequency': 'r', 'synset': 'pencil_box.n.01', 'synonyms': ['pencil_box', 'pencil_case'], 'id': 783, 'def': 'a box for holding pencils', 'name': 'pencil_box'}, {'frequency': 'r', 'synset': 'pencil_sharpener.n.01', 'synonyms': ['pencil_sharpener'], 'id': 784, 'def': 'a rotary implement for sharpening the point on pencils', 'name': 'pencil_sharpener'}, {'frequency': 'r', 'synset': 'pendulum.n.01', 'synonyms': ['pendulum'], 'id': 785, 'def': 'an apparatus consisting of an object mounted so that it swings freely under the influence of gravity', 'name': 'pendulum'}, {'frequency': 'c', 'synset': 'penguin.n.01', 'synonyms': ['penguin'], 'id': 786, 'def': 'short-legged flightless birds of cold southern regions having webbed feet and wings modified as flippers', 'name': 'penguin'}, {'frequency': 'r', 'synset': 'pennant.n.02', 'synonyms': ['pennant'], 'id': 787, 'def': 'a flag longer than it is wide (and often tapering)', 'name': 'pennant'}, {'frequency': 'r', 'synset': 'penny.n.02', 'synonyms': ['penny_(coin)'], 'id': 788, 'def': 'a coin worth one-hundredth of the value of the basic unit', 'name': 'penny_(coin)'}, {'frequency': 'f', 'synset': 'pepper.n.03', 'synonyms': ['pepper', 'peppercorn'], 'id': 789, 'def': 'pungent seasoning from the berry of the common pepper plant; whole or ground', 'name': 'pepper'}, {'frequency': 'c', 'synset': 'pepper_mill.n.01', 'synonyms': ['pepper_mill', 'pepper_grinder'], 'id': 790, 'def': 'a mill for grinding pepper', 'name': 'pepper_mill'}, {'frequency': 'c', 'synset': 'perfume.n.02', 'synonyms': ['perfume'], 'id': 791, 'def': 'a toiletry that emits and diffuses a fragrant odor', 'name': 'perfume'}, {'frequency': 'r', 'synset': 'persimmon.n.02', 'synonyms': ['persimmon'], 'id': 792, 'def': 'orange fruit resembling a plum; edible when fully ripe', 'name': 'persimmon'}, {'frequency': 'f', 'synset': 'person.n.01', 'synonyms': ['person', 'baby', 'child', 'boy', 'girl', 'man', 'woman', 'human'], 'id': 793, 'def': 'a human being', 'name': 'person'}, {'frequency': 'c', 'synset': 'pet.n.01', 'synonyms': ['pet'], 'id': 794, 'def': 'a domesticated animal kept for companionship or amusement', 'name': 'pet'}, {'frequency': 'c', 'synset': 'pew.n.01', 'synonyms': ['pew_(church_bench)', 'church_bench'], 'id': 795, 'def': 'long bench with backs; used in church by the congregation', 'name': 'pew_(church_bench)'}, {'frequency': 'r', 'synset': 'phonebook.n.01', 'synonyms': ['phonebook', 'telephone_book', 'telephone_directory'], 'id': 796, 'def': 'a directory containing an alphabetical list of telephone subscribers and their telephone numbers', 'name': 'phonebook'}, {'frequency': 'c', 'synset': 'phonograph_record.n.01', 'synonyms': ['phonograph_record', 'phonograph_recording', 'record_(phonograph_recording)'], 'id': 797, 'def': 'sound recording consisting of a typically black disk with a continuous groove', 'name': 'phonograph_record'}, {'frequency': 'f', 'synset': 'piano.n.01', 'synonyms': ['piano'], 'id': 798, 'def': 'a keyboard instrument that is played by depressing keys that cause hammers to strike tuned strings and produce sounds', 'name': 'piano'}, {'frequency': 'f', 'synset': 'pickle.n.01', 'synonyms': ['pickle'], 'id': 799, 'def': 'vegetables (especially cucumbers) preserved in brine or vinegar', 'name': 'pickle'}, {'frequency': 'f', 'synset': 'pickup.n.01', 'synonyms': ['pickup_truck'], 'id': 800, 'def': 'a light truck with an open body and low sides and a tailboard', 'name': 'pickup_truck'}, {'frequency': 'c', 'synset': 'pie.n.01', 'synonyms': ['pie'], 'id': 801, 'def': 'dish baked in pastry-lined pan often with a pastry top', 'name': 'pie'}, {'frequency': 'c', 'synset': 'pigeon.n.01', 'synonyms': ['pigeon'], 'id': 802, 'def': 'wild and domesticated birds having a heavy body and short legs', 'name': 'pigeon'}, {'frequency': 'r', 'synset': 'piggy_bank.n.01', 'synonyms': ['piggy_bank', 'penny_bank'], 'id': 803, 'def': "a child's coin bank (often shaped like a pig)", 'name': 'piggy_bank'}, {'frequency': 'f', 'synset': 'pillow.n.01', 'synonyms': ['pillow'], 'id': 804, 'def': 'a cushion to support the head of a sleeping person', 'name': 'pillow'}, {'frequency': 'r', 'synset': 'pin.n.09', 'synonyms': ['pin_(non_jewelry)'], 'id': 805, 'def': 'a small slender (often pointed) piece of wood or metal used to support or fasten or attach things', 'name': 'pin_(non_jewelry)'}, {'frequency': 'f', 'synset': 'pineapple.n.02', 'synonyms': ['pineapple'], 'id': 806, 'def': 'large sweet fleshy tropical fruit with a tuft of stiff leaves', 'name': 'pineapple'}, {'frequency': 'c', 'synset': 'pinecone.n.01', 'synonyms': ['pinecone'], 'id': 807, 'def': 'the seed-producing cone of a pine tree', 'name': 'pinecone'}, {'frequency': 'r', 'synset': 'ping-pong_ball.n.01', 'synonyms': ['ping-pong_ball'], 'id': 808, 'def': 'light hollow ball used in playing table tennis', 'name': 'ping-pong_ball'}, {'frequency': 'r', 'synset': 'pinwheel.n.03', 'synonyms': ['pinwheel'], 'id': 809, 'def': 'a toy consisting of vanes of colored paper or plastic that is pinned to a stick and spins when it is pointed into the wind', 'name': 'pinwheel'}, {'frequency': 'r', 'synset': 'pipe.n.01', 'synonyms': ['tobacco_pipe'], 'id': 810, 'def': 'a tube with a small bowl at one end; used for smoking tobacco', 'name': 'tobacco_pipe'}, {'frequency': 'f', 'synset': 'pipe.n.02', 'synonyms': ['pipe', 'piping'], 'id': 811, 'def': 'a long tube made of metal or plastic that is used to carry water or oil or gas etc.', 'name': 'pipe'}, {'frequency': 'r', 'synset': 'pistol.n.01', 'synonyms': ['pistol', 'handgun'], 'id': 812, 'def': 'a firearm that is held and fired with one hand', 'name': 'pistol'}, {'frequency': 'c', 'synset': 'pita.n.01', 'synonyms': ['pita_(bread)', 'pocket_bread'], 'id': 813, 'def': 'usually small round bread that can open into a pocket for filling', 'name': 'pita_(bread)'}, {'frequency': 'f', 'synset': 'pitcher.n.02', 'synonyms': ['pitcher_(vessel_for_liquid)', 'ewer'], 'id': 814, 'def': 'an open vessel with a handle and a spout for pouring', 'name': 'pitcher_(vessel_for_liquid)'}, {'frequency': 'r', 'synset': 'pitchfork.n.01', 'synonyms': ['pitchfork'], 'id': 815, 'def': 'a long-handled hand tool with sharp widely spaced prongs for lifting and pitching hay', 'name': 'pitchfork'}, {'frequency': 'f', 'synset': 'pizza.n.01', 'synonyms': ['pizza'], 'id': 816, 'def': 'Italian open pie made of thin bread dough spread with a spiced mixture of e.g. tomato sauce and cheese', 'name': 'pizza'}, {'frequency': 'f', 'synset': 'place_mat.n.01', 'synonyms': ['place_mat'], 'id': 817, 'def': 'a mat placed on a table for an individual place setting', 'name': 'place_mat'}, {'frequency': 'f', 'synset': 'plate.n.04', 'synonyms': ['plate'], 'id': 818, 'def': 'dish on which food is served or from which food is eaten', 'name': 'plate'}, {'frequency': 'c', 'synset': 'platter.n.01', 'synonyms': ['platter'], 'id': 819, 'def': 'a large shallow dish used for serving food', 'name': 'platter'}, {'frequency': 'r', 'synset': 'playpen.n.01', 'synonyms': ['playpen'], 'id': 820, 'def': 'a portable enclosure in which babies may be left to play', 'name': 'playpen'}, {'frequency': 'c', 'synset': 'pliers.n.01', 'synonyms': ['pliers', 'plyers'], 'id': 821, 'def': 'a gripping hand tool with two hinged arms and (usually) serrated jaws', 'name': 'pliers'}, {'frequency': 'r', 'synset': 'plow.n.01', 'synonyms': ['plow_(farm_equipment)', 'plough_(farm_equipment)'], 'id': 822, 'def': 'a farm tool having one or more heavy blades to break the soil and cut a furrow prior to sowing', 'name': 'plow_(farm_equipment)'}, {'frequency': 'r', 'synset': 'plume.n.02', 'synonyms': ['plume'], 'id': 823, 'def': 'a feather or cluster of feathers worn as an ornament', 'name': 'plume'}, {'frequency': 'r', 'synset': 'pocket_watch.n.01', 'synonyms': ['pocket_watch'], 'id': 824, 'def': 'a watch that is carried in a small watch pocket', 'name': 'pocket_watch'}, {'frequency': 'c', 'synset': 'pocketknife.n.01', 'synonyms': ['pocketknife'], 'id': 825, 'def': 'a knife with a blade that folds into the handle; suitable for carrying in the pocket', 'name': 'pocketknife'}, {'frequency': 'c', 'synset': 'poker.n.01', 'synonyms': ['poker_(fire_stirring_tool)', 'stove_poker', 'fire_hook'], 'id': 826, 'def': 'fire iron consisting of a metal rod with a handle; used to stir a fire', 'name': 'poker_(fire_stirring_tool)'}, {'frequency': 'f', 'synset': 'pole.n.01', 'synonyms': ['pole', 'post'], 'id': 827, 'def': 'a long (usually round) rod of wood or metal or plastic', 'name': 'pole'}, {'frequency': 'f', 'synset': 'polo_shirt.n.01', 'synonyms': ['polo_shirt', 'sport_shirt'], 'id': 828, 'def': 'a shirt with short sleeves designed for comfort and casual wear', 'name': 'polo_shirt'}, {'frequency': 'r', 'synset': 'poncho.n.01', 'synonyms': ['poncho'], 'id': 829, 'def': 'a blanket-like cloak with a hole in the center for the head', 'name': 'poncho'}, {'frequency': 'c', 'synset': 'pony.n.05', 'synonyms': ['pony'], 'id': 830, 'def': 'any of various breeds of small gentle horses usually less than five feet high at the shoulder', 'name': 'pony'}, {'frequency': 'r', 'synset': 'pool_table.n.01', 'synonyms': ['pool_table', 'billiard_table', 'snooker_table'], 'id': 831, 'def': 'game equipment consisting of a heavy table on which pool is played', 'name': 'pool_table'}, {'frequency': 'f', 'synset': 'pop.n.02', 'synonyms': ['pop_(soda)', 'soda_(pop)', 'tonic', 'soft_drink'], 'id': 832, 'def': 'a sweet drink containing carbonated water and flavoring', 'name': 'pop_(soda)'}, {'frequency': 'c', 'synset': 'postbox.n.01', 'synonyms': ['postbox_(public)', 'mailbox_(public)'], 'id': 833, 'def': 'public box for deposit of mail', 'name': 'postbox_(public)'}, {'frequency': 'c', 'synset': 'postcard.n.01', 'synonyms': ['postcard', 'postal_card', 'mailing-card'], 'id': 834, 'def': 'a card for sending messages by post without an envelope', 'name': 'postcard'}, {'frequency': 'f', 'synset': 'poster.n.01', 'synonyms': ['poster', 'placard'], 'id': 835, 'def': 'a sign posted in a public place as an advertisement', 'name': 'poster'}, {'frequency': 'f', 'synset': 'pot.n.01', 'synonyms': ['pot'], 'id': 836, 'def': 'metal or earthenware cooking vessel that is usually round and deep; often has a handle and lid', 'name': 'pot'}, {'frequency': 'f', 'synset': 'pot.n.04', 'synonyms': ['flowerpot'], 'id': 837, 'def': 'a container in which plants are cultivated', 'name': 'flowerpot'}, {'frequency': 'f', 'synset': 'potato.n.01', 'synonyms': ['potato'], 'id': 838, 'def': 'an edible tuber native to South America', 'name': 'potato'}, {'frequency': 'c', 'synset': 'potholder.n.01', 'synonyms': ['potholder'], 'id': 839, 'def': 'an insulated pad for holding hot pots', 'name': 'potholder'}, {'frequency': 'c', 'synset': 'pottery.n.01', 'synonyms': ['pottery', 'clayware'], 'id': 840, 'def': 'ceramic ware made from clay and baked in a kiln', 'name': 'pottery'}, {'frequency': 'c', 'synset': 'pouch.n.01', 'synonyms': ['pouch'], 'id': 841, 'def': 'a small or medium size container for holding or carrying things', 'name': 'pouch'}, {'frequency': 'c', 'synset': 'power_shovel.n.01', 'synonyms': ['power_shovel', 'excavator', 'digger'], 'id': 842, 'def': 'a machine for excavating', 'name': 'power_shovel'}, {'frequency': 'c', 'synset': 'prawn.n.01', 'synonyms': ['prawn', 'shrimp'], 'id': 843, 'def': 'any of various edible decapod crustaceans', 'name': 'prawn'}, {'frequency': 'c', 'synset': 'pretzel.n.01', 'synonyms': ['pretzel'], 'id': 844, 'def': 'glazed and salted cracker typically in the shape of a loose knot', 'name': 'pretzel'}, {'frequency': 'f', 'synset': 'printer.n.03', 'synonyms': ['printer', 'printing_machine'], 'id': 845, 'def': 'a machine that prints', 'name': 'printer'}, {'frequency': 'c', 'synset': 'projectile.n.01', 'synonyms': ['projectile_(weapon)', 'missile'], 'id': 846, 'def': 'a weapon that is forcibly thrown or projected at a targets', 'name': 'projectile_(weapon)'}, {'frequency': 'c', 'synset': 'projector.n.02', 'synonyms': ['projector'], 'id': 847, 'def': 'an optical instrument that projects an enlarged image onto a screen', 'name': 'projector'}, {'frequency': 'f', 'synset': 'propeller.n.01', 'synonyms': ['propeller', 'propellor'], 'id': 848, 'def': 'a mechanical device that rotates to push against air or water', 'name': 'propeller'}, {'frequency': 'r', 'synset': 'prune.n.01', 'synonyms': ['prune'], 'id': 849, 'def': 'dried plum', 'name': 'prune'}, {'frequency': 'r', 'synset': 'pudding.n.01', 'synonyms': ['pudding'], 'id': 850, 'def': 'any of various soft thick unsweetened baked dishes', 'name': 'pudding'}, {'frequency': 'r', 'synset': 'puffer.n.02', 'synonyms': ['puffer_(fish)', 'pufferfish', 'blowfish', 'globefish'], 'id': 851, 'def': 'fishes whose elongated spiny body can inflate itself with water or air to form a globe', 'name': 'puffer_(fish)'}, {'frequency': 'r', 'synset': 'puffin.n.01', 'synonyms': ['puffin'], 'id': 852, 'def': 'seabirds having short necks and brightly colored compressed bills', 'name': 'puffin'}, {'frequency': 'r', 'synset': 'pug.n.01', 'synonyms': ['pug-dog'], 'id': 853, 'def': 'small compact smooth-coated breed of Asiatic origin having a tightly curled tail and broad flat wrinkled muzzle', 'name': 'pug-dog'}, {'frequency': 'c', 'synset': 'pumpkin.n.02', 'synonyms': ['pumpkin'], 'id': 854, 'def': 'usually large pulpy deep-yellow round fruit of the squash family maturing in late summer or early autumn', 'name': 'pumpkin'}, {'frequency': 'r', 'synset': 'punch.n.03', 'synonyms': ['puncher'], 'id': 855, 'def': 'a tool for making holes or indentations', 'name': 'puncher'}, {'frequency': 'r', 'synset': 'puppet.n.01', 'synonyms': ['puppet', 'marionette'], 'id': 856, 'def': 'a small figure of a person operated from above with strings by a puppeteer', 'name': 'puppet'}, {'frequency': 'c', 'synset': 'puppy.n.01', 'synonyms': ['puppy'], 'id': 857, 'def': 'a young dog', 'name': 'puppy'}, {'frequency': 'r', 'synset': 'quesadilla.n.01', 'synonyms': ['quesadilla'], 'id': 858, 'def': 'a tortilla that is filled with cheese and heated', 'name': 'quesadilla'}, {'frequency': 'r', 'synset': 'quiche.n.02', 'synonyms': ['quiche'], 'id': 859, 'def': 'a tart filled with rich unsweetened custard; often contains other ingredients (as cheese or ham or seafood or vegetables)', 'name': 'quiche'}, {'frequency': 'f', 'synset': 'quilt.n.01', 'synonyms': ['quilt', 'comforter'], 'id': 860, 'def': 'bedding made of two layers of cloth filled with stuffing and stitched together', 'name': 'quilt'}, {'frequency': 'c', 'synset': 'rabbit.n.01', 'synonyms': ['rabbit'], 'id': 861, 'def': 'any of various burrowing animals of the family Leporidae having long ears and short tails', 'name': 'rabbit'}, {'frequency': 'r', 'synset': 'racer.n.02', 'synonyms': ['race_car', 'racing_car'], 'id': 862, 'def': 'a fast car that competes in races', 'name': 'race_car'}, {'frequency': 'c', 'synset': 'racket.n.04', 'synonyms': ['racket', 'racquet'], 'id': 863, 'def': 'a sports implement used to strike a ball in various games', 'name': 'racket'}, {'frequency': 'r', 'synset': 'radar.n.01', 'synonyms': ['radar'], 'id': 864, 'def': 'measuring instrument in which the echo of a pulse of microwave radiation is used to detect and locate distant objects', 'name': 'radar'}, {'frequency': 'f', 'synset': 'radiator.n.03', 'synonyms': ['radiator'], 'id': 865, 'def': 'a mechanism consisting of a metal honeycomb through which hot fluids circulate', 'name': 'radiator'}, {'frequency': 'c', 'synset': 'radio_receiver.n.01', 'synonyms': ['radio_receiver', 'radio_set', 'radio', 'tuner_(radio)'], 'id': 866, 'def': 'an electronic receiver that detects and demodulates and amplifies transmitted radio signals', 'name': 'radio_receiver'}, {'frequency': 'c', 'synset': 'radish.n.03', 'synonyms': ['radish', 'daikon'], 'id': 867, 'def': 'pungent edible root of any of various cultivated radish plants', 'name': 'radish'}, {'frequency': 'c', 'synset': 'raft.n.01', 'synonyms': ['raft'], 'id': 868, 'def': 'a flat float (usually made of logs or planks) that can be used for transport or as a platform for swimmers', 'name': 'raft'}, {'frequency': 'r', 'synset': 'rag_doll.n.01', 'synonyms': ['rag_doll'], 'id': 869, 'def': 'a cloth doll that is stuffed and (usually) painted', 'name': 'rag_doll'}, {'frequency': 'c', 'synset': 'raincoat.n.01', 'synonyms': ['raincoat', 'waterproof_jacket'], 'id': 870, 'def': 'a water-resistant coat', 'name': 'raincoat'}, {'frequency': 'c', 'synset': 'ram.n.05', 'synonyms': ['ram_(animal)'], 'id': 871, 'def': 'uncastrated adult male sheep', 'name': 'ram_(animal)'}, {'frequency': 'c', 'synset': 'raspberry.n.02', 'synonyms': ['raspberry'], 'id': 872, 'def': 'red or black edible aggregate berries usually smaller than the related blackberries', 'name': 'raspberry'}, {'frequency': 'r', 'synset': 'rat.n.01', 'synonyms': ['rat'], 'id': 873, 'def': 'any of various long-tailed rodents similar to but larger than a mouse', 'name': 'rat'}, {'frequency': 'c', 'synset': 'razorblade.n.01', 'synonyms': ['razorblade'], 'id': 874, 'def': 'a blade that has very sharp edge', 'name': 'razorblade'}, {'frequency': 'c', 'synset': 'reamer.n.01', 'synonyms': ['reamer_(juicer)', 'juicer', 'juice_reamer'], 'id': 875, 'def': 'a squeezer with a conical ridged center that is used for squeezing juice from citrus fruit', 'name': 'reamer_(juicer)'}, {'frequency': 'f', 'synset': 'rearview_mirror.n.01', 'synonyms': ['rearview_mirror'], 'id': 876, 'def': 'vehicle mirror (side or rearview)', 'name': 'rearview_mirror'}, {'frequency': 'c', 'synset': 'receipt.n.02', 'synonyms': ['receipt'], 'id': 877, 'def': 'an acknowledgment (usually tangible) that payment has been made', 'name': 'receipt'}, {'frequency': 'c', 'synset': 'recliner.n.01', 'synonyms': ['recliner', 'reclining_chair', 'lounger_(chair)'], 'id': 878, 'def': 'an armchair whose back can be lowered and foot can be raised to allow the sitter to recline in it', 'name': 'recliner'}, {'frequency': 'c', 'synset': 'record_player.n.01', 'synonyms': ['record_player', 'phonograph_(record_player)', 'turntable'], 'id': 879, 'def': 'machine in which rotating records cause a stylus to vibrate and the vibrations are amplified acoustically or electronically', 'name': 'record_player'}, {'frequency': 'f', 'synset': 'reflector.n.01', 'synonyms': ['reflector'], 'id': 880, 'def': 'device that reflects light, radiation, etc.', 'name': 'reflector'}, {'frequency': 'f', 'synset': 'remote_control.n.01', 'synonyms': ['remote_control'], 'id': 881, 'def': 'a device that can be used to control a machine or apparatus from a distance', 'name': 'remote_control'}, {'frequency': 'c', 'synset': 'rhinoceros.n.01', 'synonyms': ['rhinoceros'], 'id': 882, 'def': 'massive powerful herbivorous odd-toed ungulate of southeast Asia and Africa having very thick skin and one or two horns on the snout', 'name': 'rhinoceros'}, {'frequency': 'r', 'synset': 'rib.n.03', 'synonyms': ['rib_(food)'], 'id': 883, 'def': 'cut of meat including one or more ribs', 'name': 'rib_(food)'}, {'frequency': 'c', 'synset': 'rifle.n.01', 'synonyms': ['rifle'], 'id': 884, 'def': 'a shoulder firearm with a long barrel', 'name': 'rifle'}, {'frequency': 'f', 'synset': 'ring.n.08', 'synonyms': ['ring'], 'id': 885, 'def': 'jewelry consisting of a circlet of precious metal (often set with jewels) worn on the finger', 'name': 'ring'}, {'frequency': 'r', 'synset': 'river_boat.n.01', 'synonyms': ['river_boat'], 'id': 886, 'def': 'a boat used on rivers or to ply a river', 'name': 'river_boat'}, {'frequency': 'r', 'synset': 'road_map.n.02', 'synonyms': ['road_map'], 'id': 887, 'def': '(NOT A ROAD) a MAP showing roads (for automobile travel)', 'name': 'road_map'}, {'frequency': 'c', 'synset': 'robe.n.01', 'synonyms': ['robe'], 'id': 888, 'def': 'any loose flowing garment', 'name': 'robe'}, {'frequency': 'c', 'synset': 'rocking_chair.n.01', 'synonyms': ['rocking_chair'], 'id': 889, 'def': 'a chair mounted on rockers', 'name': 'rocking_chair'}, {'frequency': 'r', 'synset': 'rodent.n.01', 'synonyms': ['rodent'], 'id': 890, 'def': 'relatively small placental mammals having a single pair of constantly growing incisor teeth specialized for gnawing', 'name': 'rodent'}, {'frequency': 'r', 'synset': 'roller_skate.n.01', 'synonyms': ['roller_skate'], 'id': 891, 'def': 'a shoe with pairs of rollers (small hard wheels) fixed to the sole', 'name': 'roller_skate'}, {'frequency': 'r', 'synset': 'rollerblade.n.01', 'synonyms': ['Rollerblade'], 'id': 892, 'def': 'an in-line variant of a roller skate', 'name': 'Rollerblade'}, {'frequency': 'c', 'synset': 'rolling_pin.n.01', 'synonyms': ['rolling_pin'], 'id': 893, 'def': 'utensil consisting of a cylinder (usually of wood) with a handle at each end; used to roll out dough', 'name': 'rolling_pin'}, {'frequency': 'r', 'synset': 'root_beer.n.01', 'synonyms': ['root_beer'], 'id': 894, 'def': 'carbonated drink containing extracts of roots and herbs', 'name': 'root_beer'}, {'frequency': 'c', 'synset': 'router.n.02', 'synonyms': ['router_(computer_equipment)'], 'id': 895, 'def': 'a device that forwards data packets between computer networks', 'name': 'router_(computer_equipment)'}, {'frequency': 'f', 'synset': 'rubber_band.n.01', 'synonyms': ['rubber_band', 'elastic_band'], 'id': 896, 'def': 'a narrow band of elastic rubber used to hold things (such as papers) together', 'name': 'rubber_band'}, {'frequency': 'c', 'synset': 'runner.n.08', 'synonyms': ['runner_(carpet)'], 'id': 897, 'def': 'a long narrow carpet', 'name': 'runner_(carpet)'}, {'frequency': 'f', 'synset': 'sack.n.01', 'synonyms': ['plastic_bag', 'paper_bag'], 'id': 898, 'def': "a bag made of paper or plastic for holding customer's purchases", 'name': 'plastic_bag'}, {'frequency': 'f', 'synset': 'saddle.n.01', 'synonyms': ['saddle_(on_an_animal)'], 'id': 899, 'def': 'a seat for the rider of a horse or camel', 'name': 'saddle_(on_an_animal)'}, {'frequency': 'f', 'synset': 'saddle_blanket.n.01', 'synonyms': ['saddle_blanket', 'saddlecloth', 'horse_blanket'], 'id': 900, 'def': 'stable gear consisting of a blanket placed under the saddle', 'name': 'saddle_blanket'}, {'frequency': 'c', 'synset': 'saddlebag.n.01', 'synonyms': ['saddlebag'], 'id': 901, 'def': 'a large bag (or pair of bags) hung over a saddle', 'name': 'saddlebag'}, {'frequency': 'r', 'synset': 'safety_pin.n.01', 'synonyms': ['safety_pin'], 'id': 902, 'def': 'a pin in the form of a clasp; has a guard so the point of the pin will not stick the user', 'name': 'safety_pin'}, {'frequency': 'f', 'synset': 'sail.n.01', 'synonyms': ['sail'], 'id': 903, 'def': 'a large piece of fabric by means of which wind is used to propel a sailing vessel', 'name': 'sail'}, {'frequency': 'f', 'synset': 'salad.n.01', 'synonyms': ['salad'], 'id': 904, 'def': 'food mixtures either arranged on a plate or tossed and served with a moist dressing; usually consisting of or including greens', 'name': 'salad'}, {'frequency': 'r', 'synset': 'salad_plate.n.01', 'synonyms': ['salad_plate', 'salad_bowl'], 'id': 905, 'def': 'a plate or bowl for individual servings of salad', 'name': 'salad_plate'}, {'frequency': 'c', 'synset': 'salami.n.01', 'synonyms': ['salami'], 'id': 906, 'def': 'highly seasoned fatty sausage of pork and beef usually dried', 'name': 'salami'}, {'frequency': 'c', 'synset': 'salmon.n.01', 'synonyms': ['salmon_(fish)'], 'id': 907, 'def': 'any of various large food and game fishes of northern waters', 'name': 'salmon_(fish)'}, {'frequency': 'r', 'synset': 'salmon.n.03', 'synonyms': ['salmon_(food)'], 'id': 908, 'def': 'flesh of any of various marine or freshwater fish of the family Salmonidae', 'name': 'salmon_(food)'}, {'frequency': 'c', 'synset': 'salsa.n.01', 'synonyms': ['salsa'], 'id': 909, 'def': 'spicy sauce of tomatoes and onions and chili peppers to accompany Mexican foods', 'name': 'salsa'}, {'frequency': 'f', 'synset': 'saltshaker.n.01', 'synonyms': ['saltshaker'], 'id': 910, 'def': 'a shaker with a perforated top for sprinkling salt', 'name': 'saltshaker'}, {'frequency': 'f', 'synset': 'sandal.n.01', 'synonyms': ['sandal_(type_of_shoe)'], 'id': 911, 'def': 'a shoe consisting of a sole fastened by straps to the foot', 'name': 'sandal_(type_of_shoe)'}, {'frequency': 'f', 'synset': 'sandwich.n.01', 'synonyms': ['sandwich'], 'id': 912, 'def': 'two (or more) slices of bread with a filling between them', 'name': 'sandwich'}, {'frequency': 'r', 'synset': 'satchel.n.01', 'synonyms': ['satchel'], 'id': 913, 'def': 'luggage consisting of a small case with a flat bottom and (usually) a shoulder strap', 'name': 'satchel'}, {'frequency': 'r', 'synset': 'saucepan.n.01', 'synonyms': ['saucepan'], 'id': 914, 'def': 'a deep pan with a handle; used for stewing or boiling', 'name': 'saucepan'}, {'frequency': 'f', 'synset': 'saucer.n.02', 'synonyms': ['saucer'], 'id': 915, 'def': 'a small shallow dish for holding a cup at the table', 'name': 'saucer'}, {'frequency': 'f', 'synset': 'sausage.n.01', 'synonyms': ['sausage'], 'id': 916, 'def': 'highly seasoned minced meat stuffed in casings', 'name': 'sausage'}, {'frequency': 'r', 'synset': 'sawhorse.n.01', 'synonyms': ['sawhorse', 'sawbuck'], 'id': 917, 'def': 'a framework for holding wood that is being sawed', 'name': 'sawhorse'}, {'frequency': 'r', 'synset': 'sax.n.02', 'synonyms': ['saxophone'], 'id': 918, 'def': "a wind instrument with a `J'-shaped form typically made of brass", 'name': 'saxophone'}, {'frequency': 'f', 'synset': 'scale.n.07', 'synonyms': ['scale_(measuring_instrument)'], 'id': 919, 'def': 'a measuring instrument for weighing; shows amount of mass', 'name': 'scale_(measuring_instrument)'}, {'frequency': 'r', 'synset': 'scarecrow.n.01', 'synonyms': ['scarecrow', 'strawman'], 'id': 920, 'def': 'an effigy in the shape of a man to frighten birds away from seeds', 'name': 'scarecrow'}, {'frequency': 'f', 'synset': 'scarf.n.01', 'synonyms': ['scarf'], 'id': 921, 'def': 'a garment worn around the head or neck or shoulders for warmth or decoration', 'name': 'scarf'}, {'frequency': 'c', 'synset': 'school_bus.n.01', 'synonyms': ['school_bus'], 'id': 922, 'def': 'a bus used to transport children to or from school', 'name': 'school_bus'}, {'frequency': 'f', 'synset': 'scissors.n.01', 'synonyms': ['scissors'], 'id': 923, 'def': 'a tool having two crossed pivoting blades with looped handles', 'name': 'scissors'}, {'frequency': 'f', 'synset': 'scoreboard.n.01', 'synonyms': ['scoreboard'], 'id': 924, 'def': 'a large board for displaying the score of a contest (and some other information)', 'name': 'scoreboard'}, {'frequency': 'r', 'synset': 'scraper.n.01', 'synonyms': ['scraper'], 'id': 925, 'def': 'any of various hand tools for scraping', 'name': 'scraper'}, {'frequency': 'c', 'synset': 'screwdriver.n.01', 'synonyms': ['screwdriver'], 'id': 926, 'def': 'a hand tool for driving screws; has a tip that fits into the head of a screw', 'name': 'screwdriver'}, {'frequency': 'f', 'synset': 'scrub_brush.n.01', 'synonyms': ['scrubbing_brush'], 'id': 927, 'def': 'a brush with short stiff bristles for heavy cleaning', 'name': 'scrubbing_brush'}, {'frequency': 'c', 'synset': 'sculpture.n.01', 'synonyms': ['sculpture'], 'id': 928, 'def': 'a three-dimensional work of art', 'name': 'sculpture'}, {'frequency': 'c', 'synset': 'seabird.n.01', 'synonyms': ['seabird', 'seafowl'], 'id': 929, 'def': 'a bird that frequents coastal waters and the open ocean: gulls; pelicans; gannets; cormorants; albatrosses; petrels; etc.', 'name': 'seabird'}, {'frequency': 'c', 'synset': 'seahorse.n.02', 'synonyms': ['seahorse'], 'id': 930, 'def': 'small fish with horse-like heads bent sharply downward and curled tails', 'name': 'seahorse'}, {'frequency': 'r', 'synset': 'seaplane.n.01', 'synonyms': ['seaplane', 'hydroplane'], 'id': 931, 'def': 'an airplane that can land on or take off from water', 'name': 'seaplane'}, {'frequency': 'c', 'synset': 'seashell.n.01', 'synonyms': ['seashell'], 'id': 932, 'def': 'the shell of a marine organism', 'name': 'seashell'}, {'frequency': 'c', 'synset': 'sewing_machine.n.01', 'synonyms': ['sewing_machine'], 'id': 933, 'def': 'a textile machine used as a home appliance for sewing', 'name': 'sewing_machine'}, {'frequency': 'c', 'synset': 'shaker.n.03', 'synonyms': ['shaker'], 'id': 934, 'def': 'a container in which something can be shaken', 'name': 'shaker'}, {'frequency': 'c', 'synset': 'shampoo.n.01', 'synonyms': ['shampoo'], 'id': 935, 'def': 'cleansing agent consisting of soaps or detergents used for washing the hair', 'name': 'shampoo'}, {'frequency': 'c', 'synset': 'shark.n.01', 'synonyms': ['shark'], 'id': 936, 'def': 'typically large carnivorous fishes with sharpe teeth', 'name': 'shark'}, {'frequency': 'r', 'synset': 'sharpener.n.01', 'synonyms': ['sharpener'], 'id': 937, 'def': 'any implement that is used to make something (an edge or a point) sharper', 'name': 'sharpener'}, {'frequency': 'r', 'synset': 'sharpie.n.03', 'synonyms': ['Sharpie'], 'id': 938, 'def': 'a pen with indelible ink that will write on any surface', 'name': 'Sharpie'}, {'frequency': 'r', 'synset': 'shaver.n.03', 'synonyms': ['shaver_(electric)', 'electric_shaver', 'electric_razor'], 'id': 939, 'def': 'a razor powered by an electric motor', 'name': 'shaver_(electric)'}, {'frequency': 'c', 'synset': 'shaving_cream.n.01', 'synonyms': ['shaving_cream', 'shaving_soap'], 'id': 940, 'def': 'toiletry consisting that forms a rich lather for softening the beard before shaving', 'name': 'shaving_cream'}, {'frequency': 'r', 'synset': 'shawl.n.01', 'synonyms': ['shawl'], 'id': 941, 'def': 'cloak consisting of an oblong piece of cloth used to cover the head and shoulders', 'name': 'shawl'}, {'frequency': 'r', 'synset': 'shears.n.01', 'synonyms': ['shears'], 'id': 942, 'def': 'large scissors with strong blades', 'name': 'shears'}, {'frequency': 'f', 'synset': 'sheep.n.01', 'synonyms': ['sheep'], 'id': 943, 'def': 'woolly usually horned ruminant mammal related to the goat', 'name': 'sheep'}, {'frequency': 'r', 'synset': 'shepherd_dog.n.01', 'synonyms': ['shepherd_dog', 'sheepdog'], 'id': 944, 'def': 'any of various usually long-haired breeds of dog reared to herd and guard sheep', 'name': 'shepherd_dog'}, {'frequency': 'r', 'synset': 'sherbert.n.01', 'synonyms': ['sherbert', 'sherbet'], 'id': 945, 'def': 'a frozen dessert made primarily of fruit juice and sugar', 'name': 'sherbert'}, {'frequency': 'c', 'synset': 'shield.n.02', 'synonyms': ['shield'], 'id': 946, 'def': 'armor carried on the arm to intercept blows', 'name': 'shield'}, {'frequency': 'f', 'synset': 'shirt.n.01', 'synonyms': ['shirt'], 'id': 947, 'def': 'a garment worn on the upper half of the body', 'name': 'shirt'}, {'frequency': 'f', 'synset': 'shoe.n.01', 'synonyms': ['shoe', 'sneaker_(type_of_shoe)', 'tennis_shoe'], 'id': 948, 'def': 'common footwear covering the foot', 'name': 'shoe'}, {'frequency': 'f', 'synset': 'shopping_bag.n.01', 'synonyms': ['shopping_bag'], 'id': 949, 'def': 'a bag made of plastic or strong paper (often with handles); used to transport goods after shopping', 'name': 'shopping_bag'}, {'frequency': 'c', 'synset': 'shopping_cart.n.01', 'synonyms': ['shopping_cart'], 'id': 950, 'def': 'a handcart that holds groceries or other goods while shopping', 'name': 'shopping_cart'}, {'frequency': 'f', 'synset': 'short_pants.n.01', 'synonyms': ['short_pants', 'shorts_(clothing)', 'trunks_(clothing)'], 'id': 951, 'def': 'trousers that end at or above the knee', 'name': 'short_pants'}, {'frequency': 'r', 'synset': 'shot_glass.n.01', 'synonyms': ['shot_glass'], 'id': 952, 'def': 'a small glass adequate to hold a single swallow of whiskey', 'name': 'shot_glass'}, {'frequency': 'f', 'synset': 'shoulder_bag.n.01', 'synonyms': ['shoulder_bag'], 'id': 953, 'def': 'a large handbag that can be carried by a strap looped over the shoulder', 'name': 'shoulder_bag'}, {'frequency': 'c', 'synset': 'shovel.n.01', 'synonyms': ['shovel'], 'id': 954, 'def': 'a hand tool for lifting loose material such as snow, dirt, etc.', 'name': 'shovel'}, {'frequency': 'f', 'synset': 'shower.n.01', 'synonyms': ['shower_head'], 'id': 955, 'def': 'a plumbing fixture that sprays water over you', 'name': 'shower_head'}, {'frequency': 'r', 'synset': 'shower_cap.n.01', 'synonyms': ['shower_cap'], 'id': 956, 'def': 'a tight cap worn to keep hair dry while showering', 'name': 'shower_cap'}, {'frequency': 'f', 'synset': 'shower_curtain.n.01', 'synonyms': ['shower_curtain'], 'id': 957, 'def': 'a curtain that keeps water from splashing out of the shower area', 'name': 'shower_curtain'}, {'frequency': 'r', 'synset': 'shredder.n.01', 'synonyms': ['shredder_(for_paper)'], 'id': 958, 'def': 'a device that shreds documents', 'name': 'shredder_(for_paper)'}, {'frequency': 'f', 'synset': 'signboard.n.01', 'synonyms': ['signboard'], 'id': 959, 'def': 'structure displaying a board on which advertisements can be posted', 'name': 'signboard'}, {'frequency': 'c', 'synset': 'silo.n.01', 'synonyms': ['silo'], 'id': 960, 'def': 'a cylindrical tower used for storing goods', 'name': 'silo'}, {'frequency': 'f', 'synset': 'sink.n.01', 'synonyms': ['sink'], 'id': 961, 'def': 'plumbing fixture consisting of a water basin fixed to a wall or floor and having a drainpipe', 'name': 'sink'}, {'frequency': 'f', 'synset': 'skateboard.n.01', 'synonyms': ['skateboard'], 'id': 962, 'def': 'a board with wheels that is ridden in a standing or crouching position and propelled by foot', 'name': 'skateboard'}, {'frequency': 'c', 'synset': 'skewer.n.01', 'synonyms': ['skewer'], 'id': 963, 'def': 'a long pin for holding meat in position while it is being roasted', 'name': 'skewer'}, {'frequency': 'f', 'synset': 'ski.n.01', 'synonyms': ['ski'], 'id': 964, 'def': 'sports equipment for skiing on snow', 'name': 'ski'}, {'frequency': 'f', 'synset': 'ski_boot.n.01', 'synonyms': ['ski_boot'], 'id': 965, 'def': 'a stiff boot that is fastened to a ski with a ski binding', 'name': 'ski_boot'}, {'frequency': 'f', 'synset': 'ski_parka.n.01', 'synonyms': ['ski_parka', 'ski_jacket'], 'id': 966, 'def': 'a parka to be worn while skiing', 'name': 'ski_parka'}, {'frequency': 'f', 'synset': 'ski_pole.n.01', 'synonyms': ['ski_pole'], 'id': 967, 'def': 'a pole with metal points used as an aid in skiing', 'name': 'ski_pole'}, {'frequency': 'f', 'synset': 'skirt.n.02', 'synonyms': ['skirt'], 'id': 968, 'def': 'a garment hanging from the waist; worn mainly by girls and women', 'name': 'skirt'}, {'frequency': 'r', 'synset': 'skullcap.n.01', 'synonyms': ['skullcap'], 'id': 969, 'def': 'rounded brimless cap fitting the crown of the head', 'name': 'skullcap'}, {'frequency': 'c', 'synset': 'sled.n.01', 'synonyms': ['sled', 'sledge', 'sleigh'], 'id': 970, 'def': 'a vehicle or flat object for transportation over snow by sliding or pulled by dogs, etc.', 'name': 'sled'}, {'frequency': 'c', 'synset': 'sleeping_bag.n.01', 'synonyms': ['sleeping_bag'], 'id': 971, 'def': 'large padded bag designed to be slept in outdoors', 'name': 'sleeping_bag'}, {'frequency': 'r', 'synset': 'sling.n.05', 'synonyms': ['sling_(bandage)', 'triangular_bandage'], 'id': 972, 'def': 'bandage to support an injured forearm; slung over the shoulder or neck', 'name': 'sling_(bandage)'}, {'frequency': 'c', 'synset': 'slipper.n.01', 'synonyms': ['slipper_(footwear)', 'carpet_slipper_(footwear)'], 'id': 973, 'def': 'low footwear that can be slipped on and off easily; usually worn indoors', 'name': 'slipper_(footwear)'}, {'frequency': 'r', 'synset': 'smoothie.n.02', 'synonyms': ['smoothie'], 'id': 974, 'def': 'a thick smooth drink consisting of fresh fruit pureed with ice cream or yoghurt or milk', 'name': 'smoothie'}, {'frequency': 'r', 'synset': 'snake.n.01', 'synonyms': ['snake', 'serpent'], 'id': 975, 'def': 'limbless scaly elongate reptile; some are venomous', 'name': 'snake'}, {'frequency': 'f', 'synset': 'snowboard.n.01', 'synonyms': ['snowboard'], 'id': 976, 'def': 'a board that resembles a broad ski or a small surfboard; used in a standing position to slide down snow-covered slopes', 'name': 'snowboard'}, {'frequency': 'c', 'synset': 'snowman.n.01', 'synonyms': ['snowman'], 'id': 977, 'def': 'a figure of a person made of packed snow', 'name': 'snowman'}, {'frequency': 'c', 'synset': 'snowmobile.n.01', 'synonyms': ['snowmobile'], 'id': 978, 'def': 'tracked vehicle for travel on snow having skis in front', 'name': 'snowmobile'}, {'frequency': 'f', 'synset': 'soap.n.01', 'synonyms': ['soap'], 'id': 979, 'def': 'a cleansing agent made from the salts of vegetable or animal fats', 'name': 'soap'}, {'frequency': 'f', 'synset': 'soccer_ball.n.01', 'synonyms': ['soccer_ball'], 'id': 980, 'def': "an inflated ball used in playing soccer (called `football' outside of the United States)", 'name': 'soccer_ball'}, {'frequency': 'f', 'synset': 'sock.n.01', 'synonyms': ['sock'], 'id': 981, 'def': 'cloth covering for the foot; worn inside the shoe; reaches to between the ankle and the knee', 'name': 'sock'}, {'frequency': 'f', 'synset': 'sofa.n.01', 'synonyms': ['sofa', 'couch', 'lounge'], 'id': 982, 'def': 'an upholstered seat for more than one person', 'name': 'sofa'}, {'frequency': 'r', 'synset': 'softball.n.01', 'synonyms': ['softball'], 'id': 983, 'def': 'ball used in playing softball', 'name': 'softball'}, {'frequency': 'c', 'synset': 'solar_array.n.01', 'synonyms': ['solar_array', 'solar_battery', 'solar_panel'], 'id': 984, 'def': 'electrical device consisting of a large array of connected solar cells', 'name': 'solar_array'}, {'frequency': 'r', 'synset': 'sombrero.n.02', 'synonyms': ['sombrero'], 'id': 985, 'def': 'a straw hat with a tall crown and broad brim; worn in American southwest and in Mexico', 'name': 'sombrero'}, {'frequency': 'f', 'synset': 'soup.n.01', 'synonyms': ['soup'], 'id': 986, 'def': 'liquid food especially of meat or fish or vegetable stock often containing pieces of solid food', 'name': 'soup'}, {'frequency': 'r', 'synset': 'soup_bowl.n.01', 'synonyms': ['soup_bowl'], 'id': 987, 'def': 'a bowl for serving soup', 'name': 'soup_bowl'}, {'frequency': 'c', 'synset': 'soupspoon.n.01', 'synonyms': ['soupspoon'], 'id': 988, 'def': 'a spoon with a rounded bowl for eating soup', 'name': 'soupspoon'}, {'frequency': 'c', 'synset': 'sour_cream.n.01', 'synonyms': ['sour_cream', 'soured_cream'], 'id': 989, 'def': 'soured light cream', 'name': 'sour_cream'}, {'frequency': 'r', 'synset': 'soya_milk.n.01', 'synonyms': ['soya_milk', 'soybean_milk', 'soymilk'], 'id': 990, 'def': 'a milk substitute containing soybean flour and water; used in some infant formulas and in making tofu', 'name': 'soya_milk'}, {'frequency': 'r', 'synset': 'space_shuttle.n.01', 'synonyms': ['space_shuttle'], 'id': 991, 'def': "a reusable spacecraft with wings for a controlled descent through the Earth's atmosphere", 'name': 'space_shuttle'}, {'frequency': 'r', 'synset': 'sparkler.n.02', 'synonyms': ['sparkler_(fireworks)'], 'id': 992, 'def': 'a firework that burns slowly and throws out a shower of sparks', 'name': 'sparkler_(fireworks)'}, {'frequency': 'f', 'synset': 'spatula.n.02', 'synonyms': ['spatula'], 'id': 993, 'def': 'a hand tool with a thin flexible blade used to mix or spread soft substances', 'name': 'spatula'}, {'frequency': 'r', 'synset': 'spear.n.01', 'synonyms': ['spear', 'lance'], 'id': 994, 'def': 'a long pointed rod used as a tool or weapon', 'name': 'spear'}, {'frequency': 'f', 'synset': 'spectacles.n.01', 'synonyms': ['spectacles', 'specs', 'eyeglasses', 'glasses'], 'id': 995, 'def': 'optical instrument consisting of a frame that holds a pair of lenses for correcting defective vision', 'name': 'spectacles'}, {'frequency': 'c', 'synset': 'spice_rack.n.01', 'synonyms': ['spice_rack'], 'id': 996, 'def': 'a rack for displaying containers filled with spices', 'name': 'spice_rack'}, {'frequency': 'c', 'synset': 'spider.n.01', 'synonyms': ['spider'], 'id': 997, 'def': 'predatory arachnid with eight legs, two poison fangs, two feelers, and usually two silk-spinning organs at the back end of the body', 'name': 'spider'}, {'frequency': 'r', 'synset': 'spiny_lobster.n.02', 'synonyms': ['crawfish', 'crayfish'], 'id': 998, 'def': 'large edible marine crustacean having a spiny carapace but lacking the large pincers of true lobsters', 'name': 'crawfish'}, {'frequency': 'c', 'synset': 'sponge.n.01', 'synonyms': ['sponge'], 'id': 999, 'def': 'a porous mass usable to absorb water typically used for cleaning', 'name': 'sponge'}, {'frequency': 'f', 'synset': 'spoon.n.01', 'synonyms': ['spoon'], 'id': 1000, 'def': 'a piece of cutlery with a shallow bowl-shaped container and a handle', 'name': 'spoon'}, {'frequency': 'c', 'synset': 'sportswear.n.01', 'synonyms': ['sportswear', 'athletic_wear', 'activewear'], 'id': 1001, 'def': 'attire worn for sport or for casual wear', 'name': 'sportswear'}, {'frequency': 'c', 'synset': 'spotlight.n.02', 'synonyms': ['spotlight'], 'id': 1002, 'def': 'a lamp that produces a strong beam of light to illuminate a restricted area; used to focus attention of a stage performer', 'name': 'spotlight'}, {'frequency': 'r', 'synset': 'squid.n.01', 'synonyms': ['squid_(food)', 'calamari', 'calamary'], 'id': 1003, 'def': '(Italian cuisine) squid prepared as food', 'name': 'squid_(food)'}, {'frequency': 'c', 'synset': 'squirrel.n.01', 'synonyms': ['squirrel'], 'id': 1004, 'def': 'a kind of arboreal rodent having a long bushy tail', 'name': 'squirrel'}, {'frequency': 'r', 'synset': 'stagecoach.n.01', 'synonyms': ['stagecoach'], 'id': 1005, 'def': 'a large coach-and-four formerly used to carry passengers and mail on regular routes between towns', 'name': 'stagecoach'}, {'frequency': 'c', 'synset': 'stapler.n.01', 'synonyms': ['stapler_(stapling_machine)'], 'id': 1006, 'def': 'a machine that inserts staples into sheets of paper in order to fasten them together', 'name': 'stapler_(stapling_machine)'}, {'frequency': 'c', 'synset': 'starfish.n.01', 'synonyms': ['starfish', 'sea_star'], 'id': 1007, 'def': 'echinoderms characterized by five arms extending from a central disk', 'name': 'starfish'}, {'frequency': 'f', 'synset': 'statue.n.01', 'synonyms': ['statue_(sculpture)'], 'id': 1008, 'def': 'a sculpture representing a human or animal', 'name': 'statue_(sculpture)'}, {'frequency': 'c', 'synset': 'steak.n.01', 'synonyms': ['steak_(food)'], 'id': 1009, 'def': 'a slice of meat cut from the fleshy part of an animal or large fish', 'name': 'steak_(food)'}, {'frequency': 'r', 'synset': 'steak_knife.n.01', 'synonyms': ['steak_knife'], 'id': 1010, 'def': 'a sharp table knife used in eating steak', 'name': 'steak_knife'}, {'frequency': 'f', 'synset': 'steering_wheel.n.01', 'synonyms': ['steering_wheel'], 'id': 1011, 'def': 'a handwheel that is used for steering', 'name': 'steering_wheel'}, {'frequency': 'r', 'synset': 'step_ladder.n.01', 'synonyms': ['stepladder'], 'id': 1012, 'def': 'a folding portable ladder hinged at the top', 'name': 'stepladder'}, {'frequency': 'c', 'synset': 'step_stool.n.01', 'synonyms': ['step_stool'], 'id': 1013, 'def': 'a stool that has one or two steps that fold under the seat', 'name': 'step_stool'}, {'frequency': 'c', 'synset': 'stereo.n.01', 'synonyms': ['stereo_(sound_system)'], 'id': 1014, 'def': 'electronic device for playing audio', 'name': 'stereo_(sound_system)'}, {'frequency': 'r', 'synset': 'stew.n.02', 'synonyms': ['stew'], 'id': 1015, 'def': 'food prepared by stewing especially meat or fish with vegetables', 'name': 'stew'}, {'frequency': 'r', 'synset': 'stirrer.n.02', 'synonyms': ['stirrer'], 'id': 1016, 'def': 'an implement used for stirring', 'name': 'stirrer'}, {'frequency': 'f', 'synset': 'stirrup.n.01', 'synonyms': ['stirrup'], 'id': 1017, 'def': "support consisting of metal loops into which rider's feet go", 'name': 'stirrup'}, {'frequency': 'f', 'synset': 'stool.n.01', 'synonyms': ['stool'], 'id': 1018, 'def': 'a simple seat without a back or arms', 'name': 'stool'}, {'frequency': 'f', 'synset': 'stop_sign.n.01', 'synonyms': ['stop_sign'], 'id': 1019, 'def': 'a traffic sign to notify drivers that they must come to a complete stop', 'name': 'stop_sign'}, {'frequency': 'f', 'synset': 'stoplight.n.01', 'synonyms': ['brake_light'], 'id': 1020, 'def': 'a red light on the rear of a motor vehicle that signals when the brakes are applied', 'name': 'brake_light'}, {'frequency': 'f', 'synset': 'stove.n.01', 'synonyms': ['stove', 'kitchen_stove', 'range_(kitchen_appliance)', 'kitchen_range', 'cooking_stove'], 'id': 1021, 'def': 'a kitchen appliance used for cooking food', 'name': 'stove'}, {'frequency': 'c', 'synset': 'strainer.n.01', 'synonyms': ['strainer'], 'id': 1022, 'def': 'a filter to retain larger pieces while smaller pieces and liquids pass through', 'name': 'strainer'}, {'frequency': 'f', 'synset': 'strap.n.01', 'synonyms': ['strap'], 'id': 1023, 'def': 'an elongated strip of material for binding things together or holding', 'name': 'strap'}, {'frequency': 'f', 'synset': 'straw.n.04', 'synonyms': ['straw_(for_drinking)', 'drinking_straw'], 'id': 1024, 'def': 'a thin paper or plastic tube used to suck liquids into the mouth', 'name': 'straw_(for_drinking)'}, {'frequency': 'f', 'synset': 'strawberry.n.01', 'synonyms': ['strawberry'], 'id': 1025, 'def': 'sweet fleshy red fruit', 'name': 'strawberry'}, {'frequency': 'f', 'synset': 'street_sign.n.01', 'synonyms': ['street_sign'], 'id': 1026, 'def': 'a sign visible from the street', 'name': 'street_sign'}, {'frequency': 'f', 'synset': 'streetlight.n.01', 'synonyms': ['streetlight', 'street_lamp'], 'id': 1027, 'def': 'a lamp supported on a lamppost; for illuminating a street', 'name': 'streetlight'}, {'frequency': 'r', 'synset': 'string_cheese.n.01', 'synonyms': ['string_cheese'], 'id': 1028, 'def': 'cheese formed in long strings twisted together', 'name': 'string_cheese'}, {'frequency': 'r', 'synset': 'stylus.n.02', 'synonyms': ['stylus'], 'id': 1029, 'def': 'a pointed tool for writing or drawing or engraving, including pens', 'name': 'stylus'}, {'frequency': 'r', 'synset': 'subwoofer.n.01', 'synonyms': ['subwoofer'], 'id': 1030, 'def': 'a loudspeaker that is designed to reproduce very low bass frequencies', 'name': 'subwoofer'}, {'frequency': 'r', 'synset': 'sugar_bowl.n.01', 'synonyms': ['sugar_bowl'], 'id': 1031, 'def': 'a dish in which sugar is served', 'name': 'sugar_bowl'}, {'frequency': 'r', 'synset': 'sugarcane.n.01', 'synonyms': ['sugarcane_(plant)'], 'id': 1032, 'def': 'juicy canes whose sap is a source of molasses and commercial sugar; fresh canes are sometimes chewed for the juice', 'name': 'sugarcane_(plant)'}, {'frequency': 'f', 'synset': 'suit.n.01', 'synonyms': ['suit_(clothing)'], 'id': 1033, 'def': 'a set of garments (usually including a jacket and trousers or skirt) for outerwear all of the same fabric and color', 'name': 'suit_(clothing)'}, {'frequency': 'c', 'synset': 'sunflower.n.01', 'synonyms': ['sunflower'], 'id': 1034, 'def': 'any plant of the genus Helianthus having large flower heads with dark disk florets and showy yellow rays', 'name': 'sunflower'}, {'frequency': 'f', 'synset': 'sunglasses.n.01', 'synonyms': ['sunglasses'], 'id': 1035, 'def': 'spectacles that are darkened or polarized to protect the eyes from the glare of the sun', 'name': 'sunglasses'}, {'frequency': 'c', 'synset': 'sunhat.n.01', 'synonyms': ['sunhat'], 'id': 1036, 'def': 'a hat with a broad brim that protects the face from direct exposure to the sun', 'name': 'sunhat'}, {'frequency': 'f', 'synset': 'surfboard.n.01', 'synonyms': ['surfboard'], 'id': 1037, 'def': 'a narrow buoyant board for riding surf', 'name': 'surfboard'}, {'frequency': 'c', 'synset': 'sushi.n.01', 'synonyms': ['sushi'], 'id': 1038, 'def': 'rice (with raw fish) wrapped in seaweed', 'name': 'sushi'}, {'frequency': 'c', 'synset': 'swab.n.02', 'synonyms': ['mop'], 'id': 1039, 'def': 'cleaning implement consisting of absorbent material fastened to a handle; for cleaning floors', 'name': 'mop'}, {'frequency': 'c', 'synset': 'sweat_pants.n.01', 'synonyms': ['sweat_pants'], 'id': 1040, 'def': 'loose-fitting trousers with elastic cuffs; worn by athletes', 'name': 'sweat_pants'}, {'frequency': 'c', 'synset': 'sweatband.n.02', 'synonyms': ['sweatband'], 'id': 1041, 'def': 'a band of material tied around the forehead or wrist to absorb sweat', 'name': 'sweatband'}, {'frequency': 'f', 'synset': 'sweater.n.01', 'synonyms': ['sweater'], 'id': 1042, 'def': 'a crocheted or knitted garment covering the upper part of the body', 'name': 'sweater'}, {'frequency': 'f', 'synset': 'sweatshirt.n.01', 'synonyms': ['sweatshirt'], 'id': 1043, 'def': 'cotton knit pullover with long sleeves worn during athletic activity', 'name': 'sweatshirt'}, {'frequency': 'c', 'synset': 'sweet_potato.n.02', 'synonyms': ['sweet_potato'], 'id': 1044, 'def': 'the edible tuberous root of the sweet potato vine', 'name': 'sweet_potato'}, {'frequency': 'f', 'synset': 'swimsuit.n.01', 'synonyms': ['swimsuit', 'swimwear', 'bathing_suit', 'swimming_costume', 'bathing_costume', 'swimming_trunks', 'bathing_trunks'], 'id': 1045, 'def': 'garment worn for swimming', 'name': 'swimsuit'}, {'frequency': 'c', 'synset': 'sword.n.01', 'synonyms': ['sword'], 'id': 1046, 'def': 'a cutting or thrusting weapon that has a long metal blade', 'name': 'sword'}, {'frequency': 'r', 'synset': 'syringe.n.01', 'synonyms': ['syringe'], 'id': 1047, 'def': 'a medical instrument used to inject or withdraw fluids', 'name': 'syringe'}, {'frequency': 'r', 'synset': 'tabasco.n.02', 'synonyms': ['Tabasco_sauce'], 'id': 1048, 'def': 'very spicy sauce (trade name Tabasco) made from fully-aged red peppers', 'name': 'Tabasco_sauce'}, {'frequency': 'r', 'synset': 'table-tennis_table.n.01', 'synonyms': ['table-tennis_table', 'ping-pong_table'], 'id': 1049, 'def': 'a table used for playing table tennis', 'name': 'table-tennis_table'}, {'frequency': 'f', 'synset': 'table.n.02', 'synonyms': ['table'], 'id': 1050, 'def': 'a piece of furniture having a smooth flat top that is usually supported by one or more vertical legs', 'name': 'table'}, {'frequency': 'c', 'synset': 'table_lamp.n.01', 'synonyms': ['table_lamp'], 'id': 1051, 'def': 'a lamp that sits on a table', 'name': 'table_lamp'}, {'frequency': 'f', 'synset': 'tablecloth.n.01', 'synonyms': ['tablecloth'], 'id': 1052, 'def': 'a covering spread over a dining table', 'name': 'tablecloth'}, {'frequency': 'r', 'synset': 'tachometer.n.01', 'synonyms': ['tachometer'], 'id': 1053, 'def': 'measuring instrument for indicating speed of rotation', 'name': 'tachometer'}, {'frequency': 'r', 'synset': 'taco.n.02', 'synonyms': ['taco'], 'id': 1054, 'def': 'a small tortilla cupped around a filling', 'name': 'taco'}, {'frequency': 'f', 'synset': 'tag.n.02', 'synonyms': ['tag'], 'id': 1055, 'def': 'a label associated with something for the purpose of identification or information', 'name': 'tag'}, {'frequency': 'f', 'synset': 'taillight.n.01', 'synonyms': ['taillight', 'rear_light'], 'id': 1056, 'def': 'lamp (usually red) mounted at the rear of a motor vehicle', 'name': 'taillight'}, {'frequency': 'r', 'synset': 'tambourine.n.01', 'synonyms': ['tambourine'], 'id': 1057, 'def': 'a shallow drum with a single drumhead and with metallic disks in the sides', 'name': 'tambourine'}, {'frequency': 'r', 'synset': 'tank.n.01', 'synonyms': ['army_tank', 'armored_combat_vehicle', 'armoured_combat_vehicle'], 'id': 1058, 'def': 'an enclosed armored military vehicle; has a cannon and moves on caterpillar treads', 'name': 'army_tank'}, {'frequency': 'f', 'synset': 'tank.n.02', 'synonyms': ['tank_(storage_vessel)', 'storage_tank'], 'id': 1059, 'def': 'a large (usually metallic) vessel for holding gases or liquids', 'name': 'tank_(storage_vessel)'}, {'frequency': 'f', 'synset': 'tank_top.n.01', 'synonyms': ['tank_top_(clothing)'], 'id': 1060, 'def': 'a tight-fitting sleeveless shirt with wide shoulder straps and low neck and no front opening', 'name': 'tank_top_(clothing)'}, {'frequency': 'f', 'synset': 'tape.n.01', 'synonyms': ['tape_(sticky_cloth_or_paper)'], 'id': 1061, 'def': 'a long thin piece of cloth or paper as used for binding or fastening', 'name': 'tape_(sticky_cloth_or_paper)'}, {'frequency': 'c', 'synset': 'tape.n.04', 'synonyms': ['tape_measure', 'measuring_tape'], 'id': 1062, 'def': 'measuring instrument consisting of a narrow strip (cloth or metal) marked in inches or centimeters and used for measuring lengths', 'name': 'tape_measure'}, {'frequency': 'c', 'synset': 'tapestry.n.02', 'synonyms': ['tapestry'], 'id': 1063, 'def': 'a heavy textile with a woven design; used for curtains and upholstery', 'name': 'tapestry'}, {'frequency': 'f', 'synset': 'tarpaulin.n.01', 'synonyms': ['tarp'], 'id': 1064, 'def': 'waterproofed canvas', 'name': 'tarp'}, {'frequency': 'c', 'synset': 'tartan.n.01', 'synonyms': ['tartan', 'plaid'], 'id': 1065, 'def': 'a cloth having a crisscross design', 'name': 'tartan'}, {'frequency': 'c', 'synset': 'tassel.n.01', 'synonyms': ['tassel'], 'id': 1066, 'def': 'adornment consisting of a bunch of cords fastened at one end', 'name': 'tassel'}, {'frequency': 'c', 'synset': 'tea_bag.n.01', 'synonyms': ['tea_bag'], 'id': 1067, 'def': 'a measured amount of tea in a bag for an individual serving of tea', 'name': 'tea_bag'}, {'frequency': 'c', 'synset': 'teacup.n.02', 'synonyms': ['teacup'], 'id': 1068, 'def': 'a cup from which tea is drunk', 'name': 'teacup'}, {'frequency': 'c', 'synset': 'teakettle.n.01', 'synonyms': ['teakettle'], 'id': 1069, 'def': 'kettle for boiling water to make tea', 'name': 'teakettle'}, {'frequency': 'f', 'synset': 'teapot.n.01', 'synonyms': ['teapot'], 'id': 1070, 'def': 'pot for brewing tea; usually has a spout and handle', 'name': 'teapot'}, {'frequency': 'f', 'synset': 'teddy.n.01', 'synonyms': ['teddy_bear'], 'id': 1071, 'def': "plaything consisting of a child's toy bear (usually plush and stuffed with soft materials)", 'name': 'teddy_bear'}, {'frequency': 'f', 'synset': 'telephone.n.01', 'synonyms': ['telephone', 'phone', 'telephone_set'], 'id': 1072, 'def': 'electronic device for communicating by voice over long distances (includes wired and wireless/cell phones)', 'name': 'telephone'}, {'frequency': 'c', 'synset': 'telephone_booth.n.01', 'synonyms': ['telephone_booth', 'phone_booth', 'call_box', 'telephone_box', 'telephone_kiosk'], 'id': 1073, 'def': 'booth for using a telephone', 'name': 'telephone_booth'}, {'frequency': 'f', 'synset': 'telephone_pole.n.01', 'synonyms': ['telephone_pole', 'telegraph_pole', 'telegraph_post'], 'id': 1074, 'def': 'tall pole supporting telephone wires', 'name': 'telephone_pole'}, {'frequency': 'r', 'synset': 'telephoto_lens.n.01', 'synonyms': ['telephoto_lens', 'zoom_lens'], 'id': 1075, 'def': 'a camera lens that magnifies the image', 'name': 'telephoto_lens'}, {'frequency': 'c', 'synset': 'television_camera.n.01', 'synonyms': ['television_camera', 'tv_camera'], 'id': 1076, 'def': 'television equipment for capturing and recording video', 'name': 'television_camera'}, {'frequency': 'f', 'synset': 'television_receiver.n.01', 'synonyms': ['television_set', 'tv', 'tv_set'], 'id': 1077, 'def': 'an electronic device that receives television signals and displays them on a screen', 'name': 'television_set'}, {'frequency': 'f', 'synset': 'tennis_ball.n.01', 'synonyms': ['tennis_ball'], 'id': 1078, 'def': 'ball about the size of a fist used in playing tennis', 'name': 'tennis_ball'}, {'frequency': 'f', 'synset': 'tennis_racket.n.01', 'synonyms': ['tennis_racket'], 'id': 1079, 'def': 'a racket used to play tennis', 'name': 'tennis_racket'}, {'frequency': 'r', 'synset': 'tequila.n.01', 'synonyms': ['tequila'], 'id': 1080, 'def': 'Mexican liquor made from fermented juices of an agave plant', 'name': 'tequila'}, {'frequency': 'c', 'synset': 'thermometer.n.01', 'synonyms': ['thermometer'], 'id': 1081, 'def': 'measuring instrument for measuring temperature', 'name': 'thermometer'}, {'frequency': 'c', 'synset': 'thermos.n.01', 'synonyms': ['thermos_bottle'], 'id': 1082, 'def': 'vacuum flask that preserves temperature of hot or cold drinks', 'name': 'thermos_bottle'}, {'frequency': 'f', 'synset': 'thermostat.n.01', 'synonyms': ['thermostat'], 'id': 1083, 'def': 'a regulator for automatically regulating temperature by starting or stopping the supply of heat', 'name': 'thermostat'}, {'frequency': 'r', 'synset': 'thimble.n.02', 'synonyms': ['thimble'], 'id': 1084, 'def': 'a small metal cap to protect the finger while sewing; can be used as a small container', 'name': 'thimble'}, {'frequency': 'c', 'synset': 'thread.n.01', 'synonyms': ['thread', 'yarn'], 'id': 1085, 'def': 'a fine cord of twisted fibers (of cotton or silk or wool or nylon etc.) used in sewing and weaving', 'name': 'thread'}, {'frequency': 'c', 'synset': 'thumbtack.n.01', 'synonyms': ['thumbtack', 'drawing_pin', 'pushpin'], 'id': 1086, 'def': 'a tack for attaching papers to a bulletin board or drawing board', 'name': 'thumbtack'}, {'frequency': 'c', 'synset': 'tiara.n.01', 'synonyms': ['tiara'], 'id': 1087, 'def': 'a jeweled headdress worn by women on formal occasions', 'name': 'tiara'}, {'frequency': 'c', 'synset': 'tiger.n.02', 'synonyms': ['tiger'], 'id': 1088, 'def': 'large feline of forests in most of Asia having a tawny coat with black stripes', 'name': 'tiger'}, {'frequency': 'c', 'synset': 'tights.n.01', 'synonyms': ['tights_(clothing)', 'leotards'], 'id': 1089, 'def': 'skintight knit hose covering the body from the waist to the feet worn by acrobats and dancers and as stockings by women and girls', 'name': 'tights_(clothing)'}, {'frequency': 'c', 'synset': 'timer.n.01', 'synonyms': ['timer', 'stopwatch'], 'id': 1090, 'def': 'a timepiece that measures a time interval and signals its end', 'name': 'timer'}, {'frequency': 'f', 'synset': 'tinfoil.n.01', 'synonyms': ['tinfoil'], 'id': 1091, 'def': 'foil made of tin or an alloy of tin and lead', 'name': 'tinfoil'}, {'frequency': 'c', 'synset': 'tinsel.n.01', 'synonyms': ['tinsel'], 'id': 1092, 'def': 'a showy decoration that is basically valueless', 'name': 'tinsel'}, {'frequency': 'f', 'synset': 'tissue.n.02', 'synonyms': ['tissue_paper'], 'id': 1093, 'def': 'a soft thin (usually translucent) paper', 'name': 'tissue_paper'}, {'frequency': 'c', 'synset': 'toast.n.01', 'synonyms': ['toast_(food)'], 'id': 1094, 'def': 'slice of bread that has been toasted', 'name': 'toast_(food)'}, {'frequency': 'f', 'synset': 'toaster.n.02', 'synonyms': ['toaster'], 'id': 1095, 'def': 'a kitchen appliance (usually electric) for toasting bread', 'name': 'toaster'}, {'frequency': 'f', 'synset': 'toaster_oven.n.01', 'synonyms': ['toaster_oven'], 'id': 1096, 'def': 'kitchen appliance consisting of a small electric oven for toasting or warming food', 'name': 'toaster_oven'}, {'frequency': 'f', 'synset': 'toilet.n.02', 'synonyms': ['toilet'], 'id': 1097, 'def': 'a plumbing fixture for defecation and urination', 'name': 'toilet'}, {'frequency': 'f', 'synset': 'toilet_tissue.n.01', 'synonyms': ['toilet_tissue', 'toilet_paper', 'bathroom_tissue'], 'id': 1098, 'def': 'a soft thin absorbent paper for use in toilets', 'name': 'toilet_tissue'}, {'frequency': 'f', 'synset': 'tomato.n.01', 'synonyms': ['tomato'], 'id': 1099, 'def': 'mildly acid red or yellow pulpy fruit eaten as a vegetable', 'name': 'tomato'}, {'frequency': 'f', 'synset': 'tongs.n.01', 'synonyms': ['tongs'], 'id': 1100, 'def': 'any of various devices for taking hold of objects; usually have two hinged legs with handles above and pointed hooks below', 'name': 'tongs'}, {'frequency': 'c', 'synset': 'toolbox.n.01', 'synonyms': ['toolbox'], 'id': 1101, 'def': 'a box or chest or cabinet for holding hand tools', 'name': 'toolbox'}, {'frequency': 'f', 'synset': 'toothbrush.n.01', 'synonyms': ['toothbrush'], 'id': 1102, 'def': 'small brush; has long handle; used to clean teeth', 'name': 'toothbrush'}, {'frequency': 'f', 'synset': 'toothpaste.n.01', 'synonyms': ['toothpaste'], 'id': 1103, 'def': 'a dentifrice in the form of a paste', 'name': 'toothpaste'}, {'frequency': 'f', 'synset': 'toothpick.n.01', 'synonyms': ['toothpick'], 'id': 1104, 'def': 'pick consisting of a small strip of wood or plastic; used to pick food from between the teeth', 'name': 'toothpick'}, {'frequency': 'f', 'synset': 'top.n.09', 'synonyms': ['cover'], 'id': 1105, 'def': 'covering for a hole (especially a hole in the top of a container)', 'name': 'cover'}, {'frequency': 'c', 'synset': 'tortilla.n.01', 'synonyms': ['tortilla'], 'id': 1106, 'def': 'thin unleavened pancake made from cornmeal or wheat flour', 'name': 'tortilla'}, {'frequency': 'c', 'synset': 'tow_truck.n.01', 'synonyms': ['tow_truck'], 'id': 1107, 'def': 'a truck equipped to hoist and pull wrecked cars (or to remove cars from no-parking zones)', 'name': 'tow_truck'}, {'frequency': 'f', 'synset': 'towel.n.01', 'synonyms': ['towel'], 'id': 1108, 'def': 'a rectangular piece of absorbent cloth (or paper) for drying or wiping', 'name': 'towel'}, {'frequency': 'f', 'synset': 'towel_rack.n.01', 'synonyms': ['towel_rack', 'towel_rail', 'towel_bar'], 'id': 1109, 'def': 'a rack consisting of one or more bars on which towels can be hung', 'name': 'towel_rack'}, {'frequency': 'f', 'synset': 'toy.n.03', 'synonyms': ['toy'], 'id': 1110, 'def': 'a device regarded as providing amusement', 'name': 'toy'}, {'frequency': 'c', 'synset': 'tractor.n.01', 'synonyms': ['tractor_(farm_equipment)'], 'id': 1111, 'def': 'a wheeled vehicle with large wheels; used in farming and other applications', 'name': 'tractor_(farm_equipment)'}, {'frequency': 'f', 'synset': 'traffic_light.n.01', 'synonyms': ['traffic_light'], 'id': 1112, 'def': 'a device to control vehicle traffic often consisting of three or more lights', 'name': 'traffic_light'}, {'frequency': 'c', 'synset': 'trail_bike.n.01', 'synonyms': ['dirt_bike'], 'id': 1113, 'def': 'a lightweight motorcycle equipped with rugged tires and suspension for off-road use', 'name': 'dirt_bike'}, {'frequency': 'f', 'synset': 'trailer_truck.n.01', 'synonyms': ['trailer_truck', 'tractor_trailer', 'trucking_rig', 'articulated_lorry', 'semi_truck'], 'id': 1114, 'def': 'a truck consisting of a tractor and trailer together', 'name': 'trailer_truck'}, {'frequency': 'f', 'synset': 'train.n.01', 'synonyms': ['train_(railroad_vehicle)', 'railroad_train'], 'id': 1115, 'def': 'public or private transport provided by a line of railway cars coupled together and drawn by a locomotive', 'name': 'train_(railroad_vehicle)'}, {'frequency': 'r', 'synset': 'trampoline.n.01', 'synonyms': ['trampoline'], 'id': 1116, 'def': 'gymnastic apparatus consisting of a strong canvas sheet attached with springs to a metal frame', 'name': 'trampoline'}, {'frequency': 'f', 'synset': 'tray.n.01', 'synonyms': ['tray'], 'id': 1117, 'def': 'an open receptacle for holding or displaying or serving articles or food', 'name': 'tray'}, {'frequency': 'r', 'synset': 'trench_coat.n.01', 'synonyms': ['trench_coat'], 'id': 1118, 'def': 'a military style raincoat; belted with deep pockets', 'name': 'trench_coat'}, {'frequency': 'r', 'synset': 'triangle.n.05', 'synonyms': ['triangle_(musical_instrument)'], 'id': 1119, 'def': 'a percussion instrument consisting of a metal bar bent in the shape of an open triangle', 'name': 'triangle_(musical_instrument)'}, {'frequency': 'c', 'synset': 'tricycle.n.01', 'synonyms': ['tricycle'], 'id': 1120, 'def': 'a vehicle with three wheels that is moved by foot pedals', 'name': 'tricycle'}, {'frequency': 'f', 'synset': 'tripod.n.01', 'synonyms': ['tripod'], 'id': 1121, 'def': 'a three-legged rack used for support', 'name': 'tripod'}, {'frequency': 'f', 'synset': 'trouser.n.01', 'synonyms': ['trousers', 'pants_(clothing)'], 'id': 1122, 'def': 'a garment extending from the waist to the knee or ankle, covering each leg separately', 'name': 'trousers'}, {'frequency': 'f', 'synset': 'truck.n.01', 'synonyms': ['truck'], 'id': 1123, 'def': 'an automotive vehicle suitable for hauling', 'name': 'truck'}, {'frequency': 'r', 'synset': 'truffle.n.03', 'synonyms': ['truffle_(chocolate)', 'chocolate_truffle'], 'id': 1124, 'def': 'creamy chocolate candy', 'name': 'truffle_(chocolate)'}, {'frequency': 'c', 'synset': 'trunk.n.02', 'synonyms': ['trunk'], 'id': 1125, 'def': 'luggage consisting of a large strong case used when traveling or for storage', 'name': 'trunk'}, {'frequency': 'r', 'synset': 'tub.n.02', 'synonyms': ['vat'], 'id': 1126, 'def': 'a large vessel for holding or storing liquids', 'name': 'vat'}, {'frequency': 'c', 'synset': 'turban.n.01', 'synonyms': ['turban'], 'id': 1127, 'def': 'a traditional headdress consisting of a long scarf wrapped around the head', 'name': 'turban'}, {'frequency': 'c', 'synset': 'turkey.n.04', 'synonyms': ['turkey_(food)'], 'id': 1128, 'def': 'flesh of large domesticated fowl usually roasted', 'name': 'turkey_(food)'}, {'frequency': 'r', 'synset': 'turnip.n.01', 'synonyms': ['turnip'], 'id': 1129, 'def': 'widely cultivated plant having a large fleshy edible white or yellow root', 'name': 'turnip'}, {'frequency': 'c', 'synset': 'turtle.n.02', 'synonyms': ['turtle'], 'id': 1130, 'def': 'any of various aquatic and land reptiles having a bony shell and flipper-like limbs for swimming', 'name': 'turtle'}, {'frequency': 'c', 'synset': 'turtleneck.n.01', 'synonyms': ['turtleneck_(clothing)', 'polo-neck'], 'id': 1131, 'def': 'a sweater or jersey with a high close-fitting collar', 'name': 'turtleneck_(clothing)'}, {'frequency': 'c', 'synset': 'typewriter.n.01', 'synonyms': ['typewriter'], 'id': 1132, 'def': 'hand-operated character printer for printing written messages one character at a time', 'name': 'typewriter'}, {'frequency': 'f', 'synset': 'umbrella.n.01', 'synonyms': ['umbrella'], 'id': 1133, 'def': 'a lightweight handheld collapsible canopy', 'name': 'umbrella'}, {'frequency': 'f', 'synset': 'underwear.n.01', 'synonyms': ['underwear', 'underclothes', 'underclothing', 'underpants'], 'id': 1134, 'def': 'undergarment worn next to the skin and under the outer garments', 'name': 'underwear'}, {'frequency': 'r', 'synset': 'unicycle.n.01', 'synonyms': ['unicycle'], 'id': 1135, 'def': 'a vehicle with a single wheel that is driven by pedals', 'name': 'unicycle'}, {'frequency': 'f', 'synset': 'urinal.n.01', 'synonyms': ['urinal'], 'id': 1136, 'def': 'a plumbing fixture (usually attached to the wall) used by men to urinate', 'name': 'urinal'}, {'frequency': 'c', 'synset': 'urn.n.01', 'synonyms': ['urn'], 'id': 1137, 'def': 'a large vase that usually has a pedestal or feet', 'name': 'urn'}, {'frequency': 'c', 'synset': 'vacuum.n.04', 'synonyms': ['vacuum_cleaner'], 'id': 1138, 'def': 'an electrical home appliance that cleans by suction', 'name': 'vacuum_cleaner'}, {'frequency': 'f', 'synset': 'vase.n.01', 'synonyms': ['vase'], 'id': 1139, 'def': 'an open jar of glass or porcelain used as an ornament or to hold flowers', 'name': 'vase'}, {'frequency': 'c', 'synset': 'vending_machine.n.01', 'synonyms': ['vending_machine'], 'id': 1140, 'def': 'a slot machine for selling goods', 'name': 'vending_machine'}, {'frequency': 'f', 'synset': 'vent.n.01', 'synonyms': ['vent', 'blowhole', 'air_vent'], 'id': 1141, 'def': 'a hole for the escape of gas or air', 'name': 'vent'}, {'frequency': 'f', 'synset': 'vest.n.01', 'synonyms': ['vest', 'waistcoat'], 'id': 1142, 'def': "a man's sleeveless garment worn underneath a coat", 'name': 'vest'}, {'frequency': 'c', 'synset': 'videotape.n.01', 'synonyms': ['videotape'], 'id': 1143, 'def': 'a video recording made on magnetic tape', 'name': 'videotape'}, {'frequency': 'r', 'synset': 'vinegar.n.01', 'synonyms': ['vinegar'], 'id': 1144, 'def': 'sour-tasting liquid produced usually by oxidation of the alcohol in wine or cider and used as a condiment or food preservative', 'name': 'vinegar'}, {'frequency': 'r', 'synset': 'violin.n.01', 'synonyms': ['violin', 'fiddle'], 'id': 1145, 'def': 'bowed stringed instrument that is the highest member of the violin family', 'name': 'violin'}, {'frequency': 'r', 'synset': 'vodka.n.01', 'synonyms': ['vodka'], 'id': 1146, 'def': 'unaged colorless liquor originating in Russia', 'name': 'vodka'}, {'frequency': 'c', 'synset': 'volleyball.n.02', 'synonyms': ['volleyball'], 'id': 1147, 'def': 'an inflated ball used in playing volleyball', 'name': 'volleyball'}, {'frequency': 'r', 'synset': 'vulture.n.01', 'synonyms': ['vulture'], 'id': 1148, 'def': 'any of various large birds of prey having naked heads and weak claws and feeding chiefly on carrion', 'name': 'vulture'}, {'frequency': 'c', 'synset': 'waffle.n.01', 'synonyms': ['waffle'], 'id': 1149, 'def': 'pancake batter baked in a waffle iron', 'name': 'waffle'}, {'frequency': 'r', 'synset': 'waffle_iron.n.01', 'synonyms': ['waffle_iron'], 'id': 1150, 'def': 'a kitchen appliance for baking waffles', 'name': 'waffle_iron'}, {'frequency': 'c', 'synset': 'wagon.n.01', 'synonyms': ['wagon'], 'id': 1151, 'def': 'any of various kinds of wheeled vehicles drawn by an animal or a tractor', 'name': 'wagon'}, {'frequency': 'c', 'synset': 'wagon_wheel.n.01', 'synonyms': ['wagon_wheel'], 'id': 1152, 'def': 'a wheel of a wagon', 'name': 'wagon_wheel'}, {'frequency': 'c', 'synset': 'walking_stick.n.01', 'synonyms': ['walking_stick'], 'id': 1153, 'def': 'a stick carried in the hand for support in walking', 'name': 'walking_stick'}, {'frequency': 'c', 'synset': 'wall_clock.n.01', 'synonyms': ['wall_clock'], 'id': 1154, 'def': 'a clock mounted on a wall', 'name': 'wall_clock'}, {'frequency': 'f', 'synset': 'wall_socket.n.01', 'synonyms': ['wall_socket', 'wall_plug', 'electric_outlet', 'electrical_outlet', 'outlet', 'electric_receptacle'], 'id': 1155, 'def': 'receptacle providing a place in a wiring system where current can be taken to run electrical devices', 'name': 'wall_socket'}, {'frequency': 'f', 'synset': 'wallet.n.01', 'synonyms': ['wallet', 'billfold'], 'id': 1156, 'def': 'a pocket-size case for holding papers and paper money', 'name': 'wallet'}, {'frequency': 'r', 'synset': 'walrus.n.01', 'synonyms': ['walrus'], 'id': 1157, 'def': 'either of two large northern marine mammals having ivory tusks and tough hide over thick blubber', 'name': 'walrus'}, {'frequency': 'r', 'synset': 'wardrobe.n.01', 'synonyms': ['wardrobe'], 'id': 1158, 'def': 'a tall piece of furniture that provides storage space for clothes; has a door and rails or hooks for hanging clothes', 'name': 'wardrobe'}, {'frequency': 'r', 'synset': 'washbasin.n.01', 'synonyms': ['washbasin', 'basin_(for_washing)', 'washbowl', 'washstand', 'handbasin'], 'id': 1159, 'def': 'a bathroom sink that is permanently installed and connected to a water supply and drainpipe; where you can wash your hands and face', 'name': 'washbasin'}, {'frequency': 'c', 'synset': 'washer.n.03', 'synonyms': ['automatic_washer', 'washing_machine'], 'id': 1160, 'def': 'a home appliance for washing clothes and linens automatically', 'name': 'automatic_washer'}, {'frequency': 'f', 'synset': 'watch.n.01', 'synonyms': ['watch', 'wristwatch'], 'id': 1161, 'def': 'a small, portable timepiece', 'name': 'watch'}, {'frequency': 'f', 'synset': 'water_bottle.n.01', 'synonyms': ['water_bottle'], 'id': 1162, 'def': 'a bottle for holding water', 'name': 'water_bottle'}, {'frequency': 'c', 'synset': 'water_cooler.n.01', 'synonyms': ['water_cooler'], 'id': 1163, 'def': 'a device for cooling and dispensing drinking water', 'name': 'water_cooler'}, {'frequency': 'c', 'synset': 'water_faucet.n.01', 'synonyms': ['water_faucet', 'water_tap', 'tap_(water_faucet)'], 'id': 1164, 'def': 'a faucet for drawing water from a pipe or cask', 'name': 'water_faucet'}, {'frequency': 'r', 'synset': 'water_heater.n.01', 'synonyms': ['water_heater', 'hot-water_heater'], 'id': 1165, 'def': 'a heater and storage tank to supply heated water', 'name': 'water_heater'}, {'frequency': 'c', 'synset': 'water_jug.n.01', 'synonyms': ['water_jug'], 'id': 1166, 'def': 'a jug that holds water', 'name': 'water_jug'}, {'frequency': 'r', 'synset': 'water_pistol.n.01', 'synonyms': ['water_gun', 'squirt_gun'], 'id': 1167, 'def': 'plaything consisting of a toy pistol that squirts water', 'name': 'water_gun'}, {'frequency': 'c', 'synset': 'water_scooter.n.01', 'synonyms': ['water_scooter', 'sea_scooter', 'jet_ski'], 'id': 1168, 'def': 'a motorboat resembling a motor scooter (NOT A SURFBOARD OR WATER SKI)', 'name': 'water_scooter'}, {'frequency': 'c', 'synset': 'water_ski.n.01', 'synonyms': ['water_ski'], 'id': 1169, 'def': 'broad ski for skimming over water towed by a speedboat (DO NOT MARK WATER)', 'name': 'water_ski'}, {'frequency': 'c', 'synset': 'water_tower.n.01', 'synonyms': ['water_tower'], 'id': 1170, 'def': 'a large reservoir for water', 'name': 'water_tower'}, {'frequency': 'c', 'synset': 'watering_can.n.01', 'synonyms': ['watering_can'], 'id': 1171, 'def': 'a container with a handle and a spout with a perforated nozzle; used to sprinkle water over plants', 'name': 'watering_can'}, {'frequency': 'f', 'synset': 'watermelon.n.02', 'synonyms': ['watermelon'], 'id': 1172, 'def': 'large oblong or roundish melon with a hard green rind and sweet watery red or occasionally yellowish pulp', 'name': 'watermelon'}, {'frequency': 'f', 'synset': 'weathervane.n.01', 'synonyms': ['weathervane', 'vane_(weathervane)', 'wind_vane'], 'id': 1173, 'def': 'mechanical device attached to an elevated structure; rotates freely to show the direction of the wind', 'name': 'weathervane'}, {'frequency': 'c', 'synset': 'webcam.n.01', 'synonyms': ['webcam'], 'id': 1174, 'def': 'a digital camera designed to take digital photographs and transmit them over the internet', 'name': 'webcam'}, {'frequency': 'c', 'synset': 'wedding_cake.n.01', 'synonyms': ['wedding_cake', 'bridecake'], 'id': 1175, 'def': 'a rich cake with two or more tiers and covered with frosting and decorations; served at a wedding reception', 'name': 'wedding_cake'}, {'frequency': 'c', 'synset': 'wedding_ring.n.01', 'synonyms': ['wedding_ring', 'wedding_band'], 'id': 1176, 'def': 'a ring given to the bride and/or groom at the wedding', 'name': 'wedding_ring'}, {'frequency': 'f', 'synset': 'wet_suit.n.01', 'synonyms': ['wet_suit'], 'id': 1177, 'def': 'a close-fitting garment made of a permeable material; worn in cold water to retain body heat', 'name': 'wet_suit'}, {'frequency': 'f', 'synset': 'wheel.n.01', 'synonyms': ['wheel'], 'id': 1178, 'def': 'a circular frame with spokes (or a solid disc) that can rotate on a shaft or axle', 'name': 'wheel'}, {'frequency': 'c', 'synset': 'wheelchair.n.01', 'synonyms': ['wheelchair'], 'id': 1179, 'def': 'a movable chair mounted on large wheels', 'name': 'wheelchair'}, {'frequency': 'c', 'synset': 'whipped_cream.n.01', 'synonyms': ['whipped_cream'], 'id': 1180, 'def': 'cream that has been beaten until light and fluffy', 'name': 'whipped_cream'}, {'frequency': 'c', 'synset': 'whistle.n.03', 'synonyms': ['whistle'], 'id': 1181, 'def': 'a small wind instrument that produces a whistling sound by blowing into it', 'name': 'whistle'}, {'frequency': 'c', 'synset': 'wig.n.01', 'synonyms': ['wig'], 'id': 1182, 'def': 'hairpiece covering the head and made of real or synthetic hair', 'name': 'wig'}, {'frequency': 'c', 'synset': 'wind_chime.n.01', 'synonyms': ['wind_chime'], 'id': 1183, 'def': 'a decorative arrangement of pieces of metal or glass or pottery that hang together loosely so the wind can cause them to tinkle', 'name': 'wind_chime'}, {'frequency': 'c', 'synset': 'windmill.n.01', 'synonyms': ['windmill'], 'id': 1184, 'def': 'A mill or turbine that is powered by wind', 'name': 'windmill'}, {'frequency': 'c', 'synset': 'window_box.n.01', 'synonyms': ['window_box_(for_plants)'], 'id': 1185, 'def': 'a container for growing plants on a windowsill', 'name': 'window_box_(for_plants)'}, {'frequency': 'f', 'synset': 'windshield_wiper.n.01', 'synonyms': ['windshield_wiper', 'windscreen_wiper', 'wiper_(for_windshield/screen)'], 'id': 1186, 'def': 'a mechanical device that cleans the windshield', 'name': 'windshield_wiper'}, {'frequency': 'c', 'synset': 'windsock.n.01', 'synonyms': ['windsock', 'air_sock', 'air-sleeve', 'wind_sleeve', 'wind_cone'], 'id': 1187, 'def': 'a truncated cloth cone mounted on a mast/pole; shows wind direction', 'name': 'windsock'}, {'frequency': 'f', 'synset': 'wine_bottle.n.01', 'synonyms': ['wine_bottle'], 'id': 1188, 'def': 'a bottle for holding wine', 'name': 'wine_bottle'}, {'frequency': 'c', 'synset': 'wine_bucket.n.01', 'synonyms': ['wine_bucket', 'wine_cooler'], 'id': 1189, 'def': 'a bucket of ice used to chill a bottle of wine', 'name': 'wine_bucket'}, {'frequency': 'f', 'synset': 'wineglass.n.01', 'synonyms': ['wineglass'], 'id': 1190, 'def': 'a glass that has a stem and in which wine is served', 'name': 'wineglass'}, {'frequency': 'f', 'synset': 'winker.n.02', 'synonyms': ['blinder_(for_horses)'], 'id': 1191, 'def': 'blinds that prevent a horse from seeing something on either side', 'name': 'blinder_(for_horses)'}, {'frequency': 'c', 'synset': 'wok.n.01', 'synonyms': ['wok'], 'id': 1192, 'def': 'pan with a convex bottom; used for frying in Chinese cooking', 'name': 'wok'}, {'frequency': 'r', 'synset': 'wolf.n.01', 'synonyms': ['wolf'], 'id': 1193, 'def': 'a wild carnivorous mammal of the dog family, living and hunting in packs', 'name': 'wolf'}, {'frequency': 'c', 'synset': 'wooden_spoon.n.02', 'synonyms': ['wooden_spoon'], 'id': 1194, 'def': 'a spoon made of wood', 'name': 'wooden_spoon'}, {'frequency': 'c', 'synset': 'wreath.n.01', 'synonyms': ['wreath'], 'id': 1195, 'def': 'an arrangement of flowers, leaves, or stems fastened in a ring', 'name': 'wreath'}, {'frequency': 'c', 'synset': 'wrench.n.03', 'synonyms': ['wrench', 'spanner'], 'id': 1196, 'def': 'a hand tool that is used to hold or twist a nut or bolt', 'name': 'wrench'}, {'frequency': 'f', 'synset': 'wristband.n.01', 'synonyms': ['wristband'], 'id': 1197, 'def': 'band consisting of a part of a sleeve that covers the wrist', 'name': 'wristband'}, {'frequency': 'f', 'synset': 'wristlet.n.01', 'synonyms': ['wristlet', 'wrist_band'], 'id': 1198, 'def': 'a band or bracelet worn around the wrist', 'name': 'wristlet'}, {'frequency': 'c', 'synset': 'yacht.n.01', 'synonyms': ['yacht'], 'id': 1199, 'def': 'an expensive vessel propelled by sail or power and used for cruising or racing', 'name': 'yacht'}, {'frequency': 'c', 'synset': 'yogurt.n.01', 'synonyms': ['yogurt', 'yoghurt', 'yoghourt'], 'id': 1200, 'def': 'a custard-like food made from curdled milk', 'name': 'yogurt'}, {'frequency': 'c', 'synset': 'yoke.n.07', 'synonyms': ['yoke_(animal_equipment)'], 'id': 1201, 'def': 'gear joining two animals at the neck; NOT egg yolk', 'name': 'yoke_(animal_equipment)'}, {'frequency': 'f', 'synset': 'zebra.n.01', 'synonyms': ['zebra'], 'id': 1202, 'def': 'any of several fleet black-and-white striped African equines', 'name': 'zebra'}, {'frequency': 'c', 'synset': 'zucchini.n.02', 'synonyms': ['zucchini', 'courgette'], 'id': 1203, 'def': 'small cucumber-shaped vegetable marrow; typically dark green', 'name': 'zucchini'}] # noqa +# fmt: on diff --git a/annotator/oneformer/detectron2/data/datasets/lvis_v1_category_image_count.py b/annotator/oneformer/detectron2/data/datasets/lvis_v1_category_image_count.py new file mode 100644 index 0000000000000000000000000000000000000000..31bf0cfcd5096ab87835db86a28671d474514c40 --- /dev/null +++ b/annotator/oneformer/detectron2/data/datasets/lvis_v1_category_image_count.py @@ -0,0 +1,20 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# Autogen with +# with open("lvis_v1_train.json", "r") as f: +# a = json.load(f) +# c = a["categories"] +# for x in c: +# del x["name"] +# del x["instance_count"] +# del x["def"] +# del x["synonyms"] +# del x["frequency"] +# del x["synset"] +# LVIS_CATEGORY_IMAGE_COUNT = repr(c) + " # noqa" +# with open("/tmp/lvis_category_image_count.py", "wt") as f: +# f.write(f"LVIS_CATEGORY_IMAGE_COUNT = {LVIS_CATEGORY_IMAGE_COUNT}") +# Then paste the contents of that file below + +# fmt: off +LVIS_CATEGORY_IMAGE_COUNT = [{'id': 1, 'image_count': 64}, {'id': 2, 'image_count': 364}, {'id': 3, 'image_count': 1911}, {'id': 4, 'image_count': 149}, {'id': 5, 'image_count': 29}, {'id': 6, 'image_count': 26}, {'id': 7, 'image_count': 59}, {'id': 8, 'image_count': 22}, {'id': 9, 'image_count': 12}, {'id': 10, 'image_count': 28}, {'id': 11, 'image_count': 505}, {'id': 12, 'image_count': 1207}, {'id': 13, 'image_count': 4}, {'id': 14, 'image_count': 10}, {'id': 15, 'image_count': 500}, {'id': 16, 'image_count': 33}, {'id': 17, 'image_count': 3}, {'id': 18, 'image_count': 44}, {'id': 19, 'image_count': 561}, {'id': 20, 'image_count': 8}, {'id': 21, 'image_count': 9}, {'id': 22, 'image_count': 33}, {'id': 23, 'image_count': 1883}, {'id': 24, 'image_count': 98}, {'id': 25, 'image_count': 70}, {'id': 26, 'image_count': 46}, {'id': 27, 'image_count': 117}, {'id': 28, 'image_count': 41}, {'id': 29, 'image_count': 1395}, {'id': 30, 'image_count': 7}, {'id': 31, 'image_count': 1}, {'id': 32, 'image_count': 314}, {'id': 33, 'image_count': 31}, {'id': 34, 'image_count': 1905}, {'id': 35, 'image_count': 1859}, {'id': 36, 'image_count': 1623}, {'id': 37, 'image_count': 47}, {'id': 38, 'image_count': 3}, {'id': 39, 'image_count': 3}, {'id': 40, 'image_count': 1}, {'id': 41, 'image_count': 305}, {'id': 42, 'image_count': 6}, {'id': 43, 'image_count': 210}, {'id': 44, 'image_count': 36}, {'id': 45, 'image_count': 1787}, {'id': 46, 'image_count': 17}, {'id': 47, 'image_count': 51}, {'id': 48, 'image_count': 138}, {'id': 49, 'image_count': 3}, {'id': 50, 'image_count': 1470}, {'id': 51, 'image_count': 3}, {'id': 52, 'image_count': 2}, {'id': 53, 'image_count': 186}, {'id': 54, 'image_count': 76}, {'id': 55, 'image_count': 26}, {'id': 56, 'image_count': 303}, {'id': 57, 'image_count': 738}, {'id': 58, 'image_count': 1799}, {'id': 59, 'image_count': 1934}, {'id': 60, 'image_count': 1609}, {'id': 61, 'image_count': 1622}, {'id': 62, 'image_count': 41}, {'id': 63, 'image_count': 4}, {'id': 64, 'image_count': 11}, {'id': 65, 'image_count': 270}, {'id': 66, 'image_count': 349}, {'id': 67, 'image_count': 42}, {'id': 68, 'image_count': 823}, {'id': 69, 'image_count': 6}, {'id': 70, 'image_count': 48}, {'id': 71, 'image_count': 3}, {'id': 72, 'image_count': 42}, {'id': 73, 'image_count': 24}, {'id': 74, 'image_count': 16}, {'id': 75, 'image_count': 605}, {'id': 76, 'image_count': 646}, {'id': 77, 'image_count': 1765}, {'id': 78, 'image_count': 2}, {'id': 79, 'image_count': 125}, {'id': 80, 'image_count': 1420}, {'id': 81, 'image_count': 140}, {'id': 82, 'image_count': 4}, {'id': 83, 'image_count': 322}, {'id': 84, 'image_count': 60}, {'id': 85, 'image_count': 2}, {'id': 86, 'image_count': 231}, {'id': 87, 'image_count': 333}, {'id': 88, 'image_count': 1941}, {'id': 89, 'image_count': 367}, {'id': 90, 'image_count': 1922}, {'id': 91, 'image_count': 18}, {'id': 92, 'image_count': 81}, {'id': 93, 'image_count': 1}, {'id': 94, 'image_count': 1852}, {'id': 95, 'image_count': 430}, {'id': 96, 'image_count': 247}, {'id': 97, 'image_count': 94}, {'id': 98, 'image_count': 21}, {'id': 99, 'image_count': 1821}, {'id': 100, 'image_count': 16}, {'id': 101, 'image_count': 12}, {'id': 102, 'image_count': 25}, {'id': 103, 'image_count': 41}, {'id': 104, 'image_count': 244}, {'id': 105, 'image_count': 7}, {'id': 106, 'image_count': 1}, {'id': 107, 'image_count': 40}, {'id': 108, 'image_count': 40}, {'id': 109, 'image_count': 104}, {'id': 110, 'image_count': 1671}, {'id': 111, 'image_count': 49}, {'id': 112, 'image_count': 243}, {'id': 113, 'image_count': 2}, {'id': 114, 'image_count': 242}, {'id': 115, 'image_count': 271}, {'id': 116, 'image_count': 104}, {'id': 117, 'image_count': 8}, {'id': 118, 'image_count': 1758}, {'id': 119, 'image_count': 1}, {'id': 120, 'image_count': 48}, {'id': 121, 'image_count': 14}, {'id': 122, 'image_count': 40}, {'id': 123, 'image_count': 1}, {'id': 124, 'image_count': 37}, {'id': 125, 'image_count': 1510}, {'id': 126, 'image_count': 6}, {'id': 127, 'image_count': 1903}, {'id': 128, 'image_count': 70}, {'id': 129, 'image_count': 86}, {'id': 130, 'image_count': 7}, {'id': 131, 'image_count': 5}, {'id': 132, 'image_count': 1406}, {'id': 133, 'image_count': 1901}, {'id': 134, 'image_count': 15}, {'id': 135, 'image_count': 28}, {'id': 136, 'image_count': 6}, {'id': 137, 'image_count': 494}, {'id': 138, 'image_count': 234}, {'id': 139, 'image_count': 1922}, {'id': 140, 'image_count': 1}, {'id': 141, 'image_count': 35}, {'id': 142, 'image_count': 5}, {'id': 143, 'image_count': 1828}, {'id': 144, 'image_count': 8}, {'id': 145, 'image_count': 63}, {'id': 146, 'image_count': 1668}, {'id': 147, 'image_count': 4}, {'id': 148, 'image_count': 95}, {'id': 149, 'image_count': 17}, {'id': 150, 'image_count': 1567}, {'id': 151, 'image_count': 2}, {'id': 152, 'image_count': 103}, {'id': 153, 'image_count': 50}, {'id': 154, 'image_count': 1309}, {'id': 155, 'image_count': 6}, {'id': 156, 'image_count': 92}, {'id': 157, 'image_count': 19}, {'id': 158, 'image_count': 37}, {'id': 159, 'image_count': 4}, {'id': 160, 'image_count': 709}, {'id': 161, 'image_count': 9}, {'id': 162, 'image_count': 82}, {'id': 163, 'image_count': 15}, {'id': 164, 'image_count': 3}, {'id': 165, 'image_count': 61}, {'id': 166, 'image_count': 51}, {'id': 167, 'image_count': 5}, {'id': 168, 'image_count': 13}, {'id': 169, 'image_count': 642}, {'id': 170, 'image_count': 24}, {'id': 171, 'image_count': 255}, {'id': 172, 'image_count': 9}, {'id': 173, 'image_count': 1808}, {'id': 174, 'image_count': 31}, {'id': 175, 'image_count': 158}, {'id': 176, 'image_count': 80}, {'id': 177, 'image_count': 1884}, {'id': 178, 'image_count': 158}, {'id': 179, 'image_count': 2}, {'id': 180, 'image_count': 12}, {'id': 181, 'image_count': 1659}, {'id': 182, 'image_count': 7}, {'id': 183, 'image_count': 834}, {'id': 184, 'image_count': 57}, {'id': 185, 'image_count': 174}, {'id': 186, 'image_count': 95}, {'id': 187, 'image_count': 27}, {'id': 188, 'image_count': 22}, {'id': 189, 'image_count': 1391}, {'id': 190, 'image_count': 90}, {'id': 191, 'image_count': 40}, {'id': 192, 'image_count': 445}, {'id': 193, 'image_count': 21}, {'id': 194, 'image_count': 1132}, {'id': 195, 'image_count': 177}, {'id': 196, 'image_count': 4}, {'id': 197, 'image_count': 17}, {'id': 198, 'image_count': 84}, {'id': 199, 'image_count': 55}, {'id': 200, 'image_count': 30}, {'id': 201, 'image_count': 25}, {'id': 202, 'image_count': 2}, {'id': 203, 'image_count': 125}, {'id': 204, 'image_count': 1135}, {'id': 205, 'image_count': 19}, {'id': 206, 'image_count': 72}, {'id': 207, 'image_count': 1926}, {'id': 208, 'image_count': 159}, {'id': 209, 'image_count': 7}, {'id': 210, 'image_count': 1}, {'id': 211, 'image_count': 13}, {'id': 212, 'image_count': 35}, {'id': 213, 'image_count': 18}, {'id': 214, 'image_count': 8}, {'id': 215, 'image_count': 6}, {'id': 216, 'image_count': 35}, {'id': 217, 'image_count': 1222}, {'id': 218, 'image_count': 103}, {'id': 219, 'image_count': 28}, {'id': 220, 'image_count': 63}, {'id': 221, 'image_count': 28}, {'id': 222, 'image_count': 5}, {'id': 223, 'image_count': 7}, {'id': 224, 'image_count': 14}, {'id': 225, 'image_count': 1918}, {'id': 226, 'image_count': 133}, {'id': 227, 'image_count': 16}, {'id': 228, 'image_count': 27}, {'id': 229, 'image_count': 110}, {'id': 230, 'image_count': 1895}, {'id': 231, 'image_count': 4}, {'id': 232, 'image_count': 1927}, {'id': 233, 'image_count': 8}, {'id': 234, 'image_count': 1}, {'id': 235, 'image_count': 263}, {'id': 236, 'image_count': 10}, {'id': 237, 'image_count': 2}, {'id': 238, 'image_count': 3}, {'id': 239, 'image_count': 87}, {'id': 240, 'image_count': 9}, {'id': 241, 'image_count': 71}, {'id': 242, 'image_count': 13}, {'id': 243, 'image_count': 18}, {'id': 244, 'image_count': 2}, {'id': 245, 'image_count': 5}, {'id': 246, 'image_count': 45}, {'id': 247, 'image_count': 1}, {'id': 248, 'image_count': 23}, {'id': 249, 'image_count': 32}, {'id': 250, 'image_count': 4}, {'id': 251, 'image_count': 1}, {'id': 252, 'image_count': 858}, {'id': 253, 'image_count': 661}, {'id': 254, 'image_count': 168}, {'id': 255, 'image_count': 210}, {'id': 256, 'image_count': 65}, {'id': 257, 'image_count': 4}, {'id': 258, 'image_count': 2}, {'id': 259, 'image_count': 159}, {'id': 260, 'image_count': 31}, {'id': 261, 'image_count': 811}, {'id': 262, 'image_count': 1}, {'id': 263, 'image_count': 42}, {'id': 264, 'image_count': 27}, {'id': 265, 'image_count': 2}, {'id': 266, 'image_count': 5}, {'id': 267, 'image_count': 95}, {'id': 268, 'image_count': 32}, {'id': 269, 'image_count': 1}, {'id': 270, 'image_count': 1}, {'id': 271, 'image_count': 1844}, {'id': 272, 'image_count': 897}, {'id': 273, 'image_count': 31}, {'id': 274, 'image_count': 23}, {'id': 275, 'image_count': 1}, {'id': 276, 'image_count': 202}, {'id': 277, 'image_count': 746}, {'id': 278, 'image_count': 44}, {'id': 279, 'image_count': 14}, {'id': 280, 'image_count': 26}, {'id': 281, 'image_count': 1}, {'id': 282, 'image_count': 2}, {'id': 283, 'image_count': 25}, {'id': 284, 'image_count': 238}, {'id': 285, 'image_count': 592}, {'id': 286, 'image_count': 26}, {'id': 287, 'image_count': 5}, {'id': 288, 'image_count': 42}, {'id': 289, 'image_count': 13}, {'id': 290, 'image_count': 46}, {'id': 291, 'image_count': 1}, {'id': 292, 'image_count': 8}, {'id': 293, 'image_count': 34}, {'id': 294, 'image_count': 5}, {'id': 295, 'image_count': 1}, {'id': 296, 'image_count': 1871}, {'id': 297, 'image_count': 717}, {'id': 298, 'image_count': 1010}, {'id': 299, 'image_count': 679}, {'id': 300, 'image_count': 3}, {'id': 301, 'image_count': 4}, {'id': 302, 'image_count': 1}, {'id': 303, 'image_count': 166}, {'id': 304, 'image_count': 2}, {'id': 305, 'image_count': 266}, {'id': 306, 'image_count': 101}, {'id': 307, 'image_count': 6}, {'id': 308, 'image_count': 14}, {'id': 309, 'image_count': 133}, {'id': 310, 'image_count': 2}, {'id': 311, 'image_count': 38}, {'id': 312, 'image_count': 95}, {'id': 313, 'image_count': 1}, {'id': 314, 'image_count': 12}, {'id': 315, 'image_count': 49}, {'id': 316, 'image_count': 5}, {'id': 317, 'image_count': 5}, {'id': 318, 'image_count': 16}, {'id': 319, 'image_count': 216}, {'id': 320, 'image_count': 12}, {'id': 321, 'image_count': 1}, {'id': 322, 'image_count': 54}, {'id': 323, 'image_count': 5}, {'id': 324, 'image_count': 245}, {'id': 325, 'image_count': 12}, {'id': 326, 'image_count': 7}, {'id': 327, 'image_count': 35}, {'id': 328, 'image_count': 36}, {'id': 329, 'image_count': 32}, {'id': 330, 'image_count': 1027}, {'id': 331, 'image_count': 10}, {'id': 332, 'image_count': 12}, {'id': 333, 'image_count': 1}, {'id': 334, 'image_count': 67}, {'id': 335, 'image_count': 71}, {'id': 336, 'image_count': 30}, {'id': 337, 'image_count': 48}, {'id': 338, 'image_count': 249}, {'id': 339, 'image_count': 13}, {'id': 340, 'image_count': 29}, {'id': 341, 'image_count': 14}, {'id': 342, 'image_count': 236}, {'id': 343, 'image_count': 15}, {'id': 344, 'image_count': 1521}, {'id': 345, 'image_count': 25}, {'id': 346, 'image_count': 249}, {'id': 347, 'image_count': 139}, {'id': 348, 'image_count': 2}, {'id': 349, 'image_count': 2}, {'id': 350, 'image_count': 1890}, {'id': 351, 'image_count': 1240}, {'id': 352, 'image_count': 1}, {'id': 353, 'image_count': 9}, {'id': 354, 'image_count': 1}, {'id': 355, 'image_count': 3}, {'id': 356, 'image_count': 11}, {'id': 357, 'image_count': 4}, {'id': 358, 'image_count': 236}, {'id': 359, 'image_count': 44}, {'id': 360, 'image_count': 19}, {'id': 361, 'image_count': 1100}, {'id': 362, 'image_count': 7}, {'id': 363, 'image_count': 69}, {'id': 364, 'image_count': 2}, {'id': 365, 'image_count': 8}, {'id': 366, 'image_count': 5}, {'id': 367, 'image_count': 227}, {'id': 368, 'image_count': 6}, {'id': 369, 'image_count': 106}, {'id': 370, 'image_count': 81}, {'id': 371, 'image_count': 17}, {'id': 372, 'image_count': 134}, {'id': 373, 'image_count': 312}, {'id': 374, 'image_count': 8}, {'id': 375, 'image_count': 271}, {'id': 376, 'image_count': 2}, {'id': 377, 'image_count': 103}, {'id': 378, 'image_count': 1938}, {'id': 379, 'image_count': 574}, {'id': 380, 'image_count': 120}, {'id': 381, 'image_count': 2}, {'id': 382, 'image_count': 2}, {'id': 383, 'image_count': 13}, {'id': 384, 'image_count': 29}, {'id': 385, 'image_count': 1710}, {'id': 386, 'image_count': 66}, {'id': 387, 'image_count': 1008}, {'id': 388, 'image_count': 1}, {'id': 389, 'image_count': 3}, {'id': 390, 'image_count': 1942}, {'id': 391, 'image_count': 19}, {'id': 392, 'image_count': 1488}, {'id': 393, 'image_count': 46}, {'id': 394, 'image_count': 106}, {'id': 395, 'image_count': 115}, {'id': 396, 'image_count': 19}, {'id': 397, 'image_count': 2}, {'id': 398, 'image_count': 1}, {'id': 399, 'image_count': 28}, {'id': 400, 'image_count': 9}, {'id': 401, 'image_count': 192}, {'id': 402, 'image_count': 12}, {'id': 403, 'image_count': 21}, {'id': 404, 'image_count': 247}, {'id': 405, 'image_count': 6}, {'id': 406, 'image_count': 64}, {'id': 407, 'image_count': 7}, {'id': 408, 'image_count': 40}, {'id': 409, 'image_count': 542}, {'id': 410, 'image_count': 2}, {'id': 411, 'image_count': 1898}, {'id': 412, 'image_count': 36}, {'id': 413, 'image_count': 4}, {'id': 414, 'image_count': 1}, {'id': 415, 'image_count': 191}, {'id': 416, 'image_count': 6}, {'id': 417, 'image_count': 41}, {'id': 418, 'image_count': 39}, {'id': 419, 'image_count': 46}, {'id': 420, 'image_count': 1}, {'id': 421, 'image_count': 1451}, {'id': 422, 'image_count': 1878}, {'id': 423, 'image_count': 11}, {'id': 424, 'image_count': 82}, {'id': 425, 'image_count': 18}, {'id': 426, 'image_count': 1}, {'id': 427, 'image_count': 7}, {'id': 428, 'image_count': 3}, {'id': 429, 'image_count': 575}, {'id': 430, 'image_count': 1907}, {'id': 431, 'image_count': 8}, {'id': 432, 'image_count': 4}, {'id': 433, 'image_count': 32}, {'id': 434, 'image_count': 11}, {'id': 435, 'image_count': 4}, {'id': 436, 'image_count': 54}, {'id': 437, 'image_count': 202}, {'id': 438, 'image_count': 32}, {'id': 439, 'image_count': 3}, {'id': 440, 'image_count': 130}, {'id': 441, 'image_count': 119}, {'id': 442, 'image_count': 141}, {'id': 443, 'image_count': 29}, {'id': 444, 'image_count': 525}, {'id': 445, 'image_count': 1323}, {'id': 446, 'image_count': 2}, {'id': 447, 'image_count': 113}, {'id': 448, 'image_count': 16}, {'id': 449, 'image_count': 7}, {'id': 450, 'image_count': 35}, {'id': 451, 'image_count': 1908}, {'id': 452, 'image_count': 353}, {'id': 453, 'image_count': 18}, {'id': 454, 'image_count': 14}, {'id': 455, 'image_count': 77}, {'id': 456, 'image_count': 8}, {'id': 457, 'image_count': 37}, {'id': 458, 'image_count': 1}, {'id': 459, 'image_count': 346}, {'id': 460, 'image_count': 19}, {'id': 461, 'image_count': 1779}, {'id': 462, 'image_count': 23}, {'id': 463, 'image_count': 25}, {'id': 464, 'image_count': 67}, {'id': 465, 'image_count': 19}, {'id': 466, 'image_count': 28}, {'id': 467, 'image_count': 4}, {'id': 468, 'image_count': 27}, {'id': 469, 'image_count': 1861}, {'id': 470, 'image_count': 11}, {'id': 471, 'image_count': 13}, {'id': 472, 'image_count': 13}, {'id': 473, 'image_count': 32}, {'id': 474, 'image_count': 1767}, {'id': 475, 'image_count': 42}, {'id': 476, 'image_count': 17}, {'id': 477, 'image_count': 128}, {'id': 478, 'image_count': 1}, {'id': 479, 'image_count': 9}, {'id': 480, 'image_count': 10}, {'id': 481, 'image_count': 4}, {'id': 482, 'image_count': 9}, {'id': 483, 'image_count': 18}, {'id': 484, 'image_count': 41}, {'id': 485, 'image_count': 28}, {'id': 486, 'image_count': 3}, {'id': 487, 'image_count': 65}, {'id': 488, 'image_count': 9}, {'id': 489, 'image_count': 23}, {'id': 490, 'image_count': 24}, {'id': 491, 'image_count': 1}, {'id': 492, 'image_count': 2}, {'id': 493, 'image_count': 59}, {'id': 494, 'image_count': 48}, {'id': 495, 'image_count': 17}, {'id': 496, 'image_count': 1877}, {'id': 497, 'image_count': 18}, {'id': 498, 'image_count': 1920}, {'id': 499, 'image_count': 50}, {'id': 500, 'image_count': 1890}, {'id': 501, 'image_count': 99}, {'id': 502, 'image_count': 1530}, {'id': 503, 'image_count': 3}, {'id': 504, 'image_count': 11}, {'id': 505, 'image_count': 19}, {'id': 506, 'image_count': 3}, {'id': 507, 'image_count': 63}, {'id': 508, 'image_count': 5}, {'id': 509, 'image_count': 6}, {'id': 510, 'image_count': 233}, {'id': 511, 'image_count': 54}, {'id': 512, 'image_count': 36}, {'id': 513, 'image_count': 10}, {'id': 514, 'image_count': 124}, {'id': 515, 'image_count': 101}, {'id': 516, 'image_count': 3}, {'id': 517, 'image_count': 363}, {'id': 518, 'image_count': 3}, {'id': 519, 'image_count': 30}, {'id': 520, 'image_count': 18}, {'id': 521, 'image_count': 199}, {'id': 522, 'image_count': 97}, {'id': 523, 'image_count': 32}, {'id': 524, 'image_count': 121}, {'id': 525, 'image_count': 16}, {'id': 526, 'image_count': 12}, {'id': 527, 'image_count': 2}, {'id': 528, 'image_count': 214}, {'id': 529, 'image_count': 48}, {'id': 530, 'image_count': 26}, {'id': 531, 'image_count': 13}, {'id': 532, 'image_count': 4}, {'id': 533, 'image_count': 11}, {'id': 534, 'image_count': 123}, {'id': 535, 'image_count': 7}, {'id': 536, 'image_count': 200}, {'id': 537, 'image_count': 91}, {'id': 538, 'image_count': 9}, {'id': 539, 'image_count': 72}, {'id': 540, 'image_count': 1886}, {'id': 541, 'image_count': 4}, {'id': 542, 'image_count': 1}, {'id': 543, 'image_count': 1}, {'id': 544, 'image_count': 1932}, {'id': 545, 'image_count': 4}, {'id': 546, 'image_count': 56}, {'id': 547, 'image_count': 854}, {'id': 548, 'image_count': 755}, {'id': 549, 'image_count': 1843}, {'id': 550, 'image_count': 96}, {'id': 551, 'image_count': 7}, {'id': 552, 'image_count': 74}, {'id': 553, 'image_count': 66}, {'id': 554, 'image_count': 57}, {'id': 555, 'image_count': 44}, {'id': 556, 'image_count': 1905}, {'id': 557, 'image_count': 4}, {'id': 558, 'image_count': 90}, {'id': 559, 'image_count': 1635}, {'id': 560, 'image_count': 8}, {'id': 561, 'image_count': 5}, {'id': 562, 'image_count': 50}, {'id': 563, 'image_count': 545}, {'id': 564, 'image_count': 20}, {'id': 565, 'image_count': 193}, {'id': 566, 'image_count': 285}, {'id': 567, 'image_count': 3}, {'id': 568, 'image_count': 1}, {'id': 569, 'image_count': 1904}, {'id': 570, 'image_count': 294}, {'id': 571, 'image_count': 3}, {'id': 572, 'image_count': 5}, {'id': 573, 'image_count': 24}, {'id': 574, 'image_count': 2}, {'id': 575, 'image_count': 2}, {'id': 576, 'image_count': 16}, {'id': 577, 'image_count': 8}, {'id': 578, 'image_count': 154}, {'id': 579, 'image_count': 66}, {'id': 580, 'image_count': 1}, {'id': 581, 'image_count': 24}, {'id': 582, 'image_count': 1}, {'id': 583, 'image_count': 4}, {'id': 584, 'image_count': 75}, {'id': 585, 'image_count': 6}, {'id': 586, 'image_count': 126}, {'id': 587, 'image_count': 24}, {'id': 588, 'image_count': 22}, {'id': 589, 'image_count': 1872}, {'id': 590, 'image_count': 16}, {'id': 591, 'image_count': 423}, {'id': 592, 'image_count': 1927}, {'id': 593, 'image_count': 38}, {'id': 594, 'image_count': 3}, {'id': 595, 'image_count': 1945}, {'id': 596, 'image_count': 35}, {'id': 597, 'image_count': 1}, {'id': 598, 'image_count': 13}, {'id': 599, 'image_count': 9}, {'id': 600, 'image_count': 14}, {'id': 601, 'image_count': 37}, {'id': 602, 'image_count': 3}, {'id': 603, 'image_count': 4}, {'id': 604, 'image_count': 100}, {'id': 605, 'image_count': 195}, {'id': 606, 'image_count': 1}, {'id': 607, 'image_count': 12}, {'id': 608, 'image_count': 24}, {'id': 609, 'image_count': 489}, {'id': 610, 'image_count': 10}, {'id': 611, 'image_count': 1689}, {'id': 612, 'image_count': 42}, {'id': 613, 'image_count': 81}, {'id': 614, 'image_count': 894}, {'id': 615, 'image_count': 1868}, {'id': 616, 'image_count': 7}, {'id': 617, 'image_count': 1567}, {'id': 618, 'image_count': 10}, {'id': 619, 'image_count': 8}, {'id': 620, 'image_count': 7}, {'id': 621, 'image_count': 629}, {'id': 622, 'image_count': 89}, {'id': 623, 'image_count': 15}, {'id': 624, 'image_count': 134}, {'id': 625, 'image_count': 4}, {'id': 626, 'image_count': 1802}, {'id': 627, 'image_count': 595}, {'id': 628, 'image_count': 1210}, {'id': 629, 'image_count': 48}, {'id': 630, 'image_count': 418}, {'id': 631, 'image_count': 1846}, {'id': 632, 'image_count': 5}, {'id': 633, 'image_count': 221}, {'id': 634, 'image_count': 10}, {'id': 635, 'image_count': 7}, {'id': 636, 'image_count': 76}, {'id': 637, 'image_count': 22}, {'id': 638, 'image_count': 10}, {'id': 639, 'image_count': 341}, {'id': 640, 'image_count': 1}, {'id': 641, 'image_count': 705}, {'id': 642, 'image_count': 1900}, {'id': 643, 'image_count': 188}, {'id': 644, 'image_count': 227}, {'id': 645, 'image_count': 861}, {'id': 646, 'image_count': 6}, {'id': 647, 'image_count': 115}, {'id': 648, 'image_count': 5}, {'id': 649, 'image_count': 43}, {'id': 650, 'image_count': 14}, {'id': 651, 'image_count': 6}, {'id': 652, 'image_count': 15}, {'id': 653, 'image_count': 1167}, {'id': 654, 'image_count': 15}, {'id': 655, 'image_count': 994}, {'id': 656, 'image_count': 28}, {'id': 657, 'image_count': 2}, {'id': 658, 'image_count': 338}, {'id': 659, 'image_count': 334}, {'id': 660, 'image_count': 15}, {'id': 661, 'image_count': 102}, {'id': 662, 'image_count': 1}, {'id': 663, 'image_count': 8}, {'id': 664, 'image_count': 1}, {'id': 665, 'image_count': 1}, {'id': 666, 'image_count': 28}, {'id': 667, 'image_count': 91}, {'id': 668, 'image_count': 260}, {'id': 669, 'image_count': 131}, {'id': 670, 'image_count': 128}, {'id': 671, 'image_count': 3}, {'id': 672, 'image_count': 10}, {'id': 673, 'image_count': 39}, {'id': 674, 'image_count': 2}, {'id': 675, 'image_count': 925}, {'id': 676, 'image_count': 354}, {'id': 677, 'image_count': 31}, {'id': 678, 'image_count': 10}, {'id': 679, 'image_count': 215}, {'id': 680, 'image_count': 71}, {'id': 681, 'image_count': 43}, {'id': 682, 'image_count': 28}, {'id': 683, 'image_count': 34}, {'id': 684, 'image_count': 16}, {'id': 685, 'image_count': 273}, {'id': 686, 'image_count': 2}, {'id': 687, 'image_count': 999}, {'id': 688, 'image_count': 4}, {'id': 689, 'image_count': 107}, {'id': 690, 'image_count': 2}, {'id': 691, 'image_count': 1}, {'id': 692, 'image_count': 454}, {'id': 693, 'image_count': 9}, {'id': 694, 'image_count': 1901}, {'id': 695, 'image_count': 61}, {'id': 696, 'image_count': 91}, {'id': 697, 'image_count': 46}, {'id': 698, 'image_count': 1402}, {'id': 699, 'image_count': 74}, {'id': 700, 'image_count': 421}, {'id': 701, 'image_count': 226}, {'id': 702, 'image_count': 10}, {'id': 703, 'image_count': 1720}, {'id': 704, 'image_count': 261}, {'id': 705, 'image_count': 1337}, {'id': 706, 'image_count': 293}, {'id': 707, 'image_count': 62}, {'id': 708, 'image_count': 814}, {'id': 709, 'image_count': 407}, {'id': 710, 'image_count': 6}, {'id': 711, 'image_count': 16}, {'id': 712, 'image_count': 7}, {'id': 713, 'image_count': 1791}, {'id': 714, 'image_count': 2}, {'id': 715, 'image_count': 1915}, {'id': 716, 'image_count': 1940}, {'id': 717, 'image_count': 13}, {'id': 718, 'image_count': 16}, {'id': 719, 'image_count': 448}, {'id': 720, 'image_count': 12}, {'id': 721, 'image_count': 18}, {'id': 722, 'image_count': 4}, {'id': 723, 'image_count': 71}, {'id': 724, 'image_count': 189}, {'id': 725, 'image_count': 74}, {'id': 726, 'image_count': 103}, {'id': 727, 'image_count': 3}, {'id': 728, 'image_count': 110}, {'id': 729, 'image_count': 5}, {'id': 730, 'image_count': 9}, {'id': 731, 'image_count': 15}, {'id': 732, 'image_count': 25}, {'id': 733, 'image_count': 7}, {'id': 734, 'image_count': 647}, {'id': 735, 'image_count': 824}, {'id': 736, 'image_count': 100}, {'id': 737, 'image_count': 47}, {'id': 738, 'image_count': 121}, {'id': 739, 'image_count': 731}, {'id': 740, 'image_count': 73}, {'id': 741, 'image_count': 49}, {'id': 742, 'image_count': 23}, {'id': 743, 'image_count': 4}, {'id': 744, 'image_count': 62}, {'id': 745, 'image_count': 118}, {'id': 746, 'image_count': 99}, {'id': 747, 'image_count': 40}, {'id': 748, 'image_count': 1036}, {'id': 749, 'image_count': 105}, {'id': 750, 'image_count': 21}, {'id': 751, 'image_count': 229}, {'id': 752, 'image_count': 7}, {'id': 753, 'image_count': 72}, {'id': 754, 'image_count': 9}, {'id': 755, 'image_count': 10}, {'id': 756, 'image_count': 328}, {'id': 757, 'image_count': 468}, {'id': 758, 'image_count': 1}, {'id': 759, 'image_count': 2}, {'id': 760, 'image_count': 24}, {'id': 761, 'image_count': 11}, {'id': 762, 'image_count': 72}, {'id': 763, 'image_count': 17}, {'id': 764, 'image_count': 10}, {'id': 765, 'image_count': 17}, {'id': 766, 'image_count': 489}, {'id': 767, 'image_count': 47}, {'id': 768, 'image_count': 93}, {'id': 769, 'image_count': 1}, {'id': 770, 'image_count': 12}, {'id': 771, 'image_count': 228}, {'id': 772, 'image_count': 5}, {'id': 773, 'image_count': 76}, {'id': 774, 'image_count': 71}, {'id': 775, 'image_count': 30}, {'id': 776, 'image_count': 109}, {'id': 777, 'image_count': 14}, {'id': 778, 'image_count': 1}, {'id': 779, 'image_count': 8}, {'id': 780, 'image_count': 26}, {'id': 781, 'image_count': 339}, {'id': 782, 'image_count': 153}, {'id': 783, 'image_count': 2}, {'id': 784, 'image_count': 3}, {'id': 785, 'image_count': 8}, {'id': 786, 'image_count': 47}, {'id': 787, 'image_count': 8}, {'id': 788, 'image_count': 6}, {'id': 789, 'image_count': 116}, {'id': 790, 'image_count': 69}, {'id': 791, 'image_count': 13}, {'id': 792, 'image_count': 6}, {'id': 793, 'image_count': 1928}, {'id': 794, 'image_count': 79}, {'id': 795, 'image_count': 14}, {'id': 796, 'image_count': 7}, {'id': 797, 'image_count': 20}, {'id': 798, 'image_count': 114}, {'id': 799, 'image_count': 221}, {'id': 800, 'image_count': 502}, {'id': 801, 'image_count': 62}, {'id': 802, 'image_count': 87}, {'id': 803, 'image_count': 4}, {'id': 804, 'image_count': 1912}, {'id': 805, 'image_count': 7}, {'id': 806, 'image_count': 186}, {'id': 807, 'image_count': 18}, {'id': 808, 'image_count': 4}, {'id': 809, 'image_count': 3}, {'id': 810, 'image_count': 7}, {'id': 811, 'image_count': 1413}, {'id': 812, 'image_count': 7}, {'id': 813, 'image_count': 12}, {'id': 814, 'image_count': 248}, {'id': 815, 'image_count': 4}, {'id': 816, 'image_count': 1881}, {'id': 817, 'image_count': 529}, {'id': 818, 'image_count': 1932}, {'id': 819, 'image_count': 50}, {'id': 820, 'image_count': 3}, {'id': 821, 'image_count': 28}, {'id': 822, 'image_count': 10}, {'id': 823, 'image_count': 5}, {'id': 824, 'image_count': 5}, {'id': 825, 'image_count': 18}, {'id': 826, 'image_count': 14}, {'id': 827, 'image_count': 1890}, {'id': 828, 'image_count': 660}, {'id': 829, 'image_count': 8}, {'id': 830, 'image_count': 25}, {'id': 831, 'image_count': 10}, {'id': 832, 'image_count': 218}, {'id': 833, 'image_count': 36}, {'id': 834, 'image_count': 16}, {'id': 835, 'image_count': 808}, {'id': 836, 'image_count': 479}, {'id': 837, 'image_count': 1404}, {'id': 838, 'image_count': 307}, {'id': 839, 'image_count': 57}, {'id': 840, 'image_count': 28}, {'id': 841, 'image_count': 80}, {'id': 842, 'image_count': 11}, {'id': 843, 'image_count': 92}, {'id': 844, 'image_count': 20}, {'id': 845, 'image_count': 194}, {'id': 846, 'image_count': 23}, {'id': 847, 'image_count': 52}, {'id': 848, 'image_count': 673}, {'id': 849, 'image_count': 2}, {'id': 850, 'image_count': 2}, {'id': 851, 'image_count': 1}, {'id': 852, 'image_count': 2}, {'id': 853, 'image_count': 8}, {'id': 854, 'image_count': 80}, {'id': 855, 'image_count': 3}, {'id': 856, 'image_count': 3}, {'id': 857, 'image_count': 15}, {'id': 858, 'image_count': 2}, {'id': 859, 'image_count': 10}, {'id': 860, 'image_count': 386}, {'id': 861, 'image_count': 65}, {'id': 862, 'image_count': 3}, {'id': 863, 'image_count': 35}, {'id': 864, 'image_count': 5}, {'id': 865, 'image_count': 180}, {'id': 866, 'image_count': 99}, {'id': 867, 'image_count': 49}, {'id': 868, 'image_count': 28}, {'id': 869, 'image_count': 1}, {'id': 870, 'image_count': 52}, {'id': 871, 'image_count': 36}, {'id': 872, 'image_count': 70}, {'id': 873, 'image_count': 6}, {'id': 874, 'image_count': 29}, {'id': 875, 'image_count': 24}, {'id': 876, 'image_count': 1115}, {'id': 877, 'image_count': 61}, {'id': 878, 'image_count': 18}, {'id': 879, 'image_count': 18}, {'id': 880, 'image_count': 665}, {'id': 881, 'image_count': 1096}, {'id': 882, 'image_count': 29}, {'id': 883, 'image_count': 8}, {'id': 884, 'image_count': 14}, {'id': 885, 'image_count': 1622}, {'id': 886, 'image_count': 2}, {'id': 887, 'image_count': 3}, {'id': 888, 'image_count': 32}, {'id': 889, 'image_count': 55}, {'id': 890, 'image_count': 1}, {'id': 891, 'image_count': 10}, {'id': 892, 'image_count': 10}, {'id': 893, 'image_count': 47}, {'id': 894, 'image_count': 3}, {'id': 895, 'image_count': 29}, {'id': 896, 'image_count': 342}, {'id': 897, 'image_count': 25}, {'id': 898, 'image_count': 1469}, {'id': 899, 'image_count': 521}, {'id': 900, 'image_count': 347}, {'id': 901, 'image_count': 35}, {'id': 902, 'image_count': 7}, {'id': 903, 'image_count': 207}, {'id': 904, 'image_count': 108}, {'id': 905, 'image_count': 2}, {'id': 906, 'image_count': 34}, {'id': 907, 'image_count': 12}, {'id': 908, 'image_count': 10}, {'id': 909, 'image_count': 13}, {'id': 910, 'image_count': 361}, {'id': 911, 'image_count': 1023}, {'id': 912, 'image_count': 782}, {'id': 913, 'image_count': 2}, {'id': 914, 'image_count': 5}, {'id': 915, 'image_count': 247}, {'id': 916, 'image_count': 221}, {'id': 917, 'image_count': 4}, {'id': 918, 'image_count': 8}, {'id': 919, 'image_count': 158}, {'id': 920, 'image_count': 3}, {'id': 921, 'image_count': 752}, {'id': 922, 'image_count': 64}, {'id': 923, 'image_count': 707}, {'id': 924, 'image_count': 143}, {'id': 925, 'image_count': 1}, {'id': 926, 'image_count': 49}, {'id': 927, 'image_count': 126}, {'id': 928, 'image_count': 76}, {'id': 929, 'image_count': 11}, {'id': 930, 'image_count': 11}, {'id': 931, 'image_count': 4}, {'id': 932, 'image_count': 39}, {'id': 933, 'image_count': 11}, {'id': 934, 'image_count': 13}, {'id': 935, 'image_count': 91}, {'id': 936, 'image_count': 14}, {'id': 937, 'image_count': 5}, {'id': 938, 'image_count': 3}, {'id': 939, 'image_count': 10}, {'id': 940, 'image_count': 18}, {'id': 941, 'image_count': 9}, {'id': 942, 'image_count': 6}, {'id': 943, 'image_count': 951}, {'id': 944, 'image_count': 2}, {'id': 945, 'image_count': 1}, {'id': 946, 'image_count': 19}, {'id': 947, 'image_count': 1942}, {'id': 948, 'image_count': 1916}, {'id': 949, 'image_count': 139}, {'id': 950, 'image_count': 43}, {'id': 951, 'image_count': 1969}, {'id': 952, 'image_count': 5}, {'id': 953, 'image_count': 134}, {'id': 954, 'image_count': 74}, {'id': 955, 'image_count': 381}, {'id': 956, 'image_count': 1}, {'id': 957, 'image_count': 381}, {'id': 958, 'image_count': 6}, {'id': 959, 'image_count': 1826}, {'id': 960, 'image_count': 28}, {'id': 961, 'image_count': 1635}, {'id': 962, 'image_count': 1967}, {'id': 963, 'image_count': 16}, {'id': 964, 'image_count': 1926}, {'id': 965, 'image_count': 1789}, {'id': 966, 'image_count': 401}, {'id': 967, 'image_count': 1968}, {'id': 968, 'image_count': 1167}, {'id': 969, 'image_count': 1}, {'id': 970, 'image_count': 56}, {'id': 971, 'image_count': 17}, {'id': 972, 'image_count': 1}, {'id': 973, 'image_count': 58}, {'id': 974, 'image_count': 9}, {'id': 975, 'image_count': 8}, {'id': 976, 'image_count': 1124}, {'id': 977, 'image_count': 31}, {'id': 978, 'image_count': 16}, {'id': 979, 'image_count': 491}, {'id': 980, 'image_count': 432}, {'id': 981, 'image_count': 1945}, {'id': 982, 'image_count': 1899}, {'id': 983, 'image_count': 5}, {'id': 984, 'image_count': 28}, {'id': 985, 'image_count': 7}, {'id': 986, 'image_count': 146}, {'id': 987, 'image_count': 1}, {'id': 988, 'image_count': 25}, {'id': 989, 'image_count': 22}, {'id': 990, 'image_count': 1}, {'id': 991, 'image_count': 10}, {'id': 992, 'image_count': 9}, {'id': 993, 'image_count': 308}, {'id': 994, 'image_count': 4}, {'id': 995, 'image_count': 1969}, {'id': 996, 'image_count': 45}, {'id': 997, 'image_count': 12}, {'id': 998, 'image_count': 1}, {'id': 999, 'image_count': 85}, {'id': 1000, 'image_count': 1127}, {'id': 1001, 'image_count': 11}, {'id': 1002, 'image_count': 60}, {'id': 1003, 'image_count': 1}, {'id': 1004, 'image_count': 16}, {'id': 1005, 'image_count': 1}, {'id': 1006, 'image_count': 65}, {'id': 1007, 'image_count': 13}, {'id': 1008, 'image_count': 655}, {'id': 1009, 'image_count': 51}, {'id': 1010, 'image_count': 1}, {'id': 1011, 'image_count': 673}, {'id': 1012, 'image_count': 5}, {'id': 1013, 'image_count': 36}, {'id': 1014, 'image_count': 54}, {'id': 1015, 'image_count': 5}, {'id': 1016, 'image_count': 8}, {'id': 1017, 'image_count': 305}, {'id': 1018, 'image_count': 297}, {'id': 1019, 'image_count': 1053}, {'id': 1020, 'image_count': 223}, {'id': 1021, 'image_count': 1037}, {'id': 1022, 'image_count': 63}, {'id': 1023, 'image_count': 1881}, {'id': 1024, 'image_count': 507}, {'id': 1025, 'image_count': 333}, {'id': 1026, 'image_count': 1911}, {'id': 1027, 'image_count': 1765}, {'id': 1028, 'image_count': 1}, {'id': 1029, 'image_count': 5}, {'id': 1030, 'image_count': 1}, {'id': 1031, 'image_count': 9}, {'id': 1032, 'image_count': 2}, {'id': 1033, 'image_count': 151}, {'id': 1034, 'image_count': 82}, {'id': 1035, 'image_count': 1931}, {'id': 1036, 'image_count': 41}, {'id': 1037, 'image_count': 1895}, {'id': 1038, 'image_count': 24}, {'id': 1039, 'image_count': 22}, {'id': 1040, 'image_count': 35}, {'id': 1041, 'image_count': 69}, {'id': 1042, 'image_count': 962}, {'id': 1043, 'image_count': 588}, {'id': 1044, 'image_count': 21}, {'id': 1045, 'image_count': 825}, {'id': 1046, 'image_count': 52}, {'id': 1047, 'image_count': 5}, {'id': 1048, 'image_count': 5}, {'id': 1049, 'image_count': 5}, {'id': 1050, 'image_count': 1860}, {'id': 1051, 'image_count': 56}, {'id': 1052, 'image_count': 1582}, {'id': 1053, 'image_count': 7}, {'id': 1054, 'image_count': 2}, {'id': 1055, 'image_count': 1562}, {'id': 1056, 'image_count': 1885}, {'id': 1057, 'image_count': 1}, {'id': 1058, 'image_count': 5}, {'id': 1059, 'image_count': 137}, {'id': 1060, 'image_count': 1094}, {'id': 1061, 'image_count': 134}, {'id': 1062, 'image_count': 29}, {'id': 1063, 'image_count': 22}, {'id': 1064, 'image_count': 522}, {'id': 1065, 'image_count': 50}, {'id': 1066, 'image_count': 68}, {'id': 1067, 'image_count': 16}, {'id': 1068, 'image_count': 40}, {'id': 1069, 'image_count': 35}, {'id': 1070, 'image_count': 135}, {'id': 1071, 'image_count': 1413}, {'id': 1072, 'image_count': 772}, {'id': 1073, 'image_count': 50}, {'id': 1074, 'image_count': 1015}, {'id': 1075, 'image_count': 1}, {'id': 1076, 'image_count': 65}, {'id': 1077, 'image_count': 1900}, {'id': 1078, 'image_count': 1302}, {'id': 1079, 'image_count': 1977}, {'id': 1080, 'image_count': 2}, {'id': 1081, 'image_count': 29}, {'id': 1082, 'image_count': 36}, {'id': 1083, 'image_count': 138}, {'id': 1084, 'image_count': 4}, {'id': 1085, 'image_count': 67}, {'id': 1086, 'image_count': 26}, {'id': 1087, 'image_count': 25}, {'id': 1088, 'image_count': 33}, {'id': 1089, 'image_count': 37}, {'id': 1090, 'image_count': 50}, {'id': 1091, 'image_count': 270}, {'id': 1092, 'image_count': 12}, {'id': 1093, 'image_count': 316}, {'id': 1094, 'image_count': 41}, {'id': 1095, 'image_count': 224}, {'id': 1096, 'image_count': 105}, {'id': 1097, 'image_count': 1925}, {'id': 1098, 'image_count': 1021}, {'id': 1099, 'image_count': 1213}, {'id': 1100, 'image_count': 172}, {'id': 1101, 'image_count': 28}, {'id': 1102, 'image_count': 745}, {'id': 1103, 'image_count': 187}, {'id': 1104, 'image_count': 147}, {'id': 1105, 'image_count': 136}, {'id': 1106, 'image_count': 34}, {'id': 1107, 'image_count': 41}, {'id': 1108, 'image_count': 636}, {'id': 1109, 'image_count': 570}, {'id': 1110, 'image_count': 1149}, {'id': 1111, 'image_count': 61}, {'id': 1112, 'image_count': 1890}, {'id': 1113, 'image_count': 18}, {'id': 1114, 'image_count': 143}, {'id': 1115, 'image_count': 1517}, {'id': 1116, 'image_count': 7}, {'id': 1117, 'image_count': 943}, {'id': 1118, 'image_count': 6}, {'id': 1119, 'image_count': 1}, {'id': 1120, 'image_count': 11}, {'id': 1121, 'image_count': 101}, {'id': 1122, 'image_count': 1909}, {'id': 1123, 'image_count': 800}, {'id': 1124, 'image_count': 1}, {'id': 1125, 'image_count': 44}, {'id': 1126, 'image_count': 3}, {'id': 1127, 'image_count': 44}, {'id': 1128, 'image_count': 31}, {'id': 1129, 'image_count': 7}, {'id': 1130, 'image_count': 20}, {'id': 1131, 'image_count': 11}, {'id': 1132, 'image_count': 13}, {'id': 1133, 'image_count': 1924}, {'id': 1134, 'image_count': 113}, {'id': 1135, 'image_count': 2}, {'id': 1136, 'image_count': 139}, {'id': 1137, 'image_count': 12}, {'id': 1138, 'image_count': 37}, {'id': 1139, 'image_count': 1866}, {'id': 1140, 'image_count': 47}, {'id': 1141, 'image_count': 1468}, {'id': 1142, 'image_count': 729}, {'id': 1143, 'image_count': 24}, {'id': 1144, 'image_count': 1}, {'id': 1145, 'image_count': 10}, {'id': 1146, 'image_count': 3}, {'id': 1147, 'image_count': 14}, {'id': 1148, 'image_count': 4}, {'id': 1149, 'image_count': 29}, {'id': 1150, 'image_count': 4}, {'id': 1151, 'image_count': 70}, {'id': 1152, 'image_count': 46}, {'id': 1153, 'image_count': 14}, {'id': 1154, 'image_count': 48}, {'id': 1155, 'image_count': 1855}, {'id': 1156, 'image_count': 113}, {'id': 1157, 'image_count': 1}, {'id': 1158, 'image_count': 1}, {'id': 1159, 'image_count': 10}, {'id': 1160, 'image_count': 54}, {'id': 1161, 'image_count': 1923}, {'id': 1162, 'image_count': 630}, {'id': 1163, 'image_count': 31}, {'id': 1164, 'image_count': 69}, {'id': 1165, 'image_count': 7}, {'id': 1166, 'image_count': 11}, {'id': 1167, 'image_count': 1}, {'id': 1168, 'image_count': 30}, {'id': 1169, 'image_count': 50}, {'id': 1170, 'image_count': 45}, {'id': 1171, 'image_count': 28}, {'id': 1172, 'image_count': 114}, {'id': 1173, 'image_count': 193}, {'id': 1174, 'image_count': 21}, {'id': 1175, 'image_count': 91}, {'id': 1176, 'image_count': 31}, {'id': 1177, 'image_count': 1469}, {'id': 1178, 'image_count': 1924}, {'id': 1179, 'image_count': 87}, {'id': 1180, 'image_count': 77}, {'id': 1181, 'image_count': 11}, {'id': 1182, 'image_count': 47}, {'id': 1183, 'image_count': 21}, {'id': 1184, 'image_count': 47}, {'id': 1185, 'image_count': 70}, {'id': 1186, 'image_count': 1838}, {'id': 1187, 'image_count': 19}, {'id': 1188, 'image_count': 531}, {'id': 1189, 'image_count': 11}, {'id': 1190, 'image_count': 941}, {'id': 1191, 'image_count': 113}, {'id': 1192, 'image_count': 26}, {'id': 1193, 'image_count': 5}, {'id': 1194, 'image_count': 56}, {'id': 1195, 'image_count': 73}, {'id': 1196, 'image_count': 32}, {'id': 1197, 'image_count': 128}, {'id': 1198, 'image_count': 623}, {'id': 1199, 'image_count': 12}, {'id': 1200, 'image_count': 52}, {'id': 1201, 'image_count': 11}, {'id': 1202, 'image_count': 1674}, {'id': 1203, 'image_count': 81}] # noqa +# fmt: on diff --git a/annotator/oneformer/detectron2/data/datasets/pascal_voc.py b/annotator/oneformer/detectron2/data/datasets/pascal_voc.py new file mode 100644 index 0000000000000000000000000000000000000000..919cc4920394d3cb87ad5232adcbedc250e4db26 --- /dev/null +++ b/annotator/oneformer/detectron2/data/datasets/pascal_voc.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +import numpy as np +import os +import xml.etree.ElementTree as ET +from typing import List, Tuple, Union + +from annotator.oneformer.detectron2.data import DatasetCatalog, MetadataCatalog +from annotator.oneformer.detectron2.structures import BoxMode +from annotator.oneformer.detectron2.utils.file_io import PathManager + +__all__ = ["load_voc_instances", "register_pascal_voc"] + + +# fmt: off +CLASS_NAMES = ( + "aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", + "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person", + "pottedplant", "sheep", "sofa", "train", "tvmonitor" +) +# fmt: on + + +def load_voc_instances(dirname: str, split: str, class_names: Union[List[str], Tuple[str, ...]]): + """ + Load Pascal VOC detection annotations to Detectron2 format. + + Args: + dirname: Contain "Annotations", "ImageSets", "JPEGImages" + split (str): one of "train", "test", "val", "trainval" + class_names: list or tuple of class names + """ + with PathManager.open(os.path.join(dirname, "ImageSets", "Main", split + ".txt")) as f: + fileids = np.loadtxt(f, dtype=np.str) + + # Needs to read many small annotation files. Makes sense at local + annotation_dirname = PathManager.get_local_path(os.path.join(dirname, "Annotations/")) + dicts = [] + for fileid in fileids: + anno_file = os.path.join(annotation_dirname, fileid + ".xml") + jpeg_file = os.path.join(dirname, "JPEGImages", fileid + ".jpg") + + with PathManager.open(anno_file) as f: + tree = ET.parse(f) + + r = { + "file_name": jpeg_file, + "image_id": fileid, + "height": int(tree.findall("./size/height")[0].text), + "width": int(tree.findall("./size/width")[0].text), + } + instances = [] + + for obj in tree.findall("object"): + cls = obj.find("name").text + # We include "difficult" samples in training. + # Based on limited experiments, they don't hurt accuracy. + # difficult = int(obj.find("difficult").text) + # if difficult == 1: + # continue + bbox = obj.find("bndbox") + bbox = [float(bbox.find(x).text) for x in ["xmin", "ymin", "xmax", "ymax"]] + # Original annotations are integers in the range [1, W or H] + # Assuming they mean 1-based pixel indices (inclusive), + # a box with annotation (xmin=1, xmax=W) covers the whole image. + # In coordinate space this is represented by (xmin=0, xmax=W) + bbox[0] -= 1.0 + bbox[1] -= 1.0 + instances.append( + {"category_id": class_names.index(cls), "bbox": bbox, "bbox_mode": BoxMode.XYXY_ABS} + ) + r["annotations"] = instances + dicts.append(r) + return dicts + + +def register_pascal_voc(name, dirname, split, year, class_names=CLASS_NAMES): + DatasetCatalog.register(name, lambda: load_voc_instances(dirname, split, class_names)) + MetadataCatalog.get(name).set( + thing_classes=list(class_names), dirname=dirname, year=year, split=split + ) diff --git a/annotator/oneformer/detectron2/data/datasets/register_coco.py b/annotator/oneformer/detectron2/data/datasets/register_coco.py new file mode 100644 index 0000000000000000000000000000000000000000..e564438d5bf016bcdbb65b4bbdc215d79f579f8a --- /dev/null +++ b/annotator/oneformer/detectron2/data/datasets/register_coco.py @@ -0,0 +1,3 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .coco import register_coco_instances # noqa +from .coco_panoptic import register_coco_panoptic_separated # noqa diff --git a/annotator/oneformer/detectron2/data/detection_utils.py b/annotator/oneformer/detectron2/data/detection_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..99ce45f52bab8ff87dba3e9e008947eef2f7c33e --- /dev/null +++ b/annotator/oneformer/detectron2/data/detection_utils.py @@ -0,0 +1,659 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +""" +Common data processing utilities that are used in a +typical object detection data pipeline. +""" +import logging +import numpy as np +from typing import List, Union +import pycocotools.mask as mask_util +import torch +from PIL import Image + +from annotator.oneformer.detectron2.structures import ( + BitMasks, + Boxes, + BoxMode, + Instances, + Keypoints, + PolygonMasks, + RotatedBoxes, + polygons_to_bitmask, +) +from annotator.oneformer.detectron2.utils.file_io import PathManager + +from . import transforms as T +from .catalog import MetadataCatalog + +__all__ = [ + "SizeMismatchError", + "convert_image_to_rgb", + "check_image_size", + "transform_proposals", + "transform_instance_annotations", + "annotations_to_instances", + "annotations_to_instances_rotated", + "build_augmentation", + "build_transform_gen", + "create_keypoint_hflip_indices", + "filter_empty_instances", + "read_image", +] + + +class SizeMismatchError(ValueError): + """ + When loaded image has difference width/height compared with annotation. + """ + + +# https://en.wikipedia.org/wiki/YUV#SDTV_with_BT.601 +_M_RGB2YUV = [[0.299, 0.587, 0.114], [-0.14713, -0.28886, 0.436], [0.615, -0.51499, -0.10001]] +_M_YUV2RGB = [[1.0, 0.0, 1.13983], [1.0, -0.39465, -0.58060], [1.0, 2.03211, 0.0]] + +# https://www.exiv2.org/tags.html +_EXIF_ORIENT = 274 # exif 'Orientation' tag + + +def convert_PIL_to_numpy(image, format): + """ + Convert PIL image to numpy array of target format. + + Args: + image (PIL.Image): a PIL image + format (str): the format of output image + + Returns: + (np.ndarray): also see `read_image` + """ + if format is not None: + # PIL only supports RGB, so convert to RGB and flip channels over below + conversion_format = format + if format in ["BGR", "YUV-BT.601"]: + conversion_format = "RGB" + image = image.convert(conversion_format) + image = np.asarray(image) + # PIL squeezes out the channel dimension for "L", so make it HWC + if format == "L": + image = np.expand_dims(image, -1) + + # handle formats not supported by PIL + elif format == "BGR": + # flip channels if needed + image = image[:, :, ::-1] + elif format == "YUV-BT.601": + image = image / 255.0 + image = np.dot(image, np.array(_M_RGB2YUV).T) + + return image + + +def convert_image_to_rgb(image, format): + """ + Convert an image from given format to RGB. + + Args: + image (np.ndarray or Tensor): an HWC image + format (str): the format of input image, also see `read_image` + + Returns: + (np.ndarray): (H,W,3) RGB image in 0-255 range, can be either float or uint8 + """ + if isinstance(image, torch.Tensor): + image = image.cpu().numpy() + if format == "BGR": + image = image[:, :, [2, 1, 0]] + elif format == "YUV-BT.601": + image = np.dot(image, np.array(_M_YUV2RGB).T) + image = image * 255.0 + else: + if format == "L": + image = image[:, :, 0] + image = image.astype(np.uint8) + image = np.asarray(Image.fromarray(image, mode=format).convert("RGB")) + return image + + +def _apply_exif_orientation(image): + """ + Applies the exif orientation correctly. + + This code exists per the bug: + https://github.com/python-pillow/Pillow/issues/3973 + with the function `ImageOps.exif_transpose`. The Pillow source raises errors with + various methods, especially `tobytes` + + Function based on: + https://github.com/wkentaro/labelme/blob/v4.5.4/labelme/utils/image.py#L59 + https://github.com/python-pillow/Pillow/blob/7.1.2/src/PIL/ImageOps.py#L527 + + Args: + image (PIL.Image): a PIL image + + Returns: + (PIL.Image): the PIL image with exif orientation applied, if applicable + """ + if not hasattr(image, "getexif"): + return image + + try: + exif = image.getexif() + except Exception: # https://github.com/facebookresearch/detectron2/issues/1885 + exif = None + + if exif is None: + return image + + orientation = exif.get(_EXIF_ORIENT) + + method = { + 2: Image.FLIP_LEFT_RIGHT, + 3: Image.ROTATE_180, + 4: Image.FLIP_TOP_BOTTOM, + 5: Image.TRANSPOSE, + 6: Image.ROTATE_270, + 7: Image.TRANSVERSE, + 8: Image.ROTATE_90, + }.get(orientation) + + if method is not None: + return image.transpose(method) + return image + + +def read_image(file_name, format=None): + """ + Read an image into the given format. + Will apply rotation and flipping if the image has such exif information. + + Args: + file_name (str): image file path + format (str): one of the supported image modes in PIL, or "BGR" or "YUV-BT.601". + + Returns: + image (np.ndarray): + an HWC image in the given format, which is 0-255, uint8 for + supported image modes in PIL or "BGR"; float (0-1 for Y) for YUV-BT.601. + """ + with PathManager.open(file_name, "rb") as f: + image = Image.open(f) + + # work around this bug: https://github.com/python-pillow/Pillow/issues/3973 + image = _apply_exif_orientation(image) + return convert_PIL_to_numpy(image, format) + + +def check_image_size(dataset_dict, image): + """ + Raise an error if the image does not match the size specified in the dict. + """ + if "width" in dataset_dict or "height" in dataset_dict: + image_wh = (image.shape[1], image.shape[0]) + expected_wh = (dataset_dict["width"], dataset_dict["height"]) + if not image_wh == expected_wh: + raise SizeMismatchError( + "Mismatched image shape{}, got {}, expect {}.".format( + " for image " + dataset_dict["file_name"] + if "file_name" in dataset_dict + else "", + image_wh, + expected_wh, + ) + + " Please check the width/height in your annotation." + ) + + # To ensure bbox always remap to original image size + if "width" not in dataset_dict: + dataset_dict["width"] = image.shape[1] + if "height" not in dataset_dict: + dataset_dict["height"] = image.shape[0] + + +def transform_proposals(dataset_dict, image_shape, transforms, *, proposal_topk, min_box_size=0): + """ + Apply transformations to the proposals in dataset_dict, if any. + + Args: + dataset_dict (dict): a dict read from the dataset, possibly + contains fields "proposal_boxes", "proposal_objectness_logits", "proposal_bbox_mode" + image_shape (tuple): height, width + transforms (TransformList): + proposal_topk (int): only keep top-K scoring proposals + min_box_size (int): proposals with either side smaller than this + threshold are removed + + The input dict is modified in-place, with abovementioned keys removed. A new + key "proposals" will be added. Its value is an `Instances` + object which contains the transformed proposals in its field + "proposal_boxes" and "objectness_logits". + """ + if "proposal_boxes" in dataset_dict: + # Transform proposal boxes + boxes = transforms.apply_box( + BoxMode.convert( + dataset_dict.pop("proposal_boxes"), + dataset_dict.pop("proposal_bbox_mode"), + BoxMode.XYXY_ABS, + ) + ) + boxes = Boxes(boxes) + objectness_logits = torch.as_tensor( + dataset_dict.pop("proposal_objectness_logits").astype("float32") + ) + + boxes.clip(image_shape) + keep = boxes.nonempty(threshold=min_box_size) + boxes = boxes[keep] + objectness_logits = objectness_logits[keep] + + proposals = Instances(image_shape) + proposals.proposal_boxes = boxes[:proposal_topk] + proposals.objectness_logits = objectness_logits[:proposal_topk] + dataset_dict["proposals"] = proposals + + +def get_bbox(annotation): + """ + Get bbox from data + Args: + annotation (dict): dict of instance annotations for a single instance. + Returns: + bbox (ndarray): x1, y1, x2, y2 coordinates + """ + # bbox is 1d (per-instance bounding box) + bbox = BoxMode.convert(annotation["bbox"], annotation["bbox_mode"], BoxMode.XYXY_ABS) + return bbox + + +def transform_instance_annotations( + annotation, transforms, image_size, *, keypoint_hflip_indices=None +): + """ + Apply transforms to box, segmentation and keypoints annotations of a single instance. + + It will use `transforms.apply_box` for the box, and + `transforms.apply_coords` for segmentation polygons & keypoints. + If you need anything more specially designed for each data structure, + you'll need to implement your own version of this function or the transforms. + + Args: + annotation (dict): dict of instance annotations for a single instance. + It will be modified in-place. + transforms (TransformList or list[Transform]): + image_size (tuple): the height, width of the transformed image + keypoint_hflip_indices (ndarray[int]): see `create_keypoint_hflip_indices`. + + Returns: + dict: + the same input dict with fields "bbox", "segmentation", "keypoints" + transformed according to `transforms`. + The "bbox_mode" field will be set to XYXY_ABS. + """ + if isinstance(transforms, (tuple, list)): + transforms = T.TransformList(transforms) + # bbox is 1d (per-instance bounding box) + bbox = BoxMode.convert(annotation["bbox"], annotation["bbox_mode"], BoxMode.XYXY_ABS) + # clip transformed bbox to image size + bbox = transforms.apply_box(np.array([bbox]))[0].clip(min=0) + annotation["bbox"] = np.minimum(bbox, list(image_size + image_size)[::-1]) + annotation["bbox_mode"] = BoxMode.XYXY_ABS + + if "segmentation" in annotation: + # each instance contains 1 or more polygons + segm = annotation["segmentation"] + if isinstance(segm, list): + # polygons + polygons = [np.asarray(p).reshape(-1, 2) for p in segm] + annotation["segmentation"] = [ + p.reshape(-1) for p in transforms.apply_polygons(polygons) + ] + elif isinstance(segm, dict): + # RLE + mask = mask_util.decode(segm) + mask = transforms.apply_segmentation(mask) + assert tuple(mask.shape[:2]) == image_size + annotation["segmentation"] = mask + else: + raise ValueError( + "Cannot transform segmentation of type '{}'!" + "Supported types are: polygons as list[list[float] or ndarray]," + " COCO-style RLE as a dict.".format(type(segm)) + ) + + if "keypoints" in annotation: + keypoints = transform_keypoint_annotations( + annotation["keypoints"], transforms, image_size, keypoint_hflip_indices + ) + annotation["keypoints"] = keypoints + + return annotation + + +def transform_keypoint_annotations(keypoints, transforms, image_size, keypoint_hflip_indices=None): + """ + Transform keypoint annotations of an image. + If a keypoint is transformed out of image boundary, it will be marked "unlabeled" (visibility=0) + + Args: + keypoints (list[float]): Nx3 float in Detectron2's Dataset format. + Each point is represented by (x, y, visibility). + transforms (TransformList): + image_size (tuple): the height, width of the transformed image + keypoint_hflip_indices (ndarray[int]): see `create_keypoint_hflip_indices`. + When `transforms` includes horizontal flip, will use the index + mapping to flip keypoints. + """ + # (N*3,) -> (N, 3) + keypoints = np.asarray(keypoints, dtype="float64").reshape(-1, 3) + keypoints_xy = transforms.apply_coords(keypoints[:, :2]) + + # Set all out-of-boundary points to "unlabeled" + inside = (keypoints_xy >= np.array([0, 0])) & (keypoints_xy <= np.array(image_size[::-1])) + inside = inside.all(axis=1) + keypoints[:, :2] = keypoints_xy + keypoints[:, 2][~inside] = 0 + + # This assumes that HorizFlipTransform is the only one that does flip + do_hflip = sum(isinstance(t, T.HFlipTransform) for t in transforms.transforms) % 2 == 1 + + # Alternative way: check if probe points was horizontally flipped. + # probe = np.asarray([[0.0, 0.0], [image_width, 0.0]]) + # probe_aug = transforms.apply_coords(probe.copy()) + # do_hflip = np.sign(probe[1][0] - probe[0][0]) != np.sign(probe_aug[1][0] - probe_aug[0][0]) # noqa + + # If flipped, swap each keypoint with its opposite-handed equivalent + if do_hflip: + if keypoint_hflip_indices is None: + raise ValueError("Cannot flip keypoints without providing flip indices!") + if len(keypoints) != len(keypoint_hflip_indices): + raise ValueError( + "Keypoint data has {} points, but metadata " + "contains {} points!".format(len(keypoints), len(keypoint_hflip_indices)) + ) + keypoints = keypoints[np.asarray(keypoint_hflip_indices, dtype=np.int32), :] + + # Maintain COCO convention that if visibility == 0 (unlabeled), then x, y = 0 + keypoints[keypoints[:, 2] == 0] = 0 + return keypoints + + +def annotations_to_instances(annos, image_size, mask_format="polygon"): + """ + Create an :class:`Instances` object used by the models, + from instance annotations in the dataset dict. + + Args: + annos (list[dict]): a list of instance annotations in one image, each + element for one instance. + image_size (tuple): height, width + + Returns: + Instances: + It will contain fields "gt_boxes", "gt_classes", + "gt_masks", "gt_keypoints", if they can be obtained from `annos`. + This is the format that builtin models expect. + """ + boxes = ( + np.stack( + [BoxMode.convert(obj["bbox"], obj["bbox_mode"], BoxMode.XYXY_ABS) for obj in annos] + ) + if len(annos) + else np.zeros((0, 4)) + ) + target = Instances(image_size) + target.gt_boxes = Boxes(boxes) + + classes = [int(obj["category_id"]) for obj in annos] + classes = torch.tensor(classes, dtype=torch.int64) + target.gt_classes = classes + + if len(annos) and "segmentation" in annos[0]: + segms = [obj["segmentation"] for obj in annos] + if mask_format == "polygon": + try: + masks = PolygonMasks(segms) + except ValueError as e: + raise ValueError( + "Failed to use mask_format=='polygon' from the given annotations!" + ) from e + else: + assert mask_format == "bitmask", mask_format + masks = [] + for segm in segms: + if isinstance(segm, list): + # polygon + masks.append(polygons_to_bitmask(segm, *image_size)) + elif isinstance(segm, dict): + # COCO RLE + masks.append(mask_util.decode(segm)) + elif isinstance(segm, np.ndarray): + assert segm.ndim == 2, "Expect segmentation of 2 dimensions, got {}.".format( + segm.ndim + ) + # mask array + masks.append(segm) + else: + raise ValueError( + "Cannot convert segmentation of type '{}' to BitMasks!" + "Supported types are: polygons as list[list[float] or ndarray]," + " COCO-style RLE as a dict, or a binary segmentation mask " + " in a 2D numpy array of shape HxW.".format(type(segm)) + ) + # torch.from_numpy does not support array with negative stride. + masks = BitMasks( + torch.stack([torch.from_numpy(np.ascontiguousarray(x)) for x in masks]) + ) + target.gt_masks = masks + + if len(annos) and "keypoints" in annos[0]: + kpts = [obj.get("keypoints", []) for obj in annos] + target.gt_keypoints = Keypoints(kpts) + + return target + + +def annotations_to_instances_rotated(annos, image_size): + """ + Create an :class:`Instances` object used by the models, + from instance annotations in the dataset dict. + Compared to `annotations_to_instances`, this function is for rotated boxes only + + Args: + annos (list[dict]): a list of instance annotations in one image, each + element for one instance. + image_size (tuple): height, width + + Returns: + Instances: + Containing fields "gt_boxes", "gt_classes", + if they can be obtained from `annos`. + This is the format that builtin models expect. + """ + boxes = [obj["bbox"] for obj in annos] + target = Instances(image_size) + boxes = target.gt_boxes = RotatedBoxes(boxes) + boxes.clip(image_size) + + classes = [obj["category_id"] for obj in annos] + classes = torch.tensor(classes, dtype=torch.int64) + target.gt_classes = classes + + return target + + +def filter_empty_instances( + instances, by_box=True, by_mask=True, box_threshold=1e-5, return_mask=False +): + """ + Filter out empty instances in an `Instances` object. + + Args: + instances (Instances): + by_box (bool): whether to filter out instances with empty boxes + by_mask (bool): whether to filter out instances with empty masks + box_threshold (float): minimum width and height to be considered non-empty + return_mask (bool): whether to return boolean mask of filtered instances + + Returns: + Instances: the filtered instances. + tensor[bool], optional: boolean mask of filtered instances + """ + assert by_box or by_mask + r = [] + if by_box: + r.append(instances.gt_boxes.nonempty(threshold=box_threshold)) + if instances.has("gt_masks") and by_mask: + r.append(instances.gt_masks.nonempty()) + + # TODO: can also filter visible keypoints + + if not r: + return instances + m = r[0] + for x in r[1:]: + m = m & x + if return_mask: + return instances[m], m + return instances[m] + + +def create_keypoint_hflip_indices(dataset_names: Union[str, List[str]]) -> List[int]: + """ + Args: + dataset_names: list of dataset names + + Returns: + list[int]: a list of size=#keypoints, storing the + horizontally-flipped keypoint indices. + """ + if isinstance(dataset_names, str): + dataset_names = [dataset_names] + + check_metadata_consistency("keypoint_names", dataset_names) + check_metadata_consistency("keypoint_flip_map", dataset_names) + + meta = MetadataCatalog.get(dataset_names[0]) + names = meta.keypoint_names + # TODO flip -> hflip + flip_map = dict(meta.keypoint_flip_map) + flip_map.update({v: k for k, v in flip_map.items()}) + flipped_names = [i if i not in flip_map else flip_map[i] for i in names] + flip_indices = [names.index(i) for i in flipped_names] + return flip_indices + + +def get_fed_loss_cls_weights(dataset_names: Union[str, List[str]], freq_weight_power=1.0): + """ + Get frequency weight for each class sorted by class id. + We now calcualte freqency weight using image_count to the power freq_weight_power. + + Args: + dataset_names: list of dataset names + freq_weight_power: power value + """ + if isinstance(dataset_names, str): + dataset_names = [dataset_names] + + check_metadata_consistency("class_image_count", dataset_names) + + meta = MetadataCatalog.get(dataset_names[0]) + class_freq_meta = meta.class_image_count + class_freq = torch.tensor( + [c["image_count"] for c in sorted(class_freq_meta, key=lambda x: x["id"])] + ) + class_freq_weight = class_freq.float() ** freq_weight_power + return class_freq_weight + + +def gen_crop_transform_with_instance(crop_size, image_size, instance): + """ + Generate a CropTransform so that the cropping region contains + the center of the given instance. + + Args: + crop_size (tuple): h, w in pixels + image_size (tuple): h, w + instance (dict): an annotation dict of one instance, in Detectron2's + dataset format. + """ + crop_size = np.asarray(crop_size, dtype=np.int32) + bbox = BoxMode.convert(instance["bbox"], instance["bbox_mode"], BoxMode.XYXY_ABS) + center_yx = (bbox[1] + bbox[3]) * 0.5, (bbox[0] + bbox[2]) * 0.5 + assert ( + image_size[0] >= center_yx[0] and image_size[1] >= center_yx[1] + ), "The annotation bounding box is outside of the image!" + assert ( + image_size[0] >= crop_size[0] and image_size[1] >= crop_size[1] + ), "Crop size is larger than image size!" + + min_yx = np.maximum(np.floor(center_yx).astype(np.int32) - crop_size, 0) + max_yx = np.maximum(np.asarray(image_size, dtype=np.int32) - crop_size, 0) + max_yx = np.minimum(max_yx, np.ceil(center_yx).astype(np.int32)) + + y0 = np.random.randint(min_yx[0], max_yx[0] + 1) + x0 = np.random.randint(min_yx[1], max_yx[1] + 1) + return T.CropTransform(x0, y0, crop_size[1], crop_size[0]) + + +def check_metadata_consistency(key, dataset_names): + """ + Check that the datasets have consistent metadata. + + Args: + key (str): a metadata key + dataset_names (list[str]): a list of dataset names + + Raises: + AttributeError: if the key does not exist in the metadata + ValueError: if the given datasets do not have the same metadata values defined by key + """ + if len(dataset_names) == 0: + return + logger = logging.getLogger(__name__) + entries_per_dataset = [getattr(MetadataCatalog.get(d), key) for d in dataset_names] + for idx, entry in enumerate(entries_per_dataset): + if entry != entries_per_dataset[0]: + logger.error( + "Metadata '{}' for dataset '{}' is '{}'".format(key, dataset_names[idx], str(entry)) + ) + logger.error( + "Metadata '{}' for dataset '{}' is '{}'".format( + key, dataset_names[0], str(entries_per_dataset[0]) + ) + ) + raise ValueError("Datasets have different metadata '{}'!".format(key)) + + +def build_augmentation(cfg, is_train): + """ + Create a list of default :class:`Augmentation` from config. + Now it includes resizing and flipping. + + Returns: + list[Augmentation] + """ + if is_train: + min_size = cfg.INPUT.MIN_SIZE_TRAIN + max_size = cfg.INPUT.MAX_SIZE_TRAIN + sample_style = cfg.INPUT.MIN_SIZE_TRAIN_SAMPLING + else: + min_size = cfg.INPUT.MIN_SIZE_TEST + max_size = cfg.INPUT.MAX_SIZE_TEST + sample_style = "choice" + augmentation = [T.ResizeShortestEdge(min_size, max_size, sample_style)] + if is_train and cfg.INPUT.RANDOM_FLIP != "none": + augmentation.append( + T.RandomFlip( + horizontal=cfg.INPUT.RANDOM_FLIP == "horizontal", + vertical=cfg.INPUT.RANDOM_FLIP == "vertical", + ) + ) + return augmentation + + +build_transform_gen = build_augmentation +""" +Alias for backward-compatibility. +""" diff --git a/annotator/oneformer/detectron2/data/samplers/__init__.py b/annotator/oneformer/detectron2/data/samplers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..85c9f1a9df8a4038fbd4246239b699402e382309 --- /dev/null +++ b/annotator/oneformer/detectron2/data/samplers/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .distributed_sampler import ( + InferenceSampler, + RandomSubsetTrainingSampler, + RepeatFactorTrainingSampler, + TrainingSampler, +) + +from .grouped_batch_sampler import GroupedBatchSampler + +__all__ = [ + "GroupedBatchSampler", + "TrainingSampler", + "RandomSubsetTrainingSampler", + "InferenceSampler", + "RepeatFactorTrainingSampler", +] diff --git a/annotator/oneformer/detectron2/data/samplers/distributed_sampler.py b/annotator/oneformer/detectron2/data/samplers/distributed_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..cd4724eac8fbff2456bd26f95e6fea5e914b73e2 --- /dev/null +++ b/annotator/oneformer/detectron2/data/samplers/distributed_sampler.py @@ -0,0 +1,278 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import itertools +import logging +import math +from collections import defaultdict +from typing import Optional +import torch +from torch.utils.data.sampler import Sampler + +from annotator.oneformer.detectron2.utils import comm + +logger = logging.getLogger(__name__) + + +class TrainingSampler(Sampler): + """ + In training, we only care about the "infinite stream" of training data. + So this sampler produces an infinite stream of indices and + all workers cooperate to correctly shuffle the indices and sample different indices. + + The samplers in each worker effectively produces `indices[worker_id::num_workers]` + where `indices` is an infinite stream of indices consisting of + `shuffle(range(size)) + shuffle(range(size)) + ...` (if shuffle is True) + or `range(size) + range(size) + ...` (if shuffle is False) + + Note that this sampler does not shard based on pytorch DataLoader worker id. + A sampler passed to pytorch DataLoader is used only with map-style dataset + and will not be executed inside workers. + But if this sampler is used in a way that it gets execute inside a dataloader + worker, then extra work needs to be done to shard its outputs based on worker id. + This is required so that workers don't produce identical data. + :class:`ToIterableDataset` implements this logic. + This note is true for all samplers in detectron2. + """ + + def __init__(self, size: int, shuffle: bool = True, seed: Optional[int] = None): + """ + Args: + size (int): the total number of data of the underlying dataset to sample from + shuffle (bool): whether to shuffle the indices or not + seed (int): the initial seed of the shuffle. Must be the same + across all workers. If None, will use a random seed shared + among workers (require synchronization among all workers). + """ + if not isinstance(size, int): + raise TypeError(f"TrainingSampler(size=) expects an int. Got type {type(size)}.") + if size <= 0: + raise ValueError(f"TrainingSampler(size=) expects a positive int. Got {size}.") + self._size = size + self._shuffle = shuffle + if seed is None: + seed = comm.shared_random_seed() + self._seed = int(seed) + + self._rank = comm.get_rank() + self._world_size = comm.get_world_size() + + def __iter__(self): + start = self._rank + yield from itertools.islice(self._infinite_indices(), start, None, self._world_size) + + def _infinite_indices(self): + g = torch.Generator() + g.manual_seed(self._seed) + while True: + if self._shuffle: + yield from torch.randperm(self._size, generator=g).tolist() + else: + yield from torch.arange(self._size).tolist() + + +class RandomSubsetTrainingSampler(TrainingSampler): + """ + Similar to TrainingSampler, but only sample a random subset of indices. + This is useful when you want to estimate the accuracy vs data-number curves by + training the model with different subset_ratio. + """ + + def __init__( + self, + size: int, + subset_ratio: float, + shuffle: bool = True, + seed_shuffle: Optional[int] = None, + seed_subset: Optional[int] = None, + ): + """ + Args: + size (int): the total number of data of the underlying dataset to sample from + subset_ratio (float): the ratio of subset data to sample from the underlying dataset + shuffle (bool): whether to shuffle the indices or not + seed_shuffle (int): the initial seed of the shuffle. Must be the same + across all workers. If None, will use a random seed shared + among workers (require synchronization among all workers). + seed_subset (int): the seed to randomize the subset to be sampled. + Must be the same across all workers. If None, will use a random seed shared + among workers (require synchronization among all workers). + """ + super().__init__(size=size, shuffle=shuffle, seed=seed_shuffle) + + assert 0.0 < subset_ratio <= 1.0 + self._size_subset = int(size * subset_ratio) + assert self._size_subset > 0 + if seed_subset is None: + seed_subset = comm.shared_random_seed() + self._seed_subset = int(seed_subset) + + # randomly generate the subset indexes to be sampled from + g = torch.Generator() + g.manual_seed(self._seed_subset) + indexes_randperm = torch.randperm(self._size, generator=g) + self._indexes_subset = indexes_randperm[: self._size_subset] + + logger.info("Using RandomSubsetTrainingSampler......") + logger.info(f"Randomly sample {self._size_subset} data from the original {self._size} data") + + def _infinite_indices(self): + g = torch.Generator() + g.manual_seed(self._seed) # self._seed equals seed_shuffle from __init__() + while True: + if self._shuffle: + # generate a random permutation to shuffle self._indexes_subset + randperm = torch.randperm(self._size_subset, generator=g) + yield from self._indexes_subset[randperm].tolist() + else: + yield from self._indexes_subset.tolist() + + +class RepeatFactorTrainingSampler(Sampler): + """ + Similar to TrainingSampler, but a sample may appear more times than others based + on its "repeat factor". This is suitable for training on class imbalanced datasets like LVIS. + """ + + def __init__(self, repeat_factors, *, shuffle=True, seed=None): + """ + Args: + repeat_factors (Tensor): a float vector, the repeat factor for each indice. When it's + full of ones, it is equivalent to ``TrainingSampler(len(repeat_factors), ...)``. + shuffle (bool): whether to shuffle the indices or not + seed (int): the initial seed of the shuffle. Must be the same + across all workers. If None, will use a random seed shared + among workers (require synchronization among all workers). + """ + self._shuffle = shuffle + if seed is None: + seed = comm.shared_random_seed() + self._seed = int(seed) + + self._rank = comm.get_rank() + self._world_size = comm.get_world_size() + + # Split into whole number (_int_part) and fractional (_frac_part) parts. + self._int_part = torch.trunc(repeat_factors) + self._frac_part = repeat_factors - self._int_part + + @staticmethod + def repeat_factors_from_category_frequency(dataset_dicts, repeat_thresh): + """ + Compute (fractional) per-image repeat factors based on category frequency. + The repeat factor for an image is a function of the frequency of the rarest + category labeled in that image. The "frequency of category c" in [0, 1] is defined + as the fraction of images in the training set (without repeats) in which category c + appears. + See :paper:`lvis` (>= v2) Appendix B.2. + + Args: + dataset_dicts (list[dict]): annotations in Detectron2 dataset format. + repeat_thresh (float): frequency threshold below which data is repeated. + If the frequency is half of `repeat_thresh`, the image will be + repeated twice. + + Returns: + torch.Tensor: + the i-th element is the repeat factor for the dataset image at index i. + """ + # 1. For each category c, compute the fraction of images that contain it: f(c) + category_freq = defaultdict(int) + for dataset_dict in dataset_dicts: # For each image (without repeats) + cat_ids = {ann["category_id"] for ann in dataset_dict["annotations"]} + for cat_id in cat_ids: + category_freq[cat_id] += 1 + num_images = len(dataset_dicts) + for k, v in category_freq.items(): + category_freq[k] = v / num_images + + # 2. For each category c, compute the category-level repeat factor: + # r(c) = max(1, sqrt(t / f(c))) + category_rep = { + cat_id: max(1.0, math.sqrt(repeat_thresh / cat_freq)) + for cat_id, cat_freq in category_freq.items() + } + + # 3. For each image I, compute the image-level repeat factor: + # r(I) = max_{c in I} r(c) + rep_factors = [] + for dataset_dict in dataset_dicts: + cat_ids = {ann["category_id"] for ann in dataset_dict["annotations"]} + rep_factor = max({category_rep[cat_id] for cat_id in cat_ids}, default=1.0) + rep_factors.append(rep_factor) + + return torch.tensor(rep_factors, dtype=torch.float32) + + def _get_epoch_indices(self, generator): + """ + Create a list of dataset indices (with repeats) to use for one epoch. + + Args: + generator (torch.Generator): pseudo random number generator used for + stochastic rounding. + + Returns: + torch.Tensor: list of dataset indices to use in one epoch. Each index + is repeated based on its calculated repeat factor. + """ + # Since repeat factors are fractional, we use stochastic rounding so + # that the target repeat factor is achieved in expectation over the + # course of training + rands = torch.rand(len(self._frac_part), generator=generator) + rep_factors = self._int_part + (rands < self._frac_part).float() + # Construct a list of indices in which we repeat images as specified + indices = [] + for dataset_index, rep_factor in enumerate(rep_factors): + indices.extend([dataset_index] * int(rep_factor.item())) + return torch.tensor(indices, dtype=torch.int64) + + def __iter__(self): + start = self._rank + yield from itertools.islice(self._infinite_indices(), start, None, self._world_size) + + def _infinite_indices(self): + g = torch.Generator() + g.manual_seed(self._seed) + while True: + # Sample indices with repeats determined by stochastic rounding; each + # "epoch" may have a slightly different size due to the rounding. + indices = self._get_epoch_indices(g) + if self._shuffle: + randperm = torch.randperm(len(indices), generator=g) + yield from indices[randperm].tolist() + else: + yield from indices.tolist() + + +class InferenceSampler(Sampler): + """ + Produce indices for inference across all workers. + Inference needs to run on the __exact__ set of samples, + therefore when the total number of samples is not divisible by the number of workers, + this sampler produces different number of samples on different workers. + """ + + def __init__(self, size: int): + """ + Args: + size (int): the total number of data of the underlying dataset to sample from + """ + self._size = size + assert size > 0 + self._rank = comm.get_rank() + self._world_size = comm.get_world_size() + self._local_indices = self._get_local_indices(size, self._world_size, self._rank) + + @staticmethod + def _get_local_indices(total_size, world_size, rank): + shard_size = total_size // world_size + left = total_size % world_size + shard_sizes = [shard_size + int(r < left) for r in range(world_size)] + + begin = sum(shard_sizes[:rank]) + end = min(sum(shard_sizes[: rank + 1]), total_size) + return range(begin, end) + + def __iter__(self): + yield from self._local_indices + + def __len__(self): + return len(self._local_indices) diff --git a/annotator/oneformer/detectron2/data/samplers/grouped_batch_sampler.py b/annotator/oneformer/detectron2/data/samplers/grouped_batch_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..5b247730aacd04dd0c752664acde3257c4eddd71 --- /dev/null +++ b/annotator/oneformer/detectron2/data/samplers/grouped_batch_sampler.py @@ -0,0 +1,47 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import numpy as np +from torch.utils.data.sampler import BatchSampler, Sampler + + +class GroupedBatchSampler(BatchSampler): + """ + Wraps another sampler to yield a mini-batch of indices. + It enforces that the batch only contain elements from the same group. + It also tries to provide mini-batches which follows an ordering which is + as close as possible to the ordering from the original sampler. + """ + + def __init__(self, sampler, group_ids, batch_size): + """ + Args: + sampler (Sampler): Base sampler. + group_ids (list[int]): If the sampler produces indices in range [0, N), + `group_ids` must be a list of `N` ints which contains the group id of each sample. + The group ids must be a set of integers in the range [0, num_groups). + batch_size (int): Size of mini-batch. + """ + if not isinstance(sampler, Sampler): + raise ValueError( + "sampler should be an instance of " + "torch.utils.data.Sampler, but got sampler={}".format(sampler) + ) + self.sampler = sampler + self.group_ids = np.asarray(group_ids) + assert self.group_ids.ndim == 1 + self.batch_size = batch_size + groups = np.unique(self.group_ids).tolist() + + # buffer the indices of each group until batch size is reached + self.buffer_per_group = {k: [] for k in groups} + + def __iter__(self): + for idx in self.sampler: + group_id = self.group_ids[idx] + group_buffer = self.buffer_per_group[group_id] + group_buffer.append(idx) + if len(group_buffer) == self.batch_size: + yield group_buffer[:] # yield a copy of the list + del group_buffer[:] + + def __len__(self): + raise NotImplementedError("len() of GroupedBatchSampler is not well-defined.") diff --git a/annotator/oneformer/detectron2/data/transforms/__init__.py b/annotator/oneformer/detectron2/data/transforms/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e91c6cdfacd6992a7a1e80c7d2e4b38b2cf7dcde --- /dev/null +++ b/annotator/oneformer/detectron2/data/transforms/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from fvcore.transforms.transform import Transform, TransformList # order them first +from fvcore.transforms.transform import * +from .transform import * +from .augmentation import * +from .augmentation_impl import * + +__all__ = [k for k in globals().keys() if not k.startswith("_")] + + +from annotator.oneformer.detectron2.utils.env import fixup_module_metadata + +fixup_module_metadata(__name__, globals(), __all__) +del fixup_module_metadata diff --git a/annotator/oneformer/detectron2/data/transforms/augmentation.py b/annotator/oneformer/detectron2/data/transforms/augmentation.py new file mode 100644 index 0000000000000000000000000000000000000000..63dd41aef658c9b51c7246880399405a029c5580 --- /dev/null +++ b/annotator/oneformer/detectron2/data/transforms/augmentation.py @@ -0,0 +1,380 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +import inspect +import numpy as np +import pprint +from typing import Any, List, Optional, Tuple, Union +from fvcore.transforms.transform import Transform, TransformList + +""" +See "Data Augmentation" tutorial for an overview of the system: +https://detectron2.readthedocs.io/tutorials/augmentation.html +""" + + +__all__ = [ + "Augmentation", + "AugmentationList", + "AugInput", + "TransformGen", + "apply_transform_gens", + "StandardAugInput", + "apply_augmentations", +] + + +def _check_img_dtype(img): + assert isinstance(img, np.ndarray), "[Augmentation] Needs an numpy array, but got a {}!".format( + type(img) + ) + assert not isinstance(img.dtype, np.integer) or ( + img.dtype == np.uint8 + ), "[Augmentation] Got image of type {}, use uint8 or floating points instead!".format( + img.dtype + ) + assert img.ndim in [2, 3], img.ndim + + +def _get_aug_input_args(aug, aug_input) -> List[Any]: + """ + Get the arguments to be passed to ``aug.get_transform`` from the input ``aug_input``. + """ + if aug.input_args is None: + # Decide what attributes are needed automatically + prms = list(inspect.signature(aug.get_transform).parameters.items()) + # The default behavior is: if there is one parameter, then its "image" + # (work automatically for majority of use cases, and also avoid BC breaking), + # Otherwise, use the argument names. + if len(prms) == 1: + names = ("image",) + else: + names = [] + for name, prm in prms: + if prm.kind in ( + inspect.Parameter.VAR_POSITIONAL, + inspect.Parameter.VAR_KEYWORD, + ): + raise TypeError( + f""" \ +The default implementation of `{type(aug)}.__call__` does not allow \ +`{type(aug)}.get_transform` to use variable-length arguments (*args, **kwargs)! \ +If arguments are unknown, reimplement `__call__` instead. \ +""" + ) + names.append(name) + aug.input_args = tuple(names) + + args = [] + for f in aug.input_args: + try: + args.append(getattr(aug_input, f)) + except AttributeError as e: + raise AttributeError( + f"{type(aug)}.get_transform needs input attribute '{f}', " + f"but it is not an attribute of {type(aug_input)}!" + ) from e + return args + + +class Augmentation: + """ + Augmentation defines (often random) policies/strategies to generate :class:`Transform` + from data. It is often used for pre-processing of input data. + + A "policy" that generates a :class:`Transform` may, in the most general case, + need arbitrary information from input data in order to determine what transforms + to apply. Therefore, each :class:`Augmentation` instance defines the arguments + needed by its :meth:`get_transform` method. When called with the positional arguments, + the :meth:`get_transform` method executes the policy. + + Note that :class:`Augmentation` defines the policies to create a :class:`Transform`, + but not how to execute the actual transform operations to those data. + Its :meth:`__call__` method will use :meth:`AugInput.transform` to execute the transform. + + The returned `Transform` object is meant to describe deterministic transformation, which means + it can be re-applied on associated data, e.g. the geometry of an image and its segmentation + masks need to be transformed together. + (If such re-application is not needed, then determinism is not a crucial requirement.) + """ + + input_args: Optional[Tuple[str]] = None + """ + Stores the attribute names needed by :meth:`get_transform`, e.g. ``("image", "sem_seg")``. + By default, it is just a tuple of argument names in :meth:`self.get_transform`, which often only + contain "image". As long as the argument name convention is followed, there is no need for + users to touch this attribute. + """ + + def _init(self, params=None): + if params: + for k, v in params.items(): + if k != "self" and not k.startswith("_"): + setattr(self, k, v) + + def get_transform(self, *args) -> Transform: + """ + Execute the policy based on input data, and decide what transform to apply to inputs. + + Args: + args: Any fixed-length positional arguments. By default, the name of the arguments + should exist in the :class:`AugInput` to be used. + + Returns: + Transform: Returns the deterministic transform to apply to the input. + + Examples: + :: + class MyAug: + # if a policy needs to know both image and semantic segmentation + def get_transform(image, sem_seg) -> T.Transform: + pass + tfm: Transform = MyAug().get_transform(image, sem_seg) + new_image = tfm.apply_image(image) + + Notes: + Users can freely use arbitrary new argument names in custom + :meth:`get_transform` method, as long as they are available in the + input data. In detectron2 we use the following convention: + + * image: (H,W) or (H,W,C) ndarray of type uint8 in range [0, 255], or + floating point in range [0, 1] or [0, 255]. + * boxes: (N,4) ndarray of float32. It represents the instance bounding boxes + of N instances. Each is in XYXY format in unit of absolute coordinates. + * sem_seg: (H,W) ndarray of type uint8. Each element is an integer label of pixel. + + We do not specify convention for other types and do not include builtin + :class:`Augmentation` that uses other types in detectron2. + """ + raise NotImplementedError + + def __call__(self, aug_input) -> Transform: + """ + Augment the given `aug_input` **in-place**, and return the transform that's used. + + This method will be called to apply the augmentation. In most augmentation, it + is enough to use the default implementation, which calls :meth:`get_transform` + using the inputs. But a subclass can overwrite it to have more complicated logic. + + Args: + aug_input (AugInput): an object that has attributes needed by this augmentation + (defined by ``self.get_transform``). Its ``transform`` method will be called + to in-place transform it. + + Returns: + Transform: the transform that is applied on the input. + """ + args = _get_aug_input_args(self, aug_input) + tfm = self.get_transform(*args) + assert isinstance(tfm, (Transform, TransformList)), ( + f"{type(self)}.get_transform must return an instance of Transform! " + f"Got {type(tfm)} instead." + ) + aug_input.transform(tfm) + return tfm + + def _rand_range(self, low=1.0, high=None, size=None): + """ + Uniform float random number between low and high. + """ + if high is None: + low, high = 0, low + if size is None: + size = [] + return np.random.uniform(low, high, size) + + def __repr__(self): + """ + Produce something like: + "MyAugmentation(field1={self.field1}, field2={self.field2})" + """ + try: + sig = inspect.signature(self.__init__) + classname = type(self).__name__ + argstr = [] + for name, param in sig.parameters.items(): + assert ( + param.kind != param.VAR_POSITIONAL and param.kind != param.VAR_KEYWORD + ), "The default __repr__ doesn't support *args or **kwargs" + assert hasattr(self, name), ( + "Attribute {} not found! " + "Default __repr__ only works if attributes match the constructor.".format(name) + ) + attr = getattr(self, name) + default = param.default + if default is attr: + continue + attr_str = pprint.pformat(attr) + if "\n" in attr_str: + # don't show it if pformat decides to use >1 lines + attr_str = "..." + argstr.append("{}={}".format(name, attr_str)) + return "{}({})".format(classname, ", ".join(argstr)) + except AssertionError: + return super().__repr__() + + __str__ = __repr__ + + +class _TransformToAug(Augmentation): + def __init__(self, tfm: Transform): + self.tfm = tfm + + def get_transform(self, *args): + return self.tfm + + def __repr__(self): + return repr(self.tfm) + + __str__ = __repr__ + + +def _transform_to_aug(tfm_or_aug): + """ + Wrap Transform into Augmentation. + Private, used internally to implement augmentations. + """ + assert isinstance(tfm_or_aug, (Transform, Augmentation)), tfm_or_aug + if isinstance(tfm_or_aug, Augmentation): + return tfm_or_aug + else: + return _TransformToAug(tfm_or_aug) + + +class AugmentationList(Augmentation): + """ + Apply a sequence of augmentations. + + It has ``__call__`` method to apply the augmentations. + + Note that :meth:`get_transform` method is impossible (will throw error if called) + for :class:`AugmentationList`, because in order to apply a sequence of augmentations, + the kth augmentation must be applied first, to provide inputs needed by the (k+1)th + augmentation. + """ + + def __init__(self, augs): + """ + Args: + augs (list[Augmentation or Transform]): + """ + super().__init__() + self.augs = [_transform_to_aug(x) for x in augs] + + def __call__(self, aug_input) -> TransformList: + tfms = [] + for x in self.augs: + tfm = x(aug_input) + tfms.append(tfm) + return TransformList(tfms) + + def __repr__(self): + msgs = [str(x) for x in self.augs] + return "AugmentationList[{}]".format(", ".join(msgs)) + + __str__ = __repr__ + + +class AugInput: + """ + Input that can be used with :meth:`Augmentation.__call__`. + This is a standard implementation for the majority of use cases. + This class provides the standard attributes **"image", "boxes", "sem_seg"** + defined in :meth:`__init__` and they may be needed by different augmentations. + Most augmentation policies do not need attributes beyond these three. + + After applying augmentations to these attributes (using :meth:`AugInput.transform`), + the returned transforms can then be used to transform other data structures that users have. + + Examples: + :: + input = AugInput(image, boxes=boxes) + tfms = augmentation(input) + transformed_image = input.image + transformed_boxes = input.boxes + transformed_other_data = tfms.apply_other(other_data) + + An extended project that works with new data types may implement augmentation policies + that need other inputs. An algorithm may need to transform inputs in a way different + from the standard approach defined in this class. In those rare situations, users can + implement a class similar to this class, that satify the following condition: + + * The input must provide access to these data in the form of attribute access + (``getattr``). For example, if an :class:`Augmentation` to be applied needs "image" + and "sem_seg" arguments, its input must have the attribute "image" and "sem_seg". + * The input must have a ``transform(tfm: Transform) -> None`` method which + in-place transforms all its attributes. + """ + + # TODO maybe should support more builtin data types here + def __init__( + self, + image: np.ndarray, + *, + boxes: Optional[np.ndarray] = None, + sem_seg: Optional[np.ndarray] = None, + ): + """ + Args: + image (ndarray): (H,W) or (H,W,C) ndarray of type uint8 in range [0, 255], or + floating point in range [0, 1] or [0, 255]. The meaning of C is up + to users. + boxes (ndarray or None): Nx4 float32 boxes in XYXY_ABS mode + sem_seg (ndarray or None): HxW uint8 semantic segmentation mask. Each element + is an integer label of pixel. + """ + _check_img_dtype(image) + self.image = image + self.boxes = boxes + self.sem_seg = sem_seg + + def transform(self, tfm: Transform) -> None: + """ + In-place transform all attributes of this class. + + By "in-place", it means after calling this method, accessing an attribute such + as ``self.image`` will return transformed data. + """ + self.image = tfm.apply_image(self.image) + if self.boxes is not None: + self.boxes = tfm.apply_box(self.boxes) + if self.sem_seg is not None: + self.sem_seg = tfm.apply_segmentation(self.sem_seg) + + def apply_augmentations( + self, augmentations: List[Union[Augmentation, Transform]] + ) -> TransformList: + """ + Equivalent of ``AugmentationList(augmentations)(self)`` + """ + return AugmentationList(augmentations)(self) + + +def apply_augmentations(augmentations: List[Union[Transform, Augmentation]], inputs): + """ + Use ``T.AugmentationList(augmentations)(inputs)`` instead. + """ + if isinstance(inputs, np.ndarray): + # handle the common case of image-only Augmentation, also for backward compatibility + image_only = True + inputs = AugInput(inputs) + else: + image_only = False + tfms = inputs.apply_augmentations(augmentations) + return inputs.image if image_only else inputs, tfms + + +apply_transform_gens = apply_augmentations +""" +Alias for backward-compatibility. +""" + +TransformGen = Augmentation +""" +Alias for Augmentation, since it is something that generates :class:`Transform`s +""" + +StandardAugInput = AugInput +""" +Alias for compatibility. It's not worth the complexity to have two classes. +""" diff --git a/annotator/oneformer/detectron2/data/transforms/augmentation_impl.py b/annotator/oneformer/detectron2/data/transforms/augmentation_impl.py new file mode 100644 index 0000000000000000000000000000000000000000..965f0a947d7c3ff03b0990f1a645703d470227de --- /dev/null +++ b/annotator/oneformer/detectron2/data/transforms/augmentation_impl.py @@ -0,0 +1,736 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. +""" +Implement many useful :class:`Augmentation`. +""" +import numpy as np +import sys +from numpy import random +from typing import Tuple +import torch +from fvcore.transforms.transform import ( + BlendTransform, + CropTransform, + HFlipTransform, + NoOpTransform, + PadTransform, + Transform, + TransformList, + VFlipTransform, +) +from PIL import Image + +from annotator.oneformer.detectron2.structures import Boxes, pairwise_iou + +from .augmentation import Augmentation, _transform_to_aug +from .transform import ExtentTransform, ResizeTransform, RotationTransform + +__all__ = [ + "FixedSizeCrop", + "RandomApply", + "RandomBrightness", + "RandomContrast", + "RandomCrop", + "RandomExtent", + "RandomFlip", + "RandomSaturation", + "RandomLighting", + "RandomRotation", + "Resize", + "ResizeScale", + "ResizeShortestEdge", + "RandomCrop_CategoryAreaConstraint", + "RandomResize", + "MinIoURandomCrop", +] + + +class RandomApply(Augmentation): + """ + Randomly apply an augmentation with a given probability. + """ + + def __init__(self, tfm_or_aug, prob=0.5): + """ + Args: + tfm_or_aug (Transform, Augmentation): the transform or augmentation + to be applied. It can either be a `Transform` or `Augmentation` + instance. + prob (float): probability between 0.0 and 1.0 that + the wrapper transformation is applied + """ + super().__init__() + self.aug = _transform_to_aug(tfm_or_aug) + assert 0.0 <= prob <= 1.0, f"Probablity must be between 0.0 and 1.0 (given: {prob})" + self.prob = prob + + def get_transform(self, *args): + do = self._rand_range() < self.prob + if do: + return self.aug.get_transform(*args) + else: + return NoOpTransform() + + def __call__(self, aug_input): + do = self._rand_range() < self.prob + if do: + return self.aug(aug_input) + else: + return NoOpTransform() + + +class RandomFlip(Augmentation): + """ + Flip the image horizontally or vertically with the given probability. + """ + + def __init__(self, prob=0.5, *, horizontal=True, vertical=False): + """ + Args: + prob (float): probability of flip. + horizontal (boolean): whether to apply horizontal flipping + vertical (boolean): whether to apply vertical flipping + """ + super().__init__() + + if horizontal and vertical: + raise ValueError("Cannot do both horiz and vert. Please use two Flip instead.") + if not horizontal and not vertical: + raise ValueError("At least one of horiz or vert has to be True!") + self._init(locals()) + + def get_transform(self, image): + h, w = image.shape[:2] + do = self._rand_range() < self.prob + if do: + if self.horizontal: + return HFlipTransform(w) + elif self.vertical: + return VFlipTransform(h) + else: + return NoOpTransform() + + +class Resize(Augmentation): + """Resize image to a fixed target size""" + + def __init__(self, shape, interp=Image.BILINEAR): + """ + Args: + shape: (h, w) tuple or a int + interp: PIL interpolation method + """ + if isinstance(shape, int): + shape = (shape, shape) + shape = tuple(shape) + self._init(locals()) + + def get_transform(self, image): + return ResizeTransform( + image.shape[0], image.shape[1], self.shape[0], self.shape[1], self.interp + ) + + +class ResizeShortestEdge(Augmentation): + """ + Resize the image while keeping the aspect ratio unchanged. + It attempts to scale the shorter edge to the given `short_edge_length`, + as long as the longer edge does not exceed `max_size`. + If `max_size` is reached, then downscale so that the longer edge does not exceed max_size. + """ + + @torch.jit.unused + def __init__( + self, short_edge_length, max_size=sys.maxsize, sample_style="range", interp=Image.BILINEAR + ): + """ + Args: + short_edge_length (list[int]): If ``sample_style=="range"``, + a [min, max] interval from which to sample the shortest edge length. + If ``sample_style=="choice"``, a list of shortest edge lengths to sample from. + max_size (int): maximum allowed longest edge length. + sample_style (str): either "range" or "choice". + """ + super().__init__() + assert sample_style in ["range", "choice"], sample_style + + self.is_range = sample_style == "range" + if isinstance(short_edge_length, int): + short_edge_length = (short_edge_length, short_edge_length) + if self.is_range: + assert len(short_edge_length) == 2, ( + "short_edge_length must be two values using 'range' sample style." + f" Got {short_edge_length}!" + ) + self._init(locals()) + + @torch.jit.unused + def get_transform(self, image): + h, w = image.shape[:2] + if self.is_range: + size = np.random.randint(self.short_edge_length[0], self.short_edge_length[1] + 1) + else: + size = np.random.choice(self.short_edge_length) + if size == 0: + return NoOpTransform() + + newh, neww = ResizeShortestEdge.get_output_shape(h, w, size, self.max_size) + return ResizeTransform(h, w, newh, neww, self.interp) + + @staticmethod + def get_output_shape( + oldh: int, oldw: int, short_edge_length: int, max_size: int + ) -> Tuple[int, int]: + """ + Compute the output size given input size and target short edge length. + """ + h, w = oldh, oldw + size = short_edge_length * 1.0 + scale = size / min(h, w) + if h < w: + newh, neww = size, scale * w + else: + newh, neww = scale * h, size + if max(newh, neww) > max_size: + scale = max_size * 1.0 / max(newh, neww) + newh = newh * scale + neww = neww * scale + neww = int(neww + 0.5) + newh = int(newh + 0.5) + return (newh, neww) + + +class ResizeScale(Augmentation): + """ + Takes target size as input and randomly scales the given target size between `min_scale` + and `max_scale`. It then scales the input image such that it fits inside the scaled target + box, keeping the aspect ratio constant. + This implements the resize part of the Google's 'resize_and_crop' data augmentation: + https://github.com/tensorflow/tpu/blob/master/models/official/detection/utils/input_utils.py#L127 + """ + + def __init__( + self, + min_scale: float, + max_scale: float, + target_height: int, + target_width: int, + interp: int = Image.BILINEAR, + ): + """ + Args: + min_scale: minimum image scale range. + max_scale: maximum image scale range. + target_height: target image height. + target_width: target image width. + interp: image interpolation method. + """ + super().__init__() + self._init(locals()) + + def _get_resize(self, image: np.ndarray, scale: float) -> Transform: + input_size = image.shape[:2] + + # Compute new target size given a scale. + target_size = (self.target_height, self.target_width) + target_scale_size = np.multiply(target_size, scale) + + # Compute actual rescaling applied to input image and output size. + output_scale = np.minimum( + target_scale_size[0] / input_size[0], target_scale_size[1] / input_size[1] + ) + output_size = np.round(np.multiply(input_size, output_scale)).astype(int) + + return ResizeTransform( + input_size[0], input_size[1], output_size[0], output_size[1], self.interp + ) + + def get_transform(self, image: np.ndarray) -> Transform: + random_scale = np.random.uniform(self.min_scale, self.max_scale) + return self._get_resize(image, random_scale) + + +class RandomRotation(Augmentation): + """ + This method returns a copy of this image, rotated the given + number of degrees counter clockwise around the given center. + """ + + def __init__(self, angle, expand=True, center=None, sample_style="range", interp=None): + """ + Args: + angle (list[float]): If ``sample_style=="range"``, + a [min, max] interval from which to sample the angle (in degrees). + If ``sample_style=="choice"``, a list of angles to sample from + expand (bool): choose if the image should be resized to fit the whole + rotated image (default), or simply cropped + center (list[[float, float]]): If ``sample_style=="range"``, + a [[minx, miny], [maxx, maxy]] relative interval from which to sample the center, + [0, 0] being the top left of the image and [1, 1] the bottom right. + If ``sample_style=="choice"``, a list of centers to sample from + Default: None, which means that the center of rotation is the center of the image + center has no effect if expand=True because it only affects shifting + """ + super().__init__() + assert sample_style in ["range", "choice"], sample_style + self.is_range = sample_style == "range" + if isinstance(angle, (float, int)): + angle = (angle, angle) + if center is not None and isinstance(center[0], (float, int)): + center = (center, center) + self._init(locals()) + + def get_transform(self, image): + h, w = image.shape[:2] + center = None + if self.is_range: + angle = np.random.uniform(self.angle[0], self.angle[1]) + if self.center is not None: + center = ( + np.random.uniform(self.center[0][0], self.center[1][0]), + np.random.uniform(self.center[0][1], self.center[1][1]), + ) + else: + angle = np.random.choice(self.angle) + if self.center is not None: + center = np.random.choice(self.center) + + if center is not None: + center = (w * center[0], h * center[1]) # Convert to absolute coordinates + + if angle % 360 == 0: + return NoOpTransform() + + return RotationTransform(h, w, angle, expand=self.expand, center=center, interp=self.interp) + + +class FixedSizeCrop(Augmentation): + """ + If `crop_size` is smaller than the input image size, then it uses a random crop of + the crop size. If `crop_size` is larger than the input image size, then it pads + the right and the bottom of the image to the crop size if `pad` is True, otherwise + it returns the smaller image. + """ + + def __init__( + self, + crop_size: Tuple[int], + pad: bool = True, + pad_value: float = 128.0, + seg_pad_value: int = 255, + ): + """ + Args: + crop_size: target image (height, width). + pad: if True, will pad images smaller than `crop_size` up to `crop_size` + pad_value: the padding value to the image. + seg_pad_value: the padding value to the segmentation mask. + """ + super().__init__() + self._init(locals()) + + def _get_crop(self, image: np.ndarray) -> Transform: + # Compute the image scale and scaled size. + input_size = image.shape[:2] + output_size = self.crop_size + + # Add random crop if the image is scaled up. + max_offset = np.subtract(input_size, output_size) + max_offset = np.maximum(max_offset, 0) + offset = np.multiply(max_offset, np.random.uniform(0.0, 1.0)) + offset = np.round(offset).astype(int) + return CropTransform( + offset[1], offset[0], output_size[1], output_size[0], input_size[1], input_size[0] + ) + + def _get_pad(self, image: np.ndarray) -> Transform: + # Compute the image scale and scaled size. + input_size = image.shape[:2] + output_size = self.crop_size + + # Add padding if the image is scaled down. + pad_size = np.subtract(output_size, input_size) + pad_size = np.maximum(pad_size, 0) + original_size = np.minimum(input_size, output_size) + return PadTransform( + 0, + 0, + pad_size[1], + pad_size[0], + original_size[1], + original_size[0], + self.pad_value, + self.seg_pad_value, + ) + + def get_transform(self, image: np.ndarray) -> TransformList: + transforms = [self._get_crop(image)] + if self.pad: + transforms.append(self._get_pad(image)) + return TransformList(transforms) + + +class RandomCrop(Augmentation): + """ + Randomly crop a rectangle region out of an image. + """ + + def __init__(self, crop_type: str, crop_size): + """ + Args: + crop_type (str): one of "relative_range", "relative", "absolute", "absolute_range". + crop_size (tuple[float, float]): two floats, explained below. + + - "relative": crop a (H * crop_size[0], W * crop_size[1]) region from an input image of + size (H, W). crop size should be in (0, 1] + - "relative_range": uniformly sample two values from [crop_size[0], 1] + and [crop_size[1]], 1], and use them as in "relative" crop type. + - "absolute" crop a (crop_size[0], crop_size[1]) region from input image. + crop_size must be smaller than the input image size. + - "absolute_range", for an input of size (H, W), uniformly sample H_crop in + [crop_size[0], min(H, crop_size[1])] and W_crop in [crop_size[0], min(W, crop_size[1])]. + Then crop a region (H_crop, W_crop). + """ + # TODO style of relative_range and absolute_range are not consistent: + # one takes (h, w) but another takes (min, max) + super().__init__() + assert crop_type in ["relative_range", "relative", "absolute", "absolute_range"] + self._init(locals()) + + def get_transform(self, image): + h, w = image.shape[:2] + croph, cropw = self.get_crop_size((h, w)) + assert h >= croph and w >= cropw, "Shape computation in {} has bugs.".format(self) + h0 = np.random.randint(h - croph + 1) + w0 = np.random.randint(w - cropw + 1) + return CropTransform(w0, h0, cropw, croph) + + def get_crop_size(self, image_size): + """ + Args: + image_size (tuple): height, width + + Returns: + crop_size (tuple): height, width in absolute pixels + """ + h, w = image_size + if self.crop_type == "relative": + ch, cw = self.crop_size + return int(h * ch + 0.5), int(w * cw + 0.5) + elif self.crop_type == "relative_range": + crop_size = np.asarray(self.crop_size, dtype=np.float32) + ch, cw = crop_size + np.random.rand(2) * (1 - crop_size) + return int(h * ch + 0.5), int(w * cw + 0.5) + elif self.crop_type == "absolute": + return (min(self.crop_size[0], h), min(self.crop_size[1], w)) + elif self.crop_type == "absolute_range": + assert self.crop_size[0] <= self.crop_size[1] + ch = np.random.randint(min(h, self.crop_size[0]), min(h, self.crop_size[1]) + 1) + cw = np.random.randint(min(w, self.crop_size[0]), min(w, self.crop_size[1]) + 1) + return ch, cw + else: + raise NotImplementedError("Unknown crop type {}".format(self.crop_type)) + + +class RandomCrop_CategoryAreaConstraint(Augmentation): + """ + Similar to :class:`RandomCrop`, but find a cropping window such that no single category + occupies a ratio of more than `single_category_max_area` in semantic segmentation ground + truth, which can cause unstability in training. The function attempts to find such a valid + cropping window for at most 10 times. + """ + + def __init__( + self, + crop_type: str, + crop_size, + single_category_max_area: float = 1.0, + ignored_category: int = None, + ): + """ + Args: + crop_type, crop_size: same as in :class:`RandomCrop` + single_category_max_area: the maximum allowed area ratio of a + category. Set to 1.0 to disable + ignored_category: allow this category in the semantic segmentation + ground truth to exceed the area ratio. Usually set to the category + that's ignored in training. + """ + self.crop_aug = RandomCrop(crop_type, crop_size) + self._init(locals()) + + def get_transform(self, image, sem_seg): + if self.single_category_max_area >= 1.0: + return self.crop_aug.get_transform(image) + else: + h, w = sem_seg.shape + for _ in range(10): + crop_size = self.crop_aug.get_crop_size((h, w)) + y0 = np.random.randint(h - crop_size[0] + 1) + x0 = np.random.randint(w - crop_size[1] + 1) + sem_seg_temp = sem_seg[y0 : y0 + crop_size[0], x0 : x0 + crop_size[1]] + labels, cnt = np.unique(sem_seg_temp, return_counts=True) + if self.ignored_category is not None: + cnt = cnt[labels != self.ignored_category] + if len(cnt) > 1 and np.max(cnt) < np.sum(cnt) * self.single_category_max_area: + break + crop_tfm = CropTransform(x0, y0, crop_size[1], crop_size[0]) + return crop_tfm + + +class RandomExtent(Augmentation): + """ + Outputs an image by cropping a random "subrect" of the source image. + + The subrect can be parameterized to include pixels outside the source image, + in which case they will be set to zeros (i.e. black). The size of the output + image will vary with the size of the random subrect. + """ + + def __init__(self, scale_range, shift_range): + """ + Args: + output_size (h, w): Dimensions of output image + scale_range (l, h): Range of input-to-output size scaling factor + shift_range (x, y): Range of shifts of the cropped subrect. The rect + is shifted by [w / 2 * Uniform(-x, x), h / 2 * Uniform(-y, y)], + where (w, h) is the (width, height) of the input image. Set each + component to zero to crop at the image's center. + """ + super().__init__() + self._init(locals()) + + def get_transform(self, image): + img_h, img_w = image.shape[:2] + + # Initialize src_rect to fit the input image. + src_rect = np.array([-0.5 * img_w, -0.5 * img_h, 0.5 * img_w, 0.5 * img_h]) + + # Apply a random scaling to the src_rect. + src_rect *= np.random.uniform(self.scale_range[0], self.scale_range[1]) + + # Apply a random shift to the coordinates origin. + src_rect[0::2] += self.shift_range[0] * img_w * (np.random.rand() - 0.5) + src_rect[1::2] += self.shift_range[1] * img_h * (np.random.rand() - 0.5) + + # Map src_rect coordinates into image coordinates (center at corner). + src_rect[0::2] += 0.5 * img_w + src_rect[1::2] += 0.5 * img_h + + return ExtentTransform( + src_rect=(src_rect[0], src_rect[1], src_rect[2], src_rect[3]), + output_size=(int(src_rect[3] - src_rect[1]), int(src_rect[2] - src_rect[0])), + ) + + +class RandomContrast(Augmentation): + """ + Randomly transforms image contrast. + + Contrast intensity is uniformly sampled in (intensity_min, intensity_max). + - intensity < 1 will reduce contrast + - intensity = 1 will preserve the input image + - intensity > 1 will increase contrast + + See: https://pillow.readthedocs.io/en/3.0.x/reference/ImageEnhance.html + """ + + def __init__(self, intensity_min, intensity_max): + """ + Args: + intensity_min (float): Minimum augmentation + intensity_max (float): Maximum augmentation + """ + super().__init__() + self._init(locals()) + + def get_transform(self, image): + w = np.random.uniform(self.intensity_min, self.intensity_max) + return BlendTransform(src_image=image.mean(), src_weight=1 - w, dst_weight=w) + + +class RandomBrightness(Augmentation): + """ + Randomly transforms image brightness. + + Brightness intensity is uniformly sampled in (intensity_min, intensity_max). + - intensity < 1 will reduce brightness + - intensity = 1 will preserve the input image + - intensity > 1 will increase brightness + + See: https://pillow.readthedocs.io/en/3.0.x/reference/ImageEnhance.html + """ + + def __init__(self, intensity_min, intensity_max): + """ + Args: + intensity_min (float): Minimum augmentation + intensity_max (float): Maximum augmentation + """ + super().__init__() + self._init(locals()) + + def get_transform(self, image): + w = np.random.uniform(self.intensity_min, self.intensity_max) + return BlendTransform(src_image=0, src_weight=1 - w, dst_weight=w) + + +class RandomSaturation(Augmentation): + """ + Randomly transforms saturation of an RGB image. + Input images are assumed to have 'RGB' channel order. + + Saturation intensity is uniformly sampled in (intensity_min, intensity_max). + - intensity < 1 will reduce saturation (make the image more grayscale) + - intensity = 1 will preserve the input image + - intensity > 1 will increase saturation + + See: https://pillow.readthedocs.io/en/3.0.x/reference/ImageEnhance.html + """ + + def __init__(self, intensity_min, intensity_max): + """ + Args: + intensity_min (float): Minimum augmentation (1 preserves input). + intensity_max (float): Maximum augmentation (1 preserves input). + """ + super().__init__() + self._init(locals()) + + def get_transform(self, image): + assert image.shape[-1] == 3, "RandomSaturation only works on RGB images" + w = np.random.uniform(self.intensity_min, self.intensity_max) + grayscale = image.dot([0.299, 0.587, 0.114])[:, :, np.newaxis] + return BlendTransform(src_image=grayscale, src_weight=1 - w, dst_weight=w) + + +class RandomLighting(Augmentation): + """ + The "lighting" augmentation described in AlexNet, using fixed PCA over ImageNet. + Input images are assumed to have 'RGB' channel order. + + The degree of color jittering is randomly sampled via a normal distribution, + with standard deviation given by the scale parameter. + """ + + def __init__(self, scale): + """ + Args: + scale (float): Standard deviation of principal component weighting. + """ + super().__init__() + self._init(locals()) + self.eigen_vecs = np.array( + [[-0.5675, 0.7192, 0.4009], [-0.5808, -0.0045, -0.8140], [-0.5836, -0.6948, 0.4203]] + ) + self.eigen_vals = np.array([0.2175, 0.0188, 0.0045]) + + def get_transform(self, image): + assert image.shape[-1] == 3, "RandomLighting only works on RGB images" + weights = np.random.normal(scale=self.scale, size=3) + return BlendTransform( + src_image=self.eigen_vecs.dot(weights * self.eigen_vals), src_weight=1.0, dst_weight=1.0 + ) + + +class RandomResize(Augmentation): + """Randomly resize image to a target size in shape_list""" + + def __init__(self, shape_list, interp=Image.BILINEAR): + """ + Args: + shape_list: a list of shapes in (h, w) + interp: PIL interpolation method + """ + self.shape_list = shape_list + self._init(locals()) + + def get_transform(self, image): + shape_idx = np.random.randint(low=0, high=len(self.shape_list)) + h, w = self.shape_list[shape_idx] + return ResizeTransform(image.shape[0], image.shape[1], h, w, self.interp) + + +class MinIoURandomCrop(Augmentation): + """Random crop the image & bboxes, the cropped patches have minimum IoU + requirement with original image & bboxes, the IoU threshold is randomly + selected from min_ious. + + Args: + min_ious (tuple): minimum IoU threshold for all intersections with + bounding boxes + min_crop_size (float): minimum crop's size (i.e. h,w := a*h, a*w, + where a >= min_crop_size) + mode_trials: number of trials for sampling min_ious threshold + crop_trials: number of trials for sampling crop_size after cropping + """ + + def __init__( + self, + min_ious=(0.1, 0.3, 0.5, 0.7, 0.9), + min_crop_size=0.3, + mode_trials=1000, + crop_trials=50, + ): + self.min_ious = min_ious + self.sample_mode = (1, *min_ious, 0) + self.min_crop_size = min_crop_size + self.mode_trials = mode_trials + self.crop_trials = crop_trials + + def get_transform(self, image, boxes): + """Call function to crop images and bounding boxes with minimum IoU + constraint. + + Args: + boxes: ground truth boxes in (x1, y1, x2, y2) format + """ + if boxes is None: + return NoOpTransform() + h, w, c = image.shape + for _ in range(self.mode_trials): + mode = random.choice(self.sample_mode) + self.mode = mode + if mode == 1: + return NoOpTransform() + + min_iou = mode + for _ in range(self.crop_trials): + new_w = random.uniform(self.min_crop_size * w, w) + new_h = random.uniform(self.min_crop_size * h, h) + + # h / w in [0.5, 2] + if new_h / new_w < 0.5 or new_h / new_w > 2: + continue + + left = random.uniform(w - new_w) + top = random.uniform(h - new_h) + + patch = np.array((int(left), int(top), int(left + new_w), int(top + new_h))) + # Line or point crop is not allowed + if patch[2] == patch[0] or patch[3] == patch[1]: + continue + overlaps = pairwise_iou( + Boxes(patch.reshape(-1, 4)), Boxes(boxes.reshape(-1, 4)) + ).reshape(-1) + if len(overlaps) > 0 and overlaps.min() < min_iou: + continue + + # center of boxes should inside the crop img + # only adjust boxes and instance masks when the gt is not empty + if len(overlaps) > 0: + # adjust boxes + def is_center_of_bboxes_in_patch(boxes, patch): + center = (boxes[:, :2] + boxes[:, 2:]) / 2 + mask = ( + (center[:, 0] > patch[0]) + * (center[:, 1] > patch[1]) + * (center[:, 0] < patch[2]) + * (center[:, 1] < patch[3]) + ) + return mask + + mask = is_center_of_bboxes_in_patch(boxes, patch) + if not mask.any(): + continue + return CropTransform(int(left), int(top), int(new_w), int(new_h)) diff --git a/annotator/oneformer/detectron2/data/transforms/transform.py b/annotator/oneformer/detectron2/data/transforms/transform.py new file mode 100644 index 0000000000000000000000000000000000000000..de44b991d7ab0d920ffb769e1402f08e358d37f7 --- /dev/null +++ b/annotator/oneformer/detectron2/data/transforms/transform.py @@ -0,0 +1,351 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +""" +See "Data Augmentation" tutorial for an overview of the system: +https://detectron2.readthedocs.io/tutorials/augmentation.html +""" + +import numpy as np +import torch +import torch.nn.functional as F +from fvcore.transforms.transform import ( + CropTransform, + HFlipTransform, + NoOpTransform, + Transform, + TransformList, +) +from PIL import Image + +try: + import cv2 # noqa +except ImportError: + # OpenCV is an optional dependency at the moment + pass + +__all__ = [ + "ExtentTransform", + "ResizeTransform", + "RotationTransform", + "ColorTransform", + "PILColorTransform", +] + + +class ExtentTransform(Transform): + """ + Extracts a subregion from the source image and scales it to the output size. + + The fill color is used to map pixels from the source rect that fall outside + the source image. + + See: https://pillow.readthedocs.io/en/latest/PIL.html#PIL.ImageTransform.ExtentTransform + """ + + def __init__(self, src_rect, output_size, interp=Image.LINEAR, fill=0): + """ + Args: + src_rect (x0, y0, x1, y1): src coordinates + output_size (h, w): dst image size + interp: PIL interpolation methods + fill: Fill color used when src_rect extends outside image + """ + super().__init__() + self._set_attributes(locals()) + + def apply_image(self, img, interp=None): + h, w = self.output_size + if len(img.shape) > 2 and img.shape[2] == 1: + pil_image = Image.fromarray(img[:, :, 0], mode="L") + else: + pil_image = Image.fromarray(img) + pil_image = pil_image.transform( + size=(w, h), + method=Image.EXTENT, + data=self.src_rect, + resample=interp if interp else self.interp, + fill=self.fill, + ) + ret = np.asarray(pil_image) + if len(img.shape) > 2 and img.shape[2] == 1: + ret = np.expand_dims(ret, -1) + return ret + + def apply_coords(self, coords): + # Transform image center from source coordinates into output coordinates + # and then map the new origin to the corner of the output image. + h, w = self.output_size + x0, y0, x1, y1 = self.src_rect + new_coords = coords.astype(np.float32) + new_coords[:, 0] -= 0.5 * (x0 + x1) + new_coords[:, 1] -= 0.5 * (y0 + y1) + new_coords[:, 0] *= w / (x1 - x0) + new_coords[:, 1] *= h / (y1 - y0) + new_coords[:, 0] += 0.5 * w + new_coords[:, 1] += 0.5 * h + return new_coords + + def apply_segmentation(self, segmentation): + segmentation = self.apply_image(segmentation, interp=Image.NEAREST) + return segmentation + + +class ResizeTransform(Transform): + """ + Resize the image to a target size. + """ + + def __init__(self, h, w, new_h, new_w, interp=None): + """ + Args: + h, w (int): original image size + new_h, new_w (int): new image size + interp: PIL interpolation methods, defaults to bilinear. + """ + # TODO decide on PIL vs opencv + super().__init__() + if interp is None: + interp = Image.BILINEAR + self._set_attributes(locals()) + + def apply_image(self, img, interp=None): + assert img.shape[:2] == (self.h, self.w) + assert len(img.shape) <= 4 + interp_method = interp if interp is not None else self.interp + + if img.dtype == np.uint8: + if len(img.shape) > 2 and img.shape[2] == 1: + pil_image = Image.fromarray(img[:, :, 0], mode="L") + else: + pil_image = Image.fromarray(img) + pil_image = pil_image.resize((self.new_w, self.new_h), interp_method) + ret = np.asarray(pil_image) + if len(img.shape) > 2 and img.shape[2] == 1: + ret = np.expand_dims(ret, -1) + else: + # PIL only supports uint8 + if any(x < 0 for x in img.strides): + img = np.ascontiguousarray(img) + img = torch.from_numpy(img) + shape = list(img.shape) + shape_4d = shape[:2] + [1] * (4 - len(shape)) + shape[2:] + img = img.view(shape_4d).permute(2, 3, 0, 1) # hw(c) -> nchw + _PIL_RESIZE_TO_INTERPOLATE_MODE = { + Image.NEAREST: "nearest", + Image.BILINEAR: "bilinear", + Image.BICUBIC: "bicubic", + } + mode = _PIL_RESIZE_TO_INTERPOLATE_MODE[interp_method] + align_corners = None if mode == "nearest" else False + img = F.interpolate( + img, (self.new_h, self.new_w), mode=mode, align_corners=align_corners + ) + shape[:2] = (self.new_h, self.new_w) + ret = img.permute(2, 3, 0, 1).view(shape).numpy() # nchw -> hw(c) + + return ret + + def apply_coords(self, coords): + coords[:, 0] = coords[:, 0] * (self.new_w * 1.0 / self.w) + coords[:, 1] = coords[:, 1] * (self.new_h * 1.0 / self.h) + return coords + + def apply_segmentation(self, segmentation): + segmentation = self.apply_image(segmentation, interp=Image.NEAREST) + return segmentation + + def inverse(self): + return ResizeTransform(self.new_h, self.new_w, self.h, self.w, self.interp) + + +class RotationTransform(Transform): + """ + This method returns a copy of this image, rotated the given + number of degrees counter clockwise around its center. + """ + + def __init__(self, h, w, angle, expand=True, center=None, interp=None): + """ + Args: + h, w (int): original image size + angle (float): degrees for rotation + expand (bool): choose if the image should be resized to fit the whole + rotated image (default), or simply cropped + center (tuple (width, height)): coordinates of the rotation center + if left to None, the center will be fit to the center of each image + center has no effect if expand=True because it only affects shifting + interp: cv2 interpolation method, default cv2.INTER_LINEAR + """ + super().__init__() + image_center = np.array((w / 2, h / 2)) + if center is None: + center = image_center + if interp is None: + interp = cv2.INTER_LINEAR + abs_cos, abs_sin = (abs(np.cos(np.deg2rad(angle))), abs(np.sin(np.deg2rad(angle)))) + if expand: + # find the new width and height bounds + bound_w, bound_h = np.rint( + [h * abs_sin + w * abs_cos, h * abs_cos + w * abs_sin] + ).astype(int) + else: + bound_w, bound_h = w, h + + self._set_attributes(locals()) + self.rm_coords = self.create_rotation_matrix() + # Needed because of this problem https://github.com/opencv/opencv/issues/11784 + self.rm_image = self.create_rotation_matrix(offset=-0.5) + + def apply_image(self, img, interp=None): + """ + img should be a numpy array, formatted as Height * Width * Nchannels + """ + if len(img) == 0 or self.angle % 360 == 0: + return img + assert img.shape[:2] == (self.h, self.w) + interp = interp if interp is not None else self.interp + return cv2.warpAffine(img, self.rm_image, (self.bound_w, self.bound_h), flags=interp) + + def apply_coords(self, coords): + """ + coords should be a N * 2 array-like, containing N couples of (x, y) points + """ + coords = np.asarray(coords, dtype=float) + if len(coords) == 0 or self.angle % 360 == 0: + return coords + return cv2.transform(coords[:, np.newaxis, :], self.rm_coords)[:, 0, :] + + def apply_segmentation(self, segmentation): + segmentation = self.apply_image(segmentation, interp=cv2.INTER_NEAREST) + return segmentation + + def create_rotation_matrix(self, offset=0): + center = (self.center[0] + offset, self.center[1] + offset) + rm = cv2.getRotationMatrix2D(tuple(center), self.angle, 1) + if self.expand: + # Find the coordinates of the center of rotation in the new image + # The only point for which we know the future coordinates is the center of the image + rot_im_center = cv2.transform(self.image_center[None, None, :] + offset, rm)[0, 0, :] + new_center = np.array([self.bound_w / 2, self.bound_h / 2]) + offset - rot_im_center + # shift the rotation center to the new coordinates + rm[:, 2] += new_center + return rm + + def inverse(self): + """ + The inverse is to rotate it back with expand, and crop to get the original shape. + """ + if not self.expand: # Not possible to inverse if a part of the image is lost + raise NotImplementedError() + rotation = RotationTransform( + self.bound_h, self.bound_w, -self.angle, True, None, self.interp + ) + crop = CropTransform( + (rotation.bound_w - self.w) // 2, (rotation.bound_h - self.h) // 2, self.w, self.h + ) + return TransformList([rotation, crop]) + + +class ColorTransform(Transform): + """ + Generic wrapper for any photometric transforms. + These transformations should only affect the color space and + not the coordinate space of the image (e.g. annotation + coordinates such as bounding boxes should not be changed) + """ + + def __init__(self, op): + """ + Args: + op (Callable): operation to be applied to the image, + which takes in an ndarray and returns an ndarray. + """ + if not callable(op): + raise ValueError("op parameter should be callable") + super().__init__() + self._set_attributes(locals()) + + def apply_image(self, img): + return self.op(img) + + def apply_coords(self, coords): + return coords + + def inverse(self): + return NoOpTransform() + + def apply_segmentation(self, segmentation): + return segmentation + + +class PILColorTransform(ColorTransform): + """ + Generic wrapper for PIL Photometric image transforms, + which affect the color space and not the coordinate + space of the image + """ + + def __init__(self, op): + """ + Args: + op (Callable): operation to be applied to the image, + which takes in a PIL Image and returns a transformed + PIL Image. + For reference on possible operations see: + - https://pillow.readthedocs.io/en/stable/ + """ + if not callable(op): + raise ValueError("op parameter should be callable") + super().__init__(op) + + def apply_image(self, img): + img = Image.fromarray(img) + return np.asarray(super().apply_image(img)) + + +def HFlip_rotated_box(transform, rotated_boxes): + """ + Apply the horizontal flip transform on rotated boxes. + + Args: + rotated_boxes (ndarray): Nx5 floating point array of + (x_center, y_center, width, height, angle_degrees) format + in absolute coordinates. + """ + # Transform x_center + rotated_boxes[:, 0] = transform.width - rotated_boxes[:, 0] + # Transform angle + rotated_boxes[:, 4] = -rotated_boxes[:, 4] + return rotated_boxes + + +def Resize_rotated_box(transform, rotated_boxes): + """ + Apply the resizing transform on rotated boxes. For details of how these (approximation) + formulas are derived, please refer to :meth:`RotatedBoxes.scale`. + + Args: + rotated_boxes (ndarray): Nx5 floating point array of + (x_center, y_center, width, height, angle_degrees) format + in absolute coordinates. + """ + scale_factor_x = transform.new_w * 1.0 / transform.w + scale_factor_y = transform.new_h * 1.0 / transform.h + rotated_boxes[:, 0] *= scale_factor_x + rotated_boxes[:, 1] *= scale_factor_y + theta = rotated_boxes[:, 4] * np.pi / 180.0 + c = np.cos(theta) + s = np.sin(theta) + rotated_boxes[:, 2] *= np.sqrt(np.square(scale_factor_x * c) + np.square(scale_factor_y * s)) + rotated_boxes[:, 3] *= np.sqrt(np.square(scale_factor_x * s) + np.square(scale_factor_y * c)) + rotated_boxes[:, 4] = np.arctan2(scale_factor_x * s, scale_factor_y * c) * 180 / np.pi + + return rotated_boxes + + +HFlipTransform.register_type("rotated_box", HFlip_rotated_box) +ResizeTransform.register_type("rotated_box", Resize_rotated_box) + +# not necessary any more with latest fvcore +NoOpTransform.register_type("rotated_box", lambda t, x: x) diff --git a/annotator/oneformer/detectron2/engine/__init__.py b/annotator/oneformer/detectron2/engine/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..08a61572b4c7d09c8d400e903a96cbf5b2cc4763 --- /dev/null +++ b/annotator/oneformer/detectron2/engine/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from .launch import * +from .train_loop import * + +__all__ = [k for k in globals().keys() if not k.startswith("_")] + + +# prefer to let hooks and defaults live in separate namespaces (therefore not in __all__) +# but still make them available here +from .hooks import * +from .defaults import * diff --git a/annotator/oneformer/detectron2/engine/defaults.py b/annotator/oneformer/detectron2/engine/defaults.py new file mode 100644 index 0000000000000000000000000000000000000000..51d49148ca7b048402a63490bf7df83a43c65d9f --- /dev/null +++ b/annotator/oneformer/detectron2/engine/defaults.py @@ -0,0 +1,715 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +""" +This file contains components with some default boilerplate logic user may need +in training / testing. They will not work for everyone, but many users may find them useful. + +The behavior of functions/classes in this file is subject to change, +since they are meant to represent the "common default behavior" people need in their projects. +""" + +import argparse +import logging +import os +import sys +import weakref +from collections import OrderedDict +from typing import Optional +import torch +from fvcore.nn.precise_bn import get_bn_modules +from omegaconf import OmegaConf +from torch.nn.parallel import DistributedDataParallel + +import annotator.oneformer.detectron2.data.transforms as T +from annotator.oneformer.detectron2.checkpoint import DetectionCheckpointer +from annotator.oneformer.detectron2.config import CfgNode, LazyConfig +from annotator.oneformer.detectron2.data import ( + MetadataCatalog, + build_detection_test_loader, + build_detection_train_loader, +) +from annotator.oneformer.detectron2.evaluation import ( + DatasetEvaluator, + inference_on_dataset, + print_csv_format, + verify_results, +) +from annotator.oneformer.detectron2.modeling import build_model +from annotator.oneformer.detectron2.solver import build_lr_scheduler, build_optimizer +from annotator.oneformer.detectron2.utils import comm +from annotator.oneformer.detectron2.utils.collect_env import collect_env_info +from annotator.oneformer.detectron2.utils.env import seed_all_rng +from annotator.oneformer.detectron2.utils.events import CommonMetricPrinter, JSONWriter, TensorboardXWriter +from annotator.oneformer.detectron2.utils.file_io import PathManager +from annotator.oneformer.detectron2.utils.logger import setup_logger + +from . import hooks +from .train_loop import AMPTrainer, SimpleTrainer, TrainerBase + +__all__ = [ + "create_ddp_model", + "default_argument_parser", + "default_setup", + "default_writers", + "DefaultPredictor", + "DefaultTrainer", +] + + +def create_ddp_model(model, *, fp16_compression=False, **kwargs): + """ + Create a DistributedDataParallel model if there are >1 processes. + + Args: + model: a torch.nn.Module + fp16_compression: add fp16 compression hooks to the ddp object. + See more at https://pytorch.org/docs/stable/ddp_comm_hooks.html#torch.distributed.algorithms.ddp_comm_hooks.default_hooks.fp16_compress_hook + kwargs: other arguments of :module:`torch.nn.parallel.DistributedDataParallel`. + """ # noqa + if comm.get_world_size() == 1: + return model + if "device_ids" not in kwargs: + kwargs["device_ids"] = [comm.get_local_rank()] + ddp = DistributedDataParallel(model, **kwargs) + if fp16_compression: + from torch.distributed.algorithms.ddp_comm_hooks import default as comm_hooks + + ddp.register_comm_hook(state=None, hook=comm_hooks.fp16_compress_hook) + return ddp + + +def default_argument_parser(epilog=None): + """ + Create a parser with some common arguments used by detectron2 users. + + Args: + epilog (str): epilog passed to ArgumentParser describing the usage. + + Returns: + argparse.ArgumentParser: + """ + parser = argparse.ArgumentParser( + epilog=epilog + or f""" +Examples: + +Run on single machine: + $ {sys.argv[0]} --num-gpus 8 --config-file cfg.yaml + +Change some config options: + $ {sys.argv[0]} --config-file cfg.yaml MODEL.WEIGHTS /path/to/weight.pth SOLVER.BASE_LR 0.001 + +Run on multiple machines: + (machine0)$ {sys.argv[0]} --machine-rank 0 --num-machines 2 --dist-url [--other-flags] + (machine1)$ {sys.argv[0]} --machine-rank 1 --num-machines 2 --dist-url [--other-flags] +""", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument("--config-file", default="", metavar="FILE", help="path to config file") + parser.add_argument( + "--resume", + action="store_true", + help="Whether to attempt to resume from the checkpoint directory. " + "See documentation of `DefaultTrainer.resume_or_load()` for what it means.", + ) + parser.add_argument("--eval-only", action="store_true", help="perform evaluation only") + parser.add_argument("--num-gpus", type=int, default=1, help="number of gpus *per machine*") + parser.add_argument("--num-machines", type=int, default=1, help="total number of machines") + parser.add_argument( + "--machine-rank", type=int, default=0, help="the rank of this machine (unique per machine)" + ) + + # PyTorch still may leave orphan processes in multi-gpu training. + # Therefore we use a deterministic way to obtain port, + # so that users are aware of orphan processes by seeing the port occupied. + port = 2**15 + 2**14 + hash(os.getuid() if sys.platform != "win32" else 1) % 2**14 + parser.add_argument( + "--dist-url", + default="tcp://127.0.0.1:{}".format(port), + help="initialization URL for pytorch distributed backend. See " + "https://pytorch.org/docs/stable/distributed.html for details.", + ) + parser.add_argument( + "opts", + help=""" +Modify config options at the end of the command. For Yacs configs, use +space-separated "PATH.KEY VALUE" pairs. +For python-based LazyConfig, use "path.key=value". + """.strip(), + default=None, + nargs=argparse.REMAINDER, + ) + return parser + + +def _try_get_key(cfg, *keys, default=None): + """ + Try select keys from cfg until the first key that exists. Otherwise return default. + """ + if isinstance(cfg, CfgNode): + cfg = OmegaConf.create(cfg.dump()) + for k in keys: + none = object() + p = OmegaConf.select(cfg, k, default=none) + if p is not none: + return p + return default + + +def _highlight(code, filename): + try: + import pygments + except ImportError: + return code + + from pygments.lexers import Python3Lexer, YamlLexer + from pygments.formatters import Terminal256Formatter + + lexer = Python3Lexer() if filename.endswith(".py") else YamlLexer() + code = pygments.highlight(code, lexer, Terminal256Formatter(style="monokai")) + return code + + +def default_setup(cfg, args): + """ + Perform some basic common setups at the beginning of a job, including: + + 1. Set up the detectron2 logger + 2. Log basic information about environment, cmdline arguments, and config + 3. Backup the config to the output directory + + Args: + cfg (CfgNode or omegaconf.DictConfig): the full config to be used + args (argparse.NameSpace): the command line arguments to be logged + """ + output_dir = _try_get_key(cfg, "OUTPUT_DIR", "output_dir", "train.output_dir") + if comm.is_main_process() and output_dir: + PathManager.mkdirs(output_dir) + + rank = comm.get_rank() + setup_logger(output_dir, distributed_rank=rank, name="fvcore") + logger = setup_logger(output_dir, distributed_rank=rank) + + logger.info("Rank of current process: {}. World size: {}".format(rank, comm.get_world_size())) + logger.info("Environment info:\n" + collect_env_info()) + + logger.info("Command line arguments: " + str(args)) + if hasattr(args, "config_file") and args.config_file != "": + logger.info( + "Contents of args.config_file={}:\n{}".format( + args.config_file, + _highlight(PathManager.open(args.config_file, "r").read(), args.config_file), + ) + ) + + if comm.is_main_process() and output_dir: + # Note: some of our scripts may expect the existence of + # config.yaml in output directory + path = os.path.join(output_dir, "config.yaml") + if isinstance(cfg, CfgNode): + logger.info("Running with full config:\n{}".format(_highlight(cfg.dump(), ".yaml"))) + with PathManager.open(path, "w") as f: + f.write(cfg.dump()) + else: + LazyConfig.save(cfg, path) + logger.info("Full config saved to {}".format(path)) + + # make sure each worker has a different, yet deterministic seed if specified + seed = _try_get_key(cfg, "SEED", "train.seed", default=-1) + seed_all_rng(None if seed < 0 else seed + rank) + + # cudnn benchmark has large overhead. It shouldn't be used considering the small size of + # typical validation set. + if not (hasattr(args, "eval_only") and args.eval_only): + torch.backends.cudnn.benchmark = _try_get_key( + cfg, "CUDNN_BENCHMARK", "train.cudnn_benchmark", default=False + ) + + +def default_writers(output_dir: str, max_iter: Optional[int] = None): + """ + Build a list of :class:`EventWriter` to be used. + It now consists of a :class:`CommonMetricPrinter`, + :class:`TensorboardXWriter` and :class:`JSONWriter`. + + Args: + output_dir: directory to store JSON metrics and tensorboard events + max_iter: the total number of iterations + + Returns: + list[EventWriter]: a list of :class:`EventWriter` objects. + """ + PathManager.mkdirs(output_dir) + return [ + # It may not always print what you want to see, since it prints "common" metrics only. + CommonMetricPrinter(max_iter), + JSONWriter(os.path.join(output_dir, "metrics.json")), + TensorboardXWriter(output_dir), + ] + + +class DefaultPredictor: + """ + Create a simple end-to-end predictor with the given config that runs on + single device for a single input image. + + Compared to using the model directly, this class does the following additions: + + 1. Load checkpoint from `cfg.MODEL.WEIGHTS`. + 2. Always take BGR image as the input and apply conversion defined by `cfg.INPUT.FORMAT`. + 3. Apply resizing defined by `cfg.INPUT.{MIN,MAX}_SIZE_TEST`. + 4. Take one input image and produce a single output, instead of a batch. + + This is meant for simple demo purposes, so it does the above steps automatically. + This is not meant for benchmarks or running complicated inference logic. + If you'd like to do anything more complicated, please refer to its source code as + examples to build and use the model manually. + + Attributes: + metadata (Metadata): the metadata of the underlying dataset, obtained from + cfg.DATASETS.TEST. + + Examples: + :: + pred = DefaultPredictor(cfg) + inputs = cv2.imread("input.jpg") + outputs = pred(inputs) + """ + + def __init__(self, cfg): + self.cfg = cfg.clone() # cfg can be modified by model + self.model = build_model(self.cfg) + self.model.eval() + if len(cfg.DATASETS.TEST): + self.metadata = MetadataCatalog.get(cfg.DATASETS.TEST[0]) + + checkpointer = DetectionCheckpointer(self.model) + checkpointer.load(cfg.MODEL.WEIGHTS) + + self.aug = T.ResizeShortestEdge( + [cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MIN_SIZE_TEST], cfg.INPUT.MAX_SIZE_TEST + ) + + self.input_format = cfg.INPUT.FORMAT + assert self.input_format in ["RGB", "BGR"], self.input_format + + def __call__(self, original_image): + """ + Args: + original_image (np.ndarray): an image of shape (H, W, C) (in BGR order). + + Returns: + predictions (dict): + the output of the model for one image only. + See :doc:`/tutorials/models` for details about the format. + """ + with torch.no_grad(): # https://github.com/sphinx-doc/sphinx/issues/4258 + # Apply pre-processing to image. + if self.input_format == "RGB": + # whether the model expects BGR inputs or RGB + original_image = original_image[:, :, ::-1] + height, width = original_image.shape[:2] + image = self.aug.get_transform(original_image).apply_image(original_image) + image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1)) + + inputs = {"image": image, "height": height, "width": width} + predictions = self.model([inputs])[0] + return predictions + + +class DefaultTrainer(TrainerBase): + """ + A trainer with default training logic. It does the following: + + 1. Create a :class:`SimpleTrainer` using model, optimizer, dataloader + defined by the given config. Create a LR scheduler defined by the config. + 2. Load the last checkpoint or `cfg.MODEL.WEIGHTS`, if exists, when + `resume_or_load` is called. + 3. Register a few common hooks defined by the config. + + It is created to simplify the **standard model training workflow** and reduce code boilerplate + for users who only need the standard training workflow, with standard features. + It means this class makes *many assumptions* about your training logic that + may easily become invalid in a new research. In fact, any assumptions beyond those made in the + :class:`SimpleTrainer` are too much for research. + + The code of this class has been annotated about restrictive assumptions it makes. + When they do not work for you, you're encouraged to: + + 1. Overwrite methods of this class, OR: + 2. Use :class:`SimpleTrainer`, which only does minimal SGD training and + nothing else. You can then add your own hooks if needed. OR: + 3. Write your own training loop similar to `tools/plain_train_net.py`. + + See the :doc:`/tutorials/training` tutorials for more details. + + Note that the behavior of this class, like other functions/classes in + this file, is not stable, since it is meant to represent the "common default behavior". + It is only guaranteed to work well with the standard models and training workflow in detectron2. + To obtain more stable behavior, write your own training logic with other public APIs. + + Examples: + :: + trainer = DefaultTrainer(cfg) + trainer.resume_or_load() # load last checkpoint or MODEL.WEIGHTS + trainer.train() + + Attributes: + scheduler: + checkpointer (DetectionCheckpointer): + cfg (CfgNode): + """ + + def __init__(self, cfg): + """ + Args: + cfg (CfgNode): + """ + super().__init__() + logger = logging.getLogger("detectron2") + if not logger.isEnabledFor(logging.INFO): # setup_logger is not called for d2 + setup_logger() + cfg = DefaultTrainer.auto_scale_workers(cfg, comm.get_world_size()) + + # Assume these objects must be constructed in this order. + model = self.build_model(cfg) + optimizer = self.build_optimizer(cfg, model) + data_loader = self.build_train_loader(cfg) + + model = create_ddp_model(model, broadcast_buffers=False) + self._trainer = (AMPTrainer if cfg.SOLVER.AMP.ENABLED else SimpleTrainer)( + model, data_loader, optimizer + ) + + self.scheduler = self.build_lr_scheduler(cfg, optimizer) + self.checkpointer = DetectionCheckpointer( + # Assume you want to save checkpoints together with logs/statistics + model, + cfg.OUTPUT_DIR, + trainer=weakref.proxy(self), + ) + self.start_iter = 0 + self.max_iter = cfg.SOLVER.MAX_ITER + self.cfg = cfg + + self.register_hooks(self.build_hooks()) + + def resume_or_load(self, resume=True): + """ + If `resume==True` and `cfg.OUTPUT_DIR` contains the last checkpoint (defined by + a `last_checkpoint` file), resume from the file. Resuming means loading all + available states (eg. optimizer and scheduler) and update iteration counter + from the checkpoint. ``cfg.MODEL.WEIGHTS`` will not be used. + + Otherwise, this is considered as an independent training. The method will load model + weights from the file `cfg.MODEL.WEIGHTS` (but will not load other states) and start + from iteration 0. + + Args: + resume (bool): whether to do resume or not + """ + self.checkpointer.resume_or_load(self.cfg.MODEL.WEIGHTS, resume=resume) + if resume and self.checkpointer.has_checkpoint(): + # The checkpoint stores the training iteration that just finished, thus we start + # at the next iteration + self.start_iter = self.iter + 1 + + def build_hooks(self): + """ + Build a list of default hooks, including timing, evaluation, + checkpointing, lr scheduling, precise BN, writing events. + + Returns: + list[HookBase]: + """ + cfg = self.cfg.clone() + cfg.defrost() + cfg.DATALOADER.NUM_WORKERS = 0 # save some memory and time for PreciseBN + + ret = [ + hooks.IterationTimer(), + hooks.LRScheduler(), + hooks.PreciseBN( + # Run at the same freq as (but before) evaluation. + cfg.TEST.EVAL_PERIOD, + self.model, + # Build a new data loader to not affect training + self.build_train_loader(cfg), + cfg.TEST.PRECISE_BN.NUM_ITER, + ) + if cfg.TEST.PRECISE_BN.ENABLED and get_bn_modules(self.model) + else None, + ] + + # Do PreciseBN before checkpointer, because it updates the model and need to + # be saved by checkpointer. + # This is not always the best: if checkpointing has a different frequency, + # some checkpoints may have more precise statistics than others. + if comm.is_main_process(): + ret.append(hooks.PeriodicCheckpointer(self.checkpointer, cfg.SOLVER.CHECKPOINT_PERIOD)) + + def test_and_save_results(): + self._last_eval_results = self.test(self.cfg, self.model) + return self._last_eval_results + + # Do evaluation after checkpointer, because then if it fails, + # we can use the saved checkpoint to debug. + ret.append(hooks.EvalHook(cfg.TEST.EVAL_PERIOD, test_and_save_results)) + + if comm.is_main_process(): + # Here the default print/log frequency of each writer is used. + # run writers in the end, so that evaluation metrics are written + ret.append(hooks.PeriodicWriter(self.build_writers(), period=20)) + return ret + + def build_writers(self): + """ + Build a list of writers to be used using :func:`default_writers()`. + If you'd like a different list of writers, you can overwrite it in + your trainer. + + Returns: + list[EventWriter]: a list of :class:`EventWriter` objects. + """ + return default_writers(self.cfg.OUTPUT_DIR, self.max_iter) + + def train(self): + """ + Run training. + + Returns: + OrderedDict of results, if evaluation is enabled. Otherwise None. + """ + super().train(self.start_iter, self.max_iter) + if len(self.cfg.TEST.EXPECTED_RESULTS) and comm.is_main_process(): + assert hasattr( + self, "_last_eval_results" + ), "No evaluation results obtained during training!" + verify_results(self.cfg, self._last_eval_results) + return self._last_eval_results + + def run_step(self): + self._trainer.iter = self.iter + self._trainer.run_step() + + def state_dict(self): + ret = super().state_dict() + ret["_trainer"] = self._trainer.state_dict() + return ret + + def load_state_dict(self, state_dict): + super().load_state_dict(state_dict) + self._trainer.load_state_dict(state_dict["_trainer"]) + + @classmethod + def build_model(cls, cfg): + """ + Returns: + torch.nn.Module: + + It now calls :func:`detectron2.modeling.build_model`. + Overwrite it if you'd like a different model. + """ + model = build_model(cfg) + logger = logging.getLogger(__name__) + logger.info("Model:\n{}".format(model)) + return model + + @classmethod + def build_optimizer(cls, cfg, model): + """ + Returns: + torch.optim.Optimizer: + + It now calls :func:`detectron2.solver.build_optimizer`. + Overwrite it if you'd like a different optimizer. + """ + return build_optimizer(cfg, model) + + @classmethod + def build_lr_scheduler(cls, cfg, optimizer): + """ + It now calls :func:`detectron2.solver.build_lr_scheduler`. + Overwrite it if you'd like a different scheduler. + """ + return build_lr_scheduler(cfg, optimizer) + + @classmethod + def build_train_loader(cls, cfg): + """ + Returns: + iterable + + It now calls :func:`detectron2.data.build_detection_train_loader`. + Overwrite it if you'd like a different data loader. + """ + return build_detection_train_loader(cfg) + + @classmethod + def build_test_loader(cls, cfg, dataset_name): + """ + Returns: + iterable + + It now calls :func:`detectron2.data.build_detection_test_loader`. + Overwrite it if you'd like a different data loader. + """ + return build_detection_test_loader(cfg, dataset_name) + + @classmethod + def build_evaluator(cls, cfg, dataset_name): + """ + Returns: + DatasetEvaluator or None + + It is not implemented by default. + """ + raise NotImplementedError( + """ +If you want DefaultTrainer to automatically run evaluation, +please implement `build_evaluator()` in subclasses (see train_net.py for example). +Alternatively, you can call evaluation functions yourself (see Colab balloon tutorial for example). +""" + ) + + @classmethod + def test(cls, cfg, model, evaluators=None): + """ + Evaluate the given model. The given model is expected to already contain + weights to evaluate. + + Args: + cfg (CfgNode): + model (nn.Module): + evaluators (list[DatasetEvaluator] or None): if None, will call + :meth:`build_evaluator`. Otherwise, must have the same length as + ``cfg.DATASETS.TEST``. + + Returns: + dict: a dict of result metrics + """ + logger = logging.getLogger(__name__) + if isinstance(evaluators, DatasetEvaluator): + evaluators = [evaluators] + if evaluators is not None: + assert len(cfg.DATASETS.TEST) == len(evaluators), "{} != {}".format( + len(cfg.DATASETS.TEST), len(evaluators) + ) + + results = OrderedDict() + for idx, dataset_name in enumerate(cfg.DATASETS.TEST): + data_loader = cls.build_test_loader(cfg, dataset_name) + # When evaluators are passed in as arguments, + # implicitly assume that evaluators can be created before data_loader. + if evaluators is not None: + evaluator = evaluators[idx] + else: + try: + evaluator = cls.build_evaluator(cfg, dataset_name) + except NotImplementedError: + logger.warn( + "No evaluator found. Use `DefaultTrainer.test(evaluators=)`, " + "or implement its `build_evaluator` method." + ) + results[dataset_name] = {} + continue + results_i = inference_on_dataset(model, data_loader, evaluator) + results[dataset_name] = results_i + if comm.is_main_process(): + assert isinstance( + results_i, dict + ), "Evaluator must return a dict on the main process. Got {} instead.".format( + results_i + ) + logger.info("Evaluation results for {} in csv format:".format(dataset_name)) + print_csv_format(results_i) + + if len(results) == 1: + results = list(results.values())[0] + return results + + @staticmethod + def auto_scale_workers(cfg, num_workers: int): + """ + When the config is defined for certain number of workers (according to + ``cfg.SOLVER.REFERENCE_WORLD_SIZE``) that's different from the number of + workers currently in use, returns a new cfg where the total batch size + is scaled so that the per-GPU batch size stays the same as the + original ``IMS_PER_BATCH // REFERENCE_WORLD_SIZE``. + + Other config options are also scaled accordingly: + * training steps and warmup steps are scaled inverse proportionally. + * learning rate are scaled proportionally, following :paper:`ImageNet in 1h`. + + For example, with the original config like the following: + + .. code-block:: yaml + + IMS_PER_BATCH: 16 + BASE_LR: 0.1 + REFERENCE_WORLD_SIZE: 8 + MAX_ITER: 5000 + STEPS: (4000,) + CHECKPOINT_PERIOD: 1000 + + When this config is used on 16 GPUs instead of the reference number 8, + calling this method will return a new config with: + + .. code-block:: yaml + + IMS_PER_BATCH: 32 + BASE_LR: 0.2 + REFERENCE_WORLD_SIZE: 16 + MAX_ITER: 2500 + STEPS: (2000,) + CHECKPOINT_PERIOD: 500 + + Note that both the original config and this new config can be trained on 16 GPUs. + It's up to user whether to enable this feature (by setting ``REFERENCE_WORLD_SIZE``). + + Returns: + CfgNode: a new config. Same as original if ``cfg.SOLVER.REFERENCE_WORLD_SIZE==0``. + """ + old_world_size = cfg.SOLVER.REFERENCE_WORLD_SIZE + if old_world_size == 0 or old_world_size == num_workers: + return cfg + cfg = cfg.clone() + frozen = cfg.is_frozen() + cfg.defrost() + + assert ( + cfg.SOLVER.IMS_PER_BATCH % old_world_size == 0 + ), "Invalid REFERENCE_WORLD_SIZE in config!" + scale = num_workers / old_world_size + bs = cfg.SOLVER.IMS_PER_BATCH = int(round(cfg.SOLVER.IMS_PER_BATCH * scale)) + lr = cfg.SOLVER.BASE_LR = cfg.SOLVER.BASE_LR * scale + max_iter = cfg.SOLVER.MAX_ITER = int(round(cfg.SOLVER.MAX_ITER / scale)) + warmup_iter = cfg.SOLVER.WARMUP_ITERS = int(round(cfg.SOLVER.WARMUP_ITERS / scale)) + cfg.SOLVER.STEPS = tuple(int(round(s / scale)) for s in cfg.SOLVER.STEPS) + cfg.TEST.EVAL_PERIOD = int(round(cfg.TEST.EVAL_PERIOD / scale)) + cfg.SOLVER.CHECKPOINT_PERIOD = int(round(cfg.SOLVER.CHECKPOINT_PERIOD / scale)) + cfg.SOLVER.REFERENCE_WORLD_SIZE = num_workers # maintain invariant + logger = logging.getLogger(__name__) + logger.info( + f"Auto-scaling the config to batch_size={bs}, learning_rate={lr}, " + f"max_iter={max_iter}, warmup={warmup_iter}." + ) + + if frozen: + cfg.freeze() + return cfg + + +# Access basic attributes from the underlying trainer +for _attr in ["model", "data_loader", "optimizer"]: + setattr( + DefaultTrainer, + _attr, + property( + # getter + lambda self, x=_attr: getattr(self._trainer, x), + # setter + lambda self, value, x=_attr: setattr(self._trainer, x, value), + ), + ) diff --git a/annotator/oneformer/detectron2/engine/hooks.py b/annotator/oneformer/detectron2/engine/hooks.py new file mode 100644 index 0000000000000000000000000000000000000000..7dd43ac77068c908bc13263f1697fa2e3332d7c9 --- /dev/null +++ b/annotator/oneformer/detectron2/engine/hooks.py @@ -0,0 +1,690 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +import datetime +import itertools +import logging +import math +import operator +import os +import tempfile +import time +import warnings +from collections import Counter +import torch +from fvcore.common.checkpoint import Checkpointer +from fvcore.common.checkpoint import PeriodicCheckpointer as _PeriodicCheckpointer +from fvcore.common.param_scheduler import ParamScheduler +from fvcore.common.timer import Timer +from fvcore.nn.precise_bn import get_bn_modules, update_bn_stats + +import annotator.oneformer.detectron2.utils.comm as comm +from annotator.oneformer.detectron2.evaluation.testing import flatten_results_dict +from annotator.oneformer.detectron2.solver import LRMultiplier +from annotator.oneformer.detectron2.solver import LRScheduler as _LRScheduler +from annotator.oneformer.detectron2.utils.events import EventStorage, EventWriter +from annotator.oneformer.detectron2.utils.file_io import PathManager + +from .train_loop import HookBase + +__all__ = [ + "CallbackHook", + "IterationTimer", + "PeriodicWriter", + "PeriodicCheckpointer", + "BestCheckpointer", + "LRScheduler", + "AutogradProfiler", + "EvalHook", + "PreciseBN", + "TorchProfiler", + "TorchMemoryStats", +] + + +""" +Implement some common hooks. +""" + + +class CallbackHook(HookBase): + """ + Create a hook using callback functions provided by the user. + """ + + def __init__(self, *, before_train=None, after_train=None, before_step=None, after_step=None): + """ + Each argument is a function that takes one argument: the trainer. + """ + self._before_train = before_train + self._before_step = before_step + self._after_step = after_step + self._after_train = after_train + + def before_train(self): + if self._before_train: + self._before_train(self.trainer) + + def after_train(self): + if self._after_train: + self._after_train(self.trainer) + # The functions may be closures that hold reference to the trainer + # Therefore, delete them to avoid circular reference. + del self._before_train, self._after_train + del self._before_step, self._after_step + + def before_step(self): + if self._before_step: + self._before_step(self.trainer) + + def after_step(self): + if self._after_step: + self._after_step(self.trainer) + + +class IterationTimer(HookBase): + """ + Track the time spent for each iteration (each run_step call in the trainer). + Print a summary in the end of training. + + This hook uses the time between the call to its :meth:`before_step` + and :meth:`after_step` methods. + Under the convention that :meth:`before_step` of all hooks should only + take negligible amount of time, the :class:`IterationTimer` hook should be + placed at the beginning of the list of hooks to obtain accurate timing. + """ + + def __init__(self, warmup_iter=3): + """ + Args: + warmup_iter (int): the number of iterations at the beginning to exclude + from timing. + """ + self._warmup_iter = warmup_iter + self._step_timer = Timer() + self._start_time = time.perf_counter() + self._total_timer = Timer() + + def before_train(self): + self._start_time = time.perf_counter() + self._total_timer.reset() + self._total_timer.pause() + + def after_train(self): + logger = logging.getLogger(__name__) + total_time = time.perf_counter() - self._start_time + total_time_minus_hooks = self._total_timer.seconds() + hook_time = total_time - total_time_minus_hooks + + num_iter = self.trainer.storage.iter + 1 - self.trainer.start_iter - self._warmup_iter + + if num_iter > 0 and total_time_minus_hooks > 0: + # Speed is meaningful only after warmup + # NOTE this format is parsed by grep in some scripts + logger.info( + "Overall training speed: {} iterations in {} ({:.4f} s / it)".format( + num_iter, + str(datetime.timedelta(seconds=int(total_time_minus_hooks))), + total_time_minus_hooks / num_iter, + ) + ) + + logger.info( + "Total training time: {} ({} on hooks)".format( + str(datetime.timedelta(seconds=int(total_time))), + str(datetime.timedelta(seconds=int(hook_time))), + ) + ) + + def before_step(self): + self._step_timer.reset() + self._total_timer.resume() + + def after_step(self): + # +1 because we're in after_step, the current step is done + # but not yet counted + iter_done = self.trainer.storage.iter - self.trainer.start_iter + 1 + if iter_done >= self._warmup_iter: + sec = self._step_timer.seconds() + self.trainer.storage.put_scalars(time=sec) + else: + self._start_time = time.perf_counter() + self._total_timer.reset() + + self._total_timer.pause() + + +class PeriodicWriter(HookBase): + """ + Write events to EventStorage (by calling ``writer.write()``) periodically. + + It is executed every ``period`` iterations and after the last iteration. + Note that ``period`` does not affect how data is smoothed by each writer. + """ + + def __init__(self, writers, period=20): + """ + Args: + writers (list[EventWriter]): a list of EventWriter objects + period (int): + """ + self._writers = writers + for w in writers: + assert isinstance(w, EventWriter), w + self._period = period + + def after_step(self): + if (self.trainer.iter + 1) % self._period == 0 or ( + self.trainer.iter == self.trainer.max_iter - 1 + ): + for writer in self._writers: + writer.write() + + def after_train(self): + for writer in self._writers: + # If any new data is found (e.g. produced by other after_train), + # write them before closing + writer.write() + writer.close() + + +class PeriodicCheckpointer(_PeriodicCheckpointer, HookBase): + """ + Same as :class:`detectron2.checkpoint.PeriodicCheckpointer`, but as a hook. + + Note that when used as a hook, + it is unable to save additional data other than what's defined + by the given `checkpointer`. + + It is executed every ``period`` iterations and after the last iteration. + """ + + def before_train(self): + self.max_iter = self.trainer.max_iter + + def after_step(self): + # No way to use **kwargs + self.step(self.trainer.iter) + + +class BestCheckpointer(HookBase): + """ + Checkpoints best weights based off given metric. + + This hook should be used in conjunction to and executed after the hook + that produces the metric, e.g. `EvalHook`. + """ + + def __init__( + self, + eval_period: int, + checkpointer: Checkpointer, + val_metric: str, + mode: str = "max", + file_prefix: str = "model_best", + ) -> None: + """ + Args: + eval_period (int): the period `EvalHook` is set to run. + checkpointer: the checkpointer object used to save checkpoints. + val_metric (str): validation metric to track for best checkpoint, e.g. "bbox/AP50" + mode (str): one of {'max', 'min'}. controls whether the chosen val metric should be + maximized or minimized, e.g. for "bbox/AP50" it should be "max" + file_prefix (str): the prefix of checkpoint's filename, defaults to "model_best" + """ + self._logger = logging.getLogger(__name__) + self._period = eval_period + self._val_metric = val_metric + assert mode in [ + "max", + "min", + ], f'Mode "{mode}" to `BestCheckpointer` is unknown. It should be one of {"max", "min"}.' + if mode == "max": + self._compare = operator.gt + else: + self._compare = operator.lt + self._checkpointer = checkpointer + self._file_prefix = file_prefix + self.best_metric = None + self.best_iter = None + + def _update_best(self, val, iteration): + if math.isnan(val) or math.isinf(val): + return False + self.best_metric = val + self.best_iter = iteration + return True + + def _best_checking(self): + metric_tuple = self.trainer.storage.latest().get(self._val_metric) + if metric_tuple is None: + self._logger.warning( + f"Given val metric {self._val_metric} does not seem to be computed/stored." + "Will not be checkpointing based on it." + ) + return + else: + latest_metric, metric_iter = metric_tuple + + if self.best_metric is None: + if self._update_best(latest_metric, metric_iter): + additional_state = {"iteration": metric_iter} + self._checkpointer.save(f"{self._file_prefix}", **additional_state) + self._logger.info( + f"Saved first model at {self.best_metric:0.5f} @ {self.best_iter} steps" + ) + elif self._compare(latest_metric, self.best_metric): + additional_state = {"iteration": metric_iter} + self._checkpointer.save(f"{self._file_prefix}", **additional_state) + self._logger.info( + f"Saved best model as latest eval score for {self._val_metric} is " + f"{latest_metric:0.5f}, better than last best score " + f"{self.best_metric:0.5f} @ iteration {self.best_iter}." + ) + self._update_best(latest_metric, metric_iter) + else: + self._logger.info( + f"Not saving as latest eval score for {self._val_metric} is {latest_metric:0.5f}, " + f"not better than best score {self.best_metric:0.5f} @ iteration {self.best_iter}." + ) + + def after_step(self): + # same conditions as `EvalHook` + next_iter = self.trainer.iter + 1 + if ( + self._period > 0 + and next_iter % self._period == 0 + and next_iter != self.trainer.max_iter + ): + self._best_checking() + + def after_train(self): + # same conditions as `EvalHook` + if self.trainer.iter + 1 >= self.trainer.max_iter: + self._best_checking() + + +class LRScheduler(HookBase): + """ + A hook which executes a torch builtin LR scheduler and summarizes the LR. + It is executed after every iteration. + """ + + def __init__(self, optimizer=None, scheduler=None): + """ + Args: + optimizer (torch.optim.Optimizer): + scheduler (torch.optim.LRScheduler or fvcore.common.param_scheduler.ParamScheduler): + if a :class:`ParamScheduler` object, it defines the multiplier over the base LR + in the optimizer. + + If any argument is not given, will try to obtain it from the trainer. + """ + self._optimizer = optimizer + self._scheduler = scheduler + + def before_train(self): + self._optimizer = self._optimizer or self.trainer.optimizer + if isinstance(self.scheduler, ParamScheduler): + self._scheduler = LRMultiplier( + self._optimizer, + self.scheduler, + self.trainer.max_iter, + last_iter=self.trainer.iter - 1, + ) + self._best_param_group_id = LRScheduler.get_best_param_group_id(self._optimizer) + + @staticmethod + def get_best_param_group_id(optimizer): + # NOTE: some heuristics on what LR to summarize + # summarize the param group with most parameters + largest_group = max(len(g["params"]) for g in optimizer.param_groups) + + if largest_group == 1: + # If all groups have one parameter, + # then find the most common initial LR, and use it for summary + lr_count = Counter([g["lr"] for g in optimizer.param_groups]) + lr = lr_count.most_common()[0][0] + for i, g in enumerate(optimizer.param_groups): + if g["lr"] == lr: + return i + else: + for i, g in enumerate(optimizer.param_groups): + if len(g["params"]) == largest_group: + return i + + def after_step(self): + lr = self._optimizer.param_groups[self._best_param_group_id]["lr"] + self.trainer.storage.put_scalar("lr", lr, smoothing_hint=False) + self.scheduler.step() + + @property + def scheduler(self): + return self._scheduler or self.trainer.scheduler + + def state_dict(self): + if isinstance(self.scheduler, _LRScheduler): + return self.scheduler.state_dict() + return {} + + def load_state_dict(self, state_dict): + if isinstance(self.scheduler, _LRScheduler): + logger = logging.getLogger(__name__) + logger.info("Loading scheduler from state_dict ...") + self.scheduler.load_state_dict(state_dict) + + +class TorchProfiler(HookBase): + """ + A hook which runs `torch.profiler.profile`. + + Examples: + :: + hooks.TorchProfiler( + lambda trainer: 10 < trainer.iter < 20, self.cfg.OUTPUT_DIR + ) + + The above example will run the profiler for iteration 10~20 and dump + results to ``OUTPUT_DIR``. We did not profile the first few iterations + because they are typically slower than the rest. + The result files can be loaded in the ``chrome://tracing`` page in chrome browser, + and the tensorboard visualizations can be visualized using + ``tensorboard --logdir OUTPUT_DIR/log`` + """ + + def __init__(self, enable_predicate, output_dir, *, activities=None, save_tensorboard=True): + """ + Args: + enable_predicate (callable[trainer -> bool]): a function which takes a trainer, + and returns whether to enable the profiler. + It will be called once every step, and can be used to select which steps to profile. + output_dir (str): the output directory to dump tracing files. + activities (iterable): same as in `torch.profiler.profile`. + save_tensorboard (bool): whether to save tensorboard visualizations at (output_dir)/log/ + """ + self._enable_predicate = enable_predicate + self._activities = activities + self._output_dir = output_dir + self._save_tensorboard = save_tensorboard + + def before_step(self): + if self._enable_predicate(self.trainer): + if self._save_tensorboard: + on_trace_ready = torch.profiler.tensorboard_trace_handler( + os.path.join( + self._output_dir, + "log", + "profiler-tensorboard-iter{}".format(self.trainer.iter), + ), + f"worker{comm.get_rank()}", + ) + else: + on_trace_ready = None + self._profiler = torch.profiler.profile( + activities=self._activities, + on_trace_ready=on_trace_ready, + record_shapes=True, + profile_memory=True, + with_stack=True, + with_flops=True, + ) + self._profiler.__enter__() + else: + self._profiler = None + + def after_step(self): + if self._profiler is None: + return + self._profiler.__exit__(None, None, None) + if not self._save_tensorboard: + PathManager.mkdirs(self._output_dir) + out_file = os.path.join( + self._output_dir, "profiler-trace-iter{}.json".format(self.trainer.iter) + ) + if "://" not in out_file: + self._profiler.export_chrome_trace(out_file) + else: + # Support non-posix filesystems + with tempfile.TemporaryDirectory(prefix="detectron2_profiler") as d: + tmp_file = os.path.join(d, "tmp.json") + self._profiler.export_chrome_trace(tmp_file) + with open(tmp_file) as f: + content = f.read() + with PathManager.open(out_file, "w") as f: + f.write(content) + + +class AutogradProfiler(TorchProfiler): + """ + A hook which runs `torch.autograd.profiler.profile`. + + Examples: + :: + hooks.AutogradProfiler( + lambda trainer: 10 < trainer.iter < 20, self.cfg.OUTPUT_DIR + ) + + The above example will run the profiler for iteration 10~20 and dump + results to ``OUTPUT_DIR``. We did not profile the first few iterations + because they are typically slower than the rest. + The result files can be loaded in the ``chrome://tracing`` page in chrome browser. + + Note: + When used together with NCCL on older version of GPUs, + autograd profiler may cause deadlock because it unnecessarily allocates + memory on every device it sees. The memory management calls, if + interleaved with NCCL calls, lead to deadlock on GPUs that do not + support ``cudaLaunchCooperativeKernelMultiDevice``. + """ + + def __init__(self, enable_predicate, output_dir, *, use_cuda=True): + """ + Args: + enable_predicate (callable[trainer -> bool]): a function which takes a trainer, + and returns whether to enable the profiler. + It will be called once every step, and can be used to select which steps to profile. + output_dir (str): the output directory to dump tracing files. + use_cuda (bool): same as in `torch.autograd.profiler.profile`. + """ + warnings.warn("AutogradProfiler has been deprecated in favor of TorchProfiler.") + self._enable_predicate = enable_predicate + self._use_cuda = use_cuda + self._output_dir = output_dir + + def before_step(self): + if self._enable_predicate(self.trainer): + self._profiler = torch.autograd.profiler.profile(use_cuda=self._use_cuda) + self._profiler.__enter__() + else: + self._profiler = None + + +class EvalHook(HookBase): + """ + Run an evaluation function periodically, and at the end of training. + + It is executed every ``eval_period`` iterations and after the last iteration. + """ + + def __init__(self, eval_period, eval_function, eval_after_train=True): + """ + Args: + eval_period (int): the period to run `eval_function`. Set to 0 to + not evaluate periodically (but still evaluate after the last iteration + if `eval_after_train` is True). + eval_function (callable): a function which takes no arguments, and + returns a nested dict of evaluation metrics. + eval_after_train (bool): whether to evaluate after the last iteration + + Note: + This hook must be enabled in all or none workers. + If you would like only certain workers to perform evaluation, + give other workers a no-op function (`eval_function=lambda: None`). + """ + self._period = eval_period + self._func = eval_function + self._eval_after_train = eval_after_train + + def _do_eval(self): + results = self._func() + + if results: + assert isinstance( + results, dict + ), "Eval function must return a dict. Got {} instead.".format(results) + + flattened_results = flatten_results_dict(results) + for k, v in flattened_results.items(): + try: + v = float(v) + except Exception as e: + raise ValueError( + "[EvalHook] eval_function should return a nested dict of float. " + "Got '{}: {}' instead.".format(k, v) + ) from e + self.trainer.storage.put_scalars(**flattened_results, smoothing_hint=False) + + # Evaluation may take different time among workers. + # A barrier make them start the next iteration together. + comm.synchronize() + + def after_step(self): + next_iter = self.trainer.iter + 1 + if self._period > 0 and next_iter % self._period == 0: + # do the last eval in after_train + if next_iter != self.trainer.max_iter: + self._do_eval() + + def after_train(self): + # This condition is to prevent the eval from running after a failed training + if self._eval_after_train and self.trainer.iter + 1 >= self.trainer.max_iter: + self._do_eval() + # func is likely a closure that holds reference to the trainer + # therefore we clean it to avoid circular reference in the end + del self._func + + +class PreciseBN(HookBase): + """ + The standard implementation of BatchNorm uses EMA in inference, which is + sometimes suboptimal. + This class computes the true average of statistics rather than the moving average, + and put true averages to every BN layer in the given model. + + It is executed every ``period`` iterations and after the last iteration. + """ + + def __init__(self, period, model, data_loader, num_iter): + """ + Args: + period (int): the period this hook is run, or 0 to not run during training. + The hook will always run in the end of training. + model (nn.Module): a module whose all BN layers in training mode will be + updated by precise BN. + Note that user is responsible for ensuring the BN layers to be + updated are in training mode when this hook is triggered. + data_loader (iterable): it will produce data to be run by `model(data)`. + num_iter (int): number of iterations used to compute the precise + statistics. + """ + self._logger = logging.getLogger(__name__) + if len(get_bn_modules(model)) == 0: + self._logger.info( + "PreciseBN is disabled because model does not contain BN layers in training mode." + ) + self._disabled = True + return + + self._model = model + self._data_loader = data_loader + self._num_iter = num_iter + self._period = period + self._disabled = False + + self._data_iter = None + + def after_step(self): + next_iter = self.trainer.iter + 1 + is_final = next_iter == self.trainer.max_iter + if is_final or (self._period > 0 and next_iter % self._period == 0): + self.update_stats() + + def update_stats(self): + """ + Update the model with precise statistics. Users can manually call this method. + """ + if self._disabled: + return + + if self._data_iter is None: + self._data_iter = iter(self._data_loader) + + def data_loader(): + for num_iter in itertools.count(1): + if num_iter % 100 == 0: + self._logger.info( + "Running precise-BN ... {}/{} iterations.".format(num_iter, self._num_iter) + ) + # This way we can reuse the same iterator + yield next(self._data_iter) + + with EventStorage(): # capture events in a new storage to discard them + self._logger.info( + "Running precise-BN for {} iterations... ".format(self._num_iter) + + "Note that this could produce different statistics every time." + ) + update_bn_stats(self._model, data_loader(), self._num_iter) + + +class TorchMemoryStats(HookBase): + """ + Writes pytorch's cuda memory statistics periodically. + """ + + def __init__(self, period=20, max_runs=10): + """ + Args: + period (int): Output stats each 'period' iterations + max_runs (int): Stop the logging after 'max_runs' + """ + + self._logger = logging.getLogger(__name__) + self._period = period + self._max_runs = max_runs + self._runs = 0 + + def after_step(self): + if self._runs > self._max_runs: + return + + if (self.trainer.iter + 1) % self._period == 0 or ( + self.trainer.iter == self.trainer.max_iter - 1 + ): + if torch.cuda.is_available(): + max_reserved_mb = torch.cuda.max_memory_reserved() / 1024.0 / 1024.0 + reserved_mb = torch.cuda.memory_reserved() / 1024.0 / 1024.0 + max_allocated_mb = torch.cuda.max_memory_allocated() / 1024.0 / 1024.0 + allocated_mb = torch.cuda.memory_allocated() / 1024.0 / 1024.0 + + self._logger.info( + ( + " iter: {} " + " max_reserved_mem: {:.0f}MB " + " reserved_mem: {:.0f}MB " + " max_allocated_mem: {:.0f}MB " + " allocated_mem: {:.0f}MB " + ).format( + self.trainer.iter, + max_reserved_mb, + reserved_mb, + max_allocated_mb, + allocated_mb, + ) + ) + + self._runs += 1 + if self._runs == self._max_runs: + mem_summary = torch.cuda.memory_summary() + self._logger.info("\n" + mem_summary) + + torch.cuda.reset_peak_memory_stats() diff --git a/annotator/oneformer/detectron2/engine/launch.py b/annotator/oneformer/detectron2/engine/launch.py new file mode 100644 index 0000000000000000000000000000000000000000..0a2d6bcdb5f1906d3eedb04b5aa939f8269f0344 --- /dev/null +++ b/annotator/oneformer/detectron2/engine/launch.py @@ -0,0 +1,123 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +from datetime import timedelta +import torch +import torch.distributed as dist +import torch.multiprocessing as mp + +from annotator.oneformer.detectron2.utils import comm + +__all__ = ["DEFAULT_TIMEOUT", "launch"] + +DEFAULT_TIMEOUT = timedelta(minutes=30) + + +def _find_free_port(): + import socket + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # Binding to port 0 will cause the OS to find an available port for us + sock.bind(("", 0)) + port = sock.getsockname()[1] + sock.close() + # NOTE: there is still a chance the port could be taken by other processes. + return port + + +def launch( + main_func, + # Should be num_processes_per_machine, but kept for compatibility. + num_gpus_per_machine, + num_machines=1, + machine_rank=0, + dist_url=None, + args=(), + timeout=DEFAULT_TIMEOUT, +): + """ + Launch multi-process or distributed training. + This function must be called on all machines involved in the training. + It will spawn child processes (defined by ``num_gpus_per_machine``) on each machine. + + Args: + main_func: a function that will be called by `main_func(*args)` + num_gpus_per_machine (int): number of processes per machine. When + using GPUs, this should be the number of GPUs. + num_machines (int): the total number of machines + machine_rank (int): the rank of this machine + dist_url (str): url to connect to for distributed jobs, including protocol + e.g. "tcp://127.0.0.1:8686". + Can be set to "auto" to automatically select a free port on localhost + timeout (timedelta): timeout of the distributed workers + args (tuple): arguments passed to main_func + """ + world_size = num_machines * num_gpus_per_machine + if world_size > 1: + # https://github.com/pytorch/pytorch/pull/14391 + # TODO prctl in spawned processes + + if dist_url == "auto": + assert num_machines == 1, "dist_url=auto not supported in multi-machine jobs." + port = _find_free_port() + dist_url = f"tcp://127.0.0.1:{port}" + if num_machines > 1 and dist_url.startswith("file://"): + logger = logging.getLogger(__name__) + logger.warning( + "file:// is not a reliable init_method in multi-machine jobs. Prefer tcp://" + ) + + mp.start_processes( + _distributed_worker, + nprocs=num_gpus_per_machine, + args=( + main_func, + world_size, + num_gpus_per_machine, + machine_rank, + dist_url, + args, + timeout, + ), + daemon=False, + ) + else: + main_func(*args) + + +def _distributed_worker( + local_rank, + main_func, + world_size, + num_gpus_per_machine, + machine_rank, + dist_url, + args, + timeout=DEFAULT_TIMEOUT, +): + has_gpu = torch.cuda.is_available() + if has_gpu: + assert num_gpus_per_machine <= torch.cuda.device_count() + global_rank = machine_rank * num_gpus_per_machine + local_rank + try: + dist.init_process_group( + backend="NCCL" if has_gpu else "GLOO", + init_method=dist_url, + world_size=world_size, + rank=global_rank, + timeout=timeout, + ) + except Exception as e: + logger = logging.getLogger(__name__) + logger.error("Process group URL: {}".format(dist_url)) + raise e + + # Setup the local process group. + comm.create_local_process_group(num_gpus_per_machine) + if has_gpu: + torch.cuda.set_device(local_rank) + + # synchronize is needed here to prevent a possible timeout after calling init_process_group + # See: https://github.com/facebookresearch/maskrcnn-benchmark/issues/172 + comm.synchronize() + + main_func(*args) diff --git a/annotator/oneformer/detectron2/engine/train_loop.py b/annotator/oneformer/detectron2/engine/train_loop.py new file mode 100644 index 0000000000000000000000000000000000000000..0c24c5af94e8f9367a5d577a617ec426292d3f89 --- /dev/null +++ b/annotator/oneformer/detectron2/engine/train_loop.py @@ -0,0 +1,469 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +import logging +import numpy as np +import time +import weakref +from typing import List, Mapping, Optional +import torch +from torch.nn.parallel import DataParallel, DistributedDataParallel + +import annotator.oneformer.detectron2.utils.comm as comm +from annotator.oneformer.detectron2.utils.events import EventStorage, get_event_storage +from annotator.oneformer.detectron2.utils.logger import _log_api_usage + +__all__ = ["HookBase", "TrainerBase", "SimpleTrainer", "AMPTrainer"] + + +class HookBase: + """ + Base class for hooks that can be registered with :class:`TrainerBase`. + + Each hook can implement 4 methods. The way they are called is demonstrated + in the following snippet: + :: + hook.before_train() + for iter in range(start_iter, max_iter): + hook.before_step() + trainer.run_step() + hook.after_step() + iter += 1 + hook.after_train() + + Notes: + 1. In the hook method, users can access ``self.trainer`` to access more + properties about the context (e.g., model, current iteration, or config + if using :class:`DefaultTrainer`). + + 2. A hook that does something in :meth:`before_step` can often be + implemented equivalently in :meth:`after_step`. + If the hook takes non-trivial time, it is strongly recommended to + implement the hook in :meth:`after_step` instead of :meth:`before_step`. + The convention is that :meth:`before_step` should only take negligible time. + + Following this convention will allow hooks that do care about the difference + between :meth:`before_step` and :meth:`after_step` (e.g., timer) to + function properly. + + """ + + trainer: "TrainerBase" = None + """ + A weak reference to the trainer object. Set by the trainer when the hook is registered. + """ + + def before_train(self): + """ + Called before the first iteration. + """ + pass + + def after_train(self): + """ + Called after the last iteration. + """ + pass + + def before_step(self): + """ + Called before each iteration. + """ + pass + + def after_backward(self): + """ + Called after the backward pass of each iteration. + """ + pass + + def after_step(self): + """ + Called after each iteration. + """ + pass + + def state_dict(self): + """ + Hooks are stateless by default, but can be made checkpointable by + implementing `state_dict` and `load_state_dict`. + """ + return {} + + +class TrainerBase: + """ + Base class for iterative trainer with hooks. + + The only assumption we made here is: the training runs in a loop. + A subclass can implement what the loop is. + We made no assumptions about the existence of dataloader, optimizer, model, etc. + + Attributes: + iter(int): the current iteration. + + start_iter(int): The iteration to start with. + By convention the minimum possible value is 0. + + max_iter(int): The iteration to end training. + + storage(EventStorage): An EventStorage that's opened during the course of training. + """ + + def __init__(self) -> None: + self._hooks: List[HookBase] = [] + self.iter: int = 0 + self.start_iter: int = 0 + self.max_iter: int + self.storage: EventStorage + _log_api_usage("trainer." + self.__class__.__name__) + + def register_hooks(self, hooks: List[Optional[HookBase]]) -> None: + """ + Register hooks to the trainer. The hooks are executed in the order + they are registered. + + Args: + hooks (list[Optional[HookBase]]): list of hooks + """ + hooks = [h for h in hooks if h is not None] + for h in hooks: + assert isinstance(h, HookBase) + # To avoid circular reference, hooks and trainer cannot own each other. + # This normally does not matter, but will cause memory leak if the + # involved objects contain __del__: + # See http://engineering.hearsaysocial.com/2013/06/16/circular-references-in-python/ + h.trainer = weakref.proxy(self) + self._hooks.extend(hooks) + + def train(self, start_iter: int, max_iter: int): + """ + Args: + start_iter, max_iter (int): See docs above + """ + logger = logging.getLogger(__name__) + logger.info("Starting training from iteration {}".format(start_iter)) + + self.iter = self.start_iter = start_iter + self.max_iter = max_iter + + with EventStorage(start_iter) as self.storage: + try: + self.before_train() + for self.iter in range(start_iter, max_iter): + self.before_step() + self.run_step() + self.after_step() + # self.iter == max_iter can be used by `after_train` to + # tell whether the training successfully finished or failed + # due to exceptions. + self.iter += 1 + except Exception: + logger.exception("Exception during training:") + raise + finally: + self.after_train() + + def before_train(self): + for h in self._hooks: + h.before_train() + + def after_train(self): + self.storage.iter = self.iter + for h in self._hooks: + h.after_train() + + def before_step(self): + # Maintain the invariant that storage.iter == trainer.iter + # for the entire execution of each step + self.storage.iter = self.iter + + for h in self._hooks: + h.before_step() + + def after_backward(self): + for h in self._hooks: + h.after_backward() + + def after_step(self): + for h in self._hooks: + h.after_step() + + def run_step(self): + raise NotImplementedError + + def state_dict(self): + ret = {"iteration": self.iter} + hooks_state = {} + for h in self._hooks: + sd = h.state_dict() + if sd: + name = type(h).__qualname__ + if name in hooks_state: + # TODO handle repetitive stateful hooks + continue + hooks_state[name] = sd + if hooks_state: + ret["hooks"] = hooks_state + return ret + + def load_state_dict(self, state_dict): + logger = logging.getLogger(__name__) + self.iter = state_dict["iteration"] + for key, value in state_dict.get("hooks", {}).items(): + for h in self._hooks: + try: + name = type(h).__qualname__ + except AttributeError: + continue + if name == key: + h.load_state_dict(value) + break + else: + logger.warning(f"Cannot find the hook '{key}', its state_dict is ignored.") + + +class SimpleTrainer(TrainerBase): + """ + A simple trainer for the most common type of task: + single-cost single-optimizer single-data-source iterative optimization, + optionally using data-parallelism. + It assumes that every step, you: + + 1. Compute the loss with a data from the data_loader. + 2. Compute the gradients with the above loss. + 3. Update the model with the optimizer. + + All other tasks during training (checkpointing, logging, evaluation, LR schedule) + are maintained by hooks, which can be registered by :meth:`TrainerBase.register_hooks`. + + If you want to do anything fancier than this, + either subclass TrainerBase and implement your own `run_step`, + or write your own training loop. + """ + + def __init__(self, model, data_loader, optimizer, gather_metric_period=1): + """ + Args: + model: a torch Module. Takes a data from data_loader and returns a + dict of losses. + data_loader: an iterable. Contains data to be used to call model. + optimizer: a torch optimizer. + gather_metric_period: an int. Every gather_metric_period iterations + the metrics are gathered from all the ranks to rank 0 and logged. + """ + super().__init__() + + """ + We set the model to training mode in the trainer. + However it's valid to train a model that's in eval mode. + If you want your model (or a submodule of it) to behave + like evaluation during training, you can overwrite its train() method. + """ + model.train() + + self.model = model + self.data_loader = data_loader + # to access the data loader iterator, call `self._data_loader_iter` + self._data_loader_iter_obj = None + self.optimizer = optimizer + self.gather_metric_period = gather_metric_period + + def run_step(self): + """ + Implement the standard training logic described above. + """ + assert self.model.training, "[SimpleTrainer] model was changed to eval mode!" + start = time.perf_counter() + """ + If you want to do something with the data, you can wrap the dataloader. + """ + data = next(self._data_loader_iter) + data_time = time.perf_counter() - start + + """ + If you want to do something with the losses, you can wrap the model. + """ + loss_dict = self.model(data) + if isinstance(loss_dict, torch.Tensor): + losses = loss_dict + loss_dict = {"total_loss": loss_dict} + else: + losses = sum(loss_dict.values()) + + """ + If you need to accumulate gradients or do something similar, you can + wrap the optimizer with your custom `zero_grad()` method. + """ + self.optimizer.zero_grad() + losses.backward() + + self.after_backward() + + self._write_metrics(loss_dict, data_time) + + """ + If you need gradient clipping/scaling or other processing, you can + wrap the optimizer with your custom `step()` method. But it is + suboptimal as explained in https://arxiv.org/abs/2006.15704 Sec 3.2.4 + """ + self.optimizer.step() + + @property + def _data_loader_iter(self): + # only create the data loader iterator when it is used + if self._data_loader_iter_obj is None: + self._data_loader_iter_obj = iter(self.data_loader) + return self._data_loader_iter_obj + + def reset_data_loader(self, data_loader_builder): + """ + Delete and replace the current data loader with a new one, which will be created + by calling `data_loader_builder` (without argument). + """ + del self.data_loader + data_loader = data_loader_builder() + self.data_loader = data_loader + self._data_loader_iter_obj = None + + def _write_metrics( + self, + loss_dict: Mapping[str, torch.Tensor], + data_time: float, + prefix: str = "", + ) -> None: + if (self.iter + 1) % self.gather_metric_period == 0: + SimpleTrainer.write_metrics(loss_dict, data_time, prefix) + + @staticmethod + def write_metrics( + loss_dict: Mapping[str, torch.Tensor], + data_time: float, + prefix: str = "", + ) -> None: + """ + Args: + loss_dict (dict): dict of scalar losses + data_time (float): time taken by the dataloader iteration + prefix (str): prefix for logging keys + """ + metrics_dict = {k: v.detach().cpu().item() for k, v in loss_dict.items()} + metrics_dict["data_time"] = data_time + + # Gather metrics among all workers for logging + # This assumes we do DDP-style training, which is currently the only + # supported method in detectron2. + all_metrics_dict = comm.gather(metrics_dict) + + if comm.is_main_process(): + storage = get_event_storage() + + # data_time among workers can have high variance. The actual latency + # caused by data_time is the maximum among workers. + data_time = np.max([x.pop("data_time") for x in all_metrics_dict]) + storage.put_scalar("data_time", data_time) + + # average the rest metrics + metrics_dict = { + k: np.mean([x[k] for x in all_metrics_dict]) for k in all_metrics_dict[0].keys() + } + total_losses_reduced = sum(metrics_dict.values()) + if not np.isfinite(total_losses_reduced): + raise FloatingPointError( + f"Loss became infinite or NaN at iteration={storage.iter}!\n" + f"loss_dict = {metrics_dict}" + ) + + storage.put_scalar("{}total_loss".format(prefix), total_losses_reduced) + if len(metrics_dict) > 1: + storage.put_scalars(**metrics_dict) + + def state_dict(self): + ret = super().state_dict() + ret["optimizer"] = self.optimizer.state_dict() + return ret + + def load_state_dict(self, state_dict): + super().load_state_dict(state_dict) + self.optimizer.load_state_dict(state_dict["optimizer"]) + + +class AMPTrainer(SimpleTrainer): + """ + Like :class:`SimpleTrainer`, but uses PyTorch's native automatic mixed precision + in the training loop. + """ + + def __init__( + self, + model, + data_loader, + optimizer, + gather_metric_period=1, + grad_scaler=None, + precision: torch.dtype = torch.float16, + log_grad_scaler: bool = False, + ): + """ + Args: + model, data_loader, optimizer, gather_metric_period: same as in :class:`SimpleTrainer`. + grad_scaler: torch GradScaler to automatically scale gradients. + precision: torch.dtype as the target precision to cast to in computations + """ + unsupported = "AMPTrainer does not support single-process multi-device training!" + if isinstance(model, DistributedDataParallel): + assert not (model.device_ids and len(model.device_ids) > 1), unsupported + assert not isinstance(model, DataParallel), unsupported + + super().__init__(model, data_loader, optimizer, gather_metric_period) + + if grad_scaler is None: + from torch.cuda.amp import GradScaler + + grad_scaler = GradScaler() + self.grad_scaler = grad_scaler + self.precision = precision + self.log_grad_scaler = log_grad_scaler + + def run_step(self): + """ + Implement the AMP training logic. + """ + assert self.model.training, "[AMPTrainer] model was changed to eval mode!" + assert torch.cuda.is_available(), "[AMPTrainer] CUDA is required for AMP training!" + from torch.cuda.amp import autocast + + start = time.perf_counter() + data = next(self._data_loader_iter) + data_time = time.perf_counter() - start + + with autocast(dtype=self.precision): + loss_dict = self.model(data) + if isinstance(loss_dict, torch.Tensor): + losses = loss_dict + loss_dict = {"total_loss": loss_dict} + else: + losses = sum(loss_dict.values()) + + self.optimizer.zero_grad() + self.grad_scaler.scale(losses).backward() + + if self.log_grad_scaler: + storage = get_event_storage() + storage.put_scalar("[metric]grad_scaler", self.grad_scaler.get_scale()) + + self.after_backward() + + self._write_metrics(loss_dict, data_time) + + self.grad_scaler.step(self.optimizer) + self.grad_scaler.update() + + def state_dict(self): + ret = super().state_dict() + ret["grad_scaler"] = self.grad_scaler.state_dict() + return ret + + def load_state_dict(self, state_dict): + super().load_state_dict(state_dict) + self.grad_scaler.load_state_dict(state_dict["grad_scaler"]) diff --git a/annotator/oneformer/detectron2/evaluation/__init__.py b/annotator/oneformer/detectron2/evaluation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d96609e8f2261a6800fe85fcf3e1eaeaa44455c6 --- /dev/null +++ b/annotator/oneformer/detectron2/evaluation/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .cityscapes_evaluation import CityscapesInstanceEvaluator, CityscapesSemSegEvaluator +from .coco_evaluation import COCOEvaluator +from .rotated_coco_evaluation import RotatedCOCOEvaluator +from .evaluator import DatasetEvaluator, DatasetEvaluators, inference_context, inference_on_dataset +from .lvis_evaluation import LVISEvaluator +from .panoptic_evaluation import COCOPanopticEvaluator +from .pascal_voc_evaluation import PascalVOCDetectionEvaluator +from .sem_seg_evaluation import SemSegEvaluator +from .testing import print_csv_format, verify_results + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/annotator/oneformer/detectron2/evaluation/cityscapes_evaluation.py b/annotator/oneformer/detectron2/evaluation/cityscapes_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..f5be637dc87b5ca8645563a4a921144f6c5fd877 --- /dev/null +++ b/annotator/oneformer/detectron2/evaluation/cityscapes_evaluation.py @@ -0,0 +1,197 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import glob +import logging +import numpy as np +import os +import tempfile +from collections import OrderedDict +import torch +from PIL import Image + +from annotator.oneformer.detectron2.data import MetadataCatalog +from annotator.oneformer.detectron2.utils import comm +from annotator.oneformer.detectron2.utils.file_io import PathManager + +from .evaluator import DatasetEvaluator + + +class CityscapesEvaluator(DatasetEvaluator): + """ + Base class for evaluation using cityscapes API. + """ + + def __init__(self, dataset_name): + """ + Args: + dataset_name (str): the name of the dataset. + It must have the following metadata associated with it: + "thing_classes", "gt_dir". + """ + self._metadata = MetadataCatalog.get(dataset_name) + self._cpu_device = torch.device("cpu") + self._logger = logging.getLogger(__name__) + + def reset(self): + self._working_dir = tempfile.TemporaryDirectory(prefix="cityscapes_eval_") + self._temp_dir = self._working_dir.name + # All workers will write to the same results directory + # TODO this does not work in distributed training + assert ( + comm.get_local_size() == comm.get_world_size() + ), "CityscapesEvaluator currently do not work with multiple machines." + self._temp_dir = comm.all_gather(self._temp_dir)[0] + if self._temp_dir != self._working_dir.name: + self._working_dir.cleanup() + self._logger.info( + "Writing cityscapes results to temporary directory {} ...".format(self._temp_dir) + ) + + +class CityscapesInstanceEvaluator(CityscapesEvaluator): + """ + Evaluate instance segmentation results on cityscapes dataset using cityscapes API. + + Note: + * It does not work in multi-machine distributed training. + * It contains a synchronization, therefore has to be used on all ranks. + * Only the main process runs evaluation. + """ + + def process(self, inputs, outputs): + from cityscapesscripts.helpers.labels import name2label + + for input, output in zip(inputs, outputs): + file_name = input["file_name"] + basename = os.path.splitext(os.path.basename(file_name))[0] + pred_txt = os.path.join(self._temp_dir, basename + "_pred.txt") + + if "instances" in output: + output = output["instances"].to(self._cpu_device) + num_instances = len(output) + with open(pred_txt, "w") as fout: + for i in range(num_instances): + pred_class = output.pred_classes[i] + classes = self._metadata.thing_classes[pred_class] + class_id = name2label[classes].id + score = output.scores[i] + mask = output.pred_masks[i].numpy().astype("uint8") + png_filename = os.path.join( + self._temp_dir, basename + "_{}_{}.png".format(i, classes) + ) + + Image.fromarray(mask * 255).save(png_filename) + fout.write( + "{} {} {}\n".format(os.path.basename(png_filename), class_id, score) + ) + else: + # Cityscapes requires a prediction file for every ground truth image. + with open(pred_txt, "w") as fout: + pass + + def evaluate(self): + """ + Returns: + dict: has a key "segm", whose value is a dict of "AP" and "AP50". + """ + comm.synchronize() + if comm.get_rank() > 0: + return + import cityscapesscripts.evaluation.evalInstanceLevelSemanticLabeling as cityscapes_eval + + self._logger.info("Evaluating results under {} ...".format(self._temp_dir)) + + # set some global states in cityscapes evaluation API, before evaluating + cityscapes_eval.args.predictionPath = os.path.abspath(self._temp_dir) + cityscapes_eval.args.predictionWalk = None + cityscapes_eval.args.JSONOutput = False + cityscapes_eval.args.colorized = False + cityscapes_eval.args.gtInstancesFile = os.path.join(self._temp_dir, "gtInstances.json") + + # These lines are adopted from + # https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/evaluation/evalInstanceLevelSemanticLabeling.py # noqa + gt_dir = PathManager.get_local_path(self._metadata.gt_dir) + groundTruthImgList = glob.glob(os.path.join(gt_dir, "*", "*_gtFine_instanceIds.png")) + assert len( + groundTruthImgList + ), "Cannot find any ground truth images to use for evaluation. Searched for: {}".format( + cityscapes_eval.args.groundTruthSearch + ) + predictionImgList = [] + for gt in groundTruthImgList: + predictionImgList.append(cityscapes_eval.getPrediction(gt, cityscapes_eval.args)) + results = cityscapes_eval.evaluateImgLists( + predictionImgList, groundTruthImgList, cityscapes_eval.args + )["averages"] + + ret = OrderedDict() + ret["segm"] = {"AP": results["allAp"] * 100, "AP50": results["allAp50%"] * 100} + self._working_dir.cleanup() + return ret + + +class CityscapesSemSegEvaluator(CityscapesEvaluator): + """ + Evaluate semantic segmentation results on cityscapes dataset using cityscapes API. + + Note: + * It does not work in multi-machine distributed training. + * It contains a synchronization, therefore has to be used on all ranks. + * Only the main process runs evaluation. + """ + + def process(self, inputs, outputs): + from cityscapesscripts.helpers.labels import trainId2label + + for input, output in zip(inputs, outputs): + file_name = input["file_name"] + basename = os.path.splitext(os.path.basename(file_name))[0] + pred_filename = os.path.join(self._temp_dir, basename + "_pred.png") + + output = output["sem_seg"].argmax(dim=0).to(self._cpu_device).numpy() + pred = 255 * np.ones(output.shape, dtype=np.uint8) + for train_id, label in trainId2label.items(): + if label.ignoreInEval: + continue + pred[output == train_id] = label.id + Image.fromarray(pred).save(pred_filename) + + def evaluate(self): + comm.synchronize() + if comm.get_rank() > 0: + return + # Load the Cityscapes eval script *after* setting the required env var, + # since the script reads CITYSCAPES_DATASET into global variables at load time. + import cityscapesscripts.evaluation.evalPixelLevelSemanticLabeling as cityscapes_eval + + self._logger.info("Evaluating results under {} ...".format(self._temp_dir)) + + # set some global states in cityscapes evaluation API, before evaluating + cityscapes_eval.args.predictionPath = os.path.abspath(self._temp_dir) + cityscapes_eval.args.predictionWalk = None + cityscapes_eval.args.JSONOutput = False + cityscapes_eval.args.colorized = False + + # These lines are adopted from + # https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/evaluation/evalPixelLevelSemanticLabeling.py # noqa + gt_dir = PathManager.get_local_path(self._metadata.gt_dir) + groundTruthImgList = glob.glob(os.path.join(gt_dir, "*", "*_gtFine_labelIds.png")) + assert len( + groundTruthImgList + ), "Cannot find any ground truth images to use for evaluation. Searched for: {}".format( + cityscapes_eval.args.groundTruthSearch + ) + predictionImgList = [] + for gt in groundTruthImgList: + predictionImgList.append(cityscapes_eval.getPrediction(cityscapes_eval.args, gt)) + results = cityscapes_eval.evaluateImgLists( + predictionImgList, groundTruthImgList, cityscapes_eval.args + ) + ret = OrderedDict() + ret["sem_seg"] = { + "IoU": 100.0 * results["averageScoreClasses"], + "iIoU": 100.0 * results["averageScoreInstClasses"], + "IoU_sup": 100.0 * results["averageScoreCategories"], + "iIoU_sup": 100.0 * results["averageScoreInstCategories"], + } + self._working_dir.cleanup() + return ret diff --git a/annotator/oneformer/detectron2/evaluation/coco_evaluation.py b/annotator/oneformer/detectron2/evaluation/coco_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..1eef5ce6f688a749cfa35a389f6599f10df79c22 --- /dev/null +++ b/annotator/oneformer/detectron2/evaluation/coco_evaluation.py @@ -0,0 +1,722 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import contextlib +import copy +import io +import itertools +import json +import logging +import numpy as np +import os +import pickle +from collections import OrderedDict +import pycocotools.mask as mask_util +import torch +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval +from tabulate import tabulate + +import annotator.oneformer.detectron2.utils.comm as comm +from annotator.oneformer.detectron2.config import CfgNode +from annotator.oneformer.detectron2.data import MetadataCatalog +from annotator.oneformer.detectron2.data.datasets.coco import convert_to_coco_json +from annotator.oneformer.detectron2.structures import Boxes, BoxMode, pairwise_iou +from annotator.oneformer.detectron2.utils.file_io import PathManager +from annotator.oneformer.detectron2.utils.logger import create_small_table + +from .evaluator import DatasetEvaluator + +try: + from annotator.oneformer.detectron2.evaluation.fast_eval_api import COCOeval_opt +except ImportError: + COCOeval_opt = COCOeval + + +class COCOEvaluator(DatasetEvaluator): + """ + Evaluate AR for object proposals, AP for instance detection/segmentation, AP + for keypoint detection outputs using COCO's metrics. + See http://cocodataset.org/#detection-eval and + http://cocodataset.org/#keypoints-eval to understand its metrics. + The metrics range from 0 to 100 (instead of 0 to 1), where a -1 or NaN means + the metric cannot be computed (e.g. due to no predictions made). + + In addition to COCO, this evaluator is able to support any bounding box detection, + instance segmentation, or keypoint detection dataset. + """ + + def __init__( + self, + dataset_name, + tasks=None, + distributed=True, + output_dir=None, + *, + max_dets_per_image=None, + use_fast_impl=True, + kpt_oks_sigmas=(), + allow_cached_coco=True, + ): + """ + Args: + dataset_name (str): name of the dataset to be evaluated. + It must have either the following corresponding metadata: + + "json_file": the path to the COCO format annotation + + Or it must be in detectron2's standard dataset format + so it can be converted to COCO format automatically. + tasks (tuple[str]): tasks that can be evaluated under the given + configuration. A task is one of "bbox", "segm", "keypoints". + By default, will infer this automatically from predictions. + distributed (True): if True, will collect results from all ranks and run evaluation + in the main process. + Otherwise, will only evaluate the results in the current process. + output_dir (str): optional, an output directory to dump all + results predicted on the dataset. The dump contains two files: + + 1. "instances_predictions.pth" a file that can be loaded with `torch.load` and + contains all the results in the format they are produced by the model. + 2. "coco_instances_results.json" a json file in COCO's result format. + max_dets_per_image (int): limit on the maximum number of detections per image. + By default in COCO, this limit is to 100, but this can be customized + to be greater, as is needed in evaluation metrics AP fixed and AP pool + (see https://arxiv.org/pdf/2102.01066.pdf) + This doesn't affect keypoint evaluation. + use_fast_impl (bool): use a fast but **unofficial** implementation to compute AP. + Although the results should be very close to the official implementation in COCO + API, it is still recommended to compute results with the official API for use in + papers. The faster implementation also uses more RAM. + kpt_oks_sigmas (list[float]): The sigmas used to calculate keypoint OKS. + See http://cocodataset.org/#keypoints-eval + When empty, it will use the defaults in COCO. + Otherwise it should be the same length as ROI_KEYPOINT_HEAD.NUM_KEYPOINTS. + allow_cached_coco (bool): Whether to use cached coco json from previous validation + runs. You should set this to False if you need to use different validation data. + Defaults to True. + """ + self._logger = logging.getLogger(__name__) + self._distributed = distributed + self._output_dir = output_dir + + if use_fast_impl and (COCOeval_opt is COCOeval): + self._logger.info("Fast COCO eval is not built. Falling back to official COCO eval.") + use_fast_impl = False + self._use_fast_impl = use_fast_impl + + # COCOeval requires the limit on the number of detections per image (maxDets) to be a list + # with at least 3 elements. The default maxDets in COCOeval is [1, 10, 100], in which the + # 3rd element (100) is used as the limit on the number of detections per image when + # evaluating AP. COCOEvaluator expects an integer for max_dets_per_image, so for COCOeval, + # we reformat max_dets_per_image into [1, 10, max_dets_per_image], based on the defaults. + if max_dets_per_image is None: + max_dets_per_image = [1, 10, 100] + else: + max_dets_per_image = [1, 10, max_dets_per_image] + self._max_dets_per_image = max_dets_per_image + + if tasks is not None and isinstance(tasks, CfgNode): + kpt_oks_sigmas = ( + tasks.TEST.KEYPOINT_OKS_SIGMAS if not kpt_oks_sigmas else kpt_oks_sigmas + ) + self._logger.warn( + "COCO Evaluator instantiated using config, this is deprecated behavior." + " Please pass in explicit arguments instead." + ) + self._tasks = None # Infering it from predictions should be better + else: + self._tasks = tasks + + self._cpu_device = torch.device("cpu") + + self._metadata = MetadataCatalog.get(dataset_name) + if not hasattr(self._metadata, "json_file"): + if output_dir is None: + raise ValueError( + "output_dir must be provided to COCOEvaluator " + "for datasets not in COCO format." + ) + self._logger.info(f"Trying to convert '{dataset_name}' to COCO format ...") + + cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") + self._metadata.json_file = cache_path + convert_to_coco_json(dataset_name, cache_path, allow_cached=allow_cached_coco) + + json_file = PathManager.get_local_path(self._metadata.json_file) + with contextlib.redirect_stdout(io.StringIO()): + self._coco_api = COCO(json_file) + + # Test set json files do not contain annotations (evaluation must be + # performed using the COCO evaluation server). + self._do_evaluation = "annotations" in self._coco_api.dataset + if self._do_evaluation: + self._kpt_oks_sigmas = kpt_oks_sigmas + + def reset(self): + self._predictions = [] + + def process(self, inputs, outputs): + """ + Args: + inputs: the inputs to a COCO model (e.g., GeneralizedRCNN). + It is a list of dict. Each dict corresponds to an image and + contains keys like "height", "width", "file_name", "image_id". + outputs: the outputs of a COCO model. It is a list of dicts with key + "instances" that contains :class:`Instances`. + """ + for input, output in zip(inputs, outputs): + prediction = {"image_id": input["image_id"]} + + if "instances" in output: + instances = output["instances"].to(self._cpu_device) + prediction["instances"] = instances_to_coco_json(instances, input["image_id"]) + if "proposals" in output: + prediction["proposals"] = output["proposals"].to(self._cpu_device) + if len(prediction) > 1: + self._predictions.append(prediction) + + def evaluate(self, img_ids=None): + """ + Args: + img_ids: a list of image IDs to evaluate on. Default to None for the whole dataset + """ + if self._distributed: + comm.synchronize() + predictions = comm.gather(self._predictions, dst=0) + predictions = list(itertools.chain(*predictions)) + + if not comm.is_main_process(): + return {} + else: + predictions = self._predictions + + if len(predictions) == 0: + self._logger.warning("[COCOEvaluator] Did not receive valid predictions.") + return {} + + if self._output_dir: + PathManager.mkdirs(self._output_dir) + file_path = os.path.join(self._output_dir, "instances_predictions.pth") + with PathManager.open(file_path, "wb") as f: + torch.save(predictions, f) + + self._results = OrderedDict() + if "proposals" in predictions[0]: + self._eval_box_proposals(predictions) + if "instances" in predictions[0]: + self._eval_predictions(predictions, img_ids=img_ids) + # Copy so the caller can do whatever with results + return copy.deepcopy(self._results) + + def _tasks_from_predictions(self, predictions): + """ + Get COCO API "tasks" (i.e. iou_type) from COCO-format predictions. + """ + tasks = {"bbox"} + for pred in predictions: + if "segmentation" in pred: + tasks.add("segm") + if "keypoints" in pred: + tasks.add("keypoints") + return sorted(tasks) + + def _eval_predictions(self, predictions, img_ids=None): + """ + Evaluate predictions. Fill self._results with the metrics of the tasks. + """ + self._logger.info("Preparing results for COCO format ...") + coco_results = list(itertools.chain(*[x["instances"] for x in predictions])) + tasks = self._tasks or self._tasks_from_predictions(coco_results) + + # unmap the category ids for COCO + if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"): + dataset_id_to_contiguous_id = self._metadata.thing_dataset_id_to_contiguous_id + all_contiguous_ids = list(dataset_id_to_contiguous_id.values()) + num_classes = len(all_contiguous_ids) + assert min(all_contiguous_ids) == 0 and max(all_contiguous_ids) == num_classes - 1 + + reverse_id_mapping = {v: k for k, v in dataset_id_to_contiguous_id.items()} + for result in coco_results: + category_id = result["category_id"] + assert category_id < num_classes, ( + f"A prediction has class={category_id}, " + f"but the dataset only has {num_classes} classes and " + f"predicted class id should be in [0, {num_classes - 1}]." + ) + result["category_id"] = reverse_id_mapping[category_id] + + if self._output_dir: + file_path = os.path.join(self._output_dir, "coco_instances_results.json") + self._logger.info("Saving results to {}".format(file_path)) + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(coco_results)) + f.flush() + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info( + "Evaluating predictions with {} COCO API...".format( + "unofficial" if self._use_fast_impl else "official" + ) + ) + for task in sorted(tasks): + assert task in {"bbox", "segm", "keypoints"}, f"Got unknown task: {task}!" + coco_eval = ( + _evaluate_predictions_on_coco( + self._coco_api, + coco_results, + task, + kpt_oks_sigmas=self._kpt_oks_sigmas, + cocoeval_fn=COCOeval_opt if self._use_fast_impl else COCOeval, + img_ids=img_ids, + max_dets_per_image=self._max_dets_per_image, + ) + if len(coco_results) > 0 + else None # cocoapi does not handle empty results very well + ) + + res = self._derive_coco_results( + coco_eval, task, class_names=self._metadata.get("thing_classes") + ) + self._results[task] = res + + def _eval_box_proposals(self, predictions): + """ + Evaluate the box proposals in predictions. + Fill self._results with the metrics for "box_proposals" task. + """ + if self._output_dir: + # Saving generated box proposals to file. + # Predicted box_proposals are in XYXY_ABS mode. + bbox_mode = BoxMode.XYXY_ABS.value + ids, boxes, objectness_logits = [], [], [] + for prediction in predictions: + ids.append(prediction["image_id"]) + boxes.append(prediction["proposals"].proposal_boxes.tensor.numpy()) + objectness_logits.append(prediction["proposals"].objectness_logits.numpy()) + + proposal_data = { + "boxes": boxes, + "objectness_logits": objectness_logits, + "ids": ids, + "bbox_mode": bbox_mode, + } + with PathManager.open(os.path.join(self._output_dir, "box_proposals.pkl"), "wb") as f: + pickle.dump(proposal_data, f) + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info("Evaluating bbox proposals ...") + res = {} + areas = {"all": "", "small": "s", "medium": "m", "large": "l"} + for limit in [100, 1000]: + for area, suffix in areas.items(): + stats = _evaluate_box_proposals(predictions, self._coco_api, area=area, limit=limit) + key = "AR{}@{:d}".format(suffix, limit) + res[key] = float(stats["ar"].item() * 100) + self._logger.info("Proposal metrics: \n" + create_small_table(res)) + self._results["box_proposals"] = res + + def _derive_coco_results(self, coco_eval, iou_type, class_names=None): + """ + Derive the desired score numbers from summarized COCOeval. + + Args: + coco_eval (None or COCOEval): None represents no predictions from model. + iou_type (str): + class_names (None or list[str]): if provided, will use it to predict + per-category AP. + + Returns: + a dict of {metric name: score} + """ + + metrics = { + "bbox": ["AP", "AP50", "AP75", "APs", "APm", "APl"], + "segm": ["AP", "AP50", "AP75", "APs", "APm", "APl"], + "keypoints": ["AP", "AP50", "AP75", "APm", "APl"], + }[iou_type] + + if coco_eval is None: + self._logger.warn("No predictions from the model!") + return {metric: float("nan") for metric in metrics} + + # the standard metrics + results = { + metric: float(coco_eval.stats[idx] * 100 if coco_eval.stats[idx] >= 0 else "nan") + for idx, metric in enumerate(metrics) + } + self._logger.info( + "Evaluation results for {}: \n".format(iou_type) + create_small_table(results) + ) + if not np.isfinite(sum(results.values())): + self._logger.info("Some metrics cannot be computed and is shown as NaN.") + + if class_names is None or len(class_names) <= 1: + return results + # Compute per-category AP + # from https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L222-L252 # noqa + precisions = coco_eval.eval["precision"] + # precision has dims (iou, recall, cls, area range, max dets) + assert len(class_names) == precisions.shape[2] + + results_per_category = [] + for idx, name in enumerate(class_names): + # area range index 0: all area ranges + # max dets index -1: typically 100 per image + precision = precisions[:, :, idx, 0, -1] + precision = precision[precision > -1] + ap = np.mean(precision) if precision.size else float("nan") + results_per_category.append(("{}".format(name), float(ap * 100))) + + # tabulate it + N_COLS = min(6, len(results_per_category) * 2) + results_flatten = list(itertools.chain(*results_per_category)) + results_2d = itertools.zip_longest(*[results_flatten[i::N_COLS] for i in range(N_COLS)]) + table = tabulate( + results_2d, + tablefmt="pipe", + floatfmt=".3f", + headers=["category", "AP"] * (N_COLS // 2), + numalign="left", + ) + self._logger.info("Per-category {} AP: \n".format(iou_type) + table) + + results.update({"AP-" + name: ap for name, ap in results_per_category}) + return results + + +def instances_to_coco_json(instances, img_id): + """ + Dump an "Instances" object to a COCO-format json that's used for evaluation. + + Args: + instances (Instances): + img_id (int): the image id + + Returns: + list[dict]: list of json annotations in COCO format. + """ + num_instance = len(instances) + if num_instance == 0: + return [] + + boxes = instances.pred_boxes.tensor.numpy() + boxes = BoxMode.convert(boxes, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS) + boxes = boxes.tolist() + scores = instances.scores.tolist() + classes = instances.pred_classes.tolist() + + has_mask = instances.has("pred_masks") + if has_mask: + # use RLE to encode the masks, because they are too large and takes memory + # since this evaluator stores outputs of the entire dataset + rles = [ + mask_util.encode(np.array(mask[:, :, None], order="F", dtype="uint8"))[0] + for mask in instances.pred_masks + ] + for rle in rles: + # "counts" is an array encoded by mask_util as a byte-stream. Python3's + # json writer which always produces strings cannot serialize a bytestream + # unless you decode it. Thankfully, utf-8 works out (which is also what + # the pycocotools/_mask.pyx does). + rle["counts"] = rle["counts"].decode("utf-8") + + has_keypoints = instances.has("pred_keypoints") + if has_keypoints: + keypoints = instances.pred_keypoints + + results = [] + for k in range(num_instance): + result = { + "image_id": img_id, + "category_id": classes[k], + "bbox": boxes[k], + "score": scores[k], + } + if has_mask: + result["segmentation"] = rles[k] + if has_keypoints: + # In COCO annotations, + # keypoints coordinates are pixel indices. + # However our predictions are floating point coordinates. + # Therefore we subtract 0.5 to be consistent with the annotation format. + # This is the inverse of data loading logic in `datasets/coco.py`. + keypoints[k][:, :2] -= 0.5 + result["keypoints"] = keypoints[k].flatten().tolist() + results.append(result) + return results + + +# inspired from Detectron: +# https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L255 # noqa +def _evaluate_box_proposals(dataset_predictions, coco_api, thresholds=None, area="all", limit=None): + """ + Evaluate detection proposal recall metrics. This function is a much + faster alternative to the official COCO API recall evaluation code. However, + it produces slightly different results. + """ + # Record max overlap value for each gt box + # Return vector of overlap values + areas = { + "all": 0, + "small": 1, + "medium": 2, + "large": 3, + "96-128": 4, + "128-256": 5, + "256-512": 6, + "512-inf": 7, + } + area_ranges = [ + [0**2, 1e5**2], # all + [0**2, 32**2], # small + [32**2, 96**2], # medium + [96**2, 1e5**2], # large + [96**2, 128**2], # 96-128 + [128**2, 256**2], # 128-256 + [256**2, 512**2], # 256-512 + [512**2, 1e5**2], + ] # 512-inf + assert area in areas, "Unknown area range: {}".format(area) + area_range = area_ranges[areas[area]] + gt_overlaps = [] + num_pos = 0 + + for prediction_dict in dataset_predictions: + predictions = prediction_dict["proposals"] + + # sort predictions in descending order + # TODO maybe remove this and make it explicit in the documentation + inds = predictions.objectness_logits.sort(descending=True)[1] + predictions = predictions[inds] + + ann_ids = coco_api.getAnnIds(imgIds=prediction_dict["image_id"]) + anno = coco_api.loadAnns(ann_ids) + gt_boxes = [ + BoxMode.convert(obj["bbox"], BoxMode.XYWH_ABS, BoxMode.XYXY_ABS) + for obj in anno + if obj["iscrowd"] == 0 + ] + gt_boxes = torch.as_tensor(gt_boxes).reshape(-1, 4) # guard against no boxes + gt_boxes = Boxes(gt_boxes) + gt_areas = torch.as_tensor([obj["area"] for obj in anno if obj["iscrowd"] == 0]) + + if len(gt_boxes) == 0 or len(predictions) == 0: + continue + + valid_gt_inds = (gt_areas >= area_range[0]) & (gt_areas <= area_range[1]) + gt_boxes = gt_boxes[valid_gt_inds] + + num_pos += len(gt_boxes) + + if len(gt_boxes) == 0: + continue + + if limit is not None and len(predictions) > limit: + predictions = predictions[:limit] + + overlaps = pairwise_iou(predictions.proposal_boxes, gt_boxes) + + _gt_overlaps = torch.zeros(len(gt_boxes)) + for j in range(min(len(predictions), len(gt_boxes))): + # find which proposal box maximally covers each gt box + # and get the iou amount of coverage for each gt box + max_overlaps, argmax_overlaps = overlaps.max(dim=0) + + # find which gt box is 'best' covered (i.e. 'best' = most iou) + gt_ovr, gt_ind = max_overlaps.max(dim=0) + assert gt_ovr >= 0 + # find the proposal box that covers the best covered gt box + box_ind = argmax_overlaps[gt_ind] + # record the iou coverage of this gt box + _gt_overlaps[j] = overlaps[box_ind, gt_ind] + assert _gt_overlaps[j] == gt_ovr + # mark the proposal box and the gt box as used + overlaps[box_ind, :] = -1 + overlaps[:, gt_ind] = -1 + + # append recorded iou coverage level + gt_overlaps.append(_gt_overlaps) + gt_overlaps = ( + torch.cat(gt_overlaps, dim=0) if len(gt_overlaps) else torch.zeros(0, dtype=torch.float32) + ) + gt_overlaps, _ = torch.sort(gt_overlaps) + + if thresholds is None: + step = 0.05 + thresholds = torch.arange(0.5, 0.95 + 1e-5, step, dtype=torch.float32) + recalls = torch.zeros_like(thresholds) + # compute recall for each iou threshold + for i, t in enumerate(thresholds): + recalls[i] = (gt_overlaps >= t).float().sum() / float(num_pos) + # ar = 2 * np.trapz(recalls, thresholds) + ar = recalls.mean() + return { + "ar": ar, + "recalls": recalls, + "thresholds": thresholds, + "gt_overlaps": gt_overlaps, + "num_pos": num_pos, + } + + +def _evaluate_predictions_on_coco( + coco_gt, + coco_results, + iou_type, + kpt_oks_sigmas=None, + cocoeval_fn=COCOeval_opt, + img_ids=None, + max_dets_per_image=None, +): + """ + Evaluate the coco results using COCOEval API. + """ + assert len(coco_results) > 0 + + if iou_type == "segm": + coco_results = copy.deepcopy(coco_results) + # When evaluating mask AP, if the results contain bbox, cocoapi will + # use the box area as the area of the instance, instead of the mask area. + # This leads to a different definition of small/medium/large. + # We remove the bbox field to let mask AP use mask area. + for c in coco_results: + c.pop("bbox", None) + + coco_dt = coco_gt.loadRes(coco_results) + coco_eval = cocoeval_fn(coco_gt, coco_dt, iou_type) + # For COCO, the default max_dets_per_image is [1, 10, 100]. + if max_dets_per_image is None: + max_dets_per_image = [1, 10, 100] # Default from COCOEval + else: + assert ( + len(max_dets_per_image) >= 3 + ), "COCOeval requires maxDets (and max_dets_per_image) to have length at least 3" + # In the case that user supplies a custom input for max_dets_per_image, + # apply COCOevalMaxDets to evaluate AP with the custom input. + if max_dets_per_image[2] != 100: + coco_eval = COCOevalMaxDets(coco_gt, coco_dt, iou_type) + if iou_type != "keypoints": + coco_eval.params.maxDets = max_dets_per_image + + if img_ids is not None: + coco_eval.params.imgIds = img_ids + + if iou_type == "keypoints": + # Use the COCO default keypoint OKS sigmas unless overrides are specified + if kpt_oks_sigmas: + assert hasattr(coco_eval.params, "kpt_oks_sigmas"), "pycocotools is too old!" + coco_eval.params.kpt_oks_sigmas = np.array(kpt_oks_sigmas) + # COCOAPI requires every detection and every gt to have keypoints, so + # we just take the first entry from both + num_keypoints_dt = len(coco_results[0]["keypoints"]) // 3 + num_keypoints_gt = len(next(iter(coco_gt.anns.values()))["keypoints"]) // 3 + num_keypoints_oks = len(coco_eval.params.kpt_oks_sigmas) + assert num_keypoints_oks == num_keypoints_dt == num_keypoints_gt, ( + f"[COCOEvaluator] Prediction contain {num_keypoints_dt} keypoints. " + f"Ground truth contains {num_keypoints_gt} keypoints. " + f"The length of cfg.TEST.KEYPOINT_OKS_SIGMAS is {num_keypoints_oks}. " + "They have to agree with each other. For meaning of OKS, please refer to " + "http://cocodataset.org/#keypoints-eval." + ) + + coco_eval.evaluate() + coco_eval.accumulate() + coco_eval.summarize() + + return coco_eval + + +class COCOevalMaxDets(COCOeval): + """ + Modified version of COCOeval for evaluating AP with a custom + maxDets (by default for COCO, maxDets is 100) + """ + + def summarize(self): + """ + Compute and display summary metrics for evaluation results given + a custom value for max_dets_per_image + """ + + def _summarize(ap=1, iouThr=None, areaRng="all", maxDets=100): + p = self.params + iStr = " {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}" + titleStr = "Average Precision" if ap == 1 else "Average Recall" + typeStr = "(AP)" if ap == 1 else "(AR)" + iouStr = ( + "{:0.2f}:{:0.2f}".format(p.iouThrs[0], p.iouThrs[-1]) + if iouThr is None + else "{:0.2f}".format(iouThr) + ) + + aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng] + mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets] + if ap == 1: + # dimension of precision: [TxRxKxAxM] + s = self.eval["precision"] + # IoU + if iouThr is not None: + t = np.where(iouThr == p.iouThrs)[0] + s = s[t] + s = s[:, :, :, aind, mind] + else: + # dimension of recall: [TxKxAxM] + s = self.eval["recall"] + if iouThr is not None: + t = np.where(iouThr == p.iouThrs)[0] + s = s[t] + s = s[:, :, aind, mind] + if len(s[s > -1]) == 0: + mean_s = -1 + else: + mean_s = np.mean(s[s > -1]) + print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s)) + return mean_s + + def _summarizeDets(): + stats = np.zeros((12,)) + # Evaluate AP using the custom limit on maximum detections per image + stats[0] = _summarize(1, maxDets=self.params.maxDets[2]) + stats[1] = _summarize(1, iouThr=0.5, maxDets=self.params.maxDets[2]) + stats[2] = _summarize(1, iouThr=0.75, maxDets=self.params.maxDets[2]) + stats[3] = _summarize(1, areaRng="small", maxDets=self.params.maxDets[2]) + stats[4] = _summarize(1, areaRng="medium", maxDets=self.params.maxDets[2]) + stats[5] = _summarize(1, areaRng="large", maxDets=self.params.maxDets[2]) + stats[6] = _summarize(0, maxDets=self.params.maxDets[0]) + stats[7] = _summarize(0, maxDets=self.params.maxDets[1]) + stats[8] = _summarize(0, maxDets=self.params.maxDets[2]) + stats[9] = _summarize(0, areaRng="small", maxDets=self.params.maxDets[2]) + stats[10] = _summarize(0, areaRng="medium", maxDets=self.params.maxDets[2]) + stats[11] = _summarize(0, areaRng="large", maxDets=self.params.maxDets[2]) + return stats + + def _summarizeKps(): + stats = np.zeros((10,)) + stats[0] = _summarize(1, maxDets=20) + stats[1] = _summarize(1, maxDets=20, iouThr=0.5) + stats[2] = _summarize(1, maxDets=20, iouThr=0.75) + stats[3] = _summarize(1, maxDets=20, areaRng="medium") + stats[4] = _summarize(1, maxDets=20, areaRng="large") + stats[5] = _summarize(0, maxDets=20) + stats[6] = _summarize(0, maxDets=20, iouThr=0.5) + stats[7] = _summarize(0, maxDets=20, iouThr=0.75) + stats[8] = _summarize(0, maxDets=20, areaRng="medium") + stats[9] = _summarize(0, maxDets=20, areaRng="large") + return stats + + if not self.eval: + raise Exception("Please run accumulate() first") + iouType = self.params.iouType + if iouType == "segm" or iouType == "bbox": + summarize = _summarizeDets + elif iouType == "keypoints": + summarize = _summarizeKps + self.stats = summarize() + + def __str__(self): + self.summarize() diff --git a/annotator/oneformer/detectron2/evaluation/evaluator.py b/annotator/oneformer/detectron2/evaluation/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..9cddc296432cbb6f11caf3c3be98833a50778ffb --- /dev/null +++ b/annotator/oneformer/detectron2/evaluation/evaluator.py @@ -0,0 +1,224 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import datetime +import logging +import time +from collections import OrderedDict, abc +from contextlib import ExitStack, contextmanager +from typing import List, Union +import torch +from torch import nn + +from annotator.oneformer.detectron2.utils.comm import get_world_size, is_main_process +from annotator.oneformer.detectron2.utils.logger import log_every_n_seconds + + +class DatasetEvaluator: + """ + Base class for a dataset evaluator. + + The function :func:`inference_on_dataset` runs the model over + all samples in the dataset, and have a DatasetEvaluator to process the inputs/outputs. + + This class will accumulate information of the inputs/outputs (by :meth:`process`), + and produce evaluation results in the end (by :meth:`evaluate`). + """ + + def reset(self): + """ + Preparation for a new round of evaluation. + Should be called before starting a round of evaluation. + """ + pass + + def process(self, inputs, outputs): + """ + Process the pair of inputs and outputs. + If they contain batches, the pairs can be consumed one-by-one using `zip`: + + .. code-block:: python + + for input_, output in zip(inputs, outputs): + # do evaluation on single input/output pair + ... + + Args: + inputs (list): the inputs that's used to call the model. + outputs (list): the return value of `model(inputs)` + """ + pass + + def evaluate(self): + """ + Evaluate/summarize the performance, after processing all input/output pairs. + + Returns: + dict: + A new evaluator class can return a dict of arbitrary format + as long as the user can process the results. + In our train_net.py, we expect the following format: + + * key: the name of the task (e.g., bbox) + * value: a dict of {metric name: score}, e.g.: {"AP50": 80} + """ + pass + + +class DatasetEvaluators(DatasetEvaluator): + """ + Wrapper class to combine multiple :class:`DatasetEvaluator` instances. + + This class dispatches every evaluation call to + all of its :class:`DatasetEvaluator`. + """ + + def __init__(self, evaluators): + """ + Args: + evaluators (list): the evaluators to combine. + """ + super().__init__() + self._evaluators = evaluators + + def reset(self): + for evaluator in self._evaluators: + evaluator.reset() + + def process(self, inputs, outputs): + for evaluator in self._evaluators: + evaluator.process(inputs, outputs) + + def evaluate(self): + results = OrderedDict() + for evaluator in self._evaluators: + result = evaluator.evaluate() + if is_main_process() and result is not None: + for k, v in result.items(): + assert ( + k not in results + ), "Different evaluators produce results with the same key {}".format(k) + results[k] = v + return results + + +def inference_on_dataset( + model, data_loader, evaluator: Union[DatasetEvaluator, List[DatasetEvaluator], None] +): + """ + Run model on the data_loader and evaluate the metrics with evaluator. + Also benchmark the inference speed of `model.__call__` accurately. + The model will be used in eval mode. + + Args: + model (callable): a callable which takes an object from + `data_loader` and returns some outputs. + + If it's an nn.Module, it will be temporarily set to `eval` mode. + If you wish to evaluate a model in `training` mode instead, you can + wrap the given model and override its behavior of `.eval()` and `.train()`. + data_loader: an iterable object with a length. + The elements it generates will be the inputs to the model. + evaluator: the evaluator(s) to run. Use `None` if you only want to benchmark, + but don't want to do any evaluation. + + Returns: + The return value of `evaluator.evaluate()` + """ + num_devices = get_world_size() + logger = logging.getLogger(__name__) + logger.info("Start inference on {} batches".format(len(data_loader))) + + total = len(data_loader) # inference data loader must have a fixed length + if evaluator is None: + # create a no-op evaluator + evaluator = DatasetEvaluators([]) + if isinstance(evaluator, abc.MutableSequence): + evaluator = DatasetEvaluators(evaluator) + evaluator.reset() + + num_warmup = min(5, total - 1) + start_time = time.perf_counter() + total_data_time = 0 + total_compute_time = 0 + total_eval_time = 0 + with ExitStack() as stack: + if isinstance(model, nn.Module): + stack.enter_context(inference_context(model)) + stack.enter_context(torch.no_grad()) + + start_data_time = time.perf_counter() + for idx, inputs in enumerate(data_loader): + total_data_time += time.perf_counter() - start_data_time + if idx == num_warmup: + start_time = time.perf_counter() + total_data_time = 0 + total_compute_time = 0 + total_eval_time = 0 + + start_compute_time = time.perf_counter() + outputs = model(inputs) + if torch.cuda.is_available(): + torch.cuda.synchronize() + total_compute_time += time.perf_counter() - start_compute_time + + start_eval_time = time.perf_counter() + evaluator.process(inputs, outputs) + total_eval_time += time.perf_counter() - start_eval_time + + iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup) + data_seconds_per_iter = total_data_time / iters_after_start + compute_seconds_per_iter = total_compute_time / iters_after_start + eval_seconds_per_iter = total_eval_time / iters_after_start + total_seconds_per_iter = (time.perf_counter() - start_time) / iters_after_start + if idx >= num_warmup * 2 or compute_seconds_per_iter > 5: + eta = datetime.timedelta(seconds=int(total_seconds_per_iter * (total - idx - 1))) + log_every_n_seconds( + logging.INFO, + ( + f"Inference done {idx + 1}/{total}. " + f"Dataloading: {data_seconds_per_iter:.4f} s/iter. " + f"Inference: {compute_seconds_per_iter:.4f} s/iter. " + f"Eval: {eval_seconds_per_iter:.4f} s/iter. " + f"Total: {total_seconds_per_iter:.4f} s/iter. " + f"ETA={eta}" + ), + n=5, + ) + start_data_time = time.perf_counter() + + # Measure the time only for this worker (before the synchronization barrier) + total_time = time.perf_counter() - start_time + total_time_str = str(datetime.timedelta(seconds=total_time)) + # NOTE this format is parsed by grep + logger.info( + "Total inference time: {} ({:.6f} s / iter per device, on {} devices)".format( + total_time_str, total_time / (total - num_warmup), num_devices + ) + ) + total_compute_time_str = str(datetime.timedelta(seconds=int(total_compute_time))) + logger.info( + "Total inference pure compute time: {} ({:.6f} s / iter per device, on {} devices)".format( + total_compute_time_str, total_compute_time / (total - num_warmup), num_devices + ) + ) + + results = evaluator.evaluate() + # An evaluator may return None when not in main process. + # Replace it by an empty dict instead to make it easier for downstream code to handle + if results is None: + results = {} + return results + + +@contextmanager +def inference_context(model): + """ + A context where the model is temporarily changed to eval mode, + and restored to previous mode afterwards. + + Args: + model: a torch Module + """ + training_mode = model.training + model.eval() + yield + model.train(training_mode) diff --git a/annotator/oneformer/detectron2/evaluation/fast_eval_api.py b/annotator/oneformer/detectron2/evaluation/fast_eval_api.py new file mode 100644 index 0000000000000000000000000000000000000000..75458b1cf8c26500da9b6e60cb6224a3c26d6dd2 --- /dev/null +++ b/annotator/oneformer/detectron2/evaluation/fast_eval_api.py @@ -0,0 +1,121 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +import logging +import numpy as np +import time +from pycocotools.cocoeval import COCOeval + +from annotator.oneformer.detectron2 import _C + +logger = logging.getLogger(__name__) + + +class COCOeval_opt(COCOeval): + """ + This is a slightly modified version of the original COCO API, where the functions evaluateImg() + and accumulate() are implemented in C++ to speedup evaluation + """ + + def evaluate(self): + """ + Run per image evaluation on given images and store results in self.evalImgs_cpp, a + datastructure that isn't readable from Python but is used by a c++ implementation of + accumulate(). Unlike the original COCO PythonAPI, we don't populate the datastructure + self.evalImgs because this datastructure is a computational bottleneck. + :return: None + """ + tic = time.time() + + p = self.params + # add backward compatibility if useSegm is specified in params + if p.useSegm is not None: + p.iouType = "segm" if p.useSegm == 1 else "bbox" + logger.info("Evaluate annotation type *{}*".format(p.iouType)) + p.imgIds = list(np.unique(p.imgIds)) + if p.useCats: + p.catIds = list(np.unique(p.catIds)) + p.maxDets = sorted(p.maxDets) + self.params = p + + self._prepare() # bottleneck + + # loop through images, area range, max detection number + catIds = p.catIds if p.useCats else [-1] + + if p.iouType == "segm" or p.iouType == "bbox": + computeIoU = self.computeIoU + elif p.iouType == "keypoints": + computeIoU = self.computeOks + self.ious = { + (imgId, catId): computeIoU(imgId, catId) for imgId in p.imgIds for catId in catIds + } # bottleneck + + maxDet = p.maxDets[-1] + + # <<<< Beginning of code differences with original COCO API + def convert_instances_to_cpp(instances, is_det=False): + # Convert annotations for a list of instances in an image to a format that's fast + # to access in C++ + instances_cpp = [] + for instance in instances: + instance_cpp = _C.InstanceAnnotation( + int(instance["id"]), + instance["score"] if is_det else instance.get("score", 0.0), + instance["area"], + bool(instance.get("iscrowd", 0)), + bool(instance.get("ignore", 0)), + ) + instances_cpp.append(instance_cpp) + return instances_cpp + + # Convert GT annotations, detections, and IOUs to a format that's fast to access in C++ + ground_truth_instances = [ + [convert_instances_to_cpp(self._gts[imgId, catId]) for catId in p.catIds] + for imgId in p.imgIds + ] + detected_instances = [ + [convert_instances_to_cpp(self._dts[imgId, catId], is_det=True) for catId in p.catIds] + for imgId in p.imgIds + ] + ious = [[self.ious[imgId, catId] for catId in catIds] for imgId in p.imgIds] + + if not p.useCats: + # For each image, flatten per-category lists into a single list + ground_truth_instances = [[[o for c in i for o in c]] for i in ground_truth_instances] + detected_instances = [[[o for c in i for o in c]] for i in detected_instances] + + # Call C++ implementation of self.evaluateImgs() + self._evalImgs_cpp = _C.COCOevalEvaluateImages( + p.areaRng, maxDet, p.iouThrs, ious, ground_truth_instances, detected_instances + ) + self._evalImgs = None + + self._paramsEval = copy.deepcopy(self.params) + toc = time.time() + logger.info("COCOeval_opt.evaluate() finished in {:0.2f} seconds.".format(toc - tic)) + # >>>> End of code differences with original COCO API + + def accumulate(self): + """ + Accumulate per image evaluation results and store the result in self.eval. Does not + support changing parameter settings from those used by self.evaluate() + """ + logger.info("Accumulating evaluation results...") + tic = time.time() + assert hasattr( + self, "_evalImgs_cpp" + ), "evaluate() must be called before accmulate() is called." + + self.eval = _C.COCOevalAccumulate(self._paramsEval, self._evalImgs_cpp) + + # recall is num_iou_thresholds X num_categories X num_area_ranges X num_max_detections + self.eval["recall"] = np.array(self.eval["recall"]).reshape( + self.eval["counts"][:1] + self.eval["counts"][2:] + ) + + # precision and scores are num_iou_thresholds X num_recall_thresholds X num_categories X + # num_area_ranges X num_max_detections + self.eval["precision"] = np.array(self.eval["precision"]).reshape(self.eval["counts"]) + self.eval["scores"] = np.array(self.eval["scores"]).reshape(self.eval["counts"]) + toc = time.time() + logger.info("COCOeval_opt.accumulate() finished in {:0.2f} seconds.".format(toc - tic)) diff --git a/annotator/oneformer/detectron2/evaluation/lvis_evaluation.py b/annotator/oneformer/detectron2/evaluation/lvis_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..7d712ef262789edb85392cb54577c3a6b15e223e --- /dev/null +++ b/annotator/oneformer/detectron2/evaluation/lvis_evaluation.py @@ -0,0 +1,380 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +import itertools +import json +import logging +import os +import pickle +from collections import OrderedDict +import torch + +import annotator.oneformer.detectron2.utils.comm as comm +from annotator.oneformer.detectron2.config import CfgNode +from annotator.oneformer.detectron2.data import MetadataCatalog +from annotator.oneformer.detectron2.structures import Boxes, BoxMode, pairwise_iou +from annotator.oneformer.detectron2.utils.file_io import PathManager +from annotator.oneformer.detectron2.utils.logger import create_small_table + +from .coco_evaluation import instances_to_coco_json +from .evaluator import DatasetEvaluator + + +class LVISEvaluator(DatasetEvaluator): + """ + Evaluate object proposal and instance detection/segmentation outputs using + LVIS's metrics and evaluation API. + """ + + def __init__( + self, + dataset_name, + tasks=None, + distributed=True, + output_dir=None, + *, + max_dets_per_image=None, + ): + """ + Args: + dataset_name (str): name of the dataset to be evaluated. + It must have the following corresponding metadata: + "json_file": the path to the LVIS format annotation + tasks (tuple[str]): tasks that can be evaluated under the given + configuration. A task is one of "bbox", "segm". + By default, will infer this automatically from predictions. + distributed (True): if True, will collect results from all ranks for evaluation. + Otherwise, will evaluate the results in the current process. + output_dir (str): optional, an output directory to dump results. + max_dets_per_image (None or int): limit on maximum detections per image in evaluating AP + This limit, by default of the LVIS dataset, is 300. + """ + from lvis import LVIS + + self._logger = logging.getLogger(__name__) + + if tasks is not None and isinstance(tasks, CfgNode): + self._logger.warn( + "COCO Evaluator instantiated using config, this is deprecated behavior." + " Please pass in explicit arguments instead." + ) + self._tasks = None # Infering it from predictions should be better + else: + self._tasks = tasks + + self._distributed = distributed + self._output_dir = output_dir + self._max_dets_per_image = max_dets_per_image + + self._cpu_device = torch.device("cpu") + + self._metadata = MetadataCatalog.get(dataset_name) + json_file = PathManager.get_local_path(self._metadata.json_file) + self._lvis_api = LVIS(json_file) + # Test set json files do not contain annotations (evaluation must be + # performed using the LVIS evaluation server). + self._do_evaluation = len(self._lvis_api.get_ann_ids()) > 0 + + def reset(self): + self._predictions = [] + + def process(self, inputs, outputs): + """ + Args: + inputs: the inputs to a LVIS model (e.g., GeneralizedRCNN). + It is a list of dict. Each dict corresponds to an image and + contains keys like "height", "width", "file_name", "image_id". + outputs: the outputs of a LVIS model. It is a list of dicts with key + "instances" that contains :class:`Instances`. + """ + for input, output in zip(inputs, outputs): + prediction = {"image_id": input["image_id"]} + + if "instances" in output: + instances = output["instances"].to(self._cpu_device) + prediction["instances"] = instances_to_coco_json(instances, input["image_id"]) + if "proposals" in output: + prediction["proposals"] = output["proposals"].to(self._cpu_device) + self._predictions.append(prediction) + + def evaluate(self): + if self._distributed: + comm.synchronize() + predictions = comm.gather(self._predictions, dst=0) + predictions = list(itertools.chain(*predictions)) + + if not comm.is_main_process(): + return + else: + predictions = self._predictions + + if len(predictions) == 0: + self._logger.warning("[LVISEvaluator] Did not receive valid predictions.") + return {} + + if self._output_dir: + PathManager.mkdirs(self._output_dir) + file_path = os.path.join(self._output_dir, "instances_predictions.pth") + with PathManager.open(file_path, "wb") as f: + torch.save(predictions, f) + + self._results = OrderedDict() + if "proposals" in predictions[0]: + self._eval_box_proposals(predictions) + if "instances" in predictions[0]: + self._eval_predictions(predictions) + # Copy so the caller can do whatever with results + return copy.deepcopy(self._results) + + def _tasks_from_predictions(self, predictions): + for pred in predictions: + if "segmentation" in pred: + return ("bbox", "segm") + return ("bbox",) + + def _eval_predictions(self, predictions): + """ + Evaluate predictions. Fill self._results with the metrics of the tasks. + + Args: + predictions (list[dict]): list of outputs from the model + """ + self._logger.info("Preparing results in the LVIS format ...") + lvis_results = list(itertools.chain(*[x["instances"] for x in predictions])) + tasks = self._tasks or self._tasks_from_predictions(lvis_results) + + # LVIS evaluator can be used to evaluate results for COCO dataset categories. + # In this case `_metadata` variable will have a field with COCO-specific category mapping. + if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"): + reverse_id_mapping = { + v: k for k, v in self._metadata.thing_dataset_id_to_contiguous_id.items() + } + for result in lvis_results: + result["category_id"] = reverse_id_mapping[result["category_id"]] + else: + # unmap the category ids for LVIS (from 0-indexed to 1-indexed) + for result in lvis_results: + result["category_id"] += 1 + + if self._output_dir: + file_path = os.path.join(self._output_dir, "lvis_instances_results.json") + self._logger.info("Saving results to {}".format(file_path)) + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(lvis_results)) + f.flush() + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info("Evaluating predictions ...") + for task in sorted(tasks): + res = _evaluate_predictions_on_lvis( + self._lvis_api, + lvis_results, + task, + max_dets_per_image=self._max_dets_per_image, + class_names=self._metadata.get("thing_classes"), + ) + self._results[task] = res + + def _eval_box_proposals(self, predictions): + """ + Evaluate the box proposals in predictions. + Fill self._results with the metrics for "box_proposals" task. + """ + if self._output_dir: + # Saving generated box proposals to file. + # Predicted box_proposals are in XYXY_ABS mode. + bbox_mode = BoxMode.XYXY_ABS.value + ids, boxes, objectness_logits = [], [], [] + for prediction in predictions: + ids.append(prediction["image_id"]) + boxes.append(prediction["proposals"].proposal_boxes.tensor.numpy()) + objectness_logits.append(prediction["proposals"].objectness_logits.numpy()) + + proposal_data = { + "boxes": boxes, + "objectness_logits": objectness_logits, + "ids": ids, + "bbox_mode": bbox_mode, + } + with PathManager.open(os.path.join(self._output_dir, "box_proposals.pkl"), "wb") as f: + pickle.dump(proposal_data, f) + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info("Evaluating bbox proposals ...") + res = {} + areas = {"all": "", "small": "s", "medium": "m", "large": "l"} + for limit in [100, 1000]: + for area, suffix in areas.items(): + stats = _evaluate_box_proposals(predictions, self._lvis_api, area=area, limit=limit) + key = "AR{}@{:d}".format(suffix, limit) + res[key] = float(stats["ar"].item() * 100) + self._logger.info("Proposal metrics: \n" + create_small_table(res)) + self._results["box_proposals"] = res + + +# inspired from Detectron: +# https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L255 # noqa +def _evaluate_box_proposals(dataset_predictions, lvis_api, thresholds=None, area="all", limit=None): + """ + Evaluate detection proposal recall metrics. This function is a much + faster alternative to the official LVIS API recall evaluation code. However, + it produces slightly different results. + """ + # Record max overlap value for each gt box + # Return vector of overlap values + areas = { + "all": 0, + "small": 1, + "medium": 2, + "large": 3, + "96-128": 4, + "128-256": 5, + "256-512": 6, + "512-inf": 7, + } + area_ranges = [ + [0**2, 1e5**2], # all + [0**2, 32**2], # small + [32**2, 96**2], # medium + [96**2, 1e5**2], # large + [96**2, 128**2], # 96-128 + [128**2, 256**2], # 128-256 + [256**2, 512**2], # 256-512 + [512**2, 1e5**2], + ] # 512-inf + assert area in areas, "Unknown area range: {}".format(area) + area_range = area_ranges[areas[area]] + gt_overlaps = [] + num_pos = 0 + + for prediction_dict in dataset_predictions: + predictions = prediction_dict["proposals"] + + # sort predictions in descending order + # TODO maybe remove this and make it explicit in the documentation + inds = predictions.objectness_logits.sort(descending=True)[1] + predictions = predictions[inds] + + ann_ids = lvis_api.get_ann_ids(img_ids=[prediction_dict["image_id"]]) + anno = lvis_api.load_anns(ann_ids) + gt_boxes = [ + BoxMode.convert(obj["bbox"], BoxMode.XYWH_ABS, BoxMode.XYXY_ABS) for obj in anno + ] + gt_boxes = torch.as_tensor(gt_boxes).reshape(-1, 4) # guard against no boxes + gt_boxes = Boxes(gt_boxes) + gt_areas = torch.as_tensor([obj["area"] for obj in anno]) + + if len(gt_boxes) == 0 or len(predictions) == 0: + continue + + valid_gt_inds = (gt_areas >= area_range[0]) & (gt_areas <= area_range[1]) + gt_boxes = gt_boxes[valid_gt_inds] + + num_pos += len(gt_boxes) + + if len(gt_boxes) == 0: + continue + + if limit is not None and len(predictions) > limit: + predictions = predictions[:limit] + + overlaps = pairwise_iou(predictions.proposal_boxes, gt_boxes) + + _gt_overlaps = torch.zeros(len(gt_boxes)) + for j in range(min(len(predictions), len(gt_boxes))): + # find which proposal box maximally covers each gt box + # and get the iou amount of coverage for each gt box + max_overlaps, argmax_overlaps = overlaps.max(dim=0) + + # find which gt box is 'best' covered (i.e. 'best' = most iou) + gt_ovr, gt_ind = max_overlaps.max(dim=0) + assert gt_ovr >= 0 + # find the proposal box that covers the best covered gt box + box_ind = argmax_overlaps[gt_ind] + # record the iou coverage of this gt box + _gt_overlaps[j] = overlaps[box_ind, gt_ind] + assert _gt_overlaps[j] == gt_ovr + # mark the proposal box and the gt box as used + overlaps[box_ind, :] = -1 + overlaps[:, gt_ind] = -1 + + # append recorded iou coverage level + gt_overlaps.append(_gt_overlaps) + gt_overlaps = ( + torch.cat(gt_overlaps, dim=0) if len(gt_overlaps) else torch.zeros(0, dtype=torch.float32) + ) + gt_overlaps, _ = torch.sort(gt_overlaps) + + if thresholds is None: + step = 0.05 + thresholds = torch.arange(0.5, 0.95 + 1e-5, step, dtype=torch.float32) + recalls = torch.zeros_like(thresholds) + # compute recall for each iou threshold + for i, t in enumerate(thresholds): + recalls[i] = (gt_overlaps >= t).float().sum() / float(num_pos) + # ar = 2 * np.trapz(recalls, thresholds) + ar = recalls.mean() + return { + "ar": ar, + "recalls": recalls, + "thresholds": thresholds, + "gt_overlaps": gt_overlaps, + "num_pos": num_pos, + } + + +def _evaluate_predictions_on_lvis( + lvis_gt, lvis_results, iou_type, max_dets_per_image=None, class_names=None +): + """ + Args: + iou_type (str): + max_dets_per_image (None or int): limit on maximum detections per image in evaluating AP + This limit, by default of the LVIS dataset, is 300. + class_names (None or list[str]): if provided, will use it to predict + per-category AP. + + Returns: + a dict of {metric name: score} + """ + metrics = { + "bbox": ["AP", "AP50", "AP75", "APs", "APm", "APl", "APr", "APc", "APf"], + "segm": ["AP", "AP50", "AP75", "APs", "APm", "APl", "APr", "APc", "APf"], + }[iou_type] + + logger = logging.getLogger(__name__) + + if len(lvis_results) == 0: # TODO: check if needed + logger.warn("No predictions from the model!") + return {metric: float("nan") for metric in metrics} + + if iou_type == "segm": + lvis_results = copy.deepcopy(lvis_results) + # When evaluating mask AP, if the results contain bbox, LVIS API will + # use the box area as the area of the instance, instead of the mask area. + # This leads to a different definition of small/medium/large. + # We remove the bbox field to let mask AP use mask area. + for c in lvis_results: + c.pop("bbox", None) + + if max_dets_per_image is None: + max_dets_per_image = 300 # Default for LVIS dataset + + from lvis import LVISEval, LVISResults + + logger.info(f"Evaluating with max detections per image = {max_dets_per_image}") + lvis_results = LVISResults(lvis_gt, lvis_results, max_dets=max_dets_per_image) + lvis_eval = LVISEval(lvis_gt, lvis_results, iou_type) + lvis_eval.run() + lvis_eval.print_results() + + # Pull the standard metrics from the LVIS results + results = lvis_eval.get_results() + results = {metric: float(results[metric] * 100) for metric in metrics} + logger.info("Evaluation results for {}: \n".format(iou_type) + create_small_table(results)) + return results diff --git a/annotator/oneformer/detectron2/evaluation/panoptic_evaluation.py b/annotator/oneformer/detectron2/evaluation/panoptic_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..bf77fe061291f44381f8417e82e8b2bc7c5a60c6 --- /dev/null +++ b/annotator/oneformer/detectron2/evaluation/panoptic_evaluation.py @@ -0,0 +1,199 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import contextlib +import io +import itertools +import json +import logging +import numpy as np +import os +import tempfile +from collections import OrderedDict +from typing import Optional +from PIL import Image +from tabulate import tabulate + +from annotator.oneformer.detectron2.data import MetadataCatalog +from annotator.oneformer.detectron2.utils import comm +from annotator.oneformer.detectron2.utils.file_io import PathManager + +from .evaluator import DatasetEvaluator + +logger = logging.getLogger(__name__) + + +class COCOPanopticEvaluator(DatasetEvaluator): + """ + Evaluate Panoptic Quality metrics on COCO using PanopticAPI. + It saves panoptic segmentation prediction in `output_dir` + + It contains a synchronize call and has to be called from all workers. + """ + + def __init__(self, dataset_name: str, output_dir: Optional[str] = None): + """ + Args: + dataset_name: name of the dataset + output_dir: output directory to save results for evaluation. + """ + self._metadata = MetadataCatalog.get(dataset_name) + self._thing_contiguous_id_to_dataset_id = { + v: k for k, v in self._metadata.thing_dataset_id_to_contiguous_id.items() + } + self._stuff_contiguous_id_to_dataset_id = { + v: k for k, v in self._metadata.stuff_dataset_id_to_contiguous_id.items() + } + + self._output_dir = output_dir + if self._output_dir is not None: + PathManager.mkdirs(self._output_dir) + + def reset(self): + self._predictions = [] + + def _convert_category_id(self, segment_info): + isthing = segment_info.pop("isthing", None) + if isthing is None: + # the model produces panoptic category id directly. No more conversion needed + return segment_info + if isthing is True: + segment_info["category_id"] = self._thing_contiguous_id_to_dataset_id[ + segment_info["category_id"] + ] + else: + segment_info["category_id"] = self._stuff_contiguous_id_to_dataset_id[ + segment_info["category_id"] + ] + return segment_info + + def process(self, inputs, outputs): + from panopticapi.utils import id2rgb + + for input, output in zip(inputs, outputs): + panoptic_img, segments_info = output["panoptic_seg"] + panoptic_img = panoptic_img.cpu().numpy() + if segments_info is None: + # If "segments_info" is None, we assume "panoptic_img" is a + # H*W int32 image storing the panoptic_id in the format of + # category_id * label_divisor + instance_id. We reserve -1 for + # VOID label, and add 1 to panoptic_img since the official + # evaluation script uses 0 for VOID label. + label_divisor = self._metadata.label_divisor + segments_info = [] + for panoptic_label in np.unique(panoptic_img): + if panoptic_label == -1: + # VOID region. + continue + pred_class = panoptic_label // label_divisor + isthing = ( + pred_class in self._metadata.thing_dataset_id_to_contiguous_id.values() + ) + segments_info.append( + { + "id": int(panoptic_label) + 1, + "category_id": int(pred_class), + "isthing": bool(isthing), + } + ) + # Official evaluation script uses 0 for VOID label. + panoptic_img += 1 + + file_name = os.path.basename(input["file_name"]) + file_name_png = os.path.splitext(file_name)[0] + ".png" + with io.BytesIO() as out: + Image.fromarray(id2rgb(panoptic_img)).save(out, format="PNG") + segments_info = [self._convert_category_id(x) for x in segments_info] + self._predictions.append( + { + "image_id": input["image_id"], + "file_name": file_name_png, + "png_string": out.getvalue(), + "segments_info": segments_info, + } + ) + + def evaluate(self): + comm.synchronize() + + self._predictions = comm.gather(self._predictions) + self._predictions = list(itertools.chain(*self._predictions)) + if not comm.is_main_process(): + return + + # PanopticApi requires local files + gt_json = PathManager.get_local_path(self._metadata.panoptic_json) + gt_folder = PathManager.get_local_path(self._metadata.panoptic_root) + + with tempfile.TemporaryDirectory(prefix="panoptic_eval") as pred_dir: + logger.info("Writing all panoptic predictions to {} ...".format(pred_dir)) + for p in self._predictions: + with open(os.path.join(pred_dir, p["file_name"]), "wb") as f: + f.write(p.pop("png_string")) + + with open(gt_json, "r") as f: + json_data = json.load(f) + json_data["annotations"] = self._predictions + + output_dir = self._output_dir or pred_dir + predictions_json = os.path.join(output_dir, "predictions.json") + with PathManager.open(predictions_json, "w") as f: + f.write(json.dumps(json_data)) + + from panopticapi.evaluation import pq_compute + + with contextlib.redirect_stdout(io.StringIO()): + pq_res = pq_compute( + gt_json, + PathManager.get_local_path(predictions_json), + gt_folder=gt_folder, + pred_folder=pred_dir, + ) + + res = {} + res["PQ"] = 100 * pq_res["All"]["pq"] + res["SQ"] = 100 * pq_res["All"]["sq"] + res["RQ"] = 100 * pq_res["All"]["rq"] + res["PQ_th"] = 100 * pq_res["Things"]["pq"] + res["SQ_th"] = 100 * pq_res["Things"]["sq"] + res["RQ_th"] = 100 * pq_res["Things"]["rq"] + res["PQ_st"] = 100 * pq_res["Stuff"]["pq"] + res["SQ_st"] = 100 * pq_res["Stuff"]["sq"] + res["RQ_st"] = 100 * pq_res["Stuff"]["rq"] + + results = OrderedDict({"panoptic_seg": res}) + _print_panoptic_results(pq_res) + + return results + + +def _print_panoptic_results(pq_res): + headers = ["", "PQ", "SQ", "RQ", "#categories"] + data = [] + for name in ["All", "Things", "Stuff"]: + row = [name] + [pq_res[name][k] * 100 for k in ["pq", "sq", "rq"]] + [pq_res[name]["n"]] + data.append(row) + table = tabulate( + data, headers=headers, tablefmt="pipe", floatfmt=".3f", stralign="center", numalign="center" + ) + logger.info("Panoptic Evaluation Results:\n" + table) + + +if __name__ == "__main__": + from annotator.oneformer.detectron2.utils.logger import setup_logger + + logger = setup_logger() + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument("--gt-json") + parser.add_argument("--gt-dir") + parser.add_argument("--pred-json") + parser.add_argument("--pred-dir") + args = parser.parse_args() + + from panopticapi.evaluation import pq_compute + + with contextlib.redirect_stdout(io.StringIO()): + pq_res = pq_compute( + args.gt_json, args.pred_json, gt_folder=args.gt_dir, pred_folder=args.pred_dir + ) + _print_panoptic_results(pq_res) diff --git a/annotator/oneformer/detectron2/evaluation/pascal_voc_evaluation.py b/annotator/oneformer/detectron2/evaluation/pascal_voc_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..b2963e5dc5b6ed471f0c37056b35a350ea4cf020 --- /dev/null +++ b/annotator/oneformer/detectron2/evaluation/pascal_voc_evaluation.py @@ -0,0 +1,300 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +import logging +import numpy as np +import os +import tempfile +import xml.etree.ElementTree as ET +from collections import OrderedDict, defaultdict +from functools import lru_cache +import torch + +from annotator.oneformer.detectron2.data import MetadataCatalog +from annotator.oneformer.detectron2.utils import comm +from annotator.oneformer.detectron2.utils.file_io import PathManager + +from .evaluator import DatasetEvaluator + + +class PascalVOCDetectionEvaluator(DatasetEvaluator): + """ + Evaluate Pascal VOC style AP for Pascal VOC dataset. + It contains a synchronization, therefore has to be called from all ranks. + + Note that the concept of AP can be implemented in different ways and may not + produce identical results. This class mimics the implementation of the official + Pascal VOC Matlab API, and should produce similar but not identical results to the + official API. + """ + + def __init__(self, dataset_name): + """ + Args: + dataset_name (str): name of the dataset, e.g., "voc_2007_test" + """ + self._dataset_name = dataset_name + meta = MetadataCatalog.get(dataset_name) + + # Too many tiny files, download all to local for speed. + annotation_dir_local = PathManager.get_local_path( + os.path.join(meta.dirname, "Annotations/") + ) + self._anno_file_template = os.path.join(annotation_dir_local, "{}.xml") + self._image_set_path = os.path.join(meta.dirname, "ImageSets", "Main", meta.split + ".txt") + self._class_names = meta.thing_classes + assert meta.year in [2007, 2012], meta.year + self._is_2007 = meta.year == 2007 + self._cpu_device = torch.device("cpu") + self._logger = logging.getLogger(__name__) + + def reset(self): + self._predictions = defaultdict(list) # class name -> list of prediction strings + + def process(self, inputs, outputs): + for input, output in zip(inputs, outputs): + image_id = input["image_id"] + instances = output["instances"].to(self._cpu_device) + boxes = instances.pred_boxes.tensor.numpy() + scores = instances.scores.tolist() + classes = instances.pred_classes.tolist() + for box, score, cls in zip(boxes, scores, classes): + xmin, ymin, xmax, ymax = box + # The inverse of data loading logic in `datasets/pascal_voc.py` + xmin += 1 + ymin += 1 + self._predictions[cls].append( + f"{image_id} {score:.3f} {xmin:.1f} {ymin:.1f} {xmax:.1f} {ymax:.1f}" + ) + + def evaluate(self): + """ + Returns: + dict: has a key "segm", whose value is a dict of "AP", "AP50", and "AP75". + """ + all_predictions = comm.gather(self._predictions, dst=0) + if not comm.is_main_process(): + return + predictions = defaultdict(list) + for predictions_per_rank in all_predictions: + for clsid, lines in predictions_per_rank.items(): + predictions[clsid].extend(lines) + del all_predictions + + self._logger.info( + "Evaluating {} using {} metric. " + "Note that results do not use the official Matlab API.".format( + self._dataset_name, 2007 if self._is_2007 else 2012 + ) + ) + + with tempfile.TemporaryDirectory(prefix="pascal_voc_eval_") as dirname: + res_file_template = os.path.join(dirname, "{}.txt") + + aps = defaultdict(list) # iou -> ap per class + for cls_id, cls_name in enumerate(self._class_names): + lines = predictions.get(cls_id, [""]) + + with open(res_file_template.format(cls_name), "w") as f: + f.write("\n".join(lines)) + + for thresh in range(50, 100, 5): + rec, prec, ap = voc_eval( + res_file_template, + self._anno_file_template, + self._image_set_path, + cls_name, + ovthresh=thresh / 100.0, + use_07_metric=self._is_2007, + ) + aps[thresh].append(ap * 100) + + ret = OrderedDict() + mAP = {iou: np.mean(x) for iou, x in aps.items()} + ret["bbox"] = {"AP": np.mean(list(mAP.values())), "AP50": mAP[50], "AP75": mAP[75]} + return ret + + +############################################################################## +# +# Below code is modified from +# https://github.com/rbgirshick/py-faster-rcnn/blob/master/lib/datasets/voc_eval.py +# -------------------------------------------------------- +# Fast/er R-CNN +# Licensed under The MIT License [see LICENSE for details] +# Written by Bharath Hariharan +# -------------------------------------------------------- + +"""Python implementation of the PASCAL VOC devkit's AP evaluation code.""" + + +@lru_cache(maxsize=None) +def parse_rec(filename): + """Parse a PASCAL VOC xml file.""" + with PathManager.open(filename) as f: + tree = ET.parse(f) + objects = [] + for obj in tree.findall("object"): + obj_struct = {} + obj_struct["name"] = obj.find("name").text + obj_struct["pose"] = obj.find("pose").text + obj_struct["truncated"] = int(obj.find("truncated").text) + obj_struct["difficult"] = int(obj.find("difficult").text) + bbox = obj.find("bndbox") + obj_struct["bbox"] = [ + int(bbox.find("xmin").text), + int(bbox.find("ymin").text), + int(bbox.find("xmax").text), + int(bbox.find("ymax").text), + ] + objects.append(obj_struct) + + return objects + + +def voc_ap(rec, prec, use_07_metric=False): + """Compute VOC AP given precision and recall. If use_07_metric is true, uses + the VOC 07 11-point method (default:False). + """ + if use_07_metric: + # 11 point metric + ap = 0.0 + for t in np.arange(0.0, 1.1, 0.1): + if np.sum(rec >= t) == 0: + p = 0 + else: + p = np.max(prec[rec >= t]) + ap = ap + p / 11.0 + else: + # correct AP calculation + # first append sentinel values at the end + mrec = np.concatenate(([0.0], rec, [1.0])) + mpre = np.concatenate(([0.0], prec, [0.0])) + + # compute the precision envelope + for i in range(mpre.size - 1, 0, -1): + mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i]) + + # to calculate area under PR curve, look for points + # where X axis (recall) changes value + i = np.where(mrec[1:] != mrec[:-1])[0] + + # and sum (\Delta recall) * prec + ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) + return ap + + +def voc_eval(detpath, annopath, imagesetfile, classname, ovthresh=0.5, use_07_metric=False): + """rec, prec, ap = voc_eval(detpath, + annopath, + imagesetfile, + classname, + [ovthresh], + [use_07_metric]) + + Top level function that does the PASCAL VOC evaluation. + + detpath: Path to detections + detpath.format(classname) should produce the detection results file. + annopath: Path to annotations + annopath.format(imagename) should be the xml annotations file. + imagesetfile: Text file containing the list of images, one image per line. + classname: Category name (duh) + [ovthresh]: Overlap threshold (default = 0.5) + [use_07_metric]: Whether to use VOC07's 11 point AP computation + (default False) + """ + # assumes detections are in detpath.format(classname) + # assumes annotations are in annopath.format(imagename) + # assumes imagesetfile is a text file with each line an image name + + # first load gt + # read list of images + with PathManager.open(imagesetfile, "r") as f: + lines = f.readlines() + imagenames = [x.strip() for x in lines] + + # load annots + recs = {} + for imagename in imagenames: + recs[imagename] = parse_rec(annopath.format(imagename)) + + # extract gt objects for this class + class_recs = {} + npos = 0 + for imagename in imagenames: + R = [obj for obj in recs[imagename] if obj["name"] == classname] + bbox = np.array([x["bbox"] for x in R]) + difficult = np.array([x["difficult"] for x in R]).astype(bool) + # difficult = np.array([False for x in R]).astype(bool) # treat all "difficult" as GT + det = [False] * len(R) + npos = npos + sum(~difficult) + class_recs[imagename] = {"bbox": bbox, "difficult": difficult, "det": det} + + # read dets + detfile = detpath.format(classname) + with open(detfile, "r") as f: + lines = f.readlines() + + splitlines = [x.strip().split(" ") for x in lines] + image_ids = [x[0] for x in splitlines] + confidence = np.array([float(x[1]) for x in splitlines]) + BB = np.array([[float(z) for z in x[2:]] for x in splitlines]).reshape(-1, 4) + + # sort by confidence + sorted_ind = np.argsort(-confidence) + BB = BB[sorted_ind, :] + image_ids = [image_ids[x] for x in sorted_ind] + + # go down dets and mark TPs and FPs + nd = len(image_ids) + tp = np.zeros(nd) + fp = np.zeros(nd) + for d in range(nd): + R = class_recs[image_ids[d]] + bb = BB[d, :].astype(float) + ovmax = -np.inf + BBGT = R["bbox"].astype(float) + + if BBGT.size > 0: + # compute overlaps + # intersection + ixmin = np.maximum(BBGT[:, 0], bb[0]) + iymin = np.maximum(BBGT[:, 1], bb[1]) + ixmax = np.minimum(BBGT[:, 2], bb[2]) + iymax = np.minimum(BBGT[:, 3], bb[3]) + iw = np.maximum(ixmax - ixmin + 1.0, 0.0) + ih = np.maximum(iymax - iymin + 1.0, 0.0) + inters = iw * ih + + # union + uni = ( + (bb[2] - bb[0] + 1.0) * (bb[3] - bb[1] + 1.0) + + (BBGT[:, 2] - BBGT[:, 0] + 1.0) * (BBGT[:, 3] - BBGT[:, 1] + 1.0) + - inters + ) + + overlaps = inters / uni + ovmax = np.max(overlaps) + jmax = np.argmax(overlaps) + + if ovmax > ovthresh: + if not R["difficult"][jmax]: + if not R["det"][jmax]: + tp[d] = 1.0 + R["det"][jmax] = 1 + else: + fp[d] = 1.0 + else: + fp[d] = 1.0 + + # compute precision recall + fp = np.cumsum(fp) + tp = np.cumsum(tp) + rec = tp / float(npos) + # avoid divide by zero in case the first detection matches a difficult + # ground truth + prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps) + ap = voc_ap(rec, prec, use_07_metric) + + return rec, prec, ap diff --git a/annotator/oneformer/detectron2/evaluation/rotated_coco_evaluation.py b/annotator/oneformer/detectron2/evaluation/rotated_coco_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..4cf954d751dfe25367ce6059626b7118b34bb45a --- /dev/null +++ b/annotator/oneformer/detectron2/evaluation/rotated_coco_evaluation.py @@ -0,0 +1,207 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import itertools +import json +import numpy as np +import os +import torch +from pycocotools.cocoeval import COCOeval, maskUtils + +from annotator.oneformer.detectron2.structures import BoxMode, RotatedBoxes, pairwise_iou_rotated +from annotator.oneformer.detectron2.utils.file_io import PathManager + +from .coco_evaluation import COCOEvaluator + + +class RotatedCOCOeval(COCOeval): + @staticmethod + def is_rotated(box_list): + if type(box_list) == np.ndarray: + return box_list.shape[1] == 5 + elif type(box_list) == list: + if box_list == []: # cannot decide the box_dim + return False + return np.all( + np.array( + [ + (len(obj) == 5) and ((type(obj) == list) or (type(obj) == np.ndarray)) + for obj in box_list + ] + ) + ) + return False + + @staticmethod + def boxlist_to_tensor(boxlist, output_box_dim): + if type(boxlist) == np.ndarray: + box_tensor = torch.from_numpy(boxlist) + elif type(boxlist) == list: + if boxlist == []: + return torch.zeros((0, output_box_dim), dtype=torch.float32) + else: + box_tensor = torch.FloatTensor(boxlist) + else: + raise Exception("Unrecognized boxlist type") + + input_box_dim = box_tensor.shape[1] + if input_box_dim != output_box_dim: + if input_box_dim == 4 and output_box_dim == 5: + box_tensor = BoxMode.convert(box_tensor, BoxMode.XYWH_ABS, BoxMode.XYWHA_ABS) + else: + raise Exception( + "Unable to convert from {}-dim box to {}-dim box".format( + input_box_dim, output_box_dim + ) + ) + return box_tensor + + def compute_iou_dt_gt(self, dt, gt, is_crowd): + if self.is_rotated(dt) or self.is_rotated(gt): + # TODO: take is_crowd into consideration + assert all(c == 0 for c in is_crowd) + dt = RotatedBoxes(self.boxlist_to_tensor(dt, output_box_dim=5)) + gt = RotatedBoxes(self.boxlist_to_tensor(gt, output_box_dim=5)) + return pairwise_iou_rotated(dt, gt) + else: + # This is the same as the classical COCO evaluation + return maskUtils.iou(dt, gt, is_crowd) + + def computeIoU(self, imgId, catId): + p = self.params + if p.useCats: + gt = self._gts[imgId, catId] + dt = self._dts[imgId, catId] + else: + gt = [_ for cId in p.catIds for _ in self._gts[imgId, cId]] + dt = [_ for cId in p.catIds for _ in self._dts[imgId, cId]] + if len(gt) == 0 and len(dt) == 0: + return [] + inds = np.argsort([-d["score"] for d in dt], kind="mergesort") + dt = [dt[i] for i in inds] + if len(dt) > p.maxDets[-1]: + dt = dt[0 : p.maxDets[-1]] + + assert p.iouType == "bbox", "unsupported iouType for iou computation" + + g = [g["bbox"] for g in gt] + d = [d["bbox"] for d in dt] + + # compute iou between each dt and gt region + iscrowd = [int(o["iscrowd"]) for o in gt] + + # Note: this function is copied from cocoeval.py in cocoapi + # and the major difference is here. + ious = self.compute_iou_dt_gt(d, g, iscrowd) + return ious + + +class RotatedCOCOEvaluator(COCOEvaluator): + """ + Evaluate object proposal/instance detection outputs using COCO-like metrics and APIs, + with rotated boxes support. + Note: this uses IOU only and does not consider angle differences. + """ + + def process(self, inputs, outputs): + """ + Args: + inputs: the inputs to a COCO model (e.g., GeneralizedRCNN). + It is a list of dict. Each dict corresponds to an image and + contains keys like "height", "width", "file_name", "image_id". + outputs: the outputs of a COCO model. It is a list of dicts with key + "instances" that contains :class:`Instances`. + """ + for input, output in zip(inputs, outputs): + prediction = {"image_id": input["image_id"]} + + if "instances" in output: + instances = output["instances"].to(self._cpu_device) + + prediction["instances"] = self.instances_to_json(instances, input["image_id"]) + if "proposals" in output: + prediction["proposals"] = output["proposals"].to(self._cpu_device) + self._predictions.append(prediction) + + def instances_to_json(self, instances, img_id): + num_instance = len(instances) + if num_instance == 0: + return [] + + boxes = instances.pred_boxes.tensor.numpy() + if boxes.shape[1] == 4: + boxes = BoxMode.convert(boxes, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS) + boxes = boxes.tolist() + scores = instances.scores.tolist() + classes = instances.pred_classes.tolist() + + results = [] + for k in range(num_instance): + result = { + "image_id": img_id, + "category_id": classes[k], + "bbox": boxes[k], + "score": scores[k], + } + + results.append(result) + return results + + def _eval_predictions(self, predictions, img_ids=None): # img_ids: unused + """ + Evaluate predictions on the given tasks. + Fill self._results with the metrics of the tasks. + """ + self._logger.info("Preparing results for COCO format ...") + coco_results = list(itertools.chain(*[x["instances"] for x in predictions])) + + # unmap the category ids for COCO + if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"): + reverse_id_mapping = { + v: k for k, v in self._metadata.thing_dataset_id_to_contiguous_id.items() + } + for result in coco_results: + result["category_id"] = reverse_id_mapping[result["category_id"]] + + if self._output_dir: + file_path = os.path.join(self._output_dir, "coco_instances_results.json") + self._logger.info("Saving results to {}".format(file_path)) + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(coco_results)) + f.flush() + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info("Evaluating predictions ...") + + assert self._tasks is None or set(self._tasks) == { + "bbox" + }, "[RotatedCOCOEvaluator] Only bbox evaluation is supported" + coco_eval = ( + self._evaluate_predictions_on_coco(self._coco_api, coco_results) + if len(coco_results) > 0 + else None # cocoapi does not handle empty results very well + ) + + task = "bbox" + res = self._derive_coco_results( + coco_eval, task, class_names=self._metadata.get("thing_classes") + ) + self._results[task] = res + + def _evaluate_predictions_on_coco(self, coco_gt, coco_results): + """ + Evaluate the coco results using COCOEval API. + """ + assert len(coco_results) > 0 + + coco_dt = coco_gt.loadRes(coco_results) + + # Only bbox is supported for now + coco_eval = RotatedCOCOeval(coco_gt, coco_dt, iouType="bbox") + + coco_eval.evaluate() + coco_eval.accumulate() + coco_eval.summarize() + + return coco_eval diff --git a/annotator/oneformer/detectron2/evaluation/sem_seg_evaluation.py b/annotator/oneformer/detectron2/evaluation/sem_seg_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..f8bc0e901954fc0eefca6386bcf8ad31e0e66277 --- /dev/null +++ b/annotator/oneformer/detectron2/evaluation/sem_seg_evaluation.py @@ -0,0 +1,265 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import itertools +import json +import logging +import numpy as np +import os +from collections import OrderedDict +from typing import Optional, Union +import pycocotools.mask as mask_util +import torch +from PIL import Image + +from annotator.oneformer.detectron2.data import DatasetCatalog, MetadataCatalog +from annotator.oneformer.detectron2.utils.comm import all_gather, is_main_process, synchronize +from annotator.oneformer.detectron2.utils.file_io import PathManager + +from .evaluator import DatasetEvaluator + +_CV2_IMPORTED = True +try: + import cv2 # noqa +except ImportError: + # OpenCV is an optional dependency at the moment + _CV2_IMPORTED = False + + +def load_image_into_numpy_array( + filename: str, + copy: bool = False, + dtype: Optional[Union[np.dtype, str]] = None, +) -> np.ndarray: + with PathManager.open(filename, "rb") as f: + array = np.array(Image.open(f), copy=copy, dtype=dtype) + return array + + +class SemSegEvaluator(DatasetEvaluator): + """ + Evaluate semantic segmentation metrics. + """ + + def __init__( + self, + dataset_name, + distributed=True, + output_dir=None, + *, + sem_seg_loading_fn=load_image_into_numpy_array, + num_classes=None, + ignore_label=None, + ): + """ + Args: + dataset_name (str): name of the dataset to be evaluated. + distributed (bool): if True, will collect results from all ranks for evaluation. + Otherwise, will evaluate the results in the current process. + output_dir (str): an output directory to dump results. + sem_seg_loading_fn: function to read sem seg file and load into numpy array. + Default provided, but projects can customize. + num_classes, ignore_label: deprecated argument + """ + self._logger = logging.getLogger(__name__) + if num_classes is not None: + self._logger.warn( + "SemSegEvaluator(num_classes) is deprecated! It should be obtained from metadata." + ) + if ignore_label is not None: + self._logger.warn( + "SemSegEvaluator(ignore_label) is deprecated! It should be obtained from metadata." + ) + self._dataset_name = dataset_name + self._distributed = distributed + self._output_dir = output_dir + + self._cpu_device = torch.device("cpu") + + self.input_file_to_gt_file = { + dataset_record["file_name"]: dataset_record["sem_seg_file_name"] + for dataset_record in DatasetCatalog.get(dataset_name) + } + + meta = MetadataCatalog.get(dataset_name) + # Dict that maps contiguous training ids to COCO category ids + try: + c2d = meta.stuff_dataset_id_to_contiguous_id + self._contiguous_id_to_dataset_id = {v: k for k, v in c2d.items()} + except AttributeError: + self._contiguous_id_to_dataset_id = None + self._class_names = meta.stuff_classes + self.sem_seg_loading_fn = sem_seg_loading_fn + self._num_classes = len(meta.stuff_classes) + if num_classes is not None: + assert self._num_classes == num_classes, f"{self._num_classes} != {num_classes}" + self._ignore_label = ignore_label if ignore_label is not None else meta.ignore_label + + # This is because cv2.erode did not work for int datatype. Only works for uint8. + self._compute_boundary_iou = True + if not _CV2_IMPORTED: + self._compute_boundary_iou = False + self._logger.warn( + """Boundary IoU calculation requires OpenCV. B-IoU metrics are + not going to be computed because OpenCV is not available to import.""" + ) + if self._num_classes >= np.iinfo(np.uint8).max: + self._compute_boundary_iou = False + self._logger.warn( + f"""SemSegEvaluator(num_classes) is more than supported value for Boundary IoU calculation! + B-IoU metrics are not going to be computed. Max allowed value (exclusive) + for num_classes for calculating Boundary IoU is {np.iinfo(np.uint8).max}. + The number of classes of dataset {self._dataset_name} is {self._num_classes}""" + ) + + def reset(self): + self._conf_matrix = np.zeros((self._num_classes + 1, self._num_classes + 1), dtype=np.int64) + self._b_conf_matrix = np.zeros( + (self._num_classes + 1, self._num_classes + 1), dtype=np.int64 + ) + self._predictions = [] + + def process(self, inputs, outputs): + """ + Args: + inputs: the inputs to a model. + It is a list of dicts. Each dict corresponds to an image and + contains keys like "height", "width", "file_name". + outputs: the outputs of a model. It is either list of semantic segmentation predictions + (Tensor [H, W]) or list of dicts with key "sem_seg" that contains semantic + segmentation prediction in the same format. + """ + for input, output in zip(inputs, outputs): + output = output["sem_seg"].argmax(dim=0).to(self._cpu_device) + pred = np.array(output, dtype=np.int) + gt_filename = self.input_file_to_gt_file[input["file_name"]] + gt = self.sem_seg_loading_fn(gt_filename, dtype=np.int) + + gt[gt == self._ignore_label] = self._num_classes + + self._conf_matrix += np.bincount( + (self._num_classes + 1) * pred.reshape(-1) + gt.reshape(-1), + minlength=self._conf_matrix.size, + ).reshape(self._conf_matrix.shape) + + if self._compute_boundary_iou: + b_gt = self._mask_to_boundary(gt.astype(np.uint8)) + b_pred = self._mask_to_boundary(pred.astype(np.uint8)) + + self._b_conf_matrix += np.bincount( + (self._num_classes + 1) * b_pred.reshape(-1) + b_gt.reshape(-1), + minlength=self._conf_matrix.size, + ).reshape(self._conf_matrix.shape) + + self._predictions.extend(self.encode_json_sem_seg(pred, input["file_name"])) + + def evaluate(self): + """ + Evaluates standard semantic segmentation metrics (http://cocodataset.org/#stuff-eval): + + * Mean intersection-over-union averaged across classes (mIoU) + * Frequency Weighted IoU (fwIoU) + * Mean pixel accuracy averaged across classes (mACC) + * Pixel Accuracy (pACC) + """ + if self._distributed: + synchronize() + conf_matrix_list = all_gather(self._conf_matrix) + b_conf_matrix_list = all_gather(self._b_conf_matrix) + self._predictions = all_gather(self._predictions) + self._predictions = list(itertools.chain(*self._predictions)) + if not is_main_process(): + return + + self._conf_matrix = np.zeros_like(self._conf_matrix) + for conf_matrix in conf_matrix_list: + self._conf_matrix += conf_matrix + + self._b_conf_matrix = np.zeros_like(self._b_conf_matrix) + for b_conf_matrix in b_conf_matrix_list: + self._b_conf_matrix += b_conf_matrix + + if self._output_dir: + PathManager.mkdirs(self._output_dir) + file_path = os.path.join(self._output_dir, "sem_seg_predictions.json") + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(self._predictions)) + + acc = np.full(self._num_classes, np.nan, dtype=np.float) + iou = np.full(self._num_classes, np.nan, dtype=np.float) + tp = self._conf_matrix.diagonal()[:-1].astype(np.float) + pos_gt = np.sum(self._conf_matrix[:-1, :-1], axis=0).astype(np.float) + class_weights = pos_gt / np.sum(pos_gt) + pos_pred = np.sum(self._conf_matrix[:-1, :-1], axis=1).astype(np.float) + acc_valid = pos_gt > 0 + acc[acc_valid] = tp[acc_valid] / pos_gt[acc_valid] + union = pos_gt + pos_pred - tp + iou_valid = np.logical_and(acc_valid, union > 0) + iou[iou_valid] = tp[iou_valid] / union[iou_valid] + macc = np.sum(acc[acc_valid]) / np.sum(acc_valid) + miou = np.sum(iou[iou_valid]) / np.sum(iou_valid) + fiou = np.sum(iou[iou_valid] * class_weights[iou_valid]) + pacc = np.sum(tp) / np.sum(pos_gt) + + if self._compute_boundary_iou: + b_iou = np.full(self._num_classes, np.nan, dtype=np.float) + b_tp = self._b_conf_matrix.diagonal()[:-1].astype(np.float) + b_pos_gt = np.sum(self._b_conf_matrix[:-1, :-1], axis=0).astype(np.float) + b_pos_pred = np.sum(self._b_conf_matrix[:-1, :-1], axis=1).astype(np.float) + b_union = b_pos_gt + b_pos_pred - b_tp + b_iou_valid = b_union > 0 + b_iou[b_iou_valid] = b_tp[b_iou_valid] / b_union[b_iou_valid] + + res = {} + res["mIoU"] = 100 * miou + res["fwIoU"] = 100 * fiou + for i, name in enumerate(self._class_names): + res[f"IoU-{name}"] = 100 * iou[i] + if self._compute_boundary_iou: + res[f"BoundaryIoU-{name}"] = 100 * b_iou[i] + res[f"min(IoU, B-Iou)-{name}"] = 100 * min(iou[i], b_iou[i]) + res["mACC"] = 100 * macc + res["pACC"] = 100 * pacc + for i, name in enumerate(self._class_names): + res[f"ACC-{name}"] = 100 * acc[i] + + if self._output_dir: + file_path = os.path.join(self._output_dir, "sem_seg_evaluation.pth") + with PathManager.open(file_path, "wb") as f: + torch.save(res, f) + results = OrderedDict({"sem_seg": res}) + self._logger.info(results) + return results + + def encode_json_sem_seg(self, sem_seg, input_file_name): + """ + Convert semantic segmentation to COCO stuff format with segments encoded as RLEs. + See http://cocodataset.org/#format-results + """ + json_list = [] + for label in np.unique(sem_seg): + if self._contiguous_id_to_dataset_id is not None: + assert ( + label in self._contiguous_id_to_dataset_id + ), "Label {} is not in the metadata info for {}".format(label, self._dataset_name) + dataset_id = self._contiguous_id_to_dataset_id[label] + else: + dataset_id = int(label) + mask = (sem_seg == label).astype(np.uint8) + mask_rle = mask_util.encode(np.array(mask[:, :, None], order="F"))[0] + mask_rle["counts"] = mask_rle["counts"].decode("utf-8") + json_list.append( + {"file_name": input_file_name, "category_id": dataset_id, "segmentation": mask_rle} + ) + return json_list + + def _mask_to_boundary(self, mask: np.ndarray, dilation_ratio=0.02): + assert mask.ndim == 2, "mask_to_boundary expects a 2-dimensional image" + h, w = mask.shape + diag_len = np.sqrt(h**2 + w**2) + dilation = max(1, int(round(dilation_ratio * diag_len))) + kernel = np.ones((3, 3), dtype=np.uint8) + + padded_mask = cv2.copyMakeBorder(mask, 1, 1, 1, 1, cv2.BORDER_CONSTANT, value=0) + eroded_mask_with_padding = cv2.erode(padded_mask, kernel, iterations=dilation) + eroded_mask = eroded_mask_with_padding[1:-1, 1:-1] + boundary = mask - eroded_mask + return boundary diff --git a/annotator/oneformer/detectron2/evaluation/testing.py b/annotator/oneformer/detectron2/evaluation/testing.py new file mode 100644 index 0000000000000000000000000000000000000000..9e5ae625bb0593fc20739dd3ea549157e4df4f3d --- /dev/null +++ b/annotator/oneformer/detectron2/evaluation/testing.py @@ -0,0 +1,85 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import numpy as np +import pprint +import sys +from collections.abc import Mapping + + +def print_csv_format(results): + """ + Print main metrics in a format similar to Detectron, + so that they are easy to copypaste into a spreadsheet. + + Args: + results (OrderedDict[dict]): task_name -> {metric -> score} + unordered dict can also be printed, but in arbitrary order + """ + assert isinstance(results, Mapping) or not len(results), results + logger = logging.getLogger(__name__) + for task, res in results.items(): + if isinstance(res, Mapping): + # Don't print "AP-category" metrics since they are usually not tracked. + important_res = [(k, v) for k, v in res.items() if "-" not in k] + logger.info("copypaste: Task: {}".format(task)) + logger.info("copypaste: " + ",".join([k[0] for k in important_res])) + logger.info("copypaste: " + ",".join(["{0:.4f}".format(k[1]) for k in important_res])) + else: + logger.info(f"copypaste: {task}={res}") + + +def verify_results(cfg, results): + """ + Args: + results (OrderedDict[dict]): task_name -> {metric -> score} + + Returns: + bool: whether the verification succeeds or not + """ + expected_results = cfg.TEST.EXPECTED_RESULTS + if not len(expected_results): + return True + + ok = True + for task, metric, expected, tolerance in expected_results: + actual = results[task].get(metric, None) + if actual is None: + ok = False + continue + if not np.isfinite(actual): + ok = False + continue + diff = abs(actual - expected) + if diff > tolerance: + ok = False + + logger = logging.getLogger(__name__) + if not ok: + logger.error("Result verification failed!") + logger.error("Expected Results: " + str(expected_results)) + logger.error("Actual Results: " + pprint.pformat(results)) + + sys.exit(1) + else: + logger.info("Results verification passed.") + return ok + + +def flatten_results_dict(results): + """ + Expand a hierarchical dict of scalars into a flat dict of scalars. + If results[k1][k2][k3] = v, the returned dict will have the entry + {"k1/k2/k3": v}. + + Args: + results (dict): + """ + r = {} + for k, v in results.items(): + if isinstance(v, Mapping): + v = flatten_results_dict(v) + for kk, vv in v.items(): + r[k + "/" + kk] = vv + else: + r[k] = v + return r diff --git a/annotator/oneformer/detectron2/export/README.md b/annotator/oneformer/detectron2/export/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c86ff62516f4e8e4b1a6c1f33f11192933cf3861 --- /dev/null +++ b/annotator/oneformer/detectron2/export/README.md @@ -0,0 +1,15 @@ + +This directory contains code to prepare a detectron2 model for deployment. +Currently it supports exporting a detectron2 model to TorchScript, ONNX, or (deprecated) Caffe2 format. + +Please see [documentation](https://detectron2.readthedocs.io/tutorials/deployment.html) for its usage. + + +### Acknowledgements + +Thanks to Mobile Vision team at Facebook for developing the Caffe2 conversion tools. + +Thanks to Computing Platform Department - PAI team at Alibaba Group (@bddpqq, @chenbohua3) who +help export Detectron2 models to TorchScript. + +Thanks to ONNX Converter team at Microsoft who help export Detectron2 models to ONNX. diff --git a/annotator/oneformer/detectron2/export/__init__.py b/annotator/oneformer/detectron2/export/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5a58758f64aae6071fa688be4400622ce6036efa --- /dev/null +++ b/annotator/oneformer/detectron2/export/__init__.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- + +import warnings + +from .flatten import TracingAdapter +from .torchscript import dump_torchscript_IR, scripting_with_instances + +try: + from caffe2.proto import caffe2_pb2 as _tmp + from caffe2.python import core + + # caffe2 is optional +except ImportError: + pass +else: + from .api import * + + +# TODO: Update ONNX Opset version and run tests when a newer PyTorch is supported +STABLE_ONNX_OPSET_VERSION = 11 + + +def add_export_config(cfg): + warnings.warn( + "add_export_config has been deprecated and behaves as no-op function.", DeprecationWarning + ) + return cfg + + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/annotator/oneformer/detectron2/export/api.py b/annotator/oneformer/detectron2/export/api.py new file mode 100644 index 0000000000000000000000000000000000000000..cf1a27a4806ca83d97f5cd8c27726ec29f4e7e50 --- /dev/null +++ b/annotator/oneformer/detectron2/export/api.py @@ -0,0 +1,230 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +import logging +import os +import torch +from caffe2.proto import caffe2_pb2 +from torch import nn + +from annotator.oneformer.detectron2.config import CfgNode +from annotator.oneformer.detectron2.utils.file_io import PathManager + +from .caffe2_inference import ProtobufDetectionModel +from .caffe2_modeling import META_ARCH_CAFFE2_EXPORT_TYPE_MAP, convert_batched_inputs_to_c2_format +from .shared import get_pb_arg_vali, get_pb_arg_vals, save_graph + +__all__ = [ + "Caffe2Model", + "Caffe2Tracer", +] + + +class Caffe2Tracer: + """ + Make a detectron2 model traceable with Caffe2 operators. + This class creates a traceable version of a detectron2 model which: + + 1. Rewrite parts of the model using ops in Caffe2. Note that some ops do + not have GPU implementation in Caffe2. + 2. Remove post-processing and only produce raw layer outputs + + After making a traceable model, the class provide methods to export such a + model to different deployment formats. + Exported graph produced by this class take two input tensors: + + 1. (1, C, H, W) float "data" which is an image (usually in [0, 255]). + (H, W) often has to be padded to multiple of 32 (depend on the model + architecture). + 2. 1x3 float "im_info", each row of which is (height, width, 1.0). + Height and width are true image shapes before padding. + + The class currently only supports models using builtin meta architectures. + Batch inference is not supported, and contributions are welcome. + """ + + def __init__(self, cfg: CfgNode, model: nn.Module, inputs): + """ + Args: + cfg (CfgNode): a detectron2 config used to construct caffe2-compatible model. + model (nn.Module): An original pytorch model. Must be among a few official models + in detectron2 that can be converted to become caffe2-compatible automatically. + Weights have to be already loaded to this model. + inputs: sample inputs that the given model takes for inference. + Will be used to trace the model. For most models, random inputs with + no detected objects will not work as they lead to wrong traces. + """ + assert isinstance(cfg, CfgNode), cfg + assert isinstance(model, torch.nn.Module), type(model) + + # TODO make it support custom models, by passing in c2 model directly + C2MetaArch = META_ARCH_CAFFE2_EXPORT_TYPE_MAP[cfg.MODEL.META_ARCHITECTURE] + self.traceable_model = C2MetaArch(cfg, copy.deepcopy(model)) + self.inputs = inputs + self.traceable_inputs = self.traceable_model.get_caffe2_inputs(inputs) + + def export_caffe2(self): + """ + Export the model to Caffe2's protobuf format. + The returned object can be saved with its :meth:`.save_protobuf()` method. + The result can be loaded and executed using Caffe2 runtime. + + Returns: + :class:`Caffe2Model` + """ + from .caffe2_export import export_caffe2_detection_model + + predict_net, init_net = export_caffe2_detection_model( + self.traceable_model, self.traceable_inputs + ) + return Caffe2Model(predict_net, init_net) + + def export_onnx(self): + """ + Export the model to ONNX format. + Note that the exported model contains custom ops only available in caffe2, therefore it + cannot be directly executed by other runtime (such as onnxruntime or TensorRT). + Post-processing or transformation passes may be applied on the model to accommodate + different runtimes, but we currently do not provide support for them. + + Returns: + onnx.ModelProto: an onnx model. + """ + from .caffe2_export import export_onnx_model as export_onnx_model_impl + + return export_onnx_model_impl(self.traceable_model, (self.traceable_inputs,)) + + def export_torchscript(self): + """ + Export the model to a ``torch.jit.TracedModule`` by tracing. + The returned object can be saved to a file by ``.save()``. + + Returns: + torch.jit.TracedModule: a torch TracedModule + """ + logger = logging.getLogger(__name__) + logger.info("Tracing the model with torch.jit.trace ...") + with torch.no_grad(): + return torch.jit.trace(self.traceable_model, (self.traceable_inputs,)) + + +class Caffe2Model(nn.Module): + """ + A wrapper around the traced model in Caffe2's protobuf format. + The exported graph has different inputs/outputs from the original Pytorch + model, as explained in :class:`Caffe2Tracer`. This class wraps around the + exported graph to simulate the same interface as the original Pytorch model. + It also provides functions to save/load models in Caffe2's format.' + + Examples: + :: + c2_model = Caffe2Tracer(cfg, torch_model, inputs).export_caffe2() + inputs = [{"image": img_tensor_CHW}] + outputs = c2_model(inputs) + orig_outputs = torch_model(inputs) + """ + + def __init__(self, predict_net, init_net): + super().__init__() + self.eval() # always in eval mode + self._predict_net = predict_net + self._init_net = init_net + self._predictor = None + + __init__.__HIDE_SPHINX_DOC__ = True + + @property + def predict_net(self): + """ + caffe2.core.Net: the underlying caffe2 predict net + """ + return self._predict_net + + @property + def init_net(self): + """ + caffe2.core.Net: the underlying caffe2 init net + """ + return self._init_net + + def save_protobuf(self, output_dir): + """ + Save the model as caffe2's protobuf format. + It saves the following files: + + * "model.pb": definition of the graph. Can be visualized with + tools like `netron `_. + * "model_init.pb": model parameters + * "model.pbtxt": human-readable definition of the graph. Not + needed for deployment. + + Args: + output_dir (str): the output directory to save protobuf files. + """ + logger = logging.getLogger(__name__) + logger.info("Saving model to {} ...".format(output_dir)) + if not PathManager.exists(output_dir): + PathManager.mkdirs(output_dir) + + with PathManager.open(os.path.join(output_dir, "model.pb"), "wb") as f: + f.write(self._predict_net.SerializeToString()) + with PathManager.open(os.path.join(output_dir, "model.pbtxt"), "w") as f: + f.write(str(self._predict_net)) + with PathManager.open(os.path.join(output_dir, "model_init.pb"), "wb") as f: + f.write(self._init_net.SerializeToString()) + + def save_graph(self, output_file, inputs=None): + """ + Save the graph as SVG format. + + Args: + output_file (str): a SVG file + inputs: optional inputs given to the model. + If given, the inputs will be used to run the graph to record + shape of every tensor. The shape information will be + saved together with the graph. + """ + from .caffe2_export import run_and_save_graph + + if inputs is None: + save_graph(self._predict_net, output_file, op_only=False) + else: + size_divisibility = get_pb_arg_vali(self._predict_net, "size_divisibility", 0) + device = get_pb_arg_vals(self._predict_net, "device", b"cpu").decode("ascii") + inputs = convert_batched_inputs_to_c2_format(inputs, size_divisibility, device) + inputs = [x.cpu().numpy() for x in inputs] + run_and_save_graph(self._predict_net, self._init_net, inputs, output_file) + + @staticmethod + def load_protobuf(dir): + """ + Args: + dir (str): a directory used to save Caffe2Model with + :meth:`save_protobuf`. + The files "model.pb" and "model_init.pb" are needed. + + Returns: + Caffe2Model: the caffe2 model loaded from this directory. + """ + predict_net = caffe2_pb2.NetDef() + with PathManager.open(os.path.join(dir, "model.pb"), "rb") as f: + predict_net.ParseFromString(f.read()) + + init_net = caffe2_pb2.NetDef() + with PathManager.open(os.path.join(dir, "model_init.pb"), "rb") as f: + init_net.ParseFromString(f.read()) + + return Caffe2Model(predict_net, init_net) + + def __call__(self, inputs): + """ + An interface that wraps around a Caffe2 model and mimics detectron2's models' + input/output format. See details about the format at :doc:`/tutorials/models`. + This is used to compare the outputs of caffe2 model with its original torch model. + + Due to the extra conversion between Pytorch/Caffe2, this method is not meant for + benchmark. Because of the conversion, this method also has dependency + on detectron2 in order to convert to detectron2's output format. + """ + if self._predictor is None: + self._predictor = ProtobufDetectionModel(self._predict_net, self._init_net) + return self._predictor(inputs) diff --git a/annotator/oneformer/detectron2/export/c10.py b/annotator/oneformer/detectron2/export/c10.py new file mode 100644 index 0000000000000000000000000000000000000000..fde3fb71189e6f1061e83b878bfdd16add7d8350 --- /dev/null +++ b/annotator/oneformer/detectron2/export/c10.py @@ -0,0 +1,557 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import math +from typing import Dict +import torch +import torch.nn.functional as F + +from annotator.oneformer.detectron2.layers import ShapeSpec, cat +from annotator.oneformer.detectron2.layers.roi_align_rotated import ROIAlignRotated +from annotator.oneformer.detectron2.modeling import poolers +from annotator.oneformer.detectron2.modeling.proposal_generator import rpn +from annotator.oneformer.detectron2.modeling.roi_heads.mask_head import mask_rcnn_inference +from annotator.oneformer.detectron2.structures import Boxes, ImageList, Instances, Keypoints, RotatedBoxes + +from .shared import alias, to_device + + +""" +This file contains caffe2-compatible implementation of several detectron2 components. +""" + + +class Caffe2Boxes(Boxes): + """ + Representing a list of detectron2.structures.Boxes from minibatch, each box + is represented by a 5d vector (batch index + 4 coordinates), or a 6d vector + (batch index + 5 coordinates) for RotatedBoxes. + """ + + def __init__(self, tensor): + assert isinstance(tensor, torch.Tensor) + assert tensor.dim() == 2 and tensor.size(-1) in [4, 5, 6], tensor.size() + # TODO: make tensor immutable when dim is Nx5 for Boxes, + # and Nx6 for RotatedBoxes? + self.tensor = tensor + + +# TODO clean up this class, maybe just extend Instances +class InstancesList(object): + """ + Tensor representation of a list of Instances object for a batch of images. + + When dealing with a batch of images with Caffe2 ops, a list of bboxes + (instances) are usually represented by single Tensor with size + (sigma(Ni), 5) or (sigma(Ni), 4) plus a batch split Tensor. This class is + for providing common functions to convert between these two representations. + """ + + def __init__(self, im_info, indices, extra_fields=None): + # [N, 3] -> (H, W, Scale) + self.im_info = im_info + # [N,] -> indice of batch to which the instance belongs + self.indices = indices + # [N, ...] + self.batch_extra_fields = extra_fields or {} + + self.image_size = self.im_info + + def get_fields(self): + """like `get_fields` in the Instances object, + but return each field in tensor representations""" + ret = {} + for k, v in self.batch_extra_fields.items(): + # if isinstance(v, torch.Tensor): + # tensor_rep = v + # elif isinstance(v, (Boxes, Keypoints)): + # tensor_rep = v.tensor + # else: + # raise ValueError("Can't find tensor representation for: {}".format()) + ret[k] = v + return ret + + def has(self, name): + return name in self.batch_extra_fields + + def set(self, name, value): + # len(tensor) is a bad practice that generates ONNX constants during tracing. + # Although not a problem for the `assert` statement below, torch ONNX exporter + # still raises a misleading warning as it does not this call comes from `assert` + if isinstance(value, Boxes): + data_len = value.tensor.shape[0] + elif isinstance(value, torch.Tensor): + data_len = value.shape[0] + else: + data_len = len(value) + if len(self.batch_extra_fields): + assert ( + len(self) == data_len + ), "Adding a field of length {} to a Instances of length {}".format(data_len, len(self)) + self.batch_extra_fields[name] = value + + def __getattr__(self, name): + if name not in self.batch_extra_fields: + raise AttributeError("Cannot find field '{}' in the given Instances!".format(name)) + return self.batch_extra_fields[name] + + def __len__(self): + return len(self.indices) + + def flatten(self): + ret = [] + for _, v in self.batch_extra_fields.items(): + if isinstance(v, (Boxes, Keypoints)): + ret.append(v.tensor) + else: + ret.append(v) + return ret + + @staticmethod + def to_d2_instances_list(instances_list): + """ + Convert InstancesList to List[Instances]. The input `instances_list` can + also be a List[Instances], in this case this method is a non-op. + """ + if not isinstance(instances_list, InstancesList): + assert all(isinstance(x, Instances) for x in instances_list) + return instances_list + + ret = [] + for i, info in enumerate(instances_list.im_info): + instances = Instances(torch.Size([int(info[0].item()), int(info[1].item())])) + + ids = instances_list.indices == i + for k, v in instances_list.batch_extra_fields.items(): + if isinstance(v, torch.Tensor): + instances.set(k, v[ids]) + continue + elif isinstance(v, Boxes): + instances.set(k, v[ids, -4:]) + continue + + target_type, tensor_source = v + assert isinstance(tensor_source, torch.Tensor) + assert tensor_source.shape[0] == instances_list.indices.shape[0] + tensor_source = tensor_source[ids] + + if issubclass(target_type, Boxes): + instances.set(k, Boxes(tensor_source[:, -4:])) + elif issubclass(target_type, Keypoints): + instances.set(k, Keypoints(tensor_source)) + elif issubclass(target_type, torch.Tensor): + instances.set(k, tensor_source) + else: + raise ValueError("Can't handle targe type: {}".format(target_type)) + + ret.append(instances) + return ret + + +class Caffe2Compatible(object): + """ + A model can inherit this class to indicate that it can be traced and deployed with caffe2. + """ + + def _get_tensor_mode(self): + return self._tensor_mode + + def _set_tensor_mode(self, v): + self._tensor_mode = v + + tensor_mode = property(_get_tensor_mode, _set_tensor_mode) + """ + If true, the model expects C2-style tensor only inputs/outputs format. + """ + + +class Caffe2RPN(Caffe2Compatible, rpn.RPN): + @classmethod + def from_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): + ret = super(Caffe2Compatible, cls).from_config(cfg, input_shape) + assert tuple(cfg.MODEL.RPN.BBOX_REG_WEIGHTS) == (1.0, 1.0, 1.0, 1.0) or tuple( + cfg.MODEL.RPN.BBOX_REG_WEIGHTS + ) == (1.0, 1.0, 1.0, 1.0, 1.0) + return ret + + def _generate_proposals( + self, images, objectness_logits_pred, anchor_deltas_pred, gt_instances=None + ): + assert isinstance(images, ImageList) + if self.tensor_mode: + im_info = images.image_sizes + else: + im_info = torch.tensor([[im_sz[0], im_sz[1], 1.0] for im_sz in images.image_sizes]).to( + images.tensor.device + ) + assert isinstance(im_info, torch.Tensor) + + rpn_rois_list = [] + rpn_roi_probs_list = [] + for scores, bbox_deltas, cell_anchors_tensor, feat_stride in zip( + objectness_logits_pred, + anchor_deltas_pred, + [b for (n, b) in self.anchor_generator.cell_anchors.named_buffers()], + self.anchor_generator.strides, + ): + scores = scores.detach() + bbox_deltas = bbox_deltas.detach() + + rpn_rois, rpn_roi_probs = torch.ops._caffe2.GenerateProposals( + scores, + bbox_deltas, + im_info, + cell_anchors_tensor, + spatial_scale=1.0 / feat_stride, + pre_nms_topN=self.pre_nms_topk[self.training], + post_nms_topN=self.post_nms_topk[self.training], + nms_thresh=self.nms_thresh, + min_size=self.min_box_size, + # correct_transform_coords=True, # deprecated argument + angle_bound_on=True, # Default + angle_bound_lo=-180, + angle_bound_hi=180, + clip_angle_thresh=1.0, # Default + legacy_plus_one=False, + ) + rpn_rois_list.append(rpn_rois) + rpn_roi_probs_list.append(rpn_roi_probs) + + # For FPN in D2, in RPN all proposals from different levels are concated + # together, ranked and picked by top post_nms_topk. Then in ROIPooler + # it calculates level_assignments and calls the RoIAlign from + # the corresponding level. + + if len(objectness_logits_pred) == 1: + rpn_rois = rpn_rois_list[0] + rpn_roi_probs = rpn_roi_probs_list[0] + else: + assert len(rpn_rois_list) == len(rpn_roi_probs_list) + rpn_post_nms_topN = self.post_nms_topk[self.training] + + device = rpn_rois_list[0].device + input_list = [to_device(x, "cpu") for x in (rpn_rois_list + rpn_roi_probs_list)] + + # TODO remove this after confirming rpn_max_level/rpn_min_level + # is not needed in CollectRpnProposals. + feature_strides = list(self.anchor_generator.strides) + rpn_min_level = int(math.log2(feature_strides[0])) + rpn_max_level = int(math.log2(feature_strides[-1])) + assert (rpn_max_level - rpn_min_level + 1) == len( + rpn_rois_list + ), "CollectRpnProposals requires continuous levels" + + rpn_rois = torch.ops._caffe2.CollectRpnProposals( + input_list, + # NOTE: in current implementation, rpn_max_level and rpn_min_level + # are not needed, only the subtraction of two matters and it + # can be infer from the number of inputs. Keep them now for + # consistency. + rpn_max_level=2 + len(rpn_rois_list) - 1, + rpn_min_level=2, + rpn_post_nms_topN=rpn_post_nms_topN, + ) + rpn_rois = to_device(rpn_rois, device) + rpn_roi_probs = [] + + proposals = self.c2_postprocess(im_info, rpn_rois, rpn_roi_probs, self.tensor_mode) + return proposals, {} + + def forward(self, images, features, gt_instances=None): + assert not self.training + features = [features[f] for f in self.in_features] + objectness_logits_pred, anchor_deltas_pred = self.rpn_head(features) + return self._generate_proposals( + images, + objectness_logits_pred, + anchor_deltas_pred, + gt_instances, + ) + + @staticmethod + def c2_postprocess(im_info, rpn_rois, rpn_roi_probs, tensor_mode): + proposals = InstancesList( + im_info=im_info, + indices=rpn_rois[:, 0], + extra_fields={ + "proposal_boxes": Caffe2Boxes(rpn_rois), + "objectness_logits": (torch.Tensor, rpn_roi_probs), + }, + ) + if not tensor_mode: + proposals = InstancesList.to_d2_instances_list(proposals) + else: + proposals = [proposals] + return proposals + + +class Caffe2ROIPooler(Caffe2Compatible, poolers.ROIPooler): + @staticmethod + def c2_preprocess(box_lists): + assert all(isinstance(x, Boxes) for x in box_lists) + if all(isinstance(x, Caffe2Boxes) for x in box_lists): + # input is pure-tensor based + assert len(box_lists) == 1 + pooler_fmt_boxes = box_lists[0].tensor + else: + pooler_fmt_boxes = poolers.convert_boxes_to_pooler_format(box_lists) + return pooler_fmt_boxes + + def forward(self, x, box_lists): + assert not self.training + + pooler_fmt_boxes = self.c2_preprocess(box_lists) + num_level_assignments = len(self.level_poolers) + + if num_level_assignments == 1: + if isinstance(self.level_poolers[0], ROIAlignRotated): + c2_roi_align = torch.ops._caffe2.RoIAlignRotated + aligned = True + else: + c2_roi_align = torch.ops._caffe2.RoIAlign + aligned = self.level_poolers[0].aligned + + x0 = x[0] + if x0.is_quantized: + x0 = x0.dequantize() + + out = c2_roi_align( + x0, + pooler_fmt_boxes, + order="NCHW", + spatial_scale=float(self.level_poolers[0].spatial_scale), + pooled_h=int(self.output_size[0]), + pooled_w=int(self.output_size[1]), + sampling_ratio=int(self.level_poolers[0].sampling_ratio), + aligned=aligned, + ) + return out + + device = pooler_fmt_boxes.device + assert ( + self.max_level - self.min_level + 1 == 4 + ), "Currently DistributeFpnProposals only support 4 levels" + fpn_outputs = torch.ops._caffe2.DistributeFpnProposals( + to_device(pooler_fmt_boxes, "cpu"), + roi_canonical_scale=self.canonical_box_size, + roi_canonical_level=self.canonical_level, + roi_max_level=self.max_level, + roi_min_level=self.min_level, + legacy_plus_one=False, + ) + fpn_outputs = [to_device(x, device) for x in fpn_outputs] + + rois_fpn_list = fpn_outputs[:-1] + rois_idx_restore_int32 = fpn_outputs[-1] + + roi_feat_fpn_list = [] + for roi_fpn, x_level, pooler in zip(rois_fpn_list, x, self.level_poolers): + if isinstance(pooler, ROIAlignRotated): + c2_roi_align = torch.ops._caffe2.RoIAlignRotated + aligned = True + else: + c2_roi_align = torch.ops._caffe2.RoIAlign + aligned = bool(pooler.aligned) + + if x_level.is_quantized: + x_level = x_level.dequantize() + + roi_feat_fpn = c2_roi_align( + x_level, + roi_fpn, + order="NCHW", + spatial_scale=float(pooler.spatial_scale), + pooled_h=int(self.output_size[0]), + pooled_w=int(self.output_size[1]), + sampling_ratio=int(pooler.sampling_ratio), + aligned=aligned, + ) + roi_feat_fpn_list.append(roi_feat_fpn) + + roi_feat_shuffled = cat(roi_feat_fpn_list, dim=0) + assert roi_feat_shuffled.numel() > 0 and rois_idx_restore_int32.numel() > 0, ( + "Caffe2 export requires tracing with a model checkpoint + input that can produce valid" + " detections. But no detections were obtained with the given checkpoint and input!" + ) + roi_feat = torch.ops._caffe2.BatchPermutation(roi_feat_shuffled, rois_idx_restore_int32) + return roi_feat + + +class Caffe2FastRCNNOutputsInference: + def __init__(self, tensor_mode): + self.tensor_mode = tensor_mode # whether the output is caffe2 tensor mode + + def __call__(self, box_predictor, predictions, proposals): + """equivalent to FastRCNNOutputLayers.inference""" + num_classes = box_predictor.num_classes + score_thresh = box_predictor.test_score_thresh + nms_thresh = box_predictor.test_nms_thresh + topk_per_image = box_predictor.test_topk_per_image + is_rotated = len(box_predictor.box2box_transform.weights) == 5 + + if is_rotated: + box_dim = 5 + assert box_predictor.box2box_transform.weights[4] == 1, ( + "The weights for Rotated BBoxTransform in C2 have only 4 dimensions," + + " thus enforcing the angle weight to be 1 for now" + ) + box2box_transform_weights = box_predictor.box2box_transform.weights[:4] + else: + box_dim = 4 + box2box_transform_weights = box_predictor.box2box_transform.weights + + class_logits, box_regression = predictions + if num_classes + 1 == class_logits.shape[1]: + class_prob = F.softmax(class_logits, -1) + else: + assert num_classes == class_logits.shape[1] + class_prob = F.sigmoid(class_logits) + # BoxWithNMSLimit will infer num_classes from the shape of the class_prob + # So append a zero column as placeholder for the background class + class_prob = torch.cat((class_prob, torch.zeros(class_prob.shape[0], 1)), dim=1) + + assert box_regression.shape[1] % box_dim == 0 + cls_agnostic_bbox_reg = box_regression.shape[1] // box_dim == 1 + + input_tensor_mode = proposals[0].proposal_boxes.tensor.shape[1] == box_dim + 1 + + proposal_boxes = proposals[0].proposal_boxes + if isinstance(proposal_boxes, Caffe2Boxes): + rois = Caffe2Boxes.cat([p.proposal_boxes for p in proposals]) + elif isinstance(proposal_boxes, RotatedBoxes): + rois = RotatedBoxes.cat([p.proposal_boxes for p in proposals]) + elif isinstance(proposal_boxes, Boxes): + rois = Boxes.cat([p.proposal_boxes for p in proposals]) + else: + raise NotImplementedError( + 'Expected proposals[0].proposal_boxes to be type "Boxes", ' + f"instead got {type(proposal_boxes)}" + ) + + device, dtype = rois.tensor.device, rois.tensor.dtype + if input_tensor_mode: + im_info = proposals[0].image_size + rois = rois.tensor + else: + im_info = torch.tensor( + [[sz[0], sz[1], 1.0] for sz in [x.image_size for x in proposals]] + ) + batch_ids = cat( + [ + torch.full((b, 1), i, dtype=dtype, device=device) + for i, b in enumerate(len(p) for p in proposals) + ], + dim=0, + ) + rois = torch.cat([batch_ids, rois.tensor], dim=1) + + roi_pred_bbox, roi_batch_splits = torch.ops._caffe2.BBoxTransform( + to_device(rois, "cpu"), + to_device(box_regression, "cpu"), + to_device(im_info, "cpu"), + weights=box2box_transform_weights, + apply_scale=True, + rotated=is_rotated, + angle_bound_on=True, + angle_bound_lo=-180, + angle_bound_hi=180, + clip_angle_thresh=1.0, + legacy_plus_one=False, + ) + roi_pred_bbox = to_device(roi_pred_bbox, device) + roi_batch_splits = to_device(roi_batch_splits, device) + + nms_outputs = torch.ops._caffe2.BoxWithNMSLimit( + to_device(class_prob, "cpu"), + to_device(roi_pred_bbox, "cpu"), + to_device(roi_batch_splits, "cpu"), + score_thresh=float(score_thresh), + nms=float(nms_thresh), + detections_per_im=int(topk_per_image), + soft_nms_enabled=False, + soft_nms_method="linear", + soft_nms_sigma=0.5, + soft_nms_min_score_thres=0.001, + rotated=is_rotated, + cls_agnostic_bbox_reg=cls_agnostic_bbox_reg, + input_boxes_include_bg_cls=False, + output_classes_include_bg_cls=False, + legacy_plus_one=False, + ) + roi_score_nms = to_device(nms_outputs[0], device) + roi_bbox_nms = to_device(nms_outputs[1], device) + roi_class_nms = to_device(nms_outputs[2], device) + roi_batch_splits_nms = to_device(nms_outputs[3], device) + roi_keeps_nms = to_device(nms_outputs[4], device) + roi_keeps_size_nms = to_device(nms_outputs[5], device) + if not self.tensor_mode: + roi_class_nms = roi_class_nms.to(torch.int64) + + roi_batch_ids = cat( + [ + torch.full((b, 1), i, dtype=dtype, device=device) + for i, b in enumerate(int(x.item()) for x in roi_batch_splits_nms) + ], + dim=0, + ) + + roi_class_nms = alias(roi_class_nms, "class_nms") + roi_score_nms = alias(roi_score_nms, "score_nms") + roi_bbox_nms = alias(roi_bbox_nms, "bbox_nms") + roi_batch_splits_nms = alias(roi_batch_splits_nms, "batch_splits_nms") + roi_keeps_nms = alias(roi_keeps_nms, "keeps_nms") + roi_keeps_size_nms = alias(roi_keeps_size_nms, "keeps_size_nms") + + results = InstancesList( + im_info=im_info, + indices=roi_batch_ids[:, 0], + extra_fields={ + "pred_boxes": Caffe2Boxes(roi_bbox_nms), + "scores": roi_score_nms, + "pred_classes": roi_class_nms, + }, + ) + + if not self.tensor_mode: + results = InstancesList.to_d2_instances_list(results) + batch_splits = roi_batch_splits_nms.int().tolist() + kept_indices = list(roi_keeps_nms.to(torch.int64).split(batch_splits)) + else: + results = [results] + kept_indices = [roi_keeps_nms] + + return results, kept_indices + + +class Caffe2MaskRCNNInference: + def __call__(self, pred_mask_logits, pred_instances): + """equivalent to mask_head.mask_rcnn_inference""" + if all(isinstance(x, InstancesList) for x in pred_instances): + assert len(pred_instances) == 1 + mask_probs_pred = pred_mask_logits.sigmoid() + mask_probs_pred = alias(mask_probs_pred, "mask_fcn_probs") + pred_instances[0].set("pred_masks", mask_probs_pred) + else: + mask_rcnn_inference(pred_mask_logits, pred_instances) + + +class Caffe2KeypointRCNNInference: + def __init__(self, use_heatmap_max_keypoint): + self.use_heatmap_max_keypoint = use_heatmap_max_keypoint + + def __call__(self, pred_keypoint_logits, pred_instances): + # just return the keypoint heatmap for now, + # there will be option to call HeatmapMaxKeypointOp + output = alias(pred_keypoint_logits, "kps_score") + if all(isinstance(x, InstancesList) for x in pred_instances): + assert len(pred_instances) == 1 + if self.use_heatmap_max_keypoint: + device = output.device + output = torch.ops._caffe2.HeatmapMaxKeypoint( + to_device(output, "cpu"), + pred_instances[0].pred_boxes.tensor, + should_output_softmax=True, # worth make it configerable? + ) + output = to_device(output, device) + output = alias(output, "keypoints_out") + pred_instances[0].set("pred_keypoints", output) + return pred_keypoint_logits diff --git a/annotator/oneformer/detectron2/export/caffe2_export.py b/annotator/oneformer/detectron2/export/caffe2_export.py new file mode 100644 index 0000000000000000000000000000000000000000..d609c27c7deb396352967dbcbc79b1e00f2a2de1 --- /dev/null +++ b/annotator/oneformer/detectron2/export/caffe2_export.py @@ -0,0 +1,203 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import copy +import io +import logging +import numpy as np +from typing import List +import onnx +import onnx.optimizer +import torch +from caffe2.proto import caffe2_pb2 +from caffe2.python import core +from caffe2.python.onnx.backend import Caffe2Backend +from tabulate import tabulate +from termcolor import colored +from torch.onnx import OperatorExportTypes + +from .shared import ( + ScopedWS, + construct_init_net_from_params, + fuse_alias_placeholder, + fuse_copy_between_cpu_and_gpu, + get_params_from_init_net, + group_norm_replace_aten_with_caffe2, + infer_device_type, + remove_dead_end_ops, + remove_reshape_for_fc, + save_graph, +) + +logger = logging.getLogger(__name__) + + +def export_onnx_model(model, inputs): + """ + Trace and export a model to onnx format. + + Args: + model (nn.Module): + inputs (tuple[args]): the model will be called by `model(*inputs)` + + Returns: + an onnx model + """ + assert isinstance(model, torch.nn.Module) + + # make sure all modules are in eval mode, onnx may change the training state + # of the module if the states are not consistent + def _check_eval(module): + assert not module.training + + model.apply(_check_eval) + + # Export the model to ONNX + with torch.no_grad(): + with io.BytesIO() as f: + torch.onnx.export( + model, + inputs, + f, + operator_export_type=OperatorExportTypes.ONNX_ATEN_FALLBACK, + # verbose=True, # NOTE: uncomment this for debugging + # export_params=True, + ) + onnx_model = onnx.load_from_string(f.getvalue()) + + return onnx_model + + +def _op_stats(net_def): + type_count = {} + for t in [op.type for op in net_def.op]: + type_count[t] = type_count.get(t, 0) + 1 + type_count_list = sorted(type_count.items(), key=lambda kv: kv[0]) # alphabet + type_count_list = sorted(type_count_list, key=lambda kv: -kv[1]) # count + return "\n".join("{:>4}x {}".format(count, name) for name, count in type_count_list) + + +def _assign_device_option( + predict_net: caffe2_pb2.NetDef, init_net: caffe2_pb2.NetDef, tensor_inputs: List[torch.Tensor] +): + """ + ONNX exported network doesn't have concept of device, assign necessary + device option for each op in order to make it runable on GPU runtime. + """ + + def _get_device_type(torch_tensor): + assert torch_tensor.device.type in ["cpu", "cuda"] + assert torch_tensor.device.index == 0 + return torch_tensor.device.type + + def _assign_op_device_option(net_proto, net_ssa, blob_device_types): + for op, ssa_i in zip(net_proto.op, net_ssa): + if op.type in ["CopyCPUToGPU", "CopyGPUToCPU"]: + op.device_option.CopyFrom(core.DeviceOption(caffe2_pb2.CUDA, 0)) + else: + devices = [blob_device_types[b] for b in ssa_i[0] + ssa_i[1]] + assert all(d == devices[0] for d in devices) + if devices[0] == "cuda": + op.device_option.CopyFrom(core.DeviceOption(caffe2_pb2.CUDA, 0)) + + # update ops in predict_net + predict_net_input_device_types = { + (name, 0): _get_device_type(tensor) + for name, tensor in zip(predict_net.external_input, tensor_inputs) + } + predict_net_device_types = infer_device_type( + predict_net, known_status=predict_net_input_device_types, device_name_style="pytorch" + ) + predict_net_ssa, _ = core.get_ssa(predict_net) + _assign_op_device_option(predict_net, predict_net_ssa, predict_net_device_types) + + # update ops in init_net + init_net_ssa, versions = core.get_ssa(init_net) + init_net_output_device_types = { + (name, versions[name]): predict_net_device_types[(name, 0)] + for name in init_net.external_output + } + init_net_device_types = infer_device_type( + init_net, known_status=init_net_output_device_types, device_name_style="pytorch" + ) + _assign_op_device_option(init_net, init_net_ssa, init_net_device_types) + + +def export_caffe2_detection_model(model: torch.nn.Module, tensor_inputs: List[torch.Tensor]): + """ + Export a caffe2-compatible Detectron2 model to caffe2 format via ONNX. + + Arg: + model: a caffe2-compatible version of detectron2 model, defined in caffe2_modeling.py + tensor_inputs: a list of tensors that caffe2 model takes as input. + """ + model = copy.deepcopy(model) + assert isinstance(model, torch.nn.Module) + assert hasattr(model, "encode_additional_info") + + # Export via ONNX + logger.info( + "Exporting a {} model via ONNX ...".format(type(model).__name__) + + " Some warnings from ONNX are expected and are usually not to worry about." + ) + onnx_model = export_onnx_model(model, (tensor_inputs,)) + # Convert ONNX model to Caffe2 protobuf + init_net, predict_net = Caffe2Backend.onnx_graph_to_caffe2_net(onnx_model) + ops_table = [[op.type, op.input, op.output] for op in predict_net.op] + table = tabulate(ops_table, headers=["type", "input", "output"], tablefmt="pipe") + logger.info( + "ONNX export Done. Exported predict_net (before optimizations):\n" + colored(table, "cyan") + ) + + # Apply protobuf optimization + fuse_alias_placeholder(predict_net, init_net) + if any(t.device.type != "cpu" for t in tensor_inputs): + fuse_copy_between_cpu_and_gpu(predict_net) + remove_dead_end_ops(init_net) + _assign_device_option(predict_net, init_net, tensor_inputs) + params, device_options = get_params_from_init_net(init_net) + predict_net, params = remove_reshape_for_fc(predict_net, params) + init_net = construct_init_net_from_params(params, device_options) + group_norm_replace_aten_with_caffe2(predict_net) + + # Record necessary information for running the pb model in Detectron2 system. + model.encode_additional_info(predict_net, init_net) + + logger.info("Operators used in predict_net: \n{}".format(_op_stats(predict_net))) + logger.info("Operators used in init_net: \n{}".format(_op_stats(init_net))) + + return predict_net, init_net + + +def run_and_save_graph(predict_net, init_net, tensor_inputs, graph_save_path): + """ + Run the caffe2 model on given inputs, recording the shape and draw the graph. + + predict_net/init_net: caffe2 model. + tensor_inputs: a list of tensors that caffe2 model takes as input. + graph_save_path: path for saving graph of exported model. + """ + + logger.info("Saving graph of ONNX exported model to {} ...".format(graph_save_path)) + save_graph(predict_net, graph_save_path, op_only=False) + + # Run the exported Caffe2 net + logger.info("Running ONNX exported model ...") + with ScopedWS("__ws_tmp__", True) as ws: + ws.RunNetOnce(init_net) + initialized_blobs = set(ws.Blobs()) + uninitialized = [inp for inp in predict_net.external_input if inp not in initialized_blobs] + for name, blob in zip(uninitialized, tensor_inputs): + ws.FeedBlob(name, blob) + + try: + ws.RunNetOnce(predict_net) + except RuntimeError as e: + logger.warning("Encountered RuntimeError: \n{}".format(str(e))) + + ws_blobs = {b: ws.FetchBlob(b) for b in ws.Blobs()} + blob_sizes = {b: ws_blobs[b].shape for b in ws_blobs if isinstance(ws_blobs[b], np.ndarray)} + + logger.info("Saving graph with blob shapes to {} ...".format(graph_save_path)) + save_graph(predict_net, graph_save_path, op_only=False, blob_sizes=blob_sizes) + + return ws_blobs diff --git a/annotator/oneformer/detectron2/export/caffe2_inference.py b/annotator/oneformer/detectron2/export/caffe2_inference.py new file mode 100644 index 0000000000000000000000000000000000000000..deb886c0417285ed1d5ad85eb941fa1ac757cdab --- /dev/null +++ b/annotator/oneformer/detectron2/export/caffe2_inference.py @@ -0,0 +1,161 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import logging +import numpy as np +from itertools import count +import torch +from caffe2.proto import caffe2_pb2 +from caffe2.python import core + +from .caffe2_modeling import META_ARCH_CAFFE2_EXPORT_TYPE_MAP, convert_batched_inputs_to_c2_format +from .shared import ScopedWS, get_pb_arg_vali, get_pb_arg_vals, infer_device_type + +logger = logging.getLogger(__name__) + + +# ===== ref: mobile-vision predictor's 'Caffe2Wrapper' class ====== +class ProtobufModel(torch.nn.Module): + """ + Wrapper of a caffe2's protobuf model. + It works just like nn.Module, but running caffe2 under the hood. + Input/Output are tuple[tensor] that match the caffe2 net's external_input/output. + """ + + _ids = count(0) + + def __init__(self, predict_net, init_net): + logger.info(f"Initializing ProtobufModel for: {predict_net.name} ...") + super().__init__() + assert isinstance(predict_net, caffe2_pb2.NetDef) + assert isinstance(init_net, caffe2_pb2.NetDef) + # create unique temporary workspace for each instance + self.ws_name = "__tmp_ProtobufModel_{}__".format(next(self._ids)) + self.net = core.Net(predict_net) + + logger.info("Running init_net once to fill the parameters ...") + with ScopedWS(self.ws_name, is_reset=True, is_cleanup=False) as ws: + ws.RunNetOnce(init_net) + uninitialized_external_input = [] + for blob in self.net.Proto().external_input: + if blob not in ws.Blobs(): + uninitialized_external_input.append(blob) + ws.CreateBlob(blob) + ws.CreateNet(self.net) + + self._error_msgs = set() + self._input_blobs = uninitialized_external_input + + def _infer_output_devices(self, inputs): + """ + Returns: + list[str]: list of device for each external output + """ + + def _get_device_type(torch_tensor): + assert torch_tensor.device.type in ["cpu", "cuda"] + assert torch_tensor.device.index == 0 + return torch_tensor.device.type + + predict_net = self.net.Proto() + input_device_types = { + (name, 0): _get_device_type(tensor) for name, tensor in zip(self._input_blobs, inputs) + } + device_type_map = infer_device_type( + predict_net, known_status=input_device_types, device_name_style="pytorch" + ) + ssa, versions = core.get_ssa(predict_net) + versioned_outputs = [(name, versions[name]) for name in predict_net.external_output] + output_devices = [device_type_map[outp] for outp in versioned_outputs] + return output_devices + + def forward(self, inputs): + """ + Args: + inputs (tuple[torch.Tensor]) + + Returns: + tuple[torch.Tensor] + """ + assert len(inputs) == len(self._input_blobs), ( + f"Length of inputs ({len(inputs)}) " + f"doesn't match the required input blobs: {self._input_blobs}" + ) + + with ScopedWS(self.ws_name, is_reset=False, is_cleanup=False) as ws: + for b, tensor in zip(self._input_blobs, inputs): + ws.FeedBlob(b, tensor) + + try: + ws.RunNet(self.net.Proto().name) + except RuntimeError as e: + if not str(e) in self._error_msgs: + self._error_msgs.add(str(e)) + logger.warning("Encountered new RuntimeError: \n{}".format(str(e))) + logger.warning("Catch the error and use partial results.") + + c2_outputs = [ws.FetchBlob(b) for b in self.net.Proto().external_output] + # Remove outputs of current run, this is necessary in order to + # prevent fetching the result from previous run if the model fails + # in the middle. + for b in self.net.Proto().external_output: + # Needs to create uninitialized blob to make the net runable. + # This is "equivalent" to: ws.RemoveBlob(b) then ws.CreateBlob(b), + # but there'no such API. + ws.FeedBlob(b, f"{b}, a C++ native class of type nullptr (uninitialized).") + + # Cast output to torch.Tensor on the desired device + output_devices = ( + self._infer_output_devices(inputs) + if any(t.device.type != "cpu" for t in inputs) + else ["cpu" for _ in self.net.Proto().external_output] + ) + + outputs = [] + for name, c2_output, device in zip( + self.net.Proto().external_output, c2_outputs, output_devices + ): + if not isinstance(c2_output, np.ndarray): + raise RuntimeError( + "Invalid output for blob {}, received: {}".format(name, c2_output) + ) + outputs.append(torch.tensor(c2_output).to(device=device)) + return tuple(outputs) + + +class ProtobufDetectionModel(torch.nn.Module): + """ + A class works just like a pytorch meta arch in terms of inference, but running + caffe2 model under the hood. + """ + + def __init__(self, predict_net, init_net, *, convert_outputs=None): + """ + Args: + predict_net, init_net (core.Net): caffe2 nets + convert_outptus (callable): a function that converts caffe2 + outputs to the same format of the original pytorch model. + By default, use the one defined in the caffe2 meta_arch. + """ + super().__init__() + self.protobuf_model = ProtobufModel(predict_net, init_net) + self.size_divisibility = get_pb_arg_vali(predict_net, "size_divisibility", 0) + self.device = get_pb_arg_vals(predict_net, "device", b"cpu").decode("ascii") + + if convert_outputs is None: + meta_arch = get_pb_arg_vals(predict_net, "meta_architecture", b"GeneralizedRCNN") + meta_arch = META_ARCH_CAFFE2_EXPORT_TYPE_MAP[meta_arch.decode("ascii")] + self._convert_outputs = meta_arch.get_outputs_converter(predict_net, init_net) + else: + self._convert_outputs = convert_outputs + + def _convert_inputs(self, batched_inputs): + # currently all models convert inputs in the same way + return convert_batched_inputs_to_c2_format( + batched_inputs, self.size_divisibility, self.device + ) + + def forward(self, batched_inputs): + c2_inputs = self._convert_inputs(batched_inputs) + c2_results = self.protobuf_model(c2_inputs) + c2_results = dict(zip(self.protobuf_model.net.Proto().external_output, c2_results)) + return self._convert_outputs(batched_inputs, c2_inputs, c2_results) diff --git a/annotator/oneformer/detectron2/export/caffe2_modeling.py b/annotator/oneformer/detectron2/export/caffe2_modeling.py new file mode 100644 index 0000000000000000000000000000000000000000..e0128e4672bc08eb2983d3d382614c6381baefd9 --- /dev/null +++ b/annotator/oneformer/detectron2/export/caffe2_modeling.py @@ -0,0 +1,419 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import functools +import io +import struct +import types +import torch + +from annotator.oneformer.detectron2.modeling import meta_arch +from annotator.oneformer.detectron2.modeling.box_regression import Box2BoxTransform +from annotator.oneformer.detectron2.modeling.roi_heads import keypoint_head +from annotator.oneformer.detectron2.structures import Boxes, ImageList, Instances, RotatedBoxes + +from .c10 import Caffe2Compatible +from .caffe2_patch import ROIHeadsPatcher, patch_generalized_rcnn +from .shared import ( + alias, + check_set_pb_arg, + get_pb_arg_floats, + get_pb_arg_valf, + get_pb_arg_vali, + get_pb_arg_vals, + mock_torch_nn_functional_interpolate, +) + + +def assemble_rcnn_outputs_by_name(image_sizes, tensor_outputs, force_mask_on=False): + """ + A function to assemble caffe2 model's outputs (i.e. Dict[str, Tensor]) + to detectron2's format (i.e. list of Instances instance). + This only works when the model follows the Caffe2 detectron's naming convention. + + Args: + image_sizes (List[List[int, int]]): [H, W] of every image. + tensor_outputs (Dict[str, Tensor]): external_output to its tensor. + + force_mask_on (Bool): if true, the it make sure there'll be pred_masks even + if the mask is not found from tensor_outputs (usually due to model crash) + """ + + results = [Instances(image_size) for image_size in image_sizes] + + batch_splits = tensor_outputs.get("batch_splits", None) + if batch_splits: + raise NotImplementedError() + assert len(image_sizes) == 1 + result = results[0] + + bbox_nms = tensor_outputs["bbox_nms"] + score_nms = tensor_outputs["score_nms"] + class_nms = tensor_outputs["class_nms"] + # Detection will always success because Conv support 0-batch + assert bbox_nms is not None + assert score_nms is not None + assert class_nms is not None + if bbox_nms.shape[1] == 5: + result.pred_boxes = RotatedBoxes(bbox_nms) + else: + result.pred_boxes = Boxes(bbox_nms) + result.scores = score_nms + result.pred_classes = class_nms.to(torch.int64) + + mask_fcn_probs = tensor_outputs.get("mask_fcn_probs", None) + if mask_fcn_probs is not None: + # finish the mask pred + mask_probs_pred = mask_fcn_probs + num_masks = mask_probs_pred.shape[0] + class_pred = result.pred_classes + indices = torch.arange(num_masks, device=class_pred.device) + mask_probs_pred = mask_probs_pred[indices, class_pred][:, None] + result.pred_masks = mask_probs_pred + elif force_mask_on: + # NOTE: there's no way to know the height/width of mask here, it won't be + # used anyway when batch size is 0, so just set them to 0. + result.pred_masks = torch.zeros([0, 1, 0, 0], dtype=torch.uint8) + + keypoints_out = tensor_outputs.get("keypoints_out", None) + kps_score = tensor_outputs.get("kps_score", None) + if keypoints_out is not None: + # keypoints_out: [N, 4, #kypoints], where 4 is in order of (x, y, score, prob) + keypoints_tensor = keypoints_out + # NOTE: it's possible that prob is not calculated if "should_output_softmax" + # is set to False in HeatmapMaxKeypoint, so just using raw score, seems + # it doesn't affect mAP. TODO: check more carefully. + keypoint_xyp = keypoints_tensor.transpose(1, 2)[:, :, [0, 1, 2]] + result.pred_keypoints = keypoint_xyp + elif kps_score is not None: + # keypoint heatmap to sparse data structure + pred_keypoint_logits = kps_score + keypoint_head.keypoint_rcnn_inference(pred_keypoint_logits, [result]) + + return results + + +def _cast_to_f32(f64): + return struct.unpack("f", struct.pack("f", f64))[0] + + +def set_caffe2_compatible_tensor_mode(model, enable=True): + def _fn(m): + if isinstance(m, Caffe2Compatible): + m.tensor_mode = enable + + model.apply(_fn) + + +def convert_batched_inputs_to_c2_format(batched_inputs, size_divisibility, device): + """ + See get_caffe2_inputs() below. + """ + assert all(isinstance(x, dict) for x in batched_inputs) + assert all(x["image"].dim() == 3 for x in batched_inputs) + + images = [x["image"] for x in batched_inputs] + images = ImageList.from_tensors(images, size_divisibility) + + im_info = [] + for input_per_image, image_size in zip(batched_inputs, images.image_sizes): + target_height = input_per_image.get("height", image_size[0]) + target_width = input_per_image.get("width", image_size[1]) # noqa + # NOTE: The scale inside im_info is kept as convention and for providing + # post-processing information if further processing is needed. For + # current Caffe2 model definitions that don't include post-processing inside + # the model, this number is not used. + # NOTE: There can be a slight difference between width and height + # scales, using a single number can results in numerical difference + # compared with D2's post-processing. + scale = target_height / image_size[0] + im_info.append([image_size[0], image_size[1], scale]) + im_info = torch.Tensor(im_info) + + return images.tensor.to(device), im_info.to(device) + + +class Caffe2MetaArch(Caffe2Compatible, torch.nn.Module): + """ + Base class for caffe2-compatible implementation of a meta architecture. + The forward is traceable and its traced graph can be converted to caffe2 + graph through ONNX. + """ + + def __init__(self, cfg, torch_model): + """ + Args: + cfg (CfgNode): + torch_model (nn.Module): the detectron2 model (meta_arch) to be + converted. + """ + super().__init__() + self._wrapped_model = torch_model + self.eval() + set_caffe2_compatible_tensor_mode(self, True) + + def get_caffe2_inputs(self, batched_inputs): + """ + Convert pytorch-style structured inputs to caffe2-style inputs that + are tuples of tensors. + + Args: + batched_inputs (list[dict]): inputs to a detectron2 model + in its standard format. Each dict has "image" (CHW tensor), and optionally + "height" and "width". + + Returns: + tuple[Tensor]: + tuple of tensors that will be the inputs to the + :meth:`forward` method. For existing models, the first + is an NCHW tensor (padded and batched); the second is + a im_info Nx3 tensor, where the rows are + (height, width, unused legacy parameter) + """ + return convert_batched_inputs_to_c2_format( + batched_inputs, + self._wrapped_model.backbone.size_divisibility, + self._wrapped_model.device, + ) + + def encode_additional_info(self, predict_net, init_net): + """ + Save extra metadata that will be used by inference in the output protobuf. + """ + pass + + def forward(self, inputs): + """ + Run the forward in caffe2-style. It has to use caffe2-compatible ops + and the method will be used for tracing. + + Args: + inputs (tuple[Tensor]): inputs defined by :meth:`get_caffe2_input`. + They will be the inputs of the converted caffe2 graph. + + Returns: + tuple[Tensor]: output tensors. They will be the outputs of the + converted caffe2 graph. + """ + raise NotImplementedError + + def _caffe2_preprocess_image(self, inputs): + """ + Caffe2 implementation of preprocess_image, which is called inside each MetaArch's forward. + It normalizes the input images, and the final caffe2 graph assumes the + inputs have been batched already. + """ + data, im_info = inputs + data = alias(data, "data") + im_info = alias(im_info, "im_info") + mean, std = self._wrapped_model.pixel_mean, self._wrapped_model.pixel_std + normalized_data = (data - mean) / std + normalized_data = alias(normalized_data, "normalized_data") + + # Pack (data, im_info) into ImageList which is recognized by self.inference. + images = ImageList(tensor=normalized_data, image_sizes=im_info) + return images + + @staticmethod + def get_outputs_converter(predict_net, init_net): + """ + Creates a function that converts outputs of the caffe2 model to + detectron2's standard format. + The function uses information in `predict_net` and `init_net` that are + available at inferene time. Therefore the function logic can be used in inference. + + The returned function has the following signature: + + def convert(batched_inputs, c2_inputs, c2_results) -> detectron2_outputs + + Where + + * batched_inputs (list[dict]): the original input format of the meta arch + * c2_inputs (tuple[Tensor]): the caffe2 inputs. + * c2_results (dict[str, Tensor]): the caffe2 output format, + corresponding to the outputs of the :meth:`forward` function. + * detectron2_outputs: the original output format of the meta arch. + + This function can be used to compare the outputs of the original meta arch and + the converted caffe2 graph. + + Returns: + callable: a callable of the above signature. + """ + raise NotImplementedError + + +class Caffe2GeneralizedRCNN(Caffe2MetaArch): + def __init__(self, cfg, torch_model): + assert isinstance(torch_model, meta_arch.GeneralizedRCNN) + torch_model = patch_generalized_rcnn(torch_model) + super().__init__(cfg, torch_model) + + try: + use_heatmap_max_keypoint = cfg.EXPORT_CAFFE2.USE_HEATMAP_MAX_KEYPOINT + except AttributeError: + use_heatmap_max_keypoint = False + self.roi_heads_patcher = ROIHeadsPatcher( + self._wrapped_model.roi_heads, use_heatmap_max_keypoint + ) + + def encode_additional_info(self, predict_net, init_net): + size_divisibility = self._wrapped_model.backbone.size_divisibility + check_set_pb_arg(predict_net, "size_divisibility", "i", size_divisibility) + check_set_pb_arg( + predict_net, "device", "s", str.encode(str(self._wrapped_model.device), "ascii") + ) + check_set_pb_arg(predict_net, "meta_architecture", "s", b"GeneralizedRCNN") + + @mock_torch_nn_functional_interpolate() + def forward(self, inputs): + if not self.tensor_mode: + return self._wrapped_model.inference(inputs) + images = self._caffe2_preprocess_image(inputs) + features = self._wrapped_model.backbone(images.tensor) + proposals, _ = self._wrapped_model.proposal_generator(images, features) + with self.roi_heads_patcher.mock_roi_heads(): + detector_results, _ = self._wrapped_model.roi_heads(images, features, proposals) + return tuple(detector_results[0].flatten()) + + @staticmethod + def get_outputs_converter(predict_net, init_net): + def f(batched_inputs, c2_inputs, c2_results): + _, im_info = c2_inputs + image_sizes = [[int(im[0]), int(im[1])] for im in im_info] + results = assemble_rcnn_outputs_by_name(image_sizes, c2_results) + return meta_arch.GeneralizedRCNN._postprocess(results, batched_inputs, image_sizes) + + return f + + +class Caffe2RetinaNet(Caffe2MetaArch): + def __init__(self, cfg, torch_model): + assert isinstance(torch_model, meta_arch.RetinaNet) + super().__init__(cfg, torch_model) + + @mock_torch_nn_functional_interpolate() + def forward(self, inputs): + assert self.tensor_mode + images = self._caffe2_preprocess_image(inputs) + + # explicitly return the images sizes to avoid removing "im_info" by ONNX + # since it's not used in the forward path + return_tensors = [images.image_sizes] + + features = self._wrapped_model.backbone(images.tensor) + features = [features[f] for f in self._wrapped_model.head_in_features] + for i, feature_i in enumerate(features): + features[i] = alias(feature_i, "feature_{}".format(i), is_backward=True) + return_tensors.append(features[i]) + + pred_logits, pred_anchor_deltas = self._wrapped_model.head(features) + for i, (box_cls_i, box_delta_i) in enumerate(zip(pred_logits, pred_anchor_deltas)): + return_tensors.append(alias(box_cls_i, "box_cls_{}".format(i))) + return_tensors.append(alias(box_delta_i, "box_delta_{}".format(i))) + + return tuple(return_tensors) + + def encode_additional_info(self, predict_net, init_net): + size_divisibility = self._wrapped_model.backbone.size_divisibility + check_set_pb_arg(predict_net, "size_divisibility", "i", size_divisibility) + check_set_pb_arg( + predict_net, "device", "s", str.encode(str(self._wrapped_model.device), "ascii") + ) + check_set_pb_arg(predict_net, "meta_architecture", "s", b"RetinaNet") + + # Inference parameters: + check_set_pb_arg( + predict_net, "score_threshold", "f", _cast_to_f32(self._wrapped_model.test_score_thresh) + ) + check_set_pb_arg( + predict_net, "topk_candidates", "i", self._wrapped_model.test_topk_candidates + ) + check_set_pb_arg( + predict_net, "nms_threshold", "f", _cast_to_f32(self._wrapped_model.test_nms_thresh) + ) + check_set_pb_arg( + predict_net, + "max_detections_per_image", + "i", + self._wrapped_model.max_detections_per_image, + ) + + check_set_pb_arg( + predict_net, + "bbox_reg_weights", + "floats", + [_cast_to_f32(w) for w in self._wrapped_model.box2box_transform.weights], + ) + self._encode_anchor_generator_cfg(predict_net) + + def _encode_anchor_generator_cfg(self, predict_net): + # serialize anchor_generator for future use + serialized_anchor_generator = io.BytesIO() + torch.save(self._wrapped_model.anchor_generator, serialized_anchor_generator) + # Ideally we can put anchor generating inside the model, then we don't + # need to store this information. + bytes = serialized_anchor_generator.getvalue() + check_set_pb_arg(predict_net, "serialized_anchor_generator", "s", bytes) + + @staticmethod + def get_outputs_converter(predict_net, init_net): + self = types.SimpleNamespace() + serialized_anchor_generator = io.BytesIO( + get_pb_arg_vals(predict_net, "serialized_anchor_generator", None) + ) + self.anchor_generator = torch.load(serialized_anchor_generator) + bbox_reg_weights = get_pb_arg_floats(predict_net, "bbox_reg_weights", None) + self.box2box_transform = Box2BoxTransform(weights=tuple(bbox_reg_weights)) + self.test_score_thresh = get_pb_arg_valf(predict_net, "score_threshold", None) + self.test_topk_candidates = get_pb_arg_vali(predict_net, "topk_candidates", None) + self.test_nms_thresh = get_pb_arg_valf(predict_net, "nms_threshold", None) + self.max_detections_per_image = get_pb_arg_vali( + predict_net, "max_detections_per_image", None + ) + + # hack to reuse inference code from RetinaNet + for meth in [ + "forward_inference", + "inference_single_image", + "_transpose_dense_predictions", + "_decode_multi_level_predictions", + "_decode_per_level_predictions", + ]: + setattr(self, meth, functools.partial(getattr(meta_arch.RetinaNet, meth), self)) + + def f(batched_inputs, c2_inputs, c2_results): + _, im_info = c2_inputs + image_sizes = [[int(im[0]), int(im[1])] for im in im_info] + dummy_images = ImageList( + torch.randn( + ( + len(im_info), + 3, + ) + + tuple(image_sizes[0]) + ), + image_sizes, + ) + + num_features = len([x for x in c2_results.keys() if x.startswith("box_cls_")]) + pred_logits = [c2_results["box_cls_{}".format(i)] for i in range(num_features)] + pred_anchor_deltas = [c2_results["box_delta_{}".format(i)] for i in range(num_features)] + + # For each feature level, feature should have the same batch size and + # spatial dimension as the box_cls and box_delta. + dummy_features = [x.clone()[:, 0:0, :, :] for x in pred_logits] + # self.num_classess can be inferred + self.num_classes = pred_logits[0].shape[1] // (pred_anchor_deltas[0].shape[1] // 4) + + results = self.forward_inference( + dummy_images, dummy_features, [pred_logits, pred_anchor_deltas] + ) + return meta_arch.GeneralizedRCNN._postprocess(results, batched_inputs, image_sizes) + + return f + + +META_ARCH_CAFFE2_EXPORT_TYPE_MAP = { + "GeneralizedRCNN": Caffe2GeneralizedRCNN, + "RetinaNet": Caffe2RetinaNet, +} diff --git a/annotator/oneformer/detectron2/export/caffe2_patch.py b/annotator/oneformer/detectron2/export/caffe2_patch.py new file mode 100644 index 0000000000000000000000000000000000000000..9c197cac1e7d5f665b6cbda46268716b1222f217 --- /dev/null +++ b/annotator/oneformer/detectron2/export/caffe2_patch.py @@ -0,0 +1,152 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import contextlib +from unittest import mock +import torch + +from annotator.oneformer.detectron2.modeling import poolers +from annotator.oneformer.detectron2.modeling.proposal_generator import rpn +from annotator.oneformer.detectron2.modeling.roi_heads import keypoint_head, mask_head +from annotator.oneformer.detectron2.modeling.roi_heads.fast_rcnn import FastRCNNOutputLayers + +from .c10 import ( + Caffe2Compatible, + Caffe2FastRCNNOutputsInference, + Caffe2KeypointRCNNInference, + Caffe2MaskRCNNInference, + Caffe2ROIPooler, + Caffe2RPN, +) + + +class GenericMixin(object): + pass + + +class Caffe2CompatibleConverter(object): + """ + A GenericUpdater which implements the `create_from` interface, by modifying + module object and assign it with another class replaceCls. + """ + + def __init__(self, replaceCls): + self.replaceCls = replaceCls + + def create_from(self, module): + # update module's class to the new class + assert isinstance(module, torch.nn.Module) + if issubclass(self.replaceCls, GenericMixin): + # replaceCls should act as mixin, create a new class on-the-fly + new_class = type( + "{}MixedWith{}".format(self.replaceCls.__name__, module.__class__.__name__), + (self.replaceCls, module.__class__), + {}, # {"new_method": lambda self: ...}, + ) + module.__class__ = new_class + else: + # replaceCls is complete class, this allow arbitrary class swap + module.__class__ = self.replaceCls + + # initialize Caffe2Compatible + if isinstance(module, Caffe2Compatible): + module.tensor_mode = False + + return module + + +def patch(model, target, updater, *args, **kwargs): + """ + recursively (post-order) update all modules with the target type and its + subclasses, make a initialization/composition/inheritance/... via the + updater.create_from. + """ + for name, module in model.named_children(): + model._modules[name] = patch(module, target, updater, *args, **kwargs) + if isinstance(model, target): + return updater.create_from(model, *args, **kwargs) + return model + + +def patch_generalized_rcnn(model): + ccc = Caffe2CompatibleConverter + model = patch(model, rpn.RPN, ccc(Caffe2RPN)) + model = patch(model, poolers.ROIPooler, ccc(Caffe2ROIPooler)) + + return model + + +@contextlib.contextmanager +def mock_fastrcnn_outputs_inference( + tensor_mode, check=True, box_predictor_type=FastRCNNOutputLayers +): + with mock.patch.object( + box_predictor_type, + "inference", + autospec=True, + side_effect=Caffe2FastRCNNOutputsInference(tensor_mode), + ) as mocked_func: + yield + if check: + assert mocked_func.call_count > 0 + + +@contextlib.contextmanager +def mock_mask_rcnn_inference(tensor_mode, patched_module, check=True): + with mock.patch( + "{}.mask_rcnn_inference".format(patched_module), side_effect=Caffe2MaskRCNNInference() + ) as mocked_func: + yield + if check: + assert mocked_func.call_count > 0 + + +@contextlib.contextmanager +def mock_keypoint_rcnn_inference(tensor_mode, patched_module, use_heatmap_max_keypoint, check=True): + with mock.patch( + "{}.keypoint_rcnn_inference".format(patched_module), + side_effect=Caffe2KeypointRCNNInference(use_heatmap_max_keypoint), + ) as mocked_func: + yield + if check: + assert mocked_func.call_count > 0 + + +class ROIHeadsPatcher: + def __init__(self, heads, use_heatmap_max_keypoint): + self.heads = heads + self.use_heatmap_max_keypoint = use_heatmap_max_keypoint + + @contextlib.contextmanager + def mock_roi_heads(self, tensor_mode=True): + """ + Patching several inference functions inside ROIHeads and its subclasses + + Args: + tensor_mode (bool): whether the inputs/outputs are caffe2's tensor + format or not. Default to True. + """ + # NOTE: this requries the `keypoint_rcnn_inference` and `mask_rcnn_inference` + # are called inside the same file as BaseXxxHead due to using mock.patch. + kpt_heads_mod = keypoint_head.BaseKeypointRCNNHead.__module__ + mask_head_mod = mask_head.BaseMaskRCNNHead.__module__ + + mock_ctx_managers = [ + mock_fastrcnn_outputs_inference( + tensor_mode=tensor_mode, + check=True, + box_predictor_type=type(self.heads.box_predictor), + ) + ] + if getattr(self.heads, "keypoint_on", False): + mock_ctx_managers += [ + mock_keypoint_rcnn_inference( + tensor_mode, kpt_heads_mod, self.use_heatmap_max_keypoint + ) + ] + if getattr(self.heads, "mask_on", False): + mock_ctx_managers += [mock_mask_rcnn_inference(tensor_mode, mask_head_mod)] + + with contextlib.ExitStack() as stack: # python 3.3+ + for mgr in mock_ctx_managers: + stack.enter_context(mgr) + yield diff --git a/annotator/oneformer/detectron2/export/flatten.py b/annotator/oneformer/detectron2/export/flatten.py new file mode 100644 index 0000000000000000000000000000000000000000..3fcb2bf49a0adad2798a10781a42accd9571218f --- /dev/null +++ b/annotator/oneformer/detectron2/export/flatten.py @@ -0,0 +1,330 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import collections +from dataclasses import dataclass +from typing import Callable, List, Optional, Tuple +import torch +from torch import nn + +from annotator.oneformer.detectron2.structures import Boxes, Instances, ROIMasks +from annotator.oneformer.detectron2.utils.registry import _convert_target_to_string, locate + +from .torchscript_patch import patch_builtin_len + + +@dataclass +class Schema: + """ + A Schema defines how to flatten a possibly hierarchical object into tuple of + primitive objects, so it can be used as inputs/outputs of PyTorch's tracing. + + PyTorch does not support tracing a function that produces rich output + structures (e.g. dict, Instances, Boxes). To trace such a function, we + flatten the rich object into tuple of tensors, and return this tuple of tensors + instead. Meanwhile, we also need to know how to "rebuild" the original object + from the flattened results, so we can evaluate the flattened results. + A Schema defines how to flatten an object, and while flattening it, it records + necessary schemas so that the object can be rebuilt using the flattened outputs. + + The flattened object and the schema object is returned by ``.flatten`` classmethod. + Then the original object can be rebuilt with the ``__call__`` method of schema. + + A Schema is a dataclass that can be serialized easily. + """ + + # inspired by FetchMapper in tensorflow/python/client/session.py + + @classmethod + def flatten(cls, obj): + raise NotImplementedError + + def __call__(self, values): + raise NotImplementedError + + @staticmethod + def _concat(values): + ret = () + sizes = [] + for v in values: + assert isinstance(v, tuple), "Flattened results must be a tuple" + ret = ret + v + sizes.append(len(v)) + return ret, sizes + + @staticmethod + def _split(values, sizes): + if len(sizes): + expected_len = sum(sizes) + assert ( + len(values) == expected_len + ), f"Values has length {len(values)} but expect length {expected_len}." + ret = [] + for k in range(len(sizes)): + begin, end = sum(sizes[:k]), sum(sizes[: k + 1]) + ret.append(values[begin:end]) + return ret + + +@dataclass +class ListSchema(Schema): + schemas: List[Schema] # the schemas that define how to flatten each element in the list + sizes: List[int] # the flattened length of each element + + def __call__(self, values): + values = self._split(values, self.sizes) + if len(values) != len(self.schemas): + raise ValueError( + f"Values has length {len(values)} but schemas " f"has length {len(self.schemas)}!" + ) + values = [m(v) for m, v in zip(self.schemas, values)] + return list(values) + + @classmethod + def flatten(cls, obj): + res = [flatten_to_tuple(k) for k in obj] + values, sizes = cls._concat([k[0] for k in res]) + return values, cls([k[1] for k in res], sizes) + + +@dataclass +class TupleSchema(ListSchema): + def __call__(self, values): + return tuple(super().__call__(values)) + + +@dataclass +class IdentitySchema(Schema): + def __call__(self, values): + return values[0] + + @classmethod + def flatten(cls, obj): + return (obj,), cls() + + +@dataclass +class DictSchema(ListSchema): + keys: List[str] + + def __call__(self, values): + values = super().__call__(values) + return dict(zip(self.keys, values)) + + @classmethod + def flatten(cls, obj): + for k in obj.keys(): + if not isinstance(k, str): + raise KeyError("Only support flattening dictionaries if keys are str.") + keys = sorted(obj.keys()) + values = [obj[k] for k in keys] + ret, schema = ListSchema.flatten(values) + return ret, cls(schema.schemas, schema.sizes, keys) + + +@dataclass +class InstancesSchema(DictSchema): + def __call__(self, values): + image_size, fields = values[-1], values[:-1] + fields = super().__call__(fields) + return Instances(image_size, **fields) + + @classmethod + def flatten(cls, obj): + ret, schema = super().flatten(obj.get_fields()) + size = obj.image_size + if not isinstance(size, torch.Tensor): + size = torch.tensor(size) + return ret + (size,), schema + + +@dataclass +class TensorWrapSchema(Schema): + """ + For classes that are simple wrapper of tensors, e.g. + Boxes, RotatedBoxes, BitMasks + """ + + class_name: str + + def __call__(self, values): + return locate(self.class_name)(values[0]) + + @classmethod + def flatten(cls, obj): + return (obj.tensor,), cls(_convert_target_to_string(type(obj))) + + +# if more custom structures needed in the future, can allow +# passing in extra schemas for custom types +def flatten_to_tuple(obj): + """ + Flatten an object so it can be used for PyTorch tracing. + Also returns how to rebuild the original object from the flattened outputs. + + Returns: + res (tuple): the flattened results that can be used as tracing outputs + schema: an object with a ``__call__`` method such that ``schema(res) == obj``. + It is a pure dataclass that can be serialized. + """ + schemas = [ + ((str, bytes), IdentitySchema), + (list, ListSchema), + (tuple, TupleSchema), + (collections.abc.Mapping, DictSchema), + (Instances, InstancesSchema), + ((Boxes, ROIMasks), TensorWrapSchema), + ] + for klass, schema in schemas: + if isinstance(obj, klass): + F = schema + break + else: + F = IdentitySchema + + return F.flatten(obj) + + +class TracingAdapter(nn.Module): + """ + A model may take rich input/output format (e.g. dict or custom classes), + but `torch.jit.trace` requires tuple of tensors as input/output. + This adapter flattens input/output format of a model so it becomes traceable. + + It also records the necessary schema to rebuild model's inputs/outputs from flattened + inputs/outputs. + + Example: + :: + outputs = model(inputs) # inputs/outputs may be rich structure + adapter = TracingAdapter(model, inputs) + + # can now trace the model, with adapter.flattened_inputs, or another + # tuple of tensors with the same length and meaning + traced = torch.jit.trace(adapter, adapter.flattened_inputs) + + # traced model can only produce flattened outputs (tuple of tensors) + flattened_outputs = traced(*adapter.flattened_inputs) + # adapter knows the schema to convert it back (new_outputs == outputs) + new_outputs = adapter.outputs_schema(flattened_outputs) + """ + + flattened_inputs: Tuple[torch.Tensor] = None + """ + Flattened version of inputs given to this class's constructor. + """ + + inputs_schema: Schema = None + """ + Schema of the inputs given to this class's constructor. + """ + + outputs_schema: Schema = None + """ + Schema of the output produced by calling the given model with inputs. + """ + + def __init__( + self, + model: nn.Module, + inputs, + inference_func: Optional[Callable] = None, + allow_non_tensor: bool = False, + ): + """ + Args: + model: an nn.Module + inputs: An input argument or a tuple of input arguments used to call model. + After flattening, it has to only consist of tensors. + inference_func: a callable that takes (model, *inputs), calls the + model with inputs, and return outputs. By default it + is ``lambda model, *inputs: model(*inputs)``. Can be override + if you need to call the model differently. + allow_non_tensor: allow inputs/outputs to contain non-tensor objects. + This option will filter out non-tensor objects to make the + model traceable, but ``inputs_schema``/``outputs_schema`` cannot be + used anymore because inputs/outputs cannot be rebuilt from pure tensors. + This is useful when you're only interested in the single trace of + execution (e.g. for flop count), but not interested in + generalizing the traced graph to new inputs. + """ + super().__init__() + if isinstance(model, (nn.parallel.distributed.DistributedDataParallel, nn.DataParallel)): + model = model.module + self.model = model + if not isinstance(inputs, tuple): + inputs = (inputs,) + self.inputs = inputs + self.allow_non_tensor = allow_non_tensor + + if inference_func is None: + inference_func = lambda model, *inputs: model(*inputs) # noqa + self.inference_func = inference_func + + self.flattened_inputs, self.inputs_schema = flatten_to_tuple(inputs) + + if all(isinstance(x, torch.Tensor) for x in self.flattened_inputs): + return + if self.allow_non_tensor: + self.flattened_inputs = tuple( + [x for x in self.flattened_inputs if isinstance(x, torch.Tensor)] + ) + self.inputs_schema = None + else: + for input in self.flattened_inputs: + if not isinstance(input, torch.Tensor): + raise ValueError( + "Inputs for tracing must only contain tensors. " + f"Got a {type(input)} instead." + ) + + def forward(self, *args: torch.Tensor): + with torch.no_grad(), patch_builtin_len(): + if self.inputs_schema is not None: + inputs_orig_format = self.inputs_schema(args) + else: + if len(args) != len(self.flattened_inputs) or any( + x is not y for x, y in zip(args, self.flattened_inputs) + ): + raise ValueError( + "TracingAdapter does not contain valid inputs_schema." + " So it cannot generalize to other inputs and must be" + " traced with `.flattened_inputs`." + ) + inputs_orig_format = self.inputs + + outputs = self.inference_func(self.model, *inputs_orig_format) + flattened_outputs, schema = flatten_to_tuple(outputs) + + flattened_output_tensors = tuple( + [x for x in flattened_outputs if isinstance(x, torch.Tensor)] + ) + if len(flattened_output_tensors) < len(flattened_outputs): + if self.allow_non_tensor: + flattened_outputs = flattened_output_tensors + self.outputs_schema = None + else: + raise ValueError( + "Model cannot be traced because some model outputs " + "cannot flatten to tensors." + ) + else: # schema is valid + if self.outputs_schema is None: + self.outputs_schema = schema + else: + assert self.outputs_schema == schema, ( + "Model should always return outputs with the same " + "structure so it can be traced!" + ) + return flattened_outputs + + def _create_wrapper(self, traced_model): + """ + Return a function that has an input/output interface the same as the + original model, but it calls the given traced model under the hood. + """ + + def forward(*args): + flattened_inputs, _ = flatten_to_tuple(args) + flattened_outputs = traced_model(*flattened_inputs) + return self.outputs_schema(flattened_outputs) + + return forward diff --git a/annotator/oneformer/detectron2/export/shared.py b/annotator/oneformer/detectron2/export/shared.py new file mode 100644 index 0000000000000000000000000000000000000000..53ba9335e26819f9381115eba17bbbe3816b469c --- /dev/null +++ b/annotator/oneformer/detectron2/export/shared.py @@ -0,0 +1,1039 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import collections +import copy +import functools +import logging +import numpy as np +import os +from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from unittest import mock +import caffe2.python.utils as putils +import torch +import torch.nn.functional as F +from caffe2.proto import caffe2_pb2 +from caffe2.python import core, net_drawer, workspace +from torch.nn.functional import interpolate as interp + +logger = logging.getLogger(__name__) + + +# ==== torch/utils_toffee/cast.py ======================================= + + +def to_device(t, device_str): + """ + This function is a replacement of .to(another_device) such that it allows the + casting to be traced properly by explicitly calling the underlying copy ops. + It also avoids introducing unncessary op when casting to the same device. + """ + src = t.device + dst = torch.device(device_str) + + if src == dst: + return t + elif src.type == "cuda" and dst.type == "cpu": + return torch.ops._caffe2.CopyGPUToCPU(t) + elif src.type == "cpu" and dst.type == "cuda": + return torch.ops._caffe2.CopyCPUToGPU(t) + else: + raise RuntimeError("Can't cast tensor from device {} to device {}".format(src, dst)) + + +# ==== torch/utils_toffee/interpolate.py ======================================= + + +# Note: borrowed from vision/detection/fair/detectron/detectron/modeling/detector.py +def BilinearInterpolation(tensor_in, up_scale): + assert up_scale % 2 == 0, "Scale should be even" + + def upsample_filt(size): + factor = (size + 1) // 2 + if size % 2 == 1: + center = factor - 1 + else: + center = factor - 0.5 + + og = np.ogrid[:size, :size] + return (1 - abs(og[0] - center) / factor) * (1 - abs(og[1] - center) / factor) + + kernel_size = int(up_scale) * 2 + bil_filt = upsample_filt(kernel_size) + + dim = int(tensor_in.shape[1]) + kernel = np.zeros((dim, dim, kernel_size, kernel_size), dtype=np.float32) + kernel[range(dim), range(dim), :, :] = bil_filt + + tensor_out = F.conv_transpose2d( + tensor_in, + weight=to_device(torch.Tensor(kernel), tensor_in.device), + bias=None, + stride=int(up_scale), + padding=int(up_scale / 2), + ) + + return tensor_out + + +# NOTE: ONNX is incompatible with traced torch.nn.functional.interpolate if +# using dynamic `scale_factor` rather than static `size`. (T43166860) +# NOTE: Caffe2 Int8 conversion might not be able to quantize `size` properly. +def onnx_compatibale_interpolate( + input, size=None, scale_factor=None, mode="nearest", align_corners=None +): + # NOTE: The input dimensions are interpreted in the form: + # `mini-batch x channels x [optional depth] x [optional height] x width`. + if size is None and scale_factor is not None: + if input.dim() == 4: + if isinstance(scale_factor, (int, float)): + height_scale, width_scale = (scale_factor, scale_factor) + else: + assert isinstance(scale_factor, (tuple, list)) + assert len(scale_factor) == 2 + height_scale, width_scale = scale_factor + + assert not align_corners, "No matching C2 op for align_corners == True" + if mode == "nearest": + return torch.ops._caffe2.ResizeNearest( + input, order="NCHW", width_scale=width_scale, height_scale=height_scale + ) + elif mode == "bilinear": + logger.warning( + "Use F.conv_transpose2d for bilinear interpolate" + " because there's no such C2 op, this may cause significant" + " slowdown and the boundary pixels won't be as same as" + " using F.interpolate due to padding." + ) + assert height_scale == width_scale + return BilinearInterpolation(input, up_scale=height_scale) + logger.warning("Output size is not static, it might cause ONNX conversion issue") + + return interp(input, size, scale_factor, mode, align_corners) + + +def mock_torch_nn_functional_interpolate(): + def decorator(func): + @functools.wraps(func) + def _mock_torch_nn_functional_interpolate(*args, **kwargs): + if torch.onnx.is_in_onnx_export(): + with mock.patch( + "torch.nn.functional.interpolate", side_effect=onnx_compatibale_interpolate + ): + return func(*args, **kwargs) + else: + return func(*args, **kwargs) + + return _mock_torch_nn_functional_interpolate + + return decorator + + +# ==== torch/utils_caffe2/ws_utils.py ========================================== + + +class ScopedWS(object): + def __init__(self, ws_name, is_reset, is_cleanup=False): + self.ws_name = ws_name + self.is_reset = is_reset + self.is_cleanup = is_cleanup + self.org_ws = "" + + def __enter__(self): + self.org_ws = workspace.CurrentWorkspace() + if self.ws_name is not None: + workspace.SwitchWorkspace(self.ws_name, True) + if self.is_reset: + workspace.ResetWorkspace() + + return workspace + + def __exit__(self, *args): + if self.is_cleanup: + workspace.ResetWorkspace() + if self.ws_name is not None: + workspace.SwitchWorkspace(self.org_ws) + + +def fetch_any_blob(name): + bb = None + try: + bb = workspace.FetchBlob(name) + except TypeError: + bb = workspace.FetchInt8Blob(name) + except Exception as e: + logger.error("Get blob {} error: {}".format(name, e)) + + return bb + + +# ==== torch/utils_caffe2/protobuf.py ========================================== + + +def get_pb_arg(pb, arg_name): + for x in pb.arg: + if x.name == arg_name: + return x + return None + + +def get_pb_arg_valf(pb, arg_name, default_val): + arg = get_pb_arg(pb, arg_name) + return arg.f if arg is not None else default_val + + +def get_pb_arg_floats(pb, arg_name, default_val): + arg = get_pb_arg(pb, arg_name) + return list(map(float, arg.floats)) if arg is not None else default_val + + +def get_pb_arg_ints(pb, arg_name, default_val): + arg = get_pb_arg(pb, arg_name) + return list(map(int, arg.ints)) if arg is not None else default_val + + +def get_pb_arg_vali(pb, arg_name, default_val): + arg = get_pb_arg(pb, arg_name) + return arg.i if arg is not None else default_val + + +def get_pb_arg_vals(pb, arg_name, default_val): + arg = get_pb_arg(pb, arg_name) + return arg.s if arg is not None else default_val + + +def get_pb_arg_valstrings(pb, arg_name, default_val): + arg = get_pb_arg(pb, arg_name) + return list(arg.strings) if arg is not None else default_val + + +def check_set_pb_arg(pb, arg_name, arg_attr, arg_value, allow_override=False): + arg = get_pb_arg(pb, arg_name) + if arg is None: + arg = putils.MakeArgument(arg_name, arg_value) + assert hasattr(arg, arg_attr) + pb.arg.extend([arg]) + if allow_override and getattr(arg, arg_attr) != arg_value: + logger.warning( + "Override argument {}: {} -> {}".format(arg_name, getattr(arg, arg_attr), arg_value) + ) + setattr(arg, arg_attr, arg_value) + else: + assert arg is not None + assert getattr(arg, arg_attr) == arg_value, "Existing value {}, new value {}".format( + getattr(arg, arg_attr), arg_value + ) + + +def _create_const_fill_op_from_numpy(name, tensor, device_option=None): + assert type(tensor) == np.ndarray + kTypeNameMapper = { + np.dtype("float32"): "GivenTensorFill", + np.dtype("int32"): "GivenTensorIntFill", + np.dtype("int64"): "GivenTensorInt64Fill", + np.dtype("uint8"): "GivenTensorStringFill", + } + + args_dict = {} + if tensor.dtype == np.dtype("uint8"): + args_dict.update({"values": [str(tensor.data)], "shape": [1]}) + else: + args_dict.update({"values": tensor, "shape": tensor.shape}) + + if device_option is not None: + args_dict["device_option"] = device_option + + return core.CreateOperator(kTypeNameMapper[tensor.dtype], [], [name], **args_dict) + + +def _create_const_fill_op_from_c2_int8_tensor(name, int8_tensor): + assert type(int8_tensor) == workspace.Int8Tensor + kTypeNameMapper = { + np.dtype("int32"): "Int8GivenIntTensorFill", + np.dtype("uint8"): "Int8GivenTensorFill", + } + + tensor = int8_tensor.data + assert tensor.dtype in [np.dtype("uint8"), np.dtype("int32")] + values = tensor.tobytes() if tensor.dtype == np.dtype("uint8") else tensor + + return core.CreateOperator( + kTypeNameMapper[tensor.dtype], + [], + [name], + values=values, + shape=tensor.shape, + Y_scale=int8_tensor.scale, + Y_zero_point=int8_tensor.zero_point, + ) + + +def create_const_fill_op( + name: str, + blob: Union[np.ndarray, workspace.Int8Tensor], + device_option: Optional[caffe2_pb2.DeviceOption] = None, +) -> caffe2_pb2.OperatorDef: + """ + Given a blob object, return the Caffe2 operator that creates this blob + as constant. Currently support NumPy tensor and Caffe2 Int8Tensor. + """ + + tensor_type = type(blob) + assert tensor_type in [ + np.ndarray, + workspace.Int8Tensor, + ], 'Error when creating const fill op for "{}", unsupported blob type: {}'.format( + name, type(blob) + ) + + if tensor_type == np.ndarray: + return _create_const_fill_op_from_numpy(name, blob, device_option) + elif tensor_type == workspace.Int8Tensor: + assert device_option is None + return _create_const_fill_op_from_c2_int8_tensor(name, blob) + + +def construct_init_net_from_params( + params: Dict[str, Any], device_options: Optional[Dict[str, caffe2_pb2.DeviceOption]] = None +) -> caffe2_pb2.NetDef: + """ + Construct the init_net from params dictionary + """ + init_net = caffe2_pb2.NetDef() + device_options = device_options or {} + for name, blob in params.items(): + if isinstance(blob, str): + logger.warning( + ( + "Blob {} with type {} is not supported in generating init net," + " skipped.".format(name, type(blob)) + ) + ) + continue + init_net.op.extend( + [create_const_fill_op(name, blob, device_option=device_options.get(name, None))] + ) + init_net.external_output.append(name) + return init_net + + +def get_producer_map(ssa): + """ + Return dict from versioned blob to (i, j), + where i is index of producer op, j is the index of output of that op. + """ + producer_map = {} + for i in range(len(ssa)): + outputs = ssa[i][1] + for j, outp in enumerate(outputs): + producer_map[outp] = (i, j) + return producer_map + + +def get_consumer_map(ssa): + """ + Return dict from versioned blob to list of (i, j), + where i is index of consumer op, j is the index of input of that op. + """ + consumer_map = collections.defaultdict(list) + for i in range(len(ssa)): + inputs = ssa[i][0] + for j, inp in enumerate(inputs): + consumer_map[inp].append((i, j)) + return consumer_map + + +def get_params_from_init_net( + init_net: caffe2_pb2.NetDef, +) -> [Dict[str, Any], Dict[str, caffe2_pb2.DeviceOption]]: + """ + Take the output blobs from init_net by running it. + Outputs: + params: dict from blob name to numpy array + device_options: dict from blob name to the device option of its creating op + """ + # NOTE: this assumes that the params is determined by producer op with the + # only exception be CopyGPUToCPU which is CUDA op but returns CPU tensor. + def _get_device_option(producer_op): + if producer_op.type == "CopyGPUToCPU": + return caffe2_pb2.DeviceOption() + else: + return producer_op.device_option + + with ScopedWS("__get_params_from_init_net__", is_reset=True, is_cleanup=True) as ws: + ws.RunNetOnce(init_net) + params = {b: fetch_any_blob(b) for b in init_net.external_output} + ssa, versions = core.get_ssa(init_net) + producer_map = get_producer_map(ssa) + device_options = { + b: _get_device_option(init_net.op[producer_map[(b, versions[b])][0]]) + for b in init_net.external_output + } + return params, device_options + + +def _updater_raise(op, input_types, output_types): + raise RuntimeError( + "Failed to apply updater for op {} given input_types {} and" + " output_types {}".format(op, input_types, output_types) + ) + + +def _generic_status_identifier( + predict_net: caffe2_pb2.NetDef, + status_updater: Callable, + known_status: Dict[Tuple[str, int], Any], +) -> Dict[Tuple[str, int], Any]: + """ + Statically infer the status of each blob, the status can be such as device type + (CPU/GPU), layout (NCHW/NHWC), data type (float32/int8), etc. "Blob" here + is versioned blob (Tuple[str, int]) in the format compatible with ssa. + Inputs: + predict_net: the caffe2 network + status_updater: a callable, given an op and the status of its input/output, + it returns the updated status of input/output. `None` is used for + representing unknown status. + known_status: a dict containing known status, used as initialization. + Outputs: + A dict mapping from versioned blob to its status + """ + ssa, versions = core.get_ssa(predict_net) + versioned_ext_input = [(b, 0) for b in predict_net.external_input] + versioned_ext_output = [(b, versions[b]) for b in predict_net.external_output] + all_versioned_blobs = set().union(*[set(x[0] + x[1]) for x in ssa]) + + allowed_vbs = all_versioned_blobs.union(versioned_ext_input).union(versioned_ext_output) + assert all(k in allowed_vbs for k in known_status) + assert all(v is not None for v in known_status.values()) + _known_status = copy.deepcopy(known_status) + + def _check_and_update(key, value): + assert value is not None + if key in _known_status: + if not _known_status[key] == value: + raise RuntimeError( + "Confilict status for {}, existing status {}, new status {}".format( + key, _known_status[key], value + ) + ) + _known_status[key] = value + + def _update_i(op, ssa_i): + versioned_inputs = ssa_i[0] + versioned_outputs = ssa_i[1] + + inputs_status = [_known_status.get(b, None) for b in versioned_inputs] + outputs_status = [_known_status.get(b, None) for b in versioned_outputs] + + new_inputs_status, new_outputs_status = status_updater(op, inputs_status, outputs_status) + + for versioned_blob, status in zip( + versioned_inputs + versioned_outputs, new_inputs_status + new_outputs_status + ): + if status is not None: + _check_and_update(versioned_blob, status) + + for op, ssa_i in zip(predict_net.op, ssa): + _update_i(op, ssa_i) + for op, ssa_i in zip(reversed(predict_net.op), reversed(ssa)): + _update_i(op, ssa_i) + + # NOTE: This strictly checks all the blob from predict_net must be assgined + # a known status. However sometimes it's impossible (eg. having deadend op), + # we may relax this constraint if + for k in all_versioned_blobs: + if k not in _known_status: + raise NotImplementedError( + "Can not infer the status for {}. Currently only support the case where" + " a single forward and backward pass can identify status for all blobs.".format(k) + ) + + return _known_status + + +def infer_device_type( + predict_net: caffe2_pb2.NetDef, + known_status: Dict[Tuple[str, int], Any], + device_name_style: str = "caffe2", +) -> Dict[Tuple[str, int], str]: + """Return the device type ("cpu" or "gpu"/"cuda") of each (versioned) blob""" + + assert device_name_style in ["caffe2", "pytorch"] + _CPU_STR = "cpu" + _GPU_STR = "gpu" if device_name_style == "caffe2" else "cuda" + + def _copy_cpu_to_gpu_updater(op, input_types, output_types): + if input_types[0] == _GPU_STR or output_types[0] == _CPU_STR: + _updater_raise(op, input_types, output_types) + return ([_CPU_STR], [_GPU_STR]) + + def _copy_gpu_to_cpu_updater(op, input_types, output_types): + if input_types[0] == _CPU_STR or output_types[0] == _GPU_STR: + _updater_raise(op, input_types, output_types) + return ([_GPU_STR], [_CPU_STR]) + + def _other_ops_updater(op, input_types, output_types): + non_none_types = [x for x in input_types + output_types if x is not None] + if len(non_none_types) > 0: + the_type = non_none_types[0] + if not all(x == the_type for x in non_none_types): + _updater_raise(op, input_types, output_types) + else: + the_type = None + return ([the_type for _ in op.input], [the_type for _ in op.output]) + + def _device_updater(op, *args, **kwargs): + return { + "CopyCPUToGPU": _copy_cpu_to_gpu_updater, + "CopyGPUToCPU": _copy_gpu_to_cpu_updater, + }.get(op.type, _other_ops_updater)(op, *args, **kwargs) + + return _generic_status_identifier(predict_net, _device_updater, known_status) + + +# ==== torch/utils_caffe2/vis.py =============================================== + + +def _modify_blob_names(ops, blob_rename_f): + ret = [] + + def _replace_list(blob_list, replaced_list): + del blob_list[:] + blob_list.extend(replaced_list) + + for x in ops: + cur = copy.deepcopy(x) + _replace_list(cur.input, list(map(blob_rename_f, cur.input))) + _replace_list(cur.output, list(map(blob_rename_f, cur.output))) + ret.append(cur) + + return ret + + +def _rename_blob(name, blob_sizes, blob_ranges): + def _list_to_str(bsize): + ret = ", ".join([str(x) for x in bsize]) + ret = "[" + ret + "]" + return ret + + ret = name + if blob_sizes is not None and name in blob_sizes: + ret += "\n" + _list_to_str(blob_sizes[name]) + if blob_ranges is not None and name in blob_ranges: + ret += "\n" + _list_to_str(blob_ranges[name]) + + return ret + + +# graph_name could not contain word 'graph' +def save_graph(net, file_name, graph_name="net", op_only=True, blob_sizes=None, blob_ranges=None): + blob_rename_f = functools.partial(_rename_blob, blob_sizes=blob_sizes, blob_ranges=blob_ranges) + return save_graph_base(net, file_name, graph_name, op_only, blob_rename_f) + + +def save_graph_base(net, file_name, graph_name="net", op_only=True, blob_rename_func=None): + graph = None + ops = net.op + if blob_rename_func is not None: + ops = _modify_blob_names(ops, blob_rename_func) + if not op_only: + graph = net_drawer.GetPydotGraph(ops, graph_name, rankdir="TB") + else: + graph = net_drawer.GetPydotGraphMinimal( + ops, graph_name, rankdir="TB", minimal_dependency=True + ) + + try: + par_dir = os.path.dirname(file_name) + if not os.path.exists(par_dir): + os.makedirs(par_dir) + + format = os.path.splitext(os.path.basename(file_name))[-1] + if format == ".png": + graph.write_png(file_name) + elif format == ".pdf": + graph.write_pdf(file_name) + elif format == ".svg": + graph.write_svg(file_name) + else: + print("Incorrect format {}".format(format)) + except Exception as e: + print("Error when writing graph to image {}".format(e)) + + return graph + + +# ==== torch/utils_toffee/aten_to_caffe2.py ==================================== + + +def group_norm_replace_aten_with_caffe2(predict_net: caffe2_pb2.NetDef): + """ + For ONNX exported model, GroupNorm will be represented as ATen op, + this can be a drop in replacement from ATen to GroupNorm + """ + count = 0 + for op in predict_net.op: + if op.type == "ATen": + op_name = get_pb_arg_vals(op, "operator", None) # return byte in py3 + if op_name and op_name.decode() == "group_norm": + op.arg.remove(get_pb_arg(op, "operator")) + + if get_pb_arg_vali(op, "cudnn_enabled", None): + op.arg.remove(get_pb_arg(op, "cudnn_enabled")) + + num_groups = get_pb_arg_vali(op, "num_groups", None) + if num_groups is not None: + op.arg.remove(get_pb_arg(op, "num_groups")) + check_set_pb_arg(op, "group", "i", num_groups) + + op.type = "GroupNorm" + count += 1 + if count > 1: + logger.info("Replaced {} ATen operator to GroupNormOp".format(count)) + + +# ==== torch/utils_toffee/alias.py ============================================= + + +def alias(x, name, is_backward=False): + if not torch.onnx.is_in_onnx_export(): + return x + assert isinstance(x, torch.Tensor) + return torch.ops._caffe2.AliasWithName(x, name, is_backward=is_backward) + + +def fuse_alias_placeholder(predict_net, init_net): + """Remove AliasWithName placeholder and rename the input/output of it""" + # First we finish all the re-naming + for i, op in enumerate(predict_net.op): + if op.type == "AliasWithName": + assert len(op.input) == 1 + assert len(op.output) == 1 + name = get_pb_arg_vals(op, "name", None).decode() + is_backward = bool(get_pb_arg_vali(op, "is_backward", 0)) + rename_op_input(predict_net, init_net, i, 0, name, from_producer=is_backward) + rename_op_output(predict_net, i, 0, name) + + # Remove AliasWithName, should be very safe since it's a non-op + new_ops = [] + for op in predict_net.op: + if op.type != "AliasWithName": + new_ops.append(op) + else: + # safety check + assert op.input == op.output + assert op.input[0] == op.arg[0].s.decode() + del predict_net.op[:] + predict_net.op.extend(new_ops) + + +# ==== torch/utils_caffe2/graph_transform.py =================================== + + +class IllegalGraphTransformError(ValueError): + """When a graph transform function call can't be executed.""" + + +def _rename_versioned_blob_in_proto( + proto: caffe2_pb2.NetDef, + old_name: str, + new_name: str, + version: int, + ssa: List[Tuple[List[Tuple[str, int]], List[Tuple[str, int]]]], + start_versions: Dict[str, int], + end_versions: Dict[str, int], +): + """In given proto, rename all blobs with matched version""" + # Operater list + for op, i_th_ssa in zip(proto.op, ssa): + versioned_inputs, versioned_outputs = i_th_ssa + for i in range(len(op.input)): + if versioned_inputs[i] == (old_name, version): + op.input[i] = new_name + for i in range(len(op.output)): + if versioned_outputs[i] == (old_name, version): + op.output[i] = new_name + # external_input + if start_versions.get(old_name, 0) == version: + for i in range(len(proto.external_input)): + if proto.external_input[i] == old_name: + proto.external_input[i] = new_name + # external_output + if end_versions.get(old_name, 0) == version: + for i in range(len(proto.external_output)): + if proto.external_output[i] == old_name: + proto.external_output[i] = new_name + + +def rename_op_input( + predict_net: caffe2_pb2.NetDef, + init_net: caffe2_pb2.NetDef, + op_id: int, + input_id: int, + new_name: str, + from_producer: bool = False, +): + """ + Rename the op_id-th operator in predict_net, change it's input_id-th input's + name to the new_name. It also does automatic re-route and change + external_input and init_net if necessary. + - It requires the input is only consumed by this op. + - This function modifies predict_net and init_net in-place. + - When from_producer is enable, this also updates other operators that consumes + the same input. Be cautious because may trigger unintended behavior. + """ + assert isinstance(predict_net, caffe2_pb2.NetDef) + assert isinstance(init_net, caffe2_pb2.NetDef) + + init_net_ssa, init_net_versions = core.get_ssa(init_net) + predict_net_ssa, predict_net_versions = core.get_ssa( + predict_net, copy.deepcopy(init_net_versions) + ) + + versioned_inputs, versioned_outputs = predict_net_ssa[op_id] + old_name, version = versioned_inputs[input_id] + + if from_producer: + producer_map = get_producer_map(predict_net_ssa) + if not (old_name, version) in producer_map: + raise NotImplementedError( + "Can't find producer, the input {} is probably from" + " init_net, this is not supported yet.".format(old_name) + ) + producer = producer_map[(old_name, version)] + rename_op_output(predict_net, producer[0], producer[1], new_name) + return + + def contain_targets(op_ssa): + return (old_name, version) in op_ssa[0] + + is_consumer = [contain_targets(op_ssa) for op_ssa in predict_net_ssa] + if sum(is_consumer) > 1: + raise IllegalGraphTransformError( + ( + "Input '{}' of operator(#{}) are consumed by other ops, please use" + + " rename_op_output on the producer instead. Offending op: \n{}" + ).format(old_name, op_id, predict_net.op[op_id]) + ) + + # update init_net + _rename_versioned_blob_in_proto( + init_net, old_name, new_name, version, init_net_ssa, {}, init_net_versions + ) + # update predict_net + _rename_versioned_blob_in_proto( + predict_net, + old_name, + new_name, + version, + predict_net_ssa, + init_net_versions, + predict_net_versions, + ) + + +def rename_op_output(predict_net: caffe2_pb2.NetDef, op_id: int, output_id: int, new_name: str): + """ + Rename the op_id-th operator in predict_net, change it's output_id-th input's + name to the new_name. It also does automatic re-route and change + external_output and if necessary. + - It allows multiple consumers of its output. + - This function modifies predict_net in-place, doesn't need init_net. + """ + assert isinstance(predict_net, caffe2_pb2.NetDef) + + ssa, blob_versions = core.get_ssa(predict_net) + + versioned_inputs, versioned_outputs = ssa[op_id] + old_name, version = versioned_outputs[output_id] + + # update predict_net + _rename_versioned_blob_in_proto( + predict_net, old_name, new_name, version, ssa, {}, blob_versions + ) + + +def get_sub_graph_external_input_output( + predict_net: caffe2_pb2.NetDef, sub_graph_op_indices: List[int] +) -> Tuple[List[Tuple[str, int]], List[Tuple[str, int]]]: + """ + Return the list of external input/output of sub-graph, + each element is tuple of the name and corresponding version in predict_net. + + external input/output is defined the same way as caffe2 NetDef. + """ + ssa, versions = core.get_ssa(predict_net) + + all_inputs = [] + all_outputs = [] + for op_id in sub_graph_op_indices: + all_inputs += [inp for inp in ssa[op_id][0] if inp not in all_inputs] + all_outputs += list(ssa[op_id][1]) # ssa output won't repeat + + # for versioned blobs, external inputs are just those blob in all_inputs + # but not in all_outputs + ext_inputs = [inp for inp in all_inputs if inp not in all_outputs] + + # external outputs are essentially outputs of this subgraph that are used + # outside of this sub-graph (including predict_net.external_output) + all_other_inputs = sum( + (ssa[i][0] for i in range(len(ssa)) if i not in sub_graph_op_indices), + [(outp, versions[outp]) for outp in predict_net.external_output], + ) + ext_outputs = [outp for outp in all_outputs if outp in set(all_other_inputs)] + + return ext_inputs, ext_outputs + + +class DiGraph: + """A DAG representation of caffe2 graph, each vertice is a versioned blob.""" + + def __init__(self): + self.vertices = set() + self.graph = collections.defaultdict(list) + + def add_edge(self, u, v): + self.graph[u].append(v) + self.vertices.add(u) + self.vertices.add(v) + + # grab from https://www.geeksforgeeks.org/find-paths-given-source-destination/ + def get_all_paths(self, s, d): + visited = {k: False for k in self.vertices} + path = [] + all_paths = [] + + def _get_all_paths_util(graph, u, d, visited, path): + visited[u] = True + path.append(u) + if u == d: + all_paths.append(copy.deepcopy(path)) + else: + for i in graph[u]: + if not visited[i]: + _get_all_paths_util(graph, i, d, visited, path) + path.pop() + visited[u] = False + + _get_all_paths_util(self.graph, s, d, visited, path) + return all_paths + + @staticmethod + def from_ssa(ssa): + graph = DiGraph() + for op_id in range(len(ssa)): + for inp in ssa[op_id][0]: + for outp in ssa[op_id][1]: + graph.add_edge(inp, outp) + return graph + + +def _get_dependency_chain(ssa, versioned_target, versioned_source): + """ + Return the index list of relevant operator to produce target blob from source blob, + if there's no dependency, return empty list. + """ + + # finding all paths between nodes can be O(N!), thus we can only search + # in the subgraph using the op starting from the first consumer of source blob + # to the producer of the target blob. + consumer_map = get_consumer_map(ssa) + producer_map = get_producer_map(ssa) + start_op = min(x[0] for x in consumer_map[versioned_source]) - 15 + end_op = ( + producer_map[versioned_target][0] + 15 if versioned_target in producer_map else start_op + ) + sub_graph_ssa = ssa[start_op : end_op + 1] + if len(sub_graph_ssa) > 30: + logger.warning( + "Subgraph bebetween {} and {} is large (from op#{} to op#{}), it" + " might take non-trival time to find all paths between them.".format( + versioned_source, versioned_target, start_op, end_op + ) + ) + + dag = DiGraph.from_ssa(sub_graph_ssa) + paths = dag.get_all_paths(versioned_source, versioned_target) # include two ends + ops_in_paths = [[producer_map[blob][0] for blob in path[1:]] for path in paths] + return sorted(set().union(*[set(ops) for ops in ops_in_paths])) + + +def identify_reshape_sub_graph(predict_net: caffe2_pb2.NetDef) -> List[List[int]]: + """ + Idenfity the reshape sub-graph in a protobuf. + The reshape sub-graph is defined as matching the following pattern: + + (input_blob) -> Op_1 -> ... -> Op_N -> (new_shape) -─┐ + └-------------------------------------------> Reshape -> (output_blob) + + Return: + List of sub-graphs, each sub-graph is represented as a list of indices + of the relavent ops, [Op_1, Op_2, ..., Op_N, Reshape] + """ + + ssa, _ = core.get_ssa(predict_net) + + ret = [] + for i, op in enumerate(predict_net.op): + if op.type == "Reshape": + assert len(op.input) == 2 + input_ssa = ssa[i][0] + data_source = input_ssa[0] + shape_source = input_ssa[1] + op_indices = _get_dependency_chain(ssa, shape_source, data_source) + ret.append(op_indices + [i]) + return ret + + +def remove_reshape_for_fc(predict_net, params): + """ + In PyTorch nn.Linear has to take 2D tensor, this often leads to reshape + a 4D tensor to 2D by calling .view(). However this (dynamic) reshaping + doesn't work well with ONNX and Int8 tools, and cause using extra + ops (eg. ExpandDims) that might not be available on mobile. + Luckily Caffe2 supports 4D tensor for FC, so we can remove those reshape + after exporting ONNX model. + """ + from caffe2.python import core + + # find all reshape sub-graph that can be removed, which is now all Reshape + # sub-graph whose output is only consumed by FC. + # TODO: to make it safer, we may need the actually value to better determine + # if a Reshape before FC is removable. + reshape_sub_graphs = identify_reshape_sub_graph(predict_net) + sub_graphs_to_remove = [] + for reshape_sub_graph in reshape_sub_graphs: + reshape_op_id = reshape_sub_graph[-1] + assert predict_net.op[reshape_op_id].type == "Reshape" + ssa, _ = core.get_ssa(predict_net) + reshape_output = ssa[reshape_op_id][1][0] + consumers = [i for i in range(len(ssa)) if reshape_output in ssa[i][0]] + if all(predict_net.op[consumer].type == "FC" for consumer in consumers): + # safety check if the sub-graph is isolated, for this reshape sub-graph, + # it means it has one non-param external input and one external output. + ext_inputs, ext_outputs = get_sub_graph_external_input_output( + predict_net, reshape_sub_graph + ) + non_params_ext_inputs = [inp for inp in ext_inputs if inp[1] != 0] + if len(non_params_ext_inputs) == 1 and len(ext_outputs) == 1: + sub_graphs_to_remove.append(reshape_sub_graph) + + # perform removing subgraph by: + # 1: rename the Reshape's output to its input, then the graph can be + # seen as in-place itentify, meaning whose external input/output are the same. + # 2: simply remove those ops. + remove_op_ids = [] + params_to_remove = [] + for sub_graph in sub_graphs_to_remove: + logger.info( + "Remove Reshape sub-graph:\n{}".format( + "".join(["(#{:>4})\n{}".format(i, predict_net.op[i]) for i in sub_graph]) + ) + ) + reshape_op_id = sub_graph[-1] + new_reshap_output = predict_net.op[reshape_op_id].input[0] + rename_op_output(predict_net, reshape_op_id, 0, new_reshap_output) + ext_inputs, ext_outputs = get_sub_graph_external_input_output(predict_net, sub_graph) + non_params_ext_inputs = [inp for inp in ext_inputs if inp[1] != 0] + params_ext_inputs = [inp for inp in ext_inputs if inp[1] == 0] + assert len(non_params_ext_inputs) == 1 and len(ext_outputs) == 1 + assert ext_outputs[0][0] == non_params_ext_inputs[0][0] + assert ext_outputs[0][1] == non_params_ext_inputs[0][1] + 1 + remove_op_ids.extend(sub_graph) + params_to_remove.extend(params_ext_inputs) + + predict_net = copy.deepcopy(predict_net) + new_ops = [op for i, op in enumerate(predict_net.op) if i not in remove_op_ids] + del predict_net.op[:] + predict_net.op.extend(new_ops) + for versioned_params in params_to_remove: + name = versioned_params[0] + logger.info("Remove params: {} from init_net and predict_net.external_input".format(name)) + del params[name] + predict_net.external_input.remove(name) + + return predict_net, params + + +def fuse_copy_between_cpu_and_gpu(predict_net: caffe2_pb2.NetDef): + """ + In-place fuse extra copy ops between cpu/gpu for the following case: + a -CopyAToB-> b -CopyBToA> c1 -NextOp1-> d1 + -CopyBToA> c2 -NextOp2-> d2 + The fused network will look like: + a -NextOp1-> d1 + -NextOp2-> d2 + """ + + _COPY_OPS = ["CopyCPUToGPU", "CopyGPUToCPU"] + + def _fuse_once(predict_net): + ssa, blob_versions = core.get_ssa(predict_net) + consumer_map = get_consumer_map(ssa) + versioned_external_output = [ + (name, blob_versions[name]) for name in predict_net.external_output + ] + + for op_id, op in enumerate(predict_net.op): + if op.type in _COPY_OPS: + fw_copy_versioned_output = ssa[op_id][1][0] + consumer_ids = [x[0] for x in consumer_map[fw_copy_versioned_output]] + reverse_op_type = _COPY_OPS[1 - _COPY_OPS.index(op.type)] + + is_fusable = ( + len(consumer_ids) > 0 + and fw_copy_versioned_output not in versioned_external_output + and all( + predict_net.op[_op_id].type == reverse_op_type + and ssa[_op_id][1][0] not in versioned_external_output + for _op_id in consumer_ids + ) + ) + + if is_fusable: + for rv_copy_op_id in consumer_ids: + # making each NextOp uses "a" directly and removing Copy ops + rs_copy_versioned_output = ssa[rv_copy_op_id][1][0] + next_op_id, inp_id = consumer_map[rs_copy_versioned_output][0] + predict_net.op[next_op_id].input[inp_id] = op.input[0] + # remove CopyOps + new_ops = [ + op + for i, op in enumerate(predict_net.op) + if i != op_id and i not in consumer_ids + ] + del predict_net.op[:] + predict_net.op.extend(new_ops) + return True + + return False + + # _fuse_once returns False is nothing can be fused + while _fuse_once(predict_net): + pass + + +def remove_dead_end_ops(net_def: caffe2_pb2.NetDef): + """remove ops if its output is not used or not in external_output""" + ssa, versions = core.get_ssa(net_def) + versioned_external_output = [(name, versions[name]) for name in net_def.external_output] + consumer_map = get_consumer_map(ssa) + removed_op_ids = set() + + def _is_dead_end(versioned_blob): + return not ( + versioned_blob in versioned_external_output + or ( + len(consumer_map[versioned_blob]) > 0 + and all(x[0] not in removed_op_ids for x in consumer_map[versioned_blob]) + ) + ) + + for i, ssa_i in reversed(list(enumerate(ssa))): + versioned_outputs = ssa_i[1] + if all(_is_dead_end(outp) for outp in versioned_outputs): + removed_op_ids.add(i) + + # simply removing those deadend ops should have no effect to external_output + new_ops = [op for i, op in enumerate(net_def.op) if i not in removed_op_ids] + del net_def.op[:] + net_def.op.extend(new_ops) diff --git a/annotator/oneformer/detectron2/export/torchscript.py b/annotator/oneformer/detectron2/export/torchscript.py new file mode 100644 index 0000000000000000000000000000000000000000..8ce1c81e1b7abb65415055ae0d1d4b83e1ae111d --- /dev/null +++ b/annotator/oneformer/detectron2/export/torchscript.py @@ -0,0 +1,132 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import os +import torch + +from annotator.oneformer.detectron2.utils.file_io import PathManager + +from .torchscript_patch import freeze_training_mode, patch_instances + +__all__ = ["scripting_with_instances", "dump_torchscript_IR"] + + +def scripting_with_instances(model, fields): + """ + Run :func:`torch.jit.script` on a model that uses the :class:`Instances` class. Since + attributes of :class:`Instances` are "dynamically" added in eager mode,it is difficult + for scripting to support it out of the box. This function is made to support scripting + a model that uses :class:`Instances`. It does the following: + + 1. Create a scriptable ``new_Instances`` class which behaves similarly to ``Instances``, + but with all attributes been "static". + The attributes need to be statically declared in the ``fields`` argument. + 2. Register ``new_Instances``, and force scripting compiler to + use it when trying to compile ``Instances``. + + After this function, the process will be reverted. User should be able to script another model + using different fields. + + Example: + Assume that ``Instances`` in the model consist of two attributes named + ``proposal_boxes`` and ``objectness_logits`` with type :class:`Boxes` and + :class:`Tensor` respectively during inference. You can call this function like: + :: + fields = {"proposal_boxes": Boxes, "objectness_logits": torch.Tensor} + torchscipt_model = scripting_with_instances(model, fields) + + Note: + It only support models in evaluation mode. + + Args: + model (nn.Module): The input model to be exported by scripting. + fields (Dict[str, type]): Attribute names and corresponding type that + ``Instances`` will use in the model. Note that all attributes used in ``Instances`` + need to be added, regardless of whether they are inputs/outputs of the model. + Data type not defined in detectron2 is not supported for now. + + Returns: + torch.jit.ScriptModule: the model in torchscript format + """ + assert ( + not model.training + ), "Currently we only support exporting models in evaluation mode to torchscript" + + with freeze_training_mode(model), patch_instances(fields): + scripted_model = torch.jit.script(model) + return scripted_model + + +# alias for old name +export_torchscript_with_instances = scripting_with_instances + + +def dump_torchscript_IR(model, dir): + """ + Dump IR of a TracedModule/ScriptModule/Function in various format (code, graph, + inlined graph). Useful for debugging. + + Args: + model (TracedModule/ScriptModule/ScriptFUnction): traced or scripted module + dir (str): output directory to dump files. + """ + dir = os.path.expanduser(dir) + PathManager.mkdirs(dir) + + def _get_script_mod(mod): + if isinstance(mod, torch.jit.TracedModule): + return mod._actual_script_module + return mod + + # Dump pretty-printed code: https://pytorch.org/docs/stable/jit.html#inspecting-code + with PathManager.open(os.path.join(dir, "model_ts_code.txt"), "w") as f: + + def get_code(mod): + # Try a few ways to get code using private attributes. + try: + # This contains more information than just `mod.code` + return _get_script_mod(mod)._c.code + except AttributeError: + pass + try: + return mod.code + except AttributeError: + return None + + def dump_code(prefix, mod): + code = get_code(mod) + name = prefix or "root model" + if code is None: + f.write(f"Could not found code for {name} (type={mod.original_name})\n") + f.write("\n") + else: + f.write(f"\nCode for {name}, type={mod.original_name}:\n") + f.write(code) + f.write("\n") + f.write("-" * 80) + + for name, m in mod.named_children(): + dump_code(prefix + "." + name, m) + + if isinstance(model, torch.jit.ScriptFunction): + f.write(get_code(model)) + else: + dump_code("", model) + + def _get_graph(model): + try: + # Recursively dump IR of all modules + return _get_script_mod(model)._c.dump_to_str(True, False, False) + except AttributeError: + return model.graph.str() + + with PathManager.open(os.path.join(dir, "model_ts_IR.txt"), "w") as f: + f.write(_get_graph(model)) + + # Dump IR of the entire graph (all submodules inlined) + with PathManager.open(os.path.join(dir, "model_ts_IR_inlined.txt"), "w") as f: + f.write(str(model.inlined_graph)) + + if not isinstance(model, torch.jit.ScriptFunction): + # Dump the model structure in pytorch style + with PathManager.open(os.path.join(dir, "model.txt"), "w") as f: + f.write(str(model)) diff --git a/annotator/oneformer/detectron2/export/torchscript_patch.py b/annotator/oneformer/detectron2/export/torchscript_patch.py new file mode 100644 index 0000000000000000000000000000000000000000..24c69b25dbec19221bcd8fc2e928a8393dd3aaf6 --- /dev/null +++ b/annotator/oneformer/detectron2/export/torchscript_patch.py @@ -0,0 +1,406 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import os +import sys +import tempfile +from contextlib import ExitStack, contextmanager +from copy import deepcopy +from unittest import mock +import torch +from torch import nn + +# need some explicit imports due to https://github.com/pytorch/pytorch/issues/38964 +import annotator.oneformer.detectron2 # noqa F401 +from annotator.oneformer.detectron2.structures import Boxes, Instances +from annotator.oneformer.detectron2.utils.env import _import_file + +_counter = 0 + + +def _clear_jit_cache(): + from torch.jit._recursive import concrete_type_store + from torch.jit._state import _jit_caching_layer + + concrete_type_store.type_store.clear() # for modules + _jit_caching_layer.clear() # for free functions + + +def _add_instances_conversion_methods(newInstances): + """ + Add from_instances methods to the scripted Instances class. + """ + cls_name = newInstances.__name__ + + @torch.jit.unused + def from_instances(instances: Instances): + """ + Create scripted Instances from original Instances + """ + fields = instances.get_fields() + image_size = instances.image_size + ret = newInstances(image_size) + for name, val in fields.items(): + assert hasattr(ret, f"_{name}"), f"No attribute named {name} in {cls_name}" + setattr(ret, name, deepcopy(val)) + return ret + + newInstances.from_instances = from_instances + + +@contextmanager +def patch_instances(fields): + """ + A contextmanager, under which the Instances class in detectron2 is replaced + by a statically-typed scriptable class, defined by `fields`. + See more in `scripting_with_instances`. + """ + + with tempfile.TemporaryDirectory(prefix="detectron2") as dir, tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", suffix=".py", dir=dir, delete=False + ) as f: + try: + # Objects that use Instances should not reuse previously-compiled + # results in cache, because `Instances` could be a new class each time. + _clear_jit_cache() + + cls_name, s = _gen_instance_module(fields) + f.write(s) + f.flush() + f.close() + + module = _import(f.name) + new_instances = getattr(module, cls_name) + _ = torch.jit.script(new_instances) + # let torchscript think Instances was scripted already + Instances.__torch_script_class__ = True + # let torchscript find new_instances when looking for the jit type of Instances + Instances._jit_override_qualname = torch._jit_internal._qualified_name(new_instances) + + _add_instances_conversion_methods(new_instances) + yield new_instances + finally: + try: + del Instances.__torch_script_class__ + del Instances._jit_override_qualname + except AttributeError: + pass + sys.modules.pop(module.__name__) + + +def _gen_instance_class(fields): + """ + Args: + fields (dict[name: type]) + """ + + class _FieldType: + def __init__(self, name, type_): + assert isinstance(name, str), f"Field name must be str, got {name}" + self.name = name + self.type_ = type_ + self.annotation = f"{type_.__module__}.{type_.__name__}" + + fields = [_FieldType(k, v) for k, v in fields.items()] + + def indent(level, s): + return " " * 4 * level + s + + lines = [] + + global _counter + _counter += 1 + + cls_name = "ScriptedInstances{}".format(_counter) + + field_names = tuple(x.name for x in fields) + extra_args = ", ".join([f"{f.name}: Optional[{f.annotation}] = None" for f in fields]) + lines.append( + f""" +class {cls_name}: + def __init__(self, image_size: Tuple[int, int], {extra_args}): + self.image_size = image_size + self._field_names = {field_names} +""" + ) + + for f in fields: + lines.append( + indent(2, f"self._{f.name} = torch.jit.annotate(Optional[{f.annotation}], {f.name})") + ) + + for f in fields: + lines.append( + f""" + @property + def {f.name}(self) -> {f.annotation}: + # has to use a local for type refinement + # https://pytorch.org/docs/stable/jit_language_reference.html#optional-type-refinement + t = self._{f.name} + assert t is not None, "{f.name} is None and cannot be accessed!" + return t + + @{f.name}.setter + def {f.name}(self, value: {f.annotation}) -> None: + self._{f.name} = value +""" + ) + + # support method `__len__` + lines.append( + """ + def __len__(self) -> int: +""" + ) + for f in fields: + lines.append( + f""" + t = self._{f.name} + if t is not None: + return len(t) +""" + ) + lines.append( + """ + raise NotImplementedError("Empty Instances does not support __len__!") +""" + ) + + # support method `has` + lines.append( + """ + def has(self, name: str) -> bool: +""" + ) + for f in fields: + lines.append( + f""" + if name == "{f.name}": + return self._{f.name} is not None +""" + ) + lines.append( + """ + return False +""" + ) + + # support method `to` + none_args = ", None" * len(fields) + lines.append( + f""" + def to(self, device: torch.device) -> "{cls_name}": + ret = {cls_name}(self.image_size{none_args}) +""" + ) + for f in fields: + if hasattr(f.type_, "to"): + lines.append( + f""" + t = self._{f.name} + if t is not None: + ret._{f.name} = t.to(device) +""" + ) + else: + # For now, ignore fields that cannot be moved to devices. + # Maybe can support other tensor-like classes (e.g. __torch_function__) + pass + lines.append( + """ + return ret +""" + ) + + # support method `getitem` + none_args = ", None" * len(fields) + lines.append( + f""" + def __getitem__(self, item) -> "{cls_name}": + ret = {cls_name}(self.image_size{none_args}) +""" + ) + for f in fields: + lines.append( + f""" + t = self._{f.name} + if t is not None: + ret._{f.name} = t[item] +""" + ) + lines.append( + """ + return ret +""" + ) + + # support method `cat` + # this version does not contain checks that all instances have same size and fields + none_args = ", None" * len(fields) + lines.append( + f""" + def cat(self, instances: List["{cls_name}"]) -> "{cls_name}": + ret = {cls_name}(self.image_size{none_args}) +""" + ) + for f in fields: + lines.append( + f""" + t = self._{f.name} + if t is not None: + values: List[{f.annotation}] = [x.{f.name} for x in instances] + if torch.jit.isinstance(t, torch.Tensor): + ret._{f.name} = torch.cat(values, dim=0) + else: + ret._{f.name} = t.cat(values) +""" + ) + lines.append( + """ + return ret""" + ) + + # support method `get_fields()` + lines.append( + """ + def get_fields(self) -> Dict[str, Tensor]: + ret = {} + """ + ) + for f in fields: + if f.type_ == Boxes: + stmt = "t.tensor" + elif f.type_ == torch.Tensor: + stmt = "t" + else: + stmt = f'assert False, "unsupported type {str(f.type_)}"' + lines.append( + f""" + t = self._{f.name} + if t is not None: + ret["{f.name}"] = {stmt} + """ + ) + lines.append( + """ + return ret""" + ) + return cls_name, os.linesep.join(lines) + + +def _gen_instance_module(fields): + # TODO: find a more automatic way to enable import of other classes + s = """ +from copy import deepcopy +import torch +from torch import Tensor +import typing +from typing import * + +import annotator.oneformer.detectron2 +from annotator.oneformer.detectron2.structures import Boxes, Instances + +""" + + cls_name, cls_def = _gen_instance_class(fields) + s += cls_def + return cls_name, s + + +def _import(path): + return _import_file( + "{}{}".format(sys.modules[__name__].__name__, _counter), path, make_importable=True + ) + + +@contextmanager +def patch_builtin_len(modules=()): + """ + Patch the builtin len() function of a few detectron2 modules + to use __len__ instead, because __len__ does not convert values to + integers and therefore is friendly to tracing. + + Args: + modules (list[stsr]): names of extra modules to patch len(), in + addition to those in detectron2. + """ + + def _new_len(obj): + return obj.__len__() + + with ExitStack() as stack: + MODULES = [ + "detectron2.modeling.roi_heads.fast_rcnn", + "detectron2.modeling.roi_heads.mask_head", + "detectron2.modeling.roi_heads.keypoint_head", + ] + list(modules) + ctxs = [stack.enter_context(mock.patch(mod + ".len")) for mod in MODULES] + for m in ctxs: + m.side_effect = _new_len + yield + + +def patch_nonscriptable_classes(): + """ + Apply patches on a few nonscriptable detectron2 classes. + Should not have side-effects on eager usage. + """ + # __prepare_scriptable__ can also be added to models for easier maintenance. + # But it complicates the clean model code. + + from annotator.oneformer.detectron2.modeling.backbone import ResNet, FPN + + # Due to https://github.com/pytorch/pytorch/issues/36061, + # we change backbone to use ModuleList for scripting. + # (note: this changes param names in state_dict) + + def prepare_resnet(self): + ret = deepcopy(self) + ret.stages = nn.ModuleList(ret.stages) + for k in self.stage_names: + delattr(ret, k) + return ret + + ResNet.__prepare_scriptable__ = prepare_resnet + + def prepare_fpn(self): + ret = deepcopy(self) + ret.lateral_convs = nn.ModuleList(ret.lateral_convs) + ret.output_convs = nn.ModuleList(ret.output_convs) + for name, _ in self.named_children(): + if name.startswith("fpn_"): + delattr(ret, name) + return ret + + FPN.__prepare_scriptable__ = prepare_fpn + + # Annotate some attributes to be constants for the purpose of scripting, + # even though they are not constants in eager mode. + from annotator.oneformer.detectron2.modeling.roi_heads import StandardROIHeads + + if hasattr(StandardROIHeads, "__annotations__"): + # copy first to avoid editing annotations of base class + StandardROIHeads.__annotations__ = deepcopy(StandardROIHeads.__annotations__) + StandardROIHeads.__annotations__["mask_on"] = torch.jit.Final[bool] + StandardROIHeads.__annotations__["keypoint_on"] = torch.jit.Final[bool] + + +# These patches are not supposed to have side-effects. +patch_nonscriptable_classes() + + +@contextmanager +def freeze_training_mode(model): + """ + A context manager that annotates the "training" attribute of every submodule + to constant, so that the training codepath in these modules can be + meta-compiled away. Upon exiting, the annotations are reverted. + """ + classes = {type(x) for x in model.modules()} + # __constants__ is the old way to annotate constants and not compatible + # with __annotations__ . + classes = {x for x in classes if not hasattr(x, "__constants__")} + for cls in classes: + cls.__annotations__["training"] = torch.jit.Final[bool] + yield + for cls in classes: + cls.__annotations__["training"] = bool diff --git a/annotator/oneformer/detectron2/layers/__init__.py b/annotator/oneformer/detectron2/layers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..761a3d1c7afa049e9779ee9fc4d299e9aae38cad --- /dev/null +++ b/annotator/oneformer/detectron2/layers/__init__.py @@ -0,0 +1,26 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .batch_norm import FrozenBatchNorm2d, get_norm, NaiveSyncBatchNorm, CycleBatchNormList +from .deform_conv import DeformConv, ModulatedDeformConv +from .mask_ops import paste_masks_in_image +from .nms import batched_nms, batched_nms_rotated, nms, nms_rotated +from .roi_align import ROIAlign, roi_align +from .roi_align_rotated import ROIAlignRotated, roi_align_rotated +from .shape_spec import ShapeSpec +from .wrappers import ( + BatchNorm2d, + Conv2d, + ConvTranspose2d, + cat, + interpolate, + Linear, + nonzero_tuple, + cross_entropy, + empty_input_loss_func_wrapper, + shapes_to_tensor, + move_device_like, +) +from .blocks import CNNBlockBase, DepthwiseSeparableConv2d +from .aspp import ASPP +from .losses import ciou_loss, diou_loss + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/annotator/oneformer/detectron2/layers/aspp.py b/annotator/oneformer/detectron2/layers/aspp.py new file mode 100644 index 0000000000000000000000000000000000000000..14861aa9ede4fea6a69a49f189bcab997b558148 --- /dev/null +++ b/annotator/oneformer/detectron2/layers/aspp.py @@ -0,0 +1,144 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from copy import deepcopy +import fvcore.nn.weight_init as weight_init +import torch +from torch import nn +from torch.nn import functional as F + +from .batch_norm import get_norm +from .blocks import DepthwiseSeparableConv2d +from .wrappers import Conv2d + + +class ASPP(nn.Module): + """ + Atrous Spatial Pyramid Pooling (ASPP). + """ + + def __init__( + self, + in_channels, + out_channels, + dilations, + *, + norm, + activation, + pool_kernel_size=None, + dropout: float = 0.0, + use_depthwise_separable_conv=False, + ): + """ + Args: + in_channels (int): number of input channels for ASPP. + out_channels (int): number of output channels. + dilations (list): a list of 3 dilations in ASPP. + norm (str or callable): normalization for all conv layers. + See :func:`layers.get_norm` for supported format. norm is + applied to all conv layers except the conv following + global average pooling. + activation (callable): activation function. + pool_kernel_size (tuple, list): the average pooling size (kh, kw) + for image pooling layer in ASPP. If set to None, it always + performs global average pooling. If not None, it must be + divisible by the shape of inputs in forward(). It is recommended + to use a fixed input feature size in training, and set this + option to match this size, so that it performs global average + pooling in training, and the size of the pooling window stays + consistent in inference. + dropout (float): apply dropout on the output of ASPP. It is used in + the official DeepLab implementation with a rate of 0.1: + https://github.com/tensorflow/models/blob/21b73d22f3ed05b650e85ac50849408dd36de32e/research/deeplab/model.py#L532 # noqa + use_depthwise_separable_conv (bool): use DepthwiseSeparableConv2d + for 3x3 convs in ASPP, proposed in :paper:`DeepLabV3+`. + """ + super(ASPP, self).__init__() + assert len(dilations) == 3, "ASPP expects 3 dilations, got {}".format(len(dilations)) + self.pool_kernel_size = pool_kernel_size + self.dropout = dropout + use_bias = norm == "" + self.convs = nn.ModuleList() + # conv 1x1 + self.convs.append( + Conv2d( + in_channels, + out_channels, + kernel_size=1, + bias=use_bias, + norm=get_norm(norm, out_channels), + activation=deepcopy(activation), + ) + ) + weight_init.c2_xavier_fill(self.convs[-1]) + # atrous convs + for dilation in dilations: + if use_depthwise_separable_conv: + self.convs.append( + DepthwiseSeparableConv2d( + in_channels, + out_channels, + kernel_size=3, + padding=dilation, + dilation=dilation, + norm1=norm, + activation1=deepcopy(activation), + norm2=norm, + activation2=deepcopy(activation), + ) + ) + else: + self.convs.append( + Conv2d( + in_channels, + out_channels, + kernel_size=3, + padding=dilation, + dilation=dilation, + bias=use_bias, + norm=get_norm(norm, out_channels), + activation=deepcopy(activation), + ) + ) + weight_init.c2_xavier_fill(self.convs[-1]) + # image pooling + # We do not add BatchNorm because the spatial resolution is 1x1, + # the original TF implementation has BatchNorm. + if pool_kernel_size is None: + image_pooling = nn.Sequential( + nn.AdaptiveAvgPool2d(1), + Conv2d(in_channels, out_channels, 1, bias=True, activation=deepcopy(activation)), + ) + else: + image_pooling = nn.Sequential( + nn.AvgPool2d(kernel_size=pool_kernel_size, stride=1), + Conv2d(in_channels, out_channels, 1, bias=True, activation=deepcopy(activation)), + ) + weight_init.c2_xavier_fill(image_pooling[1]) + self.convs.append(image_pooling) + + self.project = Conv2d( + 5 * out_channels, + out_channels, + kernel_size=1, + bias=use_bias, + norm=get_norm(norm, out_channels), + activation=deepcopy(activation), + ) + weight_init.c2_xavier_fill(self.project) + + def forward(self, x): + size = x.shape[-2:] + if self.pool_kernel_size is not None: + if size[0] % self.pool_kernel_size[0] or size[1] % self.pool_kernel_size[1]: + raise ValueError( + "`pool_kernel_size` must be divisible by the shape of inputs. " + "Input size: {} `pool_kernel_size`: {}".format(size, self.pool_kernel_size) + ) + res = [] + for conv in self.convs: + res.append(conv(x)) + res[-1] = F.interpolate(res[-1], size=size, mode="bilinear", align_corners=False) + res = torch.cat(res, dim=1) + res = self.project(res) + res = F.dropout(res, self.dropout, training=self.training) if self.dropout > 0 else res + return res diff --git a/annotator/oneformer/detectron2/layers/batch_norm.py b/annotator/oneformer/detectron2/layers/batch_norm.py new file mode 100644 index 0000000000000000000000000000000000000000..32a1e05470065e75b6caad18d36211d27af8eec0 --- /dev/null +++ b/annotator/oneformer/detectron2/layers/batch_norm.py @@ -0,0 +1,300 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import torch +import torch.distributed as dist +from fvcore.nn.distributed import differentiable_all_reduce +from torch import nn +from torch.nn import functional as F + +from annotator.oneformer.detectron2.utils import comm, env + +from .wrappers import BatchNorm2d + + +class FrozenBatchNorm2d(nn.Module): + """ + BatchNorm2d where the batch statistics and the affine parameters are fixed. + + It contains non-trainable buffers called + "weight" and "bias", "running_mean", "running_var", + initialized to perform identity transformation. + + The pre-trained backbone models from Caffe2 only contain "weight" and "bias", + which are computed from the original four parameters of BN. + The affine transform `x * weight + bias` will perform the equivalent + computation of `(x - running_mean) / sqrt(running_var) * weight + bias`. + When loading a backbone model from Caffe2, "running_mean" and "running_var" + will be left unchanged as identity transformation. + + Other pre-trained backbone models may contain all 4 parameters. + + The forward is implemented by `F.batch_norm(..., training=False)`. + """ + + _version = 3 + + def __init__(self, num_features, eps=1e-5): + super().__init__() + self.num_features = num_features + self.eps = eps + self.register_buffer("weight", torch.ones(num_features)) + self.register_buffer("bias", torch.zeros(num_features)) + self.register_buffer("running_mean", torch.zeros(num_features)) + self.register_buffer("running_var", torch.ones(num_features) - eps) + + def forward(self, x): + if x.requires_grad: + # When gradients are needed, F.batch_norm will use extra memory + # because its backward op computes gradients for weight/bias as well. + scale = self.weight * (self.running_var + self.eps).rsqrt() + bias = self.bias - self.running_mean * scale + scale = scale.reshape(1, -1, 1, 1) + bias = bias.reshape(1, -1, 1, 1) + out_dtype = x.dtype # may be half + return x * scale.to(out_dtype) + bias.to(out_dtype) + else: + # When gradients are not needed, F.batch_norm is a single fused op + # and provide more optimization opportunities. + return F.batch_norm( + x, + self.running_mean, + self.running_var, + self.weight, + self.bias, + training=False, + eps=self.eps, + ) + + def _load_from_state_dict( + self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs + ): + version = local_metadata.get("version", None) + + if version is None or version < 2: + # No running_mean/var in early versions + # This will silent the warnings + if prefix + "running_mean" not in state_dict: + state_dict[prefix + "running_mean"] = torch.zeros_like(self.running_mean) + if prefix + "running_var" not in state_dict: + state_dict[prefix + "running_var"] = torch.ones_like(self.running_var) + + super()._load_from_state_dict( + state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs + ) + + def __repr__(self): + return "FrozenBatchNorm2d(num_features={}, eps={})".format(self.num_features, self.eps) + + @classmethod + def convert_frozen_batchnorm(cls, module): + """ + Convert all BatchNorm/SyncBatchNorm in module into FrozenBatchNorm. + + Args: + module (torch.nn.Module): + + Returns: + If module is BatchNorm/SyncBatchNorm, returns a new module. + Otherwise, in-place convert module and return it. + + Similar to convert_sync_batchnorm in + https://github.com/pytorch/pytorch/blob/master/torch/nn/modules/batchnorm.py + """ + bn_module = nn.modules.batchnorm + bn_module = (bn_module.BatchNorm2d, bn_module.SyncBatchNorm) + res = module + if isinstance(module, bn_module): + res = cls(module.num_features) + if module.affine: + res.weight.data = module.weight.data.clone().detach() + res.bias.data = module.bias.data.clone().detach() + res.running_mean.data = module.running_mean.data + res.running_var.data = module.running_var.data + res.eps = module.eps + else: + for name, child in module.named_children(): + new_child = cls.convert_frozen_batchnorm(child) + if new_child is not child: + res.add_module(name, new_child) + return res + + +def get_norm(norm, out_channels): + """ + Args: + norm (str or callable): either one of BN, SyncBN, FrozenBN, GN; + or a callable that takes a channel number and returns + the normalization layer as a nn.Module. + + Returns: + nn.Module or None: the normalization layer + """ + if norm is None: + return None + if isinstance(norm, str): + if len(norm) == 0: + return None + norm = { + "BN": BatchNorm2d, + # Fixed in https://github.com/pytorch/pytorch/pull/36382 + "SyncBN": NaiveSyncBatchNorm if env.TORCH_VERSION <= (1, 5) else nn.SyncBatchNorm, + "FrozenBN": FrozenBatchNorm2d, + "GN": lambda channels: nn.GroupNorm(32, channels), + # for debugging: + "nnSyncBN": nn.SyncBatchNorm, + "naiveSyncBN": NaiveSyncBatchNorm, + # expose stats_mode N as an option to caller, required for zero-len inputs + "naiveSyncBN_N": lambda channels: NaiveSyncBatchNorm(channels, stats_mode="N"), + "LN": lambda channels: LayerNorm(channels), + }[norm] + return norm(out_channels) + + +class NaiveSyncBatchNorm(BatchNorm2d): + """ + In PyTorch<=1.5, ``nn.SyncBatchNorm`` has incorrect gradient + when the batch size on each worker is different. + (e.g., when scale augmentation is used, or when it is applied to mask head). + + This is a slower but correct alternative to `nn.SyncBatchNorm`. + + Note: + There isn't a single definition of Sync BatchNorm. + + When ``stats_mode==""``, this module computes overall statistics by using + statistics of each worker with equal weight. The result is true statistics + of all samples (as if they are all on one worker) only when all workers + have the same (N, H, W). This mode does not support inputs with zero batch size. + + When ``stats_mode=="N"``, this module computes overall statistics by weighting + the statistics of each worker by their ``N``. The result is true statistics + of all samples (as if they are all on one worker) only when all workers + have the same (H, W). It is slower than ``stats_mode==""``. + + Even though the result of this module may not be the true statistics of all samples, + it may still be reasonable because it might be preferrable to assign equal weights + to all workers, regardless of their (H, W) dimension, instead of putting larger weight + on larger images. From preliminary experiments, little difference is found between such + a simplified implementation and an accurate computation of overall mean & variance. + """ + + def __init__(self, *args, stats_mode="", **kwargs): + super().__init__(*args, **kwargs) + assert stats_mode in ["", "N"] + self._stats_mode = stats_mode + + def forward(self, input): + if comm.get_world_size() == 1 or not self.training: + return super().forward(input) + + B, C = input.shape[0], input.shape[1] + + half_input = input.dtype == torch.float16 + if half_input: + # fp16 does not have good enough numerics for the reduction here + input = input.float() + mean = torch.mean(input, dim=[0, 2, 3]) + meansqr = torch.mean(input * input, dim=[0, 2, 3]) + + if self._stats_mode == "": + assert B > 0, 'SyncBatchNorm(stats_mode="") does not support zero batch size.' + vec = torch.cat([mean, meansqr], dim=0) + vec = differentiable_all_reduce(vec) * (1.0 / dist.get_world_size()) + mean, meansqr = torch.split(vec, C) + momentum = self.momentum + else: + if B == 0: + vec = torch.zeros([2 * C + 1], device=mean.device, dtype=mean.dtype) + vec = vec + input.sum() # make sure there is gradient w.r.t input + else: + vec = torch.cat( + [mean, meansqr, torch.ones([1], device=mean.device, dtype=mean.dtype)], dim=0 + ) + vec = differentiable_all_reduce(vec * B) + + total_batch = vec[-1].detach() + momentum = total_batch.clamp(max=1) * self.momentum # no update if total_batch is 0 + mean, meansqr, _ = torch.split(vec / total_batch.clamp(min=1), C) # avoid div-by-zero + + var = meansqr - mean * mean + invstd = torch.rsqrt(var + self.eps) + scale = self.weight * invstd + bias = self.bias - mean * scale + scale = scale.reshape(1, -1, 1, 1) + bias = bias.reshape(1, -1, 1, 1) + + self.running_mean += momentum * (mean.detach() - self.running_mean) + self.running_var += momentum * (var.detach() - self.running_var) + ret = input * scale + bias + if half_input: + ret = ret.half() + return ret + + +class CycleBatchNormList(nn.ModuleList): + """ + Implement domain-specific BatchNorm by cycling. + + When a BatchNorm layer is used for multiple input domains or input + features, it might need to maintain a separate test-time statistics + for each domain. See Sec 5.2 in :paper:`rethinking-batchnorm`. + + This module implements it by using N separate BN layers + and it cycles through them every time a forward() is called. + + NOTE: The caller of this module MUST guarantee to always call + this module by multiple of N times. Otherwise its test-time statistics + will be incorrect. + """ + + def __init__(self, length: int, bn_class=nn.BatchNorm2d, **kwargs): + """ + Args: + length: number of BatchNorm layers to cycle. + bn_class: the BatchNorm class to use + kwargs: arguments of the BatchNorm class, such as num_features. + """ + self._affine = kwargs.pop("affine", True) + super().__init__([bn_class(**kwargs, affine=False) for k in range(length)]) + if self._affine: + # shared affine, domain-specific BN + channels = self[0].num_features + self.weight = nn.Parameter(torch.ones(channels)) + self.bias = nn.Parameter(torch.zeros(channels)) + self._pos = 0 + + def forward(self, x): + ret = self[self._pos](x) + self._pos = (self._pos + 1) % len(self) + + if self._affine: + w = self.weight.reshape(1, -1, 1, 1) + b = self.bias.reshape(1, -1, 1, 1) + return ret * w + b + else: + return ret + + def extra_repr(self): + return f"affine={self._affine}" + + +class LayerNorm(nn.Module): + """ + A LayerNorm variant, popularized by Transformers, that performs point-wise mean and + variance normalization over the channel dimension for inputs that have shape + (batch_size, channels, height, width). + https://github.com/facebookresearch/ConvNeXt/blob/d1fa8f6fef0a165b27399986cc2bdacc92777e40/models/convnext.py#L119 # noqa B950 + """ + + def __init__(self, normalized_shape, eps=1e-6): + super().__init__() + self.weight = nn.Parameter(torch.ones(normalized_shape)) + self.bias = nn.Parameter(torch.zeros(normalized_shape)) + self.eps = eps + self.normalized_shape = (normalized_shape,) + + def forward(self, x): + u = x.mean(1, keepdim=True) + s = (x - u).pow(2).mean(1, keepdim=True) + x = (x - u) / torch.sqrt(s + self.eps) + x = self.weight[:, None, None] * x + self.bias[:, None, None] + return x diff --git a/annotator/oneformer/detectron2/layers/blocks.py b/annotator/oneformer/detectron2/layers/blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..1995a4bf7339e8deb7eaaffda4f819dda55e7ac7 --- /dev/null +++ b/annotator/oneformer/detectron2/layers/blocks.py @@ -0,0 +1,111 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +import fvcore.nn.weight_init as weight_init +from torch import nn + +from .batch_norm import FrozenBatchNorm2d, get_norm +from .wrappers import Conv2d + + +""" +CNN building blocks. +""" + + +class CNNBlockBase(nn.Module): + """ + A CNN block is assumed to have input channels, output channels and a stride. + The input and output of `forward()` method must be NCHW tensors. + The method can perform arbitrary computation but must match the given + channels and stride specification. + + Attribute: + in_channels (int): + out_channels (int): + stride (int): + """ + + def __init__(self, in_channels, out_channels, stride): + """ + The `__init__` method of any subclass should also contain these arguments. + + Args: + in_channels (int): + out_channels (int): + stride (int): + """ + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.stride = stride + + def freeze(self): + """ + Make this block not trainable. + This method sets all parameters to `requires_grad=False`, + and convert all BatchNorm layers to FrozenBatchNorm + + Returns: + the block itself + """ + for p in self.parameters(): + p.requires_grad = False + FrozenBatchNorm2d.convert_frozen_batchnorm(self) + return self + + +class DepthwiseSeparableConv2d(nn.Module): + """ + A kxk depthwise convolution + a 1x1 convolution. + + In :paper:`xception`, norm & activation are applied on the second conv. + :paper:`mobilenet` uses norm & activation on both convs. + """ + + def __init__( + self, + in_channels, + out_channels, + kernel_size=3, + padding=1, + dilation=1, + *, + norm1=None, + activation1=None, + norm2=None, + activation2=None, + ): + """ + Args: + norm1, norm2 (str or callable): normalization for the two conv layers. + activation1, activation2 (callable(Tensor) -> Tensor): activation + function for the two conv layers. + """ + super().__init__() + self.depthwise = Conv2d( + in_channels, + in_channels, + kernel_size=kernel_size, + padding=padding, + dilation=dilation, + groups=in_channels, + bias=not norm1, + norm=get_norm(norm1, in_channels), + activation=activation1, + ) + self.pointwise = Conv2d( + in_channels, + out_channels, + kernel_size=1, + bias=not norm2, + norm=get_norm(norm2, out_channels), + activation=activation2, + ) + + # default initialization + weight_init.c2_msra_fill(self.depthwise) + weight_init.c2_msra_fill(self.pointwise) + + def forward(self, x): + return self.pointwise(self.depthwise(x)) diff --git a/annotator/oneformer/detectron2/layers/csrc/README.md b/annotator/oneformer/detectron2/layers/csrc/README.md new file mode 100644 index 0000000000000000000000000000000000000000..778ed3da0bae89820831bcd8a72ff7b9cad8d4dd --- /dev/null +++ b/annotator/oneformer/detectron2/layers/csrc/README.md @@ -0,0 +1,7 @@ + + +To add a new Op: + +1. Create a new directory +2. Implement new ops there +3. Delcare its Python interface in `vision.cpp`. diff --git a/annotator/oneformer/detectron2/layers/csrc/ROIAlignRotated/ROIAlignRotated.h b/annotator/oneformer/detectron2/layers/csrc/ROIAlignRotated/ROIAlignRotated.h new file mode 100644 index 0000000000000000000000000000000000000000..03f4211003f42f601f0cfcf4a690f5da4a0a1f67 --- /dev/null +++ b/annotator/oneformer/detectron2/layers/csrc/ROIAlignRotated/ROIAlignRotated.h @@ -0,0 +1,115 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#pragma once +#include + +namespace detectron2 { + +at::Tensor ROIAlignRotated_forward_cpu( + const at::Tensor& input, + const at::Tensor& rois, + const float spatial_scale, + const int pooled_height, + const int pooled_width, + const int sampling_ratio); + +at::Tensor ROIAlignRotated_backward_cpu( + const at::Tensor& grad, + const at::Tensor& rois, + const float spatial_scale, + const int pooled_height, + const int pooled_width, + const int batch_size, + const int channels, + const int height, + const int width, + const int sampling_ratio); + +#if defined(WITH_CUDA) || defined(WITH_HIP) +at::Tensor ROIAlignRotated_forward_cuda( + const at::Tensor& input, + const at::Tensor& rois, + const float spatial_scale, + const int pooled_height, + const int pooled_width, + const int sampling_ratio); + +at::Tensor ROIAlignRotated_backward_cuda( + const at::Tensor& grad, + const at::Tensor& rois, + const float spatial_scale, + const int pooled_height, + const int pooled_width, + const int batch_size, + const int channels, + const int height, + const int width, + const int sampling_ratio); +#endif + +// Interface for Python +inline at::Tensor ROIAlignRotated_forward( + const at::Tensor& input, + const at::Tensor& rois, + const double spatial_scale, + const int64_t pooled_height, + const int64_t pooled_width, + const int64_t sampling_ratio) { + if (input.is_cuda()) { +#if defined(WITH_CUDA) || defined(WITH_HIP) + return ROIAlignRotated_forward_cuda( + input, + rois, + spatial_scale, + pooled_height, + pooled_width, + sampling_ratio); +#else + AT_ERROR("Detectron2 is not compiled with GPU support!"); +#endif + } + return ROIAlignRotated_forward_cpu( + input, rois, spatial_scale, pooled_height, pooled_width, sampling_ratio); +} + +inline at::Tensor ROIAlignRotated_backward( + const at::Tensor& grad, + const at::Tensor& rois, + const double spatial_scale, + const int64_t pooled_height, + const int64_t pooled_width, + const int64_t batch_size, + const int64_t channels, + const int64_t height, + const int64_t width, + const int64_t sampling_ratio) { + if (grad.is_cuda()) { +#if defined(WITH_CUDA) || defined(WITH_HIP) + return ROIAlignRotated_backward_cuda( + grad, + rois, + spatial_scale, + pooled_height, + pooled_width, + batch_size, + channels, + height, + width, + sampling_ratio); +#else + AT_ERROR("Detectron2 is not compiled with GPU support!"); +#endif + } + return ROIAlignRotated_backward_cpu( + grad, + rois, + spatial_scale, + pooled_height, + pooled_width, + batch_size, + channels, + height, + width, + sampling_ratio); +} + +} // namespace detectron2 diff --git a/annotator/oneformer/detectron2/layers/csrc/ROIAlignRotated/ROIAlignRotated_cpu.cpp b/annotator/oneformer/detectron2/layers/csrc/ROIAlignRotated/ROIAlignRotated_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2a3d3056cc71a4acaafb570739a9dd247a7eb1ed --- /dev/null +++ b/annotator/oneformer/detectron2/layers/csrc/ROIAlignRotated/ROIAlignRotated_cpu.cpp @@ -0,0 +1,522 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#include +#include "ROIAlignRotated.h" + +// Note: this implementation originates from the Caffe2 ROIAlignRotated Op +// and PyTorch ROIAlign (non-rotated) Op implementations. +// The key difference between this implementation and those ones is +// we don't do "legacy offset" in this version, as there aren't many previous +// works, if any, using the "legacy" ROIAlignRotated Op. +// This would make the interface a bit cleaner. + +namespace detectron2 { + +namespace { +template +struct PreCalc { + int pos1; + int pos2; + int pos3; + int pos4; + T w1; + T w2; + T w3; + T w4; +}; + +template +void pre_calc_for_bilinear_interpolate( + const int height, + const int width, + const int pooled_height, + const int pooled_width, + const int iy_upper, + const int ix_upper, + T roi_start_h, + T roi_start_w, + T bin_size_h, + T bin_size_w, + int roi_bin_grid_h, + int roi_bin_grid_w, + T roi_center_h, + T roi_center_w, + T cos_theta, + T sin_theta, + std::vector>& pre_calc) { + int pre_calc_index = 0; + for (int ph = 0; ph < pooled_height; ph++) { + for (int pw = 0; pw < pooled_width; pw++) { + for (int iy = 0; iy < iy_upper; iy++) { + const T yy = roi_start_h + ph * bin_size_h + + static_cast(iy + .5f) * bin_size_h / + static_cast(roi_bin_grid_h); // e.g., 0.5, 1.5 + for (int ix = 0; ix < ix_upper; ix++) { + const T xx = roi_start_w + pw * bin_size_w + + static_cast(ix + .5f) * bin_size_w / + static_cast(roi_bin_grid_w); + + // Rotate by theta around the center and translate + // In image space, (y, x) is the order for Right Handed System, + // and this is essentially multiplying the point by a rotation matrix + // to rotate it counterclockwise through angle theta. + T y = yy * cos_theta - xx * sin_theta + roi_center_h; + T x = yy * sin_theta + xx * cos_theta + roi_center_w; + // deal with: inverse elements are out of feature map boundary + if (y < -1.0 || y > height || x < -1.0 || x > width) { + // empty + PreCalc pc; + pc.pos1 = 0; + pc.pos2 = 0; + pc.pos3 = 0; + pc.pos4 = 0; + pc.w1 = 0; + pc.w2 = 0; + pc.w3 = 0; + pc.w4 = 0; + pre_calc[pre_calc_index] = pc; + pre_calc_index += 1; + continue; + } + + if (y < 0) { + y = 0; + } + if (x < 0) { + x = 0; + } + + int y_low = (int)y; + int x_low = (int)x; + int y_high; + int x_high; + + if (y_low >= height - 1) { + y_high = y_low = height - 1; + y = (T)y_low; + } else { + y_high = y_low + 1; + } + + if (x_low >= width - 1) { + x_high = x_low = width - 1; + x = (T)x_low; + } else { + x_high = x_low + 1; + } + + T ly = y - y_low; + T lx = x - x_low; + T hy = 1. - ly, hx = 1. - lx; + T w1 = hy * hx, w2 = hy * lx, w3 = ly * hx, w4 = ly * lx; + + // save weights and indices + PreCalc pc; + pc.pos1 = y_low * width + x_low; + pc.pos2 = y_low * width + x_high; + pc.pos3 = y_high * width + x_low; + pc.pos4 = y_high * width + x_high; + pc.w1 = w1; + pc.w2 = w2; + pc.w3 = w3; + pc.w4 = w4; + pre_calc[pre_calc_index] = pc; + + pre_calc_index += 1; + } + } + } + } +} + +template +void bilinear_interpolate_gradient( + const int height, + const int width, + T y, + T x, + T& w1, + T& w2, + T& w3, + T& w4, + int& x_low, + int& x_high, + int& y_low, + int& y_high) { + // deal with cases that inverse elements are out of feature map boundary + if (y < -1.0 || y > height || x < -1.0 || x > width) { + // empty + w1 = w2 = w3 = w4 = 0.; + x_low = x_high = y_low = y_high = -1; + return; + } + + if (y < 0) { + y = 0; + } + + if (x < 0) { + x = 0; + } + + y_low = (int)y; + x_low = (int)x; + + if (y_low >= height - 1) { + y_high = y_low = height - 1; + y = (T)y_low; + } else { + y_high = y_low + 1; + } + + if (x_low >= width - 1) { + x_high = x_low = width - 1; + x = (T)x_low; + } else { + x_high = x_low + 1; + } + + T ly = y - y_low; + T lx = x - x_low; + T hy = 1. - ly, hx = 1. - lx; + + // reference in forward + // T v1 = input[y_low * width + x_low]; + // T v2 = input[y_low * width + x_high]; + // T v3 = input[y_high * width + x_low]; + // T v4 = input[y_high * width + x_high]; + // T val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + + w1 = hy * hx, w2 = hy * lx, w3 = ly * hx, w4 = ly * lx; + + return; +} + +template +inline void add(T* address, const T& val) { + *address += val; +} + +} // namespace + +template +void ROIAlignRotatedForward( + const int nthreads, + const T* input, + const T& spatial_scale, + const int channels, + const int height, + const int width, + const int pooled_height, + const int pooled_width, + const int sampling_ratio, + const T* rois, + T* output) { + int n_rois = nthreads / channels / pooled_width / pooled_height; + // (n, c, ph, pw) is an element in the pooled output + // can be parallelized using omp + // #pragma omp parallel for num_threads(32) + for (int n = 0; n < n_rois; n++) { + int index_n = n * channels * pooled_width * pooled_height; + + const T* current_roi = rois + n * 6; + int roi_batch_ind = current_roi[0]; + + // Do not use rounding; this implementation detail is critical + // ROIAlignRotated supports align == true, i.e., continuous coordinate + // by default, thus the 0.5 offset + T offset = (T)0.5; + T roi_center_w = current_roi[1] * spatial_scale - offset; + T roi_center_h = current_roi[2] * spatial_scale - offset; + T roi_width = current_roi[3] * spatial_scale; + T roi_height = current_roi[4] * spatial_scale; + T theta = current_roi[5] * M_PI / 180.0; + T cos_theta = cos(theta); + T sin_theta = sin(theta); + + AT_ASSERTM( + roi_width >= 0 && roi_height >= 0, + "ROIs in ROIAlignRotated do not have non-negative size!"); + + T bin_size_h = static_cast(roi_height) / static_cast(pooled_height); + T bin_size_w = static_cast(roi_width) / static_cast(pooled_width); + + // We use roi_bin_grid to sample the grid and mimic integral + int roi_bin_grid_h = (sampling_ratio > 0) + ? sampling_ratio + : ceil(roi_height / pooled_height); // e.g., = 2 + int roi_bin_grid_w = + (sampling_ratio > 0) ? sampling_ratio : ceil(roi_width / pooled_width); + + // We do average (integral) pooling inside a bin + const T count = std::max(roi_bin_grid_h * roi_bin_grid_w, 1); // e.g. = 4 + + // we want to precalculate indices and weights shared by all channels, + // this is the key point of optimization + std::vector> pre_calc( + roi_bin_grid_h * roi_bin_grid_w * pooled_width * pooled_height); + + // roi_start_h and roi_start_w are computed wrt the center of RoI (x, y). + // Appropriate translation needs to be applied after. + T roi_start_h = -roi_height / 2.0; + T roi_start_w = -roi_width / 2.0; + + pre_calc_for_bilinear_interpolate( + height, + width, + pooled_height, + pooled_width, + roi_bin_grid_h, + roi_bin_grid_w, + roi_start_h, + roi_start_w, + bin_size_h, + bin_size_w, + roi_bin_grid_h, + roi_bin_grid_w, + roi_center_h, + roi_center_w, + cos_theta, + sin_theta, + pre_calc); + + for (int c = 0; c < channels; c++) { + int index_n_c = index_n + c * pooled_width * pooled_height; + const T* offset_input = + input + (roi_batch_ind * channels + c) * height * width; + int pre_calc_index = 0; + + for (int ph = 0; ph < pooled_height; ph++) { + for (int pw = 0; pw < pooled_width; pw++) { + int index = index_n_c + ph * pooled_width + pw; + + T output_val = 0.; + for (int iy = 0; iy < roi_bin_grid_h; iy++) { + for (int ix = 0; ix < roi_bin_grid_w; ix++) { + PreCalc pc = pre_calc[pre_calc_index]; + output_val += pc.w1 * offset_input[pc.pos1] + + pc.w2 * offset_input[pc.pos2] + + pc.w3 * offset_input[pc.pos3] + pc.w4 * offset_input[pc.pos4]; + + pre_calc_index += 1; + } + } + output_val /= count; + + output[index] = output_val; + } // for pw + } // for ph + } // for c + } // for n +} + +template +void ROIAlignRotatedBackward( + const int nthreads, + // may not be contiguous. should index using n_stride, etc + const T* grad_output, + const T& spatial_scale, + const int channels, + const int height, + const int width, + const int pooled_height, + const int pooled_width, + const int sampling_ratio, + T* grad_input, + const T* rois, + const int n_stride, + const int c_stride, + const int h_stride, + const int w_stride) { + for (int index = 0; index < nthreads; index++) { + // (n, c, ph, pw) is an element in the pooled output + int pw = index % pooled_width; + int ph = (index / pooled_width) % pooled_height; + int c = (index / pooled_width / pooled_height) % channels; + int n = index / pooled_width / pooled_height / channels; + + const T* current_roi = rois + n * 6; + int roi_batch_ind = current_roi[0]; + + // Do not use rounding; this implementation detail is critical + // ROIAlignRotated supports align == true, i.e., continuous coordinate + // by default, thus the 0.5 offset + T offset = (T)0.5; + T roi_center_w = current_roi[1] * spatial_scale - offset; + T roi_center_h = current_roi[2] * spatial_scale - offset; + T roi_width = current_roi[3] * spatial_scale; + T roi_height = current_roi[4] * spatial_scale; + T theta = current_roi[5] * M_PI / 180.0; + T cos_theta = cos(theta); + T sin_theta = sin(theta); + + AT_ASSERTM( + roi_width >= 0 && roi_height >= 0, + "ROIs in ROIAlignRotated do not have non-negative size!"); + + T bin_size_h = static_cast(roi_height) / static_cast(pooled_height); + T bin_size_w = static_cast(roi_width) / static_cast(pooled_width); + + T* offset_grad_input = + grad_input + ((roi_batch_ind * channels + c) * height * width); + + int output_offset = n * n_stride + c * c_stride; + const T* offset_grad_output = grad_output + output_offset; + const T grad_output_this_bin = + offset_grad_output[ph * h_stride + pw * w_stride]; + + // We use roi_bin_grid to sample the grid and mimic integral + int roi_bin_grid_h = (sampling_ratio > 0) + ? sampling_ratio + : ceil(roi_height / pooled_height); // e.g., = 2 + int roi_bin_grid_w = + (sampling_ratio > 0) ? sampling_ratio : ceil(roi_width / pooled_width); + + // roi_start_h and roi_start_w are computed wrt the center of RoI (x, y). + // Appropriate translation needs to be applied after. + T roi_start_h = -roi_height / 2.0; + T roi_start_w = -roi_width / 2.0; + + // We do average (integral) pooling inside a bin + const T count = roi_bin_grid_h * roi_bin_grid_w; // e.g. = 4 + + for (int iy = 0; iy < roi_bin_grid_h; iy++) { + const T yy = roi_start_h + ph * bin_size_h + + static_cast(iy + .5f) * bin_size_h / + static_cast(roi_bin_grid_h); // e.g., 0.5, 1.5 + for (int ix = 0; ix < roi_bin_grid_w; ix++) { + const T xx = roi_start_w + pw * bin_size_w + + static_cast(ix + .5f) * bin_size_w / + static_cast(roi_bin_grid_w); + + // Rotate by theta around the center and translate + T y = yy * cos_theta - xx * sin_theta + roi_center_h; + T x = yy * sin_theta + xx * cos_theta + roi_center_w; + + T w1, w2, w3, w4; + int x_low, x_high, y_low, y_high; + + bilinear_interpolate_gradient( + height, width, y, x, w1, w2, w3, w4, x_low, x_high, y_low, y_high); + + T g1 = grad_output_this_bin * w1 / count; + T g2 = grad_output_this_bin * w2 / count; + T g3 = grad_output_this_bin * w3 / count; + T g4 = grad_output_this_bin * w4 / count; + + if (x_low >= 0 && x_high >= 0 && y_low >= 0 && y_high >= 0) { + // atomic add is not needed for now since it is single threaded + add(offset_grad_input + y_low * width + x_low, static_cast(g1)); + add(offset_grad_input + y_low * width + x_high, static_cast(g2)); + add(offset_grad_input + y_high * width + x_low, static_cast(g3)); + add(offset_grad_input + y_high * width + x_high, static_cast(g4)); + } // if + } // ix + } // iy + } // for +} // ROIAlignRotatedBackward + +at::Tensor ROIAlignRotated_forward_cpu( + const at::Tensor& input, + const at::Tensor& rois, + const float spatial_scale, + const int pooled_height, + const int pooled_width, + const int sampling_ratio) { + AT_ASSERTM(input.device().is_cpu(), "input must be a CPU tensor"); + AT_ASSERTM(rois.device().is_cpu(), "rois must be a CPU tensor"); + + at::TensorArg input_t{input, "input", 1}, rois_t{rois, "rois", 2}; + + at::CheckedFrom c = "ROIAlign_forward_cpu"; + at::checkAllSameType(c, {input_t, rois_t}); + + auto num_rois = rois.size(0); + auto channels = input.size(1); + auto height = input.size(2); + auto width = input.size(3); + + at::Tensor output = at::zeros( + {num_rois, channels, pooled_height, pooled_width}, input.options()); + + auto output_size = num_rois * pooled_height * pooled_width * channels; + + if (output.numel() == 0) { + return output; + } + + auto input_ = input.contiguous(), rois_ = rois.contiguous(); + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + input.scalar_type(), "ROIAlignRotated_forward", [&] { + ROIAlignRotatedForward( + output_size, + input_.data_ptr(), + spatial_scale, + channels, + height, + width, + pooled_height, + pooled_width, + sampling_ratio, + rois_.data_ptr(), + output.data_ptr()); + }); + return output; +} + +at::Tensor ROIAlignRotated_backward_cpu( + const at::Tensor& grad, + const at::Tensor& rois, + const float spatial_scale, + const int pooled_height, + const int pooled_width, + const int batch_size, + const int channels, + const int height, + const int width, + const int sampling_ratio) { + AT_ASSERTM(grad.device().is_cpu(), "grad must be a CPU tensor"); + AT_ASSERTM(rois.device().is_cpu(), "rois must be a CPU tensor"); + + at::TensorArg grad_t{grad, "grad", 1}, rois_t{rois, "rois", 2}; + + at::CheckedFrom c = "ROIAlignRotated_backward_cpu"; + at::checkAllSameType(c, {grad_t, rois_t}); + + at::Tensor grad_input = + at::zeros({batch_size, channels, height, width}, grad.options()); + + // handle possibly empty gradients + if (grad.numel() == 0) { + return grad_input; + } + + // get stride values to ensure indexing into gradients is correct. + int n_stride = grad.stride(0); + int c_stride = grad.stride(1); + int h_stride = grad.stride(2); + int w_stride = grad.stride(3); + + auto rois_ = rois.contiguous(); + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + grad.scalar_type(), "ROIAlignRotated_forward", [&] { + ROIAlignRotatedBackward( + grad.numel(), + grad.data_ptr(), + spatial_scale, + channels, + height, + width, + pooled_height, + pooled_width, + sampling_ratio, + grad_input.data_ptr(), + rois_.data_ptr(), + n_stride, + c_stride, + h_stride, + w_stride); + }); + return grad_input; +} + +} // namespace detectron2 diff --git a/annotator/oneformer/detectron2/layers/csrc/ROIAlignRotated/ROIAlignRotated_cuda.cu b/annotator/oneformer/detectron2/layers/csrc/ROIAlignRotated/ROIAlignRotated_cuda.cu new file mode 100644 index 0000000000000000000000000000000000000000..fca186519143b168a912c880a4cf495a0a5a9322 --- /dev/null +++ b/annotator/oneformer/detectron2/layers/csrc/ROIAlignRotated/ROIAlignRotated_cuda.cu @@ -0,0 +1,443 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#include +#include +#include +#include + +// TODO make it in a common file +#define CUDA_1D_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < n; \ + i += blockDim.x * gridDim.x) + +// Note: this implementation originates from the Caffe2 ROIAlignRotated Op +// and PyTorch ROIAlign (non-rotated) Op implementations. +// The key difference between this implementation and those ones is +// we don't do "legacy offset" in this version, as there aren't many previous +// works, if any, using the "legacy" ROIAlignRotated Op. +// This would make the interface a bit cleaner. + +namespace detectron2 { + +namespace { + +template +__device__ T bilinear_interpolate( + const T* input, + const int height, + const int width, + T y, + T x) { + // deal with cases that inverse elements are out of feature map boundary + if (y < -1.0 || y > height || x < -1.0 || x > width) { + // empty + return 0; + } + + if (y < 0) { + y = 0; + } + + if (x < 0) { + x = 0; + } + + int y_low = (int)y; + int x_low = (int)x; + int y_high; + int x_high; + + if (y_low >= height - 1) { + y_high = y_low = height - 1; + y = (T)y_low; + } else { + y_high = y_low + 1; + } + + if (x_low >= width - 1) { + x_high = x_low = width - 1; + x = (T)x_low; + } else { + x_high = x_low + 1; + } + + T ly = y - y_low; + T lx = x - x_low; + T hy = 1. - ly, hx = 1. - lx; + // do bilinear interpolation + T v1 = input[y_low * width + x_low]; + T v2 = input[y_low * width + x_high]; + T v3 = input[y_high * width + x_low]; + T v4 = input[y_high * width + x_high]; + T w1 = hy * hx, w2 = hy * lx, w3 = ly * hx, w4 = ly * lx; + + T val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + + return val; +} + +template +__device__ void bilinear_interpolate_gradient( + const int height, + const int width, + T y, + T x, + T& w1, + T& w2, + T& w3, + T& w4, + int& x_low, + int& x_high, + int& y_low, + int& y_high) { + // deal with cases that inverse elements are out of feature map boundary + if (y < -1.0 || y > height || x < -1.0 || x > width) { + // empty + w1 = w2 = w3 = w4 = 0.; + x_low = x_high = y_low = y_high = -1; + return; + } + + if (y < 0) { + y = 0; + } + + if (x < 0) { + x = 0; + } + + y_low = (int)y; + x_low = (int)x; + + if (y_low >= height - 1) { + y_high = y_low = height - 1; + y = (T)y_low; + } else { + y_high = y_low + 1; + } + + if (x_low >= width - 1) { + x_high = x_low = width - 1; + x = (T)x_low; + } else { + x_high = x_low + 1; + } + + T ly = y - y_low; + T lx = x - x_low; + T hy = 1. - ly, hx = 1. - lx; + + // reference in forward + // T v1 = input[y_low * width + x_low]; + // T v2 = input[y_low * width + x_high]; + // T v3 = input[y_high * width + x_low]; + // T v4 = input[y_high * width + x_high]; + // T val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + + w1 = hy * hx, w2 = hy * lx, w3 = ly * hx, w4 = ly * lx; + + return; +} + +} // namespace + +template +__global__ void RoIAlignRotatedForward( + const int nthreads, + const T* input, + const T spatial_scale, + const int channels, + const int height, + const int width, + const int pooled_height, + const int pooled_width, + const int sampling_ratio, + const T* rois, + T* top_data) { + CUDA_1D_KERNEL_LOOP(index, nthreads) { + // (n, c, ph, pw) is an element in the pooled output + int pw = index % pooled_width; + int ph = (index / pooled_width) % pooled_height; + int c = (index / pooled_width / pooled_height) % channels; + int n = index / pooled_width / pooled_height / channels; + + const T* current_roi = rois + n * 6; + int roi_batch_ind = current_roi[0]; + + // Do not use rounding; this implementation detail is critical + // ROIAlignRotated supports align == true, i.e., continuous coordinate + // by default, thus the 0.5 offset + T offset = (T)0.5; + T roi_center_w = current_roi[1] * spatial_scale - offset; + T roi_center_h = current_roi[2] * spatial_scale - offset; + T roi_width = current_roi[3] * spatial_scale; + T roi_height = current_roi[4] * spatial_scale; + T theta = current_roi[5] * M_PI / 180.0; + T cos_theta = cos(theta); + T sin_theta = sin(theta); + + T bin_size_h = static_cast(roi_height) / static_cast(pooled_height); + T bin_size_w = static_cast(roi_width) / static_cast(pooled_width); + + const T* offset_input = + input + (roi_batch_ind * channels + c) * height * width; + + // We use roi_bin_grid to sample the grid and mimic integral + int roi_bin_grid_h = (sampling_ratio > 0) + ? sampling_ratio + : ceil(roi_height / pooled_height); // e.g., = 2 + int roi_bin_grid_w = + (sampling_ratio > 0) ? sampling_ratio : ceil(roi_width / pooled_width); + + // roi_start_h and roi_start_w are computed wrt the center of RoI (x, y). + // Appropriate translation needs to be applied after. + T roi_start_h = -roi_height / 2.0; + T roi_start_w = -roi_width / 2.0; + + // We do average (inte gral) pooling inside a bin + const T count = max(roi_bin_grid_h * roi_bin_grid_w, 1); // e.g. = 4 + + T output_val = 0.; + for (int iy = 0; iy < roi_bin_grid_h; iy++) // e.g., iy = 0, 1 + { + const T yy = roi_start_h + ph * bin_size_h + + static_cast(iy + .5f) * bin_size_h / + static_cast(roi_bin_grid_h); // e.g., 0.5, 1.5 + for (int ix = 0; ix < roi_bin_grid_w; ix++) { + const T xx = roi_start_w + pw * bin_size_w + + static_cast(ix + .5f) * bin_size_w / + static_cast(roi_bin_grid_w); + + // Rotate by theta around the center and translate + T y = yy * cos_theta - xx * sin_theta + roi_center_h; + T x = yy * sin_theta + xx * cos_theta + roi_center_w; + + T val = bilinear_interpolate(offset_input, height, width, y, x); + output_val += val; + } + } + output_val /= count; + + top_data[index] = output_val; + } +} + +template +__global__ void RoIAlignRotatedBackwardFeature( + const int nthreads, + const T* top_diff, + const int num_rois, + const T spatial_scale, + const int channels, + const int height, + const int width, + const int pooled_height, + const int pooled_width, + const int sampling_ratio, + T* bottom_diff, + const T* rois) { + CUDA_1D_KERNEL_LOOP(index, nthreads) { + // (n, c, ph, pw) is an element in the pooled output + int pw = index % pooled_width; + int ph = (index / pooled_width) % pooled_height; + int c = (index / pooled_width / pooled_height) % channels; + int n = index / pooled_width / pooled_height / channels; + + const T* current_roi = rois + n * 6; + int roi_batch_ind = current_roi[0]; + + // Do not use rounding; this implementation detail is critical + // ROIAlignRotated supports align == true, i.e., continuous coordinate + // by default, thus the 0.5 offset + T offset = (T)0.5; + T roi_center_w = current_roi[1] * spatial_scale - offset; + T roi_center_h = current_roi[2] * spatial_scale - offset; + T roi_width = current_roi[3] * spatial_scale; + T roi_height = current_roi[4] * spatial_scale; + T theta = current_roi[5] * M_PI / 180.0; + T cos_theta = cos(theta); + T sin_theta = sin(theta); + + T bin_size_h = static_cast(roi_height) / static_cast(pooled_height); + T bin_size_w = static_cast(roi_width) / static_cast(pooled_width); + + T* offset_bottom_diff = + bottom_diff + (roi_batch_ind * channels + c) * height * width; + + int top_offset = (n * channels + c) * pooled_height * pooled_width; + const T* offset_top_diff = top_diff + top_offset; + const T top_diff_this_bin = offset_top_diff[ph * pooled_width + pw]; + + // We use roi_bin_grid to sample the grid and mimic integral + int roi_bin_grid_h = (sampling_ratio > 0) + ? sampling_ratio + : ceil(roi_height / pooled_height); // e.g., = 2 + int roi_bin_grid_w = + (sampling_ratio > 0) ? sampling_ratio : ceil(roi_width / pooled_width); + + // roi_start_h and roi_start_w are computed wrt the center of RoI (x, y). + // Appropriate translation needs to be applied after. + T roi_start_h = -roi_height / 2.0; + T roi_start_w = -roi_width / 2.0; + + // We do average (integral) pooling inside a bin + const T count = roi_bin_grid_h * roi_bin_grid_w; // e.g. = 4 + + for (int iy = 0; iy < roi_bin_grid_h; iy++) // e.g., iy = 0, 1 + { + const T yy = roi_start_h + ph * bin_size_h + + static_cast(iy + .5f) * bin_size_h / + static_cast(roi_bin_grid_h); // e.g., 0.5, 1.5 + for (int ix = 0; ix < roi_bin_grid_w; ix++) { + const T xx = roi_start_w + pw * bin_size_w + + static_cast(ix + .5f) * bin_size_w / + static_cast(roi_bin_grid_w); + + // Rotate by theta around the center and translate + T y = yy * cos_theta - xx * sin_theta + roi_center_h; + T x = yy * sin_theta + xx * cos_theta + roi_center_w; + + T w1, w2, w3, w4; + int x_low, x_high, y_low, y_high; + + bilinear_interpolate_gradient( + height, width, y, x, w1, w2, w3, w4, x_low, x_high, y_low, y_high); + + T g1 = top_diff_this_bin * w1 / count; + T g2 = top_diff_this_bin * w2 / count; + T g3 = top_diff_this_bin * w3 / count; + T g4 = top_diff_this_bin * w4 / count; + + if (x_low >= 0 && x_high >= 0 && y_low >= 0 && y_high >= 0) { + atomicAdd( + offset_bottom_diff + y_low * width + x_low, static_cast(g1)); + atomicAdd( + offset_bottom_diff + y_low * width + x_high, static_cast(g2)); + atomicAdd( + offset_bottom_diff + y_high * width + x_low, static_cast(g3)); + atomicAdd( + offset_bottom_diff + y_high * width + x_high, static_cast(g4)); + } // if + } // ix + } // iy + } // CUDA_1D_KERNEL_LOOP +} // RoIAlignRotatedBackward + +at::Tensor ROIAlignRotated_forward_cuda( + const at::Tensor& input, + const at::Tensor& rois, + const float spatial_scale, + const int pooled_height, + const int pooled_width, + const int sampling_ratio) { + AT_ASSERTM(input.device().is_cuda(), "input must be a CUDA tensor"); + AT_ASSERTM(rois.device().is_cuda(), "rois must be a CUDA tensor"); + at::TensorArg input_t{input, "input", 1}, rois_t{rois, "rois", 2}; + + at::CheckedFrom c = "ROIAlignRotated_forward_cuda"; + at::checkAllSameGPU(c, {input_t, rois_t}); + at::checkAllSameType(c, {input_t, rois_t}); + at::cuda::CUDAGuard device_guard(input.device()); + + auto num_rois = rois.size(0); + auto channels = input.size(1); + auto height = input.size(2); + auto width = input.size(3); + + auto output = at::empty( + {num_rois, channels, pooled_height, pooled_width}, input.options()); + auto output_size = num_rois * pooled_height * pooled_width * channels; + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + dim3 grid(std::min( + at::cuda::ATenCeilDiv( + static_cast(output_size), static_cast(512)), + static_cast(4096))); + dim3 block(512); + + if (output.numel() == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return output; + } + + auto input_ = input.contiguous(), rois_ = rois.contiguous(); + AT_DISPATCH_FLOATING_TYPES( + input.scalar_type(), "ROIAlignRotated_forward", [&] { + RoIAlignRotatedForward<<>>( + output_size, + input_.data_ptr(), + spatial_scale, + channels, + height, + width, + pooled_height, + pooled_width, + sampling_ratio, + rois_.data_ptr(), + output.data_ptr()); + }); + cudaDeviceSynchronize(); + AT_CUDA_CHECK(cudaGetLastError()); + return output; +} + +// TODO remove the dependency on input and use instead its sizes -> save memory +at::Tensor ROIAlignRotated_backward_cuda( + const at::Tensor& grad, + const at::Tensor& rois, + const float spatial_scale, + const int pooled_height, + const int pooled_width, + const int batch_size, + const int channels, + const int height, + const int width, + const int sampling_ratio) { + AT_ASSERTM(grad.device().is_cuda(), "grad must be a CUDA tensor"); + AT_ASSERTM(rois.device().is_cuda(), "rois must be a CUDA tensor"); + + at::TensorArg grad_t{grad, "grad", 1}, rois_t{rois, "rois", 2}; + at::CheckedFrom c = "ROIAlign_backward_cuda"; + at::checkAllSameGPU(c, {grad_t, rois_t}); + at::checkAllSameType(c, {grad_t, rois_t}); + at::cuda::CUDAGuard device_guard(grad.device()); + + auto num_rois = rois.size(0); + auto grad_input = + at::zeros({batch_size, channels, height, width}, grad.options()); + + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + dim3 grid(std::min( + at::cuda::ATenCeilDiv( + static_cast(grad.numel()), static_cast(512)), + static_cast(4096))); + dim3 block(512); + + // handle possibly empty gradients + if (grad.numel() == 0) { + AT_CUDA_CHECK(cudaGetLastError()); + return grad_input; + } + + auto grad_ = grad.contiguous(), rois_ = rois.contiguous(); + AT_DISPATCH_FLOATING_TYPES( + grad.scalar_type(), "ROIAlignRotated_backward", [&] { + RoIAlignRotatedBackwardFeature<<>>( + grad.numel(), + grad_.data_ptr(), + num_rois, + spatial_scale, + channels, + height, + width, + pooled_height, + pooled_width, + sampling_ratio, + grad_input.data_ptr(), + rois_.data_ptr()); + }); + AT_CUDA_CHECK(cudaGetLastError()); + return grad_input; +} + +} // namespace detectron2 diff --git a/annotator/oneformer/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated.h b/annotator/oneformer/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated.h new file mode 100644 index 0000000000000000000000000000000000000000..3bf383b8ed9b358b5313d433a9682c294dfb77e4 --- /dev/null +++ b/annotator/oneformer/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated.h @@ -0,0 +1,35 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#pragma once +#include + +namespace detectron2 { + +at::Tensor box_iou_rotated_cpu( + const at::Tensor& boxes1, + const at::Tensor& boxes2); + +#if defined(WITH_CUDA) || defined(WITH_HIP) +at::Tensor box_iou_rotated_cuda( + const at::Tensor& boxes1, + const at::Tensor& boxes2); +#endif + +// Interface for Python +// inline is needed to prevent multiple function definitions when this header is +// included by different cpps +inline at::Tensor box_iou_rotated( + const at::Tensor& boxes1, + const at::Tensor& boxes2) { + assert(boxes1.device().is_cuda() == boxes2.device().is_cuda()); + if (boxes1.device().is_cuda()) { +#if defined(WITH_CUDA) || defined(WITH_HIP) + return box_iou_rotated_cuda(boxes1.contiguous(), boxes2.contiguous()); +#else + AT_ERROR("Detectron2 is not compiled with GPU support!"); +#endif + } + + return box_iou_rotated_cpu(boxes1.contiguous(), boxes2.contiguous()); +} + +} // namespace detectron2 diff --git a/annotator/oneformer/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_cpu.cpp b/annotator/oneformer/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c843487b5fa4e8077dd27402ec99009266ddda8d --- /dev/null +++ b/annotator/oneformer/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_cpu.cpp @@ -0,0 +1,39 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#include "box_iou_rotated.h" +#include "box_iou_rotated_utils.h" + +namespace detectron2 { + +template +void box_iou_rotated_cpu_kernel( + const at::Tensor& boxes1, + const at::Tensor& boxes2, + at::Tensor& ious) { + auto num_boxes1 = boxes1.size(0); + auto num_boxes2 = boxes2.size(0); + + for (int i = 0; i < num_boxes1; i++) { + for (int j = 0; j < num_boxes2; j++) { + ious[i * num_boxes2 + j] = single_box_iou_rotated( + boxes1[i].data_ptr(), boxes2[j].data_ptr()); + } + } +} + +at::Tensor box_iou_rotated_cpu( + // input must be contiguous: + const at::Tensor& boxes1, + const at::Tensor& boxes2) { + auto num_boxes1 = boxes1.size(0); + auto num_boxes2 = boxes2.size(0); + at::Tensor ious = + at::empty({num_boxes1 * num_boxes2}, boxes1.options().dtype(at::kFloat)); + + box_iou_rotated_cpu_kernel(boxes1, boxes2, ious); + + // reshape from 1d array to 2d array + auto shape = std::vector{num_boxes1, num_boxes2}; + return ious.reshape(shape); +} + +} // namespace detectron2 diff --git a/annotator/oneformer/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_cuda.cu b/annotator/oneformer/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_cuda.cu new file mode 100644 index 0000000000000000000000000000000000000000..952710e53041187907fbd113f8d0d0fa24134a86 --- /dev/null +++ b/annotator/oneformer/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_cuda.cu @@ -0,0 +1,130 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#include +#include +#include +#include +#include "box_iou_rotated_utils.h" + +namespace detectron2 { + +// 2D block with 32 * 16 = 512 threads per block +const int BLOCK_DIM_X = 32; +const int BLOCK_DIM_Y = 16; + +template +__global__ void box_iou_rotated_cuda_kernel( + const int n_boxes1, + const int n_boxes2, + const T* dev_boxes1, + const T* dev_boxes2, + T* dev_ious) { + const int row_start = blockIdx.x * blockDim.x; + const int col_start = blockIdx.y * blockDim.y; + + const int row_size = min(n_boxes1 - row_start, blockDim.x); + const int col_size = min(n_boxes2 - col_start, blockDim.y); + + __shared__ float block_boxes1[BLOCK_DIM_X * 5]; + __shared__ float block_boxes2[BLOCK_DIM_Y * 5]; + + // It's safe to copy using threadIdx.x since BLOCK_DIM_X >= BLOCK_DIM_Y + if (threadIdx.x < row_size && threadIdx.y == 0) { + block_boxes1[threadIdx.x * 5 + 0] = + dev_boxes1[(row_start + threadIdx.x) * 5 + 0]; + block_boxes1[threadIdx.x * 5 + 1] = + dev_boxes1[(row_start + threadIdx.x) * 5 + 1]; + block_boxes1[threadIdx.x * 5 + 2] = + dev_boxes1[(row_start + threadIdx.x) * 5 + 2]; + block_boxes1[threadIdx.x * 5 + 3] = + dev_boxes1[(row_start + threadIdx.x) * 5 + 3]; + block_boxes1[threadIdx.x * 5 + 4] = + dev_boxes1[(row_start + threadIdx.x) * 5 + 4]; + } + + if (threadIdx.x < col_size && threadIdx.y == 0) { + block_boxes2[threadIdx.x * 5 + 0] = + dev_boxes2[(col_start + threadIdx.x) * 5 + 0]; + block_boxes2[threadIdx.x * 5 + 1] = + dev_boxes2[(col_start + threadIdx.x) * 5 + 1]; + block_boxes2[threadIdx.x * 5 + 2] = + dev_boxes2[(col_start + threadIdx.x) * 5 + 2]; + block_boxes2[threadIdx.x * 5 + 3] = + dev_boxes2[(col_start + threadIdx.x) * 5 + 3]; + block_boxes2[threadIdx.x * 5 + 4] = + dev_boxes2[(col_start + threadIdx.x) * 5 + 4]; + } + __syncthreads(); + + if (threadIdx.x < row_size && threadIdx.y < col_size) { + int offset = (row_start + threadIdx.x) * n_boxes2 + col_start + threadIdx.y; + dev_ious[offset] = single_box_iou_rotated( + block_boxes1 + threadIdx.x * 5, block_boxes2 + threadIdx.y * 5); + } +} + +at::Tensor box_iou_rotated_cuda( + // input must be contiguous + const at::Tensor& boxes1, + const at::Tensor& boxes2) { + using scalar_t = float; + AT_ASSERTM( + boxes1.scalar_type() == at::kFloat, "boxes1 must be a float tensor"); + AT_ASSERTM( + boxes2.scalar_type() == at::kFloat, "boxes2 must be a float tensor"); + AT_ASSERTM(boxes1.is_cuda(), "boxes1 must be a CUDA tensor"); + AT_ASSERTM(boxes2.is_cuda(), "boxes2 must be a CUDA tensor"); + at::cuda::CUDAGuard device_guard(boxes1.device()); + + auto num_boxes1 = boxes1.size(0); + auto num_boxes2 = boxes2.size(0); + + at::Tensor ious = + at::empty({num_boxes1 * num_boxes2}, boxes1.options().dtype(at::kFloat)); + + bool transpose = false; + if (num_boxes1 > 0 && num_boxes2 > 0) { + scalar_t *data1 = boxes1.data_ptr(), + *data2 = boxes2.data_ptr(); + + if (num_boxes2 > 65535 * BLOCK_DIM_Y) { + AT_ASSERTM( + num_boxes1 <= 65535 * BLOCK_DIM_Y, + "Too many boxes for box_iou_rotated_cuda!"); + // x dim is allowed to be large, but y dim cannot, + // so we transpose the two to avoid "invalid configuration argument" + // error. We assume one of them is small. Otherwise the result is hard to + // fit in memory anyway. + std::swap(num_boxes1, num_boxes2); + std::swap(data1, data2); + transpose = true; + } + + const int blocks_x = + at::cuda::ATenCeilDiv(static_cast(num_boxes1), BLOCK_DIM_X); + const int blocks_y = + at::cuda::ATenCeilDiv(static_cast(num_boxes2), BLOCK_DIM_Y); + + dim3 blocks(blocks_x, blocks_y); + dim3 threads(BLOCK_DIM_X, BLOCK_DIM_Y); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + box_iou_rotated_cuda_kernel<<>>( + num_boxes1, + num_boxes2, + data1, + data2, + (scalar_t*)ious.data_ptr()); + + AT_CUDA_CHECK(cudaGetLastError()); + } + + // reshape from 1d array to 2d array + auto shape = std::vector{num_boxes1, num_boxes2}; + if (transpose) { + return ious.view(shape).t(); + } else { + return ious.view(shape); + } +} + +} // namespace detectron2 diff --git a/annotator/oneformer/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_utils.h b/annotator/oneformer/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..b54a5dde2ca11a74d29c4d8adb7fe1634f5baf9c --- /dev/null +++ b/annotator/oneformer/detectron2/layers/csrc/box_iou_rotated/box_iou_rotated_utils.h @@ -0,0 +1,370 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#pragma once + +#include +#include + +#if defined(__CUDACC__) || __HCC__ == 1 || __HIP__ == 1 +// Designates functions callable from the host (CPU) and the device (GPU) +#define HOST_DEVICE __host__ __device__ +#define HOST_DEVICE_INLINE HOST_DEVICE __forceinline__ +#else +#include +#define HOST_DEVICE +#define HOST_DEVICE_INLINE HOST_DEVICE inline +#endif + +namespace detectron2 { + +namespace { + +template +struct RotatedBox { + T x_ctr, y_ctr, w, h, a; +}; + +template +struct Point { + T x, y; + HOST_DEVICE_INLINE Point(const T& px = 0, const T& py = 0) : x(px), y(py) {} + HOST_DEVICE_INLINE Point operator+(const Point& p) const { + return Point(x + p.x, y + p.y); + } + HOST_DEVICE_INLINE Point& operator+=(const Point& p) { + x += p.x; + y += p.y; + return *this; + } + HOST_DEVICE_INLINE Point operator-(const Point& p) const { + return Point(x - p.x, y - p.y); + } + HOST_DEVICE_INLINE Point operator*(const T coeff) const { + return Point(x * coeff, y * coeff); + } +}; + +template +HOST_DEVICE_INLINE T dot_2d(const Point& A, const Point& B) { + return A.x * B.x + A.y * B.y; +} + +// R: result type. can be different from input type +template +HOST_DEVICE_INLINE R cross_2d(const Point& A, const Point& B) { + return static_cast(A.x) * static_cast(B.y) - + static_cast(B.x) * static_cast(A.y); +} + +template +HOST_DEVICE_INLINE void get_rotated_vertices( + const RotatedBox& box, + Point (&pts)[4]) { + // M_PI / 180. == 0.01745329251 + double theta = box.a * 0.01745329251; + T cosTheta2 = (T)cos(theta) * 0.5f; + T sinTheta2 = (T)sin(theta) * 0.5f; + + // y: top --> down; x: left --> right + pts[0].x = box.x_ctr + sinTheta2 * box.h + cosTheta2 * box.w; + pts[0].y = box.y_ctr + cosTheta2 * box.h - sinTheta2 * box.w; + pts[1].x = box.x_ctr - sinTheta2 * box.h + cosTheta2 * box.w; + pts[1].y = box.y_ctr - cosTheta2 * box.h - sinTheta2 * box.w; + pts[2].x = 2 * box.x_ctr - pts[0].x; + pts[2].y = 2 * box.y_ctr - pts[0].y; + pts[3].x = 2 * box.x_ctr - pts[1].x; + pts[3].y = 2 * box.y_ctr - pts[1].y; +} + +template +HOST_DEVICE_INLINE int get_intersection_points( + const Point (&pts1)[4], + const Point (&pts2)[4], + Point (&intersections)[24]) { + // Line vector + // A line from p1 to p2 is: p1 + (p2-p1)*t, t=[0,1] + Point vec1[4], vec2[4]; + for (int i = 0; i < 4; i++) { + vec1[i] = pts1[(i + 1) % 4] - pts1[i]; + vec2[i] = pts2[(i + 1) % 4] - pts2[i]; + } + + // When computing the intersection area, it doesn't hurt if we have + // more (duplicated/approximate) intersections/vertices than needed, + // while it can cause drastic difference if we miss an intersection/vertex. + // Therefore, we add an epsilon to relax the comparisons between + // the float point numbers that decide the intersection points. + double EPS = 1e-5; + + // Line test - test all line combos for intersection + int num = 0; // number of intersections + for (int i = 0; i < 4; i++) { + for (int j = 0; j < 4; j++) { + // Solve for 2x2 Ax=b + T det = cross_2d(vec2[j], vec1[i]); + + // This takes care of parallel lines + if (fabs(det) <= 1e-14) { + continue; + } + + auto vec12 = pts2[j] - pts1[i]; + + T t1 = cross_2d(vec2[j], vec12) / det; + T t2 = cross_2d(vec1[i], vec12) / det; + + if (t1 > -EPS && t1 < 1.0f + EPS && t2 > -EPS && t2 < 1.0f + EPS) { + intersections[num++] = pts1[i] + vec1[i] * t1; + } + } + } + + // Check for vertices of rect1 inside rect2 + { + const auto& AB = vec2[0]; + const auto& DA = vec2[3]; + auto ABdotAB = dot_2d(AB, AB); + auto ADdotAD = dot_2d(DA, DA); + for (int i = 0; i < 4; i++) { + // assume ABCD is the rectangle, and P is the point to be judged + // P is inside ABCD iff. P's projection on AB lies within AB + // and P's projection on AD lies within AD + + auto AP = pts1[i] - pts2[0]; + + auto APdotAB = dot_2d(AP, AB); + auto APdotAD = -dot_2d(AP, DA); + + if ((APdotAB > -EPS) && (APdotAD > -EPS) && (APdotAB < ABdotAB + EPS) && + (APdotAD < ADdotAD + EPS)) { + intersections[num++] = pts1[i]; + } + } + } + + // Reverse the check - check for vertices of rect2 inside rect1 + { + const auto& AB = vec1[0]; + const auto& DA = vec1[3]; + auto ABdotAB = dot_2d(AB, AB); + auto ADdotAD = dot_2d(DA, DA); + for (int i = 0; i < 4; i++) { + auto AP = pts2[i] - pts1[0]; + + auto APdotAB = dot_2d(AP, AB); + auto APdotAD = -dot_2d(AP, DA); + + if ((APdotAB > -EPS) && (APdotAD > -EPS) && (APdotAB < ABdotAB + EPS) && + (APdotAD < ADdotAD + EPS)) { + intersections[num++] = pts2[i]; + } + } + } + + return num; +} + +template +HOST_DEVICE_INLINE int convex_hull_graham( + const Point (&p)[24], + const int& num_in, + Point (&q)[24], + bool shift_to_zero = false) { + assert(num_in >= 2); + + // Step 1: + // Find point with minimum y + // if more than 1 points have the same minimum y, + // pick the one with the minimum x. + int t = 0; + for (int i = 1; i < num_in; i++) { + if (p[i].y < p[t].y || (p[i].y == p[t].y && p[i].x < p[t].x)) { + t = i; + } + } + auto& start = p[t]; // starting point + + // Step 2: + // Subtract starting point from every points (for sorting in the next step) + for (int i = 0; i < num_in; i++) { + q[i] = p[i] - start; + } + + // Swap the starting point to position 0 + auto tmp = q[0]; + q[0] = q[t]; + q[t] = tmp; + + // Step 3: + // Sort point 1 ~ num_in according to their relative cross-product values + // (essentially sorting according to angles) + // If the angles are the same, sort according to their distance to origin + T dist[24]; +#if defined(__CUDACC__) || __HCC__ == 1 || __HIP__ == 1 + // compute distance to origin before sort, and sort them together with the + // points + for (int i = 0; i < num_in; i++) { + dist[i] = dot_2d(q[i], q[i]); + } + + // CUDA version + // In the future, we can potentially use thrust + // for sorting here to improve speed (though not guaranteed) + for (int i = 1; i < num_in - 1; i++) { + for (int j = i + 1; j < num_in; j++) { + T crossProduct = cross_2d(q[i], q[j]); + if ((crossProduct < -1e-6) || + (fabs(crossProduct) < 1e-6 && dist[i] > dist[j])) { + auto q_tmp = q[i]; + q[i] = q[j]; + q[j] = q_tmp; + auto dist_tmp = dist[i]; + dist[i] = dist[j]; + dist[j] = dist_tmp; + } + } + } +#else + // CPU version + std::sort( + q + 1, q + num_in, [](const Point& A, const Point& B) -> bool { + T temp = cross_2d(A, B); + if (fabs(temp) < 1e-6) { + return dot_2d(A, A) < dot_2d(B, B); + } else { + return temp > 0; + } + }); + // compute distance to origin after sort, since the points are now different. + for (int i = 0; i < num_in; i++) { + dist[i] = dot_2d(q[i], q[i]); + } +#endif + + // Step 4: + // Make sure there are at least 2 points (that don't overlap with each other) + // in the stack + int k; // index of the non-overlapped second point + for (k = 1; k < num_in; k++) { + if (dist[k] > 1e-8) { + break; + } + } + if (k == num_in) { + // We reach the end, which means the convex hull is just one point + q[0] = p[t]; + return 1; + } + q[1] = q[k]; + int m = 2; // 2 points in the stack + // Step 5: + // Finally we can start the scanning process. + // When a non-convex relationship between the 3 points is found + // (either concave shape or duplicated points), + // we pop the previous point from the stack + // until the 3-point relationship is convex again, or + // until the stack only contains two points + for (int i = k + 1; i < num_in; i++) { + while (m > 1) { + auto q1 = q[i] - q[m - 2], q2 = q[m - 1] - q[m - 2]; + // cross_2d() uses FMA and therefore computes round(round(q1.x*q2.y) - + // q2.x*q1.y) So it may not return 0 even when q1==q2. Therefore we + // compare round(q1.x*q2.y) and round(q2.x*q1.y) directly. (round means + // round to nearest floating point). + if (q1.x * q2.y >= q2.x * q1.y) + m--; + else + break; + } + // Using double also helps, but float can solve the issue for now. + // while (m > 1 && cross_2d(q[i] - q[m - 2], q[m - 1] - q[m - 2]) + // >= 0) { + // m--; + // } + q[m++] = q[i]; + } + + // Step 6 (Optional): + // In general sense we need the original coordinates, so we + // need to shift the points back (reverting Step 2) + // But if we're only interested in getting the area/perimeter of the shape + // We can simply return. + if (!shift_to_zero) { + for (int i = 0; i < m; i++) { + q[i] += start; + } + } + + return m; +} + +template +HOST_DEVICE_INLINE T polygon_area(const Point (&q)[24], const int& m) { + if (m <= 2) { + return 0; + } + + T area = 0; + for (int i = 1; i < m - 1; i++) { + area += fabs(cross_2d(q[i] - q[0], q[i + 1] - q[0])); + } + + return area / 2.0; +} + +template +HOST_DEVICE_INLINE T rotated_boxes_intersection( + const RotatedBox& box1, + const RotatedBox& box2) { + // There are up to 4 x 4 + 4 + 4 = 24 intersections (including dups) returned + // from rotated_rect_intersection_pts + Point intersectPts[24], orderedPts[24]; + + Point pts1[4]; + Point pts2[4]; + get_rotated_vertices(box1, pts1); + get_rotated_vertices(box2, pts2); + + int num = get_intersection_points(pts1, pts2, intersectPts); + + if (num <= 2) { + return 0.0; + } + + // Convex Hull to order the intersection points in clockwise order and find + // the contour area. + int num_convex = convex_hull_graham(intersectPts, num, orderedPts, true); + return polygon_area(orderedPts, num_convex); +} + +} // namespace + +template +HOST_DEVICE_INLINE T +single_box_iou_rotated(T const* const box1_raw, T const* const box2_raw) { + // shift center to the middle point to achieve higher precision in result + RotatedBox box1, box2; + auto center_shift_x = (box1_raw[0] + box2_raw[0]) / 2.0; + auto center_shift_y = (box1_raw[1] + box2_raw[1]) / 2.0; + box1.x_ctr = box1_raw[0] - center_shift_x; + box1.y_ctr = box1_raw[1] - center_shift_y; + box1.w = box1_raw[2]; + box1.h = box1_raw[3]; + box1.a = box1_raw[4]; + box2.x_ctr = box2_raw[0] - center_shift_x; + box2.y_ctr = box2_raw[1] - center_shift_y; + box2.w = box2_raw[2]; + box2.h = box2_raw[3]; + box2.a = box2_raw[4]; + + T area1 = box1.w * box1.h; + T area2 = box2.w * box2.h; + if (area1 < 1e-14 || area2 < 1e-14) { + return 0.f; + } + + T intersection = rotated_boxes_intersection(box1, box2); + T iou = intersection / (area1 + area2 - intersection); + return iou; +} + +} // namespace detectron2 diff --git a/annotator/oneformer/detectron2/layers/csrc/cocoeval/cocoeval.cpp b/annotator/oneformer/detectron2/layers/csrc/cocoeval/cocoeval.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0a5b7b907c06720fefc77b0dfd921b8ec3ecf2be --- /dev/null +++ b/annotator/oneformer/detectron2/layers/csrc/cocoeval/cocoeval.cpp @@ -0,0 +1,507 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#include "cocoeval.h" +#include +#include +#include +#include + +using namespace pybind11::literals; + +namespace detectron2 { + +namespace COCOeval { + +// Sort detections from highest score to lowest, such that +// detection_instances[detection_sorted_indices[t]] >= +// detection_instances[detection_sorted_indices[t+1]]. Use stable_sort to match +// original COCO API +void SortInstancesByDetectionScore( + const std::vector& detection_instances, + std::vector* detection_sorted_indices) { + detection_sorted_indices->resize(detection_instances.size()); + std::iota( + detection_sorted_indices->begin(), detection_sorted_indices->end(), 0); + std::stable_sort( + detection_sorted_indices->begin(), + detection_sorted_indices->end(), + [&detection_instances](size_t j1, size_t j2) { + return detection_instances[j1].score > detection_instances[j2].score; + }); +} + +// Partition the ground truth objects based on whether or not to ignore them +// based on area +void SortInstancesByIgnore( + const std::array& area_range, + const std::vector& ground_truth_instances, + std::vector* ground_truth_sorted_indices, + std::vector* ignores) { + ignores->clear(); + ignores->reserve(ground_truth_instances.size()); + for (auto o : ground_truth_instances) { + ignores->push_back( + o.ignore || o.area < area_range[0] || o.area > area_range[1]); + } + + ground_truth_sorted_indices->resize(ground_truth_instances.size()); + std::iota( + ground_truth_sorted_indices->begin(), + ground_truth_sorted_indices->end(), + 0); + std::stable_sort( + ground_truth_sorted_indices->begin(), + ground_truth_sorted_indices->end(), + [&ignores](size_t j1, size_t j2) { + return (int)(*ignores)[j1] < (int)(*ignores)[j2]; + }); +} + +// For each IOU threshold, greedily match each detected instance to a ground +// truth instance (if possible) and store the results +void MatchDetectionsToGroundTruth( + const std::vector& detection_instances, + const std::vector& detection_sorted_indices, + const std::vector& ground_truth_instances, + const std::vector& ground_truth_sorted_indices, + const std::vector& ignores, + const std::vector>& ious, + const std::vector& iou_thresholds, + const std::array& area_range, + ImageEvaluation* results) { + // Initialize memory to store return data matches and ignore + const int num_iou_thresholds = iou_thresholds.size(); + const int num_ground_truth = ground_truth_sorted_indices.size(); + const int num_detections = detection_sorted_indices.size(); + std::vector ground_truth_matches( + num_iou_thresholds * num_ground_truth, 0); + std::vector& detection_matches = results->detection_matches; + std::vector& detection_ignores = results->detection_ignores; + std::vector& ground_truth_ignores = results->ground_truth_ignores; + detection_matches.resize(num_iou_thresholds * num_detections, 0); + detection_ignores.resize(num_iou_thresholds * num_detections, false); + ground_truth_ignores.resize(num_ground_truth); + for (auto g = 0; g < num_ground_truth; ++g) { + ground_truth_ignores[g] = ignores[ground_truth_sorted_indices[g]]; + } + + for (auto t = 0; t < num_iou_thresholds; ++t) { + for (auto d = 0; d < num_detections; ++d) { + // information about best match so far (match=-1 -> unmatched) + double best_iou = std::min(iou_thresholds[t], 1 - 1e-10); + int match = -1; + for (auto g = 0; g < num_ground_truth; ++g) { + // if this ground truth instance is already matched and not a + // crowd, it cannot be matched to another detection + if (ground_truth_matches[t * num_ground_truth + g] > 0 && + !ground_truth_instances[ground_truth_sorted_indices[g]].is_crowd) { + continue; + } + + // if detected instance matched to a regular ground truth + // instance, we can break on the first ground truth instance + // tagged as ignore (because they are sorted by the ignore tag) + if (match >= 0 && !ground_truth_ignores[match] && + ground_truth_ignores[g]) { + break; + } + + // if IOU overlap is the best so far, store the match appropriately + if (ious[d][ground_truth_sorted_indices[g]] >= best_iou) { + best_iou = ious[d][ground_truth_sorted_indices[g]]; + match = g; + } + } + // if match was made, store id of match for both detection and + // ground truth + if (match >= 0) { + detection_ignores[t * num_detections + d] = ground_truth_ignores[match]; + detection_matches[t * num_detections + d] = + ground_truth_instances[ground_truth_sorted_indices[match]].id; + ground_truth_matches[t * num_ground_truth + match] = + detection_instances[detection_sorted_indices[d]].id; + } + + // set unmatched detections outside of area range to ignore + const InstanceAnnotation& detection = + detection_instances[detection_sorted_indices[d]]; + detection_ignores[t * num_detections + d] = + detection_ignores[t * num_detections + d] || + (detection_matches[t * num_detections + d] == 0 && + (detection.area < area_range[0] || detection.area > area_range[1])); + } + } + + // store detection score results + results->detection_scores.resize(detection_sorted_indices.size()); + for (size_t d = 0; d < detection_sorted_indices.size(); ++d) { + results->detection_scores[d] = + detection_instances[detection_sorted_indices[d]].score; + } +} + +std::vector EvaluateImages( + const std::vector>& area_ranges, + int max_detections, + const std::vector& iou_thresholds, + const ImageCategoryInstances>& image_category_ious, + const ImageCategoryInstances& + image_category_ground_truth_instances, + const ImageCategoryInstances& + image_category_detection_instances) { + const int num_area_ranges = area_ranges.size(); + const int num_images = image_category_ground_truth_instances.size(); + const int num_categories = + image_category_ious.size() > 0 ? image_category_ious[0].size() : 0; + std::vector detection_sorted_indices; + std::vector ground_truth_sorted_indices; + std::vector ignores; + std::vector results_all( + num_images * num_area_ranges * num_categories); + + // Store results for each image, category, and area range combination. Results + // for each IOU threshold are packed into the same ImageEvaluation object + for (auto i = 0; i < num_images; ++i) { + for (auto c = 0; c < num_categories; ++c) { + const std::vector& ground_truth_instances = + image_category_ground_truth_instances[i][c]; + const std::vector& detection_instances = + image_category_detection_instances[i][c]; + + SortInstancesByDetectionScore( + detection_instances, &detection_sorted_indices); + if ((int)detection_sorted_indices.size() > max_detections) { + detection_sorted_indices.resize(max_detections); + } + + for (size_t a = 0; a < area_ranges.size(); ++a) { + SortInstancesByIgnore( + area_ranges[a], + ground_truth_instances, + &ground_truth_sorted_indices, + &ignores); + + MatchDetectionsToGroundTruth( + detection_instances, + detection_sorted_indices, + ground_truth_instances, + ground_truth_sorted_indices, + ignores, + image_category_ious[i][c], + iou_thresholds, + area_ranges[a], + &results_all + [c * num_area_ranges * num_images + a * num_images + i]); + } + } + } + + return results_all; +} + +// Convert a python list to a vector +template +std::vector list_to_vec(const py::list& l) { + std::vector v(py::len(l)); + for (int i = 0; i < (int)py::len(l); ++i) { + v[i] = l[i].cast(); + } + return v; +} + +// Helper function to Accumulate() +// Considers the evaluation results applicable to a particular category, area +// range, and max_detections parameter setting, which begin at +// evaluations[evaluation_index]. Extracts a sorted list of length n of all +// applicable detection instances concatenated across all images in the dataset, +// which are represented by the outputs evaluation_indices, detection_scores, +// image_detection_indices, and detection_sorted_indices--all of which are +// length n. evaluation_indices[i] stores the applicable index into +// evaluations[] for instance i, which has detection score detection_score[i], +// and is the image_detection_indices[i]'th of the list of detections +// for the image containing i. detection_sorted_indices[] defines a sorted +// permutation of the 3 other outputs +int BuildSortedDetectionList( + const std::vector& evaluations, + const int64_t evaluation_index, + const int64_t num_images, + const int max_detections, + std::vector* evaluation_indices, + std::vector* detection_scores, + std::vector* detection_sorted_indices, + std::vector* image_detection_indices) { + assert(evaluations.size() >= evaluation_index + num_images); + + // Extract a list of object instances of the applicable category, area + // range, and max detections requirements such that they can be sorted + image_detection_indices->clear(); + evaluation_indices->clear(); + detection_scores->clear(); + image_detection_indices->reserve(num_images * max_detections); + evaluation_indices->reserve(num_images * max_detections); + detection_scores->reserve(num_images * max_detections); + int num_valid_ground_truth = 0; + for (auto i = 0; i < num_images; ++i) { + const ImageEvaluation& evaluation = evaluations[evaluation_index + i]; + + for (int d = 0; + d < (int)evaluation.detection_scores.size() && d < max_detections; + ++d) { // detected instances + evaluation_indices->push_back(evaluation_index + i); + image_detection_indices->push_back(d); + detection_scores->push_back(evaluation.detection_scores[d]); + } + for (auto ground_truth_ignore : evaluation.ground_truth_ignores) { + if (!ground_truth_ignore) { + ++num_valid_ground_truth; + } + } + } + + // Sort detections by decreasing score, using stable sort to match + // python implementation + detection_sorted_indices->resize(detection_scores->size()); + std::iota( + detection_sorted_indices->begin(), detection_sorted_indices->end(), 0); + std::stable_sort( + detection_sorted_indices->begin(), + detection_sorted_indices->end(), + [&detection_scores](size_t j1, size_t j2) { + return (*detection_scores)[j1] > (*detection_scores)[j2]; + }); + + return num_valid_ground_truth; +} + +// Helper function to Accumulate() +// Compute a precision recall curve given a sorted list of detected instances +// encoded in evaluations, evaluation_indices, detection_scores, +// detection_sorted_indices, image_detection_indices (see +// BuildSortedDetectionList()). Using vectors precisions and recalls +// and temporary storage, output the results into precisions_out, recalls_out, +// and scores_out, which are large buffers containing many precion/recall curves +// for all possible parameter settings, with precisions_out_index and +// recalls_out_index defining the applicable indices to store results. +void ComputePrecisionRecallCurve( + const int64_t precisions_out_index, + const int64_t precisions_out_stride, + const int64_t recalls_out_index, + const std::vector& recall_thresholds, + const int iou_threshold_index, + const int num_iou_thresholds, + const int num_valid_ground_truth, + const std::vector& evaluations, + const std::vector& evaluation_indices, + const std::vector& detection_scores, + const std::vector& detection_sorted_indices, + const std::vector& image_detection_indices, + std::vector* precisions, + std::vector* recalls, + std::vector* precisions_out, + std::vector* scores_out, + std::vector* recalls_out) { + assert(recalls_out->size() > recalls_out_index); + + // Compute precision/recall for each instance in the sorted list of detections + int64_t true_positives_sum = 0, false_positives_sum = 0; + precisions->clear(); + recalls->clear(); + precisions->reserve(detection_sorted_indices.size()); + recalls->reserve(detection_sorted_indices.size()); + assert(!evaluations.empty() || detection_sorted_indices.empty()); + for (auto detection_sorted_index : detection_sorted_indices) { + const ImageEvaluation& evaluation = + evaluations[evaluation_indices[detection_sorted_index]]; + const auto num_detections = + evaluation.detection_matches.size() / num_iou_thresholds; + const auto detection_index = iou_threshold_index * num_detections + + image_detection_indices[detection_sorted_index]; + assert(evaluation.detection_matches.size() > detection_index); + assert(evaluation.detection_ignores.size() > detection_index); + const int64_t detection_match = + evaluation.detection_matches[detection_index]; + const bool detection_ignores = + evaluation.detection_ignores[detection_index]; + const auto true_positive = detection_match > 0 && !detection_ignores; + const auto false_positive = detection_match == 0 && !detection_ignores; + if (true_positive) { + ++true_positives_sum; + } + if (false_positive) { + ++false_positives_sum; + } + + const double recall = + static_cast(true_positives_sum) / num_valid_ground_truth; + recalls->push_back(recall); + const int64_t num_valid_detections = + true_positives_sum + false_positives_sum; + const double precision = num_valid_detections > 0 + ? static_cast(true_positives_sum) / num_valid_detections + : 0.0; + precisions->push_back(precision); + } + + (*recalls_out)[recalls_out_index] = !recalls->empty() ? recalls->back() : 0; + + for (int64_t i = static_cast(precisions->size()) - 1; i > 0; --i) { + if ((*precisions)[i] > (*precisions)[i - 1]) { + (*precisions)[i - 1] = (*precisions)[i]; + } + } + + // Sample the per instance precision/recall list at each recall threshold + for (size_t r = 0; r < recall_thresholds.size(); ++r) { + // first index in recalls >= recall_thresholds[r] + std::vector::iterator low = std::lower_bound( + recalls->begin(), recalls->end(), recall_thresholds[r]); + size_t precisions_index = low - recalls->begin(); + + const auto results_ind = precisions_out_index + r * precisions_out_stride; + assert(results_ind < precisions_out->size()); + assert(results_ind < scores_out->size()); + if (precisions_index < precisions->size()) { + (*precisions_out)[results_ind] = (*precisions)[precisions_index]; + (*scores_out)[results_ind] = + detection_scores[detection_sorted_indices[precisions_index]]; + } else { + (*precisions_out)[results_ind] = 0; + (*scores_out)[results_ind] = 0; + } + } +} +py::dict Accumulate( + const py::object& params, + const std::vector& evaluations) { + const std::vector recall_thresholds = + list_to_vec(params.attr("recThrs")); + const std::vector max_detections = + list_to_vec(params.attr("maxDets")); + const int num_iou_thresholds = py::len(params.attr("iouThrs")); + const int num_recall_thresholds = py::len(params.attr("recThrs")); + const int num_categories = params.attr("useCats").cast() == 1 + ? py::len(params.attr("catIds")) + : 1; + const int num_area_ranges = py::len(params.attr("areaRng")); + const int num_max_detections = py::len(params.attr("maxDets")); + const int num_images = py::len(params.attr("imgIds")); + + std::vector precisions_out( + num_iou_thresholds * num_recall_thresholds * num_categories * + num_area_ranges * num_max_detections, + -1); + std::vector recalls_out( + num_iou_thresholds * num_categories * num_area_ranges * + num_max_detections, + -1); + std::vector scores_out( + num_iou_thresholds * num_recall_thresholds * num_categories * + num_area_ranges * num_max_detections, + -1); + + // Consider the list of all detected instances in the entire dataset in one + // large list. evaluation_indices, detection_scores, + // image_detection_indices, and detection_sorted_indices all have the same + // length as this list, such that each entry corresponds to one detected + // instance + std::vector evaluation_indices; // indices into evaluations[] + std::vector detection_scores; // detection scores of each instance + std::vector detection_sorted_indices; // sorted indices of all + // instances in the dataset + std::vector + image_detection_indices; // indices into the list of detected instances in + // the same image as each instance + std::vector precisions, recalls; + + for (auto c = 0; c < num_categories; ++c) { + for (auto a = 0; a < num_area_ranges; ++a) { + for (auto m = 0; m < num_max_detections; ++m) { + // The COCO PythonAPI assumes evaluations[] (the return value of + // COCOeval::EvaluateImages() is one long list storing results for each + // combination of category, area range, and image id, with categories in + // the outermost loop and images in the innermost loop. + const int64_t evaluations_index = + c * num_area_ranges * num_images + a * num_images; + int num_valid_ground_truth = BuildSortedDetectionList( + evaluations, + evaluations_index, + num_images, + max_detections[m], + &evaluation_indices, + &detection_scores, + &detection_sorted_indices, + &image_detection_indices); + + if (num_valid_ground_truth == 0) { + continue; + } + + for (auto t = 0; t < num_iou_thresholds; ++t) { + // recalls_out is a flattened vectors representing a + // num_iou_thresholds X num_categories X num_area_ranges X + // num_max_detections matrix + const int64_t recalls_out_index = + t * num_categories * num_area_ranges * num_max_detections + + c * num_area_ranges * num_max_detections + + a * num_max_detections + m; + + // precisions_out and scores_out are flattened vectors + // representing a num_iou_thresholds X num_recall_thresholds X + // num_categories X num_area_ranges X num_max_detections matrix + const int64_t precisions_out_stride = + num_categories * num_area_ranges * num_max_detections; + const int64_t precisions_out_index = t * num_recall_thresholds * + num_categories * num_area_ranges * num_max_detections + + c * num_area_ranges * num_max_detections + + a * num_max_detections + m; + + ComputePrecisionRecallCurve( + precisions_out_index, + precisions_out_stride, + recalls_out_index, + recall_thresholds, + t, + num_iou_thresholds, + num_valid_ground_truth, + evaluations, + evaluation_indices, + detection_scores, + detection_sorted_indices, + image_detection_indices, + &precisions, + &recalls, + &precisions_out, + &scores_out, + &recalls_out); + } + } + } + } + + time_t rawtime; + struct tm local_time; + std::array buffer; + time(&rawtime); +#ifdef _WIN32 + localtime_s(&local_time, &rawtime); +#else + localtime_r(&rawtime, &local_time); +#endif + strftime( + buffer.data(), 200, "%Y-%m-%d %H:%num_max_detections:%S", &local_time); + return py::dict( + "params"_a = params, + "counts"_a = std::vector( + {num_iou_thresholds, + num_recall_thresholds, + num_categories, + num_area_ranges, + num_max_detections}), + "date"_a = buffer, + "precision"_a = precisions_out, + "recall"_a = recalls_out, + "scores"_a = scores_out); +} + +} // namespace COCOeval + +} // namespace detectron2 diff --git a/annotator/oneformer/detectron2/layers/csrc/cocoeval/cocoeval.h b/annotator/oneformer/detectron2/layers/csrc/cocoeval/cocoeval.h new file mode 100644 index 0000000000000000000000000000000000000000..db246e49a026b7cd989b305f4d3d98100be3c912 --- /dev/null +++ b/annotator/oneformer/detectron2/layers/csrc/cocoeval/cocoeval.h @@ -0,0 +1,88 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#pragma once + +#include +#include +#include +#include +#include + +namespace py = pybind11; + +namespace detectron2 { + +namespace COCOeval { + +// Annotation data for a single object instance in an image +struct InstanceAnnotation { + InstanceAnnotation( + uint64_t id, + double score, + double area, + bool is_crowd, + bool ignore) + : id{id}, score{score}, area{area}, is_crowd{is_crowd}, ignore{ignore} {} + uint64_t id; + double score = 0.; + double area = 0.; + bool is_crowd = false; + bool ignore = false; +}; + +// Stores intermediate results for evaluating detection results for a single +// image that has D detected instances and G ground truth instances. This stores +// matches between detected and ground truth instances +struct ImageEvaluation { + // For each of the D detected instances, the id of the matched ground truth + // instance, or 0 if unmatched + std::vector detection_matches; + + // The detection score of each of the D detected instances + std::vector detection_scores; + + // Marks whether or not each of G instances was ignored from evaluation (e.g., + // because it's outside area_range) + std::vector ground_truth_ignores; + + // Marks whether or not each of D instances was ignored from evaluation (e.g., + // because it's outside aRng) + std::vector detection_ignores; +}; + +template +using ImageCategoryInstances = std::vector>>; + +// C++ implementation of COCO API cocoeval.py::COCOeval.evaluateImg(). For each +// combination of image, category, area range settings, and IOU thresholds to +// evaluate, it matches detected instances to ground truth instances and stores +// the results into a vector of ImageEvaluation results, which will be +// interpreted by the COCOeval::Accumulate() function to produce precion-recall +// curves. The parameters of nested vectors have the following semantics: +// image_category_ious[i][c][d][g] is the intersection over union of the d'th +// detected instance and g'th ground truth instance of +// category category_ids[c] in image image_ids[i] +// image_category_ground_truth_instances[i][c] is a vector of ground truth +// instances in image image_ids[i] of category category_ids[c] +// image_category_detection_instances[i][c] is a vector of detected +// instances in image image_ids[i] of category category_ids[c] +std::vector EvaluateImages( + const std::vector>& area_ranges, // vector of 2-tuples + int max_detections, + const std::vector& iou_thresholds, + const ImageCategoryInstances>& image_category_ious, + const ImageCategoryInstances& + image_category_ground_truth_instances, + const ImageCategoryInstances& + image_category_detection_instances); + +// C++ implementation of COCOeval.accumulate(), which generates precision +// recall curves for each set of category, IOU threshold, detection area range, +// and max number of detections parameters. It is assumed that the parameter +// evaluations is the return value of the functon COCOeval::EvaluateImages(), +// which was called with the same parameter settings params +py::dict Accumulate( + const py::object& params, + const std::vector& evalutations); + +} // namespace COCOeval +} // namespace detectron2 diff --git a/annotator/oneformer/detectron2/layers/csrc/cuda_version.cu b/annotator/oneformer/detectron2/layers/csrc/cuda_version.cu new file mode 100644 index 0000000000000000000000000000000000000000..6dfe1b90c1f65c443681813fd3e3386c9faa3360 --- /dev/null +++ b/annotator/oneformer/detectron2/layers/csrc/cuda_version.cu @@ -0,0 +1,26 @@ +// Copyright (c) Facebook, Inc. and its affiliates. + +#include + +namespace detectron2 { +int get_cudart_version() { +// Not a ROCM platform: Either HIP is not used, or +// it is used, but platform is not ROCM (i.e. it is CUDA) +#if !defined(__HIP_PLATFORM_HCC__) + return CUDART_VERSION; +#else + int version = 0; + +#if HIP_VERSION_MAJOR != 0 + // Create a convention similar to that of CUDA, as assumed by other + // parts of the code. + + version = HIP_VERSION_MINOR; + version += (HIP_VERSION_MAJOR * 100); +#else + hipRuntimeGetVersion(&version); +#endif + return version; +#endif +} +} // namespace detectron2 diff --git a/annotator/oneformer/detectron2/layers/csrc/deformable/deform_conv.h b/annotator/oneformer/detectron2/layers/csrc/deformable/deform_conv.h new file mode 100644 index 0000000000000000000000000000000000000000..965c1bfd47b58f9802d1c3fd69a5962517b2da61 --- /dev/null +++ b/annotator/oneformer/detectron2/layers/csrc/deformable/deform_conv.h @@ -0,0 +1,377 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#pragma once +#include + +namespace detectron2 { + +#if defined(WITH_CUDA) || defined(WITH_HIP) +int deform_conv_forward_cuda( + at::Tensor input, + at::Tensor weight, + at::Tensor offset, + at::Tensor output, + at::Tensor columns, + at::Tensor ones, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + int im2col_step); + +int deform_conv_backward_input_cuda( + at::Tensor input, + at::Tensor offset, + at::Tensor gradOutput, + at::Tensor gradInput, + at::Tensor gradOffset, + at::Tensor weight, + at::Tensor columns, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + int im2col_step); + +int deform_conv_backward_parameters_cuda( + at::Tensor input, + at::Tensor offset, + at::Tensor gradOutput, + at::Tensor gradWeight, // at::Tensor gradBias, + at::Tensor columns, + at::Tensor ones, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + float scale, + int im2col_step); + +void modulated_deform_conv_cuda_forward( + at::Tensor input, + at::Tensor weight, + at::Tensor bias, + at::Tensor ones, + at::Tensor offset, + at::Tensor mask, + at::Tensor output, + at::Tensor columns, + int kernel_h, + int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const bool with_bias); + +void modulated_deform_conv_cuda_backward( + at::Tensor input, + at::Tensor weight, + at::Tensor bias, + at::Tensor ones, + at::Tensor offset, + at::Tensor mask, + at::Tensor columns, + at::Tensor grad_input, + at::Tensor grad_weight, + at::Tensor grad_bias, + at::Tensor grad_offset, + at::Tensor grad_mask, + at::Tensor grad_output, + int kernel_h, + int kernel_w, + int stride_h, + int stride_w, + int pad_h, + int pad_w, + int dilation_h, + int dilation_w, + int group, + int deformable_group, + const bool with_bias); + +#endif + +inline int deform_conv_forward( + at::Tensor input, + at::Tensor weight, + at::Tensor offset, + at::Tensor output, + at::Tensor columns, + at::Tensor ones, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + int im2col_step) { + if (input.is_cuda()) { +#if defined(WITH_CUDA) || defined(WITH_HIP) + TORCH_CHECK(weight.is_cuda(), "weight tensor is not on GPU!"); + TORCH_CHECK(offset.is_cuda(), "offset tensor is not on GPU!"); + return deform_conv_forward_cuda( + input, + weight, + offset, + output, + columns, + ones, + kW, + kH, + dW, + dH, + padW, + padH, + dilationW, + dilationH, + group, + deformable_group, + im2col_step); +#else + AT_ERROR("Detectron2 is not compiled with GPU support!"); +#endif + } + AT_ERROR("This operator is not implemented on CPU"); +} + +inline int deform_conv_backward_input( + at::Tensor input, + at::Tensor offset, + at::Tensor gradOutput, + at::Tensor gradInput, + at::Tensor gradOffset, + at::Tensor weight, + at::Tensor columns, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + int im2col_step) { + if (gradOutput.is_cuda()) { +#if defined(WITH_CUDA) || defined(WITH_HIP) + TORCH_CHECK(input.is_cuda(), "input tensor is not on GPU!"); + TORCH_CHECK(weight.is_cuda(), "weight tensor is not on GPU!"); + TORCH_CHECK(offset.is_cuda(), "offset tensor is not on GPU!"); + return deform_conv_backward_input_cuda( + input, + offset, + gradOutput, + gradInput, + gradOffset, + weight, + columns, + kW, + kH, + dW, + dH, + padW, + padH, + dilationW, + dilationH, + group, + deformable_group, + im2col_step); +#else + AT_ERROR("Detectron2 is not compiled with GPU support!"); +#endif + } + AT_ERROR("This operator is not implemented on CPU"); +} + +inline int deform_conv_backward_filter( + at::Tensor input, + at::Tensor offset, + at::Tensor gradOutput, + at::Tensor gradWeight, // at::Tensor gradBias, + at::Tensor columns, + at::Tensor ones, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + float scale, + int im2col_step) { + if (gradOutput.is_cuda()) { +#if defined(WITH_CUDA) || defined(WITH_HIP) + TORCH_CHECK(input.is_cuda(), "input tensor is not on GPU!"); + TORCH_CHECK(offset.is_cuda(), "offset tensor is not on GPU!"); + return deform_conv_backward_parameters_cuda( + input, + offset, + gradOutput, + gradWeight, + columns, + ones, + kW, + kH, + dW, + dH, + padW, + padH, + dilationW, + dilationH, + group, + deformable_group, + scale, + im2col_step); +#else + AT_ERROR("Detectron2 is not compiled with GPU support!"); +#endif + } + AT_ERROR("This operator is not implemented on CPU"); +} + +inline void modulated_deform_conv_forward( + at::Tensor input, + at::Tensor weight, + at::Tensor bias, + at::Tensor ones, + at::Tensor offset, + at::Tensor mask, + at::Tensor output, + at::Tensor columns, + int kernel_h, + int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const bool with_bias) { + if (input.is_cuda()) { +#if defined(WITH_CUDA) || defined(WITH_HIP) + TORCH_CHECK(weight.is_cuda(), "weight tensor is not on GPU!"); + TORCH_CHECK(bias.is_cuda(), "bias tensor is not on GPU!"); + TORCH_CHECK(offset.is_cuda(), "offset tensor is not on GPU!"); + return modulated_deform_conv_cuda_forward( + input, + weight, + bias, + ones, + offset, + mask, + output, + columns, + kernel_h, + kernel_w, + stride_h, + stride_w, + pad_h, + pad_w, + dilation_h, + dilation_w, + group, + deformable_group, + with_bias); +#else + AT_ERROR("Detectron2 is not compiled with GPU support!"); +#endif + } + AT_ERROR("This operator is not implemented on CPU"); +} + +inline void modulated_deform_conv_backward( + at::Tensor input, + at::Tensor weight, + at::Tensor bias, + at::Tensor ones, + at::Tensor offset, + at::Tensor mask, + at::Tensor columns, + at::Tensor grad_input, + at::Tensor grad_weight, + at::Tensor grad_bias, + at::Tensor grad_offset, + at::Tensor grad_mask, + at::Tensor grad_output, + int kernel_h, + int kernel_w, + int stride_h, + int stride_w, + int pad_h, + int pad_w, + int dilation_h, + int dilation_w, + int group, + int deformable_group, + const bool with_bias) { + if (grad_output.is_cuda()) { +#if defined(WITH_CUDA) || defined(WITH_HIP) + TORCH_CHECK(input.is_cuda(), "input tensor is not on GPU!"); + TORCH_CHECK(weight.is_cuda(), "weight tensor is not on GPU!"); + TORCH_CHECK(bias.is_cuda(), "bias tensor is not on GPU!"); + TORCH_CHECK(offset.is_cuda(), "offset tensor is not on GPU!"); + return modulated_deform_conv_cuda_backward( + input, + weight, + bias, + ones, + offset, + mask, + columns, + grad_input, + grad_weight, + grad_bias, + grad_offset, + grad_mask, + grad_output, + kernel_h, + kernel_w, + stride_h, + stride_w, + pad_h, + pad_w, + dilation_h, + dilation_w, + group, + deformable_group, + with_bias); +#else + AT_ERROR("Detectron2 is not compiled with GPU support!"); +#endif + } + AT_ERROR("This operator is not implemented on CPU"); +} + +} // namespace detectron2 diff --git a/annotator/oneformer/detectron2/layers/csrc/deformable/deform_conv_cuda.cu b/annotator/oneformer/detectron2/layers/csrc/deformable/deform_conv_cuda.cu new file mode 100644 index 0000000000000000000000000000000000000000..2072bb856ec40b61c3826cead2fb7bb7c971a089 --- /dev/null +++ b/annotator/oneformer/detectron2/layers/csrc/deformable/deform_conv_cuda.cu @@ -0,0 +1,1223 @@ +// Copyright (c) Facebook, Inc. and its affiliates. + +// modified from +// https://github.com/open-mmlab/mmdetection/blob/master/mmdet/ops/dcn/src/deform_conv_cuda.cpp +// Original license: Apache 2.0 + +// modify from +// https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/blob/mmdetection/mmdet/ops/dcn/src/deform_conv_cuda.c +// Original license: Apache 2.0 + +#include + +#include "deform_conv.h" + +#include +#include + +namespace detectron2 { + +void deformable_im2col( + const at::Tensor data_im, + const at::Tensor data_offset, + const int channels, + const int height, + const int width, + const int ksize_h, + const int ksize_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int parallel_imgs, + const int deformable_group, + at::Tensor data_col); + +void deformable_col2im( + const at::Tensor data_col, + const at::Tensor data_offset, + const int channels, + const int height, + const int width, + const int ksize_h, + const int ksize_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int parallel_imgs, + const int deformable_group, + at::Tensor grad_im); + +void deformable_col2im_coord( + const at::Tensor data_col, + const at::Tensor data_im, + const at::Tensor data_offset, + const int channels, + const int height, + const int width, + const int ksize_h, + const int ksize_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int parallel_imgs, + const int deformable_group, + at::Tensor grad_offset); + +void modulated_deformable_im2col_cuda( + const at::Tensor data_im, + const at::Tensor data_offset, + const at::Tensor data_mask, + const int batch_size, + const int channels, + const int height_im, + const int width_im, + const int height_col, + const int width_col, + const int kernel_h, + const int kenerl_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int deformable_group, + at::Tensor data_col); + +void modulated_deformable_col2im_cuda( + const at::Tensor data_col, + const at::Tensor data_offset, + const at::Tensor data_mask, + const int batch_size, + const int channels, + const int height_im, + const int width_im, + const int height_col, + const int width_col, + const int kernel_h, + const int kenerl_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int deformable_group, + at::Tensor grad_im); + +void modulated_deformable_col2im_coord_cuda( + const at::Tensor data_col, + const at::Tensor data_im, + const at::Tensor data_offset, + const at::Tensor data_mask, + const int batch_size, + const int channels, + const int height_im, + const int width_im, + const int height_col, + const int width_col, + const int kernel_h, + const int kenerl_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int deformable_group, + at::Tensor grad_offset, + at::Tensor grad_mask); + +void shape_check( + at::Tensor input, + at::Tensor offset, + at::Tensor* gradOutput, + at::Tensor weight, + int kH, + int kW, + int dH, + int dW, + int padH, + int padW, + int dilationH, + int dilationW, + int group, + int deformable_group) { + TORCH_CHECK( + weight.ndimension() == 4, + "4D weight tensor (nOutputPlane,nInputPlane,kH,kW) expected, " + "but got: %s", + weight.ndimension()); + + TORCH_CHECK(weight.is_contiguous(), "weight tensor has to be contiguous"); + + TORCH_CHECK( + kW > 0 && kH > 0, + "kernel size should be greater than zero, but got kH: %d kW: %d", + kH, + kW); + + TORCH_CHECK( + (weight.size(2) == kH && weight.size(3) == kW), + "kernel size should be consistent with weight, ", + "but got kH: %d kW: %d weight.size(2): %d, weight.size(3): %d", + kH, + kW, + weight.size(2), + weight.size(3)); + + TORCH_CHECK( + dW > 0 && dH > 0, + "stride should be greater than zero, but got dH: %d dW: %d", + dH, + dW); + + TORCH_CHECK( + dilationW > 0 && dilationH > 0, + "dilation should be greater than 0, but got dilationH: %d dilationW: %d", + dilationH, + dilationW); + + int ndim = input.ndimension(); + int dimf = 0; + int dimh = 1; + int dimw = 2; + + if (ndim == 4) { + dimf++; + dimh++; + dimw++; + } + + TORCH_CHECK( + ndim == 3 || ndim == 4, + "3D or 4D input tensor expected but got: %s", + ndim); + + long nInputPlane = weight.size(1) * group; + long inputHeight = input.size(dimh); + long inputWidth = input.size(dimw); + long nOutputPlane = weight.size(0); + long outputHeight = + (inputHeight + 2 * padH - (dilationH * (kH - 1) + 1)) / dH + 1; + long outputWidth = + (inputWidth + 2 * padW - (dilationW * (kW - 1) + 1)) / dW + 1; + + TORCH_CHECK( + nInputPlane % deformable_group == 0, + "input channels must divide deformable group size"); + + if (outputWidth < 1 || outputHeight < 1) + AT_ERROR( + "Given input size: (%ld x %ld x %ld). " + "Calculated output size: (%ld x %ld x %ld). Output size is too small", + nInputPlane, + inputHeight, + inputWidth, + nOutputPlane, + outputHeight, + outputWidth); + + TORCH_CHECK( + input.size(1) == nInputPlane, + "invalid number of input planes, expected: %d, but got: %d", + nInputPlane, + input.size(1)); + + TORCH_CHECK( + (inputHeight + 2 * padH >= kH && inputWidth + 2 * padW >= kW), + "input image is smaller than kernel"); + + TORCH_CHECK( + (offset.size(2) == outputHeight && offset.size(3) == outputWidth), + "invalid spatial size of offset, expected height: %d width: %d, but " + "got height: %d width: %d", + outputHeight, + outputWidth, + offset.size(2), + offset.size(3)); + + TORCH_CHECK( + (offset.size(1) == deformable_group * 2 * kH * kW), + "invalid number of channels of offset"); + + if (gradOutput != NULL) { + TORCH_CHECK( + gradOutput->size(dimf) == nOutputPlane, + "invalid number of gradOutput planes, expected: %d, but got: %d", + nOutputPlane, + gradOutput->size(dimf)); + + TORCH_CHECK( + (gradOutput->size(dimh) == outputHeight && + gradOutput->size(dimw) == outputWidth), + "invalid size of gradOutput, expected height: %d width: %d , but " + "got height: %d width: %d", + outputHeight, + outputWidth, + gradOutput->size(dimh), + gradOutput->size(dimw)); + } +} + +int deform_conv_forward_cuda( + at::Tensor input, + at::Tensor weight, + at::Tensor offset, + at::Tensor output, + at::Tensor columns, + at::Tensor ones, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + int im2col_step) { + // todo: resize columns to include im2col: done + // todo: add im2col_step as input + // todo: add new output buffer and transpose it to output (or directly + // transpose output) todo: possibly change data indexing because of + // parallel_imgs + + shape_check( + input, + offset, + NULL, + weight, + kH, + kW, + dH, + dW, + padH, + padW, + dilationH, + dilationW, + group, + deformable_group); + + input = input.contiguous(); + offset = offset.contiguous(); + weight = weight.contiguous(); + + int batch = 1; + if (input.ndimension() == 3) { + // Force batch + batch = 0; + input.unsqueeze_(0); + offset.unsqueeze_(0); + } + + // todo: assert batchsize dividable by im2col_step + + long batchSize = input.size(0); + long nInputPlane = input.size(1); + long inputHeight = input.size(2); + long inputWidth = input.size(3); + + long nOutputPlane = weight.size(0); + + long outputWidth = + (inputWidth + 2 * padW - (dilationW * (kW - 1) + 1)) / dW + 1; + long outputHeight = + (inputHeight + 2 * padH - (dilationH * (kH - 1) + 1)) / dH + 1; + + TORCH_CHECK((offset.size(0) == batchSize), "invalid batch size of offset"); + + output = output.view( + {batchSize / im2col_step, + im2col_step, + nOutputPlane, + outputHeight, + outputWidth}); + columns = at::zeros( + {nInputPlane * kW * kH, im2col_step * outputHeight * outputWidth}, + input.options()); + + if (ones.ndimension() != 2 || + ones.size(0) * ones.size(1) < outputHeight * outputWidth) { + ones = at::ones({outputHeight, outputWidth}, input.options()); + } + + input = input.view( + {batchSize / im2col_step, + im2col_step, + nInputPlane, + inputHeight, + inputWidth}); + offset = offset.view( + {batchSize / im2col_step, + im2col_step, + deformable_group * 2 * kH * kW, + outputHeight, + outputWidth}); + + at::Tensor output_buffer = at::zeros( + {batchSize / im2col_step, + nOutputPlane, + im2col_step * outputHeight, + outputWidth}, + output.options()); + + output_buffer = output_buffer.view( + {output_buffer.size(0), + group, + output_buffer.size(1) / group, + output_buffer.size(2), + output_buffer.size(3)}); + + for (int elt = 0; elt < batchSize / im2col_step; elt++) { + deformable_im2col( + input[elt], + offset[elt], + nInputPlane, + inputHeight, + inputWidth, + kH, + kW, + padH, + padW, + dH, + dW, + dilationH, + dilationW, + im2col_step, + deformable_group, + columns); + + columns = columns.view({group, columns.size(0) / group, columns.size(1)}); + weight = weight.view( + {group, + weight.size(0) / group, + weight.size(1), + weight.size(2), + weight.size(3)}); + + for (int g = 0; g < group; g++) { + output_buffer[elt][g] = output_buffer[elt][g] + .flatten(1) + .addmm_(weight[g].flatten(1), columns[g]) + .view_as(output_buffer[elt][g]); + } + } + + output_buffer = output_buffer.view( + {output_buffer.size(0), + output_buffer.size(1) * output_buffer.size(2), + output_buffer.size(3), + output_buffer.size(4)}); + + output_buffer = output_buffer.view( + {batchSize / im2col_step, + nOutputPlane, + im2col_step, + outputHeight, + outputWidth}); + output_buffer.transpose_(1, 2); + output.copy_(output_buffer); + output = output.view({batchSize, nOutputPlane, outputHeight, outputWidth}); + + input = input.view({batchSize, nInputPlane, inputHeight, inputWidth}); + offset = offset.view( + {batchSize, deformable_group * 2 * kH * kW, outputHeight, outputWidth}); + + if (batch == 0) { + output = output.view({nOutputPlane, outputHeight, outputWidth}); + input = input.view({nInputPlane, inputHeight, inputWidth}); + offset = offset.view({offset.size(1), offset.size(2), offset.size(3)}); + } + + return 1; +} + +int deform_conv_backward_input_cuda( + at::Tensor input, + at::Tensor offset, + at::Tensor gradOutput, + at::Tensor gradInput, + at::Tensor gradOffset, + at::Tensor weight, + at::Tensor columns, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + int im2col_step) { + shape_check( + input, + offset, + &gradOutput, + weight, + kH, + kW, + dH, + dW, + padH, + padW, + dilationH, + dilationW, + group, + deformable_group); + + input = input.contiguous(); + offset = offset.contiguous(); + gradOutput = gradOutput.contiguous(); + weight = weight.contiguous(); + + int batch = 1; + + if (input.ndimension() == 3) { + // Force batch + batch = 0; + input = input.view({1, input.size(0), input.size(1), input.size(2)}); + offset = offset.view({1, offset.size(0), offset.size(1), offset.size(2)}); + gradOutput = gradOutput.view( + {1, gradOutput.size(0), gradOutput.size(1), gradOutput.size(2)}); + } + + long batchSize = input.size(0); + long nInputPlane = input.size(1); + long inputHeight = input.size(2); + long inputWidth = input.size(3); + + long nOutputPlane = weight.size(0); + + long outputWidth = + (inputWidth + 2 * padW - (dilationW * (kW - 1) + 1)) / dW + 1; + long outputHeight = + (inputHeight + 2 * padH - (dilationH * (kH - 1) + 1)) / dH + 1; + + TORCH_CHECK((offset.size(0) == batchSize), 3, "invalid batch size of offset"); + gradInput = gradInput.view({batchSize, nInputPlane, inputHeight, inputWidth}); + columns = at::zeros( + {nInputPlane * kW * kH, im2col_step * outputHeight * outputWidth}, + input.options()); + + // change order of grad output + gradOutput = gradOutput.view( + {batchSize / im2col_step, + im2col_step, + nOutputPlane, + outputHeight, + outputWidth}); + gradOutput.transpose_(1, 2); + + gradInput = gradInput.view( + {batchSize / im2col_step, + im2col_step, + nInputPlane, + inputHeight, + inputWidth}); + input = input.view( + {batchSize / im2col_step, + im2col_step, + nInputPlane, + inputHeight, + inputWidth}); + gradOffset = gradOffset.view( + {batchSize / im2col_step, + im2col_step, + deformable_group * 2 * kH * kW, + outputHeight, + outputWidth}); + offset = offset.view( + {batchSize / im2col_step, + im2col_step, + deformable_group * 2 * kH * kW, + outputHeight, + outputWidth}); + + for (int elt = 0; elt < batchSize / im2col_step; elt++) { + // divide into groups + columns = columns.view({group, columns.size(0) / group, columns.size(1)}); + weight = weight.view( + {group, + weight.size(0) / group, + weight.size(1), + weight.size(2), + weight.size(3)}); + gradOutput = gradOutput.view( + {gradOutput.size(0), + group, + gradOutput.size(1) / group, + gradOutput.size(2), + gradOutput.size(3), + gradOutput.size(4)}); + + for (int g = 0; g < group; g++) { + columns[g] = columns[g].addmm_( + weight[g].flatten(1).transpose(0, 1), + gradOutput[elt][g].flatten(1), + 0.0f, + 1.0f); + } + + columns = + columns.view({columns.size(0) * columns.size(1), columns.size(2)}); + gradOutput = gradOutput.view( + {gradOutput.size(0), + gradOutput.size(1) * gradOutput.size(2), + gradOutput.size(3), + gradOutput.size(4), + gradOutput.size(5)}); + + deformable_col2im_coord( + columns, + input[elt], + offset[elt], + nInputPlane, + inputHeight, + inputWidth, + kH, + kW, + padH, + padW, + dH, + dW, + dilationH, + dilationW, + im2col_step, + deformable_group, + gradOffset[elt]); + + deformable_col2im( + columns, + offset[elt], + nInputPlane, + inputHeight, + inputWidth, + kH, + kW, + padH, + padW, + dH, + dW, + dilationH, + dilationW, + im2col_step, + deformable_group, + gradInput[elt]); + } + + gradOutput.transpose_(1, 2); + gradOutput = + gradOutput.view({batchSize, nOutputPlane, outputHeight, outputWidth}); + + gradInput = gradInput.view({batchSize, nInputPlane, inputHeight, inputWidth}); + input = input.view({batchSize, nInputPlane, inputHeight, inputWidth}); + gradOffset = gradOffset.view( + {batchSize, deformable_group * 2 * kH * kW, outputHeight, outputWidth}); + offset = offset.view( + {batchSize, deformable_group * 2 * kH * kW, outputHeight, outputWidth}); + + if (batch == 0) { + gradOutput = gradOutput.view({nOutputPlane, outputHeight, outputWidth}); + input = input.view({nInputPlane, inputHeight, inputWidth}); + gradInput = gradInput.view({nInputPlane, inputHeight, inputWidth}); + offset = offset.view({offset.size(1), offset.size(2), offset.size(3)}); + gradOffset = + gradOffset.view({offset.size(1), offset.size(2), offset.size(3)}); + } + + return 1; +} + +int deform_conv_backward_parameters_cuda( + at::Tensor input, + at::Tensor offset, + at::Tensor gradOutput, + at::Tensor gradWeight, // at::Tensor gradBias, + at::Tensor columns, + at::Tensor ones, + int kW, + int kH, + int dW, + int dH, + int padW, + int padH, + int dilationW, + int dilationH, + int group, + int deformable_group, + float scale, + int im2col_step) { + // todo: transpose and reshape outGrad + // todo: reshape columns + // todo: add im2col_step as input + + shape_check( + input, + offset, + &gradOutput, + gradWeight, + kH, + kW, + dH, + dW, + padH, + padW, + dilationH, + dilationW, + group, + deformable_group); + + input = input.contiguous(); + offset = offset.contiguous(); + gradOutput = gradOutput.contiguous(); + + int batch = 1; + + if (input.ndimension() == 3) { + // Force batch + batch = 0; + input = input.view( + at::IntList({1, input.size(0), input.size(1), input.size(2)})); + gradOutput = gradOutput.view( + {1, gradOutput.size(0), gradOutput.size(1), gradOutput.size(2)}); + } + + long batchSize = input.size(0); + long nInputPlane = input.size(1); + long inputHeight = input.size(2); + long inputWidth = input.size(3); + + long nOutputPlane = gradWeight.size(0); + + long outputWidth = + (inputWidth + 2 * padW - (dilationW * (kW - 1) + 1)) / dW + 1; + long outputHeight = + (inputHeight + 2 * padH - (dilationH * (kH - 1) + 1)) / dH + 1; + + TORCH_CHECK((offset.size(0) == batchSize), "invalid batch size of offset"); + + columns = at::zeros( + {nInputPlane * kW * kH, im2col_step * outputHeight * outputWidth}, + input.options()); + + gradOutput = gradOutput.view( + {batchSize / im2col_step, + im2col_step, + nOutputPlane, + outputHeight, + outputWidth}); + gradOutput.transpose_(1, 2); + + at::Tensor gradOutputBuffer = at::zeros_like(gradOutput); + gradOutputBuffer = gradOutputBuffer.view( + {batchSize / im2col_step, + nOutputPlane, + im2col_step, + outputHeight, + outputWidth}); + gradOutputBuffer.copy_(gradOutput); + // gradOutput is not contiguous, so we do reshape (instead of view) next + gradOutputBuffer = gradOutputBuffer.reshape( + {batchSize / im2col_step, + nOutputPlane, + im2col_step * outputHeight, + outputWidth}); + + gradOutput.transpose_(1, 2); + gradOutput = + gradOutput.view({batchSize, nOutputPlane, outputHeight, outputWidth}); + + input = input.view( + {batchSize / im2col_step, + im2col_step, + nInputPlane, + inputHeight, + inputWidth}); + offset = offset.view( + {batchSize / im2col_step, + im2col_step, + deformable_group * 2 * kH * kW, + outputHeight, + outputWidth}); + + for (int elt = 0; elt < batchSize / im2col_step; elt++) { + deformable_im2col( + input[elt], + offset[elt], + nInputPlane, + inputHeight, + inputWidth, + kH, + kW, + padH, + padW, + dH, + dW, + dilationH, + dilationW, + im2col_step, + deformable_group, + columns); + + // divide into group + gradOutputBuffer = gradOutputBuffer.view( + {gradOutputBuffer.size(0), + group, + gradOutputBuffer.size(1) / group, + gradOutputBuffer.size(2), + gradOutputBuffer.size(3)}); + columns = columns.view({group, columns.size(0) / group, columns.size(1)}); + gradWeight = gradWeight.view( + {group, + gradWeight.size(0) / group, + gradWeight.size(1), + gradWeight.size(2), + gradWeight.size(3)}); + + for (int g = 0; g < group; g++) { + gradWeight[g] = gradWeight[g] + .flatten(1) + .addmm_( + gradOutputBuffer[elt][g].flatten(1), + columns[g].transpose(1, 0), + 1.0, + scale) + .view_as(gradWeight[g]); + } + gradOutputBuffer = gradOutputBuffer.view( + {gradOutputBuffer.size(0), + gradOutputBuffer.size(1) * gradOutputBuffer.size(2), + gradOutputBuffer.size(3), + gradOutputBuffer.size(4)}); + columns = + columns.view({columns.size(0) * columns.size(1), columns.size(2)}); + gradWeight = gradWeight.view( + {gradWeight.size(0) * gradWeight.size(1), + gradWeight.size(2), + gradWeight.size(3), + gradWeight.size(4)}); + } + + input = input.view({batchSize, nInputPlane, inputHeight, inputWidth}); + offset = offset.view( + {batchSize, deformable_group * 2 * kH * kW, outputHeight, outputWidth}); + + if (batch == 0) { + gradOutput = gradOutput.view({nOutputPlane, outputHeight, outputWidth}); + input = input.view({nInputPlane, inputHeight, inputWidth}); + } + + return 1; +} + +void modulated_deform_conv_cuda_forward( + at::Tensor input, + at::Tensor weight, + at::Tensor bias, + at::Tensor ones, + at::Tensor offset, + at::Tensor mask, + at::Tensor output, + at::Tensor columns, + int kernel_h, + int kernel_w, + const int stride_h, + const int stride_w, + const int pad_h, + const int pad_w, + const int dilation_h, + const int dilation_w, + const int group, + const int deformable_group, + const bool with_bias) { + shape_check( + input, + offset, + NULL, + weight, + kernel_h, + kernel_w, + stride_h, + stride_w, + pad_h, + pad_w, + dilation_h, + dilation_w, + group, + deformable_group); + + TORCH_CHECK(input.is_contiguous(), "input tensor has to be contiguous"); + TORCH_CHECK(weight.is_contiguous(), "weight tensor has to be contiguous"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + + const int channels_out = weight.size(0); + const int channels_kernel = weight.size(1); + const int kernel_h_ = weight.size(2); + const int kernel_w_ = weight.size(3); + + if (kernel_h_ != kernel_h || kernel_w_ != kernel_w) + AT_ERROR( + "Input shape and kernel shape wont match: (%d x %d vs %d x %d).", + kernel_h_, + kernel_w, + kernel_h_, + kernel_w_); + if (channels != channels_kernel * group) + AT_ERROR( + "Input shape and kernel channels wont match: (%d vs %d).", + channels, + channels_kernel * group); + + const int height_out = + (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; + const int width_out = + (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + // mask shape check + TORCH_CHECK( + (mask.size(2) == height_out && mask.size(3) == width_out), + "invalid spatial size of mask, expected height: %d width: %d, but " + "got height: %d width: %d", + height_out, + width_out, + mask.size(2), + mask.size(3)); + + TORCH_CHECK( + (mask.size(1) == deformable_group * kernel_h * kernel_w), + "invalid number of channels of mask"); + + if (ones.ndimension() != 2 || + ones.size(0) * ones.size(1) < height_out * width_out) { + // Resize plane and fill with ones... + ones = at::ones({height_out, width_out}, input.options()); + } + + // resize output + output = output.view({batch, channels_out, height_out, width_out}).zero_(); + // resize temporary columns + columns = at::zeros( + {channels * kernel_h * kernel_w, 1 * height_out * width_out}, + input.options()); + + output = output.view( + {output.size(0), + group, + output.size(1) / group, + output.size(2), + output.size(3)}); + + for (int b = 0; b < batch; b++) { + modulated_deformable_im2col_cuda( + input[b], + offset[b], + mask[b], + 1, + channels, + height, + width, + height_out, + width_out, + kernel_h, + kernel_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + deformable_group, + columns); + + // divide into group + weight = weight.view( + {group, + weight.size(0) / group, + weight.size(1), + weight.size(2), + weight.size(3)}); + columns = columns.view({group, columns.size(0) / group, columns.size(1)}); + + for (int g = 0; g < group; g++) { + output[b][g] = output[b][g] + .flatten(1) + .addmm_(weight[g].flatten(1), columns[g]) + .view_as(output[b][g]); + } + + weight = weight.view( + {weight.size(0) * weight.size(1), + weight.size(2), + weight.size(3), + weight.size(4)}); + columns = + columns.view({columns.size(0) * columns.size(1), columns.size(2)}); + } + + output = output.view( + {output.size(0), + output.size(1) * output.size(2), + output.size(3), + output.size(4)}); + + if (with_bias) { + output += bias.view({1, bias.size(0), 1, 1}); + } +} + +void modulated_deform_conv_cuda_backward( + at::Tensor input, + at::Tensor weight, + at::Tensor bias, + at::Tensor ones, + at::Tensor offset, + at::Tensor mask, + at::Tensor columns, + at::Tensor grad_input, + at::Tensor grad_weight, + at::Tensor grad_bias, + at::Tensor grad_offset, + at::Tensor grad_mask, + at::Tensor grad_output, + int kernel_h, + int kernel_w, + int stride_h, + int stride_w, + int pad_h, + int pad_w, + int dilation_h, + int dilation_w, + int group, + int deformable_group, + const bool with_bias) { + shape_check( + input, + offset, + &grad_output, + weight, + kernel_h, + kernel_w, + stride_h, + stride_w, + pad_h, + pad_w, + dilation_h, + dilation_w, + group, + deformable_group); + + TORCH_CHECK(input.is_contiguous(), "input tensor has to be contiguous"); + TORCH_CHECK(weight.is_contiguous(), "weight tensor has to be contiguous"); + + const int batch = input.size(0); + const int channels = input.size(1); + const int height = input.size(2); + const int width = input.size(3); + + const int channels_kernel = weight.size(1); + const int kernel_h_ = weight.size(2); + const int kernel_w_ = weight.size(3); + if (kernel_h_ != kernel_h || kernel_w_ != kernel_w) + AT_ERROR( + "Input shape and kernel shape wont match: (%d x %d vs %d x %d).", + kernel_h_, + kernel_w, + kernel_h_, + kernel_w_); + if (channels != channels_kernel * group) + AT_ERROR( + "Input shape and kernel channels wont match: (%d vs %d).", + channels, + channels_kernel * group); + + const int height_out = + (height + 2 * pad_h - (dilation_h * (kernel_h - 1) + 1)) / stride_h + 1; + const int width_out = + (width + 2 * pad_w - (dilation_w * (kernel_w - 1) + 1)) / stride_w + 1; + + // mask shape check + TORCH_CHECK( + (mask.size(2) == height_out && mask.size(3) == width_out), + "invalid spatial size of mask, expected height: %d width: %d, but " + "got height: %d width: %d", + height_out, + width_out, + mask.size(2), + mask.size(3)); + + TORCH_CHECK( + (mask.size(1) == deformable_group * kernel_h * kernel_w), + "invalid number of channels of mask"); + + if (ones.ndimension() != 2 || + ones.size(0) * ones.size(1) < height_out * width_out) { + // Resize plane and fill with ones... + ones = at::ones({height_out, width_out}, input.options()); + } + + grad_input = grad_input.view({batch, channels, height, width}); + columns = at::zeros( + {channels * kernel_h * kernel_w, height_out * width_out}, + input.options()); + + grad_output = grad_output.view( + {grad_output.size(0), + group, + grad_output.size(1) / group, + grad_output.size(2), + grad_output.size(3)}); + + for (int b = 0; b < batch; b++) { + // divide int group + columns = columns.view({group, columns.size(0) / group, columns.size(1)}); + weight = weight.view( + {group, + weight.size(0) / group, + weight.size(1), + weight.size(2), + weight.size(3)}); + + for (int g = 0; g < group; g++) { + columns[g].addmm_( + weight[g].flatten(1).transpose(0, 1), + grad_output[b][g].flatten(1), + 0.0f, + 1.0f); + } + + columns = + columns.view({columns.size(0) * columns.size(1), columns.size(2)}); + weight = weight.view( + {weight.size(0) * weight.size(1), + weight.size(2), + weight.size(3), + weight.size(4)}); + + // gradient w.r.t. input coordinate data + modulated_deformable_col2im_coord_cuda( + columns, + input[b], + offset[b], + mask[b], + 1, + channels, + height, + width, + height_out, + width_out, + kernel_h, + kernel_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + deformable_group, + grad_offset[b], + grad_mask[b]); + // gradient w.r.t. input data + modulated_deformable_col2im_cuda( + columns, + offset[b], + mask[b], + 1, + channels, + height, + width, + height_out, + width_out, + kernel_h, + kernel_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + deformable_group, + grad_input[b]); + + // gradient w.r.t. weight, dWeight should accumulate across the batch and + // group + modulated_deformable_im2col_cuda( + input[b], + offset[b], + mask[b], + 1, + channels, + height, + width, + height_out, + width_out, + kernel_h, + kernel_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + deformable_group, + columns); + + columns = columns.view({group, columns.size(0) / group, columns.size(1)}); + grad_weight = grad_weight.view( + {group, + grad_weight.size(0) / group, + grad_weight.size(1), + grad_weight.size(2), + grad_weight.size(3)}); + if (with_bias) + grad_bias = grad_bias.view({group, grad_bias.size(0) / group}); + + for (int g = 0; g < group; g++) { + grad_weight[g] = + grad_weight[g] + .flatten(1) + .addmm_(grad_output[b][g].flatten(1), columns[g].transpose(0, 1)) + .view_as(grad_weight[g]); + if (with_bias) { + grad_bias[g] = + grad_bias[g] + .view({-1, 1}) + .addmm_(grad_output[b][g].flatten(1), ones.view({-1, 1})) + .view(-1); + } + } + + columns = + columns.view({columns.size(0) * columns.size(1), columns.size(2)}); + grad_weight = grad_weight.view( + {grad_weight.size(0) * grad_weight.size(1), + grad_weight.size(2), + grad_weight.size(3), + grad_weight.size(4)}); + if (with_bias) + grad_bias = grad_bias.view({grad_bias.size(0) * grad_bias.size(1)}); + } + grad_output = grad_output.view( + {grad_output.size(0) * grad_output.size(1), + grad_output.size(2), + grad_output.size(3), + grad_output.size(4)}); +} + +} // namespace detectron2 diff --git a/annotator/oneformer/detectron2/layers/csrc/deformable/deform_conv_cuda_kernel.cu b/annotator/oneformer/detectron2/layers/csrc/deformable/deform_conv_cuda_kernel.cu new file mode 100644 index 0000000000000000000000000000000000000000..f299c7add116685e9c87a187a85ea63f9f808867 --- /dev/null +++ b/annotator/oneformer/detectron2/layers/csrc/deformable/deform_conv_cuda_kernel.cu @@ -0,0 +1,1288 @@ +// Copyright (c) Facebook, Inc. and its affiliates. + +// modified from +// https://github.com/open-mmlab/mmdetection/blob/master/mmdet/ops/dcn/src/deform_conv_cuda_kernel.cu +// Original license: Apache 2.0 +// clang-format off + +// modify from +// https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/blob/mmdetection/mmdet/ops/dcn/src/deform_conv_cuda_kernel.cu + +/*! + ******************* BEGIN Caffe Copyright Notice and Disclaimer ***************** + * + * COPYRIGHT + * + * All contributions by the University of California: + * Copyright (c) 2014-2017 The Regents of the University of California (Regents) + * All rights reserved. + * + * All other contributions: + * Copyright (c) 2014-2017, the respective contributors + * All rights reserved. + * + * Caffe uses a shared copyright model: each contributor holds copyright over + * their contributions to Caffe. The project versioning records all such + * contribution and copyright details. If a contributor wants to further mark + * their specific copyright on a particular contribution, they should indicate + * their copyright solely in the commit message of the change when it is + * committed. + * + * LICENSE + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + *AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + *IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE + *FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + *DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + *SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + *CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + *OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + *OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * CONTRIBUTION AGREEMENT + * + * By contributing to the BVLC/caffe repository through pull-request, comment, + * or otherwise, the contributor releases their content to the + * license and copyright terms herein. + * + ***************** END Caffe Copyright Notice and Disclaimer ********************* + * + * Copyright (c) 2018 Microsoft + * Licensed under The MIT License [see LICENSE for details] + * \file modulated_deformable_im2col.cuh + * \brief Function definitions of converting an image to + * column matrix based on kernel, padding, dilation, and offset. + * These functions are mainly used in deformable convolution operators. + * \ref: https://arxiv.org/abs/1703.06211 + * \author Yuwen Xiong, Haozhi Qi, Jifeng Dai, Xizhou Zhu, Han Hu, Dazhi Cheng + */ + +#include +#include +#include +#include +#include +#include + +using namespace at; + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; i < (n); \ + i += blockDim.x * gridDim.x) + + +namespace { + +const int CUDA_NUM_THREADS = 1024; +const int kMaxGridNum = 65535; + +inline int GET_BLOCKS(const int N) { + return std::min(kMaxGridNum, (N + CUDA_NUM_THREADS - 1) / CUDA_NUM_THREADS); +} + +} + +template +__device__ scalar_t deformable_im2col_bilinear( + const scalar_t* bottom_data, + const int data_width, + const int height, + const int width, + scalar_t h, + scalar_t w) { + int h_low = floor(h); + int w_low = floor(w); + int h_high = h_low + 1; + int w_high = w_low + 1; + + scalar_t lh = h - h_low; + scalar_t lw = w - w_low; + scalar_t hh = 1 - lh, hw = 1 - lw; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + v1 = bottom_data[h_low * data_width + w_low]; + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + v2 = bottom_data[h_low * data_width + w_high]; + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + v3 = bottom_data[h_high * data_width + w_low]; + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + v4 = bottom_data[h_high * data_width + w_high]; + + scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + +template +__device__ scalar_t get_gradient_weight( + scalar_t argmax_h, + scalar_t argmax_w, + const int h, + const int w, + const int height, + const int width) { + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || + argmax_w >= width) { + // empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + if (h == argmax_h_low && w == argmax_w_low) + weight = (h + 1 - argmax_h) * (w + 1 - argmax_w); + if (h == argmax_h_low && w == argmax_w_high) + weight = (h + 1 - argmax_h) * (argmax_w + 1 - w); + if (h == argmax_h_high && w == argmax_w_low) + weight = (argmax_h + 1 - h) * (w + 1 - argmax_w); + if (h == argmax_h_high && w == argmax_w_high) + weight = (argmax_h + 1 - h) * (argmax_w + 1 - w); + return weight; +} + +template +__device__ scalar_t get_coordinate_weight( + scalar_t argmax_h, + scalar_t argmax_w, + const int height, + const int width, + const scalar_t* im_data, + const int data_width, + const int bp_dir) { + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || + argmax_w >= width) { + // empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + + if (bp_dir == 0) { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_w_low + 1 - argmax_w) * + im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += -1 * (argmax_w - argmax_w_low) * + im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += (argmax_w_low + 1 - argmax_w) * + im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_w - argmax_w_low) * + im_data[argmax_h_high * data_width + argmax_w_high]; + } else if (bp_dir == 1) { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_h_low + 1 - argmax_h) * + im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += (argmax_h_low + 1 - argmax_h) * + im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += -1 * (argmax_h - argmax_h_low) * + im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_h - argmax_h_low) * + im_data[argmax_h_high * data_width + argmax_w_high]; + } + + return weight; +} + +template +__global__ void deformable_im2col_gpu_kernel( + const int n, + const scalar_t* data_im, + const scalar_t* data_offset, + const int height, + const int width, + const int kernel_h, + const int kernel_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, + const int num_channels, + const int deformable_group, + const int height_col, + const int width_col, + scalar_t* data_col) { + CUDA_KERNEL_LOOP(index, n) { + // index index of output matrix + const int w_col = index % width_col; + const int h_col = (index / width_col) % height_col; + const int b_col = (index / width_col / height_col) % batch_size; + const int c_im = (index / width_col / height_col) / batch_size; + const int c_col = c_im * kernel_h * kernel_w; + + // compute deformable group index + const int deformable_group_index = c_im / channel_per_deformable_group; + + const int h_in = h_col * stride_h - pad_h; + const int w_in = w_col * stride_w - pad_w; + scalar_t* data_col_ptr = data_col + + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + // const scalar_t* data_im_ptr = data_im + ((b_col * num_channels + c_im) * + // height + h_in) * width + w_in; + const scalar_t* data_im_ptr = + data_im + (b_col * num_channels + c_im) * height * width; + const scalar_t* data_offset_ptr = data_offset + + (b_col * deformable_group + deformable_group_index) * 2 * kernel_h * + kernel_w * height_col * width_col; + + for (int i = 0; i < kernel_h; ++i) { + for (int j = 0; j < kernel_w; ++j) { + const int data_offset_h_ptr = + ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col; + const int data_offset_w_ptr = + ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + + w_col; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + scalar_t val = static_cast(0); + const scalar_t h_im = h_in + i * dilation_h + offset_h; + const scalar_t w_im = w_in + j * dilation_w + offset_w; + if (h_im > -1 && w_im > -1 && h_im < height && w_im < width) { + // const scalar_t map_h = i * dilation_h + offset_h; + // const scalar_t map_w = j * dilation_w + offset_w; + // const int cur_height = height - h_in; + // const int cur_width = width - w_in; + // val = deformable_im2col_bilinear(data_im_ptr, width, cur_height, + // cur_width, map_h, map_w); + val = deformable_im2col_bilinear( + data_im_ptr, width, height, width, h_im, w_im); + } + *data_col_ptr = val; + data_col_ptr += batch_size * height_col * width_col; + } + } + } +} + + +template +__global__ void deformable_col2im_gpu_kernel( + const int n, + const scalar_t* data_col, + const scalar_t* data_offset, + const int channels, + const int height, + const int width, + const int kernel_h, + const int kernel_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, + const int deformable_group, + const int height_col, + const int width_col, + scalar_t* grad_im) { + CUDA_KERNEL_LOOP(index, n) { + const int j = (index / width_col / height_col / batch_size) % kernel_w; + const int i = + (index / width_col / height_col / batch_size / kernel_w) % kernel_h; + const int c = + index / width_col / height_col / batch_size / kernel_w / kernel_h; + // compute the start and end of the output + + const int deformable_group_index = c / channel_per_deformable_group; + + int w_out = index % width_col; + int h_out = (index / width_col) % height_col; + int b = (index / width_col / height_col) % batch_size; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + + const scalar_t* data_offset_ptr = data_offset + + (b * deformable_group + deformable_group_index) * 2 * kernel_h * + kernel_w * height_col * width_col; + const int data_offset_h_ptr = + ((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out; + const int data_offset_w_ptr = + ((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t cur_inv_h_data = h_in + i * dilation_h + offset_h; + const scalar_t cur_inv_w_data = w_in + j * dilation_w + offset_w; + + const scalar_t cur_top_grad = data_col[index]; + const int cur_h = (int)cur_inv_h_data; + const int cur_w = (int)cur_inv_w_data; + for (int dy = -2; dy <= 2; dy++) { + for (int dx = -2; dx <= 2; dx++) { + if (cur_h + dy >= 0 && cur_h + dy < height && cur_w + dx >= 0 && + cur_w + dx < width && abs(cur_inv_h_data - (cur_h + dy)) < 1 && + abs(cur_inv_w_data - (cur_w + dx)) < 1) { + int cur_bottom_grad_pos = + ((b * channels + c) * height + cur_h + dy) * width + cur_w + dx; + scalar_t weight = get_gradient_weight( + cur_inv_h_data, + cur_inv_w_data, + cur_h + dy, + cur_w + dx, + height, + width); + atomicAdd(grad_im + cur_bottom_grad_pos, weight * cur_top_grad); + } + } + } + } +} + + +template +__global__ void deformable_col2im_coord_gpu_kernel( + const int n, + const scalar_t* data_col, + const scalar_t* data_im, + const scalar_t* data_offset, + const int channels, + const int height, + const int width, + const int kernel_h, + const int kernel_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, + const int offset_channels, + const int deformable_group, + const int height_col, + const int width_col, + scalar_t* grad_offset) { + CUDA_KERNEL_LOOP(index, n) { + scalar_t val = 0; + int w = index % width_col; + int h = (index / width_col) % height_col; + int c = (index / width_col / height_col) % offset_channels; + int b = (index / width_col / height_col) / offset_channels; + // compute the start and end of the output + + const int deformable_group_index = c / (2 * kernel_h * kernel_w); + const int col_step = kernel_h * kernel_w; + int cnt = 0; + const scalar_t* data_col_ptr = data_col + + deformable_group_index * channel_per_deformable_group * batch_size * + width_col * height_col; + const scalar_t* data_im_ptr = data_im + + (b * deformable_group + deformable_group_index) * + channel_per_deformable_group / kernel_h / kernel_w * height * width; + const scalar_t* data_offset_ptr = data_offset + + (b * deformable_group + deformable_group_index) * 2 * kernel_h * + kernel_w * height_col * width_col; + + const int offset_c = c - deformable_group_index * 2 * kernel_h * kernel_w; + + for (int col_c = (offset_c / 2); col_c < channel_per_deformable_group; + col_c += col_step) { + const int col_pos = + (((col_c * batch_size + b) * height_col) + h) * width_col + w; + const int bp_dir = offset_c % 2; + + int j = (col_pos / width_col / height_col / batch_size) % kernel_w; + int i = + (col_pos / width_col / height_col / batch_size / kernel_w) % kernel_h; + int w_out = col_pos % width_col; + int h_out = (col_pos / width_col) % height_col; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + const int data_offset_h_ptr = + (((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out); + const int data_offset_w_ptr = + (((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + + w_out); + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + scalar_t inv_h = h_in + i * dilation_h + offset_h; + scalar_t inv_w = w_in + j * dilation_w + offset_w; + if (inv_h <= -1 || inv_w <= -1 || inv_h >= height || inv_w >= width) { + inv_h = inv_w = -2; + } + const scalar_t weight = get_coordinate_weight( + inv_h, + inv_w, + height, + width, + data_im_ptr + cnt * height * width, + width, + bp_dir); + val += weight * data_col_ptr[col_pos]; + cnt += 1; + } + + grad_offset[index] = val; + } +} + + +namespace detectron2 { + +void deformable_im2col( + const at::Tensor data_im, + const at::Tensor data_offset, + const int channels, + const int height, + const int width, + const int ksize_h, + const int ksize_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int parallel_imgs, + const int deformable_group, + at::Tensor data_col) { + // num_axes should be smaller than block size + // todo: check parallel_imgs is correctly passed in + int height_col = + (height + 2 * pad_h - (dilation_h * (ksize_h - 1) + 1)) / stride_h + 1; + int width_col = + (width + 2 * pad_w - (dilation_w * (ksize_w - 1) + 1)) / stride_w + 1; + int num_kernels = channels * height_col * width_col * parallel_imgs; + int channel_per_deformable_group = channels / deformable_group; + + at::cuda::CUDAGuard device_guard(data_im.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + data_im.scalar_type(), "deformable_im2col_gpu", ([&] { + const scalar_t* data_im_ = data_im.data_ptr(); + const scalar_t* data_offset_ = data_offset.data_ptr(); + scalar_t* data_col_ = data_col.data_ptr(); + + deformable_im2col_gpu_kernel<<< + GET_BLOCKS(num_kernels), + CUDA_NUM_THREADS, + 0, + stream>>>( + num_kernels, + data_im_, + data_offset_, + height, + width, + ksize_h, + ksize_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + channel_per_deformable_group, + parallel_imgs, + channels, + deformable_group, + height_col, + width_col, + data_col_); + })); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("error in deformable_im2col: %s\n", cudaGetErrorString(err)); + } +} + + +void deformable_col2im( + const at::Tensor data_col, + const at::Tensor data_offset, + const int channels, + const int height, + const int width, + const int ksize_h, + const int ksize_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int parallel_imgs, + const int deformable_group, + at::Tensor grad_im) { + // todo: make sure parallel_imgs is passed in correctly + int height_col = + (height + 2 * pad_h - (dilation_h * (ksize_h - 1) + 1)) / stride_h + 1; + int width_col = + (width + 2 * pad_w - (dilation_w * (ksize_w - 1) + 1)) / stride_w + 1; + int num_kernels = + channels * ksize_h * ksize_w * height_col * width_col * parallel_imgs; + int channel_per_deformable_group = channels / deformable_group; + + at::cuda::CUDAGuard device_guard(data_col.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + data_col.scalar_type(), "deformable_col2im_gpu", ([&] { + const scalar_t* data_col_ = data_col.data_ptr(); + const scalar_t* data_offset_ = data_offset.data_ptr(); + scalar_t* grad_im_ = grad_im.data_ptr(); + + deformable_col2im_gpu_kernel<<< + GET_BLOCKS(num_kernels), + CUDA_NUM_THREADS, + 0, + stream>>>( + num_kernels, + data_col_, + data_offset_, + channels, + height, + width, + ksize_h, + ksize_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + channel_per_deformable_group, + parallel_imgs, + deformable_group, + height_col, + width_col, + grad_im_); + })); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf("error in deformable_col2im: %s\n", cudaGetErrorString(err)); + } +} + + +void deformable_col2im_coord( + const at::Tensor data_col, + const at::Tensor data_im, + const at::Tensor data_offset, + const int channels, + const int height, + const int width, + const int ksize_h, + const int ksize_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int parallel_imgs, + const int deformable_group, + at::Tensor grad_offset) { + int height_col = + (height + 2 * pad_h - (dilation_h * (ksize_h - 1) + 1)) / stride_h + 1; + int width_col = + (width + 2 * pad_w - (dilation_w * (ksize_w - 1) + 1)) / stride_w + 1; + int num_kernels = height_col * width_col * 2 * ksize_h * ksize_w * + deformable_group * parallel_imgs; + int channel_per_deformable_group = + channels * ksize_h * ksize_w / deformable_group; + + at::cuda::CUDAGuard device_guard(data_col.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + data_col.scalar_type(), "deformable_col2im_coord_gpu", ([&] { + const scalar_t* data_col_ = data_col.data_ptr(); + const scalar_t* data_im_ = data_im.data_ptr(); + const scalar_t* data_offset_ = data_offset.data_ptr(); + scalar_t* grad_offset_ = grad_offset.data_ptr(); + + deformable_col2im_coord_gpu_kernel<<< + GET_BLOCKS(num_kernels), + CUDA_NUM_THREADS, + 0, + stream>>>( + num_kernels, + data_col_, + data_im_, + data_offset_, + channels, + height, + width, + ksize_h, + ksize_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + channel_per_deformable_group, + parallel_imgs, + 2 * ksize_h * ksize_w * deformable_group, + deformable_group, + height_col, + width_col, + grad_offset_); + })); +} + +} // namespace detectron2 + + +template +__device__ scalar_t dmcn_im2col_bilinear( + const scalar_t* bottom_data, + const int data_width, + const int height, + const int width, + scalar_t h, + scalar_t w) { + int h_low = floor(h); + int w_low = floor(w); + int h_high = h_low + 1; + int w_high = w_low + 1; + + scalar_t lh = h - h_low; + scalar_t lw = w - w_low; + scalar_t hh = 1 - lh, hw = 1 - lw; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + v1 = bottom_data[h_low * data_width + w_low]; + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + v2 = bottom_data[h_low * data_width + w_high]; + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + v3 = bottom_data[h_high * data_width + w_low]; + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + v4 = bottom_data[h_high * data_width + w_high]; + + scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + +template +__device__ scalar_t dmcn_get_gradient_weight( + scalar_t argmax_h, + scalar_t argmax_w, + const int h, + const int w, + const int height, + const int width) { + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || + argmax_w >= width) { + // empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + if (h == argmax_h_low && w == argmax_w_low) + weight = (h + 1 - argmax_h) * (w + 1 - argmax_w); + if (h == argmax_h_low && w == argmax_w_high) + weight = (h + 1 - argmax_h) * (argmax_w + 1 - w); + if (h == argmax_h_high && w == argmax_w_low) + weight = (argmax_h + 1 - h) * (w + 1 - argmax_w); + if (h == argmax_h_high && w == argmax_w_high) + weight = (argmax_h + 1 - h) * (argmax_w + 1 - w); + return weight; +} + +template +__device__ scalar_t dmcn_get_coordinate_weight( + scalar_t argmax_h, + scalar_t argmax_w, + const int height, + const int width, + const scalar_t* im_data, + const int data_width, + const int bp_dir) { + if (argmax_h <= -1 || argmax_h >= height || argmax_w <= -1 || + argmax_w >= width) { + // empty + return 0; + } + + int argmax_h_low = floor(argmax_h); + int argmax_w_low = floor(argmax_w); + int argmax_h_high = argmax_h_low + 1; + int argmax_w_high = argmax_w_low + 1; + + scalar_t weight = 0; + + if (bp_dir == 0) { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_w_low + 1 - argmax_w) * + im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += -1 * (argmax_w - argmax_w_low) * + im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += (argmax_w_low + 1 - argmax_w) * + im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_w - argmax_w_low) * + im_data[argmax_h_high * data_width + argmax_w_high]; + } else if (bp_dir == 1) { + if (argmax_h_low >= 0 && argmax_w_low >= 0) + weight += -1 * (argmax_h_low + 1 - argmax_h) * + im_data[argmax_h_low * data_width + argmax_w_low]; + if (argmax_h_low >= 0 && argmax_w_high <= width - 1) + weight += (argmax_h_low + 1 - argmax_h) * + im_data[argmax_h_low * data_width + argmax_w_high]; + if (argmax_h_high <= height - 1 && argmax_w_low >= 0) + weight += -1 * (argmax_h - argmax_h_low) * + im_data[argmax_h_high * data_width + argmax_w_low]; + if (argmax_h_high <= height - 1 && argmax_w_high <= width - 1) + weight += (argmax_h - argmax_h_low) * + im_data[argmax_h_high * data_width + argmax_w_high]; + } + + return weight; +} + +template +__global__ void modulated_deformable_im2col_gpu_kernel( + const int n, + const scalar_t* data_im, + const scalar_t* data_offset, + const scalar_t* data_mask, + const int height, + const int width, + const int kernel_h, + const int kernel_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, + const int num_channels, + const int deformable_group, + const int height_col, + const int width_col, + scalar_t* data_col) { + CUDA_KERNEL_LOOP(index, n) { + // index index of output matrix + const int w_col = index % width_col; + const int h_col = (index / width_col) % height_col; + const int b_col = (index / width_col / height_col) % batch_size; + const int c_im = (index / width_col / height_col) / batch_size; + const int c_col = c_im * kernel_h * kernel_w; + + // compute deformable group index + const int deformable_group_index = c_im / channel_per_deformable_group; + + const int h_in = h_col * stride_h - pad_h; + const int w_in = w_col * stride_w - pad_w; + + scalar_t* data_col_ptr = data_col + + ((c_col * batch_size + b_col) * height_col + h_col) * width_col + w_col; + // const float* data_im_ptr = data_im + ((b_col * num_channels + c_im) * + // height + h_in) * width + w_in; + const scalar_t* data_im_ptr = + data_im + (b_col * num_channels + c_im) * height * width; + const scalar_t* data_offset_ptr = data_offset + + (b_col * deformable_group + deformable_group_index) * 2 * kernel_h * + kernel_w * height_col * width_col; + + const scalar_t* data_mask_ptr = data_mask + + (b_col * deformable_group + deformable_group_index) * kernel_h * + kernel_w * height_col * width_col; + + for (int i = 0; i < kernel_h; ++i) { + for (int j = 0; j < kernel_w; ++j) { + const int data_offset_h_ptr = + ((2 * (i * kernel_w + j)) * height_col + h_col) * width_col + w_col; + const int data_offset_w_ptr = + ((2 * (i * kernel_w + j) + 1) * height_col + h_col) * width_col + + w_col; + const int data_mask_hw_ptr = + ((i * kernel_w + j) * height_col + h_col) * width_col + w_col; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t mask = data_mask_ptr[data_mask_hw_ptr]; + scalar_t val = static_cast(0); + const scalar_t h_im = h_in + i * dilation_h + offset_h; + const scalar_t w_im = w_in + j * dilation_w + offset_w; + // if (h_im >= 0 && w_im >= 0 && h_im < height && w_im < width) { + if (h_im > -1 && w_im > -1 && h_im < height && w_im < width) { + // const float map_h = i * dilation_h + offset_h; + // const float map_w = j * dilation_w + offset_w; + // const int cur_height = height - h_in; + // const int cur_width = width - w_in; + // val = dmcn_im2col_bilinear(data_im_ptr, width, cur_height, + // cur_width, map_h, map_w); + val = dmcn_im2col_bilinear( + data_im_ptr, width, height, width, h_im, w_im); + } + *data_col_ptr = val * mask; + data_col_ptr += batch_size * height_col * width_col; + // data_col_ptr += height_col * width_col; + } + } + } +} + +template +__global__ void modulated_deformable_col2im_gpu_kernel( + const int n, + const scalar_t* data_col, + const scalar_t* data_offset, + const scalar_t* data_mask, + const int channels, + const int height, + const int width, + const int kernel_h, + const int kernel_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, + const int deformable_group, + const int height_col, + const int width_col, + scalar_t* grad_im) { + CUDA_KERNEL_LOOP(index, n) { + const int j = (index / width_col / height_col / batch_size) % kernel_w; + const int i = + (index / width_col / height_col / batch_size / kernel_w) % kernel_h; + const int c = + index / width_col / height_col / batch_size / kernel_w / kernel_h; + // compute the start and end of the output + + const int deformable_group_index = c / channel_per_deformable_group; + + int w_out = index % width_col; + int h_out = (index / width_col) % height_col; + int b = (index / width_col / height_col) % batch_size; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + + const scalar_t* data_offset_ptr = data_offset + + (b * deformable_group + deformable_group_index) * 2 * kernel_h * + kernel_w * height_col * width_col; + const scalar_t* data_mask_ptr = data_mask + + (b * deformable_group + deformable_group_index) * kernel_h * kernel_w * + height_col * width_col; + const int data_offset_h_ptr = + ((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out; + const int data_offset_w_ptr = + ((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + w_out; + const int data_mask_hw_ptr = + ((i * kernel_w + j) * height_col + h_out) * width_col + w_out; + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t mask = data_mask_ptr[data_mask_hw_ptr]; + const scalar_t cur_inv_h_data = h_in + i * dilation_h + offset_h; + const scalar_t cur_inv_w_data = w_in + j * dilation_w + offset_w; + + const scalar_t cur_top_grad = data_col[index] * mask; + const int cur_h = (int)cur_inv_h_data; + const int cur_w = (int)cur_inv_w_data; + for (int dy = -2; dy <= 2; dy++) { + for (int dx = -2; dx <= 2; dx++) { + if (cur_h + dy >= 0 && cur_h + dy < height && cur_w + dx >= 0 && + cur_w + dx < width && abs(cur_inv_h_data - (cur_h + dy)) < 1 && + abs(cur_inv_w_data - (cur_w + dx)) < 1) { + int cur_bottom_grad_pos = + ((b * channels + c) * height + cur_h + dy) * width + cur_w + dx; + scalar_t weight = dmcn_get_gradient_weight( + cur_inv_h_data, + cur_inv_w_data, + cur_h + dy, + cur_w + dx, + height, + width); + atomicAdd(grad_im + cur_bottom_grad_pos, weight * cur_top_grad); + } + } + } + } +} + +template +__global__ void modulated_deformable_col2im_coord_gpu_kernel( + const int n, + const scalar_t* data_col, + const scalar_t* data_im, + const scalar_t* data_offset, + const scalar_t* data_mask, + const int channels, + const int height, + const int width, + const int kernel_h, + const int kernel_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int channel_per_deformable_group, + const int batch_size, + const int offset_channels, + const int deformable_group, + const int height_col, + const int width_col, + scalar_t* grad_offset, + scalar_t* grad_mask) { + CUDA_KERNEL_LOOP(index, n) { + scalar_t val = 0, mval = 0; + int w = index % width_col; + int h = (index / width_col) % height_col; + int c = (index / width_col / height_col) % offset_channels; + int b = (index / width_col / height_col) / offset_channels; + // compute the start and end of the output + + const int deformable_group_index = c / (2 * kernel_h * kernel_w); + const int col_step = kernel_h * kernel_w; + int cnt = 0; + const scalar_t* data_col_ptr = data_col + + deformable_group_index * channel_per_deformable_group * batch_size * + width_col * height_col; + const scalar_t* data_im_ptr = data_im + + (b * deformable_group + deformable_group_index) * + channel_per_deformable_group / kernel_h / kernel_w * height * width; + const scalar_t* data_offset_ptr = data_offset + + (b * deformable_group + deformable_group_index) * 2 * kernel_h * + kernel_w * height_col * width_col; + const scalar_t* data_mask_ptr = data_mask + + (b * deformable_group + deformable_group_index) * kernel_h * kernel_w * + height_col * width_col; + + const int offset_c = c - deformable_group_index * 2 * kernel_h * kernel_w; + + for (int col_c = (offset_c / 2); col_c < channel_per_deformable_group; + col_c += col_step) { + const int col_pos = + (((col_c * batch_size + b) * height_col) + h) * width_col + w; + const int bp_dir = offset_c % 2; + + int j = (col_pos / width_col / height_col / batch_size) % kernel_w; + int i = + (col_pos / width_col / height_col / batch_size / kernel_w) % kernel_h; + int w_out = col_pos % width_col; + int h_out = (col_pos / width_col) % height_col; + int w_in = w_out * stride_w - pad_w; + int h_in = h_out * stride_h - pad_h; + const int data_offset_h_ptr = + (((2 * (i * kernel_w + j)) * height_col + h_out) * width_col + w_out); + const int data_offset_w_ptr = + (((2 * (i * kernel_w + j) + 1) * height_col + h_out) * width_col + + w_out); + const int data_mask_hw_ptr = + (((i * kernel_w + j) * height_col + h_out) * width_col + w_out); + const scalar_t offset_h = data_offset_ptr[data_offset_h_ptr]; + const scalar_t offset_w = data_offset_ptr[data_offset_w_ptr]; + const scalar_t mask = data_mask_ptr[data_mask_hw_ptr]; + scalar_t inv_h = h_in + i * dilation_h + offset_h; + scalar_t inv_w = w_in + j * dilation_w + offset_w; + if (inv_h <= -1 || inv_w <= -1 || inv_h >= height || inv_w >= width) { + inv_h = inv_w = -2; + } else { + mval += data_col_ptr[col_pos] * + dmcn_im2col_bilinear( + data_im_ptr + cnt * height * width, + width, + height, + width, + inv_h, + inv_w); + } + const scalar_t weight = dmcn_get_coordinate_weight( + inv_h, + inv_w, + height, + width, + data_im_ptr + cnt * height * width, + width, + bp_dir); + val += weight * data_col_ptr[col_pos] * mask; + cnt += 1; + } + // KERNEL_ASSIGN(grad_offset[index], offset_req, val); + grad_offset[index] = val; + if (offset_c % 2 == 0) + // KERNEL_ASSIGN(grad_mask[(((b * deformable_group + + // deformable_group_index) * kernel_h * kernel_w + offset_c / 2) * + // height_col + h) * width_col + w], mask_req, mval); + grad_mask + [(((b * deformable_group + deformable_group_index) * kernel_h * + kernel_w + + offset_c / 2) * + height_col + + h) * + width_col + + w] = mval; + } +} + + +namespace detectron2 { + +void modulated_deformable_im2col_cuda( + const at::Tensor data_im, + const at::Tensor data_offset, + const at::Tensor data_mask, + const int batch_size, + const int channels, + const int height_im, + const int width_im, + const int height_col, + const int width_col, + const int kernel_h, + const int kenerl_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int deformable_group, + at::Tensor data_col) { + // num_axes should be smaller than block size + const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = channels * batch_size * height_col * width_col; + + at::cuda::CUDAGuard device_guard(data_im.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + data_im.scalar_type(), "modulated_deformable_im2col_gpu", ([&] { + const scalar_t* data_im_ = data_im.data_ptr(); + const scalar_t* data_offset_ = data_offset.data_ptr(); + const scalar_t* data_mask_ = data_mask.data_ptr(); + scalar_t* data_col_ = data_col.data_ptr(); + + modulated_deformable_im2col_gpu_kernel<<< + GET_BLOCKS(num_kernels), + CUDA_NUM_THREADS, + 0, + stream>>>( + num_kernels, + data_im_, + data_offset_, + data_mask_, + height_im, + width_im, + kernel_h, + kenerl_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + channel_per_deformable_group, + batch_size, + channels, + deformable_group, + height_col, + width_col, + data_col_); + })); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf( + "error in modulated_deformable_im2col_cuda: %s\n", + cudaGetErrorString(err)); + } +} + +void modulated_deformable_col2im_cuda( + const at::Tensor data_col, + const at::Tensor data_offset, + const at::Tensor data_mask, + const int batch_size, + const int channels, + const int height_im, + const int width_im, + const int height_col, + const int width_col, + const int kernel_h, + const int kernel_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int deformable_group, + at::Tensor grad_im) { + const int channel_per_deformable_group = channels / deformable_group; + const int num_kernels = + channels * kernel_h * kernel_w * batch_size * height_col * width_col; + + at::cuda::CUDAGuard device_guard(data_col.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + data_col.scalar_type(), "modulated_deformable_col2im_gpu", ([&] { + const scalar_t* data_col_ = data_col.data_ptr(); + const scalar_t* data_offset_ = data_offset.data_ptr(); + const scalar_t* data_mask_ = data_mask.data_ptr(); + scalar_t* grad_im_ = grad_im.data_ptr(); + + modulated_deformable_col2im_gpu_kernel<<< + GET_BLOCKS(num_kernels), + CUDA_NUM_THREADS, + 0, + stream>>>( + num_kernels, + data_col_, + data_offset_, + data_mask_, + channels, + height_im, + width_im, + kernel_h, + kernel_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + channel_per_deformable_group, + batch_size, + deformable_group, + height_col, + width_col, + grad_im_); + })); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf( + "error in modulated_deformable_col2im_cuda: %s\n", + cudaGetErrorString(err)); + } +} + +void modulated_deformable_col2im_coord_cuda( + const at::Tensor data_col, + const at::Tensor data_im, + const at::Tensor data_offset, + const at::Tensor data_mask, + const int batch_size, + const int channels, + const int height_im, + const int width_im, + const int height_col, + const int width_col, + const int kernel_h, + const int kernel_w, + const int pad_h, + const int pad_w, + const int stride_h, + const int stride_w, + const int dilation_h, + const int dilation_w, + const int deformable_group, + at::Tensor grad_offset, + at::Tensor grad_mask) { + const int num_kernels = batch_size * height_col * width_col * 2 * kernel_h * + kernel_w * deformable_group; + const int channel_per_deformable_group = + channels * kernel_h * kernel_w / deformable_group; + + at::cuda::CUDAGuard device_guard(data_col.device()); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_FLOATING_TYPES_AND_HALF( + data_col.scalar_type(), "modulated_deformable_col2im_coord_gpu", ([&] { + const scalar_t* data_col_ = data_col.data_ptr(); + const scalar_t* data_im_ = data_im.data_ptr(); + const scalar_t* data_offset_ = data_offset.data_ptr(); + const scalar_t* data_mask_ = data_mask.data_ptr(); + scalar_t* grad_offset_ = grad_offset.data_ptr(); + scalar_t* grad_mask_ = grad_mask.data_ptr(); + + modulated_deformable_col2im_coord_gpu_kernel<<< + GET_BLOCKS(num_kernels), + CUDA_NUM_THREADS, + 0, + stream>>>( + num_kernels, + data_col_, + data_im_, + data_offset_, + data_mask_, + channels, + height_im, + width_im, + kernel_h, + kernel_w, + pad_h, + pad_w, + stride_h, + stride_w, + dilation_h, + dilation_w, + channel_per_deformable_group, + batch_size, + 2 * kernel_h * kernel_w * deformable_group, + deformable_group, + height_col, + width_col, + grad_offset_, + grad_mask_); + })); + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) { + printf( + "error in modulated_deformable_col2im_coord_cuda: %s\n", + cudaGetErrorString(err)); + } +} + +} // namespace detectron2 diff --git a/annotator/oneformer/detectron2/layers/csrc/nms_rotated/nms_rotated.h b/annotator/oneformer/detectron2/layers/csrc/nms_rotated/nms_rotated.h new file mode 100644 index 0000000000000000000000000000000000000000..12aca388e47b12dafd20999f2991a9d42f4b904b --- /dev/null +++ b/annotator/oneformer/detectron2/layers/csrc/nms_rotated/nms_rotated.h @@ -0,0 +1,39 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#pragma once +#include + +namespace detectron2 { + +at::Tensor nms_rotated_cpu( + const at::Tensor& dets, + const at::Tensor& scores, + const double iou_threshold); + +#if defined(WITH_CUDA) || defined(WITH_HIP) +at::Tensor nms_rotated_cuda( + const at::Tensor& dets, + const at::Tensor& scores, + const double iou_threshold); +#endif + +// Interface for Python +// inline is needed to prevent multiple function definitions when this header is +// included by different cpps +inline at::Tensor nms_rotated( + const at::Tensor& dets, + const at::Tensor& scores, + const double iou_threshold) { + assert(dets.device().is_cuda() == scores.device().is_cuda()); + if (dets.device().is_cuda()) { +#if defined(WITH_CUDA) || defined(WITH_HIP) + return nms_rotated_cuda( + dets.contiguous(), scores.contiguous(), iou_threshold); +#else + AT_ERROR("Detectron2 is not compiled with GPU support!"); +#endif + } + + return nms_rotated_cpu(dets.contiguous(), scores.contiguous(), iou_threshold); +} + +} // namespace detectron2 diff --git a/annotator/oneformer/detectron2/layers/csrc/nms_rotated/nms_rotated_cpu.cpp b/annotator/oneformer/detectron2/layers/csrc/nms_rotated/nms_rotated_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d7556e645b604aa83d86cc702b783fd8ecedffcc --- /dev/null +++ b/annotator/oneformer/detectron2/layers/csrc/nms_rotated/nms_rotated_cpu.cpp @@ -0,0 +1,75 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#include "../box_iou_rotated/box_iou_rotated_utils.h" +#include "nms_rotated.h" + +namespace detectron2 { + +template +at::Tensor nms_rotated_cpu_kernel( + const at::Tensor& dets, + const at::Tensor& scores, + const double iou_threshold) { + // nms_rotated_cpu_kernel is modified from torchvision's nms_cpu_kernel, + // however, the code in this function is much shorter because + // we delegate the IoU computation for rotated boxes to + // the single_box_iou_rotated function in box_iou_rotated_utils.h + AT_ASSERTM(dets.device().is_cpu(), "dets must be a CPU tensor"); + AT_ASSERTM(scores.device().is_cpu(), "scores must be a CPU tensor"); + AT_ASSERTM( + dets.scalar_type() == scores.scalar_type(), + "dets should have the same type as scores"); + + if (dets.numel() == 0) { + return at::empty({0}, dets.options().dtype(at::kLong)); + } + + auto order_t = std::get<1>(scores.sort(0, /* descending=*/true)); + + auto ndets = dets.size(0); + at::Tensor suppressed_t = at::zeros({ndets}, dets.options().dtype(at::kByte)); + at::Tensor keep_t = at::zeros({ndets}, dets.options().dtype(at::kLong)); + + auto suppressed = suppressed_t.data_ptr(); + auto keep = keep_t.data_ptr(); + auto order = order_t.data_ptr(); + + int64_t num_to_keep = 0; + + for (int64_t _i = 0; _i < ndets; _i++) { + auto i = order[_i]; + if (suppressed[i] == 1) { + continue; + } + + keep[num_to_keep++] = i; + + for (int64_t _j = _i + 1; _j < ndets; _j++) { + auto j = order[_j]; + if (suppressed[j] == 1) { + continue; + } + + auto ovr = single_box_iou_rotated( + dets[i].data_ptr(), dets[j].data_ptr()); + if (ovr >= iou_threshold) { + suppressed[j] = 1; + } + } + } + return keep_t.narrow(/*dim=*/0, /*start=*/0, /*length=*/num_to_keep); +} + +at::Tensor nms_rotated_cpu( + // input must be contiguous + const at::Tensor& dets, + const at::Tensor& scores, + const double iou_threshold) { + auto result = at::empty({0}, dets.options()); + + AT_DISPATCH_FLOATING_TYPES(dets.scalar_type(), "nms_rotated", [&] { + result = nms_rotated_cpu_kernel(dets, scores, iou_threshold); + }); + return result; +} + +} // namespace detectron2 diff --git a/annotator/oneformer/detectron2/layers/csrc/nms_rotated/nms_rotated_cuda.cu b/annotator/oneformer/detectron2/layers/csrc/nms_rotated/nms_rotated_cuda.cu new file mode 100644 index 0000000000000000000000000000000000000000..2a3db5c62e7a2da52ccf5bac980653c943d630fd --- /dev/null +++ b/annotator/oneformer/detectron2/layers/csrc/nms_rotated/nms_rotated_cuda.cu @@ -0,0 +1,145 @@ +// Copyright (c) Facebook, Inc. and its affiliates. +#include +#include +#include +#include +#ifdef WITH_CUDA +#include "../box_iou_rotated/box_iou_rotated_utils.h" +#endif +// TODO avoid this when pytorch supports "same directory" hipification +#ifdef WITH_HIP +#include "box_iou_rotated/box_iou_rotated_utils.h" +#endif + +using namespace detectron2; + +namespace { +int const threadsPerBlock = sizeof(unsigned long long) * 8; +} + +template +__global__ void nms_rotated_cuda_kernel( + const int n_boxes, + const double iou_threshold, + const T* dev_boxes, + unsigned long long* dev_mask) { + // nms_rotated_cuda_kernel is modified from torchvision's nms_cuda_kernel + + const int row_start = blockIdx.y; + const int col_start = blockIdx.x; + + // if (row_start > col_start) return; + + const int row_size = + min(n_boxes - row_start * threadsPerBlock, threadsPerBlock); + const int col_size = + min(n_boxes - col_start * threadsPerBlock, threadsPerBlock); + + // Compared to nms_cuda_kernel, where each box is represented with 4 values + // (x1, y1, x2, y2), each rotated box is represented with 5 values + // (x_center, y_center, width, height, angle_degrees) here. + __shared__ T block_boxes[threadsPerBlock * 5]; + if (threadIdx.x < col_size) { + block_boxes[threadIdx.x * 5 + 0] = + dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 0]; + block_boxes[threadIdx.x * 5 + 1] = + dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 1]; + block_boxes[threadIdx.x * 5 + 2] = + dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 2]; + block_boxes[threadIdx.x * 5 + 3] = + dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 3]; + block_boxes[threadIdx.x * 5 + 4] = + dev_boxes[(threadsPerBlock * col_start + threadIdx.x) * 5 + 4]; + } + __syncthreads(); + + if (threadIdx.x < row_size) { + const int cur_box_idx = threadsPerBlock * row_start + threadIdx.x; + const T* cur_box = dev_boxes + cur_box_idx * 5; + int i = 0; + unsigned long long t = 0; + int start = 0; + if (row_start == col_start) { + start = threadIdx.x + 1; + } + for (i = start; i < col_size; i++) { + // Instead of devIoU used by original horizontal nms, here + // we use the single_box_iou_rotated function from box_iou_rotated_utils.h + if (single_box_iou_rotated(cur_box, block_boxes + i * 5) > + iou_threshold) { + t |= 1ULL << i; + } + } + const int col_blocks = at::cuda::ATenCeilDiv(n_boxes, threadsPerBlock); + dev_mask[cur_box_idx * col_blocks + col_start] = t; + } +} + +namespace detectron2 { + +at::Tensor nms_rotated_cuda( + // input must be contiguous + const at::Tensor& dets, + const at::Tensor& scores, + double iou_threshold) { + // using scalar_t = float; + AT_ASSERTM(dets.is_cuda(), "dets must be a CUDA tensor"); + AT_ASSERTM(scores.is_cuda(), "scores must be a CUDA tensor"); + at::cuda::CUDAGuard device_guard(dets.device()); + + auto order_t = std::get<1>(scores.sort(0, /* descending=*/true)); + auto dets_sorted = dets.index_select(0, order_t); + + auto dets_num = dets.size(0); + + const int col_blocks = + at::cuda::ATenCeilDiv(static_cast(dets_num), threadsPerBlock); + + at::Tensor mask = + at::empty({dets_num * col_blocks}, dets.options().dtype(at::kLong)); + + dim3 blocks(col_blocks, col_blocks); + dim3 threads(threadsPerBlock); + cudaStream_t stream = at::cuda::getCurrentCUDAStream(); + + AT_DISPATCH_FLOATING_TYPES( + dets_sorted.scalar_type(), "nms_rotated_kernel_cuda", [&] { + nms_rotated_cuda_kernel<<>>( + dets_num, + iou_threshold, + dets_sorted.data_ptr(), + (unsigned long long*)mask.data_ptr()); + }); + + at::Tensor mask_cpu = mask.to(at::kCPU); + unsigned long long* mask_host = + (unsigned long long*)mask_cpu.data_ptr(); + + std::vector remv(col_blocks); + memset(&remv[0], 0, sizeof(unsigned long long) * col_blocks); + + at::Tensor keep = + at::empty({dets_num}, dets.options().dtype(at::kLong).device(at::kCPU)); + int64_t* keep_out = keep.data_ptr(); + + int num_to_keep = 0; + for (int i = 0; i < dets_num; i++) { + int nblock = i / threadsPerBlock; + int inblock = i % threadsPerBlock; + + if (!(remv[nblock] & (1ULL << inblock))) { + keep_out[num_to_keep++] = i; + unsigned long long* p = mask_host + i * col_blocks; + for (int j = nblock; j < col_blocks; j++) { + remv[j] |= p[j]; + } + } + } + + AT_CUDA_CHECK(cudaGetLastError()); + return order_t.index( + {keep.narrow(/*dim=*/0, /*start=*/0, /*length=*/num_to_keep) + .to(order_t.device(), keep.scalar_type())}); +} + +} // namespace detectron2 diff --git a/annotator/oneformer/detectron2/layers/csrc/vision.cpp b/annotator/oneformer/detectron2/layers/csrc/vision.cpp new file mode 100644 index 0000000000000000000000000000000000000000..c9a2cd4f20e6f58be1c5783d67c64232dd59b560 --- /dev/null +++ b/annotator/oneformer/detectron2/layers/csrc/vision.cpp @@ -0,0 +1,117 @@ +// Copyright (c) Facebook, Inc. and its affiliates. + +#include +#include "ROIAlignRotated/ROIAlignRotated.h" +#include "box_iou_rotated/box_iou_rotated.h" +#include "cocoeval/cocoeval.h" +#include "deformable/deform_conv.h" +#include "nms_rotated/nms_rotated.h" + +namespace detectron2 { + +#if defined(WITH_CUDA) || defined(WITH_HIP) +extern int get_cudart_version(); +#endif + +std::string get_cuda_version() { +#if defined(WITH_CUDA) || defined(WITH_HIP) + std::ostringstream oss; + +#if defined(WITH_CUDA) + oss << "CUDA "; +#else + oss << "HIP "; +#endif + + // copied from + // https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/cuda/detail/CUDAHooks.cpp#L231 + auto printCudaStyleVersion = [&](int v) { + oss << (v / 1000) << "." << (v / 10 % 100); + if (v % 10 != 0) { + oss << "." << (v % 10); + } + }; + printCudaStyleVersion(get_cudart_version()); + return oss.str(); +#else // neither CUDA nor HIP + return std::string("not available"); +#endif +} + +bool has_cuda() { +#if defined(WITH_CUDA) + return true; +#else + return false; +#endif +} + +// similar to +// https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/Version.cpp +std::string get_compiler_version() { + std::ostringstream ss; +#if defined(__GNUC__) +#ifndef __clang__ + +#if ((__GNUC__ <= 4) && (__GNUC_MINOR__ <= 8)) +#error "GCC >= 4.9 is required!" +#endif + + { ss << "GCC " << __GNUC__ << "." << __GNUC_MINOR__; } +#endif +#endif + +#if defined(__clang_major__) + { + ss << "clang " << __clang_major__ << "." << __clang_minor__ << "." + << __clang_patchlevel__; + } +#endif + +#if defined(_MSC_VER) + { ss << "MSVC " << _MSC_FULL_VER; } +#endif + return ss.str(); +} + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("get_compiler_version", &get_compiler_version, "get_compiler_version"); + m.def("get_cuda_version", &get_cuda_version, "get_cuda_version"); + m.def("has_cuda", &has_cuda, "has_cuda"); + + m.def("deform_conv_forward", &deform_conv_forward, "deform_conv_forward"); + m.def( + "deform_conv_backward_input", + &deform_conv_backward_input, + "deform_conv_backward_input"); + m.def( + "deform_conv_backward_filter", + &deform_conv_backward_filter, + "deform_conv_backward_filter"); + m.def( + "modulated_deform_conv_forward", + &modulated_deform_conv_forward, + "modulated_deform_conv_forward"); + m.def( + "modulated_deform_conv_backward", + &modulated_deform_conv_backward, + "modulated_deform_conv_backward"); + + m.def("COCOevalAccumulate", &COCOeval::Accumulate, "COCOeval::Accumulate"); + m.def( + "COCOevalEvaluateImages", + &COCOeval::EvaluateImages, + "COCOeval::EvaluateImages"); + pybind11::class_(m, "InstanceAnnotation") + .def(pybind11::init()); + pybind11::class_(m, "ImageEvaluation") + .def(pybind11::init<>()); +} + +TORCH_LIBRARY(detectron2, m) { + m.def("nms_rotated", &nms_rotated); + m.def("box_iou_rotated", &box_iou_rotated); + m.def("roi_align_rotated_forward", &ROIAlignRotated_forward); + m.def("roi_align_rotated_backward", &ROIAlignRotated_backward); +} +} // namespace detectron2 diff --git a/annotator/oneformer/detectron2/layers/deform_conv.py b/annotator/oneformer/detectron2/layers/deform_conv.py new file mode 100644 index 0000000000000000000000000000000000000000..49547238bcc67775bbeb97184467c3c8eee8df60 --- /dev/null +++ b/annotator/oneformer/detectron2/layers/deform_conv.py @@ -0,0 +1,514 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import math +from functools import lru_cache +import torch +from torch import nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.utils import _pair +from torchvision.ops import deform_conv2d + +from annotator.oneformer.detectron2.utils.develop import create_dummy_class, create_dummy_func + +from .wrappers import _NewEmptyTensorOp + + +class _DeformConv(Function): + @staticmethod + def forward( + ctx, + input, + offset, + weight, + stride=1, + padding=0, + dilation=1, + groups=1, + deformable_groups=1, + im2col_step=64, + ): + if input is not None and input.dim() != 4: + raise ValueError( + "Expected 4D tensor as input, got {}D tensor instead.".format(input.dim()) + ) + ctx.stride = _pair(stride) + ctx.padding = _pair(padding) + ctx.dilation = _pair(dilation) + ctx.groups = groups + ctx.deformable_groups = deformable_groups + ctx.im2col_step = im2col_step + + ctx.save_for_backward(input, offset, weight) + + output = input.new_empty( + _DeformConv._output_size(input, weight, ctx.padding, ctx.dilation, ctx.stride) + ) + + ctx.bufs_ = [input.new_empty(0), input.new_empty(0)] # columns, ones + + if not input.is_cuda: + # TODO: let torchvision support full features of our deformconv. + if deformable_groups != 1: + raise NotImplementedError( + "Deformable Conv with deformable_groups != 1 is not supported on CPUs!" + ) + return deform_conv2d( + input, offset, weight, stride=stride, padding=padding, dilation=dilation + ) + else: + cur_im2col_step = _DeformConv._cal_im2col_step(input.shape[0], ctx.im2col_step) + assert (input.shape[0] % cur_im2col_step) == 0, "im2col step must divide batchsize" + + _C.deform_conv_forward( + input, + weight, + offset, + output, + ctx.bufs_[0], + ctx.bufs_[1], + weight.size(3), + weight.size(2), + ctx.stride[1], + ctx.stride[0], + ctx.padding[1], + ctx.padding[0], + ctx.dilation[1], + ctx.dilation[0], + ctx.groups, + ctx.deformable_groups, + cur_im2col_step, + ) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + input, offset, weight = ctx.saved_tensors + + grad_input = grad_offset = grad_weight = None + + if not grad_output.is_cuda: + raise NotImplementedError("Deformable Conv is not supported on CPUs!") + else: + cur_im2col_step = _DeformConv._cal_im2col_step(input.shape[0], ctx.im2col_step) + assert (input.shape[0] % cur_im2col_step) == 0, "im2col step must divide batchsize" + + if ctx.needs_input_grad[0] or ctx.needs_input_grad[1]: + grad_input = torch.zeros_like(input) + grad_offset = torch.zeros_like(offset) + _C.deform_conv_backward_input( + input, + offset, + grad_output, + grad_input, + grad_offset, + weight, + ctx.bufs_[0], + weight.size(3), + weight.size(2), + ctx.stride[1], + ctx.stride[0], + ctx.padding[1], + ctx.padding[0], + ctx.dilation[1], + ctx.dilation[0], + ctx.groups, + ctx.deformable_groups, + cur_im2col_step, + ) + + if ctx.needs_input_grad[2]: + grad_weight = torch.zeros_like(weight) + _C.deform_conv_backward_filter( + input, + offset, + grad_output, + grad_weight, + ctx.bufs_[0], + ctx.bufs_[1], + weight.size(3), + weight.size(2), + ctx.stride[1], + ctx.stride[0], + ctx.padding[1], + ctx.padding[0], + ctx.dilation[1], + ctx.dilation[0], + ctx.groups, + ctx.deformable_groups, + 1, + cur_im2col_step, + ) + + return grad_input, grad_offset, grad_weight, None, None, None, None, None, None + + @staticmethod + def _output_size(input, weight, padding, dilation, stride): + channels = weight.size(0) + output_size = (input.size(0), channels) + for d in range(input.dim() - 2): + in_size = input.size(d + 2) + pad = padding[d] + kernel = dilation[d] * (weight.size(d + 2) - 1) + 1 + stride_ = stride[d] + output_size += ((in_size + (2 * pad) - kernel) // stride_ + 1,) + if not all(map(lambda s: s > 0, output_size)): + raise ValueError( + "convolution input is too small (output would be {})".format( + "x".join(map(str, output_size)) + ) + ) + return output_size + + @staticmethod + @lru_cache(maxsize=128) + def _cal_im2col_step(input_size, default_size): + """ + Calculate proper im2col step size, which should be divisible by input_size and not larger + than prefer_size. Meanwhile the step size should be as large as possible to be more + efficient. So we choose the largest one among all divisors of input_size which are smaller + than prefer_size. + :param input_size: input batch size . + :param default_size: default preferred im2col step size. + :return: the largest proper step size. + """ + if input_size <= default_size: + return input_size + best_step = 1 + for step in range(2, min(int(math.sqrt(input_size)) + 1, default_size)): + if input_size % step == 0: + if input_size // step <= default_size: + return input_size // step + best_step = step + + return best_step + + +class _ModulatedDeformConv(Function): + @staticmethod + def forward( + ctx, + input, + offset, + mask, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1, + deformable_groups=1, + ): + ctx.stride = stride + ctx.padding = padding + ctx.dilation = dilation + ctx.groups = groups + ctx.deformable_groups = deformable_groups + ctx.with_bias = bias is not None + if not ctx.with_bias: + bias = input.new_empty(1) # fake tensor + if not input.is_cuda: + raise NotImplementedError("Deformable Conv is not supported on CPUs!") + if ( + weight.requires_grad + or mask.requires_grad + or offset.requires_grad + or input.requires_grad + ): + ctx.save_for_backward(input, offset, mask, weight, bias) + output = input.new_empty(_ModulatedDeformConv._infer_shape(ctx, input, weight)) + ctx._bufs = [input.new_empty(0), input.new_empty(0)] + _C.modulated_deform_conv_forward( + input, + weight, + bias, + ctx._bufs[0], + offset, + mask, + output, + ctx._bufs[1], + weight.shape[2], + weight.shape[3], + ctx.stride, + ctx.stride, + ctx.padding, + ctx.padding, + ctx.dilation, + ctx.dilation, + ctx.groups, + ctx.deformable_groups, + ctx.with_bias, + ) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + if not grad_output.is_cuda: + raise NotImplementedError("Deformable Conv is not supported on CPUs!") + input, offset, mask, weight, bias = ctx.saved_tensors + grad_input = torch.zeros_like(input) + grad_offset = torch.zeros_like(offset) + grad_mask = torch.zeros_like(mask) + grad_weight = torch.zeros_like(weight) + grad_bias = torch.zeros_like(bias) + _C.modulated_deform_conv_backward( + input, + weight, + bias, + ctx._bufs[0], + offset, + mask, + ctx._bufs[1], + grad_input, + grad_weight, + grad_bias, + grad_offset, + grad_mask, + grad_output, + weight.shape[2], + weight.shape[3], + ctx.stride, + ctx.stride, + ctx.padding, + ctx.padding, + ctx.dilation, + ctx.dilation, + ctx.groups, + ctx.deformable_groups, + ctx.with_bias, + ) + if not ctx.with_bias: + grad_bias = None + + return ( + grad_input, + grad_offset, + grad_mask, + grad_weight, + grad_bias, + None, + None, + None, + None, + None, + ) + + @staticmethod + def _infer_shape(ctx, input, weight): + n = input.size(0) + channels_out = weight.size(0) + height, width = input.shape[2:4] + kernel_h, kernel_w = weight.shape[2:4] + height_out = ( + height + 2 * ctx.padding - (ctx.dilation * (kernel_h - 1) + 1) + ) // ctx.stride + 1 + width_out = ( + width + 2 * ctx.padding - (ctx.dilation * (kernel_w - 1) + 1) + ) // ctx.stride + 1 + return n, channels_out, height_out, width_out + + +deform_conv = _DeformConv.apply +modulated_deform_conv = _ModulatedDeformConv.apply + + +class DeformConv(nn.Module): + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + deformable_groups=1, + bias=False, + norm=None, + activation=None, + ): + """ + Deformable convolution from :paper:`deformconv`. + + Arguments are similar to :class:`Conv2D`. Extra arguments: + + Args: + deformable_groups (int): number of groups used in deformable convolution. + norm (nn.Module, optional): a normalization layer + activation (callable(Tensor) -> Tensor): a callable activation function + """ + super(DeformConv, self).__init__() + + assert not bias + assert in_channels % groups == 0, "in_channels {} cannot be divisible by groups {}".format( + in_channels, groups + ) + assert ( + out_channels % groups == 0 + ), "out_channels {} cannot be divisible by groups {}".format(out_channels, groups) + + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = _pair(stride) + self.padding = _pair(padding) + self.dilation = _pair(dilation) + self.groups = groups + self.deformable_groups = deformable_groups + self.norm = norm + self.activation = activation + + self.weight = nn.Parameter( + torch.Tensor(out_channels, in_channels // self.groups, *self.kernel_size) + ) + self.bias = None + + nn.init.kaiming_uniform_(self.weight, nonlinearity="relu") + + def forward(self, x, offset): + if x.numel() == 0: + # When input is empty, we want to return a empty tensor with "correct" shape, + # So that the following operations will not panic + # if they check for the shape of the tensor. + # This computes the height and width of the output tensor + output_shape = [ + (i + 2 * p - (di * (k - 1) + 1)) // s + 1 + for i, p, di, k, s in zip( + x.shape[-2:], self.padding, self.dilation, self.kernel_size, self.stride + ) + ] + output_shape = [x.shape[0], self.weight.shape[0]] + output_shape + return _NewEmptyTensorOp.apply(x, output_shape) + + x = deform_conv( + x, + offset, + self.weight, + self.stride, + self.padding, + self.dilation, + self.groups, + self.deformable_groups, + ) + if self.norm is not None: + x = self.norm(x) + if self.activation is not None: + x = self.activation(x) + return x + + def extra_repr(self): + tmpstr = "in_channels=" + str(self.in_channels) + tmpstr += ", out_channels=" + str(self.out_channels) + tmpstr += ", kernel_size=" + str(self.kernel_size) + tmpstr += ", stride=" + str(self.stride) + tmpstr += ", padding=" + str(self.padding) + tmpstr += ", dilation=" + str(self.dilation) + tmpstr += ", groups=" + str(self.groups) + tmpstr += ", deformable_groups=" + str(self.deformable_groups) + tmpstr += ", bias=False" + return tmpstr + + +class ModulatedDeformConv(nn.Module): + def __init__( + self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + deformable_groups=1, + bias=True, + norm=None, + activation=None, + ): + """ + Modulated deformable convolution from :paper:`deformconv2`. + + Arguments are similar to :class:`Conv2D`. Extra arguments: + + Args: + deformable_groups (int): number of groups used in deformable convolution. + norm (nn.Module, optional): a normalization layer + activation (callable(Tensor) -> Tensor): a callable activation function + """ + super(ModulatedDeformConv, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = stride + self.padding = padding + self.dilation = dilation + self.groups = groups + self.deformable_groups = deformable_groups + self.with_bias = bias + self.norm = norm + self.activation = activation + + self.weight = nn.Parameter( + torch.Tensor(out_channels, in_channels // groups, *self.kernel_size) + ) + if bias: + self.bias = nn.Parameter(torch.Tensor(out_channels)) + else: + self.bias = None + + nn.init.kaiming_uniform_(self.weight, nonlinearity="relu") + if self.bias is not None: + nn.init.constant_(self.bias, 0) + + def forward(self, x, offset, mask): + if x.numel() == 0: + output_shape = [ + (i + 2 * p - (di * (k - 1) + 1)) // s + 1 + for i, p, di, k, s in zip( + x.shape[-2:], self.padding, self.dilation, self.kernel_size, self.stride + ) + ] + output_shape = [x.shape[0], self.weight.shape[0]] + output_shape + return _NewEmptyTensorOp.apply(x, output_shape) + + x = modulated_deform_conv( + x, + offset, + mask, + self.weight, + self.bias, + self.stride, + self.padding, + self.dilation, + self.groups, + self.deformable_groups, + ) + if self.norm is not None: + x = self.norm(x) + if self.activation is not None: + x = self.activation(x) + return x + + def extra_repr(self): + tmpstr = "in_channels=" + str(self.in_channels) + tmpstr += ", out_channels=" + str(self.out_channels) + tmpstr += ", kernel_size=" + str(self.kernel_size) + tmpstr += ", stride=" + str(self.stride) + tmpstr += ", padding=" + str(self.padding) + tmpstr += ", dilation=" + str(self.dilation) + tmpstr += ", groups=" + str(self.groups) + tmpstr += ", deformable_groups=" + str(self.deformable_groups) + tmpstr += ", bias=" + str(self.with_bias) + return tmpstr + + +try: + from annotator.oneformer.detectron2 import _C +except ImportError: + # TODO: register ops natively so there is no need to import _C. + _msg = "detectron2 is not compiled successfully, please build following the instructions!" + _args = ("detectron2._C", _msg) + DeformConv = create_dummy_class("DeformConv", *_args) + ModulatedDeformConv = create_dummy_class("ModulatedDeformConv", *_args) + deform_conv = create_dummy_func("deform_conv", *_args) + modulated_deform_conv = create_dummy_func("modulated_deform_conv", *_args) diff --git a/annotator/oneformer/detectron2/layers/losses.py b/annotator/oneformer/detectron2/layers/losses.py new file mode 100644 index 0000000000000000000000000000000000000000..850a852a2f0986d4d1ce89a526d96db42c76e44f --- /dev/null +++ b/annotator/oneformer/detectron2/layers/losses.py @@ -0,0 +1,133 @@ +import math +import torch + + +def diou_loss( + boxes1: torch.Tensor, + boxes2: torch.Tensor, + reduction: str = "none", + eps: float = 1e-7, +) -> torch.Tensor: + """ + Distance Intersection over Union Loss (Zhaohui Zheng et. al) + https://arxiv.org/abs/1911.08287 + Args: + boxes1, boxes2 (Tensor): box locations in XYXY format, shape (N, 4) or (4,). + reduction: 'none' | 'mean' | 'sum' + 'none': No reduction will be applied to the output. + 'mean': The output will be averaged. + 'sum': The output will be summed. + eps (float): small number to prevent division by zero + """ + + x1, y1, x2, y2 = boxes1.unbind(dim=-1) + x1g, y1g, x2g, y2g = boxes2.unbind(dim=-1) + + # TODO: use torch._assert_async() when pytorch 1.8 support is dropped + assert (x2 >= x1).all(), "bad box: x1 larger than x2" + assert (y2 >= y1).all(), "bad box: y1 larger than y2" + + # Intersection keypoints + xkis1 = torch.max(x1, x1g) + ykis1 = torch.max(y1, y1g) + xkis2 = torch.min(x2, x2g) + ykis2 = torch.min(y2, y2g) + + intsct = torch.zeros_like(x1) + mask = (ykis2 > ykis1) & (xkis2 > xkis1) + intsct[mask] = (xkis2[mask] - xkis1[mask]) * (ykis2[mask] - ykis1[mask]) + union = (x2 - x1) * (y2 - y1) + (x2g - x1g) * (y2g - y1g) - intsct + eps + iou = intsct / union + + # smallest enclosing box + xc1 = torch.min(x1, x1g) + yc1 = torch.min(y1, y1g) + xc2 = torch.max(x2, x2g) + yc2 = torch.max(y2, y2g) + diag_len = ((xc2 - xc1) ** 2) + ((yc2 - yc1) ** 2) + eps + + # centers of boxes + x_p = (x2 + x1) / 2 + y_p = (y2 + y1) / 2 + x_g = (x1g + x2g) / 2 + y_g = (y1g + y2g) / 2 + distance = ((x_p - x_g) ** 2) + ((y_p - y_g) ** 2) + + # Eqn. (7) + loss = 1 - iou + (distance / diag_len) + if reduction == "mean": + loss = loss.mean() if loss.numel() > 0 else 0.0 * loss.sum() + elif reduction == "sum": + loss = loss.sum() + + return loss + + +def ciou_loss( + boxes1: torch.Tensor, + boxes2: torch.Tensor, + reduction: str = "none", + eps: float = 1e-7, +) -> torch.Tensor: + """ + Complete Intersection over Union Loss (Zhaohui Zheng et. al) + https://arxiv.org/abs/1911.08287 + Args: + boxes1, boxes2 (Tensor): box locations in XYXY format, shape (N, 4) or (4,). + reduction: 'none' | 'mean' | 'sum' + 'none': No reduction will be applied to the output. + 'mean': The output will be averaged. + 'sum': The output will be summed. + eps (float): small number to prevent division by zero + """ + + x1, y1, x2, y2 = boxes1.unbind(dim=-1) + x1g, y1g, x2g, y2g = boxes2.unbind(dim=-1) + + # TODO: use torch._assert_async() when pytorch 1.8 support is dropped + assert (x2 >= x1).all(), "bad box: x1 larger than x2" + assert (y2 >= y1).all(), "bad box: y1 larger than y2" + + # Intersection keypoints + xkis1 = torch.max(x1, x1g) + ykis1 = torch.max(y1, y1g) + xkis2 = torch.min(x2, x2g) + ykis2 = torch.min(y2, y2g) + + intsct = torch.zeros_like(x1) + mask = (ykis2 > ykis1) & (xkis2 > xkis1) + intsct[mask] = (xkis2[mask] - xkis1[mask]) * (ykis2[mask] - ykis1[mask]) + union = (x2 - x1) * (y2 - y1) + (x2g - x1g) * (y2g - y1g) - intsct + eps + iou = intsct / union + + # smallest enclosing box + xc1 = torch.min(x1, x1g) + yc1 = torch.min(y1, y1g) + xc2 = torch.max(x2, x2g) + yc2 = torch.max(y2, y2g) + diag_len = ((xc2 - xc1) ** 2) + ((yc2 - yc1) ** 2) + eps + + # centers of boxes + x_p = (x2 + x1) / 2 + y_p = (y2 + y1) / 2 + x_g = (x1g + x2g) / 2 + y_g = (y1g + y2g) / 2 + distance = ((x_p - x_g) ** 2) + ((y_p - y_g) ** 2) + + # width and height of boxes + w_pred = x2 - x1 + h_pred = y2 - y1 + w_gt = x2g - x1g + h_gt = y2g - y1g + v = (4 / (math.pi**2)) * torch.pow((torch.atan(w_gt / h_gt) - torch.atan(w_pred / h_pred)), 2) + with torch.no_grad(): + alpha = v / (1 - iou + v + eps) + + # Eqn. (10) + loss = 1 - iou + (distance / diag_len) + alpha * v + if reduction == "mean": + loss = loss.mean() if loss.numel() > 0 else 0.0 * loss.sum() + elif reduction == "sum": + loss = loss.sum() + + return loss diff --git a/annotator/oneformer/detectron2/layers/mask_ops.py b/annotator/oneformer/detectron2/layers/mask_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..990d04abbb120e40fe07a21d024dfead471bc998 --- /dev/null +++ b/annotator/oneformer/detectron2/layers/mask_ops.py @@ -0,0 +1,275 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import numpy as np +from typing import Tuple +import torch +from PIL import Image +from torch.nn import functional as F + +__all__ = ["paste_masks_in_image"] + + +BYTES_PER_FLOAT = 4 +# TODO: This memory limit may be too much or too little. It would be better to +# determine it based on available resources. +GPU_MEM_LIMIT = 1024**3 # 1 GB memory limit + + +def _do_paste_mask(masks, boxes, img_h: int, img_w: int, skip_empty: bool = True): + """ + Args: + masks: N, 1, H, W + boxes: N, 4 + img_h, img_w (int): + skip_empty (bool): only paste masks within the region that + tightly bound all boxes, and returns the results this region only. + An important optimization for CPU. + + Returns: + if skip_empty == False, a mask of shape (N, img_h, img_w) + if skip_empty == True, a mask of shape (N, h', w'), and the slice + object for the corresponding region. + """ + # On GPU, paste all masks together (up to chunk size) + # by using the entire image to sample the masks + # Compared to pasting them one by one, + # this has more operations but is faster on COCO-scale dataset. + device = masks.device + + if skip_empty and not torch.jit.is_scripting(): + x0_int, y0_int = torch.clamp(boxes.min(dim=0).values.floor()[:2] - 1, min=0).to( + dtype=torch.int32 + ) + x1_int = torch.clamp(boxes[:, 2].max().ceil() + 1, max=img_w).to(dtype=torch.int32) + y1_int = torch.clamp(boxes[:, 3].max().ceil() + 1, max=img_h).to(dtype=torch.int32) + else: + x0_int, y0_int = 0, 0 + x1_int, y1_int = img_w, img_h + x0, y0, x1, y1 = torch.split(boxes, 1, dim=1) # each is Nx1 + + N = masks.shape[0] + + img_y = torch.arange(y0_int, y1_int, device=device, dtype=torch.float32) + 0.5 + img_x = torch.arange(x0_int, x1_int, device=device, dtype=torch.float32) + 0.5 + img_y = (img_y - y0) / (y1 - y0) * 2 - 1 + img_x = (img_x - x0) / (x1 - x0) * 2 - 1 + # img_x, img_y have shapes (N, w), (N, h) + + gx = img_x[:, None, :].expand(N, img_y.size(1), img_x.size(1)) + gy = img_y[:, :, None].expand(N, img_y.size(1), img_x.size(1)) + grid = torch.stack([gx, gy], dim=3) + + if not torch.jit.is_scripting(): + if not masks.dtype.is_floating_point: + masks = masks.float() + img_masks = F.grid_sample(masks, grid.to(masks.dtype), align_corners=False) + + if skip_empty and not torch.jit.is_scripting(): + return img_masks[:, 0], (slice(y0_int, y1_int), slice(x0_int, x1_int)) + else: + return img_masks[:, 0], () + + +# Annotate boxes as Tensor (but not Boxes) in order to use scripting +@torch.jit.script_if_tracing +def paste_masks_in_image( + masks: torch.Tensor, boxes: torch.Tensor, image_shape: Tuple[int, int], threshold: float = 0.5 +): + """ + Paste a set of masks that are of a fixed resolution (e.g., 28 x 28) into an image. + The location, height, and width for pasting each mask is determined by their + corresponding bounding boxes in boxes. + + Note: + This is a complicated but more accurate implementation. In actual deployment, it is + often enough to use a faster but less accurate implementation. + See :func:`paste_mask_in_image_old` in this file for an alternative implementation. + + Args: + masks (tensor): Tensor of shape (Bimg, Hmask, Wmask), where Bimg is the number of + detected object instances in the image and Hmask, Wmask are the mask width and mask + height of the predicted mask (e.g., Hmask = Wmask = 28). Values are in [0, 1]. + boxes (Boxes or Tensor): A Boxes of length Bimg or Tensor of shape (Bimg, 4). + boxes[i] and masks[i] correspond to the same object instance. + image_shape (tuple): height, width + threshold (float): A threshold in [0, 1] for converting the (soft) masks to + binary masks. + + Returns: + img_masks (Tensor): A tensor of shape (Bimg, Himage, Wimage), where Bimg is the + number of detected object instances and Himage, Wimage are the image width + and height. img_masks[i] is a binary mask for object instance i. + """ + + assert masks.shape[-1] == masks.shape[-2], "Only square mask predictions are supported" + N = len(masks) + if N == 0: + return masks.new_empty((0,) + image_shape, dtype=torch.uint8) + if not isinstance(boxes, torch.Tensor): + boxes = boxes.tensor + device = boxes.device + assert len(boxes) == N, boxes.shape + + img_h, img_w = image_shape + + # The actual implementation split the input into chunks, + # and paste them chunk by chunk. + if device.type == "cpu" or torch.jit.is_scripting(): + # CPU is most efficient when they are pasted one by one with skip_empty=True + # so that it performs minimal number of operations. + num_chunks = N + else: + # GPU benefits from parallelism for larger chunks, but may have memory issue + # int(img_h) because shape may be tensors in tracing + num_chunks = int(np.ceil(N * int(img_h) * int(img_w) * BYTES_PER_FLOAT / GPU_MEM_LIMIT)) + assert ( + num_chunks <= N + ), "Default GPU_MEM_LIMIT in mask_ops.py is too small; try increasing it" + chunks = torch.chunk(torch.arange(N, device=device), num_chunks) + + img_masks = torch.zeros( + N, img_h, img_w, device=device, dtype=torch.bool if threshold >= 0 else torch.uint8 + ) + for inds in chunks: + masks_chunk, spatial_inds = _do_paste_mask( + masks[inds, None, :, :], boxes[inds], img_h, img_w, skip_empty=device.type == "cpu" + ) + + if threshold >= 0: + masks_chunk = (masks_chunk >= threshold).to(dtype=torch.bool) + else: + # for visualization and debugging + masks_chunk = (masks_chunk * 255).to(dtype=torch.uint8) + + if torch.jit.is_scripting(): # Scripting does not use the optimized codepath + img_masks[inds] = masks_chunk + else: + img_masks[(inds,) + spatial_inds] = masks_chunk + return img_masks + + +# The below are the original paste function (from Detectron1) which has +# larger quantization error. +# It is faster on CPU, while the aligned one is faster on GPU thanks to grid_sample. + + +def paste_mask_in_image_old(mask, box, img_h, img_w, threshold): + """ + Paste a single mask in an image. + This is a per-box implementation of :func:`paste_masks_in_image`. + This function has larger quantization error due to incorrect pixel + modeling and is not used any more. + + Args: + mask (Tensor): A tensor of shape (Hmask, Wmask) storing the mask of a single + object instance. Values are in [0, 1]. + box (Tensor): A tensor of shape (4, ) storing the x0, y0, x1, y1 box corners + of the object instance. + img_h, img_w (int): Image height and width. + threshold (float): Mask binarization threshold in [0, 1]. + + Returns: + im_mask (Tensor): + The resized and binarized object mask pasted into the original + image plane (a tensor of shape (img_h, img_w)). + """ + # Conversion from continuous box coordinates to discrete pixel coordinates + # via truncation (cast to int32). This determines which pixels to paste the + # mask onto. + box = box.to(dtype=torch.int32) # Continuous to discrete coordinate conversion + # An example (1D) box with continuous coordinates (x0=0.7, x1=4.3) will map to + # a discrete coordinates (x0=0, x1=4). Note that box is mapped to 5 = x1 - x0 + 1 + # pixels (not x1 - x0 pixels). + samples_w = box[2] - box[0] + 1 # Number of pixel samples, *not* geometric width + samples_h = box[3] - box[1] + 1 # Number of pixel samples, *not* geometric height + + # Resample the mask from it's original grid to the new samples_w x samples_h grid + mask = Image.fromarray(mask.cpu().numpy()) + mask = mask.resize((samples_w, samples_h), resample=Image.BILINEAR) + mask = np.array(mask, copy=False) + + if threshold >= 0: + mask = np.array(mask > threshold, dtype=np.uint8) + mask = torch.from_numpy(mask) + else: + # for visualization and debugging, we also + # allow it to return an unmodified mask + mask = torch.from_numpy(mask * 255).to(torch.uint8) + + im_mask = torch.zeros((img_h, img_w), dtype=torch.uint8) + x_0 = max(box[0], 0) + x_1 = min(box[2] + 1, img_w) + y_0 = max(box[1], 0) + y_1 = min(box[3] + 1, img_h) + + im_mask[y_0:y_1, x_0:x_1] = mask[ + (y_0 - box[1]) : (y_1 - box[1]), (x_0 - box[0]) : (x_1 - box[0]) + ] + return im_mask + + +# Our pixel modeling requires extrapolation for any continuous +# coordinate < 0.5 or > length - 0.5. When sampling pixels on the masks, +# we would like this extrapolation to be an interpolation between boundary values and zero, +# instead of using absolute zero or boundary values. +# Therefore `paste_mask_in_image_old` is often used with zero padding around the masks like this: +# masks, scale = pad_masks(masks[:, 0, :, :], 1) +# boxes = scale_boxes(boxes.tensor, scale) + + +def pad_masks(masks, padding): + """ + Args: + masks (tensor): A tensor of shape (B, M, M) representing B masks. + padding (int): Number of cells to pad on all sides. + + Returns: + The padded masks and the scale factor of the padding size / original size. + """ + B = masks.shape[0] + M = masks.shape[-1] + pad2 = 2 * padding + scale = float(M + pad2) / M + padded_masks = masks.new_zeros((B, M + pad2, M + pad2)) + padded_masks[:, padding:-padding, padding:-padding] = masks + return padded_masks, scale + + +def scale_boxes(boxes, scale): + """ + Args: + boxes (tensor): A tensor of shape (B, 4) representing B boxes with 4 + coords representing the corners x0, y0, x1, y1, + scale (float): The box scaling factor. + + Returns: + Scaled boxes. + """ + w_half = (boxes[:, 2] - boxes[:, 0]) * 0.5 + h_half = (boxes[:, 3] - boxes[:, 1]) * 0.5 + x_c = (boxes[:, 2] + boxes[:, 0]) * 0.5 + y_c = (boxes[:, 3] + boxes[:, 1]) * 0.5 + + w_half *= scale + h_half *= scale + + scaled_boxes = torch.zeros_like(boxes) + scaled_boxes[:, 0] = x_c - w_half + scaled_boxes[:, 2] = x_c + w_half + scaled_boxes[:, 1] = y_c - h_half + scaled_boxes[:, 3] = y_c + h_half + return scaled_boxes + + +@torch.jit.script_if_tracing +def _paste_masks_tensor_shape( + masks: torch.Tensor, + boxes: torch.Tensor, + image_shape: Tuple[torch.Tensor, torch.Tensor], + threshold: float = 0.5, +): + """ + A wrapper of paste_masks_in_image where image_shape is Tensor. + During tracing, shapes might be tensors instead of ints. The Tensor->int + conversion should be scripted rather than traced. + """ + return paste_masks_in_image(masks, boxes, (int(image_shape[0]), int(image_shape[1])), threshold) diff --git a/annotator/oneformer/detectron2/layers/nms.py b/annotator/oneformer/detectron2/layers/nms.py new file mode 100644 index 0000000000000000000000000000000000000000..1019e7f4c8c58f2def34a019e4c3a0573c5f69bb --- /dev/null +++ b/annotator/oneformer/detectron2/layers/nms.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +import torch +from torchvision.ops import boxes as box_ops +from torchvision.ops import nms # noqa . for compatibility + + +def batched_nms( + boxes: torch.Tensor, scores: torch.Tensor, idxs: torch.Tensor, iou_threshold: float +): + """ + Same as torchvision.ops.boxes.batched_nms, but with float(). + """ + assert boxes.shape[-1] == 4 + # Note: Torchvision already has a strategy (https://github.com/pytorch/vision/issues/1311) + # to decide whether to use coordinate trick or for loop to implement batched_nms. So we + # just call it directly. + # Fp16 does not have enough range for batched NMS, so adding float(). + return box_ops.batched_nms(boxes.float(), scores, idxs, iou_threshold) + + +# Note: this function (nms_rotated) might be moved into +# torchvision/ops/boxes.py in the future +def nms_rotated(boxes: torch.Tensor, scores: torch.Tensor, iou_threshold: float): + """ + Performs non-maximum suppression (NMS) on the rotated boxes according + to their intersection-over-union (IoU). + + Rotated NMS iteratively removes lower scoring rotated boxes which have an + IoU greater than iou_threshold with another (higher scoring) rotated box. + + Note that RotatedBox (5, 3, 4, 2, -90) covers exactly the same region as + RotatedBox (5, 3, 4, 2, 90) does, and their IoU will be 1. However, they + can be representing completely different objects in certain tasks, e.g., OCR. + + As for the question of whether rotated-NMS should treat them as faraway boxes + even though their IOU is 1, it depends on the application and/or ground truth annotation. + + As an extreme example, consider a single character v and the square box around it. + + If the angle is 0 degree, the object (text) would be read as 'v'; + + If the angle is 90 degrees, the object (text) would become '>'; + + If the angle is 180 degrees, the object (text) would become '^'; + + If the angle is 270/-90 degrees, the object (text) would become '<' + + All of these cases have IoU of 1 to each other, and rotated NMS that only + uses IoU as criterion would only keep one of them with the highest score - + which, practically, still makes sense in most cases because typically + only one of theses orientations is the correct one. Also, it does not matter + as much if the box is only used to classify the object (instead of transcribing + them with a sequential OCR recognition model) later. + + On the other hand, when we use IoU to filter proposals that are close to the + ground truth during training, we should definitely take the angle into account if + we know the ground truth is labeled with the strictly correct orientation (as in, + upside-down words are annotated with -180 degrees even though they can be covered + with a 0/90/-90 degree box, etc.) + + The way the original dataset is annotated also matters. For example, if the dataset + is a 4-point polygon dataset that does not enforce ordering of vertices/orientation, + we can estimate a minimum rotated bounding box to this polygon, but there's no way + we can tell the correct angle with 100% confidence (as shown above, there could be 4 different + rotated boxes, with angles differed by 90 degrees to each other, covering the exactly + same region). In that case we have to just use IoU to determine the box + proximity (as many detection benchmarks (even for text) do) unless there're other + assumptions we can make (like width is always larger than height, or the object is not + rotated by more than 90 degrees CCW/CW, etc.) + + In summary, not considering angles in rotated NMS seems to be a good option for now, + but we should be aware of its implications. + + Args: + boxes (Tensor[N, 5]): Rotated boxes to perform NMS on. They are expected to be in + (x_center, y_center, width, height, angle_degrees) format. + scores (Tensor[N]): Scores for each one of the rotated boxes + iou_threshold (float): Discards all overlapping rotated boxes with IoU < iou_threshold + + Returns: + keep (Tensor): int64 tensor with the indices of the elements that have been kept + by Rotated NMS, sorted in decreasing order of scores + """ + return torch.ops.detectron2.nms_rotated(boxes, scores, iou_threshold) + + +# Note: this function (batched_nms_rotated) might be moved into +# torchvision/ops/boxes.py in the future + + +@torch.jit.script_if_tracing +def batched_nms_rotated( + boxes: torch.Tensor, scores: torch.Tensor, idxs: torch.Tensor, iou_threshold: float +): + """ + Performs non-maximum suppression in a batched fashion. + + Each index value correspond to a category, and NMS + will not be applied between elements of different categories. + + Args: + boxes (Tensor[N, 5]): + boxes where NMS will be performed. They + are expected to be in (x_ctr, y_ctr, width, height, angle_degrees) format + scores (Tensor[N]): + scores for each one of the boxes + idxs (Tensor[N]): + indices of the categories for each one of the boxes. + iou_threshold (float): + discards all overlapping boxes + with IoU < iou_threshold + + Returns: + Tensor: + int64 tensor with the indices of the elements that have been kept + by NMS, sorted in decreasing order of scores + """ + assert boxes.shape[-1] == 5 + + if boxes.numel() == 0: + return torch.empty((0,), dtype=torch.int64, device=boxes.device) + boxes = boxes.float() # fp16 does not have enough range for batched NMS + # Strategy: in order to perform NMS independently per class, + # we add an offset to all the boxes. The offset is dependent + # only on the class idx, and is large enough so that boxes + # from different classes do not overlap + + # Note that batched_nms in torchvision/ops/boxes.py only uses max_coordinate, + # which won't handle negative coordinates correctly. + # Here by using min_coordinate we can make sure the negative coordinates are + # correctly handled. + max_coordinate = ( + torch.max(boxes[:, 0], boxes[:, 1]) + torch.max(boxes[:, 2], boxes[:, 3]) / 2 + ).max() + min_coordinate = ( + torch.min(boxes[:, 0], boxes[:, 1]) - torch.max(boxes[:, 2], boxes[:, 3]) / 2 + ).min() + offsets = idxs.to(boxes) * (max_coordinate - min_coordinate + 1) + boxes_for_nms = boxes.clone() # avoid modifying the original values in boxes + boxes_for_nms[:, :2] += offsets[:, None] + keep = nms_rotated(boxes_for_nms, scores, iou_threshold) + return keep diff --git a/annotator/oneformer/detectron2/layers/roi_align.py b/annotator/oneformer/detectron2/layers/roi_align.py new file mode 100644 index 0000000000000000000000000000000000000000..163462e1f194e1e4100da92d76d9516f7cc22e35 --- /dev/null +++ b/annotator/oneformer/detectron2/layers/roi_align.py @@ -0,0 +1,74 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from torch import nn +from torchvision.ops import roi_align + + +# NOTE: torchvision's RoIAlign has a different default aligned=False +class ROIAlign(nn.Module): + def __init__(self, output_size, spatial_scale, sampling_ratio, aligned=True): + """ + Args: + output_size (tuple): h, w + spatial_scale (float): scale the input boxes by this number + sampling_ratio (int): number of inputs samples to take for each output + sample. 0 to take samples densely. + aligned (bool): if False, use the legacy implementation in + Detectron. If True, align the results more perfectly. + + Note: + The meaning of aligned=True: + + Given a continuous coordinate c, its two neighboring pixel indices (in our + pixel model) are computed by floor(c - 0.5) and ceil(c - 0.5). For example, + c=1.3 has pixel neighbors with discrete indices [0] and [1] (which are sampled + from the underlying signal at continuous coordinates 0.5 and 1.5). But the original + roi_align (aligned=False) does not subtract the 0.5 when computing neighboring + pixel indices and therefore it uses pixels with a slightly incorrect alignment + (relative to our pixel model) when performing bilinear interpolation. + + With `aligned=True`, + we first appropriately scale the ROI and then shift it by -0.5 + prior to calling roi_align. This produces the correct neighbors; see + detectron2/tests/test_roi_align.py for verification. + + The difference does not make a difference to the model's performance if + ROIAlign is used together with conv layers. + """ + super().__init__() + self.output_size = output_size + self.spatial_scale = spatial_scale + self.sampling_ratio = sampling_ratio + self.aligned = aligned + + from torchvision import __version__ + + version = tuple(int(x) for x in __version__.split(".")[:2]) + # https://github.com/pytorch/vision/pull/2438 + assert version >= (0, 7), "Require torchvision >= 0.7" + + def forward(self, input, rois): + """ + Args: + input: NCHW images + rois: Bx5 boxes. First column is the index into N. The other 4 columns are xyxy. + """ + assert rois.dim() == 2 and rois.size(1) == 5 + if input.is_quantized: + input = input.dequantize() + return roi_align( + input, + rois.to(dtype=input.dtype), + self.output_size, + self.spatial_scale, + self.sampling_ratio, + self.aligned, + ) + + def __repr__(self): + tmpstr = self.__class__.__name__ + "(" + tmpstr += "output_size=" + str(self.output_size) + tmpstr += ", spatial_scale=" + str(self.spatial_scale) + tmpstr += ", sampling_ratio=" + str(self.sampling_ratio) + tmpstr += ", aligned=" + str(self.aligned) + tmpstr += ")" + return tmpstr diff --git a/annotator/oneformer/detectron2/layers/roi_align_rotated.py b/annotator/oneformer/detectron2/layers/roi_align_rotated.py new file mode 100644 index 0000000000000000000000000000000000000000..2a523992e7c736262ad5a158f209aae7875f6f0b --- /dev/null +++ b/annotator/oneformer/detectron2/layers/roi_align_rotated.py @@ -0,0 +1,100 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import torch +from torch import nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.utils import _pair + + +class _ROIAlignRotated(Function): + @staticmethod + def forward(ctx, input, roi, output_size, spatial_scale, sampling_ratio): + ctx.save_for_backward(roi) + ctx.output_size = _pair(output_size) + ctx.spatial_scale = spatial_scale + ctx.sampling_ratio = sampling_ratio + ctx.input_shape = input.size() + output = torch.ops.detectron2.roi_align_rotated_forward( + input, roi, spatial_scale, output_size[0], output_size[1], sampling_ratio + ) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + (rois,) = ctx.saved_tensors + output_size = ctx.output_size + spatial_scale = ctx.spatial_scale + sampling_ratio = ctx.sampling_ratio + bs, ch, h, w = ctx.input_shape + grad_input = torch.ops.detectron2.roi_align_rotated_backward( + grad_output, + rois, + spatial_scale, + output_size[0], + output_size[1], + bs, + ch, + h, + w, + sampling_ratio, + ) + return grad_input, None, None, None, None, None + + +roi_align_rotated = _ROIAlignRotated.apply + + +class ROIAlignRotated(nn.Module): + def __init__(self, output_size, spatial_scale, sampling_ratio): + """ + Args: + output_size (tuple): h, w + spatial_scale (float): scale the input boxes by this number + sampling_ratio (int): number of inputs samples to take for each output + sample. 0 to take samples densely. + + Note: + ROIAlignRotated supports continuous coordinate by default: + Given a continuous coordinate c, its two neighboring pixel indices (in our + pixel model) are computed by floor(c - 0.5) and ceil(c - 0.5). For example, + c=1.3 has pixel neighbors with discrete indices [0] and [1] (which are sampled + from the underlying signal at continuous coordinates 0.5 and 1.5). + """ + super(ROIAlignRotated, self).__init__() + self.output_size = output_size + self.spatial_scale = spatial_scale + self.sampling_ratio = sampling_ratio + + def forward(self, input, rois): + """ + Args: + input: NCHW images + rois: Bx6 boxes. First column is the index into N. + The other 5 columns are (x_ctr, y_ctr, width, height, angle_degrees). + """ + assert rois.dim() == 2 and rois.size(1) == 6 + orig_dtype = input.dtype + if orig_dtype == torch.float16: + input = input.float() + rois = rois.float() + output_size = _pair(self.output_size) + + # Scripting for Autograd is currently unsupported. + # This is a quick fix without having to rewrite code on the C++ side + if torch.jit.is_scripting() or torch.jit.is_tracing(): + return torch.ops.detectron2.roi_align_rotated_forward( + input, rois, self.spatial_scale, output_size[0], output_size[1], self.sampling_ratio + ).to(dtype=orig_dtype) + + return roi_align_rotated( + input, rois, self.output_size, self.spatial_scale, self.sampling_ratio + ).to(dtype=orig_dtype) + + def __repr__(self): + tmpstr = self.__class__.__name__ + "(" + tmpstr += "output_size=" + str(self.output_size) + tmpstr += ", spatial_scale=" + str(self.spatial_scale) + tmpstr += ", sampling_ratio=" + str(self.sampling_ratio) + tmpstr += ")" + return tmpstr diff --git a/annotator/oneformer/detectron2/layers/rotated_boxes.py b/annotator/oneformer/detectron2/layers/rotated_boxes.py new file mode 100644 index 0000000000000000000000000000000000000000..03f73b3bb99275931a887ad9b2d8c0ac9f412bf3 --- /dev/null +++ b/annotator/oneformer/detectron2/layers/rotated_boxes.py @@ -0,0 +1,21 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import absolute_import, division, print_function, unicode_literals +import torch + + +def pairwise_iou_rotated(boxes1, boxes2): + """ + Return intersection-over-union (Jaccard index) of boxes. + + Both sets of boxes are expected to be in + (x_center, y_center, width, height, angle) format. + + Arguments: + boxes1 (Tensor[N, 5]) + boxes2 (Tensor[M, 5]) + + Returns: + iou (Tensor[N, M]): the NxM matrix containing the pairwise + IoU values for every element in boxes1 and boxes2 + """ + return torch.ops.detectron2.box_iou_rotated(boxes1, boxes2) diff --git a/annotator/oneformer/detectron2/layers/shape_spec.py b/annotator/oneformer/detectron2/layers/shape_spec.py new file mode 100644 index 0000000000000000000000000000000000000000..8dac3c59b96576710656abebe9b5eac25868abbb --- /dev/null +++ b/annotator/oneformer/detectron2/layers/shape_spec.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class ShapeSpec: + """ + A simple structure that contains basic shape specification about a tensor. + It is often used as the auxiliary inputs/outputs of models, + to complement the lack of shape inference ability among pytorch modules. + """ + + channels: Optional[int] = None + height: Optional[int] = None + width: Optional[int] = None + stride: Optional[int] = None diff --git a/annotator/oneformer/detectron2/layers/wrappers.py b/annotator/oneformer/detectron2/layers/wrappers.py new file mode 100644 index 0000000000000000000000000000000000000000..4367f9ab50ce3ea47616e5c4c43ac4b78164b128 --- /dev/null +++ b/annotator/oneformer/detectron2/layers/wrappers.py @@ -0,0 +1,162 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +""" +Wrappers around on some nn functions, mainly to support empty tensors. + +Ideally, add support directly in PyTorch to empty tensors in those functions. + +These can be removed once https://github.com/pytorch/pytorch/issues/12013 +is implemented +""" + +import warnings +from typing import List, Optional +import torch +from torch.nn import functional as F + +from annotator.oneformer.detectron2.utils.env import TORCH_VERSION + + +def shapes_to_tensor(x: List[int], device: Optional[torch.device] = None) -> torch.Tensor: + """ + Turn a list of integer scalars or integer Tensor scalars into a vector, + in a way that's both traceable and scriptable. + + In tracing, `x` should be a list of scalar Tensor, so the output can trace to the inputs. + In scripting or eager, `x` should be a list of int. + """ + if torch.jit.is_scripting(): + return torch.as_tensor(x, device=device) + if torch.jit.is_tracing(): + assert all( + [isinstance(t, torch.Tensor) for t in x] + ), "Shape should be tensor during tracing!" + # as_tensor should not be used in tracing because it records a constant + ret = torch.stack(x) + if ret.device != device: # avoid recording a hard-coded device if not necessary + ret = ret.to(device=device) + return ret + return torch.as_tensor(x, device=device) + + +def check_if_dynamo_compiling(): + if TORCH_VERSION >= (1, 14): + from torch._dynamo import is_compiling + + return is_compiling() + else: + return False + + +def cat(tensors: List[torch.Tensor], dim: int = 0): + """ + Efficient version of torch.cat that avoids a copy if there is only a single element in a list + """ + assert isinstance(tensors, (list, tuple)) + if len(tensors) == 1: + return tensors[0] + return torch.cat(tensors, dim) + + +def empty_input_loss_func_wrapper(loss_func): + def wrapped_loss_func(input, target, *, reduction="mean", **kwargs): + """ + Same as `loss_func`, but returns 0 (instead of nan) for empty inputs. + """ + if target.numel() == 0 and reduction == "mean": + return input.sum() * 0.0 # connect the gradient + return loss_func(input, target, reduction=reduction, **kwargs) + + return wrapped_loss_func + + +cross_entropy = empty_input_loss_func_wrapper(F.cross_entropy) + + +class _NewEmptyTensorOp(torch.autograd.Function): + @staticmethod + def forward(ctx, x, new_shape): + ctx.shape = x.shape + return x.new_empty(new_shape) + + @staticmethod + def backward(ctx, grad): + shape = ctx.shape + return _NewEmptyTensorOp.apply(grad, shape), None + + +class Conv2d(torch.nn.Conv2d): + """ + A wrapper around :class:`torch.nn.Conv2d` to support empty inputs and more features. + """ + + def __init__(self, *args, **kwargs): + """ + Extra keyword arguments supported in addition to those in `torch.nn.Conv2d`: + + Args: + norm (nn.Module, optional): a normalization layer + activation (callable(Tensor) -> Tensor): a callable activation function + + It assumes that norm layer is used before activation. + """ + norm = kwargs.pop("norm", None) + activation = kwargs.pop("activation", None) + super().__init__(*args, **kwargs) + + self.norm = norm + self.activation = activation + + def forward(self, x): + # torchscript does not support SyncBatchNorm yet + # https://github.com/pytorch/pytorch/issues/40507 + # and we skip these codes in torchscript since: + # 1. currently we only support torchscript in evaluation mode + # 2. features needed by exporting module to torchscript are added in PyTorch 1.6 or + # later version, `Conv2d` in these PyTorch versions has already supported empty inputs. + if not torch.jit.is_scripting(): + # Dynamo doesn't support context managers yet + is_dynamo_compiling = check_if_dynamo_compiling() + if not is_dynamo_compiling: + with warnings.catch_warnings(record=True): + if x.numel() == 0 and self.training: + # https://github.com/pytorch/pytorch/issues/12013 + assert not isinstance( + self.norm, torch.nn.SyncBatchNorm + ), "SyncBatchNorm does not support empty inputs!" + + x = F.conv2d( + x, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups + ) + if self.norm is not None: + x = self.norm(x) + if self.activation is not None: + x = self.activation(x) + return x + + +ConvTranspose2d = torch.nn.ConvTranspose2d +BatchNorm2d = torch.nn.BatchNorm2d +interpolate = F.interpolate +Linear = torch.nn.Linear + + +def nonzero_tuple(x): + """ + A 'as_tuple=True' version of torch.nonzero to support torchscript. + because of https://github.com/pytorch/pytorch/issues/38718 + """ + if torch.jit.is_scripting(): + if x.dim() == 0: + return x.unsqueeze(0).nonzero().unbind(1) + return x.nonzero().unbind(1) + else: + return x.nonzero(as_tuple=True) + + +@torch.jit.script_if_tracing +def move_device_like(src: torch.Tensor, dst: torch.Tensor) -> torch.Tensor: + """ + Tracing friendly way to cast tensor to another tensor's device. Device will be treated + as constant during tracing, scripting the casting process as whole can workaround this issue. + """ + return src.to(dst.device) diff --git a/annotator/oneformer/detectron2/model_zoo/__init__.py b/annotator/oneformer/detectron2/model_zoo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6204208198d813728cf6419e8eef4a733f20c18f --- /dev/null +++ b/annotator/oneformer/detectron2/model_zoo/__init__.py @@ -0,0 +1,10 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +""" +Model Zoo API for Detectron2: a collection of functions to create common model architectures +listed in `MODEL_ZOO.md `_, +and optionally load their pre-trained weights. +""" + +from .model_zoo import get, get_config_file, get_checkpoint_url, get_config + +__all__ = ["get_checkpoint_url", "get", "get_config_file", "get_config"] diff --git a/annotator/oneformer/detectron2/model_zoo/model_zoo.py b/annotator/oneformer/detectron2/model_zoo/model_zoo.py new file mode 100644 index 0000000000000000000000000000000000000000..74e11b292a725cb22a7d5b001ed30b589b74598e --- /dev/null +++ b/annotator/oneformer/detectron2/model_zoo/model_zoo.py @@ -0,0 +1,213 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import os +from typing import Optional +import pkg_resources +import torch + +from annotator.oneformer.detectron2.checkpoint import DetectionCheckpointer +from annotator.oneformer.detectron2.config import CfgNode, LazyConfig, get_cfg, instantiate +from annotator.oneformer.detectron2.modeling import build_model + + +class _ModelZooUrls(object): + """ + Mapping from names to officially released Detectron2 pre-trained models. + """ + + S3_PREFIX = "https://dl.fbaipublicfiles.com/detectron2/" + + # format: {config_path.yaml} -> model_id/model_final_{commit}.pkl + CONFIG_PATH_TO_URL_SUFFIX = { + # COCO Detection with Faster R-CNN + "COCO-Detection/faster_rcnn_R_50_C4_1x": "137257644/model_final_721ade.pkl", + "COCO-Detection/faster_rcnn_R_50_DC5_1x": "137847829/model_final_51d356.pkl", + "COCO-Detection/faster_rcnn_R_50_FPN_1x": "137257794/model_final_b275ba.pkl", + "COCO-Detection/faster_rcnn_R_50_C4_3x": "137849393/model_final_f97cb7.pkl", + "COCO-Detection/faster_rcnn_R_50_DC5_3x": "137849425/model_final_68d202.pkl", + "COCO-Detection/faster_rcnn_R_50_FPN_3x": "137849458/model_final_280758.pkl", + "COCO-Detection/faster_rcnn_R_101_C4_3x": "138204752/model_final_298dad.pkl", + "COCO-Detection/faster_rcnn_R_101_DC5_3x": "138204841/model_final_3e0943.pkl", + "COCO-Detection/faster_rcnn_R_101_FPN_3x": "137851257/model_final_f6e8b1.pkl", + "COCO-Detection/faster_rcnn_X_101_32x8d_FPN_3x": "139173657/model_final_68b088.pkl", + # COCO Detection with RetinaNet + "COCO-Detection/retinanet_R_50_FPN_1x": "190397773/model_final_bfca0b.pkl", + "COCO-Detection/retinanet_R_50_FPN_3x": "190397829/model_final_5bd44e.pkl", + "COCO-Detection/retinanet_R_101_FPN_3x": "190397697/model_final_971ab9.pkl", + # COCO Detection with RPN and Fast R-CNN + "COCO-Detection/rpn_R_50_C4_1x": "137258005/model_final_450694.pkl", + "COCO-Detection/rpn_R_50_FPN_1x": "137258492/model_final_02ce48.pkl", + "COCO-Detection/fast_rcnn_R_50_FPN_1x": "137635226/model_final_e5f7ce.pkl", + # COCO Instance Segmentation Baselines with Mask R-CNN + "COCO-InstanceSegmentation/mask_rcnn_R_50_C4_1x": "137259246/model_final_9243eb.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_1x": "137260150/model_final_4f86c3.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x": "137260431/model_final_a54504.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_50_C4_3x": "137849525/model_final_4ce675.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_50_DC5_3x": "137849551/model_final_84107b.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_3x": "137849600/model_final_f10217.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_101_C4_3x": "138363239/model_final_a2914c.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_101_DC5_3x": "138363294/model_final_0464b7.pkl", + "COCO-InstanceSegmentation/mask_rcnn_R_101_FPN_3x": "138205316/model_final_a3ec72.pkl", + "COCO-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_3x": "139653917/model_final_2d9806.pkl", # noqa + # New baselines using Large-Scale Jitter and Longer Training Schedule + "new_baselines/mask_rcnn_R_50_FPN_100ep_LSJ": "42047764/model_final_bb69de.pkl", + "new_baselines/mask_rcnn_R_50_FPN_200ep_LSJ": "42047638/model_final_89a8d3.pkl", + "new_baselines/mask_rcnn_R_50_FPN_400ep_LSJ": "42019571/model_final_14d201.pkl", + "new_baselines/mask_rcnn_R_101_FPN_100ep_LSJ": "42025812/model_final_4f7b58.pkl", + "new_baselines/mask_rcnn_R_101_FPN_200ep_LSJ": "42131867/model_final_0bb7ae.pkl", + "new_baselines/mask_rcnn_R_101_FPN_400ep_LSJ": "42073830/model_final_f96b26.pkl", + "new_baselines/mask_rcnn_regnetx_4gf_dds_FPN_100ep_LSJ": "42047771/model_final_b7fbab.pkl", # noqa + "new_baselines/mask_rcnn_regnetx_4gf_dds_FPN_200ep_LSJ": "42132721/model_final_5d87c1.pkl", # noqa + "new_baselines/mask_rcnn_regnetx_4gf_dds_FPN_400ep_LSJ": "42025447/model_final_f1362d.pkl", # noqa + "new_baselines/mask_rcnn_regnety_4gf_dds_FPN_100ep_LSJ": "42047784/model_final_6ba57e.pkl", # noqa + "new_baselines/mask_rcnn_regnety_4gf_dds_FPN_200ep_LSJ": "42047642/model_final_27b9c1.pkl", # noqa + "new_baselines/mask_rcnn_regnety_4gf_dds_FPN_400ep_LSJ": "42045954/model_final_ef3a80.pkl", # noqa + # COCO Person Keypoint Detection Baselines with Keypoint R-CNN + "COCO-Keypoints/keypoint_rcnn_R_50_FPN_1x": "137261548/model_final_04e291.pkl", + "COCO-Keypoints/keypoint_rcnn_R_50_FPN_3x": "137849621/model_final_a6e10b.pkl", + "COCO-Keypoints/keypoint_rcnn_R_101_FPN_3x": "138363331/model_final_997cc7.pkl", + "COCO-Keypoints/keypoint_rcnn_X_101_32x8d_FPN_3x": "139686956/model_final_5ad38f.pkl", + # COCO Panoptic Segmentation Baselines with Panoptic FPN + "COCO-PanopticSegmentation/panoptic_fpn_R_50_1x": "139514544/model_final_dbfeb4.pkl", + "COCO-PanopticSegmentation/panoptic_fpn_R_50_3x": "139514569/model_final_c10459.pkl", + "COCO-PanopticSegmentation/panoptic_fpn_R_101_3x": "139514519/model_final_cafdb1.pkl", + # LVIS Instance Segmentation Baselines with Mask R-CNN + "LVISv0.5-InstanceSegmentation/mask_rcnn_R_50_FPN_1x": "144219072/model_final_571f7c.pkl", # noqa + "LVISv0.5-InstanceSegmentation/mask_rcnn_R_101_FPN_1x": "144219035/model_final_824ab5.pkl", # noqa + "LVISv0.5-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_1x": "144219108/model_final_5e3439.pkl", # noqa + # Cityscapes & Pascal VOC Baselines + "Cityscapes/mask_rcnn_R_50_FPN": "142423278/model_final_af9cf5.pkl", + "PascalVOC-Detection/faster_rcnn_R_50_C4": "142202221/model_final_b1acc2.pkl", + # Other Settings + "Misc/mask_rcnn_R_50_FPN_1x_dconv_c3-c5": "138602867/model_final_65c703.pkl", + "Misc/mask_rcnn_R_50_FPN_3x_dconv_c3-c5": "144998336/model_final_821d0b.pkl", + "Misc/cascade_mask_rcnn_R_50_FPN_1x": "138602847/model_final_e9d89b.pkl", + "Misc/cascade_mask_rcnn_R_50_FPN_3x": "144998488/model_final_480dd8.pkl", + "Misc/mask_rcnn_R_50_FPN_3x_syncbn": "169527823/model_final_3b3c51.pkl", + "Misc/mask_rcnn_R_50_FPN_3x_gn": "138602888/model_final_dc5d9e.pkl", + "Misc/scratch_mask_rcnn_R_50_FPN_3x_gn": "138602908/model_final_01ca85.pkl", + "Misc/scratch_mask_rcnn_R_50_FPN_9x_gn": "183808979/model_final_da7b4c.pkl", + "Misc/scratch_mask_rcnn_R_50_FPN_9x_syncbn": "184226666/model_final_5ce33e.pkl", + "Misc/panoptic_fpn_R_101_dconv_cascade_gn_3x": "139797668/model_final_be35db.pkl", + "Misc/cascade_mask_rcnn_X_152_32x8d_FPN_IN5k_gn_dconv": "18131413/model_0039999_e76410.pkl", # noqa + # D1 Comparisons + "Detectron1-Comparisons/faster_rcnn_R_50_FPN_noaug_1x": "137781054/model_final_7ab50c.pkl", # noqa + "Detectron1-Comparisons/mask_rcnn_R_50_FPN_noaug_1x": "137781281/model_final_62ca52.pkl", # noqa + "Detectron1-Comparisons/keypoint_rcnn_R_50_FPN_1x": "137781195/model_final_cce136.pkl", + } + + @staticmethod + def query(config_path: str) -> Optional[str]: + """ + Args: + config_path: relative config filename + """ + name = config_path.replace(".yaml", "").replace(".py", "") + if name in _ModelZooUrls.CONFIG_PATH_TO_URL_SUFFIX: + suffix = _ModelZooUrls.CONFIG_PATH_TO_URL_SUFFIX[name] + return _ModelZooUrls.S3_PREFIX + name + "/" + suffix + return None + + +def get_checkpoint_url(config_path): + """ + Returns the URL to the model trained using the given config + + Args: + config_path (str): config file name relative to detectron2's "configs/" + directory, e.g., "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml" + + Returns: + str: a URL to the model + """ + url = _ModelZooUrls.query(config_path) + if url is None: + raise RuntimeError("Pretrained model for {} is not available!".format(config_path)) + return url + + +def get_config_file(config_path): + """ + Returns path to a builtin config file. + + Args: + config_path (str): config file name relative to detectron2's "configs/" + directory, e.g., "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml" + + Returns: + str: the real path to the config file. + """ + cfg_file = pkg_resources.resource_filename( + "detectron2.model_zoo", os.path.join("configs", config_path) + ) + if not os.path.exists(cfg_file): + raise RuntimeError("{} not available in Model Zoo!".format(config_path)) + return cfg_file + + +def get_config(config_path, trained: bool = False): + """ + Returns a config object for a model in model zoo. + + Args: + config_path (str): config file name relative to detectron2's "configs/" + directory, e.g., "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml" + trained (bool): If True, will set ``MODEL.WEIGHTS`` to trained model zoo weights. + If False, the checkpoint specified in the config file's ``MODEL.WEIGHTS`` is used + instead; this will typically (though not always) initialize a subset of weights using + an ImageNet pre-trained model, while randomly initializing the other weights. + + Returns: + CfgNode or omegaconf.DictConfig: a config object + """ + cfg_file = get_config_file(config_path) + if cfg_file.endswith(".yaml"): + cfg = get_cfg() + cfg.merge_from_file(cfg_file) + if trained: + cfg.MODEL.WEIGHTS = get_checkpoint_url(config_path) + return cfg + elif cfg_file.endswith(".py"): + cfg = LazyConfig.load(cfg_file) + if trained: + url = get_checkpoint_url(config_path) + if "train" in cfg and "init_checkpoint" in cfg.train: + cfg.train.init_checkpoint = url + else: + raise NotImplementedError + return cfg + + +def get(config_path, trained: bool = False, device: Optional[str] = None): + """ + Get a model specified by relative path under Detectron2's official ``configs/`` directory. + + Args: + config_path (str): config file name relative to detectron2's "configs/" + directory, e.g., "COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml" + trained (bool): see :func:`get_config`. + device (str or None): overwrite the device in config, if given. + + Returns: + nn.Module: a detectron2 model. Will be in training mode. + + Example: + :: + from annotator.oneformer.detectron2 import model_zoo + model = model_zoo.get("COCO-InstanceSegmentation/mask_rcnn_R_50_FPN_1x.yaml", trained=True) + """ + cfg = get_config(config_path, trained) + if device is None and not torch.cuda.is_available(): + device = "cpu" + if device is not None and isinstance(cfg, CfgNode): + cfg.MODEL.DEVICE = device + + if isinstance(cfg, CfgNode): + model = build_model(cfg) + DetectionCheckpointer(model).load(cfg.MODEL.WEIGHTS) + else: + model = instantiate(cfg.model) + if device is not None: + model = model.to(device) + if "train" in cfg and "init_checkpoint" in cfg.train: + DetectionCheckpointer(model).load(cfg.train.init_checkpoint) + return model diff --git a/annotator/oneformer/detectron2/modeling/__init__.py b/annotator/oneformer/detectron2/modeling/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ce9ddac2f3006c7ee422aab7239060190a9d95d1 --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/__init__.py @@ -0,0 +1,64 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from annotator.oneformer.detectron2.layers import ShapeSpec + +from .anchor_generator import build_anchor_generator, ANCHOR_GENERATOR_REGISTRY +from .backbone import ( + BACKBONE_REGISTRY, + FPN, + Backbone, + ResNet, + ResNetBlockBase, + build_backbone, + build_resnet_backbone, + make_stage, + ViT, + SimpleFeaturePyramid, + get_vit_lr_decay_rate, + MViT, + SwinTransformer, +) +from .meta_arch import ( + META_ARCH_REGISTRY, + SEM_SEG_HEADS_REGISTRY, + GeneralizedRCNN, + PanopticFPN, + ProposalNetwork, + RetinaNet, + SemanticSegmentor, + build_model, + build_sem_seg_head, + FCOS, +) +from .postprocessing import detector_postprocess +from .proposal_generator import ( + PROPOSAL_GENERATOR_REGISTRY, + build_proposal_generator, + RPN_HEAD_REGISTRY, + build_rpn_head, +) +from .roi_heads import ( + ROI_BOX_HEAD_REGISTRY, + ROI_HEADS_REGISTRY, + ROI_KEYPOINT_HEAD_REGISTRY, + ROI_MASK_HEAD_REGISTRY, + ROIHeads, + StandardROIHeads, + BaseMaskRCNNHead, + BaseKeypointRCNNHead, + FastRCNNOutputLayers, + build_box_head, + build_keypoint_head, + build_mask_head, + build_roi_heads, +) +from .test_time_augmentation import DatasetMapperTTA, GeneralizedRCNNWithTTA +from .mmdet_wrapper import MMDetBackbone, MMDetDetector + +_EXCLUDE = {"ShapeSpec"} +__all__ = [k for k in globals().keys() if k not in _EXCLUDE and not k.startswith("_")] + + +from annotator.oneformer.detectron2.utils.env import fixup_module_metadata + +fixup_module_metadata(__name__, globals(), __all__) +del fixup_module_metadata diff --git a/annotator/oneformer/detectron2/modeling/anchor_generator.py b/annotator/oneformer/detectron2/modeling/anchor_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..04127c4af440b4623427b4c0911ee299166d1d7d --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/anchor_generator.py @@ -0,0 +1,386 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import collections +import math +from typing import List +import torch +from torch import nn + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.layers import ShapeSpec, move_device_like +from annotator.oneformer.detectron2.structures import Boxes, RotatedBoxes +from annotator.oneformer.detectron2.utils.registry import Registry + +ANCHOR_GENERATOR_REGISTRY = Registry("ANCHOR_GENERATOR") +ANCHOR_GENERATOR_REGISTRY.__doc__ = """ +Registry for modules that creates object detection anchors for feature maps. + +The registered object will be called with `obj(cfg, input_shape)`. +""" + + +class BufferList(nn.Module): + """ + Similar to nn.ParameterList, but for buffers + """ + + def __init__(self, buffers): + super().__init__() + for i, buffer in enumerate(buffers): + # Use non-persistent buffer so the values are not saved in checkpoint + self.register_buffer(str(i), buffer, persistent=False) + + def __len__(self): + return len(self._buffers) + + def __iter__(self): + return iter(self._buffers.values()) + + +def _create_grid_offsets( + size: List[int], stride: int, offset: float, target_device_tensor: torch.Tensor +): + grid_height, grid_width = size + shifts_x = move_device_like( + torch.arange(offset * stride, grid_width * stride, step=stride, dtype=torch.float32), + target_device_tensor, + ) + shifts_y = move_device_like( + torch.arange(offset * stride, grid_height * stride, step=stride, dtype=torch.float32), + target_device_tensor, + ) + + shift_y, shift_x = torch.meshgrid(shifts_y, shifts_x) + shift_x = shift_x.reshape(-1) + shift_y = shift_y.reshape(-1) + return shift_x, shift_y + + +def _broadcast_params(params, num_features, name): + """ + If one size (or aspect ratio) is specified and there are multiple feature + maps, we "broadcast" anchors of that single size (or aspect ratio) + over all feature maps. + + If params is list[float], or list[list[float]] with len(params) == 1, repeat + it num_features time. + + Returns: + list[list[float]]: param for each feature + """ + assert isinstance( + params, collections.abc.Sequence + ), f"{name} in anchor generator has to be a list! Got {params}." + assert len(params), f"{name} in anchor generator cannot be empty!" + if not isinstance(params[0], collections.abc.Sequence): # params is list[float] + return [params] * num_features + if len(params) == 1: + return list(params) * num_features + assert len(params) == num_features, ( + f"Got {name} of length {len(params)} in anchor generator, " + f"but the number of input features is {num_features}!" + ) + return params + + +@ANCHOR_GENERATOR_REGISTRY.register() +class DefaultAnchorGenerator(nn.Module): + """ + Compute anchors in the standard ways described in + "Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks". + """ + + box_dim: torch.jit.Final[int] = 4 + """ + the dimension of each anchor box. + """ + + @configurable + def __init__(self, *, sizes, aspect_ratios, strides, offset=0.5): + """ + This interface is experimental. + + Args: + sizes (list[list[float]] or list[float]): + If ``sizes`` is list[list[float]], ``sizes[i]`` is the list of anchor sizes + (i.e. sqrt of anchor area) to use for the i-th feature map. + If ``sizes`` is list[float], ``sizes`` is used for all feature maps. + Anchor sizes are given in absolute lengths in units of + the input image; they do not dynamically scale if the input image size changes. + aspect_ratios (list[list[float]] or list[float]): list of aspect ratios + (i.e. height / width) to use for anchors. Same "broadcast" rule for `sizes` applies. + strides (list[int]): stride of each input feature. + offset (float): Relative offset between the center of the first anchor and the top-left + corner of the image. Value has to be in [0, 1). + Recommend to use 0.5, which means half stride. + """ + super().__init__() + + self.strides = strides + self.num_features = len(self.strides) + sizes = _broadcast_params(sizes, self.num_features, "sizes") + aspect_ratios = _broadcast_params(aspect_ratios, self.num_features, "aspect_ratios") + self.cell_anchors = self._calculate_anchors(sizes, aspect_ratios) + + self.offset = offset + assert 0.0 <= self.offset < 1.0, self.offset + + @classmethod + def from_config(cls, cfg, input_shape: List[ShapeSpec]): + return { + "sizes": cfg.MODEL.ANCHOR_GENERATOR.SIZES, + "aspect_ratios": cfg.MODEL.ANCHOR_GENERATOR.ASPECT_RATIOS, + "strides": [x.stride for x in input_shape], + "offset": cfg.MODEL.ANCHOR_GENERATOR.OFFSET, + } + + def _calculate_anchors(self, sizes, aspect_ratios): + cell_anchors = [ + self.generate_cell_anchors(s, a).float() for s, a in zip(sizes, aspect_ratios) + ] + return BufferList(cell_anchors) + + @property + @torch.jit.unused + def num_cell_anchors(self): + """ + Alias of `num_anchors`. + """ + return self.num_anchors + + @property + @torch.jit.unused + def num_anchors(self): + """ + Returns: + list[int]: Each int is the number of anchors at every pixel + location, on that feature map. + For example, if at every pixel we use anchors of 3 aspect + ratios and 5 sizes, the number of anchors is 15. + (See also ANCHOR_GENERATOR.SIZES and ANCHOR_GENERATOR.ASPECT_RATIOS in config) + + In standard RPN models, `num_anchors` on every feature map is the same. + """ + return [len(cell_anchors) for cell_anchors in self.cell_anchors] + + def _grid_anchors(self, grid_sizes: List[List[int]]): + """ + Returns: + list[Tensor]: #featuremap tensors, each is (#locations x #cell_anchors) x 4 + """ + anchors = [] + # buffers() not supported by torchscript. use named_buffers() instead + buffers: List[torch.Tensor] = [x[1] for x in self.cell_anchors.named_buffers()] + for size, stride, base_anchors in zip(grid_sizes, self.strides, buffers): + shift_x, shift_y = _create_grid_offsets(size, stride, self.offset, base_anchors) + shifts = torch.stack((shift_x, shift_y, shift_x, shift_y), dim=1) + + anchors.append((shifts.view(-1, 1, 4) + base_anchors.view(1, -1, 4)).reshape(-1, 4)) + + return anchors + + def generate_cell_anchors(self, sizes=(32, 64, 128, 256, 512), aspect_ratios=(0.5, 1, 2)): + """ + Generate a tensor storing canonical anchor boxes, which are all anchor + boxes of different sizes and aspect_ratios centered at (0, 0). + We can later build the set of anchors for a full feature map by + shifting and tiling these tensors (see `meth:_grid_anchors`). + + Args: + sizes (tuple[float]): + aspect_ratios (tuple[float]]): + + Returns: + Tensor of shape (len(sizes) * len(aspect_ratios), 4) storing anchor boxes + in XYXY format. + """ + + # This is different from the anchor generator defined in the original Faster R-CNN + # code or Detectron. They yield the same AP, however the old version defines cell + # anchors in a less natural way with a shift relative to the feature grid and + # quantization that results in slightly different sizes for different aspect ratios. + # See also https://github.com/facebookresearch/Detectron/issues/227 + + anchors = [] + for size in sizes: + area = size**2.0 + for aspect_ratio in aspect_ratios: + # s * s = w * h + # a = h / w + # ... some algebra ... + # w = sqrt(s * s / a) + # h = a * w + w = math.sqrt(area / aspect_ratio) + h = aspect_ratio * w + x0, y0, x1, y1 = -w / 2.0, -h / 2.0, w / 2.0, h / 2.0 + anchors.append([x0, y0, x1, y1]) + return torch.tensor(anchors) + + def forward(self, features: List[torch.Tensor]): + """ + Args: + features (list[Tensor]): list of backbone feature maps on which to generate anchors. + + Returns: + list[Boxes]: a list of Boxes containing all the anchors for each feature map + (i.e. the cell anchors repeated over all locations in the feature map). + The number of anchors of each feature map is Hi x Wi x num_cell_anchors, + where Hi, Wi are resolution of the feature map divided by anchor stride. + """ + grid_sizes = [feature_map.shape[-2:] for feature_map in features] + anchors_over_all_feature_maps = self._grid_anchors(grid_sizes) + return [Boxes(x) for x in anchors_over_all_feature_maps] + + +@ANCHOR_GENERATOR_REGISTRY.register() +class RotatedAnchorGenerator(nn.Module): + """ + Compute rotated anchors used by Rotated RPN (RRPN), described in + "Arbitrary-Oriented Scene Text Detection via Rotation Proposals". + """ + + box_dim: int = 5 + """ + the dimension of each anchor box. + """ + + @configurable + def __init__(self, *, sizes, aspect_ratios, strides, angles, offset=0.5): + """ + This interface is experimental. + + Args: + sizes (list[list[float]] or list[float]): + If sizes is list[list[float]], sizes[i] is the list of anchor sizes + (i.e. sqrt of anchor area) to use for the i-th feature map. + If sizes is list[float], the sizes are used for all feature maps. + Anchor sizes are given in absolute lengths in units of + the input image; they do not dynamically scale if the input image size changes. + aspect_ratios (list[list[float]] or list[float]): list of aspect ratios + (i.e. height / width) to use for anchors. Same "broadcast" rule for `sizes` applies. + strides (list[int]): stride of each input feature. + angles (list[list[float]] or list[float]): list of angles (in degrees CCW) + to use for anchors. Same "broadcast" rule for `sizes` applies. + offset (float): Relative offset between the center of the first anchor and the top-left + corner of the image. Value has to be in [0, 1). + Recommend to use 0.5, which means half stride. + """ + super().__init__() + + self.strides = strides + self.num_features = len(self.strides) + sizes = _broadcast_params(sizes, self.num_features, "sizes") + aspect_ratios = _broadcast_params(aspect_ratios, self.num_features, "aspect_ratios") + angles = _broadcast_params(angles, self.num_features, "angles") + self.cell_anchors = self._calculate_anchors(sizes, aspect_ratios, angles) + + self.offset = offset + assert 0.0 <= self.offset < 1.0, self.offset + + @classmethod + def from_config(cls, cfg, input_shape: List[ShapeSpec]): + return { + "sizes": cfg.MODEL.ANCHOR_GENERATOR.SIZES, + "aspect_ratios": cfg.MODEL.ANCHOR_GENERATOR.ASPECT_RATIOS, + "strides": [x.stride for x in input_shape], + "offset": cfg.MODEL.ANCHOR_GENERATOR.OFFSET, + "angles": cfg.MODEL.ANCHOR_GENERATOR.ANGLES, + } + + def _calculate_anchors(self, sizes, aspect_ratios, angles): + cell_anchors = [ + self.generate_cell_anchors(size, aspect_ratio, angle).float() + for size, aspect_ratio, angle in zip(sizes, aspect_ratios, angles) + ] + return BufferList(cell_anchors) + + @property + def num_cell_anchors(self): + """ + Alias of `num_anchors`. + """ + return self.num_anchors + + @property + def num_anchors(self): + """ + Returns: + list[int]: Each int is the number of anchors at every pixel + location, on that feature map. + For example, if at every pixel we use anchors of 3 aspect + ratios, 2 sizes and 5 angles, the number of anchors is 30. + (See also ANCHOR_GENERATOR.SIZES, ANCHOR_GENERATOR.ASPECT_RATIOS + and ANCHOR_GENERATOR.ANGLES in config) + + In standard RRPN models, `num_anchors` on every feature map is the same. + """ + return [len(cell_anchors) for cell_anchors in self.cell_anchors] + + def _grid_anchors(self, grid_sizes): + anchors = [] + for size, stride, base_anchors in zip(grid_sizes, self.strides, self.cell_anchors): + shift_x, shift_y = _create_grid_offsets(size, stride, self.offset, base_anchors) + zeros = torch.zeros_like(shift_x) + shifts = torch.stack((shift_x, shift_y, zeros, zeros, zeros), dim=1) + + anchors.append((shifts.view(-1, 1, 5) + base_anchors.view(1, -1, 5)).reshape(-1, 5)) + + return anchors + + def generate_cell_anchors( + self, + sizes=(32, 64, 128, 256, 512), + aspect_ratios=(0.5, 1, 2), + angles=(-90, -60, -30, 0, 30, 60, 90), + ): + """ + Generate a tensor storing canonical anchor boxes, which are all anchor + boxes of different sizes, aspect_ratios, angles centered at (0, 0). + We can later build the set of anchors for a full feature map by + shifting and tiling these tensors (see `meth:_grid_anchors`). + + Args: + sizes (tuple[float]): + aspect_ratios (tuple[float]]): + angles (tuple[float]]): + + Returns: + Tensor of shape (len(sizes) * len(aspect_ratios) * len(angles), 5) + storing anchor boxes in (x_ctr, y_ctr, w, h, angle) format. + """ + anchors = [] + for size in sizes: + area = size**2.0 + for aspect_ratio in aspect_ratios: + # s * s = w * h + # a = h / w + # ... some algebra ... + # w = sqrt(s * s / a) + # h = a * w + w = math.sqrt(area / aspect_ratio) + h = aspect_ratio * w + anchors.extend([0, 0, w, h, a] for a in angles) + + return torch.tensor(anchors) + + def forward(self, features): + """ + Args: + features (list[Tensor]): list of backbone feature maps on which to generate anchors. + + Returns: + list[RotatedBoxes]: a list of Boxes containing all the anchors for each feature map + (i.e. the cell anchors repeated over all locations in the feature map). + The number of anchors of each feature map is Hi x Wi x num_cell_anchors, + where Hi, Wi are resolution of the feature map divided by anchor stride. + """ + grid_sizes = [feature_map.shape[-2:] for feature_map in features] + anchors_over_all_feature_maps = self._grid_anchors(grid_sizes) + return [RotatedBoxes(x) for x in anchors_over_all_feature_maps] + + +def build_anchor_generator(cfg, input_shape): + """ + Built an anchor generator from `cfg.MODEL.ANCHOR_GENERATOR.NAME`. + """ + anchor_generator = cfg.MODEL.ANCHOR_GENERATOR.NAME + return ANCHOR_GENERATOR_REGISTRY.get(anchor_generator)(cfg, input_shape) diff --git a/annotator/oneformer/detectron2/modeling/backbone/__init__.py b/annotator/oneformer/detectron2/modeling/backbone/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5b3358a4061b143c78eba8e7bf81fe9f7ffac1aa --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/backbone/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .build import build_backbone, BACKBONE_REGISTRY # noqa F401 isort:skip + +from .backbone import Backbone +from .fpn import FPN +from .regnet import RegNet +from .resnet import ( + BasicStem, + ResNet, + ResNetBlockBase, + build_resnet_backbone, + make_stage, + BottleneckBlock, +) +from .vit import ViT, SimpleFeaturePyramid, get_vit_lr_decay_rate +from .mvit import MViT +from .swin import SwinTransformer + +__all__ = [k for k in globals().keys() if not k.startswith("_")] +# TODO can expose more resnet blocks after careful consideration diff --git a/annotator/oneformer/detectron2/modeling/backbone/backbone.py b/annotator/oneformer/detectron2/modeling/backbone/backbone.py new file mode 100644 index 0000000000000000000000000000000000000000..04f3c3c009d972bcab46eaeab33a8bfcc05b726c --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/backbone/backbone.py @@ -0,0 +1,74 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from abc import ABCMeta, abstractmethod +from typing import Dict +import torch.nn as nn + +from annotator.oneformer.detectron2.layers import ShapeSpec + +__all__ = ["Backbone"] + + +class Backbone(nn.Module, metaclass=ABCMeta): + """ + Abstract base class for network backbones. + """ + + def __init__(self): + """ + The `__init__` method of any subclass can specify its own set of arguments. + """ + super().__init__() + + @abstractmethod + def forward(self): + """ + Subclasses must override this method, but adhere to the same return type. + + Returns: + dict[str->Tensor]: mapping from feature name (e.g., "res2") to tensor + """ + pass + + @property + def size_divisibility(self) -> int: + """ + Some backbones require the input height and width to be divisible by a + specific integer. This is typically true for encoder / decoder type networks + with lateral connection (e.g., FPN) for which feature maps need to match + dimension in the "bottom up" and "top down" paths. Set to 0 if no specific + input size divisibility is required. + """ + return 0 + + @property + def padding_constraints(self) -> Dict[str, int]: + """ + This property is a generalization of size_divisibility. Some backbones and training + recipes require specific padding constraints, such as enforcing divisibility by a specific + integer (e.g., FPN) or padding to a square (e.g., ViTDet with large-scale jitter + in :paper:vitdet). `padding_constraints` contains these optional items like: + { + "size_divisibility": int, + "square_size": int, + # Future options are possible + } + `size_divisibility` will read from here if presented and `square_size` indicates the + square padding size if `square_size` > 0. + + TODO: use type of Dict[str, int] to avoid torchscipt issues. The type of padding_constraints + could be generalized as TypedDict (Python 3.8+) to support more types in the future. + """ + return {} + + def output_shape(self): + """ + Returns: + dict[str->ShapeSpec] + """ + # this is a backward-compatible default + return { + name: ShapeSpec( + channels=self._out_feature_channels[name], stride=self._out_feature_strides[name] + ) + for name in self._out_features + } diff --git a/annotator/oneformer/detectron2/modeling/backbone/build.py b/annotator/oneformer/detectron2/modeling/backbone/build.py new file mode 100644 index 0000000000000000000000000000000000000000..63a4aaced2c2869294d2b16f4b95cdfdd01259b7 --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/backbone/build.py @@ -0,0 +1,33 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from annotator.oneformer.detectron2.layers import ShapeSpec +from annotator.oneformer.detectron2.utils.registry import Registry + +from .backbone import Backbone + +BACKBONE_REGISTRY = Registry("BACKBONE") +BACKBONE_REGISTRY.__doc__ = """ +Registry for backbones, which extract feature maps from images + +The registered object must be a callable that accepts two arguments: + +1. A :class:`detectron2.config.CfgNode` +2. A :class:`detectron2.layers.ShapeSpec`, which contains the input shape specification. + +Registered object must return instance of :class:`Backbone`. +""" + + +def build_backbone(cfg, input_shape=None): + """ + Build a backbone from `cfg.MODEL.BACKBONE.NAME`. + + Returns: + an instance of :class:`Backbone` + """ + if input_shape is None: + input_shape = ShapeSpec(channels=len(cfg.MODEL.PIXEL_MEAN)) + + backbone_name = cfg.MODEL.BACKBONE.NAME + backbone = BACKBONE_REGISTRY.get(backbone_name)(cfg, input_shape) + assert isinstance(backbone, Backbone) + return backbone diff --git a/annotator/oneformer/detectron2/modeling/backbone/fpn.py b/annotator/oneformer/detectron2/modeling/backbone/fpn.py new file mode 100644 index 0000000000000000000000000000000000000000..a5a9e8ce1a5ad2e3e07111731185a60855e59b22 --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/backbone/fpn.py @@ -0,0 +1,268 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import math +import fvcore.nn.weight_init as weight_init +import torch +import torch.nn.functional as F +from torch import nn + +from annotator.oneformer.detectron2.layers import Conv2d, ShapeSpec, get_norm + +from .backbone import Backbone +from .build import BACKBONE_REGISTRY +from .resnet import build_resnet_backbone + +__all__ = ["build_resnet_fpn_backbone", "build_retinanet_resnet_fpn_backbone", "FPN"] + + +class FPN(Backbone): + """ + This module implements :paper:`FPN`. + It creates pyramid features built on top of some input feature maps. + """ + + _fuse_type: torch.jit.Final[str] + + def __init__( + self, + bottom_up, + in_features, + out_channels, + norm="", + top_block=None, + fuse_type="sum", + square_pad=0, + ): + """ + Args: + bottom_up (Backbone): module representing the bottom up subnetwork. + Must be a subclass of :class:`Backbone`. The multi-scale feature + maps generated by the bottom up network, and listed in `in_features`, + are used to generate FPN levels. + in_features (list[str]): names of the input feature maps coming + from the backbone to which FPN is attached. For example, if the + backbone produces ["res2", "res3", "res4"], any *contiguous* sublist + of these may be used; order must be from high to low resolution. + out_channels (int): number of channels in the output feature maps. + norm (str): the normalization to use. + top_block (nn.Module or None): if provided, an extra operation will + be performed on the output of the last (smallest resolution) + FPN output, and the result will extend the result list. The top_block + further downsamples the feature map. It must have an attribute + "num_levels", meaning the number of extra FPN levels added by + this block, and "in_feature", which is a string representing + its input feature (e.g., p5). + fuse_type (str): types for fusing the top down features and the lateral + ones. It can be "sum" (default), which sums up element-wise; or "avg", + which takes the element-wise mean of the two. + square_pad (int): If > 0, require input images to be padded to specific square size. + """ + super(FPN, self).__init__() + assert isinstance(bottom_up, Backbone) + assert in_features, in_features + + # Feature map strides and channels from the bottom up network (e.g. ResNet) + input_shapes = bottom_up.output_shape() + strides = [input_shapes[f].stride for f in in_features] + in_channels_per_feature = [input_shapes[f].channels for f in in_features] + + _assert_strides_are_log2_contiguous(strides) + lateral_convs = [] + output_convs = [] + + use_bias = norm == "" + for idx, in_channels in enumerate(in_channels_per_feature): + lateral_norm = get_norm(norm, out_channels) + output_norm = get_norm(norm, out_channels) + + lateral_conv = Conv2d( + in_channels, out_channels, kernel_size=1, bias=use_bias, norm=lateral_norm + ) + output_conv = Conv2d( + out_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=use_bias, + norm=output_norm, + ) + weight_init.c2_xavier_fill(lateral_conv) + weight_init.c2_xavier_fill(output_conv) + stage = int(math.log2(strides[idx])) + self.add_module("fpn_lateral{}".format(stage), lateral_conv) + self.add_module("fpn_output{}".format(stage), output_conv) + + lateral_convs.append(lateral_conv) + output_convs.append(output_conv) + # Place convs into top-down order (from low to high resolution) + # to make the top-down computation in forward clearer. + self.lateral_convs = lateral_convs[::-1] + self.output_convs = output_convs[::-1] + self.top_block = top_block + self.in_features = tuple(in_features) + self.bottom_up = bottom_up + # Return feature names are "p", like ["p2", "p3", ..., "p6"] + self._out_feature_strides = {"p{}".format(int(math.log2(s))): s for s in strides} + # top block output feature maps. + if self.top_block is not None: + for s in range(stage, stage + self.top_block.num_levels): + self._out_feature_strides["p{}".format(s + 1)] = 2 ** (s + 1) + + self._out_features = list(self._out_feature_strides.keys()) + self._out_feature_channels = {k: out_channels for k in self._out_features} + self._size_divisibility = strides[-1] + self._square_pad = square_pad + assert fuse_type in {"avg", "sum"} + self._fuse_type = fuse_type + + @property + def size_divisibility(self): + return self._size_divisibility + + @property + def padding_constraints(self): + return {"square_size": self._square_pad} + + def forward(self, x): + """ + Args: + input (dict[str->Tensor]): mapping feature map name (e.g., "res5") to + feature map tensor for each feature level in high to low resolution order. + + Returns: + dict[str->Tensor]: + mapping from feature map name to FPN feature map tensor + in high to low resolution order. Returned feature names follow the FPN + paper convention: "p", where stage has stride = 2 ** stage e.g., + ["p2", "p3", ..., "p6"]. + """ + bottom_up_features = self.bottom_up(x) + results = [] + prev_features = self.lateral_convs[0](bottom_up_features[self.in_features[-1]]) + results.append(self.output_convs[0](prev_features)) + + # Reverse feature maps into top-down order (from low to high resolution) + for idx, (lateral_conv, output_conv) in enumerate( + zip(self.lateral_convs, self.output_convs) + ): + # Slicing of ModuleList is not supported https://github.com/pytorch/pytorch/issues/47336 + # Therefore we loop over all modules but skip the first one + if idx > 0: + features = self.in_features[-idx - 1] + features = bottom_up_features[features] + top_down_features = F.interpolate(prev_features, scale_factor=2.0, mode="nearest") + lateral_features = lateral_conv(features) + prev_features = lateral_features + top_down_features + if self._fuse_type == "avg": + prev_features /= 2 + results.insert(0, output_conv(prev_features)) + + if self.top_block is not None: + if self.top_block.in_feature in bottom_up_features: + top_block_in_feature = bottom_up_features[self.top_block.in_feature] + else: + top_block_in_feature = results[self._out_features.index(self.top_block.in_feature)] + results.extend(self.top_block(top_block_in_feature)) + assert len(self._out_features) == len(results) + return {f: res for f, res in zip(self._out_features, results)} + + def output_shape(self): + return { + name: ShapeSpec( + channels=self._out_feature_channels[name], stride=self._out_feature_strides[name] + ) + for name in self._out_features + } + + +def _assert_strides_are_log2_contiguous(strides): + """ + Assert that each stride is 2x times its preceding stride, i.e. "contiguous in log2". + """ + for i, stride in enumerate(strides[1:], 1): + assert stride == 2 * strides[i - 1], "Strides {} {} are not log2 contiguous".format( + stride, strides[i - 1] + ) + + +class LastLevelMaxPool(nn.Module): + """ + This module is used in the original FPN to generate a downsampled + P6 feature from P5. + """ + + def __init__(self): + super().__init__() + self.num_levels = 1 + self.in_feature = "p5" + + def forward(self, x): + return [F.max_pool2d(x, kernel_size=1, stride=2, padding=0)] + + +class LastLevelP6P7(nn.Module): + """ + This module is used in RetinaNet to generate extra layers, P6 and P7 from + C5 feature. + """ + + def __init__(self, in_channels, out_channels, in_feature="res5"): + super().__init__() + self.num_levels = 2 + self.in_feature = in_feature + self.p6 = nn.Conv2d(in_channels, out_channels, 3, 2, 1) + self.p7 = nn.Conv2d(out_channels, out_channels, 3, 2, 1) + for module in [self.p6, self.p7]: + weight_init.c2_xavier_fill(module) + + def forward(self, c5): + p6 = self.p6(c5) + p7 = self.p7(F.relu(p6)) + return [p6, p7] + + +@BACKBONE_REGISTRY.register() +def build_resnet_fpn_backbone(cfg, input_shape: ShapeSpec): + """ + Args: + cfg: a detectron2 CfgNode + + Returns: + backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`. + """ + bottom_up = build_resnet_backbone(cfg, input_shape) + in_features = cfg.MODEL.FPN.IN_FEATURES + out_channels = cfg.MODEL.FPN.OUT_CHANNELS + backbone = FPN( + bottom_up=bottom_up, + in_features=in_features, + out_channels=out_channels, + norm=cfg.MODEL.FPN.NORM, + top_block=LastLevelMaxPool(), + fuse_type=cfg.MODEL.FPN.FUSE_TYPE, + ) + return backbone + + +@BACKBONE_REGISTRY.register() +def build_retinanet_resnet_fpn_backbone(cfg, input_shape: ShapeSpec): + """ + Args: + cfg: a detectron2 CfgNode + + Returns: + backbone (Backbone): backbone module, must be a subclass of :class:`Backbone`. + """ + bottom_up = build_resnet_backbone(cfg, input_shape) + in_features = cfg.MODEL.FPN.IN_FEATURES + out_channels = cfg.MODEL.FPN.OUT_CHANNELS + in_channels_p6p7 = bottom_up.output_shape()["res5"].channels + backbone = FPN( + bottom_up=bottom_up, + in_features=in_features, + out_channels=out_channels, + norm=cfg.MODEL.FPN.NORM, + top_block=LastLevelP6P7(in_channels_p6p7, out_channels), + fuse_type=cfg.MODEL.FPN.FUSE_TYPE, + ) + return backbone diff --git a/annotator/oneformer/detectron2/modeling/backbone/mvit.py b/annotator/oneformer/detectron2/modeling/backbone/mvit.py new file mode 100644 index 0000000000000000000000000000000000000000..50667a8a836b933666761cc09d4175e64098c8aa --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/backbone/mvit.py @@ -0,0 +1,448 @@ +import logging +import numpy as np +import torch +import torch.nn as nn + +from .backbone import Backbone +from .utils import ( + PatchEmbed, + add_decomposed_rel_pos, + get_abs_pos, + window_partition, + window_unpartition, +) + +logger = logging.getLogger(__name__) + + +__all__ = ["MViT"] + + +def attention_pool(x, pool, norm=None): + # (B, H, W, C) -> (B, C, H, W) + x = x.permute(0, 3, 1, 2) + x = pool(x) + # (B, C, H1, W1) -> (B, H1, W1, C) + x = x.permute(0, 2, 3, 1) + if norm: + x = norm(x) + + return x + + +class MultiScaleAttention(nn.Module): + """Multiscale Multi-head Attention block.""" + + def __init__( + self, + dim, + dim_out, + num_heads, + qkv_bias=True, + norm_layer=nn.LayerNorm, + pool_kernel=(3, 3), + stride_q=1, + stride_kv=1, + residual_pooling=True, + window_size=0, + use_rel_pos=False, + rel_pos_zero_init=True, + input_size=None, + ): + """ + Args: + dim (int): Number of input channels. + dim_out (int): Number of output channels. + num_heads (int): Number of attention heads. + qkv_bias (bool: If True, add a learnable bias to query, key, value. + norm_layer (nn.Module): Normalization layer. + pool_kernel (tuple): kernel size for qkv pooling layers. + stride_q (int): stride size for q pooling layer. + stride_kv (int): stride size for kv pooling layer. + residual_pooling (bool): If true, enable residual pooling. + use_rel_pos (bool): If True, add relative postional embeddings to the attention map. + rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. + input_size (int or None): Input resolution. + """ + super().__init__() + self.num_heads = num_heads + head_dim = dim_out // num_heads + self.scale = head_dim**-0.5 + + self.qkv = nn.Linear(dim, dim_out * 3, bias=qkv_bias) + self.proj = nn.Linear(dim_out, dim_out) + + # qkv pooling + pool_padding = [k // 2 for k in pool_kernel] + dim_conv = dim_out // num_heads + self.pool_q = nn.Conv2d( + dim_conv, + dim_conv, + pool_kernel, + stride=stride_q, + padding=pool_padding, + groups=dim_conv, + bias=False, + ) + self.norm_q = norm_layer(dim_conv) + self.pool_k = nn.Conv2d( + dim_conv, + dim_conv, + pool_kernel, + stride=stride_kv, + padding=pool_padding, + groups=dim_conv, + bias=False, + ) + self.norm_k = norm_layer(dim_conv) + self.pool_v = nn.Conv2d( + dim_conv, + dim_conv, + pool_kernel, + stride=stride_kv, + padding=pool_padding, + groups=dim_conv, + bias=False, + ) + self.norm_v = norm_layer(dim_conv) + + self.window_size = window_size + if window_size: + self.q_win_size = window_size // stride_q + self.kv_win_size = window_size // stride_kv + self.residual_pooling = residual_pooling + + self.use_rel_pos = use_rel_pos + if self.use_rel_pos: + # initialize relative positional embeddings + assert input_size[0] == input_size[1] + size = input_size[0] + rel_dim = 2 * max(size // stride_q, size // stride_kv) - 1 + self.rel_pos_h = nn.Parameter(torch.zeros(rel_dim, head_dim)) + self.rel_pos_w = nn.Parameter(torch.zeros(rel_dim, head_dim)) + + if not rel_pos_zero_init: + nn.init.trunc_normal_(self.rel_pos_h, std=0.02) + nn.init.trunc_normal_(self.rel_pos_w, std=0.02) + + def forward(self, x): + B, H, W, _ = x.shape + # qkv with shape (3, B, nHead, H, W, C) + qkv = self.qkv(x).reshape(B, H, W, 3, self.num_heads, -1).permute(3, 0, 4, 1, 2, 5) + # q, k, v with shape (B * nHead, H, W, C) + q, k, v = qkv.reshape(3, B * self.num_heads, H, W, -1).unbind(0) + + q = attention_pool(q, self.pool_q, self.norm_q) + k = attention_pool(k, self.pool_k, self.norm_k) + v = attention_pool(v, self.pool_v, self.norm_v) + + ori_q = q + if self.window_size: + q, q_hw_pad = window_partition(q, self.q_win_size) + k, kv_hw_pad = window_partition(k, self.kv_win_size) + v, _ = window_partition(v, self.kv_win_size) + q_hw = (self.q_win_size, self.q_win_size) + kv_hw = (self.kv_win_size, self.kv_win_size) + else: + q_hw = q.shape[1:3] + kv_hw = k.shape[1:3] + + q = q.view(q.shape[0], np.prod(q_hw), -1) + k = k.view(k.shape[0], np.prod(kv_hw), -1) + v = v.view(v.shape[0], np.prod(kv_hw), -1) + + attn = (q * self.scale) @ k.transpose(-2, -1) + + if self.use_rel_pos: + attn = add_decomposed_rel_pos(attn, q, self.rel_pos_h, self.rel_pos_w, q_hw, kv_hw) + + attn = attn.softmax(dim=-1) + x = attn @ v + + x = x.view(x.shape[0], q_hw[0], q_hw[1], -1) + + if self.window_size: + x = window_unpartition(x, self.q_win_size, q_hw_pad, ori_q.shape[1:3]) + + if self.residual_pooling: + x += ori_q + + H, W = x.shape[1], x.shape[2] + x = x.view(B, self.num_heads, H, W, -1).permute(0, 2, 3, 1, 4).reshape(B, H, W, -1) + x = self.proj(x) + + return x + + +class MultiScaleBlock(nn.Module): + """Multiscale Transformer blocks""" + + def __init__( + self, + dim, + dim_out, + num_heads, + mlp_ratio=4.0, + qkv_bias=True, + drop_path=0.0, + norm_layer=nn.LayerNorm, + act_layer=nn.GELU, + qkv_pool_kernel=(3, 3), + stride_q=1, + stride_kv=1, + residual_pooling=True, + window_size=0, + use_rel_pos=False, + rel_pos_zero_init=True, + input_size=None, + ): + """ + Args: + dim (int): Number of input channels. + dim_out (int): Number of output channels. + num_heads (int): Number of attention heads in the MViT block. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool): If True, add a learnable bias to query, key, value. + drop_path (float): Stochastic depth rate. + norm_layer (nn.Module): Normalization layer. + act_layer (nn.Module): Activation layer. + qkv_pool_kernel (tuple): kernel size for qkv pooling layers. + stride_q (int): stride size for q pooling layer. + stride_kv (int): stride size for kv pooling layer. + residual_pooling (bool): If true, enable residual pooling. + window_size (int): Window size for window attention blocks. If it equals 0, then not + use window attention. + use_rel_pos (bool): If True, add relative postional embeddings to the attention map. + rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. + input_size (int or None): Input resolution. + """ + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = MultiScaleAttention( + dim, + dim_out, + num_heads=num_heads, + qkv_bias=qkv_bias, + norm_layer=norm_layer, + pool_kernel=qkv_pool_kernel, + stride_q=stride_q, + stride_kv=stride_kv, + residual_pooling=residual_pooling, + window_size=window_size, + use_rel_pos=use_rel_pos, + rel_pos_zero_init=rel_pos_zero_init, + input_size=input_size, + ) + + from timm.models.layers import DropPath, Mlp + + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + self.norm2 = norm_layer(dim_out) + self.mlp = Mlp( + in_features=dim_out, + hidden_features=int(dim_out * mlp_ratio), + out_features=dim_out, + act_layer=act_layer, + ) + + if dim != dim_out: + self.proj = nn.Linear(dim, dim_out) + + if stride_q > 1: + kernel_skip = stride_q + 1 + padding_skip = int(kernel_skip // 2) + self.pool_skip = nn.MaxPool2d(kernel_skip, stride_q, padding_skip, ceil_mode=False) + + def forward(self, x): + x_norm = self.norm1(x) + x_block = self.attn(x_norm) + + if hasattr(self, "proj"): + x = self.proj(x_norm) + if hasattr(self, "pool_skip"): + x = attention_pool(x, self.pool_skip) + + x = x + self.drop_path(x_block) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + return x + + +class MViT(Backbone): + """ + This module implements Multiscale Vision Transformer (MViT) backbone in :paper:'mvitv2'. + """ + + def __init__( + self, + img_size=224, + patch_kernel=(7, 7), + patch_stride=(4, 4), + patch_padding=(3, 3), + in_chans=3, + embed_dim=96, + depth=16, + num_heads=1, + last_block_indexes=(0, 2, 11, 15), + qkv_pool_kernel=(3, 3), + adaptive_kv_stride=4, + adaptive_window_size=56, + residual_pooling=True, + mlp_ratio=4.0, + qkv_bias=True, + drop_path_rate=0.0, + norm_layer=nn.LayerNorm, + act_layer=nn.GELU, + use_abs_pos=False, + use_rel_pos=True, + rel_pos_zero_init=True, + use_act_checkpoint=False, + pretrain_img_size=224, + pretrain_use_cls_token=True, + out_features=("scale2", "scale3", "scale4", "scale5"), + ): + """ + Args: + img_size (int): Input image size. + patch_kernel (tuple): kernel size for patch embedding. + patch_stride (tuple): stride size for patch embedding. + patch_padding (tuple): padding size for patch embedding. + in_chans (int): Number of input image channels. + embed_dim (int): Patch embedding dimension. + depth (int): Depth of MViT. + num_heads (int): Number of base attention heads in each MViT block. + last_block_indexes (tuple): Block indexes for last blocks in each stage. + qkv_pool_kernel (tuple): kernel size for qkv pooling layers. + adaptive_kv_stride (int): adaptive stride size for kv pooling. + adaptive_window_size (int): adaptive window size for window attention blocks. + residual_pooling (bool): If true, enable residual pooling. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool): If True, add a learnable bias to query, key, value. + drop_path_rate (float): Stochastic depth rate. + norm_layer (nn.Module): Normalization layer. + act_layer (nn.Module): Activation layer. + use_abs_pos (bool): If True, use absolute positional embeddings. + use_rel_pos (bool): If True, add relative postional embeddings to the attention map. + rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. + window_size (int): Window size for window attention blocks. + use_act_checkpoint (bool): If True, use activation checkpointing. + pretrain_img_size (int): input image size for pretraining models. + pretrain_use_cls_token (bool): If True, pretrainig models use class token. + out_features (tuple): name of the feature maps from each stage. + """ + super().__init__() + self.pretrain_use_cls_token = pretrain_use_cls_token + + self.patch_embed = PatchEmbed( + kernel_size=patch_kernel, + stride=patch_stride, + padding=patch_padding, + in_chans=in_chans, + embed_dim=embed_dim, + ) + + if use_abs_pos: + # Initialize absoluate positional embedding with pretrain image size. + num_patches = (pretrain_img_size // patch_stride[0]) * ( + pretrain_img_size // patch_stride[1] + ) + num_positions = (num_patches + 1) if pretrain_use_cls_token else num_patches + self.pos_embed = nn.Parameter(torch.zeros(1, num_positions, embed_dim)) + else: + self.pos_embed = None + + # stochastic depth decay rule + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] + dim_out = embed_dim + stride_kv = adaptive_kv_stride + window_size = adaptive_window_size + input_size = (img_size // patch_stride[0], img_size // patch_stride[1]) + stage = 2 + stride = patch_stride[0] + self._out_feature_strides = {} + self._out_feature_channels = {} + self.blocks = nn.ModuleList() + for i in range(depth): + # Multiply stride_kv by 2 if it's the last block of stage2 and stage3. + if i == last_block_indexes[1] or i == last_block_indexes[2]: + stride_kv_ = stride_kv * 2 + else: + stride_kv_ = stride_kv + # hybrid window attention: global attention in last three stages. + window_size_ = 0 if i in last_block_indexes[1:] else window_size + block = MultiScaleBlock( + dim=embed_dim, + dim_out=dim_out, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + drop_path=dpr[i], + norm_layer=norm_layer, + qkv_pool_kernel=qkv_pool_kernel, + stride_q=2 if i - 1 in last_block_indexes else 1, + stride_kv=stride_kv_, + residual_pooling=residual_pooling, + window_size=window_size_, + use_rel_pos=use_rel_pos, + rel_pos_zero_init=rel_pos_zero_init, + input_size=input_size, + ) + if use_act_checkpoint: + # TODO: use torch.utils.checkpoint + from fairscale.nn.checkpoint import checkpoint_wrapper + + block = checkpoint_wrapper(block) + self.blocks.append(block) + + embed_dim = dim_out + if i in last_block_indexes: + name = f"scale{stage}" + if name in out_features: + self._out_feature_channels[name] = dim_out + self._out_feature_strides[name] = stride + self.add_module(f"{name}_norm", norm_layer(dim_out)) + + dim_out *= 2 + num_heads *= 2 + stride_kv = max(stride_kv // 2, 1) + stride *= 2 + stage += 1 + if i - 1 in last_block_indexes: + window_size = window_size // 2 + input_size = [s // 2 for s in input_size] + + self._out_features = out_features + self._last_block_indexes = last_block_indexes + + if self.pos_embed is not None: + nn.init.trunc_normal_(self.pos_embed, std=0.02) + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + nn.init.trunc_normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def forward(self, x): + x = self.patch_embed(x) + + if self.pos_embed is not None: + x = x + get_abs_pos(self.pos_embed, self.pretrain_use_cls_token, x.shape[1:3]) + + outputs = {} + stage = 2 + for i, blk in enumerate(self.blocks): + x = blk(x) + if i in self._last_block_indexes: + name = f"scale{stage}" + if name in self._out_features: + x_out = getattr(self, f"{name}_norm")(x) + outputs[name] = x_out.permute(0, 3, 1, 2) + stage += 1 + + return outputs diff --git a/annotator/oneformer/detectron2/modeling/backbone/regnet.py b/annotator/oneformer/detectron2/modeling/backbone/regnet.py new file mode 100644 index 0000000000000000000000000000000000000000..a9d5b1c8c2d71abccedca7c2cca1117588407e9f --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/backbone/regnet.py @@ -0,0 +1,452 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Implementation of RegNet models from :paper:`dds` and :paper:`scaling`. + +This code is adapted from https://github.com/facebookresearch/pycls with minimal modifications. +Some code duplication exists between RegNet and ResNets (e.g., ResStem) in order to simplify +model loading. +""" + +import numpy as np +from torch import nn + +from annotator.oneformer.detectron2.layers import CNNBlockBase, ShapeSpec, get_norm + +from .backbone import Backbone + +__all__ = [ + "AnyNet", + "RegNet", + "ResStem", + "SimpleStem", + "VanillaBlock", + "ResBasicBlock", + "ResBottleneckBlock", +] + + +def conv2d(w_in, w_out, k, *, stride=1, groups=1, bias=False): + """Helper for building a conv2d layer.""" + assert k % 2 == 1, "Only odd size kernels supported to avoid padding issues." + s, p, g, b = stride, (k - 1) // 2, groups, bias + return nn.Conv2d(w_in, w_out, k, stride=s, padding=p, groups=g, bias=b) + + +def gap2d(): + """Helper for building a global average pooling layer.""" + return nn.AdaptiveAvgPool2d((1, 1)) + + +def pool2d(k, *, stride=1): + """Helper for building a pool2d layer.""" + assert k % 2 == 1, "Only odd size kernels supported to avoid padding issues." + return nn.MaxPool2d(k, stride=stride, padding=(k - 1) // 2) + + +def init_weights(m): + """Performs ResNet-style weight initialization.""" + if isinstance(m, nn.Conv2d): + # Note that there is no bias due to BN + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + m.weight.data.normal_(mean=0.0, std=np.sqrt(2.0 / fan_out)) + elif isinstance(m, nn.BatchNorm2d): + m.weight.data.fill_(1.0) + m.bias.data.zero_() + elif isinstance(m, nn.Linear): + m.weight.data.normal_(mean=0.0, std=0.01) + m.bias.data.zero_() + + +class ResStem(CNNBlockBase): + """ResNet stem for ImageNet: 7x7, BN, AF, MaxPool.""" + + def __init__(self, w_in, w_out, norm, activation_class): + super().__init__(w_in, w_out, 4) + self.conv = conv2d(w_in, w_out, 7, stride=2) + self.bn = get_norm(norm, w_out) + self.af = activation_class() + self.pool = pool2d(3, stride=2) + + def forward(self, x): + for layer in self.children(): + x = layer(x) + return x + + +class SimpleStem(CNNBlockBase): + """Simple stem for ImageNet: 3x3, BN, AF.""" + + def __init__(self, w_in, w_out, norm, activation_class): + super().__init__(w_in, w_out, 2) + self.conv = conv2d(w_in, w_out, 3, stride=2) + self.bn = get_norm(norm, w_out) + self.af = activation_class() + + def forward(self, x): + for layer in self.children(): + x = layer(x) + return x + + +class SE(nn.Module): + """Squeeze-and-Excitation (SE) block: AvgPool, FC, Act, FC, Sigmoid.""" + + def __init__(self, w_in, w_se, activation_class): + super().__init__() + self.avg_pool = gap2d() + self.f_ex = nn.Sequential( + conv2d(w_in, w_se, 1, bias=True), + activation_class(), + conv2d(w_se, w_in, 1, bias=True), + nn.Sigmoid(), + ) + + def forward(self, x): + return x * self.f_ex(self.avg_pool(x)) + + +class VanillaBlock(CNNBlockBase): + """Vanilla block: [3x3 conv, BN, Relu] x2.""" + + def __init__(self, w_in, w_out, stride, norm, activation_class, _params): + super().__init__(w_in, w_out, stride) + self.a = conv2d(w_in, w_out, 3, stride=stride) + self.a_bn = get_norm(norm, w_out) + self.a_af = activation_class() + self.b = conv2d(w_out, w_out, 3) + self.b_bn = get_norm(norm, w_out) + self.b_af = activation_class() + + def forward(self, x): + for layer in self.children(): + x = layer(x) + return x + + +class BasicTransform(nn.Module): + """Basic transformation: [3x3 conv, BN, Relu] x2.""" + + def __init__(self, w_in, w_out, stride, norm, activation_class, _params): + super().__init__() + self.a = conv2d(w_in, w_out, 3, stride=stride) + self.a_bn = get_norm(norm, w_out) + self.a_af = activation_class() + self.b = conv2d(w_out, w_out, 3) + self.b_bn = get_norm(norm, w_out) + self.b_bn.final_bn = True + + def forward(self, x): + for layer in self.children(): + x = layer(x) + return x + + +class ResBasicBlock(CNNBlockBase): + """Residual basic block: x + f(x), f = basic transform.""" + + def __init__(self, w_in, w_out, stride, norm, activation_class, params): + super().__init__(w_in, w_out, stride) + self.proj, self.bn = None, None + if (w_in != w_out) or (stride != 1): + self.proj = conv2d(w_in, w_out, 1, stride=stride) + self.bn = get_norm(norm, w_out) + self.f = BasicTransform(w_in, w_out, stride, norm, activation_class, params) + self.af = activation_class() + + def forward(self, x): + x_p = self.bn(self.proj(x)) if self.proj else x + return self.af(x_p + self.f(x)) + + +class BottleneckTransform(nn.Module): + """Bottleneck transformation: 1x1, 3x3 [+SE], 1x1.""" + + def __init__(self, w_in, w_out, stride, norm, activation_class, params): + super().__init__() + w_b = int(round(w_out * params["bot_mul"])) + w_se = int(round(w_in * params["se_r"])) + groups = w_b // params["group_w"] + self.a = conv2d(w_in, w_b, 1) + self.a_bn = get_norm(norm, w_b) + self.a_af = activation_class() + self.b = conv2d(w_b, w_b, 3, stride=stride, groups=groups) + self.b_bn = get_norm(norm, w_b) + self.b_af = activation_class() + self.se = SE(w_b, w_se, activation_class) if w_se else None + self.c = conv2d(w_b, w_out, 1) + self.c_bn = get_norm(norm, w_out) + self.c_bn.final_bn = True + + def forward(self, x): + for layer in self.children(): + x = layer(x) + return x + + +class ResBottleneckBlock(CNNBlockBase): + """Residual bottleneck block: x + f(x), f = bottleneck transform.""" + + def __init__(self, w_in, w_out, stride, norm, activation_class, params): + super().__init__(w_in, w_out, stride) + self.proj, self.bn = None, None + if (w_in != w_out) or (stride != 1): + self.proj = conv2d(w_in, w_out, 1, stride=stride) + self.bn = get_norm(norm, w_out) + self.f = BottleneckTransform(w_in, w_out, stride, norm, activation_class, params) + self.af = activation_class() + + def forward(self, x): + x_p = self.bn(self.proj(x)) if self.proj else x + return self.af(x_p + self.f(x)) + + +class AnyStage(nn.Module): + """AnyNet stage (sequence of blocks w/ the same output shape).""" + + def __init__(self, w_in, w_out, stride, d, block_class, norm, activation_class, params): + super().__init__() + for i in range(d): + block = block_class(w_in, w_out, stride, norm, activation_class, params) + self.add_module("b{}".format(i + 1), block) + stride, w_in = 1, w_out + + def forward(self, x): + for block in self.children(): + x = block(x) + return x + + +class AnyNet(Backbone): + """AnyNet model. See :paper:`dds`.""" + + def __init__( + self, + *, + stem_class, + stem_width, + block_class, + depths, + widths, + group_widths, + strides, + bottleneck_ratios, + se_ratio, + activation_class, + freeze_at=0, + norm="BN", + out_features=None, + ): + """ + Args: + stem_class (callable): A callable taking 4 arguments (channels in, channels out, + normalization, callable returning an activation function) that returns another + callable implementing the stem module. + stem_width (int): The number of output channels that the stem produces. + block_class (callable): A callable taking 6 arguments (channels in, channels out, + stride, normalization, callable returning an activation function, a dict of + block-specific parameters) that returns another callable implementing the repeated + block module. + depths (list[int]): Number of blocks in each stage. + widths (list[int]): For each stage, the number of output channels of each block. + group_widths (list[int]): For each stage, the number of channels per group in group + convolution, if the block uses group convolution. + strides (list[int]): The stride that each network stage applies to its input. + bottleneck_ratios (list[float]): For each stage, the ratio of the number of bottleneck + channels to the number of block input channels (or, equivalently, output channels), + if the block uses a bottleneck. + se_ratio (float): The ratio of the number of channels used inside the squeeze-excitation + (SE) module to it number of input channels, if SE the block uses SE. + activation_class (callable): A callable taking no arguments that returns another + callable implementing an activation function. + freeze_at (int): The number of stages at the beginning to freeze. + see :meth:`freeze` for detailed explanation. + norm (str or callable): normalization for all conv layers. + See :func:`layers.get_norm` for supported format. + out_features (list[str]): name of the layers whose outputs should + be returned in forward. RegNet's use "stem" and "s1", "s2", etc for the stages after + the stem. If None, will return the output of the last layer. + """ + super().__init__() + self.stem = stem_class(3, stem_width, norm, activation_class) + + current_stride = self.stem.stride + self._out_feature_strides = {"stem": current_stride} + self._out_feature_channels = {"stem": self.stem.out_channels} + self.stages_and_names = [] + prev_w = stem_width + + for i, (d, w, s, b, g) in enumerate( + zip(depths, widths, strides, bottleneck_ratios, group_widths) + ): + params = {"bot_mul": b, "group_w": g, "se_r": se_ratio} + stage = AnyStage(prev_w, w, s, d, block_class, norm, activation_class, params) + name = "s{}".format(i + 1) + self.add_module(name, stage) + self.stages_and_names.append((stage, name)) + self._out_feature_strides[name] = current_stride = int( + current_stride * np.prod([k.stride for k in stage.children()]) + ) + self._out_feature_channels[name] = list(stage.children())[-1].out_channels + prev_w = w + + self.apply(init_weights) + + if out_features is None: + out_features = [name] + self._out_features = out_features + assert len(self._out_features) + children = [x[0] for x in self.named_children()] + for out_feature in self._out_features: + assert out_feature in children, "Available children: {} does not include {}".format( + ", ".join(children), out_feature + ) + self.freeze(freeze_at) + + def forward(self, x): + """ + Args: + x: Tensor of shape (N,C,H,W). H, W must be a multiple of ``self.size_divisibility``. + + Returns: + dict[str->Tensor]: names and the corresponding features + """ + assert x.dim() == 4, f"Model takes an input of shape (N, C, H, W). Got {x.shape} instead!" + outputs = {} + x = self.stem(x) + if "stem" in self._out_features: + outputs["stem"] = x + for stage, name in self.stages_and_names: + x = stage(x) + if name in self._out_features: + outputs[name] = x + return outputs + + def output_shape(self): + return { + name: ShapeSpec( + channels=self._out_feature_channels[name], stride=self._out_feature_strides[name] + ) + for name in self._out_features + } + + def freeze(self, freeze_at=0): + """ + Freeze the first several stages of the model. Commonly used in fine-tuning. + + Layers that produce the same feature map spatial size are defined as one + "stage" by :paper:`FPN`. + + Args: + freeze_at (int): number of stages to freeze. + `1` means freezing the stem. `2` means freezing the stem and + one residual stage, etc. + + Returns: + nn.Module: this model itself + """ + if freeze_at >= 1: + self.stem.freeze() + for idx, (stage, _) in enumerate(self.stages_and_names, start=2): + if freeze_at >= idx: + for block in stage.children(): + block.freeze() + return self + + +def adjust_block_compatibility(ws, bs, gs): + """Adjusts the compatibility of widths, bottlenecks, and groups.""" + assert len(ws) == len(bs) == len(gs) + assert all(w > 0 and b > 0 and g > 0 for w, b, g in zip(ws, bs, gs)) + vs = [int(max(1, w * b)) for w, b in zip(ws, bs)] + gs = [int(min(g, v)) for g, v in zip(gs, vs)] + ms = [np.lcm(g, b) if b > 1 else g for g, b in zip(gs, bs)] + vs = [max(m, int(round(v / m) * m)) for v, m in zip(vs, ms)] + ws = [int(v / b) for v, b in zip(vs, bs)] + assert all(w * b % g == 0 for w, b, g in zip(ws, bs, gs)) + return ws, bs, gs + + +def generate_regnet_parameters(w_a, w_0, w_m, d, q=8): + """Generates per stage widths and depths from RegNet parameters.""" + assert w_a >= 0 and w_0 > 0 and w_m > 1 and w_0 % q == 0 + # Generate continuous per-block ws + ws_cont = np.arange(d) * w_a + w_0 + # Generate quantized per-block ws + ks = np.round(np.log(ws_cont / w_0) / np.log(w_m)) + ws_all = w_0 * np.power(w_m, ks) + ws_all = np.round(np.divide(ws_all, q)).astype(int) * q + # Generate per stage ws and ds (assumes ws_all are sorted) + ws, ds = np.unique(ws_all, return_counts=True) + # Compute number of actual stages and total possible stages + num_stages, total_stages = len(ws), ks.max() + 1 + # Convert numpy arrays to lists and return + ws, ds, ws_all, ws_cont = (x.tolist() for x in (ws, ds, ws_all, ws_cont)) + return ws, ds, num_stages, total_stages, ws_all, ws_cont + + +class RegNet(AnyNet): + """RegNet model. See :paper:`dds`.""" + + def __init__( + self, + *, + stem_class, + stem_width, + block_class, + depth, + w_a, + w_0, + w_m, + group_width, + stride=2, + bottleneck_ratio=1.0, + se_ratio=0.0, + activation_class=None, + freeze_at=0, + norm="BN", + out_features=None, + ): + """ + Build a RegNet from the parameterization described in :paper:`dds` Section 3.3. + + Args: + See :class:`AnyNet` for arguments that are not listed here. + depth (int): Total number of blocks in the RegNet. + w_a (float): Factor by which block width would increase prior to quantizing block widths + by stage. See :paper:`dds` Section 3.3. + w_0 (int): Initial block width. See :paper:`dds` Section 3.3. + w_m (float): Parameter controlling block width quantization. + See :paper:`dds` Section 3.3. + group_width (int): Number of channels per group in group convolution, if the block uses + group convolution. + bottleneck_ratio (float): The ratio of the number of bottleneck channels to the number + of block input channels (or, equivalently, output channels), if the block uses a + bottleneck. + stride (int): The stride that each network stage applies to its input. + """ + ws, ds = generate_regnet_parameters(w_a, w_0, w_m, depth)[0:2] + ss = [stride for _ in ws] + bs = [bottleneck_ratio for _ in ws] + gs = [group_width for _ in ws] + ws, bs, gs = adjust_block_compatibility(ws, bs, gs) + + def default_activation_class(): + return nn.ReLU(inplace=True) + + super().__init__( + stem_class=stem_class, + stem_width=stem_width, + block_class=block_class, + depths=ds, + widths=ws, + strides=ss, + group_widths=gs, + bottleneck_ratios=bs, + se_ratio=se_ratio, + activation_class=default_activation_class + if activation_class is None + else activation_class, + freeze_at=freeze_at, + norm=norm, + out_features=out_features, + ) diff --git a/annotator/oneformer/detectron2/modeling/backbone/resnet.py b/annotator/oneformer/detectron2/modeling/backbone/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..34d6edf2e2ec3515ed1a395658ded85c280000b0 --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/backbone/resnet.py @@ -0,0 +1,694 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import numpy as np +import fvcore.nn.weight_init as weight_init +import torch +import torch.nn.functional as F +from torch import nn + +from annotator.oneformer.detectron2.layers import ( + CNNBlockBase, + Conv2d, + DeformConv, + ModulatedDeformConv, + ShapeSpec, + get_norm, +) + +from .backbone import Backbone +from .build import BACKBONE_REGISTRY + +__all__ = [ + "ResNetBlockBase", + "BasicBlock", + "BottleneckBlock", + "DeformBottleneckBlock", + "BasicStem", + "ResNet", + "make_stage", + "build_resnet_backbone", +] + + +class BasicBlock(CNNBlockBase): + """ + The basic residual block for ResNet-18 and ResNet-34 defined in :paper:`ResNet`, + with two 3x3 conv layers and a projection shortcut if needed. + """ + + def __init__(self, in_channels, out_channels, *, stride=1, norm="BN"): + """ + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + stride (int): Stride for the first conv. + norm (str or callable): normalization for all conv layers. + See :func:`layers.get_norm` for supported format. + """ + super().__init__(in_channels, out_channels, stride) + + if in_channels != out_channels: + self.shortcut = Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=stride, + bias=False, + norm=get_norm(norm, out_channels), + ) + else: + self.shortcut = None + + self.conv1 = Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=stride, + padding=1, + bias=False, + norm=get_norm(norm, out_channels), + ) + + self.conv2 = Conv2d( + out_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=False, + norm=get_norm(norm, out_channels), + ) + + for layer in [self.conv1, self.conv2, self.shortcut]: + if layer is not None: # shortcut can be None + weight_init.c2_msra_fill(layer) + + def forward(self, x): + out = self.conv1(x) + out = F.relu_(out) + out = self.conv2(out) + + if self.shortcut is not None: + shortcut = self.shortcut(x) + else: + shortcut = x + + out += shortcut + out = F.relu_(out) + return out + + +class BottleneckBlock(CNNBlockBase): + """ + The standard bottleneck residual block used by ResNet-50, 101 and 152 + defined in :paper:`ResNet`. It contains 3 conv layers with kernels + 1x1, 3x3, 1x1, and a projection shortcut if needed. + """ + + def __init__( + self, + in_channels, + out_channels, + *, + bottleneck_channels, + stride=1, + num_groups=1, + norm="BN", + stride_in_1x1=False, + dilation=1, + ): + """ + Args: + bottleneck_channels (int): number of output channels for the 3x3 + "bottleneck" conv layers. + num_groups (int): number of groups for the 3x3 conv layer. + norm (str or callable): normalization for all conv layers. + See :func:`layers.get_norm` for supported format. + stride_in_1x1 (bool): when stride>1, whether to put stride in the + first 1x1 convolution or the bottleneck 3x3 convolution. + dilation (int): the dilation rate of the 3x3 conv layer. + """ + super().__init__(in_channels, out_channels, stride) + + if in_channels != out_channels: + self.shortcut = Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=stride, + bias=False, + norm=get_norm(norm, out_channels), + ) + else: + self.shortcut = None + + # The original MSRA ResNet models have stride in the first 1x1 conv + # The subsequent fb.torch.resnet and Caffe2 ResNe[X]t implementations have + # stride in the 3x3 conv + stride_1x1, stride_3x3 = (stride, 1) if stride_in_1x1 else (1, stride) + + self.conv1 = Conv2d( + in_channels, + bottleneck_channels, + kernel_size=1, + stride=stride_1x1, + bias=False, + norm=get_norm(norm, bottleneck_channels), + ) + + self.conv2 = Conv2d( + bottleneck_channels, + bottleneck_channels, + kernel_size=3, + stride=stride_3x3, + padding=1 * dilation, + bias=False, + groups=num_groups, + dilation=dilation, + norm=get_norm(norm, bottleneck_channels), + ) + + self.conv3 = Conv2d( + bottleneck_channels, + out_channels, + kernel_size=1, + bias=False, + norm=get_norm(norm, out_channels), + ) + + for layer in [self.conv1, self.conv2, self.conv3, self.shortcut]: + if layer is not None: # shortcut can be None + weight_init.c2_msra_fill(layer) + + # Zero-initialize the last normalization in each residual branch, + # so that at the beginning, the residual branch starts with zeros, + # and each residual block behaves like an identity. + # See Sec 5.1 in "Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour": + # "For BN layers, the learnable scaling coefficient γ is initialized + # to be 1, except for each residual block's last BN + # where γ is initialized to be 0." + + # nn.init.constant_(self.conv3.norm.weight, 0) + # TODO this somehow hurts performance when training GN models from scratch. + # Add it as an option when we need to use this code to train a backbone. + + def forward(self, x): + out = self.conv1(x) + out = F.relu_(out) + + out = self.conv2(out) + out = F.relu_(out) + + out = self.conv3(out) + + if self.shortcut is not None: + shortcut = self.shortcut(x) + else: + shortcut = x + + out += shortcut + out = F.relu_(out) + return out + + +class DeformBottleneckBlock(CNNBlockBase): + """ + Similar to :class:`BottleneckBlock`, but with :paper:`deformable conv ` + in the 3x3 convolution. + """ + + def __init__( + self, + in_channels, + out_channels, + *, + bottleneck_channels, + stride=1, + num_groups=1, + norm="BN", + stride_in_1x1=False, + dilation=1, + deform_modulated=False, + deform_num_groups=1, + ): + super().__init__(in_channels, out_channels, stride) + self.deform_modulated = deform_modulated + + if in_channels != out_channels: + self.shortcut = Conv2d( + in_channels, + out_channels, + kernel_size=1, + stride=stride, + bias=False, + norm=get_norm(norm, out_channels), + ) + else: + self.shortcut = None + + stride_1x1, stride_3x3 = (stride, 1) if stride_in_1x1 else (1, stride) + + self.conv1 = Conv2d( + in_channels, + bottleneck_channels, + kernel_size=1, + stride=stride_1x1, + bias=False, + norm=get_norm(norm, bottleneck_channels), + ) + + if deform_modulated: + deform_conv_op = ModulatedDeformConv + # offset channels are 2 or 3 (if with modulated) * kernel_size * kernel_size + offset_channels = 27 + else: + deform_conv_op = DeformConv + offset_channels = 18 + + self.conv2_offset = Conv2d( + bottleneck_channels, + offset_channels * deform_num_groups, + kernel_size=3, + stride=stride_3x3, + padding=1 * dilation, + dilation=dilation, + ) + self.conv2 = deform_conv_op( + bottleneck_channels, + bottleneck_channels, + kernel_size=3, + stride=stride_3x3, + padding=1 * dilation, + bias=False, + groups=num_groups, + dilation=dilation, + deformable_groups=deform_num_groups, + norm=get_norm(norm, bottleneck_channels), + ) + + self.conv3 = Conv2d( + bottleneck_channels, + out_channels, + kernel_size=1, + bias=False, + norm=get_norm(norm, out_channels), + ) + + for layer in [self.conv1, self.conv2, self.conv3, self.shortcut]: + if layer is not None: # shortcut can be None + weight_init.c2_msra_fill(layer) + + nn.init.constant_(self.conv2_offset.weight, 0) + nn.init.constant_(self.conv2_offset.bias, 0) + + def forward(self, x): + out = self.conv1(x) + out = F.relu_(out) + + if self.deform_modulated: + offset_mask = self.conv2_offset(out) + offset_x, offset_y, mask = torch.chunk(offset_mask, 3, dim=1) + offset = torch.cat((offset_x, offset_y), dim=1) + mask = mask.sigmoid() + out = self.conv2(out, offset, mask) + else: + offset = self.conv2_offset(out) + out = self.conv2(out, offset) + out = F.relu_(out) + + out = self.conv3(out) + + if self.shortcut is not None: + shortcut = self.shortcut(x) + else: + shortcut = x + + out += shortcut + out = F.relu_(out) + return out + + +class BasicStem(CNNBlockBase): + """ + The standard ResNet stem (layers before the first residual block), + with a conv, relu and max_pool. + """ + + def __init__(self, in_channels=3, out_channels=64, norm="BN"): + """ + Args: + norm (str or callable): norm after the first conv layer. + See :func:`layers.get_norm` for supported format. + """ + super().__init__(in_channels, out_channels, 4) + self.in_channels = in_channels + self.conv1 = Conv2d( + in_channels, + out_channels, + kernel_size=7, + stride=2, + padding=3, + bias=False, + norm=get_norm(norm, out_channels), + ) + weight_init.c2_msra_fill(self.conv1) + + def forward(self, x): + x = self.conv1(x) + x = F.relu_(x) + x = F.max_pool2d(x, kernel_size=3, stride=2, padding=1) + return x + + +class ResNet(Backbone): + """ + Implement :paper:`ResNet`. + """ + + def __init__(self, stem, stages, num_classes=None, out_features=None, freeze_at=0): + """ + Args: + stem (nn.Module): a stem module + stages (list[list[CNNBlockBase]]): several (typically 4) stages, + each contains multiple :class:`CNNBlockBase`. + num_classes (None or int): if None, will not perform classification. + Otherwise, will create a linear layer. + out_features (list[str]): name of the layers whose outputs should + be returned in forward. Can be anything in "stem", "linear", or "res2" ... + If None, will return the output of the last layer. + freeze_at (int): The number of stages at the beginning to freeze. + see :meth:`freeze` for detailed explanation. + """ + super().__init__() + self.stem = stem + self.num_classes = num_classes + + current_stride = self.stem.stride + self._out_feature_strides = {"stem": current_stride} + self._out_feature_channels = {"stem": self.stem.out_channels} + + self.stage_names, self.stages = [], [] + + if out_features is not None: + # Avoid keeping unused layers in this module. They consume extra memory + # and may cause allreduce to fail + num_stages = max( + [{"res2": 1, "res3": 2, "res4": 3, "res5": 4}.get(f, 0) for f in out_features] + ) + stages = stages[:num_stages] + for i, blocks in enumerate(stages): + assert len(blocks) > 0, len(blocks) + for block in blocks: + assert isinstance(block, CNNBlockBase), block + + name = "res" + str(i + 2) + stage = nn.Sequential(*blocks) + + self.add_module(name, stage) + self.stage_names.append(name) + self.stages.append(stage) + + self._out_feature_strides[name] = current_stride = int( + current_stride * np.prod([k.stride for k in blocks]) + ) + self._out_feature_channels[name] = curr_channels = blocks[-1].out_channels + self.stage_names = tuple(self.stage_names) # Make it static for scripting + + if num_classes is not None: + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.linear = nn.Linear(curr_channels, num_classes) + + # Sec 5.1 in "Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour": + # "The 1000-way fully-connected layer is initialized by + # drawing weights from a zero-mean Gaussian with standard deviation of 0.01." + nn.init.normal_(self.linear.weight, std=0.01) + name = "linear" + + if out_features is None: + out_features = [name] + self._out_features = out_features + assert len(self._out_features) + children = [x[0] for x in self.named_children()] + for out_feature in self._out_features: + assert out_feature in children, "Available children: {}".format(", ".join(children)) + self.freeze(freeze_at) + + def forward(self, x): + """ + Args: + x: Tensor of shape (N,C,H,W). H, W must be a multiple of ``self.size_divisibility``. + + Returns: + dict[str->Tensor]: names and the corresponding features + """ + assert x.dim() == 4, f"ResNet takes an input of shape (N, C, H, W). Got {x.shape} instead!" + outputs = {} + x = self.stem(x) + if "stem" in self._out_features: + outputs["stem"] = x + for name, stage in zip(self.stage_names, self.stages): + x = stage(x) + if name in self._out_features: + outputs[name] = x + if self.num_classes is not None: + x = self.avgpool(x) + x = torch.flatten(x, 1) + x = self.linear(x) + if "linear" in self._out_features: + outputs["linear"] = x + return outputs + + def output_shape(self): + return { + name: ShapeSpec( + channels=self._out_feature_channels[name], stride=self._out_feature_strides[name] + ) + for name in self._out_features + } + + def freeze(self, freeze_at=0): + """ + Freeze the first several stages of the ResNet. Commonly used in + fine-tuning. + + Layers that produce the same feature map spatial size are defined as one + "stage" by :paper:`FPN`. + + Args: + freeze_at (int): number of stages to freeze. + `1` means freezing the stem. `2` means freezing the stem and + one residual stage, etc. + + Returns: + nn.Module: this ResNet itself + """ + if freeze_at >= 1: + self.stem.freeze() + for idx, stage in enumerate(self.stages, start=2): + if freeze_at >= idx: + for block in stage.children(): + block.freeze() + return self + + @staticmethod + def make_stage(block_class, num_blocks, *, in_channels, out_channels, **kwargs): + """ + Create a list of blocks of the same type that forms one ResNet stage. + + Args: + block_class (type): a subclass of CNNBlockBase that's used to create all blocks in this + stage. A module of this type must not change spatial resolution of inputs unless its + stride != 1. + num_blocks (int): number of blocks in this stage + in_channels (int): input channels of the entire stage. + out_channels (int): output channels of **every block** in the stage. + kwargs: other arguments passed to the constructor of + `block_class`. If the argument name is "xx_per_block", the + argument is a list of values to be passed to each block in the + stage. Otherwise, the same argument is passed to every block + in the stage. + + Returns: + list[CNNBlockBase]: a list of block module. + + Examples: + :: + stage = ResNet.make_stage( + BottleneckBlock, 3, in_channels=16, out_channels=64, + bottleneck_channels=16, num_groups=1, + stride_per_block=[2, 1, 1], + dilations_per_block=[1, 1, 2] + ) + + Usually, layers that produce the same feature map spatial size are defined as one + "stage" (in :paper:`FPN`). Under such definition, ``stride_per_block[1:]`` should + all be 1. + """ + blocks = [] + for i in range(num_blocks): + curr_kwargs = {} + for k, v in kwargs.items(): + if k.endswith("_per_block"): + assert len(v) == num_blocks, ( + f"Argument '{k}' of make_stage should have the " + f"same length as num_blocks={num_blocks}." + ) + newk = k[: -len("_per_block")] + assert newk not in kwargs, f"Cannot call make_stage with both {k} and {newk}!" + curr_kwargs[newk] = v[i] + else: + curr_kwargs[k] = v + + blocks.append( + block_class(in_channels=in_channels, out_channels=out_channels, **curr_kwargs) + ) + in_channels = out_channels + return blocks + + @staticmethod + def make_default_stages(depth, block_class=None, **kwargs): + """ + Created list of ResNet stages from pre-defined depth (one of 18, 34, 50, 101, 152). + If it doesn't create the ResNet variant you need, please use :meth:`make_stage` + instead for fine-grained customization. + + Args: + depth (int): depth of ResNet + block_class (type): the CNN block class. Has to accept + `bottleneck_channels` argument for depth > 50. + By default it is BasicBlock or BottleneckBlock, based on the + depth. + kwargs: + other arguments to pass to `make_stage`. Should not contain + stride and channels, as they are predefined for each depth. + + Returns: + list[list[CNNBlockBase]]: modules in all stages; see arguments of + :class:`ResNet.__init__`. + """ + num_blocks_per_stage = { + 18: [2, 2, 2, 2], + 34: [3, 4, 6, 3], + 50: [3, 4, 6, 3], + 101: [3, 4, 23, 3], + 152: [3, 8, 36, 3], + }[depth] + if block_class is None: + block_class = BasicBlock if depth < 50 else BottleneckBlock + if depth < 50: + in_channels = [64, 64, 128, 256] + out_channels = [64, 128, 256, 512] + else: + in_channels = [64, 256, 512, 1024] + out_channels = [256, 512, 1024, 2048] + ret = [] + for (n, s, i, o) in zip(num_blocks_per_stage, [1, 2, 2, 2], in_channels, out_channels): + if depth >= 50: + kwargs["bottleneck_channels"] = o // 4 + ret.append( + ResNet.make_stage( + block_class=block_class, + num_blocks=n, + stride_per_block=[s] + [1] * (n - 1), + in_channels=i, + out_channels=o, + **kwargs, + ) + ) + return ret + + +ResNetBlockBase = CNNBlockBase +""" +Alias for backward compatibiltiy. +""" + + +def make_stage(*args, **kwargs): + """ + Deprecated alias for backward compatibiltiy. + """ + return ResNet.make_stage(*args, **kwargs) + + +@BACKBONE_REGISTRY.register() +def build_resnet_backbone(cfg, input_shape): + """ + Create a ResNet instance from config. + + Returns: + ResNet: a :class:`ResNet` instance. + """ + # need registration of new blocks/stems? + norm = cfg.MODEL.RESNETS.NORM + stem = BasicStem( + in_channels=input_shape.channels, + out_channels=cfg.MODEL.RESNETS.STEM_OUT_CHANNELS, + norm=norm, + ) + + # fmt: off + freeze_at = cfg.MODEL.BACKBONE.FREEZE_AT + out_features = cfg.MODEL.RESNETS.OUT_FEATURES + depth = cfg.MODEL.RESNETS.DEPTH + num_groups = cfg.MODEL.RESNETS.NUM_GROUPS + width_per_group = cfg.MODEL.RESNETS.WIDTH_PER_GROUP + bottleneck_channels = num_groups * width_per_group + in_channels = cfg.MODEL.RESNETS.STEM_OUT_CHANNELS + out_channels = cfg.MODEL.RESNETS.RES2_OUT_CHANNELS + stride_in_1x1 = cfg.MODEL.RESNETS.STRIDE_IN_1X1 + res5_dilation = cfg.MODEL.RESNETS.RES5_DILATION + deform_on_per_stage = cfg.MODEL.RESNETS.DEFORM_ON_PER_STAGE + deform_modulated = cfg.MODEL.RESNETS.DEFORM_MODULATED + deform_num_groups = cfg.MODEL.RESNETS.DEFORM_NUM_GROUPS + # fmt: on + assert res5_dilation in {1, 2}, "res5_dilation cannot be {}.".format(res5_dilation) + + num_blocks_per_stage = { + 18: [2, 2, 2, 2], + 34: [3, 4, 6, 3], + 50: [3, 4, 6, 3], + 101: [3, 4, 23, 3], + 152: [3, 8, 36, 3], + }[depth] + + if depth in [18, 34]: + assert out_channels == 64, "Must set MODEL.RESNETS.RES2_OUT_CHANNELS = 64 for R18/R34" + assert not any( + deform_on_per_stage + ), "MODEL.RESNETS.DEFORM_ON_PER_STAGE unsupported for R18/R34" + assert res5_dilation == 1, "Must set MODEL.RESNETS.RES5_DILATION = 1 for R18/R34" + assert num_groups == 1, "Must set MODEL.RESNETS.NUM_GROUPS = 1 for R18/R34" + + stages = [] + + for idx, stage_idx in enumerate(range(2, 6)): + # res5_dilation is used this way as a convention in R-FCN & Deformable Conv paper + dilation = res5_dilation if stage_idx == 5 else 1 + first_stride = 1 if idx == 0 or (stage_idx == 5 and dilation == 2) else 2 + stage_kargs = { + "num_blocks": num_blocks_per_stage[idx], + "stride_per_block": [first_stride] + [1] * (num_blocks_per_stage[idx] - 1), + "in_channels": in_channels, + "out_channels": out_channels, + "norm": norm, + } + # Use BasicBlock for R18 and R34. + if depth in [18, 34]: + stage_kargs["block_class"] = BasicBlock + else: + stage_kargs["bottleneck_channels"] = bottleneck_channels + stage_kargs["stride_in_1x1"] = stride_in_1x1 + stage_kargs["dilation"] = dilation + stage_kargs["num_groups"] = num_groups + if deform_on_per_stage[idx]: + stage_kargs["block_class"] = DeformBottleneckBlock + stage_kargs["deform_modulated"] = deform_modulated + stage_kargs["deform_num_groups"] = deform_num_groups + else: + stage_kargs["block_class"] = BottleneckBlock + blocks = ResNet.make_stage(**stage_kargs) + in_channels = out_channels + out_channels *= 2 + bottleneck_channels *= 2 + stages.append(blocks) + return ResNet(stem, stages, out_features=out_features, freeze_at=freeze_at) diff --git a/annotator/oneformer/detectron2/modeling/backbone/swin.py b/annotator/oneformer/detectron2/modeling/backbone/swin.py new file mode 100644 index 0000000000000000000000000000000000000000..d5a651d6f4d2933e8f329bd13c04286488f25753 --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/backbone/swin.py @@ -0,0 +1,695 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Implementation of Swin models from :paper:`swin`. + +This code is adapted from https://github.com/SwinTransformer/Swin-Transformer-Object-Detection/blob/master/mmdet/models/backbones/swin_transformer.py with minimal modifications. # noqa +-------------------------------------------------------- +Swin Transformer +Copyright (c) 2021 Microsoft +Licensed under The MIT License [see LICENSE for details] +Written by Ze Liu, Yutong Lin, Yixuan Wei +-------------------------------------------------------- +LICENSE: https://github.com/SwinTransformer/Swin-Transformer-Object-Detection/blob/461e003166a8083d0b620beacd4662a2df306bd6/LICENSE +""" + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint + +from annotator.oneformer.detectron2.modeling.backbone.backbone import Backbone + +_to_2tuple = nn.modules.utils._ntuple(2) + + +class Mlp(nn.Module): + """Multilayer perceptron.""" + + def __init__( + self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.0 + ): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +def window_partition(x, window_size): + """ + Args: + x: (B, H, W, C) + window_size (int): window size + Returns: + windows: (num_windows*B, window_size, window_size, C) + """ + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + return windows + + +def window_reverse(windows, window_size, H, W): + """ + Args: + windows: (num_windows*B, window_size, window_size, C) + window_size (int): Window size + H (int): Height of image + W (int): Width of image + Returns: + x: (B, H, W, C) + """ + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +class WindowAttention(nn.Module): + """Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. + Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__( + self, + dim, + window_size, + num_heads, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + ): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + + # define a parameter table of relative position bias + self.relative_position_bias_table = nn.Parameter( + torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads) + ) # 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(self.window_size[0]) + coords_w = torch.arange(self.window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer("relative_position_index", relative_position_index) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + nn.init.trunc_normal_(self.relative_position_bias_table, std=0.02) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, mask=None): + """Forward function. + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None + """ + B_, N, C = x.shape + qkv = ( + self.qkv(x) + .reshape(B_, N, 3, self.num_heads, C // self.num_heads) + .permute(2, 0, 3, 1, 4) + ) + q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = q @ k.transpose(-2, -1) + + relative_position_bias = self.relative_position_bias_table[ + self.relative_position_index.view(-1) + ].view( + self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1 + ) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute( + 2, 0, 1 + ).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class SwinTransformerBlock(nn.Module): + """Swin Transformer Block. + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__( + self, + dim, + num_heads, + window_size=7, + shift_size=0, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop=0.0, + attn_drop=0.0, + drop_path=0.0, + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + ): + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size" + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention( + dim, + window_size=_to_2tuple(self.window_size), + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop, + ) + + if drop_path > 0.0: + from timm.models.layers import DropPath + + self.drop_path = DropPath(drop_path) + else: + self.drop_path = nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop + ) + + self.H = None + self.W = None + + def forward(self, x, mask_matrix): + """Forward function. + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + mask_matrix: Attention mask for cyclic shift. + """ + B, L, C = x.shape + H, W = self.H, self.W + assert L == H * W, "input feature has wrong size" + + shortcut = x + x = self.norm1(x) + x = x.view(B, H, W, C) + + # pad feature maps to multiples of window size + pad_l = pad_t = 0 + pad_r = (self.window_size - W % self.window_size) % self.window_size + pad_b = (self.window_size - H % self.window_size) % self.window_size + x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b)) + _, Hp, Wp, _ = x.shape + + # cyclic shift + if self.shift_size > 0: + shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) + attn_mask = mask_matrix + else: + shifted_x = x + attn_mask = None + + # partition windows + x_windows = window_partition( + shifted_x, self.window_size + ) # nW*B, window_size, window_size, C + x_windows = x_windows.view( + -1, self.window_size * self.window_size, C + ) # nW*B, window_size*window_size, C + + # W-MSA/SW-MSA + attn_windows = self.attn(x_windows, mask=attn_mask) # nW*B, window_size*window_size, C + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C) + shifted_x = window_reverse(attn_windows, self.window_size, Hp, Wp) # B H' W' C + + # reverse cyclic shift + if self.shift_size > 0: + x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2)) + else: + x = shifted_x + + if pad_r > 0 or pad_b > 0: + x = x[:, :H, :W, :].contiguous() + + x = x.view(B, H * W, C) + + # FFN + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + return x + + +class PatchMerging(nn.Module): + """Patch Merging Layer + Args: + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, dim, norm_layer=nn.LayerNorm): + super().__init__() + self.dim = dim + self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False) + self.norm = norm_layer(4 * dim) + + def forward(self, x, H, W): + """Forward function. + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + """ + B, L, C = x.shape + assert L == H * W, "input feature has wrong size" + + x = x.view(B, H, W, C) + + # padding + pad_input = (H % 2 == 1) or (W % 2 == 1) + if pad_input: + x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2)) + + x0 = x[:, 0::2, 0::2, :] # B H/2 W/2 C + x1 = x[:, 1::2, 0::2, :] # B H/2 W/2 C + x2 = x[:, 0::2, 1::2, :] # B H/2 W/2 C + x3 = x[:, 1::2, 1::2, :] # B H/2 W/2 C + x = torch.cat([x0, x1, x2, x3], -1) # B H/2 W/2 4*C + x = x.view(B, -1, 4 * C) # B H/2*W/2 4*C + + x = self.norm(x) + x = self.reduction(x) + + return x + + +class BasicLayer(nn.Module): + """A basic Swin Transformer layer for one stage. + Args: + dim (int): Number of feature channels + depth (int): Depths of this stage. + num_heads (int): Number of attention head. + window_size (int): Local window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. + Default: None + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__( + self, + dim, + depth, + num_heads, + window_size=7, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop=0.0, + attn_drop=0.0, + drop_path=0.0, + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False, + ): + super().__init__() + self.window_size = window_size + self.shift_size = window_size // 2 + self.depth = depth + self.use_checkpoint = use_checkpoint + + # build blocks + self.blocks = nn.ModuleList( + [ + SwinTransformerBlock( + dim=dim, + num_heads=num_heads, + window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer, + ) + for i in range(depth) + ] + ) + + # patch merging layer + if downsample is not None: + self.downsample = downsample(dim=dim, norm_layer=norm_layer) + else: + self.downsample = None + + def forward(self, x, H, W): + """Forward function. + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + """ + + # calculate attention mask for SW-MSA + Hp = int(np.ceil(H / self.window_size)) * self.window_size + Wp = int(np.ceil(W / self.window_size)) * self.window_size + img_mask = torch.zeros((1, Hp, Wp, 1), device=x.device) # 1 Hp Wp 1 + h_slices = ( + slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None), + ) + w_slices = ( + slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None), + ) + cnt = 0 + for h in h_slices: + for w in w_slices: + img_mask[:, h, w, :] = cnt + cnt += 1 + + mask_windows = window_partition( + img_mask, self.window_size + ) # nW, window_size, window_size, 1 + mask_windows = mask_windows.view(-1, self.window_size * self.window_size) + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill( + attn_mask == 0, float(0.0) + ) + + for blk in self.blocks: + blk.H, blk.W = H, W + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x, attn_mask) + else: + x = blk(x, attn_mask) + if self.downsample is not None: + x_down = self.downsample(x, H, W) + Wh, Ww = (H + 1) // 2, (W + 1) // 2 + return x, H, W, x_down, Wh, Ww + else: + return x, H, W, x, H, W + + +class PatchEmbed(nn.Module): + """Image to Patch Embedding + Args: + patch_size (int): Patch token size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__(self, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None): + super().__init__() + patch_size = _to_2tuple(patch_size) + self.patch_size = patch_size + + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + if norm_layer is not None: + self.norm = norm_layer(embed_dim) + else: + self.norm = None + + def forward(self, x): + """Forward function.""" + # padding + _, _, H, W = x.size() + if W % self.patch_size[1] != 0: + x = F.pad(x, (0, self.patch_size[1] - W % self.patch_size[1])) + if H % self.patch_size[0] != 0: + x = F.pad(x, (0, 0, 0, self.patch_size[0] - H % self.patch_size[0])) + + x = self.proj(x) # B C Wh Ww + if self.norm is not None: + Wh, Ww = x.size(2), x.size(3) + x = x.flatten(2).transpose(1, 2) + x = self.norm(x) + x = x.transpose(1, 2).view(-1, self.embed_dim, Wh, Ww) + + return x + + +class SwinTransformer(Backbone): + """Swin Transformer backbone. + A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted + Windows` - https://arxiv.org/pdf/2103.14030 + Args: + pretrain_img_size (int): Input image size for training the pretrained model, + used in absolute postion embedding. Default 224. + patch_size (int | tuple(int)): Patch size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + depths (tuple[int]): Depths of each Swin Transformer stage. + num_heads (tuple[int]): Number of attention head of each stage. + window_size (int): Window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. + drop_rate (float): Dropout rate. + attn_drop_rate (float): Attention dropout rate. Default: 0. + drop_path_rate (float): Stochastic depth rate. Default: 0.2. + norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm. + ape (bool): If True, add absolute position embedding to the patch embedding. Default: False. + patch_norm (bool): If True, add normalization after patch embedding. Default: True. + out_indices (Sequence[int]): Output from which stages. + frozen_stages (int): Stages to be frozen (stop grad and set eval mode). + -1 means not freezing any parameters. + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__( + self, + pretrain_img_size=224, + patch_size=4, + in_chans=3, + embed_dim=96, + depths=(2, 2, 6, 2), + num_heads=(3, 6, 12, 24), + window_size=7, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.2, + norm_layer=nn.LayerNorm, + ape=False, + patch_norm=True, + out_indices=(0, 1, 2, 3), + frozen_stages=-1, + use_checkpoint=False, + ): + super().__init__() + + self.pretrain_img_size = pretrain_img_size + self.num_layers = len(depths) + self.embed_dim = embed_dim + self.ape = ape + self.patch_norm = patch_norm + self.out_indices = out_indices + self.frozen_stages = frozen_stages + + # split image into non-overlapping patches + self.patch_embed = PatchEmbed( + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None, + ) + + # absolute position embedding + if self.ape: + pretrain_img_size = _to_2tuple(pretrain_img_size) + patch_size = _to_2tuple(patch_size) + patches_resolution = [ + pretrain_img_size[0] // patch_size[0], + pretrain_img_size[1] // patch_size[1], + ] + + self.absolute_pos_embed = nn.Parameter( + torch.zeros(1, embed_dim, patches_resolution[0], patches_resolution[1]) + ) + nn.init.trunc_normal_(self.absolute_pos_embed, std=0.02) + + self.pos_drop = nn.Dropout(p=drop_rate) + + # stochastic depth + dpr = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] # stochastic depth decay rule + + # build layers + self.layers = nn.ModuleList() + for i_layer in range(self.num_layers): + layer = BasicLayer( + dim=int(embed_dim * 2**i_layer), + depth=depths[i_layer], + num_heads=num_heads[i_layer], + window_size=window_size, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[sum(depths[:i_layer]) : sum(depths[: i_layer + 1])], + norm_layer=norm_layer, + downsample=PatchMerging if (i_layer < self.num_layers - 1) else None, + use_checkpoint=use_checkpoint, + ) + self.layers.append(layer) + + num_features = [int(embed_dim * 2**i) for i in range(self.num_layers)] + self.num_features = num_features + + # add a norm layer for each output + for i_layer in out_indices: + layer = norm_layer(num_features[i_layer]) + layer_name = f"norm{i_layer}" + self.add_module(layer_name, layer) + + self._freeze_stages() + self._out_features = ["p{}".format(i) for i in self.out_indices] + self._out_feature_channels = { + "p{}".format(i): self.embed_dim * 2**i for i in self.out_indices + } + self._out_feature_strides = {"p{}".format(i): 2 ** (i + 2) for i in self.out_indices} + self._size_devisibility = 32 + + self.apply(self._init_weights) + + def _freeze_stages(self): + if self.frozen_stages >= 0: + self.patch_embed.eval() + for param in self.patch_embed.parameters(): + param.requires_grad = False + + if self.frozen_stages >= 1 and self.ape: + self.absolute_pos_embed.requires_grad = False + + if self.frozen_stages >= 2: + self.pos_drop.eval() + for i in range(0, self.frozen_stages - 1): + m = self.layers[i] + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + nn.init.trunc_normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + @property + def size_divisibility(self): + return self._size_divisibility + + def forward(self, x): + """Forward function.""" + x = self.patch_embed(x) + + Wh, Ww = x.size(2), x.size(3) + if self.ape: + # interpolate the position embedding to the corresponding size + absolute_pos_embed = F.interpolate( + self.absolute_pos_embed, size=(Wh, Ww), mode="bicubic" + ) + x = (x + absolute_pos_embed).flatten(2).transpose(1, 2) # B Wh*Ww C + else: + x = x.flatten(2).transpose(1, 2) + x = self.pos_drop(x) + + outs = {} + for i in range(self.num_layers): + layer = self.layers[i] + x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww) + + if i in self.out_indices: + norm_layer = getattr(self, f"norm{i}") + x_out = norm_layer(x_out) + + out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous() + outs["p{}".format(i)] = out + + return outs diff --git a/annotator/oneformer/detectron2/modeling/backbone/utils.py b/annotator/oneformer/detectron2/modeling/backbone/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2b89a4c3fbe079a77fd0cef947cf9ada787fc55d --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/backbone/utils.py @@ -0,0 +1,186 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +import math +import torch +import torch.nn as nn +import torch.nn.functional as F + +__all__ = [ + "window_partition", + "window_unpartition", + "add_decomposed_rel_pos", + "get_abs_pos", + "PatchEmbed", +] + + +def window_partition(x, window_size): + """ + Partition into non-overlapping windows with padding if needed. + Args: + x (tensor): input tokens with [B, H, W, C]. + window_size (int): window size. + + Returns: + windows: windows after partition with [B * num_windows, window_size, window_size, C]. + (Hp, Wp): padded height and width before partition + """ + B, H, W, C = x.shape + + pad_h = (window_size - H % window_size) % window_size + pad_w = (window_size - W % window_size) % window_size + if pad_h > 0 or pad_w > 0: + x = F.pad(x, (0, 0, 0, pad_w, 0, pad_h)) + Hp, Wp = H + pad_h, W + pad_w + + x = x.view(B, Hp // window_size, window_size, Wp // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + return windows, (Hp, Wp) + + +def window_unpartition(windows, window_size, pad_hw, hw): + """ + Window unpartition into original sequences and removing padding. + Args: + x (tensor): input tokens with [B * num_windows, window_size, window_size, C]. + window_size (int): window size. + pad_hw (Tuple): padded height and width (Hp, Wp). + hw (Tuple): original height and width (H, W) before padding. + + Returns: + x: unpartitioned sequences with [B, H, W, C]. + """ + Hp, Wp = pad_hw + H, W = hw + B = windows.shape[0] // (Hp * Wp // window_size // window_size) + x = windows.view(B, Hp // window_size, Wp // window_size, window_size, window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, Hp, Wp, -1) + + if Hp > H or Wp > W: + x = x[:, :H, :W, :].contiguous() + return x + + +def get_rel_pos(q_size, k_size, rel_pos): + """ + Get relative positional embeddings according to the relative positions of + query and key sizes. + Args: + q_size (int): size of query q. + k_size (int): size of key k. + rel_pos (Tensor): relative position embeddings (L, C). + + Returns: + Extracted positional embeddings according to relative positions. + """ + max_rel_dist = int(2 * max(q_size, k_size) - 1) + # Interpolate rel pos if needed. + if rel_pos.shape[0] != max_rel_dist: + # Interpolate rel pos. + rel_pos_resized = F.interpolate( + rel_pos.reshape(1, rel_pos.shape[0], -1).permute(0, 2, 1), + size=max_rel_dist, + mode="linear", + ) + rel_pos_resized = rel_pos_resized.reshape(-1, max_rel_dist).permute(1, 0) + else: + rel_pos_resized = rel_pos + + # Scale the coords with short length if shapes for q and k are different. + q_coords = torch.arange(q_size)[:, None] * max(k_size / q_size, 1.0) + k_coords = torch.arange(k_size)[None, :] * max(q_size / k_size, 1.0) + relative_coords = (q_coords - k_coords) + (k_size - 1) * max(q_size / k_size, 1.0) + + return rel_pos_resized[relative_coords.long()] + + +def add_decomposed_rel_pos(attn, q, rel_pos_h, rel_pos_w, q_size, k_size): + """ + Calculate decomposed Relative Positional Embeddings from :paper:`mvitv2`. + https://github.com/facebookresearch/mvit/blob/19786631e330df9f3622e5402b4a419a263a2c80/mvit/models/attention.py # noqa B950 + Args: + attn (Tensor): attention map. + q (Tensor): query q in the attention layer with shape (B, q_h * q_w, C). + rel_pos_h (Tensor): relative position embeddings (Lh, C) for height axis. + rel_pos_w (Tensor): relative position embeddings (Lw, C) for width axis. + q_size (Tuple): spatial sequence size of query q with (q_h, q_w). + k_size (Tuple): spatial sequence size of key k with (k_h, k_w). + + Returns: + attn (Tensor): attention map with added relative positional embeddings. + """ + q_h, q_w = q_size + k_h, k_w = k_size + Rh = get_rel_pos(q_h, k_h, rel_pos_h) + Rw = get_rel_pos(q_w, k_w, rel_pos_w) + + B, _, dim = q.shape + r_q = q.reshape(B, q_h, q_w, dim) + rel_h = torch.einsum("bhwc,hkc->bhwk", r_q, Rh) + rel_w = torch.einsum("bhwc,wkc->bhwk", r_q, Rw) + + attn = ( + attn.view(B, q_h, q_w, k_h, k_w) + rel_h[:, :, :, :, None] + rel_w[:, :, :, None, :] + ).view(B, q_h * q_w, k_h * k_w) + + return attn + + +def get_abs_pos(abs_pos, has_cls_token, hw): + """ + Calculate absolute positional embeddings. If needed, resize embeddings and remove cls_token + dimension for the original embeddings. + Args: + abs_pos (Tensor): absolute positional embeddings with (1, num_position, C). + has_cls_token (bool): If true, has 1 embedding in abs_pos for cls token. + hw (Tuple): size of input image tokens. + + Returns: + Absolute positional embeddings after processing with shape (1, H, W, C) + """ + h, w = hw + if has_cls_token: + abs_pos = abs_pos[:, 1:] + xy_num = abs_pos.shape[1] + size = int(math.sqrt(xy_num)) + assert size * size == xy_num + + if size != h or size != w: + new_abs_pos = F.interpolate( + abs_pos.reshape(1, size, size, -1).permute(0, 3, 1, 2), + size=(h, w), + mode="bicubic", + align_corners=False, + ) + + return new_abs_pos.permute(0, 2, 3, 1) + else: + return abs_pos.reshape(1, h, w, -1) + + +class PatchEmbed(nn.Module): + """ + Image to Patch Embedding. + """ + + def __init__( + self, kernel_size=(16, 16), stride=(16, 16), padding=(0, 0), in_chans=3, embed_dim=768 + ): + """ + Args: + kernel_size (Tuple): kernel size of the projection layer. + stride (Tuple): stride of the projection layer. + padding (Tuple): padding size of the projection layer. + in_chans (int): Number of input image channels. + embed_dim (int): embed_dim (int): Patch embedding dimension. + """ + super().__init__() + + self.proj = nn.Conv2d( + in_chans, embed_dim, kernel_size=kernel_size, stride=stride, padding=padding + ) + + def forward(self, x): + x = self.proj(x) + # B C H W -> B H W C + x = x.permute(0, 2, 3, 1) + return x diff --git a/annotator/oneformer/detectron2/modeling/backbone/vit.py b/annotator/oneformer/detectron2/modeling/backbone/vit.py new file mode 100644 index 0000000000000000000000000000000000000000..07b5e2073ae80859be59d1142394929b504cf427 --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/backbone/vit.py @@ -0,0 +1,524 @@ +import logging +import math +import fvcore.nn.weight_init as weight_init +import torch +import torch.nn as nn + +from annotator.oneformer.detectron2.layers import CNNBlockBase, Conv2d, get_norm +from annotator.oneformer.detectron2.modeling.backbone.fpn import _assert_strides_are_log2_contiguous + +from .backbone import Backbone +from .utils import ( + PatchEmbed, + add_decomposed_rel_pos, + get_abs_pos, + window_partition, + window_unpartition, +) + +logger = logging.getLogger(__name__) + + +__all__ = ["ViT", "SimpleFeaturePyramid", "get_vit_lr_decay_rate"] + + +class Attention(nn.Module): + """Multi-head Attention block with relative position embeddings.""" + + def __init__( + self, + dim, + num_heads=8, + qkv_bias=True, + use_rel_pos=False, + rel_pos_zero_init=True, + input_size=None, + ): + """ + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + qkv_bias (bool: If True, add a learnable bias to query, key, value. + rel_pos (bool): If True, add relative positional embeddings to the attention map. + rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. + input_size (int or None): Input resolution for calculating the relative positional + parameter size. + """ + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim**-0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.proj = nn.Linear(dim, dim) + + self.use_rel_pos = use_rel_pos + if self.use_rel_pos: + # initialize relative positional embeddings + self.rel_pos_h = nn.Parameter(torch.zeros(2 * input_size[0] - 1, head_dim)) + self.rel_pos_w = nn.Parameter(torch.zeros(2 * input_size[1] - 1, head_dim)) + + if not rel_pos_zero_init: + nn.init.trunc_normal_(self.rel_pos_h, std=0.02) + nn.init.trunc_normal_(self.rel_pos_w, std=0.02) + + def forward(self, x): + B, H, W, _ = x.shape + # qkv with shape (3, B, nHead, H * W, C) + qkv = self.qkv(x).reshape(B, H * W, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + # q, k, v with shape (B * nHead, H * W, C) + q, k, v = qkv.reshape(3, B * self.num_heads, H * W, -1).unbind(0) + + attn = (q * self.scale) @ k.transpose(-2, -1) + + if self.use_rel_pos: + attn = add_decomposed_rel_pos(attn, q, self.rel_pos_h, self.rel_pos_w, (H, W), (H, W)) + + attn = attn.softmax(dim=-1) + x = (attn @ v).view(B, self.num_heads, H, W, -1).permute(0, 2, 3, 1, 4).reshape(B, H, W, -1) + x = self.proj(x) + + return x + + +class ResBottleneckBlock(CNNBlockBase): + """ + The standard bottleneck residual block without the last activation layer. + It contains 3 conv layers with kernels 1x1, 3x3, 1x1. + """ + + def __init__( + self, + in_channels, + out_channels, + bottleneck_channels, + norm="LN", + act_layer=nn.GELU, + ): + """ + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + bottleneck_channels (int): number of output channels for the 3x3 + "bottleneck" conv layers. + norm (str or callable): normalization for all conv layers. + See :func:`layers.get_norm` for supported format. + act_layer (callable): activation for all conv layers. + """ + super().__init__(in_channels, out_channels, 1) + + self.conv1 = Conv2d(in_channels, bottleneck_channels, 1, bias=False) + self.norm1 = get_norm(norm, bottleneck_channels) + self.act1 = act_layer() + + self.conv2 = Conv2d( + bottleneck_channels, + bottleneck_channels, + 3, + padding=1, + bias=False, + ) + self.norm2 = get_norm(norm, bottleneck_channels) + self.act2 = act_layer() + + self.conv3 = Conv2d(bottleneck_channels, out_channels, 1, bias=False) + self.norm3 = get_norm(norm, out_channels) + + for layer in [self.conv1, self.conv2, self.conv3]: + weight_init.c2_msra_fill(layer) + for layer in [self.norm1, self.norm2]: + layer.weight.data.fill_(1.0) + layer.bias.data.zero_() + # zero init last norm layer. + self.norm3.weight.data.zero_() + self.norm3.bias.data.zero_() + + def forward(self, x): + out = x + for layer in self.children(): + out = layer(out) + + out = x + out + return out + + +class Block(nn.Module): + """Transformer blocks with support of window attention and residual propagation blocks""" + + def __init__( + self, + dim, + num_heads, + mlp_ratio=4.0, + qkv_bias=True, + drop_path=0.0, + norm_layer=nn.LayerNorm, + act_layer=nn.GELU, + use_rel_pos=False, + rel_pos_zero_init=True, + window_size=0, + use_residual_block=False, + input_size=None, + ): + """ + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads in each ViT block. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool): If True, add a learnable bias to query, key, value. + drop_path (float): Stochastic depth rate. + norm_layer (nn.Module): Normalization layer. + act_layer (nn.Module): Activation layer. + use_rel_pos (bool): If True, add relative positional embeddings to the attention map. + rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. + window_size (int): Window size for window attention blocks. If it equals 0, then not + use window attention. + use_residual_block (bool): If True, use a residual block after the MLP block. + input_size (int or None): Input resolution for calculating the relative positional + parameter size. + """ + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + use_rel_pos=use_rel_pos, + rel_pos_zero_init=rel_pos_zero_init, + input_size=input_size if window_size == 0 else (window_size, window_size), + ) + + from timm.models.layers import DropPath, Mlp + + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + self.norm2 = norm_layer(dim) + self.mlp = Mlp(in_features=dim, hidden_features=int(dim * mlp_ratio), act_layer=act_layer) + + self.window_size = window_size + + self.use_residual_block = use_residual_block + if use_residual_block: + # Use a residual block with bottleneck channel as dim // 2 + self.residual = ResBottleneckBlock( + in_channels=dim, + out_channels=dim, + bottleneck_channels=dim // 2, + norm="LN", + act_layer=act_layer, + ) + + def forward(self, x): + shortcut = x + x = self.norm1(x) + # Window partition + if self.window_size > 0: + H, W = x.shape[1], x.shape[2] + x, pad_hw = window_partition(x, self.window_size) + + x = self.attn(x) + # Reverse window partition + if self.window_size > 0: + x = window_unpartition(x, self.window_size, pad_hw, (H, W)) + + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + if self.use_residual_block: + x = self.residual(x.permute(0, 3, 1, 2)).permute(0, 2, 3, 1) + + return x + + +class ViT(Backbone): + """ + This module implements Vision Transformer (ViT) backbone in :paper:`vitdet`. + "Exploring Plain Vision Transformer Backbones for Object Detection", + https://arxiv.org/abs/2203.16527 + """ + + def __init__( + self, + img_size=1024, + patch_size=16, + in_chans=3, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4.0, + qkv_bias=True, + drop_path_rate=0.0, + norm_layer=nn.LayerNorm, + act_layer=nn.GELU, + use_abs_pos=True, + use_rel_pos=False, + rel_pos_zero_init=True, + window_size=0, + window_block_indexes=(), + residual_block_indexes=(), + use_act_checkpoint=False, + pretrain_img_size=224, + pretrain_use_cls_token=True, + out_feature="last_feat", + ): + """ + Args: + img_size (int): Input image size. + patch_size (int): Patch size. + in_chans (int): Number of input image channels. + embed_dim (int): Patch embedding dimension. + depth (int): Depth of ViT. + num_heads (int): Number of attention heads in each ViT block. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool): If True, add a learnable bias to query, key, value. + drop_path_rate (float): Stochastic depth rate. + norm_layer (nn.Module): Normalization layer. + act_layer (nn.Module): Activation layer. + use_abs_pos (bool): If True, use absolute positional embeddings. + use_rel_pos (bool): If True, add relative positional embeddings to the attention map. + rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. + window_size (int): Window size for window attention blocks. + window_block_indexes (list): Indexes for blocks using window attention. + residual_block_indexes (list): Indexes for blocks using conv propagation. + use_act_checkpoint (bool): If True, use activation checkpointing. + pretrain_img_size (int): input image size for pretraining models. + pretrain_use_cls_token (bool): If True, pretrainig models use class token. + out_feature (str): name of the feature from the last block. + """ + super().__init__() + self.pretrain_use_cls_token = pretrain_use_cls_token + + self.patch_embed = PatchEmbed( + kernel_size=(patch_size, patch_size), + stride=(patch_size, patch_size), + in_chans=in_chans, + embed_dim=embed_dim, + ) + + if use_abs_pos: + # Initialize absolute positional embedding with pretrain image size. + num_patches = (pretrain_img_size // patch_size) * (pretrain_img_size // patch_size) + num_positions = (num_patches + 1) if pretrain_use_cls_token else num_patches + self.pos_embed = nn.Parameter(torch.zeros(1, num_positions, embed_dim)) + else: + self.pos_embed = None + + # stochastic depth decay rule + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)] + + self.blocks = nn.ModuleList() + for i in range(depth): + block = Block( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + drop_path=dpr[i], + norm_layer=norm_layer, + act_layer=act_layer, + use_rel_pos=use_rel_pos, + rel_pos_zero_init=rel_pos_zero_init, + window_size=window_size if i in window_block_indexes else 0, + use_residual_block=i in residual_block_indexes, + input_size=(img_size // patch_size, img_size // patch_size), + ) + if use_act_checkpoint: + # TODO: use torch.utils.checkpoint + from fairscale.nn.checkpoint import checkpoint_wrapper + + block = checkpoint_wrapper(block) + self.blocks.append(block) + + self._out_feature_channels = {out_feature: embed_dim} + self._out_feature_strides = {out_feature: patch_size} + self._out_features = [out_feature] + + if self.pos_embed is not None: + nn.init.trunc_normal_(self.pos_embed, std=0.02) + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + nn.init.trunc_normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def forward(self, x): + x = self.patch_embed(x) + if self.pos_embed is not None: + x = x + get_abs_pos( + self.pos_embed, self.pretrain_use_cls_token, (x.shape[1], x.shape[2]) + ) + + for blk in self.blocks: + x = blk(x) + + outputs = {self._out_features[0]: x.permute(0, 3, 1, 2)} + return outputs + + +class SimpleFeaturePyramid(Backbone): + """ + This module implements SimpleFeaturePyramid in :paper:`vitdet`. + It creates pyramid features built on top of the input feature map. + """ + + def __init__( + self, + net, + in_feature, + out_channels, + scale_factors, + top_block=None, + norm="LN", + square_pad=0, + ): + """ + Args: + net (Backbone): module representing the subnetwork backbone. + Must be a subclass of :class:`Backbone`. + in_feature (str): names of the input feature maps coming + from the net. + out_channels (int): number of channels in the output feature maps. + scale_factors (list[float]): list of scaling factors to upsample or downsample + the input features for creating pyramid features. + top_block (nn.Module or None): if provided, an extra operation will + be performed on the output of the last (smallest resolution) + pyramid output, and the result will extend the result list. The top_block + further downsamples the feature map. It must have an attribute + "num_levels", meaning the number of extra pyramid levels added by + this block, and "in_feature", which is a string representing + its input feature (e.g., p5). + norm (str): the normalization to use. + square_pad (int): If > 0, require input images to be padded to specific square size. + """ + super(SimpleFeaturePyramid, self).__init__() + assert isinstance(net, Backbone) + + self.scale_factors = scale_factors + + input_shapes = net.output_shape() + strides = [int(input_shapes[in_feature].stride / scale) for scale in scale_factors] + _assert_strides_are_log2_contiguous(strides) + + dim = input_shapes[in_feature].channels + self.stages = [] + use_bias = norm == "" + for idx, scale in enumerate(scale_factors): + out_dim = dim + if scale == 4.0: + layers = [ + nn.ConvTranspose2d(dim, dim // 2, kernel_size=2, stride=2), + get_norm(norm, dim // 2), + nn.GELU(), + nn.ConvTranspose2d(dim // 2, dim // 4, kernel_size=2, stride=2), + ] + out_dim = dim // 4 + elif scale == 2.0: + layers = [nn.ConvTranspose2d(dim, dim // 2, kernel_size=2, stride=2)] + out_dim = dim // 2 + elif scale == 1.0: + layers = [] + elif scale == 0.5: + layers = [nn.MaxPool2d(kernel_size=2, stride=2)] + else: + raise NotImplementedError(f"scale_factor={scale} is not supported yet.") + + layers.extend( + [ + Conv2d( + out_dim, + out_channels, + kernel_size=1, + bias=use_bias, + norm=get_norm(norm, out_channels), + ), + Conv2d( + out_channels, + out_channels, + kernel_size=3, + padding=1, + bias=use_bias, + norm=get_norm(norm, out_channels), + ), + ] + ) + layers = nn.Sequential(*layers) + + stage = int(math.log2(strides[idx])) + self.add_module(f"simfp_{stage}", layers) + self.stages.append(layers) + + self.net = net + self.in_feature = in_feature + self.top_block = top_block + # Return feature names are "p", like ["p2", "p3", ..., "p6"] + self._out_feature_strides = {"p{}".format(int(math.log2(s))): s for s in strides} + # top block output feature maps. + if self.top_block is not None: + for s in range(stage, stage + self.top_block.num_levels): + self._out_feature_strides["p{}".format(s + 1)] = 2 ** (s + 1) + + self._out_features = list(self._out_feature_strides.keys()) + self._out_feature_channels = {k: out_channels for k in self._out_features} + self._size_divisibility = strides[-1] + self._square_pad = square_pad + + @property + def padding_constraints(self): + return { + "size_divisiblity": self._size_divisibility, + "square_size": self._square_pad, + } + + def forward(self, x): + """ + Args: + x: Tensor of shape (N,C,H,W). H, W must be a multiple of ``self.size_divisibility``. + + Returns: + dict[str->Tensor]: + mapping from feature map name to pyramid feature map tensor + in high to low resolution order. Returned feature names follow the FPN + convention: "p", where stage has stride = 2 ** stage e.g., + ["p2", "p3", ..., "p6"]. + """ + bottom_up_features = self.net(x) + features = bottom_up_features[self.in_feature] + results = [] + + for stage in self.stages: + results.append(stage(features)) + + if self.top_block is not None: + if self.top_block.in_feature in bottom_up_features: + top_block_in_feature = bottom_up_features[self.top_block.in_feature] + else: + top_block_in_feature = results[self._out_features.index(self.top_block.in_feature)] + results.extend(self.top_block(top_block_in_feature)) + assert len(self._out_features) == len(results) + return {f: res for f, res in zip(self._out_features, results)} + + +def get_vit_lr_decay_rate(name, lr_decay_rate=1.0, num_layers=12): + """ + Calculate lr decay rate for different ViT blocks. + Args: + name (string): parameter name. + lr_decay_rate (float): base lr decay rate. + num_layers (int): number of ViT blocks. + + Returns: + lr decay rate for the given parameter. + """ + layer_id = num_layers + 1 + if name.startswith("backbone"): + if ".pos_embed" in name or ".patch_embed" in name: + layer_id = 0 + elif ".blocks." in name and ".residual." not in name: + layer_id = int(name[name.find(".blocks.") :].split(".")[2]) + 1 + + return lr_decay_rate ** (num_layers + 1 - layer_id) diff --git a/annotator/oneformer/detectron2/modeling/box_regression.py b/annotator/oneformer/detectron2/modeling/box_regression.py new file mode 100644 index 0000000000000000000000000000000000000000..3cd5668d9a72edd34df4f458f90ac72553abb955 --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/box_regression.py @@ -0,0 +1,369 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import math +from typing import List, Tuple, Union +import torch +from fvcore.nn import giou_loss, smooth_l1_loss +from torch.nn import functional as F + +from annotator.oneformer.detectron2.layers import cat, ciou_loss, diou_loss +from annotator.oneformer.detectron2.structures import Boxes + +# Value for clamping large dw and dh predictions. The heuristic is that we clamp +# such that dw and dh are no larger than what would transform a 16px box into a +# 1000px box (based on a small anchor, 16px, and a typical image size, 1000px). +_DEFAULT_SCALE_CLAMP = math.log(1000.0 / 16) + + +__all__ = ["Box2BoxTransform", "Box2BoxTransformRotated", "Box2BoxTransformLinear"] + + +@torch.jit.script +class Box2BoxTransform(object): + """ + The box-to-box transform defined in R-CNN. The transformation is parameterized + by 4 deltas: (dx, dy, dw, dh). The transformation scales the box's width and height + by exp(dw), exp(dh) and shifts a box's center by the offset (dx * width, dy * height). + """ + + def __init__( + self, weights: Tuple[float, float, float, float], scale_clamp: float = _DEFAULT_SCALE_CLAMP + ): + """ + Args: + weights (4-element tuple): Scaling factors that are applied to the + (dx, dy, dw, dh) deltas. In Fast R-CNN, these were originally set + such that the deltas have unit variance; now they are treated as + hyperparameters of the system. + scale_clamp (float): When predicting deltas, the predicted box scaling + factors (dw and dh) are clamped such that they are <= scale_clamp. + """ + self.weights = weights + self.scale_clamp = scale_clamp + + def get_deltas(self, src_boxes, target_boxes): + """ + Get box regression transformation deltas (dx, dy, dw, dh) that can be used + to transform the `src_boxes` into the `target_boxes`. That is, the relation + ``target_boxes == self.apply_deltas(deltas, src_boxes)`` is true (unless + any delta is too large and is clamped). + + Args: + src_boxes (Tensor): source boxes, e.g., object proposals + target_boxes (Tensor): target of the transformation, e.g., ground-truth + boxes. + """ + assert isinstance(src_boxes, torch.Tensor), type(src_boxes) + assert isinstance(target_boxes, torch.Tensor), type(target_boxes) + + src_widths = src_boxes[:, 2] - src_boxes[:, 0] + src_heights = src_boxes[:, 3] - src_boxes[:, 1] + src_ctr_x = src_boxes[:, 0] + 0.5 * src_widths + src_ctr_y = src_boxes[:, 1] + 0.5 * src_heights + + target_widths = target_boxes[:, 2] - target_boxes[:, 0] + target_heights = target_boxes[:, 3] - target_boxes[:, 1] + target_ctr_x = target_boxes[:, 0] + 0.5 * target_widths + target_ctr_y = target_boxes[:, 1] + 0.5 * target_heights + + wx, wy, ww, wh = self.weights + dx = wx * (target_ctr_x - src_ctr_x) / src_widths + dy = wy * (target_ctr_y - src_ctr_y) / src_heights + dw = ww * torch.log(target_widths / src_widths) + dh = wh * torch.log(target_heights / src_heights) + + deltas = torch.stack((dx, dy, dw, dh), dim=1) + assert (src_widths > 0).all().item(), "Input boxes to Box2BoxTransform are not valid!" + return deltas + + def apply_deltas(self, deltas, boxes): + """ + Apply transformation `deltas` (dx, dy, dw, dh) to `boxes`. + + Args: + deltas (Tensor): transformation deltas of shape (N, k*4), where k >= 1. + deltas[i] represents k potentially different class-specific + box transformations for the single box boxes[i]. + boxes (Tensor): boxes to transform, of shape (N, 4) + """ + deltas = deltas.float() # ensure fp32 for decoding precision + boxes = boxes.to(deltas.dtype) + + widths = boxes[:, 2] - boxes[:, 0] + heights = boxes[:, 3] - boxes[:, 1] + ctr_x = boxes[:, 0] + 0.5 * widths + ctr_y = boxes[:, 1] + 0.5 * heights + + wx, wy, ww, wh = self.weights + dx = deltas[:, 0::4] / wx + dy = deltas[:, 1::4] / wy + dw = deltas[:, 2::4] / ww + dh = deltas[:, 3::4] / wh + + # Prevent sending too large values into torch.exp() + dw = torch.clamp(dw, max=self.scale_clamp) + dh = torch.clamp(dh, max=self.scale_clamp) + + pred_ctr_x = dx * widths[:, None] + ctr_x[:, None] + pred_ctr_y = dy * heights[:, None] + ctr_y[:, None] + pred_w = torch.exp(dw) * widths[:, None] + pred_h = torch.exp(dh) * heights[:, None] + + x1 = pred_ctr_x - 0.5 * pred_w + y1 = pred_ctr_y - 0.5 * pred_h + x2 = pred_ctr_x + 0.5 * pred_w + y2 = pred_ctr_y + 0.5 * pred_h + pred_boxes = torch.stack((x1, y1, x2, y2), dim=-1) + return pred_boxes.reshape(deltas.shape) + + +@torch.jit.script +class Box2BoxTransformRotated(object): + """ + The box-to-box transform defined in Rotated R-CNN. The transformation is parameterized + by 5 deltas: (dx, dy, dw, dh, da). The transformation scales the box's width and height + by exp(dw), exp(dh), shifts a box's center by the offset (dx * width, dy * height), + and rotate a box's angle by da (radians). + Note: angles of deltas are in radians while angles of boxes are in degrees. + """ + + def __init__( + self, + weights: Tuple[float, float, float, float, float], + scale_clamp: float = _DEFAULT_SCALE_CLAMP, + ): + """ + Args: + weights (5-element tuple): Scaling factors that are applied to the + (dx, dy, dw, dh, da) deltas. These are treated as + hyperparameters of the system. + scale_clamp (float): When predicting deltas, the predicted box scaling + factors (dw and dh) are clamped such that they are <= scale_clamp. + """ + self.weights = weights + self.scale_clamp = scale_clamp + + def get_deltas(self, src_boxes, target_boxes): + """ + Get box regression transformation deltas (dx, dy, dw, dh, da) that can be used + to transform the `src_boxes` into the `target_boxes`. That is, the relation + ``target_boxes == self.apply_deltas(deltas, src_boxes)`` is true (unless + any delta is too large and is clamped). + + Args: + src_boxes (Tensor): Nx5 source boxes, e.g., object proposals + target_boxes (Tensor): Nx5 target of the transformation, e.g., ground-truth + boxes. + """ + assert isinstance(src_boxes, torch.Tensor), type(src_boxes) + assert isinstance(target_boxes, torch.Tensor), type(target_boxes) + + src_ctr_x, src_ctr_y, src_widths, src_heights, src_angles = torch.unbind(src_boxes, dim=1) + + target_ctr_x, target_ctr_y, target_widths, target_heights, target_angles = torch.unbind( + target_boxes, dim=1 + ) + + wx, wy, ww, wh, wa = self.weights + dx = wx * (target_ctr_x - src_ctr_x) / src_widths + dy = wy * (target_ctr_y - src_ctr_y) / src_heights + dw = ww * torch.log(target_widths / src_widths) + dh = wh * torch.log(target_heights / src_heights) + # Angles of deltas are in radians while angles of boxes are in degrees. + # the conversion to radians serve as a way to normalize the values + da = target_angles - src_angles + da = (da + 180.0) % 360.0 - 180.0 # make it in [-180, 180) + da *= wa * math.pi / 180.0 + + deltas = torch.stack((dx, dy, dw, dh, da), dim=1) + assert ( + (src_widths > 0).all().item() + ), "Input boxes to Box2BoxTransformRotated are not valid!" + return deltas + + def apply_deltas(self, deltas, boxes): + """ + Apply transformation `deltas` (dx, dy, dw, dh, da) to `boxes`. + + Args: + deltas (Tensor): transformation deltas of shape (N, k*5). + deltas[i] represents box transformation for the single box boxes[i]. + boxes (Tensor): boxes to transform, of shape (N, 5) + """ + assert deltas.shape[1] % 5 == 0 and boxes.shape[1] == 5 + + boxes = boxes.to(deltas.dtype).unsqueeze(2) + + ctr_x = boxes[:, 0] + ctr_y = boxes[:, 1] + widths = boxes[:, 2] + heights = boxes[:, 3] + angles = boxes[:, 4] + + wx, wy, ww, wh, wa = self.weights + + dx = deltas[:, 0::5] / wx + dy = deltas[:, 1::5] / wy + dw = deltas[:, 2::5] / ww + dh = deltas[:, 3::5] / wh + da = deltas[:, 4::5] / wa + + # Prevent sending too large values into torch.exp() + dw = torch.clamp(dw, max=self.scale_clamp) + dh = torch.clamp(dh, max=self.scale_clamp) + + pred_boxes = torch.zeros_like(deltas) + pred_boxes[:, 0::5] = dx * widths + ctr_x # x_ctr + pred_boxes[:, 1::5] = dy * heights + ctr_y # y_ctr + pred_boxes[:, 2::5] = torch.exp(dw) * widths # width + pred_boxes[:, 3::5] = torch.exp(dh) * heights # height + + # Following original RRPN implementation, + # angles of deltas are in radians while angles of boxes are in degrees. + pred_angle = da * 180.0 / math.pi + angles + pred_angle = (pred_angle + 180.0) % 360.0 - 180.0 # make it in [-180, 180) + + pred_boxes[:, 4::5] = pred_angle + + return pred_boxes + + +class Box2BoxTransformLinear(object): + """ + The linear box-to-box transform defined in FCOS. The transformation is parameterized + by the distance from the center of (square) src box to 4 edges of the target box. + """ + + def __init__(self, normalize_by_size=True): + """ + Args: + normalize_by_size: normalize deltas by the size of src (anchor) boxes. + """ + self.normalize_by_size = normalize_by_size + + def get_deltas(self, src_boxes, target_boxes): + """ + Get box regression transformation deltas (dx1, dy1, dx2, dy2) that can be used + to transform the `src_boxes` into the `target_boxes`. That is, the relation + ``target_boxes == self.apply_deltas(deltas, src_boxes)`` is true. + The center of src must be inside target boxes. + + Args: + src_boxes (Tensor): square source boxes, e.g., anchors + target_boxes (Tensor): target of the transformation, e.g., ground-truth + boxes. + """ + assert isinstance(src_boxes, torch.Tensor), type(src_boxes) + assert isinstance(target_boxes, torch.Tensor), type(target_boxes) + + src_ctr_x = 0.5 * (src_boxes[:, 0] + src_boxes[:, 2]) + src_ctr_y = 0.5 * (src_boxes[:, 1] + src_boxes[:, 3]) + + target_l = src_ctr_x - target_boxes[:, 0] + target_t = src_ctr_y - target_boxes[:, 1] + target_r = target_boxes[:, 2] - src_ctr_x + target_b = target_boxes[:, 3] - src_ctr_y + + deltas = torch.stack((target_l, target_t, target_r, target_b), dim=1) + if self.normalize_by_size: + stride_w = src_boxes[:, 2] - src_boxes[:, 0] + stride_h = src_boxes[:, 3] - src_boxes[:, 1] + strides = torch.stack([stride_w, stride_h, stride_w, stride_h], axis=1) + deltas = deltas / strides + + return deltas + + def apply_deltas(self, deltas, boxes): + """ + Apply transformation `deltas` (dx1, dy1, dx2, dy2) to `boxes`. + + Args: + deltas (Tensor): transformation deltas of shape (N, k*4), where k >= 1. + deltas[i] represents k potentially different class-specific + box transformations for the single box boxes[i]. + boxes (Tensor): boxes to transform, of shape (N, 4) + """ + # Ensure the output is a valid box. See Sec 2.1 of https://arxiv.org/abs/2006.09214 + deltas = F.relu(deltas) + boxes = boxes.to(deltas.dtype) + + ctr_x = 0.5 * (boxes[:, 0] + boxes[:, 2]) + ctr_y = 0.5 * (boxes[:, 1] + boxes[:, 3]) + if self.normalize_by_size: + stride_w = boxes[:, 2] - boxes[:, 0] + stride_h = boxes[:, 3] - boxes[:, 1] + strides = torch.stack([stride_w, stride_h, stride_w, stride_h], axis=1) + deltas = deltas * strides + + l = deltas[:, 0::4] + t = deltas[:, 1::4] + r = deltas[:, 2::4] + b = deltas[:, 3::4] + + pred_boxes = torch.zeros_like(deltas) + pred_boxes[:, 0::4] = ctr_x[:, None] - l # x1 + pred_boxes[:, 1::4] = ctr_y[:, None] - t # y1 + pred_boxes[:, 2::4] = ctr_x[:, None] + r # x2 + pred_boxes[:, 3::4] = ctr_y[:, None] + b # y2 + return pred_boxes + + +def _dense_box_regression_loss( + anchors: List[Union[Boxes, torch.Tensor]], + box2box_transform: Box2BoxTransform, + pred_anchor_deltas: List[torch.Tensor], + gt_boxes: List[torch.Tensor], + fg_mask: torch.Tensor, + box_reg_loss_type="smooth_l1", + smooth_l1_beta=0.0, +): + """ + Compute loss for dense multi-level box regression. + Loss is accumulated over ``fg_mask``. + + Args: + anchors: #lvl anchor boxes, each is (HixWixA, 4) + pred_anchor_deltas: #lvl predictions, each is (N, HixWixA, 4) + gt_boxes: N ground truth boxes, each has shape (R, 4) (R = sum(Hi * Wi * A)) + fg_mask: the foreground boolean mask of shape (N, R) to compute loss on + box_reg_loss_type (str): Loss type to use. Supported losses: "smooth_l1", "giou", + "diou", "ciou". + smooth_l1_beta (float): beta parameter for the smooth L1 regression loss. Default to + use L1 loss. Only used when `box_reg_loss_type` is "smooth_l1" + """ + if isinstance(anchors[0], Boxes): + anchors = type(anchors[0]).cat(anchors).tensor # (R, 4) + else: + anchors = cat(anchors) + if box_reg_loss_type == "smooth_l1": + gt_anchor_deltas = [box2box_transform.get_deltas(anchors, k) for k in gt_boxes] + gt_anchor_deltas = torch.stack(gt_anchor_deltas) # (N, R, 4) + loss_box_reg = smooth_l1_loss( + cat(pred_anchor_deltas, dim=1)[fg_mask], + gt_anchor_deltas[fg_mask], + beta=smooth_l1_beta, + reduction="sum", + ) + elif box_reg_loss_type == "giou": + pred_boxes = [ + box2box_transform.apply_deltas(k, anchors) for k in cat(pred_anchor_deltas, dim=1) + ] + loss_box_reg = giou_loss( + torch.stack(pred_boxes)[fg_mask], torch.stack(gt_boxes)[fg_mask], reduction="sum" + ) + elif box_reg_loss_type == "diou": + pred_boxes = [ + box2box_transform.apply_deltas(k, anchors) for k in cat(pred_anchor_deltas, dim=1) + ] + loss_box_reg = diou_loss( + torch.stack(pred_boxes)[fg_mask], torch.stack(gt_boxes)[fg_mask], reduction="sum" + ) + elif box_reg_loss_type == "ciou": + pred_boxes = [ + box2box_transform.apply_deltas(k, anchors) for k in cat(pred_anchor_deltas, dim=1) + ] + loss_box_reg = ciou_loss( + torch.stack(pred_boxes)[fg_mask], torch.stack(gt_boxes)[fg_mask], reduction="sum" + ) + else: + raise ValueError(f"Invalid dense box regression loss type '{box_reg_loss_type}'") + return loss_box_reg diff --git a/annotator/oneformer/detectron2/modeling/matcher.py b/annotator/oneformer/detectron2/modeling/matcher.py new file mode 100644 index 0000000000000000000000000000000000000000..2504d17a4f9707d7cdd8d47a6cb5a2faf3c397fd --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/matcher.py @@ -0,0 +1,127 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from typing import List +import torch + +from annotator.oneformer.detectron2.layers import nonzero_tuple + + +# TODO: the name is too general +class Matcher(object): + """ + This class assigns to each predicted "element" (e.g., a box) a ground-truth + element. Each predicted element will have exactly zero or one matches; each + ground-truth element may be matched to zero or more predicted elements. + + The matching is determined by the MxN match_quality_matrix, that characterizes + how well each (ground-truth, prediction)-pair match each other. For example, + if the elements are boxes, this matrix may contain box intersection-over-union + overlap values. + + The matcher returns (a) a vector of length N containing the index of the + ground-truth element m in [0, M) that matches to prediction n in [0, N). + (b) a vector of length N containing the labels for each prediction. + """ + + def __init__( + self, thresholds: List[float], labels: List[int], allow_low_quality_matches: bool = False + ): + """ + Args: + thresholds (list): a list of thresholds used to stratify predictions + into levels. + labels (list): a list of values to label predictions belonging at + each level. A label can be one of {-1, 0, 1} signifying + {ignore, negative class, positive class}, respectively. + allow_low_quality_matches (bool): if True, produce additional matches + for predictions with maximum match quality lower than high_threshold. + See set_low_quality_matches_ for more details. + + For example, + thresholds = [0.3, 0.5] + labels = [0, -1, 1] + All predictions with iou < 0.3 will be marked with 0 and + thus will be considered as false positives while training. + All predictions with 0.3 <= iou < 0.5 will be marked with -1 and + thus will be ignored. + All predictions with 0.5 <= iou will be marked with 1 and + thus will be considered as true positives. + """ + # Add -inf and +inf to first and last position in thresholds + thresholds = thresholds[:] + assert thresholds[0] > 0 + thresholds.insert(0, -float("inf")) + thresholds.append(float("inf")) + # Currently torchscript does not support all + generator + assert all([low <= high for (low, high) in zip(thresholds[:-1], thresholds[1:])]) + assert all([l in [-1, 0, 1] for l in labels]) + assert len(labels) == len(thresholds) - 1 + self.thresholds = thresholds + self.labels = labels + self.allow_low_quality_matches = allow_low_quality_matches + + def __call__(self, match_quality_matrix): + """ + Args: + match_quality_matrix (Tensor[float]): an MxN tensor, containing the + pairwise quality between M ground-truth elements and N predicted + elements. All elements must be >= 0 (due to the us of `torch.nonzero` + for selecting indices in :meth:`set_low_quality_matches_`). + + Returns: + matches (Tensor[int64]): a vector of length N, where matches[i] is a matched + ground-truth index in [0, M) + match_labels (Tensor[int8]): a vector of length N, where pred_labels[i] indicates + whether a prediction is a true or false positive or ignored + """ + assert match_quality_matrix.dim() == 2 + if match_quality_matrix.numel() == 0: + default_matches = match_quality_matrix.new_full( + (match_quality_matrix.size(1),), 0, dtype=torch.int64 + ) + # When no gt boxes exist, we define IOU = 0 and therefore set labels + # to `self.labels[0]`, which usually defaults to background class 0 + # To choose to ignore instead, can make labels=[-1,0,-1,1] + set appropriate thresholds + default_match_labels = match_quality_matrix.new_full( + (match_quality_matrix.size(1),), self.labels[0], dtype=torch.int8 + ) + return default_matches, default_match_labels + + assert torch.all(match_quality_matrix >= 0) + + # match_quality_matrix is M (gt) x N (predicted) + # Max over gt elements (dim 0) to find best gt candidate for each prediction + matched_vals, matches = match_quality_matrix.max(dim=0) + + match_labels = matches.new_full(matches.size(), 1, dtype=torch.int8) + + for (l, low, high) in zip(self.labels, self.thresholds[:-1], self.thresholds[1:]): + low_high = (matched_vals >= low) & (matched_vals < high) + match_labels[low_high] = l + + if self.allow_low_quality_matches: + self.set_low_quality_matches_(match_labels, match_quality_matrix) + + return matches, match_labels + + def set_low_quality_matches_(self, match_labels, match_quality_matrix): + """ + Produce additional matches for predictions that have only low-quality matches. + Specifically, for each ground-truth G find the set of predictions that have + maximum overlap with it (including ties); for each prediction in that set, if + it is unmatched, then match it to the ground-truth G. + + This function implements the RPN assignment case (i) in Sec. 3.1.2 of + :paper:`Faster R-CNN`. + """ + # For each gt, find the prediction with which it has highest quality + highest_quality_foreach_gt, _ = match_quality_matrix.max(dim=1) + # Find the highest quality match available, even if it is low, including ties. + # Note that the matches qualities must be positive due to the use of + # `torch.nonzero`. + _, pred_inds_with_highest_quality = nonzero_tuple( + match_quality_matrix == highest_quality_foreach_gt[:, None] + ) + # If an anchor was labeled positive only due to a low-quality match + # with gt_A, but it has larger overlap with gt_B, it's matched index will still be gt_B. + # This follows the implementation in Detectron, and is found to have no significant impact. + match_labels[pred_inds_with_highest_quality] = 1 diff --git a/annotator/oneformer/detectron2/modeling/meta_arch/__init__.py b/annotator/oneformer/detectron2/modeling/meta_arch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6b0668157052ce7b796ef50bc7ee85361e7605b9 --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/meta_arch/__init__.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +from .build import META_ARCH_REGISTRY, build_model # isort:skip + +from .panoptic_fpn import PanopticFPN + +# import all the meta_arch, so they will be registered +from .rcnn import GeneralizedRCNN, ProposalNetwork +from .dense_detector import DenseDetector +from .retinanet import RetinaNet +from .fcos import FCOS +from .semantic_seg import SEM_SEG_HEADS_REGISTRY, SemanticSegmentor, build_sem_seg_head + + +__all__ = list(globals().keys()) diff --git a/annotator/oneformer/detectron2/modeling/meta_arch/build.py b/annotator/oneformer/detectron2/modeling/meta_arch/build.py new file mode 100644 index 0000000000000000000000000000000000000000..98a08f06ac58b21f3a738132b8c3ac62f51fa538 --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/meta_arch/build.py @@ -0,0 +1,25 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import torch + +from annotator.oneformer.detectron2.utils.logger import _log_api_usage +from annotator.oneformer.detectron2.utils.registry import Registry + +META_ARCH_REGISTRY = Registry("META_ARCH") # noqa F401 isort:skip +META_ARCH_REGISTRY.__doc__ = """ +Registry for meta-architectures, i.e. the whole model. + +The registered object will be called with `obj(cfg)` +and expected to return a `nn.Module` object. +""" + + +def build_model(cfg): + """ + Build the whole model architecture, defined by ``cfg.MODEL.META_ARCHITECTURE``. + Note that it does not load any weights from ``cfg``. + """ + meta_arch = cfg.MODEL.META_ARCHITECTURE + model = META_ARCH_REGISTRY.get(meta_arch)(cfg) + model.to(torch.device(cfg.MODEL.DEVICE)) + _log_api_usage("modeling.meta_arch." + meta_arch) + return model diff --git a/annotator/oneformer/detectron2/modeling/meta_arch/dense_detector.py b/annotator/oneformer/detectron2/modeling/meta_arch/dense_detector.py new file mode 100644 index 0000000000000000000000000000000000000000..461c370fe9e5fab5c634b029d5176cf4dc68de2f --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/meta_arch/dense_detector.py @@ -0,0 +1,294 @@ +import numpy as np +from typing import Dict, List, Optional, Tuple +import torch +from torch import Tensor, nn + +from annotator.oneformer.detectron2.data.detection_utils import convert_image_to_rgb +from annotator.oneformer.detectron2.layers import move_device_like +from annotator.oneformer.detectron2.modeling import Backbone +from annotator.oneformer.detectron2.structures import Boxes, ImageList, Instances +from annotator.oneformer.detectron2.utils.events import get_event_storage + +from ..postprocessing import detector_postprocess + + +def permute_to_N_HWA_K(tensor, K: int): + """ + Transpose/reshape a tensor from (N, (Ai x K), H, W) to (N, (HxWxAi), K) + """ + assert tensor.dim() == 4, tensor.shape + N, _, H, W = tensor.shape + tensor = tensor.view(N, -1, K, H, W) + tensor = tensor.permute(0, 3, 4, 1, 2) + tensor = tensor.reshape(N, -1, K) # Size=(N,HWA,K) + return tensor + + +class DenseDetector(nn.Module): + """ + Base class for dense detector. We define a dense detector as a fully-convolutional model that + makes per-pixel (i.e. dense) predictions. + """ + + def __init__( + self, + backbone: Backbone, + head: nn.Module, + head_in_features: Optional[List[str]] = None, + *, + pixel_mean, + pixel_std, + ): + """ + Args: + backbone: backbone module + head: head module + head_in_features: backbone features to use in head. Default to all backbone features. + pixel_mean (Tuple[float]): + Values to be used for image normalization (BGR order). + To train on images of different number of channels, set different mean & std. + Default values are the mean pixel value from ImageNet: [103.53, 116.28, 123.675] + pixel_std (Tuple[float]): + When using pre-trained models in Detectron1 or any MSRA models, + std has been absorbed into its conv1 weights, so the std needs to be set 1. + Otherwise, you can use [57.375, 57.120, 58.395] (ImageNet std) + """ + super().__init__() + + self.backbone = backbone + self.head = head + if head_in_features is None: + shapes = self.backbone.output_shape() + self.head_in_features = sorted(shapes.keys(), key=lambda x: shapes[x].stride) + else: + self.head_in_features = head_in_features + self.register_buffer("pixel_mean", torch.tensor(pixel_mean).view(-1, 1, 1), False) + self.register_buffer("pixel_std", torch.tensor(pixel_std).view(-1, 1, 1), False) + + @property + def device(self): + return self.pixel_mean.device + + def _move_to_current_device(self, x): + return move_device_like(x, self.pixel_mean) + + def forward(self, batched_inputs: List[Dict[str, Tensor]]): + """ + Args: + batched_inputs: a list, batched outputs of :class:`DatasetMapper` . + Each item in the list contains the inputs for one image. + For now, each item in the list is a dict that contains: + + * image: Tensor, image in (C, H, W) format. + * instances: Instances + + Other information that's included in the original dicts, such as: + + * "height", "width" (int): the output resolution of the model, used in inference. + See :meth:`postprocess` for details. + + Returns: + In training, dict[str, Tensor]: mapping from a named loss to a tensor storing the + loss. Used during training only. In inference, the standard output format, described + in :doc:`/tutorials/models`. + """ + images = self.preprocess_image(batched_inputs) + features = self.backbone(images.tensor) + features = [features[f] for f in self.head_in_features] + predictions = self.head(features) + + if self.training: + assert not torch.jit.is_scripting(), "Not supported" + assert "instances" in batched_inputs[0], "Instance annotations are missing in training!" + gt_instances = [x["instances"].to(self.device) for x in batched_inputs] + return self.forward_training(images, features, predictions, gt_instances) + else: + results = self.forward_inference(images, features, predictions) + if torch.jit.is_scripting(): + return results + + processed_results = [] + for results_per_image, input_per_image, image_size in zip( + results, batched_inputs, images.image_sizes + ): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + r = detector_postprocess(results_per_image, height, width) + processed_results.append({"instances": r}) + return processed_results + + def forward_training(self, images, features, predictions, gt_instances): + raise NotImplementedError() + + def preprocess_image(self, batched_inputs: List[Dict[str, Tensor]]): + """ + Normalize, pad and batch the input images. + """ + images = [self._move_to_current_device(x["image"]) for x in batched_inputs] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors( + images, + self.backbone.size_divisibility, + padding_constraints=self.backbone.padding_constraints, + ) + return images + + def _transpose_dense_predictions( + self, predictions: List[List[Tensor]], dims_per_anchor: List[int] + ) -> List[List[Tensor]]: + """ + Transpose the dense per-level predictions. + + Args: + predictions: a list of outputs, each is a list of per-level + predictions with shape (N, Ai x K, Hi, Wi), where N is the + number of images, Ai is the number of anchors per location on + level i, K is the dimension of predictions per anchor. + dims_per_anchor: the value of K for each predictions. e.g. 4 for + box prediction, #classes for classification prediction. + + Returns: + List[List[Tensor]]: each prediction is transposed to (N, Hi x Wi x Ai, K). + """ + assert len(predictions) == len(dims_per_anchor) + res: List[List[Tensor]] = [] + for pred, dim_per_anchor in zip(predictions, dims_per_anchor): + pred = [permute_to_N_HWA_K(x, dim_per_anchor) for x in pred] + res.append(pred) + return res + + def _ema_update(self, name: str, value: float, initial_value: float, momentum: float = 0.9): + """ + Apply EMA update to `self.name` using `value`. + + This is mainly used for loss normalizer. In Detectron1, loss is normalized by number + of foreground samples in the batch. When batch size is 1 per GPU, #foreground has a + large variance and using it lead to lower performance. Therefore we maintain an EMA of + #foreground to stabilize the normalizer. + + Args: + name: name of the normalizer + value: the new value to update + initial_value: the initial value to start with + momentum: momentum of EMA + + Returns: + float: the updated EMA value + """ + if hasattr(self, name): + old = getattr(self, name) + else: + old = initial_value + new = old * momentum + value * (1 - momentum) + setattr(self, name, new) + return new + + def _decode_per_level_predictions( + self, + anchors: Boxes, + pred_scores: Tensor, + pred_deltas: Tensor, + score_thresh: float, + topk_candidates: int, + image_size: Tuple[int, int], + ) -> Instances: + """ + Decode boxes and classification predictions of one featuer level, by + the following steps: + 1. filter the predictions based on score threshold and top K scores. + 2. transform the box regression outputs + 3. return the predicted scores, classes and boxes + + Args: + anchors: Boxes, anchor for this feature level + pred_scores: HxWxA,K + pred_deltas: HxWxA,4 + + Returns: + Instances: with field "scores", "pred_boxes", "pred_classes". + """ + # Apply two filtering to make NMS faster. + # 1. Keep boxes with confidence score higher than threshold + keep_idxs = pred_scores > score_thresh + pred_scores = pred_scores[keep_idxs] + topk_idxs = torch.nonzero(keep_idxs) # Kx2 + + # 2. Keep top k top scoring boxes only + topk_idxs_size = topk_idxs.shape[0] + if isinstance(topk_idxs_size, Tensor): + # It's a tensor in tracing + num_topk = torch.clamp(topk_idxs_size, max=topk_candidates) + else: + num_topk = min(topk_idxs_size, topk_candidates) + pred_scores, idxs = pred_scores.topk(num_topk) + topk_idxs = topk_idxs[idxs] + + anchor_idxs, classes_idxs = topk_idxs.unbind(dim=1) + + pred_boxes = self.box2box_transform.apply_deltas( + pred_deltas[anchor_idxs], anchors.tensor[anchor_idxs] + ) + return Instances( + image_size, pred_boxes=Boxes(pred_boxes), scores=pred_scores, pred_classes=classes_idxs + ) + + def _decode_multi_level_predictions( + self, + anchors: List[Boxes], + pred_scores: List[Tensor], + pred_deltas: List[Tensor], + score_thresh: float, + topk_candidates: int, + image_size: Tuple[int, int], + ) -> Instances: + """ + Run `_decode_per_level_predictions` for all feature levels and concat the results. + """ + predictions = [ + self._decode_per_level_predictions( + anchors_i, + box_cls_i, + box_reg_i, + self.test_score_thresh, + self.test_topk_candidates, + image_size, + ) + # Iterate over every feature level + for box_cls_i, box_reg_i, anchors_i in zip(pred_scores, pred_deltas, anchors) + ] + return predictions[0].cat(predictions) # 'Instances.cat' is not scriptale but this is + + def visualize_training(self, batched_inputs, results): + """ + A function used to visualize ground truth images and final network predictions. + It shows ground truth bounding boxes on the original image and up to 20 + predicted object bounding boxes on the original image. + + Args: + batched_inputs (list): a list that contains input to the model. + results (List[Instances]): a list of #images elements returned by forward_inference(). + """ + from annotator.oneformer.detectron2.utils.visualizer import Visualizer + + assert len(batched_inputs) == len( + results + ), "Cannot visualize inputs and results of different sizes" + storage = get_event_storage() + max_boxes = 20 + + image_index = 0 # only visualize a single image + img = batched_inputs[image_index]["image"] + img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format) + v_gt = Visualizer(img, None) + v_gt = v_gt.overlay_instances(boxes=batched_inputs[image_index]["instances"].gt_boxes) + anno_img = v_gt.get_image() + processed_results = detector_postprocess(results[image_index], img.shape[0], img.shape[1]) + predicted_boxes = processed_results.pred_boxes.tensor.detach().cpu().numpy() + + v_pred = Visualizer(img, None) + v_pred = v_pred.overlay_instances(boxes=predicted_boxes[0:max_boxes]) + prop_img = v_pred.get_image() + vis_img = np.vstack((anno_img, prop_img)) + vis_img = vis_img.transpose(2, 0, 1) + vis_name = f"Top: GT bounding boxes; Bottom: {max_boxes} Highest Scoring Results" + storage.put_image(vis_name, vis_img) diff --git a/annotator/oneformer/detectron2/modeling/meta_arch/fcos.py b/annotator/oneformer/detectron2/modeling/meta_arch/fcos.py new file mode 100644 index 0000000000000000000000000000000000000000..150726a459b99c1aa26213043b8e609213218201 --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/meta_arch/fcos.py @@ -0,0 +1,328 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import logging +from typing import List, Optional, Tuple +import torch +from fvcore.nn import sigmoid_focal_loss_jit +from torch import nn +from torch.nn import functional as F + +from annotator.oneformer.detectron2.layers import ShapeSpec, batched_nms +from annotator.oneformer.detectron2.structures import Boxes, ImageList, Instances, pairwise_point_box_distance +from annotator.oneformer.detectron2.utils.events import get_event_storage + +from ..anchor_generator import DefaultAnchorGenerator +from ..backbone import Backbone +from ..box_regression import Box2BoxTransformLinear, _dense_box_regression_loss +from .dense_detector import DenseDetector +from .retinanet import RetinaNetHead + +__all__ = ["FCOS"] + +logger = logging.getLogger(__name__) + + +class FCOS(DenseDetector): + """ + Implement FCOS in :paper:`fcos`. + """ + + def __init__( + self, + *, + backbone: Backbone, + head: nn.Module, + head_in_features: Optional[List[str]] = None, + box2box_transform=None, + num_classes, + center_sampling_radius: float = 1.5, + focal_loss_alpha=0.25, + focal_loss_gamma=2.0, + test_score_thresh=0.2, + test_topk_candidates=1000, + test_nms_thresh=0.6, + max_detections_per_image=100, + pixel_mean, + pixel_std, + ): + """ + Args: + center_sampling_radius: radius of the "center" of a groundtruth box, + within which all anchor points are labeled positive. + Other arguments mean the same as in :class:`RetinaNet`. + """ + super().__init__( + backbone, head, head_in_features, pixel_mean=pixel_mean, pixel_std=pixel_std + ) + + self.num_classes = num_classes + + # FCOS uses one anchor point per location. + # We represent the anchor point by a box whose size equals the anchor stride. + feature_shapes = backbone.output_shape() + fpn_strides = [feature_shapes[k].stride for k in self.head_in_features] + self.anchor_generator = DefaultAnchorGenerator( + sizes=[[k] for k in fpn_strides], aspect_ratios=[1.0], strides=fpn_strides + ) + + # FCOS parameterizes box regression by a linear transform, + # where predictions are normalized by anchor stride (equal to anchor size). + if box2box_transform is None: + box2box_transform = Box2BoxTransformLinear(normalize_by_size=True) + self.box2box_transform = box2box_transform + + self.center_sampling_radius = float(center_sampling_radius) + + # Loss parameters: + self.focal_loss_alpha = focal_loss_alpha + self.focal_loss_gamma = focal_loss_gamma + + # Inference parameters: + self.test_score_thresh = test_score_thresh + self.test_topk_candidates = test_topk_candidates + self.test_nms_thresh = test_nms_thresh + self.max_detections_per_image = max_detections_per_image + + def forward_training(self, images, features, predictions, gt_instances): + # Transpose the Hi*Wi*A dimension to the middle: + pred_logits, pred_anchor_deltas, pred_centerness = self._transpose_dense_predictions( + predictions, [self.num_classes, 4, 1] + ) + anchors = self.anchor_generator(features) + gt_labels, gt_boxes = self.label_anchors(anchors, gt_instances) + return self.losses( + anchors, pred_logits, gt_labels, pred_anchor_deltas, gt_boxes, pred_centerness + ) + + @torch.no_grad() + def _match_anchors(self, gt_boxes: Boxes, anchors: List[Boxes]): + """ + Match ground-truth boxes to a set of multi-level anchors. + + Args: + gt_boxes: Ground-truth boxes from instances of an image. + anchors: List of anchors for each feature map (of different scales). + + Returns: + torch.Tensor + A tensor of shape `(M, R)`, given `M` ground-truth boxes and total + `R` anchor points from all feature levels, indicating the quality + of match between m-th box and r-th anchor. Higher value indicates + better match. + """ + # Naming convention: (M = ground-truth boxes, R = anchor points) + # Anchor points are represented as square boxes of size = stride. + num_anchors_per_level = [len(x) for x in anchors] + anchors = Boxes.cat(anchors) # (R, 4) + anchor_centers = anchors.get_centers() # (R, 2) + anchor_sizes = anchors.tensor[:, 2] - anchors.tensor[:, 0] # (R, ) + + lower_bound = anchor_sizes * 4 + lower_bound[: num_anchors_per_level[0]] = 0 + upper_bound = anchor_sizes * 8 + upper_bound[-num_anchors_per_level[-1] :] = float("inf") + + gt_centers = gt_boxes.get_centers() + + # FCOS with center sampling: anchor point must be close enough to + # ground-truth box center. + center_dists = (anchor_centers[None, :, :] - gt_centers[:, None, :]).abs_() + sampling_regions = self.center_sampling_radius * anchor_sizes[None, :] + + match_quality_matrix = center_dists.max(dim=2).values < sampling_regions + + pairwise_dist = pairwise_point_box_distance(anchor_centers, gt_boxes) + pairwise_dist = pairwise_dist.permute(1, 0, 2) # (M, R, 4) + + # The original FCOS anchor matching rule: anchor point must be inside GT. + match_quality_matrix &= pairwise_dist.min(dim=2).values > 0 + + # Multilevel anchor matching in FCOS: each anchor is only responsible + # for certain scale range. + pairwise_dist = pairwise_dist.max(dim=2).values + match_quality_matrix &= (pairwise_dist > lower_bound[None, :]) & ( + pairwise_dist < upper_bound[None, :] + ) + # Match the GT box with minimum area, if there are multiple GT matches. + gt_areas = gt_boxes.area() # (M, ) + + match_quality_matrix = match_quality_matrix.to(torch.float32) + match_quality_matrix *= 1e8 - gt_areas[:, None] + return match_quality_matrix # (M, R) + + @torch.no_grad() + def label_anchors(self, anchors: List[Boxes], gt_instances: List[Instances]): + """ + Same interface as :meth:`RetinaNet.label_anchors`, but implemented with FCOS + anchor matching rule. + + Unlike RetinaNet, there are no ignored anchors. + """ + + gt_labels, matched_gt_boxes = [], [] + + for inst in gt_instances: + if len(inst) > 0: + match_quality_matrix = self._match_anchors(inst.gt_boxes, anchors) + + # Find matched ground-truth box per anchor. Un-matched anchors are + # assigned -1. This is equivalent to using an anchor matcher as used + # in R-CNN/RetinaNet: `Matcher(thresholds=[1e-5], labels=[0, 1])` + match_quality, matched_idxs = match_quality_matrix.max(dim=0) + matched_idxs[match_quality < 1e-5] = -1 + + matched_gt_boxes_i = inst.gt_boxes.tensor[matched_idxs.clip(min=0)] + gt_labels_i = inst.gt_classes[matched_idxs.clip(min=0)] + + # Anchors with matched_idxs = -1 are labeled background. + gt_labels_i[matched_idxs < 0] = self.num_classes + else: + matched_gt_boxes_i = torch.zeros_like(Boxes.cat(anchors).tensor) + gt_labels_i = torch.full( + (len(matched_gt_boxes_i),), + fill_value=self.num_classes, + dtype=torch.long, + device=matched_gt_boxes_i.device, + ) + + gt_labels.append(gt_labels_i) + matched_gt_boxes.append(matched_gt_boxes_i) + + return gt_labels, matched_gt_boxes + + def losses( + self, anchors, pred_logits, gt_labels, pred_anchor_deltas, gt_boxes, pred_centerness + ): + """ + This method is almost identical to :meth:`RetinaNet.losses`, with an extra + "loss_centerness" in the returned dict. + """ + num_images = len(gt_labels) + gt_labels = torch.stack(gt_labels) # (M, R) + + pos_mask = (gt_labels >= 0) & (gt_labels != self.num_classes) + num_pos_anchors = pos_mask.sum().item() + get_event_storage().put_scalar("num_pos_anchors", num_pos_anchors / num_images) + normalizer = self._ema_update("loss_normalizer", max(num_pos_anchors, 1), 300) + + # classification and regression loss + gt_labels_target = F.one_hot(gt_labels, num_classes=self.num_classes + 1)[ + :, :, :-1 + ] # no loss for the last (background) class + loss_cls = sigmoid_focal_loss_jit( + torch.cat(pred_logits, dim=1), + gt_labels_target.to(pred_logits[0].dtype), + alpha=self.focal_loss_alpha, + gamma=self.focal_loss_gamma, + reduction="sum", + ) + + loss_box_reg = _dense_box_regression_loss( + anchors, + self.box2box_transform, + pred_anchor_deltas, + gt_boxes, + pos_mask, + box_reg_loss_type="giou", + ) + + ctrness_targets = self.compute_ctrness_targets(anchors, gt_boxes) # (M, R) + pred_centerness = torch.cat(pred_centerness, dim=1).squeeze(dim=2) # (M, R) + ctrness_loss = F.binary_cross_entropy_with_logits( + pred_centerness[pos_mask], ctrness_targets[pos_mask], reduction="sum" + ) + return { + "loss_fcos_cls": loss_cls / normalizer, + "loss_fcos_loc": loss_box_reg / normalizer, + "loss_fcos_ctr": ctrness_loss / normalizer, + } + + def compute_ctrness_targets(self, anchors: List[Boxes], gt_boxes: List[torch.Tensor]): + anchors = Boxes.cat(anchors).tensor # Rx4 + reg_targets = [self.box2box_transform.get_deltas(anchors, m) for m in gt_boxes] + reg_targets = torch.stack(reg_targets, dim=0) # NxRx4 + if len(reg_targets) == 0: + return reg_targets.new_zeros(len(reg_targets)) + left_right = reg_targets[:, :, [0, 2]] + top_bottom = reg_targets[:, :, [1, 3]] + ctrness = (left_right.min(dim=-1)[0] / left_right.max(dim=-1)[0]) * ( + top_bottom.min(dim=-1)[0] / top_bottom.max(dim=-1)[0] + ) + return torch.sqrt(ctrness) + + def forward_inference( + self, + images: ImageList, + features: List[torch.Tensor], + predictions: List[List[torch.Tensor]], + ): + pred_logits, pred_anchor_deltas, pred_centerness = self._transpose_dense_predictions( + predictions, [self.num_classes, 4, 1] + ) + anchors = self.anchor_generator(features) + + results: List[Instances] = [] + for img_idx, image_size in enumerate(images.image_sizes): + scores_per_image = [ + # Multiply and sqrt centerness & classification scores + # (See eqn. 4 in https://arxiv.org/abs/2006.09214) + torch.sqrt(x[img_idx].sigmoid_() * y[img_idx].sigmoid_()) + for x, y in zip(pred_logits, pred_centerness) + ] + deltas_per_image = [x[img_idx] for x in pred_anchor_deltas] + results_per_image = self.inference_single_image( + anchors, scores_per_image, deltas_per_image, image_size + ) + results.append(results_per_image) + return results + + def inference_single_image( + self, + anchors: List[Boxes], + box_cls: List[torch.Tensor], + box_delta: List[torch.Tensor], + image_size: Tuple[int, int], + ): + """ + Identical to :meth:`RetinaNet.inference_single_image. + """ + pred = self._decode_multi_level_predictions( + anchors, + box_cls, + box_delta, + self.test_score_thresh, + self.test_topk_candidates, + image_size, + ) + keep = batched_nms( + pred.pred_boxes.tensor, pred.scores, pred.pred_classes, self.test_nms_thresh + ) + return pred[keep[: self.max_detections_per_image]] + + +class FCOSHead(RetinaNetHead): + """ + The head used in :paper:`fcos`. It adds an additional centerness + prediction branch on top of :class:`RetinaNetHead`. + """ + + def __init__(self, *, input_shape: List[ShapeSpec], conv_dims: List[int], **kwargs): + super().__init__(input_shape=input_shape, conv_dims=conv_dims, num_anchors=1, **kwargs) + # Unlike original FCOS, we do not add an additional learnable scale layer + # because it's found to have no benefits after normalizing regression targets by stride. + self._num_features = len(input_shape) + self.ctrness = nn.Conv2d(conv_dims[-1], 1, kernel_size=3, stride=1, padding=1) + torch.nn.init.normal_(self.ctrness.weight, std=0.01) + torch.nn.init.constant_(self.ctrness.bias, 0) + + def forward(self, features): + assert len(features) == self._num_features + logits = [] + bbox_reg = [] + ctrness = [] + for feature in features: + logits.append(self.cls_score(self.cls_subnet(feature))) + bbox_feature = self.bbox_subnet(feature) + bbox_reg.append(self.bbox_pred(bbox_feature)) + ctrness.append(self.ctrness(bbox_feature)) + return logits, bbox_reg, ctrness diff --git a/annotator/oneformer/detectron2/modeling/meta_arch/panoptic_fpn.py b/annotator/oneformer/detectron2/modeling/meta_arch/panoptic_fpn.py new file mode 100644 index 0000000000000000000000000000000000000000..1ca5f19a0ce0099a49aad8bb6b659355c4f6e200 --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/meta_arch/panoptic_fpn.py @@ -0,0 +1,269 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + +import logging +from typing import Dict, List +import torch +from torch import nn + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.structures import ImageList + +from ..postprocessing import detector_postprocess, sem_seg_postprocess +from .build import META_ARCH_REGISTRY +from .rcnn import GeneralizedRCNN +from .semantic_seg import build_sem_seg_head + +__all__ = ["PanopticFPN"] + + +@META_ARCH_REGISTRY.register() +class PanopticFPN(GeneralizedRCNN): + """ + Implement the paper :paper:`PanopticFPN`. + """ + + @configurable + def __init__( + self, + *, + sem_seg_head: nn.Module, + combine_overlap_thresh: float = 0.5, + combine_stuff_area_thresh: float = 4096, + combine_instances_score_thresh: float = 0.5, + **kwargs, + ): + """ + NOTE: this interface is experimental. + + Args: + sem_seg_head: a module for the semantic segmentation head. + combine_overlap_thresh: combine masks into one instances if + they have enough overlap + combine_stuff_area_thresh: ignore stuff areas smaller than this threshold + combine_instances_score_thresh: ignore instances whose score is + smaller than this threshold + + Other arguments are the same as :class:`GeneralizedRCNN`. + """ + super().__init__(**kwargs) + self.sem_seg_head = sem_seg_head + # options when combining instance & semantic outputs + self.combine_overlap_thresh = combine_overlap_thresh + self.combine_stuff_area_thresh = combine_stuff_area_thresh + self.combine_instances_score_thresh = combine_instances_score_thresh + + @classmethod + def from_config(cls, cfg): + ret = super().from_config(cfg) + ret.update( + { + "combine_overlap_thresh": cfg.MODEL.PANOPTIC_FPN.COMBINE.OVERLAP_THRESH, + "combine_stuff_area_thresh": cfg.MODEL.PANOPTIC_FPN.COMBINE.STUFF_AREA_LIMIT, + "combine_instances_score_thresh": cfg.MODEL.PANOPTIC_FPN.COMBINE.INSTANCES_CONFIDENCE_THRESH, # noqa + } + ) + ret["sem_seg_head"] = build_sem_seg_head(cfg, ret["backbone"].output_shape()) + logger = logging.getLogger(__name__) + if not cfg.MODEL.PANOPTIC_FPN.COMBINE.ENABLED: + logger.warning( + "PANOPTIC_FPN.COMBINED.ENABLED is no longer used. " + " model.inference(do_postprocess=) should be used to toggle postprocessing." + ) + if cfg.MODEL.PANOPTIC_FPN.INSTANCE_LOSS_WEIGHT != 1.0: + w = cfg.MODEL.PANOPTIC_FPN.INSTANCE_LOSS_WEIGHT + logger.warning( + "PANOPTIC_FPN.INSTANCE_LOSS_WEIGHT should be replaced by weights on each ROI head." + ) + + def update_weight(x): + if isinstance(x, dict): + return {k: v * w for k, v in x.items()} + else: + return x * w + + roi_heads = ret["roi_heads"] + roi_heads.box_predictor.loss_weight = update_weight(roi_heads.box_predictor.loss_weight) + roi_heads.mask_head.loss_weight = update_weight(roi_heads.mask_head.loss_weight) + return ret + + def forward(self, batched_inputs): + """ + Args: + batched_inputs: a list, batched outputs of :class:`DatasetMapper`. + Each item in the list contains the inputs for one image. + + For now, each item in the list is a dict that contains: + + * "image": Tensor, image in (C, H, W) format. + * "instances": Instances + * "sem_seg": semantic segmentation ground truth. + * Other information that's included in the original dicts, such as: + "height", "width" (int): the output resolution of the model, used in inference. + See :meth:`postprocess` for details. + + Returns: + list[dict]: + each dict has the results for one image. The dict contains the following keys: + + * "instances": see :meth:`GeneralizedRCNN.forward` for its format. + * "sem_seg": see :meth:`SemanticSegmentor.forward` for its format. + * "panoptic_seg": See the return value of + :func:`combine_semantic_and_instance_outputs` for its format. + """ + if not self.training: + return self.inference(batched_inputs) + images = self.preprocess_image(batched_inputs) + features = self.backbone(images.tensor) + + assert "sem_seg" in batched_inputs[0] + gt_sem_seg = [x["sem_seg"].to(self.device) for x in batched_inputs] + gt_sem_seg = ImageList.from_tensors( + gt_sem_seg, + self.backbone.size_divisibility, + self.sem_seg_head.ignore_value, + self.backbone.padding_constraints, + ).tensor + sem_seg_results, sem_seg_losses = self.sem_seg_head(features, gt_sem_seg) + + gt_instances = [x["instances"].to(self.device) for x in batched_inputs] + proposals, proposal_losses = self.proposal_generator(images, features, gt_instances) + detector_results, detector_losses = self.roi_heads( + images, features, proposals, gt_instances + ) + + losses = sem_seg_losses + losses.update(proposal_losses) + losses.update(detector_losses) + return losses + + def inference(self, batched_inputs: List[Dict[str, torch.Tensor]], do_postprocess: bool = True): + """ + Run inference on the given inputs. + + Args: + batched_inputs (list[dict]): same as in :meth:`forward` + do_postprocess (bool): whether to apply post-processing on the outputs. + + Returns: + When do_postprocess=True, see docs in :meth:`forward`. + Otherwise, returns a (list[Instances], list[Tensor]) that contains + the raw detector outputs, and raw semantic segmentation outputs. + """ + images = self.preprocess_image(batched_inputs) + features = self.backbone(images.tensor) + sem_seg_results, sem_seg_losses = self.sem_seg_head(features, None) + proposals, _ = self.proposal_generator(images, features, None) + detector_results, _ = self.roi_heads(images, features, proposals, None) + + if do_postprocess: + processed_results = [] + for sem_seg_result, detector_result, input_per_image, image_size in zip( + sem_seg_results, detector_results, batched_inputs, images.image_sizes + ): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + sem_seg_r = sem_seg_postprocess(sem_seg_result, image_size, height, width) + detector_r = detector_postprocess(detector_result, height, width) + + processed_results.append({"sem_seg": sem_seg_r, "instances": detector_r}) + + panoptic_r = combine_semantic_and_instance_outputs( + detector_r, + sem_seg_r.argmax(dim=0), + self.combine_overlap_thresh, + self.combine_stuff_area_thresh, + self.combine_instances_score_thresh, + ) + processed_results[-1]["panoptic_seg"] = panoptic_r + return processed_results + else: + return detector_results, sem_seg_results + + +def combine_semantic_and_instance_outputs( + instance_results, + semantic_results, + overlap_threshold, + stuff_area_thresh, + instances_score_thresh, +): + """ + Implement a simple combining logic following + "combine_semantic_and_instance_predictions.py" in panopticapi + to produce panoptic segmentation outputs. + + Args: + instance_results: output of :func:`detector_postprocess`. + semantic_results: an (H, W) tensor, each element is the contiguous semantic + category id + + Returns: + panoptic_seg (Tensor): of shape (height, width) where the values are ids for each segment. + segments_info (list[dict]): Describe each segment in `panoptic_seg`. + Each dict contains keys "id", "category_id", "isthing". + """ + panoptic_seg = torch.zeros_like(semantic_results, dtype=torch.int32) + + # sort instance outputs by scores + sorted_inds = torch.argsort(-instance_results.scores) + + current_segment_id = 0 + segments_info = [] + + instance_masks = instance_results.pred_masks.to(dtype=torch.bool, device=panoptic_seg.device) + + # Add instances one-by-one, check for overlaps with existing ones + for inst_id in sorted_inds: + score = instance_results.scores[inst_id].item() + if score < instances_score_thresh: + break + mask = instance_masks[inst_id] # H,W + mask_area = mask.sum().item() + + if mask_area == 0: + continue + + intersect = (mask > 0) & (panoptic_seg > 0) + intersect_area = intersect.sum().item() + + if intersect_area * 1.0 / mask_area > overlap_threshold: + continue + + if intersect_area > 0: + mask = mask & (panoptic_seg == 0) + + current_segment_id += 1 + panoptic_seg[mask] = current_segment_id + segments_info.append( + { + "id": current_segment_id, + "isthing": True, + "score": score, + "category_id": instance_results.pred_classes[inst_id].item(), + "instance_id": inst_id.item(), + } + ) + + # Add semantic results to remaining empty areas + semantic_labels = torch.unique(semantic_results).cpu().tolist() + for semantic_label in semantic_labels: + if semantic_label == 0: # 0 is a special "thing" class + continue + mask = (semantic_results == semantic_label) & (panoptic_seg == 0) + mask_area = mask.sum().item() + if mask_area < stuff_area_thresh: + continue + + current_segment_id += 1 + panoptic_seg[mask] = current_segment_id + segments_info.append( + { + "id": current_segment_id, + "isthing": False, + "category_id": semantic_label, + "area": mask_area, + } + ) + + return panoptic_seg, segments_info diff --git a/annotator/oneformer/detectron2/modeling/meta_arch/rcnn.py b/annotator/oneformer/detectron2/modeling/meta_arch/rcnn.py new file mode 100644 index 0000000000000000000000000000000000000000..7cacf065ed2803686f80c8e6f562ebfeb5d584d5 --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/meta_arch/rcnn.py @@ -0,0 +1,341 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import numpy as np +from typing import Dict, List, Optional, Tuple +import torch +from torch import nn + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.data.detection_utils import convert_image_to_rgb +from annotator.oneformer.detectron2.layers import move_device_like +from annotator.oneformer.detectron2.structures import ImageList, Instances +from annotator.oneformer.detectron2.utils.events import get_event_storage +from annotator.oneformer.detectron2.utils.logger import log_first_n + +from ..backbone import Backbone, build_backbone +from ..postprocessing import detector_postprocess +from ..proposal_generator import build_proposal_generator +from ..roi_heads import build_roi_heads +from .build import META_ARCH_REGISTRY + +__all__ = ["GeneralizedRCNN", "ProposalNetwork"] + + +@META_ARCH_REGISTRY.register() +class GeneralizedRCNN(nn.Module): + """ + Generalized R-CNN. Any models that contains the following three components: + 1. Per-image feature extraction (aka backbone) + 2. Region proposal generation + 3. Per-region feature extraction and prediction + """ + + @configurable + def __init__( + self, + *, + backbone: Backbone, + proposal_generator: nn.Module, + roi_heads: nn.Module, + pixel_mean: Tuple[float], + pixel_std: Tuple[float], + input_format: Optional[str] = None, + vis_period: int = 0, + ): + """ + Args: + backbone: a backbone module, must follow detectron2's backbone interface + proposal_generator: a module that generates proposals using backbone features + roi_heads: a ROI head that performs per-region computation + pixel_mean, pixel_std: list or tuple with #channels element, representing + the per-channel mean and std to be used to normalize the input image + input_format: describe the meaning of channels of input. Needed by visualization + vis_period: the period to run visualization. Set to 0 to disable. + """ + super().__init__() + self.backbone = backbone + self.proposal_generator = proposal_generator + self.roi_heads = roi_heads + + self.input_format = input_format + self.vis_period = vis_period + if vis_period > 0: + assert input_format is not None, "input_format is required for visualization!" + + self.register_buffer("pixel_mean", torch.tensor(pixel_mean).view(-1, 1, 1), False) + self.register_buffer("pixel_std", torch.tensor(pixel_std).view(-1, 1, 1), False) + assert ( + self.pixel_mean.shape == self.pixel_std.shape + ), f"{self.pixel_mean} and {self.pixel_std} have different shapes!" + + @classmethod + def from_config(cls, cfg): + backbone = build_backbone(cfg) + return { + "backbone": backbone, + "proposal_generator": build_proposal_generator(cfg, backbone.output_shape()), + "roi_heads": build_roi_heads(cfg, backbone.output_shape()), + "input_format": cfg.INPUT.FORMAT, + "vis_period": cfg.VIS_PERIOD, + "pixel_mean": cfg.MODEL.PIXEL_MEAN, + "pixel_std": cfg.MODEL.PIXEL_STD, + } + + @property + def device(self): + return self.pixel_mean.device + + def _move_to_current_device(self, x): + return move_device_like(x, self.pixel_mean) + + def visualize_training(self, batched_inputs, proposals): + """ + A function used to visualize images and proposals. It shows ground truth + bounding boxes on the original image and up to 20 top-scoring predicted + object proposals on the original image. Users can implement different + visualization functions for different models. + + Args: + batched_inputs (list): a list that contains input to the model. + proposals (list): a list that contains predicted proposals. Both + batched_inputs and proposals should have the same length. + """ + from annotator.oneformer.detectron2.utils.visualizer import Visualizer + + storage = get_event_storage() + max_vis_prop = 20 + + for input, prop in zip(batched_inputs, proposals): + img = input["image"] + img = convert_image_to_rgb(img.permute(1, 2, 0), self.input_format) + v_gt = Visualizer(img, None) + v_gt = v_gt.overlay_instances(boxes=input["instances"].gt_boxes) + anno_img = v_gt.get_image() + box_size = min(len(prop.proposal_boxes), max_vis_prop) + v_pred = Visualizer(img, None) + v_pred = v_pred.overlay_instances( + boxes=prop.proposal_boxes[0:box_size].tensor.cpu().numpy() + ) + prop_img = v_pred.get_image() + vis_img = np.concatenate((anno_img, prop_img), axis=1) + vis_img = vis_img.transpose(2, 0, 1) + vis_name = "Left: GT bounding boxes; Right: Predicted proposals" + storage.put_image(vis_name, vis_img) + break # only visualize one image in a batch + + def forward(self, batched_inputs: List[Dict[str, torch.Tensor]]): + """ + Args: + batched_inputs: a list, batched outputs of :class:`DatasetMapper` . + Each item in the list contains the inputs for one image. + For now, each item in the list is a dict that contains: + + * image: Tensor, image in (C, H, W) format. + * instances (optional): groundtruth :class:`Instances` + * proposals (optional): :class:`Instances`, precomputed proposals. + + Other information that's included in the original dicts, such as: + + * "height", "width" (int): the output resolution of the model, used in inference. + See :meth:`postprocess` for details. + + Returns: + list[dict]: + Each dict is the output for one input image. + The dict contains one key "instances" whose value is a :class:`Instances`. + The :class:`Instances` object has the following keys: + "pred_boxes", "pred_classes", "scores", "pred_masks", "pred_keypoints" + """ + if not self.training: + return self.inference(batched_inputs) + + images = self.preprocess_image(batched_inputs) + if "instances" in batched_inputs[0]: + gt_instances = [x["instances"].to(self.device) for x in batched_inputs] + else: + gt_instances = None + + features = self.backbone(images.tensor) + + if self.proposal_generator is not None: + proposals, proposal_losses = self.proposal_generator(images, features, gt_instances) + else: + assert "proposals" in batched_inputs[0] + proposals = [x["proposals"].to(self.device) for x in batched_inputs] + proposal_losses = {} + + _, detector_losses = self.roi_heads(images, features, proposals, gt_instances) + if self.vis_period > 0: + storage = get_event_storage() + if storage.iter % self.vis_period == 0: + self.visualize_training(batched_inputs, proposals) + + losses = {} + losses.update(detector_losses) + losses.update(proposal_losses) + return losses + + def inference( + self, + batched_inputs: List[Dict[str, torch.Tensor]], + detected_instances: Optional[List[Instances]] = None, + do_postprocess: bool = True, + ): + """ + Run inference on the given inputs. + + Args: + batched_inputs (list[dict]): same as in :meth:`forward` + detected_instances (None or list[Instances]): if not None, it + contains an `Instances` object per image. The `Instances` + object contains "pred_boxes" and "pred_classes" which are + known boxes in the image. + The inference will then skip the detection of bounding boxes, + and only predict other per-ROI outputs. + do_postprocess (bool): whether to apply post-processing on the outputs. + + Returns: + When do_postprocess=True, same as in :meth:`forward`. + Otherwise, a list[Instances] containing raw network outputs. + """ + assert not self.training + + images = self.preprocess_image(batched_inputs) + features = self.backbone(images.tensor) + + if detected_instances is None: + if self.proposal_generator is not None: + proposals, _ = self.proposal_generator(images, features, None) + else: + assert "proposals" in batched_inputs[0] + proposals = [x["proposals"].to(self.device) for x in batched_inputs] + + results, _ = self.roi_heads(images, features, proposals, None) + else: + detected_instances = [x.to(self.device) for x in detected_instances] + results = self.roi_heads.forward_with_given_boxes(features, detected_instances) + + if do_postprocess: + assert not torch.jit.is_scripting(), "Scripting is not supported for postprocess." + return GeneralizedRCNN._postprocess(results, batched_inputs, images.image_sizes) + return results + + def preprocess_image(self, batched_inputs: List[Dict[str, torch.Tensor]]): + """ + Normalize, pad and batch the input images. + """ + images = [self._move_to_current_device(x["image"]) for x in batched_inputs] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors( + images, + self.backbone.size_divisibility, + padding_constraints=self.backbone.padding_constraints, + ) + return images + + @staticmethod + def _postprocess(instances, batched_inputs: List[Dict[str, torch.Tensor]], image_sizes): + """ + Rescale the output instances to the target size. + """ + # note: private function; subject to changes + processed_results = [] + for results_per_image, input_per_image, image_size in zip( + instances, batched_inputs, image_sizes + ): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + r = detector_postprocess(results_per_image, height, width) + processed_results.append({"instances": r}) + return processed_results + + +@META_ARCH_REGISTRY.register() +class ProposalNetwork(nn.Module): + """ + A meta architecture that only predicts object proposals. + """ + + @configurable + def __init__( + self, + *, + backbone: Backbone, + proposal_generator: nn.Module, + pixel_mean: Tuple[float], + pixel_std: Tuple[float], + ): + """ + Args: + backbone: a backbone module, must follow detectron2's backbone interface + proposal_generator: a module that generates proposals using backbone features + pixel_mean, pixel_std: list or tuple with #channels element, representing + the per-channel mean and std to be used to normalize the input image + """ + super().__init__() + self.backbone = backbone + self.proposal_generator = proposal_generator + self.register_buffer("pixel_mean", torch.tensor(pixel_mean).view(-1, 1, 1), False) + self.register_buffer("pixel_std", torch.tensor(pixel_std).view(-1, 1, 1), False) + + @classmethod + def from_config(cls, cfg): + backbone = build_backbone(cfg) + return { + "backbone": backbone, + "proposal_generator": build_proposal_generator(cfg, backbone.output_shape()), + "pixel_mean": cfg.MODEL.PIXEL_MEAN, + "pixel_std": cfg.MODEL.PIXEL_STD, + } + + @property + def device(self): + return self.pixel_mean.device + + def _move_to_current_device(self, x): + return move_device_like(x, self.pixel_mean) + + def forward(self, batched_inputs): + """ + Args: + Same as in :class:`GeneralizedRCNN.forward` + + Returns: + list[dict]: + Each dict is the output for one input image. + The dict contains one key "proposals" whose value is a + :class:`Instances` with keys "proposal_boxes" and "objectness_logits". + """ + images = [self._move_to_current_device(x["image"]) for x in batched_inputs] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors( + images, + self.backbone.size_divisibility, + padding_constraints=self.backbone.padding_constraints, + ) + features = self.backbone(images.tensor) + + if "instances" in batched_inputs[0]: + gt_instances = [x["instances"].to(self.device) for x in batched_inputs] + elif "targets" in batched_inputs[0]: + log_first_n( + logging.WARN, "'targets' in the model inputs is now renamed to 'instances'!", n=10 + ) + gt_instances = [x["targets"].to(self.device) for x in batched_inputs] + else: + gt_instances = None + proposals, proposal_losses = self.proposal_generator(images, features, gt_instances) + # In training, the proposals are not useful at all but we generate them anyway. + # This makes RPN-only models about 5% slower. + if self.training: + return proposal_losses + + processed_results = [] + for results_per_image, input_per_image, image_size in zip( + proposals, batched_inputs, images.image_sizes + ): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + r = detector_postprocess(results_per_image, height, width) + processed_results.append({"proposals": r}) + return processed_results diff --git a/annotator/oneformer/detectron2/modeling/meta_arch/retinanet.py b/annotator/oneformer/detectron2/modeling/meta_arch/retinanet.py new file mode 100644 index 0000000000000000000000000000000000000000..46e0fda48254f2d1e6b8c796e00467df669e4216 --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/meta_arch/retinanet.py @@ -0,0 +1,439 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import math +from typing import List, Tuple +import torch +from fvcore.nn import sigmoid_focal_loss_jit +from torch import Tensor, nn +from torch.nn import functional as F + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.layers import CycleBatchNormList, ShapeSpec, batched_nms, cat, get_norm +from annotator.oneformer.detectron2.structures import Boxes, ImageList, Instances, pairwise_iou +from annotator.oneformer.detectron2.utils.events import get_event_storage + +from ..anchor_generator import build_anchor_generator +from ..backbone import Backbone, build_backbone +from ..box_regression import Box2BoxTransform, _dense_box_regression_loss +from ..matcher import Matcher +from .build import META_ARCH_REGISTRY +from .dense_detector import DenseDetector, permute_to_N_HWA_K # noqa + +__all__ = ["RetinaNet"] + + +logger = logging.getLogger(__name__) + + +@META_ARCH_REGISTRY.register() +class RetinaNet(DenseDetector): + """ + Implement RetinaNet in :paper:`RetinaNet`. + """ + + @configurable + def __init__( + self, + *, + backbone: Backbone, + head: nn.Module, + head_in_features, + anchor_generator, + box2box_transform, + anchor_matcher, + num_classes, + focal_loss_alpha=0.25, + focal_loss_gamma=2.0, + smooth_l1_beta=0.0, + box_reg_loss_type="smooth_l1", + test_score_thresh=0.05, + test_topk_candidates=1000, + test_nms_thresh=0.5, + max_detections_per_image=100, + pixel_mean, + pixel_std, + vis_period=0, + input_format="BGR", + ): + """ + NOTE: this interface is experimental. + + Args: + backbone: a backbone module, must follow detectron2's backbone interface + head (nn.Module): a module that predicts logits and regression deltas + for each level from a list of per-level features + head_in_features (Tuple[str]): Names of the input feature maps to be used in head + anchor_generator (nn.Module): a module that creates anchors from a + list of features. Usually an instance of :class:`AnchorGenerator` + box2box_transform (Box2BoxTransform): defines the transform from anchors boxes to + instance boxes + anchor_matcher (Matcher): label the anchors by matching them with ground truth. + num_classes (int): number of classes. Used to label background proposals. + + # Loss parameters: + focal_loss_alpha (float): focal_loss_alpha + focal_loss_gamma (float): focal_loss_gamma + smooth_l1_beta (float): smooth_l1_beta + box_reg_loss_type (str): Options are "smooth_l1", "giou", "diou", "ciou" + + # Inference parameters: + test_score_thresh (float): Inference cls score threshold, only anchors with + score > INFERENCE_TH are considered for inference (to improve speed) + test_topk_candidates (int): Select topk candidates before NMS + test_nms_thresh (float): Overlap threshold used for non-maximum suppression + (suppress boxes with IoU >= this threshold) + max_detections_per_image (int): + Maximum number of detections to return per image during inference + (100 is based on the limit established for the COCO dataset). + + pixel_mean, pixel_std: see :class:`DenseDetector`. + """ + super().__init__( + backbone, head, head_in_features, pixel_mean=pixel_mean, pixel_std=pixel_std + ) + self.num_classes = num_classes + + # Anchors + self.anchor_generator = anchor_generator + self.box2box_transform = box2box_transform + self.anchor_matcher = anchor_matcher + + # Loss parameters: + self.focal_loss_alpha = focal_loss_alpha + self.focal_loss_gamma = focal_loss_gamma + self.smooth_l1_beta = smooth_l1_beta + self.box_reg_loss_type = box_reg_loss_type + # Inference parameters: + self.test_score_thresh = test_score_thresh + self.test_topk_candidates = test_topk_candidates + self.test_nms_thresh = test_nms_thresh + self.max_detections_per_image = max_detections_per_image + # Vis parameters + self.vis_period = vis_period + self.input_format = input_format + + @classmethod + def from_config(cls, cfg): + backbone = build_backbone(cfg) + backbone_shape = backbone.output_shape() + feature_shapes = [backbone_shape[f] for f in cfg.MODEL.RETINANET.IN_FEATURES] + head = RetinaNetHead(cfg, feature_shapes) + anchor_generator = build_anchor_generator(cfg, feature_shapes) + return { + "backbone": backbone, + "head": head, + "anchor_generator": anchor_generator, + "box2box_transform": Box2BoxTransform(weights=cfg.MODEL.RETINANET.BBOX_REG_WEIGHTS), + "anchor_matcher": Matcher( + cfg.MODEL.RETINANET.IOU_THRESHOLDS, + cfg.MODEL.RETINANET.IOU_LABELS, + allow_low_quality_matches=True, + ), + "pixel_mean": cfg.MODEL.PIXEL_MEAN, + "pixel_std": cfg.MODEL.PIXEL_STD, + "num_classes": cfg.MODEL.RETINANET.NUM_CLASSES, + "head_in_features": cfg.MODEL.RETINANET.IN_FEATURES, + # Loss parameters: + "focal_loss_alpha": cfg.MODEL.RETINANET.FOCAL_LOSS_ALPHA, + "focal_loss_gamma": cfg.MODEL.RETINANET.FOCAL_LOSS_GAMMA, + "smooth_l1_beta": cfg.MODEL.RETINANET.SMOOTH_L1_LOSS_BETA, + "box_reg_loss_type": cfg.MODEL.RETINANET.BBOX_REG_LOSS_TYPE, + # Inference parameters: + "test_score_thresh": cfg.MODEL.RETINANET.SCORE_THRESH_TEST, + "test_topk_candidates": cfg.MODEL.RETINANET.TOPK_CANDIDATES_TEST, + "test_nms_thresh": cfg.MODEL.RETINANET.NMS_THRESH_TEST, + "max_detections_per_image": cfg.TEST.DETECTIONS_PER_IMAGE, + # Vis parameters + "vis_period": cfg.VIS_PERIOD, + "input_format": cfg.INPUT.FORMAT, + } + + def forward_training(self, images, features, predictions, gt_instances): + # Transpose the Hi*Wi*A dimension to the middle: + pred_logits, pred_anchor_deltas = self._transpose_dense_predictions( + predictions, [self.num_classes, 4] + ) + anchors = self.anchor_generator(features) + gt_labels, gt_boxes = self.label_anchors(anchors, gt_instances) + return self.losses(anchors, pred_logits, gt_labels, pred_anchor_deltas, gt_boxes) + + def losses(self, anchors, pred_logits, gt_labels, pred_anchor_deltas, gt_boxes): + """ + Args: + anchors (list[Boxes]): a list of #feature level Boxes + gt_labels, gt_boxes: see output of :meth:`RetinaNet.label_anchors`. + Their shapes are (N, R) and (N, R, 4), respectively, where R is + the total number of anchors across levels, i.e. sum(Hi x Wi x Ai) + pred_logits, pred_anchor_deltas: both are list[Tensor]. Each element in the + list corresponds to one level and has shape (N, Hi * Wi * Ai, K or 4). + Where K is the number of classes used in `pred_logits`. + + Returns: + dict[str, Tensor]: + mapping from a named loss to a scalar tensor storing the loss. + Used during training only. The dict keys are: "loss_cls" and "loss_box_reg" + """ + num_images = len(gt_labels) + gt_labels = torch.stack(gt_labels) # (N, R) + + valid_mask = gt_labels >= 0 + pos_mask = (gt_labels >= 0) & (gt_labels != self.num_classes) + num_pos_anchors = pos_mask.sum().item() + get_event_storage().put_scalar("num_pos_anchors", num_pos_anchors / num_images) + normalizer = self._ema_update("loss_normalizer", max(num_pos_anchors, 1), 100) + + # classification and regression loss + gt_labels_target = F.one_hot(gt_labels[valid_mask], num_classes=self.num_classes + 1)[ + :, :-1 + ] # no loss for the last (background) class + loss_cls = sigmoid_focal_loss_jit( + cat(pred_logits, dim=1)[valid_mask], + gt_labels_target.to(pred_logits[0].dtype), + alpha=self.focal_loss_alpha, + gamma=self.focal_loss_gamma, + reduction="sum", + ) + + loss_box_reg = _dense_box_regression_loss( + anchors, + self.box2box_transform, + pred_anchor_deltas, + gt_boxes, + pos_mask, + box_reg_loss_type=self.box_reg_loss_type, + smooth_l1_beta=self.smooth_l1_beta, + ) + + return { + "loss_cls": loss_cls / normalizer, + "loss_box_reg": loss_box_reg / normalizer, + } + + @torch.no_grad() + def label_anchors(self, anchors, gt_instances): + """ + Args: + anchors (list[Boxes]): A list of #feature level Boxes. + The Boxes contains anchors of this image on the specific feature level. + gt_instances (list[Instances]): a list of N `Instances`s. The i-th + `Instances` contains the ground-truth per-instance annotations + for the i-th input image. + + Returns: + list[Tensor]: List of #img tensors. i-th element is a vector of labels whose length is + the total number of anchors across all feature maps (sum(Hi * Wi * A)). + Label values are in {-1, 0, ..., K}, with -1 means ignore, and K means background. + + list[Tensor]: i-th element is a Rx4 tensor, where R is the total number of anchors + across feature maps. The values are the matched gt boxes for each anchor. + Values are undefined for those anchors not labeled as foreground. + """ + anchors = Boxes.cat(anchors) # Rx4 + + gt_labels = [] + matched_gt_boxes = [] + for gt_per_image in gt_instances: + match_quality_matrix = pairwise_iou(gt_per_image.gt_boxes, anchors) + matched_idxs, anchor_labels = self.anchor_matcher(match_quality_matrix) + del match_quality_matrix + + if len(gt_per_image) > 0: + matched_gt_boxes_i = gt_per_image.gt_boxes.tensor[matched_idxs] + + gt_labels_i = gt_per_image.gt_classes[matched_idxs] + # Anchors with label 0 are treated as background. + gt_labels_i[anchor_labels == 0] = self.num_classes + # Anchors with label -1 are ignored. + gt_labels_i[anchor_labels == -1] = -1 + else: + matched_gt_boxes_i = torch.zeros_like(anchors.tensor) + gt_labels_i = torch.zeros_like(matched_idxs) + self.num_classes + + gt_labels.append(gt_labels_i) + matched_gt_boxes.append(matched_gt_boxes_i) + + return gt_labels, matched_gt_boxes + + def forward_inference( + self, images: ImageList, features: List[Tensor], predictions: List[List[Tensor]] + ): + pred_logits, pred_anchor_deltas = self._transpose_dense_predictions( + predictions, [self.num_classes, 4] + ) + anchors = self.anchor_generator(features) + + results: List[Instances] = [] + for img_idx, image_size in enumerate(images.image_sizes): + scores_per_image = [x[img_idx].sigmoid_() for x in pred_logits] + deltas_per_image = [x[img_idx] for x in pred_anchor_deltas] + results_per_image = self.inference_single_image( + anchors, scores_per_image, deltas_per_image, image_size + ) + results.append(results_per_image) + return results + + def inference_single_image( + self, + anchors: List[Boxes], + box_cls: List[Tensor], + box_delta: List[Tensor], + image_size: Tuple[int, int], + ): + """ + Single-image inference. Return bounding-box detection results by thresholding + on scores and applying non-maximum suppression (NMS). + + Arguments: + anchors (list[Boxes]): list of #feature levels. Each entry contains + a Boxes object, which contains all the anchors in that feature level. + box_cls (list[Tensor]): list of #feature levels. Each entry contains + tensor of size (H x W x A, K) + box_delta (list[Tensor]): Same shape as 'box_cls' except that K becomes 4. + image_size (tuple(H, W)): a tuple of the image height and width. + + Returns: + Same as `inference`, but for only one image. + """ + pred = self._decode_multi_level_predictions( + anchors, + box_cls, + box_delta, + self.test_score_thresh, + self.test_topk_candidates, + image_size, + ) + keep = batched_nms( # per-class NMS + pred.pred_boxes.tensor, pred.scores, pred.pred_classes, self.test_nms_thresh + ) + return pred[keep[: self.max_detections_per_image]] + + +class RetinaNetHead(nn.Module): + """ + The head used in RetinaNet for object classification and box regression. + It has two subnets for the two tasks, with a common structure but separate parameters. + """ + + @configurable + def __init__( + self, + *, + input_shape: List[ShapeSpec], + num_classes, + num_anchors, + conv_dims: List[int], + norm="", + prior_prob=0.01, + ): + """ + NOTE: this interface is experimental. + + Args: + input_shape (List[ShapeSpec]): input shape + num_classes (int): number of classes. Used to label background proposals. + num_anchors (int): number of generated anchors + conv_dims (List[int]): dimensions for each convolution layer + norm (str or callable): + Normalization for conv layers except for the two output layers. + See :func:`detectron2.layers.get_norm` for supported types. + prior_prob (float): Prior weight for computing bias + """ + super().__init__() + + self._num_features = len(input_shape) + if norm == "BN" or norm == "SyncBN": + logger.info( + f"Using domain-specific {norm} in RetinaNetHead with len={self._num_features}." + ) + bn_class = nn.BatchNorm2d if norm == "BN" else nn.SyncBatchNorm + + def norm(c): + return CycleBatchNormList( + length=self._num_features, bn_class=bn_class, num_features=c + ) + + else: + norm_name = str(type(get_norm(norm, 32))) + if "BN" in norm_name: + logger.warning( + f"Shared BatchNorm (type={norm_name}) may not work well in RetinaNetHead." + ) + + cls_subnet = [] + bbox_subnet = [] + for in_channels, out_channels in zip( + [input_shape[0].channels] + list(conv_dims), conv_dims + ): + cls_subnet.append( + nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) + ) + if norm: + cls_subnet.append(get_norm(norm, out_channels)) + cls_subnet.append(nn.ReLU()) + bbox_subnet.append( + nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=1, padding=1) + ) + if norm: + bbox_subnet.append(get_norm(norm, out_channels)) + bbox_subnet.append(nn.ReLU()) + + self.cls_subnet = nn.Sequential(*cls_subnet) + self.bbox_subnet = nn.Sequential(*bbox_subnet) + self.cls_score = nn.Conv2d( + conv_dims[-1], num_anchors * num_classes, kernel_size=3, stride=1, padding=1 + ) + self.bbox_pred = nn.Conv2d( + conv_dims[-1], num_anchors * 4, kernel_size=3, stride=1, padding=1 + ) + + # Initialization + for modules in [self.cls_subnet, self.bbox_subnet, self.cls_score, self.bbox_pred]: + for layer in modules.modules(): + if isinstance(layer, nn.Conv2d): + torch.nn.init.normal_(layer.weight, mean=0, std=0.01) + torch.nn.init.constant_(layer.bias, 0) + + # Use prior in model initialization to improve stability + bias_value = -(math.log((1 - prior_prob) / prior_prob)) + torch.nn.init.constant_(self.cls_score.bias, bias_value) + + @classmethod + def from_config(cls, cfg, input_shape: List[ShapeSpec]): + num_anchors = build_anchor_generator(cfg, input_shape).num_cell_anchors + assert ( + len(set(num_anchors)) == 1 + ), "Using different number of anchors between levels is not currently supported!" + num_anchors = num_anchors[0] + + return { + "input_shape": input_shape, + "num_classes": cfg.MODEL.RETINANET.NUM_CLASSES, + "conv_dims": [input_shape[0].channels] * cfg.MODEL.RETINANET.NUM_CONVS, + "prior_prob": cfg.MODEL.RETINANET.PRIOR_PROB, + "norm": cfg.MODEL.RETINANET.NORM, + "num_anchors": num_anchors, + } + + def forward(self, features: List[Tensor]): + """ + Arguments: + features (list[Tensor]): FPN feature map tensors in high to low resolution. + Each tensor in the list correspond to different feature levels. + + Returns: + logits (list[Tensor]): #lvl tensors, each has shape (N, AxK, Hi, Wi). + The tensor predicts the classification probability + at each spatial position for each of the A anchors and K object + classes. + bbox_reg (list[Tensor]): #lvl tensors, each has shape (N, Ax4, Hi, Wi). + The tensor predicts 4-vector (dx,dy,dw,dh) box + regression values for every anchor. These values are the + relative offset between the anchor and the ground truth box. + """ + assert len(features) == self._num_features + logits = [] + bbox_reg = [] + for feature in features: + logits.append(self.cls_score(self.cls_subnet(feature))) + bbox_reg.append(self.bbox_pred(self.bbox_subnet(feature))) + return logits, bbox_reg diff --git a/annotator/oneformer/detectron2/modeling/meta_arch/semantic_seg.py b/annotator/oneformer/detectron2/modeling/meta_arch/semantic_seg.py new file mode 100644 index 0000000000000000000000000000000000000000..b4be86864c11c2b73a56a879746fce18a88260af --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/meta_arch/semantic_seg.py @@ -0,0 +1,267 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import numpy as np +from typing import Callable, Dict, Optional, Tuple, Union +import fvcore.nn.weight_init as weight_init +import torch +from torch import nn +from torch.nn import functional as F + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.layers import Conv2d, ShapeSpec, get_norm +from annotator.oneformer.detectron2.structures import ImageList +from annotator.oneformer.detectron2.utils.registry import Registry + +from ..backbone import Backbone, build_backbone +from ..postprocessing import sem_seg_postprocess +from .build import META_ARCH_REGISTRY + +__all__ = [ + "SemanticSegmentor", + "SEM_SEG_HEADS_REGISTRY", + "SemSegFPNHead", + "build_sem_seg_head", +] + + +SEM_SEG_HEADS_REGISTRY = Registry("SEM_SEG_HEADS") +SEM_SEG_HEADS_REGISTRY.__doc__ = """ +Registry for semantic segmentation heads, which make semantic segmentation predictions +from feature maps. +""" + + +@META_ARCH_REGISTRY.register() +class SemanticSegmentor(nn.Module): + """ + Main class for semantic segmentation architectures. + """ + + @configurable + def __init__( + self, + *, + backbone: Backbone, + sem_seg_head: nn.Module, + pixel_mean: Tuple[float], + pixel_std: Tuple[float], + ): + """ + Args: + backbone: a backbone module, must follow detectron2's backbone interface + sem_seg_head: a module that predicts semantic segmentation from backbone features + pixel_mean, pixel_std: list or tuple with #channels element, representing + the per-channel mean and std to be used to normalize the input image + """ + super().__init__() + self.backbone = backbone + self.sem_seg_head = sem_seg_head + self.register_buffer("pixel_mean", torch.tensor(pixel_mean).view(-1, 1, 1), False) + self.register_buffer("pixel_std", torch.tensor(pixel_std).view(-1, 1, 1), False) + + @classmethod + def from_config(cls, cfg): + backbone = build_backbone(cfg) + sem_seg_head = build_sem_seg_head(cfg, backbone.output_shape()) + return { + "backbone": backbone, + "sem_seg_head": sem_seg_head, + "pixel_mean": cfg.MODEL.PIXEL_MEAN, + "pixel_std": cfg.MODEL.PIXEL_STD, + } + + @property + def device(self): + return self.pixel_mean.device + + def forward(self, batched_inputs): + """ + Args: + batched_inputs: a list, batched outputs of :class:`DatasetMapper`. + Each item in the list contains the inputs for one image. + + For now, each item in the list is a dict that contains: + + * "image": Tensor, image in (C, H, W) format. + * "sem_seg": semantic segmentation ground truth + * Other information that's included in the original dicts, such as: + "height", "width" (int): the output resolution of the model (may be different + from input resolution), used in inference. + + + Returns: + list[dict]: + Each dict is the output for one input image. + The dict contains one key "sem_seg" whose value is a + Tensor that represents the + per-pixel segmentation prediced by the head. + The prediction has shape KxHxW that represents the logits of + each class for each pixel. + """ + images = [x["image"].to(self.device) for x in batched_inputs] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors( + images, + self.backbone.size_divisibility, + padding_constraints=self.backbone.padding_constraints, + ) + + features = self.backbone(images.tensor) + + if "sem_seg" in batched_inputs[0]: + targets = [x["sem_seg"].to(self.device) for x in batched_inputs] + targets = ImageList.from_tensors( + targets, + self.backbone.size_divisibility, + self.sem_seg_head.ignore_value, + self.backbone.padding_constraints, + ).tensor + else: + targets = None + results, losses = self.sem_seg_head(features, targets) + + if self.training: + return losses + + processed_results = [] + for result, input_per_image, image_size in zip(results, batched_inputs, images.image_sizes): + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + r = sem_seg_postprocess(result, image_size, height, width) + processed_results.append({"sem_seg": r}) + return processed_results + + +def build_sem_seg_head(cfg, input_shape): + """ + Build a semantic segmentation head from `cfg.MODEL.SEM_SEG_HEAD.NAME`. + """ + name = cfg.MODEL.SEM_SEG_HEAD.NAME + return SEM_SEG_HEADS_REGISTRY.get(name)(cfg, input_shape) + + +@SEM_SEG_HEADS_REGISTRY.register() +class SemSegFPNHead(nn.Module): + """ + A semantic segmentation head described in :paper:`PanopticFPN`. + It takes a list of FPN features as input, and applies a sequence of + 3x3 convs and upsampling to scale all of them to the stride defined by + ``common_stride``. Then these features are added and used to make final + predictions by another 1x1 conv layer. + """ + + @configurable + def __init__( + self, + input_shape: Dict[str, ShapeSpec], + *, + num_classes: int, + conv_dims: int, + common_stride: int, + loss_weight: float = 1.0, + norm: Optional[Union[str, Callable]] = None, + ignore_value: int = -1, + ): + """ + NOTE: this interface is experimental. + + Args: + input_shape: shapes (channels and stride) of the input features + num_classes: number of classes to predict + conv_dims: number of output channels for the intermediate conv layers. + common_stride: the common stride that all features will be upscaled to + loss_weight: loss weight + norm (str or callable): normalization for all conv layers + ignore_value: category id to be ignored during training. + """ + super().__init__() + input_shape = sorted(input_shape.items(), key=lambda x: x[1].stride) + if not len(input_shape): + raise ValueError("SemSegFPNHead(input_shape=) cannot be empty!") + self.in_features = [k for k, v in input_shape] + feature_strides = [v.stride for k, v in input_shape] + feature_channels = [v.channels for k, v in input_shape] + + self.ignore_value = ignore_value + self.common_stride = common_stride + self.loss_weight = loss_weight + + self.scale_heads = [] + for in_feature, stride, channels in zip( + self.in_features, feature_strides, feature_channels + ): + head_ops = [] + head_length = max(1, int(np.log2(stride) - np.log2(self.common_stride))) + for k in range(head_length): + norm_module = get_norm(norm, conv_dims) + conv = Conv2d( + channels if k == 0 else conv_dims, + conv_dims, + kernel_size=3, + stride=1, + padding=1, + bias=not norm, + norm=norm_module, + activation=F.relu, + ) + weight_init.c2_msra_fill(conv) + head_ops.append(conv) + if stride != self.common_stride: + head_ops.append( + nn.Upsample(scale_factor=2, mode="bilinear", align_corners=False) + ) + self.scale_heads.append(nn.Sequential(*head_ops)) + self.add_module(in_feature, self.scale_heads[-1]) + self.predictor = Conv2d(conv_dims, num_classes, kernel_size=1, stride=1, padding=0) + weight_init.c2_msra_fill(self.predictor) + + @classmethod + def from_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): + return { + "input_shape": { + k: v for k, v in input_shape.items() if k in cfg.MODEL.SEM_SEG_HEAD.IN_FEATURES + }, + "ignore_value": cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE, + "num_classes": cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES, + "conv_dims": cfg.MODEL.SEM_SEG_HEAD.CONVS_DIM, + "common_stride": cfg.MODEL.SEM_SEG_HEAD.COMMON_STRIDE, + "norm": cfg.MODEL.SEM_SEG_HEAD.NORM, + "loss_weight": cfg.MODEL.SEM_SEG_HEAD.LOSS_WEIGHT, + } + + def forward(self, features, targets=None): + """ + Returns: + In training, returns (None, dict of losses) + In inference, returns (CxHxW logits, {}) + """ + x = self.layers(features) + if self.training: + return None, self.losses(x, targets) + else: + x = F.interpolate( + x, scale_factor=self.common_stride, mode="bilinear", align_corners=False + ) + return x, {} + + def layers(self, features): + for i, f in enumerate(self.in_features): + if i == 0: + x = self.scale_heads[i](features[f]) + else: + x = x + self.scale_heads[i](features[f]) + x = self.predictor(x) + return x + + def losses(self, predictions, targets): + predictions = predictions.float() # https://github.com/pytorch/pytorch/issues/48163 + predictions = F.interpolate( + predictions, + scale_factor=self.common_stride, + mode="bilinear", + align_corners=False, + ) + loss = F.cross_entropy( + predictions, targets, reduction="mean", ignore_index=self.ignore_value + ) + losses = {"loss_sem_seg": loss * self.loss_weight} + return losses diff --git a/annotator/oneformer/detectron2/modeling/mmdet_wrapper.py b/annotator/oneformer/detectron2/modeling/mmdet_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..5a60958cdc07e0170e4dfe02684bce259d42bdbc --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/mmdet_wrapper.py @@ -0,0 +1,273 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import itertools +import logging +import numpy as np +from collections import OrderedDict +from collections.abc import Mapping +from typing import Dict, List, Optional, Tuple, Union +import torch +from omegaconf import DictConfig, OmegaConf +from torch import Tensor, nn + +from annotator.oneformer.detectron2.layers import ShapeSpec +from annotator.oneformer.detectron2.structures import BitMasks, Boxes, ImageList, Instances +from annotator.oneformer.detectron2.utils.events import get_event_storage + +from .backbone import Backbone + +logger = logging.getLogger(__name__) + + +def _to_container(cfg): + """ + mmdet will assert the type of dict/list. + So convert omegaconf objects to dict/list. + """ + if isinstance(cfg, DictConfig): + cfg = OmegaConf.to_container(cfg, resolve=True) + from mmcv.utils import ConfigDict + + return ConfigDict(cfg) + + +class MMDetBackbone(Backbone): + """ + Wrapper of mmdetection backbones to use in detectron2. + + mmdet backbones produce list/tuple of tensors, while detectron2 backbones + produce a dict of tensors. This class wraps the given backbone to produce + output in detectron2's convention, so it can be used in place of detectron2 + backbones. + """ + + def __init__( + self, + backbone: Union[nn.Module, Mapping], + neck: Union[nn.Module, Mapping, None] = None, + *, + output_shapes: List[ShapeSpec], + output_names: Optional[List[str]] = None, + ): + """ + Args: + backbone: either a backbone module or a mmdet config dict that defines a + backbone. The backbone takes a 4D image tensor and returns a + sequence of tensors. + neck: either a backbone module or a mmdet config dict that defines a + neck. The neck takes outputs of backbone and returns a + sequence of tensors. If None, no neck is used. + output_shapes: shape for every output of the backbone (or neck, if given). + stride and channels are often needed. + output_names: names for every output of the backbone (or neck, if given). + By default, will use "out0", "out1", ... + """ + super().__init__() + if isinstance(backbone, Mapping): + from mmdet.models import build_backbone + + backbone = build_backbone(_to_container(backbone)) + self.backbone = backbone + + if isinstance(neck, Mapping): + from mmdet.models import build_neck + + neck = build_neck(_to_container(neck)) + self.neck = neck + + # "Neck" weights, if any, are part of neck itself. This is the interface + # of mmdet so we follow it. Reference: + # https://github.com/open-mmlab/mmdetection/blob/master/mmdet/models/detectors/two_stage.py + logger.info("Initializing mmdet backbone weights...") + self.backbone.init_weights() + # train() in mmdet modules is non-trivial, and has to be explicitly + # called. Reference: + # https://github.com/open-mmlab/mmdetection/blob/master/mmdet/models/backbones/resnet.py + self.backbone.train() + if self.neck is not None: + logger.info("Initializing mmdet neck weights ...") + if isinstance(self.neck, nn.Sequential): + for m in self.neck: + m.init_weights() + else: + self.neck.init_weights() + self.neck.train() + + self._output_shapes = output_shapes + if not output_names: + output_names = [f"out{i}" for i in range(len(output_shapes))] + self._output_names = output_names + + def forward(self, x) -> Dict[str, Tensor]: + outs = self.backbone(x) + if self.neck is not None: + outs = self.neck(outs) + assert isinstance( + outs, (list, tuple) + ), "mmdet backbone should return a list/tuple of tensors!" + if len(outs) != len(self._output_shapes): + raise ValueError( + "Length of output_shapes does not match outputs from the mmdet backbone: " + f"{len(outs)} != {len(self._output_shapes)}" + ) + return {k: v for k, v in zip(self._output_names, outs)} + + def output_shape(self) -> Dict[str, ShapeSpec]: + return {k: v for k, v in zip(self._output_names, self._output_shapes)} + + +class MMDetDetector(nn.Module): + """ + Wrapper of a mmdetection detector model, for detection and instance segmentation. + Input/output formats of this class follow detectron2's convention, so a + mmdetection model can be trained and evaluated in detectron2. + """ + + def __init__( + self, + detector: Union[nn.Module, Mapping], + *, + # Default is 32 regardless of model: + # https://github.com/open-mmlab/mmdetection/tree/master/configs/_base_/datasets + size_divisibility=32, + pixel_mean: Tuple[float], + pixel_std: Tuple[float], + ): + """ + Args: + detector: a mmdet detector, or a mmdet config dict that defines a detector. + size_divisibility: pad input images to multiple of this number + pixel_mean: per-channel mean to normalize input image + pixel_std: per-channel stddev to normalize input image + """ + super().__init__() + if isinstance(detector, Mapping): + from mmdet.models import build_detector + + detector = build_detector(_to_container(detector)) + self.detector = detector + self.detector.init_weights() + self.size_divisibility = size_divisibility + + self.register_buffer("pixel_mean", torch.tensor(pixel_mean).view(-1, 1, 1), False) + self.register_buffer("pixel_std", torch.tensor(pixel_std).view(-1, 1, 1), False) + assert ( + self.pixel_mean.shape == self.pixel_std.shape + ), f"{self.pixel_mean} and {self.pixel_std} have different shapes!" + + def forward(self, batched_inputs: List[Dict[str, torch.Tensor]]): + images = [x["image"].to(self.device) for x in batched_inputs] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors(images, size_divisibility=self.size_divisibility).tensor + metas = [] + rescale = {"height" in x for x in batched_inputs} + if len(rescale) != 1: + raise ValueError("Some inputs have original height/width, but some don't!") + rescale = list(rescale)[0] + output_shapes = [] + for input in batched_inputs: + meta = {} + c, h, w = input["image"].shape + meta["img_shape"] = meta["ori_shape"] = (h, w, c) + if rescale: + scale_factor = np.array( + [w / input["width"], h / input["height"]] * 2, dtype="float32" + ) + ori_shape = (input["height"], input["width"]) + output_shapes.append(ori_shape) + meta["ori_shape"] = ori_shape + (c,) + else: + scale_factor = 1.0 + output_shapes.append((h, w)) + meta["scale_factor"] = scale_factor + meta["flip"] = False + padh, padw = images.shape[-2:] + meta["pad_shape"] = (padh, padw, c) + metas.append(meta) + + if self.training: + gt_instances = [x["instances"].to(self.device) for x in batched_inputs] + if gt_instances[0].has("gt_masks"): + from mmdet.core import PolygonMasks as mm_PolygonMasks, BitmapMasks as mm_BitMasks + + def convert_mask(m, shape): + # mmdet mask format + if isinstance(m, BitMasks): + return mm_BitMasks(m.tensor.cpu().numpy(), shape[0], shape[1]) + else: + return mm_PolygonMasks(m.polygons, shape[0], shape[1]) + + gt_masks = [convert_mask(x.gt_masks, x.image_size) for x in gt_instances] + losses_and_metrics = self.detector.forward_train( + images, + metas, + [x.gt_boxes.tensor for x in gt_instances], + [x.gt_classes for x in gt_instances], + gt_masks=gt_masks, + ) + else: + losses_and_metrics = self.detector.forward_train( + images, + metas, + [x.gt_boxes.tensor for x in gt_instances], + [x.gt_classes for x in gt_instances], + ) + return _parse_losses(losses_and_metrics) + else: + results = self.detector.simple_test(images, metas, rescale=rescale) + results = [ + {"instances": _convert_mmdet_result(r, shape)} + for r, shape in zip(results, output_shapes) + ] + return results + + @property + def device(self): + return self.pixel_mean.device + + +# Reference: show_result() in +# https://github.com/open-mmlab/mmdetection/blob/master/mmdet/models/detectors/base.py +def _convert_mmdet_result(result, shape: Tuple[int, int]) -> Instances: + if isinstance(result, tuple): + bbox_result, segm_result = result + if isinstance(segm_result, tuple): + segm_result = segm_result[0] + else: + bbox_result, segm_result = result, None + + bboxes = torch.from_numpy(np.vstack(bbox_result)) # Nx5 + bboxes, scores = bboxes[:, :4], bboxes[:, -1] + labels = [ + torch.full((bbox.shape[0],), i, dtype=torch.int32) for i, bbox in enumerate(bbox_result) + ] + labels = torch.cat(labels) + inst = Instances(shape) + inst.pred_boxes = Boxes(bboxes) + inst.scores = scores + inst.pred_classes = labels + + if segm_result is not None and len(labels) > 0: + segm_result = list(itertools.chain(*segm_result)) + segm_result = [torch.from_numpy(x) if isinstance(x, np.ndarray) else x for x in segm_result] + segm_result = torch.stack(segm_result, dim=0) + inst.pred_masks = segm_result + return inst + + +# reference: https://github.com/open-mmlab/mmdetection/blob/master/mmdet/models/detectors/base.py +def _parse_losses(losses: Dict[str, Tensor]) -> Dict[str, Tensor]: + log_vars = OrderedDict() + for loss_name, loss_value in losses.items(): + if isinstance(loss_value, torch.Tensor): + log_vars[loss_name] = loss_value.mean() + elif isinstance(loss_value, list): + log_vars[loss_name] = sum(_loss.mean() for _loss in loss_value) + else: + raise TypeError(f"{loss_name} is not a tensor or list of tensors") + + if "loss" not in loss_name: + # put metrics to storage; don't return them + storage = get_event_storage() + value = log_vars.pop(loss_name).cpu().item() + storage.put_scalar(loss_name, value) + return log_vars diff --git a/annotator/oneformer/detectron2/modeling/poolers.py b/annotator/oneformer/detectron2/modeling/poolers.py new file mode 100644 index 0000000000000000000000000000000000000000..109ab47eb975b2302966eeb698ac6b4aff5e0a4d --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/poolers.py @@ -0,0 +1,263 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import math +from typing import List, Optional +import torch +from torch import nn +from torchvision.ops import RoIPool + +from annotator.oneformer.detectron2.layers import ROIAlign, ROIAlignRotated, cat, nonzero_tuple, shapes_to_tensor +from annotator.oneformer.detectron2.structures import Boxes +from annotator.oneformer.detectron2.utils.tracing import assert_fx_safe, is_fx_tracing + +""" +To export ROIPooler to torchscript, in this file, variables that should be annotated with +`Union[List[Boxes], List[RotatedBoxes]]` are only annotated with `List[Boxes]`. + +TODO: Correct these annotations when torchscript support `Union`. +https://github.com/pytorch/pytorch/issues/41412 +""" + +__all__ = ["ROIPooler"] + + +def assign_boxes_to_levels( + box_lists: List[Boxes], + min_level: int, + max_level: int, + canonical_box_size: int, + canonical_level: int, +): + """ + Map each box in `box_lists` to a feature map level index and return the assignment + vector. + + Args: + box_lists (list[Boxes] | list[RotatedBoxes]): A list of N Boxes or N RotatedBoxes, + where N is the number of images in the batch. + min_level (int): Smallest feature map level index. The input is considered index 0, + the output of stage 1 is index 1, and so. + max_level (int): Largest feature map level index. + canonical_box_size (int): A canonical box size in pixels (sqrt(box area)). + canonical_level (int): The feature map level index on which a canonically-sized box + should be placed. + + Returns: + A tensor of length M, where M is the total number of boxes aggregated over all + N batch images. The memory layout corresponds to the concatenation of boxes + from all images. Each element is the feature map index, as an offset from + `self.min_level`, for the corresponding box (so value i means the box is at + `self.min_level + i`). + """ + box_sizes = torch.sqrt(cat([boxes.area() for boxes in box_lists])) + # Eqn.(1) in FPN paper + level_assignments = torch.floor( + canonical_level + torch.log2(box_sizes / canonical_box_size + 1e-8) + ) + # clamp level to (min, max), in case the box size is too large or too small + # for the available feature maps + level_assignments = torch.clamp(level_assignments, min=min_level, max=max_level) + return level_assignments.to(torch.int64) - min_level + + +# script the module to avoid hardcoded device type +@torch.jit.script_if_tracing +def _convert_boxes_to_pooler_format(boxes: torch.Tensor, sizes: torch.Tensor) -> torch.Tensor: + sizes = sizes.to(device=boxes.device) + indices = torch.repeat_interleave( + torch.arange(len(sizes), dtype=boxes.dtype, device=boxes.device), sizes + ) + return cat([indices[:, None], boxes], dim=1) + + +def convert_boxes_to_pooler_format(box_lists: List[Boxes]): + """ + Convert all boxes in `box_lists` to the low-level format used by ROI pooling ops + (see description under Returns). + + Args: + box_lists (list[Boxes] | list[RotatedBoxes]): + A list of N Boxes or N RotatedBoxes, where N is the number of images in the batch. + + Returns: + When input is list[Boxes]: + A tensor of shape (M, 5), where M is the total number of boxes aggregated over all + N batch images. + The 5 columns are (batch index, x0, y0, x1, y1), where batch index + is the index in [0, N) identifying which batch image the box with corners at + (x0, y0, x1, y1) comes from. + When input is list[RotatedBoxes]: + A tensor of shape (M, 6), where M is the total number of boxes aggregated over all + N batch images. + The 6 columns are (batch index, x_ctr, y_ctr, width, height, angle_degrees), + where batch index is the index in [0, N) identifying which batch image the + rotated box (x_ctr, y_ctr, width, height, angle_degrees) comes from. + """ + boxes = torch.cat([x.tensor for x in box_lists], dim=0) + # __len__ returns Tensor in tracing. + sizes = shapes_to_tensor([x.__len__() for x in box_lists]) + return _convert_boxes_to_pooler_format(boxes, sizes) + + +@torch.jit.script_if_tracing +def _create_zeros( + batch_target: Optional[torch.Tensor], + channels: int, + height: int, + width: int, + like_tensor: torch.Tensor, +) -> torch.Tensor: + batches = batch_target.shape[0] if batch_target is not None else 0 + sizes = (batches, channels, height, width) + return torch.zeros(sizes, dtype=like_tensor.dtype, device=like_tensor.device) + + +class ROIPooler(nn.Module): + """ + Region of interest feature map pooler that supports pooling from one or more + feature maps. + """ + + def __init__( + self, + output_size, + scales, + sampling_ratio, + pooler_type, + canonical_box_size=224, + canonical_level=4, + ): + """ + Args: + output_size (int, tuple[int] or list[int]): output size of the pooled region, + e.g., 14 x 14. If tuple or list is given, the length must be 2. + scales (list[float]): The scale for each low-level pooling op relative to + the input image. For a feature map with stride s relative to the input + image, scale is defined as 1/s. The stride must be power of 2. + When there are multiple scales, they must form a pyramid, i.e. they must be + a monotically decreasing geometric sequence with a factor of 1/2. + sampling_ratio (int): The `sampling_ratio` parameter for the ROIAlign op. + pooler_type (string): Name of the type of pooling operation that should be applied. + For instance, "ROIPool" or "ROIAlignV2". + canonical_box_size (int): A canonical box size in pixels (sqrt(box area)). The default + is heuristically defined as 224 pixels in the FPN paper (based on ImageNet + pre-training). + canonical_level (int): The feature map level index from which a canonically-sized box + should be placed. The default is defined as level 4 (stride=16) in the FPN paper, + i.e., a box of size 224x224 will be placed on the feature with stride=16. + The box placement for all boxes will be determined from their sizes w.r.t + canonical_box_size. For example, a box whose area is 4x that of a canonical box + should be used to pool features from feature level ``canonical_level+1``. + + Note that the actual input feature maps given to this module may not have + sufficiently many levels for the input boxes. If the boxes are too large or too + small for the input feature maps, the closest level will be used. + """ + super().__init__() + + if isinstance(output_size, int): + output_size = (output_size, output_size) + assert len(output_size) == 2 + assert isinstance(output_size[0], int) and isinstance(output_size[1], int) + self.output_size = output_size + + if pooler_type == "ROIAlign": + self.level_poolers = nn.ModuleList( + ROIAlign( + output_size, spatial_scale=scale, sampling_ratio=sampling_ratio, aligned=False + ) + for scale in scales + ) + elif pooler_type == "ROIAlignV2": + self.level_poolers = nn.ModuleList( + ROIAlign( + output_size, spatial_scale=scale, sampling_ratio=sampling_ratio, aligned=True + ) + for scale in scales + ) + elif pooler_type == "ROIPool": + self.level_poolers = nn.ModuleList( + RoIPool(output_size, spatial_scale=scale) for scale in scales + ) + elif pooler_type == "ROIAlignRotated": + self.level_poolers = nn.ModuleList( + ROIAlignRotated(output_size, spatial_scale=scale, sampling_ratio=sampling_ratio) + for scale in scales + ) + else: + raise ValueError("Unknown pooler type: {}".format(pooler_type)) + + # Map scale (defined as 1 / stride) to its feature map level under the + # assumption that stride is a power of 2. + min_level = -(math.log2(scales[0])) + max_level = -(math.log2(scales[-1])) + assert math.isclose(min_level, int(min_level)) and math.isclose( + max_level, int(max_level) + ), "Featuremap stride is not power of 2!" + self.min_level = int(min_level) + self.max_level = int(max_level) + assert ( + len(scales) == self.max_level - self.min_level + 1 + ), "[ROIPooler] Sizes of input featuremaps do not form a pyramid!" + assert 0 <= self.min_level and self.min_level <= self.max_level + self.canonical_level = canonical_level + assert canonical_box_size > 0 + self.canonical_box_size = canonical_box_size + + def forward(self, x: List[torch.Tensor], box_lists: List[Boxes]): + """ + Args: + x (list[Tensor]): A list of feature maps of NCHW shape, with scales matching those + used to construct this module. + box_lists (list[Boxes] | list[RotatedBoxes]): + A list of N Boxes or N RotatedBoxes, where N is the number of images in the batch. + The box coordinates are defined on the original image and + will be scaled by the `scales` argument of :class:`ROIPooler`. + + Returns: + Tensor: + A tensor of shape (M, C, output_size, output_size) where M is the total number of + boxes aggregated over all N batch images and C is the number of channels in `x`. + """ + num_level_assignments = len(self.level_poolers) + + if not is_fx_tracing(): + torch._assert( + isinstance(x, list) and isinstance(box_lists, list), + "Arguments to pooler must be lists", + ) + assert_fx_safe( + len(x) == num_level_assignments, + "unequal value, num_level_assignments={}, but x is list of {} Tensors".format( + num_level_assignments, len(x) + ), + ) + assert_fx_safe( + len(box_lists) == x[0].size(0), + "unequal value, x[0] batch dim 0 is {}, but box_list has length {}".format( + x[0].size(0), len(box_lists) + ), + ) + if len(box_lists) == 0: + return _create_zeros(None, x[0].shape[1], *self.output_size, x[0]) + + pooler_fmt_boxes = convert_boxes_to_pooler_format(box_lists) + + if num_level_assignments == 1: + return self.level_poolers[0](x[0], pooler_fmt_boxes) + + level_assignments = assign_boxes_to_levels( + box_lists, self.min_level, self.max_level, self.canonical_box_size, self.canonical_level + ) + + num_channels = x[0].shape[1] + output_size = self.output_size[0] + + output = _create_zeros(pooler_fmt_boxes, num_channels, output_size, output_size, x[0]) + + for level, pooler in enumerate(self.level_poolers): + inds = nonzero_tuple(level_assignments == level)[0] + pooler_fmt_boxes_level = pooler_fmt_boxes[inds] + # Use index_put_ instead of advance indexing, to avoid pytorch/issues/49852 + output.index_put_((inds,), pooler(x[level], pooler_fmt_boxes_level)) + + return output diff --git a/annotator/oneformer/detectron2/modeling/postprocessing.py b/annotator/oneformer/detectron2/modeling/postprocessing.py new file mode 100644 index 0000000000000000000000000000000000000000..82bbad25cdc5afbde9a3af47174c97ed473cd5f0 --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/postprocessing.py @@ -0,0 +1,100 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import torch +from torch.nn import functional as F + +from annotator.oneformer.detectron2.structures import Instances, ROIMasks + + +# perhaps should rename to "resize_instance" +def detector_postprocess( + results: Instances, output_height: int, output_width: int, mask_threshold: float = 0.5 +): + """ + Resize the output instances. + The input images are often resized when entering an object detector. + As a result, we often need the outputs of the detector in a different + resolution from its inputs. + + This function will resize the raw outputs of an R-CNN detector + to produce outputs according to the desired output resolution. + + Args: + results (Instances): the raw outputs from the detector. + `results.image_size` contains the input image resolution the detector sees. + This object might be modified in-place. + output_height, output_width: the desired output resolution. + Returns: + Instances: the resized output from the model, based on the output resolution + """ + if isinstance(output_width, torch.Tensor): + # This shape might (but not necessarily) be tensors during tracing. + # Converts integer tensors to float temporaries to ensure true + # division is performed when computing scale_x and scale_y. + output_width_tmp = output_width.float() + output_height_tmp = output_height.float() + new_size = torch.stack([output_height, output_width]) + else: + new_size = (output_height, output_width) + output_width_tmp = output_width + output_height_tmp = output_height + + scale_x, scale_y = ( + output_width_tmp / results.image_size[1], + output_height_tmp / results.image_size[0], + ) + results = Instances(new_size, **results.get_fields()) + + if results.has("pred_boxes"): + output_boxes = results.pred_boxes + elif results.has("proposal_boxes"): + output_boxes = results.proposal_boxes + else: + output_boxes = None + assert output_boxes is not None, "Predictions must contain boxes!" + + output_boxes.scale(scale_x, scale_y) + output_boxes.clip(results.image_size) + + results = results[output_boxes.nonempty()] + + if results.has("pred_masks"): + if isinstance(results.pred_masks, ROIMasks): + roi_masks = results.pred_masks + else: + # pred_masks is a tensor of shape (N, 1, M, M) + roi_masks = ROIMasks(results.pred_masks[:, 0, :, :]) + results.pred_masks = roi_masks.to_bitmasks( + results.pred_boxes, output_height, output_width, mask_threshold + ).tensor # TODO return ROIMasks/BitMask object in the future + + if results.has("pred_keypoints"): + results.pred_keypoints[:, :, 0] *= scale_x + results.pred_keypoints[:, :, 1] *= scale_y + + return results + + +def sem_seg_postprocess(result, img_size, output_height, output_width): + """ + Return semantic segmentation predictions in the original resolution. + + The input images are often resized when entering semantic segmentor. Moreover, in same + cases, they also padded inside segmentor to be divisible by maximum network stride. + As a result, we often need the predictions of the segmentor in a different + resolution from its inputs. + + Args: + result (Tensor): semantic segmentation prediction logits. A tensor of shape (C, H, W), + where C is the number of classes, and H, W are the height and width of the prediction. + img_size (tuple): image size that segmentor is taking as input. + output_height, output_width: the desired output resolution. + + Returns: + semantic segmentation prediction (Tensor): A tensor of the shape + (C, output_height, output_width) that contains per-pixel soft predictions. + """ + result = result[:, : img_size[0], : img_size[1]].expand(1, -1, -1, -1) + result = F.interpolate( + result, size=(output_height, output_width), mode="bilinear", align_corners=False + )[0] + return result diff --git a/annotator/oneformer/detectron2/modeling/proposal_generator/__init__.py b/annotator/oneformer/detectron2/modeling/proposal_generator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3f4e4df7645c67b7a013295207b98fe70b2e574c --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/proposal_generator/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .build import PROPOSAL_GENERATOR_REGISTRY, build_proposal_generator +from .rpn import RPN_HEAD_REGISTRY, build_rpn_head, RPN, StandardRPNHead + +__all__ = list(globals().keys()) diff --git a/annotator/oneformer/detectron2/modeling/proposal_generator/build.py b/annotator/oneformer/detectron2/modeling/proposal_generator/build.py new file mode 100644 index 0000000000000000000000000000000000000000..255cd4d0a852f70eeba79e6630f1703ed901963c --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/proposal_generator/build.py @@ -0,0 +1,24 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from annotator.oneformer.detectron2.utils.registry import Registry + +PROPOSAL_GENERATOR_REGISTRY = Registry("PROPOSAL_GENERATOR") +PROPOSAL_GENERATOR_REGISTRY.__doc__ = """ +Registry for proposal generator, which produces object proposals from feature maps. + +The registered object will be called with `obj(cfg, input_shape)`. +The call should return a `nn.Module` object. +""" + +from . import rpn, rrpn # noqa F401 isort:skip + + +def build_proposal_generator(cfg, input_shape): + """ + Build a proposal generator from `cfg.MODEL.PROPOSAL_GENERATOR.NAME`. + The name can be "PrecomputedProposals" to use no proposal generator. + """ + name = cfg.MODEL.PROPOSAL_GENERATOR.NAME + if name == "PrecomputedProposals": + return None + + return PROPOSAL_GENERATOR_REGISTRY.get(name)(cfg, input_shape) diff --git a/annotator/oneformer/detectron2/modeling/proposal_generator/proposal_utils.py b/annotator/oneformer/detectron2/modeling/proposal_generator/proposal_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b5579f43f04e4442f897e20672e4ad5b784c029b --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/proposal_generator/proposal_utils.py @@ -0,0 +1,205 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import math +from typing import List, Tuple, Union +import torch + +from annotator.oneformer.detectron2.layers import batched_nms, cat, move_device_like +from annotator.oneformer.detectron2.structures import Boxes, Instances + +logger = logging.getLogger(__name__) + + +def _is_tracing(): + # (fixed in TORCH_VERSION >= 1.9) + if torch.jit.is_scripting(): + # https://github.com/pytorch/pytorch/issues/47379 + return False + else: + return torch.jit.is_tracing() + + +def find_top_rpn_proposals( + proposals: List[torch.Tensor], + pred_objectness_logits: List[torch.Tensor], + image_sizes: List[Tuple[int, int]], + nms_thresh: float, + pre_nms_topk: int, + post_nms_topk: int, + min_box_size: float, + training: bool, +): + """ + For each feature map, select the `pre_nms_topk` highest scoring proposals, + apply NMS, clip proposals, and remove small boxes. Return the `post_nms_topk` + highest scoring proposals among all the feature maps for each image. + + Args: + proposals (list[Tensor]): A list of L tensors. Tensor i has shape (N, Hi*Wi*A, 4). + All proposal predictions on the feature maps. + pred_objectness_logits (list[Tensor]): A list of L tensors. Tensor i has shape (N, Hi*Wi*A). + image_sizes (list[tuple]): sizes (h, w) for each image + nms_thresh (float): IoU threshold to use for NMS + pre_nms_topk (int): number of top k scoring proposals to keep before applying NMS. + When RPN is run on multiple feature maps (as in FPN) this number is per + feature map. + post_nms_topk (int): number of top k scoring proposals to keep after applying NMS. + When RPN is run on multiple feature maps (as in FPN) this number is total, + over all feature maps. + min_box_size (float): minimum proposal box side length in pixels (absolute units + wrt input images). + training (bool): True if proposals are to be used in training, otherwise False. + This arg exists only to support a legacy bug; look for the "NB: Legacy bug ..." + comment. + + Returns: + list[Instances]: list of N Instances. The i-th Instances + stores post_nms_topk object proposals for image i, sorted by their + objectness score in descending order. + """ + num_images = len(image_sizes) + device = ( + proposals[0].device + if torch.jit.is_scripting() + else ("cpu" if torch.jit.is_tracing() else proposals[0].device) + ) + + # 1. Select top-k anchor for every level and every image + topk_scores = [] # #lvl Tensor, each of shape N x topk + topk_proposals = [] + level_ids = [] # #lvl Tensor, each of shape (topk,) + batch_idx = move_device_like(torch.arange(num_images, device=device), proposals[0]) + for level_id, (proposals_i, logits_i) in enumerate(zip(proposals, pred_objectness_logits)): + Hi_Wi_A = logits_i.shape[1] + if isinstance(Hi_Wi_A, torch.Tensor): # it's a tensor in tracing + num_proposals_i = torch.clamp(Hi_Wi_A, max=pre_nms_topk) + else: + num_proposals_i = min(Hi_Wi_A, pre_nms_topk) + + topk_scores_i, topk_idx = logits_i.topk(num_proposals_i, dim=1) + + # each is N x topk + topk_proposals_i = proposals_i[batch_idx[:, None], topk_idx] # N x topk x 4 + + topk_proposals.append(topk_proposals_i) + topk_scores.append(topk_scores_i) + level_ids.append( + move_device_like( + torch.full((num_proposals_i,), level_id, dtype=torch.int64, device=device), + proposals[0], + ) + ) + + # 2. Concat all levels together + topk_scores = cat(topk_scores, dim=1) + topk_proposals = cat(topk_proposals, dim=1) + level_ids = cat(level_ids, dim=0) + + # 3. For each image, run a per-level NMS, and choose topk results. + results: List[Instances] = [] + for n, image_size in enumerate(image_sizes): + boxes = Boxes(topk_proposals[n]) + scores_per_img = topk_scores[n] + lvl = level_ids + + valid_mask = torch.isfinite(boxes.tensor).all(dim=1) & torch.isfinite(scores_per_img) + if not valid_mask.all(): + if training: + raise FloatingPointError( + "Predicted boxes or scores contain Inf/NaN. Training has diverged." + ) + boxes = boxes[valid_mask] + scores_per_img = scores_per_img[valid_mask] + lvl = lvl[valid_mask] + boxes.clip(image_size) + + # filter empty boxes + keep = boxes.nonempty(threshold=min_box_size) + if _is_tracing() or keep.sum().item() != len(boxes): + boxes, scores_per_img, lvl = boxes[keep], scores_per_img[keep], lvl[keep] + + keep = batched_nms(boxes.tensor, scores_per_img, lvl, nms_thresh) + # In Detectron1, there was different behavior during training vs. testing. + # (https://github.com/facebookresearch/Detectron/issues/459) + # During training, topk is over the proposals from *all* images in the training batch. + # During testing, it is over the proposals for each image separately. + # As a result, the training behavior becomes batch-dependent, + # and the configuration "POST_NMS_TOPK_TRAIN" end up relying on the batch size. + # This bug is addressed in Detectron2 to make the behavior independent of batch size. + keep = keep[:post_nms_topk] # keep is already sorted + + res = Instances(image_size) + res.proposal_boxes = boxes[keep] + res.objectness_logits = scores_per_img[keep] + results.append(res) + return results + + +def add_ground_truth_to_proposals( + gt: Union[List[Instances], List[Boxes]], proposals: List[Instances] +) -> List[Instances]: + """ + Call `add_ground_truth_to_proposals_single_image` for all images. + + Args: + gt(Union[List[Instances], List[Boxes]): list of N elements. Element i is a Instances + representing the ground-truth for image i. + proposals (list[Instances]): list of N elements. Element i is a Instances + representing the proposals for image i. + + Returns: + list[Instances]: list of N Instances. Each is the proposals for the image, + with field "proposal_boxes" and "objectness_logits". + """ + assert gt is not None + + if len(proposals) != len(gt): + raise ValueError("proposals and gt should have the same length as the number of images!") + if len(proposals) == 0: + return proposals + + return [ + add_ground_truth_to_proposals_single_image(gt_i, proposals_i) + for gt_i, proposals_i in zip(gt, proposals) + ] + + +def add_ground_truth_to_proposals_single_image( + gt: Union[Instances, Boxes], proposals: Instances +) -> Instances: + """ + Augment `proposals` with `gt`. + + Args: + Same as `add_ground_truth_to_proposals`, but with gt and proposals + per image. + + Returns: + Same as `add_ground_truth_to_proposals`, but for only one image. + """ + if isinstance(gt, Boxes): + # convert Boxes to Instances + gt = Instances(proposals.image_size, gt_boxes=gt) + + gt_boxes = gt.gt_boxes + device = proposals.objectness_logits.device + # Assign all ground-truth boxes an objectness logit corresponding to + # P(object) = sigmoid(logit) =~ 1. + gt_logit_value = math.log((1.0 - 1e-10) / (1 - (1.0 - 1e-10))) + gt_logits = gt_logit_value * torch.ones(len(gt_boxes), device=device) + + # Concatenating gt_boxes with proposals requires them to have the same fields + gt_proposal = Instances(proposals.image_size, **gt.get_fields()) + gt_proposal.proposal_boxes = gt_boxes + gt_proposal.objectness_logits = gt_logits + + for key in proposals.get_fields().keys(): + assert gt_proposal.has( + key + ), "The attribute '{}' in `proposals` does not exist in `gt`".format(key) + + # NOTE: Instances.cat only use fields from the first item. Extra fields in latter items + # will be thrown away. + new_proposals = Instances.cat([proposals, gt_proposal]) + + return new_proposals diff --git a/annotator/oneformer/detectron2/modeling/proposal_generator/rpn.py b/annotator/oneformer/detectron2/modeling/proposal_generator/rpn.py new file mode 100644 index 0000000000000000000000000000000000000000..e37860dd6edb7a3cf493def2ae60a424b4dfc357 --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/proposal_generator/rpn.py @@ -0,0 +1,533 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from typing import Dict, List, Optional, Tuple, Union +import torch +import torch.nn.functional as F +from torch import nn + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.layers import Conv2d, ShapeSpec, cat +from annotator.oneformer.detectron2.structures import Boxes, ImageList, Instances, pairwise_iou +from annotator.oneformer.detectron2.utils.events import get_event_storage +from annotator.oneformer.detectron2.utils.memory import retry_if_cuda_oom +from annotator.oneformer.detectron2.utils.registry import Registry + +from ..anchor_generator import build_anchor_generator +from ..box_regression import Box2BoxTransform, _dense_box_regression_loss +from ..matcher import Matcher +from ..sampling import subsample_labels +from .build import PROPOSAL_GENERATOR_REGISTRY +from .proposal_utils import find_top_rpn_proposals + +RPN_HEAD_REGISTRY = Registry("RPN_HEAD") +RPN_HEAD_REGISTRY.__doc__ = """ +Registry for RPN heads, which take feature maps and perform +objectness classification and bounding box regression for anchors. + +The registered object will be called with `obj(cfg, input_shape)`. +The call should return a `nn.Module` object. +""" + + +""" +Shape shorthand in this module: + + N: number of images in the minibatch + L: number of feature maps per image on which RPN is run + A: number of cell anchors (must be the same for all feature maps) + Hi, Wi: height and width of the i-th feature map + B: size of the box parameterization + +Naming convention: + + objectness: refers to the binary classification of an anchor as object vs. not object. + + deltas: refers to the 4-d (dx, dy, dw, dh) deltas that parameterize the box2box + transform (see :class:`box_regression.Box2BoxTransform`), or 5d for rotated boxes. + + pred_objectness_logits: predicted objectness scores in [-inf, +inf]; use + sigmoid(pred_objectness_logits) to estimate P(object). + + gt_labels: ground-truth binary classification labels for objectness + + pred_anchor_deltas: predicted box2box transform deltas + + gt_anchor_deltas: ground-truth box2box transform deltas +""" + + +def build_rpn_head(cfg, input_shape): + """ + Build an RPN head defined by `cfg.MODEL.RPN.HEAD_NAME`. + """ + name = cfg.MODEL.RPN.HEAD_NAME + return RPN_HEAD_REGISTRY.get(name)(cfg, input_shape) + + +@RPN_HEAD_REGISTRY.register() +class StandardRPNHead(nn.Module): + """ + Standard RPN classification and regression heads described in :paper:`Faster R-CNN`. + Uses a 3x3 conv to produce a shared hidden state from which one 1x1 conv predicts + objectness logits for each anchor and a second 1x1 conv predicts bounding-box deltas + specifying how to deform each anchor into an object proposal. + """ + + @configurable + def __init__( + self, *, in_channels: int, num_anchors: int, box_dim: int = 4, conv_dims: List[int] = (-1,) + ): + """ + NOTE: this interface is experimental. + + Args: + in_channels (int): number of input feature channels. When using multiple + input features, they must have the same number of channels. + num_anchors (int): number of anchors to predict for *each spatial position* + on the feature map. The total number of anchors for each + feature map will be `num_anchors * H * W`. + box_dim (int): dimension of a box, which is also the number of box regression + predictions to make for each anchor. An axis aligned box has + box_dim=4, while a rotated box has box_dim=5. + conv_dims (list[int]): a list of integers representing the output channels + of N conv layers. Set it to -1 to use the same number of output channels + as input channels. + """ + super().__init__() + cur_channels = in_channels + # Keeping the old variable names and structure for backwards compatiblity. + # Otherwise the old checkpoints will fail to load. + if len(conv_dims) == 1: + out_channels = cur_channels if conv_dims[0] == -1 else conv_dims[0] + # 3x3 conv for the hidden representation + self.conv = self._get_rpn_conv(cur_channels, out_channels) + cur_channels = out_channels + else: + self.conv = nn.Sequential() + for k, conv_dim in enumerate(conv_dims): + out_channels = cur_channels if conv_dim == -1 else conv_dim + if out_channels <= 0: + raise ValueError( + f"Conv output channels should be greater than 0. Got {out_channels}" + ) + conv = self._get_rpn_conv(cur_channels, out_channels) + self.conv.add_module(f"conv{k}", conv) + cur_channels = out_channels + # 1x1 conv for predicting objectness logits + self.objectness_logits = nn.Conv2d(cur_channels, num_anchors, kernel_size=1, stride=1) + # 1x1 conv for predicting box2box transform deltas + self.anchor_deltas = nn.Conv2d(cur_channels, num_anchors * box_dim, kernel_size=1, stride=1) + + # Keeping the order of weights initialization same for backwards compatiblility. + for layer in self.modules(): + if isinstance(layer, nn.Conv2d): + nn.init.normal_(layer.weight, std=0.01) + nn.init.constant_(layer.bias, 0) + + def _get_rpn_conv(self, in_channels, out_channels): + return Conv2d( + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + activation=nn.ReLU(), + ) + + @classmethod + def from_config(cls, cfg, input_shape): + # Standard RPN is shared across levels: + in_channels = [s.channels for s in input_shape] + assert len(set(in_channels)) == 1, "Each level must have the same channel!" + in_channels = in_channels[0] + + # RPNHead should take the same input as anchor generator + # NOTE: it assumes that creating an anchor generator does not have unwanted side effect. + anchor_generator = build_anchor_generator(cfg, input_shape) + num_anchors = anchor_generator.num_anchors + box_dim = anchor_generator.box_dim + assert ( + len(set(num_anchors)) == 1 + ), "Each level must have the same number of anchors per spatial position" + return { + "in_channels": in_channels, + "num_anchors": num_anchors[0], + "box_dim": box_dim, + "conv_dims": cfg.MODEL.RPN.CONV_DIMS, + } + + def forward(self, features: List[torch.Tensor]): + """ + Args: + features (list[Tensor]): list of feature maps + + Returns: + list[Tensor]: A list of L elements. + Element i is a tensor of shape (N, A, Hi, Wi) representing + the predicted objectness logits for all anchors. A is the number of cell anchors. + list[Tensor]: A list of L elements. Element i is a tensor of shape + (N, A*box_dim, Hi, Wi) representing the predicted "deltas" used to transform anchors + to proposals. + """ + pred_objectness_logits = [] + pred_anchor_deltas = [] + for x in features: + t = self.conv(x) + pred_objectness_logits.append(self.objectness_logits(t)) + pred_anchor_deltas.append(self.anchor_deltas(t)) + return pred_objectness_logits, pred_anchor_deltas + + +@PROPOSAL_GENERATOR_REGISTRY.register() +class RPN(nn.Module): + """ + Region Proposal Network, introduced by :paper:`Faster R-CNN`. + """ + + @configurable + def __init__( + self, + *, + in_features: List[str], + head: nn.Module, + anchor_generator: nn.Module, + anchor_matcher: Matcher, + box2box_transform: Box2BoxTransform, + batch_size_per_image: int, + positive_fraction: float, + pre_nms_topk: Tuple[float, float], + post_nms_topk: Tuple[float, float], + nms_thresh: float = 0.7, + min_box_size: float = 0.0, + anchor_boundary_thresh: float = -1.0, + loss_weight: Union[float, Dict[str, float]] = 1.0, + box_reg_loss_type: str = "smooth_l1", + smooth_l1_beta: float = 0.0, + ): + """ + NOTE: this interface is experimental. + + Args: + in_features (list[str]): list of names of input features to use + head (nn.Module): a module that predicts logits and regression deltas + for each level from a list of per-level features + anchor_generator (nn.Module): a module that creates anchors from a + list of features. Usually an instance of :class:`AnchorGenerator` + anchor_matcher (Matcher): label the anchors by matching them with ground truth. + box2box_transform (Box2BoxTransform): defines the transform from anchors boxes to + instance boxes + batch_size_per_image (int): number of anchors per image to sample for training + positive_fraction (float): fraction of foreground anchors to sample for training + pre_nms_topk (tuple[float]): (train, test) that represents the + number of top k proposals to select before NMS, in + training and testing. + post_nms_topk (tuple[float]): (train, test) that represents the + number of top k proposals to select after NMS, in + training and testing. + nms_thresh (float): NMS threshold used to de-duplicate the predicted proposals + min_box_size (float): remove proposal boxes with any side smaller than this threshold, + in the unit of input image pixels + anchor_boundary_thresh (float): legacy option + loss_weight (float|dict): weights to use for losses. Can be single float for weighting + all rpn losses together, or a dict of individual weightings. Valid dict keys are: + "loss_rpn_cls" - applied to classification loss + "loss_rpn_loc" - applied to box regression loss + box_reg_loss_type (str): Loss type to use. Supported losses: "smooth_l1", "giou". + smooth_l1_beta (float): beta parameter for the smooth L1 regression loss. Default to + use L1 loss. Only used when `box_reg_loss_type` is "smooth_l1" + """ + super().__init__() + self.in_features = in_features + self.rpn_head = head + self.anchor_generator = anchor_generator + self.anchor_matcher = anchor_matcher + self.box2box_transform = box2box_transform + self.batch_size_per_image = batch_size_per_image + self.positive_fraction = positive_fraction + # Map from self.training state to train/test settings + self.pre_nms_topk = {True: pre_nms_topk[0], False: pre_nms_topk[1]} + self.post_nms_topk = {True: post_nms_topk[0], False: post_nms_topk[1]} + self.nms_thresh = nms_thresh + self.min_box_size = float(min_box_size) + self.anchor_boundary_thresh = anchor_boundary_thresh + if isinstance(loss_weight, float): + loss_weight = {"loss_rpn_cls": loss_weight, "loss_rpn_loc": loss_weight} + self.loss_weight = loss_weight + self.box_reg_loss_type = box_reg_loss_type + self.smooth_l1_beta = smooth_l1_beta + + @classmethod + def from_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): + in_features = cfg.MODEL.RPN.IN_FEATURES + ret = { + "in_features": in_features, + "min_box_size": cfg.MODEL.PROPOSAL_GENERATOR.MIN_SIZE, + "nms_thresh": cfg.MODEL.RPN.NMS_THRESH, + "batch_size_per_image": cfg.MODEL.RPN.BATCH_SIZE_PER_IMAGE, + "positive_fraction": cfg.MODEL.RPN.POSITIVE_FRACTION, + "loss_weight": { + "loss_rpn_cls": cfg.MODEL.RPN.LOSS_WEIGHT, + "loss_rpn_loc": cfg.MODEL.RPN.BBOX_REG_LOSS_WEIGHT * cfg.MODEL.RPN.LOSS_WEIGHT, + }, + "anchor_boundary_thresh": cfg.MODEL.RPN.BOUNDARY_THRESH, + "box2box_transform": Box2BoxTransform(weights=cfg.MODEL.RPN.BBOX_REG_WEIGHTS), + "box_reg_loss_type": cfg.MODEL.RPN.BBOX_REG_LOSS_TYPE, + "smooth_l1_beta": cfg.MODEL.RPN.SMOOTH_L1_BETA, + } + + ret["pre_nms_topk"] = (cfg.MODEL.RPN.PRE_NMS_TOPK_TRAIN, cfg.MODEL.RPN.PRE_NMS_TOPK_TEST) + ret["post_nms_topk"] = (cfg.MODEL.RPN.POST_NMS_TOPK_TRAIN, cfg.MODEL.RPN.POST_NMS_TOPK_TEST) + + ret["anchor_generator"] = build_anchor_generator(cfg, [input_shape[f] for f in in_features]) + ret["anchor_matcher"] = Matcher( + cfg.MODEL.RPN.IOU_THRESHOLDS, cfg.MODEL.RPN.IOU_LABELS, allow_low_quality_matches=True + ) + ret["head"] = build_rpn_head(cfg, [input_shape[f] for f in in_features]) + return ret + + def _subsample_labels(self, label): + """ + Randomly sample a subset of positive and negative examples, and overwrite + the label vector to the ignore value (-1) for all elements that are not + included in the sample. + + Args: + labels (Tensor): a vector of -1, 0, 1. Will be modified in-place and returned. + """ + pos_idx, neg_idx = subsample_labels( + label, self.batch_size_per_image, self.positive_fraction, 0 + ) + # Fill with the ignore label (-1), then set positive and negative labels + label.fill_(-1) + label.scatter_(0, pos_idx, 1) + label.scatter_(0, neg_idx, 0) + return label + + @torch.jit.unused + @torch.no_grad() + def label_and_sample_anchors( + self, anchors: List[Boxes], gt_instances: List[Instances] + ) -> Tuple[List[torch.Tensor], List[torch.Tensor]]: + """ + Args: + anchors (list[Boxes]): anchors for each feature map. + gt_instances: the ground-truth instances for each image. + + Returns: + list[Tensor]: + List of #img tensors. i-th element is a vector of labels whose length is + the total number of anchors across all feature maps R = sum(Hi * Wi * A). + Label values are in {-1, 0, 1}, with meanings: -1 = ignore; 0 = negative + class; 1 = positive class. + list[Tensor]: + i-th element is a Rx4 tensor. The values are the matched gt boxes for each + anchor. Values are undefined for those anchors not labeled as 1. + """ + anchors = Boxes.cat(anchors) + + gt_boxes = [x.gt_boxes for x in gt_instances] + image_sizes = [x.image_size for x in gt_instances] + del gt_instances + + gt_labels = [] + matched_gt_boxes = [] + for image_size_i, gt_boxes_i in zip(image_sizes, gt_boxes): + """ + image_size_i: (h, w) for the i-th image + gt_boxes_i: ground-truth boxes for i-th image + """ + + match_quality_matrix = retry_if_cuda_oom(pairwise_iou)(gt_boxes_i, anchors) + matched_idxs, gt_labels_i = retry_if_cuda_oom(self.anchor_matcher)(match_quality_matrix) + # Matching is memory-expensive and may result in CPU tensors. But the result is small + gt_labels_i = gt_labels_i.to(device=gt_boxes_i.device) + del match_quality_matrix + + if self.anchor_boundary_thresh >= 0: + # Discard anchors that go out of the boundaries of the image + # NOTE: This is legacy functionality that is turned off by default in Detectron2 + anchors_inside_image = anchors.inside_box(image_size_i, self.anchor_boundary_thresh) + gt_labels_i[~anchors_inside_image] = -1 + + # A vector of labels (-1, 0, 1) for each anchor + gt_labels_i = self._subsample_labels(gt_labels_i) + + if len(gt_boxes_i) == 0: + # These values won't be used anyway since the anchor is labeled as background + matched_gt_boxes_i = torch.zeros_like(anchors.tensor) + else: + # TODO wasted indexing computation for ignored boxes + matched_gt_boxes_i = gt_boxes_i[matched_idxs].tensor + + gt_labels.append(gt_labels_i) # N,AHW + matched_gt_boxes.append(matched_gt_boxes_i) + return gt_labels, matched_gt_boxes + + @torch.jit.unused + def losses( + self, + anchors: List[Boxes], + pred_objectness_logits: List[torch.Tensor], + gt_labels: List[torch.Tensor], + pred_anchor_deltas: List[torch.Tensor], + gt_boxes: List[torch.Tensor], + ) -> Dict[str, torch.Tensor]: + """ + Return the losses from a set of RPN predictions and their associated ground-truth. + + Args: + anchors (list[Boxes or RotatedBoxes]): anchors for each feature map, each + has shape (Hi*Wi*A, B), where B is box dimension (4 or 5). + pred_objectness_logits (list[Tensor]): A list of L elements. + Element i is a tensor of shape (N, Hi*Wi*A) representing + the predicted objectness logits for all anchors. + gt_labels (list[Tensor]): Output of :meth:`label_and_sample_anchors`. + pred_anchor_deltas (list[Tensor]): A list of L elements. Element i is a tensor of shape + (N, Hi*Wi*A, 4 or 5) representing the predicted "deltas" used to transform anchors + to proposals. + gt_boxes (list[Tensor]): Output of :meth:`label_and_sample_anchors`. + + Returns: + dict[loss name -> loss value]: A dict mapping from loss name to loss value. + Loss names are: `loss_rpn_cls` for objectness classification and + `loss_rpn_loc` for proposal localization. + """ + num_images = len(gt_labels) + gt_labels = torch.stack(gt_labels) # (N, sum(Hi*Wi*Ai)) + + # Log the number of positive/negative anchors per-image that's used in training + pos_mask = gt_labels == 1 + num_pos_anchors = pos_mask.sum().item() + num_neg_anchors = (gt_labels == 0).sum().item() + storage = get_event_storage() + storage.put_scalar("rpn/num_pos_anchors", num_pos_anchors / num_images) + storage.put_scalar("rpn/num_neg_anchors", num_neg_anchors / num_images) + + localization_loss = _dense_box_regression_loss( + anchors, + self.box2box_transform, + pred_anchor_deltas, + gt_boxes, + pos_mask, + box_reg_loss_type=self.box_reg_loss_type, + smooth_l1_beta=self.smooth_l1_beta, + ) + + valid_mask = gt_labels >= 0 + objectness_loss = F.binary_cross_entropy_with_logits( + cat(pred_objectness_logits, dim=1)[valid_mask], + gt_labels[valid_mask].to(torch.float32), + reduction="sum", + ) + normalizer = self.batch_size_per_image * num_images + losses = { + "loss_rpn_cls": objectness_loss / normalizer, + # The original Faster R-CNN paper uses a slightly different normalizer + # for loc loss. But it doesn't matter in practice + "loss_rpn_loc": localization_loss / normalizer, + } + losses = {k: v * self.loss_weight.get(k, 1.0) for k, v in losses.items()} + return losses + + def forward( + self, + images: ImageList, + features: Dict[str, torch.Tensor], + gt_instances: Optional[List[Instances]] = None, + ): + """ + Args: + images (ImageList): input images of length `N` + features (dict[str, Tensor]): input data as a mapping from feature + map name to tensor. Axis 0 represents the number of images `N` in + the input data; axes 1-3 are channels, height, and width, which may + vary between feature maps (e.g., if a feature pyramid is used). + gt_instances (list[Instances], optional): a length `N` list of `Instances`s. + Each `Instances` stores ground-truth instances for the corresponding image. + + Returns: + proposals: list[Instances]: contains fields "proposal_boxes", "objectness_logits" + loss: dict[Tensor] or None + """ + features = [features[f] for f in self.in_features] + anchors = self.anchor_generator(features) + + pred_objectness_logits, pred_anchor_deltas = self.rpn_head(features) + # Transpose the Hi*Wi*A dimension to the middle: + pred_objectness_logits = [ + # (N, A, Hi, Wi) -> (N, Hi, Wi, A) -> (N, Hi*Wi*A) + score.permute(0, 2, 3, 1).flatten(1) + for score in pred_objectness_logits + ] + pred_anchor_deltas = [ + # (N, A*B, Hi, Wi) -> (N, A, B, Hi, Wi) -> (N, Hi, Wi, A, B) -> (N, Hi*Wi*A, B) + x.view(x.shape[0], -1, self.anchor_generator.box_dim, x.shape[-2], x.shape[-1]) + .permute(0, 3, 4, 1, 2) + .flatten(1, -2) + for x in pred_anchor_deltas + ] + + if self.training: + assert gt_instances is not None, "RPN requires gt_instances in training!" + gt_labels, gt_boxes = self.label_and_sample_anchors(anchors, gt_instances) + losses = self.losses( + anchors, pred_objectness_logits, gt_labels, pred_anchor_deltas, gt_boxes + ) + else: + losses = {} + proposals = self.predict_proposals( + anchors, pred_objectness_logits, pred_anchor_deltas, images.image_sizes + ) + return proposals, losses + + def predict_proposals( + self, + anchors: List[Boxes], + pred_objectness_logits: List[torch.Tensor], + pred_anchor_deltas: List[torch.Tensor], + image_sizes: List[Tuple[int, int]], + ): + """ + Decode all the predicted box regression deltas to proposals. Find the top proposals + by applying NMS and removing boxes that are too small. + + Returns: + proposals (list[Instances]): list of N Instances. The i-th Instances + stores post_nms_topk object proposals for image i, sorted by their + objectness score in descending order. + """ + # The proposals are treated as fixed for joint training with roi heads. + # This approach ignores the derivative w.r.t. the proposal boxes’ coordinates that + # are also network responses. + with torch.no_grad(): + pred_proposals = self._decode_proposals(anchors, pred_anchor_deltas) + return find_top_rpn_proposals( + pred_proposals, + pred_objectness_logits, + image_sizes, + self.nms_thresh, + self.pre_nms_topk[self.training], + self.post_nms_topk[self.training], + self.min_box_size, + self.training, + ) + + def _decode_proposals(self, anchors: List[Boxes], pred_anchor_deltas: List[torch.Tensor]): + """ + Transform anchors into proposals by applying the predicted anchor deltas. + + Returns: + proposals (list[Tensor]): A list of L tensors. Tensor i has shape + (N, Hi*Wi*A, B) + """ + N = pred_anchor_deltas[0].shape[0] + proposals = [] + # For each feature map + for anchors_i, pred_anchor_deltas_i in zip(anchors, pred_anchor_deltas): + B = anchors_i.tensor.size(1) + pred_anchor_deltas_i = pred_anchor_deltas_i.reshape(-1, B) + # Expand anchors to shape (N*Hi*Wi*A, B) + anchors_i = anchors_i.tensor.unsqueeze(0).expand(N, -1, -1).reshape(-1, B) + proposals_i = self.box2box_transform.apply_deltas(pred_anchor_deltas_i, anchors_i) + # Append feature map proposals with shape (N, Hi*Wi*A, B) + proposals.append(proposals_i.view(N, -1, B)) + return proposals diff --git a/annotator/oneformer/detectron2/modeling/proposal_generator/rrpn.py b/annotator/oneformer/detectron2/modeling/proposal_generator/rrpn.py new file mode 100644 index 0000000000000000000000000000000000000000..8535dcd992bc4a83ea05d285f0ec5fae1271f41d --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/proposal_generator/rrpn.py @@ -0,0 +1,209 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import itertools +import logging +from typing import Dict, List +import torch + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.layers import ShapeSpec, batched_nms_rotated, cat +from annotator.oneformer.detectron2.structures import Instances, RotatedBoxes, pairwise_iou_rotated +from annotator.oneformer.detectron2.utils.memory import retry_if_cuda_oom + +from ..box_regression import Box2BoxTransformRotated +from .build import PROPOSAL_GENERATOR_REGISTRY +from .proposal_utils import _is_tracing +from .rpn import RPN + +logger = logging.getLogger(__name__) + + +def find_top_rrpn_proposals( + proposals, + pred_objectness_logits, + image_sizes, + nms_thresh, + pre_nms_topk, + post_nms_topk, + min_box_size, + training, +): + """ + For each feature map, select the `pre_nms_topk` highest scoring proposals, + apply NMS, clip proposals, and remove small boxes. Return the `post_nms_topk` + highest scoring proposals among all the feature maps if `training` is True, + otherwise, returns the highest `post_nms_topk` scoring proposals for each + feature map. + + Args: + proposals (list[Tensor]): A list of L tensors. Tensor i has shape (N, Hi*Wi*A, 5). + All proposal predictions on the feature maps. + pred_objectness_logits (list[Tensor]): A list of L tensors. Tensor i has shape (N, Hi*Wi*A). + image_sizes (list[tuple]): sizes (h, w) for each image + nms_thresh (float): IoU threshold to use for NMS + pre_nms_topk (int): number of top k scoring proposals to keep before applying NMS. + When RRPN is run on multiple feature maps (as in FPN) this number is per + feature map. + post_nms_topk (int): number of top k scoring proposals to keep after applying NMS. + When RRPN is run on multiple feature maps (as in FPN) this number is total, + over all feature maps. + min_box_size(float): minimum proposal box side length in pixels (absolute units wrt + input images). + training (bool): True if proposals are to be used in training, otherwise False. + This arg exists only to support a legacy bug; look for the "NB: Legacy bug ..." + comment. + + Returns: + proposals (list[Instances]): list of N Instances. The i-th Instances + stores post_nms_topk object proposals for image i. + """ + num_images = len(image_sizes) + device = proposals[0].device + + # 1. Select top-k anchor for every level and every image + topk_scores = [] # #lvl Tensor, each of shape N x topk + topk_proposals = [] + level_ids = [] # #lvl Tensor, each of shape (topk,) + batch_idx = torch.arange(num_images, device=device) + for level_id, proposals_i, logits_i in zip( + itertools.count(), proposals, pred_objectness_logits + ): + Hi_Wi_A = logits_i.shape[1] + if isinstance(Hi_Wi_A, torch.Tensor): # it's a tensor in tracing + num_proposals_i = torch.clamp(Hi_Wi_A, max=pre_nms_topk) + else: + num_proposals_i = min(Hi_Wi_A, pre_nms_topk) + + topk_scores_i, topk_idx = logits_i.topk(num_proposals_i, dim=1) + + # each is N x topk + topk_proposals_i = proposals_i[batch_idx[:, None], topk_idx] # N x topk x 5 + + topk_proposals.append(topk_proposals_i) + topk_scores.append(topk_scores_i) + level_ids.append(torch.full((num_proposals_i,), level_id, dtype=torch.int64, device=device)) + + # 2. Concat all levels together + topk_scores = cat(topk_scores, dim=1) + topk_proposals = cat(topk_proposals, dim=1) + level_ids = cat(level_ids, dim=0) + + # 3. For each image, run a per-level NMS, and choose topk results. + results = [] + for n, image_size in enumerate(image_sizes): + boxes = RotatedBoxes(topk_proposals[n]) + scores_per_img = topk_scores[n] + lvl = level_ids + + valid_mask = torch.isfinite(boxes.tensor).all(dim=1) & torch.isfinite(scores_per_img) + if not valid_mask.all(): + if training: + raise FloatingPointError( + "Predicted boxes or scores contain Inf/NaN. Training has diverged." + ) + boxes = boxes[valid_mask] + scores_per_img = scores_per_img[valid_mask] + lvl = lvl[valid_mask] + boxes.clip(image_size) + + # filter empty boxes + keep = boxes.nonempty(threshold=min_box_size) + if _is_tracing() or keep.sum().item() != len(boxes): + boxes, scores_per_img, lvl = (boxes[keep], scores_per_img[keep], lvl[keep]) + + keep = batched_nms_rotated(boxes.tensor, scores_per_img, lvl, nms_thresh) + # In Detectron1, there was different behavior during training vs. testing. + # (https://github.com/facebookresearch/Detectron/issues/459) + # During training, topk is over the proposals from *all* images in the training batch. + # During testing, it is over the proposals for each image separately. + # As a result, the training behavior becomes batch-dependent, + # and the configuration "POST_NMS_TOPK_TRAIN" end up relying on the batch size. + # This bug is addressed in Detectron2 to make the behavior independent of batch size. + keep = keep[:post_nms_topk] + + res = Instances(image_size) + res.proposal_boxes = boxes[keep] + res.objectness_logits = scores_per_img[keep] + results.append(res) + return results + + +@PROPOSAL_GENERATOR_REGISTRY.register() +class RRPN(RPN): + """ + Rotated Region Proposal Network described in :paper:`RRPN`. + """ + + @configurable + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if self.anchor_boundary_thresh >= 0: + raise NotImplementedError( + "anchor_boundary_thresh is a legacy option not implemented for RRPN." + ) + + @classmethod + def from_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): + ret = super().from_config(cfg, input_shape) + ret["box2box_transform"] = Box2BoxTransformRotated(weights=cfg.MODEL.RPN.BBOX_REG_WEIGHTS) + return ret + + @torch.no_grad() + def label_and_sample_anchors(self, anchors: List[RotatedBoxes], gt_instances: List[Instances]): + """ + Args: + anchors (list[RotatedBoxes]): anchors for each feature map. + gt_instances: the ground-truth instances for each image. + + Returns: + list[Tensor]: + List of #img tensors. i-th element is a vector of labels whose length is + the total number of anchors across feature maps. Label values are in {-1, 0, 1}, + with meanings: -1 = ignore; 0 = negative class; 1 = positive class. + list[Tensor]: + i-th element is a Nx5 tensor, where N is the total number of anchors across + feature maps. The values are the matched gt boxes for each anchor. + Values are undefined for those anchors not labeled as 1. + """ + anchors = RotatedBoxes.cat(anchors) + + gt_boxes = [x.gt_boxes for x in gt_instances] + del gt_instances + + gt_labels = [] + matched_gt_boxes = [] + for gt_boxes_i in gt_boxes: + """ + gt_boxes_i: ground-truth boxes for i-th image + """ + match_quality_matrix = retry_if_cuda_oom(pairwise_iou_rotated)(gt_boxes_i, anchors) + matched_idxs, gt_labels_i = retry_if_cuda_oom(self.anchor_matcher)(match_quality_matrix) + # Matching is memory-expensive and may result in CPU tensors. But the result is small + gt_labels_i = gt_labels_i.to(device=gt_boxes_i.device) + + # A vector of labels (-1, 0, 1) for each anchor + gt_labels_i = self._subsample_labels(gt_labels_i) + + if len(gt_boxes_i) == 0: + # These values won't be used anyway since the anchor is labeled as background + matched_gt_boxes_i = torch.zeros_like(anchors.tensor) + else: + # TODO wasted indexing computation for ignored boxes + matched_gt_boxes_i = gt_boxes_i[matched_idxs].tensor + + gt_labels.append(gt_labels_i) # N,AHW + matched_gt_boxes.append(matched_gt_boxes_i) + return gt_labels, matched_gt_boxes + + @torch.no_grad() + def predict_proposals(self, anchors, pred_objectness_logits, pred_anchor_deltas, image_sizes): + pred_proposals = self._decode_proposals(anchors, pred_anchor_deltas) + return find_top_rrpn_proposals( + pred_proposals, + pred_objectness_logits, + image_sizes, + self.nms_thresh, + self.pre_nms_topk[self.training], + self.post_nms_topk[self.training], + self.min_box_size, + self.training, + ) diff --git a/annotator/oneformer/detectron2/modeling/roi_heads/__init__.py b/annotator/oneformer/detectron2/modeling/roi_heads/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d13e9c57235b982f3e0645bc316de2b75755dfda --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/roi_heads/__init__.py @@ -0,0 +1,29 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .box_head import ROI_BOX_HEAD_REGISTRY, build_box_head, FastRCNNConvFCHead +from .keypoint_head import ( + ROI_KEYPOINT_HEAD_REGISTRY, + build_keypoint_head, + BaseKeypointRCNNHead, + KRCNNConvDeconvUpsampleHead, +) +from .mask_head import ( + ROI_MASK_HEAD_REGISTRY, + build_mask_head, + BaseMaskRCNNHead, + MaskRCNNConvUpsampleHead, +) +from .roi_heads import ( + ROI_HEADS_REGISTRY, + ROIHeads, + Res5ROIHeads, + StandardROIHeads, + build_roi_heads, + select_foreground_proposals, +) +from .cascade_rcnn import CascadeROIHeads +from .rotated_fast_rcnn import RROIHeads +from .fast_rcnn import FastRCNNOutputLayers + +from . import cascade_rcnn # isort:skip + +__all__ = list(globals().keys()) diff --git a/annotator/oneformer/detectron2/modeling/roi_heads/box_head.py b/annotator/oneformer/detectron2/modeling/roi_heads/box_head.py new file mode 100644 index 0000000000000000000000000000000000000000..1e598af4f08af6618997607e1633f2b842eb6da0 --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/roi_heads/box_head.py @@ -0,0 +1,118 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import numpy as np +from typing import List +import fvcore.nn.weight_init as weight_init +import torch +from torch import nn + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.layers import Conv2d, ShapeSpec, get_norm +from annotator.oneformer.detectron2.utils.registry import Registry + +__all__ = ["FastRCNNConvFCHead", "build_box_head", "ROI_BOX_HEAD_REGISTRY"] + +ROI_BOX_HEAD_REGISTRY = Registry("ROI_BOX_HEAD") +ROI_BOX_HEAD_REGISTRY.__doc__ = """ +Registry for box heads, which make box predictions from per-region features. + +The registered object will be called with `obj(cfg, input_shape)`. +""" + + +# To get torchscript support, we make the head a subclass of `nn.Sequential`. +# Therefore, to add new layers in this head class, please make sure they are +# added in the order they will be used in forward(). +@ROI_BOX_HEAD_REGISTRY.register() +class FastRCNNConvFCHead(nn.Sequential): + """ + A head with several 3x3 conv layers (each followed by norm & relu) and then + several fc layers (each followed by relu). + """ + + @configurable + def __init__( + self, input_shape: ShapeSpec, *, conv_dims: List[int], fc_dims: List[int], conv_norm="" + ): + """ + NOTE: this interface is experimental. + + Args: + input_shape (ShapeSpec): shape of the input feature. + conv_dims (list[int]): the output dimensions of the conv layers + fc_dims (list[int]): the output dimensions of the fc layers + conv_norm (str or callable): normalization for the conv layers. + See :func:`detectron2.layers.get_norm` for supported types. + """ + super().__init__() + assert len(conv_dims) + len(fc_dims) > 0 + + self._output_size = (input_shape.channels, input_shape.height, input_shape.width) + + self.conv_norm_relus = [] + for k, conv_dim in enumerate(conv_dims): + conv = Conv2d( + self._output_size[0], + conv_dim, + kernel_size=3, + padding=1, + bias=not conv_norm, + norm=get_norm(conv_norm, conv_dim), + activation=nn.ReLU(), + ) + self.add_module("conv{}".format(k + 1), conv) + self.conv_norm_relus.append(conv) + self._output_size = (conv_dim, self._output_size[1], self._output_size[2]) + + self.fcs = [] + for k, fc_dim in enumerate(fc_dims): + if k == 0: + self.add_module("flatten", nn.Flatten()) + fc = nn.Linear(int(np.prod(self._output_size)), fc_dim) + self.add_module("fc{}".format(k + 1), fc) + self.add_module("fc_relu{}".format(k + 1), nn.ReLU()) + self.fcs.append(fc) + self._output_size = fc_dim + + for layer in self.conv_norm_relus: + weight_init.c2_msra_fill(layer) + for layer in self.fcs: + weight_init.c2_xavier_fill(layer) + + @classmethod + def from_config(cls, cfg, input_shape): + num_conv = cfg.MODEL.ROI_BOX_HEAD.NUM_CONV + conv_dim = cfg.MODEL.ROI_BOX_HEAD.CONV_DIM + num_fc = cfg.MODEL.ROI_BOX_HEAD.NUM_FC + fc_dim = cfg.MODEL.ROI_BOX_HEAD.FC_DIM + return { + "input_shape": input_shape, + "conv_dims": [conv_dim] * num_conv, + "fc_dims": [fc_dim] * num_fc, + "conv_norm": cfg.MODEL.ROI_BOX_HEAD.NORM, + } + + def forward(self, x): + for layer in self: + x = layer(x) + return x + + @property + @torch.jit.unused + def output_shape(self): + """ + Returns: + ShapeSpec: the output feature shape + """ + o = self._output_size + if isinstance(o, int): + return ShapeSpec(channels=o) + else: + return ShapeSpec(channels=o[0], height=o[1], width=o[2]) + + +def build_box_head(cfg, input_shape): + """ + Build a box head defined by `cfg.MODEL.ROI_BOX_HEAD.NAME`. + """ + name = cfg.MODEL.ROI_BOX_HEAD.NAME + return ROI_BOX_HEAD_REGISTRY.get(name)(cfg, input_shape) diff --git a/annotator/oneformer/detectron2/modeling/roi_heads/cascade_rcnn.py b/annotator/oneformer/detectron2/modeling/roi_heads/cascade_rcnn.py new file mode 100644 index 0000000000000000000000000000000000000000..69b837be7a7c2202fe4f94b7212b49678fe06c1e --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/roi_heads/cascade_rcnn.py @@ -0,0 +1,299 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from typing import List +import torch +from torch import nn +from torch.autograd.function import Function + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.layers import ShapeSpec +from annotator.oneformer.detectron2.structures import Boxes, Instances, pairwise_iou +from annotator.oneformer.detectron2.utils.events import get_event_storage + +from ..box_regression import Box2BoxTransform +from ..matcher import Matcher +from ..poolers import ROIPooler +from .box_head import build_box_head +from .fast_rcnn import FastRCNNOutputLayers, fast_rcnn_inference +from .roi_heads import ROI_HEADS_REGISTRY, StandardROIHeads + + +class _ScaleGradient(Function): + @staticmethod + def forward(ctx, input, scale): + ctx.scale = scale + return input + + @staticmethod + def backward(ctx, grad_output): + return grad_output * ctx.scale, None + + +@ROI_HEADS_REGISTRY.register() +class CascadeROIHeads(StandardROIHeads): + """ + The ROI heads that implement :paper:`Cascade R-CNN`. + """ + + @configurable + def __init__( + self, + *, + box_in_features: List[str], + box_pooler: ROIPooler, + box_heads: List[nn.Module], + box_predictors: List[nn.Module], + proposal_matchers: List[Matcher], + **kwargs, + ): + """ + NOTE: this interface is experimental. + + Args: + box_pooler (ROIPooler): pooler that extracts region features from given boxes + box_heads (list[nn.Module]): box head for each cascade stage + box_predictors (list[nn.Module]): box predictor for each cascade stage + proposal_matchers (list[Matcher]): matcher with different IoU thresholds to + match boxes with ground truth for each stage. The first matcher matches + RPN proposals with ground truth, the other matchers use boxes predicted + by the previous stage as proposals and match them with ground truth. + """ + assert "proposal_matcher" not in kwargs, ( + "CascadeROIHeads takes 'proposal_matchers=' for each stage instead " + "of one 'proposal_matcher='." + ) + # The first matcher matches RPN proposals with ground truth, done in the base class + kwargs["proposal_matcher"] = proposal_matchers[0] + num_stages = self.num_cascade_stages = len(box_heads) + box_heads = nn.ModuleList(box_heads) + box_predictors = nn.ModuleList(box_predictors) + assert len(box_predictors) == num_stages, f"{len(box_predictors)} != {num_stages}!" + assert len(proposal_matchers) == num_stages, f"{len(proposal_matchers)} != {num_stages}!" + super().__init__( + box_in_features=box_in_features, + box_pooler=box_pooler, + box_head=box_heads, + box_predictor=box_predictors, + **kwargs, + ) + self.proposal_matchers = proposal_matchers + + @classmethod + def from_config(cls, cfg, input_shape): + ret = super().from_config(cfg, input_shape) + ret.pop("proposal_matcher") + return ret + + @classmethod + def _init_box_head(cls, cfg, input_shape): + # fmt: off + in_features = cfg.MODEL.ROI_HEADS.IN_FEATURES + pooler_resolution = cfg.MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION + pooler_scales = tuple(1.0 / input_shape[k].stride for k in in_features) + sampling_ratio = cfg.MODEL.ROI_BOX_HEAD.POOLER_SAMPLING_RATIO + pooler_type = cfg.MODEL.ROI_BOX_HEAD.POOLER_TYPE + cascade_bbox_reg_weights = cfg.MODEL.ROI_BOX_CASCADE_HEAD.BBOX_REG_WEIGHTS + cascade_ious = cfg.MODEL.ROI_BOX_CASCADE_HEAD.IOUS + assert len(cascade_bbox_reg_weights) == len(cascade_ious) + assert cfg.MODEL.ROI_BOX_HEAD.CLS_AGNOSTIC_BBOX_REG, \ + "CascadeROIHeads only support class-agnostic regression now!" + assert cascade_ious[0] == cfg.MODEL.ROI_HEADS.IOU_THRESHOLDS[0] + # fmt: on + + in_channels = [input_shape[f].channels for f in in_features] + # Check all channel counts are equal + assert len(set(in_channels)) == 1, in_channels + in_channels = in_channels[0] + + box_pooler = ROIPooler( + output_size=pooler_resolution, + scales=pooler_scales, + sampling_ratio=sampling_ratio, + pooler_type=pooler_type, + ) + pooled_shape = ShapeSpec( + channels=in_channels, width=pooler_resolution, height=pooler_resolution + ) + + box_heads, box_predictors, proposal_matchers = [], [], [] + for match_iou, bbox_reg_weights in zip(cascade_ious, cascade_bbox_reg_weights): + box_head = build_box_head(cfg, pooled_shape) + box_heads.append(box_head) + box_predictors.append( + FastRCNNOutputLayers( + cfg, + box_head.output_shape, + box2box_transform=Box2BoxTransform(weights=bbox_reg_weights), + ) + ) + proposal_matchers.append(Matcher([match_iou], [0, 1], allow_low_quality_matches=False)) + return { + "box_in_features": in_features, + "box_pooler": box_pooler, + "box_heads": box_heads, + "box_predictors": box_predictors, + "proposal_matchers": proposal_matchers, + } + + def forward(self, images, features, proposals, targets=None): + del images + if self.training: + proposals = self.label_and_sample_proposals(proposals, targets) + + if self.training: + # Need targets to box head + losses = self._forward_box(features, proposals, targets) + losses.update(self._forward_mask(features, proposals)) + losses.update(self._forward_keypoint(features, proposals)) + return proposals, losses + else: + pred_instances = self._forward_box(features, proposals) + pred_instances = self.forward_with_given_boxes(features, pred_instances) + return pred_instances, {} + + def _forward_box(self, features, proposals, targets=None): + """ + Args: + features, targets: the same as in + Same as in :meth:`ROIHeads.forward`. + proposals (list[Instances]): the per-image object proposals with + their matching ground truth. + Each has fields "proposal_boxes", and "objectness_logits", + "gt_classes", "gt_boxes". + """ + features = [features[f] for f in self.box_in_features] + head_outputs = [] # (predictor, predictions, proposals) + prev_pred_boxes = None + image_sizes = [x.image_size for x in proposals] + for k in range(self.num_cascade_stages): + if k > 0: + # The output boxes of the previous stage are used to create the input + # proposals of the next stage. + proposals = self._create_proposals_from_boxes(prev_pred_boxes, image_sizes) + if self.training: + proposals = self._match_and_label_boxes(proposals, k, targets) + predictions = self._run_stage(features, proposals, k) + prev_pred_boxes = self.box_predictor[k].predict_boxes(predictions, proposals) + head_outputs.append((self.box_predictor[k], predictions, proposals)) + + if self.training: + losses = {} + storage = get_event_storage() + for stage, (predictor, predictions, proposals) in enumerate(head_outputs): + with storage.name_scope("stage{}".format(stage)): + stage_losses = predictor.losses(predictions, proposals) + losses.update({k + "_stage{}".format(stage): v for k, v in stage_losses.items()}) + return losses + else: + # Each is a list[Tensor] of length #image. Each tensor is Ri x (K+1) + scores_per_stage = [h[0].predict_probs(h[1], h[2]) for h in head_outputs] + + # Average the scores across heads + scores = [ + sum(list(scores_per_image)) * (1.0 / self.num_cascade_stages) + for scores_per_image in zip(*scores_per_stage) + ] + # Use the boxes of the last head + predictor, predictions, proposals = head_outputs[-1] + boxes = predictor.predict_boxes(predictions, proposals) + pred_instances, _ = fast_rcnn_inference( + boxes, + scores, + image_sizes, + predictor.test_score_thresh, + predictor.test_nms_thresh, + predictor.test_topk_per_image, + ) + return pred_instances + + @torch.no_grad() + def _match_and_label_boxes(self, proposals, stage, targets): + """ + Match proposals with groundtruth using the matcher at the given stage. + Label the proposals as foreground or background based on the match. + + Args: + proposals (list[Instances]): One Instances for each image, with + the field "proposal_boxes". + stage (int): the current stage + targets (list[Instances]): the ground truth instances + + Returns: + list[Instances]: the same proposals, but with fields "gt_classes" and "gt_boxes" + """ + num_fg_samples, num_bg_samples = [], [] + for proposals_per_image, targets_per_image in zip(proposals, targets): + match_quality_matrix = pairwise_iou( + targets_per_image.gt_boxes, proposals_per_image.proposal_boxes + ) + # proposal_labels are 0 or 1 + matched_idxs, proposal_labels = self.proposal_matchers[stage](match_quality_matrix) + if len(targets_per_image) > 0: + gt_classes = targets_per_image.gt_classes[matched_idxs] + # Label unmatched proposals (0 label from matcher) as background (label=num_classes) + gt_classes[proposal_labels == 0] = self.num_classes + gt_boxes = targets_per_image.gt_boxes[matched_idxs] + else: + gt_classes = torch.zeros_like(matched_idxs) + self.num_classes + gt_boxes = Boxes( + targets_per_image.gt_boxes.tensor.new_zeros((len(proposals_per_image), 4)) + ) + proposals_per_image.gt_classes = gt_classes + proposals_per_image.gt_boxes = gt_boxes + + num_fg_samples.append((proposal_labels == 1).sum().item()) + num_bg_samples.append(proposal_labels.numel() - num_fg_samples[-1]) + + # Log the number of fg/bg samples in each stage + storage = get_event_storage() + storage.put_scalar( + "stage{}/roi_head/num_fg_samples".format(stage), + sum(num_fg_samples) / len(num_fg_samples), + ) + storage.put_scalar( + "stage{}/roi_head/num_bg_samples".format(stage), + sum(num_bg_samples) / len(num_bg_samples), + ) + return proposals + + def _run_stage(self, features, proposals, stage): + """ + Args: + features (list[Tensor]): #lvl input features to ROIHeads + proposals (list[Instances]): #image Instances, with the field "proposal_boxes" + stage (int): the current stage + + Returns: + Same output as `FastRCNNOutputLayers.forward()`. + """ + box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals]) + # The original implementation averages the losses among heads, + # but scale up the parameter gradients of the heads. + # This is equivalent to adding the losses among heads, + # but scale down the gradients on features. + if self.training: + box_features = _ScaleGradient.apply(box_features, 1.0 / self.num_cascade_stages) + box_features = self.box_head[stage](box_features) + return self.box_predictor[stage](box_features) + + def _create_proposals_from_boxes(self, boxes, image_sizes): + """ + Args: + boxes (list[Tensor]): per-image predicted boxes, each of shape Ri x 4 + image_sizes (list[tuple]): list of image shapes in (h, w) + + Returns: + list[Instances]: per-image proposals with the given boxes. + """ + # Just like RPN, the proposals should not have gradients + boxes = [Boxes(b.detach()) for b in boxes] + proposals = [] + for boxes_per_image, image_size in zip(boxes, image_sizes): + boxes_per_image.clip(image_size) + if self.training: + # do not filter empty boxes at inference time, + # because the scores from each stage need to be aligned and added later + boxes_per_image = boxes_per_image[boxes_per_image.nonempty()] + prop = Instances(image_size) + prop.proposal_boxes = boxes_per_image + proposals.append(prop) + return proposals diff --git a/annotator/oneformer/detectron2/modeling/roi_heads/fast_rcnn.py b/annotator/oneformer/detectron2/modeling/roi_heads/fast_rcnn.py new file mode 100644 index 0000000000000000000000000000000000000000..a81c58ea863f32a24ed7d5ad3b2e4e4416c6a0ab --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/roi_heads/fast_rcnn.py @@ -0,0 +1,569 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +from typing import Callable, Dict, List, Optional, Tuple, Union +import torch +from torch import nn +from torch.nn import functional as F + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.data.detection_utils import get_fed_loss_cls_weights +from annotator.oneformer.detectron2.layers import ShapeSpec, batched_nms, cat, cross_entropy, nonzero_tuple +from annotator.oneformer.detectron2.modeling.box_regression import Box2BoxTransform, _dense_box_regression_loss +from annotator.oneformer.detectron2.structures import Boxes, Instances +from annotator.oneformer.detectron2.utils.events import get_event_storage + +__all__ = ["fast_rcnn_inference", "FastRCNNOutputLayers"] + + +logger = logging.getLogger(__name__) + +""" +Shape shorthand in this module: + + N: number of images in the minibatch + R: number of ROIs, combined over all images, in the minibatch + Ri: number of ROIs in image i + K: number of foreground classes. E.g.,there are 80 foreground classes in COCO. + +Naming convention: + + deltas: refers to the 4-d (dx, dy, dw, dh) deltas that parameterize the box2box + transform (see :class:`box_regression.Box2BoxTransform`). + + pred_class_logits: predicted class scores in [-inf, +inf]; use + softmax(pred_class_logits) to estimate P(class). + + gt_classes: ground-truth classification labels in [0, K], where [0, K) represent + foreground object classes and K represents the background class. + + pred_proposal_deltas: predicted box2box transform deltas for transforming proposals + to detection box predictions. + + gt_proposal_deltas: ground-truth box2box transform deltas +""" + + +def fast_rcnn_inference( + boxes: List[torch.Tensor], + scores: List[torch.Tensor], + image_shapes: List[Tuple[int, int]], + score_thresh: float, + nms_thresh: float, + topk_per_image: int, +): + """ + Call `fast_rcnn_inference_single_image` for all images. + + Args: + boxes (list[Tensor]): A list of Tensors of predicted class-specific or class-agnostic + boxes for each image. Element i has shape (Ri, K * 4) if doing + class-specific regression, or (Ri, 4) if doing class-agnostic + regression, where Ri is the number of predicted objects for image i. + This is compatible with the output of :meth:`FastRCNNOutputLayers.predict_boxes`. + scores (list[Tensor]): A list of Tensors of predicted class scores for each image. + Element i has shape (Ri, K + 1), where Ri is the number of predicted objects + for image i. Compatible with the output of :meth:`FastRCNNOutputLayers.predict_probs`. + image_shapes (list[tuple]): A list of (width, height) tuples for each image in the batch. + score_thresh (float): Only return detections with a confidence score exceeding this + threshold. + nms_thresh (float): The threshold to use for box non-maximum suppression. Value in [0, 1]. + topk_per_image (int): The number of top scoring detections to return. Set < 0 to return + all detections. + + Returns: + instances: (list[Instances]): A list of N instances, one for each image in the batch, + that stores the topk most confidence detections. + kept_indices: (list[Tensor]): A list of 1D tensor of length of N, each element indicates + the corresponding boxes/scores index in [0, Ri) from the input, for image i. + """ + result_per_image = [ + fast_rcnn_inference_single_image( + boxes_per_image, scores_per_image, image_shape, score_thresh, nms_thresh, topk_per_image + ) + for scores_per_image, boxes_per_image, image_shape in zip(scores, boxes, image_shapes) + ] + return [x[0] for x in result_per_image], [x[1] for x in result_per_image] + + +def _log_classification_stats(pred_logits, gt_classes, prefix="fast_rcnn"): + """ + Log the classification metrics to EventStorage. + + Args: + pred_logits: Rx(K+1) logits. The last column is for background class. + gt_classes: R labels + """ + num_instances = gt_classes.numel() + if num_instances == 0: + return + pred_classes = pred_logits.argmax(dim=1) + bg_class_ind = pred_logits.shape[1] - 1 + + fg_inds = (gt_classes >= 0) & (gt_classes < bg_class_ind) + num_fg = fg_inds.nonzero().numel() + fg_gt_classes = gt_classes[fg_inds] + fg_pred_classes = pred_classes[fg_inds] + + num_false_negative = (fg_pred_classes == bg_class_ind).nonzero().numel() + num_accurate = (pred_classes == gt_classes).nonzero().numel() + fg_num_accurate = (fg_pred_classes == fg_gt_classes).nonzero().numel() + + storage = get_event_storage() + storage.put_scalar(f"{prefix}/cls_accuracy", num_accurate / num_instances) + if num_fg > 0: + storage.put_scalar(f"{prefix}/fg_cls_accuracy", fg_num_accurate / num_fg) + storage.put_scalar(f"{prefix}/false_negative", num_false_negative / num_fg) + + +def fast_rcnn_inference_single_image( + boxes, + scores, + image_shape: Tuple[int, int], + score_thresh: float, + nms_thresh: float, + topk_per_image: int, +): + """ + Single-image inference. Return bounding-box detection results by thresholding + on scores and applying non-maximum suppression (NMS). + + Args: + Same as `fast_rcnn_inference`, but with boxes, scores, and image shapes + per image. + + Returns: + Same as `fast_rcnn_inference`, but for only one image. + """ + valid_mask = torch.isfinite(boxes).all(dim=1) & torch.isfinite(scores).all(dim=1) + if not valid_mask.all(): + boxes = boxes[valid_mask] + scores = scores[valid_mask] + + scores = scores[:, :-1] + num_bbox_reg_classes = boxes.shape[1] // 4 + # Convert to Boxes to use the `clip` function ... + boxes = Boxes(boxes.reshape(-1, 4)) + boxes.clip(image_shape) + boxes = boxes.tensor.view(-1, num_bbox_reg_classes, 4) # R x C x 4 + + # 1. Filter results based on detection scores. It can make NMS more efficient + # by filtering out low-confidence detections. + filter_mask = scores > score_thresh # R x K + # R' x 2. First column contains indices of the R predictions; + # Second column contains indices of classes. + filter_inds = filter_mask.nonzero() + if num_bbox_reg_classes == 1: + boxes = boxes[filter_inds[:, 0], 0] + else: + boxes = boxes[filter_mask] + scores = scores[filter_mask] + + # 2. Apply NMS for each class independently. + keep = batched_nms(boxes, scores, filter_inds[:, 1], nms_thresh) + if topk_per_image >= 0: + keep = keep[:topk_per_image] + boxes, scores, filter_inds = boxes[keep], scores[keep], filter_inds[keep] + + result = Instances(image_shape) + result.pred_boxes = Boxes(boxes) + result.scores = scores + result.pred_classes = filter_inds[:, 1] + return result, filter_inds[:, 0] + + +class FastRCNNOutputLayers(nn.Module): + """ + Two linear layers for predicting Fast R-CNN outputs: + + 1. proposal-to-detection box regression deltas + 2. classification scores + """ + + @configurable + def __init__( + self, + input_shape: ShapeSpec, + *, + box2box_transform, + num_classes: int, + test_score_thresh: float = 0.0, + test_nms_thresh: float = 0.5, + test_topk_per_image: int = 100, + cls_agnostic_bbox_reg: bool = False, + smooth_l1_beta: float = 0.0, + box_reg_loss_type: str = "smooth_l1", + loss_weight: Union[float, Dict[str, float]] = 1.0, + use_fed_loss: bool = False, + use_sigmoid_ce: bool = False, + get_fed_loss_cls_weights: Optional[Callable] = None, + fed_loss_num_classes: int = 50, + ): + """ + NOTE: this interface is experimental. + + Args: + input_shape (ShapeSpec): shape of the input feature to this module + box2box_transform (Box2BoxTransform or Box2BoxTransformRotated): + num_classes (int): number of foreground classes + test_score_thresh (float): threshold to filter predictions results. + test_nms_thresh (float): NMS threshold for prediction results. + test_topk_per_image (int): number of top predictions to produce per image. + cls_agnostic_bbox_reg (bool): whether to use class agnostic for bbox regression + smooth_l1_beta (float): transition point from L1 to L2 loss. Only used if + `box_reg_loss_type` is "smooth_l1" + box_reg_loss_type (str): Box regression loss type. One of: "smooth_l1", "giou", + "diou", "ciou" + loss_weight (float|dict): weights to use for losses. Can be single float for weighting + all losses, or a dict of individual weightings. Valid dict keys are: + * "loss_cls": applied to classification loss + * "loss_box_reg": applied to box regression loss + use_fed_loss (bool): whether to use federated loss which samples additional negative + classes to calculate the loss + use_sigmoid_ce (bool): whether to calculate the loss using weighted average of binary + cross entropy with logits. This could be used together with federated loss + get_fed_loss_cls_weights (Callable): a callable which takes dataset name and frequency + weight power, and returns the probabilities to sample negative classes for + federated loss. The implementation can be found in + detectron2/data/detection_utils.py + fed_loss_num_classes (int): number of federated classes to keep in total + """ + super().__init__() + if isinstance(input_shape, int): # some backward compatibility + input_shape = ShapeSpec(channels=input_shape) + self.num_classes = num_classes + input_size = input_shape.channels * (input_shape.width or 1) * (input_shape.height or 1) + # prediction layer for num_classes foreground classes and one background class (hence + 1) + self.cls_score = nn.Linear(input_size, num_classes + 1) + num_bbox_reg_classes = 1 if cls_agnostic_bbox_reg else num_classes + box_dim = len(box2box_transform.weights) + self.bbox_pred = nn.Linear(input_size, num_bbox_reg_classes * box_dim) + + nn.init.normal_(self.cls_score.weight, std=0.01) + nn.init.normal_(self.bbox_pred.weight, std=0.001) + for l in [self.cls_score, self.bbox_pred]: + nn.init.constant_(l.bias, 0) + + self.box2box_transform = box2box_transform + self.smooth_l1_beta = smooth_l1_beta + self.test_score_thresh = test_score_thresh + self.test_nms_thresh = test_nms_thresh + self.test_topk_per_image = test_topk_per_image + self.box_reg_loss_type = box_reg_loss_type + if isinstance(loss_weight, float): + loss_weight = {"loss_cls": loss_weight, "loss_box_reg": loss_weight} + self.loss_weight = loss_weight + self.use_fed_loss = use_fed_loss + self.use_sigmoid_ce = use_sigmoid_ce + self.fed_loss_num_classes = fed_loss_num_classes + + if self.use_fed_loss: + assert self.use_sigmoid_ce, "Please use sigmoid cross entropy loss with federated loss" + fed_loss_cls_weights = get_fed_loss_cls_weights() + assert ( + len(fed_loss_cls_weights) == self.num_classes + ), "Please check the provided fed_loss_cls_weights. Their size should match num_classes" + self.register_buffer("fed_loss_cls_weights", fed_loss_cls_weights) + + @classmethod + def from_config(cls, cfg, input_shape): + return { + "input_shape": input_shape, + "box2box_transform": Box2BoxTransform(weights=cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_WEIGHTS), + # fmt: off + "num_classes" : cfg.MODEL.ROI_HEADS.NUM_CLASSES, + "cls_agnostic_bbox_reg" : cfg.MODEL.ROI_BOX_HEAD.CLS_AGNOSTIC_BBOX_REG, + "smooth_l1_beta" : cfg.MODEL.ROI_BOX_HEAD.SMOOTH_L1_BETA, + "test_score_thresh" : cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST, + "test_nms_thresh" : cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST, + "test_topk_per_image" : cfg.TEST.DETECTIONS_PER_IMAGE, + "box_reg_loss_type" : cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_TYPE, + "loss_weight" : {"loss_box_reg": cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_LOSS_WEIGHT}, # noqa + "use_fed_loss" : cfg.MODEL.ROI_BOX_HEAD.USE_FED_LOSS, + "use_sigmoid_ce" : cfg.MODEL.ROI_BOX_HEAD.USE_SIGMOID_CE, + "get_fed_loss_cls_weights" : lambda: get_fed_loss_cls_weights(dataset_names=cfg.DATASETS.TRAIN, freq_weight_power=cfg.MODEL.ROI_BOX_HEAD.FED_LOSS_FREQ_WEIGHT_POWER), # noqa + "fed_loss_num_classes" : cfg.MODEL.ROI_BOX_HEAD.FED_LOSS_NUM_CLASSES, + # fmt: on + } + + def forward(self, x): + """ + Args: + x: per-region features of shape (N, ...) for N bounding boxes to predict. + + Returns: + (Tensor, Tensor): + First tensor: shape (N,K+1), scores for each of the N box. Each row contains the + scores for K object categories and 1 background class. + + Second tensor: bounding box regression deltas for each box. Shape is shape (N,Kx4), + or (N,4) for class-agnostic regression. + """ + if x.dim() > 2: + x = torch.flatten(x, start_dim=1) + scores = self.cls_score(x) + proposal_deltas = self.bbox_pred(x) + return scores, proposal_deltas + + def losses(self, predictions, proposals): + """ + Args: + predictions: return values of :meth:`forward()`. + proposals (list[Instances]): proposals that match the features that were used + to compute predictions. The fields ``proposal_boxes``, ``gt_boxes``, + ``gt_classes`` are expected. + + Returns: + Dict[str, Tensor]: dict of losses + """ + scores, proposal_deltas = predictions + + # parse classification outputs + gt_classes = ( + cat([p.gt_classes for p in proposals], dim=0) if len(proposals) else torch.empty(0) + ) + _log_classification_stats(scores, gt_classes) + + # parse box regression outputs + if len(proposals): + proposal_boxes = cat([p.proposal_boxes.tensor for p in proposals], dim=0) # Nx4 + assert not proposal_boxes.requires_grad, "Proposals should not require gradients!" + # If "gt_boxes" does not exist, the proposals must be all negative and + # should not be included in regression loss computation. + # Here we just use proposal_boxes as an arbitrary placeholder because its + # value won't be used in self.box_reg_loss(). + gt_boxes = cat( + [(p.gt_boxes if p.has("gt_boxes") else p.proposal_boxes).tensor for p in proposals], + dim=0, + ) + else: + proposal_boxes = gt_boxes = torch.empty((0, 4), device=proposal_deltas.device) + + if self.use_sigmoid_ce: + loss_cls = self.sigmoid_cross_entropy_loss(scores, gt_classes) + else: + loss_cls = cross_entropy(scores, gt_classes, reduction="mean") + + losses = { + "loss_cls": loss_cls, + "loss_box_reg": self.box_reg_loss( + proposal_boxes, gt_boxes, proposal_deltas, gt_classes + ), + } + return {k: v * self.loss_weight.get(k, 1.0) for k, v in losses.items()} + + # Implementation from https://github.com/xingyizhou/CenterNet2/blob/master/projects/CenterNet2/centernet/modeling/roi_heads/fed_loss.py # noqa + # with slight modifications + def get_fed_loss_classes(self, gt_classes, num_fed_loss_classes, num_classes, weight): + """ + Args: + gt_classes: a long tensor of shape R that contains the gt class label of each proposal. + num_fed_loss_classes: minimum number of classes to keep when calculating federated loss. + Will sample negative classes if number of unique gt_classes is smaller than this value. + num_classes: number of foreground classes + weight: probabilities used to sample negative classes + + Returns: + Tensor: + classes to keep when calculating the federated loss, including both unique gt + classes and sampled negative classes. + """ + unique_gt_classes = torch.unique(gt_classes) + prob = unique_gt_classes.new_ones(num_classes + 1).float() + prob[-1] = 0 + if len(unique_gt_classes) < num_fed_loss_classes: + prob[:num_classes] = weight.float().clone() + prob[unique_gt_classes] = 0 + sampled_negative_classes = torch.multinomial( + prob, num_fed_loss_classes - len(unique_gt_classes), replacement=False + ) + fed_loss_classes = torch.cat([unique_gt_classes, sampled_negative_classes]) + else: + fed_loss_classes = unique_gt_classes + return fed_loss_classes + + # Implementation from https://github.com/xingyizhou/CenterNet2/blob/master/projects/CenterNet2/centernet/modeling/roi_heads/custom_fast_rcnn.py#L113 # noqa + # with slight modifications + def sigmoid_cross_entropy_loss(self, pred_class_logits, gt_classes): + """ + Args: + pred_class_logits: shape (N, K+1), scores for each of the N box. Each row contains the + scores for K object categories and 1 background class + gt_classes: a long tensor of shape R that contains the gt class label of each proposal. + """ + if pred_class_logits.numel() == 0: + return pred_class_logits.new_zeros([1])[0] + + N = pred_class_logits.shape[0] + K = pred_class_logits.shape[1] - 1 + + target = pred_class_logits.new_zeros(N, K + 1) + target[range(len(gt_classes)), gt_classes] = 1 + target = target[:, :K] + + cls_loss = F.binary_cross_entropy_with_logits( + pred_class_logits[:, :-1], target, reduction="none" + ) + + if self.use_fed_loss: + fed_loss_classes = self.get_fed_loss_classes( + gt_classes, + num_fed_loss_classes=self.fed_loss_num_classes, + num_classes=K, + weight=self.fed_loss_cls_weights, + ) + fed_loss_classes_mask = fed_loss_classes.new_zeros(K + 1) + fed_loss_classes_mask[fed_loss_classes] = 1 + fed_loss_classes_mask = fed_loss_classes_mask[:K] + weight = fed_loss_classes_mask.view(1, K).expand(N, K).float() + else: + weight = 1 + + loss = torch.sum(cls_loss * weight) / N + return loss + + def box_reg_loss(self, proposal_boxes, gt_boxes, pred_deltas, gt_classes): + """ + Args: + proposal_boxes/gt_boxes are tensors with the same shape (R, 4 or 5). + pred_deltas has shape (R, 4 or 5), or (R, num_classes * (4 or 5)). + gt_classes is a long tensor of shape R, the gt class label of each proposal. + R shall be the number of proposals. + """ + box_dim = proposal_boxes.shape[1] # 4 or 5 + # Regression loss is only computed for foreground proposals (those matched to a GT) + fg_inds = nonzero_tuple((gt_classes >= 0) & (gt_classes < self.num_classes))[0] + if pred_deltas.shape[1] == box_dim: # cls-agnostic regression + fg_pred_deltas = pred_deltas[fg_inds] + else: + fg_pred_deltas = pred_deltas.view(-1, self.num_classes, box_dim)[ + fg_inds, gt_classes[fg_inds] + ] + + loss_box_reg = _dense_box_regression_loss( + [proposal_boxes[fg_inds]], + self.box2box_transform, + [fg_pred_deltas.unsqueeze(0)], + [gt_boxes[fg_inds]], + ..., + self.box_reg_loss_type, + self.smooth_l1_beta, + ) + + # The reg loss is normalized using the total number of regions (R), not the number + # of foreground regions even though the box regression loss is only defined on + # foreground regions. Why? Because doing so gives equal training influence to + # each foreground example. To see how, consider two different minibatches: + # (1) Contains a single foreground region + # (2) Contains 100 foreground regions + # If we normalize by the number of foreground regions, the single example in + # minibatch (1) will be given 100 times as much influence as each foreground + # example in minibatch (2). Normalizing by the total number of regions, R, + # means that the single example in minibatch (1) and each of the 100 examples + # in minibatch (2) are given equal influence. + return loss_box_reg / max(gt_classes.numel(), 1.0) # return 0 if empty + + def inference(self, predictions: Tuple[torch.Tensor, torch.Tensor], proposals: List[Instances]): + """ + Args: + predictions: return values of :meth:`forward()`. + proposals (list[Instances]): proposals that match the features that were + used to compute predictions. The ``proposal_boxes`` field is expected. + + Returns: + list[Instances]: same as `fast_rcnn_inference`. + list[Tensor]: same as `fast_rcnn_inference`. + """ + boxes = self.predict_boxes(predictions, proposals) + scores = self.predict_probs(predictions, proposals) + image_shapes = [x.image_size for x in proposals] + return fast_rcnn_inference( + boxes, + scores, + image_shapes, + self.test_score_thresh, + self.test_nms_thresh, + self.test_topk_per_image, + ) + + def predict_boxes_for_gt_classes(self, predictions, proposals): + """ + Args: + predictions: return values of :meth:`forward()`. + proposals (list[Instances]): proposals that match the features that were used + to compute predictions. The fields ``proposal_boxes``, ``gt_classes`` are expected. + + Returns: + list[Tensor]: + A list of Tensors of predicted boxes for GT classes in case of + class-specific box head. Element i of the list has shape (Ri, B), where Ri is + the number of proposals for image i and B is the box dimension (4 or 5) + """ + if not len(proposals): + return [] + scores, proposal_deltas = predictions + proposal_boxes = cat([p.proposal_boxes.tensor for p in proposals], dim=0) + N, B = proposal_boxes.shape + predict_boxes = self.box2box_transform.apply_deltas( + proposal_deltas, proposal_boxes + ) # Nx(KxB) + + K = predict_boxes.shape[1] // B + if K > 1: + gt_classes = torch.cat([p.gt_classes for p in proposals], dim=0) + # Some proposals are ignored or have a background class. Their gt_classes + # cannot be used as index. + gt_classes = gt_classes.clamp_(0, K - 1) + + predict_boxes = predict_boxes.view(N, K, B)[ + torch.arange(N, dtype=torch.long, device=predict_boxes.device), gt_classes + ] + num_prop_per_image = [len(p) for p in proposals] + return predict_boxes.split(num_prop_per_image) + + def predict_boxes( + self, predictions: Tuple[torch.Tensor, torch.Tensor], proposals: List[Instances] + ): + """ + Args: + predictions: return values of :meth:`forward()`. + proposals (list[Instances]): proposals that match the features that were + used to compute predictions. The ``proposal_boxes`` field is expected. + + Returns: + list[Tensor]: + A list of Tensors of predicted class-specific or class-agnostic boxes + for each image. Element i has shape (Ri, K * B) or (Ri, B), where Ri is + the number of proposals for image i and B is the box dimension (4 or 5) + """ + if not len(proposals): + return [] + _, proposal_deltas = predictions + num_prop_per_image = [len(p) for p in proposals] + proposal_boxes = cat([p.proposal_boxes.tensor for p in proposals], dim=0) + predict_boxes = self.box2box_transform.apply_deltas( + proposal_deltas, + proposal_boxes, + ) # Nx(KxB) + return predict_boxes.split(num_prop_per_image) + + def predict_probs( + self, predictions: Tuple[torch.Tensor, torch.Tensor], proposals: List[Instances] + ): + """ + Args: + predictions: return values of :meth:`forward()`. + proposals (list[Instances]): proposals that match the features that were + used to compute predictions. + + Returns: + list[Tensor]: + A list of Tensors of predicted class probabilities for each image. + Element i has shape (Ri, K + 1), where Ri is the number of proposals for image i. + """ + scores, _ = predictions + num_inst_per_image = [len(p) for p in proposals] + if self.use_sigmoid_ce: + probs = scores.sigmoid() + else: + probs = F.softmax(scores, dim=-1) + return probs.split(num_inst_per_image, dim=0) diff --git a/annotator/oneformer/detectron2/modeling/roi_heads/keypoint_head.py b/annotator/oneformer/detectron2/modeling/roi_heads/keypoint_head.py new file mode 100644 index 0000000000000000000000000000000000000000..bcf92dc8ab553beef98f4b8ddde639ed9d4ff0cc --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/roi_heads/keypoint_head.py @@ -0,0 +1,272 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from typing import List +import torch +from torch import nn +from torch.nn import functional as F + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.layers import Conv2d, ConvTranspose2d, cat, interpolate +from annotator.oneformer.detectron2.structures import Instances, heatmaps_to_keypoints +from annotator.oneformer.detectron2.utils.events import get_event_storage +from annotator.oneformer.detectron2.utils.registry import Registry + +_TOTAL_SKIPPED = 0 + + +__all__ = [ + "ROI_KEYPOINT_HEAD_REGISTRY", + "build_keypoint_head", + "BaseKeypointRCNNHead", + "KRCNNConvDeconvUpsampleHead", +] + + +ROI_KEYPOINT_HEAD_REGISTRY = Registry("ROI_KEYPOINT_HEAD") +ROI_KEYPOINT_HEAD_REGISTRY.__doc__ = """ +Registry for keypoint heads, which make keypoint predictions from per-region features. + +The registered object will be called with `obj(cfg, input_shape)`. +""" + + +def build_keypoint_head(cfg, input_shape): + """ + Build a keypoint head from `cfg.MODEL.ROI_KEYPOINT_HEAD.NAME`. + """ + name = cfg.MODEL.ROI_KEYPOINT_HEAD.NAME + return ROI_KEYPOINT_HEAD_REGISTRY.get(name)(cfg, input_shape) + + +def keypoint_rcnn_loss(pred_keypoint_logits, instances, normalizer): + """ + Arguments: + pred_keypoint_logits (Tensor): A tensor of shape (N, K, S, S) where N is the total number + of instances in the batch, K is the number of keypoints, and S is the side length + of the keypoint heatmap. The values are spatial logits. + instances (list[Instances]): A list of M Instances, where M is the batch size. + These instances are predictions from the model + that are in 1:1 correspondence with pred_keypoint_logits. + Each Instances should contain a `gt_keypoints` field containing a `structures.Keypoint` + instance. + normalizer (float): Normalize the loss by this amount. + If not specified, we normalize by the number of visible keypoints in the minibatch. + + Returns a scalar tensor containing the loss. + """ + heatmaps = [] + valid = [] + + keypoint_side_len = pred_keypoint_logits.shape[2] + for instances_per_image in instances: + if len(instances_per_image) == 0: + continue + keypoints = instances_per_image.gt_keypoints + heatmaps_per_image, valid_per_image = keypoints.to_heatmap( + instances_per_image.proposal_boxes.tensor, keypoint_side_len + ) + heatmaps.append(heatmaps_per_image.view(-1)) + valid.append(valid_per_image.view(-1)) + + if len(heatmaps): + keypoint_targets = cat(heatmaps, dim=0) + valid = cat(valid, dim=0).to(dtype=torch.uint8) + valid = torch.nonzero(valid).squeeze(1) + + # torch.mean (in binary_cross_entropy_with_logits) doesn't + # accept empty tensors, so handle it separately + if len(heatmaps) == 0 or valid.numel() == 0: + global _TOTAL_SKIPPED + _TOTAL_SKIPPED += 1 + storage = get_event_storage() + storage.put_scalar("kpts_num_skipped_batches", _TOTAL_SKIPPED, smoothing_hint=False) + return pred_keypoint_logits.sum() * 0 + + N, K, H, W = pred_keypoint_logits.shape + pred_keypoint_logits = pred_keypoint_logits.view(N * K, H * W) + + keypoint_loss = F.cross_entropy( + pred_keypoint_logits[valid], keypoint_targets[valid], reduction="sum" + ) + + # If a normalizer isn't specified, normalize by the number of visible keypoints in the minibatch + if normalizer is None: + normalizer = valid.numel() + keypoint_loss /= normalizer + + return keypoint_loss + + +def keypoint_rcnn_inference(pred_keypoint_logits: torch.Tensor, pred_instances: List[Instances]): + """ + Post process each predicted keypoint heatmap in `pred_keypoint_logits` into (x, y, score) + and add it to the `pred_instances` as a `pred_keypoints` field. + + Args: + pred_keypoint_logits (Tensor): A tensor of shape (R, K, S, S) where R is the total number + of instances in the batch, K is the number of keypoints, and S is the side length of + the keypoint heatmap. The values are spatial logits. + pred_instances (list[Instances]): A list of N Instances, where N is the number of images. + + Returns: + None. Each element in pred_instances will contain extra "pred_keypoints" and + "pred_keypoint_heatmaps" fields. "pred_keypoints" is a tensor of shape + (#instance, K, 3) where the last dimension corresponds to (x, y, score). + The scores are larger than 0. "pred_keypoint_heatmaps" contains the raw + keypoint logits as passed to this function. + """ + # flatten all bboxes from all images together (list[Boxes] -> Rx4 tensor) + bboxes_flat = cat([b.pred_boxes.tensor for b in pred_instances], dim=0) + + pred_keypoint_logits = pred_keypoint_logits.detach() + keypoint_results = heatmaps_to_keypoints(pred_keypoint_logits, bboxes_flat.detach()) + num_instances_per_image = [len(i) for i in pred_instances] + keypoint_results = keypoint_results[:, :, [0, 1, 3]].split(num_instances_per_image, dim=0) + heatmap_results = pred_keypoint_logits.split(num_instances_per_image, dim=0) + + for keypoint_results_per_image, heatmap_results_per_image, instances_per_image in zip( + keypoint_results, heatmap_results, pred_instances + ): + # keypoint_results_per_image is (num instances)x(num keypoints)x(x, y, score) + # heatmap_results_per_image is (num instances)x(num keypoints)x(side)x(side) + instances_per_image.pred_keypoints = keypoint_results_per_image + instances_per_image.pred_keypoint_heatmaps = heatmap_results_per_image + + +class BaseKeypointRCNNHead(nn.Module): + """ + Implement the basic Keypoint R-CNN losses and inference logic described in + Sec. 5 of :paper:`Mask R-CNN`. + """ + + @configurable + def __init__(self, *, num_keypoints, loss_weight=1.0, loss_normalizer=1.0): + """ + NOTE: this interface is experimental. + + Args: + num_keypoints (int): number of keypoints to predict + loss_weight (float): weight to multiple on the keypoint loss + loss_normalizer (float or str): + If float, divide the loss by `loss_normalizer * #images`. + If 'visible', the loss is normalized by the total number of + visible keypoints across images. + """ + super().__init__() + self.num_keypoints = num_keypoints + self.loss_weight = loss_weight + assert loss_normalizer == "visible" or isinstance(loss_normalizer, float), loss_normalizer + self.loss_normalizer = loss_normalizer + + @classmethod + def from_config(cls, cfg, input_shape): + ret = { + "loss_weight": cfg.MODEL.ROI_KEYPOINT_HEAD.LOSS_WEIGHT, + "num_keypoints": cfg.MODEL.ROI_KEYPOINT_HEAD.NUM_KEYPOINTS, + } + normalize_by_visible = ( + cfg.MODEL.ROI_KEYPOINT_HEAD.NORMALIZE_LOSS_BY_VISIBLE_KEYPOINTS + ) # noqa + if not normalize_by_visible: + batch_size_per_image = cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE + positive_sample_fraction = cfg.MODEL.ROI_HEADS.POSITIVE_FRACTION + ret["loss_normalizer"] = ( + ret["num_keypoints"] * batch_size_per_image * positive_sample_fraction + ) + else: + ret["loss_normalizer"] = "visible" + return ret + + def forward(self, x, instances: List[Instances]): + """ + Args: + x: input 4D region feature(s) provided by :class:`ROIHeads`. + instances (list[Instances]): contains the boxes & labels corresponding + to the input features. + Exact format is up to its caller to decide. + Typically, this is the foreground instances in training, with + "proposal_boxes" field and other gt annotations. + In inference, it contains boxes that are already predicted. + + Returns: + A dict of losses if in training. The predicted "instances" if in inference. + """ + x = self.layers(x) + if self.training: + num_images = len(instances) + normalizer = ( + None if self.loss_normalizer == "visible" else num_images * self.loss_normalizer + ) + return { + "loss_keypoint": keypoint_rcnn_loss(x, instances, normalizer=normalizer) + * self.loss_weight + } + else: + keypoint_rcnn_inference(x, instances) + return instances + + def layers(self, x): + """ + Neural network layers that makes predictions from regional input features. + """ + raise NotImplementedError + + +# To get torchscript support, we make the head a subclass of `nn.Sequential`. +# Therefore, to add new layers in this head class, please make sure they are +# added in the order they will be used in forward(). +@ROI_KEYPOINT_HEAD_REGISTRY.register() +class KRCNNConvDeconvUpsampleHead(BaseKeypointRCNNHead, nn.Sequential): + """ + A standard keypoint head containing a series of 3x3 convs, followed by + a transpose convolution and bilinear interpolation for upsampling. + It is described in Sec. 5 of :paper:`Mask R-CNN`. + """ + + @configurable + def __init__(self, input_shape, *, num_keypoints, conv_dims, **kwargs): + """ + NOTE: this interface is experimental. + + Args: + input_shape (ShapeSpec): shape of the input feature + conv_dims: an iterable of output channel counts for each conv in the head + e.g. (512, 512, 512) for three convs outputting 512 channels. + """ + super().__init__(num_keypoints=num_keypoints, **kwargs) + + # default up_scale to 2.0 (this can be made an option) + up_scale = 2.0 + in_channels = input_shape.channels + + for idx, layer_channels in enumerate(conv_dims, 1): + module = Conv2d(in_channels, layer_channels, 3, stride=1, padding=1) + self.add_module("conv_fcn{}".format(idx), module) + self.add_module("conv_fcn_relu{}".format(idx), nn.ReLU()) + in_channels = layer_channels + + deconv_kernel = 4 + self.score_lowres = ConvTranspose2d( + in_channels, num_keypoints, deconv_kernel, stride=2, padding=deconv_kernel // 2 - 1 + ) + self.up_scale = up_scale + + for name, param in self.named_parameters(): + if "bias" in name: + nn.init.constant_(param, 0) + elif "weight" in name: + # Caffe2 implementation uses MSRAFill, which in fact + # corresponds to kaiming_normal_ in PyTorch + nn.init.kaiming_normal_(param, mode="fan_out", nonlinearity="relu") + + @classmethod + def from_config(cls, cfg, input_shape): + ret = super().from_config(cfg, input_shape) + ret["input_shape"] = input_shape + ret["conv_dims"] = cfg.MODEL.ROI_KEYPOINT_HEAD.CONV_DIMS + return ret + + def layers(self, x): + for layer in self: + x = layer(x) + x = interpolate(x, scale_factor=self.up_scale, mode="bilinear", align_corners=False) + return x diff --git a/annotator/oneformer/detectron2/modeling/roi_heads/mask_head.py b/annotator/oneformer/detectron2/modeling/roi_heads/mask_head.py new file mode 100644 index 0000000000000000000000000000000000000000..1b5465e413195aa21733157af4e1ae3a2b897e7c --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/roi_heads/mask_head.py @@ -0,0 +1,298 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from typing import List +import fvcore.nn.weight_init as weight_init +import torch +from torch import nn +from torch.nn import functional as F + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.layers import Conv2d, ConvTranspose2d, ShapeSpec, cat, get_norm +from annotator.oneformer.detectron2.layers.wrappers import move_device_like +from annotator.oneformer.detectron2.structures import Instances +from annotator.oneformer.detectron2.utils.events import get_event_storage +from annotator.oneformer.detectron2.utils.registry import Registry + +__all__ = [ + "BaseMaskRCNNHead", + "MaskRCNNConvUpsampleHead", + "build_mask_head", + "ROI_MASK_HEAD_REGISTRY", +] + + +ROI_MASK_HEAD_REGISTRY = Registry("ROI_MASK_HEAD") +ROI_MASK_HEAD_REGISTRY.__doc__ = """ +Registry for mask heads, which predicts instance masks given +per-region features. + +The registered object will be called with `obj(cfg, input_shape)`. +""" + + +@torch.jit.unused +def mask_rcnn_loss(pred_mask_logits: torch.Tensor, instances: List[Instances], vis_period: int = 0): + """ + Compute the mask prediction loss defined in the Mask R-CNN paper. + + Args: + pred_mask_logits (Tensor): A tensor of shape (B, C, Hmask, Wmask) or (B, 1, Hmask, Wmask) + for class-specific or class-agnostic, where B is the total number of predicted masks + in all images, C is the number of foreground classes, and Hmask, Wmask are the height + and width of the mask predictions. The values are logits. + instances (list[Instances]): A list of N Instances, where N is the number of images + in the batch. These instances are in 1:1 + correspondence with the pred_mask_logits. The ground-truth labels (class, box, mask, + ...) associated with each instance are stored in fields. + vis_period (int): the period (in steps) to dump visualization. + + Returns: + mask_loss (Tensor): A scalar tensor containing the loss. + """ + cls_agnostic_mask = pred_mask_logits.size(1) == 1 + total_num_masks = pred_mask_logits.size(0) + mask_side_len = pred_mask_logits.size(2) + assert pred_mask_logits.size(2) == pred_mask_logits.size(3), "Mask prediction must be square!" + + gt_classes = [] + gt_masks = [] + for instances_per_image in instances: + if len(instances_per_image) == 0: + continue + if not cls_agnostic_mask: + gt_classes_per_image = instances_per_image.gt_classes.to(dtype=torch.int64) + gt_classes.append(gt_classes_per_image) + + gt_masks_per_image = instances_per_image.gt_masks.crop_and_resize( + instances_per_image.proposal_boxes.tensor, mask_side_len + ).to(device=pred_mask_logits.device) + # A tensor of shape (N, M, M), N=#instances in the image; M=mask_side_len + gt_masks.append(gt_masks_per_image) + + if len(gt_masks) == 0: + return pred_mask_logits.sum() * 0 + + gt_masks = cat(gt_masks, dim=0) + + if cls_agnostic_mask: + pred_mask_logits = pred_mask_logits[:, 0] + else: + indices = torch.arange(total_num_masks) + gt_classes = cat(gt_classes, dim=0) + pred_mask_logits = pred_mask_logits[indices, gt_classes] + + if gt_masks.dtype == torch.bool: + gt_masks_bool = gt_masks + else: + # Here we allow gt_masks to be float as well (depend on the implementation of rasterize()) + gt_masks_bool = gt_masks > 0.5 + gt_masks = gt_masks.to(dtype=torch.float32) + + # Log the training accuracy (using gt classes and 0.5 threshold) + mask_incorrect = (pred_mask_logits > 0.0) != gt_masks_bool + mask_accuracy = 1 - (mask_incorrect.sum().item() / max(mask_incorrect.numel(), 1.0)) + num_positive = gt_masks_bool.sum().item() + false_positive = (mask_incorrect & ~gt_masks_bool).sum().item() / max( + gt_masks_bool.numel() - num_positive, 1.0 + ) + false_negative = (mask_incorrect & gt_masks_bool).sum().item() / max(num_positive, 1.0) + + storage = get_event_storage() + storage.put_scalar("mask_rcnn/accuracy", mask_accuracy) + storage.put_scalar("mask_rcnn/false_positive", false_positive) + storage.put_scalar("mask_rcnn/false_negative", false_negative) + if vis_period > 0 and storage.iter % vis_period == 0: + pred_masks = pred_mask_logits.sigmoid() + vis_masks = torch.cat([pred_masks, gt_masks], axis=2) + name = "Left: mask prediction; Right: mask GT" + for idx, vis_mask in enumerate(vis_masks): + vis_mask = torch.stack([vis_mask] * 3, axis=0) + storage.put_image(name + f" ({idx})", vis_mask) + + mask_loss = F.binary_cross_entropy_with_logits(pred_mask_logits, gt_masks, reduction="mean") + return mask_loss + + +def mask_rcnn_inference(pred_mask_logits: torch.Tensor, pred_instances: List[Instances]): + """ + Convert pred_mask_logits to estimated foreground probability masks while also + extracting only the masks for the predicted classes in pred_instances. For each + predicted box, the mask of the same class is attached to the instance by adding a + new "pred_masks" field to pred_instances. + + Args: + pred_mask_logits (Tensor): A tensor of shape (B, C, Hmask, Wmask) or (B, 1, Hmask, Wmask) + for class-specific or class-agnostic, where B is the total number of predicted masks + in all images, C is the number of foreground classes, and Hmask, Wmask are the height + and width of the mask predictions. The values are logits. + pred_instances (list[Instances]): A list of N Instances, where N is the number of images + in the batch. Each Instances must have field "pred_classes". + + Returns: + None. pred_instances will contain an extra "pred_masks" field storing a mask of size (Hmask, + Wmask) for predicted class. Note that the masks are returned as a soft (non-quantized) + masks the resolution predicted by the network; post-processing steps, such as resizing + the predicted masks to the original image resolution and/or binarizing them, is left + to the caller. + """ + cls_agnostic_mask = pred_mask_logits.size(1) == 1 + + if cls_agnostic_mask: + mask_probs_pred = pred_mask_logits.sigmoid() + else: + # Select masks corresponding to the predicted classes + num_masks = pred_mask_logits.shape[0] + class_pred = cat([i.pred_classes for i in pred_instances]) + device = ( + class_pred.device + if torch.jit.is_scripting() + else ("cpu" if torch.jit.is_tracing() else class_pred.device) + ) + indices = move_device_like(torch.arange(num_masks, device=device), class_pred) + mask_probs_pred = pred_mask_logits[indices, class_pred][:, None].sigmoid() + # mask_probs_pred.shape: (B, 1, Hmask, Wmask) + + num_boxes_per_image = [len(i) for i in pred_instances] + mask_probs_pred = mask_probs_pred.split(num_boxes_per_image, dim=0) + + for prob, instances in zip(mask_probs_pred, pred_instances): + instances.pred_masks = prob # (1, Hmask, Wmask) + + +class BaseMaskRCNNHead(nn.Module): + """ + Implement the basic Mask R-CNN losses and inference logic described in :paper:`Mask R-CNN` + """ + + @configurable + def __init__(self, *, loss_weight: float = 1.0, vis_period: int = 0): + """ + NOTE: this interface is experimental. + + Args: + loss_weight (float): multiplier of the loss + vis_period (int): visualization period + """ + super().__init__() + self.vis_period = vis_period + self.loss_weight = loss_weight + + @classmethod + def from_config(cls, cfg, input_shape): + return {"vis_period": cfg.VIS_PERIOD} + + def forward(self, x, instances: List[Instances]): + """ + Args: + x: input region feature(s) provided by :class:`ROIHeads`. + instances (list[Instances]): contains the boxes & labels corresponding + to the input features. + Exact format is up to its caller to decide. + Typically, this is the foreground instances in training, with + "proposal_boxes" field and other gt annotations. + In inference, it contains boxes that are already predicted. + + Returns: + A dict of losses in training. The predicted "instances" in inference. + """ + x = self.layers(x) + if self.training: + return {"loss_mask": mask_rcnn_loss(x, instances, self.vis_period) * self.loss_weight} + else: + mask_rcnn_inference(x, instances) + return instances + + def layers(self, x): + """ + Neural network layers that makes predictions from input features. + """ + raise NotImplementedError + + +# To get torchscript support, we make the head a subclass of `nn.Sequential`. +# Therefore, to add new layers in this head class, please make sure they are +# added in the order they will be used in forward(). +@ROI_MASK_HEAD_REGISTRY.register() +class MaskRCNNConvUpsampleHead(BaseMaskRCNNHead, nn.Sequential): + """ + A mask head with several conv layers, plus an upsample layer (with `ConvTranspose2d`). + Predictions are made with a final 1x1 conv layer. + """ + + @configurable + def __init__(self, input_shape: ShapeSpec, *, num_classes, conv_dims, conv_norm="", **kwargs): + """ + NOTE: this interface is experimental. + + Args: + input_shape (ShapeSpec): shape of the input feature + num_classes (int): the number of foreground classes (i.e. background is not + included). 1 if using class agnostic prediction. + conv_dims (list[int]): a list of N>0 integers representing the output dimensions + of N-1 conv layers and the last upsample layer. + conv_norm (str or callable): normalization for the conv layers. + See :func:`detectron2.layers.get_norm` for supported types. + """ + super().__init__(**kwargs) + assert len(conv_dims) >= 1, "conv_dims have to be non-empty!" + + self.conv_norm_relus = [] + + cur_channels = input_shape.channels + for k, conv_dim in enumerate(conv_dims[:-1]): + conv = Conv2d( + cur_channels, + conv_dim, + kernel_size=3, + stride=1, + padding=1, + bias=not conv_norm, + norm=get_norm(conv_norm, conv_dim), + activation=nn.ReLU(), + ) + self.add_module("mask_fcn{}".format(k + 1), conv) + self.conv_norm_relus.append(conv) + cur_channels = conv_dim + + self.deconv = ConvTranspose2d( + cur_channels, conv_dims[-1], kernel_size=2, stride=2, padding=0 + ) + self.add_module("deconv_relu", nn.ReLU()) + cur_channels = conv_dims[-1] + + self.predictor = Conv2d(cur_channels, num_classes, kernel_size=1, stride=1, padding=0) + + for layer in self.conv_norm_relus + [self.deconv]: + weight_init.c2_msra_fill(layer) + # use normal distribution initialization for mask prediction layer + nn.init.normal_(self.predictor.weight, std=0.001) + if self.predictor.bias is not None: + nn.init.constant_(self.predictor.bias, 0) + + @classmethod + def from_config(cls, cfg, input_shape): + ret = super().from_config(cfg, input_shape) + conv_dim = cfg.MODEL.ROI_MASK_HEAD.CONV_DIM + num_conv = cfg.MODEL.ROI_MASK_HEAD.NUM_CONV + ret.update( + conv_dims=[conv_dim] * (num_conv + 1), # +1 for ConvTranspose + conv_norm=cfg.MODEL.ROI_MASK_HEAD.NORM, + input_shape=input_shape, + ) + if cfg.MODEL.ROI_MASK_HEAD.CLS_AGNOSTIC_MASK: + ret["num_classes"] = 1 + else: + ret["num_classes"] = cfg.MODEL.ROI_HEADS.NUM_CLASSES + return ret + + def layers(self, x): + for layer in self: + x = layer(x) + return x + + +def build_mask_head(cfg, input_shape): + """ + Build a mask head defined by `cfg.MODEL.ROI_MASK_HEAD.NAME`. + """ + name = cfg.MODEL.ROI_MASK_HEAD.NAME + return ROI_MASK_HEAD_REGISTRY.get(name)(cfg, input_shape) diff --git a/annotator/oneformer/detectron2/modeling/roi_heads/roi_heads.py b/annotator/oneformer/detectron2/modeling/roi_heads/roi_heads.py new file mode 100644 index 0000000000000000000000000000000000000000..d554a3878e7d9fa49971128ad260c3e831b70c65 --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/roi_heads/roi_heads.py @@ -0,0 +1,877 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import inspect +import logging +import numpy as np +from typing import Dict, List, Optional, Tuple +import torch +from torch import nn + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.layers import ShapeSpec, nonzero_tuple +from annotator.oneformer.detectron2.structures import Boxes, ImageList, Instances, pairwise_iou +from annotator.oneformer.detectron2.utils.events import get_event_storage +from annotator.oneformer.detectron2.utils.registry import Registry + +from ..backbone.resnet import BottleneckBlock, ResNet +from ..matcher import Matcher +from ..poolers import ROIPooler +from ..proposal_generator.proposal_utils import add_ground_truth_to_proposals +from ..sampling import subsample_labels +from .box_head import build_box_head +from .fast_rcnn import FastRCNNOutputLayers +from .keypoint_head import build_keypoint_head +from .mask_head import build_mask_head + +ROI_HEADS_REGISTRY = Registry("ROI_HEADS") +ROI_HEADS_REGISTRY.__doc__ = """ +Registry for ROI heads in a generalized R-CNN model. +ROIHeads take feature maps and region proposals, and +perform per-region computation. + +The registered object will be called with `obj(cfg, input_shape)`. +The call is expected to return an :class:`ROIHeads`. +""" + +logger = logging.getLogger(__name__) + + +def build_roi_heads(cfg, input_shape): + """ + Build ROIHeads defined by `cfg.MODEL.ROI_HEADS.NAME`. + """ + name = cfg.MODEL.ROI_HEADS.NAME + return ROI_HEADS_REGISTRY.get(name)(cfg, input_shape) + + +def select_foreground_proposals( + proposals: List[Instances], bg_label: int +) -> Tuple[List[Instances], List[torch.Tensor]]: + """ + Given a list of N Instances (for N images), each containing a `gt_classes` field, + return a list of Instances that contain only instances with `gt_classes != -1 && + gt_classes != bg_label`. + + Args: + proposals (list[Instances]): A list of N Instances, where N is the number of + images in the batch. + bg_label: label index of background class. + + Returns: + list[Instances]: N Instances, each contains only the selected foreground instances. + list[Tensor]: N boolean vector, correspond to the selection mask of + each Instances object. True for selected instances. + """ + assert isinstance(proposals, (list, tuple)) + assert isinstance(proposals[0], Instances) + assert proposals[0].has("gt_classes") + fg_proposals = [] + fg_selection_masks = [] + for proposals_per_image in proposals: + gt_classes = proposals_per_image.gt_classes + fg_selection_mask = (gt_classes != -1) & (gt_classes != bg_label) + fg_idxs = fg_selection_mask.nonzero().squeeze(1) + fg_proposals.append(proposals_per_image[fg_idxs]) + fg_selection_masks.append(fg_selection_mask) + return fg_proposals, fg_selection_masks + + +def select_proposals_with_visible_keypoints(proposals: List[Instances]) -> List[Instances]: + """ + Args: + proposals (list[Instances]): a list of N Instances, where N is the + number of images. + + Returns: + proposals: only contains proposals with at least one visible keypoint. + + Note that this is still slightly different from Detectron. + In Detectron, proposals for training keypoint head are re-sampled from + all the proposals with IOU>threshold & >=1 visible keypoint. + + Here, the proposals are first sampled from all proposals with + IOU>threshold, then proposals with no visible keypoint are filtered out. + This strategy seems to make no difference on Detectron and is easier to implement. + """ + ret = [] + all_num_fg = [] + for proposals_per_image in proposals: + # If empty/unannotated image (hard negatives), skip filtering for train + if len(proposals_per_image) == 0: + ret.append(proposals_per_image) + continue + gt_keypoints = proposals_per_image.gt_keypoints.tensor + # #fg x K x 3 + vis_mask = gt_keypoints[:, :, 2] >= 1 + xs, ys = gt_keypoints[:, :, 0], gt_keypoints[:, :, 1] + proposal_boxes = proposals_per_image.proposal_boxes.tensor.unsqueeze(dim=1) # #fg x 1 x 4 + kp_in_box = ( + (xs >= proposal_boxes[:, :, 0]) + & (xs <= proposal_boxes[:, :, 2]) + & (ys >= proposal_boxes[:, :, 1]) + & (ys <= proposal_boxes[:, :, 3]) + ) + selection = (kp_in_box & vis_mask).any(dim=1) + selection_idxs = nonzero_tuple(selection)[0] + all_num_fg.append(selection_idxs.numel()) + ret.append(proposals_per_image[selection_idxs]) + + storage = get_event_storage() + storage.put_scalar("keypoint_head/num_fg_samples", np.mean(all_num_fg)) + return ret + + +class ROIHeads(torch.nn.Module): + """ + ROIHeads perform all per-region computation in an R-CNN. + + It typically contains logic to + + 1. (in training only) match proposals with ground truth and sample them + 2. crop the regions and extract per-region features using proposals + 3. make per-region predictions with different heads + + It can have many variants, implemented as subclasses of this class. + This base class contains the logic to match/sample proposals. + But it is not necessary to inherit this class if the sampling logic is not needed. + """ + + @configurable + def __init__( + self, + *, + num_classes, + batch_size_per_image, + positive_fraction, + proposal_matcher, + proposal_append_gt=True, + ): + """ + NOTE: this interface is experimental. + + Args: + num_classes (int): number of foreground classes (i.e. background is not included) + batch_size_per_image (int): number of proposals to sample for training + positive_fraction (float): fraction of positive (foreground) proposals + to sample for training. + proposal_matcher (Matcher): matcher that matches proposals and ground truth + proposal_append_gt (bool): whether to include ground truth as proposals as well + """ + super().__init__() + self.batch_size_per_image = batch_size_per_image + self.positive_fraction = positive_fraction + self.num_classes = num_classes + self.proposal_matcher = proposal_matcher + self.proposal_append_gt = proposal_append_gt + + @classmethod + def from_config(cls, cfg): + return { + "batch_size_per_image": cfg.MODEL.ROI_HEADS.BATCH_SIZE_PER_IMAGE, + "positive_fraction": cfg.MODEL.ROI_HEADS.POSITIVE_FRACTION, + "num_classes": cfg.MODEL.ROI_HEADS.NUM_CLASSES, + "proposal_append_gt": cfg.MODEL.ROI_HEADS.PROPOSAL_APPEND_GT, + # Matcher to assign box proposals to gt boxes + "proposal_matcher": Matcher( + cfg.MODEL.ROI_HEADS.IOU_THRESHOLDS, + cfg.MODEL.ROI_HEADS.IOU_LABELS, + allow_low_quality_matches=False, + ), + } + + def _sample_proposals( + self, matched_idxs: torch.Tensor, matched_labels: torch.Tensor, gt_classes: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Based on the matching between N proposals and M groundtruth, + sample the proposals and set their classification labels. + + Args: + matched_idxs (Tensor): a vector of length N, each is the best-matched + gt index in [0, M) for each proposal. + matched_labels (Tensor): a vector of length N, the matcher's label + (one of cfg.MODEL.ROI_HEADS.IOU_LABELS) for each proposal. + gt_classes (Tensor): a vector of length M. + + Returns: + Tensor: a vector of indices of sampled proposals. Each is in [0, N). + Tensor: a vector of the same length, the classification label for + each sampled proposal. Each sample is labeled as either a category in + [0, num_classes) or the background (num_classes). + """ + has_gt = gt_classes.numel() > 0 + # Get the corresponding GT for each proposal + if has_gt: + gt_classes = gt_classes[matched_idxs] + # Label unmatched proposals (0 label from matcher) as background (label=num_classes) + gt_classes[matched_labels == 0] = self.num_classes + # Label ignore proposals (-1 label) + gt_classes[matched_labels == -1] = -1 + else: + gt_classes = torch.zeros_like(matched_idxs) + self.num_classes + + sampled_fg_idxs, sampled_bg_idxs = subsample_labels( + gt_classes, self.batch_size_per_image, self.positive_fraction, self.num_classes + ) + + sampled_idxs = torch.cat([sampled_fg_idxs, sampled_bg_idxs], dim=0) + return sampled_idxs, gt_classes[sampled_idxs] + + @torch.no_grad() + def label_and_sample_proposals( + self, proposals: List[Instances], targets: List[Instances] + ) -> List[Instances]: + """ + Prepare some proposals to be used to train the ROI heads. + It performs box matching between `proposals` and `targets`, and assigns + training labels to the proposals. + It returns ``self.batch_size_per_image`` random samples from proposals and groundtruth + boxes, with a fraction of positives that is no larger than + ``self.positive_fraction``. + + Args: + See :meth:`ROIHeads.forward` + + Returns: + list[Instances]: + length `N` list of `Instances`s containing the proposals + sampled for training. Each `Instances` has the following fields: + + - proposal_boxes: the proposal boxes + - gt_boxes: the ground-truth box that the proposal is assigned to + (this is only meaningful if the proposal has a label > 0; if label = 0 + then the ground-truth box is random) + + Other fields such as "gt_classes", "gt_masks", that's included in `targets`. + """ + # Augment proposals with ground-truth boxes. + # In the case of learned proposals (e.g., RPN), when training starts + # the proposals will be low quality due to random initialization. + # It's possible that none of these initial + # proposals have high enough overlap with the gt objects to be used + # as positive examples for the second stage components (box head, + # cls head, mask head). Adding the gt boxes to the set of proposals + # ensures that the second stage components will have some positive + # examples from the start of training. For RPN, this augmentation improves + # convergence and empirically improves box AP on COCO by about 0.5 + # points (under one tested configuration). + if self.proposal_append_gt: + proposals = add_ground_truth_to_proposals(targets, proposals) + + proposals_with_gt = [] + + num_fg_samples = [] + num_bg_samples = [] + for proposals_per_image, targets_per_image in zip(proposals, targets): + has_gt = len(targets_per_image) > 0 + match_quality_matrix = pairwise_iou( + targets_per_image.gt_boxes, proposals_per_image.proposal_boxes + ) + matched_idxs, matched_labels = self.proposal_matcher(match_quality_matrix) + sampled_idxs, gt_classes = self._sample_proposals( + matched_idxs, matched_labels, targets_per_image.gt_classes + ) + + # Set target attributes of the sampled proposals: + proposals_per_image = proposals_per_image[sampled_idxs] + proposals_per_image.gt_classes = gt_classes + + if has_gt: + sampled_targets = matched_idxs[sampled_idxs] + # We index all the attributes of targets that start with "gt_" + # and have not been added to proposals yet (="gt_classes"). + # NOTE: here the indexing waste some compute, because heads + # like masks, keypoints, etc, will filter the proposals again, + # (by foreground/background, or number of keypoints in the image, etc) + # so we essentially index the data twice. + for (trg_name, trg_value) in targets_per_image.get_fields().items(): + if trg_name.startswith("gt_") and not proposals_per_image.has(trg_name): + proposals_per_image.set(trg_name, trg_value[sampled_targets]) + # If no GT is given in the image, we don't know what a dummy gt value can be. + # Therefore the returned proposals won't have any gt_* fields, except for a + # gt_classes full of background label. + + num_bg_samples.append((gt_classes == self.num_classes).sum().item()) + num_fg_samples.append(gt_classes.numel() - num_bg_samples[-1]) + proposals_with_gt.append(proposals_per_image) + + # Log the number of fg/bg samples that are selected for training ROI heads + storage = get_event_storage() + storage.put_scalar("roi_head/num_fg_samples", np.mean(num_fg_samples)) + storage.put_scalar("roi_head/num_bg_samples", np.mean(num_bg_samples)) + + return proposals_with_gt + + def forward( + self, + images: ImageList, + features: Dict[str, torch.Tensor], + proposals: List[Instances], + targets: Optional[List[Instances]] = None, + ) -> Tuple[List[Instances], Dict[str, torch.Tensor]]: + """ + Args: + images (ImageList): + features (dict[str,Tensor]): input data as a mapping from feature + map name to tensor. Axis 0 represents the number of images `N` in + the input data; axes 1-3 are channels, height, and width, which may + vary between feature maps (e.g., if a feature pyramid is used). + proposals (list[Instances]): length `N` list of `Instances`. The i-th + `Instances` contains object proposals for the i-th input image, + with fields "proposal_boxes" and "objectness_logits". + targets (list[Instances], optional): length `N` list of `Instances`. The i-th + `Instances` contains the ground-truth per-instance annotations + for the i-th input image. Specify `targets` during training only. + It may have the following fields: + + - gt_boxes: the bounding box of each instance. + - gt_classes: the label for each instance with a category ranging in [0, #class]. + - gt_masks: PolygonMasks or BitMasks, the ground-truth masks of each instance. + - gt_keypoints: NxKx3, the groud-truth keypoints for each instance. + + Returns: + list[Instances]: length `N` list of `Instances` containing the + detected instances. Returned during inference only; may be [] during training. + + dict[str->Tensor]: + mapping from a named loss to a tensor storing the loss. Used during training only. + """ + raise NotImplementedError() + + +@ROI_HEADS_REGISTRY.register() +class Res5ROIHeads(ROIHeads): + """ + The ROIHeads in a typical "C4" R-CNN model, where + the box and mask head share the cropping and + the per-region feature computation by a Res5 block. + See :paper:`ResNet` Appendix A. + """ + + @configurable + def __init__( + self, + *, + in_features: List[str], + pooler: ROIPooler, + res5: nn.Module, + box_predictor: nn.Module, + mask_head: Optional[nn.Module] = None, + **kwargs, + ): + """ + NOTE: this interface is experimental. + + Args: + in_features (list[str]): list of backbone feature map names to use for + feature extraction + pooler (ROIPooler): pooler to extra region features from backbone + res5 (nn.Sequential): a CNN to compute per-region features, to be used by + ``box_predictor`` and ``mask_head``. Typically this is a "res5" + block from a ResNet. + box_predictor (nn.Module): make box predictions from the feature. + Should have the same interface as :class:`FastRCNNOutputLayers`. + mask_head (nn.Module): transform features to make mask predictions + """ + super().__init__(**kwargs) + self.in_features = in_features + self.pooler = pooler + if isinstance(res5, (list, tuple)): + res5 = nn.Sequential(*res5) + self.res5 = res5 + self.box_predictor = box_predictor + self.mask_on = mask_head is not None + if self.mask_on: + self.mask_head = mask_head + + @classmethod + def from_config(cls, cfg, input_shape): + # fmt: off + ret = super().from_config(cfg) + in_features = ret["in_features"] = cfg.MODEL.ROI_HEADS.IN_FEATURES + pooler_resolution = cfg.MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION + pooler_type = cfg.MODEL.ROI_BOX_HEAD.POOLER_TYPE + pooler_scales = (1.0 / input_shape[in_features[0]].stride, ) + sampling_ratio = cfg.MODEL.ROI_BOX_HEAD.POOLER_SAMPLING_RATIO + mask_on = cfg.MODEL.MASK_ON + # fmt: on + assert not cfg.MODEL.KEYPOINT_ON + assert len(in_features) == 1 + + ret["pooler"] = ROIPooler( + output_size=pooler_resolution, + scales=pooler_scales, + sampling_ratio=sampling_ratio, + pooler_type=pooler_type, + ) + + # Compatbility with old moco code. Might be useful. + # See notes in StandardROIHeads.from_config + if not inspect.ismethod(cls._build_res5_block): + logger.warning( + "The behavior of _build_res5_block may change. " + "Please do not depend on private methods." + ) + cls._build_res5_block = classmethod(cls._build_res5_block) + + ret["res5"], out_channels = cls._build_res5_block(cfg) + ret["box_predictor"] = FastRCNNOutputLayers( + cfg, ShapeSpec(channels=out_channels, height=1, width=1) + ) + + if mask_on: + ret["mask_head"] = build_mask_head( + cfg, + ShapeSpec(channels=out_channels, width=pooler_resolution, height=pooler_resolution), + ) + return ret + + @classmethod + def _build_res5_block(cls, cfg): + # fmt: off + stage_channel_factor = 2 ** 3 # res5 is 8x res2 + num_groups = cfg.MODEL.RESNETS.NUM_GROUPS + width_per_group = cfg.MODEL.RESNETS.WIDTH_PER_GROUP + bottleneck_channels = num_groups * width_per_group * stage_channel_factor + out_channels = cfg.MODEL.RESNETS.RES2_OUT_CHANNELS * stage_channel_factor + stride_in_1x1 = cfg.MODEL.RESNETS.STRIDE_IN_1X1 + norm = cfg.MODEL.RESNETS.NORM + assert not cfg.MODEL.RESNETS.DEFORM_ON_PER_STAGE[-1], \ + "Deformable conv is not yet supported in res5 head." + # fmt: on + + blocks = ResNet.make_stage( + BottleneckBlock, + 3, + stride_per_block=[2, 1, 1], + in_channels=out_channels // 2, + bottleneck_channels=bottleneck_channels, + out_channels=out_channels, + num_groups=num_groups, + norm=norm, + stride_in_1x1=stride_in_1x1, + ) + return nn.Sequential(*blocks), out_channels + + def _shared_roi_transform(self, features: List[torch.Tensor], boxes: List[Boxes]): + x = self.pooler(features, boxes) + return self.res5(x) + + def forward( + self, + images: ImageList, + features: Dict[str, torch.Tensor], + proposals: List[Instances], + targets: Optional[List[Instances]] = None, + ): + """ + See :meth:`ROIHeads.forward`. + """ + del images + + if self.training: + assert targets + proposals = self.label_and_sample_proposals(proposals, targets) + del targets + + proposal_boxes = [x.proposal_boxes for x in proposals] + box_features = self._shared_roi_transform( + [features[f] for f in self.in_features], proposal_boxes + ) + predictions = self.box_predictor(box_features.mean(dim=[2, 3])) + + if self.training: + del features + losses = self.box_predictor.losses(predictions, proposals) + if self.mask_on: + proposals, fg_selection_masks = select_foreground_proposals( + proposals, self.num_classes + ) + # Since the ROI feature transform is shared between boxes and masks, + # we don't need to recompute features. The mask loss is only defined + # on foreground proposals, so we need to select out the foreground + # features. + mask_features = box_features[torch.cat(fg_selection_masks, dim=0)] + del box_features + losses.update(self.mask_head(mask_features, proposals)) + return [], losses + else: + pred_instances, _ = self.box_predictor.inference(predictions, proposals) + pred_instances = self.forward_with_given_boxes(features, pred_instances) + return pred_instances, {} + + def forward_with_given_boxes( + self, features: Dict[str, torch.Tensor], instances: List[Instances] + ) -> List[Instances]: + """ + Use the given boxes in `instances` to produce other (non-box) per-ROI outputs. + + Args: + features: same as in `forward()` + instances (list[Instances]): instances to predict other outputs. Expect the keys + "pred_boxes" and "pred_classes" to exist. + + Returns: + instances (Instances): + the same `Instances` object, with extra + fields such as `pred_masks` or `pred_keypoints`. + """ + assert not self.training + assert instances[0].has("pred_boxes") and instances[0].has("pred_classes") + + if self.mask_on: + feature_list = [features[f] for f in self.in_features] + x = self._shared_roi_transform(feature_list, [x.pred_boxes for x in instances]) + return self.mask_head(x, instances) + else: + return instances + + +@ROI_HEADS_REGISTRY.register() +class StandardROIHeads(ROIHeads): + """ + It's "standard" in a sense that there is no ROI transform sharing + or feature sharing between tasks. + Each head independently processes the input features by each head's + own pooler and head. + + This class is used by most models, such as FPN and C5. + To implement more models, you can subclass it and implement a different + :meth:`forward()` or a head. + """ + + @configurable + def __init__( + self, + *, + box_in_features: List[str], + box_pooler: ROIPooler, + box_head: nn.Module, + box_predictor: nn.Module, + mask_in_features: Optional[List[str]] = None, + mask_pooler: Optional[ROIPooler] = None, + mask_head: Optional[nn.Module] = None, + keypoint_in_features: Optional[List[str]] = None, + keypoint_pooler: Optional[ROIPooler] = None, + keypoint_head: Optional[nn.Module] = None, + train_on_pred_boxes: bool = False, + **kwargs, + ): + """ + NOTE: this interface is experimental. + + Args: + box_in_features (list[str]): list of feature names to use for the box head. + box_pooler (ROIPooler): pooler to extra region features for box head + box_head (nn.Module): transform features to make box predictions + box_predictor (nn.Module): make box predictions from the feature. + Should have the same interface as :class:`FastRCNNOutputLayers`. + mask_in_features (list[str]): list of feature names to use for the mask + pooler or mask head. None if not using mask head. + mask_pooler (ROIPooler): pooler to extract region features from image features. + The mask head will then take region features to make predictions. + If None, the mask head will directly take the dict of image features + defined by `mask_in_features` + mask_head (nn.Module): transform features to make mask predictions + keypoint_in_features, keypoint_pooler, keypoint_head: similar to ``mask_*``. + train_on_pred_boxes (bool): whether to use proposal boxes or + predicted boxes from the box head to train other heads. + """ + super().__init__(**kwargs) + # keep self.in_features for backward compatibility + self.in_features = self.box_in_features = box_in_features + self.box_pooler = box_pooler + self.box_head = box_head + self.box_predictor = box_predictor + + self.mask_on = mask_in_features is not None + if self.mask_on: + self.mask_in_features = mask_in_features + self.mask_pooler = mask_pooler + self.mask_head = mask_head + + self.keypoint_on = keypoint_in_features is not None + if self.keypoint_on: + self.keypoint_in_features = keypoint_in_features + self.keypoint_pooler = keypoint_pooler + self.keypoint_head = keypoint_head + + self.train_on_pred_boxes = train_on_pred_boxes + + @classmethod + def from_config(cls, cfg, input_shape): + ret = super().from_config(cfg) + ret["train_on_pred_boxes"] = cfg.MODEL.ROI_BOX_HEAD.TRAIN_ON_PRED_BOXES + # Subclasses that have not been updated to use from_config style construction + # may have overridden _init_*_head methods. In this case, those overridden methods + # will not be classmethods and we need to avoid trying to call them here. + # We test for this with ismethod which only returns True for bound methods of cls. + # Such subclasses will need to handle calling their overridden _init_*_head methods. + if inspect.ismethod(cls._init_box_head): + ret.update(cls._init_box_head(cfg, input_shape)) + if inspect.ismethod(cls._init_mask_head): + ret.update(cls._init_mask_head(cfg, input_shape)) + if inspect.ismethod(cls._init_keypoint_head): + ret.update(cls._init_keypoint_head(cfg, input_shape)) + return ret + + @classmethod + def _init_box_head(cls, cfg, input_shape): + # fmt: off + in_features = cfg.MODEL.ROI_HEADS.IN_FEATURES + pooler_resolution = cfg.MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION + pooler_scales = tuple(1.0 / input_shape[k].stride for k in in_features) + sampling_ratio = cfg.MODEL.ROI_BOX_HEAD.POOLER_SAMPLING_RATIO + pooler_type = cfg.MODEL.ROI_BOX_HEAD.POOLER_TYPE + # fmt: on + + # If StandardROIHeads is applied on multiple feature maps (as in FPN), + # then we share the same predictors and therefore the channel counts must be the same + in_channels = [input_shape[f].channels for f in in_features] + # Check all channel counts are equal + assert len(set(in_channels)) == 1, in_channels + in_channels = in_channels[0] + + box_pooler = ROIPooler( + output_size=pooler_resolution, + scales=pooler_scales, + sampling_ratio=sampling_ratio, + pooler_type=pooler_type, + ) + # Here we split "box head" and "box predictor", which is mainly due to historical reasons. + # They are used together so the "box predictor" layers should be part of the "box head". + # New subclasses of ROIHeads do not need "box predictor"s. + box_head = build_box_head( + cfg, ShapeSpec(channels=in_channels, height=pooler_resolution, width=pooler_resolution) + ) + box_predictor = FastRCNNOutputLayers(cfg, box_head.output_shape) + return { + "box_in_features": in_features, + "box_pooler": box_pooler, + "box_head": box_head, + "box_predictor": box_predictor, + } + + @classmethod + def _init_mask_head(cls, cfg, input_shape): + if not cfg.MODEL.MASK_ON: + return {} + # fmt: off + in_features = cfg.MODEL.ROI_HEADS.IN_FEATURES + pooler_resolution = cfg.MODEL.ROI_MASK_HEAD.POOLER_RESOLUTION + pooler_scales = tuple(1.0 / input_shape[k].stride for k in in_features) + sampling_ratio = cfg.MODEL.ROI_MASK_HEAD.POOLER_SAMPLING_RATIO + pooler_type = cfg.MODEL.ROI_MASK_HEAD.POOLER_TYPE + # fmt: on + + in_channels = [input_shape[f].channels for f in in_features][0] + + ret = {"mask_in_features": in_features} + ret["mask_pooler"] = ( + ROIPooler( + output_size=pooler_resolution, + scales=pooler_scales, + sampling_ratio=sampling_ratio, + pooler_type=pooler_type, + ) + if pooler_type + else None + ) + if pooler_type: + shape = ShapeSpec( + channels=in_channels, width=pooler_resolution, height=pooler_resolution + ) + else: + shape = {f: input_shape[f] for f in in_features} + ret["mask_head"] = build_mask_head(cfg, shape) + return ret + + @classmethod + def _init_keypoint_head(cls, cfg, input_shape): + if not cfg.MODEL.KEYPOINT_ON: + return {} + # fmt: off + in_features = cfg.MODEL.ROI_HEADS.IN_FEATURES + pooler_resolution = cfg.MODEL.ROI_KEYPOINT_HEAD.POOLER_RESOLUTION + pooler_scales = tuple(1.0 / input_shape[k].stride for k in in_features) # noqa + sampling_ratio = cfg.MODEL.ROI_KEYPOINT_HEAD.POOLER_SAMPLING_RATIO + pooler_type = cfg.MODEL.ROI_KEYPOINT_HEAD.POOLER_TYPE + # fmt: on + + in_channels = [input_shape[f].channels for f in in_features][0] + + ret = {"keypoint_in_features": in_features} + ret["keypoint_pooler"] = ( + ROIPooler( + output_size=pooler_resolution, + scales=pooler_scales, + sampling_ratio=sampling_ratio, + pooler_type=pooler_type, + ) + if pooler_type + else None + ) + if pooler_type: + shape = ShapeSpec( + channels=in_channels, width=pooler_resolution, height=pooler_resolution + ) + else: + shape = {f: input_shape[f] for f in in_features} + ret["keypoint_head"] = build_keypoint_head(cfg, shape) + return ret + + def forward( + self, + images: ImageList, + features: Dict[str, torch.Tensor], + proposals: List[Instances], + targets: Optional[List[Instances]] = None, + ) -> Tuple[List[Instances], Dict[str, torch.Tensor]]: + """ + See :class:`ROIHeads.forward`. + """ + del images + if self.training: + assert targets, "'targets' argument is required during training" + proposals = self.label_and_sample_proposals(proposals, targets) + del targets + + if self.training: + losses = self._forward_box(features, proposals) + # Usually the original proposals used by the box head are used by the mask, keypoint + # heads. But when `self.train_on_pred_boxes is True`, proposals will contain boxes + # predicted by the box head. + losses.update(self._forward_mask(features, proposals)) + losses.update(self._forward_keypoint(features, proposals)) + return proposals, losses + else: + pred_instances = self._forward_box(features, proposals) + # During inference cascaded prediction is used: the mask and keypoints heads are only + # applied to the top scoring box detections. + pred_instances = self.forward_with_given_boxes(features, pred_instances) + return pred_instances, {} + + def forward_with_given_boxes( + self, features: Dict[str, torch.Tensor], instances: List[Instances] + ) -> List[Instances]: + """ + Use the given boxes in `instances` to produce other (non-box) per-ROI outputs. + + This is useful for downstream tasks where a box is known, but need to obtain + other attributes (outputs of other heads). + Test-time augmentation also uses this. + + Args: + features: same as in `forward()` + instances (list[Instances]): instances to predict other outputs. Expect the keys + "pred_boxes" and "pred_classes" to exist. + + Returns: + list[Instances]: + the same `Instances` objects, with extra + fields such as `pred_masks` or `pred_keypoints`. + """ + assert not self.training + assert instances[0].has("pred_boxes") and instances[0].has("pred_classes") + + instances = self._forward_mask(features, instances) + instances = self._forward_keypoint(features, instances) + return instances + + def _forward_box(self, features: Dict[str, torch.Tensor], proposals: List[Instances]): + """ + Forward logic of the box prediction branch. If `self.train_on_pred_boxes is True`, + the function puts predicted boxes in the `proposal_boxes` field of `proposals` argument. + + Args: + features (dict[str, Tensor]): mapping from feature map names to tensor. + Same as in :meth:`ROIHeads.forward`. + proposals (list[Instances]): the per-image object proposals with + their matching ground truth. + Each has fields "proposal_boxes", and "objectness_logits", + "gt_classes", "gt_boxes". + + Returns: + In training, a dict of losses. + In inference, a list of `Instances`, the predicted instances. + """ + features = [features[f] for f in self.box_in_features] + box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals]) + box_features = self.box_head(box_features) + predictions = self.box_predictor(box_features) + del box_features + + if self.training: + losses = self.box_predictor.losses(predictions, proposals) + # proposals is modified in-place below, so losses must be computed first. + if self.train_on_pred_boxes: + with torch.no_grad(): + pred_boxes = self.box_predictor.predict_boxes_for_gt_classes( + predictions, proposals + ) + for proposals_per_image, pred_boxes_per_image in zip(proposals, pred_boxes): + proposals_per_image.proposal_boxes = Boxes(pred_boxes_per_image) + return losses + else: + pred_instances, _ = self.box_predictor.inference(predictions, proposals) + return pred_instances + + def _forward_mask(self, features: Dict[str, torch.Tensor], instances: List[Instances]): + """ + Forward logic of the mask prediction branch. + + Args: + features (dict[str, Tensor]): mapping from feature map names to tensor. + Same as in :meth:`ROIHeads.forward`. + instances (list[Instances]): the per-image instances to train/predict masks. + In training, they can be the proposals. + In inference, they can be the boxes predicted by R-CNN box head. + + Returns: + In training, a dict of losses. + In inference, update `instances` with new fields "pred_masks" and return it. + """ + if not self.mask_on: + return {} if self.training else instances + + if self.training: + # head is only trained on positive proposals. + instances, _ = select_foreground_proposals(instances, self.num_classes) + + if self.mask_pooler is not None: + features = [features[f] for f in self.mask_in_features] + boxes = [x.proposal_boxes if self.training else x.pred_boxes for x in instances] + features = self.mask_pooler(features, boxes) + else: + features = {f: features[f] for f in self.mask_in_features} + return self.mask_head(features, instances) + + def _forward_keypoint(self, features: Dict[str, torch.Tensor], instances: List[Instances]): + """ + Forward logic of the keypoint prediction branch. + + Args: + features (dict[str, Tensor]): mapping from feature map names to tensor. + Same as in :meth:`ROIHeads.forward`. + instances (list[Instances]): the per-image instances to train/predict keypoints. + In training, they can be the proposals. + In inference, they can be the boxes predicted by R-CNN box head. + + Returns: + In training, a dict of losses. + In inference, update `instances` with new fields "pred_keypoints" and return it. + """ + if not self.keypoint_on: + return {} if self.training else instances + + if self.training: + # head is only trained on positive proposals with >=1 visible keypoints. + instances, _ = select_foreground_proposals(instances, self.num_classes) + instances = select_proposals_with_visible_keypoints(instances) + + if self.keypoint_pooler is not None: + features = [features[f] for f in self.keypoint_in_features] + boxes = [x.proposal_boxes if self.training else x.pred_boxes for x in instances] + features = self.keypoint_pooler(features, boxes) + else: + features = {f: features[f] for f in self.keypoint_in_features} + return self.keypoint_head(features, instances) diff --git a/annotator/oneformer/detectron2/modeling/roi_heads/rotated_fast_rcnn.py b/annotator/oneformer/detectron2/modeling/roi_heads/rotated_fast_rcnn.py new file mode 100644 index 0000000000000000000000000000000000000000..0d4cb8d50a8eeecb13bb6d9c9b8f021bed605cbc --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/roi_heads/rotated_fast_rcnn.py @@ -0,0 +1,271 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import numpy as np +import torch + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.layers import ShapeSpec, batched_nms_rotated +from annotator.oneformer.detectron2.structures import Instances, RotatedBoxes, pairwise_iou_rotated +from annotator.oneformer.detectron2.utils.events import get_event_storage + +from ..box_regression import Box2BoxTransformRotated +from ..poolers import ROIPooler +from ..proposal_generator.proposal_utils import add_ground_truth_to_proposals +from .box_head import build_box_head +from .fast_rcnn import FastRCNNOutputLayers +from .roi_heads import ROI_HEADS_REGISTRY, StandardROIHeads + +logger = logging.getLogger(__name__) + +""" +Shape shorthand in this module: + + N: number of images in the minibatch + R: number of ROIs, combined over all images, in the minibatch + Ri: number of ROIs in image i + K: number of foreground classes. E.g.,there are 80 foreground classes in COCO. + +Naming convention: + + deltas: refers to the 5-d (dx, dy, dw, dh, da) deltas that parameterize the box2box + transform (see :class:`box_regression.Box2BoxTransformRotated`). + + pred_class_logits: predicted class scores in [-inf, +inf]; use + softmax(pred_class_logits) to estimate P(class). + + gt_classes: ground-truth classification labels in [0, K], where [0, K) represent + foreground object classes and K represents the background class. + + pred_proposal_deltas: predicted rotated box2box transform deltas for transforming proposals + to detection box predictions. + + gt_proposal_deltas: ground-truth rotated box2box transform deltas +""" + + +def fast_rcnn_inference_rotated( + boxes, scores, image_shapes, score_thresh, nms_thresh, topk_per_image +): + """ + Call `fast_rcnn_inference_single_image_rotated` for all images. + + Args: + boxes (list[Tensor]): A list of Tensors of predicted class-specific or class-agnostic + boxes for each image. Element i has shape (Ri, K * 5) if doing + class-specific regression, or (Ri, 5) if doing class-agnostic + regression, where Ri is the number of predicted objects for image i. + This is compatible with the output of :meth:`FastRCNNOutputLayers.predict_boxes`. + scores (list[Tensor]): A list of Tensors of predicted class scores for each image. + Element i has shape (Ri, K + 1), where Ri is the number of predicted objects + for image i. Compatible with the output of :meth:`FastRCNNOutputLayers.predict_probs`. + image_shapes (list[tuple]): A list of (width, height) tuples for each image in the batch. + score_thresh (float): Only return detections with a confidence score exceeding this + threshold. + nms_thresh (float): The threshold to use for box non-maximum suppression. Value in [0, 1]. + topk_per_image (int): The number of top scoring detections to return. Set < 0 to return + all detections. + + Returns: + instances: (list[Instances]): A list of N instances, one for each image in the batch, + that stores the topk most confidence detections. + kept_indices: (list[Tensor]): A list of 1D tensor of length of N, each element indicates + the corresponding boxes/scores index in [0, Ri) from the input, for image i. + """ + result_per_image = [ + fast_rcnn_inference_single_image_rotated( + boxes_per_image, scores_per_image, image_shape, score_thresh, nms_thresh, topk_per_image + ) + for scores_per_image, boxes_per_image, image_shape in zip(scores, boxes, image_shapes) + ] + return [x[0] for x in result_per_image], [x[1] for x in result_per_image] + + +@torch.no_grad() +def fast_rcnn_inference_single_image_rotated( + boxes, scores, image_shape, score_thresh, nms_thresh, topk_per_image +): + """ + Single-image inference. Return rotated bounding-box detection results by thresholding + on scores and applying rotated non-maximum suppression (Rotated NMS). + + Args: + Same as `fast_rcnn_inference_rotated`, but with rotated boxes, scores, and image shapes + per image. + + Returns: + Same as `fast_rcnn_inference_rotated`, but for only one image. + """ + valid_mask = torch.isfinite(boxes).all(dim=1) & torch.isfinite(scores).all(dim=1) + if not valid_mask.all(): + boxes = boxes[valid_mask] + scores = scores[valid_mask] + + B = 5 # box dimension + scores = scores[:, :-1] + num_bbox_reg_classes = boxes.shape[1] // B + # Convert to Boxes to use the `clip` function ... + boxes = RotatedBoxes(boxes.reshape(-1, B)) + boxes.clip(image_shape) + boxes = boxes.tensor.view(-1, num_bbox_reg_classes, B) # R x C x B + # Filter results based on detection scores + filter_mask = scores > score_thresh # R x K + # R' x 2. First column contains indices of the R predictions; + # Second column contains indices of classes. + filter_inds = filter_mask.nonzero() + if num_bbox_reg_classes == 1: + boxes = boxes[filter_inds[:, 0], 0] + else: + boxes = boxes[filter_mask] + scores = scores[filter_mask] + + # Apply per-class Rotated NMS + keep = batched_nms_rotated(boxes, scores, filter_inds[:, 1], nms_thresh) + if topk_per_image >= 0: + keep = keep[:topk_per_image] + boxes, scores, filter_inds = boxes[keep], scores[keep], filter_inds[keep] + + result = Instances(image_shape) + result.pred_boxes = RotatedBoxes(boxes) + result.scores = scores + result.pred_classes = filter_inds[:, 1] + + return result, filter_inds[:, 0] + + +class RotatedFastRCNNOutputLayers(FastRCNNOutputLayers): + """ + Two linear layers for predicting Rotated Fast R-CNN outputs. + """ + + @classmethod + def from_config(cls, cfg, input_shape): + args = super().from_config(cfg, input_shape) + args["box2box_transform"] = Box2BoxTransformRotated( + weights=cfg.MODEL.ROI_BOX_HEAD.BBOX_REG_WEIGHTS + ) + return args + + def inference(self, predictions, proposals): + """ + Returns: + list[Instances]: same as `fast_rcnn_inference_rotated`. + list[Tensor]: same as `fast_rcnn_inference_rotated`. + """ + boxes = self.predict_boxes(predictions, proposals) + scores = self.predict_probs(predictions, proposals) + image_shapes = [x.image_size for x in proposals] + + return fast_rcnn_inference_rotated( + boxes, + scores, + image_shapes, + self.test_score_thresh, + self.test_nms_thresh, + self.test_topk_per_image, + ) + + +@ROI_HEADS_REGISTRY.register() +class RROIHeads(StandardROIHeads): + """ + This class is used by Rotated Fast R-CNN to detect rotated boxes. + For now, it only supports box predictions but not mask or keypoints. + """ + + @configurable + def __init__(self, **kwargs): + """ + NOTE: this interface is experimental. + """ + super().__init__(**kwargs) + assert ( + not self.mask_on and not self.keypoint_on + ), "Mask/Keypoints not supported in Rotated ROIHeads." + assert not self.train_on_pred_boxes, "train_on_pred_boxes not implemented for RROIHeads!" + + @classmethod + def _init_box_head(cls, cfg, input_shape): + # fmt: off + in_features = cfg.MODEL.ROI_HEADS.IN_FEATURES + pooler_resolution = cfg.MODEL.ROI_BOX_HEAD.POOLER_RESOLUTION + pooler_scales = tuple(1.0 / input_shape[k].stride for k in in_features) + sampling_ratio = cfg.MODEL.ROI_BOX_HEAD.POOLER_SAMPLING_RATIO + pooler_type = cfg.MODEL.ROI_BOX_HEAD.POOLER_TYPE + # fmt: on + assert pooler_type in ["ROIAlignRotated"], pooler_type + # assume all channel counts are equal + in_channels = [input_shape[f].channels for f in in_features][0] + + box_pooler = ROIPooler( + output_size=pooler_resolution, + scales=pooler_scales, + sampling_ratio=sampling_ratio, + pooler_type=pooler_type, + ) + box_head = build_box_head( + cfg, ShapeSpec(channels=in_channels, height=pooler_resolution, width=pooler_resolution) + ) + # This line is the only difference v.s. StandardROIHeads + box_predictor = RotatedFastRCNNOutputLayers(cfg, box_head.output_shape) + return { + "box_in_features": in_features, + "box_pooler": box_pooler, + "box_head": box_head, + "box_predictor": box_predictor, + } + + @torch.no_grad() + def label_and_sample_proposals(self, proposals, targets): + """ + Prepare some proposals to be used to train the RROI heads. + It performs box matching between `proposals` and `targets`, and assigns + training labels to the proposals. + It returns `self.batch_size_per_image` random samples from proposals and groundtruth boxes, + with a fraction of positives that is no larger than `self.positive_sample_fraction. + + Args: + See :meth:`StandardROIHeads.forward` + + Returns: + list[Instances]: length `N` list of `Instances`s containing the proposals + sampled for training. Each `Instances` has the following fields: + - proposal_boxes: the rotated proposal boxes + - gt_boxes: the ground-truth rotated boxes that the proposal is assigned to + (this is only meaningful if the proposal has a label > 0; if label = 0 + then the ground-truth box is random) + - gt_classes: the ground-truth classification lable for each proposal + """ + if self.proposal_append_gt: + proposals = add_ground_truth_to_proposals(targets, proposals) + + proposals_with_gt = [] + + num_fg_samples = [] + num_bg_samples = [] + for proposals_per_image, targets_per_image in zip(proposals, targets): + has_gt = len(targets_per_image) > 0 + match_quality_matrix = pairwise_iou_rotated( + targets_per_image.gt_boxes, proposals_per_image.proposal_boxes + ) + matched_idxs, matched_labels = self.proposal_matcher(match_quality_matrix) + sampled_idxs, gt_classes = self._sample_proposals( + matched_idxs, matched_labels, targets_per_image.gt_classes + ) + + proposals_per_image = proposals_per_image[sampled_idxs] + proposals_per_image.gt_classes = gt_classes + + if has_gt: + sampled_targets = matched_idxs[sampled_idxs] + proposals_per_image.gt_boxes = targets_per_image.gt_boxes[sampled_targets] + + num_bg_samples.append((gt_classes == self.num_classes).sum().item()) + num_fg_samples.append(gt_classes.numel() - num_bg_samples[-1]) + proposals_with_gt.append(proposals_per_image) + + # Log the number of fg/bg samples that are selected for training ROI heads + storage = get_event_storage() + storage.put_scalar("roi_head/num_fg_samples", np.mean(num_fg_samples)) + storage.put_scalar("roi_head/num_bg_samples", np.mean(num_bg_samples)) + + return proposals_with_gt diff --git a/annotator/oneformer/detectron2/modeling/sampling.py b/annotator/oneformer/detectron2/modeling/sampling.py new file mode 100644 index 0000000000000000000000000000000000000000..5c55fbf9f3cd985a179aeb8ad6ced524a31c3f6c --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/sampling.py @@ -0,0 +1,54 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import torch + +from annotator.oneformer.detectron2.layers import nonzero_tuple + +__all__ = ["subsample_labels"] + + +def subsample_labels( + labels: torch.Tensor, num_samples: int, positive_fraction: float, bg_label: int +): + """ + Return `num_samples` (or fewer, if not enough found) + random samples from `labels` which is a mixture of positives & negatives. + It will try to return as many positives as possible without + exceeding `positive_fraction * num_samples`, and then try to + fill the remaining slots with negatives. + + Args: + labels (Tensor): (N, ) label vector with values: + * -1: ignore + * bg_label: background ("negative") class + * otherwise: one or more foreground ("positive") classes + num_samples (int): The total number of labels with value >= 0 to return. + Values that are not sampled will be filled with -1 (ignore). + positive_fraction (float): The number of subsampled labels with values > 0 + is `min(num_positives, int(positive_fraction * num_samples))`. The number + of negatives sampled is `min(num_negatives, num_samples - num_positives_sampled)`. + In order words, if there are not enough positives, the sample is filled with + negatives. If there are also not enough negatives, then as many elements are + sampled as is possible. + bg_label (int): label index of background ("negative") class. + + Returns: + pos_idx, neg_idx (Tensor): + 1D vector of indices. The total length of both is `num_samples` or fewer. + """ + positive = nonzero_tuple((labels != -1) & (labels != bg_label))[0] + negative = nonzero_tuple(labels == bg_label)[0] + + num_pos = int(num_samples * positive_fraction) + # protect against not enough positive examples + num_pos = min(positive.numel(), num_pos) + num_neg = num_samples - num_pos + # protect against not enough negative examples + num_neg = min(negative.numel(), num_neg) + + # randomly select positive and negative examples + perm1 = torch.randperm(positive.numel(), device=positive.device)[:num_pos] + perm2 = torch.randperm(negative.numel(), device=negative.device)[:num_neg] + + pos_idx = positive[perm1] + neg_idx = negative[perm2] + return pos_idx, neg_idx diff --git a/annotator/oneformer/detectron2/modeling/test_time_augmentation.py b/annotator/oneformer/detectron2/modeling/test_time_augmentation.py new file mode 100644 index 0000000000000000000000000000000000000000..625f8ba9a01275df64967c097912538337ec91dc --- /dev/null +++ b/annotator/oneformer/detectron2/modeling/test_time_augmentation.py @@ -0,0 +1,307 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +import numpy as np +from contextlib import contextmanager +from itertools import count +from typing import List +import torch +from fvcore.transforms import HFlipTransform, NoOpTransform +from torch import nn +from torch.nn.parallel import DistributedDataParallel + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.data.detection_utils import read_image +from annotator.oneformer.detectron2.data.transforms import ( + RandomFlip, + ResizeShortestEdge, + ResizeTransform, + apply_augmentations, +) +from annotator.oneformer.detectron2.structures import Boxes, Instances + +from .meta_arch import GeneralizedRCNN +from .postprocessing import detector_postprocess +from .roi_heads.fast_rcnn import fast_rcnn_inference_single_image + +__all__ = ["DatasetMapperTTA", "GeneralizedRCNNWithTTA"] + + +class DatasetMapperTTA: + """ + Implement test-time augmentation for detection data. + It is a callable which takes a dataset dict from a detection dataset, + and returns a list of dataset dicts where the images + are augmented from the input image by the transformations defined in the config. + This is used for test-time augmentation. + """ + + @configurable + def __init__(self, min_sizes: List[int], max_size: int, flip: bool): + """ + Args: + min_sizes: list of short-edge size to resize the image to + max_size: maximum height or width of resized images + flip: whether to apply flipping augmentation + """ + self.min_sizes = min_sizes + self.max_size = max_size + self.flip = flip + + @classmethod + def from_config(cls, cfg): + return { + "min_sizes": cfg.TEST.AUG.MIN_SIZES, + "max_size": cfg.TEST.AUG.MAX_SIZE, + "flip": cfg.TEST.AUG.FLIP, + } + + def __call__(self, dataset_dict): + """ + Args: + dict: a dict in standard model input format. See tutorials for details. + + Returns: + list[dict]: + a list of dicts, which contain augmented version of the input image. + The total number of dicts is ``len(min_sizes) * (2 if flip else 1)``. + Each dict has field "transforms" which is a TransformList, + containing the transforms that are used to generate this image. + """ + numpy_image = dataset_dict["image"].permute(1, 2, 0).numpy() + shape = numpy_image.shape + orig_shape = (dataset_dict["height"], dataset_dict["width"]) + if shape[:2] != orig_shape: + # It transforms the "original" image in the dataset to the input image + pre_tfm = ResizeTransform(orig_shape[0], orig_shape[1], shape[0], shape[1]) + else: + pre_tfm = NoOpTransform() + + # Create all combinations of augmentations to use + aug_candidates = [] # each element is a list[Augmentation] + for min_size in self.min_sizes: + resize = ResizeShortestEdge(min_size, self.max_size) + aug_candidates.append([resize]) # resize only + if self.flip: + flip = RandomFlip(prob=1.0) + aug_candidates.append([resize, flip]) # resize + flip + + # Apply all the augmentations + ret = [] + for aug in aug_candidates: + new_image, tfms = apply_augmentations(aug, np.copy(numpy_image)) + torch_image = torch.from_numpy(np.ascontiguousarray(new_image.transpose(2, 0, 1))) + + dic = copy.deepcopy(dataset_dict) + dic["transforms"] = pre_tfm + tfms + dic["image"] = torch_image + ret.append(dic) + return ret + + +class GeneralizedRCNNWithTTA(nn.Module): + """ + A GeneralizedRCNN with test-time augmentation enabled. + Its :meth:`__call__` method has the same interface as :meth:`GeneralizedRCNN.forward`. + """ + + def __init__(self, cfg, model, tta_mapper=None, batch_size=3): + """ + Args: + cfg (CfgNode): + model (GeneralizedRCNN): a GeneralizedRCNN to apply TTA on. + tta_mapper (callable): takes a dataset dict and returns a list of + augmented versions of the dataset dict. Defaults to + `DatasetMapperTTA(cfg)`. + batch_size (int): batch the augmented images into this batch size for inference. + """ + super().__init__() + if isinstance(model, DistributedDataParallel): + model = model.module + assert isinstance( + model, GeneralizedRCNN + ), "TTA is only supported on GeneralizedRCNN. Got a model of type {}".format(type(model)) + self.cfg = cfg.clone() + assert not self.cfg.MODEL.KEYPOINT_ON, "TTA for keypoint is not supported yet" + assert ( + not self.cfg.MODEL.LOAD_PROPOSALS + ), "TTA for pre-computed proposals is not supported yet" + + self.model = model + + if tta_mapper is None: + tta_mapper = DatasetMapperTTA(cfg) + self.tta_mapper = tta_mapper + self.batch_size = batch_size + + @contextmanager + def _turn_off_roi_heads(self, attrs): + """ + Open a context where some heads in `model.roi_heads` are temporarily turned off. + Args: + attr (list[str]): the attribute in `model.roi_heads` which can be used + to turn off a specific head, e.g., "mask_on", "keypoint_on". + """ + roi_heads = self.model.roi_heads + old = {} + for attr in attrs: + try: + old[attr] = getattr(roi_heads, attr) + except AttributeError: + # The head may not be implemented in certain ROIHeads + pass + + if len(old.keys()) == 0: + yield + else: + for attr in old.keys(): + setattr(roi_heads, attr, False) + yield + for attr in old.keys(): + setattr(roi_heads, attr, old[attr]) + + def _batch_inference(self, batched_inputs, detected_instances=None): + """ + Execute inference on a list of inputs, + using batch size = self.batch_size, instead of the length of the list. + + Inputs & outputs have the same format as :meth:`GeneralizedRCNN.inference` + """ + if detected_instances is None: + detected_instances = [None] * len(batched_inputs) + + outputs = [] + inputs, instances = [], [] + for idx, input, instance in zip(count(), batched_inputs, detected_instances): + inputs.append(input) + instances.append(instance) + if len(inputs) == self.batch_size or idx == len(batched_inputs) - 1: + outputs.extend( + self.model.inference( + inputs, + instances if instances[0] is not None else None, + do_postprocess=False, + ) + ) + inputs, instances = [], [] + return outputs + + def __call__(self, batched_inputs): + """ + Same input/output format as :meth:`GeneralizedRCNN.forward` + """ + + def _maybe_read_image(dataset_dict): + ret = copy.copy(dataset_dict) + if "image" not in ret: + image = read_image(ret.pop("file_name"), self.model.input_format) + image = torch.from_numpy(np.ascontiguousarray(image.transpose(2, 0, 1))) # CHW + ret["image"] = image + if "height" not in ret and "width" not in ret: + ret["height"] = image.shape[1] + ret["width"] = image.shape[2] + return ret + + return [self._inference_one_image(_maybe_read_image(x)) for x in batched_inputs] + + def _inference_one_image(self, input): + """ + Args: + input (dict): one dataset dict with "image" field being a CHW tensor + + Returns: + dict: one output dict + """ + orig_shape = (input["height"], input["width"]) + augmented_inputs, tfms = self._get_augmented_inputs(input) + # Detect boxes from all augmented versions + with self._turn_off_roi_heads(["mask_on", "keypoint_on"]): + # temporarily disable roi heads + all_boxes, all_scores, all_classes = self._get_augmented_boxes(augmented_inputs, tfms) + # merge all detected boxes to obtain final predictions for boxes + merged_instances = self._merge_detections(all_boxes, all_scores, all_classes, orig_shape) + + if self.cfg.MODEL.MASK_ON: + # Use the detected boxes to obtain masks + augmented_instances = self._rescale_detected_boxes( + augmented_inputs, merged_instances, tfms + ) + # run forward on the detected boxes + outputs = self._batch_inference(augmented_inputs, augmented_instances) + # Delete now useless variables to avoid being out of memory + del augmented_inputs, augmented_instances + # average the predictions + merged_instances.pred_masks = self._reduce_pred_masks(outputs, tfms) + merged_instances = detector_postprocess(merged_instances, *orig_shape) + return {"instances": merged_instances} + else: + return {"instances": merged_instances} + + def _get_augmented_inputs(self, input): + augmented_inputs = self.tta_mapper(input) + tfms = [x.pop("transforms") for x in augmented_inputs] + return augmented_inputs, tfms + + def _get_augmented_boxes(self, augmented_inputs, tfms): + # 1: forward with all augmented images + outputs = self._batch_inference(augmented_inputs) + # 2: union the results + all_boxes = [] + all_scores = [] + all_classes = [] + for output, tfm in zip(outputs, tfms): + # Need to inverse the transforms on boxes, to obtain results on original image + pred_boxes = output.pred_boxes.tensor + original_pred_boxes = tfm.inverse().apply_box(pred_boxes.cpu().numpy()) + all_boxes.append(torch.from_numpy(original_pred_boxes).to(pred_boxes.device)) + + all_scores.extend(output.scores) + all_classes.extend(output.pred_classes) + all_boxes = torch.cat(all_boxes, dim=0) + return all_boxes, all_scores, all_classes + + def _merge_detections(self, all_boxes, all_scores, all_classes, shape_hw): + # select from the union of all results + num_boxes = len(all_boxes) + num_classes = self.cfg.MODEL.ROI_HEADS.NUM_CLASSES + # +1 because fast_rcnn_inference expects background scores as well + all_scores_2d = torch.zeros(num_boxes, num_classes + 1, device=all_boxes.device) + for idx, cls, score in zip(count(), all_classes, all_scores): + all_scores_2d[idx, cls] = score + + merged_instances, _ = fast_rcnn_inference_single_image( + all_boxes, + all_scores_2d, + shape_hw, + 1e-8, + self.cfg.MODEL.ROI_HEADS.NMS_THRESH_TEST, + self.cfg.TEST.DETECTIONS_PER_IMAGE, + ) + + return merged_instances + + def _rescale_detected_boxes(self, augmented_inputs, merged_instances, tfms): + augmented_instances = [] + for input, tfm in zip(augmented_inputs, tfms): + # Transform the target box to the augmented image's coordinate space + pred_boxes = merged_instances.pred_boxes.tensor.cpu().numpy() + pred_boxes = torch.from_numpy(tfm.apply_box(pred_boxes)) + + aug_instances = Instances( + image_size=input["image"].shape[1:3], + pred_boxes=Boxes(pred_boxes), + pred_classes=merged_instances.pred_classes, + scores=merged_instances.scores, + ) + augmented_instances.append(aug_instances) + return augmented_instances + + def _reduce_pred_masks(self, outputs, tfms): + # Should apply inverse transforms on masks. + # We assume only resize & flip are used. pred_masks is a scale-invariant + # representation, so we handle flip specially + for output, tfm in zip(outputs, tfms): + if any(isinstance(t, HFlipTransform) for t in tfm.transforms): + output.pred_masks = output.pred_masks.flip(dims=[3]) + all_pred_masks = torch.stack([o.pred_masks for o in outputs], dim=0) + avg_pred_masks = torch.mean(all_pred_masks, dim=0) + return avg_pred_masks diff --git a/annotator/oneformer/detectron2/projects/README.md b/annotator/oneformer/detectron2/projects/README.md new file mode 100644 index 0000000000000000000000000000000000000000..95afe7ff8c8a9bd2f56621fcc3c1bdac11c256a9 --- /dev/null +++ b/annotator/oneformer/detectron2/projects/README.md @@ -0,0 +1,2 @@ + +Projects live in the [`projects` directory](../../projects) under the root of this repository, but not here. diff --git a/annotator/oneformer/detectron2/projects/__init__.py b/annotator/oneformer/detectron2/projects/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b2d0540b93ebbad78d6ff2cc0adc0fe8375816c2 --- /dev/null +++ b/annotator/oneformer/detectron2/projects/__init__.py @@ -0,0 +1,34 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import importlib.abc +import importlib.util +from pathlib import Path + +__all__ = [] + +_PROJECTS = { + "point_rend": "PointRend", + "deeplab": "DeepLab", + "panoptic_deeplab": "Panoptic-DeepLab", +} +_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent / "projects" + +if _PROJECT_ROOT.is_dir(): + # This is true only for in-place installation (pip install -e, setup.py develop), + # where setup(package_dir=) does not work: https://github.com/pypa/setuptools/issues/230 + + class _D2ProjectsFinder(importlib.abc.MetaPathFinder): + def find_spec(self, name, path, target=None): + if not name.startswith("detectron2.projects."): + return + project_name = name.split(".")[-1] + project_dir = _PROJECTS.get(project_name) + if not project_dir: + return + target_file = _PROJECT_ROOT / f"{project_dir}/{project_name}/__init__.py" + if not target_file.is_file(): + return + return importlib.util.spec_from_file_location(name, target_file) + + import sys + + sys.meta_path.append(_D2ProjectsFinder()) diff --git a/annotator/oneformer/detectron2/projects/deeplab/__init__.py b/annotator/oneformer/detectron2/projects/deeplab/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..dcd88ff0c09d630577e3ac9f8afb5324a80a7be4 --- /dev/null +++ b/annotator/oneformer/detectron2/projects/deeplab/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .build_solver import build_lr_scheduler +from .config import add_deeplab_config +from .resnet import build_resnet_deeplab_backbone +from .semantic_seg import DeepLabV3Head, DeepLabV3PlusHead diff --git a/annotator/oneformer/detectron2/projects/deeplab/build_solver.py b/annotator/oneformer/detectron2/projects/deeplab/build_solver.py new file mode 100644 index 0000000000000000000000000000000000000000..19ab244380e8bcbb15c37f467bb58bc4f8dc17ec --- /dev/null +++ b/annotator/oneformer/detectron2/projects/deeplab/build_solver.py @@ -0,0 +1,27 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import torch + +from annotator.oneformer.detectron2.config import CfgNode +from annotator.oneformer.detectron2.solver import LRScheduler +from annotator.oneformer.detectron2.solver import build_lr_scheduler as build_d2_lr_scheduler + +from .lr_scheduler import WarmupPolyLR + + +def build_lr_scheduler(cfg: CfgNode, optimizer: torch.optim.Optimizer) -> LRScheduler: + """ + Build a LR scheduler from config. + """ + name = cfg.SOLVER.LR_SCHEDULER_NAME + if name == "WarmupPolyLR": + return WarmupPolyLR( + optimizer, + cfg.SOLVER.MAX_ITER, + warmup_factor=cfg.SOLVER.WARMUP_FACTOR, + warmup_iters=cfg.SOLVER.WARMUP_ITERS, + warmup_method=cfg.SOLVER.WARMUP_METHOD, + power=cfg.SOLVER.POLY_LR_POWER, + constant_ending=cfg.SOLVER.POLY_LR_CONSTANT_ENDING, + ) + else: + return build_d2_lr_scheduler(cfg, optimizer) diff --git a/annotator/oneformer/detectron2/projects/deeplab/config.py b/annotator/oneformer/detectron2/projects/deeplab/config.py new file mode 100644 index 0000000000000000000000000000000000000000..5f5e45a9124e61c12d90cfc5032b268496891a4a --- /dev/null +++ b/annotator/oneformer/detectron2/projects/deeplab/config.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. + + +def add_deeplab_config(cfg): + """ + Add config for DeepLab. + """ + # We retry random cropping until no single category in semantic segmentation GT occupies more + # than `SINGLE_CATEGORY_MAX_AREA` part of the crop. + cfg.INPUT.CROP.SINGLE_CATEGORY_MAX_AREA = 1.0 + # Used for `poly` learning rate schedule. + cfg.SOLVER.POLY_LR_POWER = 0.9 + cfg.SOLVER.POLY_LR_CONSTANT_ENDING = 0.0 + # Loss type, choose from `cross_entropy`, `hard_pixel_mining`. + cfg.MODEL.SEM_SEG_HEAD.LOSS_TYPE = "hard_pixel_mining" + # DeepLab settings + cfg.MODEL.SEM_SEG_HEAD.PROJECT_FEATURES = ["res2"] + cfg.MODEL.SEM_SEG_HEAD.PROJECT_CHANNELS = [48] + cfg.MODEL.SEM_SEG_HEAD.ASPP_CHANNELS = 256 + cfg.MODEL.SEM_SEG_HEAD.ASPP_DILATIONS = [6, 12, 18] + cfg.MODEL.SEM_SEG_HEAD.ASPP_DROPOUT = 0.1 + cfg.MODEL.SEM_SEG_HEAD.USE_DEPTHWISE_SEPARABLE_CONV = False + # Backbone new configs + cfg.MODEL.RESNETS.RES4_DILATION = 1 + cfg.MODEL.RESNETS.RES5_MULTI_GRID = [1, 2, 4] + # ResNet stem type from: `basic`, `deeplab` + cfg.MODEL.RESNETS.STEM_TYPE = "deeplab" diff --git a/annotator/oneformer/detectron2/projects/deeplab/loss.py b/annotator/oneformer/detectron2/projects/deeplab/loss.py new file mode 100644 index 0000000000000000000000000000000000000000..3a43087b7c1a2b4d2b249fad117724dbd0f14fdd --- /dev/null +++ b/annotator/oneformer/detectron2/projects/deeplab/loss.py @@ -0,0 +1,40 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import torch +import torch.nn as nn + + +class DeepLabCE(nn.Module): + """ + Hard pixel mining with cross entropy loss, for semantic segmentation. + This is used in TensorFlow DeepLab frameworks. + Paper: DeeperLab: Single-Shot Image Parser + Reference: https://github.com/tensorflow/models/blob/bd488858d610e44df69da6f89277e9de8a03722c/research/deeplab/utils/train_utils.py#L33 # noqa + Arguments: + ignore_label: Integer, label to ignore. + top_k_percent_pixels: Float, the value lies in [0.0, 1.0]. When its + value < 1.0, only compute the loss for the top k percent pixels + (e.g., the top 20% pixels). This is useful for hard pixel mining. + weight: Tensor, a manual rescaling weight given to each class. + """ + + def __init__(self, ignore_label=-1, top_k_percent_pixels=1.0, weight=None): + super(DeepLabCE, self).__init__() + self.top_k_percent_pixels = top_k_percent_pixels + self.ignore_label = ignore_label + self.criterion = nn.CrossEntropyLoss( + weight=weight, ignore_index=ignore_label, reduction="none" + ) + + def forward(self, logits, labels, weights=None): + if weights is None: + pixel_losses = self.criterion(logits, labels).contiguous().view(-1) + else: + # Apply per-pixel loss weights. + pixel_losses = self.criterion(logits, labels) * weights + pixel_losses = pixel_losses.contiguous().view(-1) + if self.top_k_percent_pixels == 1.0: + return pixel_losses.mean() + + top_k_pixels = int(self.top_k_percent_pixels * pixel_losses.numel()) + pixel_losses, _ = torch.topk(pixel_losses, top_k_pixels) + return pixel_losses.mean() diff --git a/annotator/oneformer/detectron2/projects/deeplab/lr_scheduler.py b/annotator/oneformer/detectron2/projects/deeplab/lr_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..9e15b0e19d03e955406fa1555d0d4f9d31d505c4 --- /dev/null +++ b/annotator/oneformer/detectron2/projects/deeplab/lr_scheduler.py @@ -0,0 +1,62 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import math +from typing import List +import torch + +from annotator.oneformer.detectron2.solver.lr_scheduler import LRScheduler, _get_warmup_factor_at_iter + +# NOTE: PyTorch's LR scheduler interface uses names that assume the LR changes +# only on epoch boundaries. We typically use iteration based schedules instead. +# As a result, "epoch" (e.g., as in self.last_epoch) should be understood to mean +# "iteration" instead. + +# FIXME: ideally this would be achieved with a CombinedLRScheduler, separating +# MultiStepLR with WarmupLR but the current LRScheduler design doesn't allow it. + + +class WarmupPolyLR(LRScheduler): + """ + Poly learning rate schedule used to train DeepLab. + Paper: DeepLab: Semantic Image Segmentation with Deep Convolutional Nets, + Atrous Convolution, and Fully Connected CRFs. + Reference: https://github.com/tensorflow/models/blob/21b73d22f3ed05b650e85ac50849408dd36de32e/research/deeplab/utils/train_utils.py#L337 # noqa + """ + + def __init__( + self, + optimizer: torch.optim.Optimizer, + max_iters: int, + warmup_factor: float = 0.001, + warmup_iters: int = 1000, + warmup_method: str = "linear", + last_epoch: int = -1, + power: float = 0.9, + constant_ending: float = 0.0, + ): + self.max_iters = max_iters + self.warmup_factor = warmup_factor + self.warmup_iters = warmup_iters + self.warmup_method = warmup_method + self.power = power + self.constant_ending = constant_ending + super().__init__(optimizer, last_epoch) + + def get_lr(self) -> List[float]: + warmup_factor = _get_warmup_factor_at_iter( + self.warmup_method, self.last_epoch, self.warmup_iters, self.warmup_factor + ) + if self.constant_ending > 0 and warmup_factor == 1.0: + # Constant ending lr. + if ( + math.pow((1.0 - self.last_epoch / self.max_iters), self.power) + < self.constant_ending + ): + return [base_lr * self.constant_ending for base_lr in self.base_lrs] + return [ + base_lr * warmup_factor * math.pow((1.0 - self.last_epoch / self.max_iters), self.power) + for base_lr in self.base_lrs + ] + + def _compute_values(self) -> List[float]: + # The new interface + return self.get_lr() diff --git a/annotator/oneformer/detectron2/projects/deeplab/resnet.py b/annotator/oneformer/detectron2/projects/deeplab/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..28455d123a12f887400c19c263d08cc2ed08522e --- /dev/null +++ b/annotator/oneformer/detectron2/projects/deeplab/resnet.py @@ -0,0 +1,158 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import fvcore.nn.weight_init as weight_init +import torch.nn.functional as F + +from annotator.oneformer.detectron2.layers import CNNBlockBase, Conv2d, get_norm +from annotator.oneformer.detectron2.modeling import BACKBONE_REGISTRY +from annotator.oneformer.detectron2.modeling.backbone.resnet import ( + BasicStem, + BottleneckBlock, + DeformBottleneckBlock, + ResNet, +) + + +class DeepLabStem(CNNBlockBase): + """ + The DeepLab ResNet stem (layers before the first residual block). + """ + + def __init__(self, in_channels=3, out_channels=128, norm="BN"): + """ + Args: + norm (str or callable): norm after the first conv layer. + See :func:`layers.get_norm` for supported format. + """ + super().__init__(in_channels, out_channels, 4) + self.in_channels = in_channels + self.conv1 = Conv2d( + in_channels, + out_channels // 2, + kernel_size=3, + stride=2, + padding=1, + bias=False, + norm=get_norm(norm, out_channels // 2), + ) + self.conv2 = Conv2d( + out_channels // 2, + out_channels // 2, + kernel_size=3, + stride=1, + padding=1, + bias=False, + norm=get_norm(norm, out_channels // 2), + ) + self.conv3 = Conv2d( + out_channels // 2, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=False, + norm=get_norm(norm, out_channels), + ) + weight_init.c2_msra_fill(self.conv1) + weight_init.c2_msra_fill(self.conv2) + weight_init.c2_msra_fill(self.conv3) + + def forward(self, x): + x = self.conv1(x) + x = F.relu_(x) + x = self.conv2(x) + x = F.relu_(x) + x = self.conv3(x) + x = F.relu_(x) + x = F.max_pool2d(x, kernel_size=3, stride=2, padding=1) + return x + + +@BACKBONE_REGISTRY.register() +def build_resnet_deeplab_backbone(cfg, input_shape): + """ + Create a ResNet instance from config. + Returns: + ResNet: a :class:`ResNet` instance. + """ + # need registration of new blocks/stems? + norm = cfg.MODEL.RESNETS.NORM + if cfg.MODEL.RESNETS.STEM_TYPE == "basic": + stem = BasicStem( + in_channels=input_shape.channels, + out_channels=cfg.MODEL.RESNETS.STEM_OUT_CHANNELS, + norm=norm, + ) + elif cfg.MODEL.RESNETS.STEM_TYPE == "deeplab": + stem = DeepLabStem( + in_channels=input_shape.channels, + out_channels=cfg.MODEL.RESNETS.STEM_OUT_CHANNELS, + norm=norm, + ) + else: + raise ValueError("Unknown stem type: {}".format(cfg.MODEL.RESNETS.STEM_TYPE)) + + # fmt: off + freeze_at = cfg.MODEL.BACKBONE.FREEZE_AT + out_features = cfg.MODEL.RESNETS.OUT_FEATURES + depth = cfg.MODEL.RESNETS.DEPTH + num_groups = cfg.MODEL.RESNETS.NUM_GROUPS + width_per_group = cfg.MODEL.RESNETS.WIDTH_PER_GROUP + bottleneck_channels = num_groups * width_per_group + in_channels = cfg.MODEL.RESNETS.STEM_OUT_CHANNELS + out_channels = cfg.MODEL.RESNETS.RES2_OUT_CHANNELS + stride_in_1x1 = cfg.MODEL.RESNETS.STRIDE_IN_1X1 + res4_dilation = cfg.MODEL.RESNETS.RES4_DILATION + res5_dilation = cfg.MODEL.RESNETS.RES5_DILATION + deform_on_per_stage = cfg.MODEL.RESNETS.DEFORM_ON_PER_STAGE + deform_modulated = cfg.MODEL.RESNETS.DEFORM_MODULATED + deform_num_groups = cfg.MODEL.RESNETS.DEFORM_NUM_GROUPS + res5_multi_grid = cfg.MODEL.RESNETS.RES5_MULTI_GRID + # fmt: on + assert res4_dilation in {1, 2}, "res4_dilation cannot be {}.".format(res4_dilation) + assert res5_dilation in {1, 2, 4}, "res5_dilation cannot be {}.".format(res5_dilation) + if res4_dilation == 2: + # Always dilate res5 if res4 is dilated. + assert res5_dilation == 4 + + num_blocks_per_stage = {50: [3, 4, 6, 3], 101: [3, 4, 23, 3], 152: [3, 8, 36, 3]}[depth] + + stages = [] + + # Avoid creating variables without gradients + # It consumes extra memory and may cause allreduce to fail + out_stage_idx = [{"res2": 2, "res3": 3, "res4": 4, "res5": 5}[f] for f in out_features] + max_stage_idx = max(out_stage_idx) + for idx, stage_idx in enumerate(range(2, max_stage_idx + 1)): + if stage_idx == 4: + dilation = res4_dilation + elif stage_idx == 5: + dilation = res5_dilation + else: + dilation = 1 + first_stride = 1 if idx == 0 or dilation > 1 else 2 + stage_kargs = { + "num_blocks": num_blocks_per_stage[idx], + "stride_per_block": [first_stride] + [1] * (num_blocks_per_stage[idx] - 1), + "in_channels": in_channels, + "out_channels": out_channels, + "norm": norm, + } + stage_kargs["bottleneck_channels"] = bottleneck_channels + stage_kargs["stride_in_1x1"] = stride_in_1x1 + stage_kargs["dilation"] = dilation + stage_kargs["num_groups"] = num_groups + if deform_on_per_stage[idx]: + stage_kargs["block_class"] = DeformBottleneckBlock + stage_kargs["deform_modulated"] = deform_modulated + stage_kargs["deform_num_groups"] = deform_num_groups + else: + stage_kargs["block_class"] = BottleneckBlock + if stage_idx == 5: + stage_kargs.pop("dilation") + stage_kargs["dilation_per_block"] = [dilation * mg for mg in res5_multi_grid] + blocks = ResNet.make_stage(**stage_kargs) + in_channels = out_channels + out_channels *= 2 + bottleneck_channels *= 2 + stages.append(blocks) + return ResNet(stem, stages, out_features=out_features).freeze(freeze_at) diff --git a/annotator/oneformer/detectron2/projects/deeplab/semantic_seg.py b/annotator/oneformer/detectron2/projects/deeplab/semantic_seg.py new file mode 100644 index 0000000000000000000000000000000000000000..36c2643397f6eeb5412ed333c7de79ded926a6d1 --- /dev/null +++ b/annotator/oneformer/detectron2/projects/deeplab/semantic_seg.py @@ -0,0 +1,348 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from typing import Callable, Dict, List, Optional, Tuple, Union +import fvcore.nn.weight_init as weight_init +import torch +from torch import nn +from torch.nn import functional as F + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.layers import ASPP, Conv2d, DepthwiseSeparableConv2d, ShapeSpec, get_norm +from annotator.oneformer.detectron2.modeling import SEM_SEG_HEADS_REGISTRY + +from .loss import DeepLabCE + + +@SEM_SEG_HEADS_REGISTRY.register() +class DeepLabV3PlusHead(nn.Module): + """ + A semantic segmentation head described in :paper:`DeepLabV3+`. + """ + + @configurable + def __init__( + self, + input_shape: Dict[str, ShapeSpec], + *, + project_channels: List[int], + aspp_dilations: List[int], + aspp_dropout: float, + decoder_channels: List[int], + common_stride: int, + norm: Union[str, Callable], + train_size: Optional[Tuple], + loss_weight: float = 1.0, + loss_type: str = "cross_entropy", + ignore_value: int = -1, + num_classes: Optional[int] = None, + use_depthwise_separable_conv: bool = False, + ): + """ + NOTE: this interface is experimental. + + Args: + input_shape: shape of the input features. They will be ordered by stride + and the last one (with largest stride) is used as the input to the + decoder (i.e. the ASPP module); the rest are low-level feature for + the intermediate levels of decoder. + project_channels (list[int]): a list of low-level feature channels. + The length should be len(in_features) - 1. + aspp_dilations (list(int)): a list of 3 dilations in ASPP. + aspp_dropout (float): apply dropout on the output of ASPP. + decoder_channels (list[int]): a list of output channels of each + decoder stage. It should have the same length as "in_features" + (each element in "in_features" corresponds to one decoder stage). + common_stride (int): output stride of decoder. + norm (str or callable): normalization for all conv layers. + train_size (tuple): (height, width) of training images. + loss_weight (float): loss weight. + loss_type (str): type of loss function, 2 opptions: + (1) "cross_entropy" is the standard cross entropy loss. + (2) "hard_pixel_mining" is the loss in DeepLab that samples + top k% hardest pixels. + ignore_value (int): category to be ignored during training. + num_classes (int): number of classes, if set to None, the decoder + will not construct a predictor. + use_depthwise_separable_conv (bool): use DepthwiseSeparableConv2d + in ASPP and decoder. + """ + super().__init__() + input_shape = sorted(input_shape.items(), key=lambda x: x[1].stride) + + # fmt: off + self.in_features = [k for k, v in input_shape] # starting from "res2" to "res5" + in_channels = [x[1].channels for x in input_shape] + in_strides = [x[1].stride for x in input_shape] + aspp_channels = decoder_channels[-1] + self.ignore_value = ignore_value + self.common_stride = common_stride # output stride + self.loss_weight = loss_weight + self.loss_type = loss_type + self.decoder_only = num_classes is None + self.use_depthwise_separable_conv = use_depthwise_separable_conv + # fmt: on + + assert ( + len(project_channels) == len(self.in_features) - 1 + ), "Expected {} project_channels, got {}".format( + len(self.in_features) - 1, len(project_channels) + ) + assert len(decoder_channels) == len( + self.in_features + ), "Expected {} decoder_channels, got {}".format( + len(self.in_features), len(decoder_channels) + ) + self.decoder = nn.ModuleDict() + + use_bias = norm == "" + for idx, in_channel in enumerate(in_channels): + decoder_stage = nn.ModuleDict() + + if idx == len(self.in_features) - 1: + # ASPP module + if train_size is not None: + train_h, train_w = train_size + encoder_stride = in_strides[-1] + if train_h % encoder_stride or train_w % encoder_stride: + raise ValueError("Crop size need to be divisible by encoder stride.") + pool_h = train_h // encoder_stride + pool_w = train_w // encoder_stride + pool_kernel_size = (pool_h, pool_w) + else: + pool_kernel_size = None + project_conv = ASPP( + in_channel, + aspp_channels, + aspp_dilations, + norm=norm, + activation=F.relu, + pool_kernel_size=pool_kernel_size, + dropout=aspp_dropout, + use_depthwise_separable_conv=use_depthwise_separable_conv, + ) + fuse_conv = None + else: + project_conv = Conv2d( + in_channel, + project_channels[idx], + kernel_size=1, + bias=use_bias, + norm=get_norm(norm, project_channels[idx]), + activation=F.relu, + ) + weight_init.c2_xavier_fill(project_conv) + if use_depthwise_separable_conv: + # We use a single 5x5 DepthwiseSeparableConv2d to replace + # 2 3x3 Conv2d since they have the same receptive field, + # proposed in :paper:`Panoptic-DeepLab`. + fuse_conv = DepthwiseSeparableConv2d( + project_channels[idx] + decoder_channels[idx + 1], + decoder_channels[idx], + kernel_size=5, + padding=2, + norm1=norm, + activation1=F.relu, + norm2=norm, + activation2=F.relu, + ) + else: + fuse_conv = nn.Sequential( + Conv2d( + project_channels[idx] + decoder_channels[idx + 1], + decoder_channels[idx], + kernel_size=3, + padding=1, + bias=use_bias, + norm=get_norm(norm, decoder_channels[idx]), + activation=F.relu, + ), + Conv2d( + decoder_channels[idx], + decoder_channels[idx], + kernel_size=3, + padding=1, + bias=use_bias, + norm=get_norm(norm, decoder_channels[idx]), + activation=F.relu, + ), + ) + weight_init.c2_xavier_fill(fuse_conv[0]) + weight_init.c2_xavier_fill(fuse_conv[1]) + + decoder_stage["project_conv"] = project_conv + decoder_stage["fuse_conv"] = fuse_conv + + self.decoder[self.in_features[idx]] = decoder_stage + + if not self.decoder_only: + self.predictor = Conv2d( + decoder_channels[0], num_classes, kernel_size=1, stride=1, padding=0 + ) + nn.init.normal_(self.predictor.weight, 0, 0.001) + nn.init.constant_(self.predictor.bias, 0) + + if self.loss_type == "cross_entropy": + self.loss = nn.CrossEntropyLoss(reduction="mean", ignore_index=self.ignore_value) + elif self.loss_type == "hard_pixel_mining": + self.loss = DeepLabCE(ignore_label=self.ignore_value, top_k_percent_pixels=0.2) + else: + raise ValueError("Unexpected loss type: %s" % self.loss_type) + + @classmethod + def from_config(cls, cfg, input_shape): + if cfg.INPUT.CROP.ENABLED: + assert cfg.INPUT.CROP.TYPE == "absolute" + train_size = cfg.INPUT.CROP.SIZE + else: + train_size = None + decoder_channels = [cfg.MODEL.SEM_SEG_HEAD.CONVS_DIM] * ( + len(cfg.MODEL.SEM_SEG_HEAD.IN_FEATURES) - 1 + ) + [cfg.MODEL.SEM_SEG_HEAD.ASPP_CHANNELS] + ret = dict( + input_shape={ + k: v for k, v in input_shape.items() if k in cfg.MODEL.SEM_SEG_HEAD.IN_FEATURES + }, + project_channels=cfg.MODEL.SEM_SEG_HEAD.PROJECT_CHANNELS, + aspp_dilations=cfg.MODEL.SEM_SEG_HEAD.ASPP_DILATIONS, + aspp_dropout=cfg.MODEL.SEM_SEG_HEAD.ASPP_DROPOUT, + decoder_channels=decoder_channels, + common_stride=cfg.MODEL.SEM_SEG_HEAD.COMMON_STRIDE, + norm=cfg.MODEL.SEM_SEG_HEAD.NORM, + train_size=train_size, + loss_weight=cfg.MODEL.SEM_SEG_HEAD.LOSS_WEIGHT, + loss_type=cfg.MODEL.SEM_SEG_HEAD.LOSS_TYPE, + ignore_value=cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE, + num_classes=cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES, + use_depthwise_separable_conv=cfg.MODEL.SEM_SEG_HEAD.USE_DEPTHWISE_SEPARABLE_CONV, + ) + return ret + + def forward(self, features, targets=None): + """ + Returns: + In training, returns (None, dict of losses) + In inference, returns (CxHxW logits, {}) + """ + y = self.layers(features) + if self.decoder_only: + # Output from self.layers() only contains decoder feature. + return y + if self.training: + return None, self.losses(y, targets) + else: + y = F.interpolate( + y, scale_factor=self.common_stride, mode="bilinear", align_corners=False + ) + return y, {} + + def layers(self, features): + # Reverse feature maps into top-down order (from low to high resolution) + for f in self.in_features[::-1]: + x = features[f] + proj_x = self.decoder[f]["project_conv"](x) + if self.decoder[f]["fuse_conv"] is None: + # This is aspp module + y = proj_x + else: + # Upsample y + y = F.interpolate(y, size=proj_x.size()[2:], mode="bilinear", align_corners=False) + y = torch.cat([proj_x, y], dim=1) + y = self.decoder[f]["fuse_conv"](y) + if not self.decoder_only: + y = self.predictor(y) + return y + + def losses(self, predictions, targets): + predictions = F.interpolate( + predictions, scale_factor=self.common_stride, mode="bilinear", align_corners=False + ) + loss = self.loss(predictions, targets) + losses = {"loss_sem_seg": loss * self.loss_weight} + return losses + + +@SEM_SEG_HEADS_REGISTRY.register() +class DeepLabV3Head(nn.Module): + """ + A semantic segmentation head described in :paper:`DeepLabV3`. + """ + + def __init__(self, cfg, input_shape: Dict[str, ShapeSpec]): + super().__init__() + + # fmt: off + self.in_features = cfg.MODEL.SEM_SEG_HEAD.IN_FEATURES + in_channels = [input_shape[f].channels for f in self.in_features] + aspp_channels = cfg.MODEL.SEM_SEG_HEAD.ASPP_CHANNELS + aspp_dilations = cfg.MODEL.SEM_SEG_HEAD.ASPP_DILATIONS + self.ignore_value = cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE + num_classes = cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES + conv_dims = cfg.MODEL.SEM_SEG_HEAD.CONVS_DIM + self.common_stride = cfg.MODEL.SEM_SEG_HEAD.COMMON_STRIDE # output stride + norm = cfg.MODEL.SEM_SEG_HEAD.NORM + self.loss_weight = cfg.MODEL.SEM_SEG_HEAD.LOSS_WEIGHT + self.loss_type = cfg.MODEL.SEM_SEG_HEAD.LOSS_TYPE + train_crop_size = cfg.INPUT.CROP.SIZE + aspp_dropout = cfg.MODEL.SEM_SEG_HEAD.ASPP_DROPOUT + use_depthwise_separable_conv = cfg.MODEL.SEM_SEG_HEAD.USE_DEPTHWISE_SEPARABLE_CONV + # fmt: on + + assert len(self.in_features) == 1 + assert len(in_channels) == 1 + + # ASPP module + if cfg.INPUT.CROP.ENABLED: + assert cfg.INPUT.CROP.TYPE == "absolute" + train_crop_h, train_crop_w = train_crop_size + if train_crop_h % self.common_stride or train_crop_w % self.common_stride: + raise ValueError("Crop size need to be divisible by output stride.") + pool_h = train_crop_h // self.common_stride + pool_w = train_crop_w // self.common_stride + pool_kernel_size = (pool_h, pool_w) + else: + pool_kernel_size = None + self.aspp = ASPP( + in_channels[0], + aspp_channels, + aspp_dilations, + norm=norm, + activation=F.relu, + pool_kernel_size=pool_kernel_size, + dropout=aspp_dropout, + use_depthwise_separable_conv=use_depthwise_separable_conv, + ) + + self.predictor = Conv2d(conv_dims, num_classes, kernel_size=1, stride=1, padding=0) + nn.init.normal_(self.predictor.weight, 0, 0.001) + nn.init.constant_(self.predictor.bias, 0) + + if self.loss_type == "cross_entropy": + self.loss = nn.CrossEntropyLoss(reduction="mean", ignore_index=self.ignore_value) + elif self.loss_type == "hard_pixel_mining": + self.loss = DeepLabCE(ignore_label=self.ignore_value, top_k_percent_pixels=0.2) + else: + raise ValueError("Unexpected loss type: %s" % self.loss_type) + + def forward(self, features, targets=None): + """ + Returns: + In training, returns (None, dict of losses) + In inference, returns (CxHxW logits, {}) + """ + x = features[self.in_features[0]] + x = self.aspp(x) + x = self.predictor(x) + if self.training: + return None, self.losses(x, targets) + else: + x = F.interpolate( + x, scale_factor=self.common_stride, mode="bilinear", align_corners=False + ) + return x, {} + + def losses(self, predictions, targets): + predictions = F.interpolate( + predictions, scale_factor=self.common_stride, mode="bilinear", align_corners=False + ) + loss = self.loss(predictions, targets) + losses = {"loss_sem_seg": loss * self.loss_weight} + return losses diff --git a/annotator/oneformer/detectron2/solver/__init__.py b/annotator/oneformer/detectron2/solver/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7e36c64f60f38f41d01dd2c9fb30364489a03841 --- /dev/null +++ b/annotator/oneformer/detectron2/solver/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .build import build_lr_scheduler, build_optimizer, get_default_optimizer_params +from .lr_scheduler import ( + LRMultiplier, + LRScheduler, + WarmupCosineLR, + WarmupMultiStepLR, + WarmupParamScheduler, +) + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/annotator/oneformer/detectron2/solver/build.py b/annotator/oneformer/detectron2/solver/build.py new file mode 100644 index 0000000000000000000000000000000000000000..5e526df1e05b1ad8943c18cc7a1e5e43c58d57c8 --- /dev/null +++ b/annotator/oneformer/detectron2/solver/build.py @@ -0,0 +1,310 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +import itertools +import logging +from collections import defaultdict +from enum import Enum +from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Type, Union +import torch +from fvcore.common.param_scheduler import ( + CosineParamScheduler, + MultiStepParamScheduler, + StepWithFixedGammaParamScheduler, +) + +from annotator.oneformer.detectron2.config import CfgNode +from annotator.oneformer.detectron2.utils.env import TORCH_VERSION + +from .lr_scheduler import LRMultiplier, LRScheduler, WarmupParamScheduler + +_GradientClipperInput = Union[torch.Tensor, Iterable[torch.Tensor]] +_GradientClipper = Callable[[_GradientClipperInput], None] + + +class GradientClipType(Enum): + VALUE = "value" + NORM = "norm" + + +def _create_gradient_clipper(cfg: CfgNode) -> _GradientClipper: + """ + Creates gradient clipping closure to clip by value or by norm, + according to the provided config. + """ + cfg = copy.deepcopy(cfg) + + def clip_grad_norm(p: _GradientClipperInput): + torch.nn.utils.clip_grad_norm_(p, cfg.CLIP_VALUE, cfg.NORM_TYPE) + + def clip_grad_value(p: _GradientClipperInput): + torch.nn.utils.clip_grad_value_(p, cfg.CLIP_VALUE) + + _GRADIENT_CLIP_TYPE_TO_CLIPPER = { + GradientClipType.VALUE: clip_grad_value, + GradientClipType.NORM: clip_grad_norm, + } + return _GRADIENT_CLIP_TYPE_TO_CLIPPER[GradientClipType(cfg.CLIP_TYPE)] + + +def _generate_optimizer_class_with_gradient_clipping( + optimizer: Type[torch.optim.Optimizer], + *, + per_param_clipper: Optional[_GradientClipper] = None, + global_clipper: Optional[_GradientClipper] = None, +) -> Type[torch.optim.Optimizer]: + """ + Dynamically creates a new type that inherits the type of a given instance + and overrides the `step` method to add gradient clipping + """ + assert ( + per_param_clipper is None or global_clipper is None + ), "Not allowed to use both per-parameter clipping and global clipping" + + def optimizer_wgc_step(self, closure=None): + if per_param_clipper is not None: + for group in self.param_groups: + for p in group["params"]: + per_param_clipper(p) + else: + # global clipper for future use with detr + # (https://github.com/facebookresearch/detr/pull/287) + all_params = itertools.chain(*[g["params"] for g in self.param_groups]) + global_clipper(all_params) + super(type(self), self).step(closure) + + OptimizerWithGradientClip = type( + optimizer.__name__ + "WithGradientClip", + (optimizer,), + {"step": optimizer_wgc_step}, + ) + return OptimizerWithGradientClip + + +def maybe_add_gradient_clipping( + cfg: CfgNode, optimizer: Type[torch.optim.Optimizer] +) -> Type[torch.optim.Optimizer]: + """ + If gradient clipping is enabled through config options, wraps the existing + optimizer type to become a new dynamically created class OptimizerWithGradientClip + that inherits the given optimizer and overrides the `step` method to + include gradient clipping. + + Args: + cfg: CfgNode, configuration options + optimizer: type. A subclass of torch.optim.Optimizer + + Return: + type: either the input `optimizer` (if gradient clipping is disabled), or + a subclass of it with gradient clipping included in the `step` method. + """ + if not cfg.SOLVER.CLIP_GRADIENTS.ENABLED: + return optimizer + if isinstance(optimizer, torch.optim.Optimizer): + optimizer_type = type(optimizer) + else: + assert issubclass(optimizer, torch.optim.Optimizer), optimizer + optimizer_type = optimizer + + grad_clipper = _create_gradient_clipper(cfg.SOLVER.CLIP_GRADIENTS) + OptimizerWithGradientClip = _generate_optimizer_class_with_gradient_clipping( + optimizer_type, per_param_clipper=grad_clipper + ) + if isinstance(optimizer, torch.optim.Optimizer): + optimizer.__class__ = OptimizerWithGradientClip # a bit hacky, not recommended + return optimizer + else: + return OptimizerWithGradientClip + + +def build_optimizer(cfg: CfgNode, model: torch.nn.Module) -> torch.optim.Optimizer: + """ + Build an optimizer from config. + """ + params = get_default_optimizer_params( + model, + base_lr=cfg.SOLVER.BASE_LR, + weight_decay_norm=cfg.SOLVER.WEIGHT_DECAY_NORM, + bias_lr_factor=cfg.SOLVER.BIAS_LR_FACTOR, + weight_decay_bias=cfg.SOLVER.WEIGHT_DECAY_BIAS, + ) + sgd_args = { + "params": params, + "lr": cfg.SOLVER.BASE_LR, + "momentum": cfg.SOLVER.MOMENTUM, + "nesterov": cfg.SOLVER.NESTEROV, + "weight_decay": cfg.SOLVER.WEIGHT_DECAY, + } + if TORCH_VERSION >= (1, 12): + sgd_args["foreach"] = True + return maybe_add_gradient_clipping(cfg, torch.optim.SGD(**sgd_args)) + + +def get_default_optimizer_params( + model: torch.nn.Module, + base_lr: Optional[float] = None, + weight_decay: Optional[float] = None, + weight_decay_norm: Optional[float] = None, + bias_lr_factor: Optional[float] = 1.0, + weight_decay_bias: Optional[float] = None, + lr_factor_func: Optional[Callable] = None, + overrides: Optional[Dict[str, Dict[str, float]]] = None, +) -> List[Dict[str, Any]]: + """ + Get default param list for optimizer, with support for a few types of + overrides. If no overrides needed, this is equivalent to `model.parameters()`. + + Args: + base_lr: lr for every group by default. Can be omitted to use the one in optimizer. + weight_decay: weight decay for every group by default. Can be omitted to use the one + in optimizer. + weight_decay_norm: override weight decay for params in normalization layers + bias_lr_factor: multiplier of lr for bias parameters. + weight_decay_bias: override weight decay for bias parameters. + lr_factor_func: function to calculate lr decay rate by mapping the parameter names to + corresponding lr decay rate. Note that setting this option requires + also setting ``base_lr``. + overrides: if not `None`, provides values for optimizer hyperparameters + (LR, weight decay) for module parameters with a given name; e.g. + ``{"embedding": {"lr": 0.01, "weight_decay": 0.1}}`` will set the LR and + weight decay values for all module parameters named `embedding`. + + For common detection models, ``weight_decay_norm`` is the only option + needed to be set. ``bias_lr_factor,weight_decay_bias`` are legacy settings + from Detectron1 that are not found useful. + + Example: + :: + torch.optim.SGD(get_default_optimizer_params(model, weight_decay_norm=0), + lr=0.01, weight_decay=1e-4, momentum=0.9) + """ + if overrides is None: + overrides = {} + defaults = {} + if base_lr is not None: + defaults["lr"] = base_lr + if weight_decay is not None: + defaults["weight_decay"] = weight_decay + bias_overrides = {} + if bias_lr_factor is not None and bias_lr_factor != 1.0: + # NOTE: unlike Detectron v1, we now by default make bias hyperparameters + # exactly the same as regular weights. + if base_lr is None: + raise ValueError("bias_lr_factor requires base_lr") + bias_overrides["lr"] = base_lr * bias_lr_factor + if weight_decay_bias is not None: + bias_overrides["weight_decay"] = weight_decay_bias + if len(bias_overrides): + if "bias" in overrides: + raise ValueError("Conflicting overrides for 'bias'") + overrides["bias"] = bias_overrides + if lr_factor_func is not None: + if base_lr is None: + raise ValueError("lr_factor_func requires base_lr") + norm_module_types = ( + torch.nn.BatchNorm1d, + torch.nn.BatchNorm2d, + torch.nn.BatchNorm3d, + torch.nn.SyncBatchNorm, + # NaiveSyncBatchNorm inherits from BatchNorm2d + torch.nn.GroupNorm, + torch.nn.InstanceNorm1d, + torch.nn.InstanceNorm2d, + torch.nn.InstanceNorm3d, + torch.nn.LayerNorm, + torch.nn.LocalResponseNorm, + ) + params: List[Dict[str, Any]] = [] + memo: Set[torch.nn.parameter.Parameter] = set() + for module_name, module in model.named_modules(): + for module_param_name, value in module.named_parameters(recurse=False): + if not value.requires_grad: + continue + # Avoid duplicating parameters + if value in memo: + continue + memo.add(value) + + hyperparams = copy.copy(defaults) + if isinstance(module, norm_module_types) and weight_decay_norm is not None: + hyperparams["weight_decay"] = weight_decay_norm + if lr_factor_func is not None: + hyperparams["lr"] *= lr_factor_func(f"{module_name}.{module_param_name}") + + hyperparams.update(overrides.get(module_param_name, {})) + params.append({"params": [value], **hyperparams}) + return reduce_param_groups(params) + + +def _expand_param_groups(params: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + # Transform parameter groups into per-parameter structure. + # Later items in `params` can overwrite parameters set in previous items. + ret = defaultdict(dict) + for item in params: + assert "params" in item + cur_params = {x: y for x, y in item.items() if x != "params"} + for param in item["params"]: + ret[param].update({"params": [param], **cur_params}) + return list(ret.values()) + + +def reduce_param_groups(params: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + # Reorganize the parameter groups and merge duplicated groups. + # The number of parameter groups needs to be as small as possible in order + # to efficiently use the PyTorch multi-tensor optimizer. Therefore instead + # of using a parameter_group per single parameter, we reorganize the + # parameter groups and merge duplicated groups. This approach speeds + # up multi-tensor optimizer significantly. + params = _expand_param_groups(params) + groups = defaultdict(list) # re-group all parameter groups by their hyperparams + for item in params: + cur_params = tuple((x, y) for x, y in item.items() if x != "params") + groups[cur_params].extend(item["params"]) + ret = [] + for param_keys, param_values in groups.items(): + cur = {kv[0]: kv[1] for kv in param_keys} + cur["params"] = param_values + ret.append(cur) + return ret + + +def build_lr_scheduler(cfg: CfgNode, optimizer: torch.optim.Optimizer) -> LRScheduler: + """ + Build a LR scheduler from config. + """ + name = cfg.SOLVER.LR_SCHEDULER_NAME + + if name == "WarmupMultiStepLR": + steps = [x for x in cfg.SOLVER.STEPS if x <= cfg.SOLVER.MAX_ITER] + if len(steps) != len(cfg.SOLVER.STEPS): + logger = logging.getLogger(__name__) + logger.warning( + "SOLVER.STEPS contains values larger than SOLVER.MAX_ITER. " + "These values will be ignored." + ) + sched = MultiStepParamScheduler( + values=[cfg.SOLVER.GAMMA**k for k in range(len(steps) + 1)], + milestones=steps, + num_updates=cfg.SOLVER.MAX_ITER, + ) + elif name == "WarmupCosineLR": + end_value = cfg.SOLVER.BASE_LR_END / cfg.SOLVER.BASE_LR + assert end_value >= 0.0 and end_value <= 1.0, end_value + sched = CosineParamScheduler(1, end_value) + elif name == "WarmupStepWithFixedGammaLR": + sched = StepWithFixedGammaParamScheduler( + base_value=1.0, + gamma=cfg.SOLVER.GAMMA, + num_decays=cfg.SOLVER.NUM_DECAYS, + num_updates=cfg.SOLVER.MAX_ITER, + ) + else: + raise ValueError("Unknown LR scheduler: {}".format(name)) + + sched = WarmupParamScheduler( + sched, + cfg.SOLVER.WARMUP_FACTOR, + min(cfg.SOLVER.WARMUP_ITERS / cfg.SOLVER.MAX_ITER, 1.0), + cfg.SOLVER.WARMUP_METHOD, + cfg.SOLVER.RESCALE_INTERVAL, + ) + return LRMultiplier(optimizer, multiplier=sched, max_iter=cfg.SOLVER.MAX_ITER) diff --git a/annotator/oneformer/detectron2/solver/lr_scheduler.py b/annotator/oneformer/detectron2/solver/lr_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..d6aed2bb20c418bf6cc5594c1244b241796d7086 --- /dev/null +++ b/annotator/oneformer/detectron2/solver/lr_scheduler.py @@ -0,0 +1,246 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import math +from bisect import bisect_right +from typing import List +import torch +from fvcore.common.param_scheduler import ( + CompositeParamScheduler, + ConstantParamScheduler, + LinearParamScheduler, + ParamScheduler, +) + +try: + from torch.optim.lr_scheduler import LRScheduler +except ImportError: + from torch.optim.lr_scheduler import _LRScheduler as LRScheduler + +logger = logging.getLogger(__name__) + + +class WarmupParamScheduler(CompositeParamScheduler): + """ + Add an initial warmup stage to another scheduler. + """ + + def __init__( + self, + scheduler: ParamScheduler, + warmup_factor: float, + warmup_length: float, + warmup_method: str = "linear", + rescale_interval: bool = False, + ): + """ + Args: + scheduler: warmup will be added at the beginning of this scheduler + warmup_factor: the factor w.r.t the initial value of ``scheduler``, e.g. 0.001 + warmup_length: the relative length (in [0, 1]) of warmup steps w.r.t the entire + training, e.g. 0.01 + warmup_method: one of "linear" or "constant" + rescale_interval: whether we will rescale the interval of the scheduler after + warmup + """ + end_value = scheduler(warmup_length) # the value to reach when warmup ends + start_value = warmup_factor * scheduler(0.0) + if warmup_method == "constant": + warmup = ConstantParamScheduler(start_value) + elif warmup_method == "linear": + warmup = LinearParamScheduler(start_value, end_value) + else: + raise ValueError("Unknown warmup method: {}".format(warmup_method)) + super().__init__( + [warmup, scheduler], + interval_scaling=["rescaled", "rescaled" if rescale_interval else "fixed"], + lengths=[warmup_length, 1 - warmup_length], + ) + + +class LRMultiplier(LRScheduler): + """ + A LRScheduler which uses fvcore :class:`ParamScheduler` to multiply the + learning rate of each param in the optimizer. + Every step, the learning rate of each parameter becomes its initial value + multiplied by the output of the given :class:`ParamScheduler`. + + The absolute learning rate value of each parameter can be different. + This scheduler can be used as long as the relative scale among them do + not change during training. + + Examples: + :: + LRMultiplier( + opt, + WarmupParamScheduler( + MultiStepParamScheduler( + [1, 0.1, 0.01], + milestones=[60000, 80000], + num_updates=90000, + ), 0.001, 100 / 90000 + ), + max_iter=90000 + ) + """ + + # NOTES: in the most general case, every LR can use its own scheduler. + # Supporting this requires interaction with the optimizer when its parameter + # group is initialized. For example, classyvision implements its own optimizer + # that allows different schedulers for every parameter group. + # To avoid this complexity, we use this class to support the most common cases + # where the relative scale among all LRs stay unchanged during training. In this + # case we only need a total of one scheduler that defines the relative LR multiplier. + + def __init__( + self, + optimizer: torch.optim.Optimizer, + multiplier: ParamScheduler, + max_iter: int, + last_iter: int = -1, + ): + """ + Args: + optimizer, last_iter: See ``torch.optim.lr_scheduler.LRScheduler``. + ``last_iter`` is the same as ``last_epoch``. + multiplier: a fvcore ParamScheduler that defines the multiplier on + every LR of the optimizer + max_iter: the total number of training iterations + """ + if not isinstance(multiplier, ParamScheduler): + raise ValueError( + "_LRMultiplier(multiplier=) must be an instance of fvcore " + f"ParamScheduler. Got {multiplier} instead." + ) + self._multiplier = multiplier + self._max_iter = max_iter + super().__init__(optimizer, last_epoch=last_iter) + + def state_dict(self): + # fvcore schedulers are stateless. Only keep pytorch scheduler states + return {"base_lrs": self.base_lrs, "last_epoch": self.last_epoch} + + def get_lr(self) -> List[float]: + multiplier = self._multiplier(self.last_epoch / self._max_iter) + return [base_lr * multiplier for base_lr in self.base_lrs] + + +""" +Content below is no longer needed! +""" + +# NOTE: PyTorch's LR scheduler interface uses names that assume the LR changes +# only on epoch boundaries. We typically use iteration based schedules instead. +# As a result, "epoch" (e.g., as in self.last_epoch) should be understood to mean +# "iteration" instead. + +# FIXME: ideally this would be achieved with a CombinedLRScheduler, separating +# MultiStepLR with WarmupLR but the current LRScheduler design doesn't allow it. + + +class WarmupMultiStepLR(LRScheduler): + def __init__( + self, + optimizer: torch.optim.Optimizer, + milestones: List[int], + gamma: float = 0.1, + warmup_factor: float = 0.001, + warmup_iters: int = 1000, + warmup_method: str = "linear", + last_epoch: int = -1, + ): + logger.warning( + "WarmupMultiStepLR is deprecated! Use LRMultipilier with fvcore ParamScheduler instead!" + ) + if not list(milestones) == sorted(milestones): + raise ValueError( + "Milestones should be a list of" " increasing integers. Got {}", milestones + ) + self.milestones = milestones + self.gamma = gamma + self.warmup_factor = warmup_factor + self.warmup_iters = warmup_iters + self.warmup_method = warmup_method + super().__init__(optimizer, last_epoch) + + def get_lr(self) -> List[float]: + warmup_factor = _get_warmup_factor_at_iter( + self.warmup_method, self.last_epoch, self.warmup_iters, self.warmup_factor + ) + return [ + base_lr * warmup_factor * self.gamma ** bisect_right(self.milestones, self.last_epoch) + for base_lr in self.base_lrs + ] + + def _compute_values(self) -> List[float]: + # The new interface + return self.get_lr() + + +class WarmupCosineLR(LRScheduler): + def __init__( + self, + optimizer: torch.optim.Optimizer, + max_iters: int, + warmup_factor: float = 0.001, + warmup_iters: int = 1000, + warmup_method: str = "linear", + last_epoch: int = -1, + ): + logger.warning( + "WarmupCosineLR is deprecated! Use LRMultipilier with fvcore ParamScheduler instead!" + ) + self.max_iters = max_iters + self.warmup_factor = warmup_factor + self.warmup_iters = warmup_iters + self.warmup_method = warmup_method + super().__init__(optimizer, last_epoch) + + def get_lr(self) -> List[float]: + warmup_factor = _get_warmup_factor_at_iter( + self.warmup_method, self.last_epoch, self.warmup_iters, self.warmup_factor + ) + # Different definitions of half-cosine with warmup are possible. For + # simplicity we multiply the standard half-cosine schedule by the warmup + # factor. An alternative is to start the period of the cosine at warmup_iters + # instead of at 0. In the case that warmup_iters << max_iters the two are + # very close to each other. + return [ + base_lr + * warmup_factor + * 0.5 + * (1.0 + math.cos(math.pi * self.last_epoch / self.max_iters)) + for base_lr in self.base_lrs + ] + + def _compute_values(self) -> List[float]: + # The new interface + return self.get_lr() + + +def _get_warmup_factor_at_iter( + method: str, iter: int, warmup_iters: int, warmup_factor: float +) -> float: + """ + Return the learning rate warmup factor at a specific iteration. + See :paper:`ImageNet in 1h` for more details. + + Args: + method (str): warmup method; either "constant" or "linear". + iter (int): iteration at which to calculate the warmup factor. + warmup_iters (int): the number of warmup iterations. + warmup_factor (float): the base warmup factor (the meaning changes according + to the method used). + + Returns: + float: the effective warmup factor at the given iteration. + """ + if iter >= warmup_iters: + return 1.0 + + if method == "constant": + return warmup_factor + elif method == "linear": + alpha = iter / warmup_iters + return warmup_factor * (1 - alpha) + alpha + else: + raise ValueError("Unknown warmup method: {}".format(method)) diff --git a/annotator/oneformer/detectron2/structures/__init__.py b/annotator/oneformer/detectron2/structures/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c2942fc58e3fce82e690eafc2de0204816e94cc2 --- /dev/null +++ b/annotator/oneformer/detectron2/structures/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .boxes import Boxes, BoxMode, pairwise_iou, pairwise_ioa, pairwise_point_box_distance +from .image_list import ImageList + +from .instances import Instances +from .keypoints import Keypoints, heatmaps_to_keypoints +from .masks import BitMasks, PolygonMasks, polygons_to_bitmask, ROIMasks +from .rotated_boxes import RotatedBoxes +from .rotated_boxes import pairwise_iou as pairwise_iou_rotated + +__all__ = [k for k in globals().keys() if not k.startswith("_")] + + +from annotator.oneformer.detectron2.utils.env import fixup_module_metadata + +fixup_module_metadata(__name__, globals(), __all__) +del fixup_module_metadata diff --git a/annotator/oneformer/detectron2/structures/boxes.py b/annotator/oneformer/detectron2/structures/boxes.py new file mode 100644 index 0000000000000000000000000000000000000000..fd396f68645db1d6946056eed868ffcc02cd7a22 --- /dev/null +++ b/annotator/oneformer/detectron2/structures/boxes.py @@ -0,0 +1,425 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import math +import numpy as np +from enum import IntEnum, unique +from typing import List, Tuple, Union +import torch +from torch import device + +_RawBoxType = Union[List[float], Tuple[float, ...], torch.Tensor, np.ndarray] + + +@unique +class BoxMode(IntEnum): + """ + Enum of different ways to represent a box. + """ + + XYXY_ABS = 0 + """ + (x0, y0, x1, y1) in absolute floating points coordinates. + The coordinates in range [0, width or height]. + """ + XYWH_ABS = 1 + """ + (x0, y0, w, h) in absolute floating points coordinates. + """ + XYXY_REL = 2 + """ + Not yet supported! + (x0, y0, x1, y1) in range [0, 1]. They are relative to the size of the image. + """ + XYWH_REL = 3 + """ + Not yet supported! + (x0, y0, w, h) in range [0, 1]. They are relative to the size of the image. + """ + XYWHA_ABS = 4 + """ + (xc, yc, w, h, a) in absolute floating points coordinates. + (xc, yc) is the center of the rotated box, and the angle a is in degrees ccw. + """ + + @staticmethod + def convert(box: _RawBoxType, from_mode: "BoxMode", to_mode: "BoxMode") -> _RawBoxType: + """ + Args: + box: can be a k-tuple, k-list or an Nxk array/tensor, where k = 4 or 5 + from_mode, to_mode (BoxMode) + + Returns: + The converted box of the same type. + """ + if from_mode == to_mode: + return box + + original_type = type(box) + is_numpy = isinstance(box, np.ndarray) + single_box = isinstance(box, (list, tuple)) + if single_box: + assert len(box) == 4 or len(box) == 5, ( + "BoxMode.convert takes either a k-tuple/list or an Nxk array/tensor," + " where k == 4 or 5" + ) + arr = torch.tensor(box)[None, :] + else: + # avoid modifying the input box + if is_numpy: + arr = torch.from_numpy(np.asarray(box)).clone() + else: + arr = box.clone() + + assert to_mode not in [BoxMode.XYXY_REL, BoxMode.XYWH_REL] and from_mode not in [ + BoxMode.XYXY_REL, + BoxMode.XYWH_REL, + ], "Relative mode not yet supported!" + + if from_mode == BoxMode.XYWHA_ABS and to_mode == BoxMode.XYXY_ABS: + assert ( + arr.shape[-1] == 5 + ), "The last dimension of input shape must be 5 for XYWHA format" + original_dtype = arr.dtype + arr = arr.double() + + w = arr[:, 2] + h = arr[:, 3] + a = arr[:, 4] + c = torch.abs(torch.cos(a * math.pi / 180.0)) + s = torch.abs(torch.sin(a * math.pi / 180.0)) + # This basically computes the horizontal bounding rectangle of the rotated box + new_w = c * w + s * h + new_h = c * h + s * w + + # convert center to top-left corner + arr[:, 0] -= new_w / 2.0 + arr[:, 1] -= new_h / 2.0 + # bottom-right corner + arr[:, 2] = arr[:, 0] + new_w + arr[:, 3] = arr[:, 1] + new_h + + arr = arr[:, :4].to(dtype=original_dtype) + elif from_mode == BoxMode.XYWH_ABS and to_mode == BoxMode.XYWHA_ABS: + original_dtype = arr.dtype + arr = arr.double() + arr[:, 0] += arr[:, 2] / 2.0 + arr[:, 1] += arr[:, 3] / 2.0 + angles = torch.zeros((arr.shape[0], 1), dtype=arr.dtype) + arr = torch.cat((arr, angles), axis=1).to(dtype=original_dtype) + else: + if to_mode == BoxMode.XYXY_ABS and from_mode == BoxMode.XYWH_ABS: + arr[:, 2] += arr[:, 0] + arr[:, 3] += arr[:, 1] + elif from_mode == BoxMode.XYXY_ABS and to_mode == BoxMode.XYWH_ABS: + arr[:, 2] -= arr[:, 0] + arr[:, 3] -= arr[:, 1] + else: + raise NotImplementedError( + "Conversion from BoxMode {} to {} is not supported yet".format( + from_mode, to_mode + ) + ) + + if single_box: + return original_type(arr.flatten().tolist()) + if is_numpy: + return arr.numpy() + else: + return arr + + +class Boxes: + """ + This structure stores a list of boxes as a Nx4 torch.Tensor. + It supports some common methods about boxes + (`area`, `clip`, `nonempty`, etc), + and also behaves like a Tensor + (support indexing, `to(device)`, `.device`, and iteration over all boxes) + + Attributes: + tensor (torch.Tensor): float matrix of Nx4. Each row is (x1, y1, x2, y2). + """ + + def __init__(self, tensor: torch.Tensor): + """ + Args: + tensor (Tensor[float]): a Nx4 matrix. Each row is (x1, y1, x2, y2). + """ + if not isinstance(tensor, torch.Tensor): + tensor = torch.as_tensor(tensor, dtype=torch.float32, device=torch.device("cpu")) + else: + tensor = tensor.to(torch.float32) + if tensor.numel() == 0: + # Use reshape, so we don't end up creating a new tensor that does not depend on + # the inputs (and consequently confuses jit) + tensor = tensor.reshape((-1, 4)).to(dtype=torch.float32) + assert tensor.dim() == 2 and tensor.size(-1) == 4, tensor.size() + + self.tensor = tensor + + def clone(self) -> "Boxes": + """ + Clone the Boxes. + + Returns: + Boxes + """ + return Boxes(self.tensor.clone()) + + def to(self, device: torch.device): + # Boxes are assumed float32 and does not support to(dtype) + return Boxes(self.tensor.to(device=device)) + + def area(self) -> torch.Tensor: + """ + Computes the area of all the boxes. + + Returns: + torch.Tensor: a vector with areas of each box. + """ + box = self.tensor + area = (box[:, 2] - box[:, 0]) * (box[:, 3] - box[:, 1]) + return area + + def clip(self, box_size: Tuple[int, int]) -> None: + """ + Clip (in place) the boxes by limiting x coordinates to the range [0, width] + and y coordinates to the range [0, height]. + + Args: + box_size (height, width): The clipping box's size. + """ + assert torch.isfinite(self.tensor).all(), "Box tensor contains infinite or NaN!" + h, w = box_size + x1 = self.tensor[:, 0].clamp(min=0, max=w) + y1 = self.tensor[:, 1].clamp(min=0, max=h) + x2 = self.tensor[:, 2].clamp(min=0, max=w) + y2 = self.tensor[:, 3].clamp(min=0, max=h) + self.tensor = torch.stack((x1, y1, x2, y2), dim=-1) + + def nonempty(self, threshold: float = 0.0) -> torch.Tensor: + """ + Find boxes that are non-empty. + A box is considered empty, if either of its side is no larger than threshold. + + Returns: + Tensor: + a binary vector which represents whether each box is empty + (False) or non-empty (True). + """ + box = self.tensor + widths = box[:, 2] - box[:, 0] + heights = box[:, 3] - box[:, 1] + keep = (widths > threshold) & (heights > threshold) + return keep + + def __getitem__(self, item) -> "Boxes": + """ + Args: + item: int, slice, or a BoolTensor + + Returns: + Boxes: Create a new :class:`Boxes` by indexing. + + The following usage are allowed: + + 1. `new_boxes = boxes[3]`: return a `Boxes` which contains only one box. + 2. `new_boxes = boxes[2:10]`: return a slice of boxes. + 3. `new_boxes = boxes[vector]`, where vector is a torch.BoolTensor + with `length = len(boxes)`. Nonzero elements in the vector will be selected. + + Note that the returned Boxes might share storage with this Boxes, + subject to Pytorch's indexing semantics. + """ + if isinstance(item, int): + return Boxes(self.tensor[item].view(1, -1)) + b = self.tensor[item] + assert b.dim() == 2, "Indexing on Boxes with {} failed to return a matrix!".format(item) + return Boxes(b) + + def __len__(self) -> int: + return self.tensor.shape[0] + + def __repr__(self) -> str: + return "Boxes(" + str(self.tensor) + ")" + + def inside_box(self, box_size: Tuple[int, int], boundary_threshold: int = 0) -> torch.Tensor: + """ + Args: + box_size (height, width): Size of the reference box. + boundary_threshold (int): Boxes that extend beyond the reference box + boundary by more than boundary_threshold are considered "outside". + + Returns: + a binary vector, indicating whether each box is inside the reference box. + """ + height, width = box_size + inds_inside = ( + (self.tensor[..., 0] >= -boundary_threshold) + & (self.tensor[..., 1] >= -boundary_threshold) + & (self.tensor[..., 2] < width + boundary_threshold) + & (self.tensor[..., 3] < height + boundary_threshold) + ) + return inds_inside + + def get_centers(self) -> torch.Tensor: + """ + Returns: + The box centers in a Nx2 array of (x, y). + """ + return (self.tensor[:, :2] + self.tensor[:, 2:]) / 2 + + def scale(self, scale_x: float, scale_y: float) -> None: + """ + Scale the box with horizontal and vertical scaling factors + """ + self.tensor[:, 0::2] *= scale_x + self.tensor[:, 1::2] *= scale_y + + @classmethod + def cat(cls, boxes_list: List["Boxes"]) -> "Boxes": + """ + Concatenates a list of Boxes into a single Boxes + + Arguments: + boxes_list (list[Boxes]) + + Returns: + Boxes: the concatenated Boxes + """ + assert isinstance(boxes_list, (list, tuple)) + if len(boxes_list) == 0: + return cls(torch.empty(0)) + assert all([isinstance(box, Boxes) for box in boxes_list]) + + # use torch.cat (v.s. layers.cat) so the returned boxes never share storage with input + cat_boxes = cls(torch.cat([b.tensor for b in boxes_list], dim=0)) + return cat_boxes + + @property + def device(self) -> device: + return self.tensor.device + + # type "Iterator[torch.Tensor]", yield, and iter() not supported by torchscript + # https://github.com/pytorch/pytorch/issues/18627 + @torch.jit.unused + def __iter__(self): + """ + Yield a box as a Tensor of shape (4,) at a time. + """ + yield from self.tensor + + +def pairwise_intersection(boxes1: Boxes, boxes2: Boxes) -> torch.Tensor: + """ + Given two lists of boxes of size N and M, + compute the intersection area between __all__ N x M pairs of boxes. + The box order must be (xmin, ymin, xmax, ymax) + + Args: + boxes1,boxes2 (Boxes): two `Boxes`. Contains N & M boxes, respectively. + + Returns: + Tensor: intersection, sized [N,M]. + """ + boxes1, boxes2 = boxes1.tensor, boxes2.tensor + width_height = torch.min(boxes1[:, None, 2:], boxes2[:, 2:]) - torch.max( + boxes1[:, None, :2], boxes2[:, :2] + ) # [N,M,2] + + width_height.clamp_(min=0) # [N,M,2] + intersection = width_height.prod(dim=2) # [N,M] + return intersection + + +# implementation from https://github.com/kuangliu/torchcv/blob/master/torchcv/utils/box.py +# with slight modifications +def pairwise_iou(boxes1: Boxes, boxes2: Boxes) -> torch.Tensor: + """ + Given two lists of boxes of size N and M, compute the IoU + (intersection over union) between **all** N x M pairs of boxes. + The box order must be (xmin, ymin, xmax, ymax). + + Args: + boxes1,boxes2 (Boxes): two `Boxes`. Contains N & M boxes, respectively. + + Returns: + Tensor: IoU, sized [N,M]. + """ + area1 = boxes1.area() # [N] + area2 = boxes2.area() # [M] + inter = pairwise_intersection(boxes1, boxes2) + + # handle empty boxes + iou = torch.where( + inter > 0, + inter / (area1[:, None] + area2 - inter), + torch.zeros(1, dtype=inter.dtype, device=inter.device), + ) + return iou + + +def pairwise_ioa(boxes1: Boxes, boxes2: Boxes) -> torch.Tensor: + """ + Similar to :func:`pariwise_iou` but compute the IoA (intersection over boxes2 area). + + Args: + boxes1,boxes2 (Boxes): two `Boxes`. Contains N & M boxes, respectively. + + Returns: + Tensor: IoA, sized [N,M]. + """ + area2 = boxes2.area() # [M] + inter = pairwise_intersection(boxes1, boxes2) + + # handle empty boxes + ioa = torch.where( + inter > 0, inter / area2, torch.zeros(1, dtype=inter.dtype, device=inter.device) + ) + return ioa + + +def pairwise_point_box_distance(points: torch.Tensor, boxes: Boxes): + """ + Pairwise distance between N points and M boxes. The distance between a + point and a box is represented by the distance from the point to 4 edges + of the box. Distances are all positive when the point is inside the box. + + Args: + points: Nx2 coordinates. Each row is (x, y) + boxes: M boxes + + Returns: + Tensor: distances of size (N, M, 4). The 4 values are distances from + the point to the left, top, right, bottom of the box. + """ + x, y = points.unsqueeze(dim=2).unbind(dim=1) # (N, 1) + x0, y0, x1, y1 = boxes.tensor.unsqueeze(dim=0).unbind(dim=2) # (1, M) + return torch.stack([x - x0, y - y0, x1 - x, y1 - y], dim=2) + + +def matched_pairwise_iou(boxes1: Boxes, boxes2: Boxes) -> torch.Tensor: + """ + Compute pairwise intersection over union (IOU) of two sets of matched + boxes that have the same number of boxes. + Similar to :func:`pairwise_iou`, but computes only diagonal elements of the matrix. + + Args: + boxes1 (Boxes): bounding boxes, sized [N,4]. + boxes2 (Boxes): same length as boxes1 + Returns: + Tensor: iou, sized [N]. + """ + assert len(boxes1) == len( + boxes2 + ), "boxlists should have the same" "number of entries, got {}, {}".format( + len(boxes1), len(boxes2) + ) + area1 = boxes1.area() # [N] + area2 = boxes2.area() # [N] + box1, box2 = boxes1.tensor, boxes2.tensor + lt = torch.max(box1[:, :2], box2[:, :2]) # [N,2] + rb = torch.min(box1[:, 2:], box2[:, 2:]) # [N,2] + wh = (rb - lt).clamp(min=0) # [N,2] + inter = wh[:, 0] * wh[:, 1] # [N] + iou = inter / (area1 + area2 - inter) # [N] + return iou diff --git a/annotator/oneformer/detectron2/structures/image_list.py b/annotator/oneformer/detectron2/structures/image_list.py new file mode 100644 index 0000000000000000000000000000000000000000..86c8b9512a5fd8abda7fdf058a63b19f809e46f6 --- /dev/null +++ b/annotator/oneformer/detectron2/structures/image_list.py @@ -0,0 +1,129 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from __future__ import division +from typing import Any, Dict, List, Optional, Tuple +import torch +from torch import device +from torch.nn import functional as F + +from annotator.oneformer.detectron2.layers.wrappers import move_device_like, shapes_to_tensor + + +class ImageList(object): + """ + Structure that holds a list of images (of possibly + varying sizes) as a single tensor. + This works by padding the images to the same size. + The original sizes of each image is stored in `image_sizes`. + + Attributes: + image_sizes (list[tuple[int, int]]): each tuple is (h, w). + During tracing, it becomes list[Tensor] instead. + """ + + def __init__(self, tensor: torch.Tensor, image_sizes: List[Tuple[int, int]]): + """ + Arguments: + tensor (Tensor): of shape (N, H, W) or (N, C_1, ..., C_K, H, W) where K >= 1 + image_sizes (list[tuple[int, int]]): Each tuple is (h, w). It can + be smaller than (H, W) due to padding. + """ + self.tensor = tensor + self.image_sizes = image_sizes + + def __len__(self) -> int: + return len(self.image_sizes) + + def __getitem__(self, idx) -> torch.Tensor: + """ + Access the individual image in its original size. + + Args: + idx: int or slice + + Returns: + Tensor: an image of shape (H, W) or (C_1, ..., C_K, H, W) where K >= 1 + """ + size = self.image_sizes[idx] + return self.tensor[idx, ..., : size[0], : size[1]] + + @torch.jit.unused + def to(self, *args: Any, **kwargs: Any) -> "ImageList": + cast_tensor = self.tensor.to(*args, **kwargs) + return ImageList(cast_tensor, self.image_sizes) + + @property + def device(self) -> device: + return self.tensor.device + + @staticmethod + def from_tensors( + tensors: List[torch.Tensor], + size_divisibility: int = 0, + pad_value: float = 0.0, + padding_constraints: Optional[Dict[str, int]] = None, + ) -> "ImageList": + """ + Args: + tensors: a tuple or list of `torch.Tensor`, each of shape (Hi, Wi) or + (C_1, ..., C_K, Hi, Wi) where K >= 1. The Tensors will be padded + to the same shape with `pad_value`. + size_divisibility (int): If `size_divisibility > 0`, add padding to ensure + the common height and width is divisible by `size_divisibility`. + This depends on the model and many models need a divisibility of 32. + pad_value (float): value to pad. + padding_constraints (optional[Dict]): If given, it would follow the format as + {"size_divisibility": int, "square_size": int}, where `size_divisibility` will + overwrite the above one if presented and `square_size` indicates the + square padding size if `square_size` > 0. + Returns: + an `ImageList`. + """ + assert len(tensors) > 0 + assert isinstance(tensors, (tuple, list)) + for t in tensors: + assert isinstance(t, torch.Tensor), type(t) + assert t.shape[:-2] == tensors[0].shape[:-2], t.shape + + image_sizes = [(im.shape[-2], im.shape[-1]) for im in tensors] + image_sizes_tensor = [shapes_to_tensor(x) for x in image_sizes] + max_size = torch.stack(image_sizes_tensor).max(0).values + + if padding_constraints is not None: + square_size = padding_constraints.get("square_size", 0) + if square_size > 0: + # pad to square. + max_size[0] = max_size[1] = square_size + if "size_divisibility" in padding_constraints: + size_divisibility = padding_constraints["size_divisibility"] + if size_divisibility > 1: + stride = size_divisibility + # the last two dims are H,W, both subject to divisibility requirement + max_size = (max_size + (stride - 1)).div(stride, rounding_mode="floor") * stride + + # handle weirdness of scripting and tracing ... + if torch.jit.is_scripting(): + max_size: List[int] = max_size.to(dtype=torch.long).tolist() + else: + if torch.jit.is_tracing(): + image_sizes = image_sizes_tensor + + if len(tensors) == 1: + # This seems slightly (2%) faster. + # TODO: check whether it's faster for multiple images as well + image_size = image_sizes[0] + padding_size = [0, max_size[-1] - image_size[1], 0, max_size[-2] - image_size[0]] + batched_imgs = F.pad(tensors[0], padding_size, value=pad_value).unsqueeze_(0) + else: + # max_size can be a tensor in tracing mode, therefore convert to list + batch_shape = [len(tensors)] + list(tensors[0].shape[:-2]) + list(max_size) + device = ( + None if torch.jit.is_scripting() else ("cpu" if torch.jit.is_tracing() else None) + ) + batched_imgs = tensors[0].new_full(batch_shape, pad_value, device=device) + batched_imgs = move_device_like(batched_imgs, tensors[0]) + for i, img in enumerate(tensors): + # Use `batched_imgs` directly instead of `img, pad_img = zip(tensors, batched_imgs)` + # Tracing mode cannot capture `copy_()` of temporary locals + batched_imgs[i, ..., : img.shape[-2], : img.shape[-1]].copy_(img) + + return ImageList(batched_imgs.contiguous(), image_sizes) diff --git a/annotator/oneformer/detectron2/structures/instances.py b/annotator/oneformer/detectron2/structures/instances.py new file mode 100644 index 0000000000000000000000000000000000000000..c9579bce2730f42e256c6eed99d9014d09304c99 --- /dev/null +++ b/annotator/oneformer/detectron2/structures/instances.py @@ -0,0 +1,194 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import itertools +import warnings +from typing import Any, Dict, List, Tuple, Union +import torch + + +class Instances: + """ + This class represents a list of instances in an image. + It stores the attributes of instances (e.g., boxes, masks, labels, scores) as "fields". + All fields must have the same ``__len__`` which is the number of instances. + + All other (non-field) attributes of this class are considered private: + they must start with '_' and are not modifiable by a user. + + Some basic usage: + + 1. Set/get/check a field: + + .. code-block:: python + + instances.gt_boxes = Boxes(...) + print(instances.pred_masks) # a tensor of shape (N, H, W) + print('gt_masks' in instances) + + 2. ``len(instances)`` returns the number of instances + 3. Indexing: ``instances[indices]`` will apply the indexing on all the fields + and returns a new :class:`Instances`. + Typically, ``indices`` is a integer vector of indices, + or a binary mask of length ``num_instances`` + + .. code-block:: python + + category_3_detections = instances[instances.pred_classes == 3] + confident_detections = instances[instances.scores > 0.9] + """ + + def __init__(self, image_size: Tuple[int, int], **kwargs: Any): + """ + Args: + image_size (height, width): the spatial size of the image. + kwargs: fields to add to this `Instances`. + """ + self._image_size = image_size + self._fields: Dict[str, Any] = {} + for k, v in kwargs.items(): + self.set(k, v) + + @property + def image_size(self) -> Tuple[int, int]: + """ + Returns: + tuple: height, width + """ + return self._image_size + + def __setattr__(self, name: str, val: Any) -> None: + if name.startswith("_"): + super().__setattr__(name, val) + else: + self.set(name, val) + + def __getattr__(self, name: str) -> Any: + if name == "_fields" or name not in self._fields: + raise AttributeError("Cannot find field '{}' in the given Instances!".format(name)) + return self._fields[name] + + def set(self, name: str, value: Any) -> None: + """ + Set the field named `name` to `value`. + The length of `value` must be the number of instances, + and must agree with other existing fields in this object. + """ + with warnings.catch_warnings(record=True): + data_len = len(value) + if len(self._fields): + assert ( + len(self) == data_len + ), "Adding a field of length {} to a Instances of length {}".format(data_len, len(self)) + self._fields[name] = value + + def has(self, name: str) -> bool: + """ + Returns: + bool: whether the field called `name` exists. + """ + return name in self._fields + + def remove(self, name: str) -> None: + """ + Remove the field called `name`. + """ + del self._fields[name] + + def get(self, name: str) -> Any: + """ + Returns the field called `name`. + """ + return self._fields[name] + + def get_fields(self) -> Dict[str, Any]: + """ + Returns: + dict: a dict which maps names (str) to data of the fields + + Modifying the returned dict will modify this instance. + """ + return self._fields + + # Tensor-like methods + def to(self, *args: Any, **kwargs: Any) -> "Instances": + """ + Returns: + Instances: all fields are called with a `to(device)`, if the field has this method. + """ + ret = Instances(self._image_size) + for k, v in self._fields.items(): + if hasattr(v, "to"): + v = v.to(*args, **kwargs) + ret.set(k, v) + return ret + + def __getitem__(self, item: Union[int, slice, torch.BoolTensor]) -> "Instances": + """ + Args: + item: an index-like object and will be used to index all the fields. + + Returns: + If `item` is a string, return the data in the corresponding field. + Otherwise, returns an `Instances` where all fields are indexed by `item`. + """ + if type(item) == int: + if item >= len(self) or item < -len(self): + raise IndexError("Instances index out of range!") + else: + item = slice(item, None, len(self)) + + ret = Instances(self._image_size) + for k, v in self._fields.items(): + ret.set(k, v[item]) + return ret + + def __len__(self) -> int: + for v in self._fields.values(): + # use __len__ because len() has to be int and is not friendly to tracing + return v.__len__() + raise NotImplementedError("Empty Instances does not support __len__!") + + def __iter__(self): + raise NotImplementedError("`Instances` object is not iterable!") + + @staticmethod + def cat(instance_lists: List["Instances"]) -> "Instances": + """ + Args: + instance_lists (list[Instances]) + + Returns: + Instances + """ + assert all(isinstance(i, Instances) for i in instance_lists) + assert len(instance_lists) > 0 + if len(instance_lists) == 1: + return instance_lists[0] + + image_size = instance_lists[0].image_size + if not isinstance(image_size, torch.Tensor): # could be a tensor in tracing + for i in instance_lists[1:]: + assert i.image_size == image_size + ret = Instances(image_size) + for k in instance_lists[0]._fields.keys(): + values = [i.get(k) for i in instance_lists] + v0 = values[0] + if isinstance(v0, torch.Tensor): + values = torch.cat(values, dim=0) + elif isinstance(v0, list): + values = list(itertools.chain(*values)) + elif hasattr(type(v0), "cat"): + values = type(v0).cat(values) + else: + raise ValueError("Unsupported type {} for concatenation".format(type(v0))) + ret.set(k, values) + return ret + + def __str__(self) -> str: + s = self.__class__.__name__ + "(" + s += "num_instances={}, ".format(len(self)) + s += "image_height={}, ".format(self._image_size[0]) + s += "image_width={}, ".format(self._image_size[1]) + s += "fields=[{}])".format(", ".join((f"{k}: {v}" for k, v in self._fields.items()))) + return s + + __repr__ = __str__ diff --git a/annotator/oneformer/detectron2/structures/keypoints.py b/annotator/oneformer/detectron2/structures/keypoints.py new file mode 100644 index 0000000000000000000000000000000000000000..b93ebed4f6554e67ba9bde8d3af90e8dbb3246b6 --- /dev/null +++ b/annotator/oneformer/detectron2/structures/keypoints.py @@ -0,0 +1,235 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import numpy as np +from typing import Any, List, Tuple, Union +import torch +from torch.nn import functional as F + + +class Keypoints: + """ + Stores keypoint **annotation** data. GT Instances have a `gt_keypoints` property + containing the x,y location and visibility flag of each keypoint. This tensor has shape + (N, K, 3) where N is the number of instances and K is the number of keypoints per instance. + + The visibility flag follows the COCO format and must be one of three integers: + + * v=0: not labeled (in which case x=y=0) + * v=1: labeled but not visible + * v=2: labeled and visible + """ + + def __init__(self, keypoints: Union[torch.Tensor, np.ndarray, List[List[float]]]): + """ + Arguments: + keypoints: A Tensor, numpy array, or list of the x, y, and visibility of each keypoint. + The shape should be (N, K, 3) where N is the number of + instances, and K is the number of keypoints per instance. + """ + device = keypoints.device if isinstance(keypoints, torch.Tensor) else torch.device("cpu") + keypoints = torch.as_tensor(keypoints, dtype=torch.float32, device=device) + assert keypoints.dim() == 3 and keypoints.shape[2] == 3, keypoints.shape + self.tensor = keypoints + + def __len__(self) -> int: + return self.tensor.size(0) + + def to(self, *args: Any, **kwargs: Any) -> "Keypoints": + return type(self)(self.tensor.to(*args, **kwargs)) + + @property + def device(self) -> torch.device: + return self.tensor.device + + def to_heatmap(self, boxes: torch.Tensor, heatmap_size: int) -> torch.Tensor: + """ + Convert keypoint annotations to a heatmap of one-hot labels for training, + as described in :paper:`Mask R-CNN`. + + Arguments: + boxes: Nx4 tensor, the boxes to draw the keypoints to + + Returns: + heatmaps: + A tensor of shape (N, K), each element is integer spatial label + in the range [0, heatmap_size**2 - 1] for each keypoint in the input. + valid: + A tensor of shape (N, K) containing whether each keypoint is in the roi or not. + """ + return _keypoints_to_heatmap(self.tensor, boxes, heatmap_size) + + def __getitem__(self, item: Union[int, slice, torch.BoolTensor]) -> "Keypoints": + """ + Create a new `Keypoints` by indexing on this `Keypoints`. + + The following usage are allowed: + + 1. `new_kpts = kpts[3]`: return a `Keypoints` which contains only one instance. + 2. `new_kpts = kpts[2:10]`: return a slice of key points. + 3. `new_kpts = kpts[vector]`, where vector is a torch.ByteTensor + with `length = len(kpts)`. Nonzero elements in the vector will be selected. + + Note that the returned Keypoints might share storage with this Keypoints, + subject to Pytorch's indexing semantics. + """ + if isinstance(item, int): + return Keypoints([self.tensor[item]]) + return Keypoints(self.tensor[item]) + + def __repr__(self) -> str: + s = self.__class__.__name__ + "(" + s += "num_instances={})".format(len(self.tensor)) + return s + + @staticmethod + def cat(keypoints_list: List["Keypoints"]) -> "Keypoints": + """ + Concatenates a list of Keypoints into a single Keypoints + + Arguments: + keypoints_list (list[Keypoints]) + + Returns: + Keypoints: the concatenated Keypoints + """ + assert isinstance(keypoints_list, (list, tuple)) + assert len(keypoints_list) > 0 + assert all(isinstance(keypoints, Keypoints) for keypoints in keypoints_list) + + cat_kpts = type(keypoints_list[0])( + torch.cat([kpts.tensor for kpts in keypoints_list], dim=0) + ) + return cat_kpts + + +# TODO make this nicer, this is a direct translation from C2 (but removing the inner loop) +def _keypoints_to_heatmap( + keypoints: torch.Tensor, rois: torch.Tensor, heatmap_size: int +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Encode keypoint locations into a target heatmap for use in SoftmaxWithLoss across space. + + Maps keypoints from the half-open interval [x1, x2) on continuous image coordinates to the + closed interval [0, heatmap_size - 1] on discrete image coordinates. We use the + continuous-discrete conversion from Heckbert 1990 ("What is the coordinate of a pixel?"): + d = floor(c) and c = d + 0.5, where d is a discrete coordinate and c is a continuous coordinate. + + Arguments: + keypoints: tensor of keypoint locations in of shape (N, K, 3). + rois: Nx4 tensor of rois in xyxy format + heatmap_size: integer side length of square heatmap. + + Returns: + heatmaps: A tensor of shape (N, K) containing an integer spatial label + in the range [0, heatmap_size**2 - 1] for each keypoint in the input. + valid: A tensor of shape (N, K) containing whether each keypoint is in + the roi or not. + """ + + if rois.numel() == 0: + return rois.new().long(), rois.new().long() + offset_x = rois[:, 0] + offset_y = rois[:, 1] + scale_x = heatmap_size / (rois[:, 2] - rois[:, 0]) + scale_y = heatmap_size / (rois[:, 3] - rois[:, 1]) + + offset_x = offset_x[:, None] + offset_y = offset_y[:, None] + scale_x = scale_x[:, None] + scale_y = scale_y[:, None] + + x = keypoints[..., 0] + y = keypoints[..., 1] + + x_boundary_inds = x == rois[:, 2][:, None] + y_boundary_inds = y == rois[:, 3][:, None] + + x = (x - offset_x) * scale_x + x = x.floor().long() + y = (y - offset_y) * scale_y + y = y.floor().long() + + x[x_boundary_inds] = heatmap_size - 1 + y[y_boundary_inds] = heatmap_size - 1 + + valid_loc = (x >= 0) & (y >= 0) & (x < heatmap_size) & (y < heatmap_size) + vis = keypoints[..., 2] > 0 + valid = (valid_loc & vis).long() + + lin_ind = y * heatmap_size + x + heatmaps = lin_ind * valid + + return heatmaps, valid + + +@torch.jit.script_if_tracing +def heatmaps_to_keypoints(maps: torch.Tensor, rois: torch.Tensor) -> torch.Tensor: + """ + Extract predicted keypoint locations from heatmaps. + + Args: + maps (Tensor): (#ROIs, #keypoints, POOL_H, POOL_W). The predicted heatmap of logits for + each ROI and each keypoint. + rois (Tensor): (#ROIs, 4). The box of each ROI. + + Returns: + Tensor of shape (#ROIs, #keypoints, 4) with the last dimension corresponding to + (x, y, logit, score) for each keypoint. + + When converting discrete pixel indices in an NxN image to a continuous keypoint coordinate, + we maintain consistency with :meth:`Keypoints.to_heatmap` by using the conversion from + Heckbert 1990: c = d + 0.5, where d is a discrete coordinate and c is a continuous coordinate. + """ + + offset_x = rois[:, 0] + offset_y = rois[:, 1] + + widths = (rois[:, 2] - rois[:, 0]).clamp(min=1) + heights = (rois[:, 3] - rois[:, 1]).clamp(min=1) + widths_ceil = widths.ceil() + heights_ceil = heights.ceil() + + num_rois, num_keypoints = maps.shape[:2] + xy_preds = maps.new_zeros(rois.shape[0], num_keypoints, 4) + + width_corrections = widths / widths_ceil + height_corrections = heights / heights_ceil + + keypoints_idx = torch.arange(num_keypoints, device=maps.device) + + for i in range(num_rois): + outsize = (int(heights_ceil[i]), int(widths_ceil[i])) + roi_map = F.interpolate(maps[[i]], size=outsize, mode="bicubic", align_corners=False) + + # Although semantically equivalent, `reshape` is used instead of `squeeze` due + # to limitation during ONNX export of `squeeze` in scripting mode + roi_map = roi_map.reshape(roi_map.shape[1:]) # keypoints x H x W + + # softmax over the spatial region + max_score, _ = roi_map.view(num_keypoints, -1).max(1) + max_score = max_score.view(num_keypoints, 1, 1) + tmp_full_resolution = (roi_map - max_score).exp_() + tmp_pool_resolution = (maps[i] - max_score).exp_() + # Produce scores over the region H x W, but normalize with POOL_H x POOL_W, + # so that the scores of objects of different absolute sizes will be more comparable + roi_map_scores = tmp_full_resolution / tmp_pool_resolution.sum((1, 2), keepdim=True) + + w = roi_map.shape[2] + pos = roi_map.view(num_keypoints, -1).argmax(1) + + x_int = pos % w + y_int = (pos - x_int) // w + + assert ( + roi_map_scores[keypoints_idx, y_int, x_int] + == roi_map_scores.view(num_keypoints, -1).max(1)[0] + ).all() + + x = (x_int.float() + 0.5) * width_corrections[i] + y = (y_int.float() + 0.5) * height_corrections[i] + + xy_preds[i, :, 0] = x + offset_x[i] + xy_preds[i, :, 1] = y + offset_y[i] + xy_preds[i, :, 2] = roi_map[keypoints_idx, y_int, x_int] + xy_preds[i, :, 3] = roi_map_scores[keypoints_idx, y_int, x_int] + + return xy_preds diff --git a/annotator/oneformer/detectron2/structures/masks.py b/annotator/oneformer/detectron2/structures/masks.py new file mode 100644 index 0000000000000000000000000000000000000000..80a72a53c8d9e832ca3598ed90db7a10dc3c2f6c --- /dev/null +++ b/annotator/oneformer/detectron2/structures/masks.py @@ -0,0 +1,534 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import copy +import itertools +import numpy as np +from typing import Any, Iterator, List, Union +import pycocotools.mask as mask_util +import torch +from torch import device + +from annotator.oneformer.detectron2.layers.roi_align import ROIAlign +from annotator.oneformer.detectron2.utils.memory import retry_if_cuda_oom + +from .boxes import Boxes + + +def polygon_area(x, y): + # Using the shoelace formula + # https://stackoverflow.com/questions/24467972/calculate-area-of-polygon-given-x-y-coordinates + return 0.5 * np.abs(np.dot(x, np.roll(y, 1)) - np.dot(y, np.roll(x, 1))) + + +def polygons_to_bitmask(polygons: List[np.ndarray], height: int, width: int) -> np.ndarray: + """ + Args: + polygons (list[ndarray]): each array has shape (Nx2,) + height, width (int) + + Returns: + ndarray: a bool mask of shape (height, width) + """ + if len(polygons) == 0: + # COCOAPI does not support empty polygons + return np.zeros((height, width)).astype(bool) + rles = mask_util.frPyObjects(polygons, height, width) + rle = mask_util.merge(rles) + return mask_util.decode(rle).astype(bool) + + +def rasterize_polygons_within_box( + polygons: List[np.ndarray], box: np.ndarray, mask_size: int +) -> torch.Tensor: + """ + Rasterize the polygons into a mask image and + crop the mask content in the given box. + The cropped mask is resized to (mask_size, mask_size). + + This function is used when generating training targets for mask head in Mask R-CNN. + Given original ground-truth masks for an image, new ground-truth mask + training targets in the size of `mask_size x mask_size` + must be provided for each predicted box. This function will be called to + produce such targets. + + Args: + polygons (list[ndarray[float]]): a list of polygons, which represents an instance. + box: 4-element numpy array + mask_size (int): + + Returns: + Tensor: BoolTensor of shape (mask_size, mask_size) + """ + # 1. Shift the polygons w.r.t the boxes + w, h = box[2] - box[0], box[3] - box[1] + + polygons = copy.deepcopy(polygons) + for p in polygons: + p[0::2] = p[0::2] - box[0] + p[1::2] = p[1::2] - box[1] + + # 2. Rescale the polygons to the new box size + # max() to avoid division by small number + ratio_h = mask_size / max(h, 0.1) + ratio_w = mask_size / max(w, 0.1) + + if ratio_h == ratio_w: + for p in polygons: + p *= ratio_h + else: + for p in polygons: + p[0::2] *= ratio_w + p[1::2] *= ratio_h + + # 3. Rasterize the polygons with coco api + mask = polygons_to_bitmask(polygons, mask_size, mask_size) + mask = torch.from_numpy(mask) + return mask + + +class BitMasks: + """ + This class stores the segmentation masks for all objects in one image, in + the form of bitmaps. + + Attributes: + tensor: bool Tensor of N,H,W, representing N instances in the image. + """ + + def __init__(self, tensor: Union[torch.Tensor, np.ndarray]): + """ + Args: + tensor: bool Tensor of N,H,W, representing N instances in the image. + """ + if isinstance(tensor, torch.Tensor): + tensor = tensor.to(torch.bool) + else: + tensor = torch.as_tensor(tensor, dtype=torch.bool, device=torch.device("cpu")) + assert tensor.dim() == 3, tensor.size() + self.image_size = tensor.shape[1:] + self.tensor = tensor + + @torch.jit.unused + def to(self, *args: Any, **kwargs: Any) -> "BitMasks": + return BitMasks(self.tensor.to(*args, **kwargs)) + + @property + def device(self) -> torch.device: + return self.tensor.device + + @torch.jit.unused + def __getitem__(self, item: Union[int, slice, torch.BoolTensor]) -> "BitMasks": + """ + Returns: + BitMasks: Create a new :class:`BitMasks` by indexing. + + The following usage are allowed: + + 1. `new_masks = masks[3]`: return a `BitMasks` which contains only one mask. + 2. `new_masks = masks[2:10]`: return a slice of masks. + 3. `new_masks = masks[vector]`, where vector is a torch.BoolTensor + with `length = len(masks)`. Nonzero elements in the vector will be selected. + + Note that the returned object might share storage with this object, + subject to Pytorch's indexing semantics. + """ + if isinstance(item, int): + return BitMasks(self.tensor[item].unsqueeze(0)) + m = self.tensor[item] + assert m.dim() == 3, "Indexing on BitMasks with {} returns a tensor with shape {}!".format( + item, m.shape + ) + return BitMasks(m) + + @torch.jit.unused + def __iter__(self) -> torch.Tensor: + yield from self.tensor + + @torch.jit.unused + def __repr__(self) -> str: + s = self.__class__.__name__ + "(" + s += "num_instances={})".format(len(self.tensor)) + return s + + def __len__(self) -> int: + return self.tensor.shape[0] + + def nonempty(self) -> torch.Tensor: + """ + Find masks that are non-empty. + + Returns: + Tensor: a BoolTensor which represents + whether each mask is empty (False) or non-empty (True). + """ + return self.tensor.flatten(1).any(dim=1) + + @staticmethod + def from_polygon_masks( + polygon_masks: Union["PolygonMasks", List[List[np.ndarray]]], height: int, width: int + ) -> "BitMasks": + """ + Args: + polygon_masks (list[list[ndarray]] or PolygonMasks) + height, width (int) + """ + if isinstance(polygon_masks, PolygonMasks): + polygon_masks = polygon_masks.polygons + masks = [polygons_to_bitmask(p, height, width) for p in polygon_masks] + if len(masks): + return BitMasks(torch.stack([torch.from_numpy(x) for x in masks])) + else: + return BitMasks(torch.empty(0, height, width, dtype=torch.bool)) + + @staticmethod + def from_roi_masks(roi_masks: "ROIMasks", height: int, width: int) -> "BitMasks": + """ + Args: + roi_masks: + height, width (int): + """ + return roi_masks.to_bitmasks(height, width) + + def crop_and_resize(self, boxes: torch.Tensor, mask_size: int) -> torch.Tensor: + """ + Crop each bitmask by the given box, and resize results to (mask_size, mask_size). + This can be used to prepare training targets for Mask R-CNN. + It has less reconstruction error compared to rasterization with polygons. + However we observe no difference in accuracy, + but BitMasks requires more memory to store all the masks. + + Args: + boxes (Tensor): Nx4 tensor storing the boxes for each mask + mask_size (int): the size of the rasterized mask. + + Returns: + Tensor: + A bool tensor of shape (N, mask_size, mask_size), where + N is the number of predicted boxes for this image. + """ + assert len(boxes) == len(self), "{} != {}".format(len(boxes), len(self)) + device = self.tensor.device + + batch_inds = torch.arange(len(boxes), device=device).to(dtype=boxes.dtype)[:, None] + rois = torch.cat([batch_inds, boxes], dim=1) # Nx5 + + bit_masks = self.tensor.to(dtype=torch.float32) + rois = rois.to(device=device) + output = ( + ROIAlign((mask_size, mask_size), 1.0, 0, aligned=True) + .forward(bit_masks[:, None, :, :], rois) + .squeeze(1) + ) + output = output >= 0.5 + return output + + def get_bounding_boxes(self) -> Boxes: + """ + Returns: + Boxes: tight bounding boxes around bitmasks. + If a mask is empty, it's bounding box will be all zero. + """ + boxes = torch.zeros(self.tensor.shape[0], 4, dtype=torch.float32) + x_any = torch.any(self.tensor, dim=1) + y_any = torch.any(self.tensor, dim=2) + for idx in range(self.tensor.shape[0]): + x = torch.where(x_any[idx, :])[0] + y = torch.where(y_any[idx, :])[0] + if len(x) > 0 and len(y) > 0: + boxes[idx, :] = torch.as_tensor( + [x[0], y[0], x[-1] + 1, y[-1] + 1], dtype=torch.float32 + ) + return Boxes(boxes) + + @staticmethod + def cat(bitmasks_list: List["BitMasks"]) -> "BitMasks": + """ + Concatenates a list of BitMasks into a single BitMasks + + Arguments: + bitmasks_list (list[BitMasks]) + + Returns: + BitMasks: the concatenated BitMasks + """ + assert isinstance(bitmasks_list, (list, tuple)) + assert len(bitmasks_list) > 0 + assert all(isinstance(bitmask, BitMasks) for bitmask in bitmasks_list) + + cat_bitmasks = type(bitmasks_list[0])(torch.cat([bm.tensor for bm in bitmasks_list], dim=0)) + return cat_bitmasks + + +class PolygonMasks: + """ + This class stores the segmentation masks for all objects in one image, in the form of polygons. + + Attributes: + polygons: list[list[ndarray]]. Each ndarray is a float64 vector representing a polygon. + """ + + def __init__(self, polygons: List[List[Union[torch.Tensor, np.ndarray]]]): + """ + Arguments: + polygons (list[list[np.ndarray]]): The first + level of the list correspond to individual instances, + the second level to all the polygons that compose the + instance, and the third level to the polygon coordinates. + The third level array should have the format of + [x0, y0, x1, y1, ..., xn, yn] (n >= 3). + """ + if not isinstance(polygons, list): + raise ValueError( + "Cannot create PolygonMasks: Expect a list of list of polygons per image. " + "Got '{}' instead.".format(type(polygons)) + ) + + def _make_array(t: Union[torch.Tensor, np.ndarray]) -> np.ndarray: + # Use float64 for higher precision, because why not? + # Always put polygons on CPU (self.to is a no-op) since they + # are supposed to be small tensors. + # May need to change this assumption if GPU placement becomes useful + if isinstance(t, torch.Tensor): + t = t.cpu().numpy() + return np.asarray(t).astype("float64") + + def process_polygons( + polygons_per_instance: List[Union[torch.Tensor, np.ndarray]] + ) -> List[np.ndarray]: + if not isinstance(polygons_per_instance, list): + raise ValueError( + "Cannot create polygons: Expect a list of polygons per instance. " + "Got '{}' instead.".format(type(polygons_per_instance)) + ) + # transform each polygon to a numpy array + polygons_per_instance = [_make_array(p) for p in polygons_per_instance] + for polygon in polygons_per_instance: + if len(polygon) % 2 != 0 or len(polygon) < 6: + raise ValueError(f"Cannot create a polygon from {len(polygon)} coordinates.") + return polygons_per_instance + + self.polygons: List[List[np.ndarray]] = [ + process_polygons(polygons_per_instance) for polygons_per_instance in polygons + ] + + def to(self, *args: Any, **kwargs: Any) -> "PolygonMasks": + return self + + @property + def device(self) -> torch.device: + return torch.device("cpu") + + def get_bounding_boxes(self) -> Boxes: + """ + Returns: + Boxes: tight bounding boxes around polygon masks. + """ + boxes = torch.zeros(len(self.polygons), 4, dtype=torch.float32) + for idx, polygons_per_instance in enumerate(self.polygons): + minxy = torch.as_tensor([float("inf"), float("inf")], dtype=torch.float32) + maxxy = torch.zeros(2, dtype=torch.float32) + for polygon in polygons_per_instance: + coords = torch.from_numpy(polygon).view(-1, 2).to(dtype=torch.float32) + minxy = torch.min(minxy, torch.min(coords, dim=0).values) + maxxy = torch.max(maxxy, torch.max(coords, dim=0).values) + boxes[idx, :2] = minxy + boxes[idx, 2:] = maxxy + return Boxes(boxes) + + def nonempty(self) -> torch.Tensor: + """ + Find masks that are non-empty. + + Returns: + Tensor: + a BoolTensor which represents whether each mask is empty (False) or not (True). + """ + keep = [1 if len(polygon) > 0 else 0 for polygon in self.polygons] + return torch.from_numpy(np.asarray(keep, dtype=bool)) + + def __getitem__(self, item: Union[int, slice, List[int], torch.BoolTensor]) -> "PolygonMasks": + """ + Support indexing over the instances and return a `PolygonMasks` object. + `item` can be: + + 1. An integer. It will return an object with only one instance. + 2. A slice. It will return an object with the selected instances. + 3. A list[int]. It will return an object with the selected instances, + correpsonding to the indices in the list. + 4. A vector mask of type BoolTensor, whose length is num_instances. + It will return an object with the instances whose mask is nonzero. + """ + if isinstance(item, int): + selected_polygons = [self.polygons[item]] + elif isinstance(item, slice): + selected_polygons = self.polygons[item] + elif isinstance(item, list): + selected_polygons = [self.polygons[i] for i in item] + elif isinstance(item, torch.Tensor): + # Polygons is a list, so we have to move the indices back to CPU. + if item.dtype == torch.bool: + assert item.dim() == 1, item.shape + item = item.nonzero().squeeze(1).cpu().numpy().tolist() + elif item.dtype in [torch.int32, torch.int64]: + item = item.cpu().numpy().tolist() + else: + raise ValueError("Unsupported tensor dtype={} for indexing!".format(item.dtype)) + selected_polygons = [self.polygons[i] for i in item] + return PolygonMasks(selected_polygons) + + def __iter__(self) -> Iterator[List[np.ndarray]]: + """ + Yields: + list[ndarray]: the polygons for one instance. + Each Tensor is a float64 vector representing a polygon. + """ + return iter(self.polygons) + + def __repr__(self) -> str: + s = self.__class__.__name__ + "(" + s += "num_instances={})".format(len(self.polygons)) + return s + + def __len__(self) -> int: + return len(self.polygons) + + def crop_and_resize(self, boxes: torch.Tensor, mask_size: int) -> torch.Tensor: + """ + Crop each mask by the given box, and resize results to (mask_size, mask_size). + This can be used to prepare training targets for Mask R-CNN. + + Args: + boxes (Tensor): Nx4 tensor storing the boxes for each mask + mask_size (int): the size of the rasterized mask. + + Returns: + Tensor: A bool tensor of shape (N, mask_size, mask_size), where + N is the number of predicted boxes for this image. + """ + assert len(boxes) == len(self), "{} != {}".format(len(boxes), len(self)) + + device = boxes.device + # Put boxes on the CPU, as the polygon representation is not efficient GPU-wise + # (several small tensors for representing a single instance mask) + boxes = boxes.to(torch.device("cpu")) + + results = [ + rasterize_polygons_within_box(poly, box.numpy(), mask_size) + for poly, box in zip(self.polygons, boxes) + ] + """ + poly: list[list[float]], the polygons for one instance + box: a tensor of shape (4,) + """ + if len(results) == 0: + return torch.empty(0, mask_size, mask_size, dtype=torch.bool, device=device) + return torch.stack(results, dim=0).to(device=device) + + def area(self): + """ + Computes area of the mask. + Only works with Polygons, using the shoelace formula: + https://stackoverflow.com/questions/24467972/calculate-area-of-polygon-given-x-y-coordinates + + Returns: + Tensor: a vector, area for each instance + """ + + area = [] + for polygons_per_instance in self.polygons: + area_per_instance = 0 + for p in polygons_per_instance: + area_per_instance += polygon_area(p[0::2], p[1::2]) + area.append(area_per_instance) + + return torch.tensor(area) + + @staticmethod + def cat(polymasks_list: List["PolygonMasks"]) -> "PolygonMasks": + """ + Concatenates a list of PolygonMasks into a single PolygonMasks + + Arguments: + polymasks_list (list[PolygonMasks]) + + Returns: + PolygonMasks: the concatenated PolygonMasks + """ + assert isinstance(polymasks_list, (list, tuple)) + assert len(polymasks_list) > 0 + assert all(isinstance(polymask, PolygonMasks) for polymask in polymasks_list) + + cat_polymasks = type(polymasks_list[0])( + list(itertools.chain.from_iterable(pm.polygons for pm in polymasks_list)) + ) + return cat_polymasks + + +class ROIMasks: + """ + Represent masks by N smaller masks defined in some ROIs. Once ROI boxes are given, + full-image bitmask can be obtained by "pasting" the mask on the region defined + by the corresponding ROI box. + """ + + def __init__(self, tensor: torch.Tensor): + """ + Args: + tensor: (N, M, M) mask tensor that defines the mask within each ROI. + """ + if tensor.dim() != 3: + raise ValueError("ROIMasks must take a masks of 3 dimension.") + self.tensor = tensor + + def to(self, device: torch.device) -> "ROIMasks": + return ROIMasks(self.tensor.to(device)) + + @property + def device(self) -> device: + return self.tensor.device + + def __len__(self): + return self.tensor.shape[0] + + def __getitem__(self, item) -> "ROIMasks": + """ + Returns: + ROIMasks: Create a new :class:`ROIMasks` by indexing. + + The following usage are allowed: + + 1. `new_masks = masks[2:10]`: return a slice of masks. + 2. `new_masks = masks[vector]`, where vector is a torch.BoolTensor + with `length = len(masks)`. Nonzero elements in the vector will be selected. + + Note that the returned object might share storage with this object, + subject to Pytorch's indexing semantics. + """ + t = self.tensor[item] + if t.dim() != 3: + raise ValueError( + f"Indexing on ROIMasks with {item} returns a tensor with shape {t.shape}!" + ) + return ROIMasks(t) + + @torch.jit.unused + def __repr__(self) -> str: + s = self.__class__.__name__ + "(" + s += "num_instances={})".format(len(self.tensor)) + return s + + @torch.jit.unused + def to_bitmasks(self, boxes: torch.Tensor, height, width, threshold=0.5): + """ + Args: see documentation of :func:`paste_masks_in_image`. + """ + from annotator.oneformer.detectron2.layers.mask_ops import paste_masks_in_image, _paste_masks_tensor_shape + + if torch.jit.is_tracing(): + if isinstance(height, torch.Tensor): + paste_func = _paste_masks_tensor_shape + else: + paste_func = paste_masks_in_image + else: + paste_func = retry_if_cuda_oom(paste_masks_in_image) + bitmasks = paste_func(self.tensor, boxes.tensor, (height, width), threshold=threshold) + return BitMasks(bitmasks) diff --git a/annotator/oneformer/detectron2/structures/rotated_boxes.py b/annotator/oneformer/detectron2/structures/rotated_boxes.py new file mode 100644 index 0000000000000000000000000000000000000000..aacfc730dfdf4b6bed5f8c861b720db7656f1cab --- /dev/null +++ b/annotator/oneformer/detectron2/structures/rotated_boxes.py @@ -0,0 +1,505 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import math +from typing import List, Tuple +import torch + +from annotator.oneformer.detectron2.layers.rotated_boxes import pairwise_iou_rotated + +from .boxes import Boxes + + +class RotatedBoxes(Boxes): + """ + This structure stores a list of rotated boxes as a Nx5 torch.Tensor. + It supports some common methods about boxes + (`area`, `clip`, `nonempty`, etc), + and also behaves like a Tensor + (support indexing, `to(device)`, `.device`, and iteration over all boxes) + """ + + def __init__(self, tensor: torch.Tensor): + """ + Args: + tensor (Tensor[float]): a Nx5 matrix. Each row is + (x_center, y_center, width, height, angle), + in which angle is represented in degrees. + While there's no strict range restriction for it, + the recommended principal range is between [-180, 180) degrees. + + Assume we have a horizontal box B = (x_center, y_center, width, height), + where width is along the x-axis and height is along the y-axis. + The rotated box B_rot (x_center, y_center, width, height, angle) + can be seen as: + + 1. When angle == 0: + B_rot == B + 2. When angle > 0: + B_rot is obtained by rotating B w.r.t its center by :math:`|angle|` degrees CCW; + 3. When angle < 0: + B_rot is obtained by rotating B w.r.t its center by :math:`|angle|` degrees CW. + + Mathematically, since the right-handed coordinate system for image space + is (y, x), where y is top->down and x is left->right, the 4 vertices of the + rotated rectangle :math:`(yr_i, xr_i)` (i = 1, 2, 3, 4) can be obtained from + the vertices of the horizontal rectangle :math:`(y_i, x_i)` (i = 1, 2, 3, 4) + in the following way (:math:`\\theta = angle*\\pi/180` is the angle in radians, + :math:`(y_c, x_c)` is the center of the rectangle): + + .. math:: + + yr_i = \\cos(\\theta) (y_i - y_c) - \\sin(\\theta) (x_i - x_c) + y_c, + + xr_i = \\sin(\\theta) (y_i - y_c) + \\cos(\\theta) (x_i - x_c) + x_c, + + which is the standard rigid-body rotation transformation. + + Intuitively, the angle is + (1) the rotation angle from y-axis in image space + to the height vector (top->down in the box's local coordinate system) + of the box in CCW, and + (2) the rotation angle from x-axis in image space + to the width vector (left->right in the box's local coordinate system) + of the box in CCW. + + More intuitively, consider the following horizontal box ABCD represented + in (x1, y1, x2, y2): (3, 2, 7, 4), + covering the [3, 7] x [2, 4] region of the continuous coordinate system + which looks like this: + + .. code:: none + + O--------> x + | + | A---B + | | | + | D---C + | + v y + + Note that each capital letter represents one 0-dimensional geometric point + instead of a 'square pixel' here. + + In the example above, using (x, y) to represent a point we have: + + .. math:: + + O = (0, 0), A = (3, 2), B = (7, 2), C = (7, 4), D = (3, 4) + + We name vector AB = vector DC as the width vector in box's local coordinate system, and + vector AD = vector BC as the height vector in box's local coordinate system. Initially, + when angle = 0 degree, they're aligned with the positive directions of x-axis and y-axis + in the image space, respectively. + + For better illustration, we denote the center of the box as E, + + .. code:: none + + O--------> x + | + | A---B + | | E | + | D---C + | + v y + + where the center E = ((3+7)/2, (2+4)/2) = (5, 3). + + Also, + + .. math:: + + width = |AB| = |CD| = 7 - 3 = 4, + height = |AD| = |BC| = 4 - 2 = 2. + + Therefore, the corresponding representation for the same shape in rotated box in + (x_center, y_center, width, height, angle) format is: + + (5, 3, 4, 2, 0), + + Now, let's consider (5, 3, 4, 2, 90), which is rotated by 90 degrees + CCW (counter-clockwise) by definition. It looks like this: + + .. code:: none + + O--------> x + | B-C + | | | + | |E| + | | | + | A-D + v y + + The center E is still located at the same point (5, 3), while the vertices + ABCD are rotated by 90 degrees CCW with regard to E: + A = (4, 5), B = (4, 1), C = (6, 1), D = (6, 5) + + Here, 90 degrees can be seen as the CCW angle to rotate from y-axis to + vector AD or vector BC (the top->down height vector in box's local coordinate system), + or the CCW angle to rotate from x-axis to vector AB or vector DC (the left->right + width vector in box's local coordinate system). + + .. math:: + + width = |AB| = |CD| = 5 - 1 = 4, + height = |AD| = |BC| = 6 - 4 = 2. + + Next, how about (5, 3, 4, 2, -90), which is rotated by 90 degrees CW (clockwise) + by definition? It looks like this: + + .. code:: none + + O--------> x + | D-A + | | | + | |E| + | | | + | C-B + v y + + The center E is still located at the same point (5, 3), while the vertices + ABCD are rotated by 90 degrees CW with regard to E: + A = (6, 1), B = (6, 5), C = (4, 5), D = (4, 1) + + .. math:: + + width = |AB| = |CD| = 5 - 1 = 4, + height = |AD| = |BC| = 6 - 4 = 2. + + This covers exactly the same region as (5, 3, 4, 2, 90) does, and their IoU + will be 1. However, these two will generate different RoI Pooling results and + should not be treated as an identical box. + + On the other hand, it's easy to see that (X, Y, W, H, A) is identical to + (X, Y, W, H, A+360N), for any integer N. For example (5, 3, 4, 2, 270) would be + identical to (5, 3, 4, 2, -90), because rotating the shape 270 degrees CCW is + equivalent to rotating the same shape 90 degrees CW. + + We could rotate further to get (5, 3, 4, 2, 180), or (5, 3, 4, 2, -180): + + .. code:: none + + O--------> x + | + | C---D + | | E | + | B---A + | + v y + + .. math:: + + A = (7, 4), B = (3, 4), C = (3, 2), D = (7, 2), + + width = |AB| = |CD| = 7 - 3 = 4, + height = |AD| = |BC| = 4 - 2 = 2. + + Finally, this is a very inaccurate (heavily quantized) illustration of + how (5, 3, 4, 2, 60) looks like in case anyone wonders: + + .. code:: none + + O--------> x + | B\ + | / C + | /E / + | A / + | `D + v y + + It's still a rectangle with center of (5, 3), width of 4 and height of 2, + but its angle (and thus orientation) is somewhere between + (5, 3, 4, 2, 0) and (5, 3, 4, 2, 90). + """ + device = tensor.device if isinstance(tensor, torch.Tensor) else torch.device("cpu") + tensor = torch.as_tensor(tensor, dtype=torch.float32, device=device) + if tensor.numel() == 0: + # Use reshape, so we don't end up creating a new tensor that does not depend on + # the inputs (and consequently confuses jit) + tensor = tensor.reshape((0, 5)).to(dtype=torch.float32, device=device) + assert tensor.dim() == 2 and tensor.size(-1) == 5, tensor.size() + + self.tensor = tensor + + def clone(self) -> "RotatedBoxes": + """ + Clone the RotatedBoxes. + + Returns: + RotatedBoxes + """ + return RotatedBoxes(self.tensor.clone()) + + def to(self, device: torch.device): + # Boxes are assumed float32 and does not support to(dtype) + return RotatedBoxes(self.tensor.to(device=device)) + + def area(self) -> torch.Tensor: + """ + Computes the area of all the boxes. + + Returns: + torch.Tensor: a vector with areas of each box. + """ + box = self.tensor + area = box[:, 2] * box[:, 3] + return area + + # Avoid in-place operations so that we can torchscript; NOTE: this creates a new tensor + def normalize_angles(self) -> None: + """ + Restrict angles to the range of [-180, 180) degrees + """ + angle_tensor = (self.tensor[:, 4] + 180.0) % 360.0 - 180.0 + self.tensor = torch.cat((self.tensor[:, :4], angle_tensor[:, None]), dim=1) + + def clip(self, box_size: Tuple[int, int], clip_angle_threshold: float = 1.0) -> None: + """ + Clip (in place) the boxes by limiting x coordinates to the range [0, width] + and y coordinates to the range [0, height]. + + For RRPN: + Only clip boxes that are almost horizontal with a tolerance of + clip_angle_threshold to maintain backward compatibility. + + Rotated boxes beyond this threshold are not clipped for two reasons: + + 1. There are potentially multiple ways to clip a rotated box to make it + fit within the image. + 2. It's tricky to make the entire rectangular box fit within the image + and still be able to not leave out pixels of interest. + + Therefore we rely on ops like RoIAlignRotated to safely handle this. + + Args: + box_size (height, width): The clipping box's size. + clip_angle_threshold: + Iff. abs(normalized(angle)) <= clip_angle_threshold (in degrees), + we do the clipping as horizontal boxes. + """ + h, w = box_size + + # normalize angles to be within (-180, 180] degrees + self.normalize_angles() + + idx = torch.where(torch.abs(self.tensor[:, 4]) <= clip_angle_threshold)[0] + + # convert to (x1, y1, x2, y2) + x1 = self.tensor[idx, 0] - self.tensor[idx, 2] / 2.0 + y1 = self.tensor[idx, 1] - self.tensor[idx, 3] / 2.0 + x2 = self.tensor[idx, 0] + self.tensor[idx, 2] / 2.0 + y2 = self.tensor[idx, 1] + self.tensor[idx, 3] / 2.0 + + # clip + x1.clamp_(min=0, max=w) + y1.clamp_(min=0, max=h) + x2.clamp_(min=0, max=w) + y2.clamp_(min=0, max=h) + + # convert back to (xc, yc, w, h) + self.tensor[idx, 0] = (x1 + x2) / 2.0 + self.tensor[idx, 1] = (y1 + y2) / 2.0 + # make sure widths and heights do not increase due to numerical errors + self.tensor[idx, 2] = torch.min(self.tensor[idx, 2], x2 - x1) + self.tensor[idx, 3] = torch.min(self.tensor[idx, 3], y2 - y1) + + def nonempty(self, threshold: float = 0.0) -> torch.Tensor: + """ + Find boxes that are non-empty. + A box is considered empty, if either of its side is no larger than threshold. + + Returns: + Tensor: a binary vector which represents + whether each box is empty (False) or non-empty (True). + """ + box = self.tensor + widths = box[:, 2] + heights = box[:, 3] + keep = (widths > threshold) & (heights > threshold) + return keep + + def __getitem__(self, item) -> "RotatedBoxes": + """ + Returns: + RotatedBoxes: Create a new :class:`RotatedBoxes` by indexing. + + The following usage are allowed: + + 1. `new_boxes = boxes[3]`: return a `RotatedBoxes` which contains only one box. + 2. `new_boxes = boxes[2:10]`: return a slice of boxes. + 3. `new_boxes = boxes[vector]`, where vector is a torch.ByteTensor + with `length = len(boxes)`. Nonzero elements in the vector will be selected. + + Note that the returned RotatedBoxes might share storage with this RotatedBoxes, + subject to Pytorch's indexing semantics. + """ + if isinstance(item, int): + return RotatedBoxes(self.tensor[item].view(1, -1)) + b = self.tensor[item] + assert b.dim() == 2, "Indexing on RotatedBoxes with {} failed to return a matrix!".format( + item + ) + return RotatedBoxes(b) + + def __len__(self) -> int: + return self.tensor.shape[0] + + def __repr__(self) -> str: + return "RotatedBoxes(" + str(self.tensor) + ")" + + def inside_box(self, box_size: Tuple[int, int], boundary_threshold: int = 0) -> torch.Tensor: + """ + Args: + box_size (height, width): Size of the reference box covering + [0, width] x [0, height] + boundary_threshold (int): Boxes that extend beyond the reference box + boundary by more than boundary_threshold are considered "outside". + + For RRPN, it might not be necessary to call this function since it's common + for rotated box to extend to outside of the image boundaries + (the clip function only clips the near-horizontal boxes) + + Returns: + a binary vector, indicating whether each box is inside the reference box. + """ + height, width = box_size + + cnt_x = self.tensor[..., 0] + cnt_y = self.tensor[..., 1] + half_w = self.tensor[..., 2] / 2.0 + half_h = self.tensor[..., 3] / 2.0 + a = self.tensor[..., 4] + c = torch.abs(torch.cos(a * math.pi / 180.0)) + s = torch.abs(torch.sin(a * math.pi / 180.0)) + # This basically computes the horizontal bounding rectangle of the rotated box + max_rect_dx = c * half_w + s * half_h + max_rect_dy = c * half_h + s * half_w + + inds_inside = ( + (cnt_x - max_rect_dx >= -boundary_threshold) + & (cnt_y - max_rect_dy >= -boundary_threshold) + & (cnt_x + max_rect_dx < width + boundary_threshold) + & (cnt_y + max_rect_dy < height + boundary_threshold) + ) + + return inds_inside + + def get_centers(self) -> torch.Tensor: + """ + Returns: + The box centers in a Nx2 array of (x, y). + """ + return self.tensor[:, :2] + + def scale(self, scale_x: float, scale_y: float) -> None: + """ + Scale the rotated box with horizontal and vertical scaling factors + Note: when scale_factor_x != scale_factor_y, + the rotated box does not preserve the rectangular shape when the angle + is not a multiple of 90 degrees under resize transformation. + Instead, the shape is a parallelogram (that has skew) + Here we make an approximation by fitting a rotated rectangle to the parallelogram. + """ + self.tensor[:, 0] *= scale_x + self.tensor[:, 1] *= scale_y + theta = self.tensor[:, 4] * math.pi / 180.0 + c = torch.cos(theta) + s = torch.sin(theta) + + # In image space, y is top->down and x is left->right + # Consider the local coordintate system for the rotated box, + # where the box center is located at (0, 0), and the four vertices ABCD are + # A(-w / 2, -h / 2), B(w / 2, -h / 2), C(w / 2, h / 2), D(-w / 2, h / 2) + # the midpoint of the left edge AD of the rotated box E is: + # E = (A+D)/2 = (-w / 2, 0) + # the midpoint of the top edge AB of the rotated box F is: + # F(0, -h / 2) + # To get the old coordinates in the global system, apply the rotation transformation + # (Note: the right-handed coordinate system for image space is yOx): + # (old_x, old_y) = (s * y + c * x, c * y - s * x) + # E(old) = (s * 0 + c * (-w/2), c * 0 - s * (-w/2)) = (-c * w / 2, s * w / 2) + # F(old) = (s * (-h / 2) + c * 0, c * (-h / 2) - s * 0) = (-s * h / 2, -c * h / 2) + # After applying the scaling factor (sfx, sfy): + # E(new) = (-sfx * c * w / 2, sfy * s * w / 2) + # F(new) = (-sfx * s * h / 2, -sfy * c * h / 2) + # The new width after scaling tranformation becomes: + + # w(new) = |E(new) - O| * 2 + # = sqrt[(sfx * c * w / 2)^2 + (sfy * s * w / 2)^2] * 2 + # = sqrt[(sfx * c)^2 + (sfy * s)^2] * w + # i.e., scale_factor_w = sqrt[(sfx * c)^2 + (sfy * s)^2] + # + # For example, + # when angle = 0 or 180, |c| = 1, s = 0, scale_factor_w == scale_factor_x; + # when |angle| = 90, c = 0, |s| = 1, scale_factor_w == scale_factor_y + self.tensor[:, 2] *= torch.sqrt((scale_x * c) ** 2 + (scale_y * s) ** 2) + + # h(new) = |F(new) - O| * 2 + # = sqrt[(sfx * s * h / 2)^2 + (sfy * c * h / 2)^2] * 2 + # = sqrt[(sfx * s)^2 + (sfy * c)^2] * h + # i.e., scale_factor_h = sqrt[(sfx * s)^2 + (sfy * c)^2] + # + # For example, + # when angle = 0 or 180, |c| = 1, s = 0, scale_factor_h == scale_factor_y; + # when |angle| = 90, c = 0, |s| = 1, scale_factor_h == scale_factor_x + self.tensor[:, 3] *= torch.sqrt((scale_x * s) ** 2 + (scale_y * c) ** 2) + + # The angle is the rotation angle from y-axis in image space to the height + # vector (top->down in the box's local coordinate system) of the box in CCW. + # + # angle(new) = angle_yOx(O - F(new)) + # = angle_yOx( (sfx * s * h / 2, sfy * c * h / 2) ) + # = atan2(sfx * s * h / 2, sfy * c * h / 2) + # = atan2(sfx * s, sfy * c) + # + # For example, + # when sfx == sfy, angle(new) == atan2(s, c) == angle(old) + self.tensor[:, 4] = torch.atan2(scale_x * s, scale_y * c) * 180 / math.pi + + @classmethod + def cat(cls, boxes_list: List["RotatedBoxes"]) -> "RotatedBoxes": + """ + Concatenates a list of RotatedBoxes into a single RotatedBoxes + + Arguments: + boxes_list (list[RotatedBoxes]) + + Returns: + RotatedBoxes: the concatenated RotatedBoxes + """ + assert isinstance(boxes_list, (list, tuple)) + if len(boxes_list) == 0: + return cls(torch.empty(0)) + assert all([isinstance(box, RotatedBoxes) for box in boxes_list]) + + # use torch.cat (v.s. layers.cat) so the returned boxes never share storage with input + cat_boxes = cls(torch.cat([b.tensor for b in boxes_list], dim=0)) + return cat_boxes + + @property + def device(self) -> torch.device: + return self.tensor.device + + @torch.jit.unused + def __iter__(self): + """ + Yield a box as a Tensor of shape (5,) at a time. + """ + yield from self.tensor + + +def pairwise_iou(boxes1: RotatedBoxes, boxes2: RotatedBoxes) -> None: + """ + Given two lists of rotated boxes of size N and M, + compute the IoU (intersection over union) + between **all** N x M pairs of boxes. + The box order must be (x_center, y_center, width, height, angle). + + Args: + boxes1, boxes2 (RotatedBoxes): + two `RotatedBoxes`. Contains N & M rotated boxes, respectively. + + Returns: + Tensor: IoU, sized [N,M]. + """ + + return pairwise_iou_rotated(boxes1.tensor, boxes2.tensor) diff --git a/annotator/oneformer/detectron2/tracking/__init__.py b/annotator/oneformer/detectron2/tracking/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..21078ae822b04b71dbd8b056b5993d173eaf6bff --- /dev/null +++ b/annotator/oneformer/detectron2/tracking/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .base_tracker import ( # noqa + BaseTracker, + build_tracker_head, + TRACKER_HEADS_REGISTRY, +) +from .bbox_iou_tracker import BBoxIOUTracker # noqa +from .hungarian_tracker import BaseHungarianTracker # noqa +from .iou_weighted_hungarian_bbox_iou_tracker import ( # noqa + IOUWeightedHungarianBBoxIOUTracker, +) +from .utils import create_prediction_pairs # noqa +from .vanilla_hungarian_bbox_iou_tracker import VanillaHungarianBBoxIOUTracker # noqa + +__all__ = [k for k in globals().keys() if not k.startswith("_")] diff --git a/annotator/oneformer/detectron2/tracking/base_tracker.py b/annotator/oneformer/detectron2/tracking/base_tracker.py new file mode 100644 index 0000000000000000000000000000000000000000..bec640746d4fa40ae4a4020e88300e601b95ea3d --- /dev/null +++ b/annotator/oneformer/detectron2/tracking/base_tracker.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# Copyright 2004-present Facebook. All Rights Reserved. +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.utils.registry import Registry + +from ..config.config import CfgNode as CfgNode_ +from ..structures import Instances + +TRACKER_HEADS_REGISTRY = Registry("TRACKER_HEADS") +TRACKER_HEADS_REGISTRY.__doc__ = """ +Registry for tracking classes. +""" + + +class BaseTracker(object): + """ + A parent class for all trackers + """ + + @configurable + def __init__(self, **kwargs): + self._prev_instances = None # (D2)instances for previous frame + self._matched_idx = set() # indices in prev_instances found matching + self._matched_ID = set() # idendities in prev_instances found matching + self._untracked_prev_idx = set() # indices in prev_instances not found matching + self._id_count = 0 # used to assign new id + + @classmethod + def from_config(cls, cfg: CfgNode_): + raise NotImplementedError("Calling BaseTracker::from_config") + + def update(self, predictions: Instances) -> Instances: + """ + Args: + predictions: D2 Instances for predictions of the current frame + Return: + D2 Instances for predictions of the current frame with ID assigned + + _prev_instances and instances will have the following fields: + .pred_boxes (shape=[N, 4]) + .scores (shape=[N,]) + .pred_classes (shape=[N,]) + .pred_keypoints (shape=[N, M, 3], Optional) + .pred_masks (shape=List[2D_MASK], Optional) 2D_MASK: shape=[H, W] + .ID (shape=[N,]) + + N: # of detected bboxes + H and W: height and width of 2D mask + """ + raise NotImplementedError("Calling BaseTracker::update") + + +def build_tracker_head(cfg: CfgNode_) -> BaseTracker: + """ + Build a tracker head from `cfg.TRACKER_HEADS.TRACKER_NAME`. + + Args: + cfg: D2 CfgNode, config file with tracker information + Return: + tracker object + """ + name = cfg.TRACKER_HEADS.TRACKER_NAME + tracker_class = TRACKER_HEADS_REGISTRY.get(name) + return tracker_class(cfg) diff --git a/annotator/oneformer/detectron2/tracking/bbox_iou_tracker.py b/annotator/oneformer/detectron2/tracking/bbox_iou_tracker.py new file mode 100644 index 0000000000000000000000000000000000000000..2b7e2579364b20969db884a5785cb5c650d760ac --- /dev/null +++ b/annotator/oneformer/detectron2/tracking/bbox_iou_tracker.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 +# Copyright 2004-present Facebook. All Rights Reserved. +import copy +import numpy as np +from typing import List +import torch + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.structures import Boxes, Instances +from annotator.oneformer.detectron2.structures.boxes import pairwise_iou + +from ..config.config import CfgNode as CfgNode_ +from .base_tracker import TRACKER_HEADS_REGISTRY, BaseTracker + + +@TRACKER_HEADS_REGISTRY.register() +class BBoxIOUTracker(BaseTracker): + """ + A bounding box tracker to assign ID based on IoU between current and previous instances + """ + + @configurable + def __init__( + self, + *, + video_height: int, + video_width: int, + max_num_instances: int = 200, + max_lost_frame_count: int = 0, + min_box_rel_dim: float = 0.02, + min_instance_period: int = 1, + track_iou_threshold: float = 0.5, + **kwargs, + ): + """ + Args: + video_height: height the video frame + video_width: width of the video frame + max_num_instances: maximum number of id allowed to be tracked + max_lost_frame_count: maximum number of frame an id can lost tracking + exceed this number, an id is considered as lost + forever + min_box_rel_dim: a percentage, smaller than this dimension, a bbox is + removed from tracking + min_instance_period: an instance will be shown after this number of period + since its first showing up in the video + track_iou_threshold: iou threshold, below this number a bbox pair is removed + from tracking + """ + super().__init__(**kwargs) + self._video_height = video_height + self._video_width = video_width + self._max_num_instances = max_num_instances + self._max_lost_frame_count = max_lost_frame_count + self._min_box_rel_dim = min_box_rel_dim + self._min_instance_period = min_instance_period + self._track_iou_threshold = track_iou_threshold + + @classmethod + def from_config(cls, cfg: CfgNode_): + """ + Old style initialization using CfgNode + + Args: + cfg: D2 CfgNode, config file + Return: + dictionary storing arguments for __init__ method + """ + assert "VIDEO_HEIGHT" in cfg.TRACKER_HEADS + assert "VIDEO_WIDTH" in cfg.TRACKER_HEADS + video_height = cfg.TRACKER_HEADS.get("VIDEO_HEIGHT") + video_width = cfg.TRACKER_HEADS.get("VIDEO_WIDTH") + max_num_instances = cfg.TRACKER_HEADS.get("MAX_NUM_INSTANCES", 200) + max_lost_frame_count = cfg.TRACKER_HEADS.get("MAX_LOST_FRAME_COUNT", 0) + min_box_rel_dim = cfg.TRACKER_HEADS.get("MIN_BOX_REL_DIM", 0.02) + min_instance_period = cfg.TRACKER_HEADS.get("MIN_INSTANCE_PERIOD", 1) + track_iou_threshold = cfg.TRACKER_HEADS.get("TRACK_IOU_THRESHOLD", 0.5) + return { + "_target_": "detectron2.tracking.bbox_iou_tracker.BBoxIOUTracker", + "video_height": video_height, + "video_width": video_width, + "max_num_instances": max_num_instances, + "max_lost_frame_count": max_lost_frame_count, + "min_box_rel_dim": min_box_rel_dim, + "min_instance_period": min_instance_period, + "track_iou_threshold": track_iou_threshold, + } + + def update(self, instances: Instances) -> Instances: + """ + See BaseTracker description + """ + instances = self._initialize_extra_fields(instances) + if self._prev_instances is not None: + # calculate IoU of all bbox pairs + iou_all = pairwise_iou( + boxes1=instances.pred_boxes, + boxes2=self._prev_instances.pred_boxes, + ) + # sort IoU in descending order + bbox_pairs = self._create_prediction_pairs(instances, iou_all) + # assign previous ID to current bbox if IoU > track_iou_threshold + self._reset_fields() + for bbox_pair in bbox_pairs: + idx = bbox_pair["idx"] + prev_id = bbox_pair["prev_id"] + if ( + idx in self._matched_idx + or prev_id in self._matched_ID + or bbox_pair["IoU"] < self._track_iou_threshold + ): + continue + instances.ID[idx] = prev_id + instances.ID_period[idx] = bbox_pair["prev_period"] + 1 + instances.lost_frame_count[idx] = 0 + self._matched_idx.add(idx) + self._matched_ID.add(prev_id) + self._untracked_prev_idx.remove(bbox_pair["prev_idx"]) + instances = self._assign_new_id(instances) + instances = self._merge_untracked_instances(instances) + self._prev_instances = copy.deepcopy(instances) + return instances + + def _create_prediction_pairs(self, instances: Instances, iou_all: np.ndarray) -> List: + """ + For all instances in previous and current frames, create pairs. For each + pair, store index of the instance in current frame predcitions, index in + previous predictions, ID in previous predictions, IoU of the bboxes in this + pair, period in previous predictions. + + Args: + instances: D2 Instances, for predictions of the current frame + iou_all: IoU for all bboxes pairs + Return: + A list of IoU for all pairs + """ + bbox_pairs = [] + for i in range(len(instances)): + for j in range(len(self._prev_instances)): + bbox_pairs.append( + { + "idx": i, + "prev_idx": j, + "prev_id": self._prev_instances.ID[j], + "IoU": iou_all[i, j], + "prev_period": self._prev_instances.ID_period[j], + } + ) + return bbox_pairs + + def _initialize_extra_fields(self, instances: Instances) -> Instances: + """ + If input instances don't have ID, ID_period, lost_frame_count fields, + this method is used to initialize these fields. + + Args: + instances: D2 Instances, for predictions of the current frame + Return: + D2 Instances with extra fields added + """ + if not instances.has("ID"): + instances.set("ID", [None] * len(instances)) + if not instances.has("ID_period"): + instances.set("ID_period", [None] * len(instances)) + if not instances.has("lost_frame_count"): + instances.set("lost_frame_count", [None] * len(instances)) + if self._prev_instances is None: + instances.ID = list(range(len(instances))) + self._id_count += len(instances) + instances.ID_period = [1] * len(instances) + instances.lost_frame_count = [0] * len(instances) + return instances + + def _reset_fields(self): + """ + Before each uodate call, reset fields first + """ + self._matched_idx = set() + self._matched_ID = set() + self._untracked_prev_idx = set(range(len(self._prev_instances))) + + def _assign_new_id(self, instances: Instances) -> Instances: + """ + For each untracked instance, assign a new id + + Args: + instances: D2 Instances, for predictions of the current frame + Return: + D2 Instances with new ID assigned + """ + untracked_idx = set(range(len(instances))).difference(self._matched_idx) + for idx in untracked_idx: + instances.ID[idx] = self._id_count + self._id_count += 1 + instances.ID_period[idx] = 1 + instances.lost_frame_count[idx] = 0 + return instances + + def _merge_untracked_instances(self, instances: Instances) -> Instances: + """ + For untracked previous instances, under certain condition, still keep them + in tracking and merge with the current instances. + + Args: + instances: D2 Instances, for predictions of the current frame + Return: + D2 Instances merging current instances and instances from previous + frame decided to keep tracking + """ + untracked_instances = Instances( + image_size=instances.image_size, + pred_boxes=[], + pred_classes=[], + scores=[], + ID=[], + ID_period=[], + lost_frame_count=[], + ) + prev_bboxes = list(self._prev_instances.pred_boxes) + prev_classes = list(self._prev_instances.pred_classes) + prev_scores = list(self._prev_instances.scores) + prev_ID_period = self._prev_instances.ID_period + if instances.has("pred_masks"): + untracked_instances.set("pred_masks", []) + prev_masks = list(self._prev_instances.pred_masks) + if instances.has("pred_keypoints"): + untracked_instances.set("pred_keypoints", []) + prev_keypoints = list(self._prev_instances.pred_keypoints) + if instances.has("pred_keypoint_heatmaps"): + untracked_instances.set("pred_keypoint_heatmaps", []) + prev_keypoint_heatmaps = list(self._prev_instances.pred_keypoint_heatmaps) + for idx in self._untracked_prev_idx: + x_left, y_top, x_right, y_bot = prev_bboxes[idx] + if ( + (1.0 * (x_right - x_left) / self._video_width < self._min_box_rel_dim) + or (1.0 * (y_bot - y_top) / self._video_height < self._min_box_rel_dim) + or self._prev_instances.lost_frame_count[idx] >= self._max_lost_frame_count + or prev_ID_period[idx] <= self._min_instance_period + ): + continue + untracked_instances.pred_boxes.append(list(prev_bboxes[idx].numpy())) + untracked_instances.pred_classes.append(int(prev_classes[idx])) + untracked_instances.scores.append(float(prev_scores[idx])) + untracked_instances.ID.append(self._prev_instances.ID[idx]) + untracked_instances.ID_period.append(self._prev_instances.ID_period[idx]) + untracked_instances.lost_frame_count.append( + self._prev_instances.lost_frame_count[idx] + 1 + ) + if instances.has("pred_masks"): + untracked_instances.pred_masks.append(prev_masks[idx].numpy().astype(np.uint8)) + if instances.has("pred_keypoints"): + untracked_instances.pred_keypoints.append( + prev_keypoints[idx].numpy().astype(np.uint8) + ) + if instances.has("pred_keypoint_heatmaps"): + untracked_instances.pred_keypoint_heatmaps.append( + prev_keypoint_heatmaps[idx].numpy().astype(np.float32) + ) + untracked_instances.pred_boxes = Boxes(torch.FloatTensor(untracked_instances.pred_boxes)) + untracked_instances.pred_classes = torch.IntTensor(untracked_instances.pred_classes) + untracked_instances.scores = torch.FloatTensor(untracked_instances.scores) + if instances.has("pred_masks"): + untracked_instances.pred_masks = torch.IntTensor(untracked_instances.pred_masks) + if instances.has("pred_keypoints"): + untracked_instances.pred_keypoints = torch.IntTensor(untracked_instances.pred_keypoints) + if instances.has("pred_keypoint_heatmaps"): + untracked_instances.pred_keypoint_heatmaps = torch.FloatTensor( + untracked_instances.pred_keypoint_heatmaps + ) + + return Instances.cat( + [ + instances, + untracked_instances, + ] + ) diff --git a/annotator/oneformer/detectron2/tracking/hungarian_tracker.py b/annotator/oneformer/detectron2/tracking/hungarian_tracker.py new file mode 100644 index 0000000000000000000000000000000000000000..bb2b368ca0483319616dfbe5919554e5d360dd49 --- /dev/null +++ b/annotator/oneformer/detectron2/tracking/hungarian_tracker.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# Copyright 2004-present Facebook. All Rights Reserved. +import copy +import numpy as np +from typing import Dict +import torch +from scipy.optimize import linear_sum_assignment + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.structures import Boxes, Instances + +from ..config.config import CfgNode as CfgNode_ +from .base_tracker import BaseTracker + + +class BaseHungarianTracker(BaseTracker): + """ + A base class for all Hungarian trackers + """ + + @configurable + def __init__( + self, + video_height: int, + video_width: int, + max_num_instances: int = 200, + max_lost_frame_count: int = 0, + min_box_rel_dim: float = 0.02, + min_instance_period: int = 1, + **kwargs + ): + """ + Args: + video_height: height the video frame + video_width: width of the video frame + max_num_instances: maximum number of id allowed to be tracked + max_lost_frame_count: maximum number of frame an id can lost tracking + exceed this number, an id is considered as lost + forever + min_box_rel_dim: a percentage, smaller than this dimension, a bbox is + removed from tracking + min_instance_period: an instance will be shown after this number of period + since its first showing up in the video + """ + super().__init__(**kwargs) + self._video_height = video_height + self._video_width = video_width + self._max_num_instances = max_num_instances + self._max_lost_frame_count = max_lost_frame_count + self._min_box_rel_dim = min_box_rel_dim + self._min_instance_period = min_instance_period + + @classmethod + def from_config(cls, cfg: CfgNode_) -> Dict: + raise NotImplementedError("Calling HungarianTracker::from_config") + + def build_cost_matrix(self, instances: Instances, prev_instances: Instances) -> np.ndarray: + raise NotImplementedError("Calling HungarianTracker::build_matrix") + + def update(self, instances: Instances) -> Instances: + if instances.has("pred_keypoints"): + raise NotImplementedError("Need to add support for keypoints") + instances = self._initialize_extra_fields(instances) + if self._prev_instances is not None: + self._untracked_prev_idx = set(range(len(self._prev_instances))) + cost_matrix = self.build_cost_matrix(instances, self._prev_instances) + matched_idx, matched_prev_idx = linear_sum_assignment(cost_matrix) + instances = self._process_matched_idx(instances, matched_idx, matched_prev_idx) + instances = self._process_unmatched_idx(instances, matched_idx) + instances = self._process_unmatched_prev_idx(instances, matched_prev_idx) + self._prev_instances = copy.deepcopy(instances) + return instances + + def _initialize_extra_fields(self, instances: Instances) -> Instances: + """ + If input instances don't have ID, ID_period, lost_frame_count fields, + this method is used to initialize these fields. + + Args: + instances: D2 Instances, for predictions of the current frame + Return: + D2 Instances with extra fields added + """ + if not instances.has("ID"): + instances.set("ID", [None] * len(instances)) + if not instances.has("ID_period"): + instances.set("ID_period", [None] * len(instances)) + if not instances.has("lost_frame_count"): + instances.set("lost_frame_count", [None] * len(instances)) + if self._prev_instances is None: + instances.ID = list(range(len(instances))) + self._id_count += len(instances) + instances.ID_period = [1] * len(instances) + instances.lost_frame_count = [0] * len(instances) + return instances + + def _process_matched_idx( + self, instances: Instances, matched_idx: np.ndarray, matched_prev_idx: np.ndarray + ) -> Instances: + assert matched_idx.size == matched_prev_idx.size + for i in range(matched_idx.size): + instances.ID[matched_idx[i]] = self._prev_instances.ID[matched_prev_idx[i]] + instances.ID_period[matched_idx[i]] = ( + self._prev_instances.ID_period[matched_prev_idx[i]] + 1 + ) + instances.lost_frame_count[matched_idx[i]] = 0 + return instances + + def _process_unmatched_idx(self, instances: Instances, matched_idx: np.ndarray) -> Instances: + untracked_idx = set(range(len(instances))).difference(set(matched_idx)) + for idx in untracked_idx: + instances.ID[idx] = self._id_count + self._id_count += 1 + instances.ID_period[idx] = 1 + instances.lost_frame_count[idx] = 0 + return instances + + def _process_unmatched_prev_idx( + self, instances: Instances, matched_prev_idx: np.ndarray + ) -> Instances: + untracked_instances = Instances( + image_size=instances.image_size, + pred_boxes=[], + pred_masks=[], + pred_classes=[], + scores=[], + ID=[], + ID_period=[], + lost_frame_count=[], + ) + prev_bboxes = list(self._prev_instances.pred_boxes) + prev_classes = list(self._prev_instances.pred_classes) + prev_scores = list(self._prev_instances.scores) + prev_ID_period = self._prev_instances.ID_period + if instances.has("pred_masks"): + prev_masks = list(self._prev_instances.pred_masks) + untracked_prev_idx = set(range(len(self._prev_instances))).difference(set(matched_prev_idx)) + for idx in untracked_prev_idx: + x_left, y_top, x_right, y_bot = prev_bboxes[idx] + if ( + (1.0 * (x_right - x_left) / self._video_width < self._min_box_rel_dim) + or (1.0 * (y_bot - y_top) / self._video_height < self._min_box_rel_dim) + or self._prev_instances.lost_frame_count[idx] >= self._max_lost_frame_count + or prev_ID_period[idx] <= self._min_instance_period + ): + continue + untracked_instances.pred_boxes.append(list(prev_bboxes[idx].numpy())) + untracked_instances.pred_classes.append(int(prev_classes[idx])) + untracked_instances.scores.append(float(prev_scores[idx])) + untracked_instances.ID.append(self._prev_instances.ID[idx]) + untracked_instances.ID_period.append(self._prev_instances.ID_period[idx]) + untracked_instances.lost_frame_count.append( + self._prev_instances.lost_frame_count[idx] + 1 + ) + if instances.has("pred_masks"): + untracked_instances.pred_masks.append(prev_masks[idx].numpy().astype(np.uint8)) + + untracked_instances.pred_boxes = Boxes(torch.FloatTensor(untracked_instances.pred_boxes)) + untracked_instances.pred_classes = torch.IntTensor(untracked_instances.pred_classes) + untracked_instances.scores = torch.FloatTensor(untracked_instances.scores) + if instances.has("pred_masks"): + untracked_instances.pred_masks = torch.IntTensor(untracked_instances.pred_masks) + else: + untracked_instances.remove("pred_masks") + + return Instances.cat( + [ + instances, + untracked_instances, + ] + ) diff --git a/annotator/oneformer/detectron2/tracking/iou_weighted_hungarian_bbox_iou_tracker.py b/annotator/oneformer/detectron2/tracking/iou_weighted_hungarian_bbox_iou_tracker.py new file mode 100644 index 0000000000000000000000000000000000000000..e9b40f8a9c269029e220d5dfa8df1e8372d05007 --- /dev/null +++ b/annotator/oneformer/detectron2/tracking/iou_weighted_hungarian_bbox_iou_tracker.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +# Copyright 2004-present Facebook. All Rights Reserved. + +import numpy as np +from typing import List + +from annotator.oneformer.detectron2.config import CfgNode as CfgNode_ +from annotator.oneformer.detectron2.config import configurable + +from .base_tracker import TRACKER_HEADS_REGISTRY +from .vanilla_hungarian_bbox_iou_tracker import VanillaHungarianBBoxIOUTracker + + +@TRACKER_HEADS_REGISTRY.register() +class IOUWeightedHungarianBBoxIOUTracker(VanillaHungarianBBoxIOUTracker): + """ + A tracker using IoU as weight in Hungarian algorithm, also known + as Munkres or Kuhn-Munkres algorithm + """ + + @configurable + def __init__( + self, + *, + video_height: int, + video_width: int, + max_num_instances: int = 200, + max_lost_frame_count: int = 0, + min_box_rel_dim: float = 0.02, + min_instance_period: int = 1, + track_iou_threshold: float = 0.5, + **kwargs, + ): + """ + Args: + video_height: height the video frame + video_width: width of the video frame + max_num_instances: maximum number of id allowed to be tracked + max_lost_frame_count: maximum number of frame an id can lost tracking + exceed this number, an id is considered as lost + forever + min_box_rel_dim: a percentage, smaller than this dimension, a bbox is + removed from tracking + min_instance_period: an instance will be shown after this number of period + since its first showing up in the video + track_iou_threshold: iou threshold, below this number a bbox pair is removed + from tracking + """ + super().__init__( + video_height=video_height, + video_width=video_width, + max_num_instances=max_num_instances, + max_lost_frame_count=max_lost_frame_count, + min_box_rel_dim=min_box_rel_dim, + min_instance_period=min_instance_period, + track_iou_threshold=track_iou_threshold, + ) + + @classmethod + def from_config(cls, cfg: CfgNode_): + """ + Old style initialization using CfgNode + + Args: + cfg: D2 CfgNode, config file + Return: + dictionary storing arguments for __init__ method + """ + assert "VIDEO_HEIGHT" in cfg.TRACKER_HEADS + assert "VIDEO_WIDTH" in cfg.TRACKER_HEADS + video_height = cfg.TRACKER_HEADS.get("VIDEO_HEIGHT") + video_width = cfg.TRACKER_HEADS.get("VIDEO_WIDTH") + max_num_instances = cfg.TRACKER_HEADS.get("MAX_NUM_INSTANCES", 200) + max_lost_frame_count = cfg.TRACKER_HEADS.get("MAX_LOST_FRAME_COUNT", 0) + min_box_rel_dim = cfg.TRACKER_HEADS.get("MIN_BOX_REL_DIM", 0.02) + min_instance_period = cfg.TRACKER_HEADS.get("MIN_INSTANCE_PERIOD", 1) + track_iou_threshold = cfg.TRACKER_HEADS.get("TRACK_IOU_THRESHOLD", 0.5) + return { + "_target_": "detectron2.tracking.iou_weighted_hungarian_bbox_iou_tracker.IOUWeightedHungarianBBoxIOUTracker", # noqa + "video_height": video_height, + "video_width": video_width, + "max_num_instances": max_num_instances, + "max_lost_frame_count": max_lost_frame_count, + "min_box_rel_dim": min_box_rel_dim, + "min_instance_period": min_instance_period, + "track_iou_threshold": track_iou_threshold, + } + + def assign_cost_matrix_values(self, cost_matrix: np.ndarray, bbox_pairs: List) -> np.ndarray: + """ + Based on IoU for each pair of bbox, assign the associated value in cost matrix + + Args: + cost_matrix: np.ndarray, initialized 2D array with target dimensions + bbox_pairs: list of bbox pair, in each pair, iou value is stored + Return: + np.ndarray, cost_matrix with assigned values + """ + for pair in bbox_pairs: + # assign (-1 * IoU) for above threshold pairs, algorithms will minimize cost + cost_matrix[pair["idx"]][pair["prev_idx"]] = -1 * pair["IoU"] + return cost_matrix diff --git a/annotator/oneformer/detectron2/tracking/utils.py b/annotator/oneformer/detectron2/tracking/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..78d19984f772c030982402d52307f303b84f98b4 --- /dev/null +++ b/annotator/oneformer/detectron2/tracking/utils.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +import numpy as np +from typing import List + +from annotator.oneformer.detectron2.structures import Instances + + +def create_prediction_pairs( + instances: Instances, + prev_instances: Instances, + iou_all: np.ndarray, + threshold: float = 0.5, +) -> List: + """ + Args: + instances: predictions from current frame + prev_instances: predictions from previous frame + iou_all: 2D numpy array containing iou for each bbox pair + threshold: below the threshold, doesn't consider the pair of bbox is valid + Return: + List of bbox pairs + """ + bbox_pairs = [] + for i in range(len(instances)): + for j in range(len(prev_instances)): + if iou_all[i, j] < threshold: + continue + bbox_pairs.append( + { + "idx": i, + "prev_idx": j, + "prev_id": prev_instances.ID[j], + "IoU": iou_all[i, j], + "prev_period": prev_instances.ID_period[j], + } + ) + return bbox_pairs + + +LARGE_COST_VALUE = 100000 diff --git a/annotator/oneformer/detectron2/tracking/vanilla_hungarian_bbox_iou_tracker.py b/annotator/oneformer/detectron2/tracking/vanilla_hungarian_bbox_iou_tracker.py new file mode 100644 index 0000000000000000000000000000000000000000..eecfe2f31e65147aec47704b9e775e82d9f5fa9a --- /dev/null +++ b/annotator/oneformer/detectron2/tracking/vanilla_hungarian_bbox_iou_tracker.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +# Copyright 2004-present Facebook. All Rights Reserved. + +import numpy as np +from typing import List + +from annotator.oneformer.detectron2.config import CfgNode as CfgNode_ +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.structures import Instances +from annotator.oneformer.detectron2.structures.boxes import pairwise_iou +from annotator.oneformer.detectron2.tracking.utils import LARGE_COST_VALUE, create_prediction_pairs + +from .base_tracker import TRACKER_HEADS_REGISTRY +from .hungarian_tracker import BaseHungarianTracker + + +@TRACKER_HEADS_REGISTRY.register() +class VanillaHungarianBBoxIOUTracker(BaseHungarianTracker): + """ + Hungarian algo based tracker using bbox iou as metric + """ + + @configurable + def __init__( + self, + *, + video_height: int, + video_width: int, + max_num_instances: int = 200, + max_lost_frame_count: int = 0, + min_box_rel_dim: float = 0.02, + min_instance_period: int = 1, + track_iou_threshold: float = 0.5, + **kwargs, + ): + """ + Args: + video_height: height the video frame + video_width: width of the video frame + max_num_instances: maximum number of id allowed to be tracked + max_lost_frame_count: maximum number of frame an id can lost tracking + exceed this number, an id is considered as lost + forever + min_box_rel_dim: a percentage, smaller than this dimension, a bbox is + removed from tracking + min_instance_period: an instance will be shown after this number of period + since its first showing up in the video + track_iou_threshold: iou threshold, below this number a bbox pair is removed + from tracking + """ + super().__init__( + video_height=video_height, + video_width=video_width, + max_num_instances=max_num_instances, + max_lost_frame_count=max_lost_frame_count, + min_box_rel_dim=min_box_rel_dim, + min_instance_period=min_instance_period, + ) + self._track_iou_threshold = track_iou_threshold + + @classmethod + def from_config(cls, cfg: CfgNode_): + """ + Old style initialization using CfgNode + + Args: + cfg: D2 CfgNode, config file + Return: + dictionary storing arguments for __init__ method + """ + assert "VIDEO_HEIGHT" in cfg.TRACKER_HEADS + assert "VIDEO_WIDTH" in cfg.TRACKER_HEADS + video_height = cfg.TRACKER_HEADS.get("VIDEO_HEIGHT") + video_width = cfg.TRACKER_HEADS.get("VIDEO_WIDTH") + max_num_instances = cfg.TRACKER_HEADS.get("MAX_NUM_INSTANCES", 200) + max_lost_frame_count = cfg.TRACKER_HEADS.get("MAX_LOST_FRAME_COUNT", 0) + min_box_rel_dim = cfg.TRACKER_HEADS.get("MIN_BOX_REL_DIM", 0.02) + min_instance_period = cfg.TRACKER_HEADS.get("MIN_INSTANCE_PERIOD", 1) + track_iou_threshold = cfg.TRACKER_HEADS.get("TRACK_IOU_THRESHOLD", 0.5) + return { + "_target_": "detectron2.tracking.vanilla_hungarian_bbox_iou_tracker.VanillaHungarianBBoxIOUTracker", # noqa + "video_height": video_height, + "video_width": video_width, + "max_num_instances": max_num_instances, + "max_lost_frame_count": max_lost_frame_count, + "min_box_rel_dim": min_box_rel_dim, + "min_instance_period": min_instance_period, + "track_iou_threshold": track_iou_threshold, + } + + def build_cost_matrix(self, instances: Instances, prev_instances: Instances) -> np.ndarray: + """ + Build the cost matrix for assignment problem + (https://en.wikipedia.org/wiki/Assignment_problem) + + Args: + instances: D2 Instances, for current frame predictions + prev_instances: D2 Instances, for previous frame predictions + + Return: + the cost matrix in numpy array + """ + assert instances is not None and prev_instances is not None + # calculate IoU of all bbox pairs + iou_all = pairwise_iou( + boxes1=instances.pred_boxes, + boxes2=self._prev_instances.pred_boxes, + ) + bbox_pairs = create_prediction_pairs( + instances, self._prev_instances, iou_all, threshold=self._track_iou_threshold + ) + # assign large cost value to make sure pair below IoU threshold won't be matched + cost_matrix = np.full((len(instances), len(prev_instances)), LARGE_COST_VALUE) + return self.assign_cost_matrix_values(cost_matrix, bbox_pairs) + + def assign_cost_matrix_values(self, cost_matrix: np.ndarray, bbox_pairs: List) -> np.ndarray: + """ + Based on IoU for each pair of bbox, assign the associated value in cost matrix + + Args: + cost_matrix: np.ndarray, initialized 2D array with target dimensions + bbox_pairs: list of bbox pair, in each pair, iou value is stored + Return: + np.ndarray, cost_matrix with assigned values + """ + for pair in bbox_pairs: + # assign -1 for IoU above threshold pairs, algorithms will minimize cost + cost_matrix[pair["idx"]][pair["prev_idx"]] = -1 + return cost_matrix diff --git a/annotator/oneformer/detectron2/utils/README.md b/annotator/oneformer/detectron2/utils/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9765b24a730b77556104187ac3ef5439ab0859fd --- /dev/null +++ b/annotator/oneformer/detectron2/utils/README.md @@ -0,0 +1,5 @@ +# Utility functions + +This folder contain utility functions that are not used in the +core library, but are useful for building models or training +code using the config system. diff --git a/annotator/oneformer/detectron2/utils/__init__.py b/annotator/oneformer/detectron2/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9020c2df23e2af280b7bb168b996ae9eaf312eb8 --- /dev/null +++ b/annotator/oneformer/detectron2/utils/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Facebook, Inc. and its affiliates. diff --git a/annotator/oneformer/detectron2/utils/analysis.py b/annotator/oneformer/detectron2/utils/analysis.py new file mode 100644 index 0000000000000000000000000000000000000000..d63e14bcb6d9582df8a647c9a2ca46f2f7e4cd1d --- /dev/null +++ b/annotator/oneformer/detectron2/utils/analysis.py @@ -0,0 +1,188 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# -*- coding: utf-8 -*- + +import typing +from typing import Any, List +import fvcore +from fvcore.nn import activation_count, flop_count, parameter_count, parameter_count_table +from torch import nn + +from annotator.oneformer.detectron2.export import TracingAdapter + +__all__ = [ + "activation_count_operators", + "flop_count_operators", + "parameter_count_table", + "parameter_count", + "FlopCountAnalysis", +] + +FLOPS_MODE = "flops" +ACTIVATIONS_MODE = "activations" + + +# Some extra ops to ignore from counting, including elementwise and reduction ops +_IGNORED_OPS = { + "aten::add", + "aten::add_", + "aten::argmax", + "aten::argsort", + "aten::batch_norm", + "aten::constant_pad_nd", + "aten::div", + "aten::div_", + "aten::exp", + "aten::log2", + "aten::max_pool2d", + "aten::meshgrid", + "aten::mul", + "aten::mul_", + "aten::neg", + "aten::nonzero_numpy", + "aten::reciprocal", + "aten::repeat_interleave", + "aten::rsub", + "aten::sigmoid", + "aten::sigmoid_", + "aten::softmax", + "aten::sort", + "aten::sqrt", + "aten::sub", + "torchvision::nms", # TODO estimate flop for nms +} + + +class FlopCountAnalysis(fvcore.nn.FlopCountAnalysis): + """ + Same as :class:`fvcore.nn.FlopCountAnalysis`, but supports detectron2 models. + """ + + def __init__(self, model, inputs): + """ + Args: + model (nn.Module): + inputs (Any): inputs of the given model. Does not have to be tuple of tensors. + """ + wrapper = TracingAdapter(model, inputs, allow_non_tensor=True) + super().__init__(wrapper, wrapper.flattened_inputs) + self.set_op_handle(**{k: None for k in _IGNORED_OPS}) + + +def flop_count_operators(model: nn.Module, inputs: list) -> typing.DefaultDict[str, float]: + """ + Implement operator-level flops counting using jit. + This is a wrapper of :func:`fvcore.nn.flop_count` and adds supports for standard + detection models in detectron2. + Please use :class:`FlopCountAnalysis` for more advanced functionalities. + + Note: + The function runs the input through the model to compute flops. + The flops of a detection model is often input-dependent, for example, + the flops of box & mask head depends on the number of proposals & + the number of detected objects. + Therefore, the flops counting using a single input may not accurately + reflect the computation cost of a model. It's recommended to average + across a number of inputs. + + Args: + model: a detectron2 model that takes `list[dict]` as input. + inputs (list[dict]): inputs to model, in detectron2's standard format. + Only "image" key will be used. + supported_ops (dict[str, Handle]): see documentation of :func:`fvcore.nn.flop_count` + + Returns: + Counter: Gflop count per operator + """ + old_train = model.training + model.eval() + ret = FlopCountAnalysis(model, inputs).by_operator() + model.train(old_train) + return {k: v / 1e9 for k, v in ret.items()} + + +def activation_count_operators( + model: nn.Module, inputs: list, **kwargs +) -> typing.DefaultDict[str, float]: + """ + Implement operator-level activations counting using jit. + This is a wrapper of fvcore.nn.activation_count, that supports standard detection models + in detectron2. + + Note: + The function runs the input through the model to compute activations. + The activations of a detection model is often input-dependent, for example, + the activations of box & mask head depends on the number of proposals & + the number of detected objects. + + Args: + model: a detectron2 model that takes `list[dict]` as input. + inputs (list[dict]): inputs to model, in detectron2's standard format. + Only "image" key will be used. + + Returns: + Counter: activation count per operator + """ + return _wrapper_count_operators(model=model, inputs=inputs, mode=ACTIVATIONS_MODE, **kwargs) + + +def _wrapper_count_operators( + model: nn.Module, inputs: list, mode: str, **kwargs +) -> typing.DefaultDict[str, float]: + # ignore some ops + supported_ops = {k: lambda *args, **kwargs: {} for k in _IGNORED_OPS} + supported_ops.update(kwargs.pop("supported_ops", {})) + kwargs["supported_ops"] = supported_ops + + assert len(inputs) == 1, "Please use batch size=1" + tensor_input = inputs[0]["image"] + inputs = [{"image": tensor_input}] # remove other keys, in case there are any + + old_train = model.training + if isinstance(model, (nn.parallel.distributed.DistributedDataParallel, nn.DataParallel)): + model = model.module + wrapper = TracingAdapter(model, inputs) + wrapper.eval() + if mode == FLOPS_MODE: + ret = flop_count(wrapper, (tensor_input,), **kwargs) + elif mode == ACTIVATIONS_MODE: + ret = activation_count(wrapper, (tensor_input,), **kwargs) + else: + raise NotImplementedError("Count for mode {} is not supported yet.".format(mode)) + # compatible with change in fvcore + if isinstance(ret, tuple): + ret = ret[0] + model.train(old_train) + return ret + + +def find_unused_parameters(model: nn.Module, inputs: Any) -> List[str]: + """ + Given a model, find parameters that do not contribute + to the loss. + + Args: + model: a model in training mode that returns losses + inputs: argument or a tuple of arguments. Inputs of the model + + Returns: + list[str]: the name of unused parameters + """ + assert model.training + for _, prm in model.named_parameters(): + prm.grad = None + + if isinstance(inputs, tuple): + losses = model(*inputs) + else: + losses = model(inputs) + + if isinstance(losses, dict): + losses = sum(losses.values()) + losses.backward() + + unused: List[str] = [] + for name, prm in model.named_parameters(): + if prm.grad is None: + unused.append(name) + prm.grad = None + return unused diff --git a/annotator/oneformer/detectron2/utils/collect_env.py b/annotator/oneformer/detectron2/utils/collect_env.py new file mode 100644 index 0000000000000000000000000000000000000000..bb25d297ee83c70fd244762e1a7fd554c1fa4b69 --- /dev/null +++ b/annotator/oneformer/detectron2/utils/collect_env.py @@ -0,0 +1,246 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import importlib +import numpy as np +import os +import re +import subprocess +import sys +from collections import defaultdict +import PIL +import torch +import torchvision +from tabulate import tabulate + +__all__ = ["collect_env_info"] + + +def collect_torch_env(): + try: + import torch.__config__ + + return torch.__config__.show() + except ImportError: + # compatible with older versions of pytorch + from torch.utils.collect_env import get_pretty_env_info + + return get_pretty_env_info() + + +def get_env_module(): + var_name = "DETECTRON2_ENV_MODULE" + return var_name, os.environ.get(var_name, "") + + +def detect_compute_compatibility(CUDA_HOME, so_file): + try: + cuobjdump = os.path.join(CUDA_HOME, "bin", "cuobjdump") + if os.path.isfile(cuobjdump): + output = subprocess.check_output( + "'{}' --list-elf '{}'".format(cuobjdump, so_file), shell=True + ) + output = output.decode("utf-8").strip().split("\n") + arch = [] + for line in output: + line = re.findall(r"\.sm_([0-9]*)\.", line)[0] + arch.append(".".join(line)) + arch = sorted(set(arch)) + return ", ".join(arch) + else: + return so_file + "; cannot find cuobjdump" + except Exception: + # unhandled failure + return so_file + + +def collect_env_info(): + has_gpu = torch.cuda.is_available() # true for both CUDA & ROCM + torch_version = torch.__version__ + + # NOTE that CUDA_HOME/ROCM_HOME could be None even when CUDA runtime libs are functional + from torch.utils.cpp_extension import CUDA_HOME, ROCM_HOME + + has_rocm = False + if (getattr(torch.version, "hip", None) is not None) and (ROCM_HOME is not None): + has_rocm = True + has_cuda = has_gpu and (not has_rocm) + + data = [] + data.append(("sys.platform", sys.platform)) # check-template.yml depends on it + data.append(("Python", sys.version.replace("\n", ""))) + data.append(("numpy", np.__version__)) + + try: + import annotator.oneformer.detectron2 # noqa + + data.append( + ("detectron2", detectron2.__version__ + " @" + os.path.dirname(detectron2.__file__)) + ) + except ImportError: + data.append(("detectron2", "failed to import")) + except AttributeError: + data.append(("detectron2", "imported a wrong installation")) + + try: + import annotator.oneformer.detectron2._C as _C + except ImportError as e: + data.append(("detectron2._C", f"not built correctly: {e}")) + + # print system compilers when extension fails to build + if sys.platform != "win32": # don't know what to do for windows + try: + # this is how torch/utils/cpp_extensions.py choose compiler + cxx = os.environ.get("CXX", "c++") + cxx = subprocess.check_output("'{}' --version".format(cxx), shell=True) + cxx = cxx.decode("utf-8").strip().split("\n")[0] + except subprocess.SubprocessError: + cxx = "Not found" + data.append(("Compiler ($CXX)", cxx)) + + if has_cuda and CUDA_HOME is not None: + try: + nvcc = os.path.join(CUDA_HOME, "bin", "nvcc") + nvcc = subprocess.check_output("'{}' -V".format(nvcc), shell=True) + nvcc = nvcc.decode("utf-8").strip().split("\n")[-1] + except subprocess.SubprocessError: + nvcc = "Not found" + data.append(("CUDA compiler", nvcc)) + if has_cuda and sys.platform != "win32": + try: + so_file = importlib.util.find_spec("detectron2._C").origin + except (ImportError, AttributeError): + pass + else: + data.append( + ("detectron2 arch flags", detect_compute_compatibility(CUDA_HOME, so_file)) + ) + else: + # print compilers that are used to build extension + data.append(("Compiler", _C.get_compiler_version())) + data.append(("CUDA compiler", _C.get_cuda_version())) # cuda or hip + if has_cuda and getattr(_C, "has_cuda", lambda: True)(): + data.append( + ("detectron2 arch flags", detect_compute_compatibility(CUDA_HOME, _C.__file__)) + ) + + data.append(get_env_module()) + data.append(("PyTorch", torch_version + " @" + os.path.dirname(torch.__file__))) + data.append(("PyTorch debug build", torch.version.debug)) + try: + data.append(("torch._C._GLIBCXX_USE_CXX11_ABI", torch._C._GLIBCXX_USE_CXX11_ABI)) + except Exception: + pass + + if not has_gpu: + has_gpu_text = "No: torch.cuda.is_available() == False" + else: + has_gpu_text = "Yes" + data.append(("GPU available", has_gpu_text)) + if has_gpu: + devices = defaultdict(list) + for k in range(torch.cuda.device_count()): + cap = ".".join((str(x) for x in torch.cuda.get_device_capability(k))) + name = torch.cuda.get_device_name(k) + f" (arch={cap})" + devices[name].append(str(k)) + for name, devids in devices.items(): + data.append(("GPU " + ",".join(devids), name)) + + if has_rocm: + msg = " - invalid!" if not (ROCM_HOME and os.path.isdir(ROCM_HOME)) else "" + data.append(("ROCM_HOME", str(ROCM_HOME) + msg)) + else: + try: + from torch.utils.collect_env import get_nvidia_driver_version, run as _run + + data.append(("Driver version", get_nvidia_driver_version(_run))) + except Exception: + pass + msg = " - invalid!" if not (CUDA_HOME and os.path.isdir(CUDA_HOME)) else "" + data.append(("CUDA_HOME", str(CUDA_HOME) + msg)) + + cuda_arch_list = os.environ.get("TORCH_CUDA_ARCH_LIST", None) + if cuda_arch_list: + data.append(("TORCH_CUDA_ARCH_LIST", cuda_arch_list)) + data.append(("Pillow", PIL.__version__)) + + try: + data.append( + ( + "torchvision", + str(torchvision.__version__) + " @" + os.path.dirname(torchvision.__file__), + ) + ) + if has_cuda: + try: + torchvision_C = importlib.util.find_spec("torchvision._C").origin + msg = detect_compute_compatibility(CUDA_HOME, torchvision_C) + data.append(("torchvision arch flags", msg)) + except (ImportError, AttributeError): + data.append(("torchvision._C", "Not found")) + except AttributeError: + data.append(("torchvision", "unknown")) + + try: + import fvcore + + data.append(("fvcore", fvcore.__version__)) + except (ImportError, AttributeError): + pass + + try: + import iopath + + data.append(("iopath", iopath.__version__)) + except (ImportError, AttributeError): + pass + + try: + import cv2 + + data.append(("cv2", cv2.__version__)) + except (ImportError, AttributeError): + data.append(("cv2", "Not found")) + env_str = tabulate(data) + "\n" + env_str += collect_torch_env() + return env_str + + +def test_nccl_ops(): + num_gpu = torch.cuda.device_count() + if os.access("/tmp", os.W_OK): + import torch.multiprocessing as mp + + dist_url = "file:///tmp/nccl_tmp_file" + print("Testing NCCL connectivity ... this should not hang.") + mp.spawn(_test_nccl_worker, nprocs=num_gpu, args=(num_gpu, dist_url), daemon=False) + print("NCCL succeeded.") + + +def _test_nccl_worker(rank, num_gpu, dist_url): + import torch.distributed as dist + + dist.init_process_group(backend="NCCL", init_method=dist_url, rank=rank, world_size=num_gpu) + dist.barrier(device_ids=[rank]) + + +if __name__ == "__main__": + try: + from annotator.oneformer.detectron2.utils.collect_env import collect_env_info as f + + print(f()) + except ImportError: + print(collect_env_info()) + + if torch.cuda.is_available(): + num_gpu = torch.cuda.device_count() + for k in range(num_gpu): + device = f"cuda:{k}" + try: + x = torch.tensor([1, 2.0], dtype=torch.float32) + x = x.to(device) + except Exception as e: + print( + f"Unable to copy tensor to device={device}: {e}. " + "Your CUDA environment is broken." + ) + if num_gpu > 1: + test_nccl_ops() diff --git a/annotator/oneformer/detectron2/utils/colormap.py b/annotator/oneformer/detectron2/utils/colormap.py new file mode 100644 index 0000000000000000000000000000000000000000..14ded1659b40b161358c4aaf9cc84ffe0ffafe64 --- /dev/null +++ b/annotator/oneformer/detectron2/utils/colormap.py @@ -0,0 +1,158 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +""" +An awesome colormap for really neat visualizations. +Copied from Detectron, and removed gray colors. +""" + +import numpy as np +import random + +__all__ = ["colormap", "random_color", "random_colors"] + +# fmt: off +# RGB: +_COLORS = np.array( + [ + 0.000, 0.447, 0.741, + 0.850, 0.325, 0.098, + 0.929, 0.694, 0.125, + 0.494, 0.184, 0.556, + 0.466, 0.674, 0.188, + 0.301, 0.745, 0.933, + 0.635, 0.078, 0.184, + 0.300, 0.300, 0.300, + 0.600, 0.600, 0.600, + 1.000, 0.000, 0.000, + 1.000, 0.500, 0.000, + 0.749, 0.749, 0.000, + 0.000, 1.000, 0.000, + 0.000, 0.000, 1.000, + 0.667, 0.000, 1.000, + 0.333, 0.333, 0.000, + 0.333, 0.667, 0.000, + 0.333, 1.000, 0.000, + 0.667, 0.333, 0.000, + 0.667, 0.667, 0.000, + 0.667, 1.000, 0.000, + 1.000, 0.333, 0.000, + 1.000, 0.667, 0.000, + 1.000, 1.000, 0.000, + 0.000, 0.333, 0.500, + 0.000, 0.667, 0.500, + 0.000, 1.000, 0.500, + 0.333, 0.000, 0.500, + 0.333, 0.333, 0.500, + 0.333, 0.667, 0.500, + 0.333, 1.000, 0.500, + 0.667, 0.000, 0.500, + 0.667, 0.333, 0.500, + 0.667, 0.667, 0.500, + 0.667, 1.000, 0.500, + 1.000, 0.000, 0.500, + 1.000, 0.333, 0.500, + 1.000, 0.667, 0.500, + 1.000, 1.000, 0.500, + 0.000, 0.333, 1.000, + 0.000, 0.667, 1.000, + 0.000, 1.000, 1.000, + 0.333, 0.000, 1.000, + 0.333, 0.333, 1.000, + 0.333, 0.667, 1.000, + 0.333, 1.000, 1.000, + 0.667, 0.000, 1.000, + 0.667, 0.333, 1.000, + 0.667, 0.667, 1.000, + 0.667, 1.000, 1.000, + 1.000, 0.000, 1.000, + 1.000, 0.333, 1.000, + 1.000, 0.667, 1.000, + 0.333, 0.000, 0.000, + 0.500, 0.000, 0.000, + 0.667, 0.000, 0.000, + 0.833, 0.000, 0.000, + 1.000, 0.000, 0.000, + 0.000, 0.167, 0.000, + 0.000, 0.333, 0.000, + 0.000, 0.500, 0.000, + 0.000, 0.667, 0.000, + 0.000, 0.833, 0.000, + 0.000, 1.000, 0.000, + 0.000, 0.000, 0.167, + 0.000, 0.000, 0.333, + 0.000, 0.000, 0.500, + 0.000, 0.000, 0.667, + 0.000, 0.000, 0.833, + 0.000, 0.000, 1.000, + 0.000, 0.000, 0.000, + 0.143, 0.143, 0.143, + 0.857, 0.857, 0.857, + 1.000, 1.000, 1.000 + ] +).astype(np.float32).reshape(-1, 3) +# fmt: on + + +def colormap(rgb=False, maximum=255): + """ + Args: + rgb (bool): whether to return RGB colors or BGR colors. + maximum (int): either 255 or 1 + + Returns: + ndarray: a float32 array of Nx3 colors, in range [0, 255] or [0, 1] + """ + assert maximum in [255, 1], maximum + c = _COLORS * maximum + if not rgb: + c = c[:, ::-1] + return c + + +def random_color(rgb=False, maximum=255): + """ + Args: + rgb (bool): whether to return RGB colors or BGR colors. + maximum (int): either 255 or 1 + + Returns: + ndarray: a vector of 3 numbers + """ + idx = np.random.randint(0, len(_COLORS)) + ret = _COLORS[idx] * maximum + if not rgb: + ret = ret[::-1] + return ret + + +def random_colors(N, rgb=False, maximum=255): + """ + Args: + N (int): number of unique colors needed + rgb (bool): whether to return RGB colors or BGR colors. + maximum (int): either 255 or 1 + + Returns: + ndarray: a list of random_color + """ + indices = random.sample(range(len(_COLORS)), N) + ret = [_COLORS[i] * maximum for i in indices] + if not rgb: + ret = [x[::-1] for x in ret] + return ret + + +if __name__ == "__main__": + import cv2 + + size = 100 + H, W = 10, 10 + canvas = np.random.rand(H * size, W * size, 3).astype("float32") + for h in range(H): + for w in range(W): + idx = h * W + w + if idx >= len(_COLORS): + break + canvas[h * size : (h + 1) * size, w * size : (w + 1) * size] = _COLORS[idx] + cv2.imshow("a", canvas) + cv2.waitKey(0) diff --git a/annotator/oneformer/detectron2/utils/comm.py b/annotator/oneformer/detectron2/utils/comm.py new file mode 100644 index 0000000000000000000000000000000000000000..a9ea9a9f578c5704d1e7ff563ef156e9133ab465 --- /dev/null +++ b/annotator/oneformer/detectron2/utils/comm.py @@ -0,0 +1,238 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +""" +This file contains primitives for multi-gpu communication. +This is useful when doing distributed training. +""" + +import functools +import numpy as np +import torch +import torch.distributed as dist + +_LOCAL_PROCESS_GROUP = None +_MISSING_LOCAL_PG_ERROR = ( + "Local process group is not yet created! Please use detectron2's `launch()` " + "to start processes and initialize pytorch process group. If you need to start " + "processes in other ways, please call comm.create_local_process_group(" + "num_workers_per_machine) after calling torch.distributed.init_process_group()." +) + + +def get_world_size() -> int: + if not dist.is_available(): + return 1 + if not dist.is_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank() -> int: + if not dist.is_available(): + return 0 + if not dist.is_initialized(): + return 0 + return dist.get_rank() + + +@functools.lru_cache() +def create_local_process_group(num_workers_per_machine: int) -> None: + """ + Create a process group that contains ranks within the same machine. + + Detectron2's launch() in engine/launch.py will call this function. If you start + workers without launch(), you'll have to also call this. Otherwise utilities + like `get_local_rank()` will not work. + + This function contains a barrier. All processes must call it together. + + Args: + num_workers_per_machine: the number of worker processes per machine. Typically + the number of GPUs. + """ + global _LOCAL_PROCESS_GROUP + assert _LOCAL_PROCESS_GROUP is None + assert get_world_size() % num_workers_per_machine == 0 + num_machines = get_world_size() // num_workers_per_machine + machine_rank = get_rank() // num_workers_per_machine + for i in range(num_machines): + ranks_on_i = list(range(i * num_workers_per_machine, (i + 1) * num_workers_per_machine)) + pg = dist.new_group(ranks_on_i) + if i == machine_rank: + _LOCAL_PROCESS_GROUP = pg + + +def get_local_process_group(): + """ + Returns: + A torch process group which only includes processes that are on the same + machine as the current process. This group can be useful for communication + within a machine, e.g. a per-machine SyncBN. + """ + assert _LOCAL_PROCESS_GROUP is not None, _MISSING_LOCAL_PG_ERROR + return _LOCAL_PROCESS_GROUP + + +def get_local_rank() -> int: + """ + Returns: + The rank of the current process within the local (per-machine) process group. + """ + if not dist.is_available(): + return 0 + if not dist.is_initialized(): + return 0 + assert _LOCAL_PROCESS_GROUP is not None, _MISSING_LOCAL_PG_ERROR + return dist.get_rank(group=_LOCAL_PROCESS_GROUP) + + +def get_local_size() -> int: + """ + Returns: + The size of the per-machine process group, + i.e. the number of processes per machine. + """ + if not dist.is_available(): + return 1 + if not dist.is_initialized(): + return 1 + assert _LOCAL_PROCESS_GROUP is not None, _MISSING_LOCAL_PG_ERROR + return dist.get_world_size(group=_LOCAL_PROCESS_GROUP) + + +def is_main_process() -> bool: + return get_rank() == 0 + + +def synchronize(): + """ + Helper function to synchronize (barrier) among all processes when + using distributed training + """ + if not dist.is_available(): + return + if not dist.is_initialized(): + return + world_size = dist.get_world_size() + if world_size == 1: + return + if dist.get_backend() == dist.Backend.NCCL: + # This argument is needed to avoid warnings. + # It's valid only for NCCL backend. + dist.barrier(device_ids=[torch.cuda.current_device()]) + else: + dist.barrier() + + +@functools.lru_cache() +def _get_global_gloo_group(): + """ + Return a process group based on gloo backend, containing all the ranks + The result is cached. + """ + if dist.get_backend() == "nccl": + return dist.new_group(backend="gloo") + else: + return dist.group.WORLD + + +def all_gather(data, group=None): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors). + + Args: + data: any picklable object + group: a torch process group. By default, will use a group which + contains all ranks on gloo backend. + + Returns: + list[data]: list of data gathered from each rank + """ + if get_world_size() == 1: + return [data] + if group is None: + group = _get_global_gloo_group() # use CPU group by default, to reduce GPU RAM usage. + world_size = dist.get_world_size(group) + if world_size == 1: + return [data] + + output = [None for _ in range(world_size)] + dist.all_gather_object(output, data, group=group) + return output + + +def gather(data, dst=0, group=None): + """ + Run gather on arbitrary picklable data (not necessarily tensors). + + Args: + data: any picklable object + dst (int): destination rank + group: a torch process group. By default, will use a group which + contains all ranks on gloo backend. + + Returns: + list[data]: on dst, a list of data gathered from each rank. Otherwise, + an empty list. + """ + if get_world_size() == 1: + return [data] + if group is None: + group = _get_global_gloo_group() + world_size = dist.get_world_size(group=group) + if world_size == 1: + return [data] + rank = dist.get_rank(group=group) + + if rank == dst: + output = [None for _ in range(world_size)] + dist.gather_object(data, output, dst=dst, group=group) + return output + else: + dist.gather_object(data, None, dst=dst, group=group) + return [] + + +def shared_random_seed(): + """ + Returns: + int: a random number that is the same across all workers. + If workers need a shared RNG, they can use this shared seed to + create one. + + All workers must call this function, otherwise it will deadlock. + """ + ints = np.random.randint(2**31) + all_ints = all_gather(ints) + return all_ints[0] + + +def reduce_dict(input_dict, average=True): + """ + Reduce the values in the dictionary from all processes so that process with rank + 0 has the reduced results. + + Args: + input_dict (dict): inputs to be reduced. All the values must be scalar CUDA Tensor. + average (bool): whether to do average or sum + + Returns: + a dict with the same keys as input_dict, after reduction. + """ + world_size = get_world_size() + if world_size < 2: + return input_dict + with torch.no_grad(): + names = [] + values = [] + # sort the keys so that they are consistent across processes + for k in sorted(input_dict.keys()): + names.append(k) + values.append(input_dict[k]) + values = torch.stack(values, dim=0) + dist.reduce(values, dst=0) + if dist.get_rank() == 0 and average: + # only main process gets accumulated, so only divide by + # world_size in this case + values /= world_size + reduced_dict = {k: v for k, v in zip(names, values)} + return reduced_dict diff --git a/annotator/oneformer/detectron2/utils/develop.py b/annotator/oneformer/detectron2/utils/develop.py new file mode 100644 index 0000000000000000000000000000000000000000..e8416984954f7b32fc269100620e3c0d0d0f9585 --- /dev/null +++ b/annotator/oneformer/detectron2/utils/develop.py @@ -0,0 +1,59 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +""" Utilities for developers only. +These are not visible to users (not automatically imported). And should not +appeared in docs.""" +# adapted from https://github.com/tensorpack/tensorpack/blob/master/tensorpack/utils/develop.py + + +def create_dummy_class(klass, dependency, message=""): + """ + When a dependency of a class is not available, create a dummy class which throws ImportError + when used. + + Args: + klass (str): name of the class. + dependency (str): name of the dependency. + message: extra message to print + Returns: + class: a class object + """ + err = "Cannot import '{}', therefore '{}' is not available.".format(dependency, klass) + if message: + err = err + " " + message + + class _DummyMetaClass(type): + # throw error on class attribute access + def __getattr__(_, __): # noqa: B902 + raise ImportError(err) + + class _Dummy(object, metaclass=_DummyMetaClass): + # throw error on constructor + def __init__(self, *args, **kwargs): + raise ImportError(err) + + return _Dummy + + +def create_dummy_func(func, dependency, message=""): + """ + When a dependency of a function is not available, create a dummy function which throws + ImportError when used. + + Args: + func (str): name of the function. + dependency (str or list[str]): name(s) of the dependency. + message: extra message to print + Returns: + function: a function object + """ + err = "Cannot import '{}', therefore '{}' is not available.".format(dependency, func) + if message: + err = err + " " + message + + if isinstance(dependency, (list, tuple)): + dependency = ",".join(dependency) + + def _dummy(*args, **kwargs): + raise ImportError(err) + + return _dummy diff --git a/annotator/oneformer/detectron2/utils/env.py b/annotator/oneformer/detectron2/utils/env.py new file mode 100644 index 0000000000000000000000000000000000000000..40634c17c73273ac8927632be164f466cfe7d1fa --- /dev/null +++ b/annotator/oneformer/detectron2/utils/env.py @@ -0,0 +1,170 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import importlib +import importlib.util +import logging +import numpy as np +import os +import random +import sys +from datetime import datetime +import torch + +__all__ = ["seed_all_rng"] + + +TORCH_VERSION = tuple(int(x) for x in torch.__version__.split(".")[:2]) +""" +PyTorch version as a tuple of 2 ints. Useful for comparison. +""" + + +DOC_BUILDING = os.getenv("_DOC_BUILDING", False) # set in docs/conf.py +""" +Whether we're building documentation. +""" + + +def seed_all_rng(seed=None): + """ + Set the random seed for the RNG in torch, numpy and python. + + Args: + seed (int): if None, will use a strong random seed. + """ + if seed is None: + seed = ( + os.getpid() + + int(datetime.now().strftime("%S%f")) + + int.from_bytes(os.urandom(2), "big") + ) + logger = logging.getLogger(__name__) + logger.info("Using a generated random seed {}".format(seed)) + np.random.seed(seed) + torch.manual_seed(seed) + random.seed(seed) + os.environ["PYTHONHASHSEED"] = str(seed) + + +# from https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path +def _import_file(module_name, file_path, make_importable=False): + spec = importlib.util.spec_from_file_location(module_name, file_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + if make_importable: + sys.modules[module_name] = module + return module + + +def _configure_libraries(): + """ + Configurations for some libraries. + """ + # An environment option to disable `import cv2` globally, + # in case it leads to negative performance impact + disable_cv2 = int(os.environ.get("DETECTRON2_DISABLE_CV2", False)) + if disable_cv2: + sys.modules["cv2"] = None + else: + # Disable opencl in opencv since its interaction with cuda often has negative effects + # This envvar is supported after OpenCV 3.4.0 + os.environ["OPENCV_OPENCL_RUNTIME"] = "disabled" + try: + import cv2 + + if int(cv2.__version__.split(".")[0]) >= 3: + cv2.ocl.setUseOpenCL(False) + except ModuleNotFoundError: + # Other types of ImportError, if happened, should not be ignored. + # Because a failed opencv import could mess up address space + # https://github.com/skvark/opencv-python/issues/381 + pass + + def get_version(module, digit=2): + return tuple(map(int, module.__version__.split(".")[:digit])) + + # fmt: off + assert get_version(torch) >= (1, 4), "Requires torch>=1.4" + import fvcore + assert get_version(fvcore, 3) >= (0, 1, 2), "Requires fvcore>=0.1.2" + import yaml + assert get_version(yaml) >= (5, 1), "Requires pyyaml>=5.1" + # fmt: on + + +_ENV_SETUP_DONE = False + + +def setup_environment(): + """Perform environment setup work. The default setup is a no-op, but this + function allows the user to specify a Python source file or a module in + the $DETECTRON2_ENV_MODULE environment variable, that performs + custom setup work that may be necessary to their computing environment. + """ + global _ENV_SETUP_DONE + if _ENV_SETUP_DONE: + return + _ENV_SETUP_DONE = True + + _configure_libraries() + + custom_module_path = os.environ.get("DETECTRON2_ENV_MODULE") + + if custom_module_path: + setup_custom_environment(custom_module_path) + else: + # The default setup is a no-op + pass + + +def setup_custom_environment(custom_module): + """ + Load custom environment setup by importing a Python source file or a + module, and run the setup function. + """ + if custom_module.endswith(".py"): + module = _import_file("detectron2.utils.env.custom_module", custom_module) + else: + module = importlib.import_module(custom_module) + assert hasattr(module, "setup_environment") and callable(module.setup_environment), ( + "Custom environment module defined in {} does not have the " + "required callable attribute 'setup_environment'." + ).format(custom_module) + module.setup_environment() + + +def fixup_module_metadata(module_name, namespace, keys=None): + """ + Fix the __qualname__ of module members to be their exported api name, so + when they are referenced in docs, sphinx can find them. Reference: + https://github.com/python-trio/trio/blob/6754c74eacfad9cc5c92d5c24727a2f3b620624e/trio/_util.py#L216-L241 + """ + if not DOC_BUILDING: + return + seen_ids = set() + + def fix_one(qualname, name, obj): + # avoid infinite recursion (relevant when using + # typing.Generic, for example) + if id(obj) in seen_ids: + return + seen_ids.add(id(obj)) + + mod = getattr(obj, "__module__", None) + if mod is not None and (mod.startswith(module_name) or mod.startswith("fvcore.")): + obj.__module__ = module_name + # Modules, unlike everything else in Python, put fully-qualitied + # names into their __name__ attribute. We check for "." to avoid + # rewriting these. + if hasattr(obj, "__name__") and "." not in obj.__name__: + obj.__name__ = name + obj.__qualname__ = qualname + if isinstance(obj, type): + for attr_name, attr_value in obj.__dict__.items(): + fix_one(objname + "." + attr_name, attr_name, attr_value) + + if keys is None: + keys = namespace.keys() + for objname in keys: + if not objname.startswith("_"): + obj = namespace[objname] + fix_one(objname, objname, obj) diff --git a/annotator/oneformer/detectron2/utils/events.py b/annotator/oneformer/detectron2/utils/events.py new file mode 100644 index 0000000000000000000000000000000000000000..d9a68b6b5b90cdef1ccdaffa4eb2225f3ab21e29 --- /dev/null +++ b/annotator/oneformer/detectron2/utils/events.py @@ -0,0 +1,534 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import datetime +import json +import logging +import os +import time +from collections import defaultdict +from contextlib import contextmanager +from typing import Optional +import torch +from fvcore.common.history_buffer import HistoryBuffer + +from annotator.oneformer.detectron2.utils.file_io import PathManager + +__all__ = [ + "get_event_storage", + "JSONWriter", + "TensorboardXWriter", + "CommonMetricPrinter", + "EventStorage", +] + +_CURRENT_STORAGE_STACK = [] + + +def get_event_storage(): + """ + Returns: + The :class:`EventStorage` object that's currently being used. + Throws an error if no :class:`EventStorage` is currently enabled. + """ + assert len( + _CURRENT_STORAGE_STACK + ), "get_event_storage() has to be called inside a 'with EventStorage(...)' context!" + return _CURRENT_STORAGE_STACK[-1] + + +class EventWriter: + """ + Base class for writers that obtain events from :class:`EventStorage` and process them. + """ + + def write(self): + raise NotImplementedError + + def close(self): + pass + + +class JSONWriter(EventWriter): + """ + Write scalars to a json file. + + It saves scalars as one json per line (instead of a big json) for easy parsing. + + Examples parsing such a json file: + :: + $ cat metrics.json | jq -s '.[0:2]' + [ + { + "data_time": 0.008433341979980469, + "iteration": 19, + "loss": 1.9228371381759644, + "loss_box_reg": 0.050025828182697296, + "loss_classifier": 0.5316952466964722, + "loss_mask": 0.7236229181289673, + "loss_rpn_box": 0.0856662318110466, + "loss_rpn_cls": 0.48198649287223816, + "lr": 0.007173333333333333, + "time": 0.25401854515075684 + }, + { + "data_time": 0.007216215133666992, + "iteration": 39, + "loss": 1.282649278640747, + "loss_box_reg": 0.06222952902317047, + "loss_classifier": 0.30682939291000366, + "loss_mask": 0.6970193982124329, + "loss_rpn_box": 0.038663312792778015, + "loss_rpn_cls": 0.1471673548221588, + "lr": 0.007706666666666667, + "time": 0.2490077018737793 + } + ] + + $ cat metrics.json | jq '.loss_mask' + 0.7126231789588928 + 0.689423680305481 + 0.6776131987571716 + ... + + """ + + def __init__(self, json_file, window_size=20): + """ + Args: + json_file (str): path to the json file. New data will be appended if the file exists. + window_size (int): the window size of median smoothing for the scalars whose + `smoothing_hint` are True. + """ + self._file_handle = PathManager.open(json_file, "a") + self._window_size = window_size + self._last_write = -1 + + def write(self): + storage = get_event_storage() + to_save = defaultdict(dict) + + for k, (v, iter) in storage.latest_with_smoothing_hint(self._window_size).items(): + # keep scalars that have not been written + if iter <= self._last_write: + continue + to_save[iter][k] = v + if len(to_save): + all_iters = sorted(to_save.keys()) + self._last_write = max(all_iters) + + for itr, scalars_per_iter in to_save.items(): + scalars_per_iter["iteration"] = itr + self._file_handle.write(json.dumps(scalars_per_iter, sort_keys=True) + "\n") + self._file_handle.flush() + try: + os.fsync(self._file_handle.fileno()) + except AttributeError: + pass + + def close(self): + self._file_handle.close() + + +class TensorboardXWriter(EventWriter): + """ + Write all scalars to a tensorboard file. + """ + + def __init__(self, log_dir: str, window_size: int = 20, **kwargs): + """ + Args: + log_dir (str): the directory to save the output events + window_size (int): the scalars will be median-smoothed by this window size + + kwargs: other arguments passed to `torch.utils.tensorboard.SummaryWriter(...)` + """ + self._window_size = window_size + from torch.utils.tensorboard import SummaryWriter + + self._writer = SummaryWriter(log_dir, **kwargs) + self._last_write = -1 + + def write(self): + storage = get_event_storage() + new_last_write = self._last_write + for k, (v, iter) in storage.latest_with_smoothing_hint(self._window_size).items(): + if iter > self._last_write: + self._writer.add_scalar(k, v, iter) + new_last_write = max(new_last_write, iter) + self._last_write = new_last_write + + # storage.put_{image,histogram} is only meant to be used by + # tensorboard writer. So we access its internal fields directly from here. + if len(storage._vis_data) >= 1: + for img_name, img, step_num in storage._vis_data: + self._writer.add_image(img_name, img, step_num) + # Storage stores all image data and rely on this writer to clear them. + # As a result it assumes only one writer will use its image data. + # An alternative design is to let storage store limited recent + # data (e.g. only the most recent image) that all writers can access. + # In that case a writer may not see all image data if its period is long. + storage.clear_images() + + if len(storage._histograms) >= 1: + for params in storage._histograms: + self._writer.add_histogram_raw(**params) + storage.clear_histograms() + + def close(self): + if hasattr(self, "_writer"): # doesn't exist when the code fails at import + self._writer.close() + + +class CommonMetricPrinter(EventWriter): + """ + Print **common** metrics to the terminal, including + iteration time, ETA, memory, all losses, and the learning rate. + It also applies smoothing using a window of 20 elements. + + It's meant to print common metrics in common ways. + To print something in more customized ways, please implement a similar printer by yourself. + """ + + def __init__(self, max_iter: Optional[int] = None, window_size: int = 20): + """ + Args: + max_iter: the maximum number of iterations to train. + Used to compute ETA. If not given, ETA will not be printed. + window_size (int): the losses will be median-smoothed by this window size + """ + self.logger = logging.getLogger(__name__) + self._max_iter = max_iter + self._window_size = window_size + self._last_write = None # (step, time) of last call to write(). Used to compute ETA + + def _get_eta(self, storage) -> Optional[str]: + if self._max_iter is None: + return "" + iteration = storage.iter + try: + eta_seconds = storage.history("time").median(1000) * (self._max_iter - iteration - 1) + storage.put_scalar("eta_seconds", eta_seconds, smoothing_hint=False) + return str(datetime.timedelta(seconds=int(eta_seconds))) + except KeyError: + # estimate eta on our own - more noisy + eta_string = None + if self._last_write is not None: + estimate_iter_time = (time.perf_counter() - self._last_write[1]) / ( + iteration - self._last_write[0] + ) + eta_seconds = estimate_iter_time * (self._max_iter - iteration - 1) + eta_string = str(datetime.timedelta(seconds=int(eta_seconds))) + self._last_write = (iteration, time.perf_counter()) + return eta_string + + def write(self): + storage = get_event_storage() + iteration = storage.iter + if iteration == self._max_iter: + # This hook only reports training progress (loss, ETA, etc) but not other data, + # therefore do not write anything after training succeeds, even if this method + # is called. + return + + try: + avg_data_time = storage.history("data_time").avg( + storage.count_samples("data_time", self._window_size) + ) + last_data_time = storage.history("data_time").latest() + except KeyError: + # they may not exist in the first few iterations (due to warmup) + # or when SimpleTrainer is not used + avg_data_time = None + last_data_time = None + try: + avg_iter_time = storage.history("time").global_avg() + last_iter_time = storage.history("time").latest() + except KeyError: + avg_iter_time = None + last_iter_time = None + try: + lr = "{:.5g}".format(storage.history("lr").latest()) + except KeyError: + lr = "N/A" + + eta_string = self._get_eta(storage) + + if torch.cuda.is_available(): + max_mem_mb = torch.cuda.max_memory_allocated() / 1024.0 / 1024.0 + else: + max_mem_mb = None + + # NOTE: max_mem is parsed by grep in "dev/parse_results.sh" + self.logger.info( + str.format( + " {eta}iter: {iter} {losses} {non_losses} {avg_time}{last_time}" + + "{avg_data_time}{last_data_time} lr: {lr} {memory}", + eta=f"eta: {eta_string} " if eta_string else "", + iter=iteration, + losses=" ".join( + [ + "{}: {:.4g}".format( + k, v.median(storage.count_samples(k, self._window_size)) + ) + for k, v in storage.histories().items() + if "loss" in k + ] + ), + non_losses=" ".join( + [ + "{}: {:.4g}".format( + k, v.median(storage.count_samples(k, self._window_size)) + ) + for k, v in storage.histories().items() + if "[metric]" in k + ] + ), + avg_time="time: {:.4f} ".format(avg_iter_time) + if avg_iter_time is not None + else "", + last_time="last_time: {:.4f} ".format(last_iter_time) + if last_iter_time is not None + else "", + avg_data_time="data_time: {:.4f} ".format(avg_data_time) + if avg_data_time is not None + else "", + last_data_time="last_data_time: {:.4f} ".format(last_data_time) + if last_data_time is not None + else "", + lr=lr, + memory="max_mem: {:.0f}M".format(max_mem_mb) if max_mem_mb is not None else "", + ) + ) + + +class EventStorage: + """ + The user-facing class that provides metric storage functionalities. + + In the future we may add support for storing / logging other types of data if needed. + """ + + def __init__(self, start_iter=0): + """ + Args: + start_iter (int): the iteration number to start with + """ + self._history = defaultdict(HistoryBuffer) + self._smoothing_hints = {} + self._latest_scalars = {} + self._iter = start_iter + self._current_prefix = "" + self._vis_data = [] + self._histograms = [] + + def put_image(self, img_name, img_tensor): + """ + Add an `img_tensor` associated with `img_name`, to be shown on + tensorboard. + + Args: + img_name (str): The name of the image to put into tensorboard. + img_tensor (torch.Tensor or numpy.array): An `uint8` or `float` + Tensor of shape `[channel, height, width]` where `channel` is + 3. The image format should be RGB. The elements in img_tensor + can either have values in [0, 1] (float32) or [0, 255] (uint8). + The `img_tensor` will be visualized in tensorboard. + """ + self._vis_data.append((img_name, img_tensor, self._iter)) + + def put_scalar(self, name, value, smoothing_hint=True): + """ + Add a scalar `value` to the `HistoryBuffer` associated with `name`. + + Args: + smoothing_hint (bool): a 'hint' on whether this scalar is noisy and should be + smoothed when logged. The hint will be accessible through + :meth:`EventStorage.smoothing_hints`. A writer may ignore the hint + and apply custom smoothing rule. + + It defaults to True because most scalars we save need to be smoothed to + provide any useful signal. + """ + name = self._current_prefix + name + history = self._history[name] + value = float(value) + history.update(value, self._iter) + self._latest_scalars[name] = (value, self._iter) + + existing_hint = self._smoothing_hints.get(name) + if existing_hint is not None: + assert ( + existing_hint == smoothing_hint + ), "Scalar {} was put with a different smoothing_hint!".format(name) + else: + self._smoothing_hints[name] = smoothing_hint + + def put_scalars(self, *, smoothing_hint=True, **kwargs): + """ + Put multiple scalars from keyword arguments. + + Examples: + + storage.put_scalars(loss=my_loss, accuracy=my_accuracy, smoothing_hint=True) + """ + for k, v in kwargs.items(): + self.put_scalar(k, v, smoothing_hint=smoothing_hint) + + def put_histogram(self, hist_name, hist_tensor, bins=1000): + """ + Create a histogram from a tensor. + + Args: + hist_name (str): The name of the histogram to put into tensorboard. + hist_tensor (torch.Tensor): A Tensor of arbitrary shape to be converted + into a histogram. + bins (int): Number of histogram bins. + """ + ht_min, ht_max = hist_tensor.min().item(), hist_tensor.max().item() + + # Create a histogram with PyTorch + hist_counts = torch.histc(hist_tensor, bins=bins) + hist_edges = torch.linspace(start=ht_min, end=ht_max, steps=bins + 1, dtype=torch.float32) + + # Parameter for the add_histogram_raw function of SummaryWriter + hist_params = dict( + tag=hist_name, + min=ht_min, + max=ht_max, + num=len(hist_tensor), + sum=float(hist_tensor.sum()), + sum_squares=float(torch.sum(hist_tensor**2)), + bucket_limits=hist_edges[1:].tolist(), + bucket_counts=hist_counts.tolist(), + global_step=self._iter, + ) + self._histograms.append(hist_params) + + def history(self, name): + """ + Returns: + HistoryBuffer: the scalar history for name + """ + ret = self._history.get(name, None) + if ret is None: + raise KeyError("No history metric available for {}!".format(name)) + return ret + + def histories(self): + """ + Returns: + dict[name -> HistoryBuffer]: the HistoryBuffer for all scalars + """ + return self._history + + def latest(self): + """ + Returns: + dict[str -> (float, int)]: mapping from the name of each scalar to the most + recent value and the iteration number its added. + """ + return self._latest_scalars + + def latest_with_smoothing_hint(self, window_size=20): + """ + Similar to :meth:`latest`, but the returned values + are either the un-smoothed original latest value, + or a median of the given window_size, + depend on whether the smoothing_hint is True. + + This provides a default behavior that other writers can use. + + Note: All scalars saved in the past `window_size` iterations are used for smoothing. + This is different from the `window_size` definition in HistoryBuffer. + Use :meth:`get_history_window_size` to get the `window_size` used in HistoryBuffer. + """ + result = {} + for k, (v, itr) in self._latest_scalars.items(): + result[k] = ( + self._history[k].median(self.count_samples(k, window_size)) + if self._smoothing_hints[k] + else v, + itr, + ) + return result + + def count_samples(self, name, window_size=20): + """ + Return the number of samples logged in the past `window_size` iterations. + """ + samples = 0 + data = self._history[name].values() + for _, iter_ in reversed(data): + if iter_ > data[-1][1] - window_size: + samples += 1 + else: + break + return samples + + def smoothing_hints(self): + """ + Returns: + dict[name -> bool]: the user-provided hint on whether the scalar + is noisy and needs smoothing. + """ + return self._smoothing_hints + + def step(self): + """ + User should either: (1) Call this function to increment storage.iter when needed. Or + (2) Set `storage.iter` to the correct iteration number before each iteration. + + The storage will then be able to associate the new data with an iteration number. + """ + self._iter += 1 + + @property + def iter(self): + """ + Returns: + int: The current iteration number. When used together with a trainer, + this is ensured to be the same as trainer.iter. + """ + return self._iter + + @iter.setter + def iter(self, val): + self._iter = int(val) + + @property + def iteration(self): + # for backward compatibility + return self._iter + + def __enter__(self): + _CURRENT_STORAGE_STACK.append(self) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + assert _CURRENT_STORAGE_STACK[-1] == self + _CURRENT_STORAGE_STACK.pop() + + @contextmanager + def name_scope(self, name): + """ + Yields: + A context within which all the events added to this storage + will be prefixed by the name scope. + """ + old_prefix = self._current_prefix + self._current_prefix = name.rstrip("/") + "/" + yield + self._current_prefix = old_prefix + + def clear_images(self): + """ + Delete all the stored images for visualization. This should be called + after images are written to tensorboard. + """ + self._vis_data = [] + + def clear_histograms(self): + """ + Delete all the stored histograms for visualization. + This should be called after histograms are written to tensorboard. + """ + self._histograms = [] diff --git a/annotator/oneformer/detectron2/utils/file_io.py b/annotator/oneformer/detectron2/utils/file_io.py new file mode 100644 index 0000000000000000000000000000000000000000..09f7dffdb36199350bba57bd3b4e9e8babb40594 --- /dev/null +++ b/annotator/oneformer/detectron2/utils/file_io.py @@ -0,0 +1,39 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from iopath.common.file_io import HTTPURLHandler, OneDrivePathHandler, PathHandler +from iopath.common.file_io import PathManager as PathManagerBase + +__all__ = ["PathManager", "PathHandler"] + + +PathManager = PathManagerBase() +""" +This is a detectron2 project-specific PathManager. +We try to stay away from global PathManager in fvcore as it +introduces potential conflicts among other libraries. +""" + + +class Detectron2Handler(PathHandler): + """ + Resolve anything that's hosted under detectron2's namespace. + """ + + PREFIX = "detectron2://" + S3_DETECTRON2_PREFIX = "https://dl.fbaipublicfiles.com/detectron2/" + + def _get_supported_prefixes(self): + return [self.PREFIX] + + def _get_local_path(self, path, **kwargs): + name = path[len(self.PREFIX) :] + return PathManager.get_local_path(self.S3_DETECTRON2_PREFIX + name, **kwargs) + + def _open(self, path, mode="r", **kwargs): + return PathManager.open( + self.S3_DETECTRON2_PREFIX + path[len(self.PREFIX) :], mode, **kwargs + ) + + +PathManager.register_handler(HTTPURLHandler()) +PathManager.register_handler(OneDrivePathHandler()) +PathManager.register_handler(Detectron2Handler()) diff --git a/annotator/oneformer/detectron2/utils/logger.py b/annotator/oneformer/detectron2/utils/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..d77d42cbe86366e5d91e93311f92bb166c304184 --- /dev/null +++ b/annotator/oneformer/detectron2/utils/logger.py @@ -0,0 +1,237 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import atexit +import functools +import logging +import os +import sys +import time +from collections import Counter +import torch +from tabulate import tabulate +from termcolor import colored + +from annotator.oneformer.detectron2.utils.file_io import PathManager + +__all__ = ["setup_logger", "log_first_n", "log_every_n", "log_every_n_seconds"] + + +class _ColorfulFormatter(logging.Formatter): + def __init__(self, *args, **kwargs): + self._root_name = kwargs.pop("root_name") + "." + self._abbrev_name = kwargs.pop("abbrev_name", "") + if len(self._abbrev_name): + self._abbrev_name = self._abbrev_name + "." + super(_ColorfulFormatter, self).__init__(*args, **kwargs) + + def formatMessage(self, record): + record.name = record.name.replace(self._root_name, self._abbrev_name) + log = super(_ColorfulFormatter, self).formatMessage(record) + if record.levelno == logging.WARNING: + prefix = colored("WARNING", "red", attrs=["blink"]) + elif record.levelno == logging.ERROR or record.levelno == logging.CRITICAL: + prefix = colored("ERROR", "red", attrs=["blink", "underline"]) + else: + return log + return prefix + " " + log + + +@functools.lru_cache() # so that calling setup_logger multiple times won't add many handlers +def setup_logger( + output=None, distributed_rank=0, *, color=True, name="detectron2", abbrev_name=None +): + """ + Initialize the detectron2 logger and set its verbosity level to "DEBUG". + + Args: + output (str): a file name or a directory to save log. If None, will not save log file. + If ends with ".txt" or ".log", assumed to be a file name. + Otherwise, logs will be saved to `output/log.txt`. + name (str): the root module name of this logger + abbrev_name (str): an abbreviation of the module, to avoid long names in logs. + Set to "" to not log the root module in logs. + By default, will abbreviate "detectron2" to "d2" and leave other + modules unchanged. + + Returns: + logging.Logger: a logger + """ + logger = logging.getLogger(name) + logger.setLevel(logging.DEBUG) + logger.propagate = False + + if abbrev_name is None: + abbrev_name = "d2" if name == "detectron2" else name + + plain_formatter = logging.Formatter( + "[%(asctime)s] %(name)s %(levelname)s: %(message)s", datefmt="%m/%d %H:%M:%S" + ) + # stdout logging: master only + if distributed_rank == 0: + ch = logging.StreamHandler(stream=sys.stdout) + ch.setLevel(logging.DEBUG) + if color: + formatter = _ColorfulFormatter( + colored("[%(asctime)s %(name)s]: ", "green") + "%(message)s", + datefmt="%m/%d %H:%M:%S", + root_name=name, + abbrev_name=str(abbrev_name), + ) + else: + formatter = plain_formatter + ch.setFormatter(formatter) + logger.addHandler(ch) + + # file logging: all workers + if output is not None: + if output.endswith(".txt") or output.endswith(".log"): + filename = output + else: + filename = os.path.join(output, "log.txt") + if distributed_rank > 0: + filename = filename + ".rank{}".format(distributed_rank) + PathManager.mkdirs(os.path.dirname(filename)) + + fh = logging.StreamHandler(_cached_log_stream(filename)) + fh.setLevel(logging.DEBUG) + fh.setFormatter(plain_formatter) + logger.addHandler(fh) + + return logger + + +# cache the opened file object, so that different calls to `setup_logger` +# with the same file name can safely write to the same file. +@functools.lru_cache(maxsize=None) +def _cached_log_stream(filename): + # use 1K buffer if writing to cloud storage + io = PathManager.open(filename, "a", buffering=1024 if "://" in filename else -1) + atexit.register(io.close) + return io + + +""" +Below are some other convenient logging methods. +They are mainly adopted from +https://github.com/abseil/abseil-py/blob/master/absl/logging/__init__.py +""" + + +def _find_caller(): + """ + Returns: + str: module name of the caller + tuple: a hashable key to be used to identify different callers + """ + frame = sys._getframe(2) + while frame: + code = frame.f_code + if os.path.join("utils", "logger.") not in code.co_filename: + mod_name = frame.f_globals["__name__"] + if mod_name == "__main__": + mod_name = "detectron2" + return mod_name, (code.co_filename, frame.f_lineno, code.co_name) + frame = frame.f_back + + +_LOG_COUNTER = Counter() +_LOG_TIMER = {} + + +def log_first_n(lvl, msg, n=1, *, name=None, key="caller"): + """ + Log only for the first n times. + + Args: + lvl (int): the logging level + msg (str): + n (int): + name (str): name of the logger to use. Will use the caller's module by default. + key (str or tuple[str]): the string(s) can be one of "caller" or + "message", which defines how to identify duplicated logs. + For example, if called with `n=1, key="caller"`, this function + will only log the first call from the same caller, regardless of + the message content. + If called with `n=1, key="message"`, this function will log the + same content only once, even if they are called from different places. + If called with `n=1, key=("caller", "message")`, this function + will not log only if the same caller has logged the same message before. + """ + if isinstance(key, str): + key = (key,) + assert len(key) > 0 + + caller_module, caller_key = _find_caller() + hash_key = () + if "caller" in key: + hash_key = hash_key + caller_key + if "message" in key: + hash_key = hash_key + (msg,) + + _LOG_COUNTER[hash_key] += 1 + if _LOG_COUNTER[hash_key] <= n: + logging.getLogger(name or caller_module).log(lvl, msg) + + +def log_every_n(lvl, msg, n=1, *, name=None): + """ + Log once per n times. + + Args: + lvl (int): the logging level + msg (str): + n (int): + name (str): name of the logger to use. Will use the caller's module by default. + """ + caller_module, key = _find_caller() + _LOG_COUNTER[key] += 1 + if n == 1 or _LOG_COUNTER[key] % n == 1: + logging.getLogger(name or caller_module).log(lvl, msg) + + +def log_every_n_seconds(lvl, msg, n=1, *, name=None): + """ + Log no more than once per n seconds. + + Args: + lvl (int): the logging level + msg (str): + n (int): + name (str): name of the logger to use. Will use the caller's module by default. + """ + caller_module, key = _find_caller() + last_logged = _LOG_TIMER.get(key, None) + current_time = time.time() + if last_logged is None or current_time - last_logged >= n: + logging.getLogger(name or caller_module).log(lvl, msg) + _LOG_TIMER[key] = current_time + + +def create_small_table(small_dict): + """ + Create a small table using the keys of small_dict as headers. This is only + suitable for small dictionaries. + + Args: + small_dict (dict): a result dictionary of only a few items. + + Returns: + str: the table as a string. + """ + keys, values = tuple(zip(*small_dict.items())) + table = tabulate( + [values], + headers=keys, + tablefmt="pipe", + floatfmt=".3f", + stralign="center", + numalign="center", + ) + return table + + +def _log_api_usage(identifier: str): + """ + Internal function used to log the usage of different detectron2 components + inside facebook's infra. + """ + torch._C._log_api_usage_once("detectron2." + identifier) diff --git a/annotator/oneformer/detectron2/utils/memory.py b/annotator/oneformer/detectron2/utils/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..bd494780b9dbbd1571688cd270bb9b53d113c13e --- /dev/null +++ b/annotator/oneformer/detectron2/utils/memory.py @@ -0,0 +1,84 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import logging +from contextlib import contextmanager +from functools import wraps +import torch + +__all__ = ["retry_if_cuda_oom"] + + +@contextmanager +def _ignore_torch_cuda_oom(): + """ + A context which ignores CUDA OOM exception from pytorch. + """ + try: + yield + except RuntimeError as e: + # NOTE: the string may change? + if "CUDA out of memory. " in str(e): + pass + else: + raise + + +def retry_if_cuda_oom(func): + """ + Makes a function retry itself after encountering + pytorch's CUDA OOM error. + It will first retry after calling `torch.cuda.empty_cache()`. + + If that still fails, it will then retry by trying to convert inputs to CPUs. + In this case, it expects the function to dispatch to CPU implementation. + The return values may become CPU tensors as well and it's user's + responsibility to convert it back to CUDA tensor if needed. + + Args: + func: a stateless callable that takes tensor-like objects as arguments + + Returns: + a callable which retries `func` if OOM is encountered. + + Examples: + :: + output = retry_if_cuda_oom(some_torch_function)(input1, input2) + # output may be on CPU even if inputs are on GPU + + Note: + 1. When converting inputs to CPU, it will only look at each argument and check + if it has `.device` and `.to` for conversion. Nested structures of tensors + are not supported. + + 2. Since the function might be called more than once, it has to be + stateless. + """ + + def maybe_to_cpu(x): + try: + like_gpu_tensor = x.device.type == "cuda" and hasattr(x, "to") + except AttributeError: + like_gpu_tensor = False + if like_gpu_tensor: + return x.to(device="cpu") + else: + return x + + @wraps(func) + def wrapped(*args, **kwargs): + with _ignore_torch_cuda_oom(): + return func(*args, **kwargs) + + # Clear cache and retry + torch.cuda.empty_cache() + with _ignore_torch_cuda_oom(): + return func(*args, **kwargs) + + # Try on CPU. This slows down the code significantly, therefore print a notice. + logger = logging.getLogger(__name__) + logger.info("Attempting to copy inputs of {} to CPU due to CUDA OOM".format(str(func))) + new_args = (maybe_to_cpu(x) for x in args) + new_kwargs = {k: maybe_to_cpu(v) for k, v in kwargs.items()} + return func(*new_args, **new_kwargs) + + return wrapped diff --git a/annotator/oneformer/detectron2/utils/registry.py b/annotator/oneformer/detectron2/utils/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..4b01e9007c2578a7b5ae555c926cc06c8a3010f9 --- /dev/null +++ b/annotator/oneformer/detectron2/utils/registry.py @@ -0,0 +1,60 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +from typing import Any +import pydoc +from fvcore.common.registry import Registry # for backward compatibility. + +""" +``Registry`` and `locate` provide ways to map a string (typically found +in config files) to callable objects. +""" + +__all__ = ["Registry", "locate"] + + +def _convert_target_to_string(t: Any) -> str: + """ + Inverse of ``locate()``. + + Args: + t: any object with ``__module__`` and ``__qualname__`` + """ + module, qualname = t.__module__, t.__qualname__ + + # Compress the path to this object, e.g. ``module.submodule._impl.class`` + # may become ``module.submodule.class``, if the later also resolves to the same + # object. This simplifies the string, and also is less affected by moving the + # class implementation. + module_parts = module.split(".") + for k in range(1, len(module_parts)): + prefix = ".".join(module_parts[:k]) + candidate = f"{prefix}.{qualname}" + try: + if locate(candidate) is t: + return candidate + except ImportError: + pass + return f"{module}.{qualname}" + + +def locate(name: str) -> Any: + """ + Locate and return an object ``x`` using an input string ``{x.__module__}.{x.__qualname__}``, + such as "module.submodule.class_name". + + Raise Exception if it cannot be found. + """ + obj = pydoc.locate(name) + + # Some cases (e.g. torch.optim.sgd.SGD) not handled correctly + # by pydoc.locate. Try a private function from hydra. + if obj is None: + try: + # from hydra.utils import get_method - will print many errors + from hydra.utils import _locate + except ImportError as e: + raise ImportError(f"Cannot dynamically locate object {name}!") from e + else: + obj = _locate(name) # it raises if fails + + return obj diff --git a/annotator/oneformer/detectron2/utils/serialize.py b/annotator/oneformer/detectron2/utils/serialize.py new file mode 100644 index 0000000000000000000000000000000000000000..ed45065184f0512ef65c8f38d398de553ce576ca --- /dev/null +++ b/annotator/oneformer/detectron2/utils/serialize.py @@ -0,0 +1,32 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# import cloudpickle + + +class PicklableWrapper(object): + """ + Wrap an object to make it more picklable, note that it uses + heavy weight serialization libraries that are slower than pickle. + It's best to use it only on closures (which are usually not picklable). + + This is a simplified version of + https://github.com/joblib/joblib/blob/master/joblib/externals/loky/cloudpickle_wrapper.py + """ + + def __init__(self, obj): + while isinstance(obj, PicklableWrapper): + # Wrapping an object twice is no-op + obj = obj._obj + self._obj = obj + + # def __reduce__(self): + # s = cloudpickle.dumps(self._obj) + # return cloudpickle.loads, (s,) + + def __call__(self, *args, **kwargs): + return self._obj(*args, **kwargs) + + def __getattr__(self, attr): + # Ensure that the wrapped object can be used seamlessly as the previous object. + if attr not in ["_obj"]: + return getattr(self._obj, attr) + return getattr(self, attr) diff --git a/annotator/oneformer/detectron2/utils/testing.py b/annotator/oneformer/detectron2/utils/testing.py new file mode 100644 index 0000000000000000000000000000000000000000..3c3f001a260c3df20f610f0336678d505fdce5aa --- /dev/null +++ b/annotator/oneformer/detectron2/utils/testing.py @@ -0,0 +1,478 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import io +import numpy as np +import os +import re +import tempfile +import unittest +from typing import Callable +import torch +import torch.onnx.symbolic_helper as sym_help +from packaging import version +from torch._C import ListType +from torch.onnx import register_custom_op_symbolic + +from annotator.oneformer.detectron2 import model_zoo +from annotator.oneformer.detectron2.config import CfgNode, LazyConfig, instantiate +from annotator.oneformer.detectron2.data import DatasetCatalog +from annotator.oneformer.detectron2.data.detection_utils import read_image +from annotator.oneformer.detectron2.modeling import build_model +from annotator.oneformer.detectron2.structures import Boxes, Instances, ROIMasks +from annotator.oneformer.detectron2.utils.file_io import PathManager + + +""" +Internal utilities for tests. Don't use except for writing tests. +""" + + +def get_model_no_weights(config_path): + """ + Like model_zoo.get, but do not load any weights (even pretrained) + """ + cfg = model_zoo.get_config(config_path) + if isinstance(cfg, CfgNode): + if not torch.cuda.is_available(): + cfg.MODEL.DEVICE = "cpu" + return build_model(cfg) + else: + return instantiate(cfg.model) + + +def random_boxes(num_boxes, max_coord=100, device="cpu"): + """ + Create a random Nx4 boxes tensor, with coordinates < max_coord. + """ + boxes = torch.rand(num_boxes, 4, device=device) * (max_coord * 0.5) + boxes.clamp_(min=1.0) # tiny boxes cause numerical instability in box regression + # Note: the implementation of this function in torchvision is: + # boxes[:, 2:] += torch.rand(N, 2) * 100 + # but it does not guarantee non-negative widths/heights constraints: + # boxes[:, 2] >= boxes[:, 0] and boxes[:, 3] >= boxes[:, 1]: + boxes[:, 2:] += boxes[:, :2] + return boxes + + +def get_sample_coco_image(tensor=True): + """ + Args: + tensor (bool): if True, returns 3xHxW tensor. + else, returns a HxWx3 numpy array. + + Returns: + an image, in BGR color. + """ + try: + file_name = DatasetCatalog.get("coco_2017_val_100")[0]["file_name"] + if not PathManager.exists(file_name): + raise FileNotFoundError() + except IOError: + # for public CI to run + file_name = PathManager.get_local_path( + "http://images.cocodataset.org/train2017/000000000009.jpg" + ) + ret = read_image(file_name, format="BGR") + if tensor: + ret = torch.from_numpy(np.ascontiguousarray(ret.transpose(2, 0, 1))) + return ret + + +def convert_scripted_instances(instances): + """ + Convert a scripted Instances object to a regular :class:`Instances` object + """ + assert hasattr( + instances, "image_size" + ), f"Expect an Instances object, but got {type(instances)}!" + ret = Instances(instances.image_size) + for name in instances._field_names: + val = getattr(instances, "_" + name, None) + if val is not None: + ret.set(name, val) + return ret + + +def assert_instances_allclose(input, other, *, rtol=1e-5, msg="", size_as_tensor=False): + """ + Args: + input, other (Instances): + size_as_tensor: compare image_size of the Instances as tensors (instead of tuples). + Useful for comparing outputs of tracing. + """ + if not isinstance(input, Instances): + input = convert_scripted_instances(input) + if not isinstance(other, Instances): + other = convert_scripted_instances(other) + + if not msg: + msg = "Two Instances are different! " + else: + msg = msg.rstrip() + " " + + size_error_msg = msg + f"image_size is {input.image_size} vs. {other.image_size}!" + if size_as_tensor: + assert torch.equal( + torch.tensor(input.image_size), torch.tensor(other.image_size) + ), size_error_msg + else: + assert input.image_size == other.image_size, size_error_msg + fields = sorted(input.get_fields().keys()) + fields_other = sorted(other.get_fields().keys()) + assert fields == fields_other, msg + f"Fields are {fields} vs {fields_other}!" + + for f in fields: + val1, val2 = input.get(f), other.get(f) + if isinstance(val1, (Boxes, ROIMasks)): + # boxes in the range of O(100) and can have a larger tolerance + assert torch.allclose(val1.tensor, val2.tensor, atol=100 * rtol), ( + msg + f"Field {f} differs too much!" + ) + elif isinstance(val1, torch.Tensor): + if val1.dtype.is_floating_point: + mag = torch.abs(val1).max().cpu().item() + assert torch.allclose(val1, val2, atol=mag * rtol), ( + msg + f"Field {f} differs too much!" + ) + else: + assert torch.equal(val1, val2), msg + f"Field {f} is different!" + else: + raise ValueError(f"Don't know how to compare type {type(val1)}") + + +def reload_script_model(module): + """ + Save a jit module and load it back. + Similar to the `getExportImportCopy` function in torch/testing/ + """ + buffer = io.BytesIO() + torch.jit.save(module, buffer) + buffer.seek(0) + return torch.jit.load(buffer) + + +def reload_lazy_config(cfg): + """ + Save an object by LazyConfig.save and load it back. + This is used to test that a config still works the same after + serialization/deserialization. + """ + with tempfile.TemporaryDirectory(prefix="detectron2") as d: + fname = os.path.join(d, "d2_cfg_test.yaml") + LazyConfig.save(cfg, fname) + return LazyConfig.load(fname) + + +def min_torch_version(min_version: str) -> bool: + """ + Returns True when torch's version is at least `min_version`. + """ + try: + import torch + except ImportError: + return False + + installed_version = version.parse(torch.__version__.split("+")[0]) + min_version = version.parse(min_version) + return installed_version >= min_version + + +def has_dynamic_axes(onnx_model): + """ + Return True when all ONNX input/output have only dynamic axes for all ranks + """ + return all( + not dim.dim_param.isnumeric() + for inp in onnx_model.graph.input + for dim in inp.type.tensor_type.shape.dim + ) and all( + not dim.dim_param.isnumeric() + for out in onnx_model.graph.output + for dim in out.type.tensor_type.shape.dim + ) + + +def register_custom_op_onnx_export( + opname: str, symbolic_fn: Callable, opset_version: int, min_version: str +) -> None: + """ + Register `symbolic_fn` as PyTorch's symbolic `opname`-`opset_version` for ONNX export. + The registration is performed only when current PyTorch's version is < `min_version.` + IMPORTANT: symbolic must be manually unregistered after the caller function returns + """ + if min_torch_version(min_version): + return + register_custom_op_symbolic(opname, symbolic_fn, opset_version) + print(f"_register_custom_op_onnx_export({opname}, {opset_version}) succeeded.") + + +def unregister_custom_op_onnx_export(opname: str, opset_version: int, min_version: str) -> None: + """ + Unregister PyTorch's symbolic `opname`-`opset_version` for ONNX export. + The un-registration is performed only when PyTorch's version is < `min_version` + IMPORTANT: The symbolic must have been manually registered by the caller, otherwise + the incorrect symbolic may be unregistered instead. + """ + + # TODO: _unregister_custom_op_symbolic is introduced PyTorch>=1.10 + # Remove after PyTorch 1.10+ is used by ALL detectron2's CI + try: + from torch.onnx import unregister_custom_op_symbolic as _unregister_custom_op_symbolic + except ImportError: + + def _unregister_custom_op_symbolic(symbolic_name, opset_version): + import torch.onnx.symbolic_registry as sym_registry + from torch.onnx.symbolic_helper import _onnx_main_opset, _onnx_stable_opsets + + def _get_ns_op_name_from_custom_op(symbolic_name): + try: + from torch.onnx.utils import get_ns_op_name_from_custom_op + + ns, op_name = get_ns_op_name_from_custom_op(symbolic_name) + except ImportError as import_error: + if not bool( + re.match(r"^[a-zA-Z0-9-_]*::[a-zA-Z-_]+[a-zA-Z0-9-_]*$", symbolic_name) + ): + raise ValueError( + f"Invalid symbolic name {symbolic_name}. Must be `domain::name`" + ) from import_error + + ns, op_name = symbolic_name.split("::") + if ns == "onnx": + raise ValueError(f"{ns} domain cannot be modified.") from import_error + + if ns == "aten": + ns = "" + + return ns, op_name + + def _unregister_op(opname: str, domain: str, version: int): + try: + sym_registry.unregister_op(op_name, ns, ver) + except AttributeError as attribute_error: + if sym_registry.is_registered_op(opname, domain, version): + del sym_registry._registry[(domain, version)][opname] + if not sym_registry._registry[(domain, version)]: + del sym_registry._registry[(domain, version)] + else: + raise RuntimeError( + f"The opname {opname} is not registered." + ) from attribute_error + + ns, op_name = _get_ns_op_name_from_custom_op(symbolic_name) + for ver in _onnx_stable_opsets + [_onnx_main_opset]: + if ver >= opset_version: + _unregister_op(op_name, ns, ver) + + if min_torch_version(min_version): + return + _unregister_custom_op_symbolic(opname, opset_version) + print(f"_unregister_custom_op_onnx_export({opname}, {opset_version}) succeeded.") + + +skipIfOnCPUCI = unittest.skipIf( + os.environ.get("CI") and not torch.cuda.is_available(), + "The test is too slow on CPUs and will be executed on CircleCI's GPU jobs.", +) + + +def skipIfUnsupportedMinOpsetVersion(min_opset_version, current_opset_version=None): + """ + Skips tests for ONNX Opset versions older than min_opset_version. + """ + + def skip_dec(func): + def wrapper(self): + try: + opset_version = self.opset_version + except AttributeError: + opset_version = current_opset_version + if opset_version < min_opset_version: + raise unittest.SkipTest( + f"Unsupported opset_version {opset_version}" + f", required is {min_opset_version}" + ) + return func(self) + + return wrapper + + return skip_dec + + +def skipIfUnsupportedMinTorchVersion(min_version): + """ + Skips tests for PyTorch versions older than min_version. + """ + reason = f"module 'torch' has __version__ {torch.__version__}" f", required is: {min_version}" + return unittest.skipIf(not min_torch_version(min_version), reason) + + +# TODO: Remove after PyTorch 1.11.1+ is used by detectron2's CI +def _pytorch1111_symbolic_opset9_to(g, self, *args): + """aten::to() symbolic that must be used for testing with PyTorch < 1.11.1.""" + + def is_aten_to_device_only(args): + if len(args) == 4: + # aten::to(Tensor, Device, bool, bool, memory_format) + return ( + args[0].node().kind() == "prim::device" + or args[0].type().isSubtypeOf(ListType.ofInts()) + or ( + sym_help._is_value(args[0]) + and args[0].node().kind() == "onnx::Constant" + and isinstance(args[0].node()["value"], str) + ) + ) + elif len(args) == 5: + # aten::to(Tensor, Device, ScalarType, bool, bool, memory_format) + # When dtype is None, this is a aten::to(device) call + dtype = sym_help._get_const(args[1], "i", "dtype") + return dtype is None + elif len(args) in (6, 7): + # aten::to(Tensor, ScalarType, Layout, Device, bool, bool, memory_format) + # aten::to(Tensor, ScalarType, Layout, Device, bool, bool, bool, memory_format) + # When dtype is None, this is a aten::to(device) call + dtype = sym_help._get_const(args[0], "i", "dtype") + return dtype is None + return False + + # ONNX doesn't have a concept of a device, so we ignore device-only casts + if is_aten_to_device_only(args): + return self + + if len(args) == 4: + # TestONNXRuntime::test_ones_bool shows args[0] of aten::to can be onnx::Constant[Tensor] + # In this case, the constant value is a tensor not int, + # so sym_help._maybe_get_const(args[0], 'i') would not work. + dtype = args[0] + if sym_help._is_value(args[0]) and args[0].node().kind() == "onnx::Constant": + tval = args[0].node()["value"] + if isinstance(tval, torch.Tensor): + if len(tval.shape) == 0: + tval = tval.item() + dtype = int(tval) + else: + dtype = tval + + if sym_help._is_value(dtype) or isinstance(dtype, torch.Tensor): + # aten::to(Tensor, Tensor, bool, bool, memory_format) + dtype = args[0].type().scalarType() + return g.op("Cast", self, to_i=sym_help.cast_pytorch_to_onnx[dtype]) + else: + # aten::to(Tensor, ScalarType, bool, bool, memory_format) + # memory_format is ignored + return g.op("Cast", self, to_i=sym_help.scalar_type_to_onnx[dtype]) + elif len(args) == 5: + # aten::to(Tensor, Device, ScalarType, bool, bool, memory_format) + dtype = sym_help._get_const(args[1], "i", "dtype") + # memory_format is ignored + return g.op("Cast", self, to_i=sym_help.scalar_type_to_onnx[dtype]) + elif len(args) == 6: + # aten::to(Tensor, ScalarType, Layout, Device, bool, bool, memory_format) + dtype = sym_help._get_const(args[0], "i", "dtype") + # Layout, device and memory_format are ignored + return g.op("Cast", self, to_i=sym_help.scalar_type_to_onnx[dtype]) + elif len(args) == 7: + # aten::to(Tensor, ScalarType, Layout, Device, bool, bool, bool, memory_format) + dtype = sym_help._get_const(args[0], "i", "dtype") + # Layout, device and memory_format are ignored + return g.op("Cast", self, to_i=sym_help.scalar_type_to_onnx[dtype]) + else: + return sym_help._onnx_unsupported("Unknown aten::to signature") + + +# TODO: Remove after PyTorch 1.11.1+ is used by detectron2's CI +def _pytorch1111_symbolic_opset9_repeat_interleave(g, self, repeats, dim=None, output_size=None): + + # from torch.onnx.symbolic_helper import ScalarType + from torch.onnx.symbolic_opset9 import expand, unsqueeze + + input = self + # if dim is None flatten + # By default, use the flattened input array, and return a flat output array + if sym_help._is_none(dim): + input = sym_help._reshape_helper(g, self, g.op("Constant", value_t=torch.tensor([-1]))) + dim = 0 + else: + dim = sym_help._maybe_get_scalar(dim) + + repeats_dim = sym_help._get_tensor_rank(repeats) + repeats_sizes = sym_help._get_tensor_sizes(repeats) + input_sizes = sym_help._get_tensor_sizes(input) + if repeats_dim is None: + raise RuntimeError( + "Unsupported: ONNX export of repeat_interleave for unknown " "repeats rank." + ) + if repeats_sizes is None: + raise RuntimeError( + "Unsupported: ONNX export of repeat_interleave for unknown " "repeats size." + ) + if input_sizes is None: + raise RuntimeError( + "Unsupported: ONNX export of repeat_interleave for unknown " "input size." + ) + + input_sizes_temp = input_sizes.copy() + for idx, input_size in enumerate(input_sizes): + if input_size is None: + input_sizes[idx], input_sizes_temp[idx] = 0, -1 + + # Cases where repeats is an int or single value tensor + if repeats_dim == 0 or (repeats_dim == 1 and repeats_sizes[0] == 1): + if not sym_help._is_tensor(repeats): + repeats = g.op("Constant", value_t=torch.LongTensor(repeats)) + if input_sizes[dim] == 0: + return sym_help._onnx_opset_unsupported_detailed( + "repeat_interleave", + 9, + 13, + "Unsupported along dimension with unknown input size", + ) + else: + reps = input_sizes[dim] + repeats = expand(g, repeats, g.op("Constant", value_t=torch.tensor([reps])), None) + + # Cases where repeats is a 1 dim Tensor + elif repeats_dim == 1: + if input_sizes[dim] == 0: + return sym_help._onnx_opset_unsupported_detailed( + "repeat_interleave", + 9, + 13, + "Unsupported along dimension with unknown input size", + ) + if repeats_sizes[0] is None: + return sym_help._onnx_opset_unsupported_detailed( + "repeat_interleave", 9, 13, "Unsupported for cases with dynamic repeats" + ) + assert ( + repeats_sizes[0] == input_sizes[dim] + ), "repeats must have the same size as input along dim" + reps = repeats_sizes[0] + else: + raise RuntimeError("repeats must be 0-dim or 1-dim tensor") + + final_splits = list() + r_splits = sym_help._repeat_interleave_split_helper(g, repeats, reps, 0) + if isinstance(r_splits, torch._C.Value): + r_splits = [r_splits] + i_splits = sym_help._repeat_interleave_split_helper(g, input, reps, dim) + if isinstance(i_splits, torch._C.Value): + i_splits = [i_splits] + input_sizes[dim], input_sizes_temp[dim] = -1, 1 + for idx, r_split in enumerate(r_splits): + i_split = unsqueeze(g, i_splits[idx], dim + 1) + r_concat = [ + g.op("Constant", value_t=torch.LongTensor(input_sizes_temp[: dim + 1])), + r_split, + g.op("Constant", value_t=torch.LongTensor(input_sizes_temp[dim + 1 :])), + ] + r_concat = g.op("Concat", *r_concat, axis_i=0) + i_split = expand(g, i_split, r_concat, None) + i_split = sym_help._reshape_helper( + g, + i_split, + g.op("Constant", value_t=torch.LongTensor(input_sizes)), + allowzero=0, + ) + final_splits.append(i_split) + return g.op("Concat", *final_splits, axis_i=dim) diff --git a/annotator/oneformer/detectron2/utils/tracing.py b/annotator/oneformer/detectron2/utils/tracing.py new file mode 100644 index 0000000000000000000000000000000000000000..75661131505cee2eecd0b1c9dabcd4d7bd5453b2 --- /dev/null +++ b/annotator/oneformer/detectron2/utils/tracing.py @@ -0,0 +1,71 @@ +import inspect +import torch + +from annotator.oneformer.detectron2.utils.env import TORCH_VERSION + +try: + from torch.fx._symbolic_trace import is_fx_tracing as is_fx_tracing_current + + tracing_current_exists = True +except ImportError: + tracing_current_exists = False + +try: + from torch.fx._symbolic_trace import _orig_module_call + + tracing_legacy_exists = True +except ImportError: + tracing_legacy_exists = False + + +@torch.jit.ignore +def is_fx_tracing_legacy() -> bool: + """ + Returns a bool indicating whether torch.fx is currently symbolically tracing a module. + Can be useful for gating module logic that is incompatible with symbolic tracing. + """ + return torch.nn.Module.__call__ is not _orig_module_call + + +@torch.jit.ignore +def is_fx_tracing() -> bool: + """Returns whether execution is currently in + Torch FX tracing mode""" + if TORCH_VERSION >= (1, 10) and tracing_current_exists: + return is_fx_tracing_current() + elif tracing_legacy_exists: + return is_fx_tracing_legacy() + else: + # Can't find either current or legacy tracing indication code. + # Enabling this assert_fx_safe() call regardless of tracing status. + return False + + +@torch.jit.ignore +def assert_fx_safe(condition: bool, message: str) -> torch.Tensor: + """An FX-tracing safe version of assert. + Avoids erroneous type assertion triggering when types are masked inside + an fx.proxy.Proxy object during tracing. + Args: condition - either a boolean expression or a string representing + the condition to test. If this assert triggers an exception when tracing + due to dynamic control flow, try encasing the expression in quotation + marks and supplying it as a string.""" + # Must return a concrete tensor for compatibility with PyTorch <=1.8. + # If <=1.8 compatibility is not needed, return type can be converted to None + if not is_fx_tracing(): + try: + if isinstance(condition, str): + caller_frame = inspect.currentframe().f_back + torch._assert( + eval(condition, caller_frame.f_globals, caller_frame.f_locals), message + ) + return torch.ones(1) + else: + torch._assert(condition, message) + return torch.ones(1) + except torch.fx.proxy.TraceError as e: + print( + "Found a non-FX compatible assertion. Skipping the check. Failure is shown below" + + str(e) + ) + return torch.zeros(1) diff --git a/annotator/oneformer/detectron2/utils/video_visualizer.py b/annotator/oneformer/detectron2/utils/video_visualizer.py new file mode 100644 index 0000000000000000000000000000000000000000..591c3ad3d551c421e923378fbc48fb44facc7257 --- /dev/null +++ b/annotator/oneformer/detectron2/utils/video_visualizer.py @@ -0,0 +1,287 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import numpy as np +from typing import List +import pycocotools.mask as mask_util + +from annotator.oneformer.detectron2.structures import Instances +from annotator.oneformer.detectron2.utils.visualizer import ( + ColorMode, + Visualizer, + _create_text_labels, + _PanopticPrediction, +) + +from .colormap import random_color, random_colors + + +class _DetectedInstance: + """ + Used to store data about detected objects in video frame, + in order to transfer color to objects in the future frames. + + Attributes: + label (int): + bbox (tuple[float]): + mask_rle (dict): + color (tuple[float]): RGB colors in range (0, 1) + ttl (int): time-to-live for the instance. For example, if ttl=2, + the instance color can be transferred to objects in the next two frames. + """ + + __slots__ = ["label", "bbox", "mask_rle", "color", "ttl"] + + def __init__(self, label, bbox, mask_rle, color, ttl): + self.label = label + self.bbox = bbox + self.mask_rle = mask_rle + self.color = color + self.ttl = ttl + + +class VideoVisualizer: + def __init__(self, metadata, instance_mode=ColorMode.IMAGE): + """ + Args: + metadata (MetadataCatalog): image metadata. + """ + self.metadata = metadata + self._old_instances = [] + assert instance_mode in [ + ColorMode.IMAGE, + ColorMode.IMAGE_BW, + ], "Other mode not supported yet." + self._instance_mode = instance_mode + self._max_num_instances = self.metadata.get("max_num_instances", 74) + self._assigned_colors = {} + self._color_pool = random_colors(self._max_num_instances, rgb=True, maximum=1) + self._color_idx_set = set(range(len(self._color_pool))) + + def draw_instance_predictions(self, frame, predictions): + """ + Draw instance-level prediction results on an image. + + Args: + frame (ndarray): an RGB image of shape (H, W, C), in the range [0, 255]. + predictions (Instances): the output of an instance detection/segmentation + model. Following fields will be used to draw: + "pred_boxes", "pred_classes", "scores", "pred_masks" (or "pred_masks_rle"). + + Returns: + output (VisImage): image object with visualizations. + """ + frame_visualizer = Visualizer(frame, self.metadata) + num_instances = len(predictions) + if num_instances == 0: + return frame_visualizer.output + + boxes = predictions.pred_boxes.tensor.numpy() if predictions.has("pred_boxes") else None + scores = predictions.scores if predictions.has("scores") else None + classes = predictions.pred_classes.numpy() if predictions.has("pred_classes") else None + keypoints = predictions.pred_keypoints if predictions.has("pred_keypoints") else None + colors = predictions.COLOR if predictions.has("COLOR") else [None] * len(predictions) + periods = predictions.ID_period if predictions.has("ID_period") else None + period_threshold = self.metadata.get("period_threshold", 0) + visibilities = ( + [True] * len(predictions) + if periods is None + else [x > period_threshold for x in periods] + ) + + if predictions.has("pred_masks"): + masks = predictions.pred_masks + # mask IOU is not yet enabled + # masks_rles = mask_util.encode(np.asarray(masks.permute(1, 2, 0), order="F")) + # assert len(masks_rles) == num_instances + else: + masks = None + + if not predictions.has("COLOR"): + if predictions.has("ID"): + colors = self._assign_colors_by_id(predictions) + else: + # ToDo: clean old assign color method and use a default tracker to assign id + detected = [ + _DetectedInstance(classes[i], boxes[i], mask_rle=None, color=colors[i], ttl=8) + for i in range(num_instances) + ] + colors = self._assign_colors(detected) + + labels = _create_text_labels(classes, scores, self.metadata.get("thing_classes", None)) + + if self._instance_mode == ColorMode.IMAGE_BW: + # any() returns uint8 tensor + frame_visualizer.output.reset_image( + frame_visualizer._create_grayscale_image( + (masks.any(dim=0) > 0).numpy() if masks is not None else None + ) + ) + alpha = 0.3 + else: + alpha = 0.5 + + labels = ( + None + if labels is None + else [y[0] for y in filter(lambda x: x[1], zip(labels, visibilities))] + ) # noqa + assigned_colors = ( + None + if colors is None + else [y[0] for y in filter(lambda x: x[1], zip(colors, visibilities))] + ) # noqa + frame_visualizer.overlay_instances( + boxes=None if masks is not None else boxes[visibilities], # boxes are a bit distracting + masks=None if masks is None else masks[visibilities], + labels=labels, + keypoints=None if keypoints is None else keypoints[visibilities], + assigned_colors=assigned_colors, + alpha=alpha, + ) + + return frame_visualizer.output + + def draw_sem_seg(self, frame, sem_seg, area_threshold=None): + """ + Args: + sem_seg (ndarray or Tensor): semantic segmentation of shape (H, W), + each value is the integer label. + area_threshold (Optional[int]): only draw segmentations larger than the threshold + """ + # don't need to do anything special + frame_visualizer = Visualizer(frame, self.metadata) + frame_visualizer.draw_sem_seg(sem_seg, area_threshold=None) + return frame_visualizer.output + + def draw_panoptic_seg_predictions( + self, frame, panoptic_seg, segments_info, area_threshold=None, alpha=0.5 + ): + frame_visualizer = Visualizer(frame, self.metadata) + pred = _PanopticPrediction(panoptic_seg, segments_info, self.metadata) + + if self._instance_mode == ColorMode.IMAGE_BW: + frame_visualizer.output.reset_image( + frame_visualizer._create_grayscale_image(pred.non_empty_mask()) + ) + + # draw mask for all semantic segments first i.e. "stuff" + for mask, sinfo in pred.semantic_masks(): + category_idx = sinfo["category_id"] + try: + mask_color = [x / 255 for x in self.metadata.stuff_colors[category_idx]] + except AttributeError: + mask_color = None + + frame_visualizer.draw_binary_mask( + mask, + color=mask_color, + text=self.metadata.stuff_classes[category_idx], + alpha=alpha, + area_threshold=area_threshold, + ) + + all_instances = list(pred.instance_masks()) + if len(all_instances) == 0: + return frame_visualizer.output + # draw mask for all instances second + masks, sinfo = list(zip(*all_instances)) + num_instances = len(masks) + masks_rles = mask_util.encode( + np.asarray(np.asarray(masks).transpose(1, 2, 0), dtype=np.uint8, order="F") + ) + assert len(masks_rles) == num_instances + + category_ids = [x["category_id"] for x in sinfo] + detected = [ + _DetectedInstance(category_ids[i], bbox=None, mask_rle=masks_rles[i], color=None, ttl=8) + for i in range(num_instances) + ] + colors = self._assign_colors(detected) + labels = [self.metadata.thing_classes[k] for k in category_ids] + + frame_visualizer.overlay_instances( + boxes=None, + masks=masks, + labels=labels, + keypoints=None, + assigned_colors=colors, + alpha=alpha, + ) + return frame_visualizer.output + + def _assign_colors(self, instances): + """ + Naive tracking heuristics to assign same color to the same instance, + will update the internal state of tracked instances. + + Returns: + list[tuple[float]]: list of colors. + """ + + # Compute iou with either boxes or masks: + is_crowd = np.zeros((len(instances),), dtype=bool) + if instances[0].bbox is None: + assert instances[0].mask_rle is not None + # use mask iou only when box iou is None + # because box seems good enough + rles_old = [x.mask_rle for x in self._old_instances] + rles_new = [x.mask_rle for x in instances] + ious = mask_util.iou(rles_old, rles_new, is_crowd) + threshold = 0.5 + else: + boxes_old = [x.bbox for x in self._old_instances] + boxes_new = [x.bbox for x in instances] + ious = mask_util.iou(boxes_old, boxes_new, is_crowd) + threshold = 0.6 + if len(ious) == 0: + ious = np.zeros((len(self._old_instances), len(instances)), dtype="float32") + + # Only allow matching instances of the same label: + for old_idx, old in enumerate(self._old_instances): + for new_idx, new in enumerate(instances): + if old.label != new.label: + ious[old_idx, new_idx] = 0 + + matched_new_per_old = np.asarray(ious).argmax(axis=1) + max_iou_per_old = np.asarray(ious).max(axis=1) + + # Try to find match for each old instance: + extra_instances = [] + for idx, inst in enumerate(self._old_instances): + if max_iou_per_old[idx] > threshold: + newidx = matched_new_per_old[idx] + if instances[newidx].color is None: + instances[newidx].color = inst.color + continue + # If an old instance does not match any new instances, + # keep it for the next frame in case it is just missed by the detector + inst.ttl -= 1 + if inst.ttl > 0: + extra_instances.append(inst) + + # Assign random color to newly-detected instances: + for inst in instances: + if inst.color is None: + inst.color = random_color(rgb=True, maximum=1) + self._old_instances = instances[:] + extra_instances + return [d.color for d in instances] + + def _assign_colors_by_id(self, instances: Instances) -> List: + colors = [] + untracked_ids = set(self._assigned_colors.keys()) + for id in instances.ID: + if id in self._assigned_colors: + colors.append(self._color_pool[self._assigned_colors[id]]) + untracked_ids.remove(id) + else: + assert ( + len(self._color_idx_set) >= 1 + ), f"Number of id exceeded maximum, \ + max = {self._max_num_instances}" + idx = self._color_idx_set.pop() + color = self._color_pool[idx] + self._assigned_colors[id] = idx + colors.append(color) + for id in untracked_ids: + self._color_idx_set.add(self._assigned_colors[id]) + del self._assigned_colors[id] + return colors diff --git a/annotator/oneformer/detectron2/utils/visualizer.py b/annotator/oneformer/detectron2/utils/visualizer.py new file mode 100644 index 0000000000000000000000000000000000000000..9b1619f2ec7ca86da02b32d15e6c6db86b1f688f --- /dev/null +++ b/annotator/oneformer/detectron2/utils/visualizer.py @@ -0,0 +1,1267 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import colorsys +import logging +import math +import numpy as np +from enum import Enum, unique +import cv2 +import matplotlib as mpl +import matplotlib.colors as mplc +import matplotlib.figure as mplfigure +import pycocotools.mask as mask_util +import torch +from matplotlib.backends.backend_agg import FigureCanvasAgg +from PIL import Image + +from annotator.oneformer.detectron2.data import MetadataCatalog +from annotator.oneformer.detectron2.structures import BitMasks, Boxes, BoxMode, Keypoints, PolygonMasks, RotatedBoxes +from annotator.oneformer.detectron2.utils.file_io import PathManager + +from .colormap import random_color + +logger = logging.getLogger(__name__) + +__all__ = ["ColorMode", "VisImage", "Visualizer"] + + +_SMALL_OBJECT_AREA_THRESH = 1000 +_LARGE_MASK_AREA_THRESH = 120000 +_OFF_WHITE = (1.0, 1.0, 240.0 / 255) +_BLACK = (0, 0, 0) +_RED = (1.0, 0, 0) + +_KEYPOINT_THRESHOLD = 0.05 + + +@unique +class ColorMode(Enum): + """ + Enum of different color modes to use for instance visualizations. + """ + + IMAGE = 0 + """ + Picks a random color for every instance and overlay segmentations with low opacity. + """ + SEGMENTATION = 1 + """ + Let instances of the same category have similar colors + (from metadata.thing_colors), and overlay them with + high opacity. This provides more attention on the quality of segmentation. + """ + IMAGE_BW = 2 + """ + Same as IMAGE, but convert all areas without masks to gray-scale. + Only available for drawing per-instance mask predictions. + """ + + +class GenericMask: + """ + Attribute: + polygons (list[ndarray]): list[ndarray]: polygons for this mask. + Each ndarray has format [x, y, x, y, ...] + mask (ndarray): a binary mask + """ + + def __init__(self, mask_or_polygons, height, width): + self._mask = self._polygons = self._has_holes = None + self.height = height + self.width = width + + m = mask_or_polygons + if isinstance(m, dict): + # RLEs + assert "counts" in m and "size" in m + if isinstance(m["counts"], list): # uncompressed RLEs + h, w = m["size"] + assert h == height and w == width + m = mask_util.frPyObjects(m, h, w) + self._mask = mask_util.decode(m)[:, :] + return + + if isinstance(m, list): # list[ndarray] + self._polygons = [np.asarray(x).reshape(-1) for x in m] + return + + if isinstance(m, np.ndarray): # assumed to be a binary mask + assert m.shape[1] != 2, m.shape + assert m.shape == ( + height, + width, + ), f"mask shape: {m.shape}, target dims: {height}, {width}" + self._mask = m.astype("uint8") + return + + raise ValueError("GenericMask cannot handle object {} of type '{}'".format(m, type(m))) + + @property + def mask(self): + if self._mask is None: + self._mask = self.polygons_to_mask(self._polygons) + return self._mask + + @property + def polygons(self): + if self._polygons is None: + self._polygons, self._has_holes = self.mask_to_polygons(self._mask) + return self._polygons + + @property + def has_holes(self): + if self._has_holes is None: + if self._mask is not None: + self._polygons, self._has_holes = self.mask_to_polygons(self._mask) + else: + self._has_holes = False # if original format is polygon, does not have holes + return self._has_holes + + def mask_to_polygons(self, mask): + # cv2.RETR_CCOMP flag retrieves all the contours and arranges them to a 2-level + # hierarchy. External contours (boundary) of the object are placed in hierarchy-1. + # Internal contours (holes) are placed in hierarchy-2. + # cv2.CHAIN_APPROX_NONE flag gets vertices of polygons from contours. + mask = np.ascontiguousarray(mask) # some versions of cv2 does not support incontiguous arr + res = cv2.findContours(mask.astype("uint8"), cv2.RETR_CCOMP, cv2.CHAIN_APPROX_NONE) + hierarchy = res[-1] + if hierarchy is None: # empty mask + return [], False + has_holes = (hierarchy.reshape(-1, 4)[:, 3] >= 0).sum() > 0 + res = res[-2] + res = [x.flatten() for x in res] + # These coordinates from OpenCV are integers in range [0, W-1 or H-1]. + # We add 0.5 to turn them into real-value coordinate space. A better solution + # would be to first +0.5 and then dilate the returned polygon by 0.5. + res = [x + 0.5 for x in res if len(x) >= 6] + return res, has_holes + + def polygons_to_mask(self, polygons): + rle = mask_util.frPyObjects(polygons, self.height, self.width) + rle = mask_util.merge(rle) + return mask_util.decode(rle)[:, :] + + def area(self): + return self.mask.sum() + + def bbox(self): + p = mask_util.frPyObjects(self.polygons, self.height, self.width) + p = mask_util.merge(p) + bbox = mask_util.toBbox(p) + bbox[2] += bbox[0] + bbox[3] += bbox[1] + return bbox + + +class _PanopticPrediction: + """ + Unify different panoptic annotation/prediction formats + """ + + def __init__(self, panoptic_seg, segments_info, metadata=None): + if segments_info is None: + assert metadata is not None + # If "segments_info" is None, we assume "panoptic_img" is a + # H*W int32 image storing the panoptic_id in the format of + # category_id * label_divisor + instance_id. We reserve -1 for + # VOID label. + label_divisor = metadata.label_divisor + segments_info = [] + for panoptic_label in np.unique(panoptic_seg.numpy()): + if panoptic_label == -1: + # VOID region. + continue + pred_class = panoptic_label // label_divisor + isthing = pred_class in metadata.thing_dataset_id_to_contiguous_id.values() + segments_info.append( + { + "id": int(panoptic_label), + "category_id": int(pred_class), + "isthing": bool(isthing), + } + ) + del metadata + + self._seg = panoptic_seg + + self._sinfo = {s["id"]: s for s in segments_info} # seg id -> seg info + segment_ids, areas = torch.unique(panoptic_seg, sorted=True, return_counts=True) + areas = areas.numpy() + sorted_idxs = np.argsort(-areas) + self._seg_ids, self._seg_areas = segment_ids[sorted_idxs], areas[sorted_idxs] + self._seg_ids = self._seg_ids.tolist() + for sid, area in zip(self._seg_ids, self._seg_areas): + if sid in self._sinfo: + self._sinfo[sid]["area"] = float(area) + + def non_empty_mask(self): + """ + Returns: + (H, W) array, a mask for all pixels that have a prediction + """ + empty_ids = [] + for id in self._seg_ids: + if id not in self._sinfo: + empty_ids.append(id) + if len(empty_ids) == 0: + return np.zeros(self._seg.shape, dtype=np.uint8) + assert ( + len(empty_ids) == 1 + ), ">1 ids corresponds to no labels. This is currently not supported" + return (self._seg != empty_ids[0]).numpy().astype(bool) + + def semantic_masks(self): + for sid in self._seg_ids: + sinfo = self._sinfo.get(sid) + if sinfo is None or sinfo["isthing"]: + # Some pixels (e.g. id 0 in PanopticFPN) have no instance or semantic predictions. + continue + yield (self._seg == sid).numpy().astype(bool), sinfo + + def instance_masks(self): + for sid in self._seg_ids: + sinfo = self._sinfo.get(sid) + if sinfo is None or not sinfo["isthing"]: + continue + mask = (self._seg == sid).numpy().astype(bool) + if mask.sum() > 0: + yield mask, sinfo + + +def _create_text_labels(classes, scores, class_names, is_crowd=None): + """ + Args: + classes (list[int] or None): + scores (list[float] or None): + class_names (list[str] or None): + is_crowd (list[bool] or None): + + Returns: + list[str] or None + """ + labels = None + if classes is not None: + if class_names is not None and len(class_names) > 0: + labels = [class_names[i] for i in classes] + else: + labels = [str(i) for i in classes] + if scores is not None: + if labels is None: + labels = ["{:.0f}%".format(s * 100) for s in scores] + else: + labels = ["{} {:.0f}%".format(l, s * 100) for l, s in zip(labels, scores)] + if labels is not None and is_crowd is not None: + labels = [l + ("|crowd" if crowd else "") for l, crowd in zip(labels, is_crowd)] + return labels + + +class VisImage: + def __init__(self, img, scale=1.0): + """ + Args: + img (ndarray): an RGB image of shape (H, W, 3) in range [0, 255]. + scale (float): scale the input image + """ + self.img = img + self.scale = scale + self.width, self.height = img.shape[1], img.shape[0] + self._setup_figure(img) + + def _setup_figure(self, img): + """ + Args: + Same as in :meth:`__init__()`. + + Returns: + fig (matplotlib.pyplot.figure): top level container for all the image plot elements. + ax (matplotlib.pyplot.Axes): contains figure elements and sets the coordinate system. + """ + fig = mplfigure.Figure(frameon=False) + self.dpi = fig.get_dpi() + # add a small 1e-2 to avoid precision lost due to matplotlib's truncation + # (https://github.com/matplotlib/matplotlib/issues/15363) + fig.set_size_inches( + (self.width * self.scale + 1e-2) / self.dpi, + (self.height * self.scale + 1e-2) / self.dpi, + ) + self.canvas = FigureCanvasAgg(fig) + # self.canvas = mpl.backends.backend_cairo.FigureCanvasCairo(fig) + ax = fig.add_axes([0.0, 0.0, 1.0, 1.0]) + ax.axis("off") + self.fig = fig + self.ax = ax + self.reset_image(img) + + def reset_image(self, img): + """ + Args: + img: same as in __init__ + """ + img = img.astype("uint8") + self.ax.imshow(img, extent=(0, self.width, self.height, 0), interpolation="nearest") + + def save(self, filepath): + """ + Args: + filepath (str): a string that contains the absolute path, including the file name, where + the visualized image will be saved. + """ + self.fig.savefig(filepath) + + def get_image(self): + """ + Returns: + ndarray: + the visualized image of shape (H, W, 3) (RGB) in uint8 type. + The shape is scaled w.r.t the input image using the given `scale` argument. + """ + canvas = self.canvas + s, (width, height) = canvas.print_to_buffer() + # buf = io.BytesIO() # works for cairo backend + # canvas.print_rgba(buf) + # width, height = self.width, self.height + # s = buf.getvalue() + + buffer = np.frombuffer(s, dtype="uint8") + + img_rgba = buffer.reshape(height, width, 4) + rgb, alpha = np.split(img_rgba, [3], axis=2) + return rgb.astype("uint8") + + +class Visualizer: + """ + Visualizer that draws data about detection/segmentation on images. + + It contains methods like `draw_{text,box,circle,line,binary_mask,polygon}` + that draw primitive objects to images, as well as high-level wrappers like + `draw_{instance_predictions,sem_seg,panoptic_seg_predictions,dataset_dict}` + that draw composite data in some pre-defined style. + + Note that the exact visualization style for the high-level wrappers are subject to change. + Style such as color, opacity, label contents, visibility of labels, or even the visibility + of objects themselves (e.g. when the object is too small) may change according + to different heuristics, as long as the results still look visually reasonable. + + To obtain a consistent style, you can implement custom drawing functions with the + abovementioned primitive methods instead. If you need more customized visualization + styles, you can process the data yourself following their format documented in + tutorials (:doc:`/tutorials/models`, :doc:`/tutorials/datasets`). This class does not + intend to satisfy everyone's preference on drawing styles. + + This visualizer focuses on high rendering quality rather than performance. It is not + designed to be used for real-time applications. + """ + + # TODO implement a fast, rasterized version using OpenCV + + def __init__(self, img_rgb, metadata=None, scale=1.0, instance_mode=ColorMode.IMAGE): + """ + Args: + img_rgb: a numpy array of shape (H, W, C), where H and W correspond to + the height and width of the image respectively. C is the number of + color channels. The image is required to be in RGB format since that + is a requirement of the Matplotlib library. The image is also expected + to be in the range [0, 255]. + metadata (Metadata): dataset metadata (e.g. class names and colors) + instance_mode (ColorMode): defines one of the pre-defined style for drawing + instances on an image. + """ + self.img = np.asarray(img_rgb).clip(0, 255).astype(np.uint8) + if metadata is None: + metadata = MetadataCatalog.get("__nonexist__") + self.metadata = metadata + self.output = VisImage(self.img, scale=scale) + self.cpu_device = torch.device("cpu") + + # too small texts are useless, therefore clamp to 9 + self._default_font_size = max( + np.sqrt(self.output.height * self.output.width) // 90, 10 // scale + ) + self._instance_mode = instance_mode + self.keypoint_threshold = _KEYPOINT_THRESHOLD + + def draw_instance_predictions(self, predictions): + """ + Draw instance-level prediction results on an image. + + Args: + predictions (Instances): the output of an instance detection/segmentation + model. Following fields will be used to draw: + "pred_boxes", "pred_classes", "scores", "pred_masks" (or "pred_masks_rle"). + + Returns: + output (VisImage): image object with visualizations. + """ + boxes = predictions.pred_boxes if predictions.has("pred_boxes") else None + scores = predictions.scores if predictions.has("scores") else None + classes = predictions.pred_classes.tolist() if predictions.has("pred_classes") else None + labels = _create_text_labels(classes, scores, self.metadata.get("thing_classes", None)) + keypoints = predictions.pred_keypoints if predictions.has("pred_keypoints") else None + + if predictions.has("pred_masks"): + masks = np.asarray(predictions.pred_masks) + masks = [GenericMask(x, self.output.height, self.output.width) for x in masks] + else: + masks = None + + if self._instance_mode == ColorMode.SEGMENTATION and self.metadata.get("thing_colors"): + colors = [ + self._jitter([x / 255 for x in self.metadata.thing_colors[c]]) for c in classes + ] + alpha = 0.8 + else: + colors = None + alpha = 0.5 + + if self._instance_mode == ColorMode.IMAGE_BW: + self.output.reset_image( + self._create_grayscale_image( + (predictions.pred_masks.any(dim=0) > 0).numpy() + if predictions.has("pred_masks") + else None + ) + ) + alpha = 0.3 + + self.overlay_instances( + masks=masks, + boxes=boxes, + labels=labels, + keypoints=keypoints, + assigned_colors=colors, + alpha=alpha, + ) + return self.output + + def draw_sem_seg(self, sem_seg, area_threshold=None, alpha=0.8): + """ + Draw semantic segmentation predictions/labels. + + Args: + sem_seg (Tensor or ndarray): the segmentation of shape (H, W). + Each value is the integer label of the pixel. + area_threshold (int): segments with less than `area_threshold` are not drawn. + alpha (float): the larger it is, the more opaque the segmentations are. + + Returns: + output (VisImage): image object with visualizations. + """ + if isinstance(sem_seg, torch.Tensor): + sem_seg = sem_seg.numpy() + labels, areas = np.unique(sem_seg, return_counts=True) + sorted_idxs = np.argsort(-areas).tolist() + labels = labels[sorted_idxs] + for label in filter(lambda l: l < len(self.metadata.stuff_classes), labels): + try: + mask_color = [x / 255 for x in self.metadata.stuff_colors[label]] + except (AttributeError, IndexError): + mask_color = None + + binary_mask = (sem_seg == label).astype(np.uint8) + text = self.metadata.stuff_classes[label] + self.draw_binary_mask( + binary_mask, + color=mask_color, + edge_color=_OFF_WHITE, + text=text, + alpha=alpha, + area_threshold=area_threshold, + ) + return self.output + + def draw_panoptic_seg(self, panoptic_seg, segments_info, area_threshold=None, alpha=0.7): + """ + Draw panoptic prediction annotations or results. + + Args: + panoptic_seg (Tensor): of shape (height, width) where the values are ids for each + segment. + segments_info (list[dict] or None): Describe each segment in `panoptic_seg`. + If it is a ``list[dict]``, each dict contains keys "id", "category_id". + If None, category id of each pixel is computed by + ``pixel // metadata.label_divisor``. + area_threshold (int): stuff segments with less than `area_threshold` are not drawn. + + Returns: + output (VisImage): image object with visualizations. + """ + pred = _PanopticPrediction(panoptic_seg, segments_info, self.metadata) + + if self._instance_mode == ColorMode.IMAGE_BW: + self.output.reset_image(self._create_grayscale_image(pred.non_empty_mask())) + + # draw mask for all semantic segments first i.e. "stuff" + for mask, sinfo in pred.semantic_masks(): + category_idx = sinfo["category_id"] + try: + mask_color = [x / 255 for x in self.metadata.stuff_colors[category_idx]] + except AttributeError: + mask_color = None + + text = self.metadata.stuff_classes[category_idx] + self.draw_binary_mask( + mask, + color=mask_color, + edge_color=_OFF_WHITE, + text=text, + alpha=alpha, + area_threshold=area_threshold, + ) + + # draw mask for all instances second + all_instances = list(pred.instance_masks()) + if len(all_instances) == 0: + return self.output + masks, sinfo = list(zip(*all_instances)) + category_ids = [x["category_id"] for x in sinfo] + + try: + scores = [x["score"] for x in sinfo] + except KeyError: + scores = None + labels = _create_text_labels( + category_ids, scores, self.metadata.thing_classes, [x.get("iscrowd", 0) for x in sinfo] + ) + + try: + colors = [ + self._jitter([x / 255 for x in self.metadata.thing_colors[c]]) for c in category_ids + ] + except AttributeError: + colors = None + self.overlay_instances(masks=masks, labels=labels, assigned_colors=colors, alpha=alpha) + + return self.output + + draw_panoptic_seg_predictions = draw_panoptic_seg # backward compatibility + + def draw_dataset_dict(self, dic): + """ + Draw annotations/segmentations in Detectron2 Dataset format. + + Args: + dic (dict): annotation/segmentation data of one image, in Detectron2 Dataset format. + + Returns: + output (VisImage): image object with visualizations. + """ + annos = dic.get("annotations", None) + if annos: + if "segmentation" in annos[0]: + masks = [x["segmentation"] for x in annos] + else: + masks = None + if "keypoints" in annos[0]: + keypts = [x["keypoints"] for x in annos] + keypts = np.array(keypts).reshape(len(annos), -1, 3) + else: + keypts = None + + boxes = [ + BoxMode.convert(x["bbox"], x["bbox_mode"], BoxMode.XYXY_ABS) + if len(x["bbox"]) == 4 + else x["bbox"] + for x in annos + ] + + colors = None + category_ids = [x["category_id"] for x in annos] + if self._instance_mode == ColorMode.SEGMENTATION and self.metadata.get("thing_colors"): + colors = [ + self._jitter([x / 255 for x in self.metadata.thing_colors[c]]) + for c in category_ids + ] + names = self.metadata.get("thing_classes", None) + labels = _create_text_labels( + category_ids, + scores=None, + class_names=names, + is_crowd=[x.get("iscrowd", 0) for x in annos], + ) + self.overlay_instances( + labels=labels, boxes=boxes, masks=masks, keypoints=keypts, assigned_colors=colors + ) + + sem_seg = dic.get("sem_seg", None) + if sem_seg is None and "sem_seg_file_name" in dic: + with PathManager.open(dic["sem_seg_file_name"], "rb") as f: + sem_seg = Image.open(f) + sem_seg = np.asarray(sem_seg, dtype="uint8") + if sem_seg is not None: + self.draw_sem_seg(sem_seg, area_threshold=0, alpha=0.5) + + pan_seg = dic.get("pan_seg", None) + if pan_seg is None and "pan_seg_file_name" in dic: + with PathManager.open(dic["pan_seg_file_name"], "rb") as f: + pan_seg = Image.open(f) + pan_seg = np.asarray(pan_seg) + from panopticapi.utils import rgb2id + + pan_seg = rgb2id(pan_seg) + if pan_seg is not None: + segments_info = dic["segments_info"] + pan_seg = torch.tensor(pan_seg) + self.draw_panoptic_seg(pan_seg, segments_info, area_threshold=0, alpha=0.5) + return self.output + + def overlay_instances( + self, + *, + boxes=None, + labels=None, + masks=None, + keypoints=None, + assigned_colors=None, + alpha=0.5, + ): + """ + Args: + boxes (Boxes, RotatedBoxes or ndarray): either a :class:`Boxes`, + or an Nx4 numpy array of XYXY_ABS format for the N objects in a single image, + or a :class:`RotatedBoxes`, + or an Nx5 numpy array of (x_center, y_center, width, height, angle_degrees) format + for the N objects in a single image, + labels (list[str]): the text to be displayed for each instance. + masks (masks-like object): Supported types are: + + * :class:`detectron2.structures.PolygonMasks`, + :class:`detectron2.structures.BitMasks`. + * list[list[ndarray]]: contains the segmentation masks for all objects in one image. + The first level of the list corresponds to individual instances. The second + level to all the polygon that compose the instance, and the third level + to the polygon coordinates. The third level should have the format of + [x0, y0, x1, y1, ..., xn, yn] (n >= 3). + * list[ndarray]: each ndarray is a binary mask of shape (H, W). + * list[dict]: each dict is a COCO-style RLE. + keypoints (Keypoint or array like): an array-like object of shape (N, K, 3), + where the N is the number of instances and K is the number of keypoints. + The last dimension corresponds to (x, y, visibility or score). + assigned_colors (list[matplotlib.colors]): a list of colors, where each color + corresponds to each mask or box in the image. Refer to 'matplotlib.colors' + for full list of formats that the colors are accepted in. + Returns: + output (VisImage): image object with visualizations. + """ + num_instances = 0 + if boxes is not None: + boxes = self._convert_boxes(boxes) + num_instances = len(boxes) + if masks is not None: + masks = self._convert_masks(masks) + if num_instances: + assert len(masks) == num_instances + else: + num_instances = len(masks) + if keypoints is not None: + if num_instances: + assert len(keypoints) == num_instances + else: + num_instances = len(keypoints) + keypoints = self._convert_keypoints(keypoints) + if labels is not None: + assert len(labels) == num_instances + if assigned_colors is None: + assigned_colors = [random_color(rgb=True, maximum=1) for _ in range(num_instances)] + if num_instances == 0: + return self.output + if boxes is not None and boxes.shape[1] == 5: + return self.overlay_rotated_instances( + boxes=boxes, labels=labels, assigned_colors=assigned_colors + ) + + # Display in largest to smallest order to reduce occlusion. + areas = None + if boxes is not None: + areas = np.prod(boxes[:, 2:] - boxes[:, :2], axis=1) + elif masks is not None: + areas = np.asarray([x.area() for x in masks]) + + if areas is not None: + sorted_idxs = np.argsort(-areas).tolist() + # Re-order overlapped instances in descending order. + boxes = boxes[sorted_idxs] if boxes is not None else None + labels = [labels[k] for k in sorted_idxs] if labels is not None else None + masks = [masks[idx] for idx in sorted_idxs] if masks is not None else None + assigned_colors = [assigned_colors[idx] for idx in sorted_idxs] + keypoints = keypoints[sorted_idxs] if keypoints is not None else None + + for i in range(num_instances): + color = assigned_colors[i] + if boxes is not None: + self.draw_box(boxes[i], edge_color=color) + + if masks is not None: + for segment in masks[i].polygons: + self.draw_polygon(segment.reshape(-1, 2), color, alpha=alpha) + + if labels is not None: + # first get a box + if boxes is not None: + x0, y0, x1, y1 = boxes[i] + text_pos = (x0, y0) # if drawing boxes, put text on the box corner. + horiz_align = "left" + elif masks is not None: + # skip small mask without polygon + if len(masks[i].polygons) == 0: + continue + + x0, y0, x1, y1 = masks[i].bbox() + + # draw text in the center (defined by median) when box is not drawn + # median is less sensitive to outliers. + text_pos = np.median(masks[i].mask.nonzero(), axis=1)[::-1] + horiz_align = "center" + else: + continue # drawing the box confidence for keypoints isn't very useful. + # for small objects, draw text at the side to avoid occlusion + instance_area = (y1 - y0) * (x1 - x0) + if ( + instance_area < _SMALL_OBJECT_AREA_THRESH * self.output.scale + or y1 - y0 < 40 * self.output.scale + ): + if y1 >= self.output.height - 5: + text_pos = (x1, y0) + else: + text_pos = (x0, y1) + + height_ratio = (y1 - y0) / np.sqrt(self.output.height * self.output.width) + lighter_color = self._change_color_brightness(color, brightness_factor=0.7) + font_size = ( + np.clip((height_ratio - 0.02) / 0.08 + 1, 1.2, 2) + * 0.5 + * self._default_font_size + ) + self.draw_text( + labels[i], + text_pos, + color=lighter_color, + horizontal_alignment=horiz_align, + font_size=font_size, + ) + + # draw keypoints + if keypoints is not None: + for keypoints_per_instance in keypoints: + self.draw_and_connect_keypoints(keypoints_per_instance) + + return self.output + + def overlay_rotated_instances(self, boxes=None, labels=None, assigned_colors=None): + """ + Args: + boxes (ndarray): an Nx5 numpy array of + (x_center, y_center, width, height, angle_degrees) format + for the N objects in a single image. + labels (list[str]): the text to be displayed for each instance. + assigned_colors (list[matplotlib.colors]): a list of colors, where each color + corresponds to each mask or box in the image. Refer to 'matplotlib.colors' + for full list of formats that the colors are accepted in. + + Returns: + output (VisImage): image object with visualizations. + """ + num_instances = len(boxes) + + if assigned_colors is None: + assigned_colors = [random_color(rgb=True, maximum=1) for _ in range(num_instances)] + if num_instances == 0: + return self.output + + # Display in largest to smallest order to reduce occlusion. + if boxes is not None: + areas = boxes[:, 2] * boxes[:, 3] + + sorted_idxs = np.argsort(-areas).tolist() + # Re-order overlapped instances in descending order. + boxes = boxes[sorted_idxs] + labels = [labels[k] for k in sorted_idxs] if labels is not None else None + colors = [assigned_colors[idx] for idx in sorted_idxs] + + for i in range(num_instances): + self.draw_rotated_box_with_label( + boxes[i], edge_color=colors[i], label=labels[i] if labels is not None else None + ) + + return self.output + + def draw_and_connect_keypoints(self, keypoints): + """ + Draws keypoints of an instance and follows the rules for keypoint connections + to draw lines between appropriate keypoints. This follows color heuristics for + line color. + + Args: + keypoints (Tensor): a tensor of shape (K, 3), where K is the number of keypoints + and the last dimension corresponds to (x, y, probability). + + Returns: + output (VisImage): image object with visualizations. + """ + visible = {} + keypoint_names = self.metadata.get("keypoint_names") + for idx, keypoint in enumerate(keypoints): + + # draw keypoint + x, y, prob = keypoint + if prob > self.keypoint_threshold: + self.draw_circle((x, y), color=_RED) + if keypoint_names: + keypoint_name = keypoint_names[idx] + visible[keypoint_name] = (x, y) + + if self.metadata.get("keypoint_connection_rules"): + for kp0, kp1, color in self.metadata.keypoint_connection_rules: + if kp0 in visible and kp1 in visible: + x0, y0 = visible[kp0] + x1, y1 = visible[kp1] + color = tuple(x / 255.0 for x in color) + self.draw_line([x0, x1], [y0, y1], color=color) + + # draw lines from nose to mid-shoulder and mid-shoulder to mid-hip + # Note that this strategy is specific to person keypoints. + # For other keypoints, it should just do nothing + try: + ls_x, ls_y = visible["left_shoulder"] + rs_x, rs_y = visible["right_shoulder"] + mid_shoulder_x, mid_shoulder_y = (ls_x + rs_x) / 2, (ls_y + rs_y) / 2 + except KeyError: + pass + else: + # draw line from nose to mid-shoulder + nose_x, nose_y = visible.get("nose", (None, None)) + if nose_x is not None: + self.draw_line([nose_x, mid_shoulder_x], [nose_y, mid_shoulder_y], color=_RED) + + try: + # draw line from mid-shoulder to mid-hip + lh_x, lh_y = visible["left_hip"] + rh_x, rh_y = visible["right_hip"] + except KeyError: + pass + else: + mid_hip_x, mid_hip_y = (lh_x + rh_x) / 2, (lh_y + rh_y) / 2 + self.draw_line([mid_hip_x, mid_shoulder_x], [mid_hip_y, mid_shoulder_y], color=_RED) + return self.output + + """ + Primitive drawing functions: + """ + + def draw_text( + self, + text, + position, + *, + font_size=None, + color="g", + horizontal_alignment="center", + rotation=0, + ): + """ + Args: + text (str): class label + position (tuple): a tuple of the x and y coordinates to place text on image. + font_size (int, optional): font of the text. If not provided, a font size + proportional to the image width is calculated and used. + color: color of the text. Refer to `matplotlib.colors` for full list + of formats that are accepted. + horizontal_alignment (str): see `matplotlib.text.Text` + rotation: rotation angle in degrees CCW + + Returns: + output (VisImage): image object with text drawn. + """ + if not font_size: + font_size = self._default_font_size + + # since the text background is dark, we don't want the text to be dark + color = np.maximum(list(mplc.to_rgb(color)), 0.2) + color[np.argmax(color)] = max(0.8, np.max(color)) + + x, y = position + self.output.ax.text( + x, + y, + text, + size=font_size * self.output.scale, + family="sans-serif", + bbox={"facecolor": "black", "alpha": 0.8, "pad": 0.7, "edgecolor": "none"}, + verticalalignment="top", + horizontalalignment=horizontal_alignment, + color=color, + zorder=10, + rotation=rotation, + ) + return self.output + + def draw_box(self, box_coord, alpha=0.5, edge_color="g", line_style="-"): + """ + Args: + box_coord (tuple): a tuple containing x0, y0, x1, y1 coordinates, where x0 and y0 + are the coordinates of the image's top left corner. x1 and y1 are the + coordinates of the image's bottom right corner. + alpha (float): blending efficient. Smaller values lead to more transparent masks. + edge_color: color of the outline of the box. Refer to `matplotlib.colors` + for full list of formats that are accepted. + line_style (string): the string to use to create the outline of the boxes. + + Returns: + output (VisImage): image object with box drawn. + """ + x0, y0, x1, y1 = box_coord + width = x1 - x0 + height = y1 - y0 + + linewidth = max(self._default_font_size / 4, 1) + + self.output.ax.add_patch( + mpl.patches.Rectangle( + (x0, y0), + width, + height, + fill=False, + edgecolor=edge_color, + linewidth=linewidth * self.output.scale, + alpha=alpha, + linestyle=line_style, + ) + ) + return self.output + + def draw_rotated_box_with_label( + self, rotated_box, alpha=0.5, edge_color="g", line_style="-", label=None + ): + """ + Draw a rotated box with label on its top-left corner. + + Args: + rotated_box (tuple): a tuple containing (cnt_x, cnt_y, w, h, angle), + where cnt_x and cnt_y are the center coordinates of the box. + w and h are the width and height of the box. angle represents how + many degrees the box is rotated CCW with regard to the 0-degree box. + alpha (float): blending efficient. Smaller values lead to more transparent masks. + edge_color: color of the outline of the box. Refer to `matplotlib.colors` + for full list of formats that are accepted. + line_style (string): the string to use to create the outline of the boxes. + label (string): label for rotated box. It will not be rendered when set to None. + + Returns: + output (VisImage): image object with box drawn. + """ + cnt_x, cnt_y, w, h, angle = rotated_box + area = w * h + # use thinner lines when the box is small + linewidth = self._default_font_size / ( + 6 if area < _SMALL_OBJECT_AREA_THRESH * self.output.scale else 3 + ) + + theta = angle * math.pi / 180.0 + c = math.cos(theta) + s = math.sin(theta) + rect = [(-w / 2, h / 2), (-w / 2, -h / 2), (w / 2, -h / 2), (w / 2, h / 2)] + # x: left->right ; y: top->down + rotated_rect = [(s * yy + c * xx + cnt_x, c * yy - s * xx + cnt_y) for (xx, yy) in rect] + for k in range(4): + j = (k + 1) % 4 + self.draw_line( + [rotated_rect[k][0], rotated_rect[j][0]], + [rotated_rect[k][1], rotated_rect[j][1]], + color=edge_color, + linestyle="--" if k == 1 else line_style, + linewidth=linewidth, + ) + + if label is not None: + text_pos = rotated_rect[1] # topleft corner + + height_ratio = h / np.sqrt(self.output.height * self.output.width) + label_color = self._change_color_brightness(edge_color, brightness_factor=0.7) + font_size = ( + np.clip((height_ratio - 0.02) / 0.08 + 1, 1.2, 2) * 0.5 * self._default_font_size + ) + self.draw_text(label, text_pos, color=label_color, font_size=font_size, rotation=angle) + + return self.output + + def draw_circle(self, circle_coord, color, radius=3): + """ + Args: + circle_coord (list(int) or tuple(int)): contains the x and y coordinates + of the center of the circle. + color: color of the polygon. Refer to `matplotlib.colors` for a full list of + formats that are accepted. + radius (int): radius of the circle. + + Returns: + output (VisImage): image object with box drawn. + """ + x, y = circle_coord + self.output.ax.add_patch( + mpl.patches.Circle(circle_coord, radius=radius, fill=True, color=color) + ) + return self.output + + def draw_line(self, x_data, y_data, color, linestyle="-", linewidth=None): + """ + Args: + x_data (list[int]): a list containing x values of all the points being drawn. + Length of list should match the length of y_data. + y_data (list[int]): a list containing y values of all the points being drawn. + Length of list should match the length of x_data. + color: color of the line. Refer to `matplotlib.colors` for a full list of + formats that are accepted. + linestyle: style of the line. Refer to `matplotlib.lines.Line2D` + for a full list of formats that are accepted. + linewidth (float or None): width of the line. When it's None, + a default value will be computed and used. + + Returns: + output (VisImage): image object with line drawn. + """ + if linewidth is None: + linewidth = self._default_font_size / 3 + linewidth = max(linewidth, 1) + self.output.ax.add_line( + mpl.lines.Line2D( + x_data, + y_data, + linewidth=linewidth * self.output.scale, + color=color, + linestyle=linestyle, + ) + ) + return self.output + + def draw_binary_mask( + self, binary_mask, color=None, *, edge_color=None, text=None, alpha=0.5, area_threshold=10 + ): + """ + Args: + binary_mask (ndarray): numpy array of shape (H, W), where H is the image height and + W is the image width. Each value in the array is either a 0 or 1 value of uint8 + type. + color: color of the mask. Refer to `matplotlib.colors` for a full list of + formats that are accepted. If None, will pick a random color. + edge_color: color of the polygon edges. Refer to `matplotlib.colors` for a + full list of formats that are accepted. + text (str): if None, will be drawn on the object + alpha (float): blending efficient. Smaller values lead to more transparent masks. + area_threshold (float): a connected component smaller than this area will not be shown. + + Returns: + output (VisImage): image object with mask drawn. + """ + if color is None: + color = random_color(rgb=True, maximum=1) + color = mplc.to_rgb(color) + + has_valid_segment = False + binary_mask = binary_mask.astype("uint8") # opencv needs uint8 + mask = GenericMask(binary_mask, self.output.height, self.output.width) + shape2d = (binary_mask.shape[0], binary_mask.shape[1]) + + if not mask.has_holes: + # draw polygons for regular masks + for segment in mask.polygons: + area = mask_util.area(mask_util.frPyObjects([segment], shape2d[0], shape2d[1])) + if area < (area_threshold or 0): + continue + has_valid_segment = True + segment = segment.reshape(-1, 2) + self.draw_polygon(segment, color=color, edge_color=edge_color, alpha=alpha) + else: + # TODO: Use Path/PathPatch to draw vector graphics: + # https://stackoverflow.com/questions/8919719/how-to-plot-a-complex-polygon + rgba = np.zeros(shape2d + (4,), dtype="float32") + rgba[:, :, :3] = color + rgba[:, :, 3] = (mask.mask == 1).astype("float32") * alpha + has_valid_segment = True + self.output.ax.imshow(rgba, extent=(0, self.output.width, self.output.height, 0)) + + if text is not None and has_valid_segment: + lighter_color = self._change_color_brightness(color, brightness_factor=0.7) + self._draw_text_in_mask(binary_mask, text, lighter_color) + return self.output + + def draw_soft_mask(self, soft_mask, color=None, *, text=None, alpha=0.5): + """ + Args: + soft_mask (ndarray): float array of shape (H, W), each value in [0, 1]. + color: color of the mask. Refer to `matplotlib.colors` for a full list of + formats that are accepted. If None, will pick a random color. + text (str): if None, will be drawn on the object + alpha (float): blending efficient. Smaller values lead to more transparent masks. + + Returns: + output (VisImage): image object with mask drawn. + """ + if color is None: + color = random_color(rgb=True, maximum=1) + color = mplc.to_rgb(color) + + shape2d = (soft_mask.shape[0], soft_mask.shape[1]) + rgba = np.zeros(shape2d + (4,), dtype="float32") + rgba[:, :, :3] = color + rgba[:, :, 3] = soft_mask * alpha + self.output.ax.imshow(rgba, extent=(0, self.output.width, self.output.height, 0)) + + if text is not None: + lighter_color = self._change_color_brightness(color, brightness_factor=0.7) + binary_mask = (soft_mask > 0.5).astype("uint8") + self._draw_text_in_mask(binary_mask, text, lighter_color) + return self.output + + def draw_polygon(self, segment, color, edge_color=None, alpha=0.5): + """ + Args: + segment: numpy array of shape Nx2, containing all the points in the polygon. + color: color of the polygon. Refer to `matplotlib.colors` for a full list of + formats that are accepted. + edge_color: color of the polygon edges. Refer to `matplotlib.colors` for a + full list of formats that are accepted. If not provided, a darker shade + of the polygon color will be used instead. + alpha (float): blending efficient. Smaller values lead to more transparent masks. + + Returns: + output (VisImage): image object with polygon drawn. + """ + if edge_color is None: + # make edge color darker than the polygon color + if alpha > 0.8: + edge_color = self._change_color_brightness(color, brightness_factor=-0.7) + else: + edge_color = color + edge_color = mplc.to_rgb(edge_color) + (1,) + + polygon = mpl.patches.Polygon( + segment, + fill=True, + facecolor=mplc.to_rgb(color) + (alpha,), + edgecolor=edge_color, + linewidth=max(self._default_font_size // 15 * self.output.scale, 1), + ) + self.output.ax.add_patch(polygon) + return self.output + + """ + Internal methods: + """ + + def _jitter(self, color): + """ + Randomly modifies given color to produce a slightly different color than the color given. + + Args: + color (tuple[double]): a tuple of 3 elements, containing the RGB values of the color + picked. The values in the list are in the [0.0, 1.0] range. + + Returns: + jittered_color (tuple[double]): a tuple of 3 elements, containing the RGB values of the + color after being jittered. The values in the list are in the [0.0, 1.0] range. + """ + color = mplc.to_rgb(color) + vec = np.random.rand(3) + # better to do it in another color space + vec = vec / np.linalg.norm(vec) * 0.5 + res = np.clip(vec + color, 0, 1) + return tuple(res) + + def _create_grayscale_image(self, mask=None): + """ + Create a grayscale version of the original image. + The colors in masked area, if given, will be kept. + """ + img_bw = self.img.astype("f4").mean(axis=2) + img_bw = np.stack([img_bw] * 3, axis=2) + if mask is not None: + img_bw[mask] = self.img[mask] + return img_bw + + def _change_color_brightness(self, color, brightness_factor): + """ + Depending on the brightness_factor, gives a lighter or darker color i.e. a color with + less or more saturation than the original color. + + Args: + color: color of the polygon. Refer to `matplotlib.colors` for a full list of + formats that are accepted. + brightness_factor (float): a value in [-1.0, 1.0] range. A lightness factor of + 0 will correspond to no change, a factor in [-1.0, 0) range will result in + a darker color and a factor in (0, 1.0] range will result in a lighter color. + + Returns: + modified_color (tuple[double]): a tuple containing the RGB values of the + modified color. Each value in the tuple is in the [0.0, 1.0] range. + """ + assert brightness_factor >= -1.0 and brightness_factor <= 1.0 + color = mplc.to_rgb(color) + polygon_color = colorsys.rgb_to_hls(*mplc.to_rgb(color)) + modified_lightness = polygon_color[1] + (brightness_factor * polygon_color[1]) + modified_lightness = 0.0 if modified_lightness < 0.0 else modified_lightness + modified_lightness = 1.0 if modified_lightness > 1.0 else modified_lightness + modified_color = colorsys.hls_to_rgb(polygon_color[0], modified_lightness, polygon_color[2]) + return tuple(np.clip(modified_color, 0.0, 1.0)) + + def _convert_boxes(self, boxes): + """ + Convert different format of boxes to an NxB array, where B = 4 or 5 is the box dimension. + """ + if isinstance(boxes, Boxes) or isinstance(boxes, RotatedBoxes): + return boxes.tensor.detach().numpy() + else: + return np.asarray(boxes) + + def _convert_masks(self, masks_or_polygons): + """ + Convert different format of masks or polygons to a tuple of masks and polygons. + + Returns: + list[GenericMask]: + """ + + m = masks_or_polygons + if isinstance(m, PolygonMasks): + m = m.polygons + if isinstance(m, BitMasks): + m = m.tensor.numpy() + if isinstance(m, torch.Tensor): + m = m.numpy() + ret = [] + for x in m: + if isinstance(x, GenericMask): + ret.append(x) + else: + ret.append(GenericMask(x, self.output.height, self.output.width)) + return ret + + def _draw_text_in_mask(self, binary_mask, text, color): + """ + Find proper places to draw text given a binary mask. + """ + # TODO sometimes drawn on wrong objects. the heuristics here can improve. + _num_cc, cc_labels, stats, centroids = cv2.connectedComponentsWithStats(binary_mask, 8) + if stats[1:, -1].size == 0: + return + largest_component_id = np.argmax(stats[1:, -1]) + 1 + + # draw text on the largest component, as well as other very large components. + for cid in range(1, _num_cc): + if cid == largest_component_id or stats[cid, -1] > _LARGE_MASK_AREA_THRESH: + # median is more stable than centroid + # center = centroids[largest_component_id] + center = np.median((cc_labels == cid).nonzero(), axis=1)[::-1] + self.draw_text(text, center, color=color) + + def _convert_keypoints(self, keypoints): + if isinstance(keypoints, Keypoints): + keypoints = keypoints.tensor + keypoints = np.asarray(keypoints) + return keypoints + + def get_output(self): + """ + Returns: + output (VisImage): the image output containing the visualizations added + to the image. + """ + return self.output diff --git a/annotator/oneformer/oneformer/__init__.py b/annotator/oneformer/oneformer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..39ebcd384f616ae2ba170407cee3267d461a5914 --- /dev/null +++ b/annotator/oneformer/oneformer/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from . import data # register all new datasets +from . import modeling + +# config +from .config import * + +# models +from .oneformer_model import OneFormer \ No newline at end of file diff --git a/annotator/oneformer/oneformer/config.py b/annotator/oneformer/oneformer/config.py new file mode 100644 index 0000000000000000000000000000000000000000..78879b1edd2a9edec5cdaf8e3cc1fd471c3a57be --- /dev/null +++ b/annotator/oneformer/oneformer/config.py @@ -0,0 +1,239 @@ +# -*- coding: utf-8 -*- +# Copyright (c) Facebook, Inc. and its affiliates. +from annotator.oneformer.detectron2.config import CfgNode as CN + +__all__ = ["add_common_config", "add_oneformer_config", "add_swin_config", + "add_dinat_config", "add_beit_adapter_config", "add_convnext_config"] + +def add_common_config(cfg): + """ + Add config for common configuration + """ + # data config + # select the dataset mapper + cfg.INPUT.DATASET_MAPPER_NAME = "oneformer_unified" + # Color augmentation + cfg.INPUT.COLOR_AUG_SSD = False + # We retry random cropping until no single category in semantic segmentation GT occupies more + # than `SINGLE_CATEGORY_MAX_AREA` part of the crop. + cfg.INPUT.CROP.SINGLE_CATEGORY_MAX_AREA = 1.0 + # Pad image and segmentation GT in dataset mapper. + cfg.INPUT.SIZE_DIVISIBILITY = -1 + + cfg.INPUT.TASK_SEQ_LEN = 77 + cfg.INPUT.MAX_SEQ_LEN = 77 + + cfg.INPUT.TASK_PROB = CN() + cfg.INPUT.TASK_PROB.SEMANTIC = 0.33 + cfg.INPUT.TASK_PROB.INSTANCE = 0.66 + + # test dataset + cfg.DATASETS.TEST_PANOPTIC = ("",) + cfg.DATASETS.TEST_INSTANCE = ("",) + cfg.DATASETS.TEST_SEMANTIC = ("",) + + # solver config + # weight decay on embedding + cfg.SOLVER.WEIGHT_DECAY_EMBED = 0.0 + # optimizer + cfg.SOLVER.OPTIMIZER = "ADAMW" + cfg.SOLVER.BACKBONE_MULTIPLIER = 0.1 + + # wandb + cfg.WANDB = CN() + cfg.WANDB.PROJECT = "unified_dense_recognition" + cfg.WANDB.NAME = None + + cfg.MODEL.IS_TRAIN = False + cfg.MODEL.IS_DEMO = True + + # text encoder config + cfg.MODEL.TEXT_ENCODER = CN() + + cfg.MODEL.TEXT_ENCODER.WIDTH = 256 + cfg.MODEL.TEXT_ENCODER.CONTEXT_LENGTH = 77 + cfg.MODEL.TEXT_ENCODER.NUM_LAYERS = 12 + cfg.MODEL.TEXT_ENCODER.VOCAB_SIZE = 49408 + cfg.MODEL.TEXT_ENCODER.PROJ_NUM_LAYERS = 2 + cfg.MODEL.TEXT_ENCODER.N_CTX = 16 + + # mask_former inference config + cfg.MODEL.TEST = CN() + cfg.MODEL.TEST.SEMANTIC_ON = True + cfg.MODEL.TEST.INSTANCE_ON = False + cfg.MODEL.TEST.PANOPTIC_ON = False + cfg.MODEL.TEST.DETECTION_ON = False + cfg.MODEL.TEST.OBJECT_MASK_THRESHOLD = 0.0 + cfg.MODEL.TEST.OVERLAP_THRESHOLD = 0.0 + cfg.MODEL.TEST.SEM_SEG_POSTPROCESSING_BEFORE_INFERENCE = False + cfg.MODEL.TEST.TASK = "panoptic" + + # TEST AUG Slide + cfg.TEST.AUG.IS_SLIDE = False + cfg.TEST.AUG.CROP_SIZE = (640, 640) + cfg.TEST.AUG.STRIDE = (426, 426) + cfg.TEST.AUG.SCALE = (2048, 640) + cfg.TEST.AUG.SETR_MULTI_SCALE = True + cfg.TEST.AUG.KEEP_RATIO = True + cfg.TEST.AUG.SIZE_DIVISOR = 32 + + # pixel decoder config + cfg.MODEL.SEM_SEG_HEAD.MASK_DIM = 256 + # adding transformer in pixel decoder + cfg.MODEL.SEM_SEG_HEAD.TRANSFORMER_ENC_LAYERS = 0 + # pixel decoder + cfg.MODEL.SEM_SEG_HEAD.PIXEL_DECODER_NAME = "BasePixelDecoder" + cfg.MODEL.SEM_SEG_HEAD.SEM_EMBED_DIM = 256 + cfg.MODEL.SEM_SEG_HEAD.INST_EMBED_DIM = 256 + + # LSJ aug + cfg.INPUT.IMAGE_SIZE = 1024 + cfg.INPUT.MIN_SCALE = 0.1 + cfg.INPUT.MAX_SCALE = 2.0 + + # MSDeformAttn encoder configs + cfg.MODEL.SEM_SEG_HEAD.DEFORMABLE_TRANSFORMER_ENCODER_IN_FEATURES = ["res3", "res4", "res5"] + cfg.MODEL.SEM_SEG_HEAD.DEFORMABLE_TRANSFORMER_ENCODER_N_POINTS = 4 + cfg.MODEL.SEM_SEG_HEAD.DEFORMABLE_TRANSFORMER_ENCODER_N_HEADS = 8 + +def add_oneformer_config(cfg): + """ + Add config for ONE_FORMER. + """ + + # mask_former model config + cfg.MODEL.ONE_FORMER = CN() + + # loss + cfg.MODEL.ONE_FORMER.DEEP_SUPERVISION = True + cfg.MODEL.ONE_FORMER.NO_OBJECT_WEIGHT = 0.1 + cfg.MODEL.ONE_FORMER.CLASS_WEIGHT = 1.0 + cfg.MODEL.ONE_FORMER.DICE_WEIGHT = 1.0 + cfg.MODEL.ONE_FORMER.MASK_WEIGHT = 20.0 + cfg.MODEL.ONE_FORMER.CONTRASTIVE_WEIGHT = 0.5 + cfg.MODEL.ONE_FORMER.CONTRASTIVE_TEMPERATURE = 0.07 + + # transformer config + cfg.MODEL.ONE_FORMER.NHEADS = 8 + cfg.MODEL.ONE_FORMER.DROPOUT = 0.1 + cfg.MODEL.ONE_FORMER.DIM_FEEDFORWARD = 2048 + cfg.MODEL.ONE_FORMER.ENC_LAYERS = 0 + cfg.MODEL.ONE_FORMER.CLASS_DEC_LAYERS = 2 + cfg.MODEL.ONE_FORMER.DEC_LAYERS = 6 + cfg.MODEL.ONE_FORMER.PRE_NORM = False + + cfg.MODEL.ONE_FORMER.HIDDEN_DIM = 256 + cfg.MODEL.ONE_FORMER.NUM_OBJECT_QUERIES = 120 + cfg.MODEL.ONE_FORMER.NUM_OBJECT_CTX = 16 + cfg.MODEL.ONE_FORMER.USE_TASK_NORM = True + + cfg.MODEL.ONE_FORMER.TRANSFORMER_IN_FEATURE = "res5" + cfg.MODEL.ONE_FORMER.ENFORCE_INPUT_PROJ = False + + # Sometimes `backbone.size_divisibility` is set to 0 for some backbone (e.g. ResNet) + # you can use this config to override + cfg.MODEL.ONE_FORMER.SIZE_DIVISIBILITY = 32 + + # transformer module + cfg.MODEL.ONE_FORMER.TRANSFORMER_DECODER_NAME = "ContrastiveMultiScaleMaskedTransformerDecoder" + + # point loss configs + # Number of points sampled during training for a mask point head. + cfg.MODEL.ONE_FORMER.TRAIN_NUM_POINTS = 112 * 112 + # Oversampling parameter for PointRend point sampling during training. Parameter `k` in the + # original paper. + cfg.MODEL.ONE_FORMER.OVERSAMPLE_RATIO = 3.0 + # Importance sampling parameter for PointRend point sampling during training. Parametr `beta` in + # the original paper. + cfg.MODEL.ONE_FORMER.IMPORTANCE_SAMPLE_RATIO = 0.75 + +def add_swin_config(cfg): + """ + Add config forSWIN Backbone. + """ + + # swin transformer backbone + cfg.MODEL.SWIN = CN() + cfg.MODEL.SWIN.PRETRAIN_IMG_SIZE = 224 + cfg.MODEL.SWIN.PATCH_SIZE = 4 + cfg.MODEL.SWIN.EMBED_DIM = 96 + cfg.MODEL.SWIN.DEPTHS = [2, 2, 6, 2] + cfg.MODEL.SWIN.NUM_HEADS = [3, 6, 12, 24] + cfg.MODEL.SWIN.WINDOW_SIZE = 7 + cfg.MODEL.SWIN.MLP_RATIO = 4.0 + cfg.MODEL.SWIN.QKV_BIAS = True + cfg.MODEL.SWIN.QK_SCALE = None + cfg.MODEL.SWIN.DROP_RATE = 0.0 + cfg.MODEL.SWIN.ATTN_DROP_RATE = 0.0 + cfg.MODEL.SWIN.DROP_PATH_RATE = 0.3 + cfg.MODEL.SWIN.APE = False + cfg.MODEL.SWIN.PATCH_NORM = True + cfg.MODEL.SWIN.OUT_FEATURES = ["res2", "res3", "res4", "res5"] + cfg.MODEL.SWIN.USE_CHECKPOINT = False + ## Semask additions + cfg.MODEL.SWIN.SEM_WINDOW_SIZE = 7 + cfg.MODEL.SWIN.NUM_SEM_BLOCKS = 1 + +def add_dinat_config(cfg): + """ + Add config for NAT Backbone. + """ + + # DINAT transformer backbone + cfg.MODEL.DiNAT = CN() + cfg.MODEL.DiNAT.DEPTHS = [3, 4, 18, 5] + cfg.MODEL.DiNAT.OUT_FEATURES = ["res2", "res3", "res4", "res5"] + cfg.MODEL.DiNAT.EMBED_DIM = 64 + cfg.MODEL.DiNAT.MLP_RATIO = 3.0 + cfg.MODEL.DiNAT.NUM_HEADS = [2, 4, 8, 16] + cfg.MODEL.DiNAT.DROP_PATH_RATE = 0.2 + cfg.MODEL.DiNAT.KERNEL_SIZE = 7 + cfg.MODEL.DiNAT.DILATIONS = [[1, 16, 1], [1, 4, 1, 8], [1, 2, 1, 3, 1, 4], [1, 2, 1, 2, 1]] + cfg.MODEL.DiNAT.OUT_INDICES = (0, 1, 2, 3) + cfg.MODEL.DiNAT.QKV_BIAS = True + cfg.MODEL.DiNAT.QK_SCALE = None + cfg.MODEL.DiNAT.DROP_RATE = 0 + cfg.MODEL.DiNAT.ATTN_DROP_RATE = 0. + cfg.MODEL.DiNAT.IN_PATCH_SIZE = 4 + +def add_convnext_config(cfg): + """ + Add config for ConvNeXt Backbone. + """ + + # swin transformer backbone + cfg.MODEL.CONVNEXT = CN() + cfg.MODEL.CONVNEXT.IN_CHANNELS = 3 + cfg.MODEL.CONVNEXT.DEPTHS = [3, 3, 27, 3] + cfg.MODEL.CONVNEXT.DIMS = [192, 384, 768, 1536] + cfg.MODEL.CONVNEXT.DROP_PATH_RATE = 0.4 + cfg.MODEL.CONVNEXT.LSIT = 1.0 + cfg.MODEL.CONVNEXT.OUT_INDICES = [0, 1, 2, 3] + cfg.MODEL.CONVNEXT.OUT_FEATURES = ["res2", "res3", "res4", "res5"] + +def add_beit_adapter_config(cfg): + """ + Add config for BEiT Adapter Backbone. + """ + + # beit adapter backbone + cfg.MODEL.BEiTAdapter = CN() + cfg.MODEL.BEiTAdapter.IMG_SIZE = 640 + cfg.MODEL.BEiTAdapter.PATCH_SIZE = 16 + cfg.MODEL.BEiTAdapter.EMBED_DIM = 1024 + cfg.MODEL.BEiTAdapter.DEPTH = 24 + cfg.MODEL.BEiTAdapter.NUM_HEADS = 16 + cfg.MODEL.BEiTAdapter.MLP_RATIO = 4 + cfg.MODEL.BEiTAdapter.QKV_BIAS = True + cfg.MODEL.BEiTAdapter.USE_ABS_POS_EMB = False + cfg.MODEL.BEiTAdapter.USE_REL_POS_BIAS = True + cfg.MODEL.BEiTAdapter.INIT_VALUES = 1e-6 + cfg.MODEL.BEiTAdapter.DROP_PATH_RATE = 0.3 + cfg.MODEL.BEiTAdapter.CONV_INPLANE = 64 + cfg.MODEL.BEiTAdapter.N_POINTS = 4 + cfg.MODEL.BEiTAdapter.DEFORM_NUM_HEADS = 16 + cfg.MODEL.BEiTAdapter.CFFN_RATIO = 0.25 + cfg.MODEL.BEiTAdapter.DEFORM_RATIO = 0.5 + cfg.MODEL.BEiTAdapter.WITH_CP = True + cfg.MODEL.BEiTAdapter.INTERACTION_INDEXES=[[0, 5], [6, 11], [12, 17], [18, 23]] + cfg.MODEL.BEiTAdapter.OUT_FEATURES = ["res2", "res3", "res4", "res5"] \ No newline at end of file diff --git a/annotator/oneformer/oneformer/data/__init__.py b/annotator/oneformer/oneformer/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..63ba265b1effc69f1eef16e57a04db8902ee347e --- /dev/null +++ b/annotator/oneformer/oneformer/data/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from . import datasets diff --git a/annotator/oneformer/oneformer/data/bpe_simple_vocab_16e6.txt b/annotator/oneformer/oneformer/data/bpe_simple_vocab_16e6.txt new file mode 100644 index 0000000000000000000000000000000000000000..aff3ec8f3d5f3d6c859b5751eda5e870dffd7eaf --- /dev/null +++ b/annotator/oneformer/oneformer/data/bpe_simple_vocab_16e6.txt @@ -0,0 +1,262145 @@ +"bpe_simple_vocab_16e6.txt#version: 0.2 +i n +t h +a n +r e +a r +e r +th e +in g +o u +o n +s t +o r +e n +o n +a l +a t +e r +i t +i n +t o +r o +i s +l e +i c +a t +an d +e d +o f +c h +o r +e s +i l +e l +s t +a c +o m +a m +l o +a n +a y +s h +r i +l i +t i +f or +n e +ð Ł +r a +h a +d e +o l +v e +s i +u r +a l +s e +' s +u n +d i +b e +l a +w h +o o +d ay +e n +m a +n o +l e +t o +ou r +i r +g h +w it +i t +y o +a s +s p +th is +t s +at i +yo u +wit h +a d +i s +a b +l y +w e +th e +t e +a s +a g +v i +p p +s u +h o +m y +. . +b u +c om +s e +er s +m e +m e +al l +c on +m o +k e +g e +ou t +en t +c o +f e +v er +a r +f ro +a u +p o +c e +gh t +ar e +s s +fro m +c h +t r +ou n +on e +b y +d o +t h +w or +er e +k e +p ro +f or +d s +b o +t a +w e +g o +h e +t er +in g +d e +b e +ati on +m or +a y +e x +il l +p e +k s +s c +l u +f u +q u +v er +ðŁ ĺ +j u +m u +at e +an d +v e +k ing +m ar +o p +h i +.. . +p re +a d +r u +th at +j o +o f +c e +ne w +a m +a p +g re +s s +d u +no w +y e +t ing +y our +it y +n i +c i +p ar +g u +f i +a f +p er +t er +u p +s o +g i +on s +g r +g e +b r +p l +' t +m i +in e +we e +b i +u s +sh o +ha ve +to day +a v +m an +en t +ac k +ur e +ou r +â Ģ +c u +l d +lo o +i m +ic e +s om +f in +re d +re n +oo d +w as +ti on +p i +i r +th er +t y +p h +ar d +e c +! ! +m on +mor e +w ill +t ra +c an +c ol +p u +t e +w n +m b +s o +it i +ju st +n ing +h ere +t u +p a +p r +bu t +wh at +al ly +f ir +m in +c a +an t +s a +t ed +e v +m ent +f a +ge t +am e +ab out +g ra +no t +ha pp +ay s +m an +h is +ti me +li ke +g h +ha s +th an +lo ve +ar t +st e +d ing +h e +c re +w s +w at +d er +it e +s er +ac e +ag e +en d +st r +a w +st or +r e +c ar +el l +al l +p s +f ri +p ho +p or +d o +a k +w i +f re +wh o +sh i +b oo +s on +el l +wh en +il l +ho w +gre at +w in +e l +b l +s si +al i +som e +ðŁ Ĵ +t on +d er +le s +p la +ï ¸ +e d +s ch +h u +on g +d on +k i +s h +an n +c or +. . +oun d +a z +in e +ar y +fu l +st u +ou ld +st i +g o +se e +ab le +ar s +l l +m is +b er +c k +w a +en ts +n o +si g +f e +fir st +e t +sp e +ac k +i f +ou s +' m +st er +a pp +an g +an ce +an s +g ood +b re +e ver +the y +t ic +com e +of f +b ack +as e +ing s +ol d +i ght +f o +h er +happ y +p ic +it s +v ing +u s +m at +h om +d y +e m +s k +y ing +the ir +le d +r y +u l +h ar +c k +t on +on al +h el +r ic +b ir +vi e +w ay +t ri +d a +p le +b ro +st o +oo l +ni ght +tr u +b a +re ad +re s +ye ar +f r +t or +al s +c oun +c la +t ure +v el +at ed +le c +en d +th ing +v o +ic i +be st +c an +wor k +la st +af ter +en ce +p ri +p e +e s +i l +âĢ ¦ +d re +y s +o ver +i es +ðŁ ij +com m +t w +in k +s un +c l +li fe +t t +a ch +l and +s y +t re +t al +p ol +s m +du c +s al +f t +' re +ch e +w ar +t ur +ati ons +ac h +m s +il e +p m +ou gh +at e +st ar +wee k +! !! +c lu +th ere +n er +t om +s el +ï¸ ı +wor ld +v es +c am +go t +in ter +of f +u m +ton ight +o ther +h ou +loo k +j e +i d +si on +be au +at t +el i +or t +re c +f f +st er +su pp +g en +be en +il y +te am +m m +i c +pe op +it t +at s +on ly +mb er +en g +b ri +m p +k now +b ur +b ar +in s +lo w +sh e +ro w +â Ŀ +t ro +peop le +vi a +lo w +ag a +be t +x t +f ac +ch ar +e ar +w al +s en +f am +b le +n ati +is h +n or +g ame +li ve +s co +le y +d on +ic k +b all +ver y +the se +p an +i a +at ing +c r +a re +g ir +ma ke +st re +sho w +. " +f l +u p +d r +than ks +il li +w om +st s +i g +s ur +ever y +c ur +vie w +le t +in to +mo st +n a +in di +g ar +ha d +s ou +v ed +an t +iti on +ma de +f ol +un i +it ed +ðŁ ı +ic al +th r +read y +ch ec +d ra +k es +boo k +e p +si c +mor ning +ne ws +c au +c t +w ell +an c +pho to +th an +or s +bir th +g g +ou t +ne xt +som e +en ing +stor y +ch ri +do wn +hom e +f fe +fre e +d a +b or +f il +ci al +than k +si de +le ar +qu e +l ine +t en +at es +ye ars +m y +pho to +beau ti +ri ght +n u +for m +shi p +b an +th er +d ays +g am +as on +g y +ðŁ İ +birth day +se t +ic k +e t +st ill +com ing +ta ke +ðŁ ĩ +b b +s ol +s on +d en +e p +mu sic +the m +de n +wh y +f oo +c ra +am az +w n +h ol +t ting +w r +u e +ma g +c ro +l an +c lo +b ra +a k +s ing +c al +re ad +' ve +jo h +b ab +d ri +b lo +bi g +er ic +in t +t or +tr y +l a +le g +hou se +m ic +v al +beauti ful +l itt +chec k +ne w +ver s +s w +ar i +pla y +h er +âĢ ĵ +w in +m a +con gr +sch ool +f un +. @ +he al +ic h +d el +wh ere +l on +ke t +tw o +mu ch +wat ch +v en +d ed +a st +k ed +b as +go ing +m p +e ver +w ays +ro o +de sig +l y +s ed +to p +l in +ch an +to o +it ing +d ent +gh ts +t y +sp o +ne ed +b lu +in st +be ing +âĿ ¤ +w el +l s +hi m +m ay +st ing +n a +el y +litt le +g a +n at +tom or +m c +h on +w ant +a ir +pi c +am eric +p er +le ss +wee k +ve l +a h +c ap +ch am +g er +ti m +tomor row +ne ss +st ate +h al +ser v +z e +o s +p at +v is +ex c +s in +f f +c ity +c en +an y +b el +su mm +t in +w ould +loo king +k o +ce le +fam ily +m er +po w +hel p +bu s +c o +c le +sel f +en s +ic s +th o +an i +ch o +le ad +b s +t wee +th ink +for e +ch il +vi de +di d +al e +ch i +v il +en ds +w ing +p as +' ll +v ol +s a +g s +man y +j ec +be fore +gra ph +n y +ur ing +w il +d d +bu il +f av +st ed +tr an +l ing +ou d +d ge +fi el +nati onal +st a +c er +w ere +in a +se ason +c ou +n ed +amaz ing +ti ons +cele br +n s +a th +he ad +s day +d ar +lo c +v in +an other +g oo +s at +n y +jo in +pre s +s es +s ing +an a +in ing +.. .. +c our +ï¸ ı +ac t +cau se +li ght +am s +t a +b al +f c +hi gh +off ici +t t +chri st +d ic +d ay +ra l +h or +: ) +vi si +n am +o b +ma s +gh t +re ally +t un +fin d +thr ough +por t +u t +ti ve +st y +n e +or e +ðŁĺ Ĥ +supp ort +ne ver +ev en +ðŁ Ķ +h a +y a +l d +u k +r an +j am +wi th +me di +d es +ne y +ch ing +al e +h y +k in +! ! +d y +pl ace +al so +b le +wh ich +bl ack +b li +s ay +par k +pl ay +ir e +vide o +week end +a il +ke y +p t +w ard +fri day +d in +ine ss +g ro +b en +al ways +t ball +ag o +m il +c y +pro duc +di sc +un der +ple ase +sp or +fu ll +e y +ðŁ Ļ +is e +iti es +c at +k no +u se +fo re +k er +ar t +hi gh +op en +s an +e f +our s +sh ed +st ri +d ro +aga in +i m +ðŁ ĵ +en jo +fu n +ge tting +p en +g er +c li +an y +ever y +e u +wom en +â ľ +e st +c ould +r y +" @ +th ou +sh a +comm un +b er +d ents +di s +wh ile +aw ay +di o +h am +g la +d ate +k a +mis s +un ch +w on +in f +roo m +g a +re al +ex per +di rec +sh ould +sp r +g ol +l ong +bet ter +or i +e y +i ence +il s +z z +h an +f ound +v s +â Ļ +po st +ti c +par t +m en +ren ce +ce ss +v ic +s il +sho p +ðŁĺ Ĥ +f ood +v al +sti c +y ou +s ays +e lec +st ar +o c +l and +i d +c tion +fiel d +s of +st art +wat er +fri ends +on es +ðŁ Į +f la +f ar +wh ite +par ty +in st +gr ou +t v +every one +m ent +j a +ch a +pr in +an ts +d uring +l at +l ar +we st +th en +k a +y oun +in sp +in te +we en +visi t +aga inst +re le +he ad +c es +to wn +loo ks +th re +re gi +ren t +pro jec +gir l +se ar +w o +m om +c ar +h un +pu bli +d i +p le +c all +c ri +u m +for d +per fe +fri end +h ard +ssi on +te st +pla ying +ar ound +be cause +ke ts +me et +sat ur +ar ti +wor k +j un +v en +r un +me mber +por t +su per +t wit +s am +el s +t ly +ad v +ati ve +at h +s ure +av ail +la r +s qu +ar ds +ev ent +m en +l l +o ver +lo gy +it al +tim es +m al +b ack +c oo +ma king +st ru +â ģ +it u +sh ar +g an +c as +s n +summ er +pic ture +f an +h in +christ mas +c y +pr oud +cham pi +desig n +pp ing +ho pe +c a +avail able +ma y +we d +photo graph +spe cial +sal e +sto p +er y +a we +al ity +hi story +am a +pre si +b ru +wor king +d one +d r +k en +fe at +w ood +ate st +sun day +mo vi +vel y +s le +f ace +sp ec +stu dents +b y +ha m +sp on +bus iness +d at +i e +i p +so ci +g lo +h and +re cor +r s +me e +ke ep +p ur +heal th +sh e +com ple +go d +da vi +col lec +li st +r a +clu b +t ers +in clu +th ings +pl an +â ĺ +joh n +sh ing +at ul +so on +blu e +g or +satur day +w on +congr atul +se e +âĿ¤ ï¸ı +tho se +ðŁĺ į +fin al +d ou +it h +o wn +ro ad +t our +a st +indi a +ti l +n d +f er +fav or +su l +lear n +fir e +ju st +grou p +a h +r ac +bo dy +u r +c are +à ¸ +p lo +o h +po s +gi ve +te ch +su b +c ent +er ing +y m +il ity +f ic +lon don +v ir +gu ys +b a +ðŁ ¤ +bab y +sc re +ðŁĺ į +tru mp +un der +chan ge +i an +col le +ss es +l er +ss ed +n ice +ann oun +pow er +s ar +a king +min i +s li +s wee +k ar +fu l +c ru +ac tion +a ther +) . +st and +de vel +a a +g an +le ft +lo l +re l +tran s +m ents +in t +e f +man ag +di g +gen er +do wn +p au +ti v +k u +th ur +k en +st on +f ans +tal k +twee t +t oo +sty le +pro te +se con +fr on +awe some +g l +p al +ne t +s or +la u +g on +sin ce +t ty +ser ies +me mor +b eli +fil m +di d +di es +o t +congratul ations +p ra +e ve +w oo +offici al +su c +in cre +b on +par t +pp ed +cla ss +si ve +bo y +cu l +perfe ct +t ou +d am +wel come +foo tball +h i +p ap +wa it +ad a +congr ats +youn g +exc ited +re ce +j an +v a +re d +st ra +medi a +' d +do es +le t +mu l +ill s +gre en +m el +to ge +fu ture +ye ster +vers ity +for m +ta in +i de +ch es +ki ds +qu i +ha ha +de ta +bi g +favor ite +gir ls +con tin +do m +sear ch +u al +a ir +d ers +mon th +c er +yester day +commun ity +ad e +do g +vil le +ic es +d eli +sy ste +ru n +is m +he art +c up +en ti +fe w +presi dent +e ds +un til +fe sti +o k +f lo +sa id +ol e +me d +tra vel + £ +ph one +toge ther +fa st +lo t +gam es +sh ir +bet ween +y es +th ers +do ing +m ac +at or +b and +fol low +projec t +devel op +di ffe +con fe +spe ci +ca st +y s +bo ard +r d +i al +sh oo +r am +ha ving +sh are +fol low +on e +n ame +m r +pu t +disc u +or y +c ame +ou s +s ite +twit ter +t b +t it +fin ally +z ed +su per +com pan +us ing +all s +li st +r is +sho t +g al +t ar +de l +joh n +âĢ Ķ +some thing +ra m +inte re +wh e +b it +ðŁ į +stre et +oun d +a i +tic kets +movi e +re al +k y +ta king +o pp +c c +l am +m oun +in ve +bl ack +us ed +on line +y or +loc al +gu e +c ks +o w +ge st +bo ys +illi on +con t +re ci +in ed +eu ro +no w +se en +p h +te ach +de f +sou th +su ch +aw ard +mu st +is su +ca re +fe el +p lu +l atest +spor ts +we b +te x +e ment +s k +fi c +w an +te ch +o t +bo x +n er +fre e +t al +a sh +c ase +ho t +won der +mee ting +er a +ch all +ðŁ IJ +jo b +il i +c ool +j our +th s +m o +f el +di e +mic ha +e le +te am +serv ice +st and +ma kes +p ing +ear ly +com es +e k +ho li +v ers +ag ue +s au +thre e +mon day +fa shi +some one +th ro +se a +b ad +supp or +tur n +ur y +m ing +photograph y +n ic +mar k +pre tty +ss ing +wat ching +me mb +ar ri +coun ty +be ach +fr an +cen ter +pol ice +b at +publi c +t an +pre ss +s af +s y +ge ts +ro y +n ers +y our +bu y +st ers +sho w +as ed +chil dre +af ric +in es +sp ace +sc ri +h all +pa in +ar ing +hom e +m ur +heal th +ch ed +s and +rece i +gu y +e a +americ an +re si +childre n +- - +i ri +ing ton +coun try +ro ss +le n +ann a +boo ks +b c +e ce +d om +lo vely +k h +pe t +g y +g ri +st age +off ice +ro ck +m on +b ay +t able +su n +m ed +th in +l or +f low +( @ +uni versity +stor e +fron t +goo d +z a +vo te +nor th +he y +an im +or der +mi d +with out +a de +re member +mar ket +? ? +mu s +tra ining +e duc +bu t +co ver +st an +sc en +b la +bre ak +l ou +s ame +g old +a in +o s +bo th +l it +ver n +a i +al bu +p a +enjo y +be g +ell ing +thur sday +inf o +s an +americ a +ha ir +te l +mar ch +con cer +colle ge +confe rence +ap p +h our +ch ang +â ļ +s our +ol s +we ather +w ar +p hi +festi val +secon d +cu te +pr ac +en er +str y +le a +pol it +s av +se n +o w +m i +ne ar +ou ght +z e +co ffe +w illi +d an +se y +davi d +e se +f an +de ci +the at +no v +ati on +tr ac +sc i +re view +c el +e m +u n +ju ly +or ig +ti on +d ru +form er +st ay +af ter +in v +too k +dat a +b al +tu es +d an +ev ening +ðŁĺĤ ðŁĺĤ +d ol +u res +pro vi +t s +e st +sig n +j ac +u k +s ong +ye t +bo w +in du +j ap +h oo +po int +any one +z y +i st +h ur +it al +buil ding +wom an +ch ur +j er +per for +co ach +le ague +ce ss +ne t +i mag +nati on +br it +qu e +aw ards +ag es +wor ks +c ed +man ce +l ate +ig n +mon ey +tru e +i i +t ell +pl ac +p ac +as y +wor ld +be hin +im port +read ing +gra m +gi ving +me t +h it +for ward +st om +pres ent +jun e +so cial +no on +mar t +hal f +s we +go vern +k er +deta ils +li sh +_ _ +ac y +si a +ber t +f all +! !!! +) , +th i +d iti +sp ort +k ing +f it +st af +c at +mu se +cen tr +y er +con tro +b loo +wal k +ac tu +did n +li m +lear ning +re search +wed ne +au th +h ours +k y +f ar +h en +.. .. +it ch +ri l +str ong +sk y +que sti +jam es +r on +d g +f ur +c in +do es +app ro +mar ke +tu res +ful ly +ch at +behin d +te m +fin i +mis sion +b att +fe el +he av +every thing +b ar +w ish +pre mi +i ma +exper ience +e ach +re port +swee t +tic s +spr ing +re spon +syste m +vic tor +l in +sa w +al ready +gh ter +f le +ã ĥ +br ing +albu m +- - +ell s +st an +to m +inter national +w ent +an ni +mat ch +pp er +st one +sm all +ra in +fashi on +are a +v an +ag ram +k o +thou ght +wor th +v an +m er +coffe e +it es +g n +arti st +c on +ar ch +c ir +se cre +gr ound +is o +h and +co m +bri dge +h s +x i +l ink +pu l +sp l +r ace +f li +ri ver +g as +di sco +d al +play er +f it +photo s +it y +o k +j or +tr a +ap ril +ad s +a di +sol u +beau ty +do or +me ss +up date +ali a +sch o +en ed +mom ent +sco t +sc ience +i or +ti es +ac ross +ous ly +sh es +does n +p age +wat er +m illion +cla ssi +l ic +ca st +form ation +micha el +ell o +s mo +in ts +vi sion +op ening +ld n +au str +tues day +win ner +po ssi +r ound +shir t +di t +b o +u es +il led +al ong +tri p +star ting +im pro +k an +per son +no t +re co +ne eds +c le +li e +re st +r ing +win ter +si mp +mo m +be er +fac e +tor s +us a +collec tion +ge or +se ssion +tr ying +la s +la ke +j en +orig in +stu dent +se cur +v in +pic s +ex pe +com p +gon na +e qu +b ad +le y +a u +memb ers +bre ak +w all +gi c +din ner +bu l +insp ir +r i +min d +ic a +win ning +tal king +t ren +s is +t en +wonder ful +s now +he ar +th om +no thing +gu i +st in +blo g +fe st +b un +le e +war ds +ch ance +dre ss +re n +pau l +p es +tech no +ru ssi +c ard +e ast +mar i +w ine +t i +la w +str ic +k i +ap e +au gu +pro fe +as h +cour se +ma il +ren tly +d un +m un +lo ve +is land +dri ve +s l +end ed +ma in +lo st +nat ure +âĿ¤ ï¸ı +ch ic +re por +p in +pr o +st ation +ce p +ta kes +compan y +go es +on d +ma ch +ra dio +d ad +ro ck +j a +p ay +champi on +e e +in de +tt a +ati c +t ab +beli eve +ener gy +z i +t at +wor d +on ce +re sul +y l +and re +an o +inst agram +clo se +t am +cu stom +w a +con om +sho ws +li fe +k in +ro b +t age +n ation +al most +list en +sa ve +re li +ac e +mar y +tre e +for get +j ack +wa iting +direc tor +h ill +bor n +te mp +f l +st e +on a +sing le +wedne sday +un ited +in o +@ _ +ne l +celebr ate +en ding +de al +j i +can ada +hu ge +tr ack +âĢ ¢ +f y +fan ta +an g +yor k +rele ase +p un +ep iso +wor ds +t our +p ack +i gh +classi c +perfor mance +ke t +after noon +recor d +win s +pro ble +âĿ ¤ +f our +b ed +ban k +d ance +s la +cal led +mi ght +a p +pa st +ðŁ ļ +diffe rent +it e +gi ft +ssi ve +chur ch +c us +pro gram +ho tel +ic e +ma d +secur ity +en ge +d c +en ough +st a +e ty +de ad +g un +he ar +m ir +hu man +gre ss +oun ds +pi ece +bre aking +gar den +fi ght +vie ws +f ish +star ted +run ning +gre en +ser i +s m +as k +d or +de ath +e conom +er i +ir d +s er +l unch +âģ ¦ +bo x +nat u +ba se +b an +f al +glo bal +wil d +wo w +out side +mo ve +le ad +an al +muse um +on g +ha w +pow er +than k +b ac +char ac +cam pa +dig ital +r o +op er +de v +w ol +p ati +f a +m ale +pap er +ill ing +c s +â ĥ +educ ation +ta ken +e ffe +m ou +s ad +" . +bas ed +staf f +inclu ding +li ving +a c +ch ina +mo b +stor m +lu ck +ph il +o o +y n +tra vel +k el +ti al +pr ice +boo k +import ant +bi o +p ool +ny c +f ab +lo ad +? ! +chall enge +cr y +ser ve +we ar +bu s +ta in +nu mber +ro r +k at +i z +th ough +ho sp +m m +fa ir +ut es +ho t +po p +fi ed +cam p +develop ment +li br +c ali +em s +âģ¦ @ +b ol +is ed +stand ing +mo del +it a +g le +bro wn +ima ge +ve red +for ce +o il +par tic +sh u +da ily +la w +se c +cla ss +cam p +holi day +cl in +k ers +pres ent +gam e +incre di +er ship +inter view +b ill +du e +and y +ab o +in nov +ke y +ac ade +p il +mo der +st ars +br and +f er +wee ks +con si +pr e +sa fe +wr it +di um +la unch +marke ting +ann ual +as si +cour t +la dy +c ted +and a +in side +chil d +opp or +sm ith +centr e +gu e +âģ © +f ren +st y +for t +ent ly +is n +ke ep +to ber +on y +bo y +al d +col la +de mo +le vel +com pet +ad o +b our +fanta stic +m ate +s u +sou th +oppor tun +vers ary +lat er +bu d +face book +la un +ster n +p it +! " +ma j +gr am +tb t +fi re +happ y +a ks +wh ole +actu ally +ill er +ell a +lo ts +al ex +an ge +lan ds +ðŁĺ Ń +en ter +r ou +episo de +p ed +in ten +sh ire +wh o +pl an +h o +ca ke +we st +mag az +fre sh +c c +n ar +ch ris +wr iting +w er +n om +l o +mi dd +dre am +o l +ti onal +de b +> > +be come +s i +gr and +all ing +hi stor +ri de +i red +saf e +que en +ci l +in tro +vi l +d ani +.. . +ar tic +st at +sh ort +or ing +sel fi +mis si +do c +b it +g all +b om +i re +se lec +d ition +ðŁĶ ¥ +fri end +be at +gh ting +ðŁĺ Ĭ +pe ace +ex hi +ant a +ab ility +il lu +j on +qu ality +tri bu +m es +play ers +fa ir +cu t +c ab +suc cess +b i +su s +pro mo +sch e +an ge +ic o +comm it +cat ch +ill a +kin d +feel ing +qu o +s ay +anni versary +spo t +mo ther +an e +p end +your self +op s +app le +min utes +p o +gr and +ri es +ha ha +care er +ed ition +de c +ric k +am i +concer t +iti ve +ge ous +d ly +t te +adv ent +i g +li ghts +ak er +sk y +âĥ £ +r ay +fini shed +w ay +s d +ac coun +ðŁĴ ķ +ck y +ch el +lit er +pain ting +lo s +st un +techno logy +n as +ma r +b il +afric a +ki e +ey es +gol f +plu s +ni a +it ec +serv ices +wed ding +kno wn +te le +.. ... +star ts +pa ren +w ants +ati onal +mon ths +win do +fav our +er t +magaz ine +ex clu +re ve +b c +origin al +e ss +n al +an ti +st ro +t ice +stu dy +à ¤ +v ac +nation al +fi ve +ra in +ve ment +u te +ver se +em er +ar my +possi ble +gue ss +val ley +ther n +cro w +m r +col or +on to +pic k +cle ar +dar k +t ac +wan ted +it ting +can cer +govern ment +di e +ri se +z ing +col d +f oun +stu dio +str ation +bro ther +a head +sh el +mic ro +ic ally +d au +sig ned +vi ol +a x +as se +i o +w re +spl ay +ch ick +augu st +pl at +ti ps +sp i +hu man +e asy +lo gi +mi ke +gro w +ag re +w w +sh ad +mo tiv +wi de +tur ns +om g +v ar +de fin +su g +j im +ðŁĶ ¥ +t d +campa ign +nam ed +re tweet +co p +t v +le av +k is +dou ble +s mar +issu e +vil la +in formation +li es +sto ck +n t +di stric +sh or +mi x +er o +se p +me x +see ing +li ve +re min +co de +g ur +s c +wil d +l un +h ood +spo t +fa ther +fore ver +up d +tra f +f ly +ne ed +gra du +tra in +ma ke +s ab +be y +si ze +lead er +tal ks +e u +lo g +fo x +gor geous +le ss +le ts +sur pri +my self +no te +li ves +f ru +lo ved +se ver +de m +j i +so c +h old +do gs +n i +â ŀ +lea ve +air port +ben ef +ex pl +shi ps +comple te +ach i +gre at +vin tage +j ack +ro c +woo d +pri v +off er +ey e +ver sion +te a +co ach +off ic +w ell +g en +s at +h h +you th +o x +? " +m t +mi x +g g +d le +natu ral +buil d +break fast +thin king +theat re +mo on +ber g +go als +geor ge +en e +exc ell +il ing +tun e +y ed +g ate +m it +net work +jo e +h ello +f b +tu be +we aring +ath le +stru c +har d +gla ss +g ers +thro w +g es +b t +indu stry +manag ement +ali st +go al +stre am +y el +a vi +ici ous +o thers +s ki +chri sti +bir d +e sc +m in +tr o +l t +j an +im p +ri ghts +sh a +or gan +cent ral +ar a +ro ll +favour ite +che ster +el se +p ay +car s +m ine +ste p +prac tice +maj or +h ang +ðŁĺ ĺ +n on +v ari +eng ine +vol un +di a +i led +arch itec +p ink +d s +th y +wa sh +web site +ba g +contro l +el li +f ra +an sw +d ence +y u +r on +ol a +g in +dr in +li c +cou ple +sp ar +g on +cre ate +c t +celebr ating +de ep +e at +te e +vo ice +dro p +vis it +at ors +sta dium +f t +w is +ro l +gra de +fam il +po ints +re pre +w as +traf fic +jap an +or g +hon or +tex as +man u +âĻ ¥ +safe ty +re r +b ag +em plo +rele ased +re gu +ak a +n av +ro le +sen ior +spec t +cro ss +lin es +be st +p ack +s in +ti e +mis sing +sun set +li ber +is ing +j ay +sk i +champion ship +ac tiv +la dies +play ed +y y +pu bl +al o +pri de +s r +pa ki +lu x +sur vi +ck ed +e ts +cho col +austr alia +par is +mi les +h at +ment al +al a +me an +mob ile +en a +in si +f ound +chi ef +t ag +incredi ble +re turn +à © +goo gle +fren ch +cre w +hal lo +ali an +j az +ch er +sil ver +nor th +eng lish +base ball +c af +lim ited +follow ing +app reci +ear th +k ir +ve mber +w ed +p tion +g ed +oc tober +fl ori +c r +en cy +ga ve +lor d +stu ff +ber ry +po st +sm ile +bro ad +st ate +gg er +me ans +ic y +gu n +y o +ma ster +bur g +han ds +ni e +/ / +uni on +brit ish +big gest +distric t +am ing +h il +o ce +per son +pas s +en vir +scho ols +arri ved +anc es +insp ired +ex pla +be n +libr ary +bo tt +am p +ste ph +cont act +b ang +m s +cali for +t old +batt le +b b +chic ago +âľ ¨ +str ate +sh i +de ce +- ) +ad d +la b +j ones +leg end +cast le +ing er +st ance +be l +ur a +re fu +lead ers +po t +se x +h ic +artic le +ki d +fr ance +x x +ex e +gui de +volun te +pr int +al i +ce o +twee ts +w x +scen e +vol u +ant i +h an +as soci +shar ing +ro se +mini ster +sh er +in ste +cle an +demo cr +po ster +sk in +p sy +pro per +cra zy +i am +o re +in i +any thing +po d +mo ving +cl ick +ex plo +com b +cra ft +f i +bloo d +is ra +publ ic +d ent +ol ym +eng land +a si +ch er +fac t +envir on +har ry +g one +me dic +enjo ying +just ice +j r +indi an +wi fe +s ound +t es +dra wing +p al +ide a +cr it +ju li +il er +war m +cl ar +thou ghts +def en +coun cil +intro duc +di ed +jan u +an i +s end +li er +m l +intere sting +tra de +win d +b ay +s ac +anc y +sour ce +b es +org ani +ar ly +lar ge +ff ici +ta g +u t +de sp +o es +tit le +sy m +pic tures +op en +wom en +sho wing +ri a +le ast +lead ership +cur rent +elec tr +val ent +list ening +c key +gener al +de ser +du ce +; ) +c ent +ðŁĺį ðŁĺį +sco tt +po or +selfi e +ev ents +i on +wr ong +de v +h ill +sep te +cul ture +l ine +sor ry +s ent +si ster +ce pt +k ri +no vember +ar i +announ ce +z ation +br an +g ent +d u +l en +per s +f m +mart in +o p +e mb +om e +midd le +suc cess +pe ter +janu ary +f lu +rac ing +d av +bi ke +ðŁı » +pe t +shoo t +profe ssi +feat uring +septe mber +now playing +sta ur +z a +on ic +qu ick +bas ke +spe aking +mil it +z er +chick en +b ell +s ad +co ast +lo ving +y ers +d j +pan el +ver age +s wit +ic ks +b ou +califor nia +s am +paren ts +er o +k illed +ph ys +jo bs +mi gr +an th +e mo +hallo ween +and er +c m +compet ition +e ag +s ket +sp ir +may be +exclu sive +app e +jour ney +scre en +for d +i o +h ate +u g +sou l +her o +soci ety +sy n +gu it +n h +d j +as es +im pre +ti me +sal es +d d +f ts +summ it +stun ning +om s +tur ned +cle an +sof t +be at +re staur +de red +en ces +ma gic +di o +sh ine +gu est +health y +exhi b +stor ies +po pu +n is +el a +bel ow +fun ny +resul ts +s ne +cur rently +ar d +down load +f light +m al +f ine +p ad +ch u +ent ed +h at +ðŁij ı +ste ve +j o +mar k +r at +b all +p c +p on +b by +o li +ar ts +as ure +bow l +att ack +mi c +de ar +ran ge +en ter +chocol ate +br illi +ac cess +, " +? ?? +ch ap +con st +t n +mat ter +blu e +gall ery +em p +work shop +lead ing +y ours +baske tball +w anna +th u +_ _ +mar ri +sle ep +bi a +ch e +ma d +imp act +o wn +si r +chan nel +euro pe +e sp +k itch +hosp ital +w ra +roy al +f s +ne u +qu ar +ne y +ac ks +ch ase +pp y +st al +at ely +ti m +dece mber +r are +per form +cre am +we ight +ch oo +ni ght +ha ven +fr anc +kh an +buil t +hel ping +tru st +ty pe +gol den +ta x +s now +s wi +di sa +questi ons +ve y +li ght +c n +cl oud +thom as +ag ed +sh ou +te ams +gr an +re ason +a a +you tube +v p +pi zz +manag er +bur y +cre dit +tre at +ma x +i k +ma in +g ing +de ad +pro bab +ye ah +ã Ĥ +br and +so li +pl ant +ta yl +gir l +ðŁĺ Ń +nam ent +au to +mess age +ko re +n ur +ter r +ag u +ma p +sen ting +lo ves +gi ves +g ab +z en +ro bert +con fir +w ars +o m +sta in +cam era +and er +won der +a b +ca p +s old +su it +wal king +contin ue +effe c +dau ghter +d anc +cha in +mul ti +ki d +y an +champi on +v o +ta ins +ho st +min i +mis sed +re sc +ly n +fin ish +del icious +s as +tayl or +i b +pro mis +produc ts +moun tain +flori da +regi ster +tre at +rec ent +fe male +boo th +mat t +ve hic +s op +mo tor +suppor ting +phi c +ex tre +dr ink +lan e +th ird +p s +con stru +ce re +far m +ðŁİ ī +tu red +ðŁij ī +c ats +a j +gi e +shoo ting +as ked +paki stan +am e +m b +g il +leg al +squ are +in vol +dra w +oo oo +!! !! +opportun ity +p y +e i +b ts +teach er +charac ter +john son +br on +ly wood +ch ine +c ing +c ine +d ge +gam ing +russi a +ci a +quo te +ric h +go v +flow ers +sp iri +st in +grow th +ðŁı ¼ +comm er +j uni +mu m +r an +s na +a ren +c b +ac tor +col or +si t +pa ir +ch i +bo w +acade my +hel d +r ang +me tal +y l +ac tive +probab ly +t ch +need ed +spe e +cho ice +ital y +ry an +ðŁĩ º +flow er +v it +m n +found ation +b ak +si ons +ne igh +f loo +he ard +re mo +fre sh +ing ing +re f +to wn +cl ou +je sus +spiri t +cou ldn +z es +ðŁĴ Ļ +willi ams +pro ce +moder n +pro cess +sho es +cre ated +tri c +issu es +ann e +att en +de but +h r +n it +sti g +a po +e ps +z u +ã Ģ +si x +car ds +lan gu +fam ous +tour nament +se l +e bay +y n +st on +k ick +announ ced +k am +vo c +brilli ant +hou se +che ese +war ri +mus ic +ho ckey +ðŁĺĤ ðŁĺĤ +sk ills +au tom +smar t +med ical +mon y +e x +gu ar +gi ve +pers onal +ven tion +al li +pre ss +flo or +m c +victor y +hi m +simp le +th or +ðŁĩº ðŁĩ +ta il +lu cky +ale x +qu ite +bo t +ssi ons +chall eng +c ann +amaz on +h ell +b ought +) : +ed y +secre t +produc tion +inde pend +de fe +ad ded +p r +p ag +be d +gre atest +with in +j ay +ðŁ ¥ +ire land +re ly +s d +te xt +dri ving +pro gram +spe ed +col um +str on +à © +fore st +â ĸ +mach ine +co in +sc ar +oun t +bi e +¡ ï¸ı +por tra +comm on +wre st +recei ved +kno w +inve st +pl ans +ac cor +ad op +ter y +re ali +p p +k al +art work +me an +go d +inste ad +an ci +motiv ation +as ing +inspir ation +up coming +polit ical +euro pe +m ers +heav y +ðŁij į +fe bru +scot land +ou gh +b t +bo ss +sche du +spe ak +n ick +u red +in o +e k +ri sk +tor y +pres ents +b on +ru g +st ates +exhib ition +il o +m ill +br ought +: -) +tou ri +com e +offici ally +champi ons +do ors +re p +po se +ex tra +k ings +soc cer +squ ad +app lic +at a +some times +t ari +excell ent +ðŁĺ ĺ +stra ight +car ol +ri p +âĢ į +gra phic +m ol +elec tion +febru ary +as ons +l i +di r +m t +n ick +u su +m rs +com ics +inst itu +cor por +v i +ðŁĻ ı +tu ral +di se +ac ci +we are +am ong +sho pping +t ill +wh at +cha ir +sp an +chine se +innov ation +jo y +k it +cent ury +ob ama +ph ili +f c +re ach +c iti +ul ous +n on +d ang +happ ening +bur n +p el +or ange +d v +k ick +cla im +ing ham +ph y +no v +pod cast +wh i +ni ghts +ear lier +be ar +la h +exc iting +or a +gi ven +s lo +memor ies +contin ues +produc t +gh o +c d +kno ws +ðŁİ ī +publi shed +discu ss +y ard +i phone +tri es +w all +fe b +are n +tru th +win ners +tu re +diti onal +milit ary +proble m +m and +do g +lo ss +c ric +can adi +ve ter +villa ge +" , +y r +un g +don ald +ag ing +bir ds +sci enti +le s +th is +regi on +tic al +itt en +il a +ðŁĺ İ +d ad +di am +abo ve +st ren +li t +p ir +la b +fo cus +bus y +d ur +app ly +s ma +auth or +ac i +exe cu +dom in +re la +jack son +at o +wash ington +ðŁĻ Į +k ill +popu lar +ce ment +ro ad +e ating +loc ation +v ent +ar re +n an +cu sto +advent ure +or din +spor t +ul t +lo ck +questi on +dri ver +land sc +on i +k ins +p d +jor dan +te red +k k +a f +chil d +s p +just in +en i +s elling +z o +wh it +bo ston +partic ip +sig ning +happ ened +he at +m am +dre ams +lo ws +gra ph +the day +head ing +br o +ble ssed +vi c +ve gas +h d +in ning +ro man +and ro +den ti +u se +c it +pro gress +writ er +bo b +ff s +gro wing +b ly +aw are +ex am +sp ent +be t +sc ore +bey ond +do cu +ad el +s f +cou ra +colla bor +in c +priv ate +bo at +* * +z one +p ha +b ill +to tal +plan ning +to wards +plac es +pre view +cre ative +dam n +ide as +se ems +po ten +say ing +di splay +s w +a qu +lou is +by e +li l +e mail +we stern +ger many +ell er +re s +f ant +ment ary +de als +ric hard +jer sey +stren g +ra d +pizz a +mon d +w are +l ac +g i +ar chi +c d +yel low +rec ently +re ach +à ¹ +kitch en +desig ned +tr y +g al +restaur ant +at ure +w w +j as +l ma +ðŁij Į +pa in +av o +min ute +sch ol +ther ap +tic ket +d ry +jap an +diti ons +ter ri +sel ves +happ en +t up +ma g +cop y +sh er +free dom +f ile +speci ally +tor onto +lo ad +g ary +re y +answ er +lo y +cau ght +pri ze +u ne +fic ation +ni ger +sy d +tou ch +feat ure +jaz z +recor ds +him self +di sh +ro ber +spot ted +ma ster +wa ve +fin als +bu ll +for um +al d +re comm +ch a +a e +d oo +inst ru +tru ly +l g +in k +bro thers +de st +j im +m it +clo sed +is on +tri ed +s anta +af fe +w an +hor se +g row +camp us +rel ation +nati ve +jour n +go v +o ct +k it +b ound +part ner +re ma +crow d +! ) +c alls +ra il +qu ali +solu tion +con test +con vers +sn ap +b ase +in iti +ta x +y e +ent repre +it or +constru ction +foo d +present ed +n ings +cli mate +k m +mo del +b j +blo ck +present ation +dre am +fi x +c alling +bus ine +con gress +under stand +we b +val ue +ï¸ı âĥ£ +mex ico +it ely +ki m +char ity +ref lec +bl an +fl ying +anal y +famil ies +b and +reci pe +celebr ation +ac cep +ar y +to t +g b +intere sted +cap tain +âĻ ¥ +ti p +ab sol +bra z +inve stig +o logy +de c +tru ck +ver ing +c lear +don t +go tta +ad vis +beg ins +ma ss +de scri +blo ck +k im +davi d +son gs +memor ial +feat ures +su stain +' . +gra b +jo se +v a +con serv +se ts +man chester +fi ghting +de gre +ag a +in d +sle ep +pos ition +ha ir +sig ns +pol icy +it o +al ert +st am +sp end +w y +absol ut +d m +anim al +my ster +success ful +proble ms +ro bo +k ay +gar den +p d +may or +d ale +t ol +off ers +vis iting +friend ly +tre es +offic er +accoun t +ke vin +ðŁij į +gi ant +contin u +con su +tr act +n fl +ðŁĺ Ĭ +h q +b ility +a ar +dis ney +te en +on ed +wh ite +tra iler +de dic +al one +absolut ely +dig ital +willi am +in ation +s wa +e e +enti re +ger man +ro ll +h its +co st +st ay +th a +ali ve +accor ding +co t +liter ally +her it +re ti +haha ha +exper i +li kes +g t +ste el +__ __ +ch air +christi an +to wer +diffe rence +m d +tre ss +mi d +prin ce +afric an +fe der +foo t +car ri +ser ved +r ice +sh all +feat ured +ck er +rec ru +po e +sen se +ni fic +com edy +cont ent +f at +po sted +con tribu +tim ate +li ver +mb le +inter net +ag e +europe an +cl ing +gla d +ff ic +sc o +ak es +el le +ter min +ton y +p ale +col our +seri ous +pat ri +movi es +b m +professi onal +ad o +al u +br inging +f alls +isra el +ter m +langu age +bro ok +man n +commun ic +can not +ac ti +p he +y an +entrepre ne +tur key +log ical +lon g +ar m +ur s +work ers +ing ly +gg s +ri c +tu al +recei ve +op ens +ge ar +soci al +fe et +c king +ad ver +fin an +fe els +sp la +h r +ea ster +bra in +ã ģ +fi g +le dge +ne arly +prote ct +ma ssive +e th +aw a +ðŁĺ ģ +y rs +aware ness +defin itely +k n +imag ine +k u +syste ms +ðŁij ı +f as +li k +provi de +am o +disco ver +inf lu +ma ker +g az +fit ness +stre et +er s +te d +w c +ys is +pos itive +hel ped +que st +andre w +bra d +b in +hang ing +l ing +bri ght +se ction +ma ss +ðŁĻ Į +follow ers +ho sting +tem por +fla g +a ve +let ter +k ur +re qui +of ten +cry p +su ff +âļ ½ +russi an +treat ment +al le +ha y +l an +keep ing +hol y +power ful +pre dic +fun d +e specially +windo w +je wel +il y +ðŁĴ ľ +gener ation +app a +seri ously +o d +ðŁĺĤðŁĺĤ ðŁĺĤ +cer ti +iri sh +ðŁij Į +mi ami +be th +v ity +se cu +che f +cri me +graph y +ma x +arti sts +re volu +gu ard +spee ch +u c +upd ates +fac es +st ant +chang ed +repor ts +low er +pe ar +n c +k il +loo ked +spe aker +s f +re spect +ok ay +oce an +s itting +architec ture +tra il +se at +i ra +le g +japan ese +d am +u lar +sw im +polit ics +finan cial +ol d +mou th +at temp +de stin +fi shing +atten tion +me m +chang es +deci ded +reli gi +g in +c av +z z +ad am +ma c +wr ite +beg in +sc ul +al ter +is s +ath on +imag es +m oo +jo ined +ðŁĺ ī +âŀ ¡ï¸ı +pas sed +mu sli +h ir +lar gest +cam er +com ic +gh ted +rug by +bur gh +gg ing +te sting +pre par +lau gh +al ed +impro ve +beli ev +adv ice +sha res +he art +tur ning +s b +t el +caf e +n es +dani el +pat ter +t z +se tt +par k +c and +st ick +happ ens +bri an +ne west +e pic +ad or +ki es +war ning +anim als +custo m +ar c +di an +gol d +cor e +t f +c ity +pan ts +re ality +con fi +in ju +fo x +gu il +k new +âĺ º +cor rec +itu de +d den +. # +re duc +pas s +f on +y a +ow ner +re turns +n c +e ast +ap ol +in sur +th o +si m +juni or +be e +ang el +att le +elec tric +hor ror +cra sh +e ye +pat h +sou thern +emplo ye +ge o +t an +ha z +r ally +ðŁı » +proper ty +was n +enjo yed +gre y +g as +bre w +nor thern +hol ding +g p +ta ke +ch art +ly n +dr ama +z o +pa id +throw back +cu p +discu ssion +down town +w ill +le w +b is +t ary +bre ad +up on +r ate +teach ers +it ation +anc ed +cy cle +choo se +d c +ir an +co w +da ve +ra ise +prin cess +fa ith +- > +indu stri +sp ain +guit ar +fac ts +m n +sp en +cour te +go tt +projec ts +au di +o sc +pe ter +s and +intere st +happ iness +ven ue +sol di +surpri se +poten tial +per io +custom er +i i +g ni +manu fac +e co +bro ken +sing er +vel s +wal es +hu s +in j +f our +tal ent +d ying +mat the +fil m +jo ining +s ell +j ar +lma o +sur ger +bb c +sour ces +au stin +ni k +char les +f am +prin ci +ange l +cas h +lo t +o red +pla ys +pl ate +don e +memor y +br ings +n ba +solu tions +teach ing +gr ace +cir cu +hel ps +foun der +mar y +expl ore +de cor +par ts +ch o +inte gr +ha u +is es +pu tting +in er +r it +v y +mic hel +blu es +every day +for ms +bi o +ye ar +p in +t ter +spr ing +) ) +po t +al ing +perform ing +sh an +plan et +mus ical +head s +it alian +stru gg +âĢį âĻ +w ings +pu mp +h h +tr ou +a id +pri me +ear th +pa int +mon t +am y +bb c +fab ulous +fru it +andro id +bour ne +cere mony +enti al +? ? +deb ate +on ing +dra ft +sol ar +t x +j am +cor n +!! !!! +bro o +mil k +po sed +o hi +mo vement +b ren +part ner +p g +et te +ar ies +sh out +n g +leav ing +t ells +sen s +ta ste +kel ly +wor l +gy m +ric h +e gy +pi d +ma s +â Ĥ +courte sy +fran k +incre ase +wr itten +pp ers +re l +ha i +s as +s ound +tt i +w ich +ri ver +.. ." +a g +fel low +ro me +sm all +gen cy +ic an +lux ury +pro of +me t +wild life +mom ents +ra ther +cor ner +com pe +canadi an +lik ely +therap y +li am +econom ic +indi e +rou te +fi ght +ho pe +se tting +ant ly +cro ss +fant asy +de e +sket ch +comp li +ym i +ru les +engine ering +fig ure +ro w +. , +f w +syd ney +w ou +t ation +dre w +us es +the re +sp read +struc ture +pat rick +appa rently +ro s +h ills +w we +ann y +com mission +di v +f ying +con sul +anal ysis +ex i +ten nis +vehic le +ðŁĺŃ ðŁĺŃ +as s +high ly +op ened +b ann +ðŁĴ Ļ +mp h +wi shing +v or +fi f +give away +r r +ra y +je ss +g at +ic ymi +x it +high est +yor k +pi e +invol ved +high er +ri e +mal ay +int elli +desp ite +che e +sar ah +be an +reco gni +ar sen +tal ented +pas sion +ic h +ab c +lead s +dise ase +v is +se c +pre senting +m illi +hol e +sho ts +de part +surger y +gov t +b in +du al +e vi +lon ger +ev ol +scre en +portra it +et c +lo se +ch at +p en +p i +om a +s ick +er c +compan ies +en try +plan e +gr y +ven e +liver pool +premi ere +sha red +a red +fil ms +ir a +holi days +cric ket +ici an +v ing +. ) +ul timate +di vision +con duc +se pt +for ces +mon t +s mart +disa pp +sun shine +in d +b less +ma de +col ors +fran k +ir on +bott le +s go +m ood +j ason +er ic +bir th +te en +respon se +tar get +state ment +fe ar +th el +al um +ar ab +bl in +direc tion +ste ps +er ial +wor ked +at l +ðŁĴ ķ +fel t +pol i +scen es +hom es +b ell +e at +ate ful +t in +l ace +fol ks +p se +an n +wis dom +fa v +but ter +s r +are as +sm oo +bi z +dg es +app o +mo re +the m +effe ct +windo ws +sun ny +cap ital +tot ally +c ities +gr ant +mb ers +s low +au tu +il ities +w ro +ri sing +st ics +viol ence +i gh +qu ot +h it +t c +herit age +bu ff +ne s +z ar +den tial +ex ac +ed ge +de ep +aren a +be came +benef its +mar ks +mb er +a z +am es +pre ci +dra gon +re g +d ings +do s +ðŁĴ ª +n el +s ity +me al +di st +leg end +pur chase +pic al +st ick +f at +du ba +profe ss +car to +pro f +coun tries +respon si +se qu +fa b +tribu te +hon ored +prac tic +pur ple +an ton +pa red +t ough +summ er +environ ment +s ons +ðŁĻ ı +m ps +gi es +her oes +t elling +hen ry +f en +know ledge +Ģ ï¸ı +f r +ne g +u re +ac king +hear ts +s oo +hol lywood +ju mp +sau ce +schedu le +tur n +yo ga +cre ating +c ket +cre ek +â Ń +custom ers +ma dri +gu l +asse mb +moun t +c ell +to p +st al +dav is +t wi +sig n +premi er +iti ons +he aring +un k +pati ents +app ear +heav en +al ty +doc tor +a e +plat form +je ff +ðŁĵ · +regi onal +bi d +box ing +ex ten +or ity +a w +w ise +il le +sever al +bi e +s itu +sy ria +âľ ħ +remin der +enter tain +li on +part ners +in n +ph ar +f au +pl s +expe cted +sug ar +deci sion +s b +ch ron +associ ation +leav es +vis ited +sh ap +ðŁĴ ĸ +fur ther +h ann +w i +run s +l er +fun ding +fil led +.. .... +tin y +han g +or g +co ol +se min +ðŁı Ĩ +spon s +nav y +sa int +dru g +d al +r oun +co vered +tra ditional +invest ment +de te +al ism +f low +n is +sun rise +fe at +f ted +we ird +je re +ve gan +medic ine +an o +ac cu +deli very +temp le +chang ing +wil son +phili pp +re fe +n d +is er +g ay +r and +ati ves +t ely +p and +intelli g +g are +am bas +de mon +commit tee +strate gy +refu ge +bud get +prote c +pi er +ex press +nom in +econom y +al low +ic on +gal ax +o h +indi vi +dem and +vir gin +lu ke +ali sts +man i +s mi +ju dge +ent y +mic hi +resul t +am ed +spe aks +' , +hou ston +sh in +b ing +fl y +ch em +au to +v as +ge t +ar m +thank s +d in +gan g +x x +si on +loc ated +p l +jo sh +in fo +jo ins +adver ti +ot d +el d +si e +re asons +v ent +ðŁĩºðŁĩ ¸ +â ł +convers ation +stu di +ðŁĶ¥ ðŁĶ¥ +go s +s ounds +un it +mu sc +ge l +ack ed +pac i +co s +de re +u u +a o +la m +inspir ing +ar ms +tw are +mat ters +ad dic +du de +ex t +cri sis +b ath +me et +sing h +expe ct +del hi +resc ue +wor st +au g +shi pping +ser ving +st o +dar k +ac es +histor ic +landsc ape +desig ner +b illion +gr ateful +wa ke +e ve +m iller +hou sing +dy nam +is co +be ha +sh op +pr ou +e as +a sia +e ding +k on +depart ment +aw ar +mar ine +in ci +photograph er +ta pe +lo go +r ings +d it +-- -- +vin yl +w c +vo ting +se ven +ambas sad +dal las +t u +com ment +k ra +b les +w ag +u d +au dio +stri ke +offici al +o ts +me tho +to ols +ra di +al an +hun t +wat ched +a ke +fa ke +drin king +mer ry +m l +b day +ri o +ni ke +c ant +re pe +co stu +mur der +ak ers +ch ers +ou ts +beg inning +so s +ad es +n in +not es +wro te +sol o +c i +li ghting +ur ban +bre xit +att end +shir ts +pla yo +ac tress +pl ic +stand ard +quot es +par ade +anci ent + © +tur ing +re e +pri mary +fla sh +citi z +mat es +ste in +z i +clin ton +sk in +gen e +hu m +g ar +t le +y i +fo cu +de an +pl ants +cy ber +b u +om e +ho p +ad dress +ti x +gi fts +relation ship +sub scri +fe ed +exac tly +haw ks +ex o +stre ss +s n +arre sted +an e +sof tware +z ero +the me +mu mb +im migr +mi a +make up +ple asure +uni vers +har b +eng ine +ap er +r in +br a +institu te +le ather +al th +sing ing +co s +gh ty +me as +st ic +si de +insur ance +co t +pit ch +moun tains +cri min +su pre +valent ine +at er +wou ldn +sc ale +rel ated +re gar +star tup +pack ed +mi ke +week ly +p ts +coun t +ha r +gott en +min d +ber lin +con ditions +swit ch +cor n +sa ve +g li +emer gency +tun ed +sto ck +discu ssing +every body +s day +whe ther +wrest ling +ec es +gen der +ch en +ðŁij Ģ +madri d +mar athon +e gg +i er +th x +as king +kore a +wol f +ay a +g m +g au +at ory +v r +gra ss +k illing +b ble +ur o +un i +e th +sh ore +th en +re ale +bot tom +ex erc +k ar +or ies +ad ri +san ds +se x +. ' +volunte ers +per form +par liam +inclu de +deli ghted +execu tive +fu el +kis s +ã ħ +char ge +h u +ca kes +ve t +g lu +agre e +pr ices +n au +h l +g ru +ra j +streng th +b ic +sp ending +al es +av en +b last +: ( +yo f +nor mal +si x +qu ick +se a +d aw +mee ts +lo vers +upd ated +po tat +comple ted +coo k +opportun ities +p ure +organ ic +tem per +c am +avo id +par king +duba i +and o +di stri +to y +comple tely +don ald +tri al +bas s +b oun +back ground +v as +mar vel +lu m +ru s +t ool +com missi +throw back +fin ding +is lam +! ? +st op +e vil +or al +resi dents +i denti +o ak +ðŁİ ¶ +l il +span ish +chap ter +sto pped +direc t +ho sted +pic ked +lab our +lew is +defen se +à ® +health care +wh is +mat h +pe ak +ra ised +fi x +bu ll +th ir +chel sea +fol k +tr e +can di +pau l +ei ther +ad am +poe try +jewel ry +ðŁ ¦ +pr ay +Ø § +g c +o z +wi shes +fore ign +sun g +lear ned +en e +n ing +micha el +illu stration +legend ary +w av +b au +ðŁļ ¨ +cal end +stre ets +â Ĩ +mon ster +bu ck +g r +scho ol +ba th +wa ste +ne ck +ha wa +be ach +re plac +jec t +on er +fac tory +coun t +ðŁĵ ¸ +mor gan +der ing +se an +steph en +de p +no vel +vide os +ic al +press ure +arsen al +ex pre +ir s +tren ding +ss a +fla sh +re sear +thr ough +profess or +scul p +to s +gg ed +mm a +be e +a pe +hun ter +am i +he i +pla stic +bu cks +uni verse +le gen +niger ia +ple ased +ri s +thin ks +autu mn +i ds +d is +anth ony +ðŁı ½ +ak ed +gla sses +fin ance +z er +k as +con tract +nu mbers +sh aw +partner ship +t il +laun ched +s al +victor ia +theat er +usu al +nam es +perio d +eli za +i th +bar cel +ro cks +bag s +mat e +distri bu +j on +di ffic +ali zed +cur ren +sco red +b ha +du blin +ro se +in ted +soli d +beha vi +wal ker +simp ly +garden s +head ed +in i +ohi o +we ap +f o +gl en +e state +ran dom +th under +thr u +k ill +jac ket +it i +entertain ment +thanks giving +ent al +en coura +el o +a ther +tan k +high lights +f ting +ru le +model s +bor der +bj p +hus band +in done +ken ya +be ars +al o +n inten +pi x +str o +or ders +sal ad +ro ads +n or +l ation +sop hi +ðŁı ¼ +pi eces +b one +min s +inclu des +nu tr +phi l +s ent +fun dra +ga in +bor ough +n ad +mon day +activ ity +it ems +be coming +ken ne +de tro +car di +gue sts +u x +world wide +sever e +new s +thank ful +fic tion +ve ge +m all +si an +er al +inj ury +le e +men u +danc ing +scot ti +exam ple +( # +na i +studi os +ba i +ðŁĴ Ľ +j av +diam ond +vin ce +ric k +prote ction +lin col +cham ps +appro ach +d ar +m ile +clou ds +je ff +in fin +l ers +p les +pe ace +go p +âĻ ¡ +tech n +str a +a verage +ef fort +introduc ing +di versity +austr alian +am p +boo st +s ke +pati ent +appreci ate +ici ans +pu r +f ell +woo ds +illu str +ðŁ ĸ +ag ency +ac tions +brit ain +under way +se attle +el and +ag o +f ill +stre aming +pro test +challeng es +ky o +et sy +coo king +exper t +ru ss +rain bow +commer cial +sp in +be ats +c ry +val u +el i +th row +gr ams +le vels +michi gan +c ad +ador able +const itu +w s +pu b +mid night +th at +net fli +braz il +die go +regu lar +jo y +âĤ ¬ +li qu +ea stern +k ni +fl at +n p +bro wn +w er +se y +tt ers +ac ting +v anc +cy cling +program me +ra w +comple x +tat too +throwback thursday +se ssions +ro oms +si ght +speci es +bom b +lau gh +ke eps +mo on +offic ers +con ver +t r +ha sh +t ack +ri ous +ad ap +a j +reco gn +ex po +sug ge +confir med +rol ling +dre ssing +ic t +fri day +ph ones +ri dge +con cept +ro y +ke ys +ef for +c ate +k ne +ev en +l ay +commun ities +mo d +n az +every where +al ab +bit coin +ban ks +out door +feder al +sto res +h p +c al +m ely +sig nific +be ar +re public +clo ser +al lah +pic k +x d +pal ace +ch ill +b am +er ous +un a +al len +out standing +olym pic +supp ly +fi gu +v au +l p +char lie +un es +> >> +legen ds +ici al +co ast +benef it +mul ti +f its +far mers +am ount +si sters +har ve +hon ey +que en +b ers +pl ann +âŃ IJ +m u +barcel ona +al ber +stat us +re main +ex tra +c andy +vi ous +âľ Į +o v +warri ors +-- > +ju mp +am ar +x mas +stu dies +i ors +k or +don ate +pre p +fi sh +im a +pain ted +ad mini +co splay +spor ts +dro ps +fi ghter +evi dence +ðŁĴ ª +la ke +ro b +cine ma +pro file +à ± +stan ds +leg acy +sh ape +ro of +ci vil +i ans +sy l +sh am +vo ted +re tail +ph illi +li sted +du ty +n b +th es +f are +au ction +ffici al +stor ms +d p +l oun +sh ops +al y +ani me +multi ple +ðŁĺį ðŁĺį +psy cho +je an +ap art +candi date +gg y +con f +jose ph +w ick +me at +fr ame +c l +for got +ph y +f ing +li ed +re p +se ed +f all +u fc +nu t +lin d +mo de +fiel ds +en ce +s ley +ðŁ¤ Ķ +ch ill +follow ed +announ ces +cor ru +tro phy +them selves +ac le +al du +k ong +l on +s v +bro ke +ander son +ta i +stor y +tempor ary +activ ities +k ati +ari z +cry stal +spo ke +extre mely +tra ding +ðŁĴ ļ +à ¼ +in ch +ed in +out fit +equ ip +ma di +form ed +be ef +po p +ti ger +this day +ti red +neigh b +re tro +is a +un t +t as +kan sas +de st +secon ds +ta y +hur ric +o u +galax y +dad dy +bro w +bur ger +en ced +de sk +ac cur +secre tary +el ite +k ab +ch in +touri sm +bud dy +ici de +dre ssed +u d +vac ation +che ers +com for +charac ters +j et +bu ying +l ins +n ap +reale state +li e +af c +i ii +f ame +n r +b at +ag ent +ma kers +âĢ ¼ +sec tor +op ti +le on +di et +pra yer +hi p +mi r +le x +br y +an a +pas sing +w en +reco very +ak i +po pul +res ort +mar ia +stu ck +read s +ti er +perfe c +netfli x +p oo +cham p +o c +re duce +we red +comm ents +cla im +acci dent +s ag +h ack +sal t +kin da +k iller +i os +z y +ex change +lec ture +eng er +ic king +t au +reve als +pri son +z om +gh an +u l +jour nal +i ot +tr in +jon a +govern or +cap e +quar ter +spec tive +impre ssive +bab ies +t x +m ill +o y +har ri +jo int +su e +collabor ation +tren d +revolu tion +re new +alum ni +ge tt +sh ell +sun day +ent u +ni c +donald trump +block chain +paci fic +expla ins +sp y +ad voc +par adi +to f +star ring +p av +fe ed +br ac +smo ke +ham p +y am +to kyo +si mon +d h +e ffici +phys ical +n j +ell i +s low +gradu ate +americ ans +ti fy +f red +ap ore +fin ds +rob in +we t +not ice +se mi +un ve +k om +pil ot +scre ening +da ily +ðŁĴ Ĺ +roy al +sp a +vo tes +n ag +wh ate +att ending +exper im +ad dition +k ate +sto l +m ali +foo t +chri st +ch an +de e +lic en +glo bal +mo ore +ti a +bri gh +myster y +y ay +âĿ¤ï¸ı âĿ¤ï¸ı +cre ati +me chan +clo ck +di c +âĢ Ķ +pp er +al ph +through out +al low +re sources +selec tion +ham il +bb q +aa aa +virgin ia +dis ney +en g +so red +drin ks +f ancy +consi der +end a +jan e +hand made +du l +on tari +i us +s ville +color ado +whate ver +whe el +promis e +ne ver +desig ns +ab ly +sex ual +vanc ou +at i +con vention +cul tural +sing apore +pro mo +load ed +gla sgo +pp l +n oo +ke e +ste m +men tion +i do +cru ise +ri ding +be comes +be y +âļ½ ï¸ı +tw in +dedic ated +na sh +de si +work out +jen ni +i v +grou ps +rela x +pho eni +li ft +mix ed +m ck +p c +mu st +me tro +ci es +y ar +a im +ang er +i e +rec y +marri ed +dro pped +eng ag +le st +ambassad or +op h +de s +w ick +assi stant +nat ur +fa il +l td +shor t +k ap +sha w +bi gger +rema ins +crit ical +sur vey +co verage +er son +win d +n b +bil ly +let es +ac ts +jim my +at lan +al and +t c +import ance +dam age +f g +stor age +tw t +bon d +bal ance +cr ying +pu ppy +vo te +pu sh +ðŁĴ ľ +pol y +me l +lon don +terr ori +effec tive +corpor ate +atl anta +jac o +nas a +gre ek +sen ate +i sh +ev a +intellig ence +effor ts +al co +k un +h all +di ag +claim s +fir st +h b +ba e +v ul +pu ll + ° +se par +spe ed +vic ti +on thisday +audi ence +r ates +te ach +fil ming +bu sh +son g +y um +br un +ra ine +aw a +par ks +ð Ŀ +ra bb +ra ch +ra id +reach ed +ra il +mo ves +selec ted +fr i +ra ising +om y +st ones +su k +franc isco +cas es +cap it +con fu +w tf +po ke +equip ment +gre g +ess ential +off ering +ne x +pi es +be c +cre ation +chair man +cro wn +w al +john ny +shi ft +ne ck +ban g +bir d +ðŁĺ ı +du ck +re serve +de pu +ma sters +over all +no tic +ju ice +sne ak +che er +cla sses +eag les +n ca +car pet +ci vil +coach es +har ris +u ps +b alls +dec or +mar tin +ro s +v ice +announ cement +who se +ti gers +ste red +c ts +dr am +ste el +youn g +inst all +supp o +recor ding +de ck +se ats +l der +ang le +bo t +sty les +elec tions +for tun +n ab +but ter +ari an +ka sh +in ner +ou red +be ast +we i +ic onic +exper ts +ne cess +b eng +jam es +li a +gre ece +ðŁĵ · +ðŁĺ ģ +good bye +m itch +tw ice +mumb ai +ste am +ru sh +med al +ne tt +fashi on +t ar +r s +sav ing +ric ul +l m +sleep ing +brook lyn +mis s +sen ding +disco vered +sp here +of theday +k icks +missi ons +w right +er n +ght ly +i ous +mel bourne +star tu +mo ved +car ry +d ak +ag ues +bel gi +e ma +way ne +do t +er ie +pe l +it unes +matthe w +no body +est ab +cal m +win ds +lu c +prep are +tren ds +exerc ise +adv ant +ðŁĴ ¯ +athle tics +app s +c tions +adv ance +laun ches +litt le +real donaldtrump +eliza beth +carol ina +hu b +hi dden +n w +us er +pol l +great er +mo st +f ed +p at +life style +s ati +sco res +marri age +l r +aven ue +de serve +ri f +ðŁ Ĺ +wat ch +champion ships +gr ay +en ni +cot ton +g om +whe re +pack age +su m +ab solu +new ly +foo ds +ty ler +assemb ly +musli m +ban k +re memb +op tions +produc er +land o +fun ds +u pper +shad ow +pro gre +co p +ing e +leg s +detro it +hill ary +jo se +gi ants +sou p +sustain able +t us +clo thes +roc king +n z +min ne +mat eri +bru ce +ear t +ca sting +independ ent +thou sands +ta h +de cl +veter ans +li ons +wra p +âĢ ¦ +de ss +bl ing +st ine +e ggs +o on +clo sing +z ay +at t +bac on +fa il +ariz ona +de pre +gho st +new sp +w ers +vi p +li ked +id ent +volunte er +ad ult +pu pp +cir cle +mat erial +degre e +gro wn +boo m +calend ar +su r +vie wing +ath letes +ch and +re ll +asi an +en tr +vol ley +victi ms +bo dy +m ama +trans fer +ge ek +in dic +sav ed +ma i +g ent +it s +loun ge +k ol +the ory +situ ation +is lands +ar th +z oo +floo d +vi ously +show ed +parliam ent +ch ev +el ine +at trac +ab ad +ta il +h rs +lu s +por tu +gor y +provi des +to ys +de ath +in fe +an ce +g le +li am +lo ver +hu d +dv d +reve aled +g w +re ment +ca the +l ying +ra dio +der by +stor s +che mi +hosp it +âľ ¨ +' : +ilo ve +le mon +re public +s ni +ne ss +do or +re action +pre gn +fla v +schol ar +spo tify +is ation +vis ual +aw are +spon sored +jo ke +less ons +leg is +lo ck +si mil +ðŁĺ ĭ +kin d +la y +ma h +ho ping +vancou ver +as er +clean ing +gal a +thre at +la p +ach e +ro mance +ex pen +re post +z am +e pi +mir ror +o ak +ad ul +bat man +s lu +l c +vie wed +re views +d ates +indone sia +acti vi +off en +lea f +i si +ag ricul +costu me +s ites +spir itu +appear ance +ir y +st air +applic ation +spec tac +ic ity +ski es +hand le +pun k +paradi se +t n +de al +provi ding +do c +recei ving +bre w +micro soft +à ¶ +fer r +me tro +th ail +y um +car ter +à ¡ +gent le +bre aks +coo per +show case +cu tting +egy pt +bab y +semin ar +gl ori +ss on +fa ve +re hear +lo tte +la dy +al as +pre p +deli vered +nu clear +ir o +engag ement +at ta +con ven +z an +gl ory +hol ds +busine sses +str ange +sch e +it self +gra d +mar kets +f alling +st ats +ge on +bu dd +li s +she et +thi si +co lo +deser t +regi stration +ig n +expla in +inter ior +la ws +writ ers +spr ings +k r +fri ed +blo om +inf ra +a o +cre d +pa st +line up +bo o +bre a +boo ts +celebr ity +att acks +bro ok +ev es +ex cu +cher ry +oo p +fas cin +boy friend +se as +n ine +effec ts +po wered +k ha +ðŁĺ Ģ +sh out +con dition +i j +her o +enter pri +win ter +applic ations +sho e +g el +batt le +pro grams +w art +ðŁĴ ¥ +ra p +ho l +dang erous +di a +coun ter +ric s +i or +k night +co at +emo tional +at ures +d as +whe el +fore cast +tran sport +glasgo w +king dom +prepar ing +im medi +ff in +awar ded +prin ting +ro man +fight ers +any more +bel t +p ine +win e +x i +employe es +logi es +al led +de mo +birth day +ange les +lo g +dri vers +neck lace +k ath +s it +athle te +ef s +s burg +pur pose +resi stance +rele ases +t is +vari ous +deli ver +ch al +s anc +opp o +cra w +neu ro +dr a +suppor ters +sna p +diffic ult +swe ar +logi st +pa th +attemp t +à ¥ +swim ming +ste ve +hur t +inclu ded +b ap +wa re +ðŁĴ ĭ +end ers +ja ke +le eds +cli mb +l b +im ple +li sa +clo thing +ðŁĺ İ +d t +com pla +sw ing +stra w +v als +k le +us ers +stor m +cu ts +ontari o +p an +hand some +i ow +ar gu +chec king +scotti sh +Ķ ï¸ı +si er +em ma +po d +patter n +de sh +en h +ed ward +t ing +k h +hal f +lincol n +mo ther +al leg +r c +volley ball +d n +g ay +all y +le ton +gro ve +l oud +adv anced +re spec +cli ent +supre me +thail and +ho w +gi g +to i +do t +dol lar +ðŁij ĩ +p it +r b +h n +produc ed +gg ers +âĨ Ĵ +ml b +can vas +fin eart +us d +in the +p son +actu al +s l +t b +ip ad +en sure +u mb +w d +sk a +mar s +k end +f eli +th ing +count down +absolu te +r out +dra l +p y +inju red +min t +hun ting +mm er +s age +li gh +ac ity +ex pan +mur ray +ar o +sec ure +four th +eag le +reli ef +st akes +industri al +clar k +under standing +see m +pl enty +sil ver +cla u +thre at +sa il +pro duce +ab str +is is +b r +eng ers +wor ry +bie ber +s j +just in +reali ze +ky le +esp n +fil ter +s ch +ty pes +game dev +d ing +twit ter +soldi ers +p om +car bon +y ards +child hood +ri ed +ke l +ele ph +t ons +key note +qui et +wi re +po sting +is sa +repre senting +bac ks +alex ander +celebr ates +ta ining +| | +ch or +esc ape +pe ek +ti ves +fiel d +ssi e +im pac +spons or +r c +we dd +cann ab +si des +trac ks +com par +con trac +techn ical +bi ble +expl oring +sh are +tra v +n ate +ill o +sc ru +m ingham +gun s +of the +sh ame +se es +ca tho +ac cess +ce l +repor ted + » +mari o +p ad +hope fully +ou se +y on +disapp o +ol o +p itt +pa c +ga p +cru sh +s g +k le +ge m +emp ire +dir ty +a is +avi ation +ze aland +fac ing +high way +d anny +spi der +ot ta +ðŁĺ Ħ +w y +col ours +in fl +co sts +olym pics +au s +h m +ho ward +pas ses +lau ren +mu sh +op in +r ho +disc ount +oper ation +em ily +mm m +cham ber +d il +to yo +shi p +sam u +pic tured +un ic +po l +keep er +carto on +st en +ig nor +n ations +n l +ta sting +deta il +offici als +mo tor +franc is +ed itor +ðŁij ĩ +pe ts +rang ers +t g +r n +w ri +nic hol +i se +spo ts +ani e +chec k +tri ple +ku mar +spe akers +ic ing +pre pared +ab use +friend ship +mon th +swi m +air e +sc ent +hamil ton +indi an +j es +yum my +te ars +da wn +i zed +worl ds +ðŁ ķ +b illi +st one +n hs +ba sic +p or +st le +ir on +ol der +cle vel +e ing +ðŁĺįðŁĺį ðŁĺį +prin ts +fir m +air craft +fin est +devel op +aar on +t z +gra ham +own ers +fo li +less on +qu es +bab e +cra ft +ph en +ju n +bir mingham +v ine +ll er +i an +fineart america +evol u +st ab +im per +war d +com ic +wi z +inv ited +du ke +mat ch +por ts +ro ger +diag no +ke pt +te st +vis u +r hy +so c +to x +b aker +sur face +co vers +man s +b its +x box +ff le +n an +gar d +h art +wat ers +v illa +re tro +light ning +catho lic +democr acy +neigh bor +pen n +cr an +jona than +la ura +vi bes +su b +coach ing +clear ly +uk raine +bra ve +commit ment +t all +mar t +ra p +mo di +sco tt +bro s +show er +ðŁı ¾ +âĺº ï¸ı +cou sin +appro ach +br e +com pos +hil ari +phil ly +g ad +quick ly +ri an +t m +vir tual +hou ses +k t +phoeni x +w ire +ff y +b unch +anc ing +tal e +snap chat +star ter +h t +k icking +ap art +th y +) ! +blo gger +it z +com fort +ang els +w ash +" : +ar gent +re quest +hon est +mi ghty +bo bby +k g +ro l +thou se +ex po +h c +tab les +mag ical +po sts +de m +n w +or lando +ab er +* ** +ðŁĺ ľ +environ mental +trans formation +mi le +w ic +hir ing +ma ine +bo ar +r ying +ti s +nit ure +twee ted +anton io +opin ion +fin ale +di y +f is +th in +trou ble +le go +fi les +qu art +sp a +curren cy +cli mate +fan art +rail way +sp ace +ban ds +dani el +mo tion +l eng +hol der +oc cu +mar ie +cathe dral +bu zz +bi es +nas car +bm w +bat tery +char lotte +doc tor +zz le +se ven +in san +d dy +st en +lab or +thr illed +se ren +docu mentary +wav es +cer tain +can did +allow ed +ninten do +star wars +ta p +home made +d les +ther ing +bre e +emp ty +pi ano +pos iti +coun try +por k +pu ts +per ry +m atic +spot light +ti st +or ities +we alth +c p +bar bar +commit ted +as sau +pro fit +e ight +hu l +fini shing +run ner +ss o +insp ec +char ged +christ op +lo sing +co al +ho o +ele v +de le +mo ham +don ation +c able +clin ic +j in +manag ed +ter ing +â ¬ +ur ban +depu ty +bb er +bur n +acade mic +o tt +sta ke +it er +sto wn +ack er +advent ures +ad ams +gre g +pro m +vo l +ac qu +con gre +pa int +citiz ens +c all +af ford +v c +as ks +the tic +independ ence +â Ľ +h itting +bl on +fu ture +â ı +in no +gen e +bo ards +di stance +se t +re mem +th al +pre vent +l ang +ob jec +su sp +mat t +in duc +bor o +pi one +re di +vir tu +prin ted +sco pe +shar k +suc ce +a stron +il legal +j ag +c ting +ine e +at o +rob in +nutr ition +b f +du tch +b n +fur niture +for gotten +at ar +ru p +hy per +bran ch +communic ation +degre es +on ia +un cle +promo te +or che +wi i +j s +but ton +ma jor +c bs +bri stol +premi um +ordin ary +e dit +m g +we ed +st even +: ' +gu s +te s +cap tured +dru gs +do w +wr ites +bi shop +whe els +ali zation +disco very +w r +rach el +ne il +hy dr +cu test +entreprene ur +kore an +ore gon +ul ty +perfec tly +suppor ted +histor ical +t wins +ell y +we l +de vil +in come +scienti sts +de leg +h en +on i +ic ed +gi o +cur ry +reve al +e g +buff alo +n ol +op era +camer on +haha haha +j ab +gradu ation +cra ig +r al +i f +organi zation +le ge +g ang +su d +edin burgh +l ack +fli es +g ate +thr ones +q b +the real +e leg +pp in +c les +jam ie +tn am +cryp to +ou l +p ages +a se +roo ts +stu pid +a did +boo t +prote in +s ap +si um +su s +end or +fun ction +don t +en na +ch y +squ e +wor ker +m tv +e a +k an +ðŁĴ ļ +mu s +professi on +t to +oper ations +al lo +c tor +inv ite +sc and +ou th +z im +lin ks +cli ents +sam sung +discu sses +n ell +ul tra +some where +ste wart +ine t +de z +b out +fac tor +ti an +tr ans +jere my +d b +ðŁĩ ¬ +or n +develop ing +spo l +coo per +ma u +rememb ering +tre k +famil y +sen iors +fo ster +att ended +w ing +trans form +ele mentary +hor iz +li sting +malay sia +it ch +warri or +philipp ines +russ ell +m end +initi ative +cre ep +to ps +br iti +a ur +shar p +adverti sing +ug ly +achi ev +materi als +bu g +dev ice +bon us +fac ility +col e +nh l +y as +plann ed +pol e +excell ence +tr ick +con fl +r p +achi eve +lo an +swa g +jess ica +ho we +p our +sc u +z oo +r ated +dre sses +re bel +mex ican +co ordin +me ss +atlan tic +t l +osc ar +wal ks +phar mac +investig ation +... # +cc i +eas ily +monday motivation +y ment +au ti +for ced +ar med +colle agues +pap ers +pro per +sha ke +bu c +le an +exhi bit +e vement +co tt +bi z +sp er +k ent +sw an +/ @ +girl friend +haw k +âĺ Ģï¸ı +mon o +ðŁĴ Ľ +stat ue +ðŁĺ ³ +ra s +te eth +preci ous +t ile +p am +swi ft +v ali +no se +dr unk +experi ences +come back +gen ius +wor se +sh ef +ra d +ed it +hon our +au spol +lar ry +h ire +gor don +achi evement +.... .... +su icide +alter native +su p +sur roun +sha ke +ke ith +pe pper +tur k +crimin al +be ck +su m +w alls +cn n +an tic +of fe +col li +win es +high light +hawa ii +emb ar +l fc +ðŁĩ ® +m v +> > +at mo +wor d +car l +shout out +bre wing +ì Ŀ +do f +s ic +hot test +col on +hh h +shu t +low ing +volu me +apart ment +agre ement +de stro +we e +religi ous +iow a +ro d +land ing +re present +ðŁĵ· : +la s +usu ally +h l +c ac +sal v +al ong +laugh ing +be ans +remin ds +pha se +some body +ma sk +ran ked +dest roy +sc i +âĢ¼ ï¸ı +gab ri +le o +ro a +fa iled +si l +refuge es +re vi +r ing +ber ries +coo kies +y y +conserv ation +sh ab +human s +de termin +a in +ni all +as su +mb a +fro m +extre me +vic es +commer ce +ght ful +or dered +suppor ts +re cap +v or +dro pping +correc t +pay ing +mean ing +n j +qui z +" # +busine ss +ðŁĩ® ðŁĩ +indi gen +du st +box es +bl ind +x xx +zz y +ðŁĩ¬ ðŁĩ +ss els +s ant +dd le +hilari ous +desig n +wonder ing +vehic les +k re +ju d +rece ption +par ker +Ã Ń +pri vi +hy dro +sof tball +pol lu +lo cked +ba h +e ar +scri pt +di vi +br ace +geor ge +the ast +bel o +j al +tion ary +dent al +roc ket +pur ch +sh ak +manufac turing +e z +it is +con cep +tb all +ch s +direc ted +pra yers +oo k +phil os +vari ety +che ss +ser ver +g and +bal ti +ðŁĵ ¸ +sel y +cru z +spectac ular +bur ning +re present +i z +t one +mer ce +h ell +bed room +estab li +bo l +com mon +ãĥ » +ab or +kit ty +hei ghts +re pair +willi am +qu ake +alab ama +popul ation +re v +re tt +i sts +n ite +le m +a ha +clevel and +r m +po ver +ob se +mon tre +man ia + ® +con ne +car ni +sh ah +f y +u a +sc or +strugg le +bo b +' ' +appro pri +deci de +ff ed +ca ster +s ort +hun gry +dra g +ا Ù +gr ounds +d w +sli ghtly +car din +dead line +bron ze +web in +bar ry +sil ence +e uro +op tion +ear n +ðŁĴ ĸ +howe ver +na ren +na ils +bath room +v ine +ph d +min ing +gar age +( ) +shou lder +defe at +di r +o v +liber ty +ple as +x on +com pre +a v +j in +ab les +sil ent +fam ili +vis its +di pl +ha bit +milli ons +regar ding +innov ative +sen ator +r ts +v on +k l +wh il +requi red +âĿ Ħ +lu v +presi dential +po cket +hun dre +sho wn +fro zen +to ward +fa st +confi dence +r ough +indivi dual +qu et +ðŁı ½ +dom e +fi fa +engine er +z en +re mix +ðŁĺ ĥ +pl ant +min or +robin son +as y +pul led +cer tain +potat o +( : +pre s +oc ca +w it +it em +si e +d ating +thom pson +own ed +an u +vi e +te dly +good night +ex cept +ðŁĮ Ł +ira q +ki e +ren ces +li p +simil ar +sau di +vi g +arth ur +pic ks +mil an +hon da +ma xi +o g +ste st +ar ch +analy tics +ba sti +pear l +ter ry +hor se +ast ro +ac ce +laun ching +inter national +s no +ta sty +den ver +ir l +pe te +tor n +advant age +var sity +" " +sol e +g c +lan g +demon str +ol ds +un ity +ne ts +insp ire +cre te +nash ville +nel son +e ter +wal k +hy un +m ack +tre as +see king +ra ge +bru sh +ab and +whil st +co con +h ong +shel ter +i p +possi bly +so o +it ed +â Ħ +rac es +war ming +qu in +tele vision +mat ches +ra pi +ment al +pal m +jenni fer +rol ls +indi ana +b ars +cat ching +resc u +candid ates +fa re +âł Ģ +se o +vie tnam +alph a +michel le +visi ble +re gre +wn ed +app le +li p +f fe +li z +york shire +ha il +se asons +be gan +m d +k c +la p +fascin ating +hel p +ur y +u ms +nu ts +se m +along side +bri dge +ori al +o ve +world cup +briti sh +comfor table +i ve +hot els +fair s +hor ri +so x +d ining +stre am +bar ri +ss y +w im +ter ms +v u +pe re +l ens +wal ked +r or +l ars +shi eld +dou bt +pro to +cro ssing +me ant +medi um +ad ding +e b +che ap +fun c +pap er +bran ds +ry an +feed back +col lins +un known +tro pical +sand wich +fal len +for mu +selec t +lo ads +answ ers +or i +mag a +d or +du o +ali e +dru m +ur i +de er +sou l +sh ut +âĺ º +sto len +don ated +bu zz +patri ots +ha l +na sty +nomin ated +mon te +ki a +th ri +ing u +te sts +pe tro +ðŁij ij +ho sts +ne st +to pic +pat ch +m my +hu gh +ab ilities +ma the +s miles +g b +ag enda +insi ghts +chi p +ph an +fail ure +dg ers +ha i +signific ant +sho ck +ru ral +gl am +figu res +pot us +o ta +mini stry +appe ars +fe ar +r h +americ an +h att +son y +fi res +e di +n ou +e qui +wh en +univers al +mad ness +i x +sculp ture +b ach +t to +swe den +et a +en to +develop ed +month ly +ma ps +ra h +le d +del ta +sa ints +is lam +ben ch +fif th +v ard +so cks +wel coming +j e +tur ner +v b +ad i +nor way +ad y +hurric ane +por sche +tra dition +ex am +newsp aper +lu ci +a ver +ide al +d na +madi son +ðŁ § +wit ness +ac ou +insi ght +si mon +robo t +sna ke +n bc +ac o +ro ss +sh ment +religi on +ch ann +in su +camp bell +inst alled +we ather +hor ses +ol i +rober t +k az +ðŁı Ģ +veter an +th read +quar ter +ea sier +cap ture +hi pho +law rence +roman tic +pas sion +cl ay +ox ford +th ai +stu dying +fi a +elec ted +most ly +c b +tu mb +âĢįâĻ Ĥ +x l +sh an +fa ster +ev ans +sli de +sh ri +see k +mi es +chemi stry +pump kin +tu m +, , +ro om +fi red +li ps +pres ence +af f +brew ery +arri ve +sw ag +photo graph +pen gu +chi ps +at tor +val ues +accur ate +con temporary +princi pal +cannab is +ari o +any where +gi a +democr ats +buil dings +li ved +ap s +neg ative +m are +bal lo +li on +diam on +loo k +re form +tom my +il la +tre ats +hundre ds +port land +wor thy +ex cep +ar ia +ido l +be er +cd n +y u +aw k +ðŁĩ ¨ +c ells +à ³ +ident ity +dra wn +de vil +f inger +th am +ðŁij Ĭ +ear ned +fin tech +dol ph +twee ting +evolu tion +ðŁĵ į +est im +m vp +n one +ðŁĩºðŁĩ ¸ +toyo ta +au x +mar in +b old +l bs +ste ak +mur phy +it able +lou is +sol ve +pi a +sk ir +ill ino +webin ar +ban ana +lo v +th on +vo ters +afford able +defe ated +lm fa +air lines +super b +any way +deb t +bo red +ver si +me tal +responsi ble +m k +s se +f ay +cau sed +f p +recomm end +pla za +spor ting +alli ance +au stri +n n +t ours +surpri sed +arti f +th under +sur ve +wor e +bri ef +necess ary +z ie +ash ley +dra ke +r t +kni fe +im mun +char ges +a the +bri de +rep ly +g av +broad cast +pu er +brace let +cap acity +harve st +id k +perfor man +d ding +il ers +par a +jam a +pro vince +ch in +id ers +har i +te aser +ch en +re stor +r at +fl at +col om +ðŁĴ ŀ +ðŁĩ¨ ðŁĩ +smoo th +r t +p itch +stay ing +isra eli +t cot +per spective +do ck +open er +lo vel +x o +class room +l ington +go al +kenne dy +sh am +sp aces +mitch ell +home coming +uk i +claim ed +recru it +ing o +mu fc +mon it +g roo +resi dent +per cent +per man +otta wa +int ment +an xi +stand ards +wor ship +sche me +f x +pot ter +bi an +athle tic +af gh +s se +sat ell +par ties +âĿ¤ âĿ¤ +infra structure +rela x +mo du +wor n +smo king +y ach +practic es +wc w +am b +dome stic +tay lor +k entu +provi ded +mo di +ve g +" ... +ob serv +ðŁĺ © +be ard +m our +an gry +ðŁĺ ± +startu ps +woo den +di ve +na il +anti que +ro ses +torn ado +m at +^ ^ +su spect +far m +de vices +me ga +tu l +scholar ship +ge e +disa ster +arri val +po in +mar c +kati e +bb ed +fal se +deser ves +ric hard +ju ana +fre y +tion ed +hy bri +r w +sar ah +ach i +c ure +o le +mor ris +ch ic +broad way +la bel +pa k +pover ty +gol f +e red +f u +er ies +be es +alo gue +st el +wire less +je wish +ti de +blo cked +life time +b har +sp lit +am ster +th i +jo shu +br unch +ha ps +s for +oo ps +ka poor +hi king +suppo sed +ro of +re as +tra in +ti ght +tru mp +bas ically +r r +ea red +see ds +entr ance +c p +wi e +son ic +vic tim +he re +e h +ear rings +sal mon +arc tic +an ne +dou gla +corru ption +hann ah +ha sn +vo ices +con ce +att a +fle et +clin ical +democr atic +ton y +st ood +le f +twit ch +a il +honest ly +incre ased +dro me +don na +accep ted +visit ors +ap ar +ad or +p ar +jer ry +ra i +brand on +ab u +!! !!!! +me me +in gh +glori ous +b hu +pu mp +j ol +li ke +fi sher +ma z +ag an +destin ation +play list +le tters +gen u +br ace +celebr ated +bann er +r he +dra gon +ðŁĺ ħ +sig nature +gre y +âľ Ķï¸ı +al ice +be red +ph er +ber n +ca th +ga thering +sc oring +influ ence +sm iling +de pt +lo cal +a x +ac u +reti rement +hon or +her self +chem ical +asse ss +y all +fre qu +appreci ation +ac a +cho ir +cu z +so il +c il +repor ting +u h +enterpri se +gr at +jaco b +ru m +fe e +j ak +sp in +bi kes +phi a +ste re +p is +bloo d +t att +ra ft +war ren +sh eri +back stage +mar sh +hash tag +ther ine +re in +game day +guar an +reci pes +min ds +stron ger +issu ed +bic y +n ak +ment ed +sc ary +u x +pre vious +tt le +th ats +ac tors +u ma +tin a +bun ny +promo tion +u ss +oli ver +montre al +what s +appreci ated +la kes +excu se +kno wing +pri zes +musc le +shad es +sco t +ing redi +electr onic +ju an +comb at +s ri +e h +turk ish +l om +stri kes +pri son +re e +po pe +vi d +ol dest +dol l +sw iss +certi fied +cli p +re turning +lat or +le igh +tt es +wat son +heal ing +el im +per haps +ha ss +k au +d der +mou se +new castle +indigen ous +wel comes +co le +tau ght +no ise +appe ar +jo e +can on +wedne sday +u tah +c tive +dri ven +i v +c ell +stri p +ac c +focu sed +ar rest +sto cks +wo o +â Ĺ +notic ed +shad o +di spla +ter ror +bor ne +secon d +que ens +wo ke +ja il +no tt +cam bridge +har t +se af +fa x +ac cept +âĺ ħ +goo ds +k at +t win +h s +thou sand +s ins +su ite +amp ton +ar n +rele v +ric har +hoo ps +n bc +class ic +p ab +soldi er +de plo +le ans +install ation +cla sh +le ban +ee e +ti re +belo ved +fu sion +travel ing +ne i +coo kie +glo be +phys ics +s q +co l +wol ves +d l +ex it +" - +foo tball +le af +ster ling +hi de +minne so +fresh man +natu re +indi e +supp lies +bri s +iri sh +ink tober +doo dle +ic op +mess ages +adul ts +recor ded +fix ed +ar do +offe red +under ground +dr one +p ine +ma inten +and re +ham mer +s x +r ound +hi ke +bra d +ro me +fu ll +on ey +ro ws +colum bia +archi ves +appro ved +bat ch +illino is +recogn ition +shou ldn +fo g +nca a +ke vin +human ity +al though +pow ers +p ou +s ar +pe st +alco hol +con sci +phil adel +en o +t m +ok la +cate gory +particip ate +accu sed +bri ef +po em +clu bs +consul t +ja b +big data +amster dam +ac ing +certi fic +n u +d at +impro ved +and y +campa ig +pale stin +p ace +mo bi +feel ings +wol f +bra in +pro pos +inter active +prin ce +inde x +c is +cha e +peace ful +co vering +ac o +cour ses +mon key +re place +b l +bloo dy +tal es +brigh ton +neighbor hood +g ates +spiritu al +af raid +bre ast +b ones +ðŁij ī +vide o +w au +tou ch +inju ries +car l +ri x +une x +âĢ ¢ +fre d +consi dered +thu si +an ch +on y +u sa +graph ics +ac re +ðŁĺ © +com memor +com mod +go ti +guar dian +star bucks +pre vention +haha haha +admini stration +portu gal +fac ulty +bet a +ul a +al bert +bre ath +er i +le tting +tr ic +ment ation +incredi bly +ten nes +v d +ðŁĻ Ī +ed die +br ick +gr ill +bt w +wat ches +resear chers +t ney +ni e +p as +a ster +vi br +poke mon +ch rome +go at +pitt s +il ly +festi ve +y d +can al +ðŁ Ĩ +fi es +car los +re que +partic i +tra ins +sam ple +temper ature +sym ph +pic king +in door +z ers +playo ffs +____ ____ +ap es +ly rics +islam ic +performan ces +d ick +spar k +se as +hom a +gr ound +disc i +employe e +com mu +alas ka +al an +fe ast +dg ing +ban king +manu el +slow ly +tru cks +mc car +oo o +sc rat +orche stra +indivi du +m x +bre ath +stair s +equ ality +bla ke +loc ations +cocon ut +balti more +aa a +l c +ðŁı Ĩ +har vey +resi st +immigr ation +adid as +fil i +re f +lg bt +mo s +pp i +ken ny +terr or +ban e +apol is +s g +social media +ka i +hon est +as sas +bol lywood +âĢįâĻ Ģï¸ı +ferr ari +hor n +cryp to +bo om +mainten ance +i di +s man +w l +ext ended +in sul +ve s +go sp +tr i +pi g +tar ge +cel er +st ati +sm h +ri dic +appe al +? ) +con clu +cos me +she ep +christop her +en thusi +po lish +me ts +oun ded +sustain ability +creati vity +con crete +ra i +ali en +ble ss +te es +clu b +ro t +bo s +ex ist +perfe ction +lu ck +rock y +expen sive +mean while +happy birthday +pre t +thr iller +ca ve +playo ff +som er +l u +le x +def ence +am writing +home less +pro phe +ch et +past or +ðŁ¤ £ +land er +ww w +Ģ ï¸ı +tic a +! # +o tic +rad ar +po sters +pow der +po li +ha un +tra p +bl in +assau lt +shor ts +re y +sh y +squ ir +rac ist +gar lic +fu r +remo te +sm ell +impre ssed +fing ers +âł Ģ +din o +le ment +s nu +promo ting +str ing +produc tive +b age +ma son +ra z +direc tly +j k +ev al +ðŁij Ĭ +doc tors +co w +ri der +st v +re move +w u +na than +ro d +n r += > +affe cted +inve st +mp tion +g inger +o d +agricul ture +s que +mu g +coun ting +ke e +mag nific +coo k +ani stan +roo t +plac ed +sym po +gh ana +un d +che er +thro wing +secre ts +f illing +opti mi +butter fly +bu bb +ðŁĺ ī +terri ble +d g +sil k +obse ssed +lo u +ai de +sal ute +mon u +philadel phia +scienti fic +i st +u ae +dess ert +bott les +can yon +ðŁĺ Ī +car ib +o ther +w ich +re source +guil ty +un d +le on +e ss +kan e +el e +tra iner +he im +an te +man age +roo kie +tre ated +po ses +rs vp +cau ses +aw ak +je well +le tt +on ics +tit les +cardi ff +g aga +bu mp +use ful +? ! +loo se +bb ing +: : +argent ina +de bu +cy cl +wh el +dis gu +j el +k ills +bio logy +ex ter +tra sh +bo dies +tr am +circu it +expe ct +la ds +w ells +sho t +ge e +naren dr +fa stest +b ent +b ills +mar shall +h ats +intro duce +citi zen +im possible +gi b +az z +net working +r ant +thin k +in dy +st ops +f theday +bri an +* * +amo di +dom e +coura ge +pac king +af fairs +g n +si zed +ent ary +pol and +swit zer +afgh anistan +w u +ten der +subscri be +mo sco +att end +republic an +hon ey +âĢ ĭ +si mul +we ster +foo die +or o +midd le +ab t +co pies +ma je +narendr amodi +ty pical +inspir ational +vit am +wis con +cu bs +tiv ity +h ali +e ars +k ay +d are +mari juana +cu rious +an ia +tom ato +re mind +ðŁĩ · +sc ared +cou p +po et +land ed +ri d +wra pped +mor ri +climb ing +e ws +fe eding +con tra +tho logy +gri d +ti vely +read er +la ser +di ving +di g +lat in +ti ed +shake spe +o ci +ad m +show ers +chu ck +mar cus +oo s +kne e +o live +ow l +dy lan +an no +g ym +deci sions +well ness +arri ves +sati s +chri s +thur s +ðŁ¤ £ +inter views +thank you +switzer land +over night +journ alist +ser ves +vol can +.... ... +plo t +nic ol +car rying +mag ne +tre asure +ex p +be ver +ðŁĺ ¢ +mar ty +mo le +don ations +recogni zed +b h +du s +sh ann +al do +success fully +ent e +ðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤ +cab inet +cu is +tit led +d as +so l +strate gies +deli vering +ad ds +ani an +ne ther +ðŁĴ ĥ +con tain +su its +pa irs +to dd +rel la +ro pe +ci o +cro p +paint ings +su z +re jec +bu st +d h +fra ud +m h +contro l +je al +destroy ed +al lows +wo ol +minneso ta +om en +j u +sympo sium +d af +lim it +accoun ts +load ing +inter n +re solution +hol land +qu al +meet ings +gra ve +cam ping +v am +re nov +liber al +am ber +gre e +hu mb +fe ver +el ing +broo ks +à ² +be th +ad ed +al t +ro e +perform ed +jo sh +frank lin +nic ole +de ss +bb s +m g +net works +min im +al t +weap ons +gu y +jas on +g ha +harb our +at on +pra ise +kentu cky +bel fast +st icks +blo ss +ho pes +an thro +famili ar +wa it +ch ile +depre ssion +la x +je ts +le ice +recei ves +si er +an k +de x +inde ed +fle xi +fab ric +lam b +hel icop +am anda +âĢĶ âĢĶ +compe te +sn ack +techno logies +sy rian +mom s +mu ham +cho sen +an at +dev on +shar ks +re t +fundra iser +selfi es +st ations +communic ations +tennes see +tu tor +ro t +valu able +dynam ic +nur se +i ed +earth quake +deser ved +a ve +sar a +stre tch +dougla s +ne pal +à § +ob viously +d ame +ra pe +any body +k w +pat rol +hol ders +h anna +info graphic +ec o +be ating +stan ley +bo ats +ri bb +e z +wit ch +inv a +ac id +boar ding +- @ +gi l +da ve +care ers +opp os +l loy +in ter +do pe +re su +j agu +sh ade +in dy +on ist +rel ations +ag en +ab le +inci dent +me ter +shar ma +id r +pro ve +immedi ately +tro ops +am an +g low +gaz a +blo cks +person al +chron ic +all er +si d +sh r +whats app +lu cy +ar chae +ho u +journ alism +our selves +go t +the med +shap ed +we ak +cas ual +leng th +sla m +ab bey +e v +coun ter +est a +reci pi +cha pel +expan sion +sel f +suff ering +sp ice +n z +sp art +desp er +boo king +quart ers +y on +ðŁĴ Ĺ +p k +continu ed +- # +man hatt +tal ked +sh en +com bo +hybri d +je ans +liqu id +se al +re tweets +ac celer +collec tive +t as +: )) +profession als +ra w +o tt +su san +ir ing +okla homa +re ven +survi val +cre ator +tran sit +st ac +sur f +i k +ed iting +ch illing +bai ley +ste al +ra ble +pa rent +hun ger +sn app +collec t +philos oph +dedic ation +c f +c m +le ep +repe at +re ha +un fortun +a er +a ero +abstr act +mon itor +ag ents +bu l +sci ence +harb or +drag ons +floo ding +ac compli +d ash +juli a +the red +tues day +cy ber +b low +ta ined +le m +refe rence +pp o +ne goti +char le +con nor +au lt +access ories +commissi oner +rain y +re ar +advis ory +luc as +ma id +co al +k av +pol o +ðŁı ¾ +tran sport +mar gare +straw berry +bur ns +gre ens +ne v +partici pants +col in +belgi um +col our +in form +d ell +br on +cal y +kick off +strate gic +re union +hon ors +li b +egy p +âŃIJ ï¸ı +hy po +si zes +regi stered +bet es +relax ing +bloo m +inten se +valent ines +insan e +w wii +p x +tri o +bla de +wiscon sin +con e +plat in +ali ze +ra ven +incre asing +indi ans +il ian +bl u +rabb it +exten sion +je f +au di +fer ry +s ell +a day +us b +swe at +cham pag +metho d +mem ph +assi st +s by +ca pe +remo ved +mag n +v t +r ams +f bi +tack le +phe w +h on +motor cycle +su spec +eleph ant +sub ject +let te +da iry +whe at +awk ward +ac t +tro l +mit ted +zay n +sheri ff +ene my +con s +ke tt +bul ls +ev alu +bt c +satell ite +ho lo +por ter +dia betes +bet ter +rele asing +sur f +: - +se basti +collec ting +en cing +e thi +go ds +al ley +health y +m ills +sma sh +co pper +cr ack +read ers +sp ac +licen se +bas ket +bang la +en tic +om i +m ere +si vely +anim ation +lan es +dent ally +chill in +fi e +k aren +dep th +li pse +n g +ri p +mel o +sand y +ðŁijı ðŁijı +vin cent +nu t +hu g +who le +cre ates +? ??? +âĿ¤ï¸ı âĿ¤ï¸ı +bak ed +up grade +rober ts +har a +carib bean +auth entic +mb s +mosco w +attor ney +wi ki +ch lo +hu ll +cor k +" ! +sty lish +ðŁĵ¸ : +di ary +impro ving +ex pand +bri ght +pollu tion +k nights +person ality +chec ked +fac ilities +z el +bow ling +gu er +ðŁİ Ĥ +on going +un its +hoo k +be ck +confl ict +to dd +far ming +educ ational +k ak +cla y +stro ke +bel ly +explo re +mill enni +th m +loo p +sm s +consi st +cir ca +br yan +d ab +youn ger +soli dar +pp a +experi enced +b ella +bo ard +shef field +steph en +consu mer +sub mit +spon sor +t ang +ag gre +comb ined +trac king +sand ers +b az +survi ve +fer red +equ al +se p +re ed +str ong +priv acy +st ap +un g +ac ry +pa sta +pir ates +ag er +fair y +du p +introduc ed +wi p +let s +spr ay +ðŁĵ º +gre w +a sts +pitts burgh +new york +jo ey +lau ren +tra de +ch op +pi pe +cla ire +behavi or +v ap +cre ws +lap top +ðŁ¤ Ĺ +che ster +disci pl +d f +out doors +k s +go ver +super star +cas ino +far mer +; -) +re turned +ðŁı Ī +ma il +roa sted +co sta +v ill +pe z +gard ening +distribu tion +sh ining +inve stors +ra sp +dec ades +reali zed +bar n +p ti +st able +ut d +pan thers +m ens +b n +ca de +bu cket +yn n +when ever +wa ke +da is +ber nie +lo dge +ju lie +atmo sphere +ðŁĺĺ ðŁĺĺ +major ity +par ti +exc it +cu t +me h +musli ms +be gun +fli ghts +vene ss +ce me +po sing +so le +g ou +dark ness +pe ach +cel tic +auth ority +grand ma +ful ness +smi th +speci fic +gar cia +co ins +good ness +aldu b +recru iting +den nis +gar y +sle eve +weap on +pl z +disco ver +harri son +recruit ment +ja i +ch im +com pared +tom s +mo thers +am y +archi ve +t ask +ben jam +se g +law yer +al um +inve sting +mi e +che z +j p +a ke +fl am +wall paper +âĻ¥ ï¸ı +t ton +che st +favor ites +we igh +coo lest +r ating +relev ant +lo gan +ma ple +run ners +pri or +peop le +ma ur +terrori st +te sted +carni val +su spen +me asure +m v +cyber security +app ren +terror ism +o z +v ital +ni es +gon z +fun ded +twi st +assess ment +die sel +en for +colum n +ad dressing +ca sts +pay ment +x ton +fi er +, ' +la st +ne e +un less +clo se +sk ill +cuis ine +fun eral +ti les +a un +k ru +relation ships +ðŁĴ ¯ +ev ent +âĢįâĻĤ ï¸ı +kind ness +pro posed +acou stic +a es +defen der +dan ce +h tt +w at +vo y +ðŁ¤ ĺ +au s +cli ff +sear ching +beauti fully +in qu +at l +speci alist +ðŁIJ ¶ +da i +tra ils +class ics +inst ant +v ous +re venue +mar ch +kir k +fr inge +fire works +tri via +âĺ ħ +tr action +wal ter +mo to +l ily +att itude +cli mb +sc an +sav ings +c w +fa ith +cred its +ab led +gra ff +auto graph +he he +ran ch +ha d +ro gers +ðŁĮ ¹ +f in +re qu +fol k +ad ditional +lyn n +u ber +dol lars +lo gic +wor th +so m +the sis +p ound +bi c +st ur +cer am +spen cer +en tered +v amp +organi zed +âľ Ī +pp s +tr on +merce des +no ti +compet itive +do w +ous ness +vic tor +gr illed +na i +pu tin +ab ra +bl ame +alex and +anim al +dec ent +p ent +inter ior +:' ) +but ler +bal let +ðŁĴ Ķ +albu ms +down s +la d +si r +pla in +p ers +blon de +dis c +paki stan +se ment +ga a +w age +ch as +man i +co ps +terr it +lo l +lau ghter +ri vers +magnific ent +lam p +w b +new sle +char ts +ble ssing +p unch +lon gest +fl oral +cu tie +fare well +sto pping +mb b +bu d +chee se +de cla +si m +mc donald +de ter +you th +t ch +fre der +kin dle +fer n +at or +as leep +p ond +spr int +p ounds +la zy +gh e +fundra ising +dead ly +gran de +dou g +he y +lin da +consi dering +i um +gol den +vi k +auth ors +di ss +u ally +appropri ate +mor ning +y le +hon oring +foli o +be c +re bec +fin land +formu la +corn wall +sh ay +cau sing +bl end +sig nal +t ent +kash mir +nation als +har mony +sc out +acce ssi +he ight +medi eval +impro vement +ke es +prac tical +car d +de par +hu n +om ing +cal gary +ste l +bu bble +gur u +ma h +unex pe +n h +ed a +me at +i ge +si o +god dess +in ches +tun es +br itt +sti on +ra j +âĻ « +mer cy +ðŁĴ ĺ +sen ds +i est +pol ici +val e +reduc ed +as ap +vi jay +defen sive +celebr ations +ri ders +med itation +har mon +g ing + ¡ +program ming +in au +sud den +m h +replac ement +sk u +j ar +gra des +ta st +k itt +brand ing +k aw +boo t +f ought +p ays +g f +iz ation +ho p +k k +activi st +v end +coast al +cha os +ðŁĶ ´ +se me +bill board +li fting +cu mb +sc al +ðŁĸ ¤ +stru ck +l v +indie dev +beat en +jun gle +al right +destin y +m ing +k c +ch ances +om an +q atar +cra f +tra ined +pri x +char m +o tive +s mu +e c +and ers +hand ed +al ban +certain ly +arri ving +i ze +sa i +tr ack +pain ter +hu mble +appo intment +head line +manag ing +mo d +as pe +andre a +à ¤ +ethi op +un ited +exi st +bal i +k ad +n t +d red +re x +recogni ze +tam pa +be ers +ati a +he els +no te +transport ation +tur tle +re de +hipho p +sp icy +sp urs +⬠ĩ +cor p +ther n +to ast +hur ry +proper ties +ma ge +mar co +ele ments +bou ti +syn drome +ms g +develop er +gra ders +he im +re sil +off ices +del ay +di men +vin tag +barbar a +ðŁĺ ± +vene zu +cu lar +fac ed +bar n +ðŁĺ Ĩ +survi vor +wor m +confu sed +passion ate +Ø ± +identi fy +electr icity +sou ls +brad ley +repor tedly +lun ch +shel f +eli a +swee t +smoo th +emplo yment +am el +manhatt an +ste am +oun ts +ye p +li ving +un e +descri be +ca res +man ila +sha wn +ac ted +bas h +st even +re st +pet ition +div ine +wel sh +rac e +platin um +ðŁĮ ¸ +p b +extra ordinary +solidar ity +m all +on ion +schedu led +game of +fer gu +de ms +nor m +p k +tri als +polici es +publi shing +st ole +fron t +charac ter +van ia +ex ce +sti e +sc a +resi dential +sa iling +ðŁĶ¥ðŁĶ¥ ðŁĶ¥ +spons ors +th ick +champag ne +she pher +continu ing +ven ice +per th +na p +a ster +y ak +un limited +cho ices +ne o +hi v +repor ter +bru ssels +f old +dy s +se mi +la wn +it alia +wi fi +as k +em ed +fr ame +monit oring +ste ad +i da +gr in +is a +fli p +re stric +offen sive +atta ched +di sh +wh y +philli ps +gre et +p als +mix tape +v ou +fiel der +spar k +alber ta +g len +ca sh +s ri +u ri +ro dri +entreprene urs +climate change +p sy +d le +em ents +lin ked +nether lands +acci dentally +oppos ition +vel vet +ra ys +c w +om o +m f +lmfa o +newsle tter +: ) +toi let +liter ature +di sp +phili p +uni form +sudden ly +head er +cool er +-- - +prou d +bri g +nis san +scienti st +j ah +con centr +pac ks +appo inted +so ap +eng age +cho se +âĻ ¡ +se tup +jeal ous +har ry +g ation +tun nel +te mp +osc ars +dec ade +recomm ended +child ren +ab a +anxi ety +ve ments +sal on +pho too +organi z +mach ines +ab s +vil le +hy pe +ti ff +emer ging +av geek +[ # +contribu tion +bra dy +re sto +g mail +fit z +photo shoot +hel met +h t +eleg ant +ug anda +nur sing +or leans +pen n +na h +foo tage +em a +w o +w ad +concer ns +ve re +re mark +who ever +str ang +p t +qu it +sh ang +histor y +s ick +perman ent +ill ness +col d +visi on +he m +ar row +con vic +pin k +oc cup +bal d +ex hau +u of +am o +on t +ãĥ » +adop t +la id +smo ked +inter pre +ess enti +associ ated +b d +bb y +fi er +inst all +dipl om +con diti +c f +w ak +any a +gr aci +fi sher +s ss +ap r +il it +mus ician +symph ony +cor d +h ack +le gi +l v +bless ings +hum or +sc ra +e ti +min ster +trav elling +bu sh +jewell ery +li me +!! ! +pregn ant +pe e +lo b +cap ital +ip a +pen cil +la bor +duc ks +prou dly +wedd ing +dere k +m w +pe g +valent ine +an gu +re treat +pro spect +dang er +vul ner +up set +, # +sr k +x im +thur sday +n fl +kis ses +re ds +cr ack +re ward +c u +ko k +me te +aband oned +it t +me als +sp ell +stan bul +del ays +ru m +le op +gu m +no va +super man +ch ick +m is +dram atic +inno cent +r ounds +re c +auti sm +bangla desh +mor al +mo vie +sp oo +k la +âĥ £ +ou ting +mess i +ab road +loo kin +a im +q i +st ack +colla ge +à ¯ +hud son +sc an +ho e +ch au +oc cur +comm ander +ho les +ðŁİ Ħ +bi as +v on +stick er +ma k +responsi bility +colum bus +sa int +ed mon +rac ism +far ms +w en +gul f +may o +!!!! !!!! +corpor ation +ba chel +el a +inter nal +je ep +fol lows +di alogue +de rer +smart phone +he len +rich mond +equ ity +s land +b g +ne ar +av i +memph is +we ir +discu ssed +bad ge +p up +mi stake +phen omen +un ite +ðŁ Ľ +de pic +ri des +in augu +n at +sof twitter +comb ination +gosp el +âļ ¾ +ad mission +retro gaming +ðŁIJ ¾ +sch u +mb o +jun ction +al arm +à ¦ +gr ac +kh ali +k ul +m ale +cap tion +wi sh +te re +cor ps +ru bber +play station +er in +effici ent +l or +jo kes +in ary +nor man +lu is +inaugu ral +ch ed +âļ½ ï¸ı +di p +to e +str at +aa c +am u +pi er +co tt +comm and +tt en +sn oo +cu be +clo ses +class ical +s word +expre ssion +reach ing +n app +co st +affe ct +ric o +gi f +brea the +tri be +or tho +h ay +l g +fri es +n m +hi ding +richar ds +en de +mic ro +capit ol +cop y +ro m +regi me +mary land +tax i +di al +embar ra +un believ +ch t +v s +elim in +o dd +pen ny +sound track +l ings +trans ition +rema ining +a is +mali k +? !? +rand om +def end +ul tra +tru m +danc er +st ol +dri ve +a ver +ro ast +defin ition +se an +excit ement +partic ul +su rely +sh av +ber y +di shes +com m +is ol +i am +ob li +gho st +hugh es +chi efs +b as +conserv ative +speci al +fe min +sh ri +n ancy +inte l +tu ne +ðŁĩ ª +jo el +gg le +mo to +ðŁĺ Ķ +bu ck +d ag +antic ip +mont ana +gu id +fro g +ec raft +op e +dri ves +nu mer +x y +color ful +wednesday wisdom +illu min +bey on +inau gur +deep ly +pre fer +for tune +coo ked +ti ble +âĺ ķ +swe ater +it ter +tt y +u i +gi e +com plic +~ ~ +tax es +cu ps +di verse +sam anth +âłĢ âłĢ +ba king +sy mp +wa i +be half +mer cur +travel s +ðŁİī ðŁİ +or ia +eng aged +jump ing +reti red +n aked +p uni +speed way +sci ences +rehear sal +on ym +dy ou +pl ates +r ati +kri sh +jaz z +car ol +ra f +pen alty +tim eline +ru by +engine ers +ra f +bel le +do se +che on +esc ap +me g +ran k +or d +me gan +mer ch +ec lipse +âĺº ï¸ı +ple dge +kir k +per si +leice ster +sa k +w k +saf ely +yy y +je t +promis ed +j c +en ne +no ah +re no +re a +ðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤ +tra il +ðŁij Ģ +f d +soo o +ri min +w k +ภ² +i al +x ox +bis cu +d ale +fan dom +particip ating +fla g +privi lege +pe ach +mach ine +bo ston +gro ss +o g +mir acle +adop tion +u ss +mon sters +be ij +clar ke +pu shing +pra ying +ar o +d n +ell is +apol lo +od ds +refuge e +to w +b p +ðŁĩ¬ðŁĩ § +h end +app eared +memb ership +pe an +du m +viol ent +v y +potat oes +aw w +greet ings +t ts +ac on +sh ane +photograph ed +cra b +temper atures +cu ba +c fc +wel com +he l +in nings +m k +co de +kno ck +gra ss +swe dish +p ta +ick y +v at +lin ing +s q +sa p +ar c +announ cing +sk ins +cit yof +br ing +co x +gam er +it arian +i da +h d +ros se +sad ly +ge o +âļ ¡ï¸ı +tag s +fa ther +chan ge +l ance +whis key +adel aide +te c +stick ers +marke t +class y +bad ass +flo rence +lin er +fro st +k ate +ac on +scand al +es sex +ðŁĺ ı +vi vi +dr ill +blo ggers +recomm end +d ha +ac res +ro ma +bu y +gro cer +er ia +ma har +ff er +patter ns +ver i +com pu +st ev +ang a +ment or +do o +it ali +cdn poli +on ly +conduc t +elec tro +de f +wh ale +prepar ation +bicy cle +vi ral +turn out +bra ss +qu ad +hospit ality +pack aging +den cy +ceme tery +abo ard +dre aming +pic ture +t all +inv ent +ad mi +o e +tem ps +qu an +fun dam +pro mp +resi dence +mu d +sour i +âĦ ¢ +graff iti +gi f +d nd +com p +s war +pe eps +pale stine +devil s +san g +assi stance +bi ke +missi ssi +inter viewed +ne phew +dru ms +v and +gentle men +n sw +inst a +leban on +ee ee +oli via +ver y +rou gh +industri es +m ation +ðŁĺ Ĵ +bar rel +n ay +po ps +moder n +ill y +are st +on ents +protec ting +v ans +e o +vi kings +restaur ants +re ck +jac kie +andre w +w illing +he ath +citiz en +disc rimin +๠Ī +stu art +m ys +hi p +tran sp +" ? +te x +su shi +ke d +cro ssed +dist ur +pe dia +f ate +some how +mo th +proce ssing +is s +r in +u ts +yy c +ver t +lg bt +re id +on to +arab ia +habit at += = +stre ak +simp son +addic tion +wim ble +deli vers +challeng ing +ðŁİ ¶ +fran ch +e du +s me +ai ds +hur st +th am +tari an +remem bered +palestin ian +fe es +tru m +sket ch +ur u +fit ting +jes se +ðŁĶ¥ ðŁĶ¥ +---- ---- +ba ch +ici a +colo red +da h +associ ate +int el +s eller +p u +stu ffed +ac s +b s +sh in +cooper ation +certific ate +ab u +ingredi ents +re v +in ge +el der +christi an +bun dle +th ic +dir t +beij ing +comm it +ted dy +ed u +to day +s field +w yn +confir ms +lo o +j v +ene ss +al pha +vir us +ari um +gr ind +bri dges +introduc tion +pol ls +bac ter +z ach +termin al +ra iders +fla vor +zom bie +vo d +sp reading +gameof thrones +effici ency +lat ely +ale m +twee t +cri mes +cl er +de y +dg ed +hy un +pay ments +cir cus +ðŁĺŃ ðŁĺŃ +mis souri +lu b +episo des +c age +po s +mat ching +tumb lr +lin ed +ge st +am bi +nar r +ing ton +regu l +blo wn +is le +co co +on don +joshu a +tour ing +sm a +sau sage +best friend +bo eing +desi re +sav age +ra pper +de vo +te ar +take over +cow boys +po ker +par ag +pp e +h int +we ars +se th +ro les +l anc +man ga +form at +fl yer +c ay +mo or +ba ke +spla sh +v ad +ker ala +proce eds +sil ly +reflec tion +di str +wi d +su it +ci vic +yan kees +by n +migr ation +di stin +or ch +fe mini +quali fying +tu ri +o be +hun dred +cra p +wan g +mathe mat +bu re +expo sure +fergu son +seme ster +re serv +pl ym +a hu +fac ial +wa x +wor ried +ca b +vi o +as a +co d +to pics +p cs +hal o +rescu ed +horiz on +ar k +âļ ª +hol ly +el f +ul ti +pu p +quali fied +attend ance +ati vely +destro y +y c +for th +photoo ftheday +c ents +ic eland +meas ures +de sk +port folio +artic les +direc tors +dat ab +e w +creep y +oun ding +hon oured +mi st +j it +men tioned +port able +iti c +d ann +friday feeling +am id +ti ger +scri p +helicop ter +hard ware +expl or +work place +austri a +beat les +ber nar +spi der +disc o +cul t +lim its +shor tly +fin al +nin ja +lu ke +le bron +wal mart +o il +van illa +shi re +ye g +ak y +c s +bl er +collec ted +t g +rol led +speci als +b ff +pier re +sh im +vi er +flash back +restor ation +individu als +pro d +fre aking +tu rer +o a +re fre +mor oc +gre et +re yn +care ful +our ing +u sh +is d +g ill +vie w +thunder storm +b led +pic nic +guar di +pi g +ar k +syl vania +bann ed +u cl +vi jay +ori um +av engers +believ es +eu r +monu ment +concer ned +la bs +ber g +a ap +vi sh +sing les +can cel +z el +ar ab +ru th +too th +ar ta +sh af +chair s +r ack +dise ases +crow d +cl y +fle x +christ ma +artif icial +tom at +fin e +dra ws +advoc ate +fran ce +Ù Ĭ +ðŁĺ ³ +heav y +s our +compre hen +no ble +aa p +hin du +cor al +g ars +ow en +n l +st all +yel low +mar ina +in ver +suppor t +tou gh +promis es +pi e +master piece +sco re +for ce +mor tg +crypto currency +o x +r ors +rock in +pro vin +ho g +no stal +oak land +pat rick +inclu sion +tra ffic +ah med +a ha +lux ury +con secu +de mon +âĸ º +b lowing +st ag +: " +encoura ge +ben e +sku ll +do dge +bu ster +kin son +wit ne +er ror +lo west +fel low +à ° +sh re +bl ur +vir gin +compos er +sli p +mor nings +ga ins +tab le +gra in +ari st +braz ilian +w we +tu es +ribb on +an ag +di st +sac rif +em brace +entreprene ur +af fili +de o +t ali +touri st +fat al +ì Ĭ +autom atic +ðŁĩ µ +we ak +wel fare +confir m +benjam in +fi ghts +alleg ed +me ad +strugg ling +pro secu +che f +à ¨ +propos al +er n +ðŁĺ Ħ +dy k +on gs +hon g +m ack +mel on +on ent +ru sh +d ap +tol er +pro pag +c ze +trans lation +wal let +cott age +sa il +constitu tion +ðŁĴ Ģ +mun ici +fav or +storm hour +i h +ðŁĺ Į +approach ing +pin ned +j ed +niger ian +n ach +sh at +particul arly +mc don +camer as +anni e +admini str +he at +electr ical +char ming +gib son +bouti que +ex posed +ac tor +pil low +beach es +genu ine +margare t +ben nett +lou isi +pos itions +el y +shin y +ten tion +architec t +ren tal +ac qui +goo gle +sub way +mom ent +ðŁļ ¨ +ri m +metho ds +cy cli +nor folk +Ù Ī +over whel +ra pid +we ar +happy birthday +progre ssive +ðŁĴ ¥ +co gn +pap a +f ool +philosoph y +pol ar +jim my +wi g +ðŁĴ ĭ +oper ating +reduc tion +ph i +fla gs +to the +o di +a res +k oo +k ang +ar kansas +ash ton +wimble don +sci fi +attrac tive +mississi ppi +logi sts +ral ph +la bel +gradu ates +ma ha +home town +âľĮ ï¸ı +foun ded +on the +li z +trans l +mini mum +pre sti +ta m +gener ations +re bel +journ alists +par am +mc m +acry lic +death s +tes la +w t +bry ant +jer us +i stanbul +muham mad +ri ley +k ris +work shops +is o +coun ts +stre t +prote cted +trin ity +man ual +r hin +r il +pleas ant +le mon +ner d +har der +dar ren +bur y +ra h +bas is +mi gu +occa sion +li sts +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ı +e b +de cre +hamp ton +ìĿ ´ +tra vis +trans form +puer to +nh l +av oc +tri ps +unexpe cted +ve t +di dyou +bar ber +st ages +m son +re presented +for t +l al +pp le +nic ely +ignor e +qu il +qu inn +h k +carri er +remin ded +am ong +pass enger +el len +gue z +sc ape +mu ral +youn gest +ma sh +d ill +rout ine +stain less +jack son +gand hi +th al +on ers +edit orial +convers ations +sd ale +autom ation +i ke +า ภ+ðŁĩ ª +hau l +la ying +men tions +am en +abor tion +i bi +coun ties +ca therine +man ds +jam e +roll er +au t +n am +o logical +cep tion +ran king +tox ic +sn acks +victor ian +bang kok +psycho logy +re g +ang ela +respon d +sty le +sophi e +dak ota +achiev ed +mar ked +imper ial +in as +glo ves +sli m +confi dent +att acked +gg er +lon ely +valentine sday +re b +craft beer +orig in +zim bab +ce iling +te ens +other wise +w b +f ers +day sof +advis or +y ah +âĻ ª +en der +republic ans +av a +skir t +pi pel +chi e +jan e +ja x +ðŁĺ ĭ +âľ Ĭ +j ays +bre tt +bal o +cru cial +d har +as is +de au +lloy d +chat ting +âĿĦ ï¸ı +rel ay +remark able +n s +we t +bris bane +ðŁĶ ´ +tion ally +f k +la yer +house hold +consecu tive +es is +pend ant +st ir +crit ic +su gar +photo shop +pa res +arti stic +do dgers +c un +cra fted +am end +bo at +âŃIJ ï¸ı +egyp tian +sa w +tra ge +small er +ox y +pa ired +nex t +i res +tac o +o y +u c +st i +a erial +: // +dr o +dot com +gg ins +r pg +ay e +le an +stri ker +lo bby +prote sts +pri ority +congre ss +am ate +inv it +r ington +mom my +th us +allow ing +pione er +enfor cement +g ori +tal k +dra g +du mb +bul let +san ge +er y +tar gets +ðŁĩ ¦ +he ather +consi der +seaf ood +ve st +ris ks +% . +p g +sac red +he ating +kick ed +tto t +. - +chan di +co ven +po ol +pul se +i a +ro ster +shakespe are +es a +car go +pean ut +tro op +ac tion +tab let +home work +cast le +stru ction +mus icians +free zing +bu tt +justin bieber +j j +bah rain +an them +au dit +didyou know +na vig +guid ance +âĸ ¶ +tur f +n un +fic ations +ye men +char ging +x c +bron cos +su bur +p ale +bor ing +among st +for the +em per +om fg +p j +expe cting +ðŁĴ « +st l +ad min +expect ations +sw an +shoo t +oooo o +min ent +ãĢ IJ +wall ace +stan g +satur day +adop ted +dou bles +hom ie +ome z +d han +vent ure +surroun ding +fi le +mob ility +de es +w ski +broo ke +emb ro +re members +kar a +test im +bo tan +m tv +sacrif ice +jerus alem +d l + ´ +proper ly +ili on +as i +leg it +co pe +m cla +recy cling +lar ger +ðŁĴ ĵ +pat ric +gener ous +ja red +p f +mol ly +thom as +ju dges +h b +sor ts +bl vd +o ven +enter ing +plan es +be et +integr ation +boo ked +fre ed +ver n +ash es +to pped +de pot +welcom ed +ren a +m ick +d and +see ks +gam er +ran kings +ren e +mu t +whis ky +fire fighters +gu es +ga ther +tour ney +de men +y ang +new ton +autom otive +back yard +deta iled +mi st +to bac +fi ber +un usual +grat itude +sp are +ne ys +: * +per i +flo ating +fin alist +don ating +dre ss +bro ad +be the +econom ics +tai wan +ed wards +plu g +pra iri +val en +bab a +f ad +an as +har per +dis order +app lied +p att +bi kin +li ver +cu ri +carol ine +ann er +juli an +wal king +mal col +screen shot +co ding +skin care +activi sts +myster ious +ex act +blo cking +mercur y +bat ter +du mp +âľ Į +en se +li sh +ridic ulous +prote sters +ðŁĻ Ī +lu st +swe at +as s +ali ke +co dy +re ments +win ds +as pir +vi enna +pra y +.. .@ +bo i +cand le +assi sts +te e +der son +p ony +f ence +con spir +âĺħ âĺħ +oo th +e pic +ba rely +a unt +b am +diamon ds +end less +scre ens +can cer +gr o +p st +pro spec +mo sque +help ful +ou ri +bro ther +gu jar +cri sti +ine z +to wers +ad dresses +gra y +bur ton +re tweeted +ðŁ¤ Ķ +n ity +du ck +super vis +jo an +kin der +sanc tu +pi ed +âı ° +ł ï¸ı +m ati +reven ge +ce ster +eli fe +desig ners +back ed +bo li +wei ght +cou ch +su res +s its +shri mp +la gos +auth orities +os ity +hol ly +compu ting +fac tors +ab e +pan els +ram ad +sent ence +missi on +hol m +r b +d ads +shang hai +mon ey +she ets +sk ate +thre w +cup cakes +infin ite +l is +practic ing +ess ay +ka i +as ci +mo b +u gh +hol mes +re gg +ik h +mo ck +collec tions +pe p +o va +sal t +nan dez +co y +thre ats +tex ts +cin nam +pregn ancy +pen ding +stam p +flow er +g is +agre ed +pay ne +ro ver +ph ra +sof t +f fin +fa thers +pass engers +aw ays +al a +h es +li van +in s +samu el +ingu i +h of +j j +chen nai +cat al +om ic +he ath +ni ece +pump ed +integr ated +are l +no m +produc tivity +wan ting +vis a +di ana +tw il +it v +cam ps +ro wing +d ley +black and +gu ards +b ells +re verse +vi be +ric ky +mo ss +ny t +âĺ Ģï¸ı +el le +tro y +cu dd +ev an +women s +fo to +mi stakes +wick ed +mi l +c led +me mes +co smo +schol ar +ren o +ðŁĺ Ģ +v ents +# âĢ¦ +terrori sts +ca sey +cardin als +ðŁĺĬ ðŁĺĬ +venezu ela +bol a +liter acy +t w +en o +con tains +au stin +fin anci +ev an +har vard +origin ally +chev ro +her ald +nott ingham +manag ers +âŀ ¡ +accep ting +wal sh +tutor ial +entrepreneur ship +yach t +requi rements +glen n +pe de +unfortun ately +ach ing +dais y +gi an +night mare +âĿ Ĺ +r ina +b art +ema ils +oppo site +who m +sa ke +pu zzle +da shi +par ty +blan ket +bus es +lo re +beau ty +reas on +pun jab +winds or +func tional +exi sting +hel lo +gli mp +con vin +la k +scre aming +rebec ca +bli ss +north west +infin ity +cosme tics +pul ling +coffe e +pl ing +op ho +colom bia +interior design +( + +emo tions +sa c +sun glasses +sav es +d f +six th +al y +ðŁĺ » +de en +dev ast +polit icians +lac rosse +g u +pe i +jav a +comb ine +coal ition +er ts +survi v +ch ad +stri an +n n +de vi +coun c +concer n +contro ller +bre ast +j ury +tu m +introduc es +la di +mobi le +al z +ste ady +nur ses +h acking +on line +oce an +ðŁİ Ħ +a am +ju ven +ic c +louisi ana +ar te +street art +is on +wn s +fr m +p anda +no ir +main tain +del ay +symp toms +thor n +ge ome +ter n +carri ed +p ru +pan or +as sy +per u +clou d +sp ra +pe di +e ste +tag ged +ðŁĺ Ŀ +shado ws +naz i +ا٠Ħ +cor ri +âĻ¥ âĻ¥ +j ad +ðŁĩ « +form al +spo ken +ðŁĮ ŀ +enjo y +lo pez +out look +in ho +w ander +Ù ħ +ma ya +pe e +d ine +ãĢ ij +brief ing +suppor ter +ar ily +ght ers +natur ally +doctor who +j en +v ar +new year +re se +si mm +re x +con sequ +tomat oes +bur st +bra vo +bur gers +cr acking +nor theast +bi om +mush room +mar que +dou ble +ni er +v ag +tw enty +key board +win ni +jama ica +par ish +: - +mental health +ali zing +ren der +wa king +ðŁİ Ĥ +g ly +na than +wa shing +mel issa +jun g +loy al +chil i +song writer +guit arist +bo wie +neighb ors +onym ous +as set +ta i +head quarters +ðŁĮ Ī +i hear +ci gare +sur g +) " +re pl +dar ling +ðŁĻ Ħ +z ak +sa re +ãħ ĭ +mic key +ware house +mass age +ine es +did nt +i w +hur ts +eng aging +mag ic +women in +k itten +mor s +c art +tit ans +colle ague +compe ting +er an +k hal +mar ble +dem and +del ight +et ary +bli zz +lou ise +m ls +fini shes +experim ent +conduc ted +electr onics +itt ers +car ing +wh ats +sym bol +jun g +e cu +pi x +con text +char ger +ðŁĺ ĩ +re ig +fra g +ë ĭ +ch ad +tru e +ker ry +def ending +a int +au ton +check out +bar nes +less ly +d t +m me +clou dy +second ary +are z +_ : +app a +const ant +" ) +ve ts +jo b +i ent +ðŁĺŃðŁĺŃ ðŁĺŃ +m j +fren ch +di ver +davi es +hh hh +e book +๠ī +mar iti +bree ze +susp ended +mat o +vi et +ra hu +se i +bol t +en ary +le is +kar l +fr amed +expla ining +ab c +de aling +nat o +ja ke +exp and +leon ard +establi shed +du b +ar men +el led +voc al +nichol as +ori ent +k yo +illustr ated +ah h +danc ers +milli on +ge ta +po pp +as u +mur dered +gi ble +sto ked +gri ffin +maxi mum +adri an +en counter +ther o +david son +ðŁį » +holi day +ev o +asse ts +car son +memor able +âļ ½ +ob am +represent ative +cb d +tr icks +vo gue +vo ice +mm mm +sebasti an +cli f +ath y +par alle +ðŁ¤ · +pa k +ev acu +e ats +ا Ø +tou ched +organ ised +spir its +can ad +gui ded +frame work +ðŁĮ Ł +pe d +natur al +ag ar +replac ed +anch or +ti t +sha h +organ is +super ior +r n +ch ro +eric a +st ill +cor on +chu ck +loc ks +or gan +ro sen +sc am +ben ed +/ # +ke en +tre vor +vamp ire +sor ted +! ' +af ford +in tro +gr ace +ðŁĺ ľ +sau r +kick starter +influ en +v u +y up +po c +ðŁİ ¥ +a ar +s ang +tre k +et sy +tb h +scre am +chevro let +pix el +shepher d +an or +gabri el +tw ood +sd cc +me ters +develop ers +clo sure +v w +twit ch +ì Ĺ +se oul +pr ice +ho g +n ish +hill ary +scrat ch +in cen +wag on +dis ability +pan ther +ch ats +g d +wit z +sus sex +l ate +den mark +ger ald +cancel led +net te +i x +nav al +bap tist +te t +y ad +ma th +ho y +r andy +po int +intel lec +fru its +w ool +gu in +pr on +the ft +con dem +mar ry +n ola +architec ts +cin cin +roc kets +gentle man +ex plan +t ate +do e +ra ises +wild life +w l +insi der +blan c +w p +for sale +ny c +po well +unbeliev able +pen s +goo dies +mu stang +p ens +st ays +squ ash +xox o +near by +ever ton +co co +le agu +k han +stu d +south west +con struc +s worth +cro atia +le a +su ms +aim s +e an +van ess +iti ous +pa thy +arc ade +b end +sugge sts +sac ram +roy als +ri er +em ir +in cl +an k +clar k +ri ght +vac c +ठ¾ +tan e +li b +u sc +sal es +hu h +s ally +ver a +p ga +gro ws +dru m +tre e +eth ics +sug gest +is ab +se aled +pre viously +anim ated +ab du +ri ses +glo b +pre dat +scar f +del ic +om ar +ll i +sx sw +py thon +ne bra +fun k +reflec t +pav ilion +tic ally +ch asing +bak ery +inva sion +ko h +believ ed +co hen +con qu +cra fts +nat i +cle ver +govern ance +sam ples +fa ils +â Ķ +ti mo +r itu +stri king +inclu sive +sho cking +can t +requi res +dra wings +à¸ Ń +purch ased +du m +z ach +war ner +con sole +man sion +foun tain +circu m +e sh +is land +mil k +pro fits +hali fax +ri val +âľĪ ï¸ı +jen ny +sand ra +ny e +k elly +y al +qu ad +no s +inste in +fin alists +mid fielder +cu e +excep tional +a an +sa pp +gett in +sa a +f ati +sl ice +vol k +s wal +la sting +sum mary +it as +sm o +s z +âĺ Ĩ +ip l +fl ames +ene ws +ha v +hoo die +pitch er +win dy +re vol +centr al +ton ite +ðŁİī ðŁİī +sol ved +mil wau +organiz ations +wee ts +re fin +s th +ãĥ ¼ +el in +ton a +cinnam on +ðŁİ ¨ +ðŁİ ģ +ron aldo +pen insu +ome ga +el ds +desig ning +e igh +blu et +ben z +nu g +ash a +robo ts +su dan +choo sing +en do +ser ge +clo sely +hand y +fing er +be ing +ar te +survi ved +fl ame +mile stone +gu t +d war +fu tures +é e +el o +fri dge +eli c +ou ch +u b +p v +tit an +col lar +st ation +nev ada +aur ora +r d +dun can +âģ ł +bri en +mar sh +Ð ¾ +to tal +ch ry +s ers +su ffe +ra chel +colle ge +to days +cour ts +ch it +re united +gym na +gen esis +be side +re presentation +ch ant +collec tor +ra k +ath ens +ni gh +mun ich +langu ages +fl u +particip ation +__ _ +c v +spec trum +so da +co ver +refe ren +ab bo +ap a +public ation +ed m +mon ica +ar my +ðŁļ Ģ +div or +dr y +stre ams +robo tics +ci der +bull ying +appro val +sto ke +plat forms +sier ra +ex tin +i b +ha yes +succe ed +suff er +at ically +da i +lyn ch +h ound +del ines +ack now +d ated +exclu sively +he res +fac ilit +dam aged +char ter +la kers +fal con +unve iled +wel ove +e ase +pati ence +l one +gent le +gene tic +produc ing +g our +shann on +bil ities +zimbab we +p int +dau ghters +liter ary +bel le +cl am +surroun ded +k any +ne il +pir ate +rang er +hb d +nat alie +bel ong +olym pi +emb assy +sc ol +en er +ak in +lo ren +b h +: / +di va +den im +hi pp +ðŁĩµ ðŁĩ +arn old +? ' +we ren +em power +dis abled +man or +rasp berry +b af +aw ful +dru mmer +kar dashi +n ash +machine learning +ch u +rebel s +tim ing +mon roe +ton gue +ran ge +pup ils +re ss +amaz on +b z +har ley +pal mer +ballo on +s ings +ic ec +j b +c ers +g ps +whi st +ri se +l t +oo oo +c attle +shoo ter +vod ka +uc l +mt g +le sli +jon as +di spo +at ric +ste in +vintag e +fir ms +flo yd +cow boy +soo oo +is aac +war craft +disney land +beauti ful +be am +franch ise +bu n +k ag +an on +tur bo +swee p +made in +kar achi +dete ctive +penn sylvania +contro versi +vitam in +a side +chron ic +descri bes +remo val +ha h +ap er +ten ed +u to +bad ly +mir ac +f ry +ye a +in jec +ther mal +comp act +th or +te ed +ur gent +l ite +g illi +sop hom +ic o +che m +p m +for k +fre ak +ch ak +recipi ent +i y +ni k +model ing +c ans +ðŁı Ģ +del ux +se am +surviv ors +rad ical +investig ating +reli able +f m +tur t +ligh thouse +to ol +go wn +) ) +bo ts +auto graph +a id +bu ffe +h mm +horri ble +ssi onal +ann i +๠Ģ +k its +sch i +eter nal +hu ss +sens itive +r u +tast es +chec ks +im o +por tion +sk ate +e den +half time +fri ed +ri hanna +ti se +fl ick +ca in +s gt +âľ Ķ +sh au +sta ined +ra ffle +dro ve +sal man +princi ples +sh o +ar u +je ss +gu ine +gar bage +my an +jel ly +dis ru +z ia +q ld +ent ries +la v +fle w +ad mit +objec ts +comp are +ny times +cann es +p n +suff ol +ro c +d ana +e gg +hi st +coun sel +' ! +phy si +imag ination +ad just +explo sion +plym outh +hor ror +elli ott +bour ne +de x +bre ed +au dio +lob ster +disappo inted +nation wide +( ( +incre ases +austr ali +ce dar +star ing +rac ial +e is +g mt +visi ons +stay ed +discu ssions +de an +cur tis +mai den +stel lar +happ iest +h wy +pre season +car av +mon days +hospit als +glimp se +schol ars +ja i +ter race +ann a +goo se +gra ded +lot us +hun g +grocer y +stam ps +emper or +sc oop +in ser +c as +exist ence +he al +fal cons +mar vel +reduc ing +terri fic +magne tic +perfor ms +bar re +p us +tre ating +ic on +w h +decla red +tra uma +do d +come dian +nik on +bu gs +as m +mont gom +ibi za +comprehen sive +ha s +san ti +fellow ship +da sh +p sal +louis ville +sp y +fau lt +d the +fi led +vi sta +de sc +fe ars +you tu +sp s +es p +ri g +cri me +ber ger +wonder land +k ent +in formed +stev ens +my th +ast on +ir i +visit or +at ri +produc ers +al la +person ally +separ ate +agen cies +af ri +il an +spo ke +n ina +squ ad +di ves +de pend +li v +fier ce +enter taining +cha in +sc at +bor ders +pal ette +sp ro +os is +der by +tobac co +zi o +willi e +ju vent +zoo m +hol y +enti rely +af e +mart inez +be ds +pe a +bull dogs +ðŁĩª ðŁĩ +ib m +ne on +ethiop ia +team mates +plan ting +tw er +any time +for bes +ó n +run way +ner vous +ro ger +p ile +ch anc +apo caly +u w +o i +dr ought +territ ory +br ick +cre atures +go in +w aff +gre n +sou theast +je an +am bul +ed ited +stra p +c v +aar on +ãĥ» ãĥ» +t su +descri ption +kin dly +clu tch +im mer +en or +women sday +or ange +ra g +ob vious +hy der +chann els +man go +me yer +ra ining +ge tty +pil gri +coordin ator +up load +ninten do +don uts +san chez +app arel +j r +zz i +, @ +jeff erson +accessi ble +great ly +e id +initi al +budd ha +par is +ma scot +â¬ĩ ï¸ı +sch war +si ri +sp inning +mortg age +e cho +end ange +ge dly +chlo e +enh ance +kar nat +k ry +explo res +ðŁĴ ģ +af fair +ic als +all a +dar t +dolph ins +diffe rences +squir rel +au gh +dr ones +ell en +re store +pa w +un for +pi ke +hil ton +colla b +consu mers +co inci +out comes +pp p +a q +coup on +li est +si ms +k ho +av es +spo on +pu dding +cor byn +hat ers +ex ams +sla ve +. ! +p sa +app les +tam il +se d +co ke +zz o +lo sange +car bon +cla ir +... ) +k hu +cra ig +explor ation +sanctu ary +su e +al way +demen tia +won ders +super hero +pakistan i +brown s +bluet ooth +lo cker +mar c +ev entu +delux e +rodri guez +âĿ¤ âĿ¤ +ro bb +ðŁĴ ¦ +lin ux +ten s +intellig ent +se ed +vo ter +s ler +pe aks +inter n +teen age +peninsu la +hand ling +ti e +cou sins +wen dy +me e +à¹Ģ ภ+din o +ðŁĴ ° +ðŁĺ ĥ +ze e +s bury +trage dy +b k +bo re +z in +war ns +idi ot +tou ching +contin ental +tac os +saf ari +wa shed +po dium +morri son +fore sts +c bc +al on +partic ular +be ads +inv ented +lo ch +li ghter +where ver +i de +docu ments +a we +k r +no where +min er +st it +ro x +contribu te +har dy +cl an +ob ject +ca it +ðŁĴķ ðŁĴķ +happ ier +vege tables +t art +g ag +nom inee +heav ily +pan ic +j d +there sa +at m +u ph +s fc +su ri +drin k +n al +re vel +k l +avoc ado +nom ination +ma donna +shar on +malcol m +control led +sh ers +revi val +legis lation +shoo ts +n in +comm entary +pro s +human rights +str anger +mit ch +pipel ine +leg ally +th u +gil bert +tol l +gran ted +gh s +ir anian +refre shing +du k +ab i +pri me +jose ph +mo sa +stati stics +produc tions +mer ry +pat el +sa x +human itarian +struc tures +e missions +town s +fre el +ster ing +rat ings +alle gedly +cab in +st l +w ade +fl yers +tri m +promis ing +z u +bal lot +compar ison +free ze +ou ter +great ness +as sign +snow y +r ale +tor ies +med iter +kno ck +consult ant +cincin nati +analy st +sc oo +je ws +appro xim +pu re +portra its +cy rus +ation al +lo ans +acqu is +el u +accep table +uni on +water color +ru st +batt les +per fu +seas onal +ser ial +mind set +ri ot +fel d +enni al +clo set +pri est +tan ks +int l +scre w +bu m +ab dul +ou x +expla ined +ric a +imag ing +law yers +bu ried +ãĥ»ãĥ» ãĥ» +ear l +âĢ ķ +l ton +resto red +stri pes +fo ss +de mands +ste aling +alex is +mun d +ak er +ur us +war dro +hu gs +gen re +e go +Ù Ħ +particip ated +bab es +ban quet +ti ous +he mi +ds b +lo st +milwau kee +jen ner +ge m +ou tra +lo ses +id i +re ps +ðŁİ § +regu lation +fla w +f ang +vibr ant +ram p +ra ins +well being +so viet +vie wers +de po +libr aries +bi go +ser y +g ill +de struction +co z +c x +bri dal +al ds +plan ted +amate ur +lu d +che ering +show cas +pro file +i u +ver tical +pack ers +wiz ard +ski p +s light +be au +air ways +mu ch +re ra +ðŁĮ Ĭ +ab sor +pati o +pack ages +s ells +ment ally +ðŁĺ ¢ +reyn olds +k are +tri bun +wal t +kn it +ta ste +sur rey +boun ce +cre ature +b are +bet ting +su re +mi ley +laugh s +al ore +cy n +t l +arti st +ann ah +war mer +dynam ics +lunch time +mariti me +vulner able +ðŁĴ ĥ +wol ver +dur ham +const antly +am in +si bl +: @ +bul let +k ach +angel o +wil der +doo m +desk top +law suit +k ca +hen derson +inv iting +bet ty +ta wards +ra fa +le aked +and i +ge ms +af l +vel o +mediter ran +pro be +to tten +steph anie +sn ation +com be +q s +over come +assas sin +ra v +fil ip +winni peg +sh il +determin ed +k as +ou tre +regre t +gui des +aa a +ðŁĺ Ī +wi ves +mani fe +er ly +sm y +sh ima +x ing +pix el +jac ob +ac commod +to y +on o +po o +ti er +an swe +ðŁĴ ģ +ro sa +le ase +bel ongs +th ar +eventu ally +nei ther +go a +ski ing +at ra +ag h +broad casting +f ury +py ram +d ice +volk swag +wom ens +provi der +bom bs +miss ile +whi p +d ick +nor we +back up +el der +mat ure +concer ts +gi ous +sque e +good morning +bra ves +^ _ +au ssie +lun a +mal es +he ck +for tn +rome o +steel ers +p n +pe er +re presents + « +kat y +migu el +requ ire +cha ins +l ur +immedi ate +ti mber +âĸ¶ ï¸ı +advoc acy +ex port +an z +tiff any +auth or +ðŁİ Ī +du des +chil ly +hi d +har m +bu g +mon ster +terri er +tu c +story telling +ta k +in ti +immigr ants +b is +reach es +com passion +john ny +contribu tions +ðŁIJ ¶ +mechan ical +impre ssion +ran ks +ko be +men ting +bloss om +pab lo +buil der +bom bing +tw el +sul livan +om o +pe te +de mi +ku dos +w bb +t gif +mass ach +neighb or +che fs +eng ines +pun e +ga ined +phan tom +s days +ext end +gr an +cent ers +jac qu +dat asci +sleep y +el vis +answe red +s lot +con y +flexi ble +ti ally +le tics +% , +andre ws +si ble +mom ma +vin o +do x +invit ational +twil ight +j ade +ill ery +joh ns +f ou +p v +-- -> +break down +billi on +prin ter +mon d +c bc +mag gie +legi on +du b +kur t +po or +paren ting +regi ons +bikin i +be ware +si onal +au burn +kid ding +amp les +sp an +con tempor +c ic +ha bits +ak o +pre fe +bud dies +it z +em ily +person nel +moun tain +ver sus +ðŁĺ ¬ +ear ning +s ink +dar i +u u +s win +i ster +bru tal +n ac +kat a +clo th +am and +ðŁĶ Ĺ +ne o +alu min +week ends +nebra ska +co des +delay ed +brun o +pro ven +in c +i ght +fl an +or o +lam bert +regu lat +w f +massach use +kardashi an +bern ard +fi esta +volcan o +grand pa +anc a +d re +st itu +mean ing +fo am +au ck +at ed +r l +hot el +pers ons +dy nasty +ell or +ma i +am ne +sty ling +avi er +e g +vege tarian +, âĢ¦ +foun ders +sta in +g d +cy cles +sky line +trac tor +exi sts +tra l +kid ney +mar il +inst ag +se tte +addic t +tri angle +flash back +controversi al +z on +p ins +i as +tr ay +town ship +deleg ates +sp am +h ms +cr ane +peop les +o lo +fac tion +but es +on ica +deleg ation +new profile +eli er +mc a +w and +g ely +losange les +ber ke +ti ve +dis rup +zz a +cas a +jor dan +ford shire +ga thered +ic hi +atten dees +à¸Ń ภ+pe ppers +co in +bour bon +ern ity +ro tary +behavi our +jere my +team work +compli ance +tre mend +ðŁĩ § +bu hari +cam bo +bu yers +ha gen +bu ds +bay ern +mon te +sm ells +an za +ath lon +descri bed +work force +gi ving +ap i +invest ments +da il +sel ena +datab ase +th um +mor tal +stu dent +bu yer +do ver +gar ten +att le +loy alty +gen oci +holo cau +theat ers +ru ling +ven us +pat ent +ch un +ab by +awa ke +mass acre +bang alore +break ing +simm ons +ju sti +hal e +ed chat +gg les +haw k +mar king +head lines +stro m +co ve +breath taking +med als +hair cut +christ ine +tele graph +gujar at +ju ra +can e +sho re +propag anda +mu eller +.... .... +sa vi +stom ach +thro ws +ta b +war m +j ong +reno wned +hi r +ra is +mush rooms +guaran teed +bo a +m j +revolu tionary +certi fication +bru ins +jo in +w es +pas sport +c g +sex u +cap able +w v +ton es +jac kets +ac compan +spin ach +fore ver +bla ir +wat ts +g l +cou ples +prairi e +newprofile pic +logi stics +massachuse tts +jagu ar +o id +we al +under water +mo z +y i +ma ths +myan mar +pre ps +suffe red +tr ace +wal i +ah hh +bor g +st itch +cu lin +real ise +infe ction +discrimin ation +sh ame +an kle +hu mid +y t +brac ket +tru ck +tri u +ea ster +commun ity +post card +invol ving +ty ler +car amel +over view +ex amples +integr ity +base ment +instru ments +ani um +at us +gh er +laun dry +achi eve +gen eva +pr icing +hyder abad +beli ef +me ta +j aw +accoun ting +lead er +cristi ano +cou ture +cy p +vis ed +, ,, +k nu +h ick +break er +br am +ra b +mo or +ham as +gradu ating +pupp ies +ak h +ta h +ach es +ri e +op ini +g ta +re ign +tra gic +re ver +p ill +pine apple +tou ches +da re +le ys +il o +inter iors +sc outs +bar t +en zie +don o +bro ck +christi ans +ense mble + · +cine mas +new port +air line +win ston +le igh +cont ents +pre scri +ur ge +tr out +fic ally +il ia +sub si +are r +âļ¾ ï¸ı +w ounded +ðŁĻ Ĥ +pe pper +ðŁĴ ŀ +fit ted +af f +re sur +thursday thoughts +z ero +archae ology +di v +je e +i on +awa iting +co zy +beauti es +bal d +dat a +gri zz +stal k +kin ds +cle ared +jess ic +regu lar +ali ens +plac e +bo s +bi zar +thisi s +ðŁĴ Ģ +totten ham +ma fia +s lam +ari ana +car roll +back pack +care y +uni v +r g +pe p +dig it +tatt oos +ag on +volunte ering +diffe ren +consu mption +ka thr +head phones +t shirt +o b +ele ment +re tail +sh ru +al gori +contain er +consci ous +fi l +com ing +ra sh +u rope +def ine +gi or +femini st +flow ing +rout es +gl aci +fer t +somer set +ant es +twee ps +$ $ +h our +endange red +year sof +ro h +po pped +bac king +ba sil +bra ke +mon aco +lgbt q +pra gue +ut ility +cas si +gate way +haun ted +sch ul +ðŁİ µ +shou ld +walking dead +comple ting +dann y +montgom ery +pengu in +ss i +mer chandi +ðŁij ij +chur ch +h ates +cap tain +brea thing +ce t +fair ly +approach es +compan ion +surpri sing +kany e +pe y +hin di +targe ted +lor ds +de ut +di gging +ger man +ru t +ener gy +close st +y un +apo logi +ภ± +s ack +ru p +dd y +port al +d ough +b ats +ðŁĵ ° +at ur +graph er +pi res +mo tors +ðŁĮ ¹ +j c +dan g +tu k +clu e +us c +pag e +d less +bro ws +ju s +ad ing +re marks +oo m +car dio +ste fan +arm strong +âĢ¢ âĢ¢ +ni est +belgi an +bi op +so y +lo f +í ĥ +q t +flashback friday +ce e +ģ ภ+wre ck +mar ines +amend ment +wardro be +vo y +bur ned +guit ars +ra inf +li fel +ssi l +oun ce +exter nal +c key +me sh +she ikh +inv itation +sugge sti +pop corn +phenomen al +an onymous +tun a +chic ago +o val +del y +loc als +( & +pro f +no vel +fin der +spar ks +la ven +in fu +nic ks +qu ant +ra e +exe c +dist ingui +st ances +mu tual +sh al +unve ils +edmon ton +zan ia +a dio +vie wer +brad ford +audit orium +qu is +re act +htt p +l ero +chee ky +impac ts +ta k +ed t +desper ate +t ay +ì Ħ +sett le +bar gain +resu me +un ite +thro wn +ke st +se ys +mar ching +am it +decl ine +sch ar +me tr +stan ford +lin ke +ber ra +dol ls +rug by +jam i +b or +road trip +dino saur +mi k +sun der +re m +b k +over seas +nau ghty +imple mentation +iam srk +lun cheon +fir ing +mi ami +pere z +the e +z on +gi fted +con version +ceram ic +¡ ï¸ı +pe dro +ì Ĩ +v ick +! @ +he ed +si d +b w +docu ment +pl un +gr ants +fant asy +predic tions +vali d +car ved +gradu ated +ðŁijį ðŁı» +nation ally +ch y +af l +re sso +blan k +ri vals +j ig +e ties +om ics +une mp +b ound +sk o +inspec tion +par al +high s +cri sp +b ans +ob a +[ @ +co spla +costu mes +rec all +mou th +ni gel +b ts +ter a +ko v +do cs +west minster +dic t +gra vity +kar i +ro gue +t ted +war k +ida ho +w end +aw i +queen sland +proce sses +cli ffe +m ick +com pens +op ol +the y +cl ari +wiki pedia +salman khan +haz ard +pre ston +swee test +pd f +che es +tr ilo +south africa +bur nt +( $ +con tain +t p +sub mitted +sound cloud +at u +re z +word press +corru pt +n f +ma ker +í ķ +par as +adv ent +ri al +ca fe +fo ssil +!!!! !!! +co ws +c j +sp ur +institu tions +land mark +ent it +re ut +h is +alz heim +we mb +regg ae +mo squ +st at +identi fied +deal er +re am +re land +ten sion +ðŁĩ © +wra pping +deep er +fr at +red dit +ar is +moroc co +.. " +b low +ma pping +pri orities +ing a +swa p +re wards +conspir acy +creati ve +c j +congre ssional +vau lt +ple x +sophom ore +shad ow +ele ss +ðŁĺ ħ +dar ts +aldu b +anno ying +pro ps +n as +alumin um +h bo +offen se +j ill +oni ons +la ur +ta e +har dest +sh ro +ga ining +meas ure +ed tech +cyp rus +tar a +ang eli +car lo +go on +all i +im plic +ju pit +resil ience +ha il +bal anced +) ... +joy ce +gr a +th eli +defin ed +shi pped +main ly +min a +l m +sac ri +o ber +p im +claim ing +ent ers +co rey +bo k +cri ed +cool ing +dani elle +pharmac y +thor ough +ca ke +k lo +outre ach +z ens +digital marketing +val ent +sn p +her b +mr w +caf é +cap tures +no tre +triu mph +pan cakes +cu mber +spi ke +d ation +bi gg +sp er +crit ical +am al +too th +foun ding +a stro +' # +quan tum +th ames +un c +pri de +air bus +kno cked +un defeated +mediterran ean +cal cu +clo wn +sens or +ham mer +for give +cu shi +ber ry +maje stic +elec t +polit an +g ta +k ari +bur ke +sea hawks +volkswag en +re i +landsc apes +cas u +grand father +list ened +/ / +star trek +rainf all +fur ry +vi er +star k +rif le +ff a +leg es +hillary clinton +min us +correc tly +architec tural +pre ce +up side +box er +ðŁĻĮ ðŁı¼ +is ai +de t +pro vo +tis sue +spoo ky +ve led +re con +prospec ts +que bec +âļ « +ig no +anat omy +shap es +w p +p interest +hor e +an es +pick up +ti p +pra desh +hu gh +co e +po k +gram my +well ington +sti gate +ri gh +lea p +king ston +scen ic +go sh +v ani +au g +s ary +zi er +bure au +lin son +con te +fra gr +all an +g aw +lan a +colli sion +surve ill +ren ais +ar range +s ali +do in +br ance +bren dan +our se +in coming +suspen sion +à ´ +l la +educ ators +in tri +da e +bio graphy +bul gar +villa in +go thic +rw anda +e w +may or +meet up +democr at +mor gan +su dden +te sco +car rot +bom ber +mck in +re ne +fun day +agricul tural +haha h +show time +form ing +col a +scor pi +quo te +po ppy +s life +d az +tu b +ne n +mo t +ðŁĺ » +s ore +elder ly +o ve +skin ny +um i +anc o +man ship +we re +g v +k ah +fol ding +ne at +samanth a +dan ish +uk rain +humid ity +nu tri +jak arta +cand les +oooo oooo +at ile +streng th +i bra +bap ti +charle ston +fr ames +girl s +clear ing +glu ten +# # +super natural +ju bi +ph one +he in +dr un +le ak +invest or +y er +dom ain +ball room +mi sh +app li +off shore +bla ze +dor o +âĺķ ï¸ı +win ery +shar if +ad ore +n ir +saf er +si gh +as cri +strong ly +trac y +ck er +ol l +faith ful +ey ed +deli ghtful +vis m +karnat aka +tit an +wh ar +jer seys +re fur +heav en +gri p +pan ama +pre li +glu ten +o dd +cont ent +pon ti +tion ing +e commerce +feder ation +flaw less +ge ar +ti res +by r +pol ice +cu ban +tri butes +tic ul +chur ches +nur sery +di aries +muse ums +snapp ed +i van +wi ght +touri sts +ramad an +t rent +prophe t +won dered +focu sing +hi d +ic ons +i q +ambul ance +pi st +fun niest +time less +sr ilan +bu ys +ki ds +colour ful +a shi +ch ir +mu m +ðŁĵ ļ +let ter +x en +reut ers +pre serve +in ting +ste p +fu ji +uni ver +i u +show down +po ems +surveill ance +suspec ted +ta e +sol ving +tom b +mother sday +car pen +recru it +pil ots +bro c +mix ing +fri days +ty r +represent atives +tra pped +abdu l +free style +clu ster +âļ łï¸ı +k d +sk ill +pit t +ex o +commer ci +muse um +loc ally +g ina +no bel +immun e +fr ac +cap su +main ed +attemp ts +bull dog +be spoke +sing ers +sp elling +seg ment +nat ures +tic k +lip stick +clean er +gett able +preci sion +âĢ¼ ï¸ı +th ood +re ef +no pe +bill y +di gi +mu si +ri val +figu red +tal ity +sun ny +ber k +aw ww +awa its +un real +co pen +asy lum +ex otic +bu en +mo ck +en able +arch y +fr a +pla stic +al mond +amp li +displa ys +abbo tt +s me +x p +ðŁĻ ĥ +graph ic +i ved +mar a +cau tion +lea ks +en berg +ul u +unic orn +cann on +appren tic +ðŁĺĺ ðŁĺĺ +b ball +wil low +at ics +am as +manufac turer +campaig ns +port ers +flo ors +l su +ty pe +ke j +honor ary +it im +to le +min ecraft +d x +ma sh +ri o +consequ ences +ron ald +go ssi +suffol k +mu se +r bi +live music +i van +ðŁİ ¤ +le u +patri ot +man it +lan ca +home decor +de ar +sig ma +ti de +str ings +v ita +sequ el +try na +inve stigate +bor is +ve gan +barri er +mind fulness +web b +hu stle +in da +tan zania +str ay +tex as +c ag +diagno sis +wom an +g w +ob session +l ative +nu fc +fl ynn +moment um +sof a +wal d +vege table +tu cker +supp er +se ab +ar ro +se ag +ven ting +counc ill +sp lat +cal cul +.. # +com fy +odi sha +sto pp +war fare +ca es +à ¨ +co y +price less +in sec +ðŁĺ Ľ +contro ls +empower ment +datasci ence +per pe +gen ic +e res +tru deau +man o +sla very +expand ing +ma he +fa iling +s aga +photograph s +cre st +re on +surf ing +hi e +ðŁį Ģ +ja e +fel lows +south ampton +sol om +ce ster +tab ility +hor n +se ct +he e +cole man +at las +explo rer +consul tation +copy right +organi zing +den ied +mon keys +noo dles +br is +fl or +dou gh +bon ds +sho cked +eco system +care fully +w m +apart ments +cur ve +san diego +must ard +comm en +cere mon +e ch +ru th +ðŁĻĮ ðŁı» +hawa i +fil med +te ar +as ingly +ca ir +wat t +instru ment +ou tta +ye ol +river side +ë ° +. : +nor wich +alo g +migr ants +new man +ri de +spr ink +targe ting +beli eve +tor ch +reflec ts +per mission +ff man +ene mies +bas ics +se ized +sun days +le i +hass an +en do +h c +st ad +le ments +kk kk +nan o +shar k +man a +on ic +treat ments +ear ly +collabor ative +shu ttle +bran ches +mis ses +mained cm +ap ers +ky le +carri e +leis ure +sh et +bir ding +adv ances +ðŁĵ Ŀ +popu lar +di ane +a be +re war +neigh bour +k pop +remem brance +play ground +ru b +krish na +e bola +inqu iry +ep a +lu min +organ isation +abra ham +norm ally +pre ten +jan et +w t +ðŁĴ İ +encoura ging +a stic +bu mp +syd ney +s z +ss ss +gar rett +ðŁĵ » +consul ting +roman ia +spo tting +chanc ellor +ar ma +presti gious +ðĿ IJ +t ad +cry st +compe tit +rati o +cat aly +bro w +j ur +vi king +commu te +y day +la yers +du mb +esc al +genoci de +f ill +gu pta +ste pping +se i +fo to +wild cats +col i +projec t +ear nings +st r +ge ons +comple tion +b m +decor ated +craw ford +af ghan +sc are +visi bility +hi b +direc tion +stro ll +christ ina +alter nate +cl are +sty list +be hold +s ance +leop ard +acqui red +narr ative +ash i +the a +?? ?? +pe as +at ch +sli des +le en +renew able +eng lish +qu ir +co aster +r x +fo ols +match day +mis m +amaz ing +z ig +ke ting +won t +to wel +di ab +sta ke +n m +mel t +e than +gra pe +polit ician +sm en +í ĺ +re o +wedd ings +cat cher +or acle +me mo +ðŁĮ ´ +ec k +rob bie +norwe gian +oper ator +am or +se wing +ju l +x ie +u v +fif ty +me ga +tatt oo +liber als +u pri +traffic king +richard son +su v +ki p +mess y +tremend ous +gl ou +cour tney +la d +stere o +my ers +i dio +^_ ^ +man ning +dy e +w d +thr one +jun k +as u +provin cial +k ook +wr c +fine art +hamp shire +renais sance +b red +fall out +s j +sn l +al am +tor ture +fy i +sh ines +pa w +ch ar +hen ry +c row +aci ous +di an +pa ige +ba re +stock holm +scen ery +ðŁĩ · +jef frey +pu sh +decor ation +ne d +cu te +brig ade +laven der +inv ites +e sports +vo ir +dri ed +tran spl +sur geon +no vels +pul ls +son y +lun ar +man e +i vy +fru str +dor set +sa i +tor res +ssi on +shut down +suggesti ons +writ ing +e o +battle field +u ga +ðŁIJ ¾ +vac u +spl ac +g it +u g +high land +% ) +mer maid +sacram ento +ta ils +p w +ka h +t ell +enh anced +ì ķ +auck land +cru el +ðŁ¤ © +au dre +sail or +gram mar +g love +de on +infl am +fresh ly +k ell +zi p +christi e +mil d +di xon +instru ctor +g ence +ãħ ł +sub jec +constitu tional +crow ds +in visible +ru ins +da k +si p +pla que +p ouring +comple x +z ine +ste ad +f let +trans mission +lo way +ar un +incre asingly +au d +transp aren +cro wned +sc oun +blizz ard +lux u +fi ers +achieve ments +hun ters +rock ed +bas in +vio let +pro ves +achiev ing +pro sper +se ga +flo at +vi an +xi v +pol ic +tur a +approxim ately +wander lust +keep ers +geta way +co d +pol is +br yan +col ts +tal ents +yo gur +gluten free +wri st +gr y +cze ch +ðŁİ Ī +ev ille +ðŁı Ī +to x +dani els +am er +bi ds +weare one +me tab +g t +boy z +pd x +pos session +pu shed +shr ine +reali stic +tri gger +na vi +ru mors +n af +jen kins +tr un +comm uni +Ã Ĺ +gam ers +arm or +moham med +bal cony +y ah +stron gest +rhy thm +unfor gettable +k p +ho bb +custo dy +greg or +r ita +aes thetic +il ation +sponsor ing +n ay +kid napp +sh s +ra jas +me g +signific antly +butt ons +la c +ver sions +essenti als +opini ons +k ro +d printing +wi dely +d k +ur an +y al +reque sted +c n +cur ric +plu m +gr un +v m +dev on +m yo +rel ation +juvent us +rou ge +min ority +min es +jupit er +n ine +oxy gen +fran kie +une sco +fab ric +disgu sting +sal man +dete ction +lan ka +d ac +ðŁĩ« ðŁĩ· +argu ment +shel ves +cel tics +rober to +pi gs +he dge +fau l +pow ering +butter flies +fi r +re make +att i +com o +emp ha +kend all +poke mon +se ating +d ans +bald win +ðŁij » +lesli e +one direction +ti mber +im an +fon t +e der +di on +ste ph +for mat +gre gory +pro p +he x +ru in +sor y +inf er +n aw +bar ak +sd gs +kar ao +lu sh +v ander +end ent +g is +a fro +soc cer +ay an +t uni +lun g +da yof +alex a +mar ath +addic ted +ag ile +hy gi +light weight +ì § +mand ela +jo ey +anc y +hu m +bi r +memor ial +jim in +ging er +v ak +jav ascri +cro ps +orig ins +d ari +pi per +im port +aggre ssive +predic tion +re pairs +cr acker +voy age +ni ke +mu mmy +linke din +country side +bor der +gla ss +per t +s als +sho e +autograph ed +wal nut +colle gi +sal ary +pa iring +ðŁĮ ¸ +cath ol +swee the +defe ats +streng then +roof top +impro vements +barri ers +ur u +t ally +ru led +ðŁĨ ļ +nai ja +emo ji +per cent +gi o +pro bs +on ce +adm its +pa ths +li ar +day tona +pe ters +cal i +cal li +mu g +o sa +ap h +ab y +hy de +eth nic +pla ins +ol f +haha hahaha +holi c +?! ?! +su bli +bl acks +mo t +gh ton +lo vin +b rent +bar u +l ati +de w +ate au +q a +pain ful +bu sters +st atic +ðŁĩ¨ðŁĩ ¦ +note book +out fits +si es +r f +floo ds +Ñ Ģ +thro at +su ici +ro vers +beng al +pre pares +blo g +mini ature +Ø ¨ +am phi +com b +r sp +in timate +green e +Ì ĩ +al tar +surg ical +ves sel +... ? +gav in +g ator +threat ened +z ar +rob bery +di er +promo ted +y g +x s +su bs +inter viewing +threat ening +do zen +me ado +water fall +nintendo switch +cal um +mini sters +dro p +univers ities +war ned +tac tics +ðŁĩ ² +refu se +ad ju +v ast +ðŁĺ ´ +mc fc +lib ya +no filter +distribu ted +re ser +ron nie +de co +javascri pt +mon k +intere sts +fle x +mar tha +sti es +oo d +ðŁ¤£ ðŁ¤£ +e un +b ali +g omez +sti mul +moder ate +d ity +ir is +stra w +consist ent +direc tions +adop t +sal sa +cro o +reco vered +black friday +lan caster +accep t +weareone exo +buil ds +free man +air plane +diti on +bel ong +jam ie +pit ching +li f +om in +cri spy +pre pping +ve g +chan g +accompli shed +graci as +dolph in +elec tor +culin ary +super bowl +wal a +pur suit +black berry +be an +cardin al +pro ved +immigr ant +stric tly +holocau st +pass age +ha us +cou p +pur se +har ass +< < +le ed +ado be +st ad +legis lat +par ked +pri yan +sil va +kri st +s the +fun ky +ig a +sett lement +ph s +t mrw +stre ssed +hun t +ho ckey +treas ures +cham bers +ol u +hu t +mar ley +tex ture +wilder ness +mm ing +poten tially +om aha +ju dy +to es +spo iler +distingui shed +feli x +ah u +recommend ations +zom bies +hit ler +tri ple +colla pse +motiv ated +ulti mat +gg ling +so y +ci gar +fo ren +vine yard +gl itter +fin dings +colon ial +hun ter +eri k +den s +beet le +lot te +sub tle +s matter +tru sted +experim ental +nam ents +ðŁĺ Ĩ +regi on +acquis ition +bre eding +quarter back +am reading +oo td +ru de +initi atives +st out +hy ung +out come +al fred +mic s +exper tise +bacter ia +pengu ins +jump er +valen cia +bar k +ing day +sell ers +contrac ts +hou ston +commissi oned +adap tation +swan sea +santi ago +common wealth +ju dging +sub mission +sco rer +tom my +ñ o +ex quis +fil ing +explan ation +alli son +wemb ley +ri dge +chev y +san tos +own ership +cogn itive +favour ites +sh ed +phil anthro +dele ted +go dd +s nor +gui delines +ff ing +je ep +cli ps +sw amp +an or +guil d +bol ton +spring field +munici pal +goal keeper +ye on +ðŁĺįðŁĺį ðŁĺįðŁĺį +ãħĭ ãħĭ +water front +gra ve +contempor ary +ar ity +ÃŃ a +sle eps +sy rup +al am +pi re +co yo +moto gp +ty son +kej ri +cir cul +sing ly +cr unch +complic ated +nostal gia +k op +mo ve +k ale +mac ro +mid west +h ans +tri bal +nu de +௠į +bey once +congratul ate +cat er +leagu e +ðŁĻ Ĭ +la dder +cra shed +tech nic +karao ke +harass ment +ro ts +experi encing +kri sten +ðŁĩ ³ +ðŁ¤ Ĺ +reflec tions +guin ness +illustr ator +ðŁĻı ðŁı» +cen ter +nar row +comm ons +regul ations +Ù Ĩ +har m +cro ft +cu ssion +hong kong +st ical +intern ship +zo e +cho p +hoo ds +estim ated +batter ies +berke ley +smooth ie +shau n +cro s +~ ~ +cam pe +hu mp +b g +proto type +cl ick +shaw n +re viewed +tem pl +p f +jed i +blo gs +ray mond +as th +ba h +av ail +scot ch +leaf s +nik ki +to k +hol low +ur ges +of t +un like +lat in +u e +cat ering +mil i +alter nati +ma ver +Ð ¸ +ag le +pre order +lu x +cu cu +ðŁijı ðŁijı +t art +âĿ¤âĿ¤ âĿ¤ +arab ic +rapi dly +ar rang +all en +travel tuesday +pa ws +flo ws +st ability +flu id +ca pp +can berra +uu uu +sp ani +demon stration +m la +plac ement +m w +presi dents +awe som +bever ly +ani st +ne al +father sday +referen dum +la hore +o aks +deb bie +half way +gho sts +de bor +matthe ws +fi at +t fw +pre sen +rob i +de d +bro ck +laugh ed +am ounts +bam boo +kinder garten +eat en +mtv hottest +break out +u sic +fra ser +legis lative +p ang +modu le +sam my +go ver +ear ns +expe dition +gar h +concep ts +char lie +la va +bachel or +veg gies +deter mine +el lie +un locked +fru it +dal la +cou pe +wash ington +depo sit +iv ory +pau la +chic ag +gu cci +ðŁİ ĥ +cul tiv +pier ce +li fted +stu mb +re cover +musc les +conduc ting +cb s +mcla ren +sophi a +cel lu +oce ans +up loaded +game play +mal dives +kim ber +avo i +rac er +ca ine +cav s +h ana +li ga +ra ven +inter vention +inaugur ation +oo h +at traction +merchandi se +tune in +li king +juni ors +int ended +att acking +aqu arium +i wd +comp onents +sur ing +cent u +yogur t +ðŁı ĥ +show room +op tical +ty our +ju dge +yi eld +an to +pl c +transparen cy +recy cled +chi ef +ar om +ambassad ors +plan et +âĿĦ ï¸ı +om ed +vaness a +cour t +mar gar +hal ey +v r +reg ina +pd ates +hi span +live stream +âģ £ +ya hoo +gal la +secu red +w ir +bene ath +off l +n il +am b +ye g +out let +u te +pe ep +lind say +bent ley +... ! +he el +trilo gy +vo s +ty re +there fore +tor onto +ab i +simp li +ja e +exten sive +eleph ants +s or +orient ation +im peach +re play +constru cted +peter son +pa is +por ted +custom s +colla p +ad u +high lands +sal em +shel by +ko vic +stra in +ro sie +sen ators +snap s +bo bb +suz uki +bla des +k p +lo lo +gener ate +si ght +ma e +struc tural +predic t +jump ed +ah mad +sun g +just ice +gla m +vol vo +jubi lee +de tention +lo sses +pu ri +every time +Ð ° +ra o +ed ge +li mer +rese mb +har old +re tri +sacri fic +surpri ses +am c +srilan ka +bar bie +men s +fin n +ag s +ukrain ian +em brac +î IJ +flav ors +hom er +lau re +ou th +pr iced +ver de +fir m +ah s +cu b +tre y +par anor +pro fit +in dv +who a +har sh +al ot +crit ics +hu bby +fi gur +gi ra +ca stro +chan el +in put +origin als +ten ant +yy yy +ture rs +lincol n +co on +lear n +ch ou +ac are +o les +din er +hy p +bizar re +mc r +let sgo +decor ating +ðŁĮ İ +al ison +ar vin +f d +reha b +mccar thy +lot tery +da h +minne apolis +eli gible +diagno sed +emer ald +destin ations +s ans +or y +bla zers +n v +ba il +digital art +no c +mal ta +sol ar +pi pes +alleg ations +no ck +po pe +bri d +premi er +n x +present ations +ef a +bo ws +val ve +opp onent +Į ë +visu al +ing le +cate gor +e ter +po is +dan i +at tract +neu tral +th ene +cra shes +fred die +ut ili +c st +awak ening +slo ven +quali fy +pro of +fair y +le v +fre ight +enjo ys +cup cake +flav our +â ķ +protec tive +ðŁijı ðŁı» +is u +ad mir +h mmm +continu ous +ai res +rap tors +showcas ing +y uk +pa ste +follow er +instru ctions +sp ru +@ __ +the o +debu ts +ve tte +sto w +es of +ach ed +sul tan +sand wich +som alia +franc o +car ne +flu ffy +al pine +jas mine +he ated +viol in +ple ss +divor ce +per former +phi es +port sm +dar a +kir by +lo p +chill i +for th +sky pe +ðŁĩ®ðŁĩ ¹ +celebr ities +ed y +ve e +po ison +ey el +gra bs +ssi c +un o +wester n +rail road +am er +numer ous +s v +fo w +fi st +âĢ ĭ +reque sts +mar tial +em my +accept ance +lau ra +ภ´ +er up +hyun dai +out lander +u tt +wrest le +esp resso +demand ing +g dp +geo graphy +sas kat +tro ll +confe der +su es +se m +be ts +t ful +to sh +teach es +col oured +gal way +mac y +dis orders +bb cra +at em +fen der +lit ter +e sh +provi ders +renov ation +nomin ate +ps g +nomin ations +jen na +shar p +some day +z ur +bra ins +che shire +pre y +hu go + ¿ +to ken +r v +car r +tac tical +zel da +kay la +fern ando +photograph ers +j our +umb rella +woo dy +congress man +du mp +le vy +ju an +d azz +sign als +la in +an u +mic hel +por ch +al den +sibl ings +y ale +pe el +sw ick +gg in +ll c +k ale +s con +il d +pat reon +re el +qu in +wit t +mar ty +moo dy +ton i +der y +g ators +speci fically +dd in +ly on +tr ick +meado ws +p j +bor gh +vi k +tu r +bron x +pu ff +lan tern +ðŁ¤ ¦ +g ently +be stie +fac t +refu sed +fas ci +mp y +ðŁĶ µ +cross over +mead ow +indian apolis +duc ation +sle y +loo m +mix er +new music +film maker +prosper ity +li m +week end +cre amy +neu tr +lu ther +h v +nor thern +tw o +h ra +cat ches +appear ances +ha bit +kitt ens +n v +illa c +inf an +regar dless +liz ard +dun k +cur tain +ac om +in tu +ve z +e min +fl ats +calend ars +em power +ru ined +hun gary +vi d +we x +u lum +aber deen +o sa +k t +ma ssi +se emed +s den +' ? +tele phone +de fi +insp ires +me ow +z ones +bl ind +pl y +tuc son +advent ure +ge d +oy ster +ðŁijıðŁijı ðŁijı +out put +tt t +metal lic +sma sh +ucl a +sco ts +perfe ct +lu cy +regular ly +sp ic +rel ative +ath ers +mis e +batt ling +deci des +mat a +occu pied +random ly +cat softwitter +gi an +ball y +al ties +al lies +im men +sy rac +ðŁĴľ ðŁĴľ +l lan +au r +k ut +lam ar +affe cts +n ra +star war +ðŁ¤ ĺ +sc ram +en chan +pro cess +luxu rious +ar ray +sher lock +comp ati +dor f +stre ss +m su +s with +sal a +sof instagram +fo il +under stood +qu ay +r p +c ade +ja w +en ab +en coun +ðŁİī : +do ck +satur n +mu ll +lay out +ra rely +happ ily +fix ture +or ph +over looking +her bs +m itt +pil lar +nol an +pe tty +str y +u i +mu k +o res +o vers +á µ +re creation +we sley +ri t +kejri wal +sto cking +g v +subscri bers +moo se +ma e +ber t +opp re +assign ment +u ro +high lighting +cal vin +we igh +cambo dia +av on +ke m +dis abilities +read y +char gers +p ads +iz ing +illi an +tru ste +col leges +associ ates +alban y +mil ton +cr on +bu r +har dly +si ghts +anti ques +e cho +surpri singly +ha iti +cap t +ph p +op io +ine quality +equ al +ken y +sch mid +autograph s +ren t +qu er +cit rus +challeng ed +te c +epi de +fe st +z hou +li me +citizen ship +cry stal +convin ced +mess enger +copen hagen +âĿĹ ï¸ı +war ran +develop ments +ï¸ı âĥ£ +fore x +hi ro +sne akers +xi de +vi va +stere o +bat ting +ss el +ho st +beng al +critic ism +q c +cr un +attemp ted +ry e +determin ation +cre ations +d read +label s +pos se +anc er +joh an +si ster +partner ships +les bian +k st +guaran tee +bar o +fix ing +ma son +m ous +chem icals +t less +bio diversity +par o +bhar at +ac ol +refu ge +en te +t iti +dys sey +respon ds +lef to +in er +se vel +rahu l +ol ine +frank fur +cho reo +enjoy able +c to +strugg les +wood land +heavy weight +gen s +rece p +ac cred +ðŁĺ ¡ +trans formed +list en +at op +n k +sur ge +be re +gover nor +prison ers +clau de +t ill +mu lator +emo tion +water loo +star t +ðŁĩ º +clean ed +grand mother +fear less +afric an +astron omy +ðŁı ģ +ภĻ +the world +su itable +anth ony +k and +tt en +meaning ful +disc lo +jaco bs +à ¸ +tom linson +ghe tti +ty pho +sub stan +as co +te k +nag ar +mu d +am on +vacc ine +f ty +fle sh +no el +infl ation +portu gue +glam our +tra m +v re +te qu +roun dup +w yn +rejec ted +mosa ic +si ghting +cal f +o ta +com position +go pro +gonz ale +e ed +b ard +tu e +effec tively +we en +al to +ri bs +rel ate +thir sty +fu rious +di m +ch ard +perfu me +s ny +chur chill +k of +master class +wa ve +ðŁĶ µ +er in +own s +to be +sk illed +te m +go f +en i +tor i +cra zy +l ick +resi stant +ici al +ag ar +! : +g ali +del aware +bl itz +koh li +pu ck +avail ability +hi malay +influ ential +cro chet +victor i +read ing +ho bby +vie t +j as +en gra +sk ul +ðŁĩ² ðŁĩ +educ ate +tech no +distric ts +blu es +se tt +seven th +lear ns +ee ee +apocaly pse +hang out +cru el +mu tu +bru h +hel en +she er +c tion +kle in +tex ans +ce real +sh ine +ne red +gra s +am bro +f ella +hin du +matthe w +li ma +mir anda +je wel +so ho +euro vision +neighb ours +chand ler +be sides +ðŁ¥ ° +ast ros +thu mbs +ren ault +ra ve +hi red +ðŁĸ ¤ +it ary +z or +bla zer +k ine +ea u +kat y +dc comics +pe c +ro dgers +water proof +kill ers +super int +pre serv +as so +brew ers +promo tional +sc am +villa ges +sket ches +ju icy +for life +au dit +so lo +fundam ental +len e +philipp ine +t end +conserv atives +sponsor ship +dd le +a ine +h tc +os i +hul k +w af +ภĻ +evalu ation +ant ine +sle e +robert son +roo sevel +ag i +sophi stic +emplo yers +bubb les +ko wski +inter action +sh u +bou le +ic an +j are +han k +leg itim +k nicks +kar ma +recei ver +per ks +u h +sta ir +sun i +labor atory +gra ves +voc als +oo t +c ture +thri ve +tic o +ãĥ ³ +b w +carto ons +mcdon alds +dra w +y ung +pl er +li d +eth ical +groo ve +ent a +international womensday +pat ron +wor ries +ðŁİ ħ +ðŁij ĭ +ka therine +di az +tor i +bach chan +tru st +min eral +ic om +buil ders +bor n +col oring +lat te +ca se +revolu tion +tra der +ox id +chi pot +inst antly +sou thern +se hun +pro b +her nandez +lis bon +hu awe +p ong +me a +ro oney +wheel chair +ke en +be tt +cor in +regulat ory +di splac +ka ren +sch em +sun sets +wh ales +remin is +he p +hi de +mar cel +pand ora +do yle +th fc +ot to +no kia +trans gender +ko v +hawai ian +sha ve +so vere +exc er +nick i +pu g +st or +ro th +wee t +leg al +dig nity +po w +hom age +ðŁĩ³ ðŁĩ +s re +can on +la x +wo ah +quart z +ñ a +gree ting +flick r +nai robi +advoc ates +an c +vi i +eu gene +th ra +c re +el an +pen sion +th letics +ton i +re agan +x v +sto re +ben ch +har lem +todd ler +sent enced +âĻ¥ ï¸ı +glob ally +che aper +u f +ma m +nic o +ik u +tho u +ni st +dam i +th ala +rho des +sal e +bow ls +â Ī +las vegas +sanc tions +adm ire +mat ched +un able +travel er +ele ven +straw berries +âĢĶâĢĶ âĢĶâĢĶ +stu dio +jac ques +im s +valu ed +s no +cheese cake +n xt +e os +s x +f x +ton ic +hat ch +chic ks +gra ds +hand ic +r ory +as p +ri pped +denti st +n en +lu fc +âľ Ĭ +di ge +hop kins +sher man +f da +for all +ash ley +str and +h y +liqu or +buffe t +ess ence +phar ma +suri ya +ðŁĴĻ ðŁĴĻ +festi vals +z an +re fresh +pur ple +uni forms +kenne th += ) +as an +hel sin +transform ers +k ali +person alized +chal k +bo bby +â Į +the mes +depar ture +prin t +illustr ations +qui et +agre es +gri ff +Ø ³ +m iti +toge ther +conven ience +ab ar +car lo +turt les +info sec +some what +ar lington +scholar ships +emir ates +mu ms +st ella +auton om +fe ather +g ore +nom inees +fragr ance +Ñ Ĥ +w ong +thea stern +gr e +z illa +is i +bump er +go o +do zens +ab duc +âļª ï¸ı +o ils +don ors +sil icon +i pod +fortn ite +ðŁĴ ¨ +tor o +spark ling +consci ousness +pal a +nu m +moun ted +ffin s +thi eves +team mate +pra b +om er +ta pes +bo d +mit su +ste w +e re +p bs +tu sc +lo we +ra de +parliam entary +h m +ed gar +ðŁijĩ ðŁijĩ +to a +a gh +hon i +s late +ge ek +ap t +hard t +ta p +horiz on +grow th +make over +hi l +paper back +id an +reha bil +gi u +possi bilities +let tu +fran co +bo ss +ach er +does nt +mo e +ta ker +huss ain +ml k +di l +th ia +ham a +real ised +raven s +curric ulum +m ith +k night +ted x +r v +isai ah +cumb ria +birth days +f ing +pre z +mu barak +exquis ite +clear ance +y en +par i +ev o +à º +modi fied +app lying +imple ment +disco vering +chap man +indie game +dis k +crowd funding +mach in +li vel +sty led +âĿ Į +ma king +rehear sals +nutr iti +subscri ption +and ro +cre ators +car ries +ky lie +cam den +appren tice +tax pay +c ca +tuesday thoughts +pis sed +er man +dete c +freed om +mer i +.. ! +psal m +sun light +per spec +be ings +book store +rock star +fun ctions +p ence +fav es +z n +obam acare +sp ill +coven try +pi geon +pi vo +ba it +kol kata +av al +don or +wa h +privi leg +tra ditions +rajas than +ten ess +portugue se +yn es +tack les +de fic +tor n +pol ling +thor ne +in a +bened ict +bar ry +cal ories +ver dict +save the +nor ton +off ice +main stream +impro ves +fr on +respon ding +real tor +scotti sh +de clar +r l +shi v +supp lier +re sting +swee ts +qu i +. âĢ¦ +whit ney +startu p +thank you +teach er +h alls +ha ve +hand made +pro ving +quar tet +ro chester +li an +virtu al +mend es +of icial +mid lands +x box +meas uring +o vo +accommod ation +bri des +collegi ate +intellec tual +in car +ni ag +ðŁį · +sf w +coco a +co ats +civil ians +presi dency +mat rix +sweethe art +tri athlon +wag ner +ra dic +plann er +the o +execu tion +k um +the walkingdead +sc ar +ro tation +blo gging +bom b +re son +bb les +st are +assi sted +e do +brand ed +war nings +thor pe +acknow le +satis fied +sho res +ri d +dor a +phys ically +bi gh +appro ves +ha h +ric al +vers atile +pret end +lu m +ab hi +ye e +sp it +ãĢ Į +dj s +ash tra +j t +ven ues +gram mys +cy clo +tr acker +over watch +repl ica +el yn +nr l +lind sey +hom o +ballo ons +kitch en +si s +am os +ende av +ðŁĴ » +a rec +thu g +hoo ked +hr c +new york +bur gh +americ as +patric ia +ug u +ap athy +ha st +psy chi +cor k +petro l +ðŁİ ¬ +ak u +po pping +psycho logical +au x +g ma +cad illac +wa ste +auth ent +bri stol +nam e +que er +to ber +jer ry +com in +ch ant +privileg ed +op ar +lo ser +tex t +mar ker +stri es +equ ally +ak i +christ mas +gare th +ble w +em ma +imag in +se als +che at +conditi oning +j ana +ren s +dar ies +o asis +disc ounts +coun cil +i ka +shir ley +vou cher +al ps +w x +q r +dri ft +attemp ting +ut c +Ø ª +gonzale z +m f +jo ker +paralle l +pa re +aspe cts +proce du +n p +am a +rale igh +bright en +gu ire +radi ation +cre scent +ho b +il le +str and +v ore +n ard +che st +di wali +av atar +al der +d ling +pa thetic +ðŁĴ ĺ +spir it +jor ge +film making +ðŁĻı ðŁĻı +challeng er +b j +down town +ht ml +ade qu +twi sted +in ely +( ' +wra ps +oper ational +y ne +n us +mag net +market place +health ier +snap shot +dam on +inter ven +fe derer +ow ls +biscu its +j p +ro deo +blue berry +lec tion +fron tier +summ ers +re yes +pede strian +go l +caf fe +refur bi +bou lder +me ghan +speci alty +la ss +e i +suspec ts +appro x +rr r +ra th +st im +cru shed +he d +wh un +lo af +cr ore +river a +gene tics +so ck +wa sted +ny pd +answ ering +do ve +bel la +ol in +du n +fi ji +pre tty +spar kle +y un +j d +euro pa +li fts +am ber +mu r +te k +boy d +roy alty +in do +ri b +go tham +ti est +inst alling +ke mp +the photo +cos mic +) )) +whole sale +loy ment +eas y +su ing +sett led +af p +pro ver +suppor tive +re es +ne ath +deli ber +c é +wel come +pic oftheday +new born +pat ty +sun s +si est +fl int +diffe rently +spo ilers +troop er +g ins +cor y +look out +equi pped +ta pe +to by +resear cher +u sh +ke yes +al ma +induc tion +k w +k har +sl ick +bri de +e ur +cra ving +book ings +ch es +tr unk +vern on +sp her +cryst als +rel atively +pom pe +uni ons +val ley +par a +w ant +ok c +de af +ser gio +len non +sh ay +cr a +v at +he e +t we +liqu id +pol y +ðŁİ ģ +b ent +be aring +motor sport +bar be +te sti +han i +fin ancing +astron aut +water colour +ri sh +comic con +gar t +wr ong +ber n +it an +ste pped +fil ters +c low +me x +dem ons +all o +expand ed +comm and +et ers +go ats +si ri +y r +pot tery +mari on +i le +el an +san to +person a +du ke +hom eless +li ghted +wheel er +chang er +cab bage +sur real +ham burg +sma shed +str an +k not +i art +ob i +be dro +di al +th ick +b ingo +fu s +vacu um +con ve +ati ve +accur acy +accoun t +re fer +ri z +spider man +ban a +r ite +u b +ab s +medic al +lin k +si em +> >>> +be tra +g lowing +re actions +pupp et +spa ghetti +ang s +re medi +pray for +roy ce +char lotte +£ ï¸ı +gh et +affe cting +ro de +soci alist +mo ses +az i +o it +re porters +cd t +ap ing +s nat +minim al +wa ist +sie ge +>> >> +ri g +schmid t +h are +ec a +thor n +he mp +es the +cly de +th a +don ut +moham ed +ling erie +le gg +carpen ter +perform ers +de a +imag ined +cur se +la sh +ct r +agu a +ro ar +gr i +ro le +j fk +resur rec +roosevel t +maril yn +sm alle +will is +wa ited +char ities +the res +li k +origin al +car i +c ough +cru ci +la gun +contra st +k ou +arm our +re moving +t ent +maz da +bri ghter +thi ef +cor ner +tequ ila +buzz ing +al bi +p am +az ure +disc oun +pixel art +possi bility +ham ont +tra des +bu da +hi ve +vers y +fin ch +tran spa +em i +terri fying +in qui +g ba +sub stitu +collec ti +plac ing +cin dy +k ann +pa tho +diamon d +mour inho +guine a +anthro po +air s +pu mps +ì ļ +pas o +cur ling +an ita +resi dency +ne wh +jo on +cigare tte +que ue +ex trac +gam es +spl en +ex press +public ly +bon nie +tribun e +ba ek +reason able +c or +timo thy +she eran +Ä ± +f dn +su tton +concentr ation +carav an +x avier +al ger +cy lin +freder ick +ner ve +pe ak +lettu ce +j ail +pre game +kav an +up graded +eco logy +squad ron +gra pes +goo g +pa stry +ðŁĹ £ +ãĥ¼ ãĥ +mil ano +awa z +presen ter +ðŁĮ ¿ +her d +king s +tem plate +fl our +h v +k ley +i ya +spe c +at er +frankfur t +co ch +tex ting +del i +communi st +regi ment +ele anor +anticip ated +ðŁijĮ ðŁı» +thephoto hour +ran o +survi ving +simul ation +daw son +ar in +aqu a +m or +âĢ¦ . +cin o +ira qi +sh az +dun dee +we s +dra u +hann ah +s news +occup ation +ste en +x m +ang les +sett ings +gur u +kno x +or ca +shap ing +w ent +dr illing +zz ie +br i +kis sing +fin d +ma ine +âŃIJï¸ı âŃIJï¸ı +ðŁĮ į +lar ry +bu sted +ta vern +acti vely +- " +replac ing +no d +un lock +. " +âŀ ¤ +affili ate +to w +l n +happy newyear +di f +j m +green wich +contro versy +daw g +con dol +sav annah +compens ation +touch down +te o +amb itious +embro i +convic ted +iart g +bar ack +tr ance +testim ony +au dition +thum b +my ths +be x +que z +orch id +den y +entit led +hoo d +gr ant +in box +blue jays +r illa +smalle st +bur den +in famous +divi ded +boun daries +t ter +el t +wy oming +be verage +me sm +one ws +budd hist +y ana +as sad +is ms +bar rett +predic ted +back to +tw it +e there +cap tains +escap ed +ay o +lam borgh +gard ner +la ps +k al +adverti sement +insec ts +na po +am en +ac y +r and +g k +te h +k athle +tri dge +pan cake +at ro +pyram id +bu la +paral ym +gau ge +en cies +tom y +biscu it +but cher +quali fier +coun ty +ke i +po ols +dar ker +should ers +ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ +sp re +( " +writ ers +g m +ðŁİ ĵ +k nit +hu ff +mt b +philli es +o st +den is +g art +licen sed +inter face +ex cel +d well +from the +co fficial +az zi +appear ing +fore st +n ana +ke ith +manufac turers +beck ham +) ? +e se +col ony +delic ate +ut ter +mc in +transpl ant +pre ferred +par d +ari e +hu b +po ds +perspec tives +pic t +del u +app er +be than +p mo +crimin als +femin ism +sh ack +circum stances +fel las +prote sting +wa x +sugge sted +t ator +dre w +om ni +fa ke +kath y +re b +del ine +ber ni +mi sty +ðŁij © +er able +break through +men swear +millenni als +chan yeol +la z +inser t +rep lies +phra se +n x +ihear tawards +audre y +gran ite +rac ec +ori e +ter ra +innov ations +britt any +at eral +pe ar +bio logical +sh ments +institu tion +m sn +frequ ency +d man +neg lec +t f +ste fan +fox news +ty po +comm s +sequ ence +car men +wh ites +econom ist +exe ter +se um +re sorts +cas ually +bun de +divi de +Ø ¹ +ga g +cre ed +reti re +cau cus +rapi ds +wrestle mania +tul sa +sunder land +fundam ent +o di +yam aha +v ary +intri gu +el se +be acon +an gie +tra ded +tran sm +g ents +kn itting +gal ac +ðĿ Ĺ +u to +sea side +hol t +re rs +far go +train ers +mon soon +b ale +sou ght +mad die +h w +co li +fr an +fav s +ðŁĴ Ķ +int ent +r ally +s bs +lemon ade +barack obama +bre ad +stick y +explo sive +chel ten +t j +as soc +ram en +hom ies +v log +mi ster +lor d +âĢįâĻ Ģï¸ı +aly ssa +sketch book +ru mble +cat ch +migr ant +discipl ine +un likely +chronic les +fl ora +sl ams +am id +s boro +coo p +ju mps +tran qu +mel is +sof ia +en ri +gab e +sy ri +nicol as +cha i +w v +be cky +foo ty +ta o +suppo se +ðŁĺįðŁĺį ðŁĺįðŁĺį +plu sh +ri sh +ðŁ¤ ĵ +k ha +satur days +ac cent +he c +lim it +carl ton +wi red +taylor swift +ðŁĺ ij +sq l +har ro +recipi ents +g at +go p +th of +amaz ed +gh an +ðŁıĨ ðŁıĨ +por to +cla re +di stant +na c +ohi o +ðŁĻı ðŁı¼ +mt n +anti bio +dino sa +me sa +par tial +b v +lear nt +lov ato +questi on +ex tract +gossi p +gi bb +niag ara +ðŁij ¨ +displa yed +so oner +ste vie +nug gets +ml n +bro m +tur b +give aways +stu pi +bl ink +c ili +conven ient +mo h +vi ve +f ric +cau se +cham ber +cu les +ne arest +is se +small biz +t j +canadi ans +smar ter +bra sil +ra re +que tte +w ha +cand le +at omic +ðŁijį ðŁijį +warri or +relax ed +stri ps +ne ur +k ka +r fc +jen sen +reco vering +respon ses +sal am +ortho dox +acti ve +ell ers +n it +âŃ IJ +metro politan +centu ries +vi da +gra ding +transpa rent +sim ple +do ts +superint endent +elev ator +autom ated +red skins +ima m +summer time +jona than +ge aring +michel le +confl ic +m ice +to te +publi sh +pa x +) - +na iled +á ´ +tele scope +ser bia +ba b +ape u +st ically +sen ti +r ats +isol ated +grou p +hat red +paranor mal +stan ley +ali on +safe ty +l s +ठ° +nex us +alexand ra +mas ks ++ + +tr on +au k +brother hood +brow se +mix es +sim one +mu sk +appro ve +lo la +ex p +per th +fu turi +un seen +d m +chel se +sc outing +o we +portsm outh +k ram +mi ze +di spen +su p +d lc +adver t +tere sa +is le +cy cle +met all +shi elds +marin ers +ra z +ing en +fun d +an go +jon es +o ka +mad den +broc coli +domin ic +situ ations +mer o +cric ke +puni shment +d b +sha king +ðŁĺ ļ +m q +ari ans +le h +cla w +we ds +d ure +ni el +j elly +gour met +tra ders +le vi +w ages +kne es +wi se +heaven ly +avi d +melo dy +z ack +ban anas +apprentic e +pro p +fun ny +o de +respec ted +me gan +fe wer +dra fted +med it +gra pe +us army +cru sad +vo cali +prepar ations +non sense +us age +th r +ro th +wiz ards +insi de +promo tions +mon a +red sox +si g +eleg ance +ch ia +univer sal +ãĢ į +ra ja +un ga +pol lin +filip ino +ak a +t sun +ik on +bi king +decor ations +z ac +cade ts +hum our +ag m +re ppin +vac cin +elo ve +u w +dia be +galla gher +az er +do l +a while +pro minent +wel sh +t ann +' ) +bi en +wa g +in al +c wc +wic ket +ur st +q anon +x e +out door +dun n +star r +co logy +ric ky +u efa +reb ounds +s music +inf ant +ðŁĻ ĭ +so p +u mber +hand ing +beg in +sor ting +ha sh +sp ati +re k +buda pest +black hawks +dele te +ro m +can did +auth ori +de bris +spe cul +inter section +marri ott +im ran +ðŁĺģ ðŁĺģ +cru ises +ram sey +rafa el +aware ness +vas cular +beyon cé +ru g +ðŁĺ Į +festi v +ar am +s able +bas il +p ill +flo oring +un beaten +implic ations +u f +w ound +for ge +poin ting +po ts +popular ity +ðŁijı ðŁı» +mani pul +s lots +deb ates +abs ence +ver mont +never forget +wri st +gl oria +ren ce +hu sk +mel ting +ðŁİ Ł +br aces +tim ely +transform ing +am ps +ma k +po e +ah an +gener ally +nd p +ale ppo +unic ef +pro fs +nor d +ma sk +jackson ville +v v +sh ells +bloom ing +oper ators +char coal +ne ville +ma gi +chi p +sam a +ir an +re forms +accu mul +ru e +æ ľ +web sites +ga on +devast ating +sto s +glaci er +ra pp +chipot le +pr a +or ous +rom ney +seas on +decor ative +c isco +dit ch +compla in +ll o +assu me +ðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤ +n els +cent ric +ft w +car rots +tat a +can ter +per ience +li ers +demo s +bl unt +oper ate +reserv ations +le ah +sub stance +di son +an te +elec tion +v ue +squ are +non profit +ca a +f su +y am +ãĤ ¤ +v ladi +comple tes +mar i +philli p +ne ill +er as +ka it +men do +mahar ashtra +g p +dan e +provi dence +ther apeu +juven ile +me mo +in corpor +aa aa +seven teen +teen ager +à £ +or ns +wi de +cu teness +tw d +ff les +bar a +com edy +over time +y az +bar on +unemp loyment +ðŁij ĭ +exter ior +den se +cent res +match up +history month +artif icial +qu it +e sk +war n +cr itic +j af +ðŁĵ ² +inform ative +fu els +recy cle +nam ing +stri pe +sol ic +mole cular +dee pi +con vo +s sel +na e +de scent +ti z +accoun tability +ter ry +r ito +sl ay +em o +dem ol +sens ation +co v +tor e +round table +y ol +excu ses +ॠį +tur quo +hh hh +pod casts +cele b +me ssi +li o +man n +contribu ted +u z +gener ator +ele ts +veg gie +indu l +en suring +detro it +pun jab +tran spor +instru ction +ad d +por cel +pan eli +cir cles +persi st +clay ton +sp n +dog softwitter +is nt +sp r +retail ers +p w +hun gar +el ena +mon aster +gu atem +je ssie +an z +ra shi +fle e +car ving +fau x +l al +hen ri +d jo +du ll +s ana +lar a +glo be +cri mson +com pass +pau se +na b +lion el +ba ths +u fo +invent ory +sin gh +sat an +ðŁĩ ¸ +ce ments +in form +gener ated +bi den +av g +tas ks +de er +sa u +ja iled +pa stel +sc c +na il +steel e +per is +lamborgh ini +pur sue +mar gin +u ch +bo sch +dra in +cl ara +bo m +lat ino +web ster +rose mary +r ha +s oun +billion aire +not ch +percent age +con or +' " +hom es +earth day +h ort +big gest +di sin +wal ton +edit ors +im ma +om ar +equi valent +pharmac eu +ah med +cam eo +han ni +under rated +ge ment +micro bi +v oo +honor able +obe sity +âļ ¡ï¸ı +limer ick +invol vement +st agram +boule vard +bur g +blackand white +liber ation +fi ve +inter im +sm m +rival ry +cap abilities +stat ements +thu mb +ve d +sw ans +bar ber +e que +seren a +hel m +noo dle +sam pling +n awaz +sing le +thunder storms +sh on +in ev +ë ¯ +to pp +orch ard +bi an +ðŁĺ Ķ +door step +salv ation +marke ting +r ons +cle mson +ra vi +in take +stand with +sin a +ha iku +ple y +elector al +ph illy +la ys +electr ic +cap turing +u pp +er gy +believ ing +cul tures +es day +inva sive +ed ed +spee ch +end ur +viet nam +boy cott +pe de +deli ver +ðŁĴĸ ðŁĴĸ +mer chant +st ir +den ies +poc kets +o ti +cu ddle +ro land +mm ed +den ed +lear ners +hoo p +sour cing +h acked +di m +environ ments +ben son +jud icial +wor cester +pear ls +govern ments +arri vals +cor ners +tun ing +la bour +y m +or dering +le wi +i fe +hygi ene +thou ghtful +indone sian +campaig ning +princi ple +assau l +ru bb +at v +wil ly +en tre +il i +ph on +du ties +âĻ¥ âĻ¥ +sn akes +lo op +am ar +conver tible +bon ding +ment oring +max well +ethere um +destro ying +ax is +ca iro +fin nish +sho ck +ðŁĺ IJ +cal eb +com a +pe dal +co re +contin ent +el son +temp o +helsin ki +ac p +tack ling +st ated +bl a +dou b +sma shing +a ja +camer on +disru ption +warm th +being salmankhan +bullet in +o de +syrac use +ar an +mc gregor +bul k +an ton +confir mation +sp ine +im ran +instru c +jac ks +chi o +pal m +str e +embarra ssing +un t +elimin ate +to ss +c ise +a ws +oni sts +sh inee +jo s +ho se +li vely +opp onents +mo vements +recogni zing +sandwich es +sh akes +exerc ises +se at +profe ssion +merry christmas +lu gg +adopt dont +mar vin +byr ne +un le +he t +ku wait +rah man +aspe ct +humb led +gen es +f and +long time +) ; +cam pu +an gus +ðŁijį ðŁı¼ +q uran +sle eves +s lic +¸ ë +twel ve +your e +i ke +go gh +b st +dic tionary +reflec ting +to on +yar n +em bed +ðŁı ´ +re serves +floo ded +ver iz +du sk +estab lish +pro li +au d +ritu al +or bit +declar ation +recor dings +cam o +cas sette +good luck +cu tter +bo p +b ho +che ating +paci fic +ma res +tim er +col t +tr ous +tomor row +han sen +ci e +w ang +ban i +circu lar +ac ute +far mer +co ys +p se +ir ving +w j +haw kins +b ison +ur day +cru ising +o te +k ath +whi stle +your selves +ant is +sla sh +thorough ly +ke sh +ser ie +ex em +en ig +guil d +sh red +ho gan +ap o +ä ¸ +pu zz +ne tball +au ssi +panor ama +ws j +av is +ar ming +hum ph +brow ser +cri es +fo ggy +mat te +ðŁĮ » +it er +tal lest +by ron +cap tiv +je su +any ways +flag ship +p ton +we y +fay ette +financi al +f oul +solom on +jenni fer +cucu mber +ar gue +tex tile +wrest ler +john ston +pa stor +ðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃ +cac tus +edi ble +re served +ric hie +met res +ingredi ent +h ella +un to +ch ol +cele bs +po ets +gra ham +hay den +coinci dence +b aw +communic ate +flet cher +/ - +tole do +ecu ador +coun sel +s laughter +line ar +at p +os u +jo el +ev ed +conqu er +ru stic +plic ity +recogn ise +room mate +cr acked +jas per +ph er +ðŁĮ º +wo ven +mo ist +ff c +ste ering +ni sh +stand ings +frequ ent +ar di +haz el +as msg +bau m +d art +si dd +nat h +ch ero +card board +c ss +n sfw +pa ir +ðŁĺį ðŁĺĺ +occur red +homeless ness +mal one +ph e +xi a +pad dy +decl are +theat re +b f +per sian +ta d +ax e +susp icious +lam b +mu cho +sen ior +st as +k ite +st ing +gra d +k af +wat ering +Ø ¯ +spi ral +th ms +educ ator +jer ome +of c +clo ck +su l +pe mb +.... ..... +park way +de aux +restric tions +m ons +need le +e j +le agues +water melon +am an +pl enary +max im +w ab +coming soon +bry ce +vi gil +super market +fortun ate +turquo ise +presi dent +li v +inter ns +feel in +fix tures +stun t +st aged +premi eres +lo k +prac titi +shor tage +log ne +ve c +con cor +roc ke +li g +com posed +syn thetic +di p +cam ila +ch is +j ou +su san +eye brows +supp lement +satis faction +moham mad +ti bet +house of +pu n +as sam +shado whun +psy ched +se duc +mand atory +her bert +sc allo +stream ers +proto col +block buster +produc es +sch nei +lau rel +tri be +time hop +pl a +mod elling +tv time +mtv stars +wi dow +me tric +ch am +con do +flow ering +ale c +d ms +inten sity + ¨ +mccar tney +islam abad +k b +f fi +ph al +anal og +f ond +h acks +positi vity +treat y +sub marine +conne ct +sel en +categor ies +cu b +organi ze +si k +quote oftheday +remin ding +am or +loc king +ðŁijı ðŁı¼ +comp ound +et te +b out +rec ur +fe rence +mi zz +tren d +hip ster +for tress +forth coming +preli min +o dyssey +ang p +del ici +even ings +ðŁĶ ¹ +i q +d w +da ir +kathr yn +christian ity +moon light +ha b +wh oo +f bf +se th +genu inely +pa x +char ity +deplo yed +b nb +bu cs +ju dg +con ge +plant ation +im press +car a +sc lub +sco py +land ers +compla ints +b ama +re build +x y +real ism +sh our +le in +brac elets +mer a +assas sin +an chor +ðŁijĮ ðŁı¼ +lin en +con fron +chronic le +comm ent +cat alog +il les +gor ge +me try +jung kook +love my +sent in +se em +fit ness +alli ed +ts man +digital transformation +pr an +lo ft +min ton +alden richards +en vel +cher ish +certain ty +zz z +rhin o +per kins +en rich +cape town +ome ter +sec tions +ske leton +def enders +ðŁĺ Ŀ +pen c +bri t +ja h +capital ism +ðŁ¥ ĩ +baz aar +re me +ex t +kk k +conver t +stor my +b ye +kar an +chry sler +ad os +pre ssed +syn c +ation day +dang er +bad ges +refu ses +em powering +ly m +ex ports +adoptdont shop +ðŁĩ ¯ +th c +awa ited +focu ses +fin ed +o at +haha hah +âģ © +n family +fi ona +luck ily +thr illing +ty ping +out break +di es +he u +craw l +ne sses +o ath +scri pts +gee ks +ðŁIJ Ŀ +p b +mathemat ics +al is +________ ________ +gymna stics +acti vism +recommend ation +gre n +wa in +cour ty +n apol +cau li +hor nets +g als +jo ckey +dir ty +at ar +enor mous +pe st +greg ation +an os +ii ii +def ends +black historymonth +at x +mb c +lugg age +wit ch +co b +la sts +cu m +gg g +ba thing +n ar +ce bu +ðŁį ĥ +navig ation +min e +re jo +ðŁİ Ģ +gif tide +re ta +use less +pu ll +defic it +al lu +ati me +it v +tr illion +pu e +ac ies +proce dure +l ori +jen ny +c ad +ul ously +dr ac +promo tes +ing the +can u +woo hoo +na omi +zar dari +ts u +be ir +sd g +le ver +we ber +ab ud +lun d +crow ded +deplo yment +ter rain +ken ny +ho f +witne ssed +lo ch +j k +bul ly +w ren +poe try +do ff +ww i +mo red +din i +cul ture +promp t + ¥ +maur ice +to pps +r m +cor respon +ab out +jewel s +gi br +eag le +ðŁĺĺ ðŁĺĺðŁĺĺ +l ending +sou ven +ç Ķ +contemporary art +establi shment +j ong +âĢ¦ " +gat or +patri otic +mc coy +v ape +human e +feli z +coach ella +re posting +ste als +fu ller +n ering +at ra +( - +bla ke +he ather +wor ms +discipl inary +rede mption +y ard +am in +" @_ +d nc +t ds +k appa +ne wark +comm its +spe ars +j ams +t and +msn bc +inter medi +aim ed +at ic +teen th +observ ation +kash mir +kavan augh +ou l +san francisco +re u +bel ated +cho w +pass word +st ills +deta ined +sar i +day ton +dar ren +itali an +ar th +amu sic +ar bit +w m +v m +he m +dou g +my r +a sho +pre v +vin d +bra h +sta g +ภµ +pre views +gu k +con taining +leon ardo +sad dle +ru shing +st av +lon gh +gam bling +ve gas +reserv ation +end ale +bal a +fl a +vari ant +he dge +bulgar ia +nat ali +we aver +sol st +encoura ged +ap c +as parag +ne st +cycli sts +fe l +ìĬ ¤ +overwhel ming +pey ton +j it +a post +mb le +ble eding +neighbour hood +a very +expre ssions +mac donald +gi gs +mon ds +illu sion +n ct +cam ero +over head +my th +ol y +vi o +et v +lau rie +unve iling +pri or +con n +iron man +di ff +day in +crit ici +con go +re vision +wal e +direc tor +p ines +black pink +gar ner +cur ated +manit oba +h ac +common ly +bar ton +.... # +mor tality +live smatter +philos op +shor ter +con vince +fre ak +vend ors +insi ghtful +el ly +sens ors +e led +s berg +weight loss +u kip +sp ur +priv ate +qu a +ss c +, ... +supervis or +advis er +amaz ingly +less er +at es +mah on +oooo oo +sar as +pmo india +waff le +un ders +toler ance +sculp tures +her sh +kno cking +smo ke +cathol ic +gri m +tra veled +fli p +ge off +dinosa urs +sle pt +scar let +ok i +compla int +ob sc +nam i +la g +cross fit +u fc +mc cain +refe ree +sad ness +pen ny +li eu +mo de +ki er +vol s +w is +el on +she a +ba o +son ia +cla ire +em manuel +moist ure +di gest +vi ii +t eller +ch on +access ory +night club +foss il +aw an +hu sky +ab original +brand on +ffici ent +cou gars +ste d +ad mitted +igno red +content marketing +ag as +v ase +execu ted +negoti ations +she ad +n and +tab lets +go th +ts al +d fw +on ep +protec tor +sp ho +gaz ette +andre as +ss er +comp ilation +ha v +contain ers +bro ker +soc al +porcel ain +hy uk +air ing +ðŁĴ ° +publi sher +scen ario +spart ans +re viewing +itu des +ed el +pear son +ba sh +mau i +a ad +ðŁĮ Ĭ +li u +ul ate +program mes +fav our +web design +real ty +motiv ational +cro sses +' ... +bus ch +adjust able +ar jun +mist ak +dimen sion +pi stol +weigh s +en y +unve il +indy car +gor don +f ade +fran ken +qual ities +bet t +loc ate +ker r +sp c +confu sion +ne e +luck y +bas es +dep ends +fire fighter +ol a +re t +mar oon +ðŁĶ Ĭ +w am +defin ing +whe at +bi l +é s +b hai +psy ch +ta u +ic ans +thi k +ob ile +inspec tor +ìĨ Įë +ill on +go s +ev angel +fa i +si st +voc ation +bur ge +chi stan +renew ed +enthusi asm +en ting +ag ri +ike a +m sc +aero space +sens iti +memo ir +hosp ice +co caine +der ry +mechan ics +Ħ ภ+tin o +reduc es +collec tors +in justice +supp re +v ana +ab un +nap a +su sa +os lo +e ff +en core +lic ence +ched dar +z al +moun t +ðŁĴ IJ +threat ens +!! " +archi e +fu tsal +scu ba +jo s +gn on +se xi +s official +compar ing +domin ant +tof theday +fa it +propos als +gi ft +y as +cn c +l r +ha b +reser voir +beli efs +gener al +mar ti +t d +est e +ì ł +wi l +ðŁij ¯ +ðŁĶ « +sp x +et work +excer pt +e instein +hir o +sil hou +team ed +per ception +corri dor +mental health +hin ts +ben ny +induc ted +sw x +wi desp +spe ak +cher yl +dru g +ðŁĺ ķ +h f +asparag us +myster ies +fitz gerald +off er +therap ist +care er +dam aging +ts d +per u +wei bo +y ay +phoeni x +disc re +mac book +bar ker +stig ma +sp read +roc kies +kang ar +bri dg +pa i +bi shop +ta iled +capsu le +ðŁĴ ĵ +ge of +roy ale +short listed +o ste +ash amed +ch app +key e +cl a +screen shot +austri an +nati ve +en ight +juli et +michel e +ðŁĮ ´ +travel ers +pi l +football er +win chester +ðŁĻ Ħ +azer bai +gold eng +organis ations +interpre tation +predat or +ofthe week +lo gan +pok é +mari e +cal la +t nt +cin de +ge tic +fit fam +gra v +ow ens +ðŁĮ ± +shoot out +sal is +commissi ons +co he +p tic +ni xon +hi a +amb ition +mar ine +cruel ty +t k +cru de +sal ty +jim a +mon go +ir ony +on wards +arre sts +strang ers +ig er +cycli st +ra g +exten ds +tra dio +bour g +mo i +el la +e able +lex us +au l +der a +histor ian +mor ton +ti ff +man ner +ko t +d k +po inted +mar qu +a an +en ey +du blin +on poli +em ili +secre t +fl o +âļ ¡ +ba j +ste ep +accompan ied +rum ours +dev i +purch asing +fi g +pu b +sch oo +autonom ous +go alie +x ia +autom atically +re vers +ter o +fu ku +titan ic +shoo k +sand als +see kers +exc av +nor dic +bigo live +ba ke +r att +z ak +ne p +ðŁĺ ¤ +cand y +billi ons +book worm +pp et +à ³ +sur faces +sc ars +phil ip +do gg +ci gars +co te +transl ated +cur ator +sin dh +han gover +bre wer +on es +el ton +ðŁĴª ðŁı¼ +mar cu +elli ot +righ te +di oce +ru ss +rail ways +grand son +as cen +apo logy +awa it +mob ili +re spir +parti san +oli vi +stri ke +yo o +white house +expre ssed +pu ps +bed ford +cul tur +fro gs +fly ing +cav ali +c ds +fri ger +street photography +re solve +tali ban +kan g +cru shing +ju m +ðŁĺ Ĵ +william son +tan g +cur ly +t man +veter an +fa ire +artificial intelligence +un anim +pre n +back drop +fr ances +oc cer +doro thy +work ing +ar thr +conver ted +day light +serv ant +pad dle +compla ining +thir ty +nad al +ak u +ibra him +ad dressed +p iss +green house +batt alion +si mulator +out lets +embroi dery +ðŁĵ ± +fis cal +ger ard +sas sy +ðŁİī ðŁİīðŁİī +vent ures +mer it +public ity +ðŁij Ī +sophistic ated +c tu +conven tional +condol ences +isra el +tra dition +ar an +te ss +gla d +ðŁĺĬ ðŁĺĬ +correc tion +ge on +am d +or ship +be ast +ch ment +ì ŀ +nic o +wk nd +wel s +cushi on +beli e +vo c +idio ts +under neath +pu ma +corn ell +en ation +lu l +swa ch +ab ig +u rer +mi e +form erly +ca f +er nal +chor us +juli us +sen ator +âľ į +wh ir +salv ador +ph d +uni fied +boo ster +graph ical +w rec +son ny +mi z +dere rs +s all +ven s +tusc any +wi d +y ong +kur ds +w az +trol ls +mac ro +cat urday +pre ssing +sa sha +cent ennial +gu sts +em c +be fore +den ise +cu st +ðŁĵ ¢ +lo oo +base l +eng land +y olo +ar du +manife sto +do ha +ì ľ +kni ves +bourne mouth +bi bl +bar b +al icia +Ø © +com er +cycl one +g it +ane ws +character i +vent ura +in tra +sf giants +hu t +be a +dar win +ell er +al v +re ese +bl y +kar an +conclu sion +man ny +fla kes +unite blue +nad u +co pp +ed ges +lanca shire +i als +o tta +philipp e +l ent +che e +ment ors +festi val +an ism +compli mentary +r j +pu g +d ine +we i +cli ffs +sar my +ti veness +treas ury +il and +after math +rabb i +ou n +bou quet +herit age +zi on +sur render +shen an +in ks +kar l +gh ty +pol icing +exam ination +ce y +per su +measure ment +hydro gen +lu han +âłĢâłĢ âłĢâłĢ +war i +о Ð +j y +fow ler +mis h +al fre +âĺ ij +bb naija +cat alogue +recogn ised +sa ver +hu skies +col in +mun do +si va +p ng +discoun ted +man utd +fre sno +de vin +prelimin ary +tro phies +pla stics +du g +pro cu +indi go +g ard +dy lan +pit ches +ground breaking +in son +bl ac +an thology +f h +expl ic +r ard +admi ral +so chi +la shes +splen did +en vy +ad v +sex y +festiv ities +stic king +bi b +thr ill +op p +ari el +botan ical +endur ance +fe males +br icks +vat ican +black pool +ber mu +br ough +roll er +bi d +sue de +sloven ia +mm ing +ml b +med alist +di ans +rehabil itation +ne on +s go +li thu +ram os +z ed +pi anist +inten sive +broad band +stu dy +peter sburg +lu ca +ah hhh +phys ician +dill on +tele com +gri ef +mu n +ac ro +si ded +s ly +blo ws +classic cars +tri um +ar gy +? : +h ri +marsh mal +âĢ ĵ +to pping +war saw +tran sc +preserv ation +b av +re friger +experim ents +ä º +gl it +sli ga +g age +fac tor +flav ours +br ony +sp o +cook book +carri age +aw ay +ny fw +on ian +w g +simp sons +ro lex +ðŁı ¿ +cro sby +ãħ ¤ +cre di +syn dic +pu bs +ali fe +poor ly +mac ed +ðŁĺ ŀ +behin dthe +w enger +n ats +ðŁİ Ł +rubb ish +procedu res +typho on +opho bia +er do +fu el +vi era +bu mps +millenni um +new zealand +lec tures +it on +mil ky +respon ded +ê ° +landsc ape +.. @ +bo ther +âĸ ¶ +z hang +huawe i +tu ition +s worn +in u +y or +pa olo +au ditions +ab il +malay sian +ho ps +fe athers +mp le +au ts +ã o +boun ty +ic he +ì ĺ +sh q +pin ot +ge ars +disapp ear +video games +t na +alzheim er +ðŁĮ ŀ +a ji +under wear +swit ching +sign age +o scar +ec on +dro w +cl int +pl ated +gun dy +emb lem +ho es +ici st +nel ly +juni or +road show +miner als +at le +alexand ria +ac claimed +v ell +shi va +ad he +en ne +amne sty +h ounds +councill or +ðŁĴ ¦ +aes the +part nering +influ enced +mag no +fl are +extin ction +civil ian +maje sty +va il +law makers +rac ks +mc c +ori an +sp ices +er rors +may er +co ca +pa i +s ooooo +reti ring +ba thro +ðŁĻĮ ðŁĻĮ +âĸ ª +su f +endor sement +buil ding +broo ch +pal la +arvin d +ag ent +kar ate +r hi +c tv +ta ine +um m +ba x +reig ns +uni of +enterpri ses +adel e +fla ke +at tire +bru ce +ba hamas +gra vy +sa in +che ek +tri vi +lo v +e en +bb lo +lady gaga +itt a +. "- +du stin +observ atory +eigh th +bloom berg +kh s +f cc +gi st +commemor ate +ve er +sexu ality +ed c +nic ole +vac ancy +u ser +son a +:' ( +dipl oma +t end +up grades +Å Ł +jura ssic +cardi ac +dr s +widesp read +à ł +dail ies +vend or +sim plicity +wi der +len ses +supp lements +de pos +ob served +vin es +parti ally +renew al +collabor ate +ali g +fin ity +ph u +zz y +pe tit +ðŁĵ ħ +z in +i gu +sm ack +fall on +ðŁĵ £ +back wards +comp onent +o so +compati ble +bin ding +zur ich +thom e +w ounds +ly ric +fresh men +sne aky +fi bro +di et +emplo yer +in sect +h ated +sch er +raz or +n sw +boo ker +califor ni +av fc + ° +preten ding +pep si +al is +un titled +k art +grand parents +e the +o ck +lux emb +visu als +small business +abdul lah +min ho +su baru +h ra +reve aling +heart breaking +clar ity +am g +sl r +** ** +âŀ ĸ +recor d +ici ary +min ded +ye h +exce ssive +knu ck +icec ream +tru th +ev ic +ta stic +ant arc +ren dering +, , +mit t +loren zo +st patrick +bound ary +zi g +vo cab +osa ka +fur n +tu n +gu l +s ounding +blo gger +utter ly +g af +adv ancing +l cd +mar gin +lifel ong +solst ice +sh ra +wa its +ple ar +bre ach +en ligh +ad er +itt le +c ation +ho on +stu died +?? ??? +k ash +ev angeli +ps l +wei ghts +met als +ty res +tur no +wi e +car b +g ale +se al +sun ite +am ic +patter son +á n +eu ph +up stairs +quali fiers +khali fa +apple music +ìĨĮë ħ +vau ghan +al ter +cru iser +mu a +t ana +kat rina +id ols +spo iled +secre tly +fi bre +part nered +um es +gi ov +com et +screenshot saturday +k eller +fil tr +fe t +con way +pe u +bad minton +gi d +m ound +don key +bu ff +lea ther +lar gely +bro ch +int ments +am use +r k +sto ve +impac ted +con t +cr acks +prison er +bar i +contrac tor +ori oles +domin ate +pol ar +am elia +dr c +ðŁijĮ ðŁijĮ +vi st +su arez +injec tion +blo oms +ðŁļ¨ ðŁļ¨ +sti ff +pay pal +sno wing +thur sdays +goo se +we dge +educ ated +weak ness +de cker +abud ha +bree zy +Û Į +hope ful +o bi +rai der +gh am +de u +se ve +par tly +fu t +infu sed +mer ri +than e +some time +hu e +me in +cre dit +sli ding +ran de +cher ry +dead pool +sh ol +ar am +under wood +sky e +distur bing +m nt +poli shed +guardi ans +ha dn +pic asso +ari us +ak shay +ir ri +j h +happ en +la kh +dal ton +at the +s well +mar sha +re h +cour s +j kt +top us +serv ice +r ink +hack ers +dono van +hor o +tc m +may hem +cha se +dev ops +ken sing +sc up +sh ere +quali fication +c live +ton g +n ancy +mar is +der dale +ber man +cinde rella +jol ly +ci c +loo t +collecti bles +hom icide +g ge +epide mic +su ites +mu ddy +gi mme +e rec +- * +tal la +lis le +embro ide +ðŁĩ© ðŁĩª +veriz on +ve ctor +be anie +arti san +ga in +flo res +vi gil +u so +ðŁĻı ðŁı½ +grin ding +gh er +air ports +respon sive +shaf t +can cel +ceremon ies +e me +at ari +bru shes +eag er +bo hemi +children s +yan kee +ma a +suspen se +mor an +mac ar +sun flower +cre w +vo id +ke ar +fashi oned +jen nings +sunday funday +sub missions +me ad +her man +wa i +crit ically +le um +baek hyun +for cing +co bra +ãģ ® +acqu ire +al k +ge ology +pri mar +import antly +ire z +bunde sliga +curi osity +sen a +stric t +con soli +win ters +ven om +chelten ham +ðŁį º +cen a +t at +ba in +glo ver +under cover +as ses +car n +memorial day +am eli +i rene +ch on +syn thesis +spe edy +mitsu bi +sla yer +compos ite +under stands +pe w +inter rup +hen ri +mor row +an om +thof july +g lee +thre e +ðŁĺ ® +and hi +ch att +renew ables +ye s +trans fers +!!!! !!!! +bab u +du ter +lo ops +pe ers +o ilers +pau lo +ic ation +h mu +war a +mer cer +hom eland +fu ji +ale y +year book +re m +re en +ab sur +bo is +] : +caes ar +shot gun +kur dish +o ren +ra e +anci es +ty pic +f h +def ault +re plic +lu k +trans actions +r ys +infan try +ðŁį ¾ +cho w +chick ens +ba gh +wy att +ay e +gg i +bre ws +ed itions +mi ra +commen cement +pre su +peris cope +ic hi +guatem ala +zam bia +pain ts +wit ches +wan i +un dere +cro y +vo ws +us mc +hear ted +theat res +shu ffle +le vel +mul tic +squee ze +fer n +app et +post al +mal t +on board +ld nt +co o +s sc +k ac +ðŁĺ ĩ +sc rap +mar cos +deal ers +ann u +mill er +co ve +ul ary +vladi mir +be ef +th ur +pick led +se same +bengal uru +mo tt +kathle en +hi st +no tor +dr ank +du chess +snow fall +e ff +tin y +j n +sy our +speci alists +scot us +bay lor +eve rest +mali bu +pre m +harm ful +l ali +b ates +g ye +differen ti +and ra +geome try +el over +black out +== == +ko ta +inter act +asi an +la yo +samu rai +fi del +exhau sted +gla di +pd t +spher ic +anti qu +guit ar +stu ri +ho pper +ang le +f ills +sla p +mi th +rod ney +ong i +in som +pre venting +cassi dy +ap ho +ore gon +lo in +ham mond +contribu ting +f n +gar ri +ori on +comp elling +escap ing +aim ing +plu mb +bi stro +be asts +concer ning +bo e +do pp +shop local +stumb led +âĤ ¹ +naz is +âĢįâĻĤ ï¸ı +gest ure +war ts +us open +hi ggins +char li +hang s +bom bers +° : +fe eds +c ch +st il +nic ola +ðŁĵ º +clam ation +tro pic +af ro +ou k +expen ses +der rick +al ine +fa w +reg ard +im er +sat in +thi um +ry der +pear l +te ss +mm mmm +sen ses +ðŁĩ ¹ +positi ve +exhau st +occu r +nor ris +lil ly +is les +direc ting +yo fficial +count less +sam ar +on stage +flo ck +mir rors +arch er +mo i +k d +vi v +in os +si kh +le i +sen sory +br its +kno x +chest nut +op y +coli seum +z af +di vin +adap ter +:) )) +tem ple +ku n +hel mets +t df +gu ide +m old +o ids +lu ther +he is +monaster y +sp ree +k lu +brit ney +jagu ars +gre ats +c cc +ky rie +machin ery +cric ket +re ro +ab o +aspir ing +semi finals +ale ss +sig natures +var d +me th +her bal +hol den +king dom +ap or +reg gie +ore o +palestin ians +em mys +sec tional +ro i +ney mar +qu el +cu ll +l ka +haz el +estim ate +ul ties +go w +be a +purch ases +bel ts +protec ts +m é +gue ssing +bb o +clau dia +fr acking +jon ny +el k +cel tic +al mighty +ra je +courty ard +ig i +can es +ðŁĴª ðŁı» +bank rup +le thal +âľĮ ï¸ı +graphic design +vad er +penc ils +rough ly +dan te +m fg +const ell +cam el +j b +bloss oms +en to +balo chistan +cine mato +ill ard +jer sey +con sent +dent ed +con templ +sch er +hol i +lou gh +st our +a yo +begin ners +cur b +v hs +a jax +du ff +av eng +dom est +commit ting +ai red +cha p +hedge hog +disappo inting +freel ance +in land +char ms +ðŁĺį âĿ¤ï¸ı +ai sh +m x +buck le +ti dal +per mit +bo ating +ra cha +kend rick +b ello +b hi +ple a +estim ates +l b +apo logies +jay a +bb l +ast oni +inter state +main taining +el bow +mu p +ep it +ðŁĺ ¡ +viol ations +def end +be h +sl c +am ir +pur i +ti um +fi fa +blur ry +scri m +ðŁĻı ðŁı¾ +ma ple +rel atives +âĺ Ŀ +cho c +con nor +⾨ ⾨ +whi sp +list ings +ma ze +than king +ri dd +grass roots +shi fting +desper ately +gor illa +den i +ju les +stra th +g ley +ja in +bu ick +t anner +ðŁĴ Ŀ +ga e +pri m +it ors +n ano +separ ation +armen ia +bor deaux +ðŁ ħ +pj net +bu rial +e bon +glo ss +re new +gri er +spe eds +comic books +sym boli +pur poses +ãħł ãħł +spati al +no table +ci on +n ps +ho ffman +nor man +rt g +du sty +situ ated +tr an +k fc +em en +nic kel +hast ings +sett ling +gr it +l ena +w aw +art s +gu m +ca regi +le wis +sapp hire +rememb er +embed ded +t lc +bl at +serge ant +el sa +boot camp +bow man +photo graphic +pill ars +direction ers +classi fied +no is +ve er +barre ls +wh oop +ðŁĺ± ðŁĺ± +fe male +petro leum +medi a +e fc +poké mon +ठķ +enthusi astic +var un +pro files +pedi atric +acci dents +con rad +jan g +jo jo +ac or +ob server +l f +live stock +for gi +fo s +el m +an and +go e +c ere +avoi ding +gri t +om an +thank fully +scat tered +nick y +cylin der +chees y +di ver +mahe sh +cav es +ear liest +qu inte +subjec ts +b end +gul f +vocali st +glu e +pat ches +un stopp +sny der +demonstr ating +pi o +hor ns +wic kets +and the +r ama +yo on +stra ight +bed time +or ang +bul lets +sa urus +min ers +inci dents +! ... +ðŁİ ¸ +ag ers +hand les +stat es +in ity +d ons +incredi ble +emin em +avi v +ru dy +moz art +folk lore +appli ances +mt l +fre y +di as +hu a +page ant +stri ve +im prison +bul lish +r ana +al erts +bb mas +hy per +derby shire +re cre +re dd +debor ah +cosmo s +law son +mel anie +psy cho +ho or +doo dles +sni per +shad y +man tle +canadi an +new year +inter actions +separ ated +cor ds +spiritu ality +ap u +it o +p ct +pel osi +rebel lion +se iz +wor cester +sec tors +ul i +san ta +Ð µ +ðŁĩªðŁĩ ¸ +bi ased +class ical +gam ma +dee plear +emer ge +back er +sur ance +hand crafted +ðŁİ ¥ +franc is +mill an +ic i +cro wn +wo w +stri ped +un fair +relax ation +³ ï¸ı +embrac ing +she alth +pale o +martin i +dist illery +wr ink +or k +na th +hay ley +cour thouse +si ber +sa di +quiet ly +mel t +m sm +me h +smart phones +rel ent +pp ing +war wick +co logne +gli a +cot ton +pro g +lon e +ip sw +star ters +expan ds +u mp +su ed +ski pper +infe ctions +ing le +à ¡ +cler k +demonstr ate +ac ar +ðŁĺĤðŁĺĤ ðŁĺĤ +ti bet +bun s +alo m +demol ition +ssi a +g st +[ ] +so ar +âĺ Ģ +ðŁĺ ª +ðŁĵ Ĭ +dee pest +beyon d +are t +att ends +activ ated +di mit +âļª ï¸ı +high lighted +magaz ines +rum or +az za +steph ens +dol ph +sho ckey +mat s +we av +mel an +serv ers +tra um +ku sh +æ Ĺ +bab ys +pa z +a al +la use +break ers +canter bury +ul ture +mi ri +euro s +tane ous +impre ssions +du tch +il d +gh i +pur due +adequ ate +l p +sy ner +ang ler +du rable +gal ore +ro wn +mg mt +ðŁĵ Į +lu cia +âĺij ï¸ı +zay n +bor row +. ( +north umber +cru sh +eng a +su sh +extra vag +t out +ma hal +ali stic +ther mo +gall eries +es se +chi bi +attrac tions +lex ington +legislat ure +docu mented +resi den +brow nies +w f +st ool +plan ets +sho ppers +conduc tor +ms p +tr icky +fru ity +end ra +feel the +whi pped +hair style +re fer +oo k +oc topus +audi ences +ku mar +after no +op tim +c fl +ni p +gen i +alpha bet +ann ab +lam in +accep ts +l ng +ðŁĺ « +t ine +ac om +cheer leaders +t k +gr on +v g +k ung +ja x +dha bi +r ss +mack enzie +beir ut +clean up +gy psy +st ell +bur ger +hurric anes +educ ation +st ina +âĻ¡ âĻ¡ +unfortun ate +jere mi +bad ger +at ers +: âĢ¦ +ter ra +subli me +stu d +y mca +mr u +duter te +bren nan +bul b +mel o +yl on +hack er +c red +gu d +as an +pad illa +embroide red +vietnam ese +pione ers +projec tion +re boot +id c +an ey +pri mer +suff ers +win ding +p on +sto day +mor n +u ch +all in +adid as +eliza beth +tu ck +o graphy +ðŁļ Ģ +be g +os borne +ghet to +r h +cn n +ir ma +ma kin +cab les +mur ders +oc ks +inst a +al as +si k +cu ff +la re +foo dies +o vic +at om +geome tric +em pathy +ภµ +cent enary +newsp apers +administr ative +ðŁİ Ĭ +sti ve +contrac tors +le tt +tas mania +awesom eness +den sity +ve en +prince ton +frequ ently +re ject +gh i +modu lar +ceram ics +sh ag +ki wi +can vas +sweat shirt +an j +ti mm +napol i +il er +appe als +hamil ton +ma yo +we ave +arrang ed +whar f +occu py +b vb +as aki +ot ter +nor m +vi es +de tox +tion al +dere k +id ad +ad missions +constitu ency +u pper +woo t +allo y +se ve +lu b +un comfortable +ed win +ab re +d wight +ar che +virtu ally +sp ol +pri e +ai i +er r +swit ch +bar ack +se ok +cou l +wn t +pou l +o live +caffe ine +cardi ff +notor ious +de mp +ex cess +bar r +t ford +a jay +bump ed +my thology +shel ley +fal con +shakespe are +must angs +no ted +bon e +civil ization +sy d +par sons +un official +hy ped +sp ends +oppo sed +v ings +space x +noti fication +deci ding +bio tech +out si +sal ah +! . +fe d +ss y +c ms +bad gers +cr o +ela ine +n ba +dy our +n ant +honey moon +climb ed +conom y +ath a +m ell +ne bula +nature photography +juli e +bm x +inve sted +mon o +lieu tenant +wat kins +techn ician +o se +ka e +ì Ľ +mc queen +pre ach +trav eller +flexi bility +ze bra +reta iler +p ant +ben der +brand t +squ id +war rant +veri fied +cas s +pier cing +hon ours +t ying +mor ris +kis sed +op rah +panor amic +me i +splat oon +wich ita +ari as +gal li +indy ref +good times +athe ist +confe ssion +ow ski +re pping +ad ditions +mechan ism +z im +j ans +su f +cho pped +beg innings +vitam ins +ãħ¤ ãħ¤ +or th +po les +ru b +antarc tica +indie film +web cam +ket ch +bre tt +cle ment +her on +defe ating +hydr o +buc ket +wand ering +sid ney +future of +b inge +on ies +knock out +administr ator +syn the +l ent +jan i +bar ley +premier league +ner ds +cr m +bra s +bot any +evol ved +rot ter +ro wed +tum or +weal thy +Â Ń +mon arch +li shed +da hl +ðŁİ ĥ +bu ch +ken yan +Ø § +red ness +assemb led +se mit +hud der +shro p +ran i +lear ning +mor y +iti a +geo graphic +worl dof +f b +pho sp +boo gie +am ped +? ... +che w +dwar f +ar us +s sen +ru sty +recru its +h k +gar de +app lause +vol umes +invol ves +ta c +hand bag +trans late +ffe l +se ym +aqu atic +trans fer +zo di +and r +acade mia +cr ater +te z +ar se +adap t +col oni +snow man +mal i +hang in +di schar +oy sters +pho e +colon el +w ba +hispan ic +thri ving +sh y +ag les +sales force +cre me +so les +la fayette +â ī +ter ia +ach a +sp erson +go go +car ly +the ore +am ore +vo x +af t +ãĤ ¹ +stap le +mu ffin +di agram +ino x +su stained +av ent +me ta +arbit r +dec ay +ado le +Ð ½ +ec ol +ph o +n k +o cu +gr anny +ç a +luxemb our +stad t +alber to +le vit +am as +d x +or phan +co bb +as c +lo gy +immen se +chan ts +off line +p ent +bre x +w inger +plan e +i el +nichol s +ca thy +nar uto +low ed +/ // +ignor ance +cat astro +you ts +sch en +buil d +haz i +s ine +critical role +du g +dete ct +lo gs +en amel +stpatrick sday +ed die +co pa +cigare ttes +ho ff +kay a +la goon +ra pha +air borne +choo se +puer tor +ke v +gui ding +fro sty +bor ough +mir a +ðŁİ Ĭ +cade t +anu sh +yo gi +e ger +fl ing +slo pe +nin th +we ston +foot wear +f n +may weather +a am +pla in +stair case +witne sses +work outs +ro bust +dex ter +co hort +ðŁļ Ĺ +sp ell +ha ze +o om +organ ising +wild fire +cont acts +av on +min o +upd ating +ðŁį » +li thium +ing ual +k is +au ga +lo com +de duc +u da +th ak +boy le +mp er +hot tie +eri k +re vised +is la +travel photography +oo za +en qui +confe rences +clo ver +g room +cur ves +live on +per f +displac ed +bo log +xx xx +ðŁĺ© ðŁĺ© +te al +ve ssels +rain forest +cal ci +pan ther +gira ffe +ta sted +imag ery +pad res +day time +bas s +ri pe +opio id +nu e +vin yl +invent or +sen s +process or +mu t +gad gets +bibl ical +shann on +jacqu eline +car y +the resistance +ali en +n vi +co sy +bi har +fo ley +ren d +mu gs +fa ken +cl one +ni allo +gra bbed +chi hu +power house +n tt +chero kee +spon ge +imple menting +rh ine +le one +ðŁį Ģ +pret tiest +infra red +impro v +swit ched +tu bes +con tr +bl k +projec ted +be aver +yo t +bbcra dio +thi gh +per secu +apologi ze +w ack +po ster +oli ver +az a +lou d +( ?) +f the +women shi +spar row +blu sh +us able +sc ales +it ative +peu ge +ne eding +legg ings +glam orous +mat ur +c z +wat t +da b +tam ar +et sym +bau er +heart felt +h n +else where +bir ch +alu mini +hu ck +e me +j l +traf ford +d z +por tions +ana sta +arthr itis +esp n +ber gen +viol ation +yo shi +c z +northumber land +clo sures +ðŁĩ¯ ðŁĩ +smi ley +r w +tel ugu +inten si +gre gg +ve ga +dun geon +south bound +ba il +domin ican +semi final +chap ters +h itch +van ity +trans iti +recomm ends +sati sf +bar ca +queen s +( ( +de struc +stra it +ra vi +dess erts +in tru +har am +k os +fo e +fat ty +pais ley +magn itude +dri dge +com ey +schem es +vision ary +our t +down loaded +ðŁĻĮ ðŁı½ +gd pr +lan i +p wc +gu ad +nic est +stake holders +re ferred +george town +arvind kejriwal +schnei der +in doors +all star +strand ed +gen der +ze pp +ma sses +ðŁIJ ± +pati ently +bl dg +z ab +we arab +vi vid +he ck +d ella +sy mb +je opar +la ger +à ª +comb ines +ne c +br ay +flo p +tx wx +jo ys +pon t +pro found +sur round +mad hu +ma ble +ay r +te as +n sa +open ly +er nest +ãĥ © +to po +g na +anti oxid +ti an +e tr +c ello +ma thi +gener osity +b iting +man ic +kel sey +chee ks +ten der +w th +pron oun +ultimat ely +gu sta +ari anag +ger ry +ble ed +red dy +mic h +mitsubi shi +oper ated +sex ually +ma u +cl lr +vi ds +co c +mel ted +ðŁĮ Ī +q ld +ite ch +instru mental +end game +ðŁĵ ĸ +ener gi +brow nie +tam il +at in +domin ated +pra ises +fire place +sens ational +men a +k arti +un prece +ru pt +ori ental +mc cor +tour naments +scen ter +re eves +prescri ption +sam e +fra u +tru ffle +em bo +roman s +bla sts +techno logical +pr at +b sb +y ar +tren dy +ac l +al ad +ðŁį ģ +o hh +bankrup t +tho ven +regar ds +is er +war wick +vine yards +real m +niallo fficial +do ta +ge mini +to do +v able +¨ ¨ +la u +wre ath +ju ve +nat asha +le ver +lor i +hor ser +cc tv +air bnb +es anders +sin clair +ema biggest +high school +con test +optimi stic +t te +ðŁĴķ ðŁĴķ +ss d +ye e +hel ena +con sen +ric ks +jes se +an ic +ðŁİ ¯ +re acts +ro be +independ ence +vol tage +m ington +s ant +à¸Ļ ภ+-------- -------- +sentin el +ke tt +rehear sing +aaaa aaaa +sof the +stir ling +sear ch +wi gan +stand out +sna il +pent agon +Ä ģ +ch lor +cru st +net any +chemi st +disapp eared +ric ardo +sp iders +bo se +war ren +me ssing +bann ers +gu el +par ach +ma id +coun ted +epi le +bon fire +speech less +se tter +meas ured +rejec ts +nik ki +le ster +foren sic +fab rics +alo ha +pre served +wat ford +deta iling +dar th +bo u +car ly +... ' +tail gate +noti fications +å ¤ +pas sive +trous ers +balo ch +ro ther +typic ally +à ¥ +sp it +wi z +sic ily +technic ally +ex pose +st age +hu bb +cre am +cap s +po ke +sle ek +ju ne +tempor arily +de z +awak ens +l ame +_ - +ji ha +tues days +advis ed +advis ors +exi sted +dis agree +news room +lo sers +world tour +dr ying +al di +har ness +foot print +hobb it +p mln +i ro +que red +asse ss +gaz e +sa b +th ian +í Ĭ +ti f +ob serve +ev il +dra wer +swee p +cor y +co dy +kyo to +cal lum +n inj +lau rent +be i +sket ching +custom ized +du r +regre ts +knox ville +ìķ Ħ +mess aging +grac ie +abun dance +bi dding +bre wed +fl ouri +therapeu tic +alt itude +ho gs +bur ner +elec tro +wonder fully +he ater +post pon +li very +r all +ad as +a ac +sau l +brook lyn +play house +âĻ¥âĻ¥ âĻ¥ +char itable +in y +z ah +compet itions +be av +plu gged +o is +do om +astron om +speci alized +max i +ta ps +cellu lar +depre ssed +folklore thursday +cri b +e mul +ë° © +fi gh +ru z +car lisle +spe ar +side walk +de i +depend ent +lac es +nh s +ðŁĮ Ļ +reali zing +net work +ric he +re gin +re fresh +st ral +pa thology +pla id +psyched elic +hin d +u ka +algori thm +lin king +progre ssi +fe y +d ade +hydr ated +b ant +fam ed +cot sw +bo ise +as c +rac ing +ja vier +ww en +mar lins +poo p +swe pt +toni ghts +we f +ani me +slo vak +âŀĸ âŀĸ +cla us +lem me +cli ppers +re ls +arianag rande +r te +ko t +thal apathy +hungar ian +zu ma +y von +is u +jour neys +clin ics +be be +ww f +n ws +super heroes +er it +sle ague +identi fication +mo tto +ba i +sour ced +ill er +ap i +pri se +unprece dented +dam as +tuni sia +dra in +undere stim +e ther +quarter ly +rewar ding +al ham +wolver ine +cab ine +hyp no +nad ine +hav ana +da e +ðŁĵ Ī +dr on +read ings +b ati +pic o +mer ci +iti an +wal kers +el ope +mi key +god zilla +bur lington +abu ja +social ism +at ility +sh ell +harry potter +g no +ab ur +re leg +fel ici +ro gen +neuro science +inst in +ath am +vou chers +j arre +fu se +def ici +monte rey +de port +mid day +pp ard +fre ed +ame ter +wil t +n ingham +pr att +liber ty +slo gan +o to +pr i +co ated +c pd +ne tt +il las +mal awi +evol ve +accessi bility +ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ +or nament +b p +el is +son line +chi ro +fl ick +ib m +ar ak +en ables +gar land +san e +cu ties +tri p +rotter dam +n ys +lam ps +lu cas +bo g +ra ils +travel led +hic ks +en u +sab ha +scru b +hi er +hart ford +fo o +fer nandez +tre vor +mat tress +appo intments +ale j +fe i +o logist +saf ar +oc ta +sr c +sha un +ambi ent +dri c +bi ker +she e +must ache +h ta +bo one +her ty +car dio +bra kes +rec ital +consi sts +overwhel med +cau l +robb ins +im it +al th +ur l +bi bli +on ne +black livesmatter +diffic ulties +tel ang +tall er +ðŁĵ Ĩ +deb ating +bur rito +mo vember +strength ening +bo e +te stam +mirac les +base ball +re nee +ðŁijī ðŁı» +al fa +âĺ ĺ +unstopp able +ec s +g mo +giftide as +path way +fen cing +ðŁİ ¤ +b ham +ra s +sk o +d led +thel ast +magn um +bin ary +wil de +wil der +wh ati +barbe cue +h ism +can oe +kur di +eli ve +advant ages +mad ame +bi er +mis sing +enter tain +air force +y ama +c is +hash tags +j is +ve il +dream y +ten se +may ward +ch ateau +hunt ington +âļ ĵ +v all +up on +bl ouse +dun es +ðŁĺ ´ +fert ility +m ole +curren cies +st u +ber lin +toa sted +div as +wal t +lar k +por a +hit ter +um er +chil led +bal ancing +fa is +y in +or tiz +east enders +h ate +ur al +ap ril +tim el +à ± +per o +sto cked +respec ts +th t +best friends +giving tuesday +be ad +inv ent +im i +nap les +comb ining +tok ens +thir st +ma sc +par rot +sp u +dent on +* -* +t res +subur ban +wid th +si ve +con tender +siri us +lo k +troop ers +outra ge +tur bo +frag ile +me ssed +do h +disc ord +netany ahu +re sign +forgi veness +mo han +mun ch +cam ou +identi fying +enab ling +hot ter +thorn ton +jai pur +ar ya +ðŁı» âĢįâĻĢï¸ı +mu staf +maj ors +o ke +du ffy +roh ing +til t +ðŁĩ®ðŁĩ ³ +rock star +she ep +hend rix +ra v +in vention +do u +lagun a +gru mpy +sw is +im pe +) ' +you ths +bun ker +st ache +oppo se +indi es +acceler ate +ml p +ed en +w ann +k ail +akshay kumar +su pt +pol ym +midd leton +extra ordin +wil son +australi an +alumini um +way ne +alum nus +mat ics +gri m +er nie +opp a +competit ors +rand all +h ence +decla res +pre aching +sha he +can e +sustain able +stap les +le dge +ad ena +doctor al +bur gundy +decor ate +ren dered +ri sen +pr ank +di or +bee thoven +flo or +ac com +to t +ho dg +touri sm +say in +objec tive +mar kers +premi ership +en abled +camou fla +gi ant +Ñ ģ +smo key +ric ket +pan g +de pending +s ation +evol ving +inter cep +cen sus +tof the +re en +mendo za +trum pet +marke ters +an it +ðŁĻ Ĭ +north western +v la +foto gra +blackand white +che wan +wi g +tro om +ginger bread +k n +ro mero +n fc +or chi +fun ko +sour ce +f s +ra ped +o st +tar ot +ann ually +ðŁĺ ¬ +r ill +del av +.. !! +se s +can n +medic are +ph el +ape x +guardi an +rema ined +r pm +a ñ +story month +instag ood +neighb our +p ing +sem ite +my stic +as cot +mat er +hand ful +dang ers +ti d +ana heim +opol y +sh allow +nami bia +tor ia +procu rement +big bang +announ cements +prosecu tor +beng als +sal le +en roll +ga stro +sugge stion +ba k +ha ul +budd hism +berni esanders +flu te +fati gue +cyn thia +cho i +ir win +gu a +str ous +h p +ba p +satisf ying +play a +ðŁİ ¼ +inst ap +al ice +t p +irri gation +ðŁĩ¬ðŁĩ § +in tric +clu es +ple x +sa x +he pat +dump ed +signific ance +by u +medic ation +pro v +tough est +corn ish +âŀ ľ +kel ley +u v +si zz +si bling +me st +di stor +diplom atic +aun tie +b hat +son ic +bren da +pump kins +ro ch +black burn +ur ged +shi a +arrange ments +floo d +sa unders +lec turer +nou ri +popul ations +diplom acy +consist ently +ðŁ¤ Ļ +t mund +cauli flower +l ily +vocab ulary +vari eties +coo ker +up town +qu ent +mo sa +re inde +velo city +spru ce +social medi +i ber +volun tary +proce ssed +bal tic +y ang +leban ese +d p +dol ly +arrange ment +y uri +cran berry +kal yan +elev ation +cli ff +pu shes +ìĬ ¤ +sil ic +co wx +eter nity +sla ves +vine gar +glou cester +con tained +breaking news +aga inst +renov ated +norm andy +hero in +ys m +mo ds +gre ek +un di +tren ch +v h +encoura ges +head ache +gr ange +: ' +ever green +Ù Ĭ +reck on +ab used +th ru +cho ice +ti dy +col der +scho ice +ha in +bru m +li ars +bre it +yor ker +sh ack +he idi +micha els +sco pic +fasci st +play ful +ca c +yas ss +sh ad +.. ? +qu en +ram irez +clif ton +pr s +best fan +âģ ł +gener ating +head set +disappo intment +abstr act +bo iled +paren thood +azerbai jan +exhib iting +bom bay +oli vier +ko so +un lea +mat ernity +iz er +si ves +r hu +col l +saskat chewan +fre akin +de k +na g +stab ili +ðŁį ķ +organi zer +bo sses +ar u +u va +at able +ta un +after wards +fert ili +ver ge +az i +mor ph +๠ģภ+jer k +cosme tic +ko w +stru st +ap ache +post cards +for mul +ì ĭ +spin al +jack pot +elec tri +Ã Ń +lo y +gra der +diab lo +ar di +he sit +f w +arch ery +pa sh +the ories +repe al +re live +per cy +âĺ Ĩ +im in +syn chron +sham poo +coup ons +o to +la i +thou ght +luxembour g +mo v +ðŁĺ ¥ +ge mma +se ated +m ga +strat ford +un certainty +shi fts +est o +fo ol +fire arms +cor rie +ki ki +appa rent +p ills +olym pia +fi d +elev ated +de cks +ignor ing +av alan +ro v +whist le +p tsd +milit ants +robo tic +pac ers +quil t +bankrupt cy +lic h +per cussion +celebr ity +al s +( ; +su t +pokemon go +h g +off s +gibr altar +scre ams +billi e +gen ome +mar in +be ams +arch bishop +em in +bedro oms +g ated +ol ly +warran ty +at own +cudd les +gun na +k ic +vi ve +cy mru +nar row +pro b +le o +refe rences +manufac tured +cho pper +brun swick +sem is +don ia +r ye +man o +hur ting +? # +hol li +investig ations +c els +ðŁĵ ŀ +le ster +temp les +sto rey +mc mahon +toi lets +wo of +ï¸ İ +le verage +at om +night mares +victor ious +haun ting +custom er +ag i +yo ongi +mon ty +ver onica +w ur +inti mid +blan kets +volu tion +j m +âĺ İ +am on +jud ith +ðŁĺİ ðŁĺİ +distr acted +dri p +hurric ane +and es +revel ation +tro op +ab leg +col lin +tibet an +wor rying +inter nationally +eat er +camero on +brad or +y uk +ðŁĴĹ ðŁĴĹ +tra k +slo pes +ci er +ne a +ol er +ta ka +albi on +volcan ic +am n +a fi +ob stac +face time +ger ing +n pr +metall ica +organ ic +ðŁĴ ¡ +ki dd +d ances +pemb ro +wash er +m its +om er +emo tionally +tan go +ip o +do cks +scan ning +spec s +tho m +the ology +emer gen +om i +g pa +selec tions +un necessary +ima ge +ter s +induc ed +gi gan +rent als +supp lied +m fa +shan kar +lat er +pa jam +cla ve +Ù ģ +ma hin +carl son +avi an +ano va +kati e +aj ith +design ated +chocol ates +investig ators +gla zed +prin cess +er ry +ra gn +ou rable +hr u +sun dance +peuge ot +steam punk +gh lin +gre ase +hi res +z ap +per ce +j ill +tom e +he hehe +joy ful +mae stro +ni shed +gene alo +v ich +p its +fox es +good man +emer son +lo bes +con verse +o ats +thom son +ra him +mal ware +ah i +man kind +re sin +im g +sw ood +kin der +sc roll +ar a +sak ura +ro bbed +xi on +ny a +c ism +ce dar +be in +mour ning +tor to +heath row +done gal +bar b +hydr ation +k or +elim ination +su pdates +hill s +appe ti +star red +ko m +gw en +dd d +cra y +sc anner +personal ised +seren ity +re design +meta ph +box ed +judg ment +no se +ë ¹ +er ad +ac ne +supp liers +ener getic +v om +as ap +ðŁĶ ¸ +ir vine +hat ch +la ss +ad ren +waff les +accur ately +ici o +itt le +se un +occup y +web cam +thene w +ent es +ga i +j w +accoun table +vis or +ir rit +licen sing +hudder sfield +gen ie +ðŁİ ¾ +atmo spheric +ten sions +spart an +clif ford +ol an +north bound +ame en +cen sor +u el +ster y +$ $ +far rell +hy ster +cl t +se dan +rep lied +descri bing +micro wave +sla b +pro sp +assi sting +ru bio +e than +hh hhh +gu ay +z man +ra ise +roll ing +o e +n ile +ambro se +scar borough +hero ic +coo ks +mor t +chop ra +ðŁĮ · +to b +shav ing +stac ey +dor m +motor sports +wi ki +fol ds +sp iced +stress ful +liter al +fu dge +pe ggy +wa ite +tre sses +se sh +pr ic +ðŁİ ħ +fri ght +r va +mumb ai +po m +tt v +cel lar +tom e +andro id +dor is +tsun ami +tin der +o ec +m wc +dor tmund +no thin +l iti +so u +believe in +at u +kno cks +mag ni +ss sss +ro hit +ine ws +ang i +m andy +ke ttle +intermedi ate +av ant +cur l +endor sed +ori o +ur t +consider ation +wi res +shel ters +b ino +vik ram +imple mented +ly dia +bu k +paro dy +c news +under graduate +canu cks +sam i +polit ically +ro tten +gh z +tex tiles +over load +moder ni +recre ational +fli r +bat on +typo graphy +ov ation +intrigu ing +pilgri mage +al ge +ad ays +tcm party +sp elled +cur ls +boo ze +ste m +ann es +ir ls +spon ge +sho pper +sig nation +bra ss +mi stress +le ah +beg inner +lau derdale +augu st +pre school +ta ping +tai pei +execu tives +b d +rhe tor +esc or +immun o +deeplear ning +stat ues +it us +manu script +ly ric +cor vette +mol ly +la ge +de p +cn bc +le st +je ssi +fi fe +griff ith +oppo sing +ran g +dr ills +respec tful +p ity +d ell +har ding +play boy +blo ke +shut out +k ili +o sp +se attle +bc poli +mis es +journ als +team ing +es ther +fre ddy +Ķ ï¸ı +metr ics +no tre +gar ry +for ty +navi gate +perio ds +bened ic +j id +da w +ance stors +restor ing +con g +aller gy +tit anium +c ence +lean ing +ab bas +v ast +uc f +roof ing +e man +seve rely +vo gue +ve au +in bound +d z +tane ously +stret ching +man chester +dr yer +dav is +kan th +the game +it ted +re tain +el les +conge stion +frat ernity +ol lie +lo ki +fre ely +cho o +pon y +sc ep +tab ly +bal t +rock n +di me +lo gging +ðŁį · +ad u +ha voc +water ford +char is +swee tie +run ning +ner d +erdo gan +z ara +weigh ing +fif ty +pre cise +low ell +kurdi stan +r yo +or th +syn th +lin ers +phenomen on +art illery +il legally +constru ct +nostal gic +gar th +al ta +shel ton +a sean +w ander +dur ban +di versi +bon o +cl on +le man +sh un +obstac les +appet ite +fe eder +respir atory +di xie +formu la +an to +so ber +extin ct +au c +ing les +legitim ate +; ; +min nie +ipsw ich +dram atically +ðŁijı ðŁı¼ +ingh am +milit ary +mon et +us navy +for k +dun no +play er +q otd +st oo +ex or +ethiop ian +film fest +pe red +c ate +sau di +in ner +sin cere +tion ality +ale e +de eds +cooper ative +ir onic +cro cod +br ary +post season +cam per +can ary +e in +exten sions +nb d +sher wood +spo kane +hu mp +jit su +ê ¹ +dar yl +p si +stab bed +offer ings +expe cts +cav al +body building +fr aming +f ca +ye arly +bom bed +sk il +resear ching +jud iciary +gree ted +tu dor +mil o +innov ate +ðŁĺ Ľ +r hs +ru by +contribu tor +fam er +soci ally +m lin +fi ery +ut ter +beau t +it os +de voted +rain bow +bar ney +pe ren +ar jun +r na +gab by +ut i +hann ity +pick le +ser v +qu akes +pp e +fe m +wh itec +j n +victor ies +ðŁ§ ¡ +gol fer +congratul ates +resul ting +mechan ic +ur ve +cen tered +kie v +an s +in cub +< < +c mo +bestfan army +dap h +en ham +on cology +ku sh +t xt +ori ented +fashion able +c sr +sa hara +r ack +pd p +han son +ภĩ +ti ers +ra r +pan am +in sky +sa hi +testam ent +asth ma +in her +fisher ies +or der +ho we +gall on +ep is +suz anne +drow ning +paneli sts +ðŁĺ ² +ë ¦ +al ach +commemor ative +at tribu +ðŁij » +mo o +visi onal +week sary +gu st +ak in +poin te +ee e +di spar +ni pp +dent al +st all +pi an +bor e +ul ster +tic k +ir r +tae hyung +micro phone +bermu da +ga ard +el er +plumb ing +hu gely +âļ« ï¸ı +race way +cam bridge +mar cel +burn ley +to ast +holly wood +fa sting +me red +hib ition +ca pped +benef icial +ow ning +cont amin +arab ian +to on +cap ac +hul u +sm ir +nutri ents +se in +graph s +con ditional +ðŁij ħ +or ac +play in +nor the +tor nad +mar ian +ju mbo +lex i +incredible india +road to +uk one +confu sing +sp h +shan k +pi ed +mq m +positi vely +sher ry +path ways +consi ders +tof u +argu ments +resil ient +che tt +with dra +ter o +ated ly +sw ana +he b +fli ght +har ley +decre ase +kind le +book shop +³ ï¸ı +marty rs +sm ur +mc cl +concer to +sti me +rejo ice +app lau +cle ment +mer kel +jai me +im mortal +isle of +mar co +youtu ber +stal king +me too +st ack +sp ouse +u st +lu v +âļ¾ ï¸ı +eque strian +ev ing +fl in +nick name +the big +as ar +st acks +wal ker +bor a +kidnapp ed +hur ling +humb old +rec alls +co pper +ann is +se o +mer ger +mu ir +ad dy +ðŁĴª ðŁĴª +be x +cr acy +con an +congratul ation +mid st +âĻ ¬ +for bi +op tic +cr ate +crocod ile +mad agas +secur ing +ast on +o gue +savi or +salis bury +love it +fuji film +cast les +as st +ar rows +sp acious +tr s +poly vore +progre ssion +m ri +nel son +bi m +indic ator +o da +pe pe +re signation +gu t +sne aker +log ically +az y +are lla +te aring +jo shi +ssion ism +q pr +mari ah +p x +ble ed +mi an +med ley +we iss +ker ry +gat ory +at al +madi son +av enger +nab y +pl and +gi les +fresh water +d ington +ta j +demonstr ates +n tv +bul bs +sunday morning +pe ake +souven ir +wa h +ton nes +m kt +complex ity +con den +ross i +b ing +y ds +su k +n go +mid land +ol y +life is +ri pple +mo reno +dd ers +tu s +á ĥ +bou l +x a +hol dings +wn y +shadowhun ters +ke i +asp ire +m ous +ow en +so ak +skir ts +moun taine +stor ming +ch rome +ri ots +sar ato +amaz e +less ness +nav ar +crit eria +ra fa +indul ge +ay er +por to +nam o +........ ........ +yi elds +val le +j h +mac ron +sa ins +dur ant +tra ilers +wo t +confeder ate +sh rin +id ol +form ally +ten e +motor cycles +than g +no de +bang er +dal y +p ats +enroll ment +au ctions +at al +ar bor +lo gos +de arest +trans action +dom ingo +fle a +ser mon +de ck +sin cere +questi oning +juli o +was p +pre tz +armen ian +k ham +inflam mation +picture sque +acci dental +film makers +ðŁĺ ļ +ðŁĴ į +ca sey +so b +yee zy +good will +parag ra +ss ly +fe ather +dy ed +assassin ation +na de +b cs +app lies +femin ine +fe u +ext ent +depu ties +l ack +psy chic +go i +kill ings +pse u +ðŁ¤ ª +un c +mar l +tan e +mck enna +sur fer +influ ences +free way +hack ney +mal aria +el and +te au +rema stered +Ø ± +raz or +gg y +cor ro +lak sh +fla ir +honest y +hoor ay +de pp +am c +wedne sdays +q a +ed its +- $ +se villa +dou bled +human ities +c cot +som os +r ine +af a +si oux +re construction +wel ding +th reads +am ish +encoura gement +po der +bo ck +bal m +p tions +stand up +accompli shments +guar ding +convic tion +ac ion +napo leon +depic ting +att ack +su i +wear able +âĸª ï¸ı +pot ter +esc ort +vis e +to ts +bo on +event profs +angu lar +womenshi storymonth +bar row +sch i +ac comp +ti k +l end +kensing ton +wol fe +st acked +cra shing +exhi bit +wing ed +sab rina +ma sa +k ms +alway s +et t +pla sma +counsel ing +pick les +nfl draft +mr s +inev itable +coura geous +staf ford +writers life +ho s +e j +gh yun +trade mark +adri an +influen cer +coron ation +ra ging +explo red +usa f +excep tion +eu x +tan ker +sw ami +pac ket +ðŁij¨ âĢį +f en +she en +a ero +j l +re gal +nw t +au ster +meh ta +char ge +a ste +b ate +inf eld +racec ourse +collap sed +fle ece +z il +al lie +alternati ves +geor ges +ðŁĵ į +quir ky +fc b +nat geo +philanthro py +bra i +every day +ðŁIJ ° +ach ers +ja an +fin es +q i +fisher man +distin ct +gri mes +nation alist +comm ence +ro wn +âĢ ³ +z ing +f ter +hr w +baro que +bl ender +kitt y +hoo ks +c ited +w anda +consen sus +reinde er +an and +supp ly +me ds +v n +ol ph +rat chet +shel don +secur ities +ë°© íĥ +cro m +mosqu ito +j eric +im mac +dimen sions +â ¤ +di ssi +sponge bob +dami en +steven son +jo anne +del ish +yi kes +than x +surve ys +postpon ed +alco holic +al ised +ðŁĻı ðŁı» +do ch +sen tim +mered ith +com pares +b ago +happy days +mo ss +ãħ ĭ +ne c +gn ment +frustr ated +comb in +ri v +ec lec +col lo +compli ment +actor slife +ct to +nic ar +op hon +apar the +man t +ja de +trol ley +optimi zation +eye on +eco logical +qui st +ep he +ॠĩ +cin co +appo ints +old school +c pr +behavi oral +min aj +:- ( +tag ging +ev al +jo aqu +ðŁĺ « +ha k +de me +jama ican +so s +hy att +hand book +libr arian +hanni bal +pump ing +ch om +f man +ga i +hu ll +respon ders +green ville +n us +vau gh +ðŁİī ðŁİī +ta xi +gold berg +man tra +te ase +forbi dden +metho dist +ati vity +* *** +ec t +mc gr +Ħ ëĭ +se b +amid st +disapp ear +thy ro +phili ps +er ina +v icious +stream er +million aire +ma p +str ick +hack athon +gh a +ed ic +mi ka +pe ck +ill i +anto ine +ar ca +op tic +ma ure +ðŁĩ¦ ðŁĩº +cla shes +man ly +âĺ ģ +al var +and res +me i +el m +ww ww +al tered +l te +ê¹ Ģ +mo jo +for rest +thal ai +non t +spee ches +acknow ledge +ign ite +x factor +ðŁ¥ Ĥ +mead ow +disru pt +debu ted +scrim mage +pharmaceu tical +fi dd +found ations +philosop her +et al +publi shers +bo ys +c ke +ru gged +opti mism +re be +phil harmon +nar cis +ral lies +lu is +go blue +fol ded +un acceptable +optim al +li sa +pol aro ++ . +en za +âĿ £ï¸ı +mon opoly +grace ful +dair y +du a +diffic ulty +judge ment +o si +mer sey +flu x +new found +ter ns +dimen sional +in vic +al ba +am it +abudha bi +alger ia +autom obile +the ad +lo tion +acceler ator +vac ant +iti on +lu f +al ic +pl l +bla zing +ba z +sen e +ðŁij ¼ +villa ins +direc tory +eis en +to ck +broch ure +ri pp +hb d +zayn malik +nic he +lo lol +certific ates +mor se +fac up +x ham +un wanted +im ports +carne gie +fan sign +mo u +r alph +destroy er +sw ing +trek king +cili ation +pit bull +g aps +ho well +defin itive +mc le +f ps +et z +bol ly +lyn n +gan o +at ure +fur suit +co il +na v +but ts +tro jans +eu re +en ko +sch umer +horri fic +install ment +br b +subur bs +a bel +vi r +de sh +cun ningham +ðŁIJ » +span n +sch we +ke mp +tr u +ste alth +qu es +le w +deli ghts +ko ch +hu mili +cr iti +il t +sp ells +mi ley +car ic +ðŁį ´ +lc fc +substitu te +oun g +? !! +af fir +predic table +class of +er r +cy press +chand ra +age ing +__ __ +ther land +don caster +el in +yo shi +sail ors +har ris +jo anna +niger ians +h ers +pla gue +pro cra +k no +can ton +busine s +un h +pra kash +c in +bow en +co ating +m als +be gging +smith son +ponti ac +sp ies +dam ian +pl ine +und ant +al ta +one ss +shame less +da q +bb m +wal es +stam pede +ser um +Ù Ĩ +cataly st +x n +ab sc +free zer +ch un +ari os +mc cre +fore head +he ars +damas cus +tac oma +ardu ino +encoun ters +stan ton +lg b +ab as +" .. +ke te +drac ula +ele m +g ne +zepp elin +la brador +pul p +op tional +or n +russi ans +san itation +hil ary +etsym ntt +pen alties +au st +ig ans +olympi an +medic aid +vers ace +va pe +re stra +pe ep +sexi est +st alls +di le +the a +punjab i +pupp y +tuesday motivation +ðŁĵ ļ +the flash +roc ket +mo dest +chihu ahu +on na +k sa +hur dles +ca ve +fail ures +sp lit +bo ho +gur l +disappo int +ho ward +nug get +fran z +stal ert +kaz akh +for getting +sch ri +ag ate +am at +eve rett +du et +veter inary +juli an +ch ills +bra ve +ghost busters +lan do +gre ets +profit able +d é +ti r +ze e +om en +pd x +gray son +har i +fix es +stab bing +swim mer +symb ols +compli ments +po se +func tioning +th nx +gi r +corpor ations +bar low +lo e +off season +distin ctive +marvel ous +nik on +enri que +ky u +ja ws +amo to +lom bar +travel blogger +fa h +ouri sm +tri stan +so e +ce ase +ðŁı ħ +z ac +mck enzie +taxpay ers +swim suit +bl o +les ley +kan sas +w ks +ki el +provo king +my les +str ing +kangar oo +galac tic +fif th +s ke +we ir +ll is +mat ory +ðŁĩ ¿ +un ci +re productive +roo ting +ti des +gad get +.... ...... +alex ander +bow ler +scre w +apo log +eri ka +wal ters +shet ty +lan e +ban ter +as ant +me so +v ain +" "" +us i +fer din +accomp lish +man sfield +bom bar +collabor ating +cla p +it ure +s da +smo ky +na k +im person +car la +com ra +bur gl +lo co +ti es +in hi +trac ey +se is +diss er +rr rr +dra y +prote ct +cor ona +hun ger +ck en +c eli +trou bled +predat ors +fic tional +shav ed +riche st +metab oli +ful ham +gro oming +mono chrome +wa sting +as co +ast e +ti sta +remedi es +ung soo +south end +perman ently +bu mble +procra stin +ident ical +practic ally +ma scul +su ke +assu red +val erie +devi ant +grizz lies +thi er +pur a +ne pal +not ts +bil ateral +spo il +car mel +cine matic +ph l +ni fty +ma o +hypo cri +la ser +pan try +mathemat ical +el isa +coordin ation +bel mont +a it +radi ant +bo iler +man g +f ag +cr c +h ams +br in +â¬ĩ ï¸ı +famil ia +âĿ £ +sab er +ru pert +gg an +rit z +mic h +sal ford +le vi +gra l +ðŁĴ ¤ +n ino +ce d +business man +ul tr +sim ply +compre ssion +pa ins +hal t +ë°©íĥ Ħ +landsc aping +n f +croo ked +er d +itt in +ddle ston +sur passed +ino a +da g +bl en +exten ding +at ing +al gae +ball er +u mar +snoo ker +col lu +flo wn +thu b +ridic ulously +ki sh +op le +di re +as ser +ari sto +sc iss +h ating +trou ble +syl via +suc cul +plo ts +sincere ly +al er +laure ate +br ack +att n +rif les +me to +collec tible +cu omo +conte stant +consist ency +ant z +rang es +abig ail +de b +mini ster +grow ers +an oo +hoo ver +dream er +nu cle +resear ch +mi y +sha hid +ma v +d honi +cin i +do j +hin dus +part ying +dal i +alon so +inform al +clark son +it ton +ki an +cit yo +mor i +la sted +as pen +libr ary +susp ici +qu at +den ial +fol der +ch ori +swee ping +eni x +ðŁį Ĥ +Ø Ń +nas car +handmade hour +mou l +heat wave +em er +exam ine +ib n +gr ind +po v +tion ist +m bo +she ila +integr ate +om es +take away +cer v +con nie +tic ket +ce led +bi en +visu ally +madagas car +sor ry +gu i +park run +tra its +la be +pois oning +ॠĢ +vi able +bohemi an +denti stry +bad os +spr outs +mask ed +te ddy +ðŁĺ · +sa f +sa as +ji ang +ti ght +spe aker +withdra wal +bc n +as signed +class rooms +fle ming +ðŁĴ « +super girl +tot als +table top +e books +horizon tal +cra z +flu sh +j ard +c dc +er son +ãħ ł +green wood +ni h +co x +ad a +lit re +go ing +v icky +cur ved +lou ie +gra ins +hy e +lon ge +reme dy +tra inee +san jay +super stars +ma ser +man u +s age +wh l +ðŁĺĤ ðŁĺŃ +ðŁijį ðŁı» +m sd +en z +rab hu +j oo +gh u +ac er +e po +resurrec tion +justice for +bl ended +mo da +avalan che +france sco +re spective +g s +ye ast +wel ch +devo tion +ge tin +athe ism +am ic +carol yn +lo c +ld nont +ave c +us da +le gged +bra very +b lower +cow boy +he h +sti ble +buff al +chann el +run chat +âĺķ ï¸ı +ide ology +best seller +y oo +pe anu +bon ne +fel ic +edi son +fr actu +naren dra +pp ets +seym our +ri viera +he ctor +necess arily +bi anca +soci eties +the best +w g +sent ences +win k +vacc ines +pal ooza +jam ming +as f +mp us +agre ements +ec k +ba c +hon ore +com pul +wild cat +im posed +yo ga +hud son +can celed +l ich +fu zzy +es que +ch uk +w vu +se k +fli pping +r hon +wi shed +wh a +cap ability +len ovo +ìĨĮëħ Ħëĭ +vi vo +tv d +nor a +sil k +pas adena +yo semite +valu ation +clo cks +u ber +mr c +dar kest +au bre +ss o +bell y +wrest lers +kill in +lou der +buck ley +ge el +ad on +un s +appe aling +ðŁij ¯ +semit ism +list ens +fit z +ãĥ³ ãĥ +ny lon +ar ty +seem ingly +hal a +su ited +et y +she ds +mu ffins +ap ric +um ents +u ta +jam mu +chelse afc +star z +yo ko +roo t +clean sing +di ar +pione ering +ihear tradio +dig iti +fin dyour +can o +ðŁĴ İ +z ol +spac ecraft +six ers +moi sturi +b ile +ti sts +hor ton +rang ing +colum bi +mete oro +senti ment +ep l +foo th +text book +drain age +r ly +sc ue +imran khan +ðŁĴ ¸ +margar ita +ed dy +predic ts +gamer gate +advis e +growth hacking +love you +ug and +v f +beng hazi +s later +ne wor +ch el +independence day +p np +cul len +hoo dies +num bered +brit t +t sa +kl tu +s ages +mom o +onep lus +col l +gu ts +w ta +mesm eri +enh ancing +chiro prac +j is +teen agers +m one +constell ation +sweep stakes +e ze +slovak ia +la ye +pear ce +wa ver +po gba +k ron +sur geons +mar x +ti d +gg a +desc end +p ours +upri sing +wal la +sab bath +bachel ore +mack in +k am +peter borough +hor a +ðŁĮŁ ðŁĮŁ +think big +r j +hy drau +sp al +univers it +ðŁı ī +mail online +league of +ten ants +w ally +lan ce +heav ens +dd r +bol ts +am ir +i phone +ci gar +en du +re i +el abor +r inging +john son +characteri stics +sal oon +algori thms +tal kin +m tn +di ve +region als +ff ice +hat i +deviant art +so tto +shir o +l ama +k we +f aded +por ting +tu mmy +est ates +buen os +ðŁ¦ ģ +beli ever +pen etr +dar n +sp ite +can opy +fashi oni +t illa +pet als +eli jah +bra wl +marty r +ë°©íĥĦ ìĨĮëħĦëĭ +mid town +eric h +d apper +sm town +me gam +ww w +le le +on s +cat fish +fir th +fossil friday +ball park +th aw +pot ent +illi e +cre ep +car p +so ap +gun dam +infe c +yy yyy +ठ¨ +z ag +rit t +calcu lator +bo ca +ok o +to ad +threat en +refin ed +olym pic +accompli shment +bacter ial +a ji +tat um +feli z +she ed +j at +th ic +jam al +ðĿ ĺ +lin a +ðŁIJ ¯ +jo king +yot po +pin ch +ak ron +her b +motiv ation +li a +ho stage +cre ek +gam ble +russ ell +patt i +fo tos +c pc +bro ken +back the +cla ys +u mm +stock ton +mat ernal +ü r +la kel +cent ury +be k +infe cted +ภ¡ +smack down +man ned +ta hoe +sm es +bas a +su la +augu sta +. * +rohing ya +gre ed +counsel or +silhou ette +gra vit +cla use +' - +bo bc +occa sions +now adays +dic tat +be ard +n ally +brigh test +kab ul +inc india +dhan ush +archae ological +che ape +mizz ou +d hi +ov ski +bax ter +asse mble +à ¢ +gi gi +ac am +wis ely +haz ard +north ampton +âľĪ ï¸ı +me th +bla sting +re unite +mu lus +ali zes +t read +mil a +ed ward +ko va +pe sto +ðŁij ¶ +vit z +hydrau lic +refurbi shed +mo tel +isab ella +hom me +sever ance +uph ol +mis erable +f ari +lat ter +ef er +crack ers +es l +ac io +yy j +in an +ec b +z ind +pan as +tru cking +re ed +sh aker +burge ss +em pire +ag nes +n ington +art works +fr s +ti le +bi ome +eu n +ch ong +americ ana +god father +go blin +i shi +! ). +temp ted +gen omics +mand ate +ck y +ðŁĴĻ ðŁĴĽ +som ali +br andy +in ven +spoke sperson +pc b +yu an +h g +fa z +starwar s +ro wan +blue grass +don g +d day +trin idad +er ton +ban ning +re tention +cu red +tober fest +re set +we is +deta ched +behindthe scenes +immun ity +ph a +bra y +ðŁij ½ +ran cho +ram say +est onia +nd tv +] . +cab aret +tar o +d v +show cases +plu m +ðŁij ¸ +son oma +pre pa +memor ab +e stu +drive way +u les +magn us +x r +nn n +much as +en ge +stre amed +fore stry +audio book +tro y +reck less +kil om +ru ler +ra k +proce ssion +i ons +po ole +noc tur +wh s +farm house +per a +par me +hypocri sy +s ics +v ant +cas k +holi stic +au st +Ð ¿ +in do +ðŁij© âĢį +di so +disp atch +ol sen +make it +en nis +cent re +ar range +ðŁĮ ¼ +sal ted +ea siest +f ate +reg atta +mo zz +ac an +sin i +g ically +ch ops +chick en +work in +ha gg +invol ve +wee ds +book day +wake up +ky r +michel in +fu ss +re juven +vac ancies +incar cer +m st +sc ents +sovere ign +kick er +à § +bo d +âĢĶ > +sa h +mob il +shrop shire +oph one +dress er +mis suni +hep burn +i mo +foli age +diagno stic +as san +cycl ing +guil t +c sa +puertor ico +win elover +wake field +do ggy +k he +pa pp +co g +al lot +cu ck +poe tic +mi o +re vit +mag ician +ç ¥ +ant enna +west wood +mber g +lux e +oat meal +Ø ¬ +te at +ffe e +sear ches +l ly +plu to +el on +let tering +inno cence +fa i +ann on +telang ana +ma it +neu ral +can ni +ar oma +a stor +fe x +co cac +mon etary +f ent +un sure +' @ +indi rec +teh ran +isol ation +li bs +make up +merce des +ff y +he tero +de o +sco m +cur sed +veteran sday +franken stein +shre ws +de co +ge ese +lefto ver +ha did +vari able +acade mics +carol in +under going +vari ation +na h +ssi er +gamer sunite +pur suing +emer ged +ll ers +control ling +ro aring +mete or +vol t +daw gs +be aver +is life +bathro oms +aci onal +pre vent +lake district +in als +y ani +gra bbing +sac ks +le z +sw ay +k ool +time s +klo pp +la de +con cord +resul ted +revi ve +recon ciliation +ol and +az z +gir o +mand arin +de en +nutriti onal +is coming +van i +aw www +der ived +love your +stop the +shou ting +nov ak +ðŁĻĮ ðŁı¾ +lo af +displa ying +sunday with +ma guire +ch eri +ðŁı Ł +re match +qu ic +Ú © +y in +ðŁĺ ¹ +ili ve +z ip +our ke +down loads +sw at +missi ss +care rs +t ment +proper ty +hahahaha haha +gi bbs +sur rey +ar ise +tic ism +sti a +ir ling +fro g +co se +bas sist +fore ig +lea u +pil lows +hol la +eli e +disclo sure +peanu ts +inte ch +ww c +plun ge +trium ph +cor i +sli ppers +ðŁĻı ðŁĻı +neutr ality +ma re +hair y +gang ster +hu mming +cust ard +mer lin +ale a +s by +dam p +mo han +ver bal +j st +gu tted +b jor +un finished +ðŁĩ¯ðŁĩ µ +un happy +âļ« ï¸ı +by pass +at su +fis cher +sa v +afric ans +re use +mid way +demo lished +ger rard +her cules +Ä Ł +medic ines +cl icking +sur round +jo ong +wav ing +tri bes +wet lands +offici el +argu ing +l le +do va +su zy +club house +ne gro +ob tain +ga o +gl ance +assi st +ch os +ãĤ ¢ +âĺ ķ +adri d +occur s +st ans +par don +livel i +emplo yed +re visit +ff xiv +bb le +ne aring +min er +ðŁĺ ¹ +giov anni +up to +mar vell +mar se +to wels +cb n +engine ered +y elling +spart an +si ans +ðŁĻĮ ðŁı¼ +se v +coyo te +sta di +t cm +app en +shenan igans +open access +so aked +ma squ +le vine +stro kes +l k +aparthe id +hipho p +char don +may may +ha asan +stri pped +fr o +scri ption +f ton +h f +pri sons +marsh al +ķ ãĤ +an cho +com promise +classi fication +buzz feed +bblo ggers +deser ving +) / +s way +ob o +camp ers +poder nfamily +p oured +bri e +squir rels +se ize +: # +le k +ti mb +st acy +nas daq +repe atedly +br at +mi ghty +competit or +mah one +de si +o ke +bm w +shi e +f cb +cheape st +minim alist +par amount +n ate +har as +insan ity +lat eral +ment ality +mo zam +ta pped +yad av +u sp +b way +the od +bil t +ra ids +em press +adap ted +pat ron +nut shell +ag ra +be aded +sundaywith marsha +vi king +proce ed +main tained +thinkbig sundaywithmarsha +sn es +mus ica +to wer +ch ab +bo k +sm t +insul t +harve sting +windo w +ru ther +be ige +dec al +indic ate +ma iling +ri ft +po le +ander son +ch oral +sp ride +l ili +ev elyn +imrankhan pti +.... " +ke red +un dp +water falls +se ars +le mans +world series +ri el +ani e +app ar +score rs +lam p +a than +phys icians +qu inoa +refu sing +vu itton +unle ash +s la +pat i +shou ts +inten tions +fo amed +europe an +neighbor hoods +me er +man son +du h +br at +con es +bow l +kazakh stan +ठ¿ +in appropriate +del hi +ketch up +ful ton +s ys +consul t +gar field +to go +f ml +f led +b ds +facilit ate +ree bok +selfi e +elev ate +activ ate +bi ble +ca wx +b ys +cam ille +sy ou +sk ool +her t +w bc +ple dges +recor der +po sh +ac re +so aking +mat il +v sco +shoot ings +pla r +e con +ðŁĻĮ ðŁı» +rashi d +u bi +ðŁ¤ ¤ +sw inging +wi pe +rap tor +m su +music video +dur ham +at tic +apar ty +fe tus +activ ation +aa z +motiv ate +ðŁĴķ ðŁĴķðŁĴķ +j al +ठ® +ag on +sche er +stal ker +fo ster +az zo +tele gram +vi gor +s laugh +screen shots +entrepre neu +kri stin +inten tion +ch illi +fr action +don a +ge a +tc u +s ite +la k +em il +d nt +bor o +wil kinson +re cu +ato day +t anya +bl anco +cd n +brilli antly +g cc +ac c +evacu ated +ther ine +den ny +cait lin +she pard +pou ch +hand held +sou theastern +ha a +à ´ +re solutions +led ger +sr in +r ar +shat tered +chim ney +im with +mete or +hand led +ra ke +town send +en han +shi py +duc t +tw x +inflam matory +war hammer +theat rical +gro s +sk ar +sco tty +ni el +tit o +tin i +conne ction +_ . +goldeng lobes +sha q +ðŁı ³ï¸ı +hall way +fron ts +effec tiveness +gla ston +d hs +ex pi +to h +c pl +sc s +re o +ha g +resemb lance +hor an +abu sive +qu er +virtu e +cho lester +a q +shan e +m ce +carri ers +di stress +re wind + ¡ +voo doo +int act +ann o +ðŁĺ ¤ +pi led +adi a +ãĥ ³ +en ow +di gs +light ly +goo fy +turb ine +governor s +con te +re open +pa h +i ve +cra fting +swee ps +jo di +an de +zu cker +kaw aii +o ko +v ai +out line +kri sti +ts n +insp o +qu int +fil thy +lyn ne +listen ers +depar ting +or d +t weed +, & +ale k +sel fish +nor ther +recogni zes +i ps +be s +a ed +w ills +pe at +surround ings +mon uments +ais le +be cker +la v +quant ity +v ah +helicop ters +tu cked +alv arez +sha pe +o bey +ad diti +road side +m ite +bl ers +ep age +j au +ignor ant +b ins +lu lu +x o +c fo +ee eee +apprentice ship +shef fiel +to i +ho k +faken ews +deplo y +aid an +husk ers +ãĢ İ +west brook +mi ster +confi gur +car r +fic a +proceed ings +ha w +ste ak +mur derer +pay day +a jo +p vc +don ates +bi af +nom nom +be it +k ali +x rp +ahmed abad +se mic +che y +x tra +an twer +head lining +squ ares +roun ded +flu ore +bol d +disa sters +am oo +gener ic +cran es +brief ly +gi g +auster ity +anticip ation +for ti +treas urer +cann y +ce cil +dete cted +check list +ภ§ +pam ela +bar bados +an field +hear ty +tx lege +peren ni +arro g +ing ram +âĹ ı +ty ne +spo on +r ation +am ba +m be +cam el +h hs +york shire +reflec tive +fre aks +to k +ju do +partic les +du bs +ban jo +accred itation +prover bs +over dose +inte gral +gu ang +mc s +super car +af b +al vin +ail s +x tre +st aging +tw ent +rabb its +mar o +inste m +dol l +cr ay +sant ana +ble ach +mini ons +che ap +man t +di vers +catal onia +lo is +mat ri +cou gar +kay ak +e gre +p so +a ia +å ® +char lton +tr acked +sc ari +pe tt +f wd +x in +gra vel +br ic +bigg boss +ar den +hu gging +pal ms +st v +li mb +the movie +handic ap +ri me +z ai +stu b +indi a +lithu ania +rhy th +p ita +maced onia +high ered +brid get +schwar z +ske let +hi kes +ant arctic +c ps +mash up +Ð ° +n ell +chand ra +he ir +an us +sher idan +mi mi +muse u +bec ca +an ir +bar rie +dioce se +compar able +ðŁı³ï¸ı âĢį +yuk on +me p +hor mon +mer ic +al f +con quered +christ church +ðŁĴĻ ðŁĴĻ +hazard ous +poo h +cont ing +retro spective +par ame +na ir +con sor +ho tra +astoni shing +cater pillar +u man +ti sm +t vs +serv ic +croy don +mor ales +c g +cu m +te ur +scan ada +s all +magno lia +el ise +th our +à® ¿ +ag omez +phel ps +ë°©íĥĦìĨĮëħĦëĭ ¨ +wh os +weav ing +si sd +pro poses +cro ws +pre sale +econom ies +bernar do +sha hid +air show +mc cann +hor ticul +nr l +du el +mongo lia +tou lou +requi rement +struc tured +ed i +o lives +he a +cu ter +Ð º +enthusi ast +harri et +domin ion +sub mer +ðŁį ĥ +sa ab +nes burg +mo ff +def ended +bur t +rewar ded +gold man +op tics +khali d +house holds +buc kets +ce cil +che ss +substan tial +ef l +oper ation +evalu ate +st n +rece ssion +l ll +tom as +tru ths +ak bar +s words +p act +embarra ss +ha o +ay urve +scrip ture +ny cc +op t +di ameter +sc ented +organi zers +re lat +ha e +dream ers +de se +ðŁĮ » +restric ted +n ale +r hp +dol an +mun ster +ha ired +consult ants +jo ints +hu mil +d ill +relent less +t é +af il +ut ilities +japan ese +condem n +pet ite +colli de +q f +peach es +cou rier +l ore +âĺİ ï¸ı +reli ability +ch uk +ðŁĻ ĥ +stu res +ge ther +ho stel +bi er +- _- +â ĩ +e ze +ta ilo +di ent +blu ff +chu ffed +pil ip +mon arch +e em +bu chan +b ick +op au +ku ps +ภ¢ +pist ons +sp ins +m and +ce st +bur ne +v ile +cher ries +bec kett +need les +pan ch +ë Ĥ +haha h +trou bles +insi sts +do you +g mc +mor tar +deleg ate +in n +g anda +sin atra +ठ¤ +spee ding +pu pil +pre mises +ali gnment +pi kach +as us +j alan +Ø µ +lime stone +fol kl +parme san +ce il +mo y +shawn mendes +ac up +hu st +ot es +med ina +ma di +gta v +censor ship +ar g +swe eney +sy kes +col o +foot steps +cann ed +adv ance +gta online +healthy living +ðŁį ¾ +a ig +p ality +oc s +he brew +im minent +berk shire +jeremi ah +out going +bak er +entr ata +ma ids +gro ves +bo c +a del +m fw +con science +arm ys +nut ella +conte stalert +novel ist +la h +ban ker +marque z +ðŁı ¡ +to ff +out age +gr p +ðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃ +musc le +du dley +nvi dia +mi di +m uni +ess ays +dat ac +car ter +ภ£ +t ans +i ves +public ations +al er +ok wx +il u +cu tt +har p +out law +luther an +br ill +bo lic +do well +green land +be sties +path i +pay ton +gue st +har den +ðŁ¤ © +ann ed +evacu ation +po ised +mc der +b han +o i +envel ope +ci d +ca vi +ta pas +book review +grey hound +âĻ ª +fe ud +lun gs +for te +rai der +ff er +oni x +dep end +yn wa +rel ating +de vs +ðŁĴ IJ +acqui res +d ha +j yo +priv ati +can ine +k b +cra b +sar din +imag ining +k j +em por +down hill +ne z +ta eyeon +nick imin +gb p +à µ +w ap +sec co +ma shed +ðŁĴ¥ ðŁĴ¥ +augu stine +diss ol +dic tator +â ĵ +vi per +ed fringe +vau x +hard work +book let +no x +chi ff +ðŁĴ ¨ +observ ations +xbox one +u sher +ke er +lu p +dal las +cal gary +ma dra +di ous +k bs +wood ward +hero ine +lu mber +sea world +o ws +mc ke +maver ick +gu la +cross roads +fan g +s ade +nik ol +chee tah +me c +pp g +er ick +ðŁİ µ +tox ic +bj j +viol a +sp ire +ch ino +tra vis +institu tional +ha as +low ry +w ac +ea e +hu mid +mp ton +ru ck +je w +c ine +zim mer +se f +bhar at +fre es +aam ir +ðŁĴ ħ +z inc +wan e +multi player +royal wedding +e el +preci pit +qu ery +kimber ly +isa bel +ful fill +ig an +vau l +pan e +sc y +dig it +gun n +u tah +dog day +fi on +xia omi +da c +el ast +cha vez +ro blo +g ine +ten th +ab h +ke to +hur dle +na dia +memorab ilia +ha bs +qu an +h w +hv ac +pix ar +ec cle +kram er +accu ses +ðŁĴļ ðŁĴļ +per se +mean time +wa hl +atle tico +âĢ¢âĢ¢ âĢ¢âĢ¢ +ott oman +no vo +k us +conne cted +tru sts +d mv +spen cer +rahu lg +do ve +sto kes +bolog na +enthusi asts +à ª +rockstar games +ted cruz +du ras +s acked +late x +immer sive +cer t +lu cin +princi pals +fa res +sa ils +far n +am ent +saf fron +quent in +check point +fer ris +ex cur +ðŁijī ðŁı¼ +bai ley +se h +ter re +mad am +s band +wan derers +cumber batch +yy c +digit ally +blackandwhite photography +roll in +moroc can +ðŁĮ ħ +din ner +d well +to om +m ye +ez ra +cp fc +war hol +me er +jon ah +no aa +s gate +so on +secu lar +g ating +ti o +dri ver +si ssy +assan ge +ta th +ed mund +bobc ats +ra ji +po stage +stu ds +m gm +kat o +edin burgh +meet the +shir t +fa a +mens fashion +sp reads +wi m +car ts +phoe be +j ars +bot swana +Ù Ĥ +ed war +sk ar +ri ve +gu sty +c tv +ferdin and +su therland +nickimin aj +k v +si us +bee ch +re z +desi res +on ial +camp o +quar ry +lor raine +gil more +ig gy +µ ï¸ı +ho pping +avi z +ðŁĮ º +uni sex +dedic ate +att itudes +ste er +jun kie +rail way +y b +whi sper +key an +k us +ju g +di x +a ins +sum mon +ov ich +sy ed +her ald +ma ison +me ded +wild flower +main land +ri sky +ru kh +over looked +ki c +destro ys +nam an +ki p +z ano +champion sleague +ban dit +quin cy +smi le +cal vin +open ings +ta pp +ol ulu +spec tro +accred ited +ap k +pra ised +bar nett +pol len +premi ered +selen agomez +tou red +screen ings +uu u +mis o +en se +adam lambert +guel ph +har yana +hu tto +le ar +l tc +po ached +brex it +æ Ŀ +tt c +pa vement +mon gers +ro e +ad ers +ling ton +particip ant +ca red +ga il +y ates +lan tic +dash board +jo o +feli pe +ssi onist +bu m +s end +a eri +thu gs +luci fer +a he +dete ctor +fil ly +gas oline +ham per +hump day +the ta +the band +fore casts +o hhh +lo bb +hol l +cp u +az u +ad ar +hai ley +bu b +car t +quo ted +an archy +pan cre +twit art +al den +st ash +the less +or ni +belie bers +mor mon +partic le +avi ation +⬠Ĩ +webcam toy +sad dened +cru is +ham let +n ct +roll ins +marque e +saw yer +reli ance +a ura +di ec +soo thing +sig nings +ak is +à ³ +at kins +aer op +ðŁĮ ¿ +y ab +sh ari +con nol +du bbed +manufac ture +convin cing +feelthe bern +ra u +pu lit +on ec +gem stone +ur ging +bag u +ga h +aci ds +fi anc +zodi ac +sn oop +her rera +initi ated +ven ge +profess ors +pro di +stron ger +e mission +bb a +hal le +ta pp +haw an +wh im +compe ted +myr tle +ir port +cold play +ach e +ske p +m son +ss ic +calli graphy +swim mers +me y +pp c +thri ft +po c +re places +commu ter +âģ¦ âģ¦@ +go ers +lo gue +para dig +bas kets +sensiti vity +joh an +atl antis +& & +suit case +anxi ous +l h +str i +gal loway +stre ad +war den +gr ounded +ffici ency +li feat +reli c +disgu ise +island ers +f cofficial +classical music +b mc +en field +bi que +oak ley +bat man +sla ying +ner ves +mul tit +calci um +projec tor +scott sdale +ant ino +gri ps +kim mel +des mond +prote stors +hi atus +metaboli sm +conclu ded +press er +ti pping +sli de +e to +hun ting +aus open +ri k +pp ery +innov ators +pitch ers +ag ger +fun gi +z ad +proli fic +rockn roll +bl ames +ct ar +stam ford +q ad +mozz arella +insan ely +den ver +ph ouse +nom ad +ï ¿ +s ris +pro du +hen ley +pag an +am trak +ru bi +in cl +tu tor +sco tia +wo es +sing apo +fun nel +turn bull +know ledge +gri mm +real madrid +we are +missi les +con sol +emo jis +sne ak +smi ths +ru iz +br ou +i el +ha ver +ðŁĮ ļ +kin gof +basil ica +circul ation +prin ters +ta pping +ri dley +dra gged +ha j +writ er +fundament als +personal ities +me tre +stereo types +bur le +best of +n ffc +ha th +mini stries +a ali +trac ing +pav ed +ł ï¸ı +g ic +insp ire +tu g +ha re +repe ated +ex pon +lol li +rho de +pre cin +install ations +instag ram +az ar +i es +sole ly +du kes +mission ary +van guard +fursuit friday +on d +pol ari +ma st +har an +jos é +jack ed +ec oun +al ities +ne ph +ra vel +moder ated +sco w +s fb +uru guay +as o +ni g +au du +p ints +lat ina +ben z +m itting +char ted +mat ology +cit ro +biop ic +ðŁij Ń +djo kovic +fox y +agu il +so to +an ada +sin king +sc rap +hair s +bethan y +fact friday +ðŁIJ IJ +unlea shed +) ( +contra dic +ram on +coast line +y ong +sn sd +li gan +p ome +mit age +ge tt +wat i +ri sk +so aring +bru sh +f pl +av an +å Ĩ +lar son +sh ear +mul til +blu r +multi media +chun ky +par i +n ani +weir d +cholester ol +char les +dream ed +tan ning +puzz les +fr am +hand ball +ch ag +beli ze +al u +bang s +Ñ Ħ +detec tives +mc g +ish q +bo thered +saf c +mp ing +ten eri +g ays +sail or +an gi +mul ticul +gue ssed +ros é +high ways +bro om +chatt anoo +- ' +see ker +on ed +at f +lu c +> < +bar i +per cep +jewel ry +as ph +sor row +sl ing +mam moth +jac kie +ë § +wilt shire +sa o +can cell +im paired +tor ial +bre ed +guy en +jud ice +tit le +pro spective +applic ants +ðŁį Ĭ +epis cop +e id +b yo +stock ings +ðŁĴĥ ðŁĴĥ +ll p +sna g +keep it +l ough +ol son +matur ity +!! !" +cop ter +i sha +bl i +wil mington +tr youts +th ai +ðŁ¥ ³ +pe bble +kra ft +f p + º +ssi vely +li vin +contest ants +tex tures +jo an +h dr +film festival +prov ence +wi do +op end +c si +sto wn +cro ati +ad just +host ile +analy sts +il an +cu ppa +bru m +newfound land +good win +me tt +mall orca +plu gs +bu k +bb hutto +wrest le +sa ire +sho pped +for za +le head +vi vo +ba st +ro xy +reg is +hard working +hon olulu +desp air +young sters +ni g +impro mp +roll tide +de emed +tre ason +ru shed +for ged +ff f +pikach u +bri ggs +do it +ac cent +la us +gla ze +compet ent +a ho +photo g +mid field +le go +har vard +min orities +re illy +slic ed +once upon +initi ally +financi ally +landscape photography +har dro +qu o +mm ers +par kinson +smu gg +read iness +bru tally +glou cester +mp ed +bbhutto zardari +mur der +ye d +dat aviz +sr t +dow ning +bi ans +m ü +fle ck +fli pped +s ly +brilli ance +ri m +k um +bubb a +ko i +knit ted +sor g +ma is +ðŁĮ ² +ti ss +su stain +sen su +ak han +zi est +exam ines +chardon nay +user name +short list +re bs +on o +dar ing +hard wood +che que +righte ous +light ening +dir k +shra dd +du ra +down stairs +sh al +ami gos +ru ff +s law +ri es +red nation +man us +ðŁĩ§ ðŁĩ· +distin ction +u bun +dur an +mi gra +thi ans +la ver +domest ic +k x +jaz zy +justi fy +belong ing +insul ation +color stv +drun ken +chann eling +qu and +xi ii +enligh ten +kan o +fati ma +teen choice +terri fied +p ba +as ley +met museum +dun e +pack er +ki o +ðŁĴľ ðŁĴľ +bo iler +fas cism +ar mored +back grounds +in mates +embarra ssed +defin es +th d +we go +silic one +lo on +el ding +bor rowed +he mp +ak sh +kaw asaki +br y +de af +kill er +dispo sal +ðŁĩ ° +glaston bury +un covered +o xide +po ff +d ant +k j +ku ro +dri zzle +peop les +fe e +pro pri +dd lovato +pi ggy +ot is +aller gies +u bis +pengu in +ser a +vi z +prosp erous +ici des +tornad oes +sene gal +web cast +sto red +enchan ted +bb cone +bay area +entrepreneu rial +rednation rising +experim enting +ang an +lot to +they re +por e +er p +seren e +east wood +bro kers +bar ge +stal lion +timber lake +tailo red +dy stop +b ate +lat ors +di xit +bran son +dynam o +ky lie +shame ful +bt wn +spring time +mix ture +s ounded +lu ton +dad es +mal a +op ra +en ic +rahulg andhi +se wer +~~ ~~ +ky u +nor theastern +ca er +bc u +nir vana +kitch ens +ous y +al m +river dale +hid den +fl int +sp d +pat rons +katy perry +au gh +exhib itions +sm c +shu ts +at ore +da in +some thing +ber th +bo g +por ter +gen to +con cussion +ang lic +ro we +gr illing +scar lett +master ing +mor nin +comm ented +si me +si zing +christ y +ce os +st m +at ry +tari ffs +vac ation +pre judice +p su +paren tal +far age +can a +cap com +koso vo +you re +men stru +stal in +grape fruit +br an +che sa +dav en +exc el +!! ) +๠Į +distribu tor +ce a +bride sma +millenni al +wa in +ob serving +mis ery +plan etary +expo sing +bra ised +comp ton +don gha +q l +spring steen +th ul +syl ve +cab o +pal ad +niel sen +gaz ing +ba ja +r oud +orchi ds +johan nesburg +se man +d ji +oper ative +affe ction +eclec tic +at c +mut ant +aw x +nic e +mel bourne +indu lg +tu lip +dias pora +wel p +big gie +mississ auga +retri ever +or an +tam my +c ta +hipp o +seas oned +ger mans +eng v +marvell ous +im f +rela ys +mon tan +maur iti +me ister +as surance +reig ning +su fficient +han e +no thing +pos se +nav y +in love +brigh ton +en qu +ch ung +sweat y +es c +cal ed +man s +nicar agua +sl ices +mo cha +washington post +bb n +dam ned +grow ing +en burg +lo an +me s +wh oops +believ ers +spi el +vo daf +l at +s led +cricke ter +brown e +golf ers +bar ra +wat chers +lu igi +sw amy +mom s +pit ched +san tor +cr s +si re +sc amp +bo de +ste war +jon ny +ent ity +pac qui +mind ful +min india +bear ded +temp t +scorpi on +eat on +authori zed +ar to +s vp +op athy +cch ini +house music +disney world +âĢĶ @ +pro pose +di y +expen se +ten g +pupp ets +sm el +d aca +per ry +fin n +boo sting +lefto vers +cou gs +satell ites +man y +az e +g ong +fi e +metho do +fer ries +ðŁ¤Ķ ðŁ¤Ķ +explore rs +load er +attrac ted +il ton +godd amn +pi azza +doc tr +sav ing +paragra ph +visu alization +may ors +work flow +ack les +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ +ठ¸ +twer k +clu t +lo ver +te ases +si an +o te +deter ior +accor d +l fw +swar ovski +nat al +tra ps +k ina +analy ze +laye red +bever ages +un it +ran som +pe shaw +dest ined +astro logy +si pping +miley cyrus +cam ino +marshmal low +bli ss +out back +fa q +int oler +humil ity +po ppin +hallo ween +mon tene +op hy +nu n +tattoo ed +a as +ðŁĮ ³ +dale y +qual ity +du sa +fisher men +swi f +ter rac +st au +le in +trol ling +ship ment +garden er +march madness +head band +gr t +bur nett +w and +!!!! !!!!! +gh e +du x +hu d +war ner +ðŁĩ ¦ +ex ile +rescu e +rat a +d han +duc ati +dro wn +bl ends +spi e +alli gator +simul taneously +broo ke +u ke +k har +comm union +ri ka +ford fc +chin atown +you rown +me y +can al +syste matic +de pri +ox ford +an il +w ut +equ ation +be z +fle ur +the good +lang ley +ad ity +ed ith +al fie +о ÑĤ +en cry +br ill +ex emp +ce sar +mb ling +ab ri +sc icom +j ing +school ing +mi ka +mechan isms +impromp tu +rhe a +moo re +crime a +be sto +wri ght +el ders +ro ds +kam al +folkl ore +be et +mini on +reli eve +thr o +team usa +pas cal +made with +boli via +itt i +free bies +desi red +best selling +l iness +la den +ke ane +mi sts +hipp ie +atta chment +@ / +se w +flan agan +âĿĹ ï¸ı +supre mac +stl cards +si as +q u +rh ys +ste ep +val leys +v w +pav ing +disp at +al ison +por te +id u +new sc +soc ket +mo s +co star +re vo +prote ins +stanley cup +m cal +ear ring +se cs +mc lean +cap ric +nick elo +ad en +v c +shou se +adap tive +maxi mize +entertain er +pro se +gri ffi +six teen +lam ar +mi rage +saudi arabia +awe ather +ru st +in filtr +fashion week +ðŁĺĬðŁĺĬ ðŁĺĬ +selec tive +bubb le +a den +fen nel +deci sive +m ta +mock ing +mb les +st amp +mu le +bernar do +gr in +po tt +j ingle +vet tel +colom bian +cam o +motivation monday +ba han +p ly +dh ary +k ami +x men +sleep er +gar a +my sti +confi dential +conflic ts +p neu +ce s +insur tech +clean se +me rely +va is +tu x +the great +shar on +ma j +hol a +eco systems +aj ay +aa j +hu sh +har mon +backto school +wiki leaks +reflec ted +ðŁĺ ĵ +commemor ating +ac et +buck ingham +messi ah +tu ous +hor net +to be +d q +he ine +mi g +pl ate +nichol son +sp ie +cumber land +nor mal +pho bia +happy halloween +city fc +mc el +gilli an +ke to +lu de +de mise +su ga +str ate +mcgr ath +visit scotland +foo led +cb r +gc se +col ori +po td +missuni verse +fin ances +ma poli +for ks +Ø ´ +cann on +medic inal +ðŁĹ ĵ +kh o +wre ck +pan to +bag el +gu ll +syndic ate +ic y +pr c +ki en +zi ka +ti sh +pe ta +c co +li za +ch ut +ex traction +el g +gl i +fu eled +pos it +respec tively +leice ster +br ink +vulner ability +im ported +e sha +ðŁ¦ ħ +r ural +re ll +gam ing +atlan tic +aband on +no ah +re solved +pro state +aller gic +ps d +âĺ ¹ +dun geon +fang irl +illumin ated +m hs +white sox +d ently +ck o +endor se +over ly +dazz ling +prior iti +night life +ut il +be have +flam en +east bound +ðŁĴ Ł +ilove you +gov uk +mozam bique +alle gi +dr i +testim onial +ath s +ì§ Ģ +mm y +shab by +pro secco +friend ships +cal am +dam ages +off set +jura ssic +jun o +arre ll +ðŁĴ © +interven tions +dare devil +car ver +run away +ran e +truste es +ha ute +dep ths +ðŁİ Ń +me in +sacrific es +con cier +ne sting +i zzy +me tam +ilove my +ur ine +du lu +mal hotra +ve ins +night ly +co at +an di +he witt +lon el +ci ble +wr ite +jen nie +sant ac +ĸ ï¸ı +str ato +singapo re +sop rano +kri sten +cheer ful +flee twood +fa iri +m eli +wa st +tur nt +sfor sale +sc rolling +angel ina +ren dition +jeric ho +nick y +or b +fla vo +patri ot +ash eville +sick ness +re fund +aggre ssion +b pl +ãĥ ĥ +elu sive +thi story +hang er +bu ffs +vil las +at kinson +sp h +ja it +decl ined +wo k +supre macy +oo tball +ey ang +ðŁİ ĵ +s ford +ath i +consu me +road ster +e so +u pro +reci pe +au f +uc i +ar on +oo oh +cs go +re ich +mc d +min ute +ladi es +pun k +rut gers +mee k +ariz on +ta j +land lord +de gra +autu mn +lyn x +us f +b hi +fairy tale +dongha e +bet sy +explo ded +chen nai +op a +pro tag +br ant +ðŁĵ °: +g f +pal li +ðŁı¼ âĢįâĻĢï¸ı +su t +ill ini +colum nist +shir tless +de centr +sear ched +ec or +bu ggy +s ack +ðŁĺĤ ðŁĺŃ +de t +ther i +or naments +bring back +to v +quarter finals +ic he +con stra +gi er +buchan an +vi x +kay aking +mu stread +swal low +mel b +sc af +op al +may oral +har at +ðŁ¦ ĭ +schedu les +id f +ha gue +ro z +a ah +d mc +du plic +ca che +orph an +frac ture +rec on +ch av +bun nies +al ain +mustaf a +ðŁİ Ļ +vac ations +dynam ite +tex ted +broad caster +ðŁĴ £ +ste amed +rock er +di etary +luxury travel +inaugur ated +sa wards +vaugh n +lincoln shire +click ed +kra ja +f anc +remo ves +layo ffs +mc far +bre eds +win nie +jon ghyun +incen tive +vari ations +pat ton +atur day +persist ent +pr un +pi ers +dal es +æ ĸ +breast feeding +r ance +ta wa +Ĥ âĸ +mur doch +cap tive +thi stle +nic a +commod ity +cou ldnt +board walk +graci ous +practiti oners +n gc +scru m +ner o +camoufla ge +col on +he i +phys icist +saturday morning +ten er +si won +colum ns +bru ne +y vr +ba ir +reti res +hal am +cab er +shaz am +min u +cas cade +milk shake +gri d +d ren +vin cent +so dium +plat ter +cheer leader +chen ko +y ak +elimin ated +ty po +y man +re think +âĿ Ĺ +ts ville +bernardo kath +ex tr +ðŁĺģ ðŁĺģðŁĺģ +ta o +re per +mo ths +em powered +c iting +transpor ted +mon ks +san at +cle ars +bachelore tte +camp bell +racha el +har le +hand ler +climb s +inter ference +rele ase +sh and +r bs +hr h +ãģ ª +val le +r é +sli me +w akes +chu bby +slo an +el ves +ath en +attor neys +micro scope +ston er +sc aling +o be +c out +se man +mid week +bal sam +ðŁĺį âĿ¤ +ti ful +v ish +lo tta +ri pping +re mn +ti re +le ap +ha vent +la by +hi mach +whisp ers +we in +ðŁİ ¸ +wild flowers +se le +u cc +li ability +az ine +sw ings +k ya +ta ir +re main +e do +flo ps +poc ket +grand ad +exam iner +gr is +ffe ct +ðŁijĬ ðŁı» +stud ded +heart beat +de acon +firm ly +infec tious +ste f +out lines +le asing +cla ws +sen se +tab s +hoo t +mo sul +spa wn +co a +hog warts +ve in +alban ia +manu el +b ino +vaux hall +scot land +go bucks +mat ty +phy sio +tor ino +const able +investig ated +s lower +mistak en +bay er +wild fires +vo ic +x on +time to +chas sis +bar ric +pi on +bald head +woo k +regi str +dra fts +b hs +li gue +l ick +staf fordshire +baf ta +dar ry +je anne +ven ding +cor p +⼠³ï¸ı +kid dos +fen way +ca o +west bound +ðŁĺ Ļ +dv r +quick er +bla h +goo die +ðŁĴĭ ðŁĴĭ +vo x +esp er +fac ade +cor relation +red bull +rou p +decl ining +chi ve +mc gee +tur o +in der +f eller +fu g +il ysm +mar di +peshaw ar +ki eran +ine ma +meat balls +pe ck +depre ssing +sen sing +gi z +dd ington +spring watch +ro aming +yellow stone +horse shoe +am man +week day +ol or +ðŁ¥ ° +boo sts +spr int +scar ves +je e +bee tro +cl an +all the +ìĦ ¸ë +enlighten ment +ado be +re generation +? @ +cont ag +yach ts +to u +mor a +en voy +r ani +go li +dhanush kraja +wood working +streng ths +se di +disc s +ar ina +sc on +lit e +ano ther +ðŁ¥ Ĭ +ye men +gu ern +sav vy +lo yed +biom ed +heart break +comra des +milli e +pat ch +un f +jar vis +bl aming +commemor ation +ge y +å ¥ +cardio vascular +alig ned +docu ment +. ? +aesthe tics +em u +the irs +le h +ps ic +si f +pl ateau +ex pend +domin ating +rob es +mauriti us +excep tionally +hom er +discover ies +bra un +ten nant +insul in +ðŁİ ® +car bs +te as +? !" +zi e +franco is +brow sing +th ol +cla rence +hel per +ob tained +cas sie +le es +! , +pome gran +hu bs +presti ge +] [ +mach er +bott led +pun ch +pi pe +o ch +gall ons +deliver ies +u ra +un day +mon de +depic ts +re gency +outra geous +khal ed +car o +he arti +za g +develop mental +over coming +stati stical +flavo red +for ds +cre atives +lau rence +di as +sun screen +in ked +pre acher +n ul +impac ting +auti stic +âļ Ķï¸ı +o ss +pel icans +cele ste +v b +ru mp +mc gra +fair fax +hu mor +bbc news +row ling +cal der +seam less +ag ne +p ti +mix ed +t shirts +mer ci +b tob +women instem +genealo gy +pre ven +l our +cra dle +gi use +Ð ¾ +chron o +fair ness +chocol ate +tor y +as da +pre scott +stret ched +al man +u il +re charge +in tre +ob st +hosp ital +hay ward +teneri fe +fried man +vap ing +confe ssions +ye ah +bal li +luck now +cor pse +sculp tor +amp ton +t pp +indic ates +sur plus +tru man +ðĿ Ļ +sin ha +in vo +sovere ign +ke v +establi shing +engra ved +assu ming +ðŁı ģ +sou za +fab i +ton ed +oun ge +del oit +dow ney +no ble +om or +car tridge +ðŁı IJ +u hur +hol loway +succe sses +r sa +âĦ ¢ +ma zz +tw d +disc ourse +. < +y at +satis fy +com pri +ठ¹ +graph ite +disser tation +ar ter +í Ķ +b ally +zom bi +ly ons +a ic +u bc +pra da +e il +da x +cla i +grand daughter +extravag anza +chall enge +ðŁ¤ ŀ +po ver +primar ily +dad dy +man a +bi kers +inqui ries +da un +fel ine +gener ative +he f +benef iting +lind sey +pol ka +demonstr ated +al le +rand y +o su +low key +weir dest +red bull +our y +n ous +wood stock +cre denti +nic er +g ado +aly ss +ap h +prepa redness +station ary +incorpor ated +dy er +sarato ga +cele sti +: " +antibio tics +or gs +inde fin +ap ron +и Ð +fif teen +no f +ðŁĶ Ŀ +ph x +te ga +m z +organiz ational +on air +band ung +pleas ures +mor i +secre tari +rac coon +ca shi +pil ates +k on +geof frey +la o +kam p +depart ments +back packing +an am +à « +crack down +aun ty +on do +li zzie +ph ers +cu n +ðŁĩ ± +k pop +pu t +inten tional +connol ly +bar clays +hs fb +swin don +u ku +s ally +a int +âľ ħ +pen ang +up lifting +epile psy +inter ro +bun gal +go ku +blue berries +ठ¦ +u ssia +sil ky +mou red +i stic +bri efs +me ats +go b +ch aser +state wide +pra sad +gl itch +ar in +ban ff +memb er +ðŁĺŃ âĿ¤ï¸ı +lo ving +hall a +ภ¡ +smo kers +yak u +scicom m +physi o +sw ol +lem ons +gel ato +ch ool +capit als +ki stan +ti ghts +spi kes +trav ellers +ik lan +commissi oning +ar ine +emabiggest fans +empha sis +front line +pad dock +destruc tive +ba ha +l inger +je wish +shet land +mc gin +mon key +ko z +s one +raj ini +te h +y en +c vs +masqu er +gir ly +we sle +was nt +bro dy +termin ator +gil le +mag gi +bir die +jeopar dy +cu bic +vm ware +intric ate +an up +to pia +east on +sab res +investig ates +bu sting +bil ingual +valent ino +in format +fer re +advent ur +hydr ate +for sy +az iz +san to +e de +whist ler +continu ously +d ham +un used +ji had +addic tive +vi dy +do b +i do +fi ed +ni versary +n one +fu er +ðŁĺį ðŁĺĺ +coven ant +prin table +immac ulate +o em +cl t +serv ants +consu med +un released +sc um +pack aged +me re +ìĦ¸ë ¸ +to by +ta f +spo ons +me al +f ball +fair field +jan et +silver stone +dart mouth +follow me +voy ager +kom bat +anni ver +ene w +mag dal +ho ve +sa th +grizz ly +car di +gart ner +sand y +kan ye +post ure +po ign +im pulse +radio logy +horiz ons +si am +aish war += => +no che +tr is +el yn +com me +du i +ce c +councill ors +cudd ling +creep ing +loc ke +manag es +trans ferred +ne cks +di er +dan o +v ick +lun ches +d he +en sures +cri ss +ul ster +bann on +cont enders +sp am +sweet ness +med al +hon duras +arc tic +ultra sound +in fr +disco vers +ei ffel +ca sters +ru ben +du st +awe ed +atri um +lest we +se ared +ðŁĵº : +ty ne +ex changes +little mix +l le +astron auts +hersh ey +work day +kno b +so v +re signs +today show +der man +an th +af c +ta ster +sw oo +sa eed +per ing +narrow ly +rn li +best buy +panas onic +obst acle +farmer s +ðŁİ Ļ +pa wan +ki est +ang ers +absur d +oh my +sin o +pist achi +sp ice +giu li +prime time +ko w +k ens +ex agger +! ?! +u ba +midd les +ju dd +e jec +slam med +pen sions +of a +re create +b hp +xx l +liver pool +thre sh +pur ity +ni eu +hol ics +wr ath +ra do +gli o +am ma +dile mma +cr u +lets go +.... @ +âĿ ĵ +sugge sting +tru mps +hor us +f v +ic om +refer ring +predic tive +tar ts +ge tte +so ck +glo ssy +pin ky +al ec +thy me +ou ra +thero ad +pe tr +cr am +p fi +dv n +me ier +incen tives +tun nels +mobi l +rec ap +extra s +upri ght +rev amp +per severance +, - +ot p +mir ror +ar wx +ger ry +ma her +g or +hom epage +am is +ag ra +made le +best friend +sirius xm +bun dles +admir ing +t dsb +ðŁį ģ +ch as +slow ing +ro h +wall papers +âĢ¦ / +tek ken +gang s +tal a +lind say +shou l +line backer +tool kit +ur anium +caly p +ab rams +mat thi +ðŁı ¿ +hon ourable +da yo +ver sail +tan k +st c +fr itz +spl end +pat ag +anno yed +on day +devast ated +chattanoo ga +national ism +mas sey +jen n +tail or +dev gn +org ans +zu cchini +on fox +sat ire +wex ford +dis grace +no to +vol ta +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ı +à ¶ +home owners +poin ter +m cr +au sten +day sto +mo ons +pal ma +gra zing +e so +influen cers +shahid kapoor +compli ant +measure ments +develop s +y d +par l +p vt +rand olph +tor tured +ger ald +eli as +deepi kap +war mup +hick ory +g ap +co ffin +am our +re neg +moun ting +seven s +ig le +hi er +dec ad +tri ght +esc apes +wer ner +t fl +ful filled +ni ger +sour dough +re aper +choo ses +spin ner +week nd +fil tered +sh uk +kat i +old ham +open source +kh anna +at elier +conne c +opho bic +gla s +complic ations +ar son +counc ils +sm ol +as sy +lur king +ling ui +han ks +e in +Ù ħ +ru gs +n guyen +nou veau +men ace +le v +alad din +ru ining +round about +k m +con or +shoo ps +may day +traum atic +prab has +ka iser +k ita +rou ter +pe dro +re tar +stun ner +spani sh +distur bed +acade my +e learning +wit ty +sen g +fer al +av y +sta b +ke aton +ur du +ko to +hu i +coo ke +ari an +the personal +u ma +se ap +a sting +rhetor ic +hand writing +munici pality +consor tium +ðŁIJ Ł +glasgo w +ra ya +eli za +polym er +bro th +prac ti +correspon dent +addic ts +gay le +ail ing +o fe +p li +hear tw +st itch +sight ings +prie sts +sam o +slo th +good wood +roc co +sab c +summ it +l ace +pres ley +itt en +cin cy +thepersonal network +s week +pe gas +af con +regi stry +ci m +le th +dic ap +cand ice +flu ent +sm ack +pede stri +al oud +car ac +priyan kach +p gh +ir ons +dol ce +lat via +dece ased +thero ck +cla p +cen e +fo am +morris sey +gre t +essenti ally +com cast +be agle +argu es +ing ed +- âĢ¦ +sa g +ha san +ðŁĻ Ĩ +ðŁį ° +nh ra +kann ada +indic ators +on er +bri xton +at as +screen play +sor ority +sha heed +he em +class mates +tain ment +es i +breast cancer +zucker berg +aur or +en cia +ref ers +kae per +vor tex +com part +lym ph +photograph ing +ste ff +rest ling +par sley +mom ento +th man +lac king +du tt +ocu lus +fin o +fren zy +ra sc +der n +dis missed +noo k +met gala +sh ill +rapha el +maver icks +exhib its +eag erly +c pa +amen ities +. âłĢ +exo dus +ern st +lit a +deal t +womens march +i ain +score board +campe ones +c en +ti ki +garri son +fidel ity +bra g +road map +psy chop +lo e +ble u +ðŁijĬ ðŁı¼ +sau vi +spr inger +temp tation +ru dolph +ac ura +wic z +parach ute +stro l +len ny +zi k +dom s +nb af +al pac +vivi an +ro ve +pre et +perpe tu +sna ke +air soft +infl atable +prin ces +ati e +ffe y +pati ent +m ire +chel le +sl ack +groo vy +# : +up loading +!!!!!!!! !!!!!!!! +siem ens +provi sion +v fx +need y +f ats +to poli +bhu tto +sa thletics +alu ms +t winning +south western +adop ting +last night +man ne +la ga +tw ell +ac ia +-- -- +eye wear +hur ley +fle e +sa ch +pe cker +cost ly +is k +cr ates +polic y +ero sion +in go +wer k +ðŁIJ į +torto ise +therap ies +inter net +chihuahu a +ri ps +fre i +ed or +tai ji +t fc +do d +demp sey +christ in +chen g +hi ps +gra eme +com passionate +cavali ers +histor ic +soul ful +crimin al +ja c +vin ci +expi red +sur at +turi smo +k ona +se aweed +ber ts +le ica +expre ssing +a al +wor t +break fast +her ring +am used +rhu barb +mar tian +cospla yer +y ash +stri al +ra ul +refer ral +dw ts +j w +ad ler +cur tains +gu r +val ence +tyr one +sw fc +coach ed +re born +diabe tic +cho ke +nor folk +investig ative +ðŁĴ¯ ðŁĴ¯ +z id +v mas +phi e +objec tives +âľ ĭ +over due +di vers +mat su +ðŁİŁ ï¸ı +casu alties +ภ§ +al k +stand ardi +re alist +arti facts +pand or +ke x +in vin +( !) +ine y +par aly +mr t +fay e +the voice +on ga +de ed +skin ner +az wx +speci men +priyankach opra +nu evo +bar kley +toulou se +resu mes +football ers +cit i +fe tch +è re +lestwe forget +ðŁĻ ĭ +ch unk +dri fting +manipul ation +equ als +pu tt +ky ungsoo +âĿ¤ï¸ı # +ela stic +par ano +fo y +do ping +cin cy +ss ler +interrup ted +al ay +ado res +ame thy +con voy +ãĢ ı +Ĭ ãģ +black list +gener als +sa chin +bru shed +oun ces +non stop +illi ams +bt sarmy +u av +ru ff +bur ma +bi k +defen ce +schul tz +bo asts +lonel iness +go re +trans forms +alum na +@ @ +ra ppers +ne hru +car o +himalay an +wearab les +ge h +pepper mint +re development +flam ingo +cos by +big baldhead +ag ri +bare foot +sco pes +re gram +gh ana +ðŁİ « +i heart +sa die +carri e +microbi al +ku ala +sk ater +quer que +âĻ © +gen res +reas oning +ch ased +as o +sli pped +en can +vam os +ker s +ad verse +mo il +commod ities +with you +sil ent +hy pe +an de +am ination +whi spe +lit z +âļ½ï¸ı âļ½ï¸ı +ri ff +pp y +lam bs +gan esh +ab sent +regu lator +marse ille +en roll +par cel +wa p +by rd +ðŁĩ Ń +tu ber +country music +par l +contro llers +responsi bilities +we y +ch ate +montene gro +chic o +mil an +l ms +tra inees +appropri ately +un certain +popp ies +ed sheeran +nutr itious +gar o +deut sch +awe some +ãĥ ¼ +comfor tably +land marks +et i +re usable +daniel le +ro sal +co les +just ic +c cs +f anny +ni m +mc u +clin ch +at ene +mer ge +im db +ang lo +uc cino +pan ini +an not +bur berry +feat ure +predic ting +fashioni sta +s ask +imag inary +mm o +south sudan +spe ar +hu bble +jo inthe +coyo tes +sli go +ko dak +sit com +polaro id +roo ted +corru p +ðŁĻĮ ðŁĻĮ +bris ban +at z +ah l +re my +tal ent +aval on +ra da +pau line +locom otive +go ons +ne mo +maser ati +ic u +stu tt +histor ically +sm b +pres by +avo id +so oners +rhine stone +w ad +ri sing +tro t +mo des +reg ent +optimi ze +re ece +sm u +ver ti +newyork city +cor tez +ra c +in case +sin c +fiel ding +e tta +tiff any +al monds +sad dle +k rat +mat ter +g low +star ving +gl o +cra ppy +sl ur +st d +monit ors +recei pt +maymay entrata +mc il +un is +rain bows +cal dwell +pacqui ao +j op +a fe +hoo k +es sen +wiz ard +medi an +fla ws +com s +âĿ Ħ +ing h +ha ynes +anton io +tem plates +ou ter +na w +cardi gan +bel grade +ðŁĴ ī +hom o +a ise +ro pes +no ve +what you +tri gge +concep tion +ad ukone +na di +fri ars +sw er +adju sted +hot line +san ity +kau r +down loading +c gi +ten or +eth nic +app alach +ภ¸ +pa g +gol ds +on set +investig ator +car tel +peace fully +jarre tt +cat alan +poli o +n um +fru stration +dhar ma +my life +âľĮ ðŁı» +aber deen +mu sa +bin der +spark ly +fle eing +instin ct +co ping +domin ance +ill ers +er a +u conn +lo oms +living ston +gal i +he s +c ma +bel a +se ley +mon k +la ch +mar x + ´ +m erica +woman in +es sex +ra ina +jim i +nep tune +z ack +chine se +mart ins +chand elier +her n +with us +ear l +asph alt +modu les +st p +ul la +psychi atric +mile age +captiv ating +si der +men to +mor t +tran ce +tal bot +ab by +ì ĥ +âľĮ ðŁı¼ +j ak +daw n +turn up +scre wed +fe ds +blue print +ðŁĴĸ ðŁĴĸ +har sh +er os +insom nia +ban kers +ta emin +mis conduct +hu mber +gi di +edu ardo +con a +musc ular +consu ming +ra sh +don nie +di pped +col lie +samu el +melt down +ðŁĺįðŁĺį ðŁĺį +me z +exam ining +schwar tz +pri stine +ðŁIJ Ŀ +ve it +ful filling +an esthe +gue sses +dra ft +som me +soli d +pati onal +ho ped +evolu tionary +all er +enter tained +sli ps +lud wig +conclu des +sen sible +bon net +cra ze +tra s +haz ards +const antine +ed ics +star trek +to c +occu pational +in cheon +deepikap adukone +pizz as +new comer +de part +oppre ssion +ebon y +foss ils +tro jan +el en +ste aks +k hou +positi oning +ug by +red cross +ak h +dol ce +us mnt +pp en +dil ig +ma vs +call er +cost ello +⼠Ħ +dy n +thing s +rhin os +a xi +sar kar +con vocation +att ers +ss ss +fun gus +eu gen +russ o +squ at +w sb +eli on +william sburg +s off +defici ency +be arer +o kin +key stone +t wain +cal ming +break able +wa res +horser acing +com bs +bun ting +u it +t land +ðŁĴĻðŁĴĻ ðŁĴĻ +ga stron +sab ot +ick ers +commissi oners +sen ate +ii ot +ath ena +nit rogen +an tony +ero tic +di alo +mis sou +hypo cr +âľ Ī +kaeper nick +can v +d roo +clevel and +o sh +mon sta +stefan o +^ ) +sh ul +po ison +ha e +commerci als +ma ul +nit ro +co worker +alo e +vap or +t ents +russi an +qu id +question able +mid get +po ker +girl friends +sin the +erit rea +ten ure +depos its +buc keyes +spot ter +theod ore +trin ity +joaqu in +u cci +follow the +caf c +mp a +ðŁIJ » +plo tting +dom ino +ta ek +sion ally +dicap rio +pa p +car mel +ig er +bt cc +beth le +www bigbaldhead +foo die +bagh dad +mason ry +off ended +à · +ภģ +sc ro +vers es +ori ent +ar ches +pi yu +know your +gre e +ta kers +gu ard +dish on +bucket list +bha fc +war dly +ðŁİīðŁİ Ĭ +leigh ton +pe w +stra y +assaul ted +in hal +ly fe +amar keting +l x +kat z +ubun tu +me o +carto onist +turno ver +mi z +dis like +mul len +mo f +bl and +hi des +emer ges +chori zo +truste e +ma hog +lan sing +paralym pic +fa int +fa una +ch al +sn ar +cat h +bent on +cast illo +sli ppery +apric ot +oec d +bar o +l z +he ming +clow ns +co workers +peru vian +commu ters +y ell +ðŁļ ´ +under ing +v j +tt p +fli pk +w ana +soc ent +Ĥâĸ Ĥâĸ +ठĤ +oo sa +jag ger +di sm +e less +d ham +cali f +a official +ec lip +harro gate +gra pp +com rade +n tr +concentr ate +thi ghs +bit coin +bel arus +ë ĵ +end uring +now watching +industri al +pi p +ar on +ar at + ® +whit by +oooo ooo +sa ree +tic als +mis leading +yo on +year s +sle igh +roman ian +sciss ors +vam pires +ac up +ab ba +th weeksary +cent ri +fl ye +u o +c bi +bu ena +sin d +mar ino +bur r +re building +ठ² +anniver saire +ac ca +ðŁĴĢ ðŁĴĢ +gett ing +tu lips +wolf pack +âľį ï¸ı +more than +ta kin +ðŁ¤ĺ ðŁı» +u be +mon ic +dou bts +mo wer +co balt +don ne +specul ation +argu ably +kak u +htt ps +prosecu tion +din ah +stam atic +disclo sed +bever ly +fl wx +cra bs +extraordin aire +war mest +imper i +o logists +trac es +par c +lake side +am r +ter i +hour ly +domin ation +ar row +shrews bury +ance stry +wr angler +trigge red +pen sac +roo ster +survi ves +a on +bo ko +val or +love is +la g +pe y +fo cal +out laws +bl anc +artic ho +wit s +marsh all +die go +support small +u ca +sa h +je et +syn ago +gover ning +ðŁĴ ¬ +sal ads +cre ate +miri am +cen sored +ami de +no u +z eta +allegi ance +* ) +bl m +ric an +pa stors +oly mpus +blo c +whir l +star ry +pr one +y k +p ne +congratul ating +be v +so ber +love island +sa ir +an ing +tutor ials +q e +lun d +in ist +cle ver +taxpay er +ali z +wren ch +dd ling +cap ri +h pa +ðŁı» âĢįâĻĤï¸ı +na j +o j +futuri stic +jelly fish +ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ +cel ery +plan k +fil a +ne me +un healthy +lec tions +ðŁ§ ¡ +rit chie +n ws +mi kha +wonder woman +âĢ İ +hip stamatic +ka g +ðŁĴľðŁĴľ ðŁĴľ +poul try +mo w +wor ds +lo ff +ðŁ¤£ ðŁ¤£ +relat able +re mixes +keny atta +ke m +re signed +fo d +stra igh +j lo +hu tch +box ers +colle en +mag s +instruc tional +ko l +attrac ts +pra g +account ant +go ggles +br u +th ole +mar row +leu ke +oc to +pon ds +bubb ly +he ist +ìĹ ij +im p +a har +ha unt +hall mark +psy ch +kkkk kkkk +col umb +jump suit +cost co +si delines +ag gies +over turned +ni b +key chain +fu k +f af +mi am +assist ants +cy cled +ri der +dam mit +red wings +mag es +kin s +ì Ĥ +ho d +son t +carol ine +" ' +cu le +bra id +fel ony +ar ities +ruther ford +depic tion +isab elle +ro ach +k day +fifth harmony +em y +li gam +bari sta +albu querque +gro ss +ðŁį º +oo ks +ðŁij ¼ +dun can +try in +jag s +g ould +li tho +âģ £ +а Ð +sam my +tun g +cas ser +apo lo +aaaa a +man g +as ics +sh en +p ye +tur bul +ss p +saint sfc +on lin +n anny +he ster +do z +ภĶ +th read +ren ts +kh and +ðŁĴª ðŁı½ +un conditional +rob son +car re +ph on +sacrific ed + £ +auto s +par ker +oc a +log in +kee gan +hard cover +dough nuts +ðŁĮ İ +spit fire +refresh ments +saskat oon +commod ore +j f +rub ber +halam adrid +child care +stra da +io m +ri k +dak ar +ther mom +cro pped +gar u +ali k +ven i +i ft +si ka +ritu als +z ul +e ch + © +su dan +l land +i me +do cker +ì ¤ +fe ared +fa o +wal ter +no g +mutu als +l h +ali gn +mon ia +concep tart +ðŁĻı ðŁı¼ +sco e +compet ence +sw ine +ly me +laun ch +green er +abstract art +inqu is +gran ada +ga elic +flu ff +d backs +grave yard +ba be +acade mic +adventur ous +joh ann +~ ! +bi bi +| # +pl ings +gett y +as b +âĿ¤ï¸ı @ +staf f +religi ons +bang or +world bookday +me gh +de vin +ash ore +meri dian +gi thub +qui z +all stars +be stest +ir resi +ack er +do te +war rington +pol ly +newor leans +cr ou +wi gs +che y +smithson ian +la sag +de tour +bor is +stra ps +mari ah +inten tionally +ko h +ðŁį ¸ +ssi an +mar issa +cor al +episcop al +casu alty +tom o +supply chain +sam p +on go +ro o +cavi ar +p fw +clau dio +buff alo +s ations +mat ty +snap back +l ds +al arms +mat te +âĺ Ķï¸ı +conditi oner +d ors +he x +fi zz +a stri +sus sex +secur ity +qa eda +all star +cocac ola +as one +cl icks +sc ans +mu te +he avier +ðŁİ § +âĺ ŀ +lv l +book boost +youtu be +fla shes +f jor +c su +explo de +do dge +cair n +gonz ales +th ill +pel le +hart ley +renew able +re tin +e stre +costar ica +shipy ard +nc fc +pri ya +a ghan +an ath +plu gin +co rey +re bound +or u +kat rin +hor mone +gi m +mahin dra +s sus +park land +har per +fanta stic +infer no +ep ilo +wrest ling +fe ct +c it +ac oun +to ssed +monu mental +char tered +bu st +pe tra +âĮ ļ +wildflower hour +sweat ers +* . +bl er +ate ch +go wan +demo graphic +bra l +suici de +renov ations +vu el +sin ister +ar mani +miso gy +ph arrell +nap s +un iting +crusad ers +cor gi +insu red +than i +no or +g q +d ada +bicy cles +snu ggle +sch an +ten berg +ss al +fe mme +bo il +½ ï¸ı +re ap +occur ring +hus sein +divi d +sto ke +sh alom +na ia +o lic +frustr ating +Ù ĩ +ig s +gro ver +scen arios +n ds +bru tality +med alli +bu on +sas s +skate boarding +ony x +lor ry +ny u +gau tam +mm ings +gu g +end i +lo thian +comm ando +chal k +ph ora +asse ssing +ti gh +crun chy +ad ay +is l +ci ara +pilgri ms +kam al +p to +brit anni +t ani +sm c +l ure +app store +ab y +golf ing +cl c +fa u +an as +shu tting +regul ated +carn age +scow boys +all enge +c ma +humbold t +rel le +ku mb +her i +refin ery +sound check +d wayne +bos nia +i sp +the alth +anni v +relev ance +my a +bag gage +dre ad +s bc +th ed +bu h +hi jab +lo id +ke w +c te +respec t +lovel ies +cu bes +celebr ate +dir t +sav ers +_ , +gar ment +pulit zer +mas jid +beat port +al arts +encry ption +s ner +ple ads +found ry +sym metry +ru mi +birth place +scallo ps +supp le +pivo tal +t ati +no de +so d +pro xim +tr ics +col dest +bren t +mand u +cla ir +e ach +and alu +hi ddleston +ðŁIJ º +mel ts +v ance +pin n +se ments +scre ened +sa chs +o bl +ic ha +âĺĺ ï¸ı +school ers +heal ed +lo gged +ðŁ¤ĺ ðŁı¼ +ic us +bore dom +b ish +b ffs +tal king +sure sh +hoo kem +de on +de fl +ei leen +ðŁį ķ +women intech +ri sotto +rang er +adverti se +ภģภ+tel ly +la go +dart moor +d ong +sk ates +lo go +un ner +mail box +ma sala +lo oooo +amethy st +che wing +c bb +australi ans +rc mp +game art +# ... +kor n +extre mism +fruit ful +anci ent +pu bg +pol ite +wh it +mur als +m gr +line man +dav ao +ste ms +ten nis +av age +tu pac +gigan tic +hs bc +auto biography +up the +ี à¹Ī +re gal +fig uring +ku l +mis sy +hoo p +gra s +for ums +back lash +abduc ted +p nw +min ic +bu tt +bott oms +at on +ven g +ðŁĮ ı +del aney +prab hu +fan club +over haul +health ye +sy no +aa f +ren amed +kim i +un cle +man city +se u +qu anti +este em +um in +en zo +mel vin +under go +j har +far ah +coast ers +humph rey +mh z +children s +^ . +d hi +disrup tive +integr ating +r nb +over sized +a ide +ne au +docu mentation +ðŁijĢ ðŁijĢ +pal o +hear th +ri yad +pun ctu +abc news +secu res +boy band +bir ch +ju co +tra ff +legislat ors +bay a +ãĤ ¯ +no ises +collec ts +s warm +k ner +bi shops +stur geon +snapp ing +mo l +fre aky +chair person +tro p +lyn ch +car cin +art sy +e sto +cha i +fl ur +inv ali +sau sages +im el +j or +fun fact +wit ter +puni shed +ac ons +h ya +re versi +em c +dif fu +z x +sp aw +cla d +d mit +hol land +fre sco +pay roll +ab undant +stu ffing +mor o +c ny +boy cott +wend y +ele ven +pro voc +pil ot +tr x +be ad +climate action +ri on +assi e +ì ĸ +o sm +islam ic +ho ar +good reads +al ici +afterno ons +spoke sman +jo lie +it as +masc ara +âĻ© âĻ« +pre vail +beetro ot +lu jah +k li +dod ger + » +ru le +l n +scre am +ho bart +col bert +r tc +er m +pat ro +quo ting +s live +que st +non fiction +semin ary +prosecu tors +ve st +express way +g ge +nau tical +et f +ðŁİīðŁİ Ĭ +dur ation +cha ired +the film +fab io +she h +can o +ðŁĴª ðŁı» +with draw +! :) +cor pus +phen om +yel p +la wn +ent om +snapp er +but te +pin ball +pro xy +libr e +alle vi +n ada +gabri el +fo wl +eure ka +daph ne +tu nes +pun ched +wh ore +jo g +ren tial +man ners +o pe +wh ufc +gu th +revol t +sne aker +philharmon ic +ho ste +sovereign ty +ðŁĻıðŁĻı ðŁĻı +fish ing +sci art +fe ta +i pp +dump ing +kel own +gir i +dig its +sal u +san jay +twee ters +sp as +col chester +sc ab +ma dd +๠Ħภ+Ä ĩ +ged don +march for +do p +maure en +un plugged +di do +fashion blogger +up a +mex ic +tar y +pol ye +jame son +v t +grin der +mad dy +consult ancy +¬ ë +leagueof legends +ac cents +um ni +jane iro +tu ss +h ens +ampli fier +to shi +pret tier +pre vents +new town +red wood +vant age +ball ard +ar tof +a she +a sion +lac ey +ap at +gro ve +ภĦ +rw and +real tors +tra itor +bed ding +ö r +zi on +fla shing +cam pan +boom er +secretari at +ab ol +liti gation +cont amination +se dly +shred ded +in for +do herty +bench mark +ro che +skate board +sho vel +i zz +to pper +o ster +laby rin +autu m +k ong +hum mus +vi z +tech news +kla us +am using +socialmedi amarketing +i des +cast ell +ste e +underestim ate +cal ab +pa ign +b illing +unanim ously +g mb +fly fishing +hath away +commerci al +colour ing +skul ls +pivo t +te p +tb c +motor way +x press +construc tive +pu k +under lying +kir sten +mani ac +cha o +se ma +chiff on +ðŁijĮ ðŁı» +ver ona +kom o +stan doff +wi ped +c ated +bla ir +wor kin +m sc +bethle hem +swi pe +unexpe c +pe es +pe tri +orig ami +ðŁij ħ +mex ico +flav or +ru dd +cannab is +mar u +ri ddle +wor shi +sil on +sch at +ap se +tang er +bi ous +e er +questi oned +o zar +dan k +angle sey +char an +bak u +compe ten +re pri +bat ter +sa xon +cal ves +leng ths +$ $$ +âŀ ¡ï¸ı +immer sion +ga unt +car ry +cy to +b anda +shu tt +experi ence +el gin +mous se +ta z +ê µ +in correct +en z +b ham +mor on +so ver +ar un +ti pped +la ble +de arly +bau tista +í Ļ +mor tal +woo p +dt la +sho cks +dav os +ðŁĵ Ŀ +swim wear +her man +ðŁijĩ ðŁijĩ +z ir +neglec ted +grac ed +campu ses +av s +ar ora +swach hb +live pd +ac cra +enqui ries +shoo ters +kur t +vancou ver +brad ley +gar da +g ü +ol la +attrac ting +up ton +ne win +lu mia +furn ace +ev ers +e on +sw a +roo kies +a oc +v ss +bris ket +tor ch +yo da +heart land +tac o +ph ony +food bank +ab bey +bab ylon +u y +gre ate +expre sses +d andy +sc apes +survi vor +ron d +e ci +ha vin +ab el +chil dish +tor que +wav y +ur self +kanye west +year of +ale stine +o brien +al fon +sk ag +kore an +anchor age +val eri +de w +ðŁİ ¨ +land slide +car ole +christ en +go phers +af i +priyan ka +q q +power of +it te +pc so +tw ol +pr y +intellec tu +guer rero +pi les +wish list +w ren +time table +ë ı +prodi gy +gibb ons +. / +ne ur +anz ac +mur ray +vie st +pla ster +la ir +art gallery +inter continental +g br +bell ator +nam joon +mam mals +am el +y aw +saras ota +cam ar +bud ding +sum mari +aco sta +la sh +ey ou +post graduate +instruc tors +ti g +const ant +were wolf +ic os +cla s +glen n +bud ge +ðŁĻ Ĥ +er ta +sta ins +persecu tion +cumb ri +o ch +syner gy +hu ang +scand in +mid terms +comment ator +regar ded +perpe tual +bo iling +al p +lan ge +sch le +fac eli +twee ta +ri dden +ok toberfest +charlotte sville +ik lan +jo u +ch atham +b sc +ðŁį ¦ +stra uss +mel low +xx xx +happy hour +re actor +ww er +distr action +at orial +ðŁĴª ðŁı¼ +twin peaks +fay ette +a or +ko k +bro om +sy fy +ou se +am ag +Ø · +ubis oft +lu lu +hall mark +stu art +it ya +si deline +venge ance +re lu +sex ism +boun cing +un ites +gu stav +te ssa +stu mp +pro clamation +ima x +divid end +col by +ðŁį İ +play wright +un safe +co smo +ðŁĩ²ðŁĩ ½ +cup board +constitu ents +ang lia +ram page +ðŁĺįðŁĺį ðŁĺįðŁĺįðŁĺį +than ked +take aways +shro ff +de bat +kh ur +conduc ts +format s +à © +port age +graph ers +u ten +pre m +mo ines +condem ns +s ous +l ps +f cs +deal ership +leuke mia +bure au +ski d +guardi ola +ca ster +thir d +avoi ded +en cyclo +c sr +vi xx +analy zing +she ar +dulu th +shap iro +chan ting +stre sses +as be +mil itia +ãĥ ª +col lin +arsen e +sure sh +teach ings +yi xing +sh ill +nu des +sv u +clear water +war ped +pro life +artist son +it u +versail les +galax y +ax el +spring st +cal a +hu hu +sc u +commit ments +exe ter +poign ant +mo tion +conserv atory +row dy +rec alled +mu sk +emb elli +so the +âĺ Ģ +sto pper +sch ild +to pe +el mo +zi el +j om +barn sley +snow den +on tour +jour ney +hills borough +par ole +w ts +mo ving +ag ility +tiv o +ff ers +kindle unlimited +g wen +ann an +ah mad +tex tured +hepat itis +dra m +insi ders +tis sues +ãĥ Ħ +fc barcelona +cr atic +na acp +pe can +f gm +custom ize +concer t +g sm +pe g +p one +justin trudeau +super cars +happy holidays +bu lar +ado x +lap tops +digital health +destin ation +gradu ally +áĥ ¦ +popp y +ss l +inhi bit +star light +of fro +glo omy +x per +hal der +im plants +le to +hass el +a as +un told +en ci +liber ia +or an +con tests +il ah +sma g +sc out +mari anne +cr yo +schedu ling +lo s +kan e +stutt gart +ne se +law rence +da in +pho tom +car ou +ภ£ +g wy +national dogday +roa sting +band camp +kentu cky +stret ches +ke rel +ca she +ãĤ ¸ +sta x +tran si +dog gie +at ric +hal le +ci vic +brow ning +lein ster +cat day +high land +joy ous +in cumb +or lando +ro mo +col ton +del ta +car ab +ro tc +aster oid +goose bumps +mo logy +yo ko +an ds +tomor rows +red carpet +sm p +ca sio +ðŁ¤£ðŁ¤£ ðŁ¤£ +se au +rejec tion +rot ating +bi partisan +th un +mat i +bon i +ol l +ener gye +do it +l j +mother hood +lou ise +neck laces +el ite +ni x +l cs +en v +gl u +le sh +cran k +su sie +m clau +so tu +crow ley +rat ri +use d +bre ton +alfre do +ye o +travel pics +ti pp +elli son +sax ophone +me red +heu ghan +ta ine +f es +vi ro +suppo sedly +i as +dige stive +y le +li zzy +wildlife photography +bri anna +west field +ra ined +am her +ðŁĺĦ ðŁĺĦ +distribu te +bott om +pre serving +oil and +craf ty +de scen +col ling +shakespeare sunday +r wc +ang led +ci an +t ations +mon tage +me yers +france sca +ðŁĮ · +wi ggins +san ford +volunte er +car ra +bar k +vari ed +pl in +am u +kap il +rock ers +qu ind +br ane +in mate +ent al +impro vis +michi gan +re tweeting +progre ssing +mercedes benz +smo ker +physi ology +dor ado +watt pad +h wa +sr bachchan +w ga +vol atility +hi re +ac ap +wn ba +hein z +stit ches +kidnapp ing +bur ys +lim b +f itters +thumb nail +ton e +mir and +desi rable +ad dison +tar an +tamil nadu +spec tator +soci ology +amit shah +remo tely +âĻ ¦ +ham id +r ds +g lee +smooth ly +sch ro +er c +lali ga +he als +us f +ni shi +d hu +un il +h le +tro mb +bhu tan +pilip inas +se ung +whit man +te y +min ce +snow boarding +re au +k ker +av o +zach ary +ran veer +ti k +gover n +qu al +beck y +anthropo logy +att en +grocer ies +de bit +war p +sil icon +hawa ii +ðŁĴ ħ +pomegran ate +pe er +orang es +people schoice +end ure +ðŁĴĽ ðŁĴĽ +ãĤ¹ ãĥ +ac ial +a haha +stu k +imper ial +bl ond +pow der +kno ts +vin ce +wood lands +den a +watch in +mat cha +ma hat +galax ies +middles brough +k ö +stre e +resc ues +wal do +lero y +desp ic +real ities +tm nt +ha q +un o +pe c +bolly wood +blin ds +design thinking +he ms +and hra +ab sen +fan s +ste ch +shire hour +bla ine +shak ti +pu rely +ðŁı ı +tra fal +ke ynes +gr ate +to bias +spon taneous +satur ated +caval ry +pri sc +ðŁĺ ij +wh t +pas si +~~ ~ +vir at +patt inson +la o +weir do +sym pathy +ju da +occa sionally +cred ited +stat u +es co +hil ly +esc ape +dischar ge +se er +may nard +sud bury +z lat +or al +we er +encoun tered +sm elling +over sight +ê ¸ +that cher +mack ay +you can +fre ep +freed oms +prophe cy +ho e +ishq ba +dra ke +qu its +pel led +tur k +o vi +wesle yan +new music +leg g +ch eng +h illi +ay y +pan ties +ad versity +ad jac +vaccin ation +ju ke +ga c +exce ed +time sof +sta ining +ep cot +v ital +up ward +bethe sda +apar k +ma hi +camp fire +enchan ting +rha pso +h z +na ver +fa x +vali dation +ac ad +ny r +as ym +coordin ated +depar ted +all ery +var ies +spr ite +chap lin +ss occer +s wat +bre t +relu ct +tunes app +super star +reminis cing +o co +home grown +dough nut +un canny +la pd +thyro id +! âĿ¤ï¸ı +botan ic +bre s +sp ade +i ste +echo es +du lil +bur sting +qui ero +ðŁij İ +loy ola +amuse ment +ha ils +sleep y +burgl ary +âľ ı +ro gue +cot land +mo ors +low er +wic ked +ðŁĶ Ĭ +compet iti +argent ine +yvon ne +karti keyan +ili ary +gat sby +precin ct +six ty +na ji +cam s +practiti oner +ðŁĺ³ ðŁĺ³ +pu ne +neg li +juli en +inv aded +cali br +cla m +duba i +mu k +lan tic +produc t +fe dex +ï¸ı : +eu ra +dari us +s ling +virtual reality +home stead +ðŁı³ï¸ıâĢį ðŁĮĪ +pac ed +in ha +pul mon +la zy +premi ering +ma stered +in he +con gregation +ba jo +sport ing +new jersey +hor ny +lma oo +leng thy +du t +yo gh +swe aring +philosoph ical +pap ua +in ski +know les +dy ke +âĢ ² +to ken +mc guire +ri ot +probab ility +mc con +gro s +su mat +c ite +da a +on da +mad dow +che w +board games +spar ked +re claimed +ad hd +ny se +imwith her +equ inox +boo ths +balsam ic +ha zy +dor chester +ag os +se aw +moder ator +seri ea +ander sen +pilgri m +âŃIJ âŃIJ +itch en +hal li +x ton +nathan iel +mun ition +celesti al +ga f +zo om +mark le +pen thouse +cal e +s fa +bar king +tu cket +em ery +cal orie +li que +ad ar +mc nam +tor tilla +wood pecker +mo town +bad ger +ayr shire +scram ble +dd ay +cra ziest +per rie +cho co +cast e +i ot +wre cked +selec ting +uss r +gra ft +pun t +lab ou +ir st +ba ek +Û Į +su ki +que u +ach at +te ster +aug mented +wc vb +sin ks +ðŁĵ » +ra ke +inter ne +be cause +belle vue +une arth +light en +ðŁĺ £ +turn around +labe led +unemp loyed +twitter kurds +le ia +h ye +great er +ðŁIJ İ +tim ed +i red +e tt +limit ations +cab e +s out +bee ch +anni hil +re trac +yo ona +ang er +den nis +supp lying +di z +" ( +sc ur +gun man +su ho +sauvi gnon +ภ¥ +wi ley +land on +choreo graphy +pre historic +ðŁı ĥ +var gas +assess ments +pinn acle +di i +chamber lain +ì Ī +v p +present ers +deut sche +sun shine +sal utes +r one +bu siest +- .- +motor ists +hemi sphere +al wx +ps p +ow a +den ying +cho c +gu tier +han uk +mus kete +jait ley +se wage +t ame +thin kers +shi m +se quo +pap ar +middle east +k wa +ke g +patag onia +no y +bar ça +take off +he a +à ¬ +n sc +g dc +ðŁij Ī +mou stache +mel ania +thr a +â¬Ĩ ï¸ı +pier ced +ze us +fon ts +ber a +it iner +q atar +contr ary +ire land +i fy +ou los +commun al +fin s +un paid +pa a +ðŁijĩ ðŁı» +ri os +ou p +f iller +cafe teria +à¸ Ń +kas i +cali ber +z ulu +v sco +ts ford +dragon fly +smo kin +pi st +psycho logist +diplom at +we bs +buc cane +à® ¾ +motiv ational +du ne +ba e +c fs +with out +er on +i ac +ate e +pen sion +fra zier +en sis +sk is +par ting +ger y +territ ories +nach os +eni ght +ever lasting +msd honi +tel e +sp un +po di +sab ah +environ mentally +ce ase +beau mont +mar ta +kel vin +ho ff +sun il +n da +co b +sh ale +ree dus +un boxing +u bio +re opened +n all +capsu les +mar r +himalay as +swee ter +ja z +f mr +twee ter +dha ka +na u +de mi +d fs +ta urus +fad ing +it utes +ci p +over flow +jef frey +don ny +car tunesapp +ðŁį ij +prefe cture +danc ed +c pt +ple asing +ital k +earth quakes +ul ation +hi o +ãĢ ĭ +ant an +nutri ent +de ere +selec ts +enrich ment +r iti +tram pol +bl amed +j ia +contribu tors +chesa peake +pi geons +tribun al +mad uro +w su +ilo ve +effici ently +dar cy +war ms +ar ra +ec u +ho wer +strugg led +rajini kanth +ðŁĺ¢ ðŁĺ¢ +hou sing +str at +eli x +disp ro +raf fic +thi erry +na sty +c fb +staf fing +al ma +back ers +hen son +sky walker +reale state +roo s +ness y +chan ce +cair ns +c ci +pe dal +ly ft +cross word +wait er +only in +kru ger +k ir +alej andro +car tier +car rera +re paired +ou at +un clear +un breakable +today in +qu eries +jo dy +gen ital +win ner +to l +kelown a +fascin ated +ãĥ ¬ +sris ri +squ ared +spr ung +negoti ate +priv ately +av en +>> >>> +g ical +gav in +chester field +zu mba +or r +nat alia +impeach ment +mn l +car at +criti que +credi ble +trac y +tan i +musi k +jig saw +gam bia +tol kien +fe u +as per +sav ory +fo xx +f itt +mar lon +l rt +v ell +p br +imprison ed +i om +chu l +wind shield +kay e +ba a +chor d +s art +al gon +minister ial +nat geo +la zio +nor ms +ðŁijį ðŁijį +lic king +fut bol +un sung +dalla scowboys +sh red +distur b +dev ine +be ards +ch f +b day +ro sso +ig or +ay i +si ren +k air +sti les +ro f +mag nets +un cover +mou se +bang ing +si ghted +spe ople +impac t +row land +kir a +environ ment +love the +p sis +mish ra +gl endale +ca jun +o che +de ception +sex ist +stra ws +s ga +buff er +apost le +sp l +pop up +ðŁļ Ĺ +r g +up er +ball in +i dy +occa sional +national park +ðŁı Ĭ +u an +innov ation +ภ« +te aparty +re tte +counter fe +b ha +rec s +ig en +ðŁĮ IJ +humming bird +cu r +ha ven +la zar +pue blo +: : +zi onist +op ath +inver ness +promo ter +carto on +cabine ts +mahog any +surve ying +r ational +feel ing +testi fy +so w +oc on +ภ¢ +ne el +mar is +sol itary +che mo +rad cliffe +sim ons +ros ary +new er +jo die +re tali +pra wn +pad dy +hen ge +k ala +im plant +at y +bren twood +par adox +ene z +re designed +p our +wy d +al de +௠ģ +sol d +biomed ical +๠Ĥ +tt tt +mat teo +ys er +new ton +de bun +ner dy +loo l +wo on +elisa beth +ec c +wh i +ach o +salv age +sal aries +qu ity +navig ating +oph thal +con soles +re built +o pec +ast ers +sho red +set list +kathr yn +rhy mes +re visiting +ash ish +li ft +re post +sole il +âı ± +weal th +sa at +we c +king james +flipk art +field work +se gu +mo dal +bu b +are rs +ðŁį Ĵ +clo oney +pad dington +necess ity +guth rie +pen te +li mo +jo sie +ar tin +en c +l hs +betra yal +info graphics +i er +mo a +hear ings +bon jour +sym bolic +ag ro +wed ges +krist ina +wild flower +athle tic +photograph y +pe sh +ca hill +chi lean +gou l +fi oren +ðŁij ¶ +z il +sk im +bad oo +deli a +tre ble +n cc +ðŁĩ¦ ðŁĩ +a house +bul lock +sol itude +ا٠Ĩ +can cers +futureof work +hu tch +water shed +war mongers +sp illed +colom bo +mo th +associ ations +weigh ed +global goals +not just +christ i +tor g +swe ating +man eu +clu sters +âĢ¼ï¸ı âĢ¼ï¸ı +ta ped +ul y +tru sting +yu suf +te in +ra b +, ,,, +sin ai +audi ble +explic it +cro wns +sch iz +at least +ðŁĹ £ +de bra +je suit +ene gger +z hen +one sie +i it +ss f +gur gaon +chak ra +bear cats +k ran +k awa +reque sting +han over +g end +sor os +mer cy +lovel y +do omed +tim my +ku z +ul l +ab ram +sa ison +ãĥ « +clean ers +re mo +circu its +bar red +o th +mo ist +madele ine +gall o +u j +per mits +hea viest +car ols +az te +gior gio +flo ats +decl aring +us rc +min at +craf ts +pri ma +conven i +nickelo deon +danc ing +ceremon ial +blo gg +tw p +anglic an +she k +k nick +( (( +hubb ard +harve y +hit man +fen g +we some +for za +s word +op us +bro m +gi bility +z al +m unch +dance hall +gre edy +hd mi +re birth +ðŁĺĭ ðŁĺĭ +s world +figur ine +com post +k f +engra ving +gior no +st ana +k man +ham ster +compos ers +aj e +func tionality +pol k +is ons +air planes +te se +hor rors +musc at +gi ven +sp ence +ðŁĩ¸ ðŁĩ +eli ot +ach illes +fre ck +crypto currencies +sou ther +hal o +bor neo +polit ic +hahahaha h +up state +si ena +obsc ure +hau sen +lloy d +happy friday +motor bike +bon a +americ as +hol s +- ( +spor ty +un aware +reven ues +christop her +bank sy +av an +ev apor +com press +eyel iner +to dos +buff y +renewable energy +ly rical +ar chan +rapi st +fair trade +lma ooo +beat z +pro active +la pse +ir ical +revers al +po de +mcin tyre +mac au +ãĥ ķãĤ +nash grier +f sa +g all +çĶ Ł +perpe tr +il ya +configur ation +% ; +str ange +rac i +ภĩ +pic kups +kov sky +mam mal +w ps +g able +compar ative +z h +save our +da vey +on etsy +mu ssels +mis er +cri stina +electr on +cra ve +lo ren +precipit ation +m z +ðŁį « +vin cen +snow board +no ida +ah n +marin ated +g tr +town hall +min is +bethe l +adv an +su ra +shi el +fur ry +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤ +lyn d +so il +sc ence +sen eca +shar jah +dick ens +credenti als +av ar +per k +requ iring +pre fer +j ian +de ca +r ach +ing for +del e +be ep +ðŁĴ » +cis ely +hu ddle +green sboro +haw king +ho ax +hang ar +ç ľ +mis o +lo vin +gre ta +ab ad +logi e +at an +snow flake +mahe sh +fear the +al kal +bobb lehead +ba hn +ju dged +fu tu +feli x +ðŁį ĵ +pi ke +der iv +notic es +au er +dis super +or da +wi pes +am ino +stri kers +foo tb +dram as +pun ching +score less +heming way +bi h +bal lad +chat ter +am mo +kle in +fabric ation +kari m +z end +hi sto +vol ta +rock y +marke ter +xtre me +sequ encing +paradig m +cle ats +boom ing +âģł âģł +block ade +promp ts +yogh urt +pur pose +nu r +regu late +nois y +ing rid +bird watching +bar tender +Ù ĥ +wor dof +cha otic +shor ty +el dest +z app +onceupon atime +fl yo +rit os +mike quind +ðŁIJ ´ +regi stering +. ] +ad ol +gg gg +pur ge +kid lit +ar bor +val ves +synago gue +o th +unanim ous +veri fication +dar rell +ãģ Ħ +vander bilt +tape stry +pro sper +did dy +dra fting +de cep +marqu is +st int +michael jackson +pee led +men us +bb b +sc are +ema il +wri gley +it is +f ell +some thin +bar ra +ed gar +di pping +pu ddle +sla de +lear ner +jal en +ðŁ§ IJ +the daily +mikequind azzi +ju x +iq bal +mckin ney +ra iser +ef an +dr one +cat o +pic ket +cro we +l att +uk o +giuse ppe +hin i +synthe si +ponti fex +song writing +to d +swit ches +din ners +h q +gabri elle +pensac ola +cir cle +expo ses +ev s +riyad h +pro men +o ck +sa j +cit ation +brew co +jo si +ep aper +dri f +point less +tang led +cri pp +line ups +fairi es +daz e +mour n +bla dder +sal z +bur undi +book mark +the people +sub sequ +princi pal +sk er +court ney +a oki +rac ers +ad m +mom a +critical role +hou n +shed ding +sa ka +ace ous +mck ay +hus bands + ½ +me da +accu sations +ro sel +nc is +witne ssing +or ama +go ds +hil ton +el man +ÃŃ n +meg ap +cra ven +announ cer +crit eri +sheffiel dissuper +milit ant +consu l +hoo ded +aby ss +b x +ma dam +lo cu +mary am +manic ure +grat is +ac tresses +ros ario +this dayin +king ly +gn ome +cel ine +r ous +he el +lil ac +vish al +ab h +thor ns +s ls +ne al +construc ting +be ren +s lang +ma ins +far ra +sar ko +pai ge +gu iller +l ala +ice berg +nou n +plann ers +u mmm +ou ses +ill ary +ma an +box ing +zi pper +srin agar +migu el +o str +mp o +responsi bly +lan terns +appli ance +x b +gren ade +neglec t +dy sle +ham mock +ne ctar +wit cher +r gv +di ence +ser bian +seed ed +cru z +bi sh +sp he +e q +sky rim +alge bra +phil ately +bungal ow +ge off +y ves +demand ed +consider ations +the vamp +pawan kalyan +co ded +grit ty +erup tion +se infeld +uni denti +ëĭ Ī +wor m +ac us +se ung +dun g +ro land +su d +di visions +ab lanc +shor test +j f +p oun +plant based +be to +tough er +mc o +don et +mark us +v fl +ðŁı ł +open ing +co ward +caber net +o xi +burle sque +sand ra +su mo +consi st +tho t +cay man +motor ola +gutier rez +d slr +y w +no bel +nov ice +moms demand +grun ge +sp or +d cc +pre sses +sli st +allot ment +voc ational +ft c +pu ja +lo ven +utt arak +tan dem +sh ep +come dians +anat om +cant wait +healthye ating +west side +mar gins +chi ang +asbe stos +stupi dity +proble matic +fit bit +: $ +ceil ings +shu a +protec tions +bio tic +beng ali +re sts +bien nale +tim o +cul min +e minent +affe ction +unbeliev ably +individu ally +canvas sing +wh itt +nov asco +chin son +h pe +go w +gloucester shire +pa o +thresh old +chev ron +s ine +we ther +pp ie +aqu ino +antwer p +âĸ ¬ +po on +inst af +equ ine +cinemato graphy +nbaf inals +vali ant +kil kenny +te rence +syste mic +sr l +p ound +made ira +pl ough +tre cht +mat ed +mp d +ransom ware +ph in +li qui +bb ce +boom er +i standwith +con ju +r te +nar a +foo lish +da shing +vier nes +br ite +da u +juni per +ai da +you now +ra zer +de i +repe ating +comfor ting +adjac ent +e to +ca sted +chat ur +mu er +syn th +san itary +mac le +independ ent +law ful +e erie +h or +ðŁĴ Ń +am rit +vel o +station ery +mu f +may may +contempl ating +elabor ate +gre gor +dri es +ac col +ภļ +schwarz enegger +ill nesses +day break +follow back +collu sion +electr onic +jo vi +hiro shima +ta w +hom ec +mic ah +qu itting +fro sting +ben fica +hel i +s ical +pic cad +corpor ate +ment orship +you are +sing er +shi va +ru ne +ing er +ri um +play able +doo p +wil low +ter re +ni p +at d +war bler +profession ally +er ase +proce ed +pedestri ans +mis chief +ben ding +alas kan +c kett +mo p +dd les +shut ter +ge ared +atene o +ma deline +g ations +o sha +der ick +sw ild +an gry +pat ents +hun k +decre ased +fr y +ðŁĴĸðŁĴĸ ðŁĴĸ +sal on +quant ities +d ario +ni gel +ku ma +jen n +happ ye +xx x +rex perience +pro s +au sch +rele ssly +ham burger +fuku shima +er ne +stat ec +ren d +may field +j one +lef ty +bern stein +sm il +gener ates +fore station +band its +ta yo +r ca +ac ci +rodri go +kn app +elo vers +vege tation +u ral +le ft +ħ ï¸ı +worl dre +sur i +embar k +w son +ba you +mu ller +mo vers +ðŁķ º +presby ter +l f +cre e +bat b +sal am +demonstr ations +an ec +n pc +it ics +to graphy +re inst +thur st +tal e +off ences +smart city +bro tha +ofthe year +in valuable +ear n +ðŁijı ðŁı½ +kre mlin +gra dy +town fc +guern sey +ma ha +contag ious +dre x +be en +( £ +nati vity +k tm +somer halder +comp ounds +íķ ĺ +" âĢ¦ +af g +ott news +h ound +fire fly +cil an +donet sk +volunte ered +ak ira +è ª +sing ul +st h +dro wned +mand o +he ir +ðŁİīðŁİ Ī +tax is +y uki +vel d +k ans +el k +ran ts +hash tag +t eng +ro g +a at +gru b +e ber +in india +colo ssus +sig ni +so ever +mile stones +der o +differen tial +phu ket +master mind +an gh +mel ani +bro ker +actor vijay +stun ned +continu ity +af fl +vo cal +perenni al +fianc é +in complete +hun ts +re issue +domin ates +tur meric +ro am +ri on +bag ged +nas sau +fu t +x ox +national trust +jo ye +san o +hearth stone +dis respect +le es +h se +siber ian +offe e +re stock +wolf gang +re gan +plan o +un wind +re par +mil le +] , +skul l +fat ally +concep tual +ðŁĮ ² +f é +ber to +b ms +u a +mag na +notre dame +le te +la undering +heartw arming +buffe tt +go at +pe abo +wind mill +v ac +continu ally +az alea +mem brane +can cels +make yourown +athe red +p to +tor pe +ðŁĺ ł +ðŁĴ § +sc ares +le aking +z et +pix els +ac i +kh il +marath i +ðŁĻı ðŁı½ +u la +tam u +chandi garh +z agre +aa b +pronoun ced +aubre y +sand er +pun ta +har low +ic elan +celebr atory +so t +unci ation +stru ly +mc dowell +deepi ka +remin ders +my stical +ct c +chat ted +s ica +bar gains +ch hat +ru bin +m net +oiland gas +pel ican +o at +mor ality +k our +i h +nu clear +gc u +ric her +vene zia +m ma +le ith +ac company +rich mond +sports net +ba ahu +smu ggling +mm i +ðŁĩ®ðŁĩ ª +twi sts +sahi b +.... . +amb itions +il lo +histor ical +fo rec +show biz +pon ies +chas ers +remo del +will ing +prince sses +am ple +cushi ons +ac les +lot r +da ch +an the +in corporate +new bury +ki ri +fried rich +ab v +ball ers +alber t +ðŁij Ń +let i +nan op +ci de +anal o +n sf +)) )) +griffi ths +valen ci +ro ano +fun run +babys itting +ca day +ent re +u ck +slu g +tic al +the sims +ro ar +car ney +g am +sto we +fi d +bun ny +sham rock +pe cu +mol ina +go cougs +con tributes +transform ation +mo y +v aj +sever y +antioxid ants +thir teen +sight seeing +l j +reversi ble +odd ly +hoo kah +nou vel +hal al +fe i +stab les +mul t +ho pped +bra ids +inter change +ghana ian +ww ww +eth no +con junction +ago v +ye ti +earth and +ts p +con serve +heir loom +metaph or +woo f +tor io +self less +n wa +em ilia +yl ene +y xe +gi ar +moder ating +pro bz +b fi +ne er +du mmy +hanuk kah +we bber +k v +eye brow +dag ger +su mp +ra ges +ork ney +tb o +hal sey +assign ments +tr onic +scri b +co on +an war +# âĢİ +jal ape +flori da +qu aid +haw keyes +âĻ¡ âĻ¡ +street car +ro g +dat lantic +gran ola +un changed +expect ation +Ù ĩ +mar lin +gu mmy +ðŁĻı ðŁı¾ +awareness month +oil painting +mu th +per ch +jun to +villa gers +mor g +che ated +web comic +the future +d ps +la kings +men tioning +vo or +ident ities +accor d +mc gu +l pga +rum our +massi vely +m pls +heal y +d ate +sp oli +re visited +on t +al and +scru tiny +lakel and +bl ending +< / +an kara +jami edor +metab olic +f ences +ann y +å ħ +semic on +oo tt +space ship +wack y +le ta +ap ac +she e +in herit +do res +ðŁĩ¨ðŁĩ ¦ +gent e +tw ick +ri ms +gal ve +de ville +king fisher +scorpi o +ow l +al ar +vari an +ðŁĹ ĵ +vene tian +star dust +then orth +q ing +har rington +consul ate +spectac le +ho bbs +tur ks +gre er +mat ing +ðŁİ Ģ +ðŁĮ Ģ +direc ts +í ĭ +pompe o +vo iced +la os +tz u +pro me +pri sm +mer c +fortun ately +bc fc +mcdon nell +not sorry +smi led +t ba +for war +mid term +dar by +we instein +up grading +wol ff +bron co +cab ello +ðŁ¥ ĩ +fi able +shar pe +bat tered +sat o +myth ical +instap ic +pre pped +eni um +e spo +di aper +explan ations +who pping +ragn ar +pe el +antibio tic +l acks +harri son +li sm +au l +qu ail +martin a +sent encing +sc ams +di di +tr onics +ãħł ãħł +go ff +za in +param ore +cha ined +clin ton +li ff +cott ages +em on +reve rend +consu mer +ce an +t any +lum pur +e bay +sto ol +ðŁĺ» ðŁĺ» +ta pro +h ath +modern art +just ine +prover b +app y +tra x +mani fest +am bu +nai k +pe pp +r sd +mer chants +kitch ener +shi fted +li zz +âĺħâĺħ âĺħâĺħ +âĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶ +uto pia +tom o +ou ted +com ers +chiroprac tic +book club +cin dy +pro hibition +se uss +ë¯ ¼ +thin kin +rr rr +go fund +t ack +om b +catastro phic +ling u +guild ford +bo td +ॠĭ +plan ter +^ ^ +win k +kath mandu +sto ppers +smooth ies +re efs +hin d +bell amy +Ħ ë +waste water +vo or +nat l +! ] +re el +y ap +scoo by +work space +corin thians +bl un +obli gation +g bbo +dy son +cra vings +ell ington +dap l +wre xham +earthand clouds +uk runchat +positi oned +kal b +four square +jo ck +im pending +even ing +ath y +pro claimed +c ites +ann apolis +san i +mar th +ir l +accom mo +ka a +fin a +y aa +di sper +ec ar +bha k +will y +ðŁĺĢ ðŁĺĢ +mcder mott +mo j +gener ational +u said +train ing +lon ely +lo res +impe cc +âĢ IJ +beav ers +ma ki +he b +aap l +å ı +wolver hampton +leader board +me u +c fa +easter n +hu r +civil war +ou rage +hor ned +le high +awar ds +evi dent +gi gab +r ous +ma del +ro byn +ur gently +k ors +en as +heis man +bam bam +fab ian +f om +evalu ating +assemb ly +out sourcing +hun tsville +ðŁĶ ª +justi fied +cashi er +sp aper +buc keye +analy tical +illumin ati +au tho +o j +sha de +geel ong +wh ey +he aton +terri bly +ele k +un charted +sd live +moto cross +her mes +dar shan +dar lington +cash mere +gri pping +cilan tro +pun ish +... : +ðŁĴ Ħ +inst ance +der i +lo bal +muk her +sp ar +thin ker +fre mont +com piled +color ado +vig ne +sm d +whe ad +villa ge +le ek +formula e +ta res +persist ence +?? ???? +ped ago +he z +alzheim ers +vul ture +off ence +is great +suff ra +kick in +h mmmm +broad way +ï¸ı @ +art i +alli son +endor ses +ry u +lolli pop +soy bean +kend all +cer a +inv ade +( ðŁĵ·: +conver ter +car pets +ho bo +fr it +pe ac +es qu +ern an +ou f +an il +di ffer +ch ing +bre cht +sp g +daven port +stra va +sever n +n gos +stor ians +fe te +parame dic +j hb +al amo +sne aking +gold coast +roof s +isi l +depic ted +projec tions +nu mb +o ss +ep i +glu cose +zid ane +infin iti +íĺ Ħ +ran som +ton ics +fal k +g ler +ou tw +re ss +week ly +the on +n ole +ðŁĩªðŁĩ º +vol ley +sum mar +neg ativity +sam son +ye w +aus votes +ju l +ju dy +f art +pra yed +pal ate +multicul tural +double header +cycl ones +pier re +ãģ ¨ +âĺ łï¸ı +rt w +conver ting +wir ral +l ari +ir relevant +austin mahone +an che +ya an +sd f +$ . +explo ding +ulti mate +prof ici +gofund me +cell ence +ep stein +bul lied +sep tic +à® ¤ +lu mber +cu ff +vsco cam +pl or +ภ¥ +se ok +ro to +venezu elan +sor ta +spir ited +daniel padilla +team sisd +radio active +icelan dic +ðŁĴ ¤ +ver e +accommo date +shi pp +ot ter +ol ina +e go +su la +san antonio +de as +simil arities +âļ ¾ +y om +bro ward +å ° +can cun +veri fy +on te +candle light +ìł ķ +inf ants +az am +ðŁĺ ° +le ven +un stable +bloom ington +x ford +con tour +y p +innov ator +histor ies +po y +lolo lol +ex pires +cat alo +bill boards +an ab +el ic +novasco tia +fa ire +ìĿ ´ +rock well +gr ille +az tec +joh or +ur struly +fi ren +dun lop +id le +port man +jo es +tx hsfb +hol m +cham ele +under world +lo ss +ti em +therap ists +past ure +pa ste +ing now +vul can +ra gon +lar kin +o shi +ho co +child hood +umb rel +success or +kath y +iz en +° ï¸ı +share holders +ol ga +ai b +he ap +fl aming +ro u +air tel +rat t +z ane +vo w +thor ough +sn ag +par th +un conscious +ve y +new release +gh ee +croati an +facilit ating +swan son +astor ia +to logy +master y +ðŁ¤ ij +bil bao +trou pe +the ori +chey enne +ro tt +shore line +gra sso +master chef ++ ) +vi x +ellen show +as g +an ak +ku ya +safar ilive +debu ting +blu m +list ener +v ins +book shelf +smart cities +makeyourown lane +; ; +ðŁIJ ¯ +ri zz +on ward +bull dog +bear ish +vir uses +fri gh +lin den +we iser +sn t +gon a +dre sden +fl anders +cu k +wheel ing +ba u +atu esday +surf ers +swi ft +mc call +arbitr ation +aw d +mon c +b ine +at x +re fr +mi ro +po sey +n are +rit ter +âģ ¦ +play book +blow out +sports manship +s oooooo +malay alam +gri ms +bur bank +infin ity +sar gent +oit nb +joseph ine +ski pping +par kin +excur sion +semin ars +jo har +par tridge +post game +ll ll +blan che +temp ting +m na +lu ka +is ers +to ffee +bar ron +he mmings +sa e +go hawks +cu pid +li mbs +con se +un common +z ada +head shot +so ils +pione er +mam ma +sem itic +pan dey +jamiedor nan +spl its +vel a +son i +ra ff +t mobile +âŀ ĸ +pra wns +lit er +enjo yment +egg plant +tu b +cultur al +us ic +suspici on +sy cam +summ ed +ma du +ho ck +up wards +eye ing +ri ve +assas sins +âĤ ¬ +out fy +chi ves +t ner +la is +por ridge +sad dest +w cc +vick i +sna ils +biz italk +mill an +ðŁĮ į +sam oa +j ing +mi key +gu j +chel ms +eli gibility +arma da +thro p +surger ies +ãĤ ¿ +mo hawk +ex its +me m +is lington +c me +land fill +kait lyn +ðŁİ ¼ +combin ations +tomorrow land +ver b +cor a +pre cisely +na om +ðŁĨ ķ +shr ink +sof tly +merce de +mand el +poo dle +ball erina +sop h +jux ta +y at +ary an +hesit ate +lo wered +gu lar +dungeon sand +ron an +my ri +sp f +men opau +gra sp +pa thi +fe asi +fla w +shi story +ste ward +gg le +fay re +cli que +credi bility +yo g +sec tion +mu sko +se ville +no tt +cal m +mate o +indic ted +fi ba +by l +lin o +u kin +!! # +enig ma +siri us +bu sc +ðŁį Ĭ +mac kerel +psal ms +a at +tomorrow spaper +ðŁĺ ĸ +p fc +........ ... +shre k +mul let +o sh +danger ously +immen sely +am ur +ðŁį Ĥ +pro por +sy a +london marathon +abo ve +obli gatory +pro v +ra cha +alex is +pri mary +sh h +ether net +d stv +cou gar +un lucky +ni l +steak house +mel a +fc bayern +cause way +ca therine +fluore scent +nx t +to kyo +au sp +releg ation +qui zz +shored itch +proud tobe +promo s +inter acting +home brew +da esh +w pg +stead ily +provin ces +bal lots +i ah +al to +< << +you u +ri ley +prefe rence +tra verse +incen se +am munition +ho dges +# @ +hail state +tart an +witch craft +vent ilation +liber tarian +! âĢ¦ +ow es +% ! +ong chang +bru shing +le ic +fi ber +under attack +down load +ex pir +hy o +pompe y +mc bride +y ag +stre e +com bat +ten ding +ai ra +gug gen +ab ra +in na +fli ps +aw al +m ach +dol lar +inspir ations +z um +o du +it ty +video game +aqu aman +har u +bel fast +je b +but ch +us gs +calcu lus +go yal +mor gen +x finity +stand up +contrac ep +sab re +na be +in secure +gener ously +epit ome +l w +t ca +narr atives +don nell +pand as +ber gh +tu t +ker al +fel icity +br ampton +quinte t +nom ore +ðŁĶ ij +lo i +alham dulil +ðŁĶ¥ ðŁĶĹ +ston er +shaw l +clin ical +bren dan +gon e +fla wed +tri ppy +j g +al location +po aching +ve vo +mo cks +lef tist +bon uses +condem ned +abil ity +st ating +microbi ome +bio logist +for you +wahl berg +ss or +ift ar +w ul +ÑĦ оÑĤ +pom er +me me +ver te +tre ll +tra it +in let +hormon es +deliber ately +vill ar +battle ship +p bl +tw enti +ho kies +dal ail +say a +may fair +han s +die ts +⾨ ⾨ +od in +hot spur +pap i +k ana +k amp +fin na +flo tus +ti ans +unic orns +tribe ca +chang ers +fore ground +out a +inv aders +gett ys +tomorrowspaper stoday +mac millan +hand written +w fp +u de +state of +base d +âĺģ ï¸ı +cas m +psy ched +histor ians +fol d +d da +ag grav +p ans +green way +au sv +ðŁĺ ¶ +shradd ha +inde x +be sti +zim mer +t ness +eye shadow +ot te +go ts +distribu ting +pro min +yo l +ace a +tram rahim +hoo per +supre me +jam min +intu itive +quali fications +sli m +sid di +jay ne +tri pping +g tx +pun s +e manuel +om g +mid summer +in to +succul ent +ri en +new mexico +o or +hoo king +in f +ðŁ¤ Ŀ +flir ting +na hi +g friend +t ps +hel ix +z s +on ie +ct f +kri s +irresi stible +fla p +ðŁijıðŁı» ðŁijıðŁı» +us wnt +ru d +ram ps +pin oy +ot w +lol z +low ering +favor ite +t mc +phra ses +her mi +aver aging +em br +ben o +estu ary +sle eve +ribb ons +ta sh +ภ¹ +x f +aw gs +sun ited +brew eries +anir ud +pun ches +ol die +ip ads +wi fey +land lords +d ji +gun ner +íķ ´ +tex an +ex op +cas sandra +s off +ðŁļ « +igh ton +bak ers +awareness week +v all +ear p +bts bbmas +apologi zes +âļĵ ï¸ı +was ps +states man +snat ch +watch dog +ra fi +after party +spi ke +j er +peri ph +r nc +mu ll +le en +shi es +li eu +urstruly mahesh +mer ton +de sai +shi f +ðŁĮ ± +pe dic +gos ling +arrang ing +ww g +gen y +you uu +netfli x +e ttes +k wi +bernar dino +am iga +Ø ¨ +kashmir i +t ings +emer itus +de cat +ab domin +dc i +pha ses +d jan +be am +op ry +i shed +the ellenshow +the st +habit ats +to ons +mclau ghlin +ri pper +micro biology +tal aga +clu eless +ss u +cro che +bro mance +longe vity +zagre b +prev ented +tra ve +spo ilt +darry l +migra ine +al cat +dd dd +vi v +ser pent +mat tel +jam a +con quest +î Ħ +sam sung +presbyter ian +ket ch +fire fox +mo tif +le c +cho pping +cher no +j ann +ðŁIJ ° +pro lon +wake up +conver gence +mersey side +heart broken +lo oming +hal lucin +mai ze +commun ism +mo h +twitter storians +serge y +res eller +favor able +ed gy +re iter +mal aga +live me +ka hn +pul sion +big g +kim kardashian +ati o +tyr anny +ru ption +q ant +pro ven +by z +pu shaw +kri stin +e er +tar dis +ri z +awak en +mi ko +un documented +path finder +indirec t +resemb les +h ler +conce aled +scand al +re im +d nb +cr itters +attend ant +apprentice ships +aa u +scre amed +l su +fa h +har bour +ed d +bat sman +li ss +mi sha +spani el +it f +advan cement +fa c +close up +cecil ia +medi c +narcis si +lav ish +gi ac +ma ys +le it +wine wednesday +pushaw ard +let to +curren ts +bug atti +out ine +w j +un do +ler osis +devo tional +ðŁij « +on na +fais al +sa una +himach al +am ii +à® ® +di zzy +screen writing +ph x +sp n +ick i +ag irl +fi shes +wb z +pi m +bo ar +ac id +! .. +rocke feller +n ga +dra stically +simpli fy +dru mming +autum nal +gur mee +lor de +jo ann +give up +b our +am ura +der land +sim pler +wat son +tri dent +concor dia +bel lum +bre k +dum plings +vi on +dungeonsand dragons +sp ri +ascen sion +wil datlantic +u st +rob ins +legi on +insi st +jar o +gue ss +so b +bigh it +pool side +negoti ating +mc gill +bil d +techn icians +miti gation +ajay devgn +b to +ant en +cosmo politan +ðŁĺĬðŁĺĬ ðŁĺĬðŁĺĬ +patri oti +temp er +promen ade +nav ajo +nam m +wrink les +dc fc +le ach +bru nette +r f +cout inho +al ti +tradition ally +op tome +na z +accord ingly +rec ard +de ets +sw ell +po sure +whit ening +strang er +illi on +here ford +u wu +ro bber +cotsw olds +cl en +gor ge +nam aste +re lish +gri ff +adren aline +bla sio +val e +ê ² +toler ate +rail minindia +jen sen +ho ven +el lu +ob sole +eisen hower +unidenti fied +than niversary +body guard +Ø ¯ +i dge +sch al +stock port +sn i +re taining +po po +pix ie +oli thic +ki er +ha jj +sa z +cor bin +!!!! !!!!!! +v it +me gat +de h +circu it +af fleck +theore tical +hope less +u ab +slu mp +b ice +jam med +let stalk +can i +side ways +labyrin th +re fs +ha hn +jare d +ðŁį ¹ +jam bo +ph yl +enhan cement +c tr +ful lest +se ye +do ba +cho ic +yo s +cb j +andr é +re watch +pri ma +doctr ine +for gets +u hm +ar ound +u le +art lovers +shi raz +har th +ex tor +Å ¡ +unexpec tedly +eli us +y x +em my +se ac +ðŁijĩðŁijĩ ðŁijĩ +correc ted +com bu +wom anc +cou gh +what son +publi shes +divers ity +back bone +lock down +mesmeri zing +nor te +ma b +desig ner +í ģ +ra gh +mole cules +get outside +the beatles +semicon duc +nach o +lun es +ham mers +sul tan +o on +fe ren +att ach +ar qu +uttarak hand +s ash +; - +tre ad +i ko +ar thur +scandin avian +r ation +ga el +charge able +fish y +v ma +hand bags +char a +ay ne +de fam +sett lers +qad ri +pal ais +in wx +apocaly ptic +poo ja +a es +at ories +proof ing +n lp +ts la +v ina +li do +dee phouse +informat ics +v v +pp ings +di ss +à ¯ +uhur u +st ony +betra yed +b aff +my ra +as pen +allow ance +tam ara +ci f +cor bett +ser ge +di go +ambi gu +pain ters +p cr +p ca +nom s +lo ft +ve e +opend ata +ðŁIJ ± +alex andre +identi fies +fantasy football +re production +brom ley +ware agle +mm er +p ss +cu es +ay at +hut chinson +sar ac +jack man +ira h +ap ink +col s +aussi es +ex ecs +day ton +ðŁĻ Ĩ +im v +har am +chuck le +authent icity +ar do +incub ator +ภª +photo shopped +embrac ed +fight for +gor man +zz zz +schol astic +cri sps +te apo +mid night +ga ine +col lier +s ate +de tte +å Ń +imag ine +i ff +tw ili +i fication +teat ro +nor ma +es ur +emergen cies +rise up +r inger +hass le +cait lyn +tranqu il +vers a +se b +over look +gin i +bo go +se re +may ne +henri k +contamin ated +rhapso dy +pro portion +wildatlantic way +âģ© . +organis ers +tran e +stand ard +sper m +laun cher +ric ci +her ts +paper work +showcas ed +mer yl +pen a +p imp +disa strous +^. ^ +phar a +x is +fron tal +sw irl +sp ills +swag ger +smart watch +sizz ling +savi our +cat ar +bb cr +refurbi shment +dr is +citro en +absor b +patrioti sm +il leg +chro mo +fresh ers +ru s +lim iting +ef ish +down ed +man dir +hazel nut +p all +mac on +disappear ing +quali fies +bo on +bar racks +am ine +gen dere +ðŁļ ĺ +j es +ãĥ Ń +qu ito +middle weight +sch au +quad ru +aci ones +limit less +ðŁijĮ ðŁı½ +ch man +ar av +regulat ors +it up +batter sea +mil ford +g z +tic king +gh ou +cru shes +tu tu +dread ful +fam ine +for change +dalail ama +ðŁĴ į +whit aker +hash mi +h us +vo d +bet te +aa ah +iso o +ðŁ¥ Ī +ha ar +la ine +b v +all day +spr out +indie games +free bie +gree ks +but ler +ill in +ha al +ware ness +si ma +public health +gam a +wa a +oun g +goo oo +okin awa +off enders +im pose +ho c +young ster +story teller +sc ap +figh ter ++ , +whit es +music monday +re za +go ducks +bri a +mi um +cas per +cru mbs +a ad +marti alarts +ch p +ri gged +tn g +harve sted +sa k +do jo +mill wall +b nw +oc d +histor yof +t mr +si rens +fan ci +caregi vers +vir a +son i +recur ring +acknowle dged +ðŁı Ł +oph ile +bu cky +stre ssing +roo k +di gger +vi val +san do +fle et +si ers +sel caday +refre shed +anti fa +a que +po lo +disappear ance +de mb +âĮļ ï¸ı +ren ted +ber ger +g mb +cu la +ss al +goo dy +u hh +marcel o +w anna +soft ware +shop small +turt le +tom as +fri sco +ðŁĺį ðŁĴķ +jim enez +c su +day z +an do +wyn ne +choreo grapher +cerv ical +trail blazers +ed g +zend aya +travel blog +el s +whole some +co g +lab out +ar ney +del le +su isse +ma si +ine se +om be +fi ddle +re claim +pa u +wat cher +sla in +ber ty +opti mum +el ites +min is +tur key +patro ls +ger ard +au reli +wild ly +wal tz +br gy +w ob +cre st ++ ++ +ve z +fro sted +davi do +the x +param edics +p into +han k +du pont +ur g +fo stering +micro poetry +spec tre +---- > +ne uro +fri da +music al +galve ston +e ffic +sc ape +pal azzo +th all +pro visional +p js +au re +ðŁĶ ľ +mam amoo +kit ties +cre e +wa k +lo ool +lu pus +cn blue +à º +ðŁİ ¬ +rac ed +tro se +om as +stri de +co ors +⤠µï¸ı +in comparable +cy ril +broad er +arec lipse +ðŁį Ķ +inter val +ti ru +co working +w aco +a ham +a bee +flouri sh +the times +ol ini +kick boxing +lu cer +at la +as un +casser ole +mi aw +lobb ying +jan ice +cir que +re flex +le ary +sanat omy +tem pest +se mb +mur dering +us av +ro bo +on et +p cc +nati ves +life of +sa ha +ruth less +rel ates +appeti zer +pye ongchang +nor d +er u +a thing +ug ly +pl ying +bran ce +organ ise +kend ra +dat o +chees es +par ma +burn out +a stra +pre toria +adjust ment +uk u +sl o +li ken +fav ors +cli ve +be ets +snow donia +go tv +sy n +open house +pan i +portra yed +sl ated +me cca +ren al +supportsmall streamers +staf fs +da o +bi ker +vik tor +tit us +admi red +ðŁĵ ± +hurric an +he ats +gl ory +photo genic +mer i +de por +burn ham +or angu +dj ing +impre ssionism +ign ition +ca i +w ynn +de pe +cove ted +colla gen +sau s +or nam +administr ators +ss on +nh politics +hahahaha hahahaha +aspir ations +r gb +swol len +so we +sc r +diver gent +hou ghton +han oi +d ory +ni ki +land ry +b cci +ðŁijĮ ðŁijĮ +is mail +tri pod +her d +bhat t +dress age +tab by +ingu ish +hur on +à³ į +à ł +to das +evangel ical +chor ds +st john +slo ppy +marty r +face book +ali ght +sen sei +kath niel +r ites +zi one +u o +revel ations +weight lifting +pan o +nc wx +ac ton +à® ķ +Ø ² +som a +à¸ Ĺ +respec ting +mar che +fore man +be tty +ki k +shi bu +po on +argy le +k swx +et z +mar bella +brac kets +stand by +fire side +defi ance +v ex +britanni a +in habit +appo int +piyu sh +le ash +sci ento +fla sk +sen na +> : +at roc +sand erson +id lib +dhan ush +ðŁĺ Ļ +en thr +hit ch +de dly +al ley +dor k +mon do +cudd ly +mis sin +ye sss +night ing +j pn +w ary +ump ire +ma z +ê ³ +bab s +ĭ ãģ +stan ford +posse ssed +exce eded +ðŁĶ ¶ +wall art +tra p +j il +hi bis +sp ying +scri be +khali l +trans lator +lu mb +di zed +ch c +super vision +shut ter +ja g +_ * +yester days +ms f +hi hi +gonz aga +gille spie +vive k +ec static +this morning +ch us +ed es +ston ed +be es +ðŁĩ¹ ðŁĩ +tur in +ho ver +at rics +ster n +sam heughan +auti sm +mi ya +eye witness +writ ings +travel tips +chut ney +px rtg +keny ans +my stic +k rit +/ $ +red head +world ly +am us +op la +le ve +gab bana +se en +o clock +gang a +keen an +sc ent +ol dies +go green +corner stone +comp ly +con cours +ðŁİ¶ ðŁİ¶ +ha an +con fis +aw son +cle op +î Ģ +su zu +sau té +al gar +subscri ber +este emed +ãĤ¤ ãĥ +worth while +mel rose +flo ck +bri ghtly +viol inist +p ere +sli pping +and co +si gh +ha van +cu lo +m sa +fibro sis +matil da +ra fting +aw ard +ë ª +mm mm +ge aux +ste iner +sin n +help ers +beet les +ai mee +tai wan +pistachi o +mac beth +m zan +descend ants +on sale +in r +il m +grou se +sa ig +mo w +bi gre +adjust ments +tu la +mathe w +transl ates +mu h +bol lah +ðŁĴĽ ðŁĴĻ +amo res +ab outs +bomb shell +bla ster +x avi +s ns +k roger +ga ther +erad ic +daf t +chem o +ben ches +ðŁĩ© ðŁĩ +ut v +our a +n ko +gator ade +biaf ra +ok state +im danielpadilla +dom ains +open ingday +kid do +do i +ric e +day care +mac millan +ba thurst +cheer leading +ðŁ¦ ģ +cash back +k won +hob bies +exem pl +ries ling +âļ ª +ag les +ny s +every thing +nav is +ad di +magne sium +faceli ft +ark ham +grand es +extre mist +don at +vit ality +pump kin +be tta +sl td +arti san +li by +pe aked +ah hhhh +mary am +assi m +un sc +ment e +al aya +low ers +ar as +gri ev +le ip +gr ati +cri ses +spr ints +exe cute +w to +ms d +mag ical +re viewer +spark les +juke box +ðŁĺĤ âĿ¤ï¸ı +pay back +licen ses +dun kin +bel t +lake wood +h ateful +bud gets +rev amped +ph erson +ky iv +went worth +ro sen +cru ise +gi ggle +def star +assassin scre +ym outh +win kle +w fc +band wagon +b kk +w iring +kear ney +south side +pe tit +! ðŁĺį +nor dic +mir za +mu gabe +v l +scon es +k tv +sand al +du c +m alls +ðŁĴŀ ðŁĴŀ +it c +al ay +im pair +un rest +flo ss +c é +ab ou +var ying +muse o +ser ver +di ya +hibis cus +ero y +mer ritt +fin dom +f pp +un usually +go tt +conting ent +ali aa +ball on +jo l +hi ked +zy me +ay r +ag n +ga z +perio dic +spar ty +practi sing +lin ton +tal is +cy pri +womanin biz +radio disney +ðŁĮ ¼ +jump ers +endo cr +ðŁļ¨ ðŁļ¨ +and on +shar apo +mi er +ma sonic +fac tories +vi en +bb ers +ìĽ IJ +hol d +ke bab +be ak +approach ed +ac milan +mun ro +ko sher +excell ency +negoti ation +walt disneyworld +cr ouch +te asing +suppre ssion +en ya +b ce +transformation tuesday +cal lie +vis was +p gat +ic ted +end ings +esc u +recru ited +it fc +collabor ations +g ino +snu ck +ausch witz +i fc +x ii +ke sha +ger vais +clo ak +x l +sa ad +prob ation +pre cau +mac in +anasta si +le k +e azy +daysof code +mariah carey +yo g +stit ched +boy friends +sh ar +ph ile +ag u +twin kle +phi shing +week ender +ic ton +gurmee tramrahim +al ton +l eness +all an +pen ultimate +kry stal +go u +lan de +dis mant +ab using +nor se +pat erson +ed mun +ap an +xi umin +sk el +cat walk +re act +wal led +t angle +br yn +ve to +super moon +cas ablanc +appreci ates +ski d +bo th +catal ina +ele ague +cyber monday +cau tious +ðŁ¤ ĵ +nov o +hamp ton +ha ye +jose f +var an +lo bos +roano ke +orph ans +tt in +squ ads +ishqba aaz +black panther +e tu +k sh +cru mble +cess na +reli eved +scul ly +pollin ators +explore canada +ki es +kam loops +kir an +pri mal +sett lements +hot spot +brain storming +ce dric +bi ennial +sh ant +âĻ¡âĻ¡ âĻ¡ +do on +hear n +walk way +fe m +ve al +deport ation +tox ins +elimin ating +descen ding +by the +bla sphe +ha sta +comple ment +as cent +ri ga +provo st +âĸ ª +wee ping +anti semitism +employe e +unearth ed +pin o +natali e +bla d +ang ola +lock heed +in ian +ag r +ni ster +im pala +m ke +fan atic +âĺħ âĺħ +ðŁij ¸ +lu ch +simpli fied +gall ery +econom ic +cy borg +con i +sel ma +in ception +ko ala +dv ds +cre sted +m mor +visi ble +n sd +ðŁĻĮ ðŁı½ +w under +refriger ator +re opening +e era +carou sel +as p +balli stic +victor y +mo tive +tre y +sharapo va +si i +mon ter +int end +west chester +sp e +cy mb +vi dal +ll ama +uni v +fin er +crafts manship +jazz fest +b ch +ag gio +n cc +lamb da +tranqu ility +cis co +ba den +so bbing +of i +go ta +ru mored +war med +ore an +ac ton +mar ci +gh ani +âľ ĵ +as sorted +pembro ke +pen elope +da f +at ty +aim o +pretz el +carni val +than os +ko chi +mer sal +ham radio +ar twit +cas c +guer rilla +kush ner +k app +al ise +todd lers +steward ship +o tti +ter ri +tem pe +rest less +vit o +zay ed +rsp b +pi on +hi ppo +haw thorne +in as +am ily +nut cracker +lo p +d ali +tro pic +ðŁ¤ ł +ul o +jare dle +py rene +pale o +usa ir +m ould +it ated +gene tically +biom ass +ðŁĩ³ðŁĩ ± +do dd +practic ed +monarch s +un manned +m buhari +am al +photo gra +ko ol +bren don +ju ices +cu re +world bank +poin ters +ðŁĴ Ŀ +tur f +le ds +bor ussia +bapti sm +warwick shire +moun ts +gay o +be gg +co pied +asi ans +k g +moder nist +gi d +front man +concentr ated +y t +sc avenger +iron ically +adi c +ps n +ðŁ¥ ī +cultur ally +yu v +mac arthur +fertili zer +be withyou +ri gor +min ors +z oning +âĸ ł +ri r +adole scent +vin ny +ren g +sand stone +gu et +we sth +ple dged +lac ed +sp ide +v ai +ty coon +seiz ure +du p +appalach ian +ro k +cathol ics +sey chel +posse ss +la ger +jo di +cham p +stra s +d ina +cent uri +cal der +blur ay +ðŁĩ¨ðŁĩ ³ +mo do +an nette +youtu bers +chap s +ang ling +label ing +a qui +pk wy +ly le +bi sexual +lit ur +dug out +li bby +grey sanatomy +sub stances +august us +rall ying +fi del +ing ue +äº º +hallmark channel +tooth brush +m á +adi rond +ag gi +ðŁĵį : +cru sade +tax ation +k z +i ver +dou bling +room ie +wa b +en rolled +az on +a ju +grand children +as df +ðŁ¥ º +mat ic +ough ton +utili ze +ðŁĴ £ +pon der +rais in +dys function +co bain +butter nut +e man +su red +dri an +and friends +with the +on omy +heine ken +bri dal +leader ship +pyram ids +deutsch land +jo cel +bo wel +y qr +horse power +be acon +ing eni +gra dient +fer mented +mo om +thing y +pot assi +wrist band +bor d +bo died +ðŁĺŃ ðŁĺį +ma pp +ka u +cyber punk +ph ish +loo king +co ates +ap ur +am ie +uk labour +at in +g la +adop table +shel by +v illi +ri ya +m ingly +cli mber +bumble bee +ðŁĺ ¸ +c sd +âĿ ¥ +hospit alized +c ki +hat er +ch r +re tina +it a +fan base +beat rice +gwy ne +go ss +fo s +favor ited +swachhb harat +mal ade +mon mouth +" [ +si van +sh hh +command ing +sains burys +wee d +g man +ss w +rep tile +iv y +tro pics +roll ers +over cast +ex position +masquer ade +man crush +wa ist +spr inter +sle et +le vin +j pg +_ ( +o pel +explo it +ap a +po we +wrec king +jong in +or b +er ick +bo sco +pra ising +ber tr +to wing +in security +ku t +resto cked +rr p +prescri bed +trafal gar +per t +g ases +app rais +g har +music als +âĸ¬ âĸ¬ +mc fad +ag ony +conditi on +equi p +shi k +atra vel +ðŁĩ¿ ðŁĩ¦ +ke h +abduc tion +pe oria +wil kins +g ms +as d +ev i +ðŁĴĹ ðŁĴĹðŁĴĹ +u z +mo c +halle lujah +guad alu +lou vre +dra wing +go ve +ph ant +fri e +web dev +program mer +z able +games com +clari fy +li th +kin ky +âĿ £ +labour doorstep +son ata +ju ris +mai den +vi adu +buch arest +conditi oned +capit alist +u de +ps b +sp ca +lul la +footh ills +kay o +bon d +wom b +roun der +ce sar +bur sts +ap ra +sw oon +sab rin +fra grant +cle arer +ku brick +cli max +jour no +ag le +ðŁı½ âĢįâĻĢï¸ı +poo ch +hal e +sol it +sal mon +organis ms +bron son +art en +hodg son +alo ve +vent ure +bb i +ae a +ðŁIJ ¢ +ld n +d nr +o zone +el las +man ny +azz ur +un beat +tru ffles +th ong +ma ñ +las ers +ley e +gettys burg +back packs +or is +ma ison +craw ling +la bra +cl ing +dra gging +ste al +dou bt +de van +ck ers +agent sof +photo bomb +elon musk +abo y +dist ances +story line +sp i +nor than +europe ans +wh ale +ser pent +ðŁļ ² +fi or +tr it +ox o +awar ding +class mate +su fc +smar test +rich es +pr k +big foot +ar mb +bi polar +dw elling +om ars +k wan +gri me +m eng +freder ick +navar ro +sorry notsorry +jaredle to +pa ve +sl ack +barn sley +att ar +evic tion +accumul ation +o ir +cat chy +wel ter +vik as +has see +nik ita +mo yes +mathe ws +shi v +gat wick +pro filing +compan ions +mar rake +an tics +ðŁĻĮðŁĻĮ ðŁĻĮ +se se +bo i +bart lett +poison ous +ab uses +ym m +kam pala +guggen heim +imv kohli +dol om +bre e +thro ttle +gare th +fitz patrick +un ya +par ad +mar got +j nr +we a +potassi um +p nc +disgu ised +cra sh +ren ergy +ill ic +coup led +ni els +ci ones +æĹ ¥ +im ent +despic able +d ye +what cha +conne ctions +paralym pics +gaunt let +wait rose +suici dal +star ship +vap or +st ou +law maker +coo led +si mo +then o +offro ad +ja den +bas que +vick y +lu kaku +centr o +tri sh +strate gist +medic ations +hor st +b fc +gra il +sharp ly +ad itya +tom b +kau fman +tri pad +sam ba +pastor al +brit ney +sag an +hill side +mas ons +sar a +z one +x u +to tes +rob bie +app en +mon tag +der o +short film +charis matic +tat ors +ki ba +and ri +al arming +split ting +ic ar +th ug +scari est +sylve ster +an an +u trecht +a difference +me ade +bu ster +air strikes +cu ffs +account ants +ðŁĺ¡ ðŁĺ¡ +new t +bo tt +issu ing +cl ancy +wwen etwork +kyu hyun +rese mble +pajam as +sin k +kin ney +sul ph +or k +li es +la gh +or ton +ra hul +d sc +we will +re am +collo qui +shar ia +hec tic +sar casm +land er +tm z +endor f +ro z +ham mered +fri s +w adi +pope francis +he it +flash light +un born +op es +hol iness +ðŁIJ ¦ +nach t +im sa +gr acing +bj p +ver ts +c sc +home owner +a que +bigo try +anni e +bag h +âĿ¤ï¸ı ðŁĺį +car i +thom p +dispo sable +cardio logy +pat ented +hh hhhh +ld r +stephen son +cro res +fan ning +cli mat +ðŁijį ðŁijįðŁijį +ðŁijį ðŁı¼ +aer on +piccad illy +bank rupt +sil via +emplo y +don ny +commen ting +screen writer +io ta +ce an +anc ers +tu an +street wear +ठ¯ +sk ine +esp a +asi f +os ce +she ppard +more cam +bott le +der s +orac le +google play +aver aged +edmon ton +steph an +sister hood +cru sted +stag gering +methodo logy +congress woman +c abo +tri ggers +mil ky +gli de +tooth paste +room mates +nu ff +gu am +sprink les +alternati ve +wat fordfc +uof t +hal ey +cont acted +bun dy +pro stitu +gh ar +pre ston +on site +hil ar +g ts +c att +hamp stead +? ?! +ðŁĩ§ ðŁĩ +bbc qt +aless andro +resi st +ma idan +t ko +shad ing +pin up +gal lo +sin u +at ec +fun k +ac lu +stri des +rhy me +wet land +bbc springwatch +t ins +wild card +st our +flamen co +pau la +onto logy +gang sta +am ade +ãĤ « +t bs +skelet al +run ner +jard in +harri er +hun ted +z hen +believein film +de mean +au diti +re start +chon dri +âĿ¤ï¸ı ðŁĴĻ +mcla ren +ga b +sh um +au sa +lewi sham +y pg +k jv +fur nished +dor o +bon ded +mor ty +lat itude +_ ) +lo va +water ways +vin ai +shor th +drun k +c ay +ay ana +kap lan +capp uccino +spr o +life boat +has bro +spol ice +tor on +do ing +dam n +sh ree +foun tains +ent ation +mar u +boar der +to pless +j ada +chan ning +ul ls +en closure +gib son +fractu red +brit ton +à ¶ +t ous +por th +dra f +tra iling +mar gate +eli fe +down ward +lin n +gla des +girl power +ak rish +u ki +ron da +ts c +appreci ationday +vis ing +lo om +ðŁį ³ +mex ican +ar gos +y ya +jad ine +south port +d end +si sta +rede em +men g +bra xton +antioxid ant +s key +mp g +fin ding +vibr ation +ce u +kh art +di mini +cl ine +shel ly +hin es +ī ï¸ı +to pical +no ver +ma xx +prim itive +illustr ate +b ounds +tren ton +join tly +breed ers +u chi +wakeup america +b ada +ðŁĹ £ï¸ı +gu acam +sp heres +pere gr +youth ful +lo lo +bir min +t ly +jeremy corbyn +defe cts +co sm +a rent +v aa +bag els +medi ac +cori ander +ic ago +g haz +ab bas +re model +struc turing +pu m +out law +ad ani +r bc +gul ls +n li +confu se +ðŁijĩ ðŁı¼ +vil a +mcnam ara +correc tions +mug hal +ser i +re gain +ss b +lea ve +haha hah +gran de +di stressed +re chargeable +ho a +hou sed +sti l +attribu ted +opath ic +di ps +pri t +head phone +conclu de +pil o +he t +ut sa +nit in +je m +sni ppet +tutor ing +op er +sun k +en sla +cha u +ac orn +quinte ss +ran kin +affili ated +our lives +cl int +se ater +isa ac +ba shing +sme ar +nur se +doo dling +" ; +sa ku +atroc ities +im am +g fs +viol ating +comm end +brad shaw +er ville +b illed +b be +thul hu +i phones +moo se +di os +re w +me thane +strang ely +whis ky +ti ghtly +spiel berg +radi us +notic ing +wi f +ig nati +i fa +ap is +w ali +ha itian +bu shes +y z +v l +ex ited +asse l +tru ec +dom en +ash er +in king +newyear seve +hend ricks +bat i +ìĿ´ ì +rich ter +mon santo +con line +agre at +ðŁ¤ ¯ +master pieces +ar n +rough s +cle ve +se v +fashi ons +to ya +sh ail +cop eland +aqu ari +dec als +are you +y aya +a str +fon t +ml m +ar ca +pp or +pol lock +xper ia +conserv ation +chain saw +ag gie +?! ?!? +si le +sh on +ìĹ IJ +note books +marque tte +de us +bb led +spic er +mc cabe +nor wich +modi fication +boo sted +stru m +sales man +bang le +nis san +hez bollah +brea sts +a af +anth us +sk er +ow ed +her os +gi fs +fo sters +eat ers +du es +_ / +lymph oma +sf am +me gal +afri di +ag ic +p amp +jeal ousy +ðŁijĮ ðŁı¼ +calcul ate +napp ing +g ale +ðŁ¦ Ħ +lub bock +assu med +ren ting +íĥ ľ +subur b +ãĤ · +tech nic +u cla +in front +gar net +ster oids +stri ving +ho war +mo ver +le ton +bull do +is in +ci ao +sn z +fore front +d ams +mid wife +ma wards +cla pton +we in +subsi dies +spr oud +rother ham +phan tom +ar ach +spi el +rac ket +sel amat +no on +l bc +enti ally +ðŁĴ ¸ +sil ve +m oud +kine tic +y asi +ðŁİ © +o ol +mi ku +i za +fer a +flo ren +barber shop +groo t +z est +ne ars +stan is +z and +police man +juris dic +form ations +appar atus +sp d +arti fact +to sc +motiv ating +womanc rush +re dro +diagno stics +ra za +out fitters +el xn +dod gy +ry n +sh d +ortho don +ol de +jay anti +bal ances +quic kest +can ton +friday reads +! * +na a +a ak +ðŁĶ · +behavi ors +rasp berries +ä » +polit ical +cam il +å ľ +di k +ast ounding +lie be +novel ty +tur moil +sul ly +spring break +hon ouring +cc g +ðŁı Ĵ +my little +ky c +pro ms +ðŁķ Ĭ +à ¨ +bi ge +av ril +ðŁĩµðŁĩ ° +mari on +as ants +sur ya +oc tag +luf than +ac ron +fayette ville +ti que +love s +en ca +de kalb +ta ver +de vote +aux iliary +joh annes +tread mill +ay an +qu r +donald son +cher yl +" .... +s ven +kir sty +gun ners +ra dish +o ahu +v sky +i ble +con course +b ps +elo qu +ash ford +te bow +roblo x +ma da +dri ving +th day +spro ject +m ms +band ed +. !! +libr arians +flan nel +intoler ance +her al +ç µ +neme sis +list a +tar ak +cry pt +star plus +vish nu +sc ale +cr is +% ), +j illian +regg ae +pegas us +ol in +ip ment +man ic +l fc +godd ard +ite am +parl our +anch ors +lee minho +talla hassee +ant it +d ho +kid ney +y ash +batt led +az ad +gar is +faul kner +sni ff +papar azzi +ed m +phy llis +con tested +aa ay +se ca +k ton +vel ve +rain ier +for um +tam pab +ho sp +trac tors +ox fordshire +no tion +guang zhou +ðŁĺ ¯ +ref ill +wednesday motivation +sli der +mukher jee +pr att +fon taine +alph on +af ar +ts i +pest icides +fi ends +mo cking +bra w +tran sat +do ses +co res +hom ophobia +docu menting +zlat an +con doms +s é +sun set +kun st +ton ga +ภª +v ation +sp ray +chow der +ra ps +palla dium +nor wood +music history +hoo ker +si si +osp rey +ph ys +conce ded +bob cat +ar mad +ze it +Ù Ħ +ðŁĺģ ðŁĺģ +mer idi +ðŁĩ· ðŁĩº +corn wall +! ), +touch downs +ze it +chal et +mm m +al che +gor illa +fo ss +ati ku +lumin ous +ivan ka +be ek +sta res +sw iss +âĿ¤âĿ¤ âĿ¤âĿ¤ +scru bs +me ath +gusta v +jo gging +confe tti +as os +ers fc +breit bart +applic able +autho red +ya ho +h in +displac ement +j v +ðŁĮ¹ ðŁĮ¹ +ot c +non profits +diec ast +gu sto +inte stin +c ages +me en +lu kas +moon ey +ðŁĺ · +very day +tor ah +is sion +wa c +lever aging +ish able +cu se +le wood +may an +turn table +ju ice +tru sty +tu p +eti quette +supervis ors +stu n +gu zman +confe ren +ric o +fe ast +back ward +pol aris +mic he +jo g +h ing +field house +vel ing +sho cker +esc ence +ठ¾ +vi be +anasta sia +mar ched +kill ing +Ķ ë +fe tt +exop lan +... ( +snow day +lo h +ir ani +la khs +del a +po caly +boom ers +dictat orship +ac er +tur keys +quarter final +muskete ers +ðŁĴĽ ðŁĴļ +sf x +museum week +sc ala +ri sis +( ðŁĵ· +ãĢ Ĥ +z ies +bo eh +hu es +lu sci +dol a +impeach trump +roo d +don caster +tor re +hero es +fo yer +tar i +blur red +ke w +frank ly +dro id +ap al +Ð ¼ +y af +bre t +par agu +cac ao +ðŁĻĮ ðŁı¾ +ru e +head aches +shaw ty +char ley +pal er +go wns +correc tional +ðŁĺ© ðŁĺ© +breaking bad +ol ing +da p +endeav our +cit adel +tra d +incumb ent +medit ate +foo ted +ðŁĴ µ +shab bat +dayof the +wil lem +gal way +to red +marri age +f illion +sleeve less +aud itor +jin young +invin cible +kad una +a and +volcan oes +mon eti +indie gogo +buccane ers +ðŁijī ðŁı½ +ãĢ Ĥ +lay ton +cuck oo +hu mber +buzz er +Ï ī +to re +stra ins +sto m +pa ine +s we +du ff +z ou +si mi +li pp +ur n +se agu +ðŁĶ ® +sun dae +hi c +ðŁĺ ¨ +bull pen +u per +flyo ver +al dridge +glo bes +ali es +ken zie +ge es +y cle +sp lin +mag enta +j ha +bal u +gh orn +ti pper +wick er +taste of +con clave +ch ale +inv asi +cat er +dio xide +me gab +win n +at p +transform ative +nest led +hi g +bri dging +lil ies +chee red +bad dest +sc rolls +real is +dipl o +ðŁĶ « +conce ssion +prefe rences +explo des +er gon +introduc tory +ine au +ch af +som es +land rover +spir ation +sex y +sco recard +illustr ates +soul mate +wi en +inter disciplinary +fore casting +ent ities +glu ed +en lar +cur t +percep tions +boot leg +mi re +asho k +v az +hor ne +cal le +ac ulture +ther oy +night time +oc al +character design +ar mist +ðŁĺı ðŁĺı +yah oo +ac eae +to se +even to +sou t +nay anth +wh om +v are +ri gging +gen us +hi ve +com mands +sti e +day a +ethan ol +en f +hi fi +flu ence +cle mson +re invent +thermom eter +humor ous +emer ging +aci ón +ðŁĺĺ ðŁĺį +s ity +haw ke +accompan ying +t ility +ðŁĺ ª +re cess +protag onist +l ery +dun dal +int l +britt any +q bs +off the +marri ages +how to +viol ated +adel aide +wit t +lanc er +pak v +hu me +st ade +bra gging +ou tright +ad c +super st +real time +cu res +garden ers +ero ck +dale jr +ver o +bar tol +mo ti +mc fly +v pn +st ink +over rated +guer ra +e tis +ath ome +twd family +th ab +tn x +rafa el +family travel +x ley +sat anic +equ ations +ru dy +wal dorf +stan i +tu be +meas les +zimmer man +obli gations +i ously +bow ser +trans former +sho ppe +shak en +gh ouse +to d +ke tball +share holder +mar ca +kp mg +ak an +given chy +coast al +au th +roller coaster +mar ches +coordin ate +cine ma +apprentic es +par lor +mit o +men on +consider able +bar re +glo ss +enh ances +jaz eera +fal mouth +thra sh +stat en +k zn +eng el +samanth ap +flo ppy +sal om +ðŁıĨ ðŁıĨ +w ack +deliber ate +osc ill +herit ag +du sted +orni thology +pad dle +fer ns +bar un +cl ans +anticip ate +a ay +mat ically +é ĩ +tu mble +post man +unic ef +tro tter +op d +leaf let +ge ist +cease fire +scre ws +cre ation +wal nuts +longh orns +under statement +ab b +proxim ity +na x +un ity +turn pike +orda ined +dub step +chak ra +me ch +love her +look alike +donne in +vir on +Ù Ī +bang ers +vari ants +out dated +in ta +cri sto +sp elt +food and +f on +stefan i +margin al +hu tton +ti ara +tel ford +qu en +fair grounds +que tta +mikha il +heal er +v ball +ty re +under grad +gl end +hom ers +scri bed +main tains +po che +mis sal +mar ko +u as +á n +sh p +con vey +pad re +sab a +pu glia +madhu ri +pa xton +chap lain +n ago +ca si +... !!! +fli rt +sal eh +k are +di re +stam ped +extre me +ðŁĺĥ ðŁĺĥ +ho ppy +guadalu pe +advant aged +eu char +p low +un n +mac qu +port land +cla sh +pe s +lou bout +y p +keep ing +arca dia +fran kie +fi u +de th +encyclo pedia +si ze +inve sts +ðŁį © +geo logical +fran ç +con front +ðŁĺ ¥ +d ys +af m +tex an +graph ene +repost app +ac f +ur sula +gaz a +dd led +fu m +wsb tv +m be +fron tiers +chrono graph +ke s +inter faith +tab oo +spar ta +won do +flori st +em braces +ca w +no el +arch ers +ðŁIJ · +roman o +ban an +sh akers +melo dies +geo thermal +se phora +ìļ ° +оР´ +pro c +hand shake +pan de +popul ated +slow down +hor tons +registr ations +un deni +lan ts +pas sover +thak ur +li ef +adhe sive +pe tal +micro scopy +memph is +confir ming +air drop +mesm er +perce ived +ming le +lifel ine +gh j +worcester shire +pas sions +ach er +el lar +ah o +firen ze +bar ang +letter man +hat field +lu cha +je ter +e shop +william s +horo scope +pre de +east bourne +dur ga +di version +al trin +seis mic +premi osm +nar co +ti r +ori g +or m +land fall +ci ous +lin do +max ine +x ico +tra y +os wald +c ba +ric otta +n cr +mar au +ภ² +gladi ator +ch ery +lun g +u me +po psic +lon ging +can als +ta ya +decentr alized +sho pp +pres sures +mahar aj +eti had +wal greens +succe ssion +sign aling +li g +staf fer +north korea +def ying +as ma +de g +peri meter +oak ville +m sk +balti more +rece ip +de ple +ðŁĺŃ ðŁĺĤ +jambo ree +> .< +rsp b +puni sher +consider ably +in tothe +pari sian +acceler ated +polye ster +low es +fr ying +sauté ed +mou ths +seychel les +ra x +go dis +dak ota +house wives +the me +mat inee +black bird +ye sung +pre fers +pelle gr +in ated +trun ks +stronger together +re pet +re pairing +ped als +toler ant +her r +dun ne +indic ation +decat ur +b tv +exhibit ors +ik on +friday motivation +bra gg +live tweet +al ves +womens art +foreig ners +wal lets +min dy +lan ey +bb in +tv miaw +lif ter +tar get +tam e +dr ou +astro photography +mp c +g pu +nord strom +fric tion +run off +lov able +sp nfamily +ext ingui +bloo dy +sch el +arti stry +sw ish +scar ce +ph ils +max im +pos sum +com promised +sty li +sc fc +is sa +birmin gham +sket ched +angel ica +ordin ance +je ts +conqu er +ðŁĺ IJ +online shopping +s ori +reason ably +nue stro +ar turo +ch l +benef ici +spho to +wel t +ni kk +ðŁ¤ ŀ +dan ao +for mid +as se +af irst +âľ Ĥ +gil lette +as sor +an onym +sel ca +fe mi +bear able +y and +ar mory +cre pe +celtic fc +bra vo +in expensive +de lec +ge cko +new market +snow flakes +kab ir +con tra +can ning +mor pho +gar wal +ðŁĴĥ ðŁı» +fight ing +mu tation +woo dy +ju gg +gr aces +premiosm tvmiaw +kenne dy +gu p +sa e +op ha +off spring +fini sher +bet ts +span ning +mar j +h one +sh ing +contin ents +samanthap rabhu +un related +l acy +explo sions +benjam in +sophi e +no ting +micro soft +as sen +a hoy +i ker +ho fer +mo e +ah madi +yan n +an ak +ma hi +be u +aha h +creep er +baahu bali +am at +pri ory +haw keye +deloit te +sko da +print making +assemb ling +mirac ulous +no ch +sw o +leg a +oper ates +border lands +eli e +stron gh +rep tiles +pir ate +un fold + ¯ +qual comm +un predictable +ot r +rose wood +direc tional +counsel ors +corn ell +liber ated +j ad +ir regular +bulgar ian +high ness +vodaf one +sw ild +mini mize +gra zie +๠ĩ +r stats +stre ep +ome tric +humb le +lu mp +l ille +b ü +home depot +tripad visor +ki wan +a via +er z +ex ico +du f +blu men +mi zing +ar ma +in im +con stan +sor a +ju al +au n +tw ell +tren ches +her a +r k +po plar +recipe oftheday +ll an +bhu ban +short ages +ing don +bridge water +ðŁIJ ĺ +fortn ite +cam den +un cture +pro w +colon ies +t ks +n go +b hm +live pd +spl ace +sli ke +happye aster +ter rence +revol ver +j ed +yy yy +office of +m ts +exist ential +r ourke +explore bc +sse d +pri est +vix en +si ding +k pa +a har +ju ic +ob struc +foren sics +uk mfg +cancell ation +we ary +ab q +ele c +pri zed +deb ts +me zz +salv atore +m dc +gre tte +c gc +th on +snow storm +ts ch +cook ery +å ¹ +wa xing +n acional +mur s +ra ve +cap es +ger main +dri pping +sub mitting +ome lette +iter ation +aj es +shim mer +fu eling +ðŁĩ§ ðŁĩª +li po +bo bble +un follow +islam ist +hi ber +cat s +agentsof shield +sen si +____ _ +ster ia +inst al +ausp icious +har row +over land +femini sts +inst ant +char iot +blind ness +sp ed +sc arec +nu it +mini atures +ho seok +glo ck +fifa worldcup +e te +dis m +we iner +ex foli +ear ts +ภĶ +my art +man il +iss ant +form a +in cu +buffal ob +in tim +mc cul +anj ali +po po +un doub +hil a +fun gal +thank ful +fu tur +en dish +ren ds +th ar +she ff +ring o +nichol ls +io wa +po tom +cl ams +ãģ Ħ +acon f +stadi ums +di mp +di k +residen ces +do v +caric ature +seagu ll +kl m +confe ss +sla pped +cele b +turb ines +pp v +nur ture +el ab +.... .# +tu ff +de press +al far +amii bo +di spon +e wing +que er +friend s +for re +âĺ ¼ +sw t +aqu arius +head liner +cur d +fi gs +o tters +love fl +kare em +go vegan +fri yay +consol ation +at ri +ì§ Ħ +âĺĿ ï¸ı +poly ne +gu ed +o ya +la us +intestin al +cam illa +scal p +pi r +leed s +horri fying +bore tum +dand elion +fer rer +ell ic +as x +so ren +re loaded +ale ague +navig ator +ine tte +add ams +al chemist +ak shay +dystop ian +awe c +n aya +al isa +ai led +ag or +avi ator +ali zer +smo bile +findyour park +cop ying +to ddy +sh ti +mon ger +cal houn +nap kin +break up +y atra +se thu +ric hi +eras mus +fer ry +am ore +prac tise +bo bo +power point +oo se +li ffe +chin a +sh ka +fad navis +du ane +war on +fal se +ðŁļ Ĥ +wa shes +disc ip +==== ==== +g k +ab b +stub born +medi eval +p ci +ðŁį ª +maril yn +h yo +man di +cr i +prede cess +continu ation +om usic +s lat +wh al +mall ory +bon n +shen zhen +ca i +âĺ ĥ +sa fest +for wards +dra wers +bla sted +sle e +mor phe +mb ta +dumb ass +ÑĦоÑĤ о +alhamdulil lah +ec lub +al beit +heal ey +ayurve da +adverti sed +cro cs +itt les +bry son +be i +nj pw +honore e +fu sed +ðŁĶ ĺ +mul tin +n aga +de parts +ko p +kin o +jhar khand +ed na +ax le +mil ton +supremac ist +marrake ch +domin ic +tran script +] [# +: ). +wo c +sur rounds +o gil +leaf lets +co well +whe w +tru de +proli fer +succe s +sports man +con dom +po che +k up +imprison ment +{ } +scram bled +å Ľ +ka ine +cell phone +metam or +con i +remn ants +ee z +down pour +afterno on +exerc ising +ber ser +architec ture +wick low +m ns +is p +bo c +n iss +mn wild +stu mble +r si +lu ffy +sil en +dd ad +bul lies +haw ker +bb cc +scu ba +e pp +que ts +for aging +pal let +ha di +cinemato grapher +cat chers +to aster +k hi +lite coin +kid lit +amher st +maur icio +ip ad +mar malade +fe y +don nelly +g to +est as +cere bral +ant grasso +zz led +vir gil +swa pped +ðŁĺħ ðŁĺħ +no dapl +greate st +nhl bruins +fra ser +b mo +ane w +. âĿ¤ï¸ı +se gregation +remark ably +mccor mick +lo gger +er as +contrac ting +âłĢ âłĢ +yor ks +uku lele +touch screen +de cked +ben n +south wark +ra vin +nu mis +ðŁ¤ Ļ +ru t +gre co +eth ic +red neck +ar r +t cs +ih ri +ðŁĩ« ðŁĩ· +l k +inher ited +zy k +viadu ct +marty red +hi gu +ss n +be in +street style +fer gie +bank of +æĹ ¥ +stake holder +exempl ary +cre ss +ess a +ero tica +intre pid +gom es +bra un +bethan y +bang tan +pulmon ary +m illing +doctor ate +trump russia +ठ° +s ani +bl att +pla u +depri ved +t le +ful ly +bour n +st ak +lufthan sa +kio sk +far oo +def y +bad an +ðŁĺĺ âĿ¤ï¸ı +rit z +tri sha +ran ds +middle sex +arab s +pro j +sport scenter +repe ats +iv f +bleed blue +as sure +o bs +territ orial +ele n +bever ley +ann ah +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ı +z l +for good +science fiction +gla u +son ya +pri th +st weets +mix ers +mari o +ant elope +writing community +went z +den ham +be di +sf o +harley davidson +look book +immuno therapy +or phe +es ville +ed ged +tas k +sb ball +corro sion +kilom eters +co sting +play back +ke ke +di visi +u ter +re location +yel led +pen g +up beat +ser ve +âļ ł +hal en +stir ring +reh man +en v +schu macher +frag ment +alkal ine +sb k +resil i +share point +rol lover +tra sh +counter part +âĻ « +ob itu +à ½ +ãĤ ¹ +mul berry +ðŁİ Ĩ +auton omy +spra ying +nat l +love you +fran ki +nu k +esc ar +can teen +ali baba +de plor +mole cule +pu d +fort night +blon die +sp hin +portra yal +ta che +bu te +consi sting +freep alestine +c sp +im mort +d ns +ðŁĴ¥ ðŁĴ¥ +tour de +coo king +archi val +ga thers +bit t +b anc +pre mature +snow ball +poetry day +lou dly +fug itive +ed ay +em ra +ðŁĩ¸ ðŁĩª +sci en +node js +jur gen +je ong +band ana +un is +fox sports +v andy +pro visions +wee p +tu k +i ko +h oun +zig gy +z r +fil let +bat a +tin k +con e +we want +k ilo +hor ace +sl t +sc t +stay tuned +victor ia +umb ria +att acker +ingham shire +fright ening +no ir +fr at +con tempt +lia ison +ho i +br ink +tr ill +ni agar +kick ass +dun das +not my +rho de +bu mble +no xi +fa g +spec tators +mancrush monday +jin ping +distr act +dais y +wal den +portra it +ar thistory +vol tron +ev el +is c +ac m +r ite +na o +de ported +swe ats +ru fus +lo bo +labor day +gam o +ihri thik +bl it +abdomin al +ãħ¤ãħ¤ ãħ¤ãħ¤ +i it +e q +bu sy +allu arjun +un disclosed +de ton +pro create +ki l +ðŁİĤ ðŁİĤ +mitch ell +ki i +inherit ance +al p +jo burg +pat rolling +compul sory +un signed +ni am +l ga +eshop suk +tr illi +ma w +appreci ating +rock ab +mañ ana +an tal +mal vern +roy o +grand prix +sut ton +go ftheday +dig i +ãħĭãħĭ ãħĭãħĭ +t les +varan asi +erec ted +discip les +cont act +ðŁĺ µ +li d +⬠ĩ +scen tre +radi ator +ing tips +trans itions +thursday motivation +chem ical +separ ati +sal is +mi m +geo graphical +book fest +/ . +âľ ĭ +v ae +cur rie +ag garwal +acceler ation +the ses +lg m +u mass +pro portions +nat a +ani ans +ku ch +be acons +ap r +@ # +ðŁĴª ðŁı¾ +nu ke +sher aton +ki o +ma kati +polit ico +mor ale +ì Ļ +econom ically +gg ly +ss en +pa stries +intern ships +vic ente +fanta ken +aveng ers +accu se +slee pover +indic ated +the dream +ster one +ren ders +fro st +ou i +gre gg +d ore +⾨ ⾨⾨ +pu gs +sat y +nu mb +hems worth +tam i +la ssic +schi ff +igle sias +ag awa +] " +re shi +game stop +divor ced +theat er +clau di +un conventional +prophe ts +ac in +twel f +tow ering +t ml +sc lerosis +k wan +ge ts +distur b +na ira +ener g +pir acy +pru itt +noti fied +hen na +bra m +ground water +bl s +opti mis +$ ) +luci e +biz hour +fang irling +gr ills +or l +ver se +c ina +law less +artistson twitter +tele vised +marshmal lows +radio head +bar r +m fc +bre vi +mmor pg +g aya +âĸ « +sub titles +j t +disney land +to bago +nh m +groo ve +fi awec +" / +ba o +scra bble +om ni +ff l +um c +si mba +ali er +ter rell +plu me +mi di +dig nit +co c +bru t +ad ata +alche my +d sm +ðŁĺĨ ðŁĺĨ +win try +spa res +cu er +conclu sions +to ys +od or +fl ann +gar vey +scrip tions +inspec tions +cat ap +ang lo +st louis +heim er +at ay +tr ich +en yc +chil ds +vent il +mont p +guiller mo +circu lare +z ell +mode led +craf tsman +al ina +stimul ation +cashe w +ju das +best of +to ire +susp ends +scol lege +real ising +by tes +bloo ds +as si +ðŁĴ ¿ +o hs +ðŁį ĭ +scallo p +ठµ +gi fting +camo gie +wil kes +o zzy +ðŁ¤ ¤ +ver onic +sav oy +deme tri +baby girl +ðŁĺį ðŁĺŃ +so x +cly de +induc tee +count down +self care +ठľ +vi ka +tor re +phd chat +pe ars +aw h +suff rage +le sn +admir ation +mp p +shark week +schul z +santor ini +clo ver +( * +stras bourg +ex iting +so yu +finger print +che a +ãĢ ľ +vin dic +song writers +so a +prou der +nam a += )) +simple st +delici ously +gil les +u q +mn wx +ep p +sh un +ken nel +fall on +ðŁIJ £ +sin d +tra gically +out es +modern ism +co ke +gy n +spi on +âĺ¹ ï¸ı +le am +compress or +apolog ise +twent yon +fan atics +âĻ » +sco tsman +sa wa +ko u +as er +ภļ +welter weight +phen om +twick enham +stri a +p out +ka z +gi am +cd p +ho y +emplo y +red mond +ภĦภ+sm ere +trance family +proto cols +pie ce +lu iz +iter acy +carl s +united states +har med +phd life +ch aw +foot prints +l é +cho ker +z ana +sli pper +eric sson +insul ting +articho ke +advis ing +acquis itions +op or +mut ations +re ar +ॠģ +pod cast +wi ther +kun g +íĺ ¸ +win slow +di apers +ðŁĵ¸ @ +ec ker +col lar +hu ey +gi ro +mono gram +kas ich +si veness +malay si +arom atic +gre s +gali leo +u ji +rob b +dr m +none theless +as a +: > +lo a +l np +at work +ag t +laksh mi +pipel ines +id al +stre l +re all +chain z +stone wall +san sk +ðŁı ´ +pied mont +hoste ss +ci u +t é +analy ses +wil helm +scott y +rw by +mosqu it +use mb +qu ins +ðŁij İ +tu cker +s conf +speci fications +psychi atry +broo kes +s ils +ol af +de to +co di +cli p +fil th +womancrush wednesday +go to +ang erous +be ale +w tc +paneli st +ne x +lar sen +emili o +tab leau +h itters +conce ived +americ ani +or tega +mar di +Ñ ĥ +pain tball +thir sty +new yorker +etis ation +go ss +we aker +u gh +tro ll +har ga +du al +ght ning +at ine +ðŁĺİ ðŁĺİðŁĺİ +cook out +pyrene es +po ss +authent ication +sports wear +yun ho +kir o +archi pel +shen ko +ren der +nov ation +divin ity +ðŁij £ +su fi +humb ling +ge opol +devote es +wait ress +tr ough +py ro +i ba +bl ing +gra f +epilo ts +bt r +of tball +bas king +domin os +so om +r ath +sher yl +qu el +astronom ical +wel d +track list +sig nee +slee pless +com man +ch ron +summ on +pure michigan +cri spr +sli p +la gi +ra q +um u +thal ap +char med +scru mp +quad copter +ski p +peter sen +mun i +ðŁĮ ¾ +mon aghan +tra ys +ick ed +canad aday +te gr +ï¿ ½ +hot ness +heavy metal +ab ar +gop debate +az ul +spider man +sun flowers +ľ ë +web comics +bar d +Ð ² +nichol as +slu sh +ram an +mark ham +ffici al +ff ler +íĬ ¸ +ple ss +anush ka +to to +sk aters +pro wrestling +compet es +ay ala +myster y +thr ills +mp g +independ ently +y ul +imper ative +formid able +tire less +st acking +ton gues +mal tese +pot ts +mat ti +char ting +chill out +super nova +ome o +sky sports +nu tty +ðŁĹĵ ï¸ı +ro han +insp ired +concier ge +ser ra +ma kk +gal at +chi pp +ye v +ì £ +reim bur +op ul +kimber ley +i eee +bre men +ch itec +or in +nak u +bon kers +foo ty +emer gence +ðŁĨ ĺ +sti p +serge i +zo ey +ai me +wou ld +dy es +destin y +vinai grette +dri er +circulare conomy +an archi +ss r +sch el +cin er +gro om +determin ing +gar min +cal ais +incarcer ation +bu kit +no i +chelms ford +mckin ley +chi pped +belong ed +tu mors +str oud +mi i +influen za +wwen xt +tun dra +tele communications +cat sofinstagram +t ages +beat ty +o du +ml kday +oo per +dang le +ak ley +cru mb +anti gua +ti mbers +rou hani +ðŁĴª ðŁĴªðŁĴª +ha fi +... !! +w cs +coo p +sn c +lit res +ãĢ Ĭ +ha z +co z +k ant +green field +cur ti +y ale +flye agles +what soever +wor thing +rou lette +flyeagles fly +un da +a inted +stand ing +lusci ous +h pc +effic acy +ash land +me ghan +ky wx +n pr +bath tub +ac os +h ani +mar cor +man tis +da isi +bo ba +ab bie +mu til +vi al +spy der +po z +g ti +el fie +nigh tw +metro id +anton i +mad die +dh ry +dar lings +ten ds +taek wondo +atlan ta +me ow +chlo e +ãĥ İ +ym es +siber ia +k con +gu es +mar iner +fac il +azz le +[ ... +han nover +bav aria +vir go +te uk +u sps +) # +wall a +sam pson +need less +ver bally +hay ley +bow led +pi us +lam pard +ham string +vol vo +road safety +cho king +sor bet +a hem +healthy food +brai ded +horticul ture +cr ative +che ek +ad do +the force +ko ko +schiz oph +j ie +w ada +twentyon epilots +h bcu +pro ton +pau ls +lou isa +lat am +kyr gy +com pac +sd k +sap i +?? ? +liber alism +ep silon +ai den +w usa +spra yed +baske tball +kim ono +blue wave +ali as +ë§ Ī +mug shot +ce c +do gre +ad ora +ðŁĵ· @ +kra kow +intrigu ed +exhau sting +astron omer +ven ison +lady bug +ci v +bra e +us m +bri be +acup uncture +pembro ke +ke ating +chi e +y ad +t si +sm i +see ding +gate shead +lis boa +gy p +canv ass +ðŁĶ´ âļªï¸ı +op i +ni r +soci etal +ly te +ati es +c sm +ar tery +al in +aka poor +abstr acts +âĢ¦ âĢ¦ +teen wolf +ne we +travel gram +sentim ental +per ched +han del +ho ek +f ay +coordin ating +anim ate +man ian +effor t +jer ky +f ck +adri enne +ma bly +tra ding +my el +spi ro +sol a +stor ing +over drive +monday morning +dream team +pul se +bon di +ber nie +pgat our +tri poli +son am +plat t +âļ ¡ +ag roup +îIJ Ĵ +inv ading +v cu +k ell +ñ os +un dead +pod casting +mercede sam +mana fort +cor tex +que so +impecc able +pal mer +wil doz +sport sc +guacam ole +dispen ser +cate gori +stun ts +per il +invit ations +dune din +xi e +achi eves +saf er +pre ds +ph an +knuck les +k ak +igno res +lovemy job +aru ba +ound ation +datac enter +co vert +gr ing +cou ple +ا ر +vol i +mc cle +arti sans +lu do +kal am +arom a +under taker +hu la +wiz kid +gu mb +god frey +bakers field +ker n +engine er +car ve +pal in +guaran tees +pe bbles +b ays +zi eg +fin k +â¬ĩï¸ı â¬ĩï¸ı +down pours +ro chelle +rasp berry +ðŁĺ ® +gra phies +stom p +caf es +ari zed +utt ar +cal vary +dri e +crusad er +bus an +tux edo +si u +seam us +cul tured +blan chard +town house +ge red +butter milk +flu ctu +roger federer +hel i +ðŁ¦ ĥ +u ous +ram esh +mu ppets +email marketing +ye ss +br ice +ri zio +pel o +donnein arte +u rable +inve stin +bump ing +raji v +sav a +thro wer +fore x +o hhhh +th rust +pull man +r fid +sep sis +le ed +fri ght +roun ding +ne b +ph ins +ai sha +utili zing +squ ats +gold smith +j ic +bo ks +vau s +i po +exclu sion +tari ff +po kes +min al +land s +en force +washington dc +or char +g x +mar ys +ey our +aussi e +bak ers +un popular +latin os +lar ge +pu tnam +bol o +wa de +pel o +di zz +ob struction +fla ppy +weare the +depend ence +pajam a +e te +y ann +e wan +disc la +a ay +kar ina +e ic +an trim +w soc +neg atively +kai do +fotogra fia +dh ru +colo ssal +mcle od +k wang +mani pu +ex hilar +us atoday +summer slam +co les +tapro om +unbeat able +de ma +tic ks +k ling +fil s +campaig ners +ภķ +brew ster +audu bon +qu ay +ch s +ki gali +d ler +strength ens +som al +sign ingday +gol ds +pig ment +orche stral +g q +lin kin +ðŁı ĩ +ta w +algar ve +ho v +ear le +gold fish +am ig +ex er +ben in +dru id +ðŁIJ ¸ +she m +quat tro +mer cen +men te +incorpor ating +bon anza +state fair +en de +concep tions +e es +âĻ¥ï¸ı âĻ¥ï¸ı +d son +fire arm +orb ital +we h +multi p +fo b +requi em +p light +thou se +sa id +oc re +remem brance +n old +chi pping +be v +er t +ca thy +sy m +ri ggs +m ley +dialo gues +sl ender +how l +gau teng +wd w +to bi +smo kes +im plo +b pm +ad n +mom basa +cap sul +bloom field +artic ul +cle o +goog led +flu ffy +l ard +en zyme +ve sti +ibra hi +fl ame +e mea +out ages +dispro por +ble ak +an sel +ick er +st louis +stock market +good friday +sau lt +stal led +pro m +ep som +b é +the se +sau ces +me w +lit fest +pre d +re u +kar ak +si enna +ell in +bio technology +ï¸ıâĥ£ - +tac tic +sa in +por k +mon za +ka j +lu sh +compart ment +chang ing +shraddha kapoor +fo al +ar tem +cu ando +can ola +ori ente +me sse +d ited +br c +box er +bbc two +s st +ment day +em ing +de wey +kof i +âŀĸâŀĸ âŀĸâŀĸ +reali zation +smo l +tw ood +san je +flag staff +ber wick +cor set +can ary +whistle blower +et ched +com posing +squee zed +bow er +auto desk +ne h +mathi eu +ba ja +Å Ĥ +hy dra +da im +am eri +insi sted +mer lot +gar ros +heart news +gaine sville +cut ler +bo de +ðŁĺī ðŁĺī +lew es +scoun try +g sa +us u +cc m +god awgs +phara oh +cra e +mor ley +hyp noti +f ades +neur ons +fu zz +ing co +high landers +star k +vig ne +pac kets +amar illo +reu ben +insul ts +bas ic +vec tor +n me +ac ruz +tro s +transm itter +ðŁĺ ŀ +interpre t +ðŁĺ ² +pre quel +mc gowan +dis semin +ðŁĴĺ ðŁĴĺ +mascul inity +indie gamedev +ali ve +te t +pe tal +ema iled +ar med +ko o +he er +ba ird +super junior +metro polis +delav in +decl ines +stit utes +Û ģ +p tbo +g lan +cho res +e aling +chri ssy +ste mc +vi an +assassin ated +pron ounce +illeg als +discover y +cav ill +fri fotos +f al +so i +sabot age +t int +p dc +ðŁİīðŁİ Ī +ãĤ Ĭãģ +ji o +endeav or +in sig +commit tees +she arer +me tz +mar rying +h dd +g by +fre t +tri sh +pu l +scrip ted +sa ki +l w +ke ye +shim i +nan aimo +ca h +à « +tem pered +ici an +du gg +dish washer +air field +s rugby +gr inch +y st +r ms +mahat ma +lan kan +disc ar +dige stion +no des +l ls +om ic +gu tter +tis garh +feder ico +election day +bo he +master card +fire ball +âľ Ķï¸ı +oy ster +p ong +do k +en route +m vc +beat the +ali stair +shu b +sh aming +cherno byl +ghi bli +the s +pin ion +d bs +sal ts +ic tion +epi ph +nc pol +in convenience +whit ley +inspec ting +wood ley +wi ener +skil let +no les +m ca +h ina +a sha +willing ness +well ness +tam ed +show time +dis advantaged +ber nat +us n +mission aries +coun selling +arrog ant +quant itative +leg alization +ho dge +energye fficiency +cameron dallas +pos sessions +p bb +harris burg +v g +hindu ism +happy thanksgiving +fi b +re acting +tweeta picture +pol iti +mu ppet +hur rah +pac e +coast guard +guar ded +as am +par ry +fore very +x q +oom f +ke anu +j ind +ri st +customer service +sac red +ðŁĺ º +ton er +occur rence +mat u +val dez +red d +is ak +power rangers +pe asant +raj ini +abra ham +e mil +car do +tr il +hair styles +obsole te +sam pler +direc tive +delavin kisses +ver ton +glo s +sp ay +paler mo +com ets +man ziel +chicag of +ski pped +pic torial +h ant +b mi +a ol +re opens +pad dling +devo s +fra ud +bas eline +que ues +sp ired +sn are +eu ve +descri ptions +daisi es +ca ching +gall eria +tri mmed +stin o +recy cla +ic ular +bir ken +raw lings +fli x +chic as +b gt +lik eli +argy ll +thel ove +ga ston +bl anca +ha k +f one +sailor moon +h aci +ima c +fl yn +de can +bel les +ap ic +zo g +taun ton +con stance +lasag na +ker nel +in ka +har bor +collec tively +calcul ated +av ille +shil pa +pur du +gi mm +fun er +a est +pembroke shire +nighting ale +n unes +hyper tension +hu bert +sli ders +infer tility +comm ended +transat lantic +metr ical +!! @ +Å Ł +ss g +bac ca +inver ted +fun factfriday +it ans +albu m +acqu ainted +ri er +whel an +sar ab +mu e +snoo ze +pi ff +agre eing +sp itting +jer maine +n ye +âľı ï¸ı +am bush +ze ph +con greg +univers ity +s app +wann abe +pat rice +ib d +do glo +fri dges +sun d +king ston +ar gon +kam en +hardro ck +ds ley +do lores +ì ° +ota ku +pi ping +be having +âŃIJï¸ıâŃIJï¸ı âŃIJï¸ı +blue bird +an sari +teapo t +fire work +cro p +log ans +ty ped +thick ness +ig ers +c fp +dys functional +contra sting +et ty +aston martin +tx st +dra grace +at tributes +marath on +manu scripts +john stone +ðŁĺ± ðŁĺ± +bo er +ay u +aru gula +poo rest +con du +assu mption +anag h +no h +delav in +sit ter +g ö +mor ow +kick start +com i +gl acial +ghe ad +ba in +ker shaw +en dof +fre ud +om at +i af +hu g +sign up +each other +defin ite +tu bing +shak ira +ðŁijı ðŁı½ +uu uu +sw in +sham bles +ol as +sk ell +brit ain +kn w +clu tter +om y +j ens +hang ed +city scape +scra ps +un locking +dead liest +er no +breast cancer +a it +inspec t +fu ri +ðŁĴ Į +ku d +ju le +or ah +mi ds +m dt +bur gring +r attle +pu sa +stal k +cle ans +iss ance +z ek +worth it +nam eis +musko ka +council man +urban art +bar rac +un solved +tu l +g ita +white board +soy beans +em ent +cont i +saturday motivation +conveni ently +doc king +t ado +âı © +sp ino +puppy love +po f +fabric ated +robb ers +adop ts +ti fied +kk r +indulg ence +notic eable +macqu arie +chap el +sensu al +ki ko +melan oma +lore tta +li ance +ab en +sp lus +ga al +ac ele +lib dems +compar isons +ðŁĮ µ +rhy thms +mer y +en capsul +nap ier +ðŁijĮ ðŁijĮðŁijĮ +ðŁij IJ +plat z +fre sno +re formed +ran bir +el it +the best +bhu shan +vin nie +impro vised +s ittin +re created +e ba +ec ker +ac rob +pon te +cor d +gi ddy +eur usd +fe ver +intu ition +gar i +dum mies +bud weiser +amend ments +te tra +sch nit +ay as +mar ys +ci st +k ani +ker mit +ðŁĺ±ðŁĺ± ðŁĺ± +tin ker +strol ling +di visional +niger i +omin ous +menstru al +kar ab +k hy +bw fc +pan handle +l illi +well er +stra pped +son the +transfer ring +ethe real +sne aks +ru dol +gab les +jac king +cin code +for tune +canadi ens +con for +ab normal +frank lin +tit a +mu la +persi st +cu ties +ki el +ðŁĩ± ðŁĩ +her mann +aw k +fi asco +ko to +we ta +hi ker +budd y +preven tive +mcgra w +game boy +forsy th +top shop +si ob +sad h +in tram +follow art +so aps +dragon ball +ou x +morri son +๠ĥ +lu bric +adul thood +morri sons +âļ łï¸ı +her mo +ta ka +stall one +mis use +team gb +ra gha +con fined +at y +hom ophobic +nw o +sky news +ho ya +ac rosse +wi iu +pur ée +jed dah +ðŁ¤ § +advis ers +ph ine +an is +scrump tious +ë° ķ +c ke +vin y +ter m +s dc +o do +home school +vas c +leop ards +debor ah +illic it +cur ran +as roma +nau ght +mar ig +brand i +em p +ðŁĺį ðŁijĮ +î Į +su spend +lu z +initi ation +sch aft +jensen ackles +craw ler +post doc +des ks +trail blazer +den omin +tri x +no ise +po et +± ï¸ı +s mug +vol atile +proof s +pharmac ist +sardin ia +mash able +kim chi +co ed +schal ke +doo dled +c sw +sh ur +ro x +do k +chris brown +mathemat ician +ab ound +ang elic +rock ford +d ole +yor kers +ms n +g man +xavi er +bor rowing +mark ings +longh orn +k ja +diver ted +mm it +euph oria +ay yy +te a +pa h +ck i +un cut +li ven +ky ung +fan art +mer ing +red ding +amo vie +gri di +c thulhu +schol arly +ju dah +th bewithyou +eu calyp +ðŁIJ ķ +hert fordshire +cour troom +by u +auc tioned +ple ase +mar cia +ê° ĵ +succe eded +el as +arvin d +t lot +saig on +re tt +ra kesh +fd ny +as en +se bring +gladi ators +you know +v lad +gol a +par ap +ÑĢ и +sab cnews +one team +oh l +sun e +ri j +cd c +star gate +run down +plat o +ph c +chat ter +ra viol +mn f +mand ala +li et +ภķ +mari a +hun gover +consoli dation +fer rell +tradition al +ilove art +gal ap +ðŁı Į +que zon +espa ña +ðŁĩ¨ðŁĩ Ń +ho bby +steam boat +mali gn +guil lau +pro hi +its me +íĥ Ģ +in scription +al z +mari an +k ade +mm on +adju sting +ne sts +intern ally +ci r +vik ram +mal ala +k ph +fel icia +the real +cap tivity +at is +marcor ubio +kale ido +che v +mano j +le more +gent ri +vi ps +tro pe +" âĢĶ +pair ings +mal nutrition +fr ay +desig nation +brun omars +az e +tor rential +pan zer +ga il +under the +the ological +schizoph re +dazz le +freder ic +mo par +ad illa +so ggy +ra un +medi ocre +colo rec +i fe +p inst +blu ef + ² +world water +gir oud +clar inet +ad olf +tar antino +receip ts +assu mp +ðŁij Ł +coffe es +âľĬ ðŁı¾ +du plex +s of +r x +lin o +timber wolves +pan dit +mo tm +e ga +ay ama +ach s +outsi der +ll en +co er +til ly +cheese burger +ma ds +ple dis +emp ty +national parks +az iz +p mi +jun kies +f ener +sq n +è s +gener ation +cleop atra +bhuban es +mosqu es +ty free +popp ins +tw c +or well +n age +ka whi +hol low +dal ai +¨¨ ¨¨ +ou ro +m health +gi on +az o +vis as +reneg ade +re ic +w sop +ðŁĴļ ðŁĴĽ +e chel +tox icity +mü n +bun k +stimul ating +asth our +\ ' +ep h +ende mic +cn bc +shrin king +peabo dy +michel angelo +can yon +wal e +su mi +si ders +inu it +? . +profession alism +dr acing +plat oon +p ons +out bound +maple leafs +de sol +cen cy +a than +ver ma +ru bbing +ok an +ðŁij ł +mull ins +authent ic +Å į +alman ac +ga ia +bb q +on imo +ke h +ty a +tou ts +y av +re posit +, . +wi ght +se eyou +cal lof +done sia +bar gaining +gr anth +sd su +amphi theater +p su +re watching +wine tasting +peak district +dete cting +thur man +phe e +èª ķ +u mich +re r +sculp ted +go le +name sake +ðŁĶ ģ +serv icing +bau gh +pu gh +pen cil +dar th +munch kin +at orium +ten ers +sun y +rolling stones +mag ing +star rer +i dris +fe instein +ag ron +âĺºï¸ı âĺºï¸ı +supervis ed +chamele on +aggre gate +succe ssive +mo gul +inst yle +pol dark +custom e +ohio state +ha ya +ci des +broker age +angel ou +fifa wwc +de forestation +al ton +pam ph +hu gged +ho bo +change able +ku ber +bur roughs +demon etisation +cape cod +vers atility +or ice +le ila +womenin science +tu a +he dges +embarrass ment +ali fe +so ars +ni ghter +hy mn +gi pp +chas u +tech s +ni all +k illa +hi ka +cam els +valu e + ¢ +sc oops +mah moud +clu sive +adri ana +pac o +oz il +un as +transl ations +whispe rer +s bi +bu xton +bio tics +indi ffe +ken ney +k lar +et ching +barra best +inst ability +se ine +vo tel +blo gged +whis key +my space +t ant +lan dia +give back +illu s +aw ak +ac ab +f bloggers +cloud computing +blat ant +syri ans +band ra +sty n +an em +ke ted +kar thik +barun sob +pin ot +gu bernat +gay e +arti ste +i fied +conven tions +hu an +geni uses +eeee ee +fol ly +somer ville +pride month +ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ +chemo therapy +paul s +bak ar +ìĦ¸ë¸ IJ +taiwan ese +fol lo +c ss +re ign +nn nn +fla un +catastro phe +iti es +frag ments +extre mists +ym oun +car men +eze kiel +conne cting +se h +man ta +remodel ing +we ymouth +at oms +ce m +ne well +lu mi +the open +mo c +mili band +g land +z shq +mag gie +mani acs +m sp +ad y +cre ams +le anne +e sta +py g +af finity +pray er +dun bar +ligh troom +ac adi +wyn onna +roman tic +state dept +sick le +wh os +lam o +et our +fin ity +shru b +shar pen +pun dit +ed on +af ore +mar s +jeff ery +ter ps +medal list +kath arine +accu sing +ta z +roy d +from home +confron tation +alle gh +ðŁijī ðŁijī +refresh er +ran veer +never land +jo jo +lu crative +en am +ca ver +pa edi +man jaro +flu ids +the ssal +oppre ssed +mu ss +joh anna +Ø ® +cn g +buil dthe +sett les +s ith +fu ego +cl amp +ar ag +pay er +ted x +mand y +inter stellar +fr c +ch and +b cc +mo lo +len til +johan sson +grims by +nature lovers +ðŁļ¨ ðŁļ¨ðŁļ¨ +shin de +x in +international dayof +transiti onal +sat a +cad dy +wo d +if u +ha ys +holl yo +j ang +ir c +co im +grad able +" " +ðŁį ´ +ঠ¾ +a el +n yo +west lake +time out +sof i +phenom ena +cultiv ation +ag no +un armed +so t +con j +gen o +royal navy +nutriti on +fair mont +ti relessly +sn g +re ty +mic a +lu cent +slo ane +droo l +riz al +od ell +critici zed +. '" +la ze +deser ted +co der +pra s +l illian +itiner ary +dav y +an ap +whi pping +hobo ken +kare ena +çľ Ł +vi us +ter n +nan tucket +mis understood +bu laga +st ant +chin ook +z am +reli es +d ss +ed mond +sket chy +m ell +fe x +rec tor +dist ill +day dream +wine maker +ri pley +billion aires +hel ene +ati f +cul prit +bertr and +wou ldnt +ma pped +v ak +gla dly +parliam ent +kidlit art +ware ness +goli ath +âĨ ĵ +view point +tat ted +fu ls +dor sey +ang lers +li ds +ki ya +bow les +be h +b ite +compati bility +ance stral +pro x +beha ved +gubernat orial +ch field +sab an +z h +teen y +shibu ya +holli day +pan cy +âĿĦï¸ı âĿĦï¸ı +seun gri +? , +ðŁĩ¦ ðŁĩ· +im itation +impac tful +any i +gene vie +añ os +bate man +gli der +af ar +ra sheed +effor tless +sh war +dach sh +er un +at os +kin i +ch d +kha ki +k lin +felici dades +bel o +as l +to ppers +fin ley +stac ey +rigor ous +kar ting +le ppard +car michael +be ret +c se +ak hi +mer ingue +ab an +ha ke +ger i +er jee +re sto +comm anders +pr it +fl or +ad ven +ex termin +remain der +å IJ +es g +martin o +lulla by +| @ +mi gn +in store +big bang +cor di +cau ley +ante bellum +dg ate +cro ck +span dex +scaf folding +ore os +ê°ĵ ìĦ¸ë¸IJ +pom ona +ma uro +uni versi +re mi +af ootball +t ant +sm alls +ne h +worl do +tropic al +mor ph +jav elin +gla r +arqu itec +reminis cent +tu bs +spide y +make u +syl la +progressi ves +blo t +shor ten +keep in +ch ak +ang st +super food +decad ent +ston y +neuro logical +ar boretum +ann ak +fe ma +per cu +dis respectful +small biz +lo x +co om +c sc +bs bi +pre valence +him ss +esp an +mo ga +fr ampton +sky map +mas se +levi athan +( ). +noctur nal +car ameli +ang or +amne sia +outsi ders +she alth +rhin o +ant ag +ag io +ðŁĴ° ðŁĴ° +take me +kab addi +c si +m sh +coch rane +thessal oni +sil a +ha us +du sting +obe se +mack lemore +mani sh +len in +m dc +gro wn +shef field +s rs +ke le +car son +ch um +dah lia +can tore +opp o +how ling +cyber crime +sur realism +sc ran +fa iz +thre n +rac ists +r out +pk not +se mana +sin i +mc cull +ma chi +alfon so +y b +sar dar +kend rick +den g +reci pro +on f +doom sday +bri bery +custom iz +art is +c pi +ðŁĻĪ ðŁĻĪ +sla va +let te +en s +âĿ¤ï¸ı ðŁĺĺ +cra yon +ad an +tr c +migr ate +simp son +row ers +king sley +farmers market +shee han +ne phe +bor non +car ton +mic key +all ure +u lu +sli pknot +heb do +gui do +dog celebration +online marketing +acceler ating +) .. +origin ated +macar oni +ed tech +out field +mit z +disc us +adverti ser +man or +ha shi +descri p +cap ita +ful bright +recep tor +con n +con ey +spion age +r attle +pre st +u li +blog post +acker ay +) âĢ¦ +red velvet +mat th +inspir ing +b sd +ker ri +po con +mil lar +re pur +accent ure +ä ¹ +ram bo +ragnar ok +dele ting +british museum +pat ory +leip zig +flori an +sci fi +in ers +br ate +yo y +melis sa +ab er +ma sa +po te +mosquit oes +transpl ant +r pa +; )) +bast ille +yl an +joye ux +melo dic +cap tions +atri st +roch dale +gott i +pew die +cuties aturday +who is +aqu aculture +tiv a +sp el +he ss +ha ji +fred die +co per +brand o +v k +photo book +* , +my dayin +micha ela +brune i +sr ini +in te +Ä ± +de ol +d fc +separ ately +bun d +ve sts +to c +me ck +rein forced +constra ints +car roll +sq ft +re ver +cam per +bird man +in action +gener ators +triumph ant +pe sts +o vo +gy pt +al amo +sc aled +suresh pp +sd n +is mo +gi os +) @ +justic eleague +restaur ant +gab i +den gue +next gen +exemp li +ap ex +inspir ational +down side +kid z +u pl +et na +alvar o +fel dman +bar net +m ha +es ch +bloo ded +>>>> >>>> +kan i +ho fficial +casablanc a +bir ds +ty ga +sw amp +o day +new castle +nb ap +ci sion +cho ols +af lo +ne p +mon ton +ak b +super model +down time +th os +sc wx +snoo py +ag greg +yo ke +nor cal +we tt +prolon ged +me tast +beat er +f ta +t lap +disgu sted +y h +voice over +itch y +ip c +ðŁİ ¾ +phe asant +stra its +ram pant +j g +fer til +assu res +fortun es +sal inas +liz ards +kett le +i bs +cyn thi +he g +mc cr +soccer oos +happen ings +cor den +ðŁĺĤ ðŁijĮ +t ches +egre t +wolver ines +congratul ated +ho gg +bott ling +wr i +fer ri +bo sch +af ire +og den +s jo +j dm +sv t +con tex +tol lywood +min k +me se +super sonic +op oulos +å ¸ +âĶ ģ +knuck le +gu ise +gam i +chu cky +z inger +radi al +compla ined +bo da +fe tal +discipl ines +cor ro +ðŁĩ®ðŁĩ ¹ +op ted +filtr ation +ad nan +em cee +mi stre +insom ni +fer gus +tra jec +on don +med tech +tanger ine +madra s +gru e +cab s +z hu +sureshpp rabhu +insul ated +day swild +pp m +band ai +v day +s ff +squ id +lo thing +not dead +expre ssive +cu ll +ala stair +x u +up front +fish ers +en es +um d +dis missal +sti er +sel s +lu st +re active +prote ster +eyel ashes +al im +goo de +gre eng +da ir +com pen +anush ka +proto typing +ma pu +bear ings +ðŁIJ Ł +for me +bsbi botany +timo thy +out skirts +am bed +are tha +wend ell +stre aks +ni m +k pk +sne e +fit ter +quo ta +p ate +win ning +ðŁį Ń +sho pping +ma inst +cul ver +ste vie +mcfad den +counter parts +gren fell +fol som +dor set +tech crunch +⬠ħï¸ı +tip tuesday +us l +tre x +geor gie +ranveer official +lic ks +se wn +k f +' âĢ¦ +jap s +p ate +orth op +fe sta +stra s +mon tal +hammer smith +fore most +wido ws +mad re +ite z +mito chondri +lig ans +z ona +cari bou +m ss +andre i +weather channel +gh c +: ... +ta ft +awe ather +al isation +bru tal +bliss ful +nik ola +mal icious +q m +mpg vip +bro die +bl itz +applau d +dri bb +v ague +dog go +transl ating +interpre ted +hat ched +ge tyour +benefici aries +spar ring +caes ars +aw illiams +la hat +bro ke +ti mp +virtu es +rel ying +pie tro +k tn +ici sts +pab lo +lou i +a ag +pn pp +cha st +pul ses +fini sh +usair force +type writer +thomp son +dog s +ut to +ãģ į +sand al +new ly +do ge +z w +wan kers +ne gr +mu cha +determin es +black fish +sk unk +mu ps +instru ment +phy to +daysto go +skin ned +hai der +con ten +ðŁIJ¾ ðŁIJ¾ +we iler +undoub tedly +chair ing +wall is +sh ard +zind abad +adul t +absor ption +pre sto +deplo ying +drum mond +battle front +seag ulls +how dy +juda ism +des de +part ition +âľ Ŀ +no logy +national bestfriend +lesn ar +film fare +co asts +christen sen +ac an +mb u +co pped +ru bble +sw c +fun nier +far ther +where as +nano technology +with stand +pil low +bow ers +to pe +it ly +con fit +ma kar +comfor ts +bo sh +cli pper +bal la +sti k +mil b +safe guard +musi que +eas port +ya z +pad ded +bad er +fore ign +chop in +archi ve +o ka +tran sporting +tml talk +aj it +consequ ence +sc roo +ff o +collabor ated +pug chat +ye mi +jav ed +au burn +o of +ma w +sau cer +miti gate +i les +evangeli st +ter ie +re cl +indic tment +cat a +bright ness +may the +whim sical +un lv +key word +cu min +med way +west world +tra w +im posing +form ity +coul ter +ab z +ny pd +grass i +kel sey +qld pol +clock work +f dr +di anne +âĺ ij +ad h +p ann +bra vely +ae ge +un lawful +ver di +pocaly pse +phar o +kar la +reson ance +ma stiff +la dak +bu u +ma iled +hi i +craw ley +tor rent +mach ado +liby an +effort lessly +fal sely +q vist +ke ef +craf thour +cheri shed +val kyrie +s ari +kal amaz +be he +ðŁĮ Ļ +th im +ro ddy +col trane +but chers +ach im +wk end +awk ward +cab rera +:) ))) +fran c +decl an +con dos +a ja +pandor amusic +char ter +ph ill +mon trose +hatch back +handic app +gre aves +eucalyp tus +ut most +t son +bur ton +mid wives +in cur +ðŁĺį # +moo d +compre ssed +tom a +must ang +mo g +as ana +te stic +sho tel +in sol +cor sair +nh q +ben ny +sm ma +kap ur +in con +jon as +ener gies +don al +as ad +se z +n pa +archi ved +stimul ate +do p +hy d +gri eving +ãĥ Ī +ron a +why te +tree house +ss ell +sand ro +ko bo +ther most +se clu +hi ya +ge ez +mam as +prisc illa +flav oured +fas s +w old +maker space +cospla y +p tv +happy valentinesday +sequo ia +love craft +gu an +d tm +ci i +yoko hama +pos thum +re q +ðŁĶµ âļªï¸ı +galat asar +dol by +hamp tons +disturb ance +stone henge +ok c +disrup ting +month sary +jun gle +head lights +du stin +micro sof +happy mothersday +ko ko +gra zi +te sto +na idu +mal ay +ari al +ru mb +ab oo +har man +tra pe +spo ils +je ho +go dly +lock screen +z un +pi ous +ma gento +l enders +prob able +corpor al +m our +aw al +su a +call me +ton ne +go vin +devast ation +x j +gear box +war lock +per me +it ate +gaza underattack +du val +paras ite +clement e +le th +i va +fro zen +tho les +to bin +cair n +s ill +luc kiest +conver ts +st ale +pan cra +euro pale +wis dom +sch ur +ì ¶ +verti go +bi j +u bc +nu re +righte ousness +mt c +factor y +ver st +revers ed +hur i +hee chul +fab er +ar r +ul ous +ven om +ph at +green ery +bra dy +à ¦ +: (( +never giveup +di sha +mo ta +health care +dun ham +dex po +den zel +bb ins +f ics +wh am +mc g +eli an +wat a +str alia +tel lu +pe sky +spin off +ar moured +re acted +do fficial +te du +sag ar +mor ally +paralle led +fi os +dow ner +dau gh +re do +world cup +tari q +bar ne +glaci ers +oc cult +barbar ian +her mosa +!! !) +y ur +inter nation +p ss +sit u +p int +american air +sw am +dopp ler +ðŁĴĻ ðŁĴľ +cincode mayo +le van +hell enic +mc ne +ju di +yu h +st x +qu are +ðŁĺĤ . +sti g +g els +mot ley +hard work +euro zone +e ad +ç¥ Ń +seab ir +ci us +la id +alpac a +presu mably +pewdie pie +boo ted +am ari +tam ine +sol ace +bar row +acade mies +x ian +om ination +dun geons +b ma +de ity +ai k +stab il +hir a +affection ate +ving ne +new port +ãħĭ ãħĭ +thir ds +re tains +aroma therapy +ski er +ni ma +do pe +cr inge +con domin +to or +anim ator +sar aj +seas cape +minim alism +lake shore +calla way +berg man +à¤ Ĺ +whisp ering +stupi d +ri ghtful +requ is +ir n +se va +ut pol +tuber culo +squ ish +de but +govern mental +christ ine +all man +weap on +s ito +bur i +lo lita +leaf y +fu ch +tin ted +mck en +a hahaha +ðŁĩµðŁĩ ¹ +repe al +ne gan +ðŁķ Ĭ +tail gating +game insight +ðŁıŁ ï¸ı +yaku za +z t +ti ring +pro posing +bow lers +tra itors +ak shi +cler gy +cit o +up sets +tu scal +symph onic +sil ently +shu ff +black well +ðŁĺĤ ) +ko be +rober to +ri dg +dc u +mer ino +ft p +east side +. ~ +nb l +mn leg +ts for +frau dul +ca pping +in my +gymna st +ston es +ss in +twe aks +shag gy +oak land +dem sin +sang ria +mm va +hen nessy +down ton +ri ghtly +in it +aga ve +ob last +northe ast +friend ship +dal a +tro phy +ðŁij ½ +mag in +margar itas +ê · +ww fc +fa sh +di ke +cu d +char t +ðŁij ® +refuge es +jop lin +n cs +imp y +firm ware +pas cu +flam in +health tech +bell letstalk +w aka +ol ls +la go +co wan +bombar dier +sh ome +ðŁĻ ħ +mc master +na ve +well s +u ta +tell ers +mis fits +kap il +face off +af firm +a pro +whit epaper +super yacht +speci mens +al located +... , +- __ +ka w +dachsh und +djo ker +s work +qui ere +or um +ðŁIJ ł +som m +c mt +ingh our +skin ny +lgb ti +gi ggles +break away +resear ched +par ity +my al +ms l +re tained +si vity +make inindia +sol ves +defam ation +wal tham +sri racha +road way +concep tu +al in +iw ant +å Ī +del ft +tender loin +ga ins +faul ts +sw ire +st ellen +pol lo +dy ne +bornon thisday +asdf ghj +sq l +sali m +advis es +vo ip +ìĹij ìĨ +un touched +she il +ontari o +uph ill +so bre +de shi +nov ella +du tton +craw fish +ا٠Ĩ +ma a +tw ine +kal in +ðŁĩµðŁĩ Ń +ye ss +brook s +hoo siers +ton ka +umbrel las +ay ers +ate am +acqu iring +su ction +ä n +wi es +tari ans +soci o +mat tb +shepher ds +o so +charity tuesday +s logans +ninj as +al bat +by te +bash ir +trampol ine +mydayin la +i ja +bas el +ror y +gol die +fi rec +un noticed +pecu liar +sch a +ker son +mour ns +liquid ity +qu ipment +hi bs +ar s +aeron au +slide show +sla bs +delici ousness +sk itchen +hta fc +full erton +cre ighton +aer ob +procrastin ation +az ores +white hall +uss occer +medi ation +djoker nole +and me +um en +noxi ous +jo ss +ili fe +anni vers +sudan ese +et res +under mine +whole foods +diso be +kor i +ade le +eli z +can ti +al on +gymna sium +sarko die +meteoro logist +yl de +ste en +stamp collecting +nas al +lo tt +fran ks +ex ol +ack i +good year +animal rights +y les +vio lets +mm es +s thel +ra pping +tu scan +wai ver +tur ner +eat local +northe asthour +anim ations +tom morow +t sh +ff ame +bra e +pe tron +glam our +br yn +d cs +bal es +ðŁĶ ¶ +bro v +bre v +b ons +physi que +car ne +x e +elix ir +vol ved +l oma +ìľ ł +æ ĺ +van u +ri gs +bal ance +va res +bon ita +sprink le +perfec to +di on +le ak +calcu tta +o ba +d ma +c mon +tun er +pneu monia +bo gus +apolo ge +cl ough +bor ne +)) )) +revi ved +o varian +ner f +c legg +fan fest +cho u +reali zes +mc n +li gu +leg alize +just saying +for ster +bo sni +k hi +in dom +hei del +en cryp +si ss +ed di +mar bles +brisban e +y ing +pre paid +wal sall +cooper ate +orche str +mar isa +ho wie +che wy +bren ner +andro meda +e gan +sto cki +cav endish +ag an +ban o +de ir +go g +bl k +re thinking +ch ig +rhe u +sni p +p eng +semin ole +m swx +an nex +lyn da +lewisham ilton +cu mul +tb l +dolph in +agu ero +........ .... +pre lude +at our +gr anger +too ting +ro tun +dis ar +home items +da res +**** **** +ðŁij Ĩ +compre h +jin x +as well +iri e +circul ating +ðŁIJ ¥ +over board +cultiv ate +rhe tt +oriente ering +ca k +bal kans +s itt +jas min +britney spears +ro tor +se aling +g bc +oc ci +f as +eman cip +com er +war time +tic kle +son ny +pac es +log g +at rix +sr p +g win +do bbs +uz be +the wanted +dru sh +ex tru +m icky +honore es +dar win +re dux +mm j +ram i +jalape ño +io c +do ver +ju ju +whit ney +s eng +en ly +au ch +archipel ago +vigil ant +man gal +wil dest +parano id +hal i +bb ly +sanc tioned +real ms +con co +u ddin +c sk +play time +libr a +sav ag +oc tane +rec tan +re turn +par rish +mor rha +cc p +c mu +sa iled +se vent +ro sie +pil ing +he w +boar ded +seg ments +neph ro +( . +cr ats +bak es +ðŁį ¸ +back tothe +sibl ing +kirk land +ke o +gu wa +bre ads +ðŁĺľ ðŁĺľ +t q +haras sed +ga u +wil bur +j isoo +ep er +li sam +tri ppin +sh ino +ru kh +beast mode +cho a +inst aweather +rich land +gar i +fe z +cowboy snation +fur suit +k run +a en +sycam ore +se gun +ent ennial +di h +o ax +demsin philly +ðŁĻ Ģ +sn hl +pen nies +pass words +ma kin +ty e +d eng +kni gh +jeep life +hel pline +a for +zz zz +ste amy +pic ker +iter ate +happen ingnow +ki b +bloom berg +martyr dom +bul ly +assor tment +a hora +zo e +no i +illu stri +agar wal +p sc +electr onica +recruit er +gar diner +rad ha +naf ta +dot net +pi ero +geor g +bel s +ðŁĺĤ ðŁĺį +tuberculo sis +run nin +mor is +haul ing +ev oc +bre thren +sha ir +frame works +a stu +ri gid +ku ma +kre me +jin nah +insu rers +ny u +f ere +nol lywood +good vibes +- ... +toi le +sk ril +instaweather pro +cze ch +pa vel +one piece +nike plus +fi let +cav ity +ðŁı½ âĢįâĻĤï¸ı +ðŁİ £ +dra stic +dail ys +siam ese +re bu +oste o +lar k +f re +sh elling +p é +glad ys +ðŁıĢ ðŁıĢ +gusta ve +submer ged +grand stand +att u +won t +f pv +b ley +jon i +ang ames +weigh ted +al ou +ठ¶ +les bians +f j +anni es +am l +dor ia +dav in +be ta +can c +madewith unity +ha j +bad lands +mu l +blu ec +pa wn +cov ington +neuro logy +htt weets +dysle xia +thel ove +ne at +fork lift +autom ate +une ven +monte ss +he in +ha g +rel ics +competiti veness +can elo +mar tens +bullet proof +sk ittles +g ya +pri mo +americ afirst +woo o +abor tions +?? !! +ma che +ld ers +rl ly +preli ms +direc t +cour se +swa in +super cell +ec centric +sting ray +ple ts +wil cox +west in +okan agan +kir an +car bo +bomb ings +ra rest +bo h +gaw d +di gg +mo ana +enti rety +en closed +dodge ball +par ton +milky way +at r +thorough bred +re ally +qant as +epiph any +ine e +aero smith +spi eth +ar thro +ell ini +du bu +bra ving +âļ½ âļ½ +re structuring +illumin ate +equ ili +mp i +ash ton +pony tail +ma scots +flat tering +cru m +ast a +à® ° +stranger things +bar nab +ر ÙĬ +make shift +got cha +will am +cho irs +kilom etres +gho sh +eu than +dol ly +un ning +the ar +cre we +w sw +j ace +dis miss +ke an +ho ta +kh at +~ > +thir u +ren dez +hart man +tee ssi +cas ca +z ah +hydr ange +fo d +aw p +mzan si +thick er +nago ya +ne va +sti que +cast el +dam ian +there by +ji ang +ale k +music islife +ra q +calla han +gou ache +somal iland +sean hannity +ra heem +lo se +elo ve +whar ton +rectan gular +illustr ating +har ne +auti sma +scra pped +ell and +decre e +nag pur +ki pp +so re +n md +ma as +gun a +gart ner +bel li +then ight +je on +gendere quality +gi ver +a el +gar ments +ne u +mardi gras +mar sden +ro wer +pollu ted +camer aman +vin od +be asley +cro c +ji u +hollyo aks +anesthe sia +al les +ste ward +lati mes +ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ +tic ian +gor ia +come dic +ðŁ¤Ķ ðŁ¤ĶðŁ¤Ķ +nai ve +sli ons +ł Ī +bur glar +ðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃðŁĺŃ +york shi +se ñ +fan boy +lau rel +inci dence +potom ac +rober ta +presi den +pr yor +os bourne +w ku +te me +pal ae +ðŁ¥ º +re boun +itu de +red dish +k hand +coloni alism +north carolina +ðĿ Ĵ +manne quin +lady bird +ta sty +knowledge able +g shore +ðŁĮ Į +à® © +qu aker +salz burg +med alists +chy na +bridesma id +ma ori +ro p +outra ged +in adequate +truck ers +al ana +ìĿ ¼ +ri x +oooo oooo +command ments +lam beth +aa j +eco friendly +bla z +morecam be +boun cy +rou x +rai ded +mi zed +sh c +gaw x +labor atories +ru bs +rest room +consult ations +ca jun +virgin i +so ir +rev ue +ple in +wag er +ç ¹ +we do +growing up +! ðŁĺĬ +face ted +sin ners +ho vering +ti ene +seas oning +an ja +leg go +il is +fla x +dev o +ash ram +mati sse +ker i +go wer +bo tox +mar shes +unh cr +ts m +opti mus +dun i +stu ffs +so k +order ly +n bad +islam ophobia +raviol i +fab er +cre ds +won ka +in fusion +over weight +daily news +assi mil +acol lege +medalli on +kili manjaro +sti ff +tham es +sun ken +th ard +my dubai +hilari ously +han nel +plu mber +fair view +separ ating +rasc al +qui en +necess ities +confeder ation +ll ll +: ] +weak nesses +bron co +ra ffles +el ot +ãĤ¸ ãĥ +advent calendar +ðŁİ ¹ +stra vel +tun ic +k su +im peach +e spionage +! - +di ment +cur rant +bio de +commu ting +by ron +ðŁĴĵ ðŁĴĵ +shad ed +tr uro +cray ons +ar ne +h sc +fre aked +dram ati +fle ek +u cd +marl borough +^ - +cross ings +mal o +black ops +bin ance +cho ked +chen ey +pl o +ge stures +val edic +ryan air +rem ington +v cs +mc kee +ec z +be gs +nail art +mayor of +happy fathersday +war t +pet itions +n ingly +clean energy +bro x +sl alom +exist ent +ab ay +ug liest +tom p +stom a +sel by +goal scorer +ben ji +overwhel mingly +lan s +semiconduc tor +south korea +re scheduled +sk yl +en listed +dow ski +si del +rosen berg +nas ser +white head +pri us +har are +en n +ry der +í Ĥ +mon g +clas ico +transpor ter +po tty +is me +** *** +vic e +sk it +ode ssa +l mp +her n +raci ally +pin oy +paragu ay +obitu ary +go es +bu cha +side walks +angu lar +un constitutional +transiti oning +i bu +gu ys +un packing +oooo oo +black girl +ber gs + ¯ +wordof theday +trump train +thunder bolt +m si +fasci sts +ठ¬ +t sk +collap ses +raje sh +loveis love +migr ating +set back +ðŁĺĬ âĿ¤ï¸ı +t els +safety first +nar rated +jae joong +un answered +lique ur +en nes +dal go +bill ings +salt water +mer maids +lon gs +clap ham +we arec +pic collage +n ach +h ace +pois oned +lo th +ag na +adel rey +guar dia +poli shing +peace keeping +d all +p isa +la pland +process ors +de andre +so bs +p once +dra ins +c be +ðŁİ¥ : +spla sh +meat ball +fon tana +worcester shirehour +ne v +bri sk +b int +ac r +po x +cay enne +skril lex +j fc +hahahaha hahaha +gla s +en gul +tempor al +oni zed +con cre +com pose +vibr ations +plant ers +fer t +criticalrole fanart +t bli +sch allenge +huck abee +munici pal +iam bic +radi os +ne vis +dura bility +mc cla +horse back +inst itutes +ful fill +atta ch +ate ur +ak an +resi sting +illumin ation +hand le +hair care +om ent +macle od +ka iser +g no +bear down +ly f +gl omer +distor tion +z m +san k +roo sters +is now +as ports +ag en +wo ken +st george +ro mper +my le +econom ists +ru to +t will +health and +d ito +ws l +tair p +pra kash +mic heal +h ts +w rights +kat su +fioren tina +defen seman +d itch +var sity +texan scheer +ba ham +sc anned +we il +seduc tive +ðŁijį ðŁı½ +fu e +er win +dav ison +ter ran +moo ds +wool f +re source +@ . +cu sh +ðŁį ° +regre ssion +cur led +la zer +jo anne +ab bott +mo z +down ers +mm mmmm +valent ina +k hair +dream t +cro ok +che k +ste aming +nephe ws +cl eric +as ober +indefin itely +w ye +us news +joy ce +flu shing +wynonna earp +ron do +kis s +hot dog +bar ns +sax ophon +far ley +gas p +decre asing +al way +pe x +l sd +shi ft +p outine +ra zz +rescu ing +ni ko +ho ch +cc l +u aap +n ts +m car +il wx +conqu ering +ket tering +stur dy +delay ing +sto k +vani shed +cath ar +bin gham +in v +ic hiro +he mo +budge ting +[... ] +be ss +sebasti an +slow ed +ðĿ ij +musli m +stun s +acton climate +ve a +se ton +rose tta +oun t +hard in +flu id +ca w +ðŁ¥ Ĥ +yach t +un l +sp hy +provoc ative +or ic +is back +__ _ +nicol as +gy an +loo se +fl in +reb ate +: :: +! "@ +com icon +she ff +down stream +chic hester +beach life +mom life +diabe te +ar ra +van e +ok u +ye o +man go +try out +app ell +he irs +arjun a +dd u +na veen +movi c +soci alists +s back +criteri on +soyu z +k her +da z +yol anda +wine oclock +re ina +one w +leon ard +en dez +u bs +support local +facilit ated +carameli zed +b pa +vuel ta +my tho +m ami +spe are +nbap layoffs +fe vre +nick jonas +im print +c so +craig slist +la salle +gi deon +ha doop +dis regard +w ud +tu c +ma gee +acou stics +ta a +qui e +pol a +cr t +dw yer +dis sec +capit ol +men tion +kn oll +he igh +fin ders +plac ements +l se +indi ra +gur i +madhuri dixit +kingdom s +iambic pent +geor gina +je ky +conflic ting +bay an +aga tha +uph old +dr on +vic ar +ex pat +periph eral +pe ssi +fa f +ance stor +? .. +wid get +pun c +comm enced +beav s +air waves +ad dis +po a +de sses +co den +vu e +ru pee +kar in +spo ck +m sy +ภ° +pr ick +fill more +ti fication +thing sto +sar de +em ile +pere ira +n ad +bright ening +arre sting +wo king +usc g +sp ill +raspberry pi +hu go +ite c +is ma +cuff links +optimi zed +oc c +mi wx +en ka +el ited +afford able +sa kh +coron ado +ho h +at ul +ai oli +jim cantore +accoun ted +vin ay +her mit +groo ves +ran ch +r illa +we tter +ou tof +veter in +ni kov +ki an +fair banks +ram apho +n iti +k ko +ru sty +ne stle +tv xq +shahe er +âĿ¤âĿ¤ âĿ¤âĿ¤ +penn ant +gem stones +dem debate +ðŁIJ Ĭ +auton ews +support indiefilm +mach o +ve x +new sat +ne ti +conce ssions +can died +yof the +mac au +den ds +cricke ters +san iti +mari ano +gh at +ar toftheday +¡ ľ +e gos +gen oa +chat bots +bri er +al labout +mon ty +spi ed +r tr +comfor t +sni ppets +real time +gra in +exam ined +en lightening +tt u +god bless +release the +sing ular +ki ans +ha ka +sor ren +defe ct +mar g +equ ities +d orian +su ka +per l +aishwar ya +pul lover +preci sion +fair way +ne ve +rive ting +vill anova +en com +ak o +passion ately +europale ague +siem pre +x vi +enligh tened +c fr +âĺħâĺħ âĺħâĺħ +wast eland +is f +new comers +emergen cy +amphi theatre +- . +text books +figur ative +tre mb +pe sc +ab hin +ab bot +ac acia +har ds +por sche +kau ai +el isa +car rick +abo u +elli er +be ch +neu tron +galap agos +ru ben +in nis +how to +nun s +sab ine +i ac +clin ched +no tori +fi ves +cairn gor +per i +gr c +ðŁĴ¯ ðŁĴ¯ +mal m +twelf th +di ff +rout ines +marty n +lin den +synthesi zer +nu mber +game cube +fal kirk +byz antine +queu ing +gr ill +scal able +char red +rou ting +her bali +gri zz +ðŁĺŃðŁĺŃ ðŁĺŃ +tol l +termin als +l pc +ab d +war mups +remo vable +¯ \ +vi go +pap aya +ne ve +lov ingly +jo kers +ib les +sse tt +poten ti +pel e +gi gi +sadi q +leg acy +son o +ru pees +retar ded +ele e +par r +fi ance +ey re +say ers +pend ants +mak nae +al bans +adap ting +p ff +pu berty +ji u +ing rad +hypocr ite +diplom ats +phys ical +rob by +bon sai +ãģ · +f att +catal unya +âľ ĸï¸ı +ro ma +more land +so e +conver sions +stl blues +shol m +gra ssy +pra do +on u +assaul ting +> _ +sett es +dis graceful +aph ra +âļ½ï¸ı âļ½ï¸ı +ठª +kil n +goal tender +s ru +philanthro pist +b als +th n +stu den +sando val +dogre scue +eli ons +asse ssed +lar go +hec tares +sh rm +sa if +cle avage +no ches +n ene +fat alities +cur ing +clean ser +al es +p vp +south bank +pizz eria +marsh als +kni fe +an dover +tbli ghtning +sr sly +ou te +digi mon +timesof india +prome the +le bo +f su +wit z +rever e +man as +mam ba +ch ica +gu an +exhibit or +csr racing +d ere +xx xxx +gu sta +story time +ston ey +organ ics +and u +se am +min ogue +anushka sharma +ab a +ðŁİĻ ï¸ı +ugand an +chro matic +as sn +document aries +sh t +ru paul +loy d +k ats +e us +ite ch +me dusa +pan ty +kel logg +et to +talla de +sha a +do st +p ms +mari ana +je ster +croo ks +ðŁĶ ¬ +min danao +ind hoven +ðŁ¤ ª +le xi +tv n +jan is +co te +ãģ Ĩ +ser rano +iw m +ðŁIJ ¬ +k ke +distribu tors +cap u +counterfe it +camp site +ag gie +ðŁĺ ¼ +chhat tisgarh +~ @ +state u +san di +prevent able +cl s +can ne +mm c +i ver +sa haran +pal is +night out +do s +ap ia +absc bn +manag erial +aro se +mo wx +aro sa +ðŁĮ ³ +under dog +remo ver +astronom ers +lent ils +su scep +smoo ther +pend leton +fau cet +e mory +dal mati +af cb +tic us +exem pt +en rol +d heim +ðŁIJ º +restric tion +star fish +sto w +snor kel +thunder birds +she ad +homo sexual +dy n +as li +andre tti +dou che +dom o +tar mac +slu mber +pr onto +first dayof +mini ature +mari achi +argu s +recomm ending +mobi les +in ce +illustri ous +or c +adver ts +gr its +wea sel +pag oda +over pass +gre ys +maxi mus +arma gh +wood land +sun ni +ðŁĴ ī +ë Ŀ +ti one +soci o +ho s +ðŁ¤Ĺ ðŁ¤Ĺ +wind sor +subsequ ent +munch ies +id h +exclu ding +e mi +cu th +z ai +week days +law suits +barn ard +Ø ª +pe tting +net es +mul ligan +pharmac ists +ra quel +e ton +cran ston +gil ded +cle ary +ce ph +ra a +pam per +lombar di +as in +sher ry +pro d +for te +ari anism +buffalob ills +æľ ¬ +ðŁĶ¥ # +uu u +just ices +car ina +nat in +mas low +dro oling +cog nac +cam ber +el ong +r dr +in en +convic tions +am use +tro ck +harm less +visit ation +gen omic +bl and +beno it +chim p +tuscal oosa +gre asy +x po +gil t +se q +per mitted +christma seve +book s +mu e +old school +human right +be ati +ðŁĶ Ŀ +sh at +sculp ting +h wan +fern andes +sci utto +fu entes +endeav ors +maid stone +un paralleled +shou ted +queen of +mer c +band ic +ve da +sel angor +pi le +ja han +intimid ating +disapp ears +cl ich +za ha +w urst +hi v +fod ils +cor dless +aaaa aa +hy dra +bel inda +e els +bu f +su staining +rugby league +no c +brig itte +( ðŁĵ¸: +tromb one +soo the +smo g +ad p +stab le +ing ley +diagno se +ms g +we ss +tic keting +one e +nsw pol +e up +auto psy +adity anath +sun down +river front +si ya +p is +hier archy +dur ango +di jk +ren shaw +he aps +epide mi +david bowie +interne tof +dd i +nation ality +mb ar +air y +win der +w alia +elli ott +c x +bav arian +pl att +an tw +wi wx +sof ter +ne ha +h eller +th and +dani ela +bo ast +degra dation +ðŁĴ¦ ðŁĴ¦ +transform ing +man e +av ut +ðŁĺĪ ðŁĺĪ +vo ter +the e +t ate +pu ff +in door +sop roud +boy ce +boris johnson +wait in +immun ology +ðŁıĨðŁıĨ ðŁıĨ +âĿ Į +street food +liz asober +cavali er +c elia +need le +motor ing +g ato +, ) +ra de +harve st +t ms +jar pad +on ey +air men +v re +impair ment +abhi shek +snoo p +l ant +fam ously +bl ou +s ze +g ander +un touch +tu f +dee jay +col lateral +b ind +ðŁļ © +pin ning +ic n +' ; +the economist +ul tram +worldwater day +ti poff +the i +feed ers +campa ign +sc umb +day weekend +yo m +pe dic +h ough +ps v +pl in +on de +boston marathon +az zy +* _* +con ley +thi ago +hoo o +gal erie +luci d +je tt +gl itz +final fantasy +achiev ers +y ung +peregr ine +op hi +dam es +biom ar +âĺĢï¸ı âĺĢï¸ı +sk c +l ics +fl ank +ar rahman +ho of +uphol stery +t ats +wo z + ¿ +snor ing +ra er +l ju +ap d +pl ating +kan u +im ation +fragr ances +m ra +mor ay +mo tt +im muni +hearti es +bho pal +tim ers +g ata +color way +car nation +win get +si ghs +s ville +optimi st +chate au +olympi ans +ci o +singer songwriter +ny o +fi bers +bur ch +ag ro +mil ne +ig bo +cr amer +ation als +dan ube +pad ma +nor mani +en forced +bre ck +boeh ner +ar den +sur rendered +pros thetic +om a +ha iled +calcul ations +w fa +bi b +fcb live +fon da +west coast +que sts +friend ly +to wie +fit ch +bal ot +star dom +scrat ching +ho sa +thi ka +o ven +stro ke +out post +pharmaceu ticals +hi kari +mu y +af d +fallon tonight +squ at +or u +dra ined +chocol at +ë¯ ¼ +wor ths +ri b +mu j +that s +residen te +it el +boo st +mi gos +mul led +la a +etsy shop +don keys +me k +p tc +flin ders +e hs +ro hit +mu ir +g ad +compos itions +åĨ Ļ +combu stion +i kh +yemen i +wav ed +gar ci +ak os +oo ds +fu sion +se que +s lan +pl ur +kic chasu +shenan do +s ams +worl den +horo witz +with me +mic robes +k ki +ðŁĴĶ ðŁĴĶ +w su +patch work +fre er +y aki +the art +symboli sm +mil er +bt n +ma bu +side kick +motiv ates +sag itt +natur als +serv iced +ps ori +pa ola +qu ig +i badan +gi ggs +ë ³ +sciento logy +si oux +salam at +d res +cad bury +d hawan +ci ón +_ ' +swa pping +maris ka +james bond +explo sives +ay les +af er +s agu +cen sor +tom a +jeff erson +ring ed +par tist +ir responsible +aguil ar +vac ay +equ itable +altrin cham +ac ur +man ish +ger min +schoo led +pu tter +ed ad +nav al +toast y +sol areclipse +dish u +coy ne +ac co +mu ck +mar an +el os +len der +cro ix +worth less +ha ber +gun men +ðŁį ĵ +zen ith +t enders +hur st +hol tz +itali ans +car low +u cd +characteri stic +bun g +av l +u th +sa sia +rs l +red man +neighbor ing +green peace +sti ps +follow party +y gk +en os +omni bus +na issance +chri ssy +secu re +call back +ji hoon +memor y +block er +l anta +daf fodils +bil t +ffer ty +fau st +ie c +nipp les +so g +m nd +jagu ar +bol dly +ab poli +pro position +gun sense +evan sville +cu tters +we go +dou n +do x +stal lions +ka j +shi ppers +j awa +vol o +le ven +pap rika +kov ich +jor di +induc tees +app alling +dial ysis +allevi ate +âĢĶ âĢĶ +pie ter +mid wi +q tr +juli ette +inter mission +haw ks +act ment +one ill +k lin +vam ps +fam ous +cou ld +autom obi +da an +west end +elli p +nh c +mel anch +web series +ton gue +snat ched +smy th +tan gible +sl i +e asing +bar stool +over lay +afford ability +ting ed +ter as +ay ush +wanna one +rh ine +dan a +sh ana +kend al +fer tile +w ir +repl eni +lar vae +is ro +con vos +ab brevi +u cc +hun gry +bur rows +ag er +nav i +mat in +du per +cer n +ma don +ķ ï¸ı +é ģ +tu ps +hy att +sh ep +friday night +wis er +hei di +hat ton +p gh +foun tain +wrist bands +ahmadi yya +aeri al +subscri bed +so los +m ace +sla yed +for fe +dul ce +christ mass +arun jaitley +viol ate +ob stru +ni eces +w vu +idy l +fa ze +pre serves +infr inge +premi ers +inter vals +agen cy +( © +stand alone +di mes +bo er +param eters +ge tit +ðŁĺĺðŁĺĺ ðŁĺĺðŁĺĺ +tu lane +for given +scol l +mb ps +smash bros +rob bi +prima vera +ali st +ghost ly +ay at +ye ats +impre ssionist +ear phones +caul field +wai kiki +sal ute +sc ou +mu ay +louis vuitton +bak hta +ado g +inven tions +hur d +forec lo +stream line +thalai var +ch snews +will ard +t sn +euro parl +cru sher +my sore +gro wer +ra ping +pat ti +g den +sm w +muf ti +kid man +ab r +soun ders +skep tical +ðŁĶ İ +sun dar +i me +fer g +feather weight +ar lington +pas qu +ag azine +wearab le +nati c +mccl ure +inter mitt +hor de +six ties +car te +bha v +ze al +experi ential +ador ned +som mer +eno te +hypo thesis +stin ky +pro to +dead lines +vo gel +mus ings +monc ton +gu ter +f le +aci on +voice of +ta sha +inhabit ants +type face +s ba +bts x +ðŁĶ Ĵ +wor x +u hc +jo ko +cell ars +gor o +continu um +... & +weather cee +ha p +sr k +ris ers +lonely planet +un named +co eur +ðŁį Į +the world +ili ke +fa sten +ami go +ri ba +ramapho sa +staf fers +had ley +? ?" +fi ore +sal ut +hu ff +bez os +Ñ ĭ +ra der +kam ala +in line +fill ers +um atic +all in +shat ter +re in +o ku +ch ases +fla gged +baby metal +water stones +ts b +cut out +op hel +aam a +rockab illy +sto lic +jet blue +ich ick +down ton +uzbe kistan +pat na +la q +gr ange +) _/ +subsi di +sc p +newsc ast +it sa +twee tyour +e mor +archae ologists +uni fication +por ta +q x +protec tors +pro hib +charis ma +car tag +ren fre +scul pt +guwa hati +de ma +boo p +unf pa +dex ter +lay la +alleg es +sou ps +never again +l ys +cal c +bar oness +visu alize +ger ber +absor bed +i ers +a han +fon tein +detec tors +verst appen +sv c +formul ated +ac dc +li x +in competent +bh k +lour des +water house +snow ed +appreci ative +sig ma +lizasober ano +pen ned +pay check +tall inn +fanc afe +par isi +av alley +vi g +ru fc +hard ship +so cute +po ise +ì ¹ +roth schild +k ly +???? ???? +l hp +il ay +f hs +am ad +ide als +brad bury +bal boa +nic ot +kid nap +wol ve +tas manian +op t +matthi as +ãĥ³ ãĤ +super markets +mylittle pony +me lee +li ster +gr oun +fe dora +kind ness +en en +bra hms +¯\ _( +ros well +mar lene +ic u +re formation +or ail +he brides +dispar ities +terrac otta +swal lows +re id +influ encing +flu or +den e +tum our +blon des +thunder bird +sh eva +moga dishu +ka b +cre eps +i ving +ene ed +anno y +âĶ Ģ +intri gue +enqu iry +ar aj +tur al +kuber netes +end lessly +divi dends +tor a +ti sh +commemor ates +un ra +tri b +pon ty +ne m +diss ent +brew ingco +ðŁĺ ½ +nor mali +bi of +( ... +chil len +ì£ ¼ +mell on +av is +mccor mack +ing ra +enrich ed +custome rexperience +testo sterone +snu g +sett i +ger onimo +inqui rer +bre aches +very thing +bloom ing +mu ra +dispo s +bi de +de va +shade sof +in trin +sh ev +s ven +nayanth ara +gan esha +c ws +ber ta +label led +use um +nick named +ma han +car uso +ap ur +ðŁij Ĩ +w q +orphan age +discar ded +mag nu +lu e +je on +bridge port +pac ing +mercur y +( ðŁĵ¸ +marx ist +amphi bious +transplant ation +stit ching +then burg +gradu al +ãĤ Į +ro ft +ma ils +ine c +guy ana +dopp elg +ver o +re write +head less +harb augh +gate way +car sforsale +sw i +st is +mach t +un de +sura baya +stap leton +nur turing +mil ner +ya o +lma oooo +ko sh +arsen al +k ame +er ry +ar royo +dis misses +ru bbed +rc b +lew d +dil u +and or +vi de +ur in +inter sec +ha ar +al b +year swith +app leton +é al +ul livan +suc cu +monter rey +d mx +artem is +ron nie +farm land +s football +gro tto +anth i +ãĢ ģ +à® Ł +vid ya +jimmy fallon +ൠį +t zer +gravit ational +w thr +u hhh +e hr +tin ker +ti juana +scran ton +ram charan +bar clay +re van +m si +ka p +wr s +we thenorth +tor al +sat u +gro m +fac ep +erick son +z yn +se dge +oo dle +spur sofficial +ds p +sic ilian +soli hull +recei vers +ladak h +hend rick +ther i +presi ding +mc guinness +litt ers +gun nar +gh oul +wi b +n tv +kar o +fro ck +b lau +ampli fy +all is +ul lah +memo irs +kh loe +intercep tions +pet day +lo oney +con fin +ch ay +piyush goyal +frequ encies +ut z +event ual +warm ly +obli vion +an ka +ta it +âĿ¤ï¸ı . +director ial +ru lers +prince s +mu ck +stur ridge +deu ce +abri dged +bagu ette +un cles +pen du +min ding +forre ster +av ila +wall er +wall street +ment or +hin o +high way +crom well +fanart friday +mb i +co yle +a hi +tro ve +spie gel +pay tm +mcin tosh +jan sen +nit i +nash ville +len o +leicester shire +le gos +dic t +ðŁĵ ½ +sp ad +beverly hills +sy rah +separ ates +z ain +un fit +dra gs +tan ia +over flowing +hri thik +haw thorn +z ani +mac far +fi de +to tem +pe ds +fundament ally +cal ico +sin ner +j ä +hil de +ds d +ten ay +ta hit +mil f +lie b +inform ing +up lift +ra el +mortg ages +lec t +ii ii +guillau me +compos ites +old smobile +l end +gar th +com mish +bapti zed +scorpi ons +ru cker +bringback our +alli ance +thalap athy +tal i +sp ans +eri dge +wither spoon +lin da +sky lar +kor n +hom s +Ä į +sil enced +caf fe +ar ty +dist inguish +to wed +pun g +jessic a +ear nest +beau fort +t ama +study abroad +si khs +new bie +nav ratri +mar ble +loun ging +lit ter +dal it +so sa +iz es +gra de +com promising +tr iton +de tta +v j +chau ffe +spec tral +powe red +montess ori +artic ulate +hal ton +al co +ye y +mn twins +acoun ty +ðŁijı ðŁı¾ +âī Ī +mad men +kal a +gru m +chi k +ati s +su me +akh tar +job search +high lighter +bo ath +âĦ ¹ +tar zan +lam bo +âĽĦ ï¸ı +ox fam +dump ster +pretz els +mac os +incl ined +fac tual +adverti sers +shu i +pu ree +ml pfi +anti dote +cap o +pa str +merc ado +but ton +ar min +ag g +lol la +horri bly +er rands +christop he +time snow +monday motiv +li ss +scand als +mc i +dispropor tion +âĺ İ +sur pass +samar itan +so tho +pu rest +fl att +trivi atuesday +delec table +leop old +hermi one +chou dhary +en rich +¡ ¡ +subsi diary +ine qualities +bachel or +auto immune +la kota +i hop +ad jec +the simpsons +sh es +se k +gret chen +up stream +hin akhan +coper nic +x tina +lu g +tough ness +e ad +cli pped +bi us +sl v +fah ren +dee pak +ca u +x an +im mature +dig ni +bo bs +shred ding +but tery +accommod ations +de ven +chun ks +super league +sky bet +kil dare +je et +ë į +ce k +wrec ks +pro pane +oh l +tb d +quo i +trum pp +mi mo +reluct ant +ver ne +o ic +ma gh +ar nau +se ver +li dge +stair way +kicchasu deep +ðŁĶ º +mach ining +aama admi +ot i +c da +al it +pan y +inst alls +ac ct +e shop +di em +hard well +fulfill ment +sc afe +qu ack +extrac ts +swee tened +fi ghton +f di +d inger +wal tham +us ur +refe rees +seok jin +gran n +af rin +th n +sch af +par cels +bet is +amar ine +nom an +kh tar +mor itz +cou pling +bar ons +ðŁIJ ¸ +à ¸ +sl p +sad ler +x ander +tri ad +mc millan +kh z +divi ding +ìĹijìĨ Į +dar yl +zed d +le ys +pla ques +flu ori +tipper ary +on nell +di dier +lang ford +im c +the sun +bir dies +ar cha +ye ssss +t di +dar ia +cand ace +al tam +pal aces +ch it +sant am +event ful +book of +ad b +mon stax +cre ole +co el +âĸ ½ +we aren +sten nis +she ath +ati sm +gron ingen +mlpfi m +le pre +wrong ly +rsp ca +rendez vous +acknowle dging +pel vic +solic itor +sla ys +nue stra +lo d +is lander +fer oci +fashion show +ra ss +dge on +adole scents +sma shes +negli gence +grate ful +ved ere +sw oop +ing l +apol ice +vand alism +gan n +jo ao +di supdates +zimbab we +under age +radi ance +w of +bour geo +pla s +cr ani +gh ue +wrec kem +warran ts +re form +jim mie +at wood +ys l +neil himself +l bj +i man +tan to +nois se +ver bs +equip o +al together +mam ent +l ice +dou glass +tier ney +pri med +j hal +furn itu +braz ili +v ill +past els +n ison +u ff +paral ysis +jay e +im po +ðŁij ģ +strate gically +pakistan is +was sup +super bike +thank u +tru elove +sha ikh +israel is +vi p +to g +li en +la ker +grey hounds +cul ars +bian chi +balot elli +ar ran +loo s +str ates +he bron +ar vo +sunder land +the al +tomb stone +sand man +c pac +thanks giving +love him +lat ino +an in +aka if +ĭ ãĤ +tor quay +di est +alli anz +ðŁĺ ķ +golf club +cl lr +wal cott +sch nau +promp ted +nomin ating +len nox +val et +mon ro +may ward +e ph +ðŁĶ Ķ +inter oper +r da +re flex +arm chair +ê° ķ +stri pper +por ti +ph arm +ham za +ni reland +ne ue +h pv +port foli +sun burn +fris bee +be al +bapti ste +x h +ty m +pr ati +o vers +haz rat +deser t +der ry +us ky +em mett +ach arya +)_/ ¯ +shu d +may a +ham ill +ra im +nr c +fitt ings +cur vy +ðŁı ĩ +ster ling +ॠĢ +wal kin +short cuts +mil ly +ast ur +alpha be +pl i +pe z +miss you +rad ford +ml g +ta eyang +notjust lakes +du mps +seren dip +le ur +ra ving +e ster +de priv +absc bn +ðŁijĩ ðŁı» +scar city +o cr +mean ings +cap t +da hl +fer mentation +bri oche +to win +out lander +massi mo +en cro +ðŁ¥ ³ +buil t +po tam +kir i +tm w +monit ored +k ites +peoples vote +gray son +íģ ¬ +afri ka +a dies +i vote +gy ne +g annon +di x +c mc +ou ral +fox andfriends +bel i +ig ne +gl an +katrin akaif +co politics +qual itative +p si +lu cci +disc oura +âĺ ® +kel li +gau tam +carac as +reale st +pu la +in us +hill top +make aw +atten borough +tw y +r arity +peck ham +ma hon +corn elius +clin icians +ton line +tb i +paradi se +ka si +inev it +fresh ness +colling wood +lun atic +defen se +cop d +in fra +wain wright +sains bury +alab am +te ma +lac o +chec ker +releg ated +tren t +stal ks +huff post +bhubanes war +ast ral +share your +prim rose +hi me +cat an +end ment +en dow +cle mens +mal oney +hil ary +game time +den ise +collabor ators +b wo +radic als +gue tta +ici on +au a +snap matic +sat chel +excav ation +base man +s ão +gn ation +fel d +surve y +shah zad +ma st +anirud hofficial +tru cker +ot ago +geo graph +ethe l +âļ¡ï¸ı âļ¡ï¸ı +s ver +mu tt +internetof things +ancho red +wh ouse +bang la +bal main +ç¹ ĭãģ +break fa +á Ģ +twi ster +te tris +ca v +stag s +g z +au b +stor med +hel ens +yar mouth +st asy +gustav o +co sc +vin son +up p +sc ricket +assump tions +app e +nu h +u er +pre mise +n aga +e amon +coron ary +na f +north side +el mer +ro tar +out lining +el f +re surg +kat elyn +in can +hyster ia +ce e +am bani +pro lly +Į ãĤĬãģ +ax es +san jose +rem brandt +mag pie +even ly +scor sese +qu aint +f g +b buk +indian football +weare all +spd wy +pis ces +ec g +âĺħâĺħâĺħâĺħ âĺħ +pre orders +: | +ni pple +sal azar +ju me +jail break +min n +bas sett +ze tta +jef free +ad jun +tic on +san diego +drink local +chol era +solic itors +o bo +com post +ni an +wr a +tre ach +ic ic +profession al +del ve +leg ate +histor ia +cro issant +con noisse +nam o +palli ative +chem trails +i ority +global warming +comic art +behavi oural +re sted +li as +cli mates +Ł ãģĦ +rut land +nou rish +menopau se +hot ties +demen ti +ve spa +mel ville +anal ogue +tz man +str ung +im perfect +gl are +cir cling +ros berg +rec o +oc ity +lo ire +em be +do ssier +ne el +nan do +me a +gal vani +fin esse +ag p +berke ley +asi m +âĺº âĺº +quil ted +ish ere +un matched +po tion +for z +at re +selfi es +juli ana +ðŁļ ¶ +âĸ º +mel ton +âłĢâłĢâłĢâłĢ âłĢâłĢâłĢâłĢ +spin rilla +pur cell +ed p +at leti +tony awards +ra ja +pro gno +mol ten +stu ff +p ally +nobel prize +âĻ» ï¸ı +spiritu al +spe ake +sa sha +bri um +tru ss +critici ze +assassinscre ed +yor uba +u lo +fire man +workin progress +ef cc +fla res +ro bot +hi kers +cl l +shado wing +pat sy +leh man +c ns +å ± +guad al +à± į +ra pe +r honda +paralle ls +son ja +langu age +land ings +z ola +cr amps +bur ning +apprais al +jol la +ham m +kas a +gul ly +f go +uly sses +ri be +ðŁĴ Ħ +ib u +eti enne +bri ar +fin ely +comb ating +y ql +go tham +we chat +to paz +primar ies +l se +iz z +hel e +dispon ible +cy stic +bel ichick +th rush +kansas city +ge om +soli di +red bubble +by stand +cambridge shire +par fait +ast le +ow o +ind ore +stom ping +sm elly +ðŁ¤ ĸ +locom o +adm itting +hol me +clock wise +min sk +mc co +for get +ev p +cam ra +ab ella +yo tes +universit yof +mé xico +silver ado +ric ket +crom bie +pu j +eradic ate +deli ght +y go +glam ping +vic a +du ggan +coun ters +cf d +sc our +react js +pu ram +paras ites +in ki +vill en +stel la +li mbo +ang as +k cr +ðŁĴļðŁĴļ ðŁĴļ +vap ori +mum ford +oli gar +à ¼ +al oo +boo ties +ad r +k elli +dru mmers +av ici +nature uk +ron al +in trac +un splash +le che +g oma +el ine +envir o +bi onic +bu eno +mi k +av in +star ling +em powers +cake day +boy cot +ðŁĴļ ðŁĴļ +ðŁĮ¸ ðŁĮ¸ +v ach +m ci +fractu res +ger i +sk ing +exclu ded +lu ce +ja ve +ig gy +evi den +aki stan +a wn +mor als +luci fer +ha ban +tumb ling +sunday motivation +mo sley +captain america +sch icago +the one +mo td +d ts +ðŁIJ ¼ +rep ell +ii i +locu st +geo spatial +mer sey +immer se +desc end +ber nade +j s +boat sales +win der +cran k +sing leton +candid acy +ben a +ðŁı» âĢį +high lander +ol t +k prs +healthy lifestyle +four teen +end the +ith aca +circul ated +r ans +pre valent +ha vas +splend or +roo ster +kalamaz oo +jewell ers +enne dy +rou sey +es y +cann ons +ornam ental +// // +ren don +win ne +mol ding +eid mubarak +coun tess +simon a +ha wa +fo es +du ster +sb u +por tray +mar ries +goo dday +cho co +achi ever +ðŁĺ¹ ðŁĺ¹ +pre neur +tr amp +tom i +n bat +garden chat +farra khan +ever glades +ab ru +sou sa +se ce +homes wee +terre strial +bar it +sri devi +ol u +mel inda +f rick +can dies +ðŁĺŃ ðŁĴķ +qu reshi +family fun +exor cist +cardin al +ny t +dies el +cu mulus +capric orn +si ology +lor na +dou gie +an die +super sport +c fl +п ÑĢи +say ang +pe ek +ภĬ +lo be +j em +ing lis +gg led +c sn +amne sty +chu ps +ba es +sau er +ðŁı IJ +mongo lian +en et +back street +dr illed +acce ssing +ce o +b se +ai ken +pur r +wor sen +whe res +war k +testi fying +bu ri +bla st +aw g +ðŁĵ ĭ +re defining +hear ing +u ci +c mp +bon i +tail oring +ta ji +noc chi +em t +stephen king +ne et +compla ins +campaig ner +luci ano +twili ght +ti esto +pas sports +flo yd +cathe dr +na ked +caregi ver +b coz +ade cides +ku ri +ly k +br aries +dren ched +disc lose +ðŁĴª ðŁı½ +le blanc +je tty +gar ty +chip mun +b su +rhyth mic +ic z +fri d +anne x +ame x +solo ist +lanc ers +arro whead +speci fication +simul ated +na is +inver te +bo wing +wor ship +f z +abo ss +sha q +ì¶ ķ +challeng ers +an arch +aamaadmi party +ãħĭãħĭ ãħĭ +suffol k +so corro +sn ell +cla dding +absor bing +shaw a +particip ates +ðŁį Ķ +book stores +bak u +seap ort +ko jima +gab y +pack ard +electr ician +let it +mo wing +fa wad +young jae +hot mail +men ing +u rie +intim acy +con ti +: ") +lifeis good +in ciner +i dri +craz iness +jour nos +fran chi +bott len +al da +ff es +k x +south we +air a +clay ton +sco ti +f j +bri ga +ðŁ¤ĺ ðŁı» +demonstr ators +y z +stor k +na q +casc ades +travel chat +plat a +pad ma +fran ci +at tain +bat girl +lom bard +hoo s +d dos +neon atal +discla imer +r ss +r ant +di sen +tex aste +so cal +frac tal +cam ry +stri fe +sn acking +mu h +sant ander +mor ons +gra f +par ades +hu ston +dru pal +mi ento +kir stel +hy de +vom it +forti fied +sphin x +da v +bir yani +win nings +s baseball +mer ged +lovel ondon +ling ering +dream big +car leton +liveli hood +djan go +astri d +gri ds +down e +bru ised +s ne +scarec row +hel ium +f nc +bi ggs +an ter +restor ative +em pires +ab del +life style +kiwan is +colloqui um +me en +pr ick +anti que +ze b +mi mic +edmon ds +ðŁijĬ ðŁijĬ +q ing +pp el +mc gill +interpre ting +âŀ ķ +rash ad +do ka +narr ator +electro magnetic +ash by +sau ra +iran deal +âģ īï¸ı +krish nan +in di +ff en +bre a +os man +multin ational +chi ppe +recruit ers +aus biz +p ounding +re gen +cur sor +refu sal +mac s +in ak +ax ial +wa ifu +up cycled +hindu stan +cas sini +carly le +scrat ches +re ef +man atee +eat ery +ðŁĵ ¢ +un condition +sen pai +on ther +comic book +pro sciutto +de mar +mi se +ma ge +fre ec +aye sha +al der +android games +ley ton +ho ck +door way +chicagof ire +aali yah +sw elling +bi x +. ðŁĺĤ +evan kirstel +torpe do +kon stant +genevie ve +ma ia +ha user +do torg +hide ous +fi k +sp raw +e ek +z appa +wan dered +' ' +ra jan +bam bi +( $) +wid ening +tool box +sa ir +illumin ating +pra ys +out patient +i w +day o +lo b +sw fl +sha des +gu ms +coo kin +ko di +gri ffin +traum ati +ste a +slaugh tered +god bless +air time +pseu do +b sa +hau led +ar if +à¸Ńภĩ +le l +wc po +mil iti +char ters +worl da +ru k +k gs +digital india +is able +idyl lic +esp ino +marie tta +e bo +team canada +ab our +wil ton +rock stars +fav ored +phys ic +wrink le +tb r +d print +ball arat +ad al +z ey +ðŁĺį ðŁĶ¥ +tom lin +mt r +pal sy +fener bah +tight en +phil ia +ir oning +ry u +b ant +enqu ire +ca ir +abur ger +tru n +green berg +chau han +ir ina +sh ani +trend setter +pre tt +zaf ar +alo ve +v ici +pan ic +no o +lu stre +disrup ted +bal lis +son sof +mon si +inst ac +ake st +ëĭ ¤ +kw ame +horror movies +distric t +sau cy +mb an +ar mies +with drawn +med ics +loft us +er oom +be kind +ar ns +all on +un ison +davi ds +cr at +nicot ine +so or +sm x +on co +cospla ying +zombi es +har ms +e ger +ro sy +moon shine +fe in +ce tt +du brov +reg ents +ben itez +ðŁijıðŁı¼ ðŁijıðŁı¼ +ste c +m alia +prioriti ze +ic eland +ft se +v amo +lam ont +homo sexuality +bre es +regu i +cb p +te j +sky sports +deter gent +sha sta +de rel +conserv ancy +colori zed +accol ades +vis o +show your +nan ow +bice ps +us ability +bi m +dailys ketch +pearl jam +stran gest +mega deth +broad casts +bar ren +ar ton +chri ss +confi gu +lu res +is the +e ul +railway ana +global health +gi anni +u aap +s lum +consci ously +ab re +n up +bud get +v ada +e sch +real ness +er ased +th unt +be z +armist ice +ðŁij ¹ +sh run +o led +driver less +ðŁ¤· ðŁı»âĢįâĻĢï¸ı +won dr +sk an +sal aam +mother land +h wang +gen o +gang nam +tw right +endor sing +en ic +ador ation +pau sed +patric ks +do cked +plat te +ff xv +ethnic ity +auto show +side show +after life +re located +orphan ed +food network +dare to +and ra +sla ps +v live +swim s +re imagined +mist le +re vise +real ity +bhar ti +ðŁĴĻ ðŁĴĽ +late st +prou dest +gra sses +lan yard +fresh est +carcin oma +anom aly +zieg ler +sum ner +ly rix +gor g +is d +av el +swild life +me squ +john cena +euro league +sab er +master ful +yar ra +cogn ition +jacob son +abo lic +sir loin +shuk la +moj ito +su pere +st weet +me z +e sa +rudol f +gur a +where you +tt m +win s +trust worthy +ny k +bra den +table top +good food +es on +be k +lingui stic +gra ys +ch ath +h cs +mon i +de ans +cu ssions +ch ell +slo ws +he mi +d app +shar pie +boo sters +a os +str ack +se dona +mu eller +hard wick +or nate +thor a +sal ud +o twol +ch um +mi ho +for age +thel ittle +tear ful +ones elf +min dy +sm g +gmb h +emer ald +ðŁĶ´ âļªï¸ı +tu tti +recep tions +re vising +i brox +tope ka +sal ami +expan se +i books +dob son +cli o +at s +ðŁļ Į +mo ha +is ance +shu tters +moo t +jan ine +marvel comics +jor dani +pos er +kenne th +hy ung +de ja +ase ball +speci ality +eu ston +classic car +had ith +ðŁIJ ī +chas ing +iz o +gros ven +ag lia +thisdayin history +t row +om ile +hu ar +by n +sal ine +div ine +demon ic +ty ran +han dover +revit alization +pa ella +cryp tic +se dg +m end +dun kirk +bre d +wal d +sport scar +a ard +whe aton +da ener +k lan +br t +bakhta war +spi res +schu bert +ro ti +poli sh +o se +ag ame +wonder con +prote stant +bo sa +ðŁĺ Ł +d ü +joy ride +ger trude +âĿ Ŀ +gil a +v h +tw a +tra v +swal lowed +star ve +la in +ent ren +rei ki +su kh +cra ic +az u +web page +kee fe +hypo the +hir sch +hel le +camp ground +w amy +tra vi +sha hi +san deep +ru i +han uman +dw p +reposit ory +no or +no ff +un real +p ell +black history +har vick +ma scar +pay ee +pa sha +gastron omy +d ÃŃ +ai g +rosen thal +open day +embelli shed +t tip +sun bathing +go pack +end ome +ï¸ı # +invali d +final four +st fu +squish y +ra sta +mo sch +jam esc +die trich +sel a +mel b +el vi +t dp +sun i +sli t +j ha +bi za +spi ked +l li +l illard +vam pi +syno psis +az har +kendrick lamar +ĮãĤĬãģ ŁãģĦ +heart less +country file +air play +arrog ance +pre e +virtu oso +ãħłãħł ãħłãħł +raj u +le bu +for ward +tu g +dro s +mondaymotiv aton +concep cion +thel o +pad i +looo ol +ÑĢ од +it ss +eth ical +end uro +__ : +expend iture +mon ste +mas king +terri ers +ib is +e mber +cu mple +punctu ation +pi per +ir vin +ade e +yy yyyy +flash backs +cel sius +don nie +bo gota +ben evol +the script +shil pa +pro se +fin dia +ze ke +ne ko +do ves +blues lyrix +fro sh +sowe to +mp lo +al ai +sab i +raq qa +wf tv +stro ller +ian somerhalder +ðŁĶ ª +an on +mo seley +! ?!? +sta king +mol y +car tri +c sg +ast or +transc end +ma er +de ux +cow girl +sas k +pun ter +ma ken +o ates +love tt +grow ler +sag in +v n +ssi ble +officeof rg +y mc +sab ar +faul ty +ap ha +ak on +ðŁij « +snow don +ae w +raise the +ðĿ ĵ +grue some +clement ine +sp ing +lat a +worlden viron +mi mic +can aria +bakhtawar bz +ao a +fal a +ãĤ Ń +avi va +you uuu +thi gh +la dders +gu mbo +tz ky +fu zz +plastic pollution +est ate +strength ened +k ant +dr in +cal vert +transform ational +frigh tened +mac lean +elited angerous +ear thy +t son +to da +j nu +.. , +mic hal +i ban +je ong +is real +sim coe +exclu sives +blue bells +ben e +te u +pil sner +pens ke +athe ists +m pu +cartag ena +ðŁĴĹ ðŁĴĹ +million aires +kk kk +it ar +subscri ptions +remo te +ma fi +hin ton +w cc +ho k +ds b +ab leton +sevent y +pun ks +e indhoven +sh one +mcfar lane +lim popo +empha si +à ¼ +sin fo +pe tre +man grove +ch ino +ber tie +play lists +push awards +p af +deb bie +c do +r ino +ðŁı¾ âĢįâĻĤï¸ı +fol ke +bon nar +th ine +sl an +hal ter +evi e +aw some +vul tures +spar ky +seiz ures +âľ Ķ +ram one +ine ffe +al n +pro ctor +ast ra +the voice +gro te +sci on +dead line +am aya +tain ted +patter ned +exce eding +cross fit +kay lee +drop box +ru shes +tack led +mo by +retro gamer +n cbd +benef itting +shay kh +guild hall +gen try +dream cast +dread ed +bun dled +th aw +revol ving +n pt +kylie jenner +imagin ative +ron i +over came +family time +ds burg +car naval +relation ship +recogni zable +cor oner +ho le +fan fic +emir ates +bur ritos +analy se +thin ner +ne es +galli poli +bl r +cat woman +-- >> +au lt +ada ily +nau ghty +ili o +solit aire +mtv br +jocel yn +arun ach +rep ent +south gate +hy acin +essenti al +fent on +and um +it or +go pal +sl inger +po sei +aw il +wi elding +ra ila +eli as +a sto +à ¤ +tend ency +str ata +ker t +< - +im acele +da es +sti mulus +han ley +fit nes +ec stasy +lim ous +ha iling +ðŁ¤ Ń +chis wick +tar ies +sla v +pul i +moderni zation +black mail +b ingham +h fx ++ + +ðŁĩ®ðŁĩ ³ +ni v +we a +profess or +k off +bol ster +su ave +sequ ences +pepper oni +not te +dre n +ãģ¨ ç¹ĭãģ +hs v +o ga +ap tly +z ad +excel si +rin ka +mol dova +min n +ma bel +conferen cing +bas ing +of er +ob si +hamill himself +care less +brief ed +inhe rent +par ish +dub nation +town sville +sar awak +gee ky +doncaster isgreat +was abi +gu p +phen o +dra inthe +carrie underwood +ble eds +bbc world +ane w +alta f +dul wich +ani ston +w ti +sumat ra +gra fton +bl n +me ster +bode ga +re go +es q +an jo +sump tuous +mai sie +ï¿ ½ +wil t +jak ob +el vis +se pul +mu ster +air pollution +president e +happy monday +exten sively +fl ondon +t ls +play ing +pe ed +din ho +var dy +pi ka +n iro +au cus +ðŁį ¦ +nu ll +el ondon +juvent us +imag ines +dis ab +lit o +d ura +work places +promo te +mc caf +wood work +waw x +à® ª +tt ino +shar i +sem per +better together +ðŁijĬ ðŁı» +ze bra +pon dering +en chil +ho m +cosm ic +tan z +mo cked +ec cc +ath ed +abo lish +prop eller +paris agreement +assemb lies +indu stry +fraudul ent +pe sa +chang min +ax x +ðŁĴ µ +irr ational +cu sa +ramad han +octa via +on elove +jac ki +bar ak +taxi der +seri ous +nathan fillion +mc en +ch k +po part +grav ity +copp ola +reading fc +illu sions +j ig +ww x +re sh +ex porting +buzz ard +âĻ ¤ +p cm +lan apar +ko s +arom as +antal ya +ww dc +ven a +phil a +ball in +ðŁij Ħ +quin ta +ma o +f ery +eigh ty +sentim ents +safe guarding +r wa +pu ffs +luc ille +de cath +sl u +nu gent +de ter +braz il +ze iss +super bowl +subsi dy +alter n +hi dalgo +enz ymes +ä ½ +tag ne +hair dresser +adri en +walk out +oppo ses +can tina +bed side +af an +ðŁĶ Ĺ +prophe tic +dan es +un successful +super charged +pk k +exem ption +hart le +secu lar +cli pping +br s +united way +c net +pat chy +ha gan +e en +âļ ľ +var a +sym pathi +never trump +affir mation +om f +ny cfc +ma ja +sur ro +keer th +up scale +sandal wood +mon archy +kno bs +å ĭ +po tholes +hunger games +ter races +na sir +coun sell +welcome to +wa q +se aman +m ita +stun ningly +on theroad +in ability +) !! +bon go +ant v +sp ut +worldenviron mentday +resu sc +y td +fi m +eun hyuk +sa chin +rose anne +cler mont +ape c +am ina +v ening +n antes +al most +sin us +ex as +ty l +ti en +ple ad +lanc s +bur naby +re k +jo om +observ ers +disco graphy +cl g +âĻ ¦ +sn ack +r ti +o ily +crystal li +bru te +web development +topp ings +la f +an is +ad der +reli ving +car lin +battle of +we g +syri an +pon t +n dc +lagh ate +yu ma +sp p +p iti +ro bbing +mart ing +rey kja +raj put +nc ds +kie wicz +âĢ¢ âĢ¢ +vam pire +substan tially +opio ids +nepal i +k line +ar oo +under stand +lit t +u it +thro mbo +sar ies +qu ot +b alling +t tr +s gh +philip p +br ant +ac l +m ello +whit taker +. ; +defi ant +b gc +repl ying +mir ren +metamor pho +sch wab +bul ge +utili zed +pick ering +par don +d sa +ภĪ +doo ley +cumul ative +Ð » +ur gency +e mir ++ /- +¦ Ī +ot as +âı ³ +station ed +grape vine +ar ac +karan johar +f ancy +sau l +coo gs +lgbt q +ا٠ħ +jav i +u mmer +pl l +den is +dai pur +pu ffin +lewi sham +fand om +co pe +ves matter +s ve +hel pless +deo dor +ostr ich +kaz an +friday the +con dor +v x +sophom ores +rob les +cu tt +cli mbers +ë¦ ¬ +sle g +sn f +mac ys +hydr ating +grou pe +po yn +mou lin +hg tv +lmfa ooo +sulph ur +asdfghj kl +annab elle +hump back +bra ved +viswas am +multi purpose +hu midi +escor ted +barb ican +f ad +cor sa +ðŁ¤ « +pi ppa +here to +can y +ser gi +or cas +o vie +ed ou +s any +glob alization +man cini +food truck +f is +defi brill +sch re +sma fia +love wins +la ut +k aka +hol lande +game on +resurg ence +out side +olympi ad +int an +abstr action +rapi d +pal om +cal le +jas min +attack ers +swag g +mit ra +ky lo +à® ² +her mitage +gor do +e ira +so sfam +roll out +exc ite +sy nod +mer rill +c als +as sa +liveli hoods +ju ve +the black +gopack go +ant lers +alban ian +wool ly +qu iche +puri fication +are th +smar thome +ne k +all blacks +mex icans +is m +ger ms +comple xion +mar ck +u shi +ðŁIJ IJ +char l +ca stic +till erson +giuli ani +biode gradable +mal bec +bo is +ju bil +im es +r ame +gene tic +esp nu +ch ley +so ho +go pher +g sc +buu ren +cu be +bridesma ids +webin ars +to e +mani pur +viol ently +notic ias +ex changing +chi ev +replac eable +muay thai +bu ss +sp il +instal ment +div ya +cait lin +o lim +fil tering +whirl wind +sta red +prior it +pr am +pompe ii +mono logue +k ite +bu ka +âĢ¦ .. +vac cine +bre ro +woz ni +sol ent +re ferr +my rt +gridi ron +galatasar ay +fro ze +clare mont +ðŁ¥ ĥ +victori as +ssel dorf +pa stures +net neutrality +ch or +ðŁij ģ +ಠ¿ +we ho +symp tom +jo sel +in ous +dragon con +power ball +p te +four thofjuly +ec la +ear buds +where abouts +salt life +depriv ation +ch ter +wi ggle +syste m +ps st +ch az +d any +ri mo +oax aca +lanapar rilla +barcel on +melanch oly +way back +ho tro +n si +l illy +kur o +ja han +intellec t +board game +ðŁı Ĭ +sneak peek +k prc +jail s +cand el +zan zi +mor timer +star ch +ra gs +p fa +long live +k art +gir ona +cro cker +christop h +precau tions +war ship +per m +paren t +van gogh +gif ford +allegh eny +ra yn +ut m +sten cil +rec alling +pen ney +z azzle +ìĥ Ŀ +hin ds +aren as +nu ev +law ler +gu in +do this +ðŁij ķ +ì¶ķ íķĺ +we g +ti b +ri din +complex es +turbul ent +pe sos +de marcus +vall arta +sam sun +kis ses +hein rich +deport es +wil ms +ur d +then ext +inki gayo +ho wi +fir sts +carri age +clean liness +mas war +is ch +ax el +si zzle +road house +fr ans +ent ourage +co bble +boo th +benedic t +tal on +fc u +year ofthe +ray on +raider nation +fo yle +ko val +pi anos +l pg +bur mese +man ure +geo caching +cosc ino +b np +fer ra +stro phy +mar ais +ce es +legen dof +kat niss +eno ch +av ed +you know +d prk +ðŁĺ¢ ðŁĺ¢ +sp un +pro st +sor rows +cent red +ke a +gal icia +? ðŁ¤Ķ +ÑĢод а +bou chard +ðŁĴĻ ðŁĴľ +yu i +seed lings +jon ah +reco vers +ny rd +board room +su ma +my japs +tun g +sha i +ir gc +eli o +wag ons +ka shi +polic emen +john nie +ale coscino +shop ify +dot ted +de tri +va w +to fficial +in your +chal mers +trac ed +no vi +by es +ari el +nipp on +la pel +gri ez +b gs +fool ing +d ita +vijay sethu +nm wx +as ot +kr anti +hel m +ve di +sic kest +mo chi +k abo +shru bs +he red +b sp +sq m +ham r +dul kar +anth a +nr f +avoid ance +at en +publi x +be arers +nas i +ha p +h ells +ðŁĸ ¥ +ภ· +thelast jedi +oh wx +ðŁį « +wa hoo +there se +rec aps +ss nhq +bird photography +v ay +pet ti +pau lo +bel vedere +( * +gr l +du vet +c pec +sa it +por sch +meas urable +avi ators +fre mantle +bre en +on om +me and +life saving +eu ref +en don +embar as +aira sia +el is +dun kin +star magic +s ill +porto bello +ki efer +ex e +mu ted +ãģ ¦ +we thepeople +logi a +liber al +theforce awakens +min ed +haun ts +freck les +care taker +s india +âķ IJ +dev lin +list on +direction er +oh n +fi garo +em manuel +du bois +cl ones +bru ise +ðŁİĪ ðŁİī +disin fe +der matology +as r +s watch +dis comfort +tam anna +pi day +mack en +k atic +delu sional +shaw nee +gu d +al bino +p ali +din gh +cucu mbers +coffe y +anticip ating +treas ured +web summit +shel tered +sav or +pedago gy +m gs +sh ma +s bu +den ali +cam pos +bubble gum +o ir +le aps +y ler +r one +sansk rit +min t +meat less +futuri st +du de +a vel +prote sted +squ ire +z aki +sz n +har court +cycl one +bour dain +gather ings +d ant +advent urer +parag on +alt man +dd ing +ban erjee +snorkel ing +mother well +mis sy +en der +glo ws +ki wis +chick pea +por o +e fron +app t +u y +speci fied +gab by +e strada +com bos +bour bon +vin i +var un +steph ani +key words +car vings +amit abh +wr ought +tw al +re els +clu bbing +ubi quit +cri t +ambed kar +æ Ļ +prun ing +vaccin ated +boe ing +s ks +lo ona +hypno sis +edel man +pho l +he w +colo sse +mckin sey +u on +to te +sacrific ing +ox i +n ang +e mu +пÑĢи ÑĢода +m th +kers wednesday +argu ed +timel apse +ris king +regul ating +ni gh +likeli hood +cu bic +au ction +rein for +pi stor +no ses +ye l +snu ggles +pe i +jean ette +ta ku +ri th +guy z +ภŀ +y te +ver ted +pay soff +jau regui +hoo ligans +procedu ral +mi b +har dy +el eng +chec kers +all ine +the met +prou dof +keerth yofficial +collabor ator +ni u +infl icted +adv ani +re twee +memor iam +f icial +ti ghter +sal em +re viewers +br ics +ben digo +am ell +tur kish +sush maswar +paul son +pal awan +mol lie +stitch er +s burgh +ir u +hay dn +en ers +aro a +u zzi +saraj evo +hel a +apol lo +nine ty +vac a +sp on +vent u +jel ena +hei fer +avo ids +sp ine +pri ze +mar ist +re creating +me de +woo den +find lay +ro fl +n di +compreh end +yu go +y ü +to work +u fos +son ar +pi ston +recor ding +tent ative +art forsale +pel lets +fre do +ÙĪ ر +mu ses +custom ization +pro found +is ner +ide ally +si am +plan kton +cm dr +man ger +fran ken +customiz able +ठ® +walk away +swi vel +vast ly +no ton +lex a +ex moor +z as +tan te +reduc tions +lol ly +hip sters +benef ited +ë ² +ww www +mascul ine +fi ji +dre y +ph ill +ane ous +nic ol +men dez +disapp ro +ch ner +through s +shen mue +east man +ðŁIJ İ +yu ck +under tale +re ys +go beavs +eng en +c na +mer r +bir k +ãģ¨ç¹ĭãģ ĮãĤĬãģŁãģĦ +âĥ£ @ +yn na +ste ed +offen der +at um +vani shing +presi denti +love them +g nocchi +fri ggin +per il +mad hya +ag ne +dee jay +mar nock +m tb +fold able +@ ___ +stand re +bron x +bow ski +fin ite +cro ckett +b sf +ge tit +seren awilliams +mir o +ignati us +sla y +rin se +fon due +sel dom +s more +gan i +dy ce +dmit ry +cru mb +late post +pri mark +oh ana +flor als +do a +remembrance day +d ds +azi one +toon ami +air port +æĿ ± +th ad +fi st +dine sh +dr who +ad words +admi rer +pro je +kyrgy z +à « +manife station +le wan +j ic +thi bau +le ased +van ity +nouri shed +never theless +aug mente +fu elled +che ad +wil shere +ru di +p z +my co +mor ro +herbali fe +hardro ck +de man +dre ality +sp ades +ce vic +bha i +bar on +ultimat efan +hou news +to bi +stru t +ke el +affili ation +the masters +sm al +hu e +este ban +con v +om nic +datab ases +co v +ter ti +st g +snoop dogg +metab ol +leth bridge +ðŁı» âĢįâĻĢï¸ı +year ling +residente vil +nws l +iy aki +griez mann +c ous +ðŁĵĿ : +tor ian +sam i +ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ðŁĶ¥ +g are +alli ances +whit field +we ther +refin ing +coy i +kra ken +ðŁĺĺ âĿ¤ +singul arity +lil i +h ns +bol dand +waw rinka +misogy ny +lo vers +c q +b dg +ad ona +gar ter +women of +sc d +recogn ising +mun a +str ou +sign alling +lare do +hell boy +alek sand +un available +pedi atric +as in +mer ia +ri shi +futuri sm +w ye +polari zed +e we +pro pel +in forms +cre ase +~ " +arti ston +like for +heidel berg +er ra +life in +len ny +inter rupt +cohe rent +ca z +vick ers +le veled +f bs +cab ins +bu mmed +apost les +we h +ten don +souven irs +infu ri +pier ce +asse t +m las +go th +di ggin +ann as +yl or +th waite +sw el +pan era +mur derers +croo ked +bs go +ac u +a on +re an +one of +ko hl +bloo dh +pest icide +lost dog +fle xing +ëĤ ĺ +su pra +eter nally +ðŁļ Ļ +pa olo +ol an +mom o +is elle +captain marvel +s lou +mistak enly +akhi lesh +mer t +il inan +bu on +bal kan +mir ro +mill en +der ail +dam on +tit i +bi os +re don +pic ard +par te +ðŁ¤ Ł +Ø º +son ics +fir sth +dd c +veg ans +tur ban +ni gan +lot tie +lyn don +star buck +pink floyd +life styles +am ara +a she +r sc +val a +sm er +cw gc +cli ent +buen as +jag an +coo ps +ðŁijij ðŁijij +speci alizes +snag ged +g lar +ben net +wildlife wednesday +bow den +pi k +art in +empor ium +ar l +re ba +pas ser +disappo ints +additi ve +âľĬ ðŁı½ +bay er +missou la +ha skell +comm ences +ni x +ne man +explo ited +plastic surgery +cc d +aso cial +vo t +sie gel +fro ome +kap am +far a +e ha +pro bes +mw f +meet ing +p bb +ak ins +mistle toe +kingdom hearts +for kids +ec r +bal e +escor ts +adidas originals +k wa +k ts +hallo ffame +ðŁĺį . +wag s +pot ted +o wing +honey comb +he fty +uro logy +mer le +b pd +stri pping +re ich +k state +gu ay +yon ge +shak ti +g loom +bat t +son om +n ery +el ba +blan ks +hel le +triple ts +bom bay +ak arta +ab ia +transm itted +rol f +ja is +angular js +fi erc +m ss +trac e +ॠĩ +tom bs +old man +kom bucha +fo l +e health +cere als +are lli +in ari +ðŁĴ © +wo l +liber ties +fa wn +af firm +nun avut +hyster ical +k drama +art es +âĢ¢âĢ¢âĢ¢âĢ¢ âĢ¢âĢ¢âĢ¢âĢ¢ +valent in +man slaughter +gal es +eo in +energi zed +del s +with draws +st les +sar castic +ram esh +incredi bles +lock hart +ya wn +ultimatefan live +oooooooo oooooooo +mu en +guru dev +te er +pe eling +new snow +lingui stics +direc tv +ag end +uni lever +ru ger +han dedly +ero se +li mel +the c +royal ties +fini shers +nr g +m gt +fid get +com ps +bac on +aggre ssively +ab it +ch â +tar de +slu gger +q anda +gre ening +d ats +ensla ved +spec tor +o ye +fre ef +b hand +stop brexit +mis conceptions +cav a +ðŁĺįðŁĺįðŁĺįðŁĺį ðŁĺįðŁĺįðŁĺįðŁĺį +multit asking +hou sel +ferre ira +cen time +ank les +jo dh +hel ly +fro me +out tuesday +nar nia +bal aji +l bloggers +jyo ti +ðŁį ĩ +lan cia +cap ri +y ap +nat ash +down fall +." âĢĶ +à ® +ligam ent +coat ings +ai ded +hi ko +fall ing +encryp ted +yeg food +infringe ment +cu di +ce p +ðŁĺį ðŁĺĤ +tra d +super rugby +ed win +wh iche +vi meo +lay ne +in vigor +he he +dubrov nik +bie ber +u tr +sham an +op ers +ham ill +en ig +di f +ar um +scrap book +min h +diver gence +mckin non +life time +guter res +wil le +ple as +patt y +mic ron +k z +dom aine +ru sher +m ds +ches ney +screw driver +âģ© , +sle dge +hau er +chan a +stam ina +sprink ler +pl n +he ff +bol ton +om on +car rington +accor dion +jor ge +inter ception +in puts +gu ll +tran scription +vanu atu +it ical +eth os +tic h +spac ey +pee king +u mi +ha ger +psycho tic +illi an +illi a +bonnar oo +an ese +pu c +laghate parth +en hall +econom ical +dre dge +% - +u we +tu bular +scoun cil +pe asants +fl er +tumb ler +he p +ford ham +row ley +initi als +ev asion +er nation +plu gins +coch ran +c attle +acid ity +ðŁİĬ ðŁİī +re grann +jump man +ef ace +x ma +patri archy +esco bar +cristi an +tip ton +nu eva +hack ney +back seat +kill arney +aid an +sta dion +simul taneous +ida ho +a je +u th +figu re +clo s +bur k +volun tar +rec ite +macfar lane +cur few +bou do +w gn +sti x +sla p +scrat ched +philli p +jour ne +ex pelled +wa z +u ke +tati ana +ou e +ho pp +dimit ri +ðŁĵ £ +mato logist +electri fying +blu ffs +bill smafia +az cardinals +y aa +x mas +shar a +r ith +g ills +dre s +bar ton +authori zation +imperi alism +home of +to do +foot path +band width +visit spain +moh sin +erup ted +mi ki +insig nia +mike l +ss h +ger a +bank holiday +aw an +t weak +star craft +e al +construc tion +skelet ons +le ep +ine m +bar clay +ship wreck +monsi eur +yo h +ron t +form ative +ser o +le p +horse man +hoo sier +haz mat +cylin ders +cen ti +ðŁĴ¥ðŁĴ¥ ðŁĴ¥ +re em +na ire +mus ically +gras shopper +est onian +termin ology +ro main +blogger rt +tox in +stan ce +cultiv ated +an ast +ðŁIJ į +shi mano +go pher +ene i +recycla ble +gam ification +fight for +c q +avoc ados +ke ys +eli ke +gly cer +shak ur +mobili zation +gal ley +expla in +ex changed +pe th +obe dience +illa ge +en nis +ãĥ ŀ +wi v +walla bies +ma ar +ig ers +fin tech +fin alized +wo j +meaning less +in field +onna ise +e et +bron te +pass ages +ðŁij § +strick land +northern lights +lom ond +h tc +wr ay +shi fter +di alog +ðŁį į +>> >>>> +te atime +ste ch +sic huan +qu ill +fran ca +comple mentary +bar rington +marcu s +mal am +goo oo +for sa +elec tra +af s +âĹ Ĩ +tri fe +sn azzy +fo lia +and olan +after dark +wood son +stra de +litt lest +o gun +con wy +co wards +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤ +íĬ ¸ +se ul +mur phy +dun ks +kapil shar +jo achim +wom ack +equal ity +aver ages +a ine +ðŁ¦ Ī +tac ular +dis ability +u ked +mid century +bar thol +teas ers +tab ern +nj caa +sp out +op i +ku bball +bl om +so ar +popu lism +meth yl +ðŁijĬ ðŁı¼ +o spre +alo ils +ðŁĵ ĸ +ðŁĮ ļ +x er +sp illing +publ ica +car dam +adi sh +sa cha +p kg +bu da +lyric ist +i bc +gru mp +ho ver +hal ep +anti body +anem one +âĻ¥âĻ¥ âĻ¥âĻ¥ +m cl +litho graph +cc u +s fest +path ic +calli ster +otta wa +gun sn +rut ger +hali but +en vision +differenti ate +ðŁļĢ ðŁļĢ +pir an +lat el +uc n +trou bad +ra ine +fierc ely +learn english +lea se +wex mondays +em it +dray ton +bur rell +scuba diving +hol ler +dr u +clo cked +w ral +ap ro +trans lucent +w bo +patri arch +mo ja +lan nister +fish ery +ne derland +mil dly +mi rai +ma ko +ja p +ðŁĺ©ðŁĺ© ðŁĺ© +pro statec +p anna +ar ama +under taking +tomp kins +ne op +soli ds +sav oury +e ames +cut lery +wood bridge +steam er +ri zzo +wild cat +rat na +lamin ated +kin eni +jal ap +ai des +acknowle dges +?! ?!?! +! ðŁİī +w afc +mag gio +ha ves +dar je +of i +gr il +v asi +bru x +mo hd +fake speare +arn old +r mb +for be +wal leye +ro di +therapeu tics +strate gi +ob ste +mu dder +download able +dd ings +d ca +asi angames +campe on +appropri ation +th century +ram atta +dra ped +bul lion +mu c +one x +se greg +ophel ia +bod ily +âĿ¤ ðŁĺį +wi zar +te ased +ade my +to id +sur a +lazar us +sn ickers +ma se +lo h +bow ed +bibli o +x change +har lan +gho shal +flavor ful +bha gat +alle z +whiche ver +ten stein +disc er +organ iser +mt g +dream liner +t se +hok kaido +mo k +indulg ent +hick man +blin ded +al yn +aaa ah +sp ool +lough borough +inter pret +et v +aristo tle +optimi zing +avici i +madu rai +ju li +naw az +mat chups +ab ide +paint ing +w elling +vel i +octag on +in scribed +po king +plac er +life cycle +kili g +g sp +eli ves +cle ments +na sheed +me sut +incarcer ated +dist illed +wal ang +delic acy +del gado +che z +ch ita +ad ero +tu x +pati l +o do +abh cosmetics +tv c +p bc +in accurate +hardwork paysoff +ball er +quot ation +merchandi sing +ga stri +defen ses +dro gba +bex hill +ban kno +win ona +si eg +p gs +hahah ha +agu chi +su bram +mirac le +de sch +li bre +ba cher +ent ine +bbcra di +lou dest +r ps +pi erc +fr yer +storm trooper +rafael nadal +pas co +exhau stion +epic onetsy +rc tid +kel lie +ga ines +d bz +sm riti +s bridge +lim ited +cla w +technic al +bio graphical +ado red +ภ° +exclu de +ac adia +key boards +fur man +so ca +sur u +ni ps +sw aps +server less +run e +pu ffy +north ampton +nish ings +hen der +cartri dges +gun shot +ðŁĵ ¹ +fil ament +respon dents +pey ton +mountaine er +mer ging +life span +intimid ation +p afc +nl wx +expan sive +pur r +f ck +ca e +at ti +tele thon +so hn +mend el +lo pes +dor i +un broken +te red +tast ings +in active +disin tegr +t assel +share the +pi ano +is lay +air space +z awa +ricci ardo +ming ton +fresh er +cur ry +re vs +pharo ah +h mv +exhilar ating +wh oo +lin kin +kri spy +competen cy +ste wards +ne bu +kat su +ad mins +baz ar +as ar +giving back +s summit +song z +lin us +raj kumar +farm ington +fanta sia +ðŁĺ´ ðŁĺ´ +so bri +lis se +barry more +pri sm +blo b +sen ew +mono xide +exp ire +eigh teen +di pper +xi ao +kil t +hin ch +bbc sport +bam boo +p ter +ex al +ðŁ¦ ĭ +ham lin +expe ditions +star gazing +food security +wy lie +ul f +st ingly +on storm +lo eb +bro ome +bn ha +pancre atic +eli ve +!!!!!!!! !!! +ther apper +ortho pedic +avengers endgame +antit rust +ìļ ° +go te +om d +off side +gy llen +win eries +white water +ad l +lu pita +exce eds +consi sted +chew bacca +ash leigh +nhl jets +is san +sh ld +hay at +cran berries +ðŁ¤ĺ ðŁı½ +rock the +spring training +fall out +dairy free +wa j +un decided +so wn +rc n +north wales +htt r +fu mble +d its +comp elled +popu list +min ted +blan chett +. '' +pro pulsion +m illa +au berg +her tz +h ta +u daipur +serendip ity +azte cs +als ace +ðŁIJ ij +lu n +sho es +char li +gar za +ðŁĴ Ł +pro biotics +fox tv +ol is +mi ff +loc alized +diffu ser +si gue +fun ko +rend ous +ðŁĴ ij +jeky ll +ha bib +fre ya +fjor d +ex porter +to sa +store day +maj id +ba the +cham paign +ðŁĵ Ĭ +der ma +h ittin +gor illas +emo te +ac ic +mad ly +lland ud +kru eger +eleven th +ash raf +umm it +at as +per sie +mo tives +i ona +finger tips +ss m +pon te +bri g +rb is +tu sk +ps vita +jor dyn +ci el +bas ket +are d +arbitr ary +go ed +chron o +sand box +performan ce +na ke +ant our +vas quez +quad rant +mat tis +ìĪ ĺ +sa har +numis matics +ma this +tr ams +pot w +as quez +? !!! +thro b +of life +_ ! +pan tone +mcil roy +er u +ma sto +endu red +co vent +ab hi +physio therapy +civil ized +ant asy +snap dragon +on screen +micro bio +l cc +di mple +sl ough +ma ven +col m +villar real +or p +fr ye +bar u +v tg +perio dic +concor de +childrens books +ym ru +re mark +je w +u tica +seclu ded +rogue one +ag li +why we +ro bu +nur sing +lu ster +automobi les +ic um +cl i +sagin aw +pean ut +ec ra +transp ho +bl ins +aw wwww +âĻ¦ ï¸ı +jere z +inst ances +sy on +s de +wp xi +rob ben +man x +journ al +erne sto +belle ville +as ur +wal rus +h j +cab le +blizz con +bean ies +vic inity +te igen +ta ire +pa v +navi dad +extr ater +bun gie +bbc papers +algon quin +zanzi bar +out fielder +mer ced +m q +kon ami +sho ton +hor rendous +ad vo +spoo k +nbc sn +tu tors +en tos +sur name +pay ers +mul der +be be +radic ally +bu eno +re brand +it ching +fer o +zo u +la i +gon g +weather network +rick son +recon naissance +f sc +differenti ation +be stin +y q +st as +lon gre +pro fan +mar ac +opol is +ba its +ab se +sy r +ph us +u don +schizophre nia +reg gi +jen a +deto xi +com plac +z b +pr t +ibrahi movic +bm j +seduc tion +oooo h +gonz alo +w ham +sha p +deser ts +callof duty +ðŁķ º +photo booth +bri m +si en +scri pt +cas par +line age +x ero +may bell +co ta +carls bad +ðŁĴĥ ðŁĴĥ +im ba +the car +law x +il k +g ana +sli d +ma halo +g ant +enri que +te i +jo ck +bla de +h la +i hs +west on +trans it +au bam +lone some +kobe bryant +fun ky +v andy +sh aka +an an +person alization +rede emed +hat ter +day s +par ac +living stone +for man +de mar +ðŁijıðŁijı ðŁijıðŁijı +per sia +pe der +ðŁĩµðŁĩ ± +reli ever +ith appen +dc p +den burg +Û Ĵ +k assi +un pleasant +ij u +far o +car mar +ke ren +ha u +scot tie +s bury +r sc +pistor ius +mp ire +mo at +mar uti +lion s +back country +rash ford +haras sing +ze etv +t la +tor so +sau d +ent ang +ef abz +toshi ba +resi des +âĿ ŀ +r ct +mohan lal +memor andum +hor ner +bon neville +g sd +exoplan et +blasphe my +am et +ðŁĴľ ðŁĴĽ +spo iling +ma as +ka sey +coim bat +ðŁį Ĵ +tu ske +su zan +still water +mit z +keep the +gosp el +dum best +distr actions +ch lori +ãĥ ī +sophistic ation +mm u +lithu anian +bell ingham +ðŁijĢ ðŁijĢðŁijĢ +strongh old +mon aco +k ad +dog sofinstagram +ðŁij Ļ +west ward +sedi ment +pal met +ko de +ki do +nom ads +ff ff +augmente dreality +ðŁĺĺ ðŁĴķ +upro ar +ty rant +sty lus +sli e +deli rium +occu pancy +hat t +hair stylist +ear tist +spal ding +never mind +read able +p met +fac ts +ot to +she im +sch am +go thenburg +ex it +ty n +tam worth +roof tops +mutu ally +j mu +fis k +cun ning +re news +me tic +an tho +mcel roy +con tre +ab ank +mi les +deser veit +dear born +ab ir +cruci ble +character ized +tahit i +mur cia +che tte +uni vision +pres se +love e +im pun +ast ana +a au +o vs +loo sely +ell ing +echel on +connor s +n th +ty ch +jim bo +cor don +app reh +. ðŁĺį +jiu jitsu +acol lins +sushmaswar aj +strike outs +proto types +ascen ding +argent inian +ren ner +# ' +j y +ðŁĶ¥ðŁĶ¥ ðŁĶ¥ +nanop articles +iz ers +! ðŁĺĤ +por cup +edwar dian +dx b +.. !!! +mil king +f ours +the d +ðŁ¦ ħ +writing tips +sim ms +ele mental +whis kers +rain er +ou che +influ x +å¥ ½ +snap chats +pi eter +g awa +c nt +ley n +slaugh ter +k lay +ger m +bon ne +ðŁı¼ âĢįâĻĤï¸ı +wic ke +i at +border line +* .* +ent on +ou ss +yas min +tow son +roll s +ho ho +bant am +skill z +cl o +sf u +conden sed +school boy +london ers +ãĢ ij +vand als +sat oshi +ðŁĵ» : +sin cer +ni etz +i awx +grey son +graph ed +gabri ela +je p +ha di +fron tier +ellu lar +con fluence +ðŁĮ ł +white out +mer it +shi ra +sculp tural +incar nation +life guard +mi de +bar rio +attribu tion +app re +re eve +mag ically +din al +broad casters +tend encies +su bb +reykja vik +min ts +goe the +shi i +aubam eyang +:- / +ี à¹ī +eat ing +du mbo +oc key +ber tha +am ata +aa g +evacu ate +hu tt +dr f +what aburger +tb acks +li vin +ap an +vo a +vi kas +grand mas +inter fere +dor itos +bon ner +f gc +pi ñ +per mitting +limel ight +de anna +le ela +ha st +fahren heit +ale ssi +ðŁĻ ĩ +lie b +dee zer +cul tura +vo ss +pa si +ma ud +is it +bent on +din ers +theroy al +refu eling +ent ro +sky f +mar ital +ke ene +super power +rebec ca +inform ational +hi deo +co wardly +ãģ· ãĤĮ +u sha +t ere +summ ons +ar da +or land +freel ancer +bbce arth +v agu +in sh +blo or +pot luck +poche ttino +che ats +wondr ous +euchar ist +canc elling +st es +esc ent +en den +ssi es +sand usky +bi anco +oppor tuni +liqui ds +kyrgyz stan +ai ah +gn i +mo vin +ina via +coo kie +âĢĵ âĢĵ +ol icity +ðŁį ½ +un filtered +dre ary +bbc breakfast +amar ia +rais ins +ðŁİĤ ðŁİī +sand ler +gan j +fe in +music awards +ne ta +flur ry +er re +bri ana +posei don +mul an +execu tive +dhar thm +ch ins +thirsty thursday +jam as +bar th +tn f +tac ob +k hor +mi ma +fil ms +ington post +epit om +ec w +cor ral +weak ened +ak ov +shi pper +m line +la sal +bra iner +aw m +ðŁĴĻ âĿ¤ï¸ı +twi g +this girl +man of +re count +lan zar +for ci +dischar ged +world news +mon strous +in toxic +fo ie +demean or +af firms +hal ves +che shire +se of +lanca ster +g enders +star r +chick fila +new england +jay den +ðŁĺĤ @ +sha allah +ase efabz +flamin gos +confron ted +chi anti +a om +cab ot +af loo +pi kes +leav ers +h cc +chap o +young stown +re solu +okla hom +o ons +lamin ate +cash less +ðŁĺ³ ðŁĺ³ +z aw +sa ires +rebel li +in adver +ben i +tra c +hun tsman +ðŁİĦ ðŁİħ +mer may +gi b +die u +ce ases +ðŁĺĤ # +mind less +i der +a tho +wheat ley +profit ability +un attended +in ec +han sika +backthe blue +st f +drau ght +anto inette +v ah +se ash +b ina +cl r +ari zation +ben to +ภĪ +ze man +inspec ted +ar agon +ðŁijĮ ðŁı¾ +tack y +rich ly +race track +anthe ms +abbo tsford +sheri ffs +i ah +en ough +e strang +road ways +bun k +sh anti +jurisdic tion +gur us +far r +ad on +in cogn +home improvement +dal am +col lars +co hn +be da +ai re +wester ly +avo te +spin ners +sp res +occup ying +sch rei +reinfor cement +es er +sun rise +mc manus +gold stein +gg gg +ann on +yo s +re patri +hud gens +data analytics +ag us +ðŁį ¿ +pol l +just e +gi annis +star struck +dundal k +z ap +sk ol +un miss +u man +t cr +plat y +pac man +na an +pleas antly +ob vs +corrup ted +am ari +sho ve +nau til +shi van +sh reve +go bears +we akest +bren tford +st us +pre v +basing stoke +reper toire +am ala +ç § +ch ong +c aged +bil al +! ~ +yo w +wan derer +l ila +en clave +ae c +æ µ +don ne +ðŁĴĥ ðŁı» +tru ce +he il +scor ching +iri descent +ob taining +fon dly +centuri on +buff on +seren ade +break the +sap s +ny gov +la zi +\ ( +puer to +neu tered +ta sia +racec ar +hic key +gan gu +ðŁĴ ĩ +ran cher +cla se +ðŁĶ´ ðŁĶµ +ho b +bi zz +ding le +tw restling +go go +freder icton +block chain +tu ary +perce ive +jo int +es u +emabiggest fans +bis a +win ton +re counts +re launch +m ths +ar ises +ad kins +mo tions +la wns +eno cide +reminis ce +ra pun +w kr +fass bender +e manu +sexu al +hi ppy +wine house +f dc +care r +al ai +profound ly +our o +mon toya +mee e +is cher +imp lies +fifty shades +ym on +together we +isleof wight +cru e +am zn +âļ « +me ps +haun ted +got vintage +ter son +pet smart +sell out +ne cked +entom ology +eng ar +deal er +alo re +ðŁĩ¹ ðŁĩ· +par tum +limer ick +f ates +dwell ers +diag rams +ðŁİĪ ðŁİĪ +pl ore +in ca +divisi ve +blow ers +wel les +predecess or +infin ite +theore m +hot dogs +americani dol +dam e +cap ers +reco ver +lolla palooza +in correctly +colle en +brac ing +observ ance +o ise +mr n +gran ville +estrang ed +íĭ ´ +replac ements +je sus +d st +wild wood +ta f +sar ri +horser adish +am ax +cor by +con d +cit rix +t ze +sa ic +i os +mon gering +ðŁijı ðŁı¾ +jeffree star +bar ometer +avo y +yu le +var na +v ÃŃa +paraly zed +under went +ge tter +dau phin +stra r +aberdeen shire +organ ism +ing an +fei sty +ri da +worldof warcraft +tic ker +sho u +ri ff +craft beer +thur ston +s abo +meatless monday +migr atory +ma jo +gro sse +ag chat +net te +essenti aloils +chau dhary +teddy bears +archan gel +rotun da +re us +ham ad +con vent +britt le +mar che +lo han +inti mi +eu cli +b ole +s ra +ho d +m fs +di sts +cha stain +z or +she k +canne slions +l ends +cal um +bru in +alam eda +ach ri +privi leges +indie music +fel ton +po ty +cor so +ri shi +ha bi +a see +weir dly +r ho +myster iously +low down +fur s +fe t +e die +ro sh +inqu ire +vulner abilities +sil o +nation alists +ad iti +tra pp +ti i +scrat ch +ag ora +psy che +davi de +book marks +ðŁĴĽ ðŁĴĽ +re former +lu tz +ðŁĺ» ðŁĺ» +long island +awar dee +po stu +d printed +succul ents +poo rer +l da +r cc +ivote btsbbmas +cath letics +ti psy +quin ce +pupp yday +âĸ« ï¸ı +tz el +sel fridges +i onic +wab ash +turbul ence +leam ington +tt ttt +obsi dian +o hara +legitim ately +spa in +mor al +equal iser +ap g +watch ful +w ls +h ng +ro shan +mart es +falk lands +d hl +tri angles +sta un +south bank +ren ame +quo ti +god desses +col gate +z ant +trail running +summ its +dd ick +ac ad +sc g +medi ated +ko hl +here wego +discrimin ate +sat irical +publ ici +g tc +dre dd +auto sport +si ps +correspon dence +ash win +dragon ball +ðŁ§ Ģ +ship ments +gly co +fe a +pur ses +le er +gie g +ba bel +ni on +n ca +ko a +go on +rec a +female bloggerrt +elector ate +da x +ic ulture +elli a +tun i +tor til +le tour +coimbat ore +activ ating +news night +had dock +free shipping +cano eing +ay n +ocean side +nick el +jame stown +fri gate +depend ency +cho wk +cataly tic +backstreet boys +Ð ´ +ele ment +^- ^ +zen it +ro a +fortun a +fi zz +ac lub +ÙĬ Ø© +in tra +hy ena +do dging +archi bald +mari us +ing enu +steph anie +scand inavia +ma ier +joy ner +christ ening +b td +sug ary +men e +immer sed +dom ain +ðŁı ī +pap al +ic ann +ta hir +me jor +it ys +inter fer +im pul +allo ys +" ). +z ance +an ar +tam ar +coy big +au ghter +manhatt an +ko di +wedd inghour +gla zing +bh f +depor tivo +any c +nouri shing +no tify +j py +de dition +big brother +work station +r allied +ob u +impun ity +gyllen haal +you rown +sm ite +n du +s le +o am +home opathy +gro ssing +pa e +le mb +was ser +audre y +ðŁĩ· ðŁĩ +sho pee +par que +ophthal mology +ðŁ¤ĺ ðŁı¼ +thou ses +t itu +st illness +nygov cuomo +no ta +disa ster +car den +b sl +ðŁı ħ +re po +r ate +hil da +ck en +g pi +crit ter +u hd +deadline day +tom hiddleston +sem pre +mull in +make americ +ar id +am t +n se +n ch +moz illa +food waste +co or +sagitt arius +po el +e try +c fc +kil o +av ant +pist ols +mis sive +bah rain +fa e +drin ker +war mers +sv eng +po co +its the +inter ce +pra dhan +id sday +tain able +sub marines +magn us +bo ye +am are +pen it +g fx +aren e +ãĥ ĩ +su rah +jay son +in ch +bo yer +o sun +str ati +scrip tures +master che +ster ili +program med +kn its +inj uring +sea of +reli ant +p ina +mix tapes +man tri +jind al +hac kett +bio shock +v ash +sp m +light saber +w icks +rune scape +vari ables +dimp les +ol yn +hol lis +getty images +galax ys +ed l +trajec tory +thr illers +positi ves +kit esur +del le +feel good +shan kar +ma is +is lip +ricky gervais +ingeni ous +rr bc +si p +acro polis +p buh +mesmer ising +bernar d +too t +restric t +murder ous +fo i +dul les +belie ber +sha an +ph ant +hamp den +ha ye +ch ro +ðŁ¤· âĢįâĻĤï¸ı +vi endo +mag pies +habit at +fl icks +stan za +pu tney +new smel +nd n +m ity +contrac ted +uked chat +sp ouses +plu ms +l icious +quan tum +j hope +mm r +cu sd +usa in +section als +bar bers +re vered +d ite +aw ine +mc daniel +pur dy +pelo ton +out lined +ben ito +pedic ure +moisturi zer +clif ton +prece dent +ital y +bi x +tro ye +tren ding +shan ks +le tic +wilms low +ta ir +kry p +en u +kar thi +hoar ding +surve yor +inst aweather +ri ffs +evic ted +town e +ordin ation +lux or +tampab ay +guine as +fu mes +no ck +ki ara +en visi +no e +geor gi +cruel tyfree +whe eled +te mb +mi aa +bu oy +abbas i +mc col +jas per +it als +author itarian +ma ura +tang y +mu ssel +hi gg +chlor ine +al vin +whi ps +re side +hra ya +ed ging +utt ar +ide l +du d +wo p +summon ed +ìĻ Ģ +å į +si kh +en viro +tan kers +nbc news +le bone +gw r +con ia +colosse um +rod ney +par atro +nau ghton +fe athered +chand ler +au se +! âĿ¤ +ni ko +miami heat +collap sing +ib f +gaf fer +father hood +camp ed +ro gan +hi jacked +coordin ates +am il +ðŁĺĴ ðŁĺĴ +e ther +water gate +leg er +d wy +c tly +acry lic +whole sal +ven kate +shadow ed +hor sham +bangla deshi +to ed +inst atravel +opt outside +aar p +far ce +ag in +!! !# +rap ture +lou th +mon ti +jiha di +initi ate +gro hl +u do +tech nicol +ou pe +but ti +ðŁIJ ´ +nar ayan +car la +ma kh +indi visible +ground hog +yn c +sin bajo +ban tam +wc f +sug g +pin di +them atic +rit i +kk h +val i +ty ou +lebu hraya +ko witz +sla sher +kit kat +cy pher +val u +us man +rock ville +kar ni +do re +í Ľ +fer ret +ðŁĺĬ ðŁijį +wood ford +statu tory +love and +tar p +referr als +discipl ined +yach ting +ktv u +dec king +au m +ph or +key west +a ina +ped tour +ge ti +sla shed +cric kets +gr ated +steph an +lewan dowski +intru der +al c +ðŁĺĦ ðŁĺĦðŁĺĦ +merci ful +lok sab +con sign +ab m +o shawa +fi eds +di jon +y ass +wre aths +well come +tath lon +mitt al +age of +rein force +dra ining +coy b +ac ec +inten sely +hagg is +fle mish +wool worths +partici patory +lan y +convic t +stereo type +ðŁ¦ ĩ +re sale +len i +hol ster +âĺĨ âĺĨ +âĺ ¹ +renew ing +par ted +batt ers +weak en +erup ts +sun il +nouvel le +lemon grass +tour e +h x +ç ¾ +schi phol +mess ina +han bin +daener ys +butter cream +gu o +con roy +bla k +ad ic +ach en +Ë ĺ +tran sylvania +radi of +te ren +dr fc +b ber +ay ing +alcat raz +w ld +mill ard +ìĿ ¸ +super fan +ph am +gh wa +fre ight +µ ï¸ı +infer ior +libr o +goo o +cam bria +six es +quintess ential +mat ern +j ours +hy mns +gen a +wil de +white chapel +shav en +q q +slu dge +eat clean +mariti me +ka if +bjor n +pire lli +ja sh +i gi +whis kerswednesday +the originals +sch illing +ph in +jo ke +jal sa +gen ial +rod ite +for ge +ad er +ðŁijĩ ðŁı½ +deb ated +ðŁĴĻ ðŁĴļ +woo ded +mun oz +dism al +condem ning +ant ara +saturday night +re consider +ðŁĵ ² +ol amide +hit achi +harro ds +ta way +ja a +ing uk +au c +az ette +as bury +ultra s +ri que +de ca +al oft +om ba +mi gh +me sh +fa ze +sp ital +v ado +r z +mori arty +tu ck +tou m +mon stro +sain te +ru skin +re discovered +par ais +mocking bird +cf b +tu sk +model led +black berries +spo wer +j ale +hot spots +bri m +" ," +yor ke +ap ri +mi eux +carlo s +welove you +firsth and +es thetic +rox as +j me +ho i +sch mitt +u chi +orangu tan +lead ing +def o +weekend vibes +refriger ation +inter viewer +faroo q +... :) +wy combe +rejec ting +red knapp +pi que +twee tab +middle town +palad in +balti stan +ðŁĩ³ðŁĩ ¬ +mc phee +bl medieval +ide o +e special +cc fc +ath ai +am pa +su ss +story tellers +min hyuk +tier ra +ðŁIJ § +span king +silver man +read ily +dep t +ambi ance +ðŁĴĭ ðŁĴĭ +xi x +sug ars +meteoro logical +hat chet +foreig ner +vive kan +tag ore +res ent +breath es +tele coms +pancra s +du l +ya ar +ar is +r mc +my er +jo bs +with draw +back story +u mich +sebasti en +nu est +standardi zed +sli ve +si ac +sc alli +lu be +lmfa oo +mel ons +be than +å¤ § +muer tos +hon k +din os +ãĤ ³ +team india +pet co +mo ren +fe aring +bb can +me le +kne el +gunsn roses +bau haus +ygo fficial +ygofficial blink +music fest +de marco +aro d +acce ssed +obse ssive +o con +nel lie +kil da +je well +power lifting +on en +á s +bal ism +dan ke +wol fen +pro logue +nar rows +hol o +geor die +confron ting +cab ana +loubout in +s anti +image comics +foo fighters +wester nu +g fuel +disci ple +ðŁĺī ) +su h +sc illy +next gen +eg u +aflo at +xi an +pang ilinan +di en +b ca +co ons +spo d +s dg +fall en +dol la +ðŁĶ´ âļ«ï¸ı +ä ¼ +tor rance +nc isla +ta wny +jen ni +fitness motivation +bl ount +fascin ation +p da +ip f +aege an +van o +se vered +pol s +physi ological +ju ggling +gu ev +calcul ation +sli mming +fe mmes +em pan +daw g +sto v +poly technic +municipal ities +gre tzky +defin itions +correc ting +s family +rock and +on my +homeswee thome +wt cc +sc at +mo co +lar sson +kag ame +corn bread +lc bo +head shots +fire house +d news +uc as +tem pe +son ne +di ggs +bo ilers +anti bodies +sibling sday +hobb es +fly knit +li se +ze sty +substitu tion +sic em +revolution ize +mu rad +besto wed +mill ers +liveon k +interpre ter +sant abar +queen stown +event brite +d by +chur chill +sp at +pal oma +eura sian +bu at +beau x +vor ous +naz areth +daz ed +al me +rit a +con ch +col t +hamp ers +g su +ad j +professi ons +b wi +ac b +â ĭ +univers ally +trou bling +conve y +ck ley +we asley +tra der +so td +scra ppy +nelson mandela +rup tly +pe ele +every body +conse cr +short bread +sh rou +o sama +ch ach +bino culars +pl en +nam i +k la +ce tte +wine wankers +ste f +oxy gen +ha ag +yu zu +wh olly +tri gg +me cha +subjec ted +inhibit ors +repre ssion +manipu late +cur ly +black man +u red +convers ation +bag ging +at el +vote for +eli brary +vis age +ta per +st ani +prote in +pe mber +niger ian +gle ason +behin d +trick ed +haw ley +ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ +psychi atrist +consoli dated +bru gge +ge twell +es opha +chine senew +ach t +s fu +fe mal +turn bull +mirro red +bobb i +ben id +ado ss +vit ch +man hunt +log itech +fa king +cul t +wor st +dy na +desc ended +pu ig +fre dri +chrome book +af fe +vam os +moo c +m le +lach lan +all for +ë¯ ¸ +à® µ +ye ee +paul mccartney +as au +a sive +the great +son fire +pre k +photo journalism +meh ra +le tta +h ss +dh ury +persecu ted +ha f +demo graphics +beet le +sk ath +shah rukh +k lim +esp añ +sleep ing +opp s +mun dial +extrac ted +ðŁ¥ ģ +ta ur +jeep mafia +inser ts +igu ana +fthe week +do tes +secre tary +rin poche +favor it +corri dors +eli ers +birth s +la ban +drop out +cav an +o zz +mar adona +lec turing +fan fiction +ele anor +desper ation +character ization +bu sier +or die +holo gram +son ville +av geeks +eat bulaga +" ~ +rox anne +t asked +sp k +sam ir +respec table +ha ku +di ane +bey e +fanta sies +win news +uten sils +spy ro +red mi +mer son +i be +cro ok +co pa +wa vering +ðŁĮĬ ðŁĮĬ +zz ard +selfi sh +scroo ge +p ml +bu ms +art basel +syrac use +sarac ens +n itt +har rowing +ah c +worlda idsday +strat ton +sav ages +fur nishings +billi ards +o ia +m ola +inten ds +coy g +var ma +f sb +the queen +teessi de +re locate +no one +interoper ability +fam u +planet arium +nit ro +d lr +cor an +ìĿ´ ìĬ¤ +shoul da +man an +car melo +gh o +ðŁİĦ ðŁİģ +stee ple +her zog +dev our +chan te +arun del +rio ja +box office +bo v +tri b +sn r +re for +ne wn +blake shelton +sul li +eng ages +treas ure +o yo +bi za +. _. +ãģ ĵ +oo w +ch able +brand y +ich t +âĮ ļ +z ines +shar per +plym outh +mam mo +hydr ates +el lo +do e +centri fu +ob j +laus anne +eli st +con genital +under armour +ent ree +critici zing +vogue magazine +cast ell +aga in +a ab +ld f +co ined +well done +un planned +swee ty +q p +loy al +iz ations +ir ror +ch is +sw ann +me w +custom ised +cream ery +cevic he +wrong ful +stellen bosch +n ella +house mates +e hr +c sn +tor m +pseu do +moo dy +un folding +tel aviv +small business +montp ellier +manu ally +best sellers +gin ny +leop ard +ed in +un heard +hi ero +thero ad +gr l +apho to +americ ano +nap kins +gall ant +embo ssed +avi sta +sar ts +prosecu ted +food safety +tan aka +f v +cav alli +swe den +sc ourt +bar naby +tart are +hear st +butti gieg +af icion +abo de +mtvbr kpop +flouri shing +pol ly +or son +blue sky +sound tracks +mountaine ers +ym ount +ro jo +davi e +. ðŁĺĬ +sa de +op ed +mah ler +re gs +ram ones +lanzar ote +indu s +black rock +vo cab +the hill +ni us +go ya +ru l +tin es +mu ne +cl ic +dynam ic +aggrav ated +on or +mur ph +par ka +indigen ous +ready for +boldand beautiful +au t +somer se +so good +road torio +bb t +sau k +air strike +âĥ£ - +speaker ryan +fli er +. @_ +ven detta +fre en +chap er +san ay +p fei +nu dity +mr x +h ha +ro ku +re dar +fuch sia +war ships +d fb +chau dhry +ra wal +granth am +an gio +tab loid +stran ds +portfoli os +an ning +work load +ho to +head light +general hospital +chri se +later gram +ga v +h out +bi dder +show man +sha we +servic emen +bra vest +ach y +te de +pran ks +juli anne +ema iling +car do +testim oni +supreme court +calder on +st le +wh in +tro jan +ma honey +co u +! < +gen con +bh atia +am us +vo ting +far ah +be van +å · +lin c +ka shi +gif tsfor +fas o +du tta +institu t +code x +tongue outtuesday +olo gy +nat ty +ju gger +de cency +ch ul +aw o +mont clair +gol o +g lyn +ðŁĺĭ ðŁĺĭðŁĺĭ +qu antic +n ics +h bt +cal eb +tra vers +thisi sus +shi sha +deodor ant +cr d +ac ao +ðŁĴĽ ðŁĴļ +y il +endow ment +z ur +for ts +mar tech +fifty shades +ci v +aqu atics +accumul ated +---- - +un published +poro shenko +iz u +gn all +le mur +ilo ilo +glad stone +esqu ire +sk aya +revi ving +nigh thaw +:- )) +national puppyday +mi amid +kamal haasan +guest list +gentri fication +dale k +water way +t gt +sle dding +math chat +hu da +elan ds +cap aldi +bm g +pong al +fac tions +ðŁı Ħ +p ham +el ton +, .. +op ium +lake view +excelsi or +an ic +fin serv +ent i +true story +pa id +special olympics +me tte +ta pper +ship building +z brush +tosc ana +t ants +straight forward +k sh +ca hon +bra him +simul ations +gu mp +car til +distr acting +pa is +mu rak +gre t +ma hama +eque stri +emra an +at k +la galaxy +ho ku +can to +bo gart +inher it +colli ded +carol inas +adon is +years ago +roo ts +girl sin +title ist +itch ell +fat ality +clo ths +center piece +tis land +mi ker +u bu +sh k +in tran +cob bler +bor ns +z em +sub po +expon ential +car p +uri g +panty hose +pa wan +mac cle +brigh tens +aliz ations +the weeknd +t down +t ash +ferr ara +âľĤ ï¸ı +mee k +gro omed +bar am +pl ough +letter press +edit ori +imo gen +gregor y +g mos +bree der +reduc ation +lic hen +he er +distor ted +beat tie +yum m +spla y +paras itic +brook field +ver pool +thri ves +sto ves +ru ssel +cor r +ar min +profici ency +jack y +debat enight +wh iting +nure mberg +denti sts +baja j +ari ka +vivi and +pne fc +sr h +sick ening +cu lar +å ¼ +mil let +gar age +mc murray +infin itely +aw as +anti virus +par fum +gorilla z +qui x +it sal +hair line +bo ta +ë ¸ +yan ne +ven kat +ro ta +kel a +kath niel +èªķ ç¥Ń +sch ne +deriv atives +dakota johnson +ip v +bus a +ìĦ¸ë¸ IJ +un intended +in dra +pro pelled +ne olithic +hil o +hi ves +gwin nett +co tta +can aver +b ne +magi strate +es ri +zam an +weir dos +short cut +mocking jay +ðŁİĦ ðŁİĦ +so h +wh ip +spec tra +rober ts +rob ber +promin ently +ecz ema +bu stle +b cli +sk ol +jordani an +ev ich +æĸ ° +ro jas +mizz ou +sa shimi +install er +gu chi +pon cho +hi man +democr ati +al be +pp ies +chlori de +bly th +âı °: +yo yo +ss ard +sp at +mad dox +salam ander +boun ced +asu mmit +al mer +scru tin +am editing +transform ations +tag line +neur al +mu tton +br d +ayurve dic +re vel +humili ation +ik aze +benz ema +natur alist +mac cosmetics +fi a +ram on +pre domin +li son +goo de +ce res +materi al +herald sun +cannon ball +bob dylan +bo thering +s gb +know ingly +che ung +cha z +hand gun +chinesenew year +un treated +rabb it +place bo +ble ssed +ay am +ire ann +grosven or +b light +nu ne +co stal +c span +sa an +sol l +ra jam +k q +gary en +ben nington +on tag +muse veni +black jack +o casio +mol asses +inter cept +gh ent +fu rever +bla y +aqu i +tele cast +s ats +nat gas +ho v +neighbour ing +mag ell +escal ated +newmusic friday +per ish +bru tus +lav rov +jo dy +gira ffes +bha gav +stig mati +pais ajes +it ra +bi ases +un control +hil fi +ðŁĴģ ðŁı¼ +stom ars +© ï¸ı +ur inary +:" > +s cone +grapp ling +af ran +ace way +nau t +level ing +bre ather +aud acity +loo ting +drex el +game changer +stam pe +p mo +marchfor ourlives +ger t +cre amer +ron son +gu z +ðŁį İ +k ast +hd tv +accompan iment +trade show +sacram ento +prolifer ation +wh n +facilit ator +om bre +en y +ìķ Ħ +ve h +gi ri +bal let +ðŁĹ ½ +ðŁĨ ĺ +take the +promo ters +mu ff +iti vely +crun chy +prose cute +gu antan +! ⾨ +lex ie +kar un +joshu ad +fit spo +bagu io +........ ..... +voluntar ily +sti gers +pron unciation +loo ted +ju ke +er adio +dum pling +barit one +neu er +mac cab +in ations +at te +๠ij +nbad raft +df w +chil i +bc ps +amrit sar +ta vi +ro tor +ra bi +gh os +de smo +ca g +fan meeting +ram ona +au tam +waver ley +tu sa +t se +say noto +pra ise +es mith +time piece +o jo +k ü +cu ffed +z um +juli et +vege ta +pen tax +is inha +ni ño +mall ard +gran ting +ðĿ Ł +tag gart +south land +pas se +maryam n +grum man +boot strap +amo tor +soci edad +nath alie +x es +tr out +mo ji +mar go +g ld +ma hal +bal enci +ten n +pedi gree +na omi +las vegas +ke ssel +gun fire +ak kineni +ten e +grand master +un ru +k sen +rebe kah +mon e +kol lywood +reci eved +fire fighting +cha o +de led +memor i +fr nds +b gm +shaf ts +saxophon ist +ry n +st fc +mis chiev +annu al +y vette +var i +tah rir +perth news +o bey +naji b +isab el +z ler +van de +somm elier +sole mn +interven e +con cise +. âĻ¥ +ref rain +kri sta +conver ge +trife cta +trac er +predat ory +pu li +i ow +brass erie +sco ts +pti official +bon ni +bl t +sung min +regi strar +re define +pa stime +gy u +canad as +blue bell +ye revan +south australia +sli ves +en id +vi ole +e el +death metal +avent ura +:) :) +ple dging +i ucn +daf i +bet ween +to workday +sa ur +be le +super store +hair cuts +fil ter +an ore +sp resso +shiel d +digni fied +b fa +so jour +br in +sham eless +harri ers +er ab +au ld +tight ening +prevent ative +si star +narco tics +yst wy +s nyc +ir u +for real +b ends +fala fel +si vak +free port +ce ch +say ings +don ut +dial ec +x ml +wom ani +ott city +ke re +book lovers +london is +augu ste +wonder ful +win elovers +๠Ĭ +pe da +miner va +ar de +" !! +or biting +nationalbestfriend day +flur ries +ang kor +z d +strick en +photo volta +nan dos +hou dini +fu ente +chad wick +ce rer +wh ack +terti ary +ny pl +low carb +hai ley +d ness +bla c +thar aman +re treats +pic chu +mari am +l ale +decre ases +b he +ðŁijĮ # +ou sa +o ye +nhl draft +ly on +car u +expect ancy +back log +audio books +sur ges +provin ci +pa ol +gr ate +ðŁĺİ ðŁĺİ +r nd +parais o +kee pp +hu l +ap ed +ðŁij ĵ +po tters +eeee eeee +we work +tom i +quil ting +in dra +haw t +anarchi st +pit falls +co stab +zom bie +re flexi +mar low +hen rie +gra ff +dribb ble +am joy +v ases +unex plo +tri mmer +bl ic +the or +le sley +kh urst +fix er +fili ppo +cli que +av tweeps +sc alia +festival of +mc govern +ku hn +hol z +cor ning +ym pics +villi ers +solu ble +hook up +black ed +elimin ates +t va +f endi +dent e +alger ian +re uniting +sel le +pe au +news feed +southwe stair +pendu lum +air man +ut en +in humane +gol an +av a +inci pal +d fid +bla ze +cd nag +mor bi +gal lup +wyn dham +open stack +h isd +on shore +analo gy +v ado +jo t +l la +go of +dum fries +macmillan cancer +em ur +wra pper +par mi +log ical +indi ana +lo bby +kit ts +ki z +ðŁİŁ : +vid con +phy sed +jacqu i +follow friday +cancer day +er g +____ __ +ðŁĽ Ĵ +un lock +suf fo +must go +ay lor +so va +shi gh +scienti fically +sar k +pitts burgh +barb our +arri ve +aren ergy +hon da +ãĤ· ãĥ +mother board +li ps +g ac +fore ster +ffe cts +e si +de stin +r ini +mu les +daun ting +it zer +de sal +to ad +main z +lin burg +group on +< -- +bu en +gipp sland +inv ader +hatt ers +fel la +eat in +del orean +bau m +ma un +je ez +indefin ite +ro gu +bru t +z ay +hamilton musical +emb le +sla x +he es +full moon +ant an +li one +ðŁijĬ ðŁı½ +mac kie +tk ts +loksab ha +aw i +smur f +man che +british gp +sha hi +lon sdale +hom bre +wav eleng +scoun ty +in ja +in de +darje eling +ðŁ¤ ¡ +nietz sche +nb n +win frey +pre ached +cap er +t pa +replic ate +di ii +Ì ¶ +su u +speci alizing +rep ent +jp morgan +hul me +clow n +min ster +bo ise +ðŁĻĦ ðŁĻĦ +m soc +the fancy +m re +president sday +pau ly +new delhi +jan elle +heritage month +car pool +car pe +nab i +mau rizio +es ki +bern hard +th tr +oun ced +kirk wood +in comes +ido li +coo ley +art deco +ðŁ¤ ij +waltham stow +mut ants +mal ema +son aksh +pan theon +lucer ne +intro vert +out take +dean ambrose +child birth +megap ix +gh outa +ap m +pan o +illi es +ba ez +red nose +le ston +x ero +sfor life +mid land +ir re +er th +bad al +ren ault +re spite +am ani +come on +fuku oka +b q +chai kovsky +ðŁ¤ ¨ +tab lec +an sel +war frame +sul try +sobri ety +bridge stone +arm and +ðŁĩ©ðŁĩ ° +ste u +s ny +gun ned +ji b +fo u +exac er +aper ture +ye on +k sat +gir lies +â ij +fo h +feroci ous +pep si +hay den +bry ce +ðŁĺ £ +shahe en +n mapp +mu shar +clo vis +bri bes +po sh +music festival +injec ted +ìĦ ± +li pa +sla via +ar l +an et +ðŁĮŁðŁĮŁ ðŁĮŁ +z we +meer kat +expe dition +oni k +df wwx +bat ches +kisses delavin +inter faces +ino v +cast or +âĶģ âĶģ +south park +new sday +go bble +anton i +al us +wick ham +st ly +guantan amo +fan cies +str on +moo g +ira q +i yer +cl p +vis cer +vo ten +k lon +atmo s +zach ary +michelle obama +ph ine +inven tive +bal four +s ita +remo deled +he ed +breath able +ju ju +weak ening +ol den +hel lu +g ast +extre mes +school er +perfe cted +hil al +dl su +cau ca +go to +bal fe +ab id +selec tor +yo t +surve yed +llandud no +sc ann +bour ke +us d +tow nof +j at +drin kers +ðŁĴ¯ðŁĴ¯ ðŁĴ¯ +rr rrr +maccle sfield +cor als +bra king +tr icia +collec tive +bet sy +w un +sonaksh isinha +coin base +chelten ham +usemb assy +myri ad +mc pherson +con ni +c mos +sj sharks +perth shire +kno wn +bump y +è me +supp ress +thel ma +fey eno +ðŁĴĢ ðŁĴĢ +ss erie +makk ah +bru ssel +ðŁĮ ® +mil os +sv b +emb ank +ta yy +u le +top gear +j ira +ch affe +bra dio +an ac +lu la +za a +evalu ated +ar ic +Ħ Ī +ा _ +ru ck +buy local +sag awards +k sleg +def aul +chant al +butter fly +ha vens +car ats +can lab +br k +dou x +bee hive +new bury +jodh pur +free hold +ferr ari +y ells +uncondition ally +play through +nanow rimo +dic tate +ar mor +swi fts +sc e +huss le +say ed +ro cha +at en +abil ene +ar mi +d tv +action able +tri pp +sn k +od inga +w kyc +time out +roo ks +myal gia +insul ted +an am +ts ar +o leg +me tt +j ble +escal ation +qui eter +house wife +experim entation +u ary +to ssing +re mixed +la ird +it arianism +extrater re +z are +k tor +pay load +ber ge +restra int +bethe change +be w +çĶŁ èªķç¥Ń +f ells +r ta +persu ade +line art +b do +adop tive +ðŁĩ¬ðŁĩ · +ìľ ¤ +ke ssler += = +gran ds +v aish +sa fi +emil ie +car twright +and ale +ye st +w or +po ts +pam el +boomer ang +lju bl +ham ish +el g +christ y +ðŁĶ Ł +spectro scopy +po fficial +m yeon +Ê » +sto ols +nab bed +insh allah +gi da +c sl +li dar +exper tly +deterior ating +bru ges +sati va +testi fies +py th +hero ines +chi me +facep alm +street fighter +ph oo +may onnaise +canni bal +ðŁļ Ĥ +wat ered +ðŁĺ § +cor rea +lunch box +hybri ds +s fs +is an +cul tu +zoo logy +ric ci +pi pers +be spoke +asc end +ðŁĺĬ # +stopp age +ana esthe +prostitu tion +w mc +regu lars +oce ano +comm a +shenando ah +regin ald +nas a +cohe sion +bli mp +z as +tag li +sm al +ra ga +min or +gabri ella +moja ve +m crae +earth ly +sail boat +gad kari +worth ington +lin cs +itch ard +cit ra +sor cer +racha el +pag i +ne ta +good news +ed ly +wee t +ab storm +realtime chem +over heard +g ish +barang ay +ritz carlton +miche le +hir st +gosp ur +bu sts +par rots +ke ira +hal la +bot ched +ai o +æ ¸ +su pri +ot b +hass an +sl ick +sb p +ni o +shru ti +ba it +: * +ng l +hall o +di age +qu arri +qq q +lud low +hel mut +ge al +establi shments +ax a +melan in +di ri +da red +aless andra +met cal +car val +bru ises +li u +lat ch +lap is +jurassic world +chalk board +bo sworth +batman v +awareness day +ðŁĸ ¥ +sm tl +second stomars +hen ne +pra s +fidd ler +ec ast +ve sp +kh ill +fai ths +acqu a +sold out +francis can +dat enight +h st +te acup +muham mad +manu als +ar cs +iel ts +hr t +m ro +ii fa +flu ke +ar lene +yeo vil +nut meg +lo dging +scre e +oli vier +jac que +international catday +innov ate +doglo vers +comprehen sion +bea stie +stu bbs +sol is +inter pol +hal ted +bly the +andre y +âģ£ âģ£ +schan nel +chance therapper +pott iteam +norther nireland +chee tos +belong ings +mer ida +jan o +oce ania +fear less +le ung +la pping +iver son +huff ingtonpost +hu ts +de akin +d ili +prick ly +kids deserveit +id p +est es +co sa +wi c +ne wal +confron ts +bar bi +appreci ation +Ð ± +rati os +r pi +monste renergy +apple watch +yu l +sas uke +pe gs +bow tie +ute p +salu ting +po res +home boy +char cu +ca it +ಠ¾ +mon tr +li ams +gy ms +ad in +ha slam +easy jet +col le +beyon dthe +stu co +my n +gospur sgo +ge ophy +sk a +rock land +ence phal +dispo se +ภ± +tol ls +pew ter +nom ore +div yan +californi an +undeni able +tra ver +par ri +infl ated +eu v +downton abbey +com au +n su +minis eries +tor t +prepar atory +maryamn sharif +ga els +ðŁĺ ł +pic kers +nan jing +ex u +bun ches +ðŁı ĭ +raf ale +ko sci +d of +pale ttes +on is +new sl +micro services +bar code +à¥Ģ _ +rat che +jun ta +j and +drainthe swamp +anno y +sc ards +pc gaming +aveng er +pax east +hur ray +condomin ium +sheri ff +li ra +hard back +far ts +demo lish +assaul ts +w dy +vo ort +tion ism +philanthro pic +j ci +inim itable +ft b +swar aj +ri so +qu ah +pi ps +pe so +cor olla +rolling stone +peach tree +carl ton +be b +austr al +tacob ell +ro ver +murak ami +del mar +sun dar +jeho vah +hilfi ger +emraan hashmi +emabiggestfans justinbieber +dis qualified +vi val +fren chie +brian may +bingham ton +ttr pg +refur b +il let +da ver +bath ed +bar rel +s ra +i vo +am ak +wearable tech +shahrukh khan +ne ander +le il +gren ada +ðŁĺį âĿ¤ï¸ı +swif tly +sho wr +re posted +ad il +î ģ +fir sta +easport s +aaa ay +& @ +wolf sburg +s sports +li dl +ab an +sports biz +s na +pr ank +po i +em bodies +sky papers +re ek +mc neil +el ow +dolom ites +lec ar +lau ri +grass land +at ica +hypocr ites +so ya +ro scoe +pow dered +nom nomnom +mixed media +ic p +grand kids +tray von +seaf ront +mach ina +bueno saires +multi ply +wr x +ro chester +on et +kar u +k awar +an ed +aber crombie +shak y +emp irical +bal or +anti microbial +pula ski +n ance +mi a +heart breaker +gal lop +rock away +er is +joy train +ĤâĸĤâĸ ĤâĸĤâĸ +cl un +gi z +sal ve +pan eer +out now +boni fac +wr y +sel fle +rattle snake +pi al +gh g +gastri c +walk through +nc l +ju arez +ja un +seam lessly +bur j +shim mering +outw ard +m chale +ðŁĺĤ ðŁ¤£ +stead fast +hu y +tra pping +to a +thre es +j ello +innov ating +friend lies +cor re +tic le +thi est +ot tery +mis information +hill crest +gam bino +what son +bel las +the cable +penn ington +op sis +mon ash +water fowl +storm water +ne tting +body builder +aber ystwy +ka therine +hartle pool +execu tions +vi m +sha ve +lich field +insi ght +jim mie +emb raer +cody simpson +giftide a +fu s +ci g +mand i +schwar z +ro tt +dad i +bent ley +ang ed +zar ago +worl dr +train or +pushaward skath +he iser +withdraw als +pri mera +mi gnon +diar rhea +vm world +o dom +ky ra +u mass +sp ud +ou li +c gi +ro de +quizz es +moon rise +il ty +hedge hogs +gil bert +ar ising +pi ers +p ada +fellow ships +cardam om +anni ka +un humanrights +sunday thoughts +kid neys +gand hi +mar guer +ari sts +un ny +ti ka +law d +kind red +fra pp +no sed +real madri +our i +i fi +car am +char m +sp ared +la do +ke pler +bree der +earn hardt +winder mere +viviand sena +progressi ve +mus th +jag ann +amp bell +affili ates +rick shaw +ha in +compri sed +happ ine +cambo dian +ro tting +bru nel +ê¸ ° +shreve port +ri gg +phan toms +far rah +a die +todayin history +of er +e ssi +tre ss +st am +sn d +la tham +for giving +bi ff +winter iscoming +wa hi +w ut +tor rey +silver ware +jai me +flu tter +co ders +be p +k hel +br ates +one ment +b bling +âĻª âĻª +right to +net de +e ster +ver ano +stay cation +motor home +ag ood +ì¶ķíķĺ íķ´ +ภĽ +xen ob +ven ice +sw ap +ol lins +mon i +li ka +imran khan +der m +saf aris +mon tic +better than +pa edo +may flower +hypno tic +communi sts +clari on +band o +âĶ Ģ +i j +schro eder +pre achers +ity day +b ini +oo lie +m wa +k ula +alber ta +phys icists +mi asan +do gg +whit tier +down under +dono ghue +se vere +margin alized +gb v +che eri +wat an +o an +too t +stric tly +in verse +chau n +b hic +x plo +un ner +tun ia +ro be +persu asion +dog ma +swal low +infe sted +dor an +asu ka +tortil las +mi ya +ago sto +eri als +ag ric +âĢĵ âĢ¦ +twer king +sales force +d mk +cre pes +u me +stac y +smar ts +repet itive +dul quer +ke sel +aur ang +way ans +s wind +world s +cho y +rede emer +uc sf +starwar sday +lager feld +expre ssionism +cir rus +t sum +vote blue +kaleido scope +hydrange a +chick peas +íĤ ¤ +å ī +zion ism +ty son +new som +fal k +toa sting +schrei ber +recu per +fiel ders +Î ± +{ } +thor acic +ic d +ar se +adverti sements +sink hole +liber ate +bis marck +ag ed +sylla bus +del a +cl ary +shadow y +mom my +lim ite +gr s +âŃIJï¸ı âŃIJï¸ı +uk ah +good all +f ong +envel opes +de paul +ufc fight +pe to +cur b +critic ised +wa key +steven age +hen ny +err ari +der p +canaver al +mu sta +incl ine +e bb +b ks +ter ic +re defined +ðŁĵ § +tw alk +sor cerer +me c +cali bre +ðŁĺ ĵ +se co +k andi +jun i +egyp tians +depar tures +ãĥ¼ ãĤ +ti mess +ilo ven +come true +art museum +apo the +ap l +on ation +kangar oos +x avi +sh om +pu i +na am +bom ber +tc g +ply wood +no re +h ame +electri fication +blu stery +un rival +un authorized +plu ral +lu t +gilli es +ecu ad +co quit +av rilla +yol k +pou los +her nia +est one +pe aking +id w +hispan ic +ah l +wor shipping +pe arly +ðŁĺĢ ðŁĺĢðŁĺĢ +ap s +turnbull malcolm +se av +mc l +ma koto +leu ven +b sr +zu c +te u +sc le +ph onic +shat ner +san z +caul dron +war ra +po k +pierc ings +i aaf +grande ur +fran ck +dis section +affir ming +py le +ham el +equ alizer +bernade tte +syour bestfriend +stumb ling +prith vi +polym ers +mcal len +jun ky +hu gger +da vide +ver itas +mp f +mc neill +ley land +jo zi +candy monday +bas u +mon g +list o +hair spray +sun dance +film photography +far row +u sta +oci ety +me mento +fe o +ab ruptly +c ska +ti vi +on en +calcul ating +shi rec +sequ in +mal ang +gen sen +é n +up take +r tx +free the +wi per +sym bi +co qu +ภ´ +z oned +te ak +id ps +alon zo +qu ish +oc chio +artiston twitter +summ ery +su is +sil as +j alli +forci bly +del ano +cu d +blogg ing +air craft +thri ft +pal myra +ber liner +under gone +fu j +v ray +on b +mand olin +kra ut +butter cup +br ats +termin ation +penn state +moff at +mo dem +ashi on +ðŁijį # +thin king +re publi +ou ta +beh ance +ra ha +loc ket +parag li +l hr +crow der +ma gu +light ning +jo c +fire bird +any l +an vil +sus anna +r ö +feder al +cas ket +bo or +ðŁIJ ¢ +tion ists +pushawardskath niels +jog ja +ha kim +ðŁĩ®ðŁĩ ± +kendall jenner +fri es +boo l +boath ouse +ðŁIJ ĺ +ðŁį Ĺ +s lin +kash yap +i que +care free +ðŁĴ¨ ðŁĴ¨ +hu ber +do b +bre con +an acon +pho bic +deta inees +ðŁĩ³ðŁĩ ´ +un ter +sea horse +man c +la ila +cy tes +author ised +wi pe +stor ia +mi yaz +k ling +isol ate +he brews +gu cci +australian open +tex plain +dis continued +crow ding +mer its +all ar +ðŁĸ ķ +tiem po +spo ti +balenci aga +l cm +kay la +der mot +cor dill +ou ille +oh m +le thal +free zes +ut b +she pp +rou te +dar y +bha van +breath less +ash anti +aci fic +ne followers +kristi an +on c +mary lebone +ber ks +x posure +unmiss able +tul ly +tor bay +raven s +kar thi +ad vers +ðŁĴª ðŁı¾ +us z +sc is +mil ion +at ura +peach y +cra m +ev ils +pel ham +paradi so +meteor ite +kra vitz +yo te +confis cated +bru ck +pla sh +mur ano +maro ons +ðŁĴ ¡ +yn x +pick le +lovin it +k ra +r ns +ann apur +u ct +le ander +lan adelrey +gab on +numer ical +mens style +ma pp +ju g +gli ding +steve aoki +fric kin +food fest +ch int +y pres +sidd harth +butter field +i ff +ad jour +w gr +tam my +me kong +it forward +g td +cryst alline +sur faced +super cross +dilig ence +v z +sd al +s fm +in version +sni ffing +pun to +le vis +ka jol +ini esta +the future +squ all +end alz +di me +con tention +ce sc +shin y +band age +nom adic +dif fusion +aver y +stir red +rig by +os mo +hard ships +wh or +s sp +dis ks +ðŁį © +ìĦ¸ë¸IJ íĭ´ +wil ding +yy j +ovie do +n pp +excep tions +sever ity +made by +harve ster +del inqu +pedi atrics +human trafficking +appre hen +w lax +thermost at +wi gnall +d pr +woo oo +tra u +gor such +east ward +conclu ding +team jesus +fla k +cc r +sa sh +man te +hi k +vag ab +pur sued +legis lator +di ri +ray mond +nu dge +mun dane +s ars +mccon augh +ck in +âľĮ ðŁı½ +pho p +me yer +haci enda +feasi bility +sapp hire +mu gh +ly di +lo west +ers ville +god speed +gabri elle +d agen +beer fest +bang alore +ra ff +n pl +lu kas +ach an +sno ws +ml c +hu mming +ten ter +resi dual +mou ssa +le andro +ke strel +d reads +resu med +hr m +com ix +agreat again +un loading +lovel ife +jack ass +cu yaho +wh ining +power shell +n gs +front page +barbar ic +uni q +ol phins +intensi fies +ea c +dy sp +seabir ds +tann ed +sti el +ho ws +aaj ith +mc avoy +á ¹ +windo ws +wh u +muham med +ide ological +mi de +j ingle +bbcra dio +ultra violet +next door +lei den +con un +an thro +air way +wa irport +tr p +race day +l ml +g ough +in stig +den berg +es ther +meat y +da vie +co founder +tour mal +shir ley +ob noxious +loo sing +ðŁįĢ ðŁįĢ +⾨ # +spiritu ally +sc rob +go for +coffee day +ðŁıĪ ðŁıĪ +i em +extra dition +sett ers +demb ele +tur nip +mathi as +liken ess +roo st +i en +ero ck +dro ppin +mu ay +feyeno ord +bon ang +sv g +ous ell +mar vin +cas ing +mor ata +edi bles +co a +av n +ta ken +ice man +t cc +sand berg +id gaf +consider ate +ps f +ay y +scho en +hake em +excer pts +no elle +inevit ably +blumen thal +wh yi +under taken +sp ub +oni um +daw kins +pro tip +âĺ Ħ +troye sivan +t ye +stati stic +sm l +ðŁĮ § +ger anium +ver watch +yo ak +world wide +volta ire +ns x +na iling +mo ira +band ar +lay ering +kin dest +ef fi +cham plain +apo stolic +ta vares +lero ck +appeti zers +ac centu +;- ; +w awa +or ning +on der +easports fifa +ar p +ðŁĺĬ ðŁĴķ +up setting +str inger +sho ggi +lu pin +l ny +su bor +pr itz +mor o +hil i +tro ye +scor p +her story +ent ral +ch ine +mar ques +hop kin +mo g +h cafc +g j +y aaaa +ru moured +iti ans +cotton wood +basti on +nine teen +mish acollins +men i +handicapp ed +alt coin +min der +at socialmedia +allen town +ak on +ðŁĺĿ ðŁĺĿ +gw u +ay ah +cannab ino +anth on +air stream +i wc +cbc nl +ðŁĴĥ ðŁı¼ +w soccer +se ong +aad haar +l anger +ì ¦Ī +the bachelorette +t chaikovsky +pep tide +p sl +agri business +oun i +scat ter +nul li +inhibit or +vie ira +ra iling +law ley +ðŁİī ðŁİĤ +ì ² +su tter +mi u +husk er +har rys +con cac +c ates +as ak +ãĥ Ł +serpent ine +santa fe +pat taya +modi fy +jay hawks +h ors +brain storm +be sik +wat h +qu on +creep y +u ic +sc aring +peel schools +ðŁį ª +sh yam +rou se +gov ts +go pal +par th +maxim ise +ken il +hhhh hhh +health iest +so or +r acker +bb on +vintag ec +the w +marl boro +d any +aven ues +ag it +ro sh +sc ania +pr itchard +p mb +glass ware +your bestfriend +whist ling +la e +indigen ou +brad ford +co q +bloom sbury +spirit of +op eng +flick er +cre ed +confi dently +aw fully +um n +hermo so +tom y +sn ape +kar ma +wa isi +nw dogrescue +mon mouth +de fun +bu ren +west gate +s show +goog ling +gibb on +deci der +q vc +pat ra +m chen +bra ille +wh opp +de bac +one al +willy levy +white side +the red +im patient +saat chi +depic t +war na +pick ens +gh um +fi bon +opla sty +director ate +wh ittle +kim my +gru dge +al tu +simil arity +eng ro +cham onix +alic ante +secre cy +re master +pyg my +low ski +gujar ati +figur ines +at uri +agar cia +ultra sonic +out breaks +inno cents +under goes +acou stic +nhl blackhawks +dan ville +ðŁ¥ Ģ +holo graphic +f able +cum ple +ev ens +acqu aint +she ba +the drum +equili brium +sincer ity +premi ums +jelly belly +buildthe wall +and rade +staur ant +savethe date +re election +prescri bing +kno tt +some ones +cook ware +sal ford +popsic le +dr ury +c age +ag gi +portra ying +pande mic +pan tom +v d +ero es +backtothe future +ë ħ +trans gre +suici depre +stay safe +o bas +ju ma +heigh tened +endome tri +a jo +v yn +nd t +lif es +tu ll +dic tion +chilli es +calla ghan +take out +su bbed +stephen curry +sol i +promp tly +aw ang +a theatre +ni th +d ney +aji th +abas ketball +sk it +mol ded +duc tion +an ker +ìķĦ ìĿ´ì +world sbk +syn onymous +rr r +ro dent +ash win +Ñĭ Ð +ge ton +real talk +mul ch +j ani +dray mond +ast in +harmon ic +h ms +dwar fs +ambi ence +ab laze +th grade +ra kh +mc david +bar bic +pre t +mun ster +kis sim +indic a +cy r +ac nl +ðŁĩªðŁĩ ¸ +turno vers +rae es +plu to +m hr +lec tric +kon en +ca stan +mitz vah +bo wie +une asy +pode sta +phy lo +im moral +hour sof +decath lon +c br +kham enei +ja in +ex tric +cu shing +z hao +y id +plo ver +nor ge +yak ima +women shealth +to ff +gil mour +ch ay +าภ£ +visit wales +art fair +al en +willam ette +lu zon +elli e +blin ders +the john +co lette +o zzie +drun ken +bur kina +adirond ack +rescu ers +pay out +mar ge +ju ly +car parts +su shi +goo dison +ag wa +cor doba +box set +ad un +.. ) +le sotho +layo ver +ke an +al b +ठľ +son net +mus ke +mach i +i sto +bran de +syn ony +li oness +ak ia +texaste ch +stun g +hang ers +commerci ally +du mas +uni magin +spar king +ri f +z ic +tabern acle +g aff +creati vely +coura ge +arma geddon +ðŁIJ · +s st +gerald ine +ss chool +son am +ne ha +man c +j query +eleph ant +ejec ted +cam i +yy z +cle m +x games +wi ft +sw we +ra bi +back in +man j +hol t +ho ist +fire stone +ðŁĵ ¦ +ur anus +la ing +ðŁĩ » +nfl network +insp ace +god win +clari fication +tre spas +multi plic +hack er +ðŁį ¹ +pol enta +heat ers +mk tg +mercedesam gf +ãĥ Ĺ +wwer oman +gu ing +gfuel energy +ภ· +u ste +di ony +cu sack +cor ned +( - +thex files +v news +sind hu +le high +fun times +fo g +exp ats +a beach +dun fer +deduc tion +beauti full +ol us +modi fications +mul la +not ation +wweroman reigns +thal aajith +kar im +harmon ica +salv ador +oc co +plan tain +faith fulness +prefer ably +car th +!! ? +womenin film +so br +enterpri se +che at +ab del +sar coma +mca fee +chu a +museu mof +black stone +ar af +un dies +smugg lers +yo se +ten dulkar +preci p +fc v +trac ey +in voice +am bo +theo logi +li ye +chronic pain +bash ar +war burton +the more +sol dering +ho sse +gine bra +g ly +flash y +é ĥ +schu ster +livepd nation +ind ler +bon jovi +black ened +silhou ettes +gar go +ni les +mu zik +gau rav +chant illy +recl ining +mc cur +lou doun +har old +ad ha +f ata +ali l +tb f +v am +twenti eth +thisi sy +the bachelor +lan ark +sni der +bar an +fi sts +cra i +al go +pl ice +or ang +gen ds +cor nish +ste dt +di shing +ci on +rel li +in bound +cent en +va z +sc ia +une th +mock up +lac s +dr an +design museum +cos mon +c dr +bull seye +s ds +pamph let +fi zzy +silicon valley +barthol ome +' .. +tra e +pett is +osh kosh +o ast +mal ice +body suit +all uring +pu tra +no ki +ar news +wil lows +urban a +radi sson +podesta emails +ne apol +j timberlake +ti q +om ents +cc c +what wedo +mat is +ign acio +ho ss +hill song +gra be +fran kel +e us +cre epi +benedict cumberbatch +âľĮ ðŁı» +ra bies +mc m +batmanv superman +sym path +s ry +roland garros +ku ch +gross man +du als +coco on +bri scoe +rebelli ous +pride of +mi mosa +k ola +hear th +gil more +caled onia +c md +py jamas +am end +ðŁĻ ħ +hau te +ev r +ðŁį ij +ðŁĩ« ðŁĩ +vo res +marj orie +in explic +dat piff +spr in +rub ens +lam ent +apo d +re stores +ra hm +madein italy +cas ed +ca pre +bang les +ag ile +refresh ment +parkin sons +gri eve +expon entially +gr yl +drin kin +ठ¸ +sch la +snap shots +mis on +sf v +nov i +cun y +the snp +kin ks +josi ah +é r +megam an +m dm +blu eli +x ena +par ab +maker s +cle f +ðŁĺ ¸ +t cr +pa io +cron in +the boss +scar y +ran os +ko e +daim ler +wy man +te es +s beer +ise ach +in is +and an +ðŁĴª ðŁĴª +ë¹ Ħ +stal wart +ni shing +jun k +gu s +perfec ting +new x +ir us +co preps +supp er +suc cumb +mosch ino +hi ggs +ãĥ ĸ +shan ahan +mark t +lor a +hou thi +ex c +or dan +ko d +gro in +just doit +bell ar +rho a +psori asis +ma arten +insu fficient +impac twrestling +g aff +du stry +summer of +news week +mur a +is la +do yle +ma ic +luke bryan +fibro myalgia +ر ÙĪ +m la +kar am +ju d +evoc ative +ठļ +tro tters +tri pped +ple aded +fall in +et fs +venom ous +mcconaugh ey +flam boy +chang i +good morning +fri gid +th aman +re claim +bo leyn +ãĤ ¦ +recon c +ke sh +el sie +bed fordshire +be ss +sub continent +kat erina +bo z +thessaloni ki +termin ated +rag ons +intro s +dr r +cre ss +brief case +blin ks +ran bir +perfu mes +exc ited +ever ton +cou k +c pp +yr kkh +sk u +ri va +kit sch +di pa +do do +bo ho +ticket master +ling en +lau er +dat sun +ðŁĶĹ : +m ro +gon dola +ci elo +chapp ell +fit r +ski ps +nc ga +mur dock +multi disciplinary +ki wi +cer os +cac ti +vene er +on u +k ars +evangeli on +Ñ ı +titan fall +secu rely +eyel ash +îIJ Ĵ +s watches +heal ing +ton ya +n q +mi stry +high e +cab rio +m ö +kinder gar +in nate +vi pers +nucle us +mac key +al pine +ox y +mor tem +fol ders +a fest +á ŀ +repur posed +green belt +de port +west port +pu sb +news brisbane +arquitec tura +set life +mag ick +macar ons +dark horse +vau x +mu zaf +ðŁij ° +ì§ Ħ +pro wl +gon i +edmun ds +vie jo +lau rier +enqui rer +embank ment +ðŁĮ ĥ +ro mel +ma ury +line a +k lee +bis ons +b able +we athers +o deon +de volution +cordi ally +bu ch +sti an +o varies +lov ell +cru iser +c th +v ay +un nie +tro w +t ler +ben az +- £ +nas asocial +meto ffice +gu en +clu msy +? ¿ +or ps +jac ket +in nes +regi men +mah mood +kam ala +fi end +da al +co as +å ½ +twitter less +tao iseach +buk hari +panther pride +delight fully +book case +pan tera +ms ley +mesqu ite +here by +he morrha +gun control +du ma +colla red +av l +ador n +vaul ts +teme cula +sky diving +play maker +mur ug +lat vian +here fordshire +god mother +till man +shoo ting +mar it +mal function +fr inge +tu bed +nab show +ed dy +do ge +diag onal +as mith +好 ãģį +sti est +spectac ul +pinst ri +pi pp +d sw +ðŁĮ Ŀ +nam in +mb ur +propri etary +gener ale +dic ed +ba hia +ðŁĺĬ âĿ¤ +urban ism +pe ps +dri scoll +u tt +cay ne +tul ku +national siblingsday +ya an +v adi +together ness +o en +juli en +cam pion +ðŁį ī +ye ahh +wo e +t alia +lepre chaun +p ice +fin i +de ver +carri ages +we aves +scho les +ra deon +lil o +j cc +icec ream +hagg ard +el ks +cir cled +yl le +tu cci +ic loud +dr an +analy zed +ðŁĴĽðŁĴĽ ðŁĴĽ +âĢĭ , +win x +sonam akapoor +s fl +ni ka +lock out +injec tions +erad ication +bio chemistry +rot ate +rang ersfc +playo verwatch +kr iti +hand lers +win ks +mis ss +k su +best fiends +ðŁijī ðŁı¾ +âĶĢ âĶĢ +super iority +kri sti +flan ked +alt coins +mimo sas +hallo ws +yo i +tro ller +re pay +ny g +ie a +fol lic +ðŁij ¾ +tele caster +pro claim +fear ful +whi z +mosa ics +improvis ation +bic entennial +we sley +pad ukone +every ones +ain en +lat i +lac ma +gram mer +fore arm +de ir +colum bian +tyne side +sh ingles +rou sing +rand le +cru mbling +tu pelo +glo s +cor mor +bosni an +rac ine +k ington +ha ines +children sday +at un +analy zer +at ch +meat loaf +amaz es +isa acs +corner back +os wego +multi ple +electro cu +admi rable +sho als +red mayne +lo sa +mcdon ough +ker ber +te ddington +rh one +plu mp +ne stor +kw h +hat ching +girl z +bel uga +.... ? +ðŁijĭ ðŁı» +y ms +ble achers +ang es +tor tu +refugees welcome +pu th +vul can +nu i +mad hy +doubt ful +dami en +yu u +si ppin +ky la +ospre ys +mache te +lad bro +sh era +scoo ped +jy p +z co +u bi +smugg led +dre d +cl ondon +der berg +e hl +du mont +de de +è ne +s bb +pru dential +life saver +key notes +bal t +un settling +pu ente +out fit +leg acies +exam inations +red hawks +manipul ated +gaz ebo +tou hou +medical marijuana +ing week +gi bb +zero hunger +rac king +tu ba +sun a +seaw ol +w pc +oz an +cav ite +broo d +wool wich +vol de +un fur +shadowhun ter +jo bless +har var +food blogger +ca wards +ta hs +st b +no wh +jo es +h j +cahon tas +oper ahouse +mi ght +flag ler +b ch +sp ire +bun gee +b x +ri fle +cu rie +ba ines +ru pauls +) ." +vi vac +health it +wel lesley +throw down +sa ver +ri vier +x ray +nap erville +induc ing +charcu terie +berser k +ba at +spart acus +noo b +dead ass +bel e +vi ri +niam h +mountain ous +si xx +qu a +te sters +prince ton +in q +ber gam +democr acy +bre am +aun ts +re is +pet r +par ramatta +nic ht +arte facts +un just +jet pack +in venting +filip inos +farn ham +do il +chu cks +ac ross +sp ass +r anc +hundre d +euro sport +slam ming +house mate +gam bit +d ÃŃa +azzur ri +stam ping +nar ra +campe on +suici des +colon ization +be zel +xin jiang +stand still +hiero gly +gou da +cam bs +thr ill +star vation +night shift +adi l +spher ical +loc alization +clean tech +zarago za +wor ka +spec k +sou the +lip sticks +cb t +ak im +ag es +st ica +un k +pion ship +shell fish +kyr gios +far is +sty lish +squ aw +kel p +id w +cap stone +w gi +trol led +pe ppa +gam mon +anti och +; ( +z ations +un realistic +ss cot +slu g +ke ats +en th +ad iti +uni onist +ol or +ke ita +exagger ated +briti shar +Ø £ +suzan ne +so z +n gr +campu s +bri o +pet iti +eh ler +ci k +ab io +ubiquit ous +Å į +wi gan +plac id +bank holiday +my sql +mc nally +fire wall +bay lor +bar stool +az ur +âĿ¤ ðŁĺĺ +mid as +ãĥ ¡ +sun downs +sto gether +sequ ins +m va +c ph +ðŁĩ¦ðŁĩ ¹ +trail er +so wing +am ary +mol lu +mackin tosh +al di +wil fred +vaccine swork +lo ls +dial ect +cas inos +militi as +go thic +fort worth +calibr ation +br ine +ble ached +ke k +n ss +har u +acry lics +mar ou +ger sh +bor ac +sam ar +rome o +mr p +light ing +ab p +spra sad +main line +flav our +bo sp +alber to +the show +santa anit +plu s +n fu +morning joe +m chu +gi mb +water loo +ut z +motor ized +mar icop +inst adaily +rever sing +mm ons +cen ta +salv ation +jaco by +inquis ition +he id +bantam weight +sun d +stri p +sime on +fu tile +carpen try +al ondon +ðŁĵ ¡ +p ma +the hobbit +mo ab +keep sake +for mmva +water mark +free iran +folke stone +drif twood +sen sor +maybell ine +for s +fer ous +ane mia +glen coe +atl ant +at lee +incre ibles +cor t +refuge e +elli ot +Î ± +tim or +tann er +take down +m nu +ha bad +proud to +nu tt +hann on +castle vania +timm er +restric tive +l tv +delu sion +ay la +a ann +ze al +j ant +im bi +bat smen +um o +ther on +smir k +per ishable +d wind +aa ja +pla giar +lu dic +kesel owski +clin ically +reck oning +mountaine ering +conj uring +yo gi +west land +toma hawk +montr éal +jaf fa +b de +ra fts +n lc +avrilla vigne +ux design +sun roof +ram s +gw yn +die ter +awak ened +ab l +sur realist +der mat +ban get +the cat +latin x +gar nett +ay or +wel der +state house +love joy +gir lie +coquit lam +refu el +po u +man candymonday +ma q +bus by +tt f +picture oftheday +ak ade +yi pp +y ere +wi p +tre search +li ya +wh ol +dig ic +bel lied +abar th +will ough +vil nius +tellu ride +kar at +anth rax +t work +as say +ach am +wil shire +rain drops +l are +gigab it +do san +ab p +ðŁį ¬ +tr ynna +orthop ae +h inter +go irish +gian carlo +gas ol +chat bot +where is +si h +holli ster +cli c +abc network +dress ers +fe asting +elev ate +constitu ent +adventure time +sr iti +el ou +de soto +dav i +contain ment +lo di +ko da +gl in +wr itten +wind chill +out spoken +make adifference +j annah +" -- +tro t +summer fest +sil os +joom la +game design +ar go +ru pp +perio d +new quay +mitt ens +ici ally +emplo ys +du bious +bail out ++ @ +ðŁĮ § +âĺ Ľ +special ties +pan ini +mb ridge +gar nier +du els +anton ia +u j +ph u +aw at +robo cop +mac abre +dom en +band ing +anat oli +ad n +nam co +laco ste +buy out +fav ourable +esc o +sexu als +kait lin +en try +ad ly +yang on +win ston +wau gh +pati sserie +ozar k +kristi an +kha shoggi +g mm +embar king +mo aning +mal kin +j el +di ggers +bee keeping +whirl pool +hor gan +bb cin +ðŁ¦ Ĭ +ðŁ¤· ðŁı¼âĢįâĻĢï¸ı +suffra gette +mer u +dro ck +cru fts +woo dru +pi ero +om bud +esp ana +advis ories +aby ss +us ar +ren ato +jean ine +endocr ine +. âĿ¤ +ðŁĺį @ +ìĥĿ ìĿ¼ +wand sworth +slo vak +reli ance +in competence +ey oung +ap as +a sen +s lander +ljubl jana +iti ve +ar gent +tion day +reson ate +more house +en chant +b sg +ri vers +n ils +m da +indul ging +gal le +sav annah +no k +mn h +lu h +hi berni +tor turing +le b +girls who +dro gh +adri atic +shar pen +swa sti +se urope +i fs +gi mpo +eri e +amade us +ipf conline +ðŁĺ© ðŁĺĤ +tr l +as syrian +ðŁĻ Ģ +vi ene +data protection +dream catcher +thro win +red undant +pen rith +n ne +amal gam +sense less +par v +national guard +kne eling +guine ap +fa qs +cy an +ãĥ IJ +whi le +loun ge +sik kim +makeu partist +instin cts +ha ji +cot to +vil i +mb l +com mo +mi ga +lu s +ar mp +ŀ ï¸ı +æ Ī +platin um +na am +lukebryan online +gulf stream +ad der +tot ally +pal i +wel i +alter ing +ts x +par ake +mon ol +air lift +sym pathetic +su pa +recep tors +pat a +orchar ds +op m +lo dged +ky i +bru n +villen euve +ko e +electro ly +dead mau +a ed +sharp ton +re branding +nu est +hub spot +hemp stead +gw end +bourgeo is +wn w +living thedream +friday night +orthopae dic +kx ly +is and +f co +f ada +bla s +all l +: + +r cb +mi kel +live streaming +din ing +de ford +she esh +lon nie +ho ard +zar if +thevamp sband +spiro smar +spirosmar garis +n hi +ft k +biome tric +bas f +auberg ine +acti vision +vari ability +pi ans +med an +l nk +ira h +t pc +r tv +ofi eld +dr aco +bri c +x perience +we stin +santabar bara +quadru ple +connec tivity +bru ssel +marriage equality +dat am +concac af +ë ¬ +w acom +truth ful +sw irling +sher lock +archae ologist +aque duct +york town +ton k +ten n +sti letto +jo on +ab ril +f ft +boiler up +? ðŁĺĤ +shi sh +deci mal +unle ash +pl at +ec risis +nar c +suff ice +jellybelly friday +it an +inv ades +ctr l +santaanit apark +le aping +invic tus +ful fil +x ic +under stated +l é +higu ain +ct is +bo realis +annihil ation +z hu +ul rich +shar ing +pul w +eth andolan +vard han +timber land +corin ne +spac ef +resili ency +pu k +inspec tors +cer ve +beli us +avent ure +par ris +pag ing +hy land +debac le +first look +bast ille +Ľ ï¸ı +ðŁĵ ° +ðŁĮŁ ðŁĮŁ +rel i +raje ev +fand oms +val verde +med ford +vo wed +v amp +sweat pants +dee z +par nell +glen wood +bur ners +road works +no ire +lek ki +ðŁĺ³ ðŁĺĤ +sus que +sp as +s dr +launch pad +de tto +sa q +cam po +ðŁĺŃ ðŁĴĶ +vi va +ne g +jol ly +di anna +waf fle +trick le +th w +scumb ag +henrie tta +foo lish +expo s +caf er +bil awal +âĢ¢âĢ¢ âĢ¢ +stri be +se ward +n de +lou th +cyn ical +bat o +m ily +inclu sive +hai yan +aj ar +ðŁĺĬ . +me redi +d pt +can tab +ven n +gan e +di was +bird club +tr ina +o gs +mon ic +he en +de mented +she ik +noman ss +itu nes +gly pho +ðŁİ ¯ +y ous +wi fe +vom iting +om gro +tax onomy +ri eu +berlin ale +ad ag +tur ret +maul ana +mag icians +ag ul +xx i +the age +shel ter +gru ber +cri mson +bal di +ab sin +h inge +me ij +loc a +ge iger +dri guez +atten tive +dit ched +cat nip +íĬ ¸ë +loaf monday +joko wi +ce bu +chur n +breeder scup +stap le +lef tists +train ings +fu ku +e bb +colon els +â Ĭ +whist les +shon en +mc ge +vel asquez +tes lamo +lom o +car rey +in ton +kent on +isleof man +aaj tak +ven ous +tuske gee +original funko +jewelry onetsy +ðĿ ķ +per ak +eye balls +dom ingu +ath al +ðŁı İ +tg dn +ta v +spam ming +ren ters +not ably +kav anagh +pp ert +m db +fox sport +ex ec +besik tas +auth am +ka iju +ðŁĮ Ħ +utili zation +spo of +indic es +hin der +gir ard +deep en +anag ar +ðŁĶ ¹ +termin us +s wore +rita ora +li ven +bra sil +alham bra +ðŁijıðŁı» ðŁijıðŁı» +ton ews +ore gano +boat eng +joh ann +bu mmer +ba ston +à® ķ +then ation +spac ec +cru m +bu sch +sarah g +lo we +aw arri +âĪ ļ +zel o +wayback wednesday +tent acles +l hh +jo ec +eras mu +de witt +rick man +dill ard +curi os +poin ty +po thole +mc nair +he mat +dr m +de fies +w sb +plant ations +ha im +pal trow +up i +ter ies +shor tened +al ac +pon der +la ker +best fandom +ambul ances +safe way +pas ser +melo dy +ima r +spo tty +in der +hear tedly +ge ss +bi ga +ðŁij Ĺ +fl ack +bott as +y ara +si b +disci ple +ti dal +sol ve +lon a +âľĬ ðŁı¼ +strato caster +k rs +eng age +du chen +buon giorno +ঠ° +pi geon +lets dothis +fe et +ci roc +ðŁIJ Ī +som ers +ko ch +i ain +al m +veg am +re pu +promethe us +pede stal +ke swick +i ol +ori z +cotsw old +a er +Į Ģ +æ ° +head sets +al ona +volde mort +gor d +fu se +dont care +ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ +spl at +port als +lets get +inform ation +en ugu +attend ants +th ackeray +progre sses +ke igh +alpha bets +ðŁķ Ĵ +sand ton +derel ict +ìĬ¤ íĥĢ +un familiar +super human +ri an +insur gency +cor rug +trage dies +si sland +del ilah +and aman +fu chs +br ati +adam son +par as +live by +even ting +ç ķ +ra go +le z +il ook +bou lders +bo j +tom brai +ring ton +ma ul +fi que +complic it +wel beck +gryl ls +discrimin atory +une p +scra ping +pan a +ocean ic +mat tered +ಠ° +tropic ana +house wares +bell i +agu irre +un censored +mult itude +mon god +met life +kapilshar mak +gal len +confin ement +tru est +new bies +chil ton +cal cio +ballis life +________________ ________________ +âĺĥ ï¸ı +santam onica +open ness +faw kes +co leg +bo yn +ðŁĶ į +r caf +pr in +pic colo +dev oured +av at +adequ ately +ìĬ ¹ +thi e +mc ity +madi ba +le mmy +in ject +farm ing +it el +beauti fu +ৠĩ +Ù ¾ +miz uno +en rolling +du mber +aqu inas +wo ws +sque aky +l ons +impro per +esk om +emancip ation +bar ba +a hahah +âĺĺ ï¸ı +mc mur +eye sight +dissi p +cairngor ms +baf ana +s movie +li ang +ger d +andalu cia +am mon +yl de +t mi +s group +poly mer +newin dia +li i +te w +le ge +go ha +for ay +dissol ve +th ks +so ire +lan dis +go blins +glau coma +jble fevre +d cu +th ony +p tx +margare t +mal in +íĶ ¼ +li shing +cough ing +conce aler +und p +sw ir +g te +sil ica +ro asters +po go +ou sted +in play +bird sof +hist med +dep tof +bon g +ric key +mad man +fundra isers +e al +portsm outh +mu th +predic tor +iz one +compens ate +sh inju +po achers +nbc dfw +ci ano +ðŁı ° +uof a +po cus +open ers +insi dious +a the +yi el +sup date +pel let +n sc +f fr +cha e +½ ï¸ı +lo m +l fa +kissim mee +hafi z +å ¿ +tr ich +elec tive +brant ley +mi g +mee ee +lun ar +laver ne +cor related +carto graphy +ar au +z az +yi p +viol ates +negoti ated +law ton +âĢĭ âĢĭ +to ads +reno ir +follow your +arma an +w apo +th yr +n gu +mark sand +rein force +pension ers +pa f +mu kesh +fer ro +çĶ ° +ven u +re run +man zan +fin earts +bray den +x m +wag yu +un bearable +ri deau +ec m +c pm +b itt +ðŁĻĥ ðŁĻĥ +ye ahhh +temp ura +re view +noo o +moder ates +li ef +lat ory +deplor able +co yr +re gas +gov ina +dv or +angh ami +seri es +pal lets +lin d +sha olin +colli sions +than a +l lu +jume irah +honey well +compan y +ve dic +twenti es +t was +snu ggling +la f +gossi p +bow yer +ba si +vigil ance +sni pe +senti ent +represent ations +formul ation +adven tist +âĨ Ĵ +t sa +ss ors +isu zu +bon ham +vi vegam +liver more +join us +ðŁĮ ¶ +stage coach +con tre +clique art +ðŁĵ Ī +ðŁĴ ² +par son +ful ham +ª ï¸ı +omgro bots +bridg end +wink ler +waver ly +ton to +slu gs +glit tering +ni d +dog sof +ah hhhhh +thisis queensland +pro wess +pale y +n ga +gangu ly +dor mant +agchat oz +vi acom +song bird +ron ny +after school +un insured +ther a +bc afc +. "@ +ja o +ip cc +hef ner +gen dered +cou ch +be there +v ann +retali ation +moder ation +j pl +mac adam +dan sk +y us +mu ri +car amel +bro mpton +ar mando +agu stin +.... ! +ski y +kitty loafmonday +ido t +en son +ha vill +extravag ant +ðŁĴŀ ðŁĴŀ +the op +un done +ephe sians +nott inghamshire +nec tar +ch q +bai x +te z +stream lined +fl oun +all timel +republic day +mer curi +cc w +ak ash +ðŁijĭ ðŁı¼ +twi gs +tul le +ti ram +red ford +ne ttle +el ms +bu gger +fitz roy +! ( +ver ve +bottom less +blu shing +valedic torian +tin iest +recy cle +ju di +ather ton +time for +ti mi +kis umu +fron ted +e ola +digiti zation +cu ster +baz aar +tri angular +st ann +paedi atric +mer can +ma ren +gv prakash +wind screen +un pack +la do +finan ce +saf rica +cron ulla +bit ty +bel ter +be bop +âĢ¼ï¸ı âĢ¼ï¸ı +my x +ker man +dd ell +bringbackour girls +sau ce +rac al +pap a +nu f +fa red +cartil age +c renshaw +vas a +rele ss +book ish +w mata +ite x +dor al +astur geon +tremend ously +info sys +fan fare +die ting +ðŁĺ ° +suscep tible +sex po +ry erson +mo fo +yel len +var nish +ðŁĸ¤ ðŁĸ¤ +ðŁIJ ® +mo sh +lif toff +kamala harris +crow ning +# . +âĩ Ĵ +tu f +pac er +shaf fer +en lighten +swe ars +apolo getic +yi elding +un opened +n vr +ken ner +jump start +en sured +à° ¾ +t mt +pack ham +cd mx +swer ve +sprink led +day dreaming +boudo ir +nicol asturgeon +be im +motor speedway +ane ur +acron ym +mer cer +facil itation +d ass +as il +,, ,, +tb ol +ba er +auto correct +won ky +the garden +remn ant +mv ps +mun ity +ling o +kar am +is ma +dignit aries +boy hood +st inger +marath ons +lo fficial +jo ero +flat bread +er aser +court side +y ville +n ila +li mo +im ho +tho se +pre viewing +missou ri +explo its +cry in +( ~ +y ooo +sal ma +po cahontas +ex uber +an ad +united we +pun cture +explo iting +deci sion +cauli ffe +be curious +⼠³ï¸ı +z av +newh ome +carbon ate +bust ling +bts fanart +az ur +ade bay +ac cli +kit t +c love +bur lon +ภĤ +new town +im perfec +hit z +depe che +carne gie +twitter blades +qu art +nu isance +ih sa +t series +knu tsford +doug all +at ourism +and beyond +bli ster +am es +prob ate +ex ported +ca icos +toast masters +noo oo +fa kes +pe at +maa stric +ha rem +bha g +aus vind +preli m +chippe wa +b ni +po g +pa wx +t day +e ep +benedic tine +trigg ering +e chi +v k +pretty little +har k +mam at +el more +cu ad +ar nab +j hs +c mp +v ra +stor mers +lan ier +jar rod +ice hockey +wren ching +wreck age +om ia +na shik +ar co +sveng oolie +en tran +bake off +thisi smy +sw r +grateful dead +mus sen +m ff +fal co +dor se +win n +prin cer +ðŁĺįðŁĺįðŁĺįðŁĺį ðŁĺįðŁĺį +fir mino +yu z +sk op +mum mies +mor si +l ca +art pop +ame er +qu ant +confe ction +cag li +mccla in +se ye +s int +ro bi +ra aj +á ¶ +way ward +mi mi +back side +ãģ Ĭ +regi sters +gru po +dis graced +i ghts +analy sing +advance ments +trum peter +tr ice +stlouis blues +sapp oro +ofthe month +j son +j cc +c of +bo real +anz ac +ro ch +pan tal +ny rangers +n nam +ic arus +dre au +ë Ķ +mo an +ðŁĴķ # +yann ick +pope in +ma sha +house keeping +gy o +giz mo +book stagram +samu i +ex xon +cri sto +chi sholm +sas qu +ric cardo +rel ativity +lu i +d ori +we can +super cup +se aton +func tional +chil is +sf r +p wd +le eu +l ha +ide as +and a +fli gh +ash u +tou rof +starwars rebels +per alta +on an +descri be +daf fo +se ma +monaco gp +k ink +himalay a +gi za +chim pan +law school +j z +im mobili +dick erson +chan ey +chain smokers +and hra +vir al +over take +madein usa +et ano +ca ther +quin ton +ik onic +co pley +anc entre +amal fi +ðŁĺį ðŁĻĮ +super bly +q z +hu is +sier rale +my name +em ph +yi sts +snap seed +self love +r cs +shout gamers +newsl ine +gn r +ec co +ca vern +ha pp +environ mental +dream in +ag il +! ðŁĺĺ +winter fest +sh hhh +s ye +sch on +mlb draft +bla ise +dunfer mline +be aming +a star +ðŁĺ ¿ +tu fts +har inge +f ü +aq sa +abu bakar +!!! @ +wad ing +fla panthers +dun dee +bo hol +rejuven ation +ers week +cor se +wag ga +tor o +tat er +sa ira +o tra +mck night +for thelove +t tawa +baff led +lex us +davis cup +sw ick +penetr ation +b dn +whe res +twitch tv +pr ing +heal the +f ms +tm j +pir lo +me zzo +man ley +lovel ive +hu ffman +ðŁĮ ¶ +wa w +to toro +cur tiss +chi u +bil is +ti kka +r td +im poster +edic ine +olive ira +neat ly +é Ľ +wells fargo +s mbb +f ick +alltimel ow +shim la +sat l +rein venting +pen h +san te +nu kes +im pal +bohe mia +ðŁijıðŁı»ðŁijıðŁı» ðŁijıðŁı» +stin son +bt l +ëı Ļ +ul tron +sand ing +n th +mir amar +bre l +bre cken +re draw +evi dently +e zy +re unites +miami beach +indian army +crunchy roll +Ø§Ø ª +l of +gg en +fl ay +do ol +wil ds +th ir +rent ine +ren nie +napp y +lesli e +ag ec +wood side +vi zag +ro ker +over loaded +esta dio +ภĬ +un u +kah lo +hi king +ðŁIJ ³ +ta king +ol m +ingenu ity +el p +common wealth +baha dur +wiz khalifa +stray kids +southbank centre +show er +humph ries +de vol +cor ny +micha ele +fon do +anc er +è ī +ron in +ar ou +proud ly +pp d +donnie wahlberg +copy writing +cap a +bro d +un imel +ome let +le berry +eccle si +cla ret +ter ro +si fy +lon dres +legend sof +doub ted +ö z +sony tv +rebe cc +vote yes +tv show +sun burst +de y +benef it +z ico +t cd +rejuven ate +don ato +to is +im porting +fa kh +e fe +and ela +zind agi +love y +high school +gordon ramsay +fur ries +f cim +chi vas +ax ed +p mc +jay am +brew dog +gam y +captiv ated +shout outs +sab bat +ru fai +lat enight +descrip tive +s racing +pr p +nad in +mushar raf +grump y +gn arly +font sunday +fon dant +classi fy +ðŁĴĥ ðŁı½ +ryder cup +pne umatic +i phon +ra glan +mam bo +gilli an +enig matic +cor dova +spoti fy +har ish +emo tes +ar gh +m bi +love to +cur ve +ad ore +po sa +pa inte +be gum +> @ +ro che +mag i +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı +x lr +stoo ges +newsline weather +wc l +linkin park +bush wick +hei ght +cla pping +capp ella +bad i +loo t +def con +super hero +shore ham +mc c +k lam +ale ducation +é Ł +the democrats +sher ri +dioce san +d mb +sen sex +lovel iest +ai ko +âŃIJï¸ıâŃIJï¸ı âŃIJï¸ıâŃIJï¸ıâŃIJï¸ı +gra z +cla sp +chec o +ar nie +stra d +dar ou +britt ney +bra h +festi ve +er ley +the blacklist +tb ay +pau lin +basti an +affir med +stre isand +gan esh +stat ute +re load +lu l +id is +youcan texplain +nu tt +migr ated +zi ps +pro dig +ma geddon +for ging +ðŁĺ ¨ +st mas +plu gging +dur o +correc tive +t elly +sj p +pi et +anu i +adap tations +v ant +myel oma +cap one +sier ra +black water +zeph yr +yon kers +thr ac +screen cap +pa seo +mi kes +lock wood +h rd +er rol +colum bus +ab al +pp t +indv aus +char lo +par aphra +daniel e +r joseph +hir sch +carol yn +thro ated +sli mming +adi os +v logs +mun ching +j akes +fi k +bar rage +shan gri +pin occhio +pa kh +min as +icha el +diversi fied +caes ar +ome try +ham ble +cuyaho ga +bai leys +seat belt +jeopar di +brown sville +scandal ous +oni ans +ble aching +found ation +the le +rye o +kaf ka +ja ja +feder ic +fat al +best price +bandic oot +ðŁĺĤ ðŁĻĪ +kor o +fac to +dispen sary +br ation +ur ray +makeameric agreatagain +wit ness +toyo ta +pat ties +black board +ad is +te rer +ss chat +sh alt +record storeday +la da +gi ann +íĽ Ī +un holy +kh ana +godis good +palis ades +he for +ci ve +hered itary +hay wood +cor ker +spr ingh +sand i +re du +natu ro +many vids +jessi e +âĵ Ĵ +schnit zel +o vie +gren ades +gat es +ab ed +ms ft +medic ally +led ore +l ousy +mentalhealth awareness +glee son +col ly +cabrio let +wee e +sp end +snow mobile +hi j +Ï ĥ +tal kies +rich ness +jor dy +giu lia +acti v +do pam +alleg ation +your life +sk elton +v ny +mu riel +lat t +inaugur ates +foreclo sure +tain er +harne ssing +aguil era +x rp +coo lidge +car ta +ser gio +news radio +k tr +sol arenergy +r sprasad +home design +ho stages +hat a +al ali +thal er +a sturi +tri pura +hydro power +free bie +escal ating +m ha +getin volved +protec tour +od is +mus ician +mu le +u wa +ter iyaki +rip city +race horse +loaf ers +kha o +fi vb +bal con +an ou +ðŁĽ « +vis ayas +sh all +fire flies +ठķ +re morse +pho tonics +let splay +imp lied +hes itation +gur ney +ol om +une ar +pi d +mo gg +itt al +âĿ¤ï¸ı ðŁİī +ma ku +ar man +mo ke +han ts +cor fu +ä¸ ĸ +digit alization +ti ana +su bo +schu yl +e redi +ven cy +v room +ro ars +growth mindset +cosme tic +chir p +stra u +seh wag +ric ha +pin ellas +elo tti +dur and +deta chment +qu ando +mau sole +ma su +black wood +aspir ation +bell ator +shi ka +mar oc +ki ra +pi k +gta photographers +gand alf +sto y +spee do +mand alay +fan o +un ice +sol ange +po pper +den ch +ne warri +cel ta +d lers +ce tta +cc f +black smith +bhan gra +w anders +hide away +employ ability +z te +under take +tw tr +team building +ta pa +virtu alization +pro vo +eigh ties +che ery +ay u +we ber +per ro +inspirational quotes +d hoo +aj ara +ðŁIJ ł +sub du +bill clinton +am oun +stro oms +soldi er +mouth watering +malay a +legitim acy +gr ats +syl vi +sleep ers +boar ders +ðŁĺĤ ðŁĺĺ +up loads +sports news +ske wers +referen cing +fo dder +ea a +remo s +ra ss +n ann +cor azon +alas ka +shak h +pig mentation +incogn ito +as ca +miamid olphins +le land +ig t +gn es +boo s +cla ps +major ing +di als +---------------- ---------------- +regi mes +pe an +emul ate +mer ga +med hat +head liners +go h +con di +wi gg +ser af +ric kie +bor ty +âľ § +re is +cel eri +âĿ£ ï¸ı +ye ez +ni ki +draft kings +diet itian +at weets +ampli fied +nee son +mac ross +dystop ia +bor ges +blat ter +b ade +direc to +bha skar +sch ae +kar my +scot spirit +moment ous +an ation +lm k +kne e +in da +ig g +bo tham +barber a +toi lette +r tl +metcal fe +lin x +clo thed +vo ila +se well +region al +ple dge +intere stingly +f nl +ru f +mun di +bur sary +bout i +âĺ Ķï¸ı +âĸ¶ ï¸İ +sto liveby +star let +pic stitch +car wash +aw ar +round house +margar it +manag eable +bon ito +us ab +on n +flow ed +cher che +s ju +kensing ton +jo yed +cal e +???? ??? +zu mel +wir th +re pul +gary barlow +coel ho +âı ±ï¸ı +work er +sp t +siob han +segu in +s gp +glypho sate +clin ching +charlie hebdo +bati k +uph eld +just o +her ne +euro star +mccull ough +fent anyl +diamond backs +âĺ Ķ +indian express +jis ung +hen e +conve yor +appe tit +yn g +over kill +kim mel +in it +y ff +st ine +le urs +her ring +ear ths +or ne +leis u +go pro +sky way +scri bble +by nature +water colors +tin tin +inter ruption +br rr +wide screen +shake down +knit wear +karab akh +id as +cor der +tt our +sac king +good ell +thisisy sr +goo ey +german town +fa w +cat amar +ðŁĴĥðŁĴĥ ðŁĴĥ +vijay a +ti ber +sa ad +ha are +seul gi +mischiev ous +isu per +hellu va +confe ssed +litur gy +in man +ce tti +tuni sian +tic ut +sur ging +sau er +ry erson +popu p +mb k +br itten +v f +gray ling +abstr ac +us p +un fa +so rely +ri ona +r dc +of sted +ju icy +horri fied +grac eland +fa k +justi fication +jam esp +bat u +. âģ¦@ +tu i +te jas +sul fur +indirec tly +a ford +under privileged +northan ts +m countdown +ji ve +ha des +lac quer +humbold t +gi ggling +jun ho +digital painting +aph rodite +ab di +tel us +pric ey +hahahaha hah +fibon acci +dis mantle +ar ne +an ine +willough by +motivational quotes +mid west +inter lude +ge re +be come +s illy +felic it +tap tap +st ings +ilovemy job +cospla yers +bra u +votel abour +sto ver +ru ddy +meh boo +hon e +gift for +phosp hate +it ano +do sa +babys itter +kri sty +dele on +dd ard +confu cius +stewar t +s worth +com ed +arn hem +a ño +ym er +smo re +pur suits +flee ting +dat ing +sav anna +delic acies +comic book +co arse +cir ca +calam ity +bro ads +pre natal +jo c +cyclo ps +c lec +yam amoto +un b +pu sd +plu mmer +per ils +gu mi +ath ar +val o +timel ines +ense mbles +b sc +maha jan +wa sim +techn ics +sorcer y +jo bo +havill and +themore youknow +ta ki +rest ful +mer thyr +cro ck +wee ps +reneg ades +lear nings +g lands +ti dying +ec tomy +dol la +ya g +rev it +kar ts +f natic +apologi zed +win dy +w bc +si esta +pe ony +gor an +autisma wareness +af fi +wh aling +v ps +train spotting +ra kul +mel anie +mal let +ky our +? "@ +reti rees +ip kk +in hale +electronic music +cur ators +pocon o +l sc +an son +u si +lu r +hide out +twi sting +samp led +jun su +ing s +dead ly +auditi oning +happy bday +emplo ying +ti voli +nic u +fu sil +ak am +palmet to +ofthe wild +my kon +mahar aja +deut sch +selec t +nat ura +me ddling +land is +i have +comm encing +.... .@ +ðŁį Ł +mer gers +m dp +ben i +ne led +lino cut +kn ack +j pm +battle ground +at ter +rat on +pente cost +organis ational +decan ter +cn r +boo zy +bap u +al ve +fast pitch +ðŁ¤· âĢįâĻĢï¸ı +z hang +token sale +hear tof +ha den +rapun zel +lar o +fro yo +bikin is +siddi qui +scri pps +pi ec +lin de +story board +red lands +op o +mb r +grace fully +c de +th enews +cas pian +and ali +ä¸ Ģ +ro ald +optome try +medic i +ken g +der ma +ðŁİĥ ðŁij» +mar chi +bi al +al ab +âĢĭ . +red wine +k gb +gun violence +gi vin +fan page +adri ve +ëª ¬ +tel stra +fl t +biome chan +ðŁİīðŁİ ģ +dram a +اÙĦ Ùħ +ti rol +de ferred +be sts +spr ouse +o hh +chead le +im balance +gy n +cruis in +ci m +un h +tra pp +ob scene +ir fan +impre gn +deut sche +cymb als +ob an +au er +atal anta +roo k +men zies +g went +entran ts +co pac +or tho +marksand spencer +lee ks +lac tose +spac ed +sh ak +pale ontology +ine mas +e an +bi bi +alban y +ìĭ ľ +ko in +be ssie +ar dent +latt ice +be sie +ade h +# _ +surro gate +sand hu +rain water +k hun +cau sal +be wit +at las +agu e +water polo +l ts +jam ess +ae on +sch ofield +motor cyclist +ge force +dre y +wai vers +sh us +no excuses +di ade +sweethe arts +macin tosh +l indi +ar junk +am writing +æ Ľ +luci an +ink jet +citi es +survi val +gam ep +g j +ðŁijĩ ðŁı¼ +zz ah +objec tions +dit to +ìĤ ¬ë +trump care +lof ty +tool ing +intrin sic +fenerbah ce +cle men +will power +su tra +sim pl +save shadowhunters +san aa +pri mate +or bit +kel sen +asho k +artisan al +ê¸ ° +tri stan +shre ya +midwi fery +lit ers +la dd +fla x +bry ant +nau sea +la vo +ul m +sid hu +she ena +gott lieb +cr n +we id +âĸ Ī +motor ist +ma vic +laut ner +endo wed +spar rows +ka de +ip p +o vens +ti led +stin ks +keen eland +kath a +c te +mass ages +interro gation +ðŁı ĸ +sen sanders +fish in +dro se +ðŁĴģ ðŁı» +sustain ably +sh ant +propor tional +mis cell +kh l +chemi sts +m ra +her pes +f lux +disgu st +bon nie +artin fo +~ $ +sau dis +pollu tants +op ia +mo fficial +dark side +cultiv ating +civil isation +champion ing +bl f +armed force +ðŁĺ³ðŁĺ³ ðŁĺ³ +tian jin +lar avel +fe men +baf tas +absen tee +ra onic +provo ked +pla gued +cool ers +uc davis +sand er +s books +di orama +un ab +sub division +pritz ker +pa sty +j ks +account ancy +tri bul +re tta +er ty +! ðŁĴķ +ðŁıĨðŁıĨ ðŁıĨðŁıĨ +ri beye +theli al +nin ja +g ls +cer ro +usa id +pu ma +pascu al +chev y +brati slava +bra ga +bi gs +ap nea +åĨĻ 羣 +sd p +marguer ite +khu sh +vec chio +glit ter +el issa +dumb ledore +car gill +ann am +trium phs +templ erun +ru min +lny hbt +cla sse +êµ Ń +ri ri +gun ning +boy e +am ento +limite dedition +gra w +gan ache +ðŁĮ ½ +resemb ling +on tv +moti fs +i mani +hel ms +epo xy +clear ances +ba ha +this day +re eling +gur o +fi eri +faw cett +chec kered +ti v +narcissi stic +i tha +guil le +go e +dart ford +comment ators +cl out +ch illa +ky li +hun d +ro maine +jum bo +fil ip +de au +tyler rjoseph +the un +orphan black +om ans +in manila +tho reau +sa ar +ra bin +en heim +tn t +state parks +kour tney +we th +kair a +ec r +gas par + ¸ +olu tion +me ars +home town +execu ting +defic its +car bide +blan ey +stri dge +sh r +ho tty +grow yourown +fal cao +îIJĴ îIJĴ +âĺģ ï¸ı +un wavering +more tz +hoo dy +h ine +dain ty +bak ing +ภŀ +salom on +disin formation +pu sha +la the +ad di +abi ding +zig gler +sic ilia +mening itis +hol ling +aus gp +ri mes +barrac uda +al do +af tra +pe di +lith gow +analy tic +vanc ity +se f +pc as +c ya +afric a +w ella +ra ys +n cat +fe ez +don i +din amo +breast stroke +truec rime +tofthe week +south ampton +el ina +zain ab +sw az +ph elan +kri stine +k lit +er ation +bud d +wrist watch +the week +simil arly +qui k +over throw +naku ul +itali ano +bigg bos +se ashore +arnau d +le p +fan site +ding o +cler ks +cas illas +jo die +de generation +ãģª ãģĨ +her ze +adjun ct +ac ard +ðŁĴĻ ðŁĴļ +à¤ Ń +wa al +rad hika +chim es +ti pp +o or +ki ye +he c +ba hu +ab at +sam us +inver ter +indi spen +he ge +ภģ +l ff +bi ele +mu ja +indone si +f wa +beat rix +bbc football +sa ks +q c +cont acting +att enti +vivi en +taf fairs +so crates +sen e +se mper +co bb +ðŁĴĭ ðŁĴĭðŁĴĭ +ladbro kes +á » +will ingly +p are +p able +occi dental +ich es +can tor +rox bury +fre ddy +ed ger +de capit +wgr z +ma im +ku sen +ulti ma +solar power +schan el +dishon est +beck ons +yad av +over ton +ne ale +me sses +darou sey +auc tione +ap aaja +ðŁ¤¦ âĢįâĻĤï¸ı +u tri +.... .. +taco tuesday +prote as +introduc tions +pi ds +gat linburg +pam pering +marig old +f isa +con tor +clean est +short stop +ne em +pin kie +let our +kathle en +imp ly +ðŁĴ · +work out +ro que +estim ation +countdown to +conom ics +al tar +q e +jesu schri +grayson dolan +st ines +smith field +shi gh +ir replaceable +spr itz +mon op +lo real +lab elling +fli ppin +ðŁIJ Ĵ +sher pa +jan el +embr yo +bir dy +ball game +vul gar +un f +spra ined +cr f +come back +col man +att y +at a +ar lo +u alberta +technicol or +rush more +respon der +ou re +obli ged +mt ve +mac c +wood house +vent uring +sen john +light ers +her bi +wall ace +exo tic +ag em +virgin ity +the ma +mart ine +mar ang +lee teuk +ho list +watch men +s ile +le ck +kan ji +hoo ters +st ile +n ala +e am +clari fies +am é +voc ations +succe eding +jesu is +conqu ers +uc berkeley +nz v +cas o +bar ist +bar bed +patri arch +p cos +ben ign +re read +mn ang +fly weight +fc live +cre t +bar ks +lione sses +benid orm +bc z +an je +ti wari +ol li +nak amura +ha san +fun imation +co ss +bir der +anna bel +tim er +rn cin +real ale +ra vine +no s +ðŁĩ° ðŁĩ· +te ke +sa hab +bal tic +âļ Ķï¸ı +sur g +ka hit +tru ss +stein berg +replic ation +elev ators +ðŁļ ´ +ma ki +har ming +h si +del ena +t ke +sime one +pat ina +shutter stock +she ars +aller ton +ai ka +temple ton +raf ters +perio don +note worthy +mongod b +man fred +ko wal +stu b +join ery +an gie +u kenyatta +shel ving +kenil worth +instru cted +ta ey +retri eve +interpret ations +first world +d si +biop sy +benef iciary +saf a +philosoph ers +pand ya +ner i +bow ery +wel ly +the cw +spr ites +ner v +mon orail +jac uzzi +de mysti +con sort +program mers +news desk +l sa +hong kong +home and +ed ale +dor p +dar in +vali date +tear drop +syn ap +repeal the +premi um +nik hil +blan k +ai ding +squee zing +rncin cle +ret ard +park sand +bru ck +wr u +su zi +specul ative +hew lett +cru st +as m +app end +¢ ħ +tele mundo +st aley +sn rtg +samu els +jae ger +farn borough +womenin business +ron darousey +min es +au de +shi ba +m ne +estre lla +swim ming +ë¹ Ħ +tam an +it é +cuth bert +casu al +ag ing +offici ating +li gan +fo iled +ver i +ms r +dr ay +usf ws +ta heri +se thi +ç ī +uneth ical +kick ers +hi jab +ak ash +pur po +pellegr ini +neu mann +man del +de ver +ap u +ìĭ ł +z ha +indic ations +imag en +clean india +car pe +so fe +mart ine +stra pping +wit ter +no tin +fic ent +bbcc ricket +tur ally +cou rier +tri xie +sw am +i ab +alfar omeo +stal ked +so h +oooo ooooo +miasan mia +con f +thisgirl can +tar rant +re reading +present ly +pow ys +nj devils +mart i +fe b +cerv antes +tam bo +retro games +lang ston +kell er +ar nol +ठµ +shinju ku +sasqu atch +dan ica +akrish na +so ko +cat ter +car park +inde cent +er gy +bur ley +brother ly +xi v +post doctoral +polyne sian +suspec ting +mass a +make ithappen +fab ri +cu ti +cor sica +bor den +un impressed +sli gh +he don +gon zo +fic o +eloqu ent +dic key +podcast addict +le ona +jewel ers +wic ca +uniof oxford +u den +gene res +ban shee +u ya +she khar +doll house +blu eno +af alls +wra ith +ta fe +moun ds +j ct +in clement +fun g +fluori de +! âĻ¥ +raji v +b me +waz e +squad goals +pre ak +hand painted +c sgo +sat h +leg oland +in la +d pi +c actu +aband on +tali b +janet jackson +ãģ Ĺ +khal sa +gl c +c fm +ab ang +ali sha +we m +sur passes +ou st +nai as +max ima +lind bergh +lic o +it syour +h ä +gul li +anacon da +woodru ff +pr m +h é +anonym ously +sun nah +scat tering +sc int +sal mond +pe king +j cb +ed ine +diversi fication +ari on +all state +t ley +gam bler +b hatt +ra ku +pit ts +j enga +ri di +pun dits +papp u +now spinning +ha drian +az ure +autom o +aran eta +a stray +il m +yong guk +wel ded +parano ia +explic itly +co f +ðŁİīðŁİ ģ +som uch +post partum +ler c +gu ava +enhan cements +ber gen +con glomer +âļ½ï¸ıâļ½ï¸ı âļ½ï¸ı +milli gan +% ). +âľĮ ðŁı¼ +sh yam +ry man +megat ron +koh ler +de schanel +chel sea +zoo topia +wr t +valle jo +tri pp +positive vibes +irrit ating +book fair +aac r +san it +di sk +depeche mode +classi fieds +ðŁij ¦ +ven erable +ra ves +fav re +ek iti +quarter backs +he ma +h sd +g ach +con template +l ant +kh atta +inter sections +harle quin +ncc u +m dr +pp ro +il legitimate +gre be +ler man +ite ach +cspan wj +voy ages +sat x +rednose day +oo king +dan ic +char lene +a hor +ty sm +gri ffins +cheng du +boo ka +âĢ ¼ +ye h +ur ra +tari o +pou ches +dd ddd +staf fed +revo ked +ran chers +ou z +oni ka +share d +n bs +li mp +etsy seller +cl one +visi e +ksat news +good life +cow l +chic o +н ÑĭÐ +vigil ante +skate park +re it +mar av +man ja +le et +co der +ðŁį ĭ +w gn +ld c +duck lings +don de +avoid able +í Ī +over lord +mp r +anc elotti +intermitt ent +colli der +ste ins +sque ak +dispar ity +colorec tal +clark sville +ado u +Û ģ +women swear +di bu +bar ts +sa ws +recogn ises +re pa +m cauliffe +hear tache +good music +gg ish +Å ¾ +tech house +aci dic +re pro +mal low +d printer +cro tch +kir sten +kan pur +il kley +fan i +ev ry +dit ches +cher ie +bat angas +lost girl +liam payne +la be +rid dim +den ni +? !) +vo o +tw c +s za +fo als +fic tion +lim ate +dab s +mc f +er na +smriti irani +espo sito +duc ts +blackgirl magic +weare one +sen ec +sch ie +nath singh +fo ia +en or +don bas +ç İ +wi jk +over lap +gron kowski +full er +e ssie +ðŁĺ¤ ðŁĺ¤ +ye ye +loksabha elections +guev ara +bi es +âĢ¦ âĢ¦ +z he +pic a +homer un +con stip +apo stro +refr actor +sa an +o gun +man sions +re sh +nam ely +high er +evan ston +up town +ring side +naz arene +ir regular +invent ors +pope ye +mnang agwa +fern and +sp litter +mor ten +asc end +whi ff +cr ink +be ste +at to +shad y +el n +bjor k +ðŁĴ Ĭ +no tting +england rugby +be tawards +we all +dan is +Í ¡ +mo she +miff lin +lu walia +al un +li pid +ha van +cmp unk +bur row +underestim ated +ste yn +pul pit +pir i +p nr +jer ks +fin ney +tra volta +roman ce +buck wheat +black metal +st oops +intra day +huff post +ale state +so ver +pat o +obstru cted +jen na +fi o +bo den +bill on +my fox +la hi +kar na +bartholome w +vin o +lovel and +ecu men +whitec aps +the king +qu il +human oid +alab ad +g ome +fri ar +c bre +broch ures +we si +mal anga +dr brianmay +assi si +sav our +prefe ct +mt ns +inter changeable +hex ag +gar nish +c ce +amar o +ðŁį ħ +gu sting +dismant ling +bay one +tv showtime +sun ion +rad ha +ozar ks +nautil us +la youts +did sbury +dance floor +suicide squad +ok preps +j ura +alum nae +ron ni +ma f +george tte +coordin ators +bad u +ðŁı Ŀ +ted ness +re prise +pain fully +jo ie +hill billy +thisi sd +ss er +oo ka +mm g +meredi th +klim t +kat an +blood borne +bab u +trivi al +th rif +o cla +mo yo +milk weed +âľĬ ðŁı¾ +sul u +occu pants +farm life +cru sty +sing in +r tr +penn state +met adata +fab ulously +wee zer +schnau zer +rash tra +ni moy +h cp +freebie friday +do le +sti me +jef fre +gro ban +broad ly +bil oxi +woo hyun +u zi +ny jets +fi ver +vado dara +it ake +check up +supremac ists +ro mp +the sunday +contin ence +liz quen +ha ut +ellip tical +sho o +pan ty +pa as +immigr ation +park service +os age +i sta +homes ick +cyber attack +colom bi +boy ne +ðŁIJ Ĭ +ty lero +iron maiden +hand a +ch off +al ya +ry o +red acted +mo ts +intern ment +die hard +>_ < +ðŁĺ© ðŁĺį +m sle +li bel +f ant +embar go +soc kets +ski ers +photo journalist +mchen ry +bou cher +ric ard +jayam ravi +dock lands +annot ated +ag ata +ðŁĶ » +prev ails +new ington +las sen +hydr ant +te o +rough riders +murug adoss +at su +afil m +admini stered +v fw +calam ari +bay less +sw ung +sag er +ple ated +mo dric +hi my +golf clubs +citizen science +rehear se +pic kett +le aky +polit ely +gra zia +sk or +sar an +cl inging +âĢ ł +ther oo +squir t +on your +bag gy +at oll +th ys +coul son +vivi enne +s anya +recogn isable +gr u +scoti abank +milk shakes +fundra ise +d mu +bu tters +ra wr +lin dy +al ed +sk am +ryeo wook +referen ced +quad r +cri sp +bio informatics +âľ © +che wed +sm acked +commend ation +classic rock +ðŁĵ ½ï¸ı +star ved +pu ddles +do sing +bru e +zah ra +wo king +sun rises +stro p +sal ons +lif ters +cor coran +ala ina +kom onews +ken yon +gre tsch +ge zi +floor ball +fi q +tw om +re clamation +ineffe ctive +cop tic +cit ic +ch ute +unle ashes +ran s +mo sc +joe biden +sw il +osi ris +ni h +embo diment +cruci fied +ãĥķãĤ ¡ +schu mann +horri d +pri mer +northampton shire +je b +graph ing +back street +ìĺ ģ +wood row +tar garyen +t bl +sky ler +ru ffle +joc keys +info s +deir dre +bosp horus +âĻ¡ âĻ¥ +p led +chu an +bi got +ren nes +ra va +parmi gi +chi ar +vide om +stau b +exc ali +ex clamation +city council +barnab as +se du +ker ri +is che +fr actions +fly by +bau er +where in +ra ge +ou lton +mu ah +co stac +co lec +char mer +capit an +secular ism +mumb a +hu k +hen e +blon de +so dia +r tb +de coding +cad ence +art an +ðŁĺ ĸ +too cute +tn c +chen na +brux elles +à® ¤ +t ft +ss ssss +sp ital +poun der +p ch +mega star +in junction +al ent +æľ Ī +x k +tro pez +tombrai der +m mi +amp i +tac tile +sel ina +ma sai +good bye +dod ger +al fred +vote maine +qu ads +ad die +sep ta +s brewery +ric oh +monte ith +humb ly +histo ire +me agan +low ery +del o +ab il +out numbered +er os +craz ies +r bg +pollin ator +il c +gin sburg +cy gn +ab vp +wi dest +rescue dog +ho re +y g +sof theday +rac y +la ban +i bb +ci aran +robin son +ali kes +fre n +ban bury +ball point +atten dee +ati m +ìĹ ĺ +pi p +ergon omic +ad re +rem itt +pir ates +over see +it sen +hali fax +dedic ates +cycl on +as wamy +ãĤ ¯ +mo ser +ge op +d mg +chatur thi +chag all +bu gle +ðŁĶ · +samp doria +low n +davi dg +newsmel b +fix ed +cordill era +ar ri +ðŁIJ Ļ +y ra +to pi +n aka +g é +d life +beau ts +p date +min ty +hy uk +er am +e bert +at chee +theno tori +ni kit +muham ma +fan zine +negoti ator +gad ot +bli stering +bg su +yoga day +un sw +john deere +zhen g +struc ture +porcup ine +cl ack +boun ces +qu ali +mat ador +otta w +mo es +gil git +esp lan +dy strophy +b dc +ठ¬ +sh illing +one world +on k +formu las +tru ff +teach ing +robin sons +rij ks +qu ays +her ry +fle dged +fish eye +guardian softhe +du bl +z ane +sty list +qu ack +leah y +j air +hawa i +diversi fy +darth vader +und ated +su pe +< > +p oms +memorial dayweekend +lu ms +kc mo +de hydrated +taptap fish +re com +pode sta +j ir +far med +cait rion +bor no +bc m +sk as +se dition +rub icon +noo sa +le page +haw kes +additi ves +a ol +th q +ru ta +jo burg +bas k +yan ks +mont blanc +lu a +jam mer +cho ol +rou baix +re clin +mat ured +dimit rov +ãĤ £ +m ites +âĺ Ŀ +ment alist +mam ata +g na +fla thead +canad a +caled onian +bo we +yor kie +re print +g bo +en thal +al ka +x com +u ka +tr alee +spread the +ni i +joh nathan +human kind +ha sina +du vall +craft buzz +ton ic +re lli +ra ged +jo f +free masons +____ ___ +un ter +sc aff +lee ch +extor tion +ari e +ra ghu +pin eda +att an +vehic ular +ra oul +lu pe +eng er +divi des +cr b +cap ture +bo ii +maricop a +hearti est +fair child +alter ations +no ta +mar d +s kids +no k +gui deline +deplo ys +carni vore +g sw +au ren +at ac +ani el +of c +ko vac +web site +spectac les +sn an +sher yl +rever b +nik ond +n ce +big sky +ra pes +mat o +night ly +mp n +ke sbury +jep sen +gul p +french man +bar ley +andor ra +king pin +in ns +dra shti +catar act +cand o +p co +last ly +herze govina +gr ounding +íĬ¸ ìĻĢ +testi fied +talk show +f day +even song +eat on +tor ment +love ee +cooper stown +al gi +ðŁļ Ĵ +rare disease +meth yl +lu isa +inbound marketing +ec re +t sem +ro sam +nove m +more ton +monster mmorpg +home buyers +fru gal +escal ator +de sa +boul der +bel mont +âĺ łï¸ı +twitch con +tur ners +sal ut +per vert +le ye +hiber nation +gg ggg +food service +ex porters +d assault +after market +we athered +un ravel +si di +plu ssi +mp t +ky m +bo spoli +sand piper +gole se +col itis +br al +adventure rs +ìĪ ĺ +ve do +preten ds +b da +sac re +ju m +hin ch +acci o +hy nes +back stroke +ly ce +h ve +v bs +ou m +brand enburg +âĿ¤ï¸ı ðŁĴĽ +t itude +spe cu +rapp ler +raje sh +dream land +av in +ma rek +ke ss +ho oman +pu tin +par then +hair do +aaaa a +è ģ +votemaine fpp +sha stri +remedi ation +it m +hai fa +ex xon +empha sizes +u conn +sim monds +mo ire +e scher +cru tches +vi vi +ar mitage +air ani +shi pley +prou ddad +mar v +inter nationals +bb k +prostatec ancer +mon ash +task force +god son +levan te +barric ade +tr lt +rapi sts +p vr +kimber ly +ge p +eri de +acknowle dge +will rock +u pland +aa i +willi e +une dited +tri ples +or ch +hai ku +gene see +en gel +bay e +plex us +pen zance +ko bane +f ash +cac c +xi p +scam mers +sal i +notic e +modern ity +chi en +bis cayne +nba allstar +mar ston +june au +ðŁij ¤ +van life +ro sters +r st +labra dor +brek kie +ro ssi +my x +lmfa oooo +je tta +ãĥ « +zo ya +w pg +mer chant +goul ding +food for +dosan jh +connec ticut +it to +gho on +can aan +bo le +retrac table +mar zo +lau ght +discoura ged +dan forth +a holic +th d +marri ott +in lay +duplic ate +c zar +woo fer +scre ek +ni pi +intimid ated +god ard +elu ci +ab omin +sco ping +program mable +mexico city +me tat +h mrc +h man +ashi sh +sierrale one +in sha +hero ism +gu staf +rein hart +lumber jack +gu sa +fella ini +eu f +belgi an +bab ys +an ski +yu gi +spot lights +rockand roll +k has +dark ly +but cher +bal lads +auto cherish +water man +peps ico +lo ggers +infinity war +as ach +ðŁ¦ Ħ +spring boks +ðŁĻ ģ +sp acing +la val +ffici ently +endor sements +ed am +sharp ening +sor rel +prev ailing +ove rex +nbc snl +native american +ex oner +rin gof +ml ins +disobe dience +pre car +cic ero +âĿ¤ # +per taining +mani fold +su gi +of africa +mu sh +morgan town +gel ding +co zumel +ny x +ab ili +ðŁĺĤðŁĺŃ ðŁĺĤ +world cancerday +pal acio +go k +ecoun try +âľĿ ï¸ı +ro tary +íĬ¸ìĻĢ ìĿ´ìĬ¤ +spin dle +neck line +maneu ver +mary land +i player +bal led +am ended +ne b +en rico +el dorado +dubu que +blood moon +as lam +comple te +a of +wil ma +the band +kn g +revi sta +mal ak +goha bsgo +car y +ve en +tung sten +snu b +shab aab +kil marnock +le pi +re eds +ne m +can op +ble d +vali dity +ju icing +hun a +chauffe ur +water town +wait ers +lo d +fel der +dow nie +don ate +san down +rh schel +or f +lulu lemon +kare en +galac tica +bre vard +strat us +rose ville +fall acy +-- - +w eneed +san kranti +local gov +billi e +ac cr +k illian +comm ing +worldo cean +watan abe +sothe bys +sin ful +rol i +lynch burg +integr ates +hefor she +an p +al aw +ðŁİ¶ ðŁİ¶ +maxim ilian +g ma +bre tagne +ail le +world food +turi smo +ra u +mini van +dis belief +bay ashi +us atf +skate boards +uy gh +plo ws +don bass +sugar cane +cham omile +yel lo +in quest +end ron +ani sts +te eing +cos worth +tune chi +sothe by +sle w +sher o +le f +ho wit +gly ce +joshuad un +enfor cing +baby lon +reck ons +apu ram +toler ated +resur facing +pla inti +o tr +kylie minogue +dis able +pod u +mul tiv +inter cultural +gy ro +goal keepers +cac cia +ag am +íķ ij +vic o +tri mester +ka e +coffee shop +tsem tulku +sta sia +go coogs +ds worth +wast es +boun ce +u vic +stri der +m Äģ +laur yn +kevin smith +jan ssen +ðŁį Ŀ +the one +same er +cret aceous +! ): +re gent +project management +person alize +mah di +zah n +kra f +vivi d +ten news +re re +ogil vy +tiram isu +photo bombed +n ke +cau stic +ra hi +nca adi +lauren jauregui +ki ku +i aa +dine sh +dal ton +ch ata +ba ht +x as +wi ping +ripp les +de generative +ðŁijĩðŁijĩ ðŁijĩðŁijĩ +vodaf one +thou gh +sla pping +naz ion +mer rick +intern acional +do y +bucket challenge +y au +vi stas +present a +mal t +expir ation +e ich +pu ttin +ve su +si oned +ct fu +circum stance +blu er +. ), +pra ia +mcfar land +ju eves +hat tie +du bey +myrt le +asturi as +ðŁİ ĩ +reason s +gri z +co ils +at lu +mode sto +lo gger +iber ia +af tr +v ang +transp lants +prairi e +maur it +haha a +t ink +sci fest +sc lero +random ized +glam organ +feasi ble +splin ter +blu eline +subb an +nakuul mehta +m ch +johnny depp +car cas +vin sky +hex agon +* " +women empowerment +s ooooooo +invic ta +gla de +ðŁĴĢðŁĴĢ ðŁĴĢ +we bb +desig nate +con figure +ay l +steven universe +spo il +recep tionist +plo t +na is +!!!!!!!! !!!! +ra gh +ne war +fr anta +sar aki +pi aa +obse ssing +m anda +f cc +em d +ba ins +rev ital +ki shan +fe c +ep son +def lec +cbs news +camper van +c ty +be so +at tribute +ari ya +ì¤ Ģ +tun is +out flow +not the +mykon os +clare ts +ci pri +brain wars +sign al +min ing +tr amp +ten go +jimmie johnson +christma stree +am x +a ung +v z +tran spo +spr incipal +reve rence +nam ma +key ring +ne gro +ate show +ingle wood +inadver t +impre sses +awak en +avi es +sale em +ess ence +es sel +doo dle +win ch +son e +nett les +mi me +alzheim er +vall is +insur ance +h pm +as sam +tun ity +post p +monte carlo +luxury lifestyle +ju dd +in sensitive +wax ed +was a +fand uel +chor ley +bon y +back packer +te urs +ho ag +surf board +stati st +sober ing +f anta +bob s +bar dot +ban i +ad da +rec tangle +pe tro +pa ign +n ang +influen cer +in u +hard line +gwen stefani +wood cut +s ors +for nia +cater pillars +ãĤ µ +tari anism +shel lac +nd su +illino is +el gar +cel list +ato day +ðŁĺĶ ðŁĺĶ +silver stone +log ic +fac om +weal thiest +sun tv +gy ang +com posting +candle stick +can more +z ard +x t +sc athing +ps r +north land +collec table +wa si +rip on +qu is +ge v +se men +ough lin +ob x +intoxic ated +bay side +é Ń +van poli +tim hortons +taji kistan +po i +peep ing +dri zz +contempl ation +ballo on +un folds +tre aties +aishwar yar +l hc +foodie chats +excav ator +cocon uts +apple bees +ambigu ous +repar ations +man goes +kh mer +fr itters +do wd +antonio guterres +cu a +impro vise +g fc +ec cles +carcin o +ðŁİ © +theori st +sj c +mu je +ir acing +camel ot +um mah +bachel ors +lumb ar +c sk +bi ya +re fine +queen willrock +ci os +bicy cle +bali k +à° ¿ +wil co +ow icz +loy ol +wa is +mer al +har ma +ba ili +ste e +jo ker +hein ous +bu oy +the sun +nou gat +ic ant +an anda +dji bouti +s night +relationship goals +polit ici +mb appe +google maps +ba shi +ta ap +now listening +flaun ts +ðŁĺĨ ðŁĺĨðŁĺĨ +vene to +car pent +bre do +ot l +di vert +br oughton +flat tered +bra denton +as ada +alo pe +ar b +an akin +ten cent +nad y +constant in +sb l +reinforce ments +ol b +jol li +il han +yu ri +the other +punk rock +lanark shire +bi st +ti ma +th oo +sun beam +r all +kar achi +iy ama +dhar am +k state +bat on +ay yyy +tre v +br unt +ar ashi +à® ļ +zu ka +can es +ar cade +uuuu uuuu +ma han +to pes +sen try +myster io +mis sa +deliver ance +ëĿ ¼ +un wrapped +ol li +holi sm +h pe +free view +calab ria +twent y +po g +pe cans +gru den +^ __ +ðŁĶ ¨ +os c +leng th +jef fries +ðŁĴ ı +sh el +mesmeri zed +ke gs +er at +cor r +caver ns +british library +⼠³ +pap y +al or +w age +jac lyn +co ols +bio logists +tur d +plau sible +ðŁĺĥ ðŁĺĥðŁĺĥ +yu mm +tic ho +te atro +ash lee +ve iled +umb a +obli gated +moisturi zing +magaz ine +long board +dang ling +c elli +west bury +v ä +ucl final +hou ser +chap elle +admir als +par o +mutu al +clean eating +vacu um +kh our +hab itu +foo din +fil mis +es con +sm it +awar dees +௠Ī +wan ds +thug life +mc cut +calyp so +Ù ĥ +yas a +radi om +hen ley +fron tage +critici zes +access ori +sardin es +id ge +gold finch +front end +d wi +beach front +# ï¸ıâĥ£ +âļľ ï¸ı +sh l +sh eng +rai shi +mari ota +fen der +con do +un covers +de van +ðŁĺĤ ðŁĴķ +renov ate +compri sing +tamanna ah +mash allah +humph reys +en acted +ar tu +wo e +sine ad +om ore +del phi +bre n +bo z +wellness wednesday +v sc +t iller +progre ssed +on ze +love for +gor a +don the +dear th +de ve +cas us +b ili +alla habad +ag ni +t zu +produc tion +com bed +cha un +s ce +mem bran +don ner +bla der +darkhorse comics +c zyk +ayat ol +sacram ent +pere z +pe in +n ite +ken ya +intensi fy +eleg antly +auto pilot +rou ted +iu bb +celi ac +ste infeld +moc cas +ka ther +ex hale +dad dys +sta unch +spring board +pe gg +kirk man +cin nab +aor tic +ruth wignall +moder ately +lau ds +au ral +vagu ely +spread sheet +merri mack +da thletics +we stie +pesh merga +coo pers +tan trum +t cu +ga be +criminal minds +com ix +q ur +fi q +bor g +pue bla +nis mo +cy ano +bak h +ti g +tho tel +il ford +guadal ajara +inclu sivity +atom ic +ven tric +u el +mono gram +man kato +fu q +ðŁĺĤ ðŁ¤£ +stra d +plu mage +op ath +kom en +im al +che tta +bill ing +s vr +hu mmer +d lt +water cress +pil ar +human ist +her r +ha ig +cor ino +swan ky +spo tters +sn h +medi aday +under ground +ton bridge +roblo x +re take +men in +inno cent +ton g +lar amie +be gged +arti sta +uste des +san tho +lac ro +is ations +fox y +bio sphere +temp tations +stan lee +semin al +ini go +ãĥ ĭ + º +spod cast +o shi +lloy ds +gree ce +fore word +david tennant +brain tree +tam i +sam mie +predomin antly +coffee time +clut ches +acap ul +val halla +pro ck +mytho logical +meh met +idio tic +d su +ðŁİ Ĩ +puri fier +oc ca +ma f +hoo t +over joyed +ha ke +behavi ours +ðŁĴķ ⾨ +le ena +gau l +carne gi +u ws +repell ent +maxi me +kar lie +j iro +gla ser +fr ites +earth hour +de k +se ams +l ly +gilli am +davin ci +we sh +swe de +pe tra +p tm +my b +min ig +hon dac +al f +staf ford +plu shie +pal in +metast atic +go reng +flan ery +ro java +ani x +ðŁĶ Į +ap c +activ ates +vic i +tu ally +delici ous +aw ith +air base +stabili zation +solic ited +pic ky +im pin +fo p +af x +the man +the green +pee wee +mis smar +kitchen ware +gil man +scoun dre +kra use +j nj +hend ry +eng r +dar lene +mal don +fam icom +mi er +di su +breakfa sts +اÙĦ ع +ver gne +sunshine coast +h ed +di vest +cri mean +v fc +su le +sam ay +sloven ian +is bn +borac ay +tu bby +sp are +reas suring +mu tt +intram ural +at b +aaa and +âĻ Ľ +o ge +fi u +caer philly +string ent +steel er +n pl +macadam ia +indi go +grow l +gimm ick +españ ol +z ü +th alia +soul mates +more au +cri stin +anxi ously +vod acom +pam pered +lo z +content ment +pharmac o +n sp +mon ika +mika el +mac d +da o +sw ill +sheep dog +yo st +seiz es +off shore +nor walk +lo scab +bo sque +r acked +parri kar +na st +lo tti +bi j +Ð · +tb thursday +question naire +mercen aries +evo ke +di sal +al tru +what ever +str ö +san jose +man tel +in convenient +uaap season +tow no +spec k +al bright +the girl +ha shi +em oun +dow ney +desp ise +amuse um +sm elled +rugby union +over turn +ak ron +son ora +pau ly +mont ague +min ar +mill i +suzu ka +seo hyun +portra ys +ou mi +mo hit +lor dof +ayles bury +appropri ations +en act +conclu sive +chim ic +rasc als +j rotc +ind aba +ma iler +sun dry +subjec tive +stu mps +secre tive +red blacks +qu asi +oni ze +cany ons +nav a +lor ne +for de +cocon ut +us at +shear ing +sai yan +prat ap +modu lation +flann ery +gol ding +di da +ag ong +patho gens +k ye +dow ling +chi ara +sadi q +ol ander +journ alistic +croche ted +assu mes +ai ba +s vet +re visions +ph at +elis sakh +sur passing +person ified +mon ta +in ar +wil fried +âĹ ı +x ler +ta kam +out done +hall ways +gow da +fam ou +chop sticks +antiqu ity +% )... +nak uru +forsa ken +role play +kan ban +fly in +dock ers +con jun +ìĹ ° +swee tener +sin es +re kha +imagin ed +h su +der ailed +mano j +ji mi +acab ello +âĿĹï¸ı âĿĹï¸ı +stre atham +parad ox +ma stop +like aboss +leisu rely +iti ger +faire r +er cy +ðŁĮ ¾ +ma sch +lun e +la sd +historical fiction +bol t +, ( +ta pi +jor d +d na +cant be +alder man +to sses +or deal +nur burgring +kin gh +du os +de ze +clin ches +nap avalley +indispen sable +heath ro +fro ma +dece it +ba ira +ws ers +jack al +ch ula +bal ay +sur bhic +sal at +lolo lolol +had don +fear twd +scar lets +fanta stically +eredi visie +conting ency +seven oaks +me tra +futur ama +ce phal +vi kes +termin ate +sequ els +chipmun k +val ky +stu la +se vents +lo dges +jess i +j bl +gar rett +es es +con ferred +certific ations +anter ior +te ague +ha da +âĸ ¸ +win less +ji b +grit te +ga el +z d +un covering +si mr +roy ce +gwyne th +goo ooo +conquer or +ald rich +vin ny +re generative +propag ation +cri stal +correc tness +wh m +un lv +so w +net ted +k haz +an ise +plough ing +million th +em ac +ec ki +yar ns +impo sible +good byes +ff h +fel ice +ero a +du omo +du ll +barri ster +v ity +st irs +shu gden +se per +re discover +mas sey +al fal +over ture +mole skine +ma ka +li ppers +jay len +gron k +fel dt +puni shing +gary vee +dim ple +briga dier +happy sunday +freel ancers +cajun s +bloss oming +scar let +s ities +ph en +one ida +ken nels +inadvert ently +hu sky +fi fi +bj d +?! ?! +ur chin +to s +ste pp +au tor +ãĥ ¢ +~~~~ ~~~~ +wid nes +she re +just icia +fant as +el aw +Ì Ħ +wi steria +u q +shar kn +dragon flies +al dean +go b +di bs +bo rer +petron as +doub ting +chap ar +shil oh +gabri ele +cor pu +taka shi +selfie day +pe tru +harmon ies +u ck +sty x +k ine +high gate +gang sters +ga pol +desi ree +over comes +mc cas +hersh ey +he ston +duck worth +das gupta +zam ora +har rell +am ble +visu alizing +uc u +su zie +mil dred +lu t +d wi +civil izations +preten tious +obl iter +mil len +girl boss +% : +win field +rever ber +mat ta +in ery +cro ke +ch lan +bep anna +ðŁį Į +ðŁĩºðŁĩ¸ # +squ a +se id +la ve +gh is +be headed +emer son +awk wardly +ar h +à Ł +w mu +tw i +shiva ji +on of +bun kers +tar aji +pal ace +i ang +de crimin +whit lock +wal eed +ax well +viny ls +kan al +exc ise +don ington +carl sberg +under served +solu tion +portra iture +ke urig +incorpor ates +민 íĺ¸ +velo drome +pe tta +len eck +for gott +fe fe +eu c +art book +through put +share pict +pe eing +high bury +dog strust +ðŁij IJ +w wi +m lax +dd in +con de +affordable housing +ðŁij ¿ +t fl +sho aib +ne ues +n de +mor in +flu tter +fi ver +separ able +kenne saw +irish times +ink tober +fantasy art +ed sa +shruti haasan +sc ous +cro pping +abu sh +y c +tem plar +motor cycle +gab ba +ar ic +aldub nation +af r +whispe red +va x +pollu ting +head ers +great awakening +eer ily +bal my +ida e +bon es +a fu +shi res +f ww +uni fy +press sec +consign ment +ma ken +terr ance +swe ar +prag matic +night fall +jor dans +ic han +ta hir +ma dal +g eller +alber thall +social selling +recur rent +pho ton +organi zes +fight back +evacu ations +du ci +chicag opd +bird life +ðŁĺ« ðŁĺ« +tu mul +cam pi +ðŁį Ħ +sin ski +dissol ved +an ch +yon der +rs na +river view +pul sar +pro pon +multil ingual +kab ali +be agle +air canada +sor ghum +fron trun +la val +im mortal +epi phone +de preci +cor dial +bet fair +ìĹij ìĬ¤ +ti us +ombud sman +kar p +gy psy +autom ata +you sse +tam pering +red birds +kar in +trail head +legend ary +fi shed +es l +steph ane +hill sboro +her cu +camil acabello +cam ellia +al ts +adju dic +se kar +scaff old +mel aka +ill iterate +ha sle +fa thom +campeon ato +v logger +spi zza +mccur dy +ine ed +dev ito +team em +sun nies +morning side +elec ting +ãħ ¤ +bad die +aksh mi +sport sp +program matic +ge eta +fla unt +? ). +zu ela +sun ku +spider verse +shreya ghoshal +sh ams +benevol ent +spo int +jee zy +equ ine +dunkin donuts +comp troller +arto fli +tre v +pap e +ohmy god +bou ts +bik elife +mo te +g cs +acce s +y art +ut austin +trans lators +tab or +sho wered +andre as +ak l +women who +ãģªãģĨ ãģ·ãĤĮ +ram zan +lake front +du cky +civil rights +ë l +team sters +str is +shat tering +char lot +bag an +alo a +aaaa aa +mo se +mar ce +gun powder +ge isha +ðŁĺ µ +per mac +elo pe +chri sto +april fools +wolfen stein +there of +tar n +taka hashi +poly gon +lu mix +independ ents +hier arch +fc fans +wc q +law ay +av ro +at one +tru ex +s ro +r ur +mt l +i ft +defi antly +ch itt +r vp +o bel +k mart +ela ine +bed ford +se men +sab out +bathro om +ðŁĴķ @ +khy ber +i will +business woman +tar as +param ilit +mer sal +chor lton +ak ram +w dc +universi dad +ja vier +j ara +gil as +contracep tion +cat aw +c nd +bu co +au ri +ton ight +ma vote +i fi +albat ross +vegan ism +tw ich +lu d +le ases +de ben +worsen ing +qu ia +power less +end ra +ðŁı¾ âĢįâĻĢï¸ı +ا٠ħ +ventu recap +su do +nom ad +indiffe rence +ge tup +explo ren +bay ley +u gg +pad ra +orange county +co bra +tic ked +ti ss +shrun k +k sb +ic m +gett es +aberystwy th +de ars +vas an +tab asco +life mag +fin lay +tam iz +poon am +plat former +mu sco +joy ful +it b +gol ang +ga stro +enrich ing +eli ke +p wc +kelly anne +jama ic +hal lam +bri ar +well ing +wat s +vo icing +t tering +ra vel +pa wards +mu zz +h tm +alig ning +wedding wednesday +dri fts +rolls royce +multic olor +luci o +han son +f gcu +sound garden +pan cetta +oc tober +free masonry +boun tiful +bigbang theory +behe moth +º ï¸ı +sne eze +saat chi +raw at +mobili ze +mo he +fur ther +dy bala +boli var +tan o +ghu lam +femini st +bo f +ben dy +ant in +we is +t sar +fav ours +fab regas +sh ang +pro biotic +ad mit +sol der +mo vable +dra dio +cyclo cross +australi aday +âļ Ĵ +z at +chi m +si reland +s mb +air fare +Ú ¾ +me politics +mal com +kop f +k all +chi pper +ban que +ðŁĻ ī +sol ver +in sinu +avent ador +zero waste +hell cat +universal orl +pu so +pal au +gav a +e intra +d je +arunach al +vi u +the ad +sep tic +santac ruz +mor gen +guil ford +ber tol +b dutt +world poetryday +sc m +ple thora +london traffic +cre ma +t iness +si em +el ated +before hand +b q +aw ada +nag arjuna +ev y +ðŁĮ ı +lu cha +call in +se aco +retri eval +kun al +car on +reserv a +negoti able +lang a +do fe +cab al +ani k +ðŁļ § +ðŁĺĬ ðŁĺĺ +sub station +no x +long ford +gad dafi +dic a +zi oni +mitochondri al +it on +in sp +be tray +va in +tr ts +pen di +oppre ssive +long wood +limous ine +iz umi +green light +go ggle +pl t +to kio +long beach +gene ss +visual art +s ball +pacific a +kir by +itu dinal +i den +foo ting +en sues +dit ching +sunku writer +sig nor +m news +l tr +b sn +whi de +ver des +taylor nation +sen n +n spoli +match box +mac gy +chro mato +íķij íģ¬ +q in +mo ca +la gging +b ander +sm r +red woods +de gas +ti ered +ma ko +kno tts +ble sses +bella gio +barunsob ti +ad t +sc rape +p mp +no y +mu har +whit stable +man sell +dri bble +n di +mun cie +ho yt +fiber glass +alo u +thing si +th au +sanay airani +s forever +rale igh +mccol lum +but tered +bang in +shab azz +cho ir +pension er +n ando +mu sa +kra u +intoler ant +h mcs +ka jol +ch ale +block ers +bi el +v su +sh ona +gh ol +decor ator +chi ppy +~ # +wel lies +sub hash +ou ld +oc r +o ann +ken wood +c bus +ton ing +ruck us +ro ca +pi b +op aque +for bid +britishar my +sens ation +lo bla +inde b +conne cts +ðŁİ ® +next level +jaz zy +e fe +wh l +im mortality +gym life +dopam ine +arter ies +ts g +steeple chase +resurrec ted +ka izer +hotty toddy +dg b +dal ry +attach ments +wind mills +ul cer +share ef +oy ce +n tc +loan ed +ani ka +ag r +west coast +sha ver +bolog nese +su ez +photo card +mi splac +limb augh +barbic ancentre +accor dance +ze bras +thalapathy vijay +men style +final fantasy +app ing +angel ic +raf fa +hitch in +he ct +bre da +blind spot +pre eti +mentalhealth day +dae hyun +bore rs +ãģ Ł +talk radio +Ú º +itiger shroff +end on +ra pp +ph onics +tar a +rand paul +face ts +art ful +ar anda +âĺ ĺ +stor mb +sau cony +member ships +al ur +treach erous +tis the +stan sted +re work +pre emp +hol la +gi anna +beauty andthe +âļ«ï¸ı âļªï¸ı +surbhic hand +health forall +anatom ical +ve c +trans national +or la +en n +ak ali +to tti +car mine +sub mits +projec tors +gu ten +cruz crew +sc ele +ken yan +conce de +! ðŁĶ¥ +saturday thoughts +ru k +ec er +win ing +pic mix +li mon +lau gha +iti e +fa un +bru cele +ठŁ +vocal oid +t news +miser ably +em brace +don kiss +de position +clever ly +thic ke +sky dive +play bill +her t +door i +de letes +bo asting +analy tica +sh su +pro sthe +f bc +du arte +c wa +bad as +in visible +geo logist +ec t +cas settes +ba ir +revolu tions +off end +nutriti onist +line men +dele vingne +si da +man chu +ji an +bli mey +ar gan +wildlife mag +snow drops +pre cursor +o co +neapol itan +it su +birth stone +amate urs +pin ks +ley ton +gram my +giac omo +sei u +second hand +out cast +lon er +b ites +ðŁĺŃ âĿ¤ +è Ĭ +un cg +slimming world +si o +shin ji +ou ch +kan te +ðŁİī âĿ¤ï¸ı +pe k +hu zzah +de kar +belt way +ab und +ðŁ¤Ļ ðŁı¼ +cri m +caregi ving +ara jan +q pr +fon seca +daw son +bu li +alter cation +we the +rawal pindi +messi er +je mima +den ounce +debu ssy +chiroprac tor +antiqu ities +wa hl +un packed +tri athlete +cl iche +super card +re tour +re petition +re actors +lead ers +hol lie +fa z +bad o +ta os +raj avi +multi verse +aj al +ìĹ ¬ +uof l +o ha +ka c +pd l +baro ssa +ari k +ðŁĺ¢ðŁĺ¢ ðŁĺ¢ +met calf +ciu dad +chi yaan +ash lyn +am ity +way nes +pa wn +ox nard +ìĬ¤ íĬ¸ë +es b +dod son +âĺºï¸ı âĿ¤ï¸ı +is che +ev ry +bre m +to en +pronoun s +graff iti +flat bush +che p +pig let +it ye +empha size +c bee +á Ī +tele graph +sing am +or dn +mg m +kno tted +ale igh +ranbir kapoor +mar ko +lic orice +ax ia +arti e +ðŁij ¥ +pu m +plun ging +ore illy +gastro enter +cis o +bombar d +bas qui +speci alize +nr m +nak hon +fre sco +am ity +aero bic +sin ow +re produc +pri mor +straight en +bound less +bone less +stampe ders +re tweet +quint ana +luci en +kri eger +jam z +dou che +audi ophile +abol ition +to ho +tab a +se gal +sch ö +s gov +m td +len ox +cru tch +ðŁij ł +platy pus +ing old +re think +kh attar +dri ft +apaaja iklan +ancho vy +tri este +lu mp +quen ch +on it +gill ingham +gal van +com miss +mi sty +col ton +bob cat +white horse +ric ha +pu ke +perpetr ators +no b +kt la +ira s +g wa +d pd +co tti +teas er +olym pu +franç ois +sh amed +oul ding +offici ald +inaugur ate +der went +con formity +sil vers +is ka +gat ineau +ce ce +any thing +splend our +pin tura +on am +dur bin +d gs +vali dated +se go +par ra +last fm +ka in +h cm +ak ar +ab ram +learn to +ar ke +_ ; +som mer +she amus +kam ara +flir ty +ðŁIJ ¬ +on ice +hu ma +el bows +conce al +col leg +acqu itted +tax ed +mc cau +health tips +defend ant +chee tahs +business men +re h +mat y +andro s +the musical +ob ar +ge mm +dalmati an +wob ble +spho tography +prairi es +ma dge +kail ash +fun der +datab reach +tender ness +sper ry +mc cu +debu gging +ko pp +jame is +diss atis +n cl +logan o +h mas +e ren +ac os +tumb le +natl parkservice +cross fire +bat mobile +ag iri +war ne +vi my +supervis ing +ren ko +pool ing +ma wr +dock yard +nar o +multi plex +exc l +conten tious +ab bi +好ãģį ãģª +twitter less +obli vious +kore ans +fre ne +tul si +hi ma +pasqu ale +oc clu +o key +go dof +dan ilo +ar de +ðŁ¤ § +nn nn +fur y +fi ka +ber sama +wise man +tru c +sh em +q atari +ox fam +as lan +soci ety +ri dges +mur ky +frederick sburg +f k +char ly +buck nell +atta ined +than am +mez cal +hef fron +w jc +sty lists +daffo dil +wend ys +r tz +famili es +crack le +tun nel +nit z +mur ph +ad avis +esp rit +win nin +ki pling +jer o +young blood +ru z +paranor mal +pantom ime +mat lock +inst am +gall ardo +f wy +ca en +vel le +kut i +fur thest +com ings +av b +shu dder +north am +: ^) +ðŁĴ § +swe dish +prettylittle liars +braz os +r ations +pizz a +go gators +fire wood +esto y +ad hi +lec lerc +ice bucketchallenge +hippo drome +bloodh ound +ad gpi +suc cin +sub committee +moris sette +the view +show ering +j awa +a ic +tro m +surbhichand na +ro din +restra ined +ram bl +mo bs +cubic le +then y +park our +he ute +ep c +bit coins +auc tioning +mal a +gi gem +concentr ating +er nie +bun e +tran sports +ki shore +har aju +ðŁĵ © +sweat shirts +pol yu +need ham +nc te +kh loe +fire safety +er ian +dri fter +deta chable +woof wednesday +tric ol +shu ps +dise ase +u min +story book +start ling +sg f +ma kan +ðŁIJ Ķ +agend as +hi it +dispat ched +synchron ized +shu man +radic al +pu tt +pre cure +go fficial +de code +vi ans +vi ability +v sp +tam ales +pra bhu +snor kel +eas th +bl under +: (( +wai ved +ide al +e ury +americ a +t dy +shock ingly +fran z +eric s +ce f +bal ear +ren zo +ko enig +c bbc +biome trics +suffe rers +su ch +smo kies +mur u +k hawa +i im +gha zi +íĶ Ħë +white tail +un chained +thenotori ou +sh ino +ken obi +cour gette +clint ons +al ala +sexi er +never stop +ne gros +ne ca +x cx +song book +ren ding +cal ms +amar u +ag tech +ãģ Ń +qui dd +new years +mar gher +eye ball +ati er +vivekan anda +somer set +rin ce +mu kh +ho h +col er +bu kas +⼠µï¸ı +tu cking +pi ggy +iban ez +ho skins +decep tive +click bait +bu le +world view +woo ster +wo td +stin king +dam i +pau lina +fel on +cross body +wb tv +sub han +lon zo +flat iron +burn side +win throp +tallade ga +sp angled +sf p +ro wan +real ises +wash burn +ÙĬ ÙĨ +ston ec +emp tied +ci ren +cha ise +am bu +. !!! +se tbacks +sad dam +con naught +av enge +af fluent +u ob +that ch +swin ton +sat ya +es z +equ ity +re invented +kas per +c fa +aesthe tically +mi ku +mar cy +fin anced +look up +ecoun ty +ðŁĺ º +t dih +ro pe +me ch +dy ing +book seller +aa sh +vel oci +o vi +im m +feat ured +nu g +fun g +ell sworth +sett ler +say o +mu zzle +su omi +ragn ar +mentalhealth awarenessweek +maastric ht +il in +gl und +ak ar +intellectu als +flor al +brack en +ti ps +sub ver +se duce +scu deri +nev ad +je eps +jaw an +scar red +med school +ec p +catap ult +additi ve +sm itten +q d +lock ers +like agirl +keral afloo +bub bling +ari ze +ðŁİīðŁİ Ĥ +ìŀ ¬ +ted talks +rhschel sea +pu y +ok ra +logo design +hen g +hammer head +dri bbles +gan ja +for ds +cou scous +ar gen +ë³ ´ +van ish +ne tapp +my love +anch oring +ç Ļ + ¨ +por ous +over seeing +musi k +ma gen +dar nell +r ha +per o +land a +jurassic park +fre o +bron ze +ãĥĥ ãĥ +ðŁĶ ¸ +ðŁĴIJ ðŁĴIJ +wa ch +tri gger +eng inak +d bl +ðŁĺĩ ðŁĺĩ +speake asy +solid works +sheh baz +pu sher +p ty +fat loss +discre te +di onne +ch iller +applau ds +u mp +ra staf +neg atives +macar on +islamic state +cap tion +anti aging +pember ton +long ong +issu s +res ounding +offen ses +new balance +n ley +mont auk +mc ga +dispos ition +purpo sely +ir anians +ðŁİ » +fu lani +corrug ated +ðŁĩ³ðŁĩ ¿ +ston ia +par snip +jam ison +ge is +ðŁĶ Ħ +re claiming +pleas ant +on boarding +edou ard +a ah +swee per +nu nez +mu dd +holo lens +chee z +brigh tened +âĿ ĵ +wo wed +sch ko +nis d +co ffe +ba hama +auck land +super mari +oun cing +op ting +mc clu +âĢ¦ # +sant as +sli brary +revit alize +qu ai +men acing +kkkkkkkk kkkkkkkk +der ulo +scre ech +ko enig +crowd fire +bravo tv +ay ee +ar kar +si mi +me era +jiha dists +je we +bu ss +ari ane +- , +ਠ¾ +n su +ðŁĶµ ðŁĶ´ +fi bre +ar ched +à ¥ +t so +re my +light foot +far han +embarra ss +bro derick +breath ofthewild +as ino +superst ition +new ry +mer ck +kip lier +ab ag +van g +pic to +li f +bag pipes +at ru +royal alberthall +movie review +lil ley +ju t +bang bang +. ). +grass lands +flower report +chat sworth +aam ir +syn tax +pro bing +nomanss ky +lau der +we tting +so ta +rappler dotcom +photovolta ic +pharmac ology +luc ca +le gging +gumb all +full back +dece ive +sop ranos +s bar +ru pert +com bi +clar ks +billi es +alle gro +m ce +dam an +chicago bears +cas as +vap elife +mal in +byo b +âĹ ¾ +rac er +mv p +memor ize +jiha dist +ing life +com ber +ar tex +applic ant +je a +in former +ho xton +hen ning +h ls +ðŁĩ© ðŁĩª +p q +in london +ilay athal +âľ į +sciss or +sch amp +li able +stra ining +pu ra +mon kees +let ch +kom pany +by design +mo dul +har dened +brecken ridge +wol longong +tri er +man ate +lyn ching +con cur +c sf +wood ard +ol ab +l st +jamie oliver +insur gent +wre cker +or mond +kim ball +sn ic +s ere +mal ar +gar ages +fel a +fa de +pastr ami +ic rc +hor atio +cle aver +ab be +wwer ollins +privati sation +nature guide +hol me +h eng +esk imo +may noo +lever kusen +ax l +sk sk +n our +fi do +famil le +dis i +br inger +age less +x g +pi an +path ak +ab domen +tri mble +in ns +idw publishing +focu ssed +ei ght +mandel aday +fa ve +da ire +bul ance +u mmmm +pe res +tomo da +pp ed +may all +ler ner +elder flower +bar nar +transp lan +mausole um +fier ce +alleg ing +neti zens +ky ler +il de +gam b +e as +lit es +go er +bur u +alice in +ðŁ¤· ðŁı»âĢįâĻĤï¸ı +ti pple +rupauls dragrace +pee ks +inter twin +æ ķ +y j +shan ia +techno logy +nba on +mul tan +motor head +lu kes +ken zo +mccre ery +er te +dra s +blo kes +ber nal +apple by +south carolina +new borns +tw ir +spartan burg +o cala +l ü +dwy ane +bra va +ace h +à¸ Ĺ +sv s +sula wesi +stoke city +shar ks +fo a +anti depress +ðŁĩ¬ðŁĩ Ń +sidel ined +shu l +seren geti +ll lll +kab ir +bout a +bi zzle +bam ba +que ttes +nb cla +mj hl +mg mavote +mac chi +cag r +ale ah +ðŁĺ Ĺ +whi plash +tim i +pal mas +domingu ez + · +il on +hil arity +ru a +organi st +mit ts +gas ket +ðŁıĢ ðŁıĢ +ten sor +steep le +smy rna +rand o +r ma +pl enti +fo go +aph one +ðŁijĩ ðŁı¾ +ðŁį ı +th ad +re sor +petri fied +di xi +ti gan +jalap eno +deser ve +sav age +mad sen +gre mlin +for women +depend able +conve ction +bo sc +yel yah +watch mixer +ìĺ ¤ +z ell +tur ku +soul ja +she et +sel i +ma kas +k league +an go +ak al +........ ...... +te tsu +i heart +her bst +cyber attacks +sum ter +rijks museum +raj ya +ar ba +rock ingham +mus sel +micro scopic +ke babs +cand ice +get fit +adam ant +we ing +sa shab +h gv +emp tiness +cur ation +brit natureguide +um n +ra fi +an er +viscer al +up trend +um pires +ts now +out last +le sh +ih saa +shal lots +sco pe +dan za +c vc +bac ardi +air brush +ae gis +ðŁ¥ Ĭ +tru sh +scha efer +resist bot +fior ina +bran ch +whit efish +ru l +con spic +ar ig +twe aking +tub man +ste tson +robber ies +iso l +em m +condem nation +cast ing +aud its +vish al +vand alized +oc i +gi m +work place +van buuren +nig ella +mis guided +cas cad +after ward +:- * +sub tly +car rick +ple ading +original art +omnic hannel +nancy pelosi +great lakes +glimp ses +ent ino +down right +arter ial +ðŁIJ ķ +maxi mizing +er acing +cy te +chur ros +stur gis +microbio ta +mass ac +kun al +ku b +h ny +blin ding +articul ated +an es +piers morgan +ker alab +ho cken +coles law +gas sed +d md +ðŁıģ ðŁıģ +nor bert +mi i +long ines +go de +del ray +carval ho +bou quets +bat ty +bake well +st oop +sare es +pug life +kau ffman +g ds +free speech +cul pr +basqui at +pan dian +g ws +do glover +den ces +beach y +wan ing +press freedom +home made +con stric +bhu mi +bb all +t iling +popular ly +accol ade +tar ra +sta ve +kardashi ans +jac lyn +ic ed +endange red +art finder +ðŁİ ¹ +vanity fair +tr ill +psychop ath +multi functional +lou p +jag ged +gr ama +sanctu ary +her schel +et ch +capac ities +z ora +slu ms +jad ine +bag ga +ani m +sp ress +hand some +cape town +by ers +w sc +sh reds +miyaz aki +iti st +coll ard +è ² +sal ina +pac ino +nune aton +jan ella +grass ley +ৠį +p eli +don line +comfort food +c ÃŃa +so ba +fl ys +spay ed +med ve +hol man +ãĤ Į +stef anie +nais mith +home girl +g bb +bul k +au ce +afternoon tea +ac ular +su iting +spot less +sky lar +shop kins +j ona +clo cking +car cass +bo gey +be ig +bac chus +ari es +ad k +ðŁĺ¬ ðŁĺ¬ +upro o +tri a +hom ing +han n +el vira +cdc gov +br ind +al en +wn c +with ers +ro ast +os mond +ke ele +e bs +marke ted +imperfec tions +en cin +e sses +c zar +worth iness +watch man +me ena +man ali +kat ow +historyof painting +edit or +° . +rosen stein +itsen rique +dal hou +begg ars +sym metrical +surg ical +star a +st ent +so excited +sa ac +robert pattinson +pe dd +ker o +ç Ľ +thu mp +re tribu +powered by +ober oi +hybri d +grid lock +cd m +av al +ãģ § +thenotoriou smma +sub conscious +he id +go bruins +cy ani +bo den +uc sd +sar o +el c +de ley +bol den +spie gel +re made +n anda +jodi arias +h any +too o +salis bury +op ing +cab all +an gr +... * +to ggle +shorth air +sas sy +san am +fl on +apologe tics +ac cel +shirt day +ac me +ic t +hal es +danc in +co exist +ðŁļ ¢ +è µ +yelyah williams +straw berry +fc n +ce t +sanc tion +r fa +po tt +iz om +id t +; ;; +vol a +mess engers +cher i +sfor all +rhine stones +r ann +us k +sole mn +sen tai +retro fit +mait land +ac tus +ig loo +desc ends +u day +ic c +hu ck +endometri osis +elec ts +crab tree +rakul preet +jur ors +holy rood +feder ally +di az +de mil +wash ers +ver tically +untouch able +mal o +curtiss mith +cali bur +book keeping +ym o +x men +per mian +national theatre +n assi +lein ster +c ld +aw az +apolog ises +al ysis +son u +mi dge +inte ch +gy na +come th +band stand +viol ence +survey ors +n ms +es d +dr ing +clar ke +beat on +u bud +ons laught +ne vers +m sk +ee vee +con qui +bump in +bel ted +ac ris +c our +blo em +bien ven +v mi +da de +as ic +su vs +speci fy +gaz za +fe dex +voor hees +shr ines +# âĥ£ +ito hs +del co +cruci fi +bo h +ath os +âłĢâłĢâłĢâłĢâłĢâłĢâłĢâłĢ âłĢ +tri pathi +think tank +t ta +satur ation +mmm sie +bear cat +rick and +pa z +kand ar +jo ss +al ang +sab ina +ree ce +m kh +jin der +est elle +ell on +pellegr ino +o skar +im man +n ona +k go +engag ements +dest abili +bb g +y x +shiva ay +see saw +mo tu +ger hard +al armed +houston rockets +fin ally +dra vid +corpor ates +c ô +c mo +blood shed +ti h +strong man +sol an +sn ick +r ara +pau li +n ge +horse men +ch ism +? .... +.. ' +li via +issan ts +bc g +bar one +wat ters +val eria +optome tri +jess y +j hl +com be +be toor +ðŁ¤ ¢ +ut in +iggy azalea +grou ping +commun e +columb ine +af for +sa is +panch ayat +h ro +floyd mayweather +esplan ade +z vere +sony music +no a +i vey +d onal +cher son +c ack +betoor ourke +susque hanna +kak kar +don es +derail ment +compul sive +cardi b +ca e +tim ings +ap l +after hours +ac ting +un fore +transforming india +suppre ssed +sidd har +r sm +mah rez +in capable +green wich +misplac ed +love this +insi der +biomar kers +panini america +multiplic ation +ice breaker +discre et +chec kin +人 ãģ¨ç¹ĭãģĮãĤĬãģŁãģĦ +vel cro +pre scriptions +hetero gene +dru dge +ìĬ Ī +Ø§Ø ¯ +van swar +tu pper +spar ade +m callister +e ko +ve ep +mar gi +ker k +kar a +dic ho +cos grove +val des +pu mas +off ending +k andy +hhhh hhhh +h pd +complex ities +car te +buf fo +k hun +ta char +in ky +bat es +at ms +syd ne +hri thi +der mal +ðŁĴ» : +ea sel +diss oci +bikin i +n sui +mon tes +mol loy +mo par +h di +dom a +ari ous +alphon se +âļĵ ï¸ı +wer der +uni x +seg mentation +micha l +lam beau +what the +thread ed +sa am +pfei ffer +fu sa +fr ack +aur us +te dious +nag el +ken an +island life +ge sh +cate red +bilt more +kam i +bul le +teamem mmmsie +t tu +sl b +newal bum +ma zing +gra phi +en vy +con g +we sson +sch il +gur ru +be de +aqu amarine +kand insky +emor y +den iz +ri sa +pul p +o cho +neuro surgery +le sions +h ons +big cat +sak ti +psycho sis +nsi tharaman +sw ard +le gu +fi ennes +se att +marketing tips +man groves +loop er +dh ya +quar tered +pri este +pres scon +ll amas +com elec +ri sd +r ine +pp r +dete cts +vival di +valle tta +fle sh +alfre sco +testim onies +quil ts +lat ency +k els +grun t +crimin ally +h tg +apo or +p ga +or m +ol ly +modi ji +hin ckley +na ve +n ong +heffron drive +gulf stream +gar rick +enti a +man mohan +iphone ography +flo ated +co en +c ally +armb and +te ton +tar te +ns wr +max ed +in ward +hydra ul +armin vanbuuren +hob son +creep in +re ins +kentucky derby +dream s +blou in +armaan malik +ab ana +.... ." +ten acity +ðŁı¼ âĢįâĻĢï¸ı +tam ing +oper atives +lec turers +cam us +áµ Ĵ +therap y +se dge +qu é +l ene +judi th +ac claim +on as +l ill +ben nett +sh atta +go dre +fle ury +e ath +posthum ously +pla ined +n ace +mor bid +mas ood +bac olo +mic ron +intercep tor +g acha +talk sport +requis ite +intru sion +dom es +brea the +affili ate +nyc marathon +house warming +blur b +si ren +ss an +mill on +gra inger +col by +campaig ned +kir sty +illu sion +f omo +c illa +armad illo +a better +t mc +soo young +sec tarian +rede mp +om p +chapp elle +and ar +ðŁIJ¶ ðŁIJ¶ +por tia +la pd +imit ating +do ers +cam b +bas alt +w sp +w los +tal es +lov atics +fat ah +sle eved +rand i +de foe +Ñ Ī +vo k +spraw ling +smo thered +kin ab +isu ppor +i wo +diff ering +al ine +scour ge +restra ining +kh j +joero gan +ed ina +chiyaan vikram +web ster +ty rell +take i +marting arri +j edward +e ke +dil wale +sur face +pu get +le sc +green tea +di xon +mi si +huar ache +cher y +aqu il +altern ating +my thic +lans downe +fil an +z ey +s att +o as +kerri gan +ty r +startrek discovery +ds man +bre ached +banc roft +ìĦ ¸ +lobb yists +lil ian +c ve +bull ard +bring the +st of +plane spotting +mit o +wak anda +mo wers +le la +had field +bouti que +ðŁĴ ¼ +s bc +lone star +disciple ship +qu akers +ecclesi ast +dead lift +c po +botan icals +ac al +stein way +je u +h cl +cru x +ë · +sh ani +palm beach +men non +du da +cho t +wn yc +cou pon +ca an +am f +ab users +ðŁĽ ij +ðŁĹ ³ +yu catan +w ird +tele medicine +ster oid +mayorof london +hy ay +sni pers +n andi +mo x +ga uri +xen ophobia +the arts +slo p +s val +llan elli +in consistent +do ki +demo ing +char lat +carl sen +bel lies +ìĸ ´ +ve ts +sen atorial +krit is +grun dy +golden knights +di vo +arch er +resi sted +connoisse ur +celebr ating +yousse f +par inee +de blasio +darren criss +ronal dinho +mt p +match play +entit lement +ðŁİ Ń +ภĭ +ple asee +men di +ev as +y una +that kevinsmith +red shirt +lin c +kung fu +epidemi ology +du z +sto ker +say er +mad huri +if ttt +gye om +fau lt +chan ukah +used cars +unimel b +la ha +eco logist +conserv atism +bar ro +art station +star citizen +spr outed +sho ved +shark tank +pro filed +jam i +hu xley +grote sque +be cc +ìł ľ +ภ¹ +ww t +work fromhome +ud ine +mar lowe +her bal +fur long +deb by +bou tta +ve dder +pri miti +mb t +e ia +dill on +akrish nan +wi ener +tun bridge +thy self +pav illion +om ggg +kevin hart +aw ry +tv news +si one +qu ds +n ita +loop hole +te chie +sab les +ber ing +worldocean sday +se g +pat tie +ne pale +indi o +bi anc +be ingh +air line +su ne +sj w +m wah +h ca +gre noble +gn c +council or +call a +weird ness +spo ken +sh ined +rotar act +om in +city life +vanswar pedtour +t ine +sp b +sof test +ram med +mentalhealth matters +gar ry +ex iled +adap table +smir noff +sedg wick +glori ously +bit strips +af an +tam er +q adi +origin ality +john kerry +es se +soire e +jo ggers +c gn +boo sie +se thro +le cht +in al +de generes +bog or +algori th +abo lished +scram bling +ici ones +hed ger +har ing +gen omes +bol locks +ram ble +bepanna ah +ðŁ¤Ķ ðŁ¤Ķ +the sp +t so +hof stra +stor ied +ol lege +jan os +gold wyn +donny pangilinan +ëĭ Ī +âĻ¡ âĻ¥ +yo w +sab ado +defen ces +ap ts +inter personal +el ynn +b ff +." ~ +un discovered +red deer +py ro +muhamma dali +lam on +kevin harvick +itu res +mol ds +just sarahg +irr itation +fre u +fort itude +du ality +archa ic +æ ¥ +sc loud +narcis sist +mu tiny +malign ant +du cho +culpr its +cross walk +berger on +back lit +ye sssss +tro l +sil ks +ran cher +nil sson +store front +sco ffee +pur o +fla herty +fa j +compen dium +car ds +si mu +mo sk +joe jonas +hand ker +y h +screen saver +ravi shing +hu mm +del mar +cro mer +cape cod +í Į +transi ent +taey ong +segreg ated +man ji +ki dd +jam il +cze cho +au ds +ãĥ ¯ +ma si +athle te +tu fted +tobac co +the l +bird land +transm it +thra sher +suit ably +seawol ves +ma so +lo vic +ing ford +communic ator +be gon +pr s +co ker +at ticus +tel co +stu bble +mp act +je anne +home schooling +est rogen +dt by +de hydration +com et +aper iti +work wear +tc p +pan t +men endez +air pods +tick led +me ws +may bach +li ar +inc iting +hal cy +fo m +fjor ds +wd su +saf ridi +produc er +out there +im ala +er b +butter scotch +ble tch +anc up +ãĤ ª +tan doori +shi d +p ds +ny x +insp ort +i fb +hydro gen +battle grounds +work s +meij er +mary ville +kal yan +cas sava +bo zeman +mat us +in human +ec ur +ðŁĮ µ +schoo lof +hispan ics +ga j +es qui +bt g +ac ing +pr amo +maer sk +ga iman +biza v +bir ders +whoo ping +vit ro +s ö +re telling +pal o +mar kiplier +hipp ies +cre ator +brom wich +ste ely +oo o +louth chat +nin ers +mil der +simon e +pl m +ho tt +devon shire +bon ny +victorias secret +the city +sch wei +pra bha +lil tunechi +inter galactic +cit ations +car thy +bi ow +vil lec +ut d +t st +shay ne +shakh tar +reson ates +per col +kat ana +asi ap +ak ki +shel ley +ke ston +jade ja +hutch ison +disp ers +bro mo +rai ding +o dy +n news +martingarri x +lu g +g lish +ver so +tan tal +om ag +o tak +free ing +yam in +un ser +multi family +haha ha +h sm +fi go +f ma +em bre +ab normal +nu ig +mall ya +d pa +bu i +ar no +amp shire +af fin +ab ook +pel ican +mee ks +heathro wairport +bhai jaan +ภĽ +st are +sar o +mathe son +mar ts +eucli d +w sc +seven ties +se cy +s not +motivational monday +mar que +karl sson +imit ate +if bb +houseof cards +ba sta +ðŁĩ²ðŁĩ ¾ +oc cul +na vel +manag h +ic her +ent icing +tw ente +trac ts +room ies +little big +el dor +humidi fier +depe che +su pp +si b +se ong +safridi official +nebra sk +make your +hiro shi +el khart +edi ble +du t +barrow man +balo ch +ude my +rwand an +me ts +footb alls +conun drum +ti u +p low +news stands +constell ations +ch n +lu han +khu shi +hope fuls +confe sses +ati ya +w ms +v ite +syn dro +shameless ly +khloe kardashian +hi sp +haban ero +descend ant +con scienti +black caps +ban dof +wad sworth +museu mo +ban king +anu rag +va ill +tele health +\ \ +w gc +v aqu +up cycling +k sl +aw ol +up cycle +nick names +diver se +centi pede +br indu +bur ying +bi gger +bedro ck +re solving +rang a +or icon +nikol ai +god in +excali bur +cur tin +chir anje +ab sa +wh irl +monday blogs +ll ang +bj ö +trip led +re imagining +lo ko +govern ment +craft smen +oste opor +lo bo +la vigne +grand view +v rin +v anna +s net +nomin al +ju ri +es m +cra dio +pr ingle +key chains +imagined ragons +ig ned +hill man +e ases +catch ment +ðŁĮ ª +transc end +qu ita +no sql +hav re +ðŁIJ £ +âľ ¿ +rani eri +por ta +yun nan +y ac +tam ale +ir t +gar gan +dis agreement +cy st +busine ssc +sten c +sm f +shino da +qu adri +off site +liter ate +chap ter +boun cer +asym metric +wi den +sch n +j han +ak wa +rheu mato +le de +in patient +he ide +chec ker +inf light +im pover +ha res +ayush man +ðŁı « +uter us +fly catcher +du ques +ka st +jahan gir +con vo +skin da +san sa +qu im +presu med +p ils +nbat v +mainst age +bri xham +s game +rho dod +qu ake +per ci +never hillary +love birds +loo kie +la vi +wes tham +pomer anian +ner o +montic ello +const itutes +warner bro +synth wave +nr w +fand ango +con d +grin dr +dé cor +cu h +come dies +bir kin +bap uji +smu dge +scru ffy +pan cakeday +ove se +ni d +li eve +laz iness +imple ments +ad ri +ðŁį ŀ +vi sts +ve u +risk ed +pro football +pless is +meso potam +ma ret +lu pa +koto ko +k ura +clin ic +am ends +state fb +goo ood +<< << +âĢ¢ Ì +th icc +mc do +hd fc +configu red +ck in +back ups +the mo +pol ska +insi sting +et su +sis coming +kin ect +conce iv +ar ry +go heels +vac ances +to sca +te sco +symboli zes +pnpp ro +palla vi +os born +ori ole +k sen +cro issants ++ $ +the man +li gn +jump in +hoo ligan +dictat ors +anal og +wai kato +ha vi +gis elle +fin ches +c di +ar at +tra shed +the academy +steel book +ove rest +home ward +gen ev +david son +ti bur +loo ker +brindu sab +tra shy +sl v +illustr ation +bread th +ba f +ri del +expre ssionist +co pic +clu s +ag chat +wiscon sin +sn ick +sh s +ricket ts +mlb network +han sel +dari en +chi val +wh u +sal as +phi pps +cor responding +chicago bulls +blat antly +bil a +bay watch +" :" +ìĿ ĺ +su mb +rous seau +p we +ed d +dam ning +benaz ir +bb mastop +unlea shing +hour glass +bur nie +buck les +ticho pra +tee thing +per ri +pen der +inf atu +he il +alum ni +ॠĪ +wh im +ver ge +newly weds +an ach +wo h +sj su +mi an +lom bok +j adi +ail ments +ft m +cro quet +blu ff +fa iz +chromo some +qu t +iti onist +ma dera +breastcancer awareness +b so +tra pper +tole do +o ys +fe ats +bt p +beli ve +a sey +ser t +bor i +æ Ń +tr ition +nun n +nbc thevoice +form ers +cav all +ðŁį µ +l ingham +hang zhou +we stand +inju res +gr rr +fer managh +cygn us +amster dam +t ns +spar row +ro logy +ray ner +pe onies +lu ton +huff ington +ha si +pri es +ev ol +ds l +. âģ£ +wins let +parinee tichopra +nur series +es ri +de mor +con texts +con rad +ðŁı» âĢįâĻĤï¸ı +sp rays +pres suri +don or +... ðŁĺĤ +gru b +der asach +ðŁĻ ĩ +zvere v +thi el +slo e +om w +kha di +ic hel +pun ters +f gs +commemor ated +brick ell +box eo +school house +on enote +lu men +l ye +ar ah +alex ei +ab ingdon +schol ast +magdal ene +for a +foot bridge +embo died +ble e +sm w +ren ton +mad havan +estim ating +son of +inthe world +ce ta +asau da +ঠ¿ +vue js +shar ad +sh unt +o val +local ity +first ly +de jav +whe elie +no zzle +no bu +han es +cu ban +aj ram +s radio +reen actment +play grounds +ordn ance +mu ggy +hor i +col ouri +b aka +vi ber +sle dge +ro si +off aly +im u +ende aring +concentr ations +ari th +ver me +south sea +sha ws +second life +re ac +mob i +la ff +exxon mobil +domestic violence +condol ence +cd g +bi i +ab cd +venturecap ital +thra shing +fox sports +ferra gamo +dang al +acapul co +ser rat +uphol stered +u gu +ro bs +play station +forwar ding +beautiful pakistan +x vg +tit us +su se +in sure +havas u +flam mable +ðŁĴĽðŁĴļ ðŁĴĻðŁĴľ +wh ine +tuc son +tame side +sc f +is so +afl cio +cal tech +theat lantic +taylor made +q ot +pp i +hy alur +hect are +de mir +su kho +scrap booking +sc ic +s sport +harmon izers +fol lies +che tti +med ellin +ken osha +hal ts +fuji film +b hd +epic enter +civil ity +te ac +rajam ouli +ho zier +summon ing +music news +laugh lin +friday thoughts +derasach asauda +cauca sian +z ha +total ing +sa rena +ratt lers +go se +by ul +b mc +ti st +seri ousness +kid dies +gre mlins +con testing +ë łĪ +z g +snapp y +pud sey +hor ton +ho ses +der ozan +sar ge +plastic ity +intercep ted +ðŁij ¬ +tre c +more lli +her ron +dj t +ðŁĴķðŁĴķ ðŁĴķðŁĴķ +year ning +j hu +hyacin th +che stra +ya w +sequ ential +ol ite +moo red +t assie +sop h +is brill +insec tic +fou ls +ab ook +sli ver +cripp led +transl ational +shock ers +she er +seman tic +mumbai police +accu ser +? - +the official +sam ara +jac into +fal ken +expo sures +car repair +amand a +ðŁļ Ķ +twee tup +til ted +ro phy +ske et +pamp anga +it take +eto bic +dess in +aa shi +us ga +paris attacks +ate ch +am ici +scrob bler +nintendo america +mol son +mag ne +haw es +ex pres +âļ ĸï¸ı +we got +scram bler +pra m +fic tional +elli eg +ðŁ§ ł +sw tor +quir k +karti k +s rock +ni er +land on +he dron +ber yl +^__ ^ +pin back +dar ling +c mon +and sons +al ca +severy thing +ram an +ra dy +permac ulture +be vin +see australia +man ga +kau shal +half term +fet ching +divyan ka +bureau cracy +al ena +stin i +sho vel +rho bh +raz ak +co schools +peril ofafrica +o choa +gi mp +facilit ators +blueli vesmatter +ah ly +adul ter +the art +revol ves +photogra phie +be happy +ahu e +s are +fc l +counsell or +bio gas +avi base +wh ys +v ad +santor um +les sen +don k +cover girl +bacolo d +ach en +-__ - +zir conia +roo p +brack nell +à± ģ +mis spelled +imperson ation +hand soff +( @_ +rou en +cl er +stabili ze +st t +jun aid +defibrill ator +she skinda +rox y +ra jar +pr ingles +over alls +jin ks +mchu gh +fra u +abig ail +ab adi +ro sco +re ims +ho shi +quig ley +pu rim +police uk +cu pping +aro v +a state +xero x +nz l +noctur ne +mortal kombat +clou dexpo +ain tree +hur lers +e ffing +bi athlon +al os +kin ky +hut cherson +bol l +wood bury +tart ar +sav o +q o +cou ghlin +civ ics +blogger stribe +ther oux +royal rumble +ni bbles +k ro +gar fun +west jet +track suit +syl van +sof ten +reg tech +goo oooo +bio graphies +barnsley isbrill +adam levine +ic f +guit arists +gal ing +cour tois +black hawk +ta gh +sa kes +religi ous +o er +an j +table ware +ru de +my first +mun itions +ah m +ðŁĩ«ðŁĩ ® +sli ppin +sharkn ado +gab y +early biz +ðŁı ¡ +sw ad +sorren to +koh ls +kend ra +hahahaha hahahaha +d mr +` ) +é ĸ +mel e +anten nas +work ings +i wa +ha fen +di ah +the k +prophe t +mc callum +m re +cripp ling +ate ment +ab omination +! (: +âĪ ŀ +world heritage +un reliable +t into +sho gun +que sta +ho tep +b po +al r +supple mental +mm f +it en +dor n +con current +arsen ic +martin is +cu sp +ðŁį ľ +za hid +is fun +as ahi +ðŁĨ ļ +wal kie +spo d +natural hair +blader unner +an se +it ory +infe station +gover ned +dic e +custo dian +sulli van +r ong +n dam +hi z +d ba +teen choice +sid harth +sh ami +magdal ena +john lennon +f nb +en rol +con form +unh inged +sp ay +flat ts +dar shan +to ver +si ang +one er +mo ga +lead ed +ef ur +din burgh +mezz anine +angeli que +e fl +ba ar +you ra +nbc washington +et u +disco vern +dimini shed +ten acious +precar ious +lo tu +kel e +j illo +gag reader +bre s +bal ding +u is +right now +richi e +euro maidan +dwar a +cur v +chann elling +ben zo +unreal engine +u shu +n mr +let ts +is r +fergu son +elev ations +dream works +tape red +ruff alo +pen ne +ful ton +down trend +depre ssive +actu al +vijaysethu pathi +th monthsary +fla p +de human +bol she +a sta +uchi ha +sha b +scen ic +pla gi +lan sbury +몬 ìĬ¤íĥĢ +v ri +un interrupted +sw ami +concre te +world mentalhealthday +work hard +tru ms +ser if +py on +os x +oh t +le dit +la gs +graci e +ðŁĻ ī +summer camp +karan patel +av p +ãĢ į +weather nation +the division +miser ables +liverpool fc +king sc +ju ba +holocau st +co eli +ade y +âľĮ âľĮ +un marked +swag gy +finger prints +yel lows +vo m +sm th +ri ser +on ge +no tions +vac y +tn wx +sh ala +nc state +leav eno +ec ke +dutch man +cor o +bang ed +te ver +rout inely +new schannel +hec tor +g mp +fo z +cor tina +w ce +su zy +motor spdwy +ma ye +zimbabwe an +sa ip +head ingley +glit tery +establi shes +es cotland +ander lecht +ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ +wood man +ri az +kritis anon +ko dak +ham lets +ha c +flee twood +antic or +z it +yar n +tu t +tin ashe +mand ed +dam m +d fl +comfor ter +bicy cli +u il +succe eds +pat ernity +no ds +cu sco +aband oning +white ley +weather man +shi h +marau ders +hilli ard +di rek +chor ale +ali c +ðŁİī ðŁİīðŁİīðŁİī +makeaw ish +maha thir +loch te +dro plets +cob ham +cle matis +besie ged +pan kaj +illustr ators +co burn +affl iction +travel o +ruff led +nag aland +doc trin +bul acan +aqu ap +me de +h sr +du bbing +cat z +ready to +fin alizing +e pping +defun ct +bonifac io +agu as +zo ic +taxider my +pra dhan +har ass +grad school +counter tops +clt motorspdwy +-- @ +secre taries +å ĩ +u pped +ser ap +pel t +id an +humili ated +ðŁİĦ ðŁİħ +ðŁ¤£ðŁ¤£ ðŁ¤£ðŁ¤£ +psycho logists +mil le +extraterre strial +emirates facup +chatter jee +bre con +t gs +pan el +men ag +ig le +cb se +ron ic +guide book +bystand er +valu ing +nko tb +men sah +go cards +exten der +er bil +draw string +demo day +pho t +fe red +chic har +beth une +be m +mol y +loc ke +excur sions +data set +bre ds +ab aba +ðŁij £ +stab s +scrip ting +par m +paper less +mu zi +kra sno +cle ft +accu sation +mon oc +gre wal +e dia +cb x +cal ender +besti val +v ying +uph olds +sa oir +me tic +châ teau +alle gri +yo gy +tro is +mar ley +g fuel +english heritage +emb ed +counter top +ba chi +:- ) +Ø ¶ +r sac +mill eni +ko komo +deduc tible +am ia +yam aha +total ed +th war +p br +mor den +mo yne +k attu +cali ente +tra ve +th ayer +scoo t +mc crack +gu bati +gas s +fel ices +bac a +un fairly +trust the +sb m +sadiq khan +pri mates +gen i +/ " +wou lda +un well +rac quet +pa than +me wx +hol comb +hero academia +fro do +eng vind +bet fred +ãģ £ +west end +sch ke +rat at +les worth +kashmir is +juxta position +je annie +green day +g bs +f nd +temper ate +philadel phi +fé in +eye hinakhan +ate k +ðŁĺį ðŁĴĻ +v awx +j col +er d +aj u +adirond acks +y na +tra shers +super powers +re funds +ox on +mu rah +llan o +aw ana +sly ther +kh on +cast ello +blo ch +าภĻ +up t +illa warra +gram mat +eintra cht +as aram +sprink lers +las se +haul s +chry san +cas ks +ठª +woo jin +ther mo +oppos ites +le ttes +heath en +goal less +gi ga +esper anza +anzac day +ac ul + « +te sol +obas anjo +fac es +sw w +stat en +az io +shor n +radi on +maiden head +inspec ts +ri ordan +jen son +gb h +ek u +albi on +ta ffy +slu r +rit er +nouri shment +mot ley +life guards +frei burg +cent ro +bir u +bb ci +aj english +swee tt +per shing +new haven +ake up +abru zzo +the b +sig ur +remember ing +ram pal +qu eri +conven ed +braz ak +alco holism +ॠĤ +firec racker +dat emy +dar o +ðŁį Ń +junior bachchan +dis likes +af oot +t ff +i one +deer field +cory booker +pull back +mail bag +j emma +daily photo +cur ating +tourmal ine +pupp ete +pots dam +nis ar +madi gan +bas set +un intentionally +til apia +smo s +naji brazak +xy z +squ amish +produc thunt +ni z +ellieg oulding +ch r +è ¶ +un followed +heu er +wo ww +un prepared +pe els +moul ding +mar le +bellator mma +ar rays +z uk +social justice +ri ma +ent angled +absin the +⼠° +thro ats +theat r +sligh test +shame on +notmy president +humanright sday +. :) +ðŁĺ»ðŁĺ» ðŁĺ» +your world +testimoni als +f br +cumbri aweather +| | +spr outing +san gre +roa sts +respect fully +ma el +l x +kan ata +k ells +dam nit +spur geon +pitt man +k we +geopol itics +dec can +chri sc +venkate sh +ba ad +plun kett +led zeppelin +lang don +gilli gan +fur sday +bu gg +blogging gals +u lit +sab r +ivan kat +gl ene +cd f +am m +am ble +âľĬ ðŁı» +vote snp +star gaz +cma awards +alder shot +vv v +lent en +ax is +tt r +thi bo +sunny side +pulw ama +jan us +ðŁİ · +tru er +shel fie +pas ar +lowest oft +gu ac +go diva +extraordin arily +country man +view ership +mag ma +gen g +qu into +m fl +gh d +cr p +class ico +sho veling +se same +re at +ed ining +boyn ton +bea sties +west life +trous er +ter i +myo gi +min ced +inde struc +domin ican +ê ´ +t sai +son n +sm in +jeff gordon +cypri ot +cast ro +boy with +americ orps +ac al +ðŁĴ ¢ +pa e +i shi +gau rav +evangel ism +e ic +cur ate +ti sdale +synth pop +spaw ning +role playing +national ities +hormon al +re constructed +g ne +ed way +dom i +doc u +* : +shre w +seth our +out day +mor ia +ma ther +latt es +ka it +k ri +jolli bee +di or +defe ctive +al bin +adri ano +: ((( +u cha +serv ings +il los +green brier +dye ing +congr atz +moon stone +exi les +bell ini +additive manufacturing +tin ent +ster dam +sar kar +re lin +ov c +o del +hat ties +feel s +colla ges +as sign +nik kei +my r +minim ally +bel ting +Ø ³ +stat ely +mac pherson +lob ster +hus band +ca po +bigh orn +ç ão +ver tical +sten berg +locomo tives +stocki st +norman die +mo yer +homec are +hassel blad +dad dies +cal laway +ai x +tar te +s gs +pr une +ner os +mush y +mar go +bel ton +bag ley +ai e +youn is +tom cruise +swaz iland +samar itans +nat ur +mi ata +la paro +anti bacterial +u we +ori on +loscab os +loc ates +ig p +gi ff +plun ges +maj ldr +ha ba +fac ul +cran ky +air drie +ag ron +r fk +p vd +hey ward +dat eline +bo ko +t ma +sin ce +objec tion +harmon ious +gren ache +clash of +)) ))) +ym ents +si mb +ridd ell +rb ny +mur da +may hew +ton in +swal lowing +rand wick +ob c +ig non +f ann +begg ar +ab q +suppor ting +se go +plate let +l chf +Ø§Ø ³ +wi pers +web toon +out cry +bio l +urban decay +taran tula +na uru +megh alaya +medit ating +me ren +lett ings +hol born +ðŁĴĻ # +trish trashers +ry lan +n ne +mand ated +full ness +field trip +chi sel +buil dup +ty ra +made with +ha ile +forgott en +dan gote +women smar +ti mid +ski m +si kor +rig or +reig ate +pu tty +illu m +fat ale +bra sile +bass fishing +af a +âļ ĵ +su prise +n endor +hair dressing +cd l +be cks +bart ley +wit tle +tang a +l acked +fox business +ducho vny +day time +audu bon +think able +se marang +roman ces +north umb +nl cs +io e +bt sport +ste dd +pa b +shr oud +red line +pla ge +p ell +lip ton +achiev able +take over +ru ci +o vr +mide ast +jun tos +amo ah +ve tting +v eng +ti my +new shour +le ste +indu ce +hard waj +de se +ba idu +my cleanindia +leg alized +am monia +web by +un tuk +stone ware +ap id +sol sk +satis factory +head master +fulham fc +chi dam +bere tta +ðŁĹ » +kil len +early bird +away days +ni ve +narr ation +is b +eter nal +tylero akley +tri g +scoun tdown +ol en +myogi adityanath +indi atoday +f news +engul fed +th aa +subsequ ently +music app +constantin ople +sta hl +recu er +em m +u om +stone bwo +south wales +mi zu +joy stick +hydro electric +hat trick +vivo ree +ayr ton +ðŁĺħ ðŁĺħðŁĺħ +u sch +k ham +d proud +ðŁĩ®ðŁĩ ª +ton io +lal u +kil os +hel las +gle aming +face of +east coast +the truth +ston ers +r gv +jo liet +e spar +al cs +@ âĢ¦ +sh ingle +enchil adas +cast ile +bio fuels +am il +al pin +r ile +mu da +chri so +aw ad +to b +stor mont +mat tresses +hel o +hee led +dul lah +chom p +chic os +bis que +lovely z +gali lee +co va +vir k +subli minal +phosp horus +l mu +footb alling +drogh eda +cro cus +madhy apra +graci ously +gen ova +ex pos +cruiser weight +bi ken +af amily +accr ington +tt w +ted dies +spon taneously +som o +sla sh +ben et +afri que +vand al +un till +tor ius +stadi um +nnam di +migr ant +man na +ll b +kar oo +chi les +cave man +ðŁı³ï¸ıâĢį ðŁĮĪ +separati st +ron pa +pa cha +oper a +macau lay +frank fort +fr ills +ev ade +aud iting +theli on +par take +mck ellen +man is +ka yo +dee pak +cas sp +zam be +sunday brunch +ra sa +qui p +adhe rence +s wed +le mieux +stu mp +litt les +evalu ations +amu let +ðŁĺĬ ðŁĺį +n ch +ðŁĴ¤ ðŁĴ¤ +âĻ¥ï¸ı âĻ¥ï¸ı +were wolves +ste ers +scar face +par tied +de su +creepi est +controversi es +adri ft +su mer +sou p +ri go +let stalk +irrit ated +grou pp +carni vorous +autonom ous +au e +al pes +t fa +m gb +incan descent +glo ve +cant ando +tas man +sab re +liveon komo +kapam ilya +fang s +di lem +deb bi +bah ra +moha bb +g mg +g da +ke xp +bal an +ux bridge +t of +some things +keigh ley +embarrass yourbestfriend +cho ke +nab s +am mar +adjec tive +ðŁĴĺ ðŁĴĺðŁĴĺ +vol l +pin to +nhs england +krit i +it age +collec tor +black twitter +b more +ab and +sher i +north west +mtve ma +kel so +iz ard +bur gos +ãĤ ° +wet test +ma sti +i stan +tri al +th enight +purpose ful +off ical +bbmastop social +ar g +vent ured +vas co +male ficent +har k +barre tt +re adies +quantic o +jen ks +centr alized +ye m +un tapped +un m +n bas +ivankat rump +ingl ory +haare tz +ul cers +sky nyrd +ru ms +pre cast +md w +horticul tural +geel ong +egg nog +cataly sts +y all +woo ooo +to bo +shru gs +ev in +ser mons +nau tica +it in +emb a +coloni al +bow er +blin king +bbcc in +thin ning +stu mped +sh awar +psycho therapy +o ssa +dolce gabbana +bra zen +: . +stur m +ribe iro +nbc days +zz zzz +wozni acki +with love +mag ick +id l +func tion +car li +ai ya +sp its +sn fl +os m +mo ya +hi jack +great britain +a vey +âĸ¬âĸ¬ âĸ¬âĸ¬ +u ea +stom y +quidd itch +pine apples +spoon ie +sch rader +ram blers +knuck le +gra ze +durand uran +d har +âĻ¥âĻ¥ âĻ¥âĻ¥ +patron age +nieu ws +mee ster +ij n +i is +construc ts +ðŁį ¯ +taap see +death ly +back door +aero sol +wh c +t ss +of honor +bring it +athe dral +ate c +ðŁĮ ķ +v us +tokio hotel +speck led +scon i +sa under +ra be +fairy tales +e is +av ers +ab rupt +ðŁĶ ŀ +umb c +su ren +pfi zer +love yourself +in uk +ger son +en ish +the archers +te pe +solom on +sign ite +s new +rav aged +ra ul +hon ky +ci b +chester ton +tv d +neu tro +n lt +musth ave +lu vs +han lon +coinci dentally +æ ² +projec ting +h sa +digiti zed +di min +chilli wack +kick sonfire +id ad +haraju ku +du eling +discre tion +ten ny +progno sis +pitch fork +le vee +d hy +co ven +co pic +san disk +ilook like +be sar +ar ind +try on +nor way +levit t +eun ice +w pa +scan me +quin n +met z +land au +in wood +er to +cruis ers +craw led +chap in +car nit +angel is +fl an +chel t +bri l +na in +integr ative +here sy +d app +bn pp +ut k +stam os +sco de +pen ta +name less +ka is +in elli +ill ating +sa ina +renov ating +nut anix +grand child +bo keh +bat ch +b ure +approxim ate +몬ìĬ¤íĥĢ ìĹijìĬ¤ +zam bian +fallout boy +atl traffic +un mistak +o ink +je k +ik amal +emin ence +wor ding +unimagin able +mock ery +hy man +hand er +go onies +franch ises +collabor ates +she ik +immuni zation +fre es +ayatol lah +as on +un abridged +rec iting +jen winget +du ly +& âĢ¦ +stra pless +han ey +chev alier +ber th +ansel m +acet ate +water park +vio let +s mann +s illi +of t +movi enight +do reen +collabor atively +ìŀ IJ +un confirmed +rubi k +ru di +ny knicks +longe xposure +k ur +vitam in +tra x +megapix el +lat robe +in deli +hoo oo +dream hack +dive st +deng an +cover up +comb ing +colum bu +wil kerson +lo la +flu shed +fi gue +dou in +contin ental +capit alize +baj wa +wind power +sha e +se asi +plan ks +pi i +n cbn +extin ction +ÄŁ an +tot p +rex po +oc tu +mo k +clo t +pick ford +osteopor osis +m alian +intelli gent +dimen sion +beetle juice +abre u +yo jana +touri sme +scat ter +ro per +pue de +mar tell +he sse +z ags +ta ch +sen schumer +montre al +cou ghs +ab usa +willi an +sur in +stain ed +north wood +lil ith +gun ner +ab ay +sen der +corp ses +u go +house gop +stro m +li ddell +ki ki +dir k +( {} +rela y +ma ire +cray fish +se da +id h +boy co +ðŁĻĪ ðŁĺĤ +sam son +post pone +n ra +es n +de wan +ber nabe +an thrac +ìķĦ ìĿ´ +under mining +sm v +gior dano +cor ne +ca stig +bal moral +peder sen +pap s +du e +ad here +vanc ity +ta za +t ada +le if +incre mental +house full +secre ts +eth am +ex es +r itic +keto genic +kerry washington +kean ure +du go +dra b +college gameday +co gni +ac ap +uc sb +nab il +corri gan +al ain +sh ale +s ws +im ti +bre ve +ar ai +pc gs +kaw i +har ford +gerry mand +casu als +an ish +th ap +lo aves +go alies +cle e +pash tun +ven mo +vaul ted +shi var +re gur +plum me +fun ders +t sch +rapp or +r ten +ple t +deb ilit +chil ders +black ness +black heath +az im +anthro pom +alco hol +wednesday thoughts +wan ker +lon goria +ne spresso +holland aise +artist es +ðŁij ¦ +singapore an +miam is +ent or +d lp +be ero +ak ka +united kingdom +unic orn +stan k +shi k +pres sured +person of +impre ssing +grat uit +grac ia +gang es +detroit redwings +century link +inter collegiate +boo ed +shi ki +opti ma +onthe blog +margher ita +ling us +en bc +don i +yi fan +r ba +fit test +dor ff +dep tford +dd g +woodland trust +j cu +er skine +dab o +re tr +pe eta +interpre tive +comman dos +son o +ru ffles +bi bs +mercuri al +lo pe +grim shaw +fairy tail +d ood +con nacht +bot anist +yam ato +wal ton +tri ke +sh ards +motor rad +mach u +fa had +demon eti +de h +cy ril +ch roma +bla zer +wau kee +the fan +sj s +si ro +sch iller +play wrights +geopol itical +cb l +c mb +brick yard +ëĤ ¨ +sul ts +policy makers +marx ism +el paso +dil ly +at tainment +watch ing +inser ted +bl ick +as pi +of course +la ois +a sti +ju illet +har ness +enrol ment +ðŁĻı ðŁı¿ +ðŁijĢ ðŁijĢ +hon ne +evo kes +curi ous +clo thes +tu lum +mo x +lo fc +ka os +gun point +carav an +boo boo +tran scrip +pollin ation +gas m +den ison +cam e +ãĥ ģ +obsc ur +liter ary +g ati +disneyland paris +ag ames +mn p +mitt romney +maha dev +hang a +ðŁ¤ ¬ +pre ordered +mj fam +ku al +in day +duck ling +div yas +bo v +af tere +" ), +wo bbly +transi stor +thom son +sc l +l ach +gur ley +fu tur +door bell +cau casus +ile ana +george town +be ste +ðŁļ ģ +ðŁĺĦ ðŁĺĦ +st ence +s ü +or ti +male c +islam ists +heart throb +crucifi xion +ali ster +wiz ki +cole en +app alled +sk am +sh indi +nightw ing +fix ation +tri vand +stir ling +sing ham +sh able +fro wn +cu ses +ano inted +tar yn +presu me +nu anced +meck len +ku bo +hl pf +funer als +flo at +wh edon +trans fusion +fc ps +af u +subor din +she khar +seaof thieves +plenti ful +pente costal +pa sig +beat le +squ ires +conge sted +som brero +ring ling +rein hardt +is love +bal last +annapur na +al ban +/ : +vi ent +tit ties +gro oms +du xford +dan vers +bab ar +ack erman +x factor +v ms +uniq lo +sporting kc +pen al +over run +ne arer +nad er +life hack +ko ku +cr pf +vehic le +un ners +serv o +n ta +i wan +h md +emp tying +de kker +chu bb +back yard +news flash +n st +ley ball +lam bing +jamie son +folk sy +cram med +polyu re +mpu malanga +karnat ak +ef er +w has +v age +till is +street art +nit rate +nas s +gues thouse +blan ken +save butterflies +photo bombing +pe bble +nbc sports +ke mb +jessi ej +human ism +ge ki +ern yo +dancing abc +all ard +al ford +ab r +shin hye +repent ance +lym pho +don c +di ol +no l +ठ¨ +work book +vincen zo +spra yer +mental illness +by te +ðŁĶ ° +sel var +puri fy +min zy +ce ci +cbc news +âĺ ł +win tery +toronto star +gar ret +cassp ernyo +atl é +al can +one more +hist fic +hat ches +ha se +gy ro +gamb hir +erik sen +afore ver +yl o +valu ations +sel tzer +nus ra +ðŁı ¹ +plagiar ism +per la +kun st +jon athon +inqui rer +black face +tri e +pas a +joh no +chicag oland +chi al +ag al +trin ket +fran tic +din on +cancell ations +un be +sch me +promin ence +o stro +com ical +e ads +weav ers +antwer pen +tri an +ec ole +bil bo +b su +cospla ys +conven e +cm te +barric ades +amazing phil +) ] +tat i +sh app +scis sor +north ridge +nazion ale +gro cer +eat more +ea ves +de sley +bbc weather +b vi +ðŁijıðŁı¼ ðŁijıðŁı¼ðŁijıðŁı¼ +youth day +thur rock +tensor flow +man z +katow ice +high life +deci pher +pig ments +mu mma +bu f +amar in +trouble shooting +snap deal +ol ar +jeffgordon web +dog wood +kat ya +itsenrique gil +bigo ts +ðŁļ ² +ker now +jay alali +in separable +x files +war at +mu z +mo ped +break throughs +bran ching +bouti ques +word sof +wi st +tren ded +ren aming +r hom +maced onian +keanure eves +approach able +y bridge +ve il +ty l +tamannaah speaks +sti f +photo friday +e ir +cav ities +proce eding +pix ies +key hole +eeee eee +ultimat um +stu ffer +mar sala +groo vy +dal ston +ðŁıĮ ï¸ı +vin ay +lat inas +ga is +fo les +be yer +app al +th ales +soun dof +moderni ze +ligu ria +jav a +carib bean +aa yog +wiki media +socio economic +k cr +im raina +hygi enic +the kid +stret cher +scot ch +pan cho +oo g +nat west +nam ur +ðŁĴ ĩ +re shuffle +o a +go m +es f +dill inger +bu sses +bac cal +sa al +person ali +n ought +lovers day +kew gardens +ge mini +du x +bud den +blood line +bi les +air quality +ìĤ¬ë ŀ +âĸ ² +razor back +londonis lovinit +konstant in +k vue +ima h +: ,) +spu ds +skyl ine +lux uri +loy alist +horn by +deb t +charle ston +more head +health day +ess endon +ef m +cow es +timm y +oxid ation +invest ment +inthe city +geo g +ale gre +ðŁħ °ï¸ı +waf er +ri bu +m tsu +fab ulous +zyn ski +va inglory +under whel +ri bble +men sa +kim ber +insol vency +gen ous +ck d +person as +na e +iv ory +dagen ham +ra o +mouth piece +mor ne +le mmon +gl ace +etsy social +chiranje evi +tv series +the u +sait ama +ging rich +flag day +b snl +au ra +ao i +hol brook +green ish +consult ative +win drush +water side +n ff +lovel iness +live in +for heroes +ðŁĶ ± +vo i +p ne +nol i +l all +horse hour +bre whouse +be mid +pd p +fron ten +fri eze +ar acing +æ ł +sub tle +sm ac +ah san +ts v +restric ting +li ano +is mail +fianc ée +ad oo +yn olds +pret ended +om yo +n aca +convic ts +battle ofthe +ðŁĴĥ ðŁı½ +re vo +kil lah +jad hav +gree ley +fc cc +ev in +y oooo +te al +shiv raj +rival ries +rel ational +pos ite +nct smtown +fi at +anam bra +aerop lane +# / +ðŁĩ¹ðŁĩ Ń +rein forcing +just sayin +incub ation +de u +( ...) +vern on +new swire +lan ge +hypo critical +ac ity +abu zz +star news +rhino ceros +rais ing +pm qs +pin as +ne cn +mtv lak +harry potter +att is +sof as +patho logists +oc to +mont mar +hah ha +far aday +ar murugadoss +appell ate +saku ra +imperson ator +er go +dog sare +bour go +talis man +pon dic +il legal +work flows +thn ks +sm itty +skin care +poin set +pic spam +man soor +exac to +ech lin +as at +alleg ory +y asha +u mc +re kind +rat an +pu ck +ip ur +humble isd +christ o +bel tran +az a +ab bi +vi sto +shin hwa +playo ff +pa ve +hun an +bush nell +) !!! +ðŁĺļ ðŁĺļ +st win +place tobe +non violent +lon go +kal ing +geo engineering +audit ors +è ¡ +uof l +tal ker +s borough +patho logical +or as +elm wood +bur l +bear den +b hat +relent lessly +men om +j alil +e bene +augu in +men tos +im d +fur sona +ras mussen +ran ting +kas ab +k lang +ide k +dy nasty +cbs thismorning +mt bos +ðŁĺ ½ +re worked +mali bu +lo ban +la zar +host els +do in +def ra +breit ling +bis on +an r +sa want +quin nipi +mcar thur +ally son +aler ted +y lang +tr ul +ron ald +pro ds +master son +hel io +get the +fire emblem +cup final +bre st +ðŁij Ł +y aaa +van quish +track ers +rosal ind +persu asive +new found +g sk +el ke +dev op +ci ar +buck le +aly tics +yah ya +ty me +the dailysketch +th aan +personof interest +e bel +atlu td +Ä « +tson ga +scari er +rise and +pass able +pa than +lib crib +im g +execu tion +yal it +re port +op ie +dun geness +dream home +ne ssa +monu ment +mill enium +dani sh +bert son +é Ļ +w impy +spanish gp +slic ing +n oun +la borers +ji hyo +f st +dad dario +bang or +' ." +pra ha +mau de +jacqu ard +hi ra +cook books +th wart +sor riso +me din +infe rence +gr inning +cor du +ano inting +íĺ Ħ +val do +ss oc +screen print +s ree +privati zation +national poetryday +healthand safety +er ner +the five +technic a +run es +per in +don ahue +bra c +ber nab +wizki dayo +ra bat +pyon gyang +lion el +fi da +cla us +bay are +aldub the +ðŁĴİ ðŁĴİ +suz uk +retro grade +moun ta +ma der +her ding +ðŁĶ ® +soun der +s forum +gre tel +ಠ¨ +pa the +edg baston +do h +bob bie +ðŁĴĶ ðŁĴĶ +se alife +s ree +mu gg +monte rey +no am +much os +lu red +t dc +superstar rajini +spal ace +show us +i go +faw ad +wa j +smash bro +jacob sen +dvor ak +regre tted +ral f +no b +lobby ist +isai ah +etobic oke +brant ford +bon ey +believ able +agre en +ðŁĩµ ðŁĩ· +sky fall +shilpa shinde +re spl +rail road +pau k +fun inthe +fi est +co cc +cho ck +beli ke +alli e +qu at +public schools +mar o +h ing +gloss ary +flo tilla +figu eroa +f illies +birth right +bar olo +am ag +é Ģ +tab itha +ra shi +pu tra +com or +ky un +il u +cad re +belle w +ab ort +sp fl +nick carter +naw ab +jol t +christma sin +carr illo +affirm ations + ª +yipp ee +as sail +à° ° +ske leton +river walk +per l +nin ado +mis understanding +hunting ton +holly woo +bel lows +¨ ï¸ı +unru ly +the weather +sw ar +ru stic +reggae ton +my ungsoo +muske gon +fili gree +czech republic +ch ch +un thinkable +vaccin ations +swasti ka +sol vent +ipkk nd +hel ve +aldu beb +raun er +pho en +jo ya +twi st +trade marks +spor tive +scor cher +razor backs +ra ik +infiltr ation +biow are +archi vist +ak ita +ç¥ ŀ +meek mill +kn ap +cag ayan +wh id +tu ll +sri devi +mis fit +ma v +imacele b +fo ils +cc b +bren don +bic ep +al ittle +thr ice +reg alia +ra bo +pain less +overest im +marin ara +klit schko +ig f +hr inger +gu st +captain swan +ar ay +ðŁİ º +á il +u day +co bras +caitrion am +u ig +hard top +eci g +bach mann +k wara +eric h +de bs +contra sts +turbo charged +rich man +provo ke +long mire +dilem mas +the blue +me di +ley park +fam s +e sport +bi ko +bar ium +aveng ed +allar dyce +aar hus +better call +king sbury +gn ant +friendship day +substan ti +sch ip +pep tides +mate en +اÙĦ س +tur alism +st ang +ra aj +peace keepers +li ana +exc ites +vaz quez +us gp +travel ing +pill ar +gu h +competen cies +ar tur +vo lo +jer ome +di adel +den ny +av fcofficial +u dd +mo dy +mini str +ge min +cryp tonews +chitec ture +z infan +super fast +st ace +saj id +kra zy +ðŁĵ Ģ +philipp ians +nis a +book sellers +Ä ģ +victor ian +the body +su pt +salmon ella +rat ty +jo gger +fu biz +cree ks +bled soe +ad ell +zinfan del +trape ze +si z +sho eing +le pro +ja vid +custom ed +sa ath +quar antine +mis sk +detri mental +champag ne +bi k +; _; +wa f +tiger woods +star burst +rach man +ok ada +new day +ly ca +der rick +anec dotes +stemc ells +pas cal +hu sain +clai borne +bol son +apar te +ai pac +wi k +w ch +stimul ates +morpho logy +logi stic +indom itable +gal oo +comm end +chaw la +' ( +tru jillo +lown des +log ics +liber ating +as am +arrive alive +aro ons +а н +shepher d +p bc +li po +er l +citic bs +cc sd +caitrionam balfe +br fc +se ki +it out +ish q +dil do +ati k +amar inder +tal kie +state hood +ca be +bos well +ðŁļ ij +wer th +va al +sky ping +ear phone +dilig ently +co chin +ap hi +am ente +timesof israel +sel assie +road runner +ok ay +ny der +ni ven +la ir +ce ased +categori zed +ðŁĴ Ĩ +u fo +tele scopes +om ania +cam ino +b illa +aw ning +Ĵ ï¸ı +ðŁIJ ħ +ðŁįķ ðŁįķ +wom ans +re iner +peace building +neu ter +dj ia +cyber bullying +cs x +constitu te +b the +zam bo +on ta +cal loway +steel head +one team +ini ans +i zzo +abor ted +se to +maldon ado +good day +fil mo +bre ck +hang outs +gibr an +z sa +whit more +stru p +short story +jen i +energi zing +con vening +check mate +batt en +amazon in +alfal fa +star ks +q v +ma eve +le fish +ide vad +earth capture +bn buzz +bau lt +amate ur +us l +twitch kittens +tri ms +mb bs +kodi ak +din ky +choreo graphed +ben son +ar aw +ÑĢ Ñ +real tor +fun facts +f nf +d mp +ben ue +baye sian +the old +subscri bing +ra king +official monstax +g ak +drink able +detec tive +trilli um +snow men +shah rukh +eli ds +dismant led +mo dest +lud acris +can trell +ðŁĶ Ļ +âĿ¤ï¸ı ⾨ +Ú ¯ +yw ca +tb adly +sa ha +por tof +lu cre +la ken +ha skins +vinyl records +p ima +mol o +ign ited +gau ges +f sd +ðŁĽ į +mat su +g ant +hen nes +h bo +bu sta +se tups +scor ner +reli eving +neur on +irish man +fo gle +d bn +summ a +pi ppin +micro finance +fanci ed +chair woman +brah ma +fal low +anti social +wi a +t ments +ram i +ra iney +mind blown +ly man +afgh an +billi ard +author itative +ye hun +sput nik +bombard ment +nl traffic +mar ic +ma vis +in nov +central park +bli ge +ry de +plun ged +patho logist +host ility +groove musicapp +enti st +em be +chi ba +chast ity +boul dering +bc l +accumul ating +ðŁĴļ ðŁĴĻ +smo king +sm town +pre ssie +k lik +je une +ikamal haasan +highe reducation +e music +ðŁĺı ðŁĺıðŁĺı +war ing +up c +stra chan +sp itz +rober son +nick laus +mue ang +interpre ters +cy c +casspernyo vest +cam acho +sl png +pent icton +min hyun +ki ah +i vo +energi ze +dou gal +alo ha +winter wonderland +ir win +i ar +handle bar +gal lows +est ro +en vi +trivand rum +sty rene +medi ums +gains borough +dr ina +dis agrees +d la +aud acious +wizard world +us ac +subdu ed +slaughter house +n wc +macchi ato +ham er +gat os +debun ked +contact less +c ing +z f +ver meer +trick or +opul ent +is ure +gaz i +filmis notdead +canon uk +bam ford +ske chers +shi ver +ko gi +h mi +gh ats +cor leone +su g +som uch +lo athing +l int +foot work +en list +du rian +canonuk andie +ab ot +x dd +jar gon +ban di +:) " +the only +sm n +n ha +looo ool +idevad hikari +h bl +fol l +traffic alert +kau f +dd p +ad in +so d +rom ford +re strooms +bol linger +sc cc +guardiansofthe galaxy +ash er +api ece +ÙĦ س +rod man +ren z +proce eded +hul kho +equi pe +whit worth +visual isation +under pass +thor p +tae hyun +power fully +pag ani +memor ials +mar vels +intan gible +win o +pe o +o der +ilo vel +gil christ +deep ening +chrise vans +chi ka +br ü +persi an +jer i +ful lof +em manu +cu pp +awesom ely +alvar ado +woof woofwednesday +me ticul +info wars +friend sof +fair mont +cov ina +cma fest +bul ky +agno stic +far ne +anton ov +ta pi +ofe x +men tored +e pps +a had +ãģ ķãĤ +treas on +ro dents +riz vi +pari shes +mal am +ka sey +appreh ended +absolu t +tech ed +pitt sburg +o real +mar itim +li us +laun ce +correspon dents +wat ery +s jr +ron ey +neta ji +glori fy +ar son +âĿ ķ +Ø§Ø ¦ +wood block +rt p +m th +iam jericho +hu ge +gh at +car go +a edt +wig more +shutt les +retribu tion +pinot noir +back room +abhi yan +ðŁĩ§ ðŁĩ· +sir l +se gura +latin america +ex id +be aker +arch ite +wo z +un load +student life +motiv ator +ku ta +green ock +go zo +con st +respl endent +e mulator +atten u +the zon +reser vo +mc gon +in dah +ga it +domen ico +do sage +ant ler +oh ne +inter ning +cor mier +ci ence +å ij +ss ur +red hat +ou ach +high score +exclu des +ear ls +cal u +simul ate +mur frees +kru g +gat to +christ a +an h +start shere +sin n +n wo +lo ween +g lynn +flo rentine +dra go +spi kers +shar m +north wich +liquid ation +are llo +walk about +ting ling +public art +on earth +mu ker +interrup ting +il va +de brief +cancer ous +big sean +week night +t cc +gene si +el ka +ci pher +cali ph +ti eth +re produce +koo kab +kel lo +aldub x +shoe maker +imagin able +าภ¢ +w bu +th ay +strato sphere +red stone +pla s +pimp in +mi p +lu te +hatties burg +hal lowed +fen wick +tweetapicture youcantexplain +pro gro +not ley +jaw line +dev ouring +resi due +redon do +mm t +mccaf frey +human it +gr m +dou ghty +ë¦ ¬ +wit te +til bury +men sch +intellectu ally +col ada +ar ad +ðŁĮ ¤ +understand able +gu lati +ghi story +ase m +ram ping +pr is +mt p +ic ul +gerrymand ering +fan nie +dealer ships +coc cus +carav aggio +ameli e +ra ger +twi sted +succumb ed +spino sa +ku mari +iu pui +horn sby +cro sse +c fis +t ingly +ss ohn +sab ers +red cross +over priced +ni sha +kat t +j peg +internationaldayof happiness +fau x +ym c +ug i +tn z +sw irls +strike out +st k +shred der +ninado brev +hulkho gan +gh ia +discer ning +bru yne +! .... +tac loban +r ind +major ca +le uk +grand mothers +g bu +buck inghamshire +ðŁijĮ ðŁı½ +ìĽ Įë +âļĵ ï¸ı@ +photo synthesis +jugger naut +dan te +stick y +soft bank +im mer +ha ber +! "- +y ue +ru in +id m +hing es +cricket worldcup +wisdom wednesday +tra xx +old skool +now reading +mar la +kas per +gersh win +dugg ar +ber mond +yid dish +tayl ors +mus graves +ft l +dun yan +chrome cast +ðŁ¤© ðŁ¤© +nc p +mick elson +mart en +mahar shi +contracep tive +à » +vit ae +ten ni +stal kers +mirror less +excav ations +cnbc tv +christmass y +cav ed +ste pan +sep p +oo dles +âĹ ĭ +vil les +ro ving +pan am +nen shi +l ö +bun n +âļ¡ï¸ı âļ¡ï¸ıâļ¡ï¸ı +w wa +kae mia +pre print +magi strates +kam ikaze +ka jal +inge x +equ ator +box ingday +aw ara +ser ye +not ched +membran es +mar an +humili ating +some one +sno qual +sn j +r pt +pries thood +man hole +bur ke +ðŁĺĺðŁĺĺ ðŁĺĺðŁĺĺ +ug c +pim ple +n aco +made inthe +al em +zi onists +wau kesha +nip sey +march and +// /// +⼠Ħ +to or +new ham +metamorpho sis +mag num +don ning +ci vit +bn k +v oot +the v +sar nia +sabbat ical +pa ppy +on fire +leon id +go ff +gi joe +gel atin +garfun kel +camoufla ged +air ship +paras it +nca ad +dor man +chann ing +ce bit +at ina +ðŁ¥ ħ +weare alive +rall ye +nightly news +kiba athai +fate h +codi les +amp he +yz f +su man +dm z +colli ding +assembly man +à® ¯ +year in +to ga +po tions +patric i +matter horn +ma so +kath thi +kale b +hashtag roundup +appo inting +ac g +stro be +re twt +o don +ming yu +la z +ci p +alex ey +shradd ha +schoo lo +ma do +gh ey +curren cy +mis sus +ingh ome +glob alist +di vo +cbs la +be amer +pier son +nb supdates +beau coup +bb it +anom alies +rally cross +man gan +ha zing +bred dit +sony alpha +out source +haram be +esp ys +bu x +bol and +bench marks +as ka +time travel +protect the +moor head +kar ate +jag er +gri ffey +ex port +craf ty +aw ild +arche ology +and mail +pad u +ham i +draf thouse +de ane +yugo slavia +wall paper +tyran no +quar k +lic ences +admit tedly +y re +lau rie +fore sight +al yn +ðŁIJ ¥ +yogy akarta +war gaming +tyre se +romel u +revolution izing +lil le +ing in +ah q +xi ao +ti ffin +teacher sday +sau stralia +mid western +gel e +fin chley +fantastic beasts +cla s +âĢ £ +mountain bike +l ates +ðŁĶ IJ +scu der +re direct +her bie +ge sh +frat ernal +ran chi +modu s +mar bled +g ans +f ss +compli mented +clean water +um rah +rock in +mal achi +lasag ne +gee zer +f ated +du be +de jan +as ante +anti semitic +shan ed +rou sh +resent ment +pokemon go +mi ths +catal ysis +sor cere +rati fied +jewe led +go swami +cho ppy +bra r +ar ay +a wh +zo omed +su breddit +stric ter +short cake +onlin ecraft +nes bitt +nepale se +knick ers +hotro d +d pm +bc b +ba ren +agne z +âľ ¦ +u co +s best +k dvr +ðŁį į +param ount +mar lon +machine gun +it ers +do berman +attack on +ac orns +ro stov +re gained +py r +out board +am ol +re sa +ig lobal +hill on +f kn +crowd sourcing +rc p +lo rena +e ow +d mu +ðŁijĮ ðŁĺį +tony abbott +sw b +hu blot +hom mes +gal vin +vat os +un biased +terrac ed +oc ta +mel hor +ilayathal apathy +f lead +burgl ars +electr on +cam brian +aure ate +ali b +under valued +t mr +our ce +ja er +ous al +len oir +ðŁĮ Ģ +than et +r aring +quix ote +loc ator +la porte +endocr ino +change makers +bo dh +so hail +kam il +fu sions +compri ses +ranger over +pau lie +mush room +go shen +fr nd +erc ise +bur t +strath clyde +north umbria +keepp ounding +k cal +htg awm +german y +far thest +eng lewood +block buster +wor shipped +geor g +condu it +weir der +under water +spe y +shi pp +sam aj +fon ia +ðŁĶµ âļªï¸ı +çµ µ +yehun dinon +well ington +s ood +dog gies +wa ites +ss ac +se ep +reas surance +ram sgate +di us +con fer +at too +ìĺ ģ +vaness a +us as +observ ational +na st +mis carriage +io i +ec up +af oundation +live at +gram ps +gigi hadid +end am +bu z +aspe edway +ren é +pin hole +my day +mendel ssohn +k bc +downey jr +ti gger +spe x +radio show +ft r +Ø ¹ +the series +shivan gi +senate majldr +oak wood +i mi +chuk wu +asi a +witz ki +see ley +ro deo +pin point +mod ded +home m +gor i +gb pusd +un timely +sh atta +severy where +nic hole +den ce +ðŁijıðŁijı ðŁijıðŁijı +whit t +reali dad +kin en +in or +fad er +dri fted +def leppard +ä¸ĸ çķ +sling shot +ka iz +cer o +blac kex +ap na +aaaa aaa +ðŁ¤¦ ðŁı»âĢįâĻĢï¸ı +ver acruz +she ph +pi awards +à´ ¿ +will i +visit norway +the voic +swee ties +royal airforce +pic nic +lin z +har wood +cla rendon +super foods +stri ves +ridic ul +incar nate +absa prem +toronto police +ond that +loo sen +de ws +je st +ir cle +chu ms +bri stow +zan te +mulla hs +mi as +ha bbo +à Ń +z aza +war lord +shab ab +sero tonin +ren berg +on coming +ex cre +ev ada +b cos +un dying +special ised +sirius xm +ques adilla +open science +kar olina +dil la +u fa +mir chi +juventus fc +j sa +dio de +command ed +cbs baltimore +be ys +as kar +art collector +am ira +who dat +t wn +po ppers +bl ame +u mber +sword fish +lam ini +for america +fal cone +envisi oned +der anged +wh r +t fs +seag rass +se fton +on di +kemp er +ju do +do pest +disar mament +antag onist +ali m +ak p +sm er +risk management +oo c +fan ni +eclip se +curric ular +ca stel +bal le +ate a +ar by +te quil +k ander +goal keeping +cra igh +bb h +un itarian +maas ai +just ine +be gu +arc gis +sr b +south all +mus ée +juli anna +exce ed +auditi oned +aneur ysm +ac customed +thi erry +syl vie +itu r +humming birds +fortnite game +ent ley +cro hns +cl oning +chak ras +c ely +== = +whit er +rose tte +phi r +ko bayashi +kir kle +gri st +disproportion ately +corin th +bu c +av aya +we iz +vo i +s iti +min ha +meticul ously +k si +herring bone +every one +work man +north shore +mor dor +ir ving +hammer ing +scien cen +nine ties +mar shawn +l illie +for tu +f elling +cro codiles +wi den +tiger pride +rou ss +pl r +os ab +o ski +gu zz +fen ced +end fgm +din es +tr in +sto ic +my er +k ass +k ama +eye glasses +bol an +bob marley +ation ary +air tel +pro ye +mad den +inhe rently +bush fire +ble acher +beautyandthe beast +re ak +op c +lex is +ho bi +eu genie +bar ter +bar bra +ĥ âĸ +à § +wa e +po ppa +long standing +indestruc tible +embar ked +co ppa +bel los +teslamo tors +influen ster +immac ul +hav n +chu g +apple bee +mal lo +je di +er rr +ö zil +morri stown +mo bbed +gun ter +foo tie +e ce +bun d +u ae +so ka +ra bid +lu pu +destiny thegame +class men +bo thers +ar ah +ze phy +mini stering +he ire +fo caccia +distric ting +cros stown +ðŁij© ðŁı»âĢį +sop ro +snu ggled +sho al +ch kin +ari anna +thu ggin +thor pe +mother ship +f end +cam ille +arch i +tor ches +si smo +dome ter +cur tis +tar ragon +oxi di +m ellor +i rena +head y +entr ances +bre mont +bn ppo +tro tting +sang am +no d +daugh ter +cold well +buff y +bbc proms +ann once +tt tt +em ce +culin ary +bre scia +bor uto +big p +bettercall saul +ven er +tor neo +rodri gues +de pra +stam il +ðŁı Ħ +salv aged +madein america +fu gee +comic relief +anu eva +âĺºï¸ı âĺºï¸ı +you ve +rooster teeth +mur thy +lan za +ja xx +invo ke +ign ite +fi ori +boston globe +ðŁı Ĵ +jesuschri st +cgi ar +ðŁĹ ŀ +w fd +s gc +kow loon +deloit te +the cat +sval bard +rickand morty +nun o +jun cker +implic it +har sha +ghou ls +end rick +bow es +te y +te ign +rin ks +o ce +metaph or +ish ly +intern al +in clin +day out +sil age +par ang +ny n +murfrees boro +k org +cin der +ba ji +tell in +mic rone +kang ana +id x +ðŁĴľðŁĴľ ðŁĴľðŁĴľ +te ca +sy co +madhyapra desh +expe dia +clo g +av ol +u mar +the chive +se izing +pri v +pav ili +p els +lin de +jam al +de walt +alle ys +ì§ Ģ +summer reading +orphe um +gla sto +europe antour +ci gs +beat riz +tonyabbott mhr +par le +kil au +hit ched +bere a +be holder +bas sne +arro z +í Ħ +syn ths +nfl combine +new stv +; ;) +y ams +whom ever +tor un +they re +south west +shru g +scot landis +pione ered +haul er +h ss +dav it +be fri +tro on +se pang +lo z +ea p +ay len +wri t +toile tries +lac ey +engineer ing +ch vr +wan ted +w am +silverstone uk +si al +sha sh +pe le +pan to +gor dy +g st +âŀ ¡ +upper deck +nat chez +mach e +collec tables +b cuz +ðŁIJ ij +tin ubu +snow shoe +moham ed +mal divi +mal den +thankful for +royal ascot +private equity +nine teenth +hus sey +bo ggs +zin ski +ter pen +son os +radio therapy +quic kie +pro vement +north ward +inns bruck +flash mob +expe di +boyco tting +rd guk +o ole +mar chers +mand u +griev ances +diss on +call ers +ble mi +bill gates +suni versity +sen warren +ru d +ros common +palad ins +monday thoughts +beer day +al c +n tr +kag ura +jiang su +flow ery +conce ding +che red +an ay +ĵ ãģ +âĢĵ âĢĵ +resto re +ny ard +muzaf far +ine za +esp anto +cannabis community +smithson ian +s london +regul ates +making adifference +at v +à¹ĢภĽ +ross ini +re settlement +ini k +in the +fo t +color ways +ak en +ur qu +under pants +n ns +medic ine +l lo +ik an +g acy +em body +ef ter +ban j +ar f +âĿĦï¸ıâĿĦï¸ı âĿĦï¸ı +ts r +mr ss +fi es +cor az +ago go +then orth +resolu te +nas l +magaz in +gr rr +et weets +business insider +bench marking +montmar tre +home stuck +he it +e chin +ac ai +-- " +u chicago +ph ra +m clen +cumple años +clo sing +bald win +par kes +orni tho +mi on +art sed +outer wear +farm to +endu res +dor king +ai i +per severe +o ar +knight ley +ho hoho +democr at +bernabe u +ap la +yam an +veterin arian +tin the +sonsof anarchy +shad er +hawaii an +ge tz +am bb +ðŁĮ ħ +u mu +that smy +one day +door n +cr aters +cn ni +ast ate +council member +back pain +ad r +âķIJ âķIJ +stra ined +sas su +globe andmail +geta fe +fri vol +fanta stical +allen de +thezon ecast +shipp uden +saras wati +rou ge +pu pper +mo dena +gerard way +gaz elle +du sse +dog friendly +ðŁ¤ĺ ðŁı¾ +t fi +sam mi +row dy +kinab alu +jagann ath +u ff +so da +sg d +mum taz +glo bo +faf sa +b ced +visi bly +miner al +di ra +bees wax +shail ene +presti ge +dissec ting +bm wi +b ich +an tic +wil hel +tax scam +son tour +or am +north field +k tv +her ds +fu jitsu +es v +et ch +bro ms +borough s +anchor man +smash ing +j mc +fra ppe +ct l +ðŁĵ ī +un ger +screen er +nbc nightlynews +lay zhang +hugh jackman +devo tee +defici ent +charli ze +ðŁĶ¥ @ +water crisis +vish nu +t fm +sof war +mau ve +ken ji +datemy family +cer u +bear cat +su zi +o ad +leit rim +iso de +cre scen +astro physics +wc co +w pt +tou lon +to pple +pu c +pe ay +ninj ago +manufac tures +dissi dent +col vin +u ist +tra i +t ola +swan k +splat ter +robbie williams +nu ps +mcn ulty +lacro ix +bati m +ðŁij § +ðŁİµ ðŁİ¶ +th elife +gent ina +cl at +( ), +you gov +yehundinon kibaathai +us g +re branded +mar mite +ludic rous +kc co +j ft +hur acan +huck leberry +at ingly +ðŁĩ¨ðŁĩ ´ +the storm +rec tomy +monstro sity +memor ies +kitt en +en a +dough erty +bt b +are g +ðŁĺį ðŁĺĬ +sla v +moon lit +mad d +dis advantage +comm enter +bram ble +bor rowers +bel ow +staun ton +sali va +kwi k +ken na +fal si +edge water +de ur +d souza +d me +contradic tion +stan ning +sch loss +sc tv +i aff +golden eye +archi e +ali khan +al brecht +aa as +slo west +in scriptions +girl guiding +chu ca +ðŁIJĿ ðŁIJĿ +âĢĶ # +styli zed +sam ford +ph nom +long view +jy j +bu ford +as n +wall y +li en +decad ence +dan e +bar uch +ry land +halli well +g tb +g out +fou l +f ye +cur lew +con golese +bhat tachar +ðŁĺĤ ðŁijı +sla ve +gli mmer +d alla +cy l +un fccc +tram way +ti gre +sin dhi +ob ac +ben et +beef y +strat os +pregn ancies +plan te +el ux +chennai ipl +ภ« +wa hab +out kast +manit ou +lu au +io u +hock ney +contre ras +baby sit +uh cougar +mobile app +maria sharapova +expect ant +consoli date +ton al +steep ed +kaz i +ih ate +fl acco +e show +catholic ism +ar adio +sack ville +mackin ac +i was +english man +cantbe broken +bu ble +united fc +ron de +pack er +bristol city +ãĢ ° +sol s +schur ch +sak shi +meto pera +me ang +i vor +ech l +ba an +ðŁĴŀ ðŁĴŀðŁĴŀ +vin ho +sm ttt +prince sa +por trush +aa o += ] +profici ent +lick ed +ig nis +head lined +fast ball +do ss +Ð ¶ +squir rel +pro c +plan ned +izom bie +itali angp +hu may +h town +co bh +bar un +run g +res ford +mo jo +adop tions +t la +shar d +pri vacy +ko in +don na +ceph alo +cent aur +bau d +ar cel +apol i +ak en +ç ´ +wit chy +tic ats +spo x +real food +mac e +fi fe +be ch +ðŁij Ļ +ÙĪ د +seac rest +renfre w +pi xi +neu f +entertain ers +co oney +cll rs +bo ing +well and +the office +nad ine +inspired by +dilig ent +devi ant +r pf +lin ens +k news +instap lace +em itting +dur and +uuuu u +strö m +sp hilly +o dia +nd tv +jab ba +ing t +d ceu +co et +cl v +pon ders +parliament arians +kei sha +k illi +jae hyun +c ph +an un +sho el +ly tham +em bi +break out +anim a +faith fully +cu pe +ceram ic +am way +wasser man +scotlandis now +me tics +instaplace app +g mc +ðŁ¦ ī +water proofing +resi sts +par quet +go c +brave heart +qu arium +pom ade +pit stop +ng k +for g +bankholiday weekend +resi dent +kol o +hel ic +go lov +found ational +e ine +de tain +cont end +bb f +ðŁĺĬðŁĺĬðŁĺĬðŁĺĬ ðŁĺĬðŁĺĬðŁĺĬðŁĺĬ +ãĥ Ĭ +her ts +ðŁIJ ¦ +visi oning +sk storm +la vi +ran ia +go h +fas cia +dunyan ews +chidam baram +bra zz +bain bridge +sp ook +sno d +silver stein +sav ile +plat onic +msk ajal +geta ways +fri sky +explor atory +u ll +ton ia +sy r +stru ts +spl an +rec tion +oll yofficial +n elly +malm ö +confe rence +clar o +bi bby +topp scards +ther ight +mat ricul +kajol atun +gl enda +church yard +bt m +æ ´ +utah jazz +p sat +high line +disu sed +al mo +af t +âĻ ¬ +troubad our +mull an +lie ge +k oma +he wson +with a +vent a +t ched +show me +leg ali +gift card +boy band +al las +vir at +spacef light +ou ture +lie berman +li kel +i sha +head way +ear hart +bhar ata +å £ +th as +rec ce +pa yo +gab or +foun der +cold war +car pet +ag ha +park nyc +mis a +li mass +len g +infin it +indeb ted +deep state +bloom sbury +ðŁĸ ĸ +yay yy +toy story +sb g +n wa +lar issa +gn ar +resi ding +penetr ate +fr itt +ex tro +clo ak +sens ations +rang ers +pur ring +mskajal aggarwal +hi ram +en vious +de stru +and heri +agron omy +ä¸ Ń +to va +lock heed +face timing +bon ny +ab user +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ +ro derick +out perform +ne pa +ir in +explain afil +buffalo es +ðŁ¥ ģ +uno dc +trespas sing +to h +rock music +dolce amore +on eness +lob sters +in tr +hind sight +enlar ged +brit ons +ago d +va id +tt l +there after +so ir +nat ale +ecor ps +chipot le +yo k +ga h +al va +pic s +ow sky +cin ta +anec dote +ðŁĺĽ ðŁĺĽ +rein carnation +ple aser +ni q +mackin non +can ber +wa he +ur bex +ud inese +selfies unday +sa hel +extrac ting +champion ship +buzz feed +/ £ +yasi r +smi thing +rela pse +libr i +inclu sion +fab ian +!! ... +ðŁĻıðŁı» ðŁĻıðŁı» +ne der +gu sh +blen heim +âĦ ĸ +yugi oh +seh gal +po dol +je thro +jama al +g oun +co zy +bri gid +v lc +roc keting +pin ch +maggi ore +ken a +k ony +wi ke +whistle podu +stupi dly +liv able +lion heart +lak me +jong dae +ge en +electr ics +cyani de +arc ades +to kas +mono poli +mar key +dimit ri +dale ks +convers ational +aj c +tes bury +ste pper +ri as +repl enish +ker ou +imel da +ig eria +ðŁĶ ½ +une qual +sad dles +reci fe +ms f +mal practice +io dine +dar ko +stu esday +ho dge +dc f +wal kin +spe op +pe ering +okc thunder +jac o +ha snt +ge ico +bree zes +berea vement +ðŁķ · +lu saka +irish rugby +el aw +tyr rell +thunder up +inter acted +goooo od +far oe +ver mouth +tren ch +l rc +east leigh +copernic us +ber dy +special ty +prostitu te +po ftheday +li ster +gry ffin +ed f +bra ith +boo kies +bid den +ber nier +ac ause +was s +to ku +rit u +internet marketing +electr ons +akan shag +tv r +sni ps +pac s +mu ss +ho da +fire power +drum sticks +childhood cancer +scri bbles +off i +vo gel +t gi +star s +spo st +se atur +koo tenay +gy al +ban ff +ap ush +ad mk +sp op +ri stor +nit ish +mel lo +kat erin +goo g +aw ol +ai ley +ãĤ ļ +ภĦ +wal kies +te j +jar red +g ley +" !!! +ø r +sh andy +parake et +me tering +man ju +lets movie +j ls +j aga +en shr +of s +honey bee +fire truck +ad f +an try +tom cat +p russia +li sal +ew es +du d +boy d +analy zes +real life +head teacher +a vie +yu le +wal lo +sha g +m signite +deli rious +cool in +apo s +w illa +villa inous +hac er +dece ived +cv n +wind ham +ta q +mplo tbadly +mee han +man ner +macgy ver +de pau +beer co +arch it +over looks +k ard +g litters +embe zz +dan k +atro city +ë¸ Ķë +è ĭ +tam u +sul timate +south florida +nendor oid +ge yser +di z +al ston +sul fate +ri alto +revol ve +o is +ing ress +goo od +ask mrn +waste ful +ver mic +un drafted +so as +ici c +god dess +gill an +carpent ers +stan hope +scul ture +r sr +cr ème +bri en +ðŁĶ ĵ +ob v +hi karu +flo rence +flat ter +contin ual +art vs +ðŁĩŃ ðŁĩ +z ale +ww d +ut ely +tre lief +tip ster +sto tt +st pete +muss olini +mc b +jun o +an aco +pra deep +ect ady +ãĥ ı +ঠ¨ +z uk +twe aked +hu huhu +chatur bate +bo j +a jax +subhan allah +snar ky +rw b +rever b +hu mes +gilmore girls +cel ine +âĸ ¼ +rak sha +m to +intelli gen +fla pol +co pia +can isi +bible verse +tre vino +trac i +thom e +pharmac ies +gram mar +en sign +w miwx +swach h +ra gged +par ian +p ff +gro sso +adi eu +âĢĭ ' +zor ro +swin ney +osc eola +nigeri adecides +n ma +mm d +insomni ac +in design +harley quinn +full time +er z +cer sei +male k +mac aw +cling y +beero clock +band han +ðŁĻĪ ðŁĻĪðŁĻĪ +gulfstream park +fic he +entro py +ci led +amaz e +me us +ie v +emb lem +em ate +donal dj +ban yan +akanshag autam +save america +ney mar +missi ble +leav en +capac itor +bod hi +slyther in +explainafil mplotbadly +em re +e tru +cohe rence +schen ectady +positi onal +nai arind +lao gha +jay z +bro ward +bat tista +at te +apple event +al g +te ary +replic as +polari zation +mor ton +inste iger +cotsw old +brook line +amphi bian +who res +parti san +monogram med +low land +lewi ston +ger o +cam in +boy gan +wear in +unex plained +m ke +her tford +hat cher +matu rely +i be +ac tin +wol ves +d fa +chromato graphy +arri va +sham rock +role model +legendsof tomorrow +prat chett +iom tt +franc a +makas sar +eliza bethan +cav an +ir re +ill ini +gre sham +g é +ero yale +ant m +ðŁ¤· ðŁı½âĢįâĻĢï¸ı +u vm +network rail +lo tz +la ston +inter com +conduc tors +ai p +ðŁĺį ðŁĺĭ +thin ly +mu te +jun myeon +epo ch +black cat +bir thing +tar pon +stil ton +robert downeyjr +qu int +philosop hi +kar gil +fu elling +corp gov +aw scloud +---- -> +park run +double tree +co ted +ari vera +the beat +sapphire now +pro tools +par chment +ondthat cantbebroken +ni mble +ho ts +han di +thir teenth +pow ders +musli mban +lor is +hi paa +domin ica +caramel ised +pan de +miami herald +medic are +jam est +invasi ve +half marathon +configur ations +box ingh +aug mentation +ag c +ðŁijĬ ðŁı¾ +vil leg +un natural +shoul der +roo kie +ration ale +norther ly +motor cade +mi shap +maker faire +ki ernan +ken more +jo em +instru mentation +ge stion +di ggy +cell phones +ör de +var ad +toot sie +r mt +parkin son +pakistan zindabad +ne ct +horn sup +gurru choudhary +fiel ds +cri m +blo ated +vec tors +t pu +s that +r pg +pu d +kl out +kau litz +pa sto +mar ous +man ge +kor ra +ko d +gor ka +d tc +bed lam +b anco +v cr +shakespeare an +sch atz +der ic +baby shower +mer cat +factor ing +£ £ +gi sh +dri zzled +chvr ches +cha im +! ". +tex tile +mo ssy +as v +vi enne +un suspecting +qu b +gt fc +cor tana +bal inese +v rc +un locks +prox ies +dal er +al issa +ìĿ ¸ +spon dy +s vegas +my stics +mis c +may wards +marci ano +escon dido +accumul ate +w acker +til da +sp ines +na ina +mit b +h cl +coach ing +bra v +tran mere +suicidepre vention +story lines +spin y +reci eve +kri sty +don key +cur a +comp ag +blo m +articho kes +wee b +wal luk +ssi mo +radi shes +mechan ic +aj it +ag olf +ðŁĺĢ ðŁĺĢ +wyn onna +pax ex +kon o +âĻ¦ ï¸ı +trip tych +southampton fc +old ham +god z +gibson guitar +e ka +diol ch +cu ddy +che ws +æ ı +watch list +u so +pred snhl +pad ua +of an +never forgotten +kab hi +italian food +har mac +cra g +clo gged +b ong +pay ed +norman reedus +cre ature +wc u +valu ables +run aways +cooper ating +co pious +si der +senjohn mccain +da il +uof t +under dogs +speed ster +sk ye +royal academy +prieste ss +j illian +h sn +am ey +that ched +kau l +il ver +dwell ings +dur ante +ap ache +wo burn +tanz anian +soci able +saatchi art +s wor +jo ann +ji ji +can ister +* __ +wing man +vit ale +osmo sis +organ ically +nom e +hand cuffs +ffe st +eup he +creep ers +* ! +won woo +spla shing +sear th +roll o +lud hi +head ley +fleetwood mac +differenti ated +de thr +brex ite +al drin +zi pp +sno b +sf n +hum mel +bu ona +af ox +wh ok +pilo ting +ml i +j cp +gru mps +danc er +c mg +addic ting +ad dition +hiberni an +g ham +b ame +ðŁį ¼ +sta tham +smugg ler +n tu +morning show +j ö +in love +zap ata +souther ly +shir i +obam as +letch worth +he yy +g bi +et or +ba iting +abund antly +rock port +ph ics +metall ur +dor nan +t lt +o den +lets football +kay leigh +ir ate +goog lec +fa ked +dwell er +du tt +ber k +robb o +lot ine +i yc +hey man +âĪ Ģ +me cum +jam mu +defence man +carth age +moham ad +ill an +d hol +brown lee +ven us +rapp ort +mck el +head wear +got ta +âĺħâĺħ âĺħ +we es +ven kai +n gr +m twx +jai hind +berk ley +ax eem +ye ager +w pial +matern al +gre ys +for love +daily mail +cincin nati +chau tau +awarri or +zü rich +ww os +stream er +peri shed +pearl harbor +pe ggy +online shop +me cfs +vish was +sre es +po re +ichi go +ar gento +wa ive +proble m +pat ched +nowh iring +k pk +fc porto +do on +whopp er +tri bal +sier ran +por te +bassne ctar +af ol +a sted +wit ney +inst alike +del ine +cil ic +bud light +better ment +ab lu +pe corino +and ria +ìĽĮë ĦĪ +ver t +uni er +tic elli +re max +main street +kick ball +coinci de +book lover +aver t +south america +sav ille +oriz ons +kun le +kul karni +ger ty +for better +eil ge +dimin ish +mcen roe +mb poli +kn x +im possi +grat u +doppelg än +cre atine +be safe +ðŁĴķ ðŁĺĺ +sweet water +reg ine +py aar +meh di +explan atory +dr ill +deco ded +danger ous +ci ab +uk business +oo b +mit re +metr on +kop i +gros jean +franch ising +cow ley +van qui +theologi an +re ho +publ icist +pistachi os +local news +ge ck +dor ks +ch h +ay res +river side +in continence +fear some +aster oids +men is +etsy retwt +buck ner +tr ing +tom ar +si ste +aspir in +âĽĦ ï¸ı +reclin er +product design +chim neys +alu it +å º +ti ously +r wd +pe m +nickel back +mclaren f +ly ra +inv ents +cu rio +ciren cester +and gold +� � +ãĢ IJ +ठ¹ +time sup +seag ate +purdu e +pul ley +pepper dine +ali ya +à³ ģ +z hi +ting le +po ked +hunt ley +go dogs +fa kel +" """ +take your +re writing +celeri ac +appro vals +ve rer +toom uch +mis demeanor +ly mp +gar ts +fer ia +culmin ation +bayone tta +y f +moo res +je ju +ef i +conserv ative +base camp +ab aseball +tom ar +sensi bility +r lc +pap as +jcol enc +ware houses +v ashi +transp o +spho to +poo ped +glit ches +ren i +pre et +lma ooooo +hu f +st inging +sne aked +re plays +r hi +inter racial +bu cking +tax ing +sig nature +party time +kid die +inverte brates +impro bable +gastro intestinal +to ki +tang les +om ap +loo d +gold blum +gle be +ec enter +ear ners +si belius +po los +over taken +daysof christmas +cross rail +cob webs +bir la +bil let +z x +vis i +un folded +ste ele +schwe insteiger +rak hi +mar lo +fe tty +cn u +war nock +spoke s +na shua +h ci +way lon +ph t +jan es +im al +ðŁĮŀ ðŁĮŀ +over grown +monday blues +l yo +ho yas +deter rent +crimin ology +anc on +work hard +wel don +nag y +mac omb +lin ux +dap at +ca shed +be is +th b +premi ere +ny sc +jan son +institu tion +ban ish +b hawan +al pradesh +ve lez +spo oner +gret chen +fin edining +dwar ves +drive safe +dishon ored +as so +ver anda +ty rion +stri ppers +nxt takeover +electri fied +sd b +pal moil +n sync +k ith +hart nett +gil berto +fl ate +car in +under grads +tun ga +tot w +s be +mei ji +leg ere +l lyn +head board +fru c +birken head +algon quin +wee ding +she ikh +posse sses +op ent +love able +eso teric +bigo ted +ana heim +wel le +vigor ous +tab la +sp el +par nas +land line +g po +brush less +ble au +au ger +launce ston +hr man +fav a +employee engagement +sus ana +of erta +dari us +crown e +ðŁĵ ½ +w top +te thered +man et +gu te +grim mie +forza juve +cho ppers +bere aved +andhra pradesh +al bor +weather ing +ran gi +out chea +mi zation +fro me +ed vard +bat aan +vic ar +trump jr +sea shepherd +lo cos +dun king +crypto graphy +mat ron +bo ggling +beck enham +wi x +pat ernal +hun tress +herd smen +bi sping +av ati +ac d +un married +turt leneck +tar un +ste ez +se in +lockheed martin +it takes +insur gents +botan ical +bis mil +un speak +o gre +bolson aro +bethe best +sa heb +cathedr als +captain cy +âĨ ij +t cd +se villa +ðŁĺĪ ðŁĺĪðŁĺĪ +w pi +scra pping +prote st +pi dou +mel ba +houston texans +bur dens +app ly +ìĸ ´ +typho on +r ink +j bj +flip grid +com te +carriage way +radi op +pic nics +on as +ham burgers +go red +drin king +bloom ed +sho stak +mul tim +gh en +b ss +afil m +âŃ ķï¸ı +z hi +the projec +rel ati +pr at +dj py +carl ile +sy mp +sad d +oral health +mus i +ka vi +hetero sexual +abo x +ðŁļ « +vis on +tele communication +r nr +pu la +na ir +mccrack en +hazel nuts +gre ig +flamboy ant +fiver r +aussi eed +swa c +sel va +oth ello +or ville +is ch +ing es +family history +course work +chal ice +cat ed +bli sters +xy lo +the p +sk ennedy +school girl +ki v +k nick +fu me +bri gg +bk lyn +uw madison +stumb les +stu pend +stag nant +lax is +ing en +cam corder +tzu yu +sat chat +prohi bit +heis enberg +den iz +aad mi +theli on +sty lin +reinst ated +he ur +ffici el +confi denti +ca z +âĻ¥ï¸ıâĻ¥ï¸ı âĻ¥ï¸ı +te cum +sus anne +n anna +la guardia +king sman +ing in +gil ber +f eng +ephe mera +enf ants +chom sky +chi k +bre anna +uk r +sar an +r ting +pa checo +del and +opti mise +kee gan +caliph ate +ari z +z it +west side +ul la +to it +shif ters +sh river +pan i +must fall +eth ically +br no +in cess +hu sh +happine ss +far qu +fai rest +f ka +eli m +ecumen ical +ec ity +dre dging +dc fcfans +am ichi +å Į +sp ak +sati sh +pu cks +match making +go enka +ea sement +ci der +c mu +b z +aqu as +ðŁĶ Ĩ +{ # +smoke house +separati sts +saf fron +ma at +l loren +iro quo +cor ns +cassi dy +ah met +z hou +ck man +yor i +social ite +pro mise +plo tted +metic ulous +fluore scence +se wing +sa it +hunting don +gi gat +cascad ing +atl hawks +ue fa +nor i +is son +iac aucus +gir a +der ail +bad boy +way to +too wo +se u +rath ore +pra shant +ol ay +oce ana +le vin +in dent +heavy weights +guer illa +cor o +alay sia +x ing +uc ous +storm zy +sky light +sen dai +sch s +pa il +mac miller +leo dicaprio +hell fire +g dansk +fa han +aa w +south well +ram bler +persi sts +brut alist +an men +tari k +se oul +popsic les +man sour +daes ung +carra sco +sas a +mat tie +maro on +lever aged +kom odo +h sc +fac toftheday +cur ses +aff in +ðŁĮ ĭ +à ² +yvon ne +w mur +meadow lands +gra vel +fa hey +cze chia +tupper ware +mar awi +lac azette +hero ics +far i +del aware +pe cial +pancre atic +out and +la zz +e vel +toom ey +tam pere +squid ward +sop a +shar man +er nest +black pool +ao ife +air i +y eri +sig mar +ni han +liz ard +hom ed +dry den +chu b +blac ke +aldu blo +re iss +olympi acos +love story +lent icular +lan gue +jcc aylen +i hc +ban ked +trum pets +sx m +ob inson +ma homes +k nes +dissemin ation +pe derson +or bs +ner dist +lar ies +fon te +expedition ary +ex a +dam sel +chi ang +ab on +âľĬ ðŁı½ +ta v +sil o +plan ing +ny wx +m ú +data sets +c rick +ye ay +vol e +slin ky +m srp +c pp +bex ley +ve dra +open rp +mysti que +micro phones +dra ghi +atri al +?? ) +yu ki +xenob lade +words worth +u sted +influencer marketing +ds g +as ingh +roz ay +rail ings +m ks +kan an +intimi date +bat ted +barunsob ti +asho ka +anne curtissmith +so ares +mercen ary +lg f +eyel low +embar ks +ssi ans +lung cancer +in fini +ec c +brun son +bloom er +ar can +walla by +tr ici +showr unner +r dg +net to +mi zes +erup t +dur ban +baby y +tric ycle +sam smith +pre z +pe pe +gal low +g day +dist illing +ðŁijĮ ðŁijį +ðŁIJ ļ +tw ic +see e +job seekers +why ilove +poster ior +li rr +freed man +e ge +do xy +cur y +ang al +suzu ki +mc qu +ct w +antic hri +stu f +spring er +pow ell +om aha +marri ed +go home +g vt +fox trot +uc p +ow er +fit ch +con nie +wander lust +sym bio +sunny day +mic ah +haye k +cor mac +boj angles +é es +right fully +on a +nor ah +k cc +hillary for +h mp +bru in +ath lone +action news +ì¤ Ģ +se hen +pu lau +priest ley +posthum ous +pe do +nai doc +ling ard +le ben +jum per +... - +pet tic +ob serves +nuest ros +now smoking +night cap +mc p +lan downers +k ta +eng ad +ele x +alle z +w z +un ai +md f +jard ine +al ax +wh aley +tec no +son dheim +junky ard +insu rer +e hl +de vote +car ra +ca id +ca diz +ar thi +us ch +tyour self +per n +natgeo travel +ic ism +amo g +scuderi af +que m +pow wow +kur tz +head bands +gla zer +getwell soon +gam enight +euthan asia +catap ul +asy mp +sol ano +red hill +pu ffed +ap ura +allstar game +ye sp +sal war +nag asaki +medieval twitter +loo sen +inst ax +i sti +go si +contempl ative +chanc ell +appo inte +tal ley +corn well +war dens +v ate +sori ano +rad ley +pu a +m cro +inktober day +ero th +constip ation +ðŁ¤Ļ ðŁı» +wil mer +vani er +val lado +tic ons +reconc ile +mort als +li bby +h dp +yu gyeom +ym ac +un bound +sch ach +sb sb +plastic free +napo le +mum mi +ligh test +lam ent +de vised +un intentional +sheskinda hot +saint s +place making +paste uri +ke ir +kab oom +in san +dool ittle +cassi us +te va +shutt le +row dies +dra k +bren na +blu m +ki ii +ker ton +deriv ative +bu ts +bar zani +ador bs +ad ami +zu cker +time isnow +su lli +sat i +sabar imala +min na +- >: +ðŁ¤ ķ +stu r +princer oyce +marvel studios +itali ana +hahahah ha +gg ard +flu tes +xx xxxx +pin ski +n pd +must see +immort als +gar b +fi estas +dor a +bi bles +angu ish +sco trail +pa se +lazi z +ivan ovic +chronic illness +ar me +zu lu +tech stars +ro cher +ric ar +plo y +glar ing +des antis +cancel ing +ab ha +; * +. "# +ðŁĺĬ @ +y ch +vas i +skybet champ +sha una +ke ty +goggle box +balcon ies +ag as +un tamed +t ce +ne i +montgom ery +min ot +lombar dia +brain washed +bo tha +block cha +âĺºï¸ı ðŁĴķ +te te +star tin +ric hs +new sma +fab les +fa rer +de feat +cy an +compan ion +bio diesel +ben g +ðŁĴ¦ ðŁĴ¦ðŁĴ¦ +mail man +lin cs +fc bayer +bla sters +baccal aureate +attle boro +vic eg +limass ol +isak hi +himy m +franca is +da real +bush craft +ann ou +z aky +vie ja +kirk by +ca ste +a en +zd net +fac et +ali x +è ° +raj nathsingh +n ini +mc cloud +il ano +du ous +dil jit +dal lah +cycl ical +cur acao +âĢĭ ! +wr k +west boro +rat o +r z +ne gra +land mark +john stown +interfer ing +hann an +duc ated +zin ke +tamar ind +sub title +ro eth +le at +kar ls +ka hoot +insta art +hi aleah +embi id +drum line +assess or +anore xia +~~ ~~ +wemb ley +ra jap +ouni versity +jal al +copy cat +ak as +r jd +u hhhh +re flux +mor mon +mb f +lucin da +j drf +fon taine +defend the +crack ling +ë Ł +uk housing +tb m +shi vers +pri m +per in +in for +h ba +furry art +cav ani +ale se +acce ssion +zin ta +wr ongs +worldre fugee +tam bour +reb ates +ple ases +paramilit ary +ool ong +l mc +facilit ates +espan yol +car ling +k of +hu is +brew er +a hahahaha +will son +sh ure +sain z +play hard +lu p +lau b +gour d +dun ked +vi vino +sw restling +pi en +o ks +l leg +euv res +do zier +canadi en +blood stock +bac all +ठħ +vol ver +e sti +do sto +car b +u pping +say fie +ro she +m ru +emo tional +dit alia +book shelves +ab oo +s lane +on tem +mon drian +laogha ire +ds f +conflic ted +charlotte town +ðŁĵĬ : +whites nake +uni fying +tsi pras +smar ter +fron ti +f dic +emb ers +dece iving +bel air +ac kered +ur su +taste ful +sti let +pas ok +k sc +inv ari +in gof +dar tist +clin ician +chef life +bla b +avi ary +ðŁĸ IJ +stream ing +out ings +natural beauty +gil roy +du ets +animal welfare +ad air +sounder sfc +sashab ank +ro lando +qu aker +me ses +ka jal +jesuis charlie +brucele e +u albany +re ts +install ers +ilo cos +fin ca +ar kan +ðŁĺ¡ ðŁĺ¡ +shru ti +sha qu +leak age +cy cla +bre aded +ander sson +th reading +polaro ids +korean updates +hard ball +flock bn +cw g +sauer kraut +os we +ine w +du ped +à¸ŃภĻ +un reasonable +tour billon +tat su +pab st +optimis ation +in significant +il ah +cn tr +!!!! " +sl f +ra pap +o dor +masse ffect +lac lippers +ur sing +reco very +ne ff +mon do +gymna sts +fin negan +ct m +bro wsers +w ert +pag et +ou d +ong kir +nas sar +mind ful +k si +bak i +] ) +ðŁĺľ ðŁĺľðŁĺľ +non chal +men e +ly cra +bay view +barn um +âĢ¼ï¸ı âĢ¼ï¸ıâĢ¼ï¸ı +virtu ous +tre es +pal ak +mid century +kel le +jof frey +ho ppers +tor os +se pia +pay itforward +minim ise +fresh en +el u +d pp +ce us +å° ij +yu ta +ton ian +the king +te jas +rand all +me up +gut sy +dor ms +ch ore +½ ĺ +tsu i +thu mping +s va +girl scouts +bla h +ðŁĺĤ ðŁĴĢ +stor yof +ske g +san am +kumb h +ic ts +go lem +coffee house +be more +tweet fleet +tol stoy +paul sen +film noir +deduc tions +bear ded +be len +be ke +ÙĬ ÙĪ +win win +opau lo +da is +amo eba +sun y +london derry +gil ly +davi dj +craf ter +water colours +triple h +super tuesday +papy rus +foo di +demil ov +christma stime +ber tie +v ities +to il +sky lark +mi pim +foodand wine +fl c +usa in +te ers +rom com +morph ed +ho cke +hem lock +fire storm +comp te +bo dle +vis xx +scor ched +re ams +jones boro +choo ps +cheeri os +ðŁĨ Ļ +â ħ +n ity +miri am +me trou +man de +indie comics +bree ze +boom in +bol sho +android dev +aero dynamic +sm en +kri spy +in nu +im ps +ham id +fas a +emper ors +eli sha +ëĶ Ķ +in q +i zzy +bre sson +ãĥ³ãĥ Ģ +syndic ated +sashabank swwe +ou a +kah ne +june teenth +go tigers +diljit dosanjh +can tal +ball a +asc ended +v ga +t ass +made it +hennes sey +distri butes +top man +ti ago +the game +nav as +mur at +mtvlak pop +hol ley +bau bles +am rita +/ ? +ìĤ ¬ +ëĭ ¤ +âĿ Ģ +k ä +cel lo +bio fuel +ashu tosh +adop ter +] ] +ðŁĺİ ðŁijį +sw y +pete y +home ownership +go van +ecol lege +bres lin +bbc strictly +aga i +yum i +wil lett +w isn +tr visxx +solsk jaer +so vie +sle uth +ot ley +m ame +honey bees +bal dy +ðŁĺİ # +un na +ti more +the ti +progre ss +k was +jama at +h fx +crime fiction +c ado +bad d +anu bis +ê te +nom a +h ps +vintage jewelry +to fino +stra vinsky +go ffin +gh q +ef fe +ch acha +cam eos +n ci +lim m +invic ta +fle urs +en cu +brilli ant +ban sal +alan is +wan ee +spr inting +long mont +klar oline +feed ing +ðŁ¥° ðŁ¥° +un sustainable +tw of +luci us +ld sconf +hubb a +fri o +de hra +ur in +tre me +n mc +com es +bun bury +b cu +ad ina +£ ) +za b +ske ptic +shop rite +sand o +re shaping +magne to +kel ley +ilove you +cri pple +contain er +wham my +wfa aweather +techno logist +hl man +dynam os +cru ze +ble s +ati d +ork shire +tt an +poyn ter +pac a +mar mo +hyun dai +wi eld +sabrin aann +quick books +petro chemical +nol an +les den +mc lisse +lit chfield +gur ls +done gal +debut ant +com is +am ul +vigne sh +shi ge +re portage +pru dence +poten cy +pan hellenic +me ms +kling on +impre za +eri es +cash flow +winne bago +uter ine +ss k +do bby +canv ases +vau deville +pradhan bjp +myth os +medi ocr +mattb omer +g ns +final ised +election night +d pradhanbjp +coch ran +calm ly +amag azine +ç§ ģ +wareness month +tar ga +ta hun +sche tta +mu tter +liber t +hal low +de ities +chrysan the +c ms +am ents +sabrinaann lynn +ri ghted +r wot +ko i +ha feez +g dt +ul ls +u vu +ol adi +metro parks +it ness +extingui shed +simpli stic +pl ath +menag erie +kcr w +cam bi +ðŁĺį ðŁĴĸ +worldrefugee day +kel vin +i message +emo s +ah ca +sk illing +share this +ov sky +mcin nes +inter city +innov ates +gor die +g ons +fusil iers +duques ne +artofli ving +advance d +ph ane +oli va +met is +mal loy +jer rys +fu ming +follow for +f bla +eyel id +ak enya +ac ara +yu va +x code +wyn wood +tor te +te gan +superint end +sh inde +ly ne +hay market +england cricket +at z +ðŁĴľ ðŁĴĻ +speaker pelosi +pon zi +ole miss +men ter +keo gh +do td +bun t +bdn mb +are y +mathemat icians +fi ance +ce cili +spot the +pilo ted +escor ting +av ali +ale e +ðŁ¤Ĺ ðŁ¤ĹðŁ¤Ĺ +weapon ry +trapp ist +tho l +sur rog +reci ationday +plain field +phi les +os as +mi kayla +ma ham +favour ite +bl ane +ba sto +auer bach +vit esse +one republic +di ma +caer nar +we trust +sa jid +peek aboo +mam moth +hatch ery +h re +grand dad +dail ym +correc ts +californi ans +ë ĮĢ +ster ile +land sat +in fielder +imperson ating +hypothe tical +ठĨ +ul ta +the xfactor +pas sat +nw r +he ss +atlanta falcons +ah madi +âĹ Ħ +tu lip +ti ra +this week +spar tak +ot f +l enews +at ical +newsp erth +ha ge +car box +Å « +t pb +succes strain +sel man +sal umni +mor te +mor ro +lu k +elo ise +cust serv +ðŁĸ ¼ +ri bo +f tisland +do ble +cu ma +clinical trials +bla ding +bergam ot +ti zed +si v +day ever +by o +avat ars +alle giant +ðŁĺį ðŁĺŃ +yar ra +welcome tothe +um no +tur ing +tinker bell +ter ton +swo oning +regre tting +pin nac +pat rol +iv ar +im manuel +ig lia +bar ds +vit torio +rac in +park side +ktr trs +gu el +ðŁ¤ ¯ +whitt ington +sanje ev +s festival +o rel +miami open +jan es +intensi fied +ig ar +derek theweather +barbe que +ðŁ¤ IJ +âĢ İ +yun us +ny post +abudha bigp +semin oles +ground work +fu ss +eamon n +du ol +col ympics +chi an +bo oms +ani ac +~ * +y omi +thankful thursday +tech tuesday +la ur +iphone x +ha aa +fla va +e ku +d all +cor tes +care w +blun ts +ðŁļ ¶ +îĦ Ĩ +val tter +un written +top sy +som al +re generate +man chu +hun nam +hoo ts +e com +ðŁIJ Ń +Ú Ĩ +gn or +gl ens +bartol o +avi d +antibio tic +anc i +star key +ru he +practic al +penny wise +o cular +jim marous +calvin harris +ane mon +ãģ Ĥ +tat ts +suff er +sp ics +on kyo +o jai +greg orio +big elow +ath i +an ta +ðŁĩ²ðŁĩ ½ +zo olander +sha adi +kho v +di zzy +ann um +wo m +th orium +summari zes +scam paign +kc tv +ju mb +bit z +si al +r mit +promp ting +f cm +by me +bron ies +wj z +ti ya +ri az +re sign +provinci als +go vs +brock ton +ãĤ ¬ +z orn +ola the +leh mann +juan ita +g dr +dis bur +tab ling +pi azz +per plex +milit ar +gin seng +fred di +fire birds +f ags +dig g +aug sburg +ac as +. & +ðŁ¤ Ł +stabili zer +sa thy +re gt +ni ker +mode sty +fo garty +cap p +bal oo +bal ac +ar oy +al corn +agu sta +un zi +stoo pid +pue blo +micro chip +k affe +cut lass +clin tock +tu il +startup life +speci ale +pp pp +mi ent +illumin ates +e spe +d der +pla stered +pas sive +o jo +nay lor +go te +sp and +rush ers +me b +lyn ne +kyle busch +gic lee +ee g +destroy ers +cap rice +as mr +ab f +preak ness +orphe us +ni bs +ko ji +be sse +ac anada +year sfor +thi em +san cho +philli ps +ne f +local food +ki en +inver clyde +den ergy +st paul +plu r +other worldly +li er +l q +depos ited +alope cia +ëĭĪ ìĹĺ +p bs +magne tism +long weekend +l ssc +fav oured +fa ç +ce y +ric hey +moom in +mole sted +life less +in hibition +fa k +cat lovers +blu el +le ot +beau lieu +tou ted +the fts +i frs +home stead +gg yu +cy g +written river +pe ña +ov ar +o bu +neb preps +kerou ac +forevery one +bol ling +ï £ +wu han +un til +semi finalist +persu aded +new berry +mutil ation +medi um +kr t +con da +xox oxo +ti po +lo th +j il +go eagles +dab ang +yay y +we hr +ther ings +ja eh +dang an +we pt +ol der +mcla chlan +magnus sen +dispers al +cap gemini +ðŁİģ ðŁİī +stark ville +or lan +abor tion +t outes +legal tech +lead lap +hanni gan +as ers +ver lander +sw bb +s com +little ton +fe h +empower ing +dead wood +al go +ñ as +val eyellow +patti son +d abad +buck land +ro han +pu du +ma ari +ine fficient +cra ppie +ch swx +br ca +z anne +shostak ovich +hy una +de acons +canadien smtl +byr nes +abandon ment +âļ Ķ +presi ded +predic tive +per ma +mnu chin +maccab i +ha shim +die te +antiqu es +mt m +mm da +johan sen +ex change +clut tered +ti vism +love less +id ler +bb va +am arie +all ll +air liner +yen naiarind +vern acular +tec tonic +pur gatory +photo shoots +or man +mel ina +khawa ja +ken tish +eb m +chrissy teigen +ðŁIJ ĩ +un resolved +ultram an +the heraldsun +ren ch +cab ling +bix by +beck brom +´ ï¸ı +the ism +th un +simp les +popul ous +pad ding +mar on +crun ch +catamar an +be est +zoo s +sush ma +rel les +mccr ory +ge f +ev ra +rev lon +oo ak +mit ro +ki os +ðŁĺĤ âĿ¤ +ðŁĺ¹ ðŁĺ¹ +w z +stra di +grou ped +ge x +family law +eu an +ear my +confi g +abdul la +ðŁĵ ŀ +ðŁĴĵ ðŁĴĵðŁĴĵ +she ith +quote stoliveby +of fic +alli anz +zal mi +tenny son +stor age +pol led +mac ao +lips comb +im er +dj khaled +cancer research +bbcin tro +up f +mu es +ma gritte +hyper loop +flu ency +edmonton oilers +co vey +bel low +bar ba +pau ley +etu de +e it +broo ding +at ori +ðŁĩŃ ðŁĩ· +tooth less +ha worth +ge b +bur p +bi bb +zin da +tro ts +ca shing +be ep +ðŁĩ¯ðŁĩ ² +¡ ľ +war r +shri mps +pay able +dimini shing +b tr +å ¯ +r cm +ouro cean +no ts +mil i +j deep +duc ati +bak shi +traff ickers +ta hini +pro am +ho yer +tom ic +n ce +mar ron +ki st +kemp ton +cal abas +c vd +^ )/ +ðŁı Ķ +wor sen +side board +sad o +rock on +ij in +h mc +ðŁĩ¿ ðŁĩ¦ +water islife +te ix +sty ler +sarde gna +oper atic +nl poli +hoff mann +com anche +boat ers +ðŁIJ ¼ +ï¸ıâĥ£ . +not ches +g ash +excav ated +dom me +dese cr +char ing +art and +!! ] +sur fer +mow bray +mat lab +kati ec +inside out +a shar +ðŁij Ĥ +ëı Ħ +von n +un apologetic +seven fold +ni ak +mis syou +kab uki +har tn +care ll +cal ic +bat ley +ap it +" // +tr ical +stra han +se tia +main frame +love parkrun +kait lyn +bo vine +alej andro +z us +tw oo +mal ts +dr p +cath ay +ðŁijį ðŁijĮ +zan u +video grapher +vez da +thou sand +tar an +rein vigor +inspir ation +grou per +dd dd +col ds +chur ning +be seen +automo tive +* # +ver son +numer o +michael kors +ka ala +fotogra fie +bc f +ur f +tan trums +ro sales +min ate +ki va +il in +de fied +athle ticism +tu cks +throw backs +roth ko +ro gues +per idot +new seum +national petday +erdo ÄŁan +erasmu splus +cad die +be heading +spectacul arly +sp rain +seg way +post card +mano har +ing p +he aney +schuyl kill +s anger +migra ines +m st +gor mley +ebene zer +battle born +ðŁĺı ðŁĺĤ +umber to +sm k +saturday night +palm springs +eye son +dre cords +clari fied +âĦ¢ ï¸ı +terr ors +stati stically +par tofthe +naw azu +disc ourage +bou gain +yan kee +wish ful +sla shing +oni ous +iri dium +ff el +elev ating +crew sc +craft shout +br anco +ac ri +abstract painting +bro oms +ðŁĺij ðŁĺij +un masked +super ficial +pine y +par king +our ney +lauren s +hydro pon +hand y +d ells +cel ina +au de +at ico +ðŁ§ Ļ +re di +profootball hof +nb s +fa ints +ar aja +win kle +un tol +seaf ood +scot ts +kee per +i feel +go wan +g auguin +fam ers +bü sum +brown low +am ul +ìĺ ¤ +wal liams +tsu ki +señ or +sch indler +mur phys +laugha ble +gor d +escal ate +e oc +ye swe +un solicited +spra wl +le bowski +he mel +grow lers +gar uda +ap rons +thel ine +nor throp +nab j +kin sey +hor as +dallas stars +chit ty +bu si +bar do +ul is +straight ening +sd l +ra yo +mirac ulous +ko c +har die +do y +dani ella +costu me +t ss +st iller +plu mb +on demand +cot ter +w dw +uh f +today s +s sh +s illa +roblox dev +re districting +lo de +kh or +cover ings +ba w +ali express +peace day +men o +marin ade +kear ns +how ler +har pers +au ge +alla hu +sa z +ro well +revi ves +pul is +pre ppy +illion aire +ic el +chev elle +ç Ł +vix ens +redbull racing +plac es +pal os +os ke +mid season +mendo cino +k ron +geni us +gay nor +vic tim +sn itch +hyper ion +good food +sking dom +mediocr ity +live t +ku wa +i work +ha gia +fromthe archives +chen o +bann er +ah d ++ # +relax es +mal foy +fo sse +fire places +dar pa +corin thian +ðŁı ¢ +warran ted +um d +soci et +of love +gun ther +de main +vol ts +ti zi +klo of +ith waite +har is +h ky +firsta id +bee b +av ic +mul lah +lim es +j rs +hipp os +felici ano +whe e +un plug +r ng +pren tice +mor inga +mer ah +ma sque +k mbc +j hon +fare well +bor ic +.... ..# +ðŁĺģ ðŁijį +ìĥĿìĿ¼ ì¶ķíķĺíķ´ +âĢ ¿ +mas se +hurricane harvey +de vere +cy nic +yaz idi +ro ld +pon toon +mirac ulously +h cv +gar da +g mu +der ton +d ink +coy h +av ing +âĤ ¦ +th win +mo wed +martin sville +mal lik +life sciences +kiba at +ke ym +em r +dur k +coun tering +cor vallis +bro t +baro da +w any +vijay awada +sy ty +r ill +oy ama +ole miss +mor aine +loom is +kd trey +is c +indi as +hau lage +eng le +cre s +c ct +be you +stand ar +numer acy +l pl +bru schetta +wi k +q sl +pha sed +mix cloud +fi facom +comp ile +Ì Ħ +pent agram +monday mood +financi als +do th +debilit ating +ard t +ut u +tru gby +stre tch +sp al +san tosh +me st +lo real +gen ting +cre o +air cadets +ðŁķ · +wi ff +tri os +tou rer +rhi annon +o hhhhh +kier an +james maslow +flock a +e ww +ang ar +ab di +v hf +sound system +mi ura +manipul ating +ino cul +govin da +den o +birthday girl +bad man +ak ura +ab n +нÑĭÐ µ +zand t +par dew +mo ja +misogy ni +lind ley +empire state +ejec tion +atac ama +whi ppet +tu cc +timm ons +ps x +novem bre +na it +minic amp +exc ruci +caffe inated +smoo ve +revo ke +mccar ron +inter sect +duol ingo +ch aching +brun ch +ìĹIJ ìĿ´ +е Ð +w alling +take shi +showus your +ni da +jimmy kimmel +h ri +di ed +clou ded +sap ar +ra don +practic e +pim ms +kan a +head space +gat i +frustr ations +anno ys +____ __ +ro te +per tinent +orthodon tics +be kasi +aeronau tical +wei wei +vic ki +rel son +ra he +mar ry +do ak +chan ted +bbc countryfile +uk gif +qu iri +qu ag +much ach +loren z +k roos +for your +far away +dark souls +so k +s land +k for +future ready +car din +advent uring +zambo anga +yon ghwa +u mma +kan an +hand yman +fi duci +edge wood +domestic ated +consu lar +wn d +super hit +pa cho +mono chromatic +im bu +gla ad +dar ken +cor ti +byo d +bar at +({} ) +wein berg +the chase +or ge +miy agi +j ali +in ac +en atics +ban ished +b fg +wheel ers +neo liberal +mi mics +enfor cer +we aning +py p +i pm +her ni +cla flin +chitec ts +carbo hydrates +ae i +work week +ver ity +sleepy hollow +sani bel +per due +global bc +bin ds +sty ro +show stopper +par able +i dism +hin ata +fore casted +du ffel +de cent +bott omed +bbci player +at elle +anthropo logist +see able +re creates +raven na +pu ffins +mand al +fla ps +cou sin +cool ness +che tan +cbs miami +ao te +ai leen +u twx +u di +ram bling +o ggi +ne en +meteor ology +ir respective +illi ers +domin ick +cann oli +adjour ned +âĿ¤âĿ¤ âĿ¤âĿ¤âĿ¤ +wy p +ste au +o cial +k hay +h mv +ene x +dj life +u ko +tro pes +swar tz +pu yo +play house +patient safety +labou rers +ite a +e wa +deal oftheday +bog dan +wild star +wed g +the gap +pa ki +is bell +devo id +aw w +as b +was pi +te fl +sver ige +qui eres +mo en +asi o +afri ka +vi agra +twitter verse +syour friendship +pry ce +mon on +elm hurst +bring iton +videom tv +tam er +jalli kattu +foodie friday +federic amog +extrac ur +cal len +anton y +âĺĢï¸ıâĺĢï¸ı âĺĢï¸ı +ta ichi +rishi kesh +pand ana +n ung +la ver +gaul tier +fair ways +d ello +alcat el +z le +re frac +prote omics +pent at +or bits +mor gue +maver ick +kand ahar +issu ance +intertwin ed +i ren +git mo +faz al +err atic +dor mit +beauti fy +( (: +you n +mx gp +it zy +dam en +colorec tal +co sm +chri sk +c ita +apologi zing +îĦ Ĩ +te tra +saoir se +penit enti +man deville +lo dge +ja xon +j la +aph rodi +alter bridge +ç ¦ +te dd +re fit +injec ting +di stro +brig ham +year challenge +ra he +hit men +ha bi +grammat ical +george michael +bro x +bo ck +am la +tour life +stry ker +p all +marqu ise +gher ty +el z +cla pper +cataw ba +tisthe season +scrip tw +ma doka +int ently +gee king +galac tic +do the +bl c +ap tor +anci ente +al icec +ÙĬ ÙĨ +wool y +ralph lauren +jane austen +hun ky +dry wall +chen in +at n +anticor ruption +three some +the t +metal core +li ga +lex icon +eura sia +dor sethour +daily mail +vel vet +mou lton +colle tte +bra ai +ben ning +asi acup +wor ded +social work +shu man +s ich +ment a +kin sale +i hansika +du cey +dif fuse +cur sing +cordu roy +å ² +velve ty +ur inal +tucker carlson +temper ance +fro ggy +af cv +shan ty +l ner +good rich +ge j +eli x +ef ood +b of +artvs artist +vo wel +sut cliffe +sta v +se ous +ra ines +masto don +booka day +tag alog +ridge wood +progressi vely +lock smith +kan er +dic kie +cel los +break downs +bo ssy +ba al +aveng ing +sky landers +fa jar +ci u +be aks +b ere +ðŁĴķ ðŁĺį +su fficiently +may weather +de scar +bil i +tat o +revolution aries +kwa wesome +em g +cad ill +c bre +ðŁij Ķ +wheel chair +ten a +r amaz +jo kingly +har lem +go knights +enu mer +dili p +con serving +beckbrom fl +bat avia +ag gregation +ta ip +stat ure +selfish ness +scuderiaf errari +ke u +in sati +fix it +eric garner +entr ant +âĢĶ " +Ñ ĩ +wi ley +ty nd +pa ke +infl ict +bu law +black more +sacramento kings +ol ta +ker r +gra dio +bro snan +se on +po ws +patho gen +fra zer +en large +athe on +ware housing +ur se +fil ings +dis content +che t +wri gh +vacation ing +ta rek +str angle +nic ki +ident ity +dhan i +d ary +construc tions +cagli ari +age o +a fia +wedding dress +sch ed +per rin +me ur +in art +wor c +un settled +mini bus +mat ric +christ ened +bell inger +mo xie +kar bala +e hn +at tainable +ta va +sw ells +ro da +pi stol +p z +lis burn +k roll +fcbayer nen +bel lion +ro byn +grou pie +an ant +ĥâĸ ĥâĸ +ri at +ly me +ju te +hall oumi +glu t +cor te +vas sar +then and +terro ir +ring tone +musta ine +homer oom +fu tbol +fr in +bo ba +basil don +to ews +summer fun +it sa +in ia +im plying +hark ness +gn u +deplo yments +bir dc +bar my +ì ¢ħ +sten son +roman a +note pad +me mon +cellu lite +by gone +ator y +wood lawn +thevamp s +la sses +embaras syourbestfriend +affection ately +ws j +vision aries +ren te +po iti +hann es +ha ger +ge me +distribu tions +bas sad +waveleng th +van n +or me +neighbour hoods +jersey city +fu te +car adele +bru iser +am ed +sub a +bas so +ðŁĻĮðŁı» ðŁĻĮðŁı» +ë· Ķ +â¬ĩï¸ıâ¬ĩï¸ı â¬ĩï¸ı +ss unday +sli fe +skar sg +li an +gallo ping +boc ce +tou che +ro my +ou la +n ll +mo ir +le mony +is ine +hoo ver +evacu ees +dd national +addic tions +ðŁĵ İ +ri ker +nca a +ka ia +h anya +dayofthe girl +crust ace +acrob at +white field +vill anueva +vallado lid +s mor +s ico +ro ping +open mic +gen ia +fast lane +eci gs +dod ds +board man +zin edine +u che +q at +n abo +m wa +kon rad +knock outs +i of +co lic +weight loss +w pb +shri ke +re vert +library congress +gate fold +åĨ Ĩ +se ad +sam u +piran ha +om ena +g aven +dayo ff +cray ola +y ai +under sea +shap ers +perl man +hy rule +di manche +âĻ łï¸ı +your take +sung jae +ple c +o ik +neil tyson +jam on +id le +i go +i bra +cast illa +brem ner +bot w +ðŁĺį âĿ¤ +pre fab +men orca +maine mendoza +glori fied +divyas pandana +determin ants +black sburg +ìĽ IJ +make red +ly in +ic w +filip ina +est i +del ves +dat uk +absolu te +whal ers +tt al +spaw ned +newin dian +mix er +lead off +kash etty +ha ddad +gau ahar +f ct +eis ner +ate in +w sm +th eni +school children +rel ls +orchestr ated +octa ve +ob sessions +meg adrive +jab i +ideo logies +har tt +fe sto +boo ting +band h +bacter ium +won g +tw ing +lepro sy +k vit +bye lection +bat chel +alter nat +reli ed +ke el +fresh ener +ec lan +cor i +chi ko +aw ed +anil kapoor +whis k +var o +shi o +san ia +laure ates +j rpg +guineap ig +grizz ly +doppelg anger +bar rows +a hon +ðŁijĩðŁı» ðŁijĩðŁı» +Î ´ +ye v +woo tton +sn ps +re collection +ma al +ket an +fe west +far az +cd w +bi f +shah baz +qui er +hust ling +hern don +deu ter +cl u +adul tery +ru it +gre na +gn omes +free hand +ref ills +po tting +diagno sing +ban nister +al ms +ag et +sam rat +s vi +pe tri +o virus +mid lothian +ëį ° +scal ability +me tac +just us +individu ality +usa c +the karanpatel +s bar +re shape +pa war +jf k +is sac +custom ary +bol der +Î º +r mx +pla i +ber nice +ar cana +ntv news +melan cho +h ö +bro aden +andali olo +ðŁķ ĸ +ze h +ww t +us ma +substan ce +ray man +kr ati +de ze +bron er +ðŁijį @ +ze ec +x wx +sil very +kac ey +ic ardi +et works +ba st +aap a +ðŁĶ Ľ +vap our +smu le +hu ang +end ar +dis located +cari be +) .... +ðŁĶ¥ . +vo to +tu tan +tele metry +pa karmy +opini on +o tome +nz pol +is en +e amon +co bbled +cit rine +audit ory +ween ie +uk snow +long itudinal +har pers +gni er +fasten ers +em met +cross bow +cau tionary +bre aching +yam una +wash i +swit zer +swee tly +spar is +sp ilt +nig am +indian food +fin alize +bach ata +wi w +spe wing +ra do +dh c +an ow +kis se +go frogs +alpac as +âĻ ĺ +red grave +new era +kid der +hel ios +del on +c anna +ðŁİ ª +xi on +stone man +polyure thane +ni do +mr ng +mac a +iso topes +co fc +twit pic +round trip +numer ology +n ooooo +marc marquez +just ly +ga ir +french bulldog +fish and +felix stowe +er k +bag e +ag itation +twit s +moment arily +wi st +saira jdeep +n tu +mur o +mc gi +man hood +man ford +love eee +l ta +ha dri +fer ro +doro thea +beach body +arri ba +angu illa +vape fam +spres ley +sli mmer +sday time +ophthal mo +lac ing +j ür +grin del +din ah +ce x +c sun +breath s +bra bham +war f +sp ang +ku bota +hay ne +h mo +gv su +go sa +fun nies +cre a +zak ir +stru mmer +kur tis +h kt +ch aka +bach man +b jer +adventure travel +y ves +tor r +nitt any +hi mes +cen ota +bay ern +иР² +ur ses +sn ags +saura bh +nico sia +nick ed +in shaallah +friend liest +ever note +austri angp +ar mou +anthropo logie +skag it +shrou ded +little mix +hello kitty +eli z +br é +apothe cary +amphi bians +tro p +tr t +suc ces +rup ture +metrou k +ky t +gla sto +g int +congratul atory +volunte ersweek +video clip +swoo sh +neu x +man power +format ting +fl r +fer nando +deal ings +thequeen mzansi +shawar ma +shand ong +hurricane irma +con vul +yo han +tr us +re forming +r ter +lax mi +ho hen +fu turo +down grade +dehra dun +boo ts +b ct +aaaaaaaa aaaaaaaa +ðŁĺĺ @ +shill ings +s ge +ou le +e gon +du pree +dri bbling +contradic tory +canton ese +avar ro +ze enews +y adi +was atch +th alas +rv smtown +o ap +ma dinah +ber ton +éŃ Ķ +ter yx +ss ant +sc av +realmadri den +park s +ome tre +hl f +re signing +ki ana +k cs +gal ine +el dredge +co han +anthropo cene +ðŁļ Ļ +ãĤ » +things that +ome ga +om bo +ny an +le gia +instrument als +firstdayof spring +ecuad orian +dic es +chautau qua +chas m +ðŁijį ðŁı¾ +wit ten +wang i +no bles +chan el +castle ford +bloss om +whole heartedly +v ab +un aids +pal tan +off c +meta physical +cor net +car bine +acknowledge ment +radio city +mal ach +w whl +total ity +r sp +power up +mar tel +ice day +go ings +g me +family day +es k +cb sdaytime +yam ada +wn y +spe th +os walt +man heim +make comics +in securities +ici us +ha ge +âĸº âĸº +won ho +m action +lo zano +k uk +jar ry +indi visible +in nit +go er +ff i +dut chess +cle mons +cla ssed +cham i +back end +wat u +war games +van illa +ru bin +neop rene +lo x +gly phs +develop ment +char ger +cesar o +x c +van guard +poe hler +pau ses +p sc +mis bah +mad ura +eli very +de coy +d ouro +coast path +biop hy +ìķĦìĿ´ì ½ĺ +with drawing +schwei z +sarde sairajdeep +san ji +proven ance +pic ker +nade em +he hehehe +form by +en ed +elvi spresley +ku du +ke at +kam eez +curios ities +cr amped +childre ss +wra ppers +wolf man +st ell +passion fruit +no sh +ni eve +fang irls +avon dale +z ace +sar ang +preserv atives +lo co +ig l +hand set +hai lee +ge i +g be +distin ctly +bring in +f enix +enf ant +elast ic +don o +commer ce +budd ha +wh ang +sz cz +roa dies +retin al +mc ghee +halli day +cu tie +slu m +cosmon aut +yoshi da +t ney +t ge +sm riti +d ls +at orio +ali e +ìĤ¬ëŀ ij +tink ering +ske le +rath bone +pr g +phon ec +mc w +lifetime tv +lead up +dy r +spho tos +pu ffer +prospec ting +osa urus +nv m +mor phs +maur ice +m show +le grand +iran protests +cartoon network +bet i +acrylic painting +ab id +ģภģภ+ðŁĩºðŁĩ ¦ +è res +wait ingfor +min has +leh enga +bag ans +a or +multil ateral +lig ne +hot shots +de classified +wish ers +tiss ot +mess aged +lo on +kul tur +kil ometer +ital o +fer rero +co pier +bar net +shal lot +sea view +dri ven +com press +chic ano +bou vier +âĺ ® +time flies +sal ty +rother ham +rex ha +ni al +i story +h town +chi v +afro beat +yellow knife +vil s +va sive +sin fonia +ponty pri +hou zz +di ble +âĹ ¼ +wine making +w ca +van re +scho oner +os r +na se +mi zu +klo bu +journ aling +fa ker +emmanuel macron +an jun +win t +j ari +impin v +earth athon +di ffers +c gm +supp lic +stay in +sieg fried +ni val +j ith +ho cking +u hr +shab u +hot test +g any +bigre d +ðŁ¦ Ģ +ï¸ İ +swe des +pupp etry +prin se +mc donald +fran cia +at ino +ar yn +ultr alight +the j +ra dar +pre caution +ly a +kasper sky +jeff eries +in fir +gaz zetta +face less +diver ting +chrome books +agh a +ab normally +Ù ģ +sho win +shab a +psy chic +ja unt +de formed +awan inews +a ily +unfore seen +picture ireland +n gt +down y +dalhou sie +council woman +cor nyn +bet sey +wing span +par id +ming ling +loc us +in no +husk er +fl ys +carroll ton +tr icity +scra ped +safar icom +occup ations +nawaz sharif +hoo ves +cathar ines +ag ger +à¸Ńภ£ +wad dle +syl vain +st johns +so yl +ol ds +g ack +fire men +fin o +en tex +de constructed +bc p +asser t +ب ص +wh ow +vari an +ne sta +max well +kru se +dr b +cool ant +aw kins +al et +ab rown +ðŁijĮ @ +smashbro sultimate +ir rig +cobble stone +cas anova +buzz ed +tele kom +should nt +pt p +memor ia +cham isa +alme ida +wi reless +re visits +no ize +ne go +la garde +is th +is ac +extingui sher +es an +w cd +se ful +dead lock +ðŁĺħ ðŁĺĤ +wen atchee +sla g +sen za +o in +ne hill +ko vind +kan ter +jo be +ci a +cat ers +agh i +suni ons +sop er +sli z +pac cio +mo sh +ma ddy +lo rence +herb icide +grati fication +cu it +bar bell +? ". +l pd +kil mer +car no +ball entine +sh iner +ne tta +loo kat +il ocks +iam the +ch ola +ul an +tr fc +termin ally +ori st +o gle +light bulb +zo or +web store +wait ing +render ings +poetry month +parach u +miniature monday +metro link +m ple +kre w +empha sized +car rot +íĺ ķ +vari us +roman ticism +mahesh babu +lake show +jol y +cormor ant +break in +ag ni +v av +shack leton +po of +mass ager +man ay +m br +kag awa +brew ery +att ila +ade d +pav lova +lan ning +king khan +i ata +fl our +dun ning +az awa +are th +yee haw +shel tering +se bi +ru pts +pin i +nar rates +far rar +cho kes +bo ssa +snoo ki +sep tum +p ounce +my team +my n +metaph ors +mag ine +leaven worth +lay man +lat ch +hi jacking +hard away +gu gu +godbless america +dil ip +cla r +brun o +ا٠Ħ +wer ks +vision zero +t whi +sei ko +ibm watson +emul sion +bho ys +ws vn +voice mail +v cu +robo tic +ro k +o woc +mari ecla +cric h +av c +su bi +shivangi joshi +hai da +s keeper +mid ler +kn ackered +kirk patrick +kil leen +i fc +y ala +vege tarians +sub terran +shat ter +mdc ps +max ine +mat ta +amic hele +pro claims +pri sma +h dl +for de +fo ams +end less +bill eric +an si +ti rana +smoo thing +roun ders +man ics +koo kie +invin ci +ari ad +adop ters +timess quare +ta ec +sco li +s wash +ou tt +o sho +gas co +fo c +dru pal +coy w +bus king +bat ti +ĸ ï¸ı +ðŁĺ© ðŁĺŃ +twal ker +the vote +o ha +man on +kri t +jose p +e inste +contex tual +caradele vingne +wee zy +som er +ro an +pro ton +oke anos +halo gen +german gp +choc taw +ta q +syste matically +sha shi +om ens +mad dison +focu s +ess ay +air bag +tsh wane +scho ice +pr é +hed wig +deze en +ab dic +ðŁĴ°ðŁĴ° ðŁĴ° +vaid ya +holist ic +down ing +desal ination +de are +adole scence +ou trun +nu di +má s +indie author +f agan +dof theday +conce ive +chest nuts +arch diocese +ac are +world war +gun g +g ada +cel ts +as una +à® ¨ +un cu +sun ggyu +seaf arers +red field +pis ses +odd ity +blow in +( ': +à° Ĥ +r q +nan ak +iri er +ha velo +cri key +chase elliott +c mr +bar atheon +sat the +newindian xpress +imple ment +div ul +delu ge +bla en +be el +ìĽĮëĦĪ ìĽIJ +ul timo +stained glass +ro lex +pla it +narcissi sm +mi gno +mal abar +leu kaemia +ing up +hot ch +d cr +chath am +blanc pa +ti pper +glas shouse +drag ster +dil apid +anim ators +w fla +toi let +pi o +paratro opers +mi stic +hir sh +guru gram +Ħ Ī +vie ux +sub sea +quin lan +nie der +nes n +li day +lau t +ampli fiers +ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃ +w ami +over crowded +fir s +d nd +carto onists +barre tto +wh eni +uproo ted +stun ting +spital fields +smur fs +perfor ated +n fa +kios ks +him chan +fli pper +daily deal +brand new +ðŁ¤ ¸ +íĪ ¬ +re mb +mm mmmmm +iber ian +freak y +falk land +art news +ak ha +ðŁĺ ¶ +the cure +strath cona +sel fe +omar keting +om ani +for tw +brad dock +ðŁĺĮ ðŁĺĮ +ðŁĶ´ ðŁĶ´ +san remo +hu ma +guil lotine +foot bal +dun lap +dre a +cast away +but ch +sl ant +rout ledge +on sen +litur gical +grunt led +discovern i +bou che +and am +ðŁı ¥ +tuss auds +think pad +super group +summer solstice +que sto +notice ably +fi bres +ende d +colly more +buzz in +ai k +w ate +vivi an +stav anger +re produ +pancre as +gar cetti +ceme teries +bird song +arith metic +ten is +soo thes +post modern +mul holland +cn j +bi agio +ar tapp +antichri st +yol and +so be +run time +puri fied +prou st +jo m +godd am +far id +cru yff +ðŁij ¨ +un ig +ta chi +syn chro +pa sir +ob la +lind t +le de +dist iller +cry o +ca h +atro cious +ãĥ © +x en +wi dow +veg gie +scre wing +roman reigns +ker nels +cream fields +ak ala +wri sts +work sheet +mar su +mar nie +mac o +io d +con volu +ar les +. ðŁĴķ +mo te +j ds +ers for +du ty +div ina +animal alphabets +accu weather +west minster +quin cy +pou ting +n live +lat our +ketch um +gi le +Å Ľ +wood bine +paragra phs +nad da +ac tic +white sides +ung u +som ber +min ot +lati fah +horror news +hero isl +gem ma +sky train +ni em +ne urs +mon line +jay hawk +fe cha +fast company +ce m +armedforce sday +! " +supre mely +st exas +premi o +pal mi +nie to +n ge +abe g +âĺ ģ +x r +reno wn +mor ten +ga sh +ap j +ë¹ ħ +whats the +rain y +conceiv able +af db +t live +shi itake +r mw +om alley +ke ving +stun tin +espino sa +de br +constant in +art show +ze wski +z ander +summer school +mo rena +ferr ar +d wight +boko haram +slo ths +shill ong +ky e +kry st +equ atorial +capital weather +bi onic +bc i +bb mf +arche ological +aly son +acquaint ance +!!!!!!!! !!!!! +z ina +ye ong +th ali +red car +iti zen +he cho +gri gio +du sky +de grassi +bermond sey +b nwt +aram co +ab ut +wine makers +tu al +rhetor ical +hesit ant +ay aan +at op +ap ha +sel kirk +sd v +neck tie +jo inted +jo fficial +hi bern +fle xi +dow ry +chap stick +x anth +la ren +fla shed +eg x +bin ay +agnez mo +zu mab +ra at +mat suri +ly wood +jose f +har ald +bal sam +ðŁıĥ âĢįâĻĢï¸ı +shannon leto +sa aho +s anne +mo ans +gott alent +dam us +co e +bam ber +swallow tail +snat ching +sharpen er +ligam ents +ka in +evan escence +appalach ia +à° ¨ +the ir +skag gs +sau st +partic k +lin ks +john legend +i bo +gn an +twit t +n fp +em b +doub ters +bi ak +ad ria +âı ° +segun do +sapi ens +cm h +yadav akhilesh +win i +t pt +ron d +mau rer +margi ela +man olo +jec ts +ha wn +green point +ev on +atlé tico +scam med +n nw +key less +i he +hol den +attackon titan +voo doo +thi an +tau pe +nal ang +me ath +je i +i ann +hr tech +dar lin +blackex cellence +best fans +b wa +ðŁĺī # +vo z +rup tured +mam ac +m bu +lu gar +indeli ble +he bert +al aa +seag les +ruck sack +dav y +copy writer +am ok +ab sa +ror o +q amar +new wave +multip lier +mc adams +ly chee +latel ateshow +hi ke +gen er +dra ken +cul lo +as cap +where are +radi ate +power star +ms w +hon do +gas light +bre y +az oo +at erials +ãĥ £ +she boygan +regi ster +quinnipi ac +pedro sa +mu ffs +habit able +buck head +taun ting +princi pe +na ar +hi ba +duck tales +british columbia +sug i +road block +pic kin +op tera +le os +il ford +hand picked +da shed +bal los +acceler ates +un orthodox +trend line +sy cho +hex ham +ev ita +malar key +dor mer +bri x +alici a +adel phi +ro ssa +plu mbers +newe gg +nai res +jo dha +impover ished +h mmmmm +gal en +em v +defend ants +ðŁİ ³ +way farer +to ca +ste vien +sli go +perci val +jk corden +g pl +aer of +ac es +yester year +sc as +salam anca +rey na +pv fc +p tr +harry styles +dan n +ãģ ¾ +belle za +alo y +ab alone +xian limm +hur r +hot topic +home ware +eas ports +clashof clans +ber ti +an ad +v ca +st ach +square pants +shin zo +cor ks +ðŁĨ ĵ +what syour +sh ey +ra af +pri mus +narc issus +massi ve +klobu char +jor nada +ben elux +a ou +âĿ¤ï¸ı ðŁĩºðŁĩ¸ +tre acle +tion alism +st oun +sab o +jalape ños +dar kk +ci ma +bu ku +bridge t +beha ves +wim mer +national gallery +mis conception +epi ste +b na +ani vers +us ka +u soc +ne ocon +ly e +kang in +cry baby +cler ken +car m +re ga +par ameter +over taking +nu man +mor nin +job fairy +ha f +fil o +exceed ingly +end point +b kapoor +an x +amaz in +sau teed +mal ick +lu gano +front row +di en +Ø§Ø ¨ +ys se +sti pul +sr m +sc roll +rever se +no tal +key boar +immort alized +com d +arch way +aficion ado +up heav +ta ker +m ä +hou rof +dog show +mo oring +meat less +king scol +he ter +hand maid +cani ff +bo ssing +amaz ons +x lm +sav it +ice cube +don te +woo oo +ven trilo +sy ring +sen or +pondic herry +plan ck +par ov +p ening +mcdon agh +dwind ling +dar fur +credit ors +cra zed +cr j +an ong +mar coni +devi led +carmar then +bik ram +ðŁij ª +vig or +v fb +tro ss +to th +pe u +in paradise +dev out +que tz +mi dr +hag ya +fu sing +displa y +ben teke +amir ite +ty rol +tom atic +tic ke +ro bredo +kum kumb +hurricane matthew +grand canyon +chapar ral +cat ania +car ousell +seri al +seme sters +reper cussions +ouach ita +moon shot +ic les +how doyou +d sen +comix ology +children in +richard branson +read er +p so +g dragon +far ro +ski pton +shoe gaze +ni dhi +kö ln +green wald +smu ggle +shel led +sh of +hern ando +edu ard +am is +vau lt +more llo +m ll +inter generational +i ab +don agh +bur kin +ä¸ĸçķ Į +âľĮ ðŁı¾ +venezu el +v ato +sto pover +som bra +sal ad +pav ers +i bi +beaver ton +aerial photography +aber g +åŃ IJ +wy ck +progro ck +ni vel +mc do +land rover +esc a +bis d +ðŁĵ· :@ +s gr +re stin +nar uto +longre ads +deliber ation +a ight +ðŁĺ ¦ +ssi ma +ri bbed +intro verts +end re +ah r +ðŁİ¶ ðŁİµ +á rez +squ i +park life +mo se +dal its +calm ness +bc t +angeli que +un surprisingly +un necessarily +tor ched +sw u +sof i +reimbur sement +qu inox +may e +cy stic +clt traffic +ac ed +xi ang +waz iri +supper club +se ti +pa oli +ol on +kr g +ing at +u stad +u gl +twhi ddleston +phine as +ing rosso +digital nomad +ar to +ver milion +val po +sch om +penetr ating +ky at +hand woven +fle mington +( =) +w tae +tent acion +ste em +shri e +mp l +ic am +i pan +ation ally +ðŁį ³ +th k +reti re +re mission +re done +phar ma +ove chkin +it sm +donaldj trumpjr +crack er +barist as +ari ah +app liqu +aote aroa +ab scon +west wick +veriz on +sydney fc +enthr alling +chad ha +bn n +bi stro +ðŁį ĩ +trans missions +straigh tened +mol in +letsgo bucs +jordan knight +gro ff +freel ancing +fin gered +car show +ac in +nt fc +klam ath +hitch ens +gee bung +el vin +cre amed +bourgo gne +pie monte +j su +ha bana +gran turismo +aqu at +** **** +!! . +zhe jiang +twol ves +q wer +mb urg +im partial +hor d +har ps +gr r +g illum +dar by +b ens +ap b +air lifted +pale onto +no things +gr unt +c sb +at ree +afgh ans +ðŁĩºðŁĩ¸ @ +support local +sub stitutes +gu la +ba ju +ate gate +amig as +ab ell +ve m +tw ing +o don +long hair +is ley +gu tters +gre ase +g fa +fu mi +wul f +se ase +post code +e gal +champion scup +c sis +ali yah +y rf +w saz +sr fc +me gyn +mag net +kno wns +i hs +drug store +biomechan ics +aver a +wimble don +slu ggish +si mmer +science museum +qué bec +nok xl +man do +k lub +gran bluef +dü sseldorf +col ab +ch ars +boo ger +tin nit +ra fferty +ne k +mo v +hand out +ei u +cat skills +business intelligence +boywith luv +raik konen +rachel le +pro g +mt pol +mccre ary +com pote +child marriage +aa at +âľ ¾ +zak zaky +womens rights +tre port +tramp led +no tb +m ri +lucas film +lo stin +law son +jun cke +juncke reu +ho tty +syr inge +su ds +st ooth +ka ar +ity uk +inter play +hon dar +ho gan +fu ssy +exal ted +en crusted +c bo +absor bs +ãĥ ij +ter tainment +styro foam +reali stically +n pg +men orah +mcgin n +lan dish +i ki +hr p +c chs +yo self +shi vika +petro l +morphe brushes +men os +mck agan +k uni +gob let +davi do +beau t +bart enders +ðŁį» ðŁį» +west moreland +war planes +py ne +princi pled +pen sive +par s +need to +mar salis +local e +harper collin +gi v +ap riv +al tos +zace fron +z at +takeme back +sridevi bkapoor +py ar +pla w +expend itures +de bug +ðŁĺ´ ðŁĺ´ðŁĺ´ +z ok +s itec +ne fer +n na +ki ely +co ty +anim ation +an war +ye shua +royal operahouse +nf v +cur t +beat le +........ ....... +ä ¾ +âĢ į +sy phil +sy an +op ts +lu ang +hol yoke +en tel +do terra +bl und +anag ement +alum pur +si ra +reiter ates +parad is +kpk updates +e ased +command ant +ande ren +Ļ ï¸ı +too ts +nott ingham +ley fc +ec i +ec d +comp iling +bm supdates +berdy ch +ar ron +val der +stri kingly +snoo zing +si ento +nikki haley +mar lies +ic illin +femin inity +fat boy +cal dera +bon ey +boat show +affiliate marketing +ðŁ¦ Ĩ +win nie +win dies +une ducated +mac aroons +iiii iiii +critic ise +coron el +beng a +twitter ati +p cl +n mb +les bian +jacqu eline +hom bres +encan to +dog slife +suppor tour +ral ston +cine plex +ðŁ¤ĺ ðŁ¤ĺ +work horse +tour nage +sa at +new sasia +k ish +indic ative +chat ty +cali pari +blin dly +street photo +slu mped +reservo irs +lac tic +ble ts +w tt +ta jinder +sobr ang +ro the +la uri +idi oms +hor ts +cran brook +cb f +bulaw ayo +au ro +ze a +southe ast +par ale +ing alls +drawing august +co existence +ðŁĺī . +tin sel +syn chro +stedd fod +sh et +rp gs +poppy legion +out loud +in dr +eli jah +electric vehicles +co wh +chit ra +as ahe +yu mmm +vene ws +swach h +pc p +over ride +mu z +k ada +el bert +du sty +con cussions +brazili ans +ar ame +sna il +out burst +ni hr +mun do +jean nette +har greaves +fin sbury +fa yo +dylan obrien +se ssion +sd m +sc run +procrastin ating +gol dy +brid lington +________________ ________ +tr uly +mon ies +jour no +halcy on +fer b +ex mouth +all day +soft ness +its all +hard style +bo yl +az adi +uni formed +six nations +sekar pandian +nikon usa +nc i +master of +ice bergs +hair pin +demilov ato +deben hams +crowd fund +ash croft +ang ering +: )! +stu ffers +pushawards maywards +p yo +m tu +hand ley +é » +u on +tobi as +tal aq +sig ner +ru sted +no zom +magni fying +divi der +al de +:) )))) +!! .. +ภ¶ +ठ¡ +po ons +oil field +do tty +air bags +sl urs +rapap ort +ms me +klon dike +. > +why not +tw omen +reboun ding +mi ken +ho dl +fru ition +do er +cin que +certain ties +âĢĶ - +tiss erie +theprojec ttv +se ducation +jewell ery +in between +impre ssively +hair y +floo red +flo wered +de carbon +bari atric +adar shan +ãģ ı +water ford +tre stle +tann ins +mo in +gi sts +g be +brande is +boo b +behavi or +b pi +acade m +yam aguchi +penitenti ary +mosc ato +dusse hra +democr acies +bla key +bad dies +azte ca +ar cy +tru ely +squ ab +ghazi abad +bu gging +bal vin +am nh +âŀ¡ï¸ı âŀ¡ï¸ı +าภĻ +ri aa +mennon ite +ice ps +hey wood +we fly +sig nat +shatta wale +shab irah +moor ish +men tee +hudson valley +bas mati +? ), +tigh trope +sor i +raj sekarpandian +ne manja +lu zer +fre t +en list +el dridge +e hh +bett ingtips +apple sauce +ðŁĻı âĿ¤ï¸ı +âĹ ķ +trumpp ence +sol berg +po inte +ero ar +energi zer +death penalty +ch iro +ww l +fla u +evol ves +à ¦ +ther ain +slo g +sk ova +rc mp +kumkumb hagya +go dolphin +camber well +be ading +ax ing +z ki +war blers +une qui +toowo omba +salt lake +panam ap +ny p +mc cord +light year +je fe +itu tion +hydropon ics +car paccio +sho spital +mai da +indi erock +cu enca +bati sta +all access +____ ____ +ì µ +sab e +my life +e dex +ber ne +av ings +ani ello +stor onto +pre aches +head piece +hair dressers +f sc +ex patri +dana white +ts f +tal eb +stein beck +pin der +mol l +lu ge +lil kim +jin woo +camp ing +broc ade +al locate +ï¸ıâĥ£ : +áµ ĥ +out takes +monte video +lom b +fun ke +flet ch +ê³ ł +z nation +vi Äĩ +ve sted +shabirah luwalia +pur su +o ath +nas r +mer cato +dave y += " +wi red +uni do +t ili +re ston +fren te +di aled +cf pb +/ âĢ¦ +vel our +sle u +o ren +mad ina +ken worth +kell yanne +enchant ment +b ce +av ana +ana than +! ðŁijį +ðŁijĬ ðŁijĬðŁijĬ +ðŁĮ ® +ur b +sar ap +reli a +knes set +cy p +chan neled +caball ero +bcli ons +v ella +pri sing +pine wood +n ane +insi des +gorge ously +flet cher +al jazeera +pre cep +per vasive +pen arth +mam my +kins ella +connor franta +colla bs +ahmad shahzad +w tm +mercan tile +loop ing +loo ky +i got +fa jr +s dotcom +pat naik +do brev +bor os +ad erie +stell ar +liv re +impre ssi +da hil +bas ing +wedding photography +stu pa +or z +mar ky +mag da +id len +grat in +dies els +cas ino +appe aled +machinegun kelly +m ct +beck man +at water +ëĭ¤ ëĭĪìĹĺ +tur i +st david +sre bren +smo k +pu yal +mor pinoy +inter st +you uuuu +yo der +roo i +rith vik +re payment +rat cliffe +law ren +flatt ened +cu so +ar tic +tal en +sig nees +hart mann +ev ac +dri vin +clo ves +ab lation +yy yyyyy +thro tt +th é +sw f +squ ig +jhal ak +ig nit +calabas as +al one +ðŁĺģ ðŁĺĤ +ÄŁ lu +throw ers +sway ze +srees anth +sex iness +gen ji +algi ers +z oro +roa die +posse ssing +paras ol +over watch +o dm +mal mo +ec khart +desi st +call me +) & +! ðŁĴĻ +ðŁĺĥ ðŁĺĥ +tas a +nor vina +kom o +i kaw +brutal ism +bar aka +tablec loth +out pouring +lovel ace +guar da +ga vi +circa dian +ba q +umb o +tri gon +the f +tc dsb +ta ku +sni pes +protag onists +par kin +das adarshan +cur ried +c ne +st ico +ro ja +or p +noton fire +dragonball super +dac ia +blue monday +b fs +are e +any how +adopt adog +ë ± +åŃ IJ +y ur +syl vani +rip ken +ore a +milton keynes +la it +je z +gay lord +g ase +edam ame +ba iled +v ry +si ds +rain storm +emer alds +cent ra +becky lynch +à® ³ + § +viceg and +then or +tem bre +o tw +jad ines +ain sley +petal uma +nz wine +ha emo +dor ky +ãħĭãħĭ ãħĭãħĭ +ãĥ¼ãĥ Ī +utili zes +shaned awson +ri ze +har ts +ha gar +effici encies +deu ces +def tones +centr ally +wildlife trusts +n fr +gt fo +cuis ines +boeing airplanes +ãĤ ¤ +v su +treas u +tam pon +sth lm +staf fie +simr an +sh ey +home wood +dougla s +tn tweeters +spoo ked +in ag +i pl +guang dong +culmin ating +botan ics +bha v +yl ation +very where +vel y +ten ner +ru bies +nar ita +muje res +kar ol +fa o +custo dial +uof g +ra heel +plac ard +lawn mower +ja ar +ation ist +âľ ¿ +un accompanied +sleep in +side car +qatar airways +fright fest +blu me +batt lec +tampab ay +syn gent +pend le +i bom +hu er +head gear +cosmo polit +wal ther +transpho bia +san gi +or da +hexag onal +hb cu +gryffin dor +disrup tions +ber lu +ark ham +app el +ðŁı ı +wash room +po y +pk r +new sies +mon ahan +f ene +e mas +dispo sed +the moment +shir a +kuma si +hypno therapy +dhan an +ang ler +wh et +vo u +newh ampshire +manchester united +mam as +if you +hor sey +h ma +gin sberg +de po +tran scri +tajinder bagga +oun i +lees burg +k imp +happy weekend +en coding +bru ton +broo ker +broo ches +bor k +ang lais +îĢ ¢ +st eves +sk t +negr oni +hir i +e ber +dic tion +amal fic +tho tels +som i +shap er +q asim +invigor ating +gan try +fle er +cc m +blue water +atro phy +ìĨĮëħ Ģ +tourde france +fet ched +che aters +centr icity +armp it +yu cca +tax reform +snu g +ma up +li go +hr mann +fu ses +under represented +strath more +seab ird +gulf port +dam sel +colli er +az er +a online +worldfood day +sil vio +nz d +nach a +gr illo +fair fax +book blogger +zam o +work bench +we do +traditional art +thel ight +rain forests +or phic +l ma +ko z +indiffe rent +gu apo +cw m +conspir acies +brum hour +be el +vari eg +pay et +is ang +go sport +empan adas +conver ged +am ping +wom bat +wa u +the way +merci er +mccar ty +itt y +is beautiful +hu w +was ser +s first +oni stic +mtvbrkpop bts +galvani zed +ei ghts +ðŁ¤ ł +ma ac +kel ving +grindel wald +co sas +calab ar +ar aw +# # +ðŁIJ ² +tag sfor +pur rs +nai ledit +msh sl +k ore +ham mett +ec ret +dra goon +d cm +clo i +v ics +trail blazing +loc ation +lati f +islam i +geh ry +ff xiv +dai quiri +chipotle tweets +bha gw +ab end +ðŁļ ļ +tre x +shre ya +re gen +qu illo +noon an +can ciones +âĺĢï¸ı âĺĢï¸ı +wa heed +u ggs +ni et +go da +fra il +dis gruntled +app u +anti a +ak ha +un sg +super charger +quoti ent +q l +non na +ne ely +m cauley +g fx +ford ham +far ns +⼠ħï¸ı +to ke +team moth +sr x +ordin ary +mini mizing +borough market +beckylynch wwe +az an +appro ving +yiel ded +we remember +metro polit +here ford +for rest +er ne +dar la +sp rocket +sl en +outsi der +kas kade +iam cardib +hon our +fom c +fia formulae +ev is +! ðŁĺģ +van loon +fif ties +sun gai +sil encing +pop corn +p sm +ou sh +nigh tri +naam kar +el ing +cup cake +bo te +am ac +ack le +scar lett +saf ar +pl f +n pg +msi sodia +men lo +mc ps +lu thor +h hi +b sn +ature uk +voice less +uttar pradesh +qu raishi +pover ty +l fi +kis singer +bon aparte +at eli +sur bhi +re designing +ma dan +ha id +fi stula +dra pe +car ded +asi mov +pear se +p tl +infu se +enor th +clu j +chri scol +cat riona +tr d +thingsto do +tat u +sil vi +schaf er +q at +naz ar +man ts +jab ari +fi ddle +baby boy +al politics +turi st +sur ly +re purpose +pare ce +men dy +ku ching +iso m +anime expo +ag ung +a achen +ðŁİħ ðŁı» +âľ ı +Ð ¸ +pesh awar +pe plum +n fu +liqu orice +inte stine +ingh ouse +footh ill +áµ ī +vegan uary +skep ticism +oo p +gor on +ak at +ak ai +ðŁijī ðŁijīðŁijī +the t +sport ster +ph ire +n fs +cere digi +artif icially +v rs +l bor +eri ver +cant stop +bead le +bao bab +ðŁĶ ĭ +ðŁ¥ Ī +ner dy +medi ab +fly rts +f ty +craf ters +ar dern +wl f +sr hr +s ft +mac ros +id it +hard man +ham eed +co da +boo kie +arri eta +sketch notes +pr u +o tor +granbluef antasy +co by +universal hub +there samay +spor tif +ri h +pper ton +mal le +ike ja +deut ch +audio visual +ati ans +sar ai +mik ko +fal z +dest ine +cow bell +carav ans +ðŁIJ¶ âĿ¤ï¸ı +âĤ ± +sad c +pari shi +no won +me ads +de vious +ta ÅŁ +sal one +q h +oo fficial +friday fact +easth ampton +aq a +v sk +sch ap +ras mus +ot us +osteo arthritis +orangu tans +concier to +cit ym +ah s +un loaded +sidd aram +le as +ha gger +gam bar +ðŁĴĥ ðŁı¼ +un spoken +tuesday tip +native plants +gran blue +fic ci +cart els +ðŁİħ ðŁİĦ +à ¬ +wi ggles +sheph ard +sar andon +saku rai +lumi ere +human e +dapp er +cal med +th abo +taylor made +po si +mi ston +hoo ch +freedom of +ational park +ai lee +sophi abush +sc mp +quick silver +han teo +ðŁĮ» ðŁĮ» +sab ah +remedi al +knick er +exc els +dau gherty +alex ia +sque e +matri mon +mad di +kun war +hell raiser +har uka +gi es +evolu tion +coo ke +bell at +ari elle +ak hil +active wear +tak sim +mari ab +kun dal +gar cÃŃa +con esto +click er +thir ty +sub strate +ra ye +pro league +p gc +gc se +gain with +ct n +consu mes +vi ks +stupid dope +smi a +sfor th +lifel ong +kha bib +ga ea +den o +brink ley +army selcaday +ðŁķĬ ï¸ı +ðŁİ ² +ãģ ĭãĤ +ww f +wheel er +surrog acy +squ int +marc ello +lolli pops +ic ole +chel t +travel with +read ying +fur ness +ey elids +evening standard +d ll +whe e +p ks +om gggg +logi stical +hun gama +er ve +cor ked +brig ades +book loversday +ðŁijįðŁı» ðŁijįðŁı» +wh ockey +tu ttle +son ko +ros anna +non i +in atureuk +tr f +sk ated +scri pp +mad verse +jo ked +i bc +er ri +daph ne +collec tion +aw ood +abdu laziz +ãĤ º +vas und +vapor wave +mo res +li ger +is ing +intru sive +ak mal +ä ¿ +ãĥ Ŀ +weather ly +w la +schen ker +ruth ie +eye care +eco tourism +di ap +cross stitch +benton ville +barstool bigcat +ati que +af re +ad ama +é Ŀ +ni ghty +lon i +kk u +funko pop +ev oo +ec at +che alth +az quez +polyne sia +nat ge +micro fiber +mi o +manag ement +king sof +its ch +enlar gement +emer gent +e od +barri er +acor p +teas poon +tb sp +stat t +squat ting +r fp +pas cale +p flu +ma el +jawa har +da id +con ey +vo s +sa un +goo ding +g andy +cogn itive +y dd +vis ser +tri m +su pe +so ared +six th +rizz oli +mi kas +kat arina +gulli ble +as pin +alexand ri +tri fle +tomi ho +sha in +nn nnn +mand ar +j ink +gu tenberg +discover ireland +c kie +weg mans +wedding day +v ail +so tom +music thaman +kil i +ka ke +ci el +bt cusd +be wil +âĿ ģ +under side +q b +inquis itive +har relson +gut feld +forevery child +duc ci +can ap +ag un +Į Ģ +wee per +wc ws +spe w +ri ra +pimp les +mother nature +min seok +leav y +it aryan +ir k +day um +cristo bal +cat acom +alti ma +ty pos +off beat +nc su +in tox +hur ri +gow dy +go an +edu c +d mn +ber ly +low country +in set +hom ey +help forheroes +gr out +fl ung +f enty +elimin ator +bro ly +bal th + ± +wag ner +sm ell +iv ana +in ds +hi ga +ha vas +fed cup +fe sts +f mcg +eigh teenth +daw es +can arias +âĿ¤ @ +swa hili +surrey bc +redd ick +camar aderie +animal cruelty +vali ant +shou jo +dun lop +[ [ +twitter bestfandom +sho spice +ma al +ke eler +ju les +food photography +f locks +dangan ronpa +responsi ble +oh no +octu bre +mo leg +can el +bri dle +ad ream +talla ght +qu ens +link age +la de +clam ps +al maty +ðŁıĨ # +ver dad +su i +ringof honor +mix tape +mai dens +lem ire +cen se +ber nd +aw ww +tom hanks +home opathic +dis ick +bethany mota +bahra ingp +ait ken +ðŁİ¶ ðŁİ¶ðŁİ¶ +zi pline +twi ggy +stead man +ss aint +sd d +sch ka +preven tion +mike brown +l land +!! ?? +sle m +senate gop +os an +heire ss +gi ii +fo w +bur ney +as wan +s ja +mour a +hump ty +cutt ings +cra w +an ky +sp ed +running man +pdx traffic +digital isation +deple ted +church of +staf a +ss j +soom piawards +se der +pete buttigieg +per f +le ym +burg laries +avi va +ar thouse +ðŁĴ ¿ +y lona +to cks +ss mith +sam thaan +rec en +pro bowl +overs old +euro pa +vaj payee +or say +of m +myle skennedy +methodo logies +ko jo +history teacher +gu j +dre m +cap ella +bun yan +s apol +parti do +ju gs +hun za +dan se +bobb le +yar is +skir k +laugh ing +j ell +hoursof lemans +fram ingham +da eh +ch anda +u ab +tam pons +re pair +ne ko +kw o +good time +ag in +we have +renfre w +qu att +mul cair +jeff ers +cater ham +view points +sf su +kindergar teners +gartner sym +el ong +c wl +br rrr +ðŁ§ IJ +å° ı +ro then +pil bara +olo red +my heart +mand ates +ma ith +barbe cu +adag gubati +ad oring +( £) +ðŁĩ¬ðŁĩ§ ðŁĩ¬ðŁĩ§ +tra dar +my ung +move ment +br r +blogg ers +are z +aller genic +âĿ¤ï¸ı ðŁĺĬ +womens fashion +ton kin +rakh ine +rajas than +lich tenstein +i ad +g sx +exc elled +eli se +s blog +l bi +kine sis +is ometric +enthr alled +e il +duc ing +dri zzy +clar issa +to pic +summ itt +ridd led +mag nate +fi anna +eu er +book my +ali enation +---- -- +yuv raj +von ne +tn r +ten ey +shin ing +rhe in +po to +pen ed +new book +kel len +jack sons +flat bed +el ah +cre do +cor nered +zu g +wad ers +sub hash +smol lett +p sa +mm c +mar rakesh +gir is +elast icity +disney channel +carbon ara +bi ar +anc ourt +sunny leone +mv g +mun roe +meat free +mac y +he matology +ev enti +x cel +us agi +stock bridge +star board +r pd +mad hu +le ma +boise state +african union +ê°ķ ëĭ¤ëĭĪìĹĺ +好ãģįãģª 人ãģ¨ç¹ĭãģĮãĤĬãģŁãģĦ +yu mmy +win ans +ran jan +no du +n gay +mk x +massac red +koo k +aidan turner +adi um +ðŁİ¨ : +ìĭ ľë +à¥ĩ _ +sunny vale +ra jab +pr d +kat un +ign ites +harvard biz +es y +deep a +at own +ðŁĩ¨ðŁĩ ± +toronto fc +sc v +re ni +ot g +neymar jr +mar mot +kal on +io p +equ in +echo ing +c du +bis i +beau jol +barric aded +amar athon +x ps +ts wim +super car +magical kenya +l pa +kri eg +be sser +waziri stan +save slives +pro kabaddi +or t +mü ller +mi ui +ha zza +em es +animal sin +âŃIJâŃIJ âŃIJ +united nations +tc f +se gg +nsp cc +ka o +inter modal +gill is +fri ar +danis notonfire +ba hru +amen ity +like wise +jard ins +ill at +idlen omore +gwyne dd +go ol +cess ation +am ay +nat su +ga vel +fran gi +dun n +ati va +and el +tur pin +sh ind +mo hr +ma ggi +king man +heart burn +h fc +glu co +f ll +b nw +am ae +affirm ative +,, ,,, +video graphy +sal esp +n º +jo er +jap on +f ylde +bu a +anush kashetty +win chester +scon to +no tyour +m é +kual alumpur +juli anne +ju r +four seasons +dev itt +cur sive +chiang mai +asp ca +am ico +ad joining +sta c +kee ley +jo i +hal low +go y +em f +dill i +diag on +cb sd +cal o +war ring +survivor series +stol l +stay strong +qu y +moo kie +m ally +hospit able +girl problems +exquis itely +drive in +down turn +d modeling +co pping +cad y +br ough +b ould +$ , +visit portugal +subver sive +run ny +oti v +musc ulo +k illie +in habit +hand stand +fil le +ro coco +l ge +facebook live +eu vre +black friday +thrombo sis +standre ws +r gs +mie expert +lu sa +fra sier +epi genetics +bant u +artistson instagram +ðŁĴĸ ⾨ +o sos +ipo h +cardio logist +é ¦ +white wash +ri que +peter bilt +pencil drawing +numb ering +mon tag +g ell +dr t +cra shers +ani mo +íĶĦë ¡ľ +travel skills +ro ped +fore sted +ed is +br l +bo ol +sn out +sd c +reli eves +ram in +ha usa +amp us +mar itz +initi ates +dero gatory +caru ana +bol i +tian anmen +shiv sena +opho tos +ol ice +li feli +den iro +ann unciation +zar o +xxx tentacion +sto cha +spac er +sco field +rai pur +no ctis +jam mers +gra fting +dent on +baira vaa +- / +ðŁħ ± +z onal +z ille +vagab ond +shoe box +regre ssive +kel d +jo elle +home stand +can you +anup amp +play mobil +p á +kra uss +jame el +e uk +bra s +su med +sto ys +sting rays +stat ue +remember them +refe re +produ kt +o asi +mary jane +hong ki +el mira +conduc tive +char io +bu kowski +ðŁĮ¹ ðŁĮ¹ +tibet ans +tcr wp +small ville +por tal +love whereyou +ie bc +gor ham +communic ators +cl acton +cat ford +at an +ðŁĩ° ðŁĩ +stor ies +shattawale gh +re sorted +pad stow +p wr +grey cup +fou cau +for tis +curren t +co sted +chong qing +ta vish +ta illi +ste arns +le murs +iri t +gad sden +ep r +be stow +appen dix +amalfic oast +truc kee +ther un +th nk +t fios +mon tero +min ous +har yan +ep fl +dy kes +de fund +tim kaine +rose mont +li vi +la usd +jo vic +ghan ai +boston strong +am ama +z ent +ush ka +swan queen +mis match +millen nia +jaw ans +el va +an ee +admi rers +ro lo +optic gaming +much love +i ha +dot ca +dig by +ag al +adul ting +ç§ģ ãģ® +v la +tun bridge +sur bit +no ho +la fc +jerry brown +gil insky +fb pe +eli ad +do good +dark matter +ban de +yu my +w thr +temper ament +p wa +ou ija +no sey +napp a +houn slow +h cp +goo den +american art +% + +south lake +ron an +reg gie +marth as +kir in +de a +cu toff +\ \ +ðŁĹ º +ðŁĩ¦ ðŁĩª +te vez +shel don +ragh u +push kin +passi oned +nar ayan +illi ons +hol din +c sos +anupamp kher +ë ¶ +z et +vo stok +vo gt +support the +sr ar +mour ners +laun dro +k cam +co hen +class ically +pss st +hassel hoff +fa erie +black mon +ac tin +tribu te +fil me +expir y +d mc +atur n +voy age +old field +mkh itaryan +lean er +jerrybrown gov +is enough +e ft +c suf +bu ca +batter y +b gc +ðŁĮ Ľ +sae ed +ocean front +michael j +k ross +fla ky +compost able +au den +u tion +t ite +royal baby +rod ger +och re +disproportion ate +devon hour +dand en +da ar +bas ins +b tec +pe mbs +khal e +gra b +dun ia +clo gs +be ath +ter man +shut down +re pose +raj endra +quar te +national catday +mir i +ma q +lok sabha +hyper rts +ðŁij ¹ +super woman +sris ri +oro sa +mother sday +hill sdale +eni us +disc red +bel aru +bar ring +av as +amble side +ãĤ ¨ +rot ational +pseu don +podol ski +om l +no body +nap alm +high tower +cran king +brew pub +ðŁĶ ij +Ù ī +zer os +ur ham +slo pe +sl ings +om phe +mel i +flyn n +erit rean +bri and +bel lo +aly cia +ag ap +tari an +sc ant +plu cked +p lowed +olu tions +o kee +le sm +inter vie +gall er +ent ice +ax x +wo b +ugand ans +nit ty +m kii +lin field +ine pt +bo sco +autom ating +as ketball +ane gi +smar ty +lingu ine +bb cle +ðŁijī @ +tinnit us +m vs +ko ons +k lia +ic as +gg mu +de ren +camel back +wk bw +sta ad +so res +is ola +en ps +cy n +ced ric +ber gha +te f +slur pee +me thinks +le ann +hor des +energy storage +alternati vely +ai ims +ðŁĻĮðŁı¼ ðŁĻĮðŁı¼ +war an +wai sted +man ip +madri gal +de ye +bon hams +stupi dest +ri ghty +org and +just for +j na +ipo b +gra af +fab ol +diagno ses +bio dynamic +[ + +ÅŁ k +ro bby +jewell er +exacer b +spo ty +skop je +sid ney +shadowhunter stv +paw tucket +om nia +n ong +lyce um +ingu ishable +free form +concep t +alco tt +sh weta +s sex +morpinoy biga +marilyn monroe +ic ol +di allo +alli gators +sop hila +pa hang +mascar pone +frapp uccino +clar ita +zil low +up bringing +so ch +sin nott +n q +man os +j tbc +i will +han an +flamen go +far go +doctor strange +cull man +ash ima +ad den +tutan kham +seren o +ry zen +ru mba +om agh +monol ith +cu ous +const ab +un read +demon ium +bismil lah +ap as +ab aby +te g +li que +jo son +jim in +jal isco +int i +gran by +g fw +ari at +stil t +sor tie +sof ie +re th +gw ynn +flex in +fa ireland +dispen sing +cra in +ðŁİīðŁİī ðŁİīðŁİī +ãģ į +tre mors +terr ine +presidenti al +moder nis +kkkk k +gyne co +gran ul +afro jack +unic ode +soci als +sh ec +scal ise +re entry +new stalk +m wan +li abilities +le athers +gur ung +gis ele +cont ouring +ak l +a ene +tt g +ther mal +t wor +nishi kori +cor tland +co sheriff +us vi +tsn hockey +sj u +p lowing +notori ously +menopau sal +mal ted +ma thers +kettle bell +is best +constitu encies +che ts +with standing +sport sm +op al +nh k +hu mps +edmon dson +call i +arte m +ðŁ¦ ĥ +scho tt +re ticul +ke ele +hu f +hi ma +ar ton +ar os +tim m +sc su +ru mp +oc sb +healthy life +ct u +basic income +andro ids +whitec ap +reck oned +om ir +louis ville +ju mia +ó r +sanje ev +ran adaggubati +ra ki +pil la +deci duous +carab ao +bon aventure +bel ight +the fall +o ineza +keyboar dist +impeach ed +fais alabad +clow ning +brooklyn nets +app lec +petri e +n ma +i isuper +foot loose +chakra borty +cal or +ar thu +Ùģ ÙĦس +pag an +n pm +n co +lind berg +jenni e +g outes +du ma +cul ts +! ?? +tic es +scot gov +pad mav +mam a +helsin ki +fnf jb +el am +co bbles +volu sia +per oxide +om is +new lands +hound stooth +ga eilge +so happy +re agan +o en +forthe kids +ciab atta +bal ham +af un +vicegand ako +v alli +sen i +mete o +living room +ko dy +hust le +harle quins +be ka +ag new +íĶ¼ ëĭĪ +Ê ° +wr d +usain bolt +under card +theori sts +th century +nu ance +n lex +mccul loch +light ness +fridaynight lights +e stor +! ðŁĺī +ìłķ êµŃ +Ñģ ÑĤ +yn t +we ald +ster man +pro vocation +nik ko +mi gli +max on +hol lins +fli x +environment alist +en camp +cho reo +ï ¹ +wb g +sh eroes +ridd ance +paragli ding +obsc ura +james arthur +deep ens +deb au +char geon +boot suk +book shops +ar px +ami sh +am ay +vit ri +sc ad +lac key +f ateful +do your +cabo chon +blo x +ann el +am ato +we sty +sprink ling +rain coat +ong c +le vy +ii iii +dr gn +ìĿ Ģ +wet suit +we imar +mag ica +la ff +kaz akh +flo of +dul wich +dinner ware +differen t +dan and +cellu lose +ðŁĺĭ ðŁĺĭ +tri dent +the grand +scro ssed +p sac +me hr +mark us +marau der +k up +in col +emir ate +dam o +com mi +bu shy +ìĿ¸ íĶ¼ëĭĪ +wa iling +theli fe +so der +provin ce +pol ter +pe ake +opening ceremony +nca atf +kei thur +bi v +av ast +andaliolo isa +wol le +shi kari +pau ld +mon son +medit ative +iq aluit +im pur +cri st +copac abana +un true +uc b +over heating +kale y +iisuper womani +ho si +d wn +cu tee +cler ical +bro gan +bemid ji +ðŁ¥ µ +spiderman ps +sax ony +real james +mu see +ka po +j ure +iam ahmadshahzad +i y +esc uch +vic kie +razor back +nar rowed +mat ts +mangan ese +kil ometre +iisuperwomani i +da ws +beat down +bb d +ÙģÙĦس Ø· +vil as +tu dors +p ka +mis ch +len berg +ep onymous +ðŁĴ ł +slo o +k gw +freight liner +discre dit +chi er +bur sa +ben ched +uni bet +singapore gp +noo se +jingle ball +gra ppa +black cat +anu rag +west cdnag +up sc +ru ti +ros setti +r mh +philipp a +oun ty +gau di +effe cted +co ward +ðŁı į +⾨ ðŁĴķ +zer matt +startup india +shindi g +re test +prev ailed +mill ones +light box +brim stone +and ry +al ene +á º +wre tched +the g +sky pe +hungar i +depic tions +dan bury +cra zier +camer oun +as kin +academic ally +watch out +theal thy +mac ca +honey moon +f wm +esp y +dar is +ci as +canadi angp +wil ko +wa hoo +kam ran +good fellas +fluctu ations +chi o +ch ka +ìµ ľ +wc pss +siddaram aiah +gi allo +defl ategate +bla zin +a ima +san de +pit as +kraf twerk +hel ion +fi p +ar bro +ðŁĺ¡ðŁĺ¡ ðŁĺ¡ +ðŁıĢðŁıĢ ðŁıĢ +ra it +ra gaz +om on +fri sky +bur rata +ðŁıĥ âĢįâĻĤï¸ı +wa v +ukrain ians +tor ta +sn ook +simpli fying +side bar +reflex ology +rb w +qu ers +phe asants +nor r +no ws +ko tak +ko on +fu tura +epi x +ba chao +ع ÙĦ +scy cling +sa ini +resu ming +ph ons +japanese gp +gur dwara +farns worth +bitcoin cash +å ł +tori es +ti ga +lgbtq ia +la chey +ko si +im ro +gol drush +fi ley +fet sy +fer ran +air force +ðŁĺī ðŁĺīðŁĺī +tri vedi +tor tell +sch mid +panamap apers +ley te +guil ty +golov kin +cla uses +canary islands +ber mann +as qu +a he +tun de +miscell aneous +may es +chi x +ca den +av alon +ash anegi +ðŁIJ § +world war +tid bits +se w +padu cah +goo den +girls generation +gir ar +exter n +exemp tions +ch alo +bur lap +bi ja +b mt +trich y +she art +sb r +r ls +po it +muhammad u +mer cies +low poly +kame ham +jür gen +chab lis +bil ingu +bi ot +bal y +say ye +patt en +ma this +lam pp +jag r +frag ility +erec t +di maggio +creep ed +ch re +ati ous +ðŁĴĻðŁĴĻ ðŁĴĻðŁĴĻ +vi per +ss in +r ü +par do +light stick +kidder minster +con don +ab uri +ðŁİĤ ðŁİĤ +ver million +ti wa +laz uli +la sc +irregular ities +gin za +cle ve +all ace +after glow +z k +snee zing +nfl x +moun table +infiltr ated +fra ilty +fle ets +com pa +cec ile +car n +íķ ´ +us mle +sla vic +se ema +marys ville +it weet +hal ton +eng ler +dd t +book bloggers +ب ÙĬ +v na +ut sav +t ville +ha sty +gra di +fl ore +bab i +b bie +ar na +ðŁĩ®ðŁĩ © +wof ford +ta vis +summari zed +phil a +mccle llan +m xm +lit tering +emir ati +can tona +bet te +an tares +ag n +pepper corn +motor bikes +im poses +enfor ce +cal de +vin tages +kin ge +ja f +go stanford +fac ials +chand eliers +reflec tor +ptx official +paleo art +national signingday +g gio +di am +cooper atives +ple in +kun is +fc cla +deterior ation +c pe +bel cher +ann ac +an dean +thel ance +sor o +sh asa +recur rence +play set +pee bles +n ars +entertain ment +citizen s +alej andra +reli ably +reason sto +kut cher +geograph y +fran kin +fet tucc +edwar ds +delu ca +d ger +å± ± +yo e +wl wt +sierran evada +sha ha +sear le +se pp +san gh +reinst ate +nic hi +gratuit ous +gra eme +ephe meral +chep stow +c mc +ar able +>>>> >>> +y als +wi el +trum bull +shri mp +rhe tt +pen tathlon +lle well +in organic +in accessible +head dress +haz ara +e gi +contrac tual +compu ticket +swim suits +sledge hammer +pu di +marc in +li smo +la ins +cor nu +chick as +snap backs +sal vo +raz ors +ra pped +he ms +gab bi +fr ith +equestri an +ek won +desol ation +cw m +val paraiso +te lec +tam ron +super fly +situ ational +sh k +mou sep +cu da +âĺĦ ï¸ı +ske pta +qi ang +ol sson +np fl +mau i +integr ity +ig ns +design week +cor mack +busc ando +ba ile +ðŁIJ¶ ðŁIJ¾ +ãĥ ¥ +ul ina +tast ing +koo ten +ah p +unil ateral +spring bok +purple pride +ox tail +n rs +fr itter +bur ry +arch e +ak am +uk sopro +spec tion +sa kho +ratt an +po ste +new sa +kil bride +hbl psl +handmade jewelry +con nach +bill deblasio +toxic ology +ortho doxy +new car +lar ke +godof war +f of +dar m +admir alty +s bay +parthen on +paedo phile +or ly +mo et +jackson wang +j se +iroquo is +gor da +best dayever +! ðŁĺİ +zig zag +that describe +star fleet +mac allan +interven tional +es in +snu bbed +precau tionary +orlan doc +no one +gill ard +gar van +fl icking +fa ver +cal dy +ar shad +vo ix +stru del +nu ances +ne in +kro shan +ken seth +ciu dad +bel fort +aver ted +âĿĦï¸ı âĿĦï¸ı +var ney +th il +swi sher +spec ter +lom l +kin ok +itys c +frag mented +de in +bag ong +tz bach +reb be +re issued +mar gie +lu ff +fr ack +fil les +ex oske +blue birds +ðŁijıðŁı½ ðŁijıðŁı½ +thri ve +offici ale +m schar +freak show +bour n +al fie +acre age +ú n +wag en +stevien icks +span dek +si bs +scu f +s not +olive oil +liber o +kitesur fing +ers ack +car lin +capre se +é ĺ +rhy ming +red dead +pe aky +pal ay +museum modernart +mc coy +la very +la gged +ker b +griff on +eis enberg +condu cive +aa and +ãĤ ī +puc cini +north star +lg b +leadup chat +jo h +intran et +hu sk +fur nishing +with holding +new ze +mo tte +met policeuk +dec aying +baseball hall +ar utz +alien ware +à¸Ńภĩ +uit m +sur renders +super charge +ra fe +me than +ger rit +cow en +tor ri +tal lied +sriti anne +sho ah +mc master +lun ching +li sab +is my +hay abusa +fol ger +tremb lay +ni se +moss ad +kry pton +as amoah +ann enberg +t ween +ni alls +mul ls +may the +loves you +leon ardo +i ol +har kin +circum cision +æı ı +oper as +moose heads +heart beats +du athlon +assassin ate +armen iang +rott weiler +north york +micro sd +im ic +h ant +gru ff +gaff ney +cre dential +bnpp ari +beech craft +?? ?" +ti mon +lose weight +ist ically +house plants +e ef +dun k +cosmo logy +cam ara +apart ner +âĿ¤ï¸ı ðŁĴĻ +wester os +theri ver +t wee +ke yser +inser tion +f pga +expla iner +di dd +. ðŁijį +stair well +pu ckett +por ky +pat nam +fle es +der ay +tro tt +ten by +nutt all +daf bama +comm ends +car les +c pg +ðŁĩ¦ ðŁĩº +z eller +sj c +palmer ston +no bu +ni le +flu e +cr é +am jad +ìļ Ķ +up ad +q ay +pop star +ov sk +match maker +lef ties +jeb bush +in ked +ge un +f ite +eco boost +brit on +blancpa ingt +asym metrical +ðŁ¤£ðŁ¤£ ðŁ¤£ðŁ¤£ +w yer +to shi +s fire +q nh +main enatics +hou this +her ons +fem me +c tb +ī ´ +winnin gest +ro is +pon d +oned ay +mun na +ld l +ge ant +focu ssing +ev sky +do gan +ash y +weare r +th app +santho sh +sal for +ram allah +r bc +pin scher +noun s +mün chen +keithur ban +head stone +foodfor thought +brent fordfc +andalu sia +thalai vaa +suspici ously +sc tweets +oo re +kon dap +kin ase +disney parks +cru de +com mie +raje ev +mid life +me ag +mari posa +listen ing +ka ha +jet lovers +cer vic +yor ton +walk the +tell tale +son nen +sit ka +scru ff +pro vol +ki x +di ese +camar illo +au sunions +Ù IJ +t ards +resource ful +nb k +moo ch +mar ÃŃa +manof steel +il volo +ger ardo +ate x +ag allery +* ~ +yo p +tten berg +thunder ing +rab bi +play boy +pe ddling +morph ing +menstru ation +he ster +fr ito +eu se +davi dd +beauty blogger +soul talk +ny ong +mar ple +jer vis +gho se +cul ver +atle ti +un successfully +refuge ecrisis +r sf +j aff +gram bling +enter tains +c sharp +barbar ians +at well +v é +un ending +t pd +sanc tum +reverb nation +per c +only at +mee eee +lot sa +kel man +dam ask +anim o +thumb nails +prote afire +me ir +bull itt +bi ase +ann ad +tele port +sthel ens +sa way +raw ls +o cho +morg ana +h inged +gra hame +ch ha +aj as +un sc +land slides +cur ds +blizz ard +ar ching +ãĥ İ +vit t +un ison +ske e +pin ter +nu ma +melis sam +intox ication +gamep ad +fluffy fursday +emp ties +dalla sma +compens ated +wj xt +te ther +rhy me +real saltlife +ony c +meso theli +ke et +fin dyou +boun ded +yuv strong +sco il +pr y +pp f +nys na +her on +gran canaria +fra iche +d uni +aureli us +al un +al anna +tal lu +ro am +naamkar ann +mer ges +lit trell +ko b +hi dd +grave stone +ge c +ed an +duplic ation +cast ings +bio energy +bi p +thr ones +thail and +s don +pom pano +pig gies +moo res +it wednesday +is al +fl x +faz eclan +ash tray +af y +acham pionship +zz ini +true blood +sa dio +ra rer +new england +ne who +mi h +li fer +hemorrha ge +free standing +za idi +yü rek +r lfc +mo sby +mar gol +bra ined +ber gamo +ðŁį· ðŁį· +oxy moron +looo ove +instru cting +g ago +curti ss +arutz sheva +ali ga +al ake +whit eness +state police +o po +mark s +geh rig +embed ding +debun king +ca ren +ai red +âĢ º +villa ger +sweet dreams +sub tropical +produkt fang +pick up +partisan ship +mus grave +mouth ful +mob ley +empha sizing +avenger sin +ar ron +aci di +wad dell +stor tford +ra ym +po irier +keep going +kari shma +iam andalioloisa +hammer stein +crossfit games +whistle blowers +toyo tar +it w +he on +gly ph +a era +ze c +volkswag en +vivi dly +t act +stru tting +hu erta +gro ssed +giro ditalia +ep t +bb r +amazon uk +alto ona +ðŁIJ Ħ +prime video +n sm +kings way +shay mitch +rocke ted +po lok +nhl flyers +la bon +k arel +don gh +cheese steak +b tt +atay lor +ah on +vocali sts +stocki sts +soir ée +sat ay +n bac +model o +melo dious +gha stly +an en +a ita +âĿ¤ï¸ı ðŁĴĭ +sk its +mar ky +ic ke +euro cup +dge e +whal en +w cnc +the jeepmafia +tear th +murray field +lakel and +go cats +fur t +congr at +v air +ther mia +snoqual mie +sk ri +j q +geo de +f locking +av ak +art z +you sef +vi gn +tre n +t ür +rejuven ating +maje stic +lay er +im passioned +fel ted +fac up +cruci fix +rac on +prop ylene +prag ya +mer ion +maj lis +lumin osity +li fer +ko eman +gold cup +families belong +et ly +chri ssie +where i +pre sa +mues li +kit sune +i wan +hat sune +en cer +chang a +soren sen +ha ste +brook side +ade pt +ãĥ ķ +water bury +w man +new some +ana x +ðŁĮĪ ðŁĮĪ +tunbridge wells +thu le +manas sas +lur ks +le ting +k mov +ho ss +hel ton +ab ell +ðŁį Ĩ +âĺ¹ ï¸ı +sch ir +lan dof +hu ay +ga ol +ðŁİ ŀ +yl er +tre ati +th ay +tex pre +spla shes +shan ah +scar amu +rrrr rr +ody sse +kuz net +ir reversible +cu fc +con tri +w ys +stam pa +perfect shot +panam era +lom as +diab ol +chief skingdom +act sof +ðŁĺ± ðŁĺį +ti gres +sing along +silk road +mal vern +kis sonline +gujar at +desh mukh +anne cy +tele phony +re vis +pa ak +mi kk +m hc +hy nd +hol stein +ho garth +gry phon +cute animals +classic fm +ble ms +sk ind +scre amer +music producer +endo don +em en +av anti +rhode island +medicare forall +md x +la belle +ame dia +albe marle +ðŁij Ĺ +than h +si f +pre viewed +o sinbajo +mm k +is an +ge de +e del +common sense +clown fish +bas al +ana is +ìŀ ¥ +y vr +vie ws +thel and +sy l +pre schoolers +j omo +hoo sier +fotogra f +elong ated +áµ ī +pa inst +na han +cenota ph +wh p +ti vist +s words +releasethe memo +mika ela +le duc +eng li +der n +brown field +af n +a version +âĿ¤ï¸ı ðŁĴļ +wen de +mosa ic +margin ally +ma zi +ku ll +harb inger +cocac ol +ar apa +apl enty +us an +to zer +tai me +sick lec +life lessons +kirk uk +kat elyn +ga an +ag lio +å¤ © +âĸ Ķ +util ising +s fan +provol one +per n +mon astic +mi go +lu mens +hoo die +es me +dare devils +ais les +sph one +rou th +p nb +over lapping +or ds +norwe gian +ir fan +hus q +goo fball +d zi +cro ce +cap i +bu sk +us djpy +sy man +london er +ky un +hospital ised +felic itated +wildlife day +reven ant +re tic +po cky +pi zar +os d +ne ment +mi sta +manu ka +lin gh +g ned +evangel icals +cy dia +ðŁİĥ ðŁİĥ +ðŁ¦ Į +topo graphy +ss cotland +por tico +more tti +mor de +k q +colori st +assi es +ae o +a hern +seap lane +red bridge +ra ft +omo vie +l viv +kl f +jewel er +gg wp +falcon er +di plo +all american +ðŁĺ ¯ +ðŁĴ Ń +to ba +srar auf +mo ar +ma una +lic ht +kla srarauf +dora emon +bar is +ðŁĩ¨ðŁĩ ¿ +sy ch +pun tland +par ade +obsc ured +jiha dis +e tti +cap tioned +bab oon +tic on +stor i +respec tyourself +pu tri +p ce +npg london +mar cell +clari fying +chapp y +ca as +broad casted +nikk or +e bc +cli ppings +yan kovic +sh ali +sanjose sharks +salt water +re tty +prescri be +peril ous +la fd +jon o +cam ber +bra ke +// // +xin hua +tre mor +the david +scann ers +san gh +lubric ant +it za +daun tless +solo ists +nighthaw k +golf channel +go rey +gin kgo +diet itians +ann as +ðŁij¼ ðŁı¼ +âĨ IJ +wall ingford +visit england +tom ford +thrif ty +sher aton +rite ish +ridel ondon +persi sted +pan er +ma ir +ic ao +hi jack +g so +blo em +bant ams +way side +tor toi +t dot +stupend ous +park shinhye +mer man +mar zi +jab s +glasto fest +co fe +brah man +boon dock +apostro phe +ðŁijıðŁijı ðŁijı +zak aria +yar row +wor den +ph l +om my +li bt +hall am +ha yes +gun slinger +gir oux +frei ghter +cannabino ids +torn ado +ra fre +por th +or ban +newmusic alert +itsyour wales +celebr ation +ÑĢ а +tho t +tar leton +t cl +ri fe +mann heim +ludhi ana +kero sene +jud son +e qt +disrup tors +dg p +cr ac +beati ful +barunsobti says +au p +andre am +vet te +swar a +short ening +mob b +f hm +bhak ti +az usa +yu ba +ys u +wedg wood +smi reland +sm alling +sh m +n ino +hourof code +food blog +ebon y +dag gers +v da +tri pper +shi elding +our future +man ama +d bc +che x +candi ds +ìĦ ľ +аР» +su spending +stro ll +conglomer ate +xbox share +w had +uphol ding +k sp +ic sc +how lin +ha inan +giftsfor her +does nt +bellat wins +anim at +my nameis +lau ded +land t +la sik +jen morrison +instaf ood +dud ley +delic ately +biggbos stamil +a island +y ani +wro claw +sel enium +ru ise +ri poff +ra ab +prospec tus +nu bian +mis fortune +in focus +ich en +hh n +fl w +cru ces +bu hay +wol ds +too oo +row ing +pen nine +par son +magnific ence +m elia +lo or +kam oto +hi mm +good woo +buck ling +boy friend +volunteer ism +twitch tv +regi mental +pet te +new son +ne po +nar i +hen ney +gag s +ext inguish +ds w +balance forbetter +ay tay +abo realis +ye ahhhh +wash able +lang ham +info comm +fiance e +et n +chem ically +annivers aries +å ĵ +to ve +spon ges +si mbu +sh ir +sand storm +pull in +high rise +ave ga +addition ally +vi har +ne mato +micro max +fe ira +enab ler +conduc tivity +chri sg +ìĬ¤ íĬ¸ +pos ner +pha ge +ha dh +colli ery +ch els +bur lington +ant man +al v +un fao +storm troopers +smy the +sc ough +g lyn +fan expo +el kins +apat rick +ðŁĩªðŁĩ ¬ +pro fu +an them +tou ssaint +ti go +swi g +om iya +of champions +kir chen +gargo yle +f nr +eat eries +bra xton +bleed green +ano brien +! ?" +wali d +ti xs +sho wn +rr c +lo de +hungari angp +gott fried +forsy the +boo king +ab ooks +wal pole +ve the +te ds +str angel +pad dles +om gg +mun n +gri sw +gri mm +gl er +de bon +coom bs +chief tain +ur ls +take that +still life +re ch +luf tw +kang daniel +flood lights +enough isenough +bra i +a ek +ðŁĴ° ðŁĴ° +worldr x +we sthe +thr i +scree ching +obel isk +my elo +iz i +h sp +fute bol +em bry +ðŁı½ âĢįâĻĢï¸ı +ristor ante +pa isa +ny y +mobil ity +infor mant +christmas gifts +wy ch +jenmorrison live +gru eling +blu ee +viswas am +th ng +taste buds +t gi +rack ers +okla ed +mu sed +mr f +mol ine +giftfor her +fettucc ine +distingui shing +comple ments +c ce +wi x +wal trip +virgin ia +umb il +stra sse +s vit +re mi +le sp +feb vre +cu se +chri sl +.. ( +tw u +specul ate +ru tter +obe dient +mÄģ ori +mo dal +lan de +kla xon +g ere +clau stro +auto biographical +ac s +ðŁĺĬðŁĺĬ ðŁĺĬðŁĺĬ +ãĤ ĭ +star lings +lur k +black magic +bart news +ath abas +press release +phil o +pat el +ou lt +magell an +len ation +for progress +fin nair +cow girls +sat sang +penn ines +lo tro +kvit ova +gro ssly +fer ment +du chess +do is +b hu +sal ter +presu mp +lock er +krun gy +ing ar +er vin +effi gy +dur st +araf at +w ma +revolution ized +q r +po pl +lur cher +k ora +goo fing +atur ner +trans link +t ard +su baru +sn ba +ridge way +pet es +ing ram +escal ade +cup head +bul an +anchor down +ðŁĴģ ðŁı½ +ðŁijĩ ðŁı½ +ze an +wish bone +jimi hendrix +j post +injec tor +hom in +h ine +dr acon +beni off +baby doll +' ). +woo o +ur sa +thesp ian +mik kelsen +low o +ki yo +getin spired +for bidden +can aries +ble us +bibli ophile +spla shed +seis mo +s view +prohi bits +nar cos +nan twich +mil led +gian luca +cat stagram +bo ge +ar it +mid winter +josh groban +cat lover +ca sement +advoc ated +act now +zu shi +ratat ouille +mom oa +enti als +cou ches +college football +cis d +ced ars +bra gg +ìľ Ħë +wen g +nuig alway +no ona +mer rell +lu o +ju in +heg de +gun pla +di ma +dah lone +br z +bj ym +wal sh +teach ers +pap illon +mo gu +kru ger +k elling +impur ities +don ga +casca dia +bot l +ale y +.. :) +® , +sandwich ed +sag aftra +mor mons +ire ne +ir un +dru cker +alexand ra +aap i +ðŁij¨ ðŁı»âĢį +๠Ĩ +ze ko +langue doc +ju mma +infir mary +f ice +e ers +der matologist +cu cina +contradic tions +col ino +co ley +ay ya +<< << +w tsp +the who +sl inging +rappor teur +or gy +g cats +david bowie +bag gins +ab hor +soo hyun +revers es +r ya +ou de +maha bharata +igle sia +ford performance +ec mwf +dispat ches +cheno weth +ìĿ¸íĶ¼ëĭĪ íĬ¸ +ver tex +shab ana +reho both +rand yorton +pi ste +om ber +ol ondon +ob scen +ha chi +visit britain +thel ost +re affirms +mor g +m sm +let the +jan itor +im pedi +fuer te +dhoo m +cbc mb +ben and +armeniang enocide +te vin +ste aler +q aland +lec rae +lat ent +disappear ances +be to +as kew +alab use +shar ky +sc r +sat ter +sa ward +resurrec t +reper tory +ra ver +multicul turalism +liber ator +kri sh +hor acio +hell yeah +hal lie +euro basket +cin der +amar ket +tu dor +sou k +omor ph +mar tha +lo pe +hur t +bm f +advoc ate +un willing +un licensed +rabb is +puyal lup +par rot +hail wv +gri ss +en cased +confir m +chee ch +am bi +six teenth +rescue dogs +remo vals +reg gio +pl under +painte d +la ve +k pl +iti ves +exempli fies +cron ies +ë¸Ķë ŀ +vogue team +tv g +opening night +mcgin ley +god less +er asure +de bian +bo whun +ãĢ ½ï¸ı +sy bil +ri me +po tro +geaux tigers +el sin +dahlone ga +comedy central +clu be +buy back +bl end +!!! ... +w yan +turk meni +pu ro +prodig al +pit bulls +mat ric +k appa +il yn +he heh +fil er +f enders +wk d +qu ora +fau t +cou ps +co stanza +ann avy +âĿ ¯ +min x +implic ated +handker chief +ep d +e ker +bel fas +ac umen +sc al +pi de +new man +inspir on +fox tel +clean air +chihuahu as +c zak +ðŁİĦ ðŁİģ +word press +thisisd sp +signi fies +manne quins +gra ven +dingh y +bro lin +s kie +puzz led +merci less +gro mit +fier y +expand able +equi val +but toned +ðŁı¼ âĢį +yel fie +y tes +lamar r +gro m +for peace +cp t +ca str +beat box +bak elite +wrong doing +resusc itation +han ford +go bi +depart mental +cycli c +an um +á ī +ri shab +mce wan +kang an +encoun tering +bat ra +sh rey +li at +lean startup +cliff hanger +chir ping +c va +apple podcasts +ad trip +absur dity +:: :: +w ku +so das +sa ku +prioriti zing +netflix uk +mar zio +ken n +dermat itis +ðŁĴļ ðŁĴĽ +whim sy +vapori zer +tre ks +tele scopic +scam mer +pre sci +me tag +iri on +feature tte +be f +a ath +⤠µ +work ers +sur ry +ri us +myth busters +lu ll +coo gan +cerve za +beauti fication +aty pical +an az +zen fone +x xiii +ti gre +lor n +l to +eccle stone +der t +bo fficial +vir gins +uten sil +to efl +sub woofer +pur nima +nak ba +mo hawk +guatem alan +experim ent +encom passes +botan ic +bad shah +servic enow +pass age +oxid ative +me f +ma us +ho ga +gimb al +gent ile +ers ons +doro thy +burle son +boo zer +à° ² +wipe out +vari ance +sc c +o boe +gun z +gran di +country men +cohe sive +answ er +á Ĭ +way nero +ti mur +shuff ling +ple c +hunter hayes +basti en +bankno tes +vast ava +u hs +pun chy +pent a +marketing strategy +hein z +frau ds +foam posite +ee a +chry salis +bo as +ðŁĶĬ ðŁĶĬ +touri smireland +sarko zy +print maker +kian lawley +hu ddled +du sseldorf +chron ological +cat box +y ali +sch lu +sc ast +mun go +lon er +jel lies +g agar +dat or +bo ils +ber i +andro gy +wood burn +winter olympics +vene w +tre monti +supp le +south wold +plu shies +over shadowed +temb lor +st w +spra gue +re discovering +ol lie +ma thur +lall ana +k tb +ero tic +burger king +biz party +éģ ĵ +up ta +thisi se +initi ating +imag ed +go army +crooked hillary +cro well +arab i +amher st +ðŁijį ðŁĺĬ +zi ppy +x ctf +we ah +trickor treat +teix eira +reluct antly +mamat aofficial +freder ik +for gives +fly er +clair voy +acy cling +ðŁij ° +si ppy +schu yler +in voluntary +h ite +gl ories +fac ial +as you +ark ali +um bra +sta dio +o iled +nad ler +charli es +abol itionist +war io +t ct +squa shed +no way +megan e +kon ta +in doctrin +howar th +az uma +ale house +acti ves +ach an +stand outs +sear ing +haringe y +gr ana +exoplan ets +cab ar +ap ach +stre p +square enix +pin ky +pet its +ne gre +morph ine +mer on +e on +dro pper +d pw +ar aj +adi o +yu eng +sur facing +sc ampi +sa if +rut ledge +men cap +im practical +charlie puth +ze tti +saw grass +rox ie +mul ling +hu w +eng chat +dr j +club man +bo tics +av ell +u cm +tecum seh +re runs +op on +ome trics +man x +jes y +in laid +grazi ano +ch ella +cal derdale +all red +ภ¸ +sea water +mo les +lauren t +kra us +ground hog +feder alist +da es +x peri +st elle +pati os +o tra +nt ds +n bl +mitch el +kingof the +ki ley +hom en +ge tre +dise mbo +den sely +columb o +carbon ated +af ra +wa ilers +ro pa +on top +ne f +mou ss +m hs +ind veng +fan boys +bas kin +" ..... +ju manji +gr w +governor ate +gen flynn +fl i +ep au +cont ador +ca di +aa ja +zu zu +ta kingly +special ise +sc lass +s fe +ra ic +luftw affe +hack en +gar bo +and field +am li +alcan tara +tb il +tamiz ha +om m +ne ma +natural gas +li man +hand s +fil min +boom box +vol l +un apologe +swif ties +ste els +sar ro +nar rowing +j ll +foo te +ei leen +basel world +æ ī +wil kie +wes tham +turkmeni stan +th wick +t asking +sil vers +repe l +re arranged +out puts +miy amoto +miti gating +ilo va +fe ss +fast back +cul lin +che ch +cb d +! ðŁļ¨ +visu alized +tru man +se ager +sd wx +co bo +wor ry +strate gic +pok al +mi p +fro y +ex cision +ang ell +ðŁij¨âĢį ðŁij©âĢį +zz or +subterran ean +pom p +mesotheli oma +li ao +ir at +i ap +god addy +et en +dragon age +catch up +# ( +tum ours +the star +rho ads +pi gg +ori anthi +jeep ers +infra structures +har shad +hal os +ha ver +day al +b si +b se +ac yday +sl m +mu jer +mor atorium +mon go +lo is +bw f +spani ard +musculo skeletal +mis ra +i ks +!! :) +wi eder +s á +n ts +moo g +ci bo +yr sof +young and +ye son +w myb +svet lana +mam o +jy pe +dor sal +af zal +re cherche +ra gu +hy dari +hin os +acrob atic +up tick +tel ugu +so so +rad hi +mu guru +infe ct +gosp els +down es +ded ness +ðŁį¾ ðŁį¾ +xen ophobic +ra kow +pe h +medi as +ju s +dv c +ðŁIJ ŀ +track side +tess ell +rhy l +poli shes +mal nourished +lar va +icon ic +hol der +for ç +far ron +chev al +cas sis +c ich +ball ant +wester ns +tu tu +tat ters +rich t +neander thal +jo akim +insati able +hoar der +fran ky +fla v +womensmar ch +traff icked +the gathering +te eth +sho ving +per cy +on so +nic hk +nancy ajram +kir stie +jab bar +gr ingo +fair trade +demo iselle +sch a +madam e +brah ma +ag am +æĿ± äº +yoshi ki +southern most +ra po +pre eti +ou g +mc shane +mani k +kad er +car ole +bol ger +ðŁIJ ı +tattoo ing +sketch note +poker stars +ip tv +cb u +ank ita +ı @ +sat uk +regur git +ph oning +o akes +neu ville +jor don +bre uer +as sini +ap en +tran g +te w +mn g +m te +m so +ky ri +gossip girl +b ck +ab ro +wy er +venture beat +mesopotam ia +in crimin +hel mand +ha iti +ha it +gro sse +dar den +cli matic +ç§ģãģ® ä¸ĸçķĮ +wh e +sm ill +rising star +percent ages +mu ddy +modu lar +kit ted +je une +is in +incur red +fir stre +communic ated +bt sin +ðŁ¤¦ âĢįâĻĢï¸ı +zapp os +win es +tortoi se +swan sea +shoot around +ri ding +hi ga +di van +att led +stiff ness +stereo typical +satur n +philipp ine +mati as +khar toum +j dt +hend rik +consul ted +ame c +x finity +voic es +un grateful +lizz ie +lar p +eur onews +ac n +ru mah +pan icked +men u +la grange +go lions +gam eday +dk ny +winter ing +will never +te us +t vac +su ma +sanctu aries +in justices +fi ore +dance music +án dez +zeal ous +robin hood +river bank +man zano +gy i +gati ss +fritt ata +enthusi ast +clu se +ba res +to po +spartan race +sha key +sh ue +queens ferry +mur ri +i biz +hurrican ef +hi mi +flow chart +dilu ted +ac sports +ï · +p sp +oc y +inc ite +corn elia +co very +sim ile +scuder ia +rot ator +op to +har ish +easter sunday +dish one +ch back +ðŁı İ +west point +st eller +search able +pad ang +mon goose +malcol m +hanni bal +gr n +far ting +emili ano +may berry +gal braith +ex pulsion +dow ne +congreg ational +bar bie +yb nl +oladi po +nus rat +mor nington +maurit ania +kag ut +k th +ju ror +hib bert +d va +bri k +schnei der +motley crue +live your +hospital ity +gul ation +for ty +faç ade +fair port +ely sium +de ion +come backs +cli part +ad af +w pp +un restricted +top anga +tony stewart +rare disease +ou trage +men ow +gand ol +as al +ambigu ity +tt alk +sti m +re used +mar lin +il han +fle dg +eun ji +chi le +by nr ++ ! +zom ato +wed ge +newly wed +mis management +mi f +m alling +lo fts +ho or +field hockey +bally mena +bal las +íĻ Ķ +yo ff +world healthday +tin der +ship man +pa ok +nhl flames +ma kayla +bay t +sto ve +ru dder +raj at +ra iler +pen e +mercedesam g +im on +i ee +brow der +adren al +x ed +skeg ness +sill iness +sc lassic +pe pa +fit t +diss olution +copa america +book lets +ate urs +asahe b +aln wick +the hobby +taran aki +shan k +rep john +our ses +mobili zing +iso tope +gan z +ðŁķ ¯ +vis count +val les +v se +sk enya +schau b +play mate +park scanada +ma ren +ju wa +bu mb +be rea +am orph +waist coat +piti ful +na ji +l fl +ing news +t mobile +nami bian +jan sson +hand son +ev n +catbox sunday +bhak ts +spe y +sky rocket +ever t +y ab +wen n +ve k +travel photo +ti ere +ski ppy +sa ab +qui er +payo ff +miller lite +j ps +emb attled +el ma +depos itory +complac ency +co h +ansel mo +show ings +shi ro +se ger +p fl +mat ti +knight sbridge +dumb arton +con son +bee ston +asci i +worldre cord +tim mins +men ssoccer +kilau ea +jo van +deni ers +beach side +tran scripts +punjab i +pre ssion +on ov +music hall +jam ar +err one +ep en +democr atic +contra band +& ... +su mp +sk ov +ray ne +mur ali +m eric +ed ays +cu test +con roe +bra unf +ab in +ðŁĺĤ ðŁijį +syl van +procrastin ate +la var +fing al +ak ind +admi res +à¤ Ĺ +treat ers +tre mbling +ral phie +p bis +hand outs +sam o +li one +griev ance +co tes +californi a +bis cotti +stock ton +sitec ore +podi atry +in ky +ic ici +fil lets +fail te +e pub +domin ik +day trading +braunf els +um b +tro phic +tric olor +ther ules +spl c +se con +paint brush +or ry +ni zam +nat weets +metal head +mam moo +esopha geal +de celer +allthe way +ac at +yn g +sh elly +poi rot +mon cri +fra z +ðŁij ® +å¹ ´ +yul in +util ise +tak is +stilet tos +ri v +rein carn +re imagine +pl p +ner t +ki shi +kaiser tone +ga iner +dier ksb +deptof defense +cut throat +chuk ka +black friars +alam bert +ak vari +ad om +wr ld +wkr n +w pl +tale za +ta unt +spey side +punch line +original character +maz el +help ing +har ries +bl aring +bag ger +accommod ating +predecess ors +peris cope +le ish +d illa +confe ssional +th yo +sho pper +royal visit +meridi an +me is +lovethe darts +kitchen aid +iti onal +it son +hin de +ha ss +gigat own +fri ghts +feast day +ðŁIJ ĵ +si ima +ray burn +mercedes benz +ford nation +cont ented +app i +re butt +qui k +qu and +muguru za +l ss +kan colle +ip r +fo wey +direc tives +car is +can nock +at ment +air flow +west pac +wdy t +trave sty +the artof +s fi +re ale +pa ws +new song +napol itano +mess er +lo fo +ke dar +david guetta +astu te +ðŁij¸ ðŁı¼ +ta an +ferri ss +bu chan +amuse veni +sho vels +newsle tters +ho tly +hen sley +ge dd +g ingham +esc am +drum roll +car dle +âĻ ł +spir ito +speci fic +skill s +leg alizing +daw ood +bb cn +{ " +zo oming +val erian +tand on +sk un +park view +paper craft +mr c +con traction +beautiful destinations +tas er +shaw ks +salt lake +ple y +pessi mistic +jae ger +ide e +gor ges +gar am +clo thing +cl ink +cent enni +ali ab +ta sters +sit ges +mo shi +ji iva +har borough +firstworld problems +atri sts +# $ +ðŁį ¿ +un professional +tre ach +team work +s ills +gaz ette +finger scrossed +b pc +undeni ably +th g +mo hand +kagut amuseveni +haw ken +ers world +daily art +balay age +watch us +th ame +sustainable development +re iter +ol ica +lucy hale +ku ba +foucau lt +crime stoppers +cre pt +choo sel +as w +ambassad or +. ðŁĻı +u omo +ki me +check points +wing ate +sau ber +nor ton +mei ster +lec oul +kar yn +duc a +cor te +ak aya +a sic +short stories +gol ly +elli sts +bour se +strol ls +niagar afalls +newyear s +n ines +lor ain +le win +geor die +ath lon +un knowingly +han go +bo dice +bay onet +tu mi +str ick +r ity +mis susa +le el +garth brooks +f mc +as ss +s ate +ron ics +guer re +ger u +exfoli ating +a ak +woo ten +subur bia +se wer +mecklen burg +ken shin +dj o +de wi +col ston +blue star +blanc pain +transc ends +te ma +scrib able +schi ele +mo ff +is sey +indi ab +cu bed +cand i +alp has +alle ge +ðŁ ĥ +r he +p fp +new west +lack aw +h ree +cru mbles +al ap +wthr com +to kio +state side +sit is +se vern +ro mb +ico se +gri sham +fla gging +com posure +cathe ter +can ines +ðŁį Ĺ +z la +v ander +mom oland +hil ux +gar nished +coven try +bi gi +stu cco +oo ty +kac ey +guess the +goose berry +for it +death match +ali bre +af ari +ab cs +val our +sush ant +ra hal +publ ics +lati mer +k oop +h iss +go oner +g pc +f tv +con front +c ada +archi ving +apo logist +å Ĵ +Ø§Ø ¨ +stl wx +small holder +reta iling +recording studio +of sky +le le +desol ate +algorith mic +íķ ľ +wy k +vers ed +tre spass +take it +pr aline +nue stras +mat thar +li psy +ky lian +fli pp +fa wards +clar ine +all lll +sta ar +scaramu cci +reas sure +kir ch +j cp +commend able +bank roll +baf fling +angel a +ðŁĸ Į +tre mont +spook tacular +raj kot +kent a +home stay +ho even +fontaine bleau +decapit ated +ar abe +april ia +thorn hill +tat t +si bir +no limits +newze aland +naz ir +morph in +la ken +hinch cliffe +gor se +gaz prom +fit n +defici encies +d ool +bohemi an +ar ad +z ax +tambour ine +sp elman +multi modal +milleni als +melt zer +henry cavill +han ia +w zz +sever us +planned parenthood +ni b +multip lied +cal lum +be inspired +ðŁĺĤ ðŁĺ© +yq g +uk weather +laundro mat +kir stin +ip i +fair ground +di vision +d ando +be als +based god +âģ£ âģ£ +whis kies +weak ens +to watch +te pp +seash ell +pa inter +o ast +inde scribable +g ani +el rufai +devil ish +bo capsule +bha ji +yeez us +work sop +ques ad +phosp hor +mo ffe +lan z +indi scri +id d +giz modo +el pas +co als +chim era +carbo hydrate +am oment +sta at +sof tener +shrin ks +plate lets +ok la +di b +deplor ables +car ling +cal gar +breath takingly +ann n +ðŁijĮ ðŁĺĤ +ж ив +ze m +white haven +we isse +virat kohli +sc ap +fir ma +co rea +c mi +ðŁķ ° +ðŁı ij +pn p +mess er +gue sting +gran tee +gi st +che ater +bur na +ak im +uni birmingham +kan di +her tha +feli pe +b bery +super dome +os f +mid town +letter box +la far +juni o +food trucks +fish man +âĺĿ ï¸ı +west bengal +u up +spla yer +patri k +man gan +kram pus +hyalur onic +fra un +curi ou +charl ton +bike share +bah ay +studen tath +n ant +d hillon +cre ssi +ar ta +twitch streamer +snake skin +saura bh +pre maturely +frankin cense +conden ser +capp ado +tweetab ondthatcantbebroken +ti ms +man cave +jal en +hand i +cafer acer +bar ger +as ena +" > +wic can +ver de +standing rock +puri fying +paste ur +gal t +fc king +dierksb entley +car away +batt lero +asse m +ad week +ðŁIJ Ľ +us am +thor pes +supervis ory +sc lub +pas saic +mil la +form al +° ) +travel theworld +ti sha +pic t +per oni +lore to +ku y +ff m +watch this +u lam +medit ations +emb assy +bir o +wheel chairs +su pers +si me +run corn +ne to +ke ke +hun ts +donut day +ci ders +brief ings +bren ton +ãĥķãĤ¡ ãĤ¤ +we den +tumul tuous +tr ine +shaqu ille +ran goon +pal pable +geri atric +ea stere +cfb playoff +brun ner +apro pos +ðŁĩµðŁĩ ° +w fm +tee ter +od f +nov artis +ni jme +n tw +matsu moto +intersec tionality +ham ed +contex tu +avengersin finity +sd ale +nat o +mac gregor +gar ber +ele m +c ps +bay elsa +back fired +anal ge +ni u +mini aturi +li fers +ke dah +ai mee +ad dy +ðŁĺĤ ðŁĺħ +wp tv +trouble some +li ani +deep water +ðŁı ł +wor sley +w un +si sley +s fight +mai mane +long itude +ec lare +ck a +cabine try +brook lands +anastasi a +vonne gut +swat h +science week +mutu a +can oes +brun n +aishwaryar ai +vesu vius +travel bloggers +traumati zed +te din +shaf tesbury +prow ler +ni bble +mi ko +es mer +crock pot +waynero oney +un harmed +spell bound +s ram +play suit +man che +fraud sters +fore shore +du gan +ask the +vol go +sav ant +par si +ol le +look s +fu mi +fais al +exor cism +candi da +wl w +vin yasa +vent v +urban ization +tam imi +sports betting +shar ma +rejo icing +gla sse +dar aa +d fat +bb j +bankno te +anonym ity +whi zz +shiv ratri +ri vas +popo vich +mil dew +jimmy kimmel +gon er +frag mentation +e aves +affi davit +nott m +fa ires +dr l +deeplear n +de scu +care lli +bra bant +-__ _- +ðŁĵ Ħ +the hungergames +schem ing +ro tisserie +ri pa +present e +over crowding +fear lessly +cer rone +vic theatre +ukbusiness rt +substitu ted +shut outs +pau lette +pal ing +ola unch +henne pin +bow man +a was +yaw ning +with am +vs fashionshow +ver ture +tra b +th ath +st peter +ross endale +may an +heritage day +f mi +ca ith +bel gra +tavi stock +sur ged +str am +ra tha +prem rugby +ny cacc +mor ay +fiftyshades darker +fayo se +en actment +conden sation +carra gher +british vogue +bom ba +apric ots +alessi o +war tho +sex es +pra veen +lis berger +ki bum +frac tional +ew tn +conco ction +cater ina +am aker +symbi osis +supre mo +sesame street +polok wane +new burgh +khal i +k agan +di pp +broad bent +boni face +auror aborealis +ภĸ +sub way +screen printing +h ati +dc universe +vicar age +u ah +tiger zinda +stol en +space man +sam ple +pil kington +medi ator +lu mps +joyful leaders +e ason +be agles +paren ting +padra ig +obli que +ma es +inst ar +har git +gen ie +de em +cbs sports +back stop +vern ay +t sur +rock hall +night stand +musc lecar +journ alis +eff erve +do zer +darken ed +cu per +col ne +brook ings +world premiere +vel ma +south dakota +sh inning +s ically +le er +elo ck +di pika +winni peg +wa an +vaccin ate +si ms +sho x +q t +oli o +net ball +mc as +magni fique +ma ples +i kar +how ells +v ence +richar dd +pur ina +mend ra +je z +im is +h tt +forthelove of +fight night +exhi b +earth bound +e sol +butter worth +blo c +bi ol +before you +ãĤ¹ãĥ Ī +¡ ¡¡ +wc th +tom mie +th art +stra ys +speop le +sh we +sea hawk +pp et +ol ler +n si +me tv +ma kar +kur ta +k xip +k nysna +friend zone +de scan +bint ang +as pire +aquari ums +p mr +one perfectshot +non linear +nom ura +nichk hun +he yyy +faf bulldog +du ane +all u +un sig +power bi +mill brook +lion sgate +but lins +be o +al ok +suspici ons +r ins +mid nite +mam an +el way +e bi +ast i +al ah +wi ther +senn heiser +plym uni +pan icking +night photography +lecoul tre +harri et +clerken well +ci da +chick adee +car tney +cap tained +be te +am ee +ðŁ¤Ļ ðŁı½ +what chu +scream queens +k mt +en heimer +dont be +der ive +davi dar +cr and +ëĵ ľë +âļ ķï¸ı +wi thr +pe cos +mar kie +hat teras +garden a +arti stic +stony brook +pra xis +one w +gar nered +e gor +crew neck +bn f +acon ference +zoo logical +u ic +swe b +men ard +mayo clinic +lin css +hu ggins +dl f +award winning +wrest led +tranqu ill +re voir +re charging +pro fo +pro claiming +p tole +jimmykimmel live +gman ews +douche bag +cle m +ce ylon +accent ed +aaaa and +u ks +symboli ze +swan age +safety week +mo si +law fully +ira d +idit arod +hen rico +fir a +âĻ¥ . +wak o +vo ye +susp enders +probab ilities +ox ley +hye ong +gru yere +auto cad +accumul ations +(- : +ðŁĸ¥ ï¸ı +ver os +tough mudder +thwar ted +shor ten +koo zie +kameham eha +jung les +ide ation +hill ar +el tham +chem in +assini bo +ar cane +an ai +ab bot +ðŁĮ ĩ +ye un +subpo ena +selvar ag +red box +fe eney +f wc +ab is +: ( +ðŁı½ âĢį +ðŁĩ¿ ðŁĩ +| âĢ¦ +to read +mill inery +le ssi +feu dal +fajar do +cyn di +chronic le +boyl ston +beautiful day +wj sn +w md +va g +tech n +sumat ran +pre sales +mind body +likefor like +chro mium +cha it +ab aker +~ ) +pre ma +nadin elu +missmar is +men del +man sion +kle enex +ji yong +in of +entry way +bump ers +au k +ê ± +stal bans +rs v +paddle boarding +le as +evo que +enginak yürek +em al +dang elo +berg dahl +az ad +amphe tamine +ç « +z rh +x ddd +puzz ling +mont serrat +man ns +jesu ss +hatt a +canon ical +x z +lun dy +leav ed +jo dha +epic fail +el wood +du ali +conver ters +a et +!! ' +z is +wal z +v vs +slo ping +no str +mel li +graphic novel +dol o +cork screw +ul uru +the way +sto k +spell binding +ru be +ro den +re ay +gu shing +bra wn +av at +su mac +pis sarro +mano ir +ly nette +comprehen sible +absor bent +winter solstice +the q +share r +mur ali +mm urd +md l +light skin +gg g +el ny +consoli dating +command ment +bur dened +bin ders +asi atic +Î » +um bro +suicide girls +rail uk +n ale +missmaris racal +master chef +de generate +boo sh +aze alia +united way +technical analysis +t ended +spo kes +sheep skin +ram ayana +queen ie +je f +i ana +h indi +gra pple +el itist +el ap +d pc +d mv +bet cha +b ape +ðŁijĩ ðŁı¾ +ton gs +shoton iphone +reli shing +re pra +powder puff +os man +bu tty +ba ie +aco ke +the w +re making +pl atters +per jury +ni zam +movie poster +fredri k +fa sten +ener ge +el don +bird day +wy att +wh atta +uygh ur +tx motorspeedway +strau ght +sta de +pe kka +ki k +cri spin +cat t +ayushman nk +ðŁĮ ľ +weather ford +vern al +ta stiest +suspen sions +s ada +perfec tionist +jo go +del ving +con chit +. ⾨ +top friends +summar ies +ste wie +scrat cher +pre sets +mar ana +her mano +g dn +edm family +bug sy +us ical +ste tho +qu ities +lin ings +king swood +dhar ma +wil k +ou lu +ori ous +or om +optometri st +one se +er rand +end y +doo bie +coo ki +ber tram +akvari stan +torto ises +tatters alls +ric ular +p ough +ok tober +nyx cosmetics +lo oooooooo +hoi sted +ho des +dw t +dist illers +day light +coeli ac +bo bro +arra igned +tag uig +sf pd +pres sure +flaw lessly +ev geny +conscienti ous +buc ci +we can +them all +plu ck +pless ness +me v +lab ly +it te +ar v +ab lack +wt k +up tempo +stil ts +sma c +jaf fe +hur ri +hta fc +head quartered +gul ch +g ca +with drew +selfle ss +meh mood +la bo +sunderland afc +st ens +potat o +pic card +o go +mer vyn +ma se +koenig segg +ilu str +hom opho +hari bo +di ario +calvin klein +le ec +c sharp +apat ow +al bury +yellow ish +y tv +ut ley +ro san +ram snfl +national coffeeday +kuro sawa +judg ments +it si +idio m +ho led +cl ank +citiz ent +candi dat +ae g +wom p +the opening +the bhf +shar da +north van +nas scom +mm p +inqu iring +glu t +dar te +sin pics +sas cha +re pp +do go +bag gies +u ottawa +tu ber +stor my +st lv +re it +re fil +palay eroyale +omo juwa +my suru +lo li +bio science +ang ello +ace o +ãħ İ +victor ians +tyr ann +tit o +sand hill +ous se +moneti zation +mo ka +iz mir +gr ins +gentle man +disson ance +deep avali +danic apatrick +president trump +par mar +pain killers +pag asa +origin ates +nex us +aspir ants +whatever ittakes +stock well +ste alth +s de +l bf +ign an +her z +gon da +fu sc +fe dor +dra x +d arian +ca thr +ama al +yu t +spl ice +s attar +re sses +mt f +inter acts +infiltr ate +happ end +den ounces +car row +vir gil +v vip +ti bles +oce arch +cour ant +z adar +wille ms +u ze +sympath ies +revi val +pe ase +ou fc +gren fell +globe trotters +g lin +fur thering +fla pper +war ds +raven a +mit su +eu g +cated ral +bex ar +be douin +zi oso +y aaay +sg t +refin ement +mol ine +lam y +d lamini +climate strike +bythe sea +brat ton +av r +ah ill +ad an +wolf son +m ne +ci ak +char d +bright side +â¬ĩï¸ı â¬ĩï¸ı +à¸ļ าภ+sch l +saniti zer +master plan +la vish +kar ant +hull city +gur kha +gat lin +com cast +bi ar +b ww +ak bar +x eno +wo wo +spin al +per ts +ic ent +famil ial +ally brooke +à ² +z oro +ver te +se mp +sab ato +rela p +puerto vallarta +pe dre +pat ria +moo dle +make me +im porter +fish tank +f yo +co pi +bicy cling +awil son +above andbeyond +wa ar +tat a +smallbusiness saturday +rhi an +ko ya +gr ation +dict ates +d tn +be it +ðŁ¤ĺ ðŁı½ +x eon +son akshi +sch en +ratt led +pro long +g pp +fast track +dr anath +de central +copp ell +breath ed +ðŁ¥ İ +who dun +submer sible +scallo ped +r itten +mal don +l hd +just another +joseph s +hope well +fa stand +dhar na +clar ice +walk off +unspeak able +sp ac +soap box +ross r +jay son +ce ps +af faire +ad minister +x clusive +tar f +special ising +kir ill +hand som +daytime emmys +congress men +ceredigi on +ca ix +apc nigeria +al al +âĺº âĺºâĺº +ru ps +pop music +mr and +gold man +gi vens +de ffo +art sand +alu a +viv atech +tarra gona +shak in +sa irport +recap ture +pat r +mano har +law rie +hi ver +ash am +ÃŃ s +ther m +sim mon +religi ously +opul ence +nawazu ddin +mc ca +la sso +bir a +y ami +y af +tele photo +su sten +sego via +rio de +go han +f bu +ey bl +clic quot +bu x +ber ley +âŀ Ķ +wal mart +sar war +r fs +p ylon +mign ola +go pokes +cu oco +car li +appreciation week +anti um +ali yev +Ĭãģ Ĺ +wordsof wisdom +wi ggly +wa di +u do +strand ing +sto bart +shadesof grey +port noy +port illo +pa sties +mi spr +mam elo +lor ax +la ire +janos kians +ham dan +disc ern +country life +ai les +t cher +sail fish +saf i +pro fil +nothing ness +n ri +har iri +grou cho +far outa +ev m +enthusiast ically +en da +du sk +dread nought +cru mp +coul da +certi fied +bot ticelli +ba x +au me +ske tt +sc b +rose hill +mb c +isra eli +h ne +great day +folk fest +faire y +er ink +en ry +craw ford +broms grove +bo drum +travel channel +sar dar +rec tify +newsa del +micro be +inci dentally +in still +fe cal +eu gene +dru gged +b man +whoo pi +un a +twi z +street view +our day +nicar agu +mr k +mouth ed +intu it +ingra ham +groo ving +cute ee +chil tern +che ol +boomer sooner +arbro ath +to ko +te ab +smo ak +ser aph +sal ert +re w +pol k +pim ps +ma ho +ik ay +he sper +cit ru +black sabbath +short fall +mar a +ib as +easter ly +ca stiel +ìĨĮëħĢ ìĭľë +ë Ĭ +rein vention +la vin +jo ong +con cur +clu stering +bra ver +ba aa +alge bra +al ita +aberdeen fc +wholesal er +vo et +vin od +st alling +daun ted +âĺ ¯ +walk ways +sadi stic +ridd les +o ar +ne ves +match y +lex y +kine tics +gil da +ðŁĺĺ ðŁİī +sant ino +predic tability +fo kker +ana ero +vesp ers +sy ne +stock ing +self help +r bl +mak ita +ju ego +in fidelity +hei de +devi ation +cur zon +com mis +ci bc +bbc wthr +ba hai +aaaa ah +ðŁĮ ª +wer ise +tom mo +seren ading +m itten +loose women +ite e +ic arly +ha va +gop ats +ufc w +the chainsmokers +t chat +seab ass +san ju +pepp ered +or illia +ministr yof +inf ant +fortune magazine +augu sto +ais ling +ðŁ¤· ðŁı½âĢįâĻĤï¸ı +scru bbing +rac coons +mon et +mcke an +jay y +experim ented +cost as +cam my +base ment +al te +worshi ppers +wal eg +t co +sier rac +santi ago +s ø +s ily +s aga +k sd +inj ury +fi jian +exeter chiefs +d ja +com erica +bee cher +u du +ti ernan +sol eno +show jumping +purr fect +mer tens +fr p +feder alism +constab ulary +ba shed +air max +syner gies +shi da +pi ña +mis lead +ma ud +eye z +air and +z of +wizar ding +w cha +tab u +spo ssible +sol vers +red zone +nhl stats +ne iman +mile high +mc vey +lew y +laur amar +incen tivi +i stria +goti ger +en amel +bb on +alco holics +ðŁĻĦ ðŁĻĦðŁĻĦ +we as +time pieces +swee ten +st ah +rehear sed +n wc +frontrun ner +fi vb +d our +cataly zed +bron ch +blo k +ðŁİħ ðŁı¼ +ven do +ra vers +obi spo +k alli +iner tia +g ny +d ni +bi hari +anaheim ducks +altu ve +air bus +ac a +we sts +voc ally +rati fication +nj it +lar son +izz ard +i ec +gb m +city wide +call an +bob sled +bbcwthr watchers +ìľĦë ĦĪ +sun risers +pediatric ian +pan ning +nar asi +liber ian +end ic +base balls +v anian +um g +tai ko +ri sd +magno lias +le em +ken ai +fric ken +dom ed +d atta +col fax +cephal us +adopt me +what a +pre mon +mass age +go buffs +enor m +dolla sign +dal es +bon aire +bertie schip +applau ded +ann n +wind swept +ss football +recover ies +raj at +pro tru +hoo kers +bio security +ãħ¤ãħ¤ãħ¤ãħ¤ ãħ¤ãħ¤ãħ¤ãħ¤ +ton o +selvarag havan +pitt i +n ro +l pr +je vic +goog ly +char tre +ðŁĮ´ ðŁĮ´ +âłĢâłĢ âłĢ +u bere +sb d +ri vi +po conor +pan ellists +matt ingly +ken y +ibe w +foolish ness +farouta khtar +dream work +whit erab +west field +ten ors +mu sume +mo rey +md traffic +i af +easy branches +ch aff +carden as +ab vote +å ¾ +s ours +mul grew +me su +kd ka +food truck +der mal +child abuse +time share +se ti +pha se +oka for +lough lin +jan ine +around theworld +ॠĭ +rein forces +jane the +hel io +he man +dra kes +c sports +ye ee +vis iti +st john +percu ssionist +non violence +f ase +di ac +break y +" * +sn b +saf ran +pat ching +nickelo deon +intru ders +enlist ment +el les +cost ner +coo s +be sson +base less +appe ase +super se +su mit +sab ian +gene simmons +g don +frat ern +emph atic +d np +constra ined +clee thorpes +catal ans +an ae +yu en +sori bada +sky bet +saw dust +s film +nag ano +n ari +le ong +la is +in eligible +idi bia +go dav +disper se +bur man +an jel +re za +pough keep +ph oned +me du +ka ori +ive co +com uni +chinese gp +chim ps +twin kies +o ise +natge ophotos +na irn +mitochondri a +ju hi +cy lind +churchill downs +christma siscoming +atta ching +ar ras +. "" +timb aland +the hedgehog +sustainable fashion +summ ing +more los +me tta +man tan +kut ch +evan s +dazz led +stu ssy +royal family +roeth lisberger +prism atic +jam shed +ge s +brou ssard +blue angels +b mo +ann af +alis son +al gal +ë ī´ +wal ang +scar ab +m ingo +fruc tose +force fully +eu w +cri er +bai k +ar ter +alphabe tical +al lot +waz ir +to ffe +opio id +non existent +nephro logy +mc at +ing it +har ts +dad life +tx h +twit ters +tross achs +ss oa +so koto +rein ce +real bread +ray theon +ragha v +periodic ally +mayo gaa +gio vin +ed on +down graded +de pay +costac offee +colli ers +canu ck +vo tre +onthe move +margarit aville +kw az +gour met +foo dre +exo tics +de grom +daeh wi +ðŁĮ¹ðŁĮ¹ ðŁĮ¹ +te dros +ss rajamouli +ru ble +p news +ot one +ny i +fu ge +dam an +dal ert +as bury +allow ances +tel la +t dr +spir ulina +rugby united +rel ly +pass ers +oooo oh +medic ated +evangel ine +enti al +conditi oners +âĺ Ĥ +scoli osis +h ro +gift guide +g ally +dv f +cru mlin +moy nihan +mo disar +master classes +mac ular +be cau +bair stow +aun e +us gbc +thelion king +overwhel m +foo ter +and ler +she ard +ridge field +na as +n so +m sia +leg on +c sp +bo zo +autism speaks +as ch +ðŁĩ¯ ðŁĩµ +âĿ¤ . +» » +zo ella +syphil is +shim ura +sen tosa +new er +m clou +kri spies +im fc +gar h +g hazi +charle se +by d +ush ers +spread sheets +sel in +projec tile +p gm +over turns +must aches +mun son +muchach os +mol on +itss sr +ino is +fanc am +d cc +bu dge +pe gged +ing dom +cymb al +tul are +kryp tonite +ino va +feed the +f eni +ci ster +na eun +individu alized +fi h +fer al +ef fie +d so +???? ???? +syman tec +ss f +sma ug +si bal +okee cho +md pi +ku di +ho wer +gar gano +a pren +âĭ Ĩ +y is +w tv +thorn ton +subsi dized +speed wagon +pas so +mat ted +hargit ay +grave send +gi dd +friday fun +detec table +wild lands +w soc +tw is +san ji +sam bora +sal via +fakh ri +bella thorne +ak var +scint illating +ne er +n usa +m pl +leg iti +ku a +guer re +grou ch +en baum +ej f +col la +wind hoek +ut dfc +trey songz +stra damus +ro sar +mol ler +lordof therings +ill ar +drex el +dot tie +di straught +chaper one +bring your +bay shore +am ur +um ph +stock port +sitt ing +radi sson +ok al +jol lof +hor net +havelo ck +de j +cab bie +a arti +° , +van de +sch wan +let cher +lero ck +j mu +dw ells +dis qualification +bru s +amaze balls +ðŁ¤ ® +sc ac +radi ates +grow ling +ge th +et ter +dis fru +colo ssians +cd w +an arkali +alde burgh +ag ot +s west +or ro +on l +max x +imman composer +fro mmy +dam nation +d int +beer week +tribu to +til ak +t da +savethe children +pim lico +mississi pp +mar gau +ak ana +ag ami +âī § +wool ley +reven ge +over size +k res +ir ce +h news +et x +con yers +bill shorten +ban v +at el +v sphere +sule iman +stack able +petro v +pale y +pal atine +pa arl +le ch +kil patrick +k shs +ju v +hit am +ash down +abomin able +var k +uni an +u wi +thel u +shoot film +sand lot +pau sing +l lega +hor nb +íķ ľ +ठ¤ +Ùħ ر +y ha +wzz m +way back +t suk +stom achs +star i +pizz ahu +pa sted +nameis nani +kan to +car ley +be ur +ðŁĴ¸ ðŁĴ¸ +yn j +us army +sen eg +roa ster +mo rel +inthe park +ff acup +cre an +billshorten mp +ann arbor +abo y +rock wood +pill sbury +lu go +explor ations +broom field +az mi +atul a +akvar yum +show en +mc nab +d ws +wa see +nijme gen +john kasich +f pc +cr at +êµ ¬ +ื à¹Ī +velo ve +rose bud +orche stras +mortg age +flate arth +dailym irror +charle stown +bra ff +bo ku +bel kin +ãģ « +ร าภ+ti is +sacrif icial +lo esch +vide omarketing +un dul +supe rel +sh as +musi q +ki era +kam en +jam ey +encan ta +den u +ar cus +æ Ĵ +sor kin +son ali +ros alie +pushaward sliz +no ord +iam specialized +cap tioning +ðŁļĢðŁļĢ ðŁļĢ +sange et +rashtra pati +rain yday +paralym pian +must n +kun e +gen z +en viable +ef b +ami ens +à® ± +t de +re painted +ma zer +lay up +keh lani +jor gensen +der g +con chita +bloem fontein +all yn +synony ms +sten house +sli my +shal ini +den ier +assi stive +aquari en +am bar +subram anian +rebu ke +mam mam +ing ers +h itt +dog fish +cr l +am are +te uil +soci alize +shi z +rar ities +e ire +cincy tennis +benet ton +aven atti +ëĵ Ģ +un geneva +saan ich +r sa +poconor aceway +p liers +inter rupts +dark room +bau man +affe ctive +tou ro +tag aytay +sw ole +sc n +o ston +min ah +lam pung +coni ston +biken yc +bali ye +win i +spec trum +h ick +ely se +pet ter +i sel +emb assies +dj iglobal +dec ca +chal amet +an ony +ta ar +stemc ell +po sium +muen chen +bblo grt +app dev +anirud h +ad ah +toler able +sula iman +sec network +rhon j +prece ded +ob vi +kp mg +exclu sive +cou steau +une arth +space walk +pen der +il k +fari ous +excited ly +common place +bin ge +alec ki +a ert +w mma +transc ei +sw amin +sch ec +s anga +lec tive +ki pp +gl itch +f any +elli s +eal ing +di man +ãĤ¹ ãĤ¿ +ÙĨ ÙĪ +ville a +ver ily +putra jaya +head land +h elly +ë ŀ +un announced +techno logically +pushawardsliz quens +phra im +mar z +ma scot +kindness matters +hu ski +her ren +amary llis +a isa +sten osis +shi ite +mv fc +ml p +mirand alambert +me jia +lo ger +like able +ge vents +cold field +bu de +appli que +^ * +windows ill +ste mming +sql server +sh ur +mschar lotte +mscharlotte wwe +katerin burg +i spr +hinter land +fre i +er asing +concentr ates +blood bath +bk lyn +ari ka +st mary +prime minister +parap hern +pa ket +om ie +mun d +medic a +law yer +ka poor +gotiger sgo +enorm ously +dop ening +cur l +ang irl +ðŁĩŃ ðŁĩº +vo tered +oooooooo oo +om bré +neer aj +n vey +marcel lus +mar illion +el fon +dro z +ane a +abre ak +wont stop +sof love +sher idan +sch utz +ry ne +old town +kr p +jype twice +int end +ghanai ans +flying tr +doppelgän ger +bro lly +agn olo +ðŁ¥ ´ +ìĦ Ŀ +yn drome +y ate +mic keym +life coach +en ke +cap that +b ne +stere ophon +pal mo +la et +franc ine +bm x +âī ¦ +whit eri +til ting +post production +knicker bo +em boli +umbrel la +ri i +refu elling +rally together +ne th +matri arch +lg r +fore shadowing +eye witness +ðŁĺį ⾨ +u can +ty rants +pav es +omic ron +mir r +medit ated +gal atians +dro m +cabine t +buy now +skill ful +sha v +pit bull +meand ering +indic tments +gu tt +f ens +br ity +bar f +ìĦ ± +su st +sn ort +sky ward +reincarn ated +posit ano +neuro pathy +mag and +lit tered +line backers +jule p +car tons +ben shapiro +ax l +ðŁIJ ĭ +rejec ted +o ssi +gai ther +en sue +b gg +uncontrol lably +sur bhi +so de +sha an +re join +pre e +higg in +cav s +yu b +w hal +use rexperience +spoon ful +sli ght +sar in +sachin ita +rhodod endron +rep til +rel enting +refere eing +paral lax +mün de +lea shed +il ms +col onia +chow dhury +cer i +ap are +and son +ðŁİ ¢ +ìĬ¤ íĦ +åľ Ł +work loads +up ers +tenter den +snapp ers +sm acking +she v +redd itch +ilo v +dinosa ur +bi jou +bankof america +wag tail +vi se +ud hay +pic turing +festiv us +expe c +ep o +encro ach +co ding +ba ad +ಠ¦ +wye th +sc raw +ove re +n ena +l z +j anie +gar g +e de +arti fic +window sphone +ver dun +under standings +to g +silver ton +shack les +ho ppin +fa zio +election results +cbsd fw +ca pel +bio ethics +wrong fully +vel i +singul ar +pe sh +o chs +kat er +kar li +hango vers +flo pped +financial inclusion +fin ns +ff en +eart g +e sche +dy na +consecr ated +ce u +sam bo +s zy +reyn old +mat uring +lol ly +lau d +gel man +gear sofwar +g sl +fledg ling +epilo gue +cal led +bo ssier +zo id +yas in +whos next +stabili zed +spo res +spi ky +rol lie +ra vic +prinse sachinita +ph ds +mun g +mamelo di +maker bot +fur by +fin der +ct fc +brun ello +avengersinfinity war +ac cru +ab us +ðŁı Ŀ +ìļ © +âľĪï¸ı âľĪï¸ı +sp u +se pe +se aboard +power puff +impre ssion +gold end +ft f +e gy +drink water +b int +affl icted +Ñ ı +sch on +respect the +ram ming +piñ ata +park lands +math ur +la vuelta +far ia +disney cruise +deci dedly +simul cast +que bec +p ge +mit te +lc pl +ill ing +har oon +eu pol +enh ancer +der gaard +ard more +accli mati +á ĭ +wat e +tat oo +sh g +od b +la gan +equi pping +dhru v +cystic fibrosis +al aac +ðŁĺĴ ðŁĺĴðŁĺĴ +âĪ Ĵ +win theday +total itarian +it sm +elle smere +de kho +daugh try +childrenin need +by s +bak it +tallade gas +supple mentary +stu ck +pav lo +obla stoma +n jo +mix x +lan ez +krat os +kay aks +gar ret +favor it +civil ised +am pl +ac ra +¨¨¨¨ ¨¨¨¨ +wor ley +tri omphe +st ak +porto fino +pin ec +percent ile +om ari +kus ama +inverte brate +guild wars +gu id +ei b +bo gs +analy sed +san thanam +rang ed +le j +gains bourg +feel goodfriday +den hall +cros scountry +confeder acy +cen trum +blak ely +belgi angp +ðŁIJ¾ ðŁIJ¾ +ðŁĮ Ń +y aaa +up time +sound wave +renfrew shire +pati ala +mi m +k adi +hum bug +hey day +fox woods +fab rizio +ely sian +deterior ated +cover version +afrika ans +Ì ² +sn it +slo t +samsmith world +r dj +py aar +black hole +bar man +abstrac texpre +xox ox +where by +m raz +green est +fly be +dro wns +cu mu +bla m +al af +ain sworth +trump shutdown +sk at +set to +sc outed +mal ton +law lor +fini shed +emo tive +dynam ite +ar shi +ano e +жив оÑĤ +sing let +sar torial +ni shes +hel big +hart ford +boy le +ðŁį £ +z c +tuss le +sti ves +skir mish +red to +phen ology +matil das +jen son +integr a +heart ily +dolly parton +breit bartnews +b mp +ðŁĶ¥ ðŁĺį +ðŁĮ¸ðŁĮ¸ ðŁĮ¸ +way v +si stine +poughkeep sie +oro ssi +loc kett +hindu tva +dead man +aqu it +ðŁį ¬ +âŀĸâŀĸâŀĸâŀĸ âŀĸâŀĸâŀĸâŀĸ +Ñ Ģ +uni onists +ther oe +sm elt +r natweets +kal u +family guy +exagger ation +des ic +chate aux +birdc age +bic ol +anc tuary +ad nan +" @__ +went worth +u ros +se ss +se ss +power ment +mi sia +mar ku +gen itals +flo g +distill ation +bun dt +bor tles +w ile +scalli ons +sat t +imperial college +gu v +aerob ics +çµµ æıı +pope yes +pi sta +neglec ting +ik ki +house boat +ge ary +don er +spear head +sol aris +ob ili +eur on +dun stable +ë¸Ķëŀ Ļ +un claimed +spoo ky +persi mmon +it smy +fight in +ar ley +z eni +th yl +shav es +predic tably +me ach +may day +ma sti +hq trivia +bien venue +be bo +âĿ¤ï¸ı ðŁĺŃ +ô me +ve tch +val lec +v dc +spru it +pat ent +o she +guru ji +do ch +cor tical +cashe ws +bu eller +bau chi +super ior +sand r +r cr +ir in +hrithi kroshan +embr yos +dom ens +do per +cha peau +ðŁij» ðŁİĥ +yl ine +y us +un am +su kk +stoner fam +recep tive +os p +in ke +hil ia +green energy +gor od +cap er +c co +b wc +redro ck +ra ekwon +g yo +eu bank +complac ent +bedro om +ðŁijī ðŁijĪ +⼠Ī +живоÑĤ нÑĭе +water melons +total divas +spring dale +sp edes +slu shy +re ve +nur ser +men ez +bil lab +ad l +ç IJ +term ites +r fu +lo ll +ip u +cr acing +chas se +zi va +trilli ons +red fish +pat on +long champ +li sd +fol lo +fin ex +do goftheday +ce do +adap tor +wil lem +transiti oned +swee teners +ps vr +na agin +la was +kar no +guad ag +gal ena +exclu si +conspir ing +ber d +any ang +andr ze +tur an +stra yed +spl urge +personal finance +nat bynature +legendof zelda +food travelchat +delu ded +conce al +besto fetsy +ac companies +ab al +numer als +mb laq +dar rows +anach ron +ame thi +af ca +water color +under mines +sh ish +paraphern alia +ke gan +index es +hydraul ics +cl onal +campan ia +c bb +ber gh +======== ======== +................ ................ +the par +taste fully +scoo ping +s fc +om atic +mi q +lv g +itunes music +eng ar +du la +dra ch +dn cin +bloomberg tv +bever ley +bak r +and ha +âľħ âľħ +o bel +mah endra +la j +kun o +khatta k +k rug +hu iz +fen n +dn ce +colino donoghue +blaz blue +éĩ İ +vas eline +un cw +ts w +snow shoeing +refin eries +pho s +muer te +jumbo tron +in ners +im mu +e br +bri d +bram ley +bab son +at lus +a om +sim ha +rip tide +oh saa +dam pen +d te +bahrain i +vibr ating +st marys +redar my +gui dores +g di +fu k +bo bber +aler ting +( ^ +ver ton +retar dant +let tered +in vis +ha dd +gr instead +e wok +before and +âĺºï¸ı âĺºï¸ıâĺºï¸ı +yu me +thatdescribe syourfriendship +super lative +sovie ts +oro ck +lar cen +hy gge +hon duran +hilli er +hat in +h pm +est an +decentr alization +at ology +andre a +wi pro +typho id +stub born +scalli on +levit ation +esc u +dis sect +car done +bro dy +ay ew +alab a +ab ras +íĤ¤ ì¦Ī +sil i +rock band +rin con +mo cs +kick back +ju ssie +ar ayan +alai kum +ðŁĺ ¼ +ãģ¦ ãĤ +str ans +ship sinpics +ree ze +mat z +ko th +gun metal +ds n +di ved +cur ley +contamin ants +catch ing +tyne mouth +my k +mor neau +bud gie +apolog ised +adam s +ðŁĻĭ âĢįâĻĢï¸ı +ãħ ¡ +work life +mult nom +la fferty +dove cameron +a em +í ļ +æ ¨ +why dont +sur fs +st ü +repor ter +rec al +photograph yday +p isco +ko y +gram ma +dong woo +cor t +astro logical +ðŁĩª ðŁĩº +you were +u zu +ti dings +red bul +pre set +lamp shade +inthe air +icic les +hol zer +gi psy +gc p +cli x +bible study +w sr +the dog +tas sels +movi star +kur ti +im ed +icon ocla +fire dept +dg in +ant illes +a awards +sugar loaf +ric ken +motiv ations +ili st +hep worth +fan meet +do an +davi ds +chron ology +bol in +at g +[ !] +weh be +tortell ini +team dairy +new cast +manate es +mag alu +fre itas +forwar ded +college of +buffal osab +spor trelief +sotom ayor +nbaon tnt +matthew mercer +governor ship +al ger +wol fe +tit ch +stephen athome +ru pa +p onic +origin ating +nbc universal +info tech +eu logy +car ters +bum garner +ance y +yeg dt +wind surfing +st ons +poz nan +not ary +music is +men shealth +l pt +ha pur +el or +crun ching +terr arium +royal society +par ke +ner a +muru gan +mem grizz +joshu agarcia +hin ted +harmon y +ga ur +flu me +el rey +doc ket +be ga +twitter nature +s water +pu gli +ordin ator +one sies +mu kun +cru mp +bur leigh +ar chil +aftere ffects +stro mberg +pim ento +meh ndi +lo bal +kin near +intech nology +holiday season +con summ +cli ffe +cer f +buffalosab res +? âĢ¦ +topo logy +su ga +sne ver +skep tics +shinde shil +ru h +mar at +ll or +hear thealth +ha vil +bhar ati +ar ang +weare united +w kyt +o tro +minne tonka +mal ag +g sc +Ĺ ï¸ı +un rwa +twitternature community +seym our +se ar +r nr +q ab +linkin bio +ku an +ha ku +ch aco +butt ler +bc wine +sket chers +shake up +ram m +pol on +photo aday +mosqu itos +fotograf ÃŃa +fli ers +encephal itis +el as +du page +terra pin +sath ish +har at +g ell +fe dor +disc ard +co ole +am ph +adop ta +ye z +ty dollasign +the win +sub trac +roy ston +once abc +od p +i im +fa kis +diplom as +bru ising +vene ers +tu i +thesunday times +shop e +moneti ze +mo ol +mann kibaat +khil adi +ipsw ich +electrocu ted +el do +cyber space +car naby +ãĤ ¢ +tech week +swing in +stocha stic +mall ory +li r +land fills +kala hari +fa of +à° ķ +this is +rap sheet +radi ating +ra pha +p me +niti aayog +ne gara +mand al +kra bi +iam k +hin ting +erup tions +dmit ri +ab ington +up mc +tc b +raj nath +multi function +lec ted +grin ds +dj ian +cad bury +burge ss +bron z +ang la +ac mawards +yah weh +pu ss +lei bo +lanc elot +bang kok +back field +b sm +as ce +whit mer +tou n +pre ju +max preps +j crew +ed camp +deport ations +cho cs +beat sby +ash worth +za heer +val ery +tr ini +sy sad +sun dial +sti p +sange les +san gu +roman esque +le al +lam ents +hit is +equi fax +clu tch +chi apas +af sc +zig lar +un qualified +tend in +stanis laus +rock chalk +ri vet +rhon y +ra ppa +man tras +fromthe east +dy ck +boy f +bi ome +ba strop +à´ ¾ +tw ise +perenni als +multiple sclerosis +mccar thy +disper sed +dau phine +ber ner +aubre y +xen on +ss outh +sar ahah +par in +muker ji +lu ci +hyo yeon +evangeli sta +ce asing +an dis +tim on +lu sk +f ha +esof instagram +duke u +tex tual +steff en +sagu aro +ridic ule +re unification +leap day +kra ine +idol ssa +hot shot +financial services +envy us +con templates +al ters +ðŁĺ· ðŁĺ· +ðŁĴ¨ ðŁĴ¨ðŁĴ¨ +ãĤ Ĭ +tu gs +sl er +pro wrestling +po ck +patri zi +nadi ya +hahahaha h +be as +wan ska +sle azy +ri ku +rad nor +r sv +nadinelu stre +man galore +kil gore +inno va +green leaf +ad mon +å¥ ³ +u ously +sung woon +sho d +sal erno +roller derby +r tm +pitt a +pau line +ni mitz +moores ville +lan ark +jav its +indv pak +hi the +here after +gri pped +encin itas +edtech chat +do pen +demo lishing +beck ford +ban h +ðŁĹ ŀï¸ı +ud ice +taste less +promp ter +nat ter +mi el +ii hf +han over +guj rat +dis dain +b news +aw c +ab g +ãĤ ½ +âĿ ® +y fm +transm itters +tigh tens +stel ter +sc ouse +sal liance +ir v +ick a +fa inted +dethr oned +bo tte +sa hil +rhon a +proof ed +juven iles +isuppor t +gh ton +fli r +champion ed +c span +alde hyde +zam alek +waf ers +sul tans +sn apple +re capping +n daa +gov t +followfor follow +discrimin ated +dg c +brid led +âĸĪ âĸĪ +for mance +fac ades +du pe +de mir +bl fc +biomar ker +sin st +ry ka +ple i +ny m +nur tured +moi stu +mal aika +gh ill +eli os +court ship +cal mer +an ey +ag ye +yose ob +ved anta +uss ell +um l +trick ster +th ali +pen and +pe et +ob er +loo kers +ia as +gam ba +ethno graphy +bor dering +bal er +an en +walk man +then ation +ri dding +pen rose +la ssie +hydro ponic +east coast +wwe universe +tom boy +to ir +ro dan +p th +on ef +care ss +bee z +the comedy +son goftheday +sab or +rten ews +ro hr +peak y +pare des +in come +gre l +en is +chocol atier +cas sa +aon b +an f +ampli fication +accom plice +wel by +stre wn +sand well +o for +kim on +kim my +k dp +ik al +hoo pla +gan as +ei steddfod +drum stick +demonstr ator +centrifu gal +bl chat +ìĦ Ŀ +vit er +ssy dney +nan om +deter red +anim ating +aeronau tics +ab ull +tick ling +testic les +soo t +sax ena +qu ine +pet us +mousep ad +jo ols +german shepherd +b th +alabam af +ðŁļ ¬ +ðŁĩ¸ðŁĩ ¬ +uof glasgow +tra bajo +th ics +rap tor +pro stitutes +orlandoc itysc +heart disease +first nations +bo ces +ãĥ¼ãĥ Ī +âĩ ¨ +yueng ling +talladegas upers +tab ula +ske l +re affirm +pan es +ir k +d oun +chan tel +bron t +wether by +spec savers +sch ema +precin cts +pan acea +inf eri +gint ama +fir stal +fin sup +e studi +de in +c á +yu van +the bear +paley fest +page ants +krist off +har dik +ha shanah +cr g +bu do +amli ventv +a jan +ðŁķ ¸ +ठĸ +susten ance +onlin ed +nostr ils +mol ar +f sl +ente bbe +de ed +chival ry +bib chat +aj mal +adju sts +[ !!] +ðŁĺŃ ðŁĴĸ +w mn +qu ang +pil lai +misogyni stic +mar bs +its me +holy spirit +h se +critic ising +co ff +cm w +chel seaf +ch abad +ad ry +uru gu +tom bo +pl u +mass acres +jack o +it l +id capthat +hl f +go red +chri ssi +av ani +anthrac ite +am ous +t ity +su ggs +se maine +safar icom +po z +mey dan +medi al +kan en +je taime +il ver +gu adel +gre nier +duchen ne +ale ssia +abra sive +wind fall +t itious +ra yy +mind blowing +le b +kati a +in charge +fu d +chit ra +alvin foo +re dress +me gha +ha grid +du champ +cudd led +buc ke +woman hood +vey ron +pat ton +ou is +lar ch +j x +fla via +bran ched +bas ses +agron om +reach er +ram ses +ra han +prohib iting +pl er +pe eve +oo zing +luke warm +kru sty +hai lee +el d +ardu ous +' .... +watchthis space +vi ot +road runners +q mjhl +pel le +ned bank +mos cone +mam et +lit is +kosci elny +j uri +j ra +in am +han zo +hahah haha +gamer girl +consumer ism +chipp enham +centuri ons +as ya +ancho vies +ste ver +sk r +roo ker +que be +organ za +nar ry +l itu +kl cc +accompli shing +Î ´ +u she +sw d +official helly +montre ux +len ingrad +ic ola +her kim +fuer te +e wn +dilapid ated +dau s +colli son +cold war +boo g +à³ Ĩ +to dor +ter mite +shine down +on ye +mer ck +law of +garden design +fighter z +de grading +bra u +ange red +al labou +wra h +to logist +smallbiz satuk +s wati +mon gol +mari age +man uk +gold finger +em mal +cit rix +ar rhyth +quadr atic +pat chou +mcil roy +iteach math +art v +Ø ¢ +valdo sta +to ks +ste ppin +sal gado +moo k +maz ar +irish times +comment ating +brown ish +ac ism +ãĤ § +play list +ol f +lucha underground +kol b +gc f +ðŁijij ðŁijij +show rooms +rafre darrows +on nbc +mew two +kondap aar +jud as +j illa +goal scorers +g autham +dump trump +de bra +cov fefe +chur ro +t ando +ly medi +ergon omics +capit alists +capecod times +ðŁįģ ðŁįĤ +ws of +squ ish +om c +meghan markle +lha sa +jan ney +hust ings +photo set +kis an +gard ner +ben zo +bat am +z ito +sub ju +sar k +pun itive +maure en +kaw ai +groupp alestine +fi j +en lists +ch ini +bang a +w abi +vit ali +valder rama +sou thea +p ku +om x +flori an +cn d +bt u +ast ley +am ai +ach amp +heath ens +go lobos +dan ia +cn rs +authori ze +ar oo +. [ +wonder full +w pl +taun ts +sonom achat +pi otr +pan ache +mc n +exper t +do than +alex i +ðŁį ī +íĶĦë¡ľ ëĵĢ +u calgary +tigerzinda hai +spin nin +shar inge +migr ations +mac don +ma ssie +key pad +karls ruhe +ili g +har issa +ha vok +figur ation +d ld +cle arest +broad cast +brit pop +biom ed +att t +arto is +zh eng +slu tty +ser c +ro fficial +plex es +pe du +moul ds +le ek +dak ot +dais uke +chry so +bon fires +tick les +stun t +sikor sky +gr d +def rau +chimpan zee +bha sin +worshi ping +w ylde +w ole +thejohn abraham +s re +ra ig +pinst ripe +orient birdclub +mc morris +lumin aries +lou ch +la shing +gro omer +elo we +clut ching +cal ving +accessori ze +ðŁİī @ +the todayshow +t ld +spectro metry +pa ka +minot aur +man gi +karant acker +hay stack +fr d +ef en +diabe tics +bul i +av s +andr és +al ty +x k +uni e +sof itel +shi do +riteish d +mystic ism +kundal ini +ho te +ho sen +hin kle +good luck +go gi +fried rich +con gle +chap lains +bur net +ang lian +é « +ston ey +rede eming +random ness +pr sa +ober on +newh ouse +gonz á +den im +del ph +con ic +an kit +wolf ram +wine bar +unmistak able +power play +nag ging +lincss kies +gh h +desk tops +bore anaz +as port +ad wala +íĺ ¸ +theyre theone +sal dana +nes se +ci an +chemi stry +can is +b hc +zoo t +x an +sylve ster +ici dal +hmo india +gav i +gam ma +g itt +critic isms +bi do +be bold +aashi qui +tu ff +street life +ro mp +monk fish +mal evol +looo ve +k cl +gad get +d bu +ben carson +ail a +ì ¡ +re playing +noc turn +labe ouf +j hb +game on +ast aire +% ? +ðŁĺī ðŁĺĤ +ri yad +nyc parks +nm su +ly mph +kwan zaa +in sg +hack saw +gh nessy +dand ruff +basti an +au ber +atla ssian +al icious +wel ker +ris sur +pra h +pit ino +mt w +la thtr +jong suk +in subcontinent +ev elyn +dav ina +cri bs +cre u +cit ys +chin chilla +canter bury +adhe sives +tower of +su ite +rapp ler +op h +new sin +don ot +co ts +bair n +ãĥ© ãĤ¤ãĥ +w aging +sl acker +siem en +sand bags +of e +ig ars +hygi en +hcl dr +fuerte ventura +fore see +f td +f sm +ev ict +bun g +at tica +whitecap sfc +ugl iness +ko hn +in animate +gaf fi +fe yn +empire fox +dv ent +co inde +chuck d +aber gaven +ðŁĻıðŁĻı ðŁĻıðŁĻı +verse oftheday +titan ic +microsof tedu +l atives +eri ka +au f +adjec tives +ðŁĶ´ âļª +z ari +xi jinping +vir ul +the ville +tar ot +su va +s magazine +ri ggins +py e +isi er +der ick +barn staple +thu man +sprin ters +r mu +mexic ana +loo ters +lan i +jaeh wan +hi me +fr u +east end +cr amp +char izard +out ons +ni ppy +f xx +d agu +sky cam +ner kondapaar +chu gging +argent ino +alab i +âĿ¤ ðŁĴĻ +u waterloo +redi aries +mi da +jar os +in ching +hon i +gold ilocks +dra pes +d td +bi ed +anemon es +aku mari +ak hil +yam an +vel ife +surin ame +ru ud +r hd +kill zone +i my +hur a +es inc +cric keting +cor busier +bridg ford +ble sse +as sur +; " +wal lah +up r +thor in +sc bwi +re mus +ol oured +news stand +new sonline +mal li +mahar ash +li tho +jun ga +il ies +first friday +cu evas +clo sets +bur j +bac c +b hs +ae sop +a alto +wembley stadium +wal len +under graduates +stag g +pla stering +le l +ity fc +it ur +im gur +homec ooking +hear se +g se +eski mos +dr ys +dailymail uk +bi ot +arav ind +ðŁĶ § +âĿ¤ï¸ı ðŁIJ¶ +âĿ İ +tapi oca +syn cing +sw p +mcgin ty +i wata +hon ing +de graded +boy kin +aurang abad +aun ties +vienne se +unexplo red +pal u +look alikes +ham sters +for taleza +ed am +diction aries +care y +ty ree +tom tom +stra vel +re aring +periph ery +mcle llan +ju hu +i je +gd x +dent ures +au gie +architec tures +am ador +ac at +yu g +ve he +sh is +sall ye +kut v +impossi ble +chat t +billeric ay +war birds +turn in +tol y +the mb +sc lothing +nbc bayarea +lun areclipse +li be +kin ross +et es +dar ke +advant age +wing ers +stri ve +ru se +modi fying +mcilroy rory +hi ght +hair loss +critic ises +bob bi +autonomous vehicles +ar go +̲ Ì +sub bar +spar kle +sar dine +ran aut +nu c +na sional +lo kom +impeach kavanaugh +folk lor +defen sively +bigg in +ave da +غ ر +ubere ats +sy mon +mimic king +ini um +eatmore fish +ca zor +bo ds +a fore +< --- +ðŁı Ĥ +winni pe +tooth ed +seren aded +har ic +drow sy +domin oes +dog finder +costab rava +bob sleigh +bich on +all iteracy +ðŁĻĭ âĢįâĻĤï¸ı +ಠ² +out lier +n ites +lanca shire +idi ocy +guz mand +far ris +caernar fon +bar ney +az eroth +au dra +amazon prime +x haka +valent in +tumb led +t ph +retro spect +rajap ak +ni kes +nad i +lu br +giov anna +elek tra +de ku +cl b +cash man +art lover +anap hy +! ðŁĴľ +⾨ @ +west virginia +nur ses +mac on +hul k +heath ers +ach ak +ðŁ¤· ðŁı¾âĢįâĻĤï¸ı +sp ore +ling ers +kid ar +har poon +gran dopening +chel an +anaero bic +à ° +toyotar acing +tar on +rays baseball +pilot life +ori vera +kur u +c wu +alan te +ab ate +wil ber +tou can +the fosters +shar key +r illo +lo per +life goals +jam ba +gall atin +coin collecting +bhatt i +è¶ ĬãģĹ +utri ents +s rd +po h +o ds +fun ding +fili pe +digit ale +cycling life +c vt +aband ons +tem pah +tar sands +stat a +sher bet +prosthe tics +pi ppen +ne sted +le va +ferr in +da ho +af ina +sports radio +sam edi +li ffey +lex is +gen eve +cal lu +bri st +bar ty +bar ic +you suf +it up +woking ham +wizard ry +we ster +si di +pan sy +me des +ke ya +hilary duff +de barge +crani al +win esof +symph onies +she hu +re sp +mis ano +lin der +infer nal +engro ssed +dallasma vs +cron kite +ðŁ§ ļ +ãĥ Ĩ +se vent +sd avis +pru d +olu min +hog manay +gin n +et ted +cul kin +corro bor +x ti +we ck +ud der +sta ines +reig ned +particul ate +nu mmer +gro sser +gro g +gon awaz +f bc +encan ta +ce i +(( (( +ventric ular +tr k +ta al +o ong +no vena +n cr +lob bies +ini showen +in oue +i up +hallo we +fore seeable +con done +vegan food +pr ally +moun tb +mi ki +jake tapper +gra iny +gil i +gh s +gaw ker +forever more +experi en +ex asper +ep lus +chuck les +cervic al +anom tv +ah old +ðŁİŁ ï¸ı: +sphy nx +shon da +ra khan +pel vis +kil burn +ic or +as at +york ville +travel diaries +th ack +shan th +sear cy +n dr +looooo oo +lip gloss +it achi +hartn ell +gar dent +chriscol fer +ch ies +bor d +bla ken +nep tunia +my switzerland +mu mmy +d de +cl twx +ac ek +: < +sho ba +rico chet +mark up +fy re +fire rescue +christ en +al eta +zo oms +youre welcome +wi gw +unis outh +twil dlife +sun ning +sin tra +seed ling +ru gg +public safety +pitch ero +mm ff +mid fielders +kn k +hyuk jae +fif teenth +emb al +bra zier +ðŁĶ¥ ðŁĴ¯ +sp itta +pa chel +jour dan +gold mine +flip board +eric o +az adi +ë¹ Ī +à® µ +visit london +reco il +que t +oc up +ni vea +new combe +k ome +foss ili +duck dynasty +dev ents +csharp corner +cheek bones +aishwaryarai bachchan +ðŁ¤ ¡ +æ ĥ +âĹ ¡ +yar aj +tre llis +stra f +myrtle beach +ligh thouses +cr unk +ðŁļĢ ðŁļĢ +ê° Ģ +unsc athed +tt ur +team sky +real y +pin na +orthodon tic +nike sb +let me +lean ed +gro en +dono hue +bra sh +traw ler +taxi ing +ros sum +photo art +pakh tun +origin ate +nu ovo +more over +man ti +machi av +long fellow +inj ure +hen y +ces are +am v +ðŁļ ĺ +t lv +ru grats +reg als +pad alecki +lun ga +kh wa +jan ette +fc i +de tours +cle ese +ðŁĺĻ ðŁĺĻ +ãĢ ı +top gun +peak challenge +le thar +institu te +hemat ite +fri sk +( ´ +ðŁįº ðŁįº +vote fifthharmony +un checked +th rash +sassu olo +ra kyat +proof reading +new deal +ma ree +lo ins +letour yorkshire +godd aughter +elsin ore +companion ship +bon fire +big time +beast fromtheeast +ðŁij ¬ +el salvador +asse sses +amo ore +ahar ashtra +adul tswim +swan sofficial +star c +se wa +sa xo +old man +ga on +centime ters +bluef in +bet way +ast wood +art sakh +are al +ag ee +ag ape +ðŁijį ðŁijı +vul cano +unrival led +tues news +se khar +sac char +oni an +kau n +im position +goul burn +fru m +free man +fou led +fin all +eger ton +dri e +x uan +victoria beckham +ver min +trun k +tam aram +super mario +need for +mess in +me ar +io g +fe ces +ce tera +cab os +tren tino +re paint +on etv +off screen +niger ia +mccon nell +kin ship +fore igno +christma scountdown +bag well +çİ ĭ +yo kai +yar os +wad dington +ur band +real hughjackman +r wy +ou ette +mo res +llang ol +fly thew +dl r +bis choff +al ak +اÙĦ ج +vent ur +tab bed +st ls +seam aster +ratt ler +pro cure +nott s +con forming +ðŁİ · +william stown +var ou +tranquill ity +th rissur +sn ark +sevilla fc +pe asy +paper backs +law an +day uk +app iah +uri ah +som mes +showyour hits +sc ancer +mal inga +lauren ce +hurricanef lorence +bride tobe +bri and +blind folded +beg g +azzur ro +ðŁ¤ ¼ +tu stin +scy the +ma din +luxury homes +ker atin +gw yn +ff d +dam o +bt ts +be cer +Î ² +wid gets +var ner +tbil isi +shock wave +sa hl +rock wall +qu eria +kel le +invasive species +flam ing +ve tri +surf boards +sukho i +ox one +mm l +fr act +c sul +ಠ¤ +w ss +sar u +ro by +ra bin +myan c +erup ting +des ro +ci aa +ac ro +thyro idism +schla fly +parksand rec +mut ated +lifeis strange +gh y +ford mustang +dor ney +cat o +body guards +ani els +è¶ĬãģĹ ãģ® +sl g +s iting +resear ches +lofo ten +i and +cop ha +assemb lage +; - +wo t +tcd dublin +sten ch +no sy +net worked +ma eda +gher kin +cuper tino +com o +wre ak +shel f +padmav ati +mon ti +lol lies +ho tb +entren ched +tron dheim +srini vas +shor ty +shiv n +projec trun +low ly +lin wood +kier on +eth el +es ce +ðŁĺī ðŁĺĺ + ³ +wil ts +unc tad +smar ties +pat t +ne jm +mad hav +jayalali thaa +g tv +the city +o gle +mu sing +mcke own +matri x +f sf +Ñ ĭ +poinset tia +magne tic +fle as +ed hi +ed bookfest +bow o +ba har +x lt +working together +wo a +with refugees +ss chools +score line +run for +regre tt +ha der +e it +case study +ad ot +ab ha +ðŁĺĨ ðŁĺĨ +trac tor +sub culture +special ises +san u +pl tw +mis led +mari kina +maneu vers +hoo ps +gri me +fort lauderdale +dy spla +cel o +aw am +at our +Ë ĺ +william sport +sk int +å¹ ´ +t anna +shou ses +rheumato id +pla sty +pa wa +oscar pistorius +nott ingh +m we +lor raine +kar tel +i dont +har te +ghost adventures +g listening +ep som +a acc +ãĥ ł +âĶ Ĭ +watch dogs +time x +spec t +sp aul +salute to +rin se +qant as +plur alism +neil son +mo ine +maha bharat +mad don +electroly tes +du ches +adap ters +ا٠Ĭ +valen zuela +r ca +pit man +o ars +micro plastics +ho tt +ho ti +dou ma +dimple verse +der nier +commo dores +b boy +wor ri +seung yoon +or is +no ban +men shealth +i dy +hi g +greg orian +f sprint +conj ure +cazor la +but chery +ad versary +x amarin +thorn berry +t ü +sw ann +sta al +santac lar +repe aled +quin tu +qu é +per tur +mé tis +man ning +ic es +go ji +agne tic +ðŁijĭ ðŁijĭ +zo d +wal dron +tree hill +spo p +ig or +hal ley +cotton candy +ar kansas +acceler ators +vis alia +tr iceps +qing dao +od ac +li key +lat enight +itv corrie +empor ia +electron ically +cer ritos +b ns +are cords +ad du +ðŁIJ Ĥ +ve dalam +spar se +on tap +monoc le +la il +gn t +car dia +cap sic +bou w +bear dsley +bas i +plan ds +pi et +personal trainer +ow er +ol as +janu ary +jack and +environment alists +dr qadri +dog fish +vuel ve +th waites +steff i +schul man +les ville +food tech +stephen fry +pos ers +cur so +cor bin +br uni +ðŁ¤ Ń +tweet like +sme tics +rene e +post malone +pat ter +p sni +or no +ll am +i du +en tro +bl ica +b nd +the park +son atas +prime day +paw some +official bsb +dro wn +danger field +beach clean +ðŁĺį ðŁĴľ +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ +wefly asone +wa seem +rac kete +pri des +op u +grand slam +dolphin project +cun ard +zi ppo +wi les +sh b +san toro +muse o +me mos +inde xing +dri est +bronch itis +arte yart +world peace +theat ric +ke ely +inve stor +far ru +dam el +criti qu +coron et +channing tatum +body work +as ser +éĩ ij +road ing +quin te +nation ale +gu di +great ful +fu gees +adri enne +sal ish +quot as +qu elle +pro team +neo liberalism +n elle +khal ee +jaw bone +impair ments +go ggle +dul la +di ari +black adder +ven ge +spro gram +she w +science magazine +lind or +h pi +forthe people +fac esof +fab rice +ef ra +d ne +cate rer +canisi us +bu sca +bran den +bibli ote +bee keeper +ation matters +arri a +ðŁĴĸðŁĴĸ ðŁĴĸðŁĴĸ +y au +sub ways +state ofthe +sher ri +rho ea +patchou li +lu mpy +la vor +hal wa +creek side +coler aine +car bure +bloom in +albu s +al bers +ta ha +ren ata +polter geist +net gear +kaz e +dazz les +cre mation +chan baek +cal oo +mour ne +kor o +ha ight +gas se +fi m +eg linton +desi der +chri sm +bat tering +ak onda +ðŁķ ¶ +zip an +sen tedcruz +rin go +re me +or cs +mist ook +marthas vineyard +lu th +li vid +iti l +er tz +tag h +step dad +staten island +rol la +riode janeiro +province town +lu lar +ken e +expe l +boom town +bh vn +Î µ +sh as +se is +quatt ro +p fe +over use +moder ne +hype beast +folk music +fish tail +ca jon +ang ora +ðŁĴĶðŁĴĶ ðŁĴĶ +ðŁij§ âĢį +no filter +mc gann +lam or +hist stm +el los +cre we +art nouveau +am atsu +ac cs +a em +ðŁĺ Ĺ +ðŁĸ Į +yuk o +turk ana +torch wood +spi ffy +si ii +sel fridge +roc ca +ro chel +mat er +life with +len i +kil le +ij s +hard ness +ben net +t ml +son es +sic ili +road to +pric hard +p va +midd ay +chihu ly +back fires +ak il +ade v +& / +âľ ª +wf my +supere agles +rang as +py ri +pix ar +pan khurst +la hore +ho stel +expend ables +el nino +circu lar +bizar ro +be bold +ais d +tre ce +sr f +orland om +o ed +ny time +munster rugby +invent ories +gate house +gar m +camer a +be then +asser tive +ìĨ ¡ +âĿ¤ï¸ı ðŁ§¡ +we p +w td +t mp +coven ey +ceil idh +born to +aw f +autom akers +asi l +ðŁĺįðŁĺįðŁĺįðŁĺį ðŁĺįðŁĺįðŁĺį +wol l +thedaily show +t dm +sher man +scru ggs +samo an +rear view +over takes +mad max +geo logy +condi ments +by num + ¤ +wood fc +tan gi +san rio +oo ster +le u +k wu +hiber nate +cay man +bewit ched +ali bi +you ll +y fc +win ed +warat ahs +spit fires +sne ha +love dublin +impul sive +ibra xton +hop kins +har o +blue jacket +bee be +ar paio +ðŁij Ħ +Ø§Ø ¹ +ve on +tham mer +sh ta +pudu cherry +pitch perfect +me ine +me gs +li mp +j sc +fu dd +can et +bel k +acro bat +ó w +west meath +sa opaulo +pro jet +lam ba +frit illary +er by +dg al +deliver oo +ye eeee +vul garis +start led +repe ater +ray ban +ra val +por que +o han +ni eves +mur ica +kenne tt +haar lem +gro he +constitu ted +best boyband +èģ ´ +y aga +turn key +sx m +su raj +sk poli +s di +psycho social +nar cole +n and +level up +leis ure +kis d +jam ia +house work +cra dle +compost ela +comp iler +anne marie +aleksand r +su bic +season ally +king sland +jam b +jal and +f blogger +drey fus +din ed +cron enberg +conspic uous +co ton +ca pps +bo hra +bo gum +bal aya +americ anc +u mic +pau s +o kie +mul roney +mer maid +melo drama +lis more +it ations +im mol +ful mer +br ining +bol ero +bin h +ast y +we standwith +thunder ous +stub hub +ro by +r kc +path um +o ac +nb r +mun ir +legi ons +jeon ghan +habit ation +ge ht +cappado cia +( !!!) +èĭ ± +yn om +the grammys +tab lo +rec er +pu ller +ny ack +new beginnings +maynoo th +inf low +en stein +de ano +cr in +confection ery +berlu sconi +ash raf +aby te +âŃIJï¸ıâŃIJï¸ı âŃIJï¸ıâŃIJï¸ı +wing sup +syri za +presci ent +new sad +nare sh +lis zt +gre ath +extra pol +divest ment +dis orderly +cu st +body weight +ave don +walk able +red fern +push kar +pro kof +mind charity +marin elife +dul u +as son +win kel +to ky +the p +t pr +refu ges +phoe be +ec fc +comic con +bro od +br m +asam sakti +adulter ated +qu illa +pol anco +po vers +no shame +montan o +kaz u +ham mocks +gu ana +el v +der ange +delle mc +bic on +bi or +bean stalk +ve tt +saur on +or bust +ol ic +li zzy +ik at +hand cuffed +fa p +ani an +ac ell +à¤ Ń +whatyou eat +syty cd +star cinema +s net +rat cha +om b +john green +jit as +h si +fun time +e ac +dad lani +clean air +bay e +zo a +wo on +wh smith +vo wels +un secured +steph on +st é +per v +i aea +ger maine +dis respected +birthday boy +ba on +as gard +âľĬ âľĬ +us atoday +tri age +tho ts +kipp ur +fam y +equal pay +dncin phl +del ph +dd w +al qaeda +" + +௠ĩ +str zok +sh ome +on ward +kas auti +hy ping +excell ence +caric atures +bblo gger +ay n +winter time +sac co +or no +musical theatre +la cher +juni per +happy place +ero de +dt p +color ad +rak uten +pla ss +no ite +mccul lum +hosi ery +ether idge +enrique iglesias +dru id +dra gan +com unic +( = +ðŁıĪ ðŁıĪðŁıĪ +uncu lus +twee ty +squaw ka +p russian +oro ville +m pe +ker i +colin morgan +ay ush +_ âģ© +. âĢĶ +à ¹ +shed d +she en +rose dale +pa chy +mix mag +incen diary +gil git +cat ac +bold ness +ambu shed +alco a +ü n +so con +rati fy +plu ri +on air +fl p +eng els +eb ner +bron zer +bro s +alig ns +ķ ï¸ı +ì Ķ +prosecu ting +profo sinbajo +obscur ity +n tl +lu ger +gonzá lez +epile psy +bo fa +ali fornia +' !! +ãĤ Ĵ +ठ² +wwe supercard +wt f +win ch +wee vil +twit song +tv guide +supp os +spir o +i see +ate ez +vy as +soom pi +ny dailynews +hand loom +co bble +bolsho i +az ing +aw olf +an kit +ðŁ¤¦ ðŁı»âĢįâĻĤï¸ı +way land +track andfield +tear oom +scoundre ls +po b +over fishing +en ia +bar bosa +alicec ooper +) * +we day +in activity +hel lish +dor dog +axi om +ë¸ĶëŀĻ íķijíģ¬ +thra shed +su tter +stra iner +so ren +ram o +ope ia +nikki sixx +ky derby +flori dian +callaway golf +c ml +bran ford +bird house +baby face +the cho +simon cowell +move able +meatfree monday +lo red +laun chers +ko alas +kingscol leg +ja v +gor gonz +femin a +car mona +an sky +z ep +verte brae +time y +skill susa +shir in +ser gi +re gan +pha il +north gate +mo eller +keralafloo ds +ke swick +iti e +har psic +fin cher +dc l +carmar then +amit abh +alzheimer ssoc +ab jp +pace maker +ore m +lyca productions +en sued +ee c +donald glover +bot tega +wy z +run d +pour ri +o dys +my ron +le ti +la dd +jc ps +heal ers +greys abc +fair mount +bru v +anton in +ajay i +ê² ½ +spr inge +press forprogress +p ase +lo ons +kellyanne polls +ic are +fre da +fox conn +de france +ag all +ac ne +[ âĢ¦ +ðŁĮ ij +v gc +show ground +pound land +olympi que +manife sted +kar as +jack fruit +instru ct +in our +il ab +hel sing +al meria +ì§Ģ 민 +Ð ´ +ver ock +tl m +oc elot +gas pari +data security +cher ub +c vb +birth ed +bel voir +bar rack +bak lava +ad min +ðŁĺģ . +un er +tech ni +su ena +rot ated +penny dreadful +pel tier +mic ally +f naf +cipri ani +auto car +any day +ÃŃ o +vignesh shivn +sr u +re ttes +mb p +marsh alls +legi ble +labrador ite +e az +day ne +con val +ci se +chimic hurri +black currant +bar y +ba ale +ash burn +ðŁļĹ ðŁĴ¨ +y adi +su bang +save money +on in +nhl allstar +k fc +grin ders +gates foundation +âģ© ! +ti dy +sky lines +mor land +full house +ex l +every man +to ft +pp l +perpetr ated +nand ini +mines weeper +love of +ingra ham +en elson +da as +cam pari +ann ul +a art +ðŁļ ¦ +stun tman +spr inging +nou vel +million dollar +in hib +her der +entang lement +di spl +com batting +battle star +whel p +tru ssell +srebren ica +rt é +o tica +mumb les +er st +coc teau +uc f +summer ville +suggesti ve +g pus +escar p +ed son +dg asm +cap ta +ab ir +zak har +woo kie +victi mized +thu pp +the book +stra it +sports man +scher zer +raj kum +own it +mc cour +le ib +hor ia +holy field +excel ente +est ilo +el am +e ben +coyo te +amazon ian +rocket league +ritten house +public lands +mat ador +manife sting +kar n +afri end +w yl +w lc +t mb +qui que +patriarch al +p sac +j ago +gi bby +de colon +contra ption +brid ger +astoni shed +å ³ +water sports +timeout london +ten o +quanti fy +nap time +moh fw +know sley +hei fers +gasco igne +free k +ei ffel +collar bone +brazili an +vi dad +uc davis +r itt +open street +moun ties +min ton +kryst led +k uni +hol ton +flash point +duali pa +will be +v alls +ry t +re issues +na die +luhan sk +l pool +guil le +di strust +des man +apo o +ðŁIJ ¤ +sin tl +rever ie +ma kon +le ve +jak ob +hor ni +dd b +cam ryn +ë ¡ľ +ymoun tains +wedding photographer +vit oria +tome try +tal ons +sche in +ran jit +pau lin +past el +or atory +neve rending +mon fils +library life +li si +indi atv +bin ks +bi da +ai kido +victor s +tur day +sport smen +shru gged +sal ves +re gn +peer less +pate k +jj ba +guern sey +exuber ant +black berry +wel fare +stu deb +quay side +nar ay +lou don +f wx +dise ño +cel e +bohe me +awe b +antic o +anthony f +ador ably +aamir khan +stock pile +pe tty +pas se +pa stu +moderni zed +man z +le vers +jun tas +gras shoppers +a stre +w ta +thegreat awakening +pas coe +ng k +jen o +d se +cla vic +& ' +we faq +we care +sun spot +rai den +optic al +once acard +jav dekar +house made +first born +erec tion +de angelo +bal ah +alp ina +ðŁĴķðŁĴķ ðŁĴķðŁĴķ +sla shes +ra ucous +pal ac +hey man +gh ir +fo yles +crou ching +changing lives +au la +as ghar +apol lon +ab stin +san kal +monro via +l alo +kangan aran +car yl +birth time +am ano +ðŁĴĽ ðŁĴľ +unapologe tically +som o +sh of +sey i +prop tech +ong ed +ni hon +nas ri +mat tia +man ik +lo gar +jur ong +it ti +hay dock +don russ +dis respecting +carnegie hall +ano dized +© ï¸ı +tex po +ss an +robin williams +pun che +mon ero +mo hair +manit ou +interst iti +home red +fsprint monday +tan u +su bbing +shiel ded +ratt ling +rak itic +quest love +man orama +look north +jun hyung +isma el +grumpy cat +fro lic +escal ators +dÃŃ as +de bar +colleg elife +cle v +brick work +bom er +all ama +y urt +vul a +spreadthe word +ped ics +lom ax +l uring +kr u +human resources +he dging +har wich +goo ber +crutch low +cly ne +y v +thessaloni ans +spir als +plant ar +hang man +hai fa +gyp sum +gl p +gameof thrones +advo care +wa yof +star times +or a +occu pies +misogy nist +ku d +kello gg +g under +foli um +emily bett +big blue +aw oman +sp aring +sop p +par ound +mind body +mail boxes +la zer +j so +great british +gi lead +f ba +ch aves +ce vents +c so +aw oke +al aw +ak ong +young thug +ru ll +poly styrene +pe ñ +oko ye +lip stick +ke fir +hi x +flu oro +di aling +am ana +traxx as +su j +stra m +sas soon +mm snippets +han if +fiduci ary +co stan +blu shes +av ale +af p +/ ' +è ĩ +wedding venue +univer se +shrin ers +reli ef +ob in +mike the +mat os +jo ckey +jam in +intric ately +il da +gli ders +ex xx +yu lia +wg me +w pli +sc lo +ra ker +patter n +ob it +master works +landsc aped +l sp +l ers +kav ita +ih g +fly past +extracur ricular +end anger +cape breton +bra x +bor row +action figures +w bbl +tal kers +sau cers +re adi +mam mal +m sport +i ee +g hal +back light +b ww +ak ane +sti ve +my peakchallenge +il ana +sand ed +pi ety +ke es +hur l +harry shum +eag s +dro ids +do v +city news +brai ding +barclay scenter +band era +ภŁ +y ada +wedd le +varou fakis +swel tering +special needs +sjo fficial +sax ons +riaz theboss +rashtrapati bhvn +mi ms +kro q +har nesses +g sma +freder ik +dy an +colo res +centri st +brain wash +be ow +ay ton +ax o +aureli a +ace vedo +ç¾ İ +ا٠ģ +yas mine +stel vio +scoo by +mul van +i got +endo scopy +dil bert +ðŁĴħ ðŁı» +x drive +power train +h de +foster care +eloqu ently +carbon dale +wh all +un ing +ti fying +superintend ents +sm fh +of tware +mu tts +krystled souza +far thing +transm itting +sig net +portrait photography +o varian +kit sap +kar ya +d alian +b sb +b ho +ari zing +ãĥ³ãĥĢ ãĥ¼ +à¹ģภ¥ +sam bal +new in +music ed +monaster ies +marke tresearch +lovel o +di op +deta inee +whe ate +sol er +sa wyer +red ales +lan es +dan zig +bac chan +b iss +austr alis +ab acus +what vegan +upp sala +tull amore +soci ological +s van +ru ffin +nepo tism +ms gs +ke mi +ka hu +ex pun +ec ks +ðŁĺģ ðŁĺģðŁĺģðŁĺģ +whit church +w elive +un block +u cr +tow path +sen er +rede ye +r ch +pear land +o afc +lamb ton +imagin ations +fashion wk +daily doodle +ay man +apart ment +í Ĺ +tahir ul +sei do +ok on +o jos +mu dra +mor tuary +is thenew +fore fathers +fern dale +del phine +carre four +bor gs +ðŁ¥ ¶ +tie fling +th and +sel hurst +re ya +nei ge +mi ha +medic board +jann at +i movie +hol dsworth +gu en +gat ari +garden ia +cho bani +ca sta +ben nie +yule tide +r se +proud coach +lu te +josh ane +gu era +gl ac +far mb +exu des +eng le +battle fields +ap akistan +ÙĨ ا +ss gt +shar pest +power shot +mar fa +laur am +harryshum jr +go ole +espan ol +dis d +cas sel +cam ise +argon ne +yos bourne +uk bloggers +ta kay +m strong +lubric ants +kine siology +kas am +game sh +eu ri +disc golf +dev ans +cha os +auto desk +âĿ ¯ +wim ming +wa id +valiant comics +simul ating +po pes +on dor +mari anna +lop sided +isab ela +game maker +flead h +easter weekend +bhu mi +bar ges +ani shin +íĺ ģ +sand ara +may i +lo es +kin son +godav ari +b fast +avi onics +ab elle +. ðŁĺī +loveyour petday +kru sh +import ers +fro mage +east side +e ey +c mom +bo die +bl x +ul cer +tm h +sa ito +reti ree +ps al +pret ties +maritim es +magalu f +m fm +jenson button +in am +car hartt +bun da +avi gnon +need a +ip f +ic le +dews bury +bar ker +andre ww +ðŁĺĿ ðŁĺĿðŁĺĿ +ther mos +sonic thehedgehog +m nc +la vine +glo u +car club +ag gs +ac tional +ac cou +ab cf +y atta +vis co +ver bena +syner gi +pri mes +phar ao +p ella +ner uda +mo tos +guel ph +cor re +bang erz +aw l +auth ackeray +all saints +ae v +un circulated +ste ading +precision medicine +o stomy +must see +mediac ell +kwe si +ju al +im c +ghet to +fla vi +em pathetic +dip brow +criti ques +cri si +ador n +ðŁ¤ ¬ +stap h +rol le +re tur +bab yyy +ye et +wild horses +wel wyn +stop her +see ger +reiter ated +nl ds +lo ge +head ings +gorgonz ola +en caustic +di u +di sta +dam me +ch iron +bike to +ðŁĮ¸ ðŁĮ¸ +war is +usa hockey +the placetobe +snow boarder +sheskindahot vma +sal ome +owen sboro +k ler +im perfection +if ta +house keeper +gu v +game changers +est amos +bu kol +bom ba +tol ling +steal thy +sta x +sketch up +sc lu +pol and +mis cha +jin ja +gre go +da star +as al +ar arat +ãĤ¤ ãĥ© +ಠ® +tw irl +t ws +pu cci +par ading +kal am +is fahan +himach alpradesh +et l +copy righted +co heed +ar kin +ah n +ag ad +ack o +ac op +ðŁļ ľ +ðŁij Ĥ +âĺºï¸ı ⾨ +yan ew +un divided +ull man +t q +st ds +pa sion +minim alistic +menis cus +jo st +ich thyo +gol e +wsb k +spo kane +leav ing +kan n +iter ative +cel ica +bl arney +ðŁĴ Ĩ +zz les +womenshi story +vick sburg +un p +swa b +sof tball +ro or +pamp ers +pa ch +ni ya +neutr ino +it f +haver ford +groo vin +fa thom +f mx +art space +ab ounds +âľĬ ðŁı¼ +t ams +n ham +ju ggle +jol ene +brandy wine +augu stin +ðŁĴ£ ðŁĴ£ +ëī´ ìĿ´ +z is +sm d +pa del +ni ec +man fro +ke iser +grown up +blo opers +bar tow +ad hi +run ways +rang i +portu gal +papp as +men der +mal aw +ex ert +amwriting fantasy +çĶ » +åĪ Ĩ +trek ker +tele matics +sf d +pap an +ou tro +optic ians +niker unning +lmfa ooooo +lal it +iso bel +fair play +expen sed +canary wharf +call for +be ster +ah li +zambe zi +ut ara +stru mp +sal to +pen y +om id +obstruc ting +ne re +kre bs +glyce mic +ext ant +dominican republic +cour ting +ar re +x eno +ren ta +new video +make the +horn bill +gu ero +fut sal +fertili zers +d di +constan tia +ó ¾ +trustthe process +tid bit +te ese +st ler +seri als +pr ate +lan ai +ge ta +feu er +bun dling +tent acle +silen cer +short comings +safe guards +pal atable +pag ano +missi t +epile ptic +ed h +de santiago +bur k +alab ang +wsoc tv +worka holics +we iss +uto pian +ster t +om ma +loo o +lol la +ho on +gre ggs +beat y +we br +up d +un committed +trivi um +t ce +pine hurst +maple wood +gor gon +ek ta +anthonyf joshua +son ar +oste opathic +gru elling +director siva +d th +boye ga +boli vian +tan gel +strate gy +st paul +shel burne +sch mitz +pla c +pal me +niti sh +mom mies +human a +fern anda +faver sham +di ana +chu ckie +âŃ ķ +vig nette +sail ing +pre show +li gt +kar loff +hou sen +h ft +em pres +be vel +be sh +âĺĥ ï¸ı +ta iling +silk screen +pri mal +off erman +mil dura +king sport +ferr ous +dg en +chair manship +ðŁĴľ # +sp outs +sil ove +schme ichel +s lau +ri ken +mc clintock +lu strous +k lau +jumper day +go atee +global isation +ari ef +after shock +zi ki +we aver +smo m +sa si +recor ders +ra is +pear son +ip as +i pe +humber side +f ce +buck y +bo ars +wis s +re ine +prob st +ph ong +intellectu al +handic rafts +fd fs +enterpri sing +cocc al +cic lismo +carr ara +b vs +ak c +ðŁļ¨ðŁļ¨ ðŁļ¨ðŁļ¨ +yy ah +web shop +sym biotic +stu bby +me phi +mb ang +e sea +but chered +u vm +revol ting +mac ca +hhhh hhhh +gun tur +el be +dragonball z +catch phrase +at tic +an ee +vo e +vio lette +unra veling +tu ms +subur ban +struc turally +stre ls +se ch +re si +puj ols +pras anna +om arosa +nar ro +lumber jacks +ja an +free book +boss man +black ish +av ali +ãĥķãĤ¡ãĤ¤ ãĥ³ãĥĢãĥ¼ +vo ce +search light +rejuven ated +pr n +mar th +goal scoring +gi vers +ga w +fat ter +vac ated +ts ons +ta pia +shri ya +oswe stry +op ter +now all +mas ss +lang kawi +janethe virgin +carlo sp +budd hi +brit ains +be eps +ak han +w ff +prep on +navar re +kas ar +gran dest +elev ation +ele on +bra ithwaite +beaujol ais +and ante +าภģ +yard age +stal emate +ol lywood +ne ssie +mam matus +inf alli +de ering +crude oil +angu s +am ex +!!!!!!!! !!!!!! +olm sted +lin n +iri ses +in xs +impac ton +faul k +curren tly +ba az +wan ews +ske wed +shind ong +re po +p itu +oo ops +mar zipan +mar te +make sme +h bs +gedd es +g bm +first time +es ks +asi ago +sar copha +nal oxone +kab out +ing rid +globe trotter +c sir +back fire +ba reng +y w +sa adi +q asi +opportun ity +ni khil +ms v +mau ri +iron mantri +iron man +ingraham angle +indie authors +grey son +íĭ ° +x mr +studio green +se mma +ridicul ousness +rec tory +kon do +inf ancy +el clasico +deli very +an caster +ach in +Ì ¶ +sv pol +sun o +sp anner +solidar ity +ro hn +eag les +d ti +clau dette +â¬Ĩ ï¸ı +xi amen +word play +wiki artapp +v ath +sur fri +pat anjali +ortho pedics +ingu ish +gle am +eu elections +epi genetic +cold water +ay a +ant agon +aer om +ade a +ab as +Í Ļ +ze eland +sig graph +phon o +of com +mar ni +inve stec +im not +gener alized +fromthe past +ad ob +âĿ¯ âĿ¯ +what doyou +theo dor +stitu te +sac s +k app +di mas +cos ford +carry on +book ends +ai mim +Í ľ +red sox +k mc +jun cture +inhal er +harro gate +afro beats +vin cere +subli mation +so ton +pe per +mid week +mi zer +medve dev +lombar do +fineart photography +col eridge +co i +chu b +cardi omyo +bro phy +balear ic +aster isk +ar und +alabamaf tbl +wrest lec +un loved +time ter +sav vy +ro sas +recy cles +r cc +mi speedway +mat aram +lund qvist +lo vie +fare awards +classi que +boo know +tr n +pu ddings +post o +magdal en +dav entry +carnival cruise +bureau crats +beforeand after +b dp +we tzel +spring field +mosco w +hepat itis +gre cia +game development +dro it +diversi fying +class room +ut an +up dat +s itters +port ada +ou bli +novo tel +nag ar +kw ang +krat om +croy don +ax on +à¸Ļ à¸Ļ +ye vents +wan aka +tion less +t sm +shab bos +refriger ated +ra ku +om f +mari en +lead right +kla as +k ering +jen kinson +inte x +gro te +galaxy note +delu sions +chu mp +toys rus +st pi +spell man +som atic +red und +mci ver +k pi +inqui res +icon ography +dro ck +astro loger +abc tv +u cir +su mer +retri eved +per vez +nik las +kar olin +infl at +hol l +hand guns +good beer +food festival +divin ation +dash cam +bbc doctorwho +b caa +łĪ ìĿ´ +ì¤ ij +tri pel +sizz ler +ro op +q na +m gh +lindsey graham +limo ges +j anna +goo oooo +ghe ads +curi ously +cl á +cantal oupe +brook haven +blin der +barri os +yas sss +ws room +winter fell +v cf +su spiri +st end +ro omy +r aro +marchfor science +har dracing +fc p +fat wa +end zone +dol lop +ru dra +rio ting +poul ter +poche tte +on ds +o ge +lu igi +impro v +g bf +del as +can tik +all you +wasee mb +sno hom +poster ity +pash mina +nb avote +mg k +de shaun +clark gregg +cav ing +ั à¹ī +ਠ° +ug m +mil os +live sport +ho vers +gam blers +four th +form en +fire up +far ing +execu tes +dumb bell +su cht +sny der +sle d +scorpi on +rie gel +fe asts +f dp +di b +conne ctor +cb doil +ar gon +âĢ¦ ) +u ft +ss ou +re trace +ms b +lone star +kin shasa +jam balaya +fan z +cyber ne +seneg alese +ne ther +mid ori +law enforcement +jawahar lal +harper collins +burning man +at itis +adol phe +æ ĭ +wheel barrow +tan ah +si ff +saw mill +rose bowl +own er +nt l +nathan sykes +morten sen +kan sai +kajal aggarwal +he user +es auce +cet ace +༠ĭ +welcome back +tin c +superel mo +repri eve +prokof iev +pis mo +go vic +en j +corri ere +bel ushi +ali za +ur ya +tb g +se va +nd l +jaf ri +ghost face +fino alla +de war +colli des +au sten +ãĢ ľ +ton ey +though toftheday +ta kara +sher ingham +shakti arora +pal ak +mut ant +mk r +lo ony +kno tt +in cest +gul u +cri ssc +central parknyc +c ca +bar bs +x ander +supp s +sky y +samsung mobile +nail polish +mak toum +le da +lar der +four nier +dicho tomy +bigre d +as un +ale sso +t dim +suz hou +song stress +pla sm +mind sets +keer thy +ju k +i in +che sts +anivers ario +after shave +teacher life +star fire +shi vani +peck ers +pancreatic cancer +kana wha +inst inc +htc vive +bulldo zer +bliss fully +angel o +ÙĪ ÙĦ +Ë ļ +ze es +time frame +she etz +ser o +sent ral +pt g +nc is +nak o +lo pez +hive works +hany u +f pa +enab lement +electr ically +cam ilo +caled on +ade yemi +team love +revolution ise +mach ined +fili pp +fate go +d hu +chri sp +bon afi +b sw +tus ks +refresh ingly +muhar ram +high roller +fre eland +dat as +cru ella +twee dy +see b +per k +merit orious +lu do +l wc +in dc +ic hoo +hare wood +bur rowing +bur ris +back waters +al don +sun devil +sne st +ph es +part i +ka ha +eng l +con cu +bed azz +ðŁĴħ ðŁı¼ +tin ts +t mg +shan tae +nighthaw ks +ni es +miraculous ladybug +ming us +ma kings +lhh atl +joy news +i ums +bystand ers +!!! ] +âļ½ âļ½ +wo g +vive k +tr attor +to bler +simpli fies +sc er +new z +lam my +jay ne +ham my +hair less +gra u +gat uck +fri ghtful +e au +delinqu ent +chech nya +ari sen +ali ons +! âĿ¤ï¸ı +ઠ¾ +ਠ¿ +ve get +town sh +te entit +tc as +soo k +sl acking +roman tics +rock steady +orange ville +neural networks +motor show +maya wati +ma hia +lu sit +isi ah +er ken +ch allah +a one +Ð ³ +ö ster +u aw +the matte +si go +ro bust +mo hali +mi staking +maje ed +le sion +jc penney +fung icide +dy k +comic bookday +boba fett +row d +potenti als +post punk +jig s +inf lows +inf ar +en vis +ds r +der py +big ten +vast u +signi fy +puer ta +poo le +lindi s +lim itation +i sps +dor is +co satu +chromo somes +boo the +al arm +ðŁĮ Į +weare uk +vare la +sun glass +sec def +savethe bees +s love +per missions +mi zor +macro photography +girl friend +emmanu elle +des don +cl m +chesa peake +cal is +bo ps +ðŁ¤ ľ +v st +no vices +me son +love our +itten den +it r +ir oned +clu ster +char i +cap sized +ave tt +asy lum +arrang es +ab angan +zi er +yo d +u sos +te el +sne ek +ru der +ori el +mcne ese +kill the +kid ston +jam my +inexplic able +ho th +griffi th +galax ie +death stroke +but i +ðŁĺį ðŁĺİ +wick er +thi essen +san gha +puni shable +pre ma +me u +interpre ts +ida h +harsh vardhan +gen naro +ff p +exhau stive +e ke +cha hal +catacom bs +amaal mallik +Ù Ĥ +ut r +ul in +tab lo +south paw +sor or +road blocks +ren zi +pre term +lead generation +he dy +gun shots +feyn man +e phraim +de grade +d ama +b cc +am und +af in +ðŁĴª # +ðŁİħ ðŁı» +yand r +tari q +stre it +store ys +sky blue +o connor +naz ism +moun i +macar oon +i bex +gen ces +gee tha +free press +dayo ff +cle at +bad awi +an ko +ðŁļ Ĵ +âĢ ı@ +tornad os +stra sburg +post ale +om ers +nim bus +murri eta +il yn +hou ser +equ ate +eclip sed +dis liked +aleksand ar +al oo +aku fo +âĦ ĥ +ty ron +t with +sport sday +ml w +is l +in experienced +hom ily +ex xx +depri ve +deon tay +can ter +bin di +arab iya +adap tion +tu belight +lo ong +le ith +ku wtk +ke ta +ka izen +fro ch +el ish +el if +dun ya +diec ast +communic ates +black outs +armen ians +åĭ Ŀ +á ķ +ri ke +park land +ou tri +new bridge +national donutday +hender sonville +hailee steinfeld +d antes +co ffman +z ano +t lp +summari ze +sud hir +sole dad +rami fications +pee te +otak on +ng ss +loop holes +le der +insomni ac +h pu +gag ne +dhe er +const ables +boom studios +block age +bas sey +as ad +al ittle +ac le +رÙĪ ب +whi pp +tweet master +tg it +re manded +moff itt +ky an +kno wh +kidnapp ers +ki di +ji won +in abudhabi +drive time +cic ada +chitt agong +challenge cup +soul less +pos i +morning star +manny pacquiao +fre eride +cap tor +bro se +bak ker +alli um +yy y +xen ia +we iland +vad achen +se dent +luxury realestate +lac er +kanganaran aut +ire x +hager stown +format ted +fashioni stas +ec f +dema go +clone wars +cha day +book reviews +ay eee +at trition +asu per +am d +actor leeminho +abp newstv +ãĢ ĭ +wr p +su al +stead y +shim on +re gener +photo sof +n tsc +martin sville +k vb +haw key +fish bowl +fil thy +din ge +dar on +cher on +barn acle +archan a +ut ilit +us abasketball +shack le +ol itics +mel endez +lat h +graph ically +geor gin +dv b +dig itali +cô te +cover dale +che quer +assimil ation +under appreciated +todd ler +qu arti +pig lets +p ounded +mom entary +int eli +ent um +dal glish +chop ard +aqu at +al lam +ra scal +ole mis +mat era +late show +heff ernan +ex ols +en emy +com stock +ze gna +wel la +viol ins +t ti +studeb aker +scotti a +pash tun +on wisconsin +nca as +n lin +mis anthro +lul z +deco der +con lon +cau sa +ay s +anaesthe sia +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ı +vine yard +on it +ko ep +ho echlin +heart land +haw thorn +free mason +blood lines +bbc f +a ot +! âłĢ +ye bo +ple ader +ne ssi +matt j +ke fal +door s +cylind rical +crumb led +conve ctive +bruck ner +waseemb adami +vici ously +tu bbs +treat able +slide share +shop lifting +port landia +nor cross +mix ology +leap frog +inte stines +hw dsb +harness racing +ha igh +glo bin +ge au +flori dians +eli quid +dread locks +collap sible +west co +sang at +rooi bos +mobile apps +her vey +feature friday +et us +cor relates +biz kit +ank let +ta ff +stick er +sof la +r tf +proc ter +photom ode +ou ge +me sa +mal acca +lar ks +kot lin +ki hyun +inaugur ating +godre j +car le +cal o +black listed +? ", +âĶ Ī +trouble maker +tol le +spot lighting +s gr +rep sol +rangas thal +pan demonium +fr é +forti fication +custo dians +capu chin +alab aster +w ound +un sere +sal tire +now ruz +n tsb +move on +mahi rakhan +head start +distin ctions +annex ation +ðŁĴĿ ðŁĴĿ +yas ser +westham utd +th ong +sab road +ro ld +po bl +oc p +loun ges +l fp +go ffs +franco phone +devo tions +d ni +alex i +tur ous +taj mahal +se ad +ram ach +neo classical +l anny +hal p +ðŁļ £ +superst itious +show y +relay forlife +pi sto +part ite +no dding +it n +gra smere +for zam +fire emblem +ethno graphic +die hl +der i +dat i +cam bma +br anca +apo li +structure fire +spark ling +n sta +kosci us +joo st +horizon zero +gun nar +der ives +dc ps +cre mat +choc ta +cay uga +ble mish +bi olumin +bar co +al ky +a health +u aa +sub mariner +quot able +ponder osa +nan omat +meen akshi +jojo ba +im als +gu ia +dig is +der ich +corti sol +coach j +bra gs +benef itted +ðŁĩµðŁĩ ¸ +ðŁ¥ ij +wol fie +tn es +quick en +lov u +jam m +it sin +fe es +because of +ðŁİī # +sa ket +oli a +ke ds +happy diwali +ful fills +ent omo +cle xa +anc ity +alon ga +! ;) +william shatner +sunday supper +sun ita +sh anna +rep in +mar l +madd i +kis sy +ke mal +k fan +jak ub +hoff enheim +ha ko +fron tera +danden ong +cooper tire +cau tiously +bon gs +b him +angr ily +aldu bi +z p +snu ggly +sas kia +preci ous +prakash javdekar +infin iti +in roads +cur bs +cat i +bu stin +black women +ben j +ballo oning +bali k +ðŁļ ĵ +thin blueline +ther yan +the justice +shipy ards +li bros +j ase +gre tna +fi ba +che khov +avl news +une sco +tr g +ro dol +ri ppin +pit to +pad lock +no tan +nic he +ink ling +haver hill +cro hn +chicago tribune +back flip +ty p +thursday thought +social good +re ise +pw g +nor r +nepal earthquake +min y +metho d +living wage +jon gup +duke of +cub ism +b ough +âľĬ ðŁı¿ +âĥ£ . +youn gs +yaz oo +whi sper +tre cords +su erte +me tax +long list +kub ica +indv nz +ey y +cla ren +bread crumbs +zig gy +yu suke +win king +tt rell +pa pel +m sps +gb ps +wag on +ut p +thel ondon +tang ent +standard news +south beach +sece ssion +fri c +felici dad +ev alley +en bridge +cour sera +chro m +canni balism +burn ham +bill i +beau champ +accent ure +ðŁIJ Ī +u mesh +to vey +smile day +pass the +lar i +jas mine +hodg kin +gaf fe +forest service +f mc +enth used +dis advantages +cur ren +bl ended +ë © +¿ ? +univers es +tweetab ond +tau po +s las +pan ahon +occu piers +il len +ec y +dro ppings +boo yah +bar as +ba cha +af r +ภ³ +wi u +prece dence +o gier +nca aw +manohar parrikar +mac aque +io a +gh man +france sc +burj khalifa +bog dan +av ala +trans for +sto go +pon to +n sn +ch achi +catholic twitter +yy t +un environment +sof ar +pen ance +mole station +massi f +line ker +kcam exico +buil der +bak ken +apic ture +si res +sav oy +relin qui +mu ñ +mag con +k her +i aa +fo ther +fam a +editori als +chel sey +âĿĹ âĿĹ +z inn +w ud +sel fre +schau mburg +lo carno +dar cy +co star +austr o +âĿ ® +Å ¡ +wa ha +to end +sus annah +stag nation +shi vering +sau mya +ra gin +paul wesley +maldivi an +combin ator +aven ue +) ~ +âĺķï¸ı âĺķï¸ı +whitec ol +well come +tra iled +ste pup +smar kets +savo ie +ren ard +rabb itohs +prefe rential +panas onic +new forest +ka den +fu en +fix e +far ber +ba ig +an jan +zu kic +uni watch +under cut +sul tana +retri evers +ramad an +pi xiv +ob b +jun ctions +hall oran +endor ser +decl an +cu c +cardi ac +be ath +ba al +assi dy +as pires +adver sely +wee kes +un l +training camp +thene w +ter med +rec itation +pu cker +ome gle +ki drau +ho cus +gru ff +electr ical +doctr ination +cra bby +cosme tology +ce spedes +carni vores +br yo +blit zer +be hr +b nc +am bo +actu ator +ther aces +state bound +star ts +dil ated +burn in +bir b +you saf +social impact +re gaining +n ku +holy week +h mc +crack in +clich é +ato ire +wine and +ufcfight pass +sal va +pen den +maryam rajavi +jay e +hide ki +ar nett +ì¹ ´ +âĿ® âĿ® +xxxx xxxx +tomor ow +to yo +sun mi +pur sues +mini figure +hu ez +clu bb +beingh uman +aqu ila +ab ah +" -@ +y tfc +triple t +sun belt +stan dee +shaw na +rece ding +p sh +master minds +llewell yn +im post +geograph ically +youn ger +wer k +perplex ed +ju z +ima d +hep tathlon +hay man +eye sore +du hamel +brock lesnar +ॠ¤ +o scopy +mor in +lour d +hil lel +flo re +ero berts +doge coin +board game +am ond +ðŁĺį ðŁĺ© +scru m +ly ke +ju illi +ji yeon +griss om +fou ling +fo addo +fidel is +ero ded +en so +e wr +down sizing +aie sec +twi x +the final +pla sti +nit o +national selfieday +malay sians +li re +lepi dop +ess o +dab bing +complic ity +bre merton +ÙĪر Ø© +zay ns +wau sau +stim es +slou ch +pis gah +pf ft +n ans +medalli sts +kar ol +en sburg +da aa +arch daily +alpha retta +upheav al +pp hoto +pan ig +fla ir +eras er +emper or +bla dder +bir t +! ", +âĹ¾ ï¸ı +un met +reu ter +ran cid +pert wee +lin ds +kansas speedway +jodha akbar +jersey shore +heral ded +deple tion +cu taway +complic ation +cal trans +ant ar +un see +u tic +tw ar +sp en +ram say +nav an +j bs +hu mm +herman os +forever home +com prise +. ðŁijĮ +ãģ ¤ +win tour +ren tal +real paige +pin ching +pharao hs +or den +onthe go +newyears resolution +mend ler +k da +is ler +inde pendi +im iss +i eee +han ky +da emon +& # +ðŁĻĪ ðŁĻĪ +u gg +primor dial +nam mshow +mamac ita +ku i +k gf +emer ick +d ach +cm shehbaz +ðŁįĬ ðŁįĬ +vis ite +tun atuesday +shindeshil pas +one direction +his ar +hazel wood +haun tingly +h ley +gif fords +fo gg +ed le +bram hall +blac kest +anxi eties +yer ba +tem pran +sound ly +imperi alist +go les +ghaz al +disp el +coffe elover +vy bz +small pox +predictive analytics +ou tofthe +no kid +mr robot +gi udice +euph oric +ele gy +da sher +czecho slovakia +censor ing +burn snight +vic eroy +ukgif thour +ts am +snow mass +sa ar +robb in +rho d +mce voy +mc cauley +joy ann +i bar +for two +f cn +charlat ans +c ga +at onement +af oo +ad ine +ðŁĸ ¼ +ðŁĮ ł +video production +v kenya +v fl +ue ber +neutr als +nay ak +lili ana +la se +herb aceous +he en +chau vin +bit sy +ann ap +w aged +ven kat +tail ors +t shirt +spor ted +royal caribbean +pe tunia +myr na +in visi +af o +ab dsc +world wildlifeday +ti gress +th unders +h na +gon or +class y +aval dez +pois ons +par abolic +nba a +n ado +mcnab b +loc sin +hallucin o +gri gor +enter a +din ero +d icky +vis itt +ul ver +sk al +short wave +rae us +pleas anton +ou za +kre m +ith i +ipad pro +hi sham +fill ings +wh yy +vio gnier +u led +thir l +scu tt +m ck +kab a +k one +i shaq +home front +eclip ses +deau ville +caith ness +ber son +aviation photography +ðŁĸ¤ðŁĸ¤ ðŁĸ¤ +varad kar +shan kly +rev ell +re kindle +li sac +lazar o +g si +david duchovny +climate change +circul ate +can twell +can field +bir ra +Ùħ ÙĨ +john legere +j att +henri que +fro mb +concer ted +cic i +aller gens +za id +w ena +virgin trains +sp ina +oro zco +oak dale +lec tor +ir r +inquis itor +ine ff +how th +hof mann +gum mies +gilli ana +gel dof +final ising +christ church +Ù ¹ +y id +wal ken +tro ck +tr ong +tar d +sasha alexander +me che +latest news +gay atri +f mf +bi ffle +su ha +shore birds +ol do +lackaw anna +l grw +i ic +gw adar +cra ve +con ga +b pl +ar ao +æľ ¨ +vend ra +silve stre +row land +m power +ja vi +ec lair +compart ments +aaron carter +wh s +tol o +sy denham +sti pe +skarsg ard +ric kie +ric ans +pay al +outrage ously +inciner ator +iam joshuagarcia +hin dered +herr mann +gi ga +daw ns +cf da +bla zed +bene detto +ba sha +red heads +ko da +i ano +gh hh +fre ep +cu tty +acknowle dg +abre w +ðŁ¤¤ ðŁ¤¤ +âļłï¸ı âļłï¸ı +p ä +oc cit +l gg +l bd +kuwa iti +gb v +ap roud +******** ******** +ug lier +to know +swin burne +surg ically +myo ttawa +ke tamine +c mn +an amor +aci vil +ìĨĮëħ Ģ +trump ed +testic ular +spen der +singh a +ror schach +pad am +men i +le vinson +kron os +ko ta +hi stam +harb ha +ear ner +do dd +capital one +ap ne +ab es +wc s +ven ation +the dark +sp iller +richar dar +plac enta +me tv +lov ren +itt ing +in du +down syndrome +des mond +bukol asar +agn st +. âĢ¢ +uk ti +spe t +se ton +requ ited +olive t +hobb it +car is +bas ra +ban yana +ak ing +án chez +ya el +y eni +un faithful +tahirul qadri +stetho scope +santa fe +rot ations +nar whal +n berg +muse u +mc caw +ja official +infuri ating +hispanic heritagemonth +gir dle +daw ning +con anobrien +bri sto +black coffee +ðŁĽ į +ten sile +ss un +she edy +ili stic +gi on +c mb +bhattachar ya +b æ +am boy +ton do +tom oz +sel ba +per gola +okeecho bee +hou wx +he ya +ground sman +griev ous +co tab +arma an +app g +stop rush +se ach +russi agate +me gyn +in doctrination +he ee +handi work +gas sing +fred rick +enab lers +diabol ical +chicago history +bukolasar aki +uro pa +tun ney +steel er +on der +ly sm +ger wen +en tails +as says +ಠķ +Ùħ ÛĮ +vampi rediaries +se ig +ran ches +pa vements +over sees +mu ffler +medi ev +human sof +go yal +ear nest +decep tively +back packers +at lan +ww p +turn ing +st elling +sel atan +s you +pur y +masquer ading +luxu ries +har ington +gil d +findyou re +eve sham +esc row +ery kah +endeav ours +comfor ted +chant elle +ari ver +am ey +wit ted +stal ingrad +spill way +sli mbridge +shel ved +real isation +meer ut +green lantern +grat z +flat tery +ef ly +cy pru +count ing +bookmy show +at ini +al bi +âļ¾ï¸ı âļ¾ï¸ı +vi render +vari ety +sun da +quot ations +public speaking +office space +mi ffy +mad house +eun hae +el td +dro ss +buck ler +⼠µ +vi bes +up grade +u ga +rag time +par ro +knu st +ke ven +insul ating +ici on +gar ba +el ph +del t +ax s +vis conti +r tx +pos al +my heroacademia +mm in +groundhog day +cuer vo +childhood memories +cag ney +bear grylls +b awa +whatvegan seat +un wto +un officially +thibau t +the struggle +raj on +pho spho +jesuss aves +gur ud +fur tado +fligh tradar +es mol +el eni +bletch ley +an otte +ad ol +ãĥķãĤ¡ãĤ¤ãĥ³ãĥĢãĥ¼ è¶ĬãģĹãģ® +usp to +ten ured +por ch +oo ze +ne go +nbc newyork +mal ty +logger head +av il +âĸ ij +tw g +theod or +quesad illas +pit ted +mountain top +ml s +megap lane +ma sc +just listed +jo ana +geo science +fos goodwood +del o +conve yed +b ö +as ado +vibr ancy +sey fried +scott s +pam pl +nation wide +ma shaallah +log i +ko st +go z +gb k +foot note +dau m +bam bino +arcel or +adel ine +y j +u ppers +transc endent +thi stle +stran s +ri mmer +moon lighting +lie ch +inexplic ably +in motion +head quarter +guadel oupe +el um +dordog ne +diamond back +c td +ðŁijĬ ðŁı½ +with friends +spot light +splat form +so beys +shin obi +scu ffle +re sorting +peri pher +kak ar +is sy +hel ga +cmom aharashtra +bo logy +app legate +w os +sway ing +ma uk +hun ny +grand rapids +gee ky +fore caster +e ate +ys r +to si +she af +sec ts +repatri ation +real es +piyu sh +pi as +mexico gp +incon gru +dee side +custom ise +chi kan +ÙĤ رÙĪب +wil frid +wi sher +topi ary +south yorkshire +s ge +real tree +nd is +marilyn manson +m of +kas parov +geof frey +for ges +aw anda +west midlands +ti ppy +sit ar +sho al +shi zu +room ing +puzz le +mur illo +jes sup +hur tful +by un +bront ë +b wa +work manship +w nep +um ami +the apprentice +smoo ch +se hun +se ff +reve rent +pleas ance +mm k +her mine +al art +thevoice uk +psychop aths +or gul +mor peth +inger soll +hydro xy +hiphop tamizha +ge dy +cha pe +at rol +ย ย +wine day +wa o +u news +tiger shroff +som nath +ramm stein +pre neurs +peter pan +pan jang +national championship +man gal +for gery +dru ids +ðŁįĤ ðŁįģ +rl wc +pin ner +nan ce +guar dra +bridg water +bor de +black star +atta ining +ag ates +ðŁį ¨ +yau pdates +tru dy +tortoise shell +sch ell +meg aphone +joey logano +jin ki +induc es +go oners +gigab yte +gack t +escal ates +classic movies +wet land +tumb les +istan bul +i ic +fran tically +fore runner +esh war +childre nof +ba aghi +al ente +âĢ¦ : +shab bat +ou ise +ic j +gi i +fav ela +cran ked +co iled +bul king +ap am +af ood +sir f +se to +pop culture +perpetu ally +li dia +la pp +kai ros +deaf ening +cr ate +cool more +conve ys +ðŁijĮ ðŁı¾ +vic hy +sm acks +letsgo pens +hu mong +her m +emma us +e and +cali per +ah hhhhhh +\( ^ +~ !! +yar ch +univer so +ste me +sp ader +pe pit +pe ct +night sky +gla dwell +florida keys +curb side +con joined +c ville +bb z +visit devon +sky net +sas so +sand ford +plu mes +impost or +homes forsale +hair dontcare +disru pts +cin elli +suppre ssing +speak up +seren di +phil bin +pdp nig +non leagu +ne ville +mtp spride +mar ci +len non +ld p +kam an +front line +ever ly +do cker +co vet +clo ser +ìĬ¤íĬ¸ë łĪìĿ´ +æ Ħ +win it +welove history +webcam s +w mp +ur bang +sw ood +stro wman +port ad +ot ts +obse ss +mic ra +ir repar +helen clar +campaign for +beck in +bar ros +al barn +ad vi +var vatos +ske eter +sit c +is z +hygien ist +en on +cro ke +crich ton +car nou +ag round +âĪ ĩ +wh izz +welove you +stour bridge +sentin els +regal o +official pdpnig +myn tra +gonawaz go +bu sker +bio economy +angle sey +.. & ++ / +z vezda +union strong +te man +sav o +polit as +pe tron +orig i +op ro +mit ty +hen nig +g aku +foren sic +far c +and an +tb n +speci ale +sm ite +siri ano +press ley +ox lade +kingscolleg elon +kau shik +holi est +fu sel +fre di +dalmati ans +bc b +bat woman +- ____ +vegas born +un supervised +ste le +m ft +k md +ho shi +go vols +four nette +chaun cey +annivers ary +srilan kan +mil ad +lac ombe +jake miller +gall a +ff r +ec to +cu bby +be cher +ri gg +kra ft +hin dr +ha tha +fundament alist +erat ops +dun yaupdates +cor niche +congre s +bumble bees +block party +absur dly +wh et +ve ta +un bridled +ulls water +tri star +str inging +mickeym ouse +ic ar +hope lessly +gu zzi +fire proof +dul ce +d co +common core +cis se +bol ted +a bet +zebra fish +t ze +t pm +swi ped +sean flanery +san ford +pas ay +network marketing +mitch el +mike shinoda +mediacell ppp +mach en +krispy kreme +key ston +k hon +humm els +grow thegame +camino desantiago +za z +peace maker +pay pig +le yofficial +iu cn +gri l +diony sus +dinner time +de ming +dc fcofficial +bi ba +amity ville +ðŁĺ¤ ðŁĺ¤ +ë° Ķ +watch tower +va v +tel eno +sto the +shi pping +rhe in +p ft +ma aa +m sci +itsal labout +h bc +fri sco +east lake +dark siders +croati a +char n +brah min +bol ing +sie h +sh allo +s ram +ney land +nature lover +mh saa +martinsville swy +m pi +ling ua +li za +inst one +ic up +hu x +emerging markets +bur d +amo tors +! ðŁijĮ +ðŁį ² +âļ½ï¸ı ðŁıĨ +âĺº âĻ¥ +tc s +taek ook +sh ina +q n +ple te +ni mb +ml k +mil by +li ma +kel ce +j ingles +groovy bruce +for aged +ct g +conspir ators +wind breaker +war lords +tol uca +lan o +joyann reid +il uv +hoku sai +getty museum +extermin ation +cro f +ball ina +ðŁķ ij +à¸Ńภ¢ +ofc trendsetter +mu star +illi st +christi esinc +cancer awareness +éŁ ³ +ภĤ +yuk on +w fu +vintage style +tel kom +pho tol +mi stresses +ll cool +ku an +i fly +espino za +duff erin +dono van +chil terns +cal train +bio grapher +ber nese +belie ber +aw ami +ann als +alber tans +zing is +wv wx +rat ch +mu ms +ku lit +kav a +inter view +i hl +emra an +d li +c ll +broad church +bl ks +aw wwwww +annot ations +vent i +ru bles +pl ondon +ol lo +mor avian +inter cess +intensi fying +gra vely +game week +flint stone +dis missing +dash boards +cr outons +belt line +ðŁĴªðŁı» ðŁĴªðŁı» +ðŁ§ Ģ +will ys +twitch affiliate +st nt +re mun +out weigh +noo ks +min dia +i led +home screen +furnitu re +be om +azerbai jani +alley way +? ] +ðŁĵ ķ +ta chy +shuttle worth +quit os +peri winkle +miti gated +lu mix +itv news +i ep +gaz pacho +gay i +en trusted +em me +cc n +caf f +bombar ded +bo chum +tul sa +p gr +mic ror +kur z +ku da +jacob i +insur rection +el ko +east london +ak ure +women artists +spin ning +park dale +n gi +lud wig +amar k +w pri +rashi da +pod caster +o ster +ku chen +it es +gh sa +follow us +defe cted +deci mated +comple anno +c qc +beach wear +aw ash +po ff +mitch am +mc clo +in cit +hol ic +ev ander +dy no +dur an +do co +co efficient +calgary stampede +big south +animal kingdom +æ ± +wy ch +te les +sud hir +stu ar +gw ang +g lynne +eu in +dav ila +bir i +agu as +ðŁİĪ ðŁİĪ +z adeh +to st +techno logists +sy th +suzi day +sk l +rein hard +mis soni +mat ely +hell er +fa jitas +eu ch +er go +cm j +bo hs +vape on +trace ability +tom holland +stac ie +sapp hires +sam phire +re ge +prin temps +nic hol +multiv it +mari el +lo tions +li ddle +it stime +indi as +gi rish +foreign policy +far ina +em bry +disc ord +aj style +tw ye +set te +physical therapy +je ze +fu ele +fat bike +ex hor +cap ra +b chs +re dox +picture books +me gaz +me gar +jo sep +gre ent +cre ating +ce v +b fi +adebay or +âĢ » +uw m +te g +t ni +ro es +res tha +re fra +pc w +oc n +mole cular +mi it +kirkle es +int x +gyp sies +gu da +gr anger +ger on +fle ur +fin kel +fe ely +evi an +end ran +cat lady +biar ritz +ban ky +youtu ber +yanew ade +tic ia +supre mes +ross lyn +pi pit +passi v +nc state +mr chuckd +mayor ship +mati sm +euchar istic +dw yanewade +d sc +cur mu +boy z +am ours +ac ms +yu rion +tricol our +sa al +ren aud +ramesh laus +photoo ftheweek +ol vi +occlu sion +moul ded +kour tney +illumin ations +ali garh +alas dair +ye yes +turn tables +sil ence +sharpen ed +re sp +patric io +pal omar +kre we +internation ale +emb ert +desmo ines +ar ap +amotor sport +abergaven ny +ðŁİ ¡ +townsh end +over doses +mile split +middle bury +less on +i pa +hat ter +george galloway +fla vin +disappo intments +conchit aw +cantab ria +blur ring +be ec +ast m +work able +ty sons +swing ers +in humans +dormit ory +dc ms +daryl dixon +da shi +burn um +ym iller +vill alo +ri bery +play room +m ander +jo a +digiti zing +dar ley +cloi sters +brad paisley +up enn +stay strong +re edy +macken zie +k stp +j inn +it g +instrument alist +buff ed +bl acc +abra hams +zu l +thisi scle +rear don +photo cards +per verted +ou ble +kh ati +k ly +is kcon +gha da +fr itz +fa ites +en etwork +east bay +dev con +d bt +country wide +columb a +clo the +alex ab +ิ à¹ī +wy then +vinay ak +sch ick +ru fu +making amur +go pi +ec kel +door mat +ca vie +aw ines +wan ath +vege tarian +van halen +sa wareness +rocket man +pan di +mom mas +lloren te +kavan augh +k ı +ic ro +de pts +conti gu +claudi us +ven u +su ki +scru pul +sap a +min ce +man to +gau lle +ba esy +ati mes +ap alach +an ana +ug x +save alife +nu ri +kre uk +honey dew +ha de +fre q +farra gut +famili arity +doo t +de cou +ba ils +ba ier +az har +and ri +uni form +th ylene +spokes woman +sp ate +shi on +presi dio +on key +ncaadi i +min woo +it yof +gal ette +fur nish +fe de +cedar point +brin dle +bigh it +accu mulator +........ ........ +âĿ¤ï¸ı ðŁĴķ +ver na +tri vikram +sel y +nik os +moff att +make sthe +j x +i spower +hen ce +elis se +da unt +cric info +back less +aj k +ab ner +ðŁĸ ĸ +us w +sei ya +pry dz +p mest +on us +mic honne +kap uso +jets go +jaf ar +hen don +goo domens +em its +egre gious +conchitaw urst +bird guides +ber to +aur i +ajstyle sorg +vo tto +spr out +seri e +re finance +protectour care +omo hoy +ne os +national poetrymonth +min strel +min eo +lighting design +kin caid +foo dis +far b +div as +davidbowie real +d wc +confu ses +ala unch +ðŁĮŁðŁĮŁ ðŁĮŁðŁĮŁðŁĮŁ +ëĿ ¼ +å Ŀ +w aker +ri xton +philli ppe +o au +kan ka +do cher +them out +sm it +seaco ast +scott morrison +ro magna +mc part +life savers +l bw +hu me +er v +ding dong +con us +cha at +ad al +aco yotes +ðŁĺĬ ðŁİī +áµ Ĺ +tour ny +sp ic +secon dly +samu rai +po te +per ovsk +parkrun uk +paralym pic +idri selba +ha pi +gra v +gr rl +fawad khan +dj snake +chil ife +capt cha +vol voo +t ci +sab ri +pul ver +onlined ating +newarri vals +melis sab +j ala +domest ically +devon life +aper ol +amare tto +ad olph +ðŁİ « +ìĿ ¼ +wake boarding +multi plying +moffe tt +mais y +jo z +in hab +germin ation +di dit +cre pu +ad ella +ðŁıĥ ðŁıĥ +wen sley +twel ve +spur red +spoo ks +spear mint +snow mageddon +selec tors +sc ms +par si +new listing +large mouth +kak ashi +cab elas +bry ony +ba as +ash ken +apriv acy +ðŁĺģ # +ðŁij» ðŁij» +war si +ru te +mapp le +foot paths +elan tra +cc cc +bluejacket snhl +ãĤ ± +treati se +stret chy +re pro +o din +it sk +ino y +hal ei +gy e +fronten ac +captiv ate +ax o +amy jackson +a pre +ðŁĺĢ # +zoo ey +w faa +then et +squ awk +schu le +s warming +plan tains +official livepd +kam u +hom et +hel pin +har de +g sb +chas in +tu ts +philosophi es +osc ale +orchi d +oceano graphy +mod ding +l sh +firstre spon +educ ate +demo ted +comp ad +co gs +âĨ ĵ +wolf pack +wen del +w gs +tooth brushes +th ame +ra hat +pet ter +pad ova +mel ly +mar aming +ma sen +ky lee +invest iture +how es +hol st +blockcha intechnology +z app +star wood +san born +sam osa +muh lenberg +mc ve +loop y +ir ty +hyper x +fri el +cr ony +bo bi +best ofthe +ber o +bee feat +ano che +wil w +ta pering +self defense +le vis +ivo ire +iti ba +edinburgh uni +camber ley +boo ster +as soc +alta f +adaf ruit +ðŁĮ ½ +ye ssi +ver ma +san dia +refrac tory +re organization +ph ul +pe ti +me izu +hat ec +gc ses +gam ous +ai za +tw ick +ti ki +star boy +sign or +sh pk +practic ality +om me +mail chimp +ma bry +ju ssi +ju a +ho ke +extru sion +dirt bag +chin os +ìĬ¤íĬ¸ëłĪìĿ´ íĤ¤ì¦Ī +su fficiency +slat tery +saving lives +ri ss +pro bin +new bery +iz ind +it p +hb r +daz n +bio sciences +ber line +ahu ja +w yo +um f +tx ed +otta waf +or land +napole onic +moly neux +main event +lax man +kut ty +ic hung +husq varna +huawe i +ferrari friday +ak syon +. ): +tru sses +scottmorrison mp +repe aling +r dy +pizz agate +phil at +malach ite +l q +ju ss +den se +car path +ðŁĩ ¾ +wr f +who scored +volgo grad +visit greece +vi el +v cc +tagsfor likes +shi fty +rotor ua +opportun e +naq vi +na eem +mo stre +le mu +hill ingdon +cup boards +bon ed +absen tia +ห à¸Ļ +tri pe +te th +schoice awards +ranveer singh +ilford photo +hl mann +deep sea +ber us +b ta +aku mar +âĺ Ħ +t pl +studio life +stimul i +sec pompeo +repur posing +re strain +organ ises +op l +midcentury modern +forza horizon +co cos +bik elife +beckin sale +ar aki +Ñ Į +wany ama +w chs +vi ens +tribul ations +rip ening +pro grama +photo challenge +l era +embry onic +de yes +t th +spread love +snu ff +sand bar +sac a +s ánchez +ri mini +ph ants +juic er +for sberg +eh ren +chocolate day +c to +b anna +aust ell +ar mand +ðŁIJ ī +ze i +ur als +ob f +barnsley fc +a hou +wind ward +su sie +sketch books +regi o +pha l +par an +ign iting +gri a +cb m +bc w +al thea +âĩ ¢ +safe co +rand stad +pra dio +makeup by +howit zer +foo sball +fe eh +design showcase +cla rence +avi an +ani dhi +welcome home +ush l +recomm s +parv ati +n pf +lindseygraham sc +glen non +gain z +domen ica +dis agreements +dar row +c ff +au chi +type script +the alth +ste rer +ss ens +sine ma +re treating +plin th +ic l +hen do +den euve +colour pop +ch t +burger day +bor rows +ano v +ad age +the white +str ano +realjames woods +onlyin mn +modernis ation +mast ectomy +late ch +khu malo +horizonzero dawn +g ak +fu ja +flo cked +fier cest +ex pat +etsy chaching +coon hound +cal endu +ban ting +ðŁİ £ +treason ous +ten ancy +statist ician +occup ational +nun ca +lau da +gram ercy +flu ttering +ble vins +acidi fication +ðŁĺĤðŁĺĤ ðŁĺŃ +west mid +w inging +uk parliament +sc tr +mach akos +go jetsgo +early years +eagle pride +da fuq +cot te +cospla yed +carol la +cap y +bo leh +aa rena +wi dens +thor ragnarok +pa jh +n ellis +g tf +ìŬ ìŀIJ +ro ddick +lev ant +ky rie +emb er +c gs +amae chi +wh cd +soo ty +sni ffer +sl c +seem ing +madri d +it ating +is lav +girls with +electr icians +do ings +dal en +d gy +cu ss +con fer +busc emi +brat wurst +an sar +abstractexpre ssionism +a india +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ +ul li +tele vision +sol on +pri ming +persi b +pap ryka +ol ac +o bie +n ade +multic olored +im mune +hu x +holy head +hen k +fel led +diam ante +cla udine +char l +but z +( âĢ¦ +yurion ice +vel as +uk r +the beast +sy ah +st ourism +so co +snohom ish +sin o +siddhar tha +rho dium +ren n +red shift +na ise +mutil ated +khale esi +ka abil +j aded +hen chman +ga st +discern ment +court ne +big green +bi plane +be ano +anti po +tu lisa +sion ary +re written +oy u +ll oris +kra sinski +jan ice +franc s +familiesbelong together +cricket aus +co si +bu sey +ade m +tal o +show girls +sen ess +pad hyay +op el +moss berg +man zo +interior decor +f les +day y +cu e +ber ia +ठ§ +tou chy +tennes sean +mac lachlan +fa inting +den omination +ci encia +ca poe +all caps +al ade +è Ī +ठ· +the tford +so bel +radio thon +postp ones +ine urope +harve sts +edin burg +dexter ity +dar rin +bho j +til ley +th ore +roman cing +realpaige wwe +r atti +proble ms +nor se +kre wella +ja ket +ir cus +half penny +chit o +bun ton +bu u +boun ty +y rs +ste u +ro ald +resto cking +qld votes +ou i +nay arivera +labou rer +is ak +i bo +coc ina +aer on +ad show +sur rendering +len nox +hindu stani +gop taxscam +car itas +byr ne +bo gle +always learning +æ¥ ½ +titan up +starmagic ball +sna res +multnom ah +lap u +interce ssion +her barium +greg son +furi ously +fly wheel +eck ler +del tag +d aco +break outs +alaska air +îIJ ĺ +âļ¡ âļ¡ +zam boni +suit cases +stri pe +roto world +no bility +inf ill +dynam ically +ch é +band aid +angel i +a org +sheh zad +s office +pu pa +ph or +pajh wok +pag er +natu red +mis used +iam ejf +hast ing +e chop +dever akonda +daniel ricciardo +cr one +coet zee +ar ms +ar as +ðŁĺİ ðŁijĮ +ë¦ ° +æĸ ¹ +ั à¹Ī +wonder fu +why not +ta re +su mit +slo an +sa shes +perin atal +man ek +guer lain +gar man +egyp tology +do ghouse +appar ition +ac cre +uten a +umm er +skim ming +ri re +nf pa +ne tapp +mi sta +mang aniello +don nell +cit adel +axl rose +ad obo +acrobat ics +aci os +ðŁĶ ¬ +uk ong +tro yer +thi one +t ase +s fashion +par ikh +mono graph +ja ish +hor ford +h qs +foot sc +col ini +ðŁ§ ĺ +ure a +stol tenberg +ma an +index ed +hydro ly +hin kley +adel rio +ab solution +Ø « +til burg +sa ppy +replic ated +ree bok +pat erno +nand am +i pe +home base +coun tr +cbee bies +boot le +uof sc +seduc ed +scan lon +priorit ise +pc n +or ks +occupy hk +nan ay +mil o +mao ist +genoci dal +ex tractor +con lan +bro k +av ir +app an +war ra +stri x +sedi mentary +real ty +ps w +paris jetaime +m can +li um +li aquat +jo shi +inquirer dotnet +ha gen +este fan +coinci dental +bett is +ðŁħ ° +winter ishere +re unions +nu ss +mix tures +mir amichi +la pi +ko pe +kir chner +kel i +emur phy +de tti +d q +confe ssion +ç ģ +âĿ¤ï¸ı ðŁĻı +w de +u stream +swamp y +sub missive +sleep wear +out back +on ond +in corporation +fan friday +desch utes +cush man +cour tenay +blon don +ÙģÙĦسط ÙĬÙĨ +st george +sp ag +sic le +sh ins +resul t +ka ik +j ms +get cha +fo lie +depo k +cri p +bit stamp +azz am +an bar +ãĢ ° +u pm +ph thal +over tones +game show +es r +duf fie +cas sad +si ssi +scru bbed +ron n +ren n +path os +mur mur +kul deep +gril led +gal o +ez ra +ext end +dav an +cl f +beha r +avant garde +am far +ach ill +volvo car +visit florida +sea ford +prote stor +pl dt +gir ling +fc d +bi zzy +ash benzo +aquap onics +wh alley +vet ted +te ment +tal ks +super drug +sun coast +sav elives +sam y +pro duk +lec tronics +je suit +ii hf +gan sett +dat aprivacy +circu mc +c ch +be o +bay ley +ak ot +twi sty +ton ians +skir ting +s golf +phil pott +k mh +in world +go or +du s +devon port +d ago +community engagement +ac ta +w wed +scot parl +sc ut +s bt +richardar mitage +o den +nol l +jan mash +hy an +green houses +goo p +excruci ating +dur i +das ilva +d ich +c pr +algebra ic +war am +th ins +sho ddy +sad der +plum lee +pay wall +od our +mp ton +kic au +kan sen +jubi lee +exce ssively +euro fighter +dri ps +diage o +daf t +cat alan +cam girl +ðĿ ļ +ra kim +ot ay +mob ster +leg lass +kp is +en el +crush wednesday +chakra var +ar ranger +xtre me +sali ent +kar lo +her manus +chlor ophy +bang sie +un affected +plein air +pink ish +p yl +n cle +head lamp +hay worth +fun a +fore urope +floun der +cork gaa +compress ors +an anya +yoo chun +u shistory +strike force +ste ine +social security +re makes +pim ping +nau ght +moh sin +mem bered +le yes +ihs net +d ner +clu mp +âĸ ½ +z sl +si red +krp kab +capital reports +bu ries +angry birds +twin ned +roy er +pu reb +pand ering +ne gi +men tees +joh ny +gru bby +b ine +vie wings +vel asco +tec no +sk is +pun ny +primary care +po key +par lay +llangol len +ju ghead +graphic designer +fashion friday +endocrino logy +b way +aldubx dtby +thought fully +soft ening +my ard +judge me +h unch +g ss +g sm +carmarthen shire +vol k +sp atula +sin dh +shan mu +kn are +ip d +har penden +h up +fab ul +encom passing +ann amari +with held +vin t +to ty +si den +ren se +regi me +pv sindhu +middle school +mel ine +mau er +lu thier +kk ar +justi fying +int an +ill iteracy +her ty +gil len +geek and +electro de +dry ness +can ons +bar ca +arab ella +up for +tre mble +thematte spinosa +social enterprise +scoo p +re to +ran es +pu tts +projectrun way +pp h +lar ue +jar re +homo gene +geo logic +faj ita +don do +den nison +demetri us +beth nal +a hahahah +t fclive +s iti +roe buck +pen ni +pal tz +ol af +kat zen +home office +expan sions +eat aly +can te +boyband ph +ap titude +ambu latory +wa pp +t mom +saddle back +re focus +ngo zi +mb ach +mah mud +loaf er +iam dj +ga etz +d wn +clash royale +be resford +an keny +agi ikay +à¸Ħภ£ +wack en +ther ide +sher win +rule of +ren u +pre pare +naf isa +mix on +ke sari +girls rock +classi fying +bri e +boo ker +blo x +star gate +sor rell +sop e +sam hain +rocke f +nozom i +likel y +inter locking +grace ffa +gq magazine +du bbo +die ppe +coinde sk +broad stairs +ano a +anaco stia +al au +temper a +sque al +selfle ssly +rivier amaya +rhe um +molon labe +lin i +ko st +incu rable +fe ige +ca water +ber is +barre ra +wol l +ta reen +ome thing +humong ous +ha ut +g wa +eu k +e pa +confidenti ality +be az +ðŁĺį ðŁIJ¶ +ðŁIJ ® +x lp +sm dh +ju pp +hierarch ical +f sr +ev a +eng g +disinfe ct +dis arm +bot ton +bab ble +ðŁİ º +word less +virtual photography +twin kling +tennes se +sch wal +sam p +rece ssed +ou b +lec tion +j id +ha up +co ho +carlosp ena +swachhb harat +stip end +salesp erson +rb g +prow se +occup ant +medi ate +fix ers +cu ore +ar fa +ðŁĻı ðŁĻĮ +à ª +tori kelly +si fication +nu e +neuro scientist +live wire +lin dale +lan vin +jeremi h +herkim er +cutt ing +cho k +cab ral +bil ic +big timer +âĻ¥ .âĻ¥ +volvoo cean +sten o +neha kakkar +hoo da +for o +disney springs +dam per +chap lain +cal zone +brad man +alyss avaldez +whir ling +wh ack +wall flower +wake board +un turned +stab ler +sp dc +par am +pad man +odd ities +l cl +imti az +g mr +eugen io +di feren +c ella +aeri als +young life +see in +pran ab +laksh mi +kent uc +john mayer +free download +fod map +top golf +tomoda chilife +to ole +siste ma +n ny +mcgin nis +kru mah +ishi i +hyung won +geri atrics +f fa +cl erics +aqu a +worklife balance +w bir +the saurus +sibl ingday +ru mbles +ridd ler +pro let +ow t +mc q +ma dar +lo isa +gue te +ds max +dir tiest +di stru +coni fer +cen tering +prote ge +nat alya +kur di +ho ka +gan pati +feeh ily +cad ero +ag ric +⬠Ĩ +wor sens +war rick +the k +south end +s yed +plo rer +p ili +n é +im ura +hen sel +fili buster +city walk +brexit shambles +ba ked +amic us +wild land +wag en +thad deus +sh atters +lu po +kar u +ig al +hutch ins +har ie +bra wl +bay liss +baske tb +ðŁĺ¥ ðŁĺ¥ +westin ghouse +un knowns +timo th +pi sts +man gos +mal di +kasauti izind +ite ja +interior designer +hor ry +dro mo +ch all +cavie zel +[âĢ¦ ] +viro logy +spice jet +software development +se uss +school girls +mac an +l bp +koval ev +hitch hiker +fi brill +face app +dra sh +circum ference +annot ation +wow za +super annu +snow mob +si sco +pel opon +lor ries +gar neau +chicag omed +c mx +brasil ia +an ok +al fa +( ^ +thro p +shr ink +man si +l las +kitchen design +is lington +install ments +gab a +dele tion +twee ts +ran jit +post gre +ny j +monte go +lipo suction +kut z +ha que +gir lon +ba illy +arre tt +agre eable +ðŁIJ ½ +wrist let +vadi m +tat ar +sibir sk +l boro +hoo ah +ho wrah +ey o +bi ere +b do +al ev +vo ight +mete ors +lymph atic +live well +li shes +kr ill +i barra +greet ing +en ric +eldo ret +bren den +angel ico +afore mentioned +èģ´ ãģĦ +we ald +un isa +sta dia +o cha +n iner +mait re +i ki +cur bing +chri scor +bo gnor +bird ing +baton rouge +allo a +t ka +shoo p +reser ve +par tie +opportuni stic +ma scher +ka di +ep il +conce des +b hn +wo ks +t lr +t dd +shul tz +ne vin +lou ie +hyde park +hero d +az am +al q +â¬ĩ â¬ĩ +web developer +uc sf +te dge +suppor tyour +rastaf ari +lin ic +is ai +ic ri +hege mony +beow ulf +artic ulation +ap ir +see k +sap lings +muja hi +mid del +ku do +inter sex +high tech +af am +# - +yearin review +qui ver +post grad +pho today +ni vers +my baby +mi ed +meche len +ky ush +inclu siveness +gn ss +dv t +dastar dly +dal key +clo th +chi evo +awe e +tre llo +ti reland +pupp et +metro s +j hun +horizon te +gom be +gastro pub +gari baldi +fon dness +e ley +do tch +dart moor +cott rell +axi os +asi e +Ø§Ø ± +youtube gaming +wash wizards +t andy +sumer ian +ser b +secrets anta +pedal board +mizor am +li days +draken sberg +dic kies +courte ous +car mack +bor on +af k +êµ ¬ +wilde beest +victor inox +shan nen +seun ghoon +py rex +proje kt +pri x +moccas in +kuz ma +floren tino +bachelore tte +wan k +ve sh +ug ar +s network +my new +mari u +manu ela +k nut +jo ão +indv saus +il ham +hal ford +goode ats +dg r +bun ker +blues kies +amas sa +ðŁı į +wh urst +smar tly +sh rap +seaw all +school boy +or ator +not d +ma ac +live music +in yo +howar du +dor sett +audio logy +ðŁĶ« ðŁĶ« +vote kathryn +vol ga +sta shed +serendi pit +se led +p cd +lam y +ko dal +glo b +cm f +chrysanthe mum +chocol at +black box +spo iler +sof itel +smo ck +pla st +pa this +obstruc tive +krist of +histo logy +h ird +flu i +feather stone +ch aya +box x +toys for +robe son +postcard sto +n gin +merri man +kh oury +exist enti +bo les +be b +bb qs +ìľ ł +twitter support +sw asan +sh ura +raven claw +jp nadda +findyoure pic +d out +cutt lefish +ye shiva +r giii +parliament arian +o gg +modi in +marath a +hoo ping +do gra +d ard +char dy +cal on +ap ati +ak ennedy +we f +soc keye +shane filan +sedent ary +san aya +ri gi +re produced +r ought +orthodon tist +ner dv +en raged +do young +calu met +bor rego +boo gie +aa as +zom ba +world tour +under belly +un ob +tom m +mal tings +hr d +ex ple +cu le +cis d +chi bok +chan son +business news +brune ttes +bluff ton +aqui fer +range finder +makingamur derer +ma go +gran ular +don n +cressi da +cr ans +capac itors +c dr +arizon acoyotes +ve es +ta ko +su pa +slope style +seat belts +pron oun +men thol +knigh thood +key ed +ji ffy +issa quah +ide o +ende ar +chop e +bo red +an at +z any +uss sa +se mple +rider ship +mariecla ire +kra v +drys dale +deb i +congre so +c cu +ðŁ¥ Ĺ +ç¥ ĸ +worm hole +teen agec +standardi zation +perse phone +perfect game +ough s +l pool +hahahaha hahahahaha +gw ali +do dged +cru tch +co ped +clo gging +be ver +band mates +ven to +ti fo +rd x +pavili ons +nip sey +is las +il frac +han sol +grisw old +emanu ele +devo e +bull ring +at ala +ãĥ¼ãĥ ³ +win ecountry +stal warts +not ations +macin tyre +job fair +je sper +in ne +holm firth +hoi sting +geh lot +gagar in +fla red +bou logne +aw o +agas si +afran klin +xox ox +wn l +waw ine +wash u +top model +tain ers +su cha +si aa +ough ta +kil ig +guana ju +go do +gar n +fly air +ff ff +dec i +bri dle +bran ning +blu estone +bl r +ìĺ Ī +synchron icity +spl unk +mel chi +mc tavish +loot crate +icic le +hot wheels +go y +die z +dar vish +ble ep +arab ica +Í Ł +z ka +schi av +my cology +kumar aswamy +edwin a +ati v +^_ ^ +ðŁij ĸ +y ac +wey bridge +ti mor +si ke +roman i +ro ther +quint an +ham pi +fla c +co vent +cham in +c sm +bour get +ðŁĺİ . +wind ell +waf a +stan ak +seab rook +san chez +russi angp +re arrange +pen tium +paki stani +nfl playoffs +mo hit +mari am +mar ne +four four +conesto ga +co ff +bus quets +ar jen +ðŁĴľðŁĴľ ðŁĴľðŁĴľ +ðŁ¤ ³ +virender sehwag +valeri e +semi finalists +lower case +khu sh +in vocation +hc sm +dunlop btcc +bla u +barb ary +auctione er +ac cu +x lix +water spout +w pd +vand enberg +swe en +in soles +del os +cutt ack +car ru +byr ds +black widow +ath in +a ç +yol ks +tan go +sto cke +no is +mk t +miya ke +mc dougall +manish malhotra +fon d +fare ham +by l +and is +an gui +ad as +ðŁĮ ¬ +wil ley +swim dive +shoo ter +se wers +sch efter +os b +mm b +jane oineza +jami es +colli sion +chron ically +bo jan +aro ss +ts l +tories out +sens ical +ol ins +official r +life quotes +karnat aka +hir u +cir cas +amo vie +sports book +sangi ovese +ravin dra +prof iting +pro gen +pois son +ji day +bm wm +this week +synchron ised +sou ff +people of +o campo +norwich cityfc +mt k +mor phic +lor o +k home +identi fiable +ic ula +flint stones +bibli ography +à´ ¤ +vent e +unite the +ter ill +pamph lets +nas aj +md g +l ı +ker rang +k bc +fer ran +cu bans +biz awards +un winding +swe g +ru mmy +resur faced +ocon ee +nat as +jo iner +i oc +gra ys +chop on +carol ing +be p +terrori zing +slack hq +sch mal +ra du +ponte fract +pi aget +p yn +o gp +o edi +el ven +digital signage +an ight +a arts +$ ... +world rugby +wb ko +ti verton +th ati +tar tu +sk ink +sharinge conomy +sal lie +recipro cal +propon ent +poe tics +o q +novo sibirsk +nb stv +mini stry +me unier +lyn ched +je tair +in fighting +hello bangsie +book list +as apo +ðŁĴĺ ðŁĴĺ +ìł Ħ +wy den +wareness day +ta wat +t ph +sky hawks +personali zed +me za +int ouch +fü r +franca ise +dejav u +de spo +bur ks +astro dome +v mc +uncontrol lable +th ie +spike lee +park city +matri mony +ho pen +h x +brook ing +bre aux +ap ace +alicein wonderland +aj am +ac pa +; ) +âĤ¬ . +ve sta +tri d +offici ate +natu recomms +mient ras +lan gh +im measurable +gif tof +fash ola +candle lit +bal der +baj rangi +agre en +y aar +tee pee +re structure +rc si +mis cre +lu rie +libert adores +li ssa +geordie shore +gent leness +flo gging +bre win +bahu bali +and ere +ad ana +ãģ £ +vil lar +ucl an +tyr one +tro dden +sub zero +scol lection +nu bia +na in +men de +jubil ant +gre gh +freedom day +fin ery +deuter onomy +byu football +brain erd +bour sin +ben ven +belgra via +ar una +app state +trattor ia +polye thylene +nikon photography +marc ella +fn f +film twitter +far si +back hand +è ı +vel t +twin k +sau sage +sar ban +reproduc ción +mo dis +jo ta +cad do +ad don +vis akha +thu mper +sy x +p dr +oil sands +mar oo +m soccer +hen a +glen fidd +ethical fashion +emo ticon +cam embert +be il +ar ro +ab x +vian ney +sweat er +su bar +sh w +raredisease day +meang irls +le win +im planted +hun han +ha kk +fit life +ei gen +ear a +bur de +bloss omed +Ù İ +u con +ty o +sta p +pru dent +p fs +jar man +fire stone +blund ell +ðŁijıðŁı¾ ðŁijıðŁı¾ +nam ed +kei ko +ju b +ide x +hemo philia +everton fc +ede ma +d be +cor gis +ðŁİ » +to que +rambl ings +por zingis +or chy +mi rad +land mines +kom ets +hi ddle +go team +fyo dor +escarp ment +du k +dn f +bro deur +as ki +an ks +a ee +un framed +rich land +ra di +ma qu +leinster rugby +kali mantan +hit ching +economic times +dump ty +craw ls +asado waisi +as oci +as and +to bey +poetry community +official bhafc +mon alisa +jag er +ha x +h ff +flat ware +duc king +di vi +bio chemical +ðŁĴ ij +í Ŀ +su o +sl k +predic ament +nerdv ana +m live +le von +gaither sburg +com ox +by water +ðŁıĨ @ +vaul ting +to ta +thel onious +pre cari +ios dev +hon king +her nan +h ice +enchil ada +en reproducción +da ed +bi ki +bau ble +band it +we c +venge ful +tobler one +tay ler +schar ity +revit alizing +r vs +r rs +love craf +k age +ei bar +dysle xic +cro lla +chit ral +ðŁijijðŁijij ðŁijij +x vii +wil la +tang lewood +ta iga +su football +squ ier +sas sen +per rier +n ld +ko lo +conservation ist +c fe +block busters +an ah +ü ber +sun ba +sty our +smil in +pillow talk +le pas +kru pp +hosp ices +hel ipad +fil i +dro sophila +bo som +yennaiarind haal +uk in +standup comedy +sni ping +sand castle +qu avo +nom bre +n la +man tar +gu bler +gr ano +elo y +d bh +cy r +car pal +bor i +air france +aali zardari +ðŁĩ° ðŁĩª +yak o +un women +sundance fest +small mouth +seash ells +o waisi +mul doon +cuis inart +bo gie +bas soon +an jan +rock o +po ste +pim entel +pe avey +nos fer +kir che +inter pol +haji me +en l +ar ak +ðŁĺ¹ðŁĺ¹ ðŁĺ¹ +еР¹ +Ï Ĩ +woo fers +vo tive +ver dant +u leta +trum pe +ship wrecks +shim my +sc ats +salut ations +s anna +pat ani +nag s +indi gn +gaffi gan +eag an +cr v +bad r +ant and +annu ity +the afric +terrori st +sol ana +rape seed +poo ping +m chs +fast food +emul ation +elev ates +de sean +wel yn +w yo +th birthday +speed boat +pinstri pe +oneof akind +maritz burg +k hai +j nj +gil ani +chri sw +ay our +ap il +a ini +ðŁİ Ĺ +v ln +ther sc +sw en +restor ations +reiter ate +photo call +ob p +ny p +m hp +fil mb +d aps +ðŁIJ Į +z ec +uniof nottingham +tra shing +stra ub +sequ al +ry back +ro thes +mummi fied +millenni um +marsh field +j cs +is art +hugh es +gau cho +defen sible +ce mented +bor land +bon nets +ðŁİĤðŁİĤ ðŁİĤ +wonder wall +wim ps +vivo ipl +tallu lah +taec yeon +sport sawards +sher brooke +q sa +pin ck +ph r +oun ty +nu ala +kung fu +hel sing +dalry mple +ate acher +animal crossing +afc wimbledon +] - +seven teenth +saip an +ku o +ka an +in ta +huss ain +epi thelial +den iso +as kan +wam bach +su ko +son oran +sn ola +pr ong +plu g +nb cs +mt u +logar ith +local es +kelle her +kat ch +flu ff +cr yer +cont ours +con jec +ce real +calendu la +a icc +åij ¨ +tent atively +tempran illo +succu mb +south ward +raj jat +r fl +par ham +ny our +my p +mur ry +ligh thear +in time +gag gle +f lim +city hall +ceme x +brexite ers +bi glo +at ly +ห ล +women insport +un invited +town es +the botanics +sensu ality +sc el +pre occupied +onlin ec +men ai +long term +le ich +land y +ig ong +conservation ists +black light +az aren +architec tural +ðŁijĪ ðŁı» +u et +tu red +stal ybridge +r za +perfectgame usa +pap adop +motion less +mil t +di el +cre at +black birds +bal eno +att itude +about lastnight +ãģ ¯ +respir ation +re defines +pic c +pale stine +now tv +m cuban +lo ka +gman etwork +chi z +angu age +alli ed +alicia keys +w ning +us se +the people +sd t +reson ant +nyc mayor +n bt +hoo pers +don ned +do sed +d illi +centre piece +blog spot +tu so +t mo +md na +land rieu +kann ur +ka rena +in slee +giu lio +alle lu +ak un +thejustice dept +simm ering +ro ly +o ki +nh at +metal work +hou ten +contag ion +aka worldwide +å İ +ãĥķãĤ¡ãĤ¤ãĥ³ãĥĢãĥ¼è¶ĬãģĹãģ® ç§ģãģ®ä¸ĸçķĮ +under tones +sun daes +pi os +on de +o intment +mo bo +kev lar +ket te +ing lori +ic ano +i ag +hay festival +doctor ow +chir ps +bill board +! ðŁijı +âĢ¼ ï¸İ +year n +ven er +ul te +treat yoself +ton ys +som os +s man +oreilly factor +laparo scopic +hah haha +free se +domin ator +chau cer +ch lamy +birdsof prey +armed forces +aero dynamics +ad ors +vol com +vancouver island +the killers +ob fusc +mú sica +lil bthe +lilbthe basedgod +gor akh +fool proof +etsy gifts +cho d +bu e +ac p +ðŁĺ© âĿ¤ï¸ı +war r +vapor max +sr tc +sen ayan +ri man +onond aga +on ference +metro plex +mcgill u +kath ie +kak o +je tting +get out +fuja irah +fertil iser +ex propri +eli ght +don tt +car jacking +bi ri +bal de +y ella +wil ton +wheat grass +vani shes +thel on +sedi ments +pu yol +postcardsto voters +mu to +miss america +ley la +len ovo +justi fies +in co +ear plugs +bur o +blue prints +b school +ver and +ou k +ny giants +jo vo +deter rence +dc cc +con diment +an l +wor cs +v di +tt d +moor land +lun acy +inti mately +idio syn +bod min +belli ge +. ðŁĺİ +work sheets +wil led +ulster rugby +th july +teen age +super janella +sty lings +sh ingly +p spk +ost asis +om sk +n acc +mi ren +ko bi +im ola +fe f +bil le +bb mp +ae ther +! ðŁĴ¥ +tear gas +tapp an +sig ourney +sam ira +pap hos +kat weets +hocken heim +gen ghis +gate keeper +acap ella +âľĮï¸ı âľĮï¸ı +unrival ed +sla ven +russell crowe +py rr +poo ja +ni z +mike tyson +lero i +lan sman +fran sch +end violence +don y +dian ade +bour que +b tv +anci ents +ab dallah +ðŁĵį @ +ðŁĴµ ðŁĴµ +z os +wozni ak +wer ri +sin jar +in visibility +cor si +cen to +ar ine +adebay o +⼠Ī +pur p +n bab +mari ee +ma sta +ly les +l chs +i ak +de gan +creu set +co ppin +blu eri +bag us +ai on +wh ut +urban fantasy +stephen amell +snod grass +sand hurst +pool party +plat form +plan king +p ona +no sleep +mr sa +luci ana +live show +jais almer +it smore +finish line +film maker +fc f +e bol +destru ct +an sele +suppre ssor +spit zer +real berta +pl iny +nr t +nfl pa +lal aland +eric hards +bil tong +and ai +ak ro +war hawks +redund ancy +q ian +pu shups +grou pies +gel lar +com inghome +clan destine +chait anya +an ys +ab aya +tx dot +su ble +r du +migh tiest +mccre e +jurisdic tions +hu dd +hans ard +flo rent +d ce +colla bro +ch oma +bar sand +adi se +v ago +tic keted +th p +st lucia +snow pack +sher borne +ration ing +promote horror +mobil ise +luxury hotel +kand la +k awak +ho se +he dley +dr yers +cre scent +ab or +w sa +su le +sister ly +re bar +ramaz an +long mire +inhal ation +dissol ves +commerci alization +cha ine +carri on +cam erica +boo galoo +big deal +b let +aspir ant +ur gh +tiru pati +sl acrosse +sb ach +poor people +oo gie +ki ambu +jab lon +how ls +b hardwaj +b ds +ant u +a aw +ðŁĶ Ķ +é ľ +van v +plussi ze +link later +lin lithgow +kla ss +finoalla fine +envir ons +bren nan +appeti zing +$$ $$ +$ ! +wa pol +tu fc +ther o +sivak arthi +si va +plastic free +my hero +la gh +fau sto +ev c +cross overs +bn sf +bern thal +au li +ìĿ ĺ +tin sley +ti sch +straigh tener +scotty mccreery +rece p +pun ky +no to +in pics +happy day +criteri um +bikelife dotch +worldcup final +to let +shin kansen +popu late +orche stration +naku foaddo +lindis farne +lat akia +integr ations +ig c +ib g +hopp us +food lover +eng ard +du ds +df b +depau w +bel af +bc n +bar c +ba sie +as sad +af resh +ðŁĺĺðŁĺĺ ðŁĺĺðŁĺĺðŁĺĺ +ya ay +sar sour +ric ardo +prophe cies +power boat +om r +newh omes +magic thegathering +m dr +lokom otiv +li ii +jä ger +ju egos +iter ations +inclin ation +ig ne +gro gan +fu sco +cran ks +be sos +âģ© ' +tric eratops +spic y +spac ers +scri bbled +reach able +over ground +microsoft teams +m hm +g mt +future is +fell ini +fel ines +fab s +cri ssy +ca il +book worms +bo do +ar rington +ðŁĺı ðŁĺı +y ameen +sa kamoto +re ared +nu ys +market cap +mail lot +inhi bit +filmo graphy +falcon ry +engag em +de faced +car at +buc keye +bay front +bangalore ashram +atp worldtour +am un +ad om +y ate +mediac ityuk +j fl +gun ung +fre s +che on +bagh dadi +bab at +aug ment +ari sta +alk official +ê· ¸ë +wie sel +trin idad +sof summer +orp ington +nose bleed +jay me +foot locker +em pathi +bo bi +anti bes +ansele lg +aerob atic +ðŁİ ĩ +ãĥ¼ãĥ « +âĺĨ âĺĨ +water works +water logged +no bar +n cd +ka huna +is ar +fli rts +d mb +cp us +coo kers +cas co +c fi +band ain +ayo dhya +aj man +surf in +o carina +gu tter +fries land +cast rol +bon plan +be so +à¹Ħภ¡ +ven ter +spr oul +sport back +sp j +parti zan +oc ket +mathur a +m fl +ha poel +gre i +g mf +dru p +cover art +contra dict +ay ub +anselelg ort +abse il +war bird +tro ma +ph ro +nerv ously +kw ch +kun j +j illy +id b +hundred th +hal alan +dece it +ca wl +bon t +tash kent +ph lebo +march forlife +mar red +l cr +krish namur +he bei +fra g +bill ballentine +bha gya +august ana +anastasia beverlyhills +amc cartney +íĻ Ķ +th all +ta thletics +stu es +st anc +re als +ol ino +n tn +jet lag +hi ii +aller gy +wn bl +suit ors +sin bad +scotland team +sal combe +roll back +rey no +point less +pl ou +octo pu +n xp +hy po +happy bday +bou dreau +alla ah +ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ +world star +the wire +t gc +sun trust +sore ness +sk ap +sci o +re started +low lands +lewis ville +gastron omic +expir ing +defl ation +de tox +cu s +blu est +ti rade +schol l +prof en +pol itic +po sen +par cs +liber tad +la ye +jan ic +her ing +hae morrha +en cino +cal ori +andre j +anatoli a +ðŁĮŁ # +unfor gi +tab i +shad i +opar ks +my world +ho dor +azaren ka +ðŁĵ ľ +vic firth +tre vi +tran shuman +squir tle +speci alization +read e +ra kes +one treehill +mu khtar +mar ymount +kaz u +k wai +ic er +gla ssy +forfe it +felic ity +f els +dual shock +de fra +cer i +as phy +ang ri +ঠ¬ +worldof outlaws +ta or +slic er +na si +mis se +lock the +jen kin +friday sfor +fasten er +dream force +bloom sday +bl ck +biop harma +apple jack +: /// +. ðŁĴĻ +val ero +uri be +un doing +tw w +to tnes +tic les +tech fest +sun nier +stream lining +s gi +offen sively +lund gren +lc sm +inhal ing +enci as +cr on +comment aries +code ine +cly ro +br n +bo wel +tutankham un +spru ce +sky walk +pang olin +mod ine +men ta +lan cet +horizon tally +gu rira +grant gust +gas par +freder icks +curv ature +cam bio +ask is +éŃĶ éģĵ +u cs +tcm ff +sen or +pu dge +mal dini +kirk d +ken berg +kam pung +iri a +cotab ato +bay onne +anesthe si +and ron +ðŁij © +zen o +stock man +sk an +r pc +on canvas +mtvbrkpop exo +ci re +band ages +aurang zeb +a star +ðŁĻĮðŁĻĮ ðŁĻĮðŁĻĮ +ðŁĻĮðŁı¾ ðŁĻĮðŁı¾ +à· Ĭ +yeg wx +w mt +volvoocean race +vari etal +simpli fication +rock climbing +mis matched +h nd +geome trical +dh sgov +danger uss +center ville +ble acher +ar quette +afc bournemouth +âļ Ĵ +ty bee +tre d +thur good +oast al +nax os +mainst ay +kar rue +joaqu in +iff i +ic re +hover board +evic tions +dun more +d ado +both ell +twee zers +se ep +ry lee +power ade +mur phys +lang chat +execution er +camp out +bice ster +æ · +à¸ŃภĻ +zo ey +le ach +guide dogs +fi bro +ef x +bi el +be fitting +ðŁį ı +workhard playhard +sojour n +sav am +rss org +ni i +nay eon +li han +h bu +gab bar +eng lund +e space +citizent vkenya +aw ah +ab bit +-- # +thor ny +squ aring +ra wal +lou der +li sar +lex ington +intro spective +guin ness +daily telegraph +co pland +cen o +vi zio +transplan ted +shi z +ros ari +ml khattar +indie dev +huck nall +fe ct +embr yo +dy l +di go +dg n +am dg +al locations +ðŁ¦ ĩ +ãĥ Ĵ +sym one +season ably +no stra +mee ke +loud speaker +jam bo +bo got +beck ley +scep tical +sa hm +parame tric +oc tor +nn pc +logan henderson +limb urg +lak as +gor ton +drone photography +deb ate +char me +cele stron +can tu +avi on +ab in +éŃĶéģĵ ç¥ĸ +éŃĶéģĵç¥ĸ å¸ +x terra +we x +wal worth +tra di +team coopertire +sin aloa +r anda +pu tih +physio therapist +over falls +or omo +ny islanders +mmi w +la ki +go av +gee ked +fan e +enu ff +dr at +al evel +yu vi +vol pe +stren uous +shaq iri +shaf i +oun a +mu fasa +ju i +happy ness +em pe +eg l +degra sse +comple tions +chili peppers +bi dders +ðŁĸ¤ ðŁĸ¤ +ys gol +y ano +ur at +un attractive +scienti fic +sam pai +re vels +ou ma +nic ation +ly don +in voices +fer v +expe dite +dangeruss wilson +cumb rian +cram ming +caption this +bas spro +an au +ak it +air crafts +vis ita +shi ps +out grown +north cote +kair i +k nut +ini sts +dio go +dc united +cur ler +cellu loid +baller inas +arapa hoe +yn h +up ma +true blue +t pg +ste via +sever ino +rajjat tokas +r ational +pol kad +os in +nasaj pl +mann schaft +manipul ative +kayo de +k ray +hur ry +euph or +educ ates +cu cam +cor don +black hat +b side +ane o +ai admk +adri anna +tie breaker +south side +param singh +koko bop +kaffe e +in croy +icon f +health news +distin c +bigtimer ush +wi aa +tore tto +to wel +thelittle things +tele fon +smor gas +smo ked +positive thinking +on theroad +music production +mar ab +kasautiizind agiikay +hahahaha hahah +fr antz +empty the +e ffin +cu eto +cross ing +ace of +ðŁijħ ðŁĴ¦ +âķ ± +sorcere ss +o ad +ni aller +nh ms +mu gged +metro pole +kau st +hit man +croche ting +crimin ality +⼠½ï¸ı +sme ars +mir ko +leep ace +kirkd borne +kaz oo +kan ina +ham sa +ennis killen +e scri +di shed +cr na +bo iz +best nine +âĿ¤ï¸ı ðŁĺĤ +wise st +vou cher +vac uu +tac ks +south land +ridic uled +rasp utin +port ability +pat ine +mus ket +la ya +gh ese +free form +flu ently +ferre tti +chand on +ch oux +bli p +betra ying +bet tie +arte ta +all night +un ica +toom any +te mer +switch foot +sme ared +s vs +quir ks +prin sloo +northern soul +excit ing +dwar f +do tte +de ven +corro sive +ci b +certi fy +bam burgh +ak op +si bly +sherlock holmes +ro to +re turno +re launched +outlaw ed +nat archives +mon so +lo tt +lo ire +detroit pistons +del ly +coward ice +at ur +vi ki +tion ate +te v +speci fics +se ale +parth samthaan +mist ral +mea india +lent z +lasal le +is g +gu llo +cull en +che tte +billab ong +at at +aspir ational +air jordan +web gl +wa stage +underwhel ming +ufcfight night +tre yarch +su cess +ru islip +op tus +no regrets +mar r +loo e +it sy +handic raft +graph ql +fen cer +ener o +dun hill +dre search +be thel +ari ed +tim peake +scandal abc +re visi +pu ffin +ph um +memor ized +ka han +hale storm +ep al +dublin airport +w ca +vic er +thaw ks +so to +shu ck +pureb red +pere ra +mug ello +l fcc +ki vu +fre hley +e brd +clack amas +che vel +ðŁijį . +woo young +sti hl +show times +jane way +e be +cir c +blast fromthepast +big ben +bel grave +v force +skim med +san ofi +r cn +out number +mont fort +major ly +f bi +cob blers +cave at +asse ur +adel sol +wel o +ur ns +tor ii +mor rho +man nered +lymedi sease +dg t +âľĿ ï¸ı +woo ps +ur ch +tane ja +tal al +snor kelling +sas se +ric one +nu thin +n art +me ck +lan tern +ker ridge +i ram +her re +followyour nola +flexi ble +ep at +e day +dun de +de id +dal v +cul lum +coinci des +bern hardt +be friend +and han +tik tok +ranch life +por phy +ol ito +new southwales +nak a +move mber +len z +kim bo +kids books +ken nington +k ase +fan shawe +ce ballos +capac itive +bro ek +bom a +battle for +ba ap +ðŁĹ£ ðŁĹ£ +ãĤ¸ãĥ ¥ +war field +vas ily +t sc +se mo +prince of +le tic +cack ling +bett ina +b ms +è Ĺ +äº º +{ @ +scor ch +sau v +s disease +rev amping +piyush goyal +pe chak +nac ion +mn statefair +mag al +la fleur +isol ating +i au +gal gadot +cd ne +bill maher +beg ley +ba io +west australia +vul gar +tv b +tar lac +stenc ils +sch lei +normali ze +me anie +jo elo +hereto stay +dy ne +cre at +chartre use +aj mer +storm team +river o +re spe +paner ai +pale mbang +lu ty +lo athe +judg mental +huff le +hr c +hh c +eu il +c illian +br p +all natural +aar p +yo gis +xan adu +uw f +topp led +se ers +ophi les +mo sque +m ame +ju er +jan asen +guanaju ato +employ ment +bryan cranston +berne se +bbcintro ducing +ad eni +Ø ¸ +yan k +wr angling +wind farm +webr tc +tr one +timber line +the cube +team chevy +tac tical +pur p +o steen +ing rown +gilgit baltistan +gau thier +fee ble +der went +bra chy +ami ento +ðŁIJ ³ +vy ch +the boy +sky blues +po acher +nico lette +na az +dit ka +Î ¹ +youn gin +was ps +tu ks +stat ic +makeaw ish +houseof commons +her sh +ecr chat +de ji +ac ru +xi u +vs sea +u vic +tt weets +sthel en +pr ana +oc ado +ob jet +negli gent +ko tor +kar yak +flax seed +daf fy +conve x +aristo crat +whist ler +vas cular +theone show +standre w +south field +screen writers +kan hai +athe i +to you +sam y +sag rada +ring ers +opp enheimer +mono gatari +m wave +j angan +gil gamesh +dai ley +d ancy +boo by +bbc looknorth +sw asth +sound design +sci am +sadh na +pret enders +out en +mis sm +ma guk +ike da +gil lette +el fman +defl ated +col u +buddhi sts +av u +with pride +wc bs +t mb +t ld +sydney swans +swan ted +st acker +pratt pratt +nan oscale +lil acs +ju ul +high street +fren d +fer ru +de ve +con klin +un relenting +trans actional +stro mb +s ach +religious freedom +n tm +n ela +lu i +h iller +flo tation +en cy +disrup tor +ci er +cas per +bul la +bet ti +w tn +ursu line +up silon +thur mond +splat fest +sal o +p gc +mm h +makesthe dreamwork +lean in +ka ji +gro ping +g cc +d ening +col ter +ar al +anni gan +a week +ðŁĻĮ ðŁĺį +x abi +u wc +true social +timb ersfc +rich mon +prattpratt pratt +ny am +lo thian +leot ard +j ma +itu te +ek ay +echin acea +dur acell +ìĹ ° +tro feo +she tra +shang hai +sab i +qu inter +nhl canes +me rer +ly nyrd +lin del +lawof attraction +lam ela +kho sla +has set +finger nails +end angering +dro plet +dies er +cont ac +center pieces +a sharma +ðŁijĩðŁijĩ ðŁijĩðŁijĩ +vero truesocial +segun da +plum met +pan ch +mal le +li sav +hi bit +h ru +g ct +bon amassa +blu th +backto work +aphi ds +ti bility +sc ount +ra pt +place holder +lane way +fo stered +fo red +fi el +emplo i +eme ka +ca k +ante ater +!!!! ! +the ist +tech o +tec mo +sw best +su da +sky hawk +se itz +s academy +pra j +plac ards +pivo ts +mountain biking +jum mah +jj f +ig ata +eu co +de constructing +d ft +al mond +weis z +vi j +to li +south wark +slo tted +ra gin +pro actively +obste trics +north woods +nh of +jeune sse +ae um +tv p +thero ck +sym metric +so afx +seag lass +non league +night crawler +m de +ky uu +kl ance +kab balah +cri sis +chemical physics +anarch ism +å¤ ľ +tr m +smo res +sa xton +re construct +pettic oat +out scored +mini mum +luci atas +luciatas san +loy alists +ligh thouse +lake ville +industri e +ic aew +i ie +ho gging +fro mm +ephe sus +dur rell +blood shot +beech wood +american cancer +ach allenge +v cg +tom ellis +tempor ada +sel la +morri gan +lom ography +li der +googlec loud +ger ie +fe ild +ev os +cine world +bha bhi +amy schumer +afsc me +vic toire +vi a +sub i +na sir +multi ples +lu stig +lat timore +k cb +i din +guy ss +di stressing +ðŁijį ðŁı½ +wil f +tom bola +tigh tened +sl peeps +sig ye +sham rocks +sat z +qu ec +no gales +new ss +natur ale +k ss +k cap +et fo +epic ure +bbc four +barrier reef +ab on +ãĥ Ģ +tw os +ro id +re eve +natu rema +mal ac +m sh +i jo +extermin ate +chou han +cas i +yn or +tele visions +storm doris +spor adic +soli hull +soci alizing +sh amp +pnpp atrol +out fest +it orial +idh lig +how land +ch ur +belgi que +and ran +w mf +tan nehill +ta ye +s thu +ro que +rik ki +ra dium +pat er +pac sun +p cusa +obli ge +ob vi +n sf +mi es +mc busted +lingu ist +li ppy +di ms +ce g +canni bals +candid ly +barre tto +scholast ic +q fs +propri etor +paci fier +offici alu +nott m +mexic ano +mar yann +la hm +grand parent +forz amotorsport +formula oneworld +burn leyofficial +bax ter +apal mer +ab loh +ðŁĸ Ĭ +wh ittle +throwback thursdays +sla yers +ma key +lauramar ano +athan asi +ap el +vo is +vi ves +tur nips +snor e +pl ines +or do +mac rame +ir b +hl n +global dev +fuss ball +evol ve +epit aph +dang les +dalrymple will +carn elian +as cd +ana esthetic +Ê ĸ +un du +shabbat shalom +ridd ick +ol ney +li da +lal un +im possibly +her at +groom smen +gay le +co ffs +capoe ira +can ta +bak eries +vik ki +tu ra +t ne +pl zz +per ky +peace and +ord way +n anc +la vin +doo d +digi byte +com promises +co bbs +at am +vik tor +ser aph +re arranging +pil sen +marque tte +mari ob +fic us +do pey +d ng +cur ries +ce ec +caf cl +wee ee +urugu ayan +ru ffi +pre ppers +h ü +gob ind +ga stown +baham ut +attrac tiveness +ad ra +zar ia +wis p +sang ster +ribb le +mo ises +martin luther +leagu er +le one +kat v +inlove with +encamp ment +d ct +ba di +âĥ£ : +senior night +rosel and +rand al +pampl ona +link ages +inspec tor +ha bibi +equ is +dam ing +cat chin +been ie +ba haha +al cu +ac ar +èªķ çĶŁ +war ri +tom morrow +ti oga +te sla +sh rooms +orb ital +mulvan ey +mu gging +ku i +distingui shes +abnormal ities +a verse +wb w +vit us +trac ie +the end +t week +speed master +sag ging +re tainer +panch oli +n po +ing ame +in sk +har apan +dif fraction +custom izing +buckle up +are search +tweet whatyoueat +shi pla +pon ting +or us +north america +lucer o +lam i +kit z +green y +de composition +dabang g +belo ve +asper ger +ap ai +antidepress ants +ac tory +) ". +yor ku +yo h +te res +si ft +red bird +movie awards +li mon +dispat cher +compet ition +à´ ¨ +tin dall +skele tor +qv cuk +pnppatrol plan +licen sure +letter kenny +leaf leting +grate fully +gorge ousness +er ste +b fd +ave tt +aloy sius +ow d +ol ine +nom akeup +n tas +man ch +jer oen +had don +gri ggs +golden retriever +fact check +digit ised +dd h +bella donna +ðŁĺģ ðŁĺį +w sd +the z +prith vi +ou en +or ford +mush taq +ma b +ger a +frank ston +fab led +f rick +deleg ations +æ © +xti andela +per fil +ong we +mp v +jammu and +il op +geekand sundry +fi dge +feder ated +da we +cli f +black veil +tu scar +span ky +or ob +moline ux +mar ano +ma pa +hol tz +fret board +ec ac +dup atta +biscu it +bij oux +am illo ++ : +volunteer week +vac ate +v fd +self portrait +north dakota +mull ingar +make overs +he ke +h ct +ever a +deliber ations +chickas aw +bo bbing +big daddy +aro ck +ৠģ +tb ar +sanc tity +ny cha +mgm grand +jones y +jan go +fri st +di fun +chouhan shivraj +ad agio +âĺĢï¸ı # +y bor +upl b +ti fa +s fans +ri ven +pol yam +ol am +gn am +fre dd +dog toworkday +cr an +cin que +bee keepers +be ÅŁ +at au +ar la +an ah +y ura +te rence +te ck +su ge +re insurance +play store +l ile +ker ns +hy the +h tx +gn ani +centenni al +bu ter +ash ville +agre at +ach u +a see +ಠ¸ +sus ang +super dry +sp rime +sc ity +re aping +out sourced +obstru ct +green room +fe heroes +fa in +cla pped +cab in +be inn +av ai +ðŁijī # +vector stock +teamwork makesthedreamwork +sha urya +le ch +kristen stewart +in between +gin ny +fy c +fre er +fr inged +em itted +di ba +cross bones +cow ichan +conve ying +bolshe vik +band i +alexab liss +ador o +ðŁĺį ðŁİī +wan amaker +ve ena +sr v +nit rous +mor aes +loving life +kay ak +iq rtg +hil d +competiti vely +cleveland clinic +cit ron +ar aya +ëĤ ĺ +uc sc +recor deli +puli sic +pis cat +mow ry +ma est +ke p +is ko +fal lujah +dun a +cor byn +zeit geist +wedding planner +spor ttv +schem atic +ram ya +ra ji +napo leon +muen ster +m se +le bron +judas priest +imper ium +did nt +brecon beacons +bobb in +bex ley +bel k +à® ª +ÙĪ ÙĬ +z yl +y con +west africa +spac es +ory x +oran je +of w +odys sey +n ür +japanese food +il lest +grind core +gla dy +fre ude +exerc ised +diar mid +da th +curren sy +awe struck +andrew lincoln +ðŁĴĽ ðŁĸ¤ +ãĥķãĤ © +yof theday +vinyl collection +vancity reynolds +un compromising +su de +spi ele +sing karaoke +rout ers +rei sen +red bulls +priv at +ma day +live strong +k mc +har land +goo ch +fi an +bit moji +aj or +ach ar +ðŁĮ· ðŁĮ· +work force +soci opath +pro fun +mer kley +love team +le itch +kin z +inflat ables +ge y +e esc +chat ta +al dini +æ ¼ +w aven +reich stag +off erson +nat west +moo s +mid nigh +gubler nation +grind in +goal tending +du jour +com an +charlo tten +bm th +blooming dales +appal achi +:- ). +ðŁĺĺðŁĺĺ ðŁĺĺ +will smith +unexplo ded +thegood life +tar ver +sy es +sush mit +stop adani +sh or +ol and +mon di +meet sworld +ka isoo +indv sl +fra ses +car in +bo ve +ðŁķ Ľ +ti pi +sustain ab +strang ling +provi den +ol den +ner ium +merr ily +janmash tami +in famy +docher ty +cassi eclare +carnit as +car ing +all thing +al at +y onex +worsen ed +w aff +tr onix +ste y +name is +mole sting +ma gg +low rider +lo it +jab er +id ling +i et +cra bb +beau regard +au tor +yousaf zai +un structured +syl lable +perman ente +o gu +nu b +kyrie irving +kele chi +he ther +f sh +csr classics +chees in +chandra babu +bar am +ðŁİīðŁİĬ ðŁİĪ +tew kesbury +super coach +prison break +onco logist +ocu lu +itz man +in house +go dot +ge aux +fah lo +disneyland today +bre gman +tor mented +nou ve +margau x +mar kov +loo kit +kimp ton +ish mael +goss elin +denti al +cultural heritage +cin ch +capsic um +ðŁ¦ ij +æ ģ +yumy um +studi ous +social media +seong wu +sat ory +q an +par rott +mac ey +funer al +fiery verse +e bit +congle ton +com as +chario ts +but thole +ap te +an et +ador ning +x hosa +un women +un dy +talk like +dhar mendra +da ele +con fines +cad dy +bury fc +av oce +altru ism += ))) +/ ... +t assi +scu f +ri a +renew als +rec ited +que re +psyched el +ow ar +geck os +egyp tian +ye p +seri ou +rosel le +public relations +oak man +me theny +make money +ll ins +k alo +ho sea +hau ghton +ha gel +gram matically +at ro +armist iceday +worldof tanks +vindic ated +triumph ed +ti eri +oni us +mon cler +mo ps +mis ed +mat ures +i gem +hilton hotels +geo logists +dishone sty +din ning +but te +alger ie +ðŁĴĻ @ +sym pathis +repleni shment +md k +mau mee +margin alised +manil ow +kar ta +im passe +hy vee +green away +d st +ba hl +ap ic +aerof lot +visakha patnam +the wall +style blogger +smoke free +sig mund +omo to +leg room +jig gy +ja unes +gai ety +free code +express o +ek man +drou ghts +cu i +chall a +ber nan +am pang +way sto +vol ante +ti redness +sen gupta +scoun cil +ro amed +mt k +linden wood +l alo +k lose +jac que +ilhan mn +hoot suite +ci pd +ampli fy +ðŁĴ¯ ðŁĶ¥ +ðŁĴ ³ +tor rington +ne farious +muj he +l ó +krug man +ki mani +jammuand kashmir +in h +im en +g bt +fla vio +dee mable +bo sh +blues fest +bi on +as awa +ðŁĩ²ðŁĩ ¦ +var icose +ton ous +surbit on +ssi veness +pre form +post docs +n sr +n le +jun a +iz akaya +gul liver +futu rec +fa ster +e of +bastille dan +apo cry +ðŁĺİ ðŁĶ¥ +ðŁĺĤ " +up start +ro ff +ran cho +paw paw +llll ll +kor an +humid or +her c +haw tin +goo gl +chic ory +car ro +ax les +announ cers +what sfor +per use +p ml +drag me +dis array +belo it +bar neys +ìķ ¼ +ì° ¬ +western australia +tweet perth +petru cci +oo ga +mc enter +mb d +lawrence ville +lak sa +kre is +ke own +kar at +fro licking +fac ulties +ed ra +dism ay +de kh +davi doff +cur atorial +brue gel +acro ft +- : +ðŁĹ ºï¸ı +y ny +sp reader +share thelove +lu ca +lic a +flower power +d ka +clo tted +aton in +am ori +ðŁIJ ¨ +wood peckers +titu lar +sudo ku +sd proud +pol ynom +mu sso +mi mo +figur atively +cor nea +ak iss +Ñģ к +za j +te eming +social es +n sp +mo pping +le bo +id ina +i want +harm reduction +har ian +darm stadt +arre st +âļªï¸ı âļ«ï¸ı +wedding planning +und ate +social care +sale speople +re ck +ra che +megyn kelly +me ille +ger r +enor th +cani sters +c mof +bi u +ðŁIJ ľ +uf fi +realestate agent +ny times +mor ial +mi shima +ken do +je suits +insp ain +hyun jin +gastroenter ology +eiffel tower +cheltenham races +à® ħ +we k +v logging +shoo m +rom mel +repre ssed +newho pe +nar rating +n cd +metal gear +gloss op +ger aint +fa is +ed ition +e book +coron as +car tman +accor ds +youn gg +un certainties +suspiri a +sal vini +preeti katweets +peru gia +ke p +in shore +guin nes +gi ger +family business +bin aural +au try +acron yms +---- --- +ãĥ ĵ +x viii +val di +urban o +star ry +pla ster +fli rt +zir con +un defined +tre st +the gold +su árez +sle ds +sk elly +moderni zing +mer lin +li ere +lam u +j hel +gol lum +cr ysis +chu la +cali pers +ca ille +bri x +bou lton +big finish +bc r +bar tending +world class +welove our +te emu +sed ation +sabot aging +q lik +pos ada +mother ing +jer ker +hello love +cinnab on +can poli +autom aker +ðŁĻıðŁı¼ ðŁĻıðŁı¼ +wm ns +vand alised +ul trac +mon soon +miz uki +legis l +ju ried +je anie +intro spection +ho ggard +cor rine +br ynn +bell erin +astro physicist +a bed +à¹ĢภĶ +won ton +whok new +un scheduled +the que +sig a +ricky rozay +pp p +llcool j +keer thy +kat sina +k cc +hop scotch +defin ately +d att +ðŁ¤Ķ ðŁĺĤ +æĿ İ +we ill +shirec c +qu orum +nd x +kha imah +ist g +ge et +fle ischer +fidd ling +exclu sions +electro lyte +dispar ate +boric ua +ar mas +tu delft +te ous +li de +leg ality +jil lette +f hp +boy scouts +ar jan +al ami +௠į +tat ler +steve harvey +shrap nel +sf g +sensiti zation +miss world +le me +industri alization +ic ting +har man +fol ate +den haag +clo sings +ch are +aru sha +ado c +us j +to ying +sof life +sna pe +pé rez +poorpeople scampaign +no le +looooo ol +jit singh +il b +gi ans +dot son +do sh +bra il +battlero yale +ðŁĩ·ðŁĩ ´ +ë Ħ +Å Ħ +this s +sn v +reddead redemption +pal me +pa wel +no witzki +nar ayana +mobile photography +m sin +live at +le ones +jaclyn hill +euph rates +engv pak +dc ad +co ffins +book launch +ðŁĥ ı +strat for +speed paint +s zi +ram in +perpetr ator +paint job +ol m +mr m +hal ved +flint shire +eri o +blaken ey +bin ky +aqui les +y out +watch able +w sf +the carlospena +roy le +ri ers +py d +piyushgoyal offc +magni fication +iso c +hurrican es +diet z +c wu +br ich +border collie +bis son +æ ŀ +val ance +u kh +truck in +ter y +rick ards +pest control +natu res +mo fos +m vb +gruff alo +ful tron +e ben +doo s +dar bar +car melo +busines stips +bou din +transpho bic +schae ffer +pre cords +mee tups +isaac son +e talk +dr g +barsand melody +aye sha +au dley +ash tanga +amar anth +ðŁĺ¬ ðŁĺ¬ðŁĺ¬ +ðŁIJ ¹ +shap s +r dp +mol lywood +kun dra +ki ba +dig vijaya +cycla des +co il +back gammon +b more +wensley dale +u ar +the house +tb b +sha o +nor ri +mer alco +l ée +is our +her ak +go x +consecr ation +chrisg packham +chester field +animo sity +! ðŁĺĦ +ìĥ ¤ +ya ad +v x +ta ren +syn dergaard +road kill +nat chito +mountain view +min ec +lighthear ted +leg is +illi er +grand daughters +ay ed +aqu il +ðŁĮĬðŁĮĬ ðŁĮĬ +w gbh +typo graphic +the be +ta cha +suc re +spr att +rom toronto +ol leyball +my st +lack luster +kal ash +ilfrac ombe +il ley +hon ed +heyman hustle +gu ill +go tha +crystal lo +bho omi +âĿ¤ï¸ı ðŁĩºðŁĩ¸ +ঠ² +z oni +ucir vine +t ga +ro vani +nipsey hussle +lun atics +les vos +kidrau hl +jovo vich +comic s +beck yg +arbor day +ad tech +ðŁĶ´ âļª +umbil ical +tan que +swag gin +stor ch +show off +sallye aves +picture book +my rr +jo ele +hor chata +el dr +dil iman +cmof karnataka +choose day +al ish +ver itable +tre jo +ran gel +rail roads +ny sut +morphe us +masterche fau +mani ac +kowal ski +jaz mine +ic ahn +credit unions +cra d +ann ation +yn ski +wilhel mina +sare an +nosfer atu +gri ffs +dias por +d jash +d iller +ct p +contigu ous +bottlen ose +baha sa +âĸ¶ ï¸ı +stal bert +profan ity +pharmac y +oc chi +ju co +ishi da +fe mur +di minu +comple mented +clo ts +bal akrishna +asv px +art net +ah ed +ag b +stanak atic +show girl +resc o +res ell +re group +pra vin +mt news +mb m +li ais +kell erman +kaz uki +gr ater +dis gaea +dere rs +def lect +concer tos +bha dra +beig nets +anak ar +ê° Ģ +stall ings +photo gs +music fans +mon gol +min now +mam ie +ib jp +e ta +cd ma +cath al +c mt +arun ning +aquit aine +win ery +to res +super latives +recep tac +par ched +loun ger +ja ap +i ia +hill billies +grey stone +ge tover +fashion ably +ad eno +yay yyy +west bourne +su stains +star buck +so so +sh ner +rave ena +oned rive +k town +in ar +gw g +gir ardi +cec ily +c ations +advers aries +иР´ +yeo vil +v allo +spas ms +so ton +ra bble +r ch +q gis +n bt +lake s +lady smith +is y +iol ani +iam j +drif ters +compar atively +cli pper +business owner +birth date +battle field +ym ur +winter classic +vic ari +sub species +spe er +sor ia +sion er +si mcity +o glu +mar cell +jeremi ah +ho pi +gar vin +further more +flo ssing +dogfish beer +discoun t +denomin ator +block chains +b fp +ah at +ðŁķ IJ +trow bridge +stool presidente +sky rocketing +sho tt +shan gril +ro pp +par ine +news line +m cly +le sia +kun duz +kon o +k fm +ic er +har twell +eng in +char ot +bel per +as yn +alter ation +a ish +æ ³ +transcend ental +sugar free +semiconduc tors +sau vage +red devils +mun dy +msle amichele +mo her +milwau kee +mclen nan +ll ws +j lin +gur meet +g tm +farm ville +f bb +burge oning +belly dance +ba sti +athabas ca +aran sas +a historyof +thisi sm +tek no +stif tung +south asia +prom posal +orient ated +needle work +local business +le iter +if as +ho cane +gran ary +domin ion +bo go +bar fi +abdul lahi +zane tti +woo len +si fting +natur ally +lu ongo +jaland har +interrup tions +ge u +game plan +fro cks +foun ders +facup final +dem convention +d ici +coup é +circu ses +bar gain +à® £ +up an +tram mell +tab led +seag ames +rest itution +q igong +pull out +opar ty +no p +ko dan +juli a +hal stead +ga the +dani il +bat su +b ng +ab ca +âĢ¦ ? +vali dating +transcei ver +re touching +mindy kaling +la gu +ke mba +hi ght +fibrill ation +dei ros +cor man +con spired +arcelor mittal +âĢ ¹ +z ata +yorkshire hour +ventil ated +ueber tangel +to ile +ter us +rho da +prank ster +m ston +lumin ary +kk rv +ker rang +gru bb +bu ki +bo one +aque ous +âģł # +young people +wi ig +wh ich +wax aha +synony m +summer lin +struc tural +saddle worth +rush die +pher om +p mr +oli go +og den +ne hemi +michel in +israel ites +hip ster +go duke +fu gue +evacu ating +de fer +cb schicago +wi v +spart ner +simon son +selec ta +rat liff +ra zz +plainti ffs +lu coz +kar st +iw news +hone ys +f sen +dinah jane +cec elia +ðŁį Ł +vote leave +tom daley +tibur on +srini vasan +roth well +mon dial +man chin +lovecraf tian +l mc +ha ving +gun i +den man +de ga +chu y +bru k +blue devils +ageo fultron +a ie +( !!) +wir ral +tm f +skybet league +ra ds +pk d +neil young +lad ys +is ys +ion ian +inhal ed +hoodie allen +ellic ott +car sten +al bay +adi da +acci dent +Ï Ħ +visual ise +tre viso +tra che +speed run +ra joy +prospec t +orlandom agic +nokid hungry +margare tat +kri ss +ik onics +grrr l +go hoos +g sf +do ty +applau ding +ac tu +ëĵ ľ +suffra gettes +star gat +jonas brothers +it alien +g luck +deton ated +can andai +bo st +begon ia +beef cake +bann at +anderson cooper +affor ded +travel guide +stell amccartney +re spawn +panig ale +one il +ny ongo +nike football +mow gli +montan amoment +mid size +kel antan +jamm wal +ha se +golds mith +fo den +da ren +child hoo +ald ine +adri en +ðŁĶ¶ ðŁĶ· +ðŁ¦ į +ss eries +spear headed +se xt +sad hana +ram bam +pe ta +oligar chs +mc court +loc s +ðŁĺį ðŁĴķ +и Ñı +~ âĻ¡ +yee zy +wil ks +tcc andler +que tball +posse ssive +moff ice +medi at +materi alism +jon ath +hat su +flu ous +craf turday +car re +b hala +am hq +veloci raptor +teen vogue +table tennis +se away +pre amp +pn pd +mc clean +labon te +invic tus +ic r +help desk +exclu sivity +etsy uk +episo dic +dat sy +bu teo +ðŁĮ Ĩ +ye a +sky box +sing let +pi f +or te +om ara +man alo +mac tic +li sd +feder ica +fati h +ener gia +el ines +coden ame +cho ckey +birth da +w ssu +ver bier +ush ering +talk to +t me +ro swell +neuro surgeon +ne pen +national siblingday +mess y +mascher ano +k vy +iy i +hong bin +flutter shy +chi i +ay go +y amaz +whit ford +un welcome +si yak +scri bes +sad lers +re imer +r cr +paw sox +parale gal +my picmix +moo ts +kirk caldy +k rum +ische mic +int z +gui da +gh es +gb w +fransch hoek +finn balor +east on +blu ish +atthe disco +âľ īï¸ı +ye huda +wi jn +wag ging +terri er +swar th +state champs +star fighter +schec ter +sas soc +pod casters +omor phic +ma dy +ine bri +go pack +de tv +d xy +cra ss +chag rin +bur den +ay m +app soc +al haji +z wolle +theore tically +tel ford +ri bera +problems night +po lis +mel ind +ish an +indi anc +ga ana +food allergy +equine hour +dream z +bi mbo +alou ettes +wal dor +tri angle +ste k +ra imi +qu ell +nieu we +nickelodeon tv +mohabb atein +lot l +liech tenstein +ir p +gu stin +decor ators +cl ack +bha ira +y cles +we music +train wreck +stam kos +sar tre +ru h +remin i +pizar ro +mu scul +liven ation +jazz festival +il ence +i ffy +constitu tionally +b ld +ìĤ ¬ +åī £ +stra ppy +sever ing +priv y +oo zes +nightw ish +hom ely +grin nell +fantastic four +du vernay +ce ts +ay den +ar pur +apar na +andrew smp +wyn n +vet med +town homes +tips for +tat oo +ste t +sa iy +rock hampton +pro choice +pnpd pcr +organd onation +n ago +meg ali +k po +jan ef +i mex +het field +gen et +free diving +fis ker +fe tu +ep n +democr atically +chap book +cas sper +carto oning +betra ys +ðŁİ ± +west bank +vis es +som ali +sivakarthi keyan +sc athedral +reflec tivity +postgre sql +o fus +no da +mu kh +mitch um +m fab +hyster ically +gi ano +force ful +debun k +cru ised +cic ely +brain washing +ak aran +ab ul +rash tra +pneu mo +oun tain +manit owoc +lo ic +it all +ik or +id n +hu ppert +gg gggg +z ite +thir st +te an +strang led +peanut butter +pc gamer +lo ta +kurt busch +ko stas +kib ben +jer main +gab bott +yas u +t pe +ry and +platt sburgh +nicole scher +nb nnews +mr james +kauf mann +it san +get outdoors +gam on +eugen ia +car man +bon heur +anti polo +ðŁ¤¦ ðŁı¼âĢįâĻĢï¸ı +âłĢâłĢ âłĢâłĢ +ÙĦ ÙĬ +ut as +super man +pickle ball +optimis ed +les ford +ko tt +journey man +gra bber +co inte +bra eden +bj s +atur k +ad ler +ðŁĴĻ âĿ¤ +won ga +wi er +wartho g +tribul ation +tan ker +stan for +shev chenko +regar der +r int +pun ya +nab y +mill ican +ha er +ev alon +dwar ka +cyclon enation +boo gi +blu ed +tra vail +so aker +plainti ff +mar kh +loreal paris +kovac s +fis ch +di ab +brew master +az ole +rugby worldcup +ny lon +nat t +jas si +igu anas +flap jack +energ ised +ed x +buccane er +baz ooka +ati l +ar dee +ðŁĮ ¬ +wil mot +the stage +super massive +seva sto +sc rit +river trust +podi ums +part iti +montag ne +mer chan +meetthe team +loubout inworld +kindness day +heb den +dur kin +cynic ism +cape x +ag ulation +abid jan +ðŁİī ðŁĴķ +yo sef +un avoidable +sting y +soyl ent +shar am +re using +offic er +mend enhall +je eves +hi day +day soff +bay swater +ban ned +ative art +april fool +apple wood +app easement +allelu ia +tri o +trax source +ss mb +re publica +raz r +p ingu +ouri er +mcgra th +magn ac +k mph +irrit able +ing roup +harvard med +hak una +gre nad +ero se +ed clv +door steps +counter terrorism +andis les +à¹ĦภĶ +whiterab bit +wh ill +vad ra +tooth pick +te mber +suspen seful +shar pens +natchito ches +minute men +mick y +mer ge +libr arian +laha ina +jugg ler +james on +in ker +gen x +fin de +engra ver +chi yaan +amon day +aband oned +a ami +twitter clarets +ter baru +spen ce +shav ings +sf moma +par ke +id ly +grena dier +bu ko +ðŁĺĥ ðŁijį +ðŁķ ¯ +tuesday trivia +ro el +mul la +min ami +luf kin +heart s +gine tta +g ff +dise ased +cute emergency +cor dell +christma sday +cer ts +authent ically +ap ta +am stel +wilber force +was sily +var am +se daris +naz ar +mori ah +kis ser +k ba +high heels +hh s +give blood +ging ers +eti salat +ener gie +dopp el +dex perience +cole gio +chester fc +bha iya +ag l +we w +stu y +ss ang +sal ento +psy trance +pan ko +paign ton +im pt +hoo se +goooo ood +erink rakow +design boom +clon tarf +b per +afc cup +abhi shek +wether spoons +ventil ator +tweet deck +stap ler +pow r +plo vers +nur i +northan t +mc garry +ma ur +lang ley +kla ine +justi fiable +habitu al +g soc +fin est +extre mer +exc elling +en coder +eil ish +duc kie +bon ucci +bct gb +si bley +red hat +philharmon ie +pe cs +mete o +m ound +liter acy +io ka +i hr +hyper bolic +happy holi +ess er +con temp +cau cuses +bm th +ym urray +when in +tw irling +sex ting +scar ring +ru den +ru bi +rom ney +ridge back +ok ka +off ends +ny mag +kla ge +fix ings +excav ating +digit isation +am alia +zam fara +w kc +unc aged +tele tub +purpose fully +mex po +mass governor +kha di +cor neal +bin son +allot ments +abur ro +âĿĹï¸ı âĿĹï¸ıâĿĹï¸ı +wicked ness +vaj al +tw im +tt weet +tru es +tan jung +sin ned +rain dance +priest ly +pra e +p fi +non sensical +meta irie +le omin +ha ase +g nac +eth ic +dou gi +bucci gross +bre y +a etv +/ = +zu bair +zephy r +vo id +un ed +sc ani +sav oir +recom end +mic ho +mer ch +lo cum +jun os +instagram mer +gago sian +eri ous +cau tions +best photo +an abolic +ag ame +âĿ¤ï¸ı ðŁIJ¾ +vol ks +up vc +terra zzo +spl icing +rte one +mc cray +g pm +emoun tains +east lothian +du bz +dmit ri +den ning +c sic +blood matters +baku gou +arame tta +al pa +âĻ £ +travel chat +tayy ip +su et +rebutt al +prote a +pontypri dd +pi ac +per d +lu ker +hypo allergenic +haha haa +fun friday +elisse joson +at rump +tom maso +slo ver +on omics +metz ger +lor ca +lek ker +ja ipur +inf ood +gl ent +full metal +cucam onga +cu taneous +cp as +coron ation +cal abre +bul ging +b ko +ap sa +* -- +yo ta +wo ke +util ised +tin cture +rhon dda +pc f +ngay on +mic hi +margaretat wood +ld i +hi ther +guil ds +cleve don +bank side +af ans +- >> +vers i +v ld +under classmen +tri an +te v +stone bridge +smi ley +rinse fm +real me +re affirmed +pla st +oo dyssey +nei stat +medalli ons +mc kibben +mbe ki +hashi moto +ha zzard +ha ther +ele y +ak ko +af ashion +western sahara +villeg as +su man +nor e +monte cito +mari bor +m ch +em watson +bu la +bas sy +bar ratt +yorkshi redales +ware ham +v pd +selfless ness +refil lable +om aker +mb l +fear nley +ea b +de marc +che quered +br ze +ame ga +." -- +yar mou +x series +ri gan +pig mented +patrizi arametta +pa ppa +of ah +mu cus +lets gor +leader boards +eff ingham +drive ways +dragon sden +cl n +cit ron +chi esa +bron wyn +brack en +bl v +are id +ami stad +ae oli +ae com +а к +wax wing +sz abo +openg olf +o berlin +mac ul +inf omer +ho de +ge ert +encapsul ates +cro mpton +con centric +bil le +bc jr +as gar +aired ale +usa a +tra gus +to pher +reed timmer +rare books +per verse +mo star +lom an +ll m +je p +ir ang +fi br +f mg +e ir +child line +book challenge +bon o +augu stin +at night +anup am +ðŁĺ² ðŁĺ² +what sup +u avs +t ittle +sw amps +st petersburg +so shi +mar ni +je je +inde mn +humili ate +do ped +cate chism +bur bs +awkward ness +ðŁĻĬ ðŁĻĬ +ðŁIJ¸ âĺķï¸ı +âľ ĸ +world league +vi di +theatre day +tal is +t be +sterili zation +shaf er +scal er +plan ar +nhl ducks +mapple thorpe +get covered +esopha gus +em el +cir o +braw ler +bottlen eck +ðŁĺį ðŁijį +ðŁı¾ âĢįâĻĤï¸ı +ಠĤ +Ø ² +vin eland +thr iller +side burns +se cours +pit ting +nu tz +nj pw +mogu ls +mee ch +ine a +houston dynamo +grav ure +gor ba +glyn de +fri en +daw are +commemor ations +bay max +ðŁ¤ « +xx v +tran quil +th um +spad ina +sol ly +mo ti +metast asis +mapu to +len se +im on +hilde brand +h sj +fur man +etsy finds +esmer alda +e goti +d fo +cham a +bri el +bor dered +ðŁĴ« ðŁĴ« +wido wed +thro bbing +themo in +ta it +synchro tron +stand er +skate boarder +samu ell +pa ire +free town +csi ro +ber ners +bar buda +squ ash +so well +raf ter +rad ine +oregon ian +northern most +mo hic +master fully +jar on +inter sectional +hass am +fla grant +emce eing +captiv a +buck led +ze ki +ye oman +welsh rugbyunion +tur ney +tam aki +stro llers +nn r +merri am +lien hardracing +hi pp +ev ander +ers burg +erik sson +cn b +bas ker +aphra gm +the year +stereo typing +sensor y +rovani emi +lo gues +kre mer +four teenth +bri ann +bow ling +bio logically +bang z +b har +arch uleta +a et +ðŁĺ ¿ +ðŁĶ´ âļ«ï¸ı +swit cher +se gre +ne da +mountb atten +la dle +catar acts +b cl +varieg ated +sou d +she is +rad ars +mistre ated +mc cal +gam el +g pab +conte ssa +chri sj +che ques +ch saa +bun nings +ambi ente +~ < +ye ol +under mined +trans lat +or to +ol oughlin +off load +neuro logist +mu ba +men ong +m cre +letic ia +iz u +hence forth +gai ther +e ws +cer berus +car ia +boy george +ac entre +zen o +w ür +vanessa hudgens +sushi l +pla z +ma za +kar dash +di va +di aphragm +cloud appsoc +acci dently +ðŁĴ Ī +Ø§Ø Ń +sw illiams +stie boys +sterling silver +si xx +s bee +re td +northyork moors +min olta +migr ation +ma shing +ma sam +lo ach +indiedev hour +ga is +ep al +ec l +bye bye +bic i +at elli +asen sio +anti o +ala stro +à° ¤ +un ir +to asts +specific ity +sma sher +shop keeper +ram ada +oni e +n ph +meet s +lular oe +li sto +kaf tan +j mi +fon tan +cardiff uni +bur ro +! ðŁĻĮ +vigor ously +themoin montrose +thel asto +t sang +slur p +sed ans +regre so +mun k +lar ds +ha sil +fra p +flin ching +dragon s +disappro val +del aire +chu cking +be coz +anarchi sts +ðŁĵ¸ :@ +wic ke +v axx +tex oma +tall a +summ ers +su si +ro wy +provoc ateur +pg achampionship +oko toks +o tv +magick ingdom +khome ini +hs sports +here tic +happ py +h ito +gbm fg +de paul +come di +coffee morning +cannon dale +bra ini +au robin +till am +plann ing +ph ir +panic atthedisco +mc pe +kanka kee +ful tz +fm radio +dissatis fied +con gru +bat ors +ambro sio +adol fo +acom be +æĴ ® +ãĤ Ī +y ona +tri as +to yn +thefuture is +pen icillin +os lo +mt gs +menong autham +med tronic +inf om +her ve +gau l +essence fest +blackveil brides +amas sed +aficion ados +aesthe tic +wo de +wal lop +ts d +thermo dynamics +school games +ram dev +pal patine +hom an +go vind +g va +fe il +el v +bjor n +av u +aaa at +ðŁĻĪ ðŁĻī +twin cities +tri alling +si ue +relax in +rapi de +kal o +gover ment +gl ick +fun fair +fick le +e ed +dre vival +che o +bull terrier +berk lee +ðŁĩºðŁĩ ¬ +çµ µ +tr yan +substan tive +sol heim +sh into +scotland hour +s oooooooo +ro he +ril ke +pro mi +nam az +mini figures +fraud ster +engad get +bb b +aperiti f +... "@ +$ - +ðŁĴ¯ % +» . +west cott +smo sh +odd ball +mee ker +la wards +hacken sack +fr act +fer menting +fac s +er rant +call the +buen os +broad ening +bar bo +afl w +ac sa +⾨ ðŁĴ« +woo din +ton awanda +sin ise +re ka +mu rad +kl is +ingl és +ij f +ham els +gre gabbott +f mp +egyp tair +egor aptor +csgo giveaway +contrac t +bar nes +together stronger +su ze +slo tt +rani al +lamar cus +hick ory +exploren l +beach club +yy ccc +sw all +suc on +storm chasers +sound scape +objec tively +nov ich +ni kel +neur onal +me aghan +manny mua +iber ico +fer ty +fa thead +dol lywood +dareto dream +d jen +cr pd +courier mail +baon pdx +vi vre +thomas rhett +seal ant +sa arc +qu asi +pac o +macken zi +k ole +john lewis +head rest +gn ini +generale lection +ben affleck +zul fiq +tac om +spel un +run dle +pr ana +la pped +kni ghted +gold fields +can oe +bellar ke +ba hr +amo led +acro ix +willi ston +wen ch +vig ny +ti the +se cul +sad r +pick ler +ne pean +may r +karrue che +is sf +han solo +fri zz +flood waters +fari dabad +dar ya +coden ew +cell ar +batchel or +ar co +ak t +* ... +ðŁijıðŁijı ðŁijıðŁijıðŁijı +æł ¼ +we will +un ch +sni ped +sat yan +ren fro +re ena +rd ma +ra am +iti ative +hear to +h mong +ght me +cine max +bon obo +atta ches +af tab +) âĢĶ +ðŁĴļ âĿ¤ï¸ı +ðŁIJ µ +âŀ ° +ç e +we gian +vin i +trans continental +tear down +tar as +tam agot +semb lance +pet care +notre ally +money maker +lu can +jazz club +her tz +great barrierreef +dec can +bogot á +a az +ï· º +twili o +tug boat +top brew +se ren +santac laus +roman empire +pr ite +pay outs +n sg +n att +gun d +bon nies +back woods +ante tok +an kh +ali f +able me +ver dic +van camp +tynd all +se vier +scele bration +ro darte +pe scat +par affin +kir wan +isi dro +io sa +hun chback +gas quet +fl it +el rod +cott ag +camero onian +buck s +at wain +ðŁijĮ ðŁijı +ðŁį ħ +sunrise on +shen hua +r vd +pr yn +on side +nom i +mour n +kno t +kha l +iri es +independi ente +guer ri +ffe t +cal lo +ðŁĵ Į +âĢ ³ +sj m +s inter +recipro city +pir at +pi do +nuclear ban +nag le +ingh e +golf club +goldman sachs +geography teacher +g mw +g inge +fu g +france sco +cor bis +cli theroe +bas co +alta ir +al of +ag over +tu do +tam per +ster il +say s +ri ss +pr unes +l ck +in decisive +guide d +gou lash +gold schmidt +geaux cajuns +fo is +dr ona +ct x +anup am +all things +achri st +ðŁĮ´ âĺĢï¸ı +ãģķãĤ ĵ +ve taffairs +sar is +qwer ty +ore illy +mcgu iness +je c +ir lam +h vac +for an +follow up +elix ir +clau sen +bram all +bighit ent +baum gartner +y mm +van ce +ta pur +s fa +pre ity +mach el +got g +dess ert +client ele +br una +bo ylan +al td +spy der +spirit week +semper fi +re developed +r ko +pre face +mc adoo +mal kovich +m mu +kanan askis +iw obi +ell yn +dream ville +dick y +coo lio +char maine +canal rivertrust +brown back +brac ed +a ena +tal kin +sw ot +si raj +say n +ryan gosling +ole um +mil denhall +ka dir +gram m +eng ined +dont try +death bed +cor sets +close the +aro or +amaz ement +al akshmi +é u +upp olice +tem be +stev o +scan lan +reco de +ma pper +lux e +ke yn +hr v +horror story +flaun ting +em s +dor je +dignit as +dar ul +chor ley +chav o +b hoy +ar us +ac ram +ðŁĹ ½ +uof cincy +universit yo +te aday +sal k +pin kerton +mc all +man oa +ma kat +ke wl +it x +ili us +ibu profen +go el +gi glio +f and +bau mann +bastille day +un balanced +ter rence +shot els +row ena +ra she +pein ture +moham med +mis sc +gau che +daniel son +cud litz +counter act +ca vern +ah soka +after show +wh ot +unner ving +to ko +sho pe +rise of +red friday +pobl ano +noble sville +naturema pr +mam malian +ma goo +know le +jam shed +go k +fo wl +dh ana +dand elions +cu ador +colleen b +co ba +bug ti +au guri +ap ad +am be +и н +vin ton +to vote +sentim ent +red chilli +rar itan +ra quel +min ter +kore atown +ha bl +final ise +fa ison +engra vings +ab at +éŃĶéģĵç¥ĸå¸ Ī +yo gan +x anax +we er +wahl burgers +town ships +stra gg +ste er +ste de +sel ive +my st +lu des +liv onia +kite boarding +kemp inski +joy fully +j hu +ig ner +go har +centr ic +bri bed +bla zes +ag rit +ver mon +u cle +sc ard +por g +plex ig +no plac +man nion +j abo +i aff +crest wood +co org +co horts +cla dd +can ard +bi kel +bannat yne +ban n +ðŁijĩðŁı¼ ðŁijĩðŁı¼ +zi ers +yesp lease +su fi +spell ings +quar ant +pa di +ki ff +end gunviolence +e ggers +con signed +ce au +brum bies +brit o +aldi uk +ad sor +abo lish +win itwednesday +thre elions +tech ies +snat ches +sei ze +pu is +ol mos +min chin +mce wen +mar ner +malam ute +made myday +labour day +da ar +cott age +ban u +ak land +ðŁĻĮ ðŁĻı +Å § +the wine +shuff le +s map +road work +re defin +mud slide +leon ie +head waters +hay don +clyde bank +cas in +cardiff cityfc +ber li +bar bour +au ston +ab us +ç Į +yi k +wa pping +sun der +scen ter +par snips +no bby +jen i +icom be +hpm kt +gla sne +ga han +fre ier +co is +bru baker +vis ite +te sta +te flon +roman tically +my c +kir tan +get some +carry on +asian et +_ âĢ¦ +wat terson +waste management +room ba +red ick +re mou +r team +prince harry +pr ca +post ings +new mexico +net galley +mp loyment +mil ano +cry ing +cd b +á´ ĩ +z ang +weather proof +tang ling +strat ford +sleep out +shown u +nir mala +n krumah +mon iz +lan et +del onge +box ster +bij lani +ag upta +a quar +yoon min +win the +un afraid +ug ent +supervis e +sn u +shak ib +sal taire +ru sk +pre fabricated +pentat onix +pe ston +na stur +l pin +go dal +faith full +enshr ined +crusad es +aldu bb +al ok +whole meal +riz in +re dri +or ta +nutr i +kelly file +gen k +farm shop +erken ci +du ffle +dev endra +ci gn +bc ity +av ram +ale u +ye ung +unic ycle +sp rang +si ones +ri parian +plu ton +pierce the +pha sing +od dest +non o +natur alized +n ls +my favorite +k ran +ic bm +hom i +gro cers +gov christie +four some +fl keys +d ally +ayour b +yor g +when you +tw ang +ti als +r tel +nationalbestfriend sday +mcgu igan +kath i +invo king +ev ading +dor tiz +col borne +bur qa +balu chistan +and proud +am ba +adidas uk +âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢ âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢âĢ¢ +uk is +tra ore +then fl +quarri e +origin ator +om nis +m gh +knare sborough +it d +ho yle +donat ella +cho ses +capit alization +are pa +ar ua +un cann +twitter afterdark +over the +ley va +le ggy +john wick +her pe +ev ora +der mato +a wat +ðŁķ Ĺ +âĿ¤ï¸ı ðŁĸ¤ +ภ¿ +wil ted +the tonyawards +sig s +sha hr +sel leck +re van +pan eling +jun ket +id g +gol dent +gaz ian +don gle +car fax +at w +| || +wheate ar +whe ein +tabletop games +speed way +solic iting +shk reli +ser ia +s ann +pol anski +on ara +iw f +indi st +iam nagarjuna +gre ville +fan zone +ee ek +do vey +bhar atiya +astro turf +antetok oun +amazon music +all saint +al amy +v ora +tv t +sy fy +rob ison +ra zzle +pli skova +offshore wind +no id +nain ital +ma dog +inter reg +il bert +hot elier +gu gli +chri shem +chas ka +ath om +and om +vo st +ter p +sun tory +summari zing +stor mer +steve jobs +st x +sc dsb +po tre +news boys +mc crae +luc ite +it at +excu sable +daz s +colon na +b andy +war game +w ans +valenci ano +sa chet +phx traffic +phila union +mumb aim +mar gret +fon dation +explo realberta +defe c +david caruso +a egy +ðŁĶ » +미 ìĬ¤íĦ +vir gen +ren dra +n ack +mon deo +live the +l sch +j lt +di ka +con dors +berry man +anc illary +acor n +о ÑĢÑ +wythen shawe +tobe apartner +tai z +street light +star lin +si u +pro sser +ph is +on track +m wh +humanit arian +travel ogue +trans duc +theop ap +seman tics +sat work +sahi h +pas schen +nik ka +narra gansett +na thy +man ado +m po +l sc +kcap inoy +kcapinoy star +i dd +ge ass +g onal +fair field +d alia +clean up +chor ong +ay ang +yn n +x fm +wil a +ren dez +ra vish +qui ff +puppete er +nat asha +inst on +fi f +e star +do shi +cu zzo +corre ia +cole man +annoy ance +.. ðŁĺĤ +á l +wi ps +try st +top chef +spoken word +sel ah +madel yn +lg fa +give me +e wood +don ington +ci gna +chry stal +calic ut +zimmer mann +tre et +spon tane +sour is +sag et +palli ative +mo jit +htafc dotcom +dre yer +dev ore +cycl o +cor by +bey hadh +banque ting +aber ry +ãĥ ł +wo ong +tess er +ss sssss +shop boys +screen caps +sa dies +obliter ated +ni as +mel ty +knock down +ka ji +ep f +din i +british council +ðŁij¯ ðŁij¯ +ðŁĩ®ðŁĩ ± +ðŁ¥ ĥ +wu v +un opposed +sw enson +stu ffy +spee der +raw ford +r gc +prayfor gaza +pg achamp +p nc +oni sta +mtv la +military monday +k he +fiel ded +engagem entr +en amor +cas sell +cad res +arund hati +.. ?? +⼠¹ +war ks +ver ny +theopap hitis +subtle ty +stat us +spro blems +spin n +simu lators +sail boats +rais man +oc are +mw angi +london symphony +freddie gray +con way +class act +bebe rexha +air bender +u yo +the music +re did +queu eing +leav in +kitchen rocknroll +hau d +glo ck +fe ile +be vy +bass master +barretto julia +band on +abar ça +a ep +¨ ë² +z ko +we support +trol leys +transcend ence +tal esof +silver lake +sharp shooter +schwe itzer +real gdt +oh yeah +life blood +king sme +heart attack +glori etta +extre mity +cro y +com motion +collu ded +col fer +checker board +cath ay +buen dia +am uses +aa ahhh +. ðŁĺĺ +ðŁĶ ľ +wo hoo +twitter vforce +rot ates +qu els +pizzahu t +pan tai +or me +man gesh +happy saturday +h kg +ge station +communic able +coast lines +âĺĨâĺĨ âĺĨ +y ooooo +thiru van +steve austin +ni azi +gg anu +em w +d itt +buff ering +am ma +ðŁĨ ķ +wha aaat +vs gb +spe ight +re sis +m se +j ho +ib aka +fro ot +evalon goria +din klage +bio hazard +beli a +ac as +ðŁij ķ +Ð º +tw r +sysad min +sun burn +rrrr rrrr +pr ater +kyush u +go by +consequ ential +come together +beÅŁ ik +bab b +annak endrick +ðŁ¤ ĸ +x rd +too good +seal er +re ira +ra ut +pet tit +own tv +ol ler +mountain dog +mis sp +goodbeer tweet +european union +efur niture +dra dio +disc ern +call ous +âī¦ ) +ut mb +spur rier +soli der +or bison +od g +mic a +ktn kenya +koep ka +ic ca +gau lt +g x +g dn +for ts +fil mawards +eu tical +ea g +dier ks +cannabino id +bul bas +;; ;; +ðŁĸ IJ +vit toria +up lift +under writing +sne ad +sn ell +re distribution +p do +no akes +narayan an +j vc +gram ophone +franç ais +ell ery +convey ancing +bi ked +aw we +ab ulous +wan te +sh wara +pay son +lu mumba +lifeat att +le ics +iron fist +gr int +figh to +copper head +aqu are +ÙģÙĦسطÙĬÙĨ ÙĬ +we make +t ys +qu t +play as +off a +ne revs +must apha +meta physics +mammoo tty +legali zeit +jun oon +jan n +flatt ening +du ral +cam a +bub ba +antand dec +actu allyn +aar ons +ðŁį § +á IJ +wi zz +twin peak +sle wis +parishi oners +oak ham +mai du +jessica jones +bay town +az s +ates sen +anc ing +ðŁĻĮ ðŁı¿ +ðŁĺĢ ðŁijį +ঠ® +ske w +fi af +da sha +cladd agh +bino cular +bal le +az ria +v ented +ts laq +sn m +pen chant +mod ality +gand hin +frivol ous +del am +cc na +ang an +am os +alente jo +across america +y ore +twee ter +the clash +ny lons +needfor speed +mag got +lion king +har id +h sieh +fabi en +ul hassan +ui design +ste vi +sl ats +retwee et +radio graphy +por poise +man cuso +lap wing +ki bble +gram pian +fai ers +ec nl +dun phy +disney pixar +de eney +ca pote +ðŁ¦ Ī +Ì · +v int +tyranno saurus +tu gal +sw amped +su strans +small town +seag al +salvation army +ready stock +kri ders +hen an +groom ers +earth lings +ce da +bom i +actuallyn ph +vand al +sch rö +polic eng +nbc blacklist +mul ca +jack johnson +eeee eeee +bri elle +brazil gp +b ages +woo gie +wat tle +ve ley +tede schi +tape stries +stain less +sb s +pri yad +parish ilton +nam pa +mor rell +melo dic +kam o +impro ve +hill climb +eur or +dev ant +dal umni +chi ellini +al chem +ak ashi +vote trump +steel heads +six pence +po wn +offici ated +new yor +magnum photos +lin dy +la yed +int ar +immortal ised +hall fame +f hd +cor dy +ba a +ar ru +ðŁĵį # +âĮ Ľï¸ı +tt b +ra pper +pier cer +pe m +nomin ates +marathon training +le vert +kodal ine +el ford +e gl +doyle stown +ay re +as suring +yo tu +vel lum +up sers +tg f +supple mentation +phy sorg +never stops +mean est +maple story +kid dy +incu bus +goav sgo +fic h +cot illard +carmelo anthony +c ny +c me +az pi +âľ ° +suf jan +sneaker head +sher if +sa har +rum mage +rub instein +remitt ance +rail a +phant asm +onyour side +mccut chen +main streaming +it ag +hoss ain +end or +de briefing +cou ros +boo tie +bharat anen +baesy stem +aud ited +am un +ðŁĨĺ ðŁĨĺ +ঠķ +v apes +superannu ation +ry anc +rec ourse +re working +pom pidou +pok hara +nma ahc +equip ments +do ha +cham bray +ba ste +year lings +vap ors +tom kins +tom hardy +san s +quo tes +pixel ated +mur tagh +md ma +mau led +erec tile +dd j +brah man +blood stream +alway sin +ai kman +whad dup +un authorised +topbrew stues +sea horses +remitt ances +ra id +play ers +lee son +joh nam +ipan ema +dust bin +devan te +ab hay +! ðŁĺĢ +un ni +tar heel +o jib +mal lett +machin ist +got chu +gb l +e ish +discrimin ating +bc d +az tec +avi c +ðŁĴ¥ # +âĢ¼ï¸ı # +wool len +timm c +sun se +st oughton +sethro gen +ro tten +ro sey +over lords +night shade +mou ld +min c +mi ele +line of +lifeli ke +glut tony +fla galine +fan made +e art +destin o +desc artes +bun dy +artist as +we bradio +ty agi +there in +su si +sp rit +side by +ro isin +pt bo +pro bed +passchen daele +n ich +man as +jor dy +gwend olyn +far rington +ef ury +eamon n +cu v +buzz y +ut tered +t ally +surbhi jyoti +stu m +shar an +q v +pre tender +ji kook +hol ger +gh is +co axial +che wie +blue moon +ash bourne +up cycle +tes acker +sy monds +silent film +service design +pre go +pa wns +one ttes +nc ss +monmouth shire +lum ley +level led +fun nels +flint watercrisis +flick ering +edel weiss +croke park +cloud flare +cis neros +b appa +un protected +sp anned +som in +score sheet +look outs +libr ar +jen der +jas am +g land +french open +disclo sures +az ura +ðŁĺĬ ðŁijĮ +ðŁijı ðŁijį +wel a +vit ra +spine less +my way +le anne +lat ics +kri ssy +k va +inge sted +hu bris +h me +furnitu redesign +f md +discre tionary +d mm +comple to +bc sm +balo gun +womanin bizhour +som mers +pd m +ol um +o sho +ne en +mobili zed +me gas +incess ant +gu aj +ga th +fa ste +ed un +col lies +arche type +ad us +ç Ħ +yo yo +ul lo +re wilding +mac ron +m peg +kk un +ji ju +for senate +er ud +edi son +com ey +ðŁĵ± : +æ Ģ +un worthy +talk ative +sc rolled +s ment +rainbow six +pin up +p tv +nc w +hager ty +di xie +cor delia +coles windell +ch ito +c pim +ali ef +ðŁļ ª +âľı ï¸ı +âĢĶâĢĶ âĢĶ +x ss +ww ltv +tv l +sel van +ra gini +ph ore +par ry +o show +mar ref +mam aya +high field +fis ch +e amad +dg allery +dail ye +ck ont +ce ce +buon anotte +be ary +ðŁĺĤðŁĺŃ ðŁĺĤðŁĺŃ +sw ope +snow drop +sin dhu +pet worth +mur row +mou st +manife stations +gra dation +gorba chev +gh ul +fc dallas +euro trip +dw b +dom ic +datasci entist +ali sta +ac ps +ðŁij µ +ve mos +tur nar +the first +su var +nord ics +dizz iness +dit ko +complic ate +come on +cogni zant +citro en +am ory +ðŁĩ®ðŁĩ ¸ +z ela +y are +super fans +r ry +meas ura +mari ok +je ux +green newdeal +gi um +d zeko +bicycli st +approxim ation +appli ed +actu ally +ðŁIJ Ķ +wwi i +under lined +so ty +slur ry +sho ta +scol ded +o ona +no ord +naturale za +loveyour self +kim ura +hack man +go sh +dru mand +de jec +chri sco +cel le +apr ile +ad ot +åĨĻ羣 好ãģįãģªäººãģ¨ç¹ĭãģĮãĤĬãģŁãģĦ +un provoked +tt ps +step father +sen tra +ro hini +rabb a +personal isation +mirr oring +mc mullen +lun ges +lat itudes +koon tz +kevin jonas +jimmy johns +forzam ilan +car bons +ach enko +ye sh +worl dd +war sz +use fulness +su pra +sol as +rapp el +mo sth +ki is +im bec +efan dom +drou ght +co ax +bur saries +black bear +best oftheday +ar up +ðŁĴĸ ðŁĴķ +woo commerce +waist line +tr ini +super liga +recur ve +ra ho +nj ca +nas r +mesmer ised +mer tesacker +lu ce +j illi +im mobile +de commissioning +bo ta +] ... +vo j +tibet an +sponsor ships +sp ad +roger scup +re filled +pune eth +olivier awards +nether land +n whl +kil kenny +kedar nath +kap a +ha shem +follow train +eth yl +dar my +cr ps +bay ard +wre tch +w mag +super girl +su an +prece ding +ni uk +multi faceted +mali ka +insp i +fr b +emble matic +cap uto +bur ren +xim en +ul ta +smo key +si zable +remain ers +mesu to +men zel +mc daniels +is kandar +fuel cell +fron ds +bu xton +ari ba +americas cup +am iz +ðŁĴ ² +x is +ur chins +sur fl +sn p +see it +or ra +nf örde +lat ex +kre m +ir v +hel der +fore t +ecker nförde +drum heads +car nal +ðŁİŁ ï¸ı +whati f +vas ili +succu bus +s wales +ret ford +mon di +ma ina +lun ge +la shed +indie wire +gla sper +fresh eyes +forec ourt +fan k +don n +disturb ances +denomin ations +boy ish +arav ind +( âĤ¬ +⼠Ķï¸ı +whe elie +u plands +scru ises +pet s +me chat +mac am +like mike +lgbt qi +jo li +ido sis +iat se +he di +er oo +eamad den +ê¹Ģ ìŀ¬ +ste ered +rh s +pop sugar +n ape +mun nar +ling field +leban on +lan te +kend ricks +jelly bean +igh ton +ho dder +gor ky +give sback +dayin wa +cor tic +c caa +buzz ards +ar awa +aaron rodgers +ãģĵ ãĤĮ +yearsof onedirection +wood wind +true to +sal inity +re sin +pl ural +nor cal +liz quen +kay ne +gu rion +gi org +gallo ps +conti go +chil de +car issa +ye oh +win ky +w nu +son parade +show case +sho walter +ru ston +nicolas maduro +newarri val +monster mile +kumar an +kad ri +jim cramer +gu lab +gravity falls +g chq +esper ance +cur lers +chamin ade +brad field +travelchat sa +tor rens +rh swis +ree se +mal vi +lof ton +law firm +kp cc +it ab +fer i +el lum +diversity and +counter point +chrishem sworth +chaplain cy +biz journal +bi sp +bi elsa +at cha +assur ances +ak ay +aer lingus +ya yoi +sode xo +reme ber +ord nance +or ation +lin donesia +jo sey +hast ily +go pin +fan atic +el oun +depend encies +comp ounding +az aki +al wefaq +ðŁĺī ðŁijį +ðŁį Ĩ +venkai ah +stimul ated +pp act +pmr outine +papp ar +mel oni +mc gur +j itters +it sc +harsh ly +ham ish +el ca +dece mber +de wy +copper field +bha kt +be more +apple seed +all yn +aby smal +ðŁĺħ ðŁĺħ +ys sen +tu q +to ei +thor ax +se din +sar cast +po way +or se +jan asena +cityo flondon +cat lin +car lie +bie bs +bc fc +ap y +[ !!!] +:- ))) +trav elling +raun chy +pim ped +kat ja +ju tland +h pl +first day +crew life +colla bo +che ong +che chen +bl ink +bernab éu +ban c +win x +ur gent +tul u +sof c +repri ses +pe pin +optimis ing +gau chos +com bo +chang wat +bo ca +b dm +audi sport +ðŁįĢ ðŁįĢ +é« ĺ +white house +sav in +r so +p bo +k de +illi brand +g sr +conver ging +conduc tion +adequ acy +ab ane +wood all +tha ic +tant alizing +soren to +satis fies +rush theband +rhyth m +ner c +ma ilers +jin hwan +exem plar +en acting +dar r +d ars +ball o +agr itech +ðŁĺı ðŁijĮ +wide body +u ow +tur ley +sab u +red waver +perse us +out do +nam c +mm el +las z +kne cht +interne tradio +haw kn +ey fs +dur bar +aegy o +. -. +w awa +venkaiah naidu +sure fire +stone walluk +ru slan +royal enfield +pollu te +natur alization +mo oning +li otta +iow ac +he yer +eli ver +don th +cal ma +bri anne +am ission +action news +vish war +treach ery +talk back +sav chenko +ri pon +pur vis +no e +mne monic +kol kat +k oni +johnny cash +jam el +gall i +fer nie +extr alife +eeee eats +dom ani +dann er +cy b +bel fry +ðŁİ ¿ +zil djian +yam aham +tur lock +to play +si sa +rho c +passiv haus +paratro oper +ju ara +insectic ide +fat boy +brigh ouse +be cket +ao e +wel lian +tim tebow +thegirl gang +su c +sto watch +sp iti +octa vi +jen g +jac aranda +improvis ing +hoo doo +gry phons +fri t +be ane +ðŁ¤ Ĵ +yo ka +wo gan +witha view +un controlled +tw oman +ti z +thereal taraji +rams bottom +ra bles +pen ce +pe per +mi hal +man ti +mal to +ja u +ig ar +ice service +hosse in +gen italia +g age +fascin ator +baz os +abyss rium +we bex +viole tta +une lected +un ashamed +sor row +ram akrishna +pe f +pay a +na ev +mor gon +l th +j iri +f sp +ethnic ities +elle magazine +co leg +ali bab +ëª ¨ +⾨ ðŁİĤ +up fronts +stoner nation +stack house +retali ate +ram apo +preity zinta +osc illo +n pc +instam ood +in ck +hun ks +hi b +fluor ite +disc losing +br g +appropri ated +amé rica +y pe +way anad +vi ñ +v le +trin kets +to to +syn bio +stru th +se wed +r ce +pain killer +night mare +loan ee +implic ation +guer in +fi i +deb out +dal le +clut tering +astra zen +as saf +afric ana +# ## +ðŁĩ¨ðŁĩ ³ +⤠µï¸ı +tex turing +steel workers +star man +son n +scho on +roo de +nit in +mi ah +inten ding +happen in +hali m +gun fight +ge ffen +de pot +che tt +am sa +ðŁ¤£ ðŁĺĤ +yess ssss +sha ina +scen e +sb spop +rol lin +penand ink +our n +ok ami +mer cure +me thu +mari ya +en closures +dmn takeover +athle ta +aggreg ator +wash out +sunday sunsets +re watched +nr cs +ma shi +lynd sey +k adam +ik ka +i sen +gc n +fl un +ent wi +discipl in +antic a +. _ +ðŁĸ¤ ðŁĴĽ +vit is +ur laub +trans at +tra inee +tom petty +the powerof +next generation +mo is +mac er +liam gallagher +lev elling +k aga +int ell +gh ard +dol man +cu ten +cla ves +cam ill +bur well +ag ia +accu sers +à´ ķ +zak k +yan cey +wi jaya +w rest +ven ables +te sonline +sha z +se gal +ri r +pin us +phone tic +nor s +natgeo wild +le asure +hi an +ham mar +goo gl +ga den +el che +cab ot +bu lova +bah n +an agram +agency life +ðŁĺ« ðŁĺ« +u ña +tro wel +tam im +se me +pap u +mfab oston +marin as +ha de +evapor ation +com miser +bor sch +bor ja +yo del +toho ku +ssi ve +new marke +mine head +mar wan +mal ari +m mb +kor fball +im part +hedger ow +he uri +gab bar +elpas o +e wu +cour chevel +col qu +char ol +buzz word +ab vp +visual novel +tac s +san ghi +ph all +per kin +op hia +mexican food +math ilde +li do +har grove +gor abbit +fun house +envir on +e der +de sen +confi dant +b ya +ay k +ant ina +an anth +ภ® + ¦ +yang tze +tagli atelle +sr w +sor ley +sk ellington +sever in +s oooo +mo ku +mar ri +iphon ex +invo kes +guil len +get to +enti rl +en cel +e bro +digg ity +cr itch +ci morelli +ðŁĴľ ðŁĴĽ +yo ho +su deep +so cool +sine k +see ker +roy soc +ro ps +re mington +re lo +paul walker +ode tte +martine z +lec ture +laban pilipinas +ken z +hibern ating +flag pole +fight club +fer nan +ab ack +tam iya +stone hill +stin k +sell ing +re treated +pig tails +pe eler +par ten +n ku +loaf ing +ko vo +i sie +ferr aris +cdne con +c ti +bi le +ber cow +bar ing +augh n +ace res +ter se +sten de +rizzoli andisles +ri son +rav iteja +ph q +lo ews +jaw ad +gim me +fridaysfor future +cal cite +by line +z aya +west mont +v ce +tt ac +t ø +super show +stel ena +scape goat +mesuto zil +mer s +livel ove +g end +g ann +fun kad +evan cho +conver sing +ak uma +ðŁĴ¤ ðŁĴ¤ +wh itten +ti gnes +skysports news +sex press +rum maging +ov ary +mu v +ma homies +ha chette +gi gging +gi gg +fel ting +con vivi +blo or +acoun cil +à² Ĺ +Ú© ا +z ner +sc w +rose mary +rhswis ley +rabin dranath +polari zing +mel atonin +len nie +leed sr +ke zia +infan try +he k +gen nady +ey oung +change theworld +bu te +bay bay +assemb les +ðŁ¤ ¢ +wise words +we ws +was aga +sw v +run k +pul s +mon iker +mer o +hur ried +garden ersworld +frisky friday +ev b +cn c +c mv +c ati +actionnews jax +w st +shot guns +scottish labour +sau stin +new single +merr ill +ma jer +kitesur f +impecc ably +grand fathers +go bi +glu ta +fe moral +fa thered +e sports +cre spo +bhagw at +au coin +aram irez +ang u +after care +w aca +trac eable +sav oring +purple reign +pas qual +may ans +maidu guri +li ens +im t +ful bright +f ram +domin atrix +be my +ai ww +wal sall +w res +ti ri +ti ed +sch engen +reasons why +luxury life +le pore +kn itters +he k +bibliote ca +bene factor +bedazz led +bbc three +ad g +ðŁĴĻðŁĴĻ ðŁĴĻðŁĴĻ +si stas +sch alk +roch mn +r pu +pic ton +paper weight +over se +mat zo +masi h +gwali or +gau r +football manager +flin toff +fitz ro +dal and +crescen do +bow ery +ateli er +ark ana +antetokoun mpo +ws fa +wi zz +st angs +ro v +poo kie +parid hi +my lo +itu ne +hu ed +gorabbit ohs +fred do +ed ical +dj mag +beacon sfield +( > +z ep +wab bit +u om +stu bb +stap o +singular ity +p gp +nehemi ah +music education +ho key +gun nison +fri zzy +feed ly +chap man +ch alo +bien nal +belaru sian +aga ins +> " +ì¹ ľ +ton gan +th ais +stor me +seque stration +s fra +psycho active +ol ph +mi dat +marc jacobs +mar ini +m ellen +layo ff +kan chi +hi hihi +gul zar +equ us +can va +bellar mine +bad minton +anag rams +ðŁķ ĺ +ç © +valenci acf +tanger ang +ss ociety +shaw shank +sche rer +sc ity +red v +ra whide +petr us +od as +nsc aa +man am +lock yer +lar ams +kiri shima +im petus +gu lag +french gp +cu bano +bil lo +aw in +asser tion +tre f +the expanse +raisethe wage +o smo +melancho lic +luci an +koo pa +cor relate +colourpop co +c zer +bis ky +beck with +all ga +al ang +ðŁij¶ ðŁı¼ +whydont wemusic +unfor giving +str ath +sell in +ron paul +ri sm +qu ino +music day +mat ata +legion of +heat nation +gro ats +fawad chaudhry +ebit da +chriscor nell +adam awa +าภĩ +z n +waq ar +vel e +treat yourself +so cratic +pie tro +net suite +leon ards +lam bert +kyrgy z +k cb +ike ji +he f +gfx coach +fat tah +fashion blog +chi story +b sk +ðŁĺľ ðŁĺľ +v lm +shaw cross +plo p +pixel s +indy star +in compatible +home brewing +fri eda +dun gare +consumm ate +cha eyoung +brow ski +are llano +ar sh +anni es +- _ +ðŁı½ âĢįâĻĤï¸ı +teddy bear +humane society +geo graphers +for sake +de cap +com plying +col onists +car ay +bu is +ðŁij¶ ðŁı» +veer am +tra jan +to ch +shyam alan +ri ki +pre neur +pin wheel +per v +o sei +its not +iiii ii +hydro logy +haram bee +gossi ping +fix in +ec mo +be art +ar x +agra wal +ãĥ § +univers itas +tremb lant +to saurus +shin ki +sci oto +om itted +my asu +lou ghton +hypo thermia +ee as +cre mated +az ale +as as +!!!!!!!!!!!!!!!! !!!!!!!!!!!!!!!! +wi po +suffer ings +ro dr +nouri shes +noban nowall +me irion +kle ine +kan aan +ho ek +form entera +eng vaus +ck c +celine dion +ber sih +ae us +a hotels +z ke +sch im +poo dles +po sible +on ted +meri jaan +luke evans +lor re +li ber +ili pp +her nan +haw kesbury +eugen ics +bates motel +assam ese +win denergy +ver tic +th aus +stan field +peru ana +new ick +george takei +g mp +fer me +dis su +day mond +cha el +cer ia +ai shu +ĤâĸĤâĸĤâĸĤâĸ ĤâĸĤâĸĤâĸĤâĸ +ðŁĴĥ ðŁı¾ +un specified +tor ry +sp ousal +sapol iceservice +re funded +p andy +master sof +madel ine +m rd +julian assange +gon u +dep ended +delmar va +de commissioned +de activated +com usic +bir fday +ance day +alber tsons +al man +a etna +yas min +whoo pie +vidy ut +tv land +too led +sutton nick +pro mon +nudi branch +lm g +kh ans +jac inda +heterogene ous +e zio +com pass +calibr ated +bio logics +ben j +back channel +v aya +tra yvon +suffe rer +snow white +scre amin +pledge music +palad ins +mo va +m mo +lan k +kr rish +cd ns +brown s +bah nhof +babylon ian +b á +ath ia +arm oury +ad ou +abo lishing +== => +th us +quen ching +proud mom +pre phoo +pheno type +par atha +monte iro +ex hal +evalu ates +drop the +ashi on +all my +ðŁĴ Ĥ +you m +writers community +tye ight +tren to +stu bs +southea stasia +some place +per using +pepsi ipl +no ggin +mc curry +gustaf son +fon si +chri sy +ar peg +abram ovich +... ðŁĺį +âŃIJ âŃIJ +uni for +u af +tap ings +tank ard +sn cf +small town +sid mouth +se ta +rosen baum +rc f +pay sage +oo th +mer v +kit o +ka ito +jewish press +go bolts +fen way +fc bb +clu stered +cap lan +bo ater +beur re +bar nyard +anti viral +z uri +thorough breds +super boy +pha ges +par ibas +our team +ol denburg +mccas kill +ky ga +ku la +kk city +jo jos +girlswho code +far cry +da ren +clon mel +ar ci +alo y +> .> +ðŁĺį ðŁĴĺ +viron ment +ul s +u ssa +run ny +riv ington +pp u +over worked +loc i +is si +gradi ents +gol i +go eags +gla iz +csul b +cn l +ash field +am k +ab ject +ðŁIJ ª +you thin +table lands +ta de +sewer age +sau sal +ro la +py rite +palm dale +mur doch +love capetown +ka unas +hur l +gib ber +ge stal +fu mbles +eu banks +down tempo +dae bak +cra ves +cal stampeders +caber net +be yourself +bay field +val ken +tow ner +terri ble +syrian refugees +sun lit +splen did +saf b +rapha elite +pseudon ym +p mp +night light +lo rele +kiran ks +in off +horror movie +gr und +gilli vray +col son +cas well +bu mba +ØŃ Ùħ +ver batim +she h +scu le +regime change +r cd +pla id +os sett +mobil isation +mel d +marcel a +leg day +ick ness +fo lic +farm bureau +f hc +dee pp +cli ft +cla dies +bu gged +adel phia +shel p +ok av +nav ami +mclou ghlin +femal eartist +ed r +e juice +dissec ted +din er +clean ing +x tra +we tt +wa ii +w vb +vi ana +ve das +tal le +stat er +sale en +sa iler +s do +onas sis +o der +nitish kumar +new garden +lea ke +ho kie +h ds +ge w +fu qua +dor g +chlorophy ll +brain y +ai bo +âľį ï¸ı +âĸĶ âĸĶ +๠ĭ +venezuel ans +teren gganu +syl lab +sling ers +shar pe +san ce +re es +mor k +mill man +mick le +lo ser +jor dyn +horse shoes +gr ath +dre cht +dar ley +bow en +ar beit +aero drome +aditi rao +ðŁĻĮðŁı½ ðŁĻĮðŁı½ +® . +w dr +voll mer +vis sa +vermic elli +v ÃŃ +un de +son nen +po da +md wx +ky an +kilo grams +j ko +gran tees +gin ter +for acause +en coded +ela er +el sen +christma sparty +alk maar +оР² +wed ged +war crimes +wa ja +virgin atlantic +su bro +stabili zing +sc ab +ru f +olivi a +medi o +mc iner +je ka +im onday +ho va +hier ro +grey hound +great day +gibb ous +gay travel +footsc ray +far had +alon te +your say +tri state +the box +sed don +sb f +re sol +op als +nay sayers +mcco wn +m atia +jab ber +gg c +ft ar +fr anti +f ndn +ess endon +elyn n +at it +the on +som m +share mondays +polish girl +pic ka +pi ped +p mu +nave en +mus ch +lal it +hollywood bowl +fow ler +d ga +cor on +car leton +brow ne +b blf +as ante +wr n +vol ker +syste ms +state park +seduc ing +san de +row lands +riz wan +pakhtun khwa +kri st +ka sia +hudder s +frac turing +f yn +esmol lett +dc b +brisbane broncos +becer ra +ak ir +ðŁĩ ³ +í ĸ +Ú© ر +vil les +responsi veness +rain maker +pig skin +marti ans +mariu pol +h wc +ge h +gar ra +fre ire +flight less +di biase +ðŁİĦðŁİĦ ðŁİĦ +wwww wwww +ti pu +ti amat +succes ful +shi seido +nas akennedy +mu x +mu swell +methyl ation +live from +li est +lavin ia +jes u +glynde bourne +g atti +fro thy +coinci dences +bou lud +bizar rely +ber nad +avel ino +ast enders +Ì µ +v sa +un finished +soci ologist +seaf arer +poly glo +pa chi +ki as +ke th +karthi k +jac qui +ha ik +g cl +ferr aro +cornu copia +brock port +arte fact +aj ah +trun dle +steu ben +p gi +opportuni st +mussel burgh +mat ar +innu endo +hawk man +h under +figu arts +ey ama +exter nally +dun gy +debu tante +con sol +chand ni +bin d +au den +ak ari +af ood +ãĤ Ĥ +yu mmmm +yellow fin +volley ball +to gs +sidi be +nür burgring +newor der +len ient +lar imer +justanother dayinwa +ill aries +hamid mir +fine gael +bri enne +blog tour +be ter +bar to +ard elli +yotu bazos +tah itian +spit als +sin sider +see ing +parisi en +over hauled +op lan +mic kie +long shot +la pierre +hoag ie +heel ziggler +gi les +ge to +fossili zed +eu g +di ot +bhar ath +ðŁĺ±ðŁĺ± ðŁĺ±ðŁĺ± +ðŁ¥ ķ +ä¸ ī +still man +redchilli es +pan sies +newh orizons +mouth wash +mi shaps +mad dux +lincoln center +ju mble +here for +dr illers +congr at +chi bi +charlese sten +break water +big dog +aro se +viv ant +rac quetball +pu ffing +plei ades +par di +nar gis +michael phelps +lec ce +has sen +escuch ando +dun garvan +dou ce +de ff +cy sts +a vide +⬠ħ +tr anger +the west +suri gao +sn sw +re touch +re smi +r mnp +peaky blinders +mu bar +mi gs +mi glia +merc ado +koo ks +inu yasha +firec rackers +debau chery +cress well +cat suit +cast a +aim i +y ne +v q +unfa thom +sp ind +si sta +shay la +s ago +one minute +nu ka +n aki +leomin ster +ju iced +institu to +hi ja +das a +co so +chan ia +cav anagh +be amish +atay ulu +applic ator +y v +var ane +uni dos +tarte cosmetics +swin dle +sli eve +run disney +ren da +per ly +p ft +or gullo +on lookers +notal one +jet team +h spa +gu ic +fox boro +exoske leton +earth worm +das co +cu ppy +cro quettes +brook dale +bo lo +b ht +av ac +z sbp +y alla +vou ch +smo or +rak shab +push back +pardon ed +news watch +ic ma +god se +eu stace +er aces +caesar spalace +by city +bun ga +bu oys +al iso +ðŁĺĪ ðŁĺĪ +ðŁ¥ĩ ðŁ¥ĩ +un assuming +studi os +p md +mcclu skey +kq ed +hol len +flash cards +final y +fam erica +f ds +cre dence +commonwealth games +built by +bri xton +bar bac +ago v +å Ł +à¸Ńภ¡ +yo gini +wing stop +virtu alastro +stonebwo yb +statu ette +sc ad +san juan +plo sone +pe ga +op ah +o id +dg m +cir rho +charm ander +bur rard +anti gen +aless andro +ðŁĴĥ ðŁı¾ +west papua +w fm +u mentary +u fm +thereal stanlee +thereal pcb +tham endment +t fp +steep er +raff ling +nicolescher zy +maxim ising +list ers +kn c +ke tu +h itech +gymna stic +goo oooooooo +dra gic +de caf +cor dero +aster ix +af at +visu alizations +v uk +sadh guru +rais ers +par thi +on ah +oc at +nav rat +n pcs +minu tos +kin tyre +fun dy +extinction rebellion +broad side +bro iled +brid port +aard man +ze id +sun sport +sle igh +sg v +se on +ran ching +our e +ol p +mi stress +mi ff +me hr +lym ington +knowledge ispower +ine ke +g dl +cro ker +coom bes +centa uri +ber ber +ann alise +aditirao hydari +to pp +sk r +si do +sch ef +ross dale +redwaver ising +q as +princes scruises +pap ier +ol v +mohe gan +levit icus +dun stan +ci e +cav uto +ðŁĺį ðŁĻĪ +w mu +st pat +sd lp +samsun gg +pu tters +paper cut +ous er +ma sin +kla us +john sen +fictional death +explore archives +esc on +edel stein +dig beth +chair men +ch ert +can elo +cali entes +bath rugby +asvpx rocky +ash mi +as sed +ys d +y aqu +un seat +un detected +twee dle +style z +solidi fy +skull candy +s loot +ren ss +my y +give away +freu dian +fluid ity +fix ie +di ren +de ers +dani ele +d ta +bry de +bla kes +ben ji +un boxed +thir sk +si phon +ra val +park uk +moham mad +mish and +major a +indist inguishable +inbetween ers +immigr ated +i watch +fro d +fav reau +digital media +cra bb +convin ces +bb age +aerop ress +wor dy +thel im +pil af +pand a +mcla in +lieb herr +her me +hart land +grou puk +e wc +compli menting +chur ra +bl unt +bhand ari +andre wr +ãģ ķ +war paint +tu tto +tom ball +spur t +rescu er +rein hold +pump rules +muscle cars +jo chen +har py +gowan us +g hai +en uf +eli an +counsell ors +carcas sonne +af ters +// / +âĿ¤ï¸ı ! +âĢĵâĢĵ âĢĵâĢĵ +shoo tin +qui pe +n inger +my girl +kof a +h ach +fer ris +dy l +choic efandom +us open +ugh h +snet terton +si as +s mat +racon teur +r iting +par roto +one u +of saa +k ich +k agu +i voted +gai den +dog training +dis sement +concor d +color ing +career advice +at ori +aro tti +woo w +votekathryn fpp +un requited +uc sd +thir tyeight +tan ka +sti pp +sear chers +schar les +sandr ingham +sac o +men sbball +jun ko +j ho +fang oria +djash ba +chip set +cap tives +biom aterials +back ing +ambro sia +ald ous +ãģ Į +âĿĦ âĿĦ +wim borne +tr icking +tl ds +q azi +melbourne cup +lord ship +k loss +inti fada +gate au +f ss +edmun d +debat able +civil war +cast leton +bb els +è° · +ãĤ ® +twi zz +tere rs +sle aford +shar mar +ru es +ran gra +pro state +porti shead +pe ga +oz una +mad hav +ino id +happy anniversary +e tten +demoneti zation +cryo therapy +corru pts +bre mbo +ban us +app are +aphrodi siac +al pe +ade t +visi o +tail wind +steeler snation +som ers +rtr naps +ra si +pilip ino +o poty +montag u +merri on +lv mh +lec ter +kan chan +fabol ous +da ad +cb g +bulbas aur +# , +ðŁİĦ ðŁİĦ +âĿ¤ï¸ı ðŁĶ¥ +âĻ¥ _ +tit os +the spi +re schedule +pin o +parroto td +my k +me athe +jou sting +hockey hallfame +hcp ss +guys ss +gri gor +dat adri +dan o +dam son +aren dt +aero postale +a jan +ðŁĶ´ # +t ars +sh ola +se vier +piero gi +pe ma +in undated +heck ler +gu ignon +escap ade +em mitt +debun ks +ann nn +y ve +williams racing +shack led +rn as +reyn ard +per ros +par fu +or op +nurser y +nol te +mac as +j inj +d áil +citi bank +chocolate y +cbs sportsnet +bott i +ðŁĩµðŁĩ Ń +trot sky +tre view +the big +shoe less +s brewing +quar rel +p ellic +longh orn +jou bert +jo yof +ill is +george harrison +g win +comp il +camp agne +beth page +b gr +í İ +zzzz zz +ty pic +sarcopha gus +pre nd +mol inari +lynn wood +luci c +house party +harbha jan +hall yday +gram pa +gos ford +gator nation +endangered species +di ke +cs v +comp action +clemen cy +ca iro +c tures +ðŁĴķ ðŁİī +v ra +us ag +se gw +nh v +negoti ators +mer yl +long island +lgb thm +irrig ated +intellig ently +humay un +har row +har dik +gul bis +gera ghty +fusel age +classi est +charlotte gshore +bar tram +ban ts +ap lin +antiqu arian +all ank +ab harat +!! âĿ¤ï¸ı +âĨĴ @ +sky new +serv itude +ri mb +ra pa +port is +on ya +need ling +magno li +kath arina +eco was +bru lee +bro o +any on +anti microbi +aller gen +wham mer +western bulldogs +star key +spar ty +rheu matology +ren dell +ph un +p out +my o +lo ol +ki yoko +icy cles +hi sham +gener ale +gag non +fitness model +dev ries +con descending +christian sen +cassi opeia +bi gart +af remo +ðŁĺĤðŁĺĤ @ +take back +stimul ant +siri sh +silic ate +rh cp +prisc illa +port ation +pic kings +ph ering +mu ppet +mo tu +lost boy +liveli fe +in ordin +grind house +col bert +ch onews +!! : +ãĥ į +âĿĦï¸ı âĽĦï¸ı +zu mb +ww u +vi bram +tra verse +squ atters +sandy hook +saf f +oper able +iraq is +instru cts +hotb ed +finger less +en ame +cul ling +cla wed +cam is +be que +back splash +apocaly p +Ŀ ¼ +sand burg +resi a +repul sive +queen su +perse polis +om ag +n elli +minor ity +me sen +li sp +la ku +hor seri +ha im +extre m +d mt +am am +ðŁ¤Ĺ ðŁ¤Ĺ +zachary levi +wis bech +ut f +rule book +mel on +ko on +kh oo +k ame +jj watt +imit ates +he ine +ha vering +elk horn +co sproject +aldub big + ¬ +wat auga +queen of +photoo f +paraphra se +mol oney +mcve igh +lap sed +kim soohyun +ker o +jennifer winget +jason derulo +go goi +fish net +fest us +e tam +den i +be eld +ðŁĶ Ń +ðŁıĨðŁıĨ ðŁıĨðŁıĨ +t mm +shar ps +richardd awkins +rev d +rag doll +north port +i was +gw ent +dun away +duff mckagan +br f +as pi +acon gress +war head +w mc +v sb +tec tonics +ta ki +ste pin +slo b +re at +ram m +race forlife +perma frost +ni kova +new age +nb cc +k hair +cy pres +college bound +bungal ows +brain health +bad rin +à¸Ńภ¢ +tu h +street scape +snick er +shoe string +seacoast online +scar l +red neck +pu ddin +post war +normal cy +mobi us +man airport +l hs +krati ka +in el +hom mage +har uki +g wr +fas d +end poverty +em path +ctv news +cho wski +agu stus +ac aci +âľį ðŁı» +tad poles +sw ane +st man +sher rod +scot ties +py m +oster ia +ned bank +ma ar +leon idas +la ssi +jeze bel +je h +inform ations +feliz lunes +commu tes +ci stern +bo car +black er +akin dele +ah oops +ðŁĴĻ ⾨ +Î ¼ +way finding +w oun +tend ons +swi ping +smi thy +side kicks +red start +ra ith +pt w +pre requisite +n ti +mitt el +kw k +hand maid +fren s +boo hoo +bal ti +arte sian +ammon ite +ðŁĴIJ ðŁĴIJ +z ena +warr nam +val do +tu pper +shot show +ru mbo +poe sia +n ha +mp loyed +lion pride +l tg +kaiz er +gru mble +fin lay +end lich +egre ts +econ dev +chlo eg +alo vel +afi b +ü e +zoo keeper +we believe +vers al +ra ked +politi k +om u +n ff +mu sky +kath ak +jack kirby +j ell +iron bridge +in ab +il se +il f +en suite +de ira +change the +blah nik +bin ny +author itarianism +add ario +ab do +wildlife crime +un productive +the shelf +sou mya +soleno id +re surface +pro geny +out fitted +ne mann +lam o +innov ative +g do +forest of +fam e +am ars +admir al +ðŁĩ·ðŁĩ ¸ +wear orange +utr gv +t mann +stur t +sm ita +sit coms +sit ara +shani atwain +rangasthal am +pe dition +lo ggins +life hacks +lan sky +it sli +info tainment +hol lander +go wer +g mat +fore casters +d ack +abre ast +a if +âľ § +wester ville +theat reco +ste yer +sp ite +sam ad +ra sk +ple bis +p vam +lar ne +koo t +kill joys +ig ital +ent z +ðŁĺŃ ðŁĴĻ +world juniors +stre aking +s worthy +s les +s du +read venture +prabhu pada +pan elling +nat ick +li anne +gre cian +condomin iums +cam as +bur dock +be m +ðŁİ Ĵ +ìĨ Į +âĻ łï¸ı +wri gley +van adium +the dead +sti val +steve z +sh ink +saint john +ren ae +pres su +p anned +mat tw +ju ssi +hill song +harrison burg +exagger ating +crum pets +ash leigh +apha sia +ach il +___ ^ +wb pictures +valentine sday +un godly +ru mble +ric he +pun x +pren der +pet shopboys +mp ong +liqu ors +lion fish +ka hani +jan esville +hom icides +gar yline +fla pping +five thirtyeight +empor io +ecker t +bo hm +tab ul +t storm +sw l +starmagic phils +sound city +sof tail +so i +sheffiel duni +re joins +perform ing +oh my +mari anne +lan yards +jan oski +ab original +⾨⾨ ⾨⾨ +us ask +te tons +spani ards +sk elli +shop aholic +post box +poly propylene +or mond +lau der +last man +kr k +f art +eli k +do ff +cli m +cat life +cas sy +af ta +whol eness +wer un +tiffany young +thai food +riot fest +re starting +pill ay +lor rie +le do +inf antino +bi fur +ali gn +ac el +( +) +ðŁĶª ðŁĶª +we hr +ste ppe +stat on +si ed +sher wood +pic ar +palom ino +mp w +me her +mack y +lati sm +home wares +fre und +fin ner +false hood +ero ses +er ster +encapsul ated +en tra +ec am +brown stone +brain tu +bed and +band b +bal ven +ðŁĺª ðŁĺª +veri fying +sto sur +sp leen +scoun ty +ready tor +pe aty +pan tages +pal it +museum selfie +milit arized +ly le +ll sif +gr annies +garyline ker +ed g +ber ne +w engen +toy in +tin en +sky view +r mc +n oooooo +lib spill +leyton stone +jama is +imper man +im min +hall yu +gal es +f si +de ye +bra id +ber ths +bar z +bake house +b ttf +av illa +ðŁ¦ Ĭ +wizar do +thegreat khalid +south ie +pur ging +p ago +mu mble +mo co +mand zukic +kat v +jay araj +gav inde +fore hand +el aide +distill eries +ani el +ali enable +al cal +ak kar +advis able +unil ag +sial kot +schro der +sc or +pe ws +nh p +mon is +md anderson +les bos +kasab ian +ink l +heart strings +freder ic +eh y +drop ship +bian ca +adhe sion +vor one +tumb lers +t reading +poly carbonate +papadop oulos +on this +mer cia +ludo gore +koo ky +klu ber +he mato +gar on +depo ts +dan son +bo seman +ac q +ðŁĺį ðŁĴŀ +å Ĭ +women leaders +wi est +okav ango +mathemat ically +mar isol +jack al +gum by +el az +du is +brown university +biaf rans +ban go +wn cn +w ily +us m +um h +thra wn +sath yaraj +ricken backer +prox ima +por ches +over seer +meri den +ma jum +lt fc +leg ge +kir ke +king z +har low +cor nette +birthday y +answ all +time zone +smart contracts +si do +ro day +mendi ola +hou ma +gu ang +gran dio +dil aik +contradic ts +cardi al +cad rought +breakfast club +* ( +âľ Ŀ +âĺ ĥ +trin it +tom ato +six ty +refu tes +phant asy +perpetu ate +ol c +ny cosmos +needle point +milan ese +goog leglass +gold stone +fle tt +ev ar +de kh +cas ings +bic ic +bi ddle +at ay +ar z +ar rl +ä¾ ¡ +virgin ian +team followback +span thers +siyak eram +shu g +prince sse +po em +mu ka +metro logy +major crimes +la res +la byu +ki ffin +kar o +kah lil +gay pride +g app +fire base +every town +e su +cust exp +af faires +ðŁĴĶ ðŁĺŃ +sec toral +prod mgmt +omni um +lindsey stirling +ki pper +gar rix +freel ance +explo res +climate emergency +bu rak +b com +av eni +air fix +x jr +wan ton +un sw +tur kiye +teacher appreciationweek +sar ay +pay ment +param us +neuro degenerative +kumar vishwas +inter nets +fri gi +dy nia +did cot +de formation +asset store +antibiotic resistance +wa ver +vel ocity +ted by +tat tered +suz ette +stom per +sm outh +rs g +plant ings +ott olen +mel low +life and +lad bach +kat es +infl ate +head in +he ung +fr inges +food banks +depreci ation +chipmun ks +bro ski +ale ister +ac ito +ëĿ¼ ìĿ´ +tin ction +taf rica +sau ro +rio ters +raila odinga +queu ed +out stretched +one time +ni y +leg olas +jun ky +fo il +du as +dah mer +cell ent +bull er +bt posse +as ket +un cc +snow bird +rhin oplasty +oro ad +mala hide +lu ma +la four +king wood +kentucky weather +jun hoe +inter planetary +har ada +fla ppy +ek g +di fc +cool pix +char ade +bl ant +vene zi +sw m +sen ko +samsungg alax +run yon +party poker +parmigi ano +moder ators +me ac +lu sso +live chonews +ken nard +ig in +h mo +fren chart +exxx otica +do err +; ))) +" < +ðŁĺĺ ðŁĴĭ +ó s +worm wood +with hold +vell ore +stan cia +r ma +phil ae +mocking jay +mag en +luke cage +kur d +hearing loss +gau ri +e spor +den ounced +clean sed +cay ce +cash el +boo ing +athen aeum +art station +ais dproud +a qi +ðŁĴľ @ +v liet +tx s +tamaram c +spin ks +small wood +si th +severe weather +ny sph +morning motivation +for lorn +car ino +bul len +b bott +ðŁĴ Ĵ +ste go +smith son +res se +par ise +levit ating +haw ick +go bert +fl s +cor ding +bu ell +bbce astenders +arctic monkeys +angel us +ðŁĶ´âļªï¸ı ðŁĶµ +win stead +vor acious +up coming +tn hs +sof london +on me +o rec +munch kins +li x +kookab urra +hyper car +he sh +gow rie +gen es +film works +dev illiers +daily deals +co pilot +bad gley +alex andro +agr arian +worshi per +vor tex +up loader +tribe chat +tex ash +su che +r dm +o possum +hal ve +fer mi +e bt +der on +chee ky +andre ss +zin da +yard ley +whit acre +u os +twith aca +trophy hunting +sy a +spa ghet +pixels website +ox for +newbury port +mier coles +get ready +flor in +ev ely +city con +argent in +åĭĿ è² +tsar naev +tati an +symb ol +spar c +sou ffle +skid more +sh restha +ru pan +rehabil itate +ratche ts +pp age +pla za +pb x +op ark +ma ille +lilli es +le be +lari at +kapam ily +aaaa aaaa +)) )))) +ì¡ ° +wee i +vibr ator +ta kan +scifi art +les mis +kb tribechat +jant ar +etru scan +esc alante +edu chat +dy che +di shoom +bal once +ak ye +! ðŁĴĸ +writers fest +timi glia +time less +thi op +syn apse +sop ho +sh ula +repu di +pu rab +ori ver +of ford +monster jam +m ph +le mak +incumb ents +har dee +ety mology +eey ore +du du +do van +cou leurs +con served +codenew bie +co ton +cau sal +audubon society +alle gra +al eph +ðŁİ¤ ðŁİ¶ +worl demo +west country +touri st +tost ada +ran khan +plat te +ou el +nz herald +nu a +nouvel le +no th +new by +mo salah +kidnapp er +he morrho +har lingen +gat o +ent ric +dot com +color less +chir u +an kar +ê Ĵ +vijay antony +tt ura +senate dems +mh sm +ja une +hoo tie +han e +friend liness +fre shair +formul ations +fit ri +dr do +dee pa +c int +c illo +bathro be +. | +ಠł +y ello +white man +rust ling +per ley +mush kil +lead ville +just ina +j rm +ira ins +gro an +fu ton +de tt +damn day +d mac +ca hu +black foot +apar ks +af lock +adun ham +ðŁĴĽ âĿ¤ï¸ı +zi pped +sn avy +qu ater +mus c +mal ai +ing am +gi let +ge et +el ich +crow borough +chan woo +bobi wine +a ali +ðŁĺį ðŁĴĹ +with it +win ec +weing arten +website design +vo v +ut tox +sequ encer +sc su +save children +sat yam +sa eng +s ace +ri be +pos sums +non ton +instag rams +h da +dw drums +commercial realestate +auth oring +apple tv +ðŁ¤ŀ ðŁı» +zwe ig +wyan dotte +u bu +the dj +silve stri +sil t +pre pper +pay less +pax west +log itech +lec tured +l tw +kari sh +j dk +h tf +cheng ladbach +anup ama +a field +ðŁĴĸðŁĴĸ ðŁĴĸðŁĴĸ +v illian +tw ars +pro pped +min den +lay am +lady antebellum +ktb ffh +ha iga +e www +delic atessen +chill ico +broad moor +anthropom orphic +ale ks +sef ton +ron nies +plant sci +per ini +pay ton +orchar d +m pm +k ym +jan ovic +golden boy +f ana +dynamite comics +dorje shugden +cr inge +co ombe +ation of +activ a +ç · +you are +ren zo +man spla +kelving rove +kat ara +ham den +furn aces +favor ably +f wp +coch on +Í¡ ° +zak kw +wealth management +touch stone +resi stance +ras ool +private jet +officialu om +lc f +kit ch +keto sis +it alie +hy enas +grati fying +futur ity +b di +ðŁIJ ĩ +ëłĪ ëĵľë +vol ine +tiger s +tamaramc cleary +potter more +on elove +mc fly +m ffl +libr aries +j zarif +irish bizparty +hy thm +hon a +hail state +gal le +fail ings +brand is +br ons +w ry +ve sper +uch el +ta is +spro te +ser ato +save water +re des +psyched elic +miss guided +lin na +la vie +gui dic +green span +gor ing +gold water +fi bau +fan mily +dro go +cr c +chee to +bo son +yu ka +wnu krt +web master +state farm +sp ero +ony mus +mel aye +lou i +i fm +engro ssing +empirestate bldg +dissol ving +boo tle +blue bonnet +b mr +thr ace +ps ch +pic sart +per fom +nuf field +nar rate +may u +l tu +elder berry +dibu jo +desc ence +buoy ancy +bay lee +bald ness +ar ish +ani ght +aj hl +activ ator +z hong +vorone zh +sing a +road trip +property management +ple tt +pin inf +multic hannel +mill sap +male model +ma shaba +lubr ication +loc ate +kin nick +kid z +fe h +defund pp +dd icombe +cu tch +chun gha +bard stown +> $ +year sand +veu ve +unit edin +p ire +oxi dized +or ada +mug ler +mom sen +matur ation +kol be +gl engar +father sday +farm workers +family planning +fact sheet +dilla shaw +digic el +c whl +c sur +c ite +and read +* ^* +ðŁĵ ħ +ã Ħ +âŃIJâŃIJ âŃIJâŃIJâŃIJ +photo contest +n ü +lat te +kwaz ulu +ici ou +hy lton +ghost town +funkad elic +for theday +fen nec +eviden ced +do ce +dige sted +cor sten +back drops +un said +supermari oodyssey +retro spec +oligar ch +mispr on +la as +il logical +galli ano +flash fiction +clou dera +cab oose +brian mb +bon bon +ac ep +ðŁ¥ ī +âĻ¡âĻ¡ âĻ¡âĻ¡ +thunder cat +smoo thest +sing e +si eve +sa ko +par co +muse elou +miy uki +mag gots +led better +it ous +integr ator +grow with +gir t +fal a +day an +arden nes +ag ence +trias sic +samuell jackson +painter ly +nar d +mcm comiccon +mari anas +mahon ing +i est +i ba +hills borough +greg gutfeld +genealo gical +g ys +em pi +de spi +bla ire +anciente gypt +alo vers +ac lu +ä» Ĭ +оР³ +tt x +sy ke +q ai +pad ron +ma her +it so +fa wr +de ee +carmel ite +# " +wo i +urban isation +theli br +s daily +pri ve +my ers +k vm +ic hand +fol au +ext inguish +edger ton +daf oe +aro va +âĿ¤ ðŁĴĭ +º c +yy carts +seab ed +pal acios +me ik +mck ay +mb ti +ma stin +lu pe +love u +le ero +lc ps +ine quity +fastand furious +bund aberg +bu do +ap ort +ðŁĺį ðŁijı +willi enelson +ut el +tri force +teas dale +spell ing +shi h +san down +mer sin +kh at +incit ement +ho smer +ha ier +follow me +do ggy +deer hunting +day sun +d no +christian kane +candel abra +*__ * +te gan +slu mber +scrib bling +ran di +plat ini +n sl +mo relia +mel amine +le ee +kash ima +h nl +box y +audi olaunch +ad ro += ( +yahoo finance +sho ai +out smar +men ding +ma ks +ki ing +jami ro +gn l +gio vani +gain ers +ent rap +duplic ated +dublin town +dar lo +c sh +breastcancer awarenessmonth +baff les +åĭĿè² ł +un justly +ste mmed +smol dering +pd fs +men do +mcdon nell +lat on +lar yn +ing field +hell eni +head z +fron ting +e wen +dur rant +cas s +af oto +âľĮï¸ı # +w cm +verton ghen +ve ils +ste wed +span k +sen ews +selec tively +san usi +retin ol +re build +prou do +pene tra +oo pam +nar c +millenni al +m clo +kat z +inter ned +hot docs +for honor +fitt ingly +fan gio +fal lo +egyp t +constitu tion +chri stos +$ âĢ¦ +Î µ +watch making +wat an +verte brate +sand a +nu mba +mikethe miz +mans bestfriend +leedsr hinos +le sm +kal an +in ah +b han +admini stering +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı +ঠ¤ +غر د +z ot +sup streamers +shab by +pachel bel +mer k +ki da +inte ger +hear ne +goodwoo drevival +cro oner +cl ings +ðŁ¤· ðŁı¼âĢįâĻĤï¸ı +t kach +soo ke +sony pictures +ros acea +psycho logically +pl srt +ligh twood +lib ations +j ony +home style +geophy sics +disco verer +des se +bu ie +bbc tennis +bar ri +astoni shingly +arre ars +and drive +ad vices +x fl +weightloss journey +tyne castle +tran k +sweep stake +sw wap +russ west +la ure +icant breathe +hol len +ha ptic +go ty +fri zz +bay o +b ich +ar au +âĶģâĶģ âĶģâĶģ +vol tron +vis cose +un scripted +twe ens +tor res +tak umi +swansea uni +stav ros +rold an +rev ents +re da +rc gp +ng ong +meteoro logists +marl ene +kil ns +janel lemon +induc t +hur ler +hok age +hend rickson +happy customer +gow yo +dag h +d med +bron ner +trans figuration +topo graphic +the doors +shang ha +sam mam +sa wn +rom ita +re pub +pho logy +pe que +open gov +mul i +laz are +hon cho +ga bel +dun leavy +disembo died +as pa +aqu es +amitabh bachchan +ad dle +ðŁĵ ¹ +âĿ¤ï¸ı " +vik toria +van zant +sre sorts +spoon bill +sp az +sho sp +shau ghnessy +mdanderson news +making comics +kal amata +gri sly +gr g +go g +encroach ment +en ate +dw t +di sch +cu ra +buffo on +bi polar +b inging +ago stino +yo ssi +valent e +tram lines +tal lies +sydney is +sun dress +sci atica +may on +mali k +itt t +har ri +fr anny +drumand bass +crypto currency +community policing +com po +chan i +buch holz +ba hu +apex legends +and l +an aya +ðŁĮĬ ðŁĮĬ +âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶ +your shot +swe e +summon er +sul king +spy ware +si stance +ny eri +mon om +mar lena +m nufc +ky ler +hol z +gre alish +far han +em outh +dil ma +chenna irains +cam ren +axx ess +amy loid +wa aaay +us man +sparrow hawk +porth cawl +pad man +nysph saa +new por +n mnh +n dr +march on +magh rib +mac ie +indv spak +gar son +g and +f anny +british gt +bit o +bam berg +asiap acific +al w +al omar +è ¦ +zu ber +sausal ito +ro ath +po table +monit or +mk ts +mis searth +me ity +har mer +gin or +ei ji +charle magne +bere ts +bel tel +apo gee +an two +am cham +________ __ +stead fast +rot ana +iw gp +im b +disco s +contrac tions +cal listo +be great +an w +a eng +௠Ĩ +walking dead +th ener +rem and +mom entos +m clu +lou lou +kacey musgraves +hol ler +fh saa +det tori +de es +cu boulder +arra ign +ach icago += $ +ðŁIJ Ĺ +ðŁı¾ âĢį +zun iga +widow maker +tz at +sportsp lex +sla w +shi pin +re booted +pr news +kha kis +ka se +gam al +fro ck +da inik +bird seye +tat yana +sw ac +ste emit +mc x +ho tri +fu ries +car nes +bassad or +at ka +an mol +ak g +価 æł¼ +zy deco +z ouk +yessi r +y ck +ty cho +tot ame +tor u +too tough +syl vanian +rat i +nswr l +n gad +martin dale +m wd +legu mes +kha ya +ho je +gan net +eco x +aj s +ðŁķ ¸ +wanderlust wednesday +w ym +tech nik +sri kanth +soul calibur +sof india +ring wood +read more +psychedel ics +oak ridge +na dir +mlb tv +kareen ak +ishi kawa +climate justice +at test +abstin ence +wal low +t pi +sel e +scissor hands +s do +po der +paddy power +next level +kid der +ker alam +he ber +for this +ban ton +b him +am ali +al gore +trafal gar +thir ties +swan se +rol fe +rang oli +ke z +ja wn +go etz +glo bular +face timed +fabi ano +fab io +e cht +cu ticle +chloeg moretz +car io +blo s +bap s +art by +ale em +zi ed +whatyou do +wel ders +water ville +vt poli +tourof britain +tight ness +ten able +some on +sar g +pre sh +me gg +iow ans +french tech +ei hl +death core +da ichi +co com +chan ak +b ga +at eness +a op +( ^_ +ÙĪ ر +yp sil +tat su +taor mina +stradi varius +son ali +pk p +pil lage +or it +he wett +dug dale +chi pper +busk ers +blanken ship +tt ner +sub tweet +sh eu +season ings +ni thi +mn m +l sk +kay den +favor ita +cu touts +coloni ze +boat sforsale +ðŁĵ ¦ +ðŁ¦ ĸ +y ll +y hm +wit tenberg +t cea +steve ston +sci rocco +sal ud +portad own +n pas +lo dhi +ke hoe +k anda +jo en +ivan ov +han h +ge f +anc s +ðŁĴļ # +ye garts +vege mite +ri ofer +ri go +mb fw +mar ki +magnific ently +li steria +kar zai +dar ragh +d prin +chu ka +car as +c ys +be w +ast o +æľ ¬ +tan zan +search engine +ross on +prab hat +pan tene +noti fying +mo dig +micro scopes +lak elife +j hene +fre w +dom o +christmas gift +cc su +cas sie +anc ini +² ¨ë² +yo ffice +tr iti +the street +st aw +set zer +re doing +pun ts +on slow +nv me +night gown +narco tic +n its +hb cus +gi el +farm stead +ding ton +ct vo +cinnam on +chatta hoo +beren ice +audu sd +æľ¬ åĭĿè²ł +worldemo jiday +vie ques +truth fully +tru bisky +to ss +t auto +st event +shu ll +s ff +pol arity +off ing +must do +mc cand +magni fied +jor am +gab or +f ld +danny wood +c mm +aurobin do +wood head +women ssoccer +the state +hi roy +en suing +ac lan +w aga +u wp +u rian +tar u +swer ving +stou ff +shi mizu +pun ite +psychop ath +pap aro +n sca +moisturi ser +khand el +fu gel +fro th +cocon uto +ca in +bil al +x v +tootough totame +succin ct +scot twalker +ray mon +pom o +pluton ium +phen yl +nd f +music inc +marble head +leg it +l si +kan wal +jour n +jesu sis +gall antry +freeform tv +fo aming +fish burne +crink le +bur a +blu em +bla q +ak wyz +adap ts +a ip +trevor moran +ten i +sl ate +scuf gaming +re printed +quad ro +polar vortex +ot ol +marou ane +m fr +is this +har ker +fra zetta +fon daz +ed c +dc traffic +cor ky +cal las +bulgar i +x pro +write up +wat che +w ds +veterin arians +tuil eries +toy photography +th apa +staf fy +sha i +seng illibrand +pand an +oo ts +grocer y +goog les +fitch burg +eye on +dre es +di zation +di eng +de vise +d sl +book keeper +bean bag +aer ts +² ï¸ı +take da +stre as +sal lah +royal mail +memor izing +mar lee +m sport +inner peace +i wc +ge en +fighto wen +dra iders +de spon +chou dhury +can tar +al ola +ðŁİħ ðŁı¼ +west indies +wan stead +univ of +smi thers +sel ene +rose bank +red necks +na hyan +l pu +invasi ons +independence day +har bin +gri ddle +eri ot +ell wood +dul ous +cover reveal +bel mond +beach ed +architecture photography +al stom +¦ Ī +un rated +tu u +tri met +simon pegg +remou lade +prover bial +over berg +normal ity +mor ticia +kal ra +fuer za +fightowen sfight +diver sions +ðŁĮ¹ @ +ú l +vo igt +tal on +scu ri +saw tooth +pen ning +n sm +medi alab +m ty +elisa betta +eer ste +cute sy +ceru lean +adap tability +âľ ´ï¸ı +tumble weed +thi en +tex arkana +tar ch +t fb +st aked +seduc tive +rogu elike +oste opathy +os good +eye patch +ami x +> # +ðŁĺĤ ðŁĺģ +ðŁ§ Ł +tan ager +sun way +poly com +jar musch +j anta +iber o +gian luigi +er win +dit ty +compre ssing +brae head +bo cca +ap hor +ac ey +yeah that +wa il +vap ers +tyrann ical +st augustine +saf ari +raj ni +min to +medi acom +me ander +human ly +ga illard +fro om +dapp led +cum bia +cro co +bol le +bi ght +an itta +a stru +us ana +twit ty +tun ers +pe h +no es +ni mr +ll f +jay cee +it en +ik is +hur led +et r +e pics +dro id +dis land +car play +briel arson +ar w +anti septic +ai do +Ï Ģ +thelance t +thel ong +th ongs +tart an +sper fect +sli ghts +ser ling +prie to +mu sso +khal ed +kareenak apoor +in uk +fe res +fan atical +boy y +ban jar +( âī§ +ãħ İ +wim p +ti gris +ta ye +propag ate +pl it +pin ar +ph vote +parab ens +on ces +moo i +mark down +m fi +li fen +ko smos +ka v +jhene aiko +hello bc +ha qq +girar deau +fe ster +el gin +de q +de ena +ar agon +amazing race +ëłĪëĵľë ²¨ë² +wu thering +whid bey +tam pico +sh ami +ri day +out landish +itiner aries +gu ha +e katerina +cul vert +chronic ling +car dona +because itsthe +reci di +re evaluate +pal u +m fp +lun enburg +kr one +hyper bole +hydro carbon +fr n +fir kin +def er +bowhun ting +bird gang +ðŁij¸ ðŁı» +ठ¼ +write tip +tv shows +sag arika +ray sup +l so +hic cup +ha pe +gu len +din wid +but lers +brock ville +bang er +b lec +ad h +ðŁĺ Ł +ðŁijĮðŁı» ðŁijĮðŁı» +wf sb +we ft +twer p +stri ping +scrupul ous +sab in +ros lyn +rev ving +rack ham +ra ph +mp as +m hi +kor ver +kar k +jewel led +heath cote +dv m +duma guete +door ways +detroit become +cru soe +connoisse urs +c we +brit tain +blood pressure +blemi shes +repent ant +pi ana +otta wac +open follow +ndam endment +la homa +kicau promo +ju ste +invis align +inthe house +indi anews +ibg drgn +gri gg +gene seo +force awakens +bode gas +ayk royd +âĦ¢ . +ye eee +ulta beauty +u ol +tomas z +ter berg +tam ina +stepp en +sam m +q g +practic um +oro x +kri stie +inter governmental +in ce +g anna +dom ino +di yar +dar ter +cr amp +cad aver +bru beck +bo tu +yin ka +spaul ding +shun ter +shir di +sam ia +recidi vism +r to +pi quet +kis i +karli ek +jordan spieth +ip sos +dr x +datsy uk +cute eee +critic schoice +bernan ke +as inan +ak ery +soren son +nucle o +ne ice +mis read +max is +le am +ka ise +gur r +fre sca +cat oftheday +birk dale +wi h +w ct +vi aje +vere in +spice girls +soccer am +sar am +reflex es +kam insky +gu tta +gli ano +dick head +depu ty +as r +air con +yu val +sec gen +q af +ob in +nv wx +mellen camp +kenne dys +k pm +j ini +i mas +ent wined +end en +dyspla sia +dl m +book love +bon appetit +wahe guru +vacuum ing +tt ler +tra ut +sun moon +ro sado +new stalk +ko bani +janellemon ae +ine os +houston chron +historic england +geo technical +gag ging +g tg +do ren +cla ses +as ai +ap ride +super valu +stro ker +si mm +scar le +quo it +morbi dity +lam ma +jo anie +finger board +ethnic ally +eson oma +el ab +dou bly +attor ney +ðŁij ´ +viol ent +multim illion +l pr +k norr +fac toid +dou la +de valuation +de ker +aw allace +ðŁĩµ ðŁĩª +year sold +vol up +ver lag +tar avi +stre eter +sh bo +sclero derma +rabb in +pro x +mary kay +lic ht +l pn +ka as +j ame +haz al +gru ppe +coun table +bri gg +ah in +way a +walk for +vi dic +the dan +tar nished +stand for +semic olon +phy sic +naz im +n cb +mun ds +kre uz +i league +hy m +gli des +fees mustfall +dr ano +dis av +daw lish +classic car +ch olo +w tg +tax day +spec ks +sen feinstein +salom on +pe ppy +par kh +new ssydney +mul k +liber alis +le apt +ke di +heav ing +grand pas +gno sis +fiftyshades freed +edg ware +digiti ze +cel t +carnou stie +aa v +âĻ¥ @ +world photographyday +tack le +swane poel +sound scapes +sole watch +rise ofthe +pe dan +oregon standoff +newyear sday +mir th +kratika only +kil la +je ann +gir ly +gate ways +gall eries +et now +eli ana +dark seid +cur bed +chim amanda +bit finex +bell atrix +and company +an engineer +a beer +y fn +vit or +vag rant +su u +ryerson u +retrac ement +pal az +mtvlakpop bts +im l +gour mand +dim sum +dah lias +as li +ari sti +areth afranklin +am alie +ac r +wini fred +ven ky +uni q +un known +over done +nofilter needed +mu mm +mark sman +lin ares +land care +kel lie +ferran te +eng i +cigar chairman +y k +show grounds +sho ts +seren ades +sal inger +roman ov +resi dues +refre shes +prop els +news mornings +n mp +monte bello +mat sui +make money +lady bugs +inter scope +house plant +fugel sang +fu hrman +du que +dil ute +co ss +c atie +bar stow +abu ela +a beats +wut ang +warrior nation +v tc +uk an +trans vest +super stock +su won +su ll +sto cker +stgeorge sday +r pl +pin head +kul u +kareenakapoor khan +je wala +i ffe +heck en +glamour mag +gan ay +g ous +et an +el ric +disper sion +coch lear +bb p +analy tic +amal gam +af cf +_ @ +å· Ŀ +ud all +opioid crisis +neel am +mo j +methan ol +lang s +ka oh +iv re +ho z +he bri +ash etty +as port +arson ist +.. : +zi ja +trac ted +solemn ity +samo yed +rapha el +premon ition +pa ver +mis demean +karliek loss +jor g +hann i +ga unt +fork lifts +ee en +death wish +cw p +celebrity news +car ty +brook wood +ae gon +we bbed +polit o +ph ari +miami oh +mel le +lind gren +kath arine +ja qu +it down +i em +gradu ate +gluta thione +gardent ags +fundament alism +eu l +ck less +chi ri +br w +bla k +bayley wwe +a pres +ðŁİĤ ðŁİģ +ìĸ ij +un limited +super fund +sun id +rough ness +pam pa +on em +nong mo +ick son +hospit alization +geophy sical +fra ge +folk life +fo ley +fi z +fa wales +ek h +din ar +cc ps +ben ard +ar up +zee bo +var da +tou ting +space suit +so aks +sm rt +si gil +sam aj +potam us +mi f +ludogore ts +lu z +kit ti +jacqu ie +io logy +har tz +ex hu +estor il +duchess of +cake decorating +animal rescue +adobe summit +yo ver +work house +wil lo +stand point +rockef eller +renss elaer +pres stitutes +n ene +maison valentino +ken to +hail storm +fre era +disapp rove +co bourg +ar q +and ru +ê³ µ +Ï Ĥ +sel wyn +ocean view +i book +donat elife +dani ell +da hiya +cas sin +be em +wwed aniel +wwedaniel bryan +ta pen +stan den +rainbowsix siege +pan ip +noir alley +new brunswick +ja ir +ill ation +d ous +brad bery +bar kha +ane jo +ag ir +winter hawks +thyro id +out did +ni vin +museelou vre +lalun asan +inhi bits +anti gon +amazon kindle +un tested +thunder struck +stor ks +sp age +run the +refur bish +qu ed +oh y +law and +l me +its bayleywwe +ibc show +gr ü +dr ams +denomin ational +b fm +ale o +ðŁĴ Ĭ +ðŁİĦ # +ãĤ¹ãĥ Ī +wn e +who dey +un credited +tu va +syngent a +su ther +sax on +resu mption +po gi +pen iel +on wheels +magi karp +i ñ +harid war +funinthe philippines +frame less +fer vent +eros now +eradic ated +doo dy +di av +ce ci +br il +ber ge +are i +ard or +ar sal +ani ma +alton brown +ðŁ¤ĺ ðŁı¾ +wil ke +vi aggio +vas il +un sold +studi op +speed ometer +ment ly +lli ott +jan ata +go bucs +gar ay +g tp +fract als +et p +en bach +ci do +be bes +arc teryx +ðŁĴ ı +wha aat +ventrilo quist +vel azquez +un likely +umich football +sober ano +sideby side +sick kids +r bb +na hh +mustsee iran +mam ta +luc ina +ll in +li pped +j da +innis free +hb p +franch ised +fen nell +ever deen +deci bel +chom ping +cau s +bon illa +al pur +âĻ¥âĻ¥ âĻ¥âĻ¥âĻ¥ +voucher codes +ree ks +rand l +pot ash +north yorkshire +n sb +limon cello +lah ti +la stest +ke o +hat rick +freera if +feliz martes +extremer ules +contracep tives +cam illa +bru ch +bri bing +........ ......... +⾨ âĿ¤ï¸ı +whe c +spectro meter +rter adio +pe stle +ma am +k war +je mison +jai mie +j mm +iko yi +hav ant +gu tting +dis banded +compe l +but tocks +at birth +am ann +su su +sk en +re pos +petal ing +pechak ucha +ordnance survey +mis spelling +kit ak +ig uri +hal ting +gou ging +evis cer +es and +dept vetaffairs +con man +city v +boer ne +blooming dale +b md +astron au +ðŁijħ ðŁijħ +wwe hof +win ski +vg punite +ur b +un usable +u pan +lar gs +hf player +gri p +god ley +gad ing +co whide +weigh ting +ste gen +st ling +somerse thouse +sh um +sa kai +s é +pic h +mul o +lt gov +lan gui +kh q +juli o +j aga +he ep +fig ma +epide mics +daman sara +cold well +chro matin +chic hester +arbor ist +amo vement +agent carter +ìļ ´ +âĹ Ģ +Ï Ģ +tzat ziki +tas ker +in appropriately +host ilities +gar r +epic a +disc world +cra ps +christi ane +caf és +tipp ett +team b +strou dsburg +rose water +pu ig +n psl +mick jagger +lik able +ig inal +ic ata +huawe ip +guid ing +fil my +fer c +fearthe deer +embelli shments +cho e +ch il +blue book +big band +am sey +alli ster +\ _ +the sam +steveaustin bsr +spiderman homecoming +so bie +rich field +pr ato +parac ord +o skar +non profit +lar val +im at +daniel andrewsmp +cly des +blen heim +ban ne +avin ash +alad ha +ðŁĵ¢ ðŁĵ¢ +âĦ¢ , +zombies quad +wor f +vas sil +sc ada +profan e +over t +ny man +man zoor +li vers +ko taku +kin go +glit zy +fre igh +equi valence +env agency +en sen +en liven +den ouncing +crest view +cl ere +city beer +chanc ery +atle tic +ag ang +wra paround +v italy +to remember +stun ted +slap stick +se phar +op w +newyear new +may ak +li ev +ker sh +fou st +ef g +bur ge +au vergne +ðŁı ¹ +ëłĪëĵľë²¨ë² ³ +xen omorph +uk iss +to ic +rifle man +phal lic +per turb +leg ac +isu ke +hyper baric +hall er +elg ort +don an +con fox +car ron +cab lecar +baby daddy +aby tes +wh nt +war zone +treasu ries +swal e +scal zi +pes ach +p ous +norman ikor +montal cino +mid gets +kourtney kardash +jo dre +hy person +hi di +her by +har ra +guj ran +gu ppies +family first +f ach +con ical +ch ace +ar mie +ид е +soli dly +shay na +segw it +rough necks +may im +macro economic +ler oux +len adunham +k tg +in z +hend rick +gui mar +furry fandom +dest itute +z aire +sun fish +ro za +rac esonoma +r po +not withstanding +is chool +hydrange as +ham ra +generale lectric +fon te +fe w +fairy house +comp tia +bet as +bed head +b drm +ag ini +aaron paul +ðŁļ © +ðŁĶ¥ âĿ¤ï¸ı +ðŁĶ º +ॠIJ +w golf +te ater +ren ter +re so +qu are +parti ality +lac quered +krasno dar +go cu +dyna sties +dublin city +cle ef +ban try +advent u +abhi gya +a hai +) / +ìł ķ +yarmou k +wie der +wal est +u zo +tetsu ya +success factors +schwar tzman +ne bo +me ttle +mc man +hol kham +gandhin agar +esp nc +e tro +bo st +bel u +av ari +ðŁij ľ +zombiesquad hq +wal dman +wa cha +v tv +sh hhhh +russi a +ra itt +prest wich +normanikor dei +mar zano +lu as +liqu or +lig and +k ck +k anna +insta video +gul shan +ge oc +fright en +dr tedros +die mannschaft +cheese board +biom imic +an ze +ag oo +vi b +tu le +tele pathy +sd j +raaj jain +humber to +go ku +fourfour two +den berry +co weta +bur go +albin ism +ac án +vasund har +to bin +sol ange +sof tens +shadow run +pro mis +pre fix +paralle l +p fd +more ira +larcen y +kann ad +j cw +hl m +her ne +grand in +gi deon +g lynd +fertili zation +fc w +dun woody +deserve dly +by ung +bar king +ar har +... $ +wu shu +vital i +too ooo +s theatre +ren ditions +pto sis +par ley +o ja +na x +li vi +jetair ways +indian railways +hand washing +guidic elli +el ottery +che sters +bern ini +b cr +art fully +al mere +ab leness +zimmer n +westand together +new vision +new belgium +mit zi +lux ur +luc cio +ku mble +is aw +he sa +gom vfc +gh today +french montana +cro sley +ce b +caro winds +bal do +b ps +adder all +w gm +ve era +vai o +taur anga +ta shi +spi king +sch winn +or ica +of o +jay am +im patiently +final sweek +fe thi +fa wsl +cu pe +cla shing +brett eldredge +bookof mormon +ap al +ac tress +ac amp +>>>>>>>> >>>>>>>> +âĺĢï¸ı ðŁĮ´ +ื à¹ī +vest ment +swi pes +step mom +ste dman +sr n +sa warenessmonth +redchillies ent +pri zm +pal ais +nd football +lea ver +in paris +i pos +give blood +else vier +el lus +de ann +clu ck +ber wyn +!!!!!!!! !!!!!!! +view fromthe +un founded +to da +tin ge +ta itt +od ors +mu di +money ball +max ie +herpe tology +gad i +fortw ayne +drey fuss +dissatis faction +care rs +adel ante +âľĤ ï¸ı +è ge +ten ement +ssss ssss +snor lax +sc ymru +personal development +ma dero +kor d +ki shi +it re +implo sion +fu x +eque stria +ei vissa +chic ha +å ¨ +Ä ĵ +tu bers +tin a +the ads +tamagot chi +sy k +sun n +sd ar +sammam ish +rene ga +prun us +ponti f +pier rot +oak mont +kur i +joo heon +howard stern +he o +ge stapo +gas kets +gab bard +conven er +brazz ers +bol ly +bajrangi bhaijaan +" :" +ðŁij · +wi dened +watch maker +vit ru +un ico +rough ing +poly phen +m the +led by +in tv +h vy +end is +cumber land +cn traveler +choreo graphers +chan te +bar nier +bar di +ë n +to ting +pa stry +o hi +mis eric +m venkaiahnaidu +kum amoto +kis sme +kin es +iq rar +h gh +envisi oning +c acci +bla ss +beauty tips +ather osc +x cel +the standard +sou q +ram er +pregn ancy +periscope co +n scc +mariok art +lass ics +ko shi +k itu +flo yds +er ma +endor phins +e aux +de compression +ðŁİī ðŁĺĺ +ðŁħ ±ï¸ı +unfa zed +te ak +set ti +má laga +mur ch +let go +is n +is awesome +inf lection +he cht +g bc +fra pp +combu stible +cali ph +black mirror +at last +ðŁĻĦ ðŁĺĤ +view finder +us af +un ta +tommy hilfiger +so wer +rug ged +nav in +lu ks +loyal ty +hs football +harry and +gal eria +fic titious +fair hope +croche t +al ver +ðŁĻĥ ðŁĻĥ +wf mu +tel enor +tai wo +sb w +sam pauk +real saltlake +reac tionary +progre ssions +makeit count +is om +inst an +ib h +holo grams +ham pered +game grumps +dole zal +di wali +cutie pie +ch ung +cal low +be coming +ag irl +ae gon +welsh man +wel ove +weap onized +slaugh tering +repre ssive +q m +optic ian +milk man +kazoo ie +indemn ity +imit ated +form aldehyde +chlamy dia +car radine +bur gun +am ah +è į +اÙĦ ÙĦ +ú s +vent oux +thel aw +the bridge +swed u +sir pat +s medley +nov ate +nectar ine +muir head +mam adou +ii um +glend al +fon ten +eas ley +cre sts +col wyn +chelse ac +cart wheel +bead work +a jac +z official +un seasonably +t singh +sul fide +steam ers +ra uf +predic tors +phe v +lg c +imo gen +ic chio +hallucin ations +dev ry +crow bar +bri sco +bonafi de +ball ondor +b wv +b hr +att as +war horse +v ss +un due +teletub bies +state less +pom o +ohio state +offici ally +no ddy +n rel +mu layam +mix radio +indigenou speople +hu q +golds miths +fin ales +f ti +dra ven +corn eli +architecture mw +å ĥ +âļ ł +wi j +vietnam war +th acker +sub set +sk g +pun x +pre made +ox bow +op cw +mo ba +man an +mal lee +kra kat +inf on +hu tto +en car +ann eli +aer c +wick i +un sweetened +tar dy +shil ton +q ts +per ri +on ite +official corkgaa +nf hs +lit mus +lan ie +kul ture +kend al +ja emin +is not +innovate uk +gran ite +gavinde graw +for far +diss enting +couple goals +br ano +bill ingham +bi ffy +whitney museum +wh d +vadachen nai +tin ned +si bility +sea ver +re cast +pru sa +orthop a +multi dimensional +mick foley +k ory +it am +haifa wehbe +diar rhoea +chau d +brun el +ðŁı Ľ +ðŁ§ ľ +wallo ps +un helpful +thecar ousell +tex press +terpen es +stereophon ics +sh ic +quanti fying +play ball +nh art +lo so +keep calm +ine s +in coherent +heat map +hayes grier +grizz led +glycer in +gin u +ex changer +auto bahn +am me +am fam +ag lobal +ðŁĻıðŁı¾ ðŁĻıðŁı¾ +work ington +under scores +su ju +sti mu +n ft +mus lin +mani sha +gran ad +forza inter +end cancer +eli est +der bies +by law +bran agh +ben assi +at man +asapo fficial +v festival +tt tttt +to ps +su mo +si ak +rin ne +play test +p ba +om ission +na ke +mu ji +mu i +major lazer +holocaust memorialday +h any +guay aquil +goodday atlanta +forever alone +disco gs +cro x +con cave +calde cott +ar ap +wind surf +teenagec ancer +reali gnment +rav elling +ram adi +rall ye +qu id +prest wick +milli meter +jud icial +intric acies +fear n +dur ch +bur ly +bigart boost +as part +ãĥĥ ãĥĪ +Ê» i +tr ous +slovak ian +red land +nev ans +mi ggy +m pp +je y +haw kers +edi th +eccle ston +despic ableme +deir dre +ad sense +... ðŁ¤Ķ +. ðŁĺģ +ภĵ +à® ´ +vi ation +the usa +sirpat stew +sad at +re ylo +psycho analysis +po demos +pap aw +ncaadi ii +mc cool +may be +her z +give syou +dun blane +cnn brk +b xl +ao d +american express +aliab hatt +ìķ ¼ +Ú © +wheel base +var und +untouch ables +thu d +squ awk +riofer dy +re assured +r ted +ph p +pan do +o dot +late ch +la hiri +l sm +ken zi +eh lers +e ko +dro ves +copernic use +be tten +be ti +ame e +) _ +vibr ate +to pa +tel o +son nets +seam er +re aves +poly gamy +pan zer +milit arization +mandal orian +h ingham +elfon theshelf +dom ore +contamin ate +con s +ash ar +akam ai +áµĴ áµ +à® ĩ +zach arias +wy re +undul ating +thank fulness +star sand +r mb +need n +n our +minig olf +march of +m ings +je u +fr yday +fau l +eamadden nfl +door man +com passion +biker ide +ay yy +ðŁıģ ðŁıģ +ठ¦ +wad dup +ug lies +ti aras +ther ia +soul cycle +s bli +ran cic +psychi atrists +pis o +p ve +p tx +nu it +mar as +jama ican +illi am +holling sworth +heir looms +hallmark movie +emir ate +chimpan zees +sweett ooth +self made +ru mpus +pl on +land es +laid law +ken ham +insp ira +hh m +cook stown +con ran +cm ll +clyde sdale +c mas +berli oz +bel mar +apal ooza +antwo ord +ale wis +ت ص +ó n +westh our +v tec +un wrap +snoo ping +she es +proof read +pride and +om eters +mani p +king sville +impacton pop +hi biki +fal un +evo ice +ash ura +aer ation +a ana +wheelie wednesday +warrior s +w enders +v fa +up side +tech summit +sli ms +phil hecken +l mu +kun s +kon da +kol hapur +g pd +embroi der +demetri a +car dy +blue jay +bay es +% ... +ঠ¯ +ze alo +wend i +v ats +unforgi vable +ms ft +min haj +m hockey +kab an +ju kes +dracon ian +damsel fly +bhagav ad +bernese mountaindog +tosc ano +ter rap +te vents +te an +sl n +share able +restaur ateur +ok cupid +modern family +mei sel +mcal pine +keerthy suresh +haiku challenge +fire dup +fer i +ail sa +adop tion +wies enthal +what sin +ugh hh +te jano +tar ry +snat cher +premi os +peter man +p mot +nch saa +me ter +kit out +gre inke +fa had +dispen se +dd l +bru dda +ah lul +z uko +wh er +way s +tid ore +staf fs +ron de +mon olithic +lolo lolo +leigh ton +kaoh si +hersh el +foo s +cha sh +centri sm +cen sors +boo z +bloom ers +black top +athen ian +you saf +sun dried +stouff ville +sj ws +renais sance +re don +ra gan +plun ger +ma bles +josel ine +infu sions +hig ham +heart soup +ee b +e bike +de wine +comer cial +b va +an ons +!! "@ +ðŁĵ¸ | +ãħłãħł ãħłãħł +ö l +y aaaay +world smileday +usc is +th as +tem po +taravi kapoor +stereo scopic +shus wap +sco ff +q aw +over stock +oppre ssors +mil d +hi aw +h ous +eviden ce +ek taravikapoor +dream scape +dar an +cle burne +cla shed +ben ig +bau sch +amer on +aly son +afri day +adel son +ac og +abi des +ðŁıĥ âĢįâĻĢï¸ı +âĢ¢ ) +waron drugs +visual studio +vis ors +theart bond +th usi +swe st +sp hd +soy inka +sor lando +sadh guru +ra isa +pe ds +ozz yosbourne +occupy gezi +lo ke +ke ast +h wasa +fashion tech +em ichael +ec rash +differenti ating +collec t +chi st +bart shealth +b dd +ase an +ag ü +z ang +wales coastpath +w ten +thresh olds +tatoo ine +tan king +sch ner +rock fish +pro bono +photo gram +ne ye +nar gis +marie curi +gau d +co fe +christi ana +berk ham +bebold forchange +be ery +b chl +archi ves +ar bys +ãģĭãĤ ī +power bank +pas ko +nefer titi +nasr allah +maur ici +mary se +mah jong +j he +it aylor +free ze +fatal moves +dispat chers +buck horn +ate day +asi r +ar bour +á´ Ģ +turf grass +tu gger +track nation +topo fthe +toen ails +sof ar +pic hu +ong ar +na seem +klin smann +ip w +impro perly +im parting +good will +gat tuso +exer cise +escu ela +de jong +bridge tte +bloo p +abull dog +âĹ ĩ +௠ĭ +w bs +v elling +tu pp +spun ky +sl fl +sin f +rad he +p tero +martin truex +mad u +le mar +ka ws +je ta +it na +fu jian +cruis enor +boot sy +at la +astrazen eca +al ani +triple j +tri z +resc in +proye cto +omme gang +make rere +ko en +har iv +excell ently +cab rio +be han +ba shes +ðŁijī ðŁı¿ +the smiths +the scotsman +tak oma +sath ya +r ri +pu bg +pi anists +patri c +p bm +nov us +nj wx +mcke on +mang led +m dot +kir ti +kar yo +informat ica +his lop +gu apos +fren chy +franco ise +for business +extra dited +deci phering +cremat orium +color ation +ble ecker +bell is +ðŁĽ ł +zi o +un saturated +sarato v +ridg eline +rare bir +r sherman +orti gas +nu h +metaphor ical +m gc +jab ra +hom os +ganesh chaturthi +foam rad +de ac +d br +bbcle eds +ar ran +ðŁĴ¦ ðŁĴ¦ +z ama +vol beat +ur ple +thick ening +reduc er +pp able +nw f +ib g +geo de +form ica +dis ra +amir mateen +adar sh +' '. +ðŁĩ¨ðŁĩ Ń +whis ker +v ars +sunny deol +sar fraz +run nings +resi sters +pi relli +mustaf i +mus ici +mc combs +liken ed +i ep +hu u +hover craft +home steading +halla day +green beauty +f adi +calis then +box e +af aso +á´ ı +wunder bar +whatyou love +var nished +v aw +toom uch +st marys +pro sen +pere tti +mu so +mo bo +mistre atment +lo dd +kno tweed +im magine +hydr ants +he se +girl scout +german ic +bu cci +australian labor +addi son +v ru +v pc +s fox +repa ira +po wai +paranormal romance +o tero +no so +ni pper +men ko +i op +em poli +cory monteith +col ts +best place +at kin +ç Ī +woo s +west man +w bay +ve ti +un regulated +the blaze +tan ja +schri stmas +power plant +pocon os +pleas ure +pa sts +official csa +nabe el +me iner +lo ath +li pids +kom atsu +iy engar +hit list +em cees +car lit +arron dissement +anky lo +al bac +afric aday +ठĵ +waxaha chie +transfer able +titi an +tar des +sus ann +shel bourne +sf mta +pizz aday +per ris +paren ts +or ting +op tion +nyc council +more to +mill ington +hur witz +eve online +dayofthe year +daes ang +da bears +car nations +ar ae +ðŁĺĺ # +ðŁĺ©ðŁĺ© ðŁĺ©ðŁĺ© +ãĤ·ãĥ § +wra p +vari o +the mighty +tas mani +so go +sierrac lub +neu ve +may ed +ma demoiselle +lu ne +k lux +hoe down +glene agles +di ga +dee dee +d cd +co vers +black thorn +bass line +bad ass +alife time +æĿ± æĸ¹ +âĢ¦ ' +wild man +u pre +t are +stroll call +soor aj +smart watches +ro tors +radi ocity +q army +mi res +mi ac +me deiros +mabu hay +krau thammer +jaz zed +in ny +hai der +f dot +eli ka +chal ky +celebr ant +з аР+vas cul +sli braries +sas ol +pyl ons +pom me +pal pit +ol factory +missing person +kell in +inf j +fantasy land +er j +beÅŁik taÅŁ +b st +aw aaz +arraign ment +wood ville +su priya +seri o +sapar li +re va +palm sunday +keen um +gri eg +eye health +east sussex +dol or +derers fc +dand ridge +cor gan +contr arian +comm ited +blan ton +athei strollcall +am mon +ac te +ðŁļ¨ # +ðŁIJ Ģ +ãĥ ľ +à £ +the dream +scon ce +sc orn +ow c +n agu +mort ars +me sti +massi mo +lat or +johnlewis retail +j world +islam ophobic +ho ey +glu tes +ed avis +duf field +boiler makers +ba illie +ac ampbell +ðŁĮº ðŁĮº +wonderfu lindonesia +u iowa +tor bay +shi geru +sare coming +pu edo +im printed +homeof cricket +gh alib +drou in +di di +aprilfool sday +ðŁijĮ ðŁĴ¯ +wool lim +to lex +synchron ous +rag gedy +quarti er +penetra ble +new post +k sdk +h cr +char ds +call out +ca ip +agu in +zo ë +z un +you ve +urqu hart +thunder clap +single malt +sequ ined +mornin ge +mor tis +lu ga +lib or +li si +ken net +kell ys +jail house +iqrar ulhassan +ham mad +grey joy +gorge ous +gi el +delle mc +cun ha +?? ?) +ðŁĮ Ŀ +val lab +v é +sub surface +sl its +person ification +perfu me +olympi a +mik lan +kel pies +jo ko +h kane +doo h +dialec ts +cruisenor wegian +connach trugby +buc key +an cho +ðŁĵ£ ðŁĵ£ +âĤ © +zak at +wur z +westco aste +vac aciones +tel fer +sch ou +roll tribe +pinstri pes +on ine +musli mah +leather back +la ziest +i drc +fox la +fis cally +ever last +euro vision +ela ar +dream league +corn ing +carre ras +bu ech +bru den +bridg north +brek ky +bat ang +acu ity +wu z +thur les +thor ne +shaw k +shak o +se gel +res ins +rajapak sa +quint in +purp lish +mcgra dy +k sl +jam meh +ilo u +escap ism +e well +e mar +defra govuk +configu rable +bu dur +blue bloods +becauseitsthe cup +aw en +ashraf ghani +ðŁĴľ ⾨ +vehe mently +v ries +un se +tu tte +stan wawrinka +ro lo +re home +rashi d +pre side +one pride +ole k +n sh +mug shots +mu ggle +ma sia +li ot +ju lexis +helleni stic +ham let +gr ating +forfe iture +fat one +divor cing +de porte +british flowers +bor go +bb ca +an sa +ÃŃ guez +wor tham +tr w +tele phones +sur plu +sque aling +sab c +ken e +hu gg +g ath +face mask +elastic search +daysun til +cr t +cheese burgers +cast elli +busch gardens +bu cker +bhu tan +amber alert +all llll +accom od +ðŁĺİ ðŁĺį +ðŁıĭ ï¸ı +womenin biz +with her +wa el +tech talk +orang ery +ne ots +im plant +il ani +ha gi +fool ery +donat ello +do sis +de frost +co host +christmas jumperday +cham pa +break age +al so +academy awards +ac iam +war hawk +se ys +rodol fo +pizza express +pec toral +mon str +mag ome +itz hak +iam stevent +geop ark +fal do +f fier +embelli shment +electric vehicle +deliber ating +de mille +cat ello +ar yo +% " +ðŁĺ § +ðŁĴ Ĥ +world bank +willnever getover +uro vision +tribu ne +thu b +sky lights +re et +ol use +mccoy dl +mc bride +ly la +john shopkins +iceland air +i vi +ge tac +gam ut +fu fu +div ing +descar ga +ctv morning +corrup tion +clint bowyer +char on +bull i +boney ard +blind fold +be ca +ax e +artist ically +af ra +ðŁĮ IJ +um kc +ui uc +ten or +tam sin +ski ff +si ar +plo v +photo shopping +peter head +pe des +online business +no bis +mum ford +je bel +ilove sydney +gy ros +ema baras +di mmer +da hyun +co xon +as ds +af ton +word smith +whit by +up with +tuf nell +pi en +nr j +kak is +hr va +ghos n +get me +gar of +ft x +deta il +demp ire +bomb shells +bnppari bas +ben chers +abbrevi ation +åľ ° +woe fully +way ward +shizu oka +sche ese +qpr fc +prosecu tions +pc g +ori enting +nanop ore +mon ta +limous in +la ther +kry stal +hor ology +h ks +geo sciences +delhi police +alo gy +waz a +upperdeck hockey +shar mila +se pat +rock fest +ran ji +pd illon +musa shi +mumbai rains +mi zz +mccand less +mahesh wari +ma vin +fol lett +de young +con figuring +cbc ns +ðŁĮ ķ +ye i +what sup +vivac ious +tru tv +tre sp +tagh euer +t veit +pat oran +nswr fs +mu mps +mi reland +leed sunited +gau ze +efferve scent +de mont +cham plin +canyon lands +boss lady +bo az +at noon +⼠ªï¸ı +vel u +u wa +ter a +sportsc aster +sme g +s loop +pininf arina +pe ori +o ire +mur r +gla de +flood plain +esri uc +eco logically +cier ra +car bo +bu stos +b dr +un wrapping +ta stier +shop online +sch lo +sau dia +run r +pod squad +nm pol +moccas ins +mix ta +mi shaw +mam oru +jin der +gin obili +f kin +ep as +bu z +bel tr +bar ran +av ance +age orge +zx spectrum +thor o +ox en +noble man +mobil ised +min ato +manche go +il lop +hel ix +g local +etch ings +energy transition +dj b +dead beat +cul p +cor ps +cheap ly +carru thers +big boss +b ny +ar ray +and u +ack man +ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃðŁĺŃ +tsu basa +strong women +river bend +reale stat +nvi diage +n abc +me dea +ko el +kefal onia +kar una +invo icing +igu o +i ola +ham strings +hah n +fresh man +flat mate +ferre ts +di ppin +dez bryant +de regulation +crab apple +bo va +ari ki +ðŁĵ Ģ +ä¸Ģ æľ¬åĭĿè²ł +à¸Ļ à¹Ģภ+vijay fan +t du +sli dell +si won +serv ing +sam os +sa ker +riti ka +naw ab +mer le +jar rod +james gunn +i anc +hiro shige +hil t +green sburg +gilli brand +gibber ish +fire and +fin ch +bing ley +bi di +bati ste +» , +widen er +wash ere +wait list +traver sing +tooth ache +sr r +spir aling +space port +ra hane +prithvi raj +power line +plumme ted +pien aar +mo ven +mac i +lat oya +kil ly +gu thealth +ex olu +emp ir +ec ake +direction er +ðŁĺįðŁĺį ðŁĺĺ +z ta +y eng +ted der +surfri der +stone street +micro organisms +mar ra +man ly +lovel i +lee jongsuk +labra doodle +hy at +heen an +har v +gwyn ne +galvan ised +alam os +a ami +ë¬ ´ +toby mac +theno ise +stronger in +selfie with +se edy +sc alex +saurabh raajjain +q ash +new girl +mccut cheon +madewith paper +lo le +he ure +fet tered +el it +dur in +de cs +coffe elovers +chil tern +cal aver +ble ach +ad block +ac wri +worri some +wi than +vic enza +ushu aia +under lines +tax able +tanzan ite +t elle +sa ith +l wa +kr ant +j anc +harne ssed +hadri answall +h isa +gw p +fam itsu +ex tran +dux bury +d aci +cow boys +air tattoo +ag io +acqu ittal +south african +ski jumping +shan gr +postp onement +plo s +paro dies +nr dc +mar z +lynd hurst +le ón +ki ele +jo res +hb k +hb g +coconuto il +car bone +- [ +yaz idis +vijay an +urban sketchers +sub traction +na je +lin dam +gran it +courage ously +chillico the +carpen ter +barn stable +agu y +wo y +ur r +t sun +star ting +ro ssy +prin cely +pin c +mak si +luty ens +ker rie +ke tones +it was +isab elle +ilove dn +hubb ell +euthan ized +eid aladha +ci h +chapp ie +ce ans +anthro con +! ðŁĺĥ +zar ry +wb ff +wag amama +travel er +thi st +tbl st +sleepy head +shau kat +scar ry +qu orn +porter house +nu ck +mu gen +let splay +l mt +jo eye +hereford hour +d row +bunny men +bra ham +blow fish +biop la +bee zy +ba char +am na +à© Ģ +yemen is +y ola +tr b +sur tees +sc af +sa quon +sa j +quintess entially +pertur bed +p mg +ottolen ghi +no fthe +new ness +n elli +mar ish +k illu +g tas +dari o +can ales +adi ge +a thology +wich it +vos ges +sweet corn +shear water +schul te +sch uh +ruleof law +re played +land ing +lampp ost +hyder abad +fo co +en listing +electro pop +d fm +buoy ant +bewit ching +bethen ny +aryn go +ac offee +a world +un cool +tl chat +ther u +swin don +pang asinan +ox ing +ne stl +mp s +mic o +mb it +let them +ho ge +fore sters +fl aring +divin ely +de ano +bron n +bl m +and az +ai ai +ty us +tn bc +st j +snu bs +shun ned +see ya +rodri gue +pi ppi +mish ka +horo scopes +harb ors +gge tt +game over +fl gov +emp r +eldr itch +clemson fb +ch ri +cate rers +car ine +ban zai +authentic ated +ano ka +ali d +alex oloughlin +adam schefter +! ?!?! +were ld +wa il +rock it +pa stic +le by +kel pie +j ln +fe tch +d afc +ðŁĴ · +ðŁİĤ ðŁİĪ +z ild +wolf hound +wa st +the talk +sar geant +no reen +mom mie +jor is +hen ch +doctor s +com ida +colo red +aggreg ates +⼠ħ +âĶ ĥ +w ll +to ka +tipper ary +super villain +shou se +sach sen +practi sed +pi ons +pen sa +nk la +iq baale +infan til +huffle puff +homo gen +her ders +haz lewood +ha q +flyair nz +elg ato +du chy +demp ster +day na +d aga +cp f +biele ma +ald ana +mass y +leather head +kur z +i ar +h cc +gra sping +gan nett +explores ask +ðŁĻıðŁı½ ðŁĻıðŁı½ +Ù © +vox el +vig go +tw ay +thunder cats +thru way +star ke +shepp ey +plat ts +no val +k pr +john cornyn +h sf +gu ppy +fitzro via +fe stuk +enfor cers +de con +chall ange +bodh is +bet ch +beach comber +ðŁĶ ¨ +un corked +terr aria +stone house +slee povers +she th +puls ating +pan os +p of +julie plec +godal ming +ever more +cli pse +cal les +bio blitz +bi olo +bi ggins +bang tan +aw nings +ali stair +îģ ĸ +ypsil anti +wtn h +we ver +varun tej +si willnevergetover +san fran +sab athia +pers and +ogil vy +ne opla +nand it +man no +lac a +kar r +im plantation +i big +elin or +du bb +ar dro +af ca +y ric +west cork +vic ks +valeri o +trin am +scep tre +rusty rockets +osc uro +no daysoff +kvy at +khu x +ic ha +ho ge +haz are +han eda +gugu dan +fr mr +fireup chips +fer mil +fay sal +brit ann +bran stad +ben del +art inez +am bar +ak wx +w smv +tow ski +ti am +tam many +step mother +re dre +pas chal +n fd +motor happy +marion ette +litt lec +keeping it +gri eved +gl r +fit a +f of +coloni als +chevro let +canop ies +can ale +ad z +ðŁIJ Ĩ +ðĿIJ ¨ +ye aren +ultra book +truck load +to tt +t gr +sx c +sick o +scru bber +sant ander +republi k +post mates +pic stv +perovsk ite +most wanted +ml g +me k +labor dayweekend +kir t +ka f +heart breakers +gym khana +fl ange +es rc +d pd +bush els +ban field +ðŁij¨âĢįðŁij©âĢį ðŁij§âĢį +ula ganay +then ational +sp ly +shari ah +sh fly +radi ated +ordin ances +offe l +mar bling +link up +like me +ke effe +k tn +jack daniels +her m +henri ksen +hen shaw +han sa +guaran teeing +gi ster +fin minindia +dest abil +dental care +dad ar +d tg +bc ndp +archite k +à® ° +vincen nes +spino za +sold ado +sale sian +mal uma +lyn sey +hit s +famil ie +c ingly +> :) +âĺĿ ðŁı¼ +tam ils +photo shop +pe art +palliative care +mar tie +lagun itas +knuck lehead +j pop +inf er +ep am +cor ded +che mise +cal atra +blo ons +ðŁĺįðŁĺį ðŁĺĺðŁĺĺ +z ima +ww r +tour n +t sub +sinu log +r na +p ith +marj ory +listen in +l gus +kil im +home ostasis +hal lett +guil foyle +gri g +go har +ging i +fre ts +ca ked +bul b +bike packing +band anas +ally son +ag gro +wag oner +v te +urban i +tu tus +too faced +the musketeers +ten et +spoo fing +se es +san fl +qu ash +prism acol +per ton +pe ddle +mk z +mer cs +kon ia +inci dental +gimm icks +fur r +erkenci ku +dra wn +devil s +cont in +burkin afaso +belaf onte +ðŁ¥º ðŁ¥º +wit ting +stef ani +sat com +ole h +kru l +joshu atree +hy omin +gest ational +black bear +bird watching +bai lee +are al +ak as +ðŁĩ¾ ðŁĩª +ãĢ° ãĢ° +yaku bu +ty n +thin nest +sand bach +p msl +oo da +me so +master node +mahin dr +la ferrari +ice hogs +g pp +for ney +brant ley +bl ick +ari za +al il +ðŁĺŃ . +ðŁĺIJ ðŁĺIJ +ðŁİĥ ðŁij» +wicke dly +w fan +thecomedy store +ta pos +sensu ous +se ul +rol ly +re sent +pon ta +pleas urable +on el +military history +jhan si +je wski +infl amed +god like +eic hel +dres sup +dist al +cru d +bun gle +bol sa +boiler maker +as ch +abo ys +à© ģ +zam ani +ur man +tic ino +thegood wife +thecat reviewer +taylor caniff +sri vastava +river boat +random actsof +pen e +pc sos +or zo +ny ias +ny ers +montp elier +migh tier +la autoshow +ku li +kal ani +ju ta +in comprehensible +i believe +hunger ford +gam boa +fore sth +dee gan +day break +cr ouse +carp fishing +can i +boy zone +blo ating +bil as +bharatanen enu +an ac +amo red +ðŁĸ ķ +y kj +w mas +ti red +se apor +ro i +pe tt +op kins +multi sport +ineff able +in capac +i ji +fat ass +es lone +document ary +clip board +ani mas +ang sty +am ou +ag irls +---------------- -------- + ¼ +ucl an +the witcher +that awkward +shut tered +sen ergy +pres stv +por scher +mis aki +ke zi +inser ting +ha veli +e ap +dispo sals +di keh +da egu +cur rants +cro pper +charol ais +c una +as f +yorkshire day +villa iny +ve ga +ruben stein +rack space +pr onged +pon i +ow ings +nikol ay +ne sh +mu mu +mck end +ma ka +ly s +kis sing +k offee +jon jo +indi ec +incon spic +fl atten +ex h +essential oil +ec are +dock ery +dis illu +d wan +coer cion +^ ___^ +super gt +sne ij +real mike +par taking +michael b +man cha +lak me +jab al +green s +doyou know +dis ation +cur tain +cat skill +believe tour +amalgam ation +ag itated +un registered +sl at +sha ho +ou thouse +mel in +ku stom +jc poa +grenfell tower +es ar +er yn +eff lu +cau very +c sa +âĸ Ī +ur is +traver tine +thevoic eau +teen top +pry de +ple isto +pan tries +pack the +or r +ntv kenya +mahin dra +kar los +juli us +ih ana +harle quin +gi fu +gen stein +der ails +coted azur +control now +bi kin +bc ferries +bb ca +ðŁĺĶ ðŁĺĶðŁĺĶ +ðŁķ µï¸ı +ðŁĮ§ ï¸ı +zulfiq ar +zero es +wester nu +vit iculture +ver don +teen mom +sun oco +snat chers +shorth and +po real +net z +me do +mark ruffalo +lo kesh +led ley +knit ter +insta story +fishand chips +fat i +di mon +bk b +artstation hq +vis cous +rehabil itated +pin kett +not meus +ne scafe +j ka +gareth bale +co ch +austral asia +ale ssi +adi pose +a ou +ðŁĻı âĿ¤ +à¸ļà¸²à¸ Ĺ +viz sla +tw n +to pi +schwar tz +scep tic +red is +po sto +pieter sen +n sa +lu mi +favour it +de throne +cy d +youn gh +tan ked +strick er +strategi zing +sho pee +raj ah +nic om +march ant +kam ra +joong ki +is ur +ipad games +i fu +green space +gen cy +e sport +din c +cru k +bis sell +bec ca +ay t +aq ours +andy bvb +ðŁİ Į +Å « +win ked +welling borough +twitch sharer +seg all +re cht +p tt +or dia +open text +matthew daddario +last day +kat sucon +just ins +invo ked +dk pol +cac ophon +bre yer +beck en +az ar +ars ène +ari o +ageof sigmar +villalo bos +torch light +saltlake city +pon ty +piece of +nw l +ne gev +l ente +l ci +k wok +e gh +dise mbar +cor sage +consi sten +charli ec +bow ness +blo at +bel tand +antic oup +ðŁĴĵ ðŁĴĵ +shul man +sho ver +sha shi +retire ment +remo tes +re wa +r ci +protec tionism +ob d +mr l +mountain side +mit el +let stal +jami emc +hil de +hey its +german wings +flex es +bush el +blu dge +and hi +age ism +ab owl +whi ley +un reported +ter prise +tam pines +som bre +selfie for +sd mf +sciencen ews +radi ators +periodon tal +ny la +nascar hall +mer z +mechan ical +made of +i wish +e akins +capi strano +angel ine +ai ja +?? !!! +ty ra +th aroor +seat ac +red men +red fored +pl b +modal ities +li ms +incroy able +hol lows +gu ing +flann els +cu miklan +chel yab +bjö rn +bb its +ðŁ¥ ŀ +will ingham +v tech +trek kers +tr al +tel cos +st wee +represent in +red hot +nepal quake +nanomat erials +minn elli +lau ter +kon ta +kail ua +jo cks +hi eld +fanta stique +fal ta +echi dna +cler gy +cl oned +calatra va +be prepared +bat ts +bak o +anch al +à® ® +yu u +tennes see +ta chi +snap matic +seam stress +sab lon +rie del +ri kers +rakshab andhan +qu asar +nach baliye +morecam be +ko koro +epi k +di marco +day sout +blum house +a vision +ìĹ ´ +ãĥĥ ãĤ¯ +vish alkofficial +tho th +s girl +ro zier +painting warhammer +observ ant +nie wski +l ous +jan ey +itch at +ge bouw +gam elan +fin nan +day star +dance moms +crou ch +city police +ch up +archi medes +am aka +alien ated +world vision +usa af +tuni sie +re constructing +pag li +over seen +n lr +min of +il ish +glu m +f pd +et ze +e bu +de ft +conqui st +bc storm +bag ue +al te +ah our +ãĥ Ħ +zi en +work life +va ing +ur inals +tren to +sou ness +shopper shour +provi dent +new ood +le sean +le hill +iron wood +ic ab +ib t +ia af +ha pless +gar ag +fibro ids +dishon or +day challenge +curve ball +crow ther +cn f +cataly st +bal to +à Ł +vel aik +su ey +si ed +shav in +sant as +pro prie +keen ly +j ima +extre madura +differen ce +cham an +á ħ +zor an +y ari +x w +wh acked +un diagnosed +trade marked +shrun ken +sap ling +revel stoke +mu la +marqu and +man irat +ir reverent +international yogaday +hai kyuu +gl or +gh ari +done channel +cau k +awaken ings +ventil ating +t list +su wan +sam ir +rain drop +quat ro +pro fi +polar bear +normali zation +mcle more +mb as +math ru +mar ino +man ya +maic hard +littlebig planet +intermedi aries +i hm +her rick +hel les +emer ge +consequ ently +ay az +allga ier +vote themout +visco sity +tu mp +ti ber +tar ps +take uchi +t illery +special offer +shop now +parmigi ana +parisi enne +o sso +minneso tans +micro chipped +merci lessly +kaw an +kar li +indi gestion +in vert +han uman +hali k +guar dsman +good tobe +ger ani +ero dgers +emer cury +desp ised +cu pof +bir n +bbc wm +b town +ðŁIJ Ĵ +ws g +wi thered +who vian +vel d +thal le +t ach +sub standard +stock dale +sal ar +oba femi +g ine +fal ter +disal lowed +book blog +bl v +awesom en +âĢĶâĢĶ âĢĶâĢĶ +yellow jackets +woo ow +water brew +un tapp +speed test +sin action +sch elling +sa ic +ru pi +re fried +line sman +k se +fi v +euphe mism +do bie +bur go +anupama here +åĪĨ ä¸Ģæľ¬åĭĿè²ł +vol te +truste d +states men +pre release +pag ina +osp rey +op reps +oec d +north view +int ol +how ser +haw i +evapor ated +cros by +cor rer +cine mark +bur ling +biom es +bhag wan +bedand breakfast +b union +aure us +am z +⼠· +wre kin +winter green +walang pasok +traf ford +tn ick +sun burnt +sf jazz +remembrance sunday +r tn +pi rie +nor n +new ts +la ika +knock in +ju mi +fertili zed +f ns +el ang +change over +canandai gua +argon auts +ur s +so cin +ske wer +sioux sie +since rest +saif alikhan +jav anese +fe sto +e ren +dog gos +descan so +body building +aid s +yu to +tab ata +stry der +se red +scre ed +phon ie +phen ia +o er +mach a +jeong yeon +jen nette +in saf +gloucester rugby +da igle +bollywood news +biop rin +bay nes +autumn watch +í į +æ° ´ +ãĥ ½ +usc ap +tol u +the score +the brave +over used +myo cardial +ku biak +graph y +fast net +eth ylene +enjoy life +da ikon +chir p +be inte +ur vashi +tro cks +shi ki +se id +pri ya +pasteuri zed +ma en +lu gan +le ann +ho ddle +g we +fran zen +dyk stra +car ding +bum rah +berk shires +bed spread +ax ton +afremo v +к а +we ire +se my +ro sin +ra es +jor din +flo pping +affin itweet +abse iling +; ). +& , +ðŁĻĭ ðŁı» +ys f +wake man +teign mouth +syring es +sy p +se kar +sav ind +po co +panther a +orient al +myo pia +mind less +med twitter +man se +log on +lis ch +ju wan +inter active +integr ated +ha pha +gigan te +en compass +d ce +cr ane +col ate +chil is +chelyab insk +bridg it +ashwin i +alt press +ðŁĻı ðŁı¿ +yoak am +woo zi +vets get +vetsget scanning +spin nin +sa ida +reptil ian +pinot age +ok ayama +man hunter +kle pto +jarry d +ip sum +ii it +hi rez +ger rit +fr ill +euro millions +c mm +adic hie +ðŁĻı # +ðŁĺ ¾ +ë ł +thur t +theli ght +stra bane +sko kie +sk al +rid ley +re introduce +pro ge +paul ding +open studios +li se +la velle +go stars +ev ille +ename led +corpu schri +clau de +cir que +cin é +child like +bobs burgers +bi ersack +al jaz +wen lock +then ame +tele portation +taste the +something new +s je +ricky pdillon +py l +ox in +om kar +nau ld +mile sdavis +law al +kha bar +ke mang +jor die +homo sexuals +euro stat +d do +cor rhi +ala qsa +ðŁĨ Ĵ +transiti on +ni ge +mx n +mam iya +m ki +kings ford +he yyyy +formul ate +dar dan +d reading +cb sphilly +cashi ers +bra ver +ater alism +abbo ts +] ' +á ¥ +trick ery +ter me +spi sil +space time +simple mente +sc ac +ru sa +ra za +ore tti +mon ico +max imo +la ia +holt by +ham ann +er is +ener gie +duf ner +cha eyeon +canap és +ab t +vo re +thread less +storm watch +shre w +re loading +ph leg +lewin sky +iu fb +gel ly +cross ley +am iller +al pert +ว ว +zand voort +worship ful +woman sday +wg tn +ultr amarathon +ty la +tat ty +supportour troops +numer ic +nom en +newsp aper +net book +meridi en +magome dov +leav itt +islam orada +flouri shes +cook off +convin cingly +choco bo +camero ons +bo ggy +awa ke +allfor one +ai fe +à¸Ļà¸ Ĺ +wel comeback +trit ons +schoolboy q +pew pew +mor onic +mail day +laure us +k sr +jer maine +j rn +gun nery +ew b +be is +ap ap +ðŁļ £ +ushe red +swild cats +sp here +secon f +ra jan +pun y +pi po +ma ffe +lesp aul +lar naca +il orin +hu guen +hate m +for mosa +chri sd +c jp +bla zing +barak at +ah t +aga th +ac cts +` ` +ìĶ ¨ +wester ners +villa in +un responsive +tu scu +sof ia +sli ther +sh mu +sen doff +quarri es +ninj atur +jaff na +jacqu es +intensi fication +il ang +gu b +glad bach +ero ding +env agency +elec t +ci pes +chat el +ca ñ +btsx amas +am h +aber nathy +ðŁĹ ¡ +ta ze +sthlm tech +stein meier +sing led +sal ta +pleisto cene +pix abay +mel k +ma theny +m hu +intere sting +ever green +eng l +em sc +cz w +amir kingkhan +" @/ +whats next +the empire +swith out +ss as +say onara +save d +s women +rite sh +repleni shed +po il +pecu li +lu vin +in or +ic ac +e os +cryp tom +cra p +ch evening +bristol uni +boe heim +bewil dered +zano tti +ye wear +tre acy +tc w +scu pp +sant amaria +rc car +h gc +faith less +do como +ch ug +cal oric +bureau cratic +anth apuram +yeg traffic +wellcome trust +we ve +vac aville +ump iring +son unig +road america +qu itter +pic story +pag en +oak en +miro slav +masterche fuk +lou se +lon eliest +har ney +gal eri +ga shi +fi fam +eti ha +d tr +bab o +abi ola +. $ +wit ching +wi erd +warner bros +tou rette +seung kwan +refriger ators +post al +pics depot +par an +mega force +m golf +le imer +l wb +khe dira +je y +ha asf +gra hn +gerald ton +en actus +eat drink +cat en +brat z +bian con +b ge +wha dd +w so +the food +subbar aj +sty lo +secre tion +r do +michi el +ko b +hay ashi +haw keye +full screen +dinwid die +de fund +cullin an +cloud less +bl g +ãħłãħł ãħł +wr on +weight lifter +team dcs +te ve +student s +sr ry +san key +pinec rest +mu sta +kas ama +jan ath +fren z +forthe win +ev ga +bath time +auto zone +all out +after work +ðŁıĪ ðŁıĪ +z epp +tempe h +siob han +sho hei +rez nor +rarebir daler +peter sfield +non binary +na hl +mer gency +kar sh +gu er +etsy handmade +din the +crock ery +cri ss +broad sheet +black town +balac lava +athin ai +" @. +âĻ © +wor rell +wal le +stpi india +ro mi +rel ink +q ty +pent land +ngin x +mo xley +mar ten +mar cos +m sla +jar row +in ton +huff po +he ave +flow ers +fc cincinnati +dr kumarvishwas +brain injury +blue planet +bed ded +ann u +anag i +âĢİ # +zen desk +win dle +ven ue +the color +tg v +t bex +st pauls +san dow +parish ad +of cal +north coast +mujahi deen +mu mm +miit omo +lin ce +ksd knews +kat ona +jud waa +jou le +im all +heart ening +head canon +gou rock +golf course +fan photo +er of +dro cks +dime bag +david lynch +cur itiba +az ov +aft union +" ] +âŀ¡ï¸ı : +âĺ Ķ +wil lesden +wg tn +trek kie +sx swedu +sta vern +sh f +ru mbling +re counting +pot ence +mo stra +luch alibre +lower show +kab ila +hir ani +gavin newsom +dur r +chicag om +bi ma +ber ni +bab oy +arri e +ðŁĻĥðŁĻĥ ðŁĻĥ +اÙĦ ÙĤ +y pj +vis itu +v pl +phil omena +per pend +ot sego +on zo +n bi +metabol omics +mac ri +ll n +kap i +ju ries +indiscri minate +hosse ini +hal vor +goldeng irls +gargo yles +f bu +equ ates +du moulin +chi ma +char is +al derson +âļ½âļ½ âļ½ +âļªï¸ı ðŁĶ´ +ww wwww +vick ery +u lus +u eno +spol itics +sol aire +so hio +sar ang +saf ter +protec tion +peabo dy +mol lie +l fg +kail ash +k kriders +itu ate +ingle se +hand sets +drag way +car pedi +car ib +az el +zag ato +street z +ps g +pl atten +pal las +mapu tra +hoo ky +flame thrower +elly wood +dir tb +dig ans +cli o +chal ce +cap ris +book oftheday +am ra +am ite +wor thing +wis c +who soever +v bc +tom eter +stac i +souff lé +shoul dered +ship wrecked +sethu pathi +sch loss +o dern +jess a +ga fe +g cu +dar gah +ab dou +ðŁĴĽ ðŁĴĻðŁĴľ +women and +vern is +tam asha +re pr +rarebirdaler tuk +poiti er +live install +le ly +kla ssi +kha i +d bd +cl in +cann ery +birch wood +beck er +at ara +anamor phic +american horrorstory +tur bot +thunder dome +ter remo +size able +r fd +mus ics +mask ell +l bl +kap adia +gam bian +car vajal +bl unders +aker man +wi the +tembe akenya +si mar +ri jiju +q w +press o +pr ins +pad dlers +morgan stanley +mali ki +lin ne +in j +history museum +dat af +cou lis +cor onal +compla in +be gum +amo e +ai me +a hier +åĮ Ĺ +wc i +ve ster +uni vers +uil state +ser kis +r dg +ohh ill +ni gg +ne sco +mccla ren +li zar +k het +indi annavy +his ense +fi ably +do ze +digiti zer +cardiff city +aw ssummit +as ami +ãģ ¿ +âŀ ¨ +xk cd +w ounding +ste ff +ni shi +mun ger +mil ks +manife sts +man ju +front lines +fro id +fri ended +fall colors +du san +dream cometrue +da iries +d nc +п ей +zer man +tomor o +the aviators +sing ed +shelby ville +sch wab +port lao +plov div +pi pa +over coat +muk ka +marke table +ma zza +k lum +is as +ginu wine +fortun er +dur g +dri vel +claus ura +cardiomyo pathy +brisban eroar +bo des +bak ery +b dl +wo donga +wi shi +utr ition +ther ain +seman gat +sali h +po catello +p â +olympu suk +oc kets +inci sive +fas cial +dr dre +cushi oned +concert photography +cathar tic +carol ina +c ze +bunde stag +bas se +ban on +b fr +b afc +altrin cham +westcoaste agles +victor ias +tar aba +rock smith +rizz oli +pe cking +my lan +f pi +dazz ler +dav es +conjun c +cal f +associ ating +world book +safe ties +s fo +r jal +novel a +monash uni +mi do +meth amphetamine +luci d +le mond +k bo +innis fil +glen core +du sh +buck ingham +bo ssed +bhu shan +bb cdp +atlantic city +ac kie +ðŁ¤© ðŁ¤©ðŁ¤© +çľ Ł +wasps rugby +us fs +ur umi +o vid +monster hunter +ma aa +kant ar +jer om +i photo +ho si +beefeat er +a ef +ðŁĩ¨ðŁĩ º +ðŁ¤ Ŀ +water stone +vit ili +tail gat +squ es +pro phy +mendel sohn +lazy sunday +kaizer chiefs +joh o +illi ps +hau gh +ha kan +fortn um +dro bot +do wager +do ily +cfis d +canon photography +bry son +bri erley +an sible +am persand +tall is +summer holidays +sof icial +sa wing +river bed +q mul +po zo +onen ation +nah j +mon davi +kop p +h db +columbi a +co burg +bar rell +bankholiday monday +as sn +ang ara +andy warhol +amend i +affili ations +absen ces +ðŁĴªðŁı¼ ðŁĴªðŁı¼ +âĶ Ĥ +win sor +victor ville +uk awa +tor sion +tau t +sport sin +ski ed +shab ba +sab in +popul ace +nir up +mari elle +li mping +kaohsi ung +high ly +h fa +forç abarça +flam in +don is +de iro +character ize +cay o +b he +ano tha +ach o +ðŁĺı ðŁĺį +wind proof +uw g +up lifted +un tv +shum pert +os oph +mr kt +mi amic +me dev +maser ati +levit ate +jack s +gr at +fru tti +dee ks +cor oman +apar k +íĶ Ħ +âľĮ ðŁı½ +yor ku +tra fic +sur charge +stro gan +ster o +ste acher +shad ers +ru tte +roadto omaha +po dge +ou trigger +o wo +narrow boat +mu ggles +mo hun +ket chikan +kawar tha +j cm +ine scap +iguo dala +hang gang +gra do +colle n +cn h +cla sse +cir co +car afe +boon docks +ðŁ¤Ķ # +wheel house +out crop +nith ya +ni hil +m per +gad v +d rick +cowh erd +clear view +catalo gues +bit torrent +af el +yo be +wrong ed +u maru +tom s +stat ler +sle eds +sal uki +ropp ongi +pi ston +not so +na ilers +heterogene ity +gra sse +eddie hearn +cle e +bau delaire +yellow ston +vintage fashion +ur mston +trevor noah +ti veros +tar yn +sul ley +qot sa +pay ne +no wak +nevers ay +n zo +medical cannabis +magell anic +keepit inthe +ing rained +in l +im penetrable +i fr +for u +flouri shed +em mam +e steel +ba ikal +asse tt +app development +al con +aj ag +zo on +v anya +she kar +schizophre nic +pp pp +new books +mon real +might ily +man bij +lau trec +ke eling +ho omans +god smack +d vi +bo gut +bene detti +au ma +apprehen sive +ðŁĴģ ðŁı¼ +woo bin +sof ig +shepp arton +sa ira +pro t +petit es +ms gr +maddie ziegler +fu zzy +fu kin +f te +check lists +c fe +au der +animal planet +yas achi +wan n +ti poftheday +subsi stence +ss a +scrat chy +mor oni +kle z +estre llas +eg f +creed ence +consi dine +candy man +bull frog +am ash +ðŁĮ ¤ +Ø§Ø ª +y assi +spri ze +sick ly +sab s +ro we +ro sco +plumme ting +pi ven +oooooooo ooo +ni rav +na bel +moo cs +man ors +loit ering +lib dem +gi acom +erik son +den bigh +cur cumin +bi har +ar ob +ac cc +¿ ï¸ı +wall st +tri but +suscep tibility +sgo fficial +re solves +pig gott +p end +mb aa +john nie +job sin +iri global +inqu inn +gav askar +fl amed +es ss +ecc mid +di acon +detroitbecome human +c cio +bronco scountry +berg kamp +" ðŁĺĤ +yach ty +survivor ship +st itt +s open +pt sa +mo er +lec o +kgw news +hel la +grand national +ett one +esthe tics +dent ure +corn meal +bt f +beat plasticpollution +ani sh +amand ak +ðŁIJ¶ ðŁIJ¶ +ta pa +so iled +shou trts +nol a +no sh +n wan +love me +li zz +gene sis +gas ser +gag ner +fre donia +fa q +euph onium +dissi dents +dev t +col angelo +cirrho sis +cas sia +br ack +ap ink +adverti ses +пей заР+yo lo +u is +tay tay +sub titled +par se +ny ce +no ve +mor zine +mar imba +mac eo +lanc er +iam beckyg +i know +h ml +go lic +exoner ated +eco chic +dis agreed +cy non +bruden ell +ðŁijī ðŁı» +win sor +tv o +tul la +tranqu ili +town home +sport media +sel ig +sebasti en +sa jj +resili ent +re joint +r hum +oneminute briefs +na oki +ke shav +kam chat +kah f +jeff lemire +co rel +bo gen +... ?? +ðŁ¥ ļ +wish in +wild ness +we bbing +wales online +vent ana +trash can +spor tage +na it +ló pez +irr fan +i ago +hl hockey +had dish +fug itives +free stuff +f bm +ey ck +bi mini +anakar ylle +ail ment +acadi ana +aal borg +tae il +stylish ly +saran ac +republic an +linke din +l na +kirk ham +gor ams +google edu +getty sport +ener gi +ding ell +com eau +co pes +climate march +cal ver +bro d +as mara +ab j +ðŁĵ ¬ +าภ¡ +пейзаР¶ +w dsd +vaing lor +stap hy +soo m +rangra siya +ra bb +prie bus +pre f +pessi mism +mega store +ku ku +joey mcintyre +in fact +harsh est +ha x +dont drink +dock side +differenti ates +dalry mple +constra int +buck thorn +ami ya +ä t +yeahthat greenville +world animal +tri ppers +sav ills +quer cus +psal ter +pow dery +parine eti +over took +oscill ator +nor throp +ni igata +ms x +mine field +liber a +k att +h gs +gyne cology +glori fying +for tin +elliott wave +ech r +denver channel +b mr +as cott +ab ul +vu du +visit ca +te ve +tara strong +stone haven +sm its +show a +rep tour +rayy an +puni shes +pol dark +mu laney +mu cky +life s +lever ages +land mine +kick stand +ka aba +ig u +gu shes +green wall +gr ans +go sforth +fair ing +dies es +chape co +chaffe e +britt ney +blo bs +bat wing +av entu +angelique kerber +ì ¿ +اÙĦ ØŃ +yellow tail +stu ous +sorren tino +sc ular +pura vida +piggy back +pav an +past i +on gress +o hr +more e +laet itia +je tz +id week +fal kirk +emphasi ses +arig ato +abhin av +ìĨĮëħĢìĭľë ĮĢ +va isakhi +trinam ool +take ak +stre tton +slim line +ski ppers +ro on +ri ends +prat ap +o log +m wr +louisi an +lok pal +infalli ble +fan fave +en dish +embrace the +distor t +de wa +cher t +bull head +bry don +auto parts +åĽ ½ +wor li +tex el +reign ited +quir k +qu ila +procu red +pick ling +pan athinai +o he +nigh trts +mall ards +lob bied +lady like +kid sin +hh of +h mh +cro ft +close out +cl k +band z +agra ha +ad ela +ðŁ¤¦ ðŁı½âĢįâĻĢï¸ı +é ķ +woo die +up l +thorn bury +thiruvan anthapuram +ther mals +sh ames +se un +ren u +que ers +post natal +panch ay +mcg lynn +keralab o +jump ing +jon i +its janeoineza +glu tin +flo rez +everyday sexism +ee ting +d cre +cher o +centr alia +centr ale +bon k +beauti fying +am usic +af ern +ac ure +ðŁĺĴ ðŁĺĴ +un gu +thrash metal +pelican snba +mo ley +mis am +make over +loud speakers +lifelong learning +joy sms +ima gem +ic ho +hu lu +hover fly +foo duk +financial times +conven es +bo gan +ajith kumar +ac cola +un capped +r ws +pav illon +new job +nay yar +mccle lland +lan ter +kar n +go iu +game spot +ey yy +ero oms +dun ford +di reland +de ire +ca stron +ay es +av ar +amb iti +ale gria +ãĥ ¬ +âĶĪ âĶĪ +summer land +mor ing +march ing +ke ying +jeni fer +hun dley +hick en +have fun +fo y +bett ing +ba che +ล าภ+Ø ¥ +Ã¥ rd +werri bee +v ba +staat soper +song kran +scand i +re zz +p mf +n flu +lo onie +l rs +ku ech +krist aps +kar ang +hey it +gur l +freel ander +flo ater +dy o +be ady +. ,, +Å ĵ +wai ving +ts in +tru mbo +toen ail +star dust +se ki +richmond bc +punc tured +protest ants +propon ents +ow usu +orange burg +official wrc +o ju +nade shot +le tran +kir ti +kex po +in gest +idi ol +gh el +fictionaldeath siwillnevergetover +f pj +escape e +dianade p +v lad +titos vodka +te so +ser na +scar bto +robo tech +return march +r mr +modisar kar +man gas +ma hot +lan ky +l gs +kel sea +ing live +ho ffa +global ism +gb bo +g itex +eco logists +dev ening +darkk night +cour tau +catalo gs +bha ya +vir unga +v ta +un punished +spectac led +si ghtly +semin arians +role models +ro ble +paw nee +palmi eri +n ter +m sa +in stock +gan is +f hs +ene e +castell ano +bun cha +bl enders +र त +wca x +uuuu uu +us dt +strat um +ser gior +ri ad +prender gast +muni z +magni fy +kah n +in considerate +her ma +har nett +gye ong +ginor mous +gar ra +contempl ated +chit own +aun ay +alla h +ulster uni +sting er +sho to +sh urst +profess orship +prit am +pi af +pe dometer +momo fuku +missy elliott +mas oo +loh ri +infu sing +gustaf sson +dg ar +co ord +clock tower +cl orox +bro kaw +boxingh eads +ba hahaha +ab ba +witt ingly +uo it +steuben ville +sie te +sco l +re tract +re ce +pre scrip +north devon +lam bic +jack rabbit +dough boy +do remi +d ff +calabre se +bre i +bbc breaking +ar ce +aper tura +a jr +ൠģ +ww week +tour neys +soft shell +sc news +ori ans +one goal +nyc feelings +li ban +lash ley +k eli +interven ed +i idx +fut ility +du sit +disillu sioned +china sea +che if +ath u +am my +a ichi +" $ +ĺ ï¸ı +what com +w ani +te ahouse +tat amotors +ste ws +so led +sin us +scram bles +retri eving +pe as +p nw +owen jones +metal working +la hey +l man +l ct +kk b +k hel +go ven +for sk +fl ation +clon ak +can tt +can lit +bur han +aur is +warner archive +torpe does +te agas +stat em +sm iler +sha rec +san die +rc sd +profu sely +po spisil +opp of +ml r +li bra +ki xi +ine uk +edge of +crow ne +cri bbage +castle bar +book cover +an heuser +ì¹ľ 구 +âļ¡ï¸ı âļ¡ï¸ı +tter dam +ts itsi +tri balism +st je +snag ging +ri den +re press +r we +pre clinical +pat ap +om and +mu su +del tas +cun ard +cre wed +bath y +ar mes +am ino +íĥľ íĺķ +zaf ar +yan ke +ti was +rous seff +ro tarians +pol ina +hic cups +hall andale +g add +electro lux +eas a +duchen e +ding y +d xc +city centre +che sham +caloo can +ant am +tiem bre +rela is +refin ancing +pre stat +otter bein +mul hern +lin er +ha irex +fel ons +de ducation +surviv alist +ri u +r ra +mil ken +louis farrakhan +lam bie +kom i +hassan rouhani +harpers bazaar +gre iner +foo se +di leep +bay ne +baham ian +arin en +ðŁijĭ ðŁı½ +wild about +well man +under score +un democratic +u tta +tr h +ta eny +stri ding +sil lu +sal li +sa avn +radic alisation +rachman inov +ot en +medi al +m nc +lot sof +ku hl +kov ac +kor ner +instaf ashion +infatu ated +har un +gujran wala +fan ia +dor n +do work +confor mist +be ena +antidepress ant +alfre do +a ang +ðŁį ¤ +war ring +wal ru +vo sa +thru xton +the dragon +senior bowl +ree der +raven swood +pro té +pag ans +ome tti +o des +ng v +na hu +multit ask +mr g +market screener +great returnmarch +festi vity +chang in +cat ty +cad u +c ta +annab el +an tti +allot ted +ðŁIJ ħ +íĶĦë¡ľëĵĢ ìĬ¤ +un install +steam ship +sou le +san go +opp olo +ol ap +nicaragu an +mixta pez +las se +kran z +khair ykj +kar sten +j rc +infiltr ating +glan ville +freecode camp +fox new +el ation +edtech team +clow ney +by nes +bis bee +ad av +a unch +ðŁij ¾ +ye w +way yy +wa ir +tyler g +thursday aesthetic +stra il +st room +shor ting +read out +pri mero +pic hai +pg ce +par rilla +par dons +nat us +manirat nam +lo kal +lef thand +king fish +jun hui +isa ak +indi scre +gorakh pur +going to +far ted +east gate +demysti fying +de ek +d wan +compil ations +cel a +cath arine +book pro +bl p +! ðŁĻı +ðŁıİ ï¸ı +ãģĬ çµµæıı +âĢĵ @ +yun hyeong +yi wa +wd bj +tourism day +terri fy +specul ating +ser u +pal umbo +pal grave +or wellian +nithi in +lu pine +liais ons +h wang +gb g +fi sho +ed chat +christmasin july +antic ancer +ys k +tor c +tom es +ta pas +su st +stag gered +shanmu gh +season of +s mond +qu ally +o sun +nou ghts +mm w +leg gett +kabo b +hu mi +ha fez +h dc +fin ne +every damnday +cw l +curren ces +bige ast +ash more +ar and +ad pi +absor ber +wil ders +paparo ach +me ma +mac rae +law y +kar is +jä ger +inter nazionale +heart month +freck le +financi er +en gi +coast mag +character istically +bur ford +bour guignon +b ph +ar bonne +ap in +sty les +sl icker +s mas +ro cio +pyth ons +pur beck +pil a +patter ning +over saw +ky ung +ke gaard +im balances +hin dering +head stones +har to +ga ster +fab ia +euro p +done ws +de du +consci ous +cas bah +c mn +bever idge +ðĿIJ¨ ðĿIJ +ู à¹ī +w sh +tv line +tri ste +tany abur +solidi fied +snow balls +smir ks +size more +shu ayi +sam an +ris me +pot m +porsch es +p ining +ni sts +nar mada +n dc +ma wson +khandel wal +jeremy scott +infatu ation +for congress +fon z +anom al +ãģ Ļ +v anda +rec at +pan cies +numb ing +no doubt +newmexico true +mis mo +lat ers +lac tate +ko ck +gustav us +gat ecra +fujifilm x +force d +f bo +don ni +conven tion +ber on +âĿ¤âĿ¤âĿ¤âĿ¤ âĿ¤âĿ¤âĿ¤âĿ¤ +ठ¥ +zo v +yo semit +wolf gang +shu ster +rudy ard +pres se +ic en +fun nel +cryp tid +back roads +at aturk +ar ga +yamaz aki +trento antonio +trave münde +tele serye +shrop shire +sco ast +sb st +resc ence +pi es +op tioned +o ssie +nick las +low y +joey badass +i hi +g its +cuv ée +ct xwx +cat arina +calisthen ics +c bu +buzz horn +bil ia +âĿ¤ ðŁĺĬ +а ÑĢ +win wood +under paid +tran sa +sum lin +shan kill +separ ator +pat an +no fear +nau m +metr orail +lam poon +ke shi +ian mckellen +he garty +flan ker +encry pt +dilu tion +chattahoo chee +am ellywood +z ino +ym ca +wildlife refuge +thre es +teach able +sydneyis skyblue +swami ji +row th +need ful +nano tech +microne sia +go greek +gandhi ji +en right +edit able +cin ia +carbox y +ble u +ach ine +wall decor +vitam ind +vibr ational +rodr ÃŃguez +redd warf +re gains +mac laren +leh rer +la uper +khar kiv +j if +in usa +her mana +gur das +embezz lement +domin us +dh b +car ro +bloo died +ari sto +aravind ha +af ree +? :) +ðŁĻı . +ðŁįĢðŁįĢ ðŁįĢ +zol tan +wine time +w ach +terrap ins +tat ton +sycam ores +ssi onists +plau ction +pir an +pick ard +per sson +olu fsen +net ty +ne etu +mean s +lu ft +joo x +gu is +ful la +ev ant +bt k +boxing news +as mile +anim ity +an up +yan uko +wareness week +vel de +val ais +upl ink +sar faraz +rin gette +public library +mill icent +mcder mid +let ang +lee filters +il p +flam me +e gs +decor ates +br indi +bea stieboys +bark sdale +albor an +! / +âľ ´ +è ve +women sequal +ut ama +u igh +shoel aces +se kai +re trial +r kelly +peri yar +past as +na o +monte pul +michael gove +gode acs +eno ise +dizz ying +cho ta +che ong +begu iling +ale gacy +... âĿ¤ï¸ı +ðŁĺĤðŁĺĤ . +ðŁİĪðŁİĪ ðŁİĪ +âĺ Ĥ +workout wednesday +w ft +u zi +u ve +tn q +substanti ated +su x +st c +shankar shanmugh +plo s +play group +philharmon ia +p cie +n vc +me on +jer zy +is ds +if y +hut son +fe ig +fare ed +entrap ment +eh f +d sson +cut lets +cur ragh +chlor o +bizar readventure +ðŁİ Ĺ +wy ld +wies baden +unis wag +tur rets +tre m +sor aya +pg l +nu kem +lucy lawless +haute couture +google doodle +can nes +by ram +bo ch +az eem +albi hariv +ah sfx +!! ðŁĺį +yu ge +whit ley +west sussex +us ola +tn m +sen edd +resemb led +read abook +re develop +pri a +paraly sed +n intend +my photo +mass on +lori ent +jan ak +im prison +heart failure +go wen +gar rard +dumb bells +discre pancy +am pe +Ñ Ĩ +wr ds +we itzman +viol adavis +verma elen +tweet chat +subsidi sed +shi g +sf h +samson ite +ri sc +re dit +q doba +nv da +e let +continu a +co kes +ba jan +am iri +:(( (( +ðŁĩºðŁĩ ¾ +ver dasco +that matter +schnei derman +rebec cam +re mor +quie test +pulit zer +princi pality +pre rog +pr ssa +monta igne +lo tus +kar te +k assel +hy phen +ho ang +g aga +fruit cake +fan te +ew f +el fs +eate mu +defense less +caw ley +air d +ðŁĻĪðŁĻī ðŁĻĬ +âĻª âĻ« +th ich +t sk +su is +stat in +rr t +rand hawa +psychopath ic +petra eus +now showing +lu u +little hampton +here is +ha zi +green bay +gli ac +em pren +dun drum +di ac +demo graphy +coul thard +australian coins +apply now +a ham +.... !!! +vidyut jammwal +un leaded +un friendly +un favorable +ug by +th doctor +rac c +quiz up +q ol +peac ecorps +pas scode +oedi pus +notic ia +mari ka +mal la +ki official +khur ana +ke on +kaw ai +ka sim +hi f +ger alt +ay ut +ar ara +ap x +you go +wer the +t ila +stol ler +stam per +sf dn +sch ick +sat suma +raf bbmf +r ter +police week +pen field +p de +nit to +mono po +live united +lib dem +in sole +i official +her sey +doubt fire +dis mayed +dic hro +char don +cal v +ad ra +zi ppers +tri ste +sm alley +scal y +ram y +it ens +i one +eeee ee +du ps +de cries +criticalrole art +bug gies +boc cia +bigh ero +argent ine +alyss a +z eu +west view +vo ta +un sealed +teenchoice fox +tam ir +sympo sia +ssang yong +solve ig +ofe urope +ni pped +mol dy +kom en +ken ley +k ws +har pist +gon na +dh all +desh pande +cruci ate +coroman del +bo ssi +... < +ðŁĺį ðŁĺģ +ðŁĺĩ ðŁĺĩðŁĺĩ +unfathom able +ul ter +trit on +torto rella +thumb sup +ten ets +ten ergy +tan quer +soo kie +scuri osity +sar don +sar ath +ri mmed +polkad ot +om ak +le se +ke te +k una +jun aid +gr iner +golden rod +gig antes +ful crum +ell an +di ani +currently reading +broad view +ber al +am bode +< ) +ðŁij© ðŁı¼âĢį +âĮ £ +wr angler +week nights +under foot +twit ching +turn berry +su dir +ss is +shangr ila +ser rated +sea forth +rubber maid +rive ted +read ers +re marked +p bloggers +out flows +non verbal +ni v +nabo kov +human ity +fin den +er f +das soc +d vs +auction update +as la +angel ina +.... ..@ +( ? +the ak +ste ffy +lu blin +k win +g ago +full metal +for bes +ell ars +dd is +car mina +by d +boardgame geek +ber wick +auto bot +as ura +ap hex +ander pump +air brushed +your way +wil more +tur ki +si a +s later +ro ared +pur itan +om ori +nbc philadelphia +mor cha +mi me +ku char +ken newick +kb ps +indie book +hen do +ft th +flo of +bru sco +ben elli +asser ting +aqu atic +ठ£ +sur real +st kil +shet land +sed bergh +scots magazine +ri za +playlist live +pa il +mi us +mh k +lor as +leicester tigers +lc g +hul la +hu ms +help me +gin and +ea f +dungare es +don tw +deterior ate +dar cey +dal al +d wr +conspir acy +ch ere +bander as +all iteration +* âĢ¢ +âŀ Ł +xen overse +wheel in +u al +tur bos +sfor kids +saturday kitchen +s ja +rand olph +pr b +ou w +o in +new look +nd wx +lew ski +incur sion +gr rrr +car shal +buc ca +ban an +asset management +arau jo +apo logists +af ly +ðŁijģ ï¸ı +you got +wax es +toile try +stow away +ste adi +standup to +si ski +sel tine +school s +pho t +miamis up +lore tto +lam ba +kr ita +ib nlive +hin dley +frank ford +exfoli ation +diabe tes +de cer +cra gs +bin dings +bat cave +aj j +wo www +waveleng ths +w abe +toysfor tots +rash mika +pizz o +pha ser +ore l +musso orie +la pp +hit t +happy independenceday +ga ius +colon oscopy +ang ing +adidas football +yon o +whit eli +ton ks +stumb le +solemn ly +ru r +ragh un +qi ao +pay gap +mal an +lisav anderpump +ku be +k son +ib l +homemade tools +gw h +favor ing +ec lamp +deven ter +deceit ful +dag en +cur ling +can ey +big brother +bau s +ah ri +winter sun +willi es +wi bw +twer k +table spoon +t eli +sze chuan +super tramp +reminis ces +rally finland +quizz ing +papa johns +naco gdo +mccre a +i zzie +fitz william +fal le +de j +dar relle +canti lever +business women +bush fires +yas sin +vander pump +th als +t dn +ri ko +proce dur +opti k +omo vies +nai doo +minute man +kasey kahne +fo olin +ex chequer +corn rows +black mailed +bl at +bil derberg +ar twalk +ðŁļ Į +ðŁIJ Ļ +ëĭ ¬ +å § +ye eee +thaw ing +shak ir +ra zed +pitto drie +p atta +mccre ady +jack i +inside the +fla m +can am +camp sites +ban nock +at si +ar bo +ao g +anch ine +vand ana +sw ade +show and +rishab h +pi raeus +phar md +mat son +m gp +lau g +kal inga +injec tors +hypnoti zed +hil bert +fric king +e if +du sd +do herty +de bi +cab a +brah maputra +ber rys +ban offee +af fer +] ." +ðŁĻıðŁı» ðŁĻıðŁı»ðŁĻıðŁı» +âľĬ ðŁı» +z ele +thom es +te ide +super marine +stress ors +sing ing +si bi +self care +scalex tric +pres que +podi atrist +p mb +naval ny +mother ly +ko ku +ingh istory +do pen +cj reform +chor al +as amy +ampli fying +ali i +water ton +vw fc +unex plain +strangel ove +she sh +qu bool +pre via +person i +offic in +gn on +g ps +fi est +farah khan +engul fing +energ ys +.. ðŁĺį +ت ع +y anni +warnerbro stv +tor rid +summer sale +or bis +motor cycling +mojit os +le jeune +hippo campus +gil pin +flgov scott +fin dthe +edwards ville +dsw d +d mi +cur r +bon neau +blue eyes +b bo +am ills +win sford +weid man +upcycle d +sung yeol +shi shi +re setting +ravic hand +r ty +nt australia +dham aal +da res +d cd +cb colympics +bapti sed +bab yyyy +adi an +ðŁĽ ģ +wil ford +wak aflock +sp ink +sie ge +sad am +qash qai +phra sing +ling a +kin ka +indigenouspeople sday +il ie +gin ho +giac ometti +extru der +cand or +callthe midwife +bow erman +bague ttes +at oz +arche types +anten nae +without you +to cin +th ts +th aya +sh int +s guild +musk rat +mc gre +man aus +mag nit +lun di +lumin escent +lap ses +kin dof +je han +if ad +gu iness +greg ori +gi jon +gg is +foo dgasm +floor plan +f sg +ess ss +di marzio +dd ata +clu mps +al at +ãĤ ´ +ze en +y cling +w eli +trou p +tote bag +shre ws +scur ll +repjohn lewis +or te +ma ho +kaz i +jor dana +irrit ate +ha vi +ge c +f ici +avi e +ari jit +am rit +am gen +wre g +wit a +tor ide +ti died +shu bh +se mua +ride share +r vc +outfit oftheday +nypd news +novel ties +kid cudi +khali stan +k ren +ide c +gru p +gon nam +conne ct +confe ssing +cere bral +bal am +ash u +won k +twom bly +river ton +repaira ble +re constructive +ra west +ple at +play writing +paul kagame +nurse sday +lo dz +ghou lish +dra xler +dolom iti +de te +cru do +billy joel +atom izer +as ol +al car +y cfc +we sl +under floor +tre maine +te tr +sween ey +sket chaday +se ba +s sec +rjal ok +rin der +rally gb +pr ine +port as +jam mies +horseri ding +gra phie +gi menez +gar oppolo +gar dai +ey enews +clun y +cavan augh +cal lie +cal ey +brou gham +berline tta +ben tham +arou sed +wh aler +vo les +ty ner +twee thearts +tor na +si ap +shu ja +sar avan +sand awana +s fr +quik silver +pter odac +pm harper +ob tains +neo geo +mog wai +mid year +mi kasa +eh ne +droit wich +conservative vw +cho l +bigten network +arach no +æ¸ ĭ +ur bo +u sta +sub prime +sle aze +s ber +ru sia +neb biolo +man al +lun t +it ori +is good +ho ard +hel dens +go ve +fla gg +et at +emma watson +cas so +as aba +aro ha +am ica +alfar o +wer den +tri glycer +to ho +re ema +punx su +om nomnom +ol de +mack in +li vor +kw gt +kh ris +john c +harpsic hord +gal ent +francis co +dr g +come to +cater pillar +calcu lators +bbc world +augu stal +ad sl +tran spon +tra eger +string ed +sr hv +sowe to +sle ad +se ur +santac ruz +run happy +nhs bartshealth +ken cen +it all +hot sauce +good fellow +gian franco +ec ap +b ening +aha b +take flight +symbio te +sou da +solar panels +si gue +ru bric +ri voli +rema x +ome gam +n kandla +mores by +mik ado +migno let +may bank +man gum +makar sankranti +kam eron +i ero +hart pury +gab ay +ft nhs +ever son +come dia +color blind +be aune +bank stown +amend ola +---- ---- +ðŁĴĹ ðŁĴĹðŁĴĹðŁĴĹ +ðŁĴª ðŁijĬ +ðŁĩµðŁĩ ¹ +à º +ÑĤ е +ve sting +up keep +traw ling +team breezy +star scream +ss av +sper son +slu mps +shin ya +re package +po were +po ort +pla b +pic hand +ok kad +o brien +nu ff +n ani +illi am +harold sinnott +green party +glen elg +ge er +dreamleague soccer +diso wned +constan ce +cas sandra +al gui +ty per +tore ros +tan us +swar mapp +sin dy +shee ting +sham si +sep tiembre +sar ita +palae o +indv swi +fiel duni +david walliams +cool down +color ador +camise ta +ap ul +ad ac +wet shaving +van hansen +tw ard +tou areg +syn ced +str ang +sp w +sh acks +sati ri +ron it +reali sts +pramo d +ori ol +fann in +en nale +embro iled +cul vers +chat room +buff ing +ban e +ac m +ðŁı ĸï¸ı +writing life +vasundhar abjp +un desirable +tho ckey +ram anu +pa store +nin ian +ly tton +knu dsen +gg v +fi z +emble ms +emb arc +dispen sers +ca sid +asu tra +app y +ðŁĴŁ ðŁĴŁ +y ut +wb ball +vin icius +pre dation +pa sting +noplac elike +ne wark +mammo gram +ma ji +luch ador +ilove jax +i plauction +espar za +el ley +contempor aries +clo aked +cele stia +car ola +bt x +brave hearts +bi ghead +benand jerrys +ar len +ap it +ap at +anjun abeats +am erika +ãĤ¹ãĥ Ĺ +ا٠Ī +y aba +wau kegan +tw p +tter man +stra ddle +statec ap +rash tri +quarte ts +plat num +pax aus +morg ans +li baly +leopard stown +kro hn +it security +hun ty +here dia +gra ined +express oshow +d he +ak f +* : +á Ĺ +yanuko vych +ty ger +sun limited +shealth care +sc itech +oppre ssor +m tt +he ssen +gon gs +funny picsdepot +flip side +fam iglia +du o +cathedr al +bal anchine +af pphoto +.... ( +ðŁij Ĵ +tho o +seaf loor +san kara +rac ial +open air +ner ve +mat ryo +kilo gram +khal il +it weets +he is +embo ssing +egg man +bachar ach +att va +ðŁĺĭ ðŁĺį +ðŁĩ³ðŁĩ ± +vox dotcom +un learn +super cross +ros ita +re paid +pan ettone +nor fol +mii verse +mari ai +loud ness +ley den +j dc +fm news +fasci itis +eye glass +eloqu ence +daw ned +chron ometer +chri swe +cho i +carling ford +bhar gava +bbc mtd +bal tics +uof m +ty d +swasth abharat +stor noway +shu ffles +sen o +reson ated +re ag +no via +monster cat +mb ank +lo te +kir ito +hoo ligan +her up +h ite +fox news +early modern +derby shire +day trip +cudd le +consecu tively +bli c +black out +be mis +ar ash +âĻ¥_ âĻ¥ +vishwar oopam +vash on +trajec tories +sine ad +sat ri +pu fc +new lyn +natu rel +min tz +d pan +cru k +bor u +ta ko +se and +s america +pri yam +navar ra +monte cristo +mill is +ingh ope +hep atic +hall in +fc ity +electro chemical +dr martens +cj ad +as rc +weather ill +varund hawan +teh reek +stocke xchange +sko ol +shi be +rubi dilaik +n pe +mo ko +ma ic +indi ak +in takes +impedi ment +il ent +go tye +getin to +fing ering +clau son +c ni +bal o +ann andale +an ju +an ers +am g +al goma +womensequal ityday +tew ks +sugar land +prospec tor +mil ian +man made +li iga +laz ada +hum per +hb v +green bush +ep k +con tro +biomimic ry +ठĤ +uk tour +the happy +scro ft +punxsu taw +on the +newmarke trace +me ca +lie tta +itsmore funinthephilippines +is born +haringe y +fri sch +eye candy +electro des +con ant +co done +w br +sch y +rad wanska +newn an +nei man +nb poli +megam i +ma da +lunar newyear +lei va +inthe sky +i vs +glend ora +foreigno ffice +fashion photography +eu ticals +d kr +c st +c ftc +bri stles +bic ent +az family +ai ff +ðŁĴ¥ @ +ðŁİ ĸ +wa aay +up u +tho d +sle dging +sig ne +oire ach +nor ad +noel gallagher +new comb +ma suk +kra b +ken ner +jet star +in ert +hon ore +global ed +bur pees +bs v +bett man +at sushi +arjun bijlani +airand space +ab bin +ó r +sonunig am +se mat +ro vin +nat galleries +natgalleries sco +nar co +miz rahi +lero y +kno pe +hi ker +hang ing +comple teness +cha vez +cab ell +bil der +av m +ak y +a are +pretty much +po ta +over arching +or nl +kovac ic +ken n +kam ui +hel f +har r +ga stonia +fo h +fidd lers +fac to +aren al +âĿ ĩ +zol ciak +toyo tag +tan gier +spot ligh +spo ols +san bernardino +s burg +ra pati +p dd +n age +mu cking +j io +is cool +i mus +hassel beck +har shad +gn g +forex trading +du es +borgh ese +bi kaner +am uk +al wys +waist band +w league +tot alling +summer house +srin ath +pun gent +pe dr +pab st +mulca hy +infr inged +fir daus +bur ka +brian cox +bi ola +bc bg +ðŁĺľ ðŁĺĤ +x el +sul kin +sal ve +rafi ki +pan ky +pag lia +na aa +malibu comics +lear jet +lac una +keen eland +kau ff +her acles +hair color +fur st +cor rin +cas al +ale h +ب ÙĨ +y aaaa +vi ra +te sy +styli stic +strong bow +sport pesa +savi o +pyram id +may er +mante gna +light sout +lease hold +lan n +kit tel +kick solution +jet set +ic at +f gr +donttry this +de jesus +cw lps +character ised +buzz words +bail on +awesom esauce +asi ana +articul ating +abra sion +ðŁ¥° ðŁ¥°ðŁ¥° +ãħ¤ ãħ¤ +Ñ Ĥ +z hong +worth the +talking picstv +sen goku +r é +pic o +nit ric +mr ti +mal u +la ster +jac ke +is simo +gra fia +game audio +down for +do something +di donato +d fir +chal ked +adi t +accor hotels +ab rac +âĺĿ ðŁı» +wt kr +wl tx +vote redto +ve sper +spur lock +sc limate +s mid +recen sione +paper clip +oom ph +national birdday +ke shav +k df +ichoo se +gmt games +di mm +dan ews +clo seness +c se +al tidore +after thought +zimbabwe ans +za id +wizardo foz +un flattering +thar p +tal ong +sump ter +stein brenner +sn w +sb n +sat o +pl d +mar com +mal ina +luxury cars +kho bar +j ss +ice house +hicken looper +dead by +d loesch +cas sino +budo kan +bi zz +amar one +tic e +sou vla +sin uses +seam us +samu elson +pre poster +news rooms +mel wood +maest ros +ma gus +lyn x +kav i +ir f +hal eso +get out +ent in +dog walk +cu al +ðŁIJIJ ðŁIJIJ +âĺĢ âĺĢ +what evs +wex ler +vi stara +swag s +soc biz +sneij der +sm on +si se +pr ancing +le ff +khadi ja +j sm +hill toppers +emer il +dar nold +comp o +chan tic +can aan +bl inn +ðŁĴ© ðŁĴ© +york ton +yl i +world building +w syx +u hi +stre l +stop kavanaugh +space ships +ski i +sel as +rac oon +pri mula +platnum z +paren tal +pal ah +nim rod +min doro +mc mullin +lo in +il en +em merson +cricket merijaan +ze o +w afl +thel oud +specialty coffee +soap y +say no +sab adell +rosam und +ravi dubey +pray ersfor +patrick dempsey +ower ri +oc u +mari as +lifeis beautiful +go tto +d wee +circu latory +child less +bag ay +awol nation +analo gies +aldublo ve +ðŁĻĪ ðŁĴķ +troubad ours +tou te +timb ur +so dy +see the +rachman inoff +n tt +mol ars +mo tta +love ukweather +k ates +il keston +hol gate +hair styling +fel onies +chen ille +camp grounds +am asa +å¤ © +© @ +st ape +sl ung +righ ton +plan es +p oul +mic ha +methu en +kore y +ke ener +ke ck +jarre ll +in fidel +il ona +herb alist +ff re +dog meat +cur sed +cron k +centr a +cam rose +bright man +as ce +ac cade +abas ket +ys ers +wy se +warsz awa +vik ander +ver onika +unfinished business +su ter +steven age +startup grind +roth stein +rio olympics +name plate +myrr h +mer cu +me aux +low nz +lin seed +ir un +i aw +gi ani +fij inews +ef an +early ed +detoxi fication +deta ins +cor rado +burn sville +bri thday +bri stle +bodle ian +bj j +bau t +aude mars +as ys +ðŁĺİ @ +yan ong +trayvon martin +suf ism +stern show +stainless steel +sp all +sant ini +ripp on +panathinai kos +musko gee +loo ts +local elections +la yan +kit teh +khur shid +kel son +iron side +illi c +hick son +ha aa +gooden ough +brand en +ann ast +we ger +va o +uk news +talking dead +spi ers +sculp ture +ridg way +re sets +ra ved +nex gen +nat aka +ligh tened +lie ber +inter i +goe bbels +gal lau +free play +bu kan +at ani +a of +ðŁijĢ ðŁĶ¥ +ï¸ıâĥ£ , +sy ard +squ alls +ran deep +r nb +qui el +proudtobe abulldog +pom eroy +o brig +moe bius +kar ine +juni e +jou st +joe ys +jo k +ir y +ha is +gin o +ester o +del ands +coo t +bbcradio wales +assimil ate +and ouille +ðŁijįðŁı¼ ðŁijįðŁı¼ +wine fest +wai heke +ve sic +star tribune +sid well +scale up +sc cm +pru ett +perfec tionism +night marish +nca aw +nc f +in bkk +hirsh horn +he tero +griff en +green e +fat test +faceof mlb +el r +chuck grassley +christ oph +chip tune +c itt +brick ed +bo ga +blasphe mous +ber m +are dux +thel and +sk op +shak er +o ems +mciner ney +k ween +i ppo +gas ps +col mar +cla xton +castan eda +? ðŁĺį +ðŁ§Ļ âĢįâĻĤï¸ı +ìŀ Ī +wed ded +ve te +uka id +tribut ary +syracuse u +san pedro +on location +ngr president +mon oli +modig liani +luxemb urg +leg anes +iam will +ecclesiast ical +du plass +ded ham +comp els +blan ch +bill nye +âĿ ¦ +âĻ« âĻª +weight watchers +wax man +tede sco +te zuka +sneak peak +rec ir +ran dee +radio times +py re +oom pa +messi anic +hawks bill +ha ga +glen livet +gil mer +fabric ate +edin son +eco smetics +colorado springs +co tte +bag a +b ä +b anta +antarc tic +ambro sius +a sea +ðŁĺij ðŁĺij +th il +te avana +tam era +shann ara +sch aff +s ence +rhe e +re ta +pe al +mari ach +kri dge +ic co +fratern ities +endic ott +dere cho +dam er +cad mium +brick town +ì º +v pa +tau s +ta rek +sun downer +rose burg +pel agic +pa es +ou nos +nicol ai +lindel of +libt ards +leadership development +laure ls +hot star +goldend oodle +gi untol +dand c +cros sh +ch ym +cannab idiol +bure ss +bmw motorrad +blin ky +bel asco +apol itics +am bler +ale sha +ðŁĺ® ðŁĺ® +white boards +wa hoos +us y +stro de +sar as +pro visioning +oni giri +may ank +mal inois +low ell +ke chara +hyperson ic +herbi vore +hard castle +blue star +bio diversity +av os +and white +ware house +viol ators +v asha +tul loch +tor fa +th ony +sh iller +pun tac +proce ssions +piec ed +p ca +mayo clinic +ma shups +la goons +in suff +illustr ative +golfclub sforsale +frie sen +drizz ly +do ane +deare vanhansen +cross bar +bri on +au rea +aro berts +aqu al +ðŁĻĤ ðŁĻĤ +weis man +uz bek +traumati sed +switch board +st app +smo vement +sa arinen +re growth +ra wing +nu ke +melissam c +hun na +glasgow warriors +dict ated +bv l +balonce sto +amar al +ag dq +velo ce +the hague +tet ley +tattoo ed +son us +sc india +sar un +preemp tive +pr oro +pi dgeon +mon tel +magi k +ke ylor +ine x +h pt +f cbd +cyril ramaphosa +co ppers +chri sho +bur r +actor jiiva +trans verse +str one +stin kin +pil atus +occupy central +nephro pathy +looo ong +le ight +language learning +l rb +hy annis +di ppy +col ville +cho ate +central coast +car illion +camp y +bol dest +b hay +all ston +xplo rer +wy wx +w ur +ur so +taver na +summer nights +rock dale +re supply +qot dapp +pan etta +pal azz +oh well +monon oke +loe we +listen to +l eri +kun dp +if p +i onia +fro mm +cé sar +cu enta +col ley +be gotten +ang rier +ad are +abhor rent +! âĻ¡ +ðŁİģ ðŁİģ +za al +v sp +there min +su br +s doh +qaland ars +presi des +nup tials +mis behaving +im ams +hc mc +happy tuesday +fru iting +di abo +datam ining +augustal sina +an zi +!!! . +ļ ðĿIJ +ðŁĴķ âĿ¤ï¸ı +ðŁĩ±ðŁĩ § +ëĵ ľ +wise au +we artv +war ne +te pper +strategi sts +stargaz er +sp ann +siss oko +sal a +physical activity +newn ham +na im +n cel +me aden +mar cin +kay aker +j iz +hagger ty +gun ge +gu yan +ernie ball +di splace +de el +code pend +cay etano +ay yyyy +ar irang +adren alin +achan nel +ston eroses +sto ga +sport scars +solom ons +q hu +ph nom +palla dio +lun gu +lo i +j ari +hob goblin +gathe rer +de volved +come and +celebr at +bra inde +ba atar +avie more +as ky +: \ +ãģĬçµµæıı ãģį +uro logical +un declared +u ob +su ess +sc ura +sc ast +sam en +roo l +ri pen +raise your +ra ju +pra bang +pinar ayi +paign ton +os int +lake wood +kon an +je ffs +jacob whitesides +incu bators +ichi ban +hb l +fr illy +fo gerty +conju res +ain slie +. ðŁĴľ +wor te +wol ters +wo wow +tra gic +teleno vela +smar athon +shaw ols +sex ta +salvation army +quan tu +pinnac les +on itunes +nestl é +myelo id +mu y +mit er +meg ac +mc kee +jo van +heart break +gas ped +func tioned +freak out +endthe stigma +disab ling +carne vale +cam uto +bernar di +ðŁĺ¢ ðŁĴĶ +âľ ³ï¸ı +ÃŃ as +un ni +ter p +sin op +pre co +opi ate +men in +mandu rah +lon gu +integrity tt +hr tech +great north +fr nd +eli k +dad dys +construc tor +conce ited +can em +ðŁĺį " +su ll +oper andi +on ster +mm x +lost cat +leg less +karim loo +ju ga +j sp +hand rail +gri pen +glori ous +di mming +bury sted +bt c +be eck +am ai +algui en +youn es +ti sham +stil t +soul ful +sid cup +seg awa +p ex +open shift +mechan ically +hd x +har tigan +dhanan jay +cu atro +canal side +bon gino +berger ac +as cle +an ju +ag low +ag et +.... !! +âĺº âĿ¤ +tom er +the us +teac ups +sa urs +real mickfoley +perman ent +pa chuca +matric es +loud phillips +kof i +k ago +g cr +flu stered +de anie +bloo diest +bis u +am ni +selen ators +sc ens +rine hart +objec tivity +moving forward +masa hiro +marath oner +lom i +logitech g +koin ange +is wa +ing ues +hyun gs +hi ther +end anger +ele v +consu mables +caval cade +cap ilano +black beard +arte misia +arian ators +actor madhavan +yo c +un win +u am +shahe er +sci der +s mbs +p ish +my mixtapez +j oma +he yn +gui do +federal reserve +fair mon +dist t +direc tories +cab rini +ber ri +beau voir +be the +a head +y sle +warrnam bool +up market +tv personality +tuesday morning +schri stie +sar gon +re bus +r bu +presi den +pow ells +nfl draft +nacogdo ches +music group +kis lands +insomniac games +il or +exter iors +end res +der ot +de composing +das ani +camp agnolo +but ted +br ann +anti gone +ahistoryof paint +ठ¯ +thim ble +the stor +sul ly +starwar sthe +sc avenging +red wood +palah niuk +nove mber +mat eria +longmire posse +kerrang magazine +ing els +industri alist +i dai +ghe alth +dont miss +del any +cook man +brain child +book nerd +bland ford +backto back ++ ] +ðŁļ ¿ +ye z +v ash +un stuck +summar ises +pen manship +mumb o +minimum wage +maz ur +mar cas +k ray +id wx +gold in +follo back +ear pers +É ª +well being +var g +ubis oft +tom brady +some where +qu ire +pax south +od ar +london bridge +justin formmva +it ar +ha at +gup tas +gom or +glaci er +ge b +gan ic +cam ron +c pap +brianmb endis +brantley gilbert +bow doin +boo z +ale jo +ag at +âķ ° +th f +ta zar +sex tet +sam osas +pl is +pel tz +pedestri an +oo t +newh am +mc williams +koinange jeff +k tr +ji be +gas lamp +gar ou +fit ment +ened ict +en tail +duck face +coin age +co ale +car very +atho l +aj lee +afca jax +ðŁĵ ĺ +tra it +tor ms +stri bune +snow boards +shadowhun ters +sere mban +prefer able +pesc ara +nz mustdo +nar ine +multic oloured +les by +g att +ey al +en fin +day made +congre s +ade vi +accoun ting +! ðŁĩºðŁĩ¸ +ðŁĺ¤ðŁĺ¤ ðŁĺ¤ +Ë ļ +z illion +yel awolf +un question +thalai va +shay e +savag ery +poly cystic +nh ra +nc b +mathis on +made a +jay as +indul ged +ho well +f mt +erud ite +drash tid +d anna +cire bon +ch ander +ca ity +bay ou +ant en +alban ese +æµ · +âļ ĺ +zom at +v si +tay la +sultan ate +sagarika ghose +rt l +re eses +re charged +pla zas +pi eters +passi one +p mt +merry xmas +men of +marti al +ly can +ku antan +jojos bizarreadventure +is ac +cullo den +chef s +cam omile +bean z +an nette +a itor +ãĢ ½ï¸ı +à· Ĵ +whom ade +whi ppin +sun corp +ru lings +obl ong +marsh mello +ly re +live mixtapes +lafar ge +je anni +hot chocolate +ge ty +fu rio +for all +fall a +ez ral +eun kwang +cumber nauld +c gr +bleacher report +apo pka +al italia +agil ent +ðŁĺĢ . +ðŁĩ» ðŁĩª +wed d +tro ika +torch relay +terrori ze +t inge +t anger +stat ics +ron y +re assures +ra ze +pre so +pl am +orph ism +matthi eu +fun chal +f sn +est ation +en el +e pos +dist o +den ys +dam ore +da hi +car natic +bur un +airtel india +your self +wonder woman +wi eners +tv m +swords man +so ha +seductive sunday +pet kovic +oil oncanvas +jugg alo +hu dak +home automation +gu mmi +go ch +girlsin stem +fli m +electri fy +dig nity +commissi on +canon usa +av ro +american u +ag f +a adi +up and +under arm +team adidas +sta westland +smd streams +single track +r sn +quanti fied +pocket camp +pan kaj +oxy tocin +outlaw queen +or rin +of time +nigh tof +ke ter +k sg +jim lee +jeopardi ze +jax a +janath agar +j harden +ho isin +h kd +giuntol i +fra yed +for trump +doo zy +deli ghting +del ray +dair ways +chir ico +car crisis +c dj +arin ers +thre sher +strictly comed +sp akistan +seas capes +scal pel +ro mulus +pro po +prin z +pre clu +pat in +kis sin +kareen akapoor +gl eng +flam borough +dece mb +d andy +cli k +⼠± +âĹ » +wo u +v oom +th it +seven teen +serv ant +sar ovar +sam er +quinter o +qadi r +puj ara +publi sher +pla sia +per domo +pc bs +nau seous +n gn +lom poc +jig gly +ir refu +hero escon +he sp +ge er +f wiw +exempli fied +exemp ted +do is +d xd +bang sam +ban jo +av n +wed gie +thom ason +sway ed +sv cs +spen ny +slam min +re starts +orm skirk +meadow lark +mar scuriosity +man sa +maith ri +ma sato +li saf +ko ehler +kab e +ja key +gar lic +flori o +du pont +dd ler +ation week +arsenal fc +ye k +y ss +true detective +thel aw +sun beams +school memories +pra bowo +oi af +life times +lank ford +k ci +in sead +howdoyou pixels +fthe year +down grades +dam mam +cor champs +colle gian +aul x +ðŁijĮ . +wo h +sc um +ra ham +play hearthstone +pagen aud +nik ola +mcl ane +lucoz ade +lake tahoe +lac tation +kno p +kei fer +janu zaj +home bound +hol lowed +heat on +gor gon +fur baby +fox glove +fau cets +colbert lateshow +barnet fc +agar h +! ðŁĴª +thab uri +statue of +snee zed +singapore ans +perth glory +patho genic +orthopa edics +odysse us +mq m +k tg +far ouk +cw f +creative bizhour +bo ice +beo grad +water parks +vitam inc +re broadcast +ph enix +perfec ta +palm beach +ni mh +my croft +mu tv +liber tarians +lang dale +l tm +jad u +ine k +har macy +gun day +fair lane +entwi stle +csur ams +canni b +bu tane +aveni da +afe ty +.... ) +ë§Ī íģ¬ +wax wings +video game +un daunted +ther yman +staple scenter +plu gge +nis d +nab e +mari el +lor il +leather face +key ne +kam o +hu la +herb icides +gw apo +grego ire +flav on +fion n +fatt y +do ke +disab led +dam ion +col as +che quers +ðŁį ŀ +ðŁħ ° +ë ´ +zon da +yuz uru +whe aties +under took +u bl +tu sc +sonn tag +raz on +pu kka +pe ssoa +nab lus +mus kie +misam ental +lo pen +lar ch +lan downer +jame scharles +gra cht +gly col +gi onee +g cm +er ob +cé line +cay e +c no +air liners +ag era +abdu r +ìĺ ¨ +æ· ± +wing tip +violence against +throwback tuesday +sound s +servic es +rin aldi +pun ting +porscher aces +p wr +luzer n +ind re +for humanity +fair ford +ent rada +dan mark +ati st +ati me +and blue +à ± +wwe shop +vitili go +ur bandic +under hill +thisi sour +tex a +slan ted +remote work +radio shack +molo kai +mid somer +mb ap +jar od +ih sa +har rah +fir mer +fa ure +cla ires +car oti +c ang +b gp +assi on +app u +af fo +ðŁĺī ðŁĺī +web md +swarth more +ste g +re efer +rab ino +promo code +play land +o wain +mill ersville +le anna +kuech ly +hypo thyroidism +green build +forthe many +fair ley +er ice +di sing +cv g +busines stravel +brun e +è © +un itas +small youtuber +nal cs +move the +morde cai +micro bit +jack gilinsky +ir vine +graphi x +gra ha +g live +fri pp +disgu ising +chuck todd +amal ai +zan esville +worshi ped +wart burg +u cu +star ter +sol way +sag na +ro den +por tra +mer cad +li ane +it sd +illumin a +hu shed +fc p +experim ental +e ol +du val +chri e +belmont stakes +beis bol +ant ander +al fi +ðŁİ¸ ðŁİ¶ +ðŁĮ² ðŁĮ² +whi pple +water aid +vin b +un wittingly +str ana +sd wan +reson able +notori ous +nang arhar +li sas +lfc family +le ic +hump day +h mr +go the +evo king +ei der +detoxi fy +con cili +cin tiq +bla is +bar ris +au bin +andri y +alder weire +ðŁĽ ¡ +ðŁij» : +sand hya +quar ry +pol ley +oc currences +nvidiage force +neverstop exploring +mo onee +man ed +helenclar kundp +gag ged +descri pt +da shes +cl ang +car dano +can geo +av ond +as sa +wwe sheamus +wut ang +wewill rememberthem +we know +vibes only +van canucks +u tiful +tür kiye +th l +talk talk +summer side +str itch +roo tedin +re ous +quay le +obe ying +grand sons +cnn money +chat sworth +char tres +br att +au dia +ae ter +âĿ ķ +warrior pride +virtual assistant +va sia +tre de +talk za +sal ou +ran ce +r fi +pir zada +pd b +pa rel +os ler +oh p +need lessly +met all +meado whall +mcel wain +mccull ers +eldor aspeedway +dele phant +del tar +budge tary +alternat or +addic tion +ys jagan +wood carving +u ffici +turkish airlines +triu mp +stephen ville +silhou etted +shant anu +scottish fa +ro aches +pe dra +p mc +nu de +looooo ve +li velo +kis er +kar on +k ma +ho by +com pas +cau x +bre ch +author ship +ar mer +ðŁıĢ : +woe ful +wode house +un categorized +tiwas avage +stru ck +ros se +r ps +prithvi official +no bs +kor ma +ken zi +jone stown +jav y +il it +ga ad +fe ei +esp a +end childmarriage +do en +cooper ates +ci bility +bigg ar +alex morgan +al x +aa ahh +ìłľ ìĿ´ +u ria +t ve +so you +share my +rother hithe +pierre pinna +nts live +not en +ni ks +mark gatiss +lifeat purdue +law lessness +lati me +kru k +kn b +hyun day +gre end +din n +del aney +d tl +combat ants +bon gos +athe ist +all that +a bet +ðŁĸ Ĭ +ðŁĩ¬ðŁĩ Ń +we al +understand ably +thel ake +te kno +tamanna ah +street lights +squ an +shul er +sh inn +seth macfarlane +ro stock +ren an +plu cking +n vw +mariai rene +kor da +kad y +itti had +hov ski +hobb its +gr ates +fern and +digital artist +ball fanfave +bab bar +alessi acara +travel news +sw g +sau gus +rou n +re booting +pre is +pe ps +ota ku +nom o +mce ach +k official +k anti +in sa +glaiz aredux +fuller house +f q +cw bb +back line +actu arial +í ı +å ¦ +w pro +ver anomtv +un happiness +un developed +travi ata +synap tic +singlet ary +sharp ness +se gun +sam berg +ryan j +ro ca +pin al +pi olo +per ro +par khurst +nc sc +kri stol +kat rin +gra dy +gn ats +glyce mia +fall back +cyber news +cor netto +catching fire +bom berman +ar ris +aless and +accor di +ðĿIJ ŀ +wat kin +ty co +tri gg +ta int +sv m +street cars +store house +sh esthe +se aly +rou ges +r co +quere taro +py c +pre zi +o oni +nyo vest +mar rao +mall on +gio ia +gau dy +ecoun cil +dan ang +confe ssor +clo ister +bio engineering +bac carat +az central +ta hi +sport stalk +ri pper +phoenix raceway +mon bebe +min ds +mal ad +kyle larson +kri shan +hul ls +hi att +h fh +ge tters +follic les +duis burg +diss apo +dele te +cu bist +corn field +con ec +cat an +ag ta +ðŁĺģ âĿ¤ï¸ı +ðŁİĬ ðŁİĬ +u op +tyl desley +streaming party +st pi +smoke less +si gep +shut up +scot tho +rose gold +reinst atement +pre sti +pil i +out performed +laser jet +lale ge +kine tic +je evan +jam m +humb u +grand erson +dress ings +cr üe +cal dic +c cac +bhar ath +amy g +amoun ted +âĿĮ âĿĮ +tol entino +terrori zed +som s +sah in +real bencarson +re introduced +ovarian cancer +nam ah +monte z +mis fit +kamp ong +ice age +gum tree +gou sa +gli oblastoma +gat royd +figue res +er ror +entertain ments +ec an +dream boat +dis membered +di mble +cro m +cor win +conspir ator +colle tt +bon ney +apir ateday +ðŁĴ Į +v ts +tiff ani +sof tened +re collections +pom pad +mis adventures +li gab +kal an +intermedi ary +hammer heads +gal ata +frat ton +ea rena +din h +big bad +be hringer +bab ad +alder ley +Ä ij +swi zzle +sri ram +sp hl +so wn +rit su +r ga +pur u +pointless blog +pide mic +opinion ated +mo stafa +mel ange +heaven ly +fort nightly +first class +essay ist +e ons +crump led +cri ver +c ór +book signing +bicycli sts +bb ys +ball ston +ap arelli +amc theatres +ðŁ¤¦ ðŁı½âĢįâĻĤï¸ı +èĩ ª +ym posium +war heads +wan de +usc cb +un suitable +thomson reuters +syndro me +ste tson +smart grid +rut ger +r nc +pro gs +pre to +pre text +pla gues +pink socks +pathe tically +musk ing +memories of +list e +kylelarson racin +ja hang +hos anna +follow vintage +du ong +de arie +bol lards +bey blade +archil overs +antho logies +ãħ ľ +yo shin +yeez ys +vv vv +the mike +stone work +shuff led +par c +osp ina +mu mmers +mouse trap +maz dar +h sathletics +future leaders +f dl +don nar +disney sea +can seco +ab use +? "- +âĸª ï¸ı +าภģภ+yofthe week +uk books +to sh +to live +semen ya +seep ing +sa die +non toxic +n ss +ma dre +kin k +kim xi +ki ef +j angp +fré dé +fo low +etiha dairways +cro s +car le +bou gie +ak leg +Ø§Ø ¡ +w sf +vali antly +u pe +titus ville +th ill +sto wers +red wave +pan et +o ea +ne mt +marketing automation +m cafe +la borer +k tt +iron pigs +hero clix +gart ners +fran ke +fate h +carly rae +cab ela +c jr +brill ant +bj b +back court +babe sonparade +adri anne +åī į +z umi +uk smallbiz +t zen +rancher o +pho ton +p tc +nav ara +mea gher +maced on +juli ab +intro verted +gar issa +ga rena +f nd +eco s +do tie +diffu sers +c tober +bt vs +bi deford +al roker +ab stain +>>>> >>>> +ðŁIJ ĸ +w lv +the perfect +sun ami +retweet tuesday +ragn i +or ally +newsma kers +ne ster +lon im +in ra +ido psis +ham es +h sg +gra u +far a +fac simile +dar rel +d nb +craf ter +cla as +羣 åī£ +âĶĢâĶĢ âĶĢâĶĢ +vol tag +ts field +tech nom +t and +stal ban +st witter +peace makers +noo dle +newvision wire +monty python +iloven y +hein lein +hard son +ge sch +fri mpong +door ne +doctor who +derma bra +ban que +adder ley +ãħĭãħĭãħĭãħĭ ãħĭ +vol t +vine et +ok in +no pinion +ker n +kal goor +hender son +grey ish +dharam sala +curmu dgeon +crab be +cade t +boden see +ax i +arab ian +ðŁĴľ ðŁĴļ +world teachersday +wil m +thi eving +strol led +stre pto +sting ers +st aves +soil health +sharmar ashmi +prince strust +pontif ical +pi der +nu trac +meteor ites +mag z +ma zes +lu dd +grl probs +fresh eners +ev asive +endanger ment +diab los +coney island +can tata +bra bus +bou lt +boo oo +at rol +amazon as +al bom +ag oura +ad dress +à ¬ +worl de +whati s +ser rat +re pack +pinto fotografia +per dana +noo t +neuro muscular +intol erable +ib sen +grandio se +fe tz +e sher +drunken ly +d back +cadill ac +bou l +bex mader +ak al +âĶ » +z ir +win star +vent as +teapo ts +team hendrick +stick man +raw food +ol vera +octag onal +ms dyn +mo et +mand alu +ly ne +le im +kim brel +gill ani +ent in +eno ist +dog fight +by passing +brisban elions +bir git +au snavy +ake em +ac v +âĻ¡âĻ¥ âĻ¡ +âĻ £ +x tc +wer n +tu uk +tapen ade +smo ther +shar o +roy ale +reach higher +prin tables +paci fist +on or +o ken +mi ps +leg en +j bc +im h +hell man +gri z +cin c +carmel a +at un +accentu ate +ti pal +ti ghe +ss ds +si ba +sas aki +robb enedict +resign ations +quo c +pag anism +oshe aga +om ur +naw al +mariairene ali +mack lin +mach in +kriti ka +gran bury +glo zell +endo scopic +da im +d able +cu si +cn p +cl ann +cken ya +brown band +bo jack +ze hra +vote green +then yra +superhero day +se phi +ro sa +or issa +metro bus +magand ang +ke ster +good man +fox star +fer min +educ ational +eat well +colling woodfc +cap ed +bww ings +burn the +ban y +ag enocide +ad x +top team +sycho logy +spaw ns +she ela +ru ffin +ri mary +puni shments +pan ting +pal es +nun u +mal lika +lip man +let itia +jody highroller +id li +hot pot +franco is +fer rig +fa sted +end es +domain names +dissemin ate +disp leasure +dem ire +council ors +citi group +bougain villea +bau x +band mate +bal list +ðŁķº ðŁı» +we sto +vi als +uffi zi +sp ud +souther ners +solar impulse +shy ness +roo o +re ma +pin ker +pin ay +pal ati +pacific o +northrop grumman +matricul ation +master nodes +ln sm +laugh in +gw v +gui ana +fis cal +den ovic +clifton ville +cham ps +âĸ ł +ver hoeven +thr illist +ta vi +synthe size +stre u +shih tzu +roth fuss +rajkum marrao +per it +os orio +oak ridge +nov oro +musi al +mc comb +ik oro +i ap +gh mc +esp ero +der ri +be tul +abbey road +ur m +tony goldwyn +taker u +s onal +pet food +pare kh +or in +mo dao +men ino +love va +lince cum +le is +form is +dou lton +corn flower +ci h +chair lift +blog share +auto trader +amin u +air power +we ig +ta han +s from +re apers +peto skey +out i +meson et +lo ar +ici est +heal dsburg +he mant +gom ariners +go ong +for texas +for hire +for go +fa zer +cor nel +cancer awarenessmonth +can cion +bo shi +ab ena +म ह +umb ai +ta ffe +stas era +shar ps +sar torius +s vo +s mex +ro co +neutro gena +neg at +mac arena +light bulbs +kam ar +k aki +jor di +hc ps +fre elo +edi fice +eat er +dream job +disgu ises +cre di +ðŁĹ ŀ +w psd +talklike apirateday +rosh ni +rock lin +remun eration +re forestation +pp ls +philipp e +pan as +ni ere +ne geri +n sic +long beach +kenny chesney +joe budden +jap onica +hair ston +go ths +funer ary +fig ment +f ps +emabaras syourbestfriend +durg apu +dul ci +craw l +blo oper +blizz heroes +battle star +bamboo z +bail ando +aust intx +ë ¥ +Î ¿ +tide water +tech radar +t mn +stro op +queen ofthe +plebis cite +om ggggg +ngad c +nay an +mi il +md pi +loy alties +ili ad +il au +high point +gal ang +gail simone +fro u +epi der +dynam ism +crand all +cou ture +bilt more +adam o +ðŁijĬ ðŁĴ¥ +x prize +well er +way fair +sym ons +sky warn +men ards +ladies night +kaz oo +hin denburg +geor gi +fun da +d sk +bren ham +au ght +annu cci +ë t +team sheet +sho k +sa org +p silo +ow al +oure ux +ortho tics +ofex eter +ni ers +mil am +mcnam ee +ma def +laid back +l mbo +kiran bedi +jav on +ha vel +ew g +dol ite +dar na +chi eng +book binding +ban jos +ab gt +âľ ¯ +yoo jung +wee den +thick ens +the secret +t wa +swi zz +sor ter +sec under +resi dential +per y +palmo live +oculu sri +nerkondapaar vai +mu ddled +lif ton +knickerbo cker +ke b +gri dge +form er +fo gnini +en im +dream like +caille botte +bourne mou +bar res +abbrevi ated +[ $ +Ùĩ Ùĩ +ws dot +well fleet +we ss +uof g +pic oult +orange army +oph on +op aths +ohi ou +mar ton +l ory +keep sakes +jhb traffic +hutch ings +gendere quity +entrepreneur life +e pee +dising enu +dex trous +ðŁ¤ŀ ðŁı¼ +ãĥ© ãĥĸ +we an +t cnj +sunny side +south chinasea +solange knowles +serap him +saq ib +sa thome +re mit +r df +pan cit +our revolution +op r +my corrhi +mi ike +mamo vies +liberalis misamental +jam ila +hen dy +hatch lings +fy fe +fi aerc +er am +ecclesiast es +de forest +crystalli zed +beatsby dre +) + +é £ +âĸ ¬ +Å ¼ +west ph +un os +tu olum +stra hovski +stop watch +road tothe +pl unk +non stop +mohand as +ma saya +lik ens +leon ora +ide alism +half pipe +hak flak +en ji +desi igner +co si +bro gue +br ith +bil la +yam mer +xi u +wait angi +vell yn +temp us +scot tw +sal ukis +ren ne +rec ou +r ft +qe ii +pun an +por chetta +ot an +malcol mx +leg azpi +lady birds +ket ts +head line +grey friars +eu council +eclamp sia +bri ghts +balik papan +archie comics +a ok +Ø ´ +vs det +swit ched +sw it +stre aker +st ela +sojour ner +sam a +re ham +rak shi +prit chett +modao zushi +leaveno one +kai ley +jo sie +har sher +ham esha +hal ston +genu ine +gant t +for rent +f me +exfoli ate +exc o +dru sh +di um +chau d +carri gan +av anti +ðĿIJ ļðĿIJ +woo oooo +twit cam +twil son +schul er +pump er +pro ve +pd k +moti v +mary j +l mb +key blade +jam un +invicta fc +helen zille +gome z +ge co +fi ero +effec ting +disra eli +diction ary +core tta +compul sion +colouri st +at ella +an ant +ah at +ðŁıĥ ðŁı» +âķ Ń +wing o +turtle day +sw k +sv k +sun less +stay woke +starwar scelebration +ss k +sam bal +por gy +pollu ters +pedal ing +mo een +ming gu +mb led +lar ose +idi opathic +holy well +franco phonie +felici aday +ear piece +citro ën +car ies +business growth +bu bs +bree zy +big thing +bend tner +bank head +au ssi +arab idopsis +afa ith +upd ate +side winder +ser p +red hot +red bone +rash mi +radio x +pom pey +news agents +ne sh +kui per +ko td +karl lagerfeld +hun e +har po +frau ght +fe brero +contempor ain +ben q +ban nock +b db +aus law +annihil ated +acquaint ances +x n +wood shop +stdavid sday +row ski +re le +par acet +miro la +man college +ki sii +ken ora +inspire them +hor ati +hodg man +hal ong +fm ri +eto wah +dol led +asap ferg +ab ac +âĨ ³ +tre au +tram ps +tn b +time in +thereal luke +srk universe +skr tel +shu sh +peep les +ni yo +lom ond +lew ick +il over +hy d +hand lettering +good night +givesyou wings +giuli ano +galvani ze +forget ful +fill on +en q +echo ed +dou sed +card holders +bel ve +ar leg +af o +ðŁĽ ¬ +ì ½ +water craft +tw omen +tu bu +tren dy +ton ibraxton +thermom ix +straf fic +si su +rac lette +phal lus +n sic +m hl +ke zi +irish water +ido u +he igl +gc w +eman ating +ema scul +elias sports +con fers +bay ev +atch ison +are public +ÛĮ Úº +weare bc +van te +un lawfully +typic al +thro ck +sumed h +pro mul +pre d +phan atic +pen ge +new born +moor park +lu ang +lock up +lind as +kir sch +is wonderful +ger st +g pr +fast post +diabete suk +dc p +cy st +con nell +bo bo +big gio +witt genstein +victori abc +vi ajes +the bes +tar r +re ignite +pon yup +out performing +my be +kr ingle +keepitinthe ground +interro gated +ghos thun +gant eng +dio des +dhar mamovies +devil man +carnegi em +beu lah +ant or +ðŁĻıðŁĻı ðŁĻıðŁĻı +voten o +u icide +the social +stag n +st outs +soul music +ratt les +qu be +pru e +n online +moon byul +magni fier +mac neil +lil ia +kon stan +ily as +ik shank +hen stridge +gu cc +faze up +cor relations +cham bered +caf a +braun schwe +bb cy +b indi +am mi +ðŁĴħ ðŁı½ +vi vor +vare se +ta pir +spe ach +schiav one +sam buru +rose marie +q ms +philli pe +over cooked +on point +mtv teenwolf +mo cca +le febvre +jess y +i gen +ho to +head scarf +hazal kaya +gar cons +dru mmer +cur ricul +ap hid +am eri +ah ps +о Ñģ +whee zing +tam pered +soul j +shaz am +pap ill +nadi adwala +mor ts +mil ag +joburg za +jen nette +in at +hoe ffer +ha ylor +gol dust +fri uli +fertil ity +con spire +co ty +cityof joburgza +ch ac +bu bu +ðŁĩ¹ ðŁĩ· +will and +white washed +suri yaf +sun burned +summer vibes +sr g +si re +se gue +pur ve +po what +ou ster +opp ress +o donnell +mur i +market watch +ka ir +jazz master +j lc +hu ss +ger ais +elo g +e ject +cho sen +boston terrier +baahu bali +ઠ° +xxxx xxx +ton en +the family +sub strates +savi ors +sand banks +quel wine +pel vic +one onta +nur ul +news live +n pb +mat ar +le eroy +jmi quelwine +char ley +chante relles +brack ley +art as +ap f +aene as +aap tards +ç ¬ +tro po +til ton +tan u +ster ne +stefan ia +scar sdale +ra bia +post mortem +pen ob +nk jv +mu sha +mor rill +mal functioning +jag dish +individu alism +im presi +im at +husband ry +hiking adventures +heritag elottery +free assange +diyar bak +cro que +bear dy +west ley +truck suk +tran scribed +ti vo +shi khar +she f +pour quoi +pinck ney +nam joo +lexis nex +ladi esof +kun d +keep on +inspiredby pets +he dral +ge ss +fry denberg +dominic ana +def y +cour gettes +bnpparibas open +all size +ad lington +absc ess +ve toed +v log +us opengolf +tin ley +tech i +strictlycomed ancing +so kol +sil ences +pic u +p nd +or un +or l +no th +meet and +jennifer lawrence +huay ra +hu evos +fant abulous +fa ery +but land +bul lah +balth azar +ba ster +ar pels +v hp +un sightly +t ny +sag amore +ri jeka +resc ent +pokemon sword +mar rone +mag alona +kra bs +indic t +her mit +hem phill +erkenciku ÅŁ +e dem +den zel +d jer +bre mer +blac kie +bir nie +big boy +be si +arabe sque +aap ke +a erie +à° ¸ +ॠĪ +ö ping +week long +ur ination +un bothered +tropic a +tb k +super saturday +si rac +scri m +ru an +qu als +pe avy +ow w +nu ova +new menu +little things +lepidop tera +kil ts +ire v +frat er +footh old +du tyfree +corrup ting +cor aline +conven or +consul ts +cl amp +carrie fisher +bra himi +br annon +bab by +ap ics +an thea +à´ ® +ب ÙĬ +wou ter +west allen +vi bra +tsun dere +tin isto +thic ket +ter io +su z +sarah k +ph lox +nick els +nat sume +ma sandawana +joh na +i fl +extinguish ers +de spe +cunard line +cor ley +class man +chang eling +bio logy +ap supdate +???? ????? +: ** +ðŁĻĦ ðŁĻĦ +w fa +to we +the dress +the day +ten emos +so viet +sam ine +queens way +pho tonic +n annies +i ae +ho xton +hel met +ha be +exam en +ethere um +e ks +de ion +day project +ball mer +as me +aber dare +~ ^^ +wy clef +u wh +the magic +stan dish +st ich +ss np +rc navy +pile up +pedago gical +pav el +p ings +n mb +keyston exl +gin ni +gam ers +fer rier +ex if +du plo +dillon francis +dash berlin +chi vers +carm ella +bre chin +bloom sburg +ar nt +aji thfc +ðŁijıðŁı¼ ðŁijıðŁı¼ +ðŁı Ļ +ìĿ´ 민íĺ¸ +the alex +t ance +soho theatre +smart city +skyline skaters +ro atan +nu vo +nouvel les +ms j +jave c +is let +inspirethem retweettuesday +gov au +ge ver +g ling +dermabra sion +corn flakes +coo gee +ck nw +chul z +candy land +ðŁij© ðŁı½âĢį +wel k +tr onica +tex om +supercross live +shin ola +san at +pav arotti +pan cake +lunch break +ic g +hi bachi +head room +han gul +gir aldo +gen isys +elo pez +bed time +bar won +akin ator +ahlul bayt +í Ĩ +zac brownband +w cu +stou demire +states ville +ser p +se dia +scru b +schle singer +rick ard +refe c +propri o +pro cycling +ou ya +ma ston +health tip +h mm +gradu ations +for tify +fitz simmons +far oe +fang ir +collu ding +bit umen +aram is +ðŁļ´ âĢįâĻĤï¸ı +zig er +y ell +ve ttes +tree house +theor ville +telefon ica +sub version +sizz les +shi an +secre to +scot national +sco ps +sal ley +pflu ger +peace ful +ning bo +mor kel +mo ise +kra k +kn itting +hul man +gwyne th +ge z +fe c +f ête +c aceres +bjö rk +at twood +as cor +armp its +animalsin churches +& - +̵ Ì +wales rallygb +tink off +tb t +shin ers +rock man +ro skil +ram ped +ra bil +om agazine +nu st +ntl museum +may war +le wa +huer tas +ha gley +frigi daire +flori das +bu dg +brock ley +bly th +am ath +åı ¤ +zin nia +wyn ton +work men +wag oneer +ru stom +resi stor +patt ys +new suk +nan di +mock ups +laid back +jer rod +j oun +inglori ous +helve tica +go transit +chint skap +abdul rahman +ðŁijĬ ðŁı¾ +ðŁ¥ ĵ +z adi +wonder kid +wad er +ro vell +rafi que +pi el +or gre +o da +newyork redbulls +negro es +mother well +li gat +ki a +institu tional +im pulses +epi per +cu ba +commiss ary +big news +az is +arse guidores +wo hl +west mount +voy ag +vit i +valtter i +tinisto essel +tan tra +sto ddard +spres ents +pre quels +pran ked +ne gril +love eeee +la kh +jetz t +hel vellyn +har grave +dun barton +buck scounty +bauer hockey +aw b +asi fa +as v +art print +ar al += # +ãĢ °ï¸ı +ô ne +verme il +unfur l +tc disrupt +sat ch +sapp hi +ou da +oh c +na sties +man liness +ky w +jo sue +inter op +import ation +her alds +f nm +education forall +dimitri vegas +de ana +day yy +datam anagement +cy no +ct ags +confi dence +cho ge +chang sha +chab ot +bi gh +beck i +bae kho +b enda +ðŁĶ¥ ! +z ong +ward ell +u dc +ticket master +the ss +symboli zing +sch muck +salt illo +rive ter +pre history +pe ven +pain relief +over powered +mis understandings +mark tremonti +l ats +kemp t +ili ve +h sk +gir th +eve rest +dol enz +doglo stuk +cost adelsol +che ep +bc age +banan arama +anti etam +ì§Ģ íĽĪ +thi every +sp unk +skel ter +secunder abad +sav arkar +re upload +pt k +pelle tier +peep ers +our self +oku ta +never know +mitsubi shi +le dges +john stamos +hindr ance +harmon ize +fau n +er ases +duncan ville +classi fications +califor nication +barstool sports +ðŁĩ¸ ðŁĩ¦ +¦Ī ìĽIJ +vs ne +vil ma +v ater +think big +str al +sta thletics +speed y +selfish ly +sad dens +pride inlondon +pressuri zed +pre ece +nicol l +na ï +mm pr +mj f +mega watt +l zzy +l ks +hou d +fi zzle +cole optera +ch aca +carcas ses +yo kai +ym phony +y one +ww j +wat cha +vir g +scep ticism +rc ti +ra fin +pic ts +patron us +ni fty +mess ner +merry gold +ma hila +lor dy +hou sley +hom i +guadag nino +glo sses +gim na +fil my +di ssing +daniel j +ci f +bad ri +adju vant +trek bikes +too funny +than g +ten ney +stri b +sin ab +ru sev +rhe umatic +residen cies +ren jun +pathan kot +mil ena +lin dos +libr is +le mma +k age +hy poten +hur dler +fanci er +expo west +aug gie +ar ted +an w +accur acies +a ek +à¹Ģภ£ +ye ahs +win some +un adulterated +s nel +publici zed +pren sa +pos y +patri o +o cre +ne cker +hann a +go lightly +glenn beck +explore r +el ita +di sta +delephant day +debon air +dawg pound +cone jo +cc v +brick man +bc it +b pt +alli a +ab dn +ãģ Ĩ +âļ¡ï¸ı # +wet suits +vad os +thelasto fus +sun room +sin do +ser re +rob zombie +region ally +ra uch +prs guitars +on elast +no ct +nay apakistan +mu q +mo vin +ma ite +leavenoone behind +lake head +l vs +jau me +gre itens +gr anda +firstrespon ders +down beat +di mash +cy nd +ct c +crimin alization +chriso donnell +ch b +c ades +us al +ting gi +shon dar +s rising +russell ville +rp crd +pu es +ntlmuseum sscot +nick erson +mika el +mand an +mainten ant +listen live +leader shi +ic hin +hugh ie +hod son +gun j +first time +essendon fc +d apps +crad dock +by un +bu ehler +as signing +antiqu ated +ad dio +ab acha +/ âĤ¬ +ðŁį ½ï¸ı +è Į +zh ny +race horses +ob r +nor co +my cen +mur u +len c +ka en +j mc +j dr +iran talks +icha bod +encel adus +emmy lou +doper ahouse +dige sting +day sfor +cher u +cast elo +black book +al mirola +Í¡ °) +ze iro +xi o +un altered +tra desman +tiber ius +span ked +sha ho +sal len +rabo bank +ma der +ke ren +irresi sti +gan on +g by +far fromhome +ever e +darren atics +chennai yin +cedar ville +bo stic +bla det +why te +wa ig +vi dar +urbandic tionary +tal c +stret ford +som bor +skir mi +scam ming +rec ali +pic tou +p lies +nil erodgers +locomo tion +kar don +kag iso +iz h +hon iton +ho wie +gas ly +g ci +ent endre +b ami +yam ig +v st +tran scanada +toend abortion +spondy litis +sky rocketed +player one +oscar del +offici all +nu ms +mi umi +lo vi +land locked +ky les +juli eta +jac ke +hard ing +fine baum +ess ar +emptythe tanks +dun gannon +desp acito +cul ly +cow les +clover field +clean ses +casca is +bus k +be same +arl berg +al bie +ag onist +wolver ine +valtteri bottas +tomoda chi +the wild +the play +termin ating +tan may +tall ships +ta queria +stonebwo y +sta hp +sou rav +sh allows +ra ison +pan day +nam ath +mumb aic +mor ricone +medi ators +lon dra +h ould +e mus +demi god +dee per +deduc ted +cru ikshank +correc tor +chichar ito +cer cle +backthe brits +asper gers +as aurus +acci on +w ylde +un dress +tro ms +tee hee +raw son +pollu tes +pi ri +oro so +ni mmo +me taco +kill y +juilli ard +iihf worlds +hockey canada +gusta f +ge ddy +faul kerson +f sog +elizabeth town +crowd sourced +cnn philippines +ba aaa +ash ington +ap ni +aha h +ì ħ +udhay stalin +tra eger +te rest +spi ros +so com +pr or +petre l +or ita +not ables +nai jap +monc ada +mol t +mo sel +mediac rooks +kari joys +k ente +ig man +heal thre +goo o +fro sin +do ji +dan is +cur i +creep show +creati vity +cliff side +chil islove +cham ba +cele ste +be tro +aven ir +are se +ðŁĺĤ ðŁĻĮ +ìķĦìĿ´ì ¦ĪìĽIJ +u da +swimming pool +st impy +se ol +sar di +redu ce +passion passport +pappar delle +nit v +new collection +mil burn +make my +kam au +go friars +en core +ellen berger +den on +dal by +cri pps +cooke ville +be u +barbac oa +ari ane +ðŁĺĦ # +zi le +world music +wo wing +w br +w ades +strogan off +sax e +s mee +myri am +moon light +mo ko +mn dassoc +mirad or +lobla w +lam beth +jeff sessions +har una +h yeon +glu ing +game jam +fascin ates +donagh y +compens ating +chennaiyin fc +cas se +bi um +au die +anag an +ag d +* - +the pink +sto y +sli ppy +scham pion +sag as +ra sia +ra az +non ame +nb g +mer gan +marin ating +kr c +know sbest +is enberg +fa king +er land +day swith +coloni zed +at orian +amer sham +:) :):) +ðŁĻĪ ðŁĻĬ +ðŁİĦ âĿ¤ï¸ı +yas por +van s +under class +story boards +so true +si bos +roy ally +pi dgin +not as +mand rake +mal as +ling ual +kari us +k ry +it sabout +hugh laurie +hist sci +datac entre +d hon +bro iler +bis was +basti a +as pr +am tv +ac ry +ê²½ ìĪĺ +wol fs +ve sta +tor ship +t po +t ela +sg dq +san deep +sa ia +ru sse +randee phoo +py jama +pre owned +pax man +knowh ow +knop fler +institu ted +home building +hol sters +end polio +dun ker +ds world +do sti +dixi eland +berkham sted +bat son +bal ert +amand apalmer +all time +al ys +wi relessly +thank ateacher +superst itions +sec u +ri vets +pole star +photography isart +penguin random +olim pico +o cal +nor ovirus +much hal +mlp season +michigan stateu +matryo shka +lu ms +lu ann +kuro ko +hu th +far hank +et as +do gged +di dy +dent ity +dark wave +cruel ly +cr inging +capit alizing +bizz are +beach boys +american gods +al aves +zach braff +un important +u vb +the wrap +repleni shing +pin ang +pi guet +pha blet +per spex +pelopon nese +pe tere +paracet amol +marty n +lat ingram +ir ala +gi ada +gemin itv +gal ahad +ery thro +el stree +const antino +ch ali +car den +bro wer +yu chun +world wetland +vibr ant +uri el +tim ms +tanyabur r +tablo ids +pau lus +para ben +oz plants +manitou lin +intrin sically +i ros +hol by +gaz es +food bank +flu ted +farhank virk +co ster +brian stelter +bil o +ben atar +arch duke +well deserved +ven lo +v á +un informed +tre m +trading cards +sp lu +shondar himes +san ci +re vision +punctu ated +pott stown +ot rends +nishi da +in za +ig es +g music +firstdayof school +espo o +cath cart +c aya +ast ounded +app el +am ik +action bronson +ðŁĮ ¨ +war craft +w mmr +super models +st ich +spor k +sc ituate +pal er +leil ani +lant ana +la fit +kar th +horn church +gat land +fir ming +fal low +den za +de vising +day er +cher ubs +cab i +black comb +athle tico +any an +anten atal +å ĺ +Ì ħ +wi ese +vig no +tattoo ist +s media +s mbc +run way +resur gent +re plete +rd v +ra bly +r mp +pit ty +pis co +pan erab +nj e +lets ride +l ally +k lay +imbi be +here tics +haw kes +go bills +extra judicial +car port +bird sup +bf v +b sy +appointe es +è ¨ +ãģ ¡ +Ë ¢ +z iti +w ence +uuuu uu +stev an +skag en +sech skies +rin ds +pu ggle +oni on +ob tu +mer ito +kis sel +king maker +goo f +fab i +ex alt +drex ler +co del +can io +c sir +brook land +bre c +as king +as ado +animat ronic +andre am +alleg any +acces sto +yas u +y cc +stin the +sch aaf +pati entex +nathan son +mc vie +matt goss +lorele i +kom bi +innocent ly +illu si +hosp ice +gr dc +cw t +coronation street +c end +bi ddy +apprehen sion +anime art +ancient aliens +. âĢĭ +âĺº âĺº +you version +voteredto saveamerica +voet bal +ur c +u gs +su mn +self publishing +ro el +ref illing +re generating +peninsu lar +parker sburg +pan sexual +music uk +hus kie +glad ness +endo thelial +earth en +dram atur +bar negat +aq ha +ani tha +and al +al ag +ye quipment +un failing +tu dyk +ti mbs +th old +stra p +st ly +single handedly +sid har +red hawk +power ful +pou lton +phant as +maur ya +mal ai +load shedding +i acp +hamp shire +h tl +girl crush +fy ne +found dog +do wer +ander ton +z ink +yez idi +world photoday +whin cup +uu tah +tri ggs +sig nups +reen act +rafa h +n phc +min dedness +mc entire +kru pa +kni ghton +in town +grate fulness +gi one +en dive +c ji +b ws +are wa +allank lownz +. ") +ðŁĴ¤ðŁĴ¤ ðŁĴ¤ +⼠· +zombies quad +v fx +turn stiles +ti guan +si kka +shat ru +sel o +salv ad +red shank +r football +one ys +obam agate +kal in +k con +gree ter +extend able +beg ining +aver age +ari ann +ak om +ðŁĴģ ðŁı»âĢįâĻĢï¸ı +ðŁĩ®ðŁĩ · +ur p +uni fied +u mah +the ia +schre iner +schaf fer +san e +rejo ices +portlao ise +ntv uganda +min ke +massi mili +mari juan +lma oooooo +leis real +jo dor +immigration reform +illusion ist +i wa +h tv +fren chri +fe ction +di ure +dg ingly +d banj +criminal isation +cr w +bu p +bo ban +black women +av as +alpine stars +å ´ +you ss +y gg +tat ay +stop light +sick ened +sen de +sag ar +oculusri ft +oak ley +nor den +mash pee +liv uni +kam akura +heat stroke +gre ggy +fo ward +fc px +em ura +den n +dec ry +cap ello +buc s +bu ono +bal khan +zeit ung +younger tv +wee tab +si sy +se el +rv life +o ho +neutr alize +merri ment +m vr +long boat +lay in +kinder gartners +homeand away +historical romance +gen eliad +eric ho +asser ts +abhi man +æĺ İ +âĮ Ĵ +wordpress dotcom +winter watch +w gal +vi da +valky ria +universi ade +tt inger +tillam ook +tamar aws +stra us +she ehy +reit man +re de +pan gea +nhl playoffs +male e +ma ite +joseph morgan +ix els +ici er +fe ist +fair haven +epis co +dat av +dar ken +dal matia +cur zon +cityof pg +chec s +char lo +can tante +bas c +androgy nous +ac risis +ab iz +ãĤ¤ãĥ© ãĤ¹ãĥĪ +wh er +tube less +ten ant +tell tale +sun dog +so red +sigur dsson +sig rid +samsung mobile +rat ner +randeephoo da +quote softheday +pitts field +mu tombo +la wa +l sl +j vm +j ii +ing mar +hard woods +happy girl +grace helbig +glasne vin +footy show +fa wl +choo ks +c and +bo res +berser ker +ðŁĴ¥ðŁĴ¥ ðŁĴ¥ðŁĴ¥ +y im +vo ici +vai bhav +tt n +that cher +supri sed +su plex +si op +sen bob +sa wed +rr l +ri gat +q so +pro tour +pat oo +nat ed +mis d +mar ke +ma thon +ker at +hypno tist +go huskies +g atta +es group +em pt +dol ittle +del ancey +cour bet +confer ring +carlyrae jepsen +canon usa +beat ings +a holics +ðŁĺŀ ðŁĺŀ +verte bral +str ac +stat eline +regin eval +regineval casid +real blackcoffee +on myo +offici alo +mar dan +lar osa +k ny +in conveni +ilove it +hi ms +hand ily +fan n +fan acc +f to +ech ever +de activate +cn w +camis ole +ay outh +ar wen +all and +ab ot +ðŁĶ ĭ +wr gb +wh iny +toy fair +sun glasse +ste x +skull girls +rab bids +of hope +mumford andsons +mou ton +mel vyn +mc diarmid +le mont +ki ir +ja wor +ef c +dyr dek +broad com +basspro shops +ar bon +all kpop +ðŁĺİ ðŁĺĤ +yrs ago +un dressed +ste pson +short ness +se vic +respon dent +re decorating +pessi mist +ob en +ni f +lland aff +ke f +hurricane maria +h pp +grenad ines +ful fil +exam iners +equal payday +daysof ourlives +chec kitout +bell flower +befri ended +beaver creek +azerbai jangp +all sopp +aim an +whis ked +ur du +sho tz +seque ster +sau ter +pro tracted +oy w +oun cil +on ight +nit ride +nca as +kim brough +kh in +home forsale +gra ber +gol die +flui dic +erst while +der vish +con temp +child hoods +captain americ +cade au +c ft +bron c +bri ones +ale vel +agar den +adri atico +' ), +ðŁį ĸ +vivid sydney +up stat +re elected +re cluse +quad ra +prime knit +play fully +par ik +ny r +mill iner +mate o +kil ian +jin shi +ine quities +idin amen +flim sy +d wayne +bi dge +bare foot +bannock burn +amu st +ag ut +ade kunle +ðŁĺį ðŁĴĭ +wic ket +tur rell +tr all +stu ttering +smo thers +slu gging +sl benfica +sam ut +saj jan +re turner +ran unculus +ow asso +litho graphy +le son +jef free +ha das +gurud wara +gaspar illa +ffff ff +fah my +es ny +dha ba +de bru +content strategy +canonusa imaging +can tin +besto ws +benz ene +ame er +al mir +Ñ ħ +w uk +te ena +spand au +sl acks +shra van +se er +ru x +re can +popp unk +om arion +ob gyn +li ppi +i robot +gun ship +gu dang +good to +for gettable +el isse +dis lav +cc ma +bud da +brod sky +britt a +bon avista +bet we +arth i +ar vada +acor ta +ä¸ Ń +tweet storm +sal u +ro mu +perpend icular +partofthe pride +o dometer +moncri ef +mad lib +lur ch +kon go +jam il +injec table +hu y +gal lego +g afe +freen az +dunbarton shire +disney infinity +da han +bar ingo +ballant yne +ba je +al ors +ab devilliers +ðŁĴķ . +ðŁĩºðŁĩ ² +yar brough +whit erock +vee am +tw ales +sk ai +septe mber +ring git +red sea +ra fred +quig g +pollin i +ofthe world +madein britain +kz n +kalin ingrad +j sw +hawk nation +h la +glen rothes +em mac +ear nit +doug ducey +condo lee +as sis +ane es +acci es +worl dradio +veronic amars +tele prompter +tee public +sailor moon +rat ed +mon ast +mark it +makon nen +mad ness +leh ner +k ca +info en +gi ms +ge sso +fr amer +fi era +f sb +down ham +darshan raval +daddys girl +ab hay +vicari ously +twee p +tom aso +tik har +season ality +prime val +onec lub +nargis fakhri +me te +mag fest +fre ida +fat ma +donington parkuk +corpuschri sti +confe d +chuck ling +bridge town +b halla +anticip ates +! ðŁĺĬ +zon do +worl delephantday +wis ley +win c +unsig ne +su cess +ra gg +q ar +olim pi +linde mann +kali l +irrepar able +gab bie +free books +em lyn +e brahim +dam busters +cu pola +cham berlin +bro co +av atar +at albihariv +amar nath +af ish +.... .." +" ?? +wb ko +vel ly +tho b +streas ury +stop km +sec tor +ride sharing +plum mets +mill ville +mary beth +mar bury +mal ini +is chia +im pu +haver ford +happy womensday +gh ero +fo e +expe dited +charle voix +cc p +ca o +backthe birds +ab bs +ðŁĺĽ ðŁĺĽðŁĺĽ +Ùħ ÙĬ +walk athon +ver on +tra ktor +ton igh +tes ers +ss ons +sam and +repra p +o bra +nir mal +niq ab +national park +mat adors +mal evich +g sn +dun lo +dh fc +de tal +citi field +ce ded +cait lyn +ausv pak +art fest +appropri ating +all women +z ella +web hosting +to pra +sy s +spiel man +snapmatic art +scent ral +refr active +re frame +pat ern +magic rock +khush sundar +hemp field +gab i +g war +fort inet +dark ening +chick lit +cer velo +bag gio +ap t +ðŁĺĺ ðŁĺĬ +ب ر +´ ´ +wel ford +uck field +td garden +spi vey +septic eye +roll wave +reboun ded +raf ale +pu rohit +promon tory +plu cky +museumo flondon +mu fc +moon walk +le sham +kol lam +jessic am +head winds +fre mont +fla ked +fit ton +eto ile +brain less +be tel +ar be +ðŁİģ ðŁİĦ +âĢĭ âĢĭ +wei maran +wat ts +wagen ingen +w mo +tual atin +tro d +til de +strategi ze +stewar dess +stan sfield +propor tioned +per ot +official aldub +mun da +mo ong +mid lands +marine tte +k roc +ham idi +gri pper +gob bler +go ins +euphor bia +dreams cometrue +di adora +def lection +cyan obac +collin sville +claustro phobic +ce dia +cal lus +buri ram +as jad +à° ® +uc ine +tun s +tory canvass +sun tan +ste deli +sensi bilities +seed less +sag al +ruby rose +preten se +n fb +mon tes +lo sal +lar oche +kar isma +jen s +gru dges +fore al +excav ators +enni o +emboli sm +el dora +di able +cou se +cor ic +carr boro +aa e +a oyama +zo zeebo +zar co +val eri +uni ofexeter +tram onto +tra sk +t dk +subli me +ro ys +resurrec ting +pro vision +mari sha +mare mma +looking good +lex po +kutz town +kop itar +jo ed +jay ryan +inferi ority +hil le +gol da +fashion police +fairy land +ex im +euro pol +clif bar +cir illo +brit to +atul lah +agor as +accu radio +. » +wood turning +un disturbed +uk h +sp liff +sher rill +sh elli +sale stips +sa chi +s ld +radio logist +o sten +nan ette +miami dade +lat ic +kil roy +ki zer +kh en +ke shar +j ci +green building +g md +femen ino +empan ada +candle sticks +bye bye +bul loch +blo tter +around the +alli ving +wal u +um or +ther ton +tal war +ss mann +sha ile +run t +ro ze +p ander +ny lander +no zzles +naga i +maz das +martin i +ly ca +loving it +ki owa +eras ers +cas save +bis co +am ini +íį ¼ +ti gard +th ig +stateof mind +slu ice +sitting bourne +sham bhala +red list +quiet ness +o iq +nbas ummer +metax as +mat ts +mar ling +ma ad +li ed +j ina +inter laken +inte xas +hand shakes +gall bladder +g br +far relly +boston college +asyn chron +ar le +antiques roadshow +and ed +an ahi +ador ns +xi ang +world vision +wood hall +rutger su +ro main +pronoun cing +piec ing +nai vasha +mishaw aka +lamp work +jay ce +ivan hoe +indivisible team +idol master +gab s +final level +fe tc +f jb +di sses +decision making +cro stini +cour sing +car ves +an tof +wine spectator +ver it +un kind +spinn aker +sle p +seper ate +pre loved +ous mane +min econ +mal zahn +love day +lav azza +kl inger +kac i +for us +f ú +f se +et tore +deer hunter +cand ela +bobble heads +bel tre +ban do +bab i +b illu +acu te +z sl +wat ling +tele com +t gm +surpri se +super valu +sevier ville +sch o +sa hi +ren dang +regi a +perpetu ating +par veen +mood board +mer lo +me go +kom al +ki efer +in extric +i dar +hu ish +gon do +foot notes +cham bord +blizz ards +bbc africa +b fc +aq aba +ais d +ðŁĽ ¸ +wal nut +un selfish +uf ti +timoth ée +tewks bury +summ ation +stephen asmith +so dom +selec cion +ro ya +repa ire +prosp ero +pha i +ou ston +o zy +mel vin +love thi +lamp shades +kh t +k eng +ir ua +in cur +iam steveharvey +howe y +hom icidal +he cker +feed backs +du pon +de be +blood thirsty +ar ni +and uil +Ä « +y eng +we izen +springh ill +sp rig +sch ler +np bot +min aret +maha shivratri +littlemix offic +le van +lab ours +jj ong +iko shi +hy olyn +hat o +ha sten +d mn +cycl amen +chicag op +black heart +bl yn +barne veld +ambi valent +ðŁ¥ Ľ +w bal +tu ft +sun downers +subsi diaries +set tembre +rel td +plan ed +mar mara +mad town +liv uni +jar dim +jan is +harry hausen +eu a +est reno +do able +dissi dia +dis ordered +ca at +annoy ingly +al ax +Ä į +ww y +wr ing +ur ner +twee d +tw ire +thought fulness +sho ji +sar co +pho gat +ohio ans +ny rr +nov a +north westernu +nac ac +mour ned +mam mukka +mal tesers +lan sing +edin boro +dr ones +depra vity +conor maynard +cherry blossom +ch oli +biophy sics +asse en +( / +vi ento +sri man +sf chronicle +schol z +row lett +ribb on +ren ga +rec tal +rascal flatts +mi v +materi alize +mag say +koo p +invinci bles +imacele brity +hello ween +gor ica +gi ge +fire starter +fe p +enqui res +be jeweled +ang ana +albu mo +si sulu +sand paper +re designs +raff i +quad ril +over paid +n gw +megam all +mac ie +he avies +ha aaa +h itec +f dd +by catch +bla in +ax stv +ar ocks +ðŁ¦ģ ðŁ¦ģ +wor ke +ve stas +shin di +percep tive +p wm +ncss bethebest +navig ators +lu men +ligh tup +kak amega +jake owen +in conceivable +ha gee +green hills +got land +garda ÃŃ +docu sign +dalla spd +com mas +bra gged +biz arre +bat ov +ag nes +aam u +Ä Ł +ulaganay agan +s ited +river ina +palo alto +o shie +never more +n land +mc coys +maxim al +ho bie +h cg +frome arth +exor bit +exe ge +copy rights +clear field +clai mants +cau sation +bu stam +boo zy +bon hoeffer +bam m +aw ur +?! ?? +wholesal ers +super sunday +richar do +re packaged +pr iti +penguin ukbooks +pas aden +ot m +nigh y +mi ao +maz ari +ka oru +ju sth +incre ment +green man +glenfidd ich +for st +f ourier +este e +e speci +dallas news +cuad rado +c pl +bu chi +brace bridge +ben guet +bella ire +b heem +aro oms +abi ke +Ñģ п +toyo tac +thir u +team envyus +star sky +sol ent +smar ty +shine y +ric ki +penn sylvani +montepul ciano +me sports +kail a +j one +ine u +gun controlnow +go slings +foot fall +far rier +el ucas +el nella +de composed +ch andy +black ford +beat rix +alma gro +adden dum +ad ress +abduc t +vidy alaya +vent us +trol ley +tin tag +speci alt +roo sting +pur ported +pa sta +openstreet map +mu ang +maxim ili +led bury +kel seab +kat u +k weli +is ra +hoard ings +gc b +fu ze +friendship goals +cyr illic +creepy pasta +ce zanne +bon zo +bo thy +blur ry +aziz ansari +ami right +ys weden +woj ci +va shi +thevamps james +stee pest +shahi di +puneeth rajkumar +pack aging +over valued +mu tha +motor ised +mend i +la an +k old +jas pers +idinamen zel +i vers +gas ping +elec tors +dur rani +col li +chi est +ch utes +bmw motorsport +blo bby +wend t +week ende +us weekly +type faces +tor ts +spr i +prank sters +pancre atitis +pak ka +im pro +heart day +hab sburg +fresco es +expedi achat +car pooling +be jealous +a iga +ðŁĺĤ ðŁĺľ +ðŁĴķ ðŁĴĸ +ys c +w annab +tra ger +tor us +the bar +sy nes +swi the +subordin ate +sin clar +si ab +sel ing +scienti st +s rule +re told +r inge +profe ss +pra chi +nat al +ma soud +ma ble +lou pe +load ers +j wt +ice vic +hebri dean +fountain pen +fet ches +de ems +child labour +bo ren +adu ba +vi f +torpe do +sla inte +sar ada +ono van +maxine waters +mach u +intra venous +housel dn +gwang ju +geo graphies +gal eries +fein berg +e my +cross breed +cre ston +consisten cies +col ou +be mo +b hel +au tre +au ch +astro biology +air strip +ag andhi +advantage ous +! ðŁĴĹ +x ts +uzu maki +tin foil +teenchoice awards +tad ashi +sonymusic south +soci ale +se urat +san tee +re th +ppor tunity +newsad elaide +mol en +metallur gy +jamiro quai +ir anga +hydro therapy +g les +fran che +fra se +eri sts +dam as +biele feld +aller ini +ðŁį Ŀ +y ax +trans media +sur y +summer tour +su iza +si ra +sh ada +reminis cence +pro tists +o soy +nf ld +mar mont +magic johnson +lan c +jessic aal +hur ley +had leigh +ha dron +gui seley +fo td +b bau +au berge +acti vel +ye m +vac caro +under study +un fulfilled +un ca +su chet +seaco ast +ready playerone +ram ey +plussi ze +pai va +newer acap +min oz +m pe +li ske +legion ella +kom men +kate y +iv lp +in m +hr vat +finger ling +ea thealthy +e jer +disinfect ant +dark horse +cro que +cow bridge +ast an +ðŁĶ Ĵ +ðŁĩ» ðŁĩ³ +ðŁ¥ ¤ +ÙĦ ÙĪ +un clean +tuesday treat +transcri bing +tain an +sing hal +sher rie +shako pee +sarab are +s ward +ro ams +r ct +plane spotter +ol x +off ame +n als +muñ oz +me chs +maz inger +m hd +len ow +ku bert +know the +hann o +flat iron +er ys +en chant +conquer ors +cd x +bu shido +bonfire night +auto bots +audrey hepburn +as signs +ak ara +tit ania +sub han +stat oil +som alis +pun cher +pe sci +pal as +noir vember +mathru bhumi +li mber +fo iling +ffxiv snaps +ecoun ty +dou cet +deli c +ble tt +bar ham +aard vark +. ðŁĶ¥ +un affordable +um al +ty ke +the war +she eps +sc old +retin opathy +pol ski +l illi +k you +jan ina +indom ie +hor wood +ho gue +glob alists +era iders +embarc adero +co ddington +canvas sers +bird seye +bein sports +art an +amaz onia +am studios +allevi ation +alas kan +al vi +ðŁIJ¾ âĿ¤ï¸ı +ಠµ +à° µ +yen press +ud f +the golden +t kd +sequo yah +sap teched +ray na +ra ad +py ard +ph m +p yo +oli phant +morning news +mar den +mandalu yong +lu mina +irrefu table +i wi +e oy +di dier +desch amps +cornwall hour +brooking sinst +bor romeo +allthe time +adr ille +work spaces +train er +su th +stand swith +sc ola +ru mm +quag mire +pad er +ob or +nu er +motor ways +mohe gan +mi en +me mp +marke dly +ku chi +koth ari +kelseab allerini +gi ana +geom agnetic +fu m +fri se +en ick +di vide +cyber sec +clá sico +bro c +be fully +au stral +atu ral +yoko ono +university leeds +sti glitz +shre wd +restaur ante +oo ja +oh tani +monte zuma +mit i +marsh mell +lo zi +kkkk kk +gov mike +el ane +e pr +cra ved +cr anium +cc as +boy ce +bo gged +bill erica +ar sen +amp stead +ðŁĺĤ ðŁĺı +ðŁĮŀ ðŁĮŀ +z j +wo ve +win a +walla sey +w swan +tin ie +thr anduil +tal mud +stom ach +squ ished +small youtuber +seri en +salam anders +s ness +one big +lloyd minster +kim ble +kas sandra +joey bats +hamp son +gli zzy +gle d +gg j +es cott +erick a +e um +de gale +da che +confis cate +bul gogi +arthr itis +ali x +af er +à®ķ ள +war mb +vander meer +u in +so co +oiq fc +lu gs +ll bean +ke ma +k rush +j mp +hi x +flori stry +convolu ted +cle a +chil ies +ar vin +tin dustry +th une +syri ac +survi ve +spark lers +shaho ffice +sem ites +sag er +ry le +re kt +ra ita +quad ric +psilo cy +path ophy +oak well +ni antic +n acion +mis using +lpr tg +ler i +k music +jet ti +god wit +gn ition +fer vor +fel ter +fe mail +dream world +disc ou +destination wedding +de clutter +curly hair +ch hs +c gc +bournemou thecho +bil ge +ac ac ++ - +ðŁĺī @ +women shealth +wack y +van wa +twee tuk +te wari +te che +swal edale +summar ised +psych ics +par os +o euvres +mill ward +long march +ke k +ka sem +hower ton +g su +fon ds +de posed +crack head +bad en +arri er +ann en +ìŬìŀIJ ì¹ľêµ¬ +âľį ðŁı¼ +zax bys +z df +terremo to +tann in +se ph +rebec cas +prioriti zed +octa vio +i funny +haqq ani +eu m +ef o +dan one +d lo +cordon ed +che p +bel itt +anat oly +al h +ste iger +s friday +present able +mar ama +man on +ji th +jaf frey +ha sa +glu tamine +fre shies +foo ts +el den +dese ret +d drive +clear the +campaignfor leo +bangsam oro +angla ise +amand at +åĨĻ 羣 +wi spy +v fr +urban ist +touch line +toffe es +the ben +stri l +qubool hai +preci o +ox en +ov sk +nov ello +no yes +mar gre +lou ghe +jess ical +gid dens +gen ome +challeng er +caroti d +bly the +bl am +bi v +bam ma +bally castle +ac am +âĢ ij +zab aleta +wip wednesday +twitter india +tunnel ing +trans world +t ween +stilt skin +stab enow +sarabare illes +san desh +quizz ed +penob scot +pal ouse +pa an +of fi +mer rier +m we +k way +ia wn +em un +egg shell +cou turi +coo ker +class less +chi os +cag atayulu +bay reu +ap ie +an son +am stel +agronom ist +è ± +y gent +weare rs +vla anderen +very one +sp s +pl ers +nivers ary +neiman marcus +ma ut +la gers +kalgoor lie +gl t +ge ena +dictat orial +cwm bran +be ee +ठ« +w bal +vit ally +ulver ston +te tanus +tab oos +sthe band +sta an +sque als +seab reeze +savag ely +r mu +p be +n ke +jo ven +j mo +hypo theses +hen n +health ily +guil lo +feliz jueves +dn cle +de de +crossh airs +clow es +british airways +ami ka +alcar az +" : +ye aaah +wol ff +un reached +twiz tid +turn tab +sal im +read ership +quin ones +quanti fication +over lays +national cheese +low brow +lang ton +la fayette +horror art +gr ls +gib ney +bow tie +ble phar +bit co +band leader +anarch o +acker mann +๠Ĩ +wall ington +tab c +t md +sm ilers +ri pened +ra ging +li ri +lg v +kn oll +jak u +im be +elo him +dono stia +d hr +cyber aware +chit wood +ðŁijįðŁijį ðŁijįðŁijį +à© ĩ +trill anes +thought works +te ared +san gel +out shine +nr b +ni bbling +mueller report +mehboo ba +m jol +kali spell +inv ade +inf ante +iggy pop +high lighters +dd dddd +contra ils +coer ced +chil dri +caterpillar inc +cad dies +beef ed +bar ajas +aco in +a joy +ðŁĮ ° +wq ad +wn a +twis cone +suz an +sm kt +sche id +scham pionship +sav aus +sa hy +p sin +nj transit +nanop article +mine strone +marshmell omusic +lanc a +kings go +gas kell +friday feei +fel tham +draw something +cri s +casablanc as +ver ges +schwar ber +rr m +rise vans +revel ry +requis ites +prestat yn +ping pong +no fx +nine veh +napp ies +le up +in decision +i gre +ho ka +hass ell +hard case +gau dreau +flex ed +fat to +eber le +dissi mil +defin itively +cra ven +canu ck +best life +be better +am bridge +ach risevans +¸ .âĢ¢ +y aj +vi as +t sh +su ji +sar my +rose hip +radi ok +on gan +ner oli +mi ja +long sleeve +lis beth +er ocks +ef lower +doc ent +ching ford +cb k +byz antium +am r +! ** +ðŁĶ ĥ +ãĥ Ń +à µ +y ma +whit estone +ur k +theri dge +sandown park +p bp +nw p +no well +mr david +mill s +ma gia +little john +ku ra +ko ski +hur ston +g night +cor ina +com el +be fit +aro y +ab ney +. âĿŀ +wiki data +war minster +tro yes +todor oki +stat ins +re touched +pen ting +os metics +nin h +nick jr +min it +memory lane +man cy +l ce +kip choge +kal k +il hoon +ig ami +han rahan +fridayfeei ing +fly away +coldwell banker +co ady +cha el +bo gge +ar xiv +amar ok +af ir +acadi an +ู à¹Ī +urban outfitters +un spoiled +tab riz +sun deep +stom pin +ru ido +rep ton +re activity +rav ana +pre debut +na ito +mr in +mill wall +lind strom +ki bera +jo ve +intelli gible +inst as +indiana jones +hedge hog +fre itag +el ana +dau sa +cham ois +bil lowing +anti freeze +alice springs +ðŁį Ħ +yu su +wa xy +wa aaaa +vir tus +tin gh +soor aj +sh kh +sal aried +pray toendabortion +nor di +motor cyclists +malevol ent +ig lio +homo e +here we +ger aldo +fron d +floo daware +ep src +e prix +e bel +du pri +cu nei +americ ann +ðŁĻĮðŁı» ðŁĻĮðŁı»ðŁĻĮðŁı» +v sc +the si +ten bach +tel kom +span x +sp eller +ni am +nathan thewanted +man nar +m cla +l alla +ko at +kar pov +kar la +journey to +hu esca +ho ffer +guang xi +gone but +ek ur +egg leston +ca ire +bo hen +barr haven +avoc a +army strong +ano di +??? !!! +ðŁĮ¹ # +åİ Ł +âļ ľ +zi am +wa th +tun n +te p +scumb ags +sco ffed +rol land +right move +raim ondi +que e +pushawards jadines +notori ety +ninjatur tles +ke dar +g sg +fro wned +de bo +d da +court land +chi seled +and ad +af ri +$$ $$ +" & +Ø ° +zo i +w ska +tur lington +the young +ser ai +sec tarianism +re aper +on ico +om yel +nam ur +ink master +har vin +gle b +fatt ening +ehl ers +dwar fed +com it +cly burn +bas sa +ant one +altern ates +ðŁij ĵ +ðŁIJ ı +yn ch +tv week +tu ta +tomat illo +som mar +scho en +pv t +prop ellers +prior at +na stics +ma aran +lu lac +kin sler +ke mono +ke keke +grub hub +gridi ron +gir lin +fe hr +covent garden +boom boom +bianc adelrio +bas sin +abcf pll +ðŁĶ ¦ +yoland aph +wel ton +thel ord +ten zin +sav i +ren ée +r ne +phys ician +nu ig +nd win +michelle visage +merck x +measura bly +manfro tto +magne to +jae bum +inst at +in azuma +hurrican emichael +hali burton +g bt +disco vere +di po +cas c +blue bird +blu efish +at ali +art scouncil +andrze j +anaphy laxis +american made +albac ore +ðŁ¤ij ðŁ¤ij +zav ala +vacuu ms +shopee my +sch nee +rez oning +play makers +pin ups +part out +narcole psy +nai ro +miil kkk +man owar +kis met +hau ght +fish el +f anime +er ici +ed sel +dutt sanjay +dun ce +de music +cer novich +bor at +b days +ang li +w tp +souvla ki +rec ti +nah in +lovewhereyou live +li gon +jo hal +im movable +hil son +hesper ia +gn at +f tt +ess el +en amored +elton official +ed a +dee speak +d wa +d don +cumu lonim +be avis +an ji +af lgf +ðŁĴ« ⾨ +ðŁijĮðŁijĮ ðŁijĮðŁijĮ +ðŁ¦ ī +оР» +wm police +wan ita +v sd +uro pe +up ton +sure ty +stef on +ru sten +recir cul +press uring +p ela +mc alister +lin na +l nr +kri swu +kim jon +ish ment +industrial design +hr g +hi mesh +fer ri +del aunay +carbure tor +blu en +a home +ðŁĺī ðŁĺį +wish art +up o +tu it +tri ennial +sema show +ram iro +pur posed +private eyenews +plough ed +onthe beach +minne haha +man ne +inj al +gwend oline +geor gel +fle mington +ed b +di ouf +creation ism +cran ford +bin du +ìĹ Ĩ +è ³ +yn z +uk g +trans iting +thr itis +smu dged +si en +shin in +sher rod +rus set +roman ceno +repri sing +plan er +photo bombs +oc f +mo dic +kejri wal +k umi +hemi spheres +goo devening +financial planning +dy fi +distr ito +cor ian +cel i +bur nished +aw el +art ph +ag ging +ud l +thedaily beast +tax ic +ta kuya +stair cases +stag ger +show me +pre ble +pitu itary +pad gett +no bun +maj o +lumber ton +lime house +leagu ers +l sat +jam an +it isation +hedger ows +go pichand +g eld +doub lec +de bby +daily qotdapp +cu neta +chri schristie +chain mail +cake p +bir ks +amy klobuchar +ðŁĶ ŀ +âĢ¹ " +za ar +town hall +topo logical +timmc graw +sel on +sbu x +quick sand +pin nock +o strava +mp ath +le lo +kar ang +kag i +judge ments +ju tsu +inf antic +go kingsgo +folk art +fli pit +ever grande +dav el +cut ts +custom isable +con c +commit tee +blueno se +belfast giants +barn acles +bar nhart +b tech +ar mani +an adol +agh an +ag ie +ê² Į +é ¢ +yorkshi rec +vote uk +tur no +ther mic +stu di +sre eni +soci ete +sil ken +si rs +sami yusuf +qu acks +pren tiss +national nightout +mp k +mono logues +mo hawks +ma vi +ish tar +ing our +han kins +g force +embarrass ingly +ek ay +dil i +de boer +chester tweets +ca pper +ash mole +app or +al yankovic +after taste +(* ´ +ãĥķãĤ § +wave form +wa hid +un recognizable +sos fam +scien cer +re la +po thead +nu buck +ni st +nai ja +mot ör +mo sses +mc quarrie +mak ro +m provement +luton town +ih ra +hay y +first post +et ting +dance day +cough lan +car ti +ber cy +barca stuff +bal ms +axel rod +ar trave +amit shahoffice +âľ ĸï¸ı +Ø§Ø ² +ty c +speci es +senator collins +re wire +pepper corns +mo sman +mer ly +lo ssi +kany akumari +health ful +he pp +g wc +debr ame +coor slight +centrifu ge +budd has +bed sheets +bate son +b ingen +anurag kashyap +ãĥ³ãĥ ī +âļ Ļï¸ı +yo gesh +y alls +wh q +wait ress +tortu gas +stir rups +still born +rcb tweets +pft commenter +pc u +ow y +neer aj +mar yanne +mar ga +let us +le chon +kin t +joh ny +ja hn +ing apore +hou lt +ho dak +high ball +hh h +e fi +dosto evsky +de th +custom isation +csk v +clu bbers +anto ine +aci ously + ¢ +~ âĻ¥ +yo gap +w era +vishal dadlani +st ena +quan to +poyn ton +open university +music city +maz atlan +mag pul +lavor o +lam as +kar ak +ho wick +her me +fore told +daw ah +chak o +bron zed +bron cs +bro king +beard foundation +ba sho +an museum +a hino +Ñ Į +wood worker +wood s +woo dro +winkle voss +ve toes +tb buccaneers +t lc +spen ser +s mike +prof briancox +pomegran ates +o chi +night ers +mete ora +liber tines +kamchat ka +hel ter +grass fed +god liness +germin ate +gab o +du pes +dead heads +croatia fullof +coach es +cas sand +bram bles +biz ness +bath ory +aw ks +at ma +ðŁķ ¹ +visit canberra +unear thing +rott nest +ross iter +r tt +pau lg +moul trie +loan ing +great ormond +gil ding +ger tru +gal era +discred ited +d fe +cand ler +ani ah +ah sa +ab orig +yamig autam +y ie +the original +sun times +sh n +sal ahu +robin hood +re introduction +kap o +jan el +it each +intri gues +fas s +enter shikari +en dow +doyour job +can ova +au tres +anglo phone +ab n +ðŁ¤ Ľ +~~ > +v ally +stromb oli +star fox +smir king +s su +ring tones +ragha van +po sta +news x +mc cam +matty bra +jag ex +itali c +i see +goldeng ate +girl probs +gipp snews +fin borough +dun c +de formity +clam ations +chand an +bu ra +bree ches +ash ford +anti pasto +ಠ¡ +za hir +we rent +ty len +th inspo +ta kas +t sen +suwan nee +sor vino +sold by +sch amber +per ty +pas orob +only fans +mic hell +mc quaid +ja und +garri do +franchi sees +foo ds +entit lements +elector al +cy rano +convo ys +christma ses +bapti sms +ðŁĶ ½ +ðŁĮŀðŁĮŀ ðŁĮŀ +ÑĤ а +zipp ered +tu li +speaker boehner +slam mers +shake el +ser bs +potter head +poe tr +pen test +p noy +ophthal mic +ng u +lock herup +lance bass +l tz +in numer +granger smith +facul ty +du four +der ham +decou page +cull is +cri ps +cen tos +blackcat appreciationday +bal lester +and juliet +weare in +v ax +v ata +und son +tem er +ta ichung +sun bathe +sni ffles +re painting +nore aster +nel spruit +master s +ineffe c +har as +gn ar +ff g +end ing +em ple +ei shq +din as +deaf ness +cor in +ch g +bly ton +ann coulter +ac utely +ðŁij ¿ +walk to +wal o +shire en +restra ints +poo ches +pdd ancing +palati al +north westhour +motiv ators +may ra +j ury +in me +field park +exuber ance +cre ased +cond é +c gr +bor ing +antic li +am av +ðŁĺĤ ðŁĺİ +ãĥķãĤ £ +à± ĩ +wol sey +tu gg +so ws +pick wick +panther nation +nell ore +mul sanne +lime ade +lee ann +hul lar +here foryou +he as +gi v +fun with +fi real +fascin ate +dream weaver +daniel howell +cushi oning +cou leur +birdwatching mag +bar at +b ation +ail y +acknowledg ment +âŀ ¼ +س ÙĬ +z ine +ws bt +ur thy +u ce +trouble shoot +tin os +super natural +states boro +she ree +seaf oods +ori flame +neu man +nau d +n la +n ky +model er +mi av +le ck +intu it +hyper market +his sing +harbour front +gon ski +gam ay +dok ken +de construction +cute cats +cran field +confeder ations +co ex +cd h +car lito +c moffice +bar ga +af fa +yy am +whi shaw +trigon ometry +tal ento +rothe say +pet m +pa via +lug nuts +lu kash +lash ings +kali ko +fe men +e disto +bike shop +ape l +anc ou +zin hle +veu ve +tu ohy +to td +sue ño +star ck +smo del +rigat oni +prostate uk +ple bs +nike basketball +narasi mha +mu sty +mehboo bam +mano euvres +lief eld +invictus games +infe cting +i ber +hor sley +ho om +gau tier +fat tuesday +f pm +ezral evant +ex x +ec ity +derby day +cali gula +boc elli +besse mer +bed bugs +beat cancer +at m +arom agna +an ica +ðŁĺį âĺºï¸ı +âı ²ï¸ı +ा à¤Ĥ +w bur +ul ere +sk ap +ration ality +preci ou +pay ee +ny it +mor tified +man us +lon gue +lets gov +kerr ville +hitch hiking +good stuff +fy ingly +flood light +feu ds +ero ad +end as +donny brook +declar ations +blant yre +balloon fiesta +aki ha +ver ia +su so +sportsm ed +snoo ker +science day +reboun der +panerab read +lon ged +klez mer +inec nigeria +hol ker +grand addy +for no +fast ening +e migration +dri de +dis location +davidar chie +dar uss +che viot +bogge ss +barn stormers +bar tel +art life +angel fish +womenin music +wi union +travel alberta +ti zen +st pete +sp amal +sexy saturday +screen awards +sch rute +ru mple +om ele +nase eru +nar rati +n una +n mu +mo slem +mc minn +madeinthe usa +lu jan +kro enke +he pa +haru hi +gri pe +ear then +diverse books +dan go +ber rien +b mb +atar decer +ðŁĺļ ðŁĺļðŁĺļ +ñ ez +yo b +trump er +soci alist +sig an +scher zinger +sch au +refurbi shing +ra gga +qu ero +ncle x +massimili ano +mand alas +jaund ice +is right +ir acle +hrd ministry +grand er +gra ble +f bn +desp atch +bul bul +brasile iro +bor age +bend is +bal zac +baad shaho +aku lam +a ahh +ठ¿ +zack ryder +wr dsb +wai mea +up to +tech review +tar k +sp ick +scaf ell +sa chets +rod denberry +r ø +pl cs +pac ey +mono type +lot to +lich ens +le pto +le of +just the +juli ag +j rs +int c +in deci +ic dc +he ze +di anna +dhru va +dab ble +cumulonim bus +clairvoy ant +cat on +bu mi +bl on +ar ai +a ich +. âĻ¡ +ðŁĺģ ðŁijĮ +ðŁij µ +yar nold +umh langa +tra itor +the beer +sun aga +scar am +regar de +not to +mil ani +m me +le man +ko by +int u +hu li +energie wende +dn v +cor tona +car ted +calaver as +c scs +bro il +break dance +birthday party +wardro bes +w lu +v au +tw t +tigh test +thcentury fox +startup week +sports india +se hir +sch mu +orient ations +nv leg +midland shour +ly mm +k ps +ish am +gish wh +geode sic +est ado +emer yville +du lehill +dg ates +den ne +cou cou +bun sen +bo id +bal k +ado gs +주 ëħ +èĬ ± +zombi ea +ze ch +wre aking +synthe sized +swir led +sto o +ske in +ren ounce +photo grid +no pain +nic obar +network rail +metron ome +m di +j ski +hd v +hal gh +h war +gar l +e gp +dic o +di ggle +con ker +cat at +c myk +book makers +bo ding +ang panahon +________ _ +>> > +(( (( +we ill +val era +truck ing +tro polis +tam mi +so fu +scho ir +sch aller +readi ed +pou ty +o clock +nemt sov +mo rec +mal te +judge jeanine +gro th +f fie +brooklyn museum +ðŁİ ŀ +wake forest +tro pa +thi stime +sle ek +rival ry +q bal +pinstripe pride +op ti +me stre +kings bridge +eso ter +danand shay +cuten ess +be amed +ani ya +af owl +zhou mi +voc acy +vel and +vander bil +stan wyck +snowmob iling +sheu gs +se us +sc itech +sand hills +rit o +re serving +quintan illa +pollin ator +ph s +per p +mu ti +mehboobam ufti +matthi js +maj ic +ly tle +ki is +k oun +ili ana +go ggins +gi verny +gi anni +geo grapher +fu gazi +fir stalert +em ic +don at +cro c +cn x +city and +ch acos +canadian forces +bon nard +bleed ing +asym metry +amy peruana +> < +ðŁĴĭ âĿ¤ï¸ı +walt disney +udu pi +u sher +tread well +rit mo +rev ved +rash mi +pre ssies +pompad our +patric ia +lg d +ko sta +ko balt +kidi ki +in cis +himan shu +fi baw +fan service +dist ancing +chav an +cassave tes +aqu ab +ant ana +adventure sof +ad tr +ab ut +[ - +ðŁĴª ðŁı¿ +ðŁİĤ ðŁİĪ +weather tech +vm ware +viz media +vic votes +ut v +the mentalist +ten fold +stun na +skill fully +pl ent +other side +men sday +medical devices +li sad +kush al +kas umi +k era +juri spru +inno cuous +in conclusive +iamk sgofficial +hit z +gri ft +go pies +gam os +def ame +dd lj +copernicuse u +car low +befully informed +arach nid +ap n +amp at +air crew +âĹ ¾ +zan elowe +wh ooo +wet ter +wat c +vs den +vas u +u du +syllab les +surf side +sur ly +sg u +revital ise +palpit ations +padma avat +maup in +mano euv +len s +le beau +kne ad +insuff erable +hun s +home coming +guitar center +eu geni +equ it +e discovery +bro ma +bot net +ber ita +beinte haa +and r +ale conomics +ðĿĹ ² +âĿ¤ï¸ı ðŁİ¶ +á´ Ĺ +te tsu +t anda +symboli ses +spontane ity +sou per +shan ley +san skar +sab it +r ils +r dd +pul len +ple xing +pla guing +ntv tonight +north park +max field +madhu bala +inst illed +hea dies +hal perin +earthen ware +discou raging +crustace ans +black mailing +auror a +ar der +agro forestry +ðŁļ ķ +âŃ ķï¸ı +yorkshi repost +val lee +th ut +tar di +sp hero +skin cancer +se ms +sc ant +sach sen +s combe +ru hr +or vis +night line +nes bit +m sl +love food +kni evel +itt ance +im patience +i vr +fis alpine +ferrig no +dedic ations +collar ds +chipp endale +c ren +bbc scot +al ten +ak shar +y sa +wal ford +v so +ucl draw +time bomb +tam pa +t oun +sear cher +ran za +pedu to +p ch +nov ato +mb storm +love sick +lov sky +long worth +line han +l va +he ures +freddi emercury +er im +em conf +eli g +decent ly +brain power +astar isborn +zhu hai +z uni +wi the +un in +tortu ga +stream ys +specul ations +sol vang +smil ing +seed orf +sajid javid +nab a +mil ford +mb assy +jim carrey +jay ant +hippo potamus +har kins +gray scale +daily caller +dai go +carpedi em +calgary expo +by rn +brek ko +bre thart +br rrrr +bon is +an ther +actu alliving +a ameen +whar fe +vigil antes +u ee +top sail +the res +soul food +so cs +se op +r bd +preju dices +postpon ing +neander thals +joh ne +i pods +hal es +ed mnangagwa +cham beau +calibr ate +cal vo +bul ma +bobby bones +bo sse +bl urs +bei ber +arn auto +ðŁĺĺ ðŁĺĤ +tylerg posey +t als +sur ulere +stur gess +sat nam +robb in +ra ster +obste tric +nc dot +ms dyn +mobile marketing +mel rose +maj in +li kud +kas bah +infl ating +ether ington +dic i +at axia +ðŁıĥ âĢįâĻĤï¸ı +ðŁĩŃðŁĩ ° +âĺĢï¸ı ðŁĮĬ +wunderbar films +ta vern +sr ila +square space +sign post +riff trax +pe qu +nave ed +na stiest +local history +life skills +j br +it now +ipp o +in bev +gon salves +gargan tuan +g dm +drop kick +dr harshvardhan +d yo +conver gent +ci hr +blueno te +black girlsrock +befri ending +b nl +anadol u +alca sid +abhi she +a asa +visit novascotia +un st +tri un +tod morden +super center +stay ing +rocke ttes +ric flair +pe ac +p janic +p dac +non league +mediterran ean +lounge wear +hal al +geo chemistry +fi ra +feel good +fag ans +eff southafrica +e urs +du an +circuit ry +childrens book +caro tene +broc colini +black day +bar ret +ball antine +annu als +yyyy yyyy +tm x +testic le +nu man +men newsdesk +letsgov cu +kids fashion +kak adu +h ink +ger tie +fir me +fe v +don gho +diete tics +depri ving +coolmore stud +clu e +che etham +cat trall +c ja +bio chem +bache let +b hil +teentit ans +sw tor +strugg leisreal +stone brewingco +sto xx +rock st +nil sen +muk hi +mo thra +metro pcs +mael strom +ma zar +lo oney +land i +kay y +in aba +ikoro du +g ade +e migrated +e iger +count ach +che sil +bus i +breast feed +better with +beatthe heat +be stro +íĥ Ģ +wright sville +wom end +us ar +tees dale +t fi +scra pes +s weather +run de +repe at +pend le +pav lov +ni ang +line wed +kaz uo +grand final +gi mli +cal ton +bro kered +bo stick +bo sley +arrhyth mia +wak and +vaill ant +ul alaunch +tw op +th ac +str ated +str ack +stone chat +stad t +sh ingo +scooby doo +oci c +mp u +mira da +l np +ic ey +hh t +handle bars +gup till +e he +duplic ates +consol ing +arti slife +al x +acar son +ðŁĨļ : +âĺºï¸ı ðŁĺĺ +à ¯ +work shop +thegreen party +th xs +swoo ping +skar du +siz we +sas si +rebe kah +po es +pap p +panor amas +mou sa +mel an +matt son +lee ward +keu ken +kar un +joeye ssex +hobby craft +hal cruises +go ps +giu lietta +dog town +dat ang +bu pa +bow ties +advers arial +ðŁĺł ðŁĺł +ðŁį© ðŁį© +ðĿ ĸ +ุ à¹Ī +zakkw ylde +yo wl +un r +un jab +the am +ta shan +raven ous +rain n +pe ppery +micro grid +long line +kak ao +intellectual property +ice ster +houston isd +hon oka +gravit as +forthe planet +flu or +fin kel +en r +en cant +disc ourses +dem ers +comis key +but thead +bring ing +afl ame +war dle +tre bek +stre aky +some body +sci fit +roch dale +ro fl +restaurant week +prophe sy +over street +mill field +matti as +len to +kla asen +ke ough +jo ji +ii ac +ga there +fe ws +excep tionalism +dragon born +daw a +d ants +ch elli +canton ment +black sails +bhu tto +ban shees +au teur +ðŁĴ¯ . +ðŁ¤Ļ ðŁı¾ +zakkwylde bls +y rago +wom bats +wing ard +tb oll +plo ve +philly police +pat snation +pam ban +meren gue +ma hira +long leat +light sabers +la ine +ju pil +i believe +hour ly +fli ppers +e ffy +devere ux +deire zzor +bu shing +br ined +bor u +bi dity +bi a +adju ster +unexplain able +the block +software testing +smu ts +rim mel +pro audio +per verts +nsc lc +nab isco +manchu rian +j cr +ic ant +house democrats +ear worm +disc olor +cv m +coal ition +chanak ya +boe hm +bla stoma +aldu m +af on +? '" +... ?" +* "@ +æĹ¥ æľ¬ +whatsfor dinner +weetab ix +un masking +turn up +trade war +the sly +tako yaki +ta pps +t vo +soulj aboy +sf ed +sco tu +rick ville +pend ragon +peep er +o it +norm alized +no ël +lake michigan +hy der +haci endo +gru bs +gazian tep +fork sup +every man +disp leased +darley stallions +crime watch +ck enzie +chron i +baji rao +auror as +@ $ +wy ck +ver hof +sti le +sna red +pin arello +nick o +n we +mod ell +min ess +lyric ally +li ason +lar ra +la ges +kimber ley +kas sam +jennette mccurdy +gret sch +gl ances +feu ille +endo za +dam ir +cre er +black panther +bas er +av t +wy the +visual ising +tr oughton +see ee +raw lins +pu dong +pu ddle +pl k +per ignon +ow ill +misss aig +mc duffie +kay akers +ire l +ing ol +gol son +gay er +deep dale +crowd funded +az o +awal ker +asi f +" ?! +âĺ® ï¸ı +zion sville +tvd family +sha sha +sh ko +s act +out smart +nu z +ni ac +ner c +ma iz +la pin +kou fax +ji ani +in ing +ii b +freenaz anin +des well +cra ke +ay din +al monte +âģ¦ # +wrest les +un blocked +tre al +ten sei +ski pp +sf aye +serv in +rw f +rodeo houston +r vi +nastur tium +mike and +hon es +he gel +haz arde +get to +future stars +emo ticons +di pi +de gli +cul peper +cat oc +bu gab +bo cuse +âĻ¦ï¸ı âĻ¦ï¸ı +y se +worldwetland sday +volu metric +vey ors +un comfortably +stumble upon +sto vall +star dew +si ro +shout factory +sc ca +ro wett +metalgear solid +mal me +lam pas +khati b +imperson ate +home team +happ i +hal fords +gri maldi +fri ez +en vo +dani e +cow per +conce aling +channel newsasia +cen k +brussel s +at ak +angelina jolie +vend i +un obstructed +thermo plastic +t ds +shkh rasheed +reti ro +ps supt +phyl lum +ma us +ma doff +lyn ton +ly so +kth opkins +just giving +jou les +eze quiel +euse bio +ct ms +conce ssion +by er +book sand +a od +tu ft +thespi ans +st thomas +sower by +ran tham +news network +micro cosm +maya angelou +m ka +ko gan +inclu sions +htown takeover +hil ty +har ge +happ ys +h fm +gra zer +gd ns +digital clou +digitalclou dgal +dero ga +car vers +back hoe +art studio +ðĿĹ ¼ +x er +usu al +the kid +tal us +stu tter +sh uri +mcdo wall +match room +marcel ine +man nan +kel sie +k ler +it ol +gi onal +faysal quraishi +farqu har +cooper ated +abraham lincoln +ðŁĩ¦ ðŁĩ· +wswan derersfc +vrin davan +vivi r +vau ghan +sop ra +scal ping +quad ro +pit lane +per ip +omni potent +n tn +mobile games +lc v +kej ri +intercep ts +fil aments +ed f +bo eck +arab ic +aff le +ãĥ ª +Ú Ī +wrink led +worship ers +vibe z +vari et +to sin +sp ica +shel vey +schi aparelli +riot games +pfluger ville +perme able +online store +me igs +ly ss +jen carlo +ig r +hr r +hom elo +hi jo +hen ch +drashtid hami +courte eners +cor vus +cine mathe +ar aman +ad ur +vijay rupan +tran scen +the jazz +spill ane +son tag +smite game +smash words +sav eluci +reu ben +pay e +ol entang +nc sa +manj re +knock off +inv ite +ge sund +flash lights +fau quier +engar dens +en r +el pha +eh san +combat ant +co it +clydes dales +cir ce +chu cked +cent com +bi pasha +barbar ism +baha wal +ay ear +ar slan +an bu +ãģ® æĹ¥ +à® ľ +Ùģ ÙĬ +аР² +vigne ttes +tur namc +sab ra +ram part +ra iz +pollin ating +peuge ot +perfom ance +pe dag +out performs +new season +murch ison +ministryof sound +market places +kul tur +k lat +ham blin +fu bar +ci mm +caro ten +canon australia +bo euf +bibi mbap +bapti smal +your story +we my +vijayrupan ibjp +tad pole +sc ud +sau ter +ph ol +park race +mono gamy +mom en +mat alan +kwi buka +hol bein +hoff man +hart son +go vote +gira ud +gar cinia +fu i +critiqu ing +cotton tail +clip trends +cabe za +bethe difference +ar ancini +ðŁļ Ľ +z war +white washing +weight less +vide otrends +valtter i +ser vi +nomen cl +morris ville +milk day +male h +legal ise +le ke +kar as +incrimin ating +hydro carbons +ha ftar +gra uman +g of +diminu tive +congreg ations +cit ric +chan sung +brum mie +broke back +à ´ +vo ile +under wire +tru deau +tro i +steve angello +ste aua +sr bija +reliance jio +perthglory fc +ml scup +mez ze +lo stand +in ck +he j +haw thorns +flo ating +en cum +empathi ze +dra ping +deli o +dar wish +curricul a +cold play +co dec +bf bs +bant en +as sun +art design +anup ama +al films +] ! +ðŁİµ ðŁİµ +tz in +thisi sla +sti fle +serge ants +rose parade +restin peace +reasons why +r mac +p wn +or wx +nar rower +mystic messenger +manip al +luv urself +la gann +he mming +he brew +er furt +draw backs +coim bra +breakout artist +al ar +ag ay +actor life +.... ' +yasi el +v ff +u fs +thr ong +spider web +russian art +re fraction +paddy sday +oy ang +ne do +nai ve +lo of +lat o +kar m +interro gate +gur up +cc pa +amar avati +ðŁĴģ ðŁı» +éĥ ¨ +z r +y alo +whet stone +thri ved +tc n +pre workout +onthe hill +novi embre +navig ational +mp tourism +mol ton +l tl +ko ster +ka el +jyo thika +in un +fur stenberg +food share +fi les +famili arize +exempli fy +detoxi fying +che se +char lies +cag le +bir r +biot in +ar ounds +af am +ðŁ¤¼ âĢįâĻĤï¸ı +wh iz +wee dy +volu minous +us dcad +ud get +there is +th andi +super draft +ss os +solom id +snow plow +ru ge +rock solid +re section +raj dhani +rain ham +psy locke +pro line +passion for +pa ster +nipp ert +medi ap +ma sts +lock port +ko il +hor vath +hair brush +gi lets +g ants +far fetch +f xc +dissi pate +debrame ssing +co sco +by product +braz ile +apho bia +ah gases +/ /@ +ðŁij¯ âĢįâĻĢï¸ı +the bell +sun spots +scrim mages +ra úl +poly math +kum ite +khal ili +jon ty +j ku +hyper allergic +hugh ton +histam ine +gul la +e ib +d la +civil society +chicago ans +bu di +be ile +aviva prem +are ers +ann s +zo or +yan tra +vand ross +val k +the truth +sy oung +spu blic +se thu +rel la +refec tory +pi z +pen elope +ku shi +kangar oo +jan vier +h lc +fan sites +decep ticon +clif ford +chec kat +bl ouses +ah db +ado g +xi sts +var ia +thibau lt +the power +sh oring +scu omo +resul ts +pot belly +ne ary +low ther +indi eartist +h als +goo s +gl f +fel e +disingenu ous +dese mber +del acroix +co ya +bou le +b sm +appet ites +ye ux +walt zing +vac om +ump qua +truck er +siti o +sham u +pvam u +phar m +on ne +jap a +it ek +it ad +in bend +hugh ley +hu mi +gen too +free style +fa in +f q +di zon +agu stawestland +ach ter +ðŁĶ ģ +ðŁijĬ ðŁijĬ +ðŁ¤¦ ðŁı¾âĢįâĻĤï¸ı +⼠Ķ +âĸ · +y team +wis d +v mt +um w +tw h +tt an +ren wick +re mix +po polo +pad am +la er +kno b +jag ran +hol bycity +f dom +eng arden +bocar aton +alex x +a ÄŁ +è ij +uni ofe +u wcl +swin son +sco p +pe gida +patter dale +p fm +o if +mc ity +materi ality +m gi +la di +kick boxer +jor n +fat man +eo ghan +einste in +dc n +ch ala +bullet ins +bla sey +bird life +am ol +akade mi +ah all +acor tes +ðŁĴİðŁĴİ ðŁĴİ +ìĹIJìĿ´ íķijíģ¬ +ëī´ìĿ´ ìĬ¤íĬ¸ +س ÙĪ +wi ak +wen dover +van ovic +val an +twin kie +team youtube +so ppy +scru mmy +sa ath +rick les +ra dy +pun kin +pro j +pim m +pete wentz +pal abras +news worthy +nascar onfox +nan of +mit osis +lawn dale +la die +la bi +jay anthi +jason isbell +iti me +hol lands +h sd +gam ora +ferr and +f anned +em meline +dol by +dal oo +burling ame +ben cher +ballo u +an ational +victori apolice +v ds +tah le +sc olds +ru be +ragaz zi +paro chial +no stril +newor lean +live for +jor ts +ip b +im plore +frigh ten +em aci +e ering +don nab +dat u +ch avis +benefit beauty +ba ited +aun t +ê· ľ +wor rall +wijn aldum +wag ering +vel oc +ucd dublin +tooth picks +su raj +st ec +sriman thu +spirit day +sleeping beauty +sethro llins +rec ites +philipp s +per ú +mcle ish +mau rier +ma he +loan er +kne b +ic bc +how den +hail stones +doc fest +ck es +china open +cd baby +bper rion +azale as +alo ka +waffle house +util isation +ti b +si ge +qu aking +pre zzo +pol ling +nu its +mobile game +mb om +life size +l fb +ki ee +ke hinde +in semin +how l +hin sdale +hes key +fon dest +fe style +evil regals +den bigh +cri bbs +bhu van +be on +aga dir +after all +ye dd +un fettered +theli ving +tad alal +suche tadalal +musician ship +l ali +ky l +kath ua +kan tai +k ds +it sn +ini b +hyun joong +her ps +head wind +head lamps +he pc +gay est +fit out +es in +eric garcetti +ef it +down syndrome +con sig +bperrion ni +be see +b sd +asi k +al ki +ðŁĽ © +ঠ¦ +Ø§Ø ¯ +to ki +te ja +sy arief +sq u +rho ades +result sday +prerog ative +pik min +no z +mile post +mc nab +mass illon +is sing +ima de +gr ito +globe master +f dj +do dges +corin ne +anthropo logical ++ " +zam ir +ty nan +thenorth face +tam ang +super imposed +son d +som i +ret ards +pre ying +person able +paradig ms +micro s +mer we +kit son +kel by +hul t +hand print +fun icular +fu me +form in +fifty shadesofgrey +fari da +escap eroom +eil at +dep an +daw ar +by blos +billi kens +bed minster +bad hopper +awo lowo +anand ani +çĶ Ł +âĹ ĭ +ÅĤ aw +tib co +ta seer +stal ag +spro mo +snow shoes +sick er +sal om +s sey +roof top +pari ah +ow asp +no gain +night wear +nak amoto +mc ourt +ing lot +ha es +go tops +go dukes +g staad +fe it +ear marked +chinese food +cats rule +black lightning +bir dof +bet tere +bbc south +as sss +art for +amo ja +aler t +wh igs +ur kel +spray berry +sol vents +so v +roth man +rec ool +proc ter +pew research +p ci +nic hes +nflu k +ly all +jail avak +hybrid cloud +hahahah haha +green peace +g za +freedomof speech +eat your +e ep +den ne +commu n +cheri shing +cc gs +cbc mtl +bar as +wash ten +topo graphical +toler ating +sc ool +no ssa +nab il +n ool +muzz le +mis sed +mg lit +loop ed +jo taro +jam tarts +i up +h ys +fre re +f dm +e mile +e bp +defense men +cro fton +chis ora +cha ren +brun ton +bra vado +artex hibition +am ro +am ac +adn ams +ðŁ§ ¢ +wy vern +spam mers +shu g +schoo logy +rec to +pink berry +pak vaus +p di +or kut +nat ch +mr r +m vt +l nc +john fugelsang +hur u +hum bucker +horn sey +high more +h cd +gods end +dun nes +comm enters +ale b +administr ations +a itch +? !!!! +ðŁĺ¢ ðŁĺŃ +yu ji +worldanimal day +war by +tribun als +tow ne +snow pocalypse +s wine +run away +rob ed +rad ley +ph ang +ol lers +official wolves +n li +m andre +ke it +ke er +kasi h +kal pana +ici ously +had ley +depra ved +dd as +cough ed +co ates +cher bourg +capit ulation +al tai +ì° ½ +z ow +wool worth +wigw am +trum pre +propos itions +pork chop +over flows +ori ana +megal odon +meat packing +mcder mitt +le graph +irv ington +inde cency +ge yer +fre tless +difun dir +case in +ben oni +wat ford +un ravelling +uhcougar fb +thibo deau +sme aring +revisi onist +pi de +phylo genetic +lu an +loren zo +long meadow +livor no +kit sil +joe gibb +it syn +her ac +great war +free bsd +fermil ab +di still +cu sh +clear water +can ela +brun s +bish kek +big dat +ðŁĴ ¬ +ภĭ +war th +ver ticals +top cybernews +stat utes +sob ti +sni ffed +sky lab +shin ty +ro ane +mentor ing +mag andhi +liveon k +it k +g mo +endear ment +east africa +dom us +ci w +carlo tta +bos combe +bath gate +ë§Ī ë§Ī +æĿ ¾ +z ny +vit ara +v enda +twizz lers +tor d +spen cers +sel den +se bo +mcar dle +lasz lo +l ra +karthik subbaraj +h no +g fr +fri gates +franki ero +corn huskers +bing bing +al ite +ภ® +son us +ren derer +point illi +phil ic +novo sel +let down +l cp +har v +fuele dby +fra w +er ving +duke energy +dan ko +circumc ised +chandra yaan +carli sle +can onized +c vr +bor ine +bor ge +bl dgs +a opa +بص ÙĪرة +week of +turbo prop +tol bert +tam ika +t lan +seuss ical +scu ll +sb h +pp es +pet ar +my la +m busa +le and +jo gia +ha gi +golden hour +finger tip +er rat +dog life +cy nic +chi ral +b ty +av irus +ani shi +ag day +ðŁĻĮ ðŁijı +ðŁIJ¶ ðŁĴķ +thru ster +th ya +t wat +solic it +so tl +silver tone +signor e +shu ckers +shi bir +sha in +sepul chre +por irua +paulin ho +listo wel +la ba +ing with +ho sh +har ari +first net +dischar ging +de tr +chi quita +bullet club +bachelor inparadise +audi om +ade eb +⼠ĵ +Ê ³ +xx ii +woodin ville +wheni was +tra shes +thu man +te soro +supp ers +special ities +sean ad +sch at +ra if +professional development +mi est +mat osis +life science +kal mar +juni ata +jump a +ir ctc +hoor ah +guess who +gra fted +f da +essi en +dt lv +carpath ian +ca hoots +bi ps +alex jones +al n +wu pper +w gr +use i +ston ight +st martin +rev ellers +photogram metry +parson age +na oto +muff led +micha ud +metag en +lee ann +le ford +kow ska +kookab ur +invari ably +grand central +gar nacha +future ofeurope +frank turner +eye let +ed modo +dro d +dre sse +den nys +chil aquiles +buc co +botan ica +be ppo +bar ron +b ti +ar sed +ak ova +: } +yo ak +x al +wv tm +te er +ta pit +mo vado +mexic ali +may pac +leav ened +joegibb sracing +is sar +i ag +go away +ge ve +fedex cup +emphasi se +dis continue +den ials +costu mer +by night +busc ema +bi ju +bay eux +bar codes +alco ve +al ast +ac enter +ðŁĺľ # +whodun nit +var ga +us w +tre s +ti ko +the fallen +som an +social ising +she mar +sevasto pol +se cc +s movies +rye grass +reser vist +re joined +re factoring +py roman +ox ted +nw u +magi ster +lu cena +lind ner +ide a +hill house +gam el +franken muth +fla sher +emul si +election swith +demo lishes +creati vel +av ita +!! , +xrp community +tru sive +sneak y +sal an +og awa +j aren +hin son +en grave +e itc +datab ase +ch he +brind ley +blue jay +baux ite +amig urumi +alfre ton +actu ators +abra r +abduc tions +ðŁijĢ ðŁĺĤ +âĿ¤ï¸ı ' +⾨ ðŁĴĸ +âī ¥ +wright son +u op +tober mory +supervalu irl +sport fishing +sl ates +sin h +schedu ler +sch nabel +re cumb +rab bani +pur nell +power tothe +persu ading +natural england +mou lt +me use +l ans +haz ell +go oner +gal leon +footh ill +fav ed +exhib ition +em path +elo die +dvent ure +den arius +dementi afri +defe cation +ci x +chor ro +bar dugo +are qu +arbit rage +a defenders +yah t +to sk +tele conference +taylormade golf +tan z +shaw ls +red fm +post secondary +kri ss +kaik oura +juventus fc +ity ours +i verse +hi jinks +gri f +col gan +blueri bbon +bips luvurself +altaf hussain +alam ance +ðŁį ½ +w ux +un ic +tr ically +tam bor +success ful +spo wer +si rena +sc roun +sab ras +re itz +physic ality +park sville +par aded +oo per +ne va +mu gi +lv motorspeedway +log ica +lec at +kram er +k hama +infantic ide +food lovers +civil engineering +c wd +c boe +brown sburg +aviation lovers +ani dol +alo ck +ðŁ¦ İ +âĪ Ĩ +veri fiable +tri k +ti ang +she go +self catering +reimbur se +prefer ring +por thar +mk to +ka am +fun dac +filip ino +dre lief +chart mill +caf frey +assinibo ine +afro punk +ab lo +wutang clan +ur ry +uni formity +t ferriss +sub pop +stom pers +sand r +sam bar +sad i +robbi es +rec oup +r itch +national volunteerweek +n td +midd lew +lau f +kun ta +ke ppel +immun o +hor wath +hereis gina +goo des +faber castell +ef r +dor ic +do da +de paris +conju red +carol inians +cali stoga +ben ching +bat aclan +af cofficial +ห ม +ร ะ +wild in +ud acity +tra dio +theo do +the is +t bol +sque aks +smorgas bord +shawn michaels +pon go +no stradamus +nike store +mor sels +j ó +in exor +igne ous +ig nific +hodak otb +heb den +hay ride +ham on +ge v +gal an +chapeco ense +bun combe +bu o +bod acious +bb ard +authori zing +auctione ers +atta che +anatom ically +ak id +af oe +af fiche +ðŁĵ ¡ +ðŁIJ ļ +yu uri +ye c +u ja +trans fixed +ti bia +te cho +sol ilo +school work +roof ers +re zz +pur veyors +pu z +pokemon sunmoon +ok oro +mahat magandhi +lu e +leban ese +laura prepon +kam enri +je han +intersec ting +happy humpday +golds boro +ga im +fro lics +f ads +encar nacion +ce vic +cat sof +burysted munds +birthday yy +at ai +ap ital +alter yx +ad ua +you andme +t bb +supplic ation +so gno +rah ma +puntac ana +pat cho +pa ar +om it +naz anin +mar kovic +ma ssed +legis late +ko irala +k re +imbec ile +hot seat +he eded +gran ules +ge yman +fran chi +e pm +ds all +d tes +crepu scular +cer ise +bermu dez +ben anti +ay umi +ater ally +ap us +answ erable +an pur +ac omohoy +ãģ ij +y omi +wing field +tw all +this was +thereal sambora +smooth ness +roh tak +produ ce +pro pen +private er +pa jero +mre azi +mou la +marku ss +maneu vering +lo ki +k sw +ju ma +joshu a +jon freier +in ion +cozy mystery +cor ot +chicag ol +carl son +b ink +ampl itude +ìĭ ľ +worldradio day +wb ca +th x +stop bullying +sat mar +sa aa +rock ymoun +qu ade +press man +nabe el +ma sdar +lic eo +im passable +frod sham +f du +ex chang +eric bolling +dick en +del on +cou lee +construc tors +bri er +ber ardi +bag pipe +ba j +b pp +alder ney += - +ðŁļ´ âĢįâĻĢï¸ı +ðŁĩ¿ðŁĩ ¼ +zer rie +wer d +study in +sh is +sch wimmer +rhi z +paul smith +oscillo scope +or ca +nav as +nag isa +n np +len ore +ir repre +ima b +im alt +hand craft +gravit ate +gr anny +gam u +g bag +dis agreeing +diar mu +cor tado +bring ers +am ies +alber ts +ad han +abram son +ðŁĺĴ ðŁĺĤ +ìĦĿ ì§Ħ +young adult +y si +wx w +ther cn +roc kett +rb news +milwau ke +laun dry +jung shin +hon olu +gym time +dv rs +all livesmatter +y nez +xenob lade +u mat +sign sof +profe sional +o stric +letter box +l lap +ke in +jar ring +hon d +g tl +fraun hofer +fo ort +du brow +crown theempire +cincy wx +c ley +baw al +ann al +adon ai +ðŁĩ ¶ +yu rappa +wor l +wei gel +trin oma +syndic ation +side walk +shaan xi +sci ver +saveluci fer +sar aha +r sm +prophy laxis +pine apple +ol bermann +mo fa +lori keet +lec am +iow aspeedway +exac ting +ent or +engv nz +dream house +dove tail +door stop +d stv +cy borgs +bou gue +bal on +awo ken +æ Ĭ +âĻ¥ âĻ¡ +Ãł idhlig +uc ca +thenand alfilms +stop and +sk b +sen ryu +rovin j +pare sh +n out +lu gosi +kro eger +kon k +ki awah +iy anya +hawthorn fc +haber dash +freef all +fla iling +expon ent +en doc +drag queen +cove red +com mies +char ice +ch ima +ceme tary +at ria +are k +an jou +al ented +ac igars +) -- +vse vil +verhof stadt +var sha +val ov +un sworth +uk on +show manship +seal ofhonor +ph alan +mel y +me tered +match day +k inston +hs baseball +he tty +fulfil ment +ful ster +earl ham +dev ita +d my +contradic ting +braini ac +bon et +bau l +bar ong +astr al +app er +an tim +.... .? +Ê ¸ +world toilet +un um +tit ling +sol vable +sant an +sa bel +rall ysweden +r boy +ocla ure +numis matic +natural skincare +nano tubes +mass dot +marcel oclaure +kitt le +greatormond st +e gm +dire k +bos sho +bj c +b tw +b enders +adel le +âĿ¤ï¸ı ðŁĮ¹ +ye vsky +w aki +spon se +simil ars +scol ding +pou dre +penn dot +offe e +mol i +mc sorley +ma char +green hill +gon oles +freer ange +engagementr ing +dundal kfc +defibrill ators +che ick +cam bo +atx wx +ar mid +am uro +aber ration +ľ ï¸ı +ðŁĩ§ ðŁĩª +ðŁ¥ Ģ +wreck ers +we stover +vad is +un box +ste ver +spic tures +smo kin +shah nafisa +never winter +nad ar +mcpart lin +mayo ress +mart ÃŃ +lu cker +love cornwall +lam brusco +kol n +ismail i +if tikhar +h mh +good ale +ger ar +gau ssian +eag er +du lux +do die +bvl gari +als icebucketchallenge +ðŁĺĬ ðŁĴĻ +ਠ¨ +zack snyder +yi ppie +world turtleday +wind storm +vin oth +vene tian +ve olia +ti f +than gs +steppen wolf +schi ffer +say lor +po pp +philanthro pists +park bogum +ni va +ne ct +me hn +love ya +lo zada +hidden figures +her men +fin nish +feng shui +ex ude +com al +cabar rus +biof ilm +bam bu +all endale +ł ãģĭãĤī +ðŁĺĬ ðŁĴľ +ìĤ¬ëŀij íķ´ +tylen ol +tro it +transpor ters +to pinion +spi key +sand or +rec tangles +q ian +preposter ous +p supt +mb or +gor dhan +gl anced +figur a +ell chat +di mm +dat ors +d bo +com ically +cli pe +bige ast +bienven ido +battlestar galactica +b hal +albert son +ص ر +twitter world +tren ton +town houses +surger y +sun bird +stan niversary +sicklec ell +shrink age +sh and +semat ary +sassen ach +s food +pu lido +one way +nose bleeds +mccr ary +mar din +log book +kn z +injec ts +infl icting +hu da +hems ley +health ier +great lake +free styles +ear ley +cunei form +clark sburg +ap bio +an h +accompli shes +ðŁĺĬ ðŁĻı +âĿ¤ ï¸İ +ti ra +theroo ts +tex aco +teach out +sop with +sack ler +route master +quil ter +pyth ag +per dido +panel list +opho to +k pd +ilipp ines +how lett +hau g +g fl +faber ge +ek ka +e iz +dream actnow +corru pt +chaffe tz +bu ggers +austral asian +as us +alon dra +twim bos +to ku +so hard +sla yin +sky watch +san som +official gaa +now ay +ni mo +mori moto +local isation +jack y +for our +emb ol +ed j +dra vi +col ai +ci ene +bar rick +bal dock +bab oons +auto pha +ar nd +âľ Ĵ +x lv +wil mington +vo ids +ven ise +tw ingo +saint srugby +pep far +our ay +op n +oligar chy +nat an +n do +moun tie +madein chelsea +ma ze +humboldt strong +hipp oly +hello ooo +good by +frat elli +ev geni +es mo +ensla vement +da chau +char train +am entor +' âĢĶ +ðŁĻĭ ðŁĻĭ +ت ر +yl la +willow brook +video conferencing +then ci +tat ter +sw aff +sp illage +si pa +sev yn +sa head +ry che +pu sheen +poly phonic +oc tane +no irs +national gri +n wi +lap dog +l vc +hun i +holocau stre +h wa +guinnes spro +flash sale +dv la +dav uto +d sm +d mf +causal ity +car illon +be cuz +wa hi +tintag el +the deanambrose +tat in +shan n +searchengine optimization +pl se +petiti oning +pand o +o gi +nucle i +missi onal +magnific ent +k ks +isc out +imp d +fa kir +evil dead +emul ating +dish washing +des jardins +clothes line +caver sham +ba ikon +anno u +ani er +al mu +ah rar +a sexual +! ðŁĴļ +y aka +wish ful +vi gan +unisouth ampton +the buzz +tan amon +taffe ta +stopp ard +sin ker +sha araw +schu man +reck lessly +pro pping +maran ello +ma sal +lol ll +hoo ooo +ho ban +gas monkey +er dem +det ach +darius rucker +clean water +black heads +biop harmac +belve dere +bart lett +ask ren +ðŁĺŃ ðŁĺ© +ðŁĩ· ðŁĩº +trav eller +tor na +theop hil +succin ctly +stan bic +smith ville +sikh ism +se pa +rayn or +plas mic +over heated +optimi sts +mo si +meetthe press +mc so +lamon tagne +kirk us +in ne +har vie +hallucin ation +green ham +gre xit +gas karth +errone ous +ef arm +cook son +con over +con don +care n +burgh ley +belfas thour +be du +bash ful +ariad ne +anim alabuse +acrob ats +ab ap +wann acry +un incorporated +te b +spor tnz +sa ari +ro vio +rad ler +pra c +pi voting +ph ono +pearl man +mun day +mon ch +modern slavery +mi yu +md zs +life way +k sm +jas oos +hor ta +galac tus +fossil fuels +ex us +end it +c leg +bron fman +beef steak +ar but +akiha bara +ðŁķ ĵ +womens month +torri don +t je +spring bank +spin elli +shab bir +rock your +poc keting +parliam ents +meal prep +mathemat ica +mar q +luxury living +loo ong +lar kana +ki zomba +ig cse +him mel +high st +head lands +gl m +da ines +corn elis +bett ys +beck mann +bb ons +b sp +ar ks +am iss +. ðŁĺĤðŁĺĤ +wol cott +un accounted +sub consciously +splin ts +sa onb +ru per +pader born +n bag +mid south +march esa +lu sty +lu cario +ky go +juli ette +inter feres +hypo xia +gand u +free gle +fra ggle +far ren +fac i +cryp tos +change therules +cap iz +bru ford +b ge +at re +activ ations +ðŁĹ ¿ +wood craft +us ama +tun stall +so ch +sm cr +sin an +salfor duni +punc tual +proud moment +pr icks +poe tic +pine cone +oscill ation +nag i +my fav +mer aki +man gold +makemoney online +letter head +k hoo +interstiti al +hyper ten +hick on +gul den +grey houn +galla gher +fit tipal +ente ast +end u +ecol lection +dr v +dis band +del f +decor um +com ikaze +cob web +chatter box +c fos +bo zz +bionic le +ar ke +voc ate +vo g +vai z +v ry +twi stle +ti ba +thro es +the wave +tender ly +shaz ams +sc avengers +re organized +propag an +port meirion +n gi +my chal +manipu lator +lam enting +kr w +kilo watt +jubil ation +iron works +hon y +hiaw atha +hel plessness +he mb +gil ad +gen ovese +en actu +dor fman +csir onews +corri entes +bore ham +ben ni +bath house +ath ur +arcade fire +amon te +al tus +? ( +yi u +wk tv +wer u +vsp hi +ve stal +synthesi zers +super sporttv +stra de +sag en +ravichand ran +rai ya +o doi +medi kal +live able +le vity +koch har +jessicaal ba +heral dry +harryand meghan +glendal ough +gil ley +ed n +drive club +devi ous +denu clear +cy o +cryo genic +cho gm +bu ssel +brou ck +ar moire +aj payee +ab ta +a aya +wool sey +unearth ly +ultra fast +spinnin records +scot te +res ellers +read acrossamerica +n cea +mcqu ade +martha stewart +loosen ing +j harris +girl talkza +g bo +fin nigan +elias son +bri ley +bow land +boo bie +blue field +actu ary +ðŁIJŁ ðŁIJŁ +ë¹ħ ìĬ¤ +y pc +y ager +un skilled +u gle +ty pi +tric ity +tin ie +thomas ville +stran raer +ssf crabb +ssfcrabb itohs +sk ic +reic hen +ram e +raj deep +pu shy +pic ad +p wi +oo f +naturo pathic +n ps +mccl endon +keshar ose +jeremy clarkson +je ster +in bred +h pp +f cr +close ted +c ton +ash tabula +an cona +alla board +ìŀ IJ +wad dy +voyage urs +tanamon geau +pr r +pon ti +pgc ps +our g +metro id +lauren laverne +kri ya +kohl rabi +key bank +kar ag +kal ab +is adora +grow nups +der osa +datadri ven +dan ks +ast one +ames bury +alife style +ภį +vijaysethu offl +t shi +ron don +pu s +planet jedward +pen alized +observ able +no sso +nca c +mon santo +ke se +ka ur +in en +i fl +greyson chance +golds worthy +gocu bsgo +foolish ly +fat cat +esqui re +dise mb +bon di +body con +birk beck +battle tech +av ent +an the +z cz +w ä +w drb +une ven +un peacekeeping +u er +ti gru +the vampire +sorrow ful +ru stle +ru hr +print works +pe kin +omni present +musking um +mool ah +mid point +mi hai +mar cie +jit su +ireland sanci +ham pur +gh ome +fro gg +fix ated +fer oz +dead stock +city center +campe che +ca sia +br ampton +blitz krieg +at first +ar tur +ushu aiai +ushuaiai biza +urve di +tph online +sympathi zer +side tracked +sf gate +pal infoen +man sur +ly d +li mos +jacque es +gun sup +gli se +ge thin +fri e +fen u +dy spra +cla ym +chap a +c gl +bar rens +an isa +ī ï¸ı +world tourismday +w oc +w ladi +van itas +sudir man +shab ir +ros icky +pre eminent +potter ies +pi awur +ne eti +my fitnes +men or +mark j +love dogs +k hou +ir anian +insi stence +flexi on +exorbit ant +du ele +desk ill +dd am +cy tometry +box wood +ben avide +assimil ated +ade e +ÙĦ ÙĦ +~ ^ +y ps +w iller +vali dates +u kes +tone itup +tom ars +she ared +rush ton +plu gger +pir ro +nar sissi +lynd say +lyn c +liver ies +k jr +inner most +ho tham +herman miller +h var +fanta stico +ec anada +dd m +daily motion +bed fellows +arbitr arily +am cham +ye son +weekend reads +vast ness +tre molo +the greatest +sil oam +sar pong +sandal sresorts +r grosjean +public transport +power ing +neuro tic +nan as +moo lool +mol itor +mi am +m sr +lou w +kra v +gab f +fier ro +f mb +don lemon +del ong +boro budur +ar nav +agro ecology +ac to +wür z +wil helm +v ingly +travelo dge +thre ep +the irish +staf fel +sta ats +snic ket +p gt +ol ab +o sts +numer al +neuro diversity +mat ters +la ga +jay ant +jaw breaker +in lays +home builder +gray don +gis borne +gas pard +fethi ye +fear thewalkingdead +ek ad +crum pet +cr h +cc sd +boar dof +backin theday +ðŁĺĦ ðŁijį +ãĢ Į +vetri maaran +tri ad +tough ened +ta hu +sp litters +sher riff +polar is +pe or +or ab +one year +nam carey +mu ito +make ityours +m no +l ch +juxta posed +jussi esmollett +it works +isthenew black +irelandsanci enteast +har z +e zz +dimble by +de wayne +de mic +co ves +cav y +cancell ara +bridge gate +bougue reau +bore ham +balu strade +al righty +ðĿĹ ® +âĻª " +wj hl +wi ffle +war rington +tony the +thi opia +spir acy +sp ry +social ise +shaaraw y +se dum +roman esco +ri ssa +red dog +rare diseases +rain i +plural sight +pau per +o dense +navig ated +made ley +len ape +k ys +k afe +home buyer +eul cs +dip tych +cube sat +ch isle +car nar +be ko +baf fert +av ai +ع ÙĪد +zombiea pocalypse +wit ton +unequi vocally +tri angu +thero ar +soccer grlprobs +roch as +revi e +pic ballot +meer kats +kr z +kq ed +kon kan +ker stin +innumer able +gu is +gu ber +ely ria +bo gu +aly zer +alphabe tically +alban ians +ade cco +ðŁIJį ðŁIJį +ÃŃ k +w under +te ow +shi ga +rick man +n ph +micro brewery +mi ffed +mazdas peed +marchi sio +loo b +lea v +laugh ton +kear ny +ip aded +i think +hod der +glen more +gle aner +exper ian +co bs +cau tioned +cab bage +border less +athle isure +ale do +a ard +ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ +u tu +tw al +too ting +spr a +sky tower +sal ado +rex press +pub med +om usic +no fficial +ni hon +i ams +h mt +goo sen +giuli ana +elpha ba +dream girls +concor ds +clover dale +citru sy +bra w +boun ties +barang ay +ase prite +antw on +an ja +aller gan +ðŁĩ²ðŁĩ ¨ +zal giris +wesl aco +um sl +tree top +tarry town +tac tically +slo pp +said haram +sa wan +sa kur +pan african +not good +nick j +must be +mul lets +miss ary +mand aue +lab ors +kam er +he met +gar rity +cre ases +ch oline +bro ch +blemi shed +zapp ed +ulter ior +turf club +sur fact +su leman +sk n +pre rna +on campus +nu dging +n up +matrimon ial +lore tt +ine sday +in mar +hydro phobic +hawkn poke +eury th +em ts +dun oon +cott en +constru ed +be red +ba sher +ðŁĺĺ . +ðŁį ¼ +z ze +ying luck +y de +vit als +ve day +tropic o +tra ynor +ticon deroga +the farahkhan +st asi +ss v +sk lar +sal icy +ro sne +rackete ering +pennstate fball +p bi +o ji +o doo +new mark +my fox +mor ies +marke l +mar oney +mac ra +ke izer +kal ak +human ists +hazarde den +fis sion +fe der +dy mium +buy art +at co +ash an +al sa +al kali +ë° ° +à¹Ģภ¥ +ver bo +th air +sas ol +s vel +nu f +mil lais +mc cook +mackin aw +light speed +lethal weapon +impal ed +hoo ten +hat ers +fil o +fantasy baseball +del isle +bu cked +blo t +ax mi +ash vsevil +aristo cratic +acro stic +ac rime +< = +!!!!!!!! !!!!!!!! +ìĿ Ģ +w cyb +vau se +uw tsd +suffo cate +special report +semi o +sat anism +ric or +ra res +pm modi +livel ife +le lla +ir inha +i Äĩ +hor o +her bed +ham mam +glen view +extrac tive +down loader +christian grey +chae ology +bur nette +bow more +bor ini +boo geyman +big sby +tu ffy +th unk +son air +siddi q +sch nell +rook ery +read athon +ong ate +men shoops +melis sar +meal time +kh ough +ju sta +ger ton +fal mouth +es as +ed and +der ian +da unt +d pl +conserv ator +concert gebouw +chi raq +char leroi +bull fighting +bon efish +ban stead +ðŁı Į +æ·± å¤ľ +âľ ī +tun der +to ffs +theli st +swar ms +snap chat +sk ra +shot gun +sch ee +s be +pros thesis +pink shirtday +or adi +one health +mp ha +molin aro +moisturi ze +lac i +l á +k ult +ide m +ho bi +gam eface +fittipal di +film struck +ed ens +cow town +com media +carte ret +blan cos +barbi eri +atra vel +amal u +alway ss +al tona +ye katerinburg +y israel +wal msley +w wee +under funded +u cb +tim ento +thomas fire +sho spitals +shail ene +sev ak +ru stin +romantic suspense +provo kes +pic ku +pe rel +pe en +pap ad +paleonto logist +minneso ta +kie hls +inde pth +e ero +concer ting +co valent +ce st +cardiff devils +bro da +bra k +ash bury +apprais als +appointe e +y ir +xx vi +workin ghard +wing suit +us borne +ra shes +patientex perience +ocho cinco +ny l +new kirk +myfitnes spal +made on +l atta +hu ys +ga jah +finger nail +eart ening +drew brees +do vico +diso wn +dan ai +colori zation +catter ick +call ing +bulk head +bo kuno +b ice +aul lah +ab ai +ðŁĻĭ ðŁı¼ +ı n +wil ber +upcycled hour +swor dar +sum é +steve smith +sinn fein +si dra +rochel le +ran n +raim ondo +polit ico +ph rine +outdoor sman +ne smith +nc pa +nal edi +mo len +ma show +m fb +ke bbi +im mo +ilo ilop +ha itians +gill iland +for christmas +escar got +download fest +dev gan +colla b +clonak ilty +click able +cali endo +c wx +blackandwhite photo +bangla desh +av y +ang ala +aj in +ze b +usar ugby +twitter sisters +turk sand +tee ism +sou let +se squ +sculp tors +refu te +reas oned +q iu +pie day +oyster catcher +oo st +mo da +ma ach +kah lua +g fp +fr ys +flood lit +dul hania +dan ne +cro ton +col icchio +cle x +ch boursin +cb cs +bern at +bat um +ì ¼ +ëı Ħ +z ori +wi thering +wang ar +teas poons +ske em +s gm +recuper ating +rattle snakes +r sh +po vich +mil en +m out +ki edis +her mos +her mon +garri son +gam ist +ga etano +cu bana +cru zeiro +chig well +ato se +ar bu +annihil ate +agu ila +ðŁĺĤðŁ¤£ ðŁĺĤ +y zerman +y po +wwer aw +tw iler +tal os +sw p +sw ab +shaz ier +ry ans +r pi +pur ples +phyto plankton +pha r +pa ise +motor homes +mccour ty +lun as +lauren tian +kol i +jim caviezel +gen io +flex friday +fair lady +ema ar +down the +devan ey +cre tin +ca shire +blu ey +blo is +black jacks +barnar dos +axo lotl +ashvsevil dead +zyn ga +tomo fromearth +sk ine +se wol +reu ters +promp to +por cello +per ron +pash to +paedi atric +oo z +oo kie +lich ter +leder hosen +leather man +jets fc +ippo iloilop +im pede +howard donald +gent es +ev ms +el co +dan tonio +coch ise +chil dish +á ¸ +vish u +un berg +ti enne +t fr +sir car +ribb it +r be +pla gne +pike tty +pi rit +personal injury +my lar +lzzy hale +ku ria +frau en +foot stool +farmb ill +fac inelli +escam bia +e we +cw o +chelseac linton +char ades +and ile +ac kee +yo soy +wol d +usp oli +us na +u val +tusc ola +toy ama +the sia +tac it +super ga +strang lers +steph mcmahon +solo travel +sh ampton +sa enz +robu sta +re framing +po ch +life insurance +la day +kar apar +jur gens +guild ford +ghir ardelli +ebb s +diav olo +crowd sale +chester zoo +char r +bigg boss +big cats +bhan sali +bad boy +at sea +as sent +Ï ģ +trek ked +touch pad +tim buk +rob portman +raku ten +pre trial +party nextdoor +os g +o tom +no yer +madd ening +jon son +inlovewith switzerland +gro ped +diam andis +big little +b me +aw t +ak ingly +wat sons +wat p +vote onedirection +u hn +tweetapicture thatdescribesyourfriendship +tr anny +son gof +sim racing +she par +sebasti án +scarec rows +sal cedo +o samu +nurses week +ner ship +narsissi st +n saina +mole ster +machiav elli +li ek +lc ms +ip fw +infon doro +infondoro tweet +gre tath +gerani ums +ent v +earth pix +du bh +dimit ris +corner brook +cor respond +cic er +chir py +cheltenham festival +bom bus +b fl +ai den +adar na +aber ne +a hir +unt angle +tu shar +stein bach +señ ora +schlei fe +offen bach +nobel peace +margo tro +hi ac +ge ma +eze kiele +ed ers +cule bra +comor os +win stone +way point +war pedtour +trick ling +su dar +spring cleaning +shaf en +pin oe +pg w +ola ju +leit ner +le se +ippoiloilop np +induc tive +hhhh hhhhh +gretath unberg +go tto +exo tica +erec ting +ea res +di ary +der os +de meter +co wie +che ema +books beseen +bo to +bb do +av aro +as of +am mer +a joe +........ .......... +violenceagainst women +under weight +ter r +t sing +sty l +stu bb +spi er +slack ers +shal f +public an +plo tter +os ment +nuss baum +moor fields +money penny +mal thouse +lu ka +lament ations +jai brooks +hel t +hahahahahahahaha hahahahahahahaha +er lang +emo tion +dy son +dro it +dorse twildlife +donkey kong +di anne +craf ted +cn m +bad ham +ay c +ari ate +ðŁIJ © +âĶ Ĭ +ü rk +ymc mb +wat ley +w toc +virgin media +super sonics +sh it +sch ilis +sad da +recycla bles +pu shin +pro xi +per reg +new shq +mother f +mo red +mal ak +love struck +kre ss +imam ali +hel msley +gri sel +flemington vrc +far scape +el ge +chi rag +alphon so +. ". +ðŁĺį ðŁĶ¥ +âļ½ï¸ıâļ½ï¸ı âļ½ï¸ıâļ½ï¸ı +âī ł +à¥ĩ à¤Ĥ +y ayo +way o +vi jender +tamar braxton +tamarbraxton her +student success +spi ece +sker ries +sha shank +sche de +pun dit +pt cl +perse vered +omg its +nko tb +ms deskill +msdeskill india +mr na +motor sport +mon ics +mcgon agall +kitsil ano +k vo +jo enbc +jas s +it em +ilo g +ie h +fe est +favour ited +far in +debar linea +comp ton +cheryl cole +bree ch +brac kett +barrett jackson +ðŁ¤ ¨ +âĺİ ï¸ı +yaros lav +us ace +ti do +sw as +sn hs +plu sh +pale olithic +out set +net eller +natur alists +naje eb +mi paltan +mer ce +lock lear +ley man +ip h +ini x +in ed +fre dette +entomo logist +devo tion +can g +boat man +b bow +ak ou +;; ;; +! âĺºï¸ı +ðŁĩ©ðŁĩ ¿ +羣åī£ ãģĬçµµæııãģį +wist ful +ver afar +the time +shon en +saidharam tej +po kies +paris i +no stro +monso ons +mix up +mith un +min oru +mb ball +man to +magnac arta +kalash nikov +hy phen +fist ful +expo se +day after +co gic +co bia +cin os +ber oi +ber at +an ki +' / +tre foil +tb p +tan en +steu ben +sportsc ards +skynew saust +pri stin +pay phone +onto logical +nikel ab +milli ken +mendo ta +k lasse +indi ain +imp ounded +iam nehakakkar +hal sted +gat ling +for dre +far ne +dump er +di dion +decre es +d ph +book suk +barber ing +ace c +ðŁĻĪ ðŁĺį +ðŁij ± +åĨ į +winnie the +tomb stones +thin ned +sty a +si pho +pseu dom +propagan dist +pe ct +over tly +ound ingly +o suna +nu er +nk peace +ni gg +livel ong +li ott +l sg +gom is +farming dale +ec ko +e iri +dsw d +deton ation +de voting +clu n +burn s +bring backthe +bo pper +ber l +ale i +al ba +zo iep +yo yp +tl eroy +te res +suit ability +sk ales +sa inte +reprodu cibility +puri jagan +pre ys +pr l +phy tes +mo there +miy avi +mc vay +lo ya +ko ke +green back +goo oooooo +giant s +fl studio +feather y +extraordin ary +dow son +defaul ts +dar wen +d ms +cur tin +clark sdale +ci les +chan elle +cassin is +ain u +á Ł +ya ari +wall ach +w tol +usf ws +twing lish +turn stile +sunil grover +sensi bly +schul ich +pro claim +prana v +peter borough +perreg aux +per nod +ne use +m lt +m ko +lyn den +ky n +ho ole +hali de +ha ys +friend lier +fer ment +f sw +er ing +enti sts +dis ch +cth agod +bor iso +bo wels +bam bam +au dis +angel s +al arabiya +ðŁĴ ½ +âĿ¤ï¸ı ðŁĴĽðŁĴļðŁĴĻðŁĴľ +woo lies +wdw today +to pper +te ap +super computing +sp angler +raise high +queens ryche +pri sms +opor to +mt mte +machi da +jan kovic +in excusable +gu ile +finlay son +dram edy +doo san +dayofthe dead +col gate +caddy shack +bt toronto +ammon ium +ami ami +ðŁİ§ : +ðŁĩ¦ðŁĩ ¿ +zoiep almer +want age +w ma +voiceof obrien +ver di +ve o +towerof london +sté phane +stur ges +sti ger +soci alized +sie g +rhy s +pro bert +over thinking +ole ic +no a +mikas ingh +ki per +iv d +ing rich +hyper active +gear best +followthe whiterabbit +fic tions +ec ou +darshanraval dz +commercial isation +catalo ging +ben evento +amund sen +aim lessly +- &- +ìĥ¤ ìĿ´ +ü yü +vin er +sutter ink +sp g +rotherham iswonderful +ri au +re deemable +pl lc +paw sup +or pol +le vent +lan come +kk an +ish ra +ha stag +gue ira +fil bert +eth i +d su +buis ness +ben ik +arre sted +al lover +agin court +wwe games +unra vels +twin sies +time management +tean eck +scar fs +r cl +pur ley +power houses +po stures +pedro ia +painst akingly +ow let +ope th +motör head +mol de +hugo boss +g oud +fri a +flo tsam +el dn +dan ai +ast side +ap apa +ans ys +° ° +vikram prabhu +up endra +sli ving +sky uk +si hh +show stoppers +see king +sc imit +reading challenge +ra du +pod bean +piawur tzbach +pel ley +p wn +os n +open banking +old timer +ol anus +mel ano +kensington royal +fc x +cp im +coss ack +comic strip +comb ative +co sida +ce v +cb w +cassinis aturn +ari jitsingh +app o +amee sha +adobe max +( $ +ðŁij¸ ðŁı½ +éĩ ij +voc ates +un habitat +tur vy +tro be +today schilis +tipp ec +sir te +saf es +ru dra +red white +ph ala +nag iri +multivit amin +me wes +lu ttrell +j ole +intrac oastal +in ici +im fnews +hand books +ha ft +go lo +crypto trading +char grilled +centra al +british bakeoff +bas set +aust ere +ali wal +ad ini +yw am +wor cs +tran scribe +to winit +sou py +sie w +show piece +shamrock rovers +sc ry +rou lade +re finished +ra vin +par ter +ou tit +ole ander +oh ms +neer ja +mag all +machin ima +job seeker +fun der +distur bs +bo dine +bm ps +as oiaf +ab atement +ìĬĪ íį¼ +worldtoilet day +victor iam +su ave +state university +smi l +sentin el +punxsutaw ney +patton oswalt +pad awan +modest ly +minim al +lass iter +ku b +ingh urst +hu lt +ho ol +hierogly phs +hay ter +hag ler +flan igan +fish town +fire brand +fel dy +et b +em at +el rey +doctor sday +dc family +daily pics +bro mide +bal aton +baff in +app raiser +app omat +aad har +ÑĢа ÑĦ +your future +war burg +summer hill +ro sses +patrol man +omyel itis +may tag +madein france +ka the +insur mountable +insig ne +happy christmas +gru mp +fo yt +draw this +disser vice +cur va +brigh ter +brad lee +ar cos +ħ ï¸ı +ì ± +wand le +w sw +university challenge +tag team +stel ugu +squ otes +so dex +script chat +ril is +rap monster +r ind +pre acher +ph rey +ly love +lur gan +lark spur +fair tax +dro opy +comple t +christi es +carri bean +box office +an eta +aggi es +ðŁ¤ª ðŁ¤ª +⼠ºï¸ı +us dot +tr r +roa stery +reson ator +phenomen ally +matt jackson +ley town +j elli +ic ana +ibi za +hy n +guess work +goti t +episo de +el veli +dr kent +boor man +ber kowitz +an hui +ah me +! ðŁıĨ +ðŁļ ¤ +ðŁĴĥðŁĴĥ ðŁĴĥðŁĴĥ +âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı +water for +val d +under secretary +stren ds +sten house +soci alization +snow globe +sil van +romeo andjuliet +rhode sia +phnom penh +ou est +nu thatch +month of +mm ys +hierogly phics +fanci ful +er ac +e studio +com i +cab anas +bioprin ting +be ggin +bak ula +ausv nz +am ado +)))) )))) +wh our +weimaran er +triump hal +so ju +serv a +serp ents +scu detto +pol qc +oli v +o die +ma hou +lover boy +la sa +jeff bezos +irish examiner +indie booksbeseen +ill amas +hel ical +fear lessness +ezekiele lliott +e ys +de gu +dat are +cool er +cointe legraph +chu gg +character isation +cb insights +ca ppy +bel lucci +alt ars +? * +stevenage fc +steel series +shu b +shabby chic +se bago +scream ing +real i +plu meria +o wat +markuss chulz +jar ra +il ot +il lah +el ight +edgar wright +don cic +domestic abuse +disinfe ction +den by +bull snation +bel in +bech tel +beau te +bar bi +اÙĦ ÙĬ +wonder land +vote march +ther anos +ston ing +shipp ensburg +sch mel +sa chat +s farm +rafin ha +po inci +ode on +meryl streep +masam une +mary anne +mammo graphy +m oooo +links ys +ke ez +ka sie +gel o +deid re +cher well +cher lloyd +cardio twitter +cal z +boo zing +bar mouth +as ma +aquas cape +amus ica +ðŁij¼ ðŁı½ +á¹ £ +zak yn +xl viii +wgn news +ti as +taken ote +syner gy +stabil isation +sacri lege +ro ark +re action +radic alization +pla sters +ph ala +om ingo +new sprint +mu ffet +mr at +lique fied +le gui +iz od +ht p +frenchri viera +ca hn +arquitec tos +a ari +ðŁķ ¶ +ðŁĴ¯ # +åĽ ½ +âĿ¤ ðŁĴĽ +ا ÛĮ +vide ocon +verafar miga +v min +upper cut +uo fu +spire ites +sor bonne +sol t +smooth ed +shiv am +sand point +reli evers +lang u +i ze +how son +hil ic +glam is +g antz +fle gends +dispen saries +discoun ting +chri sb +ber tu +âĻ« âĻ« +z ky +wen zel +wbc boxing +top shelf +spike tv +spic er +sp engler +sh ate +sev co +pal mers +om undson +ner vo +marsu pial +lec kie +ld h +kili fi +kiel basa +jerry lawler +hy eri +help fully +finger printing +e son +depend ents +cyto sis +chi on +bom p +bai lee +astro gaming +assassinscre ed +ashmole anmuseum +an ouk +alejandro sanz +al ys +ak kuma +a sec +ðŁĴģ ðŁĴģ +y annis +u bon +troop ing +spamal ot +sn ood +sa kin +ruth lessly +remote sensing +r ines +pp k +pedd ler +meet southafrica +mcminn ville +mas ri +ma ggs +keyn sham +jaun ty +israel news +hell skitchen +have yoursay +haar p +diab y +delmar racing +ch ury +carav anning +can in +bur pee +ballester os +ar rambam +ar ge +ani am +and ys +an nett +ach io +y f +west mids +ultr amarine +tip tree +te mu +st z +spl int +shan ker +pil ate +pel i +panam anian +new land +mu bb +mis informed +mer sea +me ate +mc stu +may fair +lu pton +kud row +ke dge +it ae +indv wi +iam chris +en sor +dien st +d po +cyr ille +barro so +ari de +alternati verock +ak ak +ze ta +we iser +thero of +tani sha +super fruit +shin suke +sex smith +needto breathe +mr mark +mol inos +mar ussia +llan beris +kin ch +jupit er +ho thouse +glyco lic +ge auga +fu la +fish n +fa fighting +dar ned +bin ib +an airlines +주 ëĭĪ +ve in +used gov +timeto talk +tail or +suffo cation +san lucas +re ek +queen su +prece des +pen day +pad illa +oc ad +nopain nogain +man ton +lor ing +li vigno +keto diet +k anta +juli puli +heral dic +giving day +gabor one +g pt +ft fc +f ack +el agh +dc g +colli erville +brit ches +af w +ðŁijĨ ðŁijĨ +water hole +vladi mir +tec ate +switch blade +str t +sr kians +shailene woodley +re fle +rat ch +r fm +pre vost +pae dia +nas arawa +mug am +mc gi +land lady +lalunasan gre +ki ama +keen an +kan eo +j rock +im so +fl inch +fix er +fe mi +fc bayern +electric car +e chs +destin ies +cav ell +c vi +borsch t +bore lli +' [ +ðŁ¥ ĭ +Ñ ģ +Ð · +the is +thalai v +stom s +sco w +san kran +salon edel +redd it +pu paid +pas wan +mar lies +magnific at +ma ther +lisi eux +jamshed pur +its showtime +ick man +ical ondon +heck le +gang land +ep dm +cur vy +cre mona +colon el +co aster +cab rillo +cab incre +big ten +as z +alger non +Ħภ² +une vent +twin ing +ton sils +sso cial +south ridge +shop ian +pied ra +oo i +naka jima +music industry +mcafe eshow +mar sters +liber ace +green ways +g couros +for cible +durham bulls +cru mmy +cp k +clam ping +bri dged +biri yani +ac me +ðŁijĪ ðŁı¼ +é» Ħ +wash rooms +u eda +swithe xo +so ds +sel im +rau sch +port stewart +phi v +pacific rim +or on +ond ins +masu da +lan ey +kurt cobain +i ku +he ik +gir ondins +fre o +etsy specialt +do val +bur ra +ìĿ Ħ +ìĭ Ŀ +way zata +wall er +w mb +ty an +sinf oni +rud dock +rejec t +rav ish +ra skin +pre mam +poly graph +player pro +pen tel +pa ix +on dp +oh rid +nur magomedov +meadow brook +l ill +kr all +jac inda +iso u +ie o +go tit +fri dge +fer d +ec ream +disintegr ation +cultiv ar +cp b +city tv +bishop s +bel um +bb camerica +bat b +ban ega +ba isakhi +assembly woman +yoshi kiofficial +ym nastics +xmen apocalypse +wy cliffe +water world +val or +street fashion +show boat +river keeper +realestate investing +pha sma +nen agh +mur ciel +mc andrew +match less +mack en +lu ckey +less than +ic ici +holiday inn +hasi dic +gla w +g aping +di os +dha k +countdownto christmas +c dd +boy o +berk us +b kc +are sort +ap ra +an tos +>_ > +ðŁĶĬ ðŁĶĬðŁĶĬ +zet terberg +vex ed +tril ateral +trans america +the sm +str act +sti an +stern berg +spl enda +sh iso +rat ing +photograph e +ou te +maran atha +len nar +j oli +hyun sik +hoek stra +halli burton +for days +fo k +ely see +diversityand inclusion +butch ering +bro gues +bo hr +ared itor +ar ul +app ts +add ington +vintage toys +ti ppin +reson ating +mer ton +kad ar +kab ab +happy stpatricksday +gishwh es +game z +fra gran +ee baftas +drunk ard +chaper ones +asur geon +arnauto vic +ar ow +ãĥ Ļ +velo so +tur co +t ello +sth lm +spati al +rak is +raisethe bar +pr ying +or ris +ol ong +never quit +ly u +lewis burg +karun anidhi +gru po +flu tist +fabi ola +end hunger +develop mentally +dean ery +da st +cu bao +cou pland +bv m +br attleboro +bore hole +bar ked +av ino +are ma +animal lover +am ora +al vor +un tied +umb i +tory burch +thr all +sv w +snow birds +sheik ha +ri dder +re ars +ontari ans +olentang y +mott led +ma thi +leich hardt +kim f +keuken hof +janathagar age +ir say +hisham mu +he ssian +fest of +fa hd +ensen ada +enforce able +con cub +co k +ca world +bran di +boi ster +bar ram +ar pita +al dgate +wearen ato +uch ak +toa stie +te jada +spin el +show biz +seun ggi +rajas thani +r mg +mon signor +m radio +ke pa +judd apatow +isab ell +gir on +gi p +gam bier +fal vi +ero y +eli ving +dis concerting +cu ellar +cap illary +bin ski +angel os +amo sa +ðŁķ ķ +tuesday selfie +te ste +suni verse +sud ha +sap d +over population +numb ness +nicol let +min o +mid way +mag an +le bon +koz hi +kn ell +kli psch +her rin +hd pe +car crash +atlantic council +and soul +and son +am ple +ac w +x ham +vo td +vo ssen +trade smen +sper th +spear heading +sky dome +shin ichi +registr ants +prime iro +p tom +mou che +ko oning +kind ling +gad ge +ful i +fos dem +dis qualify +dan ab +dae jeon +d ja +com patriot +chlor op +blood stained +baikon ur +az in +... ðŁİ¶ +! ðŁĮŁ +ðŁĮ¿ ðŁĮ¿ +å Ģ +âĢ¦ @ +wol seley +wie be +up surge +tony robbins +swan ston +sal vi +s ä +rouss illon +raj ag +od ile +michael son +me ted +lombar dy +li san +kait lin +h sp +get z +ger ia +gathere rs +f elly +emc world +ed reform +dor mir +dish eartening +crit ter +cre ta +ash le +ar ini +an sley +actualliving scientist +âĹ ķ +winn able +uni fi +te ka +ta stic +supermari omaker +ry lands +raven el +r le +py ke +prun ed +nam rata +n qt +min ator +melissab enoist +mante ca +inform a +head hunters +hard ening +fent y +est ac +el kin +east land +dal trey +country file +comb ate +colle tti +arup group +ar ka +anno poulos +an kle +ãĥ¼ãĥ © +ãĤ ĵãģ +wein berger +vir o +thames link +skim mer +rush den +roo y +ram sey +pas sport +or atorio +nct m +mu gi +ma adi +litt lerock +la sor +junky ard +fis sure +fay dee +fare wells +draw down +dom hn +distric tn +cout ts +cori olanus +clam ped +bumble foot +ay ne +an it +amit os +vo hra +vladi vostok +tun ities +syndro mes +symp tomatic +sun pictures +sim provement +reb elling +quer rey +photosof dublin +no ther +min nows +matern ity +ko ti +jo vial +inza ghi +imp hal +idio cracy +ha bis +cele stine +call sign +c cr +?? ?! +transhuman ism +the cus +ta ek +studentath lete +sho rey +sa sharo +rv n +rish ta +pac er +new ydd +mol to +mel ting +medi atek +man jima +litt en +li bres +kuznet sov +kir yu +karak oram +kar gil +ill matic +hafi z +gan o +d bi +d acosta +chee ked +cast es +bab in +b wb +ay ia +and country +wh ig +the mba +tat ra +stro man +smash ville +score cards +rav ello +ran z +pi pi +ner ton +national day +mg sv +mali gned +le mmings +j ev +intram uros +inf a +ij en +hopkins medicine +gal ant +far ring +deta ining +boun cers +arequ ipa +ðŁ¥ § +ze etv +zal ando +vo ces +vil let +vent uri +su do +sm r +sil marillion +shap ing +see you +sam sara +rod well +resi ze +radic chio +qui ps +predat or +par ky +manu life +mahot sav +m ge +lat itude +krem lin +khe er +jay lon +i den +ha bla +gru en +fluctu ate +flappy bird +fab le +denny hamlin +centen ario +bun ce +broom sticks +an ut +ðŁļ ij +ìĥ ģ +wid ths +waf b +the word +te efury +spy ro +shi el +sch lumber +sa ppers +re launching +pal ladi +mon a +li q +kum o +jan ae +is si +im ate +g ile +e missary +coinci ding +chalce dony +cal do +apo el +an ole +aggre ssor +ðŁĹ Ŀ +à¹Ģภ¥ +w day +under pinning +troms ø +te faf +super mom +stri als +story tell +sco in +s need +roo d +plain ly +pac ade +ne brown +mtk global +margotro bbie +kir ch +khq localnews +jim mies +holo cene +he em +fu ta +fetu ses +dysp hagia +dayoff all +cu tbacks +cb ssunday +cas soulet +bul wark +ban al +an del +am sat +yas meen +will ful +west isbest +u ds +tw ice +thel ab +stove top +ste es +patho genesis +orel se +ogil vie +of the +of ten +of ici +neuro blastoma +more ll +martÃŃ nez +loss less +k lip +har land +handicapp ing +hal ima +gau ghan +gar ia +dd ance +cu man +cru sts +chiroprac tors +ben ham +baw ling +arthu rian +allevi ating +ac ole +---- --> +ðŁĺį ! +âĿ¤ï¸ı âļ½ï¸ı +ঠ¸ +yo jin +ye u +y pa +vand en +v gl +us ada +tsingh ua +thousand th +telltale games +sho jo +sco ville +sat ori +repe atable +quent in +mari me +malac añ +ma kai +lat en +kin kade +kil lam +hit maker +h th +gram app +ge um +fri z +fe ek +daysof activism +chur ned +chico pee +bu j +bet w +berk man +ah f +worl dar +women crushwednesday +vi kings +vanqui shed +trit onal +super series +sn afu +rimb aud +rally for +quie ting +music city +mattybra ps +mac ias +lovelo velove +lees ville +k chenoweth +jaz min +inter vening +ilooklike anengineer +ho pp +ho ice +hex a +gri ma +goo glen +gener a +g wan +der z +birken stock +bend re +bel en +be ira +ath ar +aerob atics +ðŁļ µ +ðŁĺŃ ðŁĴľ +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤ +walk away +vallec ano +v end +swtor family +stro s +splat tered +robo to +red wing +or gin +oak lawn +nat u +mclaren auto +m ti +it sy +immobili er +ido o +her she +gir der +gamel oft +faroe islands +east link +do bbins +be sok +atri ots +artic a +altam ont +ðŁĵ Ĩ +ëį Ķ +zar ate +world champs +wil tern +tele mark +tang le +ste ppers +sp ha +smith field +smel ter +sleep iness +shor thorn +se hat +scast aldi +ricflair nat +ricflairnat rboy +ra un +p cy +noon time +mercy hurst +lo ling +little field +it su +ich allenge +homopho be +harmon ix +gar wood +fu a +ei p +dam ani +cla es +chal dean +brass ica +barcelon a +ar tweet +al om +aj kt +ðŁĶ´ âļ« +z ita +z ari +ur der +studio ghibli +sk ys +shareyour weather +seag rant +ry k +ou at +no guchi +meeting profs +man av +ine ar +gul ly +gul lah +guan lin +grow up +footb al +explore tnq +del acruz +creation ist +asphy xi +as ings +appreh end +alton towers +_/ \_ +year books +whoo pi +thel ady +temp tress +tatian amas +tatianamas lany +tab er +sur fed +sheep shead +ru c +ro tax +ra yan +post mark +op elika +olim pia +no bigdeal +na sti +n fib +my city +moon walker +metro linx +medev ac +mal ware +li gh +l fr +jah lil +icecream day +hon ky +h ô +gas ses +from italy +fc sp +eu erasmusplus +dol i +di mond +cor bridge +chic ane +bully ing +ball z +aggie up +world champion +ven eta +var ley +v aren +swin ford +story boarding +sor um +short films +self driving +sci reports +salv age +quat re +prime time +pat mcafeeshow +pand its +news max +ma zi +lo ko +kan mani +it ri +ink drawing +il ha +horo witz +holistic health +heath y +govmike huckabee +gluco samine +fredri k +fran corchamps +fenerbah çe +er mann +en ame +dor m +doit better +ch sler +cdn hist +bat shuayi +atmo spheres +ash urst +ant s +acu pressure +? ..... +. -- +whit lam +unab ated +u ky +tu bab +tear drops +sor orities +rekind led +qu ails +pat re +pad me +north allerton +n se +my nd +me gha +k fest +ima i +hy ang +fore du +ff un +favor ito +fashion designer +esz ka +disney sm +cheese cakes +cd k +bq quinn +:-) :-) +wic kes +un supported +squ inting +sam iz +pra gati +pk ats +nab o +like this +le scott +ku fball +kar imi +fl ink +eul er +dv d +da ke +bon in +bha va +bennet ts +ay cliffe +avali able +archi vists +al bar +çī Ī +à³ ĭ +wu f +wis in +trade govuk +sha fiq +rey k +phi fer +ow ry +n chs +mi strust +la chen +kal ki +je er +industri ous +iiii ii +for ked +dot news +desp at +dd ry +cycl onic +chief tains +ce des +bra yer +boo ba +blan keted +anti gu +alach ua +ðŁĴķ ðŁĺĬ +utilit arian +uss i +tantal ising +su hr +stri per +sal thill +re wing +ophon ic +o vember +ni ble +mc griff +ma hara +ju mla +ink well +i pt +honolu lu +green grass +fer net +ear muffs +d under +coted azur +bra ves +boy sen +asun cion +al ynn +ðŁĩºðŁĩ ¦ +ภł +yo day +vi bing +up do +uc ci +so do +si rah +seattle times +rp dr +reluct ance +pol ak +ot c +mcco ist +malay an +infe rences +in nis +ha shoah +h ga +giov anni +er ith +davin cis +compag nie +char less +blue bell +bern al +art therapy +al son +zan si +ye su +yan is +transcen ding +tommy flanagan +swal well +stick tennis +sp v +ro ms +ranjit rophy +progre so +pro bosc +nord schleife +mo jang +mill work +mase kela +laura bailey +kaz im +jaeger lecoultre +in is +hol ness +hit omi +gno l +gillette stadium +gett v +front end +di aper +d tt +country living +clo jure +clark county +bbcscot weather +ba thers +ac mi +abe okuta +ðŁijĮ ðŁĴķ +⼠ª +wanna one +un complicated +tyra banks +twitter land +tro u +tre a +tb aker +re drawing +no bodies +naom ic +md gs +mar wa +mar ly +made for +m de +la sters +jen elle +hairex tensions +green ville +ev h +ec ce +eatemu pkats +du ka +discolor ation +der mott +dc publicschools +clen ched +cart mel +brow ner +brainde ad +benevol ence +bel vo +bed worth +bab ly +auto dromo +at ak +armad ale +.. "@ +æľ Ģ +æ ¯ +to ver +tn p +there tte +teagas c +spar adise +solst ice +nott oo +n kc +my oung +mass aro +man av +hin do +h fd +fur yk +estudi antes +en ay +didd ley +cul led +comp as +cham pers +britann ica +bluer idge +bill ington +be funky +b ort +aw sm +ang gun +v ab +tol ley +the don +ted to +stro king +sm acc +shivar atri +sepul veda +ry del +poly morph +phy no +personal ise +o ing +nor ml +nerc science +mk dons +methodo logical +meas ly +landsc ap +kamp f +hel fer +gui zhou +gu ins +glengar ry +fo ghorn +est ella +embo dying +day bed +categor ically +bo yes +bench tm +beltand road +bead les +be stre +aric ci +afar ber +achri stie +ðŁĺĬ ) +ðŁįķðŁįķ ðŁįķ +x bone +worshi pper +tran som +sn elling +savit ri +sar miento +sab aton +re vocation +pd sa +mol ino +min ni +men y +mar at +macas kill +lj mu +li zzo +le aker +l ti +kimf coates +jit endra +jak ks +it zel +immer sing +hend ra +gold rick +giovin co +gal vez +food blogger +comedi enne +circuit americas +chamber sburg +bech del +à° ª +z ug +yak itori +walk ley +vocab ul +serendipit ous +sa dia +rug gi +roo ke +rit zy +resur faces +ren veen +re padam +rand burg +o vitch +nko si +motherwell fc +mo tul +mis interpreted +madi rakshi +le maire +jo you +instinc tively +hench men +haddon field +flavour ful +ex a +en al +el um +dow dy +do bb +cumber some +cervical cancer +buc ssport +bird watch +wr anglers +vi ña +vesti bular +sar ma +op g +nsm q +mag ny +kitch n +khan i +in x +hu ggy +he uss +he din +gn p +gau teng +cum be +conjec ture +carshal ton +bumble bee +black cats +ber yl +be mused +bal kan +at ron +ðŁĩµðŁĩ ± +าภ¢ +visit dublin +valent i +trampol ines +thenand now +ta hi +suss man +staff suni +slo ss +sinnfein ireland +sigh tedness +scar pa +ro ga +reen acting +po hl +plai sir +paralympic sgb +now t +mill stone +midr ange +le bat +kun ming +khur ram +kat amar +kale e +i shaan +hy gro +ho jo +go z +gar ni +gamer rter +fle ming +dv s +dru ze +done gan +deb at +carri er +ben ne +ar thurs +aly syarief +al hilal +af flu +ac una +ìĺ ¹ +ãĤ ĵ +yeah hhhh +xi ong +x man +wood haven +wan athan +ut ler +u manitoba +tweet the +top soil +tam angpanahon +starwars story +ru ba +pomo doro +per le +optimi zer +multil ateralism +mor phe +man ay +luxuri ously +kon ya +jess amine +iowac aucus +in ay +guardra il +gu tt +f pl +ern akulam +dimini shes +di sa +cort landt +coo lest +cas ita +cal vados +bo ying +blen z +bio similars +billu ps +billing sley +be res +be brave +assail ants +asak usa +an coats +ac te +æ·±å¤ľ ãģ® +ãĥ¼ãĥ « +your best +womens fiction +vibr ates +ve tri +tsitsi pas +t plf +stone ham +sq a +sk inst +ru sting +polic escotland +pet zl +more ton +k rock +jeffre ys +jad on +fu fa +fe v +donnar umma +cu jo +bun ty +brighton fringe +boling brook +ar oos +anant nag +am ando +am adou +ad dle +ðŁĽ ° +ðŁĮ ī +yn ne +wire tap +war hols +vivi er +virtu a +tv dsb +travel in +sun flower +retro viral +pre schooler +mu zzy +mdc ps +mary hill +mand alay +mah n +lil durk +lan den +la paz +ko te +jac inta +gun ny +elap sed +el son +del ap +de schu +cu eva +ci ampa +ca dia +ba iling +ay ay +ati ps +amorph ous +Ò ī +we tn +vi ver +tb in +super bugs +spac ed +sh lomo +se gas +sa is +roskil de +rag out +pie tr +o euvre +north brook +mc clar +ma su +m he +litt ler +li una +kno tty +holy land +h ku +gate keepers +fcc ca +far ma +de ka +ctv atlantic +cro zier +comedy club +co se +bryan fuller +aw wwww +ðĿIJ ¢ +ì¹´ ìĿ´ +zealo ts +vikas khanna +thel ook +terra pin +south ington +sj ö +segal ink +ring wald +pur suit +porter robinson +n jor +n able +music week +maneu ver +loc as +lat vala +jo esp +jill stein +je pson +jar omir +gre ener +gh lan +den na +cre vas +ca ja +but kus +barbecu ed +ban ia +ba shers +and als +ðŁļ į +à ¾ +Ùħ ع +za har +wa af +v uni +un hurt +sasharo iz +sar gas +rag weed +pri mers +persu asi +old town +na sim +mo sey +megab us +luci ous +lle ida +ku ka +ke sler +katy cats +juventusfc en +jale el +gri mac +form es +echo smith +dw f +deser tification +chat uchak +bol land +bec kett +bad boys +australi angp +amaz igh +ðŁĺįðŁĺįðŁĺįðŁĺį ðŁĺįðŁĺįðŁĺįðŁĺį +ðŁİĵ ðŁİĵ +ìķĦ ìĬ¤íĬ¸ë +woo ing +wire frame +warwick uni +vere en +tag a +s dorf +ruther glen +ri gger +pu renew +ound le +ophthalmo logist +nov as +mess age +lang en +ki ke +jj j +instru ctive +im cr +hud l +hahahaha hha +h nc +grow ls +ffe d +extrac ellular +egal itarian +dere khough +casc ada +cand lewick +bra chi +bi shes +aspen ideas +aro ck +am gen +aguin aldo +wit wer +vi sta +taran tu +stor ie +stenhouse jr +skin ner +sj hl +sel fi +se ck +se basto +salt ash +re turners +po ach +paras ailing +pali o +ot ti +nav ya +man gala +lil uzi +l fs +kier kegaard +kei ki +hop sin +g linda +fun e +flying treasures +first dayoffall +feed feed +extrem ities +chow kidar +bright future +bra que +bodle ian +abb vie +wait resses +vo se +streu sel +squee zes +so ham +si skin +scri vener +pied ras +pan ts +oscar noms +os weiler +multin ationals +melis sag +leibo vitz +kit na +ki h +jan os +int led +industri alized +henri k +glo om +gilets jaunes +gar ratt +door darshan +cu vee +bu hr +bbcworld service +b ita +ausv sa +ard show +zero hedge +twilight zone +treas ury +swind ell +shaha dat +se ance +s lough +quer ying +plain view +pas co +p na +me tu +maxi mum +kur uk +kare v +ic f +hur tado +hiro ki +hac kett +gi spr +chit wan +chan tal +cann ae +band ar +an pr +ad ley +acivil war +ðŁĴ ¶ +water sheds +tol an +see thing +sain toftheday +reham khan +redsox nation +pol man +nicky byrne +new build +mn r +legion naires +kw ame +in lombardia +hur ghada +ge omor +fra c +fr fr +el z +e warren +counter culture +col ney +boondock saints +ble v +be proud +bar dem +alam w +ye tic +univers alist +tiffany andco +si ar +ser pong +propag ating +preneur ship +org ana +on da +neu en +ly coming +ku ga +indv sa +glee fully +fore boding +energ ise +dr congo +diamon dre +de sar +d vp +cuck oos +cop en +coo puk +camp bel +cal pe +ca shes +ber batov +band ito +bab ushka +as sos +artist s +an cora +akat suki +yearsand years +w tb +twee ting +tre l +transgre ssions +stru mming +someon eyou +ski athos +san on +saddle dome +runny mede +ru ms +poo ts +pardon my +new son +national boyfriend +melani el +mc carter +laurabailey vo +kn br +jop hie +ic se +hu ddle +gra zed +go blets +ff ner +cham el +cedar wood +car per +bex po +bakh ti +adverti sing +ðŁıĢ ðŁıĨ +ठ¶ +vicer ylle +ust fccca +sten gel +spear fishing +sheep le +p ke +nol and +mee eeee +mb ol +leon i +la than +kru m +kan agawa +gyneco logist +ga ap +flor is +eu logi +ef p +ec n +dee j +cyber men +cross an +coin treau +cody rhodes +chori sters +car ls +belle vue +asylum seekers +ðŁijį ðŁĺİ +íĻ ĺ +women day +trun ning +to god +tali esin +spo oning +ro ve +proxim al +mr bobbybones +maul din +kel is +kaf fir +iw m +il ai +go ssett +g Ãłidhlig +fly tying +f pu +est evan +dysp horia +dosto yevsky +decap itation +dai wa +cl anton +ci ut +ce menting +ag ara +ðŁIJ ľ +ðŁ¦ ij +ys z +wel led +u sso +tu li +tok us +the futurist +t illie +st itt +sofi acarson +scrib ner +rudi mental +robert kirkman +resi stors +re gon +ram ana +r nib +po du +no tes +mount sin +inhabit able +hu i +grom met +forthe city +fore stry +ding ers +dend ritic +d ins +brown stein +ay ahu +... âĢ¦ +team coco +sunday lunch +sub tract +si ssy +sen ran +sapi en +ro go +ro f +reggi ano +re integration +re imagines +pl ices +ol on +ogle thorpe +objec ted +mr b +motor cars +mathe w +ma fi +lil ah +l pm +j balvin +inge stion +gu lies +grun ts +gior gio +gi ma +fre elibrary +fmp sd +fe is +far k +fal staff +expos é +donate aphoto +deci ma +cu c +cri eff +co sti +choo k +bridge water +bab bling +aureli o +architec turally +alt itudes +alfar ome +alderweire ld +yaht zee +ver raz +upper case +tro glo +te il +st legion +sli me +sin hala +sh uk +se ps +schen n +nc ss +mart ello +mar ka +khal e +intelligent sia +ic b +happ yn +epic games +edward barber +dy nat +colour pop +cal var +bur wood +bo dom +al tright +ãĤ ĭ +young min +trab zon +tra xx +sl ington +sim mered +ren fro +pu ke +phala en +nax al +mx r +mun iland +mac i +ior i +her p +fx networks +ec to +deb ina +chrisl hayes +cbs newyork +campbell town +bat esville +av it +ash wednesday +aco b +ðŁĺĤ ðŁĴ¯ +zi ad +wo tc +tu sh +the return +tem kin +suspen der +st j +st bat +spe kker +se án +quad rup +or os +moh sen +me jores +mcbride melissa +iri s +insul ate +holl ering +hear ns +gau ging +frosin one +fe bre +entr anced +elik kol +durgapu ja +disney xd +d max +cormor ants +cis c +chief keef +c wru +butter mere +bre tta +boycot ts +be al +à± ĭ +xanth in +wannab es +w me +tr at +too thy +tom ography +tin elli +th ack +sw ati +soda stream +pul sed +preserv ative +n ced +mon ad +mo bbing +mica ela +mar it +lor awan +khu da +jam o +jackson yb +ill ine +hor wich +hen sible +gol deyes +go or +fitness friday +f na +ett ner +echo cardio +dr phil +dema tha +cas cade +can yaman +box car +bar den +b nt +, : +ðŁļ ½ +ðŁIJ Ń +Ë ¡ +york s +un hrc +un funny +trump ers +tra k +today i +thesly stallone +tab ak +shi l +seg mented +se idel +ri ple +re man +nep tun +lie v +kw ad +he swall +goe sto +gad or +g ones +g had +fer gal +cu rable +cb cott +bu tta +brick house +beat ers +an m +# âĿ¤ï¸ı +venice beach +ved monton +team o +stedeli jk +purve yor +ponto toc +pit loch +orange theworld +o isin +nw bb +ni hr +naijap als +mark r +ko tori +heit kamp +hal pern +ha use +cor s +change maker +ai ke +a hahahahaha +ච± +wil kin +wak in +v gma +un inspired +superbowl sunday +sport snation +son wheels +sha ik +sel ife +sat yar +restric ts +most beautiful +jor ia +jodor owsky +ic ho +homogene ous +hen kel +han kook +goh mert +godz illa +g we +g ative +ech os +diso bey +car milla +be head +ðŁĹ ³ï¸ı +ðŁĴģ âĢįâĻĢï¸ı +ìķĦìĬ¤íĬ¸ë ¡ľ +⼠µï¸ı +wer ner +steve martin +sn us +shr ine +shira zi +schnau zer +n ée +mat ting +ko bach +kevin spacey +keral atourism +kam iya +in nyc +green up +gi golo +gar lands +gam ec +floor boards +d gamer +coinci ded +buli mia +a starwarsstory +í ľ +âĺĢï¸ı ðŁĺİ +ಠ¯ +ç on +way yyy +takethe crown +stri b +serv al +sa vic +rown tree +mn ths +long ley +lagun abeach +kin folk +kap iti +kan tha +ev ah +disk on +dar lo +crystallo graphy +cou n +chocta ws +chis nall +cast leg +carab iner +c pa +brock well +ban bridge +. ãĢĤ +ðŁĩºðŁĩ¸ âĿ¤ï¸ı +wu ff +van den +un loads +theti ser +shop clues +ri si +procter gamble +pom pous +photo books +ph ic +mon clo +krav maga +keep fighting +k als +i ht +exacer bate +dh l +de sil +cat chand +bo te +all and +:) @ +... ??? +ðŁ¤· ðŁı»âĢįâĻĢï¸ı +wear it +underestim ating +u en +tw m +ta fa +scen eso +ru dely +real dj +per re +pe gula +par at +or ji +oluse gun +o tha +o af +loving blogs +late ef +lamin ating +kingof pop +james bourne +j gh +in ya +har ty +h py +gore tti +go team +follic le +explor atorium +e checs +dt k +climat ecrisis +can wait +brun er +beck oning +atherosc lerosis +all road +ðŁĶ Ī +wee hawken +ty rod +ti enes +tail ings +sur fing +stam ford +silli est +self driving +sco splay +sav ini +sa kit +rock of +re snick +que an +pic ci +om ac +melani ec +mccar ran +marion spekker +mag ill +mad cap +lie w +ko bold +kee ch +its great +har ring +gw ash +fire wire +fenu greek +exagger ate +down size +davi dro +comple menting +br inson +book marking +bo chy +aln wick +alek sandra +acon vention +ãĤ ² +yi dol +water line +v co +thom sen +scor ned +s martin +re union +re finishing +pu king +photom on +nr g +n lg +megapix els +learnto code +kuri haran +kis sward +inter sper +hi sten +hae chan +go won +gn ash +func tionally +e by +colla ge +chris delia +ang in +anc at +worl duk +wee ms +un ranked +traff icker +thru sters +ta wan +stü n +stephan opoulos +sp reps +sc ouse +sa hib +rec itals +promo tional +pan nier +mic he +me scal +mar rickville +mal aise +her rington +hallucin ating +h mg +grapp les +gl int +ge thard +g dg +ful via +fow les +esp er +en mity +dream scape +don ville +dab bling +d sh +cu ck +constitu tionday +consci ou +by way +barbar ap +an elli +ad vice +ðŁijı ðŁİī +âĤ Ĥ +б оР+yuk ata +yogan anda +way forward +uk ku +twi sters +tr illo +tire some +tho don +the jim +tat as +quin ce +quattro porte +pre ening +perfect wedding +pad dies +p ws +nick kyrgios +mumbai kar +mai read +l ations +jacinda ardern +his d +h mb +gi sele +gener alist +gel in +gas lighting +gainwith xtiandela +fre unde +flat ley +fabric io +expre ssion +duni ya +del ores +brou wer +bor ro +bmth official +ðŁ¤ ´ +ãĤ ĵ +under growth +tuesday tips +top news +til lage +sceneso fulster +prince george +pitloch ry +ol me +ok sana +na ha +meg acon +mat ica +kh q +kab b +fashion style +enrich es +craft brew +chair man +canes football +c inv +be sted +ba ia +ba che +at oes +acci dental +absolut ely +á ij +un spoilt +twee tin +ric heli +re trou +rat u +raj ni +ps ys +po sta +pes ce +michael buble +mc naughton +match stick +mach ar +mac dougall +lur k +loren zen +lore m +lit any +len k +kw ong +ju gular +jhel um +inescap able +ha zen +farmto table +estim ator +do seof +cri mcourt +campan ile +bri les +b hoo +ar gin +aq ib +am ending +al bin +agon izing +ðŁİ¶ ðŁİµ +twitch share +the ed +speci esday +r th +ply m +patriot snation +overwatch league +ori ya +ny quist +nick jacksonyb +mo thman +me mentos +love whatyoudo +loo ser +jan itorial +hemo globin +hass elt +galla cher +electro static +ed ch +annot ate +andl rai +aan andlrai +a inge +ðŁĹ£ ï¸ı +ðŁijĬ ðŁĴª +ðŁİ¸ ðŁİ¸ +è ¥ +years of +wu yifan +w up +theo dora +ter u +sil i +r wn +pont chartrain +on tana +mon ae +men ara +leve sque +kell ys +kay cee +k cac +jun ichi +joe manganiello +in america +hir su +graffiti art +gra fx +geor gios +g eric +focu son +esqu imalt +eas tham +drun ks +do ku +diag on +de fuse +con dit +chery lofficial +cast aways +c jc +bring onthe +back track +îIJĴ îIJĴ +ç ao +z ic +waven ey +tol les +to sser +silver back +sav an +rebel lion +random weather +our ne +neen ah +monte verdi +hill is +giz aka +ge ms +emerging tech +east end +dev yn +de humidifier +dau d +com ission +campi one +calam ities +as it +ar ney +ðŁĴķ ðŁĴĭ +ê ® +âĦ¢ : +zel da +yu suf +win dia +urin ating +the st +ste w +spring y +so ga +sardin ian +ro mer +p tar +o el +lle welyn +kyo cera +key shia +hopkin sville +green eville +exi ge +eric trump +e aly +at ale +/ \ +yric fm +wyn dham +umb rell +th s +t ented +sur names +shilpa shetty +rtel yricfm +recumb ent +re combin +pit ter +om ari +no ster +no gizaka +nig th +master work +love song +kozhi kode +jerry seinfeld +itv thismorning +i ai +give it +fu so +fre dy +dor k +dit is +cent e +ca ches +balven ie +an jaan +aj ide +z hiv +um ali +tre vi +track pad +t illy +sw o +south port +shopp ixels +sac kett +rantham bore +q as +pa olini +n lc +mu tch +mcke sson +mc ing +manu al +ma sher +illamas qua +i ww +hoste sses +grim ace +golf digest +gen dering +from within +eis ley +dur um +cr tc +con toured +book birthday +ald is +ab ú +ðŁ¥ © +zul ily +wac coe +util ity +swar med +sky tree +rm hc +pot pourri +plant life +mor rie +jock owill +j eng +im en +fly trap +eleven ses +dog gett +do po +ding wall +cup ca +col n +bic he +beat ncds +bad ge +aun tie +amer rell +âľĶï¸ı âľĶï¸ı +yves ant +web kinz +warri er +w of +tu ras +tin kle +terri fyingly +ter t +stu yvesant +sk us +river dance +revolution ising +par ing +pag al +mull ally +mck ellar +mc nish +le et +kur la +ken zie +journe ying +jen kin +hux table +hsi en +gra bbers +go leta +flav oring +fenway park +escar dio +ed da +clu ms +ci mb +bo dn +birthday cake +arc light +ðŁĺİ âĺĢï¸ı +ÅŁ e +z ef +whit elist +w azz +vi mal +ti har +specu lators +sin di +sheh la +road master +rat er +petti grew +palaz zo +over power +omar athon +no ora +mu kul +moon child +mon biot +logo type +lash kar +kau shal +ju jitsu +jor ginho +jockowill ink +ing out +haz leton +harri man +group love +dre gs +dil ation +de mary +dar lington +counter productive +castell anos +cas satt +carly fiorina +ca ines +burn ell +ar nica +. ðŁĺĢ +wo che +whitecol lar +tu tored +thal asse +star crossed +school days +ridge crest +re shma +public health +post master +pegas us +one spn +nicky romero +net zero +national geographic +mineral monday +medi as +like a +kwa k +ke pp +jordan peele +honor é +home stretch +hock en +ha en +gla ive +gany mede +franchi see +fab rica +elo ad +du rex +dou ala +bluec laws +best dressed +an andi +aic pa +ðŁijı ðŁijĮ +âĢ¦ ! +zakyn thos +uru gan +uj jain +twitch yteam +tit mouse +thereal roseanne +tel lem +t anti +suppre ssant +suffra gist +su pran +racing post +paint work +of mind +nyc dailypics +nin er +nie meyer +n va +mur ti +maris ka +m gi +lit zer +klo k +kent state +jad akiss +han dof +han de +ger shon +ge kko +ga ja +fu hr +en zi +e pe +del auren +cur ragh +colon nade +be govic +aquare lle +aor ta +william sville +what not +vote btr +tor k +ti sa +tam mie +skysports newshq +si ge +shock proof +s research +reh man +q ade +pumpkin day +per on +mp b +megh na +materi alistic +lead enhall +lav ell +lac c +intl crimcourt +immu table +hat o +dw ts +cent r +bo di +bhoj puri +any ou +ahan nock +xox o +top notch +sal lam +sal afi +red rum +pon o +pis cina +on r +old days +mus well +merry christma +matthe wk +locker bie +ki i +ic pc +hendo polis +gal rani +finger style +dr seuss +collin s +aron ofsky +annn nd +ach other +è ¾ +ãĤ¸ãĥ £ +âĹ¼ ï¸ı +Ê · +win o +what ley +vint ners +t reads +stevemartin togo +sg x +roth bury +raj kumar +quad rang +pro publica +metaphor ically +mat ey +life after +ke mer +i etour +gou ge +g fi +frees at +fis d +elisa beth +eh le +ech ino +drive train +cór doba +cla ud +cl wyd +chaf finch +bla den +bad asses +aren o +am ita +ðŁĺĺ ðŁĴľ +ðŁijı ðŁĻĮ +wor s +weare wayne +w mn +victoria justice +tsa ww +tram ore +tecno logia +tap out +ste inem +social change +shu tu +ong allery +obsole scence +new tg +napp er +missing dog +mil ked +machu picchu +lv n +lolo lololol +li day +le moyne +kann apolis +jr h +inter mountain +ibm cloud +hang eng +fruit less +emphasi sing +divi ders +dh t +daft punk +bil as +anatoli an +ðŁĴĽ ⾨ +zeec inema +y ae +under fire +tuf te +stu mpy +rochester ny +prou k +preci pice +patri ka +mon de +lincoln city +iti onists +in ism +im m +ice fishing +duba imall +den so +dead pool +datac enters +darren rovell +d hol +co field +celebrate monday +cattle men +broo dy +broad hurst +bid vest +bal sa +ath el +an ke +an acortes +ad har +action aid +... ?! +ðŁĺĤ ðŁĺĬ +åĽ ŀ +w underground +view able +seclu sion +sali da +river dogs +re cline +quick ness +ol one +nx ne +l wf +ko ba +king stown +kevin richardson +kal ani +ju u +i us +gafe summit +estate agents +e mad +deme tri +born day +bent leys +! ðŁĺį +yedd yurappa +van gi +tristate wx +se molina +ro tarian +pou ssin +phosp ital +or de +okal oosa +newtg ingrich +me ar +lo pa +laidback luke +l fe +jefferson ville +h sb +gan assi +frédé ric +fli t +fla b +expect us +escap ades +discre pancies +clo ison +car sharing +cado gan +bird news +ba shment +ag rande +/ ) +ðŁĶ´ ðŁĶ´ +wit tering +waynes boro +tobo ggan +tn l +tho rens +tele pathic +sport sperson +sho bha +se ef +rit is +raghu ram +popl ar +pol onia +po tu +park ville +nomencl ature +nas cent +mtv news +mi dem +la sco +ku h +jay ate +ip g +ill or +hor rocks +ho co +gar forth +fr itos +fish monger +ebbs fleet +dru ck +del is +color guard +ble dis +av ni +as ker +arti sth +ac ast +, $ +ðŁĻĮ . +ì§Ħ ìĺģ +zo brist +za e +w pu +v rou +ut tra +torpedo ed +sur o +ste tter +seaf oam +sam sun +ring leader +resusc itate +rare bit +r gm +pointilli sm +pen fold +o aa +mc suk +ma ug +liveon wlos +kiwan uka +john tory +hyp no +ga hanna +fr ing +end al +en ation +dere ham +decor ative +customer success +ciene ga +ci marron +char o +card game +bra am +bom bo +bal in +aircraft forsale +ab ela +... !!!! +ðŁį» ðŁį»ðŁį» +âĺºï¸ı ðŁĺį +ا٠ĩ +wai lea +unitethe union +tippec anoe +sympathi ze +suppre sses +sinus itis +shale en +sac c +pro syndicate +pra vda +ph oops +off grid +novel ists +kx ip +kol ar +k ort +intern at +h plc +gu ss +gan nets +feels good +fan on +encu entro +devil ishly +cul pepper +cro quette +cho on +ban aras +bad in +ba ji +ari falvi +ai se +ðŁijıðŁı½ðŁijıðŁı½ ðŁijıðŁı½ +ðŁİ¶ " +wi ggy +tw u +tri partite +swa in +si van +re ham +per plexing +neve ren +morro wind +mono clonal +manu kau +ic ad +i at +hy wel +ger bera +g go +fra zzled +esc ul +el lef +dj fresh +confe ction +c gp +ast anglia +aldub nation +ad vic +ðŁijĮ âĿ¤ï¸ı +yu e +wy ler +wh ome +water less +uto pi +unquestion ably +salut ation +not ching +mans bridge +man tel +mali gn +lil ongwe +jon ge +iti zation +hunger strike +hol ter +hit the +f pv +dy brand +dun st +crumb ly +cr r +city scapes +cheap trick +catalo guing +born today +ban gui +ðŁĻĮ âĿ¤ï¸ı +win net +vi xx +un detectable +to co +team canon +seatur tles +santo sh +ro timi +rake sh +p illi +mon otone +m fa +m dd +luc man +kun z +kag ami +hometown hockey +giorg i +fl ings +eu lar +down wards +brae mar +ben ihana +be kaa +b ors +b nib +ar ko +." -@ +ðŁĴĥðŁı» ðŁĴĥðŁı» +îĢ ¢ +ìŀ ¬ +wladi mir +wash ing +u tero +tom an +super soul +su iko +sk ou +s museum +pi h +pen folds +parnas se +osoy oos +omni vore +ml u +mc minn +koda chrome +kno cker +harpersbazaar us +fit for +ep ine +e bu +dublincity uni +de bre +dav a +c gt +boy zi +adap tive +sto dd +shenando ah +s ella +run a +rickman sworth +re agents +prece de +pre meditated +poster deal +ou chi +one for +noti fies +n awa +mo by +middle sex +ket one +illumin ator +ghe art +fac tored +ess aou +en visions +davi dortiz +ca iman +brain wave +ba ine +ba illi +an ari +ak id +ab ry +ðŁĵ ĵ +ê³ ¤ +âļ½ï¸ı âĿ¤ï¸ı +vesta via +un subscribe +ste med +sp hero +sound stage +sol ari +scrun chie +regen sburg +pet friendly +n aff +mell itus +m wp +lump kin +liluzi vert +lets getit +lavish ly +laver ne +l ba +jiggly puff +jd mahama +jac opo +hermos illo +grou chy +gr nd +emb u +deb ase +cyg net +couturi er +can tile +buckey enation +bel monte +ar buckle +air tight +ðŁĴĸ # +⼠° +yeg cc +ye chury +that cham +tele marketing +tat chell +tai mur +sy doperahouse +re targeting +per missible +notre dam +mani kar +kel si +k pc +jurispru dence +jab alpur +il way +holly wills +hem poil +harvard health +h cf +fro sts +eaves dropping +cu zz +con ews +cle tus +beach wood +alicein chains +af ana +abbott abad +äº ķ +à® Ļ +zombi eland +un playable +tu o +super highway +stir rup +sri sh +se tx +rock bridge +re tails +quo d +purch asers +pen shoppe +pashtun s +pan ola +ou ettes +nai robi +menin blazers +long bottom +lo sc +kü stün +keralab lasters +kaf fe +jackal ope +is sn +in consistency +hug gies +hock ley +hedon ism +hack ing +good son +go jira +gil lett +ge ther +gad fly +fanta si +expan sion +enri quez +ef es +d li +carnit ine +bom bard +bod ys +be sigye +bas ara +aggrav ating +ad hering +ab rera +; @ +ë§Īë§Ī 무 +veti ver +tri athletes +su itor +shi kha +pun tos +p aged +one and +n wilson +n atty +maternal health +lat z +l nd +kal ai +ka sim +k ka +jean ette +in accuracies +ic x +hu elva +ho ch +gur un +gou ld +expatri ate +en jin +ed show +du cked +diffic ile +culture night +contain eri +cho pper +can vey +bas splayer +ai ello +acre w +a ahhh +ðŁĺĤðŁĺĤðŁĺĤ . +ðŁį ļ +ðŁĩ±ðŁĩ ° +ï£ « +w ens +transp ired +top side +skysports f +six flags +sh art +seasi ders +sas saf +pun cak +po b +pbs kids +passer by +parfu ms +p sd +ol low +ne ill +mo sse +mid d +mag u +li thi +ke aton +k won +ho x +eleg ans +ein horn +cu base +conce aled +bourne mouth +ab ata +travel life +thank god +relev ancy +of god +nu ttin +mic hy +martin elli +london fire +lin sey +kx an +in ada +hy mes +gar min +fun do +fox borough +fal setto +eliza be +e ston +cu buffs +co dw +cast lerock +car z +cal mac +boycot ted +bor th +bal ing +apocry pha +anz stadium +ade mola +ze sco +zam or +wel t +v alla +u ve +twitch creative +ta virus +strad broke +steme ducation +soder bergh +smo ck +slight ly +shoel ace +shab alala +sabras radio +s art +rockst areditor +red bud +real c +li oni +implement ations +hiroy uki +har well +gu any +fish friday +fi af +elic it +dli ssing +dis organized +buzz city +bo panna +bap homet +bah ria +avi k +ap ids +alamo dome +al cala +the east +the bi +speak out +so rel +sel ly +sch ofe +sa thi +rice university +re joining +pon tus +out liers +on able +md politics +lun n +logi stic +liske ard +letsgo ducks +l ha +kar ach +jet son +hur n +hern ández +gautam gambhir +e hhh +dun gey +dem is +defe ction +dagu er +chol ula +cav alli +capri ati +bol les +we aned +til ts +ti bbs +thor son +th yssen +quadrup led +pho tons +ny quil +irrepre ssible +inst ab +ii p +holiday ing +hee hee +gym pie +frene mies +fero city +exhib ited +environment alism +el lish +dol ant +cw b +confor to +b ny +az n +) ", +ðŁijı # +ðŁįģ ðŁįĤ +yan dex +vent nor +thin section +tao ist +takeak nee +si sto +sebasto pol +seas ick +sat yr +ro settes +nc cc +mont clair +micro focus +ma quette +kil main +je mi +gur jar +gold frapp +g ce +dont buy +di ssed +d model +cru shers +carpe ting +buffalo trace +art work +ar men +an sett +an ha +al ah +é º +è che +wes sel +val ve +tu v +tran salt +sol ym +singapore air +sch ap +revit alized +reg is +priorit ising +min ny +mi dri +mcmaster u +loo g +kon i +know le +kin ders +internation alization +i he +fab lab +ed filmfest +dad on +credit union +ci ent +che ta +be kah +batt aglia +ag ny +ðŁĴĹ @ +âĿ¤âĿ¤ âĿ¤ +x le +son top +so ko +powhat an +pe try +our ts +mo chrie +lau drup +l co +k cl +hi ye +handmaid stale +f end +diade m +d hara +case miro +bud dah +ano vic +ad sb +ðŁ¥³ ðŁ¥³ +� � +æ Ł +zam bales +wheat en +v scot +the legend +pri mos +pri madonna +pri ddy +pas i +official marko +must watch +lou yanong +la batt +jag ga +iti m +i ha +h pr +gal d +founders brewing +ep ers +e spark +clou gh +c tic +bha gy +abo at +ðŁijĩ ðŁı¿ +ç ļ +е н +zoom in +z oil +xbox onex +wa aa +tis rael +sw art +subpo enas +stu ckey +sor did +sic ario +sant é +rog elio +radic alism +play bold +p elli +op ta +nu l +norm and +ni shan +metro losangeles +medal ofhonor +ma iam +ley den +koko da +impregn ated +im ents +gw t +gar ri +edge hill +e cet +d cy +cu esta +col lie +cali sta +boxer dog +bishop sgate +avent urine +att weet +ak kara +ðŁĴī ðŁĴī +work mates +wal lof +town beer +south shore +roof er +rib fest +ri et +reimbur sed +ra zi +prayer ful +po pa +paignton zoo +out fitter +ni hal +m fd +killu a +kc ca +iron de +h ita +ei lean +e gr +din sider +di stance +de shawn +dart mouth +cinnab ar +boo ze +bi det +! ^^ +vach eron +uttox eter +ur fa +uno cha +tri stram +ten ough +stam endment +siski you +ser ing +scottho ying +rag brai +pay day +o gl +mcstu ffins +lbor ouniversity +kill inge +imiss you +graham rahal +flip grid +electric cars +city line +br annan +bad la +av ola +annie mac +ad one +.... .! +ðŁĺ«ðŁĺ« ðŁĺ« +Ñĥ ÑĪ +wild food +wic om +war ping +walu igi +vi agem +tr oughs +speci fying +say aka +piercethe veil +orig en +nh d +ma wx +la han +k Äģ +is better +inhib iting +i wu +holy wood +h fd +goosen eck +ga ine +g our +fm sa +f also +ey news +elope ment +dew point +che tty +ca org +bas qu +am ra +after shocks +tyler perry +tread mills +then y +terri fies +stor r +statu ary +sl tchat +shuffle board +serv ation +sc our +sam cro +rif kin +rebe ca +re tweeters +peanut butter +ni ah +national beerday +nas ci +mous er +me ji +lea rena +kristen sen +ino e +he mmer +grac evander +goo ch +go socceroos +go j +e tal +domest ication +do zing +dev net +b mar +amb az +air bu +acol lier +ðŁĵ ĥ +ðŁĴ ® +èªķçĶŁ ç¥Ń +west chester +un friended +uk raine +toll way +the americans +sty dia +stein metz +sar ong +sab ir +nieu we +mor mont +marry me +ma plin +kni k +kni fed +kash thefuturist +ih f +g wil +fun n +dianna agron +bry anc +blues y +anton ella +an ar +ðŁĺŃ ðŁĴĹ +ðŁij ³ +ðŁİĵ ðŁİī +ਠ¸ +ye hu +wil bon +to tten +ti zation +ssi s +sh anti +sare gam +samaj wadi +rot ted +realtor life +qu alls +perez hilton +om gb +never settle +nationalboyfriend day +matthar dybrand +mart z +lovel ier +leg omovie +lat rell +je sh +iam diddy +hypo chondri +ho ppe +great night +firestone walker +engv sa +dr k +di of +counterfe iting +counter attack +c wc +be shear +ar chang +al mas +ag oo +ðŁĻĮðŁı¼ ðŁĻĮðŁı¼ðŁĻĮðŁı¼ +ðŁĩ±ðŁĩ » +z oll +ton da +te shwar +su ara +ste mon +se moga +sau kee +sage brush +running day +ru eda +re install +rancher os +quean bey +our home +obse ssively +mike e +ke x +k ila +in wards +ill man +he ske +hasle mere +free birds +dist anced +clich y +cen sure +cb gb +b ateau +aspart ame +alti eri +ad mu +ðŁĴķðŁĴķ ðŁĴķðŁĴķðŁĴķ +ãģĵãĤĮ èģ´ãģĦ +whoopi goldberg +the chew +thak ur +tele text +su tro +spo tt +sa iz +rak ash +prismacol or +penn jillette +one gin +molo tov +mat sunaga +ma ir +kristen bell +kil by +ki z +ine ss +in cul +immun isation +hur tigru +hur rell +holocaustre mem +he me +girl sbball +ger al +fo or +earth works +dic tation +di one +dead head +de ts +bru le +ave dra +at rust +amp as +> :( +t vin +signi fying +pon ce +n gh +mon grel +mo sko +mil ind +jor d +j df +ide olo +hari haran +han bok +g ander +et c +dere ck +ce ca +cau sally +car rol +bbc cornwall +ap co +antigon ish +allo f +aldubeb tamangpanahon +warby parker +wam u +trader joes +ter an +stock yards +soon er +season al +pra gue +most ly +medi am +kul fi +kor tri +j mb +i yan +hershe ys +her be +entr at +dur kan +digital trends +cot terill +chante relle +cen ote +cantstop wontstop +bro mpton +british cycling +bo of +bo cas +bayreu th +b oud +ayo tzin +aw ing +al ans +agra phics +ago e +ê¹Ģìŀ¬ ì¤ij +ye vans +wy mond +wu xi +vil sack +un processed +se quality +s foundation +s bts +pho g +pang s +oc b +makeover monday +loner gan +live streamed +lister ine +ku be +joe perry +iron clad +hur ls +gro ening +ff x +e set +dian amary +cyber tron +clem ence +c we +ber gin +belli ssima +a op +welove bath +up v +un consciously +the avett +tashan eishq +tari fa +t á +sport car +song sinfo +snar l +sho ves +sco tref +reli shes +prob ationary +pre ity +park head +mazer unner +lin ed +kan ak +ich u +i et +hy pere +heide gger +g ms +esteel auder +ec ook +do ws +cra yon +classic ism +ch ail +cello phane +cathar sis +bri stle +bra thwaite +boo o +ba az +ar vi +ald red +ag ama +you can +visit bath +v fs +trust pilot +stree twise +random ised +r sw +pur pu +perman ence +otol aryngo +me gumi +may pole +lo dg +lan tau +kuro da +kill switch +its jeremyscott +infar ction +gul lit +go shawk +geno type +g ó +fal tering +du ms +de be +corner stone +chert sey +bul u +bu tting +bol dt +bo ser +bay sox +al ph +ab lett +ãĥķãĤ© ãĥŃ +ÙĪ ÙĨ +wire tapping +tom eu +shapp ening +rev war +qhu beka +pla ine +or adio +maynoo th +lo td +lets gom +lee ches +larry fitzgerald +ju le +jrs bbq +humb leness +harve sters +groo te +go saints +dru zy +de ws +dag ga +con tu +bbcn wt +arts fest +anci c +yan go +woo dridge +triple crown +the ft +t ton +summer iscoming +sub text +so ori +sarah m +refriger ant +real preityzinta +re sourcing +pol amalu +out land +nihil ism +naw al +marig olds +m ite +gu ti +gro sses +geni ality +fear ne +eu referendum +ehlers dan +dig ne +cre em +co design +blon die +abbe ville +ðŁĺĺ ðŁĴĸ +âĶ ³ +w fl +un flinching +trump is +ton ton +thinsection thursday +t tam +sun land +sr j +se ast +re pped +r studio +quarant ined +pbc sd +outer hebrides +outdoor living +off re +neg ation +mm ering +martin freeman +man cup +mac ewan +lobla ws +lin ke +kav ya +kam la +k lo +histen vscot +han gry +green wood +gar in +flori sts +flash dance +fe scue +dic ey +deci mals +chris froome +cdn health +brit a +bono bos +bl ay +bil lets +ber na +barley wine +apit ol +abb formulae +⼠ı +ਠ¤ +zieg feld +winter bottom +vo cs +tou jours +team er +te em +rohing yas +ra ba +pasorob les +n cube +mel in +mega watts +medical device +kuan lin +kinder morgan +impac to +gom er +fu taba +fu shimi +emb o +elu ded +ed man +discre etly +dan afarber +cypri ots +cot illion +boister ous +be ee +appomat tox +am strad +am ers +al tro +abhin andan +wwe tlc +work site +womenin leadership +williams ruto +v rt +trede gar +tay y +sf old +se te +ril akkuma +palm trees +na day +mil in +mas ada +mad res +live and +ktg tool +fir man +etsym ktgtool +dren the +daz ai +ctv montreal +cre asey +ci en +bor as +beem ji +and ries +ach amber +ðŁĸ ĭ +Ã¥ land +wink les +wallpaper wednesday +walk ability +w amp +tre tweets +transfer news +thevampire slayer +srimanthu du +shu bh +shrews bury +rei dy +que sting +pun a +par cours +pan jabi +ori ente +opp y +ne em +magsay say +m tech +m ended +london live +log ÃŃa +leaf sforever +kry stle +krist ine +issu arez +insan e +employment law +comp ounded +clin cher +carr é +bethe one +beach volleyball +ðŁĺĤðŁĺĤ # +x press +victor i +tm gt +sulph ate +si mages +ri ffing +ni guel +ne ke +mouth feel +main net +mac tan +ke met +john shop +inter dependence +humor ist +hawk girl +gu yer +fi des +fab b +dele o +dairy queen +bartol i +anor ak + ¾ +y anga +wood men +wh is +wade bridge +vindic ation +v so +uw bb +u in +th z +sun life +su as +steam whistle +spoke mon +spaghet ti +snicker doodle +pan ik +on music +north we +meach am +lin nea +kun ingan +kann an +ith ia +id night +humboldt broncos +hick ok +hershey park +ge ol +freder ica +flu oro +ex uma +dro ols +dispar aging +con air +cler k +bu ick +bi du +as ync +argu ment +an stey +wait for +vz la +ver a +vand a +u ot +tur ke +the creator +te arooms +tam as +t shirtday +sou bry +sand castles +re mixing +pleasant ville +phi le +park wood +o gc +n cap +moris ot +mo hs +mary dale +mam avote +kap alua +justin bieber +hal sall +greate scape +far myard +empowering women +bra him +black burn +biaf ran +bar be +aff able +ó nica +zombies run +wi der +wh ence +vig our +tu bal +stock piling +stan ton +shaf ted +phra sal +o ÄŁlu +mo wat +mascar as +mark levin +lion head +j nk +j ati +he mming +gr ice +ga it +es wara +el ora +ehr lich +drawthis inyour +deu ce +dan ge +coming outday +b ise +am ad +aldu bang +al insky +aggre ssions +abbrevi ations +wol dt +vh sl +trueto atlanta +theloud house +sub ha +stat s +stanis las +stag ger +self build +saha bat +s bo +recap it +rajni kanth +puri st +paul stanley +nau lt +msg networks +mon ts +mo selle +michael strahan +le may +ky speedway +kiri bati +infin eon +ilau per +hg se +henri kh +gc as +fi bs +fain tly +encapsul ation +dham i +de mer +cynd ilauper +brisbane tennis +boss anova +arom atics +ðŁı´ âĢį +à´ Ĥ +yo d +x fre +un concerned +tu am +to gram +tar kovsky +syl het +sy mph +ste g +school boys +scho ck +sch ill +sas sa +sas i +ree led +re hydration +psy chon +pro ffe +pon zu +play testing +official asroma +n anny +mon ie +meg ach +lo wes +kot tay +kai den +k gal +inter fered +hydro cephalus +ho fe +green well +ge urope +fried lander +flick ed +fishing life +dyna stic +digital humanities +di vino +dail yui +coral reef +cit rus +chow dhry +capac ity +bro yles +ber isha +austin dillon +:) # +ç » +ste mi +shoto kan +sh xt +sem rush +poly ps +pap ag +mo dot +marketing digital +mal lows +lad ner +inver ters +im pair +frost burg +fin is +enchan tress +em ol +d kk +clean room +cit inews +c ck +beat mania +b ican +ali ef +al bee +âĿ¤ï¸ıðŁ§¡ ðŁĴĽðŁĴļðŁĴĻðŁĴľ +üyü küstün +ze v +ygg dra +y upp +web designer +ve tter +tune up +tri ffi +teng ah +spi ker +silen cio +sat ara +roll call +red cliffe +re routed +ran k +ps news +per so +n radio +n cri +mun ros +mitch grassi +mis er +mil hist +mar otta +mac edo +lo am +kim a +kenne bunk +kati punan +in wed +highland park +far ms +engul fs +chym al +catho de +back board +attach é +ane wh +akhil akkineni +è ¿ +w lan +tend on +suvar nab +strou d +signat ories +reedtimmer tvn +my t +mohegan sun +ml f +marmo set +letterbox d +lane gan +is caring +inst illing +ine jad +higgin botham +gai ag +four thbewithyou +forti fications +dj p +dige stible +dic h +bath letics +ayahu asca +z cash +yan kton +wendy williams +twom ey +ti x +tar nish +super lig +sle ft +refugee week +ra st +pan ova +nectar ines +mao ists +make it +made jski +jah re +itas ca +iam su +honour ary +ho ve +hin ojo +from is +fl atul +down wind +dom ini +dark art +cp w +ci dade +che gg +cas am +ca reless +buffy thevampireslayer +boreham wood +bc tf +ðŁĺĶ ðŁĴĶ +world star +ur ine +sunburn festival +strang eness +snyder cut +ser ums +screw ball +sauvi gnon +rust lers +rag land +q ed +po va +past imes +over stated +os l +ne cc +me ed +marydale entrat +lv mpd +lin q +ja anu +ick y +i ghter +how toge +guy fieri +gro ans +ell oni +ele k +dax ter +cor sic +bread sticks +andreww k +ai st +ãĥ ¦ +âĿ¤ï¸ı ðŁĴĽ +าภ£ +wa shoe +ver is +spoo py +spe ws +spark man +snee zes +sn hu +silver dale +seth meyers +sensiti zed +sen yor +san ton +s fe +rosal yn +play adel +pe ale +ovi ya +newcastle jetsfc +mun dine +meetand greet +m bia +looks like +jo vi +jimmy choo +j ls +isu mmit +heritage week +e chuca +dro ga +dream warrior +ce ren +captainameric acivilwar +capit alized +broad water +blu est +be ppe +bar dsley +zild jian +ye mi +weigh ty +vas ant +tu tto +tex tural +te b +tag ma +streng thin +step ney +sta u +sli pper +sel loff +radio logists +mole sey +laver ty +lan er +ky humane +katamar ayu +hay lie +god ha +exc itable +e gov +dis arm +chak rav +business week +bil son +ðŁļ Ķ +ðŁĴļ âĿ¤ +ìĦ± ìļ° +wu stl +wr angle +ve ers +val polic +true man +timbuk tu +sch ris +sam s +sam in +salt burn +re affirming +pu ffer +pp b +po yet +par ia +p life +ow ww +mono tony +lon avala +l cc +kir st +ju sty +jan ec +is ay +ino u +inci sion +ha segawa +gust avo +fn ce +fag en +eric fisher +eng rish +dino vember +dharam shala +deutsche bank +best jo +-____ - +ðŁĶ ¼ +ðŁį· ðŁį· +ðĿIJŀ ðĿIJ +ula an +typho ons +tree tops +spra ggan +sb x +retur nees +r ila +pro ff +play pen +performing arts +newar tist +nd lovu +nascar playoffs +mur rell +maren go +ma kenzie +k ays +jun ction +jhal ak +ired ell +i der +glo bu +for res +fon tina +feel slike +es n +epitom izes +e tive +disp elling +db x +cro marty +co que +clip studiop +cher as +ca os +c pd +bomba stic +bed sheet +av ca +ac im +aay yub +za o +vo cm +va sec +tic ians +thom ond +sta at +sm un +sep si +re ton +pun g +peri ences +pel ting +pay checks +ome dia +o ban +o ake +nh wx +multil ayer +morpho logical +media set +fran cona +flat mates +eu gen +elrey network +digital skills +cross walks +col ten +bv b +aw wa +andy black +anand mahindra +è » +à¹Ħà¸ Ĺ +ú a +z enger +yn w +yel chin +x plore +woun ded +t fully +sv n +sky bound +shu bert +sg l +se de +san ews +ri ds +pu chong +ost ent +live tv +keegan allen +inver sely +gle aned +f hi +edu tech +dy stonia +del r +dar rell +cor am +bou les +be vo +ar nott +anci en +alo of +water stones +u buffalo +sweat shop +sto icism +spot ted +show box +read ability +rav age +raj e +r ze +py r +mip com +mi edo +mcdonnell mp +john mcdonnellmp +head first +glu ck +for india +fon terra +fill more +du ero +cam il +anthropo genic +and sara +amandat apping +al lee +èĹ ¤ +y sp +y hu +winter thur +trabzon spor +sumat era +sta i +six pack +rum ney +re style +ple gic +pal lotta +music therapy +metal heads +maythe fourthbewithyou +matth dgamer +magi d +le pen +l sr +kim s +kala ign +jun co +jamie foxx +it ates +imagine ering +il son +go snell +glen ville +gam ble +fibre glass +el dar +doing it +classic tv +boo lean +ba yof +b ster +art scape +aj p +zind agi +visit cz +verraz ano +un scented +uhur a +thevamps brad +then ame +sh ama +serious ly +sas m +rose of +red foo +ra chi +r bf +q al +orlando pirates +mor ra +mission impossible +me hl +j elle +hilde gard +ge eth +g nd +fro sty +family friendly +chris stapleton +cb cradio +cacci atore +bur ne +bak tan +aussie grit +astro world +archit onic +an col +alban o +wil kin +vindic tive +troglo dy +ti sci +standup for +sm tv +relais chateaux +propen sity +mmmm mmmm +li ppo +gett in +get better +fil ia +em ilion +e sen +dol in +cityof vancouver +cal oun +cairngor m +bay hawks +ate er +amer it +âļªï¸ı ðŁĶµ +vi st +tit ic +sur rep +sty linson +stu sa +stego saurus +sonam kapoor +sau cep +rat aj +pv l +pike ville +pando cruises +ny phil +news space +nand ita +n du +megam ix +m musi +lmfa oooooo +jeff flake +j anne +ir n +implo de +ger da +fi ras +cra gg +con fig +cle ave +ch anna +ca stration +bor gata +bellige rent +as cs +ab kiba +âļłï¸ı âļłï¸ı +wom w +ti ous +te sh +syl vain +summar ising +soccer aid +sk or +ra om +oe il +north norfolk +mo ët +mjol nir +litt lerock +le var +le pt +kelly rowland +k hul +jo kic +io an +hedge funds +ging a +gi otto +do ink +dianamary sharpton +cow den +com d +chon dro +cal eg +c sh +ber ns +americ andream +wa ster +v ons +t fg +sud ha +stoo ge +spl ace +so chi +seal ants +photom ag +pedag oo +mc muffin +mammo grams +lat ure +i in +gram pians +gemm ell +for ager +ev illage +de toured +day fiance +cor ry +con notations +ch ink +biker un +ar ima +ado on +ìļ © +ãĥ« ãĥ +иР¸ +Ä Ļ +whywe do +villar real +twy ford +thelo cal +tc cc +squee ze +si sson +scoundre l +sam mi +planet newsspace +peek skill +p cr +one india +near pod +ms ba +malay alam +jail broken +jag gi +grou pe +go bulls +gi ds +g fe +fra ga +fox tail +fitness goals +fero ze +evolu tions +en trust +ecu rie +den pasar +culmin ated +confis cation +ci oc +chem nitz +bron cho +agu ero +.... ...# +æ £ +ãĤ¢ ãĤ¤ +ãģ¦ãĤ ĭãĤ +valentine s +turksand caicos +tubab üyüküstün +tour ing +ta von +stay positive +star rs +sa af +ro isin +pollin ate +plan ecrash +phoen ician +o des +mt z +mau ro +manag e +lafit te +l sf +ki ka +house guest +hispan ici +fin tan +dead liest +ctv windsor +cop p +coon awar +come froma +chel ly +charn wood +ch up +brown ed +ar amark +aerop lanes +z elle +willand grace +watson ottawa +v elli +un recognisable +to ti +ti bi +talk live +so st +show tim +ser co +re tort +pd ate +moren cy +mor ita +mil pitas +men udo +manag ed +m wm +kh saa +intern acion +incar nations +helle bore +gas a +as ra +alternative facts +alca zar +ðŁĴ¸ ðŁĴ¸ðŁĴ¸ +á¹ ĩ +ve sey +underthe stars +tre bu +thisi sa +sp azio +sau dades +ruf ous +represent ationmatters +re wrote +ram esses +profe ssed +pay ette +pa quette +ontari oparks +nam ish +mill in +men dip +man es +jic hang +ja ide +ice a +hos ny +glu tton +es ke +es fc +cabo sanlucas +bla ine +big fm +ben salem +bc m +ame obi +: [ +Ê Ģ +wi es +wak ulla +thelibr arians +the bo +strong sville +sportsman like +spe aring +skri pal +sc aggs +repub blica +q asr +pix ect +op har +ontari op +national lottery +ko ss +k lassen +k ere +jungle book +jav an +intervie wee +home maker +ham ara +ha ssett +gar ages +frank sinatra +fin ning +england hockey +df n +dan ske +d pg +craz ily +com patriots +but an +bor thwick +bel anger +anu sha +an b +al mighty +âĻ ¨ï¸ı +va stav +tis ans +te yana +tay to +stay lor +schen k +sanc erre +recy cling +priyan kac +pau lma +mu see +monte fiore +mmusi maimane +load out +kis an +k los +jim watsonottawa +ilo vers +i ate +hun ga +holiday shopping +h pu +glo ssier +form work +fir dous +far well +falcon pride +en our +craftbeer hour +co ren +cat on +car news +c isa +bam f +ad k +a hari +ðŁĻĢ ðŁĻĢ +wall flowers +visual arts +turk men +tanquer ay +sol des +sla dy +sch nee +sar ina +rick grimes +revolution ised +pu gets +pra ga +post ables +mt scores +moder na +min are +mar ula +man tic +long listed +llu via +lawren son +lament able +kil dare +fr and +dri dge +devon days +de gea +dar c +dang an +dal is +coral reefs +con gee +col bie +bo ji +be sity +barn well +ak zon +ĥâĸĥâĸ ĥâĸĥâĸ +Ø§Ø ² +wolf ville +um news +tran ada +tin sel +tar ah +son gh +skill set +see red +sad hu +ros anne +rober tir +po zz +po we +nb p +lasor da +l sb +kun dan +ke b +ja ques +ind superleague +in oc +haku sho +gri eves +gov ph +gi kwang +f mp +eni or +ditch book +cut scene +bergha us +bander snatch +ant ena +an adar +agap anthus +íĤ ¤ +thur ber +thoro good +spe zia +sh eckler +sea ham +sc aa +re learn +punctu ality +prop eller +prayfor paris +pen se +p caa +os er +nam ib +mic hu +mcclo skey +mac laine +legu me +la ba +kc star +ir cuit +iam rashmika +he ol +ground skeeper +flower pot +f mb +e online +du bose +di atri +corn ering +chi ari +cad ena +cab an +bussel ton +brock le +big ly +au kerman +y ellin +wal der +ut leg +tahle quah +shat tuck +se min +rosen feld +republic of +ren aul +re star +ok ine +n naji +mumbaim irror +mar joram +ki elder +jump y +john sons +iran regimechange +in bloom +hom iny +ho ssa +hal as +ey c +eff ler +dynam ix +di vot +den sities +conceal ment +cha v +buri als +bs official +bloomsbury books +armid ale +ador no +$ / +ðŁ¤ · +x wb +uc sandiego +team liquid +taze well +stefan o +ramsay z +people power +north sea +neo dymium +multit alented +m els +losange les +kh u +jo yl +idy ll +houseof lords +en gram +diver ged +dheer aj +dash on +d abo +cra bbing +cr ony +cor rs +christmass ale +am supdates +allyou nee +ðŁĵ Ħ +ti bi +ryan lochte +roysoc chem +ro ddy +por tre +ol av +oh m +o dac +newh art +nbc chicago +n no +mont parnasse +mid ges +melbourne fc +marin ecorps +magicrock brewco +lu da +kay lie +i pac +hed nes +hand els +gameover greggy +frau en +fe as +erici dle +end ura +e hi +diarmu id +cp n +blue bombers +anat ole +ab bie +. (: +åħ ¥ +ãĥĥ ãĤ¯ +ü h +yam agata +vin as +vi di +thro bs +the journey +reli efs +quiz let +preco cious +pre fall +por ation +pe irce +ox x +no gi +mallo famerica +mack ay +kin card +ke g +kam m +is af +gol maal +giveblood nhs +gha stly +gh illie +fo bs +earth worms +deo dato +craw dads +bobb ins +beile in +az ul +alab s +ach em +... ðŁĺĬ +ðŁijī ðŁı¼ +tor iyama +tnt drama +sherrod brown +scru tine +r gt +poo ps +obtu se +ny kaa +mul ching +makin ghistory +latte art +laken heath +jet stream +ir by +guine vere +fion a +eni ghts +de ped +cat girl +carnegiem ellon +vill ers +uchi da +ski v +si fu +sc a +sbspop asia +s ape +plane tearth +pic os +pet z +parsi pp +metac ritic +lu sting +le tty +in humanity +ima an +haw ke +graven hurst +grand i +glu tam +en gie +devin nunes +dev fest +cur ving +bud ha +ave o +atalbihariv ajpayee +ao ka +agu st +adri el +aco splay +* ¨ +ìķ Ī +ãĢIJ # +za die +wat ney +washington ian +unfor given +u oc +tax on +sul k +squat ter +sp asm +side bottom +shut downs +sand bag +run with +re producing +r br +pal ast +op io +novoro ssi +nin aturner +me shu +ky d +ju rist +jen nab +jami at +in chem +hy bris +gracevander waal +dav on +dak ah +care ers +carbon ite +car ley +bun a +bal four +amazon video +al cor +ac is +ðŁĺĤ ðŁĴĻ +ðŁį· # +ðŁ¤ŀ ðŁı½ +ìĥ¤ìĿ´ ëĭĪ +åIJ į +⼠½ï¸ı +wee ded +v tol +temp ers +sc ute +saf dar +reha bbing +purenew zealand +pul liam +publi b +pri sma +poun cey +polit eness +mccl at +maggi el +kil mac +juic eplus +it sha +i fa +i ava +hir t +hak one +fac tually +explore more +ell c +discover tasmania +day lights +cosmon auts +cer rito +burgess dave +bron chos +ap ter +ani mus +ale g +ab ag +ðŁĺħ ) +wo ky +wag ers +wa as +van go +un refined +u streasury +trum prally +too ele +t acked +soldby auctions +sk ank +sirac usa +pu mm +prote us +parach utes +or nothing +olympi atheatre +mus kies +making memories +li ard +ge mb +g ch +fr its +eu stis +esc congress +emotional intelligence +di mmed +culo ttes +chall is +cer ner +books thatmatter +bo das +aperiti vo +ad ice +ðŁĺķ ðŁĺķ +ãģ Ĭãģ +zip p +your mind +whe ad +v ap +u co +trav chat +thuman ity +te abreak +szcz esny +synchron ization +su stran +ssi o +ske cher +sati sh +reconc iling +rec tified +ra wi +petro bras +nu dges +nom us +mor ag +mag de +hr l +hel mer +h lp +fear on +fa strack +euro gamer +drunken ness +div ar +den nard +d cli +conserv atoire +club app +bu ty +bloom ber +bart els +bam ako +angel as +اÙĦ Ø´ +Ì Ĭ +u za +thupp akki +shu te +sandra bland +s assi +re play +pasi fika +p vi +monstro sities +malay sian +macca bees +ki ps +imper ator +gam bo +g pac +eloun or +bha g +ิ à¹Ī +whit sundays +voy agers +river head +non partisan +muri el +matsu yama +maje stically +macro phages +insu fficiency +inhab iting +ilooklike asurgeon +high smith +guitar ra +goti ges +flu ential +fai led +fac man +de ten +clement ines +chu u +carolin elucas +bre be +bigg le +ajay maken +affo gato +ac abo +ðŁĺį ðŁĺı +yu ga +will mott +wheel ies +wee l +wb ina +squ ids +skeem saam +scar cely +proud mama +poc keted +plac entia +penn zoil +paulstanley live +nak ano +mir zas +mcke ever +mc michael +mc flurry +m dn +lion sofficial +lily ach +le louch +indi gnation +horse head +frie sian +food drink +eval ley +eric church +drum set +co rel +brze zinski +bit sch +banned book +alar sson +abscbn news +yebo ah +wehr macht +w out +uk nikon +tar heels +sw y +southern charm +sof ty +smi there +sk impy +sk ane +seatur tle +se ely +s fund +s da +pois oni +ok an +mn n +k any +im possibility +ico tin +ho cks +hed berg +head stock +he aped +flo ss +eng lander +delinqu ency +debut ants +crystal palace +conven tionally +cich lid +car mody +boyzi imen +both vote +atat ürk +ann ag +ank sha +air o +abra xton +ðŁĺ³ ðŁĺį +ðŁij ŀ +âĺ ł +youn tville +y air +v cd +tv f +te gel +stor rs +si han +shi fa +shaz ia +q az +picture cumbria +pass more +pa ined +pa bebe +mar rs +lu y +ju x +jam at +happy thursday +guing amp +grand daddy +f aya +daruss alam +bol zano +bol din +bar is +ax n +ali as +under pin +tu to +tall i +styli sed +simp kins +santam onic +ry lance +preserv ation +ppy grewal +pix er +om t +oireach tas +obste tr +namish taneja +na res +miro tic +mini me +m ta +kaise ki +ities news +hay ling +government shutdown +go wildcats +ey er +dra win +dol f +discover on +di bble +del la +d jones +crun ches +clo p +ban yu +bali stic +ba at +as alonga +an tuk +alber ti +? ðŁĺı +ðŁij Ķ +ðŁ¥ ĺ +ìĪĺ íĺĦ +wa ir +unevent ful +unbe arab +sin os +scottish borders +q o +puri fiers +por i +pl ena +no pd +nj siaa +neu schwan +neck er +mun tin +mill z +ma ith +lost dogs +ld jam +laugh er +k ml +jal sauk +irrit ates +il us +hr h +hope for +ham line +ha be +google analytics +god spell +ge x +edu n +dreamwarrior pic +dave mustaine +che yne +cel so +cal umni +besti ary +bankof england +bab ie +awil de +ad achi +unir dg +tu chel +then est +swar aj +sedge field +resor t +qur anic +qual ms +ok on +nak ai +na oko +macau ley +locu sts +leve son +le sa +le oni +juven il +j ere +iv c +in ane +h acc +guyan ese +gor tat +goal ball +go kul +gho strider +eddie redmayne +ed int +cor tney +claw sup +ba ar +at ap +ar mament +anim es +wat u +v ski +tre ve +tr x +timp son +th wa +tcs nycmarathon +song joongki +sha po +sapi enza +ram pride +pdx tst +patcho gue +p ki +ori be +on uk +movie challenge +maren morris +kore as +i en +hell omy +hallo we +em mi +dar ke +co stel +chat urvedi +cassa dee +c sun +broken hearted +bel ag +an dia +اÙĦ ب +vacation rental +ultra boost +trun cated +to pline +revolution ary +ram pling +proudof you +peace keeper +pe mra +pare to +mote gi +mass aging +je de +hell fest +f illi +en cores +ell amy +deb tors +dayo faction +cs ra +bro z +book mobile +bm l +bar field +apel doorn +air conditioning +ach ts +wash times +ut ela +u ia +through the +te il +sri dhar +sal af +ruido so +ro tat +ro ka +pom pei +mc d +mar nier +mar ga +lumin eers +leagueof performance +ken z +gl ings +gau tama +forthe throne +for ger +flam es +fin landia +faul con +fall fashion +ent revista +den ard +dc moments +cold stream +chak kar +cal exico +bun i +bi annual +bar kin +ar ola +aj stream +âľĪï¸ı âľĪï¸ıâľĪï¸ı +stu cky +sn er +shol ay +pre load +modern ise +kal ki +jo gi +j pt +in def +hen rie +havan ese +hau schka +ham an +g df +em acs +devi ations +dean er +chisle hurst +campbell sville +bu emi +atal ks +ðŁĴļðŁĴļ ðŁĴļðŁĴļ +ðŁij½ ðŁij½ +you uuuuu +womenin medicine +then e +tat v +sv u +sla c +single payer +sanjeev kapoor +pale stra +ork ney +or ono +nor well +mtv india +melo die +la gov +kri ek +kashi f +jon z +jeep ney +iu bloomington +hop kinson +high wire +go st +gho sting +fren zied +even ingen +eg ham +dhan teras +cla wson +camp bells +cal poly +ashi i +alek sander +[ .] +y ero +vu eling +union ism +ucl g +u vam +u pei +tuolum ne +them is +summer field +su ss +ssi d +sni per +se jeong +sand rine +reser vists +pe ville +palme iras +of d +ni ans +ne uk +mo el +m hi +kim so +har borne +dim mable +dali o +clever ley +catchand release +brac ey +be leagu +bar bra +art prize +ar ana +weill cornell +vi dence +tax onom +swash buckling +sre mmurd +soe karno +siddi que +sergio kun +scu ff +screw drivers +sa avedra +sa ag +pen ns +overe ating +noo bs +mar vell +lingh urst +lau rier +kel ton +jo ists +horowitz la +hann aford +fro mp +flu tie +fe tty +eric sson +duc kett +ash ree +aguero sergiokun +adam horowitzla +ðŁij¨âĢį ðŁį³ +à¹Ģภŀ +x ell +vic t +tvac tress +sub vert +spam mer +sch ade +oore doo +nicol son +mu tag +man iz +mal avi +lilyach ty +ky n +kra borty +k ady +j ons +hy u +ho he +her name +h su +du ally +dru k +dai kin +car digans +bread winner +bick i +ab lack +!!! "@ +æĸ ĩ +âĨ ªï¸ı +zee music +yiel d +x ilin +to pol +t fd +stu mping +sk f +sd hc +sau té +sab le +po so +pink man +ph ere +north olt +monday night +malign aggi +lo to +le si +jon sson +indigenous x +hin n +gour ds +glo vers +gir ard +essaou ira +ci ales +broadway world +be lem +ar shad +ðŁijıðŁı½ ðŁijıðŁı½ +yugo slav +us fw +ug adi +testimoni al +star mer +sleu ths +sla ving +shrey as +she in +re drew +pan ag +pah lavi +pa kenham +neu tering +nar al +jor i +jan h +ig we +i sherwood +harge isa +gu al +gandhi jayanti +copic marker +che atham +candel aria +ba red +apat ite +ani maniacs +am phora +adel anto +abr live +yh wh +whoo p +vernis sage +trini da +sugar man +sil ber +shir ting +sande ul +ques ada +problem solving +pear le +nu disco +ni zar +nando suk +mit ri +life e +l br +ie ga +ide alistic +home away +ho ley +free mium +eli pe +ed ler +d ld +cinque terre +centre ville +cat rina +c gy +c bus +beau tician +atx traffic +ani o +american history +ag v +. ðŁĺŃ +é ħ +âľĮ âľĮâľĮ +âĸ½ ` +y arm +wkc dogshow +velve teen +travel leisure +the ists +robert carlyle +rir insider +rachel notley +ra ashi +qual it +pre conceived +par lo +out live +national hot +magall anes +ma io +lend l +le witt +kra zy +kir tland +ir am +im r +ho bos +hess le +friend shipin +far c +ev is +confe ctions +com atose +ch aga +bi hu +bas si +back strom +alexand ani +acti o +yu pp +yak ult +we work +w cr +vi enti +uk m +ti ge +the hunt +the edge +sop hy +si sig +sher bert +pur ged +pu sat +pon ca +po blac +ob ed +n any +missing persons +ling aa +lat us +kay tranada +ir m +hunter sville +gl itchy +fa ia +eve do +ea w +doc ile +detroit gp +de cryp +carrick fergus +c vo +ble h +bd ch +back links +ade st +wood chuck +w ch +u maga +tol kien +theologi ans +the bachelor +stro mbo +sk ara +si z +say re +ra ppin +past es +of it +nutri bullet +nhs ft +napo les +mu gan +mc men +mari bel +mar ker +m haw +le elan +lauren cohan +kidney disease +kell an +ke efer +kach in +j ts +j ins +ili ke +go bain +ex tram +dru pa +diver ging +cove ting +clark ston +clann ad +cel yn +carolin ian +canad are +calibr ating +bis ley +b pg +b marketing +un cooked +tumb lr +st ong +spi zz +slow food +sapp ho +rich mix +ran aayyub +por tic +pic ante +pand i +nat ely +li sette +lais sez +kingsc ross +joon dal +j tf +it weet +incub ate +ill u +hor vat +heen im +giant spride +fr action +del in +clay pool +bul ous +bo ka +bir acial +benedic tion +ame et +ë ĦĪ +yo del +wangar atta +stefan ovic +sp gh +sno p +sm oul +sk ream +re iser +re considering +pr f +out looks +msle asalonga +mi j +materi alise +lib ation +k ise +jar din +ho yas +for b +flu shes +decentral ised +car rer +barre tte +bail ona +ard ley +apol lon +antic i +ðŁ¤ĺ # +æĹ¥ æľ¬ +ÙĬ ا +uni studios +unfur led +tat to +roller blading +recu se +ponty pool +omi dy +oc cas +nav es +music ph +move over +mas roor +ick off +hunter ian +dum fries +dermal og +cy tic +boy er +ask me +an ina +aggreg ated +> . +ðŁĮ¼ ðŁĮ¼ +z omg +yorkshirec cc +uri sm +together for +th av +sub unit +su ture +schn apps +rosal ina +re investment +planet shakers +ok t +npr music +moor lands +le ff +jr nl +jaf fer +hel looo +he ere +gre endale +gho sted +fo zzy +ex moor +esthe tician +ed om +dc w +crazy richa +cham blee +catholic church +by laws +as ou +arm rest +am ada +alessand rac +ðŁķ ¹ +âģ ¿ +zer ian +une ce +u bin +stop ing +stad ler +smi k +re balance +raw story +prabha kar +path finder +pas h +mimic ry +marsh acollier +low life +ini sh +ha pa +gal legos +elimin ations +coqu elin +cli max +chi aki +boot co +ath iy +alle tti +allabou tit +active snp +,, ,,,, +ÙĨ د +zo ids +xx y +war angal +roo h +qu b +pc po +par x +nv q +mt x +mc sally +mahar ishi +ke on +islam ia +i fic +g attis +fabric ator +f pr +es am +ear tha +draw bridge +don ie +dean sgate +dak h +cul pa +cran leigh +cor fe +clon eclub +c panel +bo p +belfast cc +barre led +ìĬ ¨ +ê ¶ +âĨ ª +viole ta +toic itiesnews +stat uses +sol dered +red bird +r mf +ov ulation +no ordinary +niz ami +mi gas +lucas oil +ley enda +lear ned +laser disc +jose on +j lr +j har +id fc +harro ld +gestal t +ger m +d bm +cou g +cooking with +commun it +cardin ale +bu gg +book cases +bol stered +blended learning +bir do +bha dra +atra k +andre ea +anast acia +ÑĦоÑĤ ог +wu b +wrist watches +stake over +spir a +sexy list +sa pere +rhi wbina +rady o +quantum computing +pin son +person alizing +path finders +ny y +nag be +mat zah +margar ine +knock hill +infin ito +ic lei +ic ate +en slave +dream coat +death note +ct b +cre ar +city winery +cic illine +christi e +cat man +cas kett +bre guet +blue hens +apa thetic +ani as +ald ine +ðŁĴĭ @ +æĿ±äº ¬ +wen o +wel ter +van ek +u hhhhh +to cando +swi fty +suriyaf an +stu tz +sch eveningen +per lis +paulo coelho +over hang +lin ley +hul kenberg +hot z +he man +google foredu +funinthe sun +fl travelchat +dynam ical +dutch men +die ter +deton ate +co gan +boule h +benavide z +an ek +x js +wor sted +win mau +win ded +we wan +ta iz +step daughter +sten cia +so wed +si si +salut atorian +ryo bi +philat elic +oned ream +nx tuk +namo in +mar can +mak kal +lille hammer +ii da +guil dhall +g illy +euro group +ere bus +dy isi +disturb ingly +could be +com ex +cl and +chat field +caf cc +biancon eri +bet z +bar da +az os +aeoli an +ac tof +<< <<< +zhiv ago +virgin ians +vi vam +uni brow +ti ques +sten holme +stel e +soundof music +revi v +resc ind +poblac ion +oscill ating +oo ol +om as +nex on +new statesman +ne sham +mu ffin +mi ere +mar rero +ma este +li ans +leopard stown +lakme fashionweek +kin o +kav u +history vikings +hair salon +h cb +gaz er +f out +ex ander +esp ress +end times +dra wer +docu mental +con geniality +chi ddy +charlie hunnam +carlton fc +butter finger +beach soccer +atp finals +>> << +å¥ Ī +zam an +the hockey +sad da +roy alist +rough ed +ross man +ram parts +punc tures +por sha +peripher als +outdoor learning +night watch +nct c +mac lin +ma gher +loud oun +littlebig town +ligh tens +le quipe +lake superior +kawar thal +if sc +hit ya +her iot +gold key +gaw ain +eh fcl +ee oc +dig weed +de tractors +dat um +dami ano +cho be +auto complete +app lenews +air mail +ac char +wow zers +whit ton +to rey +stan ly +sju bb +sir leaf +rusten burg +ron en +richmix london +pun it +people first +pauly d +pan chami +no emi +ni pping +mc devitt +mayor bowser +madison ville +mac dill +levis stadium +je sper +hyun da +hydro thermal +hed lund +hadas sah +goo dread +gla res +ge ysers +frene tic +firm ness +filip inof +feder ations +exer tion +e glin +d pf +creative cloud +cre me +con an +bro thas +bill cosby +b ings +ar mer +ap on +aer os +ðŁı¼ âĢįâĻĤï¸ı +íģ ¬ +âĻ¡âĻ¡ âĻ¡âĻ¡ +war plane +te ton +te star +starwarsthe forceawakens +signat ory +sho bha +shi rer +rh ône +repre hensible +ra es +per ma +ob stin +nap ster +mo ses +marime kko +iq ra +i thome +hun tin +hun stanton +haleso wen +gal as +g ica +dis repair +bra vos +awas see +apol o +alb ace +ac ls +ðŁļ¨ : +ya q +whit ef +w bf +stewar tha +soap stone +slo th +ru el +re mender +pe che +ng v +mu ggin +me es +maken na +khoo b +it jobs +iphone photography +hu mus +honeymoon ers +go st +ge stu +fran conia +fill the +en cen +eli verpool +disp rove +din ck +cy ru +chef jose +canad ago +bom be +aloha friday +ê´ Ģ +wa chu +up swing +un occupied +tuc kered +topo f +to go +sh ills +sch itt +sch eck +royce da +ros lin +pu bl +post office +or ga +openg l +om adrid +nal u +mini mizes +meteor ic +maryj blige +mam y +jump suits +he ft +hahahaha hahahahahaha +great devondays +gen ki +fla il +epicure an +dan sby +coffee break +char tist +bun des +ape hu +ap tn +ap aches +ag ios +a aye +ðŁį ¯ +åĺ ī +trent bridge +tor rie +thi stime +ric ker +ri bena +po sses +ple be +ph iri +ni vin +mike bloomberg +meh reen +martin sburg +lu cho +kapil sibal +kant or +joey graceffa +isol de +is ks +im vu +ho be +gis ela +gene therapy +f sx +earnest ly +do by +display port +depos iting +de mba +bart lesville +bald acci +ay az +at mel +ar ang +al shon +al fc +aau w +:" "" +Ľ ï¸ı +ðŁĩ®ðŁĩ³ ðŁĩ®ðŁĩ³ +É Ļ +zo g +where fore +web isode +travis barker +te mi +synthe tic +stinger sup +spra ins +specul ated +sob scura +sleep walking +sd u +program m +ne igh +mur ata +mi is +merri weather +mar mel +lul worth +jack septiceye +i fo +he mo +guest book +fú tbol +dais o +co sponsor +charity miles +cat son +bou ton +belgi ans +avail ing +at ou +at ennis +ਠ¹ +tip toe +the biancadelrio +st d +s wr +ram pal +priyan kag +prett iness +pom mes +out grow +ny fa +nov ak +nam ik +man is +lo fi +livepd fans +liveon fox +le ol +ji ao +is les +ida hot +haverford west +esk o +elton john +eamonn holmes +dau k +constric tor +choose chicago +bu mbling +bau me +band aged +aw amba +ar it +al ongs +af finity +us ns +tor rence +the kiranbedi +teessi de +sh antan +scra pyard +rade be +r hc +outer space +nf ca +nbc chicagofire +mon zo +me da +mary poppins +k db +jug ando +indent ured +hoo ting +hard shell +ghaz ali +gal it +foo dies +em mie +ee et +ech of +dru mpf +dontdrink anddrive +dol drums +d ury +calli ope +caff è +br illo +arte mis +ao sta +and rus +alessandrac icc +! ðŁİ¶ +ðĿ Ķ +z ile +yu sef +vivi ane +vi os +v lt +v angeli +un scrupulous +trom pe +to ph +thorn bridge +the gro +stra ding +soul child +sav el +richeli eu +red ruth +pr illy +por ing +our world +on ca +nerv ousness +nap h +mc bryde +lam e +juicy j +j fc +ine fficiency +igh i +femin is +farring don +dublin ers +dj e +cli psal +cassadee pope +bodhis attva +bar bies +back page +as ab +anci o +ance stry +all rounder +afro futurism +win et +wa ah +tor um +ta vr +superlig aph +nau man +mu stered +ly sis +kra i +k las +jac kier +j hon +ima go +horn sea +hed da +ger bil +dontmiss out +conserv ators +conden sing +cad well +bru der +bra he +af in +? " +âı ª +ti fo +th ara +steam roller +shane west +sa a +rye dale +rou ts +recover able +punche stown +p bn +our perfectwedding +opio ide +on on +obl iterate +no kom +nc n +nam ara +na seer +mart ingu +mar xists +lasal lian +kar ky +int aglio +hi u +gou let +gabbar singh +fur fest +florida state +editori al +cnn news +cal tex +bush mills +blan chard +bel it +bab as +ðŁĺĪ ðŁĶ¥ +̶ Ì²Ì +vaul ter +tokus atsu +ti ddy +stanis lav +sports woman +spor tiva +sor t +so cha +q pac +prime ira +overwhel ms +out lying +ott omans +nm leg +nk f +nelson chamisa +ne gri +mother jones +mir na +love u +li gier +ku yt +in ou +groo t +great again +ghaz ni +gh unt +fal ken +er om +colon isation +cb h +c md +bra vas +bougain ville +beach day +av chenko +ar ashi +ap ac +anton elli +z its +tre lle +t sing +stom ped +sky racing +should be +shan ice +san ur +rain nwilson +outsider art +ore ver +mer kle +lon d +la ith +kiwi fruit +killinge ve +ir shad +inthe morning +international artist +goul art +gla u +fin efood +e ki +dejec ted +dead lifts +coer cive +coder re +coal itions +cli ss +class ica +cab aye +c dd +bu hler +bin dra +basto gne +as sey +white wine +water quality +the dj +solar city +sir tis +sin ning +scar ia +q rs +py thon +portugue se +pick guard +pi pi +path to +noti fs +nl m +mo sca +min sky +mat ers +hot tub +hoo f +ha ws +g agan +fo amy +fan expo +e wan +deci sively +colouri sed +cash in +care r +callof duty +blue mix +bino che +bel tane +bel ding +be are +anim ated +ðŁ¤ ® +âķ ij +Å Ĥ +xylo phone +we tin +w tw +time stamp +sunny brook +su bre +stal ker +she el +season s +rootedin oakland +privati ze +o hhhhhh +marin as +la zen +insp ite +good year +god z +family vacation +diagon ally +del hs +cru ick +becken bauer +at so +ðŁĺ· ðŁĺ· +welo vel +usarmy reserve +un surpassed +u ty +tor tie +su mi +springe quinox +sp roles +riv onia +one ill +oly nyk +not be +mar g +kurz weil +itu al +hand sworth +ham mond +haemorrha ge +gan gre +for zan +fla ke +financi ers +fashion illustration +fal i +cli mes +cin q +champion s +cecil thelion +az ion +ash burton +ðŁĽ °ï¸ı +yo ffs +wi ston +velo ster +unite here +un surprising +u fos +ton ko +the punisher +sudhir chaudhary +sh mup +rou sh +pal lett +omak ase +nod ded +ne ste +milli e +loui stom +lam ine +i believein +dun kel +der r +cap taining +bowman ville +billi ee +afric as +adon na +ãħİ ãħİ +¡ ¡ +zar alarsson +uk manufacturing +the zone +sun ity +suicidepre ven +stan n +st johnam +slo cum +re caro +pil ger +par fitt +maur itian +marac as +leon or +ki drobot +juer gen +job centre +inter tidal +hen shall +gom usic +fire blade +ers music +duck duck +di zzle +dain tree +cour teney +conden sate +com poses +cc j +cbc sports +aki ko +absen teeism +zo on +win nick +the division +talen ted +song birds +sam bit +sam adhi +rs x +rob ic +pu ka +pro tons +patron ising +omele ttes +ne hra +multil ingu +lovel an +ko le +kn au +kam at +j ica +induc tions +hiphop music +heide cker +equal ities +coat bridge +bre nebrown +bi gro +apolog ising +âĺ » +о ÑĢ +zat anna +your voice +w co +ub ens +suzuk icup +shif frin +roch ford +rob gronkowski +queen sugar +q aida +pre scot +po plin +ph ool +penetr ated +olemiss fb +ny ff +mu ggs +monro eville +min oan +magical realism +lovin dublin +lo ths +len exa +ky loren +kof app +iam amyjackson +hy ou +hednes ford +green castle +gi rish +gi gaf +fa as +du miny +dev astate +dee ley +cav allo +casey neistat +bey hive +bas sman +babat unde +bab er +ann ina +am oo +zu lia +un warranted +spen ny +re homing +ny university +neon icotin +mini mized +ly ing +lit toral +lear nin +ke van +i sto +hyper trophy +honey cutt +gre ve +gal van +ecra ft +dol phy +dar ron +dal last +calcul ates +by rd +ar jo +alu shta +abi y +ठ§ +yand ere +woo hooo +win oo +uu ut +tri plex +toad stool +the struggleisreal +sou le +se ger +sam buca +re aver +ra gi +pag ar +ozar k +orchestr ating +o pere +new forest +mo have +ma dan +lu bin +lo ha +lac ie +kr at +ka elin +isth mus +house guests +go derich +fu shi +ema w +defec tors +d ö +colour way +blues man +bac i +amers foort +aly st +ach tung +ðŁIJ « +ãĤ¯ ãĥŃ +ty ger +town post +sunday vibes +sunday business +su goi +quick enden +poinci ana +play fulness +pin ar +par p +nom o +neuro biology +mul t +mu re +metro trains +maug ham +marque ss +k maq +jin xed +james martin +il ink +edge worth +delicious food +de eps +bal lets +bail ar +tall ying +suvarnab humi +star sportsindia +shan klin +se caucus +sc alab +san che +robo calls +re organizing +pwe de +pim s +ol ate +nas pa +nam aste +n te +log is +kr antz +heck ling +hate breed +haj duk +fcv afc +em iller +ear nyour +e hc +diamondre sorts +cri mp +ci ac +car no +brun swick +bir ches +aman ecer +ad s +âĻ¥ "@ +à´ Ł +vi on +these us +the hashtag +slo van +sk d +sab yasachi +real m +rai ola +pam yu +p chs +out boards +nieu w +moor house +mid stream +ly onne +leopardstown rc +leather jacket +kha yel +j st +im pres +illini football +hyper ledger +hair streak +f agi +es x +dor je +do bro +copic markers +che son +blanc s +bit trex +ben oit +barran quilla +b dubs +av ilion +é ļ +Äģ h +wido wer +un called +tab bouleh +t tered +ste ps +sk inning +se bo +sar um +ru ka +ross er +ri ves +real joey +po pov +ped alling +mc call +man ni +ma ile +inge sting +heather ton +han ami +ger mania +fla bber +este pona +der ren +de construct +buy backs +book end +book aday +black y +bengal ur +bar bz +ay anna +an tra +ak hen +ah ra +ad disab +academic twitter +... ðŁĺ³ +å Ħ +ÙĬÙĪ Ùħ +zeal anders +wv prep +w bb +se jal +rossi gnol +pvt ltd +print makers +pict spam +peter loo +pc v +park zoo +o go +mitro vic +mis i +love d +leaders debate +ki f +ker ato +ju e +hawk pride +du it +con currently +chocol atec +calmac ferries +bu escher +bon spiel +biggle swade +belo ve +al ama +! ðŁĺĭ +yo gali +viol ator +valpolic ella +th ave +tet bury +t fo +sway am +sati rist +richar dg +raj yas +quadro phenia +pho resis +pert ama +mon roe +macro economics +lymp he +le der +jam bi +healthe quity +hatec rime +gre as +gil do +fre m +france sa +far kas +drug discovery +deepp urple +deco rex +bride groom +bodleian libs +ben es +bapti ze +anomal ous +alle mand +a design +ä¸ Ĭ +vm fa +tre von +topp ling +tay ga +steff an +ssk roughriders +sal maan +rc w +rc m +police brutality +picker ington +or ad +maxim mag +m go +lat os +lam pe +khalee j +ka an +in safi +ick le +ge in +fian akis +ff ootball +exhu med +emily deschanel +emb ellish +e br +cro fts +bis sau +beaver townbeer +be ggs +altar piece +al sop +ak kad +ab be +aak ash +@ : +ðŁĮ Ń +ðŁĩºðŁĩ¸ . +zer i +yeh rish +uefa euro +star nes +softhe week +sig i +siem reap +rou ille +rocke teer +ric ko +perse id +pac io +ol tl +monopo lies +mo ak +mill on +micro controller +lu anda +look oftheday +l nb +k adam +jan ko +idol m +ich or +hul ton +hon eye +flori dag +flor issant +ex terminator +du puis +din fo +de sco +cran bourne +con cho +ch m +cal kins +ber tel +aw u +al una +aim er +ðŁĺĬ ðŁĴĸ +ðŁĮ¹ðŁĮ¹ ðŁĮ¹ðŁĮ¹ +ðŁ¦ Ĩ +ãģ¦ãĤĭãĤ ĵãģ +vi ken +twitter carclub +twit pic +trainee ship +tis one +tal en +sh oma +sar s +remo percussion +one ok +on ville +ne whi +muntin lupa +khe de +jack a +ja se +is nt +igu al +hrvat ska +gut tering +fre et +foun dry +fied ler +fang irl +du pdates +dish patani +co za +chu seok +braunschwe ig +bo ole +ban os +are zzo +ap so +ali p +æĿ±äº ¬ +un satisfied +tra wl +tom oko +tam blyn +sto we +puff ball +n ays +marsh alling +marqu ardt +leti zia +la chie +l vt +kid naps +ke em +fur la +f uring +eli ghts +dan aper +bear s +bay ani +ball state +azadi march +aldubeb forlove +ðŁ¤ĺ ðŁ¤ĺðŁ¤ĺ +wau wat +ulaan baatar +to eing +thirl wall +then ick +the week +the queen +spe k +sham anic +res life +nuf field +mag lia +ku jo +kof fie +kat amari +jan o +ja j +is ches +hu fc +hai b +gu ice +ge man +fe tte +edch atie +dulci mer +condi viso +con dor +buck fast +blo o +bi sexuality +alar con +ðŁĺį ðŁĻı +ðŁįĵ ðŁįĵ +ìĺ¹ ìĦ±ìļ° +ë¹Ħ íĪ¬ +ver mont +un an +to kai +te uk +sports medicine +schul ze +sa hir +roy alo +que ta +pit er +pir a +pd g +ound table +nor rie +mal oof +m tw +li zer +ki yom +ji p +its just +has well +gy le +gu ar +ent rees +dd f +carto grapher +bor ger +bin ns +apple baum +ali ste +aer in +ab ile +ðŁĺ±ðŁĺ± ðŁĺ±ðŁĺ± +ym all +wol k +von ne +vivi ana +thero se +team spirit +sto at +skeleton clique +pn co +pi gging +on trent +o za +mar chi +manc unian +jum mah +i gel +hier onymus +fer rie +el ston +e per +do ig +day dreams +comi endo +allu sirish +ade p +¨ ¨ +yan del +vi shak +togetherwe can +tel ing +tann ery +seaf air +scho enberg +re appear +r hen +out bursts +or han +motor city +mans field +lilly pulitzer +lg u +le derer +fun es +fle mming +dis assembled +da stur +car ranza +cam isa +bush land +bolly woo +bick er +anae mia +an jum +al war +ðŁĺĢ @ +ìĬĪíį¼ 주ëĭĪ +ãĥ¼ãĤ º +à ¹ +wi ven +wfa achannel +w pr +vol ition +vi en +tw ani +tf n +supran ational +stre ak +star oftheday +sr c +sheikh hasina +roger s +red angel +queanbey an +qu as +penn ants +peace fulness +over passes +mg d +ku wait +ko hin +hu ber +head strong +gr b +gift shop +floo dgates +dai hatsu +cryo gen +compli ed +ame h +ðŁĴļ ðŁĴľ +welsh pool +vegas baby +v tr +tri sk +tall grass +sl soccer +sho veled +se date +school yard +sac p +sa chem +re ville +rath lin +public is +p sk +mis understand +mechan ized +later ra +khatta b +kemp ton +ke rem +karan ka +jur is +jk live +hin cap +ha ze +guitar player +gran ita +gab bert +g sx +esper ando +ero b +dom ina +di q +danai gurira +capital ise +book plate +bi ka +aus veng +arun vijay +anec do +*__ _* +ðŁĺľ ) +ðŁĴį ðŁĴį +ðŁĮŁ @ +ï£ ¿ +å® ® +âĿ¤ï¸ı ðŁĴĽðŁĴļ +âĿ¤ ðŁĴķ +wren ches +w out +ulla pool +tiger day +stan more +shop keepers +sacramento proud +nam or +maras chino +mammal watching +ma wa +ma af +lar isa +kab i +jennifer beals +irish research +idoli ze +htt yd +high mark +ga ve +frequ ented +ec j +dogg one +dic ke +de compress +dab ba +dab a +comedy fest +co production +ch igi +cent relink +br û +artsand crafts +Î ½ +valedic tory +ten do +severy one +prodi gious +pri stina +pathophy siology +pa ho +neh wal +ma estro +london life +lad die +l mr +ky aw +inter milan +hel ier +good job +fu mo +fenty beauty +ed s +don ghyun +ce f +bli ghted +as sal +wc g +waron women +von d +twee ks +toyn bee +thi ep +steacher sa +saw yers +sau dio +roche fort +quand ary +pu be +penetr ates +ouro boros +o stia +ma hersh +jim iny +in star +head lock +he v +goal posts +geor gie +fle d +faun tleroy +et oo +danaper ino +castle field +buil dyour +bar gained +ay eeee +asiap rince +arunvijay no +api ary +am peg +ðŁļ ® +x un +wh ig +ve vo +square ly +squ et +spar ingly +pas cag +olympi akos +ob and +middlew ich +kottay am +inar ow +illu st +hi rise +head hunter +hall marks +gla x +girlswho lift +fla ppers +el vish +cryptocurrency news +ci bul +car wyn +autom ation +ato dd +and still +aga sta +à¤Ĥ _ +what areyou +voyag eur +v vd +v awa +ufos facts +stra ddling +snap chatting +si pper +sch ut +ra wa +power lines +plit vice +phar r +one family +michelle malkin +me ze +me dec +mc daid +man tova +m ws +les miserables +lav anya +k assie +inter lock +ic ef +hi fi +fuj ita +fo on +e gen +dram atics +craf ton +black pink +berg dorf +beer men +audi ence +app ic +ant al +ãģĵãĤĮèģ´ãģĦ ãģ¦ãĤĭãĤĵãģ +ãģĵãĤĮèģ´ãģĦãģ¦ãĤĭãĤĵãģ łãģĭãĤī +women swrestling +wag g +wa ite +vienti ane +tu bri +tru ong +throw away +swift lang +sab ino +re acquainted +or ff +op us +ok azaki +niam h +mu ga +mo yet +mlb theshow +mag no +mad max +hay fever +gali fianakis +for president +foot ings +faiz al +esp ada +er ase +encro aching +eleon ora +dun o +dre ws +date just +com rie +cause way +caf tan +bbc wales +band t +ba day +ant el +ani on +am co +... ðŁĺĤðŁĺĤðŁĺĤ +ðŁĺĤ ðŁĺī +ðŁijĮ âĿ¤ +âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ +ye t +well spring +wander ing +visit nepal +under line +spe terson +so cean +service man +ro es +ra yo +prud hoe +pi aggio +p wp +out played +ou h +ny dd +nhl pa +n zi +more ls +mol le +masa ki +maj id +librar yof +legui zamo +larry hogan +ka hi +indu bai +hun tly +ho ki +harbour side +ha shing +grain ne +for ties +ell ar +doctors strike +cle o +cap ito +asser ted +ðŁĴĭ ðŁĴķ +ãĤ·ãĥ £ +yo suke +u houston +sue ños +spo kan +si moes +ren contre +re installed +re authorization +press conference +po logy +pl zzz +pitts ford +phono graph +over reach +nun head +nil joshi +moon beam +may ur +lugan sk +le hi +jag dish +ilo ves +h mn +griff o +gold link +fresh water +e woks +chand ran +casi mir +cannes filmfestival +bush man +bir ther +ball inger +b df +ac ke +ðŁıĬ âĢįâĻĢï¸ı +yan is +who are +ver dugo +thatawkward moment +solic itation +shine on +sg n +segu ir +science daily +sau dade +ry ley +re traction +r vr +om u +o sam +neo pets +marlies live +malt by +ly si +lisam ur +le ats +hay nes +hat chie +ham ming +gold star +flabber ga +finkel stein +e mobility +devil may +dag ens +cen te +ðŁĺİ ðŁĺİðŁĺİðŁĺİ +yyc food +ven use +tv nz +strati fication +ste pan +samar as +rec t +re te +re conditioned +pres sclub +plant based +p has +on ata +oko cha +nfu tweets +natural stone +mil ka +metabol ites +mega star +mahi ma +lums den +len sing +kool aid +insp irit +im morality +hor nady +heck led +frees avchenko +fle uri +fi annaf +fab re +dre her +de acon +dam ar +crewe alex +cosmetic surgery +com en +cham bre +card board +b sh +ap layer +ak awa +ðŁĴĽ ðŁĴĻ +âŀ Ļ +à¸Ń à¸Ń +à¸ķ à¹Ĥ +zac atec +water jet +un itary +tri athlon +ther oom +ta ze +su dah +ston ington +sta sh +sp ano +shi atsu +shak u +sen den +pa izo +neti zen +nar ok +mull en +le manoir +lali que +l rp +in fest +h day +gaiag auden +evapor ative +en ver +du tty +domest ics +c mpd +book trust +banne ker +bab ak +ar j +al anya +.. * +wig gum +warran ties +wan o +tv one +tin type +sy ah +stjohnam bulance +shi ko +on live +moon landing +legg i +latenight seth +la vas +juvent ud +hart ley +gra sped +ge ren +gales burg +fa sts +ero ica +davincis demons +d fg +coffe eco +can do +buy ing +bar ang +ac app +[ : +ðŁij º +vale ant +upri ght +thel p +subi aco +som ni +shir k +sdc india +sal vos +sag u +sad dlers +recur sive +pu w +petro s +per shore +ou dna +oro chi +opioide pidemic +nbat witter +nathan iel +mun chin +mounta inde +miscre ants +mis communication +mbar ara +lo rela +liberalismisamental disorder +le andra +kentucky mbb +je tt +jav el +iri descence +indi av +im potent +hend ri +geth se +desper ado +cra dio +cr x +col nago +cla vier +cadbury uk +c sb +ame in +âĿ¤ï¸ı , +yoga everydamnday +win with +west wood +w eri +w ce +un sound +u wo +timor leste +tat ar +street sof +strato spheric +ss man +spre sent +spit ze +soli man +si pped +sau li +pterodac tyl +nc u +nash villec +mariecuri euk +mani acal +m ni +love bug +length ening +kw aku +kud low +kpop starz +ken yon +ji ve +he gan +greyhoun dracing +go broncos +formul ating +foot balla +foodand drink +flip the +e stevez +di she +de meaning +capy bara +blu emountains +ble ek +billiee ilish +bi partisanship +az ania +army bowl +a hoe +âı ± +wild west +whit eface +ver ve +sung jong +sunday roast +sixx am +shar lene +shame fully +se dated +ra pinoe +r dra +quo ted +post paid +ous ins +obscen ity +moust aches +midat lantic +mal achy +lee man +la gan +kot ler +jalap enos +hyun bin +hing is +gi gg +gam s +gaiagauden zi +fair weather +excep ted +duck hunting +do vi +den e +de my +cork cityfc +chem bur +che ons +bokuno heroacademia +biom ime +back es +asu g +armou ries +ðŁİ¥ ðŁİ¬ +ðŁİ ŀï¸ı +ëıĻ ë°© +ãħĭãħĭãħĭãħĭ ãħĭãħĭãħĭãħĭ +व र +ym ed +wol laston +weare family +unsigne dartist +travel tip +task master +succu mbs +stimu lants +st luke +si vas +shu u +s ve +on sea +o orah +lo lli +lex mark +ke ham +kar man +jr f +jol ts +iu lia +hai le +gar land +flow y +fant agraphics +fan sof +exolu xion +ex el +espnc fb +dr iller +dogmeat trade +consen sual +codw wii +clam shell +bou illon +bosh off +be for +ar jona +ampli fies +agric ola +ab ora +ðŁIJ ¿ +with purpose +tom oe +to bar +tend rils +slam mer +richmond hill +pur ser +po el +nuer burgring +messer schmitt +mant le +m vd +kirk stall +key shawn +ke tel +inthe uk +ho ppen +god parents +gn ano +g itta +g dg +fle amarket +fi field +down state +down sides +de contamination +dae woo +ch romeo +busine sse +british tennis +bat anes +avoce t +alarm ingly +al ann +ðŁĴĭ âĿ¤ +wight man +who ami +un appreciated +tou bia +tf si +terr an +ta ven +stol tz +shemar moore +sharing iscaring +ring ling +re gi +pune et +phi fe +par ables +pandi raj +mun de +mo ke +metat ron +inver sions +ic ap +ha plo +fu uu +f ounds +el gato +desi re +d hal +coraz ones +col y +bush ra +bron ycon +black sheep +beam iller +badas steachersa +aw ang +arch digest +ad at +Ì¶Ì²Ì ¥ +whati m +was i +under performing +to tt +th ile +st anger +rod stewart +pulp fiction +polar bear +pear ld +pan ics +op s +nord see +noo bde +ni mes +ne sa +nas sau +min ette +mar maris +levit town +leng then +kaz iranga +k hid +juda ica +ic hat +go canadago +gen sler +funny bones +dyisi sit +dress making +dj ur +devi ate +cu id +crustace an +crank shaft +co bie +bar one +b hl +aven kat +ass ate +ac ab +à¹ģภļ +Ã¥ s +y ath +waynes ville +valley wx +val ens +touri sty +suk hum +splendid ly +si oning +shiv anna +ser ine +sar ahe +samar inda +sab ar +ry ano +ring master +ridg eland +rat cliff +po gues +oi shi +ne gga +nam en +mur ree +mo omin +mil ia +lin um +kit tie +ki x +i dent +g ä +ffun ded +est ars +elvi shistory +eco sport +dress maker +dc tv +costu med +con ing +chi vas +bin aries +baj payee +! ðŁĺľ +yel love +wil s +whitt ingham +sky rim +real kevin +read allaboutit +r cl +pun o +par li +na ghan +mun e +matchroom boxing +la sd +kal os +k ring +ind ye +hero esof +ham ber +gg t +fac ey +diab lo +ct vedmonton +bre de +bla vat +be ssy +attenti vely +as ot +aristo crats +ane ws +ðŁĺįðŁĺĺ âĿ¤ +á´ Ľ +yo ps +wom ad +virgil abloh +vi vel +vat raffic +va art +toly mpus +the ip +tas so +sn ak +skill india +sie ben +rod ham +pr ata +po ors +pic kn +need s +mx px +ll er +le ers +latic sofficial +la pointe +kago shima +k mb +ju anes +it our +he da +ha kka +gu gu +growing up +gold standard +fen ce +den r +cur tesy +cor ban +beh rens +am isom +air drops +- . +z ville +youknow youre +vi dor +tsaww assen +thiswas cle +suni elv +sunielv shetty +summer stage +spark ler +sise puede +sag esse +p mk +nj c +mephi sto +lam our +kg bt +kaw as +jets ons +is berg +hor muz +gif ted +fit spiration +evapor ate +el ain +dou se +chic hen +captiv ates +beleagu ered +as ms +acup unc +a ec +@ ____ +ut tam +un wise +tri xie +tag g +style awards +sati e +sap na +san gram +sam pras +ray donovan +ra zi +pt fe +pir ata +new day +n rd +mu schamp +ma user +lamb skin +ker by +iphone only +ilustr acion +ham o +glo ster +gi ddings +ful ford +films video +fe en +dri p +cred iting +clipstudiop aint +charles worth +block b +bir ge +bac o +az ia +ay ian +arrow filmsvideo +am w +ac adie +with my +us bc +to z +thre l +rock hold +rob ing +reha b +pu yat +pillow case +perry sburg +nur tures +normali zing +nee ley +ne eta +mon ona +mal m +m by +llan de +li pinski +lai ki +kris jenner +kon ga +kar ren +italian wine +gu us +ex ci +ema zing +d ya +buo yed +bick ering +bette midler +bed lington +ban ister +ban a +ation tv +agny aath +- & +ðŁĻĪ âĿ¤ï¸ı +ãĥĿ ãĥ¼ãĥĪ +ãĥ Ķ +tip ton +tal ert +switched atbirth +su tt +sp ren +ra sp +q an +pin eville +piccad illy +pf hof +over bought +nor ma +nic helle +navig ates +morbi dly +maha devan +ll r +hop wood +en field +em l +dr yeye +dee wana +car pio +biom ole +bhar ara +art sc +arre dondo +ac costed +@ ) +z ini +wee eee +union ville +ted die +st asis +spirit u +radiof ree +petro glyphs +oliver i +naj jar +mi jn +mammam ia +maid ana +jama icans +ig ang +ife anyi +ic hert +fore play +fergu sson +etsy aaa +de test +de ke +cor tisone +bon bons +bo tero +bajpayee manoj +b te +ade es +ðŁĴ¥ ðŁĶ¥ +ðŁıħ ðŁıħ +wilder ness +tabletop gaming +t bbt +submer ge +so dal +si ah +sexi est +sen n +rober tw +rit on +pro jo +pra ther +ovi ya +oj eda +oc ke +nar u +more no +mor nay +marshall town +kil meade +ja hr +independ encia +indef ati +in sel +imagin ary +halloween costume +georgi ou +edu ardo +east view +defen sor +de joria +clai rec +by erly +at c +anore xic +annex ed +ai aa +ï¸ıâĥ£ ! +ï s +ww at +wupper tal +wheel set +tru ecol +tor tas +tion news +thre sh +te vent +swamin arayan +rep elling +real ron +re prints +re directed +quare sma +pen ne +pat en +mur k +metho dman +malaysi a +lov sk +lo ir +kill ington +ke u +kat en +jo ven +janu ari +in law +hol ts +gun dam +gled hill +garof alo +free thinker +father land +fashion history +fall acies +ee sh +dom sherwood +desi ring +cm ha +brew fest +break ups +big game +batman day +an sah +alpha bet +, < +ðŁĺĤðŁĺĤðŁĺĤ @ +âľĮï¸ı @ +whit est +whang anui +us is +tsuki ji +thom ash +the toughest +summer in +sing tel +simon coveney +sidd hi +si yah +sen ile +que remos +presu mption +pair c +na at +mc cann +maro oned +mah alia +lon ga +ja sta +j ata +ill hu +hack aday +gi zz +ga em +fin ny +fault less +far rah +ali abbas +ðŁį ľ +ðŁĮ¹ âĿ¤ï¸ı +Ú© ÛĮ +س ÙĦ +zeph aniah +w dm +villa real +sydney roosters +pap illion +ne had +myster i +mult itudes +mil s +mate us +loughe ed +le var +kenny wood +house cat +ham mons +gw f +gr ic +gl ancing +frighten ingly +free bird +fetty wap +father less +fai ro +espn fantasy +dou gie +co sho +chan cel +cardo so +brooks brothers +anadar ko +âĦ ĥ +Ð ± +zu id +verand ah +upan ish +up north +tr ona +sm sports +skag way +sigh ed +shaf fir +sab ie +ry ang +re clusive +pyth agoras +pete gui +nonchal ant +mud slides +mor ant +mclu han +man school +ku gel +kirk by +ka ali +jaw ed +is f +helen sburgh +h series +fu dd +fan army +ex claimed +enter gy +dyisisit manila +di at +cru se +car m +break neck +bilingu alism +always be +aki ba +ad abra +ðŁĺ£ ðŁĺ£ +ðŁıĢ ðŁĶ¥ +ðĿĹ ¶ +âĸª ï¸İ +ye ducation +watche spn +trampol ining +tay side +so wa +sh allow +sema phore +q ew +proud teacher +perry man +onom ato +nether realm +mun y +metamor phic +man tua +legg era +le web +le ssing +le per +ke well +jw st +je el +go beach +fro gger +forever orange +edu topia +chippe was +c mh +brexit party +biz et +beat king +aw n +asap h +anal yser +ade sina +?! ?!?!? +ðŁ¦ Ģ +ëıĻë°© ìĭł +Ùĥ ÙĦ +york dale +wey burn +wex ford +ul loa +u sip +tre ed +sx onfox +sham it +sephi roth +sch in +proper t +mo ats +jain ism +illhu emin +hirsch feld +emaci ated +eddie izzard +demysti fy +deck ard +bush ey +buff ers +append ic +ant artica +adi k +yu kari +west shore +wat sky +w fs +vac ature +super moto +ste pp +roller ball +roc nation +ran elagh +r md +professor green +produc tively +person als +pa jar +nov onor +nn n +ni emi +new salert +mal ani +ma sco +lo petegui +jersey ci +inj awarrior +i aapa +gg p +dra c +comm is +coffee with +chi hiro +cc ss +bull finch +blay ney +aul kner +ar ber +ab dl +zip car +yu shin +win kel +vo wing +une n +the ory +t pc +t adi +sho witz +sherry rehman +ser ta +reci eving +r de +q rt +pollin ated +pear lv +pe th +off ood +north western +ni der +man dem +kennebunk port +keep britain +jal alabad +howi ed +folk tales +es ra +dil worth +chess ington +calab ash +br k +bal as +ato z +amaz one +adam saleh +' !!! +ðŁĩ¬ðŁĩ · +ëĵ ¤ +yl ing +wyn ter +will cox +vi vere +valentine day +transpon der +titch marsh +theavett bros +su ce +ski doo +sk oo +ros ine +rese da +perfu med +out lived +our ces +one world +nu dist +mcr museum +lake side +kaneo he +justice orelse +instant aneous +in co +hungar oring +gri moire +great reads +ghastly gastronomy +ger n +fun fun +fu c +foodblogger ai +dur ning +datav isu +cri mbo +clinical trial +cho tels +car berry +bou dre +bil ston +biblio the +bab ie +ay aka +as core +adele ke +ðŁĶĿ ðŁĶĿ +ðŁı¾ âĢįâĻĢï¸ı +we ge +washten aw +wal kon +w ads +vic enews +v hs +un shak +thisismy crew +ter adata +tan door +sw kly +stol len +sno ke +sni ffs +shali mar +seri us +sare back +sanc tified +res ch +pseudo science +philly now +matthe ws +manag ua +laun dered +hou lihan +hou l +hou ghton +hot eliers +hoo ley +go back +ero gers +elan eri +e chan +dur ance +dj sbu +dish washers +dial er +clever bot +ben ner +bas sen +ache be +? ðŁĺī +ãģŃ ãģ£ +ãģĵãĤĮèģ´ãģĦãģ¦ãĤĭãĤĵãģłãģĭãĤī ãģŃãģ£ +د ÙĬ +w bez +upperdeck sports +uk storm +trun king +three uk +tho ward +solo astarwarsstory +se gam +schul ler +sayye shaa +reposit ories +ram bling +r aro +prayfor us +poli sher +p mk +own the +mel drum +kimon os +intern als +ine pt +human ely +gar ters +g nev +fug ly +for mo +dispo sing +charlat an +cerebral palsy +bou ton +bhutan ese +assau lt +aran ch +am hara +ake a +ac cor +;- )) +ðŁĴİ ðŁĴİ +ze alot +yr insider +xrp thestandard +wend ell +tram ple +tour o +taver nier +ta fel +solic itations +sh yne +sf b +scrib blen +s brew +ren ch +ra dek +pla its +online learning +one less +one ers +ni xa +neo sho +mor d +mo bbin +md h +mb h +man tic +ma dara +kof ta +ig naz +hooten anny +gl eni +ge tin +fo ck +evin rude +en code +emanu elaneri +chon buri +blind side +bill yidol +ank ur +aaaaa and +ðŁĴIJðŁĴIJ ðŁĴIJ +ðŁİ ° +ม าภ+à° ¦ +x xiv +wri ggle +wp moy +vi ani +ve to +theme forest +super fluous +selfe steem +se lem +ru ffed +ra kul +ptole my +pro fil +old london +newh all +me isner +madam secretary +lu me +led ger +ke ir +histo gram +hear say +hay stacks +governor perry +gor ry +ghe tt +gate wood +fandom memories +easter bunny +double day +degre es +decemb erists +chal font +bus way +base ments +b ick +ah k +ìĦ¸ íĽĪ +åĩ º +zakhar ova +wh ist +wen urses +vicky kaushal +twor g +transi stors +thak kar +t vet +t no +south wick +rach id +r la +propag andi +poli sci +pl tworg +oh are +nov gorod +neuschwan stein +nano science +man nn +hyun seung +gandol fini +g adventures +descend ents +de ta +borde aux +bo se +beck on +ali dad +alar ic +å°ij 女 +ym atic +yav apai +whit bread +water keeper +tuuk ka +ti da +the greatest +tamar ack +sto well +sky sport +ring gold +ra bacher +r pe +ponti us +pc dd +oni st +mince meat +mattb ellamy +mar gera +mar cio +maite oficial +maach oops +lifestyle blogger +lake erie +kent ridge +homestead miami +gra phia +golf news +fur row +en tech +e stos +e gu +ch imp +cat rion +bl ings +big fish +b ava +armen ian +amazon ite +al tra +ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ðŁĶ¥ +à¹Ģภķ +à¸Ĥ à¸Ńà¸ĩ +z aka +warring ah +u mo +tough en +sta si +sque aking +south coast +sil at +sar dis +san te +richar diii +r mcf +punk rock +nas sr +n once +mould ings +megali thic +l ann +kamenri der +getin my +fl aco +fab e +ema e +dwi gh +deduc t +cor vo +ch ato +arche ologists +ar ys +appropri ateness +anu j +alo is +adrian peterson +ãĥĥãĥ Ī +vandy ke +toho shinki +tai res +sundar am +sal ton +ri hann +pro cor +norm ative +ly kke +lisamur kowski +ligu ori +leg oland +kwan kwas +kar lovic +kal man +insu lator +horse manship +harri ott +et d +ery thr +er ae +dg son +deple ting +den son +chen le +cand ic +ca thra +bal u +art institu +ame deo +' ?" +ðŁĺį ðŁĺĺðŁĺĺ +ðŁĶ Ľ +ðŁIJ Ħ +yog endra +ye ap +tu lia +trap music +thu ram +thom a +sling back +recuer dos +ram pur +punch bowl +prag matism +phalaen opsis +per pi +ok k +na iry +mah langu +m ht +k lip +humber college +hu gues +hal pin +hal fling +gri my +governor va +gal go +fol ger +fla sks +firstal ert +fab ius +ei sen +chi ppa +cam elli +bu sia +bag g +ba star +amb ard +aliabbas zafar +zim ba +you like +wych wood +u ac +the summit +sv d +shi vak +re publics +re assess +pro strate +pray forthe +pal it +oo td +onmyo ji +nam ma +mond ello +mo xy +m ld +lov ich +lom achenko +lia oning +le te +kas ingh +ju py +ingol stadt +hipp ie +grim lock +go wolves +gallau det +fernand ina +fac ile +ed as +cre sta +control lable +block head +bel size +bel ind +b ck +appendic itis +al locates +yu uki +vandy boys +vag anza +thom a +sy f +sy ahu +ste pup +sp ens +sch moo +scep ter +rho s +pyn chon +psycho therapist +philli e +o ii +nehad hu +n sdcindia +lu h +lo li +ke c +inve gas +hen ness +gun k +gin ous +fit fluential +en p +ek ko +dr john +dd p +cas ade +calder wood +bur kett +buck aroo +bri gan +bon ez +accade mia +ðŁĴģ ðŁı¼âĢįâĻĢï¸ı +wha aaaa +wh of +volup tuous +up ra +tu lo +trol lope +tri ano +temp el +syn tagma +sw ays +stone wall +star kid +serie atim +say an +s xt +ridge view +reflexi on +pul sing +poon ch +pearlv puri +on du +om m +official randl +nay arit +mur taza +min it +l ns +kn elt +home pod +foster thepeople +e bf +der bys +cyber war +chal king +bli gh +bambooz led +ayo ko +akzon obel +اÙĦس عÙĪد +wh ims +volk sk +titic aca +tim perley +spre ss +smu g +rhi annon +pizz azz +pile ated +percep tual +mu shi +mix uk +lori en +kin sley +indie pop +homer uns +hippo crates +gre b +gold coast +fresno state +deli ri +coffee maker +clover leaf +br ouk +bo ther +am ati +al annah +achieve men +accom plices +ãĥĿãĥ¼ãĥĪ ãĥ¬ +ze tec +zare k +xenoblade chronicles +wise guy +wi ddicombe +wan khede +vitri ol +vin daloo +ty tlive +te dd +share acoke +semb ly +sd h +say it +san kar +res ver +place sto +phan euf +pel ton +no thanks +nichol l +nathan ael +man gano +leu ko +infuri ates +hypnoti ze +frustr ates +extric ation +end points +e sau +du dek +deer foot +comprehen sively +candi de +camp den +bm h +bian che +bab ies +assembly member +any e +é ¹ +zel en +wednesday want +texom awx +sundeep kishan +suc c +six words +sergior amos +rugby canada +ruck ers +pol ys +plugge din +om ura +nic eville +min koff +meow th +kul lu +jet ski +im ca +her oku +hash emi +grammar ly +fier cen +fa him +epp group +dogsare family +d swt +cr ittenden +black son +autopha gy +aler tness +val voline +tom lin +the agu +silver smith +shou ston +shor ties +ro mina +qu ina +pub quiz +provoc ations +pad dys +ne scac +n fc +michael rapaport +mic alle +life changing +geor ger +eric ally +de wor +clari dge +chri ss +car yn +bo sman +ball ack +arbut us +altam ira +âľ Ĵï¸ı +way mo +vom ited +u ther +torna dic +storm tracker +sprote ction +rob thomas +poin tof +pl ana +pa ha +montes julia +mo sher +marke to +mark son +lar ts +ker ning +julie bishop +ig ad +he ver +fahad mustafa +em cc +do ber +d rich +cul pable +crac ow +cal ri +bur ros +blen cathra +bi on +bass inet +at vs +amand ab +ac ted +ðŁij¶ ðŁı½ +ðŁIJĿ ðŁIJĿ +ze peda +womani zer +wir tz +vaish ali +u wc +u ting +sop hos +sink able +silve ster +s mail +s lit +politici zed +po ca +pediatric ians +nick cave +nemato des +melani atrump +ly tham +lam ination +ko sher +j co +in vision +in box +heuss aff +han doff +gov tech +goldkey comics +garage band +g mi +false hoods +epic ure +en dy +dy ard +che alth +bright ling +bourn ville +blow ed +birken au +baster ds +aw bb +atla sobscura +amy freeze +ðŁĩµðŁĩ · +âĥ£ , +Ì ² +z om +tromb onist +stro me +shahe ed +secon dary +savechildren uk +ro bie +ric er +reu ter +pol perro +p ns +ny phospital +norther ners +mh eller +man up +lm k +kather yn +heart worm +gau ld +futu rology +fa wns +du du +dis aron +crash bandicoot +comb ats +co ti +christi aan +ce elo +carp inter +bm ws +blood lust +yak in +tu an +tri pling +thank smom +spot sylvania +sand vik +purch aser +no where +mohabb at +mc goldrick +kap taan +it sam +in built +illhuemin ati +i ita +hir an +haiti an +hail sham +ha young +fernand inho +feliz domingo +eli ot +drawthisinyour style +dorset mag +di wan +buy er +b wk +ang olan +ai hl +ag gy +ðŁĺĥ # +ë¹ĦíĪ¬ ë¹Ħ +wbko wx +under writer +twitter ing +sun shin +sub par +start with +ri dlr +recipro cate +ra van +paradig m +ou c +nd g +mohan ty +mo zz +mass ape +lumin aire +lgb tiq +lea therette +la th +inve sto +im in +ik awa +hard wicke +from tomorrow +equ il +eco soc +e manuel +desecr ation +confe c +cam b +bel atedly +beat nik +av atas +an sell +acoustic guitar +win star +tarek fatah +super junior +som any +scic hat +salt spring +richard son +re forma +r our +q os +prep zone +ow i +moolool aba +mati syahu +la zing +kle iner +gr und +g aki +form ality +ferri by +determin ism +cut scenes +booking com +boo oooo +blu er +barclay card +al ition +af aye +adity aroy +ðŁį¾ ðŁİī +âļ½âļ½ âļ½âļ½ +âģ¦ âģ¦âģ¦@ +а ÑĤ +yo b +v rx +un tv +time form +ti ggy +the morning +sun star +stow market +ru kia +regrett able +popul arized +per dition +pat y +on ism +nobun aga +nad ella +mis ch +jack jackjohnson +iv orian +hu mer +herb streit +he iro +had ji +four square +faste st +fanta size +extor t +dor nier +design inspiration +deep mind +dav in +co location +cin ec +catter mole +anesthe siology +ana bel +am paign +all ura +yal ta +wur litzer +wp bf +thiep val +the horror +th v +sto b +qu aye +port able +peri sic +pe ut +ot way +ob je +nehadhu pia +mol dovan +log gia +lar kin +kyli ecosmetics +kari uki +jap ati +j ory +im the +gwyneth paltrow +grave site +futu ren +fen church +ero ad +endit movement +ei mear +e inf +david price +bo hn +beau maris +au drina +amazing ness +wy nyard +w ws +ven e +velaik karan +urban planning +tyn dale +theat rics +sup my +st kitts +se si +se kol +scor chers +profit eering +pi ro +peri operative +over loading +ne edi +lan sdale +itsli verpool +hood lums +hand el +ha gs +goose island +exciting times +edge computing +edgar town +d tf +clar ks +ch ads +cau gh +burn leyfc +bur an +bis muth +bella hadid +be active +bar tok +agath achristie +ack royd +ê¹Ģ ìĪĺíĺĦ +É ´ +wil lam +tam ers +st m +sk ys +shad dix +par iv +ono ghue +minu scule +mai sha +lu pi +kat ak +ka ja +jack daw +illu sive +hh sgov +gre ta +g ool +g fk +famili esto +facts matter +dwar ven +dont mes +diof avatas +car swell +bir kett +amid ala +alu card +ak il +æ ¬ +wedge wood +ti mc +tali bk +swaff ham +spla yoffs +spectacul ar +she i +sen in +satri ani +rhode sian +rashtra pati +qual icum +qu am +potter y +pon chos +pa ja +ne et +mzan si +maul er +mai sel +k mg +jackie chan +impac tin +hon ori +gun dog +fle abag +dru mheller +dr yan +do gue +dioce seof +cur tail +creative writing +chat elaine +car der +bri z +ber zer +b itu +archae o +add ons +a all +zap at +x ue +vancouver sun +van es +the bull +th hour +tel uk +spy gate +sm rt +shaw nab +sc si +sal alah +rowy so +red cros +rainbow rowell +radisson blu +r chs +pratt ville +poiti ers +pan n +oo ke +mu ddle +mor avia +michi ana +ky r +kli ff +hu ana +henry gayle +head butt +hair ball +g tn +fu rey +fram boise +evangeli zation +ec ard +destabili ze +de me +by c +badrin ath +amp ere +Ûģ ÛĴ +ver ney +uffici ale +tb wa +sof twood +soci ale +rede ems +q tv +pu bic +per ches +peoples votemarch +oni x +ol ith +oh su +novel las +mechat ronics +lunch ables +h ings +glass door +fc pa +fanexpo canada +edex cel +dict ating +courtne ym +child splay +caloun dra +birth mark +bar na +alway swith +/ + +ze is +y se +wil fork +wee ting +variet als +uof r +twee talong +sw ash +shar ad +se millon +ru bella +rap sody +phd forum +ny ce +ntv atone +no ice +ni biru +mu ma +mic hen +meaning fully +kawarthal akes +juliebishop mp +hu at +ha ff +gri eco +gibr altar +fire fly +f nb +dakah lo +buck cherry +bcfc tweets +bak ari +au dre +ash ly +andre essen +ai sha +ad duc +âļ½ï¸ı : +x uv +tu tsi +ti go +talibk weli +squir m +sound board +rehear ses +planet side +passage way +our time +os stf +ojib we +net ter +ne ac +n kn +mu ki +mo ins +matri arch +ma stani +laven ham +kiku chi +jan am +hinojo sa +fo etus +eye shadows +enig eria +ec rowd +dimit ar +did ger +di efen +defrau ding +bull sh +broom stick +back us +ar gh +xo los +squ amous +shu ai +shi ans +samo ajoe +sam ford +s llc +priyan k +pio tro +pic slip +or don +nol de +murciel ago +lu xion +lin ds +inst it +ing laterra +idoli zed +health insurance +harmon ia +h sl +financial freedom +ffici als +eu dora +con scription +clun ky +char lee +cap i +bannedbook sweek +baff le +back lot +b mcc +ascend ant +alu ddin +ðŁijĭ ðŁı» +ðŁIJ ŀ +zi v +x tr +wa ec +ve schwab +time e +thim phu +sh ree +r sca +peto sagan +palla dino +old victheatre +ner ding +micro beads +leven shul +larch mont +ja ved +ir ation +inf p +farc ical +f ars +dono hoe +dipo log +cho m +carnar von +can cha +bli ghty +ate m +wpl glocal +vive gam +ver sum +ur ne +twitch sharing +thap ar +shri ek +shaun king +ri is +re grow +raz ia +picture house +out stand +oo stende +ntn u +nav arre +mi micha +lamin ates +kilmain ham +kah le +ic a +gan za +figu ral +far miga +fa th +extru ded +central asia +buck town +bre ton +birthday staroftheday +basil icata +arri son +ðŁijĩðŁı¾ ðŁijĩðŁı¾ +wim pey +weis sman +visit maldives +vic eland +veget ative +um my +top ten +sp ss +som meli +sky diver +saw fish +partiti oning +n goron +n anga +mono grams +mercat or +mary mcdonnell +lush cosmetics +lc u +khayel itsha +ka ke +ji ya +it b +ic ff +hol dup +fur longs +eric prydz +car ys +buk ola +biennal ear +bay i +bari atric +aged care +âĻ £ï¸ı +z older +yes allwomen +western ma +vaill antuk +u indy +tric ho +tri as +the b +tar paulin +swing man +sun kissed +stump town +star tsnow +segu ri +romney ryan +real clear +pa chu +nulli fy +newh ol +mer api +lauren ti +kiss fm +kat er +jugg alos +jarre t +guil in +crun chie +climatechange isreal +bur dick +be ales +ba tho +b gb +b dt +adventure land +! ðŁĺ± +ver on +the deverakonda +sto wed +sp alletti +rhy ne +retrac ted +rail gun +pi qua +phonec ase +patri mon +nov elli +n serc +ma dani +lat am +khu t +indv snz +heine ken +guaran ty +golden age +fru mp +enne agram +do wie +digital ocean +darwin ism +cri stine +coun ten +chicken pox +carry out +buen as +bic arbon +bi onics +asci ence +) "@ +ðŁıĨðŁıĨ ðŁıĨðŁıĨðŁıĨ +çŁ ³ +à¸Ķ à¸Ķ +xx xl +woo oooo +wom ened +wa ren +vi ere +thun ter +thor se +simon baker +selfi mprovement +scul pin +sch wer +ro hr +raffa ele +ra yu +per version +pamp as +os seo +og u +nü rn +my i +mar ter +le vo +ko kan +im kona +ili ac +goo ooooo +gal antis +fed de +exce sses +compac t +citizen ry +cign al +che am +ap lace +an jos +amazon books +a av +wash board +u tti +tre ssel +stro h +sig ler +s voice +ride with +reliance ent +promethe an +pel ly +lie ber +kan azawa +k se +john cusack +ichand ra +ic ona +h ws +go parks +fire ball +fanatic ism +fal li +de ssa +de park +de mann +cur sors +cos ito +corn us +chelseaf lowershow +c ts +book maker +bne traffic +bb mme +baw try +avi v +au se +ae b +ab lec +è Ģ +è se +yng itis +ver re +ul mer +toy land +to ck +swill neverknow +sw va +stro mae +stephen hawking +sequ im +prop elling +po index +oun de +or ourke +nor vell +m zee +liber ally +le les +kim o +hit music +getwell soon +flir ted +fi red +et ta +don go +cibul kova +ci gi +ci en +ce peda +brad street +boo dle +bis cay +antiqu ed +ðŁĴĹðŁĴĹ ðŁĴĹðŁĴĹ +âı ¬ +ÑĦоÑĤог ÑĢаÑĦ +work stations +terre bonne +ta im +subli mated +stv j +so bama +sneak ily +si sk +sell s +scimit ar +s market +re ste +queen sbury +per vaiz +nat m +moh ler +meth adone +ironde quoit +ic si +gran ollers +gil der +fu jis +frie del +db fz +dark shadows +daily nation +d live +contra sted +clau dio +ch ole +canal etto +bor na +bigh orns +behind woods +arak an +air brushing +ac afe +èµ · +z wir +z ra +wor ships +truec aller +ss x +smithere ens +sket cher +re its +rapi er +poul sen +plan b +mersey rail +ligam x +l bk +kim hyunjoong +k alia +j ld +irresisti bly +insur ing +idol atry +en ice +elder scroll +dom browski +deltag oo +bryan adams +bou ff +bis sett +ay i +ari b +ali ving +al sati +ah r +un sun +twit ches +totten ham +to var +the crown +synergi stic +supmy tempo +snor ting +share tunes +righ twing +pos adas +pen rith +pantal oons +on z +moon cake +memorial cup +mel isa +le brun +la phro +just like +jac c +ja ina +ice box +gift cards +gender gap +ge il +fish guard +ene a +di mer +delam ere +bolshe viks +bland ing +bill murray +aqu in +winnet ka +watson ville +u dp +thrott ling +te pid +syn cop +st ere +se tit +ov ac +onthe wall +mussel man +midr and +maxi mizes +master ton +madon sela +m elling +lol lywood +lap ham +la throp +l ps +ith ac +ist ana +isol ates +hydro xide +hal ligan +gaz gshore +fu u +fli c +f sm +f ado +er ji +curi os +crusad er +conquist ador +chel e +bro wed +bourgeo isie +bol lard +anxi ety +am ec +al sina +ðŁĺĿ ðŁĺĿ +ðŁIJİ ðŁIJİ +ðŁ§¡ ðŁ§¡ +ìŀ Ń +âĨ Ĺï¸ı +yo gam +yellow card +welsh government +under tone +today fm +tan uki +sx sw +sun stone +slu mping +play warframe +pe ice +nole fam +my les +lukas z +ki v +kemp ire +hel den +gopin ath +foxstar hindi +ent r +deltagoo drem +dai ya +cheer full +care em +boon do +beno ist +baz zi +babys itters +as orock +artinstitu techi +ari elle +اÙĦ Ø£ +youn t +val is +uri st +twee ple +tra uma +t ck +suiko den +sco pus +ry th +round hay +re programming +pro sely +perfect day +pau le +negr on +ne co +ministry wcd +min ka +mimicha kraborty +madein canada +ly d +hi o +her peto +gi re +escu cha +equin ox +dontmes supmytempo +de gen +day dreamer +courte san +cate blanchett +bar ro +b hour +arch enemy +ad aa +. ðŁĩºðŁĩ¸ +! ðŁĺĤðŁĺĤ +ðŁĺĻ ðŁĺĻðŁĺĻ +wharfe dale +wall send +tank top +su hail +speed ball +sl cc +sim z +si gel +sau thor +salvador an +per severing +news agent +na poli +man olo +magu fuli +lac onia +kri stal +kortri jk +jan k +jac a +ft g +free all +fik ile +fan mail +ese tter +e ese +downfor what +doug jones +d appy +cur rier +croatiafullof life +costu ming +carpe ted +c series +bugab oo +bol ing +bas sem +app l +amo tt +a eu +ðŁĺĬ ðŁĺĤ +à¹Ģภķ +ts vangi +tr itt +spelling bee +small holders +ra ph +protru ding +pine al +pee wee +min en +mag is +la kin +karyo tic +kag eyama +is ca +inst alove +hy ste +hot pants +grun gy +ge ot +g naw +free holder +don te +ci al +bur don +bump kin +brun nen +ali pay +ah hhhhhhh +acram ento +) | +york university +vi burnum +un verified +tra ylor +ting a +theafric amentor +the michael +ros ann +pre teen +pre fontaine +pe tras +oz u +out sole +mo ylan +mc adam +markus feehily +mac fad +ko ichi +kin smen +heart breaks +gil son +equestrian hour +du ta +dog walking +dam eron +cosmopolit an +communic ative +chim ney +chick a +chester be +brid ger +brick works +boat life +ad ze +ðŁĽ « +ðŁĴĢðŁĴĢ ðŁĴĢðŁĴĢ +y erin +wn yt +we sty +twit tor +so g +se ob +sa ke +post uring +mer l +mandalay bay +leonardo dicaprio +kno wit +jor din +inf lexi +im x +ic ai +hisp ano +herak lion +ha thor +gyna ecology +gu lak +gib sons +gerrit sen +fund able +disen franchised +devious maids +crow es +cour ted +commu ted +col well +are port +aer ator +ya yo +y la +we at +ukr anian +truff aut +the ming +ta way +sre ed +sl ounge +sky scanner +sk nights +shirec cc +shil pians +san tu +rv ca +raashi khanna +py con +mur mur +mei ko +kur th +keep moat +hann am +flag ell +fen wick +en sion +dra win +down force +di gger +dese gregation +d jim +bucke thead +bran n +birdseye view +bel ittle +be dri +barn hill +ag assi +ðŁĴ ¢ +áµ Ĺ +yu gi +x zi +x rt +va an +ut agawa +un sinkable +twitch play +tre sa +town ley +tod rick +studi om +sc inema +rd chat +quoted daily +poun ders +pit zer +paul pogba +pack mas +pacade mic +ole ary +ok api +nic king +key tar +kay mer +immacul ately +hu leni +har b +h jk +goldenboy boxing +glent oran +f bt +eter no +edinburgh castle +ec lark +dun lin +clean ly +ci ps +chop da +burk hardt +bis bal +bil zerian +ap in +antag onists +yu ya +untapp d +tyd fil +twee ties +tol u +tn k +the cine +sun fire +sude ikis +sub side +spur n +slee pa +shop indie +rrrr rrr +ra ii +prin ted +ofthe game +le ly +ka veri +k zoo +just kidding +igu azu +helsinki uni +forts ask +fin ra +dur den +daguer reo +clu bber +cat ahou +bo vey +as ah +vr chat +thu gg +tele film +squ ints +sk Ã¥ +shock ley +ru scha +ripp ling +ragha vendra +pyro technics +progressive house +pen ter +over de +ni ds +newport beach +muer to +med way +man te +la place +k th +jo lo +gil dan +gh b +dong woon +de frosting +d how +blue gill +. ðŁ¤Ķ +ðŁı ŀ +ãħłãħłãħłãħł ãħłãħłãħłãħł +z tv +yo el +wine glass +think geek +succumb ing +se ast +schu tt +sa ez +s fl +ri jn +reform now +pit ney +pe ch +pan jab +ol ling +off sets +noct ilu +ne ga +mu bi +mot sepe +malaw ian +loo kin +le if +kr acker +it son +in au +ev aded +ed camp +ed accessible +e ich +d ju +colorador apids +clou dof +challenge accepted +bul ger +ba hut +ar keting +a oun +ðŁĴĹ # +ðŁĴĭ # +ðŁ¤£ðŁ¤£ ðŁ¤£ðŁ¤£ðŁ¤£ +ãħ ľ +z oku +worldwar ii +wich ita +wearein puglia +ur lacher +unh cr +tranqu ille +spear fish +saunder son +sau rav +sam each +saluteto service +ruge ley +re decorated +phy te +ot so +ori sts +nex change +ne gate +me any +jere mie +holly hock +he ys +guid ry +fre sn +earth day +defl ate +con oco +choice internationalartist +check book +caric om +ble ue +baji raom +aur um +asu u +ais i +ach ille +wu vip +wishful thinking +wigan warrior +weg ner +unice findia +summ ative +recap tured +pri ve +person hood +p anga +new jersey +ne squ +mac chi +le mentary +inver urie +infra structural +independenceday india +in fringe +ham mon +goo der +gn fnr +gla ssc +gh andi +d ations +chad wick +cacophon y +ca rec +butter ball +bu gis +be ale +appic oftheweek +a store +ðŁ¤ ³ +å ģ +~ ; +yu ju +yas ssss +water down +war machine +tu ggle +ton nage +ter ly +spite ful +spang lish +sleepy time +rox ette +rome sco +ren uka +pu mbaa +popup shop +perfec tionists +palae ontology +ne ces +nba allstar +mn d +mini mus +maz u +mag lev +lea day +ka ela +jap ur +ith es +dx b +by ng +buff er +ber glund +bak in +alphon sus +ðŁĺģ ) +vi ad +train hard +thisis the +t out +river men +raz in +pla ins +peck in +ling en +kom ori +kh rush +keepbritain tidy +kate e +jit u +idle wild +heze kiah +glos birds +genetic ist +frangi pani +dev aughn +crypto graphic +c ads +bu stier +ber ber +baby z +as ari +am ass +í ħ +å± ± +~~~~ ~ +xim ena +x cited +well nes +vin ta +v gm +unci os +torre ira +then ia +tejas swi +ta inte +schrö dinger +radio activity +poo k +ou is +oo tn +medi ating +k able +iss ner +ir ri +ingui stics +in pink +in fidel +ham ble +fra s +est y +dair port +cra pe +comp lies +boat yard +batt i +an thi +am era +actu ality +west morland +we ssel +ve sely +unear ths +underwater photography +un consci +theryan adams +the hill +shu is +sed ative +ralph s +pic ardo +onther adio +o ig +o bert +mom en +ming hao +mercy me +mari kana +mac gowan +lethar gic +johno liver +har ima +haberdash ery +guzz ler +good ger +frank l +feu er +dir tier +cas l +bol sters +ay ak +am bit +! âĺº +world childrensday +west ville +un justified +u hl +turn stone +theni asharma +tatt ers +ta wi +supportsmall er +stra the +stin ker +sm ou +slam dance +skil ful +sauter nes +ram madhav +r fr +quat re +protec tor +pal am +nut ley +movie pass +mfl twitterati +margol is +mar leau +lh saa +he wer +hawai Ê»i +fellow es +f ch +etsym nt +dag ar +cheer wine +change management +bust ard +bu h +bra via +bel lec +b pp +app r +aplayer sprogram +amic ha +accomod ation +åŃ ¦ +van jones +u stin +tri x +tam ia +sward son +siar gao +shou nen +repadam schiff +rausch enberg +r ke +prop ell +pe jic +oc toberfest +o sta +new gate +mo sas +mer co +lake districtn +ko sh +kan on +jo w +indv ban +hur tin +hondar acing +ho pper +hbo boxing +geoff johns +fru sci +fis kars +felic itation +deepak chopra +ddin h +dat in +comuni dad +ch hota +carcino genic +banjar masin +aut zen +tw an +tri bals +team bts +sub buteo +sr at +se ta +sal onga +per ig +over powering +mec cano +mccl ung +mar kaz +ke ss +inve steu +impe ded +ian u +hello fresh +gtas nap +extingui shing +entom atoes +eni x +edo dyssey +e mis +country music +confront ational +bin ned +bhu v +annot ating +ambed kar +x bmc +who ah +ut sc +tou l +th birthday +super yachts +sul la +stry per +smel ting +smar tin +sal sify +roberto cavalli +pill box +pga show +narro west +mye dit +moss ley +mil y +judge mental +her universe +he ver +go leaf +global citizen +gira ffe +fluctu ating +fat ales +cura çao +clay more +cap tors +bu dak +brad pitt +bir n +ba dia +angi es +an etwork +alla gash +al tho +al it +ac aba +ab ita +yu jin +wonder ment +turbo tuesday +su che +si amo +sho peep +shopeep h +s month +pokemonsword shield +paddle board +maz har +kick started +ker ja +kear se +kan chi +k you +hoo phall +gym motivation +gro ton +flam ingo +fair bairn +eury dice +etr ics +em press +dispo sse +dev das +cy press +ce di +bat ur +bar ras +bal oney +... ðŁĺī +ðŁı ĸ +âķ ļ +ಠ¨ +yag ami +vali ente +tinie tempah +thehashtag game +ss oftball +spiegel tent +slou chy +serre mmy +sa pper +pha edra +par to +p cc +ot u +o este +northeast india +ner dalert +merri field +li ff +kookabur ras +kitch in +ital o +housel ondon +han u +h waiting +gov nl +forwar der +f ath +equi ps +conserv atories +clo velly +boost mobile +bic olor +ber ts +bald rick +art twit +accommod ates +ðŁļ ģ +âĿ£ï¸ı âĿ£ï¸ı +âĻ ¯ +z berg +yel ena +south aven +sou ther +sketch fest +sel and +sc ut +roll icking +road bike +re amer +r db +press office +na j +mother care +meet stvj +inver no +ic ould +hulla baloo +hell muth +hall marked +gri c +bath tubs +at b +ìľ ¤ +è £ +âĻ ķ +à¥ĭ _ +ÑĢ о +zen yatta +yess cotland +wh io +volksk rant +un ltd +tro oper +ti run +the cw +thal amus +tax man +st rood +sn elson +sm mw +sen ran +sec r +sch ia +photo copy +pakv nz +oak lands +o sco +nu pe +nam biar +mur t +mok sha +mam ak +lakedistrictn pa +josh lia +jay leno +inter course +gr ackle +gen ic +ge tu +ful ci +foo trest +fjor d +e cha +dhoo par +con ger +cau ght +buck wild +blo grt +band shell +azadimarch pti +amit ab +. ðŁĴĸ +% % +ðŁĽ Ģ +ëį° ìĿ´ +ëĭ¬ ìĿĺ +ãĤ¸ãĥ § +zum walt +z han +world con +wi ec +wag goner +uni vision +transl ation +the bad +temper am +sto pper +sla ger +sh q +s bj +rcb v +rb ma +rashtri ya +par mo +mano euvre +m run +lucre tia +ip sc +incol n +id ine +house hunting +gidd yup +epo ch +dern gate +dart ington +cop adelrey +co piers +chec a +catan zaro +carers week +cam phor +bustam ante +bul li +bou y +blay lock +battlefield v +bank sia +ascen ds +as san +antoni oni +americann injawarrior +am ah +alley ways +aldub meetstvj +alab ama +al amitos +worry ingly +ultra thin +u emura +tu sker +ti miso +ta kor +staphy lo +southern railuk +son ice +smoul dering +smol tz +save the +river fest +reading list +proclaim ers +po ss +ny d +ni kil +nbc svu +n of +mis behave +luxury hotels +lu kin +lipsy nc +ks la +ko ki +ju elz +ike bana +hot tie +hiker chat +hau ser +gi ang +emer g +cu shy +brat ty +bch wy +basketb al +as q +anti war +andalu sian +amherst burg +aber soch +' ] +ðŁİīðŁİī ðŁİīðŁİīðŁİī +west bridgford +v nl +so dhi +shutter fly +scar borough +sau ctions +sar far +round abouts +riseand grind +ric co +music biz +mari ko +le cker +l illa +ktn news +geel ani +fli ppy +doo kie +domin gos +close d +che o +cat rin +black star +be emer +a jun +ðŁĻĮ # +ðŁİ¬ ' +warren ton +visi bil +temple univ +t ny +sit ch +shel ove +sh ini +sequ enced +scu ola +quad ri +procrastin ator +pin kin +paper boy +naturo pathy +n ouri +milli kin +mac dowell +long man +le tras +krist offerson +kri m +ker as +ing dean +housing crisis +ham za +gra e +gh r +g sat +fac ing +eun ha +e oi +colli der +bri des +beach house +ar mit +all t +ad den +ãĥ³ãĤ ° +zacatec as +welcom escotland +wa pping +voice first +st ints +sav or +roof ing +ro shan +rec tum +out weighs +ms by +micro array +mar dy +kalaign ar +hand work +guardian ship +fore castle +fen ice +fabric ations +er rrr +ene ch +eat at +ds ound +consul ting +cm world +broad y +break dancing +bp cl +ban del +ak ere +ag pur +af lac +[ " +ðŁ¤ŀ ðŁı¾ +what make +wakand aforever +w cracing +val jean +un ge +twit cho +t li +sy o +stru thers +sha and +mar cy +ma be +l wt +heim lich +heather wick +francis cans +field work +est en +del tona +cycle way +ces spool +bur nett +barba resco +aspin all +apor te +__ . +âľ ° +zoo logist +yor ba +we ard +wall in +tur m +timmer mans +ti ong +this ss +tat ers +sw inger +son dra +si phone +secre te +pythag orean +post modernism +perfu mery +over thrown +orac lec +news man +national pizzaday +mal u +m smes +lede cky +la brin +ke elan +i spossible +holder ness +hel ge +harle ch +giov ann +f örde +er rol +epi dural +dongh yuk +detr itus +cor yn +capital tv +canon ization +blow back +berg mann +be il +ba thin +aux ili +aggi eland +ðŁıĨ . +âĿ¤ï¸ı âĺºï¸ı +yo bo +xx vii +wind ass +we sker +wcc b +war rant +tol and +tele presence +ste ren +sp ath +se sto +sau der +ry den +qantas wallabies +priya anand +paris roubaix +o dr +nick cannon +ni o +magde burg +lo thar +kenny omegam +karen ina +jon ze +je ane +iti o +heath field +ex y +den ney +courtney force +cer ys +burn aboy +beig net +bag gs +ash en +anecdo tal +wildlife art +transc ended +ta via +streng then +show place +sanji v +sal les +returno fthe +princi pia +pau lista +ok amoto +nine inch +mu tu +min smere +manjre kar +lebat ardshow +kene ally +gh or +euryth mics +dub smash +chiff chaff +chi kara +buil da +boga erts +bin nie +ber ner +asi t +ary as +anj ali +agni eszka +abhi jit +ðŁ¤¦ âĢįâĻĤï¸ı +zen on +your boy +ver ry +vanda waterfront +un trained +un original +um t +ty ro +tendin itis +stra ighter +spring ville +simon ds +shop lifters +sapere condiviso +saint paul +prowrestling ts +plum stead +pi stil +phil om +petro vic +ne vere +multic am +mo go +main ers +ma wra +kab ayan +ji ka +hall i +gar lin +fuji wara +fri ston +fast pass +dep ooja +chale ts +c illi +bre reton +best team +b ni +ag ens +whitec ourt +whistle blowing +wau n +vre eland +un dum +uk tv +tailli ght +tag ua +sé bastien +retali atory +remo dsouza +rb k +progno stic +or ic +on star +olek sandr +la al +kennyomegam anx +jer on +islam ism +insu lar +hen ning +gal indo +f ter +exple tive +docker con +design ating +dam bi +culture of +cr d +con me +chor i +car ob +bookaday uk +andy burnham +am ick +aller os +acom munity +ðŁĴĸ @ +º f +z music +yel owo +wq xr +wido do +war path +ti maru +ssc napoli +smu dging +se itan +recur sion +re homed +progen itor +pra shanth +pluto flyby +peyton list +pend ra +mus d +lati mes +kwan za +kev yn +justic ele +jazz music +in consistencies +in cas +in ator +hoy lake +half moon +dementi a +dem ure +criticalrole cosplay +cornwall is +chuk au +chhe tri +car dale +bri anc +b so +autom aton +ar tha +anti thesis +alv arez +alex ie +⼠½ +ৠĭ +yu kio +wcc b +wash cloth +vijay diwas +venuse swilliams +v li +tt ingen +tro xler +sor do +sl x +shim bun +she ene +rock a +re training +ra ha +proof reader +prolet ariat +pin ta +pedagoo friday +pe der +ortho dox +off shoot +nu gs +newyear sre +n wac +mor illo +man tas +m ure +law man +kishi moto +kenne bec +k anya +jummah mubarak +hiro mi +fan x +eye piece +esc p +eck stein +dichro ic +de aver +cra pper +cou riers +cor us +chand hok +cch l +cas key +biggreen egg +au ss +ag nus +af rance +! âĺĢï¸ı +ðŁijĮðŁı¼ ðŁijĮðŁı¼ +è le +your anon +year wood +un tie +tur nit +tor sten +thom asian +tb ats +swwap niljoshi +sop retty +simple plan +sil vana +se ppi +sat oru +s love +ph wx +mom aps +mir in +mer curio +loud wire +lexisnex is +ko do +kim mie +ki mi +ing ri +ham leys +gar misch +eu x +eng al +e wi +demon a +daily post +cul tists +colle g +c sd +c ason +body art +beer house +ald wych +alber ni +! } +yel ich +water mill +visit nc +unlock able +un attainable +u ct +traff line +tas ki +tal low +st off +space weather +snu ff +smash burger +shep p +secon do +school choice +riv lin +radic alized +pug sley +prime minister +old navy +o aths +nt pc +muja hid +migno gna +michi e +micalle f +lusit ania +looooo ool +le vies +kra ken +ive son +inte gers +immin ently +if ma +horror film +fpj b +fluid ics +f nac +es th +el b +eb bets +ci at +char it +apo ptosis +am bos +ðŁĺī âĿ¤ï¸ı +بص رÙĬ +ti ef +te i +tar ja +t ade +pd n +pascag oula +on asap +ni xie +nay e +n me +music india +mam mary +long board +krak ów +ken der +happy wednesday +girl band +fau stina +fan arts +do tting +dar m +con ium +cleve leys +broke girls +br inks +bas er +ash vili +asan try +ðŁĴ Ī +ãģķãĤĵ ãģ¨ç¹ĭãģĮãĤĬãģŁãģĦ +âĹ ĸ +upp ere +twitchtv gaming +to serve +tat litu +t savo +sigh ing +sav en +royal randwick +ri ks +r tweets +play fair +p mi +nuff said +n mi +lick er +jawor ski +ing ia +ho orn +harvar dh +god child +dol men +dog and +co lette +cit go +bru g +bo asted +blu ecol +ambi dextrous +am ins +é o +tweeta pic +thur n +street dance +stau ffer +scra ppers +pu an +protec tionist +passer sby +pa ia +oxid ant +norm als +nir bhaya +navi o +nac er +ku ip +iri g +ide alist +hishammu ddinh +gro ggy +daver amsey +chukau munna +bla gden +ben ilde +barba rella +auto cratic +asqu ith +asham edly +addisab aba +xer xes +whe alth +val po +v yn +tar ry +tann adice +sw cc +san ts +re pulse +price isright +port ola +o zo +o tros +n gan +mohan shakti +mel vins +masqu reshi +mar ven +magni fi +l mi +krat z +jeff co +je v +intothe badlands +humper dinck +hu masqureshi +hu acan +gun ter +gautam gulati +gat t +e or +crissc ross +cre ech +coonawar ra +co ed +cheer fully +arn dale +app elle +æĦ Ľ +wool rich +vin dol +uk scouting +uf confox +ta ito +t ella +stu dd +si dent +shel ve +resver atrol +prestige diesels +per fi +new world +na ja +mr peter +may fly +lit ol +liberty london +kale y +ill ya +hill sides +head bangers +guard smen +existenti alism +en k +el icious +dew tour +deflec ted +ch lo +bum per +big weekend +bhu t +az kaban +argentine an +andron icus +up turn +u sports +tri vet +the heirs +thalasse mia +ter rains +t ates +sul len +ri ram +recombin ant +re decorate +pp ps +penguin day +pas ado +nottingh ill +ne ys +mugh als +mind map +lu q +ken sal +ig inla +hyster ics +hermen eu +heg depooja +h te +h li +garra way +fu bu +f age +di de +day lily +cri stal +craw lers +chandler riggs +ch eni +campan ella +caip irinha +caf f +bre tton +boutique hotel +be swick +av ens +and black +vol are +vick er +u ca +tri um +too ley +tin caps +super bad +sti fling +steel case +sh ero +sea hawk +persist ently +painst aking +ous d +negr ito +lo vi +laksh man +know lton +hol men +gol dring +gh or +event planning +copy right +contest india +cine phile +cayman islands +caf cofficial +but thurt +bre vity +bat tier +bas ile +bare illy +ba isse +ame al +al mon +ï¸ıâĥ£ @ +ãĥĹ ãĥŃ +âķIJ âķ +á´ ĩ +Ùħ ار +wh yd +uniof reading +under used +uber facts +twitchplay spokemon +test i +sta ite +sne xt +se belius +sa aho +s ll +ru lli +refu gio +qu s +pre dominant +pr ats +potre ro +period ical +passive house +of acial +moneti zing +mo ola +me igh +mandar ake +kh ta +ker win +kaz uya +k tp +issar ae +is y +ini shi +helsing borg +ham ada +gladi olus +g ort +fa sti +equ an +el fin +dough nut +disappo int +def tly +book week +black listing +ber ahino +bad water +as bo +a italia +ðŁĺĺ ðŁĺį +ÙĬ ر +wra th +wiki how +web be +vie to +vid ere +um titi +u mmmmm +trout man +thum ps +thisi sc +tab las +sub due +stra ding +sei bert +s met +releasethe snydercut +rech ts +po gue +pagan ini +oo sthu +nz xt +na har +my re +mixed reality +knight s +kam era +interi eur +infomer cial +h plove +field museum +fati mah +ent en +discord app +detri ment +coffee addict +chicag os +bou cher +boc coni +bach ia +ave bury +arsen io +ane ury +ago toronto +ðŁĶ İ +îĦĨ îĦĨ +war ung +vo g +twit t +the strain +sil ber +set ter +schi ef +rose wood +re sta +penn live +pad docks +ot al +o iler +mis diagnosed +minec raf +le ef +la way +kin ley +ki shore +karan mehra +jail ing +j angle +glass blowing +fau stus +e bru +dont cha +dfb pokal +de spot +cutt in +blan chfield +biop sies +ani zation +alec baldwin +al var +ðŁĶ¥ ðŁĴ¥ +ë £ +z au +wood sy +u haul +te oti +taste bud +stom e +spr it +so ave +sim co +si dings +rocke try +ri kerr +plan eta +par ivar +official pdc +nu eces +nineinch nails +ma zzy +lec tern +le ffe +ke illor +j vp +industrial isation +fl cl +de od +cra vens +ben tayga +bad ness +ba stos +ar bus +ak ame +ade v +. :-) +ðŁĴĿ ðŁĴĿ +ðŁĴĢ ðŁĺĤ +ðŁijĩðŁı½ ðŁijĩðŁı½ +whomade my +weald stone +vig ner +sma shers +simon schuster +sil ang +screen grab +sch aden +pre amble +pan elled +p imper +over turning +ordin arily +mindy project +micro blading +micha ell +mf gday +malvi ya +la gos +k ado +jer ky +ine t +h fs +gigab a +fr drx +forte scue +flo s +der van +crazyricha sians +check outs +bin ary +baltic sea +andre u +a oh +ðŁĺĤðŁĺĤ ðŁĺŃðŁĺŃ +Ë ¢ +ü ck +zare en +yr dsb +venkai ah +the irc +si ssi +popul ating +orange bowl +mrjames may +modern ised +ly nott +li minal +ju alan +inter agency +indefati gable +hi ze +hc dsb +havil and +gd la +fe tta +everybody in +er p +ed westwick +dal it +cu pp +critical thinking +back man +b fw +ari m +aid u +ðŁİħ ðŁİĦðŁİģ +иР» +zou ma +y von +worl dis +woo zy +vi um +urban exploration +tour guide +tom foolery +ti mmer +thri fting +t ole +su ke +sto ch +ssav elives +spoke sperson +shear smith +run nin +ro skam +pra deep +pein tre +oo h +nc ca +me ep +mal indi +m ts +la var +iv onne +in ara +front side +fanta sizing +extrac tions +enthr oned +el ve +draw back +cre atin +coral gables +chan try +black ley +behe st +zak ir +ver gil +v san +ty pal +tran mere +tor ri +to ques +tl x +ther mally +sil iguri +sei yuu +schwarz kopf +same tha +pum per +pre production +paradox es +pan j +ok ano +neck ties +naomic ampbell +mol son +middle man +mass i +mash ima +maj ik +lur ker +ld conf +la bo +k fi +h jel +g hay +don jazzy +debat ers +de vers +dag s +cut let +cole lla +can wnt +bb ctms +b ka +arm yof +--- >> +ðŁĻĭ âĢįâĻĢï¸ı +ëĶĶ ìĺ¤ +x au +would ve +vic arious +ven tre +tre anor +tra pad +touri s +su su +shorth anded +sed alia +r sb +ninj awarrior +new scientist +nd re +mun chen +mahog any +k rane +jo balert +ice gov +heske th +gored forwomen +fish bone +fi bula +endo scope +e gh +corin na +block ages +art ag +aristo cracy +. ðŁijĩ +ðŁķ Ļ +ðŁij¨âĢįðŁij©âĢįðŁij§âĢį ðŁij¦ +yn olan +va un +u zo +the elephant +ted talk +te sty +ta shan +ta ji +swis scom +sun power +scol ari +revol ver +rash ard +ram blin +ra an +prin cen +pin afore +pai ste +onther ise +naz ca +mrin poche +monte verde +mas thead +ma at +low en +kurt angle +jol ley +inter professional +guit o +gan e +fakel ove +eyeson you +do hc +dd ha +davi dal +cul tof +cre m +ch aus +cer ca +car and +be friends +bbow t +arter ton +air venture +ai der +aham sharma +aaaa aah +a ama +ðŁĺijðŁĺij ðŁĺij +ðŁĩ¨ðŁĩ ² +z han +wy dad +wra sse +we tv +w ente +vel shi +u hp +to so +spe a +re invent +pun j +pott inger +or nel +mitsu i +mb app +lulla bies +lin ville +kam inski +k anga +int angi +hpe discover +guen ther +gi zzard +econ d +di gga +chan hassen +ben adry +bed ingfield +battle ships +auto expo +amerit rade +ãĥ» ) +âĻ¥ # +âĢĭ : +tar anto +student voice +st ents +squee gee +son ography +sol f +sher rie +sac bee +s finest +red nation +py con +procor rpcrd +pin cushion +persi sting +nat con +mvp buzz +man ews +la ssa +ja jaja +ill on +ha bra +getac tive +fa ren +er h +e redcarpet +diabe tic +cra vat +cmoffice up +chor tle +care l +car phone +car luccio +baad shah +ak ua +water boys +w mtw +v room +tis land +ste ful +s ÃŃ +ruden ess +radha krishnan +post colonial +po sies +pit fall +perpi gnan +per ching +pad dington +nfu countryside +manspla ining +madin at +lett sville +lam bretta +la under +kap atid +jour i +jale bi +invicta fights +hur on +he seltine +hard boiled +grave stones +gel der +fant asma +dry land +desper ate +den otes +colon ie +cat chit +carter sville +bl b +bird bath +bab ul +ath ome +ai zaw +าภ§ +z ade +wish youwere +westhe imer +wegot this +un tethered +the vikaskhanna +ten dering +swim bikerun +sky force +sc ouring +ro sea +ram paging +pal co +out pacing +oc el +now lan +niall horan +negoti ates +my news +moment um +max illo +l pi +in my +hplove craft +hol loman +fresh start +et su +du jardin +du ffer +d acre +cr di +cosho cton +bro ach +blo tting +be do +bal asu +ar ousal +ðŁĻĭ ðŁı»âĢįâĻĢï¸ı +ðŁİ¾ ðŁİ¾ +wt cr +un reserved +u lic +tre sco +tote ms +til den +stylist magazine +spor tin +shi van +sad af +row es +raw ling +rav ages +quim by +qu ary +pra ger +or omia +ohi op +odes za +ober oi +nz warriors +mor ano +mon stro +mign ano +maj u +lit ty +ki hei +jim ny +im thereal +ham p +hair piece +fi ggis +es ong +embol dened +di oni +cu pra +cam an +call ander +bil o +amc bride +al ow +ë¯ ¸ë +zi zi +wreck ing +wo bbles +will o +wen i +uniof bath +uki ah +to kun +the matt +shutter bug +ship men +pu ddy +prah ran +post en +pon son +pe yote +pad don +p vs +nad ya +mesmeri ze +mel ayu +m pho +lar ri +kani ka +kam el +hipp est +go panthers +first lady +farm house +dont giveup +dispen sation +concealed carry +citron ella +ches nutt +bri sk +boho chic +beer festival +ba rents +auto cross +ar ndt +accep tably +ðŁIJĿðŁIJĿ ðŁIJĿ +ðŁİĤ ðŁİģ +yaroslav l +x pert +w pt +tat u +steve carell +ski umah +sal uda +rothen berg +rever ted +re shaped +r li +people sclimate +nik ah +long wood +lew i +la jolla +la chapelle +kun da +julianne moore +jichang wook +holli ster +fruit vale +for gave +flat line +fin nick +edex cel +dou te +de tract +confec tionary +char min +bru hh +bri dgers +bo im +bo ggle +bicarbon ate +bad ami +av aram +ðŁIJ ² +Ì ¸ +wolf s +whywedo research +u moja +sukhum vit +shrie king +scoo per +pin tail +ol ha +o stap +mur al +mid gard +mck ayla +le ps +in ad +if pri +hen man +gwa ii +golden eagles +geo location +freef ood +fly high +extravag ance +e ji +dar uma +care ssing +book sfor +bon nell +ble ch +bac illus +ಠ¥ +whoo per +wh oooo +vik ramad +usch amber +us kin +trojan pride +track xc +taylor sville +sy outh +stray kids +spe wed +sco ffing +ric kety +re map +pro polis +po is +plan che +on te +o bra +milit ancy +mc ferrin +man nix +l hc +kat ter +infe ction +hun s +guil herme +forthe future +faber books +end bsl +edgar do +dl x +ce sa +brick lane +australi azoo +ameli or +agn elli +Ùħ ÙĦ +ya ounde +vidy ut +u mang +travel bug +sur in +sol vay +sim plex +shu ka +se cord +sa hu +re illy +paster nak +pag a +ol lis +lo hud +kum quat +he wit +hau gen +gren del +gl k +fo tboll +f br +dr acu +devi zes +cur rington +chun ky +brood mare +be aked +baf f +aero sols +^ ~^ +ðŁĮ © +ìŀŃ ìĬ¨ +zildjian company +you ha +x f +va ar +us amb +the fox +substance painter +stable ford +si mo +red light +pat ting +pag osa +off loading +of fu +magh reb +madele ine +lad der +l alli +jun ge +j ml +j ago +in alienable +im maturity +hell enic +hate speech +frogg att +forever young +flan ks +cruise ship +craig david +clu tha +choke hold +chick adees +carriage works +cam bie +ber tin +aw it +) [ +è ¼ +ãĥ ¨ +áĥ ļ +zi us +web fest +voice mails +tr and +stre gis +sla dies +rub ino +rafi q +quit smoking +mu gu +mo ira +mati st +master race +lin nets +law l +head hunter +gou ver +golds gym +games workshop +franci acorta +fil son +eth at +cro hn +col ander +car sofinstagram +bu ta +ble st +b wp +as occer +am bur +af air +uniof adelaide +un cler +u indy +tow nie +tin ib +the timmcgraw +tau ghtme +synthe sizing +strang ler +stac ia +shar ry +se ssed +rockn roll +re pubs +ping u +ork spolice +or éal +opp enheim +oo ther +o eb +nor bury +lang ou +her cule +hard wired +green bay +for ma +fol lette +fiercen ess +em elis +den ser +de urope +dd f +dan by +dag u +cul tist +chil la +brig antine +blood sport +amazon deals +al bon +Ù ij +wor thy +tse mrinpoche +tool bar +tiv itis +ti krit +te rex +tag es +south bay +simon sinek +sam bil +rupi ah +rough trade +ro saries +ra ffy +poz ole +polar ised +phylo geny +parsipp any +os de +ol dro +obey ed +na jaf +my lene +moo sa +michel a +mariob atali +juno awards +ju ez +jab ar +inst aphoto +getit live +foo k +fm la +far rer +eviden ces +ennis cor +dopp io +dele uze +crime prevention +cleveland art +cam elia +bl acking +be ac +bar illa +ar rr +and newton +yodel ing +ym d +vi enne +tur ban +tsvangi rai +tin o +tas chen +sli med +sex toys +schneider lin +savethe planet +rhett and +rebel de +ran bu +nau tan +mu p +leish man +le jos +ke inishi +jer k +jav ur +j pa +immol ation +h wi +gr n +fried kin +fc g +euph ori +esk en +cypri an +cosmo logical +comra de +cold ness +at erial +ash man +app raised +ai ken +ðŁĩ¸ ðŁĩª +ze bo +za hara +wa dia +unitedwe stand +underwhel med +touri sma +to pr +tam ron +ra sk +pri z +per spiration +pann u +me tan +lor demusic +ki pl +kar ia +joondal up +ilo vic +for bids +fi o +desro siers +darry l +christmas shopping +bajiraom ast +as anas +ane urin +alla gash +alber ton +acos metics +ðŁıĮï¸ı âĢįâĻĤï¸ı +ðŁį µ +ç ı +wha aaaat +w pix +vege tarianism +under lining +tam al +stun ners +squad ra +spondy lo +slow cooker +ski b +si wa +sh aya +s for +ru ts +ru iz +rhine beck +nec ke +nam ba +ludo vic +lc n +kl at +john jay +jin go +j fa +ij m +ha ki +go pi +gir van +gau ss +expat life +dont stop +cre denza +co stam +cat amount +car ville +cac i +bicic leta +abbot ts +ðŁij ³ +âĿ¤ ðŁĻı +âľħ âľħ +âĻ¥âĻ¥âĻ¥âĻ¥ âĻ¥âĻ¥âĻ¥âĻ¥ +y vette +wymond ham +wack o +tu ris +the tical +the offici +strat is +sin clair +simon j +ry uji +rest i +ra gamu +porth madog +page antry +over laid +newton ian +nbas ummer +mor phy +me drano +mass oud +mam aron +lu ig +lor can +longre ach +lon ey +lions gate +ko ga +k lara +ir ma +impe dance +hydro gel +hidd ink +gore gaon +fr ink +emb laz +dg i +dark fantasy +cq uni +cor pse +colon y +collo idal +che shunt +buck shot +boy ds +ba ab +amazon smile +ag waii +aday in +% ] +ye ye +white hurst +toron ado +thaw ed +swe eny +shri gley +rival ed +rit ts +play date +pep ys +pemb ina +pas olini +ori de +mit ral +mccau l +mar ies +mac bookpro +jarre au +incre ments +hp cl +green screen +gar dn +foun dries +doo o +dispat ching +ched in +char mers +bri ano +ble ak +be veled +aw omen +asi ancup +anar ancic +ahmad inejad +yeovil ton +will fully +water tight +water son +uv acha +standardi sed +smok ymountains +sm rookies +sha ad +se mble +scar pe +sb news +re mmurd +queen sc +phen yl +petter sson +patron ize +nest ling +mor ta +mis ssa +m elli +ly can +local ities +jo bim +i oni +home builders +har f +h sh +ga sification +fre snel +flu g +e ireann +drag con +dj embe +del son +config mgr +christ ina +ca ity +c moh +business development +baltimore sun +ani um +am adri +whot els +vp debate +tsu ki +tra ste +timiso ara +sti ffer +stat o +snoo ty +shi koku +reggae music +rede ye +pur cell +pro getto +pir s +perl mutations +par tof +oooooooo oooo +national girlfriend +montgom erie +la bi +kari ba +in ard +har deman +hal in +gurup ur +gu shue +ge stern +food bloggers +en ola +dy ckman +dee zy +d illion +con wy +cis f +ch inst +cb stv +black thorne +belag avi +az lan +ay oub +at am +ang ina +alumin a +ðŁĴĦ ðŁĴĭ +ðŁįķ ðŁįķ +âŀ Ŀ +zeec ine +z apper +wild horse +u play +tt c +terry crews +sun chat +spe ke +slu sh +se eyour +peshawar attack +pel ted +pe tul +omor row +nizam uddin +neur one +mi dair +mer win +megal om +mc duck +mariska hargitay +maine coon +kran j +kev ins +inter con +identi fier +hale akala +gro win +gre ssion +frusci ante +fo zzie +eric clapton +dri ppy +dor drecht +dip set +con nick +ban yak +anderson ville +an iso +acur ry +a afp +Ï Ĩ +wx brad +wd tn +un painted +tripp ier +travel addict +tar kan +super i +spiderman farfromhome +si ddle +shu cks +san ada +ro st +ra as +poe ia +ov ate +old bury +ne yo +n é +mor pur +mix in +k fox +ine tt +in ab +horton works +h va +go sden +fay go +er de +em h +dragon force +day e +davie ss +dark net +clo the +by ker +bi ggy +audi os +ðŁĵĪ : +ðŁ¤· âĢįâĻĤï¸ı +Ñģ к +whit son +vise grad +vel led +taste makers +stefan os +sm s +over do +nul l +ne wey +min uto +mil on +makeyour mark +mac chio +lor in +lor imer +ko bra +ke cil +jcp sky +it ag +ham achi +free ways +fo v +fl inging +dig al +blo gg +bel grano +bel court +autom ne +ar rr +ale mania +ad ow +ðŁĴĻ . +ë°ķ ì§ĢíĽĪ +young ins +with ington +us amy +un following +tunn elling +tam arin +tag us +super footy +strat en +spla shy +spec tru +smart doll +po ps +out work +of icial +nap o +mir vish +metro poli +me arns +mav tv +marke tin +m wal +levenshul me +l ho +know able +ke itel +jb hifi +interce de +hatch ling +go warriors +for duk +ex ol +ed to +dreamscape ph +dor noch +cou pero +corn el +competiti ve +comm ander +col let +cla pp +centime tres +ca is +bubba watson +bro der +bren nen +blit zed +ber als +ay on +au du +aru ah +ami able +aitch ison +a pop +ðŁķ µ +æĸ ° +âĻ » +ym ama +w q +ver tu +tribe caf +trans fusions +tit es +thereal dj +taitt inger +summar ise +shor ts +sbst df +remember when +rammadhav bjp +oco ee +ma sar +le cular +lar key +king fishers +kimjon gun +ke vor +k voa +hel berg +heal d +for success +en circled +dan kie +co fc +chain link +cb j +ca chet +b ams +australian army +all startrek +¬ë ¸ +zale z +zain imam +z anna +z ala +yuz uru +well y +un reachable +u len +tom king +tin ting +tin ny +tim bre +the history +sy sco +spr itzer +spect ating +sor in +sioux land +sher gill +shad well +sarath kumar +sang akkara +raes remmurd +quick step +night clubs +meal worms +me ager +le om +it ri +hygi eni +huss ars +goe son +gid get +ft n +fat belly +ed ta +de whurst +dann o +da q +cri sler +columbi asc +ch inn +brac co +blu emo +big ny +bexley heath +ber tone +: .. +! ðŁĴĽ +âģł âģł +à® İ +اÙĦج ز +ye hey +wool ton +wang sa +twit tter +tul low +toler ates +tik tik +suga red +star girl +south wood +skybetleague one +see ps +sajj ad +sad ar +ru ki +ri dings +pin tof +pi eta +octopu ses +non invasive +n kt +mobo awards +lat ur +its whatwedo +is of +is ba +ij esse +icha el +ice dogs +hum vee +gl sen +gb chefs +frac as +f á +en vi +ea v +dw m +dori an +dar n +come down +cla si +cib sunday +c dot +bu tuan +bob weir +am ys +agar za +ac ri +åı ° +ãĤ¸ ãĤ§ +ал ÑĥÑĪ +y agi +xia omi +wizard world +vince staples +the walkingdead +so ke +sil er +sheff docfest +rhe on +retro wave +rad ke +r mac +pu tit +nissan usa +mi sper +mawra hocane +lindsey vonn +lam an +la hinch +hu uu +fu ad +flam ini +fa hy +f cu +de aton +cy lon +cr ê +chi kun +chi bis +cel is +calri ssian +cab ra +bi let +ay esh +at tock +ade wale +ú l +z uki +vi dal +thur so +sun anda +shake shack +robert son +red un +re balancing +rag ana +psilocy bin +parti r +ny te +mer rell +mel ane +matt smith +manay unk +l leyton +ju mba +gw b +guy ssss +flori dab +e bron +dout zen +cotte sloe +cavall ari +can ti +bre ll +bi gil +bc ps +ba is +ah ha +. + +ðŁij¼ ðŁı» +ë¯ ¸ +ÙĨ ج +yak ov +word camp +we ssels +uk hashtags +two some +trac ee +tag le +sun bury +spin ney +si mas +shar at +sep tembre +ran some +plexig lass +p ng +ni bali +mor tel +mahar ani +kur nool +kr qe +kl u +kick backs +ket u +karan singh +kar tini +kaleido scopic +international womansday +ice e +gre ll +ge tolympus +forthe cure +fc s +f ends +ev angel +estu aries +chro med +cer ny +catalo gued +canad as +cab are +bri mley +b ty +acom ms +âľ Ĥ +zo vic +tar ia +stereo gum +space balls +single market +si wan +satyar thi +sar rain +rela is +re stur +parachu ting +palom afaith +nike id +nau ts +mal ti +liber tarianism +leu co +le mi +jung frau +ijesse williams +fri dakahlo +fawl ty +edint fest +deli verer +bre ra +av x +amin aj +ðŁĺģ @ +çĻ º +ãģŃ ãģĵ +Ø ¬ +алÑĥÑĪ ÑĤа +wal led +tribu taries +tobias menzies +ther lands +tax co +sc ie +repri manded +pud gy +power tools +polyglo t +poll ster +pe les +p soe +or lean +moroc can +men no +mamaron eck +loril oughlin +ko kop +keepthe ban +julian castro +hand hel +gull wing +gro k +gi k +gen na +ge ir +g ddr +der ringer +d sei +cre sc +cor vet +clavic le +christma slights +chhat rapati +chap ters +can iglia +butt ress +bhu sa +( < +ราภĦา +young justice +the athletic +success story +so pot +sav y +rosne ft +ri pert +rhi mes +pw me +pol lok +nirup am +na deau +n saa +n eng +mu sprime +melan in +mag ners +khrush chev +intrac ranial +hy ori +holiday makers +goe bel +gh el +four somes +fal well +fa sia +ev ski +dw g +coning sby +cinemathe que +cbstv studios +cardo zo +cameron mustgo +assassinscre edodyssey +angeli ka +aly te +ðŁĶ ħ +ðŁIJ¾ ðŁIJ¶ +ðŁıĨ âļ½ï¸ı +ðŁ¤¦ ðŁı»âĢįâĻĢï¸ı +à¸ŀ ร +w end +viv ah +ure thane +un funded +ul ar +u ol +tur bans +tal avera +ta di +synchron ize +stil well +reve ille +re ye +p co +o ac +mono gamous +love bird +lav any +lang ar +khair a +hay lee +hair by +food borne +fix the +dru ck +docu series +con fed +city hall +chu bby +calf skin +broad gate +bit urbo +be eline +barr head +bar ti +amar th +ãĤ« ãĥ¡ +youcan doit +wh acking +vrin da +vam si +unra veled +ty kes +toy z +toron tomar +ti zing +thor sten +the gef +ss bu +sou dal +sab tv +s only +pollu tant +pedd lers +pe geeks +pas qua +out fitting +nup es +minute beachclean +mb ar +lin oleum +li sk +lar acroft +kel apa +kab uto +jun ji +jel ani +home depot +gla swe +gla dio +ger ber +follo will +er ato +ene ws +eager ness +cultiv ator +cir c +car lil +cap rio +c ge +c fg +brand in +blu eraiders +bin das +bar g +attribu table +ðŁIJij ðŁIJij +ðŁIJ¢ ðŁIJ¢ +wal singham +w da +vy pe +sushmit asen +sp ana +seven th +sam path +rhe ma +paris nice +neal on +mu lat +mbapp é +let ons +le tra +ki sha +keye tv +keinishi kori +k hari +ji mo +honor ably +ggg boxing +ekur huleni +cbc north +cathay pacific +bu ba +bri die +ax y +ani st +andre scue +americ ang +ðŁİ¤ ðŁİ¤ +wel ds +top con +tame impala +stock hol +sta edt +sa em +rin os +religi ou +rekind ling +pr an +porthar court +play az +photo de +opp er +nl tweets +mest alla +mean ey +ku rian +ivy league +gr illin +fe kir +dig itech +dau x +chro mic +cap s +bun dle +behavi our +bas so +bang i +ar kle +agric ole +aer is +ðŁıĬ âĢįâĻĤï¸ı +ðŁĮ ¯ +âĻ¡ ~ +well park +wat tage +tor menting +taun ton +tablec loths +sub machine +su eur +sonus ood +sate en +sam claflin +sachin pilot +ro p +re plant +om gggg +od do +nor dea +nev ins +nashvillec mt +melbourne city +mari ucci +mangal uru +lind ell +jake paul +i abc +hurt ling +hood lum +h re +greek life +grapp lers +global goal +gaz ed +franchi sement +fo glio +el ms +delo ach +cu uu +coupero gers +commercial property +col um +canber ra +bob tail +blan kie +bet victor +arrow verse +anni ston +zu bin +tv onetv +t mea +sme thwick +sme e +small world +skyl er +rose mcgowan +rit ers +re productions +phleg m +nonleagu ecrowd +nandam uri +nab ila +mid shipmen +mal ham +live chat +lar ain +l ancing +ki b +inadver tent +in life +higg inson +handsom ely +green lee +goo dy +go tz +g posers +fd m +encyclo paedia +en co +doug benson +d tw +con tending +circu ito +char ly +chaffe y +ball room +au ster +arch ies +a him +££ £ +ye oman +x ls +vo da +un conference +tu rok +to dt +thro yd +the moon +tal ai +symbol ically +swar agini +suffo cated +success ors +stream lines +sh aked +sei fert +ri so +provi dence +pric eline +policy making +ply mpton +phin ney +ot ar +of ootball +much h +living room +ku cha +integr ators +great dane +gg ah +ethi er +el roy +dra is +dr mike +de seo +copen hagen +caric aturi +bill kaulitz +benadry l +ben thic +bat as +aminaj mohammed +ys i +yaa as +tø p +swoo ped +sun shade +stan tial +shre ve +sel wood +se hr +sad dening +ri op +rep elled +raj ak +promis cuous +prioriti zes +platte ville +ncaa season +nbasummer league +nak shat +ment on +lie der +kare t +ka al +jav ad +ital ics +id sa +han bury +fifa u +fass binder +dj paulyd +dispon ibles +ch ex +c wi +blin ked +bio hacking +as ur +âĿ ¥ +wh ky +uttra khand +tri bble +th ak +ta ik +sin city +sco ffe +saul nier +riot ous +ric oh +raw k +rac s +olympic torch +novosel ic +news beat +mit ter +mir field +may bin +ma har +lymphe dema +love good +limb ing +lemon twittor +kle en +k sat +img models +hil aire +ha spel +gel ert +ffr f +do se +dick enson +dicho tom +del bert +dc tech +commi sion +c slewis +b ink +aur yn +ar ica +abyss inian +ðŁIJ ĵ +æ ¢ +wicom ico +w bls +vesti ge +upad hyay +un suk +ty pist +ti en +ta kuma +soap awards +shi els +saucep an +salt ine +rut land +re eland +pre dates +pp o +plac a +os asuna +mill town +mari huana +mar ro +kush alt +kis a +john j +jess op +jailavak usa +ino ble +illumin ate +heav yequipment +h ry +g th +dim ly +de sales +chin atown +bier stadt +beat les +aser rano +ab ly +ðŁıĩ ðŁı» +ye eye +vid alia +u ip +ty ron +trump ism +trouble makers +total itarianism +the arjunbijlani +termin al +ted med +sp ack +scon ces +ru ber +rapp elling +pu ppers +plan tronics +pay s +mrs funnybones +modu lator +mill y +mandi ri +m smar +laksh mirai +kam ba +kal lie +ka un +k ome +il ab +heat sink +hawk ish +half moon +gir ders +gand ang +di vis +desig ni +cl dr +ci aran +celo teh +camp agna +bestofthe best +bar ilo +ban dido +b movie +a ala +ðŁĺį ) +ðŁijĪ ðŁijĪ + ¬ +wil bert +wi at +viking river +us ine +tou cher +t ills +sque amish +spe e +shapo valov +sgo ing +saty agraha +quin cey +pryn ce +pac i +new mont +mis ing +mi gan +metall ic +melk sham +mahon y +luc kier +lets goo +lab h +la vey +l ton +ke men +jabber wo +iphon ec +ilay araja +gi vings +fatbelly bella +ever rr +engv sl +eli ze +dre e +corr ales +conjunc tivitis +chang kyun +bunde swe +bis set +bio chemist +++ ++ +ðŁ¥ Ķ +Û Ĵ +ı m +usp hl +tow cester +sc ircle +sc apa +sam yang +ro shi +rin jani +rho des +raj cheerfull +projec tiles +oul ster +o hai +ne ce +nav jot +nat suki +mark ronson +mad dog +la hori +ke agan +inter woven +inter league +i ban +hof stra +hi k +goleaf sgo +foli gno +ex y +em r +du me +cyber jaya +crew man +cor ben +cli ma +chapp aqua +cel ly +bo ya +bi fa +bank s +apo lis +an kar +west cliff +weare thepeople +vo il +uygh urs +ur kin +tt le +taras enko +silver plate +sign ment +shin obu +set ts +sanjay uvacha +refor mist +or puw +nuev os +net care +na thi +morri sey +me tuni +li vy +laverne cox +ky te +kir kin +kei ji +it sp +irr ationally +in africa +heter onor +happ pp +gram edia +goose berries +du pre +cu bus +candidat ure +ca eli +ayotzin apa +as adu +ar os +anderson paak +๠Į +war n +v cats +under pins +u bm +tj maxx +thir deye +tele graaf +tassi e +slug fest +sla de +sh eng +rolls royce +regi ments +pheno types +petti bone +palin dro +pabst blueribbon +old car +nt sa +nepen thez +nat ali +mo tom +mal ky +la peer +kreuz berg +jyo ti +it rtg +invigor ate +honey ed +f ti +emory university +duck man +doo dler +direc tx +ding le +cloak room +chat swood +bur russ +beli al +anan si +am s +ak k +ack ley +:) ))))) +ì° ¨ +âļ«ï¸ı ðŁĶ´ +âĹ Ķ +ye tte +ye ssss +x us +w sm +vic en +v um +s link +puri fies +pleas an +nic olo +min uten +maker faire +ko blenz +isi baya +inter activity +ing club +hindo stan +hand prints +grac eland +glee ful +gi ff +gh id +father ly +equ alled +dissimil ar +cow art +bra sov +bou te +apol los +angle sea +af cw +ðŁĺĺ ðŁĴĻ +under utilized +tosk ana +timbur ton +suk kot +stu por +sou s +slu t +see mar +r mi +queen sboro +pur pler +prophe tess +p tion +or ical +mou les +lun den +little wood +lindsay ell +lid ded +langui shing +kel o +infec tious +hot box +fran sisco +fck in +evil geniuses +drum kit +drug by +dar mian +cop se +conson ant +cann elloni +can tera +bl ore +bab asaheb +** * +ðŁĶ´ ðŁĶµ +ðŁIJ¶ ðŁIJ¶ðŁIJ¶ +waig uru +w no +vincent e +us rowing +un m +toron tor +till and +spra yers +sik ora +share goodness +sal ama +pi galle +pe co +official slc +noi sey +n aci +mus ick +mo en +lu dgate +kc chiefs +jit tery +interior styling +inter lo +ib d +hum safar +hick am +her riman +harvey nichols +gu ttenberg +fire fight +fa hr +emi lee +eliza jane +doo han +disney onice +der showitz +denne hy +decep ticons +d ml +cn ty +clim ited +chess board +capu ano +bol as +birmingham mail +bird fair +back lund +b anca +apu lia +anou shka +ago ddard +. ðŁĺĤðŁĺĤðŁĺĤ +ðŁİ¶ # +ãħ¤ãħ¤ãħ¤ãħ¤ ãħ¤ãħ¤ +âĸ « +ठ¼ +worka holic +vo xbox +u daan +tur ds +tu ma +tro tz +suppor tn +stock photo +sp itt +si se +sawyer frdrx +road tor +re takes +rapp ahannock +picnic king +para pet +p nas +nordi ques +ni ente +mil agro +marit za +lu ces +lossi emouth +le ef +kyo ko +ko ppel +j lo +infini x +hu bei +godz ill +equality forall +en gulf +du uu +dopp leg +d als +bro ss +bounce back +bi weekly +bh x +bab ar +an gra +. ", +ðŁĴĥ ðŁķº +ìļ© íĻĶ +wel kom +w ya +var ghese +tri anon +time smag +tib betts +therealluke vans +ten sor +sel vam +sch open +ri zzle +re invest +pi ya +parasit ology +one one +om gom +neural gia +nat omas +mini game +mark warner +local ised +ll l +kat vond +immigr ant +ho ho +gra ig +freak ish +for mayor +fif pro +dra wl +art sy +ap w +al tec +acron is +ãĥ¼ãĥ ł +س ت +yggdra sil +upper classmen +un mask +tro ph +sk itch +sho shone +scrib bler +roof ed +re ining +phill is +o zy +mur ty +mind storms +liter atura +left field +l tb +ken ichi +k atta +gal aga +frankfur ter +fr t +fing az +down plays +dhi kr +cru s +con ning +chad li +call backs +apostro phes +adorn ment +? ðŁĺĤðŁĺĤ +é¦ Ļ +za ade +yeswe can +yellow claw +ws bradio +whel don +tori als +tip is +testar ossa +sweden in +su meet +sl sc +red skinst +re agh +nyc schools +multi racial +lu tron +ji va +hand brake +go tch +glo bies +embroi der +e iner +disney side +darke st +curragh race +cur ative +comedy night +chu chu +bur l +bry on +birch all +bar wick +b nr +aw y +an tro +americ ana +all ers +ðŁİĤðŁİĤ ðŁİĤðŁİĤ +ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ +ye aah +y ancy +water deep +un washed +uk as +tri alist +takor adi +sun tec +stanford med +shr iner +sa en +pg dm +pakv sl +of r +obrig ado +o tho +mini mums +langh orne +lam m +ksen ia +k and +je h +inyour phone +herman as +han sen +guadal canal +gru mbling +gor dons +friendshipin apicture +flabberga sted +eye witnesses +eun jung +drake bell +den ote +colle yville +cloud foundry +che sted +camar go +ba de +aren ado +alber tine +air soft +ç Ŀ +à¸Ļภ° +ଠ¾ +year ns +wire grass +thorough fare +ter idge +sco ped +s bay +rigo letto +ride out +reel z +quin ton +po ston +pa vi +may b +mar ae +kh lo +jun kers +jc de +j evon +ili ans +id fa +hub cap +hop kin +hig don +giac chino +exter nal +dn n +cru iting +cla wing +ci i +brav ura +bra z +beat a +anti fungal +ðŁĺ´ ðŁĺ´ +ðŁĴĻ ðŁĺį +zay to +tomking tk +thi am +theri ver +stu ber +strang ford +scent sy +ru la +polar bears +pen ess +no wor +marc maron +mar com +maker spaces +jor a +johnson ville +je te +ha ben +green army +gan on +fox hound +e gged +do logy +cer ys +beat itudes +bag i +ba sia +ati eri +acne studios +ðŁį¾ ðŁį¾ +âĻ Ķ +vampi rella +tri ver +th of +tee j +super cool +stabil ise +sit down +san ju +sag an +sa cher +ram bla +rag thakur +ra damel +puffer fish +play school +or pen +mel lows +long ingly +lin ney +la xton +k cp +high five +geom atics +ff vii +ent as +ched dar +ch azz +ceri dian +cardio logists +cap size +brew dog +bli the +becc les +as ko +acas sandra +yor o +y im +x lf +vijay television +verte brates +trans net +trans its +tik ki +the ahl +star lite +si achen +she reen +se el +saty ajit +ne ena +mam moths +kingsof leon +kat rine +jit ter +jd morgan +homo log +hl th +gro en +getz laf +g town +fluffy basil +fin ing +ex ofan +ever rrr +du ffy +dra bble +dis arming +de mian +cre oso +chro mia +casi ancup +capit ano +calgar ians +c ve +berl anti +behavi oral +bacter i +aw ani +antic o +ak oz +achi o +ac aster +ðŁĵ Ĵ +à¸Ńภļ +yar borough +wo je +wit che +w mp +v ire +ta kings +sp ats +rise up +quiz master +pah rump +nu mp +nikol aj +lu ar +kabo bs +ii hs +horse sho +hi spani +foun t +fl x +fif tieth +e gel +cu gat +cruci ble +close up +cate ch +cail lou +bott ura +bla ster +bally mun +as olo +array it +allo gic +... âĿ¤ +ðŁĺĤ ðŁĴĸ +ðŁİħ ðŁİħ +yyc traffic +we were +unic redit +un ashamedly +ta ppa +stre at +sar oj +roo ve +rhe um +red der +pir amal +ohio statefb +na sik +mo tha +mac ke +lo han +lapd hq +ip n +ing al +hyper local +hot point +honey pot +fli fe +fel ine +desper ados +dark stalkers +cth sfb +ch ora +cas ale +ban ar +and ering +aim high +adrian aci +adrianaci oci +aan am +!! ðŁİī +ðŁİī ðŁį¾ +⼠³ +vango gh +try p +st ice +st f +south bend +pl ats +pare sis +o sw +mon tra +mc on +lac ri +kho ff +inevit ability +high speed +he ena +hand rails +for tresses +fi eg +f wb +ellen sburg +dre th +dil g +cyst itis +chri sta +chauffe ured +bay lis +ahmadi yy +aerop lan +ðŁĩŃðŁĩ ¹ +ر ÛĮ +windy wilson +who doyou +war by +unfur ling +un ia +to action +thrift shop +the fat +substitu ting +smik kelsen +shape ways +re organize +re appearance +rad as +planet ary +p fg +oosthu izen +naz aire +nam u +moor ings +men a +mar jan +ma kha +lin dau +jay de +isle worth +hope lessness +hol combe +hel wani +flipgrid fever +env hist +ele f +cuteness overload +creatin ine +bun c +black sea +bhumi bol +beef heart +ban ta +az evedo +anton off +!! < +âĸ« ï¸ı +touri stic +tol man +syn ch +river land +rhettand link +over heat +od awson +mul gee +long form +ku k +intermedi ates +ink scape +in habits +ici de +hi stop +financial review +ef teling +door knob +dent su +de compose +cur ritu +co to +camp in +camar ines +bt fc +brendon urie +brea keven +bo iii +be ker +all sven +abhiman yu +a iders +x li +usa wrestling +uni west +tuscu lum +thar pe +tam ra +starbuck suk +sque aker +scor n +sa iga +ra shed +pra g +pla ice +pe ppe +pat audi +nr lgf +noplacelike home +mir ra +michel ada +melaniec music +lom u +live mint +la fontaine +ke at +je mber +it ai +inter nationalist +inst ago +i etf +ho gar +her an +gonor rhea +giam atti +fa jar +der mpath +den tro +dal y +co en +bigg a +anci en +âĢĭ # +yo enis +womenwho code +win elands +will mar +tu gging +too too +tai ji +stru e +sos borne +sk h +ring thebell +pro wein +pr out +play boi +pe on +pash up +nik off +mo town +mo hini +l alu +kar ne +kal ash +k rsna +institution alized +hay le +gan ito +flesh light +ess en +er ring +dram atic +dn as +di ski +darlo biz +crack nell +catholic stl +bsy bjp +bra ylon +ayian apa +(^_ ^) +âĸ¬âĸ¬âĸ¬âĸ¬ âĸ¬âĸ¬âĸ¬âĸ¬ +à¹ģภ¥ +wyck off +wu er +unison tweets +thermost ats +te stu +supersoul sunday +st annis +rim fire +pocket book +po ko +peace haven +music life +multic hoice +mi kol +mari os +lovel ocal +lake district +la ib +ku mail +kis smy +hero ileague +got thard +global edmonton +devotion als +der ik +d wane +crimin alize +cos entino +coach man +cer rado +bu mber +bo sse +animal lovers +agen arian +- < +ðŁĵļ ðŁĵļ +à´ ° +zig gur +winx club +wh yyyy +w day +vibr ato +ten news +te stino +taji k +t sp +t intern +si sal +rece de +po vic +new product +new life +melt water +manag er +malacañ ang +make peace +laz are +l our +kanhai ya +k ø +jor ah +holm del +ho wa +height en +head sup +handsome st +halei gh +green bank +fo st +en zo +downing town +dat sik +cu zco +craigh ead +clare balding +cal poly +brac o +biglittle lies +arte sia +af fluence +! ðŁİĦ +! '" +ðŁĽį ï¸ı +ðŁij¨âĢį ðŁİĵ +zz ang +vulgar ity +vom usic +tattoo artist +tab b +sos venezuela +somerse twt +shre dders +san ches +random izer +par tri +over draft +o shima +n aren +mc ateer +may ers +lund berg +lo ook +kni pe +inthe sun +instin ctive +in got +ilm fest +ilk ley +hoo ke +fer ne +evapor ates +eno y +dash ner +clamp down +chil a +boys and +ash mo +antico agul +": {" +! ..... +wre xham +wr o +top sham +til son +the istic +str unk +stell aris +sn app +screw sc +say hername +pi thy +p alli +osoph y +nor agami +ne phil +natural history +music ales +mls allstar +me ant +mari ju +ma ree +lovel ock +lionel richie +lawrence burg +kl int +k offi +jo is +jair o +ith as +is v +ine twork +in defensible +ill aria +hy men +gunder son +guan ac +gru bbs +gorge ou +golf week +go hil +gar ces +free tibet +fol a +exor c +emer aude +el abs +curios ity +char isse +cay la +can avan +brindi si +are view +antof agasta +am obile +ah saa +:' )) +wit tman +whi anw +whianw amos +watch band +the whisky +stone leigh +sher wani +sand pit +sam ira +ru ane +pmo i +picture perfect +p q +moor gate +mi kare +mass af +mandel brot +man nie +long bow +hel mond +ha ero +g ning +f áil +el mar +do tus +dish ear +di endo +cron ut +chol la +centime ter +bro adest +brigh am +bor rower +bodybuilding com +back ward +ak ah +âĿĹï¸ı âĿĹï¸ı +zor ba +ww mt +wheres mum +tru enorth +transcrip tional +tan kian +si mian +regin acassandra +pre cept +pin cus +pat ella +nbc dfw +my boy +marco polo +kar ylle +j di +iv or +i eri +good nigh +go el +geelong cats +engel bert +din ny +de ary +dae jun +ctvo ttawa +bur khart +boser oy +bel haven +be lew +bbc theoneshow +barbo za +aston ish +ari f +amp ong +ab sten +a hou +ðŁ Ħ +youknow you +ye pp +wy c +w kn +v anni +un made +tot ton +thre ef +the script +south am +shepher dess +set tee +ri fts +port adelaide +nocturn al +n ly +live s +kish or +issu u +in um +helf rich +hay wire +fe tti +fau stino +es ma +er mine +end tb +el ake +dwi vedi +dani pedrosa +cross road +co lima +carac ol +bow a +bbcradi oulster +bab bitt +anesthesi ologist +am illion +air race +abu elo +ab rand +ðŁĵ ¼ +ب د +yan mar +white people +we ts +wal do +w sh +voting rights +u albany +tiny house +tar zana +tai ba +switch back +stitt sville +son ico +si v +serv icio +schopen hauer +ronit boseroy +ril o +re aps +popul i +playadel carmen +plastic ine +paladins game +pal ang +over reacting +o ie +moore music +men lo +maxi me +kuruk shetra +kathr ine +ka ve +k ary +hu ynh +he igh +ha rena +en daal +eli ya +eclip sing +cruick shank +cra dock +body builders +bas anti +anti bullying +ali da +ab ac +, .... +ðŁįª ðŁįª +ðŁ¤· ðŁı¾âĢįâĻĢï¸ı +ภį +yello whammer +y van +world kindnessday +visit greece +u men +straw bs +stom ps +star light +sn k +sign board +shi man +sham anism +scav ino +rue ben +que ira +old house +odi ous +ny on +lam pe +es of +dur g +cripp les +cra ggy +col inas +cnn sotu +braver man +bil ton +back yards +b ffl +amp abay +af front +ðŁIJ Ĺ +å¼ ł +ze ig +xx xi +wo v +wi geon +vid ler +vi pin +ver acity +under developed +ul ata +tu tel +to omer +thor burn +star cast +spo iled +spear man +sound z +she etal +rever ting +paraphra sing +neva eh +mati es +i quique +hy p +hi ragana +giuli anarancic +gigaf actory +free sia +dl day +dance party +crystal healing +chit r +cal stat +boy den +alad din +ak shara +abc family +a edes +) = +( ): +yan k +watercolor painting +us afa +uniof york +twitcho sf +toronto argos +the brian +tad caster +su kumar +stv news +sper mum +re vi +ra ssa +progre s +pri sh +pon dered +ophon es +nom ar +news reader +nat ic +n ll +mil ik +mil ers +meni fee +leg co +l dot +ki djo +hor ley +gulf of +fi shin +cycli sme +cro re +cat dog +bu sing +bla q +bike dc +barra gan +ane ously +ali zed +ðŁij¸ ðŁı¼ +е ÑĤ +wol ford +warrior cats +w dd +thur low +thor sday +ter mine +tepp any +south devon +sky football +scraw led +ol k +ocr acoke +objec tif +o thman +mu gger +mean er +kay e +joy division +jayson dmx +iz awa +ir vin +ing olf +hun ker +huis man +ho bb +ha py +go jacks +ferry man +fern ández +eth no +don aire +deplo y +dan neel +co sette +cli mactic +ceph aly +bro chu +bon ec +aventu ras +ap mas +al mira +æ·±å¤ľãģ® 羣åī£ãģĬçµµæııãģį +ಠª +whit comb +west lands +var ina +tx la +tit el +swee ting +sun cor +sper ling +smith town +shoe shine +sg news +publisher swkly +per ver +p nd +ny ssa +nie man +ngr senate +monad nock +l pl +k ve +just acard +indi ra +hygieni sts +her mand +har dison +ham elin +ha agen +guardian news +grass root +gn ine +forthelove of +er ight +emblaz oned +dr ons +confront ations +cell ino +bar aboo +. ": +ìłľìĿ´ íĻ +vibe fm +vi krant +tu tt +torch bearer +t ve +sw elled +smo kie +skate boarders +shan ties +sean astin +paw ty +ow ay +nis sen +neuro imaging +music lover +mor ais +mk v +mi kan +mc v +mar j +maison ette +lt p +kn x +kagu ya +james roday +histor io +herni ated +go tg +fur ore +disser tations +char la +bureau crat +box score +bh cosmetics +all on +" } +! ': +ðŁĶ Į +zam brano +ymm fire +witch hunt +whoo pee +whang arei +w pf +vs was +tra ppers +tom mi +tiger nation +tere za +tab by +stevie wonder +si uc +sen si +prilly bie +pp fa +poindex ter +pnas news +pern icious +par men +oba id +mun ich +mu di +morpur go +monc ton +mi red +licen see +kis sables +karansingh grover +josef newgarden +j ke +gif ts +gad sby +fle shy +false flag +f online +del ac +cim goi +chut z +cattle ya +burn notice +bulldog pride +b dx +ar ula +ar q +ar da +anwar ibrahim +^ ____ +ðĿIJ¢ ðĿIJ +ìĿ´ ëĭ¬ìĿĺ +âĻ ł +âĢ Į +áµ IJ +~ . +wood sman +w sav +vel t +u dd +thegame awards +solo ing +schlumber ger +rox ana +quaker town +qu eda +pan tan +oz awa +ner vy +mik ita +mi jas +lang lois +kare lia +hutch ence +empire magazine +emeraude toubia +el ad +bott oming +aven u +arm strong +and ing +and az +ðŁij¼ ðŁı¼ +ãĤ ļ +à° Ł +zucker man +y th +wolve sup +winsor andnewton +w ati +so ane +sin ensis +ri les +re a +ra wr +pub schools +por o +ostric hes +ok onom +oceano graphic +oc cer +o oms +novo sti +ni dh +mcmur try +make re +let tieri +kur upt +kil winning +imp ure +ha jar +gun ya +git lab +g shock +edo ardo +dam p +chri sle +cause were +catahou la +car pus +c pe +bull horn +beck erman +bath es +arto flegends +amandak nox +ad ur +aber avon +ç ³ +yu cat +thorpe park +this way +tam p +tai ka +shu ga +s bag +r ving +pre forming +of amily +oak leigh +no fly +ne ven +mu ld +minim ising +lea key +law es +ky w +kok kin +kar nak +k lg +jessic ani +hel ms +har ini +har ika +gur preet +gun rights +grapp ler +eil idh +e one +con oc +casser oles +carol kirkwood +bost wick +bor laug +big boy +bell ing +armb ands +alamo gordo +!!! : +ðŁĩ¸ðŁĩ ® +íĺ ķ +was sail +ty vm +tu la +ten uous +su pr +str oup +soc sci +save timeless +sau ti +ri al +re positioning +re dra +rd pro +ogun quit +ober hausen +nas p +mtv awards +mo fa +men shockey +me to +mar cha +mar ah +lyric ism +ke qiang +iy ya +high key +green ies +grange mouth +geology page +f hl +de march +conver tibles +bron chi +bilas pur +az rael +ann ular +an jana +ambi ent +albu feira +ìĭĿ ìĬ¤ +w df +viter bo +tesser act +te ad +subsidi ze +spring fest +safe house +rich y +re em +pleasee e +pin ko +pen na +pe cha +often times +nuev as +ne id +n ary +n andy +mi sprint +man illa +lav atory +lakme fashionwk +la sci +jy p +it ai +international isation +induc ting +gom era +george strait +gang tok +eri g +enz ymatic +dog mom +di ppers +c ti +break beat +beau teous +bal eares +arch on +appalachi an +ak is +*-- * +âĿ Ľ +whit ep +whatsthe difference +wel ty +tipperary gaa +the boyz +tere sted +svit olina +suppor tw +sou ped +se ann +red hair +pino chet +oxi des +o dc +na hi +musli mb +mein l +mcman aman +mary borough +manish paul +left behind +ko zak +kil mister +kamp en +ing don +ic cb +hoar ders +goss age +flu min +fel is +ebb w +del fin +columbu screwsc +bum bum +b ents +ac larke +ðŁĺĭ # +ymoun i +water borne +walk den +va is +umb ridge +traste vere +they come +thei et +ten zing +ta as +scriptw riter +sam ra +rah way +psycho billy +police dept +pi pped +nc cu +motor trend +me shed +margre the +mar wah +ma tha +ly re +linna eus +libre office +ke vents +ir raw +ianu ragthakur +i vert +he ren +han well +ham ps +guer nica +go td +emelis ande +du ddy +dan ae +cr ampton +brian eno +blu t +auto immunity +ðŁĮ´ ðŁĮ´ +visit oslo +sain thood +sa chi +ru mmer +roblo we +prohib ited +pre ach +pepit o +oun tains +on dre +ob ello +maz in +mal ka +lead som +l orie +kohin oor +jhon ny +jac i +ir b +ip bes +hyperten sive +hu ri +glow y +fifam obile +fer dow +f gf +epic entre +di ao +del man +cc bc +bul mers +body armor +bett any +b bi +ay sha +apic ture +am all +yo shino +worldof dance +work tops +vic olu +vicolu dovico +ver adio +toy box +to se +roe hampton +ro ymouni +ren nial +rath mines +ram al +photo realistic +pawn shop +p dd +mo styn +micro electronics +lef th +kiz una +itec ture +hu ddling +gre nadine +giuse pp +fa khar +do tt +do decan +diffu sed +debor a +co zza +cho kers +childri ghts +brom ley +blue andgold +bh of +barbra streisand +austin carlile +ar la +amoun ting +alo ves +al vis +uku leles +susan ne +stun ning +stok ely +solar system +shear ling +se ge +sarang hae +sa kh +rv v +run around +roh de +receptac le +reas signed +quadrang le +q om +pur na +otter box +mck it +mary lin +leigh centurions +lee du +le xx +la pak +king kong +kil more +katvond beauty +he ta +grave digger +giving tuesday +enz ies +caul ker +c vp +borrow dale +ar ket +an esthetic +âĻ¥ âĺº +xzi bit +wol laton +wee se +uy uni +turk ington +tar kin +su rest +su darshan +ste ach +smoo t +smo tor +sa wh +rising stars +ripp le +rescin ded +re ak +rale y +r alu +plac ings +or vieto +ky ne +ko ren +k ure +i spy +hon neur +fire watch +es sec +du rai +deadliest catch +datasci entists +credit card +cox swain +corner backs +colin firth +bio tech +barilo che +ave e +ac ep +x japan +wr dw +world heritageday +wis sam +wheat sheaf +water marked +viver ito +town end +thro at +thom tillis +st lv +spu rious +sig ned +sar c +s who +remem br +press ings +oss ington +newsma ker +nar ang +liberal aus +li ffe +kal as +gre ati +ga jan +functional ities +fi ord +esper anto +edu tainment +der rida +dat t +com pare +co terie +chakravar thy +car vin +cap com +cam borne +bode gas +ben no +anir udd +an ted +al ula +al liston +zz top +yi annopoulos +wheres the +wee vils +watch the +visitgreece gr +tru ro +tram iner +thi stles +thames mead +tejasswi prakash +super b +sumit omo +stru st +shi vin +sh v +señ orita +sd schools +ro or +pom elo +partner ship +om ata +nad ja +motor boat +mi guna +maun gan +le land +joke ster +flat out +finish strong +dami ani +cru gby +con serv +chang elog +castle town +aw r +austri an +amotor show +afern andes +ach ei +ðŁĮ ij +vic pro +transduc er +tr ico +ti ro +tennews qld +talle res +sam aha +salt marsh +rou ble +rie woldt +regular ity +re dribb +raven scroft +public radio +nwo su +nodu les +mark hoppus +kur sk +im practical +harrison ford +dugg al +drau ght +col aba +boo kie +blu ffing +bis ch +bier mann +beatrix potter +be wick +basil isk +ashwin ravi +ar isa +alz association +all dog +al pen +abnormal ity +aber uni +tugg lenation +tu te +tranmere rovers +thermom eters +stin iog +ss oci +rosel yn +ra sul +pho be +pan aji +orange y +mango steen +kun ing +ke im +kady rov +jupil er +iti zen +industri als +ho pps +gre ca +gal ina +gaf fa +fon so +fast ened +fari bault +f mm +comment ate +cli matology +ci g +ba sten +austri ans +al bo +adi zero +ac en +!!!! !" +ðŁijİ ðŁijİ +ðŁĮ ĥ +ลาภĶ +wis sa +what would +wer ther +was son +vig ils +ud dy +tt as +tri alled +st annes +slike us +sli mer +sal ot +relin quish +rap star +ram zi +pas ch +ou chy +oc l +ngoron goro +new ye +michael rosen +meadow bank +levi ed +lea side +kr zy +ke uchel +j ita +ho sta +hay dn +han on +gri stle +girls night +evalu ator +ec afe +dylan n +did that +ci pe +chefjose andres +bud va +bu e +be hati +bailee madison +au rel +asynchron ous +af casiancup +ðŁIJ Ĥ +ðŁİīðŁİĬ ðŁİī +ãģ Ļ +à³ ĩ +Å Ħ +zen trum +wil len +water ston +vivo azzurro +universal studios +transplan ting +top coat +ta kata +she mar +sen do +sam paio +r dt +port moody +oneok rock +ny ah +nouvelle photode +nor mie +nan u +mic rons +le fevre +key note +kah neman +just a +jung frau +interro gating +glen brook +em is +ed ger +dista steful +dictionary com +de anne +dark ness +d ago +cé zanne +craft manship +cor des +chil lax +cash app +card stock +blur r +bb src +ayut thaya +arkham knight +acol lection +ac cia +ab ass +; __ +yard birds +w ice +un assisted +the how +t km +sud bury +see it +sadhguru jv +pon tar +pol son +per fusion +palak kad +nor iega +non stick +new supdate +nature news +mac ale +kin ser +julianne hough +joseph muscat +j pc +instac art +ilike italy +i dos +fri zzle +form alities +eu fau +ep isd +chap ati +bul on +bu ya +bla den +ben n +ar ita +appar itions +ant in +abed in +z ot +verdic ts +ta ffer +solan ki +snow making +sle ight +sch uk +rubber ized +rosen borg +rigi dity +ren al +rem pel +ray ama +pu ls +past is +mel man +mc dougal +marcas ite +maha vir +luxury home +li or +laphro aig +la familia +l ø +kw skenya +ki baki +kar mic +ju e +jeann ine +ii y +houston strong +hali l +gyo za +go tom +glu g +ghe gan +fric tionless +fic t +engi klan +end ales +dr jd +cro ome +cor ing +carni vals +caly x +caity lotz +bush ra +bladder cancer +bee hives +bang yongguk +ðŁĺĤðŁĺĤ ðŁijĮ +è¥ ¿ +ãĥĪ ãĤ +öster reich +xbox e +vil anova +tu valu +ti ge +thre shing +thomas sanders +smoo thie +sec tioned +schim mel +sam wise +pal en +p tb +not forgotten +mcco vey +lam peter +john ston +io anni +headhunter z +hat su +h wan +gian forte +ger wig +fin ks +elo u +eli ff +dyspra xia +do heny +dil se +chancell ors +ch ford +capital official +beat o +ba sit +ati enza +!!!! # +ìļ Ķ +us k +up stage +ulti maker +tower bridge +telly chakkar +squig gle +se iler +se bab +scape go +sau x +re manufactured +probosc is +poke dex +om et +officiald gispr +oc bc +no ya +n ort +mili eu +levit an +lehigh valley +hyper activity +hoi berg +gre ed +expend able +endoc annab +dol ci +do sha +devilmay cry +deschu tes +con ta +coc cin +ceremon iously +avi anca +architecture lovers +apar icio +al ga +] ( +ठĪ +á l +yellowston enps +x am +vadi velu +termin ates +spac emen +se gui +schitt screek +sc media +rel ent +pu esto +pic kerel +pi vx +n assim +kh c +jack als +ishi guro +isab ella +indi at +i mei +goo dd +gol maal +fashion news +face it +dor cas +cun y +cli brary +cent rep +cath mckenna +catchit kansas +black wall +basker ville +bahawal pur +ar bol +am har +acom pany +unear ned +transvest ite +tor ching +than ku +sul t +sine stro +sho ppin +sel le +power tv +polyphen ols +pel argon +pax ton +over achiever +modi fier +michi ko +me el +mcmur do +maungan ui +masch ine +mas l +li mate +ker mode +k mm +jessicani gri +inver cargill +indi es +in cep +iam r +hyung sik +hav arti +gwin nett +good game +gar and +fan bases +ef ans +ed w +dr pepper +demon stra +con migo +cioc col +cere bro +categori ze +both well +barre to +bar ia +bal rog +ari anne +anal ima +abat toir +aaas mtg +ðŁĩ¶ ðŁĩ¦ +ìĿ´ëĭ¬ìĿĺ ìĨĮëħĢ +м ак +zwir ner +westph alia +we play +vi mto +ti que +st out +st james +sp ps +siri sh +shav ers +sha we +ry ker +retro fitting +re boot +pre en +phalan x +oom ba +nas scom +moment when +men ingo +mas ked +ko wa +ke ets +jat engiklan +ir ked +intern alized +do vic +dei dre +clar ins +cis sy +c no +buck man +bt p +bin aryo +beel ze +bapti sts +auto repair +att oo +apho to +" ->: +âĿ¤ï¸ı ) +west life +vince mcmahon +v ass +us afric +twin ings +toast master +tkach uk +the blogger +shahid masoo +sen cha +savethe crew +sat night +river hawks +revi ver +rad har +poly gons +pir ated +organic food +nigh to +mur rum +mil house +mark ell +man v +lac son +ke uka +k bis +infe cts +huw stephens +e tho +counten ance +clar is +bo sso +bo sc +bit defender +bio char +and am +american legion +ab l +aaaaaaaa a +ðŁİ¶ ðŁİ¤ +ys gol +xal apa +west f +we want +trending now +ton college +the word +teppany aki +slap shot +senior year +sam m +ridlr mum +repatri ated +reic hert +phosp holi +op ines +mm en +mau ney +mangesh kar +love this +lo do +ka sha +jar ry +in operable +hol lie +fran ko +fa kih +en schede +ellen pompeo +du kakis +co ste +cl é +christ in +ant as +annab ella +alberto contador +ak io +y tretweets +wiganwarrior srl +vi render +together weare +sw abs +so a +sil sil +reticul ated +publi que +pool er +ph un +op b +okonom iyaki +offu tt +nov ara +mockingjay part +miami university +metall ics +le ade +kra bby +ke em +jos lin +ja ha +hom erton +great food +gla zier +gin ni +fram lingham +den nen +conferen cia +city lab +cari produk +bol asie +blue sky +app am +: , +ðŁķ Ķ +vainglor y +urband ale +urban photography +unconsci onable +un kle +ubi qu +twee tur +tro ppo +sunday blogshare +stewartha asr +stewarthaasr cng +ste ams +st joseph +slo ane +shi agenocide +say e +sat chat +s ze +red car +r nz +qu ill +paris fashionweek +michel s +malme sbury +loch leven +incre ible +ima gen +home remedies +he fe +hb f +fort collins +einf ach +ea i +dou jinshi +disin fect +cr antz +conspic uously +cole sprouse +chri scuomo +chin mayi +car pa +bread board +brass ard +blu sh +arne duncan +amu ck +aku prema +acet yl +ab dl +ðŁĺ¨ ðŁĺ¨ +z inga +yasi r +tup date +top ham +tirun elveli +the muppets +sukh dev +spani els +sof theworld +sn art +shi bata +shel drake +sa hb +ro dy +park ar +our moment +nau seating +nal c +magnific o +kopp ar +kin ane +inthe dark +i home +hon nold +he cho +gla vine +game over +fr ye +fitz simons +ferdow si +dal ÃŃ +d ming +char mingly +buc cle +bo akye +barang aroo +armi sen +ar cola +ap cc +? !! +ðŁİ ² +wool sey +wh b +vene cia +ti aa +team lh +str fc +sports women +sph illips +shay k +shan tel +shahidmasoo ddr +sepul tura +sc ally +sarcast ically +s run +ry oma +ru is +r bt +port es +mock umentary +mill creek +mik ki +kincard ine +itz ky +ir mo +ip tl +imbu ed +hot plate +gow dy +facil ity +epic ness +cu ffing +cohe rently +co akley +card well +bo ek +black list +bit main +b ently +b app +as su +ah shotel +adventure sin +ðŁıĥ ðŁĴ¨ +楽 天 +à¹ĢภĶ +á r +wi elds +tiff anys +tel mo +te me +swordar tonline +sk p +salt fish +ru dd +ro xx +recei vable +question naires +pros thodon +pre torius +personal ity +ok oro +occu pier +ns ffunded +mi sto +meathe ad +life house +kelly and +helly shah +grind stone +free kick +e be +didger idoo +cro tone +clan cashire +bu lu +bri ain +art ans +ab aco +ðŁ¤Ĺ âĿ¤ï¸ı +vec tra +union jworld +triple tt +tmr rw +ther ud +ten ses +te co +t god +sb dc +ryan higa +ritt z +paint september +nax als +mul vey +mu subi +mon the +mn t +mb ab +man in +lor ne +ling fieldpark +lans down +kuro saki +ke tv +hyste rectomy +har dees +gray skull +exam iner +es ack +digital banking +cric kle +collegi um +castle maine +can te +batt a +ar ds +ain bow +ว ย +wr ack +vin atieri +ti gue +t bird +sor ties +sla yer +sha hb +se sse +ro dd +ray y +nap ak +money team +mb ah +materi alized +lich field +lam er +ky alami +kut z +jo ol +j hoo +ir sc +en ron +down patrick +dispen sed +cor um +chateau briand +bnw photography +b wi +b gn +at war +ago on +aggie football +ðŁİ¬ ðŁİ¥ +ãĤ ģ +wy ke +w dr +un break +uk f +tru di +the gac +subsi dence +sports bar +sno tty +seque ira +sen ca +ol fo +nu bes +nie mann +mouse hole +mc queen +martinluther king +mand la +lal onde +jura do +jan in +it sour +intre pid +haemo philia +geo survey +fle xo +end homelessness +dom enic +clau dia +chan e +ce pa +ahu ff +ach ingly +âĢ¢ @ +yo gal +women power +wild fire +tili kum +tegan andsara +surro gates +sun ning +stig mas +ss op +sk unk +si son +shari fs +san kofa +repu gnant +represent ational +priyankag andhi +pric eless +pre phoops +practi ses +pan ahi +ob lak +now y +n do +metro centre +matt bevin +li thia +keele university +jin ny +ic orn +herald scotland +hef fron +he wn +happybirthday tome +gl bt +gear box +g prs +g ks +fol gers +far ou +epen ny +dev ent +derwent water +de val +david hasselhoff +ct m +chief tain +ch ten +caton sville +back ground +arjun kapoor +and more +all ways +admi rably +ac um +?! ?" +ðŁĻĬ ðŁĻĪ +ðŁĺĺ âĿ¤ï¸ı +ðŁĮŁ ⾨ +ðŁĮĻ ⾨ +wax work +un am +u sca +tril by +trapad rive +timber wolf +tam ba +sla vin +si ana +san ka +roe derer +random house +pro filer +print shop +perse cute +pe up +pe k +pat atas +nuclear bla +news queensland +nep tune +nar can +miller coors +loud mouth +knut son +kha der +jharris jayaraj +gender paygap +gate ch +fu dge +ev ar +d tech +care llo +bre cker +bo ingo +aspir ated +ag el +ìłķ ìļ©íĻĶ +vod kas +united airlines +under carriage +un played +un interested +tou pee +theli ghts +tb icycles +su go +ste warts +so ja +rom sey +pil ato +pe kinge +mra dam +mad han +keswick bootco +jazz fm +j q +inconspic uous +hi w +gro u +glit ched +ger mani +follow cii +fat t +dmu leicester +demarch elier +daily drawing +cyano type +cy mru +concur so +clive den +braam fontein +bl is +bir k +barri e +apo l +ante aters +am ri +ðŁĺ° ðŁĺ° +ìĬĪíį¼ì£¼ëĭĪ ìĸ´ +ب ÙĪ +yo pen +wur zel +war locks +ver um +thelast ship +ss nyder +sports writer +so bew +relati vism +quen cher +pan eled +nr p +nau gatuck +national comingoutday +nat atorium +make th +leeu warden +inn ards +gur dy +friends giving +free will +euror ack +creoso te +bottle men +bb clancashire +bas ques +bar well +ast bury +ar ja +ani i +ðŁĺĤ ðŁĺ³ +ðŁĩª ðŁĩª +íĻ © +undate able +thedivision game +syn cs +susann areid +stan ky +squad rons +spot ter +son equ +saras wat +ro ba +pandor as +p nl +mf n +llo ret +k allis +j se +i of +hym nal +gol u +g dynia +f bd +do fe +dj set +cyberne tic +bts festa +biglo tter +bb vacom +ball parks +bab el +after movie +zig go +whatson stage +wauwat osa +tur an +the vampirediaries +tele kom +su tera +scor k +satell ite +sar zo +sange eta +ready made +p town +only nikil +nen okkad +n ooooooo +michen er +leap fro +kwa ito +krau trock +kne ading +kal y +in icial +imag er +haid ar +guy ton +go blue +gang ster +fightfor wynonna +end rons +cott ingham +central ised +cau stralia +c sforall +bol la +bibl ically +ber ch +att oos +atri ot +ation ism +ashmo lean +al tura +ak lein +ad vil +ðŁļ § +௠Ģ +yan ked +wood shed +war man +vi vica +twee de +tg n +sic ily +rust ica +rox ane +rh shampton +pu dding +ol of +o ji +nas as +n tb +lloyd kaufman +kan ha +jewel ery +ii is +ig inals +grun er +glo bus +frey ja +founder sday +elgato gaming +do ting +del f +colon cancer +cn z +charlo ttes +caly pse +by passed +breath alyzer +bk n +bis d +big pharma +az kals +and as +all america +aaron goodwin +ðŁįĶ ðŁįŁ +yad kin +work mate +wab ash +vit ter +vil am +v xr +tom mcfly +sv end +surgical strike +staur ants +sh elia +rspb scotland +responsi ve +re ward +racec ars +primary rocks +pie tra +pay loads +p gf +om pidou +o lie +ne as +mel ford +lurk force +konta kt +kin nock +jor den +inf lux +im partiality +hau han +goo t +gal avant +ga ir +fe as +fay sal +fallow field +du bcity +d squared +cross dressing +cit rate +cere zo +carlil loyd +campbel town +bu hl +boo ka +black stock +bil oba +bec kie +b hullar +analge sic +ðŁĻıðŁı» âĿ¤ï¸ı +ðŁĸĮ ï¸ı +» » +yo an +ya seen +vi do +to stit +swal d +son ando +savi ours +re shared +nation ale +man co +love scotland +leg ally +had da +gist mania +full ers +day lesford +colorectal surgery +cine macon +blenheim palace +bal ach +ack bar +abram ovic +[ ðŁĵ¸: +ðŁĵĸ : +zeemusic company +with row +wh yyy +vas con +u ec +st ours +so arin +sm fc +shaleen malhotra +schö n +s global +ride ordie +resident advisor +ray gun +ra bility +pugets ound +out sold +onit sha +o iselle +n scs +myle ene +murder mystery +muba sher +mill saps +math s +la bu +ko el +kathy griffin +kam aru +jais wal +ide apad +hyatt sville +hal len +gun d +g wan +fu ma +education govuk +dress like +dh hs +curry wurst +choic etv +bu kuro +britt ana +brin ker +bandof brothers +ball player +alo pez +ภľ +yes network +wool dridge +w gc +vend redi +tre ze +tim farron +ti enda +tark anian +pender grass +nikol as +neighbor hood +nal anda +mull ens +mis elizajane +mf g +merito cracy +mccar tan +ki ren +justfor fun +jame so +il igan +go etze +fe ck +estate agent +ern ps +ec centricity +dv bbs +dragon age +com score +chris mas +can one +bul le +board ing +beren son +beagle freedom +ap tos +aleu tian +aldubbig boyz +ëħ ¸ +ãĥ³ ãĤ¿ +âı ³ +w ira +ven try +vas u +ul ita +ty le +stu ddard +street ball +space dotcom +s icist +pe to +pas sively +pap as +os muertos +objec t +nu pur +ner lens +nd b +n sk +mm vas +min da +mie shat +manag ment +mad smikkelsen +ma uri +lu hrmann +legal aid +le ander +kro es +k tu +jor daan +isai as +ib jjf +grass roots +glo bos +g ch +fr sa +fis cher +el mo +eeee eeeee +dian ap +cheap flights +bill o +art center +ab idal +ðŁĶ ² +wen k +thumb print +third man +therud ingroup +the players +stat eroom +sla st +shire en +sb snews +sambal pur +red ne +red head +pur ana +pou ille +pimper nel +par dy +nobelpeace prize +mash had +make dcli +ko erner +jen ko +jan u +hurtigru ten +how z +haric ot +g fm +for n +far yal +ephe mer +ed dies +do lore +de kha +dar ci +cl inte +chil cott +card holder +bw ana +bud u +bo ven +bed ard +be do +age ge +ðŁĺĬðŁĺĬ ðŁĺĬðŁĺĬðŁĺĬ +ðŁijij ⾨ +work around +walk offame +uni mas +the jeremy +th picture +symboli se +stor a +speciale ffect +sn hr +sil v +sel ve +scriptw riting +r su +on iz +male y +li be +last word +guanac aste +gen entech +garden of +franco tv +formal wear +fore va +fel t +ez official +d tu +cor poreal +car ice +best practices +barri sters +barbarap alvin +avi des +attend ances +anun cia +am per +ðŁ§ ł +ì£ ¼ +z and +wi eg +tw ings +tox opla +ton sill +thisise gypt +ter ras +shi ekh +scot stoun +scol lege +royal ton +rin du +pon ga +over shadow +nus antara +nh tsa +milli seconds +memor ization +mari ani +makedcli sten +kis smar +infidel s +fi ano +envel op +defe ctor +dar ts +bt n +brock man +bran dishing +blow outs +bless thefall +big cat +b vc +agover nance +add led +ac ou +îĮ ¨ +y mp +x biz +tryan uary +stan o +spl inter +sit ra +seven dust +s gh +rumm enig +rummenig ge +ri elly +pokemongo app +pla ines +ph b +na jam +mira bella +mi raj +li pe +ld lc +kat ah +janef onda +j fs +im ou +gol lanc +fragran ce +fibaw c +f so +dig as +corn walluk +congr atu +chat tering +cb so +cat fishing +bou ghton +amu thu +alli simpson +af v +ðŁļ ĵ +ðŁĺģ ðŁĺĺ +ìķĦ ìĿ´ +yil diz +yak is +y combinator +xi ons +wil kos +wb k +tor ro +thet dn +spir alling +sen ia +scur vy +sam an +quik trip +pasqual ino +pad wa +om bs +millwall fc +mer tz +marse illes +mad der +lunch time +lu th +li gero +justi fiably +j oop +i mar +george hw +for her +eli vely +ele on +e gress +dis miss +dis engaged +de ighton +coo sa +co iling +cancer care +briar cliff +bol u +boe hringer +b ft +ay lin +are ddy +app ian +am dev +am aa +alexander wang +ðŁı ¸ +y tl +wonder boy +waitress musical +w us +virtual box +v atten +ucon nwbb +thing syou +super fine +sle ater +sig mas +sen thil +schul ich +sar ia +rapi do +ra donc +ra bo +pro pyl +polly anna +plan k +martin amcbride +kip nis +kan baru +hg tv +grave yard +glutin ous +gi pson +fre dric +fou rie +eng es +e em +dix ons +compac tor +com mode +cold fusion +ce fn +brink man +b gi +an um +an issa +yaho onews +wood ward +war da +vis or +vand alizing +v ati +un recognized +tiru mala +tac king +t mi +spe te +shr ill +schwe ppes +sach se +rho dri +photography wx +per tains +p lier +new berg +mor tad +moo die +maw lid +madein england +liberty ville +kop a +jer od +itu c +go leman +franç aise +food tank +fab inho +ep y +ep le +efflu ent +e wald +de wey +cyanobac teria +compos iting +bicent enary +bergha in +au i +african ism +* @ +zac goldsmith +wood grain +under tow +tonsill itis +thorn tons +tar aw +sp harmacy +sing lets +sho gun +sa unas +ron wyden +ris ch +po de +pekinge se +pal le +on aldo +official bwfc +nic h +nak u +modi se +mi sheard +mean green +lis bon +lig gett +kate esack +jes olo +james francotv +inter red +inte st +horse shoe +galatasar ay +er ol +ener al +com bust +chesterfield fc +break aleg +bra sco +bird lovers +be ed +bar an +bann u +ar mature +ðŁĩ¬ðŁĩ ³ +åĴ Į +âĤ¬ ) +way man +w ukong +us natarchives +trans act +top tip +su varna +sty ne +star wood +stam mer +shol lywood +p illing +overs lept +ni gra +morgen stern +mesen chymal +lagun itas +kin ah +ke ik +k usa +j are +irish music +ib mb +hr derby +hou v +glen avon +fire pit +executive goth +diet iti +di rim +di pole +craig y +cor vid +by utv +biolumin escent +bett endorf +bell amy +are ly +albace te +alab amas +ac cel +` *) +ðŁĹ£ ðŁĹ£ðŁĹ£ +âľĬ ðŁı¿ +à® ¯ +y oni +work room +woon socket +ward bound +wak u +vo ree +vest ments +tro oms +tr at +til t +tham bi +t ps +swir ly +sti mulator +pr and +mis ss +milli an +live to +lasse ter +la am +jon nie +jin day +jay aram +in ui +hinch town +hal ving +gu lo +ex ton +distor ting +death wish +chri sm +ce judo +bo ker +be ens +barn find +baj rangi +agu ayo +ad waj +ãĤ¤ ãĥ³ +é ireann +wi vind +vas anth +tm x +sympathi zers +survi ving +spo sa +span os +sece de +season finale +ro si +rep els +reli quary +real life +radhi kam +pre flight +portra yals +park as +oshi om +om art +nothing but +nicol a +mir ka +mieshat ate +me as +manufac tur +man and +ma hil +l bt +kab ar +j ns +fla s +dune din +down play +de dede +de cade +creation ists +con notation +community service +college humor +co stars +budge ted +bu colic +broo king +bo sma +bo gues +bli stered +beetro ots +an aa +americas milhist +ภĸ +wonder struck +war face +tran sis +taka o +stock stowatch +sto tts +soul ard +sk han +ro ache +rho do +re as +ra pala +pu tted +pollu x +perme ability +mul timeter +mar sha +mac u +lu tter +lor ia +lobo tomy +leon hard +keith ellison +ke diri +jon snow +jew ry +jen as +hay ward +ham ari +gher ini +ey n +dj inn +displ acing +diete tic +di ggins +dal vin +cham bliss +body boarding +big man +batt en +baton relay +anz as +za ara +yogali fe +we ise +vi pul +veuve clicquot +v uk +tom ake +sv ant +spot lighted +sky r +shil paf +she pperton +shape shifter +shal war +sen to +sam martino +s ops +roun del +pharmac ological +odeon cinemas +monic a +me anders +mak elife +kom bat +ik ar +ich mann +hump ing +hu ddleston +gul marg +gha ag +gel ang +for bidding +f np +electroly sis +een adu +dra k +ches i +cen giz +buzz saw +bo hin +be cs +ave c +amas sing +all iter +air frame +accu ms +ðŁıĢ # +zil ker +z eng +wr ks +way de +us q +thom s +thenation all +take care +sriti jha +squ aw +salonedel mobile +robust ness +quen ched +quan tic +propor tionate +pen icu +patoran king +pal y +pak ora +p min +oz one +l wr +l fk +kind les +k acy +incess antly +il ene +henry ford +hay ao +ha sa +group ings +fé lix +fen ner +es ade +eri shere +elum elu +eg ghead +e mini +du kan +dragon boat +democracy now +decre pit +de akins +dal o +cy mra +contri ved +con ny +cfl gameday +calle baut +bri ous +bal aji +as agar +art sin +a oba +ģ ) +ðŁijĭ ðŁı¼ +̶̲̥ Ìħ +zo es +wh ic +wa stel +vend ome +under cover +turn tup +tri on +ti mid +tam o +tag ine +squ el +son ee +qua shed +pp c +pi ku +overestim ate +out pace +orang enation +mon tour +mcclar non +madhu r +ma bo +love quotes +libre tto +lan ham +just add +inter viewers +hi iii +gsm np +galli um +ga en +fresh ening +dre i +colorectal cancer +bo ag +bar bel +bak al +' ', +ðŁĴķ ðŁĴŀ +year long +wn it +vodafone in +virgin ie +vi ol +tran ge +the fog +tab lon +sen dak +ren ames +re vin +rdpro cordillera +pat rolled +pat ri +pan is +pal ates +ow u +movi ec +men des +marty scurll +mana watu +lsh tm +lord taylor +koso va +ka ise +i back +hill y +h sv +global surgery +fo gel +flood water +enew york +d gaf +common er +code masters +c ty +boudre aux +ba ily +aren d +ak osi +a omg +èī º +Ã Ĺ +wom ble +with a +w tvd +votel ittle +vancouver pd +usu k +tt is +tat ars +tal at +sc ath +sandwich day +roo ghaag +pre ludes +ponson by +pho tor +p falz +owat onna +national post +nab oo +n gan +mtvawards star +lo qu +krz ysz +kind t +kin ki +kateesack hoff +join in +it sad +i ws +fle ec +fearthe turtle +f sharp +ele u +e jac +dodecan ese +do better +ch z +calli grapher +c aci +bend able +aco a +ðŁĺĵ ðŁĺĵ +zz ar +w ga +uro logist +talis ker +stra ights +sin igang +si biu +shi mo +sham us +royal welsh +row son +rose ate +rambl ers +r cts +poo ley +pett iness +overex posed +moistu rising +meh ra +marce au +marac aibo +lo omed +lo las +lets fly +la em +kick apoo +in dol +hen ne +ham tram +fort nums +fir hill +de acs +ch w +cas sian +brû lée +beyond meat +bald ricks +asdf gh +aqu in +andali o +abo l +ðŁĨ ĵ +zing ano +yaw l +vs no +voc able +vill ano +v dp +twi c +sukk ur +ss lazio +sh war +rudi mentary +rab anne +pur die +pro w +po tosi +paley center +no len +ne vers +mostre que +moor side +mill wood +ly sander +li o +ki jiji +k off +jack whitehall +ing au +heart beat +head ington +he afy +hal pert +gu ano +gr f +drone stagram +drjd rooghaag +diamond platnumz +cruci ally +col ly +channel stv +black money +bint i +ber rios +avo ir +au pdates +asi as +as sar +zapp ing +visit sunshinecoast +vic ton +vi katan +tx state +the bay +that girl +team pixel +su perfe +sour a +sli gor +si gur +shepher ds +schaden freude +red cat +ram bunc +pou let +par abel +oscill ations +nouvellephotode profil +national superheroday +merchan dis +m Ã¥ +life on +lem ke +lang it +jami ele +ig na +i frc +hu st +har rod +esp ou +devon ian +determin ant +de votes +d hal +confeder ate +chino is +book sare +bid dul +bard bits +apple pay +anna beth +ÃŁ e +w la +ther it +super leggera +sign aled +seme do +ri ba +retr ato +or son +newsc aster +negli gible +n ú +mot tram +mm flint +mathe us +mal co +gran ma +fox footy +finalfantasy xv +fallo pian +ex tempor +efl cup +disp o +dat agovernance +cor g +cf h +cart wheels +beer cat +band hav +bac s +autisma warenessday +au ke +ali yu +âĻ» ï¸ı +à° Ĺ +zoo ey +we iter +tel os +tatsu ya +swag ged +stead icam +sta ithes +smi ds +scan berra +sau lt +po ppe +percol ator +pad re +on netflix +neta porter +l antz +ku mara +ig nomin +hard copy +gold medal +ga o +ford foundation +food pics +evangeli sts +empower women +d hq +cran ey +cla ys +chief ly +cat ton +cas sano +bou chon +bos elli +boni ver +beg ru +ar tw +web pack +vivi enne +var usar +u texas +then u +the ac +tarantu las +su sy +startu paus +spar ky +s vo +retar dation +ren ergy +pur wo +postu ral +over state +mor te +mj g +me ka +master card +lovewhereyou work +loven ature +li ghty +lasc elles +ing leton +ignific ant +ho ists +ers life +cu ti +choice music +chee ze +by b +ber y +ai dy +aberne thy +ðŁĺı ðŁĺĺ +ðŁij ´ +ðŁĩ§ðŁĩ ¬ +yu sha +xer ocon +wor snop +u ba +tit ch +tion able +the fox +smo k +sen so +rohit shetty +pedic ab +pardonmy take +pa katan +pa avo +om bia +mol yb +mcgur k +mat tes +ma rella +liber sek +la palma +k latt +k bs +k alla +ji de +jablon ski +indie sel +in in +her re +head banger +ha koo +g valan +en ner +dir ks +de facing +cop as +co agulation +civit as +cath ie +braw ls +br annigan +bengal uru +bas que +band saw +and d +an chez +ac anthus +ab cac +Ø Į +white christmas +weas els +up staged +unshak able +theoffici ala +sugar hill +style book +stubborn ly +sla svegas +skin en +se vi +random house +rand oms +ra vil +r bjb +q in +persi ja +parti dos +ott bike +music matters +mol ata +mohsin khan +mike lee +ku st +kru ll +k aming +hol dthe +h ance +girl sday +em sa +eli ghting +dren ching +dread lock +dow chemical +decrimin alize +decre ed +daybreak hitz +d fat +cou ric +bha sh +aw ale +ap uto +wwe cesaro +wor sham +woo oow +winnipe saukee +whopp ers +toon town +ther olling +the gn +sar to +re sourced +pw ll +mus grove +mu ke +lo sin +llll lll +lek hi +k jrh +k dc +hockey night +hat soff +ha un +dream girl +dinesh dsouza +dend robi +cum nock +charleston shooting +chad well +cer nan +cade aux +ba chan +atten tions +al stro +ac oma +ðŁĺĺ ðŁijĮ +swee test +sport in +sony uk +sobew ff +sam mich +robin lordtaylor +rei shi +q arab +ot dh +mono tonous +me time +la zier +ki ane +kat ri +jencarlo smusic +ingo b +implant able +hope and +gurupur nima +gri fter +gor kha +give hope +g adel +flo res +ever bank +ety mo +esc ada +dream chaser +doo gie +deu lo +da el +d our +core lli +beart ooth +aziz i +au ght +angle terre +amor os +ak ai +; )" +world penguinday +tu d +tor ium +thim ble +th atta +ta her +sp ag +sou red +shape of +sche i +rub ina +rookie blue +ro jos +patoo tie +nutrition ally +mutu amadri +mutuamadri dopen +mirzas ania +mark wright +lar te +koscius ko +kn olls +kit schy +jonathan r +jawa har +hi gha +good by +desi pixer +dd un +dak ima +cicer one +can ty +cair ney +bor chetta +bm alik +bingham ton +bet sey +bernie orbust +ben nys +arm field +anni a +alla hu +aba shed +à° ¡ +yuzuru hanyu +ye en +yar sk +w bt +vic mensa +ver dad +ui dai +tra pani +tim al +sul lied +sooraj pancholi +slo wh +shin su +sen ow +scar fe +sab ella +office design +n le +mubasher lucman +mick i +li bro +juli enne +jo ef +ire lands +ingh our +hierarch ies +hailey baldwin +fri mley +favorit ism +eventi de +dv sa +domhn all +dni pro +digit our +democrati zation +dec tomy +dal awa +cre en +cos me +convey ors +ch apple +car ryo +breck in +bre mer +bell sbrewery +ba aaaa +ari ely +animal health +ê´Ģ 린 +whim per +west fall +tv channel +ton ge +thel oop +tam ir +stre aked +squ ibb +span to +slip stream +quir ke +post impressionism +oper ational +mcin nis +march é +lead ingthe +john w +gu ster +feren tz +er le +dar rah +criminal ise +cre search +cre che +coo der +chap ulte +bottlen ecks +bl é +believein yourself +bel videre +bbcy ork +all ens +al ito +ai zawa +adop tee +⾨⾨ ⾨⾨ +tac a +spray paint +smashing pumpkin +simul ates +se wanee +sac state +re agent +orno car +nesham iny +nar bonne +n ssf +n lu +my bestfriend +mu sto +mari ag +ko ve +kick flip +jor gen +gen tile +fun nest +fr and +fle dge +fa waz +en more +dirt bike +dd ar +culmin ate +comp ound +cityof sydney +bran son +beauti fied +beard less +b ils +b azz +ay une +auton omic +alli on +af cu +. âģł +ðŁ¦ Į +âŀ ķ +âķ ² +âĢ¦ âĢ¦âĢ¦ +yak ut +whatdoyou think +ving ton +truss ville +tra g +te tte +taka hiro +taip ing +share humanity +rc bc +prob ation +pp reciationday +picka xe +ottawac itizen +nu ria +nn nnnn +nicol le +n was +mc nuggets +la zers +klu ivert +j va +gl bl +gat ton +for se +for ia +el speth +dun mow +de coys +dal ziel +cypres shill +cu ticles +bri mmed +break er +boge ys +ber rye +ber kle +au sag +arus sett +ap on +aom ori +ame hta +all tech +abi el +) .# +âij ł +ye gevents +virginia beach +upro ot +triun fo +thisi show +spl iced +smack down +sh appen +sco ffs +rt k +rocket ship +ro ber +re pub +ra as +pre dawn +p cb +om v +mu tya +mo sen +mel k +mckel vie +marac ana +love shot +ki zz +ka ar +joy sticks +j sf +hether ington +ground swell +giar dino +fus illi +freaky friday +famili ar +er v +emphasi sed +dj m +di rac +del ts +de ut +convers es +clo dagh +businessc ards +brown sea +bi ge +battle ford +b be +author confession +acl fest +ãĤ¸ãĥ¥ ãĥ³ +à¸ł าภ+Å Ļ +Ä ĵ +wit i +wheat belt +vi i +vector art +up g +th si +testimonial tuesday +ta kap +symph ony +summer vacation +shaheer sheikh +sh te +rums feld +revel ers +red skin +rebec cauk +rat eliff +pred nis +pablo alboran +p dsb +optic on +mou lin +men don +leib niz +leah rebeccauk +lam ia +kedge ree +ini quity +humidi fiers +gy lfi +gun n +green tech +ger ri +galli frey +gaem gyu +financi alliteracy +fall brook +fa its +ek ta +e ot +dj ent +dece iver +de ade +cor vair +com un +char my +car pool +blo feld +baj pai +att uned +alo evera +aha shi +ãĥĩ ãĤ£ +á ĵ +yas si +y str +wwe asuka +wissa hickon +whe len +wash y +vb schools +tra ba +ta ree +screen time +qu ills +oh v +nath alia +mir wa +magi que +li ppie +lebron james +land or +j lp +iam lakshmirai +hans grohe +gro se +great artwork +gar ish +fu ga +fever ish +fe tterman +fan euil +emph atically +dan um +cu u +cow ling +conco cted +center for +brig nac +baaa ack +amar ch +al shabaab +ach arts +ðŁĶµ âļª +Ë ĭ +vo it +ver min +un readable +tre sor +thi ers +ter is +simul ation +shav n +sha c +rule sof +rock er +rivi ere +r mn +pe dis +pal abra +op acity +nd su +multi level +marin os +lg w +jake quickenden +hos king +g ph +fn q +fle ish +favour ing +fa an +eye lets +ele s +dis assembly +counter measures +chikun gunya +car one +c cl +br anc +bl l +bio gen +asan chez +arch os +alvor d +ali anza +ÑĦоÑĤогÑĢаÑĦ ии +zach ariah +spe ier +so sweet +skid daw +salahu ddin +ri endo +py t +poor na +olaju won +my our +music ality +massaf elipe +mari ano +l ge +kal yn +in site +impeach obama +iaa forg +ha dden +got the +ge malto +fas ano +event planner +en forces +dream act +dit avon +deulo feu +colqu itt +chand ram +break through +b vd +b dd +asat ru +antici patory +annex e +ðŁijij ðŁĴķ +Ø§Ø ¬ +uj ala +u may +tit alia +thenew painting +tel star +stron tium +stir chley +so ti +sky lake +ske e +seduc tively +scrim shaw +scre amo +s ba +ru blev +roo ty +return to +ren ny +pre ez +par tha +oik os +menom inee +mc chicken +lucky girl +lot teries +it sar +inform ally +illusi onists +hir ono +crutch field +craig s +coz art +bulle it +bu tera +brush strokes +at r +at plc +arab e +antam onica +ancou ver +alast air +íļ ¨ +z ma +uni ya +tru k +than javur +sv l +shaf aq +scur ry +ren de +ram trucks +rabi ot +pi dge +nürn berg +megaz ord +m gwv +kra ven +jo bber +ip sa +i fla +hol li +herm ès +hear twood +geo cache +fur i +forzan apoli +forzanapoli sempre +for greatness +fly way +fl d +first weets +domin icans +chaf ing +cf meu +ca xton +botanic garden +ban trophyhunting +af on +a alt +"} ," +Ø ¶ +zz ies +yed lin +wiven hoe +web comic +wahoo wa +vers ace +un banked +tu lear +top gear +the maine +stra vels +st ps +spa int +say brook +rss ur +plat forming +padma shri +old is +o tis +nic co +lafour che +kas ai +jer king +j ba +i flix +hind marsh +heath cliff +g war +fee han +eye of +ex um +dumb ing +duc asse +du cker +col lings +cb sa +canad ensis +basqu ecountry +art adventcalendar +ap un +anil kohli +ai doo +af air +ðŁİĥ ðŁİĥ +îIJ ĺ +ëıĻë°©ìĭł 기 +âĢ¦ âĢ¦. +wi veng +whe ats +we il +uniofe astanglia +tulo witzki +thereal ryanhiga +team tolex +tan o +summer set +start the +sh nikov +ru thin +rnc m +re ssa +r ä +pri vet +prakash raaj +pe sca +part itions +param edic +om ing +neutr al +lit itz +l nd +kool haas +kar cher +inst ate +i severything +gu ma +grou se +gn b +fu p +fu jin +fou ad +flat irons +fit etv +elles se +dre ich +derange ment +da hab +cr ana +contor tionist +chin na +change thegame +cad enza +cac a +brance day +bout in +boul anger +book able +ban ke +az y +amand la +allo way +accu m +ðŁĻĨ ðŁı» +íķ ĺ +wi shaw +wal ley +ur bann +tweet meso +tweetmeso hard +startup weekend +siyah bey +shawnab ner +sc cl +s conference +ru ch +ros ella +red skin +radi ol +pv r +porth leven +pen ciled +pac esetter +mol nar +militar ism +marcel ino +love travel +kr ys +kie hl +k oning +jo p +jin an +incen ti +in ad +h gt +greeting cards +gowdy sc +g mac +g finity +free zers +ex and +eco fashion +diffu sing +database app +chandra sekhar +cc ms +boot camps +bell eri +bar bas +as sou +art databaseapp +ani pals +alph am +aer ith +zo gh +virul ent +ts j +tru x +stru n +sl vind +si by +sch ak +sanc tus +re fundable +r win +player unknown +moo rea +mar burg +lo rescence +lo ofah +kiyo saki +kin c +ket k +j ola +inthe making +inter change +in ver +i if +gru m +grei pel +go chu +glamour ous +gh ua +ele st +confi dent +color fully +cha fee +cait riona +bu rak +braz o +b pf +air n +ack a +ab oot +ðŁĩµðŁĩ ¸ +ìº IJ +æĿ±æĸ¹ ç¥ŀ +à¸ĻภĶ +wind star +web bie +un trustworthy +tand on +sm cc +sambit swaraj +sad h +palin drome +optometri sts +o the +neu feld +ne uk +mp x +me war +knight fall +kindle deals +k na +jer myn +hh shk +guille mot +gad on +fer ne +evans drumheads +disappro ves +cp chat +cin di +agassi z +! ðŁİī +ðŁĴ¨ ðŁĴ¨ +ðŁį·ðŁį· ðŁį· +ãĥģ ãĥ£ +x scape +wilt shire +wi dge +un sanitary +uk biz +uc cess +u hoh +tu areg +top drawer +the go +st oli +sle x +scro tum +sa ja +re wired +pren ton +port more +pa hoa +nazar bayev +nar stie +nann akuprema +na stia +mo issan +melo dramatic +maz dausa +mal pas +ma rea +lin ford +lan ing +jab ari +home spun +hi stri +hell bent +global compact +ge tat +fck n +fallen kingdom +eee ek +dev summit +den ice +dan z +bur bage +bou illa +blu ed +bar tra +ali brary +agen da +! :-) +ðŁĹ Ĵï¸ı +à® Ł +ठı +Ùħ ÙĪ +you ghal +win inghour +wininghour chat +west way +warnerbro sent +w music +topa xi +the crew +st offel +spu bs +so bers +shore wood +sf dc +sav ita +redri sing +pune eth +paradox ical +par ler +oscar wilde +nca avb +na dim +memor bilia +laiki pia +ki wami +jess glynne +jar rad +ibra him +holi ka +hol ston +hat ay +gran ados +gan go +ff fff +fari ed +er tl +e wu +dissip ation +dac ty +cor dray +cm hc +ci d +capital one +bon ga +bi aggi +atten burg +asco c +ano don +zhen itsyn +w tc +w kow +tit as +te agan +tad os +tablon dean +tablondean uncios +sof ts +say yid +sap business +sam c +sal ame +reform ers +ree sh +ran kine +pr grm +pic keting +pel ini +par my +out casts +organ oids +o ag +movie making +mik kel +me ti +matta pan +m du +hol lar +hhshk mohd +gu tless +fro wning +dab ang +cred itor +cnd poli +car gos +bid deford +ben field +baz o +ay da +ar nab +am ak +ðŁij ¢ +å³ ¶ +ر اÙĨ +work books +wah da +w brc +veri fies +tu j +th ao +str ato +sophi et +son al +regi stries +pk n +p libersek +nom ercy +mid sommar +mi ocic +men ial +martingu itar +love music +laugh ter +kun o +k tf +i acc +horror fan +ge ingob +gar nering +flaw less +fantas mic +familiesto gether +ex as +entre prise +enniscor thy +end all +dro yl +disgu sts +cin ders +carpinter ia +cant illon +brightling sea +ber the +be kind +bang sar +antimicrobi al +ai duk +ãĤ Ĭ +wear ing +ve ur +u gar +the mindyproject +spirit us +sp ade +sen iority +salicy lic +ryo ta +ramin karimloo +pat chett +on tein +mish ti +ly copen +li bido +lesmis official +kal is +ire les +habl ando +gu son +gaz ans +found ationday +fin ns +edel brock +dun wich +devon te +ck p +chris rock +canad agames +bok sburg +bohemi ans +bapti zing +b ju +aziz ah +au li +ar test +alter nate +al mar +air freight +abc grandstand +ðŁĺĬ ðŁĴĹ +ç elikkol +vin z +vi ajar +un mitigated +trac i +tor turous +taver ns +tal bots +stur f +star magic +shi zzle +shack elford +se vigny +sali v +sa xi +ru thi +rte sport +pro zac +nr n +nc ba +mor pho +middle ware +mc vities +man ion +kar ly +j smith +j fr +iron ore +humb ert +horse show +horse and +ha chi +gum bel +glo at +gil an +fiord land +fin u +f gw +europ car +dÃŃ az +dre view +dra v +dont judgeme +distribu tive +ches ley +che sky +bus ily +bon gani +arm less +air boat +z ut +yi ann +tren tham +trau mas +thu y +thomas mulcair +there val +tar ma +ste ichen +starbuck s +slu gger +sland ering +skelli g +sau chie +s vod +pot ent +ou zi +ou rown +oi des +ogl ala +occupational therapy +nin a +mine ola +mascul in +ly ari +luke goss +live blog +le clair +lam pang +lag atta +in boxes +hil versum +hal eem +hag man +gsx r +gor m +gio chi +ga x +frat ello +for zaf +first avenue +fil le +ern ary +el b +dw ana +dra wl +dhar am +dev ou +chab ert +carol ers +ayr shire +ato ka +anam ari +!!!! @ +âŀ ¸ +ê s +voic esof +ut g +tomorrow world +thenationall ge +th ors +tar un +syn c +re ubens +pro ck +or row +nath ali +my job +mu tin +mo el +meille ur +mcdon ell +mb ri +maryanne hobbs +ma dy +luc ado +kel lett +ke isuke +ke il +han shin +gare tt +el sa +dr ury +dis ent +digit ale +den nett +dan foss +bi wi +ap onte +ðŁĶ· ðŁĶ¶ +zom bi +yur ts +y ingly +vali er +tf xc +stu c +stepin ac +read in +rani mukerji +qu belfast +pir o +ph ole +pf as +or as +o gi +nu x +not cool +marke teers +ma bel +m res +lycopen e +lu en +ld cs +l ary +ku lam +kil ter +jumb led +h ounded +go sto +giann ini +e gger +downing street +co ola +bra am +bike way +be ve +bar aat +bak are +and le +accre tion +y one +waterloo ville +warren point +u rock +trum pet +the hindu +synes thesia +stra ya +smock alley +sin y +sidhar th +sa hl +re plug +re han +rataj kowski +pseudom onas +private ers +pre pa +per spir +ox fords +olu wa +nd weeksary +my celi +mu ell +mk hize +mi sion +mau ghan +lo onies +kn vb +kie ren +kha bar +insi stent +idiosyn cratic +i spa +hijack ers +ground nut +glo aming +gi u +fen rir +fel da +fe ts +ed ict +dra p +del ton +crook ston +chin ko +chi we +ch ch +bor gore +bio logic +berk ley +ðŁIJ° ðŁIJ° +ت ØŃ +zi kav +y awa +w fg +ved hika +un organized +un chman +step brothers +so pp +sam j +red ington +raf museum +pu shers +present you +om ial +ne wed +nation tv +n power +mor tar +modi fied +meri wether +mac ari +loch aber +lloy ds +l mm +ku h +hin ders +grl powr +gra h +go gue +fran king +fat in +f hollande +dom aine +co pro +cadu ceus +bas seth +arsen ault +anti matter +ano de +al tis +ach t +ðŁį¾ ðŁ¥Ĥ +èĬ ± +w ando +un deterred +thel p +tan cies +speed well +sp lo +so hee +snu gly +sen so +say yed +sar ak +roz elle +renega de +or ino +nutrition month +nosh aven +nemato de +mista jam +lax alt +ku ji +key sight +k ori +j manziel +human ize +hu ac +home world +hol cim +he bb +green halgh +ga elic +fur suits +franchi se +fibr ous +facts about +elin ski +du sters +dar rene +cr itt +cor to +comic market +casso wary +cameron monaghan +ca ched +brick by +atfirst sight +abraham ic +ðŁĵħ : +ze dek +yu van +winter garden +wad hwa +train sec +the tour +the american +te ac +so blessed +simon sen +refu eled +realc frampton +r ff +pe b +mor ato +min strels +mex co +lumin aires +long mires +london zoo +la ren +j z +inspiring women +ice storm +hom ony +histor i +gry phon +global fund +gasmonkey garage +fire eye +fin techs +el brus +doc tored +cy de +com au +clark son +chuk wu +char minar +chap i +cadill acs +bringyour dogtoworkday +blow up +batt ler +ap lay +an av +amazone cho +amazon studios +ðŁĵ Ĺ +ر س +youngand hungry +unbearab ly +turbo tax +tonko tsu +t gowdysc +subsi ded +si ff +shot ton +shi ma +semin ole +sar ms +ru mer +queens lander +pre zzie +nol ita +n cia +mostreque sted +michel instar +lent on +law breakers +kale o +journe yed +jay mes +jare t +in vigil +helen yu +google chrome +dun keld +dun ga +dry dock +depre ssion +dap a +clement i +char u +cer vix +career fair +brooklyn brewery +ben ita +ben de +bees ley +bb ces +bay an +bar ta +am ram +ad gita +ãĤ ¶ +âĿ¤âĿ¤âĿ¤âĿ¤ âĿ¤âĿ¤ +zen i +ye gre +whe elock +vis wanathan +video graphers +varusar ath +us at +uri jah +u ap +tric kett +tri ana +to sun +then ational +ssi e +rumb lings +rosen blatt +pil fered +pam uk +p nm +micron utrients +mic heal +match book +mac and +lumin escence +l dr +jo wl +j fl +j blm +itch y +is na +interven es +ho ag +hill head +hi der +h xh +gr fc +gle an +gas kin +f dr +envel oped +delav al +de stress +de gan +dar u +char i +cdn pse +care takers +bunk house +bu star +bty ste +bed ell +be ak +bally clare +al by +ðŁĻĮðŁı¾ ðŁĻĮðŁı¾ðŁĻĮðŁı¾ +ðŁĺ¡ðŁĺ¡ ðŁĺ¡ðŁĺ¡ +ðŁĴĻ âļ½ï¸ı +ðŁĮ ¨ +« @ +za ghe +w abc +the we +tech forgood +tam bay +standwith pp +spartan pride +souther ner +sj d +shoal haven +shan xi +ring worm +ri led +pu mice +ptar migan +ponte vedra +pk l +one shot +mon tane +mis awa +milos raonic +lo teria +little league +leam ington +l cb +kab ar +it se +here theycome +garri ott +food fight +et si +doctr ines +death by +c xo +bumber shoot +brum by +bral ette +bal and +baf tatv +awa it +anth onym +ðŁĺĤ ðŁĴģ +ðŁĩ¸ðŁĩ ¾ +yahoo sports +wei h +uni onized +tre svant +toor ak +tic ias +thought less +si thole +sfor days +sen y +sc eo +samiz ayn +sak al +s ó +rock wood +rit enour +r mt +precari ously +por tedly +ple tcher +plate au +or raine +ob server +no che +ne meth +nas b +mire ille +me zz +mb ts +massape qua +marin a +kirk wall +jr motorsports +jabo deta +inu vik +inger many +indv ssa +hend rie +glass man +george mason +fudd ruckers +fir mament +emb l +dri pped +dental health +d age +com piles +chase z +caffe ine +bug anda +ba v +al per +ago stini +) âĻ¡ +ðŁĺį ðŁĴļ +æ¸ĭ è°· +yo ked +wis niewski +wi xt +vegas strong +va asa +uni kent +tor oro +today im +thel oo +tam ari +t anni +stu der +ssen den +son da +sol zhenitsyn +sky team +sitt ings +sho liday +sand ys +rav n +ra him +que rer +oxygen ation +nbc grimm +mur alist +math are +mar ham +ma gov +le ee +kn j +k go +iy pt +inj as +in an +ibero star +her bert +gre molata +gran th +glass er +ex ul +enjo daro +end angers +eg fr +du damel +dis que +compa q +citi open +centrep ompidou +cathr yn +by om +break ou +aw u +assist ant +ash lee +ðŁĺ¢ . +ðŁĴĸ ðŁĺį +ðŁĴª ðŁĺİ +âĹ ¦ +you matter +yo glu +win tered +wid mer +vaqu ita +vap ing +un patriotic +the tom +t ation +swa thi +show off +secret smw +say les +ric ard +repro ach +rad ham +pul la +nov ae +new bold +me lections +match box +mait ra +ma sing +m views +le gic +kel ty +jocel yn +jave dakh +j hene +ink ya +har ter +gen os +fus arium +first day +fif o +faul ks +fast food +fa eries +el stra +dan se +chey enne +c mac +bohemian rhapsody +bar kan +argy ll +annast aciam +annastaciam p +ag grieved +ðŁĻĮ ðŁİī +æĺ Ł +âĿ¤ï¸ı ðŁİĦ +ਠķ +zu erich +yu helenyu +x aver +warren buffett +univers iti +tol liver +t pain +stop motion +spo etry +snor ts +shoai bmalik +ro va +re train +prep talklive +plan eson +op ale +omg adamsaleh +no dal +nike sportswear +ner vomusic +nenokkad ine +national pumpkinday +mon tero +mc cue +louistom linson +kha dim +ju ts +jens stoltenberg +ilan ds +ije bu +how ley +ho tting +he h +gun smoke +go crimson +flat land +fer o +ep g +elder scrolls +dsb math +divin er +dine en +dem me +che h +bo tes +arse hole +ali ans +ag g +acet one +ðŁĺį ðŁĺĽ +whywe march +wh als +ur prise +ul ysse +torontomar lies +st clair +spring clean +pran ayama +pneu matics +pat ux +par athy +oil seed +nie kerk +mon dele +mi strial +m ni +lu ri +kr ld +jim mi +impressi onistic +i lex +ho day +ha kata +gur ren +grow ths +gar stang +gal ician +food stagram +foo do +flori d +fitz gibbon +ff v +f atten +en chong +des con +de filed +cur ator +clou dy +bur bridge +be tters +bar cell +ban er +all good +ah rens +ach ild +Ùģ ض +zar in +wis on +val la +utri ent +twit con +timothy sykes +sur realistic +ston ecraft +she ek +shawn formmva +save me +ro many +ravi kumar +ram bles +pink out +pac north +osc a +of action +nichol ash +nev sky +mu kil +lang an +kra emer +khan academy +kar ama +john coltrane +iz quier +ingle by +hash ish +gho e +fort worth +excu sed +dundal k +dugg ery +defence hq +blackpool fc +b ingle +air amb +af le +ðŁĺĭ ðŁĴķ +ðŁİĥðŁİĥ ðŁİĥ +ðŁį¾ðŁį¾ ðŁį¾ +} } +zo id +true tothe +thic kest +the gazette +the beach +stab bings +scri bing +screen plays +savethe arctic +regi mens +qad r +pi z +pat sy +oc m +o yl +mr jame +mortad ella +m ve +kab at +jacqu elyn +ham zah +global calgary +ged ling +ga ijin +ft l +found lostdogs +fa ur +dec affe +d mo +cor nice +chutz pah +blun ted +blak elively +bhar tiya +barne sand +back pack +ba ila +an eri +ç ij +âĿ¤ ðŁijĮ +âļ« âļª +zikav irus +y eller +wp tz +woje spn +who i +way ang +ver so +tur alist +thunder bolts +te ko +tar leton +tab let +sol apur +smithsonian mag +school lunch +ron stadt +ny primary +no strand +mon isha +milit aria +mal lam +laz ily +lamb da +krasno yarsk +joker it +jer nigan +jami ec +insectic ides +home buying +hermand ad +good latte +garden ing +fu ria +frontrun ners +flatul ence +faceli fted +f naf +even tos +emul ated +dubcity council +ditavon teese +dir nt +con ure +cabincre w +ben icio +bamboo zle +badger football +arrow writers +wh anau +wa seda +w gp +w aco +vete ments +up b +t attle +sm k +pp n +pat er +pan ter +ou rense +n aco +my rick +mol ko +mo sin +live jasmin +lede sma +labra va +la joie +l son +job site +in seoul +i ale +helm holtz +hav nt +gru ene +goss amer +france s +fire walls +colo ssal +cau pdates +bloggers blast +ben edi +barbar a +au dax +as af +alhamdulil ah +al uko +ac ed +å² ¡ +ঠª +اÙĦ Ùĥ +z ani +vikas gupta +ur sus +twit chy +tube strike +try pto +trin os +the centre +tam uc +steve yeun +sp es +rashi dat +rang ga +quin ny +poo s +po stie +pe eters +oc inema +oakridge boys +o sk +n oura +mostrequested live +marche se +maj car +lou den +lazen by +lac om +high end +hal is +gon calves +go doy +fla re +fin ial +dulce maria +dr jimmy +da ag +credenti aling +co bber +charle ville +campan a +bunk ering +buck minster +brook stone +boe tt +bi jou +ay in +art majcar +are f +act ments +ac esse +ab z +^. ^ +ðŁ¤Ķ . +wah ine +w si +twit er +town square +torre vieja +super chargers +strike sback +south town +slip cover +si kes +sho ed +raven wood +parabel lum +orme au +moon ves +mil ady +matt cardle +mark cavendish +ld v +kwe k +kre ator +kh atri +jo ba +home alone +gige conomy +for tes +ff games +di ph +despon dent +del hic +cm tawards +chemi st +catac lysm +bu cher +all rise +air bourne +agli one +af g +z he +wester ner +weard ale +was dale +votelittle mixuk +vi ator +vasec tomy +thy ssen +storyof mylife +stan lee +spor ti +sin ki +shu bham +sha bet +serpent ineuk +ser an +saf ire +sa adi +red carpet +real kvb +re try +people mag +par kers +ol lg +ok awa +na al +n gp +mirand asings +june siphone +inextric ably +ibu ki +he ch +gong chan +gaunt lets +faken ews +doc ter +doc g +do tti +deci mate +dec ayed +char twell +boon ville +bal h +ayl mer +anthem game +andre arussett +ake over +af lat +adm ittance +!!!!!!!! !!!!!!!!! +ðŁIJ ĭ +íĥ ij +ãĤ¢ ãĥ¼ãĥĪ +âĺħâĺħâĺħâĺħ âĺĨ +zey nep +yn geal +wai ves +un plugging +u spa +tul si +the wcs +spar sely +shri mpton +sel fin +scoo b +re bro +quo in +preci osa +poo chie +patient care +om arab +no wra +mss arah +mon as +lack lustre +kar ter +kal au +jav ale +jac ey +in review +ifeel slovenia +global streetart +glam berts +gh ockey +gan ics +gab bi +frustr ate +facil ities +elderscroll sonline +el din +do ga +dis armed +din ium +cor uña +caul k +bu ah +ben nu +beer garden +att lee +adol phus +ab im +ðŁĶ´ðŁĶ´ ðŁĶ´ +ðŁıĮ ï¸ı +า à¹Ģภ+world pride +wil mette +vaqu eros +tresp ass +tas c +tal bert +sy al +shar ples +schu mer +roseof tralee +rob delaney +pu pu +pr dp +plec trum +obsc uring +nash villes +miz un +mc coll +maje sties +lov age +l sbu +kind hearted +kat oomba +ka a +k iting +ja j +is cuit +hot te +hell hole +fuji fil +free the +flo gged +epi pen +dulu th +dix son +defin it +de et +clich es +chi ette +cal arts +bt sport +bou dic +ben fleet +bas in +assate ague +arth quakes +ah atchee +!!!! ) +var adero +the work +te ha +story brooke +st weeksary +sa as +ri el +replic ator +rag sdale +qu okka +prophe sied +ple ats +photo walk +pastr ana +mone ys +lomond trossachs +ler ma +le eco +lac an +hair removal +globe business +gau train +fu oco +frank land +extraterre strials +eve e +dissoci ation +d bongino +cdc whistleblower +cavali eri +by your +buter in +bi dwell +ann alisa +acceler ate +ðŁĮ ľ +ze ej +x music +x clu +wur tzbach +we scra +wa qt +u dah +table mountain +sun nis +spro perty +sk cbc +shand ling +say eed +ref n +rambunc tious +radi ouk +o bre +neuro scientists +n st +mi tha +medi acom +m trench +lang en +kon ark +ke gel +jeopar dy +hurri yat +hon shu +h my +gu ant +expe d +em ay +cra ske +chou dry +choic es +ator re +ðŁĽ ³ +é ¾ +wx yz +w bai +up shaw +ude my +the tide +the adam +terror ising +t pn +sub has +spraw led +sport trust +sp cc +sol us +see der +sa hitya +ren do +ran kin +open innovation +nasci mento +nab ors +mumb ling +movi enews +media watch +maz ars +mar abou +lym pics +is key +il ing +icri sat +human ized +hamtram ck +ha ah +gra aff +gam ecenter +fr illed +fac undo +euron ext +eu caly +espn cricinfo +do oms +chil cot +ce ms +bon y +board wal +batt elle +bad diel +am bre +altru istic +allsven skan +ago a +ðŁį £ +wreck less +wing less +water ski +veggie tales +ve ge +ta pu +sk rill +sent amu +sch ism +sac ra +ric helle +reprodu cible +prioriti zation +pais ley +ny cre +nr ly +nd fb +mourn ful +me ja +mark ballas +malaysi angp +lock able +ko er +ko dai +kneb worth +kim mich +k ree +ic hu +iam sunnydeol +handic apper +geor gy +g tt +flau bert +ess i +ecker d +dar u +cole brax +cliss old +cle mmons +city year +bro ot +black kk +anti sm +anne mari +ank lets +anim ous +am bon +adam u +; ~; +åĭ ķ +zer flin +world catday +wo hooo +wedding cake +wa vey +val en +touch é +tor ito +tit re +th anda +tam anna +tal bott +stretch able +step ford +ss afa +sk ed +sch mit +reduc tive +re brands +pul ver +proble ma +pra yag +pag er +p wl +onep unchman +no elle +mar shaw +malti poo +lineof duty +lac rosse +jones es +impractical jokers +hsi ao +guill ory +gio co +galent inesday +fed monton +discer nible +cur ates +clarine tist +car ner +bori vali +blue apron +biz tips +bho gle +bas zler +b agram +ar mi +an acho +aguas calientes +âĻ Ģ +zi z +weather all +wachu sett +vo el +tu sd +tostit os +theo cracy +the toronto +tam ucc +sequ itur +saadi q +r ones +poppy appeal +p hey +ow sla +o sen +o cot +newsc asts +mol d +lo dha +ken tish +itu l +innis fail +gott alove +gonzale z +duc ation +d jr +chou han +char as +cdw social +cap tur +bra ys +aqu inta +al ber +ad dis +ach ar +ac ela +_ ] +â¬ĩï¸ıâ¬ĩï¸ı â¬ĩï¸ıâ¬ĩï¸ı +zu b +v tt +ur sinus +un lined +transis beautiful +than ts +sy e +sb kb +rebec cal +re nia +plo d +pc sd +outer banks +out standing +monte se +mono cular +mom odu +mo dano +mb y +matt mc +marie osmond +kra sner +konstan z +k shan +jis c +jen naf +j lm +i stock +hornb lower +he ure +gy atso +film club +fe ku +energy union +duckduck go +d eller +conce als +bi pin +bh g +be ren +az mi +as microbiology +anthropo logists +ðŁĩ° ðŁĩ· +ਠ¦ +wil mott +wasi mak +up in +tric ot +tri vago +ti ah +thegood place +test ing +temper ley +smo di +ship mates +sadda haq +sab se +rin cess +ram is +quick fire +pom pom +pla skett +pl td +ow ned +over hauling +old dublintown +mono chrome +lo iter +l pp +kur unzi +khoob surat +ju doka +holy cross +ho reca +hawk smoor +gr ich +go bble +exp els +er ath +elu de +d fd +clo aking +cc tr +c mac +bon homme +bo ga +bas si +bap t +ash esi +andrew scheer +all spice +ac ma +abrac adabra +ab all +ðŁĺİ ðŁĴª +à¸ķ ลาà¸Ķ +zay as +verme ulen +updat er +tweetyour lockscreen +silver tips +s design +ru chi +quintess ence +quetz alco +pul ation +pent acle +pan ton +night jar +my d +mouss aka +mount sinai +man gle +makar ova +let smo +jeong guk +i ef +hop mancup +honor arium +ha sse +go sse +g pg +fawad akhan +fair fiel +dur u +dream time +down river +culmin ates +cul shaw +co topaxi +chlo rella +chipp endales +chatsworth house +brit awards +br ina +asymp tomatic +amar aj +ai vaz +ag at +a stern +! ãĢij +âĢ ¾ +⤵ï¸ı ⤵ï¸ı +yu ppie +women lead +vape community +un deserving +t birds +sun niest +spring wood +sp reading +slow poke +see ther +sche matics +sa zer +s grove +ru drak +ran ma +quoti dien +ober st +o han +nl rb +me o +len nard +kon ica +escu dero +elli ptic +dubl iner +do brik +dike mbe +desig nations +dam pened +cun liffe +col burn +blo go +bann erman +with stood +want to +vanc o +twit tb +tu lo +trend setters +tor ists +the drive +tau b +syri acrisis +stay lor +skit tish +site wide +silen thill +show reel +rosson eri +rip curl +revo kes +por thole +north cott +no aa +musk ok +moy les +mountainde w +mopar ornocar +mont ella +middle earth +liven ation +issu er +human izing +highway man +he ee +gr attis +go bigblue +ghost writer +food love +fo tball +fabian ski +en si +dit ton +dead shot +calvar y +brown wood +best solo +b ss +apar ker +an ty +ae z +accessi ble +aa al +. )) +! ðŁĴĭ +your i +whar ton +wee tup +u tre +truff led +tro cadero +tex ts +syndic ates +street s +stal gia +skil led +shro ve +shin geki +sauvignon blanc +ro hs +rex roth +ra us +pi ggly +pc mag +pap y +pal an +out lasts +oss ining +min ecraft +mccl ane +lec on +le ws +kon zer +kat eri +j illy +j ades +instac at +henry lau +he elan +hal dane +gor n +frangi pane +ever clear +eric afernandes +el low +de sy +cat bird +brother s +breck sville +be ha +bal kh +av anza +au sty +ar ama +an ello +an berlin +about time +!!! ' +âĿ¤ï¸ı âĢį +âľĬ âľĬâľĬ +woo dgate +winoo ski +v old +turn coat +trail run +ti psy +thibo daux +theli um +the house +ter os +stoke ontrent +sp agna +soo th +sn c +sli mmed +sid ley +schle gel +ry uk +rive ters +ra kan +quiet us +probab ilistic +popu ps +pap is +oun ge +ner vosa +nat us +motor point +mit ford +mike portnoy +med field +mayo ck +lyn wood +local music +live shere +laun dre +im kristenbell +iheart media +har ps +go aztecs +gi vin +gc saa +gar bi +far sley +em ons +dw stweets +crou ton +coo te +captiv ity +cake walk +bud weiser +billion th +bath and +app sec +andrew zimmern +aj inkya +;__ ; +visu alab +us yd +univ miami +thebody shop +team exo +swi l +shadesof blue +sh murda +senran kagura +schnit zer +ra elynn +olivi awilde +objec tionable +oak ville +nis sang +n lb +mai ko +jon o +ir relevance +hollow crown +hel pin +harb ours +elle uk +dam pening +cobble stones +brooks running +boysen berry +boston herald +be all +ane mic +all new +alder leye +alan carr +ab w +!!! :) +ðŁįĮ ðŁįĮ +æ Ķ +zu ma +z v +winter land +wild side +where the +vu j +un du +tr onik +th ando +templ ars +tart ans +syl va +stur gill +screw fix +re discovery +q os +petiti oned +per ce +pan ti +oxid ants +out buildings +olivi am +nis sa +new stead +milon akis +mc neal +jobo pening +hub caps +hel med +grant chester +frigh tens +fin land +emoun tain +em eli +ef am +easter rising +cour vo +char tering +cari oca +can ing +cam of +br acci +bol tup +bobro vsky +bend iciones +bbc scotland +at tah +ag ner +åij¨ å¹´ +zapp afaye +yul in +wx w +tur vey +the clonewars +sh enton +rashidat laib +r fd +pu gin +pu edes +plat ino +pa ik +nichol a +n ough +mou ret +mal pensa +kou n +ke von +jit bumba +ji ao +jessic acap +imperman ence +ic ket +h anne +equ inix +distr acted +dis regarded +di ah +con focal +compac ted +clo ss +carat selcaday +cal lies +cai roli +bt son +bmw usa +bee ton +ambo seli +align ments +ali ghi +ya hu +y uli +water dog +w cia +tink ler +the star +thalai van +te f +subli min +squ ashes +sol arium +saveour nhs +ra gon +power breakfast +phoenix ville +nel sons +naz riya +mani folds +m spartner +jir ga +jaco bean +j rue +ite it +hon das +gu eye +go yotes +forest day +e ton +derby uni +dding ston +dac tyl +cum bre +cork town +congre gate +cincinnati zoo +chal ke +cado gan +breeze way +bol us +bernar din +avi da +al ooza +yo jimbo +yash ica +y app +wpmoy challenge +val es +up draft +sub nautica +stic ism +ss rugby +sr ally +sli thering +sli mani +ship ka +sex tu +sci fis +samsungmobile us +ram at +r nas +queen elizabeth +prou dd +playing cards +pe tes +park gate +pan vel +ola fur +mahan ati +lm f +kie ffer +ka il +harmon izing +har is +ge onews +gam bles +egoti stical +deac oness +cu sick +cruci fy +cheese man +cat amounts +bread stick +bla g +at il +as wj +absor bers +âĿ¤ï¸ı ðŁĴľ +âļ½ï¸ı ðŁĴĻ +yo gic +wil drose +vidy arthi +v artan +uch is +tw ar +think pink +team ireland +sw on +sho orn +selec tivity +sa ola +ron ces +ro it +rep mike +ra sen +pur portedly +pu skas +pu lang +plur ality +pi ppo +phosp hor +pet co +pad locks +osho di +mun cy +like it +la ssies +ko ss +kar ns +ha ki +giro la +g ado +fiel dre +ej io +dol ley +digi day +di om +destruc to +deci bels +coder dojo +cc s +brown ell +bon ic +aw adi +ath am +as pas +arag orn +an oop +." âĢĶ@ +ðŁĴª . +ðŁijĢ . +âĶ ı +visit nland +under shirt +un becoming +trans gender +thelim it +tat anka +su omen +stpauls london +shru bbery +shi fu +s bir +re dedication +play pokemon +pen man +p hair +oy al +ot tum +nam ie +n antz +mio cene +min a +marcel le +mali on +liquid ated +lampe dusa +isleof wight +iron stone +intell fusion +he ba +ellen page +e tro +dheeraj dhoopar +dance sport +cra po +co ghlan +clearthe shelters +cic adas +chec kyour +ca relessly +breaking weather +bor sa +bestsolo breakout +ati vo +ani versary +amnesty uk +al tea +ai guille +af ka +abhi jeet +aap g +] # +ðŁijŃ ðŁijŃ +ìĬĪ ê°Ģ +zed ong +wheel ing +wh oooo +westand by +v cp +un inspiring +token ization +to velo +tea shop +strongh olds +ri yal +re ino +ravin ia +not weak +marine insight +lo yee +krishnamur ti +junior doctors +jock strap +jabodeta bek +grid locked +grant cardone +gl g +ge se +ge mann +gas works +ero ticism +dru s +confed cup +ck g +christin ec +chick end +caitlin moran +brand new +bike toworkday +bengalur ufc +be sa +as ano +ðŁĽ Ĵ +ठī +zav vi +ye ster +wood field +winter ton +ucl h +sylvi a +stev el +si bat +shep ton +shab an +service members +sep ta +sassaf ras +sa ther +raf tery +press freedomday +posit ory +pi v +otr b +nino y +ne of +nag ra +n te +motor ama +mom e +met in +m tr +lis land +lac tobac +ky iv +kil bane +kat em +insinu ating +infr inging +ho pson +green blatt +gar nets +exp ander +droyl sden +cot ash +col an +clou draiders +clau ghlin +ca jon +bc p +bae za +ad hm +ðŁĺĮ ðŁĺĮðŁĺĮ +ðŁİ¶ âĿ¤ï¸ı +âĻ ¨ +x at +waq as +up standing +sy ny +skul leero +shan be +sham ir +scare mongering +s zo +ro mulan +re clai +quadru plets +pro enza +pop vinyl +plain ville +pbb abscbn +pati o +over nights +or sini +once always +old games +nam on +mer thy +may port +mat la +lo vitz +kul ick +jessicacap shaw +j wa +it c +hou ck +happy life +grey stones +fronti spiece +found pets +fer rera +fanta st +duck sunlimited +doom ben +deniz ens +costab lanca +comi enza +cli matologist +cheat sheet +brit geosurvey +ble a +ban corp +aquil ino +ann nd +aes a +ad ai +aa sen +ðŁij ĥ +word stoliveby +vesic les +vach on +unbeliev ers +un needed +sof unny +sinfoni etta +si pp +shin ny +shel agh +same er +rs gnl +real lucylawless +r wc +r ccg +pi ps +paste magazine +ni j +me tries +mc gavin +mc broom +mal d +low rie +k ericho +jam ma +in japan +i fex +heritag es +hb h +h z +green stein +gotg vol +ga ik +front page +fi endish +electrocu tion +des sen +cro ma +con sign +clasi fic +cir i +bab er +air drie +ðŁĺ± ðŁĺĤ +zi ering +zerflin music +ye ma +yam a +wk ts +wis sen +whir ly +wedding ideas +w rens +v dm +spin ster +soli dari +sj h +she tt +scot tories +sar af +roy all +raf al +po h +pirates ofthe +pd q +omg trolls +nz veng +north point +movi da +milk men +mer ri +ma ines +lecture ship +key rings +jil ted +hypnoti zing +hydro xy +heu sen +hal con +gro sir +grays lake +good life +fyn bos +equival ency +eg go +eber hard +cub bies +cri sti +coo ts +can die +brain pop +batt enberg +avi les +am fa +. ðŁĺı +ðŁijĮ ðŁĺĬ +ðŁįģ ðŁįģ +ðŁ¦ Ĥ +ðŁ¤ IJ +ãĥ ĭãĤ +âļ¾ï¸ı âļ¾ï¸ı +was l +uk team +transcend ent +tendon itis +su shil +son iq +sig g +si sodia +shut itdown +reali stic +purpu rea +pper son +pomer anz +pay o +oral history +ol g +nonchal antly +milton on +mid summer +man deep +mal uku +life boats +hor sel +glob alized +fle shed +el ta +do gger +dishear tened +di ko +den geki +cone head +cold brew +closethe camps +chino y +cbs bigbrother +ca el +bak ayoko +as per +alex andre +al chemical +al ane +ade k +~ *~ +wheelchair tennis +waj ir +vivid cloudof +vividcloudof wat +un abashed +thisi sp +stra at +stigmati sm +sel fa +ric on +pre pper +pearle scent +pac west +or geron +o kuma +liber tine +kw t +kot la +kar ai +k sn +iz he +human istic +gui se +fo ton +er au +ell sbury +ec chi +dischar ges +dead lier +daylight saving +bow len +bal ki +bail ie +avo ice +art school +air lie +yl land +ye urope +y eni +waga h +vo ya +stat ically +spo de +sou led +socin n +scholastic uk +real hiphop +queri do +ps as +pi des +pe lee +pass enger +om ri +ol ley +ni ms +n mt +mo pac +man ang +ma stro +m kiii +lili um +ky ary +ku mud +invic tu +hotch kiss +fu si +fro tterdam +fren kie +fen land +fair lawn +et ou +dor r +delu ise +concub ine +can by +baren boim +bad bad +ambi valence +alleg an +x po +wi ggle +whit ty +vegan hour +tu dung +tom asi +to too +thy atra +thankyou thursday +t dw +t aga +ss ay +srini vasa +siddi qi +sh ly +round houseldn +ra ser +r jr +nor de +myth ical +mel kwe +lin ch +kal ah +jes ús +illa i +hel ple +goodmorning world +gabrielle u +du in +crony ism +chy ler +boot strapping +at ol +ar apa +all ent +abra zo +aam c +ðŁij ¤ +ç o +wat kin +u fl +tom ica +taste maker +soo thed +shop local +sher m +sell ars +seasi de +ry delr +pro spe +pier ces +pho tonic +pe hle +p bt +own your +op ryland +one timein +npl vic +non alcoholic +mor ado +m gc +lim ou +l uni +jo well +iter ror +ink ster +gru ppo +gross mont +good boy +gaw ad +ft z +expe diti +ers onal +epsom racecourse +em men +edmon son +cri mping +con eflower +comple ment +chis inau +cal abar +brother sosborne +beautiful bc +au be +ari i +am historymuseum +alber tus +? ': +wir t +wer m +thie baud +suc i +set suna +seo tips +ro xx +ram ire +proge sterone +peter sham +pan op +p fe +origin ale +oooo ooh +notthe fake +noshaven ovember +nor i +nom ics +nd hakoo +mm ac +mau moon +lu pin +lo effler +ling erie +leyden pride +kinder sley +kinder chat +ken o +ke strels +k br +izz at +ie tta +hyper space +home schoolers +hesit ated +h nk +gre bes +gomor rah +fox ton +fairi sd +eating disorders +dart board +da ura +cy fairisd +chri smo +chero v +cetace ans +calum worthy +ben tos +amo on +ak rapo +a forum +" ...... +âľ ´ +virgin australia +vet kin +un substantiated +u tra +tun able +ston ed +st p +sp ong +sn h +shint aro +scsu huskies +robbin sdale +pre st +peds icu +n fi +meg turney +marin el +le stat +lb v +ker sey +katiec assidy +justin welby +ip c +international womenday +hy orin +for lan +f tbl +en ny +cra dling +christmas market +chand i +buffe red +bro u +ben cic +aspen cer +arsen als +aron son +air cel +ðŁĮ Ĵ +yeg music +ush our +twee tsnow +tru tv +the magicians +t ä +subjec tivity +sson oma +share it +ri vend +rap tured +pur l +public ans +prou lx +out lay +o ggy +net scape +navi mumbai +mu amba +med x +magnu son +ken es +ka jang +k ask +jay pee +j gr +its gabrielleu +imagin arium +horri fically +hello games +head long +hal ili +gul ben +good land +foul kes +fo ch +fin ned +er day +dissip ated +deadpool movie +dakima kura +d ort +chad stone +book outure +biof ilms +baf anab +b co +as besto +ali g +actor sathish +èĭ± èª +wei der +vitru vian +th appen +tele fun +tab as +summon ers +sugar ing +steel ernation +spring water +sla very +shore lines +rund gren +ro enick +piscat away +papu anew +n car +mo gu +megap lex +matt kenseth +man than +laurenti anu +la tha +hong dae +heim at +gil ford +gh ot +fle ure +fil ton +eo in +elizabeth may +el ander +dys art +death of +by ways +brush pen +bmw pga +au kee +ant il +angu lo +amyg dala +$$ . +ðŁĺĮ âĿ¤ï¸ı +ðŁİī ⾨ +weare winter +w tic +w shh +vintage hour +ven ir +the ta +ter race +tee tering +scri ms +root beer +rik mayall +ra fc +pic ayune +over party +over blown +ostap enko +optim ally +ool ers +ob verse +noble sse +myfirst tweet +mother less +micro biologist +metro schools +m ations +lipp mann +kh en +ion ization +iam ami +homes ickness +hin ton +h ce +gw u +gel b +fur lan +forum keralam +filmmaker friday +distinc tively +crowd source +by en +blon dies +bel ay +ar boreal +ak ur +a ford +ðŁĶ ĸ +water ship +voltron legendary +ul san +thereal grimmie +te sa +stonybrook u +star bound +ss si +snew video +sh we +sc bs +sar awak +sangu ine +rough er +re release +pro day +ns be +north vancouver +north side +myfox la +mur kowski +moo sonee +monte vallo +mohan raja +men ang +mehboo b +mac aques +jan itors +home kit +fond ation +family tv +emp y +e pl +dun ston +don ner +cowden beath +brun dle +blogg ingtips +blan ka +bick ley +bau r +bas sano +au rier +ari zzo +" ..." +ðŁĻıðŁı¼ ðŁĻıðŁı¼ðŁĻıðŁı¼ +ys rcp +wilson ville +wheat field +vo gels +vig na +v sl +tweeter app +special isation +song tweeterapp +sole imani +si mula +sher win +se ia +sanat an +reinvigor ate +qu arts +pou lenc +physi ologist +pas sa +own thefuture +ov ations +ol dee +oldee ire +ni ue +n pp +n joy +matthew modine +lin h +lam jarred +ke irin +kasi ak +ha ire +h pg +gn n +galatasaray sk +fran ky +eng sub +en nui +dishe veled +discrimin ates +dh s +cultiv ars +conco ctions +con tempo +cl anc +bu cc +bo bb +b aug +architec ts +ðŁĵ ¨ +ye ow +wieder sehen +web ber +vol le +visit ma +universal credit +star rett +sport bike +scre e +san jana +rssur jewala +re ssing +pr ings +plac erville +phi de +ou sp +orche strate +or ity +norri stown +nb hatt +mo yle +mil lett +meshu ggah +math ilda +ligamx eng +kra s +kip mooremusic +k ss +ju gaad +ital i +gon dor +extra dite +euro tunnel +eu chre +differenti ator +d weck +cru tcher +cor bi +co cor +co ben +cin na +cc ny +boun ding +boo ch +bit ly +b ence +ant ini +aden uga +ðŁİ« : +whi ston +vince cable +ver ratti +tur ton +tra ppin +thisi ss +thel akes +the sea +tam ago +tal cum +tal an +suc cotash +su ria +sig ab +shine e +select men +schar f +say ulita +san j +rugg ero +ren sburg +o dish +ne te +nah da +n ris +mo haj +mathi eu +mart ineau +khadi jah +kay ano +kanchi puram +ib mi +gun ma +gh all +form h +fl acci +first lady +equ alling +corrup tible +co bi +ci vet +ci era +cab alleros +bartol omeo +av ons +anjel ica +al ci +ag t +af tn +aci dosis +ðŁ¤· ðŁı¼âĢįâĻĢï¸ı +è IJ +yyc bike +x files +wit suniversity +web pages +wa ard +val in +un set +typo logy +su wanee +stockport county +startrek cbs +stan e +shor ted +ri stin +ray ados +pre hospital +pol di +pain swick +mou n +moto america +ido wu +icec aps +god splan +fre sne +for i +fich tel +fe te +f soe +embarra sses +ebon yi +disappro ving +comic palooza +cephalo pod +cassad y +bun tu +bu shi +bri ang +ana thema +alter net +adam m +ad mission +ac nn +. ¸ +ðŁı ° +yo gri +wh ey +w cb +vel ar +symph onic +star ted +spide y +se bring +sc abs +sankal p +prate ek +pleasee ee +per icles +pater noster +par ag +ob le +nr k +mun dele +maxillo facial +ma un +m jr +m clar +keeneland sales +k live +house keepers +her ta +gu mede +gor dan +fil oil +dor mouse +disaron no +di shap +d pc +col tford +cmn hospitals +caer leon +bur gas +bi shan +bha ag +as roma +armb ar +accept ances +âĨ ĺï¸ı +vit o +var gas +sv f +super fight +ste eg +sil ke +no ia +michael fassbender +mat azz +lax airport +ku bb +impal er +illa in +hyper pigmentation +hand ers +ha ken +good vibesonly +fuen girola +fixer upper +finding dory +f ale +ep ass +e sters +dr d +de mel +co ir +climb ing +cl attenburg +chab on +apalach icola +an tiv +ame en +ach ie +a sty +women entrepreneurs +way sify +us in +union town +stopthe cull +sport f +sper formance +show match +rw th +real sociedad +readabook day +ran ey +prosen jitbumba +per kins +pas cosheriff +palli ster +nic ci +mis representation +middel burg +mau dsley +lun gi +li the +le fts +hu at +hoh ner +gim let +ge tup +food land +fo gged +eco build +dee jays +chimic hanga +bug bounty +be hl +ann ada +ag st +ag fa +adren al +ðŁijį ðŁĺģ +wr l +wor tley +william levy +w eng +un pretty +u oe +tugger anong +tho da +st michael +son amoo +si apa +shar ron +shar pies +shape shift +rn li +rick ross +ri y +ra spy +q be +pe saro +parap legic +panch kula +p summit +nin anes +mis adventure +mb atha +march es +mar gery +lod ger +letit go +knife crime +kn p +ing i +i eds +fat ca +eh ring +e by +diecast malaysia +den ia +cur rumb +cran well +cooper ative +car tes +biomechan ical +bar letta +aw right +arm ing +and or +a ash +yadi er +whe en +wh k +wat o +war ding +v ns +ura wa +un determined +u dm +type setting +traw lers +the square +t world +rob it +r top +penny stocks +nic os +mk b +maroo chy +manzan ita +mag ar +luc ile +le eming +ix tapa +hamble ton +hall statt +hair i +gym shark +g omg +dun s +dendrobi um +de perfil +cister cian +ci pla +charli ed +carac al +boriso v +av geek +arn cliffe +ano u +al wa +ðŁij ĥ +ëĿ¼ìĿ´ ê´Ģ린 +ç Ĥ +е ÑĢ +| ... +z otto +wrink ly +ultr arunning +tric hat +sy ros +str itt +spider gwen +so won +sla ppy +shekhar gupta +sel u +sa hil +pre mi +plat er +phone tics +ph led +peter son +per sol +p mpt +ow x +nis bet +na ic +mor dred +mon da +moha patra +mess rs +me ola +lo chs +ligam x +just sparkles +joyl enz +journalis m +jig ga +j ore +ig d +hitch hikers +harb ison +hallucino genic +goe de +gg ing +garl icky +freak onomics +fr sc +evangeli stic +elem chat +dread central +do er +bu ju +blood drive +beau vais +bar fly +ark low +ard ine +zi pping +z id +winter storm +wedding inspiration +ward our +tothe top +that guy +tas cam +sym pho +stu dley +seattle storm +se ey +sc up +revo king +poon am +po v +philly dotcom +one z +mar antz +m tuan +lack land +ku chi +kno tes +ki do +ju mu +jo ve +je ux +jaco bl +irish music +inst ra +indo china +hy mn +greg james +gold crest +gingi vitis +electr icals +east ers +drif field +dal liance +dag upan +cra il +comic sgate +chinst rap +ce de +categor ised +cap sizes +camill eri +but chart +brom eliad +brad well +bee day +be ds +barangay ginebra +b wr +ay so +yo plait +west fal +wee bly +un restrained +team om +tang any +spo leto +so bo +silver line +redrock sco +re boots +ramach andran +pap o +ob liqu +nico ise +nanom edicine +my plate +mongol s +mo gherini +mn ts +megal ith +mac ap +ma ggy +luis fonsi +lor g +kooten ays +kne pper +king a +kar yn +k to +j ro +i hh +hra b +hit sm +guil lem +ghe or +geome tric +feed back +falsi fied +exacerb ated +durban ville +din ky +di ep +de pop +comp tes +co sima +class dojo +choreo graphing +bub sy +bet v +ana gh +adhe red +ìĦ ľ +yu rek +work group +win di +wal lowing +valley usd +un common +te rel +sky atlantic +si gi +scra wny +question time +pim co +pe sta +pap worth +ox bridge +os wald +on z +mont ville +maid stone +logi x +li pper +jc icki +isra elite +il ac +gr illes +g fr +feature film +fal las +dw news +drew scott +dd ar +davi da +craig avon +comefroma way +brox bourne +atri athlon +appu ram +ðŁĺģ ðŁĺĬ +vacuum ed +un seen +trick y +tri ght +tor ff +tachy cardia +standar disation +st day +sje arthquakes +score boards +sco ter +ronan official +rit ch +ra gu +ps ers +popul ation +paedo philes +over supply +oc dsb +ny topinion +nicky hayden +ne stor +nau gh +n devon +morgan ite +man al +makk al +lim ps +j ellic +hayley kiyoko +harro ds +ger vinho +faking it +fab u +ess ere +crime family +ci vi +che ever +charter house +cal avera +cabbage town +butter ick +biglotter y +bene teau +ba bette +at ua +ab abes +... :-) +ðŁı ĵ +west mead +west ford +voice overs +un motivated +t ander +stand free +si biza +shop front +sha am +sc ross +sawh ney +sau gatuck +re claims +r ht +quart zite +plough man +parab ola +mon ade +molyb den +marks manship +mari ja +mahesh nbhatt +ker ch +invent or +hun ke +hr n +hou sec +gary sinise +every where +equival ents +elu des +ebook deal +e ge +do ona +dex press +daniel aruah +coch lear +circum vent +chapulte pec +champ neys +blackkk lansman +ap ie +am yl +all ys +ðŁĴĽ # +ðŁij» ðŁij» +writing prompt +val ar +un tidy +sush ant +sport chek +spad eny +s gen +read the +quarter master +prati k +po th +pat aki +ng supereagles +nc bi +mand ating +mag er +loch head +kam ar +imper vious +hel len +he u +gi ang +geni ushour +ferv our +fast break +exor di +exhilar ation +europe o +envis aged +ed gars +ear drums +diabete sday +cor tese +chri smur +can ali +bap a +ðŁij¶ ðŁı¼ +ðŁĮĪðŁĮĪ ðŁĮĪ +wark worth +vra bel +vintagec lothing +u im +time a +south view +shol t +sett le +rc g +ra ws +pre positions +pr ama +pencil art +out for +os valdo +ore y +ni oh +nabeel rajab +med gar +lif ford +kol sch +kaz uma +it sb +ilove cornwalluk +heritag emw +gra p +gall eri +for dgt +flo y +fel stead +f ony +entr ée +dy sen +custom made +cole shill +car vel +cam m +brun chc +bmr tg +blue coat +biggbos stelugu +al pena +african american +afoto deperfil +-- , +âĺĦ ï¸ı +âģ¦ @_ +zz ari +wood stock +visit the +trypto phan +ton katsu +tol d +stay in +st aci +sm ale +siber ian +runner sworld +pil on +on th +nun nery +nbc chicagopd +my ke +mp cc +milli pede +mark man +info security +il aria +hul st +horri ble +gli mmering +ge she +foam cc +eure ka +e glise +e buka +d allin +cru elest +col lo +chri sma +chi arelli +buen avista +braintu mour +atlas v +athom pson +adi za +ad ance +ze sti +yosemit enps +wa fel +ve ined +v ball +urijah faber +thanksgiving with +stu pe +sn krs +slen derman +si ku +retail tuesday +repul sed +re occurring +ra ed +qu tub +plac ido +nayanth arau +mutual funds +mi fid +mer ve +mat atu +mah wah +kri stel +ket ter +indi ant +groun dup +giftsfor him +g do +fume tti +fa sig +dak ar +d bp +croo kes +com per +clue let +cit é +chris van +cc sa +british redcross +bone head +blanchard stown +bbcradio scot +bab ington +ba is +all time +] ; +. ðŁijı +æĿ±æĸ¹ç¥ŀ èµ· +âļ½ï¸ı @ +zil ch +yn i +tri sta +ti wi +the team +sy phon +stellen bosch +selec tion +sand well +ri bcage +pest ilence +mh mm +mention someone +mb ale +made me +ka uri +hero ically +har um +gon nabe +field day +ec ats +droo g +do di +devast atingly +demarc ation +daw id +com pra +co die +ck l +chil well +cherry blossoms +cb fc +bet u +arn alds +ad ger +ðĿĺ ģ +öz demir +yogri shir +year old +yakis oba +tri ennale +to pen +ti enen +thong thursday +there ale +sá bado +sso white +so yy +pu sci +privati sed +physi os +p tf +over active +ob on +modu late +member monday +mc nairy +mai read +lex press +ken worthy +ke shia +kati emc +kap it +k icc +go omba +go blin +fo lies +far mall +eye con +dispen ses +di aph +bro y +bre r +bal combe +art scentre +applec ross +aldubi yamin +aali yah +ëĤĺ ìĿ¸ +z una +wil le +stock room +sk rona +sister patriots +sd aughter +river wood +puig demont +political prisoners +pharmaceu tical +pal atin +northumb rian +ninanes bitt +newstalk fm +mole skin +le hane +ksen ias +khor asan +kat u +jab ber +j ón +it ttt +ini sta +ic kel +hahahaha hahahah +gn ite +git te +fan cave +exc ell +encum bered +eden project +dism ount +chris webby +body positive +bc sd +ash land +ang mering +ðŁĴľðŁĴľ ðŁĴľðŁĴľðŁĴľ +ठļ +Ûģ ÛĮÚº +é ry +zzzz zzz +zet as +woolsey fire +vis ital +ve ja +uniof hull +tric kier +toon z +tom be +the of +step toe +sten o +sp ake +sho sports +sam ark +s bur +priv at +power lifter +mc sweeney +longh ouse +kitchen ette +impeach able +hy nde +gu shers +gre mio +de merit +daf ydd +carcino gen +bulb ous +budd in +boy yy +bla sto +bertr and +bandhav garh +are cibo +ar shavin +al mora +! > +ðŁĺĤðŁĺŃ ðŁĺĤðŁĺŃðŁĺĤ +çĶ » +xen ophon +ver dot +ton gar +sto ssel +sanjay dutt +road side +revi ve +rebu kes +rc v +p we +over flowed +onet ough +oik awa +ni fe +nfl onfox +miya jima +mg f +madewith love +leg aspi +lac as +kae de +k fm +hilari e +heath land +haw ker +hack y +gor ill +gl vc +gate shead +fli k +energye fficient +em mer +ele mans +eg gheads +ec rew +corri b +bot any +bol sover +bar oni +b we +alpha ville +a world +zur ita +un injured +uk zn +tri kes +ti ket +thick ened +tele scoping +ta ung +sunglasse sday +sun shines +smoo ches +sh ate +service dog +rhodod endrons +republi que +rating theraces +q urbani +p ma +no amnesty +nashvilles ounds +lucas arts +love child +lot ton +li bel +kar men +kad in +idy ll +iam johnoliver +hold out +haup t +groom sman +gri mmers +gl ay +gee chee +flap jacks +fir med +f pf +europe aid +enam elled +eco sse +ecker sley +don mar +disco theque +d mar +cad w +bri oni +ballin colli +avalan ches +alt adena +ðŁĺŃ ðŁĴĺ +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤ +ðŁijĩðŁijĩ ðŁijĩðŁijĩðŁijĩ +ðŁİī ðŁį» +ðŁĩ§ðŁĩ Ń +zul fi +yogrishir amdev +wu stl +white throat +ven ere +un furnished +un changing +truck loads +the artist +summer jam +sr ite +royalwelsh show +ro don +rickie fowler +reson ance +re scheduling +r witherspoon +quar les +pin n +pil cher +p kc +p conf +omar u +official pompey +mu ting +mla denovic +mi wa +mb da +mar v +kit ana +jar l +intu bation +her metic +ha ise +gl vc +first warn +et weet +breakou tedu +bor din +bee gees +barbar ossa +as chools +an elka +amic able +americ ane +am ap +a jones +âģ£ âģ£âģ£ +zi zek +zell weger +viz caya +visit mexico +truck tuesday +trab zon +the joker +sham bolic +sali vary +rico harena +re writes +prin ny +porti mao +pha eton +par bat +p sh +norman ton +memor ise +mar got +malle able +made easy +ker ley +k pn +izhe vsk +itt f +id c +hau z +han am +fu gate +foo ters +falling water +faken ham +david labrava +dab boor +cros scut +carni vale +camer ata +bridg it +çµ IJ +âļ °ï¸ı +âĸ¶ âĸ¶ +प र +win canton +wair arapa +volcan os +veterin ary +upside down +ua allamerica +the wanted +tao yuan +t elli +szcz ec +stu g +sp c +smat tering +sel vi +se jour +scar boro +ri sc +r iner +o red +ncaas oftball +mo jom +mit sloan +mak sim +lur ay +long ton +litho graphs +ley kis +jo by +ha chette +g yor +for tress +f outs +e om +dissemin ating +datavisu alization +conoco phillips +choice awards +chitr aloka +ca afb +bur in +bis son +ben o +bbc suffolk +athur sday +aravindha sametha +ad hoc +ab lue +ðŁĮĪ ðŁĮĪ +ãĤ ģ +world class +win du +visit kent +tubri dy +tra choma +take alot +shore bird +sho ko +she aserrano +sex perience +refor med +rece ssive +re purchase +re heat +pu trid +pb con +o thr +o som +norther ner +nit rile +new relic +n endo +mwan za +mv c +moreto come +me co +lac o +hud dy +gar ners +fresh offthe +find hour +er mene +e more +dv ar +dog meat +denver post +demon ize +dan il +car sen +bouilla baisse +blo gher +at andon +alibab agroup +ðŁİĪ ðŁİĤ +ðŁį¦ ðŁį¦ +z ameen +y ata +video shoot +venkate sh +the tis +temper ing +stra thro +stodd art +sel sey +ru ises +ri ski +re house +puzz ler +pr é +po phobia +per iment +pav an +pare ja +on ur +mi sting +melkwe g +mc gee +lo omer +kumail n +gn awing +ear nt +dismiss als +degan arseguidores +dai ichi +dad u +d za +chang sub +carryon blogging +boardwal kempire +blurry face +barn wood +balo chi +b hin +amra pali +adam devine +acom pan +ab ro +~~~~~~~~ ~~~~~~~~ +wo jcicki +ve ik +user names +ton park +super bird +string fellow +stel o +sephar dic +per umal +pat ah +north ville +noctur no +naom h +nad ra +mv m +manzan illa +ku cherov +kir il +k michelle +ip al +in ster +hub bub +hr rr +gu stan +fy p +eli on +dizzy wright +disc ours +dabboor at +cu tely +brain ard +beat boxing +ban as +as kari +abo xing +you ll +world mag +w ms +urin ate +thunder snow +super bloodmoon +sm sa +skulleero z +sick les +sh inj +sesse gnon +samuel sson +ru din +road racing +river hounds +ride au +ra iz +prephoo p +porto frotterdam +polling stations +poll sters +pap el +ottawaf ury +nure yev +method ical +le ix +jame sk +in situ +ho sed +het alia +green acres +goo good +fé ile +fri bourg +frankie j +end ay +displ aces +dee red +bluestar zone +bhubanes war +beli ke +bag by +atar get +as ay +ariann ahuff +ag adh +ab ria +[ ( +âŀ¡ï¸ı âŀ¡ï¸ıâŀ¡ï¸ı +young man +weekend kav +vi das +var is +tot ter +test bed +t pp +swe tt +soo s +rut ledge +reddog susie +recer tification +priyam ani +poo t +poly vinyl +pleas ert +part ay +na iler +mount vernon +mo za +mill o +le eps +ld k +laur ita +kwas i +jay bird +is r +imag azine +il ac +grim dark +episte mology +episo dio +de colonization +chinois erie +chag as +cd v +cam bia +braz oria +br icht +blu d +bal as +athen ry +as ps +apr ès +anu radha +alt balaji +... [ +âĿ¤ï¸ı ðŁĴļ +âģ īï¸ı +ó g +ye aaaah +visit jordan +u og +u dan +tso go +trench coat +tr ite +theroy als +th atha +st illa +shaw ano +sat anist +ros lindale +red ban +ral phi +quar to +pra d +pr ang +ottawafury fc +naom ie +mysti fied +museum day +mu ff +menstru al +mediab ase +lgbt pride +leop old +ke ley +kal ank +huski es +hebden bridge +harmon ized +gye on +green lit +geo ghegan +eye opener +every child +eneed le +e ure +dian er +di ers +day fm +cru shin +credit suisse +clon dal +cle ves +ci onal +c itta +busines strip +body paint +b ous +app enz +**** *** +ðŁĺ¢ âĿ¤ï¸ı +ðŁİī ðŁijı +âľ ŀ +wheat land +westf alia +v ade +time slot +t anta +svel te +stri pey +spring summer +seeyou there +schi sto +sang er +ryan seacrest +rother y +ross ellini +r re +pu e +porter field +pj m +oo voo +o lean +nephil im +mariach is +lole sports +kim ba +jung woo +jo anc +it ll +ish ant +herma ph +heck in +hair day +gh onetv +folk tale +eo cene +el wes +ed dsworld +don nington +distor tions +digital camera +dark wing +cor tin +chi di +chat man +cath ed +catapul ted +bra g +boat load +bo ddy +blow torch +blo cky +bio bank +beat la +bb sr +au dra +andre ab +afl womens +ðŁĩ©ðŁĩ ´ +à´ ª +z eds +your story +x j +wet ness +vla dislav +vin ales +ver beek +tw irls +thor ity +t chami +super be +sug den +strat for +stormy daniels +squan dered +south bourne +semin arian +sar godha +sa ima +reven ue +re tell +rb x +rain ha +post operative +perform ative +peck ham +organ o +or ak +oh en +nyc pride +nir mal +ni ños +manic ured +manas quan +makers market +london ist +ko jo +ker to +k bd +j tc +intermitt ently +gano derma +fa ch +ev gen +equal ised +e oy +e health +e harmony +don ee +ch strfc +bint an +ar man +ajac cio +afa una +wre aked +tul sigab +ti kal +texaste ch +t ative +sweet potato +stkil dafc +spar x +slip case +shady side +sal tz +r int +pran king +phe red +par ana +od die +o we +ni lesh +nar whals +n anti +mp hil +mat in +mag asin +lu mped +kop ar +ine dible +im bo +id leness +high end +gro aning +franch is +fo ssa +fo gel +em mons +e bbing +dro ver +co ppi +chin cote +ch ory +c de +bow lin +best memoriesof +be se +ap lus +a afc +" ??? +ðŁijĮ ðŁĺĺ +âĿ¤ï¸ı âļ¾ï¸ı +à¹Ģภ§ +wre aks +w psl +unil aterally +un predictability +tu bb +tofthe year +ti o +spenny moor +snug gie +snow cone +sims bury +segas aturn +sea eagles +ro sy +retro active +qu ail +polyam ory +par tisans +od t +nuev afotodeperfil +nca agolf +na ing +music photography +mun dell +mike trout +ly sol +le doux +le awood +jit endra +hus ks +he et +hazel ton +har rod +hamp stead +gor dian +glar us +gav r +fr ing +dend ro +d ch +com pline +cit ys +character izing +cel adon +carlo ss +bur ana +bun goma +bu bby +broad beach +at g +ðŁļ¨ðŁļ¨ ðŁļ¨ðŁļ¨ +yas uo +xfinity series +v itti +ut coach +the win +tex ans +ta kia +super heros +strang le +star child +space ship +s will +rin ella +penn relays +pede france +p nt +p lim +over bearing +off setting +n cm +more t +marin elli +makemy trip +lupe fiasco +love ireland +losange le +intu itively +go jags +gear up +g wal +esp adrille +dou jin +diam ant +d ace +c sic +bas inger +aspen institute +abcac tionnews +à¹ĥ à¸Ļภ+wel e +wee gee +wal mer +vol can +vintage findhour +un branded +thab sc +th ies +sp ars +serv i +sam ard +sabo teurs +pol icec +pal apa +olac abs +n cert +mount ford +mahin da +less er +her thabsc +ger aint +ep worth +do gara +degra w +de sc +cran mer +cn co +chand rak +bur han +buenos dias +be ki +bcb tigers +bab bage +ah gase +za w +y ls +x tr +trixie mattel +tis sue +szi get +sy co +spla yed +sd ell +ri pe +ri oli +pumper nickel +on thisdayin +ofici ales +ning en +mor bid +mc v +marcu m +lets be +l wd +khy ber +k tm +jig awa +j rd +hyper inflation +halloween town +greeng ro +disinfe cting +dis figured +dal keith +co ble +bon nes +bill mckibben +bar ç +bar uch +au dic +apol lin +ab har +william ssonoma +who ville +up state +tu tera +tilland sia +thedragon prince +the high +ta kra +t ating +super smashbros +schwe izer +sabar mati +rosco smos +remo ve +read ju +r tw +plagi arized +pi u +pen rhy +n nl +museum selfieday +mmi wg +minis kirt +man ek +ma ury +kling ons +k wi +joe star +g lowed +fol l +fl att +fi estab +eric son +eleg ant +e reader +du thie +di ano +col lis +car ami +boy kins +bor gen +bon do +bafanab afana +at ack +ar tra +acu ff +æľ Ī +yor ick +wa ar +w op +u jj +tun ica +ten orio +ta house +sum me +sel in +sec ity +sarab ande +ry d +reli ves +polynom ial +phyl icia +ol ay +nish ant +minneso tal +micro fluidics +medi agroup +make ba +mach an +long livelong +ligon ier +lam prey +karma pa +kar isma +kalk brenner +je ph +hot stove +her ath +ha stie +gro pe +gor ski +ga j +fene ch +fe ckin +f tii +ev ac +epitom ises +ch age +bran che +alim ony +⼠ı +wind shields +wi a +voltronlegendary defender +tulsigab bard +taxi driver +take action +swee et +spe yer +save ourocean +sarahk silverman +s dram +power of +phys icals +pe gi +pa ks +orec chiette +o ec +na vid +mn gop +mi ers +mel lie +lei ber +kit v +kil is +katah din +kar lo +jeff ery +imagin ator +huntington beach +hor ning +hereto help +glori aste +gloriaste inem +ghost face +fru ited +fac ci +f yn +er ac +ee h +e igg +dear ne +crow snest +compad res +charen tes +ce h +bo el +back rest +b fb +ariane space +alter na +alexand r +aldub happy +al ink +abolish ice +ze in +window less +vas anth +val entia +united for +tj x +tin ctures +thrift break +the mac +terra sse +te pco +sü d +so cr +siem pre +se sports +sar gun +reyn ol +rajasthan royals +rah mat +pro shop +phra se +phai don +per abo +p chat +ouri st +om nit +nex o +nebu lizer +nand an +more days +midwi ve +ll cr +li va +leadership matters +koto bu +ko he +jupy ter +jan ec +humer us +hindu stan +hac en +h mi +gun da +general mills +ever blades +en stars +dr ang +div is +di anthus +coll ated +car ditis +bu se +best musicvideo +au secon +alt as +ale igh +al tered +acet yl +wal ney +vintagec ar +vi pass +v qa +top billing +tac tician +t dt +t ct +suf fix +stu f +stick le +st ä +shen yang +se vend +ro sat +real talk +quil ty +pittsburgh pg +pin os +pil ly +perenni al +penni less +pacio retty +or onto +onetimein nola +off al +near ly +nau sic +mob wives +mandi ble +man ou +mal ing +jun gs +jer obo +je zz +je wett +hust led +hed man +fe tes +extr atv +dob bie +defi b +cory gardner +colum nists +charlot ter +certi fying +carne iro +book addict +blogger swanted +big fat +bethany joylenz +bassad ors +bas sam +ade el +ach ina +ľ ล +ðŁĶµ âļ½ï¸ı +ä» £ +ãĥī ãĥĥãĥĪ +ãĥ ´ +ye ver +world mcqueen +vir als +usav olleyball +to scan +time tables +thof july +te agle +takay ama +sunday brunchc +su port +solidi fies +shannon r +se cops +sch y +ru bia +ri gel +ravil ious +q ah +prof ited +pom mel +patri zia +paranor man +papuanew guinea +ou ree +nu u +neck piece +nag orno +mybe belove +mou thing +mot els +mar kelle +man city +maeste g +lu jo +jo sip +ihe anacho +hi dro +han se +go jack +fr amer +favor itas +down sview +cz ynski +contemporary painting +communic ation +bbc worldcup +ay be +... ðŁĺĤðŁĺĤ +) ," +ðŁijĮ ðŁĺİ +미ìĬ¤íĦ ° +wh darts +wal kr +vin ing +val spar +u ww +u ur +truff le +toe ic +team solomid +t tos +sub ha +so tw +sebo gier +sad dl +rob ina +re drawn +re counted +rand ell +pur slane +pt safety +pron ghorn +music ology +mur doc +micro transactions +massage therapy +mani kin +man museum +mal ley +mahersh ala +lion day +la pa +il x +huy ton +gugli elmo +gu de +for my +eu less +ess ure +elec tives +ea ste +dress ler +do et +dawn richard +dani ell +dan adel +cigar citybeer +ce ment +blue peter +bio based +be vis +b kr +arat ne +all ons +ai katsu +afri kan +ab it +ãĥĪãĤ ¥ +м и +visualab stract +un agi +ty t +tri bbles +the be +stau bach +so bbed +skel mer +scribblen auts +s fi +ride for +ric snews +red poll +r nation +quetz al +quat ernary +oce ano +noso tros +moon shiners +mika elson +mer rie +mam u +macy sparade +k ue +john wall +jay awar +ir fu +hu bli +h pc +gauri khan +feren dum +ejio for +eag lenation +batt alions +bajiraomast ani +any place +ann yeong +ang atta +af ton +:) "@ +ðŁĮ Ľ +ze ek +z ato +to ph +tin ta +te thering +sustain ment +stro ma +strading cards +still well +steven son +stepan ek +stella artois +so con +shi mada +se adogs +sabot aged +rob g +recuer do +ophy te +on nit +oc ceren +nap anee +nan os +naj wak +mis sal +marvel ing +ma hu +livel ike +ko co +ka isa +jimmy buffett +flo e +fla gler +fin dley +far row +even son +encapsul ate +el pi +dor ner +dhar an +del fino +de stro +conflic t +ch arest +canadas occeren +cad i +bj sm +being united +ban anas +audios lave +ati more +>>>> >>>>> +äº ¬ +âĿ¤ï¸ı ðŁĴ¯ +wx guy +wo akes +w ts +tro hman +tn pl +spice world +soft cover +row en +rock ingham +pul teney +plo ck +pa pped +note ven +mceach ern +manipul ations +loo keast +le mn +krist offer +jazzy b +iv ories +is am +i bus +horn ung +go in +gill man +ga iney +dis oriented +de akin +d wd +chlo ë +cannabis culture +ar oldis +anglesey scmedia +am iner +ø rn +yo v +woll stonecraft +weather live +vit a +uni do +tram adol +to inspire +sothe bys +shin ge +schne ide +sc ally +sack ings +sabc newsonline +ry bak +ro ku +reiter ating +photovolta ics +part es +north star +mu ar +moore stown +mondele z +mel ter +mark ley +mariab arti +mariabarti romo +lorela i +kkkkkkkkkkkkkkkk kkkkkkkkkkkkkkkk +ki ani +jo en +ili as +ian h +hy des +high times +hen ke +healthy choices +hat ley +grand finale +gh ali +dr strange +dog sat +dj al +dictat or +cor responds +cool more +com promis +br ø +bas son +b flay +b du +arri ver +apar ty +an ting +an schutz +af tal +ðŁijĩ # +âĺºï¸ı ðŁİī +wom b +wag staff +tv i +tss aa +shigh ered +seymour duncan +ser ra +sch lager +row ney +pll tvseries +oskar blues +ola bel +nd spdwy +mo ai +me tric +marke ts +manic ures +key stone +jor ja +jo gs +hoo kups +gre entree +gi est +geaux colonels +f ford +en b +dra is +dow sett +d wood +d fi +cost of +coral ville +city lights +chicagol ndspdwy +chic lets +ch acon +cas sy +be ps +ave z +au sf +aristo cats +ao ty +alme ida +ali ghts +ale ss +adsor ption +aac ta +[ © +ðŁįĭ ðŁįĭ +ãĥ Ĭ +à± Ģ +र à¤ķ +yum miest +wi ra +what toread +uki yoe +ton hall +ti oman +th ira +stol z +spon gy +sitt ard +sick notweak +sa recool +rott entomatoes +rock well +road tri +r dr +puertor ico +primiti ve +phan togram +par igi +mar ant +led ford +ko tel +kell yo +ke aven +hyper link +huns let +humi dex +horse box +graham stown +g alls +fr l +fa eli +du sts +down ership +de pp +da aa +cy t +cu ld +ch d +by gones +au v +ar tre +ar lin +... ). +! ðŁĺŃ +zu o +yacht life +win dup +ut knoxville +trumpre sign +toma hawks +swe ated +sw pl +stress relief +sti dham +so tt +sharemy cosplay +roger son +road trips +rhe em +plen um +peat lands +our house +on cers +old hollywood +mp inoe +maz um +lessi g +lay f +l gi +krish na +kail yn +jo ek +incub ating +ilovemy dog +hei dik +h wt +gun smith +goooooo od +g vp +fu mbling +fru strat +fis ke +f mu +esp in +encanta dia +e commerce +dal ec +cou sy +cal mac +byron bay +as ner +ar len +an anth +ðŁİī . +ãĥ¼ãĥ ł +âľ Ń +zebra head +wwe ajlee +vipass ana +trade winds +teoti huacan +ta jin +stud land +skir ball +sand ymount +resc u +repell ant +pre g +pou lin +pa quin +p bd +mondi ale +mis smal +micro grids +met formin +me ber +low carb +kw gn +kil n +kiku yu +kett les +intercep tors +fr ant +fagi oli +f bk +eri ke +doo b +d mexco +clever ness +clai mant +chatt ers +bez els +ban sko +af oul +ðŁĴª ðŁĴªðŁĴªðŁĴª +ðŁijı ðŁı¿ +zo an +wur m +viz ha +v sn +up end +twin sburg +trac ee +tower hamlets +theip aper +sun rail +sm om +sheet metal +sepat u +sel bst +scri sis +sco tti +schoo se +saturday vibes +sa pe +sa ori +regi e +oti eno +ntv today +mk ultra +micro waves +me del +man ta +mam bo +liv ro +le conte +krush na +ki ku +ke it +j mw +inhal es +industri a +heral ding +her op +ha al +gros beak +grange town +fon zie +du soleil +do si +deliver ables +dar roch +chi ve +brit athletics +be yourown +b bl +ar ge +antonio brown +a ure +âļľ ï¸ı +ÙĦ ا +zero waste +ww l +with your +windy city +wi gg +wether by +wa is +wa hy +thor ns +the over +th uli +sun kist +su bal +sport stradingcards +ski l +regen cy +ray bould +pin chot +pel ag +nikki galrani +na iro +my garden +moom bah +metro fmsa +mari k +log es +li sson +kn aus +kend i +ip so +indian ola +in tha +h sin +grey lock +gan se +fraw ley +fin epix +esh agupta +ene me +disin terested +dis jointed +cre vices +council of +cap le +calvin and +bird box +big d +bar thes +are volution +al ympics +ðŁĻĮðŁĻĮ ðŁĻĮðŁĻĮ +ãĤ ª +| -/ +zing y +zar athu +young living +xfre watch +val lotton +us as +up dike +un paved +ten ey +swk play +st james +spur snation +sound bar +soho house +sn icker +smtown global +shun ning +sen shi +sem tex +s zab +recomm ender +ram bo +radio logical +pre prints +pent ag +p bal +on ni +o sn +nom er +my story +movie quotes +movi l +mc rib +mateo guidicelli +mal apit +mac phail +lat ched +land rights +kr ann +khal eda +ked by +jo deci +harro p +gethse mane +ff dp +eyewitness wv +est elle +en demol +cmoh ry +cam illo +ble p +bio similar +bat ard +bas ant +ay ud +awesomen es +alber tope +adi ya +ðŁĴĥðŁı½ ðŁĴĥðŁı½ +ðŁį IJ +ائ ر +vat icano +va id +under pressure +slu mb +shun s +she ahan +service able +seguri dad +sau da +s gin +ri am +prep sports +pol ices +overestim ated +oko boji +nick kroll +newtown ards +marchof dimes +mar sa +lip as +kitu i +king sholm +intersper sed +inter webs +iiii i +engl ert +el nik +di kh +den v +defend theland +char an +c ics +billo frights +bern ards +beg one +ag ana +ì° ¬ +ÙĨ ÙĬ +ye ong +wheat on +vid han +ve spu +v indiesel +transform er +terra form +ta kk +t lou +t ago +sun block +span ol +scri p +rudrak sha +rhyme sayers +rest lessness +pl oring +photomon tage +periodic als +pen with +ou to +os ram +o gh +mul berries +mo wn +min u +majo red +mac aws +lon gridge +lesm is +kiren rijiju +ken way +harmon ie +growingup with +googood olls +gle ich +ford models +fo gs +eto sha +e wart +drjimmy star +cou pes +chakra barti +car ms +can not +bol stering +bha vana +auto focus +af elt +a uro +ðŁIJ ¤ +å¤ ª +Ì µ +x ab +wicker sham +tu mn +ten ch +spe ts +sage summit +run about +raw ten +rap turous +ram sar +pic kets +pantan al +oun e +nu yor +nu tting +mu bad +mor tise +mc guin +mar sdd +lu cey +lingu ists +je thro +free people +forget fulness +e zzor +dis regarding +del monico +cyber man +coldwar hist +cloud security +clo vers +clo stri +cdr kelly +brew ing +bra ssy +bor del +bill ard +be quest +b pr +apothe osis +am yn +al oma +afgh ani +ae an +adidas running +a stigmatism +a hahahahah +ðŁij©âĢį ðŁĴ» +à® ļ +who dat +whit sunday +uz ha +universal is +un garo +ty rus +tur geon +terran ova +tan war +stam pede +sport sphotography +spal omas +sam pa +revers als +re tracing +qab oos +power scourt +pi vo +petro chemicals +olive garden +official ap +nh c +mer z +marque try +m brs +kir ton +kat ra +is ser +infer red +improvis ational +hu ey +hi j +haringey council +har pa +ganon dorf +gabbi garcia +full sail +fol tz +fly laxairport +fertili ze +ec a +e gc +du plessis +di w +d mh +cut ter +condolee zza +car reno +bothvote ssnp +bit ton +bi atch +bethe a +ber tens +bar ch +bac al +at ras +ashok gehlot +artist oftheday +am and +af ai +wo den +wfu v +t pf +syl mar +subjec ting +sub woofers +steve madden +station cdrkelly +sen ter +sea star +sc f +sal wa +sa af +ro sina +red path +recy cl +preston wood +poisoni vy +percu ssive +page ant +over dosing +not atarget +nb colympics +msin spire +milli second +masa shi +mary vale +leg om +kille brew +keep talking +ist ill +io annis +icici bank +ich mond +health matters +guar ana +goun a +gon ne +ginand tonic +gas coyne +first dates +fc art +f pe +f mv +du le +discoura gement +diam o +cu mann +cougar pride +c sac +blu bber +bettere veryday +archite ttura +arachno phobia +an cha +aly x +ðŁĨļ @ +éģ ĵ +à¹ĦภĽ +w basketball +ve rena +ut ani +usair ways +tupp ence +triple talaq +travel inspiration +the gentle +tan jong +stor t +sport shub +skelmer sdale +sel igman +se aley +sb ath +rncm voice +rad boud +ra jouri +ponti anak +pic he +pic anto +pass book +p illion +ng sop +music live +mul hall +moz hi +michael vaughan +mc glo +mantel piece +laun dere +hime ji +gooch land +gif tware +gfr dofficial +ft one +fre tting +cor nett +ci oni +chal ks +cadogan hall +bro mberg +ble ep +bill iton +aubre yoday +arca dian +a ile +ðŁĴ¯ ðŁijĮ +âĺķï¸ı âĺķï¸ı +z ick +york racecourse +wolver ton +tow nies +tor tures +tol ly +thor old +territ ori +tan veer +ss at +spam med +secret service +roger waters +ri pp +relo j +r car +q mc +politi que +po inter +ous life +open stack +official wmas +my coop +mam mut +liveon news +la ity +jami el +jag iel +investo pedia +in vier +i zzi +hic h +h inews +gu c +grisel da +fidel castro +fear fully +falling skies +f lec +e mison +dag mar +clu bbed +clair mont +cell press +cany oning +canad arm +bv g +buy in +bio sci +back shall +aga ins +zer g +wester ham +v and +ui w +tour downunder +to shiro +ti ao +sun din +stje pan +ste red +so tc +sarab ia +regrett ably +re activate +potter heads +p boc +on yango +night core +n mf +n drc +mye at +my gov +multi plicity +moore head +mid am +metat arsal +meath gaa +love film +kurunzi za +jas a +j mj +j mac +im parts +home business +hac cp +gr attan +equ alization +ent als +e vey +du bb +d ch +cric kho +cookie day +constitu tions +colo strum +centa urs +camp i +arche typal +ap rs +along korn +all Äģh +- .. +ðŁļ ¢ +zu iko +zeen at +you cant +wy le +the er +siri sena +rud der +ro emer +radhikam adan +pre server +pl onk +ol at +oak y +nus rat +ne pad +my all +mu thu +mar chive +lu pins +lin don +ky ra +ksenias olo +kham is +hawaii five +har pur +green side +greatnorth run +gollanc z +fioren tino +dub ga +caro wozniacki +car illo +brexit ers +blizzard of +bird art +bel den +be urs +bar anski +bab ka +b ci +az uki +ani zed +and ani +al kan +ac worth +ac credit +y know +xy litol +wee m +the mahirakhan +t illi +swa thes +s atta +rocky horror +rain ford +ple xi +parksand rec +paedi atrics +mt c +mon ro +moissan ite +list as +kri sta +jac ek +iw ate +it news +iron work +ing team +hima wari +head count +gener ali +g pf +fo kus +de porting +david warner +d pr +cut thecable +comman deered +cen taurus +cc d +candi do +ay p +art market +ah h +turan dot +thi elen +test kitchen +tar ab +sound clash +science march +s ooooooooo +ro mping +resurrec tion +print out +post gres +pere yra +parake ets +nico tero +new releases +mtl moments +medic als +lu issuarez +li pps +ju ggles +jo si +int j +hot ell +grand theft +dur arara +diyarbak ir +digital drawing +dge tt +clean energy +cd ti +cann ula +bull winkle +bu ssell +bu kas +biddul ph +ba cha +b win +av ante +anac mo +all infor +ye tu +wr al +way togo +vir us +trait orous +thyssen krupp +the shield +the hard +tal ot +synago gues +stubborn ness +stou te +smoke screen +sh anth +se ku +sapbusiness one +rei ka +re sus +ratcha thani +ratcha sima +pol twt +op f +oci gars +mini bar +may sville +laundre tte +her ded +h ool +fuji moto +fati ha +entom bed +eagle snation +dissemin ated +cuer po +co jo +clar ington +carbon fiber +blu men +blaen au +beach am +back stabbing +assess ors +ap bs +any ar +ano te +and ino +ail party +ac supdates +ðŁĶ ° +ðŁİī ðŁĺĬ +you sif +under writers +ugand adecides +tur ro +top knot +to ren +stre tto +smi f +showusyour builds +shi vam +sex ed +sensiti vities +semio tics +sand ilands +san lam +qu avo +portu gu +pla ye +pare ido +ound theworld +myo fascial +my yy +mik al +macfad yen +ko chi +kin care +kapo lei +k sports +jag gery +j assim +irraw addy +in os +healthand wellness +golf day +ghan ds +freedom works +fah mi +europe day +errone ously +do jo +cze wska +col linson +ck ing +cimm yt +ceu ticals +ca esa +bu youts +bu ssing +boys bball +bi ber +ban es +ati one +anti hero +aneury sms +alien ating +ðŁĺĶ ðŁĺĶ +ãģ ĭãģ +âij ¡ +zogh bi +v cl +v angelis +unzi pped +under current +thebig bang +tabletop day +sk en +sie bert +se fer +santander uk +sand blasted +salt dean +ru brics +rock show +recruit ment +port chester +planeson thenet +parlo phone +paralle lo +p ade +matte o +madhan karky +kevin o +kend ama +indv seng +hollowcrown fans +ho tair +gros grain +good new +gon o +getinmy belly +ge i +free agency +fair burn +dra gracing +djfresh sa +chroni xx +chad da +can u +c wr +bla ser +all right +ai zen +ac j +abhi yaan +wilder nes +wick low +up minster +trac ers +toron tom +tm koc +t lal +speci e +ske wered +si vi +sab bey +rox anne +r di +pla sen +par ma +moh enjodaro +mikas ounds +liber ates +kry sten +kingsme ad +ke sa +junior doctorsstrike +jin u +il co +ht brunch +hr sa +ho pat +har lock +hain anese +f als +electro phoresis +eic her +ei sha +e week +disappoint ingly +de kal +cour maye +com pra +cloud native +chum my +cam u +calvin ism +bra x +biz on +bee son +bart man +bab is +author isation +aqu an +ador able +ðŁĩ¸ðŁĩ ³ +ìĨ¡ 민íĺ¸ +we ta +waff en +vor on +vi vendi +verte bra +up shur +tuss aud +su hana +sply ce +serial killer +sav sl +ram mer +raj inim +queens landers +mon of +maha bali +m wi +loven orth +liber man +leap ed +lauder hill +ko se +kh long +karen gillan +ish rat +immuno deficiency +im hotep +hed rick +he st +h anny +go res +for culture +fe ets +fab bri +eun woo +ess ler +cru x +co ren +cle land +bon ing +blake griffin +bil ang +bi phobia +barram undi +baram ulla +ash faq +ar ico +al ani +ðŁĺĨ ðŁĺĤ +ðŁįĴ ðŁįĴ +ðŁĮİ ðŁĮį +writ able +wishyouwere here +where to +velt liner +u wo +tan is +swan song +star rr +sn ina +si zer +shor o +sedg ley +sar ai +sa wesome +red fin +pra shant +phil heath +or val +nise koi +mor ath +mit m +mau le +ma um +ly ssa +ludo vico +kha an +kat ec +hun ched +hix son +high wood +ha ddington +food stuffs +em pathic +econom etrics +dr w +curritu ck +cla ssing +ck t +ce tin +car neros +can ts +ca io +bus sey +assemb ler +armad illos +andu jar +ðŁĽ ı +xplo sion +wor dt +west stigers +under painting +th au +telang anacmo +tan n +suk ma +stu bble +spor to +somer ton +skol nick +sh our +sailormoon crystal +que asy +pri mo +pin sent +pickup lucifer +ot te +nationalgri dus +mu haj +moon struck +lange vin +ky umin +kapp ap +jaz i +jan ic +j yr +hu ffy +gg u +friday facts +finger lings +f bp +e ins +de dge +christin am +bor gir +boo om +blake more +black isbeautiful +badas sery +bab aji +b acy +appell ation +ali reza +al wyn +???? ?????? +ðŁįĥ ðŁįĤ +winch combe +wash ou +vez ina +tsing tao +timon ium +thermo electric +sj maas +singularity u +ron calli +rhom bus +ra by +pulmon ary +pre cht +pla smo +over protective +one k +non traditional +ngsop engardens +nest le +medic o +ling e +le baron +kam ay +ip se +hol liston +hick sville +harmon ium +gur nard +doom tree +dh ram +crime stopper +bjer g +b vr +arthro scopy +apar is +antw an +ame ss +ðŁĺŃ ðŁĺį +ãĤĪ ãĤĬ +wil sons +un challenged +thir teen +swat ting +stone age +standwith us +softhe year +social network +sd ca +sav ona +sam en +ru ggles +roc co +rick shaws +recep tionists +reak fast +rad in +r zr +pen i +nascar onnbc +mcge ady +living with +lethal bizzle +last minute +l ch +kew pie +just dance +hi jacks +gre gar +gra fts +fric ke +floren cia +first energy +dd n +cy outh +cy mbi +confi dante +chou dary +cal lista +c low +burg led +bo pp +bill haslam +believe it +am ani +ace vic +éľ ĩ +zeal ander +wh att +uz h +turbo grafx +trailer park +thro ck +thenew school +the av +tasty tuesday +sor olla +sin on +sil ico +si banda +scott sboro +saraswat ichandra +ruff o +ran sacked +ram o +puri sima +psycho tics +pin kie +pier point +pf aff +peter dutton +nucle ic +nov ambb +nico lette +nar o +metro dome +med abad +lagh ari +kol arov +king ham +ki eth +ike bukuro +id olo +get down +figure head +daz a +cros bie +conni ff +con ner +ce urope +brum hippodrome +bj m +bhogle harsha +berry hill +be ba +ang ar +amber rose +afil ms +ðŁĺĤ ðŁĴľ +ðŁĵ ¶ +аР¹ +uri ya +tsu ko +the power +takeover tuesday +seth rich +se wick +sal vulcano +ri vu +re gr +ques nel +qu re +pre mo +power book +polymer ization +poly chrome +pj morton +ouel lette +oklahom an +ok yo +oh su +occu red +nu ages +ni ku +mus yoka +moon roof +mm hg +mc cay +mall a +kre u +ki seki +kate walsh +k pu +implo ding +ib p +fund acion +du laney +dream car +dha an +cru ral +clu edo +cha hiye +butter cups +bun doran +bon go +bet ancourt +athiy ashetty +asin ine +albertope tro +aam n +ðŁĶ ¢ +주ëħ Ħ +wa ad +vilam oura +vege tal +u ag +tao bao +swi sh +shake out +shadowhunters season +sh air +selen afor +sanc tification +s ello +s dot +s bbq +ro ti +ril los +proton mail +program ing +pol len +pla sterer +peak sn +off stage +ny ra +na ima +moor croft +mil lets +mel nyk +maxi mized +marshaw right +ley ard +ler n +le elee +lat rines +jaime camil +jagdish shetty +im ur +im measurably +home show +hallo ff +h tb +go ga +exa directioners +ex omars +ell trust +ei fel +done job +der na +dark ling +chirp chirp +chatham house +call er +brad for +bon nar +ben jy +al tan +abre wing +abduc ting +âĿ Ķ +white space +up son +type cast +trump russi +to ws +theme park +stitch fix +shi i +sat er +register tovote +q lik +pivo tal +o et +multi generational +medic ina +like app +l tte +l ingly +kur ni +ku e +king fisher +jes mond +inver aray +instig ated +in ate +hit achi +har le +ha sson +gou cher +food allergies +expen sively +diamond head +cross wind +commend ations +chri stel +che ikh +chau vet +bun do +big island +ber thed +arthr itic +ardro ssan +ar dian +ad noc +???????? ???????? +ðŁı¿ âĢįâĻĤï¸ı +ðŁ§ ¤ +ðŁ¤¬ ðŁ¤¬ +âĿ¤ï¸ı ðŁijĮ +z hon +z evon +th ound +tend encia +sock ers +so hail +sk river +short falls +shi as +se alion +scab ious +sand gate +richland two +rell eno +rei ko +r rd +nokom is +my motorhead +mul und +mor iyama +monk stown +melaniel bbh +m ander +luc kett +koo per +kin ghorn +je ppe +japan town +hoku to +genevie ve +fun n +full moon +frankiej grande +firstal er +dancing onice +chan ter +brussel sairport +breakfast ofchampions +boli varian +bicon dova +betten court +arthro plasty +ðŁĻĮ ðŁijĮ +ðŁĺĤðŁĺĤðŁĺĤ # +ðŁijĢ ðŁijĢðŁijĢðŁijĢ +ðŁ¦ģ ðŁ¦ģðŁ¦ģ +ç¾ ½ +ze on +z yl +wi ed +valenci ana +val di +u cluelet +tor adio +thra ki +thick en +t sou +sy ru +sun t +solstice publish +shim mery +sha al +se ec +ru les +resurrec ts +plastic surgeon +oye depo +over reaction +oh no +nc beer +mis rule +mic ellar +listening to +line work +lin dsley +lam eness +knight ly +ki yoshi +keep your +impresi on +ignati an +ht punjab +here wego +hand cuff +half back +futuri sts +fay emi +far u +f sg +f no +e mel +e gom +donor schoose +doni zetti +dissapo inted +dau n +crickle wood +cre es +cork airport +con serve +community day +clare more +chur che +cap shaw +bubb awallace +broad head +anne e +aly and +___ ; +âĻ ļ +âĸ ¡ +à¹Ģà¸ Ĺ +yoh ji +xau usd +work top +vol ks +vijay mallya +trend setting +sou ks +sm tx +ske eters +si rr +sa very +re wiring +quil t +prideand prejudice +patriarch ate +official bantams +o wais +nike golf +mumb ait +mo gg +michael myers +mcfly harry +lebo witz +land forms +k mf +harts field +han se +gö te +gri jal +eastere gg +dri shyam +deto fficial +daily inspiration +col lation +cheer fulness +che detofficial +breed love +bie shq +bapti sta +ðŁĻıðŁı¼ âĿ¤ï¸ı +ðŁijĮ ðŁĻĮ +íĻ į +æľ ´ +âĨĵ âĨĵ +world theatreday +vo tar +stragg lers +so yeon +slo ped +skin less +sha hs +se award +restur ant +r ks +plo ad +pau ling +par appa +oung music +no life +ne ston +n sports +mn k +mil ana +me led +marin eland +manchester pride +lund quist +low party +lar kins +lanc sccc +kim woobin +kal itta +k nar +italian ate +in yong +idyll wild +he athle +ha gan +gi one +freck led +fon nbc +fer m +febre ze +excep ting +eufau la +ethiop ians +er dington +domin ika +denbigh shire +de fro +dd d +cre el +cotedazur france +comp chem +char cha +catal ano +camo sun +bu to +bel mondo +assail ant +arthro scopic +ap fel +aali ya +îIJĴîIJĴ îIJĴ +âĶ Ĺ +âĢ¢ # +ಠ¬ +x os +wre x +wig ston +vol endam +ur sery +tru c +that one +th sh +t kt +t dl +sin b +scoun try +sam riegel +s comics +rian johnson +p mn +our way +ou sting +no tw +n ko +montra chet +mo stert +michael franti +mh w +max azria +living the +lind ac +k awa +incentivi ze +he mm +hat er +gi a +free stone +exer ted +eu storm +early music +d hr +cursed child +cross road +conse il +co quette +chill ing +cam brian +bu rien +bring ithome +blou in +biom edicine +be em +bair ns +ash ab +ar yan +anti fascist +and tobago +af acts +îĢ İ +æĻ Ĥ +youth olympics +un scientific +un repentant +uk trichat +tre bles +tinsel town +tear fund +tan ak +su sanc +ste gman +sor teo +sf g +sabo teur +roald dahl +rho den +question oftheday +presump tive +pre eclampsia +pot g +plu mpton +peace ofmind +path letics +naf a +my as +merci an +mc bee +mar ka +ma ira +ly tt +ing b +holy rood +holm gren +gom ocs +evely ne +embaras sing +em plaw +doc cubus +distr acts +cuff link +cour rier +cage theelephant +bro hm +bon sall +bar tz +af tab +af fixed +adel man +ac f +ãģ Ī +é té +yar ds +wolve srl +with style +wing chun +upd f +u ecker +tor menta +te kin +te ale +straigh tedge +star alliance +st century +spy glass +saniti zed +s indi +ros alia +ren ate +raz ek +q st +pu el +no omi +nast iness +mm viverito +mini vans +meso scale +marklevin show +luzer ne +lukash enko +lol cat +lizz i +land schaft +ko th +human oids +hoo poe +guil t +green man +gi ug +gh ly +from today +fra e +fire box +dramati st +dr oner +diy network +cover alls +counsel s +coc co +cardio thoracic +bibli o +arro ba +ann un +amgen toc +af rench +adi pec +Ä ĥ +wo wwww +w ff +vau d +ultra violence +u wt +the grdc +ten shi +tat ak +sunning dale +se et +scar diff +re order +push ing +pit ti +pic colla +pe stering +pal eta +notthefake svp +non human +ndu ja +mis steps +magnifi que +ma sta +light weights +lego set +ke ti +je g +ge dung +gag ement +fert itta +female founders +fal ke +du bo +dir ksen +din kins +de con +coo ties +concu ssed +chang bin +brian mcfadden +bel ford +begon ias +bal ch +an gra +ab log +... ". +ö n +we gman +wawa w +wav ves +u bo +take o +sto cky +son der +so ts +sle n +sky sport +sik ander +sar r +ri ana +pre order +ou p +on ore +om ax +off ramp +oc sb +mira flores +miki moto +ma res +lau rene +land sberg +ku u +ke mble +k les +joelo steen +j cb +i wak +hon o +hog weed +ho reb +hec ate +hammer time +ham pering +gossi ps +for ays +fall er +fai thin +equ ating +do deca +cott le +conom ic +con flu +chim ing +brook shire +bre an +br ani +bio geography +bay u +an zio +ðŁĶµ ðŁĶµ +ðŁı Ĥ +wy te +uuu ut +ut pa +un scr +tro om +titanic belfast +than niversary +ter schelling +sw ang +sur ge +smu dges +sin ton +scal ding +sat guru +sar py +rug ge +pur itans +ou zo +mukil teo +mechanic sburg +mall or +m ø +lim ite +lets make +lari mar +jaz ak +ir uk +init towinit +har tung +ha berman +gm fb +fri as +dysen tery +cat walks +brough ty +boston ian +ben digo +at je +ashken azi +arizon ans +appliqu es +wool ery +witch ery +travi stritt +ther un +the matildas +syracuse crunch +sun life +spl m +south london +sl ac +shur r +sa hs +rush cliffe +ren cy +reali gn +rab ada +r ld +preak ness +pl anta +over growth +mumbai kars +mpo fu +mozam bican +match s +mat ra +le well +kam in +jonathanr knight +j sr +intram urals +impressi onists +go sha +gau d +for u +for gov +fireal arm +fi xx +farmers journal +cu mber +cod fish +ch agu +celebr a +bulldo zers +blackfriday deals +ber shka +as ri +ark ell +ak ie +ad asi +ã ĭ +âĻ¥ ï¸İ +âĢĶ & +yoko suka +wi sps +westh ill +wc pt +vivi ani +vir ga +un days +tad ka +sre eram +squ in +schar gers +re introducing +phlebo tomy +peaksn valleys +pe cked +paid media +or lv +oke mos +nt p +no vic +mr peaksnvalleys +mad ama +liber ators +kam er +juliag illard +j nr +im ts +igh alo +hemorrho ids +ground hopping +go do +ge ely +fromthe field +for today +female filmmakerfriday +embezz ling +duc ato +dor ris +charlton comics +chann elled +british moths +as prey +art prints +adel ta +ìĿ´ ìĬ¹ +à¹Ģภģ +work at +wolf son +we sco +vin h +un cultured +tech y +tam ayo +sonn enberg +snar ling +sma k +se vers +s ads +prish tina +ped ne +patt ymurray +nr x +moon raker +mol an +mend elson +mcgu irk +martin omalley +kwad wo +i um +hor naday +helicop ter +gar an +en ot +discover yof +chin en +cal fire +british gq +brain waves +blue tick +ber ube +bent leigh +be aware +ave iro +are va +an unci +al sen +âľĮ âĿ¤ +x illia +wwe fanart +vigne sh +twy la +the secret +the poke +schi edam +saxophon es +pop sugar +page sen +outsi delands +nu da +nationalhot dogday +naj wa +n sac +music for +met as +lovemy life +london midland +la che +jar amillo +indoctrin ated +ib min +i dream +hellomy nameis +gn om +fur lough +fin kle +fil lu +ely xion +dug ong +do ren +ding ley +desc ents +day one +chateau neuf +charlie sheen +bread fruit +ben havn +bab oxing +ba rea +apol itical +ahu ila +afro pagesen +ad news +çİĭ åĺī +âļ¡ âļ¡ +اÙĦÙĦ Ùĩ +womend eliver +wis ma +v ro +us and +uk homeoffice +trinidad andtobago +tony stark +sweat y +stay healthy +sor chestra +smo thering +rober te +pra veen +poo m +pocaly ptic +obli ging +neg ati +na hs +n mc +man ville +ma zie +longlivelong mire +leg i +le ite +k hol +jun ker +joh anne +ja vert +j kr +inte c +hir on +heyn ckes +her mosa +hair goals +h mann +gaw ande +famil ly +fai za +dental implants +de haan +cat to +cas k +brock hampton +boon dock +as sive +arc i +aguil as +ðŁı´ ðŁı´ +åľ° éľĩ +yokai watch +wer chter +vil lette +va as +ubi quity +tor in +tol ly +stlouis rams +spo elstra +sn fonnbc +sco ggin +rahe ja +phenomen ology +ph reno +pau ley +nb ach +nati vely +napp y +mun g +major ities +lin ic +ka al +jordin sparks +gro ep +god head +fo ye +flavon oids +extro vert +dal ymount +com une +co hiba +bur sas +biz jet +bar maid +ar dan +amand as +aban ks +ðŁ¤ Ľ +âĺ ī +ਠ® +wi fi +vin tner +symph onia +sv su +stadium series +shash tag +recuper ate +ran jith +pe son +nun cio +neck wear +msport ltd +month ly +mont calm +mk tg +melo y +master softhe +mar gulies +mam noon +le bar +kro m +just jared +jessic ac +im pairs +im ited +i we +ho es +hi da +gu th +gott ago +fan sclub +e stancia +do olin +dcre birth +dash t +cou plings +compac ts +cagatayulu soyy +caesa rea +bootle gger +bar rington +al ak +å® ĺ +à´ ² +Î ³ +ya el +wi bc +wheel of +web tv +wang ari +wall en +uni onize +ther ise +the kitchen +swel come +stra dale +shannonr watts +sel anne +scap ular +san y +sa kic +ry dal +ru mped +pe mba +origin ation +ok er +mo til +mccaf ferty +max mara +mathe wson +li ken +lagun e +indiec ade +high bridge +hahah hahaha +hab toor +glass works +georgehw bush +fe ud +ez ine +emm erich +em mental +econ et +e og +dy ffr +du per +dissip ating +conversation edu +comm ittal +circu lator +bi sher +barcel o +bad azz +ay un +arm chairs +and on +ah luwalia +ภŁ +yd ney +whe eze +water lo +vi xen +vi are +ver ment +us oftball +un proven +u stad +tro xell +thank sobama +ster io +senior day +sal ir +rev amps +re married +re apply +pen umbra +pedan tic +oo sh +ok no +o gres +nz lv +no ch +newbury racing +nep sac +music ares +mo efcc +me ows +love grove +lavo ie +kevin bacon +jaco bus +j aki +infl ationary +hoo g +heg seth +gulben kian +fraser burgh +enrol lees +endo vascular +disc loses +coss acks +conju gate +comple at +call ender +broc kie +br ora +biolumin escence +belle fonte +beat s +bar sha +ax schat +atho lic +ap lan +amy acker +wi politics +wann ables +visit or +u shi +tor rey +ti fully +the shining +taxider mist +sw en +str ymon +sth attweet +rush hour +rh b +rebu ked +real betis +potat oe +phil us +petul ant +pe ten +pap oose +mon aro +mo sle +ming na +lei den +lec tro +leave your +kyan ite +kre t +k cra +jls official +inordin ate +in can +i die +huguen ot +horn castle +gluten free +fore go +emerald citycon +desecr ated +deadby daylight +danny devito +cu ero +coin marketcap +cavern ous +c fi +buenas noches +behavior aleconomics +ate am +at rip +adjudic ation +aby ssal +ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ +ðŁĴħ ðŁı¾ +à® ² +world pressfreedomday +wheel don +water lily +v db +ul t +u am +sin ha +shah naz +scu ffed +scu be +sac ro +rol la +recon ciled +ran noch +pl ong +no joke +no bbs +muzaffar nagar +multilingu alism +lor ber +kis son +kill shot +jo les +insu la +hondar acing +hel lion +he vc +gri e +gracie bjj +glbl ct +flaming lips +en f +el mi +disneysm mc +craw ly +cost liest +cave men +car pi +bon ar +bal ad +ash trays +ar mm +a hia +ðŁĴŠ⾨ +ðŁ¤· âĢįâĻĢï¸ı +íĥľ 민 +ãĥ ¾ +yesu das +x sw +wor rier +vo kes +vi ano +veronic as +ve ered +tr anc +tl j +ti bor +ten i +tatt n +sober look +simon pagenaud +shin igami +sei ji +s bo +ren na +palak muchhal +mu cos +mit nick +misi denti +meta verse +ma zen +kil ner +james joyce +hol ding +hik vision +go bs +garh wal +fanta stica +e za +du lac +der mic +dan dies +conju gated +cil ia +c team +bri ssett +bf n +baw den +aim less +ðŁĺ ¦ +âĺķ âĺķ +wap si +visi th +un ironically +teach foramerica +tan ey +spring brook +showand sell +show up +scra ft +ri mac +rep tile +ram ires +radi or +pure foy +pal ong +oke hampton +ne cking +mun t +milla jovovich +mc millen +malaysi atru +lu ch +lin ker +ld b +kon trol +jame sh +info graph +infinit um +ib ers +high brow +hetero chromia +he th +grant land +gra il +flat bush +fl v +edge ley +dee wane +dan ko +cr anny +comi ket +bur rs +brum mies +b uni +air way +a press +ãĤ Ĩ +win stanley +vuj icic +vote the +vali um +v dot +un forced +tra bant +tony kanaan +time shighered +the beat +sau ton +saravan an +sarah palin +ribe ira +rhi anna +per rine +party hard +paragli der +olympic games +nas er +mckel vey +m cau +kidnapp ings +khali faof +ke gv +iso des +home ent +hal ber +great job +g cp +fresh food +dom enech +curren taffairs +cra g +cor lando +butter beer +boat sthattweet +bo wn +bbc devon +bal dur +bacchan al +actu aries +ðŁ¤Ł ðŁı» +wi kia +ver dean +un cf +ty dillon +todayi learned +teu tonic +te tus +speed ers +sou ter +soon young +shah bag +sd f +sauce do +psy ang +pri y +pote et +pheno typic +phar os +par khill +osten sibly +olin ari +ne tw +ne mi +mother of +miq balkhan +mill ay +may on +matti son +masa k +lu ss +le ci +ky la +kor man +katamarayu du +j me +glu teus +ge ylang +fitness addict +fish y +dro poff +devi ants +convolu tional +chrisy oungmusic +cat ra +carbon ation +bou ghs +beat in +bar to +b pe +as chool +aqui fers +am mons +aker r +aig ner +afri forum +ad ame +ab dus +ðŁĽ ¡ +ðŁijı @ +ìŀ Ħ +ëį°ìĿ´ ìĭĿìĬ¤ +y alla +wicke duk +we itz +u pe +tren ds +trans liter +trans am +tr boxing +theo dore +tamar indo +si def +se lek +sch ä +ra st +pot tawat +plun dered +ni wa +nap les +my na +man child +kaiser chiefs +j de +irishmusic party +invisi bles +inqui red +imbe ciles +hor ace +havas u +fever tree +fe urope +f ca +ek k +cint as +cgc comics +cat elyn +car care +bun crana +birth control +bell wether +arn prior +ðŁļ IJ +ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ +ãĥīãĥĥãĥĪ çµµ +âĢĵ # +ಠŁ +{ { +worldstar hiphop +water slide +tz ler +tur us +tre ll +tommy robinson +tech jobs +ste phie +so wer +sar the +ren tice +real deal +pugli ese +our town +nu ern +novel as +nick toons +national zoo +mro lym +mel ding +mathi eson +lam berts +killer mike +kend ell +kate spadeny +indone sian +iale gis +hillar ys +ghol m +fichtel berg +dou gan +d hok +cro wing +con oce +com y +clondal kin +carav elle +cam pion +buzz es +bur slem +bra wny +bn m +barcell ona +appen dectomy +af ranc +ac chi +! ðŁ¤Ĺ +zooey deschanel +vintage showandsell +valent ini +turno ff +the x +stren ght +so har +sankran thi +ru iser +rober tb +quar ks +promp tattn +ponti ff +open suse +mtv base +mobi kwik +misdemean ors +marcu sle +magne tics +lovin leeds +kup wara +knife point +ke un +john l +it ay +inta wak +instig ating +ha bb +gray ish +geme ente +ge zi +gar onne +football index +fe is +fal es +evel ina +divo ire +dissu ade +cartoon ish +carry all +blame less +bigsky fb +bal intawak +an in +achak zai +accumul ates +( Ëĺ +âĢĵ > +³ ´ +yng wie +weare portadelaide +wat ashi +w to +w ka +victi mization +un steady +tri st +te gu +t the +sun ing +sports net +spell check +six teen +po do +mu ta +moz army +mar ta +malaysiatru ly +lon dis +infor me +hog sme +hi bbs +fisher mans +fid ler +estre la +do ee +d ss +d mca +cu l +con gen +causewere guys +camer amen +bun go +bmar czewska +blair gowrie +bio div +bett or +asth matic +am ru +aj ade +ag bon +abrew ster +: (" +ðŁij ĸ +å® ĩ +اÙĦجز ائر +win ny +v pg +v lei +up en +tu babu +track day +thierry henry +temp ts +tath lete +sta veley +sel kie +sc ampton +s berry +reyn aldo +recu r +pundit ry +pre loaded +pis cine +per gamon +par ada +oni verse +ne ame +mom ma +mc lean +matrimon io +ma spalomas +letit snow +ke ady +i is +hri shi +ho axes +henri ette +head pieces +gun ne +green house +glen cairn +gior nata +extermin ated +exi ste +econom ia +dja fro +dav ichi +cup ar +clinte astwood +cho ti +bla is +bethe match +ba hati +al thy +@ @ +ðŁĴģ ðŁĺĤ +wl m +twi sta +trul li +tra inst +tali tha +t df +si bo +seaf oo +se j +schre ck +sal dan +ren ge +re tren +r nn +r hul +r alli +prayfor boston +pp ey +pop tarts +p bf +ouse burn +of truth +no homo +night side +n wh +mor to +mor gs +mari us +lager tha +kick z +ki g +kal len +ju len +jo zy +hur lingham +huda beauty +he ter +hardy brand +gab aldon +fill in +fe de +encephal opathy +door dash +defam ing +cr ven +cor una +colt snation +co qui +away day +andrew gillum +alderleye dge +ak r += >> +* ____ +âĮ Ľ +à¸Ńภ° +vol ve +to read +ti po +termin a +tau r +stan tec +smi ther +sar ic +salv aging +pre biotic +pim lic +perth wildcats +ow an +on evoice +old boy +oak tree +n gar +metr onomy +me sis +mar ken +malaysiatruly asia +malaysi agp +loom pa +leg er +laur in +land in +kan ak +isi dore +ische mia +induc ts +in italy +i mec +grun berg +exordi umin +cy steine +cra sher +clut ched +cho wing +bran ston +bat aille +bal main +aw in +asce tic +art scouncil +aho k +adam sandler +ðŁĹ » +ze et +yu ca +way nes +ver onese +unequi vocal +to stad +the middle +ter rel +taka ful +supp ing +sta f +st fx +sm acc +selfdriving cars +revolu cion +pga of +nf caorg +neu trinos +n gun +mor is +maz embe +mall ick +ma aj +lu tes +lovelan sing +li pol +lear ts +l hu +kodi aks +ko wicz +kit tery +kill iney +kam mer +josi ah +home schooled +hoka oneone +hm fc +ha bak +gy asi +gri bble +gen na +eter na +eller slie +disney jobs +dan slott +cu ar +counter strike +compo ser +cle re +cetace an +bag piper +bacteri opha +ðŁĺ©ðŁĺ© ðŁĺ©ðŁĺ© +y hs +woo h +will man +up close +tele hit +swi jk +subscri ption +sl ang +salt and +s bakery +real kurtangle +qual trics +proce so +poly thene +obe sity +nom an +nis sin +nal ini +n sn +n acs +muh ney +morning mika +min gh +mi thing +math schat +madel aine +ma pl +louis burg +le vu +ke by +jo ely +ingle side +indye leven +healthy skin +gu ises +gofor it +go forth +fren ds +forfe ited +epidemi ological +effe ct +dayafter art +dag estan +d to +d fa +cu mp +cri stin +con founding +co che +cho cl +celeb juice +ap m +anachron ism +aar yan +ðŁIJ¾ # +zar ya +warrior wednesday +vor m +vir ate +ul zzang +stol lery +snor ted +sin gel +sar ath +sach dev +pi az +oth ate +new sam +mr j +litt len +legend a +leelan au +le mos +last minute +jig saw +in le +hunter x +gh al +flirt atious +egg shells +deion sanders +dedic ated +dece m +de ce +cor no +chit rang +carol l +capit ale +bridge man +blan che +bi jan +back inthe +aro hit +append age +annu alized +amen hotep +usl pdl +un y +un remarkable +un dra +tun ku +tra den +thor nes +ther hino +the hobbit +stemc ell +standardi ze +so dom +shatter proof +sen mikelee +savannah guthrie +saip alla +royal marines +prem giam +pastic he +ove reem +outdoor life +out classed +nu ts +mün ster +mike o +marsu pi +li bri +l sv +l aci +kin sman +kegv raja +kabb fox +jordan b +il r +humph rey +gui led +guer rilla +green street +free ware +fl inn +exasper ated +esp en +equ animity +endo fan +dru gging +dialec tical +debby ryan +cu au +cro martie +corri da +con junto +colloqui al +co king +cam inos +calvinand hobbes +c mbs +bu loh +black watch +badger cull +as syria +apho tography +ana ïs +an ami +aerom exico +ðŁĶ Ł +⼠°ï¸ı +x em +vish wanath +um r +thenext web +st roo +smol en +sa ins +ra usch +pre ck +ple ine +p tera +p ko +over tures +other world +ol ding +no strum +natur alistic +nan ook +miseric ordia +men achem +ken go +kanchan aburi +k opin +hug day +hercu lane +har boring +gar p +dg d +dab ur +cro me +cre ve +complex ed +cabernet sauvignon +bel gie +ash vik +amorph is +alien covenant +ain tre +adjust ers +acapp ella +ac cross +z es +ys mith +yor u +yaw ns +vol leys +vill ard +unab omber +to stones +stau stell +sar daar +saq ib +sa isd +ro vers +real johngreen +raff led +premgiam aren +per ish +nu un +me hak +mccon ville +latino america +lah or +kag gle +jha sanjay +j lg +inept itude +hu ch +hooligan ism +hal les +gur nee +grac es +flo ssie +fer rum +feliz miercoles +elk ton +elabor ates +dr ale +cycle chat +cri mped +cha rente +ch add +celi bacy +biop hilia +band hu +adi r +acou stically +íĿ ¬ +virtu osity +too p +spati ally +showr unners +scree ches +resto rer +ralph northam +pit cairn +pend a +paramount pics +pa cham +our an +ou tran +monu mento +maken a +mad as +lou dobbs +loc kie +li pp +l bj +k li +howtoge taway +group chat +go bo +girl meetsworld +enh of +duches ne +donthe sash +devere aux +death row +dan ske +collin a +chin edu +cha g +cab inte +blit zen +be as +bar by +auto blog +ap su +ant unes +an chester +amy lo +al esis +ãĢij # +ع ÙĬ +yard bird +y land +wil ke +wat ty +w has +un cw +too bin +tema sek +sum thin +strathro y +sto wer +sst pierre +spiritu alism +sile sia +shower head +se st +san ghis +rod ger +rd weeksary +r tv +r fe +pon do +pin yin +phari sees +pen er +nex star +my girl +musco vy +mune er +mich keegan +mcly nd +mag elang +la gasse +kw p +kis ner +josh ramsay +ho pl +herculane um +he mel +flick er +ene al +elli psis +e mag +dy sauton +death squad +deal sweek +dead pan +darwin day +craw fords +char ro +ch ali +center point +bun gy +bomb squad +beach es +be efs +ale man +adi b +yu ca +youranon news +yaaa as +watu kee +wal li +u ge +therain bow +the wall +stu die +snu gg +se kiro +real world +ray ong +photograph er +mock tail +minu et +memori alize +me hl +mbom bela +m ella +log ins +le ka +kotobu kiya +kor u +hu on +homeand familytv +glo ssed +gh ul +gabriel macht +g was +free dia +ee u +disney studios +de res +cu star +columbi ana +co iff +cc cp +car pa +c bot +bri ze +al ks +abi erto +ab baye +, + +âĺº ~ +ymoun tain +wee tz +we taski +water tower +under pinned +un belief +tvac tor +te q +stipp ling +stargat esg +spac ek +selenafor mmva +se van +s gaa +s down +s baby +rie der +redd ing +red brick +real alicecooper +re tweeter +re ti +rain tree +r mm +por ker +pl zen +pl undering +phil ander +peckin pah +northan ts +nar ula +n ise +mic f +medit ates +mc mullan +mark os +honey man +hom ed +he eler +girl ssoccer +g ws +f ws +e op +drac ut +dog walker +del gad +cy pher +chha bra +black art +bi ety +bhagav adgita +beu tiful +ber ks +bbcradi olondon +av ori +alldog smatter +air gun +address able +wells fargo +wear your +w mg +ve ux +vamp iro +tu loy +today skid +sw illy +shadow man +reve al +real deal +r wen +preck winkle +pink ney +philipp i +oak hurst +native american +mis ation +mannar ino +mal ting +lak hani +ki mora +ken ia +ic asa +herne bay +hercu lean +guzz ling +george sstpierre +fron teras +fluori dation +exc ercise +eman ci +down trodden +do tter +cor ax +chry sler +cat fight +bot te +bo in +berg dor +barbe cues +anime girl +an ind +amak hosi +al exc +air tricity +ðŁĹ Ŀ +ðŁĴ« ðŁĴ« +çĻº å£ +âĻª âĻ« +yl icious +wf cofficial +warmb lood +tom felton +tokyo ghoul +test drive +steel work +sp ahn +skib bere +sad ako +s ldn +re selling +rawten stall +r pr +pi leggi +panet tiere +ox on +nat as +n ppa +mont lake +money laundering +ml le +marath i +last week +kent st +jer ald +intelli vision +int p +ha dou +grat itude +frie za +dis member +del una +cp j +coraz ón +con fine +chur a +charli ze +cha uns +centre point +broad city +bou let +bo al +bill fish +beath ard +be dok +av ella +as sia +app ice +ail intelligence +ðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺį ðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺį +ðŁĺĬ ðŁĺī +ðŁĵ ¥ +âĨ IJ +whoo sh +wan tt +w abbey +vom its +us band +turnar ound +tor que +sz cz +sur geon +sop inion +sam stag +sali ba +sal ary +play ground +personal growth +ov als +or leg +on no +officin alis +oc varsity +mul key +muff lerman +mapu a +le baran +lamin ator +knight fdn +ki bo +k out +k ci +iow an +inebri ated +id or +ic emen +i dia +han kar +hamidmir pak +h ran +gulf coast +graham mctavish +gr acy +gover ns +fiannaf ailparty +exy nos +esp era +elit ism +dump sters +deterior ates +cu trone +con ciliation +chor uses +chin ooks +belit ung +b wh +as ong +am dram +ì°¬ ìĹ´ +çĮ « +âĢ ı +zou is +wylde audio +w rugby +ta ina +t db +snow melt +si sko +si ris +sap in +rous se +ra si +pv d +psyched elia +pretty inpink +power shift +pol ing +ou o +oran je +one il +o ving +o gc +nieu w +new sar +never land +n walest +my bag +mv no +mur tala +muj ica +mc s +lor aine +jam nagar +j rt +image oftheday +i wk +hol gor +gt live +for climate +food pic +fik re +enda ids +elli pse +di zer +da aaaa +civil iz +cherry wood +bu ggin +brick er +boy ne +bil awal +bibl ical +bi fida +bar and +bant ay +artific ailintelligence +an ser +ah watukee +aco ach +âĻ¥ âĻ¡âĻ¥ +wedne sbury +verment ino +us rex +ther ford +ter ce +te ssie +table aux +sub contractors +sty e +six sigma +side men +sand ro +ri scoll +re ddy +raider pride +psych ometric +pro life +pal as +p sies +om are +naf ld +mis rata +mar kie +ling on +lim one +len gua +kai ba +jon axx +jo te +hr f +har tz +gra eber +go jays +go ad +ge ce +fre cce +flex or +ed inger +dot net +community radio +climat es +braw ley +boy les +boo kish +bloody mary +best memories +barnesand noble +back space +arte per +ar cia +alleg orical +! âģ£ +âĶ Ľ +اÙĦ Ø® +uof maryland +un filled +u ti +tom os +thy ne +thisi stexas +thierry neuville +ta kar +star hub +sse ur +spe ers +sim cha +shirt dress +se guro +sch ü +scar ily +ruther ford +ru v +ru skin +redar rows +re considered +oy i +onof re +oh chr +number one +nor ah +ne esh +muse et +moder no +mo ku +meet your +mat tg +ma bino +lu can +lo tro +ks worth +khalifaof islam +kendra scott +kan ae +jalli an +ik os +gym rat +flipit blue +flip flops +dur amax +drop kick +di op +david m +courtney act +cir ro +churra sco +chel sie +cambo gia +cam bus +calla o +ayush mann +apost le +ami k +am ini +air berlin +a ino +ðŁĺī ðŁĺĬ +ðŁIJ ¹ +ãģ ł +zar ina +yeeye e +yarra ville +x tian +the shilpashetty +stitch ers +six word +sco ve +sat ria +ri ek +r tu +private er +pl dr +perry ville +particul ars +one stop +no sing +miscell any +mir pur +mary lou +mart ÃŃn +mar azion +man resa +len awee +le moore +kee bler +ka is +im mo +home going +h ni +h lin +ge ma +ga seous +for dracing +ear wax +diadel osmuertos +demo tion +cri st +canadi anti +can de +cade my +burgl arized +bun ge +brock ovich +british art +bed wetting +bas o +am ater +ace hotel +aber feldy +? ": +ðŁıĥ ðŁı» +íĺ ľ +ìļ° ì§Ħ +ë± ħ +à¸ĩ à¸ĩ +yr sonair +wood berry +wob bling +weih nach +vik tori +up shot +ulcer ative +ty win +tam ithas +tamithas kov +stat ement +sta i +soul jah +shirley setia +shaw nigan +ser f +sac re +ro dri +pu tz +pu jo +press day +ph n +passive income +oura bly +omnis cient +oklahom ans +ok ri +neighbor ly +naf p +manikar nika +k gv +in vi +hell om +grat uit +fu shi +for health +extr amar +eph rata +elvis costello +eic hen +e mac +dolce vita +di eu +devol ve +d ho +d fp +car maker +brittany ferries +blon dy +ax emen +at ore +ad moni +ac q +a festival +ðŁħ ± +we ghe +val rhona +urqu ell +thre ec +the tide +siyahbey aza +shep sut +shan kumar +scen a +rwand ans +rug ge +ram butan +ph ouse +perpetu ates +o snes +no tone +nic ke +nation sleague +mid leton +michael sheen +li w +le ghorn +keep smiling +international mensday +ic entennial +i view +hon daci +h selive +green week +fal ana +energe tics +dum plin +dubga aofficial +dprin tindustry +del aine +davuto glu +cul bertson +cre vice +commu ter +colombi ais +citrix synergy +chut neys +chin as +cam girls +ca za +burden some +bu ga +biz humanrights +bhav na +are do +annab elle +aldu bin +accommod ated +wor ts +wh sv +vit tor +under lay +ta fel +son ys +smoke stack +sl á +s met +red den +re appears +q ila +pg ms +penguin india +park theatre +or dain +o ses +nic hi +musik fest +music man +meal sonwheels +mc gau +lun sford +li zumab +lan k +kw abena +known st +khal a +jamess murray +hol as +geor gia +ff ar +fer moy +femin in +en loe +ecu ador +dr ilon +disobe dient +disen chanted +dat z +cla pham +chron os +ca sei +britt an +book man +bick er +barbecu ing +az arian +artof m +apple dore +an net +an cia +ðŁĻı @ +ðŁį ¨ +ze char +wer dum +voice actor +vo lio +ve ss +the shark +tam au +sy mon +sonic drivein +shu d +s ganguly +ro tich +ro hin +rejec tions +reco ast +rebel ution +ram im +qu d +orange isthenewblack +nesqu ik +my freec +muhar raq +mr an +molybden um +men inas +media eval +mcken ney +lu iza +labor ious +kon nect +kaf r +jer u +j mb +hob bie +glen finnan +gas kins +fra gon +film school +fight scancer +di ste +con dense +burgo yne +am ach +aggre ssiveness +ðŁĴ£ ðŁĴ¥ +ðŁİ¥ ðŁİ¥ +è § +àŃ į +é lé +za f +youn ge +yo kota +war fighter +wa if +toron tonians +ti gh +the in +the avengers +termin ator +tata steel +t shep +t gh +sunid hic +simon celli +seri eb +saniti ze +san ogo +sal adin +saf ra +rece sses +r lp +pusci fer +pla ud +pan za +pan cho +offici ant +o auth +ny ama +n sr +mour nes +mil lau +mid or +miami beach +lumin ato +let arte +la pid +kre ss +iu u +ing and +g wi +flint lock +fair clough +el mbridge +dress code +domic ile +cros stalk +cooper hewitt +commissi oner +ch ah +care home +bu ggs +ay ye +ao ii +alyn ch +ðŁĮŁðŁĮŁ ðŁĮŁðŁĮŁ +ver bal +unexpec ted +tsub aki +thesun newspaper +tel lez +rdra jaofficial +rafi eh +pu er +pop sci +phe tch +personal training +p square +ox ic +over ruled +oregon coast +nine ws +national drink +mr silver +michael mas +mer ve +mat thai +mar am +machar ia +lr sd +l its +kal kan +k ch +ju eve +in kenya +hor sforth +herbi vores +har p +happen stance +han ke +fatboy slim +eus kadi +ell man +dv am +doyou even +der ren +compar tir +bulldo zed +bray don +boozy chef +bet je +ben avides +ben atia +bb ma +artofm mignola +art forum +ap les +and rich +alum n +aln mouth +ðŁĺ»ðŁĺ» ðŁĺ»ðŁĺ» +á¹ĩ a +à° ¯ +wul lie +world diabetesday +wicket keeper +unbe knownst +un ningham +track suits +sey doux +sco ff +sauchie hall +sang amon +rv hs +ridge mont +qc poli +power gen +ottaw acity +n are +mun sters +movember uk +moore field +million s +mess a +man power +m ney +ler ch +lal upra +l ere +krun g +ken i +jae beom +inf el +imp ound +h ka +go yard +go ble +gid dish +fe do +eu u +doo ku +donmar warehouse +dilip kumar +de ek +dah lin +cur tailed +cro ak +bur da +bor ah +aston martin +ard is +ar sa +am manford +abscbn sports +aalt ouniversity +? "" +ðŁĺģ ðŁĻĮ +ç¦ ı +ت ÙĪ +zi ff +ye es +water front +visi bilities +ux e +universal ity +twitch raid +tur kic +tro o +then hs +the player +tele ported +swad dling +stal kr +slow fashion +seach ange +sas usa +rub bery +remodel s +oy in +o zo +ne ee +n ght +lin dar +le ath +kal le +hert ford +hb m +gtasnap matic +fo shan +e bird +ca ius +ca haba +buck ley +bha vi +beef ing +b cel +ascen e +arcan gel +akade mie +afar mer +ðŁ¥Ĥ ðŁį¾ +âĺĤ ï¸ı +zi huat +x online +tor in +ti mus +synthe tics +splash down +sensiti ve +sd su +sau ti +satur ate +red die +reall ys +rail analysis +pyr mont +por po +plo o +pi ent +personal branding +parksandrec nbc +out lasted +or inda +myo wn +mi gori +mac lean +lun tz +lu bitsch +lon gest +life drawing +key shi +jax son +infuri ated +imp son +iate fl +har oo +feed your +ev ades +enor man +ef itness +ee z +dele m +crypto zoology +bru schi +bew ley +ando ver +abra sives +ðŁIJ Ľ +zo wie +yam mouni +want agh +trans itive +tiam owry +thsh birmingham +the os +the bigh +t dor +t cha +switch backs +ster lite +star land +so bat +sel i +se ws +s deli +ree kie +pre maturity +pre di +phi phi +oak ton +nit to +mu das +miami hurricanes +mani stee +lud ington +lethar gy +ki th +k ariz +ja ani +inthe mix +insu lators +il ter +her balism +ham bre +gü n +got game +ge di +gar rus +ff n +ed berg +du hawks +dark star +collin sworth +coates ville +cast iron +carcino gens +boo tham +augu s +ati zation +arson ists +amon tes +ì± Ħ +x el +v un +v enga +uk cyclechat +tape worm +synap ses +sul ky +sho ku +ser ang +sel ite +scu ttle +saf die +ru sev +qu anta +pan arin +outa ou +om anc +oce anc +nigeri atoday +nigeriatoday ng +neutr alized +muscle fooduk +mo halla +ming led +metallur gical +lake ontario +l jung +kun sth +kay lin +jo inte +insu bordin +insi gh +health benefits +hau pp +h mf +fox worthy +fe ingold +far mm +ex elon +english men +el frid +ef sa +de do +d ke +co sy +boo boo +belleri ve +belk nap +be ton +ard beg +air canad +! âĢĶ +âļ¡âļ¡ âļ¡ +z acu +womenin politics +wi i +volo dy +ti rico +teha chapi +tatlitu g +suppo ses +shar na +rob ford +ps one +pass iton +natural sciences +naing golan +nad al +mont ford +michael s +mer kin +luci e +liber allogic +ho ku +head banging +fur tick +fu h +french men +fam osos +expropri ation +dont stop +develop er +cu illin +cr ated +caval era +c ne +buzz tv +bul ld +bul at +bron cho +beelze bub +assy nt +american ism +ak iz +ðŁĶ¥ðŁĶ¥ ðŁĶ¥# +ðŁİĪ @ +ãĥī ãĥ© +ze bulon +z quez +wildcat pride +wee py +waste d +uniof leicester +tric ities +thu gger +sy lt +sto pe +sp dx +snar ls +sky la +shan ds +seab ourn +school mate +rt gs +ri ans +re invents +rat z +pon tel +pho e +out patients +onit suka +new comicbookday +natur ale +muse odel +mck ale +max ey +marmel yr +ld sb +lal isa +kodai kanal +jan an +irwin dale +ili b +ianh watkins +har deep +gon z +fy re +f mo +drach m +do com +day yyy +cul p +con o +co founders +buffal onews +bu bonic +bot b +be co +baske tt +az aria +authentic ate +aquat int +apost asy +aa ahhhh +ðŁĻı ðŁĴķ +zihuat anejo +youn ge +y ce +un contested +to pla +ti et +teac ake +tar ter +tan ush +swee ts +sustain ability +ste y +sense wrds +satyam ev +roman tica +ri sso +qu ip +patux ent +paolo zzi +pang olins +o ley +north lands +new start +new sand +mer ingues +man gu +liv ability +lab view +krzysz tof +kam ath +ic is +hu izen +hat shepsut +harvardh bs +granul ated +goddam mit +forzaf errari +es guerra +dun kley +dre port +drag net +do xa +dissec ts +de activating +dat es +currumb in +cel com +cau sey +cat l +buck skin +broad leaf +br aley +bis mark +bet tors +bel ge +ass wednesday +am bra +akin wun +zee shan +yu uu +wk ly +widesp read +wether sfield +wash caps +w anga +table spoons +sinab ung +si rt +si fter +se tar +se sc +remedi ed +rajinim urugan +pri ssy +preakness stakes +phu le +per du +pavili on +organic skincare +occit anie +newsmel bourne +new bern +national icecreamday +mor daunt +min ouette +mar uk +liber ation +la wny +kis co +kassi dy +intra ub +hyder abadi +gwr help +fer land +e dir +die gom +cyto logy +cru den +chrisho y +cheeri o +chann on +carpe tb +cal vi +brother ton +brooks beau +brahim çelikkol +bour ton +bo gies +au fc +areyou ready +am bala +al ker +ai ea +aa ha +ðŁĺĬðŁĺĬðŁĺĬðŁĺĬðŁĺĬðŁĺĬðŁĺĬðŁĺĬ ðŁĺĬðŁĺĬðŁĺĬðŁĺĬðŁĺĬðŁĺĬðŁĺĬðŁĺĬ +ðŁĴŀ ðŁĴķ +ᶠ¦ +е в +zam pa +wal sham +transgre ssion +su unto +staphylo coccus +sian network +showtim ena +shil o +sharec ro +shad ab +sekol ah +ru bes +q fa +pyram idal +primiti vo +pe kanbaru +par ok +no gueira +nh rc +nbc ct +me quon +lac u +la j +k srtc +in thenews +in aug +im potence +he us +grou ting +gri ot +goto ireland +gm police +fish kill +does it +di franco +date in +cu arto +copp ice +chen e +cas si +caper naum +barric hello +bally money +assi an +adam carolla +abun da +abe di +ï¸ı . +zesti ria +we yer +w sis +tom cats +todayskid swillneverknow +tex perience +te ale +tat aki +sna han +ser on +se ck +scot trade +por twood +poe tical +peck ish +nikol aus +mol la +minnesotal ynx +mad ding +lon ge +lale ttan +ki yo +kh ich +k nur +i ker +hun ni +hither to +hay ate +gen san +f co +employ able +dump y +dois neau +digit ising +di sko +cu bo +crazy ex +cnn politics +city am +cato institute +bell town +bake shop +bag dad +agen esis +?? !? +âı © +zarathu stra +woo ley +wit ched +ven ant +track oftheday +team trump +su h +sp aren +shashtag party +shang ela +sha stra +seoul day +seac ole +sas o +retri ev +real taeyang +re ya +race hub +od st +new grange +misi ones +mb ari +liqui de +lee ch +l tt +ky l +k rol +journey tomars +it raffic +inver ts +he eee +har lan +grey lag +gi ggle +gan su +for g +finu cane +ep ath +ec ach +cymra eg +crai gie +cess ories +cell cellpress +cam shaft +c wa +bren twood +bound by +bosc astle +boo tiful +archive shashtagparty +and ddd +ah man +admini sters +ac ero +ðŁijİ ðŁı» +åį ĥ +á¹ Ľ +zhang ji +z rh +woo seok +wake field +w gu +vikramad itya +v afa +thereal mike +the mark +tan guy +sug anda +sleep walker +sleep inthe +si bu +sh ool +separ ker +saf t +rock ne +rin aldo +popein philly +paraly ze +or ms +oad by +no ver +net as +nan ami +mi do +mar as +m trs +love box +light the +li mav +jap e +jam arcus +in fol +i ad +hot springs +honey crisp +heroes ofthe +fen o +dian aross +den zil +daysof horror +cloison ne +bul well +bar bet +at sc +amazing thailand +? !!? +ðŁı´âĢį âĺłï¸ı +zon der +we ren +v ct +turke stan +tree oflife +tr ate +the book +ter cer +super con +sho toftheday +seung yeon +sen nett +sel ton +ronces valles +rac kets +ra gusa +pun ia +perpetu ated +orino co +nor seman +na jee +masa ka +marian keyes +mar un +lu ff +len ght +la valle +la hor +l hhh +kstate fb +kid sc +khur rana +kais ers +k hus +joseph ine +iam jamiefoxx +gold awards +fu miya +flyn t +fil mic +face plate +en ames +dogsof twittter +cre sted +coryn rdr +con well +car ros +capp ello +c xc +buffalo ve +bri ggan +bow ditch +body kit +bentley motors +bee cham +barne ys +bah ri +arch top +aami park +ภĵ +w imp +vo tos +sven sson +sub atomic +stry dom +star lets +siyahbeyaza ÅŁk +simpson ville +shra wal +sab onis +robu sto +poli zzi +phantas mag +peru zzi +mat or +mat as +m out +kempton parkrace +kas ur +imperson ators +ic em +green stone +girl code +fur ze +f wp +episcop alian +edge combe +di kanu +dh f +dalla ssmith +cpt traffic +ci als +cen dol +bran ham +bbc lookeast +balay ya +ar get +am et +ag old +ðŁĴĽðŁĴĽ ðŁĴĽðŁĴĽ +íİ ľ +yt creators +yo gate +women mw +whin ney +vo los +val buena +up tight +twell man +tom on +t sky +surrey cricket +sun ri +stret chers +standupto cancer +sopho cles +sli mane +sl unch +sketch card +shri ram +sewick ley +rail fan +pur u +pc masterrace +ou pacademic +ottum wa +mississi ppi +meso zoic +mar lo +lee b +kin vara +jay cees +hy slop +ha sno +good friend +gen omic +fl ic +explor ing +den k +dashi ki +cri a +cine polis +ch anced +capit ul +cand acec +bre sci +bor man +ben shephard +bbcradi olin +att ari +ate pec +ate les +at alk +anir ani +al king +al hum +agon ising +ad amo +aberdeen uni +aband hu +âĿ¤ï¸ıâĿ¤ï¸ı @ +youtube india +wir t +w mv +ver nier +v va +tum water +tn n +th yl +tex ast +ten ths +st ami +second hand +sch atten +sam sclub +sal acious +s bm +red ken +qu eville +pumpkin head +psy duck +ox ox +or nette +or ke +mash al +lu tein +lo za +laure land +kat zen +ison fire +infra structure +iber ville +gu mmer +greater anglia +graf fix +gilbert son +ghid orah +euro league +er cy +environ nement +edin ner +di jk +desmo s +desig nated +dan ila +counter balance +col yer +by usn +ambigu ously +ag baje +acre ative +' '' +ðŁĴĸ ðŁĺĺ +ðŁĩ¸ðŁĩ ° +ðŁĩ³ ðŁĩ¬ +ðĿIJ ŀ +yo thi +walkr stalkr +walk s +springer spaniel +silver bird +ser pico +scott caan +ro bre +re xton +re appeared +pre vi +pc world +pash teen +pad hao +ox ton +octa vian +nico lee +nas c +naomi aklein +mon tel +mom in +metr ically +lu neta +kj ell +j sy +j app +ham mel +ha sna +ghat kopar +ge sun +free keh +fan fics +endo genous +eco system +drive in +dementiafri ends +de as +coo gler +chou inard +char din +bruce springsteen +bre ese +better pakistan +beatle mania +bangor uni +baahubali movie +b ily +az ade +av ira +at tires +as sata +ar ison +ab roads +ab road +ðŁİĦ ⾨ +yi shun +yann ick +went zville +valentini frank +uk coach +tu cum +tp ms +tap scott +so dden +sni de +sk og +same era +restor ation +re imagine +ra wl +ra iled +quir ino +pf tompkins +perse ids +p anne +over hauls +or age +nev ents +naz a +navar atri +najwak aram +musli ms +lo sing +letsgo places +lenny kravitz +jo ver +jo cko +jeremy kyle +i yl +hol anda +giu sto +fc st +fal set +entr ancing +ekad ashi +ead pool +e spiritu +dol an +defrau ded +compad re +ci pta +che teshwar +ch ra +boo ga +bbcradiolin cs +b sa +am bang +am ane +al loy +al al +acknowledge ments +ab crural +a abe +ðŁļ ĩ +âĺĢï¸ı ðŁĮ´ +yed wards +yar nell +what sapp +wall enberg +verde jo +v ka +roy e +rela pse +rajak amaraj +racing team +ra reltd +popo vic +pett us +palad ino +oy le +no we +narcissi sts +mrsilver scott +mor sel +mon tele +mis son +malag asy +lu bbers +loc alize +live son +limav ady +l pg +kell an +jordan abrewster +joni ernst +j dl +io res +ilove this +i ding +hilli ps +gu ld +go digital +form ities +fd bloggers +evapor ating +ep b +emo ore +eman ates +dro ll +di ep +cen zo +cas sart +cam h +brit e +blood donor +berg strom +ba hi +b si +an kush +an ahan +amend es +am pi +a eds +ðŁĶ« ðŁĶ« +ðŁij»ðŁij» ðŁij» +ðŁ¥ ķ +ç aÄŁ +weird world +walkrstalkr con +ti war +there sam +theater ny +spir t +smo st +schwar zer +prospec tors +por v +peli kan +pak veng +nis i +macale ster +ma sham +kler k +kat at +jak in +itt f +hu el +hoo ke +high chair +gro ver +for children +eraser head +equ itation +deep ened +châ te +chef symon +car sten +car max +ca ppa +bro gdon +book sale +alex hirsch +ak ang +($ ): +à´ ¯ +yu go +y music +whit gift +vi dhya +uki yo +tri fles +thug sof +ther ail +them home +stall worth +spe zza +sor ies +schie ffer +sain tly +ry pien +ray leigh +ran ul +pock lington +place mats +per ine +paras ols +noo dling +my sa +monaster io +min san +mill burn +mene zes +may hem +leup old +la sky +kirk sville +hopat cong +gu tman +gr á +g alia +fun kin +f ür +eu co +en um +em itter +ear wood +dogsare love +dean cain +cyg nets +bu kid +brock en +beir ut +be hn +atte station +amni otic +ðŁĴĻðŁĴĻ ðŁĴĻðŁĴĻðŁĴĻ +za beel +x ess +why im +where sw +were wolf +vin nik +uigh ur +tur cotte +trust worthiness +to di +te ff +stein hardt +sor n +son de +sne er +sk on +she in +sar aali +red headed +pri on +piotro wski +njo ku +khun nie +hae mat +grass market +grand ville +gradu ation +go ten +global halifax +gal ax +fore ducation +f du +doppleg anger +co gent +champion stour +brun y +bray ton +bhan u +be delia +at sume +as ani +ary news +alsati an +ab du +aac sb +âĿĩ ï¸ı +à¸Ĭ à¸Ħ +whar fe +weather vane +ul ani +tomo ya +tiem pos +szy man +sty lee +stafford shire +song sof +som me +snow storms +sjc drums +set anta +ser j +rober tv +pop caan +plaud its +penicu ik +pam uk +over zealous +one more +n pt +little woods +iu pu +hyo go +humay un +ha enow +graf itti +encan ews +du v +dra goons +do cometrue +dill ane +cupp layoffs +con ra +atour nament +as vegas +ari ella +arche ologist +apo yo +ali es +êµ Ń +à¤ķ र +мак ÑĢо +val c +usa g +up north +um bia +ucla football +tor rents +sur at +sto rer +stig mata +sketch pad +sik ala +ro tti +r vad +post rock +phoenix mercury +patriot sunited +ordin al +on amission +oak leaf +nu sh +nic i +ne ater +natus vincere +nancy lee +meg afauna +mav ado +mar cal +ma wes +lo belia +leah remini +laven der +kab outit +im prints +ibm z +hee renveen +happy happy +esk dale +er w +en berger +ecor p +dun dur +dig m +ci mino +cal vino +c tt +broo kie +bo ola +black smithing +best actor +bac ke +argument ative +alexander rossi +ahu a +ðŁĴĥðŁĴĥ ðŁĴĥðŁĴĥ +ê· ľ +ye va +wdr bnews +w se +vil ly +vic tu +un interesting +toll booth +tenn ent +tab de +sussex uni +ste pla +sn l +sleep day +revital ising +re aux +ravish ndtv +q ar +play tex +pi ko +pare il +nb f +mo as +mid wood +micro environment +metro west +mc dou +mar ita +kimmy schmidt +kan gra +k ile +k arti +jim beam +ish o +hu sey +heavy metal +haw kin +green ford +gra vis +foot golf +ffe stiniog +enjoy in +embroider ies +dr ys +dip tera +dio sa +d dot +co hasset +club hectare +clay don +black history +bhand arkar +belle zza +aston villa +allrise silver +ac z +> ,< +ðŁĺģðŁĺģ ðŁĺģðŁĺģ +ðŁĩ§ðŁĩ © +âĸº âĸºâĸº +zodi ac +wil ting +w tov +w ics +v xx +v loggers +ur bino +tune stweet +tu mmy +the giant +the chris +tap is +t br +supple menting +sum ma +seize the +sat suki +sa wi +reyno so +ran go +poor nima +pi gand +pfi ster +panip at +pa ani +orth coast +neuro fibro +morgan e +mcre ynolds +mari ella +llor ar +kk ah +instant aneously +i pic +hand o +goal tenders +ge don +fried chicken +fresh produce +fly tipping +ess ences +emergency medicine +e katerinburg +dwar fare +do bbin +disintegr ating +dep th +denuclear ization +de cc +cra pp +cr ame +ci ff +cheek tow +captivity kills +c ville +brawl halla +boston schools +beh ren +akrapo vic +ðŁĺ¥ ðŁĺ¥ðŁĺ¥ +ðŁij¼ ðŁı¾ +ðŁıģðŁıģ ðŁıģðŁıģ +ठģ +y vra +y ury +ven dee +vanderbil tu +txh shoops +summer time +ss aa +spre aker +ri di +renault sportf +re charges +raw linson +ranc or +rad ja +quiz nos +pv v +pokemon letsgo +nim by +news boy +nav deep +mu v +maj ed +ma khan +la q +la aa +kri stopher +j mt +inqu iry +infant ile +hor mel +hed land +heart sof +hearing dogs +head mistress +go shen +frank s +euro p +ers ley +eri ver +econom y +e waste +di ja +cu toffs +cine mac +cheap skate +chad ne +brill ante +br ana +awesomenes stv +aval anna +vote arianagrande +tri mmers +tic oke +thorn ley +team stallion +t cn +swil son +sou les +sketch january +ran son +r ge +pl v +people are +p bk +nr cc +n annie +mr nradio +mead ville +mcel hin +mar q +mal at +ma pit +londonis open +lindsay lohan +k tc +jim rome +jenny lyn +ian m +go gogo +ger lach +fre do +fra hm +form less +deschutes beer +d chs +cross ville +clemson family +chincote ague +charity day +calver ley +bombar ding +big blue +bedri dden +back water +arrested development +arq ana +ang am +aivaz ovsky +whitec ross +we tump +tu d +tribe smen +to jo +tele vangeli +te heran +tal ke +t seng +su bir +spit fire +sm hs +ri ggers +re planting +r sh +pic ke +par malee +page views +ostr ander +o ds +northe aster +nancylee grahn +mu ke +monster a +mc phail +machin ations +mac ademy +la gonda +kris meeke +ka wor +ji e +im ers +gro lsch +gaku en +fur th +fruit fulness +fe tt +fa del +duc es +con trail +brock en +bret baier +billing sgate +bha gat +au demar +> //// +ðŁį ¥ +var u +thebigbang theory +the gifted +spoo fs +slip way +schri ft +roc team +refugees gr +q aa +prest bury +op to +nerd life +naturo path +mon ac +mini stered +mercedesbenz ind +mary lou +mari st +lo hr +kol ler +ka hala +jor dison +ji hyun +iz aak +inten sively +int age +high court +h wl +glaswe gian +gil lo +gavin rossdale +edge field +cow bells +canvas sed +canadi en +bones onfox +bha bha +bell end +battleof britain +ast an +arteper larte +ahim sa +abdic ation ++ ( +ðŁİīðŁİĪ ðŁİģ +âĺºï¸ı # +wi js +u pup +twitchtv online +tree frog +tl k +tie break +think musicindia +temperam ental +sun woo +stock piles +sp atu +sco ach +sar nie +richie hawtin +reed ley +ra bu +phy ll +om ot +metal gear +metaco gnition +man ok +kun ar +klassi k +jal op +holocaust museum +hau ghey +han nie +gre sley +flag stone +explore edmonton +er at +crun cher +crimin alizing +cove do +chandra se +cas son +cari bou +cam argue +cal zona +bu daya +band uk +anton is +ami han +ðŁĻĮðŁı½ ðŁĻĮðŁı½ðŁĻĮðŁı½ +ðŁĺ¬ ðŁĺ¬ +ðŁ¤Ķ ðŁ¤ĶðŁ¤ĶðŁ¤Ķ +ðŁ¤ ¦ +Ùģ ÙĬ +yas i +welove it +visit ing +viol as +u ted +tre vel +sympho gear +stour head +se it +se ac +scrob ble +sath letic +ri q +resc a +replic ates +raz an +pat wari +our less +n tuc +muk ha +moom ba +mid hurst +medi acorp +mc kim +matu idi +massey uni +mar cou +mal ir +ma official +m ny +le disi +l ated +kri ssy +kr illin +kid zania +kell ym +jo ga +iam vikramprabhu +hi bbing +he les +har lesden +gly nis +global news +fly swiss +ex pom +ergon om +e bisu +don ell +ci j +chel sey +cha c +best day +be van +ban anagrams +are sh +am orous +ade cco +adam as +ðŁijįðŁı» ðŁijįðŁı»ðŁijįðŁı» +ðŁĮ Ħ +à¹Ħà¸Ĺ ย +x her +wild and +u zz +tom ando +todd y +tobi ko +thebody coach +swan ton +som y +sleepinthe gardn +sj k +sho esday +shat abdi +save sharks +sa sikala +roque fort +rad lett +pink ston +pe mex +os ac +on rails +om munity +nas agoddard +murdoch mysteries +mu dge +man andthe +lu ts +look down +lett ing +let z +law ry +kevin love +k assim +jagiel ka +iucn redlist +iru mugan +if n +holl man +go pleader +gear head +for hope +fmc sa +fit o +et winning +en vivo +ebay rocteam +chlo é +chic an +carryo ver +cal era +c js +breath in +bio control +be fu +ap itals +age o +action figure +. ðŁİ¶ +âĻ ¨ +اÙĦ د +ye si +womens open +willam ette +ver ns +ta ille +stein hoff +sp x +sly ly +se abor +room ful +r blx +piran has +newsp oll +news from +ne aux +na if +mother boards +mor row +marin ades +ma fal +m scott +lud low +li ii +leon el +lam ented +ky am +kube con +kov skiy +kam au +jab hat +hermi ston +gr ata +glen don +glam or +ger n +forex signals +fabi enne +evo ked +en otes +eg or +du jun +drop head +clap trap +bani shing +bak ke +bag got +b dk +ap ar +air lock +ace e +ðŁĺįðŁĺį âĿ¤ï¸ı +ðŁĮ ĭ +world championship +vote blue +virgin america +v mug +train ier +tor ture +theroyal ballet +tele fe +tasteof london +t lu +sim bel +sho tting +ro j +rain ie +pro bando +percu ssionists +nat tie +mush u +mu iden +mel tham +mat ra +liber alization +lepre chauns +la sgo +ken jeong +hiphop gods +high water +hi e +hell blazer +hei sen +ham mill +hack enberg +green vill +furlough ed +fie star +disemb ark +de cal +dap o +dag ger +comm munity +ce iba +care t +bollywood actress +blac kett +bian chi +be set +bal briggan +ar ken +an ze +a ecc +] ] +Ú ¯ +yuca ipa +yel tsin +wit bank +va sion +tothe moon +solilo quy +setti mana +seaf aring +par rett +o q +nedbank cup +nand u +mergan ser +mel as +kissing day +kin er +ha dd +godis great +disintegr ate +der rickson +cyano gen +chloe bennet +cau e +cand al +bla ded +astr on +ap ati +ak ashi +aha doop +ad jei +â İ +when ua +vo ci +travel to +t girl +super hot +squab ble +south central +so es +saunder s +ren teria +reic hs +q tum +nh lawards +n pi +my dog +mun sch +mon ki +mc morrow +mat ka +lex masters +les fic +kash miri +jim s +integr al +ic ra +holla back +ho kie +greek town +geomor phology +ge le +gallow ay +fo al +finalfantasy xiv +eri um +en coders +devo xx +depend ability +covers ong +coke studio +car ignan +brickby brick +blo b +bird watchers +bi ocon +bar low +bally hoo +bal ai +ash brook +arl ene +and back +ðŁĻĮ ðŁı¿ +ðŁĻĨ ðŁĻĨ +ðŁijĩðŁı»ðŁijĩðŁı» ðŁijĩðŁı» +ë ¡ +æ º +yan ov +x ic +wolf blitzer +wiener mobile +weare lakota +un forgotten +un aired +tt k +syri a +sunidhic hauhan +sukho thai +style uk +st fx +spi x +si op +sau ve +sam t +saddle bred +ru ga +rtop nb +ri bes +reyn old +pr ams +po kh +phar cyde +pe v +obstruc tions +o gall +mon in +military hist +medic inal +mc bain +max ell +mag ed +ly ttle +lc cs +kokan ee +kam il +josh mcdermitt +intru ding +igi ppygrewal +har twick +hand spring +ha gu +glori fication +giang inoble +gian tess +ferr ying +eric i +ec tor +ear thing +do thraki +dis liking +di eters +de sor +cu ento +columbus day +code org +chu mb +cbee bieshq +bristle cone +brazz aville +barber ton +baha dur +auto play +arun rajakamaraj +ðŁijį ðŁĺį +ìľ Ħ +ãĥ Ľ +âĹ Ģï¸ı +Ê ¸ +yo ong +vaugh n +upstat eny +ugh hhh +twitter les +tri ver +tre eline +thre dbo +the co +than die +tel i +taxi way +stir rer +st paddysday +sor cha +shorty awards +se tu +rock ing +ren ai +rejo iced +re stre +ponti fic +pitch fork +over shoot +okin awan +nam cinema +megat rends +ma hut +long legs +land rum +ky aa +king stone +kay seri +high five +gol fcart +go key +glen bow +get together +fly in +endor ff +em mett +e azi +defaul ters +deci phered +cl ymer +ce fal +cat fish +car rol +c ici +bo len +birdo bs +big city +ati sta +ar oun +ambi ka +amalgam ated +air side +adi ga +ade boye +ad ma +abram jee +ab aliga +ðŁijı ðŁĺĤ +ðŁijį ðŁı¾ +âĺ ¾ +yess sssss +wo k +wa state +vand amme +v aa +tree hugger +thr acing +thin line +the sky +synchron y +still organ +slum dog +side show +sau con +roar ing +ravi shankar +q assim +power rangers +nor din +mustar d +muscle pharm +mun dial +mo es +mc cri +mayor soffice +masse use +manal apan +lan z +kr gv +kemp tville +i aps +h co +gu inn +gol ders +fle isch +firstaler twx +fin ke +debu gger +cam i +beer and +be moans +appliqu é +alo ren +alan shearer +abr sm +ðŁį Ľ +ëŁ ¬ë¸ +à¹Ģà¸ Ń +youha donejob +yak ub +v ff +uvam en +trespass ers +theli gh +the pleasance +syco phants +sump tu +su ae +store wide +stal ent +se ago +reflexi ones +r ts +pizz i +pat in +october yet +low den +lex ico +khil af +jy he +just go +j izz +holist ically +hobby lobby +he adey +haz arika +hay at +hak yeon +fl m +fa onews +f loc +du fc +dow sing +del as +defra ud +crystalli sed +cr annies +clo tting +cl angers +chen ango +bu ku +bron tosaurus +bo sky +black head +bir cher +belo a +balde agle +baku gan +baili ffs +ac sm +ðŁĵ· | +е к +ul ang +tom omi +ta patio +sympath ise +sw azi +surve kshan +si don +sh oney +recon cili +public works +po ck +man el +lou ren +lo hud +li les +len nan +l sarsour +ka ew +jor nal +je wson +its showtimena +inde cent +gu age +fi or +envo ys +endangered speciesday +dur sley +dor chester +don nyo +de ct +co yd +cityo fedmonton +ci ao +che atin +ch ines +cavali er +careless ness +ban at +bal bo +annu ities +ad ac +yun jae +wa hh +usc annenberg +triple threat +ten ens +te go +szczec in +stock ade +si gu +sanat orium +s lon +run ic +rhy no +re directs +principal ity +petere gan +parag ould +out doo +olm stead +o yelowo +o stro +national anthem +mer lyn +man movie +mal ate +lin dros +like minded +lid luk +kent aro +it ni +gi bi +gen om +firstdayof summer +fireemblem heroes +ent iced +e hem +do lo +design by +da ine +cruci ferous +com frey +chennai floods +carami asg +cad w +blue grass +bio active +baz e +anth ill +alam at +ak ih +ab ron +, ! +ðŁijħ ðŁijħ +ðŁ¤Ļ ðŁı¼ +åĪ Ŀ +ÑĢ оР+za inal +wh ines +weare latech +wa shu +vote james +un guarded +tho s +su panova +ster i +so close +re position +rail car +paridhi sharma +on alaska +middle east +melt down +me sse +material handling +maj nu +len ka +lau dio +lamp light +lac on +kauff man +jets am +j ru +isit octoberyet +if ers +iche tti +hou le +gender queer +gad da +form ation +flick ers +firsta id +fil oni +excav ate +envel oping +eno va +ecol ab +do yen +dema io +de winter +d hp +chickend inner +canadas militaryhist +bo wi +ben dix +be ir +bat ak +bar ged +az eez +aster oid +ami el +alien ate +akhen aten +afford ably +ae jmc +.... ... +.. < +ðŁĺ¢ðŁĺ¢ ðŁĺ¢ðŁĺ¢ +ðŁĩ³ðŁĩ ´ +war saw +victim hood +under garments +teen wolf +st ook +soldi ering +sm tp +sleu thing +sebasti ano +sar tori +rock hill +rit son +port man +pipe work +pi ala +oregon state +o dai +na hm +memori alized +mc garrett +marchi onne +malin owski +lit vin +lan ter +la brie +ko do +jad en +industri alists +hobb iton +hi jos +hawaii ans +glen side +gan grape +fur lough +fre ind +flu me +fake cases +ef t +eden vale +dev itto +detroit basketball +cpim speak +ch ho +ca er +buffo on +baj rang +ayr ton +aval os +as pin +albert ville +( )! +w pp +vern azza +vene zol +umm c +super bug +spi ff +speed week +small businesses +sign syou +science irel +ridge well +retin as +realron howard +ra ka +peshawar zalmi +pag os +our pride +ou ric +orn ery +oke ke +nsw police +nor quist +ne gus +michel son +memb ers +li wa +leather necks +la vaca +hor s +har tsville +haban os +growth hack +gr andy +ghostface killah +fiddle head +f df +ever hart +dre idel +casting call +belgi um += )))) +ãĤ Ħ +zam os +wetump ka +tre lawny +to dy +t cho +sym fony +student ship +scott sdale +sar g +robertir vine +qade er +pu ked +org one +mom i +mill bank +meat six +ma ila +lucky me +liv res +line ar +li zed +le ko +kor aku +kari ya +intimi dators +hypo glycemia +hjel m +hec paris +haver i +ham monton +ge el +fin ale +fi za +exhau sts +excre ment +etu c +diaph rag +de gen +daniel radcliffe +dak is +cri mel +colo bus +cloud waterbrew +chas er +canadianti re +ber le +banan arepublic +audemar spi +ane en +alessandro michele +ak at +af ric +ðŁı ¨ +ðŁ¤¤ ðŁ¤¤ +Ø Ń +ye syes +wait t +ve greville +uk c +travel tribe +trans sexual +touch points +tam es +swee pers +standrew sday +squa shing +squ ander +sam champion +sag res +radiof requ +pri mi +pn as +pearl thusi +patron izing +pakistan army +okoro cha +ner sville +natgeo channel +may umi +mah le +lu eders +lindi we +kri zz +king smill +ju lz +jan gh +hobby ist +heidik lum +h ti +gu edes +grac enote +gel atin +f ga +enic hols +dur on +du shanbe +drawl loween +don diablo +don ar +der win +counter intuitive +chi venation +chi da +chapters indigo +cat sprotection +ca hir +bun ks +brock u +bran cusi +ber rie +bel as +alun ageorge +# ! +ðŁĮ¸ ⾨ +ç ¿ +west world +ve ach +ti ree +tesco s +tar c +smoo ths +six fields +scam ander +san n +rad ice +queen latifah +pli able +pleasant ness +petr ino +pebble beach +otw ol +nano ha +my mind +mbo ya +man katha +ma ag +m paa +jonny bones +jn dsd +jeff hardybrand +jac arand +itsa adee +impul sively +immo kalee +if d +helple ssly +gi ppy +fair born +esthe sia +eric hie +dron field +deb tor +coiff ure +chow ki +calci omer +ca ec +bom anirani +bewil dering +beer me +bbc spotlight +bally doyle +az eri +aj inky +ach im +îIJ Ħ +Î ³ +y ri +wa ils +v de +un countable +ud g +thisis whywe +the story +spl x +si dh +qu iry +pit aya +pen alised +pag al +ober wiesenthal +o ef +ny ct +n ilo +mundele in +local produce +lign ite +lad ki +kill me +ht con +hill fort +hadi se +furio sa +fr r +enh art +emul sive +di yas +depress ingly +dee b +come dian +chi f +bc place +barak ah +avell ino +anti retroviral +ade ola +ðŁ¤ĵ ðŁ¤ĵ +zy na +zeej lf +yvra irport +x tv +vis itch +vari um +tol led +tin ia +ti dd +the weekly +the forum +tex ase +shed den +run jewels +rock pool +re doute +pyg malion +product pick +pol ity +north bay +neuro sis +neuro plasticity +lohan thony +kon ec +in af +grigor dimitrov +glori ana +girl scan +fre inds +fl w +espar garo +dur ack +dun ker +dor mand +dig vijay +dici embre +de sta +de ila +daven avarro +cor se +cor da +cohe sion +chin x +bra k +bote tour +bg v +bar now +ather stone +ari as +are ports +ar mond +ang elia +and ball +amand ine +akh shan +air heads +a discoveryof +ðŁĺĥ . +ðŁijį ðŁĴª +ðŁIJ¾ ðŁIJ¾ðŁIJ¾ +zeni th +ze be +z org +yetic oolers +wy d +was illa +wa iler +ton school +tier ra +thi stor +the challenge +te si +studio teabreak +stu ffed +stay fit +south coast +somuch fun +selen ite +sac ral +rgv zoomin +rad nor +quake con +privati zed +pat z +par ul +p wo +ol lection +ol ita +nu j +nhl network +navarre te +msu bears +mr drewscott +mand ya +mal tin +ma en +lil ac +kit ano +kar awang +k po +jo whiley +in b +holling worth +hefe weizen +gor leston +geo int +for ger +felicit aciones +fe ttes +es me +di mi +d ma +cross land +choosel ove +bet tered +bet ances +be tti +az or +aq ap +anti inflammatory +annab is +amp thill +al mon +ab riel +-------------------------------- -------- +ðŁĺŃ ðŁĺĤðŁĺĤ +à ¾ +w ttw +ver ger +venkat prabhu +thi js +tar th +sym bo +sun ray +sto les +spokesperson mod +sn n +sel cas +see f +sar na +rock hurst +quavo stuntin +oc us +naom is +mo ston +min ne +medic ate +ma do +letour neau +lemon heads +lego dimensions +kwe si +kis sable +justin mcelroy +j de +inthe know +inter loper +her bes +hen o +gisele official +for glory +fel in +esp acio +elo ves +di aw +del ved +del tas +cour ts +cha w +brush fire +brook sville +bren neman +beu ys +ad dam +ðŁijij âĿ¤ï¸ı +ðŁĩ¨ðŁĩ ¾ +® : +you rad +val ois +the con +te ck +st ny +st ille +soni agandhi +ro hat +rey mysterio +real tim +re structured +raj yam +ra thod +play mates +pic tionary +nu is +non thaburi +new tech +nca alax +national dayof +moo ji +lo wn +knigh tri +kish werm +khai dino +kal er +k sb +join ville +jodie marsh +japan e +ham di +ha upt +gro es +gla iza +ge aviation +gaw ler +fir s +euri pides +e usa +don gen +cun dy +courty ard +com ent +co ad +ch bull +cast ille +can apes +bron cos +bo tch +bo stock +bas ford +bal tazar +as n +ark wright +ap hy +adju tant +activ ate +acou stical +ðŁĺģ ðŁİī +ëĵľ ë¦ +w ld +usa p +uniof newcastle +the park +te gra +still birth +south downs +sol fe +sm illie +sle vin +sink holes +sig lo +san ha +sam pha +pet z +pac ts +oto ole +ofer tas +o sco +nr j +noi sia +nc s +nar do +more than +mont ju +mohic an +mis judged +marou bra +maj olica +liber alarts +last pass +lali gas +kla van +kir u +kin ski +ka ho +k hera +ji bril +jack hammer +is ere +impro ving +hell on +h ary +g de +fan tom +erike strada +er an +duches se +dol lie +den one +delray beach +death trap +dean wye +daily kos +co ffers +cheektow aga +cancel ation +cab bages +atp challenger +ar ouse +ar ona +andre ws +al cester +adv ancing +ðŁĺĩ ðŁĺĩ +ðŁĩª ðŁĩºðŁĩ +âĢ¦ âĺº~ +vish wa +uv f +trinida dian +travelo city +thom p +thank less +te ala +t bur +swag ger +star tle +spoiler alert +shivak umar +sc oured +rosari odawson +ren tino +pun ahou +prac ing +poo ka +pi pm +peg board +nor sk +news beeps +ndtv newsbeeps +mit os +me owing +majo rette +li der +lauren tiis +lady well +ko eln +kaz a +ka ap +ingthe future +imper ious +her mans +guar di +ginger snap +frit illaria +fran cie +extin ctions +eu budget +echop lex +dru mm +drake university +d fx +cro thers +cra s +cam cor +av net +ator ia +arama ic +alyss ahar +alber te +.. âĿ¤ +ðŁĶ Ļ +ðŁijĭ ðŁijĭðŁijĭ +Ä ħ +y ll +web app +treze guet +tothe world +ther ow +sy ke +suz anna +sultan as +suff ern +stagger ingly +son ia +sh anda +radi oc +pic sher +perit oneal +nar ain +mouse sports +mole sters +mobil ising +mish mash +midri ff +manhatt ans +maggie q +mac onie +look back +legend s +karyak artas +jor an +ib on +heis man +gru e +ge tti +fex cellence +et m +equ i +en lace +e bl +dill ards +cri se +corps man +centen arian +celoteh promo +castr ated +braw ling +bobcat nation +al brighton +ac bo +unite ch +ty y +sv t +strat ahadoop +sketch fab +shik sha +sant amar +saf flower +ros ling +par ta +on gh +nett leton +neck ar +n chc +multip liers +mu ammar +mor n +mor andi +mar ma +lan igan +kook min +kin loch +jay thewanted +ip x +im jadeja +i mee +i bec +hul se +hijack er +good charlotte +g elife +frozen four +en ine +droo d +digitale conomy +dep or +day ana +conver sion +com me +coloring book +coke zero +coffee shops +chat to +cat ena +c ally +bli shing +being maryjane +bat alla +bar win +argin ine +anim alia +af gv +âļ½ï¸ı # +âļ«ï¸ı âļªï¸ı +y strad +vand am +uniform ly +un convinced +ug r +si kandar +shan u +se poy +se mir +sb nation +pp ic +phra ya +nne di +mise ducation +lune ttes +list an +la ps +kyne ton +k nightmare +iver sen +inter min +ich ner +hod desdon +ha che +h mmmmmm +grijal va +gh illi +faryal tal +fairy tale +equi pos +energ ising +dragme down +do whatyoulove +do vers +degre e +deep ing +dam med +cs j +co chin +ci fss +chem trail +char tered +bray wyatt +bo hannon +bmovie maniacs +bi et +aff suzukicup +. ^ +ðŁĩ±ðŁĩ ¹ +ëĤ¨ ì¤Ģ +âľĮ ðŁı¾ +woo kiee +wc sh +was cana +ty lor +strangle hold +sl icked +shir l +shiel ds +sexu alas +scienti a +razz aq +ran il +pra bal +penrith panthers +pedo gate +p schools +osc ia +novonor disk +nether ton +mon archi +majum dar +lan come +kkkk kkk +kell inquinn +k offee +invier no +hunke moller +gu anci +go bulldogs +for ton +fashi oning +er za +ep ine +dro se +cul ligan +canvas ses +bun gay +bre mmer +ai ge +ðŁĶ¥ ðŁĺİ +ðŁ¥ģ ðŁ¥ģ +ìłľìĿ´íĻ ī +åħ ¬ +£ £ +week nyc +une ase +trun ner +ti gray +thi ele +ta ha +super book +star fish +spre cher +spirit awards +spin ph +skin head +si rota +se agram +schoo lies +sal oons +ragamu ffin +r dn +r bl +princen arula +prelimin aries +polit icking +pe ster +par cel +od ours +nac da +loveof mylife +l fo +kri styn +kirsten bosch +kat ar +ju bail +jarre tt +jan ab +jackson hole +j ta +ig bos +geome tries +ge hl +g ising +fa ha +der u +cracker jack +com une +car ruth +blu mberg +artif ice +al jon +!! ðŁĺĬ +íĥĢ 곤 +ye aaa +wr acking +wigg lesworth +wer der +ur vive +tv official +tru cco +trium virate +trevor project +top tips +time keeping +the ol +tat eliverpool +so ak +she affer +sh us +senor ita +s agency +ri dem +red ly +poit ou +par ul +pad d +opere tta +ol ajide +na dia +montal bano +mir ando +milli meters +man in +mammoth mountain +lok mat +lit chi +lin net +lam mas +l news +kun itz +kam rankhan +ka sher +hor st +hi stone +he most +flat top +fav elas +eep ly +dou bler +don avan +dh ing +cu li +cn es +ci opp +bin ion +banyu wangi +anti guo +ðŁĮ ¡ +âķIJâķIJ âķIJâķIJ +wizard weather +whit en +vision ary +villarreal cf +tu ria +tru ek +terri e +sti vers +sm h +sign language +shi ge +resource fulness +re directing +pr x +po to +os v +no sy +no ren +nat y +mu tai +micro fiction +metro boomin +maxi mo +manchester city +long leaf +le sli +l rg +kath ir +ji denna +hydro logical +hawk moth +gir on +flo aty +feroci ously +eli da +el bourne +ed ancer +dur ango +dhananjay ang +defl ating +daw gz +cosmo drome +cir a +cas agrande +bry den +ban presto +ay ano +athletic s +ðŁijį ðŁĺĥ +zen de +winchester bros +wh erry +wen di +we intraub +way y +voter id +vi asat +vau ght +under groun +un shine +ti mbo +stit le +scare ers +rodrigue z +rob son +rag wort +probab les +pri den +power metal +politico europe +narra bri +nan dish +n hi +matsu da +mae stra +lon go +lar c +koscius zko +kak ao +iso tonic +indv s +iloven orthcoast +hwar ang +hogsme ade +haz litt +gille speterson +ga at +f ack +ever quest +en gupta +dubu is +die thyl +desp an +danielle cormack +daniel tosh +dal len +brexite er +berkeley lab +anci enne +adri anne +ach u +zuk un +zee brugge +x da +wil by +who is +vie so +vesti ges +v apo +uu l +un selfie +ul tan +ud ta +ud hr +tre stles +timeto act +the valley +taver as +tamau lipas +subram aniam +spi rome +sh ila +sal ka +res by +rate payers +rashi di +rad res +ra x +pro ser +pr ance +photo sphere +pap aver +ob is +n anyang +my music +my exand +montal ban +mil nga +mil ilani +mb als +knowle dg +kir in +kar min +kak a +k alou +juven ile +its dre +ine ers +ic cs +hou gang +hollywood studios +ger al +gar butt +esc rito +ed enton +de vere +de kat +daf ne +character ful +chapel hill +camp fires +cage warriors +be me +bag gett +appal oosa +al et +aerop ort +ðŁı ij +zi yi +ym tn +weekende dition +weather photo +ved ha +ur mila +tri shay +torfa en +tom ac +thin i +spur ring +sophi el +slu shie +skor pion +shake able +sg f +scal pers +samark and +sam man +rose hill +proud lock +or s +open air +oneteam onedream +octane render +mu ang +mollu sk +mar wood +m skar +lam in +la zo +ku ban +k man +joel mchale +haw at +fu x +fluffy guy +flu ffed +fis erv +fa ile +f ts +ero om +eat fortheplanet +ducati motor +depar dieu +dd iction +cuer adio +crven az +clean er +claren ville +capp uc +c bridge +buzz worthy +bohin j +aph ra +an stru +an hydr +am ines +alchem y +ah san +afl finals +abvp voice ++ +++ +ìĹ Ķ +âŀ ŀ +âĿĹï¸ı # +z ef +was v +vc sk +v ava +up fight +tweet my +theri pper +th impact +talent management +sub group +sh tf +richar dro +reas signment +procrastin ated +pre existing +pic tish +pe waukee +over laps +odor less +nebu lae +muzi ek +motor i +mc fall +man fred +m mot +light years +legislat ures +leaf ed +lat ches +l nt +kofi annan +ko var +jyhe ffect +isi olo +invali des +innov ator +incan tation +hu eneme +ha boob +gy an +gall erie +g ately +frivol ity +fh wa +festi va +fa ience +euph onik +en em +di rait +da eng +cocon ino +cli braries +ci um +button hole +broad ens +birthday bash +bi vou +bbca siannetwork +baz emore +battle fron +bal mer +babys at +at atime +amon ster +amo vies +aly se +̶̲̥Ìħ ÌĬ +wil letts +ustin ov +urine town +un usual +u ef +twi gg +touch screens +thevamp scon +the bold +t lb +sty ler +sto essel +stal ley +slou ching +shel le +ser kan +scrutine ering +ro erich +ram ah +pod gorica +on film +o wh +north lake +lostand found +loc atelli +leather work +le hr +la ka +kat graham +k alian +john bolton +ingle borough +hase ena +gi ps +gal lia +fo er +dio cle +de g +dac eae +criteri on +coni ferous +car rabba +briar wood +ben alma +ay meric +avi ate +amy winehouse +abomin ations +yo go +y oooooo +wal y +wa an +universit é +ulti ma +traeger grills +to vah +theo bald +tar onga +tamir rice +ste ens +seraf ina +sat c +saar land +re activated +precision ag +par la +pann ell +octa vius +noctur nes +michel leg +me agan +mcguin ty +mc bean +maha bal +law ford +lan re +la gni +la gar +kel lam +international coffeeday +inter cooler +illu sory +ili za +her y +ha zz +gol fin +gho da +gh oops +gary clark +flatt ens +disper sing +defence less +cyanogen mod +culver city +creepi er +colorado stateu +cl td +celeri o +boston symphony +ber ate +bab ri +avn awards +au tau +arts festival +apra xia +ab els +[ ðŁĵ·: +:: :: +ðŁĸ ĭ +é rables +zan upf +wb afcofficial +vibr antly +tn w +tech expo +taun ted +tall man +skill man +skel ton +sir sa +silli man +shi ek +sc ler +sc afe +roo ter +redemp tive +re works +raj guru +pwll heli +pubg mobile +pic ka +oo dh +of eng +meningo coccal +lycan thro +j cu +home bred +gi ed +gas o +game informer +ex adata +con v +co axed +christma spresents +bov ril +bo ere +bj praj +bag chi +b ition +am aj +ale ix +ah b +achieve ment +ðŁĩ ² +ðŁ¦ij # +your game +ya ÄŁ +wh oot +west lake +ut s +un tenable +the u +sucess o +su bed +soci ation +shi raishi +seb gorka +sam ana +power fm +pla smids +pil oto +phe t +per kin +pare shrawal +o gie +no ko +newtown abbey +neu tra +nc sm +mug anda +mu dd +mi stran +mention someoneyou +maul ing +mad dock +lyn g +lipol ysis +lind quist +le flore +kine se +khat am +karma kar +intel sat +in x +hear d +hay i +gi wa +genie bouchard +gear boxes +gap year +fu mbled +e utel +dustin lynch +dic embre +decaffe inated +datasci ence +corsic ana +contrac tually +cla in +center field +ce daw +car ton +be cu +bcm houston +bad alona +audiom ack +ashe boro +ar naz +appreciation month +aph mau +an zu +alli ant +af fair +ãĤ·ãĥ§ ãĥ³ +áµ ĺ +ঠ¶ +vi ff +un civilized +tx su +transfer wise +te ju +sy leena +strat com +stab ber +ss rs +solan ke +shoe boxes +scru bby +ruffi an +rou z +rom pe +ran vir +pride in +pl z +p gy +nick kristof +navig able +nan sen +n der +myo pic +mut tering +mr ricky +micha il +mccle ary +lov ski +looo oong +lof gren +lo witz +live sey +juli ano +jeff ersons +iam fat +hel ou +he pha +epic tetus +edwar des +du quette +dire wolf +confi de +cere us +build able +boudo ir +as ala +ðŁĴªðŁı½ ðŁĴªðŁı½ +wav ers +washou gal +vill an +vijayfan strends +us v +un installed +tom wolf +thereal juicyj +the sushmitasen +super vet +stall man +sany o +sam ini +reflec tion +raj ma +ra sal +power full +pareido lia +pa es +p mh +owl city +oli vos +objec ting +o jessicanigri +northern assist +mvp school +mai ka +lumber yard +lo ld +j ir +happy new +h nic +gu aje +gre tta +fin dus +family tree +est á +ep an +elli man +dre wh +cook ham +congr ats +ca del +blo ve +alighi eri +ali ber +ad ao +acu er +actu alit +ðŁĺį ðŁijij +ðŁį« ðŁį« +âķ ¯ +ع Ùħر +ö y +wi er +west dale +vish wak +ur ich +trailerpark boys +thro m +theatre royal +su bah +seat ers +scab bard +pit re +per nell +p flag +out the +nov anation +next year +moro der +jim gaffigan +hom ura +go visit +gim mie +giff gaff +fluctu ation +fidel ity +dash wood +chipper field +cen ar +ce sarean +cath leen +bur ping +bur kini +bru gh +bare illes +bad land +ba stet +ay atra +audemarspi guet +al lum +aj c +ab ie +aa an +- ,- +âĿ ĩ +Ú© ÙĪ +you rock +y ster +wr k +von en +vir u +vas ude +ubun tu +total led +tiny url +tell me +t storms +sy rie +suk uk +sterili zed +sr sg +sol ler +sb learns +rum ple +rox burgh +rose crans +ro ko +ri serva +r ancy +public sector +peter capaldi +ou glas +objec tification +oak field +nu men +norwe gians +nissang tr +ner f +my favorite +muswell hill +much music +moon dance +modern design +mind lessly +man spreading +ly gon +luc chese +ling usamy +le sabre +le mp +lam ber +ky y +kis sf +katiec ouric +kabo cha +go i +fat man +et ti +dom ide +dist as +daily monitor +cou lton +clay ne +c maa +bridgit mendler +bom an +be ate +au w +asymp to +archae ology +apple white +ak azi +ðŁijĮðŁı» # +vintage books +video games +up w +tyour back +thecine gogue +test net +tele gram +tele commuting +tal end +sw are +sugar plum +spring vale +sp line +smar ia +slee ker +side arms +shun ting +shef vaidya +sean spicer +se mis +sd pride +rae els +pet ta +pen na +peaceand love +pan em +new sal +me out +max xis +man imal +ma stic +lastweek tonight +laem mle +ke vine +kav an +k orian +k lock +inter lagos +infer tile +in nigeria +ibar aki +hump ed +heat wave +hau ck +h ili +gt sport +grand rounds +foli ar +feature me +ew york +equal ise +ee i +e am +do wag +de face +david beckham +choosel ife +ch elios +cast ille +cas que +bin nen +big time +bang bang +ay alam +aw am +am yo +alde baran +æĸ° å® +wy ang +world lionday +window pane +ve itch +van arama +tor mund +tom ania +ti ppi +ta zz +sy p +sho twell +se if +se amen +ru apehu +r ÃŃ +probin sy +poo led +poc ke +on fire +odi ham +nove dades +med ell +mad havi +ma dr +kul i +kal ina +ka stle +iphone games +ic ap +iber dro +gv k +gratu ity +gan apati +f blchat +evacu ee +erec tus +disney animation +decrimin alization +dayton abeach +dag on +dad da +chi omega +c fu +book oftheweek +bo fors +beaut yof +badla pur +av ison +accompan ist +ab hil +:) < +ðŁijį âĿ¤ï¸ı +ëĬ Ķ +ঠĨ +zun ino +y uni +weekend wisdom +virtu alized +velve eta +vap i +up turned +under a +to plo +this flag +th street +tat ting +serv atory +schnauzer gang +san kar +ri ple +re version +raro tonga +po shan +pil sener +pe ko +p kt +odd world +la schools +kr ka +kha dr +j kl +international danceday +inspire sme +gw o +goode ve +gio van +fin lit +fili ppi +fam as +co author +caman pour +by day +bun ning +bele za +ba jac +ante ce +alyssahar ad +ðŁķ ĭ +ë Ī +大 éĺ +Å ij +} . +y lum +who sunilgrover +wetaski win +wak ayama +wach fox +viol in +vi kk +vash ti +u em +tu pole +trou per +su kira +ster anko +stanley kubrick +sf bart +se z +saraali khan +roller girls +rex burg +renzo graciebjj +rc psych +ra dian +pot torff +pon dok +parkinson suk +olap lex +now drinking +ni acin +mur do +made ira +lu mb +lon ger +loire valley +live streams +le shurr +kon trol +j miller +inj kt +gol pe +gods notdead +go khale +gam an +g ando +fe ducation +eph ron +ehren reich +dougla sville +di ur +d hen +college ville +cla stic +benig no +be any +arm ley +arca dia +ale many +adop tees +________ ___ +ãĭ ¡ +ye sto +va he +u wais +trin h +tic to +the boys +ter ias +ten ma +tau ber +si rocco +sazer ac +sas city +roy ton +raven hill +r bp +pacnorth proud +oppre ssing +og gia +national sunglassesday +mc kie +marri ot +mal appuram +loveto read +lo ti +lieb man +li ddy +last ing +kin ne +kellys later +jan z +ig m +iam valc +hay ford +hasle m +gu bler +fuku yama +extric ated +emer ita +dru mb +dj ima +dis missive +day trotter +co zier +co coro +clo set +cla ud +chi gh +cer vo +bur gs +bri st +bra es +blur bs +be eler +bap at +bag o +augu r +american muscle +alway sa +ali an +a hal +a ata +................ ... +ðŁĺ·ðŁĺ· ðŁĺ· +ðŁĮ´ðŁĮ´ ðŁĮ´ + ¹ +yash hd +we z +wan go +w web +vene gas +vanc on +v pr +usatoday sports +uni k +stream ers +ster a +sodal ite +snu ka +ske tball +sho tta +sab ic +ré sumé +rise u +ra ig +perel man +pelargon ium +p iller +orn ge +o dal +ny g +north stars +nigerian army +mq tt +mis fire +mc mann +jen ner +jay da +inher iting +highland er +har an +gli dden +gh anian +fl intri +farn worth +extreme weather +duck en +do you +dhan jani +chef tom +cat us +bo ast +bestro l +bene factors +an amika +am rut +ale gend +ak tu +aaron ovitch +த à®® +wf h +tyler thecreator +tur ris +to well +tk maxx +the buffalonews +tailli ghts +swar up +sk oll +sho chu +sen ja +ridic ules +ren stein +re connect +r vt +plec tic +myfreec ams +mid ter +micro site +mechan istic +materi als +malo los +ma gog +m tweets +lo llll +kirk us +kap s +kalon zo +kal on +k gw +jais ingh +j ach +irish whiskey +internal comms +inten tioned +hyper ventilating +ho taru +god fathers +fre eyour +fortun ato +fire fall +fin ess +e migrate +dou cette +di electric +deltar une +co sh +clari on +brook vale +bjpraj nathsingh +ðŁijĬ # +ðŁijĩðŁı» ðŁijĩðŁı» +ðŁĮ¸ ðŁĴķ +ðŁĩ¨ðŁĩ¦ ðŁĩ¨ðŁĩ¦ +리 ìĤ¬ +~ ? +ye aa +wo tton +wi spa +wi ggs +white helmets +w tmj +vy rt +vindic ator +vi ste +tv writers +tuscar awas +tu mba +tir reno +stre p +splin tered +spe irs +sp readers +south borough +shant y +sen tai +seal team +se um +schwal be +sand erson +sag arika +sa ara +rs duk +ro quette +ro bey +renfro e +promo tion +pro fusion +plow man +photo realism +paula abdul +ou verture +nebu chadne +morgan ton +mccal lion +mano tick +mak is +loc ally +lily allen +lee brice +lang port +ko yama +ker mode +il ux +ic han +ic acid +geis ler +gall inari +ful da +fly te +fing las +fin an +en ki +east field +e pping +di bella +dar ing +crimson peak +chu d +chicago an +chi klis +ched i +car net +bas swood +bas konia +ba xi +auri emma +al app +air less +accou tre +ìĦ ł + µ +wick ens +vaxx ed +urban farming +trishay earwood +ther rien +sy ork +swin doll +seon ho +senec acollege +red breast +recti fier +priyan k +priorit ised +pp as +pic cal +peup le +perme ates +pau li +pan handling +pa o +pa ic +out grew +obam ain +nai as +na ep +mis quoted +master craft +mar ak +mag a +liter ati +law dy +kor oma +ked out +jan in +halle lu +guil dof +gentle manly +fu ld +frog man +fran ck +far hat +ech ols +disp uting +da best +critical care +coti ja +ci z +card captor +boudic ca +bou cle +bar ren +ball sy +at ell +ar ata +am artin +akh bar +ðŁĺ³ðŁĺ³ ðŁĺ³ðŁĺ³ +zoey deutch +y ook +wta finals +wojci ech +van illi +un kempt +town send +thar vest +swi ggy +sod bury +slic ks +ru si +ri mi +re building +pro fastpitch +prescrip tive +pp ah +persi ans +of ws +od hi +mom and +mimic o +me j +mccl anahan +marlene king +ly anna +low man +le ffler +je red +have you +haha aa +gw ire +gro b +geo g +ga ara +fv ck +fox croft +dicken sian +di pietro +d hat +cor ne +clam bake +carbon ell +ca ia +bet amax +battlefron tii +alex salmond +agre y +adelaide kane +ad hu +acade mi +ðŁij©âĢį ðŁİĵ +âŃIJï¸ıâŃIJï¸ı âŃIJï¸ıâŃIJï¸ı +yel le +tope leven +theophil us +sy t +sk mch +sd lc +sar do +ra ssi +point blank +outw ar +ou vert +orgul lo +ny it +nature is +mö tley +mo berly +melancho lia +mar cho +lumin ance +lau tern +lab out +kw ak +kru tch +kne els +k bm +ju suf +jockey club +jo inter +jer ri +intothe woods +implo ded +i mu +homos api +hap kido +g mv +for sure +fia worldrx +fel ts +fari d +far ma +fantasy sports +fan uc +ein ar +du y +choo sing +ccm hockey +cancer survivor +buil dit +bri gida +book tour +bew dley +be brand +ar onian +ðŁĺļ ðŁĺļ +à· ı +wolf dog +wo ols +vill ani +u kun +tupole v +ten no +tam al +stil bestrol +stem less +st baldricks +scholast ica +sau t +retro fitted +qu as +pas si +oste opath +noel fielding +myan mar +ly t +level and +ilove the +hunt music +hal ftone +gyro scope +guanci ale +glen bard +gentile schi +ge os +gay ath +gab es +freed elivery +fra gs +forsy thia +fc women +ex pository +elie bers +el da +ego ist +e par +ds bury +dl cs +d bradbery +cork city +construc tivism +con ut +cle ur +biodiv library +b ba +as che +and new +an ette +an er +? ¿ +ðŁĺ³ . +çĻ ½ +âĺĺï¸ı âĺĺï¸ı +vespu cci +vand alia +tri star +tall boy +sweat band +sunday night +st eck +shovel head +shop talk +separati sm +rivend ell +pho sis +pa chinko +obe ys +nomus limb +noah cyrus +nc g +mith ila +minecraf tedu +mc clinton +manic monday +m pesa +le ddy +lb gt +john r +jesusis lord +jesse b +insu re +in sti +im pa +hu tan +hoo ple +hol te +haroo bom +guany in +ger ontology +ful vio +fu li +ers ch +endodon tics +descrip tor +coaching family +clar isse +chi em +celer on +c gf +bogdan ovic +bo ku +birthday yyy +ba shi +att ell +as elfie +ar oll +an tastic +am bert +ad ink +a age +âļ¾ï¸ıâļ¾ï¸ı âļ¾ï¸ı +winni em +verti ser +unsig ned +translat able +ten newsadel +tall ent +tak har +stone gate +sky arts +sit aram +shi rai +seman tic +sal ting +rose mount +rac o +pieter maritzburg +pal encia +pa kai +non point +metro bank +manipul ates +man kiewicz +log ar +liver ied +kar din +k sy +indr ani +in trust +iam king +i kari +horni man +heav iness +he me +ge burt +gam in +gal lus +friday funday +fo ta +e tape +du barry +cryp t +cruel ty +compar ably +cle w +claym ation +che ah +ch ander +boy cie +black n +bel co +beat maker +bcli berals +arri go +acbo fficials +< ~ +ðŁĺĬ ðŁĺģ +ਠľ +¬ë ² +tul li +ter nal +spri ggs +so ce +sam smith +rutledge wood +robu chon +ri sha +potom ac +po tawat +pla que +patr oness +national tree +moombah ton +mm un +lyme regis +kill erton +jet pack +im posters +iamfat don +hf cs +haz aras +fit bit +enjoy the +eastere ggs +dismember ment +decarbon isation +crime sof +coffe yville +civil right +bu tyl +azi za +arn side +alex alltimelow +af it +adelaide oval +ad ad +âĿ Ģ +wicked tuna +vaccin ating +tu in +ta kagi +star ships +south fields +sing apura +shir ted +shi bori +sd learns +sau geen +saber cats +rep mark +r tc +promi sed +porter airlines +par r +p ome +ovi zioso +nou rish +ne ah +national burgerday +mou stak +mark akis +man sk +liqu i +la po +la goa +kuma on +ki zzy +ke ween +k dm +jal ali +inter scholastic +indi ain +i its +hunterx hunter +han alei +ghet toradio +g kn +fif ths +ff w +favor itos +exi de +duc ting +care x +camer ons +breast plate +break point +bhar per +beef y +azmi shabana +au bry +as cot +ann ick +andread ovizioso +agno lotti +ac delco +ab alan +âľ ³ +âķ ¯ +ya al +wunder bar +w jac +vers day +vas sell +twee gram +tourism goi +the emmys +the cur +the bma +tes se +sy rus +swee eet +slam my +sc lass +reck less +pu tyour +pre ter +over runs +oh man +of ra +nj t +ni bal +net i +minare ts +maim ed +magn animous +ma zer +m net +le stone +ko ei +kay lan +john varvatos +jj b +high light +hand fuls +guardian aus +go bearcats +gar dat +fort myers +flacci d +e sop +demb élé +chennai express +ce asar +bio synthesis +beren stain +baesystem sair +an ila +am per +alex avega +abur nett +% % +ë¹ħ ë±ħ +ä¼ ļ +ಠ° +world travel +wor mald +us mca +tyler j +tin fo +sw pg +sun sentinel +su tures +stre ett +ster k +sh le +schu ster +scam per +s yos +roc kie +pon ding +per usal +penn ell +noo tropic +mon tell +mee tha +mar tham +kuch rang +kor bel +kaji ado +i marleneking +hi gley +hi bbard +hei sts +haun ter +har der +gc sa +friend lys +fi daa +extinction r +er oo +e sign +draf tee +del illo +de red +de carlo +cooker y +construc tively +chula inn +cher ly +bou e +bm j +blo cs +atom ium +ann able +al resford +al con +abdel aziz +a hara +Ùħ اÙĨ +wiel ded +wang an +wal den +vin rana +track town +tit ano +te jash +subtrac ting +statist icians +st nyc +smackdown live +shop lifter +she ung +shaf qat +selec tric +sc ba +sad face +ré my +rur ouni +resto s +regal ado +re sound +rb m +pro fli +pre diabetes +pitch ford +pee phole +ostr aci +ok ita +ne bl +lau ria +la ffy +ky ong +jazz day +intro vert +immacul ata +how se +hospit alizations +ho tography +her dman +hard wood +go de +gh ulis +g ats +fox hole +f ellers +en acts +elizabeth banks +ee ep +ec ousins +dra ge +designi deas +delph inium +cor do +constitu tionality +can thus +cam ryn +bukid non +bri ers +aviation week +anti elab +am phi +ale f +agul has +a oc +Ùħ صر +é nez +ymur phy +yar o +x bl +warren sburg +walru ses +try fan +the martian +tele kinesis +stim son +soli h +shaw ol +rick santorum +por tor +plo tters +par vez +par sing +p mm +okon kwo +mu dgee +men cken +ld t +ko slow +klat en +kick starting +ker bs +jo co +in wardly +in significance +ilove makonnen +ig tv +i sher +ho vis +graphic novels +go aussies +ful cher +fon der +eu sew +equili brio +dogsat pollingstations +d tm +ce ta +can uk +c atia +bwo y +br aman +ay el +ash rae +art collectors +arch ery +amo a +adot com +" .@ +yu cky +un nao +team zay +ta ware +street pass +strad lin +speci ation +sk at +si sq +sal us +ravin der +peregr ines +p ama +ope x +o sor +nar di +nag er +mis fortunes +margin alia +mar gs +mak os +m sam +love art +lin zi +le gar +lam on +koi moi +je ppe +its thelittlethings +igh ty +hudson sbay +hei ke +hang ten +ham n +hac ia +g top +fore skin +f rica +embryo logy +el ounge +djafro jack +c gy +bin sky +bet wixt +ben alla +bas enji +baby love +b hang +ast r +ar av +amade o +altam onte +adida shoops +?! ' +"" """ +ðŁĺ³ # +⾨ . +xxxx xxxx +wjac tv +win ship +uniof herts +ubiqu iti +tit ration +sun and +soom ro +son at +sof ascore +so loway +sle aford +si stah +re ser +pro curing +porter ville +n kr +megam illions +lac ounty +ku za +kor in +koo zies +kill ary +jo ssa +it ta +iklan onlineshop +happy friendshipday +gul lane +gu zan +floof y +euro beat +enchan ted +ely xion +ec w +ec entre +cu bs +crucible theatre +crickho well +co geco +chiar afer +cal ve +burk ard +buffe ts +black love +atas cadero +ar nel +app x +ap lomb +ana am +al timeter +al pi +ðŁĺIJ ðŁĺIJ +ðŁĺµ ðŁĺµ +worldcup russia +wood sphd +win spear +wayne state +w spd +ver tes +ve ste +vas sa +uk biz +tol i +thor ror +tat ami +tan sy +smy th +sla gs +silver wood +rum chata +rsp ca +reme dy +ramim alek +q rp +presby tery +optimi zes +ol ena +nfl top +nbc agt +mo aarena +ma san +m pps +lit ton +len et +kw ana +ke z +ke il +kan war +ju ang +jar ritos +jack box +ir van +ir th +huski e +home grown +holiday sarecoming +haz bin +hagg ar +gir d +gard ell +fri go +for ca +fati hah +do to +dal more +d ci +cyber warfare +cil ento +chir k +che mex +born free +bat te +ban ham +austr alie +au spices +asp net +ann ale +ðŁ¥ ľ +wy ong +wood fordre +wom bles +war horse +wa aa +vesti bule +tre pi +then ext +the garden +sugar ray +seaw olf +sc aup +s victoria +ru pa +ro cin +ri ii +ram leela +plos biology +pang aea +oyster catchers +never too +nas m +n gee +mut are +mtn g +mr dan +mal ta +ma im +le tu +kar ratha +jol in +indy wrestling +hodg kinson +frank lyn +francis ca +dri ppin +dak tronics +con desa +co pps +claire richards +canni bal +caled onian +back flow +avent ures +ath ina +ar ve +angel cake +am be +ak hir +ai reland +agit ator +acol yte +a and +== == +ðŁļ º +ðŁĻıðŁı» ðŁĻıðŁı» +ðŁĺı ðŁĺī +ãĤ ĥ +âĢ¼ï¸ı @ +z off +yak in +tre g +the junoawards +terrorist attack +st ager +spe cht +somerse tccc +shap en +sen kamalaharris +se mo +sav ard +re ee +pamuk kale +nutriti onists +nov y +newyork times +naught iness +nassi f +mari ela +maiam itchell +lun din +love with +key noting +ion o +infu ser +hep worth +harry style +harmon iously +good win +g tlm +fragon ard +fin sub +fantastic fest +er rr +eg mont +du ende +disintegr ated +courty ards +burde tt +bur scough +bot vinnik +blin ker +bier garten +bethe legacy +bed bug +anthropo id +al ounge +agu iar +adver b +a aps +âĺ ¢ +ÅŁe hir +up adi +un moved +u pa +the district +tech uk +straight outt +sto kke +sp ittle +soun der +snap yourworld +smi les +sharks rugby +ser re +sedge moor +sead ragon +rhe sus +recycle d +queens bridge +pri aulx +on ramp +ok ko +nen y +n cat +michel ob +mari byrn +lifeand style +li sag +li ann +ley en +leon ar +lar b +lam pert +kom pas +kof c +katv ond +hu bbs +guv nor +gro o +gal o +fo zzy +fer man +el bow +el ad +dar ty +cor ton +co ahuila +be kin +atta i +atori al +arts jobs +art ill +ðŁĺŃ # +ಠķ +à« ĩ +woodfordre serve +whe ezy +war ners +uzo aduba +uni strathclyde +un yielding +u hmm +tun as +team green +t bo +super jet +su je +strongh oldgames +sth all +sp ao +smash box +se jong +scale model +saber tooth +room ate +ron ny +roll i +ro mulo +rahul kanwal +philadelphi a +par vin +nws spc +nol en +ni rav +na hhh +movie goers +mm romance +mid gley +marav illo +mal maison +lori da +lef twich +laur it +kor ine +kamen rider +johnson pga +infantry man +inc ites +ge an +for ro +ffici encies +fam ished +extern ship +dwigh thoward +chuck le +ce ed +cab bies +bla zed +bet ws +be zan +bag atelle +ard ner +arc tica +al ata +ag w +? / +ðŁĻ ģ +ðŁĺŃðŁĺŃ ðŁĺĤ +ðŁĺĮ ðŁĴķ +ðŁĴª ðŁĶ¥ +youn gli +yorkshire tea +x p +wayof life +vu vu +volody myr +vasund hara +var dar +traumati zing +to give +there sac +teddy bear +su thep +sor optimist +sol era +sin ar +sch litter +sc ram +sa bet +rode os +remedi os +re settled +ran ka +qui vering +north cutt +nigel slater +nex en +moog fest +mark tuan +longre ad +lees offer +kor ina +klay thompson +kar mann +jesse leesoffer +il ig +hynd man +harbor side +han neman +ground lings +gin ola +ghome shi +fish mongers +fc cincy +ex claim +every thin +ely sees +dark phoenix +cy tok +co incident +cityof culture +ci mo +cae sarean +bel len +bcel xn +bar m +ba eum +aren ta +z no +yel lowing +xher dan +wood tv +wester man +w th +vo ith +v sat +tow bars +tattoo art +ta phouse +t sim +st ner +ssan tos +spar za +ship ton +scru mpy +scorpi us +school bag +rat tray +ra zer +plann er +piratesofthe caribbean +pherom one +pet sy +p sla +ofor i +od ilon +ning news +ni fa +naf tali +my dog +msk ristin +mm urray +melissamc carthy +li kee +le strange +lapak gue +lan chester +la via +johan son +iter i +house off +hor ny +gu aido +g elli +flumin ense +fire fan +fine wine +film linc +famil yo +fab ry +ec am +eb or +culture trav +cl ung +ch ack +cf ds +butcher babies +bru isers +brebe uf +bo ree +blan keting +bhubaneswar buzz +be wilder +asser tions +amber jack +ag y +ðŁĺľ ðŁĺĺ +ðŁĴĽðŁĴļ ðŁĴĻðŁĴľ +ëª ħ +w ciu +tun gu +scotts bluff +public ised +press ly +pie zo +pale ale +nix ed +newhi phop +ndam ukong +narcis o +mo den +million aire +mand ers +low rance +law we +lar king +la vo +kid suk +in und +immer sive +i ste +haunted house +gov summit +fuse tv +fr inton +f king +ell ora +educ ative +deep ti +cole us +cl x +ck enna +chant ment +chamber music +carl sson +can ad +c sat +bo bm +bio diverse +bet tering +b kk +aishwaryar ai +ag no +af ol +a uni +ðŁ¤ĺðŁı» ðŁ¤ĺðŁı» +âĿ¤ " +xavier woodsphd +wp gc +we chsler +uplift ment +to zzi +ti ent +therain makers +the herd +terror monitor +ter ric +sud han +str in +stl today +ski ba +selec ter +san guine +salu ted +rum mel +republic fc +ree per +ra sc +proud tobe +pro va +pau to +ote dola +news dict +nat arajan +mor ison +mono kini +mcen tee +maris sa +man ar +ma bee +line webtoon +li rfc +lancaster uni +la due +kat o +kan del +in lan +ifu gao +if k +dswd serves +dri d +das ch +corn fields +circuit cat +brunch bookchallenge +bow ker +boat building +bar in +az ra +axis bank +assi ani +applic ators +aper fect +ape e +aha va +ðŁĺģ ðŁĴķ +ðŁ¦ ı +æ ® +à· Ķ +with iel +wil iam +w api +veteran s +u selection +tvweek logies +thru sting +suf tum +stu die +spo tt +sor ors +sajid nadiadwala +robin roberts +ri kishi +red legs +ray music +randy houser +pat or +pap ix +omon di +od endron +nebuchadne zzar +memorial u +maroochy dore +lu rid +li ese +l tu +kit up +johnshop kins +iam santhanam +i arc +hy wel +hot ch +hang ings +ha vel +glo cken +fri gging +fit oor +fish pond +esp re +e hm +dy ke +du q +dave bautista +creep iness +comb es +co ds +claustro phobia +card illo +book fairies +bo caue +billa bong +bass guitar +bart let +aw yer +assi stive +ar ry +ap acific +amo y +al ocal +? ðŁijĢ +ðŁĩ¬ðŁĩ§ # +íİľ íĥĢ곤 +ãĤ ¡ +zo calo +va it +uma ir +travel agent +traffic sa +ton opah +ticto cnews +tah j +ta dic +sport stech +spa strana +shan emc +sep tu +sarac en +re hm +py at +pic oftheweek +part ington +park ade +ou dt +news comau +neutr alizing +nar berth +mtv movieawards +mo bb +mark t +mak ina +leth waite +la france +l ng +ju if +is landia +ink lings +ide ale +hol ac +hat tori +hat day +g benga +faken ew +fa zed +english wine +dead spin +da ves +cory don +church gate +carri ef +cari bana +cabinte ely +bryn ner +br ach +bon ington +block heads +bbces sex +athle tica +am our +am by +am bie +ale aks +ðŁĵļ # +ðŁijī : +ðŁ¤¤ðŁ¤¤ ðŁ¤¤ +اÙĦع ر +اÙĦ ÙĨ +york swildlife +yaz d +wine enthusiast +whit man +wam aga +ville franche +ve sa +valdi via +triumph antly +tony hawk +tizi an +tem pest +tat jana +sli k +sier ras +shau sa +sarban andson +red state +radi olab +plan eth +pis ang +pat ino +or cia +ome i +nor mans +mohamed nasheed +ma key +lower town +lo di +len nart +landscape painting +kermode movie +juni pero +ivy bridge +il al +hel li +gb ong +ff k +distor ts +dis assemble +davi dv +cn tower +chro matics +castleg ar +carls jr +ðŁļ ļ +ðŁij ² +ðŁIJ± ðŁIJ± +ðŁĮļ ðŁĮļ +yaz idi +whit ener +wal green +waf u +wad desdon +w enda +typo graphical +tweetyour friendshipinapicture +tricol ore +tou ken +ste yr +stan wood +spring ishere +smo ky +sle gends +sham ans +sav pak +saniti zing +sal z +s borg +quintu plets +post script +pin elands +pas sau +oscar ssowhite +on ate +nu ove +non surgical +nir anjan +ni ña +nex tel +morning breeze +mono block +mo hi +metu chen +men age +man ca +mal ou +lo xley +leop oldo +ki u +ke mar +kam ani +k mf +jail bird +j suis +j pr +hu ell +g ve +fle mings +feren c +fe asted +dere chos +cop eland +chur i +bu sto +braw lers +aug menting +as pl +als fan +ag ente +after burner +ðŁĺįðŁĺĺ âĿ¤ï¸ı +zoo k +z t +ware gem +vv v +vol kov +vand alize +un mc +udo biz +trans bay +tele visa +syl vian +shafaq naaz +sh h +sf n +sey music +sel mer +roald dahl +pr ing +pick pocket +pa atleti +o leo +nid derdale +mo zzy +mo loch +mis aligned +mets at +m sca +likefor folow +li esl +laureland hardy +la im +kw ant +ko ber +k pt +jun ot +hus se +hon es +hin k +hagger ston +h cm +gr atia +gor l +ga iam +fim mel +feed youra +enor mity +em ley +eli ver +dt p +dravi dian +din an +deathwish coffee +co pics +ck lw +chilis jobs +ch rom +bu ys +baeum ler +av ul +èī ¦ +æĿ ij +zi pl +your best +way fare +wamaga isa +va alu +v tm +um on +tab oo +tab ard +super smashbrosultimate +rhi zo +ra pini +public theaterny +pal anti +pack in +mrpeter andre +lu gh +lat ching +l ici +kuznet sova +kir stie +jos lyn +jesse mccartney +j league +im pati +hei ko +he flin +hap tics +ha art +gre ely +good people +fr aley +escape the +er oute +energye u +dis continuing +der de +defin etly +de ba +cu neo +cow al +clu tter +ci one +cd f +car ma +cal amba +bu cu +ba sham +apil ot +ap sara +îIJ ł +wood cuts +try ing +truth fulness +the aaryan +theaaryan kartik +th ire +tao ism +sound proof +sho shana +serv is +sarbanandson wal +sany al +sabre tooth +re distribute +rath aus +qu ed +nat to +nam ak +midd les +michi gand +liri ano +lig ature +le ey +kay lee +kal yani +in get +gran it +goli ad +g cr +fle m +fla bby +fi qur +fat burger +faith nomore +ero ss +ep stein +dry ad +dist ant +dent ons +demic assiani +dam nnn +daily productpick +coffe ero +bishop jakes +bene tti +bdd sw +ant inous +aise bhi +ðŁĴ¯ðŁĴ¯ ðŁĴ¯ðŁĴ¯ +ðŁ¤¯ ðŁ¤¯ +Ø · +ze tt +wr wc +wi gh +west palmbeach +wa hala +usac nation +un dr +team ol +stack pole +sport stv +soap box +sk ind +simon harri +sap hir +ph ung +par ole +ow yn +oli vers +ni xon +mo ong +mi fune +mel ancon +mas ry +m ÃŃ +lord mayor +lev ellers +kk tv +kh any +ken si +islam opho +inciner ation +her mits +gi gli +friend swood +for king +enchan ts +cordy ceps +copp inger +circu s +che tna +car char +caf u +boon en +bar ter +at ab +ang lin +amitab hk +Ï ī +wak ame +votejames fpp +un dead +tor chy +thejeremy vine +thankyou for +ster nation +steph eng +stein man +spir al +smallbiz sat +seabir der +richar lison +rec enter +q ca +puffin books +pel icula +p onto +ostent atious +opini ones +ony x +ome z +new comic +neel um +nau tique +mul laney +marque es +mark martin +leigh j +kodan shausa +kirkus reviews +ka fir +k mp +it ts +ise o +hil dreth +here in +ha warden +g sw +fidd ler +fi be +dy in +dragon quest +dispos itions +dha dak +dand i +cre swell +choreo graph +ch ir +cfis d +cash cash +bridge hampton +bally more +athanasi us +asso cham +anai vanovic +ðŁĮ»ðŁĮ» ðŁĮ» +ह र +س ر +vin oodh +shom rim +sh rank +savi on +ron gai +res ents +re assembled +qing hai +produ cex +prin ting +pal am +p mpc +op ene +ole ksi +oak park +nb m +mus que +mi ér +mg l +maje ure +lu met +line out +life hacker +joz ef +its worthit +iti ka +is ki +inter facing +indy car +incur sions +in breeding +hurry up +hir ano +grand ads +gal lie +fer man +endome trial +e les +dor gohome +djan go +dear den +dand an +cu pped +connol ly +colour less +character art +bu stelo +brech ts +breakthe internet +brack ish +bm z +blue dot +athar va +ala id +acu tie +ach ange +> /// +; ' +! ": +zu mbo +yo do +whadd ya +ver band +tri pods +tre p +they ve +the travel +the offic +st vincent +squ ib +spo or +sphy nx +r pw +pull man +pray ag +pic cata +per is +open gov +ol ture +nem ours +mute math +mu ti +miner ality +map box +lland rin +kim davis +jail er +id f +hydro graphic +hul ks +hollen beck +ho bble +har ken +han ews +ha a +gor sein +gal ton +es boeck +du guid +derail leur +co wer +close thegap +cell ini +cameron newton +br dc +bo or +beste ver +bas smusic +bam teddy +author life +actu alization +è ½ +âľį ðŁı½ +yn ys +y lo +vap id +trump y +tow bar +teh sil +str s +stit ans +standard bred +spring boro +shar ona +shand on +sh room +rand hir +rah me +privati se +pierre bouvier +pa kar +oy al +o qu +nye rere +np ci +ni dra +newss yd +ne ef +me v +m stad +lis icki +jen ning +ion ic +im bula +ick x +hy phy +haley reinhart +germin ated +gag li +fo ckers +flu sh +e sai +e gi +dise gno +demo ed +clo e +clo bber +cant stop +bu ttes +bo han +bent all +ax p +ari ums +argon aut +_ " +ðŁķ ļ +worker sday +wis den +w cbd +u at +trutv jokers +tre w +teat re +subpo en +si ad +sen ation +sap ele +sag i +rival do +ri probin +re vises +pott sville +ny cw +nt fm +nh mrc +ne ches +mun tari +magnit sky +kann on +kade em +j stor +i qs +hy th +hy fr +hog wart +gra ving +godbless our +global citizen +girl hood +galler yof +fabric ant +everything nyc +engag ement +ed cam +dul ity +dri bbled +dr amar +deccan chronicle +colo gy +code of +cap elli +c do +ban jara +atop ic +ati e +allen by +al pe +ah ills +ðŁį» ðŁį» +ë£ ¨ +âĽĪ ï¸ı +yuv raj +vi ver +v ahs +un buttoned +the in +tex change +tar g +swad lin +super conducting +sugi moto +sta ghorn +social marketing +si dious +schmal tz +sarrain odu +santi gold +sag meister +ru pay +rough y +oun dup +ou ston +oppos able +operation smile +min os +mhi esboeck +medi auk +ll ys +kir st +ke io +kate upton +kara age +jack wilshere +gal adriel +fans bts +dirtb ags +dialec tic +devi ated +dah li +cull er +crystalli zation +cory ell +club foot +cal in +bm g +baby bel +ark adelphia +ann yc +am organ +ðŁĺŃ ) +ðŁı Ķ +ä¹ IJ +âľ ® +ym ack +yax ley +wit ney +win on +wh an +ween y +w angs +vu illard +uc am +triste sse +th c +sun dogs +state lessness +ssi g +rox borough +remin er +racer mag +ra hn +qu alia +prab ha +poppy seed +piac enza +one championship +official mopar +neutr ons +ne hi +n hai +mat tox +lynch ings +lyn am +ligan ds +laur ac +kam elot +jeffre ss +il am +hottest dayoftheyear +hockey fightscancer +hi ki +hass ani +glyco gen +esc ola +effec tor +dor ma +din als +daz z +coton ou +cigar life +chryso stom +chick asha +chee tah +bug les +bu tina +benalma dena +ax minster +am ref +all round +ai ri +a ing +? ~ +ðŁijį ðŁĺĢ +ëĶĶ ìĹIJ +د ÙĨ +yum miness +yu bin +vinyl junkie +tra ppe +tony bellew +tn ite +tb ay +summ ited +st ary +skyracing au +simonharri std +sig mar +shi flett +school craft +saliv ating +s thetic +rot man +roadto state +remain er +oli day +mon star +moder ns +marie ke +main street +ma ik +li hat +kat ze +j tg +iter ates +hereto create +goo dridge +gli de +glasgow cc +fati gued +eric john +easy going +diver gent +digital marketing +di zi +derma us +de chart +dad aab +collecti f +chuck wagon +car suk +camper down +bran k +bou lang +ballist ics +ash verse +aksh aya +ðŁĺĬ âĿ¤ï¸ı +ë¶ Ģ +winter meetings +white water +v aper +tur kistan +trump f +thel or +the starters +the fin +t na +sho cker +shi ppo +red gate +pun i +pr v +or kin +om aldini +og more +nj rotc +new scenter +mv mt +monu sco +med lock +lec los +lal anne +ky lec +kt bs +ker bal +j anya +isd strong +inter war +hyde park +hoo kin +hockey roos +hei den +göte borg +grant thornton +factor ial +equal marriage +e gar +dev tools +del mas +custom ers +case book +cam co +calstat ela +ca ho +c gd +botetour t +bol g +bear sears +avori az +argen teuil +al ac +aco tta +abudha bi +ðŁĮ± ðŁĮ± +west word +tar bert +tail back +sush mita +stric test +science march +scal ped +sar dan +sab zi +sa o +run nels +ro tham +revol ts +replic ant +r ously +po ti +pilli ga +out look +nu ba +n mm +n ello +mind thegap +mil ch +messi aen +me se +malign ancies +liza beth +la din +ka at +ju mat +joaqu im +jarry d +j rb +iom mi +invigor ated +har un +govin slee +gon do +gil let +g news +freddie gibbs +fre sher +follic ular +eric metaxas +elo ck +dumb asses +dri vel +do pp +diver gences +cymbi dium +cs russell +coke studio +ce sena +brig adoon +bre h +blood less +blaen avon +bhar u +ber ke +bat ok +ban sky +bac io +asser tiveness +amag ansett +alwaysin our +í ŀ +zon dervan +wild pipm +widespread panic +wear mouth +wak ka +under sized +un cooperative +thin ku +st pauls +sinter klaas +shro ve +ru bel +robin ho +ro is +pre med +po di +pen alty +old photos +o witz +memb ering +may fire +masc is +mag ness +ma bey +london stockexchange +len or +kod ama +jim mer +it ap +im f +ie i +iamj hud +hypothe tically +gur sky +gh un +gen ge +fore heads +fo i +fli ed +fire crest +droo p +do olan +dim and +de value +d to +d ks +cor po +cond ado +comp sci +commit tothe +cla ver +carmel lawwe +bru mb +bigi deas +big bear +berth old +audi of +assassin ations +art sed +ar mee +alim ent +? ;) +ðŁĺİ âľĮï¸ı +we bos +water skiing +ve ee +v af +un bowed +to yah +tm hs +the wolf +te sd +tand ingan +takra w +symboli zed +sukh bir +spring day +sell outs +sc ylla +samard zija +re published +pv m +pu kh +os w +or na +north way +nico let +n gai +mun shi +mg ma +meh ran +me is +luke bryan +lef twing +lapakgue com +lan kans +james b +id bi +ick ens +hello oooo +hardik pandya +gatecra sher +fr ates +fountain head +duc t +donnyo smond +don iveson +chi pe +ce va +car to +car makers +bosch etto +bon dy +bo cc +big by +benson hurst +bel os +bal len +b pb +b dy +avoc ado +av ailed +as d +an ay +work days +union dale +un packs +tw all +thra sher +tan ahashi +tai sha +suf field +star sports +sin ner +pri l +pml n +pie tahouse +pic ot +pf n +or on +on ette +monon ga +mit suru +maus am +mat tu +maruk mani +klu tz +k tar +jo bless +jerry garcia +javedakh tar +iq bal +he marukmani +haz an +hay nie +gun reformnow +grav lax +gol akers +get loud +germany tourism +garyclark jr +for dyce +fen di +eu metsat +endy mion +eg ler +eco bank +du ffin +du ale +do wel +co klat +car nie +cant ine +brad l +bau mer +baf inals +ath omas +ar gan +ar dy +al kan +ad journ +ad ders +.. ðŁĺĬ +ãĤ ı +wor leans +wo om +with nail +wim mera +usc apitol +tidal hifi +ti sd +thelast kingdom +the dirty +tez os +tendin opathy +team depot +takeit back +stann ard +sp itz +smy ths +sheer ness +sen sherrodbrown +river island +regu lus +ray ment +ran z +plu it +phthal ates +per ham +nu c +naruto shippuden +mountain life +missi o +lore en +leon alewis +keeptalking mh +karam bit +kar rie +ka iri +jehan gir +jed d +himm ler +himalay an +hesit ating +gen tian +garden ingtips +gam mal +fl ory +ellis ross +el low +dayin thelife +cross winds +chen junga +broom ball +bi ffy +bi ani +audio logist +ard namur +amg medikal +alien ist +al av +acceler ometer +!!!!!!!! !!!!!!!!!! +è ¢ +ze st +yearofthe dog +wn bc +wit w +water boarding +w gl +vicky gshore +v ite +un pretentious +u id +ty vek +the jimmy +suz u +street sville +staple ford +spring iscoming +sp lott +sound proofing +sol on +sla bour +si que +schu ld +sc astle +rubber band +read me +pup date +prow ling +ped die +oli vera +nip gaming +ninj at +nation alistic +nand itas +n bb +mtv awards +ms morgan +mat lin +kum kum +keepthe faith +in frequent +hms gofficial +hi xon +got ti +fla ils +final mente +figur ativeart +f po +doe sit +do var +decor a +coupon ing +corn elis +cal academy +bra swell +blake man +bel inda +b vt +arthro pod +am pion +ali ases +ah alli +ðŁĺı ðŁĺİ +zim my +z tao +y co +war band +vent ura +v ayne +ut ty +tun sil +tu mh +truss ardi +trump sarmy +tol ler +the mad +tam borine +stir fry +spor ad +spe ter +shoo da +shat ch +seabirder saturday +rwin p +rele c +rel o +quatre foil +pum phouse +perfor ce +pas sy +ot ani +noctur na +my thri +mitsu ki +mil stein +mb fw +mason ite +lunch room +lingu ini +len ora +lat rine +l rb +jes elnik +jerus alem +jan ani +itsdre desu +gan ar +gad is +fa heem +east west +east of +dues enberg +dro r +disobe ying +develop ing +damian marley +bor abora +blo que +be les +bar ka +bar gh +ava asi +ash in +and furious +ab idi +ab ab +ðŁİ ª +yes bank +welcom es +wan go +vul ture +vel dt +tr ally +tongar iro +ten aci +team tennis +table au +syos set +soo th +sli ema +short listing +se ka +save your +sap o +sabi ersack +royal court +pronoun ce +phil amuseum +notyour shield +nit ya +mugi sha +mike brewer +me yr +mat to +ma ku +lang land +klein isd +kan chana +javedakhtar jadu +is ss +ip sos +hoop fest +heli port +hard covers +hangten stories +fresh ened +for ti +fern wood +entit le +end z +doubt less +del roy +davidson msp +d é +count yof +contemporary artist +climat ec +chri sabiersack +boy ardee +balance d +bal akrishnan +back a +as as +ar ant +apo c +anton newcombe +anab elle +amul ya +alexanderwang ny +ah en +agbon lahor +aa ir +âĺķï¸ıâĺķï¸ı âĺķï¸ı +á Į +à© ° +zheng zhou +zag at +y ago +x os +wall onia +vector ing +uni fight +uk ta +turn about +tok os +ther oxy +ta wak +sent ul +rae burn +purwo kerto +psycho logical +pol der +pin ata +palen que +pa the +out bid +moustak as +motor head +mel is +me can +mcne ely +koraku en +john nies +jj horgan +ish ak +ii hm +hypo dermic +honky tonk +gat sby +fru g +dog stagram +disav ow +cut cliffe +costu mers +chal isa +bro ssard +brittany force +bl alock +bere ft +atur als +and ina +al j +ðŁĩ¨ðŁĩ · +wh acker +vio list +van doorne +v iti +to kay +thunder bay +shar q +setit off +se kou +sare humanrights +sar bj +s ard +ruth davidsonmsp +robit aille +recor ds +pu ch +phyl la +pastu red +papix ure +omo tor +nostal gie +ni endorf +morethan ever +kun ene +kin n +ki kwe +jab baw +irish rail +impro pri +hotel news +hil ti +har preet +hah hahaha +gv n +gla dden +gam est +fo cals +fit ts +ferdin ando +drg pradhan +cros stown +clou tier +chain saws +blue green +black tip +berth oud +beati fication +b hol +aw riter +auto tune +au ren +ash tead +alberte instein +ðŁĺŃ ðŁĻı +ä¼ ļ +án gel +z une +ya ad +what eva +weekend getaway +v online +tx st +to iling +tm ro +te bal +takam ine +t mk +sv illa +straigh tens +sti fled +sizz la +sh tick +sav our +reallys wara +que chua +par ys +ot amendi +oc les +o ded +nom adic +mr jake +monument sforall +mo go +mikare yesss +kup p +ku ant +kho i +it Ãł +isak son +is sima +huss ar +hobi day +hel in +hay wards +ha dy +greas elive +gho stinthe +fla shi +ev on +el mont +earth quake +e wwww +demary ius +dead locked +de bbi +dar linghurst +bathin da +bat ley +arch deacon +aquab ats +ally pally +ad lib +:' '' +" /" +ëıĦ ê²½ìĪĺ +year swithexo +win ged +weather bug +walker ville +ur qui +unab ash +tor tola +the farm +stran ahan +stag i +sbu x +sachsen ring +ron it +reminis ced +press on +pop ham +pel ÃŃ +ov w +oc c +nne ka +ni jin +nc ta +national wineday +michaelrosen yes +mar chin +lec a +lanc o +kodak black +ju mm +jo fa +its cominghome +il oni +hur dy +ho pel +garden shour +g summit +fore shadow +f hq +esqui vel +elyn ch +drupal con +con kers +cityof ct +chantic leer +chand u +ce at +car bun +bru ja +bow doin +bl under +be our +baz on +ðŁĺī ðŁijĮ +æĥ ħ +âŀ ł +à¸ķà¹Ĥ à¸Ĭà¸Ħ +whitt ing +up n +tx a +twel come +tu cano +treve cca +tobo gg +ste ig +sol heim +soff it +schlitter bahn +sar taj +sanc tioning +rott ingdean +road cycling +re vent +press room +pe ver +pav one +officialap cng +ny bg +nau tic +moong low +melis sas +ma ino +limou sines +lil wayne +la at +kul bhushan +ko ka +khu d +jo ist +jesseb watters +io tv +hi ddles +gt k +gro dd +em maj +down grading +djen vy +deliber ative +cri scy +cram lington +courmaye ur +coo ley +clay field +chiarafer ragni +ar ani +aggrav ate +access or +ðŁij¶ ðŁı» +ãĤ¢ ãĥĭ +âĹ ł +zi elinski +y ena +w ly +vic fires +v air +tro ost +the current +stray horn +sile stone +shadow of +secretari al +scott aukerman +san sebastian +ro ke +richa chadha +refu ted +real radi +pom fret +par dee +par ashar +p sia +mu li +mey te +mer guez +mc garvey +kathr in +john cleese +job son +jair us +ir na +intercep ting +hu dd +hor sing +ho yo +free entry +fi zzled +fanci est +eve t +eti had +er ace +en ae +eag leton +dynam o +de met +com passes +circle ville +chennai fc +can so +bobbi brown +baili ff +assi sted +albe do +ai yar +ðŁıĪ # +ä½ ľ +âĪ Ļ +wide out +v agov +ub hornsup +tracee ellisross +tla ib +tin toretto +the stroke +t ling +sw oops +su mba +su ir +spread thelove +scottish cup +radi x +qasi mi +puppy cat +psychon auts +oz aki +octag on +nen u +mu mia +middle burg +mari ans +lu mos +llan ish +legal isation +ken edy +jazz man +in exhau +i xd +hear tened +hand wala +go camels +g db +funk master +forbidden planet +f ous +ex cori +duc twork +dom in +diethyl stilbestrol +dic ing +den ims +democrati zing +cre spi +churchof england +boycot tisrael +be fikre +badbad notgood +ab cc +... âĻ¥ +! ?!?!? +ðŁij¨ ðŁı¼âĢį +ðŁĮ ĺ +yn ine +wing men +wence slas +villar ai +vesu vio +verti sing +vener ated +un schooling +um ra +u van +tul lahoma +stit an +steff ens +stand upp +ss outh +squee zer +si bb +shira stweet +sf k +scrutin ized +sch wan +scani auk +sbar ro +sai fu +rin ker +rever so +re aped +ray town +radiofrequ ency +pur ty +psy chos +predomin ately +personi fies +ov h +oc w +nhra onfox +nde bele +nc caa +me sc +mc faul +mc ad +matte is +man die +mag anda +lei sha +la brad +kuant antv +jen nal +ichi gan +ib n +hill view +gro ene +gran ton +go di +french ies +fi ke +e ggy +du oden +do cus +din go +design studio +che oil +carac alla +canon ically +bu ssy +bean pot +be dd +bal it +ar landa +ang liar +ag euk +a ï +- ! +ðŁĴĻðŁĴĽ ðŁĴĻðŁĴĽ +z k +ya haya +un impressive +un ge +tri ppy +thisday that +the iron +the advocate +tempel hof +swa ins +sun valley +strengthin numbers +stop arming +si ums +si swa +sciento logy +sam bulance +ringling bros +rich thekid +reinst ates +pollinator week +perre ault +perce ives +nichol son +my home +mate j +l ors +kö y +ke mah +ke betu +join ers +jar rah +jac s +i afc +he rer +green market +gone wild +fun neled +fro ad +en cyclical +em dr +elek syon +ecol lins +e ol +dow o +cortic oster +comple ta +city uk +ciopp ino +ci roc +char bel +bok ke +bel ies +bac up +ant davis +an thers +a key +ðŁIJ¶ âĿ¤ +y pur +wy combe +ws ls +wo hl +white girl +whit elaw +usc ollege +telefun ken +tami ami +tal y +tal uka +su ed +steve scalise +so in +snow leopard +sean m +remembr ances +re ticle +ravens flock +radio grapher +port auth +pic of +pen ryn +pay nter +pain tin +mec can +me ara +magen to +lan yon +lan di +jag ex +jade ite +her iot +go cougars +g pb +fri ary +for tier +faf atl +ev enter +du ress +din ks +cu bes +cro zet +cri c +cou illard +contra ptions +chad derton +carly pearce +c sps +bur dette +blau grana +ap ahm +anc ar +am ta +ajit pa +abstr acted +[ [ +ðŁĩ ± +ãĥ» ãĤļ +ãģ¤ ãĤģ +win ce +wedding season +w apa +til lot +ti ssier +ther monu +tall on +take downs +super corp +sat yam +sam paoli +sam iti +sal tzman +sak ina +ry on +rust am +rother hood +road maps +rim less +ri ze +recon cil +ra ibh +puppete ers +prud homme +photo copier +pe cks +northern powerhouse +no isi +mu deford +mor tally +mi h +mediterran eo +mcclat chy +m ells +low carbon +lo sc +len n +lark hall +labrador retriever +kres ge +k nap +just me +jun gy +j ire +i shootfilm +hex tall +g ages +fin dit +festive season +favourit ism +fab ians +emplo yer +dur dle +dic en +con tax +clu bland +city schools +city place +cit sci +chi bok +channel tv +ch nie +box sets +bo zza +bertu zzi +bert adores +bar tomeu +back tracks +ba auer +ap lu +ang ad +ag el +adal ah +yeng pluggedin +w ck +u ys +u ppp +trigger fish +tri sten +traverse theatre +the onion +tax a +statu esque +stand in +sp ia +si vag +romeo ville +re eb +po tus +pherom ones +on wx +oak lawn +ne ave +movie star +ment i +made to +mac lay +kin naman +ingh e +il mour +hasle tt +georg elopez +ge is +fixed gear +femen ina +fan nie +et ter +en oki +eisen staedt +dusk till +dr ine +detroit news +cro tty +co sta +christinec aine +carn forth +card games +car jacked +bull is +bou lev +bo gue +blackbear nation +arm stead +amur ali +amo tor +ambaz onia +yu kiko +webster hall +war ning +wa ian +vic ars +ver bi +universal pics +therolling stones +tab ith +sun unu +sky fox +should ve +scen eries +sarahpalin usa +rehabil itating +re stle +re if +re grouping +po ore +pa ine +og bon +nl proc +n ere +mn th +loosen ed +lett in +leave eu +laur yn +lalupra sad +jrn tr +jen ison +jelli ed +j hutch +if w +i justine +ho ys +hann elius +guide tti +flo ppies +fiq h +eli zondo +donnab raz +do el +dham aka +cri sper +bon as +billion rising +be ers +bar rons +bann an +ake d +ah lu +a ari +( !), +ðŁĸĬ ï¸ı +ðŁijŃ ðŁĴķ +ìļ° 주 +why alla +upri sings +under cooked +u fw +sp its +south mead +slo a +sisy phus +shr m +shon daland +pum meled +premier inn +phal ke +penn medicine +pel é +ol n +nun atsi +neder land +mon sey +mic drop +mel fest +man en +maho pac +lu mo +li kk +le ese +lat eness +kof xiv +ko hima +kh h +kab addi +k cci +iq a +in tosh +in cy +iberdro la +head ass +hal ab +go gol +g gi +fur babies +fi shies +fb live +ez ell +enric om +dun nock +don nas +crow ley +clo d +chu d +cham bo +cadw wales +bow ens +boss babe +bonne affaire +black wing +beast master +bay ani +bad dhan +ba ther +auton ation +---- -- +ðŁĵ² : +âŀ¡ï¸ı @ +yat sen +wing lets +ul in +trepi dation +thegn show +sugar bush +srk chennaifc +se ic +scru ise +roo se +ri mando +re ith +re installing +re gress +philly sports +par ser +oo ficial +offici a +nouve aux +n ton +my la +much acho +mc mahan +le wan +laura jane +kan goo +jeff co +ja ane +inspirational quote +indign ity +i iss +homen aje +high veld +g dd +fu lop +ende sa +dropkick murphys +decentral isation +daysof summer +cute eeee +com pos +certific ated +call is +bair n +bair dmp +aud peeps +ati ka +ac da +ðŁ¤Ĺ ðŁĺĺ +zeig ler +wick en +was c +u maine +twitter sphere +ti el +terpen e +tele performance +sw anny +sv f +su pramo +sap ul +sand akan +rough neck +rema sters +philli pines +pe king +op ta +mn dot +mcla urin +mar kan +ma sted +kore and +k link +indi gnant +glob ali +gentri fied +g aku +fuji mori +fright mare +euse bius +eman ate +du fy +dil o +de palma +dat is +curv aceous +cu co +cross man +crazyex girlfriend +cosme tologist +con senting +bull mastiff +bu shiri +brand sanderson +boondo ggle +ðŁİĬ ðŁİĬ +⼠¸ +z den +ye adon +whel ans +we del +wc p +w smith +w asse +ut is +ur n +thepar raeels +the bridge +tat atru +tail backs +steel works +sno biety +ru hl +ron icles +ration alize +photo g +p du +o tak +neuro sciences +narcis se +nam ethat +mo anin +metro town +merci fully +mccu sker +mas n +m ams +ky leg +ited fc +house ful +ho ye +heine mann +hed wi +har den +gri gori +gau har +frac tal +four ball +fok ker +estre lla +engineer inguk +electro house +education ist +dig cit +df v +crouch end +co sey +cin ch +chi z +ca steel +blood stone +bg ca +ben simon +barn storming +atul co +as ke +allu vial +ag co +ace tic +ðŁĺĤ ðŁĴĹ +ðŁĹ ¼ +ðŁijı . +ðŁij¨ ðŁı½âĢį +าภģ +ver sova +ur dd +tra baj +the blog +tem pos +tb day +super troopers +sto king +sep ang +secu isine +sauro pod +sas b +rose man +ray qu +pike sville +pa ig +ohmy girl +nl f +musque am +more in +mike bairdmp +mic hell +mi jo +med aled +mall u +m pos +lu bom +laugh arne +kar nal +inf en +hillaryfor prison +gri ms +fu si +fu cc +f ynn +ed ness +dry point +direc teur +dir ndl +del rio +de mond +de angelis +critch ley +conme bol +cam pa +brah mot +blu ray +bhak tapur +bal dess +art form +art challenge +aneur in +ad oodle +ðŁĩ¹ðŁĩ ³ +ãĥĿãĥ¼ãĥĪãĥ¬ ãĥ¼ãĥĪ +z acks +yil maz +yanis varoufakis +ya sho +un ang +trans cona +tran spires +tel enor +tar sier +talent less +ta kano +ta io +sp outing +si mic +seol hyun +scru ton +ruby onrails +rose au +pro hm +pro cli +po ie +plan ck +photo bucket +perspir ant +p km +northeast tweets +musc adet +micro fluidic +man ico +mak ki +ma ppa +lawn care +ko skinen +jet brains +iri ver +holtz claw +handker chiefs +fri sby +ef oundation +east lanc +dis engagement +delon ghi +de andre +creati vidad +cerys matthews +caram els +boy sin +bha gnani +beacon theatre +bak kie +azz arello +anima ux +ambas s +âŃIJâŃIJ âŃIJâŃIJ +Â Ń +yu li +y ts +x kr +x bo +wearec isco +wb afc +vs buf +tx stormchasers +tran i +tom ei +tam ela +super men +stef anik +stand aard +sor achi +si xt +shrou ds +short all +sh ound +separ ations +re negotiate +ra sha +pleasan tries +ple asee +plaster board +per chance +ost see +nsw votes +naval academy +mon donews +me sto +mar yl +ma po +m nek +ly ph +long stocking +local host +lead gen +l amaze +kit ab +k usi +k host +he ren +grati ot +go van +gen te +fri sian +fran nie +forever more +flu ker +em ine +don nam +dd an +cou verture +clen ch +citroen racing +chit ose +cast lec +cam bri +break free +bre ves +birch box +beach combing +ash ra +al mer +ðŁijį ðŁĺĤ +ðŁ¥ į +zer ot +za atar +wing wednesday +want z +wa chter +une qu +ul ti +tuf ts +thunder head +the super +the cup +ta ppin +side cars +seun ghyun +sebring raceway +sa qu +s stv +reas sur +pan jim +oro b +op ane +onec cps +on enews +o iling +ny mex +north rup +news comau +nbc boston +n ill +mo lise +mi miko +matthai g +mati gary +mane ka +man dias +mach inists +li dge +la dera +kal ama +je unes +jac c +ho shino +hick stead +haroobom kum +fresno grizzlies +f pt +erit re +eri on +en rolls +e plenary +do pa +dl ach +debor d +color ful +col wood +col oma +chuck schumer +cactu s +bot ley +am anita +ahsa pocalypse +." âĢĵ +ðŁĴľ ðŁĺĺ +ÑĢ оÑģ +Ê Ł +yel don +wit tier +white mugh +whitemugh alsfan +volcan o +viri dian +vac lav +unstopp able +un shakeable +tru ancy +thu ggish +thra shes +tel les +taye b +tamal pais +t gn +swar t +suit up +su tras +su pes +sten ciled +st roop +sin hal +see me +sati ate +san carlo +sab atini +rpg maker +regn um +red it +profess orial +politi fact +phy llo +pag nell +oxidi sed +oss off +now youknow +navy seals +myx philippines +mor r +moor man +mon tour +man eater +mai ya +lub na +london portauth +listen now +lind holm +lib bey +la sti +kol ly +kir shner +khu t +j woww +j kia +intervie wees +ham ba +green edge +ghost stories +fire light +eng al +dü rer +dv l +doppel gang +deaf blind +dab ooks +conce it +career builder +beso tted +ber had +ba reli +awww wwww +at kowski +art ner +ala ji +ðŁĵ Ļ +âĺĿ ðŁı½ +Ñ İ +yi h +writing tip +w utv +uk football +tw rp +terr ill +tech con +super cut +sto on +stabili zers +spa ghett +slay ton +sin ful +si keston +shof ar +shamp oos +shal ala +rupert murdoch +rich homi +redun dan +rec co +pre biotics +posta pocalyptic +our cog +nib bler +mur mansk +melis andre +mat agor +marcu se +man hasset +lic eu +lead on +kelly monaco +kel ham +ini us +igers italia +hor nac +hei ze +free wheel +feed lot +far falle +eng olf +em beds +el mann +el do +compan ys +coll ating +ca thouse +be mbridge +ball out +b corp +ai ya +ðŁĹ Ĵ +à¹ĢภĪ +zo har +zak u +whale watching +vern ation +up next +tek kers +t vos +t elli +swasti kas +sail ings +ricci one +rep tar +rah man +perk sof +os at +no thin +naomi wwe +mosko witz +merit age +mc ds +mar occo +mal treatment +mac leans +lu bav +lo ps +laro chelle +kun dali +kar jakin +k sk +j mi +inter media +ing u +in forum +her old +green port +fra kes +falken tire +excell ondon +eli ud +dru itt +dailypost wales +common ality +charle sd +cbc music +cat or +carri galine +black sweather +betje man +beh rend +au dl +atre yu +an cher +all ondon +all at +air date +afl swan +__ _: +ãĥ¯ ãĥ³ +zu ck +zeph yrs +was ay +w fu +the business +th ent +sp rep +sat or +sal ada +ren fe +rebel heart +rai ison +poly tunnel +pitts boro +phillysports birthday +newscomau hq +mat ure +mac coll +lam mer +keralafloo drelief +jay walking +james dashner +industrial strategy +horse meat +heal esville +he yo +gal ecki +fighting irish +fier stein +fear ne +escap ist +def ers +daed alus +costan zo +convic ting +cli st +celi bate +care sses +canop y +c tia +bru net +bir ger +bigre ds +ballincolli g +aus mus +aquar ia +an sh +al bino +. !!!! +ìļ° íĺĦ +âĿ¤ï¸ı ðŁĻĮ +who vians +who se +wey smith +vineyard vines +vex robotics +val mcdermid +under oath +un built +u dom +u cas +sy ll +swa e +stri pling +skir ted +she been +sea vey +scab ies +sag ada +repair man +read mission +rambling sloa +py att +philanthro pies +pat ently +out scoring +olympic team +off cla +nicole kidman +nes sun +nd k +mor aga +mahar ana +lsu football +low riders +lazy day +kle ptom +kas ka +je ena +j oos +inde scri +imc millan +il legible +hir sh +he tte +harri gan +happ iest +hacker space +gon y +fü h +fin dom +emer ia +ede weysmith +dynam ites +dejav u +co relle +can ker +ca ther +btv i +bm u +beath high +bb sr +as ing +ann amalai +alan dia +air dropped +. ,,, +æľ¬ æĹ¥ +vil ified +usur ped +ton sil +t we +t nie +svs zombies +stour port +sporad ically +slumb ering +sec md +sai dyes +rm wb +radio logic +proud principal +pho bias +pedi alyte +ong seongwu +nore aga +no limit +neu stadt +ne eti +ndi aye +nasty a +n mbs +me cs +massimo bottura +marcusle monis +mad child +ma ba +lo chan +ld in +kri sz +k anna +j hope +hy g +hornac ek +hau raki +griffin mcelroy +gli oma +free zone +fail sworth +ent ous +enamel ware +ema i +ec as +dre scher +de ment +dal os +cr p +cinemato graphers +bal ak +ar bo +ah re +ad oni +ðŁijį âĿ¤ +ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ãħ¤ ãħ¤ãħ¤ +« ล +west phal +watch os +valky ries +v ci +ut k +up govt +thought ful +term limits +su yy +spen sgen +skane ateles +san lorenzo +ryan phillippe +rw p +roller blade +rep aving +re j +po vetkin +per ine +oxy contin +os su +opho tography +occupy wallstreet +o dun +nth qld +nb alive +men ews +mar kel +mad catz +lov ins +loren tz +le ana +lal or +jr nal +it le +in stigate +i kes +hall mar +ha ggle +gn abry +fu git +fra k +eutel sat +dysauton omia +dre west +dew berry +demetri ous +deh li +circle of +by att +bi dad +au man +ar col +ak ure +aes q +advers ities +... ðŁĺı +... "- +ðŁIJ¶ ðŁIJ± +âŃIJï¸ı @ +wat aru +tra han +tex tron +spoke smodel +shon an +sh els +scra pple +re builds +phil mont +perish ables +patt an +pam ir +ondre j +night beat +nas s +mu ffy +mononga hela +melo di +maru ti +mall galleries +ly dia +lov ells +lodg ings +lin di +larry kim +la spal +la phil +kwankwas o +kokkin akis +je en +jayalali tha +ha sling +guerre ros +footy accums +folklore thurs +don ard +ditch ling +deli sted +dai ki +civil isations +celebrity cruise +bitcoin news +biaf ra +bi bo +beha vin +az amar +asante kotoko +as eries +ar imel +an nes +ala ia +adel ina +& " +â n +zulu land +yourad f +wa hm +tweetapicture of +tra duction +t illing +sun dries +story books +scul lion +san geetha +ru af +ro gel +respir atory +re sham +protec tyour +pra desh +pier cy +pf re +pc games +no h +night shift +nh lon +nat z +ms j +marti an +lö w +kids grove +justin ian +john abraham +itsgreat uf +ired ale +herd wick +haupp auge +happy customers +ham sik +haim theband +h end +gibb ard +fa thi +en via +ele vy +divor ces +digi mon +crawfords ville +cr its +continu o +clo thier +chow chow +change up +bukas na +btr tg +bern ina +bc beer +bal de +analog ous +am way +alz scot +allagash brewing +all lllll +aig le +ad na +ØŃÙħ د +ze h +wind blown +wid ne +ver if +val enti +u inta +tra vi +thisdaythat year +swe ta +support smallbusiness +spar alympics +so ong +shi ites +sere bii +sch litz +sa idi +s dorp +ruffi ans +rome tty +rl grime +resili ence +re possessed +pur pl +pride ful +poly gon +pearld rum +pae vey +out la +ote p +orient fc +nannakuprema tho +memphis redbirds +martin schulz +mal abo +lay den +l st +kom et +jeopardy sports +j jp +inmar sat +hode idah +ho witt +hi les +gri pes +ga iters +fric ker +ev aders +euro money +economic development +e hud +drop outs +der ie +cre tins +corro ded +ch ittenden +brutal house +biopla stics +bi zzy +bar thel +atit lan +asi er +à¸ģภľà¸¥ +z era +yam cha +wak fu +w wat +vo ted +try an +travel wi +th ais +swamp thing +stylish star +sour sop +son ido +sk erry +sham bala +sh renu +round tree +riprobin williams +re taking +penny worth +p slv +news round +na ila +manc made +mal ec +ma brouk +love me +long hairdontcare +ling er +li gon +jordanb peterson +jason voorhees +intrac ellular +infr actions +indu stri +iann one +hybri dization +golden state +glycer ine +gastro paresis +gad dis +future house +fur ukawa +fi stic +euro copter +en ye +emili orivera +dr aceway +disco lored +di paolo +dag g +com andante +clean ups +br ary +blom kamp +bir m +ber go +ba tha +ayy appa +american sniper +afl fantasy +ðŁijıðŁı¾ðŁijıðŁı¾ ðŁijıðŁı¾ +ì¢ħ íĺĦ +ãģ ĵ +wgn radio +wedding flowers +tz ka +to ye +titan books +then es +tere sa +tagli ani +so wers +slit ting +sel don +sch ul +ry ton +ro sier +rhin itis +raj shahi +r de +pretz el +offcla stro +now playing +new on +neverstop learning +marro quin +mag ers +lat kes +iy as +home decor +gl x +gg tth +febru ari +especi ales +engv wi +el mers +e tre +dj z +day book +common alities +clou ding +cat sup +bharat pur +ang y +aly ci +aer ated +abrew ery +. âĢĵ +ðŁİ¶ ðŁİ¸ +yor u +wi eland +wearethe community +vi bin +var nishes +thra shers +the mag +th warts +spil lover +sno pes +sno bbery +sin color +sc amp +sb ama +sander ling +roland smartin +riv age +r mk +q et +pre frontal +phal ar +pe pi +pal min +om lin +o dem +north london +muskete er +mo honk +mat tro +mart ellus +loren zi +kwi k +kis sme +kelli giddish +kar ri +ine fficiencies +head of +good wood +go falcons +gi ffen +gerard pique +foot step +eli roth +dou b +dog patch +dill ons +did actic +devast ator +del p +das d +cin i +bow sette +boogi ecousins +bbcradi omanc +ban z +back link +b wl +apache spark +allow able +ag rad +/ > +ðŁĺĸ ðŁĺĸ +ठĽ +y abu +wr cb +wir ksworth +wg sn +war um +twir ler +travel pic +tol er +to talk +tim ken +stroop wafel +scho colate +sam ina +sam huntmusic +rose buds +py arke +polys acchar +north lane +ni hal +neel am +morning starr +mechan ix +marin er +mal heur +lec tomy +l vr +kur ri +kud zu +koppar berg +jack lin +hodg ins +ha dow +grow led +gr rrrr +got ze +gav an +fire fox +fasig tipton +eye d +erc q +dul kalam +dragonage inquisition +down sized +devon wildlife +defend daca +de sing +de am +dark side +cross y +cich lids +cat tails +be kele +angel enos +and ris +alyssa edwards +adl fringe +ãģĤ ãģ¤ãĤģ +zur ich +yuri y +yun s +vivekan and +vit or +ven eno +ti pa +the hip +tear jerker +te eling +super max +suit er +ste ing +shape shifting +se chelt +schö n +sam pe +russi e +run town +pop tv +paren theses +oxygen ated +oph en +ne ato +nach o +mu hl +macro phage +lom o +kt family +kon ic +kha jura +je se +jam ey +jack white +ilovel u +icc worldcup +hopkin ton +hhhh hhhhhh +he mis +ga ston +fab a +ear shot +dog fighting +discar ding +dead andcompany +bru tes +bri ony +braun strowman +bo by +ben ic +bang aram +ba itul +aw ala +artif ici +appreci able +air wolf +ah rexpo +agri food +ae hoops +ac ers +zi ya +z ula +whitney houston +upp ingham +thru st +thero pod +technic als +t ftp +sy na +stand art +source book +schi ano +sch orr +rous sel +regener ative +reco leta +rc ms +puma football +pr ancer +port marnock +over world +nthqld cowboys +novel ization +nat v +nare k +me bane +l ith +jud icious +jabbaw oc +inser t +her azade +heathle dger +harvest moon +geor gio +g vc +fossil free +ericjohn salut +eng in +enam oured +du y +decolon izing +cram ping +coun tered +conju gation +burn tisland +brow sed +blue stem +be ekman +ban chory +augustin ian +asu mmer +arc as +abio dun +ab rown +) âĻ¥ +ðŁĽ Į +ðŁĩ© ðŁĩ° +ãĤ«ãĥ¡ ãĥ© +âĿ¤ï¸ı ðŁijij +zol ak +vaj ra +to learn +stom p +spe ktor +spatu las +soci opathic +sno wiest +shif nal +sheryl crow +sherri hill +shaw ne +sc ph +ru dge +quasi modo +publici dad +prou sa +poo per +pak vind +outlaw z +nil giri +niel son +new look +mon ary +mat ang +loo ses +le garda +landscape architecture +lamp kin +kish war +khali d +kello ggs +kay al +jun ya +jj ig +indi ad +im ma +hell bound +hair dryer +guest room +ge k +ga ar +fakel ove +erich mond +def r +dashi ell +cute dog +colombiais magicalrealism +co q +chor o +bren dand +boulev ards +boston childrens +ar ol +andre ou +alzheim ers +adap a += / +ìĥ ģ +ç ĭ +vw bus +virtu oso +u del +tron c +to kaj +terri gal +sj su +sioux falls +se ko +se ay +schou ler +sar chive +saf arova +ro tavirus +rin ko +revel atory +quen neville +porpo ises +play ful +picto gram +obsc ures +my ka +mon treat +mckee sport +mar ler +lu ise +loveyou guys +lion snz +lati more +l mn +kurdi stan +kumb h +khan kk +kay u +kat we +jo hari +interoper able +icu eta +hor licks +hollister co +gochu jang +gary johnson +fy life +fein inger +feel er +fal ak +e im +digital payments +devon ta +deter ring +day with +davi dy +cc x +bu le +brock way +bridg end +bon gbong +bish t +bhav nagar +bg su +abra xas +ðŁĺģ âĿ¤ +å° Ķ +welling tons +wel ch +wasay jalil +vivo prokabaddi +val dis +trump in +toc queville +spiegel man +spe sh +sof christmas +sc asino +sar ki +sangh vi +rapp aport +qui jote +pon toons +pok ken +ornitho logist +o leg +ne agh +n land +n app +min us +mcro berts +mar zia +long field +la at +iam varuntej +halei wa +gro over +flav ouring +fc j +ec topic +county wide +copy book +con ifers +chi ki +centr ality +cas erta +car snapped +car gol +cal ver +c chi +b iliary +ath ul +are m +aly sia +ali do +alber tina +agar tala +ðŁijıðŁı¾ ðŁijıðŁı¾ +оÑĢÑ ħ +yn lle +y abo +whelans live +vogue paris +total afcon +time of +thelast leg +takap una +ta vira +strati fied +stir uni +star la +south fork +sou sap +sou rire +sofi ane +so dor +sequ ity +sel fe +s watch +rebel led +qu itely +proud american +paragu ayan +par ast +oren zo +open cup +om its +new love +multiv ariate +madri dista +libe skind +leg ault +laligas antander +kap ut +jackson guitars +it back +hotel direct +hen son +gram fam +girl shbo +fra zee +flor rie +flir tation +fan book +elph instone +eh is +dun ited +d ne +chin on +cellu lo +ce sca +boeing defense +bat our +bar bee +bag ot +az ali +av ratri +ari se +a ite +å¸ Į +ठĸ +wro c +world ph +wool ston +west park +uncondition al +tour nam +teo doro +supervis es +still sanders +star time +sp bs +sof rito +rock hopper +ro kh +rid ha +red alert +rain iers +poit ras +per tamina +p ation +otolaryngo logy +nise ko +ner ds +mis chka +mh ky +mate usz +man tha +ladies football +jan elle +hide aki +henri ques +he pp +he der +gra ven +gau det +fu era +forestof dean +ford canada +fl b +fan ation +fa anews +es md +en scon +em ms +du elling +dor on +dl wp +dead bolt +de pew +cross bows +croo ds +cou th +chri sch +cheri shes +cathe ters +cam ren +brah mins +bon durant +boiler plate +beech worth +be rens +bc care +arimel ber +ðŁķ Į +ðĿĹ¼ ðĿĹ +zag ging +z ord +trelle borg +tor no +terra forming +su pano +su chi +stran gulation +ster num +steph any +sny tv +skibbere en +shak u +se tc +ro sati +reno s +purpler ain +patrick sday +pa ko +oudt shoorn +nyt mag +nfl commish +mon on +lou dre +live with +laura benanti +laundere tte +itch ell +in ac +i bad +hum tv +hisp z +hare em +gravit ation +friend less +fe re +fanni bals +ellen berg +eden bridge +eddie vedder +dioni sio +de wars +dau phin +co red +boston pizza +bernal illo +bermu dian +all man +adver torial +ac ott +ðŁİ§ ðŁİ¶ +z azu +ye le +wur ly +wright stuff +wau bon +wag gle +w net +v for +uc co +u til +tony fernandes +ti mah +thoo thu +sport z +satter field +sanjay nirupam +sam al +sa ima +ro dan +rahme manuel +po pover +or c +only one +normal isation +na thank +my tv +mr michael +mo ir +middle men +meri dge +lat eline +laker idge +kr ack +ko tter +kex perience +kelly ripa +kan gol +joko y +john boyega +jet life +inter club +holocaustremem branceday +hat ton +gro ce +go wr +est el +ep md +e fin +dun das +du fresne +dro ws +dreams docometrue +dj zinhle +daw nof +continental tire +col m +coal ville +cbr n +casei h +brail sford +az pil +an anta +alfredo flores +af all +accentu ates +ðŁļ ¬ +worl di +wes thou +vs nyg +vil i +unabash edly +tx water +twitter firstfriday +ta si +stoke sley +stati stic +sig ners +sherwin williams +ri quel +reproduc tive +ramanu jan +quiet girl +quar rel +presump tuous +plant es +penn ine +over clocking +or lov +oni sion +of ilm +neuro endocrine +nen ele +mush fiqur +mrs gandhi +mis represented +mey ler +m do +liquid ators +lawy ering +ky ron +ku z +kindle books +kan ia +k andy +jo zi +jo ffe +jin ka +itsn ice +ha sk +gil martin +g tu +g greenwald +free born +fasigtipton co +europe ana +es quad +el sword +earth ling +e asons +donnabraz ile +cipri ano +ch us +cf n +bu hay +bram mer +bowel cancer +bil an +bbc sounds +bag shaw +as ae +appet ite +al franken +advo caat +ac rl +à® ¸ +Ø ¦ +zar b +yiann ch +yale town +whas sup +wer i +weare not +water spouts +u ffs +tri plic +th ula +taxon omic +t weddle +t kp +steal ers +sa irs +raj baddhan +ra thi +que ere +pre determined +pistol cliff +pist es +papp y +omin ously +jon na +j pn +gi er +gel ati +for less +faulcon er +facebook down +estate planning +ea day +don au +dit z +dism oun +delici oso +cre ighton +cont d +constip ated +buen aventura +bram well +bb cal +barra sso +alb ini +ak om +ðŁ¥ µ +âĪ ł +ठł +Ù ı +wob bler +wist fully +wind mill +win amp +wil let +war ton +vi j +ve j +u xo +tweet bot +tra ight +toon ie +teach sdgs +sport science +silver backs +seto pati +san tal +ryu ichi +retali ated +queens gate +pri pyat +patap sco +oxy codone +obi ang +nge movies +mst rooo +mor rone +mer kovskiy +mcan ally +maz ur +marche tti +mar land +magher afelt +lof thouse +la brum +l ÃŃ +kol l +kim jong +hen ao +hemat oma +ha ren +gun dlach +grass root +gla z +frater ni +flying scotsman +en light +el ang +ek lund +edgar allan +du plicity +doub let +descu ento +dat er +d princess +d add +confir mado +compu ware +cash me +car lene +cal vins +b mi +append ages +ald ale +al tere +ðŁĺĭ . +Ëĺ ) +wn ba +waldor f +vo w +vo ges +use able +us dol +up work +unc g +u hl +tur ducken +tre loar +tran spiration +thisism ay +th st +str out +show es +sher awat +sh alo +sh ags +scho harie +scaf folds +q atif +pre requisites +ol usola +nor ther +no ss +mud honey +mapu che +man gu +liqui fied +kon op +k á¹Ľ +inst amusic +impactin vesting +hoss am +hawk wind +gour ley +glory days +gh our +gar ma +g aren +fashion design +elevation church +di ar +das om +daf a +crow ned +cross way +cou to +con chords +cl unes +casi o +billie holiday +b bered +ath as +amade us +almeida theatre +adityaroy kapur +ðŁĴľ ðŁĴķ +ðŁĴª ðŁijį +ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ðŁĩºðŁĩ¸ +à¸ģภ¥ +æ r +yourboy myles +yard stick +werthe im +vege table +uri jah +twc news +tinder box +the trade +te sto +sun hat +smal lish +sho petsy +shi vely +shah in +sb nd +say reville +religi on +refurbi shments +real change +ptole maic +po cari +people problems +pd ga +o ana +nic ar +nev ill +mud cats +mon tro +modi govt +ly cett +long tail +light nin +la br +key worth +joan jett +it chotels +hit raffic +hill walking +ha dar +ha as +frei heit +food art +exc itation +eng strom +eng landers +doun e +de ben +cuck oo +credenti aled +crai gie +chill wave +bi reland +bastian ich +ax stv +att stadium +ar ski +allu des +ake up +abe el +\ = +ðŁĺį ðŁĴĽ +ಠ¹ +~ !!! +zou ki +yougot this +yo kes +whit well +wat ton +vin ni +un talented +un go +terri fic +tar ka +ta cht +t pl +t elia +ss acramento +sou sse +shi amak +sco ast +road tom +ro mine +ratche ting +plu tarch +pin d +o sti +new work +muk ti +mth atha +moon dog +mo hm +merrill ville +meate ater +marching band +loe wen +lo cura +le thu +laz ari +kay ser +kan th +kal k +k hang +he stia +gregh unt +gol feu +giel gud +fraudul ently +foodin novation +food panda +fon thill +electro cardio +e bn +dog tooth +der singh +den a +de gener +ba hi +avon lea +ap ay +am da +aber lour +aaa inspector +what women +wa hs +vin italy +van taa +thumb ing +thac over +supple mented +south worth +soo jung +sm tm +sl comiccon +sha u +selfe mployed +se kh +r uru +q al +pâ té +pyarke aisebhi +producer life +pieter se +p wx +odai ba +ni zza +mo shing +ml m +milnga vie +memphis mayfire +maple ton +mand era +m cham +led wards +lands downe +lamp lighter +lam bof +kuchrang pyarkeaisebhi +kime ki +ka inen +it smo +it ong +hur ra +hra wi +how tobe +hol lin +hapha zard +habak kuk +h lat +gas con +fur riends +ft lauderdale +f bal +eu i +ec ts +ea id +doctor ates +dim mock +comic stitan +ci reland +celi ac +cb seven +car nahan +calgary herald +bharat anat +beach goers +awh h +ast oundingly +anglo phones +amus ingly +am cc +air indiain +ab stained +ab asi +aaj kamrankhan +a ú +த ல +Ùħ Ø© +u vc +the bel +sunba enim +stop killing +stain d +soli dity +shoulder to +sap hana +ren cont +pur an +pnpp io +o ty +new moon +necker chief +mo shenko +me ireles +ki on +ka pow +irish history +holland roden +helve tia +hear st +hack aday +greghunt mp +give thanks +frontier land +franc avilla +fit mom +filip inas +ep irus +ear lobe +ea sty +dray ton +de ssie +dam ar +co work +chapp ed +cano ga +bursas por +bri o +billy graham +bho sle +b wl +aw omen +autor ick +aeri alist +ae reo +ðŁĺĺ ðŁĴŀ +white hill +what makesme +wal ey +villarai gosa +ve i +v usi +un reasonably +toyo tires +thirdman records +thing iverse +th ami +survey monkey +super giant +stre vor +sound checking +som met +scar petta +roberts space +reall orraine +rajyas abha +pun ta +po om +pax prime +oren dabooks +oc tet +ndhakoo ttam +museumof art +mom ocon +mo du +medium ship +maher zain +laksh ya +k abc +impression able +ho bey +hc so +harring ay +gram in +governor tomwolf +gor man +gam esa +found ling +ff x +europe forculture +dra ig +defence forces +cy clin +cos grave +corel draw +choo sy +cbseven ingnews +cam ili +br ang +bow ral +benven uto +ben zodi +be ware +bak kam +ar le +annihil ator +am antes +ales ana +ðŁĴªðŁı» ðŁĴªðŁı»ðŁĴªðŁı» +ðŁĩ¹ ðŁĩŃ +ö n +wy che +will enhall +w anne +united against +un box +tou cans +tick tock +ti jer +st pauli +ro bron +richardo sman +red backs +rayqu aza +raff le +por tree +pol gar +pil o +pen testing +oy en +owy hee +ott b +o her +mitt en +marga ery +lauren german +ki miko +jay apal +j yn +it movie +im perfectly +if es +hol ing +hesp eler +han abi +h mw +great american +giga om +gan dia +g ys +fury road +fondaz ione +ex s +esp nw +er wan +dee wani +co bol +city asone +cgr teams +caul king +carpa thians +ca el +bundy ranch +buckingham palace +basketball cl +audit orio +aq s +androgy ny +ame et +am ae +ak asha +agio ia +afro centric +. ,,,, +à¸ķลาà¸Ķ à¸Ļ +õ es +wrest ler +vi zi +un carrier +to pol +the orossi +symph onie +sr kian +sonys antamonica +sing star +shab nam +seri alization +semin ario +se aland +sav on +sak har +rise together +red coats +ralph garman +rag na +ra bun +pw f +proce ssional +perform a +pan opticon +p illa +oh so +notin myname +nit t +net beans +muh te +mm ell +meck is +marab oli +libt ard +lef kada +kay leigh +jac q +it sab +ind l +immort alize +ho res +giveaway alert +gin nie +fat joe +et sch +es xi +elabor ated +ein en +dunker que +du toit +dri l +doncaster races +don elson +de kay +d magazine +cre ssy +cin o +cil ip +chi quito +byte coin +bre ga +box y +ble n +anti gravity +albu ll +adv aita +ac chedin +ab nam +ðŁĺĤ ðŁĶ¥ +ðŁįij ðŁįij +z huang +y acou +x net +we gener +vas o +up ended +tri l +tran ter +tk pict +sun star +stic ation +spor in +sp ast +sham miya +scann ell +savi ano +sal ing +sal en +rat to +polic ing +pisto ia +pic hi +pate k +pascu a +over riding +or able +oo loo +now a +mo fthe +min ote +metall ers +merge records +mabu se +ma don +kine matics +indi s +hu o +ho tr +hal t +hal dimand +greengro cer +fri sell +formula drift +f ism +equ aled +e bd +dnb tv +da awards +cool fm +coo tie +chin ua +bot ts +bi go +ba har +ba er +aw aren +ap lusk +anz acs +an cap +alon ce +< # +ı ÅŁ +yay yyyy +vin ita +un cluttered +tutel age +tu mp +thal ys +tailgat er +sydney tennis +swadlin cote +super copa +sunday times +su ber +stat ers +silk worm +sign writing +si stem +shawn mendes +self shot +scrip tural +sch rager +punch lines +puj ya +prolon ging +polit ica +park way +paraphra sed +p sus +organ o +octo gen +nle sdca +niec y +nenele akes +na sha +missmal ini +mis eries +mens golf +men to +me too +mc gillivray +mc callie +manic urist +ma pfre +kabir khankk +k mk +k him +joss stone +j re +ir ates +inter faith +heartbreak ingly +gó mez +gro at +gie ter +gel son +fel ice +fav icon +fal lows +ex i +drift less +dog matic +digital strategy +de stiel +cop ters +circ let +chiar oscuro +chi pol +ch afe +calv illo +c ssd +c sia +bristol rovers +bo ac +ath en +altrincham mkt +albat ros +al pen +ak owski +: (. +ا٠Ĥ +wing tips +vo anews +ven able +vat er +v vt +trac ead +sweet water +sw co +sam it +resi zed +pluri potent +ny jah +natas cha +music lovers +me khi +majik ninja +maer sk +lambert ville +kl al +ki drock +khar agpur +k ram +joun ieh +jor dang +is free +insol vent +i asi +hu set +fother gill +fd tn +fac tion +evapor ator +endocannab inoid +e migrant +drunk pete +dragon srugby +dom ine +dispo ses +de guzman +cra in +cot ts +consu elo +con formed +castle well +by p +box park +bal ut +bal akot +ajinky ara +abo b +aaron carpenter +ðŁĺİ âĿ¤ï¸ı +ðŁıĪ ðŁıĨ +ðŁ¥ ij +ãĢ ĵ +ภĺ +à¤ľ य +with al +wemy ss +var ni +twitpic your +toi delhi +thir uv +stac os +somers by +solar winds +sh hs +recuper ation +rastaf arian +philanthro py +parenting tips +pa ster +mort lake +mil bank +ma iri +luq man +loc cit +letsgo x +l pf +kal ma +ja is +isleof skye +is car +in habitat +hyper spectral +hincap ie +gen do +fu egos +es waran +depo is +da hn +d sp +d lb +commer z +chi quit +center line +cas als +border terrier +bord bia +booz allen +black catday +big boi +baz an +bas ilio +; ] +! ðŁĺįðŁĺį +ðŁĸ¤ âĿ¤ï¸ı +âĻ ¢ +á in +y oud +we di +v twx +uncann y +u mas +tw ba +tim lin +teng u +sw ades +stoo she +sor bo +seatt lec +school day +reflexi ve +public is +prabal gurung +pink news +peter facinelli +pan gos +ordin ated +or cid +nintendo direct +nct dream +mur phey +min ta +mill brae +lo lit +lie ben +lactobac illus +kra ig +ko dagu +kiane gan +kaz un +k so +j alo +itu d +ho gi +glove box +gilli e +gil ded +gam era +gag olf +fall foliage +exp elling +e rez +dio genes +club life +bremer haven +boyer town +bota fogo +batter sby +bas uki +b ca +ayushman bharat +autumn leaves +ashish sharma +ashi els +ant ine +air mass +ai ai +ac ine +ac abou +: -- +ãģŃãģĵ ãģĤãģ¤ãĤģ +wi el +whites and +w tv +unbreak able +team j +sub types +sodom y +si ki +shel ton +sha key +sang in +redundan cies +red an +pv ris +pow is +piccal illi +pic hon +pan ag +palae o +pad let +olympi acos +olivi acul +observ atories +nur sed +na ac +man so +ma aya +l ally +kap u +kail an +ju h +jabbawoc keez +iz er +irri gate +ire m +il led +hut ton +fu ne +ferv ently +fc v +er re +endofan era +dr jillstein +di pg +de uk +darrene spanto +d sch +criscy borg +chante use +bless ing +bang ka +at theraces +ash ridge +ane sh +am reading +ad lai +ðŁĨĺ ðŁĨĺ +zimmy deanie +ze ks +yak umar +tu bas +travel more +thu mped +tal ha +sylvain reynard +sph ilippines +shinsu ken +scri vens +sco ding +sch zimmydeanie +sale sharksrugby +riseand shine +ris kie +ram en +pune city +probinsy ano +pos iciones +photom ani +oren zi +op un +nach man +montag na +mandi sa +mal inda +less ens +le ute +le ele +le ashes +la sg +l tn +kem pe +jur ina +je ana +jack alltimelow +ip in +hô tel +hu cks +ham ma +gir onde +george clooney +ge om +equal rights +dprincess maja +dense tsu +dare bin +craigy ferg +conor mcgregor +com fiest +co dd +cbs denver +cb ca +calli son +bring on +bio ta +be utler +authori zes +arte contempor +argent inas +an anas +âĸ Ģ +ภ³ +yew tree +worththe wait +world sleepday +whar f +u yl +swen sen +swamin athan +sub contractor +shoulderto shoulder +sc j +sanc tify +saipalla vi +russ i +record ando +ram kumar +quil mes +queer ness +prosp ering +percu taneous +onetough nerd +oj hl +mon y +mo tional +ml se +meis sen +medit ation +marine biology +mandy moore +maj ik +lasd hq +la ith +l ph +knap sack +je ev +ir g +indian wells +gin toki +forever royal +for ky +evo sti +et zel +ef rain +der p +de chambeau +cul tus +con o +cataly ze +buy british +bam boom +azpil icueta +auburn dale +as fer +am adi +acc intouch +?! ?!?!?! +/ , +ðŁĩ¦ðŁĩ ² +whodun it +war hurst +wag ggs +vindol anda +tum mies +sun down +studi of +street fighter +sti ger +ssi ere +sheriff clarke +sens ational +sd all +sc v +ru ssa +real radio +radi onet +r bn +quadril ateral +power pop +per nam +our nhs +ob tainable +non descript +n ith +mir chi +mars den +lul z +ke ck +k hara +jam ba +his i +gy aru +girl sof +gent ler +flag ships +fer tig +euro wings +et w +es fest +eq nz +el ys +do by +chin at +ceram icist +car rental +canad aam +cam br +c prs +broad well +brand icar +bird watch +argy le +ange le +ale bri +zim m +za an +ximen anr +vir den +vic roads +v go +unf pa +un u +tracead kins +than thi +tech day +spin drift +s res +rat nam +prop iti +piazz olla +ou lis +nor itake +nom ean +margin alization +margarit aday +kil bey +kevin magnussen +jas na +janu ar +houston ians +heart bleed +hadi ah +go po +glen field +dub bel +drak engard +dj drama +deut scher +de formities +dau lton +dab ney +cruise chat +cri sty +coven ants +con sho +clex acon +ci ras +christy chibi +chicago theatre +can can +camcor ders +calciomer cato +butter bean +bre vet +blondie official +black horse +back beat +bab ur +au jour +almo dovar +ag hast +ab ailey +ãĤ¯ ãĥ© +à¸Ńภģ +yate ley +y ron +ve ren +vanv leet +utah ns +upp al +uni body +un lit +un follows +ti fton +thr in +theis mann +th id +team mma +tae tae +ta yo +stro bel +stam en +sop ra +songh ye +smart ness +shell shock +shak il +seeyou soon +secretary zinke +se garra +reck lessness +per ler +pen netta +pakistan inpics +ov um +oce ani +n dm +movie maniac +mol ana +mobil ink +mass u +maric el +mar duk +maj ima +mac ks +lun ched +lsu baseball +lipp man +lau zon +kwin ana +k wen +in nam +hul king +higu rashi +hair dos +gri zzle +go be +gal ashiels +fortu itous +fe ckless +diure tic +dap ur +consu mable +cn as +clo cal +chou ston +chic ana +char ney +cen g +cav apoo +cam igu +bu uuuuuuuu +bu football +bo ak +bnppari bas +black series +bee ley +ban deau +back streets +at tired +arab a +app é +alex x +affe c +ad ani +zon ta +you kai +yo ann +wy nd +with y +vidy asagar +tru stin +trou pes +tre mens +tom welling +the kinks +swo ggle +swif tie +sun river +seung cheol +scorpi o +school mates +schom burg +rock hounds +ra ynes +quir ky +qu or +pu glove +plasticfree july +pain management +na ha +ms build +modern ising +match room +ma stro +long street +long shore +let golf +lein time +lal as +jacobl atimore +instant pot +in secure +hv ff +heuri stic +hen lopen +gunter sville +fe ste +farou q +est ates +eas ily +cv l +crani o +courier journal +con formal +com a +co ffer +church ills +cd japan +carrief fisher +carol us +c ill +bryan ston +bour din +bil ity +big city +ben der +at oc +apollon ia +am coll +a ice +ðŁĺį ðŁĻĮ +ðŁį¸ ðŁį¸ +оÑĢÑħ иде +оÑĢÑħиде Ñı +yam yam +yak kopin +yakkopin ky +well ens +wan and +vit aco +trian gulation +tip toes +spread thel +revision ism +re vie +rat rod +pun net +phal f +pe pes +palestin ian +p sn +op ic +mill port +mie sha +mer cat +mee m +mb a +kl ick +haw co +h sin +guineap igs +gn um +e stra +cubes ats +cu an +cro se +ce mber +car min +bran caster +bo gummy +bankof ireland +ban ana +bac ar +ath u +at tucks +at palace +am ante +ad ang +ç¦ ı +zu ri +yo shino +woody ard +we stridge +v ought +un initiated +tx l +tuk wila +team dignitas +sustain er +su ps +storage wars +starr music +st weed +sav ann +sarah g +ric kast +remo vers +rat rams +r pp +pharmaco vigilance +pax son +op seaworld +omnic om +nissan leaf +new species +motor cross +mo gol +mishand ling +martyr sday +lay la +labrin th +la weather +kir ra +ki vanc +k ation +jun ctive +j jang +inside edition +in super +import ante +i ims +han off +ham ar +grav att +glos biz +gar w +en rile +diet mar +di derot +deven ish +cyn ics +chu ang +chester mere +bay bee +bagh datis +b ör +b go +au lani +ato v +am ate +adam sbeer +achrist mas +. ðŁĺĭ +% / +ðŁıĢ @ +ðŁ¥Ĭ ðŁ¥Ĭ +íĥľ ìĹ° +ঠľ +youn us +win dom +wil fredo +wi ve +whole grain +vit amix +v sg +u dit +ty mp +t mm +t caf +sw ed +st clair +selec tman +re validation +perfect ly +parrot fish +pacific northwest +orbit z +or miston +one um +oliviacul po +np cc +nor iko +mod der +micro film +meg gie +me sab +manner ism +maic hard +lucha elrey +looney tunes +kum on +j man +invali date +in consequential +i drive +house building +gü len +gir lish +gal pin +forever freo +f kat +every pony +event tech +even though +esper amos +eny imba +em mer +elev ator +dun ny +du ken +dream er +dow jones +df es +corpor ator +colo gic +clu m +clairvoy ance +castle reagh +c ile +bun so +brain z +bi ster +bbc sussex +bat ali +anu el +® ! +z drav +yo shimura +walsall fc +video tape +un fairness +tas i +tann enbaum +sti eg +stephani ej +sri divya +snell ville +side swipe +se amed +sa ada +ry lie +reth ym +progre sso +pen hali +over sea +nic olo +national rail +mi thra +mi go +marin ers +leap year +kress ley +kau er +j kenney +inter mezzo +incre dulous +hipho pp +han ash +gar con +fic es +ff le +faiz an +do ac +dis loyal +cu g +chem a +cann avaro +c bit +bron chial +braz enly +bra hm +bic ho +berrye ssa +ben jie +beis govuk +b young +b gu +ayo tte +ak m +ak iyama +ajinkyara hane +after buzztv +âľ ½ +ya akov +world skill +vi rago +un characteristically +u ric +u pu +tling it +thisis not +tech meme +subpoen aed +straw weight +south wind +sit well +side step +shu tt +serge y +ru tab +ro strum +river banks +ple gia +pi xie +phospho rescent +pete souza +paris review +oned ayat +official keef +notone more +n bo +ms v +moun ir +monte falco +mo xi +min nette +micho acan +metropolit ano +mall i +ly m +kn es +ke ish +john f +job seeker +iu pac +invite es +inexhau stible +indi g +here with +hal di +go blazers +go big +ev eld +et bu +dre dged +detroit ers +dar relli +crohn scol +col sen +clarence house +cas ado +cape verde +cap ac +by ars +boat race +blen der +bick le +bi ding +belo ved +bad la +azur lane +ardnamur chan +aqu avit +applic ation +ani si +ame a +after the +aa ic +. _ +ðŁĺ©ðŁĺ© ðŁĺ© +âķ ® +yad utta +vuni pola +tru die +tra bal +the wonder +the planet +ter abyte +tay schilling +swis so +sor ority +shus engupta +sati ated +s gottalent +revel ing +ref low +ranul ph +pirate pride +pen tax +pall one +order of +or lean +oli d +nag in +n tx +mor an +mi sti +mc gary +man jun +lyn de +long staff +lo sin +lo aded +lake field +krakat oa +kis ka +kim wilde +ki da +keke palmer +johnshopkins sph +ib ach +hen nie +happen ing +h ent +gunne dah +glady sb +gen elia +gaz eta +frat ellis +fra gi +for aday +ermene gildo +ear thob +du ino +dren nan +dor us +dal an +cri ollo +cot tawa +city west +callu ses +c gh +bu er +bra vi +an fc +ðŁĺį ðŁijħ +ë°Ķ ìĿ´ +âĺģï¸ı âĺģï¸ı +ÌĦ ) +ys ch +x all +wis co +western balkans +wau paca +uneven ly +tw ells +tric homes +to su +ti u +snape ee +sha har +sac d +rie ger +ri ada +revi led +reas ser +re den +power to +pla stik +phen ol +pen k +pati a +pat mos +par isa +pag ers +pa store +njpw world +nca atournament +muskete er +mini buses +lon ny +life son +ky gomusic +kinky boots +ji me +infiltr ator +in voluntarily +hom elands +getit done +gal á +fre ja +first take +enscon ced +emily vancamp +embroi dering +eli ott +dizz ee +dhe a +del ica +dana her +crusad ing +com ando +cog nos +clam oring +cine malaya +can mnt +c adel +bryn mawr +br onto +br iny +blu x +bas news +artof visuals +aroo stook +aral gon +and t +after school +adu tta +)))) ))) +ðŁĺį ðŁĺī +âľħâľħ âľħ +wim berly +vw camper +vin ter +v sm +usur per +unmistak ably +triffi ds +tis bury +ther iot +tcu football +sw elove +stair ways +spa rent +sommeli ers +slee per +six to +seal evel +rol lei +rit chey +richhomi equan +pra sar +popu laire +pl ang +pizz ic +nor disk +nom en +mid tj +mi versary +mal da +loath some +living legend +laryn go +inter twine +i play +harry con +hand maiden +gar ut +fre eney +fixed andfurious +extramar ital +esl proleague +emma bunton +east bay +d ze +courvo isier +clo onan +ce ee +bron zes +brandicar lile +bo boi +bluee conomy +black flag +be heard +b cla +are vival +angri est +amcoll surgeons +ali ft +ab bit +ðŁij ª +ðŁİĻ ï¸ı +writers room +wo d +west en +west borough +w dp +vidar bha +uu a +ut ada +tusc any +tivi sts +te eling +sun na +sla k +sc pa +sav oretti +sat nav +remember when +re locate +re interpretation +priv ada +prim and +pheno typing +pal ma +pa pered +ol ton +ohhill yes +nu tra +mol li +midnight texas +lu gger +kabat aan +jitter bug +im parted +hudson river +holo dom +go flames +gn ac +gaz e +ga hr +g sy +free day +fly board +fer rol +etsy vintage +en ca +done yet +demol itions +de su +cro cuses +cioccol ato +cin quec +chancell or +bo ic +bla blac +bharatanat yam +believ in +baw dy +bat boy +awild cats +as fc +ame morial +alli t +ab aga +ðŁ¦ ķ +yn on +wallace burg +ves na +un questionable +tu meric +thermonu clear +the wolf +th night +test es +squ ire +shaw lands +shantan um +sai ram +rock ledge +rever ts +r co +pun cheon +opportuni sts +nightri de +nau set +mon dale +mil ledge +mi kul +mcal eese +malware bytes +mail sport +ma vis +llan rw +koh samui +keep familiestogether +kat suki +jug end +j atra +itali ani +ib dp +hä agen +hipho particles +from the +frog mouth +essel dorf +du tty +dre ger +de gaard +de coupling +cun nin +ch ons +bur gin +brune tti +broo ds +bra den +blue shirts +alto cumulus +al ped +ak ira +aj anta +! ?) +ðŁļ ĥ +ðŁĵ ½ï¸ı +ðŁIJ» ðŁIJ» +yan sk +wn ers +what we +wen ham +too oooo +thorns fc +sy er +swa ths +suicidepreven tionday +splat ter +sco by +salz berg +s jobs +rwand air +rose bowl +rezz ed +relap sing +photo shooting +ou there +okano gan +oh dems +o logies +nas o +mskar inab +moz gov +metamorpho ses +lu ella +la vie +jo ice +ha seeb +grey goose +goo fs +fur rier +frene my +ern becher +edf energy +dy outh +die hards +di bley +darrelli ssa +carolin as +caf reeland +buc n +bu ro +bronz eville +broken shire +bra uer +bon n +battle bots +av alier +at ured +alstro emeria +; ___; +åħ ¨ +wim dive +wil kens +vine y +vine eth +viare ggio +umic hb +top stories +the specials +stang gang +ss s +sp cl +sonom acounty +sloven ija +shot ts +sche ider +sch auer +s fer +rob inette +river thames +re mco +rbg sydney +ram nation +raiison ai +rabino witz +ra baa +premi os +pon tins +pav o +pau sini +onof rio +oc ps +nik kie +n we +n tf +mir co +ma dowo +lotro family +leyton orientfc +kaisers lautern +in canada +humili ates +hon fleur +hol kar +h lb +gy un +gun gah +ge isel +g px +foreclo sed +fl aca +dutch gp +dic arlo +diam ine +di j +corn husker +cole co +camigu in +bu ti +bhagat singh +beu kes +baro ka +bar ne +bandain amc +bandainamc ous +al pu +a ben +zam mit +yu i +wa ks +upad hy +up slope +touch wood +the story +sun ray +sie sta +sbag ov +sales men +s games +ri mington +resi stances +rescu ecats +re we +ra hi +r ms +photo cred +pgat our +ob ye +nra show +now ski +ng ata +ne ela +nan ban +middle brooks +mahabal eshwar +mag non +lyn mouth +lud lam +lu glio +lampas as +kurni awan +ki ko +k ma +jump street +jis shusengupta +jai shankar +i ud +i acs +hvac r +hu gin +harrycon nickjr +gun play +gor bals +ger oni +free friday +fos sey +folklor ico +fly dubai +fen ian +face to +el si +do vish +dac us +couple s +con away +caricaturi st +cardiff met +bridge tt +bri sa +bit map +batt ambang +auto cracy +ðŁĺį ðŁ¤Ĺ +zi au +wg na +wer f +weneed diversebooks +wa aaaay +vi ja +uss f +un diluted +tr v +town sville +to st +sky running +shutt ling +seamus dever +sch all +save bangladesh +sal ander +sad owski +robert j +prin tz +par ai +paci fy +or moc +olong apo +north shore +nebrask ans +ne ft +mcgr ory +leighj alland +lap els +ku opio +kov acevic +king ship +kel and +kat ainen +ka appa +il ic +heli opolis +gote borg +gom bak +gi bs +fi ora +doug stanhope +do ds +desig ning +d ander +clock work +ci gal +cham orro +ca is +bu it +breast fed +bio safety +ay ck +avi ans +auto corrects +app arte +ðŁĺĤ , +ãĥ¼ ãĤ¹ +௠Ĭ +womens football +win ne +wh iotv +vo iture +ve stry +triglycer ides +tol leson +tip top +this sss +thel an +the steve +super charging +stephanie abrams +sp aci +sch noodle +sail ly +sab is +ru gger +rati mes +ra vo +r pn +putt ing +post in +pm lunch +plan b +pi pette +pen tre +pe quot +paraly zing +nasa juno +nano structures +myer scough +mskristin kreuk +moose head +mil lage +mal te +lam da +lam be +khan um +ker mit +insemin ation +in nn +ho bs +hel ford +gum drop +ger bils +frug ality +fiel dof +entor noi +en dos +diar ra +dead sea +dat ura +dam ai +commiss ar +cla fout +chill an +cau ayan +buccle uch +blu mer +bitt man +bb ish +b tb +au reo +ani sha +angan wadi +ang kor +ahmad u +afflu enza +ðŁĴļðŁĴļ ðŁĴļðŁĴļ +ì » +âĢ¦âĢ¦ âĢ¦âĢ¦ +zab riskie +x om +wie ber +wal berg +ven timiglia +vau cluse +un focused +tbur soni +so bey +ske han +sk ua +shau l +sax o +real shoaibmalik +p blchat +outwar dly +nofly zone +mu thu +min nis +me sop +marcan thony +macie j +ludic rously +kat vnews +kak inada +k lar +jai ho +j kovic +instrument alists +in atur +homec oo +hol mes +forthe holidays +fel sted +feeling good +e gging +duc tile +duc ale +diri gida +colour ways +celebr ando +brun ell +brain stormed +black box +berlin phil +be bek +bbc newsnight +baby boomers +allu arjun +all ato +ac ol +ab ridge +ab la +ðŁĴĻ ðŁıĪ +ðŁį° ðŁį° +åĨĻ羣 æĴ® +wool ford +wo wo +wo th +ve ka +tw f +tri gun +tr aralgon +tor x +to bruk +the legomovie +ted lieu +tand u +t fest +t enga +sky zoo +sa asports +s england +ri zzi +rhy med +r fra +plou ffe +pit i +op an +on ha +off f +mon ik +mon al +milit ar +mccaf fery +mb assy +lp live +leaf snation +lati fa +ká¹Ľ á¹£ +kod joe +kle i +kis ki +ke ble +jig gle +jeep wrangler +h dg +go bigred +ger ties +gal atea +food drive +f ani +eme lec +ely se +drum corps +do ire +demet özdemir +da her +criver side +crack pot +collision conf +chal upa +cele bs +bul gar +br once +bham citycouncil +baldess ari +arc am +ah lam +afl draft +ðŁĻı ðŁĺĬ +ðŁ¥º ðŁ¥º +å³ ¶ +wunder kind +wn z +warrant less +w pm +w mar +uval de +un deserved +u yu +tt ough +tru itt +tre ty +toyo tam +tit is +tel fair +tejash wi +stat esof +stan well +sp ada +shor tz +re styling +re primand +re compen +r inn +pra kan +polar isation +pil as +pega sister +over react +over hanging +ou uu +ou ple +on elife +nier automata +new ley +mor ang +mccol gan +mat an +lon oke +lau dat +la von +kra ys +ine ment +gu b +grey scale +go solar +galaxy sedge +fr acked +febru dairy +ex tol +edgarallan poe +dr david +die sen +di stefano +derry cityfc +debu chy +de hydrating +cu ffe +chu i +cas p +car stairs +bug ler +bucu resti +bc bg +baw ah +ardi les +ac tie +ðŁ¤Ł ðŁı¼ +yoko zuna +woo key +tru ax +tel om +tech trends +sw stheband +stor res +so ames +sni fter +sheet music +sav ana +sali x +robert deniro +ro tis +rhin elander +red shirts +ram age +pol oni +pluri bus +peter pan +pen nie +p live +over land +or feo +nic ek +mor ir +mix x +mie ke +me aghan +mazz ola +le du +land ol +kib butz +java ee +j den +ian wright +haber korn +gavi ria +gas lamp +fu ror +firstworld war +fen ny +f dg +end c +element ary +e trian +dit t +distill ate +destabili zing +dad ri +comer cio +comedy central +cav itation +c ó +boiler room +bick ford +bett ym +as anta +aj staff +adro it +accor ded +ðŁĮĬ âĺĢï¸ı +whati do +we tton +v ri +tro xy +the mi +t mag +srini vas +sport scotland +speaker phone +sho down +sh apps +scoo king +san dip +ro di +rail yard +q ari +poten za +polit buro +pi gg +perpetu ity +par lament +pal ani +over work +ourhome bts +ott is +ogall ala +ny cb +no ts +nis kay +new play +ne aarts +mc gla +ma fs +lu pone +lo bi +le red +kor ir +kel liber +john isner +illing worth +i ui +humble brag +ho gg +go yo +formula one +eras mus +enricom olinari +election ske +double trouble +dev ours +de generates +d models +cheong sam +baro que +at ower +affor ding +ad ate +ðŁĩ¨ðŁĩ ´ +wim pole +wh ish +waian ae +visit finland +turn downforwhat +sp lot +sexual health +ro cs +river stone +ri squ +poun ced +phoen icia +peace making +patient engagement +oz aukee +offic elife +news goldcoast +neuro feedback +moon ie +mcgill ic +mc cra +marti jn +labou m +la du +jon atan +ji ki +jam ali +ito day +inner space +hydro static +h ppy +ft c +friends forever +friend z +free charge +fish nets +evalu ators +esch atology +edu k +d win +cross er +crawl space +clayton kersh +check box +av otes +au da +an jal +alay na +al maz +ade sanya +ðŁĩ¬ðŁĩ§ðŁĩ¬ðŁĩ§ ðŁĩ¬ðŁĩ§ +ì ³ +âĺº ðŁĺĺ +x ray +universe today +tr ing +tor iam +toi indianews +ti sch +sing ler +sigi riya +se pi +rei ji +re all +ravens burger +ple dger +pk subban +photoof thenight +pete hegseth +pam plin +outsmar ted +net minder +mer amec +main line +ley burn +len ard +kha il +k bb +just ise +junk food +ju tting +ji k +hi h +har asses +gri zed +gra hn +god den +geop end +fuku da +fru gal +fre ear +f enti +en fp +ei th +east london +dan electro +crun ched +cod ghosts +co teau +cnbc africa +clu buk +carlo ta +bush master +belli ssimo +bel z +audi en +au chin +atta c +ast and +artex hibit +anat omic +alman ack +ðŁĺĦ . +ðŁĸ Ĵ +ðŁİīðŁİģ ðŁİĪ +âĻ¥ " +ॠĤ +yyyy yyyyy +u tters +u erto +u dio +tur fs +take my +social club +smur fette +sexy back +serv ite +science hub +sb way +sas u +rum sey +oper ade +nwan kwo +new belfast +never not +master piece +kho za +ke res +jan nie +james reid +insta size +hoar ded +hir scher +high snobiety +grun ting +gofor gold +gi vi +flu gha +eo sinop +ddf tennis +dal les +cro sse +corn starch +consho hocken +com posted +cod worldleague +ch ta +blake ley +bartol ome +bab os +anthon ynolan +air patrol +ðŁĺĬ ⾨ +wom yn +west pac +victor ino +v um +v tb +un appealing +uh v +tx wine +tusk sup +tram pling +tou re +tou quet +tostad as +tonythe sharky +th aa +tar por +tang s +su ger +stra sser +south bury +som bras +shand wick +sc ip +rout ledge +ri stretto +real ale +phillips burg +pett ine +palli ser +out station +olajide bt +nor cia +ni hl +monster palooza +mis shapen +midtj ylland +mi yako +mel ale +local ism +let live +ksi olajidebt +ko ston +k sr +jodre ll +jacob y +izz ah +gor khal +glori fies +gali lei +gal los +frizz ell +finns law +fe ms +en jol +elaz aro +don agha +di smo +dam mit +credit on +concentr ation +closing ceremony +ca ine +bay port +any u +an king +aku ya +aj pw +ab ndp +ðŁĩ¹ðŁĩ ¹ +youth voice +wy ss +wool ly +whir ly +westh ampton +walt zes +vill eroy +toronto library +torbay hour +ti thing +thumb elina +thrif ty +sub sides +steam works +ru ach +re dic +ow ler +napo ca +n olo +mess iness +me ira +kianegan wl +if taar +good girls +gi ef +gen icity +gar nier +gag op +frankfur t +e ban +creation ent +cr sng +circum navigation +christen dom +chapel hill +bor ax +beryl lium +bad der +austin basis +arca di +an bu +amber gris +adorn ments +aber aeron +_ # +ï¹ ı +ê¶ Į +å¿ « +âľħ # +âļ«ï¸ı âļ«ï¸ı +zechar iah +westy orkshire +well done +we zi +val tell +un aided +ul time +tur turro +tis me +tat sumi +tar heel +sub stations +su rab +ste eds +sp ittin +shinge kinok +shi vu +shef ali +sal ters +rome u +ro mps +ro mare +red book +rece des +react native +q ns +oli vo +ofcal dub +nu do +no bile +ne whe +mu tes +mbab azi +mati p +mag ali +lov atic +ley y +le mente +last chance +ko wal +ke ek +kav it +karas wisher +joem cel +jay dee +jab dulkalam +i id +horse tail +hit record +hill en +highe red +her li +gru en +ge te +fou lard +flyn avy +fled glings +fi ya +ferman agh +en ov +don tre +co be +cal ne +bat y +barbar amills +bak so +arba az +an pan +an ai +afric abiz +( ? +ðŁIJĬ ðŁIJĬ +âĢ¼ ï¸İ +xilin x +whynot us +we ho +war de +v are +ute p +upl and +thir sting +taw fiq +su ren +st wo +squ ab +simple ton +si fted +pra tham +pay an +pa ez +osc on +nytimes books +no stro +nev ado +nay antara +natural health +nas wamy +mo chis +mk bhd +man ey +lu ma +loveli braries +la so +kiefer ravena +kat sumi +johnny gargano +inju rious +ine o +implic itly +hu alien +hoo ped +high clere +hey violet +he ian +ha ath +gm hc +fore arms +fer ny +en vision +en raptured +emer al +e bi +corne jo +commun ing +cedar burg +ce ph +cam ero +cabo olture +blon ds +bar ish +armi stead +apri mary +apo sto +acom edy +ðŁĵ¸ # +writers block +warat ah +vers ic +tom thewanted +ther adio +sp row +siva ji +ru tten +ration ally +par te +opol itan +o jama +nose dive +none w +ni osh +na as +myceli um +mor oz +monster hunter +merri am +maum elle +lyric opera +lutter worth +kron o +ker ma +juxta poz +j sk +inj era +indi ac +hu ach +hd fc +hak ur +gug ino +gl or +ging ko +gi ggled +g isd +fl yy +fen ella +fearne cotton +far ha +f kf +export ing +explore mb +es for +ent ente +down field +dol gel +day sin +da st +crowd funder +compe ti +code breaker +cla rens +ci bolo +cater in +cari boo +book plugs +ble eker +big ge +az ian +aw restling +atul ations +apollo theater +ag itate +ab ird +ðŁĺĺ ðŁĴĹ +ðŁİĪ # +âĿĦ âĽĦ +âļ¾ï¸ı âĿ¤ï¸ı +ynlle th +yel p +whi pper +vote ph +vat ore +usat f +uh mbt +that girl +thal esgroup +stop gap +sto so +stepla dder +spare ribs +si mp +shadow play +ser in +secon ded +scot thall +rvad ine +ruck a +row der +river ine +rik ki +radi okc +ra chi +profit eers +pp q +por ium +polynom ials +ple ee +phlebo tom +pear sall +pam ela +oli vares +nor te +nano tube +mar mi +ma gui +lisam arie +le mming +la hari +ky sportsradio +kk ur +kelliber glund +k ents +joshu at +if bb +iaff newsdesk +i leen +ho ax +happ ie +gar rod +from scratch +evoc ation +drewest ate +done z +dam my +d tu +cha ar +casta ic +bon elli +bo jack +besee ch +aw ai +au tun +aly sha +ain f +af ate +íĶ Įë +ãĥķãĤ§ ãĤ¹ +ಠ³ +youare notalone +yash in +whit worth +vi rage +veri der +the godfather +tame ka +tad hg +swami vivekananda +st witter +squ alor +spruce grove +snow suit +shi ori +sh ilo +san aullah +sa wai +ro ssie +rizz le +rhy sme +peppe rell +ony i +o el +ne dd +naz ia +navrat ilova +nar an +nam u +mul berry +magno lia +mac duff +lu ego +law ren +la scru +kam ma +jimmy page +jc cc +jag z +implo res +hur rying +happybirthday louis +gu thri +ge sell +ge ee +films notdead +epine phrine +du mba +dra ya +dhy an +death valley +coo ol +contamin ating +colla zo +climate action +capital isation +bre zh +bouff ant +bor don +back haul +av tar +aho ya +ablanc a +abidi factor +!!!! !! +ðŁIJĺ ðŁIJĺ +ðŁįº ðŁįº +åĿ Ĥ +wood crest +weak ly +ver vet +ver ner +ver mel +valdis ere +uni mog +ton ny +tom tom +tl h +ter ization +tel ome +social housing +scho eman +schi er +sat ch +rhon da +rashe eda +od di +o ita +nt chat +ni aga +mir allas +matte orenzi +magi ster +kun de +ku mu +khich di +k scope +k ado +james dro +j rr +insight moments +ing is +indi af +in firm +h bi +gri st +fe g +ef ta +destin ed +de cluttering +clergy man +cler c +classic rock +cave ats +cap on +cam pe +billi ken +be zu +ayl sham +aye ga +art man +anglo phile +âļ ĸ +z mir +uc ps +tyre ek +twer kin +tre et +thwar ting +teacher problems +tat ras +syn cope +stall holders +sports law +rise above +rema de +ra gas +pre war +port side +po sti +phe onix +pett itte +pen ia +pe eves +ol shop +od ell +nu tty +nat acha +mun cle +morin ville +mol ting +mc gwire +mas lany +malm steen +lee way +l po +kill deer +k team +ja q +j aff +itweet museums +id pol +go bier +fp so +ecar ter +dream pop +dra go +don buri +digi fest +death less +de di +dan ton +cou pler +ch ng +cal das +black belt +ball nyc +ba ghe +aviation daily +and ini +ama al +am ira +aha a +ðŁĴļ @ +ਠ² +za x +v sk +un im +tu atara +tt ag +tan ehis +tanehis ico +stell acre +spir ited +serv us +scott sville +ro ches +re arrangement +pung gol +po ivre +pig pen +per ation +outstand ingly +oh bm +nn taleb +ner dist +na hum +ms state +mor tician +mel os +me ti +mar kin +ly k +luck now +ktn v +klepto cracy +kenne bunk +ju gend +jol aburnett +j jc +inou ye +ign ition +i ima +hope less +hodg kins +ge th +fir at +f dk +ex tolling +ei fert +divul ge +deter mini +dei mos +dam aris +cr aco +cor dy +clau del +ck ton +cityof boston +cere mos +carlit os +brass band +berto ia +bent a +b have +aw rence +as ci +architects jrnal +ann yt +ann nn +ameri go +ah ana +accur sed +aar av +ðŁĺĦ ðŁĺĦðŁĺĦðŁĺĦ +ðŁĴ£ ðŁĴ£ +zis sou +you reyes +wag le +vi bhushan +tv c +turn down +tt age +tr d +te fal +t ver +t smc +super structure +spi got +speed kills +sou derton +skip the +sil sbee +scri bd +sa wa +s anni +rotham sted +robertsspace ind +rigor ously +r ori +potawat omi +play out +pas ang +p elli +ouston bill +ot tica +one man +nor then +no id +nis qually +ni ved +new grounds +n ÃŃ +mor gado +mc dade +maq bool +main tainer +mag ines +lubric ated +llande ilo +lar on +kop er +khil ji +joemcel derry +inti mates +instac ool +ino hio +im acs +i fit +hot star +hij rah +he modi +guy kawasaki +gif ty +fre em +fr ater +expe dient +ergonom ically +eph oustonbill +en amore +don ts +dissoci ative +continu ou +collabor ative +cleve rest +cam pinas +budd hist +brin ley +bla in +bbc new +bay on +bar tle +ax c +aq at +ap om +amar tya +alo kesh +ag glomer +ac tros +ye olde +y itzhak +x or +teme scal +str acci +star chy +sel lo +scar p +san doz +sa hota +raw ley +radio graphers +pet ts +pet shop +pet sch +opha gy +off ically +nandam uri +muzzle loader +mo har +marco tte +lun atic +kk w +kil ted +ke vic +k fs +john krasinski +hill cats +hen ni +handson learning +grin ners +graci ela +fri dley +fe ted +do bre +demon izing +comm itt +code share +castig lione +cap onata +bud die +buck o +bra ult +bi alik +bhu j +bal een +an avenkat +. ðŁĴĹ +. âĺº +ðŁıĨ ðŁ¥ĩ +ìĿ´ì ¦Ī +ãģ¦ãĤ ĭ +z m +we tten +tz mann +tu gh +tra ut +tion of +thought fortheday +the field +super iors +stere ok +source books +sour beer +sob ha +sil berman +si ero +shu riken +shop house +shari f +see theworld +se fa +scu gog +sc rn +ryo suke +rr sp +rot fl +ri pley +re mick +r lm +queere ye +pur port +plu mas +planet fitness +orth west +of cr +neg ombo +mot ility +modul ating +mm sm +mcgu inn +mb n +man soura +limit er +li j +ká¹Ľá¹£ á¹ĩa +kry sta +krist all +kariz ma +kam pal +k sk +j ra +immigr ate +huck le +hood winked +ho tele +hitrecord joe +ha ine +gro veland +google io +funny ordie +free styling +fer g +ebay us +e hrs +dennis elazaro +dem itas +chilla xing +chick am +cher vil +ch yn +carto ony +bri sson +bookie bashing +blan ked +be held +bad akhshan +b ve +att in +arn age +angel man +andthe city +ad ve +ðŁĽ ¶ +ðŁĽ ¡ï¸ı +ðŁijĮ ðŁĶ¥ +ðŁı Ľï¸ı +ðŁ¤¦ ðŁı¼âĢįâĻĤï¸ı +à¹Ĥ à¸Ķ +ठĩ +ÙĦ بÙĨ +ze meckis +wil burys +waste iton +war farin +v yy +v spit +under dog +transa via +topp les +ten uta +tatatru sts +tac kett +ta wang +t iler +sol ara +slive at +scho tten +schap iro +san an +roo depo +ro gier +retro actively +redribb on +pa hal +ol lective +oc co +note cards +mb on +ma pr +lal ala +lach ine +kirkin til +kan ovic +kal at +k cac +jo cke +jau n +hoo ba +gol fo +giu sti +gh anta +gg ings +food tour +food stuff +fiestab owl +festo on +f ale +esp agne +er inn +el ax +eb ates +e vie +dover court +defen der +dav ros +d lg +corpor a +conju gal +con dra +compe te +cheo hospital +cas illa +bur ston +ar sha +amhar ic +amber ley +al waleed +al tr +âĸ½` )/ +ঠŁ +yar ratrams +wann eroo +wa cht +valle ly +ten na +story ville +steve o +ss entials +solic its +sho m +she athed +san juan +sac tive +rose mead +re stocks +per seve +pedago gies +over charging +olo pez +o reca +my house +must love +micro greens +ma zo +le eper +jim bo +itiner ant +ir radiation +intothe spiderverse +inter bank +halloween movie +gy uri +glori a +gi dley +gho tel +gett ys +fu cc +flow restling +fle ck +fl ander +fire itup +fe stering +faryaltal pur +er dahl +dumb foun +du esseldorf +down link +darwin ian +crissc ro +cor rina +cle fs +city market +broom hill +belling cat +bag at +b gg +ate y +as mus +alet tori +ak lan +aesthe tica +a out +ãħ ħ +ÙĬ Ùĩ +wor n +whole hearted +wan es +video conference +us coastguard +up church +tw addle +tuil agi +topp rog +team ster +susan sarandon +su mas +sreeni vasan +sol r +share aholic +shal er +se idler +sc avenger +sar sapar +sap uto +royal holloway +reinst ating +region alism +pulw ama +pri macy +pin sky +pat ellar +ouri st +nough ties +mous asi +mor fa +min elli +meur sault +med line +mas bate +mar voree +maine for +lo gro +lmk moviemaniac +liter ature +life uk +lief de +lhh ny +laval in +kome dia +ko tab +khar y +kau a +jes i +iro z +hilde brandt +her to +he yy +gov abbott +god wits +gl bc +gare t +galli ani +free ajstaff +foli o +fli ka +f issu +ev ir +e sus +den yer +dad dario +contemporary romance +col ler +coale sc +clafout is +cho wan +chare tte +cere bellum +by ford +boro silicate +bla bber +au ras +arou sing +alexander platz +adver bs +íĭ ° +zu hair +vol cs +ur bant +ter cera +te ye +swamp scott +sundar bans +stru tt +speech writer +sp rod +skid ded +silver a +shav uot +sci fic +schen ley +sam aria +sac republicfc +retrac ts +redro ck +re doubt +pier ro +pere a +oc sc +now bath +notb ad +nom os +moncri eff +meme history +mckend ree +mask er +lay ar +kha qan +keyne sian +ju dah +go green +gi om +fo cal +flour less +dry january +dis se +del hai +champion nat +bou tros +boere wors +blind sided +ay or +asap mob +ar monk +addle stone +aar de +âģ£ âłĢ +wax haw +wai pa +vr v +ver ville +tri ghts +trade center +thaic oup +t mac +stri via +strath co +stra des +sponge bob +so th +sen sed +sel alu +scru bbers +rwn j +roman off +robg macfarlane +ri gu +ri bbing +redskinst alk +pe dest +omic con +no ce +my girls +match worn +mass art +marc jacob +magnu sson +madef ur +machine gun +lor ton +lily aldridge +less ard +leg ate +l ä +ku cin +knu te +khlo é +ju p +infl ames +imp i +ic ep +hudson ville +ginni fer +for people +f yl +eli o +egg s +dir ge +ddp yoga +concentr ator +classic traction +charm bracelet +chamele ons +cas alettori +candacec bure +c sl +blan ch +ba bee +americana fest +alwayss unny +along with +aline a +ais ne +acqu aint +abb oud +ab ric +ðŁIJ Į +ðŁ¥ İ +zacu to +yello whead +wi ek +ween ey +turbo charger +takeme out +stock twits +stam aria +sonic boom +sky bridge +schu ette +sacri legi +ruby ph +regu s +quest live +py lori +purcell ville +ped ant +pe toftheday +pal mar +opale scent +ob ito +new age +neglec ts +national library +nash snewvideo +mt nug +marlon wayans +lu lla +leep ing +lec raft +landscape design +lanca shirehour +lac quers +kha yyam +ka beer +instig ator +infen wa +hemorrha gic +hell mann +hel g +gri se +graffiti ed +good health +good guys +get outand +gammal abs +fru ita +fr f +drive sober +direc tedby +del phi +cont es +clay gate +chill y +chal te +cen otes +cam bell +brett king +bobc ay +bar bier +auto clave +as able +ao ki +anic ole +ami ans +aj as +ag old +... ;) +Û ¶ +zen berg +wm hd +we imer +vir ging +verdi gris +vent ral +var ys +un realized +tto es +ton neau +thing ies +the aftermath +tac on +syno ptic +sul fu +st cc +she ab +sesqu icentennial +scul pts +scor ner +satch mo +rit as +restaur ateurs +psal mist +p ssa +nor o +neh ru +muell ert +mu sim +milag ros +meri vale +mend is +mas r +malli kar +le ea +lamb chop +kyo sho +k need +jel en +irishtimes sport +ino vich +ig at +horn beam +gare e +frank dangelo +fly leaf +fag in +e inem +dun donald +drows iness +dak in +cross words +conqui sta +congru ent +com al +cle ck +ch merkovskiy +bu she +bridg eland +bor gnine +armer ie +arch on +ap la +ane gan +ak ina +" ðŁİ¶ +ëį Ķë +wood bin +wit ten +wheresw ally +wh alum +u windsor +tit chfield +thex factor +thermo genic +ta dema +sw pa +super cells +sung jin +ste h +stag gs +son ning +sk ratch +si eck +segre gate +sal ak +sag at +saf ed +re kor +publici ze +po sit +pl g +me vag +man nin +man hwa +m â +loving kindness +l haynes +kp bs +klo tz +klg and +kl rahul +ke da +kawor u +ji k +hol lies +ghe ad +fertil isers +el itec +docu men +diamond league +condon ing +co td +co debase +chev in +british basketball +biopharmac eutical +bat ons +bai les +bab bo +autom atic +ati ku +amer a +ac tra +" âĢĵ +ðŁĺ´ ðŁĴ¤ +ðŁij© ðŁı¾âĢį +à¹ĥ ห +ॠ¥ +zo ar +wel d +un attached +tu co +tre mura +sub duction +slu shies +sla zenger +sad dled +rock pile +ris ke +realradi kaa +qatar gp +puri sts +plai stow +pat roller +pas anga +pa ils +one piece +ol athe +neighbour hood +nav ox +n alla +miamis burg +mar anda +makat on +magnifique france +ma guin +lake como +l bardugo +kiel burger +kaz umi +kar issa +jo ki +jag i +it cnews +independent film +ge est +force indiaf +for cen +fe br +farm worker +far low +educ ator +despat ched +che tto +brom eli +bro y +brexit vote +bond holders +ben nel +ba ink +avol leyball +antel opes +able ism +ðŁĩ¬ðŁĩ§ ðŁĩ¬ðŁĩ§ +Ù ¡ +а Ñģ +woo ed +wh Äģ +wat es +un ker +umbrell amovement +travel oregon +thi seid +the spec +swat ter +sarco idosis +ro ig +pres splay +pontel and +pharmac okine +oye z +optimi zations +omand m +of elia +n ood +my fav +mi seleccion +medium format +mal y +mal ar +main tain +lin sky +lilli put +leven thal +leeu wen +led lighting +lakedistrict pr +kundali bhagya +kep ner +kath mandu +joel madden +i pur +good all +gay marriage +fre und +espo s +dr ps +dncle ak +dal mia +da gh +d mrc +cro we +cover crops +concert tickets +cinquec ento +chri sley +camp fire +c qb +brett lee +body works +bio statistics +bell arine +battle toads +az sen +av p +astri de +alevel results +al be +ab dou +a athletics +. ðŁĴĽ +ðŁĮ» ðŁĮ» +ë¹Ħ ìĬ¤íĬ¸ +à¸ĩ าà¸Ļ +wind stream +wi u +western most +wear house +war bucks +travel photos +tom and +to liver +spet ses +solidi fying +si qui +si im +shopp es +shari ff +sar m +row boat +ri gon +rebe ka +rain storms +r rexpress +po kk +p ted +oz comiccon +oa key +nw sa +nc sl +nandit ath +myster ium +mclen don +mary landers +margaret cho +mal abon +lot so +lauren conrad +lar in +ko sky +ker ner +k vin +ju ha +ju ego +intermin able +il sa +hum dinger +ha diya +girl guiding +ghan em +gel ish +g pas +fokker dude +fat ties +drum head +di atom +detective pikachu +cor vettes +cop ali +common ers +che kov +card making +can pl +brit z +big day +betwe ens +be sta +bc w +ar kan +aj ga +ai as +ðŁĻĬ ðŁĻĬðŁĻĬ +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤ +ðŁķ ¢ +ðŁĴĢðŁĴĢ ðŁĴĢðŁĴĢ +ðŁı Ķï¸ı +wa seca +vi stic +tam pax +string y +store fronts +stellacre asy +som nia +sin america +shovel er +sc lc +sant an +rit aco +restaur an +recording acad +re ys +ram shackle +r weingarten +r ital +par lez +noy ce +nhl devils +national security +n be +mur si +mun t +mohic ans +ment um +mc taggart +lu sts +lead better +lam entation +kom unews +kis mayo +kikwe te +hexag ons +he dged +green finch +fic ht +fav ase +ever sley +et am +es muellert +east village +dor ito +cz ek +collegi ans +cn comusic +char le +aviate addict +at ac +arlington va +aldu bon +acade mie +able ton +ìĬ¤ íĬ¸ +ëĵľë¦ ¼ +ı r +yaman ashi +x rf +wichita state +vit as +u ams +ting ley +ti ft +theoffic enbc +ta rena +suther land +student loans +skir k +sidharth malhotra +scot gov +sarki sian +post marked +pnc bank +pg d +patho m +paleonto logists +pack horse +orient alism +ontario travel +o ssi +npr news +nl bm +ni eld +ne ms +my style +mercado jen +meh fil +mcc une +mc stay +le ucine +kore gaon +ko lolo +ken an +keen ey +karl stad +jimmy carr +hepat ology +hee ey +heas lip +grand stands +geta fter +gar dy +gam la +gal ante +for shaw +expatri ates +exac ta +eisen stein +dx f +di alling +detal les +cramp ons +contro ver +cn blue +carri zo +cal d +bur ri +bu thele +bu skirk +bo vary +blesse dand +black pinkin +black ink +black cap +ann alyn +ai w +african art +acom pton +ac cp +ab idi +Ļ × +íı ¬ +wide man +wallpaper mag +tro tter +travel goals +thel ad +the music +sque aked +shou p +sher on +sheep dogs +shar wanand +scur rying +sant arosa +retro computing +president scup +poly p +plu ie +per ouse +nick olas +new sday +nade em +my bcu +menom onee +mcmur ry +marau ding +lizz iec +ko echlin +kasim reed +kap ital +jun kie +jenny mccarthy +j news +hur dling +harmon ics +h cac +guardian books +goes west +gaud ÃŃ +fright fully +fin au +fade away +ethiop ian +espn nfl +e fren +du ren +dovi zioso +do gh +crohnscol itis +cro oning +cristi an +conce iving +co ghill +chibok girls +cer ne +categor ical +cancer center +bus driver +bri ens +brendan schaub +barcelon eta +ban the +ban ality +alcu dia +ache son +aam er +[ ðŁĵ· +é Ĥ +âĢ¼ï¸ıâĢ¼ï¸ı âĢ¼ï¸ıâĢ¼ï¸ı +¯ ï¸ı +winniethe pooh +wander ings +u fff +tro st +tri veni +tremura sdv +tac ar +store room +star ling +sit t +septa philly +school strike +sch ist +sam anta +sa fia +ru pp +ro yor +reli sted +pescat arian +pe ats +pan sion +pad mini +ny sa +np ca +nicol ás +ne as +naz e +mis r +mis deeds +mcn ary +man ana +magazine uk +ker chief +kemp town +kal ye +k sf +ise tan +ing len +im memorial +ig b +gravit ational +got w +gospel music +google plex +ga as +fresh ly +fren che +food awards +fis u +fi delio +festo oned +dy mond +du ell +dot ard +df ds +da emons +cabine tre +blur t +bloomberg news +blood bowl +be dra +ban de +arbitr ator +alle ppey +alan ine +adjudic ator +adjourn ment +ðŁĩ¦ðŁĩ « +ðŁ¤Ļ ðŁı» +íĺ ľ +ë°©íĥĦìĨĮëħĦëĭ ¨ +âĺħ âĺĨ +âķ ² +wwe usos +walkaway from +vi ele +usab aseball +un cp +ug k +u hh +tuesday thought +too soon +somer sault +si dec +sen et +sar ap +run d +roth bard +rhiz ome +pinnac le +ost end +nbc thisisus +n pu +mc brayer +lynn field +lv rj +lep online +ken stein +kar mel +investig ational +hoo g +harry winston +gg plot +genev amotorshow +gem stone +ge ge +e jaz +ds v +do ss +di atoms +davi on +dalla ire +cuer nav +ches nut +cam phalf +bag nall +ash wani +anstru ther +amon til +alex il +alex andros +al tor +agni hotri +ðŁĩ¨ðŁĩ ® +w wn +ventric le +valu es +va shem +v rai +une a +umph rey +stay hungry +stan age +soci ete +skip jack +skat elife +scal ar +sanjay gupta +remember ance +real deancain +re thin +pum mel +pr yan +pardon ing +n win +n cos +muzaf far +mur ang +moss man +michigan state +meteor shower +melale uca +ken wright +k all +ih san +he bobiwine +har diman +genes ys +ga e +flu fighter +fe vents +fac eyour +el ho +dro ppers +devan e +cor bet +cliff top +ce de +car stens +car mi +c add +c ack +bur son +bri son +boule z +born stein +appé tit +Ë Ĺ +ı r +zombi fied +z l +x danny +white genocide +vincent vangogh +vesper ia +ug at +u hu +today y +te cs +suki yaki +stuart broad +stret chered +stra ddles +stal ac +shrenu parikh +sch wa +salvage able +s wines +ro sato +restor ative +pu ti +pre pay +on vention +o gt +nuz locke +new friends +nanditath hakur +nai b +million miler +micro gravity +metast ases +mechanic sville +maris cos +ma bility +lav ington +la vat +konstant inos +kipp ax +ker man +ke ar +k ella +juan ito +jan kowski +j wc +irrit ant +her rero +gyna ecologist +gws giants +gu mm +god sown +glory togod +gl f +gar ang +flux us +flash card +ely sia +el yon +edwar dj +dee wan +copali bertadores +clean bandit +bu sted +bry ar +besto wing +ar chri +ðŁİħ ðŁİĦ +âĿĵ âĿĵ +à¹Ģภ¡ +Ùħ تع +Ð ³ +weav ed +we transfer +vi ale +theresac aputo +task bar +superi ore +sub compact +stracci atella +small holding +sig no +si gui +sex work +s iller +revin ed +rak awa +quality time +pu battlegrounds +play music +pi dge +oppos itions +nov ates +n ml +med ill +litt ler +lee seunggi +l lew +koval am +inv ul +in adequacy +idri ss +hyper realism +hu got +hooba stank +grandi flora +fronti ers +freder ique +fast furious +dur fee +dresslike awoman +do les +do ddle +diatri be +del ange +db z +d jor +ctv canadaam +ci karang +char tier +cease less +calend ar +buzzfeed news +bon sang +bo strom +bit c +bin omial +as dale +anti psychotic +angel e +anag aram +a betting +ðŁĺĮ ðŁijĮ +ðŁIJ¶ ðŁĺį +âĢ ł +ि à¤Ĥ +wood cut +whit mire +ver laine +ut tering +u gali +thr ongs +the hate +the ab +th ate +ter acing +star maker +sri ght +son or +sky west +se anc +rohit sharma +rl cs +real king +pu tman +petiti oner +pau lino +pas qual +p bl +nigh tofthe +mus er +mul lin +mom eter +med student +marj ori +mal ena +ma doc +kim meridge +ke ven +k wale +jun inho +juju be +jalop nik +jack russell +indian ocean +il lar +idi ot +hedon istic +gopro hero +glass art +fri ese +empirestate building +dil shan +digit ise +dark throne +chamber tin +butter fingers +bulldo gnation +bu ceo +bri sky +bil dt +at tains +app in +and home +abc sharktank +ðŁĺĺðŁĺĺ ðŁĺĺðŁĺĺðŁĺĺðŁĺĺ +ìĹIJ ìĬ¤ +ãĥ³ãĤ ° +yan ez +wood mont +wh f +wewill win +un heated +twitter chat +tra um +tra fic +town ers +the devil +svo boda +skirmi shes +schumer shutdown +sarsapar illa +sa ks +s rocks +russell terrier +re ster +raf taar +pa sic +mu rai +mou rad +mor ongo +mol u +mattj willis +mar der +man ami +mac namara +lipsync battle +lasgo w +lar is +lake shore +ksh ama +kid well +k bl +ji wa +ji az +itim es +i ese +hus key +ground work +ful le +fol ky +far ook +exor ci +euco president +en sw +criminal justice +cotton bowl +consign ments +charlies angels +bridge ton +bau ghman +bary shnikov +ayles ford +au si +au an +alo sa +ai anational +ab ren +ðŁĴľ ðŁĺį +ãĢĮ # +âĹ İ +á Ħ +yon sei +yeoman ry +visit norfolk +u staad +try pod +tru dge +this oldhouse +the g +tai moor +tac cup +swar brick +stre eter +spro g +sex i +russ el +ram stein +pur ves +proxi mus +pre heat +pon i +phi lemon +pedne kar +path way +over spending +ole a +naz o +na ks +me gay +mari amen +man é +ma bs +kor te +ju rek +jay na +itv central +i sea +he mmed +hd ty +habitu ally +groun ders +grahn ort +getafter it +fram ers +eucli dean +emin ently +dmitri y +dar vill +chi ya +brown stown +bla stic +beli zean +ba elish +aw ka +as ssss +as ke +arc adis +and now +ìľ¤ íĺ¸ +ãĥ ļ +za atari +wix om +wait iti +vor ster +var mint +v dl +u mag +tu tic +tre ffen +track field +the tru +the jam +the crown +ta we +sur man +sthelen shour +south dale +sher r +people soft +pen o +parid hi +nyc go +music ale +mont morency +men ac +logan sport +lady wood +je bat +hem line +heck man +gla drags +gan ano +fo lo +find my +ever brandy +den rele +defam atory +dam odar +dag gett +count yo +con vair +chlor inated +cher ian +capit ola +cam mie +broad dus +audi ophiles +asjad nazir +ashley tisdale +ar mon +ar ino +anth uri +amor uso +aj kumar +ab ani +ðŁĻıðŁĻı ðŁĻıðŁĻıðŁĻı +âĻ¡âĻ¥ âĻ¡âĻ¥ +women on +vol nation +vi m +va beach +tw rites +thunder ball +tag bil +super bikes +star tre +spur r +sof tie +sligor overs +shack leford +sco te +sci p +sche ffler +sar ver +sab tu +ros endale +romance books +ritaco bix +re styled +re heated +re eser +que z +pre classic +pp arty +po les +pis ani +pfei fer +our land +nak ia +n pn +mur ry +mit ro +mario bros +maharash trian +l lao +koffee with +ka at +joyou sly +jak o +ig loos +hilari o +ham bur +hadou ken +gle sias +ga et +fo ggy +experien cen +execu tor +ed dington +dre ezy +dibu j +devi ance +den nish +dash croft +cul che +congre ssion +col later +chug ach +cher rie +cel gene +camp al +bundeswe hr +bu shell +bt sat +bm iller +blit zing +be ag +av it +air crash +ai u +ðŁĺį ðŁĺ± +оР½ +un li +un accountable +tim minchin +thom ase +thi eriot +thermo dynamic +th grade +tea sel +tartu fo +shul k +shaile sh +se bum +sc df +s di +redemp tion +re pp +rach ell +r sr +r sk +r ne +que zada +p ounces +os ine +onedayat atime +officiald annyt +obye zeks +ni thin +nar iman +mostre questlive +mor ales +mat tan +mal formation +make history +madra sa +ma eve +le ant +kö ping +ku dus +kow sky +kan ti +ira k +i mari +how lin +hom ophones +ho vey +ho sur +hin ter +har ber +h sm +grims by +go va +go om +gil by +gar gle +fred rickson +faryaltalpur pk +emulsive film +e jc +dumb founded +dar ah +cold harbour +co fi +chi shti +can ara +calle jon +budg am +bronx ville +best fandom +bel sky +am mu +ak ula +ac cra +^^ ; +ðŁĺ¸ ðŁĺ¸ +ठŁ +wing back +wee don +w anger +vla sic +ur phy +u din +u apb +tri ath +tette h +tess anne +tae hyung +su kh +su bl +songhye kyo +righ thand +qu illing +pp ar +padi ham +ov ski +ous ly +ne mac +nature photo +n ason +ms q +mi mms +mer lion +lin sey +land au +ko sar +knight dale +key port +je ke +jack pots +in sensitivity +her ky +hare field +gu mption +fu ca +fl inger +fe eny +dumbfoun dead +di ak +chri shar +cep bs +bow ley +bou m +bor hood +backto school +b ica +are gion +aj dinger +.. ðŁĺĤðŁĺĤ +âĿ¤ ðŁĶ¥ +yam ba +x hnews +whu employee +wel oven +we madeit +vag us +vad u +u doh +ty pa +tor aja +thecw supergirl +subo dh +stage coach +spro bs +song kh +sm ill +sh illa +san lu +sam b +ross lare +ro syth +remember the +power couple +pou liot +nar uh +museum monday +mi hir +memp hi +marig ny +man fredi +lagunitas beer +kale igh +kalan ick +hog shead +harde sty +gr ann +ex whuemployee +educ acion +east port +e ig +dy ers +doc ents +do ones +david dobrik +dark web +crow foot +cau dill +care of +beingh uman +ay ear +ash am +ðŁĶ¥ðŁĶ¥ðŁĶ¥ . +ðŁIJ ª +zir conium +yo yogi +x co +william h +what to +wetn wild +wer n +un stall +truek ofi +travel ban +tram pal +tione ering +time lessness +tar ak +ta plin +t fo +sydney kings +sono gram +sig urd +se ang +sar da +sapul pa +roor kee +ron deau +robot nik +robbies avage +reece shearsmith +re driver +ran sit +predic ated +plas mid +par oo +pan ti +ox alis +opho tonics +nik kis +n dy +mobile money +middle ham +lul ly +lo stock +liam hemsworth +laker snation +la ppin +ki ddin +k wini +jer man +irishtimes biz +inter disciplin +im personal +ibu shi +hon us +hon eo +hemodi alysis +hehe heh +happ ines +gil der +ge z +furi kake +fun ke +ever greens +de sir +cryp ton +confirm ations +cha ve +blun tly +biz news +be gets +be eny +atsu ko +arn aldo +andre y +amontil lado +all ic +all american +aar u +Ķ 리 +ðŁĶ» ðŁĶ» +ðŁĶ ļ +âĿ¤ ðŁijį +âĸ ij +was co +w ü +vincen tian +ve rett +vat ican +v js +uf v +ts xv +the bay +t pac +swe di +sue perkins +statueof liberty +spreadthel ard +sau cer +saddl ery +richard hammond +rev ich +repri sal +redro cks +re planted +pre view +pla ka +pit on +pemb rey +part ita +oun o +op seu +ol ano +noo ff +ni stel +na jaf +multi rotor +mor ya +mail room +ma athai +lu cap +lod don +live free +lip balm +lion up +lahari music +ko kan +kin u +ki mathi +kar amoja +indi gent +howie mandel +gon avy +gn r +ginger bread +gard endale +game works +fo da +fen er +eye wall +equ ated +ef d +dul se +ds ds +dolom ite +do one +cyberne tics +convey ance +ci bola +c fo +blue berry +blue beard +benzodi az +ben simmons +ban ti +autom echan +ator res +ash ba +al tes +ad heres +) > +ðŁ§ Ľ +⾨ ðŁĴĻ +⾨ ðŁĮŁ +z vi +yn b +ye ates +yaz z +wing friday +win son +wil mar +whor ter +wal lows +vin nie +sta renergy +seag rove +santac lara +san try +s vi +ru cci +ri sque +ow h +or v +oo kies +og lesby +o conom +mv cc +mur taz +mugh al +mouse kete +motor city +mi quel +mi fi +meri bel +me vents +mang ere +lac tating +kyw newsradio +kus anagi +ki dambi +kate middleton +jar vi +j school +ish in +il divo +ib pyp +huss aini +hu mo +horror films +high quality +happybirthday niall +han gover +great yarmouth +ghi bran +ghibran official +f sv +f db +espo ir +deliver able +cuth bertson +cow litz +cosplay girl +clean power +cav afy +cauca sians +bus by +bre port +bel zer +be eper +at eca +ar ain +al bay +aki va +ak infenwa +abh or +ðŁĺį , +ãĤ¢ãĤ¤ ãĤº +ÙĤ Ø· +ziggur at +wyl lie +va aste +up country +uke lele +tre ecre +tifo si +taw awa +switch able +sno bby +smar ttv +schol ten +sar desai +sam melan +ru en +romance readers +red season +re blog +raghu bar +plo tt +plear ning +pal try +orange men +n mu +mysti fying +my att +missing you +millen ial +mari est +maku eni +lé on +lu cked +loveis intheair +llanrw st +lar gent +l ances +kinkyboots bway +kine sthetic +ke sar +kathniel asap +kap an +june bug +ju mo +je tte +jack nicklaus +in ori +hou b +heigh tens +ha shed +gu ste +grand canyon +gonnam ake +gam bi +fer re +fa thering +em met +ein audi +e ue +e et +dol ma +den dy +de say +das i +cy prob +cross gates +consumer ist +cha it +calder one +cad man +bur bidge +bridgewater hall +boul ter +bol don +blue bonnets +ble ck +best en +bent wood +andre e +amar te +alexil alas +ai a +ĵ ľ +ðŁĺľ @ +ðŁĺĤ ðŁĺŃðŁĺŃ +å¸ Ĥ +ठı +uni dad +u dvar +tr ung +theblack keys +ten dai +ss lc +ss ense +sm ed +sir k +sim mba +shim mers +rei ster +reha bbed +razz matazz +q net +pomer ol +poly tech +petiti oners +nomean sno +new tech +napo let +my house +kö lsch +kron en +kare m +justin timberlake +jig ar +ink le +impact montreal +il on +ifw twa +gru el +giga watts +ger miston +gend armerie +gaik wad +ga thletics +flip the +fan tom +fal mer +face times +du rer +disp elled +char ming +camphalf blood +bron te +bou cheron +body shop +bak sh +at raffic +ar mors +alto ids +ag ged +aad hi +ðŁĺĤ ðŁĺĨ +ì§Ģ ìĪĺ +âĻ¬ âĻ¬ +à´ µ +wi ac +wall i +wak ker +w lr +twitch retwee +tuscar ora +trebu chet +the foster +the f +steam train +soci ologists +sleep apnea +sk fc +silsil abad +shev lin +sc us +salvad orian +rosan ne +roh rer +risqu é +reality check +rad ura +poie tic +po dr +pl itt +phi volcs +p side +ori o +or re +op ride +ome times +omarab dullah +nov an +notic ias +nis ar +naf sa +mis san +mirror ball +me sha +mar mion +long boarding +ken shi +kan sa +in land +hu ja +hot spurs +holo deck +hard wear +gungah lin +guj jar +fre mantle +en trusting +dehuman izing +cupca kke +cuernav aca +comra des +com odo +budd z +brew studs +big screen +beg ay +beec roft +bas kin +balt cops +b iches +b ary +ap li +amo yo +am m +ak n +agnyaath avaasi +agam em +a arey +yellow jacket +veloc ities +vaish no +unemp loy +tu tt +tru thor +tom ás +throck morton +theo tokos +te of +sy me +snu bbing +six er +sh enoy +senn heis +seas alt +sear ly +school spirit +ripp aul +ri sca +re collect +plant svszombies +pfe ffer +on thisday +o as +mcin tire +mam bazo +lon dons +lok handwala +learn spanish +kissmar c +kathryn bernardo +k lan +j mo +hust lin +ho pie +hil fe +go lead +fl anges +ficht ner +fi ac +en ham +dsl rs +deal with +david cameron +cu tion +cn nopinion +cep at +by women +bul mer +blood stock +bar ometric +atra iler +are te +ar cata +am pu +afun ny +wood winds +wheelof fortune +voter fraud +vier ge +ur uk +u mur +to al +thel ads +sou tien +snag gle +shap ely +sever son +perth now +pa zar +orgin al +net zer +my on +mu uu +mo ha +kriti kakar +konic amin +kir ana +kill init +kaha ani +jor gie +jen in +jay den +indescri bably +imperson ations +holly j +holla day +his sed +hell omag +he te +har do +germin ating +fu uuuu +fkat wigs +endodon tic +de asy +dad eland +cop co +con strain +buthele zi +bur ridge +bour ges +beli ze +bad ged +avi ator +alfon zo +ðŁijĢ # +ì¼ Ģ +z ial +yo tam +year anniversary +yaaaa ay +whis keys +v gk +trans genic +thou ston +tend ring +sun devils +su leyman +stac cato +sp lease +shaand aar +ro sell +riquel me +reg ale +redd ington +reck i +rainy days +pupp ym +pro ces +pop stars +patrick rothfuss +pas ar +p st +our vintagehour +ol ong +odem wing +nic ode +n ces +mu my +mixtape sapp +manufac turing +man citycouncil +m he +live mixtapesapp +lach man +kalin and +k cet +jo tter +itsnice that +ise au +is eng +in dot +imit ation +ill ac +hy ssop +ho ft +hi jau +handels blatt +grand al +glen nie +gin ko +flintri dge +eric colsen +ep t +endor sers +em po +demitas se +cra dles +comm ack +col abor +coal esce +clo che +chronic life +boy chuk +book silove +aver dict +ar stech +al olan +af ron +aerop uerto +ad air +ab staining +ðŁķ °ï¸ı +ãĤ¢ãĤ¤ãĤº ãĥ¯ãĥ³ +âľħ . +âĢ IJ +ysi dro +ye senia +wu erl +world championships +whit burn +wa hhh +vi van +vamp yr +uss strike +urban design +u zu +tro tta +timp ani +than di +th street +te ato +t kr +sli vesmatter +si vas +serv ici +school problems +rugby family +rin hatch +rie beeck +ri mm +pipel ine +pab io +p tu +or show +o very +nb sliveat +me si +matagor da +mas ji +mar gam +lali espos +khair pur +jer il +in consol +hu sted +happy land +ghu ll +ghol lywood +gameofthrones season +funnel ing +frat esmd +frank lloyd +flight line +fer rel +fe bu +en visage +en vie +de gc +ctr lfc +coffee and +cha a +brun ching +brand an +beth fratesmd +ben stokes +ben nies +bear paw +bal moral +bab son +as mir +aray ner +and alus +aggrav ation +ad hikari +abe che +a bey +z ong +will kommen +wa ju +vivi section +vaj ra +uf clondon +tin us +terra ssa +tas ca +tac onic +starwars battlefrontii +slá inte +semin yak +semb ilan +sask tel +saf ina +s max +ro iland +rashid khan +ram akrishnan +pat rollers +pat ras +parnas sus +park ash +northumb ria +nlbm prez +neck i +n clr +multi billion +mid vale +meag re +me har +mace wan +m gn +lap top +la donna +kill arney +ic sa +hq nigerianarmy +hide ously +her ber +hassel baink +fra k +experi encia +empren de +du opoly +die tician +damsel flies +cre asy +col unga +coast to +chum lee +capric ious +bun ds +bow don +bla que +bha vani +bal on +baham as +ary der +apple support +alexandri ava +ae th +[ âĺħ +ðŁĺħ . +ðŁĮ® ðŁĮ® +ðĿĹ® ðĿĹ +âħ ± +ÛĮ ÙĨ +zel ena +y he +withthe stars +whir ls +valken burg +uro web +uk pr +trous seau +transport govuk +tra ppings +toulou se +tool set +to the +the i +sug ababes +so der +silve stro +scott dixon +schom burg +sam pottorff +ruggi ero +promo ciones +poo p +poke ball +palmin teri +over brook +od dy +ny ala +nolen sville +nautan ki +mumbaic ityfc +milledge ville +marc bolan +lev ski +jy pe +iner gic +incongru ous +ilu stra +hu ela +hoo ghly +hel derberg +hall man +h po +glass of +ga ale +flower bed +finner ty +explo siveness +equil ateral +dusktill dawn +digi pak +co ining +chelms ford +case work +cam brai +borne an +bang abandhu +band as +att an +ast u +are em +appe asing +ë ij +Ã¥ r +y andy +x ies +u bb +thel augh +th wonder +te di +tab atha +sixth form +shi phop +santa ana +ru ination +reg gio +rd win +r po +quil ters +petro va +pant suit +oconom owoc +nay i +mh rd +metro card +mali faux +lampp osts +ko en +kixi fylife +kivanc tatlitug +king salman +kelly services +kam ik +k low +k humbu +ji h +ist van +hom g +heritages ite +here com +fil ers +din ka +commerz bank +cantab ile +brac eface +before christmas +b smith +aren yc +are nee +af h +^ )/ +: ,( +ãĤ¦ ãĤ +âľĶ âľĶ +z ira +y ha +what are +well sville +viking pride +vex ing +u criverside +tul ly +tran spen +sue de +sta z +samuel adamsbeer +sam mon +regre tful +plane te +pel angi +par due +p td +p oma +or msby +oli vas +nieu ws +metro pol +ligh ter +lesli eo +le ms +ko val +j bg +ip zig +ing well +ing reen +hrtech conf +ho ffs +hard to +haid agwaii +gon dwana +g tlds +flying lotus +fel sted +fannie gate +fab letics +f ce +do zy +data sheet +dark skin +cyber criminals +ci pr +ca es +bryanc allen +bin go +bhar athi +bab yyyyy +ali o +' ') +ðŁijĭ ðŁı¾ +ðŁİ¯ ðŁİ¯ +web sphere +tun ji +the google +su bie +stronger than +spart ans +sno gging +slan k +sing ton +sever o +sat ta +samanth abar +sam is +rollei flex +reef tank +real matt +ra zar +pit up +pho bos +over stuffed +nil giris +n ura +má r +mun dus +money lynch +mist ura +meri enda +mel fort +mccl urkin +loom ooloo +life force +le vie +jav afx +iti zed +island er +hon ig +ho twire +grac em +di pak +di ddle +con dones +clo aks +cj m +canti eri +box art +bis ous +bi spo +ber key +av ada +arrow smith +ard ingly +ali zee +aaa ahhh +.. âĻ¥ +ðŁĹ ¨ +ðŁĵ ij +íĤ¤ ìĬ¤ +ìķĦ ìĥĿìĿ¼ì¶ķíķĺíķ´ +zsl londonzoo +ym f +yin zer +y ba +way nesburg +warren ellis +w aba +vic odin +tir mid +tick lish +thewanted music +technic ality +tatter sall +strike back +steadi er +star awards +st pm +sp hi +so hl +slu mbers +rux pin +ros ic +repr ori +re capped +pur it +poly meric +our story +no pen +ni kol +nang is +nam jin +na es +museumo fcity +museumofcity ny +mrand mrss +mouth pieces +min young +me aly +marac ay +ly ss +lu tz +lo ew +karl thefog +is lets +is at +in corruptible +hin dle +hi mup +handhel ds +hair spray +gro zny +glo ssing +giam paolo +genius football +geburt stag +fron tale +f ard +eyn sham +ever itt +epi stle +enroll ments +durham cathedral +du di +dru d +di gan +dg ins +cor din +car rou +bri ve +black shear +bell man +band ori +b ami +author uproar +as cer +ar mie +ar bre +afer ro +( ´ +ðŁļ ¥ +ðŁĴ¥ðŁĴ¥ ðŁĴ¥ðŁĴ¥ +ðŁijį ðŁĺī +๠ī +vol kan +v spc +un gan +uk natarchives +thev creations +the bus +ta ec +sy dv +sp news +sp icier +sk z +sher mer +sel am +scott disick +sc app +rain bo +pube scent +prepper talk +prece dents +pm ts +ou mar +ol ona +nationalrail enq +nati vidad +me shes +mar kowitz +majer le +lyn as +li jiang +le brons +lay ton +lar yngitis +kav os +kansas city +jade veon +international nursesday +imit ators +he eeee +ha das +gre ch +freder ico +euroleague women +em phy +em den +e town +dr oning +dopen ess +de ters +dc am +ck ay +cil wx +cali frag +cal an +bram er +board walk +ble ase +beltr ami +bb king +bathroom design +bad ajo +aj ak +ðŁļ ľ +ðŁĺ» âĿ¤ï¸ı +wy g +white hawk +w ingham +vul pes +ucc shooting +tr ys +team k +tan gram +tab comau +t sung +super f +sub titling +stown fc +sig nac +servant leadership +pop cap +plat itudes +par ow +on ero +nup tial +ni hil +ner azzurri +nab u +na dias +makere reu +le sko +kri stan +kapilshar mashow +josh dallas +jackie evancho +in sincere +impati ens +hotel school +high society +hi ace +hannibal buress +ganse voort +gaming news +fri eden +fortnite br +footbal luga +flo s +find horn +fevertree mixers +er ate +epi phan +eng aa +elo tt +east van +ea die +e migrants +dri an +das kal +daguerreo type +cha ouen +brown x +betra yer +bestjo bever +bar cia +ath ene +at ter +arch bishop +ar mini +ap aul +an sonia +å¿ ĥ +âĸ ĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸ +z man +yu vas +wa aah +v max +ume Ã¥ +tru fant +tri une +thisi shar +thegood doctor +tennesse ans +stephen ie +sph x +sol are +smu cker +shal amar +sch eck +sah ni +run cie +pra yut +poly dor +petro glyph +pardue suzanne +ottaw apolice +occas sion +neurofibro matosis +n spa +n gl +mu cci +micro systems +men na +mel ton +matil de +ly ell +loo ter +lefth ander +la sker +ku ts +kri sk +kon ig +kin na +josh duhamel +jan aki +ine er +health is +gro pius +gi bt +gar nishes +g eli +fut taim +film review +feu dal +faw ning +enter com +dolgel lau +deep u +cor bett +cho wd +body image +be are +arch iti +ar gs +angel charlie +am bs +ain sle +advi sement +ðŁĵ· # +ðŁijİ ðŁı¼ +ìł ľ +Ùħتع Ùĩ +zid is +wow selfie +wee dman +wal wal +us afootball +uni kitty +un migration +toy shop +the kingof +ten on +staw ell +som l +sno bs +she kinah +robert plant +queri da +polici a +other ness +news bayarea +new sy +nau r +mosle ms +mi mis +mel son +master stroke +mac sports +leni han +kam ryn +je x +ino z +ice wine +homopho bes +hab ism +guru kul +gu oan +gren ville +gold son +gaz oo +galli gan +fig life +ffe ct +fair way +en bush +eck lace +ea sts +did it +dhar avi +def jam +cow shed +ci me +chum ley +chry st +ca q +bor do +bojack horseman +ben ice +beauti full +base point +bar rs +aw ny +attenu ation +amazing spiderman +ali ant +al ghero +abo ston +ãĤ ³ +zak im +y ali +wx yz +wo tw +winter son +wak at +vz w +up starts +ther ise +the da +teren ure +teb butt +tan vi +ta if +supramo lecular +super speedway +stati sm +spri ghtly +sony max +slur red +sil vered +sel way +seab iscuit +sav ored +sand blasting +sa ken +ro zen +rid dims +re fitted +pi ff +parent sday +p aga +ot x +or jay +not out +mol and +lyn sey +llan elli +kit imat +kak uma +k hol +jun ks +jason manford +is anti +ili stic +il aw +holodom or +hodge podge +histor ique +hack ath +ha zem +go ed +gi ffin +gi dit +gam mar +foxsport saus +fo ord +ev it +european lms +ent rain +emmy rossum +du hh +dodeca hedron +describe your +den ting +cotton mouth +climate reality +chu ma +chi mu +cer in +car amba +bra thay +bor mio +baa ack +ar nis +ap ut +ap enas +acreative dc +a stors +a ism +zoid berg +wild nout +white bait +wal tman +ve ster +ti jd +sing leness +sil char +sh anny +se du +screen cast +rad m +rach it +polit icom +po com +play stati +open world +oc chi +nzo gs +neuro degeneration +nay ar +nan uet +na esp +muir field +mis behavin +mel asma +mc nichols +mash burn +lock smiths +khajura ho +kha chat +kalin white +is ag +inhabit ant +il def +iam x +human right +her ning +hd ds +goo sen +gol borne +glen mor +git ex +g hassan +free music +f anno +el are +drun ner +day ak +cudd alore +cre agh +co libri +cloak and +can th +bur rill +brad dy +boute flika +boul le +bor den +bomber league +bo q +benton ite +bel cher +austy nzogs +af ed +abo ts +ðŁĶ IJ +ãĤ¹ ãĤ¯ +zak opane +x na +x brl +wy dd +whe atus +wam ba +us q +ub nt +type writers +trou w +thel ens +team vic +takeyour dogtoworkday +tak appa +swell endam +suriyafan sclub +sr pt +so ori +sneaker heads +sle man +shor i +shil pash +shape shifters +se ah +purpose tour +port cullis +p inf +over achievers +ori ano +on cle +nh f +ne stea +myrt les +mudi ay +mc ki +maso chi +lingen felter +kil lem +kier ong +khali q +key arena +kas par +ka bel +k rule +john d +impres ario +hor r +hijab i +glu es +glo c +gil lam +gi de +geo chemical +garden route +flo track +enjol ras +en able +dynat race +dro more +dia vel +de crypt +cochran e +co simo +chi desai +chelsea handler +bra shear +bo ke +bo burnham +bla der +bark box +ari ens +ah rc +ðŁĻıðŁı½ ðŁĻıðŁı½ðŁĻıðŁı½ +ðŁijı ðŁĺį +ðŁİ ¢ +ìĹ ĺ +ìĦ Ń +âĿ¤ï¸ı ðŁĻıðŁı» +v expert +underground wgn +tu ckers +tr n +thereal j +the poet +ter uel +ta fe +sport suk +sk ov +shu tup +science of +sc ra +rr b +pr gm +pic cy +pen rod +over nighter +o dio +niq qa +ng f +n ssa +mz katiecassidy +mt go +mo vi +melchi or +max xi +masa yoshi +marsh land +llandrin dod +ld schurch +lam ide +l cd +he ph +harmon ization +ha sek +gt bank +gren nan +glocken spiel +gemm ill +freelo ader +financi aleducation +f hir +explore wellcome +esc an +dou ce +din do +di visible +dal t +cu ne +cry tek +coy bib +champion sle +cel os +bi wa +beer us +bal das +at cq +ark ells +ai kens +ai b +accru ed +* ' +ðŁĴĶ ðŁĺ¢ +بصرÙĬ Ùĩ +yu mmo +yo gas +yar mul +wk n +wer tz +wer bung +val anci +upro xx +uk raina +tur bidity +tin n +subtle ties +stop tpp +slim mest +slau son +si swim +sh uru +rosann apan +rosannapan sino +ram une +raf bf +pog gio +pender gast +over powers +nur u +novorossi ya +middle brook +men slax +may belle +ma po +lust ful +lim pet +l lego +kat akana +kane ko +jonah marais +ho sk +go c +gal ar +fu ck +fol le +fistic uffs +femin ino +er hs +du sh +door knobs +dishap atani +cu mia +courtau ld +conclu sively +col ina +cav o +c agua +bi ondi +anan tapur +am ena +ak ito +ak ay +aj styles +âĿ¤ ðŁİī +عÙħر اÙĨ +youth sporttrust +w aded +villet n +tweedle dee +tu bi +top scorer +thei sen +the joshuatree +t mac +sustainable living +sp la +sni ff +sim son +sik ri +sav oia +robin meade +retrac ed +po kot +pleas ingly +photo credit +ph bo +pec tor +o hashi +newport county +mon eta +mcal eer +loo ve +lic he +lemon defr +lau ge +kre ps +keepit local +jon axx +jb gill +james bay +j anda +isab ell +hunder tw +ha ii +ha chiko +gran ja +go bear +geraint thomas +fil mp +far man +fa hri +ev am +don nel +do rey +dham ma +collar bones +ch p +bun co +bronze age +bi bl +bell wood +be elite +baseball cards +aven sis +autom ator +amin ah +a é +ॠĮ +x olo +wool acombe +we wont +von erich +vi eri +var ick +tho e +sou therland +some onet +skype classroom +shor n +sc p +ry dell +ros ner +roar loud +ro wh +ro mas +pueblo s +pro tom +percol ate +perce iving +open house +mers tham +matt gaetz +mand ar +makers gonnamake +lyn brook +kri el +jo tun +jamiemc murray +j kt +i pu +hav ai +harpercollin sin +gun awan +go canes +gar side +free spirit +fer ments +fe s +en bau +emul ates +elabor ately +do of +da king +da ithi +coz mo +cla ssen +caul field +bru cer +breakfast news +bor ge +blouin artinfo +bit ting +birn baum +ber ms +ban quette +bal con +back stretch +arre ola +andre ahor +am by +ad ham +!! âĿ¤ +! ðŁĺĤðŁĺĤðŁĺĤ +ðŁ¥ ¥ +å ĸ +zachary quinto +x q +wro xham +world vegan +weal den +w sp +un ripe +un gar +tom me +timeto change +the west +the vamps +that works +tart let +sur ratt +sum ire +ste vier +ste arman +stap ley +stal ag +scor chio +ry ders +premi ère +pff ft +p timal +oven timiglia +off rance +nu uk +nex tera +mu lia +missuni verse +mil oventimiglia +magnu ms +lu ta +leslieo dom +leslieodom jr +ker nersville +karan wahi +in progress +ij ssel +ic mp +huffpo stuk +hat ted +han eke +gandu je +fuku i +franz iska +fertili zing +far zana +dundee uni +dear man +de core +day for +constell ation +collo dion +c zy +bright line +body count +ber jaya +bat an +bad am +b dc +ardo yne +ap athi +ami des +amer ks +am ery +acos grove +aco at +ðŁĺļ ðŁĴķ +ðŁĹ ³ +ðŁİ ¿ +watch me +tt m +the ultimate +terri bles +sw t +super team +st ilo +so hi +sk gaming +sar no +sac cone +ri ffle +research highlight +r me +quad rat +poe tically +ple ad +pitch wars +photo blog +parti da +o zz +n ilo +min fo +micro wave +mak hani +lily dale +let as +kal ita +inter jec +in ic +ill ana +hu et +ho arse +green head +gesch ich +gaz an +fly ingh +evacu ates +enum claw +engal uru +ence ph +en ta +drawin goftheday +diso bey +dion ne +diocle tian +der yck +del ka +colon izing +colon ist +biglottery fund +be magenta +bakh sh +badajo z +ay en +avi ones +andrew j +al rdy +al esi +ab k +. ðŁĺ¢ +* ? +ÛĮ ا +и ÑĤ +win ches +vla de +use c +ugand ac +trans world +tommy chong +tele phone +swol ves +stan sbury +simp ang +shar pay +shar king +secure the +ryo kan +rosh hashanah +ro li +ram u +ra jah +pursu ant +phu ong +per ito +nulli fication +no limit +na aman +n nt +monday night +maz al +latch ford +kol lywood +klu g +jo j +jennifer morrison +iz aya +itv calendar +ihsa state +hyacin ths +hem lock +hel y +has ler +har deep +gol drush +fri d +fam er +fab ulo +ent ic +emar osa +citizen liveat +carre rac +bronze bomber +bran ko +book week +bi ggers +b ander +ay to +astoni shment +apur pose +aber gele +* ~* +ðŁĺĶ âĿ¤ï¸ı +z aira +young k +yl c +wid de +wet more +w tennis +vu i +un bounded +to ppo +there abouts +thename is +the eagle +tangany ika +susang komen +stra x +sp enders +smu lders +sefton hour +sa gh +proje to +prodi gies +prin ze +pr al +physi ques +per rett +pas and +pan k +op rano +of dreams +new z +my choice +mur ch +ma hela +lori moreno +legal ised +le sle +lagni appe +koval chuk +ko tc +keb bel +katharine mcphee +jer ash +jar ls +jail breaking +j edin +j cole +impro ver +immigr ating +hub bard +hi es +ham am +gu shed +great taste +gl ack +gender fluid +g chs +fun ches +freak ishly +fool s +folk ways +farru ko +en sen +em elie +elite book +el bowed +edge mont +du ss +cuck old +chig non +chi asson +cheek bone +cas sar +bor am +big narstie +ben in +bas com +ash dod +and dd +alli an +aga the +a thri +ÙĦبÙĨ اÙĨ +wal she +var us +vap enation +v ink +un pad +un inhabitable +th chat +swan lake +suren dra +studentath letes +sav ino +sar te +re assemble +q we +pigg ate +pel ts +nw t +na del +n re +min tage +min shew +michael is +mat anzas +masto don +lock jaw +lock a +little steven +lil t +la gu +kr m +kla gen +kierong illen +it sag +i read +hl g +ham bone +go tribe +giar dini +g achi +fran ds +fire fighter +film challenge +exer ting +eg bert +dg ate +cy co +cloud iness +clon don +claire mont +cdn screen +bu se +brue ghel +boss day +blan ket +bird sofinstagram +bam yan +back stage +ba ah +ayesha shroff +ay oung +arth as +ak hi +ad ab +abio tic +... ðŁĺİ +Ĥâĸ Ĥ +ðŁĴŀðŁĴŀ ðŁĴŀðŁĴŀ +ðŁĮ ªï¸ı +аР½ +ziggy marley +women shockey +wc b +waist coats +vasu dev +tre bko +ti ds +sunday morning +sui vez +side man +se sac +sch lichter +satt ler +sar uman +s ã +reha sh +public power +pu shover +pre wedding +po tro +pier pont +parag on +oval tine +newh ouse +new mar +ne wr +nax al +miz ner +mal lette +lough ran +long ish +kol in +jun gian +jah res +jac q +ig an +hant si +h vr +geof froy +gelatin ous +ess am +eno shima +engel brecht +el rond +ed ling +ec m +dam it +contradic ted +cocacol aco +chero keen +chaf er +buk we +broker ages +bleed purple +asi mo +anun naki +aly ce +advance dwarfare +! ðŁĺĨ +ðŁijįðŁijį ðŁijįðŁijį +ê¹Ģ íĺĦ +ç ¨ +व स +we an +upup cronulla +uof u +unite and +tol land +th wait +tg u +tatar stan +sweet bread +sou thee +son as +sehun day +se stri +sax es +sarg syan +rickast ley +re testing +re load +pu ce +prideof britain +place bo +phil ando +os ita +open houselondon +op ine +onthis datein +now ww +ne mo +na enae +n wark +mule soft +mer horn +marquette u +lo dh +leven e +khar kov +kent on +inten tionality +inc ited +il yushin +if cc +idiosyn cra +ger vin +gear vr +fit zy +fight cancer +fam agusta +even in +er usa +en ses +econ o +dem ps +creed moor +chap ala +bro r +bol anet +banar asi +aw ski +au v +atleti english +aj ni +ah oo +ad yar +ðŁĵ¹ : +y ces +v nc +ur ru +ty ce +thr an +swach hat +sve ta +stack overflow +sol an +sof lo +silver chair +sco ggins +sco ach +sal ps +run out +re sti +re paying +poe tess +pe tula +patin kin +pa sia +of b +mu ley +mast itis +mang khut +mand arin +man ch +mach ynlleth +lo bb +life coaching +lev eller +lemon ade +kur an +juan mata +jan us +j atin +inc ised +illi dan +il and +hold fast +his ses +har ith +gregar ious +ger stein +flyo vers +fire weed +fi et +fav ourably +exu ding +denis of +dan on +cu bit +con naughton +co hoes +cas sill +cafer acer +bat u +badger monday +auto week +ash wood +a ase +! ðŁıĢ +ᶠľ +za res +yom hashoah +wear side +valanci unas +ul man +ue facom +toshi o +to pher +tinker cad +the gill +ta ino +sunny days +sub cutaneous +sell er +scant ily +sc la +rot manschool +reynol dsburg +quest nutrition +pot stocks +photogra py +over water +onelove manchester +oler ance +neuro developmental +movi miento +mel ita +mccull in +mariamen ounos +manchester arena +man zil +ly th +link öping +laurel park +la dwp +ku kri +kaz ama +kac ie +is kra +ip sy +invul ner +hyper drive +holgor sen +hol lens +hof ner +heure ux +gree kislands +girl group +fy m +for du +fon go +ff b +femini zation +faroo q +equestri agirls +e hu +drive shaft +de by +dd yer +d kb +d company +d anda +cov ado +ci enci +chop stick +cat v +ca jas +blan kly +binaryo ptions +bake along +axstv fights +an antara +ama ury +am docs +ag om +adobe xd +ðŁĴ¯ âĿ¤ï¸ı +ðŁ¤¦ âĢįâĻĢï¸ı +âľ ¡ +yn ash +y ari +william hill +wb d +walk over +ve vey +u meda +touri ster +toulou sain +tol hurst +t gp +summer readingchallenge +su kab +stra vaganza +sk aven +simul acra +show band +scra pper +sand f +san teria +ran agar +ra ghi +quick en +pen wortham +pen ick +pe tya +out crops +nebra sketball +near shore +moor hen +mo ver +mil las +may uri +mat tos +manufac tory +lic ata +kay lam +k oos +joye use +in scru +ilove heartland +hubb le +gum road +gul lies +ghostinthe shell +g ape +fon dling +fla gon +feedyoura ddiction +eve rett +dot to +der on +das uki +cover all +cor regi +coleco vision +co wra +chocol a +cf pa +car goes +blu dgeon +bla z +belu gas +alvar omaldini +ack ers +ac entral +aby ab +a hearn +vas sal +uc cio +tur ki +tu scan +trump kim +ti thes +ti fs +tech women +taxic ab +supano va +scar brough +s jam +ro mario +progressi vism +pm live +play matters +pc bc +pac ar +pa vey +p go +olan rogers +ob il +national sandwichday +missing kids +mar onite +man preet +machiav elli +li ang +le sen +kul gam +kre feld +k wood +jan zen +jack rabbits +it bp +in corri +ican ada +hypnoti sed +h js +go war +gnar led +ganano que +fore warned +fol ding +feliz viernes +fc twente +fat f +f opp +exhor tation +eff zeh +do ddle +dhar m +des boro +da ina +d de +confe x +car rell +canap é +book shop +bl anda +bbc snooker +bb pilipinas +ball inas +back spin +au strade +am lo +am am +allevi ates +alam ed +al mos +age ha +ðŁĺľ . +ì ĩ +âĻ£ ï¸ı +வ à®° +z oli +yu mmmmm +y cle +worldbook dayuk +wigan council +visu alizes +van oss +ule ma +tro icki +trans ference +traffline mum +thugg ery +tan er +t shabalala +sym bian +susang ilbert +sma de +sh ach +sc aping +save a +sal bum +royal ty +rang an +plane tofthe +patri dge +past illes +palis ade +ori ole +oom fs +nick lachey +ni go +n to +n ira +moisturi zed +miz uno +mira bel +microsoft store +mal adies +magi x +low a +logan paul +liber te +l kg +jeff probst +is ong +intrac table +has brouck +fu schia +far o +fa it +eu karyotic +erne stine +elic its +croo ke +cra c +company culture +char issa +balasu bram +at risk +aborig ines +ðŁĺĽ ðŁĺĽ +ðŁĺĢ ðŁĺĢðŁĺĢðŁĺĢ +ðŁİĢ ðŁİĢ +âļ«ï¸ı ðŁĶµ +zz i +yab ba +wr r +wit ts +virul ence +vi rar +v gs +v ache +ts wag +tromb one +tiang ong +thestroke assoc +tham rin +take on +stones our +sa ki +root sports +rand al +r tn +pr ongs +picar die +paulo aken +pash ley +par ken +ocot illo +ny f +my top +mut ate +mon oun +means business +maun dy +mastersofthe universe +lp sc +lim bic +laurajane grace +kul bhushan +kam aal +io vine +indo chine +in fielders +in dent +i ball +hok kien +head on +hallmar kies +h so +h ile +green washing +genu in +ful fils +fire red +f xx +e tre +doones bury +dg k +de car +dahl gren +cle aves +carol an +bu stan +bri ms +bra zo +blue point +bad ging +avan shi +ar vi +ani bal +andreahor wath +amon th +áµ Ĵ +wwe fastlane +wis a +willing blam +wil lock +vu ong +vox el +vee phbo +ur bain +un tam +the ma +su chen +sin ge +seth green +se co +rumb elle +robu sto +ring tail +ri yaz +re program +re ep +re assessment +pn f +over charged +ol inda +o sea +noord wijk +n gb +msc actions +mey erson +mand ap +ku ster +innov ated +hy la +heat waves +hanson music +gri es +gol u +fro mel +fer menter +fau ght +fal ooda +f arias +er for +dra sh +disc ol +desi g +co aldale +cep tional +cen tex +cas sette +car naby +bun ter +be ton +all sop +al fano +afro pop +?? @ +winstar farm +wick ford +wh h +var ic +uwe bristol +un consciousness +trans boundary +toyo da +tar ap +sty ria +spru cing +showme your +sar az +sang ita +san ja +sam achar +sali do +ru chi +rs j +rhu mba +res q +quar rels +qual y +q on +q mul +pocket knife +petro vsk +pe ur +pauloaken fold +palmet to +ni w +n flying +mor anis +lun ging +loz enge +lauter brunnen +kur ung +kerma dec +j end +inspi ral +high worth +gul let +gi ev +gh k +en ki +doppelgang ers +does burg +dil jit +dardan elles +d anta +cur belo +comd tac +bo jo +basseth ound +bac ter +as col +alapp uzha +ðŁĴķ ðŁĺĤ +ðŁij¨âĢį ðŁĴ» +with modi +wel born +vol terra +vi reo +un restored +u kem +twin n +tweetur biz +tr ounced +torre molinos +toronto pearson +tanger ines +sy dow +super chunk +stal inist +slat on +skin heads +shankar acharya +sar panch +sa be +s style +ry ar +ry ann +roo ki +r ll +q br +pure magic +pan go +p ams +over landing +or ka +opo ku +od den +migrant crisis +meg ann +me the +mck ernan +mac kem +log ism +lat rice +la hood +kings lynn +khu shi +ke mps +kac ang +k alli +ir lande +hor witz +harri smith +greek week +great place +gi psy +fu zhou +frank ish +field fare +fan shawe +en yt +don ati +di ously +cine t +chico state +car us +car nell +campan ula +breast milk +blood cancer +bhi du +beer pong +ayck bourn +arkell smusic +am boise +al fi +ae on +adu blin +accentu ated +ab ama +aaron hernandez +ðŁĴķ ðŁijĮ +ðŁij¯ ðŁĴķ +Ê Ĵ +yal c +woman ly +wit e +wel sham +vital ity +visit philly +vegas con +us an +tune z +trump now +tool ate +to bie +thru shes +the henryford +te esta +tanehisico ates +taik await +taikawait iti +steam boat +star less +spic iness +sk oll +sin siders +sem powerment +schi ppers +sam yuk +rump el +rhin eland +ren aldo +relap sed +raman singh +psycho geography +propag ated +prince ville +por osity +photom eter +pa cha +oldham hour +o eln +mumbai indians +monday funday +mikha el +micro dermabrasion +megastar chiranjeevi +mat ara +lo sey +lith onia +li em +kon oha +kim bra +kid min +kalinand myles +jer kins +jc vd +jayant sinha +ja ish +hun tel +house bound +her i +green thumb +gor an +gillian anderson +gang nam +fortu neteller +fie bre +f wi +em mas +dri vable +dress shoes +dou ches +diabo lik +cool katz +comrades race +class work +cdn film +bit bucket +be chamel +bb tv +baltimore police +ash g +arin dam +ar ul +ap sley +al sager +ais linn +acan al +? ðŁĺĬ +; # +ðŁ¤© ðŁ¤© +âĿ¤ ðŁIJ¶ +z edge +y aks +winter soldier +who you +wha thappen +vill ans +usu i +twit ta +twin kle +too m +ther midor +tex p +ste o +sj b +sher ine +sas campaigns +san er +ro mar +red minote +plat on +pet supplies +pay g +ou de +or omo +motor co +mind tree +mi ec +lon do +leon hardt +l kr +kirkintil loch +kh ouri +kbc channel +kar ima +ka ina +k dwb +justin rose +juli ani +jeff merkley +itu esday +ip ers +ie g +hyper v +hom ep +hit theroad +hi el +ham burg +gre p +flit wick +e ula +den nings +cow fish +cos ine +colton lhaynes +clen ching +ch ö +c tic +bre aze +brad leys +book smart +blood wise +ble del +bel sen +bc wildfire +aw ad +arstech nica +arashi yama +am official +am ca +a os +ðŁij¨ ðŁı¾âĢį +å°ij å¹´ +winkle man +wig go +vou ge +us kies +ul p +ton ym +tg cf +team rwb +ta iler +syru py +sun seeker +sports line +spiritu alized +ski ffle +si ds +sham im +se bald +sar ris +ru fio +romb lon +righte ously +rebel wilson +railway museum +r wang +ore x +on ti +notre dame +ne z +na shoba +moo y +mis sr +micro prompt +manhattan ville +malari aday +mac adam +luang wa +lot tery +iwant to +incapac itated +im ber +ilove mcr +iam saidharamtej +hin oday +her u +gg in +gar ver +fumi gation +foxsports west +em powered +dr ms +domin ick +den es +de safi +corin thi +conversation uk +calori fic +barley corn +ar mag +any time +allo saurus +alder grove +accoutre ments +abdul la +Ĩ ãĤ£ +ðŁĴĸðŁĴĸ ðŁĴĸðŁĴĸðŁĴĸ +ðŁĴ¯ @ +âŀ¡ï¸ı # +you dont +ye sha +y apa +wing nuts +viv int +v ted +un assail +thursday morning +the athiyashetty +sous vide +sin cil +sch ramm +sam witwer +sahi ba +sacrilegi ous +rin na +reti ef +reeper bahn +red currant +real ddp +por ate +popu lists +passi flora +oil cloth +ohio stathletics +ny ang +noor ani +nikkie tutorials +new ells +nat ak +mss oci +mi rip +metal fest +meigh an +meet inghouse +mar row +magne tized +lucy slaw +loo sens +lin tel +le sar +jon snow +jol son +jenni rivera +hand forth +game book +g bb +ex on +erock star +ent soc +elek tro +ek azarian +e ikon +dra zen +de at +dat to +d hin +cu pids +craft sy +chel a +breaking views +avon mouth +ati x +animal testing +aki shore +ad din +. } ++ ' +! ðŁijĩ +ðŁĴ ¼ +ðŁįį ðŁįį +ðŁįĥ ðŁįĥ +ðŁ¦ į +° âĢ¢ +z ele +ys p +wh ata +we sanderson +wan stead +wai the +w tr +universityof ga +un ting +u ren +ton gued +thereal buzz +tfl tph +ste ger +stanley cupfinal +sof rench +sl s +sick ert +she sha +sas an +sam plers +safe keeping +reichen bach +pal z +outh africa +oneless gun +ober yn +nation of +micror na +mat on +man ig +ma hoo +leach man +kie hl +keween aw +is db +inter locu +i spy +hor ten +hay les +gujar ate +go old +glass boro +ger not +ga tha +fi de +f wf +exal tation +erri gal +ei ko +der ain +dep ablo +d hat +cuid ado +cb ce +bur naby +bir stall +be vac +aun ch +aujour d +au sage +at tics +at kina +ar nataka +amaz ulu +al melo +al bic +wat cher +v ha +un problematic +trevor row +to kaido +sw akop +sportsc ast +so dus +slow ness +simon stown +sever a +sab io +ru hi +roun drock +ri do +rally mexico +qaw wali +ple ttenberg +pan esar +os goode +op chemtrails +nik ole +nicol aus +mu stering +monte reya +liz beth +lad son +kir una +kir ko +kenya airways +kaw ase +ka hl +k achi +jet set +j olo +izu eta +hu atulco +he yyyyy +has bro +girls nightout +ga stel +fu oris +fromthe vault +devou rer +del tat +de hydrate +day lighting +dann yo +competition time +chim ay +cantile vered +british f +boilerroom tv +bo ers +bestof british +bal sall +b mtc +az one +aw ami +avin ash +as sin +adap to +accompan iments +aa os +ðŁ¤ µ +æī ĭ +åĨĻ羣æĴ® ãģ£ +ãģ¦ ãģĦ +âŀĸ âŀĸ +Ñ į +yorkshire is +yo gab +x dddd +water slides +wad den +umb b +ther ion +syrian army +swin k +stra yer +stop yulin +slam miversary +skid row +skan ska +shopping online +shivu adda +sbli i +sanit arium +ru ess +rr g +river run +ril las +quadri plegic +pin nick +peace time +olive tti +nak ayama +m gn +li vand +kla assen +kati ele +jung lee +jumb ura +jay sean +ja en +i shin +ha ina +ha akon +gri f +fru gi +fr g +for son +for agers +esco bedo +en derby +dou bler +do bara +cry an +cor covado +cdn olympicteam +bibli a +bhar adwaj +bell tower +ay na +auti sta +, * +ðŁ¤Ļ ðŁı½ +ãĤ į +zz er +yam hill +ver sion +vande weghe +ul c +to rero +su its +street team +sho ki +severe wx +rome os +ro opa +reclai med +ph are +op ic +obam af +montereya q +megat on +mc wfc +mari adb +lu fc +labor ing +ko za +ko iv +kang nam +john paul +irfan pathan +intangi bles +imou to +i stand +home place +ho wards +halle berry +gregori us +get chu +fx cm +flo gger +fer rers +fair hurst +esk er +efra ser +diamond jubilee +de ora +cover tly +co perez +christian sburg +chri sette +ch é +carri eh +caram anga +cam illus +bur gon +bread crumb +bre izh +bbc goodfood +ask for +as wad +ap jabdulkalam +antag onistic +am jad +al mam +ak ande +adink ra +ac triz +ðŁĶµ âļ«ï¸ı +ðŁĴĻ ðŁĸ¤ +ðŁĴªðŁı¾ ðŁĴªðŁı¾ +ç ĥ +Ë Ī +é xico +ze bre +wante duk +tw oods +trivi aday +tox teth +tid dies +thu la +theofficial sbi +then i +the free +templ o +tas ers +tan f +south jersey +sou suke +sla ine +sea bees +saturday morning +ru gg +reister stown +q aim +pu jol +plant ation +pil key +physio therapists +philli pa +pashtun longmarch +par ly +ovi Äĩ +our tney +op tus +n aging +my day +multi sensory +mplo yee +mon dal +mcke chnie +lax ative +lady podsquad +kyo ku +kidney cancer +kick ing +ke iran +jeep er +je wl +jay la +iot security +influ ence +indiana fever +ij muiden +hypno therapist +hali za +graff ito +fu gu +fiji ans +exter n +ed gier +e igen +dumb ed +dick er +dae won +co housing +chab uri +bo gg +blackand gold +bal azs +ay re +av itch +au bert +angel arayner +ag nez +a ok +ç© º +ãģķ ãģı +âĨ ij +à¹ĢภĻ +zan ella +wl ky +well and +weal thier +under coat +u tin +trad able +ta pah +stra hd +sl veng +si ria +shave club +sce les +sal mo +robert glasper +rcar mitage +rak ha +ra van +pro drive +pla sma +phi sh +p eller +outside magazine +or cutt +on ard +omen i +odhi ambo +oak ham +o ai +nikola os +n music +motor coach +mccas key +macin nes +little finger +lat asha +kot ka +jo ep +jar ah +j du +iw p +ite sh +is mat +idar thritis +holli day +hal verson +ha vard +guil derland +ge ils +g vb +g sathletics +fung icides +fu mero +for pa +elling son +dor mancy +don of +dis banding +dige ster +day parade +char lam +capit alizes +cane gra +bu blé +br ingh +bi sexuals +bein ecke +bal an +bab angi +av h +august ines +ascend ancy +anishin aabe +amar ula +al able +absur dist +; ____ +ĥ ä¹ +ðŁĩ¦ðŁĩ ± +å¥ ³ +à« į +zz z +yog ya +widde combe +war i +vol ve +ul rike +tro twood +the greatescape +tha ad +tc pa +stay classy +sr il +sp hila +san abria +sab at +ry m +roberto cavalli +road shows +rep eller +railroad ing +pu ds +perme ate +penn statem +pedra za +pas sing +p nb +or nis +ny gv +nie w +mt lv +mk don +med aglia +mc beth +mc allen +lo tr +lincoln ton +lill is +laser cut +language day +ki ght +k attan +joseph son +james mcavoy +inter species +instal ments +i just +hof meyr +hite sh +het tie +he don +gorsein on +geta way +fr üh +fle dermaus +fin icky +fero pol +faber gé +f bg +excit ingly +etu des +enlar ging +el win +dun ster +de stre +de camp +dave matthew +crest line +chat win +car cross +cam bu +bree z +bo sun +b ja +aw acs +av chd +army day +ar uh +anne ke +zen do +xen arevival +wi thern +wft da +view tiful +underthe dome +tram ping +time sheets +talis ay +sycam ore +supportn wt +super villains +star gell +soul fly +so j +slow food +sig machi +sand co +salon du +sal lies +sak shi +roy ster +ri skier +re format +pau ll +pascu al +ore imo +n mm +mo ssel +mo ate +meteor garden +magne tically +mach ismo +llan gef +jer wood +jef fro +ignaz io +hyper plasia +ho ko +har n +hai den +gu ten +ge gen +gau k +forth right +foreclo sures +fin alizes +exempli fying +ep onine +elle tti +eleu thera +du ch +disaster recovery +des don +delici ou +debre cen +cool angatta +colle ton +cla sped +cit ilink +chil eno +che halis +calder cup +byd go +bus se +bonny ville +bodn ar +bifur cation +bestfandom ca +ben ko +ba qi +ay im +agamem non +.... * +... ~ +! ??? +æŃ Į +âĿ¤ï¸ı ðŁıĪ +vi ers +uz alo +uk houseoflords +tillot son +theak ston +tagbil aran +stabili zes +so de +sne deker +ski les +shan er +sen ergy +sel fy +sch ar +sal army +robusto babe +rc ade +pic ador +pear cy +pan ay +opend ays +oli vi +ntv weekendedition +ne gras +ne agle +mu cca +moneti zed +lu pino +lick in +kathy ireland +ja afar +incen sed +hail wood +great cause +goldengate bridge +gold farb +goal setting +ghost recon +ga irport +flori dal +fli ppen +fi she +far ra +en di +di staff +dah y +cri bb +cre edon +con sin +col men +co sponsored +cin donesia +brow nie +born tobe +bo gard +biffy clyro +bella vista +ba wn +aw s +alexand ru +ac opter +ac ces +aberystwy th +. ðŁĺħ +ðŁĺĤ ðŁİī +ðŁijĮ ðŁı¿ +ye eeee +yaman aka +yakin iku +weak ling +wan ji +tuss is +timeto play +sull inger +str un +sp urge +soun dary +sor te +si deb +sau ber +red day +re dy +ra che +prote c +privateer press +per lin +per ic +p shs +ode h +nbab day +mul grave +mp c +modul ated +mis steen +michi o +mevag issey +met u +mantic ore +lus combe +li vio +l bo +king smen +jj c +ichi ba +hod ler +hit less +gos set +g pk +fck oeln +fangir led +f ons +eich ler +eaz y +east vale +der ful +dau er +compos itional +cat kins +calli graphic +boy ard +bon aventura +biop ics +be such +bbcle icester +bbcal ba +av ina +alu zon +al ind +ak ry +a stringent +!! * +ðŁijĬðŁı» ðŁijĬðŁı» +ðŁİģ ðŁİĤ +ðŁİ ¡ +à² Ĺ +Ù ł +ye omans +wx ii +wo te +wi tho +wh are +vod acom +verif one +v dv +tsun amis +trav ell +trade off +tool room +stori esof +sp icc +son yes +shoed azzle +shi hab +schomburg center +sai ful +ron ni +roarloud travel +ring let +red in +rap ace +ram es +quar re +plac emat +pi gott +north jersey +ne emo +mor tons +mis direction +mick le +mi j +lead theway +le os +le mo +jitendra singh +j mp +ici ón +iam rana +i won +heel ers +heart lands +ha thi +gr ps +go griz +giuse ppe +giam battista +gener gy +ge do +g pe +eth icist +dra upadi +deleg ating +de growth +d bag +cze chs +comp toir +charle sm +bur chfield +bne i +biza sia +be ready +bds dragonflies +asli yoyo +ari ver +ar ba +appalachian trail +all hail +alge ciras +week lies +water boy +va ez +til man +thomp kins +thene therlands +su en +stalac tites +specul ates +so di +snu ffed +she reen +scotthall nwo +ri sto +ren ly +por ro +polic eug +plasen cia +odd fellows +mount joy +mo sier +manil aluzon +magen nis +ma ak +leg as +la za +katy isd +kam and +kam ali +jo key +jim ene +its no +inst illation +ideo logically +i aw +i ao +hy ams +hu berman +home wrecker +gold field +g sofa +fu or +fou z +film fareawards +fer ber +enni um +e marketer +disgu stingly +desig ned +democrati ze +cro agh +chett inad +chain ring +ce ara +candice kp +brain cancer +boom bap +bon ino +bo zak +bel more +awesome st +ad cc +aas tha +: "" +âĹ Ĩ +zepp ole +yogi babu +wide band +whoo hoo +warm ington +voc mnews +ultra sounds +twi zy +tran che +tic h +then igh +the family +t gom +sy rups +ster ns +sinu ous +shingekinok yojin +scher merhorn +ronal dre +rock s +range ela +ram il +politicom ag +pitch atpalace +ot lfp +os rs +ol dd +ok tar +ny strom +nat or +nasti ali +mis spelt +mea ford +man asi +makers mark +mahar ajas +la ddu +kir ri +ken nelly +jj author +ishqba az +inherit ors +ic fp +huntel aar +hindu rajyam +gre te +giff nock +g nu +g audio +fresno state +flori ana +fan fan +du ro +donagha dee +di bru +deb namcarey +dal at +cros scu +contu sion +commissi ons +clu cking +cimo relli +ch awal +cat sare +cas set +burun dian +burn age +brick laying +brad thor +be holden +back to +awild life +anarch ic +al ag +ab ank +a ica +ðŁĻıðŁı¾ ðŁĻıðŁı¾ðŁĻıðŁı¾ +ðŁĺ²ðŁĺ² ðŁĺ² +ðŁIJ¯ ðŁIJ¯ +ì¤ ij +âĢ ² +á rio +y stery +william devry +werder bremen +vk xip +tyran n +tren ching +tip sters +syn nara +sw right +suppre ssive +star liner +solu bility +site c +shaw nat +sardan arohit +sar kis +rene eyoung +r ÃŃo +pu jas +psycho tropic +pss sst +providen cia +pl ss +petr illo +per cen +pc cs +park town +pad ano +pachy derm +onceupon awine +natu rist +nak ama +naf s +my ki +marma duke +mait land +lu ba +letsgo peay +lefthander sday +laz lo +lave zzi +ko taro +kit z +k nt +jäger meister +joss whedon +imperson ates +haj jar +gor ving +gen au +fu to +five star +emerson college +ea org +diste mper +dau ph +cro cks +cri spy +ch ome +ce du +car vey +bo vet +bluemo on +big issue +bab oo +b hang +arche ology +ar ayana +apprais ers +ac op +ðŁĵ ® +ðŁ¤§ ðŁ¤§ +âŀ ¥ +áħ ł +wy oming +water view +war ps +vivo v +video editing +ven ceremos +us yk +urgent podr +u sia +tre stman +tb harat +sun ds +stra der +soh na +smo vie +situ ation +sim feropol +shan er +sh ying +seeyou in +se gar +se cker +roo yen +ron chi +road trippin +ren ounced +ren ji +quie ren +queensc liff +propagandi sts +pres sclub +pp opro +pitt ston +pav a +nemac olin +natu relle +mil ou +mil ani +ment alism +med star +me sni +mat tress +man ahan +lu pul +lingon berry +lewi showes +lar ga +la el +la bia +l rn +l hb +ke ce +kar is +ka ad +holac racy +hol mberg +gur t +go pe +gaz illion +gael tacht +fu tari +fo ca +flatbush zombies +fak ta +emo ji +ed by +dy dd +danadel any +cw ts +clothe spin +chop da +cbs allaccess +ca ins +c fx +bron wen +bm wx +blood letting +bilet nikoff +bike month +back tracking +artag nan +af as +yil dirim +y pf +wilke sboro +ve f +v awg +uk la +tri phop +ther itage +thar an +tem u +steno grapher +ste mple +special forces +son go +so gon +slo v +satthe table +ru ddin +rodri gue +rig sby +quint en +pro av +prize winner +pre o +pe ppe +paren thesis +onna is +one gro +on sie +omot ola +o gm +new berry +ne vil +nak ashima +n ja +mu tour +mid mowx +mic on +mic kie +mer se +menom onie +ko bus +kei sel +kaley cuoco +jointhe movement +jam fest +illi beral +hut cheson +hi ston +hazel tine +ha o +gu eu +grun wald +grig sby +gre sik +gel atine +gaale inster +every things +don ley +deten tions +davematthew sbnd +ct cs +craft speople +counting crows +connec ted +conjun ct +clinton foundation +city jet +chesapeake bay +chatter ton +car ita +can ine +bur ress +bts b +boundby blue +bbcra dmac +bas sel +bariatric surgery +ban ya +bad ou +b wp +al ara +ak ata +abduc tor +== >> +................ .... +% * +ÛĮ ÙĪ +yuv raaj +your san +world champ +wood ham +wescra ven +vin do +upri ver +tom ah +thoothu kudi +swap nil +strepto coccus +staf froom +salv ator +roof line +rel aciones +re land +pre zi +pon ton +per las +paul feig +of ac +oc elli +national bookloversday +nar alokesh +muslimb rotherhood +mull er +mu zic +monk man +manit oba +manico tti +love dit +lma ooooooo +lam ang +lac to +ker nel +k ti +intro version +i fan +gr é +gar lick +france sc +fe rens +famer tr +ec al +drown ing +d fr +cub ical +cre ak +couple t +cor b +co cho +christy clark +ce w +ce sium +c mag +buzz y +blan chette +bar que +aur at +ath ene +arri vent +arpeg gio +ang eni +ag akhan +a herne +zar beaz +wine festival +wh in +wasimak ram +waf ina +w aga +vas ai +uu h +uk ri +tra si +ton ik +th impact +syny ster +sun wolves +sri shti +smu ir +she hu +riski est +re ddin +r gd +pun it +pre ta +power pack +pol loi +pitt con +o ddle +nj morningshow +mersey side +me cc +mcelhin ney +mccar thys +market share +makeup addict +ma ula +m mos +line arity +likefor follow +kings down +ker sten +juni us +israeli pm +iah sbkb +i hub +hu sayn +hey bridge +freshoffthe boat +fra sc +faz enda +fair lie +eff i +earn shaw +eag let +duncan james +dar ton +daily quote +coo kies +ce sc +capric cio +bur s +brum pic +bie hn +battlec ry +ayrton senna +aw on +are staurant +ì ¸ +È Ļ +| âĹ +yearswith ourhomebts +xaver ian +x oom +will ington +villi ans +unassail able +un polished +u wl +track work +town usa +thelife of +the whl +the dragon +tandon raveena +t sca +sweet grass +super califrag +stabil ised +sle dder +sin ing +sin del +seis mic +seasons greetings +se futbol +sch ild +sac nas +sa ka +rohat gi +rin con +q con +pu bes +po len +per tussis +par va +orche stre +nun ney +nowh ow +ni en +nel sen +ne pom +myco bacterium +moto g +m fk +louise mensch +lan ao +kan ame +k caf +juli ssa +jose fina +j sl +ish tonka +is san +inton ation +inter group +hul bert +hou gh +hales worth +gu sti +galway hour +fre res +fag g +fa hrt +endor phin +empe zar +dad agioia +colon izers +chill ers +carrieh fletcher +car s +cali dad +brand ambassador +bear man +band anna +aw aking +austin and +assu redly +ari shaffir +analge sia +ali qui +albert dock +aizaw l +adju dged +act fl +ab sac +zit ao +zel man +ye hi +yar die +yak ov +wedd ington +wa thletics +vacu ous +v lo +use f +un labeled +un gi +ti ens +the pug +steadfast ness +star shine +son burg +soco tra +sh ays +sch mi +rencont res +rec com +property news +pret ence +post news +per roni +par que +orphan ages +nh ler +nastiali ukin +muk ta +mele hill +mee gan +md ga +mable thorpe +ll u +lator re +ky renia +ko smo +knock ers +jo bin +je melehill +hom mie +history inpics +having fun +haber sham +gon dol +gla ad +gab er +espn radio +e mon +dol lies +dhar mesh +cote divoire +coo puk +compen satory +commerci alize +berlin wall +be guiled +aper ri +alt itude +ðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃ +ðŁIJ ¡ +ÙĦ ÙĬ +yorkshire man +wk f +tu ku +thir um +tal yl +stri pers +sto ren +spiritu alist +selfies for +rum son +roano ke +reneeyoung wwe +recei ver +q or +pro petrovsk +phone book +p sm +over dosed +ou twood +oli vine +now next +moon star +mol dav +mma junkie +mi ming +man ito +man arola +leslie grace +kill joy +kash an +jon taffer +jack er +inst ag +improvis ers +hun te +gla ze +froma bove +floor board +ethe kwini +ecoun try +du mit +diaz jr +d orie +cro om +cooper age +coo ing +con oce +compart ment +bud leigh +boo throyd +bio feedback +bi ella +b ace +anti semite +an shu +album covers +al mo +ah hhhhhhhh +ag awam +af low +îIJ ij +âľĪ âľĪ +à¹ģภĶ +zu cca +wwi ii +vamp ira +up vote +tobac con +this couldbe +tenn ant +team priyanka +tarheel football +swakop mund +shi zuka +sar oj +rou ses +ran jeet +quick time +preci ousness +photo gra +pedest als +p foa +oo ool +on ghwa +o toy +newton ma +na sher +man gini +lith ops +lef thand +kur u +kot tai +kat sura +ju hl +jaqu eline +j ll +j anner +intra preneurship +hu ebner +hor loge +her zl +hein en +ha vers +gro ms +grace and +gr ze +gh hhh +gar ner +every onec +eli er +dr ington +dining room +deple te +de ul +cr ace +cou g +cont our +castell on +brit omart +bott arga +belle view +assal amu +and ras +all red +agar ic +abraham son +< $ +ðŁļ Ĩ +ðŁĴĭðŁĴĭ ðŁĴĭðŁĴĭ +âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı +⾨ ðŁĺį +yomi uri +yo k +y barra +wall kill +tour series +thu mp +the tech +the cut +tem pel +te ct +sund berg +stat cast +star ship +slee plessness +shmu el +scor ps +sat u +roes elare +re ps +re ger +power team +politici zing +po ema +peer review +pang bourne +ok anagan +nz rl +ne ils +mun ni +muff lers +man handled +luther ans +learning isfun +kur ang +kron k +kre ider +kar ad +jet ties +iz abella +ith u +islam abad +irrit ability +holl inger +hob sons +hmv tweets +hel ge +glad win +gc ms +ful ks +fol som +fever ishly +facto ids +du tifully +draf tees +divisi veness +dis ley +del barton +de cen +contro le +ci mb +ch go +br injal +bil berry +bab ad +ar yl +am oral +am j +al tice +agne tha +ag re +; âĢ¦ +. ðŁĻĮ +ðŁĺ© ðŁĴķ +é m +zam bo +women for +wa jid +vap iano +vac ca +un witting +ultra vox +tra f +the movement +te kapo +te gal +te ats +tar quin +sweet pea +super duper +stam m +spider webs +somi asis +so fus +sh wed +ser ah +scre a +scal per +rei ffel +princi pe +pag et +osi jek +om c +official melb +nu x +no hate +nerd land +ne opolitan +nc dc +montele one +min ks +marin ate +lumb ini +lom ba +live science +len iro +ky ys +invisible illness +impedi ments +hy pes +ho ony +hin kes +hair net +ha yer +free thinkers +fa ena +exege sis +doom metal +dic amillo +de compressing +dal er +commissions open +colour ful +clu cas +clayne crawford +ckin chen +chu cho +chin aman +chatur bate +ch itchat +ch allen +center stage +cas amigos +caroten oids +bon da +bigh ouse +ball state +bag el +bac trian +awe igh +arizon a +anc alerts +ain ting +adi es +acce sories +ðŁĴĽ @ +ðŁıĿ ï¸ı +ðŁİĤ ðŁİĪðŁİī +ìļ°ì£¼ ìĨĮëħĢ +éĿ Ĵ +âĸ Ħ +women kickass +water s +walker ton +ventil ate +trou ty +tau rasi +su mitra +stry cova +sla den +skor janec +ship builders +ro ble +rheumato idarthritis +presu ming +pre search +pot vin +po wr +ow ine +otto bre +olds mar +o amaru +miér coles +marav illa +manju shri +lori keets +logan ville +le ben +kron a +k ls +j th +icar agua +humm el +horn swoggle +harish kalyan +hamil tons +ha pus +gu ter +gregg sulkin +gel at +gal las +g cd +frustrat ingly +fol ly +feas ance +evic z +dipp goi +devop sdays +defence minindia +con sternation +con gi +christmas giftideas +chint z +brooklyn bowl +bran islav +blo ts +bir man +bice ster +be go +bayani han +atharva amurali +al vv +afree dom +ab aesq +a jam +ðŁĺįðŁĺį # +ðŁĴ º +ðŁĩºðŁĩ¸ , +âĺºï¸ı @ +y ig +whit ened +v se +un conquered +turkey day +ter je +tal as +t mo +sw apo +su li +step brother +special edition +scot to +rour kela +roodepo ort +roh mer +ro mag +remodel led +rang pur +posto u +petro lia +pet one +pa wl +ny ayo +nu jab +nec co +name plates +mukher ji +monsanto co +mh or +maxim illian +leic am +kyle petty +jaz min +ive agh +intellectu alism +il ka +hi mer +hawkeye football +har di +happy hanukkah +happ ppy +gi mignano +gator sfb +gar ton +gar ni +g xp +far nell +fad ers +enrol ments +ene o +do ak +dd yn +coqui halla +conver sely +colla ged +chri sr +ch acko +best actress +be mpton +bar tering +awk wafina +at kinson +ambas sac +ama ia +alar mist +ak ela +abbey dale +ðŁĻ į +ðŁĺ² ðŁĺ² +ì³ IJ +y kid +x tin +x eni +woo ooooo +was sen +ut ch +the josh +tar af +tab ac +ta sik +ta homa +star com +sk k +sh ema +seri alized +scandin avian +sc primary +sai do +s green +roun tree +ros ler +project car +paw son +pat co +panch al +ofex cellence +new writing +morninge dition +mom preneur +mlb fancave +mis step +mc naught +mar ckinchen +man crush +mad ine +macer ated +lec tionary +la ffer +kunal nayyar +korean food +ko sa +kang en +k na +jo ppa +iscar iot +houston tx +hard well +gorkhal and +gig nac +gai waterhouse +g ittin +fr w +er langer +episco pal +dpan abaker +dolce tto +der bi +danielle jonas +da official +char laine +ch iso +cat sin +canadian art +caf od +brack nell +blow n +bla sko +bivou ac +bb crb +ari a +arche age +ak c +ait c +z big +xy lem +wi wt +whiteri bbon +wha kat +web zine +votethe wanteduk +visit california +un gen +turi sts +tre o +tobacco day +the women +the hub +stjohn s +south down +som thing +sl one +sk m +sam et +rick mercer +rayn ham +pronounce able +prison planet +photo journalists +p nu +over played +op is +nw sc +newmusic monday +nbs finds +much hh +mess am +mel ky +mac cas +ly r +love reading +ling am +l ence +kirk bride +kal ai +k pix +iso sceles +iron ton +ha ggling +ha ash +gur meet +grand fathered +glori ae +gad gets +express ly +dust pan +dragme down +ding bat +d ila +d drfc +crim mins +con gas +con founded +co bal +ch asseur +c sula +c suite +better late +av lon +av ine +alpac ino +all music +. âĺºï¸ı +! âļ¡ï¸ı +! ") +ðŁij¨âĢį ðŁİ¨ +ðŁį § +ëłĪ ìĿ´ +yo gas +vel in +tor rance +ti ranga +thegill sfc +team fiji +t she +sou ci +sk oy +singh vi +se ga +sad tweet +rose berry +rob ing +r tu +prote ome +petro grad +oke h +obfusc ation +ns v +nor they +ne phi +nar din +monoun saturated +mono graphs +mon stax +minig ames +mini fig +mcg ough +marketing profs +mac ys +l md +ku mba +kot ton +ker ang +kel sang +kadam pa +jr l +jon ker +jedin ak +jag gers +initi ator +haul ers +harshad chopda +hann er +grims ley +gr on +gl ickman +get te +gapp ed +free gaza +fox catcher +fin neg +f ylde +excell ent +dol orosa +dic ally +demago gue +d aga +curve leicester +cud more +cristi anor +costam esa +chri sky +challeng ing +bor omir +ble akley +blais dell +b ols +anton ius +ak ra +ad ara +ac io +âĢ¢Ì Ģ +à° ļ +Ä ĩ +won da +westh ollywood +w spa +w new +w kwk +vin der +v card +ttac ars +tourism malaysia +to tino +suj ata +su chitra +strength training +strat us +sli ppage +sky city +si ver +she bang +ry all +ram sar +race to +prote as +plu ssed +ph wo +par li +ous ers +ohl hockey +o he +o cher +mo ho +mn beer +mm is +mi ei +mcal ester +life jackets +letsgor angers +l antis +ki stler +kak apo +inspir its +ibm research +ho an +hauser wirth +hand fasting +grosse to +gold blatt +gob stopper +fer di +en isa +el achat +ecla irs +dri ed +dr joe +del ran +de icing +cp cs +constitu tional +connie britton +complain ant +catt aneo +cas se +bur fict +bit zer +bigger than +bal cones +ba sheer +ar mah +anti perspirant +ang at +ameri kk +zor bing +z ale +yearen d +w jr +vl tava +vene tia +val li +un willingness +tu fa +terror ised +strati graphy +sol ders +sigue meyte +second chance +sax by +sa hoo +roger sville +regin ae +red water +real cand +pu gwash +por tioned +polymer ase +pi ha +patt u +palla dian +oil er +nak at +mos fet +mond son +mett ler +mccar ren +light body +la ppi +kav insky +ji van +is da +hab sio +ha iz +fu kun +feed me +ex changers +dred ger +dra gos +despan yol +dal loway +d sa +d iness +conun drums +concert master +christa bel +chip board +bro me +br dg +bot con +av ox +ar iness +aph y +an sar +allyounee dise +air cooled +adri analima +________ ____ +ðŁIJ µ +ðŁı» # +ðŁİ ¦ +ya esu +wp xi +water mel +vul kan +var an +uc w +tyler hilton +travi spastrana +ta vola +ston ep +secon trol +sciencer ocks +rot ella +ron ge +ro j +reli shed +reinvigor ated +re configured +quack enbush +pram banan +pot pie +ph ry +par ador +papel bon +on pointe +moom ins +mo the +merriam webster +meng gay +man gin +li min +legali ze +lar ousse +ku pp +ksu owls +k lon +jonah hill +inter religious +impo stors +ic ta +hako date +giz mos +gin ar +gen de +ful wood +fr itsch +fi dh +en max +ecach ockey +dun ns +cran ach +code cademy +co ste +chau tala +carol iner +cape talk +cap ela +cab anat +beat boxer +audi uk +atlan tica +ast c +ap cs +anay ake +alovel is +alexander mcqueen +af gan +acadi au +âĿĦï¸ı âĺĥï¸ı +ze ena +yy cre +year th +u ow +to wered +tegu cigal +ta jima +symboli sing +survi vethe +stipul ation +stati k +son ne +si fa +sad dler +qui roga +py torch +precep tor +pra sad +positi f +popein dc +pj l +paul weller +pas c +p me +online first +obitu aries +ni gro +n anci +mur murs +mu cous +morton pride +mid ge +men eses +meda ille +matt cohen +mass spec +mark masai +mariju ana +man se +lon ilove +live sat +lat o +lam po +kri sallen +kod wa +kellan lutz +jurassic coast +jai den +highway men +hailstate bb +go tv +gine ttacars +flys fo +fin dac +festival en +expi alido +europe a +duck dynasty +dil dos +dil auren +co habitation +chi yo +cha ing +be aman +baz i +bar ash +aw akes +ar ugby +antio quia +an ele +amas amy +am eral +ðŁ¦ĭ ðŁ¦ĭ +your schoolgames +wel ke +uncann ily +ud ha +tril ingual +transfer deadlineday +thoma stown +the soul +stu ll +si rona +sem rush +ra hu +pet l +pe ix +ol it +naï ve +mor ad +mclo vin +matthi as +masc olo +mapl eridge +leniro bredo +lati fi +l ero +ky ren +k lim +k cbs +japan travel +ir yna +inter brand +healthy recipes +hat chee +har ia +hal owc +ha stert +gu di +gam bling +free zakzaky +es ler +eli ane +ea stridge +e derson +delhai ze +davi dragan +dar agon +cir clec +cell dweller +ce sare +car om +bru mm +brand es +assi sta +as shat +al gom +aggre tsuko +ðŁĻĭ ðŁı»âĢįâĻĤï¸ı +Ùħ ÙĨ +ye an +yan anda +v cm +v awine +union pay +un damaged +trou ver +tigre sses +thalas sa +switch gear +son or +shee ba +sc ali +sau gerties +san tur +rod rick +ro miley +ri als +quart z +pro mom +plat a +pit kin +par kins +oo ker +nationalgirlfriend day +nager coil +n mr +mor ose +momen tof +mis andry +med hurst +llan tri +liann ela +li bi +kerato conus +j assi +ima p +haz le +hatt en +gy u +gri et +go sar +ge fs +g out +ffi est +f live +eb sco +dun kelman +du rie +dollar shaveclub +dg g +dear g +d elling +crum pler +cor sairs +chef chaouen +chan ov +cau l +can el +cac io +buffalob isons +bluest acks +bab olat +as ant +angies list +aj ed +acry late +ðŁĺįðŁĺįðŁĺįðŁĺį ðŁĺįðŁĺįðŁĺįðŁĺįðŁĺį +Ú© ÛĴ +zel din +ze gers +wo ai +we ish +voye ur +ve p +united center +trun chbull +thi o +th é +taylor kinney +tam bien +subr ata +sisq o +sequ ipment +seam an +sand which +roush fenway +revo ke +relinqui shed +regar dez +realcand aceo +ra ices +r px +r gr +qu ic +prem inger +pp ort +over shadows +out paced +on looker +ncaaw restling +na ish +micro algae +lefthand ed +heis risen +hee ft +hay ato +go eth +eaves drop +du stries +didier drogba +devi ka +delic ata +de trick +daw ley +davidg andy +crazy catlady +co vic +cine matic +ch ynna +cao im +cambus lang +bull fighter +ber occa +bar ging +b chy +az es +alk alo +aliqui ppa +alamo sa +al ready +ak ab +" _ +ðŁĸIJ ï¸ı +ðŁĴĻ ðŁĴĽðŁĴĻ +äº Į +z aga +york region +wrestlec on +wol a +wes warm +wag h +u je +trape zoid +tra doc +til ghman +tiger land +tegucigal pa +tan ge +sun an +sto a +spor tivo +snow mag +sme aton +sh isa +sab es +ri ffa +pri mas +pre vin +pitt en +paul ryan +p ym +ow ers +nouve au +nih ilist +mo ves +mis al +mi dea +metal hammer +mentalhealth awarenessmonth +mc mc +man united +li more +last dayof +lar is +kom an +j sut +ilu min +i frit +ho yos +hillar ys +hal va +group think +gas per +gaff es +gab ler +ga yo +fundament alists +fi en +fe ign +far ben +e sem +e ink +dad y +da quan +cr in +chrysanthe mums +chang er +bri ere +beard gang +b th +ato saurus +as pe +ary ana +aqu afina +anne boleyn +ane h +# £ +!! !? +ðŁĺįðŁĺį âĿ¤ +ãĥ©ãĤ¤ãĥ ĸ +ଠ¿ +zab al +worldr hinoday +wil burn +unru h +uma b +thom ann +tex tu +strugg lers +stie glitz +squ alid +som ar +sol nit +soci ed +sny ders +sky park +sky activ +silver thorne +second ment +ruk mini +plun gers +pele tier +oliviam unn +nzv sa +nottingh ill +no am +net jets +nam ak +mun tu +mmo pen +me ador +mat toon +marki evicz +mand hana +lost withiel +lax er +krizz kaliko +kor aput +ki aro +kho sa +kavi tha +kan in +ja imes +j mf +is lan +impin gement +i league +hoy ts +hoo pa +gram een +gov walker +go ethe +glen na +fu tp +epic fantasy +elizabeth ton +einste ins +econom ie +ec inema +dre ssel +don ya +depress ants +d indi +crani ofacial +cole haan +chad ron +catch the +cam arena +by ard +bush wack +britt en +ben aud +bel field +baw ang +bag no +aye let +ag ry +! ðŁijĬ +ìĽ Į +zukun ft +y rold +wood hull +wolf blood +us r +ten ley +tag um +svet lan +sparkling wine +shorth aired +sd v +scott lin +sam well +sai ful +r hay +q lder +prepa red +porsche retail +pnp cordillera +partici pat +onec lu +oak hill +nor ad +newyearsre solutions +ne als +mi shi +mi mes +mi koto +med lar +lg t +kumar is +kn ish +kal ayaan +kab ah +ign ace +he iner +hat maker +hail statefb +ha sen +great outdoors +grace point +gra ver +gor ockets +gar ds +g ss +g sh +f sk +entr al +en ric +elec tioneering +dover street +dou sing +chickam auga +cer ruti +cc f +c tos +bri bie +braid wood +birdlife oz +bindi irwin +betty buckley +bar ford +bachelor abc +av ital +asu mmer +z oni +won gs +vor derman +ve ces +tu ah +ter nity +ten dered +t add +sudan uprising +stragg ler +stock holm +sl ickers +sar sfield +sa ige +s red +reality tv +rap wave +prim erica +pir ating +pettic oats +pet finder +person ne +peri sher +pat to +part way +pan avision +ott traffic +opp n +o gy +nm state +nichi wa +nax os +n js +mobile payments +ml v +me her +md ina +llan twit +lam ido +kur ama +kal ua +ka head +jol yon +its amazing +indi awith +im plac +ill omania +i ge +hei sei +haw finch +gor dita +gol gi +god by +go comics +gal anthus +fol les +fin sbury +fcbayern us +expialido cious +en ath +dru bbing +drin ks +dead man +de met +dall ara +cypri en +ca ws +by z +bram bling +bethan ie +before the +bao zi +audio engineer +ash tami +asbury park +ari bo +ar nel +aed il +actually autistic +& . +ðŁ¤£ . +ye du +xbox live +world autismawarenessday +working man +with stands +win chell +touch point +ton gass +ti dus +telome res +sub ter +star lord +skil fully +singh bjp +shar row +shad dai +semi freddo +sc abby +saturn alia +sas s +re printing +qui jano +pi quant +pa ice +oun ited +news official +mir alem +meh ro +mdpi openaccess +mari onettes +mari olopez +mafi as +mac murray +lm ere +li h +len ahe +land arch +krann ert +kirk gate +kar mal +inf ern +horseshoe tavern +hol box +gram sci +good hue +go jo +g cn +frame store +fr c +fl andre +fin anc +film house +favor iting +face friday +f type +edinburgh paper +chat tisgarh +ch seats +cc ca +car mageddon +bri se +bo diam +blo dgett +bhar athan +be positive +bau mbach +ban jul +b mac +ay ered +ani official +analy zers +alan rickman +ag il +! ? +å ķ +人 人 +ãĢ° ï¸ı +اÙĦسعÙĪد ÙĬØ© +Å¡ koda +you l +womens golf +wolf mother +wi ggin +univof standrews +u cam +ty wyn +thou sando +thang ka +tb world +sø ren +su til +spay ing +so telo +shape wear +sh ali +senator durbin +sec un +sci fri +sam mons +sal di +romanceno vels +ram ai +pulwama attack +pra sa +porte ous +pike speak +pect in +pande mics +pal lid +paco ima +one troy +oaklawn racing +ny le +na heed +mont co +milehigh basketball +medell ÃŃn +mar san +lv ad +long term +l ga +kumar sanga +kri pke +kinka id +kane ki +k lima +house man +h ns +gover ness +gas mask +fuoris alone +fiu mic +fi fer +feld mann +eur activ +er man +enor rhea +doughnut day +dista ste +did as +depress ant +dark ens +d pe +cord ura +cho te +cheap side +bu dy +boy music +bor uc +bo it +bedro omed +au stra +art week +arcan um +all ay +aljon mendoza +agen cia +ac lark +ì ¯ +ëĭ ¨ +æ ļ +âģ Ħ +will its +west chase +web dotcom +wasimakram live +victori alive +uni o +they are +thedavid cook +tele path +skul duggery +scra ig +scho tt +scan arias +san o +ren y +ram ani +rail card +perl mutter +p fr +p amp +ol é +ni shimura +missouri ans +madi un +mac ke +ma the +m side +looking for +loc key +levi than +last man +kar oline +joan rivers +jil ani +it uk +insti z +hotro ds +gob smacked +gastron om +gall ina +fair view +err body +ele mis +ead t +e bru +du tiful +dre ssed +dog pile +divin ing +dermato logists +dan ese +confe u +coc ita +che bu +can ews +ca ird +bus d +bron stein +boc con +bi pod +bedra ggled +bea hero +bar dia +aw ood +au bur +ake mi +af lex +abc new +į Ķ +ðŁĴķ ðŁĮ¸ +âĿĦâĿĦ âĿĦ +z ema +xi er +wrigley field +wire shark +v pi +un hygienic +trou ve +tox ics +together weswarm +to pa +thru sts +thi stogether +th oops +sust dev +success full +sini ster +sing topra +sido arjo +si ao +schne ier +sak thi +ru bery +ronni erad +rober ther +road man +quack ery +proté gé +pep ita +passion flower +paolo igna +pa aji +off loaded +o kai +na jam +moon beams +mi ptv +merri mack +merco sur +ma pei +lu sher +lu ol +lovethe se +loc alizing +kent ville +kay fabe +kar nad +jo ka +internationaldayof peace +immen sity +ic cr +hu ub +hu ger +hod gy +history in +hfx mooseheads +hen ny +hemp stead +hear tofthe +har aj +fi bo +f pw +everyday carry +dhan bad +dan neel +cityo fla +camel phat +bro de +bra dgate +bou ffe +bla ize +bit i +be you +bay state +ay lor +awarri or +ar cu +appreci ate +ann alee +anad olu +amrit arao +alder ton +ai rey +agood day +- _ +ðŁĴķ ðŁĴľ +ðŁijįðŁı» # +âĿ ĸ +ر ا +ب ار +webdotcom tour +wc cc +vintagec ars +vandy k +us n +une as +un ley +to wa +thofjuly weekend +ter abytes +t dam +sul fon +subsidi zing +spee dup +school teacher +safaricom ltd +ril lettes +rae es +quince añ +qui vers +por us +plus net +pase ando +occi dent +nico lec +new field +ner v +nan ba +mon os +mix ologist +man ji +mait reya +macap agal +m ris +luv ly +luth uli +lon tong +keer thi +kap it +jurisdic tional +j wu +ira j +heb burn +he z +half way +fry bread +fro man +finally mario +enner dale +doo oo +dog love +do doma +cutt inge +cre c +cou che +cloud burst +chick weed +cher ri +bon esh +barbar o +bankrupt cies +as say +arthro pods +ari b +amateur radio +alternat ely +ðŁķ· ï¸ı +âĻ¥âĻ¥âĻ¥âĻ¥ âĻ¥âĻ¥ +á k +zo eter +wren tham +wa elex +vintage shop +vien na +uri as +to chigi +the te +the halo +ta req +t ÃŃ +spelun ky +sop ore +sher ald +shee trock +seun gho +scar ter +rockymoun tain +rite ish +raj in +po ka +pay nes +pai ute +ni raj +nade shiko +miy abi +mariolopez extra +ma ghull +less ons +lar z +l wcf +kre ischer +kaz mi +kas atkina +jon tron +je ers +jal sha +jake gyllenhaal +is ar +ing lou +indone sians +inci dences +in ke +import ant +goo di +fu gard +fru mpy +fiore lli +exter ior +exporting isgreat +er tu +er ole +end aken +dra gom +demean our +dell tech +del vin +decentr alize +dau b +costi gan +copper as +conversation alist +co quito +cla x +char bonneau +chan sons +carbun cle +bydgo szcz +boro witz +ben jealous +baaaa ack +atter bury +apie terse +angio graphy +am ant +af ace +ack land +!!! .. +âļłï¸ıâļłï¸ı âļłï¸ı +é cole +wvu football +winter storm +wal wal +volu tion +van e +und ÃŃ +u ark +travel stoke +trade able +thur man +the mr +texas rangers +telom ere +tan gh +tac ops +syn cro +sput tering +sk ittle +sh eck +sab bir +red coat +rat nagiri +randomactsof kindness +ra wn +ra den +pu tu +pas aran +pan ynj +p nn +omen o +om iller +new nownext +net anya +nat es +n tx +n sd +mell ark +mas oom +loch lomond +li stos +li gi +iro ko +infor mants +ince stuous +ia eaorg +ho tin +hass les +giardini era +gam enews +g town +g dot +fictional character +fanta il +er ugby +devil driver +col ac +code pink +cobble pot +cam ano +cad ell +bu ea +bourbon nais +audit ori +au bu +apa ola +angry birds +alley ne +aid c +agrit ourism +aber rant +ë§Ī ìĿ´ +ê² Įë +è Ĥ +à¸Ħภ° +м Ñĥ +yoshi ko +xy z +wheel barrows +tol worth +the ban +sul jovic +sports day +som edays +soc cere +simm ental +shakespe ar +sentai filmworks +rw m +ru ffy +ric kets +rav aging +rab ha +promo ting +port elli +pan tani +oc cip +mr sm +montre uil +mo sely +mc millian +masco ta +logarith mic +lipp ert +leicester city +legendof korra +ld u +l hot +ko loa +ke ine +ju muah +jink x +jelly beans +jazak allah +j po +indu str +herli hy +ha kimi +guerri ero +ground s +ger alds +galvani zing +ga alive +from hell +emily thornberry +electro chemistry +el ach +dad sarmy +cru i +cru cis +coden amed +ch ava +cb z +ca ol +be strong +ban quets +astronom y +as mo +amin ophen +al po +al amein +ah manson +ah haha +acet aminophen +ðŁĴķ ðŁĺŃ +zoo plankton +zi as +ze se +world pay +wor then +won pil +ve te +tw chats +the timeisnow +te ifi +tar in +ta id +so lem +shi b +sha fi +sasusa ku +sasha apieterse +rwand a +readytor ace +ramo ji +ram but +poli mi +park house +ouri sm +not acrime +niel m +nanditas weta +mer avi +matt sorum +madeby google +macle llan +m learning +lu key +kno y +kere ta +katen ash +jolo kia +jay bilas +international beerday +inf op +ic tfc +i yo +i bee +gue sted +gud run +gri ese +gin a +g mh +fire water +ela bel +eb mt +doors down +dit i +discipl ining +der ville +decla wing +dare tobe +cymb eline +coffe ecake +climate kic +ce ja +brigham womens +bo dden +black shirts +ber icht +ar cli +anti bully +analo gs +adrian grenier +ac gme +ab abe +ðŁĩªðŁĩ ¹ +âĽĦï¸ı âĿĦï¸ı +z end +yu me +yu dk +wooo hooo +wood worth +wizard ing +whereare the +w tm +vla hos +un sportsmanlike +un satisfactory +tu gue +to paris +theshark daymond +temp i +tc v +ta onb +ta ks +set lists +scul ls +sch ing +sam warburton +salu dos +sal ta +ric ho +ri via +r alls +q ah +promp t +picku plines +pc ms +p tsd +on cbs +ok ura +ok um +nit rates +multi grain +mari insky +man metuni +ktt unstall +karan vir +intelligen ce +inst ameet +in ayat +im man +il be +hobb led +ha bu +golden heart +go jackets +glas shouses +gen teel +emphy sema +dif fie +daysof biking +dan as +co es +cleg ane +chuck norris +chiwe tel +check off +capitol hill +cap n +cad den +blood and +bindas bhidu +bhi ma +bank able +ay ur +ax eman +arshad warsi +aldine isd +>/// < +. ðŁĺĦ +ìĿ´ì ¢ħ +ãĥ³ãĥ Ľ +âĿ¤ï¸ı ðŁijį +y auch +wil mer +wa ghorn +vill eraces +uniof surrey +ul k +trage dia +sp f +sof hope +sher burn +separ ators +sci atic +sch em +s big +rol les +re organisation +priest field +pie zo +ph art +pe changa +papag ayo +out growing +of ic +niskay una +moven pick +ma hir +kra bbit +kirk herbstreit +jamie cullum +jagu ares +j beil +itv be +it in +ise d +im re +ilistic expialidocious +i inet +hurricanes andy +hands free +haha ah +gon char +go texans +glblct zn +gal leys +fl ite +fidd es +en am +el gon +diso wning +dimension data +cro oz +circas urvive +ci am +chu cke +car lor +cab er +bre hm +ben folds +bel aire +be ttes +bar ner +balu strades +ava etc +austral opi +ash lynn +as ong +are sults +arc turus +americas gp +ðŁļ´ âĢįâĻĢï¸ı +ðŁIJ ½ +z aks +yl vis +wil ho +viv ace +uniwest scotland +un manageable +un convincing +tour noi +tor p +team scorpion +tach ira +su tta +su mr +smith college +silver berg +shoot filmb +she mesh +ringo starrmusic +ri sen +red more +rac a +quintu ple +pontypri ddrfc +police woman +pero dua +pen ch +pe den +no elia +mo ch +men k +mar fan +manner isms +lo tor +light show +li ège +li ker +lc w +kram nik +ke mo +johnny swim +its morissette +intran sig +id preps +ian bohen +hunter moore +flat liners +fitz maurice +fil ho +espn nba +electro mechanical +ear drum +e pperson +deceler ation +dar b +crani osac +colli oure +cmo guj +cl gaming +christmas spirit +chris riddell +c gh +bcbg maxazria +autonom ously +ascen sion +applic ability +appar at +ali ze +ab uk +ðŁįĬðŁįĬ ðŁįĬ +ðŁĩ³ðŁĩ ¿ +zulfi kar +ye ter +yc dsb +wild cards +whis king +tom ilah +timmer man +team viewer +tai yaki +superfe st +stv l +spor tat +spo sito +son der +sch ill +rosen blum +ronnierad ke +rep ays +red action +re hana +rain out +pulse ghana +power hour +plo dding +pence in +pacham ama +oxfam gb +oom men +offer up +ni ff +luci dity +lo en +las ith +kettle bells +ju da +itsb hu +itsbhu shankumar +ish war +indy lights +hill harper +hayley atwell +hal lowell +gidit raffic +gentle men +fat ou +endocrino logist +el ica +e ie +doo fus +domen ic +dating advice +dat work +critiqu ed +coach ella +chu but +chino is +castlewell an +bu jumbura +brun ning +bren del +bo try +best show +best practice +art c +al hassan +ah so +[ ðŁĵ¸ +ðŁĺ³ ðŁĺģ +ðĿĹ Ķ +åħ¥ èį +yev geny +wynn lasvegas +wind screens +wil ayah +ven nel +unob trusive +the whitechapel +ste iz +staf fe +sheff hallam +sh anta +sas ser +sar ri +re hydrate +re ep +proto ss +pink society +pe zz +new sted +ne revolution +micro processor +michael d +met ta +meh reen +me jor +mar vi +lu croy +looooooo ol +lam ento +kristian bush +karti k +jam ai +inde e +imper dible +hol ten +hed ley +gl om +for pleasure +filipinof ood +fam osa +estim ations +e bba +dr r +cor is +clo quet +cl une +christen ings +business owners +bele ive +b fo +atmo sph +atas cocita +ash grove +arm rests +arak awa +alas sn +aintre eraces +ai o +aga wea +ðŁĺį ðŁĻĮðŁı¼ +ðŁĺĬ ! +é lie +zoes aldana +z deno +y wg +wweb attle +wwe balor +wn g +wardro be +tou ma +the silver +teenage mutant +te mbo +take it +stre sort +spec tres +span akop +soyu z +sincere ly +see konk +science direct +san tay +ridge ville +reprori ghts +re pos +rc despanyol +punche stown +play apex +pitt water +perspec tiva +or ync +nbl canada +n ky +mrjame sob +motu eka +mon gery +modi fies +make a +land ry +keo ghan +keller williams +kam bing +jackson ville +is ambard +ir bid +instagram ers +imo hq +ia q +hydro plane +herewe come +heid feld +h ny +gy imah +gn oll +fan elli +extor ting +esp news +elife style +elder ry +dsound system +ding us +de wi +collu de +clo ete +chopda harshad +chau th +but tock +brother lylove +boston ians +bj arne +bir ce +beij ing +back lighting +asi des +ðŁĵ¸ ðŁĵ¸ +ðŁĴľ âĿ¤ï¸ı +za andam +yam ahar +woo kie +whitt ling +walk up +wa wel +voteuk directioners +uz ma +utd before +ti val +thir tieth +then tic +the ginger +tack er +sy c +sulfu ric +speci esi +sp ly +sear les +sar angi +ri dic +resul tados +ren gagement +pi a +pat ois +pat gt +pap es +ol ites +oil paintings +naseeru ddin +n pe +mu tiara +moul dy +mistre at +matsu o +mar ney +maq sood +lab u +katie taylor +kak i +ipp v +ing el +i wear +himan sh +hard ware +happyn avratri +haj er +gul panag +go hard +fas b +em olli +e migrating +dull ness +divo ck +davel ackie +cor ten +ching y +chi pubschools +car rack +cabanat uan +ca illat +be quia +bar kad +autom ated +artu ro +ar ot +and thenew +ale tti +afcf ta +ab ies +ðŁļĹ ðŁļĹ +ðŁ¥ Ĵ +ðŁ¥ ĭ +�� �� +민 íĺĦ +âĸ ¿ +yan agi +wa verider +un werth +tsu kasa +tourmal et +tor books +than never +tere se +ta sered +stre ga +so cc +smrt grls +singtopra chaya +si hc +sham ba +science twitter +sch wit +sch oop +ra dia +ra bah +pur rr +pon ch +pole vault +pip kin +paw patrol +pa ku +ontheroad again +not food +nau strong +nag ato +na thletics +mo id +merthy r +mass mutual +mark waid +madhu bani +light wave +liberty u +la uro +ko sha +king size +kerat osis +kati es +kall ang +k pr +inver mere +inter na +im mingham +hu ling +han ji +guardian eco +gerst mann +ger von +gary valenciano +fe sh +favor ability +f cl +ella eyre +ec co +disp els +dir venkatprabhu +dhi ve +dab bs +clt news +civic tech +christin aperri +cer ven +cat es +bor de +ber ates +beir ne +barre t +banta yan +ar nos +an ich +am ya +ais y +?? ... +( ?), +ðŁĺĤðŁĺĤðŁĺĤ ðŁĺŃ +yel m +y bn +un quen +tz laff +truec rime +tro tted +thus ly +thin q +there all +tal in +ta vious +subver ting +subo ptimal +ssi ves +sou ths +so bukwe +shu shi +shel ping +sham mi +se pe +sav ill +ri seas +ra vines +ra gazza +provin cia +pro max +pod ca +pixar coco +pd c +pakistan day +official jld +nw ob +nin os +nay y +mun ford +mirand acosgrove +me gas +li zzi +letsgoo ilers +lar ynx +ku tta +kb fc +je ph +is ce +il ola +homes for +hat chi +haf ner +guide books +god man +fre shie +for zar +for scom +ew opinion +draft day +dj p +deut z +deleg ated +del ima +de fused +cot w +coqu imbo +capri mary +canber ratimes +back plate +aval ley +anpan man +aj ie +adity amusic +ad dl +ab ounding +( ~ +yebo sfaye +w cl +vand i +utdbefore fergie +usa wp +tu lus +thel ow +ter no +stu bhu +ste urope +skull hong +sil vano +she persisted +sever in +sche coperez +sau m +san antonio +sab by +sa ik +ru ba +root stock +resi t +real alexjones +psychop athy +pere mpu +pep co +or so +one on +north men +nihon go +nassi r +mr ddyer +mix down +mer ian +la voy +l ort +ku suma +kon eru +klar na +k fw +jinkx monsoon +ivory coast +iron dale +ira o +in bar +ilai yar +i yah +hen gist +gov andals +google earth +gesun dhe +final showdown +fabi ana +dre mel +down hills +di okno +de valued +dan ker +crin oline +co pt +ci ans +chicago tonight +ch ments +ce ux +carpool karaoke +c iss +broken ness +bou jee +bat z +austin chronicle +ask force +all nighter +agri gento +acom ic +ach opra +(âī§ âĪĩ +yan asta +welsh labour +tur man +trans ni +thehobbit movie +the sal +the oc +the men +tangi pahoa +t qm +strugg le +stati st +squ ar +si donia +seep ed +second sofsummer +sa jak +regre ssed +raj inder +ozy mandias +om ai +moon base +menta wai +martinluther king +lun go +lithu b +kir sti +juju y +jal fre +j iru +j chs +iz m +indiscri min +indie horror +im por +iheart festival +fri k +ff d +feder ici +en pt +dro oms +den feld +den den +cro on +chu mash +car me +bric ol +bra inte +black rock +banan aman +au spost +atlan tique +ase chat +arti stanbul +armie hammer +ðŁĴģ ðŁı½ +미ìĬ¤íĦ ° +é¹ ¿ +⼠µ +à¹ĥ หม +zeit kultur +wre y +withern sea +win de +vi van +va beer +v afc +tow les +tomilah ren +tar ong +tapro ot +tal madge +ta uro +swachh survekshan +su ba +style chat +sp qr +sn outs +slightly stoopid +slack lining +ser ger +romp ers +re mount +rati fies +pro tag +pintof science +pen thouses +par secs +pa al +owo sso +oster ley +need more +mug sy +morten son +meek ness +matthew perry +mart inet +mal econ +lon gan +lim mud +kno pf +ke baya +ka veh +k pm +ian to +i sit +hydro chloride +hand crafts +footballa su +fancy dress +egg ert +conceptu ally +clar is +bug z +bo ity +blue and +batt lea +aw ine +audi in +aira si +ad b +//// //// +åĥ ı +yang yang +y ade +wy rd +whit ely +war mongering +vk singh +ud hampur +ud as +tra chea +thisishar drock +te ducation +sweat in +ss ch +sna ith +slo bo +shut tering +shim mer +sent e +sad dling +rubi k +real grumpycat +rail riders +pre nz +poke shopper +pink ham +phu ket +p tu +oooooooo ooooo +oat cake +nir man +neuro psychology +mm n +missouri state +minim alists +mill pond +michael brown +merrychristma severyone +md m +mc gru +mar sabit +manolo blahnik +madal yn +mac ula +mabu za +ly fe +leg ang +lady vol +jal opy +io b +il ce +honey mooning +hi pper +ha beas +glam or +gaon kar +gal ah +event i +eso k +es lav +east chester +drum beat +doo joon +dom ore +ct tee +cross fade +chun ji +celer ator +card nation +bru ich +beverley knight +bare k +bal thus +as syrians +aphor isms +ann amaria +air disaster +ado do +ac led +íĮ ¬ +z illi +uk ay +u be +ty moshenko +tu borg +the alarm +ten pin +taxi fy +sy g +surplu ses +stad ter +som eness +sealevel rise +sche er +sch aal +rox boro +roc cat +ri gi +ravel ry +rampal arjun +ra vels +qui pped +pr sa +pj paralysis +pin sider +picad illo +phthal ate +paris airshow +ornitho logical +nbag league +nar u +nap a +mon en +mistran sl +men chaca +man jit +malign ancy +ma gie +kra kauer +kirk hope +k lang +ju die +j sb +ir chi +hom efurniture +ha sn +h Ã¥ +ghosthun ters +followthe music +flor sheim +fl amb +fi ggy +ef fu +eccle sia +das ara +cumb rae +choo choo +chatter ley +cal crutchlow +bil lows +bhat nagar +bell in +baltimore riots +bal z +bai da +arm our +ar ye +alle magne +ai z +aaron ramsey +.. âĢ¦ +ðŁĻĮ ðŁĴ¯ +ðŁĩ¹ðŁĩ ¿ +⼠ĵ +wy re +vit reous +vier nes +upp ity +under classman +tw arri +ture k +tur c +transcrip tome +the bestof +tel enet +te menos +te abag +tam bu +sp ig +skid more +ro mer +recipro cated +raci alized +pre occupation +pot gieter +poo py +ply ometrics +pk b +pil z +pe kan +opi ates +na oya +morgan library +men de +m craven +lo athed +linden hurst +ky ne +kp fk +ko chad +kent wood +jun jin +jame ela +its notre +imper cep +hi sti +her vé +grand order +family feud +esc abeche +en go +e ht +du elist +dress ing +debbie gibson +da ha +ciner ama +christyclark bc +charge sheet +car less +bys she +bis saka +be partofthe +abol itionists +aber lin +abe be +* / +ðŁĻĭ ðŁı¼âĢįâĻĢï¸ı +ðŁĶ Ĩ +Î ¸ +we ater +tutto sport +ti more +thos fansbts +style icon +steven spielberg +startrek beyond +starmagic ballfanfave +south down +si pg +sh ado +sexu alized +sel vage +sea world +se fer +sa az +s angels +run dmc +rock ford +ro strevor +ren dy +pre position +plu gin +pher able +onder wijs +of instagram +no tobaccoday +main waring +mahal axmi +maguin danao +lazare v +lam mers +kot zen +kitch engarden +jor gen +jalen rose +j boss +insta artist +fc punecity +far ish +fal kner +f atti +expun gement +esc ut +dro yal +disobey ed +dis lodge +dipi kak +di pp +deliver ing +dan u +criss angel +cor liss +co safa +cic cone +cannot wait +cafe terias +brad ys +bp brewing +bi gup +banff centre +ba azi +audi r +are ason +anag lyph +alder hey +al ima +affl ic +a ars +% ' +ðŁĺį ðŁijĮ +ðŁĮ¸ ðŁĮº +⾨ ðŁĴ« +à¹Į ) +à¸Ī ะ +y uru +wo ther +warm bier +upp pp +un coated +u mph +u icc +tur un +tu ki +tra ch +team jamaica +sye da +stubble field +stock broker +shay es +shar ath +ser as +seal ink +sc illa +ruth ven +ro ker +pett itt +pac u +op hie +neighbour ly +mumbait raffic +mr tony +med u +may ur +mar dle +mahabali puram +mag ico +luke skywalker +lloyd soflondon +leigh francis +langu id +ku bla +kiz ela +killer cat +kh are +jo ell +jay anagar +id les +hiro to +great ness +gopher football +goo wls +gon d +gloriae stefan +fron to +fil ial +est evens +er g +e bels +du wa +domest icity +diss i +desp ises +dermalog ica +cor alie +convivi al +con vers +con currency +clubb rugge +chrissi e +car rs +cam by +bu caramanga +bh fo +bertol ucci +bacter iology +au gie +app leyard +accli mated +a hotel +?? ?!! +! )) +ðŁĺĭ ðŁijĮ +ðŁıį ï¸ı +ë ĮĢ +w ops +visit dubai +uk uran +trib bett +the exorcist +tel le +teen mom +tar tt +sweet man +su zie +strange ways +sta id +sho gi +shay ne +sall is +revol ted +ran ko +polic emv +ple bes +patrick stump +paint it +ot ello +official rmt +numer ique +mr mike +mer vin +men es +mat ins +ma sn +ma saba +m chapp +leven son +le wie +kill or +kate bush +karate kid +jo shy +ital yin +ip hi +hu hu +gold corp +go ge +flexi bly +fer rato +far leigh +fan of +equ ines +el fy +dra peau +dis illusion +di van +dhu pia +dach shun +d hari +clostri dium +carmel ites +c cie +bird watcher +bam bina +bab bac +ash ah +ano int +anir ban +air park +a xion +ðŁĻĬ ðŁĺĤ +ðŁĶ ± +ðŁıĮï¸ı âĢįâĻĢï¸ı +ðŁİĪ ðŁİģ +нÑĭÐ ¹ +woo craft +we blog +un caring +uli ses +textile art +ta ym +sya allah +super set +subli mely +su ttle +stay er +ss yndrome +speci fies +sor sogon +silver light +sebastien loeb +seac liff +schlo sser +rhe ingold +resul tant +restre po +real kiefer +quin ns +princess bride +per awat +pe po +pale is +oster man +oc t +nz s +nu b +national margaritaday +n dv +mushroom head +miner alogy +mi haj +men ken +mart solf +madi kizela +ma awards +lys mic +love theatreday +lau dable +lace up +jim mi +jar im +j ynx +it to +her tel +heat ley +hal ka +h mp +gram me +gam blin +fakenews media +ell racing +du duk +doddle oddle +cot tam +christ omlin +car sand +bul g +bridge head +bod hi +bo di +bas se +bas quet +ar ash +am az +alton sterling +ak elly +ail way +after shock +af neil ++ ? +ðŁħ° ï¸ı +ãĤ ĭãģ +âĿ¤ ðŁĴļ +âĨ Ķï¸ı +⬠Ľï¸ı +yu ppies +ya j +wwebattle ground +wit tiest +wgn tv +west london +vol umen +vegan foodshare +uni e +tu ka +toby turner +tim heidecker +tar ra +tachi bana +sy zy +sweet ening +stri ved +stri ders +stealth ily +sn hu +smar ch +shim on +she dit +sha hab +se woon +sch ile +que ur +pul borough +prett ily +pra bal +pir tek +p mj +or rick +oli m +noise tte +neck beard +n fo +mu sand +mother teresa +mo khtar +mm els +mar awa +mal functioned +lu cht +lean ings +lea vey +land send +lam bourn +kin na +just voot +ir ca +indian cuisine +im debina +ho are +hather sage +hass ocks +gh oul +gastel um +famili as +f fort +ecosystem services +der vi +ch era +bom bas +blau ch +ay sia +ap ala +album cover +ade sso +acoun try +ac scc +ab khaz +a hockey +⾨ ðŁİī +× Ļ× +zo y +women scycling +wel les +we ave +war ded +vi des +ver uca +ver bose +u hc +thankyou foryour +tattoo society +stargaz ers +stal lion +sma ster +six ways +se diti +scra fts +sack boy +s ated +po ix +or se +ny nj +new media +neu l +nbaf antasy +mus burger +mssarah paulson +martin borough +mag ni +liti gator +knight sbaseball +kings gate +kick itout +kath niels +justin verlander +jay sh +iron ies +ira qi +inn keeper +hart field +har lot +had low +h br +gov pencein +gohar dorgohome +god bles +gl v +gel v +flet chers +fire house +film freeway +entrain ment +emma stone +dirty dancing +deriv ation +dal ila +confor mists +clo cal +christ us +chi h +cb ball +begru dgingly +bank sters +ball sbridge +andre sen +ador ableness +ab unga +' '@ +! '' +zam zam +yu o +u ver +te als +stereok icks +song contest +sol ak +se res +se berg +sa kata +ross ana +ri gh +resu men +rel f +rachel zoe +ph ale +passy unk +of gem +o gn +nur der +nug get +nite sh +ni hotri +na diam +multit ouch +mor y +mc y +maz ouz +li pi +lar an +kinder transport +kil kee +ke els +kab we +jave dn +infin i +id hi +hy uga +huntington sdisease +horse hair +hi rai +hay field +ha kan +gp stpete +gor sky +gi us +ge mba +fullmetal alchemist +for o +fee der +emple o +em ple +eber sole +dorse taonb +digg ory +did there +de jo +countdownto kickoff +compens ates +coinci dently +coco tte +cob den +catt ar +catac lysmic +cam hs +brew gene +bounty hunter +bo ing +be ggan +band maid +baker loo +antipo des +ðŁİī âĿ¤ +ï¸ ¶ +à® Ĩ +à ¢ +yar on +we de +w pi +thugsof hindostan +theband ghost +tex astri +swi zzz +stanis law +spr inted +speed running +samanthabar ks +rx theatre +robbie amell +rav jiani +pru st +priv é +post it +port vale +pol in +p gl +p by +otta was +one kiss +o cken +nrl knights +night scape +mortal kombat +mk stalin +milli a +mar aga +lac ounty +kyo to +kristall nacht +ko hat +kan ell +kahu ku +juli ann +jas sy +j sc +im iss +hr ough +hoo doos +helms man +hand drawn +fex pression +fast n +far ingdon +et cetera +dun luce +don nas +de min +d bt +culd rose +connec tors +combat zone +ce si +ce ol +call center +bushra gohar +bu ssed +bre saola +beque athed +b inger +ar mam +alla in +alder aan +abid in +미ìĬ¤íĦ° 미ìĬ¤íĦ° +à¹Ģภª +ൠĨ +ÙĦ اÙĦ +ze us +yeezy boost +wo tc +w aked +us latino +ur bs +un hindered +un commonly +tol son +thra x +thermo graphy +ter adio +tal kabout +stere os +sousap hone +smi the +sleaford mods +shipla p +sha e +sav inghope +saddle bag +rish nan +repo stre +ptero saur +pre gaming +prabhu deva +norfol khour +nol la +no sa +nivin official +night market +n wot +mer ano +mel illa +mar ic +maha ffey +luck enbach +lastman standing +lam esa +l ome +kit we +kh ong +junior golf +jet packs +jam mf +j ela +instru ction +hud gov +herman mashaba +hatt i +han deer +glynd wr +germany diplo +gar bine +for co +fo etal +film production +feni x +farmer smkt +fa dil +ev anna +dis sections +dil an +col usa +cla sic +chauvin ist +bow li +at rain +ali asing +ag gro +ab is +ðŁİī ðŁĺį +ä¸ ĭ +u don +tul bagh +thorn dale +te ssier +sw abi +sur p +starr cast +sock i +so wore +snu ffy +snow bound +sm oments +shrin ky +sensiti vely +sel ous +scutt led +sak hi +sab cs +que ak +pri mm +paul j +palanti r +paf os +op sec +ods al +nylon mag +multi platform +mud guard +mil sap +mi mpi +mb ak +mai kel +ma gar +ly sa +lu by +lo ja +ky on +ky ns +kh waja +kab y +jo onie +jo anna +jam z +inst adog +ing lish +inflexi ble +im material +i ferous +hor u +hierogly phic +hen day +har jo +han out +h pf +gol azo +g ml +flower show +first tweet +er ko +ent res +earthob servation +e wen +discoura ges +de codes +dar rius +cu aron +col adas +clo ke +brun tsfield +bran igan +bor neo +bbc debate +baz inga +barry m +avar ice +attle borough +at wt +apparte ment +amater asu +all orca +al ti +acab an +,,,, ,,,, +ðŁĽ İ +ðŁĺ± # +ðŁıģ ðŁıĨ +íľ ĺ +å » +à· Ģ +zo ek +xl vii +win kie +whodat nation +war den +tri ge +toriam os +thetoronto sun +sur charges +superbowl xlix +steen kamp +spac estation +sko al +ship mate +sc top +ro tter +red c +ra sam +ra jai +port slade +pietro sd +phine asand +p ous +ot ary +on dine +ny ul +n fts +mu stique +moving day +mosth aun +miss mayim +med lin +mazi buko +mark twain +mario andretti +maribyrn ong +ma drona +ly dney +lu mad +lou che +lit chat +kenne saw +ke ena +k hom +jim énez +information security +i fill +horticul turist +home bush +ho enig +her riot +he era +ha chim +gregory porter +game time +forwar ders +fl itting +faroo qi +eni ans +eclec tic +drama fever +dr ramansingh +cop thorne +congress women +conglomer ates +citym j +ci pollini +cham illionaire +carrou sel +ca hors +bruich ladd +brin ton +box nation +bother some +blumen feld +billy bragg +berzer k +beau x +bchy dro +bal ears +ath oll +ar cen +an any +aircrash series +ag elim +!! (: +ðŁĸķ ðŁı¼ +âĹ Ħ +yer ush +white privilege +well stone +volun tary +universit ario +u eg +tt ttttt +tour decor +top shelf +tack ler +su kumaran +ssh rc +sli der +shin agawa +sensation ally +se vi +ro ser +razar umi +quix otic +que iroz +pru de +pri o +phantom friday +petr ichor +per ret +parishi oner +op hore +np slsoccer +needi est +n side +mu lu +micron utrient +me tter +mc duff +man imonday +mal ouf +la zi +knott s +kapil sharma +jammf warriors +human itarianism +haz ar +har rie +fire hawk +em presa +eli seo +dor val +din ardo +cro ot +could ve +coli seo +col onic +city brew +cat oosa +calle ja +calabar zon +bun tings +braz eau +blon dell +black feet +bl ach +bernad ino +be different +bay bay +bate y +bare bones +au p +antw i +and bake +allu sions +alfon so +ak anksha +ai kin +ach el +: ~) +. ðŁĺĥ +áµĹ Ê° +z wart +ye sofficial +wood lake +waver tree +w mi +ven ango +thel abel +the flaminglips +ter relle +team england +sver d +substitu tions +st impson +spirit airlines +son ik +sling er +shru gging +shootfilmb enice +shi ho +she il +ser vos +secre ted +saldan ha +ring a +ren vy +rema pping +re shoots +re developing +ram yun +protect mueller +pri s +pap en +ornam entation +op tica +nom akeup +nieuws blad +ne ema +majo rettes +maiden graffix +magn ates +maf rica +lo chy +leg ato +le tus +kucin ich +kav ala +karisma kapoor +ka stel +jen nys +inter urban +inspector ate +ik ram +i vica +hon an +hel min +har twig +gott man +fli pped +et fo +esc olar +entit les +enter towin +edel man +dont judge +death star +crof ters +continu ous +co asting +categori zation +ca inc +by poll +bush rangers +bts m +bor ch +bladen sburg +bill c +bharat ratna +bent z +bel les +bay bears +ba hari +amber heard +am re +acom b +ðŁĺį ðŁĺľ +ðŁIJ ¨ +ðŁĩ²ðŁĩ ¹ +zi z +xia obo +ww lp +wis nu +water field +wait what +util ises +tw illing +till sonburg +tear away +tau gh +stru g +state dept +spha gnum +skÃ¥ ne +shur mur +show tv +sha ar +sex to +sc art +sa kin +resi ster +rag tag +ra ffa +r bi +quit man +pony ville +pe ets +os me +nr sc +naias detroit +n pa +mrun al +mel ges +mbr srd +martin us +maril u +mal alaf +malalaf und +la ak +kø benhavn +kentuc kians +jeff d +harve ys +ha bil +h sus +gra o +fer gal +extre mo +ety pe +ent rails +enjoy globe +education week +ec centric +easport snhl +dj carnage +demand action +dead laced +co springs +cityof to +chip man +bs india +be att +ball players +b nb +attrac tor +as salam +ach amps +ðŁĶ´âļªï¸ı âļ«ï¸ı +ðŁĴ£ðŁĴ£ ðŁĴ£ +ðŁİ¼ ðŁİ¶ +ðŁįĶ ðŁįĶ +ðŁħ ¾ +ðŁ¥ ļ +âĻ § +л е +wri mo +wi ster +waipa hu +vp ns +vi ren +under staffed +u or +taip an +stacey abrams +spread able +sol art +schle ich +salt coats +recei v +rat p +pilla ging +philly inquirer +optim o +mis carriages +mac lennan +m red +m ends +lu ana +little simz +litt mann +li ge +lad yof +ky line +ki shaadi +inscru table +i bby +hi zo +he gar +hav ells +gabrielle aplin +furi ou +fr oud +foli os +e dos +du pa +disrup tive +derby swildlife +cu pa +corporate events +code y +ci beles +chu gh +christma sc +ch ly +bol te +bo brisky +bick nell +belgian beer +babangi da +avir tue +ant manandthe +ant gupta +ak alay +ade sh +ðŁıī ðŁıī +ðŁıĥ ðŁıĥ +ðŁı « +ðŁİĦ @ +âĿ¤ ⾨ +zy go +zo ho +willow dale +wall covering +vol ar +vo ie +vit icul +vi vanco +ve mula +universal orlando +un v +thingsto doin +the flag +team bonding +sur render +ste pping +song song +siren craftbrew +singh rajput +sharma fc +shab it +sen batsu +sap i +sand pipers +sab ang +revel ator +repostre gramapp +record store +re ks +r fb +push pa +pin cer +pent agon +new saus +mis kin +mat patgt +manchester derby +le ery +klagen furt +jacqueline m +j ff +itsamazing outthere +inter sport +hermeneu tics +hackney abbott +gri mmer +good cause +glenmor angie +gett leman +futureis bright +fe tter +eras able +ep as +ent pvtltd +el ahi +du mm +du lo +dh x +day i +cul berson +ctv vancouver +create space +coffe et +christma spresent +cast res +cardio ed +boul ware +bottle cap +bbc sheffield +ani seed +angel list +agit ating +ðŁıĪ : +ઠ¨ +zomat li +woodcut wednesday +white tail +water birds +vulcan salute +vibra phone +van guards +un bc +twitter bakealong +tomlin son +to lead +thre eday +the jen +th august +terror attack +sul i +squat ted +spor ty +snar f +sk eller +sher in +sac ca +public school +ph unk +or ado +oh anian +non structurefire +nol asco +neyo compound +mtv scream +mor on +mis canthus +migr ates +mi hon +meik le +mal va +mai gret +madame tussauds +m wt +lo ath +ki mo +kag ome +ka the +je im +ic ym +iber o +goodday sac +go ksuowls +glen orchy +ge tou +found cats +fog gia +fire hose +esc ing +ed rive +descrip tors +de foli +dave and +dan j +dabboorat nani +cran fielduni +cookie monster +clu bes +char meck +cele bi +cajon valleyusd +by passes +bloom quist +black outday +be vil +bac ci +av ril +at ari +ameri star +achievemen thunt +ðŁĮİðŁĮį ðŁĮı +ðŁ¤ľ ðŁ¤Ľ +ðŁ¤· ðŁı½âĢįâĻĢï¸ı +민 íĺģ +x and +wb ss +wa ine +w set +vi mal +uuuu uuu +un believer +umbrella revolution +twit terer +tur on +theatric ally +than atos +ta em +soon est +so tt +she ir +room withaview +rising star +ra sad +pro a +po pi +phi mu +peru vians +os borne +or monde +nrl finals +no ha +nag ant +mugg sy +mor ven +mir on +min ima +mand ell +mad ams +l ws +l ft +krum lov +kronen bourg +kristian sand +kk al +kat mai +ju anda +jn f +jim dunlo +j pii +idaho bit +iam cityfc +hoo chie +hh p +heli anthus +hein ze +heid sieck +ham i +hal deman +g iller +ff ice +fabri ano +di mo +com an +ch ell +carmel ita +ca cho +bradley cooper +bo garde +ayo dele +authentic ator +audi ble +associ ative +any outh +aggre ssors +âĺĨ âĺħ +à® ¨ +yu ms +world kidney +weis sen +wa thi +ur ba +u di +u delaware +tuj he +to les +titan s +the judge +th ave +sw um +st k +st elios +spo ony +sphy sics +shonen jump +shant anu +sagu enay +ry ann +rumpel stiltskin +re funding +ray bans +ram ón +prescri bes +peter lee +per sil +penc iling +pack football +osu coach +of l +nor onha +ncis nola +nati o +mycoop food +muscul ar +mt ss +mrandmrss otto +mis érables +milit arily +middle ditch +michael cohen +me thy +matth ysse +lo ba +li stic +ken po +kal im +ju go +jac quet +ith am +il n +i amp +ho ad +halvor son +gol l +glut tonous +gli al +gar ban +game and +gal bi +frog more +floun dering +fen i +exple tives +er ism +er df +engle hart +ek elly +dilauren tis +diffe red +cri ppen +cram ond +conceptu alized +co existing +churche shour +chorley wood +character izes +castle wood +ca il +bri sas +book sto +billy burke +beck les +bamboo s +bac an +bab ae +awa res +as cona +all together +air watch +air head +ad aline +abkiba ar +[ ] +ðŁĵį - +à¸Ļภ° +à¸Ī าà¸ģ +ಠ® +¿ ¿ +v lr +uel final +tri somy +theore ms +th mic +th may +temp a +squir ts +smart ass +shen ley +sf in +sen toomey +sd npa +rape culture +r gay +quin tin +q oute +pv h +pro cks +privati zing +po lecat +pasta day +oliver os +nac omics +mendi eta +me akin +lu ling +la vasa +jo erg +jar vis +it canwait +it am +ira da +hor sham +honeye ater +he j +gran lund +go ren +fo dor +extor tionate +est ée +eras cal +enchant ments +dist antly +disen chantment +dark ish +contor tion +commerci alized +chan nibal +cahu enga +c bl +bo ons +bestin theworld +bar nette +ati ja +afro man +adu ana +absur d +ðŁĴķ âĺºï¸ı +âķ Ķ +zar ra +y ss +y rp +wym t +water treatment +u ji +tuss ock +trail side +top most +than n +stgeorge spark +steve backshall +statecap ture +sound man +sn ano +signsof spring +si var +sf bay +seung youn +see eee +san antoni +ruth enium +rolo dex +rock radio +ro gic +rit chie +ri peness +reyno sa +residen cia +ren nen +publ ico +pri maver +pe dy +oscill ators +nic colo +mon gabay +mis nomer +mind share +meso therapy +mc burney +mat u +man zi +ma et +long bridge +life proof +li fi +lea ves +l ston +kno bbly +keynote speaker +kemb la +kam my +is os +inst ills +insec t +hot line +home making +hawk shead +harri stweed +han ne +h ally +guimar aes +global shapers +fu wa +fo ghat +fe asti +fashion nova +downsyndrome day +din apoli +delhi daredevils +dam one +d vo +cran ley +comm itee +co gge +cla res +bluec rew +bin brook +ben ge +be hrs +aw fulness +as pley +ak itchen +ag andhi +acre me +ðŁĸ¼ ï¸ı +íĮ Į +ãĤ ¼ +âĻ¥ ! +ÙĨÙĪ از +wwe australia +weir ton +web casting +vor l +vacation ed +u mer +trilo gy +thisday inspace +th fleet +te pee +tar ay +tar awa +tan vir +tall madge +suit land +str u +ste eves +soundary aar +soundaryaar ajni +sn oops +shish kin +shi en +share downership +shann an +sen escence +sand ton +ronaldre agan +rise withus +rid gid +pad rino +ow ler +out score +on sunday +octogen arian +nijin sky +nav ona +my vote +muf fu +mb alu +mate os +long nose +laval lee +kn ol +kell ner +kandi vali +jumm amu +jar ir +horati o +holiday party +hill grove +happy republicday +gen is +fossil fuel +far rell +evi an +eu an +entang lements +ecur tis +dum dum +dr n +detox ing +der music +de portugal +cv cc +continu ance +com illa +co sell +chri smc +chic on +cham oy +campe stre +business awards +brush stroke +brew ers +aston merrygold +app dynamics +aler ta +ala am +ag ard +? ; +Ï ĩ +zoeter meer +xx o +x ds +world malariaday +wool loomooloo +wood chucks +wiener schnitzel +wango tango +vintag es +ve le +val lu +tu pid +trans location +tin i +thescript family +ste i +stab ile +son ically +sobie ski +slife style +silver spring +shi on +shatru ghan +samaj wadi +ron is +recu sed +raas ay +pre matur +pl om +pin atu +per anak +parv athy +pal atka +napak nowhow +my tuner +mou gins +mom blogger +men cia +mam ah +ma ini +levar burton +lem picka +jill scott +j ills +interpol ation +indu blin +im balanced +i isc +i esa +gun jan +gor sk +ge ia +flo fficial +ejer cito +ee o +dye ss +duchessof cambridge +down towns +do do +di alled +dex ys +cool kids +coc cy +co working +cityof calgary +bou rassa +boric uas +bo tta +bazaar uk +azur ite +az ula +arvind swami +arke stra +adv t +acon cagua +abo iti +x brownx +win nin +weber grills +we ka +wal tz +wa chs +urs bolt +trevel yan +tra s +tiny rebel +the modern +stay warm +spel mancollege +spanakop ita +sigur ros +seab right +se culars +sach inten +recomm ence +ra ima +r stream +pe za +ou les +obin son +nickj frost +ni wot +nb channibal +mil les +meal times +me sure +lu ken +lost inspace +lean newood +l out +kry sten +ko cher +klau dia +kevin and +k one +k far +ja ana +indv ssl +i agov +hom ology +hendri ks +h sn +gu lam +googlen ext +goal post +go yang +free comicbookday +folklor ic +erd mann +di stil +dese ret +chi dding +carol in +candle box +bush veld +bus boys +bu hle +blazed rts +bill und +bi as +barrel house +ba ster +b ended +avengers ageofultron +asi d +ap hc +after word +affe ctions +abt ballet +aaaa aaa +Ñĥ к +yoo ka +ye vo +work for +vocab ly +tyler l +tru cked +the fader +the cu +ss kin +sou they +sofrench vintage +snu ggle +si us +secon daries +scul ley +sa day +reig en +re yn +re ke +r mv +pot chef +pie tra +phoenix fc +peril ously +o calypse +ny dn +ni thi +nh strust +national ffa +muzaffar abad +messi ah +mckay lamar +maritime history +m so +level er +le vine +kon gs +kin ner +jo wls +j ino +ins worth +iam bic +happy y +ha ak +gurdas pur +gun boat +gor gie +free ship +fran che +fab ray +f rain +ecu tive +da rey +cra s +corn bury +cor dings +chu bbs +chal mette +bt ch +booker prize +bo ssie +blues music +av alli +at chaf +ar tofthe +ar ff +animat ronics +andre escu +alway z +af fords +:" ( +ðŁĩ¦ ðŁĩ¹ +âļ¡ï¸ı @ +whittle sey +wal lie +u che +troll hunters +thene ws +ta ita +ss ain +sr cm +smo sh +smo cking +sivak umar +siqui jor +sham ar +sei bel +seabor ne +sac agawea +rose bery +ro pers +rh summit +repaire r +raym undo +pro bowl +powe rexpo +pluto cracy +planes walker +pic coli +pe tawawa +pat ted +pas cha +om nam +o zomatli +neces ito +n cha +my f +mu dh +mormon ism +me ddle +mcn ally +kargil vijaydiwas +kam is +kahu lui +julie anne +jor danc +johan bbt +ig loo +ic tv +hack day +gre eeen +ge tur +flat breads +fir i +far ol +fa ja +end vaw +ence inte +eci ja +dun hill +dru mn +deser ter +de press +crink led +consoli dates +code word +coach dan +cinder ford +chrisco ons +chap els +cen i +canelo ggg +cali x +by res +bruichladd ich +bram ble +bonny rigg +bo vis +bo avista +berlin marathon +bel gard +ban ker +avi dan +aun ty +animalsin churcheshour +an diego +aldu s +academ ician +ðŁ¦ IJ +ච¸ +yr ne +year inspace +womens worldcup +westhou ghton +weekendkav aar +wave forms +wai fu +w andi +vaux hall +un popular +tro mance +travel deals +tom ita +the show +the force +th ous +ta vel +ta eng +stone mason +speak man +so bot +smart glasses +small man +skincare routine +sing lec +rutab aga +rest less +pic h +out your +nx n +nas chelli +mr sd +mollu scs +mis labeled +middle boro +mi andad +merle haggard +mckaylamar oney +may son +mad tv +log jam +light and +kis story +ke steven +kar ri +kap ta +k rang +k ck +jyp nation +juice man +joh to +jobo pportunity +inter mix +icel anders +hou sman +hof burg +ho ary +her radura +guar naschelli +gra bowski +fo k +flugha fen +fin stagram +feed stock +fa pp +f ni +est ina +else worlds +ec lassic +divor ce +dess ner +dennish of +dan ette +dan adi +cork board +christma scar +chat er +chang chun +ch ics +calab ro +cad er +bul gur +bron agh +british ness +br andre +bobcay geon +athletic club +arbor iculture +ade i +ðŁĴª @ +ðĿĹ ² +اÙĦÙĬ ÙħÙĨ +yoursan jali +yong san +ym n +wood way +whi pper +vic tro +van dien +ur sula +ul and +trumpkim summit +to cross +supanova expo +summer reads +spro gress +spo onies +somer saul +show live +shi ri +seab ury +sb scycling +sat ine +saiy ans +ru pes +rotat able +real d +potchef stroom +pirat enation +penn sau +peninsu la +pa den +p é +one music +onco logists +ne scaf +montal vo +mik ha +mexican os +meat free +lor is +lear field +king o +kali ber +jan ssens +j ingo +j adap +is man +ir radiated +inconsol able +ilove her +i know +gun volt +go karna +go down +gla dos +gair loch +frees ync +fit girl +falcon heavy +f be +ere wash +end pjparalysis +dont drown +derel iction +cut man +cristi ana +col late +cli burn +ch of +cardi opul +brain picker +border wall +birce akalay +bel ga +balfour beatty +austr a +ar vid +anc ats +am ali +aler te +ade w +aaf shar +ê¹Ģ ì§Ħ +âĺ ľ +zo eller +z apho +ya ali +william j +wen do +wel com +wef tec +vo ss +unic om +ture en +treat ment +to ib +terri fically +ta kht +syste mati +stron garm +street side +sothe by +sl ps +sin field +seemar aja +sand es +rigo berto +ridem cowboys +ra dom +propell ant +play dirty +pit z +ping tour +pau d +open bsd +olympiacos bc +nived ita +neuro sci +navas ota +my morning +my fabol +myfabol ouslife +moy nahan +max planck +marri ag +marin abay +ma pes +lar dner +lan son +ja im +inhab ited +ilove snooker +identi fiers +humb les +hex es +har simr +gri es +fundra ised +fun yuns +fort ner +eun os +eat forum +dun nott +dtp traffic +don ators +de franco +ddi qui +d pac +concer ti +co rer +co i +clich és +christ man +chio dos +castle knock +car ps +bulg akov +bud ger +bro che +brand ing +book festival +be there +b haw +b ampton +at water +al bu +akon nect +aj l +aa ke +ðŁķµï¸ı âĢįâĻĤï¸ı +ðŁ¦ ĸ +é¢ ¨ +åIJ ´ +xdanny xbrownx +x olo +ww o +wil ful +whit eland +wester berg +viral video +v schi +v cs +ty phus +tumh are +sn affle +slaughter houses +sky rail +sk amp +singul arly +si bu +shan go +sc rat +sammy hagar +rob dyrdek +ridicul ing +ri i +residen cial +rescue on +re generated +qui roz +precep ts +power house +ph oney +pet amur +pap io +oru van +official sting +o cra +ned ved +ne ipa +mynew tag +mn hs +miller music +megh ann +me issner +marvel ously +mal dive +lovi poe +logi st +lad broke +knu th +kit man +ki w +kho a +kal ing +jen nam +it ler +ho comd +hin z +green living +green baum +gra fico +giar dini +getyour rescueon +franken thaler +fiumic ino +fa wk +diur nal +co vet +clam ato +christ of +carni fex +car lee +bul ges +bor in +boo g +bo rel +bio sensors +bi fold +ate aser +arsenal ladies +alf ons +afcf ylde +adap tors +ðŁĴĿðŁĴĿ ðŁĴĿ +âģ ° +าภĩ +~ âĻ¡~ +yatsen yuk +ver ne +u son +twi g +thumb tack +ten sioner +techno park +spot swood +sp icing +sat omi +s mad +ross land +ph ela +penn sbury +over subscribed +ni ek +new stoday +muang thong +lenor mand +kul iner +ke sey +kag nihotri +jen ledger +jake and +irish art +inhibit ory +in icia +hill wood +gar o +gad os +fore saw +fif teen +execu table +dj mark +cristianor onaldo +clo bbered +ch azz +cb nnews +castel let +campbell ton +buil din +bo ze +beaver tail +asta xanthin +armit stead +algo har +afri kan +adul ation +. \ +ðŁĴĻ ðŁİī +ðŁ¥ ¶ +vs stl +vou lez +val mont +va ani +unit ar +twilling ate +ts is +to kimon +thingsto do +ter ab +te ee +stock ley +sto bart +squab bling +spl atters +sla ved +si vel +shake ology +scupp ered +practice make +parsi fal +ofthe dead +nutrac euticals +nodu le +nh simprovement +national garden +musc adine +molo ko +mo jor +ment eng +ma dox +m vollmer +lu hya +live sof +lingu istically +li venews +landscap elovers +la pid +killing bay +k vs +k ral +itsan u +it starts +inci sions +id der +hust les +ho fer +higa shi +h ø +h pt +god inez +girl shoops +galá pagos +fab rik +ev genia +ear nie +e up +du bin +din am +del os +deci mation +cwu news +cry engine +cong don +child bearing +carte sian +c nu +bur st +blue ish +bir ria +babys its +aw wh +ate le +at omics +ake chi +adelaide united +. :* +ðŁĺĦ ðŁĺį +âģ¦ âģ© +your koel +wo j +vis by +val ds +un vaccinated +teen nick +sy mes +sul liv +socialmedi aday +slo opy +sexual harassment +scarlett johansson +ready set +quebe cois +procedur ally +pro do +prim roses +power man +photo kina +photo gallery +o gie +nerd core +n ams +monmouth park +mock tails +mir an +me jo +ly nd +loyol achicago +latingram mys +ku tty +kre am +kitty cat +king splace +kind ler +jait ly +itsme leighton +is amu +ir ks +imbi bing +i for +ht fc +heer den +groun der +go thia +ge un +g elling +fur red +flipp ant +fam il +dhok la +dar ab +co sis +cen e +castle hill +boo the +bla se +bi kie +balth asar +b gl +ay ling +av neil +atlan tis +apocalyp tica +anc yl +air travel +. £ +âĿ¤ï¸ı ðŁĺĺ +âĿ¤ï¸ı ðŁĺį +wheel wednesday +weiss bier +w gi +ver ging +vair amuthu +under wing +trans mog +the resia +th ell +ten ting +tech a +su pine +strive forgreatness +stock hausen +spor tuk +sound sof +sor ana +social innovation +reu ven +redi ff +re ssler +re ms +ratt ler +punctu ate +prof tim +pro am +pirelli sport +pac ked +oso gbo +oly phant +ohi of +ogle tree +oba diah +ninjat une +mu kono +mom iji +mac manus +ma od +little bits +kir lo +kipl inger +ke dle +jun pei +joen suu +jo ao +isu zu +info world +in ty +gom i +ger hart +ge ments +followthe money +fe gan +exp ended +ef light +du pdate +dr jitendrasingh +disposse ssed +deli fe +cla vin +chap ut +cesar millan +cay ley +call aloo +caix a +c sw +britt ania +bo ssi +bed nar +bay ar +base boards +autom at +as el +alo on +aldu bt +al trock +ait anya +.... , +.. :-) +you should +wxyz detroit +white ford +wh one +venkat eswara +vac tor +un seeded +un knowing +the hotel +tex ters +sunday night +strath field +stabil ising +sports fest +spor tb +spod casts +slow dive +sleep walk +simp act +shaf fie +sel ke +sau my +ren min +redd i +red wall +recal cul +qui apo +post media +pennsylvani ans +pe ary +pat toni +param our +ou u +ou twit +os ong +os mia +octo path +model life +mis samy +miha ela +mach el +ma kaya +lu thra +let in +kum amon +king arthur +kill ick +kiaro stami +kemp ston +he av +h dm +gin day +gif te +free from +fr ary +fin ola +ferry corsten +far nes +fanni bal +do something +deutsch es +de us +cute cat +com ext +co ser +cla b +cheese making +brei vik +billie jean +anni separker +ach ance +ab ajo +aac psa +âľ Ĩ +ب Ú¾ +Äģ n +xperi az +we ert +wad ala +v adi +under funding +umichb ball +umb ral +tirmid hi +thpicture inyourphone +thanks for +ter ima +t any +squ iz +spur ts +side arm +scrutin ize +river front +ring en +renew us +re tainers +pinatu bo +paul hollywood +op killingbay +nsa ids +no love +ne om +mutt ley +multil ayered +lk ld +leed snews +kno ssos +karthi keyan +jennaf ischer +jar dines +infer ring +indiscrimin ately +indi am +ik k +i ka +house holder +hollywood musicawards +history pic +hippo campal +henri quez +hat erz +ham lyn +gri ef +good in +fil mi +fern hill +fe aster +f yr +extre ma +ear ne +e ya +dog food +dianap enty +der ailing +day le +cub stalk +comic artist +colon ised +cleve metroparks +catapul ts +carbon footprint +cap m +ca am +boule var +bou n +bal dev +art gallery +arab spring +ang poet +american airlines +ail oa +af onso +ðŁĸ¼ @ +ðŁİ İ +ðŁįĢ # +âĹ ¦ +Æ ¹ +will in +und proud +tweedle dum +tu llo +tten ham +thunder bird +suis un +su joy +strans formation +star killer +splin ters +so corro +shout out +shane warne +scot winter +sacram ents +s foster +road worthy +ri sky +reli able +ra ag +pow ley +pau le +pad ula +oller ton +og ata +nascar throwbackthursday +mush in +mile split +magnu scarl +lu re +live worx +life straw +knowyour mil +ken nan +janc is +irre vocably +ho ards +hel man +he o +great esth +ed rums +de morgen +cric ut +creek fire +cr ing +course ware +cooper atively +co generation +clu sion +ck r +che rer +cb nationals +cat tail +ca rena +c und +bru ggen +bl ms +betro thed +beach party +bal ter +asu bhash +ar dsley +anu radha +akure yri +af ell +acur ator +ab sm +âĿ¤ï¸ı ðŁıĢ +zebe dee +z wei +whistler blck +voodoo doughnut +venice biennale +vatten fall +une aten +un iter +un intelligible +twent ynine +supercalifrag ilisticexpialidocious +sun tron +stock car +speciesi sm +sp ald +sis rocks +sin noh +s folk +rock house +ri dec +rescu ecat +radio one +ra ye +pul kit +phum zile +people schoiceawards +pell erin +pad dock +over capacity +omo tor +olympic day +nt live +nam bucca +naf me +na aper +mix master +mis chief +min ori +mau ka +ma ser +ma dang +lun cheons +lili ane +li gas +la irs +king fire +kic cha +karnataka elections +jalfre zi +jak arta +intensi ves +ing g +in frequently +il h +i fans +haw kin +hau gesund +han ako +gun ston +fun di +fran xx +fa xon +es mail +el za +ed out +e ig +dull ilah +duc tions +defe cte +cro at +congre sses +con agra +chemi e +chae won +ch á +cel bridge +bar zal +bal doni +av iso +arow ana +ander matt +akit aranch +activ ators +< ---- +ðŁĩªðŁĩ ¨ +âĢ¢ _ +x ly +womenshealth mag +win and +visit york +vanoss gaming +ucan ationals +tr ps +theri ault +tfo ir +tassie keith +tai sen +steve vai +spiro graph +ski ft +sak thi +sab bat +richar de +pur ton +pro tractor +pro clamations +pre witt +pn v +peat land +oo hh +official jaden +ob inna +nzv pak +mike posner +mb log +maw gan +lothian buses +lic enced +li spector +lady leshurr +k rep +ing stone +in experience +i stom +hen ric +heis ler +gu g +gor den +gali ano +field stone +far nese +emboli zation +dho ti +demon ized +declar ative +cy m +con nah +con akry +cj tfoir +cirque dusoleil +char mouth +botu linum +bis ummit +bier ce +betterlate thannever +best buddies +arri an +and al +allato ona +ak tion +ðŁĶ½ ðŁĶ½ +ðŁijĨ ðŁı» +ðŁĮ´ # +zz ani +yogap ants +wyn num +wend el +we ichert +vir u +vi raj +v le +us ica +uof nh +ul an +tito jackson +tir ta +the urban +te tten +sung kyu +straigh taway +shail a +segam at +sal miya +road race +rhy mney +re routing +ray ray +ramadan mubarak +rac v +projectcar sgame +pou le +pom pon +per na +pack wood +nor der +new type +mit ski +ma gro +look slike +li ssy +li sn +jo tted +ji ofilm +jen is +im mor +ie b +hi pped +hed don +hac er +gre asing +gop convention +go gold +gan ge +ga str +fuj ii +fil am +feeding america +en rage +e ki +de is +cu u +cir stea +ci ani +c ju +bre ich +boot loader +bitt ner +aw alk +avac ado +ant elope +âĺĢï¸ı @ +woody att +wig town +week ley +weather spoon +wann able +unconditional love +un dignified +umat illa +tour bus +syste matic +sundance tv +strong man +spec tro +spand au +skin tight +se guidores +se btsb +sau cy +sar it +roberther javec +re direction +qaland ar +priyad ar +philando castile +petr one +pe ppard +p cap +out of +ob its +news agency +neu haus +mid den +men swear +meet in +me doc +mar get +mac room +lin gered +la van +la gg +kontrol freek +ko hei +kingdom come +kai ju +jamesbay music +j lf +ins grove +in ment +i ddle +honeo ye +he anor +griff on +gra fica +gr itt +ful mar +flan king +fact maniac +eye less +ev ene +en by +did act +demo ss +deli sting +d nr +crou ched +cote depablo +chor doverstreet +brom field +bri quettes +brahmot savam +bi ek +believe survivors +be headings +bark ads +art sat +an vers +all soul +akade mi +ðŁı ¦ +âĻ Ĥ +¨¨ ¨¨ +y sen +whistlerblck cmb +wayne brady +wan s +tin ka +summer hall +subordin ates +stan ge +spor ter +spor ch +soo bin +so gni +sig no +sa rea +rogue ales +reis man +re buy +radi al +r pio +pro create +pre formed +petre ls +ori ed +on enation +nautil us +narasi m +mex it +mer kur +list enable +just ing +inju n +hyper sensitivity +hugh hewitt +hu obi +hu gg +home security +hi elo +he yes +har on +gri dley +gre ger +gine ering +face hugger +eng s +doctor ing +det lef +cyclone idai +compos ite +club legend +chil lax +cess nock +bo para +attack man +ariel helwani +absac ape +ðŁĻĮ ðŁĴķ +ðŁĵ ķ +ðŁıĨ ðŁİī +zo bel +world heartday +wo ad +wkr p +wing let +whirly ball +we bbs +vive kagnihotri +vi varium +vel vel +vat ron +v app +um unna +um ps +tu it +ti ze +the group +terry fox +stu ary +stran ja +ssi en +soni sphere +sel ma +sd ss +sau te +s gg +ryan paevey +rosen crantz +proudly sa +priv atec +phineasand ferb +phil p +petamur gatroyd +pend ennis +ofthe year +north man +ne bulous +nb nationals +nav our +natural wine +mu cc +middle port +mas lin +maj d +lotte rer +lo ks +lo den +liz ette +kla homa +kho s +ka unda +jeon buk +inatur alist +humph rys +heer len +ha gelin +gym no +green light +gra phe +gl is +flo aters +fle cked +fe cking +fair cloth +et ine +en ning +del phia +david k +d wick +co qui +city link +cb se +camp amento +cal pers +buzz i +bru un +bin dery +ber ney +bath water +at kins +ani eri +akro tiri +. ðŁĴļ +ðŁĮ³ ðŁĮ³ +za bel +z wei +viri dis +unis ys +tere sh +su id +startrek tng +sin novation +side track +sc all +re launches +re ax +proudtobe afan +pl d +phal en +pel low +pan ter +onep age +of light +nak ata +metropolit an +melb weather +me all +loh man +leni ency +lee der +ko etter +ke al +kau ka +kas sa +jr smith +jar head +irrit ations +ir responsibility +inter varsity +infin ities +idu kki +hot cakes +historio graphy +hel mi +gu bbins +gro t +gr c +gg al +gam per +ga illar +engag ed +en sley +drewestate cigar +dess er +dang a +cro ston +cater ham +carmen ere +bru ff +bra bin +bal ta +awe a +arunach alpradesh +ared cross +angpoet nyo +andre wc +alleg ori +aal u +a ow +: "@ +!! ðŁĺĤ +ðŁĻĤ ðŁĻĤ +ðŁĺĭ ðŁĺĭðŁĺĭðŁĺĭ +ãĤŃ ãĥ£ +âĦ ĵ +z wick +youngg uns +wool fe +wild london +vi kh +uc tions +tri go +too cool +thereal rvd +the view +super cute +su blux +spe sial +sp ree +shwed agon +ser da +seman asanta +se stero +s store +rou ch +railway seva +pra gy +pit o +pi red +perfec ts +panam ax +p tes +oy als +ol son +o ved +o ao +mou lder +mon bebes +mo xon +mckin lay +mck oy +mar aj +maniac ally +mag navox +ma kara +m ry +lit us +lay lee +ko komo +ker ne +kent cricket +kad ai +ka day +io ana +innu endos +ic caworld +heroesofthe storm +hasling den +go akonnect +gi gas +gh gs +fox worth +ey yyy +espark global +e es +do bro +delav an +cyber bully +cot tee +chrismur phy +cattar augus +cab ramatta +c mes +bu fc +bce agles +ancient greece +alan hinkes +ae olus +" ðŁĺĤðŁĺĤ +ðŁĺŃ ðŁĴŀ +ðŁĴª ðŁijı +âĿ£ âĿ£ +x pac +weston birt +travel writer +tou galoo +thelittle mermaid +thak sin +t dov +spri mary +spee dos +slan dered +shi ela +sharpen ers +see u +sch um +roy don +ri era +red lips +raaj je +proff itt +pig farming +pie bald +pe tani +orchestr ator +om ha +ny anza +ning xia +ne om +mind hunter +mc grew +ma sy +lux ottica +lu chad +land reth +kan o +kah ler +iron mongery +inter ment +ing machine +ine yard +im thankfulfor +her mie +gleni ster +gill on +from software +flo c +fir min +fion nuala +fin ality +feliz sabado +fabric london +f omen +evangeli zing +ev and +eric mccormack +enfor ced +end ar +el ocke +dru ga +dianer avitch +dell inger +cv shealth +cr andon +confir a +clin ica +chop p +ce ylan +cas cara +carbon iferous +cali ban +ca zen +bran chs +boo oom +bl are +bi ji +befu ddled +as car +and star +aar ad +_ ? +ðŁĴķ ðŁIJ¶ +ðŁijĬ ðŁı¿ +ãĤ¹ãĥ ŀ +ñ o +za it +z g +wx pn +wur ster +us mc +ty balt +twil d +trump ing +topic ally +tol lywood +tod ds +thar parkar +th agallery +tel lus +tan alysis +stri d +spy ros +sop hs +sn om +slo tting +shu ang +sar ker +san kt +sam harri +re interpreted +ra fe +pizzic ato +pi eds +over shot +or not +oil spill +mun den +mo ton +mi hi +mchapp yday +mc delivery +man nnn +ma sina +link building +li at +lar ter +kon ser +kaw aii +kam ina +justin suntron +journal news +jo va +jay enge +icar us +hu zur +hall berg +half time +gitt ins +gerard butler +gel derland +gay boy +film fest +fen ster +face paint +enam els +emili aromagna +edi fying +des borough +dean heller +dar b +cnn travel +cleanpower plan +ch bosky +cau cu +bill peduto +big lia +baja j +b ø +auto biographies +ash lyn +ascri be +antico agulation +anthony horowitz +anom yces +ann er +! -- +ģà¸ģภģà¸ģภ+ðŁļ Ħ +Ù ¹ +wom xn +whl hitmen +we fts +vff vishal +vet tori +ulte gra +uci wwt +track listing +tough enough +t st +stanley cupplayoffs +snail mail +shant aram +sever yday +seti awan +scaram ouche +sau ced +sa are +s freedom +ri vard +quan tock +protec tive +portman teau +phumzile unwomen +over confidence +ov ation +mon tee +mehro tra +loccit ane +live th +li ming +kra is +josh gad +jobseeker ssa +iz abel +ic inema +ham in +goo py +fei jo +ew p +est us +do good +deeper learning +dan los +cor dia +coo kier +co key +ch affin +ce as +calgary transit +bor an +bly theville +big basket +bas ili +baj payee +awal ha +auto pia +ann aya +ac ws +absacape epic +a app +__ , +ðŁĻĬ ðŁĴķ +ðŁĺĤðŁĺĤ ðŁĴĢ +ðŁĶ¥ðŁĶ¥ # +ze stan +yah u +we irs +vou vray +voi ded +untol d +ug l +twitch stream +tutic orin +trade fair +tobogg aning +toa dies +thar u +tang led +su af +strom ness +steve dave +squir ming +slo gging +sil om +sc y +rival scamp +re locations +qu onset +poly gonal +politici ze +pember ley +pavel ski +pa j +not is +nishi ki +mothere arth +mor oney +men ina +mam bas +mal k +m cu +lor ra +lessthan jake +l ko +kor dell +kal yp +josh frydenberg +heather peace +h sf +good child +ger hardt +galli fre +farrow andball +eric balfour +el kie +dr s +dj k +diplo docus +de sailly +cynthi aeri +cynthiaeri vo +corn hill +conver ge +chaz elle +caris brooke +bri ant +breaze ale +blaz ey +bend el +b pi +atta k +ambi gram +am ii +akih ito +.. âĿ¤ï¸ı +! ðŁĺįðŁĺįðŁĺį +ðŁİ§ ðŁİ§ +ðŁį» # +âĻ Ģï¸ı +ঠ¼ +zen berger +yal da +win ders +where smy +washington state +w py +w boc +verge currency +ve les +tur fed +tu beli +tra pt +thereal swizzz +the bookseller +ste ttler +si mono +selfies aturday +river city +ri ese +relati vistic +raz dan +rath farnham +radi i +pree mie +perpetu a +op teryx +omo to +om ey +nicholas ville +my pov +my haver +mor rin +mo der +mis alignment +master killercat +mar ucci +magnuscarl sen +lu gged +low veld +lou reed +liber té +let tre +lang one +l lanes +kur umi +ko jic +ki kk +janes addiction +jac burns +j sd +i uk +hus bando +hou sat +hillen brand +heuri stics +head dresses +grill z +green roof +g suite +fukun aga +f sd +episte mo +eleanor tomlinson +east ayrshire +du rag +du hok +dor inda +donthe con +dont mess +do xie +de sa +dcu o +dar zi +cric ci +chuk ku +chis wick +central america +ced rick +carab a +bra dd +beach in +ash ak +aser vice +ak ki +ðŁķº ðŁı½ +âĶ ģ +zar ah +yo len +whow ill +wether spoon +va jani +ur gess +tsogo sun +tro pea +tot tori +tokimon sta +te gu +subscription box +strath aven +ssss ssss +shepher ding +seraf in +ri ddle +rep as +rel v +refra ined +ree du +raj as +par des +offro ading +nu ss +no stri +njor oge +navi es +mun nings +ma kurdi +liver pud +kat ju +karu izawa +jamest aylor +j ca +income tax +hel sby +h ly +gu ssie +gr anti +frog fish +fro w +endeav or +effi gies +dé j +dro pp +dread fully +do go +dh w +demo l +dat aware +da cha +coven ant +compul sively +com in +cel and +brett anomyces +boys noize +awesom econ +austin aries +asha hidi +ague final +ag ris +adhe era +accordi ons +abi er +. ðŁĺ³ +ç¾½ çĶŁ +yaw key +what it +we be +wb homeent +v nd +u ht +toyotag b +th uk +tartu ffe +sub floor +sp ga +shank land +sever na +secur itization +school holidays +ripp rince +ri zza +reak tor +rachel platten +popcorn day +poly phony +pickn pay +our is +od sc +o kes +ne olith +mythri official +mu sher +mr v +mirand akerr +me tball +ma gam +m clarke +ludd ite +leim ert +leee v +kt lamor +k aus +it pro +in ol +im printing +il more +hugh ey +hot deal +grized ale +glen shee +gen est +gan esan +gab at +elfy nevans +duckdynasty ae +doing good +dc v +dau ber +cron je +cityof melbourne +chan go +cel lists +cav in +categori zing +ca ac +burn t +boast ful +auto gas +art fund +arba az +adidas us +accredit ations +ðŁĶĬ ðŁĶĬ +ìĨĮëħĢìĭľë ĮĢ +ãĥ ® +âľĮï¸ı âľĮï¸ıâľĮï¸ı +z berger +yak ima +wound care +woke up +wil dearth +wil braham +warra gul +v any +tennis channel +team di +tarpor ley +target style +tan nen +stri stan +stabil isers +software ag +shel ford +seran goon +satyan adella +ro bri +plat zman +pi at +north bridge +mun ia +mpha sis +mosque attack +mom oko +minneso tan +min eta +mickle over +mal ki +ma pre +le amichele +lb cc +land cruiser +kas ab +k ely +it ne +int endi +il ta +i ye +hyper icum +hal am +ha dj +gram ophon +gr aca +go beyond +gd xj +findac ure +fau bourg +fair light +fabric ators +estu arine +endu ro +emb ra +electr ici +el ç +doodle bug +di ye +desp airing +del dia +de hart +d tb +d com +colom be +citizens advice +chao sium +bro man +briga deiro +born thisday +boccon cini +blu enote +bike suk +berkle ecollege +baili wick +anasta sio +allin cle +air baltic +ah mar +adel phi +\( ´ +ðŁĺª ðŁĺªðŁĺª +âĤ¬ / +à© ĭ +yar ashahidi +y ria +wim berley +wak ing +wa ren +toll gate +thunder y +tb w +tag ger +t illed +sur yah +subju gation +su sd +stend hal +stel ar +stat news +srin u +seab orn +sclu bs +sch ell +samharri sorg +salt iness +rust ington +risd greatness +reque st +reflec tors +rainbow dash +ra abta +prayag raj +positi on +police state +over wintering +orient alist +orb án +opportun ism +new sham +mccarthy ism +marl borough +mand elson +mand arina +m bro +livin ghistory +linch pin +lec avalier +lazy bones +lago m +l fm +kie wit +k ops +jaz ira +hydro gels +hull kr +hu bie +har pur +h st +guard ado +gro ene +gor ies +gna sher +ger tler +gearbox software +gad wall +fri ars +ebay seller +dr ace +dhar wad +den smore +dd x +damian lewis +counter punch +cor ran +controversi ally +cho ic +chlo elu +chill on +che shi +carbon tax +bryan dechart +berk shire +beli e +be side +bar rescue +bandi pur +baghe era +badger mbb +ast irl +asdfghj kl +aro th +anciente astirl +achi bi +ace supported +a asu +ภħ +wro ble +wood hill +will unga +welcome home +usur p +un in +ty as +team cavuto +t mt +sukk ah +sque aled +so sad +seduc er +se amu +santaclar ita +ro cc +re evaluating +pre conceptions +pli ss +palladi um +ous as +on racing +on assignment +obl iteration +morethan adodo +mir ates +melind agates +mas jid +mal do +making ithappen +lc dsoundsystem +ktlamor ningnews +kry stian +kra al +kalli o +jacob hoggard +ing all +in cr +imprison ing +implic ate +i sic +henne sy +h miller +gul ated +gu dda +grin gos +good olddays +go emon +g iler +g anta +foot man +f tw +er ba +don jon +doc sis +destruc toid +dann ys +construc tivist +cler mont +car mine +canadare members +can ar +ca zeno +c be +by example +bir ney +beve ren +ben y +bate man +bat l +basketb alls +bad ar +babbac ombe +at si +an si +ame ren +alla ire +air por +ðŁĺĬ " +ðŁĹ ¨ï¸ı +âľĮï¸ı âĿ¤ï¸ı +wol len +wec twx +wa qf +uof c +u mau +tul alip +travel and +the bookof +th re +team god +tam ashii +ta phone +syco phant +sushant singhrajput +sun iv +spro tt +siss ay +shel tie +save baloch +sanji v +sa wak +roe bling +ro jak +resi stencia +r deye +pro xy +prisma drop +poli zei +pau illac +pais leypark +oaken fold +no ps +narra been +n hat +mill ilit +mill ar +mary port +maniz ales +maithri palas +lep tin +le pe +lar oc +ki bler +kex change +kati epiper +kathryn bernardo +jancis robinson +intere strates +ij tema +i vi +hunter don +hal mstad +great things +gotom eeting +gh ur +frequ ent +flori dam +et itans +ell ines +ed ington +e bulli +dwarf ism +dn vgl +diso wns +dis assembling +di vison +de um +dann apaola +d bn +cur ro +corner stones +cor vids +c ica +bye felicia +boy fie +box uk +black ery +before thestorm +bal ck +ati ma +astri d +arri aga +amar na +ag it +abdou laye +ðŁijī ðŁı½ +ìĸ ij +ãĤ ľ +´ ) +yepp oon +y gritte +tur rentine +tl f +the water +ter on +tae gi +ta ines +swat ted +stein er +snar led +shum or +senior living +seg af +sch rock +sant angelo +s dream +roman atwood +pti family +primary day +presiden cy +police media +phlebotom ist +phan tasia +p ny +om bra +olom ouc +n illy +mu lai +milit o +mel brooks +manhattan henge +mang aka +mac world +lose it +little borough +lascru ces +kill aloe +kast uri +karim nagar +je hu +isi ah +iit tala +ig da +id v +id ar +ha za +gur khas +gunnar sson +gla xos +gen oot +for ten +ey ah +eve ready +eic ma +ec v +doll ard +den park +dab bled +cre tan +co cin +circas sian +cic lavia +ci ena +christ elle +chol as +cat love +cal me +c cha +bts v +booth bay +bb bs +ax alta +ark ady +aad hi +ðŁİ ł +zoom tv +y dr +wester ni +wal drop +vil akazi +vascul itis +tw da +to taku +time machine +ti ppers +teen sy +te trick +te hama +st any +sher ilyn +rook wood +red hook +re avis +qu aver +q alam +protector ate +pro phyl +post it +petti bon +pascual inigo +p online +opent able +nu be +no tim +no onday +mytho logies +morde chai +modu l +meg af +me che +mc elderry +mar veled +man h +m ste +life jacket +lgb thi +le dyard +land form +la ko +l de +kun kel +is am +indic ts +impeach ing +im pascualinigo +icon o +i shares +hoo pin +hocken heim +gu ta +gran fondo +gou ges +gentleman jack +gam asutra +food day +fire balls +exoner ation +em n +e zi +dp wh +desider io +d jan +congreg ating +com ber +centi pedes +catoc tin +carson daly +cag ey +beli veau +ayl ward +au b +af sc +® ¤ +wy land +wash basin +vi eng +ver us +val eo +ty ronn +toko pedia +the mm +ter ram +tand berg +stell ung +staff spolice +sm iller +slen ses +sierra boggess +sarah j +russell tovey +resi zing +rap zilla +power ofthe +plu ckers +phx cc +ni zhny +ne akers +nas reen +na aaa +murder she +moko ena +mojom agazine +mit press +mechan o +mayweather mcgregor +mar clam +kimmel center +kim rhodes +kemp ner +kee ton +jun gh +invasi ves +inter dependent +ib ps +ha ie +h ref +glaxos mith +festi vely +end papers +dren ch +daily doseof +d aki +cu lotte +cross dresser +crock ford +cop ts +co zi +christen son +charmeck schools +cb ssf +castle berry +carpen tier +canad ain +can zone +c son +buss iness +bts loveyourself +bra da +ben zi +bas sembly +barin holtz +b vp +au gur +anson mount +anit aha +aag adu +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ +ðŁijĮ ðŁĺı +ðŁĩ± ðŁĩ· +å® Ŀ +âľĮ âľĮ +yu asa +y ane +wthr sunrise +wood chip +wak ely +under pin +u cino +trib live +t mp +swan wick +stre ich +soo on +single use +si ol +sha itan +redribbon week +re um +raw ness +qarab ag +pro pel +pit ttweet +pau sa +patrick t +pal mistry +ode sa +nail sea +na har +moto x +moor town +mon tre +mee se +mal vinas +live smart +lique urs +lin zer +leol aporte +learn french +ld w +kol by +klon oa +juxta positions +judy garland +jcde caux +jay weatherill +jay esh +ja el +ito to +invali dated +ini st +id hun +har ro +happybirthday justinbieber +hand bill +go live +get thelook +g alea +free wheeling +fax es +eye mouth +ep ona +e or +dupon avon +dont missit +dar ran +color ism +collier schools +col chester +cn v +chrismurphy ct +c party +bre er +barrym cosmetics +bag oong +auto crat +amne siac +all night +ak off +ab ri +a show +: *** +âľĭ ðŁı» +âľħ @ +world suicidepreventionday +wl w +widne srl +wai fs +vegan life +uk is +te ahour +ta pers +t india +swithout limits +sr q +squawk cnbc +sour puss +shal ini +sec unda +sat ra +sam bas +recomm it +press meet +pin tos +pap ayas +pa yoh +ox as +oshiom hole +or cla +oak brook +novic hok +newcast ler +mv cv +mor van +mant els +man era +makar ov +m te +lu kens +lily collins +ko tt +kja er +khur sheed +ke ach +ingex cellence +immuni zed +honor thefallen +honey badger +home wardbound +hil and +gord downie +fel den +fam bam +dl mundo +coo ky +chip day +chang s +cal ea +bun ji +bra infood +bos mayor +bon line +blephar oplasty +bee man +base board +bam enda +aw memorial +avail ab +andro gen +ana is +acoun ty +? '. +ðŁĴĹ ðŁĴĻ +ðŁį ¤ +ðŁ¤£ # +ìĹ ł +âĹı âĹı +zulu eta +yoshino ya +worldkidney day +whisper ing +wb z +water colour +w one +urban ized +uppere astside +turnaround dontdrown +try pano +tiru pur +the gautamgulati +the darkness +tan am +surface pro +spino saurus +sky land +si endo +shou sem +sebastian stan +sch ini +rob kardashian +rizzle kicks +rebec car +rawling ssports +rallyfor rivers +q ia +provision ally +play doh +plasmo dium +piñ atas +philosop hy +paulsmith design +pagli acci +om gggggg +nz dusd +nil ai +nat sumi +napp ed +my ride +mud guards +mass aman +manek agandhi +lush ness +lor rain +lor an +log anc +kno blauch +kim m +kad hal +je thawks +issu ers +intimi dator +hot stuff +har ron +hai les +habitat forhumanity +h le +gill i +ge of +finneg ans +embed ded +el er +dic amba +d mcc +con dell +chine y +chan dos +chan cey +cdn media +bu ge +bre uil +big bird +argu ello +antimicrobial resistance +anti go +ad lon +ac amp +! [ +ðŁĴĽðŁĴļ ðŁĴĻ +ðŁıĥ ðŁı»âĢįâĻĢï¸ı +çīĪ æ·±å¤ľãģ®çľŁåī£ãģĬçµµæııãģį +ç ± +ঠ° +اÙĦ Ùĩ +à ° +yo gag +xl vi +wayne twp +wap da +ven ic +van guardia +vam ped +tradition alist +ti ina +sussex wildlife +sun splash +som aiya +sin den +se sar +se ha +sco bee +sar od +sal ima +sachinten dulkar +road safetyweek +redefin ing +qu abbin +pri ory +pier i +pier head +pennsau ken +pau land +par v +off shoring +nag as +n all +mut lu +mu chi +moldav ite +mid as +mey ero +mention perfection +mat amoros +magic leap +lush ltd +lu li +le vo +le aper +laker nation +kra k +kevic lifton +kel t +jurassic world +john carter +jarim atti +jarimatti wrc +ity now +insta style +im plausible +ide alized +hand son +ha shes +ge minic +gameofthrones finale +ga eta +franken weenie +fl ou +fir angi +film news +f bic +ent ices +earth week +dut chie +dino zzo +dh aliwal +dev ina +culche th +clou seau +chas ingthe +ca pos +bush walking +ben intendi +arlington natl +ar pa +ar beloa +and ym +amy leeev +ak ini +af terel +aard var +[ ' +ðŁ¦Ħ ðŁ¦Ħ +á ĸ +à¹Ģà¸ Ĺ +zayto ven +zain ab +yo weri +van fleet +ukem plaw +uk butterflies +trige minal +ti thi +ti pico +the juan +the beachboys +speake th +smallbusiness week +selec tor +sauber f +sare made +sam er +ress ata +recl ined +rac ist +pro yuvraaj +pre serve +p outs +op lay +olu mide +o tran +navy daily +national school +n oooooooo +mu sonda +mu dge +mar cellu +lon gi +lic on +le aching +kin deren +ke sq +k be +jun ger +jeff ry +je zza +je para +james blake +jam aa +ja ago +immedi acy +herd fb +gun slingers +gore tzka +ghost signs +gav roche +gart land +gar mo +full body +film music +fi fi +ev f +e gal +e banks +e ady +di fe +dfw traffic +dar nley +chou dhry +bridge view +brick layer +box en +blast off +ba oli +atal unya +ang ood +ain yc +abscon ding +ðŁĽ łï¸ı +ðŁ¥ ĸ +ðŁ¤ ¥ +íĭ ´ +à¹ĢภĬ +ÑĢоÑģ Ñģ +wak ati +vy pe +vene z +ve ia +uh in +ty rosine +tu as +tro tman +tore ba +tol ka +tim and +tig ny +thig pen +tel lem +tas krabbit +tarong azoo +tag uchi +swedi shousem +swedishousem fia +stop brexit +sr hs +sped als +sno res +ske ena +sch ange +sal leh +ru airi +rouss os +rodney atkins +q west +people pets +ori zon +obstetr ician +nwob hm +muzaffar pur +mrdavid haye +mob sters +mo fongo +mir cea +mer y +meang irl +math letics +mac neill +kron er +kill this +kai greene +ju dit +indi sci +horse play +help the +haz ar +gw d +grave side +gram atik +gonna hate +gas olina +fre search +fair ings +fad er +every simpson +est eli +est ela +ellic ott +disney junior +dirty bird +dheer aj +de grades +cu dahy +crimin alized +coren tin +con k +con forms +col den +cloud scape +clam or +ckin ney +ch evening +bra zing +bluenote records +bat tuta +baq ir +bang erz +bal der +austin town +as inghe +al endar +@ ... +ðŁĺĤ ðŁĺĭ +ðŁijı ðŁı¿ +wild card +we broot +vand aag +tor ode +tom ie +thevamp stristan +thel auren +tan jore +syn ching +stu mpf +son no +sas sari +sall natural +ron o +rethym no +repleni shes +raig ad +pub media +port sea +paper weights +p day +or ton +oni stas +ol our +obey giant +ni it +mucos al +mu dug +mi tho +marcal mond +lit en +lec c +khal eel +juli ane +ji bs +intric acy +i han +happy dog +h vs +greg pak +good nite +gn omeo +glam med +gen erics +food coma +fie sta +far nam +er ra +empres as +e met +drud gery +doit for +disembar king +did entity +chloro form +cebu ano +catt elan +car ini +car audio +can er +bul ilit +bol lo +bidad ari +bi os +bell er +ash dlmundo +ari da +ame ba +ab iz +ab im +å´ İ +yur il +wen ig +we gen +walk with +wal le +wait akere +ve za +ut m +trainst ation +tou raine +thre shers +the celtic +than s +ter raz +stephen mulhern +start list +solidari dad +shepherd stown +sfoster nyc +se spn +say le +sau sag +sal afi +rock thered +riks bank +regre ssing +r anna +push cart +play it +pack rat +pac west +orang erie +optimi stically +omis sing +nikon europe +natur alism +nan ton +mosth and +mccar rick +lik ening +lar sen +l anna +kwe ku +ker bal +kan chan +just ino +jor dand +is no +int n +imper ing +i fam +home track +haver town +green live +fron tex +fizz les +fir stin +do ce +demetri os +dave grohl +coven an +clerk ship +chrisvan hollen +buster keaton +bri den +ban fi +aw ful +ach rafieh +ðŁĺį ðŁĺļ +ìĹIJ ìĿ´ìĬ¤ +yl g +ye mpire +wire work +wbal tv +wan ed +wan e +vegetarian week +ur h +tre mendo +trans gendered +tide as +thro ad +tar ly +st thomas +sn bc +shi bain +shadowhunter schat +se ty +schwar tzel +sch u +sch ls +scat man +saf in +sab y +rubi ks +ro whouse +rip cord +rem itting +reformat ory +re ise +ramad ank +pren up +photomani pulation +opel ousas +mill street +merr itt +me tin +man teo +latitude fest +kel sall +jummamu barak +jer ked +jen nas +jabberwo cky +ja ins +j hc +ij e +ham by +grou pers +gon za +gl one +fre eu +fe int +f book +exchang ela +evic ting +en ery +en counter +dy ersburg +dri k +dis band +cur bed +conge stive +bro th +bier zo +atem ple +asil va +ap ig +alder men +al ye +aby ne +ðŁĴĻ ðŁĺĺ +ðŁijĩ ðŁı¿ +رÙĬ ÙĨ +yalo va +x large +wr ld +wizz air +war ley +vote conservative +visual kei +ut arlington +united sportscar +uncler ush +un couth +twee ples +thi splace +tad ao +ster oline +ss ays +slu t +scrn ch +sci oscia +ro thenburg +rid wan +qu ai +play day +pic atnoon +ph rma +pen et +or mer +nascar throwback +nar dw +mong kok +minic ab +megach urch +master minded +livefor music +lauren pope +kellyand michael +jay nes +ip aul +interven tionist +icab nf +ho ggs +hiday at +heart gold +harri ett +hand crafted +girl slikeus +ge ty +gastro pod +gall icabnf +fu quay +er red +elo dge +eg mond +def ile +day sleft +dat at +cre scents +coy m +columbi ans +cent eno +car acha +bur khal +br rrrrr +bf goodrich +beauty fromitaly +ban dol +antmanandthe wasp +ag os +ab han +ðŁĸķ ðŁı» +ðŁĵº @ +è ¯ +âĺĶï¸ı âĺĶï¸ı +z army +woking fc +wink worth +we wontstop +watt ack +vfl wolfsburg +twist cone +tusk en +trap door +tha ana +swag g +sti ffs +speed ily +speak ing +sak is +ro hm +red pill +ra um +r sac +r pf +pu f +per fs +over confident +ning ton +nin comp +netflix india +nas u +mura bi +monoli ths +mon to +mo hn +mi zen +map monday +man gement +le derman +kear sarge +kam ps +jam ir +in ni +hun di +hon tiveros +hetero sexuality +guid i +gor ga +gol fuk +godzill amovie +gavin free +gateway pundit +free zing +finola hughes +fer um +explo rey +ess sss +el ft +ec ca +don agh +del arosa +defaul ted +de fac +buil tto +bu sines +brown out +blue jacket +black house +ber nies +ar ango +aqu af +anti gens +al pin +ak agi +absol ve +aben omics +ab dalla +ðŁıĨðŁıĨðŁıĨðŁıĨ ðŁıĨðŁıĨðŁıĨðŁıĨ +ãĥ³ãĥ ī +wyn ne +whomademy clothes +westworld hbo +we ct +wakaflock absm +wa a +w cac +vie jas +u hb +ti ri +ten afly +spy ker +slu gged +san frecce +sam eness +s zu +s ition +ro my +rat chaburi +ram bin +rach et +pul led +prote ase +po temkin +photo synthetic +pal imp +nr tnews +non tario +net worth +mo dica +me withoutyou +manekagandhi bjp +li ph +ler and +l sw +kryp tonian +key tnc +jor ma +jobless ness +ir reconcil +hin shaw +fleish man +event management +es bjerg +equal ising +easter ns +du bia +discu ssant +colai ste +clec linic +choice scifit +boot leg +biltmore estate +be eco +bark ada +ar ou +al aki +akim bo +ad omin +_ (: +wren n +world tbday +woo ding +winter park +u mana +twel vyy +to phat +tdam erit +t jp +stra in +so is +sel iger +sal in +reli ent +refu ting +pur ch +pu rex +pre teens +poly chro +pet sathome +oo dle +olivi ach +nistel rooy +mul lane +mon tini +moham ud +mir u +medic alassn +mcham mer +mc cly +man mohan +linke ddata +leth ality +legal news +kwi atkowski +kir ks +kim ye +kerry jane +k Åį +jor ger +jk rowling +j muk +iri dium +intersec ts +inlan dempire +infl icts +humanright s +hr k +head masters +harris jofficial +ha sak +gre ased +grass fire +grain free +gonz ag +gian luigi +future offood +fri ende +fran ch +for mas +fem icide +fa wn +err orist +encro ached +ea z +dom an +defi ance +compos itor +clar kes +chan yeol +car line +bre ss +blablac ar +bl dg +beat en +bam bang +aquari um +amer medicalassn +alge meiner +al gé +after words +ach ile +ac ic +zero ing +zay at +whit te +wb sc +tyrone gaalive +the source +strip tease +singh e +si i +shu sterman +shay carl +sa thi +reni shaw +re tto +rack mount +q rl +pray formh +pos ites +pork pie +phoe bus +pav lovic +ozz fest +out sized +orom octo +notin this +neu berger +mun k +mississipp ian +meg acity +mat um +masculin ities +le vens +la sko +kl ick +kirk cud +kar men +k auto +jodre ll +j michael +it showbiz +independ ant +incar n +ic v +hondac enter +handsom eness +guru official +gott lieb +gold member +go west +fron trow +fle isher +far uk +fabri que +excu se +ef dn +eck man +dalla stown +d hau +cu ed +childrenof syria +ch l +ca sti +bur chett +bu cees +boge yman +bl w +ber inger +belitt ling +bar ti +ay le +av owed +asi mb +art ful +ao ta +ample forth +am eland +ðŁĻĭ âĢįâĻĤï¸ı +ðŁĮ¸ ðŁĮº +ìĬ¤ íĥĢ +åı į +zom ato +yarmul ke +wx yv +wen onah +wang anui +veng aboys +vegan recipes +vau dev +ultr alive +trot sky +than ka +sun deep +summerhall ery +split svilla +sp uri +slo v +scri bbly +sau rashtra +roger moore +rel ented +reinde ers +rand ers +promo tor +pre calculus +power wall +pi ot +phy salis +phy lum +pet ch +peel policemedia +orgre ave +or rell +op ress +ob elix +n ho +mon net +mi yan +maj er +mafi keng +lon abscbn +li mped +la e +kou libaly +knowledg able +ki pping +ki me +just y +jonathan rhysme +jo gged +inter ac +imper iled +hugh hefner +ho soi +han key +finger lakes +fav pic +fart lek +epil ator +enth iran +en ext +effici ents +edinburgh rugby +dog days +defin ing +de simone +de mario +david hogg +da ou +cr z +col mc +co quet +c sea +bol an +blue jackets +bipolar disorder +bhand ara +bbc motd +as are +ari ste +allegori thmic +ah ir +afi q +ðŁĽ ij +ðŁĺı " +ðŁİīðŁİĪ ðŁİĤ +ðŁİģ ðŁİģ +ðŁĮİ . +zi kr +web masters +up show +ton ko +title ix +tim bered +thor naby +te acup +sydney siege +stroo tman +stri py +shol ing +sho lidays +ru ang +roy g +rockab ye +re wind +ram zy +pots damer +polymer clay +poin tuk +photo bomber +philadelphi ans +palae onto +nar anjo +mysti kal +mtn za +mosco u +mo do +mis spellings +min new +mar son +magister ial +mag yar +mafal da +lew drp +lei per +lav ash +la follette +kno kke +kinna ird +ja res +in and +i roc +hos le +hepatitis day +gg r +gather ing +flat woods +ev eline +en close +elek tron +ele x +e ales +drown ings +dr c +dilu ting +dhanan jay +den ner +del ario +deer hoof +ctv news +cms gov +cer c +carin thia +bun ching +bu zan +br ong +bouy gues +bles sup +betsy devos +be here +aú paatleti +axi om +attemp loyee +ati p +assassin ating +alter nacomics +aacpsa wesome +ðŁijĮðŁı¼ # +ðŁ¥ ĵ +âļ Ķ +ª ¨ +witt mann +will erton +west land +tn rs +than ior +terri o +ta hira +swisso tel +swer ve +sw ir +subhash ree +stro ther +stain er +st res +sn cc +sli b +sk orea +sil kie +san toni +red wing +re packing +rabin dra +quar rying +ps ico +proto col +pre yed +pin kri +ny ac +nether world +ner ys +ne ca +monclo va +mau ger +mal functions +makh doom +ma him +kud la +kaz ee +journ alist +jing ling +jeal ou +jack daws +itson ly +invigil ator +insecure hbo +hugg able +hans berry +h nurburgring +gy ne +gun dogan +gour lay +gods word +gi al +gerry adam +geek ery +gay lord +fun fest +four fold +fore gone +focu sses +flor ham +fl ict +east cote +e ska +devo ir +def i +def alco +dar on +dam pers +cl ace +cham akh +bos na +boo kert +be sharam +bat roun +b eller +att il +asse tto +antin ori +animal art +anc inema +alien day +." ( +åIJ ī +âĶ ĵ +wen ner +weather by +v sr +tur nitin +the gop +tai bbi +so ddy +si mak +si busi +schi avo +samp son +ro ky +relapse records +r bn +q ais +pul py +pin ce +pil ani +phosphor ylation +perig ord +pa ano +nor quay +nkc schools +nh sc +movi star +mon ge +min ie +micro sco +mer lins +medi anews +mangi one +mand rill +ma demo +m kr +llanish en +lec tio +la sher +kre uz +kho on +jon ois +jar no +jamie bower +injec tion +ho sie +ho die +hig son +hi sto +happiness day +gold thwait +gi jinka +gen c +fire star +fin ovate +es at +encephal omyelitis +dy or +discover yed +dibru garh +de souza +de aring +dat alo +commit te +comedy bangbang +chu ppah +chan gi +cactu ses +broad us +boyce avenue +bhi ma +based learning +ay aki +as key +art scenter +apocaly pto +amer acad +ac ce +ab nett +ðŁĴŁ ðŁĴŁ +é ¡ +Ø´ ÙĨ +x rs +wa vered +vi gnes +ve ering +vault festival +vaul ters +vant age +unity assetstore +triti um +tri gla +to kara +terri bly +teamgod vek +tapp en +surreal art +ston ie +so cent +sm sc +sin spire +sidd hant +shiva ji +shan mugam +sexual violence +see us +satchat wc +sarab hai +ru men +rach na +pot n +parmen tier +on stad +nyc ballet +nico lear +mun ter +mon ate +mobile gaming +milk tea +mc memorialcup +mc dormand +mark wahlberg +li muru +ko ker +kirri billi +kh da +juli es +jig gs +jarls berg +jag ran +it support +insi eme +hy i +humanitarian day +houseoff raser +hor bury +hom ie +hilli ard +gur uk +gum shoe +gladi us +g fw +fl and +fast codesign +entertainment news +donate blood +desp ues +de wing +daz ed +da pet +cri sco +cee fax +car mo +buffo ons +bletch leypark +bell shill +be ssa +be mel +bbc glos +bag shot +aw rites +autom ates +aug ment +amate uri +a iche +ðŁĺį âĻ¥ï¸ı +ðŁĺĤðŁĺĤ âĿ¤ï¸ı +ðŁ¤· ðŁı»âĢįâĻĤï¸ı +íķ´ ìļĶ +z j +yaqu ina +wiki art +whit er +whis ks +vuvu zela +van ities +tyre ke +the dar +tam ura +suppor tall +star rcade +stan ek +skam france +shiv ay +shi ed +sa chiko +rural health +rim pac +real jeff +ranveer brar +pul u +proudtobeafan of +play book +o sullivan +numb ed +nostro mo +nor rell +nas anewh +nasanewh orizons +naco tweets +naaper usur +mo hua +mc whorter +mc cants +ma ung +ls st +lmfa ooooooo +life sci +le stari +le bon +lau de +la gav +l rv +katsu ya +inund ation +internationaldayof yoga +incorri gible +impregn ate +impe y +il icious +horo vitz +home ofthe +hol beach +hirsu te +gra dle +glaxosmith kline +giorgio armani +fuji xt +fro sch +febru ary +everysimpson sever +est ad +ea ve +e isa +du rocher +du er +dil ley +ddot dc +day at +charlam agne +bob saget +billiejean king +beau sallnatural +be uk +bbc sp +aspe aks +anneli ese +ðŁĻĮ ðŁĶ¥ +ðŁİī ðŁĻĮ +ðŁİ ĸ +yon ah +yo jna +yash want +x ula +wo sa +wing less +wi zzy +vennel akishore +usas oftball +under achievers +tro gir +the journal +ter nate +tb x +super position +straightoutt acompton +stein le +sr na +south bridge +smallyoutuber army +sm be +simon mayo +sie mian +sentiment ality +run yan +pro chain +pot ch +pine tree +pa che +oh sas +ober ts +nipp on +nh n +mil nes +mehl dau +medic in +mc shay +maje wski +liannela havas +las ry +la gom +karachi kings +jun agadh +ju kka +jas o +j of +high tech +hear ttour +grey hawk +green sleeves +go hounds +get money +gee zy +fonten ot +flag bearer +est our +e wer +dor é +disillusion ment +de clin +craco via +con lin +clai rol +cine matics +char pen +c shl +bi pedal +bha jan +bere an +ber ghe +bent ong +audi q +allstar weekend +algonquin colleg +ad our +acupunc turist +acqu its +ac ast +? '' ++ % +! ðŁĴ¯ +ðŁĴĹ ðŁĴķ +ðŁĴķ âĿ¤ +ðŁijıðŁijı ðŁijıðŁijıðŁijıðŁijı +ðŁ§ ¬ +æ Į +zab ka +ware heim +wad desdon +val aafshar +un fashionable +toll cross +tizi ano +the kla +thai airways +star talk +special k +so kc +sivak or +ru ched +ron killings +ron coni +renmin bi +relent less +regal films +ramblers gb +px g +ps yop +po tra +octo bre +nikki glaser +nightri ses +ni pa +news ers +nee pawa +nct zens +nationaldrink wineday +national service +nap ed +n bam +my body +mor oso +mo su +mine iro +mdc pss +may te +marsh y +mar ya +mani fen +malay ali +mag ruder +lauren gottlieb +ku bu +ku ang +keto genic +kerryjane ellis +kemp f +inter linked +i faw +i asp +hosp s +honor is +healthis wealth +ham asaki +gri sha +gre ste +gn itive +gais ano +furi ends +fem to +fal c +ero space +em pa +ef fusion +dj am +dis orienting +delici as +cringe worthy +cordi als +commun ities +col ucci +co ble +cloakand dagger +catal un +can ai +bur ges +brill antes +bode ans +ber nou +bally bunion +atl super +app ia +and han +al awi +air speed +ab ang +a ot +... ðŁijĢ +) ..... +ðŁijĮðŁı» ðŁijĮðŁı»ðŁijĮðŁı» +ðŁ¤Ķ ? +ðŁ¤£ ) +ëī´ìĿ´ ìĬ¤íĬ¸ +âĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸ ĤâĸĤâĸĤâĸĤâĸ +y up +wwe braywyatt +ver ite +uta pri +underpin nings +un mounted +un approved +umass amherst +u lukaku +trail cam +thun ger +thu raya +theart stack +the space +stream team +stratfor duponavon +sta stny +sp ts +soo oon +snack time +shep shed +sas son +sand bank +sab miller +sa char +romel ulukaku +ritu par +rhe l +rex ona +resent ful +receptac les +rece ssions +rav nica +radi ob +press box +portu gese +pen alize +pear sons +park jihoon +nou wen +nic las +neuro modulation +naz i +men n +magdal en +ma sur +loz enges +l pie +kut less +jungy onghwa +jim ena +jan usz +hex acopter +han zi +gat ley +fri gg +f xi +exclu sionary +engar cia +drum mond +dim mak +cool sculpting +con cent +cl wb +chat roulette +centr is +bull ough +body powerexpo +bertol t +b live +auber jonois +are yes +akl transport +ac chio +ab end +.. !!!! +za hawi +z nick +ym t +y rt +wout ers +wo ven +wind surfer +wgtn phoenixfc +we suppor +wad ley +w sop +vandy football +ust angs +ty lo +tu lan +tor ic +the ye +super loud +stu cker +sk ick +shibain u +sheff trees +sh abad +semi precious +seduc es +saf aree +rw by +roche ment +red house +red beard +re sen +rayu du +rat an +rapp rochement +pure gym +pic tu +phil pot +ott en +onthe edge +ne wel +milit aries +mhu ire +me son +mci vor +less ened +kit to +keep moving +jo ye +jerobo am +je tter +hoo sick +homoe opathy +ho pa +harsh ness +happ il +fren chs +floun ders +fen nelly +facilities management +fa ired +elo isa +ek du +dj d +diss enters +delux e +de classify +de ac +daw ber +dav itt +cru dup +confor ama +competen cia +circum stantial +cat sand +bun di +bt x +bar clays +ay ame +ar lie +am soil +ado ts +aber tillery +. ?? +ðŁİ ĸï¸ı +âļ½ï¸ıâļ½ï¸ı âļ½ï¸ıâļ½ï¸ı +wel z +weather head +w vs +vener acion +tv ds +transm its +tom keene +the observer +stand together +sojour ners +snar ky +silver sun +si ssies +sci pio +schut te +rmt union +re ik +pro camps +pride to +plan ahead +pixi elott +owen thal +ome tal +no cks +music box +min esq +mex i +mcqu illan +mar teen +man gga +land man +kpop snaps +kang as +kam merer +jind abyne +j jk +j hah +inter dimensional +inter changes +hull fcofficial +hu sson +hein richs +ha aland +h way +gerryadam ssf +georgemason u +gal ert +fu ka +flex ors +exo du +esco ffier +ed gel +econ oline +e hime +del ux +deepa wali +de shields +dac orum +cu bat +cu bam +cron us +const ants +con ard +cleveland dotcom +child care +char ron +capit oline +c car +bustar hymes +brow nuk +bre lla +bo din +biop olym +beau ford +bare ly +as cari +ar aka +american apparel +ah sn +a strian +.. - +ðŁĺĤ ðŁĺĿ +çļ Ħ +âĻ¡âĻ¡ âĻ¡âĻ¡âĻ¡ +wav y +vermon ters +time frames +thaana aser +sylvester turner +sol vency +si ano +shekhar ravjiani +ser hant +save shefftrees +sar chives +repet itions +rain ing +quinceañ era +profe sor +power on +pollu ter +pl ani +pass é +par cell +palatin ate +op rah +on on +olivi ap +o et +non believers +no jhl +new aygo +n sl +mtvlakpop got +mark ham +leon sis +lake man +ken fig +ke ong +kab ale +jodi stamaria +jen shad +jam bu +j nu +ise tta +irr fank +ic p +i wu +huis genoot +home is +gr anta +gal livan +ga iter +fu med +fried le +flu ffer +fi ps +feu ding +epic ally +eh x +di elle +cle on +ci one +cer mak +cast ate +cann avale +cambr ils +cal ley +c th +boo oo +bi focals +bha u +bent grass +barnar do +barn burner +at os +as scher +ar oh +ðŁĺİ ðŁ¤ĺ +ðŁĵ± ðŁĴ» +ë°ķ ë³´ +æ ¡ +âŀ ³ +whis kered +vidy abalan +ver ity +traf fik +temb isa +team liquid +swe tha +sty led +stomach ache +shor tt +seren ely +seis mology +sec ted +reve aled +receiv ables +re decoration +ra heny +q trs +pp ers +pos its +pneumo coccal +pinkie pie +phoenix open +people mover +pan th +ou ar +or ana +nu evo +ne afl +mon eda +modular ity +moderni sts +mlb tonight +min ers +loo ds +lizziec undy +kov ach +kon tos +kess ler +k for +jo ffre +ishi hara +isaiah mustafa +ho chman +here fordfc +hell gate +haunted mansion +half life +guil den +fly back +e euu +dumb ell +duc al +drop dead +dan ilo +customiz ations +cottag elife +compre ssive +chan an +ch ena +cancer society +bren ton +blogo sphere +ben ford +ban aue +avo y +as inger +as coli +angh arad +alti plano +aaaaaaaa aa +yoshin ori +wid mark +wal kar +vincent kompany +vie tti +tumul t +tru ett +tonight alive +them self +the dogs +th wnd +tantal us +tag team +summer town +sum mere +stu tter +stra fe +sot we +sha f +sc roller +samu sic +ru do +ross ington +preemp tively +po chard +pe rel +pat en +om ak +nathank ress +nak as +mur rah +mul u +more x +mer ited +men the +mega world +may as +m guggenheim +lur ked +ken mare +ich ri +home port +hol burne +histor ias +hill brow +hidden gem +gre c +gallifre yan +foto friday +flu vanna +f we +ev asi +equ alize +en rages +emp tor +die ben +den ko +cu tty +co efficients +chak u +cazeno via +by products +bro kering +bi vins +bhar ucha +bar coding +baller ini +aren berg +anas azi +aj yothi +acet ylene +è¡ Į +âľ © +yaf ai +wy omissing +world club +world chocolateday +wh impering +web kit +uc cia +swal ec +sw k +stylist ics +stream liner +snap chatted +ske en +sig na +sh ya +see man +sange et +san ne +sal oni +safety month +s din +ritu alistic +ri fic +resi dente +record z +p dvsa +oddi see +now live +mis calculation +mian wali +megan hilty +mck illop +mcel henney +mayor ga +lovemy team +lang ridge +l jn +kos ice +kooten ay +komo traffic +kim cattrall +jimmy butler +jay y +italy magazine +instit ut +inau sp +hel ias +hasak ah +halla bol +gr illi +gabri els +fur ball +fran e +fo ti +fl sa +fbr party +fay e +east man +e esti +drizz ling +deci mating +congratul ation +co axing +chin ch +boho style +berry z +ben rector +bar ware +awar ness +awaken ing +am ruta +alle man +al enka +agron omic +( ** +ðĿĹĶ ðĿĹ +ब à¤ļ +zo sia +you decide +wpu gamers +winter set +wil kinson +wiki mania +vs fs +velvel holler +umn proud +tyler florence +to dai +thu izen +ten chi +stun a +shat in +sham schar +shamschar ania +se ele +sc ald +sam bailey +ru iter +rep tom +ra ud +pro veit +plo sive +pilip inas +paul rabil +parisi ans +os waldo +omgits alia +ney ville +new haven +neveren ding +mouth bass +milk maid +mesopotam ian +matth au +madefur you +lon dinium +lead with +kxip v +kokop elli +kne ecap +kis ar +jonathanrhysme yers +janis joplin +indian town +hou ri +hop ton +he mos +gu sted +green mount +grand designs +goss ard +gi all +fur nas +four ze +en gen +ec amp +dynamic duo +delauren tiis +cin ci +chel le +car idad +candle wood +broward schools +br wanda +bir rell +beep beep +beck ton +aver ill +alway son +afric ain +a ask +âģ ± +za jac +wor zel +wood vale +wash er +wall ander +w no +vor tices +vit to +un naturally +u shing +tutu tuesday +toronto life +tor ana +to gas +th ér +testu do +team up +st pats +serrat os +sch oten +santu ario +sa pping +ro eg +queen at +pu rer +prize fighter +pop fest +pocket ful +pc cc +ori hime +o shin +ninten dogs +niel sen +nardw uar +n bak +moul ting +more ish +mary beard +ma vi +lo bbed +kle pper +iron workers +intelligen cer +ice berg +ic lub +hor crux +hee ley +har uto +h tr +got ts +gine tte +fero ze +er isa +efan art +cor a +color fabb +co dered +cleck heaton +cater ing +cal casi +c mmi +bull fight +belle isle +bbc sport +ay uda +arrhyth mias +ani morphs +angel alessandra +an gre +ðŁĮ¶ ï¸ı +âĸ ¹ +will you +weiz en +vul va +vor st +visit zimbabwe +un taxed +un intelligent +tri est +ti bby +the butcher +t pusa +sun rays +step sister +sr ly +slu ggers +sheffield shield +sham ma +rif fo +ric es +relocate revolution +reig nit +rain iest +queen ston +por denone +out é +out ta +ob jets +no co +ni ii +neuro transmitters +nak ak +mu zy +midwive srcm +me shell +may ank +maru thi +luc is +loven otts +loren z +laparo scopy +l ld +keeping people +kakao talk +in ria +i ig +hyundai india +hu sa +happy ending +hand hygiene +ham ner +gro ssi +glutam ate +germani um +farm lands +er mita +ear ning +du ppy +dra ken +doom sday +die guito +df wairport +dekal b +damian mcginty +d zi +cu erockstar +crevas se +chloelu kasiak +chis ago +charlotten c +c pre +bossho g +bla kk +belo tti +bad agry +aver ting +arcol atheatre +$ : +ðŁij ļ +ãħĭãħĭãħĭãħĭ ãħĭãħĭ +âĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸĤâĸ ĤâĸĤ +yoko yama +xper ts +wend ouree +weaver ville +walk throughs +w mr +tu ke +tro yal +tid worth +tate modern +tal agang +tai ze +sze wski +sutton united +subhashree sotwe +stere olab +spe ared +simon kids +seap lanes +sav illage +sand ell +roman cer +rink u +rang eland +ran jha +ra ylan +public domain +pre ste +po sca +pesc ado +ol mec +offici ali +ny onya +national hugday +muffu letta +morphe tt +mo rel +me gad +makey makey +ko belco +kno ebels +klgand hoda +kil t +john barrowman +jc ps +international airport +himansh kohli +gro e +gou ged +georgi eva +ge te +gar ao +gan ado +fitz gerald +fis ica +essenti alism +er yan +enshr inement +ejec tor +dwan imation +cute dogs +char med +ce in +capitol records +cab u +blo emen +bi monthly +ber gan +bench ley +beau doin +bas sil +bab ay +antiqu ec +all eries +ali ko +# & +! ; +ৠĢ +x cp +women sempowerment +wi right +wheel sup +w pn +vre tta +vic hy +utter most +tho li +th our +ter nan +sublimin ally +spi eler +ser fs +schn app +rü gen +run ge +ru bis +ru bicon +ro bards +qu oll +ox ys +op hir +on health +of india +obliter ates +neuro pathic +n achi +muk tbharat +modi fiers +long shore +lak shad +israel in +in zam +il as +iam don +homes ite +gg b +ge va +g ptw +g lon +ferr at +feel ers +fanta stics +fam u +excruci atingly +el ain +e tri +dir ham +di bba +deploy able +conveni ences +ci fera +catamar ans +camp ina +bundes bank +bu cyrus +brae burn +blue steel +bill ys +bi kram +belvo ir +be aded +band ung +arnol fini +arbu th +ar salan +adi vasi +a open +Ŀ ¼ +ðĿIJ Ģ +Ï Ī +zone quotes +zephyr hills +wam pano +vodafone uk +varun kapoor +v op +v br +thin ker +thi phop +theroo kie +su mm +stick le +sti fel +squ ally +skil let +sc wc +s weekend +rush moor +regre tfully +pl eni +pick pockets +pap ilio +op u +nicek icks +ne hisi +nath ist +nar in +music blog +mr police +mon ette +mo che +mis reading +mick ael +men schen +me res +me please +manifen cours +man olas +mal us +maj ura +ma kwana +lions den +libe rec +lali bela +josh widdicombe +insi pid +hedon ist +hammar by +ha fan +gus set +gott schalk +gianluigi buffon +fy ffe +friday feelings +forest ville +fi jir +exo somes +euro sceptic +et cc +encapsul ating +en circling +en circle +ed ita +dri shti +deduc tive +damp ness +cu ta +craigie burn +cover let +cov campus +cor usc +con found +co oma +chest nut +che ban +car bery +byl sma +broad meadow +bea vers +bal ita +as ÃŃ +april dryan +app end +anthuri um +anit a +anci e +afi f +adv an +ðŁİĪðŁİĪ ðŁİĪðŁİĪ +è Ķ +x pla +workers comp +we at +v th +ultra verse +ul tran +traff ics +ton sil +ter ton +ter sedia +te gna +supp lier +su tch +sound ings +ser g +scre wy +sau ction +sand men +sac ad +red land +ra azi +pun ted +plastic waste +pig gie +phys is +perfor ation +para ñ +oti des +on thill +ol bia +ob es +no ori +nbs morningbreeze +my city +multim illionaire +mor ry +mo yers +mis sd +marni go +man nat +maku hari +ma bon +lin ski +less ening +ler wick +lear t +le ib +lao ag +kim mo +kar lin +kar ch +jag gesh +is op +icec ap +ic ds +i wish +hur dlers +help to +hau ghty +har mattan +grand slam +go sta +gland ular +gill er +gha foor +ge es +gal lic +future tech +fuller ton +fred perry +far rel +fair price +fa rel +e ades +dray cott +don tb +do sent +del aha +de aley +d ile +cre ated +cc news +cather in +bien ville +bel tsville +arizon awildcats +are valo +anishin abe +anari vo +ah rc +abe shinzo +ðŁį¿ ðŁį¿ +ëī´ìĿ´ìĬ¤íĬ¸ w +âī « +z ent +wear iness +venu gopal +vay ner +v q +tr ine +ton yo +to life +teres ita +summ iting +stro jans +sto ther +spor a +spoil age +speci alt +solom un +sn andani +sexualas sault +sen ig +sand ile +sam ani +s doitbetter +ribble sdale +raima sen +quim per +pre te +pra shad +pi atti +pare jas +pa hs +p anni +offro ad +o low +n kurunziza +mol ony +moham madi +miniature painting +matchbox twenty +man ofthe +levit z +lawand order +la burnum +l lyn +ke ttes +justine greening +ju dt +jmuk mrpolice +incur ring +i stan +i ren +house kinok +housat onic +hase be +happy yy +hammer fall +ham ont +guitar world +good body +go sn +f summit +et f +enthr all +em j +can ned +camer at +burkhal ter +bottom line +betsey johnson +ber kel +ben gu +basset law +bare si +art aud +anitaha snandani +alph ons +Ï ĥ +z eck +wire land +visit pembs +vi dar +valtell ina +v ti +train your +the kk +te ign +sto dden +smi dge +sinn fein +scre wattack +sci eng +schur z +ru x +rou l +remy ma +remo test +re awakening +py per +promote gamers +poli an +plough s +pi eper +patty sday +orang ina +on erous +okc fox +ngo c +music month +mid t +malaysia airlines +m enger +lord mayor +kil le +jelly roll +is worthit +ira ge +indian airforce +in tune +ikh wan +iam not +group board +gal eri +g tr +force friday +fiber optic +fast forward +far hadi +eti ka +equ aling +energe tically +e ship +dra kkar +deton ator +dain ik +d ps +cli me +christop er +che mb +chad mmurray +ch ite +cau da +c kel +bu land +breck land +bouton niere +benz ino +ben er +aval or +au tuni +attribu ting +at illa +ask men +aq eel +ap ri +ðŁı ķ +yas uda +wy lam +wa sit +vitali k +vitalik buterin +undÃŃ acomohoy +un iti +uc cs +tre yanasta +tol ong +to self +the c +sussex ccc +spr at +shin kai +sheffhallam uni +ru cks +ro eder +ri kka +r ti +qui ros +profit erole +pony tails +player pic +pitch side +phantasmag oria +pab los +ol dier +nic liffe +ni giri +min im +met zler +mesab oogie +men z +me hul +m wl +loo sed +llew yn +ki at +kei bler +kav in +imagine er +hul led +graham cassidy +gra sps +fro wns +flam beau +dn q +denver comiccon +de gradable +cow lick +corner shop +cornel west +collecti vism +clar as +cau ca +carre tera +car ril +bledis loe +bc am +barneys ny +b ator +au h +antan anarivo +alta vista +ali um +aggies alltheway +ag gers +ðĿIJ Ń +éĥ İ +⾨ ðŁĴĽ +wo erner +tune in +trinity laban +tra ina +therapy dog +thel ake +suble ase +strat ot +soci ol +sel fs +seger strom +rel ine +reb or +rani er +q ayy +pla yed +or la +o dissi +na sib +n sb +n mo +mubad ala +micro climate +micro burst +mercedesbenz uk +maj ella +lu chas +live band +lite speed +lin iers +liev re +lewi stown +lee v +laurel wood +lap wings +ku kui +ku ki +kri pp +kapoor fc +jamesmartin chef +jad in +inthe styleuk +imit ations +hakk asan +h df +follow back +faiz abad +fair child +f pd +elen nox +ei dul +eag ency +devol ver +dess au +day quil +dar ks +d pn +cou lier +cor io +comicbook art +cla use +chi marathon +cfe vs +buck fast +bu bb +bonni emc +bel fa +bar g +baek hyunday +ba best +azure stack +ar amex +aqui legia +angui shed +amas sacre +allot ment +a wor +@ _@ +ðŁĩºðŁĩ ² +ãħłãħłãħłãħł ãħł +ÄŁ rul +zaf ira +z aun +vandal ising +v ls +ur tic +un sociable +tu ti +tren cher +tra w +to create +teen ag +teacher toolkit +tan tric +sur yan +super speed +stur rock +stirling shire +stat ton +st ach +so ley +say ur +sau cier +road warriors +ro ci +ring central +property brothers +por os +poland ball +play making +pal vin +paintin goftheday +op tically +only thebest +next wave +nc bs +middle field +mat cher +man love +leonard cohen +lah ren +kne aded +kak enews +k undera +jodrell bank +j jackson +hypoten sion +hor ween +ho teles +herop anti +herald sunsport +guardra ils +gu ren +ger rans +em rata +em elia +du ffey +drivel ine +disc ards +dic ho +den izen +daylightsaving time +co iffe +cav olleyball +cal ved +bring erof +at aste +art sci +ani xon +ak d +agit ators +ac clamation +ìŀ ¬ë² +~ ( +wz tv +wr c +wordof theweek +war iow +wariow are +walla pop +vit ello +usp id +un dial +u eg +ty cho +trade offs +tele marketers +sun apee +stephaniej block +siddhar th +satisf yingly +sab ine +s story +rodi mus +re villa +re valuation +re her +pru eba +play ability +plac ido +neg ates +naxos records +nat ori +my family +mountain day +ml ml +mar va +macmillan coffee +lu sion +lely stad +kor b +kirch hoff +kc g +joy ner +jar at +jam bon +hes ston +grand standing +gotit free +gal ati +gain ful +g lynn +fi end +eter nals +e avis +drinking water +dran kin +dean winchester +dam ac +d workin +clio awards +chem all +car ver +capsu le +budd le +bru st +bra ven +bli x +bigbang fair +bab a +at ol +ap am +amu lets +alexand rite +al vey +ag ran +ac com +a different +< \ +ðŁĺı âĿ¤ï¸ı +ðŁĹ º +ðŁ¥ IJ +éĺ ² +é ad +zul lo +zeno bia +young dolph +ww ba +wr ittle +wp fd +webcomic chat +w ml +vec chia +un affiliated +tu mis +timm erman +teeling whiskey +team nosleep +ta urine +t magazine +spo st +sock council +sher burne +she imer +semi pro +santur ce +san chi +rul lah +pw ds +product management +pre fixes +phil o +nur selife +nt midlands +nike fuel +nascar on +nam al +mustang pride +mis managed +memor ie +ma say +ma den +llangef ni +legal ising +le moine +ko ike +kear n +kam ari +ju stu +joshu ak +it ap +inter serve +integr ale +ic key +i ex +hou ruk +har peth +har di +green infrastructure +goo finess +gand i +gam gee +fent ress +fact book +esk il +enh ancers +el eni +dun elm +dun boyne +direc te +delhi metro +del fina +com mbank +camof lauge +ca reca +book post +back up +art contemporain +agame of +ðŁĺı ) +ðŁĴļ ðŁįĢ +ðŁĴª ðŁı¿ +ðĿĹ² ðĿĹ +ze ek +z emo +yu an +youth soccer +yeg bike +x bond +woodro ffe +water safety +watch party +wakati pu +vv vvvv +vitri fied +trache al +stre es +stay lifted +sky ped +scare lli +reyn olds +ram navami +powder horn +pin wheels +pi qué +pearldrum corp +op hilia +ol loc +neck lines +n so +mun ya +mono pod +mo ster +megar an +lu si +look good +libra irie +latin os +lang tree +lal ah +la iney +kre jci +kh ary +jointe am +jo enichols +jason wu +hou l +hir anandani +hillsborough sch +happy feet +guar antor +gro sz +g sas +fresco s +fl ann +feudal ism +er mey +eich mann +eco tricity +earl swood +dog z +di mash +defaul ting +dc as +dal ma +cr ts +cor dis +comer ford +clu mping +clu blondon +chal u +ch ira +brocken hurst +bot to +bir thr +believe inthe +ban hart +b iller +az im +ay ako +as som +ar bi +andre ali +an asu +alver nia +aby dos +>>>> >>>>>> +ðŁĶ į +âĿ¤ï¸ı ðŁĺĬ +Ì · +yt ff +yi fei +woman shour +v bm +u ld +tuber ose +tory lanez +ti ums +tear fully +tal ky +sur plice +suk anya +spi on +space opera +snow plows +skip bayless +sk night +se sa +schick en +sar on +sand on +safter dark +s enders +ru stan +royg biv +ro tem +riot into +quirk iness +per vy +pat ay +par oles +par mer +omur ice +officiale fcc +national walking +nar ra +moon man +mo dders +megam ind +mcgru der +mar len +main ichi +m sta +love trump +le wes +kur anda +ku cko +kie sel +kair o +judge dredd +john s +jessicam au +je eva +j ado +ir regularly +ico sa +hu ra +hoste tler +gunder sen +g isa +fly saa +fin et +face painting +fa intest +enal davies +ek wo +damo daran +croque mb +cri enaldavies +ching lish +cab arca +bridg man +bon trager +at tia +ash tan +ar aku +ancien thistory +am my +ðŁĺľ ðŁĺį +ðŁĩ± ðŁĩº +íĪ ¬ë +é ¤ +Ùħ ØŃÙħد +y acob +working dog +weare x +vo ig +vinny guadagnino +vi vas +uro pean +the mandymoore +the human +the goat +the dcuniverse +tandu ay +stanford fball +someonet ell +sof tg +smo ore +sit aram +shar mony +sf x +sen al +seeyour city +schu ll +ricar doro +re xx +rail er +r caniglia +q opa +pis ser +pawn stars +pasi apac +ong p +o yo +nulli fied +non proliferation +mer rin +med ora +mari as +manife stos +man vel +majer us +ksh mr +khan h +ke unsuk +kat ery +k oury +jhun jhun +jay ma +hydraul ic +hr b +hend rix +gra bar +gn omon +globalgoal sun +german o +gab bie +g lim +fu le +food science +fl ensburg +fissu res +fire bug +die back +de wy +d illian +cor son +chee ba +cap tured +c sj +bush ings +bri ster +boston police +bole da +big sur +bel ah +bag ani +aver i +ated r +afilm fest +ab dur +ab assett +!! .... +ðŁĺ« ðŁĺŃ +ì§Ģ ì§Ħ +âĸĶâĸĶ âĸĶâĸĶ +د Ø© +® ï¸ı +y lon +wy p +white mud +water fall +volvo trucksuk +vo glio +vill alba +ven o +un sanctioned +u fi +u angel +the frog +swim mer +stage play +sport shall +sp angle +sie bel +shop ify +shar nab +selfa wareness +see is +sch ind +rs spirit +roswell park +rippaul walker +ren z +recor k +raw le +pray ing +poly amide +phar ah +par ation +p xp +on dra +oji bway +nov ick +mun ck +moy se +misty onpointe +militi amen +mikk eller +megac ities +mb am +ma sco +lu bb +long day +log mein +let seat +laur ale +la wang +kom bu +kill monger +kellie pickler +kee ble +kawas aki +k urs +k orie +james x +indra jith +imperson ated +hl ntv +han jin +goo drem +go win +gla sson +eve rette +ev ast +et is +ele fant +ela ing +effe minate +e ines +e bert +duck and +drum condra +dr ar +dif ford +den ison +deflec tor +cul to +creative commons +cityof tampa +bu chen +broad wood +brew ton +bra yan +bla x +bell ville +base men +at ops +ar gi +amstel veen +am ts +ðŁij¸ ðŁı¾ +ðŁİ¶ðŁİ¶ ðŁİ¶ðŁİ¶ +ðŁĮ´ ðŁĮĬ +⾨ ðŁĴľ +young victheatre +yal emed +vet triano +v us +undere mployed +tom blin +to love +tld sblearns +the kapilsharmashow +stu ta +sti ft +ste ss +ste at +stand by +splat t +spe ace +social studies +so est +snow y +skul ly +sing ling +sin india +sin h +sig er +sien kiewicz +sau mur +saf o +roam ing +ro das +ric er +rath bun +pt fc +pra th +patrimon io +our club +ot bc +onthisdatein hiphop +nove lette +nish at +nicol ls +ne ph +morg fair +man na +lo vel +li ggins +kar abo +k gosi +jolly llb +ignor amus +ie w +icu isine +ichi moku +ib k +iam queenlatifah +heine man +happ is +green wave +gram een +gib bet +friedrich shafen +free space +flu bber +fal ah +empath ise +eas ing +e toro +document arian +dar kie +cheek ily +chate let +chan o +caro lee +bur rus +blo tted +bla kel +billy ray +bar bizon +az man +avan zo +au ld +allan hawco +alc dsb +alac tic +ajitpa ifcc +ðŁĺĿ ðŁĺĤ +ðŁĺĪ ðŁĴ¯ +ë± Ģ +าภ§ +yu lee +wag tails +vor arlberg +venkatesh films +uz air +ur um +ul b +uk wu +trom so +trau matic +tran o +thor gan +thi splace +simon on +roo z +rand alls +pul sat +prepa rer +pope vil +pasi ones +partic le +par ana +pang kor +out building +old strathcona +od ong +mean ness +mat an +marclam onthill +mal abri +lorraine pascale +loo d +ku pang +jis ub +jessicamau boy +hus by +his cox +hepatitis c +he mer +hass ler +gly cine +giovin azzi +geof ilter +fi fita +fa shawn +eri des +drag neel +dra zzari +diver timento +di stin +chromo somal +cham s +capac it +cal ero +bu ike +biomime tic +beam sville +bar goed +bal ed +attend re +as cap +angel sof +ali zia +alex us +ag or +ðŁĺĤ ! +ر Ùħ +zan gief +wy socki +world skills +whit tier +walkin shaw +w ase +ve tta +van avond +uof oklahoma +tt ro +trip ty +tren italia +tra e +tom jones +the lead +tag ov +su di +smur der +sm sf +sinha bjp +single life +shivar ajkumar +sham al +sh ley +sgr ho +sb cs +sal thouse +sa en +ryo ko +ron na +rie sen +rid ere +resolu tion +resol utely +repatri ate +pr girl +pig man +parach inar +om ero +oku da +nad av +mr inal +meth us +materi ally +mani ka +lyric ists +loth brok +li shment +li anna +lack burn +kn app +ker plunk +ke pong +judicial watch +inter diction +inte x +infuri ate +holi daze +ho lein +hey hey +he trick +have an +gov rauner +gen tiles +film fare +fabul ou +elabor ation +dong guan +dieben korn +dc examiner +cv show +culler coats +cot chin +cook house +charpen tier +ch aux +cc ac +cat nap +cas sin +calcu lable +bb clondon +back doors +aller gist +ali fazal +air corps +af fan +- ] +ðŁĩ ¬ +ëŁ ° +â Ł +á ĺ +zi will +ye zidis +twir lers +transpa rently +thelon ely +thegold bergs +thefuture isnow +terce ira +tar kov +so low +snow boarders +sla pp +short land +she ffer +sch eu +sap on +saf fy +red fearn +ra sher +qu ale +pipe ttes +per di +orbital atk +open work +nuest news +nu get +non ie +mobili zes +miss jess +marinabay sands +mar loes +liven ed +lady land +knox villeraces +kara jan +kam pot +k tul +im possibilities +if sec +ici ous +ho bday +ha sting +great music +gal er +fur ter +fir ston +em itters +el ings +dun ham +devo xx +dawson screek +dari en +crew members +ch cs +c anne +boye tte +bot anists +big apple +b pride +ashford castle +anticli mactic +ano is +andro ll +anaco sti +am dav +allyouneedise cuador +abbots bury +ðŁĺį ðŁĺ© +ðŁĺĬ âĺº +ä½ IJ +âĤ¬ , +world hepatitisday +wm ds +wak ar +vp dd +visit virginia +u akron +tu pac +tor mentor +the mercury +tele wis +ta van +su ffic +steel y +squ onk +spru ced +so kol +skill sforlife +scou ps +sch ram +sal emma +sad d +rain coats +raf fia +pop tropica +po poff +pix lr +pernam buco +perempu an +pear lie +par key +over hearing +om pton +oly tic +nov ell +munt jac +mis amis +mi zzi +may tals +mar well +ma zu +lu beron +lil in +larry madowo +kv k +kc j +jer on +ja heim +ing ood +inaugur ationday +ic kie +ic ele +harvard chan +goo sander +gol ub +galvan ic +fle shing +fau ji +farma ajo +essex cricket +end slavery +edmonton esks +dip ty +dayo fre +danny john +classi er +cho k +cavali ere +capital ising +canti ague +can ic +c aging +bur fday +bo ley +bill burr +bic kel +bengal is +be bi +bal uch +bair ro +av raham +arte aga +and ar +amb johnbolton +ali se +ainsle year +ai shat +ðŁķº ðŁı¼ +íĨ ł +п од +whÄģ nau +wah habism +va sey +transit ory +tothe a +the tommy +sze ged +sunderland uk +straw bridge +sti q +starwars battlefront +st less +ssel fie +sp rad +si sd +shi b +sen blumenthal +se ibu +s ji +rp crd +rock starenergy +re activation +rcr racing +rang itoto +prednis one +poly phe +polit i +pier luigi +par scale +orit se +ore tte +new museum +ne shwar +mumb a +mn df +mcken zie +mammam iam +mal awi +leg alaid +lamo the +l stm +ko tb +khul na +kcr g +jing u +impresion ante +immuni zations +i vp +heli oc +guitar lessons +gu le +gra fik +gang lion +free thought +fr ds +fox football +fly hawks +far man +exclusi vo +er dman +el gl +egre mont +do ña +dim prov +delivery man +deek sha +dan patrick +croquemb ouche +contor ted +cer da +canadas nac +camren bicondova +ca inta +bs j +book giveaway +bin ney +bas ch +ban ville +ban kyw +attrac tively +arter io +army allamerican +anachron istic +aljaz skorjanec +al tv +aim ondo +adduc tor +âĻ ¤ +à¸Ńภ¥ +zoo keepers +zi ons +yuk imura +yourstory co +your dream +we ve +war c +w cag +ur dang +un nies +tour london +til ia +te dium +sudo crem +stick ley +snet terton +savit ri +sav eng +sathy am +redhot chilipeppers +reci eves +quicken loans +ple ural +pic ross +pe zzo +painte dby +monste renergy +mccau ghey +mal is +majum der +maggi enyt +luxury car +love island +le xx +kun un +kh it +kf bk +kettle well +kc streetcar +jen kyns +james spader +jamba juice +j law +irr ation +ine sti +imp eller +i ops +heroes inlife +her nehill +harbor view +guar ino +golden berg +ga itan +foot joy +flori dap +fan signing +expre ssi +el let +ei x +di des +der idder +cy clec +com ely +bush official +bung led +bun nell +bring thenoise +blue devil +biodiversity day +bi bbs +be free +barbour sville +avo ter +arr ative +arch bold +annamari aisland +afri yie +adidas za +achio te +aaron tveit +ðŁıĨ ! +ãĤ³ ãĤ¹ãĥĹ +า า +zi v +yq r +wit sel +whe al +wf sbnews +we there +vre de +ut ile +u ap +the ech +take overs +success quotes +st pancra +space art +skiv vies +sk atec +sen burg +selfi est +scru ms +sch aap +saf fel +sa win +s girls +s dot +r ino +pol yo +phil starnews +panop ly +ny td +nau tical +n eli +mis spent +milan designweek +low rey +ll g +lal ita +kus u +kabir singh +hunts ville +ho bon +heat onedirection +ha aaaa +gsuite edu +gowar iker +gheor ghe +gend ry +flavour some +eno te +emancip ated +egyp te +egg plants +ear lobes +del co +deade ye +de ee +cy le +cree per +chit toor +carbon ear +bur leigh +beach thursday +back benchers +auberg ines +asci ences +art land +ainsleyear hardt +aggreg ating +ac ter +abc worldnews +ðŁļ Ĭ +âĽ¹ ï¸ı +za ev +ys b +y ii +west malle +tow le +te hu +ta quito +t wines +success stories +shay ari +samo ans +romag noli +rock ridge +quote oftheweek +pre market +pol litt +pine town +patriot pride +p ingo +or azio +o ab +new steam +naz ir +nan ta +marsupi als +man ity +mal lock +mad vertiser +lu que +lau fer +jer os +jac enorman +is over +i ir +habi ba +graf ton +glee onfox +ghe alth +ger on +gender less +gau s +fan pic +ero driguez +died rich +dew drop +demo tiv +de mint +d cl +corey taylor +chees ing +blan es +be sty +banque trecords +ay oun +audio drama +ao tea +ai leen +ahlu wali +ðŁĺĤ ðŁĻĪ +е Ñģ +x ai +wy si +wai fus +victi mised +urban e +ul ama +tro mp +thu raj +tf sa +tele kinetic +taj hotels +syno psys +sy stolic +swag gie +supplic ations +su uu +sk j +se ph +sce wc +sa dist +runner s +q ine +pur dy +pur dey +prosecu torial +prophet muhammad +perseve res +oren tina +offic ine +muham ad +morethan just +moo ted +mond prop +med ved +man akin +ly onnais +lum pia +lo gh +lo ane +lef ty +le aux +lan xess +kry sty +kap al +kam mer +jon mitchell +jet setter +ini ana +in asia +her as +helio trope +hedwi gon +hat chett +han o +gri erson +food show +ely fe +e erascal +dist inguishable +degra zia +dan ecook +cur le +cre mer +consu ela +che ika +ch awx +bu fv +briand awkins +blac chyna +bi directional +bi dar +ber rics +bedo ya +be vans +b we +atur al +asi m +app ling +an ot +ale tta +above the +ðŁļĤ ðŁļĤ +ðĿIJĢ ðĿIJ +åĨĻ羣æĴ®ãģ£ ãģ¦ãĤĭ +ච» +y quem +y ig +wester nrly +w anner +up wardly +ts wift +transpen nine +track mania +tom colicchio +th ale +tg l +ter day +sun silk +storm track +solar storm +snow piercer +smo sis +sm su +sadh vi +sab ay +ro del +respec tability +rem parts +relient k +re tool +rain ier +ra so +r sb +pre jean +pok ora +ov on +omega watches +oat ley +o gon +nextgen atp +newhope club +nel e +mus ar +motion graphics +mo gen +min cer +ma int +lore ttal +ko yo +kal bi +k tuu +joan hart +itye vent +itali ane +intimid ates +im pac +ili er +hu k +hot newhiphop +hon ker +grave yards +gil well +ge birds +g sp +frit olay +fin nie +fer nes +ethi o +em rys +ed trilogy +e tribune +de smar +david lammy +ci man +cat lett +c ú +brandon lewis +bill z +bil by +benny benassi +be careful +ball ard +av cavolleyball +ard aw +ar not +al ur +afol abi +aff ton +^ ^) +: Â¥ +! âļ½ï¸ı +ðŁ¥° ðŁ¥° +íĪ¬ ê²Įë +íĪ¬ê²Įë įĶ +à´ ¸ +ÃŃ lia +z ila +yu yu +yler oux +world heritagesite +wild beerco +way an +voice acting +van morrison +utter ance +trumpp ro +trin abraxton +tri bu +treyanasta sio +tre main +ter fel +stur ge +stag ecraft +spy rothe +spring burn +so derberg +so bek +shir u +seman al +sar tori +row sell +ringo starr +proprie tors +pingu in +ph lo +ortho sis +nwin diana +nod away +no ons +nas pers +nand ed +n bal +mc phillips +mb ly +march against +lyn k +lenahe adey +le pp +kath akali +kat an +k las +jerry nadler +jenni es +jenkin town +ianm harding +huddle stone +harb anda +hand bells +hailey bury +gal y +emp at +em tothea +elge use +digital workplace +der mot +dav or +cri der +corr als +chin ook +cer to +cas itas +bo dh +blan ko +b fy +agre e +ãĥ¬ ãĤ¤ +x ur +wool la +white board +wheel s +wel liver +wa ves +w roc +volt meter +vishwak arma +vin cit +vasi lev +ul tural +u dders +tell your +teeth whitening +sirius x +sella field +seem y +seef eld +sch ama +sc s +sar avana +sahar anpur +s ool +rusev bul +ru pani +robic haud +ra wan +r fr +pis d +pen stemon +pauley p +par lement +over ripe +on ley +ol tre +nyy roro +naive ty +n eno +monom oy +micro controllers +mes dames +mae jor +ma zz +ma speth +love myself +lasci vious +la bored +la ber +l wc +krishnamur thy +kar lee +iki gai +iiii iii +i olo +i hab +haters gonnahate +h itta +gr itting +gobi gor +gesundhe it +ge dge +event s +et was +ental health +el van +don eright +do gging +dizz eerascal +diefen baker +de bla +dais uki +d had +cold stone +char nock +cast ellers +car a +caho kia +broad sword +be decked +b williams +av ena +asi ri +arba een +ap ad +al aka +afl don +aci fic +ðŁĺ® ðŁĺ® +ðŁĶ« ðŁĶ«ðŁĶ« +ðŁıĥ ðŁı¼âĢįâĻĢï¸ı +ми ÑĢ +иР¼ +zig bee +yu g +yaal artista +witche so +wg ci +week i +wab asha +w wel +w icking +visit belfast +vis sel +val arie +tur bat +tom are +the my +su mida +stor ie +ss oftware +sra banti +squel ch +spotify uk +site core +si pe +restaurant australia +rein ke +ra zy +ra dd +plat o +phiphi ohara +phi mu +pacific ocean +ou tique +op v +objec tors +nav neet +n dola +mu ggers +metroid vania +mel cher +malla ig +ma kino +lorettal ynn +lo cher +lar ity +laj pat +l fafighting +j dj +hor ikoshi +hel al +gri mez +grat ton +goodwoo dr +git anjali +freer ange +fra id +for gold +fe uro +f ending +eventu ality +entrepre nu +elo te +ear ache +e dom +duc los +diam eters +denomin ated +decat ur +de twiler +com ed +coler ain +clarkcounty sch +cit ad +ci pd +christian son +ce h +boom town +blu esc +bhu ti +bergen field +ber r +bench top +beck ers +bbc goodfood +back flips +ayles bury +av j +aux erre +aur ore +athen ahealth +astronom ia +apo yaalartista +and health +alli s +ai shah +ab amba +ðŁĶ¹ ðŁĶ¹ +ม ม +wr b +wbr cnews +vol ant +us fl +udta punjab +tw cnews +tun ics +to ten +steph ano +st kilda +sn am +sh appy +self pub +saver io +s bi +ru sch +ru fo +re usch +r pm +public service +pric hard +ph ills +pet chey +part sun +par tee +over spend +or ba +open forbusiness +oo ohhh +ohmy gosh +no pal +ngv melbourne +ne era +napolet ana +najam sethi +my self +monk house +mne monics +mc william +maple syrup +london fashionweek +lat v +l all +kil bourne +kelsey redmore +k mox +juxta pose +just say +i speak +hol tnbc +hardeep spuri +har rell +gro ho +gor o +gold finches +gol der +gol conda +gen cies +frommy heart +faof ish +fan ks +fan day +fall league +epitom ize +en ar +easy money +du buffet +dream team +discovery id +de orro +de berry +cycli st +ctr l +cit b +choreo graphic +chak akhan +cc ds +ca ap +c ws +brou illard +bok ki +blu earmy +au lia +army wp +anti och +and music +air ambulance +ad sk +access control +?? # +/ ( +ðŁħ ¿ï¸ı +éķ · +çĦ ¡ +æ ¾ +ãĤ¤ãĥ ī +zcz uk +years strong +wwt slimbridge +watch nz +vir tanen +to ggles +tie res +thrill ing +the wild +the posh +swer ved +st lawrence +sp ellers +solar pv +sharnab urgess +sav va +sand blast +sab ado +rp murphy +robert patrickt +ri ad +ree bie +real love +re saca +r ales +progressi verock +pe sco +parry sound +panam á +ortho gonal +onec up +novi kov +mur tha +mou lana +milk ha +medic ale +mb alaji +lo xton +lo sh +l ène +kin te +ki rov +ke bede +ka stles +jud ici +jane philpott +indi en +if we +hu h +hel me +har ks +ge su +flau tist +fidge t +er rani +e bon +director mbalaji +derren brown +dam age +csi miami +cp v +coqu ille +con ish +bur b +brad meltzer +bra cht +beat riz +as sport +any ones +:::: :::: +ðŁĺĤ âľĮ +æľ ¨ +ãģ£ ãģŁ +z aps +wel chs +w fo +un chain +tr us +the wire +ter al +tantal um +speed sters +snow cat +sn el +sin ta +shaz za +serge j +ser vient +se epage +save on +salvationarmy us +s laney +ro oney +ro ha +re winding +re settle +r uru +q ajar +put nam +pre packaged +om onday +oise au +o ah +nur nberg +nichi jou +nex press +neuro anatomy +ne peta +ne ddy +moon flower +me it +mc beal +mb g +lough rea +lorealparis usa +lit z +lau rea +la key +karapar aask +kal imba +inordin ately +ide en +hade stown +goven der +fu z +fro bots +fau ve +face down +experi ential +ero u +eb v +den mark +dai sey +creep ily +cp mumbaipolice +com pa +co bh +broad sides +boun dary +bo ddington +bfc dublin +beverly uangel +asin ac +ani max +all state +afric anist +ae ons +acol ytes +( ãĢĤ +ðŁĻĨ ðŁı¼ +ðŁĴľ . +ر ÙĬ +waterstone spicc +w baboxing +vin ci +ver rett +vel an +tumn us +ts q +tribe sman +the wiz +the mis +the dubaimall +tatt i +sun corp +stra pline +stam mering +st ks +spinning top +sidd ha +shilpash inde +seung woo +scapego ating +sar n +ryan lewis +ronde bosch +riffo tronic +rad ford +puke kohe +prodig y +positi va +o wari +no taro +ni ajax +mycorrhi zal +mr duncanjames +mil on +megalom aniac +mag ana +lipp in +lady killers +la fitness +kur tzman +ka hs +inciner ated +i pps +hump y +hg vs +gene ws +franklloyd wright +euro dance +epider mis +ellef son +dylan mcdermott +di ame +clam my +chir ped +ceee gebirds +car tel +burk holder +bu de +breckin ridge +bre de +bal dridge +b tt +awe urope +at nam +an ico +ale lection +ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ +ðŁĶ¥ ðŁıĢ +ðŁĴĢ ðŁĮ¹ +· · +ze ina +war and +vas ser +vapori zed +un roll +tori ko +tom ax +to ple +thejim cornette +te gen +tat ro +sul ting +sud americana +sporting cp +shaw aii +sham wow +sand ridge +rt pig +road safety +ramadank areem +rain goaway +q hd +pomer anians +pin ki +pers ad +per ki +parañ aque +out doing +o sto +nor semen +no ke +no ga +nh sor +net sky +ne sia +my nam +mon ferrato +mix te +mid shipman +metagen omics +mesm er +melis sal +ku va +kno tting +ki jhl +kay afm +k law +jonathan jackson +jes sen +ing ames +ihear tradi +idri ssa +iam lenovo +hij ri +gall stones +gall er +gaine sway +ga shap +g á +forte za +fh fa +eze kiel +espn fc +er b +eat sleep +dhing ra +del po +counter intelligence +core graphy +cooper ators +cons ented +cep r +cdn history +cal zaghe +c tbt +bon ham +bex tor +bergo glio +belly button +be heads +ay anda +atletic omadrid +ar tt +ap tors +anti report +am ics +ahmad is +a style ++ âĢ¦ +к и +wy rm +wt va +wit cher +wi b +whit em +vote austinmahone +van ness +ux ury +un heralded +thapp ens +tham ma +teach in +sukab umi +sub let +stoner rock +spor ti +smet ana +skills gap +sister sof +simon schuster +siden ote +sar wat +sal ade +sag gy +s alling +ric in +ren er +rebel le +ram eez +rac i +r sd +powerrangers movie +play thing +plac ers +per ico +ot ford +noctilu cent +ne ya +mist born +meng al +mcg lin +man holes +madon na +lu bs +lin ce +langu ish +lakshad weep +la pel +ku ipers +kohl schreiber +knigh thawks +keele y +ka os +journe yofficial +jayas uriya +janasen aparty +j gn +hyat tre +humanit arians +hoo ton +hamidmir geo +gree do +gre gate +fur thur +fra ise +follow andretti +fire lands +fi ving +fa ka +expedi ency +elizabeth forma +earth rise +drink responsibly +d Åį +curric ulu +ctb to +crvenaz vezda +car boy +campbell ford +buster posey +bu uuut +bra chial +bally fer +azi muth +as fa +archi v +ar leigh +american topteam +aldu barkads +alan thomas +al los +aa am +a ÅŁ +ðŁĺİ âľĮ +ðŁİĹ ðŁİĹ +ðŁĩ¹ðŁĩ ¼ +ze inab +wire duk +whati want +v res +un lincoln +tw loha +toe jam +tenn ille +st moritz +spyrothe dragon +slide shows +sivakor atala +sig nora +si byl +sh omes +segaf redo +sar ra +rock cliffe +quick book +pra japati +portsmouth uni +po ti +party goers +pap d +over views +ni asse +mul house +mc ca +master y +magn ani +ma ior +lu wak +le ura +kamau bell +justin thomas +iso pod +hot picatnoon +hank ering +handmade gifts +grilled cheese +gari funa +fon dren +fle is +fl en +fal ak +fab ri +f gb +f bb +es covedo +e disi +du lux +dri skill +dill ingham +cu bitt +cro oners +cre t +con go +cloud berry +cl ore +cau g +cardiopul monary +campaign mag +bu is +bp climited +book cover +barn old +bami yan +b ital +ave x +at aide +ark on +arco baleno +ag co +!!! ðŁĺį +ек б +y plac +wapping ers +wal on +vi ena +trust god +threes game +sher k +s musical +riseas one +reco lored +realmadri d +ram yak +ra sse +r hu +poo lesville +poco yo +pendle bury +p chidambaram +nujab es +nu tella +nh u +ne kom +moun i +moo kie +mon fort +mil ind +mei osis +marti us +manas lu +ma ff +m pr +lie tti +lego batman +le id +ladi esday +khut bah +is sei +indian wedding +illustration art +i onized +housekinok uni +hors fall +honor flight +hitch hike +hey sham +hell blade +har lin +happ n +find lay +find away +ex on +ele a +el oped +ec ards +dv usd +dru e +di ffa +co ffin +cherry tree +bol er +bi thday +bad finger +az t +au las +atthe movies +atchaf alaya +alic es +ali bis +al cac +ak uti +air co +agricul ture +ad de +a project +> . +* - +" ðŁĺĤðŁĺĤðŁĺĤ +ãħľ ãħľ +zoff any +z off +youss ou +yl ounge +wra pup +wra gg +weid ner +wc bi +wav eland +war id +vin ca +ver dade +ur am +un ce +tw fanmily +tu log +transpor table +the pc +syste ma +sur inder +stock holders +si es +she z +selec tive +schem er +sam ora +ron co +ro the +pul wam +prez ono +port ables +poly gamous +ph ac +penit ent +par aty +pa ea +over clocked +of cs +not ability +neas den +moon set +monte grappa +mini disc +min ch +midd ling +med ha +meag angood +me q +mati angi +mar jawan +m wc +lma g +lieu tenants +lesli eville +kro g +kol am +king ia +jo co +jed burgh +jaros lav +j bb +hy keham +huawe imate +hat es +grand cayman +gossip stv +fun in +for i +fer rada +enter ic +desar rollo +dead or +dan harmon +d pt +cog swell +classe ment +checkat rade +case mate +bur ts +bow line +book plates +blat chford +big mac +band aids +auti stic +any one +am rev +af fi +? ðŁĺ³ +ðŁĵ· ] +ðŁĩ²ðŁĩ ¯ +ìĺģ ìŀ¬ +willing boro +whis en +we vel +versi one +veri sty +ver ismo +traff ick +thrasher mag +talke etna +super sport +sti jl +stand down +stan te +single player +si lex +schwer in +schoon over +rin sed +r sn +per rone +pan fur +pa hat +pa changa +over dubs +or ator +omar ry +oli days +nu bra +nig guh +nie haus +niajax wwe +neuro transmitter +must achio +mc gm +man ess +madeinthe am +macro biotic +le creuset +ku y +ku coin +kon nichiwa +kiril enko +kies za +kel as +j and +intern alize +in step +in ci +hol guin +hin rich +ha german +guer rillas +gar ai +fulbright prgrm +ez el +emblem three +el ink +dolce amore +dex p +depos it +dek kers +d elli +clean sers +cer ato +cardinal nation +broad ened +bo ones +bankrup ted +avi ationday +arach nids +aper ta +ang leton +ananth kumar +amuse sme +al ster +ahmed nagar +ag akhan +ðŁĻı ðŁĺį +ðŁIJº ðŁIJº +zodi ak +wra wesome +wex tweets +vo j +vitali y +uk ltd +the katvond +th aba +sports world +sh omo +save ur +sap ere +saint arnold +rotter dam +ric heisen +re plo +radic alised +ra hr +quin cean +pumpkin spice +pu y +pron ger +project cars +prem ji +post ma +pipe stone +pe jor +pastor alists +oooooooooooooooo oooooooooooooooo +mye m +murmur ation +mu zak +mo ssel +mersal teaser +lan go +lain ie +job sfor +jee bies +je th +iri c +int vw +in hospitable +ic har +home goods +hi zb +gu u +good friends +go ke +glow ing +gla vin +gh p +ga urd +fro ese +flu vial +ffic er +explain able +ever se +ente i +du alism +dr disrespect +digital currency +clinical research +castron eves +cas ella +carter r +care uk +candi dato +c cim +btsx snl +bobby jindal +blay don +bl ount +bar tha +bal ag +baby cham +auch ter +ashab hosle +architek tur +aphor ism +an quan +ale ad +aggi ore +ðŁĮĪ ⾨ +íĿ ¬ +yuril owenthal +xen os +wal halla +wa ja +virgin islands +ving lish +un cbball +tre u +tre gan +tau bman +t enda +swachhbharat gov +super bow +stir rers +spe ake +so it +smar ine +shin er +shar leen +sambailey real +sal ati +rocky horror +roc key +re animated +proven za +pri zed +po tong +pit ney +olivier giroud +off hand +ne ath +natu rec +national doughnutday +nan ning +n xs +mercury marine +let z +laz ard +lac eration +kor ova +kok ila +know ing +katy did +kash ish +jar on +j wu +is ins +in actives +ilove hd +iam blackbear +hud speth +hero dotus +herac litus +hedwigon bway +gull foss +gen til +ge at +g men +far ag +drive tastefully +cow girl +cl c +chec kups +canni balistic +can adap +campeon as +by rom +bra infe +bon anno +bear kats +barcelona alove +bar q +author it +as ky +anu ger +ant je +adelaide hills +ad ness +ðŁ§Ł âĢįâĻĤï¸ı +ãĥª ãĥ¼ +اÙĦ ت +ze ba +zac apa +wor s +water kloof +water bed +war nes +w kamaubell +uso glu +un right +un gie +u ib +ty ro +twitch creates +tin ke +ti ppet +teenagemutant ninjaturtles +son ly +sobie c +smart card +shav a +shan awa +scro ggins +sanc timon +ralph tresvant +nick son +mobile security +min ders +me ander +mal um +mac nab +m wo +m mat +lamin ar +la grave +l wv +kier sten +juli ago +ju sko +is kander +in legis +hor sford +hiz bul +hellolove goodbye +guis borough +green biz +goddam ned +fortnite battleroyale +ferne mccann +e government +e cot +dy as +drake relays +documentary photography +diag ramming +dav ante +club sport +c gv +c bi +bull ingdon +bu ber +bram ham +boy meetsworld +blanc mange +bear skin +be well +be sth +as riel +art basel +ark ads +aran juez +anem ometer +amor im +allison holker +all out +ðŁĮ¸ðŁĮ¸ ðŁĮ¸ðŁĮ¸ +в иде +ÅĦ ski +year slater +vand ana +um theatre +ton ner +thegood fight +the hot +tempor ary +tar bell +sun screens +stu c +star ches +soa addicts +sne k +royor bison +reserve dly +regan smith +quanti fiable +pro ba +pre disposition +plane trock +pe ggle +p tur +om at +oliver heldens +ol mo +o com +no li +musco gee +monte carlo +mike sonko +middle class +mel nick +meadow sweet +mayward x +matt damon +m ols +lost girl +lec co +lang ara +lagan jae +laganjae stranja +kol asinac +ka o +iw aki +ion ut +instag ay +inglen ook +in born +hem pel +gun fighter +go devils +gil o +gan u +g morning +frei ghters +flat ness +escan aba +elli jay +dre cs +dex trose +defecte drecords +dave chappelle +cri stopher +cr m +countrymusic hof +cheese monger +cas ano +c bt +bro mo +botry tis +bau ma +bairn sdale +asadu ddin +arsen io +ano il +alco hols +ach aem +ðŁĺı . +ðŁĺĤ ... +ðŁįĤ ðŁįģ +æĢ Ŀ +༠Ħ +z fs +yo he +yankee stadium +work safe +whole food +welsham bulance +weight los +waddesdon manor +vä ster +vo isin +urban agenda +un inor +tru dell +strang ers +stoo ping +sne ering +sin ach +sihan ouk +shig ure +shek els +seab our +scott bakula +ron k +ren nais +ram pton +radion ational +pound bury +penna onb +pas alu +org sky +nw smobile +north pennaonb +non plussed +n ical +n anni +mur nau +mr rpmurphy +montene grin +love irishresearch +lino cuts +ji ro +jang keunsuk +jab ulani +indiak agame +igu al +i boschetto +hi mba +hart line +hair le +go swans +gachi bowli +fun land +fac et +epi thelium +ehr le +e wers +du va +dam bulla +da oud +cox ford +church town +cast mates +career day +bel ching +beach ampion +be amon +band ara +b to +akal pathi +aedil hai +ðŁĵ£ ðŁĵ£ðŁĵ£ +à¸Ńภģภ+ว ย +zey ne +wi fes +wac cam +var go +un ra +un organised +ti sane +threef old +spring flowers +sil vretta +sie ff +sc cs +sample sale +reg attas +rb kc +ram z +radio app +premiere night +plac ket +pet to +mur shi +mis ato +mcca ig +mbi id +mag safe +lu es +lu di +lil dicky +laye tte +kom m +knoy dart +keep init +kaf i +jose maria +johan nes +io ang +hu tong +hover flies +hol lin +hide and +heyits conrad +hey heyitsconrad +grand is +gang stas +franco eur +fol lis +fish ponds +eye shield +eye ful +e tap +e steve +dro ste +digit alized +de watering +de enday +dar in +daniel boulud +cy donia +ctv london +coreytaylor rock +cat chiest +car ano +c ten +buffoon ery +bu der +bpl boston +bootle ggers +bn sfra +bi shi +basket bol +at rack +at liens +asun glasses +ali ka +alam ocity +âĢ º +ö ller +© ï¸İ +z lo +wood ford +wol ski +wex ner +ul tim +tugue garao +trib biani +tri state +tied ye +than es +tard iness +talis mans +tagov ailoa +sym ington +supportall streamers +stay cool +star te +smer conish +ski at +sex pistols +selec table +sam aya +safe and +roc chi +ro mac +po sobiec +plac er +pi adina +pe tre +pal ash +o ort +o dium +ny ss +ny a +nic onico +new c +ne isd +naz areno +nany uki +mye ong +mu sou +mu on +mon ga +mil n +masti ffs +mal ady +mac gill +ku ih +krist perawat +ki em +khal af +k br +jan ella +j dw +inexor able +if youknow +hudders fielduni +hohen zol +hi en +harry met +guest mix +goz wift +gla b +gd ny +gbbo final +gastron omia +eric andre +enrol lee +disen franchisement +diesel punk +di thering +committothe g +cisco live +che ssie +chat to +char oen +central park +ce derberg +cad aver +bur th +brede sen +bour dieu +bern sen +barr anco +bar kat +bar bora +bag pipers +backto basics +al mos +ahmed abad +ðŁı Ń +ìŀ¬ë² Ķ +âĿ¤ ðŁĺŃ +ze olite +ye ver +y band +tech nion +tan abata +stair wells +ske w +si w +se ssional +sch wa +sc ousers +saroj ini +santa fen +sandi gan +sand rak +rug elach +rr f +roy se +ros common +rital in +recom ended +pin sp +pill man +per ahouse +pastor ius +paki stann +ou lay +ol ya +od der +north end +nit da +nas rin +mu lah +mer men +mel anom +m wai +lester holtnbc +le kin +lab be +l wwy +l bc +kyo do +kn ole +kad hal +jac kett +ja rena +isleofwight fest +irwin mitchell +indi atimes +ili ja +ia state +head scarves +he fei +grow your +gor je +gn h +gh m +frog town +fresh breakfast +float plane +far ran +ener gon +edger atedr +eating disorder +de stefano +ct il +cad en +business line +bun ko +break bot +bol l +bir thanniversary +bi japur +ber ny +bec cy +and sf +aha hha +ag is +" .# +zarbeaz b +youn gar +yoon jin +yo ji +wrestlec ade +wo ty +wha aa +we didit +war ofthe +wah habi +vic ente +ver as +uc bt +trip wire +tp mp +ti afoe +thr one +tar sus +tac t +sty mied +street league +stor g +ste eve +spi ef +son tario +sac redness +s bac +rum miku +renfro w +rc plondon +peep ed +peace able +pate kar +orcla pex +non native +nob ita +near ness +nat yam +n ells +mlb memes +mik ami +mi fa +melissa joanhart +mau die +ma kav +lynn haven +kin cade +khil afat +ke wau +kcj hoop +kav li +jire h +javel inas +jare k +itsnotre venge +itsme abir +io sh +ig ala +i vel +ho ds +hassel back +hack the +goof balls +go time +gh yll +fr nt +fen lon +faith ful +e wn +dow ding +dol lah +do stum +dis organised +di pti +deejay ing +comprehen ding +clo sers +cho ppa +ceta phil +celtic thunder +bu lut +bor relli +bnsfra ilway +bel son +bargain hunt +ari ston +ambu lance +ai ono +ack athon +, / +ðŁĺĤ ðŁĺ© +ðŁĸ¥ : +ç¾½çĶŁ çµIJ +ç¾½çĶŁçµIJ å¼ +yang gang +y csd +wasse ypur +tur u +ton le +tile hurst +the second +stu ding +stra ke +stip ends +som izi +sok cho +sick kids +sche chter +scalab rine +sam park +sach deva +s shs +rin sing +rach man +psycho analytic +prolon gs +pla iting +peven sey +param ahan +palae olithic +p ame +official ecfc +o goni +north bay +nex po +new baby +muje eb +mu go +movie fone +mer sing +mc ts +m sum +love books +le day +latel ate +ku miko +keep cal +jul lian +ice fields +hog nose +glaston bury +fs north +france se +fps bs +fau zi +ep cot +droid con +dor ota +deb u +de hydrogen +daylight savings +dan juma +cru dely +con somm +colec tivo +cc ps +calvin ist +c gp +bur jal +bricol age +bran nen +bor ra +bo ffin +basket bal +balo chi +artem chig +apro x +ap ub +ap tist +ap rc +ani ket +am bl +algonquin park +ad c +* ) +ðŁĺŃ âĿ¤ï¸ı +âĿĮâĿĮ âĿĮ +zer of +y out +wild fowl +we scott +we ide +vivac ity +unimagin ative +travel ingram +tor point +ti mat +theroyal parks +te da +swind ler +student en +starsand stripes +spu moni +spear ritt +so hot +slumber land +sidd han +shock waves +screen rant +ru da +ros eng +roof e +pren tis +porth cur +pastr nak +partner sin +paraly tic +ok tay +mountsin ainyc +mohm and +mg madvertiser +maal ouf +life forms +legiti mize +kit aro +keepingpeople safe +just announced +jugg led +jai ram +ja ket +is ky +io annina +hit makers +hard liners +h ise +galla her +frontend friday +frank linton +fra il +fay ez +fak ery +eri vera +equi v +ell ick +dwight yoakam +drum moyne +down falls +del vaux +david cassidy +cori olis +copper plate +co ggins +catch ings +bryson tiller +brig ada +bra f +blot chy +bennel ong +benedi kt +bear kat +bandain am +b sac +angela hartnett +alla re +all eni +akashi c +ab ello +åĨĻ羣æĴ®ãģ£ãģ¦ãĤĭ 人ãģ¨ç¹ĭãģĮãĤĬãģŁãģĦ +ภ¶ +اÙĦ Ø· +zim v +zan go +z opa +z ellers +young bae +yi ps +xiaomi india +water wheel +w bl +vo icec +vincen eil +v ur +us lim +ti gra +the answer +th h +tal do +tail pipe +t yee +t jc +sz il +sz ab +suz aku +stanis lav +sk zn +sk ur +sco tians +sc it +sas software +saint lucia +sa ade +rf q +repet to +realestat enews +rat zinger +rain fow +rah mani +r hq +quic kies +pro pe +pin nell +palad in +nu eve +nic ott +nau de +na stier +mö n +michael keaton +mic eli +mccor mick +mann ingham +lor di +limited run +labor de +khary payton +internet day +ho stos +hi wx +grand lodge +ginar aimondo +ga stein +g po +final say +fant ine +existenti alist +eric martsolf +elie zer +electric guitar +ei de +effic acious +double lift +dhar y +derick pauls +cream sicle +cor field +com ino +cargol ux +can ig +ca prock +ca asports +c pc +bob collymore +beaz ley +athene um +ash wag +aro ya +ace day +a oficial +$$ $. +! ðŁĴŀ +ðŁĶ ī +ðŁ¥ ª +yu ichi +writing day +wevel gem +vil joen +v ona +uq am +unis dr +tip tronic +ste v +spin all +sihanouk ville +sheffield star +sha hir +sandigan bayan +sains bury +ro amer +re manufacturing +ra des +pyro clastic +public space +pric kett +photo gr +penta ho +pamban sang +ow sley +o ton +nific ent +never yield +mel ds +mc chrystal +marath oners +ma ay +landon donovan +kor at +kipp ah +kal ish +joele mbiid +jo see +jetti son +j ri +indi ap +hurri edly +hatter sley +ham are +gully boy +gov ballnyc +gol gotha +gla ucous +ger rie +g app +fri gate +elve den +e ous +down playing +dor ados +cry ption +cor rell +cher ub +ch ome +br onews +bio g +beach head +bal dry +ashley purdy +ang ella +amy ra +alvar omor +agronom ists +acu edu +ac or +ðŁĺŃðŁĺŃ âĿ¤ï¸ı +ãĤ « +à¹Ģภ¡ +z azz +yo shim +yellow ed +work it +wh iner +weare texans +wantt omarry +w marybeard +v ong +univer sum +un do +twi ste +tic kle +tee pees +tanz anians +supportyour local +ssouth africa +sol enn +soil work +sm p +skybetleague two +roit feld +rh y +re heating +r uri +plaque mines +pil ton +pag od +outand about +ori x +nic hts +mill stream +maxplanck press +marylou mcdonald +mar aca +mac books +lul led +lo der +liv ening +li gious +letter heads +legar rette +lang tang +la by +ko co +kam in +jme bbk +jacob in +igu chi +he ffer +har aam +han si +goddam nit +gang way +friendly breaks +frankie edgar +fol ge +fire trucks +fi resi +fene stration +f ones +ei gh +du kin +don alds +discipl ine +disc or +din goes +descend ant +dere kand +den ki +ctv calgary +cre ager +con drey +chatt yman +chadwick boseman +ca storm +by akuya +bru tha +bran ded +bj arke +ash gabat +anh inga +an sp +alli en +ad at +ðŁĩ © +ì§Ģ ìĽIJ +yal sa +xx viii +world soil +wetnwild beauty +vol cker +vijay ak +vide over +urb act +un traceable +ud der +u dra +tox ico +thér èse +thanior uvan +td cs +swis best +suppor tin +superstar mahesh +stream flow +strat aconf +step child +sp iteri +simply shop +shu hada +schwarz wald +sal ur +sa heed +re penting +r under +r ry +qi bla +port ly +pooja sharma +pis ano +pel is +pel ham +pay phones +paint sville +p mac +oul ter +neo cons +n oooo +mull is +mose ley +moo rer +mik ro +me ggs +mayorof la +main er +ma hou +lul lah +loch ness +lo vis +lime scale +like us +l pp +ked zie +jo vian +jimdunlo pusa +inn smouth +i stra +i ami +high cliffe +health week +h cs +gol fer +ge tag +g enda +famili arise +faces wap +ene wsroom +discover ing +dim wit +delic ous +dan z +d hur +cuc cinelli +cor stor +chen al +che ch +chart buster +be spectacled +bar mer +balder dash +back strap +b hon +as sts +am pt +allo c +all waltrip +albumo ftheday +:(( ((( +: ), +. ðŁĴĶ +ðŁĺį ðŁĴĵ +ðŁij¶ ðŁı½ +ðŁIJ· ðŁIJ· +âĸĪâĸĪ âĸĪâĸĪ +Ùħ ا +ÌµÌ ¡ +¡ # +zzzz zzzz +virtu spro +un awares +turtle tuesday +thisise dinburgh +thenew shour +te um +stark ly +spre esy +spreesy co +sor l +shay an +seren ay +scow ling +sche herazade +saf fa +ros seau +riz k +rin us +reebok classics +red letter +re negotiation +pulkit samrat +pro geria +prevention week +pit sea +pine top +photo copying +ou dh +oppos ite +objec tified +o ire +next time +neuro marketing +nbac ares +nandish sandhu +meaning ful +man ara +labor al +l mpd +ko vu +ko sho +ko kk +kidney wk +jarryd hayne +ivan o +ip k +in stil +he ti +gucci fer +gon dry +go back +girl streamers +fl annigan +extra ño +extermin ating +eee et +dri a +chapter tweets +carl fogarty +caraba ocup +cabinetre shuffle +c ssc +c spa +buck aroos +brachi osaurus +border town +bol c +bed n +becu ase +bbc tees +bbc gw +aviva stadium +asa ints +as ara +art dielle +antibully ingweek +air wave +ag ab +afric aus +( ; +ðŁļ ĥ +ðŁĺįðŁĺį . +ðŁİ ³ +ðŁ¤ ķ +zay ed +yar gentina +y mir +whit low +var on +us sen +truss elltrust +trans dev +tran socean +tonsil lectomy +think tank +the vulcansalute +tele ports +tear sfor +tamil cinema +spur ned +span kin +southern miss +samar itan +sam mie +sa huar +ru ppert +rsp bin +rspbin theeast +ri mu +record able +re ch +q ande +pre gun +pine wood +philli ppi +patt yar +pag pag +op ry +on wu +om itting +odemwing ie +narcissi stic +n komo +mul liner +maynooth uni +m ür +leon bridges +kwe sta +jo kanovic +jessic ak +jav it +instaf reebie +in cun +housel ive +gy les +go shi +friends first +fox fire +fal vey +fal com +etz le +en cana +ela ina +echocardio graphy +dr n +do gon +demol ay +dar at +dam nn +d anya +cosmo polis +core opsis +co stin +chichar ron +chay kin +camden market +cal es +cable vision +c mi +blan chet +big query +bech ler +bar os +arti s +afl w +afam ilia +abbo ts +ðŁ¥ ŀ +åĩº æ¼ +âļ ķï¸ı +yu ria +yow za +yac ine +y ber +wunder man +wood pile +watt les +vo vin +vex illo +to simplyshop +the wilson +tetsu o +tb c +tab or +star tl +sports desk +sp icks +sol ingen +sit ton +se yes +scul tural +sam aa +sal tim +s bl +reti en +ram aswamy +pre ah +plumb ed +pin ole +pier is +ou cher +o idal +night shirt +mums net +mean sall +me uk +me op +mat tt +mart elli +mam mon +llor ona +light ship +lao tian +landsc aper +kun lun +ko jak +kca argentina +kan oo +kamp us +ju shin +jay jay +islam ization +horror nights +hol is +hill yard +ham man +ha pag +h wm +gar ima +gang stance +er ges +endur ance +east mid +dutch town +du f +diab o +depos itors +del erium +cu ssing +cor deiro +cham ac +cardiff council +cam ii +c ter +c ico +bor nin +bol ting +bol ds +ber o +ber ling +belle garde +annnn nd +anas am +adal at +ad lard +ðŁij® âĢįâĻĤï¸ı +ðŁĨ Ĺ +ë¡ľ ìļ° +zer heide +y aser +who dares +war cry +uss c +urbang arden +til brook +the talk +the danny +tar ant +t max +stere rs +spar knotes +skate board +shiv aya +shi e +sab al +rain hill +rac ism +pul lup +poul try +poly count +po lit +ping ree +parl ours +para ÃŃ +pang ppopro +ow ol +ow gr +nick ers +nfl fantasy +my elin +mr jafri +mo lesworth +ml scupp +missou rian +mary knoll +manju la +ma hala +lut fi +lam ov +ky yon +kon oe +kil donan +kers lake +ira wan +inesti mable +in accuracy +ib ti +hus sen +hk racing +gri se +gre villea +go so +glen wood +gh eroes +gar my +flower pots +en aire +ei ke +dupont pioneer +desi erto +democr ac +debr aruh +dead weight +codepend ency +center nyc +by our +british farming +bra si +bergdor fs +bel on +bail able +ay anna +auro ville +att ard +ard han +archan akalpathi +ar rc +ar dent +am oroso +ìĽ Ķ +ëŁ¬ë¸ Ķ리 +ëĵľë¦¼ ìºIJ +âľĮ # +Ï ħ +val miki +trav ag +ti pit +tetten hall +tcho tch +ta wau +t fc +switch over +sub servient +steep les +sko whe +silver dome +sign um +siesta key +shor se +shek hawat +sa cher +rob breport +re produces +pre achy +pl s +photo shops +pen shurst +pattyar quette +pakv wi +no deal +nis an +net gov +nationallibrary week +my desert +mu rid +mil v +melis sad +mcly te +mbalu la +mat ahari +mand ana +man asi +london theatre +lo san +lo ll +ler ner +laluprasad rjd +l cr +l bb +iron bound +inve sco +hye sun +horse heads +hope toun +he ise +hab bo +go browns +ghost writing +genealo gist +ge aro +fron te +fer ing +feather ing +e acc +decision day +de vice +d cr +commiser ate +chin ned +che me +cephalo pods +cal amit +bra bazon +ble k +bla ire +bin os +bc ci +balde agles +athle tically +at uk +ancient rome +adam baldwin +ache let +ac che +âĿĹ âĿĹâĿĹ +Ñ Ķ +ye ou +yah i +wy at +wor kexperience +weekend fun +vin cy +up online +under performed +try p +trombone shorty +thermo couple +targe tt +south ville +sna x +smo ving +sl acked +shehu sani +sh appen +seg agenesis +redd itor +re configuration +ra ther +practic alities +por tales +person ify +papar azzo +pal at +our nameis +ong ma +o len +o chil +mot ti +michael avenatti +mf doom +medi ag +mall rats +log ous +llantri sant +lic e +le strade +john bishop +hills boro +gy gax +gram ercy +ge tex +g nat +for yourself +fel i +faz l +epp ley +dog sin +dl h +de wal +dam isi +cu eing +chint u +chi avari +chau mont +categor ia +car ny +calcasi eu +bur gen +bell in +be aky +bas ler +azu car +aw es +at chie +assetto corsa +ask ham +an tero +amar tin +am paro +ak aroa +acom ets +ðŁĺı # +ðŁķ µ +é» Ĵ +çī © +wo ah +whit enoise +victro la +un blemished +tu de +tie polo +thehip dotcom +than gam +th ax +tele porting +su mona +stone hill +spar ro +sime to +shi ur +shi kamaru +sh emp +selec cion +scar avan +sam plings +sade ghi +s ket +ru so +ri ise +recap tcha +real skipbayless +raj path +pol ara +pap acy +pad dled +official bvb +oceano grapher +new ser +nage sh +mun ky +mise z +ming a +mi shal +mi gros +mal vasia +lionsden kxip +le be +l drs +kra vis +ker is +kab ini +jol ina +jetpack joyride +jay me +istom in +is sam +ir kut +improvis ations +i would +high line +hart ke +h ran +gu ell +google plus +godd am +go ering +gen ce +gal op +fi sto +farqu har +expon ents +electro shock +east in +du plin +dav inci +dance y +cu i +coom era +chit rib +chas eric +cat te +can tanker +burn ley +bi valve +bi key +bett ye +back draft +ay ou +af al +ach ery +ach el +acap ital +ðŁijĢ ) +ðŁįºðŁįº ðŁįº +ys ss +winter bourne +whit eville +whir ter +un reservedly +tubu les +thy depark +the terminal +the matically +ten bury +tel cel +source fed +sop o +so ji +sis land +seung min +san son +sa shi +roger splace +rockof ages +red deer +ra ult +r wr +pu cca +pre destined +polyphe mus +pastor ing +opti plex +okay player +o zzi +o ssian +non ukes +nih ilistic +newcastler aces +mont verde +min ella +mil as +micro light +mi zan +mat raffic +man ie +lux o +lucifer onfox +lu ll +lipo protein +lew ington +ler adio +ku pe +kore y +kno bby +kick off +k rank +k aci +junkyard blonde +je bb +itte faq +inside out +hun gr +hop man +hay dock +gt masters +green power +gli b +ge mein +g son +freo dockers +england hour +en vies +em x +edin travel +ecan al +ec an +e ik +des demona +de arer +daysof type +corbi jn +cor mac +conni ving +chand ini +bon su +bi bis +bab ai +auti ful +at weet +at rue +ash leym +art chat +aps virginia +am lin +almo sthe +agn ès +af rin +ðŁ¤ŀ ðŁ¤ŀ +çİĭåĺī å°Ķ +âĿ¤ ðŁĴĻ +âĸ½ âī¦) +woo oh +win er +vijay ad +us ffootball +up wind +tri gon +tor q +toni oli +tom fletcher +todd haberkorn +throw ing +ta vo +syste matics +sk int +si tharaman +si pp +ser toma +sc ouser +sat anists +sassi est +sari ka +sale hi +robbin sville +rajesh m +r du +po teau +pit b +pic anha +pall u +pa wl +official nihr +nr and +ni pa +neg ative +ne ase +mar n +mafi oso +ma zy +low light +looooo ove +ld s +lambof god +l ous +l ke +kasar ani +jon gordon +jc chasez +indoctrin ate +human o +hi bsofficial +hand sup +han ratty +ge toff +gayle king +fy ah +force ps +flume music +espan ola +du che +disbur sement +diph theria +determini stic +del or +dairy land +d phil +cle eves +cil lessen +cali bur +c eller +bu ñ +bt q +bro mine +bon dsman +ben ares +as sc +and aran +all arin +aic te +ac cen +abby wambach +ab ish +. ðŁĴª +( ?). +% !! +ðŁij¯ ðŁij¯ +ðŁĮ ĵ +ðŁ¤Ĺ # +ìĨĮëħĢìĭľëĮĢ 미ìĬ¤íĦ°ë¯¸ìĬ¤íĦ° +é ³ +áħ ł +y adv +wor c +wam pler +walks britain +ured ome +tu bb +to your +to fu +ti ran +thu is +the joint +the flash +te bowing +taw se +strick lin +startl ingly +south old +som bre +sk ala +shawnat spg +shapeof water +shan kara +se her +ri poll +ri ble +re took +raw don +prohib itive +pres sed +ply ometric +pic abia +peter j +ou thern +one ws +one ofthe +nw ambulance +normal ise +noise less +new balance +min ni +mh b +meur on +mel lowed +marat ona +mad hatter +ma str +lore ena +line ages +life ok +laat ste +jun jun +jhah neu +intelligen ces +inst ers +ini ketan +hu ddles +hool ahan +ha wi +gran ville +gor der +gh era +gang won +g oms +free taheri +f ctc +exoske letons +escam illa +ed al +du roc +du ller +dari of +cou cher +cliff hangers +circul ates +can tos +camp liberty +c aguas +bru ma +bri stowe +bodh gaya +ban ski +aze ali +au chan +anim alistic +alof thotels +alen ia +al ı +akh ter +' ?! +ı ï¸ı +âĹ ¼ +âĢ ¿ +ye vent +wxpn fm +visit causeway +victor yday +ver tex +ve ve +ve dic +un knowable +u scen +trot tier +the fla +the bottlemen +than u +tag e +t center +str ø +so xford +skin nies +sk inn +she kau +sarbj it +sam eh +saf ford +ru thie +reti rements +republi ka +rad stock +ra el +qui bble +porsche tennis +po ong +po esy +photo play +of thenight +not doneyet +ning aloo +ne ak +navy pier +mv l +mueller time +move in +mo hd +mnu kteam +mlscupp layoffs +mid line +mehreen pirzada +mariecla ire +lasti mosa +krat is +ki ev +keaven y +ir responsibly +ir me +inocul ation +indu bit +hu tter +hr x +herb als +hd fs +ha bib +gu as +graci ousness +fro myour +four ways +fly te +ey er +ever thing +ev agreen +ero ach +emce ed +embal ming +elis icki +dun ton +dow se +dg pup +ded wards +cornell feeders +ci elos +cal andra +ca shout +brü cken +bot z +blue screen +bet sie +be ene +b wo +asper g +ang or +*¨ *âĢ¢ +ðŁĺŃ @ +ðŁĶ ³ +ðŁĴŀ @ +ðŁĴĭ ðŁĺĺ +ðŁıĪ ðŁĴ¯ +ì§Ģ ìĦ± +xiv preprint +world views +wood fired +west virgini +ve atch +var as +u bah +tru eno +titan pride +thyl acine +team europe +teak wood +taver ners +tam worth +sy outh +stan zas +solan um +shrin er +sepp uku +senbob casey +sa the +rummiku b +rude boy +ripper street +prin ts +premi x +pp g +ph la +pen sami +pe ma +para professional +out weighed +or ce +onomato poeia +nu tini +nail head +n kp +mr br +min dof +meyero witz +menshealth mag +mccul ly +mat unga +maru chan +maharash tra +liqu igan +li ssie +kyyon voot +ko gar +kin ver +k xl +jon foreman +jay an +ir landa +insomniac events +in saf +in fraction +ic ps +hu el +hol tzman +hispani ola +haz uki +go ggin +get kahoot +g itter +furn iss +fuer tes +fro mmer +far on +er lich +el well +ed ney +dece ives +competi zione +cimorelli band +ce ats +caval ry +ca chorro +bu ya +binib ining +ber thing +beg u +ash gate +amicha els +amateuri sh +alanthomas doyle +Ĥ ĺ +æĺ ¥ +w das +vander burgh +us acycling +tu pou +trou ts +tor ben +top kapi +texase dm +t lg +sumb awa +sports zone +spill man +skun ks +shored itch +shi prock +schle mmer +sb k +ri ggle +reali ve +raven snation +raj pal +rais ingthe +pic eno +petco park +olloc lip +oh lone +o zi +o possums +news break +nc sen +national forest +mo doc +mar the +lucy beeco +lu ker +love scenario +love cats +longlive the +longh i +lildicky tweets +ligh thou +licen sees +levit ated +lesbian ism +ko ky +kla sa +kit bag +kill la +jonny bernthal +jobs act +jer ic +jam erson +ja har +ilove gay +hypochondri ac +huff po +hub ley +ho los +hamilton ian +gy re +gt ld +gou ste +go ren +gi gir +fri pside +flet ching +exp end +entom ophagy +ei agov +eh feuro +eas ington +e via +duke dumont +dat alake +d orion +cro ll +conval escent +conting encies +coloni alist +coal fields +coa homa +co tham +chol ars +cake boss +ber row +and win +ahon y +aene id +! ðŁıĪ +ðŁijĩ . +ðŁijĢ : +æľ ī +yu shu +vogue india +twin z +thought leadership +thaanaaser ndhakoottam +su bash +stri ated +str ats +skate parks +six er +sign alled +shel drick +sabin elisicki +r risd +panam aleaks +pan fish +pan cham +ossu ary +or sa +nr hs +norm core +nght mre +mt lg +mid section +metro sexual +memori alizing +mc get +marvin humes +mandi ra +mack illop +m ally +ler ato +le ik +lau raj +la violette +ko izumi +ko est +knight sof +jo ch +je sup +j pp +izquier do +is obar +im bru +i pra +hungar ians +hiro ko +hair transplant +gro bb +go global +ging ras +gi ak +gar st +gam mage +gali l +g sl +free thinking +et ches +ense mbl +eat right +e ski +dji bril +con fab +colle gen +chaseric emusic +chang ingthe +cha hta +cav azos +carri g +can bball +burt ka +bmw group +bio sensor +best team +bernie in +bar ona +as px +app art +ap ital +anhydr ous +angeli ves +alph ard +alped huez +aero india +ad jaye +. ðŁĴŀ +---- ---> +ðŁļ ī +yu liya +young stown +x bla +winter ized +wall ac +wak eling +vital voices +v sf +v endor +un foundation +ty coons +tw ende +trans disciplinary +trade speople +the jeff +the james +ten ko +tempe stuous +tau ren +swan seab +sun bae +su ez +su crose +sty al +stopp ages +space suits +shiz uku +sh unga +sex party +repar ation +reen actors +ree king +raym und +pre vue +parten kirchen +os se +nih director +naz arian +myo sitis +my vi +mountain tops +mar cial +mar cas +mar al +manz oni +lucybeeco conut +lie tuva +kur tv +kitt itas +karlo vy +ipsos mori +insu lin +in ale +how we +hav anna +hah nemann +hab si +guilden stern +gran dia +godis love +go viks +fru ity +free kashmir +fox nashville +fou gasse +flat test +field photofriday +fiaf arn +fiafarn borough +ey talking +entren amiento +dont lie +dis believers +din is +dick o +des met +daysto gofor +day star +d ilo +countryfile mag +co system +co ati +cle anc +classi fies +byo c +bun ting +bos stones +bon fox +ber it +bbc newcastle +bag ile +ay ani +avan tasia +ary land +ap ap +andalu cÃŃa +alyn n +all art +alder wood +afterel len +ðŁĺ ¾ +ðŁķº ðŁı» +åł ± +ãĢ Ĭ +wra ss +who re +we stray +watch u +villan elle +ve sak +tun er +tor an +ti ffin +thehistory guy +the boston +the ale +that sme +tg d +tele dyne +tal eg +tai fa +surf sup +steeler shockey +spart ner +small bone +slic e +san tucci +ro ces +reid sville +registr ars +reen acted +rad dest +quin one +pyroman iac +ps b +practic a +phra sed +petty fer +pete tong +peac elove +nikit ind +nc p +my horse +mira belle +mi era +mc gar +mal lik +mad woman +long way +l sn +knock knock +kick er +jet liner +jag d +jab a +instagram mers +hu tt +hi ji +h ts +game music +fang led +ex onthebeach +eller be +e ah +dv sn +dro gon +ded mon +dan scavino +da sty +cross field +cricci eth +commerci alism +col ls +ch c +cas ar +can im +can ball +cambridge utdfc +c whm +butler u +boer se +blow dry +bic hette +ay nrand +atis faction +anaesthe tist +af jrotc +actor sequity +ðŁķº ðŁķº +ðŁĴĸ ðŁĴĻ +âĹ ī +zi ii +you might +vie we +ver ie +u rad +tw afterhours +tro wer +tri aling +tr ono +tool shed +thermost atic +tessell ation +ta ky +star fall +sle tter +silver leaf +shra der +shannon airport +se dap +sber bank +sav at +sar aramirez +ror aima +reci be +ray ne +pro pre +pin i +oh canada +north land +ng cobo +modu lus +mis ka +memor ising +mbl science +ma pping +longu euil +lit eco +lee derville +leav ening +land seer +ki dero +jin do +jil in +hue hue +hon en +har har +green and +g tw +foodin dustry +fas lane +express andstar +eli eve +ek blad +ear marks +dwind led +doherty shannen +dism aland +di iv +del fi +dat aw +conserv atively +coffe red +cityof miami +circum navigate +chapp el +cecil ie +canad achat +calz ado +bra venation +bor chers +bell ino +be parto +bab s +b bler +ayo s +areyou withus +anto griezmann +amar cord +allarin aresh +abor tionist +!! ðŁĺĺ +à® © +zbig niew +zay d +yugo sla +wul ff +wotc staff +wn f +whit ish +w gb +ve ale +vas yl +udd hav +trin sic +travelwith kids +tou k +th yn +tele work +tan sey +tal ab +super wiki +stun i +sto ch +stat eli +so gn +sj t +simm ers +si bles +screenshot ted +scott grimes +schoolo frock +scar r +sc ahill +sali l +rpcrd pro +roc ke +ro gow +rel ton +refu elled +re cut +pro ops +phreno logy +persi mmons +pe que +pa shin +or za +op ier +north town +nan ticoke +n didi +mix to +mal ene +ma gh +lo vies +lo fted +lil loo +lac lede +khun e +kate moss +k msl +java one +jam elia +j lee +j allen +irkut sk +iner tial +house maid +holo fernes +good time +f pb +em mit +elon university +dun vegan +dannyjohn jules +cyber security +cuuu ute +confir ma +cn ni +cl ery +bul ked +bs thydepark +bio reactor +bbsr buzz +bar bar +b jo +az azel +az az +annou cement +activ ity +acar ter +Ħë ² +天 å®ĺ +zodi acs +yummy food +y omo +wild storm +wi j +wh ys +vu cic +vali k +u shio +u art +ty me +tsu kuba +te yan +tab ri +station ery +slo sh +ru ti +roy all +recruit ing +rav eng +ragin cajun +rachel wood +qu tb +push kar +play mat +pilgri m +pepp eridge +pat as +param edicine +om ran +ol uw +ok uk +ocean optimism +o bia +ny sc +num é +nicode mus +naz are +nap thine +mo tz +man they +ma kovsky +lox ahatchee +letsgo g +lebe dev +l rr +jan ey +inno trans +in p +i acon +hongkong ers +hol royd +hepat o +havi sham +h nn +h lr +gur lz +fish sci +fare ed +ez us +eti xx +dur yea +dulil lah +diy ala +come home +co ar +clear inghouse +chi bs +bringerof rain +brac er +bob sinclar +bo hl +big kid +bel ch +bar qui +automobi le +atta m +at au +app ley +abu da +ðŁĺĭ âĿ¤ï¸ı +ðŁıĨ : +âŀ¡ âŀ¡ +à¹ĢภĪ +z ill +youtube ãĤĪãĤĬ +yoshi hiro +waynes world +visit somerset +vign elli +vamos rafa +usu sa +tat is +taj ine +sundaytimes za +sub t +sne p +si evers +shraddha srinath +short sighted +ser bian +screen savers +scou gars +sacram ental +rode mics +retrospec tives +ren tz +re focused +ran sack +ram anathan +pu pe +proven cal +poth older +plast isol +pan can +ome x +om us +ogun damisi +not for +ng oma +nel ley +ne yo +nassi b +my ke +mul lagh +modisar kaar +mmm bop +mign onette +mdcpss outh +mase ko +man ni +man gia +mal y +loun g +lign in +lar bert +la ska +k me +k acy +jyo tir +jur nal +joan smalls +hi rose +grand ma +good speed +ghe en +get syou +esc ala +ent oura +du art +di yan +devolver digital +der f +cuu ute +chen ab +cap illaries +bren d +bi mal +back wash +ato cha +ask acurator +ami sha +americani zed +aly si +alma den +abbi amo +aar hu +woolla hra +wam ish +von da +vo gler +un club +ty j +su ja +ste ar +sput nik +sol are +si rot +shal om +sch marzo +rev dr +rep joe +remor seful +rei ff +pum phrey +plain sman +ou tros +os ric +niagar a +nag ata +mo watt +meadow vale +maison objet +mad han +m lambo +m elly +lor ri +lom mel +la ding +kol k +kh ooper +kar nes +je uner +jane goodall +it oi +ireland cricket +in gho +hackett stown +ha si +falset tos +encom passed +em tee +doh kyungsoo +dig nan +dej loaf +de filing +datab ank +cu kor +chro mat +chocol atem +cannon balls +c crc +bromeli ads +bro xton +brian topping +bogdan ovich +bob st +bl ini +bin da +bet elgeuse +barqui simeto +back list +arnol ds +apple live +angu ages +american football +alo c +ago stin +af ree +af d +adar ling +ðŁĶ © +yemen cantwait +wis sen +wedd ell +war child +ver aging +vapori ze +v pk +the highway +ten ov +te ter +subsi ding +sensation alism +sa fir +rv m +rockthe vote +ray lewis +pulitzer prize +promis cu +princi pally +piñ as +photography islife +pap in +ou ps +once in +ogie alcasid +ny dn +nvi di +ni fl +neon ates +nab ba +n mh +mytop college +mv sales +min ako +mean wood +mck is +mb w +mayo red +marka very +lub bock +le itz +lav ina +kope cky +ker in +herbal medicine +helen s +hann ukah +gor ged +glu z +gin ther +ger sh +ger giev +gener alization +gean inec +forthe soul +fin ds +dream girl +distracted driving +disrup t +digital disruption +dewor ming +der m +demer ara +de vey +dad jokes +dachshun ds +commiser ations +chi kar +ce z +cay lee +carbon aro +cam bro +cal verton +brezh nev +boost torbay +boo b +black power +bespoke bids +bear naise +bbce mt +ballant ines +awa is +annul led +alber tac +ak ul +acompan ies +ðŁijł ðŁijł +ðŁ¤¦ ðŁı¾âĢįâĻĢï¸ı +åħ ´ +âļ½ : +ØŃ Ø³ +в од +ya ari +world downsyndromeday +wor mer +wol fy +westandby parth +wait ressing +videover anomtv +urru tia +united health +tele porter +sre invent +snow ball +sky fire +skowhe gan +sc upper +sa amy +rush ville +rim shot +ri fai +recording artist +reading is +re framed +rakuten ichiba +racing club +r nz +presidential debate +plant in +pe dra +p ck +ornel as +opho res +nikitind heer +nakshat ra +na fe +muss orgsky +mur taugh +multic loud +multi year +mp sf +mon amour +mat ri +maidstone united +luc u +ko x +kkun dra +kerman shah +keo wee +jan ee +inck arnataka +i day +hu se +her sham +hal bert +gru y +gre asers +glynd wr +gali on +front pages +fear gal +fac ie +engel berg +du wamish +downtown halifax +dove dale +dibuj ando +di kes +consomm é +con der +code black +clu broom +class icist +circle app +chrono logically +chef stable +cere bus +center fold +bur gi +bir do +bin z +ballo ting +aron off +apple ton +algi eri +alev els +ae gina +! ðŁĴĻ +ðŁļ ¦ +ðŁĺ®ðŁĺ® ðŁĺ® +wine fair +wed lock +want sto +vol ler +vir tu +vil ify +value of +v dr +under takes +und pasiapac +un collected +top flight +tibi dabo +the point +sym monds +sub sonic +specialk brook +softg els +shi val +seid man +sand lin +san francisco +sam per +ruk mini +ru ma +rm liga +ri as +remember anceday +rachel riley +qu bit +pan elli +osteo sarcoma +or ay +ok mulgee +my god +mine shaft +mickie james +mess ines +lleg ando +kre y +king ery +ke tty +kawak ami +juli ac +jab harrymet +interchange ably +inter continental +ic lass +hyper hidro +hr zn +hijab s +head butted +hal sman +hal it +gu uu +gor m +gon iners +gar ris +game dev +fuji yama +fish sketch +fin bar +es rb +endaken nytd +du h +du el +deflec ts +corvet teracing +clat ter +cheese day +ca edchat +burton albion +broad meadows +brigg ate +bornfree onekiss +bor rell +block screening +be xt +be ddington +be ason +bati stuta +barnold swick +av d +ashi q +andreali bman +aman at +alu mp +alek sey +alcac er +ðŁİī ðŁĴĻ +ë¦ ¬ë +ãĥ¼ãĥ ī +âľ¿ ) +âĵ ¥ +â¬ĩâ¬ĩ â¬ĩ +ಠ¶ +worcs warriors +woodbin eracing +wherei root +warne münde +vy apam +vis co +ve dan +ve asey +vacation ers +us iness +tr illing +tony todd +tant amount +sydney storm +ste bbins +ss rc +sport sca +spic ey +south shields +snow bank +siriusxm nascar +sham n +se et +scot lands +sac chi +road show +ran king +rajiv pratap +rajivpratap rudy +poe sie +picto grams +pahal gam +pac y +ou bre +on oda +o ey +nil son +mill in +mi w +meridi anc +medec ine +lu tion +life ok +lasc aux +lal ah +king let +k sed +joh anne +jamesdro driguez +ivan ova +io ane +in y +ill inim +ibi zar +high town +head corn +harman preet +ham amatsu +gol ondon +gi andu +gavr ilova +game keeper +fuel your +fil ename +fficial site +etsy store +epidemi ologist +d bu +crim minds +co loc +brooklyn bridge +british ers +bour nes +bo sv +bio availability +bet ina +bell anai +band ini +bal at +alma z +... ] +ðŁĴ«ðŁĴ« ðŁĴ« +âľ ¯ +¡ ! +yudk bh +young sville +wh h +wear thebear +tp gallery +the drummer +terrac ycle +team natural +stri k +sofe vil +soc dems +sn pout +sil as +she aly +sh hhhhh +ser ape +sandiego zoo +ribo flavin +remo delling +refin er +raft beer +proud sponsor +preci ate +politic ised +pi um +phone tically +phantom ofthe +pdx eats +paint box +oon light +ome one +nz stuff +nu o +not all +no st +new bedford +my nach +mr sk +mom os +miguel ito +middle ton +mic r +metal smith +mar gi +mac c +lovenorth devon +lo lu +lin tz +leav ell +la thes +ku gler +ke ssel +k sd +jugger nauts +jn moyo +hu ie +ho sier +haz rat +ha vil +gaz elles +fet zer +fall s +eu cerin +esp ad +ear plug +dress age +dispos session +discover hongkong +dior amas +diesel gate +di ener +di abolic +den y +crypto zoic +cor tez +coa stuk +clive barker +c ays +breathe carolina +breakthe taboo +ble ating +birthday wishes +bar don +avox el +apho tos +ak ka +ait cofficial +after birth +@ ' +ðŁĺŃ âĻ¥ï¸ı +ðŁĵ¢ ðŁĵ¢ðŁĵ¢ +ðŁĴķ ðŁijij +ëĵľë¦¼ìºIJ ì³IJ +ë§ ¨ +виде о +we it +vest ager +un managed +un coordinated +tol ga +tin at +tat er +swash buckler +super mini +stra wn +sten y +st ura +spo ked +skiat ook +si phoning +she diac +seraph in +sang oma +san h +sag rado +ro td +review journal +replac e +ra ia +questi oner +nor sk +nene h +nar k +mole fe +mod sun +mini on +michel ine +men deley +martin son +m grs +luxuri ant +lt da +lig ure +le we +latin x +kish twar +kind afunny +kas kus +k assem +jewel staite +jag meet +ib r +i ut +hoo pes +hol on +here tical +henri bendel +hel lions +he sham +hatch backs +hammer fest +ham taro +half on +h bday +go abuzz +glaci ernps +gen evi +gau m +fn ma +fi rel +eric ripert +e ec +doo kudu +dfat irl +cou pla +cork citycouncil +colombi ana +camp sie +cal co +bride well +bowery ballroom +bj arke +bhic how +bb g +bare foo +baby wearing +at unga +asimb aj +arab y +ali velshi +al pines +ago today +abou tit +ab ff +ðŁķ ī +ðŁİ¼ ðŁİ¼ +âľĭ âľĭ +âĢį âĢį +à¦ Ń +ü ber +à ½ +y shop +wwe chamber +women inthe +war lick +vik ash +vid hi +veik kau +val vano +val son +tune core +sunny dale +ster ry +sk nonline +sil sden +sh ene +sestri ere +sensiti ze +se amount +santo ku +rep ented +raj shri +ra ir +prof jnmoyo +po grom +planetofthe apes +pine au +per centers +pe dder +our city +official y +o ib +nis se +neversay die +ne tro +national museum +my data +muse tte +mon ach +miumi u +maur ic +ma ska +le mur +krakat au +kid swear +khu zestan +jam min +inde sit +iah sfb +heroes reborn +hal awa +guin nessi +grow ingu +gregor i +gost kowski +go dess +gi ac +gh orns +geme ente +gaj endra +fer no +feliz navidad +farm hand +f illion +excu sing +excer pted +eno teca +emo to +du puy +dong feng +devin book +datab ricks +dar yn +d formmva +cri so +crew member +con genial +comple xions +cine phile +cer ati +cap tu +brittany snow +bree den +book snaps +ati mein +ari hant +ange bote +anatom ia +amylo idosis +air shows +acam era +... ðŁĺģ +ðŁıĭï¸ı âĢįâĻĢï¸ı +ðŁĩ®ðŁĩ ¶ +yam auchi +wo su +witcheso feast +whor se +wan del +tree beard +tom ek +thre f +te end +sympathis er +sunday fishsketch +selfiefor nash +sei denberg +sa or +ro throck +rin do +re taken +pressclub dc +photogra p +persecu ting +pepp adew +o ves +nmb co +nick lin +ni ese +mthe thwa +migli ore +mi gi +mbalula fikile +matt miller +mal ory +kuber nete +koin onia +kil syth +kcaf av +kat ong +karun chandhok +jo stling +irreconcil able +insi den +in h +himan tab +gur gling +gh ole +gashap on +fun chess +fraser valley +fozzy rock +fer rata +f wt +elec tra +dir khooper +din akaran +denver nuggets +dal ecar +ct gla +corbyn besson +cap gemini +by any +bull pup +builtby bama +bix ler +bi xi +be ma +basketball er +ballo oned +ba bey +arm wrestling +angi oplasty +al astor +ðŁĶĬ : +ðŁĴĹ ðŁİī +ì¯ Ķ +ì ´ +æŀ Ĺ +æ ½ +âĺħâĺħ âĺħ +yi annis +video taping +va stu +us ury +uhur u +tor tuous +the conjuring +te bet +tar red +t swana +t jo +sy lar +swim min +swach hata +sto ats +statist acharts +stag gers +sher rard +sever ide +sekh met +sch ad +saved bythe +salvador dali +reve led +re convene +rah saan +pu lao +pla ga +ou lou +onenation oneteam +nick swardson +necess itate +muth oot +mo at +mis calculated +micro bus +metal blade +mel uck +megan boone +magic avoxel +m co +lo hia +lindsay arnold +li ub +klu gh +jant zen +jacob o +intar sia +herald hs +he ikki +he eling +halab ja +hahn dorf +glend ale +frederic lambert +france fr +ewn updates +essex wildlife +en m +elabor ating +eco village +duck y +de beers +da rell +cypri um +cycli ste +cha sen +capric orn +can ute +boston heraldhs +book making +bing es +bike week +best coast +benven uti +bayern munich +bath sheba +ban yo +ayanna pressley +ay b +autun no +au ria +and chill +alyand fila +ah hhhhhhhh +a history +wild turkey +whit el +whi pps +weare acmilan +wat rous +vo z +ver nors +ve res +vas arely +up un +un classified +tul lius +tide way +ti soy +summer underthestars +shil le +share holding +sd xc +scor ch +rady r +ra ki +q pcr +q ip +pla sa +peranak an +pel as +osh un +or oro +olemis sbsb +newsad l +newfound glory +ne vo +naku matt +n bar +music by +mon dragon +mo tus +mith ali +mil onga +mich ed +mer ril +marvell ously +mar da +m razek +lin stead +kn auf +kla k +khu l +kc na +july talk +jiang xi +jenny han +jennifer aniston +j aka +inter lake +imam hussain +ik f +hell en +h wy +goodwoodr rc +go thers +gen ny +gad dy +g ph +flu ttered +fle abane +fire land +fact set +exhal ing +en ders +dor ada +do gon +cow abunga +country house +car ledwards +breastcancer now +borgh i +bl att +be sler +be holding +barba dian +bab ita +b hind +avro homg +asan sol +aro sende +ar ps +anastasi ab +al aga +af ur += _ +ðŁijij @ +íĻĶ ìĸij +âĿ¤ âľĮ +⼠´ +you loved +yol ande +work ington +woody allen +wingless bird +wi wa +west wind +utcoach jones +up tv +un completed +tomor ro +tiny tales +thomas gibson +the baby +te te +te jon +stu lts +stra ker +star pass +sole us +so wski +snake head +shi amak +sfor change +see whati +schoen feld +s jam +roman ovs +rodri go +qui zon +que vedo +pre cap +pr ze +phil tufnell +pew sey +par atus +om gtr +neid hart +ne ad +mis shim +mend ham +marav ich +lv phantoms +luxury watches +low ens +logan lerman +li pson +leg no +kir schner +kang al +hurl burt +hornb ills +har ting +har bou +hage dorn +gop tax +gondol as +george washington +ge an +ep sa +du gger +do woon +dar ning +damon albarn +cri swell +cli ppard +chi ens +char ya +cas selman +ca etano +bé ry +bo ch +big wigs +bie ding +besse tte +berser ia +bel ton +bel lar +battlefor azeroth +ats community +at rice +asroma en +artic ular +art lab +ak mu +abstr actions +. ðŁĮ¹ +(( ((( +ðŁij¹ ðŁij¹ +ê² ¸ +ãģ µ +âľĶ ï¸İ +⬠Ľï¸ı +à¹ĢภĤ +y cl +x lii +world congress +v rain +ull rich +ugg la +ton the +toi res +tay ong +sumb astur +stein way +si rene +sd nd +sand ar +sam mer +sal mons +saint laurent +roca wear +resist trump +resi stive +rca records +pend ent +official s +nz l +nit ourist +neuro pe +nat alee +my sis +my l +multi plexes +more mi +mir ó +mil ose +migr atory +men fashion +medi afreedom +mad ilyn +loft is +lan ghe +kunun urra +kum u +jour nee +janine gutierrez +j cl +iro h +ig travel +hotstar tweets +hotell v +hor t +hor no +hehe hehehe +hard bound +haras ser +hahahahahahahaha hahaha +graci as +gla sse +gar ys +gan grel +gal eng +fre sen +exploren b +euthan ize +dy m +dragon lance +discla imers +dialo gue +dees nider +de sco +con fla +cio online +chel ps +c mr +bu di +boo ke +bo so +aure lie +anni ele +ang ill +al bomp +aber ger +__ _. +' -' +ðŁĴĺ ðŁĴĺðŁĴĺðŁĴĺ +اÙĦ ص +yu taka +yever after +will erby +wccb charlotte +ver tices +utic acomets +toast ers +thiop ian +telepath ically +surplu sph +sun nin +sun dew +srabanti smile +spectro scopic +spe icher +sorcere rs +smc gee +skin ceuticals +sk out +scy cles +sch ans +ru tile +rang anathan +quadric eps +q com +pur pp +puk kel +praise worthy +pipp ins +phar ms +pd h +p lied +or ser +obscen ities +nu bar +new sch +n vs +multip lies +mu lders +mo ye +mo gh +mil nerton +mikeo hearn +michi gander +mer rit +mens grooming +media works +meand my +ma hr +luc kie +loy d +long acre +ligh tin +khali f +key to +kauka una +kam is +john m +jack ers +is se +irri gator +i backthebirds +hunter pence +hin cha +hempstead town +frontline pbs +from target +feren gi +ethereum classic +elder care +east grinstead +e com +du pre +di i +davy hulme +dan ity +d kc +chaud huri +charlie chaplin +char trand +cap ing +cad air +bran ton +bordel aise +bat ata +basti ons +bas al +az uka +az mat +az aar +as ug +ar wa +any an +anglo saxon +agh at +aber dour +اÙĦس ÙĪد +اÙĦ Ø¥ +yyc flood +yis roel +yeou ido +women writers +wing stour +wi they +wi ddle +war dak +virg ilio +v bi +turn ham +trety akov +to taro +to gashi +ti y +through glass +ther isk +theli ons +thehockey india +tay ang +tail gate +swar ovski +sus bureau +support thearts +su ol +staustell brew +star dew +spart ners +seque stered +seal team +say yaf +sam ovar +salford devils +ren tacar +re ding +rc pi +pur year +prophyl actic +prati bha +pi anom +photo electric +pan icky +pan et +pak is +pa ree +outaou ais +on up +ob noxi +o sso +national voter +morphett ville +montreal ers +minu tes +michel rou +mi thi +lu ps +lot tie +loch nagar +lf cladies +letra set +lemu ria +le ber +labrad ors +krish nad +kint sugi +king shead +king bird +julie benz +jualan ku +info sec +in trust +head shot +hagu pit +gest uring +gav en +gar de +fan boy +fail safe +eit our +dri o +dou dou +cowboy bebop +confeder ates +che sty +char ros +br inda +be etham +bagh lan +azi oni +ator onto +art fight +ai shi +ag ot +ðŁĺį ðŁİ¶ +ðŁĴħ ðŁı» +ðŁIJ Ģ +ãĢĤ ) +ç al +zoon otic +zol ler +yus of +wre ath +word cloud +west woo +vo gu +unic ycling +un shaven +u toledo +tsu mt +trac ie +top spin +tn luk +tar ghee +tabde eli +surab hi +su hel +st assi +ss ants +spoo kiest +sh c +semper sum +seig neur +schloss berg +sch alk +sal ton +sal terton +ry x +rulesof survival +rhy mer +re iders +razorback fb +ragha va +q bert +pro logis +pre ethi +pown ers +pied mont +pat mccr +pais ano +not l +nith yam +nausic aa +nar anja +naj ma +moon shadow +min di +m musi +m bio +long ings +li geti +lear ner +kett ner +kc se +kal m +kad u +intru sions +il th +hu ka +hachette aus +guy verhofstadt +gre ts +geminic at +galli more +fru ta +fri joles +free holders +fli ghty +feld stein +feather ston +farne islands +far cry +er ite +el ss +eg is +eee ep +du ping +dr anger +down tow +do xx +destru cted +den ge +del h +dd ale +cor vi +cog ito +chi des +chauvin ism +cha ste +c ssa +burgun dian +bri ga +bill ym +ben ali +beau ti +bear doil +attitude mag +at ca +an den +americ agreat +alo e +ab ele +aay yy +a und +a bean +- > +ðŁļ´ âĢįâĻĤï¸ı +ðŁijij # +íĻĶìĸij ìĹ° +íĸ Ī +ãĥ¼ãĥ ī +yun que +ya as +y are +x country +world cu +waccam aw +visit europe +vad m +trau main +tot c +timess qu +tier garten +teaser tuesday +t fx +stro em +spacec amp +space jam +sot tawa +sor pren +son nier +soff its +sne tt +slu is +si bel +sheir gill +schwar zman +saltim bocca +ric ken +reg bo +re structures +r np +pukkel pop +pan ta +outdoor sy +oro chi +oni des +nutri a +nau ti +mrbr tg +monsterhunter world +mo ten +mal ahat +ma shre +luen ell +legen de +kr circuit +khilaf ah +kan aka +jo sa +ind berg +illustration hq +iam episd +hr ke +his ashi +hill yes +high fashion +hello world +he g +happybirthday ntr +ha worth +gre if +goon squad +gold tone +gold star +give me +gand onor +for tum +ea aci +e iti +dunnott ar +du mmer +drum step +depre cation +dan carter +cron os +crank set +cr èche +corregi dor +con lee +con cannon +clar kk +cla sps +char lot +casi raghi +bobby brown +bl andy +bas shunter +back round +at fc +ary a +arti stre +apple tini +agye man +a are +\(´ âĸ½`)/ +ðŁİ¶ @ +âĺħâĺħâĺħâĺħ âĺħ +à¨ Ĺ +whitley bay +wall coverings +vec chi +vang sness +vali ants +v ayu +tro pang +the asian +tham iz +texastri bune +tangi ers +sul le +ss rtg +sne inton +sho dan +shep stone +ross ouw +rob brydon +ram boll +pro co +plat ine +pis cis +pier ref +pc mc +oo z +on call +ol er +nak z +megaz ine +mayan smc +ma dusa +li mca +lang ston +la chance +kon z +kitz bü +king solver +jo ell +jab o +j ji +iren se +inas al +iam an +hat chim +hack learning +gé rard +greenbay packers +gr ms +gin ty +ge sti +fl ori +fid lar +faur é +fair man +ero ads +edd ard +dex trin +dem force +d weeb +cl ann +charlotten burg +cell block +cav usoglu +buchen wald +bru tali +bra zy +blo xham +ban ja +bak ri +baj al +auntie annes +ali us +ahar on +adop tables +actu ated +ab ct +ðŁ¥ ´ +æ ¹ +zi jn +zeis slenses +y oooo +y net +y gk +wy nette +weather watchnz +wa chee +voil Ãł +un dit +ty bg +ton ig +thisis me +thing sexpo +thegop jesus +thegentle author +ter mini +ta eng +sylve stre +stmartin spress +snow drift +shell harbour +sab ia +riet veld +retar dants +photo frommyheart +orator ical +noki amobile +new art +nescaf é +nc cs +n ades +mon gre +mini x +m chi +long sight +le marche +kre tsch +kin olor +kan za +jar rah +jag nation +jack theripper +j florez +iz on +instagram med +imo gene +hurst ville +how led +hazbin hotel +ham monds +gu ston +golf life +foursquare find +fig lio +f gv +em mag +drink up +downton pbs +demel za +crisp ness +country radio +consor tia +colin dale +coal mine +cine mat +centre town +capric orns +brook ins +bridge stone +bre k +bea vernation +bav aro +bak ary +an die +ðŁij Ľ +ðŁij · +ê Ń +å¹ ³ +zu sak +wi des +web ley +twel ves +ton it +thereal elvira +tha ws +tab uk +sun stroke +stand tall +ssi p +ss music +sp angles +so bre +sj r +simone tti +sim ha +shoe horn +shak thi +scientology theaftermath +sab lefish +sa inik +rsc anderlecht +ro ser +rick warren +re er +ra zo +preclu de +pr iti +por chestra +pen i +pap al +pal lete +ori an +or acing +ob h +na he +mol l +missi e +mis se +minic lip +meadow view +mcga hn +mazz u +market news +mar re +logger heads +lin deman +lear nin +land less +kur si +klo wn +ker see +k ichi +k hia +jul lien +jon gh +john mccain +jaw ani +infor med +ine se +hob nail +hi y +har jit +h sb +gent lest +gali lean +g ili +fli xton +fleisch mann +fivb volleyball +f ants +ey rie +eurovision songcontest +er gs +el bit +eidul fitr +edir ne +du man +du das +dis quiet +dige sti +dest itution +chim ilco +cep has +burjal arab +bur stein +bor da +blo hm +black hole +birthday gift +bell ona +bee zus +be aker +bam usic +ar nation +angler fish +ame era +all indi +ale kh +air india +acar penter +ac ffi +â¬Ĩï¸ı â¬Ĩï¸ı +wother spoon +wo kv +wit nes +vi aggi +van k +va ghani +thero ses +tam zin +t yo +supportsmall streams +steven j +speci alo +shaun watson +scap ula +s notes +rap music +precious metals +pic asa +p alla +op ara +no sta +neo gaf +ne un +nca i +nam ibi +min z +me thi +me kka +marse illa +mari ette +magni fies +m lo +le akes +latel ate +lat avi +kumar akom +kruger sdorp +kla ss +kinolor ber +kaz ant +kat usha +ity live +info arqana +in justice +i bt +hyun woo +hoom um +go alie +gly cae +gastroenter ologist +fa ite +epi graph +elast ics +e jo +do ernbecher +djer ba +dis engage +d ve +cu ria +courtneym elba +colin mochrie +co stain +cic lo +cher mside +chad lindberg +candel abras +ca dem +brooks bank +brandon j +bo sques +blood horse +bior xivpreprint +bi mmer +babe station +b side +ase ball +all dev +af fric +acon v +ðŁķ¹ ï¸ı +ðŁĩªðŁĩ © +âĪ © +unitedwe zag +uch t +trac ibraxton +tl railuk +ta oris +syn cre +sto kers +spind le +skop elos +sibusi so +shu bhan +se bad +sam po +receiver ship +re ats +puertor icop +prae torian +pe tanque +paul gallen +p sap +ox ic +out gunned +on io +odi sha +nur mi +nun avik +nicol ae +nag ini +mou ski +mis appropriation +min oc +mi mmo +mi mick +lar o +lanz ini +kirin yaga +khair ul +kat lyn +kar ne +job hunt +jack lyn +imperi alists +immobili zed +ie styn +i xl +hot mess +hey don +hat illo +hardrock hotellv +got season +gon etoo +go id +gare y +games youloved +ga stric +france schi +felden krais +express ways +el yar +disbur sed +dareal amberrose +d agan +clairec mc +centen arians +cam girl +big sky +ber u +bar rack +ap rakash +and tvofficial +amé lie +aman ah +am adi +aller ia +ah en +acffi orentina +accep tability +ac ct +ðŁİīðŁİī ðŁİī +ðŁĮĢ ðŁĮĢ +ðŁ¤ ² +ðŁ¤ ° +wor ley +wood mere +way up +wau conda +watch mojo +vo h +victori ana +v aga +uy gur +up skill +unitedwe win +underestim ates +u ppi +three houses +thinku hi +te ms +switch to +swee ty +sut tie +strathe arn +space bar +south tyrol +shi rou +save dallas +ruck man +ru zzz +ro stam +rj mitte +rex dale +ratch ford +priv ates +pol sky +plata forma +permanent makeup +pa eds +or ph +or donez +one m +nordic walking +nederland se +ncaas occer +national isation +nan ite +nam ita +movie time +mo kka +mary portas +mar len +ma icon +ma gui +lo chal +lo bs +lieu t +learn lap +kur ma +king snake +kee sha +kay an +kak ak +kab am +journe ys +jonmitchell itv +ilove travel +il g +hunting don +house dems +higa shi +he ade +h dn +golondon knights +gir ton +freddy krueger +fre md +fe dez +er land +enqui red +earth wind +ear dley +dove tails +diet coke +dews bury +delici osa +de go +cycl in +cy g +cur ri +cho sun +camoufla ging +black sheep +bch l +bay bridge +b chat +av ag +ameli a +am ami +agne epath +ðŁį» ðŁİī +ðŁ¤Ĺ . +z dar +yl enko +yak u +w canada +triumph ing +tre p +thi rai +ten aya +sty lec +stren ding +sq rt +sm day +si ran +shi zz +sher itage +sh after +sf opera +say something +sanctimon ious +qay yum +q ft +proc tor +prenup tial +pocom oke +phili stine +paren tage +pap ago +pa ke +over winter +ot suka +noah syndergaard +ne bel +mukun d +mif sud +manic ure +mac bride +liverpud lian +ken o +just inf +jeke vaaste +jan jua +j anny +ib ma +i ello +hermosa beach +hau gh +han ak +gum midge +ge table +garden party +fur rows +free way +fashion addict +farqu aad +electro acoustic +ekdu jekevaaste +ds ville +dor ling +do sages +di ye +dan gel +dag ibee +crypto exchange +congreg ants +bus u +bulldo ze +britt le +bi f +ay ner +auto sports +ase d +as ka +amazon basics +ag ir +aa ve +, ..." +ðŁĵĦ : +ðŁijį ðŁijĬ +éĻ ¢ +zoro astrian +yofthe year +writer sofinstagram +wick enburg +whittle sea +whites boro +vide oc +un ction +umass boston +tho de +thibo deaux +ther ick +team trudeau +stol ler +spec tra +rush en +rose s +ram la +race goers +pre cursors +pra ful +poke stop +po vo +park hotel +ontariop cparty +oj ha +o ca +nou is +new al +neg ated +n ase +my heritage +mour ner +mit is +met so +mb achelet +mayor an +manning tree +mand ing +m hall +lu ong +love jo +li therland +le vent +ku hoops +koop man +keepit real +kap iolani +jjig ae +in costabrava +ic ss +i vin +hun n +hu tter +ho kies +halvor sen +for victori +fas ching +f itur +f ady +er gon +desic cated +cow ls +cho lec +cho ix +cash more +cal om +bush y +burn twood +buch man +bb fc +arn ason +aph ili +ai man +a stri +ðŁijıðŁı»ðŁijıðŁı» ðŁijıðŁı»ðŁijıðŁı» +ðŁİģ ðŁİĪ +ðŁİ ĭ +оÑĢÑ Ĥ +za her +z wick +yar de +wi gley +w ys +unit eds +un satisfying +un pasteurized +tz aneen +tur bom +tre en +team mtkglobal +ta fo +swan igan +survivor man +surfl iner +storme mma +stin nett +steve kingia +shri veled +sea bee +sak halin +pizz eri +per gam +orland i +o tras +noti f +new business +mis representing +matur in +malag acf +long neck +lo thes +lo tf +live atthe +lat ed +lace wing +krysten ritter +keyshi acole +kany on +k ys +k pu +inve rell +inde terminate +in anda +ill iterates +il icon +iam a +hydr ator +hu uuu +house uk +hm recipes +gra zes +gior gos +frame set +faren thold +er roll +dukin field +du lin +dor dt +door knocking +do gie +dj d +destruc ting +dapper laughs +crow ed +collabor atory +coel ac +car lino +bun mi +back handed +as see +arty originals +afell ay +ab bys +a yoo +ðŁį Ī +vin it +ur an +tu big +tri stes +timmerman seu +terrori stic +tant allon +ta ward +sub h +steph y +skull cap +simon shield +simonshield cars +scho o +saw bridge +ra jani +patmccr orync +par cells +p band +oun ie +oc md +non nie +movethe sticks +me rel +mar galla +madd alena +lef tism +kr be +kassi an +jugg lers +jam ai +itv lorraine +itson us +inhib itions +impe des +hou lton +h te +gov con +gondo lier +galo shes +g pio +fox hall +followyour heart +florian sem +edinburgh fringe +dre ssy +do et +digital selling +d boss +crazy sexy +cou lom +cost asunglasses +competit on +co zens +chri slo +char o +ce cafa +brisk ly +bol aji +bill ymiller +be edle +bad things +bab yy +ay se +ave ley +aun or +au pdate +artofli ving +anno ck +amey er +ðŁİ¸ ðŁİ¤ +ðŁ¥ ħ +âģ ¿ +wysi wyg +w dg +veri dge +vai zey +transni stria +tom mison +toi les +team db +tang an +tam an +swim mingly +super heroine +sumr all +stein ke +stall ard +stag nate +spind les +south african +sig ils +senti do +san sone +sacri lege +ry mer +ru sses +republic anism +repe als +re mise +ra uma +portre ath +phi leas +paras ail +oney oung +occu pied +nur haliza +noisi est +ni antic +music videos +mr tt +mob psycho +mo phie +michael hyatt +med ju +materi el +mat tar +mar shab +m go +loth ario +lock sley +lau ter +koz lowski +ilustr ación +hal lion +ha ddin +ha bs +green shank +golmaal again +gav riel +g auth +ft as +foot lights +fictionalcharacter si +es guer +e zy +e spectacular +directedby women +dervi shes +del lave +davids ons +d tid +congress h +conceptu alizing +cape epic +cam es +bung ling +bre arley +bhan ush +bengal fc +bbcgoodfood show +aussie wine +as pera +arti um +arro gate +arac eli +antonio selas +anthrac nose +ameri prise +am man +ðŁĮº ðŁĮº +æ² ¢ +âļ« âļª +zephyr teachout +youth fulness +yeh hai +wit trock +wal dau +verbi age +valenci ennes +us nist +usnist gov +un ambiguous +ty red +truetothe blue +tom coronel +thimpact music +tek tronix +teab ags +tanu garn +st apes +spo cus +shor r +sand box +salesforce tour +saint louis +sa hra +pic ken +personali sed +peri gee +par ky +par cc +pand al +onna ire +national triviaday +nas co +n pd +mp ca +mis interpretation +marinas diamonds +mar am +mac callum +lyric al +loo tera +live stock +lev elland +ko zy +klin ik +kat onah +karanvir bohra +k nave +k mw +jewelry designer +j ør +it ami +ira i +inde p +ill in +id omeni +i eper +hong kon +ho chul +haw ay +happis burgh +hal per +gu ji +grobb elaar +gar gamel +gan sett +fi bber +festival season +fem min +fate grandorder +fac ademy +expos itor +ela unch +dun don +discre tely +de funding +de bono +dayof service +cringe y +com enius +chun king +chou pette +chance hmiller +ca key +bottle brush +bhu mika +bewilder ment +better world +ball park +bad tam +avo d +al cin +ai mi +abud get +/ \ +æķ ı +zuc carello +whack y +westy orkspolice +westmid sfire +wai spr +uje res +track ball +tinyrebel brewco +tej pal +sw coastpath +su ku +su god +stress free +sterili ze +stake out +sm show +sini esta +sil ove +shel duck +sensu ally +scott zolak +save therhino +sam vad +ry ant +riz ky +ri sed +recompen se +ra kers +r ml +pitney bowes +pic ally +pet smart +peri stal +penrhy n +pap anasam +ol fi +ok ello +nor den +nl h +net worker +nano wires +mön chengladbach +muck le +mu sser +motorcycle live +micro structure +me soli +mclar ty +lolly daskal +len sky +leather wood +la sing +l ancy +klu ge +khar ghar +ke zar +kar yotes +kal er +jag at +j twc +in vision +hydro chlor +hockenheim ring +health sci +gu anab +green wave +gods girl +go argos +fur ugby +execution ers +est v +errat ically +eric stonestreet +engv all +egg leton +du um +dramar ama +dor gan +dok tor +de mexit +de mas +cu mmer +cryo em +cric kla +concier ge +char dt +cafe oto +c illian +buy local +bro glio +bol z +barnes ville +au tol +athle tes +asimbaj waispr +arch ways +anglic an +ane ja +aedilhai mushkil +advoc acy +adamm grant +ðŁĶ¥ , +zak is +young bloods +yam ini +work man +war danadi +wardanadi adwala +up mc +tric are +thanksgiving day +ta ches +sydne yleroux +sw c +spin ner +sirot kin +silver sea +silver fox +siberian husky +si fy +sh ac +sali endo +roman os +roble do +ro hani +rival ing +reu ther +re uss +re make +ravin dran +quil len +perpetu ation +par apan +ordin ate +or ski +ol um +ol ave +ohl draft +ogc nice +o hy +non conference +no zze +ne wald +my nydd +mer folk +man tell +main ec +mae us +ma sel +love cork +likk le +lich tenberg +leeds beckett +lee za +leaf less +le det +kott ke +kno wer +ki wa +ker im +kel se +k flay +ju uu +ju tanugarn +is me +ink blot +hoag land +he an +grou pm +gol fs +fur n +franks redhot +fj ell +fabul ousness +ever long +dv n +dra enor +dopp ler +distur bia +dirt track +demp sey +dam ini +cracker barrel +con ation +cinnam inson +chin nor +cau d +cam pos +bro ga +bor re +bh b +as under +am ine +alab amians +ak wai +afton bladet +ad alovel +acci ona +ðŁĺĩ âĿ¤ï¸ı +ðŁķ¯ ï¸ı +ðŁįº ðŁį» +zab bix +yot suba +x xix +wego hard +vivek dahiya +uver world +tro ad +timp ano +tel is +sper oni +son uk +sm f +sig gi +shrie ked +school supplies +scho on +san é +s dr +rust lang +roden bach +rock art +ro sac +riseof rest +realdj premier +ra thyatra +qui ds +porthcur no +polk adots +per io +penguin books +pe pino +pdx now +ok ays +no em +mosk va +mis ms +mcgill is +matsu oka +man zoor +lu tter +lis ner +laser scanning +la chaise +ky tv +kim s +kendra wilkinson +jessi ej +hygro meter +husse ini +hu ub +hockey tv +h ba +gui der +gb d +gagli ardi +fio rella +fel tz +equal it +eco schools +eaton ville +dou ts +dou mbia +den holm +coer ce +cli mat +cardio id +campu slife +bot an +am oureux +ðŁĶ¥ ðŁijĮ +ðŁĩ®ðŁĩ¹ ðŁĩ®ðŁĩ¹ +æĸ°å® ¿ +yam al +y strad +y pr +writ enow +wish meluck +walter scott +wak il +uf bugs +tin ney +the box +thanks dad +th ung +tat ort +ta issa +swi vels +stormb orn +sk ai +sig int +sad diq +rwen zori +rudol f +reck itt +ravens bourne +profit ably +pro mes +po go +placido domingo +pit ons +pim p +pic cin +peter tatchell +pedre ra +pay rolls +objec tifying +nj dotcom +ne j +mol ca +mini mizer +mc nicol +mc are +mat they +mahi eu +ma isons +lyn nette +lo ssing +lat form +la khan +kr ama +kent ico +kahn awake +k lax +jit ney +jazz wise +is king +inver leith +inst ad +ib ile +i ok +hy ar +horse less +home boys +gun nell +greatest showman +gla zes +fri sson +flos stradamus +flexible working +far in +falsi fying +escape es +emili o +dete ctions +de val +de pa +dand ad +curi el +cp sa +cold spring +cine main +ci dium +chanel westcoast +buzzy buzz +blat che +bind weed +bart leby +balochi stan +aw sreinvent +aus grandprix +alpha go +all ora +aaa ahhhh +: ? +!!! âĿ¤ï¸ı +ðŁıĥ ðŁı½ +룬ë¸Ķ리 ì¦Ī +âĻ¡ . +âĭ Ĩ +âĨ Ķ +Ùħ ÙĬ +zy lof +zor don +zal de +y itz +wy ne +wild bill +we ke +vali m +utter ances +un willingly +un profitable +un anticipated +ti mba +tdw sport +tail spin +t dci +syste mo +sway ne +steven sville +steven ash +stanc enation +sony prousa +sky digg +sem pre +se q +sawbridge worth +sau con +sar af +san die +sahar a +rown trees +ro tolo +ran chos +r row +pony hour +nol ans +mu ji +mm rda +mb stadium +ma illard +lovetrump shate +lic or +laugha bly +kam uy +ji j +j any +inconveni enced +hud ds +hippo cratic +has sel +hanse atic +ha aha +h mrc +h mb +gyne cologic +glo ating +gen set +gel sen +gangre ne +fr ons +flu gel +fictionalcharactersi wanttomarry +fair uz +excit er +ev elled +eni ke +ele fante +ee zy +eddy stone +eco tec +dum mett +dm register +direc tionally +del orme +deer hound +dang dut +cur se +country girl +cleve rer +church ville +chand ani +chael incl +cellulo sic +cast aldi +by one +but lin +bu ehrle +bli s +beyon ce +best wishes +ber rigan +as vit +annul ment +ama q +alli seeis +aar gh +âļ½ï¸ı ðŁĶ¥ +âĺ ŀ +á Ķ +yw ca +wood worm +wis la +what su +west lock +w use +un crowned +u os +twy nd +tran ent +tiffany haddish +the knot +thankyouforyour service +te ssy +swit che +summ erishere +sp ir +so hc +snetterton msv +sli v +shre m +serv icec +seraph ina +sathy ajyothi +s llp +ri fling +ram eau +porth os +porsche forsale +por thour +pet plan +pel uso +patter son +p had +ox uni +otran to +o delirious +nor win +nitourist board +mizu ho +mishand led +ma sini +lum by +li os +le vein +kri palu +klo sterman +kit agawa +ipp on +infl ates +ili o +hydro x +hr sb +ho vel +ho skin +hat field +hann ya +ha el +h dt +grant making +gra hams +gol in +gh and +gel fand +gd pr +fri g +f anti +ex erts +eni sta +el ara +duncan trussell +down land +dellave dova +dac ey +d ctf +commun alism +cheftom kerridge +cerv elli +burnthe stage +brink mann +bok hari +blumen auer +bis aya +berg tv +bekind to +bau dr +australopi thecus +annon dale +ak anishi +ai ac +adu cati +ðŁIJ ĥ +ðŁı³ï¸ıâĢįðŁĮĪ ðŁı³ï¸ıâĢįðŁĮĪ +âľ « +ä s +za veri +yogi adityanath +vy dra +vote demilovato +vas s +v itt +tra ver +threep ence +thiru vizha +tail bone +sway amb +suz ann +suicide awareness +sugar bowl +st illed +soldier field +smu gly +sb v +rou se +re conditioning +q cs +prin ze +pmk vy +pi planter +panam acanal +over loads +or bi +olom bia +ob x +o gn +noble st +nam san +na fees +mu mab +mkdon sfc +min ky +mi ano +men or +meet our +may hall +man tv +ma vicpro +lulu hru +l ation +kwesi arthur +kate spade +kat rant +kalki kanmani +kal barri +jocel yne +jesuschri st +it ravel +information technology +indian diplomacy +hyp notic +ho tho +hinch liffe +hilton head +hell hound +ha wards +grau pel +gonebut notforgotten +ging erly +gin kel +fox ford +external ities +er rico +ee v +di mmu +de population +de muth +darkk nightrises +d dot +cl ane +char nley +cal af +brut alized +brun ches +bram alea +bey routh +are ata +ap hex +anti e +am logic +ale sia +al quds +air tricity +:' -) +( ??) +!!! , +ðŁĶĬ ðŁİ¶ +ðŁĵ ĭ +âĪ ŀ +° ðĿĺ +yal inetwork +womenin engineering +wilkin sburg +wail uku +visibil ityday +vi alli +tr onic +total biscuit +the evil +ter ang +suff ices +spencer port +san del +sag t +robo twars +ro guer +ransom riggs +py are +pr week +plu cks +pin cho +phili stines +organiz ed +ol z +ny v +not our +nikki reed +news media +n kun +mish ra +mb and +masochi stic +mar ang +ma kel +lo kay +lil bibby +lev antine +lc dr +la ettner +kere mbur +kaw aguchi +kath iel +ka sten +k rome +jay rock +inter provincial +incub ated +hundertw asser +henry viii +he eds +gre a +gov tnz +gill ick +gener alizations +fro bisher +fow ls +foo m +fnaf hs +fly ball +fir po +explore the +ev aaa +end child +dun combe +de mption +day dream +cor vina +conve yer +confu ci +colo m +clums iness +calci fication +bra sher +bbc cambs +assor tments +asgar dian +ard agh +anag alizia +ali u +af fix +adri ane +a heim +ðŁĺįðŁĺį @ +ðŁİĦðŁİħ ðŁİģ +ðŁįĢ ðŁĴļ +âĨ ©ï¸ı +worldsoil day +whodoyou collect +wasteiton me +vol le +vel lam +v hong +v adap +ur inated +upper class +under hand +town scape +tam bur +tach ometer +t do +strat orob +steve burton +snoo t +smo ker +small streamers +sla u +siva thewanted +shan te +seok min +season able +sahuar ita +sage francis +sa ham +road burn +ram bert +rally racc +ra wat +princess leia +pla stique +pen iche +pe therton +pang lao +panam games +orig ami +or questa +na shies +mu table +mon cur +mess o +men sur +massi modu +marcel o +m travis +lovemy dog +lloyd banks +lie ben +leix lip +ky dd +jab il +is ao +ic aria +hi ero +hawks worth +grave sham +gen ation +gard eng +gar owe +fox ct +flor id +ess on +eric whitacre +ent radas +elo ka +e owyn +e akin +du dgeon +du ce +demo scene +day glo +dant dm +dani elo +dab arkads +cuti ee +country club +cele k +car onde +cal allen +buy ers +buck hurst +bor ou +boett cher +batt enburg +bas singer +arnel pineda +ar nou +antim al +anne ka +alter a +alp bach +a ethel +ðŁĴĽðŁĴĽ ðŁĴĽðŁĴĽ +ðŁĴ ł +ðŁİĵ ðŁİĵ +ìĬ¹ ìľ¤ +èµ IJ +æĽ ľ +âĺĿ ðŁı¾ +ø r +ê me +yon du +wyn on +wo ssy +wel ts +vo e +torch bearers +tit lei +the cutch +tam ines +taf sir +stick iness +stal liance +spi erre +sophistic ate +sin less +shar af +se acade +schaff hausen +roller coasters +rock slide +rapo port +prun er +poly gamist +polari zer +pol avaram +pm dd +plant science +par to +onceupon atimein +nt southwest +neme chek +ne shoba +nay la +muth arika +mercu tio +me ares +ma homet +m buy +line tte +linda ikeji +lam oureux +la yo +kum ho +kla vier +king field +kap lan +kab aka +hypnoti sm +hin king +her mann +h wb +gre ymouth +four play +ewn traffic +esp ino +epiphan ies +ela sti +del orenzo +d bf +cull ompton +cu bbon +cs ny +cra dled +cost lier +confederate flag +con formation +ci legon +chapel town +chan u +capp ie +biomed ical +bal want +b sas +as cio +argu mentation +angliar uskin +ameracad peds +acce sses +abhin aya +ab ts +ðŁĺĤ ðŁĶ« +ðŁĸ¤ ⾨ +ðŁĴ» ðŁĵ± +ãģŃ ãģĵ +what matters +war of +wan jiru +vero beach +ul bricht +tyro lean +tromb ones +tre vis +the shark +taga q +suf cofficial +stom o +starbuck scanada +ssin z +snowmag gedon +ski pp +si mard +si at +ser j +senator menendez +se il +re write +re tested +ram nath +ra sco +pre destination +point es +pe azer +online safety +o chieng +nor mand +ngu gi +naz rul +national hatday +nation all +mr bo +moon less +min niem +matt lanter +mag ner +mac omber +kuri su +ku mail +kor k +ki y +jones borough +jo ep +jack savoretti +ish u +in lets +illeg ality +hb g +gar brandt +fra ying +fasci as +e gan +de etz +contr ition +cny central +clu bb +chero kee +blues rock +bj ørn +be eville +barn field +bar ako +astor ino +ar nau +ap lay +ant ó +ant é +am ck +world notobaccoday +wi f +way uu +vir i +vi brio +un happily +tiger lily +thekk ady +the jason +tam pere +sk omer +share pict +shanemc mahon +second city +scru ples +s fair +ryu kyu +rodan the +repost by +regin aking +ram kapoor +pin oftheday +phyl lo +pe afowl +op hen +on co +ol ka +ober weis +museo ideale +mon ad +mile stone +mi kay +mel chor +matt and +man ac +mal lup +m amp +lyle lovett +leea renberg +lea den +lamba alka +la hr +kun is +in famously +hen ner +har ve +har mar +go is +ff xv +evol ver +ess ilor +dz bb +dri ppings +drew barrymore +dog pound +dis avo +dg ca +del piero +deb ary +de militarized +de mean +cor de +confu singly +comp ston +cien fuegos +chi b +cc me +carri ere +car tm +can thelp +brox towe +boss y +bone tti +ble ier +biblio thek +bhar uch +ber kut +bbcgw live +au gi +at ag +anim atic +and oni +amil ano +amar ca +a ani +ðŁĴľ ðŁİī +ëĬ Ķ +è ® +z by +yas mina +wri ggling +visit abdn +up scaling +under takers +umay yad +ukgif tam +theplayer schamp +tat ry +taq wa +tales fromthe +super grass +sunny bank +stone field +ston ef +stey ning +ste ads +skincare tips +sho v +she hr +sg pc +ru mps +rizz uto +pin tor +pick away +phy to +petit e +pax os +over tone +op atija +o gwen +nee wer +necess itates +multivit amins +mu sil +mu ip +mor ros +mo ki +mix ta +min def +mee sha +marin us +ma die +lex perience +legisl ating +king i +kenyan traffic +ka idan +k harbanda +jun krat +jon m +jb pritzker +iron work +insuper able +infini x +i jaz +hyperhidro sis +head teachers +han de +ham ster +had win +h ace +go yer +girls bb +ft u +fitz geralds +fer ring +fc united +ev d +esc ol +enligh tens +enginak yurek +ele vens +edit ore +dolore shu +deaf heaven +dark sun +cycla dic +cubic les +cu buff +cre morne +commen taire +colum b +bore as +boom stick +bluet ooth +ballinas loe +bal gow +b inga +athe a +aste iner +arsenio hall +argentin agp +aren delle +al friston +ag aves +afol ayan +a oy +a jen +) ', +ó¾ ĵ +é § +ãĤ¹ãĥĹ ãĥ© +âĺº ðŁĺĬ +Ø§Ø ´ +zil lah +za hoor +y aaaaa +world humanitarianday +w up +vi le +vampi ric +tune ful +tshirt design +track ing +ton bridge +tish omingo +the coral +th axton +terribly tinytales +teacher appreciationday +swa inson +sound set +smar my +short land +san jaya +s mid +ryan holiday +ru ddington +rspb southwest +rivers dale +respec ter +re beli +rahul dravid +posit ron +pim pin +pas sey +oromo protests +organ elles +oc ar +no ahs +ne ophyte +my idol +mur dere +mor ri +menstru ating +ma fc +lucifer onnetflix +lili an +li sted +laure en +laten igh +labour party +la plante +ko zel +kitty hawk +kar hu +ju lee +jim myeat +jeep ster +jaye hanash +ir reversi +inst yle +idoli zing +ic pd +i bro +ho sni +hlf supported +her rig +he bs +galler ist +frasc ati +exten sible +exeter college +even ko +es b +ear tists +disarm hate +din ard +de criminalisation +dc pl +cynthi anixon +cont actus +colom bians +co an +chi haya +cc cs +bra ganza +bol ter +be avoter +bank stadium +ast ell +asperg illus +asi apac +ann ée +ann aa +am bal +alonce sto +all ana +aege anairlines +abet z +ðŁļ µ +ðŁĺİ ðŁĴ¯ +ر Ø© +ÑĤ а +ze hn +ys rc +whats new +was p +vi agogo +vegas news +vassil is +v dj +ush ima +un traditional +thursday night +talkin gheads +ta was +sw ch +sw abhi +supply chain +str ina +she amo +scott borchetta +sche ch +sc ouk +ron funches +redon do +qu aide +prolifer ate +play ingnow +pe ered +ol av +ny anga +np tech +national pieday +my view +miss world +medi o +mcle od +mc gil +mac ia +ma vs +ly t +lu anne +kon nen +ka sese +juan ma +ju alan +ing ate +ildef onso +hydroly zed +hu ong +himer ose +hill ard +hege monic +gri der +gra al +gi blin +game boy +flu g +fan zines +fall is +every woman +ejec tions +early childhood +di mera +dar ting +dar ted +dar ra +comm er +city traffic +cb bus +burde kin +bro ths +brett dalton +boiler ball +bedroom tax +atsu himerose +astro labe +alfred sson +ah y +ae ats +ac lassic +" [@ +ðŁĶª ðŁĶªðŁĶª +ðŁIJ¶ ðŁĴĻ +ç Ģ +ãĤ¿ ãĥ¼ +âĮļ ï¸ı: +z addy +y stems +wa inf +vesti do +vacation mode +v pro +ul le +thur rock +super nintendo +su sp +stjepan hauser +stal act +sou tache +simi en +search engine +sc ill +sas aeng +sacred geometry +rome omiller +rn ts +refr acted +real michael +rad cliff +pri es +pre uss +porch light +official sps +o logic +nascar hometrack +nap alm +nag pal +movi mento +mon chi +missing merlin +mathe ba +ma pper +m wh +lt d +lilloo et +ky p +kodan sha +kir cher +jd illa +iv ani +indi visi +in laws +in chic +ik r +i vie +houn sou +hay ton +haku ba +gu ti +ger gen +full stack +for sey +fabulous finn +degra ff +dat aran +dam ocles +da hal +da chi +cro is +clou se +che ers +centr ica +catal ana +bur chell +bry ne +blox wich +blood worth +black business +bir tday +bi beau +andre siniesta +anaam doon +alv ina +almosthe aven +all ach +al jaz +afl vic +ad dam +achi efs +ðŁijŁ ðŁijŁ +Ø§Ø ¬ +za haha +x aver +with kids +wh yyyyy +wen dig +wee ze +w taf +vion net +ver celli +van af +tvin colour +tor ist +the resi +ten leytown +tal bum +sweat box +sty li +stre sa +spoon bills +sorry im +son yo +smur fit +sin c +sidd all +si phoned +si fton +seismo logist +ro see +res wales +rally deportugal +rainbow six +que k +port coquitlam +philosop hie +philly firedept +phil brook +oto ya +nidh hi +nab u +moo cher +mister wives +mi os +merrell twins +mccas lin +mal feasance +mac naughton +ma saru +m wr +m tam +live journal +le chu +kib worth +k bw +jason plato +j rothen +inzam am +hover boards +honey combs +homin in +her ro +hard hat +ha hohe +gou die +football remembers +evi gan +du ba +dol ing +dir hams +dinner date +cu ala +crag side +chesapeake shores +cast o +car lucci +ca rella +businessc lass +bur kes +brew day +bol tz +bb bb +ball ater +babad ook +alex isonfire +adjun ct +aber fan +/ -. +ðŁĺī ðŁĺİ +ðŁij¸ ðŁı» +ìľ¤ 기 +à¸ģภ£ +yo ichi +y ı +whitecol lar +we dont +uof alabama +unner ved +twelf thnight +tu gend +tr ca +tom son +thegoogle images +theat tic +the sse +t ms +sven ska +sug ita +sm tr +sil urian +shol me +sel insgrove +san parks +s como +ru pi +rock sound +pro bus +po stand +plastic pollutes +paediatric ian +nbc timeless +mun dry +mi stle +menac ingly +mccu bbin +mather an +ma sum +lev ins +l hl +korean war +kai ji +ka whi +jb fa +j ck +intercess ory +il wu +i hp +hine sville +high on +hene cia +happ old +hang time +hand l +hammer films +halloween party +hall sville +grim ms +go ags +glen ny +g ner +g ads +fire pro +fal les +fal lah +f nv +expan ses +ex pel +environmental justice +en zy +electro physiology +ds meu +dread naught +dor in +dian ak +cu hk +cocom artin +cis me +ce g +carol ines +car star +boney kapoor +black white +bjarke ingels +bar ringer +bar log +bad ung +adre ssing +ac snano +ac crue +!! ðŁĴķ +ðŁĵ ĺ +youlike carsuk +yanke ec +won tons +wom bs +ving o +victori ously +vec chi +under perform +un rolled +uk ur +tv n +train ingday +tow v +th ooo +tb ell +t ounge +strat asys +stein ert +sr ing +sobr ino +sli iga +siss inghurst +si pa +senator reid +sen robportman +selve dge +sel mon +sel inger +say cheese +sa wat +rekind les +re route +rc de +ra gan +qu n +pro van +pivo t +pedic ures +pancra se +pa kk +out selling +our nation +oregon ians +niam ey +nation hood +n crc +msmar vel +morad abad +ml j +mine ko +mil fs +mesm eric +men chies +me gara +max ing +mat tek +laga res +kyoku shin +kl h +kirstie alley +killing it +kee sh +kat ed +josel yn +itv anglia +ine ed +incenti vise +in den +in cin +i dra +hyper links +hu uu +hash ana +gre nell +gravitational waves +grat ton +gg able +fun dthe +fan stand +exhilar ated +em mar +eli v +elder abuse +el tz +east wick +ear ings +dire kt +dev our +democr acyday +dee puk +cy ano +cu ssler +condition ally +coco oned +club wc +civil airpatrol +car luke +by fleet +bore holes +ber satu +barrel led +bar ths +ba hir +b pd +ashi shians +ar kh +ar iss +apor ter +ao ife +alwaysinour hearts +al ofa +air nz +adeni yi +adap t +. ' +" âĢĶ@ +ðŁıĢ âĿ¤ï¸ı +ðŁĨ Ļ +èµIJ ç¦ı +ãĤ» ãĥ¼ãĥ© +Ï ĩ +year son +y res +women also +wk shp +wi ya +versic olor +ve aux +us fca +twin ny +tour é +tor adora +ton tour +tho spital +surfer girl +steadfast ly +star sof +soil association +shru thi +she epi +schei fele +saddle bags +ronnie wood +re payments +ram ayan +rad wimps +quin tino +pink pop +per missive +p sei +open the +omni verse +oma ine +ohhill no +official nichols +o sher +newyearnew you +national pancakeday +my nt +mun chau +mirac ast +me zu +me self +mass enet +mar pole +lm h +liber atore +lei fer +l ling +kuns thalle +kon tak +kon nan +kk l +kirk franklin +key board +kc as +kab u +k por +jean michel +it group +is that +indi c +hor lo +hon go +hof man +head butting +h vo +green collar +gravel ly +gin blossoms +ger tz +gar din +flu tters +ff f +eth ridge +entre pen +enf ance +ed sall +ed di +du shy +dri es +dolla z +ded rick +d aka +cre mant +coun ton +cou libaly +clap back +city beat +ca stor +buj old +bli zz +blackandwhite challenge +beach bum +bayone ts +artill ery +ap or +andy mineo +amyra dastur +alber o +alas kans +ad cs +ab io +ðŁĺĦ âĿ¤ï¸ı +ðŁİĦðŁİħ ðŁı¼ +าภ² +worm wednesday +willi mon +whip sn +watermel onday +wat kinson +wal ki +vsc in +ue hara +tu itions +truec ar +traffic butter +to ks +thunder shower +thir lmere +think progress +t dc +stenc iling +sne ddon +sky liner +skol kovo +shu maker +shi rish +sch ic +samuel milby +sacrilege sunday +ridel ots +remington leith +red lining +re tr +rachelriley rr +quetzalco atl +quer ra +pyrr ha +pan abaker +pal utena +pa ku +o wego +nz vind +nytime sarts +no bly +nj pac +nar ada +n can +moor abbin +mon tt +mollu sc +mate ys +mal oo +lym sm +level design +ku tt +ko ori +k pandey +jap on +itsmo hit +it ron +inausp icious +ike ausa +hip flask +hallucin ate +ground cover +ge tor +gar ron +full circle +for food +fel onious +ever ard +eun bi +ensla ving +encan tan +eme k +eli gh +eeee eeeeee +edg cumbe +ed research +ear this +dow deswell +dis qus +delinqu ents +deidre hall +corpu z +cooper man +co ti +chu tiya +chri ster +channel ten +casser ly +card fight +boot legging +bollywood celebs +be ith +bang lore +b wy +aun ite +as om +argu able +ar ny +aqu arian +ao v +an kush +alas vegas +af obe +ë± Ģ +å¼ł èīº +york dukes +wrigh twood +wiel dy +wi fw +west michigan +wam pum +vic oladipo +v nv +type set +tu z +ton ys +threat ened +thou sing +tellem stevedave +t px +swar oop +stown hall +spring s +spo onie +sharps burg +shan ka +san gue +ricor di +referen tial +red bank +re tall +rad han +qu ba +push chair +pre scott +poin sett +parik rama +pang u +p tr +om gosh +night sof +nh v +mu bank +mo ise +mat to +mar tavis +mar quis +lu zia +lo ofa +lin ck +let tie +king sday +killor glin +kh p +ke ad +k int +juliago erges +iv anna +isla scanarias +inge vents +hypo thalamus +hou k +hel mut +gir ll +game iro +fir stamendment +far aj +dutty paul +dp show +char sadda +caronde let +carni vor +cak ery +brook field +bottom ley +border land +bo co +bless ington +black hall +beg bie +be ynon +baudr illard +bat suit +b dw +b con +azeali abanks +ati f +arjun reddy +amar r +all ington +alig ners +ald in +ag adi +aes ar +adam j +ab bess +!! âĻ¥ +ðŁĴķ ðŁIJ¾ +âĶ Ķ +zylof on +wreck it +whir ring +web mail +v ade +tyn tes +tor qu +the change +tel o +te chi +super friends +succes fully +subter fuge +spr inci +sp era +sof antastic +so sn +sho to +seag rave +se say +sardon ic +saeed ghani +sa kae +ro ject +regener on +rad ner +qu ism +put locker +psl grandprix +produc ciones +pemb ury +patho logies +pat te +oscardel arenta +oscar s +orangu tan +ok state +nav yy +multi scale +morbi han +mor di +metal music +mccor kle +maun akea +mar stons +mar quin +lu dus +live tweeting +li di +legi bility +leg old +le jog +l liber +kin zie +khush boo +katrant zou +ju tta +iowa stateu +harvardchan sph +happy sehunday +gra pho +geschich te +gen nar +gat chaman +forz aducati +fo ie +fer hat +ev onik +ecol lec +di sunity +derekand susan +de groot +crystalli ze +cran es +cow spiracy +connach t +cassi o +c sco +bun ts +bri bri +bre aley +borough bridge +back stories +ani emi +age ing +aa si +ðŁĺ¢ # +ðŁİ¤ ðŁİ¸ +íļ Į +x design +winter jam +wen sum +us ages +un bleached +tobin heath +ti amo +thereal elp +supt chat +sun cream +stratot anker +sti en +stee zy +sock en +sm cs +skul lisland +sk od +sh ttp +sciento logist +ruthi eel +rsv p +rosen quist +play sets +pesc adero +pat u +ouel let +oregon mbb +ne ka +nan king +mu f +moto ko +motil al +mobi leye +michael raymusic +man tz +live st +lec tric +lalo alcaraz +la von +keny amoore +kan ada +k ft +jor vik +jimmyeat world +jag ad +itae won +indisci pline +il ves +i mani +huy brechts +ho ki +guille mots +gar way +ga ems +fx bg +freed omo +football family +floriansem le +fine jewelry +es un +doby ns +derby shireccc +darry n +dance able +d con +cri spi +choicemusic group +bra venew +beauty blog +b gn +b elling +az tek +atte sa +asur f +astr antia +appalachi ans +ali k +algé rie +$$ $$$ +ðŁij¨ ðŁı» +ðŁIJ» â¬ĩï¸ı +ì¯Ķ ìľĦ +âĺ® ï¸ı +» »» +y rd +whir li +wel lo +visi thel +v pp +v annes +usag ym +turntab lism +trev ally +tran sect +tipp mann +thom yorke +thisdayin hiphop +t sson +standard kenya +slug go +sign posting +shar yn +shad bolt +sens orial +sal m +russ o +qr code +promis sory +pope mobile +philip sburg +pal ance +pad mas +off it +o yor +nu cky +no suke +new y +nb pa +mun awar +mis senden +mis ma +mi gun +men sa +mc cowan +mar khor +loft in +lets makeit +lalah hathaway +ki et +kad ena +kach ina +jim mys +ir mar +ig arashi +i dr +guit arri +gou cher +go wing +ghou lies +ga er +fun ston +for free +fol ha +flux es +fis d +fab india +enew ton +elli er +deador alive +das nakz +chis ato +chicagos mayor +cal ce +ca wood +c Åĵ +c jd +bury makeup +bbc scotland +ar athon +amazon fresh +am ager +alve church +ak os +ab cland +! ðŁĺĺ +ðŁļ¨ @ +ðŁĺĨ # +ðŁĵ ı +ðŁijĬ ðŁĺİ +ðŁİħ ðŁı½ +âĥ£ ! +whel k +we believein +wal ber +visu alizer +visit malta +vi vic +ver g +ui path +tru ism +tribe town +tr ills +thur ingia +teri polo +sumbastur tt +sto ichi +sim ran +si za +showtime boxing +shooter jennings +shan elo +senior care +san onymous +san field +roi dery +revi ent +rein as +re organised +po que +patag onian +pap iss +neph ritis +mussel white +mo tes +minic ooper +mike s +medi bank +march on +magic of +lode star +lockthe mallup +ke ates +kar aka +just sold +jrothen bergtv +jo yo +j xn +ingu ide +i lean +help me +han in +h pk +gru jic +gro h +gi vel +gali bier +galeng ering +force field +femini sta +farah khan +face idibia +eng adin +emerging technologies +elor za +edinburgh napier +echop ark +dun gar +die ing +d brooks +com esa +codel ottery +ch ini +cen tered +cab re +bull pens +buil dit +bucky brooks +bre genz +blom qvist +bk al +best deal +bes sey +ber bere +b hr +as cos +argent inians +aman pour +am phor +afi fest +ade o +abren ica +a itu +.... ðŁĺĤ +. ðŁĻĦ +*-- -* +å½ ¡ +âĿ¤ï¸ıðŁĩºðŁĩ¸ âĿ¤ï¸ıðŁĩºðŁĩ¸ +๠Ī +É Ļ +ü r +z ii +yay asan +y ars +wu hu +with in +wick liffe +vian ney +ven is +van ews +tun nicliffe +to one +thing ie +the print +tech e +tbird nation +tan ita +tak ada +ta relli +still here +st pattysday +st enger +squir ted +spaghett ini +sin u +si dency +seraph ine +sediti ous +re positioned +pigg ery +pie week +pear tree +palla vi +pa ppi +pa kt +outlander home +no bilis +niko lic +nca acws +nat sci +nat reswales +n brook +msd strong +mis su +mcm ldn +love greatbritain +logo tv +lc ds +laver gne +kee sian +jeon ju +inst gram +in on +in famous +hipho pdx +ge ol +gastr itis +funn ily +function alized +fresh fields +fitz hugh +fi amma +fa shanu +f pj +en no +empy rean +dor ne +doo k +dew points +def qon +de yn +de ery +de bo +dau da +dasty ari +crom ford +contamin ant +compag nia +colec o +code pen +che twynd +cend r +catch ments +car touche +br one +andre wh +alber tab +al ders +agen et +aber tay +ë´ Ħ +天å®ĺ èµIJç¦ı +⤠µ +yuri ko +wit ty +whitte more +vo dou +veikkau sliiga +vangogh museum +unquen chable +under par +tw yman +tu er +the sound +tan noy +ta ho +super fortress +sun shades +sun o +stram rahim +stal es +spoke speople +sj suryah +sil van +sho ckey +sh na +sarah colonna +s var +ry kiel +ru tting +ric hi +ran kins +ra ub +premiere pro +pr sd +poo fy +phone pe +pau pack +p nin +ophthalmo logists +om is +olym pos +odi on +o rest +men o +md gonzales +mcge ary +mc gregor +mar itz +mad he +lu mmi +live au +list less +lake port +la porta +kono suba +kaw ashima +jeanni emai +jai mie +jaf ari +iron sides +insec tweek +incis ors +in blue +ilit ary +il ong +henry ville +hen lo +heatwave uk +hamble don +ha ymon +guitar hero +g suite +fv su +fi an +fateh pur +f tball +do bi +dl su +div ac +cor lett +com eu +cold weather +chicag omarathon +cap saic +buri ed +bur sitis +bur chard +bur cham +bell ary +be here +bbc wales +aristi de +am newsers +al rosa +adop table +abcland line +ðŁĩ«ðŁĩ· ðŁĩ«ðŁĩ· +ãĥ§ ãĥ³ +âľ ĸ +âĺ ¼ +zen tai +what chamac +wb go +way haught +vri j +vo le +vin y +villa iness +vaqu ero +vag as +ur bina +up ington +tri que +tri alists +thir i +the metal +the coop +te bay +tan abe +systems thinking +summer style +sub plot +sto koe +stay blessed +so jin +side burn +shor ror +shar maa +sen ki +sen at +sat b +rouss anne +ron kon +red star +re classified +rati fying +q nb +plan o +paulo avelino +park son +over throwing +osi ris +os ric +organ i +ocon to +o stuni +ny chash +nitt any +ni obi +mal eny +littlerock steam +little caesars +lau x +land slip +l ale +ky sen +kus adasi +kler ks +kasab ian +k adapa +j dr +irving plaza +indian oil +immune system +i kin +hu ic +homo genous +hare krishna +hae matology +gyneco logical +gou veia +glen ridding +fun kof +fl anged +en unci +dragon ette +don lon +do ble +dalla glio +dahl berg +dab aby +classic porscheforsale +celi o +carter ville +caroline flack +canu so +bun del +botan ically +bon de +bio div +barry bados +bam baat +bac ary +az le +at tested +assimil ating +ar dens +ap v +andy roddick +an music +alu cha +alph en +íĪ¬ë ª¨ +zur g +zindagi kime +ye as +wis k +w lic +vers ilia +under aged +tek la +team tigershroff +ster ner +stalact ite +sql sat +sooth sayer +slot car +simpson whnt +si mal +shil in +ru ban +rie mann +ridec annondale +rep irate +rae gan +pu tu +pos thuman +par u +paper board +ou thern +organic farming +mut ating +moment smatter +me xi +master minding +lind blom +likeli est +li mani +lax mi +la sell +ku hlman +kam az +it ars +initi o +hyper thyroidism +homosapi en +her bin +har te +gu mps +gor j +gh ela +fr q +facul dade +f icio +ent weets +e idol +din sdale +demetri ou +comic boo +colom bian +cityofla svegas +cir l +chee zit +cha an +ch ans +car tas +c elive +by x +buy do +bor owski +book covers +bog ner +blood mobile +black families +bel lowing +bay ers +ati fas +apare ce +ang ell +ane ed +ali zafar +aga a +æ İ +wj la +un tung +u wanews +tl r +tinthe park +teign bridge +ted ford +swad dle +sun gha +son risa +slay ings +sky wards +singh ania +sid deley +shir lington +sheffiel dis +sh ram +sell ersville +saw chuk +samajwadi party +saf avie +sabah info +rous er +richar dy +read just +r ph +r ck +que strian +pursuit of +pro ad +porta ferry +plu ma +pin pointed +pile driver +per ales +pc x +p ta +nor land +nit itaylor +mu hyi +mtlg azette +miscre ant +min ardi +michael dell +mcg avo +maurit shuis +maser u +man zarek +m eller +lin field +lgbthi storymonth +laur ance +lar ned +la pin +l we +kun tz +kon tra +kerembur sin +ker nan +kat alog +kali hi +k west +jessic acaban +j wp +ite aser +into cable +imacele brit +iklan ok +ident ically +i av +hic ago +hib berd +hc n +harbour ing +guj ral +gold berger +glori ou +giant srl +geo stationary +freer oll +fr nsw +fl ac +face tious +ear ned +e die +door n +dick en +di ag +dan ziger +da an +cull man +cho pp +che tri +cel luc +ce es +bobs burger +bike tour +beep ing +anis achibi +anim pact +am ical +... !" +íĪ¬ëª¨ ë¡ľìļ° +à± Ĩ +ál varo +wwe payback +wo wie +wo ts +wil lets +west en +vbm plong +uth appa +un fla +u chs +tv bs +tuesday bookblog +tre mayne +tranquili zer +thehorror master +the well +the meat +tan ay +sy ko +storm chaser +stock yard +squab bles +sn az +shi powners +shere sy +seab ream +score keeper +sci ac +sat is +sar y +sal lee +ry d +ro ks +reading forpleasure +re su +punchestown race +patek philippe +open source +oc kies +o berg +neph pearls +n alban +monti el +min chin +mill vale +mid dy +mel itta +mani a +m hairi +lor ch +ll amas +lip sky +lett res +la bru +kwang soo +khabar ovsk +jual beli +je ws +jay demarcus +javit scenter +is dead +home opener +hemato poietic +hat un +ham rick +gelsen kirchen +gam bon +gal entine +fri ess +follow westwood +flu ffiest +feasti val +e itan +dur gap +dub w +du plexes +d tx +cow rie +clap ton +choose to +charli erose +ce ili +car law +bun heads +brisbane times +bo tting +beau ly +bat arang +barric ading +ballin am +ay ler +arg erich +an ri +ak ler +ae o +acu bs +abor ting +:) ... +. ðŁĻĤ +# ðŁĵ· +ãĥ ¤ +ب ر +zag mbb +y anda +wy r +writing commmunity +water land +ver ige +ven za +un acceptably +ul nar +trick les +ten er +tc palm +tat us +sy metra +spandau ballet +soun dar +soul pepper +som ely +sk ymall +shu toff +san ath +sa ap +ro zz +ra vitz +politic slive +plac ental +pe tyr +pat ry +of ili +nrl footyshow +nether landish +nak ul +my ler +mor tes +mis land +millions missing +mil som +mc mexpo +mass statepolice +m hat +lovefor dorset +lign ano +lifel ines +leg an +koh ler +kali bo +k gun +john wayne +iri de +icec reams +i key +hy den +hol le +he ino +he dra +hat ty +grove port +gley ber +gl td +galax ia +found ing +em po +elong ation +ee ts +dre wes +dis orientation +dh ss +defendthe den +cull ens +cou per +con die +commun es +co eur +clo tilde +cheni er +ch alian +cer rit +celesti al +cast ells +capital markets +bus man +bu su +bog ans +bernou lli +bbc stargazing +bad deck +ķ × +ðŁĻı ðŁĴĻ +ðŁĶ § +ìķ ¤ +è vre +® - +wn r +w lu +voting matters +vandy baseball +vand u +u ssie +tuft suniversity +try stan +trumpe ting +thorn burg +tgom agazine +teesside uni +te dat +tan ana +table scape +tab o +sweden borg +sp kr +shrimp ers +san miguel +s draft +ri mer +regr anned +red back +reb ello +reading rocks +re bo +pub con +pri sca +pel zer +pasteuri sed +p vb +over lander +ori entex +one word +ole man +ofthe cup +ny berg +neck band +nat alis +monty don +mis spell +mis construed +mic hiru +mac phee +lo vas +liam neeson +kib beh +ke mi +kay am +kari bu +jor dann +ig y +ibm wow +hyper venom +housing forall +hor tus +hope dale +hit ach +hin ault +h mx +gar amond +for tus +food wine +evan rachelwood +en ag +elie saab +du td +del ain +dau gaard +d sport +ctil burymakeup +cit v +cho ppin +carib be +carab ini +car avel +cant ley +cam eleon +bundo ora +bun ky +boo table +beat ric +ba ren +ba ini +avon books +antipo dean +am ox +agü ero +ac cp +abc brisbane +ðŁİĦðŁİħ ðŁı» +ðŁį ± +çµµæıı ãģį +ı k +ysp sculpture +y leo +ver ul +ve eran +tz ar +tro pal +th illary +sv c +surfl ine +sura karta +supp e +soft wa +snape maltings +show ery +sen su +score stream +reignit edtrilogy +red horse +rafa ela +ra gee +profession nel +pic poet +pee zy +pe thealth +pe stered +p tas +one coin +ok y +o by +neuro surgeons +ne tease +nasty gal +my e +move to +mon na +min ki +melissag orga +mazz one +laser cutting +krish nam +jo ed +j its +j ito +i ho +hun ton +hene ghan +green machine +greek food +girl son +garbi muguruza +funny videos +fivb worldleague +feijo ada +dish evelled +depor tiva +chester bennington +capitol theatre +cantanker ous +cand ra +can epa +buil din +bla w +bhanush ali +bat avi +bad land +art craft +aren s +are x +arag ongp +anu ar +ðŁĮŁðŁĮŁ ðŁĮŁðŁĮŁ +íĪ¬ëª¨ë¡ľìļ° ë°ĶìĿ´ +å± ĭ +zi j +zaz en +zan ardi +y ip +wr ung +win ns +we ready +v ng +up lo +tom ba +to ady +themichael owen +the foo +the dave +tar ini +tal uk +skim mers +sho go +shag un +sac lay +s sts +rock land +roc ers +rob z +religi osity +rec c +rb ge +qual ityof +q assam +pol mtl +po eta +patri moine +musu em +more a +mill at +mg motor +mer ola +mennon ites +mau rie +mad ar +ll susa +ko techa +kap u +ka olin +jun as +jon im +jami efraser +ja hi +iron heart +irn bru +ing li +hu gill +her st +hell sing +grosven or +groo ved +gro za +good governance +glock ner +gl b +fore shadowed +for the +fi do +ey am +ever rrrr +eri kal +embelli shing +durham birdclub +du i +du blins +drake ford +detec torists +de vises +de vic +daniel goddard +cullen bunn +constitu ting +chival rous +chal on +car vana +cand is +canad air +bot terill +blu ster +ble asdale +black shaw +bill hemmer +bige ye +beer sheba +be get +bar ts +band aging +bak uman +as sion +ar ner +anast aci +ambedkar jayanti +alaki ja +adeno carcinoma +ad ell +ab les +( = +xo chimilco +whyilove kenya +way cross +vin land +try pophobia +thrac ian +the gun +sti ers +sports writers +southland sports +slo combe +sea vey +se wall +scu damore +sc ig +sar razin +ron ay +ri yadi +ri poste +re mastering +ram das +pop matters +pon dexter +pierre gasly +perip lus +par lance +occo quan +oak en +ni ft +nebrask an +ne tra +nar vaez +n ales +multit rack +modi fieds +meal worm +mar leen +m jc +lap sang +ky uss +kil ty +kil keel +jodi picoult +is engard +intl forestday +inter action +inten sities +in soluble +ic hen +hydrogen ated +hyatt world +horlo gerie +hollywoo dun +hin man +hard iness +gul lion +go ka +gigir ules +gerald ine +ge tta +fun ic +fu gro +franklin tn +for am +fatt oush +esqu ina +elijah wood +eck hardt +drive to +de ok +day long +da er +d allon +culin a +cu f +cro sman +circular ity +chub bies +cho ko +cere bro +c wi +bi zim +bent e +be fits +bangtan boys +archang els +am live +am agi +affe ct +abbrevi ate +ðŁĶ¥ âĿ¤ +ðŁĴļ ⾨ +ðŁĮ « +âĺº ðŁĴķ +ঠ¬ +zero ed +zadi g +yyj events +yo ker +wieg and +wi en +ve snina +vascon celos +trac tion +ti a +tat ler +sun tour +sun dara +summer holiday +silve ira +sie ger +shrop shirehour +shawn johnson +saad lamjarred +ren thusi +purple heart +proje kt +pro book +out shines +ol td +obstruc tionist +norfolk broads +nj hs +nigh trace +nh primary +mun is +min in +mee eeeee +medial iteracy +ma ps +ma ill +little italy +liqu in +lec tri +le tv +lady bower +kuz co +kot ze +kor k +klerks dorp +kin naur +kim dotcom +kevor kian +karolin ska +jig en +inu ddin +inter link +in ang +im polite +i euan +huy gens +hugh son +hel my +han ahan +gri ddles +gran ey +gai ley +fu cking +fr angelico +for tran +far oes +es band +enew ton +eat aly +dontt ell +de mag +de da +de cryption +corri dos +com té +clean and +cheese maker +cele stion +capu let +bur ren +buck nor +boh ème +b dg +atre vi +as mar +archan ataide +ant iterror +anee sh +afai ers +ad busters +aar nold +.... & +Ùģ ر +ع Ø© +î le +x av +word fest +war p +wag t +w tofficial +vor tic +van at +val se +v allen +unright swire +un changeable +ul li +tro py +tran sperth +to yed +thin gever +the troubadour +the iss +t man +sy on +swag ga +super her +su llen +starwars thelastjedi +solomon islands +scri ven +sb augh +ruf ford +rock yr +ro er +remain ders +raise it +q aid +pray in +post codelottery +pin heiro +or bach +ol lan +ny history +nou man +neville southall +napole ons +my friend +mitt i +messi eurs +memor ised +mb ro +ma hat +lil it +lal al +ko sam +kill i +katery na +ji zz +jeremy renner +jar ome +j swr +indie gam +human atm +hl h +hey dar +hermaph rodite +hell onwheels +hau ge +half girlfriend +h silverstone +gre sty +go braves +ge ir +fern down +faw ley +face off +environment ca +em ers +ear bud +e zz +decent work +co sier +cap lin +bry and +bri sca +bra j +boun der +blu ep +blair sville +berk ner +aw sten +as kaman +ari k +apar na +ag da +ab ati +a they +ðŁĴĸ ðŁİī +ye ast +y kk +wv pol +wun na +w jac +uve itis +uproo ting +under croft +un interrup +un frozen +twee tz +tor v +tam ping +steril isation +step by +sports ground +sn azz +sky trax +ski bo +sir pareshrawal +shim s +samanth afaiers +rall yargentina +premier ships +photo copied +phalar ope +pe ss +os wego +obac ter +n inet +monstr ously +mis st +mi elke +mer row +maz havil +manate e +m ku +lo ree +lic ek +le bel +lam ing +kou yate +klu ger +kk g +kis sa +kid de +kear ney +kathiel gifford +john reese +jaun ts +jason inthehouse +jap chae +ise man +har rah +groe schel +gre ying +google classroom +gear up +frequ ents +for r +fon tane +foll lowparty +esof tware +en ema +ely gutierrez +eas thar +does stuff +di arist +di anova +daily dose +cor ri +clean energyeu +chim ed +cal eta +brig it +boyle heights +boo gers +big b +ben aroya +bean town +be good +bad uk +bac oor +b dom +anjel ah +ah le +adiscoveryof witches +a ints +ðŁĩ²ðŁĩ ¾ +ðŁ¤ij ðŁ¤ij +yankee town +with you +vil lew +vi my +var di +ume boshi +trans africa +ton ino +tj thyne +thomas son +thew lis +teve rett +tch rs +su es +star live +squ ito +sinthe park +se don +rush olme +red pointer +real paulwalker +razor fish +positive energy +p tab +ou z +opend at +nottm hospitals +n ze +mile so +main tainers +main bhichow +m pa +m fr +liv vy +jun kin +jas want +ina hyattworld +in ating +hr ds +hartz ell +ha unch +guillau me +guil bert +guer as +god first +gibb o +ger ade +forti eth +flor iculture +fix ate +fashion post +extravag antly +elin asvit +elinasvit olina +e soc +e east +dares bury +cour teney +corner house +conqu erable +clo ppic +cl oris +charity water +chap books +brah mos +bol duc +bobb itt +balsam o +au tis +ant ler +ang one +alison brie +al ten +al bari +aa de +a hero +a aryan +ðŁĴ¯ " +ìĦ± ê·ľ +âĦ ħ +ر ÛĮ +yennaiarind haal +x aml +westside story +war li +vul cans +villa hermosa +val sad +ur ger +u pr +tweet dcs +tropic alia +ti w +the ho +ten ancies +sx swi +stable coin +sta ats +sped ding +smith jr +sea worthy +sculp tured +schak owsky +rowland son +re ha +quick ening +pur in +outback bowl +ot ally +old football +odi yan +neu bauer +me he +lie be +lali tha +ko tak +kari ma +kal afina +jec tion +je anna +jami ek +iv f +is elin +imperi ale +im ploring +green view +geroni mo +gay lor +gar der +gar bage +free theni +flun ked +es wc +eng lands +emanu ela +eas d +desider ata +danielle fishel +d animation +cw tvd +cr k +county champ +com pras +codepend ent +cli entes +caw thon +camero ta +bling bling +bit c +belle fontaine +bec kel +bar ritt +bally na +b dn +atti yah +an vil +american ah +ali w +alge bras +agchat nz +ag wx +afer rera +adverti sment +accessto justice +? ðŁĺĤðŁĺĤðŁĺĤ +ðŁĩµðŁĩ ¦ +íĪ¬ëª¨ë¡ľìļ°ë°ĶìĿ´ íĪ¬ê²ĮëįĶ +æĽ ¸ +âŀ ½ +ਠ° +z solt +yaz id +yas sine +work shopping +water wise +vlog mas +vampire weekend +v anga +upthe blues +uni fier +un favourable +travel massive +tram ways +tr ng +ti bial +ther oot +the machine +tann ers +sub groups +su en +storm front +stay high +stag ione +ss afe +spoo fed +space force +sfr tg +roman ians +ri per +red mon +red crescent +re ur +pac is +p jc +ot ley +only louisiana +oneyoung world +og ham +nr ma +nick les +ngan nou +newcomic sday +n mea +meta sta +mcglin chey +mcge hee +mc goo +maxime bernier +kol chak +kn eller +jun d +jiofilm fareawards +j sh +j ip +il do +i ey +hey den +ham burger +haash oficial +grimez sz +go far +g tu +ek stra +e of +déj Ãł +disney princess +democr acia +del tabc +deduc ed +daim on +county fair +con iglio +cau ld +castron ovo +car v +cap tive +c sec +butterfly count +bu bo +bandi pora +b wh +b wf +ay ad +aw ns +ard ina +angie martinez +al phe +acad music +aar yn +; ... +ðŁĩºðŁĩ¸ ! +Î ¼ +y ati +wood brothers +winne mucca +willie bosshog +who what +wassen aar +ward law +walt disney +v sauce +un grounded +tric order +tom ics +to co +timessqu arenyc +stipul ated +star cruises +square pusher +soul silver +sherrie hewson +sand bagging +s gas +ruf fudd +rudy sarzo +robb i +ro fficialsite +re shoot +ran ker +raj neesh +rac eland +q nl +precipit ate +over mars +nul lar +nu ra +nan ase +msd n +micro pub +man bookerprize +mac ba +ma sud +lun ardi +leon berger +l ve +kel on +jewelry design +il dan +i kot +hol sworthy +hol beck +gl ico +gen z +gan pati +friend ster +fish finder +fille ted +fil mcity +du si +du der +dram buie +dor ange +don kin +dol ans +culture trip +cros shair +cor as +com ma +cas sina +carth ag +car sales +burk head +bubble tea +br attle +bol inas +bell erose +been leigh +bar croft +bach ar +autograph ing +at chi +ar dea +ade ts +ade sua +ðŁĺģðŁĺģ ðŁĺģðŁĺģðŁĺģ +ðŁĩ§ðŁĩ ´ +ðŁ¤ĺ ðŁ¤ĺ +ãĥ¼ ãĥ¼ +âĸ ¡ +à´ ± +zing ers +wave splatform +wa ag +vas ari +u chicago +tran scoding +top end +the golf +tex te +terry branstad +tar o +stel ly +solo thurn +ske g +silver fish +shot put +sho g +shi kar +sch ange +sc ort +sac ro +sac aja +s del +ror ke +ro bor +re casting +psori atic +popp et +plu splus +pete carroll +paulweller hq +pau gasol +pal ghar +oran more +o jh +nutr itive +neuro genesis +nau gh +mür ren +mu ggins +mis givings +milit aire +metabol ite +mcgi vern +margol in +mar ico +l wv +kirk sey +ke mayoran +kc v +k illion +jp gaultier +jes sphillips +jen ne +iam sterdam +ho cu +hic ham +ham murabi +gb highst +g willi +fran chot +fisho s +ex trinsic +es wat +equip ment +emol uments +emo tionless +earth gang +dor ridge +dogs rule +dj awadi +distr acting +dissip ates +diri gible +deflec ting +debut antes +de mus +dan soder +d and +cwa union +crusad ersfc +commen cer +case worker +bur ling +bul wer +brum bies +ben bow +batt ler +au sting +art galleryof +ano x +american pharoah +america east +am ing +akas aka +aig is +ah lers +agor aphobia +aau p +aa sha +?! ... +" | +you bears +yor ki +ye om +y ahh +whipsn ade +vec trex +vale of +v ran +ti econ +texas football +tamron hall +tag it +t chs +surf life +su mal +stru ct +spa etzle +skylar ks +skul king +se jarah +schiz o +rp bp +ro vere +ro ques +real music +rap mag +q amish +plexig las +plagi arizing +piper perabo +orr ville +o bb +my ra +mo ela +mesoli thic +mcgr ane +may theforce +m hart +lex ical +lar ouche +ku wa +kitti wake +kim chiu +jesus lovesyou +hit ler +heritage openday +hered ity +happybirthday harrystyles +han auma +ha den +grace potter +gandhi an +gallow gate +fin barr +fa asil +f chs +elen co +do that +dha dak +del and +dd t +con ci +con al +co sham +catal angp +castle island +cant wait +bucket list +boat wright +blit ar +bisch of +big love +bee ching +barri entos +americ aferrera +am bling +aig burth +ðŁĺįðŁĺĺ ðŁĺį +ðŁĺĤðŁĺŃ ðŁĺĤðŁĺŃ +ðŁĴĹ ðŁĺĺ +ë°ķ ìļ°ì§Ħ +௠Ĥ +под вод +ye pp +y su +y fc +vande grift +u oc +tw on +tony hinchcliffe +ton elli +them here +tennesse etitans +taylor r +tanker sley +tang i +sul fates +song lines +solihull hour +sk ratch +si so +seab l +scu ffles +scarl atti +sand u +san sui +roh ith +ro erig +real don +re thought +quant it +po zi +pillow cases +patricia heaton +park hyatt +opti que +od pp +och sner +nighting ales +naz ario +nab j +my fm +mmmm mmmmm +mir inda +mil roy +mein l +mater azzi +marshab lackburn +marsha ambrosius +mar isol +ma shab +lu le +labour ing +ko dy +kil birnie +kal pa +jone sc +j ato +isch gl +irre vocable +ir c +inte res +id pd +happ year +han sford +guer rill +glu te +fu uka +friday feels +for you +feelthe force +ema k +ell ondon +e schen +din u +dep tt +cop yed +colum nar +cherokeen ation +che wer +cantin flas +bristo lian +big wet +ba shy +art photography +ai ki +.. ?! +. âĻ¥ï¸ı +ðŁĺł ðŁĺł +åĨį çĶŁ +âķ ij +zee studios +zebra wood +z aida +wal demar +w sd +ver b +un characteristic +tra vie +timp act +the orange +the dorchester +t music +swis salps +su prabhat +spar ker +shi han +seduc tress +sec or +sciento logists +scan dies +roman tique +ro q +rmb aloncesto +ril les +ri sas +ren hotels +queens borough +pretty lights +pon er +patt naik +p elion +niobi um +neon atology +nag ios +mrolym pia +moon band +mon u +mon ong +misanthro pe +minim oog +me ia +mat aro +mai k +mach us +m uring +lu mping +little monsters +leann rimes +lang at +land mass +la vy +ky gov +koky anaamdoon +kirk hammett +k bp +jl h +iq iyi +infin itive +ide sof +go cavs +fran ko +fo de +ex ome +em mert +eldor ado +east wards +du gas +dr k +di awara +de pendant +culture matters +cresc enta +com fy +christopher son +bonesh aker +bel u +banan afish +au ty +ale ssa +! "" +ðŁijĪ ðŁı½ +ðŁ¤ ļ +woo dies +vulcan ized +uofl football +un wieldy +un truths +ud t +tradition alists +the meadowlands +terror ise +tar ar +sunnin ghill +su be +stevi evan +sonuk akkar +sond heimer +small z +sen berg +se bel +schir mer +sang u +sam pal +sad ak +s gurr +rock liffe +ri ou +reverber ation +reck oner +re focusing +rand corporation +ra stas +ra gione +quer cu +push up +pru center +pr icking +plant is +pi att +pd as +pas ka +park life +moss yoak +marke ta +lo ev +ligat ures +leu cadia +lam es +l js +kur us +kos suth +kore l +kne eled +kar le +kam ini +jonathan rea +jan ke +j lt +j hms +hepworth gallery +guic ruzzz +getting old +gent a +ge tyou +friedrich shain +fast cars +dol ton +dignit ary +david luiz +da rel +cyto kines +crab meat +counter weight +containeri zed +cont ango +co senza +cn trl +chris gethard +chon dr +budd h +brun dage +brook lands +bron i +bre vis +biz expo +biomed central +bin ational +beat yesterday +bar onet +bad shahi +aza dis +aster aceae +ap aka +ann an +algon a +ðŁĴķ ðŁĴĸ +å® ī +Ë Ĭ +win now +wes thigh +vo lei +vi tha +vi kapoor +ven to +van hoo +uuuu u +usas wimming +u spolitics +u bos +tru ant +trans mutation +tom waits +theatre sports +the billy +te viot +tat asky +ta ve +sub verse +sonny rollins +sho review +shadowhun ter +sand akoz +san ad +road atlanta +ric cio +rac isme +prin c +port arlington +play things +peu geo +per man +om ark +o tomo +notice board +nin ang +negre do +musc leman +mur fit +mis behaved +mini gun +micro glia +maz dac +mand vi +man bear +leon ardi +kil martin +kam es +jand k +ja bez +ishq mein +is dn +irration ality +ioang ruffudd +insol ent +i ç +i pr +i fac +hit ches +heff ner +heff alump +han ge +go valpo +gir dwood +gal vis +ent reprises +dundee unitedfc +dom esday +dissol vethe +dis believe +di santo +desal vo +colo stomy +chocolate week +cathr ine +carto graphic +can ed +bli ps +beau dry +aus able +atlan ti +ash well +ani ven +alth off +all ender +al ini +aden osine +aber foyle +ðŁĺĤ ðŁĻĬ +ðŁĴķ ðŁĴļ +ðŁı ŀï¸ı +ðŁİĤ ðŁİģðŁİī +ìĿ Ħ +ë±Ģ ë±Ģ +ãĤ¹ãĥĹãĥ© ãĥĪãĤ¥ +à¹Ģà¸ŀ ล +youre in +you ku +wor csc +white supremacy +visit lancashire +ver steeg +un desa +uk hi +u caas +trin arock +to whee +teu fel +terrori zes +sy ms +sunba ther +stron ach +streeter ville +sl una +skill sfor +shahb az +se up +scul lers +schro ders +sch wantz +sar gento +san ae +ri ghetti +revital ised +rat nani +po pple +po logne +plin ths +occip ital +nau ta +nau mann +n jea +musta fi +movie maker +mit u +mindless robot +michel inguide +may ang +mandar ins +m fc +ly th +luch adores +lido caine +les bleus +le it +kor cula +kitzbü hel +ju ls +joh ana +imp ale +i zi +i ed +hig bee +hay war +happybirthday zayn +grati fied +gne iss +g delaurentiis +fran son +food network +flw fishing +fijir ugby +fe mm +fe ile +fas si +far zad +fa aa +f mn +epic o +ent ures +elu ding +else car +dundur n +demor alizing +dago bah +d we +cros shaven +cor of +clu mpy +care ful +car bajal +bern ays +beat ntds +aven ged +asun rise +an halt +alizafar says +ah art +ðŁĺ± . +ðŁĴªðŁı¼ ðŁĴªðŁı¼ðŁĴªðŁı¼ +ðŁİĤ ðŁİīðŁİĪ +âĻ ¢ +ภ¨ +wra ith +womb well +wil hite +weigh bridge +us b +uaap cdc +trophy tour +tran sal +th abe +tele marketer +sylvani an +strat fest +strand berg +stran sit +ss mith +spo li +soori official +so jisub +so fun +skarsg Ã¥rd +sec chi +se les +sau ls +ric hy +rem sen +rece ded +ra su +ra dd +proud foot +prin ses +pot light +pnn lab +pas os +par nelli +paol onu +paolonu tini +pa ek +oy akhil +o skal +o india +o del +nu ovi +nisar gad +nat lib +mor taza +model monday +mbe ya +mash rafe +mag adheera +low sky +low ball +life buoy +labour leadership +kun u +kham oshi +jun jud +junjud api +haz els +han wha +guinnessi reland +global forum +gibb ins +escape from +el ayne +ec at +dru mmed +dis member +di baba +dare devil +d cast +crowd strike +crow thorne +clare morris +cho va +chel seab +che ka +ce zar +cat cher +car du +canon sburg +bru yn +brace well +bil lable +bee ing +barnar d +bal ers +avi o +at tune +ashion week +ash our +arde che +anjelah johnson +aishat yler +ðŁĶ¥ : +ãĤ¹ãĥĹãĥ©ãĥĪãĤ¥ ãĥ¼ãĥ³ +á´ ħ +zi b +z iller +wry golf +whipper snapper +w als +vad er +under takings +tyger berg +treecre eper +the east +tam am +sun bed +stick ney +soli daire +sk rull +shock oe +shanelo wrygolf +sen ne +schle swig +sagan aki +sacaja wea +sab ai +rod ley +ri jk +re organising +re locates +ration ed +rat tail +r tbf +r mh +polyam orous +pitt sylvania +pe ated +paramahan sa +ni evera +ne in +morning smaria +moral ising +moon spell +miniaturi zed +metho dically +md l +may field +manife st +lie bert +lech lade +lean up +lar occa +la ggan +kazi mir +jer onimo +jam una +iri she +idhun amma +ic sd +high road +hakk asan +gc pd +gator ade +gar ro +felici ana +fe igned +enbau ms +ek kle +eber hardt +eastern most +do bre +defe cting +de fusing +cyclo tron +co enzyme +chu bb +casu arina +c mail +branden burger +bi oc +asi va +ash uto +ani ban +age ar +ab un +. ðŁĺĶ +подвод нÑĭй +ze alandia +zach ery +welcome tom +uu ren +um hb +tur chin +tor mo +tin chy +ti ousness +tel ah +teenmom og +tan gos +swag gy +sw illing +ston king +sti fler +stable mate +ss d +sle h +sky walker +sketch app +si ska +shipla ke +score hero +schiz o +saun dra +sahi vikas +sag ra +ri ppy +repre ssing +rel ic +recor e +radi ant +proud papa +preston pans +practicemake sperfect +poinset tias +pierref onds +pen tru +peep s +osa urs +oni press +o ãĦ +ni j +nhat rang +n ones +my music +mv coup +mu erta +modul ates +mikeand mike +micro bio +marcu slu +maneu vered +kent land +kas sy +jer o +jeffree star +it sas +ira e +infu ses +impeach trumpnow +im pel +i jaw +hat boro +goli sano +fe il +extric ate +emper ador +emili e +ecto plasm +ec olabel +e io +dot na +din smore +dess ins +dele veraging +daily pic +dag gubati +cut back +craniosac ral +chom p +chi ke +chau dry +cargo bike +bozz io +bonec rusher +bat is +band ari +az traffic +artic le +amo del +air link +ade x +? "... +<<<< <<<< +ðŁĶ¶ ðŁĶ· +ਠª +x om +worm holes +white shark +wau pun +wade barrett +vol z +vbmplong march +un wed +uc ce +tugend hat +tric hot +thunder showers +tempor e +teacher goals +sun trust +ste ins +stav ing +sh moo +scot amb +runder ful +rock castle +robo call +roalddahl day +ric ke +reign iting +rafa ell +ra yed +ra jo +project manager +oyakhil ome +ow ska +obse sses +oat lands +navi star +natwest business +national popcornday +nas sp +mn cs +mn as +mi qu +metal lo +mer rett +me tha +mat y +marlin spark +man gi +man ette +m kc +longh and +long now +li aqat +lam son +ko han +ket cham +kel tner +job sat +jer seys +indi um +in ay +iam harishkalyan +hate ley +hare di +green arrow +fran go +foodi sta +fili bu +fil invest +fi endish +farru gia +fan ad +fair less +en us +elo c +ecuad orean +easing wold +duncan bannatyne +de ese +cytok ine +cum by +cro xley +coopuk food +chee z +ben im +bai le +au le +apic tures +ah madre +af fil +acuer do +abandoned places +ðŁĺĶ ðŁĺŃ +Ó Ŀ +подводнÑĭй миÑĢ +water buck +w ami +unitedin blue +tson gas +tin z +the affair +teng gara +tani shq +sym bology +stevievan zandt +stere k +staff pick +sahy ounie +royal acadmusic +ro ach +riz la +refin ements +red rum +re processing +re eva +rann vijay +radio logy +r ble +pro series +per ju +over lapped +onair now +observ ances +numen era +nhsor gandonor +mt leg +missteen usa +miseleccion mx +miner alization +low enthal +lo presti +lhu illier +ld an +lay ne +ki we +kewau nee +keep it +kas ol +jonsnow c +j ast +j ale +irish wildlife +indi anoil +il san +i ai +hu c +hiday ah +herecom esthe +her sch +hal don +h sps +gn ome +gg n +g wi +g abo +fu vah +flag poles +fi ala +fab y +essential classics +em all +don da +ding bats +de vens +cor ra +che tan +celluc or +c its +button wood +brow ser +bracken ridge +bloc party +bis ola +big ja +bhu m +bey er +bell am +beg at +be sant +bb w +att proam +ar ar +al ao +ab la +/ Â¥ +ðŁĻĦ ) +ðŁĺį ðŁĺĭ +ðŁIJł ðŁIJŁ +ðŁĮ ĸ +âľĭ ðŁı¼ +ye go +y xj +windo wed +wain sco +w ady +uvamen shoops +un dip +u bd +u ach +trump crimefamily +trinarock starr +toddler life +tl ach +the arvindswami +tattoo fixers +tad ley +t mall +surfact ant +sun aina +stur te +steg ner +ssi m +so an +silen cers +shar pei +se anie +ru inous +robert semma +re mar +rachel hel +rachelhel devans +qui les +plussize fashion +pl one +pin pointing +park man +pad arn +p ty +osric chau +oskal oosa +or adea +of art +nov ation +nfldraft news +moon alice +mizzou made +millon arios +melt downs +mell on +meant tobe +ma daya +lu cks +loy ello +leic s +le ks +later ally +kyl erichards +kopp elman +kanna digas +joseph capriati +ji bes +int ouch +ihear tt +hay tor +growwith stpi +ghost of +gang bang +galactic os +full sail +eul alia +er ot +east england +e oe +delicious ly +crew kerne +com pi +ck m +caver na +bon eta +beverly hilton +bee by +be ber +bank ole +asil omar +arctic council +all wood +aldub birthday +.... ..." +ðŁĺį ðŁĴ¦ +ðŁĺģ ðŁĺİ +ðŁĵĨ : +ðŁ§¡ ðŁĸ¤ +à¸Ńà¸ Ń +Ì ´ + ¤ +wis ner +w acc +ut sw +u dr +travel port +thene igh +thel ine +stor ck +spir aled +spar ling +sp ackle +sopra se +sham il +sal ata +rosen stock +red devil +re hn +r pv +py ra +py ar +pier das +pav elka +or ac +onthe way +old times +nuern berg +nola prep +never stop +moon ey +mom ot +me ul +mar n +kuni yoshi +kun ti +koz lov +kis o +kenneth cole +kat ty +jabharrymet sejal +ist at +ik ongallery +i shaq +himantab iswa +high sch +head butts +han ley +hairy bikers +good lettsville +gol s +gi olito +ge bauer +fidge ting +fen elon +extra solar +eu logies +electro cute +eis ele +ed dine +dan s +cro cker +cre wd +cr illy +colin donnell +cin eloka +child wall +cas par +bridge way +bounce tv +black ledge +bir an +bb itt +bail outs +arty r +al tin +abkhaz ia +ðŁĺŃ ðŁĴĵ +ðŁĺī ðŁĺı +ãĤ° ãĥ© +âĿ¤ï¸ı ðŁĻıðŁı¼ +âĿ¤ ðŁĺĤ +ี à¹ĭ +ye ti +wood fin +wkr g +win ker +wi ft +wi arton +whatmake syou +west gate +war of +wahy u +w ado +view port +vic kie +vi u +vi as +versi oning +va den +tri gga +the wil +the len +the fca +th off +th iran +tes da +tart arus +t kc +sy oung +stro ked +spring fling +sil vere +si q +seed sof +sche idt +scavenger hunt +sav as +sa aksh +ru die +ru bus +py g +pure foods +pro quest +pl eno +pin as +our met +or yan +one chicago +ny gard +nbc s +n inju +my nameis +my mt +movement detroit +mil f +mary popp +mary kom +mar ym +maine alden +lu tte +lon za +lin en +li isa +leav en +k com +iron hide +il os +if j +i yd +i bey +hill topper +ham al +gro per +go big +flye thiopian +fli ms +far rukh +emolli ent +el debarge +du by +doo cy +dal gle +copp aitalia +con court +coco bolo +care ssed +brang wyn +big blu +bid vest +bi ondo +bhav ya +bernar dini +balas ore +aster ia +asse h +argent a +ar cham +appel baum +ant miner +an var +amuse ments +amer urological +amali earena +alo dia +almost there +all americ +- ? +ðŁĺĥ @ +wap iti +w snc +vigne ault +uscen susbureau +ug av +u press +trichot illomania +toc tober +that guy +terrel lowens +team hisd +synth ase +swi hart +sun de +sky blue +sel znick +se folo +sefolo sha +se ds +sal aam +saf dar +runderful runners +rubik scube +ru tan +rh statton +potro juan +pitten weem +pho tome +pa ese +not man +national vegetarianweek +mu rex +member ship +meh endi +max carver +marriott intl +mar ck +magne site +m scs +lise berg +lin ge +limit less +le long +laughing stock +klu ang +kemp o +keep fighting +jas mina +i wat +hor field +hen nigan +ha ag +gun valson +guin an +glori atrevi +girls frontline +gat as +fu chi +fri er +flavor wire +fish ball +far point +ende ared +ebay uk +dun laoghaire +dis regards +del t +del potrojuan +deb p +davi di +conceptu alize +coeli ac +cheese makers +cas eload +c thul +brit vic +be tech +bb unker +ballyfer mot +bab bler +ati veness +ary digital +angel candice +ang lian +am yl +ae wrestling +adel mar +ad li +action aid +ach am +ðŁĻıðŁı½ âĿ¤ï¸ı +ðŁĻı ⾨ +ðŁĮŁ ðŁĴ« +weet suk +ver mark +ve mma +ve iling +va sek +ud mila +tre orchy +to prank +then ba +the science +ter ps +swild things +sugar beet +spyro reignitedtrilogy +sie ber +sheryl underwood +she inspiresme +sed aka +se pan +schau m +sam ael +sab on +ryn ism +roo ves +ri zo +real adam +raz il +po per +pas sio +park chat +orochi maru +oqu endo +nycw ff +neem rana +nadias awalha +mol loy +mis interpret +mir alles +minoc qua +mesc lun +m ellis +lovel ive +liber alisation +langou stine +lan vin +kubernete sio +kro on +kor ay +kk ad +kha itan +kart al +k vad +joy riding +jon antoine +jas pal +israel ic +in man +ic sc +hyper mobility +hu main +hu ac +homelo an +hit en +halit osis +h illa +gra ves +gher itage +geo duck +garages ale +fi dm +fac a +f wd +f ali +eskil stuna +east west +dro m +do bell +cross border +crisp us +contr ite +constan za +con ca +committ ment +ci for +cher now +buil tin +br arian +bo dil +bla sian +birds all +be chara +b é +atu ta +ar land +ar ka +allianz stadium +achaem enid +a aco +ðŁĺĤ âľĮï¸ı +ãģķãĤ ĵ +ye vadu +world milkday +wil den +west town +west by +weh ner +ware ing +vi aggi +v liss +track er +toom bs +tonight show +ton yy +thick ly +the vault +texa shoo +tab o +stri de +sri rang +spice works +sm pte +ship board +s feld +ru shin +ro sol +ri var +retali ates +real chalamet +re ves +re state +rand azzo +quadr illion +particul ates +om okri +oc allis +nr r +nobu o +monong alia +mis behavior +micro cephaly +medju gorje +massimodu tti +maso chism +mangal am +mag con +maced onians +long sword +le claire +lan the +ku ril +koz ma +k wwl +k wes +jer kin +iv on +im bolc +hudders field +hor tense +hin tz +gire sun +ge ting +fun kier +fu buki +fo cke +fa hr +express yourself +explorey ukon +en ns +en contra +double speak +di xi +dev as +despo tic +dec ke +dab i +cr ys +confidenti ally +colombi ano +cj mccollum +ca ston +bul ba +bre land +brah mi +bio solids +bio logical +ber rington +bax ley +barn door +bare footed +au slan +atri z +atlan tico +arm wood +anim ates +af corse +ac tavis +.. ] +âĤ¬ ! +wy theville +wind jammers +water vliet +us bank +tu ban +tom leykis +the place +taxider mied +tan guay +tad deo +sub human +su desh +st place +spy aar +spo sed +southland strong +sou chong +sno rer +sk ene +shor tener +sh aus +seavey daniel +scra ppage +sch angelives +sar ad +remo re +ram c +ra wn +profe sses +phara onic +naro oma +myst ere +mycen ae +michal ka +micha elian +metal ocalypse +med wick +mar lyn +manu va +mah fouz +m vula +lov att +lem w +lele pons +lax ey +lam beau +kian ow +jyo thi +jy h +jorgen son +jeric o +jamesp ure +jah n +iz a +harmon icas +green vale +gin der +george monbiot +gang plank +g ice +fu ggin +fresh fish +follow meand +felic itate +e still +e bit +e ath +dero ssi +de cameron +daily pick +con gis +chath iram +bow fin +bournemouth uni +bor sch +black down +beach resort +be anie +ban ani +as mita +ar ses +aqu ad +al gos +adam sville +? ðŁĺİ +" ." +ðŁĵ· ðŁĵ· +ë°Ķ ë¹Ħ +æ¨ ª +à¤ľ र +yo gat +yellow legs +x radio +tearsfor fears +sw op +sunnyside oflife +sunday dinner +stony hurst +spartan up +sj now +sivan anda +saro yan +sar p +re buffs +quint al +plac ate +pl and +pic co +phill yd +or folk +nol anews +nationaldayof prayer +mom oka +mas vingo +mandi bular +lu kel +li ppe +li pari +kwe se +kon jac +kappa alpha +k rin +james brown +horu sheresy +hen ares +gv l +gu ba +go dric +gar ve +friday vibes +free tips +food court +energy ctr +eclip tic +ec fa +dom atic +diom edes +din ars +digital is +desper tar +dark chocolate +capy baras +brexit chaos +br st +boxingh istory +ber ga +beauty ful +bang ali +bal wyn +bal ay +back house +baby care +b worldph +azz edine +ann ville +al or +aar ushi +! ðŁİĪ +! >> +ðŁĽ ©ï¸ı +ðŁĺį ... +ðŁijħðŁijħ ðŁijħ +ðŁ¤ĺðŁı¼ ðŁ¤ĺðŁı¼ +æ° ´ +චļ +yo av +yearofthe pig +x fc +win chelsea +wal pur +w car +vere enig +vau k +us lacrosse +u bie +toku gawa +the cityof +terry mcauliffe +tech republic +tar ana +stou ffer +sm ps +si mus +shro shan +she at +ridd hi +reg ar +read up +pu le +pro sumer +pro log +po ise +pm kk +phyto chemicals +pay with +pan agi +on do +olivi ad +oke reke +nicholash oult +museodel prado +mur gatroyd +munchau sen +mol ite +micro chips +mar yan +lino type +kne en +karti ka +jen kirkman +jeet endra +jaguar uk +is spyaar +in ko +ilove him +illino i +hydrochlor ic +hu ws +home opath +high grove +hell yer +har tono +har ol +gur preet +guil tless +gin oclock +gari ff +furi ous +fried berg +form ichetti +food market +ever ts +esp ie +epo que +eco logy +east anglia +dom swildthings +dam nnnn +craw daddy +commen sur +chab ahar +can ecor +box all +boing boing +black canary +bear de +bake ware +b schweinsteiger +ayush man +auckland uni +ad wa +abdic ate +???????? ??? +: ^ +ðŁĩ§ ðŁĩ¦ +ë© ´ +Ê ľ +wood lot +weare lebanon +ut sw +to ona +ticketmaster ire +the boat +te vis +sverd lovsk +stra ÃŁe +so bol +si aya +shab at +seat on +schle ck +scan ty +royal marsden +ri ki +ri bas +pole mic +pig spotter +pi mi +pi ave +pf dj +ped doc +pe urope +on kar +o wino +n ali +mun dra +mc w +mali q +luci da +lanc s +kus inews +ko y +ko ech +ke okuk +kali sta +kale em +ju icec +ise tte +io dp +inchic ore +in shape +ic ade +huck aby +hones dale +hann u +ha vet +global port +gl v +gess ner +food por +fi di +fathom events +fan cast +faceto face +ev g +epic mealtime +el sey +el oping +ek strom +ego centric +e red +duter te +dil f +der ms +dback s +dam u +da si +cy ma +cul po +cu ca +cro hns +camo zzi +caer philly +bt sout +brum ley +boy yyy +boro on +book selling +big al +bel go +bar soom +az oid +aver maet +ash bery +arun del +ar mine +ar anya +anthe mic +am mann +all smiles +ak dha +ab sr +aal st +⼠¸ +á į +yo z +yan an +y reka +witchesofeast end +wedne sfield +weare somerset +vote on +vir now +vi anna +un amid +ts live +tre gi +tonik roos +titi ously +the af +ten enbaums +tas neem +tapp reciationday +t pw +super conductivity +sub tweets +su render +stor ify +spor tier +sma son +simply thebest +side real +shiny pokemon +sher bourne +saty ricon +rule making +rid doch +ri ving +pu lli +or le +non such +my din +molca jete +mirwa iz +min yan +mile sa +mar imo +ma it +li mi +levi en +lang er +lam i +la vern +kirk wood +kir aly +k lansman +jun cos +joy pad +joy land +jo akim +it slav +irish men +ic eni +hum drum +hound mouth +ho palong +heu ston +gu u +glan mire +gas ometer +gamer tag +fried land +explic ation +end stigma +emm ure +emili aclarke +el bowing +dic kel +deli st +de oxys +cyber threats +crou p +cor reo +clark stown +charli ecox +cham ploo +celand ine +car rel +bren den +blo que +bir m +bha sha +bake sale +album launch +abo ttle +ab loy +îIJ Ķ +âĺĢ âĻ¥ +yam asaki +writ es +whodares wins +whi ther +w punj +w lb +voron oi +vo th +ventur ini +vat ica +tom u +three peat +thereal deal +the spy +th ant +super sized +sun ited +su kan +squ ely +spher ion +spd way +sker ritt +sk ook +sher borne +sen tertainment +sam is +sa kov +rou leau +roh nert +rad bury +ra var +quadri foglio +portugu esa +pipi stre +petro leu +outri ders +orn stein +occul tist +msp bj +mr sc +mou awad +ly ons +love animals +leibo witz +lam otta +kam lesh +kach a +ju tro +jeopardi zing +jasmin bhasin +ira p +interro bang +hor y +home towns +haul in +h sy +h ll +guacam elee +gu adi +gi di +freedomo fexpression +fred hutch +foun din +fore stal +for justice +file sharing +fe alty +fati ma +east ville +dream ies +dio ts +di yar +di bbs +des sus +demon puppy +co bar +chry soprase +chan soo +call out +cali gari +c fas +bo pping +bio data +bin son +bil dung +big money +bell field +bd world +bce fa +aver n +ar ice +an cur +amb ition +ale wife +ag uri +adrien broner +aarhu suni +ðŁĺį ðŁĴ¯ +âĦ¢ ! +à¹ģภķ +za j +with ings +wau seon +ugandac ranes +tux edos +tr ung +to z +tin u +thegro vela +the csp +ter f +sunds vall +sty lers +shirt waist +shel ar +secon dam +sea field +scele br +sar aya +salu brious +ru fino +ru dhram +ri ott +rc ds +rajkum mar +ra don +p ity +ot su +official gtfc +o gg +ny nj +mosthand some +mor se +moon walkers +mikha ilo +methus elah +mans ard +mal one +lumi ère +lo visa +lin as +lift ing +leg omar +k sa +josh ritter +intran ets +indent ation +hv m +hin ck +ha gg +ha dir +guide dog +gu ymon +green town +fro man +friday funny +film day +falcon srugby +en q +efferve scence +di wa +deli us +defin e +de cker +crow child +cor ks +coelac anth +chi ku +chi eri +brian ab +breast feed +brand er +brainfe eder +bra il +bol am +bassen thwaite +annihil ating +angel in +ag ala +ðŁĸ¤ # +ëĭ Ŀ +à¹Ģภŀ +zz era +za rena +x ambassadors +von der +under studies +taste ofthe +tal pur +summer all +sudden link +starfish club +spro ducts +ship ti +se ite +scot papers +sc pol +s br +rw u +rajah mundry +r dio +pur ush +policy holders +pit an +photography lovers +pal lette +oz on +ou zel +organi sts +nu hu +ne pon +muk bang +milk music +megab ytes +map quest +make s +lux or +low enstein +lake street +k dfw +jessic abiel +interpret ative +in audible +huiz enga +hil dy +herz liya +helic onia +heil bronn +head e +hal se +grid ded +gin osa +giller prize +geo tv +ge vent +gath bandhan +g eli +followmeand lufc +focu srite +fo kin +flaun ted +fist fight +el ol +desh on +demo polis +cu zin +chil lum +cataly zing +boy ko +beet z +back logs +ar il +apple man +ang ou +an jani +abu jat +.... ...@ +ðŁĴĹ ðŁĴľ +ðŁĮº @ +yoak erson +yam pa +y azi +wood loch +wol stenholme +wip joy +winner winner +wild trails +wendy davi +wendydavi stexas +war ped +vo guing +ur yn +u rock +turf care +tow ler +timpano gos +ti gran +thic um +thear can +the charlatans +te var +super delegates +su er +star there +st event +sri kak +sk iss +single because +sin chon +shadow lands +sea ice +se gi +sb g +s see +rally cars +pott sgrove +pan macmillan +onther un +onec lick +omar ket +newhol land +nbc u +na dez +musc ari +mel loyello +medic om +mcen ery +lu mut +liveonk val +l win +kro p +kogar ah +kind l +killer instinct +kap ten +justin amash +john lewis +jamespure foy +indvs ban +hydr onic +hud ler +hoo fs +home boyz +hom ero +ho c +her ms +havasu pai +hanni fin +gu avas +gr itz +glam bert +gen bosch +gadgets now +foh len +five ways +fin ian +extra it +en dimp +elong ate +ebol are +eb be +du dh +dor st +dj arum +dishon our +dee boss +daysof halloween +dan c +cullo whee +cu ts +corstor phine +comp ere +committee man +comeon england +clay mont +chin moy +child health +char cade +capsaic in +boulang erie +bosco v +blo lly +black stars +bir bigs +bigja yoakerson +bapti stery +atre ya +ascle pias +are la +ar bs +ad elie +abre w +ðŁĴĥðŁı¾ ðŁĴĥðŁı¾ +ç¾½çĶŁçµIJå¼ ¦ +âĺº ðŁĺį +ಠµ +zuc cotti +witch finder +wester feld +w azi +um ni +tri pple +tr ons +tou bkal +tom ikoshi +these days +ten dy +summer inthecity +sp sp +ser ch +ser ata +sch ola +sauer land +sar rac +saal bach +rufu shound +retail tech +reign fc +ray son +pro jets +prefe cts +party likea +pan ca +over threw +or to +open office +od walla +oak ed +nord land +net wor +nat rona +michaelian black +look north +le us +le eland +lamar athon +la fond +l suf +ko c +kal ender +id s +hynd burn +hook sett +hill side +help fulness +help find +har din +go towv +giro lamo +game plays +franki eco +fon tes +fi des +familiar isation +falken berg +elm street +dream ily +downe ast +doors open +door ly +der mann +de fazio +cur now +cu lia +com pl +cheese steaks +ca bez +brian solis +blo ss +black dog +bhuv neshwar +ba quet +az v +arab i +americ or +aco de +ac ac +ðŁĮ ¡ï¸ı +ï¸ı , +ìķ¼ ìĥĿìĿ¼ì¶ķíķĺíķ´ +ze w +youngh usband +we kiva +war ty +walkawayfrom democrats +venetian vegas +var kala +v city +us mani +thom az +thi ya +th ae +tequil aday +team wear +te ju +taylor twellman +star link +ssc supports +sphoto aday +spaw ner +sole dad +scott sburg +saw tell +s restaurant +rukmini maitra +rte pt +ron icle +radio gram +rachel e +pushaward steam +popul arize +polymorph ism +pent on +park academy +nor berg +new shd +ne vr +namo again +n ms +mä r +my level +my int +mire poix +mi mas +mcgillic uddy +mb ali +lun ds +lovel and +like the +kitt u +k young +k tour +j ook +itslav anya +inter house +incul cate +ig ai +iam raj +he ward +ha eckel +goh pu +geor di +gallo ped +fm revolution +expre ssible +egg cellent +ed onne +ed cny +donof rio +der k +delgad illo +del anie +de cky +daw at +comb iner +bur nol +break time +bran didentity +boy an +bird ville +ber io +barn ton +ar isai +al ate +ðŁıĥ ðŁı¼ +ðŁ¤Ĺ ðŁĴķ +z ro +yo gare +whatchamac allit +wallen paupack +trol lied +tren chard +transi ents +tom burke +thereal t +the island +tal umni +ta pan +swis sair +swil kins +stargate command +stand on +spon sored +spe ights +small cap +shadow ofthe +shad ri +sel t +sandakoz hi +ro chon +ray ford +projec tionist +produ its +pre press +plat ina +pak tia +padra ic +p mma +p fau +or lé +open water +official brianab +o sian +o ols +o ben +note toself +night breed +nic ho +nephro tic +nar aco +mrricky whittle +mou le +mb na +makeup geek +mai ja +ktul news +ko ji +jo v +in syaallah +iam rapaport +i wl +humor ously +han scom +gut mann +gran ato +gene w +geek wire +gardat raffic +gale ano +fossil ised +fer lin +equi ano +energy drink +en chong +emo re +eg ill +education matters +diony sos +desc endent +dele momodu +d oooo +cur rin +con is +com pr +chuck led +chester man +chess men +cau dle +car apace +capit aine +cap tor +cam eco +cam bs +buck master +buck by +bo ad +bfi player +ashwag andha +army worm +ar mata +app raising +ap ical +all ingham +å¼łèīº åħ´ +zo hra +yor gos +wing ate +water wednesday +walber swick +w add +ut din +u go +trum peters +tre ver +tar lton +taco day +spat tered +social isation +smith and +sky dance +sillu strated +shop ing +shi er +shan ina +se meru +se ib +santho sh +sai baba +ru sko +rrrr rrrr +ros man +ronkon koma +richar ddin +queensu belfast +quad ran +py ros +philli pson +pe du +onep lu +o sim +north enden +nel ms +nat itude +nam rat +naji my +mu b +mor ar +mn ps +milli pedes +memor ias +megab ike +me sos +may bury +matthe wr +man nen +mai z +ma bini +look out +like apro +lec key +la veen +ku ehn +jobs report +jku at +iw g +ive y +indubit ably +i drees +hog sback +hockey family +har lee +ham e +guadag no +glene agle +ghj nancy +gastroenter itis +for more +foam us +fly boys +europa africaus +eti enne +ee red +do oly +dh cp +del ilah +dang led +cine quest +chondro itin +chas ma +cambo dians +cali x +bwp mag +blit zes +bad u +au key +as weetz +as pr +aryas milesa +ann us +ami go +ami en +ame mories +albu min +al fond +af fen +ac lass +abu ena +abscbn ballfanfave +ðŁĺ© ðŁĺĤðŁĺĤ +ðŁĴľ ðŁĴĹ +ðŁĮ¶ ðŁĮ¶ +ç º +âĻ¥ ) +á´ į +zab ar +y st +voor trekker +tull amarine +trilo gies +the union +the outdoor +tern ary +t boss +sympathis ers +supportsmaller streamers +sun leashed +stay curious +softhe sun +so kratis +so beautiful +sap hira +safavie h +rim rock +repe chage +raff lesia +prote stan +propor tionality +prior ity +parana que +pap ri +outer most +or of +one another +ole sen +nit ra +mon dial +masto dons +ma go +lord sof +long year +libr ari +li gion +kir yat +kf st +kai kai +itb berlin +iron men +in om +hohenzol lern +har bord +ha beeb +gui on +gop nik +gem elli +gar rel +full backs +fru ctis +for going +fl acqua +fin zi +family search +ei j +ed gard +eco logies +du plantis +deer stalker +chul alongkorn +chiem see +char rette +cam tasia +but ties +bo si +bitt u +abu eva +çµ IJ +âĸ Ĵ +à¹Ģล ย +zipl ining +zi all +welles bourne +vermark ter +technic alities +sylvia earle +swag gering +sv rofficialsite +suppre ssors +stru ve +storm chasing +stan way +so logy +so jo +sl ender +sim la +sicklec ell +shko dran +sens it +say uri +sang li +sa jal +round top +re constituted +ram usic +rain raingoaway +q h +presen to +poste co +positi vism +oviya asweetz +outside isfree +oscardel ahoya +oon agh +ol ata +noise maker +nj ed +neck deepuk +nbaon abscbn +mour ne +mit ton +milose vic +mi rah +medi os +markie ff +maneuver ability +lu koil +kier ans +ker shner +i sec +hymen optera +hi bou +he ye +ha dad +gold line +flo rent +elder ly +el mer +direct mail +ding man +de sha +day man +cri sil +con spires +collo id +cla pper +chi vette +bobb yl +black ball +bla a +bhan ot +bag lioni +back tuesday +b center +av ger +am axwell +air c +ad ap +;; ;;; +ðŁĺ¶ ðŁĺ¶ +è · +ö rebro +zhangji ajie +ze hr +your city +yod elling +wq ed +world traveler +wipe homophobia +wgn morningnews +wampano ag +tol puddle +thamma sat +sur ry +stri jd +sti o +stargat enow +st arer +sorren ti +sna king +sm qure +smqure shipti +sla ir +sho yu +shi mabu +secon omy +saq lain +sali sh +sa ine +s folks +rim world +retro futurism +reci proc +real oviedo +re ko +re but +ra ziel +quincean era +prohi bido +phd advice +pete holmes +persu ades +passer ine +oz turk +outla wing +ord ina +on elife +new borough +n tc +mur muring +lytt elton +louis bourg +living wage +kr l +ke fla +kar jon +it sines +iphonec ase +indiad st +hy rax +hide yoshi +gwil ym +great places +graceland tv +gd w +gas man +faz il +fawk ner +f wisd +f bun +en co +emancip ate +ec ma +dog sled +disband ment +descen der +cre flo +cor ries +coch abamba +cle ft +cheryl strayed +caw lidge +cast ella +cand our +camelli as +by field +business objects +bur tt +bump us +body line +bit mex +belle ek +bangalore mirror +back er +atic sofc +as aka +ar nell +anz hi +allu ding +ala inf +abscon ded +¦ ¬ +zam perini +wheat kings +weather aware +victorias murfit +vic uni +train z +tin er +theech ola +the kids +ta day +sw ct +sud heer +su cia +ssmusic tweet +soci ally +sens ys +sch itz +sc ct +rock ymountains +radio leary +pat inated +partri dge +par mi +over filled +offic ina +o strom +o shaw +no reen +mo anal +mini max +miam bode +mi kay +mge xp +mega fest +me glio +man gat +maison neuve +long stone +lin tas +le hua +ko ven +kla x +kan gel +justi fications +jesu si +hor rell +hard scape +h vc +feld spar +fat lanta +f andi +envi ed +ell ner +el baz +direkt vermarkter +dense st +de vel +dartmoor npa +dar ina +cu ir +colon say +clar in +cen sored +cb ssacramento +car ting +cab an +bur berry +autodesk revit +as car +amar u +amand aw +al ness +ai der +acal cio +ðŁĻĭ ðŁı» +ðŁĶ Ī +ãĤ¸ãĤ§ ãĤ¸ãĥ¥ãĥ³ +vau dan +val et +taka shi +swa ke +sw ole +suffic ed +stra iners +sprint car +spe ts +sk oog +shille lagh +shab ira +ser ran +se watch +scri me +sap ient +santi ag +saint petersburg +s against +rex all +re stom +re released +prin touts +prematur ityday +por voo +pharmac are +per kasie +pass ant +p tz +ou den +oo ch +o eming +national bank +nath and +n tra +moving on +mor os +mo hen +mmac donald +ml baz +metho de +mess enger +me geve +marion berry +l anner +ko shy +interrup ters +ic ci +hom burg +hep ner +har rass +god ble +gil bey +ge uze +gaku in +football club +end games +en sler +eb brown +dry burgh +dram men +doubler ainbow +distracting ly +devol ver +defl ationary +debbi em +dealwith barbie +david stea +cv usd +criminal ising +cleveland birds +cla vi +chugg ington +chel seam +car mike +camp ylo +boat ing +bar bey +b caas +azadis quare +atla ses +at more +asap twelvyy +angi ec +alwaysinourhearts zaynmalik +alphabe tized +ðŁıŁ : +ðŁį ¢ +ðŁĮ ¥ +à³ Ģ +ÙĬ ÙĦ +ö lln +yon der +ye x +yas un +way n +verso books +us nationalguard +us ki +ur mia +uni g +tur bin +trus cott +toc cata +thr s +the bug +team uk +te ren +songsong couple +songkh la +son ho +smith ies +sci anna +schwit ters +saram ago +sad y +s fai +ry l +ro ty +realjoey fatone +r ly +proté g +pro viso +pro start +pra k +per cheron +par took +ow ch +oto gp +osman thus +oman air +o vas +nw f +north pole +muham ed +min ota +mg tab +mens fitness +mark kanen +mar ye +main board +mag al +madra sah +lu chi +lovel ady +lap sing +kon nie +keith harkin +kear sley +ke pala +k pp +k ameen +jw marriott +j pp +impregn able +i bye +hant u +hab iting +gunner sbury +green jackets +gol and +gh es +gestu ral +gerr ards +free agent +foodies festival +fo storia +fir ste +emb leton +ellicott ville +ed aily +down payment +don nacha +deser tion +deca icdc +de wald +de jas +cri p +cook top +cole tta +claire coffee +cha yote +carri g +by er +book ham +bob bie +ber gy +bal asaheb +at wal +ant enne +ann elise +andy ar +ai x +ðŁļ ħ +ðŁijĬðŁı¼ ðŁijĬðŁı¼ +yr ds +work fare +whir led +whin chat +wg no +vivi en +virgini atech +v art +uni ge +thelast drivein +the guardian +team bcps +take off +sthel ena +re sold +rand leman +pu ga +proud mum +pper man +post es +por res +polis ario +ph oria +per icar +per der +par it +om ana +ny ro +nw sl +nra am +non conformist +nigh tow +neat ness +ne sha +nawal el +nat rev +my love +mount view +man nan +mad flavor +ma slen +lyn skey +local memphis +lili reinhart +l kn +kwa cha +knowyour rights +kil gariff +kei yn +kay ong +jarrod farmer +jaros z +jamesx reid +ine ering +in putting +honor them +groen eveld +gotta seeit +fitz museum +fad own +esp o +en hs +econ et +e sts +e star +dinis guarda +di mo +design ates +cru mpton +centr alization +celebrity awards +car aga +british rowing +bg sconf +ber ated +be art +armor y +armag nac +arch es +ar onia +al ico +akinwun miambode +aki hiko +adi az +ac ord +." âĢķ +ðŁİ¨ # +ëĭĪ ëĭ¤ +à¹Ĥ à¸Ľ +ਠ¬ +y oma +warren ville +w ft +ve ere +union pacific +unic o +u oa +thrust master +the ovon +texasedm family +tecno logÃŃa +tanam rull +tach yon +sydney mardigras +sur an +su wa +su tanamrull +steph ania +societ yof +ske ins +sil loth +sen ora +scott m +sco pa +sas sn +ru ffa +rol ston +rhay ader +rep aved +re se +r ll +queen mary +prevention month +pre amplifier +paw lowski +parole es +nit zer +night vision +nh sm +n jr +mogol lon +mo fs +midnight sun +micro meter +mel ati +max us +mar adio +mam malo +mai oc +mac isaac +maaj id +lympho blastic +loving intan +le cky +lam brewery +krzy zewski +klu wer +keep ingthe +kat anga +judd monte +jing yu +jin xing +jab ir +ist as +hyphen ated +hear ken +he slop +gu ana +gram mars +goldeng ate +go ree +gi me +ghos al +gach anja +for ged +flyer stalk +fish on +evi denti +epitom ized +e jay +den airport +dar ude +cor nic +co bby +car rum +bobsburger sfox +black shirt +bio genesis +ber litz +balli ol +ayan ami +ark ell +ar ryn +anag ara +allianz leagues +al wan +ac land +a kel +é ed +yvr shoots +ym pia +yam anote +volta ic +ven i +un saved +tug boats +travel show +too hey +toc coa +thir un +the do +ter racing +te questa +suffra gists +spoil sport +southern cameroons +sk ok +sc illy +rud nick +ru tt +respon sable +ram rod +ram ji +rajiv gandhi +quar ta +pre match +play ford +pj hl +phal le +per saud +par ool +nypd protecting +nul and +newengland patriots +new speak +nagu ib +mwal imu +mur at +mor tlach +mor ada +mistre ating +meli ke +me ment +mccon key +mc par +mar lette +man fre +man dered +maffe i +lu so +leve es +lead beater +lea g +laroc que +kve w +ke ira +kan san +jesse welle +jam u +jake bugg +irrig ating +ib w +hyper dub +hic kie +harri ette +gol la +gar ter +ga ura +fla vius +fiore llo +fas sa +ev ened +er ney +elb philharmonie +el ore +dishon orable +diac omohoy +demoneti zed +de havilland +de at +dare us +dak id +d london +d hh +cro aking +club card +citizen weekend +choo o +cell mate +car tier +br ani +body care +bo du +bam mer +b hoot +amaz o +ag bo +! )... +zip code +z aven +wed der +wat ta +wan ti +unleash the +under scored +to sti +the cove +spits bergen +sol dat +sl t +shum lin +sel ter +sav ouring +sarac eno +sab a +rit orno +ri fa +re combination +r gruppe +qos baszler +pos tharvest +pale stina +p ite +over rule +out landers +orti z +oren go +oise aux +ofcal dub +neil patel +national teaday +nac l +n bu +msdyn crm +mor ong +mon keying +mcgavo ck +lefto ver +lebo euf +jap androids +jag off +jac keted +international ism +ink pen +indi ah +ido f +iceland ic +i follow +hour cloppic +hi aasen +heu vel +hd tvs +hat cher +har ak +ham or +hahah haa +gunner gale +gra bb +gab ion +full bright +fon zo +fb x +f agar +encan tas +eisen ach +dx racer +du gard +del k +cun ningly +coming to +clock ers +citi bike +chal le +bru jas +biggboss marathi +ba el +b hit +az ira +atlant adream +at omy +assalamu alaikum +ascen der +arte mi +aqual ung +antimicrobi als +ang elle +albat ros +^ .. +ðŁ¥³ ðŁ¥³ðŁ¥³ +îIJ Ĥ +اÙĦسÙĪد اÙĨ +ı n +ye ayy +watt pad +viter bi +u et +trisk elion +trespas ser +trap star +to kin +thirdeye blind +teyan ataylor +tanush ree +tan ko +statecap ture +star vs +ss l +sq r +silic ones +sexu alabuse +sas an +sarah spain +saint sand +roth mans +retali ating +re pulsion +re mixer +re kord +rapper swil +pun intended +pri vett +peter house +or os +noti zie +nighto wl +neuro logic +nett v +nelson ville +nad in +musand am +mun jal +mug wort +mu ska +mo zzie +mc up +maxim alist +madel aine +ma thy +lone wolf +lo sail +liverpool phil +ling cod +lg n +kowal czyk +kixi fy +kis ke +killla kill +juan cho +jaw i +it ic +ip at +ing sunday +ie j +hourly wolves +hd z +hasna in +grig son +great dismal +gooner family +good afternoon +gol de +ge pp +fé lic +fuj ilove +evosti kleague +episte mic +eper nay +ende cker +eli vers +el lec +dutch mfa +dor it +dod die +dist ills +design mag +deseret news +de crying +d nl +craw shaw +cow den +corpor atism +com modification +champag nat +canon favpic +bwl ch +buli mba +buck hannon +bor le +biggest fan +bic ton +beer lovers +baseball america +ban ishment +badger up +australian story +atic ket +anesthesi ologists +always remember +ak se +ab am +' + +' $ +! ðŁĺı +ðŁĺľ ðŁijį +ðŁĺĨ ðŁĺĨðŁĺĨðŁĺĨ +ðŁĵļ : +ðŁijı âĿ¤ï¸ı +ðŁij ¥ +ðŁĮ¹ ⾨ +ÛĮ ر +Ùģ ر +yester day +xbox p +war monger +war der +vo teen +unni krishnan +universi dade +un recorded +u vu +trici a +tree planting +tof dixie +thi stown +theo broma +tal ing +swan ley +sunday times +subsidi se +struc tured +strom lo +ss ello +spre zz +sleepa way +sch art +s spx +rw jf +rose ellen +rik rankin +re ux +re faeli +re elin +rc pch +qine tiq +preju diced +pl inking +per dita +pashin yan +our nament +open shaw +odd parents +o sk +nano second +mou lay +mo pped +mission accomplished +min j +megabike shop +mari usz +madein tyo +london art +lan gue +la ks +kir ky +kar jat +kalamaz oo +kal ancho +jared padalecki +j ind +iv rea +ioni q +ingh ana +ing erie +im bi +ik kar +hu sein +ho x +hear metoo +hay good +gy nt +grim ly +glaad awards +ga steiz +fox gloves +fest ina +farquhar son +ever age +edi fication +dow den +desp res +dan so +clif den +clean ness +cle av +che vette +chatt in +car seat +cape may +building champions +brea stre +bo karo +ble y +blank fein +bed font +bastian steel +baske try +bas sc +ap gar +andhra pradesh +alt ay +ak aka +ag ius +? ', +.... !!!! +ðŁİ¤ ðŁİ¤ +ðŁĩ¿ðŁĩ¦ ðŁĩ¿ðŁĩ¦ +ãĢı # +âĹķ âĢ¿ +âģ¦ âģ¦ +zy y +xen opho +worcsc cc +wood sen +wl st +winter break +wi rele +wh on +wax works +vote tris +velocity conf +ut l +unidenti fiable +ucan r +trump ington +ti anna +the common +the actor +team wales +sur tout +strato fortress +star creative +splat z +soe urs +smolen sk +shri kant +sel vag +ryanj newman +rudder less +ro zi +pun ning +pun ic +power station +pop an +point lessly +perri kiely +o base +nu cci +nt ca +nish ino +nat sci +my the +move set +mo gi +mlbaz fallleague +mitch elton +metam ora +mazer unner +max ton +ma dra +kun zite +kov ai +kon erko +kol l +kk rha +kip sang +key z +joanne worldtour +je mez +isspyaar kokyanaamdoon +impregn ating +horizon league +homin id +he resi +hax by +har l +hal onen +ha dro +h antz +gui don +growingup in +go izueta +gener ativeart +gen pact +ge biet +fre scas +fi yah +fa stra +essex policeuk +ero ge +ebulli ent +dr itad +dr ita +dor ina +don broco +diacon ate +devan shi +deter gents +depri ves +delpiero ale +dai do +d kw +cre eds +cra ggs +cohesion policy +christo dou +chowki dar +chast ised +champion strophy +ch m +cc mariners +carac ara +bold mere +bo los +bludge oned +black thought +barto sz +bad deley +baaz igar +ayo ade +ay ase +ar don +antó nio +ann ul +ang ers +and rae +anaphy lactic +all black +air ships +ab aqu +ðŁĮ¸ # +z schech +win etour +wic b +weather ization +vocabul aries +verizon wireless +van stone +us ap +un govern +u turn +u pi +u ai +twittb laster +tuber ville +tar pons +sway am +sprint cars +sol dotna +sne ha +sian icri +screen junkies +sci oli +sci ed +san filippo +ruby wax +ro screa +registr ationday +re stive +re iro +razor light +q amar +proftim noakes +podi um +pere tz +per sis +park ways +pa estu +one planet +nucle ya +ni ppers +mu choo +mo one +mil loy +mil ak +metr is +mer chi +mediev alists +mat exp +mali gne +loren za +leop ol +l ents +ky thera +knickerbo ckers +ke hr +jd m +j tv +igle sianicri +hu ay +hi muro +haryan to +hang ars +fo erster +fei joa +fax ed +fau j +eter nia +est rena +ep ay +emc fly +eid al +eck ington +ear wig +dun robin +dormit ories +demon et +dell acqua +dear zindagi +david l +college board +cole ford +char ak +chap atti +cay den +car don +by lines +busi ek +bran del +bradley james +bny mellon +birmingham rep +bing ed +basel ines +barbar aniven +au m +as sery +arcli ght +arad wanska +an net +ali al +ak ram +active learning +academ icians +ðŁİ ± +ãģĹ ãģŁ +ãĢ°ãĢ° ãĢ°ãĢ° +âĺĢ âĺĢ +âķ ¬ +zu mi +zu mbi +world ginday +will ard +virging alactic +value investing +us in +uniter ight +unfla ppable +uk y +trending orange +timo fey +thru sday +thor sen +thel er +that chers +systemo fadown +str indberg +ste eze +spur s +spr ts +south india +solit ary +shing al +she athing +sd hawan +roseellen dix +profu mo +portugal theman +platt smouth +panther snation +o ake +nychash tags +noo tropics +nine ty +ni harika +mn u +mis directed +mik uni +mi dol +men ounos +me dre +ly sine +lucky welive +lor na +live forthe +lemu el +lawand order +lapid ary +ke ane +kat aria +k long +justfor laughs +juli elmo +jp montoya +jonny existence +jo do +israelic rimes +infra red +hol lo +ho thead +gov murphy +g uni +fir le +ff weekend +e izo +dr ferdowsi +disinfe cted +dish ware +dimit ry +de bartolo +d hh +cool stuff +confection ers +chori ster +cell biology +cable way +cab overde +bur ch +bring themhere +brand name +bra ban +big krit +bette davis +berlin station +beng t +bbcradio solent +bayare a +ayesh acurry +as pic +antagon ism +ani plex +and park +ah ram +è Ħ +âļ¾ âļ¾ +à® ³ +wholesal ing +west away +wer c +v ch +uninor thants +thread bare +theori zing +thegrand tour +tend ril +tan ews +t zi +sy lac +sy beria +str an +sq lite +so tm +smart cares +skylar astin +sj news +sigma pi +shin ned +shill ington +sen diri +santiago de +s fanart +run rig +ru hlman +rep maxinewaters +re thankfulfor +ranger over +qu ini +pla k +pho u +pap ale +paintedby carol +ob l +ny books +no graphy +no buy +nam ic +mod afin +meh dir +mask march +mar cri +manhattan beach +mahindr arise +macdon agh +lu nette +lu beck +lisa kudrow +lafar ge +lady love +la im +la greca +kin berg +kham mam +kam per +kal ine +ho chim +her k +he ins +happy canadaday +hairspray live +gobigor gohome +gi yani +ge thu +ge ren +gal let +fun kand +financial post +er ni +er cole +ele mental +ed gy +e tic +deta inment +dehuman ization +deer wood +clive standen +cl anging +ci ona +chis elled +cey lon +can tons +breath lessness +bour don +booth scountry +blue hole +berg ere +arca chon +anthon ye +an tu +amphe tamines +amo dern +al gin +agains thumanity +ad elia +.. $ +. ðŁĺľ +ðŁĺĥ ðŁĺĥðŁĺĥðŁĺĥ +ðŁijĮ ðŁĺģ +ðŁĩ°ðŁĩ ¿ +åĽ £ +ÙĤØ· ر +zodi acal +zim bra +yes yesyes +wof ford +win rate +vo guet +vin os +val ori +under scoring +tread away +travel diary +tow sky +ti rado +ti dwell +the darkknight +tess avirtue +ta ako +stenhouse muir +star date +stan kovic +sri ram +spre miere +solo ed +shin ra +samsunggalax ys +saaf ni +s fera +roman e +ring road +pur vi +prin eville +posteco glou +pashup atin +paras port +ow www +no ar +ne aly +natural remedies +msg thefilm +mobi lec +mire ya +mart is +mal at +letsmo ve +le um +lat sondheimer +la kat +kotton mouth +kerry n +kav alier +jo chard +jim morrison +it off +in hofe +ig bo +hood rich +gerry mandered +frankie boyle +fe vers +exempl ars +er ling +eddie trunk +earth ship +e got +dit ors +der spiegel +dau mier +czech gp +con des +clar abelle +chud leigh +bur gher +bro die +bot son +black cosplay +berlin er +bell more +bal bir +ann ells +alway strump +abo des +ab salom +ðŁĺĤðŁĺĤ " +⼠ª +⤠´ï¸ı +ä l +ye veryday +yan ina +wx man +world beeday +win eclub +wh sv +van etten +valenci abasket +upon us +un conquerable +tt able +tra ini +today sor +thread needle +th starmagicball +te ks +te agarden +sub space +sty ledby +simil itude +se ol +se bas +san marino +sab ana +ry les +rt fm +rev lon +red entor +recru tement +proud parents +priscilla shirer +orni th +oraclec loud +or é +ophon ia +on erichmond +ohl rangers +o clc +nam bour +nalban dian +mv hs +mudh ens +mu ze +mothersday gifts +mehdir hasan +med ce +mau rawest +man group +living in +lam port +l iciou +kon gos +ko an +kli en +ker mis +kab in +j su +islamic art +ishqmein marjawan +ire do +il lette +horse trials +he us +harpercollin suk +hahahaha a +ha fta +gv hd +gatt aca +fearthe beard +fashion bloggers +enchan ter +em pre +dritad avanzo +dor as +div omovies +dir l +dijk stra +det mer +dese crate +dar vey +cri stie +colour fully +cel in +capital onecup +bury fcofficial +bun ratty +bse ssed +bre wed +billi epiper +bil k +bi ze +bhuban eshwar +b aga +arisai g +ar gy +ai ib +ade va +ac si +ê² ½ +ಠħ +wy playhouse +wood church +wha at +w brz +vereenig ing +tin ton +thick nesses +suz annah +sturte vant +sle monade +sho ku +shil pi +sh lo +senbob corker +ribo some +rei wa +ram sden +quin ault +pigeon forge +perv ades +pav lich +paper mill +official mcafee +o sum +o sle +nis america +negre anu +need syou +moncton wildcats +mis smy +mis firing +melo di +me ko +mastodon music +lil let +light itup +lic key +li sas +land shark +land forsale +lady superstar +la ffey +kumbh mela +kin ton +khi mar +keepp ushing +kar pathos +kan ako +inglou rious +hyper ion +houston dash +hi doe +hay loft +great ocean +gou den +gor z +gla ziers +euphe misms +eug ène +etsy jewelry +ent l +drew diplomat +dot co +dor tm +dj ed +di ffer +deccan herald +daw gnation +dain ian +cry me +cr cs +coble skill +chum ps +chri sman +choose your +chao tic +chandra shekhar +chad leclos +catch fire +car spotting +camera work +born holm +black mamba +black facts +bick erton +bell hop +bar tsch +baldwin sville +aru les +appe ased +apo se +and fairness +alt bier +ah waz +adam buxton +absen tees +a aww +. >> +ðŁĺ±ðŁĺ± ðŁĺ±ðŁĺ±ðŁĺ± +ðŁĶ¹ @ +ðŁijĩðŁı¼ðŁijĩðŁı¼ ðŁijĩðŁı¼ +ðŁİ¨ ðŁİ¨ +ðŁ¥ Į +Ùħ ج +Ø§Ø Ń +you aint +yo gotti +wonder girls +winns boro +van lis +v cfd +used car +u know +timmy trumpet +the scott +the cameron +t mnt +starvs theforce +sm elly +sl vaus +sil on +shuja at +shi d +sfor za +ser res +sch ot +sage uk +saf avid +rock lahoma +pol nare +podi atrists +peter hof +ori huela +one gerton +net suke +national sister +myla pore +my boys +mu sang +moun tup +moro goro +mor bius +moh tarma +mis diagnosis +milli ebbrown +mephi stop +lu uk +lock himup +le to +lakestreet dive +ke iron +imper me +iihm hotelschool +iconocla st +i quitos +hen dee +hat box +harri eth +har pe +gt foh +griff is +gre enough +ger o +gent illy +fu gli +f cat +e qs +dino bots +dham makaya +der ricks +de vex +day anand +david gold +dal ry +dad do +cf daawards +ce sen +cam os +c ff +bran do +boot strapped +bloo duk +bb ra +bar ua +ay ane +av radio +archi plain +ar no +aquil ina +ang ling +amwriting romance +amuse ment +ale thea +al ms +.ãĢĤ .:* +. ðŁĺįðŁĺį +ðŁĴķ ðŁĴĹ +ðŁIJIJ ðŁIJIJ +à¸ģภ£ +xx xv +xtian bautista +wreckit ralph +weekend wanderlust +we build +wap is +wad er +up mann +un savory +u dang +tre vose +tear itup +tear in +su turing +stri ations +sto wing +status quo +song ket +sinter ing +sight seers +showtime pettis +sher rington +shah ri +sele kt +ritu al +ril ton +rec i +re ge +public school +pre ggo +pen iel +pasqu ale +out posts +ns j +nike women +name tag +na ama +mun ity +mor den +miller time +mel ville +ma skin +lyn ley +lei der +lang leav +kin agr +juco product +jocke ying +jo ad +jam mie +interro gations +instruction al +imbru glia +hay tham +ham mamet +hal ina +gu ma +goron talo +gam mons +g dor +fruit land +frey cinet +fox friendsfirst +fly with +fag ot +en ma +em ale +elian enko +doloreshu erta +del hin +de matteis +d zone +cullin ane +crim six +cre mon +corn cob +col ditz +c rich +blitz boks +ational day +as av +angel ino +allyson felix +ah us +ah ur +ag lobal +!!!!!!!!!!!!!!!! !!! +ðŁıģðŁıģ ðŁıģ +ãħ ¡ +âĹ » +xfactor au +wigg inton +weta workshop +western digital +wen ning +wel lesley +we comefromaway +waubon sie +var man +v ho +tou charcade +ti gn +the oxford +t ne +sun city +stan chart +square mile +spl unk +smoke purpp +small streamer +sk aff +shal houb +self storage +sei ki +se mmes +save energy +rend all +re ee +ramach andra +ram li +ram iz +public ed +psycho logy +pil ings +phillip island +par ad +pan theon +own cloud +onthe board +on cology +ome tepe +om bija +ok ta +ny mr +ne gar +mud flats +monster sandmen +mo zza +mckin stry +mccau sland +marsh man +log sdon +light fest +lam bo +l ilo +kris bryant +kkkk kkkk +ken roth +ka am +k sc +jean sfor +jaz zie +ist d +i fera +her om +gun ns +green andgold +gram pus +gig ant +getting better +fil maker +feni more +fal ters +eur gbp +en ot +droner acing +daryl matla +cy outh +compre sses +colla do +co h +christ offer +cer aweek +cas carino +cafe coffeeday +blog to +bir ley +ben ward +bab by +ay outh +as oka +art ine +anu fac +al lier +ak we +aer ate +acru ises +aa ai +ðŁİīðŁİĬ ðŁİģ +âĺºï¸ı . +zoo atl +yi v +worldvegan day +worl die +wor don +wind break +wh orl +vote remain +visual cap +virgin mary +vi stula +utd football +ut c +uro logists +u str +tre ynolds +to golese +thu li +tex til +tand ragee +ta qi +streamer network +stock sbridge +star tin +srisri speaks +spe cular +sou se +simil kameen +sil ents +shu o +sher gar +seag le +se guin +schri st +save aca +san cocho +reassur ingly +re eta +ra shad +pumm eling +petri fying +peter frampton +pel frey +pear le +pan cha +operade paris +nu fc +novo tny +nerd hq +nen ad +mrss osbourne +mou stafa +mitchell reports +mc glone +lov at +korn acki +kkrha itai +kinagr annis +kavanaugh hearings +jal di +hospital et +han ham +go geocaching +gg as +fies ole +fcbb asket +fat u +f mofficial +er witt +elo i +dro b +deline ation +davidg ilmour +dam pener +cur sory +cr acies +copy cats +colloqui ally +cognitive computing +cher moula +catal in +camp allroad +cab azon +brook ville +bro myard +brau lio +bou ley +blan co +ben brook +balear ics +as fcofficial +ar asu +ani mage +amu ffin +amou reuse +allu ded +aco x +ach med +aar tic +( ... +ðŁĹĵ : +ðŁĶ ¯ +zor k +wer de +web perf +vo coder +vi bro +vas wani +ut us +ty oung +tre mbles +tow anda +tom sk +teen aged +tab aco +t ju +supere go +su mbur +stock piled +steel man +sse au +sni pping +sicili ano +ser ina +same time +sa osin +rumin ation +roo sen +reed sburg +re cker +pound world +pine view +philosoph ically +phantomofthe opera +ph resh +pack ers +olaf ur +mill ersburg +mika ela +mere ce +matar azzo +mark g +manit obam +malevol ence +livesof ww +legal ities +km js +kirlo skar +inste in +in dri +il ai +gry m +gil gun +gau se +g bit +fun clic +funclic kearn +fu hrmann +for bernie +foot board +fohlen elf +feuer stein +fem ke +fat back +fair bank +embar kation +elk hart +earth science +dun smuir +da jj +cross gene +cro isi +co bre +cheese cloth +centr ale +cele st +cave tt +cal pine +bundel khand +buckfast leigh +brush work +bhat ure +bernar dus +bar room +b ely +auto sales +as certain +an doh +am pera +al ha +abre ak +aat state +ðŁĴ°ðŁĴ° ðŁĴ°ðŁĴ° +à¸Ľ ระ +x amar +world prematurityday +wi ens +vat ic +v to +uv xy +truth out +tra vieso +tour memories +ten ny +shu ffler +show business +sail fishos +saf tas +sad ers +s journal +ro xo +ri angle +rhy e +rhe inland +redbull grc +re q +rath coole +r nai +pyo tr +public policy +prolifer ative +ponti ac +pon chat +pokemon xy +plain moor +phenom hoops +pasalu bong +opti musprime +oph ant +oak mtg +new paltz +nan ako +na ea +myth ic +museumof nature +mob berley +million maskmarch +mil ang +mick ens +mer id +men uhin +mar quet +mal ana +macdon alds +ma ec +lou cityfc +koiv u +kn m +king don +je el +indeci siveness +immobili en +ic are +hor nish +honda indy +hipho ped +he ssel +gü zel +google playmusic +god ons +geol soc +g callen +fro mal +flu shot +ex communicated +er id +em rs +elli sville +ed ric +ec thr +eber ly +du binsky +dome stique +do tc +diamon dand +der ose +dank memes +counter clockwise +compar ti +co authors +cmo kerala +cla rem +cj hl +che gwin +camp ton +bestfriend day +ber tucci +be ier +basti de +bar camp +anti psychotics +aktu ell +afric alive +ad mi +abdel kader +ðŁĴĶ . +⾨ " +Ä Ľ +yogotti kom +y usa +we tz +wbc sd +wal vis +video taped +val ter +tw rk +tu pe +trabal ho +toronto realestate +tor vill +this world +theright thing +tha at +terric lark +terr ariums +tech re +te urope +surbit on +stigmati zed +stagn ating +sta rena +special ity +son mp +smi tha +sleep walkers +skook um +shu go +sho we +run dle +ro lie +richmon draceway +revol vers +rend collective +ration alization +qa iser +pro gr +pro dding +plo sion +official ronnies +nj politics +natwest t +nationwide arena +my ton +muhte ÅŁ +mound sville +mon zon +mon sie +milli grams +mckin ney +mason ville +mam c +ma shi +longyear byen +leonard town +la forge +la em +kor ta +kor ps +juddmonte farms +jaw ab +jas prit +jan ez +invinci bility +independ ance +hunts man +hubb alli +hawk stone +god ly +gher bitz +gc isd +from wherei +fran zese +for folloback +ex x +emp son +em sworth +elf quest +ejec ting +east bengalfc +duplic ator +dra iman +dog star +deni ece +da ith +d by +cy lons +cu ota +critch low +cott bus +compre ssible +co ono +clau demonet +cj n +chel o +cen ac +capp iel +can asta +c mcdavid +bou se +blac ki +berch tes +bein art +armam ents +alleni verson +ach ille +abujat witter +: """ +ðŁĺŃ ðŁĺ¢ +ðŁijį ðŁĺĦ +ðŁ¥ĩ @ +zi x +z wart +vi shy +ver me +vani tha +val de +tuske gee +transpor te +tra buco +tr ate +talon sup +speedway gp +slo ps +scapp oose +sa hm +rick shaw +real d +re definition +qui kr +puro lator +proud sister +profan ities +pro plan +primeminister imrankhan +plan escape +pla sm +pioneer dj +pan head +p tah +orlé ans +on ts +oe ster +nir van +ner ules +ne vs +nale o +middle earth +mcr artgallery +ma kani +live bolanet +lig aya +lifel ine +ko loff +kfc barstool +kent mere +jit sircar +iowa state +ind ade +i au +hun k +hel meted +goer ges +go kyo +gli mmer +george soros +fur a +fron tto +febr ile +equ otes +en join +dun nett +dul quer +deliber ated +cuad rilla +col bi +cih housing +chen oa +cast legate +bumblebee trust +bridge tregan +briden stine +bo real +bernar dez +bean ey +bambaat aa +ax ford +arti es +ann ach +an ang +ami da +air pics +................ ..... +ðŁijĮ ðŁĴª +ðŁ¦ģ ðŁĴĽ +íĶĦë ł +zi ba +zambe si +z lj +yav in +whites burg +wh ately +vishnu vishal +vic ent +vi ffest +ven der +van sciver +uni st +travel thursday +til o +thalap ath +ter ni +tempe sta +tal bo +ta jani +sweet sixteen +super giants +sun n +strange musicinc +sq s +se gs +sa hana +record collection +providen ciales +po ku +pips queak +pi ola +pe ja +p music +ore wa +oden kirk +no cover +new smag +n lyonne +mu jib +more head +mont copa +mo gra +mil son +mer iam +mat ted +madra ssa +lippin cott +laure tta +la stic +kis ston +jimmy havoc +jess ops +ja at +it fest +immacul ate +hyundai wrc +htcon em +hol lett +hil burn +hack en +grif ters +grati as +gole stan +gi lead +gel led +gand u +g dad +fri sch +fc thulhu +fal abella +enic ole +eni k +ellen brook +ei shockey +edmon d +eastengland amb +dn ts +design awards +del dÃŃa +deborah meaden +david coverdale +daddario andco +d subverse +ct us +coy p +cor sham +clar inda +cep tions +carol yne +bur cu +buffal os +botu lism +bo ggles +ber ard +bajac alifornia +at sf +ang let +aber avon +aa x +'' / +ðŁijģ âĢį +ë¬ ¸ +za ke +wolfe boro +wig ton +wc p +tu ch +this guy +thermo py +the creative +t dsb +swe ed +suyy ash +suhel seth +stuart scott +spru ill +sp aper +sm alaysia +sanje eda +sal ver +sab ot +sa hrawi +roo ker +rockin rio +ro dale +re spon +raider scanberra +qui sta +polnare ff +pir ouette +per rette +per mutations +pel os +pe abo +our lady +no go +ne gre +mu tti +montero sso +mo tel +lim elight +learn chinese +l á +kul ang +kore matsu +king scourt +kar dinal +jeffd sachs +jake millermusic +it staylor +ifc films +i ach +hoo drat +hol and +hoch zeit +hemb ree +gst tnhs +glo ssy +fuzz ies +fleure ast +fer ox +farmer sville +faceli fts +etsy rt +esc orial +dt phx +disc ur +dimit ra +dex change +dance mom +cov hour +colt cabana +chi me +catastro ph +bridge ville +belvedere vodka +aw ans +ad aw +à¸Ńะ à¹Ħภ+y wam +wood cutting +whim brel +w display +vi agens +usc aa +transi ence +thi as +te dy +tat acompanies +spr ite +sk om +sc ran +sag ne +sa athi +ros ana +ri sto +rec tifying +re shammiya +ram ba +pure storage +pp w +platinum games +onec ampaign +nai ste +na hid +mo cap +mirror football +mir kwood +mer ten +mad docks +love seat +light man +lavi shed +kon achan +ko hala +kel ana +keith olbermann +kam on +joy less +ipp olito +ine ve +i orio +has bronews +gv su +gu sto +gra di +ge macht +fm sphotoaday +flori dac +fin cantieri +fi sted +fast balls +fag et +extrac tors +elu ve +dun kar +diam undial +davi dr +culture club +cul turing +ct news +cor ddry +cop an +clu brooms +cl ent +chronicle herald +ce cchini +cas cia +can ari +cal en +c pac +buck ers +bu ke +bor a +bon da +bill yel +bienven idos +bat sibat +au skas +assay as +apla stic +al win +ac tics +aa ihs +ðŁĺŃðŁĺŃ ðŁĺĤðŁĺĤ +ðŁijģ ðŁijģ +ðŁ¤¦ ðŁı»âĢįâĻĤï¸ı +éĸ ĭ +ภĺ +z ast +wesley stromberg +war road +ver net +un leavened +toy maker +town coffee +through ly +thi ef +thene edle +ten enbaum +tee total +surly brewing +superse ded +sub class +stone masons +starcreative stv +sport susa +spac elab +sof teners +snake day +smo t +sk ylon +shoo jitsircar +shal ane +sell schaft +sec ateurs +schle icher +sale em +salad ino +run keeper +rsa security +rivier anay +red and +rayn aud +puer ile +pu tian +prefer red +pig sty +park minyoung +pariv artan +ophar yngeal +one buffalo +nj ic +newye arre +new a +nc su +nadi ra +mon sta +mir ren +mac robert +low ton +louder milk +lec ity +le bow +le aman +l ú +ku hns +kraft foods +kr ls +kay ag +katrin api +kath rada +kan ai +je be +istandwith maryamrajavi +io so +ill z +il ens +holly bush +hill park +hat ful +gu to +greattaste awards +fo su +embe zzled +el ses +don ite +dj f +dis barred +de mayo +cro ons +cor ney +coal face +co ire +cnn debate +chiso x +ch h +candice accola +calab rian +buil th +bo zen +bo zell +biek sa +ball arat +ba ños +any oung +anton yl +alla ges +ali maging +af gan +z ins +w lox +ur ica +un banned +ul fa +ucla anderson +trevor ombija +synthesi ser +super blue +st ith +sse afood +spre well +shar man +sau to +sap ne +ry no +run withus +ri bot +re sna +rand on +r fh +pp ar +phul kari +parac as +ou fcofficial +opun tia +nuyor ican +no taries +nd mc +mis is +miniaturi st +millen ia +metron orth +me ike +maya ali +matthe wh +man lius +man art +main sail +mac pro +lich ten +kno twork +knightri der +ken shiro +je gede +inde ed +in aga +i ths +he aded +hali mah +fundac ion +finner an +fc go +encant ada +ego ism +edm life +dis agreeable +dia q +corusc ant +colle ens +cn wk +classi sm +cha ining +cas so +car my +bus well +brasile ira +bor ba +be healthy +aubre e +at lin +assemb lers +aldu barkad +aff ing +adjour n +! "... +ðŁĺĬ ðŁĴļ +ðŁijį âļ½ï¸ı +ðŁIJ ĸ +ðŁĮ²ðŁĮ² ðŁĮ² +ت ÙĤ +yod abuda +ye ch +wat ari +w elike +viet name +up skilling +un fair +tt india +tric ep +ten gok +technom usic +t sering +sun coast +str in +stand ley +slo vo +shalane flanagan +sen bill +sch lie +sal enow +rosal ynn +rol on +ro kh +research gate +rb z +popl ars +pl sss +pend olino +panip uri +pan handler +pack y +owen smith +os se +obe di +narc issa +nak ita +na ira +mud lark +mis sher +marke tre +man vs +loss yndrome +lock ton +little foot +lim kok +law alla +lari jani +kre isberg +kkrhaitai yaar +kend ari +ken cana +jo stle +jesse metcalfe +jel avic +jarrah dale +jakob sen +itv weather +its friday +infomer cials +ig daily +ic hou +hp discover +her ridge +hengist bury +hahahaha hah +ha gio +grom mets +greatplace towork +great value +giron afc +font bonne +fic ha +ferry bridge +fabri zi +er and +enviro ment +eliesaab world +el ham +edin bur +echo stage +east lands +d sk +cut throat +crab grass +conse crate +confection er +col lor +cogge shall +canad adev +can ongate +c jeu +bug s +brune au +bru shy +brick fields +bhutto ka +be siege +bb v +basker villes +bad girl +bab alu +av ma +archa ia +ach a +ðŁIJł ðŁIJŁ +ðŁĮ· ðŁĮ· +ล ะ +yu miko +ye sequality +yacou b +world net +woman kind +wo wt +wi zzy +westhe ad +wander lei +waf ting +vat iron +use your +uni fil +trinity learns +ton ow +to graph +tech net +stock ard +stand united +stacey solomon +specu lator +sp atriots +solu cky +sol enn +shett leston +sh ky +schi pper +scheck ter +sare awesome +san deman +sa qq +por ton +pop sters +pit tie +pe gg +panagi otis +pa ille +om ay +olympia stadion +nov ac +nol in +no excuse +neuk ölln +mu et +mer yem +mem brance +master pie +ma under +ma gos +m kg +lv m +lubric ate +lu an +lock in +li ja +land graf +lalit pur +ki bana +kash gar +jerry rice +jede diah +india at +howit zers +host ing +home decoration +herbi vorous +happy baekhyunday +ham moud +hack tivist +gool sby +future ready +fin k +fam iiy +fall ston +fair ground +er langen +er ine +dle ys +dic kiev +dah lem +da shain +cu cin +cre sco +country side +cost es +conservation org +catech esis +carli vatiron +car mon +cag giano +c sas +bridle wood +bon giorno +blood donation +bla ha +bar thel +atlantic records +ath ur +artist life +ar leen +al ady +aggrav ates +acknowledg ments +abus iness +ðŁĺİ ðŁİī +ðŁĴĥðŁı¼ ðŁĴĥðŁı¼ +ðŁijij . +ðŁijı ðŁĴª +⼠ħï¸ı +ઠµ +war ble +ur va +un b +tom oh +to pal +title holder +thin d +the state +th ills +sw police +stu hl +st rob +sp reck +so ter +so bi +sick i +si skins +sand art +sadi k +rath skeller +rat li +promiscu ity +pay nes +or by +one hunga +odi hr +ocean us +obliter ating +no ct +nate diaz +n pv +mat tea +marien platz +maajid nawaz +longh urst +l tc +kra vi +kel lum +iz ak +ili fe +i bb +hun cho +hu by +holly conrad +hi ki +heng elo +hein kel +hap iness +hain ault +ha idt +ha gin +god da +gin er +gair show +g latt +flat rock +finish the +farmers weekly +extempor aneous +echo location +dema rest +daw ud +dar ragh +cul la +cru et +crack lings +cosplay girls +consol ations +cogn ate +cle eve +churra scaria +cap il +cal cin +byu cougars +bulldo zing +book keepers +bha wani +bemo repirate +bec ke +be harie +bar thele +atifas lam +asdfghjkl ñ +armb arnation +ar line +aphex twin +ambassac ats +am v +alpha phi +ak pabio +ais c +afir st +abe ba +ðŁĺľ ðŁĺĿ +èī ¯ +§ ðĿIJ +yellow vests +uhcougar mbk +tri j +tra duc +th ena +tex mex +teng ku +tele mann +tee jay +sweet breads +strade bianche +still gotit +state fairo +statefairo ftx +stabil iser +si rr +shout a +she aven +sch rank +s gallery +rush worth +repell ents +qu itters +prospe red +prefe c +pra dip +po zzi +pim pri +phy ton +photo therapy +pen ser +oni shi +obel isks +ntu sg +ni za +new school +music and +morri sh +mex ica +mer imbula +med sci +mayo tte +maru yama +ly se +livand maddie +lin z +le vering +laur sen +lance armstrong +lam ond +l nh +kun war +kate winslet +kane brown +jo cky +jag meet +intertwin ing +inci vil +iff ley +hyper thermia +houser ules +hor adio +hiji kata +geri halliwell +ga er +foxsport saz +eye bags +elk ington +ed miston +dete stable +d sps +cycle toworkday +ctm pofficial +cru mple +coo ool +consu m +conjun ctions +cle to +citizen sunited +cian jur +cho bits +cath arina +cake bread +c fia +bri ana +bow down +bor zoi +bink ley +bec tomy +beard more +bat in +astro plus +as pens +ant t +amo wry +air ra +af ten +aer oline +ab bath +__ ) +! ðŁĺ³ +âĸ¶ï¸ı âĸ¶ï¸ı +zap iro +worldnet daily +wno tweet +west village +weare here +we stre +un glamorous +trunk show +tr nava +the ju +ta wana +stap led +soul wax +so sv +sky cable +seem ly +schi pper +sch atz +say what +ru sd +ric cardo +retro game +re vy +pyro technic +ps ils +planet coaster +pl h +pesc et +peach bowl +partylikea journalist +pal icki +p no +ow news +nith yananda +newsin vids +neuro surgical +neur orad +nel o +nature za +nat su +nac cho +na hr +mun tz +mell ons +meh rang +med ary +mcdon al +man zana +limitedrun games +lie paja +leadingthe way +law i +kri eg +itie suk +ir gend +intercess or +in ba +holiday gifts +her ma +hather ley +gold fish +girl child +ge ier +gab ou +flo etry +fi yero +feliz finde +eluve itie +ef dp +ec aatstate +e ut +dy w +door bells +do tty +dise m +de francesco +de bash +david lyons +dar ma +dal meny +cy anin +cur ti +crazysexy cool +concer tina +coffee hour +clu be +chau dhary +bower swilkins +big ham +ber ri +be cos +bbc philharmonic +bar beau +azhar uddin +au vers +ar mored +antonyl ruben +ano vich +amin os +a ahs +; / +. = +ðŁij® âĢįâĻĢï¸ı +ðŁ¥ĩ ðŁıĨ +ðŁ¥ ¦ +å¥ Ī +ãĤ ¿ +âŀ « +âĬ ķ +ÌµÌ ¨ +yellow man +womenin aviation +what getsyou +wa yo +vill ano +ul rika +u oregon +the deol +tch r +tar dif +t sao +sub su +sty l +sta den +st fagans +sp az +showme the +sensation alist +sa akash +ri ky +reece mastin +recon figure +pro cida +post traumatic +par ol +pag ham +p bj +oxi meter +official cufc +ob by +now w +nordic a +night watchman +nic omaine +new project +mâ ché +music on +mol on +mi kal +me j +made well +ma dri +lati go +la tham +l illo +knight frank +kle ber +kha war +kas dan +kan turk +it tttt +infringe ments +ide ser +hero ic +head stand +hair band +gro bler +glad stone +games radar +gal angal +frederick son +for dair +fedex field +fa awards +exacer bates +et ag +equ ick +ed ream +e gra +dut cher +dah len +com alee +cli ven +clev enger +c sub +bun do +bug bear +boy sen +black hat +ben ett +bax endale +band ila +ban tering +bai xas +any ama +annivers ay +ang irls +and then +air fares +ag la +ad hy +ach al +aan p +ðŁĮĬðŁĮĬ ðŁĮĬðŁĮĬ +wul f +wo wee +weis berg +water head +wall an +w lad +vol tex +vliss ingen +valley cats +un cia +tucum cari +tor ay +thermo set +the mar +tegr ation +steal mygirl +spider woman +sil ience +sh rum +semi annual +sch utz +sbli ii +sati ety +saafni yat +run blog +runblog run +recy cler +re authorize +puli murugan +public ise +pu ella +pro sieben +pratt institute +pp ppp +play dead +phwo ar +pe ka +paradox ically +palas zczuk +pack ing +oy ston +ouis ville +o saa +noy noy +nay oung +mccul lagh +mahan ey +lu kis +lou brutus +loe wy +lodh ran +linke dua +lego league +ld m +kumb aya +k institute +just ink +jasmin ec +jahang irk +jahangirk tareen +jack russellterrier +j afridi +iz umo +iowa statefair +her bology +fun night +fuj ioka +fla yed +figh tin +ferlin ghetti +fcunited mcr +ell ina +ea sterling +don abate +distractingly sexy +cul ls +credit able +chav arria +chant el +centri sts +cav at +care tta +c gg +bu pre +bryan brothers +brick ley +bir s +bi mba +best nigh +ber l +bedn arik +bec kia +ba hahahaha +awo olf +att ara +at your +assassinscreed origins +anandi ben +aj w +af eni +ìĻ Ħë² +̵̨ ÌĦ +w st +vla do +vis cera +ven et +va stra +twitter stake +tu ckey +trigla v +thu cy +then et +thebachelor au +the ware +the see +tb snetwork +super critical +su y +sty mie +southe aster +simpl ys +shine bright +see v +seattle symphony +sean price +sam mo +salman rushdie +safe guarded +roun dups +roof less +rheum atism +retwee et +red berry +radi ation +prev ail +pr ss +ppor tunities +pick oftheday +par terre +nigerian creatives +nfl oncbs +nam ad +mo twani +mm ers +micro aggressions +mc keen +mad son +llan id +li ep +level up +le bih +laba dee +kit ale +kis sarmy +jeep family +interview mag +ic ahn +humayun saeed +ho stiles +hh v +hemorrho id +he tta +han dog +gam me +gallo per +fer ias +fam i +f assie +ephe drine +endthe fed +dra we +d sap +cr ine +cloud native +ck ickoff +chu o +cbc to +bridal shower +brick layers +bott lerock +bon it +blessedand grateful +bjor k +beour guest +be somebody +bau n +bar ge +bal lot +b dunkelman +atx festival +atech nology +anth es +andrze j +amo or +alan de +a equ +< -< +ðŁĺį ðŁĴĻ +æŃ ¦ +ãĤ³ãĤ¹ãĥĹ ãĥ¬ +âĿ ĥ +ଠ° +ॠģ +Ù Ĵ +xxx ii +wm phoenixopen +water marks +ver vain +tá naiste +tx instruments +trues dale +te thys +tai ki +supp lan +su q +sti ka +soun darya +sece ssionist +se dna +sar lacc +roer mond +qu intel +qayy im +pr inter +pole dance +pi bil +photo show +pedre gal +pav los +nt g +nexus mods +n int +musicis legend +mule shoe +ms news +moo sic +modafin il +mo fo +med center +mcgre evy +marzi apie +marcho frobots +majo relle +mae by +mack enna +log ico +lo bbing +lin thicum +lex ia +leu chars +kumb akon +kar ai +juni at +jib con +iwm duxford +israel underfire +in ol +ieee org +i fat +hygi ene +hu dgins +healthy kids +he witt +hand spun +ha qq +grant morrison +go ber +gif tedness +getre al +ge mo +fun fetti +fuji ko +fu ffle +foxnews facts +forsk olin +fish sticks +fi roz +engra ined +ec ss +e pix +dylan thomas +dun ner +d loading +d co +cute y +crested butte +ch f +cau field +cas ca +carboxy lic +canig gia +camili zers +cab allo +bi sham +beth ke +bb ctw +ba id +au techre +au ric +ash craft +ao b +andy stanley +am bers +alison moyet +ali ki +abir d +a few +-------- -- +!!! .... +ðŁĴĹ ðŁĺį +ðŁijį ðŁı¿ +ðŁıĥðŁıĥ ðŁıĥ +ÙĩÙĩ ÙĩÙĩ +your dreams +ya jam +women shoops +win rar +wein stock +walk around +wad d +w wildlife +voteen rique +vintage bikes +v wo +ti ot +the tanmay +terraz as +stre psils +stat too +stardew valley +sr ams +squ andering +spring town +sports ound +sommer ville +soci opaths +sm j +sla gle +sko da +sil ke +she ed +sad dict +riseu pred +reli ent +region of +r ham +pur nama +puertoricop ur +pu pi +prs journal +pe sch +particip ative +palmi otti +opto genetics +openstack summit +nor v +ninju tsu +mm fa +min ya +maim onides +ma grath +lim nology +libre ville +kix brooks +king andcountry +kin ole +k mi +jess amyn +jacob grimes +instaf ollow +indi aw +iaaf worlds +hy mer +hu du +heav es +havai anas +h rad +grand prix +good hart +ged dit +gary j +foreign affairs +fl ico +fil mer +fi ats +f mm +ey l +expl ora +england golf +electr oneum +el x +eh f +drunk history +drug mart +drivein mob +den een +def els +deck hand +d bel +cp sc +ck worth +chill ingly +chau vin +chasing life +cedar ssin +ca ahoops +bru gman +broad land +boat face +biopla stic +bim by +beau desert +bau x +barbar ity +bal dies +at cs +arte ducation +ardi ente +aper ry +ali ght +ac climate +a otw +ðŁĺŃ ðŁĻĮ +æĪ ¦ +zem ski +wyn newood +wil den +vel arde +uof sc +un savoury +un civil +un achievable +ty umen +transm ountain +title holders +tic os +thei hi +te tras +te sted +sunid hi +steve mcqueen +spring break +somersaul ts +shor tens +sho whome +shi awassee +scorpi os +scer vino +rowland schools +roth schil +roger sarena +rise against +rehman malik +registr ant +qad dafi +po cos +paren te +paci fism +p tn +om aki +ol un +nucleo tide +ns agov +ni mm +nhs grampian +nd h +murshi dabad +mr sam +mo dok +mentalhealth week +mat zke +mark dayton +margare th +mar kii +manag ements +mainbhichow kidar +ma ppy +long side +lips comb +lib bie +lanc ôme +la dainian +kirkcud bright +kilkenny clg +kas auli +kar ra +kalin ic +k hairy +juliab radbury +intercess ors +he che +hatsu koi +h go +god parent +go wes +football tips +fo yle +flower beds +fi ets +fal zon +eye ofthe +expres sen +ero des +erin burnett +dunkar oos +dun huang +deri sion +deare st +de keyser +cu ira +coo pers +cister ns +cho tt +chees y +che tu +cfb hall +breakthe silence +bra gh +bowl by +boat shed +black buck +bet abrand +bay ero +banyo les +atay lor +argent inos +andrewr annells +ad dres +ðŁıĢ ðŁĴ¯ +åĭķ çĶ» +ãĥ Ī +âĹ» ï¸ı +⬠ħï¸ı +zak ia +z up +yum mie +yugi oh +you and +wi gger +weing ut +w ceu +vri end +us ando +un disciplined +televangeli st +tch ad +tam bour +syl la +sum times +stur dier +stre eth +spo int +skin nier +saint seiya +rohr bach +ratli ffr +rame kin +ram pa +public ising +pre late +pr anitha +pp ance +power stroke +pi one +par aiba +pal ar +out fielders +ou can +ou ag +os aga +orang ish +oneand only +nys dec +ninja sexparty +ner is +nel la +nc gop +nationwide kids +n cu +multi ethnic +mu kuro +mon chengladbach +mil gram +may wood +maud lin +matte i +man asseh +magic mike +lud ger +ls don +lou x +ko ester +knap weed +kin dred +jas wal +inthe wild +inter no +inher its +inform atique +inf anta +ie business +ibelievein bookfairies +hok um +handicapp ers +ha id +gul ping +gra der +ging in +gautam rode +fun gu +fore achother +fle ener +eswat ini +em wangi +e step +dry lands +dream big +de bb +dd ddddd +cro kes +co vington +christop he +carl sen +caf s +bu toh +bou gh +be stia +be back +bar men +ballinam allard +ball an +baby bump +ay ake +avail s +atay de +andre wyang +anci ens +absolu teradio +abo lism +ðŁijĨ ðŁı¼ +âļ Ĵï¸ı +âĺħ # +zak ka +za po +youth work +why iteach +whis ker +wh ib +west malling +wave guide +va hid +uni veristy +un listed +turn buckle +tren diest +the joe +tend encias +te pic +t anc +sp aw +sop ran +solym pic +so ss +sl und +sky divers +sip tu +shun suke +shar ding +sep r +sen corygardner +se dang +sci on +saafniyat sahivikas +sa hn +ru dolph +rin i +reen actments +re consideration +pat era +paper making +pa wb +p gn +or molu +nac ra +n cua +montre ux +mo zo +mis sn +metat rader +meet in +me morex +me mang +man am +maksi mc +lt col +low ry +louis theroux +longhorn network +lisac im +line smen +lesley ann +lef se +kis si +kar ras +kai muki +k ago +ire ton +iam sam +i eva +i apolitics +how lite +hoo oooo +hemis fair +hay maker +hantsi wwildlife +hal den +ha sso +granti mahara +ge würz +gar r +gal us +front court +follo back +flo rey +flamboy ance +fedor ov +fau vism +e somar +dur yo +dove cot +diver ts +devi ating +dela field +dal eville +cur seof +county show +comb atives +clo yd +chula vista +chi oggia +cel er +cappiel ow +canel ones +bre sse +bc ss +aus def +au dry +ation alism +athe art +assemb lye +are r +alber obello +ahmad abad +ðŁ¤ ¶ +âĺłï¸ı âĺłï¸ı +zo x +yeah buddy +wa heed +unfor gett +to ga +tin kered +team shabira +stre atham +ssy fy +shuk ri +shar ratt +seat ac +scottish open +saras ota +sai ki +s ÃŃ +s records +ru mania +ren unciation +pru d +pen land +pc engine +partici ple +out let +new ish +marcuslu ttrell +maker oom +macin nis +m ór +lux watch +luke mitchell +lt l +lock down +len zerheide +leaveno trace +lach hab +kri shi +korean air +knock out +khalee j +kab ira +k atti +jun ked +jeril ryan +jar lath +its ramimalek +har ms +greeng rocers +greatplaces only +gra inger +go ehring +gam esof +fluor ome +elec trum +ei ps +egom aniac +dying matters +dug outs +du se +du sable +di ox +de pose +dar ao +crore pati +concor dance +compra r +com passionately +co zad +chukku vellam +cer amide +cas sio +c ds +bro cks +brani ff +bour dais +blu hm +black en +bell woods +bell mare +battlea xe +bag sof +ath enians +astro tur +ast ilbe +arec a +aqu otes +abhin av +ðŁĮŀ # +ìĻĦë² ½ +é ī +youve got +x ue +wing rove +wil des +wal ston +w ths +vide oo +u od +tsun ami +transfer ase +trans dermal +thut mose +ther oes +tee spring +sul pice +su ica +sto day +sor ge +shan kman +resi ded +r fc +prat c +pontar dawe +planet labs +pentat onic +pen tath +par ola +paper art +pan handlers +outh waite +northumb ri +no bama +ne burg +mymt brain +multi plex +mor oka +min ia +mex ia +me theridge +masse ur +man tap +mad ley +love fest +light ner +lead belly +lc s +keh na +jö rg +itunes festival +inge urope +in red +ili ya +i strian +hu ard +hack saw +green economy +goo oooooo +gom me +fun and +fsg books +franci stown +fou lds +formu lad +elast omer +dr phil +de agles +cathe dra +cat mull +carval hal +bv barmy +bur gan +brain y +boothe el +bo cuse +bmc proteam +asi ya +arti kel +annou ce +an be +ac ase +? âłĢ +ðŁĵ ģ +ãģĦ ãģĦ +âĢĭ @ +н ов +Ì ģ +à · +yu ke +yakut sk +wur z +whatgetsyou outdoors +vali dator +under performance +tusk ers +treasure rs +together werise +thor ley +then at +th l +tamil nad +tal lison +ta affe +stie fel +ste ffi +speci alized +snapp er +sic amous +shoo kt +shari bu +sh moo +safi ya +rumin ating +rosie hw +reimbur sements +r news +r le +plant agenet +pizz arelli +pipe fish +per m +pav lo +pang arap +p enty +nowor never +nin iola +niche escapes +ni mr +new sw +neo sporin +ne wry +ne co +natural ness +morein common +moni fi +miley formmva +marche shour +mar vins +madilyn bailey +laure ano +lag wagon +l pb +ko ha +kassi ede +kan ade +k cm +ju la +j hump +international tigerday +iceland foods +human factors +hugh enden +hri day +hippoly te +hin ks +hel ene +gon line +geton board +george son +gay dar +g audi +fright night +ex ter +em z +ecur ity +dro sera +do tr +digital illustration +descen sion +deep veer +crickla de +con garee +collage art +clemson univ +change slives +centen ary +catastro phes +brac keto +bi gu +bar bad +an el +ai goo +acl festival +ðŁĺĤðŁ¤£ ðŁĺĤðŁ¤£ +âĿĦ âĿĦ +âľ ĺ +zar dly +word mark +wo ori +wight link +we care +way police +wakaflock a +upnorth live +un duly +tu thill +tri stana +tes ke +temu co +suffo cates +srilan kan +spor tac +si mul +si damo +red flag +re marking +pump in +pu issance +psychop athology +pro tos +ph h +peter ock +passage ways +participat ory +pan tano +ob on +o lot +o gether +non u +no hep +ner ney +myeong dong +my haver +mountain west +min nick +mil ow +mee totaku +md lz +manicure monday +man tr +mag or +ma dad +ll ant +len inist +lan gham +kom u +killthe bill +katy perry +jet fire +jad av +ire t +iff co +hor ic +hel ston +glass jaw +gewürz traminer +gar gi +g te +fe a +favorite things +fang asm +f wr +elk grove +elic ited +ehlersdan lossyndrome +e ade +dy fed +conco cting +clay face +chronic led +chennai yin +char coal +book recommendations +bish noi +billing shurst +bene dum +bello whead +beck oned +ban ka +bal ancer +ba ju +ayyy e +av ill +aug ments +asi atique +am mar +adopta shelter +a ines +________ ____ +ðŁijĩðŁijĩðŁijĩðŁijĩ ðŁijĩðŁijĩðŁijĩðŁijĩ +ðŁİīðŁİ ĵ +ðŁİĤ ðŁį° +ت ÙĨ +wom ent +wine week +whitt led +wal ay +ve ssel +ve iw +val lur +un did +ule scu +trun cation +tortu guero +thermo forming +tat ties +take part +tac itus +sus annah +superstar dom +stor z +ste ggles +standardi zing +st com +srikak ulam +soko lov +sli abh +shin sky +scri abin +schae ffler +salud tues +s dut +s ational +roh rabacher +ro zon +ritch son +relax ant +penet angui +peat free +peach ey +par sa +palimp sest +page boy +outri der +old castle +oil fields +nw m +nh h +ne we +nal in +n swe +my ung +mot lanthe +mor ley +missi bility +mini bike +milin kovic +metabol ic +mel zer +manga art +mac queen +m schat +lu ster +live it +li ket +leh tonen +l tw +ko lej +kk kon +ked die +jo kinen +it ep +irish food +il minster +iklan bandel +i mid +hom unculus +hin kie +h pb +glen roy +gir li +game keepers +g itation +fo scar +felly chibi +duke mbb +du err +doublec lick +docu drama +do ko +death fest +de positions +de activation +dab bler +cp bl +cover taffairs +corri do +complex mag +cleanair day +cas kets +c cleaner +bushwack ers +bolo gn +boiler football +bo rea +blunder buss +blooddonor day +bi bury +bhavi or +bb onday +barn ham +barking side +ba stin +at eliers +an ata +am bis +âĺ¢ ï¸ı +à¶ Ń +س ÙĨ +Î » +wr angell +wmn news +waterlo oroad +war iner +un american +u jung +u baldo +tro ver +transc ranial +tran sunion +tor on +to history +the quint +the fire +tearitup bts +ta vy +t ldr +sta ed +sig ny +shin nie +secondam endment +se ssa +sch lad +sav iler +sav alas +sacher torte +sac onf +s fan +run rocknroll +ru dis +rit chi +rep brian +re vine +publi k +ple be +pi est +pa ide +original music +oliv arez +og ba +o ases +nun thorpe +nincomp oop +murali tharan +mu ahaha +milk wood +mic rob +mc kidd +mc clair +mad awas +ly ford +ludo vic +lec tronic +la ppe +kno l +kim mi +killy begs +keving ates +kam ran +kaha pon +ji moh +james ra +inst ants +imper io +illy ria +i vette +hook land +home biz +hei den +hans raj +han ish +guerre iro +gary player +fox e +fl ach +ferra gam +felic ita +fas i +ex other +epi dermal +duc kies +dragon fire +din iz +delaha ye +david tutera +d illy +cu pido +coupon code +cou loir +clau dy +chi a +cdn muni +caste ism +bur ano +bonifac emwangi +bb tag +bar se +b hola +av ait +autau ga +au pt +apur va +ane choic +an sip +al chemists +adul ted +< : +ðŁĺİ ðŁĺı +æ¥ ½ +à¸ŃภĶ +Î ² +zel ina +zaven tem +yar r +wil banks +why tele +wapis kat +vs ner +ven eered +vel achery +v ul +usc g +u vs +tw ich +traut man +tran sept +ti meee +ti deswell +te knik +t mobi +super ia +stone walling +stig ler +ste iff +star field +stam baugh +spar red +spac et +sp ack +sou fri +sof l +sing lish +shi mane +sheryl sandberg +shar mil +shadow land +sha hani +roo tes +resonance fm +power sports +pern ille +paramilit aries +oc tor +o ger +nullar bor +nu groho +nor val +no ton +never where +n cra +mu zzled +mous y +mal um +ly se +loubout ins +light water +kentuc kian +kapp kvew +jake canuso +jah ren +is overparty +indi anidol +imagin ation +ic cc +i hi +hat sune +hasan minhaj +ha xe +gou d +gat chalian +fox wood +fight like +excep tional +eti os +en demol +cut work +cogn itively +clo ven +cine ws +christ ofer +chic est +chan ute +cb live +can ley +by bee +bun che +blu shed +bla si +bille vans +bi erman +beyond borders +beren ger +bar ad +back firing +audi rs +asur ya +as mussen +anastaci afan +an jum +aly goni +alexis dejoria +ìĩ ¼ +âı ² +اÙĦ ÙĪ +Ë Ļ +za popan +yugosla vian +wol i +whatwomen want +wei de +we hi +var ney +use ums +ure port +universit at +tu en +tu bo +trous dale +trans genders +town sfolk +there venant +thene we +the kings +the dog +ta illon +ta ff +swee eeet +sunny d +su gg +spu bli +sport scar +sp indler +snor kels +see scandies +scu mper +sc ult +river ford +ret tig +real bobby +re eth +pri yas +pr newswire +pl sd +paul vandyk +paestu m +nois ily +ni ve +natali ec +nar ia +mondaynight football +meso america +mcfar lin +man down +ma ari +lu sby +lu key +lochal sh +lo keren +leve que +la il +kron ik +krat er +king sheath +kil beggan +khe ir +katar zyna +jag an +ini quities +iglesianicri sto +house boats +hitt ite +hin ny +hen ninger +he men +hc so +ham ley +grimac ing +giall orossi +g mac +fur qan +fre dro +fl ds +fin eness +fear th +fail over +fa ile +eth nom +er sten +entin el +eng ale +en ak +edmund ston +edmun dmc +east brook +dj k +disco ve +dev fest +deli ve +cyan ins +cro kinole +cookier un +conco ct +comman deer +co fo +cl ines +chrisl illey +chaun cy +big al +bhu pinder +bc f +bar rela +app all +anton ello +an us +ala ine +al gor +ag ena +ad du +ðŁĺī ! +ðŁı · +ðŁĩ«ðŁĩ ¯ +าภ¡ +you togive +y ine +wer ker +voteenrique fpp +ven de +vel lir +uon bi +u mut +tragic ally +tho re +thisplace matters +the dukeof +tdamerit rade +tanz ani +tan credi +syste mically +syndic ate +surrep titiously +supp l +stone m +ssi an +spitt sburgh +so al +si ame +ser am +sco vel +s made +ru pe +rt dna +rope way +ro gie +river cruise +repos ado +re blogged +raffa ello +poly clinic +pickle back +open democracy +oldro yd +ofor igin +nor berto +ni mi +neu wirth +net working +na ac +moon lite +moog musicinc +micron auts +mc gowan +may en +mat chett +margare torr +mar te +magne sia +liquid ator +likeli ke +lady bird +la fer +korn field +ki ger +ka ay +ka abi +k ount +instaf rame +indoctrin ating +ilaiyar aaja +ideolo gue +i star +hel eng +hallo f +gwen pool +gonebut neverforgotten +go bu +gly pto +fit zy +fil omena +fe vered +escul tura +duplic itous +dramatur gy +drag strip +dit zy +dev ine +den nie +demo te +defen sa +davis ville +cre use +conden ast +ck ert +city break +ciel ito +chi leans +caterin atweets +cas ady +car ai +bun du +boot legged +back thursday +axi ata +aw adh +auto immune +as af +anton ino +ah ti +adoptdont buy +ac tes +absin th +/ * +ðŁĺĦ @ +ðŁĺĤðŁĺĤðŁĺĤ " +ðŁIJ Ĩ +íĥľ ìĸij +ಠľ +à¤ľ न +ydr com +xx xiii +x wing +whit en +we ssmith +vox els +vote eff +victi misation +van ney +uper man +un truth +tu si +towel day +tom ey +to bia +timeout newyork +ti bbles +thr ought +thebig show +the point +than son +tell ico +tar onegerton +stand asone +ss ahluwali +ssahluwali amp +sports guy +si skel +shrie ks +semrush chat +semin chin +scoo ks +ro len +requ in +rapid kl +rand fish +quad ra +personalized learning +pear ce +pa ji +ott oman +ot ara +omer ase +oli ka +oko th +ob y +nisargad atta +ni est +nanta hala +nag ul +myel itis +my til +mur f +mor rilton +mis smo +mg sv +mg mt +mcel wee +mcdonald suk +may bach +matti seman +man souri +loui seminchin +london fashionwk +lef tie +le chner +kw qc +ko ho +kai lee +jor is +jo dies +jel inek +ist ure +inno cen +in nnn +impro bab +hyper visor +htown rush +how you +hoop shabit +hitman hatton +h fi +f gl +educ ación +earth watch +dr mark +cy games +coun tess +cospla yer +co if +ch ough +c tweets +buen viernes +bryan habana +bar ao +b go +ax tell +andro scoggin +am ud +a kee +ðŁĵ ¬ +Ú ij +zz ese +yaku tia +würz burg +whoo ped +v mx +un shaken +ume ed +tubeli ght +tre mper +the monkees +sweet briar +svend sen +suyyash rai +ste ilac +steilac oom +stay ton +slu mp +simple things +simon rim +showoff bydesign +sho shan +sea houses +scott brown +scari fication +sc udder +sargas so +s gameday +ru bias +reli ved +pupu sas +possi ble +pliss ken +pedo bear +pas sy +our girl +ok tober +ok cps +nit schke +neonicotin oids +mo hom +mira bilis +mat lock +mario goetze +let d +la dles +ks fa +kefla vik +just is +jon jo +jason segel +it ro +ir regulars +io va +ins angu +im pe +hu lett +host elling +hoo ter +hof stadter +gro b +globe debate +gent ly +g anger +fo sco +esche w +elm endorf +eco logical +dl na +dil ate +desp ina +de constructs +dar denne +cover crop +cor ine +comp ilers +colo s +co habiting +clam bering +cin da +christoph ers +cal away +burn sy +buech ner +bu in +broy hill +bro oo +bom ani +blacke yed +beech mont +be sse +ba ena +at mega +ash kelon +as j +an ual +a ati +ðŁĶ¸ ðŁĶ¸ +ðŁĴķ " +ðŁĩ§ðŁĩ ¾ +ðŁ¤ŀ ðŁı¼ +yi wu +wil sey +what up +warrior games +wa al +w vt +thereal dcf +the process +su see +spe ight +spar tina +sick nesses +schar pling +sar ab +root sof +regur gitation +read aloud +pun ti +pro kop +pres byo +polar ising +po kiri +pli mpton +plas mas +pla i +phon ological +penetangui shene +pashupatin ath +organic gardening +omon sters +oh ba +nimb in +nak agawa +mish al +me dy +marcou x +man sky +mal iciously +mal dita +make theroad +mac found +lon min +lipsy nc +le toya +kun ze +ku us +ker an +jack black +ja sen +io dide +ing cancer +ind sey +hydro foil +hoyas axa +hottie oftheweek +hom inem +hollywoodun dead +hawkes bay +har ner +h gf +gr itters +ger t +fp gas +foo dy +fan cams +exeter cathedral +evangel ic +euro hockey +en ve +elou ise +dul fer +du kie +dis connect +det ente +dele k +defe ctions +come shome +col ons +chatto padhyay +beyond blue +bec q +baby center +ay as +aspir ing +ari stop +apollin aire +ðŁĴŀðŁĴŀ ðŁĴŀðŁĴŀ +ðŁİīðŁİĬ ðŁİīðŁİĬ +ðŁİĢ ðŁĴķ +âĸ¶ âĸ¶âĸ¶ +ï c +yorks biz +y aiba +where thecar +wherethecar sare +villa ggio +u iz +tin ian +thru pp +thr ou +the fan +tar kenton +street photographer +stin c +ste iger +son news +soc med +scy thian +sat in +rusty wallace +rp ms +rox eter +resul tado +quo gue +qu ne +q af +pulver ized +poloni ex +part in +pa wer +on vif +on k +not ch +nj ie +new tek +n pratc +my yyy +muham madi +men ke +mar na +manipu lators +mag at +love is +lc ss +lau ding +lanvin official +kab at +jy vä +just blaze +jo bar +je sters +jan ai +inthe us +inst ax +i ben +hil le +he cking +hapha zardly +gay ton +gam bar +far th +fad al +eric holder +duck weed +dru silla +do komi +deci r +d gh +ct fu +critic ality +ci ber +ch ando +cd q +care homes +car others +c icc +basti at +autorick shaw +at olls +and ymurray +am ang +al mera +al dea +ðŁĺī ðŁĺģ +w bi +var itek +up ham +tv sn +turn age +to iled +thorn ham +the open +the agenda +tan lines +superse des +su cr +staf fing +spe di +spac ial +south westerly +smooth jazz +sharethe lex +senior itis +sb vb +sand ero +ring send +recap turing +re xit +re ids +quie ted +pushawardsteam kisses +pu entes +process ing +por ttal +pop crush +pir u +peter ptur +per illo +pat summitt +ous music +or ani +new line +mordi alloc +mo hi +mo barak +mit ron +min k +merri mac +mercury theatre +memorab ili +mar la +mac n +lm x +llu ll +lis ap +lan olin +lac our +l fd +kuns thistor +ko kesh +kate ys +jon no +jeric ho +janegoodall inst +jal on +jae suk +ic entre +hus sein +hul lah +harsh ini +happy day +hai ro +h nl +gen co +g apping +friends notfood +er len +efe cto +do wels +dit mars +despo tism +dab rowski +clair ton +cel g +car on +car hart +cal ums +bu ang +brum field +brit birdlovers +bristol baby +bbc womanshour +barbas ol +bag man +back bencher +as uk +anti dotes +ann at +anarch y +anandiben patel +am ola +agh olor +acro wn +!! ( +! ¡ +ðŁĻĪ # +âĿĦï¸ıâĿĦï¸ı âĿĦï¸ıâĿĦï¸ı +âĺĢâĺĢ âĺĢ +zoni bali +white tails +vincen zonibali +vas sy +upadhy aya +tu ya +tre dge +trave le +to dor +time z +technology news +sustainable tourism +surfact ants +sun it +strang les +shor tridge +shelar ashish +she a +sf ile +say no +sair lines +rudhram adevi +rob schneider +ri ffin +reyk jav +resusc itated +ra heel +publici zing +pin aka +peter s +obstin ate +nt fs +mix ing +mcgon igle +mc girt +mad ball +lydi ate +loy e +low der +lex y +len o +lauren goodger +kah lon +k nd +jum illa +ju mu +jaye gi +jamie whincup +inten tional +ima x +icom edy +hondac lassic +hh b +haus man +gw and +gun gor +ger mline +gaz idis +gal ad +fr st +for fun +fit girls +fish hook +exped iting +ev t +eur jpy +eryn gium +enough said +dre ll +dispro ves +ctu local +conf rence +co pra +cath ao +car olo +cann ington +c agu +break beats +brah mas +bowdoin college +bor onia +bo boli +ber nas +baw se +bate mans +bas otho +barcel one +bailey m +an andi +alb ina +affe y +ac ares +zi zou +y stem +wood cutter +william byron +west fjords +wal b +wainsco ting +ver dure +vac as +tony dungy +toly atti +toku shima +thermopy lae +tambu wal +sushmit adev +sushmitadev mp +sto hl +stag es +sick bed +shri vastava +shakespe arian +se sion +school bus +sa ille +ru disha +remedi ate +re traces +re appearing +rally australia +pu recork +poke mont +po thos +play girl +pigeon hole +out num +oli gon +ol itz +no vis +nit z +ni f +myo b +mpl s +mech warrior +mccar ver +marypopp ins +mana hawkin +ley house +leve rett +kuch h +ker ber +k mr +jo wett +jeff vandermeer +jare tt +jack sock +iter ating +inf x +inci dent +imbu e +huach uca +ho ta +he fe +google fiber +glen burn +gets old +gabri ella +fresno bee +fra se +fire department +fahri ye +f bw +exten ded +est á +er ay +e ser +duv vad +drais lv +do dig +dev co +de ak +dam pens +dam ia +cryp tids +cra dley +cham bal +ce dis +carou sel +cab g +ben hur +be ika +attrac tant +as police +are alestate +apple store +anastaciafan ily +an shuman +an op +al cide +ðŁĴª âĿ¤ï¸ı +y thing +ww fcofficial +why so +wapp reciationday +wag gin +veronic amerrell +veg fest +us apro +under story +u ha +trenton thunder +traf ine +ti med +thir roul +theatre r +teachers matter +tar boro +sym metra +sylla bic +sylla bi +swin ner +sw afford +suk hi +sper anza +snow patrol +sheridan smith +sak aguchi +s suk +s birthday +rigu eur +regg a +reach out +re issuing +psych today +ps bhumi +play boys +pirates fb +pe vsner +pav itra +parth asar +orac les +ny m +no cent +narcole ptic +name sakes +mon ken +mol vi +meur ice +massey hall +mary ville +mani festive +ly lm +le sufi +lal wani +kassiede paiva +jyo ts +jul lie +ji raiya +j ato +insol ence +imitation game +i xi +houston ian +ho ja +hick ling +hash d +harrison ville +har gis +h sh +h frs +gil bane +friend zoned +fam ke +esguer rat +equi pedefrance +el neny +e boue +disney store +di ero +denver broncos +deca inc +dat du +cu bby +coo kislands +car ri +capp elli +bro in +brewery ommegang +br acy +bor gat +book marked +boi vin +bo tv +bo hu +band itos +back fill +am erie +ablu tion +abdic ated +aar mstrong +. // +' * +ðŁĵ ĵ +ðŁĵ ¯ +ðĿĻ ¤ +Ñģп оÑĢÑĤ +z brush +ye are +wur tz +win ry +weekend warrior +we missyou +viñ ales +vent ress +vait la +un bundling +un boun +thu an +tho w +the leftovers +the bath +th ope +th ain +texas childrens +ten sing +tb one +tages spiegel +stre k +spi rou +sp atz +soooo on +sen whitehouse +semp iter +sch amps +sales woman +rober talai +revdr barber +radio humberside +ra himi +pu pusa +pro mazda +pied montese +pay oneer +pal mb +ouag adou +or dic +obam as +nor um +nor mmacdonald +nationaltree week +narrati ve +murphys boro +mulat to +min day +librarian ship +len na +leish mani +le mus +lauri ston +lan ta +kork maz +kim yoojung +kha chanov +keesh ond +jan ki +j po +in nu +ilo gy +hun ni +ho stile +har ling +giri raj +gior ni +gener is +gel nails +fr ronconi +fore shadows +first love +fair tex +fa er +dre ds +disappro ved +culver house +cillian murphy +ch x +cf trust +carpath ia +call anan +bsn l +blu ecar +bis wa +benic assim +bath ong +bal bo +aw alla +app ea +an kers +accom pli +ac ali +a art +________ ______ +. ðŁĴĭ +. âĿ¤ï¸ı +ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ ðŁĺŃðŁĺŃðŁĺŃðŁĺŃðŁĺŃ +ðŁĺĶ . +ãĤ¤ãĥ ī +âĿ¤ ! +worldre sources +wood cote +winter burn +wild hearts +wh attt +wan ka +vacation er +us marinecorps +un ac +u alr +toy drive +tom mies +thisisla fferty +teter boro +tali aferro +susanc alman +stor yo +steel city +ss di +solu te +sm ount +sig al +se ssler +se pak +rou le +recru te +re awaken +ran ald +ram nath +q ra +prie ster +phil vassar +pender yn +parame shwara +par ram +p ku +national ballet +muld row +mor bi +miniature art +mat amata +man ute +malak and +makin de +lucifer ian +ld d +kun du +kil led +jove tic +jack sboro +j é +iro ha +inven tiveness +inter school +ichi bi +ic helle +i ara +hex agon +hemlock grove +grand hyatt +get creative +fur uya +fre on +fli ppa +finsbury park +fangasm spn +evil doers +eu v +ebolare sponse +dj ay +depar tement +delas oul +dar low +cu boid +cristin avee +cen kuy +bri el +bou ma +bo sn +ban tu +bal og +an vils +allevi ated +addic tedto +absurd ities +ab ida +ðŁĺī âĿ¤ +ðŁĵ° | +ìĻĦë²½ íķľ +ê°ķ ìĬ¹ìľ¤ +zi ther +x seed +x imab +wwe uk +wit chy +win eland +wake hurst +und ine +try it +transforming lives +the visualart +t lim +stock pot +sony six +somerset levels +skysports boxing +shri vel +ser dar +sampal oc +s deep +rwc md +rudi ments +rosehill gardens +respon s +repp aul +re mp +re marriage +ram asamy +qui vira +propor tionally +pom pton +pil ote +op aline +objec tify +ny k +ni hari +nett le +nam er +nafp lio +murry sville +mul vi +mon tara +moanal ua +michelin tyres +mic hale +mi rella +metta worldpeace +mc cc +mb ury +matsu ura +mate i +maic ha +loy ola +limkok wing +len n +la quan +l senews +l bm +kro mer +kpor zee +karolin ska +jim norton +je ffe +hepha estus +gu ate +green law +gre er +g anc +fur o +foam posites +fil mes +fh su +fat in +far ge +fa shi +f km +esguerrat ommy +equ alizing +ear les +discord ant +crou cher +cook sey +con quests +commensur ate +cole engarcia +col son +charlo tt +burgun dy +broad field +bre th +bray shaw +bli ssett +bha sh +ben lovejoy +bb nai +barn sdall +atar axia +as oftball +aq w +antho cyanins +ðŁĴħ ðŁı¼ +ðŁıĪ âĿ¤ï¸ı +æ© ĭ +y uni +y il +wolf hall +whow ould +water proofed +visit ations +ver dant +universityof ky +un enthusiastic +ty b +trilli um +traverse city +travel tech +ton o +the score +tar da +sth elife +steve kubota +ste wa +sm older +simon helberg +sil sila +shor tie +seattle pd +salis bur +ri ho +rha bdom +renov ator +rene a +reggie watts +real bob +pu zo +produ c +power ups +ore k +on elxn +off ootball +o al +new track +ne te +naz em +music brainz +mun ira +mu tha +movie podsquad +moto guzzi +moon day +monte casino +mo thi +mindless bhavior +medi ates +manse hra +man sarovar +man ce +lydi ard +live well +lincoln wood +less a +les bury +lee z +led low +le evalley +laver ton +kle m +kaw ana +jan ela +jae hn +instap rnts +indianc inema +indian art +in in +ici zed +hosse in +histam ines +help dogs +hant s +hahahah hahaha +ha dee +green team +green slade +gno stic +gc titans +gaming life +ga es +far yab +f grfc +elç insangu +el off +eat my +dy ker +dou we +democrat shate +daver ayner +daverayner fund +danger ou +contra ven +clu bby +cityo fl +che tbaker +cb n +book binder +black magic +bbc newsline +bad ulla +b ici +av la +aubu sson +atl v +ashi m +ash mont +apu estas +appell ate +apalach ee +am ade +ag morethanever +abduc tors +^____ ^ ++ ... +ðŁĺ¡ðŁĺ¡ ðŁĺ¡ðŁĺ¡ +ðŁĸ į +ðŁĴĥ # +ðŁ¤Ĺ ðŁĺį +ìĸ´ ëĶĶìĹIJ +ãĤ»ãĥ¼ãĥ© ãĥ¼ãĥł +ØŃ ÙĦ +× ķ× +zhu kov +yerush alay +waz iri +ver such +tü bingen +twitter artexhibit +turtle dove +true bloodh +truebloodh bo +tri sha +thereal jrsmith +thau low +th ato +sydney airport +sp azz +societe generale +sling erland +selec tronics +se ige +san marino +run withthe +ros sett +ro dolph +ric an +respir ator +re probate +ra issa +r pk +qua hog +qu ite +pub chem +pr annoy +ponty clun +per tain +pepp as +pe ons +paw lak +ox shott +ou trunning +ol ver +ny asa +new burn +nag ur +muck ross +mor rendo +min et +mikul ski +meso american +melbourne vixens +medce zir +mar ilou +ma kk +leve tt +l gas +kyo ani +kun in +ko zik +kingston ian +ki bbe +jur upa +ii e +if not +i wo +hur acán +hor ne +home ground +hedge fund +gg d +genna io +gameof th +full set +forsy th +fo pp +fic ou +dre ddy +de conge +dance mom +dam pier +dad dio +cubam in +cr inoid +cityof hope +certi fiable +cenkuy gur +castell defels +casey veggies +cas sata +bupre nor +bru ces +bee bo +bay way +azzur ra +avi dly +amust fall +am cs +al uk +advo cat +adam antium +aby ad +ðŁĴ¥ ðŁĶ« +ðŁIJ¯ ðŁıĢ +ðŁ¥ĩ ðŁ¥Ī +ÙĪÙĬ ت +year so +we dem +w tw +vi dhan +us ick +unemploy able +un fail +ultr at +uconn nation +truth iness +tri maran +tr icon +sylac auga +swain wright +suman th +ston em +stereo typically +slam dunk +si yab +shoe making +shekhar kapur +screw vala +sau v +sar os +sal pointe +saffron walden +saar brücken +rosar ito +rho donite +q z +presiden cial +pre empt +por ritt +phen olic +over indulgence +oren da +onye ka +only one +never again +mutil ate +mox ey +moon shine +ming in +meh sud +mc coll +mari otti +mali ks +logi o +lo sf +life ison +length ened +l sx +knau ss +karak achat +k wal +k enda +joann ak +japan trip +inter laced +innocent drinks +harpur hey +han ko +gwan gh +gram mes +gener alize +gan jam +fran ka +fon da +fau gheen +f one +f cau +etu i +el rich +druck mann +dhru v +dav ao +dance team +danc elife +d poy +copper heads +chit arra +buff a +bronz ino +beat itude +baz a +ball gown +as tha +ann g +angel amer +am po +all anguage +(âī§ âĸ½âī¦) +ðŁĻĪ âĿ¤ +ðŁIJ¾ ðŁĴķ +ðŁ¤ ´ +ê¹ĢíĺĦ ì¤ij +âľ İ +мÑĥ з +zay nab +yassi r +wave form +wa hili +verbo ten +ve ery +ur anger +um ji +u stour +tur riff +trek kies +tibur ones +tal las +syri acivil +syno p +su vi +skin z +she rer +sf pride +sa wal +run dell +renault sport +randomhouse kids +quater mass +po liquin +pink print +pe ch +panch ito +os agie +ole ta +ol les +nup ur +north wick +no logies +ni dhi +nathan carter +nag ach +mortalkombat x +min nie +me vani +maki shima +lorett alynch +london bookfair +loc ked +livel iest +len a +knoll wood +kal ert +kal en +k ach +ju ss +joker to +jo sc +int ent +imacelebrit yau +home star +hip life +hier ophant +grist mill +green acre +good smile +go yette +gee khour +ge fil +fath ima +fairmon thotels +fa ar +ev onne +ero v +ener d +donnell an +dissolvethe union +desi rability +deep water +cubamin rex +crusad ers +county pd +cor red +congru ence +confis cates +comicboo khour +choco holic +children sphila +char man +changing places +cap leton +camp life +call inan +bunny ranch +bro mbor +bou chet +boom ed +bom mel +autonom ou +as ra +anton ina +alex fromtarget +. ðŁĴ¯ +ðŁĴĥðŁı» ðŁĴĥðŁı» +ðŁİĻ @ +ðŁĩ³ðŁĩ µ +ðĿĺ °ðĿĺ +ìĹĨ ìĿĦ +ห à¸į +ye tti +ye aaaa +yarra valley +wor le +vel ox +usav mex +twof old +true islam +thewine society +the strokes +stress less +slu twalk +skyl ä +ski ve +seraf ini +rice gum +refrac tometer +red light +pra geru +positive psychology +po bre +piri formis +pink day +pet worth +pac zki +nut meg +nur singh +nsw fires +neglec tful +natalie ben +n gen +med students +mc nicholas +mar ske +man gus +ku news +kou ki +kh r +kedle ston +karolinska inst +ju icio +jimene z +icy cling +hhhh hhhhhhh +hernan dez +gom ustangs +gol fing +gla k +gabbi adini +fu rence +flugel horn +fang ed +face thenation +españ ola +epi k +eca illat +dream hack +divisi e +dis loyalty +detroit lions +del mundo +de ine +daniel sahyounie +copy writers +comeu ppance +colbi ecaillat +col wyn +citi c +christ ingle +chen ault +chau rasia +chand ran +cannabis cup +caldic ott +blu elive +block aded +berry ville +ban kai +arri vat +ang k +alma da +ðŁĹ ¡ï¸ı +ðŁ¤ŀ ðŁı» +îIJ ij +ìķĦìĿ´ ìľł +ಠł +zaz ie +yir uma +womens institute +willi mantic +weak ley +we ser +vi goda +vern or +un dee +ugly dolls +u rad +u dot +town speople +tic at +thor st +thisis robthomas +there ef +t ck +t ague +stream fakel +ski ddle +silicon valley +si os +shin se +sc aife +sas sen +san vers +san sar +salati ga +ren at +refin ed +re ffing +re considers +rain man +po quo +po ley +pierce brosnan +pe adar +pay al +partiti oned +panther sihc +o kun +ny xl +new adult +neg ar +nat araj +murad ali +monoc acy +mo sphere +mel bur +mar gie +mand ra +m jordan +li diab +la prairie +kou rou +kan go +jaf far +j bl +il ri +il itch +hot list +hollywoo dimprov +hobo ken +heaven onearth +he eney +happy dance +h fe +gra ying +glo ball +gau ahar +fram ed +fire trap +far r +fanta sticks +fant in +fanarmy faceoff +fac cio +extro verted +em ption +elast ica +east wood +do ku +david cicilline +da a +current mood +cur lews +cross bill +conver gys +color block +cl ande +chi rac +catat onic +blind folds +bishop briggs +bis now +bertel smann +bas ma +bas f +b lear +amey aw +' ..... +ìĸ´ëĶĶìĹIJ ëıĦ +ê tre +yar der +weare nigeriancreatives +watch roh +wal ser +vibr anium +ungovern able +transparen cies +tit p +there se +te ela +tam ako +ta fari +sy st +stai the +sol man +she iks +shadow bringers +sb ca +say d +sav ina +sar ch +san marcos +san day +robri ggle +ridge dale +redbul luk +real z +rc z +radiom irchi +punch drunk +pro circuit +pri ddis +pretty man +pre me +power outage +paul sboro +pan tin +olivi as +obe id +neil n +naj era +mu jh +monarch sway +mizzou football +mir choff +milli metre +men elik +mc daag +max y +ma ppin +ma aan +lu ffa +long stre +linde mans +lik as +learn with +lan dic +kom odo +kim ya +jer ram +ja q +its better +intermitt ent +interlo chen +in ox +hor nell +hoo ten +hallo weekend +gen ious +gad di +ga hd +g ft +fun com +ful ls +feder line +ense mble +elli es +dwar fing +dr at +down played +dji phantom +dis lodged +dhive hi +dary ll +d marc +cow li +co sum +christinam ilian +cho ong +catalo ged +busines splan +bru mmer +bridge point +bor us +ber mejo +bel gica +angio genesis +amre zy +amp oule +am dra +albino kid +albari ño +al fieri +ad akar +abre wer +aaaa aaah +ðŁij£ ðŁij£ +ðĿIJ Ħ +ãģ¾ ãģĻ +zap atista +wu du +wit mer +wildlife bcn +wes sun +voy aging +v arez +ur anga +turquo ise +tr bam +tom chaplin +th appa +sw apan +stra tham +star music +soul calibur +silvers mithing +side of +sg ill +schem bri +s ative +s alla +russia invade +run up +ru ang +rob sten +ro so +recl ines +re os +rat emy +pro core +phoenix comicon +pen kridge +passi vity +pa ap +over reacted +oven ow +ot aru +on das +om exico +ohio an +official tf +od cast +nu die +neutro phil +neder lands +nd wbb +nay ef +morning drive +mi festival +mer itor +mc mee +max thieriot +m fy +la der +ki dero +k btx +jewelry making +ive ta +iri an +inver gordon +indi ranagar +in calculable +in aki +im pasto +i ur +i il +harrass ment +green peac +god frey +ger ome +ge ti +fibro blasts +farahkhan ali +fal aise +f z +duc o +document a +dc cheerleaders +colling e +cha har +ch abby +cell c +celi e +candy crush +cancel o +burgen land +bi anch +bhil wara +bbc southeast +bb waa +avger opoulos +ath enee +ander ssen +ambassad orial +amb am +ack y +abur ke +ab om +a amar +# * +ðŁĩ ´ +é ł +wor land +wool pack +vital is +valenci agp +us ila +ur anium +under berg +trop fest +trol ly +titch well +thunder cloud +ther ma +th acher +te jo +tan trum +ta kat +strib pol +stress or +snu fkin +shy ama +she an +shak un +sco delario +san mateo +ru shall +round ness +ri baj +revol ttv +rain forest +r ê +r vb +q au +pu tti +pontific ate +pit tura +pilla ged +patrick mahomes +o dori +ni was +mout inho +mott macdonald +mm s +min il +middle march +mi sha +mccam mon +mc z +marath wada +lil jon +le pers +koo koo +kin zinger +kid scan +karen kilgariff +k fcb +just asking +jar rar +hu ffer +holler ado +hersh iser +he le +h icky +gru dgingly +grouse mountain +gom pers +go pirates +gamep ass +fore father +fo gler +f forde +ex el +ep ad +enamor ado +en closing +effe l +ed leadership +e bikes +du ston +dipikak akar +d acs +cly dach +cho desh +chit tor +changi airport +ce men +card captor +brun ing +br cc +black hills +bench marked +bay side +bar tek +anthony anderson +ann marie +am rinder +ag y +ab era +^^ ~ +ðŁĻĭ ðŁĻĭ +ðŁĺŀ ðŁĺŀðŁĺŀ +ðŁĮ¸ ðŁįĥ +ðĿŶ ðĿĹ +âĢ¢Ì ħ +yerushalay im +yah an +wu c +will friedle +vau dre +usa o +unice fusa +u las +tony romo +to des +ti memor +team blake +tax season +tatasteel chess +supercar sunday +ss ential +soc i +so kka +so den +sig nofthe +shum ate +shoe bury +senso dyne +science matters +san ge +saf ood +respectthe water +refra ining +real kid +queen victoria +que ijo +propri ety +prisc ila +plain clothes +per vious +penn sylvanian +par od +pag es +pac to +over stayed +off ilth +o tec +non discrimination +nex tofficial +ner issa +nat sec +myal gice +mo wa +mir tha +mi w +medi amar +marth apli +marthapli mpton +m vo +lun de +lost prophets +lo pp +little miss +lille shall +kw am +kin dy +j inga +it chio +im n +hol lowing +hello goodbye +heat seekers +g loc +fashi o +fac tional +exhal ed +eric as +ear ch +doo ds +dev secops +deth klok +de pa +de form +da hm +chu gh +christ on +chaus sures +chak ri +cel aya +can ario +br ind +black country +bilo deau +bick ell +bi frost +beau té +b nhs +az ee +ay am +aug gie +aper ol +apas coe +anti ka +anti bully +ambi en +adam driver +% ." +ðŁĴİ âľ¨ +white cube +whit erock +war u +wa sten +use lessness +uc g +u cr +tier sen +sted chat +staun chly +stargate atlantis +stal la +som ma +solympic team +solidar ityday +socialist sunday +snow watch +sker k +sin t +si mage +sfor trump +sf v +sch ack +s ro +rosanne cash +return able +ra vitch +r gu +pu cke +pro scen +poign ancy +philipp ullman +perio dization +or lan +oo bleck +naz es +monk ton +mogo eng +melissa and +march foreurope +mad dog +luc ille +lim oux +lie sel +lan dex +kt k +juli as +journ ée +joe hockey +jeff burton +jav aid +interdisciplin arity +inteli gence +hon ma +heriot wat +ha eng +gwangh wam +good lad +gin apu +gif tever +gavi ota +frequ enting +fore shortening +food ways +extran eous +ex ti +evangelic alism +ers out +ern st +emc coy +elle schi +ear muff +dougi emcfly +diag ram +de splat +dar laston +d welt +creature design +cran more +cos mas +cor ia +congress i +con all +ci am +ce ma +cbc murdoch +categor ise +c wo +bo we +bir die +bettym white +b anti +as pac +aldubang pag +al music +afric at +ðŁ¤¡ ðŁ¤¡ +âŃIJï¸ı # +âı ¬ +worth itv +worthitv ma +wil ander +way towork +wal ing +w co +undere mployment +tyranno saur +tv k +thorn wood +thorn cliffe +thisiswhywe play +subl ingual +streamfakel ovenow +ss chool +sister s +simp lot +signi fied +sent a +se let +sd beer +school sout +sb n +sal tier +sag ay +sa hid +s alli +rosen ber +roma downey +ro sti +reven ants +raven loft +r maf +pk mn +pin ako +pepper pot +p drm +ok un +ok r +odd ness +oa week +o gaden +ny an +now ness +nca afootball +nab arro +n ks +melis sac +manoj sinhabjp +m jo +lyn don +lumb ering +lot ter +las key +kstate mbb +kooten ai +ko sarin +kiri baku +keik amara +kail i +josel ito +jiha dism +inter related +im rie +ice breakers +hu gel +hit oshi +hei sel +guic hard +go knight +go en +go ba +glen arm +flye st +ever r +eric bellinger +equalit ymc +endof theworld +electionswith ht +doren bos +do vo +do besity +disin terest +diferen tes +di lett +dam ming +cun diff +club america +chi pping +che mbur +chapar ro +cel cius +car lock +can ham +calamit ous +book tube +blan ches +bhu tia +ben zie +bar gen +audra equalitymc +attenti on +arte k +ar kit +amdra deon +allred md +afgan syah +a ei +^ ) +ðŁĺĻ ðŁĺĻ +ðŁĴĽ ðŁĮ» +æ¸ ¯ +âĹ ½ï¸ı +ÑģÑĤ в +yo gaw +y tb +y it +x med +wine chat +wil do +wat too +w lg +vic om +ure thra +under nourished +u sche +typo grapher +tycho brahe +tv uk +totten ville +to ffler +thu sian +theband perry +the ee +te mo +tail gates +ste vero +spre paration +spo oners +speci aleducation +snever die +slug gish +sister love +shru b +sher pao +she ema +send in +se uk +sab ahan +sa or +ring fort +recipe oftheweek +pur ports +poly carp +pisco po +pier son +pas sim +par faits +panchay ats +over hear +ouagadou gou +onu evo +oat man +nab ucco +must ela +mild may +mihal y +micro breweries +masta ace +m wi +lon o +lo bel +light sfor +kozlov sky +kn bc +king bach +ki it +khu fu +kgal agadi +kevin woo +ke msley +kan chenjunga +joell ortiz +indi os +imperman ent +ima g +hul sey +gu yon +green lawn +goo dest +go ater +flor um +fl its +fin cham +fel issa +eh ren +eft pos +del rey +david bisbal +cowboy cerrone +coug fb +contemp tible +comfort zone +combin atorial +childish ly +child day +cherno ff +chast ise +capital ised +ca j +c pps +bio shock +ber nadine +ber gia +beau mont +bat tel +ballyna hinch +al ick +air vistara +agye mang +ad ub +( ?!) +ðŁ¤ ļ +îIJ ħ +íĥľ ìļ© +ãĥĥãĥ Ĺ +zum thor +ysle ta +y cee +wood bine +wirt schaft +w div +vive ka +var dhan +v wd +under cutting +um alo +touch ph +thri fted +te ching +suntrust park +squeeze box +sou cy +sof ast +sk ims +sc nn +sauté ing +ryn chibi +rush limbaugh +ru mor +rose town +regu la +rec ca +rd pro +randi zuckerberg +ps fk +people make +pand u +p sps +ore ille +orange crush +one town +nu blu +nor ie +new t +na sho +n atti +mar ination +mahon ia +ma inde +lv cc +lep anga +leon ia +kor ban +kn acker +key leth +k soo +jo bur +jacob sartorius +itsin youtogive +itch iness +iot swc +in cant +hé ctor +hu len +hor ie +ho ce +happ ym +ha flinger +gro ttos +gra be +fur rowed +ex ab +du lais +dout or +can ady +campylo bacter +bic c +bekin dedu +bar det +bally bo +auth o +ashley graham +ardaw igs +ap uri +anaesthe tists +alliseeis gold +all port +ðŁĻĦ . +ðŁĺŃ ðŁĴĽ +ðŁĺį ðŁĴª +ðŁĺĪ ðŁıĪ +大éĺ ª +x seed +wä rt +wo wsers +wis ata +wee ke +w ene +voy ager +vl si +visit savannah +tre ve +trail finders +then ut +the party +tene brae +tar pley +suzanne lepage +super cluster +stephen son +spri gs +spi vak +spa day +sp hs +solheim cup +shy ly +sfor women +school kids +schen ck +sab ka +ryo ji +ry en +rome sh +retrac ting +resc ate +plat ense +pint u +photogra ghy +petr ina +per mutation +penc illing +parale gals +pal at +outre aches +open doors +ong al +ok bokki +oh g +nzv sl +normal ised +ni ort +ni ed +nam b +n achi +mr selfridge +moun tie +mol lis +mob ilities +min ard +mimick ed +mile tich +mc isaac +maythe fourth +man flu +mag gies +mad dest +lu zh +lom ita +ll dubs +lincoln hall +lik ar +ko kol +kim mage +kid sand +keo hane +job done +j ma +j hang +itv sport +is ingly +id man +hp noti +home opath +happybirthday salmankhan +gov markdayton +glo vers +fu ganda +ex ican +encro ach +edmun dson +draw pile +do iron +dimit rios +delhi dynamos +cor vin +conoce artistas +commo diti +co atta +buff ay +bee g +beat navy +be tula +az ari +avenged sevenfold +apan thers +ano j +aldubbig girlz +ae schy +ade ci +aban erjee +:' > +ðŁİŁ ðŁİŁ +ì¿ ł +æ Ĥ +âľ Ĵ +âļ ĸï¸ı +z ade +yu th +withr and +vicen cio +va ar +un block +ukin india +uj iri +tri g +transgre ssive +tra gos +tor v +ticke ts +the stars +the ss +the matrix +the duke +stocking stuffers +stafford shirehour +so dak +six nation +rs ch +ricardoro ssello +reli x +recou ping +rangers family +radion ow +pe jeta +paper white +overex posure +oniz uka +ny le +nh r +new market +naf i +n drf +n aldo +mulvi hill +me ha +mcmen amin +mcgon igal +mall ery +logan berry +lemn sissay +laun e +la fon +kre ay +kim bolton +ke pt +ke ary +k hem +john swinney +jfk library +javedn laghari +itune spodcasts +i ken +hor der +hop kirk +hl mbb +gup ta +goo i +gi onta +ghana starnews +gar ryo +gar ib +fun nell +farmer sin +ex ib +es na +edi ficio +east leigh +dw yn +doit big +con gradu +compati bles +clar a +chro site +charter is +charlie kirk +chale ur +calibr ations +cal dwell +brank some +bison pride +bankrup ting +ba fe +ausi ello +al tria +ab flood +y boy +x bi +women said +wol ter +vocal oids +vibr atory +v yach +ur banc +tu fo +trim pe +trent university +tread way +ti pping +ther aven +the mist +stan bury +st ale +ss sa +sor oka +so tn +sma h +slam et +simone simons +sergi om +schneide relec +schle reth +sab bagh +s wh +rockefeller fdn +ri mba +rey ra +qune itra +q ip +ponchat oula +peven sie +perenni ally +pad el +osom atsu +organ elle +optical society +none such +nether field +neca xa +nationalgarden ingweek +nas daq +n guy +ml ine +me scal +me io +mazum dar +marvel sdcc +len y +kyo ya +knuckle ball +kni fes +kiis fm +ki shang +ketogenic diet +juan jo +impe ding +i goe +houston police +house forsale +hon ks +heg gie +hart ington +ha sp +gu at +gu am +good without +glori oso +ger b +geome trically +fox ing +fl oro +fend t +fairchild garden +f assett +ever sheds +enchan te +eli zalde +ed markey +ec ar +di martino +di atom +dhal sim +dead y +cuck mere +crypto pia +co authored +chir ic +chip stead +cas as +br ymo +boo sie +bo sarts +bo ki +bla w +bi ped +baz in +bar oud +az raq +az ara +ash h +as par +are pas +anticli max +another one +acadi an +a ite +ðŁİ¶ ðŁĴķ +ðŁĩ¨ðŁĩ ± +ðŁ¤ ľ +ì² Ń +while black +wh ern +v ws +u stain +tru stuk +top secret +today yy +tn news +thucy dides +sp fx +si solak +shak y +sh net +rufu swainwright +ru bal +re buffed +rc memories +ra aga +quincy djones +q ed +pu kul +prince ssc +presidenti elle +pre da +por fi +po can +pi atek +period poverty +pen light +ol wen +nitro circus +mu ddin +movie reviews +mono culture +mill bury +mid ori +merry n +mel lowing +medal monday +ma goffin +m ld +liti gators +krish olden +kno p +kla ssy +kf dx +kalancho e +jyvä skylä +jam ón +israel icon +ina ina +in anna +i bar +hot d +gumb oot +gri gory +greatest generation +ge da +fujin on +fuji x +fric assee +formul ary +flead h +finn mark +feren c +es gr +ent p +ele azar +egg old +ecumen ism +e gle +duvvad ajag +down draft +doet inchem +dis ambigu +dan reynolds +cy c +crox teth +coatta ils +co wappreciationday +clo ve +chil ika +c conf +bud in +bon us +biz kaia +bisson nette +beard life +awas thi +as mit +as ket +ap lanet +allen west +ac ads +ab v +^ / +? ðŁĺģ +:-) :-):-) +âĿ¤ ðŁijįðŁijį +ze ke +wild land +whytele afe +wat tie +w tt +vibr ated +ve chain +tuan ku +thereal p +the wright +tenter field +ted nugent +t mv +sof honor +simon i +shutu pand +shobha a +sex change +scul ture +run tagit +rock ineve +ro zz +richarddin atale +richard garriott +rajkumar hirani +radha krishna +pugli si +politici zation +pinch beck +over think +os loff +ok ker +oc tol +nostal gi +mortar board +mi stake +mary na +mar on +lu gu +love leam +li acou +le grand +l fd +kate flannery +k fa +it ou +iran freedom +il agan +holein one +hob son +henley regatta +hari krishna +hans brough +gustav sson +grizz ley +good things +go yen +giandu ja +ge tn +game faqs +g sd +feliz diadel +f vg +f oma +ep w +earth strong +dun nell +drjoe abah +dortm under +cour trooms +ce uta +cape k +cap les +buprenor phine +blues ville +blon o +biophy sical +bill bailey +be dene +batter son +bal r +baker mckenzie +aw yers +ave yron +au solympicteam +asteroid day +anuger ah +al shabab +al gie +ain sdale +ach elle +a hed +zi yon +vu du +volu me +visiti ow +ven den +u tha +trend hunter +tore lli +tom islav +the web +ter ada +sugi zo +sol va +sker ry +seren ity +sempiter nal +seismo graph +robe spierre +road er +ro da +reverber ate +retrospec tively +red s +pre hen +po trait +po in +plat for +pi voted +pet an +per nah +parkinson sdisease +p mn +ox a +ou ard +ol ah +norway mfa +north borough +nor ceca +neel ak +n dom +myhaver photography +mu ara +mobile dev +mid c +mer ak +melo dia +me con +max am +man gom +major leagu +long last +lo hse +lfc tv +kle iman +kit ne +ju ge +jesse tyler +iy am +italian art +ir th +ice house +hill arious +hello fresh +hamilton college +haen ni +fanc on +extermin ators +experim enter +everyday is +european elections +ess ai +e wel +dy land +dramati zation +dr ra +dh m +complex con +co ate +chá vez +cay ton +car nau +bronx zoo +bo pinion +black strap +bee ches +bang ash +balac lavas +b cash +au rie +ar ayan +apoor va +alder sgate +ae mia +adi aries +ab aca +ðŁĽ ¥ +ðŁļ² ðŁļ² +ðŁĺĬ ðŁĻĮ +ðŁĴĩ ðŁı¼ +ðŁ¥ĩ ðŁ¥ĩ +çĻºå£ ² +âĶ ĥ +za ini +x company +whatiwant most +wam bui +wal ima +w cr +vi gyan +vai dy +usc gc +usc c +upp ort +unse rer +unisouth wales +un watchable +tom arnold +tit u +timo teo +the odd +templ er +te hel +tb d +superstar rajinikanth +stick ler +ste phi +state house +staf fies +spur se +soto grande +snapp in +sm g +shug borough +se ish +scher tz +sar at +saath iya +s fin +rose bush +ri sser +reci a +re forged +rat oath +rafale deal +popu lus +pollu tion +poker news +po blen +pic col +par acycling +nbab allot +mummi fication +micro credit +mas sport +man rique +make chester +mah endran +lennox town +lake placid +kunis ada +kray zie +kezi ah +kd lang +jame shar +ix a +ity council +isol ator +ingo ts +indoor oo +indi atravel +in heaven +imp ish +icom os +hy uck +ho ppy +hi ei +health talk +harmon ised +getmoney out +gb g +gay timesmag +g da +first legoleague +far ry +eth je +epi stles +eco l +dunn ville +dre we +door frame +dino bot +co asto +cha dian +cast rol +bluel ight +big daddy +bernade tte +be true +bar la +bacter io +back scratcher +b music +ati i +ast man +andyburnham mp +aitu taki +aerop orto +adam curry +abdul lah +abbey theatre +ðŁ¦ħ ðŁ¦ħ +구 구 +ಠ¯ +ঠ¸ +ym fa +ya red +weiz mann +war my +walkthe moonband +van c +vac c +tom m +the mummy +tess it +tell ings +teamdi data +survivethe ark +starvstheforce sofevil +snu ffle +snow don +sin que +sch ach +ri ah +retin oblastoma +ren mark +rein aldo +re establish +pride day +philli pp +panch gani +paedo philia +ous and +ok abe +oc ic +nu mpy +no brainer +nick le +ne eth +nair amb +moun te +mc math +maui jim +mar ring +mak ana +ma sr +m div +ly ch +loveyour melon +looknorth bbc +liken oother +le breton +kri spie +koech ner +jessie ware +iklan in +ig we +hindu ja +het chy +hed ger +haw thorne +gravit ated +googl enews +gbo wee +gas son +fsm py +free event +ess l +emor ies +ebon i +du iker +dre scue +doit yourself +di emen +derek carr +demysti fied +cultiv ates +cr cc +corr alled +colqu houn +ci ro +cha shma +car se +calci fied +cack le +ca shinin +bry ony +brad berry +blossom sband +bin tur +battlec ruiser +bat umi +avak ian +au dette +ann apolis +ang t +aiya ary +abut ment +$ ? +ðĿIJ ¡ +ãĤ¦ ãĤ£ +wi fis +wha aaa +wax editorial +w bbj +vis a +ultr arun +uk pubs +turntab list +traf low +toy show +to seland +ti fft +tequ era +tan sley +talla poo +talk sport +taka hata +sy am +spring fever +spon don +spike ball +sou tine +so bered +slo vers +she saidyes +sd ks +sal onika +rol ph +roger stv +rhodo chrosite +regular show +re culver +quiz timemor +puppy bowl +psy lli +princeton u +pri eta +pol ony +pneu moni +pipistre lle +phili pe +pan ah +oooo ol +om ance +ny havn +no ff +nhl bi +mu stan +mu dding +mot tos +moro zov +mick mars +metam aterials +mer ick +me ili +mck enzies +mc millon +marsh aling +makechester proud +love goldenheart +lo red +leg work +korta jarena +kateys agal +jic ama +it sac +influx db +industry news +ide ac +ice ice +hulu si +histor yo +hem ric +gu rel +go bearcat +gen maicha +for four +fearless records +f ö +f stvl +end note +dug anda +drum chapel +draught sman +dom ingue +def a +daf ridi +d bs +cyano gen +cor bel +copp ery +con oco +chain saw +cau then +by g +bun ka +brombor ough +break ie +book awards +back of +astr alis +ar mill +ar adia +amar ante +ac eves +" !!!! +! âĿ¤ï¸ıâĿ¤ï¸ı +ðŁĴŀ # +رÙĬ Ø§Ø +Ð µ +z the +yan ag +y thon +whitting stall +we sties +ur vivor +uphol sterer +up h +un char +u fd +twee ty +trin comalee +ti ar +thexfactor usa +thecoop way +the vet +that game +testam ents +tc f +tam bun +sus ans +su maya +sser ver +smith wick +shiv dasani +senig allia +sen randpaul +seag ull +satur no +sand inista +sa jan +rub bishes +po legate +plain well +omer ta +oku bo +og l +ncan ationals +murrum bi +model ismo +michael urie +melli fluous +media awards +massi mino +mar ja +man handling +madein nigeria +ma sami +list serv +liqui date +lin nell +lesm cke +lazz ara +laura dern +l bof +kr cg +k jer +izz et +is ser +hel pe +hard aker +ha bba +gü ell +gwanghwam un +gro ttoes +gover no +go eth +give us +gi gli +ghis laine +g football +fore taste +far yal +faith hill +es days +eidal fitr +ed ong +e im +dra ko +diffic ul +del port +db ms +dari ush +dam ekelly +craft work +cor relating +col back +cine magic +chinch wad +champion sday +ch ems +cawlidge hawkey +candid ats +c engage +butter ly +bush wacker +ber gu +be ek +ausv sl +au fen +alco co +ad woa +( ???) +# < +ðŁĴ Ĵ +ðŁijİ ðŁijİ +åľ Ĵ +å£ « +âłĢâłĢâłĢâłĢ âłĢ +âĵ ľï¸ı +you view +winnerwinner chickendinner +whe ater +wh iti +wes thuizen +vsc ancer +virtu o +ver ghese +up ci +twitter smarter +tu junga +sudhan shu +stop tb +sp acy +small sat +sel ayang +seatac airport +sa im +ro hl +rigon deaux +rec tification +re mu +r tos +r ra +pur gat +pur ba +pug ilist +promotional products +pri e +poe tica +po tala +po sso +pejor ative +paragli ders +on broadway +oh dear +nn u +ner ina +mer ger +mede ski +me les +max illary +math works +mal har +lu uu +long shaw +llanid loes +liber dade +le vey +klo ster +kamal ha +kal si +junior contract +jom alon +impropri ety +il ondon +hur ontario +hen wood +har umi +gulfof mexico +guilty pleasure +girlsin tech +girl stalk +gau cho +franchis or +foxsports det +for mel +fit na +fairtrade fortnight +e ire +dreddy tennis +do oney +desh ler +del oris +cork screws +cop al +co zz +clo gher +cir ce +cindy capo +chlor ite +cal cot +brown live +bro fur +boy ssoccer +bel al +asse tto +ask with +angar cia +amal hotra +al faj +aja hn +aguas nash +ðŁĴľðŁĴľðŁĴľðŁĴľ ðŁĴľðŁĴľðŁĴľðŁĴľ +ðŁij¨ ðŁı» +yse ali +wwww www +varsity cup +under achieving +twitter friends +tri de +till ing +thep slt +the thing +the dead +t sy +str acism +stepby step +sprow ston +spor tiv +six ers +sie gal +sf ball +seraf in +q tc +proven cher +power lunch +plur alistic +play with +percol ating +oun tain +omin ion +obl ate +noo d +nizam abad +nit rite +nakat omi +n alo +me gu +marketing agency +lo ess +liph ook +len sculture +l lew +l cu +ku en +ki ppers +kalon ji +just cause +jessamyn duke +jat ya +icen ine +hydro meter +huuu ge +hard ly +gru l +google india +genealo gists +gam ache +fw ends +fresen ius +fil mi +esp inal +ep sb +enam eling +el ar +e ala +disc oloured +deri ded +derekcarr qb +deno ting +das sey +cru ce +cross over +c spc +bu hat +british monarchy +bo zar +bo bov +blu emotion +bar dock +bar ada +ban kni +aw wal +ash burnham +ans combe +am joy +agu stina +ag ould +aeschy lus +accident als +. "... +ðŁĺįðŁĺįðŁĺį # +ðŁĴĭ ðŁĴĦ +âĪł )_ +ঠķ +y talkies +wi shy +w gw +virgin active +vindic ate +vin ous +us ba +ting ting +tin aturner +the defenders +texashoo ps +tail less +sur ya +stray dogs +sto dgy +sr p +siguemeyte sigo +shu mba +sher rin +shah jahan +sarah geronimo +round wood +ro blin +reykjav ÃŃk +reinvigor ating +reic hardt +re work +re khta +pro stration +po ona +obase ki +nr can +ni os +ngu rah +nex tup +nepon set +my bestfriends +mon dad +min dedly +mikha il +mer k +med tech +me harry +luxu ria +loo by +lo phus +le ja +kyung soo +kel lar +kath thi +kab ul +jo sap +ja ars +illi um +hydro dynamics +hot n +hornet pride +hilde sheim +hi mig +gold medal +go sch +gl itz +gin us +ger ards +gare the +free wheelin +frank ton +foodpor rn +esc ap +end malaria +eag lets +dougie poynter +don ahoe +dol fin +die antwoord +denver outlaws +deniso hare +daw sonville +cph ftw +concur rence +co omes +cand ace +call an +braun ton +book stall +black forest +bil d +bike show +beaver brook +be fallen +at kin +arma gh +ai vo +ðŁĻĮ ðŁĺĤ +ðŁĴī ðŁĴīðŁĴī +ðŁĮ © +ëį° ìĿ´ +âļ ĺ +wil ight +whern side +vexillo logy +uni as +uin tah +tyr whitt +tu lipa +too sweet +ton glen +thisi si +stich ting +so horadio +sher way +shar rock +senti miento +sch rier +salem or +sal y +rich ten +ri bault +play throughs +pi pp +pet ko +p inga +or chester +now shera +nar re +mtn ng +mor iya +mis ophonia +megan follows +me ades +mcglo in +mayan sfx +make better +la plagne +kru mm +kas al +ka pok +jamesb valentine +israelicon flict +hu sain +housing day +he dera +hass all +gran holm +go air +geet anjali +fuvah mulah +flag man +fik ri +ex ley +enjo ining +endor fer +emo ir +diss ension +dismant les +deptof ed +cy bele +curl pro +chen na +ch rit +ca hoon +c ire +buffal ony +bou dica +bo ff +bla by +bill and +b isou +ast ound +art style +anti phon +air munro +ad of +abel ardo +aa di +! âĻ¥ï¸ı +ðŁĴ¿ ðŁĴ¿ +z enga +wu bb +wou l +world rowing +word books +west more +wee z +we belos +wayne coyne +vel outé +trans lu +the ki +the bible +th ale +telegraph travel +teal over +sopp ressata +snh strust +sk ola +sh ary +set lock +seg menting +rivieranay arit +rig ney +reg ner +refin es +realkevin nash +r q +quad rants +pra soon +per otti +one thing +oklahom acity +offthe beaten +nicol am +ne ara +mj m +magick al +macle od +long lines +lo licon +lew k +lali gaen +la foret +l jones +krist ofer +koe hn +kin ahan +keigh ley +kan ab +k pe +janef allon +is w +infinite warfare +individu alised +ic wa +i spor +hy patia +hy bels +ho cker +her dy +heim dall +hag ans +haare tz +gugu lethu +fri ggen +fore seen +for tino +fidd les +fel ina +endemol shine +en snared +ema snhstrust +duck pin +disability rights +dad agiri +cur wen +cosmopolit ans +ci pher +choc cy +cau li +calam ine +brim field +breaking the +bou man +boston cannons +boon sboro +black smiths +bistro s +bing u +bin ta +bel air +baru ah +b son +azira phale +axel sen +aviation history +aspen snowmass +am bat +allan mcnish +abstract artist +aboiti z +ðŁıĮï¸ı âĢįâĻĤï¸ı +ðŁıĪ ðŁĴĻ +ðŁĩµðŁĩ ¾ +å¿« ä¹IJ +ãģ ¿ +Ø « +Ê ķ +wärt sil +world musicday +who syour +v hd +utt ara +under garment +twit pod +tun de +town afc +tn cs +ti wary +ti pu +the hu +t fk +sy ariah +sweet leaf +swar m +stu tzman +stock land +stab at +sky ferreira +sinhal ese +sin opec +si eu +shu ey +shi ppy +sens go +s itti +rob stown +ren table +reli t +release day +red ol +pv hs +protect ing +pi pits +pe ton +patrick starrr +pam grier +p isode +on thel +official rufc +obli ges +o boro +no cona +niem and +mosi ah +mont vale +mo oned +me ir +mas ke +mara sigan +ma alik +lu gia +la weekly +la flamme +kin di +ken non +ke bun +kar ur +incre mentally +haz lett +hatsune miku +han dedness +ham pion +hack le +gy r +gu inee +gor rie +goldeng lobe +go sensgo +gnar ls +gh pl +g ines +fw end +forten berry +fo gelman +films bywomen +field turf +el tham +dwt sirl +drac aena +de maria +crime wave +creati vo +coun sell +com busted +cohe sity +coffe elove +chill is +ch ancy +candy crush +c ée +brighton seo +bor rel +bmw motor +bha vesh +ben splatt +be kal +atel ink +as pac +as ama +aren a +arab israeliconflict +anti fragile +andu in +an je +amand ashi +alliter acyday +ah and +adid asu +acu ña +ðŁĸĸ ðŁı¼ +ðŁĵĢ ðŁĵĢ +ðŁĴļ ðŁĴľ +ðŁIJ¼ ðŁIJ¼ +ðŁİī ðŁĴľ +æĪ ij +youth sports +yas sssss +wu i +willem se +whi tham +whakat ane +water aiduk +wash fellowship +vie v +u va +tor doff +thomas power +sxm thehighway +swap na +stren g +sk ot +siwon choi +sie gel +senator burr +sap utra +s bisd +ry ant +residente vil +renov ates +pre ssions +pre operative +po vey +pimlic orc +pedic abs +pedal ed +operation ally +nyc mayorsoffice +nd h +mu ty +mis cast +milan ello +meso sphere +mean whi +mcgu ffey +mar ois +lou vain +lo bed +learn german +lau dato +la story +koz lov +kate bosworth +ka eding +jol ted +jen ac +jare th +jaga dish +iy er +ingeni ously +in fier +hil mar +hawk stalk +h tn +gw ana +gu shi +grim ley +go terps +globu lin +ga via +frnd z +fran con +fli bs +fili ppa +fight the +fe sting +fan meeting +f sf +ezra klein +espad rilles +ell roy +dot te +don en +do yeon +dictator ships +diab olo +deli as +concre ting +coach tom +bo young +bo lex +blue tec +as st +are scue +ar rs +ar preps +aqu ÃŃ +apla za +ander as +alali brary +ajed rez +________ _______ +// < +ðŁĩ¯ ðŁĩ² +ãĤ º +yow amu +x th +wyn koop +wind surfers +whitworth art +whisky day +visit bristol +viking cruises +vic traffic +v rij +uw sp +un tangling +un ops +un ds +tv live +tu tee +tto win +thr illa +tb it +tardi grade +tar ring +t bex +spinning fields +sky deck +sitaram yechury +shoo touts +sali f +rod da +regurgit ated +ram ez +rabbin ical +picto graph +phil brick +om ake +ok hla +net tie +ne ster +nat gallery +nan terre +mum taz +monop rint +mo ggy +mn gr +mix nine +mari ek +malk mus +mal tz +lou den +lic hen +ks music +krat ts +ki ro +isthe problem +interfer on +ill enium +ig ate +hump backs +hot toys +hir si +hi rayama +hast ings +har pal +ha iz +ha islip +green chemistry +gre nou +glam cricket +ge thi +gar ita +flamen ca +film strip +f pg +f gn +ene ts +ely as +ejec ts +den de +dc policedept +dan diya +d scanning +d endi +cs music +craigh all +community college +castro tx +campbell river +cam eo +cal shot +bre slau +bor al +bla gdon +bike to +beat bama +ange bot +amjoy show +am hs +ali bre +aen gus +: £ +-____ _- +ðŁĺIJðŁĺIJ ðŁĺIJ +ðŁIJ © +ÑĢоÑģÑģ иÑı +yucat án +vu h +vol tac +veg gi +tor ians +tho le +thing sabout +the paul +that sa +ter hune +tel p +ta war +sub type +stra iler +sto kley +stal y +son arika +smartcity expo +small ness +sm t +sk itter +sig am +shivam bu +she pley +set to +scouncil uk +run gs +rob illard +ric kert +repl y +re qs +raf san +r re +pur porting +pic acho +photo copies +pere go +pedi ment +pal anca +pak man +pag ination +on j +oda at +o denton +new roz +multi view +mtv u +mosh ood +manoj tiwar +maggi es +m ool +ludwig sburg +lique faction +leh man +kuy per +kar nazes +k see +juniper networks +james acaster +its bristolbaby +ise f +ine yards +in cel +huf worldwide +hss wi +hsin chu +heu ser +he tton +harmon isation +gry ffin +gr aco +goldsmith suol +gitt ens +ge ith +flood plains +fe en +exacerb ating +douche bags +do de +dill man +diamondand silk +de itch +cradle offilth +cor ti +carry themhome +bri mmer +bio bio +berry farm +bang sa +athlon sports +ap tac +ap athy +amit ra +ale quinox +agre ssive +accli mation +ðŁĴ¡ ðŁĴ¡ +ðŁİ¶ ðŁİ§ +ðŁ¥° âĿ¤ï¸ı +x ist +wood man +whe ads +well ston +wave front +vasi l +un solvable +ull mann +ug t +u gal +u arkansas +thejuan williams +swa deshi +st bl +south yorksbiz +so cc +sil v +si kk +service dogs +serafin owicz +semi ah +se mir +rou steing +puer tas +philly mayor +perio dismo +p daf +owen benjamin +okum ura +o esn +nutrac eutical +nu bians +ni pah +már quez +mur li +moon bow +moj ica +mine workers +midter melections +mene fee +melan son +mc tom +may sa +li ska +length ens +lady boss +l ro +kost ka +juke box +jones ville +in oki +howto trainyour +harmar superstar +hag akure +ha ch +guine afowl +greath ouse +glan z +gay don +game jobs +fu yu +fr ancy +fle dging +fl ours +femin azi +f xs +emma willis +ell ard +ei ht +du ed +dispro ved +dese greg +dat o +cr ater +citizen ship +burak deniz +brew ster +break away +bo wale +blake slee +bi gor +bel mullet +baloch genocide +ao ib +am enable +ali ando +ac ros +a shem +: , +( .) +ðŁĺį ðŁ¤¤ +ðŁı ¥ +â µ +zug spitze +xi xi +window sinsiders +wig town +weather watcher +wayfare rs +w re +vas ilis +vac ates +tiger up +ti mp +ther ocks +the challenge +te kk +taylor guitars +surrey life +sto ppin +ssc ricket +spo se +solution tree +semy on +re ber +ram co +pre tension +pre ike +port way +pig my +pen rhos +pe ci +par dub +packaging design +orch ha +nun c +nobuy uki +new fie +national champs +mo tability +mi global +mer ay +meet bros +medal en +me ki +makh ura +lur ve +london lgbtpride +letsgo dodgers +kle ys +key one +k vn +jig me +j rn +j mr +iphonex s +insom nisa +indooroo pilly +indeci pherable +i asc +houseof cubs +hoge styn +hei fetz +hare hills +ha fsa +greeng ate +gre ss +gir ma +gh earts +fl oria +exagger ates +ev re +ep festival +eliver ance +disco vers +dag ang +consequ ent +complex e +by ward +ban yan +ay un +attenti veness +arch viz +ar lette +apu blic +and ong +an ae +aldub maiden +ad it +actu alize +ac tis +aamir liaquat +ðŁĺį ðŁ¥° +ðŁħ° ðŁĨ +âľĶï¸ı âľĶï¸ıâľĶï¸ı +âķ ® +z de +wrong ness +wood chips +wal ke +vum chealth +ver kho +vape shop +van esa +vai ko +tra wick +tor ti +tobaccon ist +to pl +tim westwood +thousando aks +ther is +terrence j +technicol our +te gern +stru tter +strait jacket +spl center +shakey graves +sa stro +s ddc +run meb +ro setti +revel le +re shuffles +rash omon +ra baul +queen b +praise god +panor am +oin uma +oe ttinger +ny dia +nxt takeover +na sia +n roll +n nd +my best +morning show +ml td +mikey way +mass ena +lun ged +long live +litvin enko +law society +l hh +koe itec +ko dai +kick starts +ki gur +its been +ileague official +ide sai +ic ast +hel icon +hei sey +guest post +gor achelle +gorachelle ann +gine bra +gaw an +for der +flagell ation +five a +fin esse +epic cosmos +el ow +eight ball +dramati zed +donald duck +di dio +design milk +dar lene +curtin uni +cougar nation +convul sing +co sn +ceph alic +cav orting +cap el +ca isse +busc aglia +br ts +book storeday +baf a +ati ds +ar ling +appall ingly +agri busines +adu rai +á´ Ģ +world snakeday +wing y +warren sville +usav s +upp et +u sportsca +tru ll +toplo ader +thi z +the her +tas nim +su pts +soph more +sin ai +sil vas +se asia +sd oodle +sa ed +res or +pre seli +pr h +pope scu +pc sk +our schools +or du +op roud +oak ie +now www +new all +mov ingly +michael annett +mer amy +mahogany lox +lyn es +lin csc +li gety +lett ice +l vi +kha si +ken rick +kah ana +joanc rawford +jo achim +jab rill +itsad og +incarn ated +i fri +hy ong +heee ey +happine s +had denham +guer rier +geb hardt +funkand soul +franco phile +for lease +fing alcoco +esi ason +employee experience +eb ace +e miko +der as +d design +cu ms +cro whurst +co omer +cmb yn +chim bor +che min +chandi mal +car swithoutlimits +busines sperson +big ay +bat tic +au j +astor i +anne aling +anc ru +al g +ag ba +african us +a seem +:- )))) +à¹ĥ à¸Ī +Í ¡ +xi es +wit tig +wise guys +virgin iam +vir chakra +vel ux +ut ton +un guided +ubi q +u calgary +twy cross +twe at +tra van +tibetan buddhism +tem be +sthe series +ste ffy +serv pro +secul arists +sanctuary cities +roy an +ri ems +res ounds +raven scraig +rash guard +ranc ourt +raise d +qu ent +qu atu +punjab is +prize money +positi vely +pe ste +pba onespn +parmi gian +oy ale +over coats +ol abs +nca ab +musa fir +mm tc +mey rick +metal album +merry lands +mechan ization +me gh +mat eri +mad aba +macewan u +lu king +lu dden +liber ians +lem nos +langu ag +ku tti +klein man +keat ley +k de +jo j +jan se +irr ational +inf l +ie b +id in +ic et +i herb +hispanici ze +hij jah +hepat oc +head ship +hallo ck +hal lie +gur ps +gu fc +globe trotting +g sv +fur mint +fugli en +fit food +femmin ile +f cra +ers rock +dwind les +dv g +dos gaming +dimension ality +denti st +dee g +de mont +dar yle +corne as +contain ership +cent um +cas os +can ción +campe se +bul ski +brockle bank +biz nasty +beat son +bas le +bal derson +b my +as g +ann ua +aeter na +ab senti +ðŁĺĦ ðŁĺĤ +ðŁĺ© ðŁĻĮ +ðŁıIJ ðŁıIJ +ðŁĮ·ðŁĮ· ðŁĮ· +ê·¸ë ŀ +ÙĬ Ùħ +y news +xx oo +whirli gig +web star +waynetwp super +wat teau +twitter mirror +tre esof +tigo bba +tame side +sub junctive +stru dwick +ssi mon +ss aturday +sla b +sigh thound +sie gen +sey more +semin aries +seem s +samurai jack +sam ma +s sudan +s ros +rohit roy +rail cats +pose able +popp leton +pas cack +pan handles +oo se +nice guy +negre te +n ssn +n ough +ma kak +lo thians +live underpar +la kiss +la ha +kon ner +kochad aii +ko sinski +jeremy mckinnon +j bonamassa +iv ins +hue vember +houri hane +hop f +hok itika +ho xie +hide out +hee bie +he cla +hamlet fc +hal kidiki +fre de +fm kenya +flori das +fat cat +dulwich hamletfc +dri skell +drac aen +dou bs +demir tas +dem socialists +dc tid +creative scots +conserv ator +co ko +co by +clay born +castell ani +cas sa +car fag +ca elum +black monday +billy bob +ber ndt +ber mingham +bed was +bally shannon +au ba +ascen so +ar mel +amaz i +........ # +åIJ § +âĶ ĵ +âĢįâĻ Ĥ +ze th +ys lam +wool shed +wol fal +wal shy +w gt +voi vod +vie ux +vic ini +veri sign +vare jao +valu er +un cas +ty nes +town line +tiktik tik +tidd ly +the villain +tallapoo sa +t lax +sydne yo +stein berger +star base +spider sona +sp ini +snit ches +shapp y +se kt +sc ag +sasi kumar +samanth a +roe hampton +robust ly +referen dums +re invested +ra ghe +r á +quin livan +pul ford +proven zano +pras lin +portsmouth nh +plo rers +play more +plas monic +pil ar +peter sagal +pang an +pae onia +osor no +orel lana +on repeat +og maco +nz vaus +now den +notinthis lifetime +neil sen +nathan varni +mo sta +mam usic +mal wa +l jp +l chat +kun di +kare ga +kan ald +jo swinson +ji rou +jar k +ja ig +ite k +inter costal +indi stin +incogn ita +incivil ity +hydro codone +hocu spocus +ho pin +ha sen +go jordan +go figure +gm fus +gl unch +gi unta +gepp etto +gang ly +ga hara +enamel pin +en gro +egg less +ec ru +ea sements +durham nc +dun sfold +down range +double bass +da day +cy on +cra ine +cover story +conson ants +coach jim +co quet +clu bo +cliff central +citym all +chair persons +cav our +carls bad +canvas ser +can dia +cab elo +ca ille +brun elleschi +bore scope +bear ing +ba sile +b cra +ati q +arch s +aber crom +ðŁĴĹ ðŁĴļ +ðŁĴģ ðŁı¾ +ç ¶ +е д +zi yad +zah le +wr angles +woooo ow +wit old +westend live +von age +v fp +un toward +ulti max +tre lli +tie gs +the difference +tera hertz +tallas see +tah qu +sten ation +spe tt +song jihyo +si ong +sask power +sadi sts +ruffin o +rooster s +rom puy +rogow sky +read vocates +ra dom +quin ney +queenof scots +print ings +prayfor southkorea +pen coed +pei poli +pad den +open cv +o ab +noar lunga +nihal ani +nect ars +mu dras +milan esa +mient us +mev lana +mazdar aceway +mat tock +marquin hos +marine inst +ma uli +ma ja +llll llll +j rr +inte mper +indiab ulls +ind travel +in service +im melt +holy day +hardik patel +hack ley +green hithe +gan ic +formula ic +fin domme +figu eras +father andson +f ww +end ricks +ear ing +duvvadajag annad +du bbin +dru ms +din ary +dick heads +daf fs +craig kielburger +cra ik +chi hay +cas inor +can an +c gpa +bristo lold +bj u +bi ben +bear ance +bay nton +bag ge +ay la +as afa +are ena +ane ka +am zing +allen and +alam gir +af ound +a egyp +ðŁİĤ # +ðŁĮ¸ ðŁĮ· +ëĦ ¤ +à¹ģล ะ +wer ke +wehr lein +we igl +v anny +ush mm +tr illed +timeto shine +the worst +texas forever +ta vor +t ads +swoo pes +sumb ar +stor row +spol icy +so ontario +sme uk +sm l +sheskindahot musicvideo +sanjay azad +sa am +roz as +rock androll +ric kon +restric tor +respect for +quart ile +pul o +pu sey +pr on +pope franci +pon ts +paro died +o shea +nr genergy +nov um +no words +my m +musco vado +mom ir +mn or +min omonsters +mi zo +lucifer season +llan os +leishmani asis +lat u +lacri mosa +kk box +kid brooke +kan es +justicele agu +jung min +ji eun +jennifer nettles +jan ah +itur be +is foreveryone +inge cho +im t +hy am +horse back +hol dover +hocke ssin +gold leaf +girl strip +galle ons +fs midwest +fie vel +femini zed +fe herty +equ idad +el stern +eat the +durham cricket +dragon fruit +dima io +did st +destabil ise +de ok +dat al +dardan elle +coach able +cere bellar +byom kesh +bu bi +bro cket +bra instem +bin tulu +bin dass +bi asa +be vs +bas ah +ba hau +ba arba +asian et +ash na +ag rand +ðŁĽ £ +ðŁĻĮ âĿ¤ +ðŁĮ Ĺ +æĴ® å½ +âĻ¡ ) +âĺº . +woocraft scs +wino grand +what t +wed s +wd ney +watt bike +was sa +vit ational +v mr +us ko +un varnished +tu que +tsu chiya +tr ama +total war +tidal x +thanksgivingwith blackfamilies +ten ge +teacher scollege +switche shop +sul phu +stre lit +stil inski +st bri +sse airtricity +south hams +sour ness +sky i +sj e +shu ma +shail endra +shab elle +semi automatic +schlad ming +sc use +sc dp +sagu aros +sa ami +ror attack +robert marawa +ro tr +ro em +releg ate +redondo beach +purple army +pu tsch +pro pos +pro lapse +prized raw +pre disposed +pol unin +pay scale +pau w +pad locked +otta wab +opp as +never getsold +n cre +more fun +michal ak +meramy akrishnan +medi aboy +mb ira +maul s +malayalam review +love wildlife +lordof the +lidiab asti +la shawn +kumbakon am +keep americagreat +kann o +kamal hassan +ist h +indigen ously +iac occa +hoo pinsider +home field +holiday spirit +holi es +hershey pa +heim an +ha kun +gonz aga +gon tier +go cat +gay dos +gab in +fy rom +fe verything +endo carditis +en ita +e ev +dog mas +dis rael +da via +d ún +d mt +cá diz +cow gill +cl news +cheru bim +canoe ist +by fuglien +by doing +bu sed +brux ism +blazer nation +bio scope +bad girl +avi ds +assu age +ar ouses +apost olate +andre ward +an ura +alvaromor ata +ðŁļ ĸ +ÙĨ ÛģÛĮÚº +z ool +yep live +y ke +wunder lich +wow app +viol inists +ul haq +the resident +the age +tfl s +team cap +te yes +te bo +te alight +tau n +swa ine +suf croot +sufcroot shall +sto ken +spraw ls +spra ining +sof apaka +shots fired +semb awang +sel ft +scienti fique +sch wabe +sar nies +sap er +salv atore +read indie +ra at +preike stolen +popo va +pin ney +opar di +omgom gomg +old paths +ne ese +mo jokerto +mein ers +ma inst +lil as +li hue +legisl ated +le mmons +ld nairamb +laudat osi +lanca sters +lan thi +kontak te +knit ter +ki vi +khaleej times +kha war +ju árez +joe the +jason r +is sy +i fru +humber stone +ho tta +hi jas +han kin +hallam shire +guine y +gopo lov +gom oo +gom an +gamer oom +fords theatre +f gd +ex os +er melo +er f +dy ad +dol gopolov +dic er +deser ves +dep to +den huys +deduc ting +day es +dani yal +d tz +convul sions +cil acap +ci ri +check mark +ch aba +carter reynolds +bruce wayne +book mark +boo kexpo +bla ue +ballybo fey +b ma +ar shi +am yn +am vca +ag race +actualit é +# ) +!!! ??? +ðŁĺį ðŁĺĦ +ðŁijij ðŁIJĿ +ðŁıĥ ðŁı»âĢįâĻĤï¸ı +ðŁį´ âĿ¤ðŁijįðŁijį +ðŁ¤ijðŁ¤ij ðŁ¤ij +à« Ģ +zy x +yah shua +wsu cougfb +wolf hounds +winniem andela +white gold +wa key +video juegos +ve sti +uni leiden +tx grizzled +ts ne +the hope +the chief +than ky +th ral +tau riel +t flofficial +super sprint +su ro +sja day +si rc +shra ger +sha hn +sh lita +san ita +sali eri +s gairshow +s bb +roll pride +richarde grant +reagan rays +raza q +ra fal +qui etude +pu long +priorit isation +popp en +pit stops +pin dar +penhali gon +pe acoat +parthi v +pa ean +our country +ouar zaz +opp ur +ni um +mul roy +monterey bay +mod bus +missamy childs +mephistop heles +me gang +me dulla +mag ent +lit aford +lidiabasti anich +les age +land guard +labra da +ku tt +kodo txgrizzled +ko tatsu +ko bby +joy ceme +jac ana +ite asy +initi ations +in this +ili ani +hé lène +hur tt +hur ns +hi bari +habi bie +ha fod +h fb +gran ata +goat man +go vardhan +glac é +gi de +gal vatron +fu ster +fl out +eric wareheim +duvvadajagannad ham +do ggg +devon seron +der bez +de anc +d wain +cut the +con sensys +communic ado +chal kidiki +ch appa +c za +bot olph +barthele my +ban jar +atta wapiskat +at tax +ar vs +and rena +aman si +allo fus +agar ajan +ad ms +ðŁĴ² ðŁĴ² +ðŁİī ! +ðŁĮ Ĥ +zim bardo +z ev +yot tawa +yahoo live +xander berkeley +wo de +visi oned +u og +twir led +turbo fan +timeto fly +thun dered +thearcan agame +the fire +the bb +te et +ta wh +swee tums +sun danese +sp lish +snake pit +sidd arth +shot by +shop lift +sheamo isture +shar mel +sam bu +saint msg +ro dge +resolu te +ren to +recal cit +que eni +qu ili +q am +putin rf +prun ty +pla stica +pla gue +park lane +oligon ucle +o ingo +ne ch +nab awi +my nba +muse umm +ms ds +mihaj lovic +maxi mu +max chilton +mari ote +march esi +mam ey +lal bagh +kov ski +kj ell +kendal calling +just icec +ju re +jamaic ap +jab oo +ist p +isi o +invest ing +hypo xic +hyou ka +hay hurst +happy newyears +ham madi +gur inder +grin ded +giam bi +gi be +er mah +en yi +en ie +du charme +dis locate +desic cant +dat l +dam ping +da inese +connor army +coachdan mullen +clause witz +cel li +boat right +ble wett +bit ar +birk beck +belitt led +backin black +ay yyyyy +avanti ka +arin ze +aparthe idweek +animal crossing +an berra +allison b +al vida +ail leurs +acec ourse +ab org +aal to +æĻ º +ãĤ¢ ãĥ« +à¸Ħภ§ +à ® +yon hap +wrong doings +william and +west cork +warwick castle +w vc +vi rendra +univ groningen +union ization +under reported +to pd +tiger air +tic o +thing sin +team effort +sydney trains +supple ment +stru an +straf ford +steal in +stalag mites +smoke stacks +sic ure +scatter gories +sc avo +say z +sanjayazad sln +sand more +sag rad +raw kus +ra hel +ra ds +pro fe +pe cor +pal meri +oo ohh +nu aimi +nar vik +n alls +n afc +miz ell +miro ir +minniem ouse +michal ski +mer credi +menstru ationmatters +mctom inay +mcfar land +marine science +mar ston +luci enne +le mm +l ss +l ome +kimso hyun +ke ur +k uper +joon as +jerus ale +j ti +ishi gaki +intere st +ina itis +he si +hau ssmann +go choctaws +gi ese +folk rock +flour noy +fau t +every thin +ever day +eup en +ent endres +el ke +ec ancer +du ker +doppelgän gers +dial up +designated survivor +dela et +darkest dungeon +cogni zance +cityof hamilton +cecili o +cabo chons +ca ed +braz illian +bol ler +boboi boy +bishop scourt +bet ta +ber nama +be beto +b xb +avis itor +aro ad +ak un +ag lecam +af oods +ade ep +ache ampong +aash ish +--- >>> +*: ãĥ»ãĤļ +ðŁĩ¨ðŁĩ¦ . +ðŁ¤£ðŁ¤£ðŁ¤£ðŁ¤£ ðŁ¤£ðŁ¤£ðŁ¤£ðŁ¤£ +è ´ +zeet amil +yu ne +wri f +wil ner +whatson antv +war te +vander pumprules +us jn +tr tworld +ting gal +thisi sthelife +sy yc +superblue bloodmoon +sundar pichai +stereophon ic +sql pass +sl so +sis fur +she erin +sharon vanetten +sha adi +sarwat valim +sal al +s walk +ruth men +quiztimemor ning +quit te +pucke red +pre menstrual +post cards +porsch enewsroom +po wel +pin oys +party with +offthe grid +o wain +o esophageal +nu minous +national tequiladay +national filmawards +na stro +my vancouver +mo sport +mas vidal +mar zi +mano el +mal oy +m chi +lock en +lil nas +lang side +key pads +kas u +kam asi +jar ia +jan owicz +ink off +ib mi +hr vy +hierogly ph +guide to +green line +graphi que +grandtheft auto +gor ney +gomoo sego +god rich +go hogs +gan ley +gab en +futuren hs +fare share +el swick +ebr pd +dy scal +dro vers +don kor +domin ions +dilla hunt +dc ps +day trader +cÅĵ ur +cowli shaw +con ed +c zu +bryan bros +brumbies rugby +bleed orange +berwick shire +ay ev +atl ant +at é +at rack +astra khan +ard al +am able +aless andria +agri ppa +ade kun +ad ak +abag nale +" ): +ðŁĶ¥ ðŁĴª +ðŁĵŀ : +è° · +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ı@ +zu zana +zo zo +z idan +wy ld +willi ger +wil ms +wheel wright +vol tas +uni das +ty d +twee die +tra ub +tol ar +tiber i +thim bles +thelauren graham +thel one +thatsmy dodge +than ga +tan ey +syl ph +sty x +stan ne +ss bm +sou to +so han +sig erson +shimabu kuro +sh kov +sco smetics +schne iders +ram bler +r vw +pul ly +protein world +pran av +polari ze +phil co +p mg +p ami +op re +op an +of peace +ny senate +nou mea +north co +nac elle +na shi +mygov india +mumb led +mother sbaugh +masa ko +ma thes +m wy +m dg +loi shua +lee jonghyun +knick stape +juli ere +jose fine +jan sch +jamesra hendry +j ng +it amar +i beacon +hot dog +hoo e +hi mal +hert zog +hel plines +hand stands +gr annis +global ised +gab ardine +g weru +fred ricks +fo t +eye hate +ev cen +en eco +en dian +eli ason +electroly tic +el puig +eidol on +ed dings +drin kal +dre ric +dar vin +dani al +dan ser +clutter buck +ci k +che eta +cele br +board masters +bo bol +bi ao +ber te +back britishfarming +baby gift +at tie +ar drey +ann aw +all indiab +aj r +. ðŁİī +ðŁĮ Ĺ +íķĺ ìĿ´ +yan bu +yadv ashem +will acy +ward ley +vine et +ve eder +v tt +usafric abf +tx educhat +traffic crash +todd whitaker +ti dur +thr oneof +thisi shome +taylor momsen +t sports +t jackson +swiss re +surviv ability +sul is +sublux ation +stagn ated +sno g +sk telecom +size well +ship builder +sharks za +sam sam +saint patricksday +sacchar ine +rye ong +runner bliss +rose bay +roger stv +ran x +quoti dian +qip co +pub crawl +produc tiv +pri vee +pre y +pram ila +pra bu +past ner +own voices +oliviach ow +official rezz +nil am +night bird +mo tter +mo tet +mith ril +me guro +mc niven +mau g +mar gy +man music +lou bet +lion sclub +lar ock +l bb +ko caeli +kitt i +kid slit +khamoshi yan +ker messe +kac zynski +jane ane +imogen heap +hol douts +hel oise +gu ria +goka iger +goal mouth +glamour maguk +flower photography +fire station +fern tree +fam es +extracur ricul +eve leigh +electro plating +dup date +dun bar +dubl inohio +do i +dia stolic +den ham +da ang +cthul hu +co don +clean tech +ca haya +c ses +bu ma +bread and +bing crosby +ber ridge +base plate +ball erin +bal fron +asseen in +ashley monroe +aq r +anil kumble +am dry +alo es +allmy children +alad in +adam richman +aap ka +ðŁĻĮðŁĻĮ ðŁĻĮðŁĻĮðŁĻĮ +è© ± +å ¢ +ãĥ¢ ãĥĩ +zo ya +you then +yor kies +y ster +woj cik +while youwere +wel cher +weight lessness +web marketing +wake fulness +vibe magazine +ventrilo quism +utt aran +ur on +transpor ter +tho pe +the ken +the junglebook +th awk +terab ithia +tb in +super storm +stru ly +stellenbosch uni +squ ote +spor um +shon telle +shad rack +servici os +schisto somiasis +sch rade +sau to +reci ation +re discovers +pul leys +plat en +pensac ola +pencil sketch +pah lawan +osucoach meyer +opp er +o eh +mullagh more +mis behaviour +mil dest +mall in +madmax furyroad +mabino gi +loko ja +lic enti +l ru +kkkon colors +ker fuffle +kar thika +joseph us +ith appen +institution alization +ingate stone +iffe rent +idol producer +he iss +happy valley +ham at +h sas +geton mylevel +g cap +ful k +free to +foie gras +fly pal +fc stpauli +end yk +ehl inger +dub v +dou that +doc week +din din +die hard +die bold +di sharmony +dhan raj +deco ders +danny pudi +da ik +collar oy +clean beauty +cin zia +children sla +car share +ca che +busines speople +bt as +br kfst +bor ris +blick ling +bill inge +bell port +be sta +bau com +az c +ar sons +ap ak +anim ism +angkor wat +ang pilipino +ang am +andi ka +albu mart +ðŁĺİ ðŁijĬ +ðŁĮ ĸ +ðŁ¤ª ðŁ¤ªðŁ¤ª +ìĹIJ íĶĦ +京 éĥ +ت Ùħ +á r +yani v +wind jammer +wil ayat +week uk +us movie +un recognised +tu cuman +toi mumbai +til ford +thom asians +the ys +the forum +tetra hedron +tat weets +sunny slope +sub stratum +su bba +stubhu bcenter +stro mal +strengthen er +star ched +sri jit +sig fox +shrew sweb +show boating +scry pt +sag af +rox anna +ri ft +re ju +puertor ican +ps lon +pro meth +pin ball +pend se +pat tim +outw ards +ol atun +of ir +obl ation +nu ku +ner fed +naughty america +n ä +mw ana +mv rp +miss this +merchandis ers +mascar ol +magen ta +m sha +lu sted +lou ps +life crisis +ley endecker +levan ter +les miz +le tha +le brock +lc bern +lcbern alo +l icia +ke ko +justin baldoni +ju sco +joe bob +jeff coat +intere ss +inter bike +im no +id hu +hh d +hetero sexuals +hesit ancy +head way +guillo che +go wa +gag genau +free app +fon z +file system +fe stin +f mw +eu st +escal ope +equal sfreedom +enjoy illinois +ef fin +du sen +dro pin +drew ry +dis order +destabili zation +de sain +daysuntil qatar +daily quotes +custome rengagement +cic cio +buder im +book con +bay h +ax im +att enders +ak su +ak chod +aim es +aero gel +! ðŁİĤ +ðŁİī ðŁĴĸ +ðŁ¤· ðŁı¾âĢįâĻĤï¸ı +ìķĦìĿ´ ëĵ¤ +구구 ëĭ¨ +á´ ľ +z are +wurz els +wsu pullman +wing sof +whyi march +wan de +vill an +un reality +tru sh +trop med +treasure hunt +the thing +the acc +tetra zzini +ter ris +team titleist +tac eae +ta here +synchron izing +swoo plife +strand bookstore +steril ised +steel yard +star set +st sci +spy master +spring forward +sp ils +soulful house +social responsibility +sme ar +siss ons +sid grauman +sibb ald +shin wari +rsv ps +rough guides +roh tang +riems dyk +resin ous +rehabil itative +regurgit ate +regal movies +rainbow laces +ra ffel +pur fleet +princess diana +power systems +post menopausal +pope ye +pere grin +pan isse +pall bearer +ober land +ob st +new biggin +music scene +mun oz +morning ton +mish ere +metho dist +mel ani +make som +mac iel +m laden +lux ton +lmm fao +lie big +leeu w +ko hen +kad hi +jovan ovic +john piper +jeppe sen +it ak +io ve +in dre +huang shan +hans zimmer +han afi +hagg ai +ha rel +gri mmy +gra us +give th +gen ove +ge saffel +gar n +functional medicine +fri ede +framer ate +fo yers +felipe melo +fault line +faul ted +encryp ting +eby zio +devel ope +deser ting +deni alism +den si +deep dream +dan the +dag ny +cyno sure +cherry ville +char line +cancer bats +bur un +bram lett +boroon dara +booth by +ber gg +ban ken +bal int +ayo shi +attune ment +ar lfc +ant witter +annas ophia +acry lamide +abc de +aas l +[ ðŁĵ·] +ðŁĵ± # +èĭ±èª ŀ +zy gote +ze ts +yed chat +ye sler +yay ay +w sk +vaudre uil +vaness amerrell +v magazine +usch o +up gradation +turt len +ton ly +thir deye +team english +te very +syco ph +stri ped +staf fan +smo cks +sketch book +shu ker +sch nur +rob bert +ro ft +reve rently +refr acting +recer tified +ra yer +py romania +pix i +pente cost +parod ying +pan ka +omidy ar +offici albull +officialbull srl +no kian +myas thenia +mur g +mu stre +miti gates +minne ola +mend onca +mem ling +mc george +mb en +marke aton +le ches +laur inaitis +la ak +kol lar +kirk ley +ki xx +kelly ville +iwan rheon +institu ting +im brettdalton +hy pom +home studio +gn awed +forum nyc +fli ghted +flec ks +fl anger +fab four +edu ar +durham college +dom mett +digni dad +digital singlemarket +cultiv ators +cu zz +crewd son +creative review +cole brook +cl é +cel la +ce to +cas ares +capacit ance +bru ton +bla sco +bla sberg +big show +berg son +bel don +bblo fficial +bacteriopha ge +aqu id +anti pasti +amp oules +ag ym +afl crow +adden brooke +> "@ +ðŁĶµ âļª +åħ¥èį · +ت س +zal man +y une +xi bal +wood brook +wo burn +web isodes +war g +v dub +unci ations +twilight sparkle +troll tunga +tele sur +sy ston +studi es +stro mer +stral sund +stpat sfc +stand o +soviet union +snow falls +sle iman +slan ting +sixword story +sh oneys +sarcast ic +ruba diri +road nats +regi o +ray u +promo tocross +prati que +po prock +pear man +pe go +paul pierce +param oun +p nh +ou as +oli mar +odel rosario +ob ay +o san +nauti yal +mo ze +mau ch +m ssa +love songs +les que +lanca strian +kun ai +key t +kar amel +k ne +jonah ray +jo t +jim al +j kd +info graphie +if only +iden hout +huub design +humb lest +high boy +gul liver +gros mont +golds berry +go tr +girl probz +fro thing +fri en +floss moor +fal tered +explo sively +exemp ts +ex itos +es en +erin dale +enr anta +elstern wick +eleven se +elec tor +dig deep +de brah +david muir +dav alos +d cn +cow ens +confe ss +con traflow +chittor garh +chiro po +chima era +cer veris +cani bus +cal gon +cabare t +brandre th +bicker staff +ball i +bac cano +ati fs +atel ateshow +at resia +assn chat +anglo gold +andalu z +an kari +amdry zen +amadeus itgroup +alv ador +accred iting +ðŁı ® +ãĤ¿ ãĤ¤ +âľ ĵ +à¹ĥ à¸Ļ +ار ÛĮ +¥ o +yo wie +yan et +world wi +wigg ler +war lingham +w iry +vande mataram +vaish navi +urva shira +uch us +tour ne +toot sies +thess aly +the kenny +tgi fridays +tail piece +symbo list +suppor tin +sub sp +sp ons +sim kins +shar mon +sf wa +sar apascoe +s march +rev ans +reid hoffman +psy c +poison ings +phospholi pid +pain lessly +pa zzi +oto ño +orl pride +om il +oc cam +nur kic +ns dc +ni mue +ne ith +nah ant +mon ito +mom ina +mmb ht +missjess wright +minchin hampton +metro park +me trix +mau ra +mar ras +mani ax +mand uka +loin cloth +liber tas +lessi smore +lacer ations +kl are +kingofthe monsters +k rock +j lr +j jab +iti zens +in scribe +house martins +hesit ates +haydock races +hay makers +gra ils +glas vegas +gha uri +g pe +fran sen +for tomorrow +fle che +f nl +es war +encamp ments +ekstra klasa +eco logic +dor rance +dom ici +devi ent +cund all +collie buddz +co sin +circul ars +christian bale +cell therapy +carnau ba +capri les +cai ley +buff ington +boo dles +bo pe +biz rt +bir on +big dataanalytics +bey az +be hera +bal ne +arnau lt +apprehen sions +ani ela +ak kians +agh y +aa i +! ðŁį» +ðŁĻĮðŁı» ðŁĻĮðŁı» +ðŁij©âĢį ðŁĶ¬ +ðŁı ĭ +о ÑģÑĤ +youn gen +x one +willo spre +willospre ay +wil da +weareall harry +wahi awa +vel á +var una +van helsing +union chapel +un bound +tr ach +thread gill +the sharksza +the fish +ten chu +tellu ride +taver ner +tand ridge +ta ren +t los +t ente +stur key +steve case +sru thi +spiritu als +so lex +silver mine +sch wager +sarfar aza +redcros scanada +ra sch +py i +pru ri +pro lo +pepp ering +penguin awarenessday +ostraci zed +of ia +occupy dc +nh on +na sta +n cic +mo ty +mid america +michelrou xjr +mh ra +mam tay +maha shivaratri +madison beer +m zuzu +lul ling +lore t +lof africa +line as +ler on +lennox lewis +kri z +kle z +kh ancats +k outa +jurassicworld fallenkingdom +j rp +iv ka +iter ary +is lah +ino ids +imp ong +id wal +hoch uli +he intz +har ford +hae jin +h medabad +geith ner +gat ers +gam bled +fox hill +five star +emerson barrett +ell ena +ek ins +dj quik +confla gration +commu tative +cinemain my +ches ney +chen once +cer vez +celo sia +cas is +butt resses +birthday present +back down +as phal +ang aa +ambro ise +amandashi res +alpha phi +adam antly +! ðŁIJ¾ +ðŁĩ©ðŁĩ °: +ا٠Ĥ +â tre +zlat ko +you ville +yach tsman +y ic +xan the +whizz ing +whisen hunt +when they +wap ato +vicuni wgtn +ure sh +tul se +theo logically +theav club +swoo pe +swee tromance +star musicph +socialmedi atips +sk ul +sic b +shan aya +sequ entially +sekar ang +secretary meity +sat ana +santi bernardez +sa hay +s vic +rt j +rout t +rot j +ro ge +por lock +pis atower +pin kel +pel ota +pau se +outdoor photomag +on time +old trafford +o ore +no isier +national watermelonday +nan ai +movie twit +motor plex +mor wen +mon ceau +mom mys +milli metres +mat ina +magic man +mag tang +ly cam +love eeeee +llan do +little mermaid +lang ga +keith haring +katy b +joaquin castrotx +jo casta +jacqueline fernandez +jackson ms +j bj +istandwith ahmed +ir reverence +in shorts +hy er +hv n +huic hol +grizz nation +gra vy +gg v +gesaffel stein +fren chi +fir ings +f de +ent wine +elimin ation +ed gley +ec dc +dra gger +do sto +dis illusion +dal in +da we +cul zean +cro ats +contra bass +con ex +cap rica +bur zum +bridg ton +bri ant +brau tigan +bou bou +beau soleil +be ate +bat th +bag ong +awh hh +as ae +andy bell +amphi bia +amaz ov +alfie boe +ðŁįĬ ðŁįĬ +à´ ķ +ø ya +wu or +wonder bra +well com +wc zyk +w elive +ver ba +uniteand conquer +uni oslo +tz comics +travel india +trad cat +tra kai +to hru +tik kun +the girls +the far +tele play +team unity +te mora +taraw era +tan field +swachhbharat mission +sw aging +st ecker +ss ave +spine farm +sne yd +sn sh +sk atal +scu zz +sch lock +sb swinner +sa ale +rural crime +river dale +rel pool +ra jut +pre selection +pantal eo +nun zio +neutr alizes +nav otas +na jah +mu de +mtn l +moun a +mon agas +mind the +michaele aston +lu cus +lol lo +lo ftheday +lee h +le ta +laun dries +l losa +ktn lifeandstyle +kro k +kp fa +ko tigobba +kkw beauty +kappa sigma +iti ate +ing us +ice prince +hu mic +haywar dgallery +ha good +gre sham +gran town +goo die +glaz ersout +general news +gen gar +gell ert +flying dog +fire bird +far ri +fa hm +ey en +er hard +epile p +emo tor +dev araj +dev akshi +de colonize +couch base +coil over +cine mam +chiz uru +cellu litis +calom baris +bu fo +bre it +bill rancic +awa g +assemb lages +archam bault +ak ola +agne se +ach ines +ðŁĺľ âĿ¤ï¸ı +ðŁĵ¸ - +Ñĩ аÑģ +zet land +yel ps +wak en +vel den +vall ée +us lims +uni kent +tizi ana +thisi sco +them icky +theat ro +the frank +tam inas +ss am +sk now +sh aki +sed atives +sal ai +s rush +robin thicke +re organise +re nova +raz avi +rambo donkeykong +r anny +que ene +quag ga +power pc +po sie +peyton manning +pal frey +ori k +ok tib +o dess +nipa win +neutr alise +my cin +mesti zo +maz andaran +man ston +mamtay patnaik +lun da +lady ship +ko hi +ko chan +kav y +ka hane +jud kins +joo won +jak bar +ja si +inn keepers +in ness +hi vos +hal k +hackath ons +gue strooms +gu mmo +gas light +gal en +g bbf +future day +frick ley +flipk art +fi ef +fcgo aofficial +es fahan +edge fest +ed policy +eccle sall +earth moving +din da +diero ten +darby shire +dam os +credit score +col u +cic le +che main +bul o +bul lett +bron zy +bio m +beta wi +ben icia +bellamy young +bb bots +ball ons +baarba arde +at na +ar bury +ant age +anit adon +am cu +allu du +abil ty +ab ach +?! ?!! +ðŁijī ðŁı¾ +âĸł âĸł +âģ£âģ£ âģ£âģ£ +á´ ´ +z inho +yasi elpuig +wolfs bane +wma onedirection +wand sworth +video drome +vas ool +union square +un compromised +un adilla +uconn mbb +to shin +thatgame company +th air +tal dÃŃ +sugar land +star tv +st cuth +spi ra +spe zial +small hd +sho reline +se dm +sa dek +ross ella +ros aparks +regen bogen +president kovind +pre ico +polit icos +par ds +pa wh +oregon state +oppre sses +old pic +of fe +nu u +nr c +nic d +nay ak +monop oli +mill and +metro land +men il +mcpar land +mckend ry +matthew j +masi si +mar ge +mal acan +macmillan coffeemorning +ma thai +lü beck +lifeok tv +lie bling +le gio +laspal mas +kle mmer +ker st +kar ki +ju bal +john b +jenny packham +j ü +j oux +iv w +inti mate +incant ations +humph ry +hu mored +hu ggers +hagg adah +gra do +goo dre +fin min +ewe cisme +el ady +du tra +down south +dog spotting +dish water +cust is +cou peville +coono or +constan cio +coccy x +cam ira +bron zing +bran dish +borg warner +bom berg +blue mont +blue friday +big ness +be mine +bbvacom pass +baf tac +ash can +and care +altr arunning +allindiab akchod +acar ter +. )" +ðŁĶ ĵ +ðŁĮ¤ ï¸ı +ìĿ ¸ë +yvon near +yehrish taky +war randy +vive andalucia +ver ulam +valent ini +vale ting +ur be +tragically hip +tr ouncing +to tara +that ss +thali domide +ter ol +te ki +tam m +sto c +sti jn +sp rees +sou maya +sm oment +sloven e +science ctr +rob itu +robitu ssin +ritu ximab +ris borough +rein ert +rames waram +qui ero +quebec ers +pe ca +p ach +or man +onward lu +online gaming +old skool +no worries +my son +mor itz +mo li +mir zap +manojtiwar imp +mah out +le von +laps ley +krist tps +kor f +kom mer +kno ch +kan aya +kab u +jin ho +ist itu +invari ant +ineffec tual +indy indians +i bex +hon aker +high lin +hes burgh +hear tedness +h winkler +h fr +go dd +foo kin +ey ards +engli shri +en sayo +disin i +deser tisland +deb it +de camped +cumb res +courteney cox +cos kie +cordi ale +consul tancies +church goers +cf z +centr alize +cap as +canthelp falling +cam mell +bi vens +bbc berkshire +bb u +barnab y +ban te +av ore +astro samantha +asf andyar +arti an +army navy +apo phis +amund son +alcor con +ain tv +african fashion +afl tiger +ab oud +a this +# ? +ðŁĺİ ðŁĻĮ +ðŁĺĤ ðŁIJ¶ +ðŁĮ Ķ +ì¹ ĺ +è ĥ +ز ÙĬ +writ ting +wakeup call +wainf leet +verizon fios +undu latus +tro ad +theme park +taylor swif +tat um +t che +sy am +swo boda +sw bts +steel ers +so se +sm le +sk icks +side as +schie hallion +sar copen +sam hsa +ri ev +repaire rs +rat zenberger +raise droyal +quent intar +qual ys +pu stu +proscen ium +pressu rec +pj bowles +pang ako +pa peete +oãĦ ¥o +op ment +olo f +ok ur +nwalest weetsuk +no xu +ncl b +montju ic +milli kan +mikkeller beer +men tof +mass eria +marsh on +mar aton +male gaon +mahan adi +lyo to +lot ti +litu ation +life form +lee hi +leak ages +ku lu +knight pride +ke dron +jun it +julian clary +jud son +instig ators +hu gu +hot wife +hope college +home work +he ge +hc ska +gur dji +gargan ey +g tbicycles +firesi de +fides z +farn ham +eu ijin +erec ts +emb olden +em enike +dou dna +don di +disfru tando +dho far +dev dutt +dd orp +dani ka +cy bill +cson ka +corn dog +concer ned +chee ta +cc fest +c jb +by am +bux us +bre anne +brad don +bossi er +blue water +barre iro +b politics +auburn tigers +arden ne +ar ven +app ened +am om +aivo arm +/ /# +ðŁĴĭðŁĴĭ ðŁĴĭðŁĴĭ +ðŁij©âĢį âĿ¤ï¸ıâĢį +ðŁ¤¦ ðŁı¼âĢįâĻĢï¸ı +çĿ Ģ +ãģ ® +w sav +vegas weather +upanish ads +uniof brighton +under handed +un j +too wong +tath agata +sympathi zes +sudan massacre +stop tober +ste yn +sol or +shi moga +shark science +sen ya +sen rehmanmalik +sch wi +saqib saleem +sad ams +s vil +romp in +risk on +repri sed +rele m +re possession +re buil +rapi des +r ri +pir ouettes +oxic lean +open ai +older people +oaken shield +now icki +note pads +north fleet +nor ville +nawalel zoghbi +n mireland +mott ola +mind scape +min ter +milit ari +men kes +lympho cytes +lsuf ball +lesar cs +kow ens +kh ir +keaton stromberg +ke tsu +ke ay +jar ano +istandwith israel +iron i +iceprince zamani +ice field +hil ma +hammer sley +gay athri +gabrielle doug +g leave +fu ssell +fro mc +free ebook +f sh +exofan art +epi thet +ei dos +discipline equalsfreedom +democratshate america +defence day +curt smith +ctc mediaboy +crew men +con on +colorado live +ci arab +chem society +car rero +bur rough +boo ke +bobro ss +bj w +benedic ta +beat nuts +be ed +bank ston +bad gering +art trail +apo stol +anky losing +angel ov +an heuser +alfaj iri +agn ello +abim bola +aa ve +? ¿? +.. ?" +ðŁĺ³ ðŁĺ± +ðŁij° ðŁı» +ðŁIJ§ ðŁIJ§ +ðŁĮ¶ @ +à¹Ĥ à¸Ń +zak ariya +yu yu +ys r +ye dder +x pt +wich man +wcc w +vr la +tr ès +tom kaulitz +theceltic manor +tam baram +sun way +sp j +skatal ites +shak o +sean an +s oooooooooo +ro sca +q un +punjabi rooh +peru vian +peri vale +pe ameal +pap on +pa quito +one towatch +nic ols +nh lers +national tacoday +nate berkus +my sjaday +modern life +mis kat +mar mol +manzan ar +mahi dol +london birds +liz mcclarnon +kochadaii yaan +king swear +kin ta +kar y +jami em +j crossover +it smi +islam ujeres +instra gram +hems well +healthye ats +hav licek +haile mariam +ha itham +h kust +gun an +gul man +grze gorz +grena dian +grav ell +gli ac +glam ori +gentri fying +g illa +forge din +for theride +fe thul +f ttp +ez ri +explo rec +epilepsy awareness +ekkle sia +drexel univ +doni phan +don nee +do onan +digis coping +de sau +daisy ridley +da j +culture days +cul ley +compreh ended +cho g +chic hay +cel les +ce devita +ca ver +bun ited +bun cee +br ice +benne teau +ben sley +bb k +bal dur +aw aka +arri ola +ap ko +ane ela +andrew schulz +allahu akbar +!! ~ +çĮ « +âĺº / +youtube spac +your quote +yn l +yag nik +xeno gears +x mm +where of +u vas +u cn +txh sfb +tw ila +trum ple +ter res +tempor ally +tani sha +sten cil +stand withrand +sque aler +sla vs +si pi +ser vic +sel ah +sebasti andan +sebastiandan zig +se tya +schu ur +ryan vaughan +russ ert +ru sk +ro derick +raven symone +rac is +r bs +provoc atively +phor aone +pet ted +part time +outh florida +ob ong +nic orette +next chapter +naraco orte +my burgh +musc ling +mur ine +movie tv +mirac leman +mi b +mcpart land +martin j +mart one +mal aka +makar ska +lu kman +louise redknapp +lor ong +livel iness +ld pe +lach lan +kud zi +khar an +jo bbing +jas par +hog town +heriotwat tuni +hazle hurst +h ca +grosse st +gn oni +el niño +ek afka +e ja +du guay +dr atch +dirt car +del val +deathwish inc +de coupled +co hl +chen yuk +cha eng +bri slington +blood sugar +bezu idenhout +ball rooms +bal ak +at cc +as kins +ap mc +alphon stourism +alham dullilah +al don +akiz aka +ak ville +agu ez +ad ae +ac inemas +a hahahahahaha +ðŁĺĬ ðŁijı +îIJ ķ +ãĤ»ãĥ¼ãĥ©ãĥ¼ãĥł ãĥ¼ãĥ³ +ãĢij ãĢIJ +âŀĸâŀĸ âŀĸ +ภ¨ +zi yech +zi bah +yogate acher +yi pee +wärtsil ä +would you +wo za +weather field +wa edu +v ith +transduc ers +the silent +tait ung +sylvan as +spic inemas +sne e +sky dive +sbswinner shour +sad ashi +robin tunney +ren yc +re vul +ramire z +quit aine +queen rania +qu har +pro era +pro creation +pop tart +plo tt +p agu +ono dera +nursing school +ne men +natalie grant +na gh +mun ising +mu lino +mit so +mind fully +mc griddles +mc feely +mart lesham +mac ou +m spaint +lun cheon +lovel orn +lin ky +lat tices +laf ite +kashmir bleeds +joesp ub +jenni woww +j era +isur ppo +ine aux +in nately +ig ley +heisman trophy +hanni gram +hamble n +grac ec +geo science +gend ron +g pc +fu d +form alized +fiendish ly +fall game +evry one +et yuk +enye ama +discol our +detroit tigers +custo des +commerci alizing +clau rak +chel i +cer am +cap ability +cam ier +bra sen +bio process +beauty queen +bb claurak +bargh outi +bal amory +av ra +ar hi +ano dyne +afri kaner +a sexuality +é ns +win elife +white friars +wan z +wall ner +w enty +victori aday +v ris +ur ania +under achiever +ultr atech +twer ton +turk cell +tro th +tne du +ti ote +taminas nuka +tab ilis +sy rin +sum pah +streetfighter v +stone arena +star cross +sma shin +ske ws +seren ata +scrutin izing +sand tats +ro cc +ric ca +ri kara +reviv alists +rannvijay singha +r bm +portu mna +por s +phin sup +pen ni +out shot +oju kwu +no hotch +no cs +niz wa +n cla +my lor +music notes +motor craft +mon chele +mit am +mission possible +metaph oric +masi yiwa +manfro muncle +mackin ac +longboard stuff +len ox +le blond +le ask +lap kus +ku fuor +ku ai +knuckle heads +kis wahili +jim mer +jessica simpson +jar ama +j day +hlat ahai +hell scape +health fdn +haus mann +hacksaw ridge +h enty +gurdji eff +gentle woman +geno typing +game trailers +gal vin +freak azoid +fi ware +fcc ps +fashion trends +f anni +esp r +elo ves +early voting +ds q +drain pipe +di fy +daniel platzman +cott man +co tuit +canal plus +butterfly fish +bot elho +blackgirl nerds +bi man +betsy devo +best meow +bbc wiltshire +avan am +aspi red +ar yeh +ap mso +aor aki +andreab ocelli +anah ita +ake hlatahai +ai weiwei +agra bah +ag ada +affili atelink +ðŁ¤ ½ +å Ĥ +ãĤŃ ãĥª +ม à¸ŀ +Ø´ Ûģ +zan ussi +ys avage +y anna +wf sullivan +weare messi +wdy tya +wat usi +visit rwanda +vanilla ice +u rious +turtle beach +tu eindhoven +trique tra +tri duum +tipsfor new +thetommy dreamer +tema gami +svi zzera +suppo sing +step well +ste gall +sound box +socon sports +sho whouse +sh urt +sel fridge +sche els +rob be +ren derman +re ff +ray field +rasp bian +pu u +pressuri sed +pran it +piggy backing +par vo +pai stenation +one es +nivin pauly +nau voo +n ti +n sh +my calvins +mkt gnation +min ns +mi af +meg apolis +madawas ka +ma irie +lit mag +le them +kier ra +key akizaka +ke ila +kayak fishing +kar gentina +jä rvi +jeff mauro +jacoby shaddix +j itter +ish ta +in case +homos assa +hitch in +hi as +hat tie +gh n +ful well +foun dress +for newindia +entoura ge +embar goed +em era +ely sian +egg sy +ed wyn +e mina +dv all +disintegr ates +din na +dg classics +del toid +ded man +davi dicke +dar am +danc zuk +cur to +cro i +critic schoiceawards +cour onne +confu cian +ci ba +chi oma +cent in +camel bak +cac o +c ni +by bike +bi ii +auburn u +aru z +arro whead +ar ace +angar dens +alle gre +aaaa ay +ðŁĺĦ ðŁİī +ðŁ¤ĶðŁ¤Ķ ðŁ¤ĶðŁ¤Ķ +ëĮĢ íľĺ +âļł âļł +âĻ¥ âĺĢâĻ¥ +аР¼ +zo sa +yy ates +youtu betv +young jeezy +you the +yo shimi +y iss +xseed games +wel che +vis y +vermin tide +ty outh +tro on +tou sa +toa sties +to bu +the forks +tap low +tab ular +sur ma +stanley cup +sta o +sprezz atura +spit toon +so bra +sk ane +shou rie +shoebury ness +shal lower +send help +scrutin ise +sanctuary asia +rsac onference +retic ent +red nose +re marketing +ragg amuffin +proudof you +pres age +pot ters +phe be +pescat ore +paradise papers +orient alis +oo die +non no +nin na +musi ka +mother lode +mire les +minu it +mi des +mf h +merri on +mediev alist +masterpie cepbs +marie mont +maki as +lu ssier +little girl +li des +lan than +krispy kre +kle infeld +kiyom izu +kah in +iy ad +it awamba +iso cial +in patients +ime che +ig li +hol bert +ho ty +hi jama +hi iiii +heresi es +head hunting +hand sof +hag strom +gre z +geck o +fur tive +free scale +fre ey +fei ffer +fal le +f dle +ever man +euro bike +eti ology +et canada +ep son +engle field +el ook +din gos +cul hane +cro fting +cour tier +community spirit +co fer +christ fic +chim er +canar sie +camp day +cali bers +c gw +bu mbu +boer boel +bir nam +bi enne +bern heim +beau sejour +b appi +azi kiwe +avoce ts +as mode +arch diocesan +anu bhav +ak ri +aha fo +agar ci +ad os +acar los +ac em +# [ +ðŁį» ðŁįº +âŀĸâŀĸâŀĸâŀĸâŀĸâŀĸâŀĸâŀĸ âŀĸâŀĸâŀĸâŀĸ +yoga inspiration +x z +wer sching +visual ised +visitu ganda +victori aave +ve sel +un troubled +ump teenth +u acc +trolley bus +transp ire +tradcat knight +tr indade +the moon +td bank +tai sho +stro bo +stra ka +st g +southern stars +so quel +sky scape +shi ina +shar jeel +sha ped +sent i +sam csmusic +s loo +rumin ations +rt ve +rol f +ro ding +ric ke +rhi zom +revolution ist +recogn itions +real pigfarming +rble ipzig +r tb +pyro lysis +pv z +pre pubescent +power puff +pic hler +pd w +parti t +paralle lism +pam an +ob eng +nwm waypolice +no watch +no guera +nicole byer +ncb wa +nam it +mu res +mot son +mik ro +mi dy +mersey travel +man den +lee hom +kvo awx +kun try +kul am +kingsme adow +ki ang +ker ala +jus reign +j asons +is chae +imper atives +ij at +iced coffee +huck le +homecoo ked +his ss +herald leader +glend enning +gey sir +gener ator +gar bi +gal chenyuk +ga thon +flori dian +fel ker +f tb +estre ll +er rington +end ine +elli avram +ec khar +duchessof sussex +dry cleaning +dire tta +del tab +death stranding +de gnan +dan as +dalla sschools +ctb b +constra ining +coin drop +chic ho +cfl s +catson twitter +carol decker +care quality +call sthe +c cre +bubble uk +bre c +bin ondo +bel inelli +basel itz +baal bek +ast ars +apach ekafka +ann woll +allo yed +ale sha +ack ay +abhay deol +.... :) +ðŁĴĻ ðŁIJ¾ +ðŁĴķ ) +ðŁĮ ĩ +åĴ ² +à° ° +ö calan +ziller tal +zaid hamid +z dar +yu kie +win kelman +william zabka +wei qi +van ç +val let +u tau +twitch retweets +travel awards +trans dayof +tiru pathi +tan felix +t seva +suzu miya +stormb re +stormb lood +sto h +stan thorpe +sp ang +sou lar +snow blind +sira gusa +sign boards +shir at +she said +seic aday +sd lplive +scher r +sapp eal +russ y +rol leston +rod inger +rik ke +re gin +re enter +r nh +r ari +r allo +pon oka +politic snation +ober to +obam a +noxu bee +new scenter +near sighted +nai vely +mis leads +micro blog +manbear pig +ma dia +lo kal +legion ary +lawrence town +kir sti +kalam unda +jic ha +jh us +jennab ush +inciner ate +immuno therapies +hi ff +he wwo +har twood +hage man +gre ce +goo fy +glack ens +gin ghs +geelong addy +games don +fren sham +et ce +ent rain +em elianenko +eastern ghouta +dream ing +cosm in +com ence +clough jordan +cho zen +chicken foot +certi fies +cambridge analytica +brook house +bow sher +billyray cyrus +barrac lough +au han +ard ingen +ap ron +ann am +anj ir +anim alier +amer chemsociety +al gar +ah la +aff ney +adore delano +adam ferrara +about you +a eta +! ðŁ¤© +« à¸Ļ +with holds +wis fb +visitt ampabay +vermon ter +twit z +tw ays +turn bc +ttur tles +tt t +troy ave +tro ya +tom delonge +thestar kenya +thereal daytime +than et +te mberg +son orous +sok onews +sobot ka +sla sher +sf as +service women +sent sov +roar lions +ro dos +rith vik +reef ers +re zone +re tread +re gev +ranch itti +prin cel +pr j +porcel ain +peop lenow +pent ine +pe conic +oun try +open door +o town +nusrat chirps +nsw waratahs +nom ex +music tech +mo ville +mir tz +mh ingis +mass o +maris cal +man ero +magand ang +mac mahon +ma ah +lt museum +lolo lolololol +lin dow +le pa +lappe enranta +lanc elo +ko bi +kin ne +katy turnbc +ka inan +k yser +jessic ad +jer se +ix s +immun ity +hye sung +hut chin +hu tu +hel imed +hay ati +har n +gir paten +ged don +fro ot +en antio +e strange +diver ge +dis membering +dha hran +desk og +de wdney +cre gg +cor ry +conceiv ably +commonwealth sec +coal inga +cinde rella +chi pp +champag nes +cal man +c shs +burge s +bre don +bran ly +bombay sunshine +bli xen +blanch flower +ben cy +bal da +b ated +ax um +auchin leck +ar cing +appare ls +apho bic +aor us +antic om +annies land +am wa +al labout +actor dougjones +a high +ðŁĻĭ ðŁı½ +ðŁĺĬ ðŁijį +âĻ¥ï¸ıâĻ¥ï¸ı âĻ¥ï¸ıâĻ¥ï¸ı +ઠĤ +whi ppets +western cape +wazz u +wate rer +vigil antism +ven go +vasude van +vap elyfe +v lf +usp icious +unionchapel uk +un quiet +tre forest +tothe future +the train +tain ment +su raksha +snit zel +shaku hachi +sam pan +safe t +s frs +rh in +resc ape +recor dist +quarri ed +pom pa +pol tava +photo chromic +philippine star +pan niers +p fo +op ole +only the +omor phs +om id +o valle +not ill +not arized +ni ku +ne wi +nbad league +nas ag +n ju +mor ire +mohawk college +mo reco +max ims +man awa +madal ena +mad hur +ma pple +m jj +m cap +leather goods +kry pto +kou ign +koo yong +kil cullen +ke aly +kc ca +kal impong +ju el +john no +ittake savillage +iti k +it stime +ing res +ic and +ho tei +hen rich +hato ful +guerri eri +golf chat +golden hobiday +g crf +fronten acs +flor ina +ers dorf +emanci pator +do dos +dn rr +dal hart +courtau ld +counterfe iters +cle v +cendr illon +ce ano +cade te +butler mbb +bru hhh +bla kley +bi valves +bas cule +bal lista +b wal +army team +apprehen ding +antho logy +ala jones +ai ha +" = +ðŁĺ¬ ðŁĺĤ +ðŁĩ²ðŁĩ ° +îģ ĸ +ãĥ¼ ãĤ¢ +âģ© ) +ÑĩаÑģ Ñĭ +zen as +ys man +yer ry +wy lambrewery +whib ley +wey land +wawa see +wah ls +w valderrama +vt beer +traxx as +themicky dolenz +the weekend +te mas +swing ing +style guide +stop es +spotthe shuttle +sn stuff +sm ilo +shaw in +scrutin ised +screw tape +schom berg +sch or +sav anna +sa hel +s derot +rin ce +rene auberjonois +rei mann +raj endran +r nd +pru e +pre sta +pis ci +peli gro +p ct +ou sts +oriz aba +open water +onto pp +noise makers +no kian +national ised +moon stones +mil one +middles boro +mi sic +mar ve +manis calco +man alive +machiavelli an +lo blolly +likefor likes +lambeau field +la blab +kn aus +kasabian hq +jo vana +isthis reallife +ini ki +hel ig +hae user +gre yer +ga ethje +for bearance +fly high +fit show +fin ley +f ng +ex ito +eith ne +dul ly +dul cet +dracaen awines +doo fen +do dder +digital spy +devin der +desi ree +dejavu love +def min +cy hi +cigar illo +chun soft +chitec t +chir lane +bur bank +bo ite +blue print +blaze pizza +bir doftheday +bendel acreme +ba ÅŁ +az navour +ay inde +asian food +as aad +archa ea +anton yms +ang ham +an ett +aldub dejavulove +absor bable +a dism +) ). +ç ® +Î ¸ +é tienne +z akes +ys g +you uuuuuu +yor n +ye eun +xi umin +xen osaga +william son +wheat stone +wex po +wdy wt +wall street +wa aaa +vo tel +tric orn +tof ed +the angel +swa thed +sulph ur +stormb rian +south kensington +sout doors +segui mos +science and +school safety +sb last +saty endar +satyendar jain +sar al +ruru madrid +ru lez +rev jjackson +rei der +re packaging +r music +quentintar antino +pu lak +pop con +pati dar +pam ore +pakv sa +paddle fish +om nis +ohiostate hoops +of all +oakland raiders +nu on +normal ising +nin an +ni az +new life +naac pi +muzz les +mu min +mo pani +miy ano +minim ised +me ji +mccly mont +mar illa +lu gg +lou dand +lleg ar +lets dance +lar aine +la sports +kon ami +kel kar +kan sa +kal ym +isle ta +islamic finance +intra operative +hope solo +hog ans +ho ic +hi z +henry rollins +hell bender +gur bani +gun ga +geis inger +garban zo +free bobiwine +fo ci +fen se +evo te +eucaly pt +er yn +er skin +el makias +de standaard +croy als +cow slip +commer ical +citadel le +chri sley +chihay af +castle milk +car drona +cap ron +broad stone +bri anger +behind bars +aur on +ath saust +anjan aom +anae mic +ambu shes +*____ * +ðŁĺĺ ðŁĺī +ðŁĴĭ ðŁĺį +ม า +wi ba +vyach eslav +vel ha +usl pro +tre go +the ating +team sideline +sun yani +summer ton +social business +sloven ia +shim za +roosen daal +ro gal +rivalry week +repet itively +redbul lair +re investing +re engineering +ragn bone +radio graph +radio day +racer back +pon ferrada +pic ta +philadelphia eagles +per les +pent ameter +pan tie +ou ttv +ok leg +ok at +oil seeds +noi seymusic +no ths +nieder sachsen +negli gee +nau seated +n mh +musk ox +mukun dan +mud larking +monu mentally +monte sano +mis cavi +mind bender +mi ele +mel en +mc fe +manmohan singh +make lele +m tor +lilnas x +li etz +lanthi mos +keem star +kan aa +k ander +jimmy barnes +intre sting +high point +helpin gothers +heart ly +guj rati +gu rashi +gi zzi +ga elle +frontpage stoday +fra y +fool hardy +foo ts +fly withus +fi ume +fathersday gifts +fabi ani +ek berg +dz ong +dram ani +digital diplomacy +del ite +community matters +channel seed +cbc p +bus boy +boru ssi +bo band +blogger bees +beam line +balla gh +balbo apark +aw adhi +as ntm +adv a +] âĢ¦ +( + +æĹ ħ +áµ ĩ +á´ µ +Õ ¡ +zig go +z ta +womenin history +what ttt +web server +vla ardingen +ve eck +upper classman +uni sphere +tu zla +tu aran +trul lo +tor me +the sky +th kapoor +team bc +tam ago +stin do +ster mann +stan wick +spee dier +sol tan +sn are +siddhan thkapoor +se kai +sd news +sche ffer +scan avino +sansk riti +rud beckia +ru fa +ro barts +radiof ree +ra zing +pur wak +psychop harmac +pre recorded +pou illy +plat formers +per ino +parok ya +park stone +park jimin +param aribo +p nut +oxid ase +not today +next step +new week +na tha +mit ri +mi ha +mer lino +mehrang arh +mari ama +ma sab +m ative +laur in +lam kin +kor yo +kor sakov +kor bin +kem pe +kei go +keep texas +ka aya +jennabush hager +jangh yuk +individu alistic +il sen +il a +hor lick +hongkong protests +hill ery +happy dussehra +gur us +god hra +gli m +gle aners +geor gen +gav ilan +fre ars +for bade +flo es +fidd ly +empir ically +el iti +ea stri +disp assion +dindi gul +denni stoun +deer skin +cru ller +che sson +c gu +bora hansgrohe +bir gitta +bel ur +bel leau +be moaning +b cy +algin ate +al ae +ab elli +; ') +ðŁĴ¯ðŁĴ¯ ðŁĴ¯ðŁĴ¯ +ðŁijĢ @ +ðŁĮ ° +îģ Ĺ +âĸ ¼ +ุ à¹ī +É Ľ +wroble wski +women fashion +wimp ykid +whiteri bbonday +weizen bock +ve i +tru ee +ton ers +thepc clondon +te ed +tas is +stadi umau +spotify playlist +sper ms +sken worthy +sk omo +simple ment +sikor ski +sig nore +she tty +sch ur +sch eller +sara hin +sa stra +river cats +real cider +re ily +protestan tism +po conom +phetch aburi +per ia +pac t +or tal +o bras +new lin +ne hal +nar inder +mv choops +moc tane +mo cker +melodi festivalen +med alla +me lek +mcken zi +marcell in +lu pines +lo pped +leed scity +la ibach +kitak its +karan ja +johnjay college +jo or +ja key +international mountainday +iclass ical +hype m +hy sics +hash browns +guy brush +guay nabo +fuk rey +frit illaries +fran ki +flight path +feil ding +fav o +exotic cars +early morning +dive sting +dave east +cé dric +cul ham +cobra kai +che pauk +ce ti +cat aldo +canon bury +cake boss +by as +burtonalbion fc +brit a +bit rate +beh nke +be vins +be falls +be bel +b pm +atom os +at boshoff +ap helion +an tequera +an sin +ah il +agameof tones +afel lows +ab atic +ab aad +a weee +// âĢ¦ +. ðŁĺĮ +ðŁ¦ ¸ +world govsummit +wo ss +with ou +wil fully +van ja +un addressed +turnit up +troglody te +touch of +tor ney +therealmike epps +thegreat outdoors +the h +te ven +tan dem +syrian children +sub system +street outlaws +strat agem +stop trump +stigmati ze +stephen marley +squaw king +sport scenter +sin namon +shar ry +sen jeffmerkley +screen prints +sch elle +sap ariba +rou ghead +roo tless +rivers of +ri sco +rhy olite +re integrate +radham ohan +quin nell +purple bricks +prod ded +power slave +phi er +paralym pians +pa ther +ou an +orke stra +nudi branchs +national chocolateday +nar la +mu ta +mo q +mili aromagna +mile sluna +meren da +manipul atives +magento imagine +lor al +lin dahl +ligh ten +life as +lhot se +lgb ti +leaf cutter +ky let +ku tu +kn u +kiel ty +kex press +jer m +jak ku +inter locked +hone sty +hi yori +hau ff +goknight sgo +go kwan +fore foot +figh ters +el lement +ei du +ed icion +e ita +degen kolb +de cried +d old +costel loe +cher n +chaud hari +chang won +chancell ery +cameron diaz +cam illu +by t +blu to +blackgirl scode +bi den +ben nis +ax ess +artemchig vin +ari q +anti semites +annen berg +amir ul +ami don +af ine +ad hd +action on +ðŁĴģ ðŁı½âĢįâĻĢï¸ı +ðĿĻ¤ ðĿĻ +⼠ħ +âĹ Ķ +âĢ¢âĢ¢ âĢ¢âĢ¢ +x inc +well ings +we ga +vas arja +val ero +v lan +usa ha +twitch online +tu ille +transpo sed +topp dogg +tele casted +tatt le +tar ang +swind ling +sum mum +subha dra +stran millis +sof italy +so can +sil kair +si eving +shiro gane +sep timi +s vidler +ru bra +ro tonda +re men +rapid deal +py thian +public transit +prostatec tomy +pmp st +pav lyu +pathfinder rpg +p nca +or dsall +op fun +nau l +nau er +moto gp +mossel bay +men en +mariote stino +mar to +mar git +mar con +madein india +liby ans +le we +lang ella +khi zr +kawhi leonard +justice reform +john sentamu +indi ag +iban ezofficial +iam up +hot fix +hor sfield +homestead er +hm treasury +hiro aki +harpsic hor +hal y +gw enda +guide posts +gt chat +gladi ator +gi gu +fan cher +f zs +ew t +emirate sair +ed mundo +dna day +dept ford +deep ender +de ichmann +dariof ranchitti +d mas +d mag +cul ross +crohnscolitis uk +controver tible +concentration camps +collegi al +cla rett +chill n +chihayaf uru +cher s +cheat day +chach ki +cast agna +car rico +car load +c ck +business forsale +bn ld +black girl +bay ona +ana res +amin ta +al tag +ak ilah +ab ic +ðŁĺĺ ) +ðŁĺIJ ðŁĺĤ +ðŁij©âĢį ðŁį³ +ðŁı ĺ +ðĿIJ İ +ê· ¼ +Î · +yaaaa as +vel tins +vag ner +un tu +un quote +ty er +tsu yoshi +tro va +trail blazer +tit illating +thin kin +theware xo +thes ats +the ca +termin ations +sudha kar +star ley +sr pg +sj h +shre ck +shameon you +shah ed +sar kari +s jo +ru sthall +rol lon +rish na +revealed rec +re distributed +pull down +pu po +pro col +police dogs +phra gm +phil ando +pesc ador +perki omen +paraÃŃ so +oke x +oc ell +north westerly +nohotch nowatch +new tons +net z +ne uland +monster products +ml is +mir jam +mine ol +make amovie +leic spolice +leci thin +leaf green +lare rtu +lad broke +knox rocks +kell yl +ka ghan +ir ac +hye jin +hidden gems +her in +he ati +h pv +gone girl +goldent icket +gil ts +g won +fr ampton +fe ig +fal la +f va +f peck +engel hardt +du mans +digital artwork +deri ving +deep rai +de activates +crum mock +cric k +cow bird +cor nes +co cu +ch ely +bun kie +bravo wwhl +bo canegra +bies mulders +berchtes gaden +be moan +bath spa +bar tered +au tz +amar go +ahrc press +af te +advi see +aas ld +. ðŁĺĴ +ç· ļ +âĢĵâĢĵâĢĵâĢĵ âĢĵâĢĵâĢĵâĢĵ +Ø´ رÛĮ +é tat +y anny +withdra wal +was san +warrandy te +w ason +un worn +un sent +u vi +tween ies +tw oc +time stamps +thankyou jesus +television acad +tele phonic +t line +sz ymon +super size +st davids +spo s +sonali bendre +slur ring +sku dai +signsyou re +short lists +sher d +shar meen +sens itisation +sen nen +se uro +sd w +s guy +ru ster +rou x +roc zen +ro eland +rimm ellondon +ray al +psych ometrics +po hang +peng elly +p wned +ospre ay +osc i +oo hl +o borne +ni dd +narrow band +mo ded +micro surgery +mentalhealth month +mb tc +m anderson +ljung berg +kla ar +kha as +just keep +ju beir +jr v +jami elaing +jam ena +itt ner +immun g +ic sid +ho pia +hipp y +he itz +flower friday +fi gg +fair ouz +ex trude +english bulldog +ene w +ellip ses +ech al +ds gn +divyan katri +discolour ation +cumbri an +cour y +col ma +clair sville +chi v +castro ville +cam hnews +cafe bar +c fra +bro ilers +bl inged +ba yo +az off +aki moto +ai jobs +ac bc +abu elita +@ @@ +) ,... +! âŃIJï¸ı +yo il +yaros lav +x vs +wolves races +who i +way nel +un cor +un ceremoniously +tu omas +top ten +tom perez +tom ales +to pos +tk ss +thesse hydro +tex ti +tan doh +tac tful +ta kako +sy st +superse de +super naturally +sun ga +sultan pur +stre aty +str ung +spread love +sp ul +soweto derby +sin itta +sin dustry +shah nawaz +save theday +sab ri +robinson cano +recycl ers +recu sal +razz ano +ratat at +rainf alls +railroad ers +r fruit +quin ine +puer co +pokemont cg +phoenix es +pha ed +per rins +pent acles +paynes ville +pan ja +pac ademy +orre ry +opfun kill +oni musha +on no +o att +neo tropical +ne vi +ne kop +n guni +moon cakes +moby dick +mo est +mi msy +mesni l +meh wish +mazz ei +materi alist +mar im +lumber kings +lost found +lion heart +ko aa +kevinand bean +ker shim +kay kay +joe strummer +inu it +inst anti +in communicado +icenine kills +hik mah +hal sparks +ha igh +h fg +gle ek +ge ducation +gar io +gan gof +g listen +g kids +finding nemo +fighter wing +femen il +fee q +fal ter +f nn +eye s +ever leigh +end gbv +elevense shour +ei sts +dima pur +dilip kpandey +deep tech +de haven +ctv kitchener +cri bbing +cra pp +cm ts +climate finance +cham bly +cgp grey +cd nimm +cc isd +cart ago +byu hoops +buu ut +boris kodjoe +bon dar +bl s +bi hh +benaz ir +bar son +bal dri +bal ado +assinibo ia +arch enemy +afloo ds +afgh ani +ab stra +a jol +ðŁĻıðŁĻı ðŁĻı +ðŁķº ðŁĴĥ +ðŁı Ļï¸ı +ðŁı Ļ +ëĭĿ 맨 +ãĤ·ãĥ ¥ +âĶĥ âĶĥ +ya str +y aboy +words withfriends +wo lowitz +wit kowski +wick en +w by +vogue williams +vil helm +victoriaave yard +ve u +v gn +urban ites +tv sports +torn illo +toi business +ticketmaster uk +thisisla bor +this be +the vegan +the pursuit +the edit +tam pon +tal ay +sword smen +su par +sta ab +ss ong +song books +sli gh +sli ding +sik sika +si um +sho igu +see doil +scre ative +schre yer +sahi wal +s ican +s dublin +ro pp +resu l +pur pura +pu du +power list +po zzo +pect ations +pat labor +on nie +om es +norse up +no place +my coskie +mu lan +mi asma +me sk +mar ui +mangesh kar +magh era +lyce umtheatre +ly ns +lud vig +la sk +ke ever +kalyp so +jeff gerstmann +israel mfa +in seong +ifwe date +i fra +ho way +harlequin books +har gett +grape vine +god backs +globe arts +gil dea +gat esville +games nostalgia +extingui shes +en um +dro gue +dep ted +denni spra +de valuing +curriculu ms +cur ta +corbin wwe +coo ber +conf ounds +cli mac +chrise u +chis eling +chi at +c frs +buy tolet +bu ssa +book ended +bethe sda +bear ss +baron corbinwwe +bar dwell +bam bo +au ton +ati erra +ark dg +amin aticsofc +ace us +:* :* +ðŁĮ¸ ðŁĮ¼ +ðŁĮ¸ @ +ìľ Ħ +zin c +york beer +x vx +wow wee +wm ms +wednesday morning +walter boro +unfor getable +tor che +tom kinson +toks vig +te airra +t sos +sub tweeting +st wm +spad den +sol na +shar bour +set the +salt box +s like +royalo ak +ro keby +river cottage +ri mouski +rep gh +rat dog +raj veer +r lg +q assem +pv lonabscbn +pv cs +pre zzies +pr riya +pose hn +plor ation +pit so +petroleu mmin +per rot +people over +pedestrian ised +path art +p afa +ox ana +ott tanak +osw al +on twitter +nol ito +niag ra +neo soul +naw af +nano crystals +movie history +lor os +loch side +larra bee +lang sung +kol len +khali fah +jewelry lovers +internet archive +iima hmedabad +highend audio +helens vale +hate week +han sson +ha ig +ha ad +ha ack +gula bi +gu skenworthy +going global +genetic ists +gam bang +food trip +en jo +eli b +elan valley +ec cl +dun nigan +dreamgirl hema +doctrin al +diy sos +destre han +del coronado +def s +death lok +da ia +cs rs +copi ah +col wick +coccin elle +cla ws +chelse alfc +ce tus +car keys +campbell claret +buydo texpress +bu ma +brig itta +bb clocal +battle ments +availab lenow +as os +artemchigvin tse +arm bureau +aqu is +anodi sed +ami um +alas dair +aho y +ab bado +ab ating +_ ~ +$ ' +ðŁĺĨ ) +Ø§Ø µ +za hi +wo reit +who a +wall enda +usc ellular +up es +under written +un dyed +u verse +tü rk +tu pdates +trumppro test +tra vers +tip sare +tha ip +ter mer +tam en +tag along +t ppa +syriacivil def +sun ghoon +slee pless +sle ssor +segreg ating +scot ney +sat lanta +sandra oh +sal vi +sakura jima +ross ell +rosen stiel +ro go +ran au +ra pan +qui era +psi onic +prometh azine +program ming +prakash an +potus geeks +portra itist +popefranci sph +pod sincolor +pin ni +ph f +pe mi +out strip +or ail +oktib beha +naacpi mage +mur amasa +mr nick +mot mot +mon aca +momo chi +miumi u +militar isation +mes ra +maximo park +matthe wl +mapre duce +lucian aberger +lost pets +lif ted +kul tura +kran ji +kle o +kil ic +jrl web +johna thon +jayawar dene +jammu kashmir +ir f +ipa day +hu hn +hil tz +hi go +gun by +gru ver +gou nod +go trojans +gn k +glen ys +ger asi +gamesdon equick +food co +fit c +fantasi afest +fan boost +exasper ation +ent ally +eleanor j +el te +east midlands +dun drod +diver sionary +dimp act +dev is +das raghubar +cur lin +cotte rell +com illas +clo ds +city view +cindy crawford +chew ton +chel seag +cen c +buxton brewery +bull nose +brig itte +bore k +black clover +ben do +bar ot +as mine +arne son +alternati va +allisonb janney +alberta ferretti +ala inde +al agna +agu ada +ag ay +ag ari +aberavon rfc +:) )))))) +ðŁij» ðŁİĥ +ðŁıį ï¸ı +ìĹijìĨ Į +z ea +your city +world fish +wild heart +wi kis +wc choops +water house +vu elo +vill i +vill amor +valent inos +val co +v kontakte +ux ury +tra ill +to shack +th ampi +tchotch ke +tat an +tal liance +tab ora +ta sha +stun de +sto you +stay true +short crust +se gers +se dar +sa je +ru iner +ros ch +root sy +ro des +rc stweets +rai ji +pre ading +pli o +play your +phon ology +peter hollens +pai hia +or ri +oker lund +occul tation +o dy +nadi az +n map +mon ten +mk d +mental ities +mar mon +mad hat +mac each +love mk +logi k +listen up +lets roll +ki pla +kath bum +ju gar +josh henderson +jose fa +jen nac +iron ical +ipad mini +internation alliteracyday +infier no +indie game +in expressible +ilovemy cat +homogene ity +ho ww +ham mon +gö ttingen +gross mann +gran ule +good acre +fron twoman +fowler pga +exclu siva +eco le +do dgson +despi erta +dent sply +defe rent +dam acy +coo tes +coffeewith acop +cm hr +chim my +cam on +c tot +bur nes +brahman andam +black spot +be witch +back stag +auto bant +anjanaom kashyap +ambas sade +ak ley +afl hawk +af ly +ac wa +aa ad +a jaz +ðŁĺįðŁĺįðŁĺįðŁĺį ðŁĺį +ðŁĺ© . +ðŁĴ¡ # +ðŁĩºðŁĩ ¬ +ìļ° 리 +âĿ¤ï¸ı ðŁĸ¤ +âĿ¤âĿ¤âĿ¤âĿ¤ âĿ¤âĿ¤âĿ¤ +âĢ¦ ." +à¸ĻภĦ +zoro astri +yu ma +yeon dan +ye ayyy +x me +x bet +wra ppings +wor dings +wen digo +weight man +way yyyy +vli eland +v ites +un tainted +ul lr +ton yi +tee total +tam alajones +stro y +star lights +space ibiza +soun de +soft skills +sob server +sne x +silver hawks +shropshire star +shout cast +sch lag +saf fire +s wn +rajeev khandelwal +quick draw +queens speech +pa que +overestim ating +old time +nu al +non lge +nio br +nigh tinthe +mos shart +matthew santoro +mass ad +marou lis +mammamiam usical +mam ma +makeup forever +m lauer +lu gging +lt frb +lost cats +logan air +la sser +kristi ann +kitt in +king arturo +kil gour +ke pri +kay es +jon bonjovi +jj cc +iss ler +isa itl +ilove the +ic alli +i dents +hof en +godsown country +goat skin +glenn howerton +glaci ation +gen asis +fortune cookie +fat ou +far be +faith and +fa reg +eyewitness news +engad ine +ee ve +e ung +e jb +dra isaitl +down to +dar ity +danny mcfly +dajj al +curi al +cure ttage +coo kin +consci ences +centuri on +calli han +bot nets +booksare mybag +bol dman +bod ø +bnld italia +bit burger +beg ley +beati fic +bay stars +bac laran +ash tons +ar pit +again stracism +ag li +adulter ation +ðŁĺİðŁĺİ ðŁĺİðŁĺİ +ðŁĶ¥ ðŁĻĮ +ðŁİī ðŁıĨ +ðŁĩ®ðŁĩ³ ðŁĩ®ðŁĩ³ +íĹ Ī +ìĭ ł +zab ala +yehrishtaky akehlatahai +yastr zemski +willem stad +vul fpeck +vol vos +var lamov +union station +tt olle +travis mcelroy +tra sporti +tra qu +ta kin +ta cho +sty ling +stress awarenessday +stop childabuse +stick land +stampe ding +spets naz +sol amente +soe toro +sne st +sky ways +si rio +sec tioning +scoo bies +sch achter +saha ba +roun dy +rose music +robert shaw +reverber ating +real j +probab ly +preci ado +pou pon +param aham +ouarzaz ate +opening day +nie buhr +nh sscot +ne ds +nan ometer +n atta +my club +mi way +me chag +lysi strata +liken esses +lear d +ko zi +ken sho +kelli her +kay bee +k lr +jon quil +ja veri +indira gandhi +implac able +ima bad +hill yer +herst mon +haun tings +harima u +hal tand +haku oki +gul barga +growth hub +glo ver +gil as +gi anyar +fort son +food facts +extru sions +est eves +ero les +en gr +en gender +effec tual +e zer +dieroten bullen +dho bi +dem ith +decapit ating +danko jones +dam ania +d te +cou sine +conserv atorium +co len +co biesmulders +ci offi +chin naswamy +cheri sh +charlie brown +caru anagalizia +car leigh +capit alistic +broc ke +bi aly +belcher town +be chet +bangkok post +back road +az ek +ann ae +al vor +aa q +< == +ðŁıĬ âĢįâĻĢï¸ı +í ij +åħ ī +âľĮï¸ı ðŁĺİ +à¹ģภĽ +zab ul +yay i +wit te +whone ed +whiskey myers +vigner on +ve do +vall on +us y +the move +tech week +tafa hl +stove pipe +stone masonry +stol tz +stern ly +stephan jenkins +slam dunk +skind red +sin tered +sanmarino gp +ru thy +ru ge +rind t +ren ouncing +re tton +re introduces +re assembling +re animation +pyg mies +push kin +pso as +prince william +preston sburg +pr wx +posi donia +pol at +pick meup +pay asam +park board +on w +on nn +ny ak +noo k +nic eties +ne vel +nati xis +n reid +murph twn +mon el +medi af +me gi +mayday parade +man que +lin ne +latin a +labyrin thine +kir it +ke hl +kaina atar +joyceme yer +jo vita +jo de +jeho vah +jason aaron +jar ratt +iso propyl +hu men +hir ata +hard fest +gw tweets +gun t +gru pos +gru it +gre ssive +gra bb +gon ia +golden voice +gi stics +gb doc +garda sil +future isnow +forsa king +fo sun +farm fresh +er zur +elizabeth taylor +e learning +dyffr yn +downtown la +dd b +dawg sontop +david chang +d bl +chuck berry +cat cafe +carab obo +can sa +camp and +c giar +brit tafahl +brae head +begin ning +bc s +atlan tean +as oke +ar network +ar ale +al meda +ag ray +af shan +ad ais +ace re +accordi onist +( -) +æ° Ĺ +ãĥķãĤ©ãĥŃ ãĥ¼ +âľĬ ðŁĴ¯ +ÙĦ Ùħ +woj tek +wi recard +white stown +white girl +vish wa +urum qi +ur ora +un ceasing +today news +to ils +thomas cook +test able +sub section +stav ro +sp ta +sound view +sof ties +ske ets +sk loof +shop style +shir anui +sharmel sheikh +sh yn +sett ing +sean patrick +sax mun +satt va +saras wathi +santor a +san angelo +saf ak +sa ji +sa ari +s done +rou ler +re assembly +ram ie +ra utela +ra bih +poquo son +per gi +penguin classics +pendi entes +over statement +outside thebox +ore stes +ofcaldub koto +obnoxi ously +nic ca +ne vel +nag er +n man +n korea +n kem +moth balls +mis understands +min ang +max thewanted +ma î +lu ki +lovel ine +lol jk +loath es +liti gants +lime crime +li spe +leonardo davinci +le uk +le coq +laur at +lat vians +ku sum +kin ard +kast ner +k plc +indiana univ +incentivi zing +ic ruise +hv dc +hin chin +happy veteransday +future bass +front men +fried l +finn hour +fin ck +every child +enders game +embal med +el wyn +ec us +ec astle +e qc +dexter ous +destiny schild +deck le +death inparadise +d ric +cv x +cul two +cine plex +chak wal +cem paka +carlo sm +bul man +bro iling +bi shopp +bay ridge +bay ram +bay ah +barrow lands +barra za +b ür +auck landers +ar notts +ap inv +anastasi ades +amy lee +al fre +acri mony +abujatwitter community +! *** +âļ½ ðŁıĨ +zoo pla +zib bet +youn us +wine studio +we bassembly +w enger +vi irs +vas an +vander voort +u az +tri ang +tram pa +topol ino +time keeper +tic ide +theal thy +ted dibiase +tal lia +ta tham +steve letarte +stenc illed +ss ri +sor guk +son er +sil ky +sc olo +sal il +sabo l +s stl +round hill +ricci ardi +reag ans +psylli um +pipe band +pi ques +pen nock +pand it +pachu co +outsmar ting +out numbering +oscill ate +oo om +officialo afc +objec tor +new space +ne way +national hand +mo ch +miscavi ge +mc millin +mavis staples +mat field +mal lets +mad z +mack le +maaa an +ley afc +lee ton +leaveno doubt +le llo +ko sten +kil learn +kid wai +jaga dish +it ss +ili ff +ib sa +hun nic +howtotrainyour dragon +hoo kah +haw khurst +gr ity +gopal akrishnan +go los +garden hire +fore tell +fore sta +fitt ler +em rick +eli en +eiz amusica +echever ia +eaton town +dushy ant +dee dee +de silva +day soff +dark star +cup cakeday +cu chi +cu caracha +cric buzz +craft fair +cove guardians +corne ille +comet landing +coiffe ur +cili p +chuck anut +christian music +cha b +bre on +brandon flowers +boxing day +book promo +bh ool +be fall +bat la +bag ani +back bench +am berg +affin ities +ðŁİ¾ðŁİ¾ ðŁİ¾ +ðŁĩ¬ðŁĩ³ ðŁĩ¬ðŁĩ³ +룰 ëĭĿ맨 +âĨ ł +Ê· Ê° +zi ker +ze enew +yose op +x ul +wen die +voice over +vit aisland +ver du +ve oli +ve h +van ita +uf h +tum he +tour ner +thisis london +tac ony +ta vis +t kb +t its +subro to +spiro sis +sou tah +son taran +sh ichi +seven oak +seas pray +scep tics +sar noff +sacred heart +ru tin +ro ter +refe ction +red team +re miss +ram ones +radical islam +ra eng +r hoops +q adri +prosthodon tics +pok kiri +pi oli +peper on +pedan try +paw trol +pad res +p shr +outand about +open ssl +om mi +ol t +o eno +nu d +north cliffe +nbcdfw weather +national runningday +n dd +mu ur +mt lal +mo shi +mee thi +match week +manig ault +mand ara +man ok +ma hood +m rap +lul ls +lipp ies +li sc +lepto spirosis +leather neck +lau thor +la sek +kyam bogo +kreay shawn +kra ddick +kean university +k bh +ju gs +jo bli +jig ging +jc su +jack knifed +j cre +it anagar +island wide +in delhi +in controvertible +haltand catchfire +h än +grove town +great team +gl hf +gimna sio +gig ichibi +ger des +foam ems +flu gz +fl er +fish kin +fini ster +fer mat +fa eces +ent ury +emul sions +ell ys +cummer bund +conval escing +con form +ci encias +chri sh +che ee +change d +cat ron +carre ira +car hire +bf si +better idge +bang alow +att ys +ar ath +and then +amox icillin +amandab ynes +am eric +alternative medicine +agon cillo +ag ila +adomin icana +acor ah +abduc ts +ðŁİīðŁİģ ðŁİĤ +âĹ Ĭ +âĸ«ï¸ı âĸ«ï¸ı +young day +wei mann +w go +un workable +um kc +tv awards +the tre +the jack +tel ok +super lot +stro zzi +st als +squ atch +sports memorabilia +spar kas +sm ca +sk ata +scoff law +sar ki +ru aha +rick sen +practic able +poblen ou +plant power +piece meal +phi rek +pen pals +padma bhushan +p marca +p inger +oro go +oni er +ob ata +o gan +norman whiteside +ne atoday +nascar goeswest +napp i +mus ico +mo p +me ep +mc clay +ma drug +liv ni +len sman +le coq +lax mik +lan kershim +kim taehyung +keven ingecho +kaf ka +ka steel +ka ino +jedediah bila +jar ro +jap het +inf eld +indecent ly +in adequ +ice man +ic q +holm wood +hog fish +hatt an +hass anandani +ha go +ha gens +gun ov +gool wa +fri eze +free time +fore lock +fonte yn +financial brand +fern bank +family love +f ator +exother mic +ev or +essential mix +es ling +en hage +eg be +dm z +discoun ters +der am +dave morrissey +cri spin +cour tre +cor keveningecho +comor bid +choice hotels +ces ario +ca bel +c spi +bou vet +body by +blue coats +bhat i +beat us +be smart +bangladesh is +balo gh +b cle +as mp +ambro gio +amar ley +al dc +air transat +af rl +ab ud +âĿ¤ âĺº +áIJ ħ +Ù Ģ +Ì²Ì ħ +zom boy +zapho d +yon atan +xi omara +worldd omination +wl bt +whiteri bbon +w fc +viper room +vign ale +val o +tu bule +tu berlin +thu ll +taip ans +sur mise +stor ico +sta hl +st n +st anced +smex aminer +sip smith +si rs +shin awat +shalo tt +sche epers +sc bc +sasol sa +sam sa +roy d +rib chester +rhy dian +re assessing +pro vine +poo ch +pom on +pic ado +pi ma +pg x +periodon tist +pd ca +pann acotta +p win +om avirus +ni dal +neon trees +nal co +my viking +muj uru +mtlal ouettes +mo zes +mo ren +mcv itie +mat thes +mani x +mac donnell +ma zen +ma ass +luis ito +lord stown +light painting +levan te +legal advice +le iria +lap angan +la ze +la the +krish n +kol on +ken sal +kare y +kam ya +kad lec +jig arth +itsadog slife +is ae +iron birds +hyu koh +hm nation +he ssler +has na +gore tex +good memories +gonetoo soon +goe tta +gel dings +g mm +fr yman +foot ballis +eric a +end sleigh +ek tar +ec lub +ear p +corn market +concur red +coet zer +co ker +clap board +ciz re +cic m +care e +ca ep +bush baby +bub bler +bread basket +bre p +brazilian jiujitsu +bod ger +bi sho +beng kulu +b itti +av eline +architi zer +angel ine +all c +æĽ ² +yl u +xon r +world snooker +west cliffe +ux mal +toto wa +too early +tipp ler +ting z +thin c +tf tuesday +t itt +supporting women +sru hle +som nam +sm itty +sly ke +slur ping +si miles +semir amis +see ma +sec ca +sc g +rite sh +riseupred hawks +rho dy +res nik +qu wain +pá irc +pun kand +propag andhi +principal ities +prem chand +pr zy +potenti ometer +pol ke +plu gand +pf nicholls +park field +pall ant +ophi uchus +one happy +news reel +ner r +nat geom +na ah +n to +mu ffle +mi at +manchu ria +man ai +mac ross +ma sand +lockthe gate +lo far +le pak +kemp sville +kather yn +iucn congress +irrepar ably +hi kaye +hack able +gund agai +goodwithout god +go teamusa +ge milang +funko pops +fix it +ero i +eel am +dun raven +dro st +dra ther +dorn birn +design and +dar wyn +cut ler +co bit +chimbor azo +car oftheday +car ned +brown trout +broc kett +bre vard +bluer ising +bin n +bidvest wits +ball ena +b hari +auto drome +asphyxi ation +ar dr +appet ito +anitadon gre +am ayi +alla scala +all ontheboard +absten tion +ðŁļĢ # +ðŁĺī ðŁĴķ +çĽ ® +âĿĮ âĿĮ +áµ Ī +wyn jones +wsf cs +wrong ful +winand yourein +vill on +vel ho +urtic aria +travel writing +thework isworthit +th ye +svf social +submer ging +strelit zia +stevie j +sr va +sport speople +sport park +spiderman movie +souven ir +sott sass +shel le +senti ence +scotu spick +sco bie +samu dra +sa amy +rope walk +rickie fowlerpga +quadr ille +qu k +q nx +pr itt +planetrock radio +paul ricard +pag el +pack ag +our people +ormer od +oppos itional +opol itical +olafur arnalds +ne ared +naz gul +naveen jindal +my four +morning run +mor by +mcin ally +mari eee +mangeshkar lata +mall ord +malaw ians +makeme smile +lol lar +lindy hop +lar do +lam as +l rd +klax ons +jim jefferies +j tag +iter ature +house and +homeboyz radio +ho omin +hari prriya +hand guard +grou ted +gretsch drums +gre go +gi do +friend satthetable +football archive +f nc +en ate +em cr +e iland +dor ayaki +dioce se +daniel craig +cz k +custar d +co var +co ban +chetri sunil +cher no +canone os +c train +bub nagar +bo jana +bo dley +betra yals +beh rouz +bas sing +bar bas +bank rolled +bag no +assan te +as j +anni elennox +alab adi +ðŁĻı ðŁĩºðŁĩ¸ +ðŁĺħ ðŁĺħðŁĺħðŁĺħ +ðŁĶ® ⾨ +ðŁĴ§ ðŁĴ§ +ðŁij ± +ìĤ ° +âĺĢï¸ı . +त र +ı oÄŁlu +z up +ys l +wol v +wi da +we k +ver mette +ve trano +val on +up skirt +unt ying +u rena +ts ong +tro v +toy collector +thomas ina +the ts +the stoneroses +tf boys +ten th +telang an +ta ite +swas ft +su wa +strip elife +sport paleis +soccere x +sm pls +sin to +shop ads +sher ratt +ser ous +screen ers +scot th +sch wag +sai ko +ry croft +ro dders +rein ier +range ley +qual ino +poign antly +picador books +ph orm +pete foramerica +pescet arian +pan war +pa edo +pa chter +ornam ented +orac er +opp erman +nefer tari +nan amusic +mur ton +mu mias +mol lo +mitri one +mill field +michael muhney +mediacom poser +mass ari +lu iz +licen se +ley on +la style +kamp ar +josi elong +intra week +inner circle +iit bombay +id una +hydro dynamic +homes ites +gri mey +google cardboard +gi gged +gal sworthy +for animals +eu com +don nay +doing right +dil opho +dad aism +cu v +chi pola +callsthe heart +bristolold vic +bhattachar jee +assembly fest +anything ispossible +actor karuna +a arti +. ðŁ¤Ĺ +## ## +ðŁĺĤ âľĭ +ðĿIJ ĵ +ãģ ¨ +z nik +witch of +winter berg +wh summit +well ard +vel oute +u ter +tv dsbmath +tur p +treat day +to inton +thevoice kids +thev ra +ther ich +the intercept +the coach +tel aviv +tam sen +tal garth +super impose +str fans +sr sg +spar kes +skitch en +shel ob +se ant +sakur acon +s offer +ru dman +ru ah +ronnie o +rocksolid show +ric carton +rabbin ic +puz der +punch line +pu ft +pot latch +phil os +pacqui ao +over hear +orn ella +opend ns +national nutritionmonth +music hub +mu ybridge +moody grams +missing cat +micro plastic +me il +mari ecurie +mar williamson +mar al +man za +loch gelly +lifein pieces +li q +len ni +leader ships +lat robe +la ie +kor y +kong skullisland +jane goodall +jak cloth +intra ocular +in ves +im zaheer +hurricane season +hol lowell +hahahahah haha +gu ac +great people +gon go +fr acing +flight attendant +ferry boat +fern ley +far ang +ex im +ex hum +erec tor +ef ford +e gle +dry stone +doy enne +discred iting +dia hann +delta zeta +dam ascene +cor less +con scripted +colmc ille +col ca +chase water +carden al +canad apost +cam wowapp +cab ane +bud de +brian w +bra ding +bor st +avan ade +ask ell +ashi da +arche o +ar lo +ander sen +amazing racec +am brin +alt stadt +alam gi +aig adesign +adu que +!! ðŁĺįðŁĺį +ðŁ§¡ ðŁĴĻ +ðŁ¤Ł ðŁı½ +ìĨĮë ¯ +æĥħ åł± +âĺij ï¸ı +é dou +zahaha did +ym un +yel e +with congress +v dara +undoub ted +u din +tri pler +today sdoodle +ti ang +think and +the villa +the jay +ten se +tart u +ta kai +sycoph ancy +superlot biz +sub verted +stickle back +sky hook +sho tter +shar in +sg br +se uro +sb hs +sardon yx +sarah jane +sal er +rou e +retirement planning +ren dle +prison er +portal berni +paulricard track +param ore +para professionals +pap akura +pali ka +p tech +orion books +ne dv +n ali +mondad ori +mis fires +mil verton +mex borough +met service +man gg +mam i +m fah +lyn ds +lpr nyc +let girls +landsc apers +krish nan +ki raz +kd nuggets +kalym nos +kab badi +ka ira +k achi +ju vie +ji zzle +jab bing +infantry men +ib botson +hunter valley +gu z +grow up +gre nadi +glo scricket +glee ks +giri raj +gel in +ge ste +fre twork +fore bears +flor ance +fitz gibbons +f natic +ess m +embaras syour +electro plated +dur ness +dne pr +demon ology +daily gist +cycling weekly +cour ser +cor net +col td +closing bell +cip are +ch nl +capric or +cal houn +bur rito +bouti qa +bor ys +boo zman +bk stg +bj k +bi osis +bal en +anthony jeselnik +alem bic +accentu ating +... / +ðŁĶ¥ ⾨ +ðŁĮ ĺ +zom bo +yard girl +yar is +wisconsin ite +wid dop +west van +we sen +waste basket +uper y +united coaches +uncle blazer +un ready +un perturbed +tor tue +ton igh +tim scott +thiop hene +the tribune +the blackpanther +tend re +tat au +sumbur gh +subo tic +su eno +stre icher +st cloud +soulcalibur vi +sof light +sing en +sig man +selfiest ick +scott skomo +sad an +s fax +s den +rock field +reep ham +redd war +q ais +presiden tever +prefec tural +pol li +peri helion +panto ja +pais aje +olli elocke +ohi a +ocel ots +obe isance +nor is +ni shar +ni dho +ne fer +nancy drew +my brother +modern monday +min ories +mill ard +mc mick +matted mondson +mat kearney +masa aki +mar le +mam ared +mali c +lo chee +lay away +lau di +lam he +kungfu panda +kong sberg +kan z +kac per +jo gos +jam huri +j rod +is for +indiav spakistan +in coherently +im erick +hun nid +hu it +ht cone +hin demith +he pb +hat ake +han aa +gutt man +gopal an +gal les +fer mo +fashion jewelry +far mar +eug bil +esc c +ego istic +dy stro +din ve +der ya +dend rites +cw batb +county fa +cork ery +con descension +complic ates +co gno +co ffy +cherry belle +chelt festivals +chef ou +call acu +cal as +bou los +bobbyl lew +black ening +bishop ston +be zer +be bb +bacal hau +ba ath +b pw +at enas +assi stir +as bel +andhrapradesh cm +and c +an ap +ade al +* $ +ðŁĺĬ ðŁĺĦ +ðŁijĩðŁı¼ ðŁijĩðŁı¼ +ãĥ³ãĤ º +ãĥ ¡ +wor ton +wool er +wil bur +wick man +we it +we iner +wc sd +visit brussels +vian sk +u ah +tu gu +to kyu +thre ss +ter williger +te ef +ta va +syno logy +suomen linna +sunray sia +su lit +stewar desses +ska ite +shop ko +shin do +shi maz +she ads +shan shan +se yed +scream fest +science fair +sathi sh +sas ummit +re yer +raffa ella +purwak arta +promp tlist +pet portrait +pawl enty +paw circle +pauly shore +parat ransit +par thian +papen brook +ndr rmc +mul ligat +mc dreamy +man jeet +mahesh sharma +lap ine +lam onte +key card +ker oro +kaz ak +karl urban +kainaatar ora +jur gen +jimal khalili +james dean +j radio +inform ality +in authentic +imit ator +i haven +hin dman +healthy hair +har oun +gu ast +graveyard carz +gal lucci +gab b +fi roz +farm houses +entry ways +enjo bs +electric picnic +eddi emurphy +eagle ville +duff town +don nelley +desi sto +date able +dak otas +confi ding +co des +city rail +chi angra +charlies loth +cer t +cc funkandsoul +cas sels +carequality comm +car mello +bru e +bie gel +bachelorette abc +awe want +astr alis +app design +all td +air band +ad hs +actu alized +ðŁĹ ¡ +ðŁĸ ±ï¸ı +ðŁİĬ ðŁİĪ +ðĿĹ ¢ +áµ ĸ +ॠĥ +zoy sia +zeenew shindi +yav uz +xx l +worldof warships +wink news +westh off +web tour +we te +wal lawalla +wah peton +w ben +vol and +victory beer +vicky rubadiri +ven eration +up éry +u anl +tw iss +tur alists +tron foundation +to fun +tm ills +tin am +the legendofzelda +th ays +ter lingua +tamer amowry +taldÃŃ acomohoy +supre ma +sound ness +so su +sj l +shin sen +scot amfa +rene wal +re animate +py les +pra ja +per nice +pedr ito +pat ong +pa ura +p ello +over laying +or way +onthe green +not as +nin ful +nil i +nav o +natural hair +nan chang +must weed +museu marchive +mun sey +mill ers +mc ternan +mc spadden +mant illa +magic mike +ma dailygist +lü neburg +loren zana +ll np +le iv +kel burn +kayag nik +katie holmes +k mel +ju ric +jessica jung +jan ak +io d +i ffic +hu ai +htc tryouts +hitthe floor +hi rap +hi bi +hel low +goul den +gold ber +frie de +fire baugh +fin nis +euro pac +en livened +dul thood +dhru vasarja +de duce +david h +d si +cruis ing +crimestopper suk +chun li +cho le +chery shev +cat orce +casper vandien +bristol nhs +bre m +bo ij +bo gal +blr citytraffic +birch ington +beti bacha +bbcscotland news +att kisson +asym phony +anti histamine +anind ya +amp suk +al dez +ak pan +acer bic +- "@ +ðŁĺı @ +ë¸ Į +åł ´ +âĪ Ĵ +ย ว +اÙĦب ØŃ +wo aa +wic kett +wi on +wedding decor +voj vod +vent spils +ven ray +vel uwe +v ahan +ur key +up staging +une scap +tro yal +thi em +the ug +so car +si h +shu mai +sen ai +secre tions +se ssi +rup tures +rubi doux +restore thevra +refriger ants +qu illin +pu mba +probowl vote +privati sing +pra i +police family +po ya +po cho +oldman crew +ok ent +nomin ative +no pf +net label +nei va +n itti +mulligat awny +men asha +mar gao +manpower group +man sory +man son +magal ur +lun eng +love oz +lal ang +la on +la ire +ku duro +knot fest +kele la +kamer on +john newman +jerky xp +io ta +inter jet +ich thys +iam lenaheadey +iam cannabis +henry holt +ham ban +h mua +gur riel +gour i +g berlanti +fl n +fior ano +fight ingfor +eu taw +enz as +end ly +el c +eco watch +dyscal culia +duba ic +don mar +dil se +di ola +di kt +defl ates +de sensitized +convul sive +con la +chowd ary +ce bit +cb ct +car not +brianger vais +black hawk +ber lingo +bel is +bar jatya +ay k +aphrodi si +any who +ann d +am lw +ak s +ai jaz +ac oo +aaa ad +ðŁĩ¯ðŁĩ ´ +your heart +whati learned +wear redday +war moth +walshy fire +ver lo +varney co +van ill +us g +un balance +tre ec +toscan ini +titlei steurope +tick ell +texts anta +stay focused +stat ecraft +star garyen +so bhi +smex y +smart contract +sids riram +shi geto +shawin igan +share alittle +sells vintage +se ba +schi emer +ru su +real cj +qnl z +projec toftheday +power plants +pover a +poll star +play box +phili pham +phel tzcomics +parallelo gram +p she +over shadowing +outer hebs +olf club +oil paint +nicomaine at +naw abs +mor cheeba +mel fi +matador records +mat tr +london bronco +list ener +le ist +kim guilfoyle +ke wanee +ke ffi +karti keya +kaad hal +jo ynt +jeff botl +je gs +j re +j omin +j illa +instaf it +ib times +hull fc +hiro mu +hex agram +heff ley +hand carved +gri ego +gen aro +funic ello +from dusktilldawn +food friday +follow ing +fit and +feli u +etsy sellsvintage +eli braries +du bi +dis qualifies +ding li +design junction +deni grate +davidar quette +david b +daddys gurl +d jam +cryogen ics +con fit +commun itech +colorad oriver +ck ner +chetu mal +charliec rist +ce sky +cdn screenawards +cb ase +carto graphers +can tatas +bwa hahaha +bun yip +brigh twood +bracketo logy +blan ke +bet tel +barcel os +balla deer +bab us +ba eday +att ilio +art world +ar zu +anthropom orphism +anim ales +an ath +am ua +alye ska +ad com +(âī§âĪĩ âī¦) +âĢĶ -> +à¹ģภŁ +zel aya +z ann +x pose +wra iders +wc w +um laut +transpo se +tr é +they ll +tat sun +tal isa +sz ky +sx moctane +sob chak +sig is +shu cking +shol ders +sen ju +sel borne +se bor +se acroft +scottish highlands +saad at +rpo online +rn tata +richar da +rebel hearttour +rc sd +ra ich +r alo +pun kd +poster design +phosp hates +pett way +personal care +pe gram +panther s +pad ley +pa zza +over working +op ala +ol czyk +nic ke +neu meier +natu ren +national cookieday +mor re +maruti suzuki +lob dell +liter acyday +lip kin +lic orne +la bre +la bial +la ad +krish nar +ken ting +kemp en +kc caug +ka id +ju stre +j reed +itu din +it au +insinu ate +ich rist +hu lot +hpnoti q +home school +hay don +harrieth arman +hand lin +hal ilovic +h sts +gur purab +gamers gate +g mod +g beng +franch itti +flashi kat +figue iredo +fen ske +enter o +eas dale +dh v +d sn +cru dit +corin thia +conference usa +coloni alists +co agul +chocta what +chie fex +charlesd ickens +charle swood +cen o +call ery +cal football +bul ley +brigh twell +bree am +bol ding +bi plab +betsydevo sed +bar il +bankni fty +bani shes +ban yo +ban iyas +ba reminer +asking for +as pho +app sych +ap late +and day +ade ga +aber corn +ðŁij¸ ðŁı½ +ìĭ Ŀ +âķ ¥ +ye pa +ya eger +wrest lin +world history +window sserver +way faring +ward man +vers ac +upen patel +umass lowell +u my +u local +theodor ou +the weekend +ten ergy +tce cardinals +tas min +ta ira +srijit speaketh +smo d +slv sa +sland erous +sk cv +shag gy +sen an +se ber +rich burg +re uses +ray toro +rav ager +raim ond +proser pine +pol lok +par ami +pap illary +oy uki +osme ña +or thy +op ac +old mane +oldmane bro +ob scu +non conforming +natl prep +nam as +myri ad +mud flap +mono theism +metan oia +medi en +me xt +martinlutherking jr +map making +make out +ma hay +li gnes +li ba +lesleyann brandt +le ki +laver cup +lar ner +ku zu +joe dotie +its agreat +isa ia +intermittent fasting +in china +hy dride +gub bio +gher bo +ge tover +gd live +gar re +flame throwers +exab eliebers +enchong dee +emo de +edg iness +easter brook +dow ler +dor ing +director ship +deadly sins +curling canada +cu tes +contempl ations +cl enden +car oni +bra ggs +botan ico +blog paws +blackink crew +betibacha obe +batt lers +ban bury +avi atrix +av and +anu bhav +ann ya +anheuser busch +alo dge +alamgi rizvi +age ek +ac ti +ac rack +ðŁĻıðŁı¾ âĿ¤ï¸ı +çĽ ® +ಠ¸ +à° ľ +zhong shan +ww elive +worl dexpo +woo p +wai hi +vo t +vil lo +vid éo +vas cu +vagu eness +v usd +under value +u see +tut bury +trache ostomy +to kam +tir ith +there sa +tauto logy +synchron isation +swan berg +su ad +spl urged +sp lan +sound less +smile more +sitt we +shil ajit +sh agreen +schoo le +sab ry +ryan reynolds +roth child +rbge hort +rafsan jani +ra shan +qu be +psycho analyst +proprio ception +profess r +pro me +pp d +porcup ine +po stel +pang aniban +oubli er +ol die +moo gle +mo sler +memo ire +mc di +math ed +lon drina +lla ve +live action +le ering +law ers +ku ehne +kirk dale +julio jones +jez reel +jetti son +ip ag +induc tor +ici um +ichthyo saur +hyster ic +hu pp +hr lich +houston heights +home away +he int +hat ting +ha aga +guildof finefood +gidd iness +ghost buster +gen next +fou dre +fo res +floren zi +feu illes +fer ret +fe ve +euro a +ep is +enrique gil +end les +e tim +didd ly +dic taphone +de mining +cubat ravel +co calico +clarkk ent +chor tle +cb ellracing +bul las +bren o +birth marks +be jarano +bbcradio stoke +b tho +b kt +audi ere +atp masters +as sp +ap his +ang ol +ang ere +alex s +al tran +aic p +ag gia +ðŁĺĩ ðŁĻı +ðŁ¤Ķ @ +á´ Ħ +ze spri +ye ws +y go +world run +wind lesham +we uro +vogel song +vho enen +us olympic +ur ning +u sun +twitchretwee tr +tv network +tor rez +ti be +ther see +ther itz +teresh kova +tak firi +sy ra +splot ches +spaw n +snow barry +sight lines +si mc +shilo h +se wu +schi ff +saviler ow +san wa +ro ten +rizzoli isle +re eee +rat ap +r factor +qualityof life +pur posing +pul man +pu tten +procreate app +post i +phi the +pa quet +official charts +nor west +nit ef +nick groff +nev ada +mono hydrate +mon ckton +molson coors +mod ellers +mc tell +maj ithia +low cost +louisian agov +lahore qalandars +l be +ju lly +jeux video +j man +island records +intothe wild +im ss +ili brary +ig inal +hydro logic +hou chen +hoo ter +hom am +hi madas +her ve +her ren +hel loo +heinz vhoenen +he ey +has z +h mer +gel inas +g elliott +french quarter +forthe weekend +for honor +florian opolis +fa sho +dun t +du aa +dom itian +dj max +din k +descon to +d johnsonpga +clear skin +but u +bras sey +body board +bk v +bit pay +bigro ck +bene detta +bar ony +bal art +bad uy +ba q +at ami +as umi +andrew christian +an fa +an ever +am usa +al medalen +air busa +aero space +ðŁİĦðŁİĦ ðŁİĦðŁİĦ +ë¬ ´ +å¼ ¾ +ér ôme +zuc chero +yer caud +work table +we win +wb sdcc +vill ena +viewfromthe office +usta z +upenpatel world +un ah +uit enhage +tyler cowen +thereal morrison +thame side +ter na +tar if +tani guchi +taf ter +swag man +sunday service +stre b +sky forge +simm o +shinawat ra +sety rn +sar kis +s bank +real men +re ja +prime au +plum tree +pardub ice +pal mera +or phi +off setyrn +nak heel +multi stakeholder +michel is +mc cafe +mary stown +mar bach +mad in +mac ungie +love that +lo hn +lo ffici +li thic +les niak +legi onnaire +lee z +l lah +l ity +kor ang +ko vil +kho j +ke olis +kar ls +kab o +jordan stown +jeni fer +je maine +ire x +inquirer biz +ing ly +indul gences +i abc +hor ological +har ring +halcy on +haj ji +gru s +gra vois +gis d +gi js +getur riskon +ga jar +from space +flash forward +fight news +fer ried +fast fact +fair water +eski mo +er be +eich ner +do reen +dhi vy +david afrench +darren hayes +cyclon ef +cth k +cre f +cowh er +cor wen +copp elia +cool ing +coco ons +char vel +car lie +bro sh +bri dles +breaking dawn +bre snahan +bel grade +bandof horses +bailey may +ark ley +ar kra +agreat bigcity +ag ip +adi da +ad rs +aac ps +ðŁIJ¬ ðŁIJ¬ +ï » +éĽ » +ç Ĭ +y ati +world blooddonorday +wn es +wha thapp +weare the +uw badgers +uni ak +under lies +ulti mates +tw oway +tso tsi +triangu lum +thermal take +theover tunes +thehow sofus +thec rick +tameramowry two +t shir +symph on +surfer sparadise +suncorp stadium +styles p +stick ball +sp rockets +sne tball +sna c +smart mobility +side steps +self worth +second chances +scri be +sar mad +sam ms +saakash vili +s mullen +redefin e +re tooled +rale kha +raider strong +pur view +pu gn +pranit asubhash +perme ated +par apar +nov ations +nigh tin +never surrender +nc ell +mole hill +metho w +mar tes +mar sy +manek shaw +male m +ma zo +ma had +ma ddin +m qa +m jj +m ji +lu ter +love it +lin ha +lifel ock +le twin +le gh +kathmandu post +jou e +institutional ised +im ac +illumin ators +iko kaz +home fortheholidays +hel pu +he chos +hau liers +har nish +guer neville +gre gan +gon dal +go outside +gh and +gen thaler +gabri ell +gaben fica +g list +fox sportsnews +fly tpa +flan eur +et in +esc rit +end at +dump the +dor aville +dam aris +dale steyn +dal ley +cran brook +cor oll +come up +cla de +chol mon +chess base +che es +cein ture +carre gabenfica +cann ell +c sem +box games +bou squet +biz dev +benef icent +autom obili +ass ant +ar ati +ann ell +an tri +ambassad ress +agra ham +adren aline +ðŁĺįðŁĺį âĿ¤âĿ¤ +ðŁıĢðŁıĢ ðŁıĢðŁıĢ +ðŁĨ Ķ +æµ · +ãħ Ĥ +ãĥĪ ãĥª +ãģķãģı ãĤī +๠ģ +zell ner +z ill +ysleta isd +yach trock +xylo to +wolf hard +west combe +wedding inspo +war hammer +wal de +vir sa +victim less +van go +v uganda +v hl +uk bff +ts agov +tri ble +travan core +to ton +to rey +time scales +theword alive +the team +tex ashi +talkto pd +sulphu ric +su rer +so len +sing la +schlu pp +rubb ers +ru y +ra fc +r anta +py rus +prat ts +pos iting +pof icial +poetry society +pn brock +phragm ites +penn ard +peach y +par atriathlon +ost p +oro ville +o gh +ny mf +niki for +ndu bz +moz fest +mon cks +ment ally +mc comas +maytheforce bewithyou +may flies +mat ram +maqu illage +mag is +lyn dale +lucap as +lou rens +leu ci +le xie +le shan +le ota +law enforce +latingram my +lat ou +la a +kubla i +kpm guk +kitch ener +juicy couture +join there +jedi diah +jay and +jan ky +italian style +ig as +hul kam +horni manmuseum +himm at +hil lock +hen shah +hail southern +grass ington +gore ski +geno types +fleet management +flav ell +fiction alized +fi du +fascin atingly +far allon +f ma +ett inger +er music +el lam +du leep +drive insured +dar pan +dae mun +cy bil +cur rys +contin o +con oci +co ent +chand rika +ch ulo +cen ser +caw thra +bu shi +brandy well +bou ie +block ading +bla do +bin ky +ben be +bel gis +bau ti +bau le +baro ss +bar ite +asseenin columbus +alm shouses +ag ol +achristmas story +ìľł ëħ¸ +ë³´ ìĿ´ +ê¹Ģ ìĦĿì§Ħ +ಠ¿ +ye tta +yaf fe +wordof god +wen zhou +val astro +ur f +unitedwe dream +ula res +uit p +tit ley +tiff ani +tag on +sugar and +stru mp +stri m +stri ker +stem day +sme er +sine w +sha hada +sh older +sey dou +sahar awi +rol lup +ro stro +re im +rac quets +qu ise +ppro ject +pp ina +pol let +pis an +pha res +pak sha +our town +or kest +opp ement +oh mi +ogbon na +o ord +o brist +nin ot +ne bel +nas ca +ms fc +mol ts +mohom bi +mel hores +mal en +maithri pala +ly da +lon da +liquid ating +lick ers +less or +leard blockade +le mbo +le ash +ku fi +kam illa +kad abra +ion izer +in cle +i all +hyper text +hill town +high tea +hang u +hadd am +gut tural +gu ap +gran adilla +gr ö +goff stown +gee king +g pro +fum fum +freetheni pple +floo dgate +flat man +evil hag +euro control +epitom ised +edinburgh zoo +ed ance +dro ad +diatom aceous +di ds +datascience ctrl +dar ko +comb in +co ye +cc cu +buch tel +bogo sian +big nell +ben harper +ay ar +au teuil +as ol +arc and +am att +aga g +af forestation +ae w +aal apor +ðŁĻĪ ) +ðŁĶ Ħ +áIJ Ľ +yu mmi +yj hd +wy ant +win eco +wil pf +wil lumb +whaaaaa at +wa inaina +volunteer day +trasi meno +titu t +ti so +thetalk cbs +the jump +the homeof +tender izer +tech geeks +team ucm +ta en +sw asan +suu kyi +spra gg +spo c +sien asaints +si rois +shoton oneplus +shere met +shel man +sheepi shly +serendipit ously +seaf ire +scott gshore +scen ography +scarlett moffatt +sale town +ru salka +roman ization +ridg ely +re constructions +re classification +rap mon +quebec city +pronoun ces +pre ti +pre defined +po pin +play on +petrol heads +pen er +pa hl +or dre +of music +octavi aspencer +mr tommy +moon lights +min ott +mik los +mase go +mari af +maha yek +ly kan +lin dal +leg ant +lat eran +lamba sted +kom mt +kit amura +kill cliff +kap itan +k ace +it suki +is il +insur gentes +inf lorescence +ich it +ich ay +hu an +history day +hi eu +hai kyu +grand mom +gir alda +gi en +getto know +frederik sen +forger ies +fly day +fl stateparks +fin twit +feeling blessed +fay outh +f ya +ev elin +era worth +ent deck +e dea +dwn twn +duf fus +dro om +dic he +der mody +delhin cr +de stra +dal gety +d cau +cyber sec +cour te +conten tedly +congres sperson +col tart +brown lie +bre x +bour ret +bog side +bo esch +bel phe +barber ini +ar bab +am mer +acham pion +absor bency +:' ' +ðŁĺĦ âĿ¤ +ðŁĴĶ # +æľ Ľ +ä¼ Ĭ +âľĮ ðŁı¿ +âĸ« ï¸ı@ +á zar +zehn der +yer kes +xero x +ww wf +women surgeons +wit n +we star +vinay aka +ven ator +vari ously +un translatable +u fl +u dan +ty rr +tren tuni +tott ington +the savoy +thak or +tegern see +sur ve +smithfield foods +shi u +scott baio +saur usrex +salon pas +safety tips +run offs +restaur ation +real world +r ón +quan tumb +q ml +pulmon ology +pir atas +pi ra +oven den +on ny +ollan tay +oc weekly +o hit +musth aves +mur tabak +mul ford +muhammadi yah +mu dv +mon roy +mon geau +mo liere +mis sle +middle ages +mic ke +mc p +mb fwa +m di +lumin ate +londonbronco srl +lil ting +lau rid +lare my +kow tow +kom ando +kar z +kamalha asan +k gu +journalis mis +jo shab +intre sted +iff r +ic ty +harro dsburg +hard inge +hange ul +gott acat +gl n +gam bir +fre richs +forever with +fex po +fay az +expan se +etch ells +et ool +eluci date +el rick +east devon +distribu torship +dis affected +diar rhe +deme o +deenday al +dean s +de martini +coldwar kids +children shospital +chee zy +char cot +cas andra +car reg +cap turing +brown lee +brother ali +beck ner +bas ak +b boy +auto psies +au stal +agu da +' !" +ðŁĺİ ) +âľ ¶ +é ri +zoe trope +z alab +yoshi oka +yogal ove +x cond +winter ville +wild west +wedder burn +vi varo +uryn hill +un colored +the calm +the arti +teatro allascala +te ays +tas bih +t smith +t lo +t elic +sunday selfie +sun u +summari zation +spell book +smu ckers +sli e +sksk sk +skill share +skatec anada +sicure zza +shou ka +sel by +sc te +s again +ro loff +ren ouf +rema pped +red u +reci o +re val +plu mp +ph yl +petre scue +pest ana +part age +paper man +paper magazine +on tiveros +on enew +ne ira +nare lle +mi one +manip uri +man md +male ek +mal ays +lu bang +lo ic +le ur +lar tigue +la vis +ke sel +jal y +is ob +in ski +ice do +hoof ddorp +here come +hau ts +ha or +h mph +h dr +green ing +go pala +gad son +fon i +fed con +f hc +emer aude +dunnell on +dub ya +don don +dhu l +devinder raiji +dak hla +dais aku +ch lin +cavat appi +cast away +cap rio +ca strate +bur nit +bond holder +blo tches +birch grove +bir tles +bevil acqua +bet wn +bbc merseyside +bas ak +bapti st +b cus +at ake +anok hi +annes ley +amber ly +al eck +aero star +ador kable +adel leo +ack lam +abscbn ball +"" "" +ðŁĺ£ ðŁĺ£ðŁĺ£ +ðŁij¯ âĿ¤ï¸ı +ðŁĩ®ðŁĩ © +æĽ ´ +ÙĪ Úº +Ùĩ ÙĬ +رÙħ ض +zel mer +y atim +wr on +wies er +wheel jack +weiz hou +weare weber +vene dig +ven ham +u chu +twitter party +tru elove +trade craft +to ve +thestor mishere +stri bling +spel thorne +sk anda +selfre g +schla pp +round tables +road hog +rish tey +ray more +raj in +qu ity +qian long +ps media +produc toftheday +pre port +pou ched +pis d +phi beta +pat o +pand ita +pal le +p ni +oyor ooms +ove ts +one humanity +omni directional +omidy arnetwork +northumbria uni +non members +nidho gg +mur willumb +mtb life +mtam aryland +morg ann +men folk +mel len +markr pellegrino +mariab rink +mar tock +man abu +mainten ance +lor ds +li fy +lean in +lan awwe +kw q +killthis love +kil mallock +kag erou +k wc +ix p +is que +hol men +ho is +haven lust +guaj ira +get north +ge healthcare +fox ley +fle eces +divas confessions +desperate housewives +del phic +dame wine +d jay +coun tin +city kitchener +ce smma +cay es +carlin ville +campo ree +c ge +bluestar media +bene ath +ben askren +bee bee +beat niks +b ghs +ath iya +asi acup +ash el +ase sino +alainf cae +akh ya +aj ar +admon ition +ad harshini +å°ij女 åīį +ãĥ³ãĥ Ī +your vote +xia oming +wux ia +wat lington +wal esa +w fr +ve el +va shish +un settle +un dr +ug wu +ty rel +tr nc +thu gga +thorn dike +thal loween +tan a +tam la +t ma +str ating +stock trading +ss g +sp ick +so tu +snow mobiles +sing son +shy lock +sfor the +sfor peace +ser mon +schiz oid +sar gentina +sam arth +rac coon +qui ddick +pur s +psychedelic art +pro europe +perme ating +pere grym +our finland +ori ello +o guri +o dus +ni obe +net scout +natural products +natural ised +nato summit +mt dc +ms rachel +misssaig onuk +mis represent +metac ar +medi are +maree ba +march mont +mal pass +mal ite +loveu all +london city +local auckland +lo key +lma ker +ley endas +lau g +lancelo teh +kul wicki +khat ter +kas son +je tz +iv m +itf db +iso k +impeach ment +ik lanceloteh +hu eso +house hold +hor den +hani fa +gras strack +gam ine +free ski +fibro id +fi ds +ffe y +f fo +eur on +ernest moniz +enforced disappearances +endimp unity +dro medary +don nelly +dod son +de ba +dal edc +dak o +cur tice +cran ky +confi ance +con tender +citi sports +circu mv +cash for +carra way +cal pis +bro mas +bre al +bour ses +bo za +black jack +ben ni +being boycie +bale stier +baker mayfield +arctic circle +ar sht +angu l +anal i +allo p +al nico +al murray +ag ol +a africa +! ðŁĺĪ +ðŁij Ĵ +yo soy +whizz er +vijayak anth +tyn wald +tre volution +tre han +trac tive +tit li +thir dly +thin lizzy +th form +telang ana +sympathi zing +sub d +stu be +stgeorge s +sterili zing +soul train +ske p +shiva ay +shinsen gumi +self made +scal gary +sb j +sal ice +ri vie +reverber ates +rang elands +ral ston +rad tke +q oq +proven çal +pro khor +pi miento +perfec ter +paw ley +pack pride +oak bank +o hr +o are +no problem +news net +news bud +new sit +nav deep +nap ier +n da +my du +me ineke +master system +major league +mac nee +ma quo +ma cha +len zi +kn ack +kid lington +kat arin +kalye serye +kad al +kab ba +jud ds +ip on +ing net +in he +ikokaz ike +i pod +hy on +hu cker +heinz field +heck lers +harmon town +gul berg +go bel +gar mon +free trade +floridag ators +fan sided +escapethe fate +e bell +dox ology +deb es +day club +d ally +contex tually +conom ic +com and +clai borne +cit rul +chu ll +christma sy +cavall ini +cavall aro +cas sville +cap n +brown hill +bou ldin +blau w +birch field +bio dome +behren dt +bc v +barri ere +bar cs +bal last +b wn +austinand ally +au mont +amil car +adri any +aa sif +& && +ðŁĽ ¢ +à© Ī +Ñ Ħ +yvonnear naud +work sfor +wishi was +willy wonka +vi vos +vann in +tylerj blackburn +tow son +toler ation +to ffice +tlax cala +tfl ers +star maa +stan ko +stall er +ss diski +square d +snug bucket +si skind +shaw nee +scotamb service +sardaar gabbarsingh +san tonio +s med +roy bal +ro goff +rizzoliisle stnt +ring le +reptil ia +record keeping +pren up +poster paper +photo sby +pad dler +pa ho +outra m +out ag +our d +osso buco +omni vorous +od zilla +new blogpost +n kc +music ologist +multi sports +mor iches +mitchell vii +mil ken +mem u +me kas +mc chesney +mat tam +lun ny +locke din +lake city +kristen ledlow +ki thar +jubil ate +joy as +jal aluddin +jal ade +inuk titut +intensi fier +inocul ated +house fly +hi biya +hash em +har away +ha peville +gen ii +gaw xcond +gand ara +g way +fried mann +free wifi +fivb women +fine gan +far amir +estab rook +epau lettes +ent se +en nie +dur ations +dru mma +dil ys +dg wick +dar nall +ct k +cow ans +contact center +col y +co wed +clam pett +chuk w +chronic ler +chil led +chem ed +chappa quiddick +ch th +cen mag +campan elli +caldic ot +butter fish +bur gen +bol sena +bike ways +bi yori +az m +auchter arder +ate me +aren ds +alani z +ai ge +adi sa +ad ream +actu aliz +ab sent +... " +. ** +ðŁĩ¬ âļ½ +ðŁĩ¬âļ½ ðŁĩ± +ä¸ Ģ +âĨ Ļï¸ı +âģł âģł# +ਠµ +ÙĦ ÛĮ +Ì ³ +win tney +wi hm +wan ee +walmart strikers +us ss +tv drama +tro vatore +tor pey +tops field +tito ortiz +tic khill +thoo ver +thiscouldbe us +thelu cky +theband musical +tech tips +team razer +tc v +take offs +t sev +sub national +street ly +stay hydrated +spo to +spee do +sier ung +shu shu +shel li +serious fun +sens ori +sd h +say ani +save gaza +rock ery +rcl ens +pä rt +poy thress +poy dras +popp lewell +pet renko +oxen free +old n +official baileym +no tam +nkotb sb +murwillumb ah +mitso takis +mill wright +maz on +lt gov +lo quillo +le icht +lar us +kul dip +kanti pur +japan times +jamie kennedy +j wh +iam jojo +house sitting +hou traffic +hotair balloon +ho ian +h jr +gocat sgo +go wo +gis day +funeral care +fri m +fire lli +fer rule +feminis mis +every night +em sp +em ount +elec table +dor rit +domestic ate +dik sha +desmar ais +de ason +d juma +cz ars +crest fallen +chew tonglen +can oa +cad dis +bro zovic +bov ington +boo kex +bod kin +bo whead +bluet sunami +bel fiore +ban tul +av z +ast olfo +appellate twitter +aon tro +anti histamines +anant kumar +alton brownlive +alic ja +ale ye +al kas +air frames +ç Ń +âĺ ľ +Ùħ د +ye syou +wü rt +wy ff +wed more +versail les +ve b +u ddingston +tor ise +toni thecat +todayim wearing +tiger style +ti pps +the osophy +the most +tabri zi +sy ma +swa ins +sw ca +stac kexchange +st ex +space channel +sno ddy +sne h +septe t +scutt ling +sal zman +s min +ros al +real romadowney +ran ade +radi ative +ra bie +proto culture +presidenti rl +po ddar +phari see +pec ado +om ad +ocre gister +niobr ara +new telegraph +my i +muhaj ir +mudv ayne +mradam scott +montp ellier +missuni verso +mile ena +mezu zah +md p +mar mo +mad is +lu pit +love by +load star +lo che +lev ana +kyle kinane +ku fa +kraken rum +kne ale +kenne ally +kam as +itch er +it smar +ironwork thursday +internacion al +inag arten +imper ishable +ig ma +hon ka +home time +home economics +high nesses +har gre +h aces +gol fon +gi app +fro ebel +flo rek +flavour ings +five fold +fir str +eight fold +eckhar ttolle +dr f +dor it +don ta +di se +den ou +demo is +del im +datt atreya +darb hanga +cu k +csu mb +cran leigh +cran bury +cot in +choctawhat chee +chamele one +capp adonna +c plusplus +bur gum +buch wald +brie fest +breastfeed ingweek +au sd +ari alv +ali venews +aj na +! ðŁĮŀ +ðŁĶ Ń +ðŁij¨âĢį âļķï¸ı +åį Ĺ +ठĿ +zoom zoom +yn ich +wor sfold +wol itzer +watch word +warr iner +wad low +vÃŃ deo +ver dy +upper most +uni for +un ami +twee dle +tro pon +tril log +trans kei +time warp +th amar +stud illo +strick ler +street scapes +stim ming +spren ger +sm tickets +slo sangeles +sherry ontopp +sen ter +schem ers +sch acht +saw fly +san j +sal tram +safe cofield +rumin ant +rosen wald +ron ja +ron g +rev athi +red olu +reb sbsb +re flog +rafi k +quen elle +porphy ry +pisco tty +per vs +pale strina +omis sions +nord stro +nagach aitanya +mp naveenjindal +mosqu era +morin aga +min ski +mi ras +medi an +med scape +many ara +li bret +lewis ville +lev ated +league intheworld +laryn geal +kar ski +kap tur +john muir +jhump a +iso bel +ind say +hurricanes rugby +hu ila +hol ga +hit music +hearing aids +hahahaha hahahahah +gri quas +gold stream +goe demorgen +go ads +glam sham +geoc ities +gent ing +ge sellschaft +gaillar dia +fum fumfum +fedex forum +eo valdi +energi zes +drag sters +derby shi +dena fil +demonstr ative +dav ro +cu mnor +cis l +ci aldini +christ off +chin di +charle son +chander i +car adoc +canop ic +cal usa +by culla +bure aus +brit tri +br ini +bower bird +bor chardt +black sad +black people +bic oastal +bc bs +bad die +bab o +ba shar +av r +av aaz +ar roman +angel ini +alz forum +ake redolu +ak ame +ag ma +adjour ns +adam west +ðŁĴĻ ðŁĴĹ +Å ij +you zhny +yama shiro +xer is +x ania +wing nut +wing dings +wee eeee +wad ge +wach tel +vil lu +vic pol +ver ducci +v wt +v aka +twop ad +tu ney +town son +tinke rer +thereal kiss +the jake +thar mon +terriclark music +suf is +su roor +stu dia +stron geurope +storm bringer +sti k +stateof decay +sseairtricity lg +ss quare +spor tre +special collections +spe idel +sol di +slugger nation +seam os +saf tey +s ying +ra yel +ra jon +prun ella +pis sy +pas u +p be +ox hey +on usa +obam acare +o cl +nonlge progs +nis sim +news dc +nba history +mur naghan +mi pim +men ses +man utd +m che +lit man +leg on +lan sley +la si +jon bellion +jang ly +j wala +istandwith pp +inter news +hawaii fb +hard knocks +har sin +h jh +gö tze +gul da +go bbo +ger ona +garag erock +fu mar +fr acti +fly frontier +fightfor iowa +fal lof +esp en +eritre ans +emil yos +emc donald +em mitt +ear vin +douce ur +don nan +don ation +do en +dj akarta +design lab +del tad +del lo +damas us +d under +conse jo +clock maker +cl ancey +cic lo +chis en +capp elletti +boot legs +be iner +bar bat +bagh el +bac kedby +bab alik +baarbaarde kho +avic enna +av ner +arach ne +angel ababy +ad ss +ad ry +... ðŁĺĴ +ðŁĩ¨ðŁĩ © +ö k +zo ethe +you will +ye at +y alu +wn v +win ched +water beach +wan k +vibrant gujarat +ven ner +v mm +ut am +university sa +un buried +tourisma us +tor valds +tor reon +tit led +timber man +then ats +th aven +th april +syd filmfest +su arez +sthe best +stepp es +ste ier +sr kfc +spin master +sol aria +sleepy hollow +sj f +side effects +sh att +school book +san ko +samo thrace +sab ir +sa wah +romantic ize +ramesh waram +pur vi +preten ses +pow ter +perri go +per plex +pe si +pe akers +paul brandt +pastu re +pang eran +pablo picasso +pa xon +out burger +oun ders +ou di +occul tism +noth appy +nh tsagov +n nc +myfox ny +mye yes +mun guia +mori bund +morgan field +mis k +med calf +mar ans +mal en +lover ly +laure us +kinder hook +katherine kellyl +kar amazov +k shat +ju ist +jeth malani +jad oo +j ist +j emma +itch in +interior designers +in capacity +hoop z +hemen way +harri ss +hari pur +happil yeverafter +hanson sauctions +ha fer +gon ç +go tyourback +go bots +gin ning +fury fc +est oo +energy star +don keys +di ast +desi rables +dave doyle +d hoop +chil oe +cedarssin ai +car so +callaway golfeu +bwa haha +bri zen +blue origin +blu sher +binghamton u +betty who +bernab e +benjam ins +ben sen +beg on +av us +as om +arqu ite +arielle kebbel +anatom ist +ag ga +ad re +acro ce +ab dali +! ðŁijĢ +ðŁļ´ ðŁı» +ðŁĵ IJ +ëij IJ +ê ½ +ãĤ± ãĥ¢ +âĨIJ # +à¸Ļภķ +ÅŁ a +zdrav kost +yan ka +x tend +womenin horror +winter halter +vikram bhatt +vascul arization +un us +u media +trump s +transliter ation +tos sup +thi eme +thames water +tel lement +tal ita +susan cain +sugar creek +su ar +stylist ically +statue ttes +star ker +sn ice +sil ay +semiah moo +seam stresses +ri ma +rei ver +rat m +prosthe ses +pre date +pil chuck +photo sportnz +peter mansbridge +peder nales +pe led +ou ris +ole sya +northeast ward +night call +neur oradi +mun ns +mor ad +miss india +mesh ach +mcgee han +mass aso +mark zuckerberg +mare lli +mam baday +mah mou +m sic +m da +lo wey +lo ks +limerick clg +lily whites +le pr +lake george +kit siang +kay seri +kap aun +kam pa +k je +juli ed +in sensible +impac thub +haaretz com +h nb +green keeper +grac ey +gin tonic +gi dea +gemeente museum +gam bill +fr cs +flores ville +flit croft +fire bomb +fahriye evcen +f ch +exal ting +eu foria +el st +el ord +eir com +dowag iac +dog sitting +discur sive +depreci ating +daily motivation +cur tly +cuck field +coroll ary +colle tage +co balt +cho isi +chil las +chiangra i +ches ed +cav en +ca yoglu +bisp ham +b unions +arch dale +arag ones +anu mber +and only +amor tization +ambu shing +am ania +agu erra +adidasu prising +acces shollywood +abdi rahman +ab ow +ãģĨ ãģ +âĨ ¬ +zi u +zach ry +z ole +wk bn +wicken heiser +whe ed +weed ore +wa aaah +visitu tah +viny ladd +vic mignogna +vand ellas +va vel +usd chf +tu lu +trust towbars +tow trusttowbars +tobe your +thetribune chd +thereal joebob +thei acp +the pit +tanu ja +star bomb +sr h +snow dog +simco ecounty +shun ted +shaw tv +sh off +sarcopen ia +ru sts +roger io +rodolph e +ro skill +re dedicated +pron ovi +press news +poo h +phy tic +petro c +paper clips +pa wned +p tb +p sch +ouss ama +occit ane +new construction +neel ofa +nd cs +nai docweek +na den +musician life +mu hr +mount lake +metal detecting +mc nay +marque e +lymp stone +lisam arie +lewi sp +lang sford +kwest adakar +kramer girl +kati ep +jes see +jae jin +is may +im pri +ido lish +ho que +hail storms +goo k +goo domen +glimp sed +gio conda +gi path +gev rey +furry tails +fox baltimore +for sett +foll ome +far chitecture +f th +f dc +ey ring +es af +endor p +drumn bass +dri vers +dre p +do dy +dispar age +dilopho saurus +dbel twrites +d for +co scarelli +chi quis +cha oyang +celebr ants +black diamond +astro physical +assun ta +arkra zor +aristop hanes +archon sec +aqu is +apple pie +ance l +amazing grace +all ou +al sea +ak ert +adjunct professr +abo lishes +a pod +a anchal +.. ;) +âģ© : +y anno +ximen aduque +wro e +went zel +weap onize +water shed +w tvr +vel oce +u idaho +tweet ad +trigon ometric +tou ght +thyroid cancer +they callme +thermo polis +ther ington +space apps +snow dogs +smither man +shami sen +ser kan +sch outen +ry ce +roger stone +ro sko +rep ousse +real gilbert +re offending +racha ele +ra hab +r pharms +qui o +pu pae +presbyo pia +petr cech +offer ta +o fro +notim pressed +nor is +nil and +ne pt +natalie portman +myfav murder +msla urynhill +mil spouse +mende sarmy +mbio journal +mati ang +man k +luke pasqualino +lope zo +lo ge +le kh +lam bat +lago di +la jong +ko stov +kee sept +kay sville +isol ationism +innov atively +immuni ze +im hoff +idi omatic +i ves +husk orkut +hu ds +ho bart +har tofdixie +h dcp +gram ado +gir dles +gau hati +g slv +french ay +four che +for sa +fin don +film score +fici ent +evil twin +evan halen +eisen man +dx c +doris day +donat elli +dmy tro +deepender shooda +davi dax +cur fews +cryp ton +crank worx +corning ware +common est +commi sioner +coent rao +choose kind +choice summer +char gé +centra irl +career goals +calder ón +c tec +by re +bur styn +better future +bern inger +bel ka +beaver dam +b how +aptac sm +aphili ppe +amp ly +ame ans +am alive +all meansall +ali ers +ail and +a indonesia +________ _____ +ðŁĺ¹ðŁĺ¹ ðŁĺ¹ðŁĺ¹ +ðŁĴķ ðŁİ¶ +âĸª âĸª +Ùħ س +za atar +walk men +w ör +vis u +vin ni +video in +val ry +us outhflorida +un corrected +uma ir +u oe +tribute to +transfer talk +therun way +thebig issue +ter zo +ter ious +silver lining +si ss +seattle u +sarato gas +rit eaid +rent ola +rasal khaimah +rap allo +ple sio +pit v +pet stagram +pap adi +over by +or bo +oo st +onda atje +of change +nun chaku +nottinghill carnival +nc gov +natali ya +n ourse +my nottingham +musici d +multic ast +mobile first +mm j +mitho on +mirzap ur +mck er +mam ou +m key +luca still +lo siento +lin ate +letter men +lec tura +le maitre +kra kowski +kol usola +kim bell +kill bill +ke aggy +karl towns +ka hr +k pf +ji ffy +jam sil +iwe ala +isti klal +ingex tinction +iac eae +hurst bourne +high jump +hi miko +he ilig +goj evich +gly fada +gen n +fluor ine +fair head +epi stol +eon ni +easter ners +disin vestment +din of +dhivy adharshini +cre asing +cod ling +chri si +chees man +cer vera +cd tv +cardi gan +bread winners +bonni es +bon nett +bne storm +blu cher +black alicious +bla gojevich +ber thel +ballin robe +assn at +ashok selvan +anu ja +ambul ation +akal amusic +aho i +academ yof +; )! +! "# +ðŁĻĭ ðŁı¼ +ðŁĺĤ ðŁĺį +ð ĵ +é Ĺ +å®ĩ å® +âŀ¡ï¸ı â¬ħï¸ı +âľĶï¸ı # +âĻ § +âĹ¼ ï¸ı +اÙĦ ÙĬÙĪÙħ +youn an +yom kippur +wv lt +wies ner +white plains +when callstheheart +wee der +u loom +traver so +to wolfpack +teuk ury +ten jin +tele x +summar ily +stat work +speci ous +space ksc +sof joy +sis back +shen k +shark skin +sha ikh +sh oll +scho oners +sal mahayek +sac rum +s beach +rose anna +ride along +ricky skaggs +ri blets +remb ert +realkid poker +r bb +pub blic +pro le +pri ley +pp is +po ha +os setia +om ms +o ker +ni ketan +ni ghted +ng media +nam iss +my friend +mu ere +model ers +mo ssa +militar ised +metv startrek +mel amed +mc fee +mary queenofscots +madein uk +lucapas qualino +lit le +lim kitsiang +letthem stay +lark field +korn heiser +kn wn +ju bin +jigarth anda +james the +j ne +j gd +io st +inter missions +ingu inal +incarcer ate +in offensive +ideolo gues +id k +icahn mountsinai +hyper sport +ho dag +handof hope +hand anovic +han eef +ham dard +h cfc +guar ani +gu mmy +gratu ities +grand ly +graci ano +googl enew +gi ons +funny man +french toast +explore spaceksc +deniz li +de wx +davedoyle mma +cr pg +cle ang +chang zhou +cathe terization +catch pole +cake shop +ca rel +bur ys +bug fixes +bray ford +brand shatch +bo che +bi dens +bard sey +baeday alden +ba asha +b mb +ay et +athel stan +as cat +art smia +aro ssa +arkrazor backs +arc angelo +ar lon +af ball +> = +ðŁıĨ ðŁijı +íĹ Į +âĺºï¸ı ðŁĴĻ +ม าร +ил ан +zerot olerance +youtube channel +ye wande +yarra wonga +war sash +vote katniss +v neck +v mf +under passes +ulster gaa +tremb led +ton is +ther ave +theq arena +thau vin +sym metries +superf icially +strike apose +st azione +speci alist +sp su +skam italia +sheremet yevo +sgt pepper +se journal +salt ford +rupa huq +roc as +reuter spictures +report ing +ren k +redu cere +red panda +phithe takappa +p inet +nowon air +neu illy +nephro logist +mo tw +mike will +mide ast +meadow dale +mar kin +man teno +mal len +mac ario +ma sika +lovel ife +long well +local beer +leed smot +lay field +kom arov +ko ech +kitak yushu +kenny rogers +ju ho +j ent +i var +hira eth +hemer ocallis +har r +happybirthday srk +hann s +ha ass +green juice +good ness +galve ston +g ll +fru g +fou quet +fo aled +fi ma +faf ner +en gie +en feld +emascul ated +easthar lem +dn ssec +di stancia +di siac +degener acy +dau be +daphne oz +cloud land +chy stryder +chad mendes +cal trans +bre vi +book bub +bobb les +bis nis +big mouth +be za +autu m +ask dr +aldubarkad spreparation +alau ddin +ðŁĴŀ ðŁĺį +ðŁijĢ ðŁĴ¦ +⾨ ðŁĮĻ +yuru yuri +yo ong +yo gaf +worldof wonder +work work +whiskey town +wall work +vol stead +verdic chio +vat anen +un match +typi fies +tte okbokki +trou ville +tribe of +tre va +tra g +tom parker +the style +th ilo +te aand +su jit +su ic +su bu +sow den +small batch +simple pleasures +show masters +short stops +ser gent +secul arist +scor members +sap sec +sand ag +salli saw +ryan serhant +rubin report +ro te +ro la +richmond kickers +reza aslan +revol ved +reha shing +reedtimmer accu +razor smack +ram lee +radhar ani +rad hi +quali es +principal sinaction +pri ed +pre condition +pen so +out sell +ny ck +no zawa +nicky morgan +ni ka +neuro physiology +neeti mohan +ne dc +natural light +my lifeis +mu c +mr tom +mom in +mo je +mi ette +mdcps north +mckel din +may le +marr iner +manoeuv ring +man sel +makesom enoise +m itali +lo quat +liquid mayhem +lill ington +la veau +kulbhushan jadhav +ku lim +khel o +kala handi +julie chen +jud icially +jol anda +jay akumar +it sti +indi v +in saan +in fi +ick enham +hms qnlz +he res +hal abi +gr ackles +go di +gn an +global tv +gha jini +gang ed +gan ti +folk festival +fo aling +flo gging +fis c +fight forthe +felicit ations +fa ille +eri reland +emmy kinney +eiri k +ebony mag +dor fer +dg love +de man +dat en +dar ingly +dap to +collect sideshow +classical guitar +chichester ft +celebr ates +car lease +bund chen +br yer +boxer dog +blog con +bi modal +ba ati +arra yed +app s +ah it +ðŁı · +ðŁİ¶ ðŁİ¤ +ðŁĩºðŁĩ¸ : +ðŁ§ ¦ +íĶĦë¡ľëĵĢ ìĬ¤ +åŁ İ +ãģ£ ãģ¦ +र ह +Ê ³ +} : +yoshi moto +wn du +white party +weare mkto +urbant vuganda +transform ers +to death +the following +teddy b +taylor wimpey +tax slayer +tas lim +tableau public +stock photography +star i +ssur gery +soli dus +simon etta +sathletic shoes +sarah millican +rv h +russell p +ro hi +reign cane +realmatt lucas +rd bms +raku go +ra fat +promo si +pepp apig +patron a +pam per +or un +nor ia +nct fanart +nc sl +mc swain +mc gui +mandre ll +mal ave +mag ics +lud mila +logi stica +lipstick day +league mag +latin as +las well +lan ni +kor ch +knaus gaard +kid son +kal t +j fw +it tuesday +is mart +ing gi +ind itex +imagin atively +ic s +hoo ge +hf pa +halo ween +groove shark +gol dy +go jhl +givingtuesday ca +g weedore +forever leedsmot +featured concerts +fan u +f ci +ex clu +ever after +equ atorial +eg more +dup online +dong saeng +dirk gently +di ddy +den ter +d printers +d hole +cush nie +cruci al +cre tech +cre sa +cliffcentral com +chapter house +channing posters +career teched +cardcaptor sakura +car ney +ca jal +c wre +breath lessly +breakfast club +brachy therapy +bol in +bo sw +bay city +asi ad +arkan san +arie ge +andre wn +aldubhappy baedayalden +al ward +ahu bble +affl alo +ad yen +ach aille +^^ " +? ...... +ðŁĶ¥ðŁĶ¥ðŁĶ¥ @ +ðŁIJ¶ # +ðŁĩ¸ ðŁĩ¦ +ëĦ ¤ +ãĥ¬ ãĤ¹ +z te +z quad +z atar +ye ileen +wide field +wearethe arsenal +w ru +vo tem +ub hai +tu tta +trump lies +tro ve +travel card +tor cida +toge thers +tmb g +time zones +thro cks +thre epenny +thisi sac +tbin chat +studi ously +stagi aire +spro m +son et +sit z +sh j +sel ondon +scitech museum +sai shopkins +ricko wen +raw ski +rail freight +rachael ray +qu ade +purpose tour +pu h +prehen sile +pharmacokine tics +persian gulf +per cale +patient sfirst +pan cas +pab lito +orland ounited +o chi +nv g +next week +ne gie +ne ff +ne bext +morgan freeman +mix e +minnew aska +me p +mat aji +mashi rafael +ma atv +ler os +lapi dus +kh iri +kaz mir +jann atul +ja que +j vp +ishqba aaz +irrit ants +ir m +insi debates +imperme able +im mobility +higu chi +hallow en +gul food +grease monkey +grandcanyon nps +goto southafrica +glass work +gan apathy +g ses +fu dgy +fo gging +flee ce +fi ord +falcon i +fab ra +entom ological +em ami +ek iti +dx cc +deno sau +cyber stalking +crohns disease +cro es +cn mi +clapper board +chris thile +ch isol +cf ca +carib beans +ca zares +bo vell +bed well +bb w +barrow afc +ali sta +adhe rents +ad han +ãģ ¥ +âļ¾ï¸ı ðŁĴĻ +zelda thon +ze ze +wro th +women rights +wel ches +wald wick +ul rik +u iw +the predator +tax march +t song +süd tirol +supply chains +supere xclusive +sub frame +stoichi ometry +spe akin +som ato +so yo +sl fp +sky rockets +sh azz +sam po +sa che +rochdale hornets +rid wan +realmike wilbon +r ce +pu sam +princi p +port patrick +pl x +pe ve +patriot league +pa ston +pa sen +pa har +outw ard +oop sie +ol ero +o sak +nom es +no bi +nel les +na ipaul +multi versity +momo iro +mo sco +mit ter +mic u +megat jp +may aguez +mar ucci +man gusta +m world +luth ria +lock step +linn he +length wise +le sher +la presse +la kers +la barbera +kom ar +kla i +kasab lanka +ir um +ir lemb +intern day +inadequ acies +ima genes +ich kov +icalli han +i fixit +hulk buster +honi ara +homony ms +home staging +holly woodland +hill day +guy ane +gun ton +green sborough +gordon hayward +good girl +goo bers +go guins +glen mont +giz mo +gi velife +ga aclub +for ni +fish wick +fer l +fel tman +ethan vansciver +erne sts +er agon +emilyos ment +ella henderson +ee zer +ed wyn +down turns +dont crack +dd ino +day stil +dav o +cra ins +consumer reports +conspiracy theory +conqui stad +colmen ares +coach d +citi ess +charge dup +chal ices +cease lessly +ce tt +cav ers +cake and +br angelina +bo kan +bil ty +big star +baz il +az l +az ha +avail able +atx weather +arca ea +anthon yo +ang ga +aler mo +aid i +age uk +ag ganis +adekun legold +accou stic +,,,, ,,, +ðŁĺĬ âĺºï¸ı +ðŁijįðŁijį ðŁijįðŁijįðŁijį +íĥľ íĺĦ +å¯ º +าภª +ü re +youn ghollywood +y ia +with me +wi eden +why wellington +well played +we cker +vier tel +vi var +vas antha +vall one +vad hana +u kip +ts ss +trouble shooter +tre ece +travag anza +tomas berdych +thon dan +thal amic +thack ray +te du +stade toulousain +ssc s +sor bus +social protection +sme m +sho tover +seen u +rho dy +read yyy +pur éed +post codes +perce val +pawh uska +patti smith +over dressed +oui met +oooooooo oooooo +ollantay tambo +od da +occur ing +no aas +mun ic +modern fam +mob in +mk p +missing no +mik ko +mi sen +mayweather v +mate er +madhuri ma +ma sin +lough nane +logan square +lo or +les seps +lab oured +la font +kra h +kad jar +k tx +jos é +jon gh +john lock +jeff dunham +ist i +in yc +iheartt ally +ho ima +hedley online +hau denosau +gro yne +gor inchem +goo ssens +food city +fau zia +exp consulting +expconsulting es +elem mathchat +egi dio +edwyn collins +eat well +dym church +dof the +detro i +den z +defer ral +daphne caruanagalizia +concent rix +comorbid ities +cat elynn +bt ls +brid well +bra wijaya +boy ar +beli z +barro wraiders +bal dini +bai jiu +awa ited +aw tg +aw aki +auster litz +atra de +archae opteryx +adjudic ators +ach oli +a hy +ľ ëĵľ +z wave +your home +wo ahh +winning wednesday +westvirgini au +wan ge +wa sif +vi elle +vec tored +tx politics +tomor i +tin chystryder +thenight manager +theatre uk +stur minster +southwark cathed +schmal z +sarban es +sant illan +sam l +ring mer +ri et +rath gar +rant oul +radhamohan bjp +pun ked +planet comicon +phan tic +paul polman +os am +oet ker +o es +nott age +ne ven +multi use +mon agh +mira beau +mille miglia +micro biological +meetthe artist +medi agu +loe wen +l sr +l sh +ke aney +ka ÅŁk +jag jit +i dig +hex um +haz ama +gou ter +gentle mens +g sfc +fra sca +fr ö +flower stagram +esc ro +ell inger +ed corlando +dro oping +dor mice +ding er +dies fc +de balt +debalt seve +daw yck +darao briain +d age +co hosted +cla u +ci alis +chocol aty +chin may +cac ia +bret bielema +brahman yam +bott en +blanc as +black on +bla d +bey ers +beir ness +bab bs +anne cy +angi er +ana huac +ale gg +agger scricket +ag lew +aer u +âĮļ ï¸ı +zap ruder +z burg +xx xiv +vir gina +v ong +that boy +tele casts +tc margate +tar di +sun ye +su er +stani er +squar tet +sickkids news +si mad +shoe bill +sepul cher +sarahm gellar +sach ems +sa ura +rich woods +ress ources +real sway +reagan omics +re tellings +re marque +ra ijin +quer cetin +pyro graphy +punkand stuff +principal es +plat oons +pl ari +pin der +oz ge +over populated +ny gaard +neu romancer +nativeamerican heritagemonth +nap aracing +nach t +muriel bowser +motor mouth +mon tt +mo is +mercer sburg +maz ama +manj ari +mal c +m js +lu vr +lin oleic +kwang min +kir n +ju u +japanese art +j li +itso kay +itsmohit sehgal +ipp f +inag ur +im planting +ic tp +hil den +havean iceday +har by +han cox +gro fers +grand niece +glo p +glasgow uni +gladi atorial +fm drive +fi on +feeling festive +fair wood +f legg +er col +em rich +e bc +dr ongo +defe o +de wolf +de ux +day ang +cycle tour +cur ate +cor avin +co dsall +circuit ous +che ena +cate rer +cart lidge +can y +brook green +boo gaard +bol ick +blue bear +bin ding +bi ms +bale wa +ayurve dic +auto express +app ena +ang ai +alo gic +aj in +agu er +addic t +ad tech +aco e +ðŁĴª ðŁijĮ +ðŁijĬ ðŁijį +ç ¸ +ã ı +âĺħâĺħâĺħâĺħ : +âĺĢï¸ı âĿ¤ï¸ı +zar korel +xi en +wil kes +wfm z +wap akon +wak elin +video tron +vass allo +v wap +us military +un graded +uk ho +tusc umbia +tsumt sum +toro company +tool kits +tomar ket +thondan kani +thisis lany +ter fs +tang lin +sura u +stock wood +spor tireland +spar sh +som alian +sidd ons +shel a +sham ers +sg vn +sf symphony +selvar aj +seb agai +sant illi +rumin ants +rt ls +rr v +richardy ap +rex ford +qi ong +precipit ous +pat ta +paget paget +over abundance +olimpi ja +nu dged +nu dge +non pareil +noi settes +n ni +musi q +mur rells +mu ds +mon tac +mir s +mingh ella +maric hal +makebetter happen +ma eyoung +ludd ites +luc ban +lou reiro +lo tos +ku mano +kre ta +kha dka +jess on +je sh +jane te +in news +her javec +helioc entric +head rick +hackney wick +h lundqvist +guil lot +grun dig +grin drod +grimac es +g sma +forest fire +fin chel +explor ation +ex upéry +eraser heads +dvent ures +dun g +dor rington +dj tira +deser ters +der rek +cur du +ct buh +cra iova +colle dge +children shealth +caren cro +cal lup +c twx +brock university +br antly +big fan +beyourown boss +ben na +beautiful game +bb curdu +bat kid +barbi ere +backin time +ay sen +as cher +as aram +albatros scam +aire uropa +ag ac +adom ah +ac rm +ðŁĺĺ âĿ¤ +ðŁİ ½ +ÙĦ اÙħ +yassi zzle +wine growers +wilhelm sen +who dini +wer oll +water fowl +wai alua +w shs +vine sauce +vi lest +urban ecology +u ssi +twit ness +tro gon +touch down +techno logic +tar chives +ta eler +sudar san +stump towncoffee +stre amy +spar go +sou ra +sni k +sk ow +schmid t +sam ah +sab atino +running uk +ro gge +public education +pu ber +pri zep +pied ad +p ting +nebra ska +naz imabad +naj ran +mun di +mo ed +mitchel stown +mimi kyu +mil ke +mi yam +mann ering +manjun ath +mac iver +m ten +lyn g +la gat +klein burg +kay ako +jor dache +johnnewman music +john waters +jasmin walia +indiat vnews +iey asu +hu moured +ho fers +ham brick +gurdas maan +great comet +gamer gram +ford trucks +fi lem +fal ck +f ys +f ct +er tel +eleanorj calder +duche sses +drought lander +digital leader +di parole +dend rum +demor alized +demar com +cray ford +cp x +cosum nes +cir colo +calli ance +cal zada +braun stone +bott lings +boo ya +black men +bhu pathi +bestin the +bailey lakings +au fman +aspir a +as lef +ariad na +ar tec +apple pencil +angelcake pics +ad dd +ab mb +ðŁĺĤðŁĺĤðŁĺĤ ðŁĺŃðŁĺŃðŁĺŃ +ë ¡ľ +ç³ » +âŀ ¤ +à· ĥ +ت ÙĬ +zing erman +x eter +wright stown +woo sung +whit elock +war bling +wa chau +ve ctis +us en +ty burn +top dog +tb v +t sel +swim swam +sud afed +spectro photometer +spare parts +space exploration +south ard +smart cities +shi raz +shar an +se inen +scu tt +scout ing +sac i +rubi x +ro milly +rev engers +re marry +raghun ath +ra ver +pv da +ps itt +prescri bers +poc so +po ppo +pl zzzz +pj py +ph ua +par asy +pac em +p nj +p crm +over charge +opening soon +of ilm +o ton +ni archos +ne gin +national bossday +mzansi magic +multi state +midge ure +mb asketball +mathi as +married atfirstsight +mar low +malcol mb +ly ak +kre utz +kiri akou +kinka jou +kei thing +kean sburg +karmal oop +kalam kari +k netz +k alem +james blunt +intra squad +iller time +holo graphy +hi roh +hal tom +gri maud +glovers ville +franki ekazarian +flock hart +facial recognition +everyonec an +ere k +ep at +ec lac +earth sea +dug gie +dub fire +drew lachey +dont forget +do vid +direc ts +descendant softhesun +degu station +daniel marven +dales man +da rena +d nab +cr ary +compac ting +cle wiston +ci ones +ci ety +cat andu +carabini eri +business model +bp mn +blan ck +be ok +b hog +aye shat +apar ra +am th +alkal inity +a peoples +ÃŃ m +yu uka +yeas ayer +xmen movies +west garth +wapakon eta +vi shesh +uss ocom +tu tup +tu mon +tri poto +tor oro +tor is +therise of +thereal russellp +the progressives +terre stris +teo chew +tar ahu +tae jin +stan fill +stag gies +spn famiiy +spectacular nwt +sketch bet +sin love +sho dge +shin ies +seku low +se gui +say egh +sar dana +samanth as +rescu eme +renn sport +refugee schief +re double +rat pack +randy moss +prith vival +pric ed +power lessness +pierre pont +phosp hat +perpetr ation +pave se +parab éns +pa ole +p wb +on duty +official psl +no zaki +no wing +ne wart +na via +mu tism +modu lators +mi hir +marypoppins returns +map maker +madi ha +ma ben +longer term +logarith ms +le amy +lake hurst +ladi ators +ku shida +kate mansi +ju ster +jan ele +j heri +j hen +iso ch +ir leach +inde mni +ichi kawa +iam mr +hopl ite +hank green +gretchen carlson +gine st +ginapu stor +ford ing +fashion finds +fa den +ess ent +en ationalpark +dun given +dontcrack underpressure +dom brov +dil fer +der mis +de very +cynthi abailey +cu lum +con signing +cocor ahs +chortle town +cho ise +cheap ness +ce fas +cc bvb +cal pur +cabinet maker +cab bag +c ba +belphe gor +bag gers +av c +av am +art ford +are ola +anton iom +antal yaspor +and rada +afilm fareawards +ab ingdon +ðŁijı ðŁıĨ +âķ± âķ± +ÑĤа илан +ÑĤаилан д +ã es +yl td +wo er +whit marsh +waldor fa +voltac atalunya +vander hoof +ut me +un mastered +truman sburg +the merry +the hype +tele fon +super volcano +spad aro +sin kers +ser ral +se pak +schön brunn +scen es +sam bit +sal ter +roundrock isd +river way +reali gned +re qd +push forward +pu sch +powder ham +pie man +pi era +pen alosa +oreilly media +on dcp +of shame +o gee +no dui +new beverly +natlib scot +national policeweek +namad gi +n tom +mu du +mor ti +mon ton +min jung +mel bour +medi acity +mcgra il +mc kiernan +mazz oni +martin imonday +mar tech +ma ven +m fo +lliber tat +letter forms +le the +lar aza +king g +kids activities +k liz +judd monte +john king +jere bko +jak un +jab arda +improvis es +i versity +i hn +home theater +hoki enation +hick en +har king +gu igno +gb pjpy +g pw +francis can +fo tor +feels goodman +dragon fly +dr p +dl ls +dhe yar +depreci ate +demon ization +del ap +de ads +dd ca +d hee +cur tailing +culture l +collecti ble +co sma +clay ne +chrono graphs +che re +chas sagne +ch one +cab ras +bren da +bluecol lar +bbc so +basti da +bam bi +ballet day +balder as +bal zer +avi dheyar +archer field +anti mony +anna akana +amo on +ally girl +alco y +albu men +albu maday +ac rum +ðŁIJ¸ ðŁIJ¸ +ðŁ¥ Ħ +ë§ Ī +ç Ĩ +Ø® اÙĨ +мÑĥз Ñĭ +zo ie +your allypally +xian lim +westwoo done +wein man +war fighters +vul pix +un compressed +un acknowledged +tshep o +troglody tes +toli sso +tho tep +thisisp vris +thed appy +the esh +thats what +thanksgiving week +tal er +take backthe +takam atsu +sx ii +suki waterhouse +smol ders +slopp ily +skin health +she arers +shawnmendes the +shar jah +shak shuka +scrap the +scho eller +saveour seas +salary man +run asone +roy c +ri fat +revoke article +red sonja +re bb +rand b +ra strick +ra son +quar shie +pre so +pre k +pot coin +pol ansky +pleasee eee +peter scanavino +periodon titis +pe dley +pat aky +parvo virus +p bhushan +ow y +omi ami +official ghsa +north central +nie bla +nhlon nbcsports +new era +neko case +n tia +muswell brook +mom oh +mik kelson +microne edling +michael wsmith +mer in +mckin zie +mc wane +mark dice +mari pos +mar os +mag adi +ler ouge +le pus +lam berti +kno pp +ki kki +ki hu +ke dai +katheryn winnick +k ko +jon montag +jamiele ecurtis +ir well +infu sion +imp ru +im purity +im par +hy tner +hu ta +hs bc +hoag y +his sy +himm el +hey erdahl +hersh man +heir loom +healthy diet +he v +harts dale +har uno +gro tte +gom on +goel bjp +ge mili +fuzz ing +french wine +free state +fore vs +food park +fon o +fay oum +f gg +dessert day +david harewood +data analysis +d music +cyn wyd +cycl orama +cras sula +cor dele +chag ford +cecil erichards +catelynn lowell +cas u +cas sock +brevi ary +brave souls +boss u +bi ram +bha jans +balmoral show +bal boni +b under +aver e +artscouncil ni +ar ji +an san +an ali +ail eron +agu er +ag ical +aaaaaa and +a versa +ðŁIJ´ ðŁIJ´ +ðŁ§ ¹ +ðŁ¥ºðŁ¥º ðŁ¥º +ðĿĹ ĺ +xxx vi +ww mtnews +wood thorpe +whar ves +wel over +wag ener +vsco de +very proud +un justifiable +un burnt +ue matsu +u ef +tulip siddiq +ts ys +tri shul +trampal ji +tol tec +teacher prize +tai shi +syn crude +sunshine coasto +su sty +south offrance +sha aan +seper ated +savat age +sau veur +sam mies +sal az +s dag +ri bet +re twit +re ines +queen bee +pun to +pp ke +persu ader +pay n +pantom imes +oun try +or ko +open mic +only you +ny stag +nairo bian +my jasper +mor ny +mor ioka +michaele mann +mean smore +man ha +loy ally +loc atie +lam pre +la thi +l luis +king scote +ke mer +kaz imi +k naw +jakeand amir +it uri +in competency +hispanici magines +hen rye +he dd +he aping +hair port +ha sui +h sct +gur um +glo e +gh ard +gggg ggg +gate am +forest school +fle te +fla shover +eschen bach +erd rich +ej ad +eden derry +dy y +du su +du bc +dialec tics +del acor +defi lement +de sus +de ob +dan ede +dal arna +daddy shome +cross keys +cro mer +concili atory +col po +chri spine +cham pe +c ation +but true +brock ington +brecon beacon +brad ner +blur ted +blum spew +blom berg +bha gal +ber an +bel grad +baf tag +at allah +artic lev +arru da +army rotc +an tt +am mo +alit rophy +alam enti +aed an +ad w +ðŁĺij ðŁĺĤ +ê¹Ģì§Ħ ìļ° +м оР+Î · +z na +yun o +yu da +ym outh +working mom +wild water +whit lock +wedding fair +w woof +w sn +vo dianova +un seemly +twitter storm +tf h +textile design +t dx +straight ness +soci opolitical +shek hin +sh ung +seabour n +se aways +rock away +re zende +raj shah +quant cast +psychopharmac ology +pietr angelo +phil odendron +phe x +pengu inte +pend ence +peer j +paho kee +pa pe +od awara +net books +ner gal +neh gov +mtvbase africa +mill street +micro scale +meh wish +max ence +mash rou +mand ingo +lu ers +lokay ukta +labor atorio +kalon gan +kac z +jim inday +jan a +jagmeet singh +jack knife +inside ferrari +in hand +i vies +hi mb +hatchim als +har ang +gau mont +gar bled +fiz dale +fig tree +fidd lin +fanci ulli +fal tan +emily y +e bbs +div yak +dis da +davidax elrod +d lm +cle ws +chri qui +chatur anga +cedar cove +catch oftheday +bush whacker +building bridges +british birds +brak pan +bo snow +black swan +bi sha +bel bin +bal lester +bab bb +ase ema +am z +am diabete +am bia +ag ito +acaci as +% âĢ¦ +ðŁĺĬ ðŁĴŀ +ðŁĺĨ . +ìĺ¤ ìķĦìĿ´ +æĿ ¥ +æ ¶ +âĢįâĻ Ģ +° ! +xxxx xxxxx +xim enez +wyn berg +wom bat +wi ed +wan king +viadu ckworth +up lifts +ulla dulla +u ea +twent yman +traut mann +trade centre +towno fficial +top cat +to losa +theori zed +thenewe uropean +the torocompany +ted to +tac it +t mux +student debt +spn chi +serv ais +sen zo +saw ada +sale sians +sal twell +sa q +ro do +ri spoli +reel sofficial +re join +re home +ram lila +rak ish +purple day +pre fabs +plym stock +plot lib +pi azon +petrol head +pav on +palm tree +pal med +pa sek +p wu +ori go +one planet +nikk il +nc ad +nas ahubble +n the +mobb deep +mo bike +mira sorvino +mha iri +mechan ised +mb mbam +matta rella +mat z +manz anares +mall ari +mag dy +lo veridge +limb ed +le panto +l pm +ko suke +kelly sue +jun in +jay apura +it sco +io les +im monen +ici mod +heu res +heteronor mative +helpto buy +har tog +gu yot +gly col +ghu rair +gh in +ger tner +genoci de +gain ax +fri erson +fitness journey +fer mor +feature ttes +emc gee +el chapo +e kin +dor rell +don air +dog gie +der mer +den eb +de schi +dali da +criso stomo +council members +cornelis sen +coo lie +colli gan +codi fied +clan destin +chuk ar +cho wa +chen in +chat ard +char vet +char ged +c mma +bute town +buech el +budget travel +bel gaum +bb cred +bar ce +bah ri +bab alola +az ion +awal sh +aus stellung +as rock +alvar o +aless andr +akademi ks +ai wa +ahmadre za +aditi rathore +ðŁĻĮ @ +ðŁİīðŁİĪ ðŁİĬ +ðŁ¤ĺðŁı» # +íĶĦë¡ľëĵĢìĬ¤ x +yar ov +xpla p +wvprep fb +wicker park +wa chowski +vinay avidheyar +var os +va ide +us ace +urvashira utela +upheav als +un learned +tre i +tre as +toread or +thedavid crosby +the cloud +temple man +team on +tb ayne +tad lock +swiss made +stu mbo +stu arth +squ ill +spaci ousness +sol arec +slopp iness +sle ben +she x +se be +roy lilley +re kka +re ev +raz van +ran maru +rabbit mq +qalam oun +pre bble +pi at +perfor ate +pat ara +par ga +pam lico +pam ilya +over steer +onelast time +o tun +ni mrat +nfl kickoff +nest lings +my name +mother languageday +mini o +meyer hoff +men dips +mar iss +mal formations +m prnews +lyth goe +lopezo brador +le lia +le kha +last nite +la duke +kyr sten +kv ass +kur gan +ku kul +ks giving +klu b +keny aredcross +jou et +jewl ery +janasen ani +it sat +it oh +is ara +interce ding +inge ducation +ic ron +i priyank +hebb al +hand sup +h he +gyeon ggi +gor rell +global new +gig wise +garni delia +fun belt +fon taines +fold out +feel better +eu dat +eri elive +english town +elph ick +ed guy +eaz ye +eagle sham +e one +demonstra bly +de ya +dab bles +ctv wpg +cl on +chu ter +charlied aniels +cf kargentina +buñ uel +body language +bleak ness +beso in +bent aleb +beat it +be ab +back off +b nai +b co +b chockey +avec chia +auk land +astronom ically +as wang +ar ric +apilot seye +api zza +amar ina +alph ac +ad lv +achi mota +=_ = +ì² ľ +åŃIJ éŁ +z ary +yy cevents +yehu di +wol man +wild wednesday +wasi kowska +visit goldcoast +vi sting +unity tips +techno logie +sul phide +stre at +sovere igns +shar n +sh ats +seven logics +seton hall +screen printed +san cha +sa kuma +ra ymer +pu review +pre amps +pr cc +poké mongo +perfor ations +pe wee +pare jo +over man +ot z +oh the +oh saasports +mohit sehgal +meh med +mcfad yen +mark lanegan +marc garneau +man j +madri gals +luxembour gish +lp wan +lookat my +life ontheroad +kin dra +khar ge +kei ichi +kai grun +kaigrun witz +isu net +insinu ates +ii ed +ih me +hewit son +hc sd +gro tta +go wri +gau ck +gandol fo +gab c +g ach +fro mn +forest whitaker +fe k +family medicine +energys aving +ec sc +ear wolf +dont nod +dj mix +dis ki +dir lingusamy +dand eli +dainik bhaskar +cork city +con cisely +college basketball +clear ly +cla yo +chu giak +cho sin +chi kin +care for +brunel uni +bio systems +betibachaobe ti +bach rach +az ami +at socialmedi +ash elf +as cott +as cal +an tae +am rav +alpham ale +alli want +alle go +ak sel +$ \ +ðŁĺĽ ðŁĺį +ðŁijģâĢį ðŁĹ¨ +ðŁ¤ Ĵ +âĶĪâĶĪ âĶĪâĶĪ +zon ne +white tip +what about +weing art +un ceded +turner prize +times live +time scale +ther os +tg k +ter centenary +talyl lyn +syl viaduckworth +swing arm +substance designer +su td +su mber +stor rington +space govuk +sp et +sl bc +skate shop +sharepict chibi +sent ries +seewhati didthere +san ssou +sammy wilk +sam bha +red row +re power +ramnath kovind +profun dity +poly phia +pero gies +per vad +pan kow +o estrogen +nor tel +no break +niagar aparks +nh mla +nc se +murrumbi dgee +mo val +mnc tv +mis matches +mi ket +mex chat +mat plotlib +marco g +man nu +malacan ang +ma stung +log ers +lj mu +lis sette +lign um +lan cement +la gran +kristy na +kristiann airn +kam ila +k du +jyo tish +jud gen +jin xx +itu n +itu ation +ipp atel +intrigu ingly +inte bnlditalia +im ple +ice music +hun ziker +hi bees +hend ren +hd k +haver straw +h ico +gr r +geh ring +gar dot +foun taine +flo ret +fertil ised +fer net +felicit ates +fat rophy +etsy sale +epo ca +eh v +earl sfield +dwee zil +dunhill links +doll houses +dis respects +digital sales +dietiti ans +de spots +de shaunwatson +dak u +cr tv +count mein +const anta +co rella +clin k +chuck wendig +bri sco +blac keyed +bhak ta +benbe cula +ben nion +bar go +ba sto +astralis gg +andrea petkovic +ame z +al awine +afoo tball +a issa +:' ') +.. ??? +!!! < +ðŁijįðŁı» @ +ðĿIJİ ðĿIJ +é Ĭ +ãħłãħłãħłãħł ãħłãħł +âĻ¥ ~ +âĺºï¸ıâĺºï¸ı âĺºï¸ıâĺºï¸ı +öl nir +ê t +~ âĻª +yu bikey +yellow fever +y ato +wrigley ville +wr fc +williamand mary +wh arncliffe +war mest +wang chuk +wall dorf +wa ju +urban ity +up ending +trach tenberg +to sachat +ti ar +tho orn +the tls +te fillin +su in +stiff en +ss wiss +spru e +sol la +snow cap +snoo ks +skyblue fc +silk screened +shi rob +se bright +school sport +sarang ani +sa po +revel a +re quote +ra ppe +r ó +pyrene an +pend ine +paul k +par go +panam acity +painting silove +ot an +order now +olivi ers +nws seattle +neuro toxin +n scorp +movietv techgeeks +morning coffee +mor tales +mi ral +me demb +margare tha +march itec +mar cano +manz ini +lion sclubs +limp bizkit +ker pen +kel mscott +jjab rams +j atta +itv wales +ici m +i septaphilly +hu eco +holm strom +ho sein +ho ola +hit c +hi ley +hat ice +happyear thday +gurmeet choudhary +grown ish +gro aned +go canada +ger sen +gau cher +gar bag +gango tri +fu jitsu +foo bar +fire hawks +fer dy +fat berg +far rand +face plates +equin or +epp endorf +edchat nz +dur m +disch em +demol ition +dee z +copper belt +com pres +colored pencil +cog burn +clinton fdn +chisol m +cedarcove tv +cat zingano +can son +cam ba +brant daugherty +az aad +austin isd +at ours +astro boy +asak ura +ap ier +annual report +and dean +amal aysia +alphabe tic +albi rex +ahed tamimi +aden tro +ad har +abo tt +ðŁij©âĢį ðŁı« +ච½ +à ½ +york sambulance +yo cum +yin z +wye valley +winch more +westpac stadium +weather caster +water marking +v precords +upthe dubs +uky p +tw ts +trit ic +tourde yorkshire +thesm sd +theori ze +the weirdworld +sunshinecoasto z +stur gill +steak n +spiegel online +sper kin +siri kit +she han +se aming +sc rabb +save hannibal +rosal ine +right scon +ren du +red card +rang sit +rak shak +rac ingextinction +prin toctober +pre ppin +pre cis +ppe al +pow assan +poo ds +polychro mos +pir bright +piezo electric +perfect as +patt an +pat os +p elling +on li +oh sen +nn h +ngw sd +nd ale +nar dini +n infa +n ating +muhl ach +motivational quote +monster high +miam ipd +mer aj +meanwhi lein +lucas cruikshank +ligh tof +leapfro gging +kremlin russia +kan angill +ka or +ine au +hunnic utt +hundred ths +he ger +hay seed +gra byour +fleis chman +fen berg +fa herty +econet zimbabwe +dt by +differenti als +del ma +death valley +cp ca +clear cut +che kk +cer ium +cann ata +boycott nfl +bookweek scot +bby awards +bay ing +baske tof +ball ance +ay on +ar sh +and you +anastasi ya +amé ric +all ying +ali ke +ala ura +al mont +ad zuki +ach mad +a als +:" "> +æĪ IJ +⾨⾨ ⾨⾨⾨ +ච¯ +ઠ¤ +zi zi +zac chae +yom bo +y q +wq am +whit emountains +voteblue to +vol turi +us bankstadium +unil incoln +und mhockey +umbrella academy +uc v +tri mb +tourism week +time les +tile fish +the amy +tart ine +tang ina +tan ith +states manship +snet tisham +smu ggles +smir nov +sky copter +septimi us +schu maker +sch all +ruth lessness +ru ffins +red cap +red bus +randall stown +rad ziwill +powere dge +pol ari +periodic table +pager ank +owl boy +over print +ong ate +no bler +naz eer +national doctorsday +mor well +moe ed +min dyour +ment as +mclaren vale +max joseph +mat tz +mary mary +mapper ley +manu shic +mandi bles +mahal akshmi +ma ek +lith os +lat terly +lam onica +kö nen +konzer thaus +kir rie +kingdom of +king aroy +kess ock +kam aal +kai ja +jonesc apo +jim jonescapo +jackier obinson +ja siri +j bf +ism raceway +is sf +ing space +hou renergy +hindr ances +hay dee +hann is +h fuji +gen erico +gar ak +filli p +fe ssenden +fan boying +enor me +em placement +ec tin +dow l +dont miss +dms guild +divis adero +di sher +demarcom urray +debau ched +cs ds +cont actor +com ingof +cher iton +ce mpire +bo ilies +bo dd +blade andsoul +black all +bbclocal ite +av ito +au riga +asa hi +arizon adot +anton ine +andre s +amar ket +( âĢ¢ +ðŁĴĶðŁĴĶ ðŁĴĶðŁĴĶ +æľĢ æĸ° +ãĢį âĪł)_ +⼠º +à¸Ńะà¹Ħภ£ +£ ¨ +zef firelli +yyj arts +yu mmmm +yar darm +ya semin +x ri +world tv +wild lings +wi dgeon +whel ks +we stra +vir ile +up selling +tru enorth +time forchange +thor ning +the montydon +thai day +th june +tele mundo +surrep titious +substanti ate +su dip +steph breakfast +steier mark +steel heart +st dm +spar ta +shu ja +sha ista +sequ in +se tubal +salisbury cath +rubb ings +rollsroyce cars +re formulated +re ath +quanti fies +pur ity +pro pan +po stre +par abol +op ent +on ye +neil son +neal mccoy +my protein +mx f +mue stra +mr george +mou at +morpho genesis +modic um +mo dic +misidenti fied +michael jordan +mia universe +mer n +melbur nians +mel ded +man tooth +man kin +mac master +lou cks +litt leneck +la sk +kri sto +kpr clocal +kipla gat +ki gali +juan fran +jared kushner +jab ong +idoli zes +idesof march +i the +hun ny +howtogetaway abc +hospital isation +hn tb +hiz bullah +har pal +han sel +gy da +gun dar +gordon stoun +go bows +gerry mander +gang aa +friday focus +fly half +el h +eco school +ea sia +domain name +doing business +desh one +der ic +deni ability +debt free +day u +d itta +cush endall +cun nington +cud joe +cu ssons +cor rode +con gos +christma seve +cat rin +cast ag +carfag no +car ballo +caci que +c saba +buil ders +box of +bom beck +boe ken +beparto fit +bel lotti +barber life +b zh +b fa +autumn statement +ark hu +ard ha +arch a +ar h +analog photography +alban i +ak bari +aeron aut +ad cruz +aa viation +a abb +? ): +ðŁĺİ ðŁĺĺ +ðŁĺį ðŁĺŃðŁĺį +ê± ¸ +ล า +Ñ Ķ +zon go +zakhar chenko +y pn +won do +women sbball +wb tourlondon +wann see +vo well +vig eland +un sympathetic +un ilife +un coupling +um bel +tivo li +thibau t +the arts +techno crats +te ti +tal ente +sugar rush +sto i +st immung +spring has +spirit of +speed art +southern mis +snoo zin +sil ene +shul kin +shu pe +shoul dering +sh su +sen dero +se ery +scare dy +roy moore +ro vi +rann fl +qi yah +poly chae +phi pp +partic k +origin ators +oleksi ak +ne shat +n irs +mur ri +mr porter +morgan ville +mon dy +mike schiemer +mi fi +met zen +me ers +mari do +mar nock +man olis +m ny +luncheon ette +lud lum +lincol ns +le akers +ku bler +ko viÄĩ +kit tredge +killing sworth +ki hara +ju mble +ju cy +jay lin +jackand jack +j hr +ital yday +ish afoundation +ir regardless +ir ani +iono sphere +inter states +iman gel +ifi were +human ness +hri sto +ho ess +hick ox +gv m +goback modi +gill ings +gil key +ged ney +full time +fluoro carbon +fail ure +ex arch +eric hard +ent rapped +elliot ts +el zhi +eh ner +duci dni +du par +digg ler +diff rent +democrati sation +dc s +david love +datdu debp +culp ability +coffee bean +co yl +co ston +clean seas +chak de +capri sun +cad dis +bu ari +bry her +brock ley +bro ich +bonniemc kee +bo ey +blin kers +bel and +bari atrics +bar ad +bagu ley +at large +arri vo +and wine +all ter +ak tien +ag ario +abi erta +ab ike +aad c +ðŁĩ ° +ᶠł +à¦ Ĺ +édou ard +ze ist +yout u +yor chard +y azz +wo bbled +with syria +weather authority +we heart +wan chai +vo ynich +usk asing +un selfishly +un encumbered +ul ly +ts arist +tofthe month +te cla +te americ +sp hil +sneaker pedia +sku b +si kit +short lived +sch rodinger +sas kag +river kings +reson ators +re ordering +rashmi kamand +random ize +push button +pri ons +pre party +portrait challenge +phil lauri +pha go +people with +pee ked +pat man +oste ology +onthe spot +ontari ondp +onair romeo +omni pollo +nuclearbla steu +nu un +nsc c +mor lock +model trains +mccl urg +maxi mization +man ser +man jit +man booker +lud wi +lit as +lisal oeb +lin enews +leop ar +lennon nme +lb su +lag man +la skar +ko lod +kingdom comed +ke uk +kap uskasing +kan eda +kal kaska +k jl +john sburg +idoli sed +ide v +i muk +hind head +hem nes +ha ins +gazette er +future s +fox x +fox man +fore going +fjord norway +first snow +ff ington +expun ged +esp in +esh re +end humantrafficking +en tailed +embarrass your +ele an +dro x +drmike murdock +dow ska +di radio +def jam +deben ham +danede haan +cor darrelle +community garden +col clough +cochin ita +clear out +church man +chil lest +ch aley +cas sel +c siriano +brook sby +bron y +bo cking +blind cat +bi aus +benig ni +bat ton +baskin robbins +bang ko +bag gie +axi oms +aund h +as ba +artu ria +ango stura +and real +amwriting scifi +adobe premiere +absr dnews +abse con +: âłĢ +ðŁĵ½ ï¸ı: +æ ¤ +z auber +workout motivation +wood stown +will hill +we ste +ve don +var ta +under wear +under insured +un gal +u mofficial +tri ot +tou rers +ton go +tiv at +tish man +tic s +ti gnor +the time +the ic +tej eda +te emo +tatu aje +t acy +sö der +sur anne +space museum +sou lection +soci ali +sm ys +sky watcher +sense of +secret garden +sde france +s studio +rho dolite +rene au +recru iter +ran vir +ra oul +protom artyr +proof of +produc tive +priz ren +pretty woman +pe can +park chester +par in +opp ong +music studio +mun z +mis laid +minu ets +michael angelo +mic o +mathis fun +mar wick +mal fi +maeyoung classic +lee anne +l cem +kn h +ki ren +ki am +job vacancy +iwant one +ip cpr +inge xpo +ilu sion +il ament +ifl science +hutch ens +he parin +haryan vi +ha sani +gleneagle shotel +gir lup +ginny goodwin +fu z +frit ts +fin ito +felicity huffman +fan sn +fair hill +encroach ments +el of +e town +dö ner +dow dell +der ksen +de pasquale +czecho slovakian +cox ed +coming up +cholmon deley +centime tre +caz adores +cambi um +bur dwan +bun z +bug ü +bu gli +br amb +bo ell +blu rofficial +black fire +belle vu +beauti fu +b mbf +b cause +augu stan +atal ant +al shaq +airdrie onians +a experience +-- ( +ðŁļ¶ âĢįâĻĤï¸ı +ðŁĺŃ ðŁĺĺ +ðŁĮ¿ðŁĮ¿ ðŁĮ¿ +ìķĦìĿ´ ìĺ¤ìķĦìĿ´ +ê³ ł +âĹ ĺ +ঠ¹ +е л +ál bum +y vel +ww jd +wrath ful +wil i +wal is +vampire the +v ys +un molested +ul ars +tri aled +train to +tok ki +to tty +tn leg +tech land +team red +tar jei +summer bee +steam newrelease +ss ow +soor ma +somers worth +simulation friday +sie grist +sho velling +shag ging +servic ed +sax a +rom ford +roch dal +riv alling +ret te +regre sa +real martin +ras gulla +pru frock +picto graphs +pi ad +phal anges +parachu tist +paddy mcguinness +pa iz +out eni +oo zed +ny arla +nic anor +natu ur +muse i +mu ddling +mu ad +mr teller +mo sis +mitro vica +mispron ounced +mele ch +mechag odzilla +me es +mar soc +mali m +lon avla +lin sanity +le usm +lam elo +lake garda +kir at +kal ka +jo bi +indianoil cl +in brotherhood +hippo griff +hee jin +ham worthy +green spring +gor oth +gil ham +ge bran +gast rectomy +fe stu +es miles +easy recipes +du mble +dj shadow +dennispra ger +d ils +crimin ologist +cork coco +cop i +compa ñ +come stible +chou teau +chi uni +chagu anas +cas ali +bur sle +bruce willis +book mail +black lab +bint aro +benefit uk +ben dera +av or +at us +angu sti +akhi lesh +adam ski +activ ites +ðŁĴĹðŁĴĹ ðŁĴĹðŁĴĹðŁĴĹ +ðŁĮ± # +ê± ¸ +ç§ ĭ +æ´ ¾ +âļ½ï¸ı ðŁijį +yo shis +ww d +wis d +wage red +vishnu vardhan +vis cardi +ve k +universit ät +underthe sea +tin plate +thewhisky agogo +thec ct +the writ +terry pratchett +tech ne +team no +team fortress +tch ouk +st birthday +squ am +slim ited +sli din +skillet music +shopp ers +self defence +saxmun dham +sa ipa +s van +ru sky +rosel awn +rene sas +reen actor +re classify +radnor shire +pupp i +po dunk +plu med +plat ja +pic tus +perpe rennial +par sec +pan chi +p ined +ou saf +ori el +om al +oldd ominion +now available +no st +nga io +neu chatel +nepen thes +nca af +national french +mo dc +mid nighter +micha elek +michaelek lund +mc nee +macron utrients +ly kes +looooooo ool +lim my +li bin +land shut +lab ine +la ar +kron wall +katat onia +kad ha +jonath and +j annis +it stony +inner visions +immort elle +imer ini +ig ur +homedecor ideas +him ley +hert shour +hawk sley +hard point +har perperennial +han auer +gyp sophila +gradu ationday +gow land +girl gang +fy f +franç oise +foli ate +flogging molly +fil adel +enjoy ably +empor io +echel on +e zi +dun cle +dr michael +dp ms +daysof blackcosplay +dau gav +darren shan +d se +cri sing +cri bbins +contamin ates +cn traveller +clipper nation +cinde rel +ch ye +castell ana +carly aquilino +c vs +breath itt +brass eri +boston comiccon +bor delon +blon din +better makeroom +benedic to +bathy metry +bal tz +bac carin +au gen +aster y +asic samerica +as thana +alekh ine +acci esfc +( [ +ðŁļ ® +ðŁijį ðŁijĮ +ðŁ¥ £ +çĶ ° +âĢ¢ ~ +z s +z ette +young boy +yan ko +women supportingwomen +wat l +w bir +virgini awoolf +veer u +ultr amodern +tu ur +trun cate +tru ef +tri pe +tof te +te cn +tape worms +tac tix +ta pley +sut til +strong side +stratfor don +srisri u +spec kle +sp art +sim cox +shannon bream +shal it +sc lay +sam r +ryan leslie +royal visit +rond out +rol lovers +roc codi +reis z +re dragon +rath down +r thk +qu ello +pre science +pen ha +pen do +patt ani +ou thouses +on nnn +oftheyear hma +ob t +nigel barker +new church +nau s +nan tuc +nadiaz anelli +n spra +n mn +mustang nation +multi drug +monster monday +mon ch +mo yam +migrant sday +micro blogging +mel robbins +medi vac +mecklen burgh +me dak +max pain +lun i +lubav itch +lock ridge +liver disease +leed suni +l arios +kil twalk +ken naugh +ke mlu +katsu shika +kat anning +juxta posing +je ay +jaz baa +jal gaon +jac co +ilike samizayn +ide c +hic hi +happ s +h km +h ci +gyna ecological +gow ans +gottacat chemall +good work +gene wilder +g tourney +fu qing +fresh prince +farn don +famili arization +fairground snola +e chev +dul verton +deer hurst +dam ie +cro z +cou par +correspon dences +compe tion +coc ci +chu uya +chin skiy +chenonce au +cbr ne +car na +c cat +bu suttil +box fish +bon jovi +bin u +berk off +be ere +be com +bbc surrey +bai ze +b johnson +astro physicists +aden auer +acqui esce +acqu it +acomic con +ðŁļ ķ +ðĿĹ ľ +ä¿ ¡ +è te +yaari yan +va ghela +use fully +up asana +trudeau mustgo +transport ation +tor ock +ton kinese +ti os +thusi ast +theatre ldn +teab agging +taym or +take a +super powered +sun birds +stru b +stefano gabbana +stand on +sp liff +sp ersons +sp ds +sl int +sidd al +sher pas +roa sted +rid ler +ri gid +rheu matologist +quis ition +proro deo +prophe sies +pro static +prithvival labh +preste igne +perfect gift +peniel shin +paw ling +pan get +osle isure +osiris rex +neon ate +national trust +mrs browns +mgsv tpp +merci an +may nor +mar cher +maquo keta +man by +mall inson +lo kk +lis gar +la sley +la ho +kwq cnews +kir ya +ke vo +k lee +ju se +jebe diah +jagu are +ja ib +ing man +igh tham +iam dr +i osis +hu day +ha yy +gwand shows +gun safety +gov mattbevin +gondo liers +gilded balloon +gen u +gar di +g fd +for hillary +flo ssy +flo bots +feel ies +elvis duran +elrey theatre +edi ger +dri ss +dram as +deton ates +de broy +dad os +d sey +coy gig +chro mis +charge back +chapelhill shooting +canadian opera +cal vet +ca hier +buro happold +bu ton +bru ery +brawl stars +bra ine +border patrol +birmingham pride +beth nal +bait fish +asqu ared +ar ue +aon ach +aldubin italyday +al dia +aksh ar +ablu tions +ðŁĵĮ # +ðŁıĢ ðŁĴĻ +ᣠĴ +zacchae us +worldbook night +wil fried +west king +wat ere +wasi lewski +vent ers +trac on +tony pandy +thene ed +sy re +swe ene +sw offord +super majority +super mac +sun it +suje eth +style by +stu voice +state oforigin +ske wing +sj sj +shey enne +sen ge +school master +sch itt +saf mradio +ro secity +ric kowens +rei vers +r saf +puru lia +prep star +pon tos +photo album +pharo ahe +on at +omni potence +office dog +o townofficial +o cul +native breeds +nam askar +nach richten +my fwc +mor phia +margare ta +ma aaaa +lon gy +lin ka +lang worthy +kra hn +kale v +instac ar +inser m +hyper ventilation +hopen ot +hale wood +hah ne +gre aser +grand tour +grac eville +gon zo +go via +go bel +fun atwork +free mind +forbe stech +fold sofhonor +fiji airways +end ry +emo sh +elly se +elizabeth warren +ec is +dush ku +drinkal sace +down with +dit or +dialo gic +dg and +devop ssummit +democratic debate +dele tions +del sol +death rock +dat acom +dal zell +cute off +compu ter +ci vita +chum phon +chemain us +californi adre +bro ten +bou ch +bosch global +bor r +bon ta +bhatt arai +bhar vard +becau sey +be scot +bal ks +bal ama +bad chicks +ay ato +at rade +as kim +arro yo +agil bert +adam cole +acou sa +ac ist +a eda +ðŁĺ° ðŁĺ°ðŁĺ° +ðŁijĮðŁijĮ ðŁijĮðŁijĮ +ðŁ¤¦ ðŁı½âĢįâĻĢï¸ı +íĶĦëł Įë +ا Ûģ +zeyne pab +zan ski +zack y +worlds best +wool ard +women schampion +wom bourne +williamj hague +will l +wad ham +vari ances +va q +v angel +ukcoach calipari +uconn football +u oy +tten nis +tre sco +to pre +thisi sn +ther ob +terran ce +tam ie +swa im +sun foil +still withher +st aser +spoke smen +spencer ville +south wards +sheldon ian +seac at +saltwater fish +room ed +roman owski +rob art +receip t +re shuffling +rd grade +razor blade +ran j +pyrr hic +play dough +pide mia +par tey +par menter +p batour +ou in +oooo ps +on ald +om ish +music in +mur ci +mo gami +mispr inted +misanthro py +mine ers +me iling +mark land +m smith +liv cathedral +lex an +le ane +la fia +ko daly +kit chee +kir sch +kids first +jan cic +ite mized +ini go +img academy +icosa hedron +human itas +ho soda +hi was +he wes +greatcomet bway +german yin +genuin eness +gentle mans +gaz al +gau zy +fun tastic +fm j +filmin dustry +fbun ational +fa enza +est at +enjo ined +eh den +earth sci +e ffa +drew gulak +dow ni +do ti +div y +der oy +demon ic +cy cad +crowned heads +con text +con nally +clu te +christi any +cf g +catandu anes +canecor so +bt me +brussels attacks +briti an +book art +block house +bett in +balu sters +bacchan alia +bab ich +b awards +ash eville +ap ix +ago da +a eu +- âłĢ +ðŁĻĤ ðŁĻĤðŁĻĤ +âĿĦï¸ı # +âĻ¡ " +âķ² âķ² +ઠķ +ÅĤ o +yemen crisis +whe ath +water conservation +wa iner +vir das +vic ent +viad uc +vermel ho +vent agli +vas ko +vamp y +union station +twin ed +turn again +tunder ground +tu lio +tu azon +tomy future +the brand +the better +th ark +taun us +tal aq +tak acs +t sle +syracuse chiefs +swith amazon +stu gotz +stom orrow +sri mad +sm kc +simpl yaj +shan em +se os +se alions +sanc lemente +s meal +roby ns +rib oud +repri sals +recalcit rant +re states +quar ies +q eh +promo cion +plo it +play listed +pine grove +pin edale +party in +paraly zes +open call +op tionally +offici ates +num erically +now lin +nov itiate +new designers +neer u +ne mec +myco plasma +mister giuntoli +mil stead +marcel kittel +mag ician +m fat +look man +lat ah +lang i +la ville +la ury +ktv tamil +kra vet +kor ona +kis d +ki ai +jim breuer +jax on +indi g +hight stown +hei der +hard wa +ham ida +ha jj +ha insworth +greatday tobe +ge bre +gabbie show +friend sforlife +flori bunda +ferment ers +euro sport +es el +epic urus +engel mann +elo cution +dor tch +dj jazzy +da gher +d á +d illian +cuti eee +cup w +crp findia +consul ta +com res +collective evol +ci dg +chur ns +chumb awamba +char lier +chap eron +cf adden +ce arch +bru mmell +box ed +book talk +bla upun +be ja +bar acoa +back sliding +aver ts +audit or +at gm +apri ze +an day +amelior ate +alo se +addis combe +ab bate +: _ +% ( +ðŁļ¶ ðŁļ¶ +ðŁĺ³ ðŁĻĪ +ðŁĵļ ðŁĵĸ +ðŁĴĻ âļ¾ï¸ı +íķ © +ãĥ©ãĥĸ ãĥ©ãĤ¤ãĥĸ +Å¡ a +wroc ÅĤaw +workplace safety +wex gaa +wakat obi +unc charlotte +u ws +twic ulate +truthor dare +til lett +ti ma +thisi swa +ther y +thanks forthe +tal ley +syco phantic +subsidi sing +stopthe bans +standard bank +sri xon +spring awakening +spin out +sp outed +son ship +si ma +shra van +shel p +seok ang +sak ha +s enda +ro ki +relinqui shing +recre o +re vers +re interprets +ram se +ra so +preservation ists +pp ms +pac bio +p nj +oh pa +ob it +ny rb +nottoo young +nonleagu epaper +nichol ls +new wave +nancy sinatra +n phs +n lo +mum life +mou vement +motor park +mo dak +mo ama +ming kki +mik kel +mick o +mi mar +mi drash +meli ora +mcmick en +match boxes +mase field +mac donald +ly all +leot ards +lasvegas shooting +la hav +kon st +keeptexas red +juli usc +jointeam alpha +jar mo +j ila +inner city +in ala +ig uns +hy les +heartsof oak +hear ses +haram ain +hamilton island +guatem alans +gil boa +gh d +gb hockey +fri berg +flori daf +fe tid +extrapol ation +estac ado +erne sto +eddi evanhalen +dragonball fighterz +dragon stone +div aio +diame trically +df bharvard +decrimin alizing +dais ley +d ites +ch ko +cebu ana +cas elli +carri acou +cardo za +ca pet +bur qas +bru it +bridle way +br yl +bir tley +be toys +bais den +ax xx +astru c +as ophia +as lo +artist center +ani moto +af shin +adam stown +abra sions +ðŁĻı ðŁĺĩ +ðŁĺ¢ðŁĺ¢ ðŁĺ¢ðŁĺ¢ +ðŁıĬ ðŁı¼ +ðŁİģ ðŁİīðŁİĪ +íĶĦëłĮë ĵľ +ë°° ì§Ħìĺģ +ê³ł ë§Ī +à¹Ģ à¸Ńภ+young buck +vol com +ver itas +vam shi +ty mon +twi xt +twe ener +toxopla sma +tom riley +toby keith +tho sp +the de +te tt +tales of +suicide squad +spend in +slo at +sling sby +sky one +sjo gren +school innigeria +rv f +ru ffs +rtx rt +rfd tv +reden bacher +re past +rat as +rai b +quer cia +pu ku +principe ssa +presiden to +po tage +po stive +over acting +or uk +of lu +ode brecht +naruh ina +myprotein uk +mckis sick +matriarch al +mar ito +mand vi +madrug ada +ling ling +kin ko +kai bab +kaf a +k wave +jon favs +je g +janh vi +inau t +im pulse +ilove u +ih in +hoo oo +hoo ge +honey z +heck mond +he ita +hallucin atory +gu zzo +green horn +girard perregaux +gee zers +gadge try +fri son +foot way +erotic romance +ent onces +en trance +en shrine +el mina +ec centr +du mer +domestic workers +dok lam +dj danny +dis quieting +dis continuation +din is +digitalleader sa +diap ering +deleg iti +dav adi +d me +cow dray +copp ens +con tru +clair vaux +cf ps +cav endish +cate chi +car ina +car agh +buster love +boy shoops +bhu pend +aw restaurants +auto body +atlan te +articul ates +arri etty +an tero +amur thy +alde burgh +aic ha +adel hills +academ ical +ab negation +(^ ^) +ĸ ðĿĻ +âĨ Ĺ +Ë Ĩ +yoak land +yl ancs +yakov lev +x terra +wheeling nailers +wendy davis +vintage traffic +vari ate +transdayof visibility +to pley +the gabbieshow +tan in +swimswam news +sven son +substanti ation +stat man +st birthday +sportsday hs +so frock +sixnation srugby +sin motion +sd sc +saraha ines +ro vs +ring land +recircul ating +ray com +rav ings +rail hawks +q ri +program mer +ox chambo +ot ss +on thisdayinhistory +o dum +mo graph +mind set +mic om +mar thar +manne quin +man ak +mac gillivray +lg p +lam est +knu dson +klai peda +kenny florian +ka ap +just because +jose altuve +ivan chuk +irish film +ico splay +i lea +hu ffing +horton ville +he izer +haudenosau nee +hanover ian +h atim +guer rera +gotham ist +goe tia +glyndwr uni +glaci ated +gl hs +git ana +gec dsb +gal mudug +fizz er +fin c +febu ary +fatt ened +explore your +es ns +ep ting +dot pict +dom usic +dis locations +dd newslive +danis ordo +dai fuku +daener y +curb appeal +clear lake +cbc toronto +cat ley +case break +carned dau +carla hall +bye e +build series +book sof +bollywood flashback +bloor dale +az umi +aw newyork +at ag +as oli +as ok +artist oftheyearhma +ard ening +anton ym +anthropom orphi +anne of +anatom y +anast asi +an new +alice keeler +alber talli +alai ka +al anc +accru al +ðŁijģ ï¸ı +ðŁİģðŁİģ ðŁİģ +ë IJ +é»Ħ åŃIJéŁ +اÙĦ Ùģ +yuvraj singh +yi ff +x mend +wood workers +wing ert +wanti rna +wal ch +vol pi +vit abio +virtuo sic +vir na +vici ous +val ise +un availability +tx sen +tx f +tir zah +ting u +time code +ti ant +the bige +tardi grades +tafo ya +super conductor +su ta +stra db +stradb ally +stand byme +song do +sm tg +skylar grey +sa pperton +ry er +rush koff +rural women +recon figuring +re life +raun ds +rajon rondo +port ents +pon tard +pok é +poetr yeileen +pic one +photo weather +patron ise +patric kk +pastor a +pandor a +pand avas +ot sego +omni vores +ok san +ofthe future +numb ingly +nor by +ni mh +new snl +neph jc +my poetryeileen +mus ice +min as +micro waved +micro chip +mev simi +mccar roll +mc nerney +mash ed +mark w +liv v +l annon +ks ss +kno win +kigur umi +kho jaly +k fans +jun gler +ji han +ja quet +ja ay +isti gh +ip sf +inau ghton +hmrc govuk +heme path +have eru +harrystyle slive +hang outfest +ha ddy +gru newald +gou de +gli ese +glam rock +gla dy +gif tw +gar ms +forti ssimo +for girls +fon zi +follow spree +family matters +extram ile +er adi +entro pic +emo cione +ef its +ee es +durgap ur +dom ar +dillian whyte +di bles +derri ere +de young +conquistad ors +con cour +chip sets +chim o +chi vo +chester races +chang jo +can ticle +bur gle +braban tia +bc tv +battic aloa +bas ks +bar ve +bal raj +asympto tic +asbury park +amar illa +ald ar +agr itech +abet ted +> , +ðŁĴĢ # +ðŁĩŃ ðŁĩ· +ëĤ ´ +ãĤ¯ ãĥª +ÛĮ Ùħ +y wc +x ms +wire line +wee h +ventagli diparole +ve ining +v show +tv patrol +tt ol +tri pof +ting les +tho tel +te sch +splat tering +song pop +son dre +som one +slowh and +si man +school room +sch ek +sarah h +sain ty +sad day +rubi u +rosal inda +rig our +rat tigan +radio graphic +public enemy +post le +posse ssor +poi esis +pn k +photo synth +parap sychology +par due +pad mé +op ie +ond kar +o swin +noo h +nj tv +newcastle upon +nag ore +n gala +mu sco +mode ste +mid oriya +mi uccia +media art +matri archy +mary kay +malak off +makeit rain +light up +li ath +lehigh ton +ku lik +kozlovsky d +kiz ito +king swinford +kenner ly +kees maat +kar g +k roo +k love +just transition +jo zy +j fe +innovator s +ing arden +inf ur +hor vitz +holo type +hof meister +gregg sofficial +gre if +go zags +gl antz +gg es +gb ta +g wer +g mit +forrest griffin +farra go +eviscer ated +europe ana +dura bles +du bber +drug gie +dig nam +dan ville +d hir +coa sted +co ill +ckin non +caul drons +cambu ur +ca zz +bou dhan +bio gen +benid or +believe that +ba ai +aw ash +ask twitter +ar mers +anonym ous +ana erob +al the +al pro +al adee +afranc is +action jackson +! '. +ħ ¸ +ðŁIJį ðŁIJį +îĦ ħ +ìĦ¸ ìłķ +çĻ ¾ +à¹ĥ à¸Ĭ +à¸ķ à¸Ńà¸Ļ +à¸Ĥ à¸Ńà¸ĩ +رÙħض اÙĨ +y is +würt temberg +wokeup likethis +wind horst +wim wear +wc n +wc m +wan ag +vow les +viva an +visit korea +vi ed +vel oz +vain queur +uv ella +under sheriff +tuesday shoesday +tuber ous +traf studios +too fan +those who +the ar +teh rani +te pes +summer love +states b +sta ste +spal ted +sno bbish +shar kie +shannon poe +sh ick +se pik +sc m +say id +san sad +sa hu +s dorff +royal airforce +roth ley +remb lant +re share +plu shy +play on +pg itte +pent z +pe aker +paid leave +p bo +over hangs +oo dy +olemiss rebels +ock ham +observ ator +nel da +necess itated +n nuh +morning walk +mol as +min de +mill ner +manufactur inguk +manta she +malcol mn +maj ili +ma ute +look n +le ora +label le +kü bler +ku ha +kis sthe +ki ara +joannak rupa +j scott +ital a +irish cancer +inter library +indy statefair +in berlin +ichin ose +hyper dimension +hs j +houston flood +harrell sllc +ha kim +gup te +grenadi ers +greatesth its +game pro +fu rio +fly spicejet +fire work +fini stere +ffd pgitte +fed soc +fe bs +fanc ourt +enki du +dream wave +don wood +devdutt myth +defer ring +de jah +dand elion +d hinchcliffe +con gra +clemson tigers +ch retien +ch itten +ch assi +ceru tti +ce b +canvas back +call eri +cad wellpark +cab infe +bro aches +bon nin +bk d +bin chy +az ules +ays garth +ay ee +athen s +as mbs +aru ban +ari ffin +ar uk +am rap +all inclusive +all hiphop +al die +air tran +afro basket +abor ts +ab ounded +@ - +ðŁĺĤ ðŁĺĴ +ðŁĴĽ âĿ¤ +ðŁij ² +ðŁ¦ Ĵ +ðŁ¤ ¾ +ðĿĻ ĸðĿĻ +ë´ Ħ +é»ĦåŃIJéŁ ¬ +âĿ¤ï¸ı ðŁĴĭ +âĢ³ , +ÛĮ ر +Í Ł +zi arat +yoshi o +xia oping +x wa +wether ill +welcome tomy +w wn +voc ational +ve endam +v league +usp sa +un questioned +un counted +ucla health +ty le +tor rilla +thé âtre +tas o +taran is +tampabay rowdies +tal u +stro mat +start les +st ca +spend thrift +snu gs +sm m +slobo dan +ser fdom +se fo +scifi fri +science spo +sanger institute +roccodi spirito +rc p +random ization +plac ita +pioneer woman +pau wels +pate ley +palo alton +onetown oneteam +ohi om +obi m +o gni +nw sc +night in +new sm +naz ri +mrdan walker +mothersday gift +monstr ance +mi stery +mashre q +ma or +lene han +kur land +ku bik +ki kim +k gl +joey ryan +jason mohammad +jamaic a +interior style +indiegam elover +im modest +ik aw +i hat +hyper cube +hump ty +holme sdale +hod kinson +hk jc +hemi spheric +guigno l +granad areports +gran te +glon ass +g djb +fra port +forte an +foresth ill +fli pping +flam b +feed thefuture +experiment ally +estim ators +er manno +eo sio +e bury +divine mercy +distinc tiveness +diaspor ic +delhic apitals +dc wx +cv h +csu sm +cray ton +coon an +colom bo +chris ber +chak an +chai fetz +c fw +buy ck +bri j +bre mbo +bou zouki +be sty +barry wyman +as me +art glass +arrog antly +apologi a +any as +antony cotton +amon te +amar is +am ining +al aphilippe +after hour +ad resse +accor ding +ðŁĴĥðŁı»ðŁĴĥðŁı» ðŁĴĥðŁı» +ðŁIJIJðŁIJIJ ðŁIJIJ +⾨ ðŁijij +Ã¥ le +zu manity +you make +yearswith out +ye lection +yas ar +wine pairing +wi ffle +weekly chris +wee ee +vo tol +var vara +ud ta +touri smb +thi steam +that matters +temer loh +sw mrs +strac zynski +so ory +so hrab +sky lit +sho red +san tis +rip saw +retro s +rem nick +re breather +re attach +re appointed +q tip +po cruises +pen at +patho logic +par ichay +pang ong +neiln mukesh +nawab shah +mye verything +mor na +mo hun +men inga +mc gibbon +mar ins +mann ington +mall o +ly rik +lor dan +litter ally +lamar que +ko haku +kew science +ket ches +k pol +juli ef +jap onic +it ches +ir ano +inver keithing +inclusion ary +imti azali +il ite +high flyer +happy times +hant scricket +hani ya +han kinson +h ma +go lovin +get n +füh rer +fou che +fl on +exoner ate +entic ement +engv snz +englishri viera +emp tive +e fre +dz mm +do don +di pan +der mo +de hn +dale e +dak o +cun o +ctvnews vi +con vocations +ck x +chriso cearch +charn wood +cathao irleach +capit ulate +cac p +cab over +c elife +bun ya +brue gger +book ers +bon nier +bang olufsen +autisma warenessmonth +arc gis +and relton +an fer +amo han +amdiabete sassn +af oo +ab m +a stig +a jai +ðŁĺ¢ âĿ¤ +ë³´ìĿ´ íĶĦëłĮëĵľ +âĿ¤ï¸ı ðŁĴĻâĿ¤ï¸ı +ह म +zoom ies +zo is +zeynepab dullah +your welcome +you them +year with +yard sale +whathapp ened +wedd ingo +wc ps +w shed +ving t +vic ary +utu san +us arm +ugar te +tom ate +to ten +tin da +ti angu +thomas mtv +thelu mineers +su ya +steven rinella +speci fiers +ske ete +sk ru +sign posted +se bts +sch ut +sar son +santi am +sam ahan +safe hands +ry ou +rock xx +ro get +revul sion +resi d +re tooling +re ple +radi ophonic +r fafighting +pol lies +pf aff +patriot sfight +party bus +pal afox +pack age +ott ley +o zan +nor k +nice comms +nh se +nevere ver +multi point +mu kesh +movie posters +molen beek +mil ward +mid nap +mick jenkins +men tosa +medemb lik +mber gen +mb ag +mas ar +manu fc +mane ki +makh ni +maced on +love gwendoline +li sowski +lafour cade +l cb +kol lection +key biscayne +kar son +k dr +jo bo +j ope +insu fficiently +inol vid +ing on +ing lives +inde mann +hy pn +huuuu ge +high tide +hel mick +hari ini +har alson +ha wes +gy ar +gaslamp quarter +fer ulic +farah zeynepabdullah +fa aa +ex pul +es al +equatorial guinea +eo tters +empire ofthe +elie be +e wo +don ts +deme sne +de paola +cu o +convers as +convers ant +claiborne farm +chlorop last +chi odo +chand an +can ape +bur net +brutal ities +bio chem +bin ford +biennalear te +bibi ana +bernas coni +azz ura +au ber +ar pan +anuradha pura +anton ini +an kur +alcal de +al gha +aflo or +'' '' +ðŁĺı ðŁĺıðŁĺıðŁĺı +Ù¾ ت +zy g +yy ceats +wol v +wi di +whale bone +weare mg +water week +villa vicencio +ver in +va jazz +tre the +thisweek in +thefutureis female +the chef +ter rel +te waar +t pr +sujeeth sign +su lay +spon d +south wales +so ami +sh ko +se idl +sc rowder +sag s +sad u +s science +row les +rockingham uk +returno f +qu avers +pro mesa +police women +po kok +pet terson +pa as +or jan +only badchicks +off broadway +nwc fl +nsw labor +noble woman +no control +nic asio +my sskin +music to +mo hin +mil ord +michal is +mckit trick +mari ann +manit ob +m sau +love yall +letgirls learn +lakel ouise +kuro o +kni ghting +kel laway +kashi wa +judi dench +jon benet +jessiej decker +janet lynne +in dro +ilo vela +il ar +icon forhire +i blis +ho eness +go ans +fun palaces +fre port +finn skata +fil ion +fare ast +ev y +elasti girl +ei fs +digital uk +di gue +di bb +dar gan +czar ina +cy rene +cre ggan +cr da +cor ra +con nelly +chan y +ce elo +caper ca +boysand poets +border lines +bol lettieri +blue plaque +bar isan +b wc +b life +avi ano +av ang +auden shaw +amoe bamusic +american eagle +acro phobia +ðŁĺĤ ðŁijĮðŁı» +ðŁĺ± ðŁĺŃ +ðŁĺ« ðŁĺĤ +ðŁĵ Ķ +ðŁ§ľ âĢįâĻĢï¸ı +å¿ Ĺ +âľĪï¸ı # +áĬ ł +н а +with confidence +wh ooooo +vu ren +vik ki +vie ille +uru guay +univers als +uk garage +uigh urs +trans mute +tr ys +ti dd +theme parks +thaic ave +tent acled +t suru +style inspo +stani forth +si yah +shar ris +shah zada +sell ick +selfcare sunday +schwe iger +scar olyn +sas co +sak ta +sa jal +rosar ia +ric kett +r ale +q asem +power apps +pir ri +peter crouch +peculi arities +pap ineau +over stay +out strips +orange man +opto electronics +ny er +now isthe +no win +neu star +mur rays +mon serrat +moisturi zes +mg u +mcgi v +mat ai +mam un +main ieri +lu ft +lolo jones +land marked +lam anna +juli antina +jallian wala +isee you +in ki +in icio +id om +hurst pierpoint +heat ing +had lee +grey water +greatest leagueintheworld +gorgon io +good beer +go girl +globalnew sto +ga si +fording bridge +fla vel +f ally +estro gens +ell ingham +elder of +ef ya +ed lund +dod dridge +di ger +de mentors +dah lan +cuad ros +cu x +cro fter +chikar apro +charl bury +cape coral +canadian army +border s +blow ing +big gies +beng tsson +bel in +bank rate +av ary +ast ros +aspin wall +ash by +as wan +arrow root +animation dev +amazing places +allegi ances +air box +aff ton +acu shnet +aaa al +a jaz +ðŁĴĸ . +ðĿĺ ¢ +ðĿĺ¢ ðĿĺ +ìĥĿ ìĿ¼ +âľĮ @ +âĺºï¸ı ðŁĴĸ +à¸Ħภ£ +Ê Ķ +ô n +y nares +wmn hist +weid mann +we sh +wb g +voi ding +vis ca +vast o +val ette +uni que +unex citing +ty ana +tv fest +tutankham en +tt chelps +trip to +to see +tell eria +team nl +team gaspari +taly bont +swo ons +sugar cubes +sub bu +sto ya +slo fficial +sla ver +shire brook +sham okin +schu ett +san oma +sail cloth +royalairforce uk +ro ids +r gu +proto typical +pro bly +pork belly +pon orogo +plesio saur +plau sibility +pin chas +perro tta +peori achiefs +pen ic +pe kar +pathan amth +pas ch +parks rec +pa quita +newcastleupon tyne +neal brennan +n ze +minic omic +metro stars +me ghana +me chas +marketing land +marit imo +mari ad +magic fm +ma hel +loon athe +lo ei +life inthe +lemon de +le mmer +kro ft +juliet landau +jinder mahal +inti mated +infe sting +ind ilens +in compatibility +ii ight +i irc +hydrogen ation +hooke don +ho wol +hen in +har ip +great schools +grapp led +go bind +geb bia +garri gan +gareth cliff +fou n +fashion model +fab ula +exal ts +ex upery +em oments +dur rell +dou bloons +dl rcc +dis figu +de tt +cur dled +cre swick +conceptu alization +colour ings +claus sen +citys c +cis ne +ciné ma +cb cradio +c ón +c sio +bus boysandpoets +bru hits +bluelive smtr +bloem endaal +bi dd +bh lib +bed sit +be twood +be tong +bbcred button +as man +ar twiculate +ap ca +anth on +am ran +am orosa +am molite +alo to +alfa jor +al tro +ahu t +agu adilla +ade pitan +aag pbl +ðŁĻĮðŁı¾ ðŁĻĮðŁı¾ +ðŁĺĬ ðŁĺİ +ðŁĺĤ ðŁĻĦ +ðŁIJ¶ ðŁIJ¾ +ðŁįĵðŁįĵ ðŁįĵ +ï ĥ +ç ½ +â ¸ +zar doz +yougo girl +ymc as +wr acked +women tech +win ther +win big +wifis funeral +wel low +we we +visit noosa +un tiring +un shackled +un romantic +un pronounceable +tur keye +tric losan +tothe people +to wy +to di +thriller writers +the muppet +the money +ten ser +tan an +tack ling +stron gest +stri e +ss bn +snow bell +shu jaa +sho tel +sett in +save daredevil +sarawak ian +sac rosan +sacrosan ct +rspb minsmere +rotor craft +revol ve +re ordered +pu st +propert yoftheweek +por twine +ponder ous +plateau ed +pel z +pan mure +or ville +or dov +or am +oo sten +of ans +nor gay +niek ro +ni blett +nati v +msu ext +monsie ur +mj ölnir +miss gem +miner vois +mer pol +men ier +may enne +mati vity +mascot te +mar uf +mar ani +mad h +mad decent +m night +living ston +law lz +laut aro +kill menow +kad en +jfk airport +jc mo +jagann adh +j gp +itstony bennett +ine qu +ig aming +hyper cholester +hofstra u +gosn ells +gold hawk +gior giom +gil crease +flo m +fla pped +fau chon +es m +epo c +endeav our +emmy lou +echi um +dontb omb +do ering +den ly +demi gods +daw i +darab ont +cycle ways +colori sts +colonial pride +cold cut +co ilovers +cli ving +chow chilla +cho pped +chin ar +chicagom usical +chat elain +car mouche +buffe ted +black letter +bir dr +bi utiful +bev ington +belly ache +bel lu +beh ring +bal lew +b bog +azz opardi +asymp tote +arnold schwarzenegger +apr incipal +am ole +a ans +ðŁĻı ðŁij¼ +ðŁĴŁðŁĴŁ ðŁĴŁ +ðŁĴĻ ðŁ§¡ +íĥĢ ìĺ¤ +ãĥ¼ãĥ Ń +zo oniverse +yeee ah +winter games +white horn +wand wmusic +wal sch +vote thomasmtv +vill amaria +usd learns +ubiquit in +triglycer ide +ton n +tit in +threl fall +the mars +ta ft +str ously +ste cher +sor oti +snarky puppy +sha han +schre ier +sc ai +say i +saw amura +ru der +ref ill +redbullair race +re matches +pro sec +pray ingfor +pp ur +pointe dly +pay kel +pau sch +pat ni +parti zan +parrot let +paren thetical +pac elli +out fall +odu sports +obstruc ts +mun du +milleni um +mel gar +mar dyke +mame town +mam aw +ly cans +love sick +loose strife +lin tels +le uc +lawson official +lau toka +l ura +kour nikova +kindafunny vids +k ns +ju dic +john williams +inter solar +in kosi +ic orp +hb ddar +har ton +ha fe +green burgh +gom ti +gi gha +gar gan +ga shed +ga aru +g mn +fu trell +foxnew ssunday +floridian creat +fire news +far rant +fal ci +expression less +esof tball +endodon tist +ele gies +elderof ziyon +eg ba +denis coderre +decad al +dar ting +cro pre +cro pp +ci vit +ci az +cho wn +charlize africa +cdn tv +candycrush saga +blood stains +big machine +bert kreischer +below deck +bellin zona +bbc somerset +baltimore uprising +ay lestone +at official +alle m +ale ena +ai z +ah sa +abyss inia +ab hisar +>> @ +ðŁķ¶ ï¸ı +åĿ Ĥ +ൠĩ +zy gous +x rays +wal don +voc ation +valen cian +uwh uskies +uni sport +un hurried +umb c +tu dou +triplic ate +tr é +tony kanal +tmobi learena +the tony +tel com +stra dio +stom y +song writer +shoo ky +sheil agun +sheilagun nreid +seraf ino +scho enen +sam bhar +rspb birders +rival smike +red act +reck ell +ready for +re distributing +re dedicate +quotes forlife +pit ying +pf sense +paul weller +pa zo +p mu +ou vindo +oren burg +nws bayarea +nonchal ance +nolla ig +nihon bashi +nicol ay +newfound landers +neph rite +mushroom ing +miko yan +metho dis +mer cnews +mende leev +mcl ars +mccri mmon +mc sherry +mag loire +ly ster +low boy +lo pam +lam pley +kul p +know it +kier on +kar ly +kab lam +jol liffe +jay baer +iri st +iri ga +instinc tual +inclin ations +in line +high note +herni as +he yarnold +hartz ler +happen shere +fun day +exp ounds +et nowlive +ers baseball +en et +emili ana +em ps +el ice +e hi +dun smore +dra enei +dor mont +distribu ted +der v +cta flash +craig smith +constitu tionalism +con soled +choose cruz +cer ta +cel led +carrie ann +but ner +bro mel +ble ddyn +bienven u +bbc music +bar kov +back kk +ba injal +ay ak +av endre +auto logous +at ago +arru pe +anze kopitar +any am +anantkumar h +akshar dham +afternoon tea +afric om +advance auto +ad era +achrist mascarol +ac fc +aar thi +aa official ++ +, +ðŁĺĺ ðŁĴķ +ðŁĺ» ðŁĴķ +ðŁİīðŁİĤ ðŁİĪ +æĻ ¯ +å¤ ı +à¸ĩ à¸Ĺ +world rugby +west brom +wag gy +umph rey +u chicagop +tweetapicture that +trade shows +tr ze +tepp ei +tcr no +takar azuka +tac c +tab e +t set +super res +style tip +stocking stuffer +stanley park +sn v +silk worms +shish ir +shim omura +seattle children +sarrac enia +sandal wood +sal low +sa pped +rol linson +rick wood +rest ling +recipe blog +r tty +quarter deck +pres sphoto +pil at +pe asantry +pav in +parasit ism +or ison +o zer +ny cd +nmm greenwich +nhsm illion +newlook fashion +nan or +nam anana +myco toxins +mira sol +mika il +mccoll ough +maj ka +ma shiro +lrb ht +low ville +low ns +lou lou +lo pen +lim be +lack adais +la sto +la pride +kram ers +koss off +kingdomcomed eliverance +kindergar ten +ke ef +kaye adams +ka ve +juli amichaels +joyce didonato +jon batiste +jo ginder +jo ear +ira bad +ily ich +ich wein +iback thenats +han lin +h sing +gran ulation +gra velle +gou dy +gor ging +get fit +frank warren +footb ath +fl outing +fisher folk +fall winter +extracurricul ars +eugen emir +eugenemir man +em mit +ek c +eic hel +dis associate +dhol era +dark soul +craig millar +conte h +col locations +cod champs +christmas special +cheer sport +cc tv +call anish +ca ppo +bob seger +bin do +bin ch +bill simmons +bhak ti +belo v +be mani +bar uchel +avail ble +at se +at anas +asjad nazir +ase f +aren adublin +akro polis +abo ah +*:ãĥ»ãĤļ ⾧ +ðŁĻĬ ðŁĺį +ðŁĻĤ # +ðŁij¼ ðŁĻı +ðŁıİ ðŁĴ¨ +ðŁĩ± : +æĹ¥ ãģ® +âĢĶ âĢ¦ +à¸Ī ร +öz ge +ï a +zamor ano +zai us +za habi +yy xx +yay ayay +x ol +wool ridge +who cares +way ner +wat ters +val lum +u hmmm +traffic chief +trafficchief ng +thru xton +tholi prema +thene c +the walking +the tournament +te on +tales ofthe +take astand +takan ori +stateofthe union +sor table +sonal chauhan +ship wrights +see a +sar ria +sampo erna +sahar are +sah len +rock i +resi duals +re hire +rashmikamand anna +ra del +queri ed +putin atwar +propag ator +pre heating +pow ner +postand courier +po cs +plan cha +plain tive +pedro za +pay am +op as +on our +ol g +oh boy +official wexgaa +ny j +noto kay +night clubbing +nd p +nathan s +my at +mumb ail +monarchi st +megay acht +medical research +man cs +mali ha +mal et +lou donville +lost teddy +legion fx +lanz amiento +kpop ers +kmf dm +kindergar tener +kidero evans +khan e +kc wx +juliet telewis +jazz times +iwon a +inter actively +infiltr ators +image awards +ice age +hor ch +hc ps +gri mas +ge birge +gang star +friday mood +forevery thing +for amini +foot plate +ferru ccio +extor ted +espor te +esk ay +er rata +ejec ta +east bank +dry skin +doom patrol +dombrov skis +det ling +desi rous +dell tech +cy presses +cros scountry +cre stron +compar te +cli q +ci amis +chri mbo +chibi usa +camerat rap +cairn sgbr +c plp +brown rigg +brad t +bou w +bluem tn +bloss er +blau er +bit w +be itar +bc stx +base bal +bar ani +baini marama +bai ji +ar ow +anac apa +agribusines stalk +afric aine +................ ...... +ðŁĺį ðŁ¤© +ðŁĶµ ðŁĺĪ +ðŁİī ðŁĴĹ +ðŁ¤ µ +åѦ éĻ¢ +⼠© +à¹Ģภĭ +zz one +zapp one +z gan +yeah hhhhh +whole sale +w us +v sts +usaid transforms +ure thral +upanish ad +unite c +u dit +tim bo +templ ating +swachhat ahi +sw osu +suwa idi +suppos ition +st ma +speed boats +sol ti +sk ö +scottish canals +sab z +ro sin +riv ka +rask ass +rafc gy +ra sto +qu ing +pw res +pro di +press ler +poplar ville +pol lens +ph re +pal y +pakar my +oz ona +over seas +orda z +or ite +onero vers +ny we +no ye +nis ka +new world +mr j +mo rella +mo berg +michelinguide uk +mi ria +mez cal +marilyn ne +macer ata +lth trust +loo kahead +loc q +line berger +leek wang +leed ong +lan des +la opera +kri er +kel ani +ke di +kac chako +jen naw +jay park +jas wim +j soc +indian summer +ima gic +huber tus +hou tte +hot press +gom pa +ghat ge +frick collection +for sman +fol o +fisher cats +fifa e +etsy teamunity +eric an +eras mo +elm hurst +dollar general +dic om +dele vi +dehuman ize +degan wy +dau g +d cb +cu dd +cru cero +cord mn +cor vette +convers aciones +conceptu alart +ci gale +ci bia +christmas dinner +chill o +cedric alexander +carra ig +c ung +bush meat +bon ello +bo eser +bli k +biomole cular +bio graphic +bha agam +best buy +ber thon +beck a +be om +bbc norfolk +bar th +audi bly +att ar +assi stindo +ash ell +aro che +ar kad +aper to +ami e +am ese +al sip +adhe rent +ab ondthatcantbebroken +ðŁijį ðŁĺĺ +èij ī +è ĵ +à« ĭ +ı z +ym el +world triathlon +wind blade +volu si +vle uten +vintagetraffic usa +vaness o +vac tress +ur la +un il +terrori zer +tad worth +t sw +syl vanesso +subju gated +stel zer +st q +ssss sssss +sf alls +seren aryder +see ff +seas ickness +sav arese +sar keesian +sane wwe +ru i +ros é +rob m +ric hert +rc u +raj ar +rail ton +pun akha +pri sco +precipit ated +positi vel +pos ity +plat ting +pitt sbvb +pin ho +pi ph +people matter +ore e +ordov ician +or sett +on twitch +ol pc +nu cor +nightinthe woods +never leave +natu rism +nas w +nano bots +more ra +mis za +mign ons +met ar +mello tron +mc pd +marcjacob sintl +lostin translation +lon line +lo isa +learning disability +lak l +lak is +kw anten +ku bra +kim zolciak +khu shal +kairi sanewwe +ka dir +joeyryan online +jeet bo +je k +jake pittsbvb +ja imel +itsme marcog +iron bowl +ir da +ine miliaromagna +hug your +hosp icec +ho ttt +ho ar +hiro kazu +help ful +hawaiian air +han ae +haider alabadi +ha eun +gen cia +game zone +fre de +first place +fast way +expo sed +evely ne +end el +emerson drive +el met +ecu baseball +dou ll +clin micro +cl w +chuck liddell +ceil ing +ca zy +bur rowes +bry k +brizen orton +brac ket +blunt ness +bishop stown +betti epage +ben nell +bel ters +bap ak +az leg +aw onder +avi ka +ascend ance +artist sofinstagram +annel ies +angel ayee +andru w +all sup +air fryer +ahmed patel +afford ances +> ) +ðŁij¼ ðŁı» +ðŁIJ ¿ +ðŁįī ðŁįī +ï¹ ¡ +ઠ¸ +شرÛĮ Ùģ +zindagikime hak +yz ma +what culture +well travelled +we missyou +villa verde +ut x +ultr amar +u efi +tv illa +truste eship +threl keld +theamerican sfx +techno terrorist +te guh +tart ar +tamsen fadal +stother t +stor mon +shat tered +scrip ture +scar ff +s sci +s bla +ro ton +rithvik dhanjani +ren es +refra ins +refr act +raj skub +pvr cinemas +pur ges +prote omic +plim soll +pix ley +pc bb +pace makers +p suv +p bafinals +nin kovich +nicolas cage +nic eness +new quay +narrow boats +nadez hda +n ter +mutil ating +mon aya +mobile punch +mizu hara +michael shanks +mic ks +mc que +mati z +mas nor +mar ar +maggiel awson +luxury watch +lug nut +ling le +liacou ras +le mahieu +law firms +lam oriello +ku cera +klar man +kill ough +kemp en +kas erne +kar aj +k crew +jul liard +johnny g +joh nette +jeffrey combs +jake shears +j walsh +inve ste +insane cham +insanecham pwres +ifly mia +ic ture +hp celebration +hol lick +hodder books +hi ren +her ping +hard body +ha go +gour ami +frag mentary +fr s +fe deli +err day +eng ates +el sworth +el ster +ec lips +e mig +dream less +designer sguild +del pozo +daf na +cress kill +cosmo sdb +comman deering +coal port +check point +ch acousa +caw dor +buck den +bour bons +bb mp +bad enoch +asjadnazir sexylist +amphi polis +a acs +???????? ???? +ðŁĴļ ðŁĸ¤ +ðŁİĦ ðŁİī +à¯į _ +z ol +youthen voy +yan kees +wo tan +wish bones +wind starcruises +who woreit +wah ed +unic anberra +udd hav +tx stars +tri ppi +treva than +ti ful +thread ripper +tho p +ta irport +su hani +spring fiel +spl ant +spaceship two +son paper +so apo +sm koneru +shak en +sh ood +sh dop +sasi kumar +sar lat +sar gam +refer rer +re tractor +re appointment +pnca rena +pin z +pau die +pa wesome +our in +oc v +o lowo +nar va +nam ics +my hero +monopoli ze +mom afilm +modi fiable +mid mo +mi el +mercado libre +lon nie +lia ising +li ker +les ch +le tyour +lam brecht +lagav ulin +lac ofd +kne ast +kh eng +ke j +ke aley +jun to +jay sus +jadav pur +j wilson +j jones +j hum +ir pur +im j +ikar ia +hu ahin +hon ora +hispan ia +hex is +hersh berger +her schel +hel zberg +han ok +hal berd +ha ggin +h jal +green island +gr ttweets +gautam rode +fnd tn +fluore sc +ferro vial +fc splayoffs +expand ers +ethical hour +emocione scan +el sie +eid as +e ying +e ah +dumfries and +duck ula +diop side +dh alls +den yse +deme trio +dead space +de twe +de ignan +cynic ally +cram ton +chil ds +chead le +charter schools +cath ays +ca icedo +bé ar +bur chill +bot in +bor odin +big data +bel isle +be joy +back kkk +bab bit +ba jar +aw la +atlanta fx +at tridge +as ali +arya stark +art us +arroman ches +an ies +al sadd +ais for +ai dy +actu ation +ab ula +ðŁĺį âĺº +ðŁĺĤ ðŁĴĺ +ðŁİĻ : +çij ľ +ãħİ ãħİãħİ +zo han +zah raa +z wan +yn elson +yank sonyes +winter warmer +wheeler dealers +warand peace +war same +viv ab +tri phala +trend z +traci iguns +toyo tas +time keepers +the odd +su liman +su hr +su are +stepan akert +steal z +son owal +so yer +sko glund +sie ges +shutt ler +sep timus +se eta +scar ra +scab iosa +saxi frage +sand ers +s magic +rit ory +resent ments +re location +puff balls +pu yo +peregr ine +pd ash +on cidium +official pvfc +official pes +n spc +my tton +monop lane +mono coque +marmo zets +manu bennett +mang la +lymp ne +lund university +lr ath +ll ly +leyon hjelm +leon hart +leg alism +lees ung +le pton +lac onic +kw es +kapp as +jj rockxx +implic ates +ii faut +ia sen +hot yoga +happybirthday liam +ham lisch +h na +gyn lais +griffin shockey +gravit ating +grace jones +g thr +fel ici +fac ce +escape ment +en ballet +elic iting +el mb +e tomi +du rell +destined tobeyour +dam our +dal u +dab ke +da stan +cre sson +cn sa +chap manu +cen bank +cand ling +calum best +brent butt +bo ger +bie hl +better off +b kl +auxili aries +arbitr ators +ap cu +andri od +alu ae +. ðŁĴ¥ +à° ħ +Ø§Ø ® +zan in +yo sh +with y +witchof gric +war dy +wain scott +vir ility +vice president +v mu +une z +un delivered +to vo +tin dal +then ra +tele vise +tal ex +ta wad +sturgill simpson +stu arts +stal king +speed line +spani en +sol in +snow blower +sel k +se alife +saf ta +sa karya +ry land +root sport +robre iner +ric t +refresh ers +queen sugar +q wop +pre conception +per dida +pe plo +pe ay +par amotor +one us +om ron +of arms +nicole tti +new stuff +nat oma +my u +my co +mu iz +more life +moom oo +mol alla +mi leg +mercy ful +mckin ley +matthar vey +mark logic +mamared carpet +mal on +lor as +lom adia +leg ler +lec tro +lar go +lar don +kwe se +ku le +kom bo +k oper +journo request +jad is +isak ov +iri dology +ing ness +individu alist +i she +hor sell +hol ub +hij ra +her ford +hac i +gay romance +fre tted +fe zz +farmto school +ext enders +eric j +ecou tez +eck mann +ear lof +de values +daw na +dal ys +cro oz +coul ro +costab ile +comb ichrist +chin ni +cher one +che fe +charli eco +carav ana +cano eists +can ak +c trip +bye ee +bun b +bridge of +boom boom +bok rugby +bit ties +big things +betibachaobeti padhao +bas ri +bapti stry +bag lan +at ley +astra galus +arcen eaux +ali ona +aa ÅŁk +, '' +ðŁĺģ ðŁĺľ +ðŁĮ¹ ðŁĴĢðŁĮ¹ +ðŁĮ ¯ +ìľ ¼ +âĿ¤ï¸ı ðŁ¤Ĺ +âĿ ģ +yeon jun +without limits +wire cutter +wing s +wear pink +wal brook +wa sters +val v +un supportive +un drinkable +trevor jackson +tough er +to paz +theyoung turks +ther am +the police +the grove +the fieldhouse +the bush +tewaar aton +te ter +tanger ine +su jan +sp lines +sla voj +sigma hq +shu ld +shu ben +shen mue +shan am +schmel zer +sc x +san ai +salam on +sak on +rott we +rh us +ree zy +red line +red cliff +randal stown +rag am +progressive metal +pon da +pnin ator +ple uro +playoff final +pitch man +photo catalytic +people whom +patre se +pat erson +p flp +on elmstreet +omari hardwick +ny ard +north wales +new shindi +neuro psychiatric +nece sit +native americans +nat ter +mor as +missi ves +mis su +mirac leon +mic odelrosario +mi pi +mcil wain +mar oto +mar imar +lly welyn +leit rim +kn wa +kir ara +kevin m +kat as +kaf u +juliere ichwein +jour dain +jingo ism +jean grey +iy ar +hol loween +heil man +har in +gri bbin +gra j +getat me +gerald orivera +geo x +fu sca +fant agio +es now +em yr +egyp tologist +ed na +ec cw +dige stive +di zi +det ama +dark lines +dan cere +cos se +cor fe +comicbook men +coly tic +claudia jordan +cityof perth +cine a +cham paran +cep k +catalan referendum +cad enas +burk hard +bur ga +brook lynn +bron ski +bri anne +brett kissel +bon omo +beg ich +ayo ze +avo gad +au sout +aren aof +aontro im +anec i +amand ase +alle le +al vi +ai v +a ves +!! ⾨ +ðŁĺľ ðŁİī +ðŁĺį ðŁĺĩ +ðŁĴ¯ ðŁıĪ +ä¹ ĭ +z ation +voye uri +vj se +ver da +union saustralia +ty hafan +tul lio +things done +thi er +te men +te at +tab ler +ta veta +sy en +swi wx +suss kind +steve wal +specu lum +soft works +so thern +shu bha +shant ung +serkan cayoglu +sel lar +sclo thes +sc so +sc ira +savedbythe bell +santo shi +s garden +ro ev +ro ba +retweeet please +ren ton +prim rosehill +president duterte +polit ec +patri mony +p ctto +over holt +over drawn +ov adia +onthe water +nli reland +nfldraft scout +new smy +nail sworth +my kha +msam ber +mostbeautiful faces +monof ilament +mo lex +mississipp iriver +meister singer +maur its +mat ua +mari anor +mal vin +mal to +live authentic +laurel hurst +lan kesh +ki bby +ken ly +ke bab +janel parrish +is qua +install ation +igno u +i ipa +hol zman +heron dale +gun ships +gran do +gill ylancs +fur ler +for humboldt +flo rette +fire y +figure drawing +far g +ex horts +ent ano +eccle shall +driver les +drive on +dove cote +denzel washington +d ä +comm vault +coll ingham +cer atop +camoufla ges +bristol news +bristol city +bocc accio +blythe wood +bk twtr +bay lon +bass musician +basil an +ban za +baham ians +as sche +as l +aren ts +arbit ral +am iss +ach ra +ac press +ðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺįðŁĺį ðŁĺįðŁĺį +ðŁĩ« ðŁĩ® +ë°ķ ìĭł +âĿ¤ ðŁĴľ +y utu +wr p +wny wxguy +wiley updates +west wing +we urope +vojvod ina +vit tori +vir gie +vie tto +umphrey smcgee +um bus +tx milesplit +trespas sed +titan sof +time sunion +thest style +thener d +tam popo +ta ppers +sv ball +super user +sun dari +su sand +strath spey +stan chion +sta e +sl trib +silver side +sig gy +shop fronts +sar it +samsun gs +sakhar ov +sa ie +reg n +rec ent +re re +r head +pron ghorns +pin cers +p te +onlinec asino +nzv ban +ny liberty +motivational speaker +mol der +mil air +maw dd +marin aro +mar yo +manto vani +manj hi +mandra gora +man dos +mal loch +lynd sey +loffici el +lin tott +li gation +kick as +ki bet +katiemc grath +karti ka +joo hyuk +jinxx bvb +iz aguirre +irregular ity +inter breed +intel pro +int ents +imuk biz +hu uuuu +guil lemb +gradu ands +glan bia +gat ot +football friday +fle cha +find me +fel ly +fe ste +fe aver +euro furence +egg cup +effu sive +eagle hawk +dye sebel +diagon alley +deer field +de iv +dan ka +craig ella +country duty +cou lon +corning museum +comedynight swith +collec tives +cn sphoto +clari dges +ci bona +cher if +che se +character ise +cav ender +bra chio +br acks +booksto read +boij mans +ble m +bi zer +battle on +bat lle +baker va +bai doa +bad blood +back boris +atap ult +asi mone +al uska +ðŁĺįðŁĺįðŁĺį . +ðŁijij âĿ¤ +° ï¸ı +younus algohar +wom anist +wer neth +vs bos +vil ification +vi all +v roman +un sullied +un making +tr inians +tol booth +tele casting +taylormade tour +tau k +szyman ski +sz ka +steph ani +sse arena +sou tho +silent disco +sho taro +shah ram +se ath +scout master +sar dina +sam ani +rick world +ric cardi +ri gaud +renee graziano +rant zen +ps le +pro saic +poly unsaturated +pa jaro +p scs +online poker +ohiop yle +of cl +ober ge +o ei +nev schulman +ne dra +memor ably +mb am +lo gr +lnp fail +leu ch +leaveyour mark +le di +kin hai +ken burns +kemp es +keeping you +k roy +john leguizamo +james mar +itsall goo +ik hil +hyper dunk +husky nation +hu lud +hr in +hol landia +hick forco +heures dumans +hallucino gens +gu detama +gian marco +fren chal +fox searchlight +fivb men +farm pics +expun ge +este ban +end family +easter house +diaspor as +depress o +demar re +dave mcclure +dar l +daf tar +cro che +cour tly +cor ie +col ine +co tai +co flood +cn alive +chen yi +chasu ble +cast in +cafe cito +burger week +bri gs +bra ds +bot as +bor gman +bhil ai +bestnigh tever +beauty tip +bal ada +ato jr +anno ying +am la +al cides +ais in +ag lass +@ .@ +* ] +' ". +ëŀ © +å¤ ľ +ãĢĭ ãĢĭ +د ÙĪ +zen nor +x pe +wych avon +wom ad +wol fies +what car +well spent +w daz +vag aries +transpo sition +track man +to lima +the orie +tand f +sun sout +sun gei +stradi vari +spic ers +sn ur +smithson iann +sarah jaswim +sal ine +ryan hunter +rid ger +rico che +riche mont +re spire +puri foy +pe kalongan +pe et +paro di +pan try +p th +oc pd +obam aday +o wa +nu ked +noman sland +nekop ara +neg ating +mu lago +morecambe bay +montt remblant +mm film +mello dy +mehwish hayat +mega shot +may sles +mat tera +manag ing +ma dala +litho graphic +li gia +kon kon +kitti wakes +ki yomi +khel oindia +ker ins +kate beirness +kap ag +kam asutra +jay len +int ell +hour glass +hex ham +haz im +haupt bahnhof +hal ston +hal d +grand ches +grandches stour +ger akan +gam brel +fracti ous +form alize +forgi one +fl andres +fa kku +ex claiming +enig ma +emili eder +emilieder avin +dy ar +du ri +dough ty +dob son +desecr ating +dar ri +d hir +crou te +cr nas +coyo acan +comeon you +col ón +cling mans +che sser +charity shop +cer via +campe sinos +bu ckey +bri j +bo yet +bertrand piccard +bas sie +back breaking +bab ineaux +azu lejos +as cp +andy grammer +amba reesh +amat suri +alu va +a jose +ðŁĺģ ðŁĺī +ðŁĺ¯ ðŁĺ¯ +ðŁĴĵ ðŁĴĵðŁĴĵðŁĴĵ +ðŁİĬ ðŁİīðŁİĪ +ðŁį ² +ìĿ´ ëĮĢíľĺ +ãĢ ħ +ت Ùģ +Ã¥ r +z waan +z our +youn gre +x fce +womenin ag +whatdoyou mean +wat sapp +walker booksuk +w cdma +v annan +usa ir +un subtle +trout beck +trend micro +tor turers +tol li +thereal gokwan +the mary +the jon +thaw kins +ten so +taleg gio +ta hoe +supportlocal music +strato cumulus +sthe world +stephen mangan +speci alizations +spe el +spani er +sonny digital +snow berry +smar tt +sloane stephens +serv ative +sch is +sanantoni ofc +rum ley +rot tie +rna seq +rin at +riff raff +regal es +reen acts +re dress +rajat tokas +r alli +quiztimemorning swithamazon +quis ling +pro social +pres sey +pra p +poom sae +physi atry +pat more +p skov +or rr +ontheroad with +oneteam one +omy ces +o hhhhhhh +no bill +ni sei +n mp +my night +mp cs +mon essen +mo one +mnight shyamalan +minister io +mile le +mifi dii +meander ings +mccar ley +mb oldo +man atsu +m pire +lenny gaspari +kiran shaw +kir ko +john berman +j la +incentivi zed +in viol +in attentive +in accurately +iii ight +iam ond +hockey hub +ha ast +gray wolf +google drive +gar dein +fire andrescue +far hi +estrange ment +enew sletter +ell acruz +e fr +dul han +don giovanni +do ÄŁ +djim on +dd hi +creative mornings +corred or +congre ve +com hairle +clau diab +clar os +cinemainmy genes +cholec yst +chite ttura +chil mark +chi ya +cassa day +can lon +cal mar +bri se +bra ked +bob white +black country +atl ay +athar va +architec tsuk +anth o +anime con +alph anu +alp eng +allu de +alac rity +agri goi +afloo d +ade bate +ad tran +ðŁıĦ âĢįâĻĢï¸ı +ðŁĮ Ĵ +ว ม +ø y +yogur ts +wedding present +weare stv +wau ters +walkthe talk +ve ith +under powered +un spent +um braco +ty rian +tur aco +tsu taya +troy bakerva +tricia helfer +tre esur +toi world +thur s +team nuh +tau be +tasmani ans +tamago yaki +take that +t dg +swa ppable +stur gis +stewar tuk +stare down +ss badal +spr ats +sla shers +shee ted +shanty town +sha ji +set swana +saf ai +rep l +reg ge +rebu ts +re tz +radio deejay +r sl +property forsale +promom yshop +profit eroles +pon cho +plu shes +play for +pat mcgrath +pas su +orchestr ates +nyarla thotep +nut field +nunatsi avut +north up +norfolk va +nat asa +n maa +myviking story +must ache +munster gaa +mess inger +megan n +med ill +marsh lands +marau der +mar yan +mam ed +mag ura +ly die +lu ddington +lo iseau +li fo +lec lub +lang try +l mics +l mb +kyri akos +key club +kay i +karmen uvella +karao ke +kali uchis +kal ima +juvenile justice +josap hat +jan am +j mattmiller +j ali +ian us +hiroh ito +high est +hel s +he hir +hb hai +ham me +ha thletics +gu ts +gog com +glen campbell +ger ak +gany an +g ela +g day +frand sen +flat white +ez pass +esp ina +eri der +en claves +em mm +dran ath +dig ang +di leo +defe rence +dead lands +de marcu +culi acan +cru gby +cro sland +cor bitt +coco on +cardi stry +car ona +bunb trillog +bu ttle +borne misza +bor ley +bogal usa +ben savage +bay ly +baby steps +b inet +az ir +ational guard +ati dora +and are +alic eroberts +af sp +a strup +ðŁĽ ¸ +ðŁĴģ ðŁĴģðŁĴģ +ðŁĩµðŁĩ ¯ +ze ee +you meatsix +x ers +x amas +x adee +wing sscotland +vol lrath +us wa +ur ts +uneas iness +under lie +tucson newsnow +trues dell +troo dos +ti ro +thisis roc +the original +the incredibles +te do +taylor swift +staur ant +st illing +so j +skee ball +sher ie +shak shouka +sen io +scher rer +scand alized +rou garou +ring way +rever ies +re mer +pp ap +philom ath +pc sreeram +ou asport +ortho pe +oral b +only way +odd job +nz post +nic le +natural news +nat ation +muh sin +mis behaves +mcfe tridge +magu ey +lo ga +let r +kh qa +ker on +kel tie +kash kari +jud ger +jazz ercise +janh vikapoor +jak art +it ment +is chool +ilike it +homestead ers +hol mer +hi zon +head man +ha zes +girl gamer +ge gen +garth tander +forma zione +finn forchange +fing ers +fifae worldcup +fal o +fa sta +fa e +ey n +extern als +eve leth +er satz +e ip +du gs +dow land +dis dain +di kan +del phos +def uniak +deck ers +dal go +cumber batch +cp su +coro coro +con ade +citad els +ci otti +chitten ango +chil o +check mates +chai kin +castle ton +caram ac +car week +cann ing +c ly +bur rowed +bro sseau +bor re +ble akest +bianch ini +bi yombo +bi sta +be mba +bar gnani +au chen +as ays +arci mboldo +ar agua +alky l +affl ict +ðŁĴŀ ðŁĺĺ +ðŁİ¤ ðŁİ¼ +ðŁĩ ¼ +ðŁ§ ģ +ëŀ ľëĵľ +zim zalab +xibal ba +wt bs +wi fey +wheel man +wat in +visite den +vam ily +usic festival +usc lay +u um +u ille +twit terers +twarri or +tur pentine +tu sker +tron aut +tric entennial +trade marking +to iv +tam ira +tabas sum +symphonic metal +suz lon +staf fies +spor tives +spa ins +silver io +shu hei +shi ii +sha hin +sen ough +self magazine +sandwich ing +sal ma +rory stewartuk +richmond park +ri skin +ri ber +regurgit ating +red tour +recali br +ran del +pur va +prophet sof +play towin +perturb ation +ped lar +pas chall +p rus +oxygen os +obre gon +nu v +ni avar +nav a +nad die +na hal +mtu kudzi +meng ele +me dyo +mat tocks +martham ac +malavi ka +lu la +lot ton +loks atta +lev it +leon ov +le master +lan cey +l vo +kristian sen +kris shdop +knowyour status +khar if +ket el +keegan mkey +karapar aaÅŁk +juilli ard +jellic oe +incen dio +ik lant +ig liano +if sa +idol bday +hu bo +hen nen +harsh man +happy vday +had dix +gum pert +gu ff +good news +gill ray +general ised +g sis +g dt +fu lof +freck leton +esp y +ef alls +dy r +donald sonville +dino sau +di bi +descar gar +deltag amma +deconge stant +de commission +dau r +daily show +da ig +d guy +color adan +code breakers +chu gg +che u +champag nel +bu tyou +brdg stonearena +bo iiii +blue wave +bi shara +be kka +bal tus +bach ir +ba rend +athle ti +ar cho +ar bore +apeoples journey +ann cleeves +amor is +alone together +afflic tions +adam sss +a hal +] ? +) < +ðŁį ĸ +ðĿIJ Ī +z ando +yam un +xab ial +wy verns +winter lude +wil pon +wastel ands +war di +vote dem +voor de +vi ke +var don +use recycle +twi bi +tsun a +trum puk +tre kon +to an +the adventure +tf ny +stech nology +stay humble +smilo don +smart sheet +small things +silver stonec +shahi dafridi +se ik +saqq ara +sam mut +sal umi +sal ita +ru ok +ro tan +ro mm +ro jer +re fitting +ravi zacharias +rain bird +ra oka +quest for +puc cio +prairi eville +pra chidesai +por bandar +pitt ance +pir in +paramaham sa +pacha uri +os burn +on al +ob vio +ny ab +nsc w +north wards +multit ool +mtv star +mrsbrowns boys +mollu sc +moeed nj +mixed martialarts +men slacrosse +mari ash +mahar astra +lu gogo +london jazz +lindsay jones +laim beer +la vand +kou delka +kant é +julie tta +iw sc +itunes movies +it like +ho by +har ar +h wood +gra j +go yt +gi aa +fu seli +frederick md +fol f +father john +farewell tour +fai the +ex por +emo ting +eliza veta +effec tive +e studios +du hon +dont end +dog leg +direction less +davi da +dav ar +dang it +cuttinge dge +contac to +confl ate +citizen four +chester hour +car lier +can asto +bör se +bul taco +blu ffer +bio processing +bas adi +azu read +apor tugal +apo ker +ann av +ankylo sing +amur ray +amand af +alan walker +acl tv +ðŁĺį âĿ¤ï¸ıðŁĺĺ +ðŁIJįðŁIJį ðŁIJį +âĺ ĩ +z ameer +youtube music +x cellence +wel ham +weare indigenous +vapori zing +usarm yeurope +un mixed +tyrr hen +tyr ant +tiv ities +the vfl +tamer hosny +takay uki +sympo siums +sym metrically +subor bital +stun twoman +spag nola +sofe urope +shal u +sh mirtz +se ales +schil dren +sag enda +ron dell +repre nd +redu cible +re deployment +r dan +prize winning +plau sibly +pit tock +phil taylor +pan ji +ome gap +ni renberg +ng d +never see +nebl aruz +ne ele +navar ino +nag pur +mis sin +mini atur +min um +mikewill madeit +mic as +mccre adie +mc whirter +mc bath +ma ju +m newsradio +long don +lit vin +let sen +leaving cert +lauren koslow +kyun ki +kw un +klu ane +kimber lee +kess inger +jun aluska +ju baland +ju ab +joh ancru +jeron teng +jac e +i aap +hedgehog society +happy family +go thard +ger wyn +fur pals +fu ori +freak nik +fe ssler +f me +exemp ting +en fer +eigen mann +e golf +dont shoot +disson ant +dec agon +dataf low +dai shi +cru it +cross y +cr pc +civic si +che to +che apo +cess ary +cbc thenational +calaf ate +c sg +béar naise +breaking barriers +braun er +boost vibes +biennalear chitettura +beer wah +bassmusician mag +bar io +aza adi +authent ics +ar mc +aparna dixit +ant acid +an dex +aman ullah +am erson +all that +( / +ðŁĻĦ # +ðŁĮ¿ # +ðŁĮ§ ï¸ı +î Ķ +âĿ¤ï¸ı ðŁĵļ +âĿ¤ ðŁİ¶ +Ì « +zz oli +yp silon +worldwide web +weare warriors +ward ha +w fl +viol on +vie len +vad uz +un unu +tx plants +ti bbits +thunder wolves +temple more +tem bro +sw sh +surviv able +super conductors +star times +st robes +square enix +spark led +sole a +skunk works +sibling love +shrove tuesday +shol m +shikar pur +sam trans +sa astr +ryo hei +rose and +rit annia +remark able +recuer da +re examine +ps bl +prophe sying +pro pul +prasar bharati +ph vegas +pet abytes +pandor ica +on os +ol ak +nyct subway +nigel farage +ni verville +nag ler +mouni roy +momot aro +mn ch +ml stadium +misper ceptions +mer u +mer mhart +masch era +martin heinrich +marque se +lo ys +lauri ers +lab ella +la ven +krist offer +kle mm +kingshead thtr +kar go +kam akhya +jon pardi +jo bbers +jj redick +jar os +jane austen +ireland amtv +im my +icol lection +holm fir +haz ari +had dow +hackney council +h tu +go q +gary numan +fill an +ff rench +fen ris +fast jet +ero adtrip +elast in +eddi ep +ed cs +dustin poirier +dun gan +domin go +dis bands +den of +country lifemag +con tem +cl b +church radio +chrisky le +chri sar +chai wala +car boot +bron k +bra gi +bol lin +bob vand +bds fail +back yar +aur icula +askingfor afriend +appro vingly +anaesthe tics +amrav ati +amic ably +alief isd +academy brix +ab ai +) ãĢį +ì ¢ +á´ ĺ +Ø· بÙĬ +ž i +y alo +wil co +werthe imer +w ace +vm vc +visit lisboa +un selfishness +un rolling +un followers +umu ahia +uj world +u daya +tz is +twitter art +touris mus +ther ium +theother artfair +the taste +the mira +the call +tak am +ta deo +stre ch +stopp able +stop suicide +sto janovic +steve krohn +sle ip +sit g +shirob ako +selfie expert +scol ombia +sce aux +say yes +sarah mclachlan +rey ne +relap ses +rel in +redemp torist +re hearing +qu ev +puk aki +pharmaco genomics +perse baya +pathanamth itta +pal ko +opp i +one oh +olivi eri +office depot +odell brewing +non agon +nh out +national internday +nagas wara +muk wege +missgem collins +michel lea +mi as +mcdou gald +mar king +man isa +lost lightfest +lightitup blue +le ut +kingh enry +king kong +kil coyne +kat elynn +kali das +ka el +k ku +jen carfagno +jan nik +j izo +itsu ku +i ant +hou elle +hideand seek +helenclar knz +hah haa +great work +glau cus +ghul am +gaf as +free base +flavor flav +finger printed +expen sive +etsy trending +epau lette +eat right +dun man +doo dad +disin fo +devol ves +dele terious +danny glover +dalgle ish +crow le +cross mag +co zzi +co intelpro +clums ily +city tshwane +cinema scope +ci vari +castle man +cas al +build that +brasen ose +bblo gger +ba the +b pn +aw on +av int +ast us +ari ley +aran manai +ar sd +aontroim gaa +ammon ites +ag usan +adi po +> __ +.. ðŁĻı +à¥ĭ à¤Ĥ +z rx +yor am +wwe thebigshow +wi ds +well sboro +war museum +waldorfa storia +w enge +vi borg +ut sa +u lisse +tz ou +ty ers +ton tine +telegram dotcom +taren as +tam ala +sw oon +super glue +strike out +son ika +sme a +sho guns +sel kirk +se fi +scen ted +sauti sol +s beauty +ro sleh +rim sky +right most +ri emer +r jl +qu bits +pre sta +poem trail +ple b +peter boro +pep to +pas ca +paren teau +over reaching +outh unger +officeof ssbadal +ne wa +n pf +music business +moy ra +mi yyah +melchi zedek +mcne illy +mayor alty +masnor ioles +magn animity +lo dy +liz gillies +library journal +lecoq sportif +kt nuk +ksr nv +king lear +jes c +j degrom +isth mian +inter zone +ine music +indo siar +ik könen +i annucci +hur rr +hondaci vic +har riton +gri mbergen +global artisth +geo ca +gen iu +gar ston +funk houser +ever wood +er vine +disinfect ants +discover your +de famed +de cai +dancing withthestars +cur tiz +cruis eline +crow son +coo ver +clu bc +claws out +cent a +cas sim +caperca illie +bun an +br v +boyn ton +bou twell +bo ke +bi ent +be creative +barit ones +ash lar +as ahd +arnol fini +ard ently +architek ten +ar rangers +aman resorts +al tri +air worthy +air conditioned +against cancer +ag bu +ðŁĺį âĺĢï¸ı +ðŁĶ´âļª ðŁĶµ +ðŁĴª âĿ¤ +ðŁĮ´ ðŁĮŀ +ãĥĭãĤ ³ +âľĬ âľĬ +Ì ¯ +ér rez +ye tto +y hoo +y abe +war isan +vive ko +verul amium +vanguard news +us ra +ur ing +ur bis +un answerable +turbo diesel +ton ie +tol ani +thisis america +thirty one +the handmaidstale +test accio +te mis +super cat +squid billies +spr in +spe an +sla pper +si vo +si stani +shu le +sheab utter +shaw mut +sev res +senyor atidora +se tau +ro so +ride shimano +reic helt +re interpret +rbg canada +quin tos +q y +pw ba +poise tt +pi beta +per rie +optome tric +om onia +officially dale +ny gma +nobill nobreak +niavar dalos +nen es +nce atalk +n ado +megh wal +mc murphy +mati ja +mar ving +mal liance +ma des +looking glass +leather y +laser hairremoval +l rl +koffeewith karan +ko fori +kofori dua +kliz an +kil ims +khur d +ka hu +jr w +johnc mcginley +john h +jing yi +info com +idiosyncra sies +i thin +i olan +houelle becq +hay dar +haik us +ger aci +gar rin +fu dan +far fan +evolu cion +dumfriesand galloway +dj j +diar maid +demois elles +de yo +coo lock +chi zik +chann ell +cates by +carson kressley +c ink +bur dening +bra cho +bn pa +bludge oning +beauty salon +bb celeb +bar cal +ayre some +as dru +aro ss +ap ley +any o +al styne +al amb +agi ft +af bf +ac x +ðŁİīðŁİī ðŁİīðŁİīðŁİīðŁİī +áĥ ¦ +اÙĦÙĩ ÙĦاÙĦ +zo tti +ystrad gynlais +wi gle +ver mes +valle dupar +umb rage +trans fe +thur l +tg is +tex ture +ten ere +telli gence +tech f +takeme home +ta fc +sto gie +sn aring +silver classic +ser hat +seapor t +satyamev jayate +sat inwood +sat ins +ry un +ru beng +romantic izing +ro lin +procu re +prah lad +plari del +pen ley +pe koe +param speak +ozz ie +omnam ah +omen ico +ol az +o ip +nm fcofficial +ner ul +naj ia +multi modality +mul larkey +mul her +margin alisation +march al +manti ses +m life +leban king +le ola +la fuente +kor le +ki est +ken ickie +k mbappe +jik lian +jait dp +j churchradio +inge agle +in dor +ik pe +hop ital +hal ak +hac ket +go hawaii +gen ève +gar row +gar finkel +gam ely +filadel fia +fight ins +evil legas +er st +ec inemas +dncle aks +disembar ked +demp o +danneel harris +cv f +cre matory +cinder block +catal ina +car lyon +brown y +body power +bo quer +bli fe +barber o +av ram +ausv wi +at mo +apo loo +andy serkis +ali ano +ak dong +ag twitter +affe e +ðŁĴĸ ðŁĴŀ +ðŁıĢ . +ðŁħ± ï¸ı +ðŁ¤ º +ë°ķë³´ ê² +ãģ® åĽ½ +zhen ya +zach arie +xo yo +wra bel +wait wait +viro qua +vijay an +vi veros +vet tai +ve ut +vance joy +upro arious +un tag +ud ell +u thman +ts ars +toys betoys +ti meeee +the york +tay port +sub divisions +spor tw +spart annation +southern miss +sla ys +sh ange +se ke +scriptw riters +sch nu +sar l +sac ross +sac anime +sa stre +sa inted +s sps +rol an +re attached +ralu ca +rai ffe +public sch +premon itions +point and +phantom opera +pa ha +out world +out d +official c +offic efurniture +o tu +nong shim +new game +nerd camp +na iman +mull ica +mlb fc +mis in +maurici om +matta poisett +master man +maris ela +let toysbetoys +lei dy +lanc elin +lam ott +lac asse +kri zz +kir on +jozy altidore +jig nesh +j allow +int al +in chief +iffic ult +hotho thot +har ma +gry bau +globalartisth ma +glassof bubbly +f ong +eyel ash +exol selcaday +endow ments +en raging +ed mv +e ab +desig no +der yn +der rius +daniel la +dam ara +cou pe +constric ted +confin ing +conce tta +coma stia +com ac +coach bill +co ale +clau diag +cityo fo +chandra pur +ch anti +cedar town +cards againsthumanity +cap uch +can ol +bts world +bru eg +brand t +bitcoin talk +bcu hb +bat week +ay aw +astu rian +ast and +ar nim +appe al +ano des +am ang +alex change +akh mat +ak sar +ag utter +ac cone +) '. +ðŁĺĤ ðŁijĮðŁı¼ +ðŁĩ§ðŁĩ ¸ +îIJĴîIJĴ îIJĴîIJĴ +èª ŀ +âĬ Ļ +zz le +zen aida +yo hannes +xadee journalist +volei bol +vit rio +vitrio lic +veronic aroth +up fest +unmatch able +trumpuk visit +triple talaq +tim ryan +temp at +te sfaye +t weak +sw aby +suz ana +supply co +str acker +ste marie +stab by +sta verton +sq ld +sou la +snow board +shinj iro +shim kus +ser an +sb ks +sant amari +ryanhunter reay +run tastic +run skg +rip nelsonmandela +regi us +raoka vitha +rajag opal +radio surgery +quili brium +quaide azam +pride parade +pon nu +pollok shields +plat y +pa ju +p and +over bridge +or ona +ophy l +online auction +nus ingapore +nc pc +murphys law +mur uga +mu shishi +mu hur +mtv uk +moving quickly +mo ora +mis d +mcget tigan +mam ar +ly st +lu carelli +lex ile +lass wade +lamar ca +la vaux +kir alı +kings things +kim berlin +kar ura +ka aa +jin x +ji my +jadap smith +it gets +irre deemable +inno gy +illa warra +hydroxy cut +ho cr +hef ce +hand al +haeun dae +grass field +gha ffar +fu leight +flat ly +fi as +eu il +es adler +employee benefits +elin coln +editor schoice +drawe veryday +dor ji +donat elife +dl f +depaul u +de tik +dand unn +cur rie +cor sini +con tort +compet ently +cod man +cla hrc +cic carelli +chaper oned +bur nin +buc to +blan kety +blacklist irgc +betterthan yours +benedic tus +ball an +ate man +alexander skarsgard +acceler ant +a ach +. ðŁĺĨ +**** **** +ìĦ± ìļ´ +ëł ¤ +âij ¢ +ÌĦ âĸ½ +you too +yo sp +y it +xen akis +woo sh +wonder women +wicked wednesday +wi max +way days +watch oftheday +w ends +v me +u af +tw illiams +tu llis +tru by +thu l +tech comm +tai you +si su +scip y +scho ch +sch es +sc lera +sc ali +salam u +sa dist +robert smith +ro miti +rich ton +reminis cen +redol ent +re arranges +ransom ed +pinch punch +pilgri mages +peter king +pan americana +pakistann ature +p gb +over top +ornitho logists +ol aya +off wht +nasa earth +mé rida +my man +murdershe wrote +mu do +mor ikawa +moo i +money bag +model e +mix show +mit ra +mind fullness +min iso +meravi glia +mer an +me ols +mazz eo +martin o +mari en +mar acle +luck less +lotu ses +logic pro +ligh trail +lifeat sea +lets doit +lee kuan +lebo res +lar s +lamborgh ini +kunsthistor isches +kmc kidd +jo ely +ir retriev +in play +home renovation +hom eles +here eee +hem lines +grave tt +gr ane +gov garyjohnson +ge tsu +french polynesia +free world +four cade +fa es +f ap +everyday iloveyou +entrepre nuer +em iller +earth month +ea seof +drogh eda +dpan ikkar +dolby atmos +dirty water +devon franklin +death wing +dad s +d oud +curi eux +cowboy fb +colour ation +classic film +chri sn +ching on +chin sky +cher aw +ch acal +cav snation +camp bestival +call ard +blue dot +bere jiklian +baris al +awa ji +att inam +arizon afball +arbaaz skhan +ap nic +ame th +al ts +al fam +ade goke +:: . +ðŁıĪ @ +ðŁıĩ ðŁı¼ +âĿ¤ï¸ıâĿ¤ï¸ı # +à¸²à¸ Ĺ +zing ara +you have +wo ahhh +wind less +vietname se +vampirethe masquerade +v hope +ur ss +un noticeable +uk photoshow +tur gut +transcrip tomics +tr bin +tor ano +themac belfast +the pack +the bachelorette +ter ium +td kr +swif ty +suppos itory +sportscenter ph +sports blitz +sofar sounds +shu ck +sasikumar dir +san th +san joy +ri ess +reasonswhy welove +re tells +re programmed +ram p +plot line +play more +pharmaceu tics +pay pigs +over strand +ot ville +omni presence +ola fu +okon jo +ocon taldo +occident alis +obedi ently +ob elli +museum next +muscle car +montan ez +mis ner +mi hara +mess ick +medi atour +me ret +m for +le tti +kinok uniya +ke ion +ka hanam +kahanam oku +jim erson +jame stown +jam et +jaguar usa +it sgo +inst ay +impre ss +human i +holliday sburg +health policy +he mma +hardwell onair +haiku jam +haha hh +gucc icruise +gom en +gold mining +glen ferrie +gennar ocontaldo +g star +fox rock +fly fishing +flo o +fin alization +falsi fication +fa hadh +esta dio +equit ably +enumer ation +emil yy +em elt +ear m +dom ineering +devop sday +deborah annwoll +dam ba +da beast +chu bs +chik magalur +carsand coffee +bx tch +bri quette +bi bio +banff np +bad hairday +attenu ator +arnold palmer +apple day +andre au +americor ps +alph ington +aham ed +ag amma +adobe illustrator +adi e +ðŁĺı ðŁĴķ +ðŁĺįðŁĺį âĿ¤ï¸ıâĿ¤ï¸ı +åĦ ª +« ม +zdar sky +zah ira +yo ku +yeare ver +yar mol +yach ting +wy bor +work arounds +win free +whowhat wear +wey den +wer nick +waw g +vallu var +vallab hbhai +v crs +tran scriptions +tic hin +ti ron +the hills +th im +teh ri +team messi +sty rke +stro h +spirome try +spald ing +sothebys realty +sling in +shrews berry +shipy ard +sher ali +sha stri +se less +schem mel +sch movies +sc magazine +safe schools +riz wan +resna is +rec rimin +read more +re ws +quan ah +prince sse +pokh ran +pel u +pel otas +paw ning +pat eros +over spill +over burdened +oun is +on fleek +olatun ji +official camogie +ni ji +ng lifestyle +newton more +neu e +my d +multi mode +mis cues +mis carried +michael chiklis +micha ux +matt taven +luci fans +lucas oil +lostgirl series +li sat +leekuan yew +lakeland terrier +ki pru +kate garraway +kapp ak +jonny wilkinson +jo edon +jn co +jas in +irishcancer soc +im seth +ig ing +hurricane prep +hinchin brook +ha vo +gyne comastia +gigu ere +ghan oush +fy ing +footbal lau +fine man +fig c +fatherjohn misty +digital agency +d now +d line +curmu dge +csu sb +con comit +clay man +chi omega +cher ni +charle mont +cer van +can ino +camil amendes +brown barrie +bomb proof +bk club +bintur ong +best es +bend or +bam bini +back home +av ity +at omar +art ane +armi jo +angelique kidjo +al avi +ad hoo +a university +^ âĸ½ +ðŁĺį ðŁĻĮðŁı» +ðŁijįðŁı¼ ðŁijįðŁı¼ +âĢº âĢº +àª Ĺ +zimzalab im +zi um +yy ours +ye stur +wi vedi +we sa +vs jax +v official +un free +u hhhhhh +turf way +tuney ards +tun ny +tread le +transp acific +titus oneil +titusoneil wwe +thereal kmckidd +theposh official +thegreat gatsby +the basement +taj mahal +suf fixes +spiritual awakening +spike island +soccer dotcom +so hu +sky bus +sk ater +shar ara +se tag +rz esz +rte gaa +road tripping +ripp ed +rick springfield +rich ter +ra uk +q do +pom be +play harder +penc illed +p man +own town +odd s +ns ls +nko sa +ng am +n wed +mu ssa +mr darcy +micro car +mar wari +malm stro +mac jc +ma ung +m wo +lin in +lake view +l summit +ku ja +kirrie muir +keepfighting michael +k maw +jhal akon +ja ja +islam istheproblem +ic title +i ven +hudson weather +hov land +hol steins +hipp ea +har dens +gwy dir +gw ladys +guy zz +god manchester +go ti +g ors +fu jin +flash tweet +first grade +fac daniels +f kd +en sing +e wing +du shi +det sky +del mont +day stom +dan h +d sena +d land +cy fair +cu mali +craft bizparty +cour son +coffee script +cho han +chi ons +cep tive +car doftheday +campe ona +buli mic +bri mbank +boudhan ath +boom slang +boom bayah +bo ty +bo the +bis wa +bin us +baby led +b hh +as par +arkh angel +amnesty usa +alve olar +alle stree +ali a +afro house +adrian edmondson +actu ate +actress life +(( (: +íĻ© 민íĺĦ +èĭ± ä¼ļ +ج Ùħ +z andi +yose ary +yaf antasy +whipp any +wbb mnewsradio +wall bridge +w you +vy rn +var chives +un ak +tritic ale +tre sem +too hot +the journey +ten anted +tall ship +ta pio +t morello +syn ched +step family +stay la +staser aintv +springhas sprung +sla ppin +sl mc +sko ch +shad ings +sh kin +sergio garcia +sav vas +s gi +ru pali +ron del +ri va +requis ition +rendle sham +recon quista +raven scourt +rah mah +ragh eb +ra ir +qur an +puro resu +por gs +pneu ma +periodon tics +pen alise +one ering +od ourless +o cker +nu bbin +naf is +my hill +min v +mil azzo +me gau +manu rewa +mad h +mac es +lil ja +libby schaaf +large sse +laid ley +l trs +kur saal +kome diab +kfm za +j antz +hand cart +ha ining +gu yoseary +goldenstate warriors +gil les +ger ads +games master +flu o +flavor ings +er onen +energy access +en ro +ek im +eco logie +eart fair +e stel +disney channel +dgand yofficial +den isa +del vecchio +dead rising +cu bi +cro eso +credit cards +cor oz +cli ocup +chew able +chamber land +cad dell +book out +bla bla +bis p +bill yon +bhar ti +bec ton +ba ret +atro x +ani us +all rise +adve ction +? ðŁ¤Ķ +<<<< << +( ^- +åħ ī +ൠĭ +world fest +with heart +wat kiss +vinyladd ict +vino gra +ve dt +vander pool +uss ain +ur mi +un sophisticated +un desirables +un approachable +ty rie +tu lp +trust me +traumain formed +thedead daisies +the jagmeetsingh +thar bour +sy ro +sy chi +stop avn +steel town +st francis +shu ichi +shav ing +second home +scar f +sas ural +sandwell council +sab lan +rohan bopanna +resc ities +recor riendo +ration alist +pie stewa +peep z +pap ato +pal af +pag odas +oh s +neel ima +mother ingsunday +monet ary +massaso it +lö f +lu lay +loch ner +lo vering +leedu chat +kyungsoo day +kj show +kil ala +kh ate +kay ani +ka ist +jhalak reloaded +ing post +in opportune +in concert +image sof +ike men +halloween time +go tem +glend inning +gender bend +g afford +etihad stadiumau +el by +easter monday +don nap +diamon t +deano gorman +da sia +ctvmorning live +cron kit +cir lik +chri scar +chin ta +chi ado +cam bon +c dre +brandonj routh +bon doc +bo olin +bid ston +bi kela +begum nadiya +bab an +aun gier +artiston instagram +aqu arist +annihil ates +anim enyc +ald abra +aj m +air dro +ad ome +abor tive +ðŁĺ¬ # +ðŁĴĻ ðŁıĢ +ðŁij¼ ðŁı½ +ðŁij¨âĢį ðŁı« +ðŁİ Ĵ +à® ´ +र द +zeecine awards +wildflower ctr +water management +wai kiki +von taze +voice top +unlv mbb +un born +ul er +tweeta photo +trans acting +the jeep +thai pusam +tech sters +tasty trade +sw ich +stom ata +sta th +st mt +sol l +sla thered +side y +segu ros +sc ps +sc chat +sar ag +rn k +rend ts +reel foot +ray ana +raisingthe bar +ra pin +pu tri +prophe tic +press burger +por ty +pla xico +per r +per ps +pal matum +over age +outeni qua +oro driguez +ordin aire +open farm +nj dv +nike uk +nau ck +nar thex +nacion ales +mor ti +mone ill +moneill sf +med spa +mcclo y +mc grain +mat rons +maryam nawaz +mar gate +m ds +lease holders +laura osnes +lamin itis +l den +ko stya +kis mat +kii ara +ju bb +james r +il ham +ico in +hy me +hou rigan +heart landon +heart fulness +harper adam +google pixel +gon u +girly things +gin i +gillian jacobs +genoci des +fon dled +fi ster +fashion gram +f tr +explo der +exib ition +ex ilis +esh our +english language +edd zeko +don ot +do led +disal low +david giuntoli +dalla ss +council day +con boy +chan thaburi +cerebro vascular +cast ros +calgary police +c ÃŃ +bromel ain +bridal market +billy crystal +bij li +batman vs +ba ise +ammy virk +adil hussain +aap ki +.... ..! +ðŁĴ¦ ðŁĴ¦ðŁĴ¦ðŁĴ¦ +ä¹ ĥ +âĿ¤ï¸ı ðŁĴį +འ² +à± ª +z suz +z sc +z ali +wor g +willie geist +war shaw +want s +vi vat +ve co +vanat chathiram +van brugh +usic live +ureport ke +unchar ted +un too +uil texas +toulouse fc +tom oka +ti jani +the town +ted leo +techno polis +te itel +t zi +swith un +studio city +slovenia info +sl q +sky bar +si raj +shain anc +schipper ke +rolling loud +ro dro +rio ter +re populate +rain or +ragaz ze +rac tice +point les +plot kin +player stribune +pic ador +peewee herman +pay roll +pac os +our stars +ou sia +other kin +or rrr +ob trusive +north well +no ongar +nin jas +nic hd +newpor tri +ner dgasm +ne res +national dessertday +mu vy +mayweather pacquiao +maximili ano +masti mania +mar sy +man ap +mal ate +makelife aride +ly y +ly ria +leas owe +lea quitaine +lay un +last chance +lan ow +la quan +l porchestra +ks v +kor ach +kil dee +k nation +ju iz +johnnyg weir +jes sel +j arri +ital ks +inun date +inter weave +ing rati +ib is +i aido +ho ste +har ket +happ p +han amaru +ha ug +great memories +go tyj +go ines +gilber to +geek dom +galax ya +et ana +este fania +ellip ticals +e stro +e ecs +dur gin +dontbomb syria +do dsworth +dead beats +curric u +ct politics +configur ator +colly er +collegi ality +colec cion +co su +co dex +clau ss +ck ler +cityand colour +chi mie +cardiff bay +capital stb +callo fcthulhu +bruhits zach +bru yere +bray brook +braintu mor +bolly mastimania +bo gar +beth lem +ben ro +ben ck +bang ura +bal adi +ati as +ar curi +ankylosing spondylitis +anc elife +an eda +ambi ental +alpin ist +alexandro v +abo lishment +ab dash +a hin +.... ..? +! ðŁĶ¥ +ðŁĺĤðŁĺĤ ðŁĺį +ðŁĶ ī +ðŁĴľ ðŁĴĸ +ê·¸ëŀ ¨ +ãĥĸ ãĥ© +ãĥ³ãĥ Ĩ +à¸Ķ à¸ĸ +za ghari +ym b +yc nature +xabial onso +weather watchers +ve gal +us womensopen +unob tainable +un volunteers +u selections +u donis +twer ks +tr na +tish james +timber frame +thistle down +the stage +the palm +the champ +th ely +tee zy +tax cut +sé rie +ston efly +ssun stein +ss om +shum way +se bas +sale sian +sak shim +ry se +ry pt +rotor adar +red hook +re creations +ran jan +pu dd +pom poms +pl tw +pemb scoast +pe als +parliam en +palindro mes +p sim +over rides +onep ur +nik laus +nen ownews +mueller investigation +micro bead +mehra uli +mb alula +materi alizes +lyre bird +liv ant +lightning deal +ku roi +kp j +ko va +kims convenience +kh of +katzen berg +k under +jour dandunn +jnj news +jim ura +jeep thing +jamesp oulter +jamesmar ster +jag ga +ich ry +ia fe +i sport +huntingdon shire +horseand hound +hope fulness +honor ing +hazel dean +happy tweet +h fo +guti érrez +gu ery +grace church +go explore +ghes quiere +funky town +fly catchers +euro centric +et witter +eli sts +ed sheer +drink bodyarmor +dispro ving +definit ely +decath lete +debt trap +dan sen +d memories +cw f +csn chicago +costu m +com pre +climateaction now +chlor o +chall en +cec ina +cas ssunstein +canon usapro +cad wal +bro force +assas in +art crew +alve ar +al toon +agon isingly +aga st +. )! +- + +ðŁĺĤ ðŁĺĪ +ðĿĻ ļ +ë ķ +ç ¤ +à¹ģภ¡ +zom bs +zof ia +yan key +wokv news +wiz bi +wi pro +we buk +warwick adavis +viz wizbi +villa franca +vie tor +venice filmfestival +vehe ment +uefa championsleague +tsne deker +tri eu +toro idal +ther ight +theli st +the arto +thank youn +temer ity +swachh india +sth d +sta a +ss wim +spin abi +son oda +sock sout +so dding +simul acrum +simpl on +sign posts +side walls +shak yam +setau ket +sat ine +sand storms +sa adia +s vet +s grace +run o +r ham +pro long +pre tori +po veda +phir se +pe gues +patro clus +pal omo +over lake +out magazine +our heroes +oun ce +ori ordan +on gus +ol le +of sted +ny heter +no conversion +night spot +n ols +my twitter +mood iness +mo pa +mo dems +medit ators +me spo +max ted +max pro +mal olo +mail let +mag nier +loo ka +lo sar +lear month +landmark game +l wk +infer no +in gos +hy nd +ht ml +he wa +har ap +hal ore +gra yer +gold thorpe +gal gos +fusil ier +frei berg +forward together +fm wales +fat ai +ez idi +exp consulting +etv scandal +elevate gg +doofen shmirtz +do we +di restra +defin iti +de ely +dan ke +d smith +creepy puppet +cot man +confl ating +chinese theatres +chan tier +celebr atin +cashme recat +caravan serai +car photography +camp ello +c pos +bryce papenbrook +brand tsnedeker +boo te +bobvand illen +blon der +bizim hikaye +bil ko +bha id +bensen ville +bear sted +bath i +balla chu +aw oman +aven e +arts district +art u +apcu kingdom +anti static +andro b +alphanu meric +almer ÃŃa +agit prop +ad reno +ðŁĺĬ ðŁĺĭ +ðŁ¥ Ľ +ðĿĹ¢ ðĿĹ +ë¹ Ľ +âĹ ĩ +ze bu +yor lando +yoff ish +x anthi +work up +wetherby hour +waveform gaming +v ory +us nsa +us ami +un ning +tri on +travel noire +th burne +tb ptv +tan zi +sy b +sun trap +summers lam +sr ch +spl ant +shot left +shan tou +sapp arel +sanssou ci +sal ang +rod kar +rho desi +rh cp +re fashioned +qui am +pumpkin seed +puma southafrica +pre ux +prati ma +pog dogs +platt ner +pint size +phen mj +pau lan +pau dybala +ost ler +on style +on salenow +ol oroso +ni ddry +new burg +nast ase +mu six +moi sten +mixer streamer +mar mal +makav eli +ma fa +lou n +long ter +lon done +lime wire +la en +kor ba +kom olika +kk crvenazvezda +kav ana +jur gensen +jamesmarster sof +it sma +ip sen +ino saur +ing love +ic ml +ic amp +i wt +hi dup +gre bel +gra dings +gr antly +gl ück +gilli gan +gar aki +for freedom +ev als +esh t +du ga +do gin +dc sd +ct z +cow man +cor ke +convey ancer +contribu tory +chiric ahua +catal in +carstar phenmj +car yl +c pi +bu bur +brun dle +brown sboro +boston fire +body painting +bethnal green +ber line +bedd gelert +be zz +bakers field +avi acion +australi as +agu ita +age ant +aeroline as +$$ , +! ðŁįĢ +Ĥ ĺ +ðŁĻĮðŁı» âĿ¤ï¸ı +ðŁIJŁðŁIJŁ ðŁIJŁ +ðŁĩ±ðŁĩ ¾ +ìĺ¨ ìľł +ä» ĭ +人 rt +âĥ£ âŀĸ +ya day +x slasvegas +watch face +w ge +voyeuri stic +vo va +visual merchandising +ver wood +vas ile +up y +twopad docks +tweet link +ther ace +te be +ta ar +t go +sun danc +sting less +sosn icaragua +singleuse plastic +share file +se acombe +scre ek +sargas sum +sarai va +s enga +roosevel ts +right sarehumanrights +rejected jokes +reg icide +re ais +ra sika +ra kh +q urban +pul ps +pom pe +plat ano +pic tori +peter roskam +pase kand +pasekand paul +pan oz +no it +new mom +nd ma +napp anee +mur mu +motor hour +morph ism +men ted +matra ding +math art +mark duplass +mariob autista +man official +man gel +mal low +mal formed +madein scotland +m seto +lo bato +live jazz +lin ac +li salam +la ge +ko ken +key strokes +juan man +john elway +jagu a +indeb tedness +in ri +in conveniently +huar aches +host as +hook worms +hil ip +heartsofoak gh +haw kinson +greg jennings +gom m +god flesh +glos sum +ghar ana +freed land +flori dat +fl atlay +fin ola +fin ear +feder ation +fan euil +ent en +en quanto +efan club +edmundmc millen +distant worlds +derek theler +dc te +da esh +cu dnt +creati ven +cra ver +coton easter +confor mance +clar kie +ciar án +chrisber ghd +chi w +chekk achi +ch ach +cav anagh +cas pers +brew is +beck ys +be hm +baz za +asap nat +ar rrr +ani sta +aneurin barnard +ameans business +alex slemonade +ale cology +ad sit +a hil +ðŁij ŀ +ðŁİ¤ ðŁİ¶ +âĺ ¢ +zi el +yu ku +yam ec +xi ah +worldskill suk +wn ation +wh omes +vw beetle +vo sloo +vale ska +ul va +ucht dorf +u cros +ucros spop +tough man +thought bubbleuk +terp stra +tay m +syzy gy +swe mfa +sun ni +sun king +summ ering +stone island +steph curry +ss ch +sp insters +shim o +seen ur +sap cp +san jo +sac valley +ra bal +program ma +presidenti alelection +pic on +piano forte +pegas o +orchi daceae +open space +op rint +ocean arium +n fm +more years +mor umbi +mezz o +meu len +me up +mal dive +ma kau +lovemy family +louren co +lo hen +liket kit +kick son +kel u +kare lian +ju nee +john travolta +jersey boy +it yo +ing ian +imper o +hud hud +height ening +happ ily +gri fo +green tech +ge dsb +gaz asi +g gh +french town +fest ool +ex is +esc apology +empire strikesback +el ga +el fen +eck ner +dra che +donne come +donnecome cipare +dj booth +di jual +decor ah +daf f +cul o +cro ghan +covar rubias +cold case +cas cio +car canet +cap alaba +bur meister +bo lete +blu ess +blo k +bioshock infinite +bg motogp +bbc nottingham +baybay in +az oulay +ay eeeee +appetite for +anti ga +anil ine +angk lung +ak ap +aids walk +ah p +ag rf +!! - +ðŁĺ¨ ðŁĺ¨ +ðŁĴª ðŁĴ¯ +ðŁijij ðŁĴĸ +ðŁIJ¦ ðŁIJ¦ +ðŁıĥ ðŁı½âĢįâĻĢï¸ı +ðŁĮ¬ ï¸ı +ðŁ¤© ðŁĺį +ìĿ´ì Ķ +âŀ ¢ +ÙĤ ÙĪ +ün den +yt ff +yoko gawa +years agotoday +yacht club +winner sh +white people +weyer haeuser +wab o +vzw buzz +vine sh +umm i +u vula +u cia +traditional home +the sen +thanks givin +templ in +tal ca +tai led +syl wia +swift key +stru m +sk yn +shadow cat +sees outh +seb divo +se mu +scy cle +sar daar +san guin +rani khet +r hames +quar tey +pre emption +poldark tv +plebe ian +per ations +pen ang +pawn brokers +ox blood +onto logies +od ni +nuf fiel +north pennines +nico tortorella +mur gh +mir amax +mil bury +micro biomes +mega projects +mcgla shan +maurice benard +marr show +man cino +lo as +live inthe +legobatman movie +leaves den +le thaw +lake town +kru is +kin ison +ka fue +jen kins +jaide ep +im prisons +ichi ello +hil aria +hi mani +grizzly bear +greg grunberg +gor gas +gn z +glan ford +gam elab +gage town +fc academy +fanta st +expen ding +evan sdale +emp angeni +emily slist +em presses +ell park +eli hu +dul lahan +dr jason +day atthe +dan ariely +dam sels +csi ro +comm ing +clare mont +civari ze +ciut at +city farm +cis gender +che did +chantic leers +cal care +bur tons +bon ton +bom et +bl ough +ben haenow +ben ben +begu sarai +bee de +bak an +bag ua +as cc +ar cy +anni ster +anglic ans +alle les +al ute +agu delo +afl north +adri aan +ade osun +ab bv +ðŁĻĪ ðŁĻĪðŁĻĪðŁĻĪ +ðŁĵ· ðŁİ¥ +ì°½ 민 +å¾ IJ +z ino +youth summit +your call +wn cwx +with nature +wi eck +way station +watchme work +voor hies +views for +veri thanam +v fest +us new +ura sawa +tv j +troop a +tri fluorome +trans duction +tom or +tin dle +the press +te oh +tan zer +super ted +stor i +sth our +steam punk +sl oman +shil pi +shamp oo +sh ash +sex periment +san sevi +rugg eri +ron ix +rob shaw +ri vette +rc slt +raghe balama +rachel doesstuff +que ster +propeller head +pra z +potenti ality +po groms +ple e +pinup girl +petz old +pattim urin +pat ang +or ley +oce ancity +nur gle +nun dah +nowisthe time +nor wood +nomuslimb anever +ni vas +nef yn +mous avi +middle village +mey hem +medal of +maxim ises +mart elly +marianor ajoy +malmstro meu +ma hur +ma fu +lu bber +lovel ords +long ship +le pel +lat l +la chowski +l alab +karab akh +k oral +jewellery quarter +jay mewes +j dc +iri zarry +ini sterio +infection prevention +in eland +icy cle +iclassical com +ic re +hou f +hoo fed +hol dren +hek mat +grote sk +great apes +go thel +gi at +gadel maleh +frenchal ps +fore telling +film tv +every ones +england v +e ppo +do ar +dc is +cool um +competen ces +colon ise +co bi +city year +ci anci +child protection +candle mas +bullet proof +boat y +blv ck +being coop +band q +bally mal +bagsof help +au llo +anirudd ha +angeli e +am gu +am ab +al annam +abo i +aban kers +aaaaaaaa aaa +. âłĢâłĢ +ðŁį´ # +ìĦ ł +é¤ ¨ +à¸łà¸²à¸ ŀ +zan es +z illa +yor dano +wahe eda +w stc +volcan ology +vic eroy +trum pian +tric ycle +tre ys +tomo hiro +the profit +take s +swi pe +sumptu ously +stock holder +stead iness +star log +ss ilver +sre i +sp igen +sor presa +son ye +soccer six +small talk +sky cricket +sie wert +shimaz aki +shiki gami +sco ble +santay ana +rup turing +rose well +rosco ff +root stech +roman zi +redwhite andblue +raic hur +rai han +pv p +pp u +pharmaceutical medicine +pey to +pa zuzu +pa vic +of en +obam ba +ob om +nor lin +no confidence +nn sa +nk ana +ni pe +n cep +multic enter +mouth guard +mi az +meop ham +md v +may nes +mau de +mari jke +lun aire +lifeof desiigner +le comte +laine y +ksh b +kom in +kir win +kane wwe +jic ek +isal oni +isa ach +iran election +in attention +iit madras +hype app +hero in +hepatoc ellular +har key +gwilli mbury +gu altieri +go camel +glycer ol +gk union +gangnam style +fre ke +fre el +fel brigg +faz ura +faul ds +fau rot +ev aa +erskin eville +epic reads +echel ons +easter seal +dr ane +doc teur +disney hyperion +diplom at +diade troit +de wees +dd lc +dar ci +dant as +danne mora +cout u +chor lton +chic ou +car fest +cap sizing +c micon +british spiders +brac keted +bom ar +bo ber +blog paw +bay les +ba chill +ba cher +ath ousand +ar leta +amuk erji +ale ague +ad ames +acc tourney +ab rit +??? ?" +ðŁĺįðŁĺįðŁĺį ðŁĺĺðŁĺĺðŁĺĺ +ðŁĵļ ðŁĵļ +ðŁĴ© ðŁĴ© +ðŁİĵðŁİĵ ðŁİĵ +zapo tec +ye pes +xx yyxx +wit kin +whipp oor +vu lus +vivi ano +visit japan +var den +vanqui sher +van u +v dsl +tu xie +trivi athursday +travel africa +to kaji +teve rest +tekno logi +tech nis +ta vit +syncop ated +sw ool +super couple +sundar ban +sub mitter +stru ff +star force +sportsc asters +sp ä +smriti vidyarthi +sig sauer +shi bu +sequo ias +sc b +saul williams +sare k +rosen kavalier +ron in +ren toul +pra de +pink history +pic hardo +phyto ph +pen za +pal et +ourland our +or li +one year +nr lakl +new urbanagenda +ne iges +ne els +n stom +mom ota +made at +lyn donville +loonathe world +lo anees +liquid lipstick +lay men +lar avel +kim bo +kar thick +k sf +k ren +jun yeol +jav elin +inequ itable +imag ing +ida home +iam dbanj +hagg en +gul mohar +glar ingly +gd antes +g elo +fu xin +fren kel +frank ness +fr ys +followthe cow +electro magnetism +dublin marathon +dubai marina +desig ne +deb nam +dare ss +dar rion +continuous delivery +clarem bee +clar dy +clan william +citi bikenyc +cebu anos +c figueres +bu mming +bra p +bra ben +bier ut +beauty bloggers +beauty and +beaker head +balon mano +bal derton +back slash +back nt +ba ilo +b flo +award winner +au te +at sb +assou line +as oc +armand de +and dddd +ali jo +agu ardiente +advis or +ad alli +a events +ðŁĻĮðŁı¼ ðŁĻĮðŁı¼ +Ù ¢ +ع ر +zai batsu +z if +yog scast +www blogs +wake andbake +wag gon +vol aris +valeri o +val ores +v yan +ulster museum +transhuman ist +thri ving +thr ur +there for +the island +tc ja +taw dry +tam riel +sugar hut +stay well +stan bic +sp idad +spidad mitchell +solenn heussaff +simil ar +seo joon +sas ne +sand ino +ru zzle +restar ta +reli asson +re ten +raz ones +r khan +pri js +pp ls +pou lain +po ire +pin points +petre lla +peer age +pec kem +peace forchange +pas kal +part is +parasit oid +or adell +onz ales +ober st +nps official +nph w +nicole and +nichi ren +national loveyourpetday +mu za +mol ysis +miri anag +mirianag rassi +michelle phan +materi alizing +masa o +margare tta +make m +mag as +le ire +le ab +labor sec +ketu pat +kam ble +k croyals +jor hat +jo su +jeffreestar cosmetics +jak er +j lab +ishq subhanallah +induc er +im and +ilo cano +ick worth +ibm security +hi po +hand knit +ha em +go peng +gim sswiss +ga etan +flipk ens +fil led +eye onthe +es la +encoura ger +ely ons +dy i +dr sarah +dough boys +discover able +del imitation +dear born +cr sh +contemp tuous +coat rack +co xe +cl inger +ce fc +capp rentice +cam pero +californi awild +bur wash +bu stour +bu sker +breakfastclub am +bo dak +bit trex +barri ga +back packer +b wt +b sg +at onal +asu xx +art matters +arro char +ap sphysics +antic lock +ant sula +alo isi +ak ins +aer yn +" << +ðŁĴĭ ⾨ +ðŁİĦ âĿĦï¸ı +ðŁĮ ¨ï¸ı +ãĤ° ãĥ©ãĥĸ +âĢ¼ï¸ıâĢ¼ï¸ı âĢ¼ï¸ıâĢ¼ï¸ı +د بÙĬ +ä ki +zah ran +xe han +x force +wi za +who tel +weekend read +web toons +weare jcps +wc ps +w str +vermic om +veg ard +unzi ata +ulti mas +ul trafine +ucl ambb +twin ks +time outs +the gallery +th ence +swift ness +suc at +strato varius +stats zone +sn cla +sexy sunday +severy body +sep toria +seenur amasamy +scorpion fish +sco s +run neth +ric ki +rhi an +retic ulum +res foundation +red lion +po sco +pba ontv +parro toftheday +pap hi +pan ela +o zeri +northeast news +no wa +museu mc +motor land +mccl ary +mate ship +madhu du +maceach ern +lin ec +lie man +lette rer +le aches +lamb ily +la sairport +kon st +kle berg +kle a +kaushal army +kait lynn +kafka esque +ju raj +john carpenter +jay mcguiness +jacque storres +j sp +irish design +ire zumi +ingui de +impe tuous +id ade +ich ella +icec ap +holi bobs +hir schi +hil ts +he dren +harrypotter film +gyeong bok +gor nal +go recki +go further +gla as +garri gle +gan ap +ga illi +fren k +forte an +fish lock +fin det +fer ron +fanni emae +exfoli ator +ex claims +es mee +enti es +emmitt smith +emb ree +ell it +drummond ville +dr d +dogs nyc +diversity intech +dil um +de pon +comment ated +clu ff +chi dori +che fan +categori zes +car dell +canasto ta +bride shead +bri v +bri jesh +bre search +bold ly +bli ppar +black stone +biz women +before i +be dou +be ag +at ong +ar ges +analo gues +amare lo +abhan sali +- [( +è Ļ +æµ ľ +âŀ ° +âĶĬ âĶĬ +à² Ń +zax by +xin yi +wr ps +whiskey wednesday +wak iso +vici ou +vegan recipe +tt g +trivi umofficial +the cbso +thati sall +th iri +tee zer +tav leen +t sak +subtrac ted +sto le +steam town +ss rn +sop io +sm fm +she ppy +ser pa +se ato +scand rick +sc ir +saw telle +sac and +ro mig +rent schler +red lobster +realclear politics +re growing +rat v +projec ts +prin sen +pri show +pe lec +paw fect +pal merton +ota ki +ot di +or tunity +onthis date +ol alla +ogden sburg +of rac +ni ggling +my slf +muhyi ddin +mini figs +mikaela shiffrin +mend inger +men ot +mat es +magi sta +made inspain +ma kalu +lympho cytic +li kowski +lauren holly +lau be +kur ti +krish namo +kh uri +kh ill +j ör +j haveri +isol ationist +intothe unknown +innocent i +iheart country +horn dean +heartlandon cbc +habil itation +ha ole +ha ffner +green landic +gly de +ge ot +gario ch +fundament al +fro ots +for one +flat fish +fayouth cup +faul k +fathom s +espre sso +el na +eke inde +diversity matters +di ther +dan ja +dam ita +custom s +coa sta +chou ler +chen ery +chembur studio +cc np +cassio bury +carla harvey +cam pin +by les +buc cos +bu yon +bou lay +book tours +bene gal +bap tise +balti erra +ball ymurphy +at rac +as itis +ar vest +antagon izing +an tre +an amne +all youcan +alam in +adventu res +adel boden +ade b +ad jani +acer ola +ðŁĺħ ðŁĺį +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤðŁĺĤðŁĺĤðŁĺĤ +ðŁĴ¸ ðŁĴ¸ +ðŁIJ¯ ðŁıĪ +ðŁ¥ Ĺ +ì² ł +ãģķãģıãĤī åѦéĻ¢ +áħł áħł +ystr dy +workington town +walk outday +wakar usa +virtu emoir +video tapes +ves sey +v line +usatf outdoors +tu e +tk allday +thiem domi +theevil within +swa ard +su ster +str atten +stam ey +st live +sr ks +slo ck +sle ee +ship ston +shi rakawa +shaf ting +sbu f +sau sa +sar ja +saint motel +rockwood nyc +ro ys +rif kind +recircul ation +re shuffled +ratt us +ration alism +rand fontein +rais i +pul sation +pra x +piazz ale +p tm +p my +p few +over diagnosis +no tobacco +no fur +nav an +nas uwt +moer aki +mobile phone +men zi +men tira +men go +mel illo +mc café +maggiel indemann +made it +macdon ell +lo rene +liven ow +landing page +kyrsten sinema +kuro ki +kier nan +kid dush +kai ley +jo sten +jad er +ir za +in situ +ice breaking +ic tures +hy nix +hu huhuhu +hopenot out +hippea strum +hg hair +heis en +he ddle +hat chards +ham mock +granth am +gary oldman +gansett beer +gall antly +gal lion +fur thered +franco phones +fin ne +el iner +dol and +do ire +dne ews +demo gorgon +dementi auk +dayof pink +cv spharmacy +cor adio +co be +clyde side +clarine ts +chung king +chocolatec ake +chi antic +che c +cer ney +car pel +c sdp +bren au +bol on +black swan +bill ye +ber magui +bel loc +beart ooth +bal aram +ate gui +ast it +ash s +ascor bic +as pers +are mos +ar vid +ar rakis +an sf +amar anthus +al biz +agu st +adul terer +ðŁĺįðŁĺĺ ðŁĴķ +ðŁĺĤ ðŁĺ´ +Æ Ģ +zet tai +z c +yo a +wwii museum +walking tour +w kt +visit ms +vide ophone +ver k +uach tar +try m +to bler +ther ace +teslamo del +tech navio +taver ne +tail coat +ta quitos +stil ted +shal f +sec conf +scra wl +schri stian +saifu ddin +rosat om +ron delle +rhu dd +rebeccas ugar +re iman +quin tiles +quer é +pur don +pun tag +pro by +pre conditions +plu mer +photo gravure +phac elia +pep tic +patty jenks +pal let +open vpn +obo ist +o vale +nz x +nor ges +newsp olitics +nasa insight +narr ators +n je +musi que +msh saa +mr music +mmk maymay +mine fields +mdc p +mb laq +maxim ised +mati bo +mar cle +ma pi +ly ri +lowe red +love thy +lord minion +le froy +kri stal +jubi le +jack olan +jack nicholson +island peeps +inter county +ik ke +i win +horten sia +hor sman +hil len +he bel +he aux +hawes water +harmon ising +hard drive +hamban tota +hai k +grey town +grenou ille +gar ance +fur con +fresco baldi +film forumnyc +father like +ey in +et x +equ itter +el ens +ejac ulation +drake hogestyn +deyoung museum +dere r +deal withit +cy be +cri xus +countdown to +co chem +cl ancy +ced illo +catapul ting +bu gat +brigg s +bra ai +bow eni +beyon der +best buds +beauty by +bar a +back dated +b too +awe st +at its +ase k +arm streaty +ar amide +any a +an fal +all winner +alici avi +ðŁĺķ ðŁĺķðŁĺķ +ðŁĺ¬ ) +ï¸ıâĥ£ / +âĻ¥ï¸ı . +z atch +your life +wrecking ball +who youare +we streamers +we gs +w fyi +vite bsk +vis njic +veld hoven +vaun wilmott +us bp +un pleasantness +u stennis +tonyi ommi +toe hold +to bio +thisis what +the barn +team yoshiki +sven sk +subur gatory +sto chowa +stereo scope +st maarten +spit ality +spa stic +sor ters +sno we +sl fl +sight line +sh antz +ser gal +seng ineland +sa ung +rober ge +ring lets +ri ed +red wolves +raj ab +r km +q nd +preten tiousness +pine do +peter hook +pastry chef +par ce +on kiss +nzo ia +no vio +nie g +nfl redzone +nag ara +nab er +mookie betts +ma sten +lu ganda +love field +li j +letsgo warriors +leone an +le tham +krist novoselic +kin da +kas uri +kali ka +jupit us +jonah nro +j ide +im ing +il te +holy cow +hemorrha ging +harris ville +har less +hand cuffing +hamp us +hal ala +hagan ai +gulli bility +gran ado +glass blower +gareth malone +fu u +friday nigh +free e +fr ate +foxsport ssw +fore warning +fm hs +flu te +fer rar +dri mn +down shift +doire gaa +depreci ated +daenery stargaryen +crush monday +cmicon tent +clenden in +church down +chi kka +chalo bah +ch war +car cosa +car ap +bur ness +brick leberry +blun den +bi pa +bemid ji +believe movie +bc sn +ba ot +b prd +aur al +atomar aullo +astro cytes +asi andream +art brut +armandde brignac +al ge +aap kad +aa ap +.. ðŁĺįðŁĺį +!!!!!!!!!!!!!!!! !!!! +æķ £ +å®Ŀ çŁ³ +ÑĦ ле +ʸ áµĴáµ +z hai +ya official +woj nar +with h +winter weather +water bird +wait omo +w dam +vol kova +vi hara +v kook +us ms +up field +travel ers +trail magazine +trac tions +token ism +tobo lowsky +thequeen bean +tel stra +t atten +stutter heim +sto ichkov +sporting news +sp atch +sna k +sky liners +six tus +simp kin +shortland street +shag ari +sep tal +sen markey +salta ire +sair at +sad dar +sa sebo +ro an +rene es +real ricky +ps ons +ppc chat +po red +plent yoffish +people analytics +pensacola beach +party list +pad er +p ère +outandabout scotland +ornam entals +orn dorff +oracle database +or nately +oldschool rs +occlu ded +novo a +nine wells +nic astro +new spring +multic ore +more land +mo azzam +mescal ine +mell anox +maw lana +mart lets +malcolmn ance +mag as +lp tnhs +loh mann +lo were +ligh the +lat ins +lar usso +lam ont +laf rica +ko g +kf z +jay cutler +j klf +its complicated +inv ader +ingh ands +ing ate +in sets +in day +iam writing +hungar ian +homelo ans +hel lebores +had nt +great deals +gouver neur +glade water +ge su +galar za +fc family +extrapol ate +enic ole +el ver +d family +crypto twitter +content creator +cole moore +clear brook +chol lo +chatt ooga +ch ry +celebr ities +ce sme +catastroph ically +car ducci +cann an +cadi eux +c cia +c bos +bur atai +builda bear +bo fam +bn ha +bird wat +bio available +betten hausen +barbar ous +avogad ro +ashi elds +ap tx +ap lace +amphi bi +alli reland +ain scough +_ ... +?? .. +ðŁĵ° âŀ¡ï¸ı +ðŁİĨ ðŁİĨ +ðŁĮ² ðŁĮ² +رÙĬØ§Ø ¶ +аР´ +zad ok +z ado +young band +y ilan +wunder land +white boy +wheel an +wend ling +wade yar +wa yofthe +vo ise +vidy ar +ven era +usaid gh +u shanka +tropon in +the stra +the english +the boris +tatt va +ste agall +star sailor +sou tient +smur ray +signofthe times +shi ka +schaff ner +scani agroup +sand boxes +rough shod +romantic ized +romanceno vel +rochester rhinos +ro mul +rit ter +rebeccam inkoff +ran fur +pop choiceawards +po boy +pipe fitter +per aza +pe mp +ount down +ok aku +o ecs +o budu +nun aw +nunaw ading +nosy crow +mu mmer +monarch ies +min oring +michael jai +michaeljai white +mccar ney +march break +lu oyang +loc alist +les se +ki din +kal aw +jan ek +iwas aki +ile ostomy +i ally +honor club +high performance +hd w +hat v +hahahaha hahha +gy or +green idge +green backs +grant kirkhope +gra p +glass ell +gla ize +gied royc +get outthere +gema yel +gat z +fun niest +free access +freddie highmore +fisch ler +epas cott +epascott pruitt +ep yc +en tearth +easy recipe +dur kee +drogheda united +doc x +dis u +de mars +david zwirner +david spade +dat os +dalhou sieu +cv v +craigella chie +cosmo logist +congreg ated +ci gano +chal mer +bu ste +brew crew +bli xt +blat z +bal aban +b itta +auto somal +au ghters +ataste of +asi es +art yom +ark itek +antoni ob +adv ant +ab antu +aa aw +, ? +ðŁıĩ ðŁıĩ +ðŁįĢðŁįĢ ðŁįĢðŁįĢ +ðŁĩºðŁĩ¸ ðŁĩ¨ðŁĩ¦ +íĭ´ íĥij +ç© º +âĺºï¸ı ðŁijĮ +à¸Ļะ à¸Ħะ +мÑĥзÑĭ ка +zag ora +ye esh +y ce +wrink leintime +woman iser +western union +wee tin +weare ready +vou ches +volu mab +vikram vedha +velá zquez +u ark +tr v +toshi ko +tole dano +to wey +to de +tie de +thin ners +sylve on +studio canal +statecapture inquiry +sso good +soufri ere +snoo per +sj v +site map +shilpaf am +sharealittle sun +se hs +scic luna +sche ide +salem wgna +rugged ness +rub ins +ri ppers +re flation +rav ina +rat ana +raf finity +radi als +que tte +pro om +pregn ant +pra sh +pep so +pen pal +par da +pam m +paide ia +or it +one goro +olvi do +numan cia +ni volumab +neversay never +n dez +muvy z +music bank +mu gu +mr scot +mor and +mon than +mollusc monday +mic e +merchi ston +mati sts +marvel legends +mar oun +mahil acongress +mache tes +mac omb +ma den +ma dai +lieu tenant +lauri emit +kerber os +kan ungo +jawa an +it ng +in sar +ima izumi +ide es +iam john +hpp lay +han ous +hal us +gu ler +grounds keepers +gour cuff +gay men +gam ali +gall and +g ba +fujis awa +fireemblem threehouses +f sw +european art +esp aces +el eri +egg ar +edmonton expo +e ka +dy ersville +dor rie +dong wan +dewal ttough +dev ac +deter rents +del one +de so +dale m +d group +czecho slovak +cw the +customer journey +crystal bridges +convivi ality +classicrock mag +change iscoming +can ice +cal is +bu res +bru cie +bro din +bren an +bota iku +be fearless +ban j +baking championship +avon more +asi mmons +as mith +appenz ell +apo plectic +ao y +anticoagul ant +anti thetical +anne frank +am ined +alu cas +\ " +! ðŁijij +ðŁĻ İ +ðŁİ¸ ðŁİ¸ +ðŁįİ ðŁįİ +ðŁĮ ¦ +âĸ ³ +young love +winn dixie +wf es +wend ler +vinayavidheyar ama +ug p +tru glio +ti elemans +thorough fares +the os +the magnificent +tal ar +syna esthesia +sv cc +sul zer +su enos +stop ed +sof ya +shim amura +sebastian bach +sab riel +sa chem +s itta +rs gb +rosleh tinen +ris don +rew ery +re activating +quint us +poppy pride +plumb ing +pend leton +park slope +or tner +op ress +of nature +nm true +nirav modi +nil da +nhv natural +new mr +naz z +month sof +mis interpreting +mil sim +mer cola +medve sc +mat tern +man mademo +manmademo on +mac l +m sw +m kh +ly l +long muir +lion babe +label mates +la urus +l nh +l ne +kin gham +ke saath +kan ojo +kal ay +john john +joanne wheatley +jay inslee +iv ry +invali dation +ingel heim +inge vent +imperial nhs +ij ff +ig worldclub +ia ia +huac achina +histop athology +heckmond wike +harness racing +hack worth +ha skin +government za +goli ath +gilmore girls +gh ome +gee ta +gav ino +gang an +fredd ys +fad hil +est ren +eno chian +education ally +eb ke +e ather +dex ters +dev as +de wolfe +comp ter +cdn paralympics +callo whill +cal thorpe +boo ps +blaupun kt +be sola +bathspa uni +bair ro +ay yub +auto detailing +aureli en +atre us +at eng +ar dini +angel ita +an zo +aham m +acry l +a aba +ðŁļ ª +ðŁĩ¬ âļ½âļ½âļ½âļ½ +ðŁĩ¬âļ½âļ½âļ½âļ½ ðŁĩ±: +رÙĬ ا +zu i +zor ah +you ri +yazidi genocide +wis c +win ed +wc pss +vla da +visit southdevon +veli yi +ve irs +uu ya +tir tha +tir rell +tim horton +than asis +tar mac +sub versi +stopthe hate +so po +sl h +sch aper +satur ating +sand worm +rotten broadway +riden our +retren chment +ren ick +reed hastings +ra chana +poor am +phonec ases +phic hit +or deals +om git +oath breaker +new caledonia +ne mes +nbaallstar to +multi story +mi rab +manfred weber +mal ama +lon gy +llan fair +liver nois +lettu ce +leon is +leaf ly +lanc spolice +kor on +kon di +kn acks +kin man +keepit wild +kas er +kali sto +iz har +horse fly +helsinki airport +hand tools +hal an +gru be +gor oyals +gobi ge +game changer +fr yers +ford son +fle voland +exp ound +european parliament +essi e +el anor +do jima +diamon dd +dave berry +dab olu +cw janethevirgin +cv hs +csu football +cour tice +com hghair +ciut adella +christian keyes +c puc +bra gger +br ame +bel tz +bad ai +autom ob +auto club +auror ahu +ate st +as kl +as aro +ar sle +aphrodisi acs +anti pathy +an field +amy dumas +am atu +alou ette +aller dale +al inda +accen ting +ac tc +a wit +! ?! +ðŁįŃ ðŁį¬ +ãĥŀ ãĤ¯ãĥŃ +âĿ¤ ðŁĺįðŁĺĺ +zen der +young stars +yat ton +yar on +x tian +world balletday +wildcat nation +weir ded +w kar +vu v +ving e +un fixed +um el +tugger ah +tuck well +tu mk +traffic jam +tho ta +the ph +testic ular +tel t +teach meet +super family +summer party +staf fa +sp ouse +smashing pumpkins +sil bert +shad rach +semi otic +sa whorse +sa pps +rust ler +rudi mental +rosen berger +roots music +retr or +regi sta +re seeding +re buff +r dium +quir is +pron ti +politi ken +pol hill +phil bert +pa sion +orson welles +organ elle +om un +old photo +ojib wa +oberst dorf +nws norman +nikon nofilter +nike soccer +new hair +neonicotin oid +natu ur +nates ilver +mul downey +mousekete er +mock asin +mmm gorgeous +mid wales +mid sole +liver poole +lion sxii +len ham +lady hawke +la vers +l hw +kuchi pudi +ko les +kir stend +kho dor +kam asi +kaappa an +k rell +ju ca +jhalakon colors +itali aricci +international museumday +inte zaar +inexor ably +industri alised +igo ta +ich art +icef all +ic ol +hyper real +humidi fication +ho jicha +hin aholics +hel ia +he pi +hat ari +gun stock +grin spoon +gov za +gl ancy +for far +floren tin +fish uk +fish bourne +exorc ised +er ris +epping forest +eco c +dyne ema +double fine +dine tte +dine out +die gol +di blasio +design build +dermat ological +del age +dece dent +de el +data quality +coy f +combu sti +co sas +cl ts +cl sa +ci fuentes +chillax in +ceelo green +car ve +car milla +capital and +bun te +brook man +brompton bicycle +boul ting +bir twistle +bid co +bech stein +bal doyle +bachill erato +bab c +bab ago +an sbach +allo dds +ahut chinson +ac ey +abh brows +ðŁ¤¦ ðŁı½âĢįâĻĤï¸ı +âŃIJï¸ı ⾨ +âĿ¤ï¸ı ðŁĺįðŁĺĺ +zom er +yoak um +wsw v +worl dday +wil ier +whig ham +wag len +waglen ikhil +vn dev +visit dorset +uofl baseball +tull amore +track master +tome asure +toge t +tip sand +tin u +timp anist +thisi su +the halls +the bar +sun glas +study english +sti x +standar ds +sta il +ssel f +sper rys +space week +sp nn +sig ne +si mca +si ani +sha dia +se ann +sare en +sap ir +sale straining +sal in +saha ja +row den +ron ning +rock aways +riky rickworld +regin apolice +refu se +rau fareg +raufareg besola +r st +qui enes +q ura +preemp ted +pil chard +paren ti +pad o +p tak +ot int +ol phin +ohl sson +oce t +noy noy +nipi gon +newton abbot +mu kt +ment one +manzoor pashteen +madel a +m sco +lo ing +limb less +lethaw aii +lethawaii happen +lab as +knur led +kn agar +kel ing +kal u +k iti +k illian +just sayno +jel gava +jeansfor genes +itsm illertime +itsd anny +im v +hi a +grin dle +gir r +g winn +flo p +fil iz +family holiday +fam elab +eye works +eri ley +en livening +ed ress +echo arena +e ens +drag queens +dock less +di meo +del grosso +deci mus +dau lat +dam ron +cudd ler +crê pe +crowne plaza +club houses +cer ci +carolin apanthers +ca vies +bsc sd +bru jer +bru ins +boss ert +bet ches +bel reddevils +be est +bar chetta +ban ov +at nagar +arthi story +arre stees +ar fon +app leeduchat +ag etv +af m +ach rome +ab de +a quest +ðŁĺĦ ðŁĺĬ +ðŁĵ ¿ +ãĤŃ ãĥ¥ +áµ Ī +འ¦ +zz st +zi ppor +xfinity racing +vishak ha +video gam +vid ant +ul kom +u coms +tu am +trans figured +total dhamaal +tom ies +to stan +thrill jockey +thistle tweet +thin kaboutit +tham ilton +tank less +sz w +syr crunch +swed bank +stem pel +skoll wf +sis lands +sigh thill +side saddle +sidd hi +sick ens +si y +shec an +sh pong +seventeen mag +sen te +san iga +sabar imal +rosam ond +rescueme ohio +red star +re cher +ration alizing +radio carbon +portland me +pha e +ot directo +ore o +oo bi +on des +olo ber +olafu reliasson +no wata +no confidence +ner va +nau ert +natali a +mysti k +myo dd +mun desley +moz art +mor p +misanthro pic +mining indaba +micro economics +meat wad +me ggan +mari et +man singh +main tainable +maersk line +luc chesi +lou vered +lo ken +lightsfor liberty +korch noi +ko tal +keepit up +ke ating +justin colemoore +jere mi +j pd +j mpd +ing é +info blox +indiscre tions +ig ent +hol croft +hob house +ho tor +histor ica +ham macher +halla han +h inging +gun as +guj ju +guillemb al +guillembal ague +gi aro +gg day +foxwood sct +ew ski +ew adi +enter al +energy fc +en yo +emprende dores +eil idh +early momentsmatter +dun murry +din di +di mos +derby dell +deplo res +deon dre +crossmag len +comic fest +color run +co cas +ci dent +chun ni +christiany elich +categor isation +carrick macross +bur dine +blu stein +blaster jaxx +big ley +bernar deschi +ben enden +beau bourg +be ssel +bar camp +bar ay +ban ded +ba shir +asse tte +asam blea +ary ab +arenaof valor +architec t +ar matrading +alphabet success +ah ma +aeru ginosa +... +) ": +ó¾ Į +ðŁĺĬðŁĺĬ ðŁĺĬ +ðŁĺĤ âĺºï¸ı +ðŁĸ¤ ðŁĴĽ +ðŁij¨ðŁı»âĢį ðŁį³ +ðŁĮ¸ ðŁĮ¼ +ó k +whereare they +wed eliver +war ily +w ame +visit italy +village voice +vasilev skiy +vanguard ngr +van aqua +ur on +univers ite +uni westminster +twee tt +tt ly +trung pa +tr ino +to power +ti ae +te ana +taver ham +tam ora +tal ese +ta ps +sweet waterbrew +stjohn ssmith +steph ane +stay wild +stac ys +sou kup +snap artcrew +skerry vore +shud dering +shap ira +ser vir +secre twars +se shadri +scott j +rsh yr +row blanchard +pro sport +pho tometry +per center +pas sos +pancas ila +pag ibig +pad mal +ore illes +odon to +nottooyoung torun +nom in +nikol aj +ni eri +nau shad +mun ghana +mu tsu +mq vist +moto cycle +min kus +min do +marri ot +made ine +lon gla +lip sey +light sc +li skova +letsgo tothe +leck hampton +lane way +lacer ated +ky ros +kn oop +ke lem +k hem +k ando +juri sts +jo i +jagu war +ix elles +it sky +it mo +islam isation +ir ud +ing ale +ine very +in black +imman ent +hu se +hero e +hender shot +he tti +hart shorn +ham dy +hadd adi +ha glund +gustav son +gray wolf +global foundries +gil merton +gener alists +flat water +field stud +felic itas +fasti dious +far ney +f music +exit poll +ew ert +espou sed +elf yn +eki den +ed henry +ec ri +eas me +dynam obim +dwar ven +don nal +do therightthing +died onthisday +dfa records +del boy +dare you +da aay +cyber risk +cow drey +cost as +comm eng +cocac ola +co git +classi fier +cla dy +christopher lee +cheri moya +cap en +c py +but first +br bird +bodh ran +bin yamin +bestteam ever +benck iser +be thri +bang ar +bad ler +ayeshat akia +as com +ap ag +ama waterways +ali via +al za +ag grey +abo dy +abdomin oplasty +# âĿ¤ +! âľĮï¸ı +ðŁĽ © +ðŁĺĤ ðŁĴŀ +ðŁijĢ ⾨ +âĿ ĭ +à© Ń +z ns +wer king +wedding anniversary +wealth tech +water way +ve wy +tv f +tru cke +tropo sphere +trees for +train in +todd barry +tit os +tic ky +thisi sirish +thi o +the hype +ten niel +team alex +table mountain +swi mathon +sun studio +sun dials +sound wave +sj v +sir in +sgo p +sco v +saras wati +salman khan +saar aa +ryand un +run g +ri xton +ren ren +rat es +ran ken +r se +q wq +q sr +presta shop +pre ety +pow les +podi atric +pic her +phosphor ous +phon es +pet unias +pear ling +patoran kingfire +pac c +octa ves +oath keepers +novem bro +north wind +norah jones +ne zha +ne elix +nat ron +muse umb +miamid ade +mediamar kt +mat tum +mar acan +mait ri +mah end +love itor +lisalam panelli +les band +lass ico +ku ehl +kk t +khand a +kent wildlife +ji brin +jeopar dized +is ay +instap assport +independent artist +im bb +il ma +ian jamespoulter +i wk +hy rum +ho quiam +he ff +gor am +gon orth +gb p +game ready +gal erie +future world +fortune mpw +flu ent +fire brigade +fac king +euro athletics +easter holidays +disfigu rement +dhe ena +der ci +ded chat +dad asaheb +d kp +coulro phobia +cot tes +coach p +coach k +cl ouser +cine mas +cher ui +charity week +cha hine +bi gear +beverly wilshire +bethany hamilton +bent en +bath on +ballachu lish +bal tica +asp net +as kem +aperfect circle +amreading romance +ah is +adap tions +a beautiful +% !!! +ðŁĺĬ ðŁijįðŁı» +ðŁĮł ðŁĮł +ë°ķìĭł íĺľ +å ¡ +චº +zu cco +zan jeer +x ango +wowo wow +wh ill +wal de +villa gelife +ustad h +un screwed +try hard +tro ms +torrance coombs +thi ra +the whole +t sy +sur facep +supere xcited +studio tour +stop it +sil vic +si bi +shakespeare lives +self development +sare not +ryan bingham +rossi ya +road sides +right s +reboun ders +re posts +queen spark +pur se +pro gess +pres rajapaksa +postpon ements +pinchpunch post +per ie +pel in +pe ther +parti als +parach ute +oro adshow +organic beauty +op hora +ontari on +nuclear power +nj d +ni aaa +next bigthing +net news +nb football +nan se +na ach +my car +mo ppet +mo gador +mira belli +mil ah +michel barnier +men teri +men otti +marun ouchi +margar ito +m ram +luke combs +luci en +lu tton +lock ley +live export +li sandro +li bert +lgb trights +le toy +laver stoke +last call +lake ofthe +lac alle +kuno ichi +klin gh +king lake +kh s +kar pinski +jo sy +jamest cobbler +irish tv +ieb ckenya +i ast +husey in +hop god +hom ophone +ha ake +gy ra +gy aan +grit tier +great deal +got tes +gor st +glad ney +fv k +fu do +fit ty +fish market +expo sure +eval ue +esken azi +em eline +el berg +e mia +dri essen +don official +don kis +disin cen +di gha +di gest +dac re +cup sleeve +costar ic +church planting +chriss ununu +chelse al +cann ings +can opus +caloo sa +bü nd +brian j +br oun +best sho +bare tta +bar tos +ba ee +b sr +ap rn +am prog +al atina +ak shi +aber dyfi +ðŁij©âĢį âļķï¸ı +âĻ¨ ï¸ı +âĺºï¸ı ðŁĴĹ +ا٠ģ +ÑĪ моР+ÑĪмоР± +ÑĦле ÑĪмоб +ç ay +yor dan +y aqui +with love +westfield london +welove music +water bottle +wanted wednesday +vi zier +vi b +v rr +uttarak hand +un realistically +tur p +town sley +tor ne +top oulos +tolu ene +that searth +tam pin +table topr +sv end +stoparming israel +ste pin +stars bbl +so cap +sin city +sean pertwee +se young +sat cher +ry nn +rudy mancuso +ru pe +ray man +queen radio +publicis groupe +pop concerts +poo tie +pix ography +penny royal +pen knife +peer support +par sh +or ts +om eric +nw schicago +nic ulescu +mycen aean +mor do +ment z +meet meat +marshall ampsuk +margin alize +mana fest +man nes +loc avore +liver si +liver sedge +lan zhou +la bette +kor f +ko smo +ko hi +kiraz mevsimi +ki eu +khaled azia +key and +kevino lear +kal orama +jume irah +jon jon +jennal dewan +iba secretariat +hois ington +hermi esadler +h bos +h bc +gun ite +ground less +greta vanfleet +ger th +ge mas +galla gh +futureof mining +fur u +funke akindele +fra ss +farmer sday +f ptp +ezekiel mutua +ex ped +en ner +em tech +dy le +dy c +du gongs +dru gre +dream land +doub table +con sett +co ches +cn at +chri smar +chir la +char co +c sis +buk hara +bhaagam ath +ber nice +bed stead +bana gher +baj rang +bah rami +aust mus +atri be +astre a +ard in +ann unci +an v +ami z +alter nation +alis sa +alex y +af dn +abo yne +- :) +ðŁĺ»ðŁĺ» ðŁĺ»ðŁĺ» +ðŁĺ³ðŁĺ³ ðŁĺ³ðŁĺ³ +ðŁij¼ ðŁij¼ +ðŁĮºðŁĮº ðŁĮº +ðŁĩ ½ +âĿ¤ï¸ı ðŁij¯ +zam alek +wren ched +wra ys +war precords +valken swaard +up stream +um ri +tuk ur +to fficiel +thu rai +thor selfies +th march +terry gilliam +taylor gang +survivor au +stree twalker +stacy london +spur n +sprad lin +slv ban +sky raider +skipthe dishes +sig sauer +shaw kan +sec schoolinnigeria +sar am +sap nextgen +sad er +s folly +rune quest +rumple stiltskin +rick hoffman +redd ine +reclin ers +re gaz +re collecting +raw vegan +ra ww +ra se +poster art +po partist +pluto tv +pitti uomo +pi eve +ph ou +peng u +pand ajay +pal ert +oz ma +om ino +ok olona +official b +nicolo di +nh lan +neu ss +nba xmas +nationalpark week +msc cruise +montre aler +model town +mens conference +luci ani +lizar do +le ston +l vp +ku stom +kha w +kele mentary +ke du +kanhai yakumar +kan ellis +jiu jitsu +jimmy sheirgill +jay mi +itsme deaner +isthmian league +inge borg +ing rat +ine en +ima gens +ily as +hiwas see +hi mi +haus of +guit are +gues thouses +goodnigh tt +good loe +go gos +glycae mia +gi shu +gardner md +gan ji +fur rer +freec ell +fre twell +fe spa +f lection +exoluxion in +exasper ating +ew ington +eni um +dy al +diver ticul +distric theating +desen ho +demon te +daily life +cron aca +cousine au +con nan +cl nolan +car ral +c jk +brew pubs +braintumour org +bra cha +bom bast +ber ling +bella houston +atb financial +art price +are well +ag ib +ae gis +a you +ðŁĺĬ âľĮ +ðŁıĥ ðŁı½ +á´ ¼ +à® ī +Ø· ÙĦ +zzzz zzzz +xen omorphs +wu zhen +women ofthe +walk üre +vitamin b +v no +under nutrition +un popularity +tri fling +tre bbi +the film +tch as +tab er +suther lin +su co +stock still +sp ams +south pole +sor ti +slau ghters +ske ma +side chain +se shat +scam orza +sam pat +sab har +rival s +religi onof +rebeli on +reaper mini +reading agency +r cd +projekt red +pher i +perip ate +peculi arly +over stating +nyle dimarco +naval history +mur shid +monochrome photography +mik maq +mew seum +meetthe maker +may nil +mar indu +m clay +kuy kendall +ku ber +ku bam +kro gh +klu te +kit trell +kin tail +kan at +jungfrau joch +jd ff +jacquelinem jos +i ster +i pi +hun g +holly r +holiday giftguide +ho stin +hisp ana +hi awassee +harak iri +gue sser +gett n +gam brell +gal va +fol an +faze ley +faneuil hall +family reunion +ex pe +endear ingly +ec su +dy skine +dramatur g +dine ren +dent ition +del py +del ing +defac ement +dd k +dan sk +d hall +d alli +customer care +crudit és +cre x +cod ger +cas sells +bush kill +bre ady +bon jour +blogpaw schat +ble eder +bistro t +bel staff +beg ins +baltus rol +back k +authori zations +andre rieu +alici af +ain ley +admon ished +absolut ly +: ^ +ðŁıĨðŁıĨðŁıĨðŁıĨ ðŁıĨðŁıĨ +ðŁİµ ðŁİµðŁİµ +îĢ ij +æķ ij +ä» £ +your love +yorkshire terrier +yon as +x g +wre athed +womensmar chon +wart burg +volks fest +uofl wbb +un ama +tin ari +tessell ate +tanisha amukerji +ta zo +sugarray leonard +stur geon +sto u +stigmati zing +stay safe +squ amish +sle a +signal man +shi bani +shav asana +serv ando +sakshim alik +rp motorsports +roman die +robert mondavi +robby gordon +rhyth mand +rest day +repro health +realmichael kay +re ineke +rads one +ra ymon +r br +quiz night +qu illiam +pun ks +pul lovers +publicenemy ftp +prophetsof rage +project lit +ple yel +pil ates +pi kin +philip glass +perkin elmer +per rott +paphi ope +pa ay +ov na +orlando shooting +orlando bloom +oral care +oo j +on n +ol fe +ol dis +occup yoakland +obje tivo +nag l +my g +mul lett +mu hl +movie stars +montan er +mir pur +mi bf +meteo gib +mayored lee +masc aren +maim ing +madr yn +longu eville +llan wern +liv miami +lein en +lanes borough +k vp +judy blume +juanman santos +jo wa +j tweets +j ce +isi zulu +innov azione +indi afirst +im lay +ich on +hor an +here andnow +guar aldi +gu len +green flash +gra gg +goldie hawn +god scountry +go jay +gal ley +fromthe groundup +forestry commeng +flor ine +fan wars +exmoor np +ek ar +ecw press +dro ite +dit mas +diss enter +disney nature +conceptu alised +con volution +coachella valley +co ber +civil service +ci j +chefou cauld +center fielder +catoc tin +ca official +brendan cole +bren ta +boat swain +blan ching +biz talk +ber ic +beauty world +bbc facup +auden cia +as ahutchinson +arav alli +amor pho +alcohol ic +ak int +academic swith +< =- +éĺ² å¼¾ +éĺ²å¼¾ å°ijå¹´ +âĦĥ , +à° ¬ +zab aglione +wind row +welcome week +wb kb +voc i +vision aire +uni da +ua em +typhoon display +trow dy +traveler schamp +thesavoy london +thenation aluae +sy scoin +su v +su tah +stanlee comikaze +sky science +sjo erd +silk men +sco c +sam su +sam eth +ru dan +roo sts +rela ying +red volution +re paints +quad cities +pu pillage +pro challenge +power books +pear n +palm desert +oro ck +oran allo +olic ity +o gra +no chill +nicolodi daria +nico ya +multi show +mu ter +mo ws +mi jares +matthewk heafy +massac ring +mac art +lunch boxes +lu ber +little brown +la berge +l alande +ker jak +kaz ak +kas u +kad ai +k van +k brown +jö nk +jm gardnermd +jerry lentz +jas df +jad zia +insi dere +ing gris +infinity thegame +in bath +i yan +hu kum +hoe hn +hirez studios +hi jazi +hi ers +hermo sas +hel u +greek mythology +gravity rush +gor in +go wd +global runningday +ghan af +genie francis +gam brin +ful co +fu tch +fu king +frog let +fm qs +fest nyc +factsabout me +et xwx +en h +earth work +dy sm +dra a +displac ements +dish man +dest america +cush wake +clay aiken +ci ii +cen as +cast d +bomba dil +bog ast +blin dd +blau velt +birch mount +bing su +best coversong +beren saat +beat airpollution +az nar +au trey +attenti onal +arti equitter +anton du +amlw ch +allsaint sday +al ake +agri ppina +ðŁĺĮ ðŁĺĮ +ðŁķº ðŁı¾ +âĻ¡ # +âĪ Ĥ +Ð ¼ +wy vern +wol ken +wal esc +v ult +uper girl +union budget +un dar +ultimate warrior +tumk ur +tu ska +tro c +top story +to home +ti gno +the interview +temple stowe +tehel ka +tech ne +team kitty +tan za +syfy tv +summa recon +succul ent +stra ying +stjohnssmith sq +stasi areport +sr ms +sportsp ersons +spor te +sor an +son ate +sli mes +seu mas +ser ow +scal zo +saskat oon +sarah drew +sanjay leel +ring rose +rest ing +rehome hour +quarre ling +qc times +pontard dulais +pla sse +pharmaco therapy +pen ya +peep al +pat ou +ostro wski +opier adio +o thering +o gers +o cam +nigh tie +nicom ir +nasa history +minim alist +med len +me men +marseilla ise +magnu mpi +maf fia +lu tt +love her +ll vm +liter atur +lakshmi manchu +lady bay +kling berg +keral ites +kauff mann +ka he +jane mari +jag jaguwar +it amil +inter nist +id alp +iam d +hustle hard +hone gger +ho ws +healthcare forall +har leen +hahaha ah +gye ong +ground hogs +green keeping +gor uck +fu sible +frank caliendo +final ride +fertil isation +fer mata +fen burg +fau zi +fang amer +falli ble +ent group +ei u +dogs rock +disappro vingly +deton ating +cub ator +cor mega +college hockey +coal ash +cleveland browns +char g +cham ak +can las +caer au +cad ca +buck enham +bra ska +ber beris +bel ville +bbcwales news +bac one +ba im +b vov +b dj +aç ores +ato logical +ar qi +ambassad orship +ale storm +^ ^^ +ŀ ĺ +ðŁĺħ # +ðŁĵ» @ +ìł ģ +ë°ķë³´ê² Ģ +yetto come +year sin +xehan ort +wy f +who les +welling united +web chat +wal let +ves alius +ven oms +ve dre +vas ili +vall ance +usman ov +ur gence +u mble +ty co +transfor mable +toron tonow +tmobile careers +ti jn +thrombo cyto +thin h +thewomen stour +ther mae +the tank +techno phobe +teameric sson +summer break +subju gate +stone fc +spo ker +shard london +see torontonow +se tembro +se gh +scrap books +scoo ts +sci kit +sch or +sad ler +rhe ingau +rgv wx +rem hq +redletter media +red together +reci ous +real john +raz ine +rahe en +qpr v +pu reed +psla fc +pon tes +peri shes +pel aez +pal erts +oiland gas +ofthe seas +office max +modern baseball +meth ylene +matriarch s +masse ffect +mal com +makin wa +lough gall +lon gg +lister iosis +libr arie +lean ing +lazi o +lam u +lal oo +l by +ko zo +king ussie +kil rush +ken ward +ken ard +kazimi erz +joey lawrence +jasmin evillegas +jac key +it ree +iron age +incis or +inar team +hr ys +hopen othate +hog sett +he ili +hal let +gid do +ger vasi +ge sf +gall man +gall inu +for abetter +fi ven +f bre +exp ounding +es an +er vice +en sa +en ingly +emmy awards +elms ford +elg ouna +el ya +e dul +dk v +die ren +dee waan +ct in +con ecu +comor bidity +college signingday +cli j +cd projektred +carr fire +ca jam +business coach +bu be +bruck heimer +bob sledding +bhag wati +bakhti ari +bailli eu +bag gott +bab il +ato ken +ann ers +amit sadh +agar uda +ad w +ðŁĩ °: +çľ Į +ç« Ļ +ãĥĩ ãĤ¶ +âĺĥï¸ı âĿĦï¸ı +à« ģ +zz aro +yu rika +ye asts +women sashes +wo to +wmp dogs +wc tv +wall on +ur dan +upcoming releases +under lings +tyntes field +toronto symphony +to pen +the challengecup +sya oran +svan doorne +struc turalism +stam ens +sta ip +st b +spit ball +sl ama +sim simi +sim plic +sid he +sex trafficking +sa hai +rupauls dragcon +run ralphi +runralphi erun +rip ens +repar tee +rep co +red for +re percussion +rack ley +pu ren +prin tvilla +pever il +pen edes +pas k +pal au +orlando sentinel +one ocean +off cl +nom i +nin omiya +nh p +nant asket +na pel +mun ga +mubar akan +mu is +moving imagen +movingimagen yc +mor goth +mopar chat +mindbody spirit +mes illa +maur oranallo +marvel led +managed services +man sor +male ev +lu le +lok manya +lin ny +laver ock +ki sta +kan ta +jw marriott +jo ab +ji had +it ok +isi s +ish ti +iam lp +hot lines +holling shead +holden graber +high gate +hel wig +heil ong +gu al +go tr +gel denhuys +ge hrke +g gu +fun gi +fu raffinity +fron ing +f ended +europeana eu +esken azi +ep cs +en ciso +emul lin +e zo +dor oro +di visi +decon gest +daph nia +cu yam +cu sk +cook out +con dom +cambo dge +bri ard +board shorts +az riel +au teurs +attach ment +ar vel +am pradio +am fam +al cor +aal smeer +ðŁķ¸ ï¸ı +ðŁĴħ ðŁĴħ +ðŁ¤ŀ ðŁı½ +ãĢĤ ãĢĤ +à¸Ļ à¸Ļ +Ë Ļ +é d +zag li +y ns +y appy +wr g +wf council +war ming +ver den +veganrecipe hour +trevi thick +tough ening +tou rettes +toni storm +to ome +the boat +tay som +swag at +sur inder +sub divided +still water +steve tignor +st vincent +snow flake +smart water +show mance +sho lo +scratch ings +sal divar +riev aulx +rei des +ray uki +rache le +propo fol +proc tologist +pown all +pha ilin +pew ds +pens wordbooks +pe cora +pan telleria +orange wood +ora for +onlin enews +odon ata +o ires +now open +novi grad +nose worthy +nois y +nicomir allegro +netapp insight +nat c +ms ar +mro dofficial +mo cambo +megal opolis +mc mansion +marci a +mang ling +man marzi +making of +made tomeasure +leop ardi +knott ingley +kishore kumar +kinky bootsuk +king sley +kin tore +khu ra +kevin max +kaz eem +k haw +ja aye +infectious diseases +in with +ilovelu cy +horror family +ho stal +hh mi +h bbtv +gor acing +golden boy +garret son +gamer life +gal asso +fushi gi +fu tah +fre dol +form als +food safety +fi rish +fair phone +es am +em trains +el ver +ed app +di anat +dhe ere +de sul +da pest +corrobor ate +corri gan +color me +civ milair +cine timee +ch ich +ces spit +c andre +bro dies +bre g +bha in +barry hearn +babyled weaning +ba el +aw ai +athan s +arnau d +ar amos +aq il +appor tion +anc illa +allu sion +agu do +after burners +abc newsbrisbane +aar c +ðŁķ ĸ +å¼ µ +yne kev +yadav tejashwi +y abby +whim sically +wh ou +we sel +wayne sermon +war speaks +wander n +uru an +under dark +u uponline +u od +tze dek +ts z +trin itarian +tri fon +too ke +todays special +timp f +ti so +thing swe +theo cratic +the unit +sun news +stra thy +sto x +spi er +sho liday +ser very +sc dc +sayye sto +save ur +sa ko +rr v +rouge mont +regul arization +reas sert +re dron +ram nagar +py mat +pru den +pil fering +pan zero +pad mash +opent ennis +nys fair +niy kee +nightmare beforechristmas +nicoleand bri +n ber +mrscot teddy +mosthaun ted +mm al +michi okaku +men ti +mama hal +maha sabha +lyt chett +lov ley +lo ade +lie zel +kin zer +jah nke +jaboo wins +j emmy +ital ymfa +it shappening +is aw +instru mented +ilean adcruz +hur stresort +highlin enyc +hang nail +gor din +gon oodle +go dragons +georgi ev +garden birdwatch +gar diners +game board +g elli +forre stal +five point +fit es +ff be +fat ta +fa zak +endemolshine ind +embarrass ments +eli ud +ea se +dw ade +dump site +dou ga +donal dd +dj uri +debru ynekev +cure ton +cuff ley +crack heads +cra is +cov ell +cor c +copp ock +can de +c agon +burn outs +brü hl +brun twood +blon d +beer lover +bber ing +bb coxford +bassi sts +barone zaza +bar chester +bag al +bab b +azte cempire +ausout backnt +auction at +ate f +as rar +arkhu hro +arab ella +aldub you +:- \ +/ .\ +ðŁĺĦ ðŁĺĺ +ðŁĩºðŁĩ¸ ðŁĻı +éĺ²å¼¾å°ijå¹´ åĽ£ +à¸ķ ร + ¥ +zoo cbs +z all +yuvas ena +waz ed +ve idt +val ery +vad undee +us asa +tz ed +tt ourism +timel ine +tiger n +th inf +tetra drachm +syno vial +summer camp +stro heim +squir rela +sin ked +shar ada +samu ele +saan ich +rio ted +ri pro +queens berry +psycho somatic +pre treatment +pr il +pla isance +pickup shadowhunters +over stretched +ov ni +ostr ac +ok orafor +oc z +nikit in +neutro phils +nau seam +monic as +mis on +mini stre +mercat us +md x +mal usi +mad hura +ma shan +lu iss +live sound +li iiii +lau v +kil ledit +kap ler +ilove wpb +ibu solih +i wb +hull city +hayward sheath +ha bano +guthri e +ground nuts +green all +god father +gobier no +giar dia +gand hara +gag liano +fr ings +fox field +ff u +feel in +fe compendium +fau teuil +every little +ethno botany +et zion +esp alier +end sars +dum dum +drif twood +disin her +disch ord +digital globe +digi bytecoin +dethr oning +de mur +day job +das sler +dark wood +dan forth +commercial ise +com us +colon nades +chiff re +chem ung +cbc falc +cap ut +cap ucine +can tus +caix inha +bikini body +bha bie +batt y +av aya +ar bat +apoloo hno +anesthe tist +am bati +alo ves +aker shus +ait ken +afan art +abay omi +... ðŁĴķ +! ðŁĮ¹ +ðŁĩ¦ ðŁĩª +éĸ ¢ +ا٠ĥ +z radio +yz ors +wra yzors +wr hs +wp sproud +what more +wer x +we p +vish wan +uw madison +un suspected +tri fari +todd dammit +tegr ity +tee shirt +sx s +swear ingen +sugar foot +str ouse +sof steel +shy glizzy +scu stom +sb se +sant acon +saint field +sahy adri +saaraa alto +row se +rod taylor +ren tin +re formatted +publ ick +pro enzas +proenzas chouler +pon gal +polaro id +pic tou +pex els +peop l +pay per +parthasar athy +outre mont +or cl +optic h +ol hos +oc ket +nes j +n cats +my favourite +musk aan +montal to +mon forte +mon ami +mom bas +mi shi +me on +married life +man madhudu +lunch box +lovefor sam +long port +licht steiner +lawandorder svu +later als +kunst museum +ku czynski +korn gold +k run +k ramp +jeopardi se +j ati +ip es +iop sych +io annou +integral yoga +ini ka +igen berg +ifc center +i ppy +i hrc +hynd land +har bert +great nature +gre engage +giug no +girls that +ghost sign +getex cited +ge henna +gast ineau +garden city +frankieco cozza +fr itch +fat was +far me +ey res +etri gan +eti da +ent on +en ak +eg w +e spirito +e rebor +e migr +dis orientated +derby museums +davide igenberg +countryfile live +corin thos +constric ting +co stest +classi st +cla rey +cherui yot +ch he +castate parks +cassandra sleee +c gt +bun gend +bott ineau +bor gne +blood good +bleed in +berg gren +baku go +av ely +at ang +ari fin +aquan aut +amph it +aldub for +alco ves +agha doe +agent ur +abric ation +abri stol +ableton live +ab yan +èģ´ãģĦ ãģ¦ãģĦ +âĢ¦ ... +Ùĩ ا +³´ ìĿ´ì¦Ī +zin chenko +you willbe +wv ua +white wood +vu yo +vitabio tics +v ont +v aren +u mina +tro eg +travel quote +tr yn +ti zzy +thrombo tic +thol yoke +the fight +tar vin +taj iri +syke sville +straight outta +stock brokers +stobart group +ster ility +sta h +sky sportnz +singlec ell +sidel ining +shou sing +shi jiaz +seon gnam +seanpatrick flanery +se sam +scream ers +sch moe +scar nival +saharare porters +rr u +ro sab +right stuf +rake shroshan +rake em +r yoga +pul let +publi us +pro finet +por at +pol vo +pic cini +phi delt +per icos +pau low +owen sound +north end +niykee heaton +newsar ama +new release +neutr alised +n co +n clc +move theneedle +mort lock +model railway +mo cho +mis sour +migra ine +mewseum monday +mati gnon +mat ula +mat tawa +man eesh +lu vu +kyo jin +kurtv ile +kne ec +kir kenes +ker rie +k pcb +jones music +jo ema +jer oen +jen lisa +j vs +j anni +ix ia +intelli j +inte xt +int acct +ing my +indic ud +ignit er +hor dern +hol ms +hin ching +harvey weinstein +h ff +groo k +green burg +great times +grad life +gopher hockey +gi galert +ge sucht +gael scoil +gab p +g me +g afc +fy inglife +fortn it +forfe iting +fol i +fo day +film life +fiel den +ff ert +empath y +ek or +ecre ek +e mond +dra vet +dje mba +dis qualifying +diam anti +cush wake +commo dore +com unity +chrissi efit +che ff +centrifu ges +cal vert +brief er +bridge fc +bran de +ber minat +benef ic +be ziers +bam berger +ba jan +azi one +ax elle +as cl +are gbe +arch stl +arapa ima +ar round +anyou neversee +ann ago +ank ole +am bula +allin with +ali on +aap ko +! ðŁİīðŁİī +ðŁĴķ ðŁij¯ +ðŁıĨ ðŁıĢ +âģ Ħ +Î ¾ +zoom car +ysle ta +yaz awa +yar wood +woo do +wkr c +white horse +whatilearned today +whati slife +wh dh +wat ain +vol quez +viol encia +un moving +un luckily +tra versed +tommy wiseau +tom asso +todddammit kerns +ti mate +the zoo +the vic +the amas +ten ews +tc w +tal bot +stan es +spast icity +sm soc +sla unch +si ang +shi pper +sheik hu +shar pless +sf m +schoon maker +sales manship +ry thm +rotar act +romu aldez +retail design +rescin ding +rcmp mb +ran sacking +q ic +psin sp +program matically +phone mic +pequ annock +pe a +pc game +paras auro +ous ley +one iric +of x +objec tivism +nz inga +nwa chukwu +neck pain +n aper +myodd balls +much hhh +mr h +moom oos +mob ilit +miro slava +millin ocket +middle grade +mel co +mcdon ogh +maroon dah +marit ima +long ine +liver adio +les bi +le me +le frak +lady boy +kat zman +jo da +jen is +j ss +itsuku shima +is ap +ili z +igh ty +identity theft +hiphopp antsula +hel ichry +healthcare it +han au +ham park +gu jar +gp cr +go gulls +gang war +gal low +fu rie +fbal lus +father son +ec assidy +e zzard +dur row +du vets +doub leg +dor na +ding a +dev aki +del homme +daga ati +corn ella +cinephile photo +chamber of +cam mack +bungend ore +bun o +bott band +blood money +bit d +bend ita +bar ah +av ad +aust ins +arvin dg +ar od +anti doping +ant ar +ali ster +al vie +ai ps +aerop onics +adidas fballus +\ ( +. âľĮ +ðŁİ¼ ðŁİ¤ +é Ĵ +yo gad +yel verton +wol pert +wld life +wi ggers +wat amu +waffle crew +vere em +thunder nation +ten sioning +te cla +te cha +tang ential +tan ke +tall ships +step wise +sor ong +sn d +smy lie +silicon hbo +sil vey +shu mmels +shan ter +seton hall +se ble +scar abs +scal ps +saumy atandon +sang ay +roysoc med +revolu tapp +relax er +relax ationday +rege x +readju sting +ra kel +r jc +qui res +publi shable +pleni potenti +piti fully +par takes +oy ler +over hyped +ou ise +osa ki +olober syko +ni bel +newed ition +mv ci +mu cker +mt ps +monte agle +mobi lebanking +mello phone +megab yte +manga studio +lover ugby +london npc +lit fest +lind blad +leff erts +le dgers +lc cc +lauren lapkus +lamo ille +lam bourne +kry ten +khodor kovsky +kal enjin +jo suke +jefferson town +jc zyk +ip man +interior decorating +instam oment +idhunamma aalu +i dontw +hun do +hether sett +hau ghnessy +ha ith +h iso +gwyn ne +gu ck +gra un +gra ub +gob bling +glenfidd ich +gi jón +gi bill +fri is +fl sen +fire tv +fe delin +fc ps +eu refre +eo incol +entomo logists +enni g +du th +du melo +drop zone +dining nc +depu y +de stry +de eded +danc o +couple ts +concu ssion +col chic +cl onic +chil ena +chat tel +char mian +can be +cafe press +bt spor +bren ner +brae side +bonnie and +bear mccreary +bang on +ba stow +ba die +av ta +anti fouling +amrap ali +ak ota +accessori zed +ac rid +ab big += ' +ðŁļ Ľ +ðŁij° ðŁı¼ +ðŁij©âĢį ðŁİĵ +ðŁİī ðŁĴĥ +ðŁĮ ĵ +âĿ¤ï¸ıðŁĺĬ ðŁİĤ +Ê ĺ +ye won +yak ko +wr m +worl wide +wor x +wi elder +water proofs +vivac c +vi rens +unequ aled +tl picks +tiger sfamily +the mc +tex eira +tellem jaye +te are +tanner foust +ta are +t enta +story map +stockhol ms +standardi se +st rock +speci a +ski pper +siem ens +se ai +sdg action +rone ttes +richard j +ri ata +pel meni +peking duk +pe ffer +pas sin +pab on +ot n +oo on +one gative +ny autoshow +nj enga +niki fyinglife +new i +new car +nb alive +nanow ire +mt fc +morbid elli +marqu ina +marindu que +man gwana +lyric belfast +luzh niki +lu sive +lid combe +lib man +li ban +leve ret +latch for +lan go +l spraggan +kel si +joshab bottband +jim sterling +janemari elynch +international kissingday +id wp +i yaz +hungry house +ho ppa +heb buli +hd ms +happy pride +grand teton +gr rm +gold box +gang i +game strong +gam i +g fg +fu ente +fen oli +fal sa +eye brow +erri ble +er hardt +encant ado +em slie +edu coach +ed itio +echof ox +drew seeley +dol lis +di ene +der ay +daw it +dan an +cryogen ically +cre o +cra bbers +corrobor ated +cooki ed +citrus bowl +che b +chander paul +cham plain +car forsale +canyon fire +caloosa hatchee +bumb ashi +bl undering +billie faiers +be intheknow +bbc cov +ba bes +b ason +ay er +autom arketing +auto crats +atal ji +arri vab +antoni osab +amo c +amerikk ka +am ax +albat rosses +al ha +ail i +ah wah +aga h +affil ate +abri el +ab ase +ab ak +ðŁļĻ ðŁĴ¨ +ðŁĶ¥ ðŁİ¶ +ðŁİµ ðŁİ¶ +ðŁįª ðŁįª +ðŁ¤Ļ ðŁ¤Ļ +ì² ľ +èĪ ŀ +ਠ¸ +yl va +wo chen +western ghats +wal kleys +viveko beroi +urban iak +ultra europe +tun nell +trail way +the mbi +the evening +texas monthly +super fici +su di +squ ill +south ayrshire +soft wares +sny man +smer ch +smallstreamers connect +sk rein +silver hill +sh andi +sen sen +sea power +sat anas +sare gama +sa ren +row ville +rosen zweig +rich gang +reserv as +red bulle +re mon +q ds +prag matics +pr ound +piece hall +persuasi ons +performance art +os mania +on paper +o ae +nor il +nfl sunday +na jar +mr joe +mn timberwolves +mm cr +mel by +meghan mccain +mc moran +max g +maup assant +marriage bootcamp +margare triver +mac eda +m sal +lieben berg +leys in +le dg +la ster +kis san +kar and +johny hendricks +ji ocare +jean loup +je ant +jacob s +isa beau +intersec ted +hrishi kesh +hockey town +ho ca +hin richs +her nameis +hel enab +heart and +har ra +han z +hal di +ha sk +gun sout +godzilla kingofthemonsters +git eau +game digital +fof ana +exo genous +esc at +erzur um +digic el +deri paska +de soto +crew mates +cor ail +copper smith +consig liere +con cho +ch ingly +cau very +carra geen +candle mass +cal k +c chooks +bru mis +british summertime +bram cote +bb ar +b pr +avent ine +auctionat alive +ashley judd +ankit lal +ale cia +aimee mann +aham eed +agon zalez +abdash soul +ab ert +ðŁ§ļ âĢįâĻĢï¸ı +è½ ī +å®ĿçŁ³ ãģ®åĽ½ +âľį ðŁı¾ +âľ ª +è que +your majesty +wrps rockland +with dean +willi emullin +whole wheat +whenin rome +weather alert +wab ara +wa then +vijay deverakonda +up tuks +u or +ti sing +thoughtful thursday +the shardlondon +the digital +sur realists +sun ol +stormbre aker +stim me +ster ic +stein hauer +staip ans +sru d +sportsc asting +sports massage +sin ne +si guen +shi ppen +seet trading +save themall +sat er +sa igh +sa hr +s ft +ru as +ro mil +respon se +pu ca +propri o +pro vable +pri der +plan as +pham ous +perpetr ating +pel ita +pe ddled +parasy te +pan tha +out witted +out ils +ous seau +ot k +ol ake +ny ad +nor mann +no edit +nag araj +mire la +mi eux +mega house +me rend +mary rose +marc ille +manushic hh +mad man +m jin +lo set +lim pets +len hart +leg ance +lacey chabert +koin ange +kle ve +kesh asuxx +kc traffic +kab uk +k nac +jur mala +jaun diced +invali dates +ini photo +ilm wx +ido los +ic at +hum vees +happy yyy +grub street +go zips +go bbled +ger minal +gen re +gand his +followyour dreams +flore sta +fellow ship +eyewitness nyc +evangel os +eskenazi health +es ad +elle decor +do bel +del his +cri sfield +conge aled +comp diningnc +chicken wings +chae bae +cer atops +car michael +cairn staipans +cade tsuk +brax tons +bour seettrading +book direct +bon if +blin ka +bil our +bick more +bei sel +beau bien +beach walk +backto you +at midnight +ak hand +ad ari +aardvar ks +] @ +ðŁij® ðŁı» +âĿ¤ï¸ı ðŁĴľ +âĢ¢ *¨*âĢ¢ +Ø® ت +ÅĦ sk +yo shis +yat ai +yale britishart +woody inho +whel chel +wake ham +volvo trucks +vol land +vl tn +verti ginous +val met +v wf +united by +timb its +thy mus +thur day +the village +the face +tas kin +suc cour +sub mission +su mar +social ised +snu ffer +slav yansk +sj fc +show choir +sha ren +sas ki +s biz +real monarchs +re gol +ram mandir +ra bab +r vaidya +puer tom +poldark pbs +pninator nai +philosop hic +pay zer +parasauro lophus +paphiope dilum +otra sheffield +organiz ational +or tona +ole mia +od deven +obfusc ate +ny fwm +north london +no deal +nm fc +na ak +myfour cats +mul ga +monte go +model sown +mod cloth +mic an +met alist +mega structure +mcgoo han +marth ac +m acked +lu bumbashi +la ich +kup chak +ko bolds +ki pps +ki and +kev ich +kap uso +k wat +jet se +je j +j nd +j ml +itsal way +it um +ing rate +in expensively +hyn chus +holmen kollen +ho berman +ha inaut +grader ocks +gen ies +ge mat +francis cus +foxsports go +follow er +flat pack +fabi ano +ex clamations +epistol ary +eoincol fer +ema m +ek deewaan +ecu piratesfb +do stana +diverticul itis +discover la +disciplin arian +di benedetto +de weese +day togo +davey havok +comedy show +colo ssi +co win +clande boye +chang elings +castan o +canadapost corp +bu jang +bre slow +borge ousmusic +bin x +ber hampur +benson henderson +bas ka +artsc entre +armor ial +antigu abar +antic y +ant olin +anony me +almost famous +allo h +all thebest +aj ola +afternoontea week +^^ * +ðŁıĬ ðŁı» +ðŁı ĺï¸ı +ðŁĮļ ðŁĮļ +éº » +âľį ï¸ı: +âĢ¢ âłĢ +young ji +you se +y wood +wo begon +white marsh +whi ppy +where with +wf nz +wester lies +ween ies +we ard +wash outs +waron yemen +vi bert +var u +valley fire +v rt +uc its +tu me +travel quotes +travel agents +trade talk +themira gelv +super novae +stol t +ster ols +shereen bhan +scri pta +sanjay manjrekar +s ved +reve re +pretty boy +predic ate +port colborne +pin zon +pin el +pic tet +pas richa +pan talla +outag amie +on fc +nissan uk +newsp ic +new shoes +neve u +ner fs +nen go +nac i +mose by +mon hegan +mom ento +mo hr +misse dit +mete pec +meen u +mcin tyre +mat shummels +maje ski +mah y +mah lon +lycam obile +lumin al +lis ation +le vison +laurid sen +lar khill +lam ina +l brut +kou m +king ricochet +kin ged +kildare gaa +kell man +kc pe +kay ley +kal pat +jar vie +inst l +hob good +ho gle +he sh +hall inan +gyeongbok gung +gou k +gaz ipur +g ny +fulton coschools +front als +football league +films official +faul ting +fau s +extor tionist +erin cruz +engine ersweek +eless ness +dox ey +dis comforts +dio u +dazz lingly +cut throats +comedynightswith kapil +cle liam +chine sel +ce duna +cat olica +car ya +brexit deal +bo swell +blun kett +bill u +ber ges +ben sen +batchel der +barbic an +bar the +b bu +av athi +autum ne +au vsi +ator sk +ass ja +ar lan +amu sique +all one +ahon a +af shar +! ðŁĺĿ +ðŁĴŀ ðŁĴĸ +ðŁijĢ ðŁĺį +á´ ¥ +yol andi +yogare treat +ym ous +ym cofficial +xoxox ox +williemullin snh +wido wers +wex ham +westhigh land +war is +wa ay +uranium one +un gli +ud ham +u calgar +tweet likeagirl +tur lough +thunder cracker +thor is +ten sity +tang ere +tan pa +tal mu +suni el +sm sd +show addy +shop girl +shankar raja +sha fie +sextu plets +scroo ged +sc alling +sans tha +sag ara +rovere to +rossi o +rec all +prairie chasers +pepe aguilar +papill omavirus +pan ico +oo ey +odal isque +notan other +nol lywood +nikon owner +nightshift beer +nesj loch +mú sic +my rie +my great +my fan +mun caster +mon ier +mipim world +mat suz +man je +lo pilato +lew dness +kul m +kom iks +klingh offer +kepp ler +justin hartley +just love +jer zy +it oo +ilo pez +hermi da +harjit sajjan +h alling +gun ne +guer illa +go dan +girls not +gearo id +ge us +gar u +gale abrewer +fri endo +fish fry +et cs +esoter ica +elek tro +duck bill +dot me +distill ery +disfru ta +diamon dring +deu sto +defence men +de anda +dd firish +day tripper +d windle +d gaming +crui secontrol +cruelty free +cropre dy +cran nog +conval escence +col beck +cb buk +cat tery +budd hi +bot tes +bolt action +black dress +black beauty +beque ath +be son +bar bag +bante ay +bang ko +at tests +artific er +arn ley +aregbe sola +ap sc +an ot +alphon se +alab amade +acci ones +abat to +aat r +a ill +ðŁļĹ : +ðŁļ© ðŁļ© +ðŁĴª ðŁĻĮ +ðŁį©ðŁį© ðŁį© +éĢ ± +ãĤ¤ãĥ© ãĤ¹ãĥĪ +âľĶï¸ı . +â¬ĩ â¬ĩ +ঠ¾ +whit etv +wat ing +vin ilo +vagab onds +un welcoming +un wary +un masks +tux edo +tru g +tp wd +touri sme +touch my +too ker +toast day +to lo +the clown +teng ger +ten ino +tatt nall +t ö +sum mith +sug ata +stwee tings +stu pas +stewar ton +spray tan +sop ron +sion i +shi moda +shash lik +seh ri +sc ure +sab iha +rush cutters +rudimental uk +roy ds +rober tg +ridley scott +re ap +ranz kyle +r breich +popular ised +pol onium +po kor +perplex ity +part time +paris marathon +padman ab +osco da +oregon football +oo of +om kara +ny pa +north wood +new school +ner music +neh len +nationalvoter registrationday +nation alized +mur mer +mu u +mand ela +m sti +lw lies +le ggins +la haye +kon adu +kokol ondon +kir tley +kh oops +ke sho +jon ation +ja ane +j du +is mrm +irreversi bly +insubordin ation +insafi ans +ignati eff +hor ns +holt mann +henni ker +heilong jiang +head and +harperadam suni +h elling +gu ap +great service +good kind +gol spie +getting ready +ger be +ge deon +fuzz ed +free picks +four tet +fon ics +flann agan +fire r +fethul lah +feline friday +fan wood +f mea +em rah +dil shad +den sification +de bar +crit ch +che il +cene rentola +caram bola +cal lip +cabinfe ver +business strategy +bri stling +bre ann +biz markie +bio ethanol +big ler +bab ers +b movie +az gov +asphy xia +aqu aman +apple edu +ani ze +an az +am h +alamo bowl +ah iri +adar sh +? âĢĶ +... ðŁĺħ +-- ' +! ', +ðŁĺŃ ðŁĴĢ +ðŁĴĥ ðŁİī +ðŁij½ðŁij½ ðŁij½ +ìĹ ł +æ § +å² ¡ +ÙĥÙĦ ÙĨا +zu an +x ara +wolfal icemusic +wm tw +willo wh +wik icom +wi ggling +w inge +vill al +verdad era +veg f +ush l +tv s +the leader +the hive +tequil as +tag al +tack lers +sur feit +style blog +steins gate +star finder +ss ure +so ireland +sla pdash +sko dauk +sequ atchie +sebastian rulli +schem bechler +sal mag +revan che +repl ou +re evaluated +raf brizenorton +r fo +po lem +pat waris +p cn +only on +nulli fies +muscle tech +mul lein +mou li +mont illa +mo tho +mil ap +miguel cabrera +medi adays +mc cambridge +long life +li mbers +let aba +le yo +labu an +kemb awalker +ke len +kcr anews +kcc afc +kar lie +je evi +jason dufner +jagadish bliss +jag ged +it sv +ir one +ipriyank sharmaa +inter ac +i vig +hil sa +he ttinger +harpers bazaar +gun show +gu lli +growingup black +groupp lc +group set +gro ad +gri saille +greet ers +gor go +goodto great +go colts +gener o +gc morningdrive +gav yn +fitz water +fel ty +es ol +es ada +erry body +eeee ek +dingh ies +dhru vanatchathiram +dem party +cy syll +con rado +change severything +chal oner +chal am +cer titude +cdn sci +car quest +car lina +cabez as +bus stop +bob saniga +bo sko +bmo harris +betty white +b sor +az tlan +aruban etworks +arti stique +ar mando +apoor v +all gä +alice a +air cargo +ag ur +adalovel aceday +abot anist +abac us +a otd +a ayi +a ara +ðŁĴ©ðŁĴ© ðŁĴ© +ê¹Ģ íĥľíĺķ +âĢ¦ ( +zor ya +zoo z +x liv +wyan dot +ve tt +unic y +u waterloo +u ja +tren a +toy spotting +tower hamlet +tl ry +thorn berrys +thom ast +telekom wall +te shima +t pk +sur mount +strange music +stead ying +standwith israel +stam ets +spell caster +spedi zioni +sn ac +sin bad +silver oak +seb aceous +sau dagar +salam is +ride to +remo teness +ready togo +re asure +radioc itizen +qu ie +q magazine +pymat uning +pul sars +pu spa +profit eer +pro du +presu mes +pitt ard +peak auto +parathy roid +over consumption +ou tre +oli vers +ofthe sea +o thon +nov ae +not me +norder ney +ni me +ni hilo +network security +need ville +ne yed +narasim han +musix match +motor able +mo ger +mini mization +min ting +men ia +mcil rath +marinel itter +madi keri +lu stre +live world +lever hulme +le lang +le com +knight stown +ki evan +khim ki +kee pers +ka ag +judith light +jeremy piven +jas mith +j ji +ioc media +inver loch +im plan +il vo +ij ssel +ibrahi ma +her ff +helli well +ham mes +ha chem +greet z +greek wine +great york +friday feature +fo gg +flax man +fal chuk +fail ing +escu do +ery x +ehr hardt +dur man +dubu c +dis connected +dev arak +der yk +dar g +d green +cu ello +counter acting +cor win +construc tionist +ci brian +canadian open +breath nach +boling broke +blacke verything +black well +black catsrule +bergh olt +ber lay +begin shere +beaut ys +be eee +asi at +as selin +artgalleryof nsw +ani kan +angel haze +an gol +amal ick +adhi kravi +abar ca +a inc +ðŁĴ¯ âľĶï¸ı +ðŁIJİðŁIJİ ðŁIJİ +çĽ ¸ +æĭ ¡ +ÃŃ rez +´ âĪĢ +zam asu +ye aaaaah +wrestle mani +win driver +waffle day +vin italy +video gamer +ver wo +val ya +univer selle +tv w +tumble down +ts it +tra urig +tr ond +token ized +the doctor +tg sports +sy ch +sport pe +social ites +sham en +shakyam uni +septe m +seib old +salesforce devs +saha j +s vin +restaurant news +red hawk +rebutt als +re titled +rajiv message +prolifer ating +plumb ago +pe tu +pe leg +pd st +pale face +over running +onon dag +nn pa +netro ots +n illa +my bb +mon duk +moisturi zers +mohan lal +mo pen +mil kovich +man aka +maid ment +mah ina +lance storm +l me +kleptom aniac +kingsc ross +king stree +kindle book +jay buma +jaybuma om +jas inski +ivin ghoe +im r +ifbb pro +ic ent +huf nagel +hit sville +h th +gy am +good witch +go titans +go ffman +gear shift +fy b +fre ds +forex trader +fol s +fad l +eun an +ess chau +esp ouse +en sky +eagle man +e store +dre wett +draw tober +dj h +dizz bee +dece ase +death stalker +cra il +cour noyer +coun seled +color ant +cl wyd +chon ors +ceram ica +celi k +career tech +bryan clauson +boooo om +bol li +blooming daledc +bl ico +be tway +be muse +basili que +avi shai +astro photo +apal ace +anti ageing +anodi zing +anne hathaway +anjan avj +alvv ays +almu dena +ac ai +aak nopf +? ðŁĺŃ +; $ +ðŁĴ· ðŁĴ· +âŃ ķ +âĿ¤ï¸ı ðŁĺİ +âĿ § +ร าย +Û Ķ +Ë Ĭ +ya o +wo y +winter in +winston salem +wi king +warhorse studios +usur ping +ur ca +uni fy +un ita +un alienable +u ht +u asin +traitor trump +trac s +tr illian +tourde suisse +tomas elli +tinari wen +tiddly winks +three js +the pig +thank ss +th rap +techno cracy +te ake +super mare +summer bash +su dip +stri al +sto key +sk up +simad wasim +sha fik +see ster +se mer +s ical +ruhr gebiet +ron sexsmith +rochelle humes +ro tondo +red z +red pill +real news +re processed +raj hi +pomer antz +pho cu +pete y +per al +paulma sonnews +patho logy +palam pur +over rules +o had +nv h +northant shour +no cera +natali ep +mun ari +mu hd +mtu td +migu ero +me cum +mccour y +manushichh illar +mancity women +main landers +madelaine petsch +love hate +llll llll +lic ey +li mm +li dell +let tera +legoland florida +lag ell +la gaan +kvad rat +ku ff +ko san +kevinolear ytv +johnnie walker +jer g +jenna jameson +iran talks +im bert +illiam son +il ynx +ichthyo logy +horse sofinstagram +hor ti +holo han +gw mag +gujar att +grrm speaking +gre ave +gla xo +gh pl +gaz ania +gain fully +g ics +freedom caucus +foto sred +exu ded +es war +entre met +electric ity +el itch +ei le +ei k +ear le +e bs +dry bar +di metro +deta ching +dath lete +cze wski +common s +coach k +ce mil +can as +c summit +bur treynolds +bur row +bu ana +bharathan en +beer tography +bat tist +bas ca +auto chrome +audi os +arvi at +ap lit +an ley +alim entation +aliciavi kander +alegg ero +ak ids +... ðŁĻı +)))) )))) +$$ ! +ðŁĺĬ ðŁĴĽ +ðŁĺ±ðŁĺ± ðŁĺ± +ðŁĩ¿ðŁĩ ² +ðŁ¤¸ âĢįâĻĢï¸ı +âĿ¤ï¸ı ðŁĴĹ +à¸Ļภģ +ಠ£ +à¥Ī à¤Ĥ +ze itz +wyn ford +wr its +walla hi +votedem sout +vic parkacademy +v md +up voted +u tong +u ani +tr yn +todayin prog +there bel +tamiz h +take walks +t fm +swann anoa +stre cords +st ns +spec new +space marines +socialmedi am +sd x +scroll work +sai fi +s gbv +rin dge +railroad er +pu ddicombe +pop mech +plan thealth +piti ed +pav lik +out lasting +nff network +new ell +ne trunner +nd pldr +mu sson +mor ass +mon sef +miracle treatday +mcki bbin +maz ara +kron berg +km ss +kis on +khan de +keepit public +kam sa +k san +just ici +j co +is ymf +inter fax +ick en +hast ens +ha ka +h tweets +gre sford +ge trowdy +g lines +fu zion +fu gs +ft nqt +fre a +fo sho +flo rea +ever body +et attoo +er stalk +ent rap +empor da +ellen son +el aval +ekdeewaan atha +ee er +ea stover +e ion +drumb agus +dr al +dor rian +domest ica +dine fwr +digital learning +de baser +david ferrer +dareto achieve +da oust +croco dil +crit ica +cos ch +corks redfm +cooper ator +che me +ce deno +canary island +ca ele +brit ton +bil angpilipino +bhan ja +ben souda +bed stuy +bed der +bac sin +aval ok +arti stin +art sctr +arri age +appro che +ankush loveuall +and alex +aldub ftnqt +advance qld +ac ckickoff +ðŁ¦ Ķ +ìĹ IJ +zul ki +yel a +yayo tte +world whiskyday +wo whead +wira djuri +wim bush +wee ty +wav ell +vijay tv +vibr ators +vel indre +va ine +ucu strike +tze hn +ti german +thin ness +sunset strip +sun records +sul than +speak ership +sne ach +sl r +sky harbor +si pri +shut en +sho bu +she ilah +search andrescue +sch euer +saving system +sav ar +s shop +rut kowski +run streak +ronde santis +r pas +r ala +quadri ga +prof dev +pork ys +or dem +offer te +o hene +nw sw +noaa fisheries +ner am +ne so +n sel +mitchel musso +mega project +mccre ady +mc peak +mas ke +mary lander +mar ris +ma es +lu gh +lovel ove +lohen grin +lev ick +leigh anne +leeds museums +lazz aro +kv bohra +kra fty +kom mer +kim jun +kha e +kempe gowda +kad ı +juli ahb +jeopardi zes +jdff fn +jaw ara +jason reynolds +jar boe +indie films +il mu +hinching brooke +har ahan +hai me +h enton +grac in +goon di +girl school +gar bett +fore tells +eye em +et g +elimination chamber +eff endi +e hhhh +dy bbuk +dor ries +dom pet +dir ilis +dioce ses +defaul ter +cron an +cring y +copp icing +cle atus +clam ouring +ci elo +ced chat +career center +care ening +capit ole +can nock +cal state +busy ness +brown thomas +brown ing +bou ille +bj ör +bio y +bergen county +be abin +back pages +bab lu +aw sum +aval ance +av anna +arti k +antoniosab atojr +anan avarro +alu ckett +all ene +ak or +ach tzehn +ac anth +abou bakar +ðŁĺĤ ðŁĴĶ +ðŁĴķ ðŁİĢ +ê´ ij +ê° IJ +za hid +yel les +wra c +wer q +weis ser +vit ek +visu ally +vill ach +vandana shiva +ur win +uniof glos +tweet suk +too short +til les +thenext one +the grind +ter ai +ten sai +tele vising +tann is +strath allan +solihull moors +so lie +sj hs +sier re +shop at +sex posed +resu me +ram lal +ra zzo +ra uh +ra ggi +quatu or +pulled pork +pon za +ovo xo +nokian etworks +nbs frontline +nation alization +n qf +my lanta +monoli th +mish kin +marshall u +mal enko +ma ire +luxur yy +lu chino +lps leads +log ne +lah bati +kno tto +ke me +ke ito +kalani thi +jönk öping +jab ot +j cg +inthe woods +inter lace +in reallife +in kers +illusion ary +iam super +horri fies +hi ebert +hex ane +here comes +hep cat +henning sen +he dden +hb wx +halt whistle +ha ine +h kl +h bu +greatocean road +gi gan +gi ardina +geek out +gas prices +fra sers +for thood +first champ +extre mo +espou sing +er ci +eng irl +ds band +dram atical +disa strously +dil ruba +de freitas +daugav pils +daress alaam +dam ine +clean technica +classic motor +christopher sean +chif ley +chan teur +chag os +cesen atico +car chives +cancer prevention +camillu ddington +bwn sft +bro dgar +bo it +bil ad +ban ki +bam bara +bachelor nation +babie jaan +ba rer +b sh +aurorahu skies +ashi k +as de +ar uknews +anu ma +anamne sis +all ant +alex the +ab ug +ðŁĴķðŁĴķ ðŁĴķðŁĴķðŁĴķðŁĴķ +ðŁijĮ ⾨ +ðŁı ľ +ðŁĮ ģ +ë§ IJ +âļ Ľï¸ı +à® ± +Ùĥ Ùħ +z iro +wonderwoman film +wash spirit +ward han +w va +volve remos +user research +unstopp ables +un youthenvoy +tnt vote +tie breakers +tic hy +theblue coat +the pro +th alab +te tes +tan dil +sw u +stunt men +stat erooms +sri aurobindo +spear heads +sos fanart +sma drid +sig mak +shung ite +shelove sme +see hofer +samu raic +reg la +redro ses +rad agast +queen stennis +pink villa +pin jarra +pad an +p tofed +p list +oy j +oswald twistle +oh u +nu va +notor acism +not that +nor wic +ne ste +miz utani +mis sma +mick legate +med lab +ma sto +m ve +m pre +lit o +ley music +lev ator +le pine +la ket +kwant len +kor dei +kk onen +ki ppa +kell yayotte +jim james +j assie +it ories +io h +indian river +hom ilies +hom enor +he mmo +he do +harland williams +hang i +hak ama +hac ke +ha cha +gray bar +glob alizing +glen mark +girl ll +gi essen +gar hi +g wang +fri jns +freedom pop +f op +f bisd +ecuad orians +ea stre +e toe +dro me +dri vin +dk b +dispro ven +dir co +db sk +days gone +dan aroh +danaroh rabacher +da ira +cuse football +ct fletcher +coch lea +clan sman +chit on +chandra a +cente redness +cci w +cat kin +carpet cleaning +c mcs +bur ritt +bur rit +bur ps +bristol council +breconbeacon snp +br ous +bmoharris bank +bittrex exchange +bhagal pur +berg ü +bath chron +bad boye +ay ami +auto shrine +atlantic ocean +ash ur +any where +and ora +an r +alabamade ptofed +al smtl +al goa +ag ener +ad missible +aci do +ac sc +(** *) +ðŁĺį .. +xoxox oxo +wood cliff +wa hhhh +wa ala +vi official +val da +un revealed +un ct +uc ina +tucker ton +top or +tony t +the cottag +that momentwhen +sã opaulo +ss and +sp ates +soar ing +slu mming +shy la +shir king +shak ha +ser ino +sequ al +sea quest +scum villain +san chita +samsunggalax y +sakura ba +sag ot +rosen field +reclaim temples +re vent +re housing +radi ot +ra wer +ra kia +q music +publi ka +primary school +pic say +pha i +ph ans +pg tips +pelo tonia +panzero tti +pad auk +pa pps +p fl +oster ville +nu oc +nrl storm +nor c +nid drie +nationalcheese burgerday +nation alize +nan og +nag ma +my rin +monifi eth +mat twal +mar kups +mag ery +ma zie +m dotnews +ll cs +linsey godfrey +la don +klu mp +kings road +kher son +jon te +je had +jagi ell +inter ludes +instagram aviation +inspir a +ingeni eur +in z +ike me +hotel ympia +gu agua +glo ved +glo ats +gh ook +gam pel +fuel band +free guys +fre un +filip iniana +fil ipa +field view +felic ite +eye sof +en strom +eat local +easter by +e je +e est +dys morphic +dur ston +doge ver +dinner party +dic kie +delhi governance +dave matthews +dar waza +cor zine +coc chi +cirro cumulus +ci err +cfl draft +cash cow +call al +bunting ford +border security +bobby flay +blood wood +blo on +bible verses +bham bri +belo it +bay da +as microbe +ar lyn +ar ada +an derer +amandase ales +aerom edical +a angan +-------- - +(*´ âĪĢ +' - +ðŁĻĮðŁı¼ âĿ¤ï¸ı +ðŁ§¡ðŁ§¡ ðŁ§¡ +æ¸ £ +âĻ ¿ï¸ı +ñ an +yu kino +yim by +worlds best +wood chester +wonder la +wire tapped +was ley +wal shy +visit wiltshire +u av +tun aji +tororo sso +ti sd +thalai vi +tay lore +tan ev +tag esschau +ta vener +swift water +swayam sevak +stat ecollege +sm oooo +slu dgy +shirat aki +shic ooks +shell no +senbill nelson +scro ps +roar for +rider nation +reza ian +reu el +ren nan +reis inger +reflec tance +recali bration +re eeee +railway men +queré taro +pri am +pre stel +pran dial +post grads +po too +plas matics +peace full +organic chemistry +onther ange +o zeki +nvi dia +nuit blanche +nau fal +n mw +musik verein +murmu red +millen colin +meth ley +me ggy +matti oli +maha veer +mah mut +magno liam +lo cum +live work +li pi +li le +let son +lavany ab +larkin poe +la isse +kro ko +kor is +kal ev +k snt +k mox +jojo lion +jersey boys +jamai raja +iz nik +hypn ago +huawei mobile +hr weather +hol son +highway sengland +heteronor mativity +her berger +gy ps +gp b +go vikings +gat wick +gal al +g vs +fra zer +fotogra fo +foot ages +fogel berg +fashion diaries +europe union +eti seo +eni elsen +en no +emr gency +ell and +du bie +drac onic +di rec +di du +delta pi +deci de +dav its +dar shana +cu ta +ctfletcher isymf +cra zing +clam per +chup ke +champion sof +cap city +cand ys +cafe bustelo +c phi +but chie +bro ca +box hq +black deser +bin dle +bi si +beautifu lon +ball python +bali united +av elli +aster son +arkhangel sk +ar ail +antan en +amc clain +ak au +agen i +ag ril +afric ageo +adult work +adri atic +ðŁĺĤ ðŁijĬ +ðŁijĮðŁı½ ðŁijĮðŁı½ +âķ Ĺ +à¹ģภģ +wyff news +wwe backlash +we fly +wal deck +wa ira +video art +vaill ancourt +uri ah +upto date +up en +un ay +u shahidi +tree of +then g +thelonely island +the pig +the houseof +the first +the bronx +the bobby +tere x +temper aments +tar bes +tab u +street party +st catharines +sport said +sou bise +side hustle +si donie +septic art +seal ers +sanjayleel abhansali +sam jarvis +sam bassador +saiful lah +ross more +rish on +rise borough +ri jo +referen da +rdr hw +rdrhw ke +ra glak +que eze +qld maroons +pun tarenas +pulwam ater +porta ventura +orang i +or fc +od ham +o sma +nikon canada +new blood +mul led +missan dei +man cer +mamed yarov +maha kumbh +ly co +low fat +lov ington +lou ds +lost planet +lor rain +local syr +loc ascio +lo ker +lat vi +la thletics +kope ch +kk f +khan al +kaw amura +kang ta +ka arora +k out +k bye +k block +ju che +ip eline +inter regional +inhal ers +il ynn +howard winn +holocau stuk +ha gue +gu mbs +gu ddu +go fer +gear talk +gani zation +function alization +fri dman +forthe culture +fight likeagirl +fiba europe +fel trin +far ner +du sun +divers ities +dig na +di grassi +dela uro +del is +datt ilo +ct as +cre b +cost i +cook stove +child labor +ch haya +cer as +castell an +cam ming +callfor papers +c music +bul keley +buffer chat +bre tz +bla i +bird studie +big bend +bet to +best deals +belfast trust +behindthe scene +beard day +b koepka +astro family +ap ki +ap ath +and in +an sty +americanc rime +ak dn +/ âĢ¦/ +!! ). +ðŁĵļ âĿ¤ï¸ı +ðŁĩ¦ ðŁĩºðŁĩ +å® ¶ +Äį iÄĩ +z ater +yaq oob +yach ts +xcel energyctr +woll man +wit zel +where it +wh w +w uss +vichy sso +un assembled +u an +tttt tttt +tin ed +the same +th und +test is +tar quin +sudan ese +su jin +str acing +stati k +star zz +sleep well +silicon beach +sig nes +secur itisation +sad lier +rs one +rondesantis fl +red band +ray han +ray bucknell +qu ain +pris matic +po dia +plo de +pitch er +phirek baar +par doe +pap onmusic +ou tran +om men +nvi vo +night ma +newham london +nar done +na has +n carb +mustlove dogsnyc +moyam ee +more h +morales nbc +moral monday +mo hana +miner alo +micro dot +lough or +london underground +lin ked +ley endo +le ke +lam otte +l rc +krysty na +kore sh +kor do +kim on +ke bang +k boo +jat in +jan maat +j cf +island peeps +is gusting +ione sco +horticul turalist +her mia +hau f +gre ate +gre aney +geot agging +gan ador +g kg +felicit ating +faf nir +eye dea +eu greenweek +e gon +dun stan +duba itour +dol lmaker +deod ori +dahl strom +cy ne +coz ying +coor paroo +bun nic +brumis brill +bli sh +bla ker +bi bek +bertol ini +bear up +barri o +bad religion +atheist pics +argu in +arah ma +ar bil +anupam apar +an laby +am ans +ðŁĺĤ ðŁijĢ +ðŁİĤ ðŁİī +ðŁ¥³ ðŁİī +æ ¡ľ +å· ¥ +ãħ ħ +you love +ye osu +wool lard +with rome +whit ted +waz owski +vam usic +v icia +ut karsh +unis an +un hq +un disguised +ud fa +trebbi ano +treason summit +tot tering +tiny desk +thef dci +t mu +t ellier +sun dazed +sullivan stapleton +sublime withrome +sto on +star tyour +springe tt +spec tat +sow mya +soun ion +sm ommy +shu ghes +shin agar +sar oundtheworld +samp aguita +sal tney +sacri sty +repri mands +refu sals +ra yos +pu glaas +po den +phone bank +pet lovers +pat kar +party wear +palom ares +pab a +os nat +operation alize +ob son +nat museum +nap er +mustafi zur +mrat andhan +misogyni sts +medical school +mb b +mat tryan +manus amoa +make you +ma seno +ma iri +m shs +lollo brigida +lo ssy +legend re +le tra +la sha +kon ig +kin loss +khu zdar +ke ba +kay ah +jointhe hunt +jo equ +jennifer lopez +jazz guitar +jack reacher +ja ho +insta stories +hyper converged +hor miguero +hi ep +her zig +henry mcmaster +hard life +gü rel +gra uer +gott al +glo bo +gar nham +fv glive +fro moz +fo tom +fever few +er berg +ent zel +du ology +dri scol +dit c +di mages +di ante +deli ke +del li +dejec tion +deci siveness +de presse +dazz le +cu cur +cre ll +corpor at +constitution alist +complete streets +co ziest +chop tank +chast ises +care sse +bun naha +bull s +breath ers +bol ong +block stream +bi zi +better life +ban deira +au det +at tern +arric hannel +apportion ment +ap ti +angelas canlon +amin ata +am roth +all me +ag rant +a akin +ä¼ Ŀ +zz acnn +z scaler +ye dition +yak ub +wg x +wene ed +w kw +vacation rentals +tran z +toy town +to gar +thick ener +the sopranos +tent en +tar getting +talent acquisition +ta wil +swar ts +sw and +susan ville +suno cor +stan chions +sputnik int +spay and +so bu +sig ala +shu ter +shigh school +sg len +seri ksen +sensiti zing +sd summit +salv adore +sai bot +saber metrics +s my +ra avan +pod fix +pier zynski +perri man +pensami ento +oro der +oranje stad +ober lin +nul led +nol enation +new fane +n jac +n alu +mush ers +mou lins +mo il +mo he +militari stic +mah boob +lima hl +lift bridge +labr ys +la dolcevita +kur tz +kram atorsk +know sley +king makers +kas am +juli aroberts +je der +in ae +har jit +hack berry +ground works +greg sestero +gre ninja +golden era +go duhawks +giu giaro +girar dot +get ter +ger ken +gem sona +gang adhar +gam ingh +g wich +fish cakes +far fetched +fal sify +evidenti ary +dy spe +dun gloe +dpt student +dmv hoops +demar ini +del wp +dar lin +czar necki +customer service +clt food +cilli zzacnn +centin ela +cc bl +cag op +bu achaille +brü cke +bru gby +brit ts +bret michaels +brave st +booboo stewart +blow out +black tie +bi h +bert strips +ber ano +beforeyou go +be rel +b ance +ato cracy +ati oneu +at alan +ast age +ar li +ap lang +and ung +ambiti ously +aint no +aid il +aged don +ad avies +aba journal +ðŁĻĮ ðŁĴª +ðŁĺľ ðŁĺľðŁĺľðŁĺľ +ðŁĶ¸ @ +ðŁĩ¨ðŁĩ ¿ +ðŁ§ĺ âĢįâĻĢï¸ı +ðŁ¥ĩ # +íĶ ¼ +ë° Ķë +ä¸ ĥ +âĿ¤ï¸ı ðŁĮĪ +young leaders +yamaham otor +women inste +wir h +ve dette +user interface +unboun ders +ulster grandprix +ty ran +tri pathy +til tro +the hi +the cle +terry fox +templ enew +susan n +sp aeth +sop hy +soko lowski +skan sen +sit z +sik hi +shar am +sham balla +shadow banned +sent om +sel d +sd ick +sch emed +sal eyards +riff age +re submit +re mora +re hired +radi ans +ra unch +princeton upress +powder magazine +post traumatic +pi eno +pflu eger +parade of +pakv sind +pa q +on etwork +ome de +o esophagus +nys dot +not forsale +nom is +nol on +n gara +mu cosa +mou lya +mer q +me giddo +maxg schneider +mat tc +mal herbe +maison avendre +ma sumi +lucas dirt +laquan mcdonald +lan gholm +kam os +ka ash +k do +ju bei +ittake stwo +induc tors +ib sf +horror core +homo erotic +henness y +hase ul +hang out +hal m +gui ao +giriraj singhbjp +fuse box +free zy +faro ese +faci ally +ex ocet +enlar ges +emrgency kittens +el ul +dysp nea +dream girls +district speed +digital twin +deple tes +danny sullivan +dal ab +county durham +col v +col lum +coffeet able +chicago symphony +cherno byl +caz ador +capric e +cam utd +cal mes +bur mester +bur kha +boli de +bo wne +ber c +ben aras +bal ewadi +ase er +as obi +ang g +amp adu +all ones +aha asan +!! âĢ¦ +èĩª æķij +âĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ı# +Å¡ i +y xe +whoa jordie +werewolf wednesday +v achi +un cut +tom boyish +the legal +ten de +swanse acity +stig ma +st lukes +spiri don +spectro graph +soch aux +sn omg +shaz ad +sha henshah +sd awson +salford cityfc +s north +s back +riaz or +rallye montecarlo +rad wan +pu ting +pter anodon +port ant +popul ationday +po ple +pic fair +phyl lida +path mark +pastor alist +pap ar +p inging +or tom +oooo oooh +onor ris +nor aho +nom nom +netball worldcup +natash aleggero +mysteri esof +my happyplace +move it +moto corp +miss them +mile high +mh p +mezz otint +meme monday +mediat ama +margare ts +madhu sudan +m duk +lucre cia +lu bis +local ise +line mates +ligu rian +land onorris +lac er +l varchives +ku ne +kour i +ko kane +ko ban +kis lyak +kin daichi +kennedy nation +ke ister +kashmir floods +joh no +it vs +istitu to +ise kai +ioanni dis +inter val +ing var +in britain +immun ological +igu anodon +he mme +hart pur +gru bbing +gre gari +gor ki +g lead +free beacon +flu es +fl l +fin try +fa xe +explore victoria +eugeni des +ee ks +e frem +div yadutta +di pa +di orio +dee v +crypto spor +creative market +cor undum +coo pride +conspir atorial +congr ates +commerci alised +citizen suk +christian slater +chamber layne +cath kin +cake pops +boxerdog union +boss logic +boar drooms +bo fill +bellec i +bbca q +bali ka +bale ful +b gt +audi southafrica +atom ica +arab net +ar vis +anam era +ai guil +agu stin +adel le +ade vine +aber cynon +^ *) +. ðŁıĨ +-_ -" +! = +ðŁĴĽðŁĴļ ðŁĴĽ +ðŁIJ ģ +ì¸ Ħ +ঠ¡ +न म +x media +worm ley +wedding dresses +we own +w ps +von tae +uto pias +un salted +un ation +un adorned +ty m +ty che +tou la +tou l +thuli madonsela +ten aga +te jay +t bird +sy nec +suk ses +stron aut +stock mann +st w +spark fun +sleek makeup +sk w +si dr +shanth nu +seed bed +se mba +sau thentic +sat oko +sam gye +sa heli +rye o +roar withpride +riverfront times +reclaimed wood +reali gning +qu ants +pul le +price waterhouse +preserv ationist +pp op +padmal akshmi +ox fords +otta viani +nuis ances +nu tan +nsc ad +no filters +neuroradi ology +nan aman +mutu ality +model mgmt +mod pizza +memor able +me tten +mary rose +marthamac callum +mar wat +mai ze +lyne ham +lin sley +li fan +kro onstad +kre ms +ki dding +keffi yeh +kar ley +jman rara +jeff brazier +jeal ously +ip ython +inebri ation +incumb ency +ik ko +hun ches +ht cafe +helichry sum +he sj +gren dizer +gc v +gat ineau +fly together +find sorguk +fag us +ex ss +elaw ler +eg n +eck elibrary +di eck +deaf ened +dddd dddd +dal ston +customers atisfaction +curi ae +colorad ans +coach mike +co sc +clar kin +chuck palahniuk +chron am +chav annes +cau ser +ca rele +bur u +bren gle +bren de +bly leven +bhagav an +bh hs +bein eckelibrary +b ta +ati an +ast en +app raise +anticoagul ants +an ings +age a +adri anj +ac ros +ðŁĻĥðŁĻĥ ðŁĻĥðŁĻĥ +ðŁĶ¥ðŁĶ¥ @ +ðŁİīðŁİīðŁİīðŁİī ðŁİīðŁİīðŁİīðŁİī +ðŁĩ¹ ðŁĩ¹ +å¨ ģ +ม าà¸ģ +ठħ +you cef +yo hei +ulkom inisterio +turn blad +tren chant +thel as +the velve +the futureof +te he +tal manac +sv ill +su gru +stony brook +star sfc +ss ay +sports nutrition +solo lastyle +six point +sime on +silver agetv +sho well +seeyou again +se ena +samira wiley +saf l +ro ent +read women +ram ÃŃrez +product development +pre installed +pietra santa +peristal tic +past it +parami yer +pal ette +pai gey +oberge fell +nu ñez +nkosa zana +nas ser +nam aqu +muscul ature +mor tu +million ai +mii foto +mick conlan +mav ro +lackadais ical +l án +kro gan +karl skrona +journalismis notacrime +jimin hofe +islandpeeps birthdays +incre dulity +honest ly +her si +health systems +haj du +gu pton +great clips +gott es +go lia +ghost land +fitz carral +faru qi +fabric live +f he +everythin ge +eu dy +ere tz +eli yahu +eigh ty +ec pr +dz hok +du kat +diaz epam +de regulated +de hydrator +danny kanell +d total +cycl ine +can filmday +broadway com +brigan te +box cars +bombar dier +boin net +bo witz +bi agi +ber toni +bay le +bab li +av illage +audio technica +arm end +apo thecar +andre greipel +ambul anc +adil ray +ad nate +ðŁĻĤ . +ðŁĺį ðŁijĮðŁı» +ðŁĶĿ ðŁĶĿ +渣 åıį +渣åıį æ´¾ +渣åıįæ´¾ èĩªæķij +人 渣åıįæ´¾èĩªæķij +ziau ddin +well wishers +vyrn wy +volcan ism +volcan ic +video love +van meter +ul brich +uh manoa +twenty something +tri delta +to vic +the sportshub +th onet +tex hibit +stau dt +starry night +south pacific +sof ie +smart phone +sl h +sky ride +sir specialists +shing led +sh sm +secon ding +se any +sau jani +san miguel +road rage +ro dent +rhyth mically +radiocitizen fm +ra ymi +q j +presi dium +perfec tion +pa illard +outfit grid +out t +ok ayy +nú mero +ny ghoops +ny akun +nile postnews +ner ine +ne ate +nanse mond +nag ap +my mc +must reads +muni er +moo rel +mo sconi +mit tag +min tel +mh chat +me pratap +mass aged +marri yum +mark ku +marchitec ts +maha vishnu +mag ph +mad dox +lot z +lea ke +ld week +la force +kor bol +korbol orbo +ja jafri +itsallgoo dep +imin love +huss am +hollyj green +hirsch horn +hi do +hen party +heartw alk +gu revich +green ie +gre lle +gom ers +gan bare +g wx +g burg +fr m +fos sum +fil mcenter +feel z +fe dri +fash nerd +facility management +ero yal +ermah gerd +er so +elev ated +el ay +ec ken +dur rett +dream stime +dh any +defend ing +def our +dec ay +dani elli +cyclonef ani +co wan +caw ston +catte drale +carbon ell +breastre construction +bois vert +bhu van +ban ews +as le +ar ren +ar jan +app ar +aom ine +antag onize +andrew scott +am exico +aircanad acentre +ain z +agi letd +aften posten +af thunderbirds +abhil ash +; )))) +ðŁİ¾ ðŁİ¾ +ðŁį ® +ìŀ ī +éģ ¸ +æĶ ¾ +åĬ ł +âĻ¥ï¸ı # +ঠ¨ +zo tac +za ine +ym pathetic +yab ooks +wx pn +woo fs +wild thing +war gamer +vir ani +v its +us of +under body +u kie +tsu shima +tri pl +trabaj ando +tejas wini +te os +tb alls +tash amed +tar kington +tamar aw +taitt inger +t pay +t cl +suss an +supersport fc +st francis +springh ouse +spas modic +sonic mania +shear waters +sharp stown +ser hiy +sem plice +se vo +salam at +rubber duck +roo sa +rocke ts +ribo somes +ri bon +rac ci +projec tx +prescri ber +pendi dikan +payday loans +paol achi +pan te +off sides +north walest +norfolk show +nevers leeps +my favorit +must ad +moel fre +mirac les +me der +mayor kun +mass ac +margo twall +loy le +loyle carner +love flowers +lindsay mendez +light sticks +kur ram +kirstend unst +kar aw +kami akin +k him +jessic aes +isi ana +io anna +in vocations +in ma +ike barinholtz +iamraj choco +iam lindsayjones +hyper trophic +hummel stown +hiro o +hill enburg +he bb +ha utes +h anian +gur ram +giving soul +gh raib +gabou rey +g sburg +g ite +fy ne +freak in +for him +follo train +fire boat +fi stu +ffi on +ferr ite +fe stac +fe bbra +fall acious +f ê +f twd +esto u +escri va +er sa +ent j +efin itely +dol son +diplom atically +dhanush fans +dewsbury rams +degre ed +dan il +cn sc +charless oule +cham bery +carrageen an +bun kered +bre lade +bran dished +berser kers +bb ad +banger ter +baj aur +b sy +auburn football +akh laq +adv i +abh ors +. ðŁĺįðŁĺįðŁĺį +ðŁĶ Ģ +ðŁĴĸ ðŁĴľ +å¸Į æľĽ +yin yang +y way +willowh erb +whereis adtr +wait itu +w lv +vin ces +vi zzini +vers ity +ver icks +ur v +twi p +trustthe plan +tj es +ten newsmelb +tax us +tang mere +sun music +stor min +stor ation +sti pa +spro ductions +sol ms +singh ji +sid malhotra +si eves +shijiaz huang +sh vili +sh eller +set as +sel ine +se tra +sd am +scifi actor +san tha +sab ourin +sab bah +rte soccer +rough est +ro stec +ro mil +repeat ability +rejoin der +reducere userecycle +re interpreting +py charm +photor tg +philly chic +phill amarr +patri zio +numan official +northan ger +nor anda +ni pes +next cloud +ner s +nandit adas +nanai mo +na hhhh +mutt day +mend ous +mediab ias +mary katrantzou +marsh wood +markh enry +mar ami +man ou +lo ong +letsgo heat +lee schools +lax atives +latchfor devans +lang age +la ing +la chs +l latchfordevans +kom al +kho khar +kel vedon +ka io +juxta poses +jo josi +jagran news +ir ambulance +instagram down +hon k +hill croft +helle buyck +he mis +harmon izes +gun it +gu é +gru bbin +grac ies +go ire +go flashes +go crows +gel ora +gamer retweeters +fer ial +f sk +ec ymru +eaton ton +dubai worldcup +domin ate +din as +dic ec +declin ation +dbt india +day tour +dat elin +counter measure +cle ghorn +cent i +cancer moonshot +c mrf +buy british +bon zi +black ferns +birdstudie scan +bhatt a +ben to +belk bowl +bar ison +autom at +atal aya +ar cona +anten natv +alex bowman +akwe sasne +adore e +ablu e +ab lock +a ÃŃ +ðŁĺĺ ðŁijį +ðŁ¤Ķ ) +âĶ Ķ +win dians +wellcom elibrary +wee ter +wan chope +wad is +w week +up your +toshi ya +tomat ometer +to dom +tal u +ta haw +swwap nil +sweater weather +sul cata +stru ble +ss cho +sonequ a +so bo +shari fa +sham ilton +sex tra +scan di +san andreas +sagitt al +s graffito +rhudd lan +red hour +radi ata +pwe ase +phil ology +p ells +p cin +ortho graphic +or bea +octo pizzo +nade ch +my friends +mur re +mohun bagan +mo dit +missi le +mir u +med star +me hs +mass dcr +mac er +lukas ulic +lu isi +locker room +leeann womack +le edy +l sjnews +kol om +kar ao +jae hyo +itsabout time +ij er +iafe ssia +hu ot +hu bert +hom icide +herni ation +her f +har uko +gun dry +gu ri +grit stone +gries bach +greenman fest +gou w +golden yearsof +go pers +giorgiom oroder +gilead sciences +gh l +get stealz +fun i +fru its +front stretch +fri skies +foxy gen +fion n +fex ile +ferguson october +fair lady +f pu +exac ts +eugen iafessia +epal ace +ei ras +eco leman +do bler +deutsche telekom +desay uno +dead fall +d hand +cun anan +crimin alised +colonial williamsburg +chi haru +cherrybelle indo +challenge yourself +chad bourne +ch eva +ch azy +cdw festival +carolin ed +box nationtv +bottle shop +book facefriday +book buzz +bonda renko +bon voyage +boardof directors +bo sk +bit of +bigg ame +bi jl +between the +batter ie +az tex +athe istic +as nr +ar ki +annap aquin +ange rer +amp p +alcin dor +al tab +above thelaw +_ -_ +ðŁijįðŁı» ðŁijįðŁı» +ðŁ¤· ðŁı½âĢįâĻĤï¸ı +ðŁ¤¦ ðŁı¾âĢįâĻĤï¸ı +ìĿ´ìĶ ½ +⾨ ðŁĮ¸ +âĺ¹ï¸ı âĺ¹ï¸ı +âĹĸ | +ØŃ Ø± +z b +wildbill photo +wild wood +wi ecki +whippoor will +west law +weihnach ten +weall bleedblue +vu u +video clips +utt amav +unge rer +un desired +tw alls +the lead +tere ssa +te ich +tchouk ball +tabletopr pg +sur fcity +sudarsan sand +stan away +sque ez +soli psi +sm yl +shot crete +sen alexander +semit railer +scill onian +sat anic +s endra +queensugar own +progno stication +phil trum +pervad ing +per vasi +paris jackson +par ous +paloalton tw +palindro mic +ote y +onceuponatimein hollywood +off beat +off as +ob as +noraho donnell +non ito +mul tan +mortg ag +mer nda +melo dica +mari za +mar ki +mand ino +maddie and +live on +liti gious +kul len +ku mag +kni p +king sisland +kin neil +kather ina +kani mozhi +jan ick +is eries +iam bohemia +hon ored +hin ze +hell s +hatcher ies +har da +hammer smith +gun s +gre ases +gel ada +gefil te +front door +fri m +fother ingham +foo de +fleuri eu +fieldre cording +fer ric +ext asy +exas cale +est ados +er can +en voi +ell sberg +el vina +el sen +ec pa +early year +dub awi +do yon +direstra its +dg f +detwe iler +destruc tor +de castro +craw leytown +corpor atocracy +cor biere +contor tions +conecu h +comm ending +colle gues +char ging +chakravar ty +btsout castd +bi gos +beagle facts +be ani +bath nes +axi al +astru d +aru th +aragon ite +apr ili +ap aw +antiguabar buda +android app +amor ya +akhmat ova +ad ine +ac ot +aam c +ðŁĻĭðŁĻĭ ðŁĻĭ +ðŁĹĿ ï¸ı +ðŁĴ¯ ðŁĻĮ +ðŁijŃ âĿ¤ï¸ı +ðŁ§ ¸ +âĺĿ ðŁı¼ +à¸Ńม ส +ঠ® +Ú© پت +کپت اÙĨ +ай д +zen ia +y ami +woo dring +wee ked +wa aw +w bap +ver din +van diver +us br +un ruffled +ttly teala +traqu air +thrill ingly +the cool +ten ax +tai v +supp lant +state street +st tropez +squir ting +sports media +sn cb +sik lan +should be +shi pra +sgo t +sedi mentation +saw ley +satur nia +rock wiz +ro chefoucauld +river hawk +rene ged +record label +re aney +ran ney +pur dah +pre ordering +pla bs +paolachi occhi +oxidi zing +over spent +oko lie +official tulisa +ob serve +nystag mus +non duality +newstalk zb +net mediatama +ne ung +nazar é +my kitchen +mu toh +mon ads +mis ur +mi metic +mei sha +me thi +mc as +mau boy +marien assar +mac kidsbooks +ly dd +luka ther +lov ell +ler che +leekwang soo +lam phun +kirst jen +kansa shistory +k wwl +k ello +juli ec +john cross +jn rs +jac anews +iu chi +ion izing +invest is +introduc er +ing omar +iconocla stic +hm givingsoul +heph zibah +hen ze +hell ll +hakkasan lv +gus beef +guiller mo +geode tic +geek s +gd ha +game gear +ga day +fre ema +fox hunting +firstalert ct +fe kete +entren ch +enni als +ela dies +education fest +e wer +drau ghts +dog u +disp late +denou ement +dag i +d ack +cynthi ana +cyber tron +cur ated +ctvmorning wpg +cottes more +contextu alized +con acher +comman do +ck k +chin nery +cent aurea +bünd chen +bv n +bry ana +bole lli +bit shares +bish ley +bisd pride +bevac qua +best wood +barbar alee +bar rowland +arnotts dublin +arac elyar +antondu beke +alyci adeb +akl council +ah vaz +activ ated +aa sia +ãĥ ĺ +ãĤ°ãĥ©ãĥĸ ãĥ« +you do +y cat +wom mack +wgx anews +way laid +w bell +vu ght +vel via +v fm +ut saf +uss ari +us by +tur ku +tunaji bu +tran shuman +topo data +to ka +thedaily meal +the very +the out +te anu +tc prepzone +tas si +tacho graph +swis consin +swi veling +super coppa +ster k +steakn shake +ss aint +sports fan +splatter house +sk filmsofficial +si akam +scary farm +sar iska +s good +run ton +ross o +reverber ations +resh mi +r ba +proto zoa +pil son +ouri f +orientex press +one way +on ow +ok han +ofrac osmetics +oble zada +o group +nuclear weapons +noki amo +ne sters +narcissistic abuse +nab ard +n ta +mosh pit +mispron ouncing +miam il +mi va +megal omania +masi ello +mariano divaio +mar ve +ma uk +li sel +le azes +lamp man +l alife +konz ert +kin donesia +kh ater +keyand peele +job st +jeremy mjordan +jal ouse +itsc old +it done +ill ys +house trained +hf ma +hel ove +hay u +hann ay +guer tin +guay abera +gu sher +gaurav kapur +gary gulman +fu thark +fire pro +fil mand +fil lup +fansn stars +f pack +examiner com +evo shield +et ang +ess ington +en eu +eli zab +ear mark +dru ce +dissoci ate +diag ne +den isle +deb icki +de sfile +dau k +dam es +csic yber +cros stour +crafty chaching +cr rc +conversation us +cole tti +chy trid +chriswe id +cheri e +cbit weets +career sin +car land +campus rec +buch ner +brain tumor +blue tones +birth of +bio geochemistry +bh or +believ eland +be j +be greater +bao babs +baftac ymru +av ene +attan asio +ater ra +arrow fieldstud +ard ell +archenemy metal +ar or +apoorv amehta +anticlock wise +ans gar +and en +alto adige +alo st +al lots +af low +ac w +a story +) !!!! +ðŁĻĬ âĿ¤ï¸ı +ÑĢоР¼ +zu baida +z ima +z ager +yong bosch +yade j +wood side +whati sschool +wf my +wewill waitfor +war ford +usu f +tu chman +the fall +th fan +templenew sam +temple ogue +tat raffic +t dr +sy ne +sucr alose +stan dees +st ites +spon ging +spi ffing +sky music +skew ering +sit rep +sister ship +sis sel +sin éad +shog goth +sho b +seed sman +rspca qld +roy blunt +robber y +river plate +rin th +ride the +ric asoli +renthusi asm +re spo +re man +r anco +q ala +pr iciest +pr fm +pomer ania +plu n +pic hincha +peperon cino +pent ennis +pay in +ott weather +nom inet +national nurses +na ves +n xi +n gah +muslim pro +mud died +mu ffed +mon ark +mol lison +minim ises +mic t +me lectro +maxim alism +marn grook +macau lay +ma or +ma gri +m wu +lu zzi +long boards +like toknow +lia ise +lev ada +lauren cimorelli +lan us +lan dish +la valier +karyak arta +k len +johnny yongbosch +johncross mirror +jessicaes anchez +jeong min +jeet music +itor loseit +indign ities +ill s +il div +i bang +hox ha +hol lo +hoce ima +ho bia +high ton +hat man +h ici +h anc +gosp artans +good hew +go sox +git ano +gg m +gamesof thrones +four teen +for throad +folger library +figh ton +fi bra +ever son +ev att +elton john +eli eve +ele mento +eel grass +dream theater +dog ge +de gla +cy mo +crush ingly +cott ons +cot ler +cor m +co stal +can ari +caf o +ca cha +bushy park +bryan ferry +brun sw +break room +bow fishing +bishops stortford +be safe +bav o +bau ch +band olero +badrin ath +bab un +art mag +archer fx +am far +allman brothers +alan ritchson +ail ity +ager wal +ae ther +adul yadej +ace comiccon +ac rrm +!! âĿ¤ï¸ıâĿ¤ï¸ı +ðŁķ £ +ðŁ§¡ ðŁ§¡ +æĸ° èģ +ಠ¦ +Ø§Ø ³ +zu a +zar qa +zapat illas +yakima valley +xia olin +we bane +wa hey +w ttc +veto ing +tribecaf ilmfest +trex ate +to kor +theoxford mail +than ol +tend ance +tatt y +t lu +surviv alists +sty lis +stjohn su +st fb +st ard +spre ston +spinal cordin +sp ry +sop ris +somni um +smil k +smart data +slam online +skel os +signore lli +shi rey +senator timscott +scrutin ising +scarlet blue +sandra bullock +ru bai +ros setto +rock tober +ro dt +rip curlpro +rebu king +rapi ds +pil chards +phenom eno +pent lands +pe ee +pau los +param par +ot ps +ot actic +one dog +on gole +nh t +newton grange +naj at +n itec +n dio +moff it +mo bbs +mitch albom +min oso +mid south +mcgau ghey +mc garrigle +mc callister +mb el +mal tepe +love golf +loqu acious +libraryof bham +lea hey +jac are +j np +inthe game +imangel abassett +id leg +i heid +holly gshore +hel ado +he gg +gu ld +gro ysman +gri schuk +gri dman +gori zia +gif ford +gd p +fy vie +fu kn +foo dd +focu srs +fender gbi +f ä +f te +evan oblezada +ev amarie +esch eric +ech t +don tour +do illon +corn forth +complement arity +co drington +citywinery nyc +ch id +cbs boston +caball ito +brown fields +briano driscoll +bor dir +blakk rasta +bj t +bio steel +ber tsch +ball point +avalon hollywood +arma geddon +ap atosaurus +andy milonakis +an jar +alham bra +al ising +ade deji +ac cultur +abal os +ðŁĺĤ ðŁĺĦ +ðŁį IJ +å Ķ +न ह +Ù İ +Ñĥ Ñģ +zel dab +zap atos +zad ran +za idan +wubb zy +wrays bury +wolfies mom +wie demann +weare hiring +water course +w tem +vitamin water +vaxx ers +vander griff +unfail ingly +tun dras +tri vera +token pay +thisi sanfield +terrible towel +symbi ont +sunset sunday +stro mbo +stra bis +some time +sohail khan +smu sh +ski dder +si we +shoes andcare +sclero therapy +scint illa +s book +ruben diazjr +rome e +roisin murphy +ri singer +results with +restom od +r si +priest man +pre ll +po es +plate aus +plan te +pi va +perman ency +pastit sio +paran al +oar fish +northumb rian +no en +neil ston +ne mor +national walkoutday +national agday +n power +myfox houston +melting pot +mcal pin +marque elv +m ggs +lu co +live feed +linds borg +like fatherlike +lid strom +letsmakeit awkward +leigh onsea +koreand rama +koll witz +kitchen decor +ker naghan +kappap hi +kail as +jomalon elondon +jemi ma +inst are +ik ha +ice bridge +hrithi k +homen agem +holy grail +hockey day +henry danger +hel lier +har av +group ama +grote squely +groes beck +gri ddled +green smith +gilli ard +gi ggly +ghet tos +ghe gola +fro gotd +fraterni zing +for nature +fil oli +fb family +falcon ers +entang led +encu entr +el p +ed fest +dri g +doro thee +dg trends +des lauriers +demp sie +deid rick +davematthews band +con tiki +comingof age +coming out +chrisweid manufc +cfc w +car vell +can tone +camden fringe +c gk +bull whip +bu isson +bro der +bran scombe +bel ang +beg int +be de +ar vato +ann one +ane williams +andy priaulx +aly pse +agerwal nidhhi +af arm +ack worth +abq topes +.. ðŁĺİ +ðĿIJĦ ðĿIJ +ë ģ +ঠ¼ +ya ws +x dr +work and +wm of +wern her +vo to +vel opark +vaness am +us ky +tto vino +tre ally +time and +thr onged +the palace +the eric +tex cellence +sustran sscot +superstar life +sphin xes +speight stown +smur f +sma sters +sin uk +sang i +san ni +ru sted +ru brik +roger k +rewar i +rest lers +repar ative +rajar am +punc turing +pun go +psy chol +plagiar ised +phytoph thora +phoo ey +peace ably +pan ax +paleo diet +oni ght +on oa +offici alle +o intments +ns ford +no ell +niku man +ni alla +nag ambie +nadin en +music as +multi spectral +michael berry +metal smithing +melissa fumero +mca chicago +max xie +max ene +maris sam +mari oc +mambo ibiza +mam mu +mac fellow +ma ww +lou vre +lat onia +lar gest +kor ah +kin an +keri keri +keepit simple +ke tut +jun gen +joom eara +j pj +it snick +ifam edia +hoo gland +ho per +high speed +healthe deng +h Äģ +guy sssss +guy fawkes +gru ss +gop chairwoman +gl ers +gil strap +general issimo +futu ro +fun sies +forthe boys +fo sse +fivb grandprix +fe dup +fav ell +ey re +exhor ting +excel ent +elix irs +el wha +ein mal +eh ret +dulwich college +dro zd +der ide +de mic +dcy oung +danielj gillies +cu la +corte ge +com unica +ch mp +canvas lms +cant lie +button willow +bu low +birdwat chie +at oning +asi am +as ki +apay ao +anand an +an amosa +am ily +alyciadeb namcarey +ale es +ah m +action shot +a ung +ðŁĻĢðŁĻĢ ðŁĻĢ +ðŁĺ©ðŁĺ© ðŁĺ©ðŁĺ©ðŁĺ© +ðŁĶ´ @ +ðŁĴ ¶ +ìĥĿìĿ¼ ì¶ķíķĺ +å ŀ +ൠĤ +z aa +yyj traffic +y aaaaaaaa +x sos +world boxing +will ink +west leigh +vy se +vs galang +vichysso ise +vic times +vesti gial +un raced +ul na +trifon ov +torture report +tir so +ti j +thul in +the deal +th ach +taylor r +tam aqua +ta rell +ta etiseo +sun co +sullen berger +sug andha +stu ka +steve woz +sp aul +sophi a +sl b +skidmore college +short coming +shan ked +setag aya +sere vi +sand bars +re purchased +re publish +ram el +que ally +psycho drama +premier boxing +portsmouth news +pin kies +phyton utrients +pen ile +ot chouston +oj ala +ny mets +non white +ni a +neuro ma +musik messe +mu stan +msamber priley +miw band +me ade +maravillo so +mar um +ma kit +m no +love twitter +lm h +lik ability +les den +kn apping +kingsisland pr +ki ess +ki dron +ju ans +jam ala +jalo ta +jad yn +jab iru +irish town +infiltr ates +immigr an +ih or +highland games +her p +hard scrabble +habit ability +ha gd +gro ynes +great news +gra phology +glit ching +fraun hofer +foot long +folkl orist +fo ire +fernan dez +fener o +fe strail +el mas +eileen fisher +dolant win +diam antina +dev days +cur cuma +cor tazar +chri seriksen +chand ana +cha il +cather iner +bu tti +bro g +bro berg +bosco bel +bo dog +blaz ey +bestplace to +barist alife +bar ata +ball ito +bairro alto +bai x +asdfgh jk +art fx +anubhav sinha +ant lered +amo slee +aly th +administr ative +ad uri +above water +(... )" +ðŁĸ¤ ðŁĴĻ +ðŁĶ¥ ðŁijĢ +ë² Ī +æ ħ +⾨ : +à¸Ħว าม +zul te +yogaw ith +ye atman +y ro +x bt +womens rugby +wet plate +wes farmers +wal gett +vu ka +vivam exico +varad arajan +valla dares +up stair +under write +un palatable +uc merced +u wi +twil d +ture brevi +tri estina +the shoe +te bbit +ta ac +ste ddy +sr iti +sony liv +son iam +soli do +smar ti +smacc dub +si ron +she eler +self harm +se alth +scrit turebrevi +sar va +sapp hic +sa hur +ron din +ro anne +ris ner +rie hl +rec ant +rct council +pur rp +proxim ate +post workout +phylo genetics +photonic swest +opol ice +op ined +nucle arenergy +nh w +nan jiani +n itta +mom an +maî tre +mar ren +man asa +lumix uk +lu pica +lt ps +liti gate +lenny henry +lam arche +kol ha +kin taro +kilo watts +keen en +k holi +juju bee +jojosi wa +jan ma +jackson rathbone +itv racing +intu os +impe x +iifaut savam +ig bt +hou tbay +he yl +hass le +gy rating +gur don +gro ome +gre ns +goldcoast suns +ghou li +fu gao +fri gga +fo go +family life +factor ization +ex hume +espe jo +equi distant +eccentric ities +eb k +e ha +dunman way +do rel +dharma puri +develope ment +devel o +de jean +dating me +crop sey +corr ingham +cord ner +coquet tish +coon skin +conoc er +concert goers +colle tte +col ling +cj fl +civit avecchia +chri shol +cherry belle +catechi sts +carol peletier +car ifta +brett kavanaugh +bre cht +bor inqu +bol lington +boden kirk +bo sie +blu st +black bird +battist elli +baser unning +bad astronomer +at aris +andrewr sorkin +allthings mayo +air max +ad sby +ac tres +] ]] +! âĿ¤ +ðŁĶ¹ # +ðŁĩ³ðŁĩ ¦ +ìĭľ ìļ° +ì½ Ķ +人渣åıįæ´¾èĩªæķij ç³» +âĢ¢âĢ¢âĢ¢âĢ¢ âĢ¢ +zim cricke +yuvan shankarraja +ye tt +y entl +world market +will sasso +wil shaw +whoare you +waters meet +vit ry +vac s +v lb +u hq +tun khan +tu sh +tren tharmon +tre ta +tre law +tranqu illo +toki doki +tn r +tichin aarnold +ther anch +the di +ter tulia +tak aki +stipul ations +st mike +spla yer +sphoto grapher +speci osa +sp offord +sor b +son tv +so ichiro +so dmg +sn sd +smal en +sic her +she believes +shari ef +sh anthi +salom ons +salam one +rhein metall +ren sen +regulat ory +rc pch +ran dol +ran cocas +power women +papato etoe +oce and +nothing to +nomin ate +nebu las +mom an +mish con +michael kiwanuka +melani stic +me ddled +mckin nie +man sha +malte se +m hy +ly ly +ly cia +loveoz ya +lma oooooooo +le za +lau ch +la boe +kyle hebert +ku cing +kpop fanart +kee so +ke ema +kal uuya +jo d +jim ma +jb laudio +jau zofficial +ja ic +inver kip +integr ative +indi rty +hypercholester olemia +hoo m +hertz berg +herstmon ceux +helle borus +hau ght +h wee +grou lx +gre gy +grazi ella +graphic art +go bruno +glit tered +gay ness +game tography +ga hhh +foot patrol +food processing +flag stones +femme fatale +fc bb +eun uchs +et cher +er mac +dynasty warriors +double days +diaphrag matic +dear rings +co ing +chou chou +cho ksi +chini ot +castle blayney +bu cherer +brig man +bridesma id +br aless +bo tsford +bo ffo +beth houf +beth behrs +bbc thevoiceuk +baw l +base uk +ba sir +avoy ages +assi me +animal league +amit ri +affl icts +ad ded +ac yl +.... .( +ðŁĺ» ðŁĺ½ +ðŁĸ į +ðŁĴ« # +ðĿĻ ŀ +âĿ£ï¸ı âĿ£ï¸ıâĿ£ï¸ı +âĻ¥ ' +âĻ Ĥ +س ÙĬ +z enda +z ard +win ema +win drush +whites ands +west port +wash stand +wa haha +vander cook +umen yi +uc can +transport news +todayin moviehistory +tod dr +to kel +terry androb +swind led +sw oll +super jail +sty mon +stou dt +star power +spir iting +sol ares +smo kies +skip ton +signi fier +si or +sho lom +shaw bury +rtn ba +ren as +re awakened +rdeye girl +ranvir shorey +rais ina +ra on +quality assurance +qu agli +q pf +py rac +pu sser +prince charles +pra yuth +power trains +plagi arist +par wan +pap uan +out stripping +ot itis +or ugby +open cl +ol sen +officiale gl +o sus +o brador +nstom ar +nor bert +non compliance +nic hole +nat pe +nam usic +my dream +mv v +musketeer seurope +mor u +ml rs +miy awaki +melissamc bride +md ant +mc com +margotwall strom +mace wen +london artfair +li bb +lec ito +le then +lb ma +ksn news +kr b +kir choff +joh ri +jay araman +jam ul +iz abela +inter connected +ingrat itude +inew snow +ik onic +hy lan +house breaking +hard worker +han go +ha ston +go wild +glo ves +gie thoorn +fried richs +freshman advice +football news +fetch am +fau recia +farm boy +fan chant +ey f +eric acampbell +em me +eh sa +egoti sm +dot tir +design indaba +deep kaur +cro ssett +cps reds +cou sens +cosmic consciousness +con ine +cleliam ussari +chin ensis +chee ch +ch iri +ch aco +cast ler +care bears +cap ella +busine ssi +brack ins +bor ic +belle mare +beati fied +bal ke +bak re +bad y +atta way +astor ga +aspe cies +arsen ess +ari ff +angel ito +amiz han +after image +ðŁĩ¬ðŁĩ§ ðŁĩºðŁĩ¸ +ðĿĹ µ +à¸Ńภ¥ +à¤Ń à¤Ĺ +zy ı +zaf ra +you thre +x ux +x team +whereyou live +wex med +well l +walk highlands +vla ams +vienne tta +uro logic +une qually +udemy coupon +u gur +twitter takeover +trethe wey +tre me +tororo sso +ti sl +think like +the cab +th feb +tain ting +spo hn +spho enix +sp cs +soup kitchen +snod land +smith h +slo dge +sin thenfl +si dor +shoes ource +seg mental +sea weeds +ryth me +rothschil ds +rejuven ates +redress al +read yourworld +re solve +re med +randolph harris +proprio direct +poles itter +play style +pe ming +pdx music +pal aro +oz ora +oyin bo +oni official +omnis cience +newcastle herald +ner diest +national parkcity +n acre +min era +melani escro +mam etz +magi stral +live better +line age +li even +lau l +lab one +kentish town +kas o +jal pa +iran air +inter dental +ing time +ind welling +imple menter +heal ty +ham idou +hachim an +graf ana +grab ner +ghm conline +ga un +fre se +fre enas +fr yar +faw cett +faroo qui +ex hort +espn seattle +ep aul +ent is +ennis more +enf j +enantio selective +disen franchise +dik ko +devo to +dev camp +des jar +daniel agger +cran borne +con tends +cob ley +clun kers +cincy childrens +chrissy costanza +ce aser +cau dwell +bulgar iofficial +bu har +bu ettner +bow ler +boots riley +bi aly +bhi da +bha sk +be zier +basse terre +bac co +as kia +aro adshow +annex ing +and ys +amar jeet +am cor +al stott +aj green +against trump +afri end +ðŁĽ Ģ +íģ¬ 리 +âľĬðŁı¾ âľĬðŁı¾ +âĸªï¸ı âĸªï¸ı +áµ į +zen ko +writing prompts +wowo win +wood carver +won line +wh ith +weare james +vs nyj +vortic ity +vhong x +ur sul +uo chester +treasury mog +tra kker +toad flax +tivi dale +tiki barber +tick ner +the bull +teil hard +team got +tash ir +take control +swee zy +survivor cbs +sur lerouge +stra us +stay ner +so ad +silver point +shor thai +sho eracing +scott mgi +scottmgi mple +school pr +sc sd +saw dust +safdar jung +rugged maniac +rudi ger +ri aan +real lisamarie +re doubtable +que rel +pul sen +pul po +process o +pre mratandhan +prag matist +powder ham +peplo e +pe ine +p kane +oul son +op é +ones i +one fc +no dy +nishi oka +naves ink +nationalschool walkout +nar umi +nan oro +musee orsay +mont se +misez surlerouge +mc tiernan +mc elli +mark azi +man tou +mal vina +maketheroad ny +mah in +luc re +lon grich +legionof boom +le zz +lar wood +kum in +ku so +ko diaq +key west +kaz emi +katelyn tarver +k ourt +juli eg +john hurt +jason witten +jam my +jaars veld +infl ame +ii hm +ian ism +hum buckers +hon ble +ho ps +he dy +hair and +gy r +gurum urthy +goal u +gla ube +gin acar +geo coding +gator s +g anim +fre yr +fotogra fi +fotogra f +fil des +fam ines +fac u +extru ding +evil queen +eve myles +eu mundi +emis saries +echofox gg +driving test +despon dency +dec ile +de dal +dan ticat +cran field +cosmopolitan uk +cc ts +care mark +call ington +bur ley +bu uuuu +breakfast show +big gardenbirdwatch +bi ju +berg sabc +bb qing +bacsin szky +b ner +b inning +ashley y +app same +annex es +anat ol +am bre +al anal +akint ola +ahe gao +aflat oxin +af tv +acade me +abu dapest +abi asi +ðŁij½ ðŁij½ +ìľłëħ¸ ìľ¤íĺ¸ +âĿ¤ï¸ı ðŁĻıðŁı½ +âĵ ľ +zimcricke tv +ye ssssssss +ye sh +winston churchill +virtu ality +vap il +ur wa +unra velled +umenyi ora +ul aris +turn back +try somethingnew +tou hy +timb ale +the adelaideoval +than q +taf fair +ta imur +su id +sty gian +storm frank +stoken ewington +squad up +socio cultural +scott moir +saw ston +sarkis sian +sa kal +rival sons +ri velin +ras berry +randomactsof kindnessday +r mg +quality control +qu yen +pro foto +pri sa +porsch esauce +podu machi +pete dun +per vez +pe ir +pave ment +pat cham +pasquale totaro +parklife fest +paras ke +ous sef +ni un +never be +nam as +na ston +n intex +mu kho +mosi mann +modern home +mis singh +mis sel +menin black +meg son +mc cs +maz el +manv sale +mal achi +magnet ite +mac cag +lisar inna +leh tinen +l slofficial +kill ingly +ken suke +kat el +kab al +jol yon +jen net +jack posobiec +ja yo +j rees +iz ar +isha an +iran elections +house ch +hou sings +hol ls +health workers +gta vonline +green eyes +gover ner +gok delivers +gn cc +gi meno +gg ler +gesh wari +gene editing +gay oom +gar ment +g anti +fris bie +fo ad +fil mon +febbra io +fam i +fad den +essence mag +du sek +dread noughts +dor ton +dir rell +desp ising +daw on +damas o +dam bro +cumu lus +crop top +cri ssy +cre mate +contextu alize +coach b +ci ana +chir ality +chester fiel +char lotta +ch atime +cag nes +cab ourg +bu tan +british swimming +book bag +bir bs +big bro +bibliothe ca +bc rich +bate aux +baden horst +ba official +b awi +aur icular +arbuth not +ap assion +ann unziata +an ker +alista iro +ali bhai +ale so +aj et +ahar u +ac ts +. , +- ______ ++ £ +ðŁĺĭ ðŁį´ +ðŁĴĻ ðŁĴķ +ðŁıİ ï¸ı +ðŁį ¡ +ðŁ¤® ðŁ¤® +æĸ°èģ ŀ +âı º +ಠļ +youth power +wick steed +west fiel +wer un +vincent price +vani er +uu uh +ug bu +ucc ello +ther ing +thal afan +th jan +tele m +tast ico +su kha +star day +stand ridge +st ello +st austell +soun dre +sou my +sli berty +sin isa +shill ingford +she sh +shak ila +selet ar +secur itas +schmel ing +sau lo +roger craigsmith +ri obamba +revi vals +regal os +reas ures +rapha els +q erim +publi sh +pi got +phi bes +pe muda +pavlo vian +pau lam +over ran +ontari an +of o +occi dent +ny tt +nintendo ds +newpor trfc +neko atsume +nd as +multi role +mr cp +mo preps +metaboli ze +meccan ica +mccre esh +material design +maqu illa +mad bum +ma bou +m sleg +lolli pop +letoy aluckett +leaf ing +lady vols +l dap +key dets +kevin saunderson +kesh et +kentuc kiana +kartika aryan +karak orum +k sis +k itting +john mellencamp +jo leon +jess ore +jay shree +itstaylor yall +is at +invulner able +inoc ente +ingen io +i ys +human os +hot ti +hive mind +high am +hi fk +hall er +go local +gauth am +future learn +fun dingh +fr n +forthroad bridge +finn art +er vices +er obinson +enthr onement +ent wick +end om +earth skyscience +dug gee +drar aut +don thug +dj mo +dis aggregated +dhy ana +dhau la +demar ai +decep tions +dayof giving +dance wear +cryp sis +common ground +co digo +city pgh +chin cha +chican ery +cat ala +carolin amud +carolinamud cats +cal ex +cac ao +c vw +bulgar ians +brooke henderson +broad mead +bois set +blob fish +bing aman +bbc sportsound +bau douin +bal un +ba ws +av aris +auditi onees +at vi +at ena +aravindha sameth +arac ely +apol icy +anthro pome +andy mientus +and all +am blin +agricul tura +ado or +ac nes +above ground +# % +! âļ¾ï¸ı +ðŁļ¨ðŁļ¨ ðŁļ¨ðŁļ¨ðŁļ¨ +ðŁĺĺ " +å µ +âĪ ļ +Å ¾ +á e +wustl med +wil ken +wel burn +wal lowa +vra iment +var num +ur j +um be +turtlen ecks +trump budget +tri pa +trape zius +tingu ely +time splitters +thisisd urham +the hip +te sori +tb sofficial +tachi kawa +syn tactic +syn ge +sweet land +su mon +sten ting +sober ly +si val +shop if +shields fc +shield maiden +seren issima +seish un +secre tos +sci acca +scand y +sa uro +s diner +ron johnson +rep kevin +rear guard +real politik +peter greste +pet ard +pamban sa +p mik +osh park +oneoh trix +one ofus +nx umalo +nw l +northant sccc +no war +no rell +no ire +ni mble +neg ley +ne sian +my nah +mwa haha +musicis mylife +modern day +mo zar +mo har +mlb pa +mind q +mic mac +mf is +metho trexate +mari ane +m frost +len zie +lari ver +ky lar +kut ter +knock aert +ki shin +kak ai +ji ah +jel utong +it carlow +iron monger +il ga +iconocla sm +hen ery +hell spawn +haworth ia +har bi +ham bly +hail u +gyeong ju +gra ef +goooooo ood +fom ent +fo glia +fel ino +fam oso +ey vind +exorc ising +epi thets +elli son +electrocu ting +elas mo +e hehe +dou rif +do xford +di bang +de mentor +cotedazur now +con rail +compar ator +colson whitehead +cat alu +care en +camer ino +bur se +bry z +breit ner +bledis lo +bla ise +biome thane +ba iser +b amp +aver il +ambassador power +all mets +al acrosse +ak utagawa +abigail spencer +ab dn +// : +ðŁĺłðŁĺł ðŁĺł +ðŁijijðŁijij ðŁijijðŁijij +ëĭ ĺ +æ£ ® +âĿ ¦ +è dre +yan u +xma sparty +x lu +wy prk +wg st +western sydney +wak ana +w tam +vizi anagaram +vers ini +vander waal +tunkhan nock +toyn bee +tie out +te phra +sy nucle +sy mmes +sun care +sub in +sub contracting +stre ssed +stopthe debttrap +ss w +sper m +spee die +soci ability +small youtubers +sm x +skan sk +sinu so +shri mper +sheff council +seh ban +samaritan spurse +salish sea +sal ted +s jones +rid out +red bourn +ram shaw +predic aments +pn pi +plo ys +pitch forks +pet tai +pen ampang +pajar ito +otter burn +ot ice +oro ss +one ills +nieu wen +mr b +mo esha +mmmm mmmm +mat us +ma homie +louden swain +lipo somal +lal af +lag ana +la vette +ko bler +king and +khay al +kh ri +kat wijk +kai sen +jun ia +jelly man +jeff bullas +jash n +iri sd +ingh a +hire me +hey sel +helm sdale +hake em +haber mas +h ounding +gregg rosenthal +gary leff +garden centre +foto g +forza italia +fibro blast +fell running +fee ley +fe k +eve of +evangel inel +ero deo +er tiga +elo gic +elly awards +elef ther +eg shore +edward tufte +ecol lector +ebon ics +east nor +dungeon master +dragon ite +dig in +dhol akia +dev day +dental hygiene +defro ster +dataware house +dam avand +dal ers +cu ppy +cu entas +crew mate +colon ising +code foramerica +clip stone +citiess ky +ci at +cheese cake +cdn crown +candel ario +bunnic ula +bron wyn +bra edon +boi leau +ban co +bal alaika +attic arace +atticarace wyprk +ary newsofficial +angelsof fur +and h +an ick +amera action +alli ums +ali assime +ac rif +ðŁĺĭ ðŁĺĤ +ðŁĺĤðŁĺĤ ðŁĺ© +ðŁĴľ âĿ¤ +ðŁĴĻ ðŁĴĸ +ðŁ¤ĵ # +ëĿ¼ìĿ´ íĬ¸ +âĿ¤ï¸ı ðŁĺĩ +âĨ ĺ +Ú ¡ +wom e +wc pd +wall onne +w le +ver ti +vel ia +v ung +urdan eta +un likeable +u mia +tur l +trail head +toyo tan +the oph +tarot cards +tanan lam +su hoday +steely dan +st century +sport sturf +spin offs +sphero id +sper ry +spartans will +smo yer +sk j +sion yc +sch latter +sat am +san jac +roman ian +reminiscen ces +re animator +raw ling +ra tho +priyadar shan +prabha karan +po rec +pi tha +peoplewhom ademy +par minder +p liz +p ities +onem illion +off y +noril sk +nor rington +ne tti +ne ma +nav ys +national aviationday +mcne aly +mauriciom acri +ma ssing +little dragon +liss at +lin go +lead byexample +le ix +lar ia +l bo +ko tha +kho ya +khan om +kaz en +k radio +jyr ki +juma anewilliams +joe mantegna +ji v +its ellacruz +it trivedi +ipp olita +ic tsi +hoch stein +hipho pawards +grow ur +grac iosa +gloom haven +gag an +fore seeing +fi lets +feature less +fa ial +eviltwin brewing +er au +ei rene +edge mere +ed surge +e are +dracon is +downtown ptbo +dj clue +dine o +dem at +del tron +decrimin alized +dante basco +crou ches +cra shed +cr inan +counter acts +contest able +cnblue gt +citrul line +christo logy +chris bosh +chas ms +caring bah +car ll +bur rage +bru ticus +boxer vijender +bo water +bo letus +black enterprise +bi asi +bear sden +band ha +baby go +b br +arvindg aur +arrivat w +ar asan +apic ulture +ant y +ali zafar +ali stas +alex constancio +al imi +ajin omoto +air fields +acci on +abar are +aar ohi +a preci +... âłĢ +ðŁĺģ âľĮ +ðŁĺ½ ðŁĺ½ +ðŁ¤£ðŁ¤£ðŁ¤£ðŁ¤£ ðŁ¤£ðŁ¤£ +æŃ £ +ãĤ¤ãĥ « +ãģ¨ ãģĨ +yyyy yyyyyy +woo duk +wi bowo +wh on +warcraft movie +voter suppression +tud denham +truck n +trav ails +tom ska +timm is +the wiggles +the beauty +terr r +tal kis +ta ja +stu l +star flyer +stam u +stalag mite +st aley +ssi de +ss ongs +sp ahr +slow downs +shil don +shi rayuki +sexu alised +scul ly +sch l +sar re +ru pal +rn zaf +redhour ben +race ways +ra ichu +queen sof +que te +promo cional +premi xed +practic als +plan ica +ph rom +paradi so +p mt +over stepped +or loff +nz ta +na an +mplo yers +mosthandsome faces +mo edas +mis dproud +mey cauayan +mc vicker +matt kemp +mak ura +magic ofthecup +maci ek +love actually +lipo ic +li mber +levi mitchell +lake house +la dan +l bci +kul in +kor net +knu x +kentu ck +kab inett +ka strup +jun hong +jone ss +ji yala +jak el +jaimie alexander +j ats +ipp r +in may +in ji +il ja +ic tafrica +hy bpa +hour fitness +hoh mann +hare woodhouse +h lt +gro win +gramophon emag +graceand frankie +glo ttis +gigan det +gic lée +ger ri +gcse results +games aus +ga shes +funny pictures +fron tieres +friday morning +fo ard +fit i +fish mas +fi stral +fc zenit +event sin +esp anya +emporio armani +el ene +e comm +dre mil +don no +dof theweek +do ye +do che +dh anya +dela hunty +decarbon ization +dd avis +dcyoung fly +corruption case +commerce gov +co darmy +co creation +chi d +cf m +cest lavie +britanni c +body suits +boc cioni +be evers +be eni +bbc shropshire +bal aam +bad stuber +aspr illa +arth us +annam arie +animal liberation +alistairo vereem +ab attle +ðŁĶ¸ ðŁĶ¹ +ðŁĨĺ # +ç¾ İ +wgn america +west van +wad don +wad den +vrou wen +victor ial +valeri y +valen za +v rush +v inter +un ti +u hs +tx ts +ttm success +tre garon +tre blinka +train able +thisisco ventry +th acker +t pv +t pas +sustainable finance +string band +spen son +sm day +sk ole +sham bhu +sf ontein +seong woo +se sam +scol ts +sanc ta +sa ire +ross marquand +renew timeless +red hood +ramyak rishnan +quadru ped +publ ically +pra ya +petro ssian +perfu mer +p ccw +ourlandour news +nwa as +nor aen +n anne +myk ki +mykki blanco +musc led +morgan spurlock +monclo a +mm une +miti e +michael h +michael b +metv batman +meer schaum +marcu scooks +marak wet +m wbb +long niddry +live updates +leader less +lan phier +l xc +kup fer +kre utz +kev adamsss +karnataka world +k pae +ju ku +ji ddu +jday golf +jan ka +j ies +hye res +hu sni +hollow knight +ho yne +head house +har laxton +gym wear +gul ates +groom bridge +global britain +gh ita +gere ja +geophy sicist +geno ise +exop lane +eco sphere +early learning +dre we +direc tness +digit als +denti stry +dell matchplay +dark skies +cv payne +counter tenor +coun tri +costume design +const ancy +cn try +cityo fatlanta +chukw uma +cheshire police +cele stino +car freeday +cad dick +c dos +bul losa +bravo andy +bramb illa +boz os +bosw ellia +borough fc +boad icea +bo soms +biz boost +bau hin +ba ars +b sl +avi dson +au glaize +attend ance +asdru bal +ar ar +apar napkin +ap enn +all ornothing +air plan +afl ori +adi m +ab last +aaaaaaaaaaaaaaaa aaaaaaaaaaaaaaaa +:) âĻ¥ +ðŁĺį ðŁijĮðŁı¼ +ðŁijĮ " +ðŁIJ° ðŁIJ°ðŁIJ° +ав ÑĤ +î t +z ior +yon ex +yo ver +yestur day +wun sch +won o +wl bz +web casts +warner music +war re +wale ed +wag i +vesp asian +veliyi dai +var roa +tro ilus +traffic scotland +tmc lebanon +tla que +this ssss +tenov usc +tar nishing +stu dly +star tet +sleigh s +silk wood +sil ves +shiv puri +scr unch +s va +realbobby roode +real sarathkumar +rd mas +race for +r ci +pride fest +podumachi goalu +po theads +pnpi fugao +play backs +plane tor +patr onal +party yy +pale tta +p ssi +os sett +oce ph +neopla sms +na ic +n dy +mut tered +morri ston +min cha +mi igwe +mh day +men ai +mang ler +mah mu +madri distas +ma ham +little port +kha pa +kaz aam +jis r +jimmy bullard +jen ners +jan ia +jagann atha +in cs +il ir +iklan barison +ike auk +i ak +hypere mesis +hu maim +hotel belair +hot chner +hiphop history +hijab day +hepat ica +harvey fierstein +hampton court +hammer down +habit ants +h nn +grand hotel +gra eae +gr ing +ger rish +gab onese +fur freefriday +fss ai +folk song +fo amped +fil mc +fdm group +faw zi +es wari +ed leaders +dutch bros +dis contents +descu bre +der abad +depresse dd +dat ass +da et +cysyll te +craig lockhart +cot ter +com ba +coast walk +chis um +chees elo +chaf in +cha sti +bus er +broad institute +bre snan +bon nici +bol año +bo vines +bledislo ecup +bl ang +bharat natyam +ben it +beabin ene +bar bato +asi onal +areth usa +ar tos +allgä u +ag p +acu ba +* ---- +ðŁĺį ðŁĻĬ +ðŁIJ¶ : +ðŁį ¶ +ìłľ ëĭĪ +âĢĶâĢĶâĢĶâĢĶ âĢĶ +á Ĵ +ਠķ +zay ousaf +youth month +youn kers +y po +y enko +x liii +wigw ams +wal nu +wa aaa +viaduc ts +verkho vna +un principled +typo logies +tor telli +tin d +there stless +thecottag eneedle +the aus +tem pu +taye diggs +tam ana +take five +stry pes +stra ddled +ste geman +st chat +springer nature +sper mato +speed weeks +sk otti +shu gart +sfor good +sehban azim +scoti abank +sch an +scal pels +sc id +sas c +saf c +s music +ru pauls +restor an +razor bill +r mef +purch asable +pu cker +poly hedral +pimi enta +pen arth +paul smith +parachu ted +paci fics +pa wel +ov hd +outd channel +op tout +o ep +novel isation +north fork +noraen pure +nicholas sparks +nevel son +nay sayer +money talks +mi mir +megal o +maz da +marke tability +looking ood +london grammar +lland y +like that +lef son +la gta +l les +korbolorbo jeetbo +kho isan +kam andi +jettison ed +jefferson ian +ittake s +itso knotto +isth mian +im cc +idec tomy +id sa +hoo vering +hitch hiked +her rings +health ug +hand cut +han ap +hammer ton +h ph +h kr +gujaratt ourism +gre u +gla uber +gho sh +gar madon +future past +flo ve +fleet week +fire bombing +ene mas +en ow +ele men +eg eland +ed wi +east ney +east dulwich +drag ana +dj mustard +deep am +d ın +cre ag +cor owa +chuck comeau +chop shop +chloe fashion +catal pa +cas a +cal ac +bun tin +bree zed +boho jewelry +boer sma +bl ine +big gies +bi joy +belen enses +bat themusical +bar ito +balla gha +bal ala +asu ppor +arbor io +ano les +annalyn ne +ame dia +ambl yo +amazon primeday +allegh eny +all mendinger +all black +aaf london +:' ))) +Ī : +ðŁĺĪ ðŁĺĪðŁĺĪðŁĺĪ +ðŁĮ ©ï¸ı +ë³ µ +åº ĥ +âĻ« âĻ© +york cityfc +yemi aladee +x au +wiv pak +win ward +willi ford +whiti anga +wee ping +warriors ground +ver sus +v ly +up votes +tra verses +town head +tour ne +top up +ti money +tho ver +thewe bbyawards +theatre works +thau sen +tb ath +taxi ed +state ivlp +spring fashion +speed ed +social mobility +slur pees +si regar +shiv angi +shawne merriman +sepan x +sch aut +sat ar +sand strom +san bernadino +rose bruford +ri bisi +rhetor ics +retail therapy +requ ena +relax in +raji b +preten sions +pre ponder +pois oner +pis mo +pen tire +par olin +p crg +ony m +offer tory +ode tta +ob ando +not chback +normali zes +norfolk county +nkn kk +neph rectomy +mymorning jacket +mirpur khas +mani than +mal indo +liber ato +leeu win +later alus +lar ries +kirk up +kinder garden +khawar ij +ker kyra +k rac +juli ana +jone z +jax jones +jaket austin +itsti meto +is thebest +ingh all +ine ssa +in oo +illi gan +ii ia +i fr +hood wink +hide outs +hel enium +heck uva +health summit +hand saw +gene v +gan tries +ga ily +ful kerson +fro llo +for goes +for ages +flic omovies +first love +feder man +fc groningen +farmers guardian +eru dition +els mann +el wick +eichel berger +e online +drawing aday +dot day +dock weiler +dit v +dak en +dah le +cy farth +comhghair deas +com hal +cataly se +caer laverock +bottom sup +bobble head +bo hot +bir gitte +bien al +awe bb +avon dale +arte san +arra ial +ar bel +andrew mcmahon +an tron +an el +al let +ai zu +ad mir +ad av +ab ao +ðŁĺĶ ðŁĺ¢ +ðŁĺĪ ðŁĺĤ +ðŁİ¬ ðŁİ¬ +ëįĶ ìĩ¼ +åĩºæ¼ Ķ +ãĢ Ī +zim toti +y cp +wood burning +who weare +waller racing +vijay goelbjp +valle es +usab mnt +universit elaval +ultra wide +ttan dem +translu cency +tori bio +to phobia +tist mgmt +then ana +thecalm zone +the guy +the chri +ter ming +ten napel +team sesh +tai b +sve ti +sur u +sugar daddy +steel making +sta ver +shirt friday +ship week +shaw ns +sel v +salondu bourget +sa kari +running man +ro fe +revolu t +regg ina +recon dite +rc vd +rav allo +pr ar +poly morph +polar plunge +pla intext +pi shin +peter ose +pen ola +p lit +oro sso +one music +off white +ny penn +no tion +ne jad +nc soft +music mondays +moet blindcat +mid sized +mi rian +mer cure +mcgiv ney +mb lue +man ka +make music +mait lis +m ó +ly cée +luci dum +lu minor +lo pes +line less +larry blustein +lab neh +la galaxy +l tf +kur une +krat on +kra it +kos gei +kno we +king angi +kil lester +kaw agoe +jungfrau region +jon culshaw +joe walsh +jin soul +jennie garth +j érôme +irk some +ic in +human ization +hu mph +hoge school +herom anoj +he mam +han sal +ha wala +ha voline +gz chef +great fashionfinds +glo scathedral +getting married +ge tenough +fundra ising +free h +event inglive +escal era +eri eotters +dul cinea +dor bz +dong daemun +demago gues +demago guery +dallas lovefield +da ai +crunch ies +conservator ship +co ple +cla es +cees ay +bur kle +bur kitt +big pond +bi mb +bell one +beer bods +be ghe +baltimore county +backward ness +at le +ascle pius +as ger +angel olsen +anc re +allo ween +ai h +ace attorney +! ) +ðŁĴķ ðŁijŃ +çŁ ¥ +ر د +á ras +zoo svictoria +zo ster +ye she +west la +west elm +welove shilpashinde +vol vulus +vick erman +ur ge +un modified +twit ts +tweetad run +tu bac +trac coon +to kill +thegold bergsabc +tar oko +tan x +tal ton +summerof love +spo oling +so ss +silver stream +shari fah +rosen dahl +roman jancic +re ste +raphaels barge +q k +pur merend +public protector +privacy matters +po ggi +phant asma +pe eth +padilla bela +ot acon +olympic park +olu femi +ol ap +ojh lofficial +ness on +nam eh +n cart +mutual fund +mo sso +mitsu ko +missing people +memo irist +man ick +magne tometer +mag ination +live ira +lets do +leather craft +l enti +kumb hal +kon yaspor +king span +kay lyn +jon lovett +joer ger +janee spenson +iso ara +is k +iq ra +ing re +in coherence +hoshi arpur +horror fans +homoe opathic +hen rys +ha val +gul ley +gre ferendum +grace less +grace e +gra spop +girlsnot brides +ger main +fro mt +fr wy +foodand cosplay +fiel dy +fa hn +end ro +elvis duran +draw dinovember +double think +do sen +dl hughley +deline ate +def ene +de sam +davi dre +d top +cus rise +county sheriff +cor ts +cor sican +congradu lations +con ch +collo ids +col lon +co soc +cleveland artmuseum +circum polar +chy pre +chris martin +cherly chibi +chan in +cartm ell +car hop +canvas print +bra es +bobm ckenzie +bob marley +be ville +baby animals +bab ic +aw ade +au byn +arm let +ar dyn +al tc +?! ?!" +ðŁĶ¥ ðŁĺĪ +ðŁĴķ ðŁĴĻ +ðŁijĮðŁı¼ ðŁijĮðŁı¼ðŁijĮðŁı¼ +ðŁıįï¸ı ðŁıİï¸ı +é¾ į +âĢ ķ +á´ ı +xy mox +x lk +wood smen +wi fu +vis ity +vel vets +vallab h +valent a +v pm +twit pics +tut ti +the sheep +the edge +tand ing +stur div +stan stead +ss wans +southern california +south point +sil ento +shinse gae +shen yue +sa che +s but +ryu jin +rivu let +ripon cathedral +rep lika +rel ph +red mill +rc cs +qu ast +q wp +pro ffer +preston steve +pr yer +power grid +postu late +porto fla +po styour +po ble +pend rive +pal oo +over staying +osteo spermum +non smoker +no son +nithyam enen +nick diaz +new chapter +nav aho +moz con +mortal instruments +mongo loid +mini mathur +mene my +memori alizes +mc cu +max lucado +ma sai +low carbon +long more +l vac +konta veit +kenny g +kawar than +janet varney +inter gender +instagram mable +inge agles +ine ducation +hebrew u +heat culture +harde eville +har tal +hack tivism +haber dasher +green book +gran at +gom avs +glend ening +free from +foto speed +fh g +ffbe ww +fe iner +en dia +ela sto +ef eld +edexcel maths +ea ina +duba it +du gin +diferen cia +dic ted +dec icco +dance hall +cé cile +cuyam aca +cu bi +cru achan +corri eri +com ent +co enen +chen kova +cay de +by all +bur ro +bron son +blue october +bis leri +birdof prey +bio steel +bil kent +bad er +au sk +ast ellas +asian art +asi r +as aa +app ended +andyburnham gm +an diamo +all ard +ale urope +albic ans +afternoon express +ab ms +ab arab +ðŁĺµ ðŁĺµðŁĺµ +ðŁijıðŁijıðŁijıðŁijı ðŁijıðŁijıðŁijıðŁijı +íĦ ° +ë² Ħë +ÙħÛĮ Úº +Ø µ +|âĹ Ĺ +zumi ez +zu elo +zin aida +yuki hiro +yu qi +yajam ana +wood in +winthe dark +wave music +von leh +var de +v ram +uw sp +un thinking +un defeat +tram el +track less +tlaque paque +tion ately +threeday sgrace +thedukeof york +theater shooting +tex ter +tahqu itz +sylve stris +sustainab lec +spir alled +sock game +so di +sk imp +si ii +shot ley +shor ta +sh games +seal skin +sco d +sap hire +sant inof +sam us +sad vocacy +ry sz +ry hope +rela yed +red point +ray hudson +rainbow fish +ra gg +q x +pu zha +pr ange +po ile +ple ssy +play it +penn sville +pen nin +par ijat +p ts +osu wexmed +om ed +ole v +ol faction +odd balls +obli vi +oak engates +noo tka +nnnn nnn +newh ope +nar da +mort decai +mor sy +mor rice +money inthe +mer ca +melaniescro fano +me rec +mawdd ach +mat ley +masji ds +mary ann +manik andan +m pho +lou l +litt let +legally blonde +langu r +lam acq +kri pa +kount ze +kk as +kem merer +kel sen +kc mo +ka ichi +judge ship +jo sse +jame shet +ir ally +iphonex smax +io hk +inconspic uously +hum zayousaf +help us +he tch +hadley wickham +gold mark +go iter +global music +gh um +gau din +fro de +for me +fioren tini +fastn loud +fa im +ee ight +dine sen +di emon +defibrill ation +de vis +dare rising +d pu +cwre ign +curb you +croco dile +counter insurgency +cocoro cha +chuk chi +chil cotin +cf cs +ce berano +carb ines +car dew +captain hook +buck tail +bro se +bre gat +bra sch +blue day +berlin ers +bene field +bar mby +ascri bed +arnold sports +ar rings +angou leme +andy c +amo sun +aman zimtoti +al britton +aj la +adair sville +acre ative +a beach +_ < +. * +ðŁĴģ ðŁĴķ +åĵ ¡ +à¸Ħ à¸Ļภ+zamalek sc +yearend sale +x vid +world populationday +whi story +wen z +we got +watkin sville +wak awaka +votol atino +venew shoes +ve iga +v amovie +uk baseball +ud murt +tw ane +tre sor +then ff +the writer +the andy +ten ille +techno gym +tag oe +sun ne +stu bai +sto ically +squig gly +spreck els +speech writing +spayand neuter +shon ours +sharman joshi +semic ircle +seed and +scre es +scott stapp +school sweek +row ley +ring side +reli e +relent les +rb cs +rand olf +quercu sbooks +public history +pu tte +proudly southafrican +pol lin +pod genie +pli moth +phy no +oligo poly +oil price +of qual +nick swisher +ne vik +nay ana +mis si +michaelberry sho +mccul ley +map ada +man ische +man er +mai er +magin ot +mag das +lun ney +li zation +li ong +lewis and +len sed +le hrman +le fort +kurune gala +kof u +ker lin +ken ingau +kar ap +justi fied +jhope day +jay co +ir ama +infra sound +impact live +illi p +i spo +humaim amalick +hei ji +hahaha aa +hage mann +h unie +guys borough +greenvill enews +godre j +gen tiana +gautamrode team +g burg +friday flashback +franken heimer +fo ibles +femto second +esche ws +epoch times +ek hu +ei en +dos box +disney d +diar yo +di methyl +deep ellum +debla sionyc +debat er +dar kangel +dar cis +dan dekar +dan abrams +da hir +cull in +cri velli +cour cy +cor tel +chrisley knowsbest +ce dro +catter all +brad burn +bol dinsider +bigbrother ca +bic ameral +ben tiu +beforeyou exit +b fly +b alian +army tage +arjun official +anyan wu +ameli o +alo vely +ak arjunofficial +aj pur +ah w +acon cert +aadhi official +ðŁļ¶ âĢįâĻĢï¸ı +ðŁĴĻðŁĴļ ðŁĴĽ +åĥ ı +âĹķ ) +à®İ ன +yu ill +yha official +wimbledon final +wc ba +water logging +vo lim +vampire academy +uru zgan +un drip +thum or +suppos itories +stopthe violence +starwar sep +spe ttac +spati o +space shuttle +sono ita +somo sporto +so hyun +sneaker snstuff +slee man +sko vic +sin spired +sil vassa +si vers +showaddy waddy +sh ingen +sen ri +sco tathletics +ru cian +research impac +realjoey b +re ema +re ath +ralph macchio +rail head +pre ller +pre ggers +pottawat omie +pick oftheweek +peter sson +pas es +paris motorshow +over time +ogo pogo +nw lc +ni da +nau i +msf tedu +mr g +morethan agame +mic i +mccor mack +math letes +marriage equ +lur kin +love bts +look good +lo ben +liversi dge +line arly +le eu +lamar re +kre we +kop ing +ko gi +kn abe +khu tba +ken nys +kas sab +k vapil +k rates +joel creasey +ji ocinema +inv ar +intu itive +inad missible +impregn ates +ilo gical +iced tea +iam abotanist +holiday home +ho de +hard tail +gun barrel +green music +gre vy +goooo o +gas cony +flix bus +fatin sl +far uq +evi leye +esz ter +ent eng +enni stymon +el itch +dre ier +dranath tagore +do ddington +disdain ful +digitali sierung +di af +dest iney +del sin +de itsch +de code +data storage +d pb +cor dle +congression al +con script +community shield +commis so +clo t +bud worth +bor ovets +book sin +blue throat +blackwomen didthat +bir atnagar +bernar dsville +ber chem +beau chemin +be ppu +batmanvs superman +baseball canada +backthe pack +back tracked +ba ard +annamari abiasi +angel alan +an iche +ame v +ambigu ities +alek sei +akar ni +ahs freakshow +ah ack +acar thy +_ ^ +[ +] +ðŁĵļðŁĵļ ðŁĵļ +ðŁijĢðŁijĢ ðŁijĢðŁijĢ +ðŁ¤£ðŁ¤£ ðŁ¤£ +ë¡ľ ìłľ +âĺĨâĺĨ âĺĨâĺĨâĺĨ +âĢ¦ âĢ¦.. +âĢ¢Ì ģ) +youare loved +ye ds +x pl +wr oughton +waiting for +w luk +vic h +val halla +v tv +usopen cup +un sorted +uk volkswagen +turbo chargers +trout dale +ton na +to cco +timo thee +the bone +team ceec +suj atha +sto renvy +stje pan +sti pes +steen burgen +stay gold +ssa ints +sou tar +sno whour +sm sp +sli ghted +skotti eyoung +she sa +sha ab +seo inguk +seab ikes +se yi +screen sho +sant elli +sagrad afamilia +sabhar wal +ros set +regre ssions +proto star +pricewaterhouse coopers +pho bla +pele liu +pegas i +parathy ro +oo hhh +ong ata +oneon one +oli fants +nkem diche +need leman +my writings +my my +ms b +mr inal +mou f +molly mauk +mmun ol +mirren fc +min ichiello +milit are +mic cosu +metro tech +meridi ana +mee ce +medi ations +mead er +manu ka +maith ili +maccab iah +luch alibre +laurel schuett +lam lash +l ri +kripp arrian +kore aboo +kl n +ke phart +kang an +jazz dotorg +jay r +jam mer +hoch schule +heure use +headline pg +har mos +gre edy +gott aget +go bowling +geode sy +gam mas +ga ited +frontiers man +fish back +fair brother +eval yn +euro bond +esch aton +emal ick +el via +ehr mann +ed fu +ec le +ear gasm +dimetro don +crimesof grindelwald +coyo tes +co zi +chuck y +char ing +cat ul +by rum +buzz kill +bran te +bowhun ter +bor ton +black owned +be urope +ba sim +ba ic +atic i +ate en +associ ati +archit rave +aracelyar ambula +appar at +ankylo saurus +amar sh +am rish +all ari +al tin +al thorp +air train +ðŁĺįðŁĺįðŁĺį @ +ðŁİĤ ðŁİīðŁİģ +ðŁ¤ŀ ðŁı¾ +âĺº # +à° ® +Äį a +zen imax +yl en +with thebest +wh ood +west de +wego tem +web apps +wal czak +w mw +ut martin +under desk +un does +ug al +u bp +tweetapictureof your +tun ning +tsn bobmckenzie +to ji +the blue +tender loins +ten ures +teitel baum +sug awara +streat ley +strabis mus +str yn +squee zy +spec tre +sp afford +south afric +slay age +sil kin +sie ben +si sse +sars gaard +rule set +ro eper +rich wood +read yyyy +re ges +raw materials +ram bis +ral f +rac q +pra dy +pp cli +pon yo +philosophi zing +phil by +ph ron +peter hickman +petedun ney +petedunney xb +pend ers +p ée +o herty +nott ur +nic ee +nh week +ner vou +n br +mou l +melo che +mcgu ckin +mat tre +marie fre +mar sone +mal practices +lucas digrassi +lightning network +leg ged +leg are +la reunion +l bh +kol ton +knotts berryfarm +keepingit real +kai sha +join me +jo of +jam tour +ja ia +j collins +iwak uni +ish peming +is af +invision app +infe stations +huy ghe +home inspection +heil tsuk +hat te +greatyork show +gre sini +gram marian +gor os +good karma +golden child +goeth als +german e +garrin cha +fu jit +for tb +fla vian +fine ssed +fair funding +f ng +ey non +er hu +economic growth +e he +dive sted +dinner tonight +dia thecat +dd ya +das u +cultiv ate +com ox +college colors +cnc pts +clean in +claw diathecat +chekkachi van +celest ite +can tal +c engage +bull fights +buck lin +bronco sports +bot ello +bird softwitter +be hr +basile us +b nu +azu buike +ay al +aw con +aviation geek +athletics weekly +ashken azy +arro wheads +ar bon +ar boleda +ar bogast +am artinez +am ap +alu so +alten ango +allo graft +al bie +aege an +admoni shes +( ãĥ» +ðŁĺĦ ðŁĴķ +ðŁĺĤ ðŁĺħ +ðŁĺĢ ) +ðŁĶĶ ðŁĶĶ +म न +ÏĦ he +world liness +winnie harlow +wence sla +war ga +wall aroo +vote anc +vogue uk +very play +un ar +toile tek +thenu mbers +tell er +tan on +super ba +struc tur +strength and +spar khill +soular tistmgmt +situ ate +si ón +sheep ish +sexu alization +sen thomtillis +secondary schoolmemories +seam ers +se dimen +schoo ley +san key +s faf +s atti +re zo +re ig +re affirmation +rain sy +rail fans +qual itye +pu rie +prinze ssin +pre peration +plow shares +pla ited +ping ame +peñ arol +peripate tic +penalty rates +part yof +pa olog +orthope dist +orang eroom +od nb +o va +ntv newsnl +nin aag +ninaag dal +ni eva +ncaaw bb +na or +my phone +multi hull +mudge er +mosqu ito +miguel ferrer +mack trucks +macart ney +ma belle +liss itzky +ligab bva +life changing +lazare tto +law fare +land side +la ppy +ko chs +knight swood +kisar agi +ketu rah +ka tho +ju by +josh devin +joshdevin edrums +jack hammers +ja ise +invest ec +infection control +indie pub +i kut +hydroly sis +hu tz +hot and +hen ni +gu mmed +goo b +go flyers +gior gione +geo scientists +fu sz +flat ted +fiest y +fer us +far th +fai roz +excep tion +ent rop +embal se +elfon ashelf +ef w +ec cs +digger land +diffu ses +des sel +deal sof +de dic +cá ceres +cyber ark +cultu red +cryp topsy +consu elos +comi furo +cobal amin +clari dge +carden ales +callip ers +callacu tieout +ca ireland +c xl +c tid +bru cker +broken lizard +bofam l +bb ck +bb capprentice +band la +ban erji +ay han +avalan che +ase va +as ato +artificial inteligence +armedforce s +arant xa +arad hana +ar j +anton opoulos +anor th +allu re +adidas soccer +ðŁĺ« ðŁĺį +ðŁĴµðŁĴµ ðŁĴµðŁĴµ +íķĺ ìĦ±ìļ´ +âĿĦï¸ı âĽĦï¸ı +⬠ľï¸ı +оР± +yu rio +ys by +y ili +y ates +xbox uk +wil ts +we tan +way lon +wat to +voel ker +utt ley +uncas ville +tum se +tu dy +to hono +the pioneerwoman +the misfits +th ag +taw fik +t town +t for +suit elife +st leonards +ss mh +sports radio +sonequ amg +sol ons +sen ility +sen ado +se dinburgh +salv aje +saint es +s belike +rol fing +right towork +reo speedwagon +ra bs +r hh +quik pro +qantas airways +po sed +plom acy +pinto fotografÃŃa +pin ault +pav lovsk +patag oni +pan starrs +os garage +oc d +nue stra +nj sp +naci miento +n ack +mu thi +mu ling +moyamee haa +motor car +moth balled +mortgage broker +mohand as +ming tsai +midnap ore +ment zer +megan amram +maneuver able +man none +mal ing +lu ddy +lex xx +lat ers +laff an +la famili +kwes é +killing me +keep corbyn +kat anas +kam bi +jay wick +itsagreat day +is bt +is awa +ili festyle +iconocla sts +ic osmetics +ho ak +he med +gri ppy +gil kes +fried chicken +fol x +fle urie +five guys +faun af +f pc +f out +er bb +er anow +emo cracy +ed an +e ji +du cote +do oling +discover overberg +digit alliteracy +di eta +delta state +dean morgan +dari ya +cr ys +cou lsdon +consu mp +con signer +co vin +cart land +cakeboss buddy +bu hari +btsloveyourself tour +bru hl +blood stain +bill browder +bell x +awar ner +as pher +as ghar +art daily +argan oil +ar kush +apu lian +apol it +anz alone +andre ak +an sk +an lonsdale +alzheimers disease +alex honnold +al can +af phq +af fine +aci ar +accu satory +____ ___ +! ðŁĻĪ +! ðŁĺħ +ðŁĻĮðŁı½ ðŁĻĮðŁı½ +ðŁĺĵ ðŁĺĵðŁĺĵ +ìļ ´ +ìĹ ´ +âļªï¸ı âļªï¸ı +à®ķ à®® +É Ľ +zer land +z elig +yaa asss +y rn +wester field +wen ye +we play +war ders +visit sweden +vam ani +un icity +tto loso +tro ss +thro mb +teof ilo +teenag efanclub +tc mi +tavit ulle +symboli sed +stron k +staple hurst +stack ers +spas sky +sore head +so suke +skag it +sigmar gabriel +she ild +schur ch +sama atv +road sof +r fu +quit ted +pu san +project mangement +pla x +piercethe vic +pe ery +pan u +ohi sto +officials blessing +of death +o bie +nun nally +nt ate +naomis cott +n dia +mo xie +medit ator +mc crum +maynil ad +mariefre ttoloso +lule Ã¥ +liz otte +lawy ers +kid ap +ke ta +kart ell +k ery +justgo shoot +juic ery +ju ggy +jor jasmith +jmichael tatum +jed york +ix ora +in ata +id one +iam jer +huday dah +hu sam +hooge veen +hitch en +hich ki +guide dog +growur startup +gali za +fault lines +fau det +ever glow +escu char +esc s +elvis es +else vier +e then +dreamtheater net +doc tson +din ara +dil ara +defe atist +czer ny +cw tch +cul leton +cour tin +chur cher +chri stu +chitrang ada +card assian +can am +c wallerracing +by choice +brom ham +brite eye +boon ton +biop sycho +bezan is +basketof deplorables +b flo +auto harp +ap adilla +anthony cumia +an jem +amc talkingdead +al bia +air foil +> ; += )) +ðŁĹ ¿ +ðŁij ľ +ย ย +à¥ĩ à¤Ĥ_ +Ê ĥ +youngand therestless +york town +yearen d +waw ild +vitam ine +v hi +ut pal +uni sex +tur ma +tu pi +trafalgar square +title town +thrap ston +thisi sr +th man +tech update +team sasha +teake ttle +tay miyyah +tan ish +sye da +super nanny +st ent +splay house +sou der +son ger +solo preneur +so bral +sla gging +sivi glia +si min +shri ft +shr tampa +shen long +shamit abh +seri alisation +sen na +se where +scy thes +sat uan +rudy giuliani +ru ly +rosen do +road y +ro sequ +ri ghting +ri eti +ri dis +rhu le +retro horror +rel ents +r mbr +pun kie +pul ped +powderham castle +pou f +pot es +ph ritis +out moded +om et +nomin ally +no sler +no sleep +ne za +nau t +nap es +na hai +mystic seaport +mou stach +mj keenan +mistransl ation +miamidade county +megan tic +lun aleso +lev asseur +kin ny +ka atru +jk tourism +james franco +inten se +ingh ot +ilu stration +ili v +ij t +hub bucket +hry v +home tips +heli um +heb ner +gu tu +gle w +g jr +ford india +fire arm +ff c +fcau gsburg +er ste +ele e +e consultancy +du bay +dramati ze +dec red +de metris +dance co +dae kim +ctv winnipeg +cru ijff +cor mick +complain ants +com ico +cj spiller +ci oran +chain rings +broward county +belgis che +bel va +barat aria +bam av +bal lasts +bad luck +b mus +ashley mcbryde +ash down +ash bridge +arkan sa +ar oon +anna beth +ad dai +a bear +======== ==== +ðŁļ İ +ðŁĺį ðŁĴį +ðŁĴļ . +ðĿĺ Ģ +ãĥĹãĥŃ ãĥ¬ãĤ¹ +¬ ´ +zi ad +yu v +world t +weare mumbai +wast aken +wand bc +voxel art +vis ayan +vi bha +vau ban +upper deck +tur nhout +tur gid +tre xp +tot land +to grapher +till i +ti mu +thereal xpac +te ste +tau fik +tat amag +tainte d +t pot +sug aya +stre ater +stat i +srisri in +sport sau +south indian +sk ers +sinthe city +sim ko +silver link +shoe fie +shar dul +ser ravallo +selfie time +sear ls +scott rogowsky +rune stone +ru elle +rober th +ri perton +pv f +promoting women +progen itors +pro pe +pro fuse +priv ated +pre raphaelite +prand elli +porfi rio +porcup ines +plu ck +planet x +persi ja +palla volo +notbe infringed +norse mythology +nim rud +ngay ong +nevad ans +nego cios +nb hd +n italo +multi band +monument our +mn gt +misssaig on +miss america +mersey police +megali ths +mc f +max keiser +mali gning +maha vir +len awai +lap is +kra sny +kir ui +kil bourn +ki maka +just girlythings +juniat acollege +jo sua +jimo heir +inish more +indiscre et +implo red +impercep tible +illegal aliens +high clere +heav ed +h mos +groo vies +gre search +gr v +goo fed +going strong +gems bok +ge za +gar ant +frye burg +friez eartfair +evangelinel illy +esk ar +epi da +english ness +el ser +el ani +dres den +donthug cacti +d town +cornell mba +conden sers +co sponsors +cityof toronto +cine family +christ of +chim eric +chennai rain +cheeselo versday +cfa institute +castig lioni +caitlyn jenner +cad enet +bru gger +bra sse +bgg con +bbc questiontime +barg ello +balear ic +b hy +aur aria +atro pical +ar mco +aqu inn +aje sh +ai ro +ah met +aga o +ðŁĺĭ @ +ðŁį ļ +èĭ±ä¼ļ 話 +ç« ĭ +âľĸï¸ı âľĸï¸ı +© : +yofthe seas +yas uk +yas sa +xxx holic +writer wednesday +wh p +vil ain +uvm vermont +upgrade yourworld +un spectacular +uk oug +ug dsb +tw angy +turtle back +tt om +thestra infx +ten aglia +tb u +tam amo +t ú +suble tte +straigh teners +stor ag +spor ades +spar ql +sn ark +shi ve +sharon lawrence +ser ry +scand ale +save theworld +s sex +ri béry +res ented +remun er +reign ites +raveng lass +ra anj +quir inale +py re +pu jol +prate ek +poo jab +per ic +pay bill +paradigm shift +ouro ceans +ota valo +nyc gov +noris ring +nintend ouk +nat ya +nas ugbu +n ka +myel ination +mr chris +monkey island +mo stest +miles morales +mike pence +medi asummit +mal er +maiden hair +maddieand tae +lu ÃŃs +los ada +long case +le phants +ld ny +king sx +kd v +jet port +j alo +ira ivi +ing show +ing america +indi ameansbusiness +ig ad +ideser venewshoes +hoax er +historic preservation +heavy weight +happy memorialday +handel sman +hak am +gj allar +gastro post +gam betta +future star +football season +field school +fay yaz +famili arizing +exercise works +enam elling +en stone +ember js +electro therapy +edwardj olmos +ec ma +eb don +e badi +dex com +democrati zed +daco its +da xe +d ft +cux haven +cupp les +corpor atist +cor nick +coal brookdale +cn ooc +ci enti +children underattack +chic co +cardio vascular +californiawild fires +buss mann +bu cadi +broad heath +bri zzi +brady haran +bor des +bo en +blue hour +bibliothe ek +bi thell +bi gamy +ba sted +avery brewingco +aspir ator +armb ruster +ap aper +ang y +an vi +an sascity +ale ko +ah rq +íķĺìĿ´ ëĿ¼ìĿ´íĬ¸ +èĦ ĩ +æĿİ æķı +æ± Ł +åİŁ å® +ãĥ Ł +áķ Ĺ +à³ Ĥ +Æ ° +® ) +x posed +wre sting +west bury +von k +vic trix +vamp y +usc upstate +us enet +ur w +tu chova +trumple aks +tong ans +thestor yo +thaicave rescue +ten nison +tele graphs +tejas vi +te sta +takeover day +tab ur +t pt +sumed ang +stoltz fus +star ro +star fm +st mag +spir ates +snaggle puss +sm iting +si mel +shar min +schuyl erville +roberts dale +ricci rivero +research day +rachel dolezal +put ney +proud fan +pr jct +poli shed +pk gs +param edical +pa ko +ordin ations +or poreal +onno ghen +on asap +official steps +of ire +nie res +ni mona +next stop +nat la +ms gt +mir ta +mhat re +men angle +mcro bbie +mc kees +mc frs +mb abane +maha patra +lur ching +li gao +lent i +lenawai the +la fave +konec ranes +kirk cousins +kiralı kaÅŁk +kawak ubo +kat unews +karn ali +joh ne +jim bob +jessi es +jan fam +jac i +j md +is las +inter reg +heyit scarolyn +hen low +hari kota +hand bag +gulf coast +goo derham +gla uca +for mars +filo sofia +esche wing +eman ation +eliud kipchoge +dÃŃ adel +dou chey +dor king +din or +desp ain +den k +defen sie +dan bilzerian +cre ami +cle fts +circum venting +ci ega +card ston +car lifestyle +candid ate +buff lehead +bronx nation +brian schatz +boul den +bou ska +born and +bogdan ov +black label +birch bark +bio tope +biblio graphies +bb bb +bay ad +bas rah +bar ahona +ban kia +avi gdor +aro on +arab idol +and redrum +anan arama +an ur +afro disiac +af oa +ac cardi +abu ll +aborig ine +abo d +ab d +ðŁĺŃðŁĺŃðŁĺŃðŁĺŃ ðŁĺŃ +ðŁĺĺðŁĺĺ ðŁĺįðŁĺį +ðŁĴį ðŁĴį +ðŁį¬ ðŁįŃ +ðŁĩ ¸ +îĮ ¨ +س Ùģر +تص ÙĪÙĬ +ار ات +zbrush central +zaf er +yun is +yun amusic +yo kel +year nings +ye vich +ye as +winter fashion +win ford +wid dowson +whack ers +wendi emalick +wbr c +val ances +tot ley +thisisd avina +thenana aba +thec wu +the wedding +the register +tenovusc ancer +tau p +tan redron +tanredron cal +super capacitors +sti mac +soft pedia +sic ecream +sheikhu pura +sd or +sco wl +san den +sal vin +ro vira +ro versi +rize spor +righ twing +re pa +rc cc +ram cinemas +r gn +quarter finalist +qld labor +qamish li +premratandhan payo +poul tice +pas son +pan neer +pac eu +p ite +out matched +ol medo +of osu +new usc +ncaa icehockey +molly quinn +mo ta +michel led +me som +mcgre al +mazz ini +matchday image +masar yk +manish mischief +ma fu +little st +lind seys +ley hall +le schi +lc as +latime sopinion +lam pa +ky lee +kra jicek +koz elek +kon z +keen est +kale mia +k jel +juliusc aesar +ja ha +isma ily +inter dict +ingra ssia +in sha +in mac +ib are +hush puppies +hoy te +hom ma +hol tren +han en +hack neyed +ha doo +go kin +girl talk +gab oury +fund aci +fueledby ramen +fi zzy +ferru cci +ferra gosto +felsted school +farm market +famili esc +expul sions +evolu zione +endocr ine +east burn +e braeden +dysm enorrhea +dre ver +dont look +dj ou +dis organization +develop mental +defin able +cruci fixes +cot f +condem nable +clu mped +chy ron +chro ma +cent ar +ce gep +carav aning +broad foot +brix worth +braw ner +br d +bellige rence +bear r +barry island +bar us +bal erno +bal co +b tweets +at it +arach tober +amazon giveaway +ais ling +aiac obelli +afun eral +afgh anist +aduri z +adel itas +( !). +ðŁĴĩ ðŁı» +ðŁĮ´ âĺĢï¸ı +ðŁ§ Ķ +ðŁ¤ĵ ðŁ¤ĵðŁ¤ĵ +ðŁ¤¢ ðŁ¤¢ +ìĬ¤íĥĢ ê·¸ëŀ¨ +ì¹ ´ +ãĢ°ï¸ı ãĢ°ï¸ı +âľĪï¸ı âľĪï¸ı +اÙĦبØŃ Ø±ÙĬÙĨ +zi yar +ze me +yer im +wo hoooo +vo bis +villu puram +vijay alakshmi +vi res +v rat +tun ggal +tropic als +tri bology +trans fered +tobykeith music +to v +to chi +thur ston +tex asto +tere k +sze to +super cheap +stat u +sou rav +so what +shpong le +shink awa +shawn na +sf d +selfie olympics +scru ff +sc sk +sc lassics +raw ong +rangi ora +r Ä« +pp od +popo cate +polit icon +personal shopper +pel ayo +pau city +over shooting +ortho graphy +ny land +number less +nitalo wey +nip muc +nahai wrimo +nac all +mun dy +mor osi +mis uses +matthewl illard +mar les +mag alies +m phs +lime bike +legend aries +labru sca +la se +l bi +ks ack +kre wel +ker li +johancru yff +joakim noah +jameshet field +iz anagi +iy aa +iam laceychabert +hy pon +hor ak +hoo pla +hawai inewsnow +han shika +gur k +group news +grl powr +greed ily +grad ations +google photos +goo oooooooooooooooo +golden retrievers +gig guide +gg olf +gener alizing +fun kiest +frail ties +flower girl +f stone +er roll +en ninful +el repgh +ec ross +dubo ce +doo bie +din ma +dil dhadak +der reen +dallast nt +daily calm +d chen +cu yler +cro teau +cre on +cr nc +con scripts +compen sator +col azione +coastal living +co ley +chekkachivan thav +ce volleyball +cap acious +cad burys +ca stries +bunnaha bhain +bike towork +bhand up +ber tinelli +baske twomen +banc or +ban anab +baira va +bab ul +asho ka +as gr +apar o +all ly +all bright +alessandr ini +aku mara +ai ste +afg anistan +ad ame +ab salon +; } +!! ðŁĺĤðŁĺĤ +ðŁĮº ðŁĮ¸ +ìĽIJ íĺ¸ +ãģ Ľ +zi zz +zi k +yor ke +wx mafc +wwe bige +wo wk +win today +win co +whati f +wer kin +vide tte +vajazz le +ud h +u mineko +ty ng +tu pelo +trans vaal +total cafcl +thrur pg +the revolution +th parallel +ter cel +tal man +ta be +swam ping +super micro +su nexpress +stigmati zation +steiger wald +spur ling +sports writing +sport sclub +spoiler tv +si ru +si dle +shel bs +sharon jones +shari alaw +sen deros +sec nation +sap r +ro dden +researchimpac teu +redding power +rashi ki +r mit +que brada +puffin ess +prince albert +pp aca +pop health +point lessness +pleas ence +phoenix lp +pau lar +pat rouille +pantal one +p cl +ny che +nu dgee +neer atan +nas ar +nach es +nab bing +n he +n ck +mu umu +morning listening +moon watch +ml jet +miskat onic +mis u +milk fish +mi u +mg tow +meh di +mar athe +lucasoil stadium +london breed +lizard men +live say +lifes better +liam fox +led widge +lavanyab hardwa +kre wella +kne eler +kings north +keepcal mand +joy al +josep he +jon ni +jo edi +jeff tweedy +instagram stories +ingen ue +ine learn +how lers +hipho partist +he mera +he mant +he kate +hann ahs +gold bergs +glu cagon +gad u +gabri ele +gabby giffords +from where +friez elondon +fre she +fo shay +fernet branca +fer r +farm show +er se +end coal +echi dnas +eastlanc srly +earthwind fire +duct tape +du bas +dru cken +drive thrurpg +dick ov +di gos +devarak onda +david johnston +dab omb +conis brough +comple xo +colle geny +clu mber +chester tweetsuk +che bet +ceano thus +ce ding +cas lon +career development +caoim he +bru te +borde leau +birdwatch extra +bengal cat +bel aying +beau s +bbcsouth weather +batt in +base ballis +bapp amorya +az d +att r +an anias +akshar ahaasan +abhi jit +ðŁĺĦ ) +ðŁĺĤ "@ +âłĢâłĢâłĢâłĢ âłĢâłĢâłĢâłĢ +young music +yo ke +win chesters +whok illed +weare cue +vo gl +vi ji +ver asani +uss k +un manly +ul ite +ugand a +tri vikram +tran scom +tor ye +time snews +theaf ccl +thank god +tham esp +tele toon +tat to +ta fa +su athletics +stop commoncore +st s +spire a +speed hunters +slu sser +sir na +si put +shaaan xo +selen is +scra zy +scam mell +sar tor +sam ik +safer internetday +russ els +rogu ish +rl tw +rixton official +rivie res +richard marx +re inet +re constitution +re configurable +ra pson +priyad arsh +pot c +pen american +pat tullo +par acha +p head +ori shas +one wtc +ocu lus +norman lamb +nick vujicic +ni ed +new products +nar sis +nab l +moy cullen +moo o +million cup +micha eli +mi dian +mb aye +malay o +m trainier +m ser +loveyou all +liti gating +lind ab +lehigh u +launch party +latch key +lanca shirec +lan dover +kra thong +ko tz +kint bury +ke iz +jubin nautiyal +ip mi +inten sion +hou la +hou ght +her ber +helge son +helge sen +he ireann +harry appreciationday +gimb als +gill ingham +giftof life +gel ber +ge iser +gal z +fanta sized +ey ard +ext ella +enter ta +endless summer +eat fresh +dyne vor +drunk ards +dream ers +do ps +dig able +designby humans +dart ford +confer ment +cold front +co health +claire fontaine +citiessky lines +chin amcclain +chan dio +chae bol +cec ily +caric atured +cardiff met +car ven +cameron mathison +cab ourn +buen camino +brundle f +brazo sport +brand ao +bio art +bil stein +beacon house +ba her +añ ejo +auto group +author amish +as avi +aram kon +antiquec lique +ann eli +anand tech +amne si +ambro se +aji bade +af raz +aber lady +aaron son +a etc +ðŁĺį âĿ¤âĿ¤ +ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ +ðŁĩ·ðŁĩ ¼ +ðŁĥ ı +íķľ ë¹Ī +ìĶ ¨ +Ê ° +zen book +zato ichi +yo or +y alty +x html +womens facup +wild horse +weather wax +we some +war like +wal tons +voteblueto saveamerica +val emount +v ml +ut kal +tric home +tresp asses +tool ong +thor aco +thesquare ball +thehard way +theatre awards +the drew +thar anga +sy reeta +sur ridge +sto inis +splend ens +sp lanet +soultrain awards +sophiel ady +som ec +smart buildings +smar a +sing post +shoshan abean +shak ya +sh agu +scott pelley +sat an +sal yer +saint snation +s vein +s comedy +rober ta +ro der +rebel o +rebecca jarvis +raiffe isen +pul te +ps v +prompt advant +pro visioned +pre iss +portrait november +port ela +plan itia +pic xania +philipham monduk +path on +par ata +pan hard +ns live +novo gratz +nh b +neeratan den +nam ida +nam es +mv agusta +mun in +money gram +micro climates +mement omori +medieval art +media ev +marqu es +mark smen +mar nie +mad magazine +machine intelligence +ma pun +ly c +lum sky +lu me +lewisham council +len inism +kordo fan +ki as +kel oid +kathleen madigan +jud iciously +jose fin +jessic as +je ti +jay z +jan as +jac en +j ver +ir craft +in th +illi ans +if adnews +hopgod friday +half price +guitarri sta +grlpowr chat +great indian +golf show +god win +gobble gobble +go za +gl te +gigab ytes +g lat +g brow +frank reich +fp jr +essi g +en theo +ed p +ed iti +ec ruz +eb st +eastern cape +duke gang +dover athletic +di emer +demo dex +decryp ted +d bh +crusaders rugby +clam our +choic ed +chicago cubs +carai bes +bren chley +bree zer +born thisway +ben av +ay umu +avs nigeria +antoniob anderas +am art +all eno +al sh +adi vision +aber porth +abad don +ab hy +a bee +' ..." +ðŁĶµ âļ½ï¸ı +ðŁĵ ī +ê¹Ģ ëıĻ +ี à¹ī +ÙģÙĦسط ÙĬÙĨ +ا Úº +zwart ble +young spubs +wc pw +volk off +vijayan pinarayi +val parai +un produced +un likable +ulcer ative +u can +tä nak +toxopla smosis +tor ren +tine a +thegirls musical +texas strong +tari que +tami roman +take away +ta st +t js +sx melectro +sweat pink +suresh chandraa +sound mag +si ya +sex tape +sebad oh +scott porter +satter white +sar anya +ros ada +ro senior +recomend ado +por tholes +pon tin +plio cene +pe quod +patho fexile +pak hi +p du +off world +news busters +nat vanlis +na un +my fdot +moun troyal +micro fiche +miami bookfair +mi q +mh sc +me sabi +me ddle +mau ch +math lab +materi alised +marqu ita +mar tucci +macou pin +lleg an +limit ers +leonard cheshire +lavo z +koin news +king si +kick offs +ki am +kad aramkon +jo sho +jenni em +je wer +jac ey +j pi +isaac mizrahi +ir thday +in comparably +ima ke +hou de +ho cked +hit btc +high seas +heer babu +he dd +har kavy +ham irpur +gy c +gretchen whitmer +great guy +graff erty +golf travel +giftfor him +gh un +ger as +gau vin +gar agi +gand alf +g di +g ades +früh ling +fri derz +frederick town +fing al +fibaeurope cup +fanta sist +esti mable +ellic ottcity +ek d +ed xonline +e apc +du eled +don gs +dog treats +dish nation +die h +dev ault +des apare +dent es +davidg ill +dar cel +dad bod +da di +d km +craig lowndes +cor ris +capetown cityfc +cam poo +cal d +cairn terrier +cad ence +c mg +bwal ya +bw g +bragg art +br ica +bor nes +bon um +bon der +bm ttandem +blackdeser tonline +be vents +bb ell +bat dad +ather ton +at witter +ask am +arnolfini arts +andri ana +andre an +amo reno +amit abha +alan tudyk +al van +al ace +ad be +ab ayan +? âĿ¤ï¸ı +ðŁĺĤ ðŁĴª +ðŁĵ² | +ðŁijĮ ðŁĺĭ +ðŁIJİ ðŁIJİ +ì¤Ģ íĺķ +zom usic +z ouch +yyc music +yu saku +wwen eville +wo ggle +wick reme +westfal en +weigh troom +vr ps +vote jkt +v th +ty ard +tu ks +truckn driver +travel ist +touch wiz +thero bot +ther ag +thelaugh factory +ter ma +tel for +tel erad +teach ag +tatamag ouche +summer glau +sub humans +southern star +shaf ak +seawol ve +scu do +scorch trials +sat na +sant en +sanc on +san ghar +ruok day +rumb led +rogal and +revolver mag +railroad ed +queens land +pumpkin carving +pre sonus +pr request +pop dust +po gona +parl ons +on scott +om ir +ok mesonet +nu wara +no vica +nigh tt +nh dems +newton ville +new beer +navar one +nahu el +n md +must i +morning star +model cars +mo peds +miyaz awa +michaele merson +mi yoshi +me tron +mat angi +maker studios +madon naf +lub be +losin j +livel ounge +lead up +ladies g +kre ut +kin sey +ke et +kar wan +k bb +joven es +is bnpa +industrial music +inde acon +in dominus +henson company +hay market +hat ori +ha skel +gy al +gravit on +grant ley +good looking +glu teal +ge ffen +freak indeacon +flin der +flick inger +fi ved +femin amiss +far sighted +erlen meyer +do gen +dal beattie +daily herald +cu ong +cru mley +court sey +could nt +cotton seed +co traffic +chin ky +cefas govuk +cam mi +cam bourne +c sed +c festival +bur gee +bro cante +breakfast with +brac keting +bli brary +biggest loser +bethesda studios +barry sanders +bal in +azamar avoyages +auctione ering +att park +ar ni +app rised +ani festo +al nassr +al loc +ai mar +adu bai +adam woodyatt +ach on +>> # +ðŁĺ¥ ðŁĺ¥ +çģ « +ãģ © +а ÑĢÑ +zoethe ball +zam ana +zab ludo +xia oyu +ww cc +wre tch +wi leman +white read +wal kab +viktori ya +vikk star +un trusted +tsa on +tran stv +the knick +thar aka +t drs +sw brail +swbrail riders +sw all +susu mu +star sonice +stap ha +spring well +spinabi fida +soci os +sn ard +smar ini +sk use +sit us +ship wright +sep i +sene cio +se vin +scho or +schill inger +sc sn +satis factor +sachsen hausen +s girl +ry uki +rugge dized +rosic rucian +roman britain +rl j +reas sign +rac ia +proud father +po je +plu mping +plenipotenti ary +pick les +phosp hati +phi b +ph enter +pepso dent +partick thistle +pan ge +otter dam +or re +olym pe +nut free +not ta +norther nontario +nor fleet +noir summer +multi focal +mudgeer aba +mrporter live +movi eland +morning live +mo ssi +mir us +minu tiae +min usma +metam orph +medic os +mc glade +match es +mal amu +mahar lika +ma ppings +lik on +ler k +koro vin +kaz insky +jobo aler +jac kiel +in memory +i io +howto basic +horn beck +hm hs +hel muth +hay dens +hashem ite +happy makarsankranti +ham strung +hacket tofficial +gon calo +golden temple +ge ist +gator zon +fredo santana +fl acs +first gen +fil omeno +f heis +ev intage +en ali +ed iciones +dragon gate +dr j +do ina +disper ses +di wal +dev hynes +deni grating +del rey +de duplication +davido tunga +dab lam +d str +crow medicine +cot grave +com ate +col drain +coffe es +city westminster +chieftain cy +cherry blossom +c sto +burgh leyhouse +bu er +brom well +bir bal +berry tmr +bered seered +ben the +bc x +bar atta +bam se +bal lade +aw is +av adi +as mat +as cu +armi da +app ric +anasu ya +all llllll +ale theia +aldub xe +aldu be +ail lu +against allodds +adamcole pro +accoun tab +a ane +" ...# +ðŁ¤ ² +ãĥĸ ãĥ¬ +à³ ģ +z day +youth for +yoga pose +wain ui +wad ada +w clc +var ona +urin alysis +underdesk loser +under glaze +umich medicine +ull mark +ukrain ian +tzed akah +tw ash +trump sters +think landscape +theoutdoor city +the town +the sar +thath omas +tez pur +ter no +tas lima +tar ta +tafa wa +ta arab +super mariobros +subhash ghai +sub contract +spublic ations +social rights +slo ppy +ski jumping +sher born +sehs thebest +scuder i +sar ova +sal ah +s room +roman ova +rock on +regaz zoni +r kt +produ k +predi lection +plaster work +pi ura +pen alizing +pan ta +pan enka +pal mares +old english +oax acan +notredam ec +not fair +no vik +nic kie +nast assja +n said +myfox tampabay +mom usic +moistu rise +medi amo +mc quay +mas dar +mani yah +makeit real +ma kay +m jakbar +lpool council +lowest price +lass ic +lar mer +la win +ko conews +kn apper +key stroke +k rook +joburg theatre +jack the +hk ir +hil der +herre ra +hei jden +he ley +haz ra +hann ad +gregari ous +german shorthai +games manship +gam p +gal u +g sy +g ico +fo b +fe to +fate ha +exagger ations +esur ance +eigh th +ec wa +e thology +drug war +dru gab +donkeys anctuary +diap hanous +di maria +de cri +dai quiris +cor day +con leth +come at +co bix +cobix reyes +cc sf +cap radi +c bo +bsn sports +bot ching +bin n +bike chi +ap ke +annas ui +anc as +an aka +almir ante +al bies +ait ana +ad alberto +abar bera +. ,. +ðŁijī ðŁijī +ðĿIJĪ ðĿIJ +éĢ ļ +ãĥ³ãĥ ģ +ye siam +were the +wehave wewill +we ki +wa ht +vul gare +ura uganda +up gradable +uof regina +un righteous +u stra +twitch affilate +twee ds +tv guide +top coder +tom ber +the virdas +thankyou u +ter man +tele visual +team ajaydevgn +tali ban +tale of +tac tic +swains boro +supri sing +su raiya +stu lsa +stir rings +steph anos +stand pipe +space coast +solart vnews +snoqual mi +sn live +sl un +sh ole +se wak +science advances +sche ff +sch roth +scal ene +sal atin +saf ran +sab r +road runner +ri smo +resemb lances +replic able +replac ement +re trained +re draiders +r fef +pra chu +pol an +pesh wa +ongh ere +on ova +ome a +o dgers +new shead +ne eli +national lipstickday +n ram +n han +my heart +mur nane +multic olour +mtr cb +monte mayor +miguel tanfelix +mck ell +mark d +mar m +mango tsfield +ma bie +m brundlef +lulu lemon +louisi ana +log books +lev itt +leff ingwell +la villa +ku nedo +kra de +ken ma +karate ka +k gp +jame er +jagad guru +it ouch +ish h +indul ges +ilig ence +i prit +hymn als +ho ppo +histor ico +harmon izer +grant sville +google exper +gm fc +gemat ria +gal breath +fun c +fraction ated +fox newspolitics +fi h +fan light +engag ing +en esis +ec oli +diferen te +dhen kanal +de moralising +dayo ade +day tuesday +cy m +cra zzy +conver gences +colou red +cine spia +chis os +cer ullo +carno taurus +caric hampion +capp ellini +c pyne +bu land +bri atore +bradleys fight +bracci ano +br ck +be atie +bam asb +bag na +bad day +ay ran +au do +at ole +astro logers +around hampshire +archi vo +aqu azzura +ant ourism +almir on +airbus ds +adeci mal +acid house +abell ana +a dis +.. ~ +! ðŁİĥ +ðŁĴŀ ⾨ +ðŁĴĸ ðŁĴĭ +ðŁıı ðŁıı +ðĿĺ ¦ +⾨ ðŁĺĦ +âĺºï¸ı ðŁĴľ +ب ÙĬØ© +zee man +x cfl +will mott +wikicom mons +vibr ance +vi ste +une motional +un truthful +tom segura +to pac +the bill +ter update +team sa +te icher +tam plin +tailgat ers +ta haj +super normal +stat evb +sn mp +si men +shap er +se vil +sc as +sat akarni +sanit ised +sam bro +saf wan +rider strong +re color +rath fr +quade cooper +public diplomacy +pronoun cement +positive coach +plebe ians +plane te +ph le +pent yr +pen ko +pe red +patron ised +orgre ave +ok l +o je +nor ra +non et +nas rullah +museu mart +mou tier +mosas aur +monopoli zing +mm schocolate +mizz en +mis l +mal ott +m con +lud wick +looo ok +long den +liver ight +litt a +lit te +lion nation +lever ton +lefthand brewing +le petit +kul iner +kro pp +kid sday +khali fe +kh ound +ke shi +kat la +kasar agod +kas lo +jer am +jef ford +info sy +ho adley +himach al +heritage fun +heller town +hass ine +hard anger +hallo ates +hall yu +ha sit +h sw +gy p +gron din +graub ünden +gra af +gon dar +go girls +glas lyn +ge thealthy +gan k +ga hh +form o +form ative +forgi vable +flu season +finding carter +ferrar ir +euthan ised +err orists +emb aj +eliz acarthy +ed unn +eat clean +e max +drive safely +don ghan +dom ec +do ts +divers i +deven o +dea thanniversary +dave ed +dance onfox +dair ying +d town +cyto skeleton +ct surgery +consumm ation +consoli dations +comrade ship +col ch +chun ga +c pu +busy body +bu den +bro chette +biz school +bi bb +begley jr +beautiful places +bbcradi of +bar thez +bah ram +bachel der +ar que +al iso +agi bbons +agglomer ation +ag unn +ag news +achen bach +ach or +abo gados +ab ster +ab azar +aar au +a shot +ìł Ŀ +ãĥķ ãĥ¬ +âĩ © +ÙĪ ا +zo va +wou k +wn p +windy city +wego again +wedd in +wat esgroup +vinod khanna +ve kic +v lam +unex amined +unbeliev ing +un labelled +tur nal +transc athe +tram mel +topra k +told story +todd ington +thu ms +the union +tau fiq +tab an +super intelligence +story behind +stcuth bert +sta w +spell checker +sl rs +sk news +si pple +sh kapoor +sear a +saturdaynight live +satine phoenix +s yo +s meets +row lf +ro say +richard coles +ren c +red dot +rajag op +pust aka +pneumoni ae +pi ase +petron as +parl ors +paolog ent +paologent iloni +overex tended +over wrought +ou attara +oned bestfans +o teri +ni ère +ni gr +ne a +nal gonda +n wtf +mor tification +mo tala +miz rachi +milen io +mil bourne +mh world +meridianc u +me yn +me tri +mat ariki +mail and +mad villain +love food +lo des +letsgotothe ex +lec ricket +la sen +la fe +kt va +kon ink +kir t +khal fan +it ter +ira ja +indo logy +in bal +i ae +huffpost live +hot beds +hom eland +heaven officialsblessing +health ed +groen links +gro be +grace fulness +gau ff +fundam ent +fr anny +feed lots +farmto fork +fa bel +eve yone +etru ria +esk om +ere whon +enov aaw +dr ough +down s +dis kette +dhi raj +design miami +deck i +date time +dash lane +dasch le +dam ay +cra zz +coy q +confi dants +chain maille +chad dadon +bun ited +braehead clan +blue sea +billys later +berlin a +ber it +behere now +bbcin tune +bareminer als +bal luk +bab ii +au ll +ation alist +at tu +at reides +asah ina +as rs +arts offl +app state +americ o +aller y +al az +. ðŁĮŁ +" ....... +ðŁĻĨ ðŁĻĨðŁĻĨ +ðŁĺı ðŁĶ¥ +ðŁĺ¬ . +ðŁĮ¿ ðŁĮ¿ +èģ´ãģĦãģ¦ãģĦ ãĤĭ +âŃIJï¸ı : +âĻ¥ï¸ı @ +âĨ ª +zipp speed +yel ahan +winter sale +watch me +vir di +vic ari +val demar +tu tton +trum pusa +triti ya +tornado warning +tomp kin +to fin +then fb +theli ber +thel ab +the est +th rum +teu fel +taxcut sand +swind ells +ston i +sto povers +steve perry +ster r +star flower +ss q +sound track +sigh isoara +shutt led +shen stone +shaf aq +sense making +seis mometer +seapor ts +seaman ship +sal cido +sab u +s thouse +s arti +rub chinskiy +ri poff +rho dia +retro fits +rep jerrynadler +rec entre +purrr fect +puff puff +ple eeee +pit roda +phra o +per se +partsun known +paraly se +pal mira +paddington bear +paci ous +of london +north pole +naga oka +most awaited +mor ini +mor arji +monast ir +mo wins +mike crapo +mar janovic +m chat +lindac ohn +li zza +lever ly +legg ere +latch mere +kuus amo +ku uga +kle ist +kha yr +kevin mitnick +kazant zakis +kam araj +ka shu +jos buttler +j sk +iolan the +intere sante +inter fans +incu bus +im richardyap +im broglio +hydro logist +house boy +hol ling +hk sar +hills comedy +here ee +har thill +hal yard +gov of +gimna sia +gie sen +gg lobal +gag tweets +g mtv +g board +fu sed +fru tos +fight net +fan sday +exc elle +eph rem +ele me +e tron +down slope +dg w +desro ches +desk top +derby uni +deli jah +dar gis +crime drama +crevas ses +colo simo +cla vell +chiantic lassico +can nel +camise tas +business continuity +bur lesque +bun ched +budger igar +bu ist +brun skill +brin ker +bert man +berber ian +bar onial +balu ster +bab ymama +au mf +as del +and cream +alannam asterson +: ðŁijĩ +Ķë ¸ +ðŁĩ¬ðŁĩ ³ +ðŁ¥ § +é¦ Ļ +zwartble sie +zo omi +woo ot +whoop whoop +whitec astle +wa as +virtual tour +var dan +v uuren +unter gang +tu twiler +trevor jd +tree brewing +togetherfor yes +ti ye +the ke +ter vuren +tar pon +tail ment +tab ligh +sund quist +summer fashion +spreadlove positivity +spec news +si dec +shri venham +septu agenarian +sales management +ri u +resp ighi +recep tivity +real fpjr +rational isation +quin s +quar tering +quad rennial +pope inus +pol onius +pl sql +piet sch +pan fur +over simplified +out smarts +ourcountry red +oris kany +new cifera +ne ons +national muttday +n ji +myx musicawards +my erson +mu women +mr t +mohen jo +ml bonfox +mississipp ians +micro chipping +mic keys +mi yoko +mc gowen +mc atee +mar uko +mailand guardian +magneto sphere +mac ritchie +liz ia +life below +letting go +lets roar +lea o +lab life +la et +l anni +kok stad +ker ast +k met +jas raj +jame shut +jab u +j brown +iz ola +iam intel +houseof lies +histo grams +hal ite +good rum +gbrow ingteam +fu ÃŁ +flugz eugbil +flugzeugbil dde +flamin goes +fifty three +feile belfast +fe ir +fa sig +espn greeny +dothe work +dont get +dominick cruz +disal vo +dis ables +digi dhan +di deas +deliciously ella +deaf lympics +dai ji +cur cio +con sole +cis cou +chekkachivanthav aanam +cheer sto +casca bel +cal dron +bro dick +bird dog +ber meo +bel onghere +ar p +al mod +al ahly +aga c +abe ille +ðŁĴİ # +ðŁĩ¨ðŁĩ µ +çIJ ĥ +ç¬ ij +æĽ ² +özge gürel +zin ni +yaku pov +y for +x si +wright state +worldar chery +win chen +well head +vis erys +vin da +un glazed +to cant +tocant ins +tin ky +thi ther +themeat ly +temp ter +teg mark +take charge +tac tless +suppor tive +subsi st +su ha +stil acosmetics +ssal on +spar da +smarty rs +sky sox +shutt lesworth +seismic ity +scol lective +schnit t +sch ib +sc d +sar anda +santan acarlos +sang ma +sak ala +ris ation +rein ders +ram fam +prayfor peace +pra sadam +pleas uredome +pf leger +pe f +patmcgrath real +our pain +oss ining +on ear +omer uo +oil man +official alw +ob sse +norde ste +neutr alizer +nav s +national pet +n andy +ms mith +morris art +morphe me +mic limerick +mari b +man cos +lagar to +kish en +kin lochleven +ke ffer +kamasi w +kak eru +just ment +john nys +jamshed pur +jagga jasoos +jager meister +j nan +j ff +irantalks vienna +iphi genia +inas much +in appropriateness +il ve +iamami whoami +how z +horror hound +holly holm +hitmusic only +hell yeah +hel los +haye z +harsh deepkaur +harrogate town +ham er +g tz +foo tre +finds friday +fin edon +feminamiss india +ev aristo +eu th +eth icon +epi gram +end ays +ed begleyjr +e wha +dimm itt +denni ston +daysof oscar +dal ot +ct la +christmas decorations +chee sey +ceram ist +cbc sby +cattle man +camping world +bru hn +brad stock +birthday month +birmingham weare +bird flu +bir sa +bill kristol +bh ana +bel ice +bb log +bar kin +az b +aturi smo +ath lone +aspe tt +arnou x +anamari ecox +an una +am munitions +all blackeverything +agricul tural +af ce +' ?? +ðŁĻĪ . +ðŁĺī ðŁĺľ +ðŁĸ ± +çĢ ¬ +~~ ~ +zuk erman +zippor ah +witcher game +whor un +wee tie +wal thall +vel as +v hong +us nft +us asunrise +un sheltered +un ani +type casting +trans metropolitan +the guy +super tanker +suggesti vely +ss m +spol icing +sm itten +slo cal +slash dot +skybet league +skam ania +sing am +sig gers +si mes +shar ansky +sh loka +se dro +schi frin +sat yrs +reve sby +retail news +ran chom +rackete er +polon sky +ple dgers +pit u +phenomen al +pe tah +pari stech +or lin +ocean acidification +obu asi +now toronto +newberry library +neder lander +n acks +my father +mss ociety +mount field +mor tier +mo in +mischiev ously +micro service +micro green +mc gown +matth agan +lur kers +lowes water +love wilko +lmfa oooooooo +kwan ghee +ko tv +kau fusi +kati em +kat ori +kat ar +kare en +jorger amos +j ems +ital design +isp wp +india historypic +ignomin y +if v +hostel ry +hope world +honor our +hon d +he schel +h alie +gri sman +goondi windi +get motivated +geek girl +ge tar +fire bombed +feder ico +family guy +excel sis +eve tt +estac ada +era edta +du kraine +dr l +dou ai +denisle ary +dar la +cryo lipolysis +cruel ties +cre ady +collecti vist +codeof vets +cobra golf +co gen +club med +cloi stered +cla gue +chit osan +chau dh +championsle aguefinal +cham pur +ce mal +carpenter sville +bry na +brumb augh +broad green +brat en +bram ante +bob sledder +black lick +bi alo +bhar gav +bhan j +bbc northampton +bay at +barrela ged +bal som +baby ariel +b gh +az nude +associ ate +as ound +ar coding +alainde botton +af es +aapkad haram +** . +!! ðŁĺģ +ðŁijı ðŁĺĬ +ðŁ¤Ł ðŁı¾ +íĸ ī +é Ī +æĸ ĩ +~~~~ ~~ +zu zanna +wo wwwww +wickreme singhe +wau chope +voteuk mahomies +vis arjan +vir l +vintag ep +ve rest +ur sel +unve il +uncle bob +ud n +tu ite +tr nd +total access +the phantom +the gin +the fashion +the at +ten ter +tasty poem +sutton coldfield +stream side +stre mme +stop fundingh +sports file +speed man +sp hen +so arer +sla dy +sin ter +shi st +schne pp +sch umi +sat pura +salt spring +saj jan +s arie +rogo zin +pre volution +ponto ise +po hlman +petter sen +pd schargers +pas quier +pac ers +pac eman +p bg +our de +oste op +oscar trial +o gura +nsu kka +northant spolice +natgeom ag +mo shiri +mis quoting +mercury prize +memorial hermann +mdant sane +mad dies +mac gruber +lud milla +lu sail +lin ie +lighthouse day +lich man +li ze +lazz ari +lay zie +lar ussa +kon ings +knob creek +ki si +kevin vonerich +kayseri spor +jin ni +iso de +is may +iprit amofficial +ine ws +implement ers +ig slovenia +hy dr +hof stetter +he ma +h shanghai +green planet +glu can +ghi bran +form alism +flower oftheday +falak numa +fac il +eric topol +elo ise +ei go +ec statically +east siders +eagle son +du er +drama beans +don skoy +don ata +desafi o +de za +de ste +dani her +d Ãł +cre atu +craig melvin +cr sp +con j +comhal tas +clar us +city radio +ci fer +cho let +chlor ination +chin ad +ch itti +catal di +car al +capacity building +cagli ostro +bullet storm +bu ssiere +brisban ecity +bride and +brendand assey +brac ts +bo vard +blue bull +black heath +best man +bern stein +bamboom usiclive +ath nico +at rain +ar non +appreh ends +amanda holden +am ason +alleg ories +aha dra +active yes +-------- ---- +ðŁĺģ ðŁĺĦ +ðŁijı âĿ¤ +ëĭ¤ ìĿ´ +ঠļ +اÙĦ ر +п ÑĢ +ç ons +youvegot mail +yemen ite +year sofe +yamahar acing +x op +x fighters +wood lice +wim an +wan kel +vin cat +uplift ing +un rehearsed +ub co +tom fitton +the ek +the drake +thab iso +tes las +tag er +super sonic +stro ve +stonec old +sine ws +sieg ler +shi rov +shi ren +sc liffe +salon en +sac cessories +republic adominicana +rep your +reload able +real isations +read missions +re genesis +re confirmed +rasto gi +ram jet +raja beta +rajabeta sharad +qu itec +pra kritikakar +polic ar +pol en +po wri +pipe smoking +pint ado +photo aday +pet abyte +pend ry +pen lee +paw z +p fen +p bw +overex cited +open cast +oh god +nishi kawa +nish ad +nikonowner mag +nikol aev +nas d +mun che +mu ahahaha +miss americ +mil icevic +microne sian +me mes +mc craw +matthew berrytmr +mak am +macul ata +m commerce +litt ell +lit em +lies beth +ki hn +ken ning +ke van +kamalhaasan fans +josel u +jo ssel +jim stweetings +jeet kunedo +j sch +iy engar +iv ars +incur s +impul sivity +impal ing +ili sts +if msa +ic bms +hor i +hin da +head abovewater +he ye +haz y +gul zar +guil la +gu stas +gor get +google docs +good thing +gameofth ones +gamali el +fu ssed +floridal ottery +fe ma +far away +fac er +fab bro +expec tedly +en circles +ele wis +egg en +ef ren +easter seals +ear ful +dun ya +dou rado +dilo renzo +diade ma +deco ction +dead heading +de tre +custo dio +cuer vos +crisscro ssing +cor ral +combo ver +columbu sohio +collecti v +co yo +co vel +cn nee +cho cor +char ityevent +ch avan +brass ware +boy land +bo twin +bla gg +big play +badboye m +bad illa +ba jas +b gd +ardu c +aposto los +apo state +antiterror ism +amar ah +alex iso +al men +al bright +adam jones +ad do +ad die +ðŁĺĺ ðŁ¤Ĺ +ðŁĮµ ðŁĮµ +ðĿĻ ļ +å°ij 女 +é mile +ç i +zion nps +z ian +yan tai +wr inging +won young +wis ler +will ferrell +west lothian +wah ro +w ky +w autu +vin us +victori ous +vee jay +up fitness +ulcerative colitis +uk ku +u ÃŃ +trac ism +tiru chi +the curren +temple of +tak bir +ta wn +t ver +super crawl +stop ford +soledad obrien +sing ita +simul casting +she her +se il +sc apho +save gotham +saturdaymorning cartoons +sars fields +ridge top +ri stic +rener gracie +ram apo +rach ita +r vo +productiv ity +pp sh +popocate petl +pi edmon +period stories +pav é +paul blackthorne +pachel bels +or tolan +op tane +olom ide +nw ts +newon folksy +neuro logists +nein quarterly +nass nigeria +na qi +n rol +n ooooooooo +mysteriesof laura +mun dos +motor mistress +montac ute +mo chi +milan luthria +mens shoes +mam asap +madison square +mac taggart +ma wr +lo boc +li wanag +lev ites +legg ero +lac y +kus al +kitt anning +key largo +kemp is +kam bli +ju iciest +john lithgow +jo bert +jameson whiskey +ic auk +hu llo +hom mel +hl inger +ha su +h reat +gu sman +green lights +go pen +ga thi +freakindeacon friday +fonda theatre +flamen co +fic ation +feel better +eyehate god +esk rima +eric ks +er ag +england netball +dis continuity +daniel daekim +d so +cryptom ining +crate red +coon abarab +coonabarab ran +confit ure +com pris +collegi ate +co ssa +cleanup day +cere brum +ce qa +car donald +canc ún +c newslive +btoo om +bso azad +bra zed +blue wings +biom aterial +beer geek +aun er +ash is +ash ely +as ob +art class +arkan oid +arig atou +angr yorchard +anc c +am irah +ah all +ago sta +a ep += @ +* .. +ðŁİ¶ âĿ¤ +ðŁĮ² ðŁĮ³ +ê´ ľ +ãĥ´ ãĤ¡ +âĶģâĶģâĶģâĶģ âĶģâĶģâĶģâĶģ +ॠ¤ +عÙĬ د +zam ba +za pote +yuku stun +yelahan ka +yan jun +wizzy jr +wilson tennis +ve rey +v qg +un cy +un conventionally +ultra hd +ud k +tul lian +th att +ter c +tax ation +syl ver +swat ara +sune dison +sunday read +stri pes +stopfundingh ate +stein itz +startup grind +ssin ce +spi elt +smar athi +sle epi +slack line +shrou ding +shi o +shi geo +s ry +ross lyn +roman kemp +rll racing +rick on +rep barbaralee +ratt y +pugn acious +pit ot +pig face +photo by +pad more +or lovsky +ol ten +ol itis +nz t +nin is +mo she +mel nik +mel bs +mecan oo +me flo +mcmee kin +mb sings +mahal ingam +m sch +luther ville +london town +lilli an +lewisp ugh +lend ing +lees ang +lay asmine +laur is +last year +land skrona +l gu +key f +keon jhar +kel ner +katrinapi erson +kadı köy +jum bos +jose i +i fra +hy land +huss a +hun grier +huck ster +hondac rv +hamble tonian +haji won +gyneco logists +gou let +gol go +general aviation +gc s +gbag bo +fragi lex +fl anno +fc business +fasten ings +fag un +exi stance +eura si +est ecker +equal su +epr df +dou ses +diet l +defend our +de plore +dav pope +dark sky +dar ger +dam pa +collector ate +co sp +climax es +cho com +char n +cassand re +carbox amide +car ref +car naval +ca zij +by k +bruce prichard +bowie forever +boulevar dier +blues brothers +bluec ross +bil bao +bev hills +be venting +babest ationtv +bab ay +atz maut +arc illa +araf ah +ar ag +ann ane +am etal +ale em +ald win +ah hhhhhhhhh +af branco +acce ssp +;- ). +ðŁĺĭ ðŁĺĺ +ðŁı µ +ìĦ¸ ìļĶ +기 ìłģ +âĿ¤ ' +âľĮðŁı» âľĮðŁı» +âĺº ðŁijį +ö kull +yas mani +x lb +wo da +vote searly +voltag e +uka itis +tw ald +ton earm +thof en +the antondubeke +tessell ations +swit z +sturdiv ant +spi v +spec ts +soul sby +slo we +sj giants +side mount +si et +si delights +shrub sole +sc ities +sav aged +sare thebest +sarah grafferty +saint mirrenfc +ross all +rocin ha +robin ince +river monsters +qu ente +pur lo +pneumo thorax +pilipinas debates +pere c +pc mr +par um +paper back +pan til +nu ma +nb apra +nano composites +na shik +mt gart +mr schu +michellea z +megau pload +med line +matt le +maram ures +mar torano +mandel baum +ma ipo +m ty +m thood +lu cc +lifein apicture +lepi dolite +le ser +lan ville +l fn +kil rea +kil duff +kay in +kar war +k mp +jonas blue +jin woon +japan expo +ins berg +inge ye +infinite simal +ind ar +inam illion +husk isson +hou ss +her ter +hen nes +h fp +guitar uk +gom el +god ating +foo trace +finn jones +field ps +fe ad +fate stay +emirate spalace +elvisduran show +ele igh +dysp horic +dau bed +dal le +collo quia +co hle +clari fications +cla v +choir master +chandrase karan +cer d +cdn tech +carry cot +can ina +cajam arca +ca ino +business school +brown hills +box borough +bor ak +big west +bet d +ben well +barber motorpark +bal by +ba aba +az epam +av isa +atur ns +arav ali +am amos +a ção +! ðŁĴ« +ðŁĮĬ ðŁĮ´ +ìĿ´ìĬ¹ íĽĪ +æķ ° +âľ Ń +á´ ¬ +wi sen +wherearethey now +weston supermare +war do +w xc +vertical farming +v tu +ussk iteam +ur rea +under construction +tus shkapoor +trans versal +ton n +todd terry +the kate +tas os +subli mity +subed ar +stop smoking +stipp led +spo ol +son gever +so haib +smu n +skirk ja +shen zhou +shaun w +shaun avon +set suko +san ct +sair force +s than +quir rell +prog metal +pro mat +pr ati +port age +poli dori +phi us +per ls +paper towns +pa un +p wy +p mh +p boro +oz s +owler report +orak po +or dine +oli gom +oc ton +ny gren +now play +not done +no ory +no kes +nikifor ov +net ballers +ness man +nam aqua +nae em +movie actress +middle wood +mha feez +met life +melani ep +megan bata +meaghan rath +mari ac +mar steller +lw anga +lou kas +line t +like an +li ag +laure tte +land holders +la stampa +l bg +kk d +kim jonghyun +ken za +kat yal +kali sto +k mb +justice forall +ju mana +jan ek +is oc +ike chukwu +icon ference +ic ome +i mortal +hydro philic +hou da +hotro d +hind march +heritag er +hat pin +greenpeace uk +gre x +goo ge +giga watt +gerry dick +gay weho +gas conade +for ding +folklor ama +fitness girl +eurefre sults +err le +eric decker +emu eagles +ele giac +e use +e king +dore mus +dom biv +cu tis +crank bait +cranes bill +clou ds +clothing brand +cling man +california adventure +but chart +bur gu +bu scan +boul mer +boston logan +bdn wheatkings +bar um +b zz +ay ziggy +ath o +ash kan +ash ankar +art museums +ar lovski +ap gov +and aya +ajen ews +afro z +adam hillscomedy +ac grayling +abish mathew +ðŁķ µï¸ı +îĢ Ī: +åľ ¨ +اÙĦع اÙĦÙħ +za q +z anni +yum mies +you uuuuuuuu +writer sclub +whole someness +wg ms +wear red +wales comiccon +w cd +vit ar +vater drumsticks +van am +un winnable +uht red +ub co +tx u +thy pe +tawad ros +ta jh +swi de +sveng ali +stroke play +stre ator +stir k +sol die +slo f +sf sketchfest +sebastian kurz +s bootcamp +ro cred +rel le +rain i +qua shing +qu or +py xis +produc ers +prakash raj +pir and +patt il +pasqu ini +pari eur +p mik +outnum bered +one show +om h +oik ou +o tram +nam pak +my cu +mut ineers +mosqu eda +montre zl +mi thra +mi roh +meji as +med aka +manchester fire +main frames +mahoo sive +macn cheese +ma quettes +luc ke +lowes racing +london eye +lo lllll +laff ite +ki seop +k wid +jody watley +jo a +ja akko +ipl final +iphone sia +in memoriam +ihave adream +huiz ar +horse woman +hom eric +ho ef +hm ms +hel las +hart mut +har mandir +gu pt +gra flex +gon char +gh u +gadget show +flamin ia +faz lur +fa rell +ether ic +esh un +er rs +ecma script +dream chasers +dawg sup +dan el +cyto pathology +cx nats +cul verts +corner gas +columbi arecords +co ens +clé ment +chill ingham +cha is +bur la +budd leia +bree de +bou lud +borden town +bo ving +bit ored +billy gardell +bbcradiof oyle +bbcradi okent +bas ell +bal me +ax id +aren dal +ap ine +an tak +an ky +ak osu +air langga +ad au +' & +ðŁĺĤðŁĺĤðŁĺĤðŁĺĤ ðŁĺĤðŁĺĤ +ðŁĮŀ ðŁĮĬ +ìļ ± +é£ Ľ +è¢ « +âĢ ¡ +yaf fa +wyn ton +woo delijah +winchen don +whereyou at +whatson in +walk together +voten ow +vi seu +val las +un di +ulster bank +u dub +ty bee +truecol ors +track man +tosc ani +ti kes +ten penny +ten no +tem pests +sword sup +stra ph +st cw +sportb ild +spec ters +sophielady deparis +slipper y +shing ler +ser ved +sei gel +sal amand +sai etam +sa are +ruang bin +ru gani +ros elli +rock os +rishi kapoor +reic her +rd top +rachi eskar +rachieskar sten +power trip +pope bars +per rier +pas chim +ourtown ourteam +om igu +note card +ner vi +national chocolate +narrati vely +na ie +n kab +montan ari +me ha +man gel +ma hu +ma am +m ras +lord shiva +li ah +leg ado +learning together +lan cel +lagun as +la pe +la bra +krewel layasmine +kra ak +kh ruangbin +ker stin +ken ney +kellogg sus +kan ani +kai mana +k line +johno ates +jacc journals +j js +inn outburger +ie fun +hum are +honori fic +hog wood +history today +happy spring +hair pins +green fingers +geis mar +ge wand +fun ner +fresh radio +free education +freddi eroach +extro verts +euro pride +enation ale +elm wood +el ac +edy the +dumb arton +drop shipping +dragon pride +dorm ition +di el +desi gual +davis son +dash ti +dar yn +dal am +d wr +cu scus +ct fcofficial +cry wolf +corri endo +com yn +colla bro +co ge +co bram +co bos +clin ker +chees i +ch ci +cag ed +bulldog ge +bo ken +bluedot festival +bb alli +bar nt +baek sang +av anagh +annon ces +ancestry hour +alp ade +ak aw +air fest +aga ints +acce sori +acade mi +' ~ +ðŁĻı ðŁĴĸ +ðŁĺįðŁĺį ðŁijĮ +ðŁĴ° ðŁĴ¸ +ðŁĴªðŁı¼ # +ðŁĴªðŁı» # +ðŁijį ðŁĩºðŁĩ¸ +âĢĶâĢĶ - +ziggo dome +your struly +yoon kook +whom st +wheath ampstead +walnut creek +visit neworleans +vin ik +victor io +vas co +vallec as +ut aro +un das +ugly ducklings +u sparalympics +try sail +thereal redman +the cc +the catch +team elite +ta kato +swaraj ya +supportw restlers +suni dos +steve earle +sp ms +so gn +sksk sk +securethe border +s giving +ross moor +rosel ine +rock line +ri yan +recal gary +rec itations +read by +razorback bsb +rath drum +rak hee +qu im +pri ors +pri dgen +pre calc +prat ama +play field +perpetu al +perpetr ate +pembro lizumab +pari etal +parac el +oxford street +op ere +o sin +nyul angone +nut job +ntw ales +new gate +near sightedness +ne wapp +nal le +nahu atl +mor nig +mo gov +miy ata +mini ato +mc caleb +mass incarceration +man tu +mak ka +ma shie +lincoln motorco +left most +la hs +l fe +kru mping +kolly wud +kira kira +keys borough +ker zner +ke ad +kathy najimy +kappak app +jo st +islington bc +ingh i +ilove monet +ier ce +ic hal +i aquinta +hom os +hoho kam +hin kson +hawk shead +hahahahahahahaha hahahaha +gre mory +grazi ani +gi rod +ghanaf aofficial +galvan ising +fur se +foramini fera +food hall +fe ku +fe igns +excre tion +ex py +euro barometer +esc anada +epida urus +epic research +end ora +eh burun +e weather +du bu +disambigu ation +desen zano +de couple +d stl +craw for +courte ously +cor teo +cor bally +cooper ations +clare ss +cing ular +chir inos +childhoo dobesity +cann ady +by ler +bucadi beppo +bru tus +brand design +bl itt +big baby +bethe force +beh naz +beauty products +bbcsport scot +base balli +ban ki +b drms +ay bar +av lent +aus media +ar ghhh +ar dg +ap tnnews +anne applebaum +ak if +ait zaz +agul ati +admon ish +abelli o +ab ic +:) !! +ðŁĺĹ ðŁĺĹ +ãĤ¤ãĥ Ĭ +z illi +weg mann +wat seka +view bug +vien tos +victor wanyama +var daman +va sek +uy uki +uy ajola +urin ates +tun ned +tu lane +tranquili zers +to victory +tj dillashaw +timand sid +tho lidays +the piecehall +ted wheeler +tb s +tam o +tal anoa +supportthe swiss +sun web +su ff +statik selekt +staffies aturday +sri harikota +square up +snar ks +sm ate +sh eart +sax lokk +sat su +sas ana +santinof ontana +sa st +ru the +rr rawlings +ros sen +ron go +roel of +real denise +raz ole +provinci al +primer os +pot ties +pin ions +patron ized +pas ukan +p smith +ou ji +open signal +onepur suit +ol am +oh sc +ocean port +o pro +nouri el +nano pore +n bi +mr cc +mr ancelotti +moulin rouge +mon opol +mj biz +mic dro +mi mmi +mer anti +meg myers +matsu take +mak ge +learning disabilities +lanes boro +lamb ada +lam ers +lac to +la thers +la hn +ku lak +ke fka +jeanne ret +jazz nation +it cell +indic ting +i wai +ho dy +hec ke +ha uk +gwen n +grow ler +global trade +glaze brook +gend arme +gen nes +gen ee +ge xpo +fu er +from nov +for bach +fly guy +fivea anews +film director +feen stra +famil ys +etsu tough +eru th +eris kay +episo devi +ense i +emb run +dÅį terra +dor able +dj john +dha de +deu el +dal awang +da aaay +cul bert +cosmic gate +com ity +chi ana +chau bey +charl ene +cd cs +catt in +cat ts +canadas wonderland +busines spro +bur ts +brecken ridge +br aryn +borgh ini +bol lock +big room +bi king +bee flamb +beartooth band +be cht +baw dsey +barri ster +bar dic +babylon ians +ato dos +ati ger +artistre sidency +aristi des +arch ons +aquin nah +alesha official +ade kar +abar nes +ðŁĻ į +ðŁĹ Ĵ +ðŁij¶ ðŁij¶ +ðŁIJ¥ ðŁIJ¥ +ðĿĺ Ĥ +âĸ ĵ +ภľ +ภ© +yu ja +yl t +yan es +ya ha +wwee volution +widen er +white gate +wautu th +was sen +vy jay +voci ferous +vivienne westwood +villar rica +vili fying +ven nes +vast a +up allnight +univers iteit +un encrypted +tip the +thermo fisher +the ecnl +take meto +t bon +suc cor +stor mi +stan wix +spl ans +spar r +sna sh +sm caen +skipp ack +shireen mazari +shan ola +shanola hampton +sh pe +se gol +science innov +schu errle +sato shil +saietam hankar +sag arma +roy ndtv +rogerk ver +richar dayoade +refin ers +rec ency +rational ise +qay amat +q ps +prit char +prannoy royndtv +perri ello +paulow nia +patern alism +paro dy +ox man +no shoes +ni aid +national relaxationday +nail design +my work +mw amba +mou sc +morwen na +mit smr +mcga w +mas set +mar ai +maple leaf +man ea +m ity +liber ata +let there +le fe +land ini +la paglia +ku hlmann +king glass +king air +ke c +kal las +k schi +jol lie +jeho vahs +is so +is forever +is aca +iowa hawkeyes +info blaze +i zza +i bru +hot man +home fires +hau ter +happy jiminday +gric huk +gri eve +gold ar +go ian +ginacar ano +gad h +g police +francis ville +fer gie +evalu ative +ent ure +enamel pins +eleph ante +duc tal +du rov +dou bl +din om +diet ary +de vol +de is +construc t +comence m +ci hi +chit ter +chickas aw +chef slife +cel ty +cel anese +cau sative +camel toe +califor niag +buff o +brighton museums +black hearts +bian co +bac chae +baby shower +azu cena +at asco +as ai +arap aho +anni bale +anne music +ancho ress +an boy +alo x +al bam +ad alia +ab akery +. âĢ¢ +ðŁĺı ðŁĺĪ +ðŁĴª ðŁıĨ +ðŁİĵ # +ìħ ĺ +æĽ´ æĸ° +ãĤ £ +âłĢâłĢâłĢâłĢ âłĢâłĢâłĢ +âĿĮ âŃķï¸ı +áµ ķ +à° ¿ +اÙĦ ار +айд ан +za ke +yan ick +wu ld +woo oooooo +wis casset +westen ra +well house +warner music +visit singapore +vinyl mation +vic hai +vau tour +twitch ell +triath lons +topdrawer soccer +throm bus +the stormers +telerad yo +sta ston +sports marketing +spor tn +sli mmy +sk un +sk aro +siff ert +shi ong +she aths +schind ler +salvationarmy uk +sacred heart +sa ev +s fts +roll humps +rodri quez +research lab +regi er +rebellion racing +real jeffrey +ptc punjabi +prep xtra +portu gues +pic us +pha ya +perkins will +peng en +pe il +pavi thra +panto graph +overs atur +ontari os +ob olus +oasi s +o keefe +o date +nuc football +ni block +newzealand shooting +new pic +ne maha +nbapra yers +nb forum +natsci wk +nati vist +mut ter +multi strada +mor awi +mollu sks +micro organism +met zinger +meit ner +med anta +mayo l +mattb ourne +mar saxlokk +ma juli +m lo +m hq +lym phoe +lu fisto +ky lene +kom pakt +kirch berg +kim jiwon +kim jaejoong +kati el +kali m +kali kimaka +k sco +iyan la +ikar uga +hu u +hog ben +hay ami +gujar ati +graywolf press +gra ds +gor is +fron tal +for s +flow in +fir stuk +fiel dy +exper twitness +exhum ation +estro bel +ep silon +ee w +dotte rel +dock land +diss a +digi ov +del bonis +de math +de aky +dd able +dave hill +cosi danews +cos sette +cooking channel +coo pe +coconut grove +coale scence +claress ashields +civit an +ch asse +ch alco +candid at +bush heritage +bu kitt +btsport football +bor tz +blm national +best giftever +berlay mont +ber bera +ba ofeng +athanasi ou +art se +are iner +apenn ines +am undi +alysi areiner +alo tta +al preps +akh dar +akak atie +ahadra zamir +acram pton +ac custom +? ..." +... ðŁĺįðŁĺįðŁĺį +ðŁļ Ń +ðŁĩ¹ðŁĩ ´ +à´ ļ +Ä § +yut ani +wont forget +var ty +upri ghts +tu pa +trape zo +ton ites +tile work +thor sten +tere sina +tech live +sud ley +style dotcom +strate gi +stephen kamos +spar ke +snel grove +sharpe ville +semis onic +sbse urovision +sal sac +sa ho +s wnt +revenge ofthe +rat ory +rancher ia +q tr +piran esi +picture show +pi as +penetr ator +pattan aik +paloaltontw ks +oul try +os mani +ophy tes +ome gas +ole ds +ol inium +of texas +nfl honors +nc wts +nbak icks +n newi +mug la +monop rice +mon áe +mol fetta +mohawk austin +mo lest +mo apa +mir jana +middle school +micro graph +mecon opsis +mc menemy +mau romanzi +man gam +mal lah +mac graw +ly kos +low brow +lock n +li mas +leighton buzzard +lang lais +lale h +lac ity +kuch era +kid sto +kent in +kell yosbourne +kander steg +journe yof +johnny sauter +jenny slate +je wison +je ane +jaun pur +jame se +israeli apartheidweek +innovators mindset +infe ctive +infan cia +indigen es +in scri +iklan laku +i bike +hypo thermic +hyper loop +hy mers +har ge +guar ini +great lakes +good friday +gf wc +g pu +florid alife +fer nan +feel my +fat emi +export ation +expediti ously +evidence based +ess y +eg ge +ed oll +ec ml +dundalk stadium +down hole +do fc +deno ted +del tic +db q +d ki +couples therapy +cm da +cine bl +chth onic +chelt litfest +charity jobs +casio pea +by ington +buck nell +bu kavu +brink worth +bor do +bog of +bifur cated +bhagy ashree +bha sa +beech nut +bb clondon +battle creek +aux in +att itudinal +apu esta +angel adu +and counting +ðŁĺĪ # +ðŁĵ½ : +ðŁij¯ ðŁij¯ðŁij¯ +ìķĦ ëĿ¼ +ëįĶë ³´ìĿ´ì¦Ī +ç ¯ +âľħ , +âĺĢ ðŁĮ´ +à¤ı à¤ķ +za anse +z wer +yon tour +yo bs +yearofthe bird +year old +xx xo +women against +weapon ization +water berg +was me +vail mtn +un circumcised +ub ong +u equalsu +the movie +teh seen +taxcutsand jobsact +tar u +ta wheed +super kick +sub tler +star ves +staff picks +squ inty +spla ys +sof fic +sof america +sher mano +sher loc +sheffield sharks +shap ley +shahi da +sha hr +sh ve +ser ps +scifis unday +science isfun +sat ra +saint louis +ro kko +reyn or +re tai +pro jared +preven tive +prest wood +pre cooked +port us +pont cysyllte +pod fest +pet sitting +perturb ator +pepper mint +part agas +pal ay +ori e +oneof my +no pressure +naray anas +my soul +my hre +murdere ss +mund sson +moun ts +montour sville +messer smith +men with +medical tourism +med chem +me azza +marsh alled +man oran +man ase +les ss +le auge +lati han +lat ini +lass ical +kelving rove +kat barrell +kam ak +jor da +jesper sen +jason momoa +jal ep +ira c +ic able +i earn +han az +hacker rank +grow ingthe +grand aughter +giant stalk +gat ch +g aca +furio so +for al +flight plan +first dogon +fir dau +final score +er dal +emer ic +east carolina +duc at +du enas +dra gunov +dilett ante +di aa +dete sted +de ferment +david baldacci +cu kes +cu entos +coy gib +cos way +cof state +co let +chin apa +chimp sin +cat enary +cardu elis +buck walter +blue gills +blair stown +ben nevis +bday in +auggie pride +atwater village +as kra +army football +alis al +alebri jes +ala fia +ade adepitan +a itis +... ", +ðŁĺį ðŁĴĥ +ðŁIJ¢ ðŁIJ¢ +åº Ĺ +ãĥŁ ãĤ¢ +ãģĻ ãĤĭ +âĨ ĺ +ย à¸ĩ +ب ÙĦ +z andra +ye su +ye ho +winnipeg sd +win cities +wil m +wil dy +waz za +wal ters +victory venkatesh +vi el +vel ika +vej le +us the +tyrrhen ian +trze wik +tri xx +travis fimmel +transvest ites +tr ant +timefor wiltshire +thebachelorette finale +the vijaymallya +team rubicon +tax cuts +tab ou +stopthe pressure +spu tum +son ni +sj m +semi h +sat chell +sapphi relv +sa kon +s sum +ryan sheckler +rugby romania +rn cm +rebel des +pipp in +paul kingston +park let +paint in +on zalez +omni vore +official foxes +nive thathomas +next conf +newyork jets +mongol rally +monday nigh +mol teni +modi for +mill sand +member news +mechat ronic +mayak oba +marriageequ aility +man servant +ma rey +m str +ly udmila +lego starwars +land race +l kl +kirky ard +kir ani +kil ic +keiyn anlonsdale +kan awa +kalisto wwe +jonjo oneill +jon ne +johnny gill +jin hyuk +jd bc +james ville +jag d +ja ip +j clark +j any +iz elos +it rust +in british +ig ins +ib v +i pec +homolog ation +holi er +head ford +happybirthday suriya +h ki +gli wice +fro dd +fon dre +flag football +ey nsford +en casing +edy ta +earnest ness +dun ja +dro pou +dnrr ts +dit official +district champs +dinesh karthik +din ajpur +dere kk +dend rite +dehuman ized +craft scounciluk +counsel or +cou zens +conven to +con uk +commu tation +colon izer +co xy +ch ré +car star +can h +cam ac +black musicmonth +bhadra k +bein sportsusa +bean er +ballan trae +balag tas +bad alamenti +back ings +ay ork +as gar +all sorts +aldu bun +akin ola +ai ir +ach ap +abz love +absol ved +abol ition +a heart +(- _ +!! ??? +ðŁĺĥ ðŁĺĦ +ðŁĺº ðŁĺº +ðŁĩ¨ ðŁĩº +ðŁĨĺðŁĨĺ ðŁĨĺðŁĨĺ +âĹ¾ ï¸ı +à¸į า +à¤Ń à¤ķ +z sas +y igal +y ho +x sweat +won th +westlife music +we ahps +vikh roli +un certified +trish stratus +trishstratus com +tran slivesmatter +tor q +top cow +tho resby +sydney derby +sun bel +stol le +sti verne +stere otactic +stage hands +st ful +son ge +sl qld +show stopping +show room +shakun tala +sergi us +senator leahy +sed ski +seble febvre +se gues +sb meunier +satisfactor ily +salut ary +salmon arm +sacro iliac +rugby club +rif fi +rfr driven +red squirrel +rabin dranathtagore +pror sum +presiden cies +premi o +pe trick +pay master +oo hs +onehappy island +on radionow +on dra +old city +ni ma +nd ong +my le +multiple xing +morning ireland +monday madness +mo zi +me urs +martin scorsese +ly nieg +lou ella +little more +legac yof +le estrobel +lancashirec are +la skin +la sch +khe da +kang er +kal icious +ka be +j clayfield +ire alestate +ing season +ing ay +infinity ward +inciner ators +hv ff +hun wx +hol royd +ha gh +guine an +grand hotel +gr itty +gol fe +forhonor game +fomen ting +fla grantly +finn aly +fat so +far ley +ey d +ent ra +ec fr +earth changes +e gli +dvs bjp +dutch sassenach +dun ne +dod gy +dip day +dichotom ous +day ao +da ja +cul ter +crooz ef +crewe alex +content creation +con very +com sat +climatec entral +cin dere +choir boy +chen es +c grand +bri stly +brew erton +bragg ing +boosie official +bol on +bisp hen +barbour ville +au tent +at exas +armin ia +ari bbons +$ " +ðŁĺĬ ðŁijįðŁı» +ðŁĶµ ðŁĶµ +ðŁİ¥ ðŁİ¥ +âĿ¤ï¸ı ðŁĻĪ +ઠ¿ +Ùħ ØŃ +zip lock +zind ag +zen y +z hin +yo hanna +yan chep +xen u +ww h +wild horn +wil tz +whist ling +watch nerd +vj fan +ve eee +v alia +u vp +trans rightsarehumanrights +trac kies +tr ong +toyo tat +top bloke +tin dale +tic kers +theodo sius +thel word +t Äģ +t xi +t life +super furry +sum ner +stra thal +stan sberry +stag ey +soci alistic +simple minds +sim guru +shi pley +sch mo +sc ath +sal kinstitute +riyad h +ripp led +region alli +re gran +rab wah +quan trill +pu ya +pu tre +pp en +por tait +po yer +plang lobal +ox eye +out ly +nu mark +now ar +nord kapp +ni kh +never forget +ne spress +na ilogical +mu kul +mr sm +mo hale +mmor pgs +melli fera +mediaev alis +med hi +md phd +mascaren has +mallikar jun +ma sha +lor dand +listen ings +lets work +les doggg +lac of +kum is +klu gman +ke ch +kathleen lights +kar d +jyo tika +justy na +jimmy tatro +jere m +je eta +jamal khashoggi +jac y +ivani sevic +indistin ct +inci pient +ic kes +hush puppy +ht city +ho en +he kla +guer os +food history +flash tvwriters +fa uns +exten sibility +every can +euro tour +er da +ent ini +elitch gardens +ef endi +dublin ladiesg +dj c +dismoun ted +depu tized +d ze +curbyou renthusiasm +cook stoves +comeback home +co di +co chine +cierr aramirez +christma sat +chri sv +chap ul +castell icycling +bruce buffer +brownlee tri +bra der +boeh mer +bezer ra +believein science +bek end +band es +ban kin +ay ey +as crs +argu elles +ar ild +aniso tropic +ame i +altere go +afate hi +adri aan +ador as +. __. ++ +. +ðŁĺİ ! +ðŁıĭï¸ı âĢįâĻĤï¸ı +ðŁ§ ¨ +ப த +yo hn +yaf oods +woo de +with asmile +wild dog +wex for +wacky wednesday +vinogra dov +vici ousness +van pool +van il +vali dations +vachi er +urban ag +ur ate +up ward +unsun gheroes +tom oki +tit mouse +the full +th ah +taarab t +stopp ing +steve kornacki +sports bet +smar ta +six flag +shrey ast +shashi kapoor +sd pi +rose mond +rha egar +rei ver +ran ter +rac gp +r nt +quanti fied +py bus +pre dating +pras tha +pi rogue +pay porte +p stn +outra dio +out size +on ita +o ez +nt k +norr köping +ncp speaks +nasty y +napp ingday +nad z +nab f +mul enga +msc athy +mon tie +mo gensen +mn gov +mira bal +mg book +menshealth week +men gi +melane sian +lc willi +l ound +kap liskova +kak u +john force +joann ac +jam elle +isal oniofficial +illustration oftheday +il las +ik oma +id gi +hyun suk +hoo ker +home built +hein le +h ml +h itec +gun geon +grü ner +greater manchester +gojay sgo +girlscout cookies +gi elen +gda ÅĦsk +gateway msp +gand ini +for migration +flat worm +finalfantasy vii +fet tuc +ey yc +esp ence +erry day +ero yal +el t +dubu isson +dro ma +dre x +dont frack +dom on +do ree +diony sius +dic icco +daysof giveaways +cour ter +comuni dad +co ag +clow ne +clemen za +clack mannan +civil s +cate chist +cash less +carlos vives +call toaction +c jv +bu suk +bu soga +breaking dawn +brain awarenessweek +boeing lovers +birch mere +bic hir +benef ice +bed does +bc swx +awwwww www +au pe +ati shi +as prin +army cadetsuk +ar um +american us +ame me +amber rudd +air print +a hahahahahah +!!!!!!!!!!!!!!!! !!!!! +ðŁĺį ðŁĺĮ +ï¸ıâĥ£ âŀĸ +ãĢ ī +âı ¯ +ë ns +zha o +z hir +womensmarchon washington +we yl +wc x +walk outs +w ampus +vote selena +vainglory game +un blinking +u clou +twat ter +trelli ses +track way +tom ino +thewire magazine +theli brary +th ly +term ly +te vans +tam pers +summer of +stereo scopy +star in +spatio temporal +son u +so called +sl on +sigma beauty +shopp rs +shopprs drugmart +sevasto va +se eu +sally kohn +ror onoa +ri zz +ri xon +redband society +red cedar +re imagin +rathfr iland +raman ath +r vel +quo ter +qu ico +pur itan +pu ss +protec te +pro lac +prin cy +pr icked +pioneer sports +pi hl +pc m +paint balling +ow boy +oper cula +oli mpo +newfound land +net w +multi polar +ms me +mo hinder +metten berger +lympho id +logi stically +lock land +lek ker +ker io +jas curtissmith +jap heth +jam rud +issa char +in schools +ic ba +hil mi +hereto fore +happybirthday tomhiddleston +happy camper +hamilton police +grrr graphics +gp news +gore tober +getin the +gar anti +for sch +first nation +firel ord +fe o +exi le +everything is +escheric hia +emmit sburg +elec teds +egal itarianism +ed scifest +eag u +ds india +down burst +dj sam +dit ties +demo tt +d zn +countyo fla +cor rente +colom b +cla ssc +civic engagement +circuit spa +cer ita +cb spittsburgh +case ys +cad walla +buildthat wall +bu rian +bsnl corporate +bru dder +brian dozier +brahma stra +bor y +bo gos +blu eridge +bio log +benzo yl +be lek +bach pan +ask fc +ar haus +aq pk +animal s +andrew garfield +alex marquez +afl cat +ðŁķµï¸ı âĢįâĻĢï¸ı +ðŁijĩðŁijĩðŁijĩðŁijĩ ðŁijĩðŁijĩ +ðŁıĭ ï¸ı +ãĤ¿ ãĤª +ãģ· ãĤĮ +yu ha +yo in +y ooooooo +wish master +wis bb +win noch +wiffle ball +wh ch +was now +war sop +wak ening +w gy +ver gel +vasek pospisil +us gs +ty rer +ty lers +twilight zone +thre estoo +the dani +terri irwin +te ez +sr anieri +spla sher +sopra steria +skysports golf +si bil +shreyast alpade +shipp eo +shel lie +sg haze +serv ile +sens orium +sd ons +sark cess +sant al +sag rad +sa wang +rey mond +regent street +re clu +rayal aseema +r kd +quantumb reak +py pchat +pu lev +pryn ne +pran am +placebo world +pj vfl +pis ky +phoebe j +paul malignaggi +pau lap +par cheesi +oli an +oc ci +obse qui +o hel +nxi vm +noo b +ner ja +neil gaiman +naf to +my on +mutu alism +mur d +mil dert +micro brew +merz bow +mel don +mehe kf +mary lee +lun dell +lucer omexico +lo oooooooooooooooo +le val +le mann +kru dd +ke dua +kal itta +justinbieber updates +jo stens +jae won +jack ingram +iz anami +iv t +it our +iri sranieri +ing tons +incon el +illi m +ili kazi +icon gress +iam humayunsaeed +i ley +howit smade +hiphop culture +highfive day +he tzel +he mic +hat oum +ha aaaa +gun du +goog lead +g gie +furiou spete +food matters +fire break +ev ang +esk ridge +en rico +electoral college +ef nd +demon io +das ari +dan akil +daily bread +crosscu tters +cot ours +commen taires +cineplex movies +chu ke +chop house +chae un +cathy newman +can to +c illiers +byrne offic +but su +bush ranger +build april +blockchain news +blephar itis +belle ville +band aran +bal ram +autumne quinox +at ate +arrivab ene +anfer nee +and ance +an aw +amnesty online +aldi shodge +ald n +alberta party +aha w +ac na +abhi jeet +aashi q +. ðŁĴ« +ðŁĹ ij +ðŁķ IJ +ðĿĺ ª +ï¸ıâĥ£ % +ç ¢ +åħ « +âĿ¤ï¸ı âľĮï¸ı +É ´ +ysby ty +wolf of +wire taps +win nen +weigh ton +vis wanath +vare se +var sh +v mo +u paz +ton t +toga dia +titmouse inc +tir pitz +the goo +the en +ter psic +tax returns +tal mage +tal espin +syl vio +sword and +sty mest +street games +stateli braryn +statelibraryn sw +star lit +stan collymore +spaz zing +sk ender +schlu ss +ricky martin +ri bon +rep ta +relat edness +ream ers +re pointing +re grouped +rd brwanda +privi le +primiti ves +politic shour +plato oning +philharmon iker +pe cz +outra ging +outnumbered fnc +ound stone +ome coming +nh ss +ne ills +natu real +mustang monday +mus kets +mul ally +min ty +meth il +me pauloavelino +margol yes +mar sal +mar lean +manage ability +man joo +mal volio +louise hay +lo cky +ln dont +like sfor +lein art +laurent fabius +lau rak +lang staff +lake head +l fb +kö ni +kut u +kimjong kook +kim chee +kaz oku +jyrki katainen +just wann +indiebook sblast +hyper glycemia +hun ath +hub ner +hu en +hel wan +h pl +gru dging +goth ams +gir orosa +gf dl +geoc ello +fuen labrada +fluor inated +fal chion +fad ell +ev am +el or +ed urne +ecm records +dos barth +die sem +demon ess +de michelis +dale jarrett +cu sa +cre atec +coach t +cher an +centri petal +centre for +cc cu +cañ ada +carpetb agger +cap elin +camel ford +californi achrome +buz by +bru cel +brit onedirection +blow y +bett pride +beth hart +basso onist +bahau ddin +baaaaa ack +atleti smo +assemblye lections +asha ikh +as usual +ari ola +arac al +and harry +and alou +amo dels +affl icting +ad ric +ad ow +................ ....... +.. ðŁĺ³ +" âĿ¤ï¸ı +ðĿŁ ı +îIJ į +ìĭľìļ° 민 +âĻ¥ï¸ıâĻ¥ï¸ı âĻ¥ï¸ıâĻ¥ï¸ı +âķ Ŀ +๠Ħ +ÛĮ Ûģ +zom b +yas elf +wonder ous +we oy +viva x +victori afalls +v do +under world +ultra suede +trafford centre +thecameron boyce +the killing +the captain +tech awards +tabby cat +sw gn +stron omy +stre s +sto be +steer age +statedept spox +spo em +speci aled +son oma +social saturday +sli fer +show t +scar th +saf adi +roxeter aribbons +roman us +rock paper +rock mart +redbulle sports +red fox +ram ar +rade macher +pyri dine +pslon espn +ps ja +pra der +porttal bot +play like +pern illa +pat miletich +passe ig +park haejin +pag ode +pack mensbball +pac ini +osc e +oh sehun +od ka +octo bers +oc sd +no vos +new amsterdam +na she +n he +mz ilikazi +my future +mun ny +mu as +mrmark millar +miccosu kee +matur ities +malay alee +make ssense +mai sie +mah nke +magdal eno +madon nina +mac ul +local love +list ing +life lesson +li ot +letter boxing +lat ri +lamar z +kha rel +kapp adel +ka ard +k camp +johnny orlando +jo geshwari +it secure +is yettocome +ir vana +igh ted +ic r +hoo kahs +hell zarmy +happybirthday ssmb +go bison +gille smarini +gill mor +gau dette +future proof +fondaz ion +e tech +dtop beautyworld +diam anté +di zen +dar cie +crown royal +con ro +col er +coffee houses +cho dy +chi ao +cer y +cash master +boys lax +black rod +black holes +black dog +big ass +big as +beau pre +aw p +aw oo +au cd +asi ant +as cj +as app +artbasel miami +amo ss +aly ona +ach ro +aa shi +!! ðŁĶ¥ +ðŁĺį âĿ¤ï¸ıâĿ¤ï¸ı +ðŁĺ± âĿ¤ï¸ı +ðŁĶĿðŁĶĿ ðŁĶĿ +ðŁ§ ª +ê´ ij +ãĥ¼ãĥ ŀ +ãģ¡ ãĤĥ +स म +é on +zu ehl +zent angle +yu sh +yoko ham +yn books +yamaham otogp +x sd +wu yi +white oak +vs v +vend redi +vashish tha +ty oung +twit r +ti ssa +thelast airbender +the sz +the post +the fanatic +the batman +the barn +that ches +ter ah +tech tip +tam ente +take two +synucle in +sy strom +summer learning +sug u +still births +state visit +spir ing +sou ley +sol at +sir ken +sham arie +sd urham +sale ha +sab ena +rosenber gradio +roll chos +rock ymtn +rev richardcoles +re formatting +ra abe +pyaar ke +pu dhu +property investment +pron k +prem peh +poly morphic +plains boro +pi fy +paridhi official +owen sville +our blues +ortho tic +orange county +one hit +oncein alifetime +official fpl +och il +noordinary park +need more +na jm +n wi +my voice +mul loy +monopoli stic +minne dosa +mesqu ita +men za +me sures +marketing derby +mark bam +mari aton +magni ficient +madagas can +ma bille +ly tic +liquid metal +line game +l xi +koo b +kap an +k wo +k kinen +juice bar +jim town +jason taylor +jaku bowski +interro gator +insomni acs +in orbit +in cube +ilu lissat +il ala +hu p +hotel chocolat +high castle +hi ggle +harhar mahadev +har tt +hal cones +hain pyaarke +h eneral +guineap ig +greg cipes +gra hi +gordon sville +ghost light +george s +gari bay +gander bal +flex time +fish scale +fight newsasia +ferru ginous +et trick +ei ge +eber hart +e ile +dzhok har +du mars +direction ers +dinner with +delor aine +delac our +de merits +dari usz +dan anda +cor iginal +com port +cityof stjohns +chu eca +ch ya +bul ling +buddy holly +bru gman +bri enz +bri ans +break sthe +brazil vs +bon field +bod mer +black ham +beer news +bath lifemag +barber shop +ban y +as chaf +ane gada +and yl +always keepfighting +afghanist anyouneversee +acqu i +aco ach +ac red +abb ington +^ ; +ðŁĻĪ ðŁĺĺ +ðŁĺī " +ðŁĺĪ âĿ¤ï¸ı +ðŁĮĪ âĿ¤ï¸ı +ìĶ¨ ìĬ¤íĥĢ +åħ ĥ +âĿķ âĿķ +z ica +wiz ened +win co +white boy +whi les +video today +ves z +var um +unite foreurope +unic um +typhoon team +twcnews roc +tren holm +toiletek prem +tart aglia +ta ints +sun daze +stor ian +steff london +ste ez +so hr +sher gold +shepp ard +sean j +sealteam cbs +sd mc +scott derrickson +schwar ze +sant olan +saad hariri +s aper +rep ú +rein car +recou ped +re mon +raf typhoonteam +prokhor ov +probation ers +predic tion +pla sterers +pic public +pel sall +pe dium +park hyungsik +pac ke +p ten +or theast +op sal +op art +old fashioned +oh snap +of oundation +nu k +nsc n +noc news +nl w +nikki benz +nik phillips +ni gri +ne ek +nar singh +n ulty +mö bius +mur komen +muj taba +mt bs +mobili er +mo tti +min aur +mil grom +mei jer +meet me +me len +matt kindt +marchin march +madhu kishwar +lo fa +liv tyler +lilli putian +ligh thi +li bo +lfc tour +leone tti +lend ingclub +l te +ky k +kristy n +kar asu +k love +itsecure gamer +inv itee +inter face +in churches +im hyunsik +hyo sung +hillsong united +hiberni anfc +hann um +h wd +grime ys +green leaf +gom anga +gof theweek +ger v +ger ring +geof fre +fun hau +fra ss +fox holes +food academy +flu dd +ferr one +fau stian +fal zone +fairfiel dhalls +es net +enqu iring +ear flap +dud don +du pain +dou h +don quixote +de dan +dam in +dak shina +cro co +craw lies +cli m +che pe +ch ona +ce bo +cary atids +cartoon saloon +capta insp +cape zio +c tica +buil ten +blavat nik +bigsky mbb +bb najia +aw st +ato vic +arch icad +aniche be +alz ado +ali mi +ale agu +aco aches +, £ +ðŁĵ § +ðŁĴ° # +ðŁį» @ +ðŁĮį ðŁĮİ +ðŁĩ¿ðŁĩ ¼ +ðŁĩºðŁĩ¸ ðŁĩ¬ðŁĩ§ +ãĤ¦ãĤ © +âĺķ âĺķâĺķ +âĸ¬ âĸ¬ +window pub +will an +wester lund +wemb ury +wei z +un wired +u ih +trump world +tradition alism +tomor ro +ter apia +tan nic +swa the +stri ppy +st kitchen +st ator +spi aggia +so ay +sing appen +shermano aks +sha ima +selek tah +schir ra +sali ence +ro castle +rick steves +rhy n +regenerative medicine +rahu lr +ra zy +positivecoach us +pos is +pir aten +pi enza +phoebej tonkin +pey roux +penny mordaunt +penguin book +over comer +ott mar +orange shirtday +or dos +open to +open society +ofsted news +nomin ator +nialla ppreciationday +new ent +nab or +n ão +my banff +musk er +music box +mun dt +mtn training +mol in +miz pah +mind control +mer sch +mean green +marlon brando +market day +man ica +löf ven +lusit ano +loyal sock +l bhf +ku f +kri hanoff +kindergar tens +kgal ema +ker f +keg worth +kal ba +jonm chu +je ggings +itu mbi +i isd +hur ries +ho del +hashi motos +happy mondays +greenflash beer +gjallar horn +fun t +fu ssing +freu denberg +evening chron +evan escent +en stadion +en li +en eng +emer y +eddi ggs +eber ron +dys regulation +dumfries shire +drive srt +down pipes +dom ode +do vi +dick ory +deal in +dave eddiggs +cyfarth fa +cryp ts +cro ix +cro cody +conju gates +cma openaccess +clo va +ciarab ravo +choic est +cho es +chel on +celesti als +car acci +cape hart +buy itnow +bur nishing +bugs bunny +broad band +bra ue +bon usu +blasphe mer +bestin travel +baz oo +azu mah +at bristol +asitis official +asi asoci +apost ates +annual meeting +and ito +amar an +alys sum +alltogether now +allinwith chris +akh gar +aj opinion +ais yah +ade ma +abi bi +ab dy +[... ]" +.. ðŁĺĺ +ðŁļ£ âĢįâĻĢï¸ı +ðŁĻĪ ðŁĻĬ +ðŁĶ¥ ðŁĺĤ +ìķ ł +à± ģ +° - +zea xanthin +your quote +yak umo +wy rick +weare bangalore +we mo +war lal +wak rah +vien nois +veri fications +uw gb +tusc on +tt ank +troy trojan +tos link +til is +the struts +the square +tax ila +tahaj jud +syring a +syd al +stra sberg +stor ino +sreed har +sport news +south la +software developer +sk off +si ona +shangha inese +shack ney +scou gar +rv sed +rockstar spud +rm sothebys +ri pl +proprie torship +pro ss +photograph ically +phenter mine +phase out +pe gging +paul deveno +part ys +p wa +out put +out ines +or que +ok one +nyakun dih +nu suk +nove m +new profi +net weaver +ne ot +nb sat +napalm records +musical uk +moven pick +moss op +mo so +mi eri +mets rewind +meta search +merry man +meh tab +mar clay +maiden head +litur gies +letsgo flyers +lechu ga +lari more +lanter ne +land trust +laber into +klein hans +kidap awan +kat chi +kam boj +kal isz +k é +ju bba +jorgeramos news +j sw +iron bark +ine wa +in oran +ideac ellular +hey ne +hex adecimal +hemo dynamic +he ssen +haydn jones +hand bills +gru ene +grow the +gretsch usa +gooo al +good toknow +go sho +go sei +go il +freeall arrested +for bury +fin cen +file maker +fear rts +evolu tionists +es ben +engle bert +eli ak +dur can +dit er +dante wada +dan rather +daler mehndi +d jane +cy world +comp ell +clo ves +cic lista +chol angio +charlotter usse +car bor +cap oue +buzz r +bur goo +bra sse +bol dini +boh dan +billion aire +bha sker +bemel mans +beach vb +barbar acrampton +bar ik +aval kyrie +au brac +as cal +appare l +ak ick +aa o +\ âĺº/ +) ðŁĺĤ +ìĻ ķ +âĮ ¨ï¸ı +Õ¡ Õ +zen n +yo te +y allop +wo tt +weird beard +w pc +vogel sang +vand or +ultra sa +trump colluded +triple m +timid ity +tian men +three word +thebold type +th ops +th oooo +tess gerritsen +tejas wi +taylor kitsch +tar k +swi ffer +su hsd +started fromthe +sr f +sou dha +soon ish +son theroad +soder strom +sig ar +sennheis erusa +sch ley +sant ner +sa way +s johnson +ru lon +resi sti +raj kapoor +rad key +plow right +pic keted +pha d +per cept +per alejo +pema quid +patrio tic +paras ympathetic +pak tika +org is +orange amps +ol au +o jt +nice day +nat cap +nandamuri balakrishna +n indies +mor ghulis +monk seaton +mazel tov +mat ura +mariaton tini +man si +man kins +mali shka +male fic +mal tag +mak ran +mae gan +ma resca +love theatre +lord swood +loch gil +lily hammer +licht man +li kers +li bia +li bi +ley bridge +la vag +l ams +kon itz +kn aggs +kar lis +kam at +kal uga +kair at +ka on +jo brani +jim irsay +ja the +i sim +hywel dda +horn buckle +hi za +hekmat yar +gu energy +gratuit ously +go rebels +give way +ghay al +fishing trip +fis chetti +far da +fabi en +eus kal +es com +eco sia +du ar +denomin ators +del bene +de hesa +coup de +cor gi +constra ins +co kie +chiri qui +chesney hawkes +change your +central bank +cb university +case mates +carra untoo +ca podi +boy stown +bloo dier +ble an +bio remediation +ber til +bar tali +bar ryo +bal ko +b marshall +aw inner +aus geo +ath es +ash ami +as ako +aquaf aba +alle mands +ak havan +agno sticism +afl q +afl power +ab sar +ab ong +ðŁĺĥ ðŁĺį +ê¹Ģ ì¢ħ +Ú© ÙĪ +اÛĮ راÙĨ +ä ger +z wari +z q +young king +yo joe +y fg +wpl g +wmtw tv +weare south +vm wa +viscer ally +val ore +uni part +the storyof +the crystal +ta fen +t jr +sure tte +suffolk wildlife +su thers +su mut +squ anders +springh ead +so rey +sin fully +simm s +seme a +se phor +sarang ha +sal sha +saga ins +red turn +ram us +radi onica +pre me +polon aise +po els +playstati oneu +pi hu +phan art +palu stris +pal misano +pab udget +outdoor play +out music +ont liberal +old friends +ok amura +ode tte +nu star +news readers +neural network +n lighten +n bbj +my artwork +mscathy gonzaga +movie s +moen ch +mit tee +mi halik +menis cal +mag ine +mach loop +lon garm +live veryplay +lit era +lingu ica +lavat ories +lau ber +lat ona +lang ata +lake huron +knu d +kla ssic +kin nikuman +kad dish +jo dee +jap antour +jan ssen +is cc +interior inspo +inst al +indian ambb +in mortales +i vens +humor less +head cover +harvar dg +happy birth +hani f +haha i +gur gaon +gun smithing +great white +gra ben +good read +gim let +gg ae +germanshorthai redpointer +geor geous +g jer +g adam +flun ky +fi p +fat en +execu tors +ethno logy +est alk +el abour +ef arms +e je +dood lin +dof fro +do ted +deutsch en +determin ate +de itz +cre pe +corn u +coo tam +continu ities +columbia journ +classic films +claire holt +cl ario +châ tel +chief srugby +chal ker +ch the +center parc +caroliner hea +capric ho +can cun +can aday +cam pp +ber land +ber dan +ban chan +bab uji +ba aa +austin healey +armani exchange +ar jen +anemon efish +andre ana +andreana vedo +alu x +absten tions +aac tas +\\ \ +! ðŁĺ¡ +ðŁĺİ ðŁĮ´ +ðŁĺģ ! +ðŁĹ ¯ +ðŁĴª ðŁıĢ +âĿĹï¸ı @ +âľĮðŁı¼ # +yn hs +y gl +wise shopper +whatmakesme happy +way bill +vo key +vo isins +vampi rism +uw f +unce asingly +un mentioned +un impeded +ugly sweater +uc chi +u winnipeg +tran sur +tom ok +the odds +tex tes +tac tfully +syd fest +stopbrexit save +stin co +steven mnuchin +sor tium +solom ita +so tn +silvers miths +silke borg +schotten stein +san zo +sam winchester +rust led +ru xton +ru tt +roy ston +rival schallenge +rish fa +rise again +rig ours +ri or +repre zent +refe reed +r antanen +pwn age +pure michigan +pro mark +prithvi theatre +pride aux +pre spa +pre edy +polyphen ol +pieceof me +personi fying +palit choke +pa kor +over flying +oo ow +nifty warehouse +ne aq +nay py +nak usp +n ør +muumu u +mukun da +mor ng +month long +michael smith +metho dists +mem ri +mcallen isd +markj weather +mahindr aracing +ma wer +ma bus +lc clondon +ku leuven +klo of +kir ill +kant ner +kalin owski +k ichi +juven al +joe manchin +jesus freak +jenn colella +jem al +iwant clips +inflam mable +in ic +if nt +ida ireland +hudson sbay +hert smere +heati son +hayden byerly +han nover +h di +gre tton +giff ard +ghis lain +ge sser +gan ton +funhau steam +fun t +fuch sias +four five +fonten elle +fiber art +fc basel +family values +et pt +eri ff +earl xsweat +e map +dy er +do jos +die ffen +de files +david c +da ji +cou shatta +chi sholm +che sh +channel uk +cc dc +cash time +car fest +calder on +cald beck +c india +bway con +bre ssler +bi bis +berg quist +beal ach +bay shore +bartol om +badtam eez +az abu +ati sh +appeti sing +anti balas +andre ja +anand amayi +almaz an +alit abattle +ali sher +alexand ro +akame gak +ai ves +acon roy +ach ef +absur dism +abhishek bachchan +... ðŁijį +! ...... +ðŁĴĺ ðŁĺį +ðŁĴĥðŁı½ ðŁĴĥðŁı½ +ðŁij ¢ +ðŁıĢ ðŁĴª +îĦĨ îĦĨ +ë¶ Ī +ê° ľ +Í Ī +wind ber +what abou +wen jun +we o +ver bas +valle lunga +ush kowitz +urin ary +uni directional +twin brook +twic elight +tom udall +to doro +threestoo ges +ther im +the je +the cam +the broad +the bol +th nk +th band +teng en +tam bora +tai yo +t ft +summer festival +su santo +stark weather +sor bara +skin ks +sil denafil +shuben acadie +se mm +se j +san ilac +sam ant +salesforce ohana +sain tramrahim +said hanshika +sadhguru quotes +s mic +s key +roev wade +rif fe +re constructs +pura skar +profun do +pres nell +pra vasi +pol kas +po gs +pink hair +pepit one +pa de +p ú +orca dian +oni rose +oman is +o hed +nu age +not vivoree +no ty +nico tin +newsc lt +nct zen +nascar salutes +mrs gif +mrsgif letcher +movie actor +mour vedre +mo gha +micron ized +mi asto +me myself +max illa +matsu shima +mato sin +mandy rose +mak ens +mag ala +madele ines +ma vens +ma snow +loch end +living my +lease back +land sman +kyr sten +krish nagiri +kei ko +kap ur +kangaroo island +kade tt +ka stur +k dei +just my +jonas son +jimmer fredette +jerry can +intro biz +inf n +i era +i dium +hy del +hul stone +history matters +han dovers +hampton roads +greif swald +gold ner +gim bel +gau ci +ga res +form labs +forde ast +fil ma +fedor as +fau stine +fanta sizes +fa oi +f dn +f bn +etou ffee +entre at +en ature +elis sa +el ddis +ecol ts +demonstra ble +de regulate +de my +de and +daz dillinger +dallas comiccon +dach stein +d nam +custom shop +cuis ine +cox sac +corof in +containeri zation +com modus +ci gi +celebs godating +carrieann inaba +cap stan +campo bello +cal ama +caf fenero +bus sum +brown ells +brooklands museu +bha sma +benji bananas +bedro ck +be jewelled +be awesome +avi spa +av go +atla irport +armen trout +anikan onirose +andr é +and ur +and ale +amc kee +ab radley +a jac +ðŁĺŃ ðŁĺĤðŁĺŃðŁĺĤ +ðŁIJŁ ðŁIJŁ +ðŁĮ¸ ðŁįĥ +âĿ¤ ðŁĮ¹ +⾨ ðŁĴĹ +âļ¾ï¸ı : +иРµ +z ue +you ro +wolf song +win ecountry +wi eden +whispering bob +wal las +vinyl meplease +umi zoomi +twit te +tv at +tul fo +tribun a +tom sula +to travel +ti zzle +thisisirish food +thi amine +syd nee +supp leness +su has +sonic maxpro +somnam bu +snow line +sme x +small caps +sky high +silk road +shiv aj +shibu tani +sem la +seaw alls +seatt let +sea otter +schi ffman +s ftp +rosal ba +revent on +rec sys +re facing +r ni +plo eg +pe skov +ou trank +ott en +ong niel +one man +o sten +new mutants +ne onics +monk land +men sclothing +melane sia +medi mmune +mcga han +mary kill +mark uk +mar win +major can +magal haes +madam ex +machine tools +ma bius +lle gamos +land art +lady beard +kur up +kun gla +kend zior +k khh +je ev +it startshere +in music +in ish +igers france +hyp mic +house hotel +home chef +here fords +he hee +hay am +has bara +happ i +gu ffey +gitex techweek +git ane +ger gely +geo storm +gate keeping +gat ting +gal oob +fu ly +from heaven +for deco +feni ans +fantas ylg +fair pay +euro satory +emmas later +down able +dow en +di za +df j +der aa +de mu +dan er +daaa amn +cross on +con gs +civic a +circum navigating +champur rado +cham ling +cham ar +celebr itye +carrerac up +bun nie +bli ssed +bant z +bally mena +baby cakes +are e +antro bus +anal o +amph lett +al bro +ai ki +ah sd +. ðŁ¤£ +( ^^ +! | +ðŁĸĸ ðŁı» +ï· » +ì² ¸ +âĺºï¸ı ðŁĺĬ +à´ £ +Í Ī + ¶ +zind abaad +yur man +ys ay +wool folk +wine shop +wigan warriors +we u +wau ke +vi ole +vam o +un no +tylero akley +tu mi +tree less +tor ra +timo f +ti zi +themy scira +theben forster +the south +the hollow +tel ma +te vita +tar quini +ta kaya +t sou +sub genre +stell aracal +ss occer +sno win +simon says +show you +sep tima +sch moke +save bsoazad +sau de +saddle up +s dogs +run ciman +row en +row botham +rm hs +ri stor +reco do +re portable +re groups +re eagency +ra shaad +quick quotes +pyroman cer +puj ari +pu go +prosely tizing +pirand ello +pick pocketing +pha sic +ph ryne +peugeot sport +petro u +peter thiel +perform in +pe trac +pani agua +pac ke +pa hari +p ge +ou risme +od l +noval iches +newcastle hosps +new country +neil d +navy blues +natural medicine +mor atti +moon bin +mihon ishida +mic hon +mesh el +mck endrick +mar stellaracal +man ak +mach aca +lin thorpe +lei dos +laur diy +lamon gan +l wt +ku sa +kol hs +kis ch +ki ano +keith richards +kan sans +k upa +k club +jon kortajarena +jo ico +j bt +insta quote +ineffec tiveness +ignomin ious +ici ousness +hy yh +hoo yah +hippoly ta +health month +hal las +hagi asophia +h wi +gob blers +gn clive +gnclive well +girl sss +gan z +gad olinium +g ör +fy ffest +friday freebie +free kesha +first look +fin acle +far maid +fall river +fala hee +em mets +e kin +don julio +cran berry +coal mining +cliff avril +clas ica +church land +chugh tai +christ offerson +chinese art +chi veon +car acol +cap tian +campe sina +ca kra +bre z +black lives +bit wise +beh nam +bed ale +barry allen +bar ral +balne ario +bal krishna +badrinath ki +back road +auto dro +attle foracause +as sail +arte mi +apartment sfor +ap ba +anand skfc +aldubb attleforacause +agu ard +ad ino +ach eron +abram ov +ab ente +ðŁĺĤðŁĺĤðŁĺĤ ðŁijĮ +ðŁĵ· © +ðŁĮŀ . +ðŁĮ¼ðŁĮ¼ ðŁĮ¼ +ìĿ´ì¢ħ ìĦĿ +ãĢ½ï¸ı ðŁıĢ +zy gon +zelmer low +zak arian +zabludo wicz +y th +woo snam +won derer +w trf +w sa +vocali ze +v sop +usc s +uni kl +un tried +uclou vain +tu gger +tre gs +transcathe ter +tom rw +tom men +time slots +thursday treat +tho dari +then aked +the record +the hive +teentit an +te brau +tailor made +sur ti +sun art +step children +standupp addle +stan bridge +sr lfc +sportat orium +sense mble +sec ta +seabourn cruise +salomon running +safe space +s foods +ru ine +redwood city +re settling +re fa +ran ong +ralli art +q outes +pocon o +piero gies +pi ppy +perfect fit +pand as +p forz +ox igen +or co +ofic ina +north africa +no dame +nikk ic +nicol led +monch hichi +mon daw +mo vers +minim inter +min aya +milos z +medic aid +matosin hos +mark jin +mariash river +main aand +lyon dell +luc ci +lemb ah +lace work +la king +kschi thra +konop ka +ko tta +ko ek +ki bra +kay le +kann adiga +int nl +infr inges +in on +im ready +heavy duty +head lee +hcp sarea +gur s +gor dano +go squirrels +go getit +gilligan sisland +gil breath +fri ant +fr ath +fa thead +es rd +el j +ed elson +ec lass +dv antage +down towno +domic iliary +do ber +di enes +devo y +debbie allen +dang ly +curious er +crystal ball +cre de +coor ong +cokestudio africa +click ers +church warden +char twell +chamele on +car ica +cad aques +brown bag +brock worth +bo ere +blackpanther party +bla ker +bin der +big ride +big give +bha vi +becau ser +ballagha derreen +bah ra +bag y +ay aku +atter see +athar vaa +angel sinamerica +anc afe +an sara +amsterdam se +am elle +almod ó +ali ot +ad amp +ac tioned +ac ron +ac el +a ana ++ ). +ðŁ§ ŀ +íĶĮë Ŀ¼ +⼠± +ÅĤ a +yung blud +yo gend +wick y +weir racing +wedem boyz +wah ba +w ame +vishak ha +veen endaal +vee bha +ur schel +theros ary +ther ink +theo walcott +terrac ina +ten yc +tempor ality +tele path +teacup club +te ems +tc bc +tafel musik +sydneyo perahouse +strathal byn +stee les +splen di +span ic +sp olympic +sou treach +so tr +skylar king +shar ica +shand ur +sfu sd +se cho +saving places +sarah shahi +sansevi eria +sab aq +s deb +rosen bloom +ro jas +respe k +redbull za +re tra +re housed +ra ham +r do +pepe jeans +out growth +on fd +on aga +nurder hsv +ni ç +nhs digital +my ron +my ne +my lfc +mw ca +mu rawski +mthe mbu +mon stress +mil ledge +mcca in +maxi priest +matan uska +masay uki +mal hi +ma kabayan +ly rica +lol wut +local art +lef in +lead on +le cu +la it +kyiv post +kup i +ki anna +kcr gwx +joke day +jo ser +jeong in +jam bs +jalpa iguri +j hay +is ud +ingof the +igre ja +ic ure +i ones +hunni ford +hi mura +gui yang +guar do +guan aco +grat on +grameen phone +gossi py +googleexper tuk +gla zers +ge ers +fun s +friende ver +fri so +frees ample +free pick +fleadh cheoil +fit nes +familiesc lose +evi dential +eu h +es ung +episcop alians +em mott +ef conline +ear wigs +dougla ston +directs elling +dem swork +del onte +deadly class +de jeuner +de caro +dc shoes +darke sthour +da aaa +cra u +continuou simprovement +confuci anism +comb ate +co fi +cleo bury +cilli zza +chiz uki +chicken hour +cay abyab +cancer treatment +c src +c ml +by ung +buzz cut +bro war +bro l +bre cher +black by +billy tolley +bee zer +bam l +bam a +bake club +backedby aib +az hs +aro b +ap ass +anthonye tuggle +another magazine +an art +allegre tto +aftershock comix +ach hedin +aber tay +! ðŁĴĺ +ðŁĻĤ ðŁĻĥ +ðŁIJ± ðŁIJ±ðŁIJ± +ðŁİŁï¸ı ðŁİŁï¸ı +âĿ¤ ðŁĩºðŁĩ¸ +е м +z anda +youthem powerment +yl unch +yam assey +women with +winkel mann +wh ay +weis ner +water polo +war master +vis cabarca +vir ta +ven ia +utter back +un fussy +uk orchids +tour neur +time shift +ter kel +tay son +tamar ins +ta ipa +superbowl lii +steph i +spol sky +sor okin +soldie red +sk og +shi ken +se hs +schulich school +say ing +sagarma tha +ry leigh +rocred wings +rock n +remor seless +reed bed +re deployed +pro tips +playstation vr +pel key +parapar au +palit oy +pa heli +oz amiz +ox alate +official willow +official triumph +oc tors +non commercial +ne do +nai z +mrtommy land +model kit +men z +me costa +may ash +mato logists +maroo chy +ma hera +lucky manzano +ltgov delhi +lou rie +lin derman +leuci stic +leez agibbons +leeh sien +le ino +law making +law are +l zr +kri ge +kollywud cinema +kirkle esc +khar is +karai kudi +kapp el +jud moo +jb mauney +jay walker +j ini +itsar ush +inter vista +ine f +i six +how ler +guardian witness +guaj ardo +glow up +gis bergen +gigli otti +gertru dis +gaming pc +fran ti +fluctu ates +fishn tips +ff wd +fab aceae +fa illa +emmaslater dance +el ac +du dleys +du bbs +dremil yamassey +dog boe +de use +de oband +de kton +daniel padilla +dak shin +da hisar +d fc +corbin bleu +city bus +choisi won +ch fi +cel entano +bse india +brockle hurst +bro dus +brit actor +britactor sfan +born free +blogger life +black burne +bird land +bell labs +be fell +bb cr +bal laugh +au nee +astar oth +arag ami +app ens +an american +alzheimer sday +almodó var +al port +air ings +adeni z +acol lusion +ach ary +________ __ +ðŁĺ© ðŁĴĶ +ðŁĶµâļªï¸ı ðŁĶµâļªï¸ı +ðŁĶ¥ " +ðĿij Ĵ +ìµľ ìĬ¹ +è¡ ĵ +å®® èĦĩ +âľĮðŁı» # +اÙĨ ÙĬ +yogur tland +yarmol enko +yankeec andle +y sabel +wri ddhi +wire image +wessel mann +war daz +vis vim +uttamav illain +uchicagop ress +ubc tbirds +ty ms +tu larosa +tre bor +toyo tasa +tou reiffel +tor mey +toff oli +timber lands +tiger football +thisi sse +thero se +thelaw society +the ba +ter zi +tananlam az +sub o +stage it +spokane indians +socal gas +sj games +si vi +si dd +seta pak +savi ation +sav arin +roar ke +ro saleen +rel an +re gnier +raiz awilson +r dh +py ré +plate a +pavan wadeyar +pas sa +par ki +papad akis +panneer selvam +pand anus +orange ade +o stara +o hau +nostal gically +nicolled wallace +nde geocello +nam po +my president +mont ages +mis sa +mel bre +medline plus +mcken nitt +mat en +mariek ondo +mar oni +mar ma +ma kan +livepd fantasylg +ladies fc +l yoko +ku kush +kor angi +kom ple +ko g +kkun drra +kensington wm +ken oother +kapil mishra +k anner +jabarda sth +ic td +horn book +ha pand +grigor yan +git u +gg f +georgin io +freep sports +fred matiangi +fly fish +floren cio +fla thead +fl ers +first group +face spics +ew snow +eri ght +er got +ene sco +elek trik +e ick +dt cc +drum life +dol t +deod har +de tracts +cricket nation +coming back +cochine al +cnd world +ch our +cdw corp +can ora +call o +bu duc +brisbanecity qld +brett dennen +bi eta +bed wyn +bed narek +bar bu +backing green +b icon +ashley banjo +ar tel +an tron +an bieding +albor z +aj mal +ahl berg +abil is +abatto irs +ðŁİ© ðŁİ© +ê´ Ģ +ó ria +z art +york city +yard goats +wl ns +win nowing +win de +wilden stein +wild water +weare marshall +we thu +watch fam +washington ville +wage theft +wac ke +vocali zations +under manned +un zip +trag icomedy +tow boat +to kimeki +thor ton +thelead cnn +tar om +tann eries +sur co +sreed haran +sp inde +sony xperia +social science +smo te +sma shedit +sivas spor +shop era +shaunw keaveny +shar bino +shadow box +se malam +schro er +saturn awards +sam at +sal ameh +sac ré +roast y +ro kin +respe to +re dis +radio graphs +q ni +prescrip tion +peter parker +ox ox +oun slow +oakham ales +nor mies +nitto tire +nigh a +nd grade +nati vism +my cause +mur derby +mon arda +miss jillscott +mindful monday +middle weights +mickey hart +melody jkt +me tsu +mcfar lane +masa ku +marchfor truth +maj e +mainaand kingangi +lwk md +lec l +lans downe +lafarge holcim +ladu ree +la ina +la ffin +kwame alexander +kum manam +kro kus +kem boi +ke vitch +ke iser +kathy raven +karun akaran +jeky lland +je ga +jar lena +irri gators +in quests +in ni +ic ot +homeaway fromhome +ho way +hilari on +heu mann +he ur +harnessracing fz +happybirthday prabhas +ham bo +grybau skaite +gran ter +grammy museum +goe ttingen +girl ss +gigan tea +geor dies +fv cking +fromm ay +fran kies +fou cher +fit ba +evic ts +evangeli ze +er ol +enter ovirus +eleph anti +e eva +driverles scars +dream work +doit right +dis arms +de funded +de criminalise +ddfirish open +dat en +dar ach +daniel sen +dani alve +dance plus +d brand +cy d +cory barlog +conglomer ation +colle c +coach works +clarine t +chitra koot +chir ur +chandram ouli +c vi +burton wood +brek ke +blu et +bid ness +barry manilow +avery music +audi gier +attack uk +ar rabbi +ar ao +ar amid +anc tified +an society +amaz one +am ooo +allenand unwin +air bn +aggie pride +acc football +ac ini +abkibaar modisarkaar +^ = +ðŁĺį ðŁĴĸ +ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ðŁĶ¥ +ðŁĴª ðŁĺį +ðŁij¸ ðŁij¸ +ì° ® +ë ®¤ +ãĤ³ ãĥ¼ãĥ +âĺ ¯ï¸ı +zi ak +z wicker +working families +win dex +westernsydney u +var in +u ep +turkey hunting +tre covery +tour mnl +the mill +temer aire +sur jit +sub mit +stan konia +spinalcordin jury +south en +sor dell +son david +simon books +si ron +si bert +scot x +scoo kies +scare fest +santafen m +sanc ti +s sea +russiainvade dukraine +rs no +rose will +richard hawley +ram my +prosecu tes +procli vity +presidency za +por ur +pod saveamerica +pag cor +pa yot +ott arts +og na +o sports +nieuwe marlean +newsal bany +ne villeg +ne mea +muradali shah +mr cs +mother house +mont and +mill sap +men indee +mal asak +mako ssa +make animpact +lute fisk +lot w +li sk +li i +legionof superheroes +late comer +lat tice +lake mba +l bg +kri sa +kid life +khid mat +kelli wardaz +kari byron +kappaalpha order +k layton +jubi lees +josh malina +it ama +invest ni +internet billofrights +infec tiously +illi quid +ic mr +hy son +hot an +hen rico +heming ford +hellolove goodbye +hear taches +head erfor +gulf shores +greg j +gre eley +gor goroth +goe tt +gh in +gc sb +gaunt lett +fu xk +for ag +figure heads +feet sfriday +fantasy league +f ki +ext ols +elimin ators +eli ber +ele af +echev arria +ead weard +e anna +dz rh +dun levy +duken ation +dream fall +dor ham +dk weirracing +dj sli +dan quah +d crising +cyto plasm +cri stela +crank case +count downs +corticoster oids +con con +co readvocates +cla va +chry stia +chiw enga +charle smil +ch hath +cefal u +capp ucino +cantab rian +c wr +c atters +by uw +br or +boye tt +bir git +biopsycho social +best picture +bell icose +bbc newsbeat +bath ampton +bat alha +bang arang +bandit ry +au tour +assemb lea +artificial grass +archri val +ap fel +andreja pejic +an sal +amu jer +amph icar +american hero +am sel +am angala +adapto gen +. ðŁĺĩ +ðŁİī ðŁİ¶ +ðĿĹ ¦ +ðĿij ĸ +íģ¬ ëĤĺ +íģ¬ëĤĺ íģ +ëĵ ¤ +« « +yy ys +ya hy +x roads +whyi write +waite mata +vidyar thee +varieg ata +val gus +vaj rayana +utilit arianism +usic ology +undp nepal +ul rich +uba id +tur genev +tracy morgan +tr pg +the weather +the french +territ ory +terr atech +temp t +tele suren +telesuren glish +te odo +sv hs +style inspiration +student sfirst +stu ttered +stoo ped +ss art +spi o +sigsauer inc +sharon stone +ser ban +se akay +science center +saras wat +sandi fer +sam billings +sal mi +sak sham +rub én +room nyc +ric keys +ri gas +rei ley +radi ore +py are +punk newwave +promul gated +prob ity +prat ley +pin en +ph anie +pan in +official somo +oel wein +nws boston +no thern +netflix andchill +nbam emes +nay er +mylfc matchdayimage +my daily +my bad +multic ellular +moul ton +mer redin +men hir +meach em +mcclu sky +mal ong +luv v +looking up +logarith m +life sinked +li scard +leehsien loong +lauter bach +la pua +ko cian +kil ob +ki pedia +kar aca +k hub +jo zo +jami ed +j bf +iti me +immun ohisto +hollow ay +helic arrier +han kyu +gas ca +gallery nucleus +fore finger +fo dera +fast ener +f ttc +exer tions +ev ren +elast omers +eis ler +egh radio +ed mc +eclec ticism +dramatic tvactress +dogge rel +dile k +dic tum +dem townhall +de eming +dani o +daily artapp +d pu +cre ese +coton de +coo len +come tothe +columb arium +color ants +cio glu +chev rons +cher ini +campa ña +call ousness +bur kart +bran dishes +brain port +bps official +book design +bess borough +bed knobs +bea del +be toftheday +bas sm +b ics +aw ow +at tr +at tala +asi us +as gupta +around theworld +ando ther +amal thea +alter cations +ale u +al j +ail or +ag rill +acon lin +achi eng +abc perth +ab k +..... !! +... ðŁĻĦ +ðŁĻĮ ⾨ +ðŁĺĤðŁĺĤ ðŁĺĺ +ðĿĹ § +è¡ Į +âłĢâłĢâłĢâłĢâłĢâłĢâłĢâłĢ âłĢâłĢâłĢâłĢ +Ãł n +zi val +yuz uki +yo sa +x tz +warmest dayoftheyear +wall man +wab co +vesper tine +ver hagen +vaidy anathan +uts engage +uni one +uk ko +ud yo +ubis of +u ble +tow and +too tie +too kes +ton us +theother palace +theor yo +tham marat +team parieur +team marco +tart us +tarant ella +tar sem +supportsmaller streams +subtrac tive +string ers +stay ers +st patrick +spil sby +spati o +sor ay +slat kin +si pos +share alike +sel zer +schill aci +schan zer +ru lz +rott nest +ren ter +re start +rashe ed +quasi moto +pol ack +plac id +party poker +partic u +par ri +pall ant +paga dian +pa zz +open mind +onu cle +om ix +odu ba +oc transpo +nu zz +nevilleg aunt +nelli gan +nathan caliendo +mur ga +mor iz +monta ña +moj govuk +mc gorry +masseffect andromeda +man tia +maj ima +lu tea +lode stone +lef kowitz +laur amer +la stra +la quon +ku rashiki +kingston uni +key logger +kar upp +kali dou +just married +ju yal +john daly +ine ver +inconveni ences +holtren frew +ho efer +hasan uddin +gr rrrrr +gen eric +gab ap +fredrik stad +fra ile +fl anagan +first book +f mcc +eri ko +ell ende +ee sha +du mo +down cast +do bry +divyankatri pathi +dip onegoro +desi perkins +david le +cryp tic +cort land +cootam undra +colli o +cla vel +cin tra +ci rio +ce ann +cau dal +cary grant +can struction +by hilton +budd leja +bo gho +bl art +bis mil +birdr inging +bilingu als +biggies malls +be kar +be careful +bc boston +bar sky +bag naia +av eli +art books +around thenfl +ant farm +amand am +al over +agra deci +ach he +ab ella +a beauty +a ade +... âĻ¡ +! ðŁ¤£ +ðŁĶĽ ðŁĶĿ +ðŁİ¥ # +ìĻ Ģ +âĿ¤ , +âĺķï¸ı ðŁIJ¸ +Ùİ Ùij +zar os +wj hg +wind turbine +wide format +whit nall +whist led +wans beck +uniof greenwich +under my +u afootball +twitter arty +tun ick +tric ycles +tri ss +to fur +thankyou lord +terra zza +ter mas +tellu rium +tal abani +ta uri +ta official +supportw yo +squ anto +sp edchat +sor na +shin da +shi row +sh ym +scraw ler +scam bridge +salmag undi +ru derman +rit as +ricar dol +redbull ring +real racing +par x +pack able +onthe table +officiali rishfa +ny ana +nump ty +n ellie +mrschu reads +mi ak +makge olli +mahel ajay +mac ondo +lumi ere +live itup +legiti mizing +lamor na +lam ington +ksh mrmusic +kit kat +kin i +kim itsme +kelsey grammer +kav adhu +ji ren +ji rayu +ji ley +jer rold +isra r +inter line +insur rec +inocul ate +ino v +inf urt +in ther +in skip +ill ings +hul hu +hs live +hossein panahi +ho sford +hfx gov +here ward +hello kitty +han afu +hal flings +had do +gy ratory +goog learts +god ber +gen nie +gail kimitsme +futureis clean +footre sts +flip class +firstdogon moon +fiji water +fant v +et one +esof twitter +en ze +el ittle +ed ris +econom e +ec rs +dr pol +dog man +dirty south +dikt at +dichotom ies +deb harkness +danse use +daga anbieding +d wor +cut down +cumbri auni +crossy road +cros sen +cot to +compare the +com ley +col a +ci le +cc mfc +casc adel +cas ap +cab ella +bu chs +brugman sia +braz ell +bir dies +biblio therapy +behnaz akhgar +b spencer +az al +autum ns +arqi va +ar z +ar ques +andri ukaitis +an ini +an al +am rap +ain da +ahwah nee +adi alogue +abo xer +ab dal +... ðŁĺŃ +) . +ðŁĺĺ âĺºï¸ı +ðŁĴį âĿ¤ï¸ı +ðŁİīðŁİģ ðŁİĪ +ðŁ§IJ ðŁ§IJ +ðŁ¥ĩðŁ¥Ī ðŁ¥ī +ê´ľ ì°® +ãĥIJ ãĥ³ãĥī +âĢĵ ... +म र +Ú© Ø´ +yam ah +versi ones +usa rec +under ling +um g +turi sm +tune n +tom greenlive +tetra pak +tessell ated +tan auan +tak ami +tablo id +sub domain +student nurse +stu hr +stu bbins +strath more +ssoci al +sociol inguistics +sk la +shrews morris +shou ty +sel vin +sch unk +sa ww +s ago +rose tta +rene ef +religionof peace +refu els +reduc tase +redon da +real tristan +rad or +r ning +projec tion +profun dis +pop surrealism +plym ou +pin on +pil ley +pe mc +open weight +once more +om n +om loop +official itm +ny kv +nucle o +nove cento +nim mayash +nie miec +ni had +ni ge +ni eve +nes sus +nd sufootball +natur inaughton +nash y +nar m +mr hs +motley fool +moren te +mongre ls +mol k +mcelli gott +mark mcmorris +mani sharma +mahesh war +mahar aj +lis se +li pan +lav ant +lar ı +kar avan +kal inda +ka aris +k dramas +jul quen +ju mah +john nosta +jethro tull +jar o +it begins +inve ctive +inthe middle +instruc tables +ing bottom +in sincerity +im it +hurl but +hock omo +health grades +he mat +happy jinday +great read +gh f +ge stede +gaur ilan +g biffle +fx ck +frank ly +for charity +falci parum +explore tocreate +exfoli ates +estad ouni +en id +em cer +dylan wang +dull stroom +dete sts +daysof happiness +coo oool +cle te +cl bv +chitt y +chap leau +catch me +bush c +bronchi olitis +broad street +bo kor +big il +beltr ame +bbc panorama +bb bz +bauhin ia +bal ey +b jr +awe sum +aqu ilo +antimal arial +anti k +angrybirds movie +amon dru +al mac +ahor ra +ab os +ðŁĴķ ðŁĻĪ +ðŁĴ¯ ! +ðŁijı ðŁĻı +ðŁijĢ " +ðŁıĪ ðŁĶ¥ +ðĿĻŀ ðĿĻ +ðĿijĸ ðĿij +âĸ Ķ +Ùĥ ر +и ÑĤе +zor ds +zeit lin +ystr day +yn p +xiumin day +women folk +wind pipe +wel ding +we pa +wa ac +vladimir putin +vital ogy +uni z +unex pressed +un dressing +u tube +u alber +tor tora +tony denison +thor ny +thereal autoblog +thejeep boss +the flying +story corps +stie ber +ste mp +so al +sin fin +shiamak official +shenmue hd +sf aulkner +semantic web +sarac en +sar tain +sammy watkins +sak ya +sac town +s dept +ritu ally +ri shab +ri oux +ree de +realestate investor +rat ers +quad er +q cd +pre dated +portu k +plan chette +pla iner +pink tober +pilo thouse +par anj +packer scamp +outre ach +on elu +obli gate +npl nsw +nott jmiller +northco teuk +ni ga +ne leg +my sad +must die +mul tani +muen ch +msd honi +miner alized +mi ked +melbourne rebels +mand saur +macro monday +macleod lisa +ma bon +lunch special +love fool +lo sch +list in +lew ys +laurin burg +lamin ck +laid ler +kn auer +kingsc lere +kelly hoppen +ke mber +k heim +je anie +jan edu +jahn joyce +ja ey +j nl +j fb +it ra +irish athletics +invest ingin +ice pick +iam nathancarter +ia edchat +hutter ite +hong qiao +homi letics +hand ball +ham burglar +ha eger +group suk +gos well +gop shutdown +glycol ysis +glo ben +gi aco +gerring ong +ge bra +gar do +fruit and +fein berg +fat ma +f ager +erit age +er la +end ment +ei jun +dro ege +down hearted +domode dovo +di mock +di gression +dham mapada +dell in +daniele wski +cre aking +cour tiers +cortin as +cook with +contextu alizing +ci pe +child actor +chi usa +cent conf +ce ducation +carol i +candy floss +can adam +cab ri +blue stockings +big hair +ber lyn +battle ship +bass fishntips +aure ole +as quare +artscentre melb +arti ste +ard glass +ap ari +an holt +alph on +alham dol +al ano +aju da +abq journal +abil aa +aar ya +ðŁļĢðŁļĢ ðŁļĢðŁļĢ +ðŁĺĴ ðŁĺĴðŁĺĴðŁĺĴ +ðŁĺįðŁĺį ðŁĺŃ +ðŁĴĭ ðŁĴĸ +ðŁİĤ âĿ¤ï¸ı +ìĪĺ ì§Ģ +ç» Ł +åı £ +à¸Ĺ ำ +اÙģ Ø© +ÑĢом айдан +youknowyou lovethem +women wh +w tr +uninterrup tible +un treatable +uk g +uc susa +tyne dale +tri ston +tim mies +thener d +the breakfastclub +tel er +tail pipes +suren dran +sparkle horse +spac enews +soton bloggers +sne ers +sm lb +shopif yl +sch one +sar us +sale able +sa kay +rugby team +reviv alist +readabook sa +re sund +queen y +propul sive +prom out +pol sk +po stol +petron io +pecz wolle +pate y +palm spring +our councilday +ound le +oti um +or pik +or ne +opera holland +onlin eradio +ok ane +oj simpson +obe tten +o war +nw ssan +nor afatehi +nfl trainingcamp +ne agoe +nbaf reeagency +n vr +mosque shooting +monster girl +miumiu official +may ben +mares me +maic har +mag li +m din +lyondell basell +lo docom +le em +le corbusier +lande cho +land lines +ladies coffeehour +kn filters +kim es +kihu en +ker shaw +ker no +ju bbly +jeremy shada +jeep neys +jare cki +ja jang +isag enix +intere sse +indy fuel +hi ggi +hec kel +har io +h á +grav ina +go kart +gis ella +gir llll +ge res +gam bi +gab r +fu jimura +frog men +forthe union +ff acts +fe iler +fatta hamin +famili ars +evelyne brochu +euro dollar +eu scienceinnov +eri zed +eri ously +eosinop hilic +edward sharpe +e ppie +e jig +e gil +dy fed +dued iligence +don nat +do ges +dent i +den ili +de pil +day in +data point +dan acar +conspiracy theories +clo ying +cl andon +choc taw +charger pride +ce se +carab iners +c scc +ble e +bi planes +be zal +bat as +bar ic +bali kavadhu +awu mia +apriv ate +ad fa +acrif ice +ðŁĻĪ ðŁĻī +ðŁĩ· ðŁĩºðŁĩ +ðŁĩ²ðŁĩ ² +ìĹIJìĿ´ íĭ° +éĿ ¢ +Ùģ ت +Ø® ÙĪ +Ø« ÙĤ +zyl ka +ys w +ye sor +yar ai +ya hia +wheat croft +wap ello +want in +vo p +vir ushka +ven yc +use lessly +un tagged +tw en +tsu ji +tre zor +tn ks +thelast word +thefla bar +team r +strongman burner +stra ks +stoy show +spor tv +som ani +sof er +sneaker holics +shore ham +shar nbrook +sc broncos +says thanks +sarah jan +ru pesh +roc que +ran sparent +quarter maine +proven ce +power wolf +ph onic +peter reckell +perturb ations +perth saint +periscope tv +pere stro +party like +partnership working +par le +p vo +ori fic +on thames +on se +od deven +nt pol +my job +mon sun +moment a +mo hawke +mj h +mississ au +minority report +miner alisation +min cing +mil ius +max in +market smith +mar griet +mai ley +long town +lisan andy +lion t +lam born +lack o +kyo ka +kiku sharda +kad okawa +jehovah swit +j ú +j heel +institutional isation +ili on +i yogibabu +hu gest +green bonds +gra ze +gra da +get surrey +gell horn +gat ron +fuel ledby +freddie mac +flye ia +fer oz +f official +exoplane tapp +ex one +erin andrews +entren ching +eltonjohn dotcom +dz ire +drug policy +dre bin +decor s +de classification +dalecar negie +da than +cryo sphere +crooked media +creative coding +concert series +cel t +ce si +bra zza +border line +book ofthemonth +bobby deol +bo vespa +blue marble +bit ola +ber man +bench mark +bel man +bar bap +bad illo +az ore +at ering +and one +an dere +amdav ad +amb h +amazing world +ale ment +al verson +al caz +ac tr +ab caustralia +aash to +ðŁļ ¤ +ðŁİħðŁı¼ ðŁİĦ +ðŁİĤ ðŁį° +ðŁĩ²ðŁĩ¯ ðŁĩ²ðŁĩ¯ +ðŁĩ µ +ãĤ¹ãĤ¯ ãĥķãĤ§ãĤ¹ +âĢ¦ /âĢ¦/ +zz ap +young sheldon +ym piad +wyn and +women at +willi g +we cam +wan less +wald ner +vil ar +vi stap +vb hardwaj +vag h +us now +uri arte +ur baine +tru ssed +tru del +to god +titansof cosplay +timb res +thisi smo +think different +the empty +thames and +tec tonic +tat yan +tal aat +studi ob +star mall +spanish wine +space plane +sonyo pentennis +sonic youth +som osc +solfe ggio +smar tie +siame se +shore side +sho tof +she han +shark friday +sh man +serv ator +sen dit +saw bone +save forever +sage steele +s burning +rohit vbhardwaj +rock centernyc +river head +ricer ca +restin power +raise theroof +present ation +prepar ando +pose fx +plain smen +pic turi +photome tric +pen alizes +paint ourcountryred +out land +ou lihan +ont sciencectr +off man +ny saf +nun obetten +nix es +nik khil +nav orro +na ini +mw ff +msu bear +mont au +mittel stand +mi ahamm +medi apro +marcus rashford +male fic +ly sette +lunatic fringe +lover anch +lik elier +landol akes +ku bas +ko djia +kel an +jo bling +ji ayi +j simpson +iñ aki +im fact +ical cio +holy prophet +hk n +harms worth +happpp py +h gst +govisit donegal +gear hart +ge mc +fur r +fromthe heart +freedom for +free bet +first data +episode ix +emoun tain +drimn agh +dni propetrovsk +di ffs +dev yani +desol ated +cyto toxic +cro pland +cou pa +co yy +christi ano +char ring +cfas ummit +cel lier +catt olica +cas ely +car ron +ca they +c suf +c family +business world +bu ong +boo ooo +bluebull srugby +best cover +ber tini +b po +b hide +azam garh +arul nith +anne hill +anight club +amo u +ak sha +air lifting +ab baf +ðŁĺĺðŁĺĺðŁĺĺðŁĺĺ ðŁĺĺðŁĺĺðŁĺĺðŁĺĺ +âĻ Ŀ +| âĢ¢ +| " +youn gent +ye lem +x mpp +wuor nos +wrong doers +worldwar z +worl don +visi ón +ver su +up one +u cks +tweeds muir +twd season +tu fn +travis mills +tran sasia +tour an +tot teridge +tin man +ti ë +thelad bible +the code +thd specialt +thdspecialt yops +te poz +t way +t cb +sydney harbour +sura gi +stro zier +stay stron +star bird +squi shing +south yarra +small streamer +skan ks +sk imo +shey i +shaw kat +sha di +sece ded +se de +scul thorpe +scal endar +say antika +saras vati +sar afina +rtel atelateshow +roberts bridge +ri ser +retro game +red dragon +receipt bank +re sour +re nier +ra fan +pli ant +pinstripe bowl +picof day +pear ly +paladins art +paci fied +our planet +oikou mene +norman scat +nfl gameday +newzealand terroristattack +nat bookfest +n ées +n dogo +mur ra +mog wai +mo kel +mo bot +mitch ells +min ner +mer rick +men il +mee go +mat v +mat eu +malate sta +lund by +lon glo +less lie +leip sic +ku las +kit by +ke ala +kan kar +jeffrey deanmorgan +jan an +j iri +inter aksyon +in articulate +hibern ates +hfd cathedral +hemost asis +heidi montag +harps well +gri mble +glu ckman +gif tv +gerard cosmetics +fordair show +ford fiesta +flying lizard +fa zel +endic ott +em boss +elen aof +el row +el al +div ada +disp lease +dis neys +digital inclusion +dif fi +daniel pink +dam aja +dab u +curi g +cur vil +compli mentday +chicou timi +cep heus +cc am +casey stoney +calpur nia +by polls +bryl creem +bre mont +box elder +boom afoo +book tweeter +bolly spy +big land +bho pal +bend y +bemore bulldog +auto didact +at will +ann ayya +al thy +al ila +af zal +achil lea +aap ki +. âĺĢï¸ı +-------------------------------- ---------------- +ðŁĴĹ ðŁIJ¾ +ìĨ IJ +人渣åıįæ´¾èĩªæķijç³» 绣 +Æ Ĵ +¨¨¨¨¨¨¨¨ ¨¨¨¨ +zwar te +zu biri +zo gby +zah ra +your style +yes yet +yash ar +wei den +veloci pede +van doorn +use dom +up setters +unmiss media +un amused +u ise +ty to +tru triciahelfer +trans gress +ton bori +thum amina +the sergiogarcia +th planet +targe ted +sy kora +sw j +suppre ssants +stree ting +st patricks +sports network +sle at +shiv amo +serj tankian +seago ville +s oper +roes ler +riv kin +rin king +rel ite +red l +re go +rc pe +ray rice +que ss +puntag orda +poetry club +pl w +pet teri +parac lete +p texp +oviya army +otta way +ole k +nrl south +ng w +n ber +morro co +mic ca +meinl cymbals +mar kova +manji mup +manav gat +malai kaarora +made lein +mad ingley +mad ill +mad diec +macau gp +m sian +logro ño +little wood +leon arda +kol la +ko stic +keep grinding +jung koo +julien solomita +juilliard school +jaime murray +itsoknotto beok +ir furugby +iphonex r +interrup ter +iam kevingates +hypoten use +holm quist +histri onic +h gw +guildhall school +guant á +ground out +good trouble +goian ia +go pis +gen ix +form alin +film friday +fe tus +evry thing +eudat ap +estor il +eri sta +ep ines +emil ys +elisa bet +eli el +edward norton +ecor re +echever ria +ear ther +e kywx +dramati sation +do tan +dism ally +dia gh +di photo +den el +de ko +dann yo +dal bir +cudd yer +con fort +community first +clanc ys +charlesmil ander +cau tioning +carre ra +cad le +by noe +bro eck +brian may +blue family +bit me +bil ayer +bie bers +bi ene +bhu pen +beit bridge +bat ala +bas smaster +bapti sing +bad ak +b ico +ar trave +anu sh +ano tti +ang ley +analy tically +amor gos +amanda seyfried +ama hal +akamegak ill +air craf +adi son +[ ..] +.. ðŁĺĤðŁĺĤðŁĺĤ +## ## +ðŁĴĹ ðŁĴĭ +âĺº @ +âĦ ĵ +zen er +yeezy season +workat bmo +wil cox +weare lions +water foot +wat more +vintage finery +vanqui shing +ucbt la +tw b +tra ks +tiru vann +theatric als +the peoples +the fresh +the culture +terry ville +ter ate +syncop ation +subo dh +su steren +styx theband +spir anac +sl pp +ski e +shur tle +shu bin +scor chers +scorchers bbl +scam bill +rø de +ry uu +run day +royal nottingham +roseng arten +roo ters +ro ved +restorative justice +rec d +ram k +produc ts +pral ines +po hn +phon te +perry farrell +opp en +om entum +olivi as +ol inger +oil prices +nucle ation +noo ksack +nikkhil advani +nem rut +muzz in +muzi ek +mul ligan +mont seny +monster shockey +money lifers +mo sk +mitt a +mi thi +mchen ry +may il +mate ch +mar jane +mak ris +mac aluso +ma bley +m mol +lions roar +limb u +legend sneverdie +ku si +kqed news +king crimson +kar ok +kane shiro +k jal +ju gni +jm sdf +inform ación +in adequately +i bid +hend ryx +heat world +hard head +gu lags +grand vintagefinery +ghar afa +gar zon +gallinu le +futu h +fried rice +frey tag +forever faster +fluid ly +fli kis +flat sharkfriday +fer it +f tir +ev ng +er kan +e od +dé but +dz mm +dyou th +dogre scu +dixie chicks +disc ordia +di ge +defen siveness +dead of +dac arter +com eta +cm madhyapradesh +chi architecture +chennairain shelp +change agent +ce sta +categor ising +camp illo +c cam +bw ca +brendon hartley +breeze wood +bon soir +blue diamond +black pride +black moor +bla sé +bicycle day +bibek debroy +bergü zarkorel +ber nd +bel apur +beav an +bali ke +az ri +aud ley +att ini +atmosph eric +ash bar +as ale +arulnith itamil +arte verywhere +arom ero +appell ations +ante y +alexiso hanian +alder maston +ah g +acon ite +aard wolf +åĽŀ æķ° +åĨįçĶŁ åĽŀæķ° +öster sund +é ire +à ¿ +zoek ravitz +z inner +yak ushima +wood art +william morris +whites ell +west ling +we sti +wall onie +ver ka +van badham +uni fortheunion +u mac +to cs +tiv ation +the tapi +the hsf +thal ic +tele babad +tc boe +tar kwa +tan trik +tac chini +sustainable seafood +su chy +stin ky +spini fex +somer sethour +skil ton +sepul ch +sci enza +sang ster +road and +res su +relation ships +re so +rand on +raim und +radi kal +qu ater +q lowes +purpose less +pow pow +pix s +petri fied +paul fox +pan american +pa korn +p canada +ow ls +ou ton +orig nal +on tour +omo bile +odi ya +o tunga +nur ture +nrl bulldog +ness week +mon em +mo akley +mma worldseries +mazz arri +mayor gregor +making it +lo aiza +lie bing +ler ay +leng let +la vac +kon tor +ko thi +kill star +khan e +jo konta +jer wood +inner strength +in cirlik +in ap +im un +illion aires +iihf hockey +hu mmm +heather smusical +he arers +hard talk +happy min +gu sev +gre athe +gayath rie +fli es +feno kee +engineer sday +enamor ada +du lli +dro yal +dewy ze +depart amento +denili quin +danielle peazer +danialve sd +danacar vey +cymb al +concur ring +co ins +clari fier +chol ar +chi el +chasti sement +bug zy +boon dox +body positivity +bloodh ound +bla go +bi agini +bath bomb +baby parts +b pt +awe bster +ash all +arr ancar +aq a +apal encia +allegh en +agains thate +adventure dub +adsby google +! ðŁĮ¸ +ðŁĺ³ ðŁĺĤðŁĺĤ +ðŁĺ³ @ +ðŁĴĥðŁĴĥ ðŁĴĥðŁĴĥðŁĴĥ +æģ © +ä» Ļ +à¸Ħภ¥ +Ï Ł +~ ' +yo gar +xmas gifts +wmc actionnews +wearit pink +uof california +ukho zi +uk coastwalk +u hud +tu shy +triste za +trek segafredo +trance music +ten sioned +team wendy +take warning +sto dge +star dock +st joe +st fest +srish ty +sri hari +sony pic +sol arc +siyab onga +sirr fs +signor ia +she el +shar q +sel z +see tha +school shooting +sali k +sal aah +sad atx +rockos modernlife +rit suko +ri mas +reinde er +re applied +rat te +pri ses +pp sells +pir on +pavlyu chenkova +papri kash +palazz olo +pac kexpo +p alike +ousp lants +opent ype +o vp +novis ad +ne wex +nat ore +my jam +moun tallison +mou stapha +moo somin +mo state +mo chizuki +mi sere +mar able +madeto order +lut senko +longstre th +lon hro +league acs +le so +lac tose +l voe +l nm +ku ze +kriti k +kir kelementary +kidsin museums +kc n +jermaine dupri +jazz giants +inter weaving +instap undit +inqu ilab +illinim bb +i pupa +hyper sensitive +hoch berg +hig nett +hi shoot +hernan es +handmade withlove +hand shaking +halloff amer +gun sand +gudi padwa +gre v +grange gorman +goldber gabc +get xo +fl anne +fi dalgo +feder alists +es rc +elf cosmetics +ef ell +eccle sfield +e off +dove tailed +diplom atic +di spiriting +dead locks +david haydnjones +darren aronofsky +dak ini +critt all +cor sair +compli ance +committee woman +chili bowl +cher rapun +chab rol +cash money +cambridge up +calde ira +bu zek +bol tzmann +blood worm +bird wing +bil lah +bhag want +ber ko +ato ols +at d +asi mha +asa pro +ap ta +an erley +aha ve +access oires +ab elson +ðŁĻı ðŁĺĺ +çĪ ± +âĢ ¹ +ب Ø© +Ñĥ ÑĢ +zo tero +zach aria +z ura +young dems +ye hia +ya x +wyo sports +world day +wir th +wil les +whatsin your +vet college +us ke +upaz ila +unk rich +uni fies +uk nighted +u pike +troop ingthe +travel holic +toom any +thisgirl canuk +theak ston +ter yl +ten o +techo bloc +tali on +t de +supri yo +sunny and +sul rich +su dd +st croix +spark s +space program +space ghost +somer ford +smash mouth +sm wnyc +sin dic +shiv shankar +shear man +schmoe down +ru ark +ribon ucle +real dlhughley +radial first +rac ters +quan go +prome sas +pow a +politec nico +pol itti +playboy plus +play z +phil health +petr arca +official rfdtv +ob ay +nigel walsh +nicol lette +nic col +ni gar +natural capital +my kindof +monic ac +miner strong +mi zo +mg mre +mer in +mend acious +mcgin lay +may apur +matthewk oma +materials science +mark downs +mac al +m power +long branch +lochgil phead +llamas oft +live performance +ley ne +lac us +l sl +kumbhal garh +ku dai +ke char +kari mov +kap ila +ka inos +jone goro +john wesley +joequ esada +jam y +iwant my +it l +impre cise +ii fym +ig noble +id aniel +hue ys +housen y +hau er +hart sock +guan eng +gu entzel +gr rm +gone withthe +gn x +give back +gaw thorpe +g dc +frees kiing +fli eger +fer riter +exc ite +ev anovich +eter n +energye ast +dz ic +dou gal +dhan ak +day party +daren th +d tek +cut worm +cunnin gham +cu fi +cri men +crave online +cool ly +collin sjr +clo ture +clavic les +chelt scifest +carnaby london +calcare ous +buffe l +bs gyn +brig and +bo vey +bo il +bi eler +beverly hill +baz alge +bazalge tte +barn house +back plane +baby shambles +ba stani +b nar +atthe beach +ation ists +at aman +articul ations +argo sy +ap oria +animation history +an ted +adventuresof sabrina +abdul kadir +ðŁĺĸ ðŁĺĸðŁĺĸ +ðŁĺĴ " +ðŁĹ ij +ðŁijĬ @ +âĢ³ ) +âĢ¢ | +è ce +zo ila +zak i +zait sev +za har +wonth aggi +wcs district +wai roa +wag staffe +virgin radio +vietnam war +uwh arrie +uss enate +under state +ugly sweater +tw bb +tux ford +tsu gumi +ts ve +trophy less +tri ppie +tor ok +tooth fairy +tom baugh +toib engaluru +to wards +time is +ti mex +thun berg +theland news +tell ing +tee garden +tc palm +tb al +tantrum jas +tagteam titles +strong and +stiff key +sport sand +sou k +si kar +sher inian +shaz aming +sharon tate +sh lomi +sc inemas +sap stat +sag acious +sa ison +rut les +roun drock +rother field +rosel ia +robert liefeld +regi one +refor mists +ram adoss +punks notdead +priyadarsh ini +power bar +pon tian +phyno fino +percy jackson +pa ek +p ds +op hrys +oko h +nor ds +mugg ings +mu barak +mr ts +mou sa +mo pan +mitch inson +mis ao +mi thai +mall er +malcolmb arrett +mainde foe +madison wi +lp family +long drive +lettu ces +letsgo fish +letsgo bluejays +lahore blast +la yl +la wr +la had +knee bone +ken yc +ke yeast +kc z +just jane +jack averymusic +j music +iso pods +in chi +il f +ho sey +guanab ara +gir nar +fleec ed +fla key +fieldwork friday +ex bury +eu onymus +en snare +el acrosse +ek po +ebay deals +dge y +dee sh +daun te +dam usic +dac ascos +cou lomb +cotsw ol +coffe ec +cityof york +christodou lou +chapp atte +cav ill +capo bianco +bul kier +bru jo +brit pod +bristol museum +brady bunch +botanical garden +bom e +bol len +beau ce +be sit +be achi +ban ach +bal tas +bal jit +azhar ali +ated atbirth +ar mon +ang ellic +all ready +aerop onic +advis ory +acul ture +acri mon +aa aj +ĥâĸĥâĸĥâĸĥâĸ ĥâĸĥâĸĥâĸĥâĸ +ðŁĴĻ ðŁĺĬ +ðŁı ¢ +âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı +Î ¹ +än der +zet seguin +za im +ym g +yam en +xfiles revival +womenin football +whe ely +visit nsw +vill amil +var ur +uj fm +tur nings +tri lemma +tomas ino +ting in +ti em +thrombo embolism +thenext level +thanks you +tehseen p +taylor ville +tank girl +super mamc +stipul ates +st jinx +som d +snoo py +sloven ly +sini akova +shi garaki +shi bas +se aga +sawbone shex +salsha abilaa +sa has +ru dess +relief ph +quarry men +quadru plet +prin tr +pos thu +port ent +poli she +po ffo +pi b +ph arre +person ae +pel inka +pe psic +os mar +op tom +nr v +nerdy bookclub +nelli safb +ne zetseguin +nayanth ar +nav sari +mus thear +mon toro +mon er +men shair +men ashe +meflo quine +mazar ine +man esar +mack aye +maccab i +mac eration +lance hoyt +kol ten +ko hut +kne ipp +ki dul +keep cup +kbr shanthnu +kart arpur +kapil dev +jj project +ji ho +jared keeso +jam session +jail ene +jag at +israel houghton +isix hosa +ingex pert +im kbrshanthnu +herne hill +heck ingbottom +head water +grue somely +gregarious gus +gov tof +gored foxes +giri ja +ghat ak +gh anim +gaz coombes +gastro esophageal +gam le +gab ino +fry selectronics +for paws +for gan +flat shoes +firepro world +financial advisor +fi it +fe bo +fashion revolution +farn combe +exam inees +es op +dun shau +dontbuy thesun +di val +dday live +dark man +cole k +clayo quot +city comiccon +ci han +chop ta +chi am +catal oni +cap ensis +braue rei +bra ying +blue stein +billye ichner +benazir bhutto +bel ami +band elier +b plant +b mm +atx tvs +arbor vitae +andrew bird +and j +americane wsroom +alitabattle angel +ali h +album review +agu ars +agen der +afl footyshow +aegyp ti +aed rummer +ace to +a ahhhh +a abc +< -> +ðŁĺ» # +ðŁĵ Ł +ðŁĴľ ðŁĺĬ +âľ Ĺ +ص ب +zil los +zay lea +ys an +ye ea +y kids +y ge +wondo lowski +wn n +when the +web deals +wash post +upgrade able +underground hiphop +under stating +ukbiz lunch +ucc as +ty k +tu mut +toni morrison +themercury comau +th ast +tgi bf +techno cratic +susan oo +submer sion +su at +sty nes +stel co +start growthhack +splin ting +speake asies +smtown engsub +sh uru +ser ota +schum peter +sc atters +satra pi +s mino +ric key +rh show +re ino +re conversion +rco bsgyn +qu ynh +py mble +pure blood +pu pil +priyank sharma +ppe dup +plough shares +petal inews +on field +nswe ducation +nh scrisis +newscenter maine +na id +n be +mr tim +mm ts +mic nice +meg awith +med ine +manu script +locomo tor +lin tang +ley la +laureus sport +lane ways +kukush kin +ko lob +kl k +ken de +kaz em +ka ura +joke oftheday +joint address +j das +iy ana +is and +infirm ities +incongru ity +ibmin sight +hulkam ania +hu mawards +how live +hor stman +hin akhan +health coach +have it +hardrock cafe +ha sel +grim stoyshow +good sell +gods love +fro ilan +fm sport +fisher price +final cut +fb heroes +faul tin +fa kers +ex oxo +emph is +emo cion +elenaof avalor +ed codes +east bourne +dun dru +du ur +dog park +digital day +deve aux +del gada +deep space +day byday +daveand busters +da er +cur sos +cruis elife +crop science +cott aging +cor duff +conflu ences +cohe siveness +cel an +cat experience +caprio tti +cag iva +caf ferty +bus bee +buildseries nyc +bu et +bru el +bookof mormon +bo cage +blax land +blavat sky +benav ente +be sic +banc assurance +automotive industry +aston university +approxim ations +amy adams +am mal +alex ius +alber thall +aj arian +ac cc +. ðŁijĢ +. âľĮï¸ı +ðŁĺŃ ðŁĺįâĿ¤ï¸ı +ðŁĺ³ ) +ðŁIJ « +ðŁİīðŁİĤ ðŁİģ +ñ ol +zale ski +z aib +youth hockeyhub +yacht club +woof woof +with cnbctv +wil born +whit efly +west water +west texas +wen gie +weare ky +water colorist +wa ik +w mca +w be +vou ching +visual isations +var ro +v ants +utt lesford +us ing +un feeling +u fp +tory belleci +tiruvann amalai +ti schler +texas music +test drive +te pa +t wh +super malls +sun s +style sheet +spectro meters +sol do +sign ano +sci pol +scher zo +sar b +sanguine tti +san a +samani ego +salaf is +sa wal +ro ep +rit am +rising fc +reaction aries +rang zen +rado van +ra fer +po tty +petro logy +pel ters +par sed +opor tuni +oke fenokee +o ic +nou ak +nor doffro +naz ari +navo daya +nat andalex +nai adi +mun chie +mor uya +monu mental +mir am +mill house +mi mar +met c +mergan sers +mb un +martial law +maro cain +local artist +late junction +l cac +kur kova +krewella jahan +kias oul +kendall ville +jobsat sap +jay atv +jav elina +jar nail +jake snake +jad wal +ir rera +im mobilization +ill ich +heat pumps +har im +grou pltd +go bbi +gin sider +gen l +gautam i +fuji feed +fra is +for mby +food systems +feel on +fast ford +eye inthesky +equestri ans +ef vater +dream it +dh ani +dest inee +derek blasberg +de santo +de mel +dce aglecam +dau di +darth maul +d mi +cw fc +contin i +christen sen +chocl ate +chil duk +chatter is +challenge mtv +chall oner +ca estecker +buz dar +bo als +bloomberg nef +blon del +bil gi +bidden den +bey eler +ber ti +be filter +bc so +bath couk +ban cos +awami league +aw ls +atra vels +annu alised +angelamer kel +and ray +amorpho phallus +amai zing +agh oulihan +ac b +abid sherali +\ : +ðŁĺŃðŁĺŃ ðŁĺį +ðŁĺĮ ðŁĺį +ðĿĺ Ģ +ãģĦ ãģ +âĿ¤ï¸ı ðŁİ¸ +à¹Ĥ ม +غ ز +اÙĦ اÙħ +zu er +z lin +your face +yo hio +yn k +wissen schaft +who afro +white sox +whati wore +west norwood +wak amatsu +wa gh +w abush +vat an +uno suke +ulster wildlife +trul ly +tor il +ti sc +this weekend +thel in +thekenny johnson +theatric ality +tapa chula +tab ulation +ta waf +sunder bans +strong room +stop p +sser vice +snow drifts +sisse ton +shriner shosp +sherri saum +sheil aedrummer +shar m +schul tze +sch ow +saving species +sahl berg +ru sin +romul ans +ro tte +re der +railway stoday +ra gging +quarter backing +qu ashes +prolet arian +por che +play renegade +photo sensitive +pasteuri zation +party city +parkh rc +pa ed +os bournes +oo ks +octa hedron +oce ang +nw mt +nelly furtado +ne si +nase by +ms sen +mr sc +mon ell +mon acan +mom entu +me aker +mayur bhanj +manische witz +mag sa +mac abe +m cree +ludwig shafen +luckywelive hawaii +lodocom ello +lil lahi +lik ha +liber ate +leonard peltier +lc sw +lai kuanlin +l boro +kier vi +ki prop +kerast ase +kcb group +kay yem +kal dor +k gf +jx mmi +jvp live +jo ee +jnd newin +jn tu +ite u +inter nists +indra dhanush +ilo tt +id lesband +hou mous +hex en +hend aye +hani m +ha patra +gwent police +go ar +gli atti +gla snow +gir lof +ger land +garethe mery +freethe children +flow cytometry +flam ini +fe ye +est agram +ent reco +eng les +electro magnet +dst govza +drugab use +dri pper +dl bcl +dese gregate +democrat sare +day today +davidal angrier +david w +dae han +cornell birds +colin cowherd +co ia +co el +cloud busting +clat sop +ci endo +chilling adventuresofsabrina +cha itra +ch kala +central valley +can teens +cam t +caffe y +brueg gemann +bren ham +brace less +boy ss +blo wh +bit on +big rig +big bluerising +bharathanen enu +beauty awards +bar nicle +ballo onist +bag mati +ba rex +aur ic +au ta +asburypark press +ap ada +ande cho +ald inga +albion parkhrc +af ic +________________ _ +( _ +" --- +ðŁij º +ëĤ ł +æ¸ĭ è°· +ઠ® +ب ÛĮ +z eli +ye ley +yan o +women slibrary +willi son +whole food +whitt all +white shell +w lo +visit nh +tz ipi +tomat ina +toledo zoo +thu bb +then ow +then ex +thelong dark +tey hainpyaarke +tar buck +ta bet +t san +suc ci +stu xnet +specu laas +sock puppet +smi leys +sl mpd +shi pre +shannon od +shannonod komo +sgvn sports +se mar +scream official +sc cg +sau dara +sar athi +ry uko +righ ton +rig don +realmike fox +rass lin +radion z +r carter +quote sabout +purlo ined +pueblo bonito +projec tive +pnp benguet +pin ia +piase cki +phil star +pain free +paddy considine +oo ley +onmy way +ok ream +official ntas +of ar +occi dente +o ssia +nz greens +nov is +my chal +music magazine +msc cruises +ms as +monte squi +mon stru +mk malarkey +minof healthug +metax a +metamor ph +me sta +mazdar acing +max okream +mal gudi +maas ante +m doc +love guernsey +lee goldbergabc +lar ale +kom achi +knit ted +kee fe +joanne clifton +jingo istic +jh pie +jg quintel +jam shoro +jahang ir +indonesi agaruda +in sofar +ho thouse +hiphop news +helpin ghand +he ree +ham on +h murray +goun damani +glen rock +gi gh +floral design +film ation +fili povic +ex pl +eun jiwon +eo connor +dun church +dream theater +dirty heads +dicky johnson +di si +davidd avis +cup ation +cou va +corn elio +cor nexchange +che mun +cham bana +cb cy +catalu ña +carolo ates +carlo scon +carbon ates +car ras +c smt +businessc ard +boy ton +bo stik +bleep ed +baltimore ravens +bad ley +auror amax +audit ore +atribe called +arse holes +arr anca +ar saigh +alnwick gazette +ak ino +air bnb +adventure awaits +ach on +ab copen +;____ ; +! & +ðŁĺį âĿ¤ï¸ıðŁĺį +ðŁĺĬ ðŁIJ¶ +âĿ¤ï¸ı ðŁijı +à¹ĢภĦภ+ä n +zy lo +zel az +zam belli +yoshi e +women slax +wiz kids +whitman comics +was u +war ta +viv ant +vin u +vast atin +under city +twicelight sin +torfa en +tik vah +thisishow we +the sc +the official +the legendof +th q +tatt ler +sweeney todd +swan scombe +sw oc +suga day +su azo +studio time +stou dam +stop islam +steep ly +st bcbeer +smy ly +sky boxes +sk yo +shesthe man +sh Åį +se ak +sd in +schoen aerts +scam ps +salute our +s vea +rol lei +rock fall +ro mo +right ness +rh hs +rec afe +rang y +racing usa +q waq +pá draig +pro pp +prachu ap +poly chromatic +place res +pidge y +parthi epan +over it +oppur tunity +oo ak +one on +omat osis +o shii +o asys +nye timber +nunobetten court +nor dha +night lights +news gathering +nar annual +na ig +na ee +n clt +muss ina +mud bound +mu co +moz eliak +mousc ron +mor ges +monso onal +mj d +migo satl +michel isz +michael aconlin +melen chon +marykill speople +marvel s +mad chester +ma iga +loong abba +lincsc athedral +lad bible +kir kk +keeping familiesclose +ke ino +k hil +joedon rooney +ji om +jessi ka +janet mock +jac at +itsme angelie +itsmeangelie ofc +ing change +infer nos +independ anceday +ima ik +illustr ator +hut cheon +head wrap +hang ar +gun buster +gu end +go vikes +gla zer +gis elle +franklin ville +foo ks +focu sed +flu ting +ff mpeg +fabu list +encyclo pedic +el ak +eeeeeeee eee +ec tasis +earn history +ea si +du gald +dramati sts +don adoni +dog ar +divyak ho +digi pack +day breakers +dar da +daniel biss +cra ze +cour town +coste ssey +cooking class +chocol ati +chester ville +chester see +chast ising +charles stur +carne secca +can tin +buland shahr +book clubs +blo ater +black label +big bro +bed kar +be yo +baliunited shop +ayud ham +astri x +as agi +arthur s +art less +antan stead +annel amott +anis arahma +anime today +ang ono +amik kr +alpeng low +almam ater +air bushc +adri ve +ac athletics +!!!! !!" +ðŁļ¨ðŁļ¨ðŁļ¨ðŁļ¨ ðŁļ¨ðŁļ¨ðŁļ¨ðŁļ¨ +ðŁĺĪ ðŁıĢ +ðŁĺĩ # +ðŁĺģ ðŁijı +ðŁĶ¥ðŁĶ¥ . +ðŁĩ²ðŁĩ » +ëĪ Ī +ãĥĹ ãĥª +âķŃ âķ® +Å µ +ze idler +ya hara +with that +winter reise +weal den +wake ford +visionary art +v india +un heeded +un decorated +u vo +towand abraxton +tour nai +tlal pan +tilling hast +thur a +theun toldstory +the diamond +tah niah +tab qa +stu me +st stephen +sportsin kansas +shrimp sofficial +sf vae +sem ler +scotland rhshow +sari kaya +sanit ary +sa via +sa aya +s gu +ry stal +rock umentary +ro woon +ro stro +ro stered +rescu eme +rachael ray +pyrac antha +pulver ize +pu late +promo zione +poo jak +pin aco +perfect match +par rs +p tit +onthe air +one wyoming +ol ar +ohed chat +nyche althy +nu thatches +ni elle +nett uno +nb storm +nandamuri kalyan +mussel burgh +musco vite +mud jug +mo ated +millstreet brew +marti ka +makeyour fringe +mahar anap +mag daily +louder than +loop back +lon ers +le ya +laura bell +laurabell bundy +ko ger +kingscross n +kha os +keral ite +ker cher +kelly music +kel me +karne val +joshu agates +jim and +jas ong +itsa jeepthing +isol ators +int fi +indol ent +imagine ers +ic ole +hu meral +home of +hillary shealth +gun di +gr atulations +gior nale +gat ty +freer adio +flx wine +fluori dated +fi da +father christmas +fair lop +ec ran +dragon slayer +dr paul +do ink +derry berry +decor dova +david lloy +cra han +cla sping +chor hai +chint amani +che state +chatter ji +chao imh +chak ma +carrol lisd +cam anach +caf fi +c mk +c jones +buzz ing +bran well +bracken bury +bo twood +blom field +bla x +bemore chill +bed post +bas ant +ap lc +ansp ach +anisi mov +al chemist +ac credit +ab da +ab ali +a strong +ðŁĻıðŁı¼ ðŁĻıðŁı¼ +ðŁĴµ ðŁĴµ +æĽľ æĹ¥ +âĿ¤ï¸ı ðŁĺģ +âĥ£ % +ö mer +ze ppo +yi mou +ww norton +womeninthe arts +wom ex +wi ze +where on +waltham forest +vinyl record +vil oria +vicom te +ven dee +uta at +un ha +un alaska +ultimat ums +uli sses +u schi +twiz tid +tubabu stun +tri kala +travel manitoba +town site +tony danza +tn f +thom mo +thetrade shub +the missing +the max +ten ements +summ icron +staffordshire bullterrier +sp global +sole bury +sit all +si delight +shr u +selfiewith daughter +see tickets +sar agar +sany u +run ing +royal blooduk +richard madden +regi stro +red ale +ra be +quins rugbyunion +psycho therapists +pre eran +poke mons +pocket watch +please dont +photo ssmh +pe texpo +pau ll +patrio tically +padam see +onthe mic +on wood +on sports +om durman +nye pi +nouak chott +nati oggi +nan twich +mtv lebanon +moor thy +mono drama +mo state +mo realive +milwauke etool +mb loggers +may nard +maverick mack +mathi ascor +mat twil +mark knopfler +macand cheese +louisi anat +lolit afashion +lo ey +leit ao +leaving neverland +latavi us +l rn +kuri yama +kreu zer +koso ko +ko pec +kar oli +justin roiland +jump shot +jess alyn +jacket pride +ja hannam +ipu parliament +ingui stic +ik wy +hur v +hi ral +hesj edal +hel an +hei hachi +he ming +har bored +guantá namo +gam is +fr in +flip flop +fiat friday +fen cers +famili ares +euphe mia +ell l +el anna +e guchi +divul ges +disco teca +didyou know +devon shire +destabil ising +dent su +de tto +danger zone +da sein +cy matics +crand ell +comer mx +colossal con +clark dale +clan lupi +chil dof +cher itage +chaudhary mos +charli ea +castle derg +castle comer +cast ilian +can zoni +c cea +black butler +beng haz +bbce urovision +bar wa +b sales +aw ali +avoc at +augu stu +ani x +andre c +amil ies +am ti +alay kum +ais beautiful +ack enzie +abr antes +! ðŁĶ¥ðŁĶ¥ +ðŁĩ®ðŁĩ¹ # +ðŁĩ¨ðŁĩ¦ ' +ìĹIJìĿ´íĭ° ì¦Ī +ãģ Ŀ +âĿ¤ï¸ı : +ᶠł +à¹Ģภļ +à· Ļ +ó t +yeg biz +y tm +world wired +wol ffe +wil lesden +wi od +wel le +vl cc +ushe rette +unis sen +un constrained +u pert +twoo fus +tu loy +tu ffs +tsu en +travel photographer +tra inning +tor ridge +tokam ak +thor gerson +thol land +the vinyl +thal mic +th bday +texas roadhouse +taxonom ies +talmu dic +synop ses +super talent +sumbasturtt ooo +sumbasturttooo sday +state ofe +ssi ers +soul is +smu k +silver alert +se zer +se hat +scots man +sat elite +san alytics +saddle brook +s mets +russ diemon +rspb norfolk +rspbnorfolk linc +rou bini +ronnie fieg +rio jaw +reic hel +rei ki +rare pic +r risd +quo test +qld premier +prow lers +pre ve +plant breeding +pic torials +photogra hy +pe ya +pat ro +over simplification +our planet +osun decides +oo di +oe dema +nkab inde +ner ang +music ally +more so +mon now +mod bury +mit sun +minim i +minic ar +min son +min dover +meto we +melt zer +medi bang +m chan +ly de +ludo vica +lovethi splace +love construction +lored ana +lesc aut +leonard nimoy +lee music +lawenforce men +lan o +lali berte +l do +kush n +krono squartet +kit ching +kimberley jwalsh +ki ddle +kan aloa +jossel yn +jo vem +jes see +je witt +jani kowski +janella salvador +jane eyre +it speter +in ie +il kay +id led +hus ar +h dh +goog let +go wers +glac iology +gil le +gi x +food guide +fol ke +fnb sa +fle isch +finger picking +f cie +dz um +dyskine sia +dundee fc +double ornothing +disfru tar +din kel +desig no +deodor ants +del ille +de veau +da ay +cut more +cub scouts +cook county +common weal +clwyd tweets +circe anna +chin maya +chilli da +ce dros +cat mint +carri on +buse ireann +braun stein +bobs redmill +bir e +belvo irst +bel ing +ban kes +b iti +autor oute +an ory +all hallow +al maj +aguil ar +af fy +adri aen +ad sk +ad me +.... : +ðŁĻĢ ðŁĻĢ +âľ ¡ï¸ı +zar o +yu uko +ye u +ww c +world teamtennis +wi jn +whit sett +wesle ys +watson iot +walt frazier +w bn +vin en +vi ggen +valley field +vak il +ut la +ump ed +ul ys +twee ti +travel deeper +topdrawer london +tiny house +till ers +thi stv +ten ka +tarta kovsky +tac ek +t sparks +sutt les +survi vin +sucess ful +stau ber +spra shanth +sonnen ch +sn itching +sm supermalls +sli vers +sla bajo +shun ts +seanan mcguire +scav olini +sc ake +saraj cox +sandy ford +sand fire +sal sas +saiy uki +rosemary shrager +rip muhammadali +ri ya +rhon dd +ren ta +ren old +reen ie +rec chia +re education +r ferl +quality improvement +pur ps +pull inger +pro lapsed +peta india +pe pero +pe ahen +paste bin +oun taine +oto ko +ni ver +newport folkfest +never y +music i +mu sina +mon twood +modular synth +mississ inewa +mcin doe +mari yam +mar os +mar ing +mand ola +macca bee +loe we +lo cash +liven good +le rena +lavo isier +lam bi +koo ser +ko leos +khush want +k lun +k cee +jamesh askell +ja afari +ithac acollege +ira sci +indy to +idu bbbz +huar az +ho plr +hit sugaya +hi va +her bo +han lan +ha sc +gre tag +grand baby +gor ls +gor aiders +geni y +gargan tua +full kit +flat lands +fel id +falken spotting +ezz at +extrac orporeal +en larger +eat slee +dor si +dev log +demon ised +dash board +dar bari +dan sa +da ij +d mart +d conf +cu bing +county gov +collo quy +cat boy +castig lia +calix to +calend ers +ca via +burn sday +bud gies +bs fc +brainte aser +br ata +book bug +bo ies +blue grey +bloomberg dotorg +birthr ate +bibli ophiles +bi ra +bi ggles +au rion +at ation +asmode us +apart ners +ao i +anz ca +antho cyanin +analge sics +am maasante +alone together +agrit erra +adel ica +! ðŁĴĥ +ðŁįĴ @ +ðŁįĭðŁįĭ ðŁįĭ +ðĿĺ ¦ +ë²Ī ì§ +ñ e +zlo ty +zel f +ze ich +ye agles +yam it +ya ke +wwt worldwide +wri thing +webber naturals +wc f +ur za +uof denver +underestim ation +un oh +troopingthe colour +tre mont +tok ina +thelittle idiot +the global +termin ologies +teas ley +ta kt +sw yn +strato volcano +spokes man +spectro gram +somo za +smo squito +sme ad +sm ta +sj susp +shim my +schin kel +salvad orean +sag ged +ru man +ridge ley +rich thofen +reyn es +repudi ation +reg no +re ade +pure gold +pier paolo +part low +oyster mouth +ottawa hospital +oster tag +osh kosh +om ah +o city +neuro typical +mull aly +mu larkey +mogwai band +michigand ers +mel ua +me ppel +me iz +mars global +mark richt +mani er +ma sat +luce scu +live ga +livega aresults +lind h +li zam +league one +lancashire day +lanc shospitals +la fd +kush waha +kri os +ko hr +kick s +kati ek +kar ad +ju hi +jodie whittaker +jha adu +jan sport +jackie morrisart +j angling +irish hockey +invol vements +ing peace +immun ologist +i yam +humber side +hu gz +heem skerk +hed vig +healing crystals +he yn +gü ne +guad ar +gor zata +galent ines +g dr +fresh meat +finne more +ferr arif +fer mi +feel it +faithe vans +facebook ads +ek land +echo esof +ec atepec +duke sof +digital nomads +diane sawyer +dhaula giri +denter tainer +david schwimmer +cy f +cruz ado +cor saire +coastto coast +clon curry +charlie baker +chao tix +cel ina +cavanagh tom +c ttw +business day +bur ki +buch la +bronchi ectasis +bro sius +bor Ã¥s +black gold +big ass +ber de +bel ittles +beauty chic +az on +ash p +arti fact +andrew bogut +alexandri av +ain hoa +a amo +:- )" +ìŬìŀIJ ìķĦìĿ´ëĵ¤ +ãĢ ķ +âĿ¤ï¸ı ðŁĴĸ +yoshi kawa +world pressphoto +work with +wonder swan +wildwater stu +whata view +water stone +walker artcenter +w pr +volu bilis +ver no +veebha anand +vag rancy +tumble weeds +tubabu yukustun +tri shap +travel gumbo +tor rente +tor por +ti sca +thel y +thebusiness show +temp tation +te gid +tann ahill +sweet life +sure shra +strep tomy +spr ou +spark ly +sonic fox +sombre ros +sof nature +sob tians +so gard +sla vica +sky light +sksk sks +skill sshow +sk top +simi on +sime k +side one +sfe ir +ser rate +senator cardin +se wanee +sco tian +sch leg +scatter brain +satoshil ite +sand town +ry dell +ronnie oddo +road sbrewing +rishab h +restaur ante +ren ounces +redar row +rachel boston +pter osaurs +psycho logies +poo bah +picture h +phrase book +pepp ino +p kn +ocal an +no bin +newar knj +nelson piquet +ndom bele +n pw +n nab +mount juliet +mondaw min +moderni stic +mo chan +mindbody soul +mindbody green +midnight madness +michael irvin +me dair +matt makens +mar ny +mag adan +lol rt +listento this +le drew +lam poons +ki goma +karlo slabajo +ka et +just inj +jazz radio +janak puri +is scr +il tm +hu mn +hin de +hef ti +han ly +han ayo +ha fa +go grow +gibr al +ger ace +gau t +from dec +freen hs +fr ann +floo w +fab by +easty orkshire +du pleader +dra il +dian eneal +di wak +davi dru +craz yyy +coulom be +concor di +cl unie +cho key +char len +cha erin +central pictureh +cc mv +cat fished +carloscon dit +buy black +butchart gardens +brin apalencia +bir cham +bi ff +bhagav atam +beta thetapi +beng tson +bell roy +bare bone +bancos antander +b horsetrials +as cents +arth quake +ar matures +animal sasia +ancient art +aloe blacc +ah music +actof kindness +acceler ometers +ac lock +aa ar +... ðŁĺĺ +ðŁĹ£ ðŁĹ£ +å¤ ļ +ãĥŀ ãĥ³ +âĽ³ï¸ı âĽ³ï¸ı +âĺģï¸ı âĺģï¸ıâĺģï¸ı +à¹Ĩ à¹Ĩ +ยย ยย +z v +yo wen +y endi +y bn +wood working +winder mere +whoo kid +walker stalker +vivic afox +visit orlando +vie wing +vertic ality +ve ille +van horn +up there +uof m +uf ford +to logists +the men +suppor ter +su amico +stele com +stefan molyneux +star z +sram road +spun out +spen guins +spann ers +smoke out +smith mp +sma kers +si sts +shan el +sh ales +segre to +seen onmy +scint ill +sche iner +sanlu iso +sakura gi +s ga +rutherford ton +rubber ised +pu asa +prayfor syria +port credit +pon ent +point break +pir sig +personal brand +peri winkles +per mit +ped lars +pam bondi +open wrt +ome body +odi dev +nov as +nott z +north face +nordoffro bbins +non standard +noki a +nishar awal +ninot chka +newss vc +neg aunee +nav ed +n sen +mytwitter anniversary +mumbai metro +move more +moon rock +minim isation +micro management +met to +memo ires +mcga hee +maxi on +ly cian +lovethi steam +loud mouth +losf eliz +lopes rising +limp sfield +like ability +lan ken +kn or +kir ari +kids games +kid problems +keala settle +karish ma +jo ongi +jin bei +jfl mtl +jamie chung +iw amoto +insec tiv +injury pics +ight club +ht delhi +hon ge +heck ert +han ai +ha iri +h music +gu ill +gru mbled +gra fi +gote ana +glo winthedark +gil das +fun dus +fred vanvleet +fre as +france diplo +fo el +fix edin +fas sler +faha dh +fabi of +f é +espa illat +en fren +el rancho +el nido +el arry +ee x +dro op +dontbuy aticket +dis believing +din nington +deray davis +dem ander +dash pay +crazy horse +cor moran +cold feet +clu bre +cloak coin +clay pole +clan king +cine rea +child lessness +chat ur +cel sus +ce dentertainer +car ris +cal fire +bun inyong +bt group +brø nd +bru hs +bring itback +borro wings +booster club +bli t +bho sale +bar de +bad ha +bach alo +aw oo +angelalan sbury +ana q +alm shouse +ald well +ad dae +acec ar +ac ats += ))))) +ðŁĴ ¾ +ðŁij¨âĢį ðŁĶ¬ +ðŁIJ ¡ +ðŁİĬðŁİĬ ðŁİĬ +ðŁįªðŁįª ðŁįª +âĺĿ ðŁı» +اÙĨ ÛĮ +yu a +ys auce +yaa dein +y nasty +xmas jumperday +x ende +wkc dogs +win tle +westerni zed +welcometo fife +wearable art +vol f +v orders +unborn livesmatter +un inviting +ultra chat +tyn drum +trump taxscam +troeg sbeer +trave sties +tom ove +time suk +thisweek abc +ther uf +then asem +the blues +thatcher ism +tender foot +teacher training +super cili +stol per +stif les +ster rett +sta el +spl endo +space walks +so zo +so be +skelton sophie +she kar +shaw sheen +shar bat +scienti sm +schoolo flaw +sand nes +sa stry +ruby tandoh +ru sal +roger bezanis +ridge view +r ttowin +r nu +qu ach +q esh +pv z +pol ster +physio logically +peter kin +pe prally +pat ar +paci fying +ou gars +om n +olu wa +of thel +oci ation +oat cakes +o ja +nw fl +no stell +nhsc trust +ne utdfc +nbc sandiego +nar ges +nak aka +myri am +monte jo +mis ael +milak unis +mero ving +me hi +matte son +mathiascor mann +mal content +mah ana +ma zzi +m xc +m rad +lou sa +lob sang +lind asu +liat ris +li jk +ldn ent +lamb dal +la har +kres se +kra z +kot ze +kathe vans +ka than +ka det +jim cameron +j achat +iso topic +iow acubs +inthe usa +inspira si +ici onal +ib tiss +hoursof spa +hou gaard +homin ids +home makers +herz berg +hello ooooo +han am +hamilton ians +hal lux +h mis +greeng ables +gr itti +glyco sy +gl ace +gabby logan +frank town +fb nation +fazak erley +exti rp +en cwx +emiratesair line +emili op +ell ac +dolant wins +do blo +disbur se +devi ates +den cia +del amar +de aries +cor ton +colo colo +codi fy +christma scards +by word +bryo phytes +bro g +boath ouses +bare illy +baldri ge +ar field +anc ook +alway sab +ï¸ı âĻ +ìķ Ī +åIJį åı¤ +à® ľ +zap ier +ys london +x bit +wy kes +wsc atlanta +work flo +wool loongabba +wied lin +wake ful +vitru vius +vijay awards +vel ive +van ss +tur ri +tokyo pop +the pretty +the orlando +that sthe +team c +t zy +t byd +sun catcher +stephen colletti +stal inism +spag nu +so tis +snowmob ilers +shop my +sho ward +she kel +sharma bjp +ser zh +scru mv +sat un +sas one +s los +s line +s bt +ru ski +quarter finalists +quand t +pun kt +pp an +pin acol +pepper idge +opti ks +officialtf gm +nottm playhouse +nor dan +no ter +newtech network +naf t +n mmu +multi beam +motor ing +montesqui eu +minof culture +minofculture goi +mime sis +micror nas +mi ani +mazz anti +masa an +mal bork +m wak +ley green +le ther +la ging +kurt schlichter +ki ke +kal ona +ju dai +janele eves +jale el +itv westcountry +is ci +inthe box +impu dent +ih re +iamking promise +hy pixel +hugh i +hau dio +han kerson +gru sin +grand view +gra vid +godd d +go puram +gas sy +galactic elliot +fra ger +fow le +fon er +fest spiele +fandom btsarmy +esc aflow +dispropor tional +des barres +den ker +cred ito +cr ite +consu ls +cl aman +chimpsin socks +cheer up +che shirec +charliebaker ma +chang kat +caleb shomo +bra sh +bor ra +black work +bharat bandh +bha d +bell let +avi vauk +atra ircraft +arvid sson +ano d +anjun adeep +anim ate +angela white +alco pop +ade en +ac coya +" .- +ðŁĴľ ðŁĸ¤ +íķ ł +ìľ¤ ìķĦ +åĽ Ľ +âĿ¤ ðŁIJ¾ +ൠį +़ ा +© × +zan aco +yw pd +yu va +y rian +wood works +we stor +wak an +vel iko +vanguard ngr +vanguardngr news +vanc leave +un learning +uk crafter +uih lein +tur ay +tum mel +trelaw ney +tour nee +tho ckey +therealt boz +thereal mattkemp +te un +ta sik +swag gg +steph end +star ring +spe ace +souley mane +soo o +sonic forces +sinful sunday +simp er +silli er +shop uk +shon ibare +shay kh +shan kha +sham shad +sever ally +sef covic +se ow +scar is +scale bound +sandi acre +salty ard +ru ta +ru gosa +ronald say +rn wn +resusc itating +ratche tness +ranc agua +proudto bean +propor tionately +posh mark +plu tus +petro ff +peter gallagher +pers ico +pas crell +par amo +oun ion +orgreave justice +oar sman +no iz +nca avolleyball +nca atennis +navy seal +n lex +my scar +mor io +minn ich +mil aj +micro tia +michael jackson +ment is +mein en +matt bennett +m qi +lub wx +lo tan +leo burnett +legalize marijuana +le scu +kuchi ki +kon omi +kha sh +key amo +kel y +kare em +kal vin +in humanely +in coming +iam sandraoh +hydro electricity +high wycombe +her dsman +hel dt +healthy habits +hand printed +ham re +hait ink +guter man +gg davidjohnston +finger hut +fel tbicycles +euron cap +er and +emu fb +ed din +dran ks +di wata +desro chers +ddin shah +dayton flyers +dam ed +cry onics +cream tea +chev rier +che wy +castle berry +car leen +canni zzaro +cal c +cadi gan +buddy hield +bor rero +bn ppm +blove is +beast men +be fi +bash kor +az ed +audible uk +aten cion +asab fb +ant sy +andri us +ana am +allen ge +ag onies +ag ian +ad dax +actin ic +ach ats +acci dente +* âĢ¦ +) âĿ¤ï¸ı +ðŁİĦðŁİħ ðŁı» +âĢ ¥ +Äį ek +yoo hoo +yar wanda +wo hn +wi mey +warren dale +waja hat +vidy ape +uoft medicine +unct ad +un scramble +tri fid +travel ph +trade deadline +top starnews +the ophile +terror tuesday +tan imals +subscri bes +su chard +stone cutters +stev elu +star darshan +st mark +southpoint lv +sin fulness +shi bley +sher lyn +shepar d +sham o +sentom cotton +seems legit +secre tof +school snc +sche ster +scam era +sar ny +sal tal +saint anselm +s wn +raf fi +pu gh +proudto workatbmo +projec tre +pratt and +pm ct +pie za +ph m +peten ajarian +peop lemw +pentag onal +p wt +om pt +old spice +ol au +nyu stern +north park +news journal +new sen +near by +nas lofficial +nag ase +n gh +mor th +moder nam +mi len +melissaand joey +meli odas +me cham +man zar +man asu +ma br +m roz +ly u +lili enthal +kyo suke +ku fr +kra viz +kk c +jäger bomb +julian marley +jrr tolkien +join aap +je pang +jam ar +iv ri +in most +il ys +il ok +i give +he ymo +hak ko +h tcu +green skeeper +gravity x +gol dies +go inghome +gh arib +gart side +gab ru +fly y +fil by +fi dan +fet life +fay ad +fant ino +eno ir +ema son +em itchell +eli p +el angana +ear thers +dun igan +dor f +do j +ditt mer +dex po +cre aky +corri mal +coraz on +con fab +chang an +cha amp +cer vi +caer u +bur le +bull ington +bry anna +broom all +blo ks +billy porter +ben koku +bell ringing +bel ou +be informed +balu arte +bad sha +b surveillance +b gi +atasco sa +armend ariz +ake ley +ak ers +ag od +ag grandi +af rc +act itud +abe sigye +aa fia +? ðŁĺľ +ðŁĺĥ âĿ¤ï¸ı +ðŁĺĤ .. +ðŁķ İ +ðŁĶ¥ ðŁĴ¥ +ðŁ¤Ń ðŁ¤Ń +ìŀ¥ ìĿ´ìĶ½ +ìĽ Į +ì ¦ +æľ ± +ãĥ¼ãĥ IJ +âĨ © +à¹Ģภģ +world alzheimersday +wine review +willmott dixon +welcom ing +warri gal +waifu wednesday +w acs +von ian +vinyl collector +valu eless +tvguide magazine +tru eee +troy aikman +trap soul +ton ature +tic toc +theti den +theroyal opera +there bel +thekings fund +the feed +that swy +thal f +team tuesday +team psg +te bogo +talis manic +swar na +supriyo babul +sulay maniyah +stubb ington +straw man +stoudam ire +ssel berghe +sp ak +son amo +sm ic +shack ling +scott k +school trip +scandal ously +ryu junyeol +rukh sana +ri sin +reic hard +reflexi vity +red lines +raim undo +r ki +r kg +r ills +ps j +pre ndre +pra dera +plat in +pir atical +pig nat +picke ters +philippe gilbert +ph um +pe otone +pap olitics +pap ad +padmash ree +p sms +over stepping +osu v +ooster beek +ocam l +o izo +nc po +national pastaday +nat aly +nan kana +nag ari +n fum +myco toxin +mr h +mo ayush +mitch gerads +mis uponus +mike j +mendo za +mass generalnews +mar kes +ma kinson +m sy +m ll +lu u +lin gotto +light itup +lie bes +liciou sto +le pper +le compte +lady tron +lac aze +lab corp +ku rien +kle infeld +kin nard +k sp +japanese chin +j williams +is b +ine te +ima am +hong seok +hell yes +health food +ham med +h bogo +h barnes +guer cio +gi puz +gal is +g tc +futureof mobility +fu miko +fre jus +france sinha +flu mes +fal am +entreprenu ership +elek tronik +elast o +ed widge +early risers +dutch nat +du breuil +dont panic +do xy +digital divide +di ack +deuse x +de it +de cle +dark matter +dac apo +cow pens +conten to +coach d +clackmannan shire +cin zano +chin y +child safety +cc stca +castille jo +bo jonegoro +bmw champs +bio engineered +big dayout +beer advocate +baycity rollers +bashkor tostan +band master +ban sal +bailey sprize +bag nell +avo ices +august ine +atu cker +at avi +as kincare +ari v +ari ana +ar ten +ar pad +anne tta +angi olini +ang ini +alex ail +alber tan +agor aphobic +ago g +acar r +ab ration +ðŁijIJ ðŁı¼ +ðŁıħ ðŁıħ +ëĵ ł +ê³łë§Ī ìĽĮ +zoo sk +zam in +z eca +ysrc party +wux ian +wonder lust +win fromwithin +w chl +vanhoo ser +van ak +urbang ardening +twee zer +top speed +toi fa +to ghe +tic hen +thesam icallihan +then bhd +the plac +th ate +te sl +tar kowski +swan ee +sub plots +star ac +sse m +south church +shet ler +share thedream +shar my +shane helm +shanehelm scom +sham sher +sh amy +sewing bee +sen o +security council +se tif +se ap +sco bar +sand alphon +rosen gren +ri ina +pro pa +prime performer +pri d +pod cas +par mley +par ise +pail waan +open jdk +op lus +on hot +obi dos +oak v +night club +nec primeperformer +national nappingday +nadal ind +mush taq +mure san +mo dan +miln row +mil ing +meyer land +marang oni +mal ope +maje sty +luci ano +lobo tom +lean or +laro ja +lab and +khar al +katie qlowes +kar adeniz +k beauty +jon ty +joann alum +jiz an +jak elong +it railer +is sou +indie gaming +ill ero +ile v +i sho +huguen ots +hr block +hoch schild +heart this +har bisson +gu zheng +gu li +graven ey +go stags +gno sticism +glasgo wc +gc punknewwave +gar field +for c +foodre volution +food mag +fle sch +finan za +fin ning +fieldof dreams +extr atropical +estadouni dense +est il +ero ski +er mer +election pakistan +el mes +eh c +ed ar +e tec +donnell rawlings +don kiss +don avon +dil raju +den mead +dar le +dal um +cyber goth +cra ige +cow rote +cortel you +ci flacs +chir stmas +che in +challenging stardarshan +catholic newssvc +calvin ists +c xr +brou weri +bro sse +bir gunj +big wig +ber lage +bell oni +be ze +be sting +api er +anat oli +allo tt +ali ko +alcon bury +al ver +adobe symp +ab rin +@ _____ +ðŁĺŃ âľ¨ +ðŁįĬ ðŁıĪ +éĻ Ī +æĿ ¨ +ä¿ Ĭ +کر ÙĪ +» ðĿĹ +zi bo +wrp stoday +wey bourne +wbc baseball +water town +vo lei +viol ino +vers ation +vend itti +vee redi +value walk +valky rie +tü v +twick enham +tweetyour seat +tra ven +tow yn +too ie +tmr kt +tin sider +thereal stylesp +the quietus +talis ca +ta aa +suppor ts +sun omiya +su el +stru tters +stpaul saints +squirrela ppreciationday +speak up +sour cing +so rell +smee th +sle zak +singul arities +simul ink +shor ne +se iner +se gan +sawak ened +s was +s enger +ru mi +roman ized +rin der +rhy d +rayne spark +pushawards donkiss +pu yi +pro max +pres anctified +pou rover +pom bo +pi kappaphi +phe l +perver sely +patriot sawakened +o canada +nil an +namak kal +nam ja +mull ery +mt cc +ms ss +mol ls +mil ner +mi datlantic +mend on +medic aleducation +medi acon +mc nichol +mb raves +matricul ated +mat thu +marcin iak +magic weekend +ma assen +lun amaya +lu skin +loc all +le ury +lawn mowers +kum bia +ku wabara +knap ton +klo veradio +kizz abesigye +kirkby lonsdale +king e +kap ar +kalev ala +kad ay +k me +jumu iya +joey kramer +jhar su +j nc +ittf world +invinci ble +insinu ated +initi ald +indra prastha +ide e +ic v +hob goblins +harrogate hour +ha ab +gu lab +green party +god des +gir ouard +gb ball +gas pe +funny bone +fore sti +fla k +fin back +fan tome +fairfax county +fa ecal +f mtv +eli zed +el khound +eeve elu +ec lo +dur ley +dun ord +dublin pride +duba iland +drainthe deepstate +dpc dsb +doun reay +delight ful +del mon +daf trucksuk +crê pes +cruci form +creati va +countyof gp +cot trill +corpor ately +cooper union +chi ayi +che doke +ce of +catul lus +capodi monte +callum smith +bu cheon +brun elle +bren a +brand l +bis sell +bis cay +big dreams +bha gira +bearcat nation +be tw +bau dry +bath wick +bangerz tour +att lers +an tum +ameli ar +all am +agelim it +aga pe +ag l +ach ari +abstr acting +a hini +( âĹı +ðŁİħðŁı» ðŁİĦ +zo boomafoo +zeetv jodhaakbar +wur d +woody harrelson +wol bachia +white way +we ma +wash dc +walker hayes +w assim +val orem +v atika +us vi +turkeye lections +tro binson +thegreatest showman +the oscars +the core +telegraph news +tarheel nation +tar ge +tan it +ta zz +syd ni +surve il +sul fites +sub soil +str m +sten nett +sonthe hill +so sick +snorkel ers +skit ouring +showyour stripes +sh kar +sexu ales +se alift +scifit v +same sex +s mag +royal jordanian +rou k +rosen heim +rosen ama +rosenama junas +ron quillo +rock solid +rit zy +re orient +re negotiating +r atta +prote aceae +pronoun cements +pp s +plot lines +plen itude +on p +oliviap alermo +oakland county +northeaster ly +nikhil chinapa +night cliff +nic oti +next topmodel +near buy +nas ution +mote ts +mill woods +migun amig +micha elia +mi kazuki +mez quita +mar ls +mar lee +magicmike xxl +ma gre +london calling +lo wit +live phish +lies matter +li viu +larry king +land use +kristi ana +kib bie +khu lic +kavanaugh now +kassi us +inked girls +ic gc +hughi efury +house sitter +hon asan +hir ingnow +helg ason +hear ty +head stall +hartford courant +hang leton +ham lett +halton police +gro es +griffon ramsey +goulet pens +general isation +fore stier +fi ere +event prof +et ats +et at +et ags +eric thomas +embroide rers +el sner +eet ings +easter island +depu is +delhi ites +dan brown +dal riada +cosmopolit anism +con cious +colling wood +cl at +chinese medicine +cephalo pod +cdn press +cau stin +bull frogs +bruno tonioli +broad cloth +bombay bicycle +blood brothers +bic ultural +beren berg +ber ro +bd smovement +bar in +augu ay +ash es +ash combe +aschaf fenburg +ar groupsuk +app l +ap na +annecy festival +altam irano +aj w +aga inste +abster go +ðŁķ ¡ +ðŁĶ¥ ðŁĴ¯ +ðŁij¦ âĢį +åĭ Ł +ãģĹ ãģ¦ +âĽĦ âĿĦ +à¹Ĥà¸Ľ ร +à¹ģà¸ Ĺ +youth homelessness +y da +whid bey +wat onga +visith ampshire +vi rion +vg com +vel ux +uri en +ukcrafter shour +tur ville +tur rican +tsong khapa +tourdecor se +timber ners +tier no +ti rl +thirty something +thermo plastics +thelo af +tb q +tal aash +ta kuma +swan son +superbowl xlviii +spencer boldman +sort sintl +sleepyhollow fox +sla vik +si dro +shet terly +scroun ged +san mate +safe haven +run du +ruff ner +rod ina +ride ox +rep ens +rec chi +re zeki +prophe tically +plan um +ping ry +parrs boro +paraparau mu +p cori +or rinhatch +olive tte +oli vella +official blue +of ashion +nush rat +nun an +nie man +ne rea +ne grin +naq sh +nant garw +na hlhockey +na anum +mycause my +my journey +modern warfare +mitt agong +mis appropriated +min der +middlesex uni +micro tubules +mgmre sortsintl +mer na +maru game +malm strom +mait ake +macmillan kidsuk +lemon is +lance field +kra g +ko sty +ko li +km fm +kad aga +kacz marek +ivel isse +iti er +inter bay +intelli j +in sou +ili um +i imc +hrtech world +hoo fer +hit parade +hill crest +hapand leonard +hair growth +hail stone +haha aha +gurun anak +grac ed +gemini ds +gd st +gal v +for umb +flo tte +flash man +fl outed +fent on +fe tters +family goals +fa wnl +eviden cing +escal ada +em j +el r +eff lorescence +di ant +dev lin +depend ants +death ly +dark and +d bo +crimin alizes +crashed ice +cor robo +cop us +cm chat +ci ot +cast en +carlo sainz +british pieweek +brad wall +bor on +bird shot +biomole cules +bi bby +bestteam in +balle tic +bagat sing +bac ton +au ghtme +at the +as afo +ar ness +aqual ad +apit alist +anu media +americanc rime +ali ah +aldub thering +air worthiness +aegon championships +abu il +aa aan +ðŁĻĮ ðŁĻı +ðĿĺª ðĿĺ +å¤ ª +âĿ¤ï¸ıðŁĴĻ âĿ¤ï¸ıðŁĴĻ +âĺĢï¸ıâĺĢï¸ı âĺĢï¸ıâĺĢï¸ı +âĬ Ļ +à¹ģภģภ+zyg munt +y ula +wy lie +work space +wild scotland +weather tech +wean ling +viz quel +vit olo +vi gg +ver day +usu l +umich hockey +uk health +ty pennington +ton et +thisis fusion +ther saorg +the wee +the village +tele conferencing +tab lature +sv vsd +sun burns +sul tra +star my +spring isd +spat ter +smac cus +sku ld +sizz lin +sin space +shu ji +sfor you +scol lide +sche ana +roth gar +roo ke +rohan mehra +radi oman +rach els +ra ditional +que star +pra des +post secret +pon dy +pi lea +oto scope +ol kata +official sslazio +of books +ny jv +nord berg +nice attack +nax alite +nar ducci +nad asurf +n sula +my babies +msh sl +mom of +moist ened +mit r +min di +mid wales +mel ena +meg u +me urer +mayward for +margare tho +man ov +mal ouda +mah ra +mackin lay +luxor lv +loun gers +lo pers +lim pid +ligab an +ligaban comermx +land wehr +l alive +kumar u +khulic hana +kho si +khaz anah +kell am +jewer ly +jenniem calpine +interven tionism +ini sti +in volu +husky pride +he tt +he aver +harri sfaulkner +handmade jewellery +h laudi +gü nter +guadar rama +greeting card +goo dge +glen ns +ge tb +gann icus +fun ck +fountain sabbey +fficial page +fa thy +escut cheon +eric ans +emb erton +ei der +ecor nish +dj ze +dj david +dil bar +dia internacional +deob andi +del ice +dej oun +degre esof +de mountable +dang le +daffo dil +cp gs +corre os +cocc inea +chub buck +chic lana +cassad aga +bul lock +bu erk +bru v +bom ar +bh vr +bass rush +ban kon +ax ons +all inthe +aleu tians +agar nier +af und +act for +ab ct +a septic +ðŁĶĿ # +ðŁĵ ľ +åĴ² èī¯ +âĽı ï¸ı +âĺİ ï¸ı: +âĢ¢Ìħ _ +ص ÙĪرة +ин а +zel jko +your plate +xylo phones +xu an +wire uk +w kc +visit norway +vann elli +tone bell +to car +ti gi +thi el +the tachi +the doctors +teryl rothery +tauto u +sunsout gunsout +sunday live +sun od +su chus +st mun +sou li +sn ts +smom entum +si saket +sfra hm +sean sch +scott adam +sch ank +regin a +recycle sday +ratche t +rapo so +ram page +prof david +ppsells babyparts +policy maker +pol lex +plus blogs +pion ate +pe avine +pan ache +otta war +os by +or be +npa ie +neapol itan +natural is +nar r +nal im +nail sinc +na thuram +n lg +moto red +montan astate +meik le +marqu am +ma whinney +lu ria +loyol amar +lone some +live aboard +lection fraud +koval ainen +ki anand +kath imerini +kar dia +kad av +jur ra +jugend stil +jim parsons +ji ye +ji goku +janete van +jackierobinson day +ishin omaki +indu sind +indiant ellyawards +hus n +ho din +him zahawi +hel goland +hal mahera +hack neye +grind time +granul arity +gon injas +gobel ins +go pa +gab led +fu ke +free bets +fra bbits +fior ucci +fam z +fake facts +express js +every corner +euro scepticism +eu karyotes +es w +erik solheim +ephemer is +ellen ville +elec ciones +eg ner +dundru mtc +ducati uk +du stria +dou ge +den nys +demarcu sware +del luk +deduc tibles +decade of +debash is +de muro +cumbri ans +cor abi +con ures +col ter +char ri +ceremon ially +ce tin +catar aqui +casc ina +cas eros +carrauntoo hil +carr aro +capitul ated +brown coats +br cnews +bong bong +blood root +bir bhum +big dog +bi gr +be wafa +bar gy +awe som +au zon +aspho del +arro el +ar ace +ann erley +al annah +ah jussi +ag op +aber rations +aan and +ðŁĻıðŁı¾ . +ä¹ ħ +âĿ¤ï¸ı âĺĢï¸ı +zen kmm +y the +wood z +wood burner +wol fin +wic cans +who p +wetter ling +wer ki +wel len +weid ler +wartho gs +virtu osi +vi aj +veteran sday +utre ch +unicorn day +un seasoned +un mas +tu dela +tre acher +tou rage +tororosso spy +tit res +thi g +thelon gest +the budget +table tennis +supt king +sub structure +sri devil +sridevil ives +sridevilives forever +sportac cord +sloven sko +sha shan +sequ el +senergy drink +sal vac +sal guero +ryandun gey +rter ugby +rou st +retin ue +reti rez +recre ationally +q bl +proco pio +pro digi +pri matologist +pram ukh +phobla cht +pets alive +perver ting +personality disorder +paulo grady +pat chen +pachu lia +p drc +over step +oneof ourown +now all +new show +nat aly +nasag lenn +nanomat erial +nan ing +monast icism +mohandas pai +mobile al +mo yn +min ju +mh clg +medi us +maracan ã +mac m +lur d +loin ie +lobby day +lil ya +les b +laer dal +kyle brandt +ky naston +know thyself +kk k +kenner ley +ken ichiro +kemer ovo +kear ly +juli er +ioc prev +insu rable +industri ally +indian ad +in ot +i play +hur ford +high park +heine mann +hay akawa +hann elore +gol dglove +gi lets +ge ph +fun mi +free india +fowl kes +foresth ills +for ap +for africa +feelthe burn +fal guni +es man +eni ola +e special +dis contented +de personalization +david the +dad aist +con ow +clow nish +classic horror +cj leblanc +chri sf +cho dron +chee siest +chau sson +chan dos +chall inor +ch ado +canter o +candre va +cal stock +cad avers +buon arro +brandon heath +bra hm +block ades +ben ayoun +bad ab +ba artman +b girl +assisted living +are op +anast aciam +amusement park +amath us +alec bradley +ab dal +ðŁķ Ŀ +ðŁ¤Ļ ðŁı¾ +ãĥ ĸ +à° µ +௠ģ +zsas z +z ok +yoshim itsu +ym x +wj bf +wella pro +we aried +wal kal +vital signs +vini fera +urock radionet +tvweek mag +ti ii +the viperroom +the gift +the festival +the angry +te ds +tce lectronic +tai k +super collider +stü ssy +stre ats +ster ritory +stat ev +ssi one +spar xxx +space invaders +so ini +sheep adoodle +sexiest manalive +semper fidelis +sec ant +se sotho +scol aire +sal onica +sa repta +s monday +ry de +ribble valley +re ssed +re deploy +rail pictures +ra sm +q ag +potat os +post lethwaite +pl iner +pd news +parli ment +park boyoung +or ito +opp ermann +ooster hout +official donegal +nr ms +nor walk +nin ak +nick swag +nad himzahawi +na ho +n á +mt pa +monet arily +mo tac +miss vogueuk +mi amis +metr onews +mc curtain +massey ferguson +mar azzi +man liest +mame tro +mag si +mac alister +lu bez +lo salam +lie shout +legen ius +l wow +kri korian +kan zaki +kam pu +ka ther +ka ida +joss elin +jo su +jes shar +jeff erys +jdr f +japan gov +jackolan tern +j gi +itscold outside +intellij idea +hypnoti zes +hur u +hub bert +hu zoor +hu sen +hondac ar +hill i +har leys +h wv +gu se +grin berg +glu on +giant games +gi ously +ger hard +g gh +fre eness +flor a +fleet line +fit forlife +ev ak +europe echecs +el op +dor sa +deep veer +decem bre +david burtka +dave meltzer +ctb ase +cru ll +crit care +coron ado +coachtom herman +clark university +cityo ff +ci fa +chin chin +changeagent sa +catherine tresa +cate che +cas ula +cas kale +caroline manzo +bull riding +breck in +boving don +bot kin +bon ner +bo ie +black welder +bi yani +begum pet +bbc spoty +bad guy +ay tas +atv show +as ml +art deco +apod aca +am aa +alu z +ake sson +ak sy +ag aya +aero tropolis +ac ire +ac d +a ang +! --> +ðŁĺĺ ðŁĴĺ +ðŁĺį ðŁĺĪ +ðŁĶij ðŁĶij +ðŁİ¸ðŁİ¸ ðŁİ¸ +ðŁįĤ ðŁįĥ +âľ µ +Ùĥ Ø© +ع Ø´ +é l +youre ye +yacht smen +x fire +writer life +wilmington nc +w ne +vv pat +vroom vroom +visit richmond +v cc +untam ed +un accustomed +tusk ys +to eat +thom astur +theatre news +ter amo +taq iyya +supper sunny +sue anna +stan doff +spi key +sp uppet +solit ario +sm itt +sic ard +shurtle ff +sell wood +scout scanada +scott age +scien cel +rudolf schenker +ring sidec +ravi shed +pride seeds +popul arizing +pin elli +pfe il +pe skin +pav oni +patric ian +over doing +oli sa +of arevolution +od ley +nut case +nu us +nu fbfamily +nou ll +nor df +non committal +nin der +nil sfrahm +nes see +nc isla +nation scup +n ment +my army +mon naie +min oo +micro aggression +mechanic ville +mcl ane +mam mill +makeupforever us +ll nl +larcen ciel +koni shi +ko tto +ko sk +jiha di +jezz ine +jewish chron +jeh lum +jam rock +jack fish +hw ys +hoy les +ho el +healthy ireland +havan aclub +hate story +hash t +haiti en +gwal tney +gun fights +gre han +go wings +gha ith +ger vase +gemin id +ge u +gal im +g ct +fre chette +fr yn +fight stick +feltrin elli +fast car +en hor +el pais +ec bc +du bonnet +dragmedown musicvideo +drag ali +dou bloon +do eee +di ks +depon ia +dece m +deal z +davis jr +david suzuki +dat ar +dan agould +damekelly holmes +d hun +cycl onep +cur sus +cra pped +cool runnings +conden ses +coco abeach +co ie +bur ghs +boy ata +bound forglory +bor stal +bo bin +bin ley +biblio graphic +bethe sd +bel fer +beg awan +bbvacom pass +barbap apa +ath let +aspher ical +asic s +art smith +ang ely +am ancio +alway son +aero tech +adul terers +ach inery +acclimati se +ab util +-. -" +ðŁĺĺ ðŁĺįâĿ¤ +ðŁİĢ ðŁİĢ +âĵ Ķ +y anti +xx music +win don +visu ali +vid mar +vic is +va ad +urbann ax +un sr +ukin theusa +thisday ing +thewilson center +thereal b +the pink +tape o +tann ersville +takam ina +ta isuke +t soi +sylvan us +super humans +stu ckin +simel ane +shu ddle +sex scandal +seriou seats +sa official +s alive +ruz icka +royal freenhs +rosequ artz +roo tin +rock box +robert m +realm royale +real remyma +rawn sley +radhi ka +ra ee +puppete ering +psych ical +pro positioned +prick ly +preacher amc +pr z +poshan abhiyaan +playbook athlete +pim enta +peri ence +penn sboro +pen cak +pb ks +panch ali +ov ando +ou uuu +ot ch +op r +onthe farm +ong kong +ol szewski +nj con +nas sa +na ag +myhouse idea +mtu hky +mm ilive +mentionsomeoneyou rethankfulfor +meg acon +mann ino +man field +mal ins +mal alay +m itic +luv vie +lu to +living uk +let my +led lights +le zz +la ves +la er +kroko dil +konicamin olta +kir mani +ker mes +kelly hu +kam io +k wei +juego de +jim henson +jan netty +insider tweets +ine us +in chon +huck berry +horseback riding +her ault +he arer +har uki +good wyn +gon zi +gil ardi +gaunt let +gar lock +fur ni +from me +fle xbox +fiore tti +fence post +fac ed +e board +e anes +domestic violence +dissoci ating +dem u +de briefed +davemeltzer won +copp el +com tesse +cle lia +chiropo dist +cate ley +carlin i +can ongate +caed mon +ca thal +bru d +brett cateley +bran sford +book qw +black street +blabber mouth +bi gup +ax illary +auto gly +anan sie +an wr +an ax +amblyo pia +amar deep +aji mobi +air show +aesthe te +ad oes +ac loud +ab culture +ðŁĺĺðŁĺĺ ðŁĺį +ðŁĮ Ĩ +æĺİ æĹ¥ +åģ ¶ +âĻª âĻªâĻª +ÙĨÙĪ از +yr self +youss ouf +ylon agarcia +yeee haaa +xl center +xende sktop +wl k +vote sforwomen +volk man +vo ssen +union bank +unicy clist +tro is +tri ppe +tr b +ton kawa +tin su +three words +thom an +the wave +the party +te ann +tarahu mara +sy lum +swee thome +stick in +steep ing +stavro pol +sound transit +sophi atown +slike these +sk n +sk ille +simply santafe +simon parkin +si pri +si dero +si benik +ser dar +sch ronicle +sander stead +sabah tourism +s thorpe +s ates +ry sselberghe +red way +rec tus +re ba +ravi pudi +ravens thorpe +rath more +ranfur ly +rad itya +ra pra +ra pacious +quan zhou +prinsen gracht +photo ed +phon on +phenomen ons +people smosquito +pd mf +pdmf nb +particip a +par malat +papp ano +palaeonto logist +oli var +ol ink +o fu +npr freshair +noe mie +niç oise +ng tindia +nc statec +n toko +mun tashir +ml bn +marches afashion +marcel hirscher +macgy ver +mac chia +lung fish +leh to +kon tiki +kk as +ki pro +kerry on +kendra para +ivan ovich +inte gra +hirsch man +hin tze +hemi plegia +har ned +happyhappy labs +hanafu da +halam adri +h ure +greece central +gra gson +goodby enewton +goodbyenewton trees +go with +go scots +go cubs +gb x +football museum +fly boy +fil enames +fearthe fin +fast web +f muganda +ext ents +eure kalert +escaflow ne +erra zu +ero bbins +entourage movie +enni es +ea fifamobile +dur utti +dri se +dimp led +di ard +de coloni +de blois +dak shin +crink les +cra ving +chak otay +casano vas +cancer uk +can id +brin sley +brian stann +bravest warriors +bobb ys +black guard +bay anis +bassen dean +bas eload +bar si +bar ato +bac onf +aw ong +aru iz +armad afc +ap rice +ang ame +and aman +an ani +algar ve +acar on +??? !!!! +? !!!!! +:) :) +..... !!! +**************** **************** +ðŁij§ ðŁı» +ðŁİ ı +ðŁħ°ðŁĨ ĸ +ê·ľ íĺĦ +ê³ ¨ +âĿ¤ï¸ı ðŁĴĽðŁĴĻ +âĿ¤ ðŁijij +اÙĦ رÙĬاض +yumm my +yi fang +yas ui +xx s +xer oroadshow +white knights +watch on +w wh +var key +v ril +ut kar +ushl draft +us x +un enforceable +trekon linegame +toiv onen +the bear +th sc +tex ana +tand ang +sw is +sun gazing +suha imi +suf fused +subb u +stgeorge groves +stephanie sheh +stell arton +stel ton +spur pose +sp fc +som pting +slush hq +shu dders +shor rock +sh ring +se belum +sclero tinia +sav ban +sand ham +san ral +sadi ya +rv g +resilient cities +repre sen +reha shed +re spu +ran ee +ram rahim +pur ani +profli gate +pride chs +pretty boy +photo grams +persian food +pen men +ori ol +or ale +on ii +ome where +official allegri +of riday +oak bay +now akowski +nick mangwana +nep tun +nd lea +nb fc +nat ale +much ly +most beautiful +missal ex +mis sl +mass roots +marqu art +man oh +lu hh +london winefair +loe ws +live it +launch pad +la senza +kin nick +ker inci +kel k +kar lov +kan oa +jec t +iw lca +is ya +invent ory +institution alize +ine w +incre el +ifyou can +house judiciary +hoss ack +holroyd howe +hil ditch +happ ier +hammer son +hail mary +go ward +gm si +glo fish +ghot ki +gh im +ger ges +gee kin +gav increel +gal ov +g mu +fortean times +flyn as +flesh god +feis al +fe well +fareshare uk +extra dition +eu funded +es ame +eras ure +dy ah +down light +do ki +do dgin +discover the +digital print +desau tels +deloitte us +de ports +countr ys +coo ee +commu te +cob ber +co pai +cli ente +choo sen +choices ong +che ska +chalk paint +cele stin +cathedr ale +car leasing +ca vil +ca chaca +bé la +brum mel +box ley +bour goin +bot swan +bongbong marcos +bon te +black field +b ville +az ette +autonomous driving +au ob +are ynolds +anu grah +andre wro +an ter +an se +an m +alap ai +al mand +ak bars +ah scult +ah ine +aglew ings +af feldt +ae g +ad ush +action uk +abscon der +! ðŁİģ +ðŁijį ) +è İ +ห ล +Ø ¤ +zing t +x tension +ws ferries +wo yz +will ers +wi an +walpur gis +wac ket +w mt +visit dartmoor +vil na +vi mala +u eli +tougher together +tothe max +torin ofc +thi ong +theris enation +then ic +tammy baldwin +synec doche +swanseab ay +sw ingle +strath peffer +steadi ed +ste arn +ss afety +spo well +space shi +sou live +son s +social ism +sec nav +seapor tmrkt +seabir d +scra pp +saf fold +ro pical +rmh c +ric eless +ri sp +red sky +rath aur +ram ban +race check +princess of +pressuri zation +porsch ec +plun ket +pavlo vic +paracel sus +or go +oo chee +om oon +obstetr icians +nushrat bharucha +ni fe +nes golf +ne reus +nd preps +nar n +museumc ats +montpellier hsc +mont one +meddle some +makin en +macro scopic +m dd +ly oness +locum tenens +lee ching +laurit zen +kü n +kaw ara +k gaf +jurassic june +jk jk +jeff stinco +jann arden +jackier obinson +j atim +iamjer maindefoe +i aye +hunie pop +hov de +hfx seaportmrkt +heat seeker +hatt in +ha ssi +h wd +gutt macher +gri ha +gi rolles +geor geor +geer twil +ga jap +g pdf +flyo ver +fj all +ex positions +emirates nbd +em n +drum beats +dropou thillary +drach ma +door ley +doc tober +den dera +deal maker +curmudge only +crook ham +county fire +coun ties +cosme tically +core values +col dly +co we +co group +clay travis +ce fr +carbonell nestor +capital factory +camanach d +cal ore +buy ingit +brighton hove +bou lot +blue box +blessed ness +bel liveau +beck ler +bbc sportsday +bandar ban +ban ter +az oo +aven port +ave t +aren ga +arch uk +ar é +ant acids +andre ev +amin ute +alexail acad +alber tini +ak hali +ab ish +, _ +, '" +( .@ +éĽ Ĩ +âĿĵ âĿĵ +âĢĶ ' +Ì Ģ +Ã¥le sund +zu banski +zel alem +zar ine +zar autz +yp young +yal u +y io +wood pigeon +won de +winne shiek +whe ee +well don +wall now +w pu +view park +val pro +twin kly +traffick ing +tipsare vic +thur m +thi az +theat r +the three +thailand news +team blue +t vix +t qs +sureshra ina +stu min +stock fish +stable mates +sq d +spelun king +spar ano +sn whs +smith sburg +sleip nir +sin aia +sin a +sher locks +she w +sel b +sec nielsen +schol ly +sch ellenberg +saving abel +sau vie +sath letic +sajal aly +safaricom plc +sa pper +rowh ouses +ross mann +revan th +retro spection +repre sses +relinqui shes +red squirrel +re thinking +qui zz +q alam +pw tc +protec tively +probation er +pres stitute +pre sto +pom pano +politic isation +point swest +pla smon +per mai +pe mbe +pau leen +pang lima +palmo il +p fk +over write +or nc +oc s +nub lar +north well +no senergydrink +nine tte +nikol ina +nickswag ypyoung +nick naming +nhs leadership +nadi gar +n world +mykitchen rules +my husband +moreno valley +mo ers +mike dirnt +men stennis +mclo one +may i +mat za +magdas zubanski +machin eries +luke kuechly +lu ken +lu er +lock chain +lie bermann +lati um +la wro +l blogger +ky leigh +kp tv +ke ston +ke edy +ka elyn +k te +jonim itchell +iz ingly +inter locks +il ahi +hyacin the +house master +ho bar +hindi imposition +her omoto +hab lar +gu sm +gra e +glass making +ger old +future decoded +fur ano +fra yne +fox terrier +ff xiii +fent yx +fen cing +explo iters +eu ws +eng ill +dw mtweets +dige sters +de wis +dangerous wom +danger mouse +d do +cs ba +croy de +cre te +chi bu +chat on +ch live +cece winans +calu met +bu dig +br ama +berner ay +bed clothes +be san +bayanis andiego +average hunter +aust int +append ices +anti racism +americ andad +ame sh +al cona +afre zza +afloor test +ade mo +ac ohen +ðŁĻı ðŁĩ®ðŁĩ³ +ðŁĺĢðŁĺĢ ðŁĺĢðŁĺĢ +ðŁįĵ ðŁįĵ +ðŁį¸ ðŁį¸ðŁį¸ +íģ¬ëĤĺíģ ° +ë¥ ´ +Ạ¡ +ห ร +yul ong +wor boys +wolf ers +win ick +willi ston +vote forchange +visit snowdonia +vento aureo +tom mor +timore se +thekingof queens +the cove +the club +tamar isk +t je +sw sb +sw l +sunking brewing +sub by +spring is +splinter ing +slo bber +skep tic +sir tomjones +sid lowe +sh wrs +seun gh +saint leo +sa wano +ri par +rel lik +reh mat +real jonghyun +readyfor hillary +ra elyn +prolong ation +pre views +piccin ini +pic ea +penit ence +pe ditions +pak ula +or ai +oliver io +o tide +nh cw +new battle +national assembly +nac ott +n ttw +moreco wbell +michigan tech +mayak ovsky +matil dam +mass challenge +man inder +mali kyo +malikyo ba +lymphoe dema +lor ac +lly r +llibertat preso +lig ny +libraries week +lam ey +lam be +lab at +la duma +kre pt +ko dos +killthe trade +kelsen rally +kaatru veliyidai +ka im +jose canseco +jen te +jay asurya +jay anta +jani k +jaj pur +j mac +ish ima +iren aeus +inter cal +in sley +in bangkok +illumin ed +husker fbnation +hil dr +high ams +henrik stenson +hej duk +han pti +hac ke +h ct +grail lot +gow anda +gott alo +girl love +gar çons +gangster ism +fulle rene +fin t +fel o +est court +entdeck t +em ill +ele ssp +edinbur ght +dry ads +dramati sed +dol vett +di oxin +di ame +cultu relle +cul turi +cu uuu +cru sin +cl volley +chil koot +cha aaa +cell reports +castel vetrano +bus news +bursle don +boxer bond +book posse +bon u +bon temps +bla zin +bates burg +bashi rah +bas ar +az ha +assi r +ar me +ap hl +anti serum +anthony bourdain +antece dents +ane us +amy ra +amik kelsenrally +allian zarena +ale v +ad ma +abull dogs +aa sher +,, / +( âģ¦@ +ðŁİ¶ðŁİ¶ ðŁİ¶ðŁİ¶ +ðŁİ į +ðŁįĶ ðŁįŁ +è ĸ +åĢ ī +âĿ ľ +ج ÙĪ +y pu +wx by +wirele s +win diest +weare international +wang xian +wain scot +w top +vhs india +van os +up setter +un wholesome +u ov +u ou +turbo jet +tsur ugi +tru lia +tor ino +to er +thrott le +the office +the har +the fix +testim oni +teacher friends +taec cool +stra ss +spe ters +sinthe sky +sier pinski +shichi mi +she is +shangha ima +sales enablement +sack cloth +s gro +s followparty +routledge books +roque brune +roo th +re uk +rally x +r xr +q school +pro les +pin al +pendu la +pas ku +outdoor fun +oooooooo ooooooo +oh gov +obscen ely +obi ano +obas i +nu ffin +non conformity +no ten +nar cis +mus ick +mun daring +mormon probs +mor ale +mis sam +mend elian +me gui +man ley +lili angarcia +leapfro gged +lamb swool +ker newe +kenya power +katak lysm +juve derm +joyce caroloates +jim wkyt +jean not +jackie o +it slit +it ac +isof ix +ir ão +inj akarta +im ats +humphrey bogart +hom etour +hinter land +herit ability +haway thelads +harmon ise +har ik +gy umri +gun makers +gre glou +gio i +gastri que +g vw +fun home +fren do +fort mcmurray +for youth +fix able +essenti a +ess lingen +es news +eric vergne +er minator +ep chihuahuas +ent le +engine shed +eller ton +el sayed +ebol a +e issa +dy mph +du monde +dragon lord +do gan +dil utes +die for +desertisland discs +den een +deceler ating +country boy +cor ser +cop tic +cly ffe +cher ri +casam ance +cas ona +bukitt inggi +brujer ia +book quotes +boardwalk hall +bio terrorism +bend all +behe moth +bak assi +au clair +ast mh +arou ca +app ending +alpha deltapi +ales und +ale ksa +aldubxe bloveis +alchem illa +across the +ach ing +? !# +!!! * +ðŁĺģ âľĮï¸ı +ðŁĴĽðŁĴĻðŁĴľ ðŁĴļâĿ¤ +ðŁĴĶ ðŁĺĶ +ðŁıĴ ðŁ¥ħ +ðŁ¤¤ ðŁĺį +à¸Ńภ° +zuc chi +york dale +yan a +womenin law +wine mag +walkin stown +vote dem +violet chachki +usat sport +unitedin orange +umb reon +ultram ar +ukbusiness lunch +uef ayouth +tyson foods +ttro pez +tsh ering +toy ne +ton geren +tim the +ti anti +the market +the key +tf con +tero th +team sailer +t reader +swi ley +swach hta +sur sok +summ ilux +storytell er +ssou ths +sp res +sonymusic india +smu ller +sly james +slo viansk +sis rael +she hab +sha j +senyor a +sch aff +scan al +remington arms +remb au +rati gan +ra smu +ra gen +r hanews +r fe +pym nts +prow res +pra iz +pr ss +pi ents +phel ps +pe gan +pavlovic nbcs +pab owl +p ton +over heads +ou standing +os f +on scene +offici o +occa sion +ober heim +ob cs +nis man +ni sta +nbad raft +nat ics +nanopore conf +mur rey +mu tasa +mu dgal +mt sen +mis represents +mis classification +min oa +mi at +manne h +lun guk +lov ly +let scher +len asia +lad ha +l ff +l ario +kon ichi +kerri son +keep working +k deleon +k cur +juju tsu +joseph jett +jau har +jat inder +jann ation +insinu ation +ingh orse +indiam art +honey pots +healthand fitness +haw era +hare brained +han ge +h tb +great gatsby +gott ardo +goodnightt witter +golf ball +go ias +glow sticks +glow stick +ge etv +gb hour +gar net +g pf +fur ze +fuer zas +fu gee +fri endof +frauen kirche +forec ourts +for tun +fal me +esp guitar +epistemo logical +enumer ate +elast omeric +eh s +ed w +dulwich gallery +dublin horseshow +don lan +digiov anni +deer park +daily bot +cs gazette +cotonde tulear +col ber +clear way +celebration of +cbc sask +caul kins +cartoon hangover +carol vorders +care full +car meli +by ne +buonarro ti +bum stead +bre el +bras stown +brad by +bored oms +blow fly +bloody scotland +blo q +betro thal +beng old +be ara +basil don +barbar acom +ba set +ba his +ato records +at ok +aru sh +ar si +antron brown +aman zo +amalgam ate +alleno very +ali ghting +ale au +al vy +agu std +aerop uer +ae expo +adul tedu +ad ate +acl are +ab illion +ðŁĺįðŁĺŃ ðŁĺįðŁĺŃ +ðŁ¥ĩðŁ¥ĩ ðŁ¥ĩ +îĮ ª +å½ © +à¹ĥ à¸Ī +your vote +yo liverpool +yam l +world cafe +won kyu +women fortrump +wit tes +wilton musichall +whid don +wh oring +ward ine +w bbm +van tho +val verde +val ory +v nu +ut mb +us sey +under brush +tv mohandaspai +trapezo idal +tom az +ther mali +the kitchn +the dilipkumar +the aaf +th are +team bnn +team ashishians +ta kkar +t sin +sub heading +st lucie +spacec am +smo del +sile sian +shawn stockman +shak oor +score board +sara watkins +san siro +sai pem +rt pi +rou shy +repudi ate +remed ying +reli ve +re ik +pic poul +pc s +pan jab +p kushn +or ding +onther ocks +ocv updates +obfusc ated +ob ad +oab aab +no ac +nester ov +nb h +nak amichi +na jim +mickey avalon +melissar auch +me chi +mcla gan +mc girr +magn itudes +ma ws +ma sp +m monogram +lit rpg +leg alizes +lar a +la ppe +kid swb +key shot +kar lan +k lime +juda ic +jenny lewis +jenna elfman +inclu sively +im ls +i set +homony m +ho pley +hir ta +her lings +haynes ville +happy anniversary +guar neri +gu o +gro the +google home +gon i +global health +glade sville +gl c +georgel ucas +foot light +fluff ball +fcn antes +fan army +extrapol ated +exacerb ation +er sclub +emil ym +east sussex +early start +dy dd +du etting +dry ga +dom tar +divul ging +di po +dez ma +desol at +den er +csed week +cor zo +co tulla +clark mp +cher ney +chan del +cant waitfor +ca ha +bu ghead +brush wood +bombay times +blueridge parkway +blu eroom +bikel anes +big bear +bharath iraja +beav en +b fore +awi ki +auto bus +author sofinstagram +athen sga +asi mon +ash rafi +arrabbi ata +ann curry +ambi ga +alkal ine +algorith mically +al si +al dou +afric acom +abre go +abe dian +:) .. +ðŁĴĻðŁĴļ ðŁĴĽðŁĴľ +ðŁĴĥ @ +ðŁijĮðŁijĮ ðŁijĮðŁijĮðŁijĮ +å¯ Į +ãħ ĩ +yy o +y iz +wr ona +world changers +water gate +wal eshour +vla ar +veoli auk +usace hq +us amateur +uru bamba +up welling +ul ts +uefayouth league +tupac shakur +trishap aytas +trio works +travelo gues +trampol ene +tra ister +tor con +toni and +ton der +toiletekprem katha +today is +thec gf +tham esc +tennis australia +tat lı +tart lets +swift current +su te +speed wy +spac eneedle +soci été +snap shot +sme w +shat to +shan atics +senator wong +sd ny +schar lie +sc indi +samgye opsal +sam gyimah +ry usei +ry l +ro seee +rhe o +re organizes +rdan ational +pun ahou +pre fabrication +power glide +por tw +plan ted +pete doherty +pa chan +ou mu +on love +not given +north wich +niantic labs +newworld order +mr josh +mr f +million views +metas ploit +marucci sports +mark burnett +markburnett tv +marie hamn +mar nell +mar lowe +mar ise +malasak it +mal nourishment +mad lang +m wf +ly nah +lot fi +lion smusic +li ths +lewis capaldi +learning analytics +lead o +las dun +l arian +ku ss +ku dat +key ano +ke ely +k ch +jueve s +jet charter +jay mohr +ja eden +istigh far +isles app +ios app +inx ile +intrepid museum +iklan terbaru +hill billy +helic obacter +health data +harro wer +hannah spearritt +gu wop +grow yourown +gratu ito +grape seed +gland ore +gin ato +gho stre +geon osis +fifty shade +fas cic +far lane +extre mely +eliti sts +ed ps +dw l +dungeon family +djima vic +din ghouse +deep spac +de gate +daw nzpost +davidlove photog +dar ner +crump sall +cre gan +cochlear implant +cheru bic +chel ation +chase masterson +ch under +cellu lar +canaryisland sen +bsor ules +bro seley +blk perspectives +behavi orist +barne sy +augh rim +aqu atica +amat ata +am one +allameric agame +________ ________ +! ⾨ +ðŁĻĭâĢįâĻĢï¸ı ðŁĻĭâĢįâĻĤï¸ı +ðŁĸķ ðŁı½ +ðŁijİðŁijİ ðŁijİ +ðŁİĻ ï¸ı@ +ðŁİĨ ðŁİĩ +ì Ł +⾨ âľĶ +âĻ¥ï¸ı ⾨ +âĸº # +à¸Ļภª +ج اÙħ +ö l +z wel +xfactor final +wood bury +wild child +wan tit +wa urn +viol on +ve gam +v nl +uro logy +ur araka +upone aglewings +unc wilmington +unanim ity +ubi que +transgender ism +tipping point +thinking outloud +the church +tech nip +team blackberry +tavis smiley +tam lyn +swart berg +style me +ste ds +ste acher +sp acc +solor zano +sin ghs +side m +sha be +set by +seri ally +sean hayes +satur ates +san wool +sal ar +saints row +ru stin +ru ffing +rock face +road warrior +reprezent radio +reno omokri +reboot liberty +pronovi as +pra kriti +polar ities +pete sessions +perth and +pas sp +oo iman +onemore time +one yearof +okone do +ojama jo +o week +nit zan +ngi reland +negr oni +n fre +mus lera +mu squ +mortg aging +monsta xin +mm wave +mitch y +ming les +mil utta +memphis fb +melissa ordway +may fest +man repeller +m mie +m le +ly sa +legiti mized +la ffs +knowledge management +kiernan shipka +khal is +kawarthan ow +jean ericvergne +jason bourne +jack o +ja ked +ja ima +inf am +in sky +homep ages +ho vered +ho sch +hi the +herto genbosch +he gerty +hall marking +gyor ko +gul ick +gu apa +gre gs +good foryou +gen berger +gandalf wasme +ful bourn +fru gally +fpv racing +food fact +flo y +flo rets +fe dele +el vin +effi ong +een age +eddi reader +eag ar +div vy +distill ates +deb ello +day sss +dani eller +cork chamber +cold water +cla ggett +chick y +ce oil +capability brown +camero onians +california fires +calcu tt +cal dey +brian azzarello +bren er +boys brigade +bmwmotor ra +blen kin +bio compatible +binge watching +bin nington +big bos +ber ating +basal tic +babun aidu +as ph +anthology film +angh el +al cos +ai fam +acro pora +ab berton +ðŁĺįðŁĺį ðŁĺįðŁĺĺ +ðŁĺĤ ðŁ¤Ķ +ìķĦ íĬ¸ +æĭ¡ æķ£ +ÙĦ ÛĮ +ا٠ĩ +yl p +yad u +what sinthe +westmid shour +web socket +voo c +vm ro +viv adelrio +victori ao +veen stra +ve dran +v ch +ul ing +uk business +tur ron +trin ita +times magazine +thibau d +thewrit elist +the face +th st +ter za +team nike +ta share +swan bourne +svt foe +steph on +statueof unity +st nd +speci fier +spagnu olo +so hlhockey +small faces +sin till +shus kies +shoo fly +shakti rajan +shahi dul +sd pd +schul enburg +sch tick +sawal ha +sal soul +sag acity +s vig +royalvisit canada +rox by +roeth ke +reson ances +re eni +ram blin +pwe tty +pri mark +pramo d +polo club +plu ghole +photo chemistry +phillips academy +pete sohlhockey +pe dir +ost friesland +oke chukwu +noynoy aquino +now streaming +nive les +nikon india +neo classicism +negro ponte +ne sd +nbc nightshift +na thu +n tis +n pas +n bb +n acon +my ah +mur ari +mubar ik +mo jis +missk atie +mis wak +mirror sedge +min ow +men jadi +melb derby +masch io +mar ji +mal ine +ma quis +ly onnaise +lier se +late y +large format +kour a +kost ner +king sdale +kick the +ken gen +k bbq +jw j +justicefor benghazi +juse yo +jur nee +jasminec ain +jacqu elin +inthe clouds +id lo +hss vca +honey wood +hockey isforeveryone +he bron +ha seo +h ke +gold wing +gold mines +girl slax +gi ya +garri ga +forest dale +foot action +flash game +fiat chrysler +felipe calderon +facto tum +ew stv +ev as +er kin +emiliop ucci +elock hart +ego yan +ebel ing +e iders +discer ned +demor alize +darting tonhall +damaris cotta +dairy month +cutthroat kitchen +cu bas +correspon ded +cin ar +che ssies +canton io +bowie baysox +blue chip +blair ite +bili moria +be yourbest +bb s +ban karena +ba shaw +armc andy +an chi +amberrudd hr +alex bracing +ab ashi +ðŁļ´ ðŁı¼ +ðŁĺĺ ðŁĺģ +ðŁĺ© ðŁĺŃ +ðŁİ¬ # +ðŁĮį . +ðŁĮ¾ ðŁĮ¾ +ãħ Ĭ +ñ ana +zz ato +zer din +zer be +zan ele +z aco +xxx viii +wych wood +wha a +week endo +we cantwait +viennois erie +vide op +v laminck +uta ware +un enviable +ul le +tran shu +torye lectionfraud +topo logies +tomato e +timy the +the change +ten ali +tech cc +super bock +stra uch +ss mb +sri vastav +spor tawards +sp robz +sof ÃŃa +so hi +slow travel +sic ut +si ring +sea fishing +sc ea +sbut d +sain ttropez +saaksh isra +roust about +roadtrip tv +ro chambeau +rf k +ren a +reggie bush +rege hr +re stuar +rain men +rail budget +proble mas +pon zio +perfume genius +per loff +pap azian +ou tu +ossi pee +or vis +oh hey +o zzy +nv sd +north leach +nfl freeagency +na os +myo c +mur alists +mu scaria +mo ton +mo gens +midnight red +me ins +matt sson +mark field +map info +mang ino +lucre zia +love qub +louisianat ravel +law ther +lat u +lam pson +la ppa +krisa quino +kohi stan +kit i +john nyo +iti sprashanth +hul ley +hedon ic +hass i +hackneye mpire +greens bor +gam po +futureready libs +forti fying +follow in +fle ek +flag ship +fer id +feel like +fashion week +ey b +evel ynn +entertain ingly +embe ddings +dhan u +depor te +day pack +dar rin +d pict +consul ates +conow ingo +chi yoda +cas spi +carbon ic +car ota +call ic +c da +bryo zoan +bo sio +bir dy +bab bu +aye she +av ui +archang el +ar val +aqu igley +ap lic +anti ago +an sen +ak asi +ad owns +ad fw +ac unity +:( "@ +į ° +ðŁļĹ ðŁĴ¨ +ðŁĴģ ðŁı»âĢįâĻĤï¸ı +ìĤ¬ë ¬´ +ìĤ¬ë¬´ ìĹĺ +ãģĬ ãĤģ +à´ ³ +تص ÙħÙĬ +yasuk uni +worl dy +wool ens +wentworth miller +wemy ss +we stri +waf s +vol tige +vo et +vin c +vil de +ve aled +urban wildlife +up f +uni vienna +u kyo +twitter uk +truck fest +tro caire +trail ering +too vey +togar ashi +tn ell +tin h +the jo +the hunter +the heat +tex turi +terra za +ter im +ter batas +telfor dutd +tech nation +te gh +sy ren +sud duth +submer ge +su fis +stre gi +sten zel +stech ford +st assen +splendo red +smar thomes +slumber party +sim sim +shee sha +shant iniketan +sf ballet +semen ov +schalk wyk +say es +saty amev +sap skzn +s woman +rubeng allego +ro iz +reher sal +re analysis +rainor shine +radio aire +qu mran +port lethen +por gie +plough mans +pine iro +pat nap +pap en +palar ong +over up +om achenko +nor gaard +ni dia +new bridge +national ise +mul lum +moscow mitch +mil ko +meri weather +me official +mazz etti +marian rivera +map úa +mand ai +man at +maan karate +ma sya +lore to +lop ate +latelate toyshow +lambda chi +kwa hu +kr n +kim sey +khat ami +judgen ap +joshu as +jami elynn +jami ah +j adel +it matters +ison zo +inter med +inco terms +in mind +i anni +hoo ft +hi way +hay on +hands comb +han ske +half adams +guimar as +gos lin +gian tuk +get tested +gaw k +g ww +g stephanopoulos +foe ticide +fil in +exist ences +excell encies +evam arc +ene spanol +en ext +el tiempo +dra ugr +ditt mar +disco tek +day day +dar a +dan icim +d chat +cow ering +comple tly +color adop +chou sing +chima ira +ch n +cel ite +cas bah +caro emerald +can ta +calm down +buden holzer +brink manship +boywith luv +boh len +blogger suk +bedtime story +basketbal laus +bak ht +b craw +ato gether +asho kk +as cb +art dubai +architec tes +aram m +ar pin +ant middleton +ani poke +andro logy +alexand rina +alex a +ajax capetown +* \(^ +ðŁĺĢ ðŁĺģ +ðŁĺ¿ ðŁĺ¿ +ìĪĺ íĺ¸ +모 모 +âĺºï¸ı ðŁĺĤ +à¸Ńภ¡ +à° ¶ +Ø§Ø · +yun an +worley parsons +workflo whq +witney carson +win z +we ymouth +vu cevic +vou liag +vin expo +vic gov +venkat raman +venezi ano +urgent care +udyo kta +ud yog +twee abondthatcantbebroken +tl aloc +timel ord +ticket webuk +thorn leigh +thor sby +the press +the kevin +tech forum +tau malolo +tara weeh +tan jung +t strong +sy kess +swag gart +sw amy +storybehind myscar +stein haus +sr ini +sphy g +sonamo hapatra +son en +sn ck +sle v +silic osis +sextu ple +sev aks +se infel +sc avenge +s veri +roadsof mumbai +road ton +rho diola +re entering +raymond ville +rang arajan +quir rel +q sfp +preeti karao +polit is +per versity +pat tee +oc ele +oak wood +o scott +ny drock +nature medicine +n ads +memory monday +mak tab +ma thu +lobb an +land bou +lago sians +l day +kra bbe +karan vir +jof frey +joe thomas +intercon hotels +inter web +inform acion +indeli bly +in training +il ho +hydro geology +hun ch +hell and +hahah hahahaha +ha kai +h sia +gre port +girlsin science +gen ga +fo tis +fo sbury +flanne lette +financial domination +festivalof lights +eu ets +emanu elle +eas ilocks +driven by +dream iest +dr seuss +doy ley +dorse techo +don health +dj whookid +disav owed +dic o +desi o +dean karnazes +day music +daw illiams +crowd cube +craig leith +cor bell +constant inos +col ler +co fee +co as +cme group +cho ck +catt ell +cat news +cast ner +card room +carab ins +cali bri +bor sato +ben ighted +bea sting +be yoglu +basti dores +bah man +ba rest +avell aneda +atur days +aspl und +as oe +arquite tura +ard vark +arctic frontiers +anewh ope +ai ku +adjac ency +ad um +academicswith cats +aber daron +aaron yan +ðŁĩ¬ðŁĩ§ @ +éŁ ³ +ãı Į +zor aida +yig it +wes ther +way de +water melon +wac sports +wa fula +villano vau +vicis situ +var nado +vap elove +utel aforever +ur ro +universal ism +un constitutionally +tu fan +tru iden +trij icon +topal ov +tomor ph +tomoh on +tn ght +tit lan +tit anium +thomastur goose +tho ct +this winter +thing for +theruf friderz +the ashes +tattle tale +tarte ist +tam arab +sym pathetically +student problems +stop knifecrime +spo ints +sne ider +singh is +si menon +seman as +scri bble +sc ros +sas a +sar b +sahib zada +sahara occidental +s gc +s des +row let +rc jh +rare breed +ram anan +raising awareness +rain sford +rai sen +radu lov +proc tors +pres spass +pre season +po ori +plu ot +pennstatem hky +pal en +oxfords bs +oun ified +operaholland pk +nt w +nor tec +nikkis anderson +ne xx +nav agio +na xx +na imi +muzi k +mur aco +mu hle +mother nature +moneyinthe bank +mn dnr +mirror monday +mer ville +mcne ice +mayor slyjames +marq ise +mal var +main aure +lux us +luck yday +lu per +loisa andalio +lo si +linux foundation +l ct +ky aaa +kotze bue +kirko bangz +kier sey +keh rer +k gra +john key +jhpie go +jeanne de +jav adekar +janak pur +j ni +iti oned +inv ited +international teaday +ian ziering +i boy +hid aka +heck scher +heather dubrow +hd palooza +hallo way +gu apas +gr hs +gotoireland us +good for +fuel ing +fl ö +ferdin and +fan support +evamarc ille +er ca +emabiggest fan +em f +el bridge +eco ins +dumer vil +duck s +dove awards +double standards +discover greece +dhe er +desig nin +dec ca +dd insi +dab bin +curvil inear +counterfe its +cob hc +coachj franklin +ck w +chandra shekar +cebit aus +cabernet franc +by sea +buy social +bucc al +bu sco +brody jenner +brec bassinger +bou illet +bf hardline +ber halter +bent eng +be your +bas c +bag anac +atten bury +att aboy +as kea +ari ano +ann nnnn +alimi ballard +ad mn +acher on +> ... +" "@ +ðŁĺĺ ðŁĺľ +ðŁĴķðŁĴķðŁĴķðŁĴķ ðŁĴķðŁĴķðŁĴķðŁĴķ +íĺķ ìĽIJ +ë· Ķ +âļª âļ« +ઠľ +zin nias +zale wski +yogag irl +wyr ley +woyz eck +worth morealive +world sight +won u +wil drose +wei hen +vu ko +vendredi lecture +tz on +twinpeak s +tur non +total football +top golf +thur sley +thr on +the pi +thatsmy girl +terrific tuesday +tal uk +symp a +sunny beach +su deley +ste idl +star uk +spi ano +sperkin sdnd +soule ater +sop rou +solfe ge +smo or +sketch noting +sidec inema +shubhan gi +sh indler +sed its +se dat +sd chat +scotti e +sap inker +sab ba +river park +ri mowa +reason ableness +readju sted +ran jana +quer as +py ong +pu ggy +promo products +prayforthe world +phosphat ase +ph v +pau lone +open studio +nr sv +nose k +nis ch +ng wa +new rules +ne scafe +my wolfsong +morphe tt +mon n +mon kee +mini atura +micdro premix +mesop orous +mc pher +mc coo +mbl season +marin ka +mal asada +m do +luni z +lse plc +love youu +lo ew +libret tist +letter sto +lair g +lac on +kou ga +kid shealth +kend riya +kay han +kau shiki +kan de +june jo +ju styn +john inne +johninne scentre +jo bad +jharsu guda +jer ri +jame spear +jal na +infl iction +ine gypt +incarcer ating +im hof +i asb +hel pre +head casey +headcasey mike +hallucino gen +hal lettsville +h ilife +goo din +go sparks +global summit +ger an +freethe arctic +fre aker +for trose +for summer +fe her +fcu trecht +face time +ext gen +er awan +enew man +ellef sondavid +el il +ed ball +duc ted +dr marty +doh ring +dhar ti +des ford +dent in +dau lat +climate breakdown +ci wf +chri sperkinsdnd +chill er +chevening fco +can ids +bro mides +bid dies +bham donkeys +bed in +be idou +be honest +bat en +bare la +bac ademy +b ment +b dl +anastaciam usic +an erjee +am pex +al fas +al ben +ak hir +aed as +adam aofficial +ack in +achievement unlocked +abutil on +abdel rahman +aar sbl +?? "@ +! ðŁĶ¥ðŁĶ¥ðŁĶ¥ +! ðŁijī +! :( +ðŁĺį ðŁĺĻ +ðŁİĬ ðŁİī +ë§ ¤ +ë¦ ´ +åĵ ģ +ze ma +youngen terprise +yaqu b +xavier daniel +wit tic +win st +wag ar +vovin am +vor ov +vijay filmaker +ver ilog +vapor ware +uri m +up ends +tre sham +tre mpe +total y +tor ie +today at +thimble weed +the chase +swit chers +swati jaihind +swap meet +stru ms +stock photos +stin ch +speed prayers +southernmiss fb +so ca +simon webbe +shep newsteam +ser via +sean cardo +sam ish +saakshisra wat +sa otome +rough rider +rond ina +roadand track +righ tist +ric c +renew ui +ren lagann +re treat +rc psychic +rae bareli +py x +pron o +present ando +poo jai +pau gh +pat roberts +ovi de +over development +ometal lic +nfl pabowl +nep tunes +mt dna +mother and +moore senate +mjin nocent +mistre s +mike tirico +med itech +marky ramone +maquilla je +low theband +love tour +lift back +li arin +latt in +labor atorium +ky bourbon +kr ach +ken sit +kazakh stan +kati ed +kaan ur +jones jr +jac kel +it ain +isser lis +interrup tus +international youthday +insan ity +indo chinese +im mbm +ig awa +hydro graphy +hi mitsu +herak les +her se +hend all +he ssel +havil ah +happy jhopeday +ham ina +h wu +h wp +h kt +gy res +guy ra +gun sense +gul fair +greglou ganis +grease paint +ghaz ala +gang neung +gambrin us +foodis medicine +fon ua +fi ii +felice herrig +ero icab +ero ars +er lend +engie group +ell p +eli sir +einstein maga +efra im +earl ham +e peeps +dr marth +do towsky +dis love +dim witted +de juan +daniel s +dak os +d scc +cri an +contr alto +commun is +clergy men +christmas music +chi ded +chest nu +chazz palminteri +cd na +castle martyr +cab by +brighton pride +bm v +big and +bel mon +be oplay +bat sky +bag amoyo +australian shepherd +audio boom +ath boy +as chi +arcadia unified +ap ure +ap ate +anc ities +amand ac +alife dotowsky +al san +ake elah +af sar +adam rank +. ;) +" ¡ +!! ðŁİīðŁİī +ðŁĴļðŁĴļ ðŁĴļðŁĴļðŁĴļ +ðŁİīðŁİĬ ðŁİĪ +ðŁĮ¼ ðŁĮ¼ +ðŁ§Ł âĢįâĻĢï¸ı +íĭ° ìķĦëĿ¼ +éī Ħ +âĿ¤ï¸ı ðŁĴª +ᥠ± +о ÑĤ +your car +york theatre +ye sil +yam ma +ya hog +wt fff +writers festival +worldfood prize +wor dle +we yer +vo xx +vendee globe +var ico +vancity buzz +under development +tran scultural +tor neo +tor mentors +tiltro tor +thisisse th +thisisseth sblog +the bonni +th year +tayl an +tahaw y +sw fclive +super soft +suburban ites +str ator +speaker series +sp sm +sou ad +slo per +sla visa +skipp ered +sketch cards +show pieces +sho wi +shin han +sf hs +seg no +se chs +school meals +schel de +sau rian +sag ami +ry p +rump f +rou lston +ro ffey +riz al +renega des +ren alovelis +regionalli ga +region ale +rb k +ravin dersingh +ra ux +pu plovers +pro scribed +present an +pre heated +pr illy +poul son +po ids +pm dc +pik app +photo video +phi bbs +pc j +par ash +pa tha +p rum +p drm +our community +oph ils +ny rd +nonprofit day +non resident +nin ny +ne ira +nat oli +nalim bachia +my app +muke sham +msccruise susa +morning rush +monagh angaa +mol let +mile posts +mi thu +meaning lessness +mcken ny +mathi sen +masterchef sheep +mar one +mamat abanerjee +ly dden +lon ie +lo bat +lin foot +lgb thistory +lec ats +kilmac ud +kerry katona +ke ens +karol sevilla +kam akshi +jrod fromoz +jose cuervo +jai my +interior designideas +ino ise +inder jit +hon gi +hill erman +hemam alini +hayley mcqueen +hand maids +hal kett +gui z +gross ers +gre el +gib let +gay les +fou sey +fli k +female entrepreneur +feels badman +fanni bal +f staste +erc p +ent el +en in +eli kes +ei ichiro +ed postcards +ec kankar +duc s +dor mammu +domin ik +dogand puplovers +disp leasing +dge l +del fonics +del at +de ase +d pm +cu cks +cronkit enews +congress muktbharat +color smarathi +col avita +clen buter +cipri an +cbc news +carleton college +car nal +black nose +bior xiv +bhav nalimbachia +best t +be chard +bar ma +babs draws +b fest +az atho +avon rep +att ac +asph alt +ap sf +anton etti +annane trebko +ang om +ameth asone +am oner +ah hh +ag proud +affor dance +ad joint +: ~ +//// // +$ ] +! :)) +ðŁĴį ðŁĴİ +ðŁijįðŁı½ ðŁijįðŁı½ +zar da +yuk ito +work athome +women scienceday +wit s +where withal +vic ta +vas elines +v any +utaware ru +us fda +un competitive +un believe +twoo odley +tw ofer +truck driver +tri stin +trans acted +threep wood +the yyam +the shoe +the il +tari kh +tali m +sussex lifemag +sus open +sugi mori +su long +stati stik +sta this +spring side +smo ss +set su +sch aer +sc ana +sat oh +sar la +santa anita +sambu cus +saig on +ru perts +ru ia +roman i +robb ins +real muto +reading festival +read justment +provision als +pro kabaddi +ppl summit +poe mas +pig tail +pdp nigeria +party yyy +ow lets +news sport +neptun o +naray ang +nan tong +nai ad +mur rin +mo scon +ml h +merq ury +mc t +marcus butler +makha chkala +lucky man +loe hr +ll cc +liveyour dream +lin acre +lime stone +lifeof an +li ffs +lesmiz bway +leh ri +learn coding +le user +lan l +lake eri +lado ga +ko chunni +k dd +justicele ague +jen nar +iyanla vanzant +im pi +i stre +i ib +hondac anada +hey ns +har diest +hannah bronfman +h pr +h fc +gw m +guar in +greatbritish menu +ghu ggi +ghost story +gec dsb +ge ats +gaslight anthem +garci aparra +friend sday +fl ye +fis chl +fent anil +fair vote +ery ri +episco po +eng cro +electrocardio gram +ek mainaure +dun ces +duc ators +dub nyk +distru stful +deton ators +deceler ate +debbie reynolds +de mais +day z +d th +cuar ón +cu pich +cros well +courts matter +cool kid +chol ar +cha que +catholic faith +cap olitics +c pride +by land +bran sholme +bla zek +bicy clec +be calmed +bas combe +bae jin +bad on +auri fil +audi op +attenu ated +ar ki +ar dara +anti theism +ames waran +am paio +al isms +ag api +abo g +!!! ?? +! âĿ¤ï¸ıâĿ¤ï¸ıâĿ¤ï¸ı +ðŁĴĻ ðŁĴļðŁĴĽ +ðŁĩ´ ðŁĩ² +ìĨ Ķë +미ë Ĥĺ +é İ +åį ģ +âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶ +z abu +ye young +ye ji +y ma +wild boyz +weald stonefc +vote ariana +ven ne +v bn +ut zon +uise du +ufo table +ufo sighting +uff ington +tu le +tt itz +troph ication +travel awesome +transcendent alism +tra pper +tik va +ti sed +thisis england +teenwolf season +tech support +sundar c +suare zg +su sy +stron gheart +street food +story bots +sto c +st gile +srsg vac +southeast ward +soccer saturday +so zone +smid t +sm city +sli mey +sin claire +sd reader +scare d diff --git a/annotator/oneformer/oneformer/data/bpe_simple_vocab_16e6.txt.gz b/annotator/oneformer/oneformer/data/bpe_simple_vocab_16e6.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..36a15856e00a06a9fbed8cdd34d2393fea4a3113 --- /dev/null +++ b/annotator/oneformer/oneformer/data/bpe_simple_vocab_16e6.txt.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:924691ac288e54409236115652ad4aa250f48203de50a9e4722a6ecd48d6804a +size 1356917 diff --git a/annotator/oneformer/oneformer/data/build.py b/annotator/oneformer/oneformer/data/build.py new file mode 100644 index 0000000000000000000000000000000000000000..2611644589d6a5978c257a4e349a1b466f366c0c --- /dev/null +++ b/annotator/oneformer/oneformer/data/build.py @@ -0,0 +1,117 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from typing import Any, Callable, Dict, List, Optional, Union +import torch.utils.data as torchdata + +from annotator.oneformer.detectron2.config import configurable + + +from annotator.oneformer.detectron2.data.common import DatasetFromList, MapDataset +from annotator.oneformer.detectron2.data.dataset_mapper import DatasetMapper +from annotator.oneformer.detectron2.data.samplers import ( + InferenceSampler, +) +from annotator.oneformer.detectron2.data.build import ( + get_detection_dataset_dicts, + trivial_batch_collator +) +""" +This file contains the default logic to build a dataloader for training or testing. +""" + +__all__ = [ + "build_detection_test_loader", +] + + +def _test_loader_from_config(cfg, dataset_name, mapper=None): + """ + Uses the given `dataset_name` argument (instead of the names in cfg), because the + standard practice is to evaluate each test set individually (not combining them). + """ + if isinstance(dataset_name, str): + dataset_name = [dataset_name] + + dataset = get_detection_dataset_dicts( + dataset_name, + filter_empty=False, + proposal_files=[ + cfg.DATASETS.PROPOSAL_FILES_TEST[list(cfg.DATASETS.TEST).index(x)] for x in dataset_name + ] + if cfg.MODEL.LOAD_PROPOSALS + else None, + ) + if mapper is None: + mapper = DatasetMapper(cfg, False) + return { + "dataset": dataset, + "mapper": mapper, + "num_workers": cfg.DATALOADER.NUM_WORKERS, + "sampler": InferenceSampler(len(dataset)) + if not isinstance(dataset, torchdata.IterableDataset) + else None, + } + + +@configurable(from_config=_test_loader_from_config) +def build_detection_test_loader( + dataset: Union[List[Any], torchdata.Dataset], + *, + mapper: Callable[[Dict[str, Any]], Any], + sampler: Optional[torchdata.Sampler] = None, + batch_size: int = 1, + num_workers: int = 0, + collate_fn: Optional[Callable[[List[Any]], Any]] = None, +) -> torchdata.DataLoader: + """ + Similar to `build_detection_train_loader`, with default batch size = 1, + and sampler = :class:`InferenceSampler`. This sampler coordinates all workers + to produce the exact set of all samples. + + Args: + dataset: a list of dataset dicts, + or a pytorch dataset (either map-style or iterable). They can be obtained + by using :func:`DatasetCatalog.get` or :func:`get_detection_dataset_dicts`. + mapper: a callable which takes a sample (dict) from dataset + and returns the format to be consumed by the model. + When using cfg, the default choice is ``DatasetMapper(cfg, is_train=False)``. + sampler: a sampler that produces + indices to be applied on ``dataset``. Default to :class:`InferenceSampler`, + which splits the dataset across all workers. Sampler must be None + if `dataset` is iterable. + batch_size: the batch size of the data loader to be created. + Default to 1 image per worker since this is the standard when reporting + inference time in papers. + num_workers: number of parallel data loading workers + collate_fn: same as the argument of `torch.utils.data.DataLoader`. + Defaults to do no collation and return a list of data. + + Returns: + DataLoader: a torch DataLoader, that loads the given detection + dataset, with test-time transformation and batching. + + Examples: + :: + data_loader = build_detection_test_loader( + DatasetRegistry.get("my_test"), + mapper=DatasetMapper(...)) + + # or, instantiate with a CfgNode: + data_loader = build_detection_test_loader(cfg, "my_test") + """ + if isinstance(dataset, list): + dataset = DatasetFromList(dataset, copy=False) + if mapper is not None: + dataset = MapDataset(dataset, mapper) + if isinstance(dataset, torchdata.IterableDataset): + assert sampler is None, "sampler must be None if dataset is IterableDataset" + else: + if sampler is None: + sampler = InferenceSampler(len(dataset)) + return torchdata.DataLoader( + dataset, + batch_size=batch_size, + sampler=sampler, + drop_last=False, + num_workers=num_workers, + collate_fn=trivial_batch_collator if collate_fn is None else collate_fn, + ) \ No newline at end of file diff --git a/annotator/oneformer/oneformer/data/dataset_mappers/__init__.py b/annotator/oneformer/oneformer/data/dataset_mappers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/annotator/oneformer/oneformer/data/dataset_mappers/__init__.py @@ -0,0 +1 @@ + diff --git a/annotator/oneformer/oneformer/data/dataset_mappers/coco_unified_new_baseline_dataset_mapper.py b/annotator/oneformer/oneformer/data/dataset_mappers/coco_unified_new_baseline_dataset_mapper.py new file mode 100644 index 0000000000000000000000000000000000000000..25a460bf73e0417916d2e09e2edc1f975155024c --- /dev/null +++ b/annotator/oneformer/oneformer/data/dataset_mappers/coco_unified_new_baseline_dataset_mapper.py @@ -0,0 +1,341 @@ +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/Mask2Former/blob/main/mask2former/data/dataset_mappers/coco_panoptic_new_baseline_dataset_mapper.py +# Modified by Jitesh Jain (https://github.com/praeclarumjj3) +# ------------------------------------------------------------------------------ + +import copy +import logging + +import numpy as np +import torch + +from annotator.oneformer.detectron2.data import MetadataCatalog +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.data import detection_utils as utils +from annotator.oneformer.detectron2.data import transforms as T +from annotator.oneformer.detectron2.structures import BitMasks, Instances +from annotator.oneformer.oneformer.utils.box_ops import masks_to_boxes +from annotator.oneformer.oneformer.data.tokenizer import SimpleTokenizer, Tokenize + +__all__ = ["COCOUnifiedNewBaselineDatasetMapper"] + + +def build_transform_gen(cfg, is_train): + """ + Create a list of default :class:`Augmentation` from config. + Now it includes resizing and flipping. + Returns: + list[Augmentation] + """ + assert is_train, "Only support training augmentation" + image_size = cfg.INPUT.IMAGE_SIZE + min_scale = cfg.INPUT.MIN_SCALE + max_scale = cfg.INPUT.MAX_SCALE + + augmentation = [] + + if cfg.INPUT.RANDOM_FLIP != "none": + augmentation.append( + T.RandomFlip( + horizontal=cfg.INPUT.RANDOM_FLIP == "horizontal", + vertical=cfg.INPUT.RANDOM_FLIP == "vertical", + ) + ) + + augmentation.extend([ + T.ResizeScale( + min_scale=min_scale, max_scale=max_scale, target_height=image_size, target_width=image_size + ), + T.FixedSizeCrop(crop_size=(image_size, image_size)), + ]) + + return augmentation + + +# This is specifically designed for the COCO dataset. +class COCOUnifiedNewBaselineDatasetMapper: + """ + A callable which takes a dataset dict in Detectron2 Dataset format, + and map it into a format used by OneFormer. + + This dataset mapper applies the same transformation as DETR for COCO panoptic segmentation. + + The callable currently does the following: + + 1. Read the image from "file_name" + 2. Applies geometric transforms to the image and annotation + 3. Find and applies suitable cropping to the image and annotation + 4. Prepare image and annotation to Tensors + """ + + @configurable + def __init__( + self, + is_train=True, + *, + num_queries, + tfm_gens, + meta, + image_format, + max_seq_len, + task_seq_len, + semantic_prob, + instance_prob, + ): + """ + NOTE: this interface is experimental. + Args: + is_train: for training or inference + augmentations: a list of augmentations or deterministic transforms to apply + crop_gen: crop augmentation + tfm_gens: data augmentation + image_format: an image format supported by :func:`detection_utils.read_image`. + """ + self.tfm_gens = tfm_gens + logging.getLogger(__name__).info( + "[COCOUnifiedNewBaselineDatasetMapper] Full TransformGens used in training: {}".format( + str(self.tfm_gens) + ) + ) + + self.img_format = image_format + self.is_train = is_train + self.meta = meta + self.ignore_label = self.meta.ignore_label + self.num_queries = num_queries + + self.things = [] + for k,v in self.meta.thing_dataset_id_to_contiguous_id.items(): + self.things.append(v) + self.class_names = self.meta.stuff_classes + self.text_tokenizer = Tokenize(SimpleTokenizer(), max_seq_len=max_seq_len) + self.task_tokenizer = Tokenize(SimpleTokenizer(), max_seq_len=task_seq_len) + self.semantic_prob = semantic_prob + self.instance_prob = instance_prob + + @classmethod + def from_config(cls, cfg, is_train=True): + # Build augmentation + tfm_gens = build_transform_gen(cfg, is_train) + dataset_names = cfg.DATASETS.TRAIN + meta = MetadataCatalog.get(dataset_names[0]) + + ret = { + "is_train": is_train, + "meta": meta, + "tfm_gens": tfm_gens, + "image_format": cfg.INPUT.FORMAT, + "num_queries": cfg.MODEL.ONE_FORMER.NUM_OBJECT_QUERIES - cfg.MODEL.TEXT_ENCODER.N_CTX, + "task_seq_len": cfg.INPUT.TASK_SEQ_LEN, + "max_seq_len": cfg.INPUT.MAX_SEQ_LEN, + "semantic_prob": cfg.INPUT.TASK_PROB.SEMANTIC, + "instance_prob": cfg.INPUT.TASK_PROB.INSTANCE, + } + return ret + + def _get_semantic_dict(self, pan_seg_gt, image_shape, segments_info, num_class_obj): + instances = Instances(image_shape) + + classes = [] + texts = ["a semantic photo"] * self.num_queries + masks = [] + label = np.ones_like(pan_seg_gt) * self.ignore_label + + for segment_info in segments_info: + class_id = segment_info["category_id"] + if not segment_info["iscrowd"]: + mask = pan_seg_gt == segment_info["id"] + if not np.all(mask == False): + if class_id not in classes: + cls_name = self.class_names[class_id] + classes.append(class_id) + masks.append(mask) + num_class_obj[cls_name] += 1 + else: + idx = classes.index(class_id) + masks[idx] += mask + masks[idx] = np.clip(masks[idx], 0, 1).astype(np.bool) + label[mask] = class_id + + num = 0 + for i, cls_name in enumerate(self.class_names): + if num_class_obj[cls_name] > 0: + for _ in range(num_class_obj[cls_name]): + if num >= len(texts): + break + texts[num] = f"a photo with a {cls_name}" + num += 1 + + classes = np.array(classes) + instances.gt_classes = torch.tensor(classes, dtype=torch.int64) + if len(masks) == 0: + # Some image does not have annotation (all ignored) + instances.gt_masks = torch.zeros((0, pan_seg_gt.shape[-2], pan_seg_gt.shape[-1])) + instances.gt_bboxes = torch.zeros((0, 4)) + else: + masks = BitMasks( + torch.stack([torch.from_numpy(np.ascontiguousarray(x.copy())) for x in masks]) + ) + instances.gt_masks = masks.tensor + # Placeholder bounding boxes for stuff regions. Note that these are not used during training. + instances.gt_bboxes = torch.stack([torch.tensor([0., 0., 1., 1.])] * instances.gt_masks.shape[0]) + return instances, texts, label + + def _get_instance_dict(self, pan_seg_gt, image_shape, segments_info, num_class_obj): + instances = Instances(image_shape) + + classes = [] + texts = ["an instance photo"] * self.num_queries + masks = [] + label = np.ones_like(pan_seg_gt) * self.ignore_label + + for segment_info in segments_info: + class_id = segment_info["category_id"] + if class_id in self.things: + if not segment_info["iscrowd"]: + mask = pan_seg_gt == segment_info["id"] + if not np.all(mask == False): + cls_name = self.class_names[class_id] + classes.append(class_id) + masks.append(mask) + num_class_obj[cls_name] += 1 + label[mask] = class_id + + num = 0 + for i, cls_name in enumerate(self.class_names): + if num_class_obj[cls_name] > 0: + for _ in range(num_class_obj[cls_name]): + if num >= len(texts): + break + texts[num] = f"a photo with a {cls_name}" + num += 1 + + classes = np.array(classes) + instances.gt_classes = torch.tensor(classes, dtype=torch.int64) + if len(masks) == 0: + # Some image does not have annotation (all ignored) + instances.gt_masks = torch.zeros((0, pan_seg_gt.shape[-2], pan_seg_gt.shape[-1])) + instances.gt_bboxes = torch.zeros((0, 4)) + else: + masks = BitMasks( + torch.stack([torch.from_numpy(np.ascontiguousarray(x.copy())) for x in masks]) + ) + instances.gt_masks = masks.tensor + instances.gt_bboxes = masks_to_boxes(instances.gt_masks) + return instances, texts, label + + def _get_panoptic_dict(self, pan_seg_gt, image_shape, segments_info, num_class_obj): + instances = Instances(image_shape) + + classes = [] + texts = ["a panoptic photo"] * self.num_queries + masks = [] + label = np.ones_like(pan_seg_gt) * self.ignore_label + + for segment_info in segments_info: + class_id = segment_info["category_id"] + if not segment_info["iscrowd"]: + mask = pan_seg_gt == segment_info["id"] + if not np.all(mask == False): + cls_name = self.class_names[class_id] + classes.append(class_id) + masks.append(mask) + num_class_obj[cls_name] += 1 + label[mask] = class_id + + num = 0 + for i, cls_name in enumerate(self.class_names): + if num_class_obj[cls_name] > 0: + for _ in range(num_class_obj[cls_name]): + if num >= len(texts): + break + texts[num] = f"a photo with a {cls_name}" + num += 1 + + classes = np.array(classes) + instances.gt_classes = torch.tensor(classes, dtype=torch.int64) + if len(masks) == 0: + # Some image does not have annotation (all ignored) + instances.gt_masks = torch.zeros((0, pan_seg_gt.shape[-2], pan_seg_gt.shape[-1])) + instances.gt_bboxes = torch.zeros((0, 4)) + else: + masks = BitMasks( + torch.stack([torch.from_numpy(np.ascontiguousarray(x.copy())) for x in masks]) + ) + instances.gt_masks = masks.tensor + instances.gt_bboxes = masks_to_boxes(instances.gt_masks) + for i in range(instances.gt_classes.shape[0]): + # Placeholder bounding boxes for stuff regions. Note that these are not used during training. + if instances.gt_classes[i].item() not in self.things: + instances.gt_bboxes[i] = torch.tensor([0., 0., 1., 1.]) + return instances, texts, label + + def __call__(self, dataset_dict): + """ + Args: + dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. + + Returns: + dict: a format that builtin models in detectron2 accept + """ + dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below + image = utils.read_image(dataset_dict["file_name"], format=self.img_format) + utils.check_image_size(dataset_dict, image) + + image, transforms = T.apply_transform_gens(self.tfm_gens, image) + image_shape = image.shape[:2] # h, w + + # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, + # but not efficient on large generic data structures due to the use of pickle & mp.Queue. + # Therefore it's important to use torch.Tensor. + dataset_dict["image"] = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1))) + + if not self.is_train: + # USER: Modify this if you want to keep them for some reason. + dataset_dict.pop("annotations", None) + return dataset_dict + + # semantic segmentation + if "sem_seg_file_name" in dataset_dict: + # PyTorch transformation not implemented for uint16, so converting it to double first + sem_seg_gt = utils.read_image(dataset_dict.pop("sem_seg_file_name")).astype("double") + sem_seg_gt = transforms.apply_segmentation(sem_seg_gt) + else: + sem_seg_gt = None + + if "pan_seg_file_name" in dataset_dict: + pan_seg_gt = utils.read_image(dataset_dict.pop("pan_seg_file_name"), "RGB") + segments_info = dataset_dict["segments_info"] + + # apply the same transformation to panoptic segmentation + pan_seg_gt = transforms.apply_segmentation(pan_seg_gt) + + from panopticapi.utils import rgb2id + pan_seg_gt = rgb2id(pan_seg_gt) + + prob_task = np.random.uniform(0,1.) + + num_class_obj = {} + + for name in self.class_names: + num_class_obj[name] = 0 + + if prob_task < self.semantic_prob: + task = "The task is semantic" + instances, text, sem_seg = self._get_semantic_dict(pan_seg_gt, image_shape, segments_info, num_class_obj) + elif prob_task < self.instance_prob: + task = "The task is instance" + instances, text, sem_seg = self._get_instance_dict(pan_seg_gt, image_shape, segments_info, num_class_obj) + else: + task = "The task is panoptic" + instances, text, sem_seg = self._get_panoptic_dict(pan_seg_gt, image_shape, segments_info, num_class_obj) + + + dataset_dict["sem_seg"] = torch.from_numpy(sem_seg).long() + dataset_dict["instances"] = instances + dataset_dict["orig_shape"] = image_shape + dataset_dict["task"] = task + dataset_dict["text"] = text + dataset_dict["thing_ids"] = self.things + + return dataset_dict diff --git a/annotator/oneformer/oneformer/data/dataset_mappers/dataset_mapper.py b/annotator/oneformer/oneformer/data/dataset_mappers/dataset_mapper.py new file mode 100644 index 0000000000000000000000000000000000000000..710c81bee298e9e6b21a93742d09e720024ceeff --- /dev/null +++ b/annotator/oneformer/oneformer/data/dataset_mappers/dataset_mapper.py @@ -0,0 +1,203 @@ +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/detectron2/blob/main/detectron2/data/dataset_mapper.py +# Modified by Jitesh Jain (https://github.com/praeclarumjj3) +# ------------------------------------------------------------------------------ + +import copy +import logging +import numpy as np +from typing import List, Optional, Union +import torch + +from annotator.oneformer.detectron2.config import configurable + +from annotator.oneformer.detectron2.data import detection_utils as utils +from annotator.oneformer.detectron2.data import transforms as T +from annotator.oneformer.oneformer.data.tokenizer import SimpleTokenizer, Tokenize + +__all__ = ["DatasetMapper"] + + +class DatasetMapper: + """ + A callable which takes a dataset dict in Detectron2 Dataset format, + and map it into a format used by the model. + + This is the default callable to be used to map your dataset dict into training data. + You may need to follow it to implement your own one for customized logic, + such as a different way to read or transform images. + See :doc:`/tutorials/data_loading` for details. + + The callable currently does the following: + + 1. Read the image from "file_name" + 2. Applies cropping/geometric transforms to the image and annotations + 3. Prepare data and annotations to Tensor and :class:`Instances` + """ + + @configurable + def __init__( + self, + is_train: bool, + *, + augmentations: List[Union[T.Augmentation, T.Transform]], + image_format: str, + task_seq_len: int, + task: str = "panoptic", + use_instance_mask: bool = False, + use_keypoint: bool = False, + instance_mask_format: str = "polygon", + keypoint_hflip_indices: Optional[np.ndarray] = None, + precomputed_proposal_topk: Optional[int] = None, + recompute_boxes: bool = False, + ): + """ + NOTE: this interface is experimental. + + Args: + is_train: whether it's used in training or inference + augmentations: a list of augmentations or deterministic transforms to apply + image_format: an image format supported by :func:`detection_utils.read_image`. + use_instance_mask: whether to process instance segmentation annotations, if available + use_keypoint: whether to process keypoint annotations if available + instance_mask_format: one of "polygon" or "bitmask". Process instance segmentation + masks into this format. + keypoint_hflip_indices: see :func:`detection_utils.create_keypoint_hflip_indices` + precomputed_proposal_topk: if given, will load pre-computed + proposals from dataset_dict and keep the top k proposals for each image. + recompute_boxes: whether to overwrite bounding box annotations + by computing tight bounding boxes from instance mask annotations. + """ + if recompute_boxes: + assert use_instance_mask, "recompute_boxes requires instance masks" + # fmt: off + self.is_train = is_train + self.augmentations = T.AugmentationList(augmentations) + self.image_format = image_format + self.use_instance_mask = use_instance_mask + self.instance_mask_format = instance_mask_format + self.use_keypoint = use_keypoint + self.keypoint_hflip_indices = keypoint_hflip_indices + self.proposal_topk = precomputed_proposal_topk + self.recompute_boxes = recompute_boxes + self.task_tokenizer = Tokenize(SimpleTokenizer(), max_seq_len=task_seq_len) + self.task = task + assert self.task in ["panoptic", "semantic", "instance"] + + # fmt: on + logger = logging.getLogger(__name__) + mode = "training" if is_train else "inference" + logger.info(f"[DatasetMapper] Augmentations used in {mode}: {augmentations}") + + @classmethod + def from_config(cls, cfg, is_train: bool = True): + augs = utils.build_augmentation(cfg, is_train) + if cfg.INPUT.CROP.ENABLED and is_train: + augs.insert(0, T.RandomCrop(cfg.INPUT.CROP.TYPE, cfg.INPUT.CROP.SIZE)) + recompute_boxes = cfg.MODEL.MASK_ON + else: + recompute_boxes = False + + ret = { + "is_train": is_train, + "augmentations": augs, + "image_format": cfg.INPUT.FORMAT, + "use_instance_mask": cfg.MODEL.MASK_ON, + "instance_mask_format": cfg.INPUT.MASK_FORMAT, + "use_keypoint": cfg.MODEL.KEYPOINT_ON, + "task_seq_len": cfg.INPUT.TASK_SEQ_LEN, + "recompute_boxes": recompute_boxes, + "task": cfg.MODEL.TEST.TASK, + } + + if cfg.MODEL.KEYPOINT_ON: + ret["keypoint_hflip_indices"] = utils.create_keypoint_hflip_indices(cfg.DATASETS.TRAIN) + + if cfg.MODEL.LOAD_PROPOSALS: + ret["precomputed_proposal_topk"] = ( + cfg.DATASETS.PRECOMPUTED_PROPOSAL_TOPK_TRAIN + if is_train + else cfg.DATASETS.PRECOMPUTED_PROPOSAL_TOPK_TEST + ) + return ret + + def _transform_annotations(self, dataset_dict, transforms, image_shape): + # USER: Modify this if you want to keep them for some reason. + for anno in dataset_dict["annotations"]: + if not self.use_instance_mask: + anno.pop("segmentation", None) + if not self.use_keypoint: + anno.pop("keypoints", None) + + # USER: Implement additional transformations if you have other types of data + annos = [ + utils.transform_instance_annotations( + obj, transforms, image_shape, keypoint_hflip_indices=self.keypoint_hflip_indices + ) + for obj in dataset_dict.pop("annotations") + if obj.get("iscrowd", 0) == 0 + ] + instances = utils.annotations_to_instances( + annos, image_shape, mask_format=self.instance_mask_format + ) + + # After transforms such as cropping are applied, the bounding box may no longer + # tightly bound the object. As an example, imagine a triangle object + # [(0,0), (2,0), (0,2)] cropped by a box [(1,0),(2,2)] (XYXY format). The tight + # bounding box of the cropped triangle should be [(1,0),(2,1)], which is not equal to + # the intersection of original bounding box and the cropping box. + if self.recompute_boxes: + instances.gt_boxes = instances.gt_masks.get_bounding_boxes() + dataset_dict["instances"] = utils.filter_empty_instances(instances) + + def __call__(self, dataset_dict): + """ + Args: + dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. + + Returns: + dict: a format that builtin models in detectron2 accept + """ + dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below + # USER: Write your own image loading if it's not from a file + image = utils.read_image(dataset_dict["file_name"], format=self.image_format) + utils.check_image_size(dataset_dict, image) + + task = f"The task is {self.task}" + dataset_dict["task"] = task + + # USER: Remove if you don't do semantic/panoptic segmentation. + if "sem_seg_file_name" in dataset_dict: + sem_seg_gt = utils.read_image(dataset_dict.pop("sem_seg_file_name"), "L").squeeze(2) + else: + sem_seg_gt = None + + aug_input = T.AugInput(image, sem_seg=sem_seg_gt) + transforms = self.augmentations(aug_input) + image, sem_seg_gt = aug_input.image, aug_input.sem_seg + + image_shape = image.shape[:2] # h, w + # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, + # but not efficient on large generic data structures due to the use of pickle & mp.Queue. + # Therefore it's important to use torch.Tensor. + dataset_dict["image"] = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1))) + if sem_seg_gt is not None: + dataset_dict["sem_seg"] = torch.as_tensor(sem_seg_gt.astype("long")) + + # USER: Remove if you don't use pre-computed proposals. + # Most users would not need this feature. + if self.proposal_topk is not None: + utils.transform_proposals( + dataset_dict, image_shape, transforms, proposal_topk=self.proposal_topk + ) + + if not self.is_train: + # USER: Modify this if you want to keep them for some reason. + dataset_dict.pop("annotations", None) + dataset_dict.pop("sem_seg_file_name", None) + return dataset_dict + + if "annotations" in dataset_dict: + self._transform_annotations(dataset_dict, transforms, image_shape) + + return dataset_dict \ No newline at end of file diff --git a/annotator/oneformer/oneformer/data/dataset_mappers/oneformer_unified_dataset_mapper.py b/annotator/oneformer/oneformer/data/dataset_mappers/oneformer_unified_dataset_mapper.py new file mode 100644 index 0000000000000000000000000000000000000000..e5dadbc2e4eb1e5f06e2294bccb23057dcfdf09d --- /dev/null +++ b/annotator/oneformer/oneformer/data/dataset_mappers/oneformer_unified_dataset_mapper.py @@ -0,0 +1,375 @@ +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/Mask2Former/blob/main/mask2former/data/dataset_mappers/mask_former_panoptic_dataset_mapper.py +# Modified by Jitesh Jain (https://github.com/praeclarumjj3) +# ------------------------------------------------------------------------------ + +import copy +import logging +import os + +import numpy as np +import torch +from torch.nn import functional as F + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.data import detection_utils as utils +from annotator.oneformer.detectron2.data import transforms as T +from annotator.oneformer.detectron2.structures import BitMasks, Instances +from annotator.oneformer.detectron2.data import MetadataCatalog +from annotator.oneformer.detectron2.projects.point_rend import ColorAugSSDTransform +from annotator.oneformer.oneformer.utils.box_ops import masks_to_boxes +from annotator.oneformer.oneformer.data.tokenizer import SimpleTokenizer, Tokenize + +__all__ = ["OneFormerUnifiedDatasetMapper"] + + +class OneFormerUnifiedDatasetMapper: + """ + A callable which takes a dataset dict in Detectron2 Dataset format, + and map it into a format used by OneFormer for universal segmentation. + + The callable currently does the following: + + 1. Read the image from "file_name" + 2. Applies geometric transforms to the image and annotation + 3. Find and applies suitable cropping to the image and annotation + 4. Prepare image and annotation to Tensors + """ + + @configurable + def __init__( + self, + is_train=True, + *, + name, + num_queries, + meta, + augmentations, + image_format, + ignore_label, + size_divisibility, + task_seq_len, + max_seq_len, + semantic_prob, + instance_prob, + ): + """ + NOTE: this interface is experimental. + Args: + is_train: for training or inference + augmentations: a list of augmentations or deterministic transforms to apply + image_format: an image format supported by :func:`detection_utils.read_image`. + ignore_label: the label that is ignored to evaluation + size_divisibility: pad image size to be divisible by this value + """ + self.is_train = is_train + self.meta = meta + self.name = name + self.tfm_gens = augmentations + self.img_format = image_format + self.ignore_label = ignore_label + self.size_divisibility = size_divisibility + self.num_queries = num_queries + + logger = logging.getLogger(__name__) + mode = "training" if is_train else "inference" + logger.info(f"[{self.__class__.__name__}] Augmentations used in {mode}: {augmentations}") + + self.things = [] + for k,v in self.meta.thing_dataset_id_to_contiguous_id.items(): + self.things.append(v) + self.class_names = self.meta.stuff_classes + self.text_tokenizer = Tokenize(SimpleTokenizer(), max_seq_len=max_seq_len) + self.task_tokenizer = Tokenize(SimpleTokenizer(), max_seq_len=task_seq_len) + self.semantic_prob = semantic_prob + self.instance_prob = instance_prob + + @classmethod + def from_config(cls, cfg, is_train=True): + # Build augmentation + augs = [ + T.ResizeShortestEdge( + cfg.INPUT.MIN_SIZE_TRAIN, + cfg.INPUT.MAX_SIZE_TRAIN, + cfg.INPUT.MIN_SIZE_TRAIN_SAMPLING, + ) + ] + if cfg.INPUT.CROP.ENABLED: + augs.append( + T.RandomCrop_CategoryAreaConstraint( + cfg.INPUT.CROP.TYPE, + cfg.INPUT.CROP.SIZE, + cfg.INPUT.CROP.SINGLE_CATEGORY_MAX_AREA, + cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE, + ) + ) + if cfg.INPUT.COLOR_AUG_SSD: + augs.append(ColorAugSSDTransform(img_format=cfg.INPUT.FORMAT)) + augs.append(T.RandomFlip()) + + # Assume always applies to the training set. + dataset_names = cfg.DATASETS.TRAIN + meta = MetadataCatalog.get(dataset_names[0]) + ignore_label = meta.ignore_label + + ret = { + "is_train": is_train, + "meta": meta, + "name": dataset_names[0], + "num_queries": cfg.MODEL.ONE_FORMER.NUM_OBJECT_QUERIES - cfg.MODEL.TEXT_ENCODER.N_CTX, + "task_seq_len": cfg.INPUT.TASK_SEQ_LEN, + "max_seq_len": cfg.INPUT.MAX_SEQ_LEN, + "augmentations": augs, + "image_format": cfg.INPUT.FORMAT, + "ignore_label": ignore_label, + "size_divisibility": cfg.INPUT.SIZE_DIVISIBILITY, + "semantic_prob": cfg.INPUT.TASK_PROB.SEMANTIC, + "instance_prob": cfg.INPUT.TASK_PROB.INSTANCE, + } + return ret + + def _get_semantic_dict(self, pan_seg_gt, image_shape, segments_info, num_class_obj): + pan_seg_gt = pan_seg_gt.numpy() + instances = Instances(image_shape) + + classes = [] + texts = ["a semantic photo"] * self.num_queries + masks = [] + label = np.ones_like(pan_seg_gt) * self.ignore_label + + for segment_info in segments_info: + class_id = segment_info["category_id"] + if not segment_info["iscrowd"]: + mask = pan_seg_gt == segment_info["id"] + if not np.all(mask == False): + if class_id not in classes: + cls_name = self.class_names[class_id] + classes.append(class_id) + masks.append(mask) + num_class_obj[cls_name] += 1 + else: + idx = classes.index(class_id) + masks[idx] += mask + masks[idx] = np.clip(masks[idx], 0, 1).astype(np.bool) + label[mask] = class_id + + num = 0 + for i, cls_name in enumerate(self.class_names): + if num_class_obj[cls_name] > 0: + for _ in range(num_class_obj[cls_name]): + if num >= len(texts): + break + texts[num] = f"a photo with a {cls_name}" + num += 1 + + classes = np.array(classes) + instances.gt_classes = torch.tensor(classes, dtype=torch.int64) + if len(masks) == 0: + # Some image does not have annotation (all ignored) + instances.gt_masks = torch.zeros((0, pan_seg_gt.shape[-2], pan_seg_gt.shape[-1])) + instances.gt_bboxes = torch.zeros((0, 4)) + else: + masks = BitMasks( + torch.stack([torch.from_numpy(np.ascontiguousarray(x.copy())) for x in masks]) + ) + instances.gt_masks = masks.tensor + # Placeholder bounding boxes for stuff regions. Note that these are not used during training. + instances.gt_bboxes = torch.stack([torch.tensor([0., 0., 1., 1.])] * instances.gt_masks.shape[0]) + return instances, texts, label + + def _get_instance_dict(self, pan_seg_gt, image_shape, segments_info, num_class_obj): + pan_seg_gt = pan_seg_gt.numpy() + instances = Instances(image_shape) + + classes = [] + texts = ["an instance photo"] * self.num_queries + masks = [] + label = np.ones_like(pan_seg_gt) * self.ignore_label + + for segment_info in segments_info: + class_id = segment_info["category_id"] + if class_id in self.things: + if not segment_info["iscrowd"]: + mask = pan_seg_gt == segment_info["id"] + if not np.all(mask == False): + cls_name = self.class_names[class_id] + classes.append(class_id) + masks.append(mask) + num_class_obj[cls_name] += 1 + label[mask] = class_id + + num = 0 + for i, cls_name in enumerate(self.class_names): + if num_class_obj[cls_name] > 0: + for _ in range(num_class_obj[cls_name]): + if num >= len(texts): + break + texts[num] = f"a photo with a {cls_name}" + num += 1 + + classes = np.array(classes) + instances.gt_classes = torch.tensor(classes, dtype=torch.int64) + if len(masks) == 0: + # Some image does not have annotation (all ignored) + instances.gt_masks = torch.zeros((0, pan_seg_gt.shape[-2], pan_seg_gt.shape[-1])) + instances.gt_bboxes = torch.zeros((0, 4)) + else: + masks = BitMasks( + torch.stack([torch.from_numpy(np.ascontiguousarray(x.copy())) for x in masks]) + ) + instances.gt_masks = masks.tensor + instances.gt_bboxes = masks_to_boxes(instances.gt_masks) + return instances, texts, label + + def _get_panoptic_dict(self, pan_seg_gt, image_shape, segments_info, num_class_obj): + pan_seg_gt = pan_seg_gt.numpy() + instances = Instances(image_shape) + + classes = [] + texts = ["a panoptic photo"] * self.num_queries + masks = [] + label = np.ones_like(pan_seg_gt) * self.ignore_label + + for segment_info in segments_info: + class_id = segment_info["category_id"] + if not segment_info["iscrowd"]: + mask = pan_seg_gt == segment_info["id"] + if not np.all(mask == False): + cls_name = self.class_names[class_id] + classes.append(class_id) + masks.append(mask) + num_class_obj[cls_name] += 1 + label[mask] = class_id + + num = 0 + for i, cls_name in enumerate(self.class_names): + if num_class_obj[cls_name] > 0: + for _ in range(num_class_obj[cls_name]): + if num >= len(texts): + break + texts[num] = f"a photo with a {cls_name}" + num += 1 + + classes = np.array(classes) + instances.gt_classes = torch.tensor(classes, dtype=torch.int64) + if len(masks) == 0: + # Some image does not have annotation (all ignored) + instances.gt_masks = torch.zeros((0, pan_seg_gt.shape[-2], pan_seg_gt.shape[-1])) + instances.gt_bboxes = torch.zeros((0, 4)) + else: + masks = BitMasks( + torch.stack([torch.from_numpy(np.ascontiguousarray(x.copy())) for x in masks]) + ) + instances.gt_masks = masks.tensor + instances.gt_bboxes = masks_to_boxes(instances.gt_masks) + for i in range(instances.gt_classes.shape[0]): + # Placeholder bounding boxes for stuff regions. Note that these are not used during training. + if instances.gt_classes[i].item() not in self.things: + instances.gt_bboxes[i] = torch.tensor([0., 0., 1., 1.]) + return instances, texts, label + + def __call__(self, dataset_dict): + """ + Args: + dataset_dict (dict): Metadata of one image, in Detectron2 Dataset format. + + Returns: + dict: a format that builtin models in detectron2 accept + """ + assert self.is_train, "OneFormerUnifiedDatasetMapper should only be used for training!" + + dataset_dict = copy.deepcopy(dataset_dict) # it will be modified by code below + image = utils.read_image(dataset_dict["file_name"], format=self.img_format) + utils.check_image_size(dataset_dict, image) + + # semantic segmentation + if "sem_seg_file_name" in dataset_dict: + # PyTorch transformation not implemented for uint16, so converting it to double first + sem_seg_gt = utils.read_image(dataset_dict.pop("sem_seg_file_name")).astype("double") + else: + sem_seg_gt = None + + # panoptic segmentation + if "pan_seg_file_name" in dataset_dict: + pan_seg_gt = utils.read_image(dataset_dict.pop("pan_seg_file_name"), "RGB") + segments_info = dataset_dict["segments_info"] + else: + pan_seg_gt = None + segments_info = None + + if pan_seg_gt is None: + raise ValueError( + "Cannot find 'pan_seg_file_name' for panoptic segmentation dataset {}.".format( + dataset_dict["file_name"] + ) + ) + + aug_input = T.AugInput(image, sem_seg=sem_seg_gt) + aug_input, transforms = T.apply_transform_gens(self.tfm_gens, aug_input) + image = aug_input.image + if sem_seg_gt is not None: + sem_seg_gt = aug_input.sem_seg + + # apply the same transformation to panoptic segmentation + pan_seg_gt = transforms.apply_segmentation(pan_seg_gt) + + from panopticapi.utils import rgb2id + + pan_seg_gt = rgb2id(pan_seg_gt) + + # Pad image and segmentation label here! + image = torch.as_tensor(np.ascontiguousarray(image.transpose(2, 0, 1))) + if sem_seg_gt is not None: + sem_seg_gt = torch.as_tensor(sem_seg_gt.astype("long")) + pan_seg_gt = torch.as_tensor(pan_seg_gt.astype("long")) + + if self.size_divisibility > 0: + image_size = (image.shape[-2], image.shape[-1]) + padding_size = [ + 0, + self.size_divisibility - image_size[1], + 0, + self.size_divisibility - image_size[0], + ] + image = F.pad(image, padding_size, value=128).contiguous() + if sem_seg_gt is not None: + sem_seg_gt = F.pad(sem_seg_gt, padding_size, value=self.ignore_label).contiguous() + pan_seg_gt = F.pad( + pan_seg_gt, padding_size, value=0 + ).contiguous() # 0 is the VOID panoptic label + + image_shape = (image.shape[-2], image.shape[-1]) # h, w + + # Pytorch's dataloader is efficient on torch.Tensor due to shared-memory, + # but not efficient on large generic data structures due to the use of pickle & mp.Queue. + # Therefore it's important to use torch.Tensor. + dataset_dict["image"] = image + + if "annotations" in dataset_dict: + raise ValueError("Pemantic segmentation dataset should not have 'annotations'.") + + prob_task = np.random.uniform(0,1.) + + num_class_obj = {} + + for name in self.class_names: + num_class_obj[name] = 0 + + if prob_task < self.semantic_prob: + task = "The task is semantic" + instances, text, sem_seg = self._get_semantic_dict(pan_seg_gt, image_shape, segments_info, num_class_obj) + elif prob_task < self.instance_prob: + task = "The task is instance" + instances, text, sem_seg = self._get_instance_dict(pan_seg_gt, image_shape, segments_info, num_class_obj) + else: + task = "The task is panoptic" + instances, text, sem_seg = self._get_panoptic_dict(pan_seg_gt, image_shape, segments_info, num_class_obj) + + dataset_dict["sem_seg"] = torch.from_numpy(sem_seg).long() + dataset_dict["instances"] = instances + dataset_dict["orig_shape"] = image_shape + dataset_dict["task"] = task + dataset_dict["text"] = text + dataset_dict["thing_ids"] = self.things + + return dataset_dict diff --git a/annotator/oneformer/oneformer/data/datasets/__init__.py b/annotator/oneformer/oneformer/data/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..59ce30713f63d056107b2a06ecd434eb27a30b7d --- /dev/null +++ b/annotator/oneformer/oneformer/data/datasets/__init__.py @@ -0,0 +1,7 @@ +from . import ( + register_ade20k_panoptic, + register_cityscapes_panoptic, + register_coco_panoptic_annos_semseg, + register_ade20k_instance, + register_coco_panoptic2instance, +) diff --git a/annotator/oneformer/oneformer/data/datasets/register_ade20k_instance.py b/annotator/oneformer/oneformer/data/datasets/register_ade20k_instance.py new file mode 100644 index 0000000000000000000000000000000000000000..e32d2b0bf5e2a937ac0ecf46b76239d6bc889ab8 --- /dev/null +++ b/annotator/oneformer/oneformer/data/datasets/register_ade20k_instance.py @@ -0,0 +1,56 @@ +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/Mask2Former/blob/main/mask2former/data/datasets/register_ade20k_instance.py +# ------------------------------------------------------------------------------ + +import json +import logging +import numpy as np +import os +from PIL import Image + +from annotator.oneformer.detectron2.data import DatasetCatalog, MetadataCatalog +from annotator.oneformer.detectron2.data.datasets.coco import load_coco_json, register_coco_instances +from annotator.oneformer.detectron2.utils.file_io import PathManager + +ADE_CATEGORIES = [{'id': 7, 'name': 'bed'}, {'id': 8, 'name': 'windowpane'}, {'id': 10, 'name': 'cabinet'}, {'id': 12, 'name': 'person'}, {'id': 14, 'name': 'door'}, {'id': 15, 'name': 'table'}, {'id': 18, 'name': 'curtain'}, {'id': 19, 'name': 'chair'}, {'id': 20, 'name': 'car'}, {'id': 22, 'name': 'painting'}, {'id': 23, 'name': 'sofa'}, {'id': 24, 'name': 'shelf'}, {'id': 27, 'name': 'mirror'}, {'id': 30, 'name': 'armchair'}, {'id': 31, 'name': 'seat'}, {'id': 32, 'name': 'fence'}, {'id': 33, 'name': 'desk'}, {'id': 35, 'name': 'wardrobe'}, {'id': 36, 'name': 'lamp'}, {'id': 37, 'name': 'bathtub'}, {'id': 38, 'name': 'railing'}, {'id': 39, 'name': 'cushion'}, {'id': 41, 'name': 'box'}, {'id': 42, 'name': 'column'}, {'id': 43, 'name': 'signboard'}, {'id': 44, 'name': 'chest of drawers'}, {'id': 45, 'name': 'counter'}, {'id': 47, 'name': 'sink'}, {'id': 49, 'name': 'fireplace'}, {'id': 50, 'name': 'refrigerator'}, {'id': 53, 'name': 'stairs'}, {'id': 55, 'name': 'case'}, {'id': 56, 'name': 'pool table'}, {'id': 57, 'name': 'pillow'}, {'id': 58, 'name': 'screen door'}, {'id': 62, 'name': 'bookcase'}, {'id': 64, 'name': 'coffee table'}, {'id': 65, 'name': 'toilet'}, {'id': 66, 'name': 'flower'}, {'id': 67, 'name': 'book'}, {'id': 69, 'name': 'bench'}, {'id': 70, 'name': 'countertop'}, {'id': 71, 'name': 'stove'}, {'id': 72, 'name': 'palm'}, {'id': 73, 'name': 'kitchen island'}, {'id': 74, 'name': 'computer'}, {'id': 75, 'name': 'swivel chair'}, {'id': 76, 'name': 'boat'}, {'id': 78, 'name': 'arcade machine'}, {'id': 80, 'name': 'bus'}, {'id': 81, 'name': 'towel'}, {'id': 82, 'name': 'light'}, {'id': 83, 'name': 'truck'}, {'id': 85, 'name': 'chandelier'}, {'id': 86, 'name': 'awning'}, {'id': 87, 'name': 'streetlight'}, {'id': 88, 'name': 'booth'}, {'id': 89, 'name': 'television receiver'}, {'id': 90, 'name': 'airplane'}, {'id': 92, 'name': 'apparel'}, {'id': 93, 'name': 'pole'}, {'id': 95, 'name': 'bannister'}, {'id': 97, 'name': 'ottoman'}, {'id': 98, 'name': 'bottle'}, {'id': 102, 'name': 'van'}, {'id': 103, 'name': 'ship'}, {'id': 104, 'name': 'fountain'}, {'id': 107, 'name': 'washer'}, {'id': 108, 'name': 'plaything'}, {'id': 110, 'name': 'stool'}, {'id': 111, 'name': 'barrel'}, {'id': 112, 'name': 'basket'}, {'id': 115, 'name': 'bag'}, {'id': 116, 'name': 'minibike'}, {'id': 118, 'name': 'oven'}, {'id': 119, 'name': 'ball'}, {'id': 120, 'name': 'food'}, {'id': 121, 'name': 'step'}, {'id': 123, 'name': 'trade name'}, {'id': 124, 'name': 'microwave'}, {'id': 125, 'name': 'pot'}, {'id': 126, 'name': 'animal'}, {'id': 127, 'name': 'bicycle'}, {'id': 129, 'name': 'dishwasher'}, {'id': 130, 'name': 'screen'}, {'id': 132, 'name': 'sculpture'}, {'id': 133, 'name': 'hood'}, {'id': 134, 'name': 'sconce'}, {'id': 135, 'name': 'vase'}, {'id': 136, 'name': 'traffic light'}, {'id': 137, 'name': 'tray'}, {'id': 138, 'name': 'ashcan'}, {'id': 139, 'name': 'fan'}, {'id': 142, 'name': 'plate'}, {'id': 143, 'name': 'monitor'}, {'id': 144, 'name': 'bulletin board'}, {'id': 146, 'name': 'radiator'}, {'id': 147, 'name': 'glass'}, {'id': 148, 'name': 'clock'}, {'id': 149, 'name': 'flag'}] + + +_PREDEFINED_SPLITS = { + # point annotations without masks + "ade20k_instance_train": ( + "ADEChallengeData2016/images/training", + "ADEChallengeData2016/ade20k_instance_train.json", + ), + "ade20k_instance_val": ( + "ADEChallengeData2016/images/validation", + "ADEChallengeData2016/ade20k_instance_val.json", + ), +} + + +def _get_ade_instances_meta(): + thing_ids = [k["id"] for k in ADE_CATEGORIES] + assert len(thing_ids) == 100, len(thing_ids) + # Mapping from the incontiguous ADE category id to an id in [0, 99] + thing_dataset_id_to_contiguous_id = {k: i for i, k in enumerate(thing_ids)} + thing_classes = [k["name"] for k in ADE_CATEGORIES] + ret = { + "thing_dataset_id_to_contiguous_id": thing_dataset_id_to_contiguous_id, + "thing_classes": thing_classes, + } + return ret + + +def register_all_ade20k_instance(root): + for key, (image_root, json_file) in _PREDEFINED_SPLITS.items(): + # Assume pre-defined datasets live in `./datasets`. + register_coco_instances( + key, + _get_ade_instances_meta(), + os.path.join(root, json_file) if "://" not in json_file else json_file, + os.path.join(root, image_root), + ) + + +_root = os.getenv("DETECTRON2_DATASETS", "datasets") +register_all_ade20k_instance(_root) diff --git a/annotator/oneformer/oneformer/data/datasets/register_ade20k_panoptic.py b/annotator/oneformer/oneformer/data/datasets/register_ade20k_panoptic.py new file mode 100644 index 0000000000000000000000000000000000000000..05094a617b0103b0f0250eb32e555df994e5331b --- /dev/null +++ b/annotator/oneformer/oneformer/data/datasets/register_ade20k_panoptic.py @@ -0,0 +1,394 @@ +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/Mask2Former/blob/main/mask2former/data/datasets/register_ade20k_panoptic.py +# Modified by Jitesh Jain (https://github.com/praeclarumjj3) +# ------------------------------------------------------------------------------ + +import json +import os + +from annotator.oneformer.detectron2.data import DatasetCatalog, MetadataCatalog +from annotator.oneformer.detectron2.utils.file_io import PathManager + +ADE20K_150_CATEGORIES = [ + {"color": [120, 120, 120], "id": 0, "isthing": 0, "name": "wall"}, + {"color": [180, 120, 120], "id": 1, "isthing": 0, "name": "building"}, + {"color": [6, 230, 230], "id": 2, "isthing": 0, "name": "sky"}, + {"color": [80, 50, 50], "id": 3, "isthing": 0, "name": "floor"}, + {"color": [4, 200, 3], "id": 4, "isthing": 0, "name": "tree"}, + {"color": [120, 120, 80], "id": 5, "isthing": 0, "name": "ceiling"}, + {"color": [140, 140, 140], "id": 6, "isthing": 0, "name": "road, route"}, + {"color": [204, 5, 255], "id": 7, "isthing": 1, "name": "bed"}, + {"color": [230, 230, 230], "id": 8, "isthing": 1, "name": "window "}, + {"color": [4, 250, 7], "id": 9, "isthing": 0, "name": "grass"}, + {"color": [224, 5, 255], "id": 10, "isthing": 1, "name": "cabinet"}, + {"color": [235, 255, 7], "id": 11, "isthing": 0, "name": "sidewalk, pavement"}, + {"color": [150, 5, 61], "id": 12, "isthing": 1, "name": "person"}, + {"color": [120, 120, 70], "id": 13, "isthing": 0, "name": "earth, ground"}, + {"color": [8, 255, 51], "id": 14, "isthing": 1, "name": "door"}, + {"color": [255, 6, 82], "id": 15, "isthing": 1, "name": "table"}, + {"color": [143, 255, 140], "id": 16, "isthing": 0, "name": "mountain, mount"}, + {"color": [204, 255, 4], "id": 17, "isthing": 0, "name": "plant"}, + {"color": [255, 51, 7], "id": 18, "isthing": 1, "name": "curtain"}, + {"color": [204, 70, 3], "id": 19, "isthing": 1, "name": "chair"}, + {"color": [0, 102, 200], "id": 20, "isthing": 1, "name": "car"}, + {"color": [61, 230, 250], "id": 21, "isthing": 0, "name": "water"}, + {"color": [255, 6, 51], "id": 22, "isthing": 1, "name": "painting, picture"}, + {"color": [11, 102, 255], "id": 23, "isthing": 1, "name": "sofa"}, + {"color": [255, 7, 71], "id": 24, "isthing": 1, "name": "shelf"}, + {"color": [255, 9, 224], "id": 25, "isthing": 0, "name": "house"}, + {"color": [9, 7, 230], "id": 26, "isthing": 0, "name": "sea"}, + {"color": [220, 220, 220], "id": 27, "isthing": 1, "name": "mirror"}, + {"color": [255, 9, 92], "id": 28, "isthing": 0, "name": "rug"}, + {"color": [112, 9, 255], "id": 29, "isthing": 0, "name": "field"}, + {"color": [8, 255, 214], "id": 30, "isthing": 1, "name": "armchair"}, + {"color": [7, 255, 224], "id": 31, "isthing": 1, "name": "seat"}, + {"color": [255, 184, 6], "id": 32, "isthing": 1, "name": "fence"}, + {"color": [10, 255, 71], "id": 33, "isthing": 1, "name": "desk"}, + {"color": [255, 41, 10], "id": 34, "isthing": 0, "name": "rock, stone"}, + {"color": [7, 255, 255], "id": 35, "isthing": 1, "name": "wardrobe, closet, press"}, + {"color": [224, 255, 8], "id": 36, "isthing": 1, "name": "lamp"}, + {"color": [102, 8, 255], "id": 37, "isthing": 1, "name": "tub"}, + {"color": [255, 61, 6], "id": 38, "isthing": 1, "name": "rail"}, + {"color": [255, 194, 7], "id": 39, "isthing": 1, "name": "cushion"}, + {"color": [255, 122, 8], "id": 40, "isthing": 0, "name": "base, pedestal, stand"}, + {"color": [0, 255, 20], "id": 41, "isthing": 1, "name": "box"}, + {"color": [255, 8, 41], "id": 42, "isthing": 1, "name": "column, pillar"}, + {"color": [255, 5, 153], "id": 43, "isthing": 1, "name": "signboard, sign"}, + { + "color": [6, 51, 255], + "id": 44, + "isthing": 1, + "name": "chest of drawers, chest, bureau, dresser", + }, + {"color": [235, 12, 255], "id": 45, "isthing": 1, "name": "counter"}, + {"color": [160, 150, 20], "id": 46, "isthing": 0, "name": "sand"}, + {"color": [0, 163, 255], "id": 47, "isthing": 1, "name": "sink"}, + {"color": [140, 140, 140], "id": 48, "isthing": 0, "name": "skyscraper"}, + {"color": [250, 10, 15], "id": 49, "isthing": 1, "name": "fireplace"}, + {"color": [20, 255, 0], "id": 50, "isthing": 1, "name": "refrigerator, icebox"}, + {"color": [31, 255, 0], "id": 51, "isthing": 0, "name": "grandstand, covered stand"}, + {"color": [255, 31, 0], "id": 52, "isthing": 0, "name": "path"}, + {"color": [255, 224, 0], "id": 53, "isthing": 1, "name": "stairs"}, + {"color": [153, 255, 0], "id": 54, "isthing": 0, "name": "runway"}, + {"color": [0, 0, 255], "id": 55, "isthing": 1, "name": "case, display case, showcase, vitrine"}, + { + "color": [255, 71, 0], + "id": 56, + "isthing": 1, + "name": "pool table, billiard table, snooker table", + }, + {"color": [0, 235, 255], "id": 57, "isthing": 1, "name": "pillow"}, + {"color": [0, 173, 255], "id": 58, "isthing": 1, "name": "screen door, screen"}, + {"color": [31, 0, 255], "id": 59, "isthing": 0, "name": "stairway, staircase"}, + {"color": [11, 200, 200], "id": 60, "isthing": 0, "name": "river"}, + {"color": [255, 82, 0], "id": 61, "isthing": 0, "name": "bridge, span"}, + {"color": [0, 255, 245], "id": 62, "isthing": 1, "name": "bookcase"}, + {"color": [0, 61, 255], "id": 63, "isthing": 0, "name": "blind, screen"}, + {"color": [0, 255, 112], "id": 64, "isthing": 1, "name": "coffee table"}, + { + "color": [0, 255, 133], + "id": 65, + "isthing": 1, + "name": "toilet, can, commode, crapper, pot, potty, stool, throne", + }, + {"color": [255, 0, 0], "id": 66, "isthing": 1, "name": "flower"}, + {"color": [255, 163, 0], "id": 67, "isthing": 1, "name": "book"}, + {"color": [255, 102, 0], "id": 68, "isthing": 0, "name": "hill"}, + {"color": [194, 255, 0], "id": 69, "isthing": 1, "name": "bench"}, + {"color": [0, 143, 255], "id": 70, "isthing": 1, "name": "countertop"}, + {"color": [51, 255, 0], "id": 71, "isthing": 1, "name": "stove"}, + {"color": [0, 82, 255], "id": 72, "isthing": 1, "name": "palm, palm tree"}, + {"color": [0, 255, 41], "id": 73, "isthing": 1, "name": "kitchen island"}, + {"color": [0, 255, 173], "id": 74, "isthing": 1, "name": "computer"}, + {"color": [10, 0, 255], "id": 75, "isthing": 1, "name": "swivel chair"}, + {"color": [173, 255, 0], "id": 76, "isthing": 1, "name": "boat"}, + {"color": [0, 255, 153], "id": 77, "isthing": 0, "name": "bar"}, + {"color": [255, 92, 0], "id": 78, "isthing": 1, "name": "arcade machine"}, + {"color": [255, 0, 255], "id": 79, "isthing": 0, "name": "hovel, hut, hutch, shack, shanty"}, + {"color": [255, 0, 245], "id": 80, "isthing": 1, "name": "bus"}, + {"color": [255, 0, 102], "id": 81, "isthing": 1, "name": "towel"}, + {"color": [255, 173, 0], "id": 82, "isthing": 1, "name": "light"}, + {"color": [255, 0, 20], "id": 83, "isthing": 1, "name": "truck"}, + {"color": [255, 184, 184], "id": 84, "isthing": 0, "name": "tower"}, + {"color": [0, 31, 255], "id": 85, "isthing": 1, "name": "chandelier"}, + {"color": [0, 255, 61], "id": 86, "isthing": 1, "name": "awning, sunshade, sunblind"}, + {"color": [0, 71, 255], "id": 87, "isthing": 1, "name": "street lamp"}, + {"color": [255, 0, 204], "id": 88, "isthing": 1, "name": "booth"}, + {"color": [0, 255, 194], "id": 89, "isthing": 1, "name": "tv"}, + {"color": [0, 255, 82], "id": 90, "isthing": 1, "name": "plane"}, + {"color": [0, 10, 255], "id": 91, "isthing": 0, "name": "dirt track"}, + {"color": [0, 112, 255], "id": 92, "isthing": 1, "name": "clothes"}, + {"color": [51, 0, 255], "id": 93, "isthing": 1, "name": "pole"}, + {"color": [0, 194, 255], "id": 94, "isthing": 0, "name": "land, ground, soil"}, + { + "color": [0, 122, 255], + "id": 95, + "isthing": 1, + "name": "bannister, banister, balustrade, balusters, handrail", + }, + { + "color": [0, 255, 163], + "id": 96, + "isthing": 0, + "name": "escalator, moving staircase, moving stairway", + }, + { + "color": [255, 153, 0], + "id": 97, + "isthing": 1, + "name": "ottoman, pouf, pouffe, puff, hassock", + }, + {"color": [0, 255, 10], "id": 98, "isthing": 1, "name": "bottle"}, + {"color": [255, 112, 0], "id": 99, "isthing": 0, "name": "buffet, counter, sideboard"}, + { + "color": [143, 255, 0], + "id": 100, + "isthing": 0, + "name": "poster, posting, placard, notice, bill, card", + }, + {"color": [82, 0, 255], "id": 101, "isthing": 0, "name": "stage"}, + {"color": [163, 255, 0], "id": 102, "isthing": 1, "name": "van"}, + {"color": [255, 235, 0], "id": 103, "isthing": 1, "name": "ship"}, + {"color": [8, 184, 170], "id": 104, "isthing": 1, "name": "fountain"}, + { + "color": [133, 0, 255], + "id": 105, + "isthing": 0, + "name": "conveyer belt, conveyor belt, conveyer, conveyor, transporter", + }, + {"color": [0, 255, 92], "id": 106, "isthing": 0, "name": "canopy"}, + { + "color": [184, 0, 255], + "id": 107, + "isthing": 1, + "name": "washer, automatic washer, washing machine", + }, + {"color": [255, 0, 31], "id": 108, "isthing": 1, "name": "plaything, toy"}, + {"color": [0, 184, 255], "id": 109, "isthing": 0, "name": "pool"}, + {"color": [0, 214, 255], "id": 110, "isthing": 1, "name": "stool"}, + {"color": [255, 0, 112], "id": 111, "isthing": 1, "name": "barrel, cask"}, + {"color": [92, 255, 0], "id": 112, "isthing": 1, "name": "basket, handbasket"}, + {"color": [0, 224, 255], "id": 113, "isthing": 0, "name": "falls"}, + {"color": [112, 224, 255], "id": 114, "isthing": 0, "name": "tent"}, + {"color": [70, 184, 160], "id": 115, "isthing": 1, "name": "bag"}, + {"color": [163, 0, 255], "id": 116, "isthing": 1, "name": "minibike, motorbike"}, + {"color": [153, 0, 255], "id": 117, "isthing": 0, "name": "cradle"}, + {"color": [71, 255, 0], "id": 118, "isthing": 1, "name": "oven"}, + {"color": [255, 0, 163], "id": 119, "isthing": 1, "name": "ball"}, + {"color": [255, 204, 0], "id": 120, "isthing": 1, "name": "food, solid food"}, + {"color": [255, 0, 143], "id": 121, "isthing": 1, "name": "step, stair"}, + {"color": [0, 255, 235], "id": 122, "isthing": 0, "name": "tank, storage tank"}, + {"color": [133, 255, 0], "id": 123, "isthing": 1, "name": "trade name"}, + {"color": [255, 0, 235], "id": 124, "isthing": 1, "name": "microwave"}, + {"color": [245, 0, 255], "id": 125, "isthing": 1, "name": "pot"}, + {"color": [255, 0, 122], "id": 126, "isthing": 1, "name": "animal"}, + {"color": [255, 245, 0], "id": 127, "isthing": 1, "name": "bicycle"}, + {"color": [10, 190, 212], "id": 128, "isthing": 0, "name": "lake"}, + {"color": [214, 255, 0], "id": 129, "isthing": 1, "name": "dishwasher"}, + {"color": [0, 204, 255], "id": 130, "isthing": 1, "name": "screen"}, + {"color": [20, 0, 255], "id": 131, "isthing": 0, "name": "blanket, cover"}, + {"color": [255, 255, 0], "id": 132, "isthing": 1, "name": "sculpture"}, + {"color": [0, 153, 255], "id": 133, "isthing": 1, "name": "hood, exhaust hood"}, + {"color": [0, 41, 255], "id": 134, "isthing": 1, "name": "sconce"}, + {"color": [0, 255, 204], "id": 135, "isthing": 1, "name": "vase"}, + {"color": [41, 0, 255], "id": 136, "isthing": 1, "name": "traffic light"}, + {"color": [41, 255, 0], "id": 137, "isthing": 1, "name": "tray"}, + {"color": [173, 0, 255], "id": 138, "isthing": 1, "name": "trash can"}, + {"color": [0, 245, 255], "id": 139, "isthing": 1, "name": "fan"}, + {"color": [71, 0, 255], "id": 140, "isthing": 0, "name": "pier"}, + {"color": [122, 0, 255], "id": 141, "isthing": 0, "name": "crt screen"}, + {"color": [0, 255, 184], "id": 142, "isthing": 1, "name": "plate"}, + {"color": [0, 92, 255], "id": 143, "isthing": 1, "name": "monitor"}, + {"color": [184, 255, 0], "id": 144, "isthing": 1, "name": "bulletin board"}, + {"color": [0, 133, 255], "id": 145, "isthing": 0, "name": "shower"}, + {"color": [255, 214, 0], "id": 146, "isthing": 1, "name": "radiator"}, + {"color": [25, 194, 194], "id": 147, "isthing": 1, "name": "glass, drinking glass"}, + {"color": [102, 255, 0], "id": 148, "isthing": 1, "name": "clock"}, + {"color": [92, 0, 255], "id": 149, "isthing": 1, "name": "flag"}, +] + +ADE20k_COLORS = [k["color"] for k in ADE20K_150_CATEGORIES] + +MetadataCatalog.get("ade20k_sem_seg_train").set( + stuff_colors=ADE20k_COLORS[:], +) + +MetadataCatalog.get("ade20k_sem_seg_val").set( + stuff_colors=ADE20k_COLORS[:], +) + + +def load_ade20k_panoptic_json(json_file, image_dir, gt_dir, semseg_dir, meta): + """ + Args: + image_dir (str): path to the raw dataset. e.g., "~/coco/train2017". + gt_dir (str): path to the raw annotations. e.g., "~/coco/panoptic_train2017". + json_file (str): path to the json file. e.g., "~/coco/annotations/panoptic_train2017.json". + Returns: + list[dict]: a list of dicts in Detectron2 standard format. (See + `Using Custom Datasets `_ ) + """ + + def _convert_category_id(segment_info, meta): + if segment_info["category_id"] in meta["thing_dataset_id_to_contiguous_id"]: + segment_info["category_id"] = meta["thing_dataset_id_to_contiguous_id"][ + segment_info["category_id"] + ] + segment_info["isthing"] = True + else: + segment_info["category_id"] = meta["stuff_dataset_id_to_contiguous_id"][ + segment_info["category_id"] + ] + segment_info["isthing"] = False + return segment_info + + with PathManager.open(json_file) as f: + json_info = json.load(f) + + ret = [] + for ann in json_info["annotations"]: + image_id = ann["image_id"] + # TODO: currently we assume image and label has the same filename but + # different extension, and images have extension ".jpg" for COCO. Need + # to make image extension a user-provided argument if we extend this + # function to support other COCO-like datasets. + image_file = os.path.join(image_dir, os.path.splitext(ann["file_name"])[0] + ".jpg") + label_file = os.path.join(gt_dir, ann["file_name"]) + sem_label_file = os.path.join(semseg_dir, ann["file_name"]) + segments_info = [_convert_category_id(x, meta) for x in ann["segments_info"]] + ret.append( + { + "file_name": image_file, + "image_id": image_id, + "pan_seg_file_name": label_file, + "sem_seg_file_name": sem_label_file, + "segments_info": segments_info, + } + ) + assert len(ret), f"No images found in {image_dir}!" + assert PathManager.isfile(ret[0]["file_name"]), ret[0]["file_name"] + assert PathManager.isfile(ret[0]["pan_seg_file_name"]), ret[0]["pan_seg_file_name"] + assert PathManager.isfile(ret[0]["sem_seg_file_name"]), ret[0]["sem_seg_file_name"] + return ret + + +def register_ade20k_panoptic( + name, metadata, image_root, panoptic_root, semantic_root, panoptic_json, instances_json=None, +): + """ + Register a "standard" version of ADE20k panoptic segmentation dataset named `name`. + The dictionaries in this registered dataset follows detectron2's standard format. + Hence it's called "standard". + Args: + name (str): the name that identifies a dataset, + e.g. "ade20k_panoptic_train" + metadata (dict): extra metadata associated with this dataset. + image_root (str): directory which contains all the images + panoptic_root (str): directory which contains panoptic annotation images in COCO format + panoptic_json (str): path to the json panoptic annotation file in COCO format + sem_seg_root (none): not used, to be consistent with + `register_coco_panoptic_separated`. + instances_json (str): path to the json instance annotation file + """ + panoptic_name = name + DatasetCatalog.register( + panoptic_name, + lambda: load_ade20k_panoptic_json( + panoptic_json, image_root, panoptic_root, semantic_root, metadata + ), + ) + MetadataCatalog.get(panoptic_name).set( + panoptic_root=panoptic_root, + image_root=image_root, + panoptic_json=panoptic_json, + json_file=instances_json, + evaluator_type="ade20k_panoptic_seg", + ignore_label=255, + label_divisor=1000, + **metadata, + ) + + +_PREDEFINED_SPLITS_ADE20K_PANOPTIC = { + "ade20k_panoptic_train": ( + "ADEChallengeData2016/images/training", + "ADEChallengeData2016/ade20k_panoptic_train", + "ADEChallengeData2016/ade20k_panoptic_train.json", + "ADEChallengeData2016/annotations_detectron2/training", + "ADEChallengeData2016/ade20k_instance_train.json", + ), + "ade20k_panoptic_val": ( + "ADEChallengeData2016/images/validation", + "ADEChallengeData2016/ade20k_panoptic_val", + "ADEChallengeData2016/ade20k_panoptic_val.json", + "ADEChallengeData2016/annotations_detectron2/validation", + "ADEChallengeData2016/ade20k_instance_val.json", + ), +} + + +def get_metadata(): + meta = {} + # The following metadata maps contiguous id from [0, #thing categories + + # #stuff categories) to their names and colors. We have to replica of the + # same name and color under "thing_*" and "stuff_*" because the current + # visualization function in D2 handles thing and class classes differently + # due to some heuristic used in Panoptic FPN. We keep the same naming to + # enable reusing existing visualization functions. + thing_classes = [k["name"] for k in ADE20K_150_CATEGORIES if k["isthing"] == 1] + thing_colors = [k["color"] for k in ADE20K_150_CATEGORIES if k["isthing"] == 1] + stuff_classes = [k["name"] for k in ADE20K_150_CATEGORIES] + stuff_colors = [k["color"] for k in ADE20K_150_CATEGORIES] + + meta["thing_classes"] = thing_classes + meta["thing_colors"] = thing_colors + meta["stuff_classes"] = stuff_classes + meta["stuff_colors"] = stuff_colors + + # Convert category id for training: + # category id: like semantic segmentation, it is the class id for each + # pixel. Since there are some classes not used in evaluation, the category + # id is not always contiguous and thus we have two set of category ids: + # - original category id: category id in the original dataset, mainly + # used for evaluation. + # - contiguous category id: [0, #classes), in order to train the linear + # softmax classifier. + thing_dataset_id_to_contiguous_id = {} + stuff_dataset_id_to_contiguous_id = {} + + for i, cat in enumerate(ADE20K_150_CATEGORIES): + if cat["isthing"]: + thing_dataset_id_to_contiguous_id[cat["id"]] = i + # else: + # stuff_dataset_id_to_contiguous_id[cat["id"]] = i + + # in order to use sem_seg evaluator + stuff_dataset_id_to_contiguous_id[cat["id"]] = i + + meta["thing_dataset_id_to_contiguous_id"] = thing_dataset_id_to_contiguous_id + meta["stuff_dataset_id_to_contiguous_id"] = stuff_dataset_id_to_contiguous_id + + return meta + + +def register_all_ade20k_panoptic(root): + metadata = get_metadata() + for ( + prefix, + (image_root, panoptic_root, panoptic_json, semantic_root, instance_json), + ) in _PREDEFINED_SPLITS_ADE20K_PANOPTIC.items(): + # The "standard" version of COCO panoptic segmentation dataset, + # e.g. used by Panoptic-DeepLab + register_ade20k_panoptic( + prefix, + metadata, + os.path.join(root, image_root), + os.path.join(root, panoptic_root), + os.path.join(root, semantic_root), + os.path.join(root, panoptic_json), + os.path.join(root, instance_json), + ) + + +_root = os.getenv("DETECTRON2_DATASETS", "datasets") +register_all_ade20k_panoptic(_root) diff --git a/annotator/oneformer/oneformer/data/datasets/register_cityscapes_panoptic.py b/annotator/oneformer/oneformer/data/datasets/register_cityscapes_panoptic.py new file mode 100644 index 0000000000000000000000000000000000000000..5f2c2a69e8c396b4b6fa8eb4125d76b9d1f3a101 --- /dev/null +++ b/annotator/oneformer/oneformer/data/datasets/register_cityscapes_panoptic.py @@ -0,0 +1,199 @@ +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/detectron2/blob/main/detectron2/data/datasets/cityscapes_panoptic.py +# Modified by Jitesh Jain (https://github.com/praeclarumjj3) +# ------------------------------------------------------------------------------ + +import json +import logging +import os + +from annotator.oneformer.detectron2.data import DatasetCatalog, MetadataCatalog +from annotator.oneformer.detectron2.data.datasets.builtin_meta import CITYSCAPES_CATEGORIES +from annotator.oneformer.detectron2.utils.file_io import PathManager + +""" +This file contains functions to register the Cityscapes panoptic dataset to the DatasetCatalog. +""" + + +logger = logging.getLogger(__name__) + + +def get_cityscapes_panoptic_files(image_dir, gt_dir, json_info): + files = [] + # scan through the directory + cities = PathManager.ls(image_dir) + logger.info(f"{len(cities)} cities found in '{image_dir}'.") + image_dict = {} + for city in cities: + city_img_dir = os.path.join(image_dir, city) + for basename in PathManager.ls(city_img_dir): + image_file = os.path.join(city_img_dir, basename) + + suffix = "_leftImg8bit.png" + assert basename.endswith(suffix), basename + basename = os.path.basename(basename)[: -len(suffix)] + + image_dict[basename] = image_file + + for ann in json_info["annotations"]: + image_file = image_dict.get(ann["image_id"], None) + assert image_file is not None, "No image {} found for annotation {}".format( + ann["image_id"], ann["file_name"] + ) + label_file = os.path.join(gt_dir, ann["file_name"]) + segments_info = ann["segments_info"] + files.append((image_file, label_file, segments_info)) + + assert len(files), "No images found in {}".format(image_dir) + assert PathManager.isfile(files[0][0]), files[0][0] + assert PathManager.isfile(files[0][1]), files[0][1] + return files + + +def load_cityscapes_panoptic(image_dir, gt_dir, gt_json, meta): + """ + Args: + image_dir (str): path to the raw dataset. e.g., "~/cityscapes/leftImg8bit/train". + gt_dir (str): path to the raw annotations. e.g., + "~/cityscapes/gtFine/cityscapes_panoptic_train". + gt_json (str): path to the json file. e.g., + "~/cityscapes/gtFine/cityscapes_panoptic_train.json". + meta (dict): dictionary containing "thing_dataset_id_to_contiguous_id" + and "stuff_dataset_id_to_contiguous_id" to map category ids to + contiguous ids for training. + + Returns: + list[dict]: a list of dicts in Detectron2 standard format. (See + `Using Custom Datasets `_ ) + """ + + def _convert_category_id(segment_info, meta): + if segment_info["category_id"] in meta["thing_dataset_id_to_contiguous_id"]: + segment_info["category_id"] = meta["thing_dataset_id_to_contiguous_id"][ + segment_info["category_id"] + ] + else: + segment_info["category_id"] = meta["stuff_dataset_id_to_contiguous_id"][ + segment_info["category_id"] + ] + return segment_info + + assert os.path.exists( + gt_json + ), "Please run `python cityscapesscripts/preparation/createPanopticImgs.py` to generate label files." # noqa + + + with open(gt_json) as f: + json_info = json.load(f) + + files = get_cityscapes_panoptic_files(image_dir, gt_dir, json_info) + ret = [] + for image_file, label_file, segments_info in files: + sem_label_file = ( + image_file.replace("leftImg8bit", "gtFine").split(".")[0] + "_labelTrainIds.png" + ) + segments_info = [_convert_category_id(x, meta) for x in segments_info] + ret.append( + { + "file_name": image_file, + "image_id": "_".join( + os.path.splitext(os.path.basename(image_file))[0].split("_")[:3] + ), + "sem_seg_file_name": sem_label_file, + "pan_seg_file_name": label_file, + "segments_info": segments_info, + } + ) + assert len(ret), f"No images found in {image_dir}!" + assert PathManager.isfile( + ret[0]["sem_seg_file_name"] + ), "Please generate labelTrainIds.png with cityscapesscripts/preparation/createTrainIdLabelImgs.py" # noqa + assert PathManager.isfile( + ret[0]["pan_seg_file_name"] + ), "Please generate panoptic annotation with python cityscapesscripts/preparation/createPanopticImgs.py" # noqa + return ret + + +_RAW_CITYSCAPES_PANOPTIC_SPLITS = { + "cityscapes_fine_panoptic_train": ( + "cityscapes/leftImg8bit/train", + "cityscapes/gtFine/cityscapes_panoptic_train", + "cityscapes/gtFine/cityscapes_panoptic_train.json", + ), + "cityscapes_fine_panoptic_val": ( + "cityscapes/leftImg8bit/val", + "cityscapes/gtFine/cityscapes_panoptic_val", + "cityscapes/gtFine/cityscapes_panoptic_val.json", + ), + # "cityscapes_fine_panoptic_test": not supported yet +} + + +def register_all_cityscapes_panoptic(root): + meta = {} + # The following metadata maps contiguous id from [0, #thing categories + + # #stuff categories) to their names and colors. We have to replica of the + # same name and color under "thing_*" and "stuff_*" because the current + # visualization function in D2 handles thing and class classes differently + # due to some heuristic used in Panoptic FPN. We keep the same naming to + # enable reusing existing visualization functions. + thing_classes = [k["name"] for k in CITYSCAPES_CATEGORIES] + thing_colors = [k["color"] for k in CITYSCAPES_CATEGORIES] + stuff_classes = [k["name"] for k in CITYSCAPES_CATEGORIES] + stuff_colors = [k["color"] for k in CITYSCAPES_CATEGORIES] + + meta["thing_classes"] = thing_classes + meta["thing_colors"] = thing_colors + meta["stuff_classes"] = stuff_classes + meta["stuff_colors"] = stuff_colors + + # There are three types of ids in cityscapes panoptic segmentation: + # (1) category id: like semantic segmentation, it is the class id for each + # pixel. Since there are some classes not used in evaluation, the category + # id is not always contiguous and thus we have two set of category ids: + # - original category id: category id in the original dataset, mainly + # used for evaluation. + # - contiguous category id: [0, #classes), in order to train the classifier + # (2) instance id: this id is used to differentiate different instances from + # the same category. For "stuff" classes, the instance id is always 0; for + # "thing" classes, the instance id starts from 1 and 0 is reserved for + # ignored instances (e.g. crowd annotation). + # (3) panoptic id: this is the compact id that encode both category and + # instance id by: category_id * 1000 + instance_id. + thing_dataset_id_to_contiguous_id = {} + stuff_dataset_id_to_contiguous_id = {} + + for k in CITYSCAPES_CATEGORIES: + if k["isthing"] == 1: + thing_dataset_id_to_contiguous_id[k["id"]] = k["trainId"] + else: + stuff_dataset_id_to_contiguous_id[k["id"]] = k["trainId"] + + meta["thing_dataset_id_to_contiguous_id"] = thing_dataset_id_to_contiguous_id + meta["stuff_dataset_id_to_contiguous_id"] = stuff_dataset_id_to_contiguous_id + + for key, (image_dir, gt_dir, gt_json) in _RAW_CITYSCAPES_PANOPTIC_SPLITS.items(): + image_dir = os.path.join(root, image_dir) + gt_dir = os.path.join(root, gt_dir) + gt_json = os.path.join(root, gt_json) + + if key in DatasetCatalog.list(): + DatasetCatalog.remove(key) + + DatasetCatalog.register( + key, lambda x=image_dir, y=gt_dir, z=gt_json: load_cityscapes_panoptic(x, y, z, meta) + ) + MetadataCatalog.get(key).set( + panoptic_root=gt_dir, + image_root=image_dir, + panoptic_json=gt_json, + gt_dir=gt_dir.replace("cityscapes_panoptic_", ""), + evaluator_type="cityscapes_panoptic_seg", + ignore_label=255, + label_divisor=1000, + **meta, + ) + +_root = os.getenv("DETECTRON2_DATASETS", "datasets") +register_all_cityscapes_panoptic(_root) \ No newline at end of file diff --git a/annotator/oneformer/oneformer/data/datasets/register_coco_panoptic2instance.py b/annotator/oneformer/oneformer/data/datasets/register_coco_panoptic2instance.py new file mode 100644 index 0000000000000000000000000000000000000000..511c5b66fa1a1814baf6f83bf048622723551e7d --- /dev/null +++ b/annotator/oneformer/oneformer/data/datasets/register_coco_panoptic2instance.py @@ -0,0 +1,44 @@ +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/detectron2/blob/main/detectron2/data/datasets/builtin.py +# Modified by Jitesh Jain (https://github.com/praeclarumjj3) +# ------------------------------------------------------------------------------ + + +""" +This file registers pre-defined datasets at hard-coded paths, and their metadata. + +We hard-code metadata for common datasets. This will enable: +1. Consistency check when loading the datasets +2. Use models on these standard datasets directly and run demos, + without having to download the dataset annotations + +We hard-code some paths to the dataset that's assumed to +exist in "./datasets/". + +Users SHOULD NOT use this file to create new dataset / metadata for new dataset. +To add new dataset, refer to the tutorial "docs/DATASETS.md". +""" + +import os +from annotator.oneformer.detectron2.data.datasets.builtin_meta import _get_builtin_metadata +from annotator.oneformer.detectron2.data.datasets.coco import register_coco_instances + + +_PREDEFINED_SPLITS_COCO = { + "coco_2017_val_panoptic2instance": ("coco/val2017", "coco/annotations/panoptic2instances_val2017.json"), +} + + +def register_panoptic2instances_coco(root): + for key, (image_root, json_file) in _PREDEFINED_SPLITS_COCO.items(): + # Assume pre-defined datasets live in `./datasets`. + register_coco_instances( + key, + _get_builtin_metadata("coco"), + os.path.join(root, json_file) if "://" not in json_file else json_file, + os.path.join(root, image_root), + ) + + +_root = os.path.expanduser(os.getenv("DETECTRON2_DATASETS", "datasets")) +register_panoptic2instances_coco(_root) \ No newline at end of file diff --git a/annotator/oneformer/oneformer/data/datasets/register_coco_panoptic_annos_semseg.py b/annotator/oneformer/oneformer/data/datasets/register_coco_panoptic_annos_semseg.py new file mode 100644 index 0000000000000000000000000000000000000000..5690a2c217dc698464e8057f057c1ad2dcdf605b --- /dev/null +++ b/annotator/oneformer/oneformer/data/datasets/register_coco_panoptic_annos_semseg.py @@ -0,0 +1,367 @@ +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/Mask2Former/blob/main/mask2former/data/datasets/register_coco_panoptic_annos_semseg.py +# Modified by Jitesh Jain (https://github.com/praeclarumjj3) +# ------------------------------------------------------------------------------ + +import json +import os + +from annotator.oneformer.detectron2.data import DatasetCatalog, MetadataCatalog +from annotator.oneformer.detectron2.data.datasets import load_sem_seg +from annotator.oneformer.detectron2.data.datasets.builtin_meta import COCO_CATEGORIES +from annotator.oneformer.detectron2.utils.file_io import PathManager +import contextlib +import logging +import io +from fvcore.common.timer import Timer +import pycocotools.mask as mask_util +from annotator.oneformer.detectron2.structures import BoxMode + + +logger = logging.getLogger(__name__) + + +_PREDEFINED_SPLITS_COCO_PANOPTIC = { + "coco_2017_train_panoptic": ( + # This is the original panoptic annotation directory + "coco/panoptic_train2017", + "coco/annotations/panoptic_train2017.json", + # This directory contains semantic annotations that are + # converted from panoptic annotations. + # It is used by PanopticFPN. + # You can use the script at detectron2/datasets/prepare_panoptic_fpn.py + # to create these directories. + "coco/panoptic_semseg_train2017", + ), + "coco_2017_val_panoptic": ( + "coco/panoptic_val2017", + "coco/annotations/panoptic_val2017.json", + "coco/panoptic_semseg_val2017", + ), +} + +def load_coco_instance_json(json_file, image_root, dataset_name=None): + from pycocotools.coco import COCO + + timer = Timer() + json_file = PathManager.get_local_path(json_file) + with contextlib.redirect_stdout(io.StringIO()): + coco_api = COCO(json_file) + if timer.seconds() > 1: + logger.info("Loading {} takes {:.2f} seconds.".format(json_file, timer.seconds())) + + id_map = None + if dataset_name is not None: + meta = MetadataCatalog.get(dataset_name) + cat_ids = sorted(coco_api.getCatIds()) + cats = coco_api.loadCats(cat_ids) + # The categories in a custom json file may not be sorted. + thing_classes = [c["name"] for c in sorted(cats, key=lambda x: x["id"])] + meta.thing_classes = thing_classes + + # In COCO, certain category ids are artificially removed, + # and by convention they are always ignored. + # We deal with COCO's id issue and translate + # the category ids to contiguous ids in [0, 80). + + # It works by looking at the "categories" field in the json, therefore + # if users' own json also have incontiguous ids, we'll + # apply this mapping as well but print a warning. + if not (min(cat_ids) == 1 and max(cat_ids) == len(cat_ids)): + if "coco" not in dataset_name: + logger.warning( + """ +Category ids in annotations are not in [1, #categories]! We'll apply a mapping for you. +""" + ) + id_map = {v: i for i, v in enumerate(cat_ids)} + meta.thing_dataset_id_to_contiguous_id = id_map + + # sort indices for reproducible results + img_ids = sorted(coco_api.imgs.keys()) + # imgs is a list of dicts, each looks something like: + # {'license': 4, + # 'url': 'http://farm6.staticflickr.com/5454/9413846304_881d5e5c3b_z.jpg', + # 'file_name': 'COCO_val2014_000000001268.jpg', + # 'height': 427, + # 'width': 640, + # 'date_captured': '2013-11-17 05:57:24', + # 'id': 1268} + imgs = coco_api.loadImgs(img_ids) + # anns is a list[list[dict]], where each dict is an annotation + # record for an object. The inner list enumerates the objects in an image + # and the outer list enumerates over images. Example of anns[0]: + # [{'segmentation': [[192.81, + # 247.09, + # ... + # 219.03, + # 249.06]], + # 'area': 1035.749, + # 'iscrowd': 0, + # 'image_id': 1268, + # 'bbox': [192.81, 224.8, 74.73, 33.43], + # 'category_id': 16, + # 'id': 42986}, + # ...] + anns = [coco_api.imgToAnns[img_id] for img_id in img_ids] + total_num_valid_anns = sum([len(x) for x in anns]) + total_num_anns = len(coco_api.anns) + if total_num_valid_anns < total_num_anns: + logger.warning( + f"{json_file} contains {total_num_anns} annotations, but only " + f"{total_num_valid_anns} of them match to images in the file." + ) + + if "minival" not in json_file: + # The popular valminusminival & minival annotations for COCO2014 contain this bug. + # However the ratio of buggy annotations there is tiny and does not affect accuracy. + # Therefore we explicitly white-list them. + ann_ids = [ann["id"] for anns_per_image in anns for ann in anns_per_image] + assert len(set(ann_ids)) == len(ann_ids), "Annotation ids in '{}' are not unique!".format( + json_file + ) + + imgs_anns = list(zip(imgs, anns)) + logger.info("Loaded {} images in COCO format from {}".format(len(imgs_anns), json_file)) + + dataset_dicts = {} + + ann_keys = ["iscrowd", "bbox", "keypoints", "category_id"] + + num_instances_without_valid_segmentation = 0 + + for (img_dict, anno_dict_list) in imgs_anns: + record = {} + record["file_name"] = os.path.join(image_root, img_dict["file_name"]) + record["height"] = img_dict["height"] + record["width"] = img_dict["width"] + image_id = record["image_id"] = img_dict["id"] + + objs = [] + for anno in anno_dict_list: + # Check that the image_id in this annotation is the same as + # the image_id we're looking at. + # This fails only when the data parsing logic or the annotation file is buggy. + + # The original COCO valminusminival2014 & minival2014 annotation files + # actually contains bugs that, together with certain ways of using COCO API, + # can trigger this assertion. + assert anno["image_id"] == image_id + + assert anno.get("ignore", 0) == 0, '"ignore" in COCO json file is not supported.' + + obj = {key: anno[key] for key in ann_keys if key in anno} + if "bbox" in obj and len(obj["bbox"]) == 0: + raise ValueError( + f"One annotation of image {image_id} contains empty 'bbox' value! " + "This json does not have valid COCO format." + ) + + segm = anno.get("segmentation", None) + if segm: # either list[list[float]] or dict(RLE) + if isinstance(segm, dict): + if isinstance(segm["counts"], list): + # convert to compressed RLE + segm = mask_util.frPyObjects(segm, *segm["size"]) + else: + # filter out invalid polygons (< 3 points) + segm = [poly for poly in segm if len(poly) % 2 == 0 and len(poly) >= 6] + if len(segm) == 0: + num_instances_without_valid_segmentation += 1 + continue # ignore this instance + obj["segmentation"] = segm + + keypts = anno.get("keypoints", None) + if keypts: # list[int] + for idx, v in enumerate(keypts): + if idx % 3 != 2: + # COCO's segmentation coordinates are floating points in [0, H or W], + # but keypoint coordinates are integers in [0, H-1 or W-1] + # Therefore we assume the coordinates are "pixel indices" and + # add 0.5 to convert to floating point coordinates. + keypts[idx] = v + 0.5 + obj["keypoints"] = keypts + + obj["bbox_mode"] = BoxMode.XYWH_ABS + if id_map: + annotation_category_id = obj["category_id"] + try: + obj["category_id"] = id_map[annotation_category_id] + except KeyError as e: + raise KeyError( + f"Encountered category_id={annotation_category_id} " + "but this id does not exist in 'categories' of the json file." + ) from e + objs.append(obj) + record["annotations"] = objs + dataset_dicts[image_id] = record + + if num_instances_without_valid_segmentation > 0: + logger.warning( + "Filtered out {} instances without valid segmentation. ".format( + num_instances_without_valid_segmentation + ) + + "There might be issues in your dataset generation process. Please " + "check https://detectron2.readthedocs.io/en/latest/tutorials/datasets.html carefully" + ) + return dataset_dicts + +def get_metadata(): + meta = {} + # The following metadata maps contiguous id from [0, #thing categories + + # #stuff categories) to their names and colors. We have to replica of the + # same name and color under "thing_*" and "stuff_*" because the current + # visualization function in D2 handles thing and class classes differently + # due to some heuristic used in Panoptic FPN. We keep the same naming to + # enable reusing existing visualization functions. + thing_classes = [k["name"] for k in COCO_CATEGORIES if k["isthing"] == 1] + thing_colors = [k["color"] for k in COCO_CATEGORIES if k["isthing"] == 1] + stuff_classes = [k["name"] for k in COCO_CATEGORIES] + stuff_colors = [k["color"] for k in COCO_CATEGORIES] + + meta["thing_classes"] = thing_classes + meta["thing_colors"] = thing_colors + meta["stuff_classes"] = stuff_classes + meta["stuff_colors"] = stuff_colors + + # Convert category id for training: + # category id: like semantic segmentation, it is the class id for each + # pixel. Since there are some classes not used in evaluation, the category + # id is not always contiguous and thus we have two set of category ids: + # - original category id: category id in the original dataset, mainly + # used for evaluation. + # - contiguous category id: [0, #classes), in order to train the linear + # softmax classifier. + thing_dataset_id_to_contiguous_id = {} + stuff_dataset_id_to_contiguous_id = {} + + for i, cat in enumerate(COCO_CATEGORIES): + if cat["isthing"]: + thing_dataset_id_to_contiguous_id[cat["id"]] = i + # else: + # stuff_dataset_id_to_contiguous_id[cat["id"]] = i + + # in order to use sem_seg evaluator + stuff_dataset_id_to_contiguous_id[cat["id"]] = i + + meta["thing_dataset_id_to_contiguous_id"] = thing_dataset_id_to_contiguous_id + meta["stuff_dataset_id_to_contiguous_id"] = stuff_dataset_id_to_contiguous_id + + return meta + + +def load_coco_panoptic_json(json_file, instances_json, instances_name, image_dir, gt_dir, semseg_dir, meta): + """ + Args: + image_dir (str): path to the raw dataset. e.g., "~/coco/train2017". + gt_dir (str): path to the raw annotations. e.g., "~/coco/panoptic_train2017". + json_file (str): path to the json file. e.g., "~/coco/annotations/panoptic_train2017.json". + Returns: + list[dict]: a list of dicts in Detectron2 standard format. (See + `Using Custom Datasets `_ ) + """ + + def _convert_category_id(segment_info, meta): + if segment_info["category_id"] in meta["thing_dataset_id_to_contiguous_id"]: + segment_info["category_id"] = meta["thing_dataset_id_to_contiguous_id"][ + segment_info["category_id"] + ] + segment_info["isthing"] = True + else: + segment_info["category_id"] = meta["stuff_dataset_id_to_contiguous_id"][ + segment_info["category_id"] + ] + segment_info["isthing"] = False + return segment_info + + with PathManager.open(json_file) as f: + json_info = json.load(f) + + instance_data_dicts = load_coco_instance_json(instances_json, image_dir.replace("panoptic_", ""), instances_name) + + ret = [] + for ann in json_info["annotations"]: + image_id = int(ann["image_id"]) + # TODO: currently we assume image and label has the same filename but + # different extension, and images have extension ".jpg" for COCO. Need + # to make image extension a user-provided argument if we extend this + # function to support other COCO-like datasets. + image_file = os.path.join(image_dir, os.path.splitext(ann["file_name"])[0] + ".jpg") + label_file = os.path.join(gt_dir, ann["file_name"]) + sem_label_file = os.path.join(semseg_dir, ann["file_name"]) + segments_info = [_convert_category_id(x, meta) for x in ann["segments_info"]] + ret.append( + { + "file_name": image_file, + "image_id": image_id, + "pan_seg_file_name": label_file, + "sem_seg_file_name": sem_label_file, + "segments_info": segments_info, + "annotations": instance_data_dicts[image_id]["annotations"], + } + ) + assert len(ret), f"No images found in {image_dir}!" + assert PathManager.isfile(ret[0]["file_name"]), ret[0]["file_name"] + assert PathManager.isfile(ret[0]["pan_seg_file_name"]), ret[0]["pan_seg_file_name"] + assert PathManager.isfile(ret[0]["sem_seg_file_name"]), ret[0]["sem_seg_file_name"] + return ret + + +def register_coco_panoptic_annos_sem_seg( + name, metadata, image_root, panoptic_root, panoptic_json, sem_seg_root, instances_json, instances_name, +): + panoptic_name = name + delattr(MetadataCatalog.get(panoptic_name), "thing_classes") + delattr(MetadataCatalog.get(panoptic_name), "thing_colors") + MetadataCatalog.get(panoptic_name).set( + thing_classes=metadata["thing_classes"], + thing_colors=metadata["thing_colors"], + # thing_dataset_id_to_contiguous_id=metadata["thing_dataset_id_to_contiguous_id"], + ) + + # the name is "coco_2017_train_panoptic_with_sem_seg" and "coco_2017_val_panoptic_with_sem_seg" + semantic_name = name + "_with_sem_seg" + DatasetCatalog.register( + semantic_name, + lambda: load_coco_panoptic_json(panoptic_json, instances_json, instances_name, image_root, panoptic_root, sem_seg_root, metadata), + ) + MetadataCatalog.get(semantic_name).set( + sem_seg_root=sem_seg_root, + panoptic_root=panoptic_root, + image_root=image_root, + panoptic_json=panoptic_json, + json_file=instances_json, + evaluator_type="coco_panoptic_seg", + ignore_label=255, + label_divisor=1000, + **metadata, + ) + + +def register_all_coco_panoptic_annos_sem_seg(root): + for ( + prefix, + (panoptic_root, panoptic_json, semantic_root), + ) in _PREDEFINED_SPLITS_COCO_PANOPTIC.items(): + + prefix_instances = prefix[: -len("_panoptic")] + instances_meta = MetadataCatalog.get(prefix_instances) + image_root, instances_json = instances_meta.image_root, instances_meta.json_file + + if 'val' in instances_json: + instances_json = instances_json.replace('instances_', 'panoptic2instances_') + + register_coco_panoptic_annos_sem_seg( + prefix, + get_metadata(), + image_root, + os.path.join(root, panoptic_root), + os.path.join(root, panoptic_json), + os.path.join(root, semantic_root), + instances_json, + prefix_instances, + ) + + +_root = os.getenv("DETECTRON2_DATASETS", "datasets") +register_all_coco_panoptic_annos_sem_seg(_root) diff --git a/annotator/oneformer/oneformer/data/tokenizer.py b/annotator/oneformer/oneformer/data/tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..21103dbfdcd77a3bf19ed0489c21c1b85ac61b87 --- /dev/null +++ b/annotator/oneformer/oneformer/data/tokenizer.py @@ -0,0 +1,200 @@ +# ------------------------------------------------------------------------- +# MIT License +# +# Copyright (c) 2021 OpenAI +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# Modified by Jiarui Xu +# ------------------------------------------------------------------------- + +import gzip +import html +import os +from functools import lru_cache + +import ftfy +import regex as re +import torch + + +@lru_cache() +def default_bpe(): + return os.path.join(os.path.dirname(os.path.abspath(__file__)), 'bpe_simple_vocab_16e6.txt') + +@lru_cache() +def bytes_to_unicode(): + """Returns list of utf-8 byte and a corresponding list of unicode strings. + + The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab + if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for decent + coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup tables + between utf-8 bytes and unicode strings. And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord('!'), ord('~') + 1)) + list(range(ord('¡'), ord('¬') + 1)) + list(range(ord('®'), ord('ÿ') + 1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8 + n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """Return set of symbol pairs in a word. + + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +def basic_clean(text): + text = ftfy.fix_text(text) + text = html.unescape(html.unescape(text)) + return text.strip() + + +def whitespace_clean(text): + text = re.sub(r'\s+', ' ', text) + text = text.strip() + return text + +class Tokenize: + + def __init__(self, tokenizer, max_seq_len=77, truncate=True): + self.tokenizer = tokenizer + self.max_seq_len = max_seq_len + self.truncate = truncate + + def __call__(self, texts): + expanded_dim = False + if isinstance(texts, str): + texts = [texts] + expanded_dim = True + + sot_token = self.tokenizer.encoder['<|startoftext|>'] + eot_token = self.tokenizer.encoder['<|endoftext|>'] + all_tokens = [[sot_token] + self.tokenizer.encode(text) + [eot_token] for text in texts] + result = torch.zeros(len(all_tokens), self.max_seq_len, dtype=torch.long) + + for i, tokens in enumerate(all_tokens): + if len(tokens) > self.max_seq_len: + if self.truncate: + tokens = tokens[:self.max_seq_len] + tokens[-1] = eot_token + else: + raise RuntimeError(f'Input {texts[i]} is too long for context length {self.max_seq_len}') + result[i, :len(tokens)] = torch.tensor(tokens) + + if expanded_dim: + return result[0] + + return result + + +class SimpleTokenizer(object): + + def __init__(self, bpe_path: str = default_bpe()): + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + + with open(bpe_path, encoding='UTF-8') as f: + contents = f.readlines() + merges = [] + for cnt in contents: + merges.append(cnt.split('\n')[0]) + merges.append("") + + # merges = gzip.open(bpe_path).read().decode('utf-8').split('\n') + merges = merges[1:49152 - 256 - 2 + 1] + merges = [tuple(merge.split()) for merge in merges] + vocab = list(bytes_to_unicode().values()) + vocab = vocab + [v + '' for v in vocab] + for merge in merges: + vocab.append(''.join(merge)) + vocab.extend(['<|startoftext|>', '<|endoftext|>']) + self.encoder = dict(zip(vocab, range(len(vocab)))) + self.decoder = {v: k for k, v in self.encoder.items()} + self.bpe_ranks = dict(zip(merges, range(len(merges)))) + self.cache = {'<|startoftext|>': '<|startoftext|>', '<|endoftext|>': '<|endoftext|>'} + self.pat = re.compile( + r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", + re.IGNORECASE) + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token[:-1]) + (token[-1] + '', ) + pairs = get_pairs(word) + + if not pairs: + return token + '' + + while True: + bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float('inf'))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except: # noqa: E722 + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word) - 1 and word[i + 1] == second: + new_word.append(first + second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = ' '.join(word) + self.cache[token] = word + return word + + def encode(self, text): + bpe_tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8')) + bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' ')) + return bpe_tokens + + def decode(self, tokens): + text = ''.join([self.decoder[token] for token in tokens]) + text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors='replace').replace('', ' ') + return text \ No newline at end of file diff --git a/annotator/oneformer/oneformer/demo/colormap.py b/annotator/oneformer/oneformer/demo/colormap.py new file mode 100644 index 0000000000000000000000000000000000000000..3eff9a46d37a1926c48ef0ad6e3308128438140f --- /dev/null +++ b/annotator/oneformer/oneformer/demo/colormap.py @@ -0,0 +1,170 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +""" +An awesome colormap for really neat visualizations. +Copied from Detectron, and removed gray colors. +""" + +import numpy as np +import random +random.seed(0) + +__all__ = ["colormap", "random_color", "random_colors"] + +# fmt: off +# RGB: +# _COLORS = np.array( +# [ +# 0.000, 0.447, 0.741, +# 0.850, 0.325, 0.098, +# 0.929, 0.694, 0.125, +# 0.494, 0.184, 0.556, +# 0.466, 0.674, 0.188, +# 0.301, 0.745, 0.933, +# 0.635, 0.078, 0.184, +# 0.300, 0.300, 0.300, +# 0.600, 0.600, 0.600, +# 1.000, 0.000, 0.000, +# 1.000, 0.500, 0.000, +# 0.749, 0.749, 0.000, +# 0.000, 1.000, 0.000, +# 0.000, 0.000, 1.000, +# 0.667, 0.000, 1.000, +# 0.333, 0.333, 0.000, +# 0.333, 0.667, 0.000, +# 0.333, 1.000, 0.000, +# 0.667, 0.333, 0.000, +# 0.667, 0.667, 0.000, +# 0.667, 1.000, 0.000, +# 1.000, 0.333, 0.000, +# 1.000, 0.667, 0.000, +# 1.000, 1.000, 0.000, +# 0.000, 0.333, 0.500, +# 0.000, 0.667, 0.500, +# 0.000, 1.000, 0.500, +# 0.333, 0.000, 0.500, +# 0.333, 0.333, 0.500, +# 0.333, 0.667, 0.500, +# 0.333, 1.000, 0.500, +# 0.667, 0.000, 0.500, +# 0.667, 0.333, 0.500, +# 0.667, 0.667, 0.500, +# 0.667, 1.000, 0.500, +# 1.000, 0.000, 0.500, +# 1.000, 0.333, 0.500, +# 1.000, 0.667, 0.500, +# 1.000, 1.000, 0.500, +# 0.000, 0.333, 1.000, +# 0.000, 0.667, 1.000, +# 0.000, 1.000, 1.000, +# 0.333, 0.000, 1.000, +# 0.333, 0.333, 1.000, +# 0.333, 0.667, 1.000, +# 0.333, 1.000, 1.000, +# 0.667, 0.000, 1.000, +# 0.667, 0.333, 1.000, +# 0.667, 0.667, 1.000, +# 0.667, 1.000, 1.000, +# 1.000, 0.000, 1.000, +# 1.000, 0.333, 1.000, +# 1.000, 0.667, 1.000, +# 0.333, 0.000, 0.000, +# 0.500, 0.000, 0.000, +# 0.667, 0.000, 0.000, +# 0.833, 0.000, 0.000, +# 1.000, 0.000, 0.000, +# 0.000, 0.167, 0.000, +# 0.000, 0.333, 0.000, +# 0.000, 0.500, 0.000, +# 0.000, 0.667, 0.000, +# 0.000, 0.833, 0.000, +# 0.000, 1.000, 0.000, +# 0.000, 0.000, 0.167, +# 0.000, 0.000, 0.333, +# 0.000, 0.000, 0.500, +# 0.000, 0.000, 0.667, +# 0.000, 0.000, 0.833, +# 0.000, 0.000, 1.000, +# 0.000, 0.000, 0.000, +# 0.143, 0.143, 0.143, +# 0.857, 0.857, 0.857, +# 1.000, 1.000, 1.000 +# ] +# ).astype(np.float32).reshape(-1, 3) +# fmt: on + +_COLORS = [] + + +def gen_color(): + color = tuple(np.round(np.random.choice(range(256), size=3)/255, 3)) + if color not in _COLORS and np.mean(color) != 0.0: + _COLORS.append(color) + else: + gen_color() + + +for _ in range(300): + gen_color() + + +def colormap(rgb=False, maximum=255): + """ + Args: + rgb (bool): whether to return RGB colors or BGR colors. + maximum (int): either 255 or 1 + Returns: + ndarray: a float32 array of Nx3 colors, in range [0, 255] or [0, 1] + """ + assert maximum in [255, 1], maximum + c = _COLORS * maximum + if not rgb: + c = c[:, ::-1] + return c + + +def random_color(rgb=False, maximum=255): + """ + Args: + rgb (bool): whether to return RGB colors or BGR colors. + maximum (int): either 255 or 1 + Returns: + ndarray: a vector of 3 numbers + """ + idx = np.random.randint(0, len(_COLORS)) + ret = _COLORS[idx] * maximum + if not rgb: + ret = ret[::-1] + return ret + + +def random_colors(N, rgb=False, maximum=255): + """ + Args: + N (int): number of unique colors needed + rgb (bool): whether to return RGB colors or BGR colors. + maximum (int): either 255 or 1 + Returns: + ndarray: a list of random_color + """ + indices = random.sample(range(len(_COLORS)), N) + ret = [_COLORS[i] * maximum for i in indices] + if not rgb: + ret = [x[::-1] for x in ret] + return ret + + +if __name__ == "__main__": + import cv2 + + size = 100 + H, W = 10, 10 + canvas = np.random.rand(H * size, W * size, 3).astype("float32") + for h in range(H): + for w in range(W): + idx = h * W + w + if idx >= len(_COLORS): + break + canvas[h * size : (h + 1) * size, w * size : (w + 1) * size] = _COLORS[idx] + cv2.imshow("a", canvas) + cv2.waitKey(0) \ No newline at end of file diff --git a/annotator/oneformer/oneformer/demo/defaults.py b/annotator/oneformer/oneformer/demo/defaults.py new file mode 100644 index 0000000000000000000000000000000000000000..ba99129950ce16adba975f8138d73d6883865f42 --- /dev/null +++ b/annotator/oneformer/oneformer/demo/defaults.py @@ -0,0 +1,77 @@ +import torch +import annotator.oneformer.detectron2.data.transforms as T +from annotator.oneformer.detectron2.checkpoint import DetectionCheckpointer +from annotator.oneformer.detectron2.data import ( + MetadataCatalog, +) +from annotator.oneformer.detectron2.modeling import build_model + + +__all__ = [ + "DefaultPredictor", +] + + +class DefaultPredictor: + """ + Create a simple end-to-end predictor with the given config that runs on + single device for a single input image. + Compared to using the model directly, this class does the following additions: + 1. Load checkpoint from `cfg.MODEL.WEIGHTS`. + 2. Always take BGR image as the input and apply conversion defined by `cfg.INPUT.FORMAT`. + 3. Apply resizing defined by `cfg.INPUT.{MIN,MAX}_SIZE_TEST`. + 4. Take one input image and produce a single output, instead of a batch. + This is meant for simple demo purposes, so it does the above steps automatically. + This is not meant for benchmarks or running complicated inference logic. + If you'd like to do anything more complicated, please refer to its source code as + examples to build and use the model manually. + Attributes: + metadata (Metadata): the metadata of the underlying dataset, obtained from + cfg.DATASETS.TEST. + Examples: + :: + pred = DefaultPredictor(cfg) + inputs = cv2.imread("input.jpg") + outputs = pred(inputs) + """ + + def __init__(self, cfg): + self.cfg = cfg.clone() # cfg can be modified by model + self.model = build_model(self.cfg) + self.model.eval() + if len(cfg.DATASETS.TEST): + self.metadata = MetadataCatalog.get(cfg.DATASETS.TEST[0]) + + checkpointer = DetectionCheckpointer(self.model) + checkpointer.load(cfg.MODEL.WEIGHTS) + + self.aug = T.ResizeShortestEdge( + [cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MIN_SIZE_TEST], cfg.INPUT.MAX_SIZE_TEST + ) + + self.input_format = cfg.INPUT.FORMAT + assert self.input_format in ["RGB", "BGR"], self.input_format + + def __call__(self, original_image, task): + """ + Args: + original_image (np.ndarray): an image of shape (H, W, C) (in BGR order). + Returns: + predictions (dict): + the output of the model for one image only. + See :doc:`/tutorials/models` for details about the format. + """ + with torch.no_grad(): # https://github.com/sphinx-doc/sphinx/issues/4258 + # Apply pre-processing to image. + if self.input_format == "RGB": + # whether the model expects BGR inputs or RGB + original_image = original_image[:, :, ::-1] + height, width = original_image.shape[:2] + image = self.aug.get_transform(original_image).apply_image(original_image) + image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1)) + + task = f"The task is {task}" + + inputs = {"image": image, "height": height, "width": width, "task": task} + predictions = self.model([inputs])[0] + return predictions \ No newline at end of file diff --git a/annotator/oneformer/oneformer/demo/predictor.py b/annotator/oneformer/oneformer/demo/predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..4b2de6c7db1baca4ea5d234b1f1ae99d341eef65 --- /dev/null +++ b/annotator/oneformer/oneformer/demo/predictor.py @@ -0,0 +1,190 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# Copied from: https://github.com/facebookresearch/detectron2/blob/master/demo/predictor.py +import atexit +import bisect +import multiprocessing as mp +from collections import deque + +import cv2 +import torch + +from annotator.oneformer.detectron2.data import MetadataCatalog +from defaults import DefaultPredictor +from annotator.oneformer.detectron2.utils.video_visualizer import VideoVisualizer +from visualizer import ColorMode, Visualizer + + +class VisualizationDemo(object): + def __init__(self, cfg, instance_mode=ColorMode.IMAGE, parallel=False): + """ + Args: + cfg (CfgNode): + instance_mode (ColorMode): + parallel (bool): whether to run the model in different processes from visualization. + Useful since the visualization logic can be slow. + """ + self.metadata = MetadataCatalog.get( + cfg.DATASETS.TEST[0] if len(cfg.DATASETS.TEST) else "__unused" + ) + if 'cityscapes_fine_sem_seg_val' in cfg.DATASETS.TEST[0]: + from cityscapesscripts.helpers.labels import labels + stuff_colors = [k.color for k in labels if k.trainId != 255] + self.metadata = self.metadata.set(stuff_colors=stuff_colors) + self.cpu_device = torch.device("cpu") + self.instance_mode = instance_mode + + self.parallel = parallel + if parallel: + num_gpu = torch.cuda.device_count() + self.predictor = AsyncPredictor(cfg, num_gpus=num_gpu) + else: + self.predictor = DefaultPredictor(cfg) + + def run_on_image(self, image, task, sem_gt, pan_gt, ins_gt, box_gt): + """ + Args: + image (np.ndarray): an image of shape (H, W, C) (in BGR order). + This is the format used by OpenCV. + Returns: + predictions (dict): the output of the model. + vis_output (VisImage): the visualized image output. + """ + vis_output = None + # Convert image from OpenCV BGR format to Matplotlib RGB format. + image = image[:, :, ::-1] + vis_output = {} + + if task == 'panoptic': + visualizer = Visualizer(image, metadata=self.metadata, instance_mode=0) + predictions = self.predictor(image, "panoptic") + panoptic_seg, segments_info = predictions["panoptic_seg"] + vis_output['panoptic'] = visualizer.draw_panoptic_seg_predictions( + panoptic_seg.to(self.cpu_device), segments_info, alpha=1 + ) + + # visualizer = Visualizer(image, metadata=self.metadata, instance_mode=0) + # vis_output['pan_gt'] = visualizer.draw_panoptic_seg( + # pan_gt[0].to(self.cpu_device), pan_gt[1], alpha=1 + # ) + + if task == 'panoptic' or task == 'semantic': + visualizer = Visualizer(image, metadata=self.metadata, instance_mode=1) + predictions = self.predictor(image, "semantic") + vis_output['semantic'] = visualizer.draw_sem_seg( + predictions["sem_seg"].argmax(dim=0).to(self.cpu_device), alpha=1 + ) + + # visualizer = Visualizer(image, metadata=self.metadata, instance_mode=1) + # vis_output['gt_sem'] = visualizer.draw_sem_seg( + # sem_gt.to(self.cpu_device), alpha=1 + # ) + + if task == 'panoptic' or task == 'instance': + visualizer = Visualizer(image, metadata=self.metadata, instance_mode=2) + predictions = self.predictor(image, "instance") + instances = predictions["instances"].to(self.cpu_device) + vis_output['instance'] = visualizer.draw_instance_predictions(predictions=instances, alpha=1) + + if 'boxes' in predictions: + boxes, labels, scores = predictions["boxes"] + visualizer = Visualizer(image, False, metadata=self.metadata, instance_mode=0) + vis_output['boxes'] = visualizer.draw_box_predictions( + boxes.to(self.cpu_device), labels.to(self.cpu_device), scores.to(self.cpu_device)) + + + # visualizer = Visualizer(image, metadata=self.metadata, instance_mode=2) + # vis_output['ins_gt'] = visualizer.draw_instance_predictions(predictions=ins_gt.to(self.cpu_device), alpha=1) + # vis_output['input'] = visualizer.get_image(image) + + return predictions, vis_output + + +class AsyncPredictor: + """ + A predictor that runs the model asynchronously, possibly on >1 GPUs. + Because rendering the visualization takes considerably amount of time, + this helps improve throughput a little bit when rendering videos. + """ + + class _StopToken: + pass + + class _PredictWorker(mp.Process): + def __init__(self, cfg, task_queue, result_queue): + self.cfg = cfg + self.task_queue = task_queue + self.result_queue = result_queue + super().__init__() + + def run(self): + predictor = DefaultPredictor(self.cfg) + + while True: + task = self.task_queue.get() + if isinstance(task, AsyncPredictor._StopToken): + break + idx, data = task + result = predictor(data) + self.result_queue.put((idx, result)) + + def __init__(self, cfg, num_gpus: int = 1): + """ + Args: + cfg (CfgNode): + num_gpus (int): if 0, will run on CPU + """ + num_workers = max(num_gpus, 1) + self.task_queue = mp.Queue(maxsize=num_workers * 3) + self.result_queue = mp.Queue(maxsize=num_workers * 3) + self.procs = [] + for gpuid in range(max(num_gpus, 1)): + cfg = cfg.clone() + cfg.defrost() + cfg.MODEL.DEVICE = "cuda:{}".format(gpuid) if num_gpus > 0 else "cpu" + self.procs.append( + AsyncPredictor._PredictWorker(cfg, self.task_queue, self.result_queue) + ) + + self.put_idx = 0 + self.get_idx = 0 + self.result_rank = [] + self.result_data = [] + + for p in self.procs: + p.start() + atexit.register(self.shutdown) + + def put(self, image): + self.put_idx += 1 + self.task_queue.put((self.put_idx, image)) + + def get(self): + self.get_idx += 1 # the index needed for this request + if len(self.result_rank) and self.result_rank[0] == self.get_idx: + res = self.result_data[0] + del self.result_data[0], self.result_rank[0] + return res + + while True: + # make sure the results are returned in the correct order + idx, res = self.result_queue.get() + if idx == self.get_idx: + return res + insert = bisect.bisect(self.result_rank, idx) + self.result_rank.insert(insert, idx) + self.result_data.insert(insert, res) + + def __len__(self): + return self.put_idx - self.get_idx + + def __call__(self, image): + self.put(image) + return self.get() + + def shutdown(self): + for _ in self.procs: + self.task_queue.put(AsyncPredictor._StopToken()) + + @property + def default_buffer_size(self): + return len(self.procs) * 5 diff --git a/annotator/oneformer/oneformer/demo/visualizer.py b/annotator/oneformer/oneformer/demo/visualizer.py new file mode 100644 index 0000000000000000000000000000000000000000..469eb1ab7e95d7646a6a07bed2428a905efdc992 --- /dev/null +++ b/annotator/oneformer/oneformer/demo/visualizer.py @@ -0,0 +1,1350 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import colorsys +import logging +import math +import numpy as np +from enum import Enum, unique +import cv2 +import matplotlib as mpl +import matplotlib.colors as mplc +import matplotlib.figure as mplfigure +import pycocotools.mask as mask_util +import torch +from matplotlib.backends.backend_agg import FigureCanvasAgg +from PIL import Image + +from annotator.oneformer.detectron2.data import MetadataCatalog +from annotator.oneformer.detectron2.structures import BitMasks, Boxes, BoxMode, Keypoints, PolygonMasks, RotatedBoxes +from annotator.oneformer.detectron2.utils.file_io import PathManager +import random +random.seed(0) +from .colormap import random_color, _COLORS +logger = logging.getLogger(__name__) + +__all__ = ["ColorMode", "VisImage", "Visualizer"] + + +_SMALL_OBJECT_AREA_THRESH = 1000 +_LARGE_MASK_AREA_THRESH = 120000 +_OFF_WHITE = (1.0, 1.0, 1.0) +_BLACK = (0, 0, 0) +_RED = (1.0, 0, 0) + +_KEYPOINT_THRESHOLD = 0.05 + + +def instance_color(rgb=False, idx=1, maximum=255): + """ + Args: + rgb (bool): whether to return RGB colors or BGR colors. + maximum (int): either 255 or 1 + Returns: + ndarray: a vector of 3 numbers + """ + ret = _COLORS[idx] * maximum + if not rgb: + ret = ret[::-1] + return ret + +@unique +class ColorMode(Enum): + """ + Enum of different color modes to use for instance visualizations. + """ + + IMAGE = 0 + """ + Picks a random color for every instance and overlay segmentations with low opacity. + """ + SEGMENTATION = 1 + """ + Let instances of the same category have similar colors + (from metadata.thing_colors), and overlay them with + high opacity. This provides more attention on the quality of segmentation. + """ + IMAGE_BW = 2 + """ + Same as IMAGE, but convert all areas without masks to gray-scale. + Only available for drawing per-instance mask predictions. + """ + + +class GenericMask: + """ + Attribute: + polygons (list[ndarray]): list[ndarray]: polygons for this mask. + Each ndarray has format [x, y, x, y, ...] + mask (ndarray): a binary mask + """ + + def __init__(self, mask_or_polygons, height, width): + self._mask = self._polygons = self._has_holes = None + self.height = height + self.width = width + + m = mask_or_polygons + if isinstance(m, dict): + # RLEs + assert "counts" in m and "size" in m + if isinstance(m["counts"], list): # uncompressed RLEs + h, w = m["size"] + assert h == height and w == width + m = mask_util.frPyObjects(m, h, w) + self._mask = mask_util.decode(m)[:, :] + return + + if isinstance(m, list): # list[ndarray] + self._polygons = [np.asarray(x).reshape(-1) for x in m] + return + + if isinstance(m, np.ndarray): # assumed to be a binary mask + assert m.shape[1] != 2, m.shape + assert m.shape == ( + height, + width, + ), f"mask shape: {m.shape}, target dims: {height}, {width}" + self._mask = m.astype("uint8") + return + + raise ValueError("GenericMask cannot handle object {} of type '{}'".format(m, type(m))) + + @property + def mask(self): + if self._mask is None: + self._mask = self.polygons_to_mask(self._polygons) + return self._mask + + @property + def polygons(self): + if self._polygons is None: + self._polygons, self._has_holes = self.mask_to_polygons(self._mask) + return self._polygons + + @property + def has_holes(self): + if self._has_holes is None: + if self._mask is not None: + self._polygons, self._has_holes = self.mask_to_polygons(self._mask) + else: + self._has_holes = False # if original format is polygon, does not have holes + return self._has_holes + + def mask_to_polygons(self, mask): + # cv2.RETR_CCOMP flag retrieves all the contours and arranges them to a 2-level + # hierarchy. External contours (boundary) of the object are placed in hierarchy-1. + # Internal contours (holes) are placed in hierarchy-2. + # cv2.CHAIN_APPROX_NONE flag gets vertices of polygons from contours. + mask = np.ascontiguousarray(mask) # some versions of cv2 does not support incontiguous arr + res = cv2.findContours(mask.astype("uint8"), cv2.RETR_CCOMP, cv2.CHAIN_APPROX_NONE) + hierarchy = res[-1] + if hierarchy is None: # empty mask + return [], False + has_holes = (hierarchy.reshape(-1, 4)[:, 3] >= 0).sum() > 0 + res = res[-2] + res = [x.flatten() for x in res] + # These coordinates from OpenCV are integers in range [0, W-1 or H-1]. + # We add 0.5 to turn them into real-value coordinate space. A better solution + # would be to first +0.5 and then dilate the returned polygon by 0.5. + res = [x + 0.5 for x in res if len(x) >= 6] + return res, has_holes + + def polygons_to_mask(self, polygons): + rle = mask_util.frPyObjects(polygons, self.height, self.width) + rle = mask_util.merge(rle) + return mask_util.decode(rle)[:, :] + + def area(self): + return self.mask.sum() + + def bbox(self): + p = mask_util.frPyObjects(self.polygons, self.height, self.width) + p = mask_util.merge(p) + bbox = mask_util.toBbox(p) + bbox[2] += bbox[0] + bbox[3] += bbox[1] + return bbox + + +class _PanopticPrediction: + """ + Unify different panoptic annotation/prediction formats + """ + + def __init__(self, panoptic_seg, segments_info, metadata=None): + if segments_info is None: + assert metadata is not None + # If "segments_info" is None, we assume "panoptic_img" is a + # H*W int32 image storing the panoptic_id in the format of + # category_id * label_divisor + instance_id. We reserve -1 for + # VOID label. + label_divisor = metadata.label_divisor + segments_info = [] + for panoptic_label in np.unique(panoptic_seg.numpy()): + if panoptic_label == -1: + # VOID region. + continue + pred_class = panoptic_label // label_divisor + isthing = pred_class in metadata.thing_dataset_id_to_contiguous_id.values() + segments_info.append( + { + "id": int(panoptic_label), + "category_id": int(pred_class), + "isthing": bool(isthing), + } + ) + del metadata + + self._seg = panoptic_seg + + self._sinfo = {s["id"]: s for s in segments_info} # seg id -> seg info + segment_ids, areas = torch.unique(panoptic_seg, sorted=True, return_counts=True) + areas = areas.numpy() + sorted_idxs = np.argsort(-areas) + self._seg_ids, self._seg_areas = segment_ids[sorted_idxs], areas[sorted_idxs] + self._seg_ids = self._seg_ids.tolist() + for sid, area in zip(self._seg_ids, self._seg_areas): + if sid in self._sinfo: + self._sinfo[sid]["area"] = float(area) + + def non_empty_mask(self): + """ + Returns: + (H, W) array, a mask for all pixels that have a prediction + """ + empty_ids = [] + for id in self._seg_ids: + if id not in self._sinfo: + empty_ids.append(id) + if len(empty_ids) == 0: + return np.zeros(self._seg.shape, dtype=np.uint8) + assert ( + len(empty_ids) == 1 + ), ">1 ids corresponds to no labels. This is currently not supported" + return (self._seg != empty_ids[0]).numpy().astype(np.bool) + + def semantic_masks(self): + for sid in self._seg_ids: + sinfo = self._sinfo.get(sid) + if sinfo is None or sinfo["isthing"]: + # Some pixels (e.g. id 0 in PanopticFPN) have no instance or semantic predictions. + continue + yield (self._seg == sid).numpy().astype(np.bool), sinfo + + def instance_masks(self): + for sid in self._seg_ids: + sinfo = self._sinfo.get(sid) + if sinfo is None or not sinfo["isthing"]: + continue + mask = (self._seg == sid).numpy().astype(np.bool) + if mask.sum() > 0: + yield mask, sinfo + + +def _create_text_labels(classes, scores, class_names, is_crowd=None): + """ + Args: + classes (list[int] or None): + scores (list[float] or None): + class_names (list[str] or None): + is_crowd (list[bool] or None): + Returns: + list[str] or None + """ + labels = None + if classes is not None: + if class_names is not None and len(class_names) > 0: + labels = [class_names[i] for i in classes] + else: + labels = [str(i) for i in classes] + if scores is not None: + if labels is None: + labels = ["{:.0f}%".format(s * 100) for s in scores] + else: + labels = ["{} {:.0f}%".format(l, s * 100) for l, s in zip(labels, scores)] + if labels is not None and is_crowd is not None: + labels = [l + ("|crowd" if crowd else "") for l, crowd in zip(labels, is_crowd)] + return labels + + +class VisImage: + def __init__(self, img, scale=1.0): + """ + Args: + img (ndarray): an RGB image of shape (H, W, 3) in range [0, 255]. + scale (float): scale the input image + """ + self.img = img + self.scale = scale + self.width, self.height = img.shape[1], img.shape[0] + self._setup_figure(img) + + def _setup_figure(self, img): + """ + Args: + Same as in :meth:`__init__()`. + Returns: + fig (matplotlib.pyplot.figure): top level container for all the image plot elements. + ax (matplotlib.pyplot.Axes): contains figure elements and sets the coordinate system. + """ + fig = mplfigure.Figure(frameon=False) + self.dpi = fig.get_dpi() + # add a small 1e-2 to avoid precision lost due to matplotlib's truncation + # (https://github.com/matplotlib/matplotlib/issues/15363) + fig.set_size_inches( + (self.width * self.scale + 1e-2) / self.dpi, + (self.height * self.scale + 1e-2) / self.dpi, + ) + self.canvas = FigureCanvasAgg(fig) + # self.canvas = mpl.backends.backend_cairo.FigureCanvasCairo(fig) + ax = fig.add_axes([0.0, 0.0, 1.0, 1.0]) + ax.axis("off") + self.fig = fig + self.ax = ax + self.reset_image(img) + + def reset_image(self, img): + """ + Args: + img: same as in __init__ + """ + img = img.astype("uint8") + self.ax.imshow(img, extent=(0, self.width, self.height, 0), interpolation="nearest") + + def save(self, filepath): + """ + Args: + filepath (str): a string that contains the absolute path, including the file name, where + the visualized image will be saved. + """ + self.fig.savefig(filepath) + + def get_image(self): + """ + Returns: + ndarray: + the visualized image of shape (H, W, 3) (RGB) in uint8 type. + The shape is scaled w.r.t the input image using the given `scale` argument. + """ + canvas = self.canvas + s, (width, height) = canvas.print_to_buffer() + # buf = io.BytesIO() # works for cairo backend + # canvas.print_rgba(buf) + # width, height = self.width, self.height + # s = buf.getvalue() + + buffer = np.frombuffer(s, dtype="uint8") + + img_rgba = buffer.reshape(height, width, 4) + rgb, alpha = np.split(img_rgba, [3], axis=2) + return rgb.astype("uint8") + + +class Visualizer: + """ + Visualizer that draws data about detection/segmentation on images. + It contains methods like `draw_{text,box,circle,line,binary_mask,polygon}` + that draw primitive objects to images, as well as high-level wrappers like + `draw_{instance_predictions,sem_seg,panoptic_seg_predictions,dataset_dict}` + that draw composite data in some pre-defined style. + Note that the exact visualization style for the high-level wrappers are subject to change. + Style such as color, opacity, label contents, visibility of labels, or even the visibility + of objects themselves (e.g. when the object is too small) may change according + to different heuristics, as long as the results still look visually reasonable. + To obtain a consistent style, you can implement custom drawing functions with the + abovementioned primitive methods instead. If you need more customized visualization + styles, you can process the data yourself following their format documented in + tutorials (:doc:`/tutorials/models`, :doc:`/tutorials/datasets`). This class does not + intend to satisfy everyone's preference on drawing styles. + This visualizer focuses on high rendering quality rather than performance. It is not + designed to be used for real-time applications. + """ + + # TODO implement a fast, rasterized version using OpenCV + + def __init__(self, img_rgb, is_img=True, metadata=None, scale=1.0, instance_mode=ColorMode.IMAGE): + """ + Args: + img_rgb: a numpy array of shape (H, W, C), where H and W correspond to + the height and width of the image respectively. C is the number of + color channels. The image is required to be in RGB format since that + is a requirement of the Matplotlib library. The image is also expected + to be in the range [0, 255]. + metadata (Metadata): dataset metadata (e.g. class names and colors) + instance_mode (ColorMode): defines one of the pre-defined style for drawing + instances on an image. + """ + if is_img: + self.img = np.asarray(img_rgb).clip(0, 255).astype(np.uint8) + else: + self.img = np.zeros_like(img_rgb).clip(0, 255).astype(np.uint8) + 255 + if metadata is None: + metadata = MetadataCatalog.get("__nonexist__") + self.metadata = metadata + self.output = VisImage(self.img, scale=scale) + self.cpu_device = torch.device("cpu") + + # too small texts are useless, therefore clamp to 9 + self._default_font_size = max( + np.sqrt(self.output.height * self.output.width) // 90, 10 // scale + ) + self._instance_mode = instance_mode + self.keypoint_threshold = _KEYPOINT_THRESHOLD + + def get_image(self, img): + img = np.asarray(img).clip(0, 255).astype(np.uint8) + return VisImage(img, scale=1.0) + + def draw_box_predictions( + self, + boxes=None, + labels=None, + scores=None, + assigned_colors=None + ): + """ + Args: + boxes (Boxes, RotatedBoxes or ndarray): either a :class:`Boxes`, + or an Nx4 numpy array of XYXY_ABS format for the N objects in a single image, + or a :class:`RotatedBoxes`, + or an Nx5 numpy array of (x_center, y_center, width, height, angle_degrees) format + for the N objects in a single image, + labels (list[str]): the text to be displayed for each instance. + assigned_colors (list[matplotlib.colors]): a list of colors, where each color + corresponds to each mask or box in the image. Refer to 'matplotlib.colors' + for full list of formats that the colors are accepted in. + Returns: + output (VisImage): image object with visualizations. + """ + num_instances = 0 + boxes = self._convert_boxes(boxes) + classes = labels.tolist() + scores = scores.tolist() + labels = _create_text_labels(classes, scores, self.metadata.get("stuff_classes", None)) + num_instances = len(boxes) + assert len(labels) == num_instances + if assigned_colors is None: + # assigned_colors = [random_color(rgb=True, maximum=1) for _ in range(num_instances)] + assigned_colors = [instance_color(rgb=True, idx=i, maximum=1) for i in range(num_instances)] + if num_instances == 0: + return self.output + + # Display in largest to smallest order to reduce occlusion. + areas = None + areas = np.prod(boxes[:, 2:] - boxes[:, :2], axis=1) + + if areas is not None: + sorted_idxs = np.argsort(-areas).tolist() + # Re-order overlapped instances in descending order. + boxes = boxes[sorted_idxs] if boxes is not None else None + labels = [labels[k] for k in sorted_idxs] if labels is not None else None + assigned_colors = [assigned_colors[idx] for idx in sorted_idxs] + + for i in range(num_instances): + color = assigned_colors[i] + if boxes is not None: + self.draw_box(boxes[i], edge_color=color) + + if labels is not None: + # first get a box + if boxes is not None: + x0, y0, x1, y1 = boxes[i] + text_pos = (x0, y0) # if drawing boxes, put text on the box corner. + horiz_align = "left" + else: + continue # drawing the box confidence for keypoints isn't very useful. + # for small objects, draw text at the side to avoid occlusion + instance_area = (y1 - y0) * (x1 - x0) + if ( + instance_area < _SMALL_OBJECT_AREA_THRESH * self.output.scale + or y1 - y0 < 40 * self.output.scale + ): + if y1 >= self.output.height - 5: + text_pos = (x1, y0) + else: + text_pos = (x0, y1) + + height_ratio = (y1 - y0) / np.sqrt(self.output.height * self.output.width) + lighter_color = self._change_color_brightness(color, brightness_factor=0.7) + font_size = ( + np.clip((height_ratio - 0.02) / 0.08 + 1, 1.2, 2) + * 0.5 + * self._default_font_size + ) + self.draw_text( + labels[i], + text_pos, + color=lighter_color, + horizontal_alignment=horiz_align, + font_size=font_size, + ) + + return self.output + + + def draw_instance_predictions(self, predictions, alpha=0.8, is_text=True): + """ + Draw instance-level prediction results on an image. + Args: + predictions (Instances): the output of an instance detection/segmentation + model. Following fields will be used to draw: + "pred_boxes", "pred_classes", "scores", "pred_masks" (or "pred_masks_rle"). + Returns: + output (VisImage): image object with visualizations. + """ + boxes = predictions.pred_boxes if predictions.has("pred_boxes") else None + scores = predictions.scores if predictions.has("scores") else None + classes = predictions.pred_classes.tolist() if predictions.has("pred_classes") else None + labels = _create_text_labels(classes, scores, self.metadata.get("stuff_classes", None)) + keypoints = predictions.pred_keypoints if predictions.has("pred_keypoints") else None + + if predictions.has("pred_masks"): + masks = np.asarray(predictions.pred_masks) + masks = [GenericMask(x, self.output.height, self.output.width) for x in masks] + else: + masks = None + + if self._instance_mode == ColorMode.SEGMENTATION and self.metadata.get("stuff_colors"): + # colors = [ + # self._jitter([x / 255 for x in self.metadata.thing_colors[c]]) for c in classes + # ] + colors = [ + instance_color(rgb=True, idx=c, maximum=1) for c in classes + ] + else: + colors = None + + if self._instance_mode == ColorMode.IMAGE_BW: + self.output.reset_image( + self._create_grayscale_image( + (predictions.pred_masks.any(dim=0) > 0).numpy() + if predictions.has("pred_masks") + else None + ) + ) + + self.overlay_instances( + masks=masks, + boxes=boxes, + labels=labels, + keypoints=keypoints, + assigned_colors=colors, + alpha=alpha, + is_text=is_text, + ) + return self.output + + def draw_sem_seg(self, sem_seg, area_threshold=None, alpha=0.8, is_text=True, edge_color=_OFF_WHITE): + """ + Draw semantic segmentation predictions/labels. + Args: + sem_seg (Tensor or ndarray): the segmentation of shape (H, W). + Each value is the integer label of the pixel. + area_threshold (int): segments with less than `area_threshold` are not drawn. + alpha (float): the larger it is, the more opaque the segmentations are. + Returns: + output (VisImage): image object with visualizations. + """ + if isinstance(sem_seg, torch.Tensor): + sem_seg = sem_seg.numpy() + labels, areas = np.unique(sem_seg, return_counts=True) + sorted_idxs = np.argsort(-areas).tolist() + labels = labels[sorted_idxs] + for label in filter(lambda l: l < len(self.metadata.stuff_classes), labels): + try: + mask_color = [x / 255 for x in self.metadata.stuff_colors[label]] + except (AttributeError, IndexError): + mask_color = None + + binary_mask = (sem_seg == label).astype(np.uint8) + text = self.metadata.stuff_classes[label] + self.draw_binary_mask( + binary_mask, + color=mask_color, + edge_color=edge_color, + text=text, + alpha=alpha, + area_threshold=area_threshold, + is_text=is_text, + ) + return self.output + + def draw_panoptic_seg(self, panoptic_seg, segments_info, area_threshold=None, alpha=0.7, is_text=True,): + """ + Draw panoptic prediction annotations or results. + Args: + panoptic_seg (Tensor): of shape (height, width) where the values are ids for each + segment. + segments_info (list[dict] or None): Describe each segment in `panoptic_seg`. + If it is a ``list[dict]``, each dict contains keys "id", "category_id". + If None, category id of each pixel is computed by + ``pixel // metadata.label_divisor``. + area_threshold (int): stuff segments with less than `area_threshold` are not drawn. + Returns: + output (VisImage): image object with visualizations. + """ + pred = _PanopticPrediction(panoptic_seg, segments_info, self.metadata) + + if self._instance_mode == ColorMode.IMAGE_BW: + self.output.reset_image(self._create_grayscale_image(pred.non_empty_mask())) + + # draw mask for all semantic segments first i.e. "stuff" + for mask, sinfo in pred.semantic_masks(): + category_idx = sinfo["category_id"] + try: + mask_color = [x / 255 for x in self.metadata.stuff_colors[category_idx]] + except AttributeError: + mask_color = None + + text = self.metadata.stuff_classes[category_idx] + self.draw_binary_mask( + mask, + color=mask_color, + edge_color=_OFF_WHITE, + text=text, + alpha=alpha, + area_threshold=area_threshold, + is_text=is_text, + ) + + # draw mask for all instances second + all_instances = list(pred.instance_masks()) + if len(all_instances) == 0: + return self.output + masks, sinfo = list(zip(*all_instances)) + category_ids = [x["category_id"] for x in sinfo] + + try: + scores = [x["score"] for x in sinfo] + except KeyError: + scores = None + labels = _create_text_labels( + category_ids, scores, self.metadata.stuff_classes, [x.get("iscrowd", 0) for x in sinfo] + ) + + try: + colors = [ + self._jitter([x / 255 for x in self.metadata.stuff_colors[c]]) for c in category_ids + ] + except AttributeError: + colors = None + self.overlay_instances(masks=masks, labels=labels, assigned_colors=colors, alpha=alpha, is_text=is_text) + + return self.output + + draw_panoptic_seg_predictions = draw_panoptic_seg # backward compatibility + + def draw_dataset_dict(self, dic): + """ + Draw annotations/segmentaions in Detectron2 Dataset format. + Args: + dic (dict): annotation/segmentation data of one image, in Detectron2 Dataset format. + Returns: + output (VisImage): image object with visualizations. + """ + annos = dic.get("annotations", None) + if annos: + if "segmentation" in annos[0]: + masks = [x["segmentation"] for x in annos] + else: + masks = None + if "keypoints" in annos[0]: + keypts = [x["keypoints"] for x in annos] + keypts = np.array(keypts).reshape(len(annos), -1, 3) + else: + keypts = None + + boxes = [ + BoxMode.convert(x["bbox"], x["bbox_mode"], BoxMode.XYXY_ABS) + if len(x["bbox"]) == 4 + else x["bbox"] + for x in annos + ] + + colors = None + category_ids = [x["category_id"] for x in annos] + if self._instance_mode == ColorMode.SEGMENTATION and self.metadata.get("stuff_colors"): + colors = [ + self._jitter([x / 255 for x in self.metadata.stuff_colors[c]]) + for c in category_ids + ] + names = self.metadata.get("stuff_classes", None) + labels = _create_text_labels( + category_ids, + scores=None, + class_names=names, + is_crowd=[x.get("iscrowd", 0) for x in annos], + ) + self.overlay_instances( + labels=labels, boxes=boxes, masks=masks, keypoints=keypts, assigned_colors=colors + ) + + sem_seg = dic.get("sem_seg", None) + if sem_seg is None and "sem_seg_file_name" in dic: + with PathManager.open(dic["sem_seg_file_name"], "rb") as f: + sem_seg = Image.open(f) + sem_seg = np.asarray(sem_seg, dtype="uint8") + if sem_seg is not None: + self.draw_sem_seg(sem_seg, area_threshold=0, alpha=0.5) + + pan_seg = dic.get("pan_seg", None) + # if pan_seg is None and "pan_seg_file_name" in dic: + # with PathManager.open(dic["pan_seg_file_name"], "rb") as f: + # pan_seg = Image.open(f) + # pan_seg = np.asarray(pan_seg) + # from panopticapi.utils import rgb2id + # + # pan_seg = rgb2id(pan_seg) + if pan_seg is not None: + segments_info = dic["segments_info"] + pan_seg = torch.tensor(pan_seg) + self.draw_panoptic_seg(pan_seg, segments_info, area_threshold=0, alpha=0.5) + return self.output + + def overlay_instances( + self, + *, + boxes=None, + labels=None, + masks=None, + keypoints=None, + assigned_colors=None, + alpha=0.5, + is_text=True, + ): + """ + Args: + boxes (Boxes, RotatedBoxes or ndarray): either a :class:`Boxes`, + or an Nx4 numpy array of XYXY_ABS format for the N objects in a single image, + or a :class:`RotatedBoxes`, + or an Nx5 numpy array of (x_center, y_center, width, height, angle_degrees) format + for the N objects in a single image, + labels (list[str]): the text to be displayed for each instance. + masks (masks-like object): Supported types are: + * :class:`detectron2.structures.PolygonMasks`, + :class:`detectron2.structures.BitMasks`. + * list[list[ndarray]]: contains the segmentation masks for all objects in one image. + The first level of the list corresponds to individual instances. The second + level to all the polygon that compose the instance, and the third level + to the polygon coordinates. The third level should have the format of + [x0, y0, x1, y1, ..., xn, yn] (n >= 3). + * list[ndarray]: each ndarray is a binary mask of shape (H, W). + * list[dict]: each dict is a COCO-style RLE. + keypoints (Keypoint or array like): an array-like object of shape (N, K, 3), + where the N is the number of instances and K is the number of keypoints. + The last dimension corresponds to (x, y, visibility or score). + assigned_colors (list[matplotlib.colors]): a list of colors, where each color + corresponds to each mask or box in the image. Refer to 'matplotlib.colors' + for full list of formats that the colors are accepted in. + Returns: + output (VisImage): image object with visualizations. + """ + num_instances = 0 + if boxes is not None: + boxes = self._convert_boxes(boxes) + num_instances = len(boxes) + if masks is not None: + masks = self._convert_masks(masks) + if num_instances: + assert len(masks) == num_instances + else: + num_instances = len(masks) + if keypoints is not None: + if num_instances: + assert len(keypoints) == num_instances + else: + num_instances = len(keypoints) + keypoints = self._convert_keypoints(keypoints) + if labels is not None: + assert len(labels) == num_instances + if assigned_colors is None: + # assigned_colors = [random_color(rgb=True, maximum=1) for _ in range(num_instances)] + assigned_colors = [instance_color(rgb=True, idx=i, maximum=1) for i in range(num_instances)] + if num_instances == 0: + return self.output + if boxes is not None and boxes.shape[1] == 5: + return self.overlay_rotated_instances( + boxes=boxes, labels=labels, assigned_colors=assigned_colors + ) + + # Display in largest to smallest order to reduce occlusion. + areas = None + if boxes is not None: + areas = np.prod(boxes[:, 2:] - boxes[:, :2], axis=1) + elif masks is not None: + areas = np.asarray([x.area() for x in masks]) + + if areas is not None: + sorted_idxs = np.argsort(-areas).tolist() + # Re-order overlapped instances in descending order. + boxes = boxes[sorted_idxs] if boxes is not None else None + labels = [labels[k] for k in sorted_idxs] if labels is not None else None + masks = [masks[idx] for idx in sorted_idxs] if masks is not None else None + assigned_colors = [assigned_colors[idx] for idx in sorted_idxs] + keypoints = keypoints[sorted_idxs] if keypoints is not None else None + + for i in range(num_instances): + color = assigned_colors[i] + if boxes is not None: + self.draw_box(boxes[i], edge_color=color) + + if masks is not None: + for segment in masks[i].polygons: + self.draw_polygon(segment.reshape(-1, 2), color, alpha=alpha) + + if labels is not None: + # first get a box + if boxes is not None: + x0, y0, x1, y1 = boxes[i] + text_pos = (x0, y0) # if drawing boxes, put text on the box corner. + horiz_align = "left" + elif masks is not None: + # skip small mask without polygon + if len(masks[i].polygons) == 0: + continue + + x0, y0, x1, y1 = masks[i].bbox() + + # draw text in the center (defined by median) when box is not drawn + # median is less sensitive to outliers. + text_pos = np.median(masks[i].mask.nonzero(), axis=1)[::-1] + horiz_align = "center" + else: + continue # drawing the box confidence for keypoints isn't very useful. + # for small objects, draw text at the side to avoid occlusion + instance_area = (y1 - y0) * (x1 - x0) + if ( + instance_area < _SMALL_OBJECT_AREA_THRESH * self.output.scale + or y1 - y0 < 40 * self.output.scale + ): + if y1 >= self.output.height - 5: + text_pos = (x1, y0) + else: + text_pos = (x0, y1) + + height_ratio = (y1 - y0) / np.sqrt(self.output.height * self.output.width) + lighter_color = self._change_color_brightness(color, brightness_factor=0.7) + font_size = ( + np.clip((height_ratio - 0.02) / 0.08 + 1, 1.2, 2) + * 0.5 + * self._default_font_size + ) + if is_text: + self.draw_text( + labels[i], + text_pos, + color=lighter_color, + horizontal_alignment=horiz_align, + font_size=font_size, + ) + + # draw keypoints + if keypoints is not None: + for keypoints_per_instance in keypoints: + self.draw_and_connect_keypoints(keypoints_per_instance) + + return self.output + + def overlay_rotated_instances(self, boxes=None, labels=None, assigned_colors=None): + """ + Args: + boxes (ndarray): an Nx5 numpy array of + (x_center, y_center, width, height, angle_degrees) format + for the N objects in a single image. + labels (list[str]): the text to be displayed for each instance. + assigned_colors (list[matplotlib.colors]): a list of colors, where each color + corresponds to each mask or box in the image. Refer to 'matplotlib.colors' + for full list of formats that the colors are accepted in. + Returns: + output (VisImage): image object with visualizations. + """ + num_instances = len(boxes) + + if assigned_colors is None: + # assigned_colors = [random_color(rgb=True, maximum=1) for _ in range(num_instances)] + assigned_colors = [instance_color(rgb=True, idx=i, maximum=1) for i in range(num_instances)] + if num_instances == 0: + return self.output + + # Display in largest to smallest order to reduce occlusion. + if boxes is not None: + areas = boxes[:, 2] * boxes[:, 3] + + sorted_idxs = np.argsort(-areas).tolist() + # Re-order overlapped instances in descending order. + boxes = boxes[sorted_idxs] + labels = [labels[k] for k in sorted_idxs] if labels is not None else None + colors = [assigned_colors[idx] for idx in sorted_idxs] + + for i in range(num_instances): + self.draw_rotated_box_with_label( + boxes[i], edge_color=colors[i], label=labels[i] if labels is not None else None + ) + + return self.output + + def draw_and_connect_keypoints(self, keypoints): + """ + Draws keypoints of an instance and follows the rules for keypoint connections + to draw lines between appropriate keypoints. This follows color heuristics for + line color. + Args: + keypoints (Tensor): a tensor of shape (K, 3), where K is the number of keypoints + and the last dimension corresponds to (x, y, probability). + Returns: + output (VisImage): image object with visualizations. + """ + visible = {} + keypoint_names = self.metadata.get("keypoint_names") + for idx, keypoint in enumerate(keypoints): + + # draw keypoint + x, y, prob = keypoint + if prob > self.keypoint_threshold: + self.draw_circle((x, y), color=_RED) + if keypoint_names: + keypoint_name = keypoint_names[idx] + visible[keypoint_name] = (x, y) + + if self.metadata.get("keypoint_connection_rules"): + for kp0, kp1, color in self.metadata.keypoint_connection_rules: + if kp0 in visible and kp1 in visible: + x0, y0 = visible[kp0] + x1, y1 = visible[kp1] + color = tuple(x / 255.0 for x in color) + self.draw_line([x0, x1], [y0, y1], color=color) + + # draw lines from nose to mid-shoulder and mid-shoulder to mid-hip + # Note that this strategy is specific to person keypoints. + # For other keypoints, it should just do nothing + try: + ls_x, ls_y = visible["left_shoulder"] + rs_x, rs_y = visible["right_shoulder"] + mid_shoulder_x, mid_shoulder_y = (ls_x + rs_x) / 2, (ls_y + rs_y) / 2 + except KeyError: + pass + else: + # draw line from nose to mid-shoulder + nose_x, nose_y = visible.get("nose", (None, None)) + if nose_x is not None: + self.draw_line([nose_x, mid_shoulder_x], [nose_y, mid_shoulder_y], color=_RED) + + try: + # draw line from mid-shoulder to mid-hip + lh_x, lh_y = visible["left_hip"] + rh_x, rh_y = visible["right_hip"] + except KeyError: + pass + else: + mid_hip_x, mid_hip_y = (lh_x + rh_x) / 2, (lh_y + rh_y) / 2 + self.draw_line([mid_hip_x, mid_shoulder_x], [mid_hip_y, mid_shoulder_y], color=_RED) + return self.output + + """ + Primitive drawing functions: + """ + + def draw_text( + self, + text, + position, + *, + font_size=None, + color="g", + horizontal_alignment="center", + rotation=0, + ): + """ + Args: + text (str): class label + position (tuple): a tuple of the x and y coordinates to place text on image. + font_size (int, optional): font of the text. If not provided, a font size + proportional to the image width is calculated and used. + color: color of the text. Refer to `matplotlib.colors` for full list + of formats that are accepted. + horizontal_alignment (str): see `matplotlib.text.Text` + rotation: rotation angle in degrees CCW + Returns: + output (VisImage): image object with text drawn. + """ + if not font_size: + font_size = self._default_font_size + + # since the text background is dark, we don't want the text to be dark + color = np.maximum(list(mplc.to_rgb(color)), 0.2) + color[np.argmax(color)] = max(0.8, np.max(color)) + + x, y = position + self.output.ax.text( + x, + y, + text, + size=font_size * self.output.scale, + family="sans-serif", + bbox={"facecolor": "black", "alpha": 0.8, "pad": 0.7, "edgecolor": "none"}, + verticalalignment="top", + horizontalalignment=horizontal_alignment, + color=color, + zorder=10, + rotation=rotation, + ) + return self.output + + def draw_box(self, box_coord, alpha=1.0, edge_color="g", line_style="-"): + """ + Args: + box_coord (tuple): a tuple containing x0, y0, x1, y1 coordinates, where x0 and y0 + are the coordinates of the image's top left corner. x1 and y1 are the + coordinates of the image's bottom right corner. + alpha (float): blending efficient. Smaller values lead to more transparent masks. + edge_color: color of the outline of the box. Refer to `matplotlib.colors` + for full list of formats that are accepted. + line_style (string): the string to use to create the outline of the boxes. + Returns: + output (VisImage): image object with box drawn. + """ + x0, y0, x1, y1 = box_coord + width = x1 - x0 + height = y1 - y0 + + linewidth = 2 + + self.output.ax.add_patch( + mpl.patches.Rectangle( + (x0, y0), + width, + height, + fill=False, + edgecolor=edge_color, + linewidth=linewidth * self.output.scale, + alpha=alpha, + linestyle=line_style, + ) + ) + return self.output + + def draw_rotated_box_with_label( + self, rotated_box, alpha=0.5, edge_color="g", line_style="-", label=None + ): + """ + Draw a rotated box with label on its top-left corner. + Args: + rotated_box (tuple): a tuple containing (cnt_x, cnt_y, w, h, angle), + where cnt_x and cnt_y are the center coordinates of the box. + w and h are the width and height of the box. angle represents how + many degrees the box is rotated CCW with regard to the 0-degree box. + alpha (float): blending efficient. Smaller values lead to more transparent masks. + edge_color: color of the outline of the box. Refer to `matplotlib.colors` + for full list of formats that are accepted. + line_style (string): the string to use to create the outline of the boxes. + label (string): label for rotated box. It will not be rendered when set to None. + Returns: + output (VisImage): image object with box drawn. + """ + cnt_x, cnt_y, w, h, angle = rotated_box + area = w * h + # use thinner lines when the box is small + linewidth = self._default_font_size / ( + 6 if area < _SMALL_OBJECT_AREA_THRESH * self.output.scale else 3 + ) + + theta = angle * math.pi / 180.0 + c = math.cos(theta) + s = math.sin(theta) + rect = [(-w / 2, h / 2), (-w / 2, -h / 2), (w / 2, -h / 2), (w / 2, h / 2)] + # x: left->right ; y: top->down + rotated_rect = [(s * yy + c * xx + cnt_x, c * yy - s * xx + cnt_y) for (xx, yy) in rect] + for k in range(4): + j = (k + 1) % 4 + self.draw_line( + [rotated_rect[k][0], rotated_rect[j][0]], + [rotated_rect[k][1], rotated_rect[j][1]], + color=edge_color, + linestyle="--" if k == 1 else line_style, + linewidth=linewidth, + ) + + if label is not None: + text_pos = rotated_rect[1] # topleft corner + + height_ratio = h / np.sqrt(self.output.height * self.output.width) + label_color = self._change_color_brightness(edge_color, brightness_factor=0.7) + font_size = ( + np.clip((height_ratio - 0.02) / 0.08 + 1, 1.2, 2) * 0.5 * self._default_font_size + ) + self.draw_text(label, text_pos, color=label_color, font_size=font_size, rotation=angle) + + return self.output + + def draw_circle(self, circle_coord, color, radius=3): + """ + Args: + circle_coord (list(int) or tuple(int)): contains the x and y coordinates + of the center of the circle. + color: color of the polygon. Refer to `matplotlib.colors` for a full list of + formats that are accepted. + radius (int): radius of the circle. + Returns: + output (VisImage): image object with box drawn. + """ + x, y = circle_coord + self.output.ax.add_patch( + mpl.patches.Circle(circle_coord, radius=radius, fill=True, color=color) + ) + return self.output + + def draw_line(self, x_data, y_data, color, linestyle="-", linewidth=None): + """ + Args: + x_data (list[int]): a list containing x values of all the points being drawn. + Length of list should match the length of y_data. + y_data (list[int]): a list containing y values of all the points being drawn. + Length of list should match the length of x_data. + color: color of the line. Refer to `matplotlib.colors` for a full list of + formats that are accepted. + linestyle: style of the line. Refer to `matplotlib.lines.Line2D` + for a full list of formats that are accepted. + linewidth (float or None): width of the line. When it's None, + a default value will be computed and used. + Returns: + output (VisImage): image object with line drawn. + """ + if linewidth is None: + linewidth = self._default_font_size / 3 + linewidth = max(linewidth, 1) + self.output.ax.add_line( + mpl.lines.Line2D( + x_data, + y_data, + linewidth=linewidth * self.output.scale, + color=color, + linestyle=linestyle, + ) + ) + return self.output + + def draw_binary_mask( + self, binary_mask, color=None, *, edge_color=None, text=None, alpha=0.5, area_threshold=10, is_text=True, + ): + """ + Args: + binary_mask (ndarray): numpy array of shape (H, W), where H is the image height and + W is the image width. Each value in the array is either a 0 or 1 value of uint8 + type. + color: color of the mask. Refer to `matplotlib.colors` for a full list of + formats that are accepted. If None, will pick a random color. + edge_color: color of the polygon edges. Refer to `matplotlib.colors` for a + full list of formats that are accepted. + text (str): if None, will be drawn on the object + alpha (float): blending efficient. Smaller values lead to more transparent masks. + area_threshold (float): a connected component smaller than this area will not be shown. + Returns: + output (VisImage): image object with mask drawn. + """ + if color is None: + color = random_color(rgb=True, maximum=1) + color = mplc.to_rgb(color) + + has_valid_segment = False + binary_mask = binary_mask.astype("uint8") # opencv needs uint8 + mask = GenericMask(binary_mask, self.output.height, self.output.width) + shape2d = (binary_mask.shape[0], binary_mask.shape[1]) + + if not mask.has_holes: + # draw polygons for regular masks + for segment in mask.polygons: + area = mask_util.area(mask_util.frPyObjects([segment], shape2d[0], shape2d[1])) + if area < (area_threshold or 0): + continue + has_valid_segment = True + segment = segment.reshape(-1, 2) + self.draw_polygon(segment, color=color, edge_color=edge_color, alpha=alpha) + else: + # TODO: Use Path/PathPatch to draw vector graphics: + # https://stackoverflow.com/questions/8919719/how-to-plot-a-complex-polygon + rgba = np.zeros(shape2d + (4,), dtype="float32") + rgba[:, :, :3] = color + rgba[:, :, 3] = (mask.mask == 1).astype("float32") * alpha + has_valid_segment = True + self.output.ax.imshow(rgba, extent=(0, self.output.width, self.output.height, 0)) + + if is_text: + if text is not None and has_valid_segment: + lighter_color = self._change_color_brightness(color, brightness_factor=0.7) + self._draw_text_in_mask(binary_mask, text, lighter_color) + return self.output + + def draw_soft_mask(self, soft_mask, color=None, *, text=None, alpha=0.5): + """ + Args: + soft_mask (ndarray): float array of shape (H, W), each value in [0, 1]. + color: color of the mask. Refer to `matplotlib.colors` for a full list of + formats that are accepted. If None, will pick a random color. + text (str): if None, will be drawn on the object + alpha (float): blending efficient. Smaller values lead to more transparent masks. + Returns: + output (VisImage): image object with mask drawn. + """ + if color is None: + color = random_color(rgb=True, maximum=1) + color = mplc.to_rgb(color) + + shape2d = (soft_mask.shape[0], soft_mask.shape[1]) + rgba = np.zeros(shape2d + (4,), dtype="float32") + rgba[:, :, :3] = color + rgba[:, :, 3] = soft_mask * alpha + self.output.ax.imshow(rgba, extent=(0, self.output.width, self.output.height, 0)) + + if text is not None: + lighter_color = self._change_color_brightness(color, brightness_factor=0.7) + binary_mask = (soft_mask > 0.5).astype("uint8") + # self._draw_text_in_mask(binary_mask, text, lighter_color) + return self.output + + def draw_polygon(self, segment, color, edge_color=None, alpha=0.5): + """ + Args: + segment: numpy array of shape Nx2, containing all the points in the polygon. + color: color of the polygon. Refer to `matplotlib.colors` for a full list of + formats that are accepted. + edge_color: color of the polygon edges. Refer to `matplotlib.colors` for a + full list of formats that are accepted. If not provided, a darker shade + of the polygon color will be used instead. + alpha (float): blending efficient. Smaller values lead to more transparent masks. + Returns: + output (VisImage): image object with polygon drawn. + """ + if edge_color is None: + # make edge color darker than the polygon color + if alpha > 0.8: + edge_color = self._change_color_brightness(color, brightness_factor=-0.7) + else: + edge_color = color + edge_color = mplc.to_rgb(edge_color) + (1,) + + polygon = mpl.patches.Polygon( + segment, + fill=True, + facecolor=mplc.to_rgb(color) + (alpha,), + edgecolor=edge_color, + linewidth=max(self._default_font_size // 15 * self.output.scale, 1), + ) + self.output.ax.add_patch(polygon) + return self.output + + """ + Internal methods: + """ + + def _jitter(self, color): + """ + Randomly modifies given color to produce a slightly different color than the color given. + Args: + color (tuple[double]): a tuple of 3 elements, containing the RGB values of the color + picked. The values in the list are in the [0.0, 1.0] range. + Returns: + jittered_color (tuple[double]): a tuple of 3 elements, containing the RGB values of the + color after being jittered. The values in the list are in the [0.0, 1.0] range. + """ + color = mplc.to_rgb(color) + vec = np.random.rand(3) + # better to do it in another color space + vec = vec / np.linalg.norm(vec) * 0.5 + res = np.clip(vec + color, 0, 1) + return tuple(res) + + def _create_grayscale_image(self, mask=None): + """ + Create a grayscale version of the original image. + The colors in masked area, if given, will be kept. + """ + img_bw = self.img.astype("f4").mean(axis=2) + img_bw = np.stack([img_bw] * 3, axis=2) + if mask is not None: + img_bw[mask] = self.img[mask] + return img_bw + + def _change_color_brightness(self, color, brightness_factor): + """ + Depending on the brightness_factor, gives a lighter or darker color i.e. a color with + less or more saturation than the original color. + Args: + color: color of the polygon. Refer to `matplotlib.colors` for a full list of + formats that are accepted. + brightness_factor (float): a value in [-1.0, 1.0] range. A lightness factor of + 0 will correspond to no change, a factor in [-1.0, 0) range will result in + a darker color and a factor in (0, 1.0] range will result in a lighter color. + Returns: + modified_color (tuple[double]): a tuple containing the RGB values of the + modified color. Each value in the tuple is in the [0.0, 1.0] range. + """ + assert brightness_factor >= -1.0 and brightness_factor <= 1.0 + color = mplc.to_rgb(color) + polygon_color = colorsys.rgb_to_hls(*mplc.to_rgb(color)) + modified_lightness = polygon_color[1] + (brightness_factor * polygon_color[1]) + modified_lightness = 0.0 if modified_lightness < 0.0 else modified_lightness + modified_lightness = 1.0 if modified_lightness > 1.0 else modified_lightness + modified_color = colorsys.hls_to_rgb(polygon_color[0], modified_lightness, polygon_color[2]) + return modified_color + + def _convert_boxes(self, boxes): + """ + Convert different format of boxes to an NxB array, where B = 4 or 5 is the box dimension. + """ + if isinstance(boxes, Boxes) or isinstance(boxes, RotatedBoxes): + return boxes.tensor.detach().numpy() + else: + return np.asarray(boxes) + + def _convert_masks(self, masks_or_polygons): + """ + Convert different format of masks or polygons to a tuple of masks and polygons. + Returns: + list[GenericMask]: + """ + + m = masks_or_polygons + if isinstance(m, PolygonMasks): + m = m.polygons + if isinstance(m, BitMasks): + m = m.tensor.numpy() + if isinstance(m, torch.Tensor): + m = m.numpy() + ret = [] + for x in m: + if isinstance(x, GenericMask): + ret.append(x) + else: + ret.append(GenericMask(x, self.output.height, self.output.width)) + return ret + + def _draw_text_in_mask(self, binary_mask, text, color): + """ + Find proper places to draw text given a binary mask. + """ + # TODO sometimes drawn on wrong objects. the heuristics here can improve. + _num_cc, cc_labels, stats, centroids = cv2.connectedComponentsWithStats(binary_mask, 8) + if stats[1:, -1].size == 0: + return + largest_component_id = np.argmax(stats[1:, -1]) + 1 + + # draw text on the largest component, as well as other very large components. + for cid in range(1, _num_cc): + if cid == largest_component_id or stats[cid, -1] > _LARGE_MASK_AREA_THRESH: + # median is more stable than centroid + # center = centroids[largest_component_id] + center = np.median((cc_labels == cid).nonzero(), axis=1)[::-1] + self.draw_text(text, center, color=color) + + def _convert_keypoints(self, keypoints): + if isinstance(keypoints, Keypoints): + keypoints = keypoints.tensor + keypoints = np.asarray(keypoints) + return keypoints + + def get_output(self): + """ + Returns: + output (VisImage): the image output containing the visualizations added + to the image. + """ + return self.output \ No newline at end of file diff --git a/annotator/oneformer/oneformer/evaluation/__init__.py b/annotator/oneformer/oneformer/evaluation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..49f62369cca38a3c85884f8dea6baea674cb9060 --- /dev/null +++ b/annotator/oneformer/oneformer/evaluation/__init__.py @@ -0,0 +1,3 @@ +from .detection_coco_evaluator import * +from .coco_evaluator import * +from .cityscapes_evaluation import CityscapesInstanceEvaluator \ No newline at end of file diff --git a/annotator/oneformer/oneformer/evaluation/cityscapes_evaluation.py b/annotator/oneformer/oneformer/evaluation/cityscapes_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..19b1cb779e5f493cf75c8e6913a90da5c174735f --- /dev/null +++ b/annotator/oneformer/oneformer/evaluation/cityscapes_evaluation.py @@ -0,0 +1,201 @@ +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/detectron2/blob/main/detectron2/evaluation/cityscapes_evaluation.py +# Modified by Jitesh Jain (https://github.com/praeclarumjj3) +# ------------------------------------------------------------------------------ + +import glob +import logging +import numpy as np +import os +import tempfile +from collections import OrderedDict +import torch +from PIL import Image + +from annotator.oneformer.detectron2.data import MetadataCatalog +from annotator.oneformer.detectron2.utils import comm +from annotator.oneformer.detectron2.utils.file_io import PathManager + +from .evaluator import DatasetEvaluator + + +class CityscapesEvaluator(DatasetEvaluator): + """ + Base class for evaluation using cityscapes API. + """ + + def __init__(self, dataset_name): + """ + Args: + dataset_name (str): the name of the dataset. + It must have the following metadata associated with it: + "thing_classes", "gt_dir". + """ + self._metadata = MetadataCatalog.get(dataset_name) + self._cpu_device = torch.device("cpu") + self._logger = logging.getLogger(__name__) + + def reset(self): + self._working_dir = tempfile.TemporaryDirectory(prefix="cityscapes_eval_") + self._temp_dir = self._working_dir.name + # All workers will write to the same results directory + # TODO this does not work in distributed training + assert ( + comm.get_local_size() == comm.get_world_size() + ), "CityscapesEvaluator currently do not work with multiple machines." + self._temp_dir = comm.all_gather(self._temp_dir)[0] + if self._temp_dir != self._working_dir.name: + self._working_dir.cleanup() + self._logger.info( + "Writing cityscapes results to temporary directory {} ...".format(self._temp_dir) + ) + + +class CityscapesInstanceEvaluator(CityscapesEvaluator): + """ + Evaluate instance segmentation results on cityscapes dataset using cityscapes API. + + Note: + * It does not work in multi-machine distributed training. + * It contains a synchronization, therefore has to be used on all ranks. + * Only the main process runs evaluation. + """ + + def process(self, inputs, outputs): + from cityscapesscripts.helpers.labels import name2label + + for input, output in zip(inputs, outputs): + file_name = input["file_name"] + basename = os.path.splitext(os.path.basename(file_name))[0] + pred_txt = os.path.join(self._temp_dir, basename + "_pred.txt") + + if "instances" in output: + output = output["instances"].to(self._cpu_device) + num_instances = len(output) + with open(pred_txt, "w") as fout: + for i in range(num_instances): + pred_class = output.pred_classes[i] + classes = self._metadata.stuff_classes[pred_class] + class_id = name2label[classes].id + score = output.scores[i] + mask = output.pred_masks[i].numpy().astype("uint8") + png_filename = os.path.join( + self._temp_dir, basename + "_{}_{}.png".format(i, classes) + ) + + Image.fromarray(mask * 255).save(png_filename) + fout.write( + "{} {} {}\n".format(os.path.basename(png_filename), class_id, score) + ) + else: + # Cityscapes requires a prediction file for every ground truth image. + with open(pred_txt, "w") as fout: + pass + + def evaluate(self): + """ + Returns: + dict: has a key "segm", whose value is a dict of "AP" and "AP50". + """ + comm.synchronize() + if comm.get_rank() > 0: + return + import cityscapesscripts.evaluation.evalInstanceLevelSemanticLabeling as cityscapes_eval + + self._logger.info("Evaluating results under {} ...".format(self._temp_dir)) + + # set some global states in cityscapes evaluation API, before evaluating + cityscapes_eval.args.predictionPath = os.path.abspath(self._temp_dir) + cityscapes_eval.args.predictionWalk = None + cityscapes_eval.args.JSONOutput = False + cityscapes_eval.args.colorized = False + cityscapes_eval.args.gtInstancesFile = os.path.join(self._temp_dir, "gtInstances.json") + + # These lines are adopted from + # https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/evaluation/evalInstanceLevelSemanticLabeling.py # noqa + gt_dir = PathManager.get_local_path(self._metadata.gt_dir) + groundTruthImgList = glob.glob(os.path.join(gt_dir, "*", "*_gtFine_instanceIds.png")) + assert len( + groundTruthImgList + ), "Cannot find any ground truth images to use for evaluation. Searched for: {}".format( + cityscapes_eval.args.groundTruthSearch + ) + predictionImgList = [] + for gt in groundTruthImgList: + predictionImgList.append(cityscapes_eval.getPrediction(gt, cityscapes_eval.args)) + results = cityscapes_eval.evaluateImgLists( + predictionImgList, groundTruthImgList, cityscapes_eval.args + )["averages"] + + ret = OrderedDict() + ret["segm"] = {"AP": results["allAp"] * 100, "AP50": results["allAp50%"] * 100} + self._working_dir.cleanup() + return ret + + +class CityscapesSemSegEvaluator(CityscapesEvaluator): + """ + Evaluate semantic segmentation results on cityscapes dataset using cityscapes API. + + Note: + * It does not work in multi-machine distributed training. + * It contains a synchronization, therefore has to be used on all ranks. + * Only the main process runs evaluation. + """ + + def process(self, inputs, outputs): + from cityscapesscripts.helpers.labels import trainId2label + + for input, output in zip(inputs, outputs): + file_name = input["file_name"] + basename = os.path.splitext(os.path.basename(file_name))[0] + pred_filename = os.path.join(self._temp_dir, basename + "_pred.png") + + output = output["sem_seg"].argmax(dim=0).to(self._cpu_device).numpy() + pred = 255 * np.ones(output.shape, dtype=np.uint8) + for train_id, label in trainId2label.items(): + if label.ignoreInEval: + continue + pred[output == train_id] = label.id + Image.fromarray(pred).save(pred_filename) + + def evaluate(self): + comm.synchronize() + if comm.get_rank() > 0: + return + # Load the Cityscapes eval script *after* setting the required env var, + # since the script reads CITYSCAPES_DATASET into global variables at load time. + import cityscapesscripts.evaluation.evalPixelLevelSemanticLabeling as cityscapes_eval + + self._logger.info("Evaluating results under {} ...".format(self._temp_dir)) + + # set some global states in cityscapes evaluation API, before evaluating + cityscapes_eval.args.predictionPath = os.path.abspath(self._temp_dir) + cityscapes_eval.args.predictionWalk = None + cityscapes_eval.args.JSONOutput = False + cityscapes_eval.args.colorized = False + + # These lines are adopted from + # https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/evaluation/evalPixelLevelSemanticLabeling.py # noqa + gt_dir = PathManager.get_local_path(self._metadata.gt_dir) + groundTruthImgList = glob.glob(os.path.join(gt_dir, "*", "*_gtFine_labelIds.png")) + assert len( + groundTruthImgList + ), "Cannot find any ground truth images to use for evaluation. Searched for: {}".format( + cityscapes_eval.args.groundTruthSearch + ) + predictionImgList = [] + for gt in groundTruthImgList: + predictionImgList.append(cityscapes_eval.getPrediction(cityscapes_eval.args, gt)) + results = cityscapes_eval.evaluateImgLists( + predictionImgList, groundTruthImgList, cityscapes_eval.args + ) + ret = OrderedDict() + ret["sem_seg"] = { + "IoU": 100.0 * results["averageScoreClasses"], + "iIoU": 100.0 * results["averageScoreInstClasses"], + "IoU_sup": 100.0 * results["averageScoreCategories"], + "iIoU_sup": 100.0 * results["averageScoreInstCategories"], + } + self._working_dir.cleanup() + return ret diff --git a/annotator/oneformer/oneformer/evaluation/coco_evaluator.py b/annotator/oneformer/oneformer/evaluation/coco_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..b8ede48f623e9aacf4bbb1a9297d8941ef64aedb --- /dev/null +++ b/annotator/oneformer/oneformer/evaluation/coco_evaluator.py @@ -0,0 +1,563 @@ +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/detectron2/blob/main/detectron2/evaluation/coco_evaluation.py +# Modified by Jitesh Jain (https://github.com/praeclarumjj3) +# ------------------------------------------------------------------------------ + +import contextlib +import copy +import io +import itertools +import json +import logging +import numpy as np +import os +import pickle +from collections import OrderedDict +import pycocotools.mask as mask_util +import torch +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval +from tabulate import tabulate + +import annotator.oneformer.detectron2.utils.comm as comm +from annotator.oneformer.detectron2.config import CfgNode +from annotator.oneformer.detectron2.data import MetadataCatalog +from annotator.oneformer.detectron2.data.datasets.coco import convert_to_coco_json +from annotator.oneformer.detectron2.structures import Boxes, BoxMode, pairwise_iou +from annotator.oneformer.detectron2.utils.file_io import PathManager +from annotator.oneformer.detectron2.utils.logger import create_small_table + +from .evaluator import DatasetEvaluator + +try: + from annotator.oneformer.detectron2.evaluation.fast_eval_api import COCOeval_opt +except ImportError: + COCOeval_opt = COCOeval + + +class COCOEvaluator(DatasetEvaluator): + """ + Evaluate AP for instance detection/segmentation, AP + for keypoint detection outputs using COCO's metrics. + See http://cocodataset.org/#detection-eval and + http://cocodataset.org/#keypoints-eval to understand its metrics. + The metrics range from 0 to 100 (instead of 0 to 1), where a -1 or NaN means + the metric cannot be computed (e.g. due to no predictions made). + + In addition to COCO, this evaluator is able to support any bounding box detection, + instance segmentation, or keypoint detection dataset. + """ + + def __init__( + self, + dataset_name, + tasks=None, + distributed=True, + output_dir=None, + *, + max_dets_per_image=None, + use_fast_impl=True, + kpt_oks_sigmas=(), + allow_cached_coco=True, + ): + """ + Args: + dataset_name (str): name of the dataset to be evaluated. + It must have either the following corresponding metadata: + + "json_file": the path to the COCO format annotation + + Or it must be in detectron2's standard dataset format + so it can be converted to COCO format automatically. + tasks (tuple[str]): tasks that can be evaluated under the given + configuration. A task is one of "bbox", "segm", "keypoints". + By default, will infer this automatically from predictions. + distributed (True): if True, will collect results from all ranks and run evaluation + in the main process. + Otherwise, will only evaluate the results in the current process. + output_dir (str): optional, an output directory to dump all + results predicted on the dataset. The dump contains two files: + + 1. "instances_predictions.pth" a file that can be loaded with `torch.load` and + contains all the results in the format they are produced by the model. + 2. "coco_instances_results.json" a json file in COCO's result format. + max_dets_per_image (int): limit on the maximum number of detections per image. + By default in COCO, this limit is to 100, but this can be customized + to be greater, as is needed in evaluation metrics AP fixed and AP pool + (see https://arxiv.org/pdf/2102.01066.pdf) + This doesn't affect keypoint evaluation. + use_fast_impl (bool): use a fast but **unofficial** implementation to compute AP. + Although the results should be very close to the official implementation in COCO + API, it is still recommended to compute results with the official API for use in + papers. The faster implementation also uses more RAM. + kpt_oks_sigmas (list[float]): The sigmas used to calculate keypoint OKS. + See http://cocodataset.org/#keypoints-eval + When empty, it will use the defaults in COCO. + Otherwise it should be the same length as ROI_KEYPOINT_HEAD.NUM_KEYPOINTS. + allow_cached_coco (bool): Whether to use cached coco json from previous validation + runs. You should set this to False if you need to use different validation data. + Defaults to True. + """ + self._logger = logging.getLogger(__name__) + self._distributed = distributed + self._output_dir = output_dir + + if use_fast_impl and (COCOeval_opt is COCOeval): + self._logger.info("Fast COCO eval is not built. Falling back to official COCO eval.") + use_fast_impl = False + self._use_fast_impl = use_fast_impl + + # COCOeval requires the limit on the number of detections per image (maxDets) to be a list + # with at least 3 elements. The default maxDets in COCOeval is [1, 10, 100], in which the + # 3rd element (100) is used as the limit on the number of detections per image when + # evaluating AP. COCOEvaluator expects an integer for max_dets_per_image, so for COCOeval, + # we reformat max_dets_per_image into [1, 10, max_dets_per_image], based on the defaults. + if max_dets_per_image is None: + max_dets_per_image = [1, 10, 100] + else: + max_dets_per_image = [1, 10, max_dets_per_image] + self._max_dets_per_image = max_dets_per_image + + if tasks is not None and isinstance(tasks, CfgNode): + kpt_oks_sigmas = ( + tasks.TEST.KEYPOINT_OKS_SIGMAS if not kpt_oks_sigmas else kpt_oks_sigmas + ) + self._logger.warn( + "COCO Evaluator instantiated using config, this is deprecated behavior." + " Please pass in explicit arguments instead." + ) + self._tasks = None # Infering it from predictions should be better + else: + self._tasks = tasks + + self._cpu_device = torch.device("cpu") + + self._metadata = MetadataCatalog.get(dataset_name) + if not hasattr(self._metadata, "json_file"): + if output_dir is None: + raise ValueError( + "output_dir must be provided to COCOEvaluator " + "for datasets not in COCO format." + ) + self._logger.info(f"Trying to convert '{dataset_name}' to COCO format ...") + + cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") + self._metadata.json_file = cache_path + convert_to_coco_json(dataset_name, cache_path, allow_cached=allow_cached_coco) + + json_file = PathManager.get_local_path(self._metadata.json_file) + with contextlib.redirect_stdout(io.StringIO()): + self._coco_api = COCO(json_file) + + # Test set json files do not contain annotations (evaluation must be + # performed using the COCO evaluation server). + self._do_evaluation = "annotations" in self._coco_api.dataset + if self._do_evaluation: + self._kpt_oks_sigmas = kpt_oks_sigmas + + def reset(self): + self._predictions = [] + + def process(self, inputs, outputs): + """ + Args: + inputs: the inputs to a COCO model (e.g., GeneralizedRCNN). + It is a list of dict. Each dict corresponds to an image and + contains keys like "height", "width", "file_name", "image_id". + outputs: the outputs of a COCO model. It is a list of dicts with key + "instances" that contains :class:`Instances`. + """ + for input, output in zip(inputs, outputs): + prediction = {"image_id": input["image_id"]} + + if "instances" in output: + instances = output["instances"].to(self._cpu_device) + prediction["instances"] = instances_to_coco_json(instances, input["image_id"]) + if len(prediction) > 1: + self._predictions.append(prediction) + + def evaluate(self, img_ids=None): + """ + Args: + img_ids: a list of image IDs to evaluate on. Default to None for the whole dataset + """ + if self._distributed: + comm.synchronize() + predictions = comm.gather(self._predictions, dst=0) + predictions = list(itertools.chain(*predictions)) + + if not comm.is_main_process(): + return {} + else: + predictions = self._predictions + + if len(predictions) == 0: + self._logger.warning("[COCOEvaluator] Did not receive valid predictions.") + return {} + + if self._output_dir: + PathManager.mkdirs(self._output_dir) + file_path = os.path.join(self._output_dir, "instances_predictions.pth") + with PathManager.open(file_path, "wb") as f: + torch.save(predictions, f) + + self._results = OrderedDict() + if "instances" in predictions[0]: + self._eval_predictions(predictions, img_ids=img_ids) + # Copy so the caller can do whatever with results + return copy.deepcopy(self._results) + + def _tasks_from_predictions(self, predictions): + """ + Get COCO API "tasks" (i.e. iou_type) from COCO-format predictions. + """ + for pred in predictions: + if "segmentation" in pred: + tasks = {"segm"} + if "keypoints" in pred: + tasks.add("keypoints") + return sorted(tasks) + + def _eval_predictions(self, predictions, img_ids=None): + """ + Evaluate predictions. Fill self._results with the metrics of the tasks. + """ + self._logger.info("Preparing results for COCO format ...") + coco_results = list(itertools.chain(*[x["instances"] for x in predictions])) + tasks = self._tasks or self._tasks_from_predictions(coco_results) + + # unmap the category ids for COCO + if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"): + dataset_id_to_contiguous_id = self._metadata.thing_dataset_id_to_contiguous_id + all_contiguous_ids = list(dataset_id_to_contiguous_id.values()) + num_classes = len(all_contiguous_ids) + assert min(all_contiguous_ids) == 0 and max(all_contiguous_ids) == num_classes - 1 + + reverse_id_mapping = {v: k for k, v in dataset_id_to_contiguous_id.items()} + for result in coco_results: + category_id = result["category_id"] + assert category_id < num_classes, ( + f"A prediction has class={category_id}, " + f"but the dataset only has {num_classes} classes and " + f"predicted class id should be in [0, {num_classes - 1}]." + ) + result["category_id"] = reverse_id_mapping[category_id] + + if self._output_dir: + file_path = os.path.join(self._output_dir, "coco_instances_results.json") + self._logger.info("Saving results to {}".format(file_path)) + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(coco_results)) + f.flush() + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info( + "Evaluating predictions with {} COCO API...".format( + "unofficial" if self._use_fast_impl else "official" + ) + ) + for task in sorted(tasks): + assert task in {"segm", "keypoints"}, f"Got unknown task: {task}!" + coco_eval = ( + _evaluate_predictions_on_coco( + self._coco_api, + coco_results, + task, + kpt_oks_sigmas=self._kpt_oks_sigmas, + use_fast_impl=self._use_fast_impl, + img_ids=img_ids, + max_dets_per_image=self._max_dets_per_image, + ) + if len(coco_results) > 0 + else None # cocoapi does not handle empty results very well + ) + + res = self._derive_coco_results( + coco_eval, task, class_names=self._metadata.get("thing_classes") + ) + self._results[task] = res + + def _derive_coco_results(self, coco_eval, iou_type, class_names=None): + """ + Derive the desired score numbers from summarized COCOeval. + + Args: + coco_eval (None or COCOEval): None represents no predictions from model. + iou_type (str): + class_names (None or list[str]): if provided, will use it to predict + per-category AP. + + Returns: + a dict of {metric name: score} + """ + + metrics = { + "segm": ["AP", "AP50", "AP75", "APs", "APm", "APl"], + "keypoints": ["AP", "AP50", "AP75", "APm", "APl"], + }[iou_type] + + if coco_eval is None: + self._logger.warn("No predictions from the model!") + return {metric: float("nan") for metric in metrics} + + # the standard metrics + results = { + metric: float(coco_eval.stats[idx] * 100 if coco_eval.stats[idx] >= 0 else "nan") + for idx, metric in enumerate(metrics) + } + self._logger.info( + "Evaluation results for {}: \n".format(iou_type) + create_small_table(results) + ) + if not np.isfinite(sum(results.values())): + self._logger.info("Some metrics cannot be computed and is shown as NaN.") + + if class_names is None or len(class_names) <= 1: + return results + # Compute per-category AP + # from https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L222-L252 # noqa + precisions = coco_eval.eval["precision"] + # precision has dims (iou, recall, cls, area range, max dets) + assert len(class_names) == precisions.shape[2] + + results_per_category = [] + for idx, name in enumerate(class_names): + # area range index 0: all area ranges + # max dets index -1: typically 100 per image + precision = precisions[:, :, idx, 0, -1] + precision = precision[precision > -1] + ap = np.mean(precision) if precision.size else float("nan") + results_per_category.append(("{}".format(name), float(ap * 100))) + + # tabulate it + N_COLS = min(6, len(results_per_category) * 2) + results_flatten = list(itertools.chain(*results_per_category)) + results_2d = itertools.zip_longest(*[results_flatten[i::N_COLS] for i in range(N_COLS)]) + table = tabulate( + results_2d, + tablefmt="pipe", + floatfmt=".3f", + headers=["category", "AP"] * (N_COLS // 2), + numalign="left", + ) + self._logger.info("Per-category {} AP: \n".format(iou_type) + table) + + results.update({"AP-" + name: ap for name, ap in results_per_category}) + return results + + +def instances_to_coco_json(instances, img_id): + """ + Dump an "Instances" object to a COCO-format json that's used for evaluation. + + Args: + instances (Instances): + img_id (int): the image id + + Returns: + list[dict]: list of json annotations in COCO format. + """ + num_instance = len(instances) + if num_instance == 0: + return [] + + scores = instances.scores.tolist() + classes = instances.pred_classes.tolist() + + has_mask = instances.has("pred_masks") + if has_mask: + # use RLE to encode the masks, because they are too large and takes memory + # since this evaluator stores outputs of the entire dataset + rles = [ + mask_util.encode(np.array(mask[:, :, None], order="F", dtype="uint8"))[0] + for mask in instances.pred_masks + ] + for rle in rles: + # "counts" is an array encoded by mask_util as a byte-stream. Python3's + # json writer which always produces strings cannot serialize a bytestream + # unless you decode it. Thankfully, utf-8 works out (which is also what + # the pycocotools/_mask.pyx does). + rle["counts"] = rle["counts"].decode("utf-8") + + has_keypoints = instances.has("pred_keypoints") + if has_keypoints: + keypoints = instances.pred_keypoints + + results = [] + for k in range(num_instance): + result = { + "image_id": img_id, + "category_id": classes[k], + "score": scores[k], + } + if has_mask: + result["segmentation"] = rles[k] + if has_keypoints: + # In COCO annotations, + # keypoints coordinates are pixel indices. + # However our predictions are floating point coordinates. + # Therefore we subtract 0.5 to be consistent with the annotation format. + # This is the inverse of data loading logic in `datasets/coco.py`. + keypoints[k][:, :2] -= 0.5 + result["keypoints"] = keypoints[k].flatten().tolist() + results.append(result) + return results + +def _evaluate_predictions_on_coco( + coco_gt, + coco_results, + iou_type, + kpt_oks_sigmas=None, + use_fast_impl=True, + img_ids=None, + max_dets_per_image=None, +): + """ + Evaluate the coco results using COCOEval API. + """ + assert len(coco_results) > 0 + + if iou_type == "segm": + coco_results = copy.deepcopy(coco_results) + # When evaluating mask AP, if the results contain bbox, cocoapi will + # use the box area as the area of the instance, instead of the mask area. + # This leads to a different definition of small/medium/large. + # We remove the bbox field to let mask AP use mask area. + for c in coco_results: + c.pop("bbox", None) + + coco_dt = coco_gt.loadRes(coco_results) + coco_eval = (COCOeval_opt if use_fast_impl else COCOeval)(coco_gt, coco_dt, iou_type) + # For COCO, the default max_dets_per_image is [1, 10, 100]. + if max_dets_per_image is None: + max_dets_per_image = [1, 10, 100] # Default from COCOEval + else: + assert ( + len(max_dets_per_image) >= 3 + ), "COCOeval requires maxDets (and max_dets_per_image) to have length at least 3" + # In the case that user supplies a custom input for max_dets_per_image, + # apply COCOevalMaxDets to evaluate AP with the custom input. + if max_dets_per_image[2] != 100: + coco_eval = COCOevalMaxDets(coco_gt, coco_dt, iou_type) + if iou_type != "keypoints": + coco_eval.params.maxDets = max_dets_per_image + + if img_ids is not None: + coco_eval.params.imgIds = img_ids + + if iou_type == "keypoints": + # Use the COCO default keypoint OKS sigmas unless overrides are specified + if kpt_oks_sigmas: + assert hasattr(coco_eval.params, "kpt_oks_sigmas"), "pycocotools is too old!" + coco_eval.params.kpt_oks_sigmas = np.array(kpt_oks_sigmas) + # COCOAPI requires every detection and every gt to have keypoints, so + # we just take the first entry from both + num_keypoints_dt = len(coco_results[0]["keypoints"]) // 3 + num_keypoints_gt = len(next(iter(coco_gt.anns.values()))["keypoints"]) // 3 + num_keypoints_oks = len(coco_eval.params.kpt_oks_sigmas) + assert num_keypoints_oks == num_keypoints_dt == num_keypoints_gt, ( + f"[COCOEvaluator] Prediction contain {num_keypoints_dt} keypoints. " + f"Ground truth contains {num_keypoints_gt} keypoints. " + f"The length of cfg.TEST.KEYPOINT_OKS_SIGMAS is {num_keypoints_oks}. " + "They have to agree with each other. For meaning of OKS, please refer to " + "http://cocodataset.org/#keypoints-eval." + ) + + coco_eval.evaluate() + coco_eval.accumulate() + coco_eval.summarize() + + return coco_eval + + +class COCOevalMaxDets(COCOeval): + """ + Modified version of COCOeval for evaluating AP with a custom + maxDets (by default for COCO, maxDets is 100) + """ + + def summarize(self): + """ + Compute and display summary metrics for evaluation results given + a custom value for max_dets_per_image + """ + + def _summarize(ap=1, iouThr=None, areaRng="all", maxDets=100): + p = self.params + iStr = " {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}" + titleStr = "Average Precision" if ap == 1 else "Average Recall" + typeStr = "(AP)" if ap == 1 else "(AR)" + iouStr = ( + "{:0.2f}:{:0.2f}".format(p.iouThrs[0], p.iouThrs[-1]) + if iouThr is None + else "{:0.2f}".format(iouThr) + ) + + aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng] + mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets] + if ap == 1: + # dimension of precision: [TxRxKxAxM] + s = self.eval["precision"] + # IoU + if iouThr is not None: + t = np.where(iouThr == p.iouThrs)[0] + s = s[t] + s = s[:, :, :, aind, mind] + else: + # dimension of recall: [TxKxAxM] + s = self.eval["recall"] + if iouThr is not None: + t = np.where(iouThr == p.iouThrs)[0] + s = s[t] + s = s[:, :, aind, mind] + if len(s[s > -1]) == 0: + mean_s = -1 + else: + mean_s = np.mean(s[s > -1]) + print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s)) + return mean_s + + def _summarizeDets(): + stats = np.zeros((12,)) + # Evaluate AP using the custom limit on maximum detections per image + stats[0] = _summarize(1, maxDets=self.params.maxDets[2]) + stats[1] = _summarize(1, iouThr=0.5, maxDets=self.params.maxDets[2]) + stats[2] = _summarize(1, iouThr=0.75, maxDets=self.params.maxDets[2]) + stats[3] = _summarize(1, areaRng="small", maxDets=self.params.maxDets[2]) + stats[4] = _summarize(1, areaRng="medium", maxDets=self.params.maxDets[2]) + stats[5] = _summarize(1, areaRng="large", maxDets=self.params.maxDets[2]) + stats[6] = _summarize(0, maxDets=self.params.maxDets[0]) + stats[7] = _summarize(0, maxDets=self.params.maxDets[1]) + stats[8] = _summarize(0, maxDets=self.params.maxDets[2]) + stats[9] = _summarize(0, areaRng="small", maxDets=self.params.maxDets[2]) + stats[10] = _summarize(0, areaRng="medium", maxDets=self.params.maxDets[2]) + stats[11] = _summarize(0, areaRng="large", maxDets=self.params.maxDets[2]) + return stats + + def _summarizeKps(): + stats = np.zeros((10,)) + stats[0] = _summarize(1, maxDets=20) + stats[1] = _summarize(1, maxDets=20, iouThr=0.5) + stats[2] = _summarize(1, maxDets=20, iouThr=0.75) + stats[3] = _summarize(1, maxDets=20, areaRng="medium") + stats[4] = _summarize(1, maxDets=20, areaRng="large") + stats[5] = _summarize(0, maxDets=20) + stats[6] = _summarize(0, maxDets=20, iouThr=0.5) + stats[7] = _summarize(0, maxDets=20, iouThr=0.75) + stats[8] = _summarize(0, maxDets=20, areaRng="medium") + stats[9] = _summarize(0, maxDets=20, areaRng="large") + return stats + + if not self.eval: + raise Exception("Please run accumulate() first") + iouType = self.params.iouType + if iouType == "segm": + summarize = _summarizeDets + elif iouType == "keypoints": + summarize = _summarizeKps + self.stats = summarize() + + def __str__(self): + self.summarize() \ No newline at end of file diff --git a/annotator/oneformer/oneformer/evaluation/detection_coco_evaluator.py b/annotator/oneformer/oneformer/evaluation/detection_coco_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..5e3c37362d8153a138e20e698b009ca8a26535e2 --- /dev/null +++ b/annotator/oneformer/oneformer/evaluation/detection_coco_evaluator.py @@ -0,0 +1,723 @@ +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/detectron2/blob/main/detectron2/evaluation/coco_evaluation.py +# Modified by Jitesh Jain (https://github.com/praeclarumjj3) +# ------------------------------------------------------------------------------ + +import contextlib +import copy +import io +import itertools +import json +import logging +import numpy as np +import os +import pickle +from collections import OrderedDict +import pycocotools.mask as mask_util +import torch +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval +from tabulate import tabulate + +import annotator.oneformer.detectron2.utils.comm as comm +from annotator.oneformer.detectron2.config import CfgNode +from annotator.oneformer.detectron2.data import MetadataCatalog +from annotator.oneformer.detectron2.data.datasets.coco import convert_to_coco_json +from annotator.oneformer.detectron2.structures import Boxes, BoxMode, pairwise_iou +from annotator.oneformer.detectron2.utils.file_io import PathManager +from annotator.oneformer.detectron2.utils.logger import create_small_table + +from .evaluator import DatasetEvaluator + +try: + from annotator.oneformer.detectron2.evaluation.fast_eval_api import COCOeval_opt +except ImportError: + COCOeval_opt = COCOeval + + +class DetectionCOCOEvaluator(DatasetEvaluator): + """ + Evaluate AR for object proposals, AP for instance detection/segmentation, AP + for keypoint detection outputs using COCO's metrics. + See http://cocodataset.org/#detection-eval and + http://cocodataset.org/#keypoints-eval to understand its metrics. + The metrics range from 0 to 100 (instead of 0 to 1), where a -1 or NaN means + the metric cannot be computed (e.g. due to no predictions made). + + In addition to COCO, this evaluator is able to support any bounding box detection, + instance segmentation, or keypoint detection dataset. + """ + + def __init__( + self, + dataset_name, + tasks=None, + distributed=True, + output_dir=None, + *, + max_dets_per_image=None, + use_fast_impl=True, + kpt_oks_sigmas=(), + allow_cached_coco=True, + ): + """ + Args: + dataset_name (str): name of the dataset to be evaluated. + It must have either the following corresponding metadata: + + "json_file": the path to the COCO format annotation + + Or it must be in detectron2's standard dataset format + so it can be converted to COCO format automatically. + tasks (tuple[str]): tasks that can be evaluated under the given + configuration. A task is one of "bbox", "segm", "keypoints". + By default, will infer this automatically from predictions. + distributed (True): if True, will collect results from all ranks and run evaluation + in the main process. + Otherwise, will only evaluate the results in the current process. + output_dir (str): optional, an output directory to dump all + results predicted on the dataset. The dump contains two files: + + 1. "instances_predictions.pth" a file that can be loaded with `torch.load` and + contains all the results in the format they are produced by the model. + 2. "coco_instances_results.json" a json file in COCO's result format. + max_dets_per_image (int): limit on the maximum number of detections per image. + By default in COCO, this limit is to 100, but this can be customized + to be greater, as is needed in evaluation metrics AP fixed and AP pool + (see https://arxiv.org/pdf/2102.01066.pdf) + This doesn't affect keypoint evaluation. + use_fast_impl (bool): use a fast but **unofficial** implementation to compute AP. + Although the results should be very close to the official implementation in COCO + API, it is still recommended to compute results with the official API for use in + papers. The faster implementation also uses more RAM. + kpt_oks_sigmas (list[float]): The sigmas used to calculate keypoint OKS. + See http://cocodataset.org/#keypoints-eval + When empty, it will use the defaults in COCO. + Otherwise it should be the same length as ROI_KEYPOINT_HEAD.NUM_KEYPOINTS. + allow_cached_coco (bool): Whether to use cached coco json from previous validation + runs. You should set this to False if you need to use different validation data. + Defaults to True. + """ + self._logger = logging.getLogger(__name__) + self._distributed = distributed + self._output_dir = output_dir + + if use_fast_impl and (COCOeval_opt is COCOeval): + self._logger.info("Fast COCO eval is not built. Falling back to official COCO eval.") + use_fast_impl = False + self._use_fast_impl = use_fast_impl + + # COCOeval requires the limit on the number of detections per image (maxDets) to be a list + # with at least 3 elements. The default maxDets in COCOeval is [1, 10, 100], in which the + # 3rd element (100) is used as the limit on the number of detections per image when + # evaluating AP. COCOEvaluator expects an integer for max_dets_per_image, so for COCOeval, + # we reformat max_dets_per_image into [1, 10, max_dets_per_image], based on the defaults. + if max_dets_per_image is None: + max_dets_per_image = [1, 10, 100] + else: + max_dets_per_image = [1, 10, max_dets_per_image] + self._max_dets_per_image = max_dets_per_image + + if tasks is not None and isinstance(tasks, CfgNode): + kpt_oks_sigmas = ( + tasks.TEST.KEYPOINT_OKS_SIGMAS if not kpt_oks_sigmas else kpt_oks_sigmas + ) + self._logger.warn( + "COCO Evaluator instantiated using config, this is deprecated behavior." + " Please pass in explicit arguments instead." + ) + self._tasks = None # Infering it from predictions should be better + else: + self._tasks = tasks + + self._cpu_device = torch.device("cpu") + + self._metadata = MetadataCatalog.get(dataset_name) + if not hasattr(self._metadata, "json_file"): + if output_dir is None: + raise ValueError( + "output_dir must be provided to COCOEvaluator " + "for datasets not in COCO format." + ) + self._logger.info(f"Trying to convert '{dataset_name}' to COCO format ...") + + cache_path = os.path.join(output_dir, f"{dataset_name}_coco_format.json") + self._metadata.json_file = cache_path + convert_to_coco_json(dataset_name, cache_path, allow_cached=allow_cached_coco) + + json_file = PathManager.get_local_path(self._metadata.json_file) + with contextlib.redirect_stdout(io.StringIO()): + self._coco_api = COCO(json_file) + + # Test set json files do not contain annotations (evaluation must be + # performed using the COCO evaluation server). + self._do_evaluation = "annotations" in self._coco_api.dataset + if self._do_evaluation: + self._kpt_oks_sigmas = kpt_oks_sigmas + + def reset(self): + self._predictions = [] + + def process(self, inputs, outputs): + """ + Args: + inputs: the inputs to a COCO model (e.g., GeneralizedRCNN). + It is a list of dict. Each dict corresponds to an image and + contains keys like "height", "width", "file_name", "image_id". + outputs: the outputs of a COCO model. It is a list of dicts with key + "box_instances" that contains :class:`Instances`. + """ + for input, output in zip(inputs, outputs): + prediction = {"image_id": input["image_id"]} + + if "box_instances" in output: + instances = output["box_instances"].to(self._cpu_device) + prediction["box_instances"] = instances_to_coco_json(instances, input["image_id"]) + if "proposals" in output: + prediction["proposals"] = output["proposals"].to(self._cpu_device) + if len(prediction) > 1: + self._predictions.append(prediction) + + def evaluate(self, img_ids=None): + """ + Args: + img_ids: a list of image IDs to evaluate on. Default to None for the whole dataset + """ + if self._distributed: + comm.synchronize() + predictions = comm.gather(self._predictions, dst=0) + predictions = list(itertools.chain(*predictions)) + + if not comm.is_main_process(): + return {} + else: + predictions = self._predictions + + if len(predictions) == 0: + self._logger.warning("[COCOEvaluator] Did not receive valid predictions.") + return {} + + if self._output_dir: + PathManager.mkdirs(self._output_dir) + file_path = os.path.join(self._output_dir, "instances_predictions.pth") + with PathManager.open(file_path, "wb") as f: + torch.save(predictions, f) + + self._results = OrderedDict() + if "proposals" in predictions[0]: + self._eval_box_proposals(predictions) + if "box_instances" in predictions[0]: + self._eval_predictions(predictions, img_ids=img_ids) + # Copy so the caller can do whatever with results + return copy.deepcopy(self._results) + + def _tasks_from_predictions(self, predictions): + """ + Get COCO API "tasks" (i.e. iou_type) from COCO-format predictions. + """ + tasks = {"bbox"} + for pred in predictions: + if "keypoints" in pred: + tasks.add("keypoints") + return sorted(tasks) + + def _eval_predictions(self, predictions, img_ids=None): + """ + Evaluate predictions. Fill self._results with the metrics of the tasks. + """ + self._logger.info("Preparing results for COCO format ...") + coco_results = list(itertools.chain(*[x["box_instances"] for x in predictions])) + tasks = self._tasks or self._tasks_from_predictions(coco_results) + + # unmap the category ids for COCO + if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"): + dataset_id_to_contiguous_id = self._metadata.thing_dataset_id_to_contiguous_id + all_contiguous_ids = list(dataset_id_to_contiguous_id.values()) + num_classes = len(all_contiguous_ids) + assert min(all_contiguous_ids) == 0 and max(all_contiguous_ids) == num_classes - 1 + + reverse_id_mapping = {v: k for k, v in dataset_id_to_contiguous_id.items()} + for result in coco_results: + category_id = result["category_id"] + assert category_id < num_classes, ( + f"A prediction has class={category_id}, " + f"but the dataset only has {num_classes} classes and " + f"predicted class id should be in [0, {num_classes - 1}]." + ) + result["category_id"] = reverse_id_mapping[category_id] + + if self._output_dir: + file_path = os.path.join(self._output_dir, "coco_instances_results.json") + self._logger.info("Saving results to {}".format(file_path)) + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(coco_results)) + f.flush() + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info( + "Evaluating predictions with {} COCO API...".format( + "unofficial" if self._use_fast_impl else "official" + ) + ) + for task in sorted(tasks): + assert task in {"bbox", "keypoints"}, f"Got unknown task: {task}!" + coco_eval = ( + _evaluate_predictions_on_coco( + self._coco_api, + coco_results, + task, + kpt_oks_sigmas=self._kpt_oks_sigmas, + use_fast_impl=self._use_fast_impl, + img_ids=img_ids, + max_dets_per_image=self._max_dets_per_image, + ) + if len(coco_results) > 0 + else None # cocoapi does not handle empty results very well + ) + + res = self._derive_coco_results( + coco_eval, task, class_names=self._metadata.get("thing_classes") + ) + self._results[task] = res + + def _eval_box_proposals(self, predictions): + """ + Evaluate the box proposals in predictions. + Fill self._results with the metrics for "box_proposals" task. + """ + if self._output_dir: + # Saving generated box proposals to file. + # Predicted box_proposals are in XYXY_ABS mode. + bbox_mode = BoxMode.XYXY_ABS.value + ids, boxes, objectness_logits = [], [], [] + for prediction in predictions: + ids.append(prediction["image_id"]) + boxes.append(prediction["proposals"].proposal_boxes.tensor.numpy()) + objectness_logits.append(prediction["proposals"].objectness_logits.numpy()) + + proposal_data = { + "boxes": boxes, + "objectness_logits": objectness_logits, + "ids": ids, + "bbox_mode": bbox_mode, + } + with PathManager.open(os.path.join(self._output_dir, "box_proposals.pkl"), "wb") as f: + pickle.dump(proposal_data, f) + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info("Evaluating bbox proposals ...") + res = {} + areas = {"all": "", "small": "s", "medium": "m", "large": "l"} + for limit in [100, 1000]: + for area, suffix in areas.items(): + stats = _evaluate_box_proposals(predictions, self._coco_api, area=area, limit=limit) + key = "AR{}@{:d}".format(suffix, limit) + res[key] = float(stats["ar"].item() * 100) + self._logger.info("Proposal metrics: \n" + create_small_table(res)) + self._results["box_proposals"] = res + + def _derive_coco_results(self, coco_eval, iou_type, class_names=None): + """ + Derive the desired score numbers from summarized COCOeval. + + Args: + coco_eval (None or COCOEval): None represents no predictions from model. + iou_type (str): + class_names (None or list[str]): if provided, will use it to predict + per-category AP. + + Returns: + a dict of {metric name: score} + """ + + metrics = { + "bbox": ["AP", "AP50", "AP75", "APs", "APm", "APl"], + "keypoints": ["AP", "AP50", "AP75", "APm", "APl"], + }[iou_type] + + if coco_eval is None: + self._logger.warn("No predictions from the model!") + return {metric: float("nan") for metric in metrics} + + # the standard metrics + results = { + metric: float(coco_eval.stats[idx] * 100 if coco_eval.stats[idx] >= 0 else "nan") + for idx, metric in enumerate(metrics) + } + self._logger.info( + "Evaluation results for {}: \n".format(iou_type) + create_small_table(results) + ) + if not np.isfinite(sum(results.values())): + self._logger.info("Some metrics cannot be computed and is shown as NaN.") + + if class_names is None or len(class_names) <= 1: + return results + # Compute per-category AP + # from https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L222-L252 # noqa + precisions = coco_eval.eval["precision"] + # precision has dims (iou, recall, cls, area range, max dets) + assert len(class_names) == precisions.shape[2] + + results_per_category = [] + for idx, name in enumerate(class_names): + # area range index 0: all area ranges + # max dets index -1: typically 100 per image + precision = precisions[:, :, idx, 0, -1] + precision = precision[precision > -1] + ap = np.mean(precision) if precision.size else float("nan") + results_per_category.append(("{}".format(name), float(ap * 100))) + + # tabulate it + N_COLS = min(6, len(results_per_category) * 2) + results_flatten = list(itertools.chain(*results_per_category)) + results_2d = itertools.zip_longest(*[results_flatten[i::N_COLS] for i in range(N_COLS)]) + table = tabulate( + results_2d, + tablefmt="pipe", + floatfmt=".3f", + headers=["category", "AP"] * (N_COLS // 2), + numalign="left", + ) + self._logger.info("Per-category {} AP: \n".format(iou_type) + table) + + results.update({"AP-" + name: ap for name, ap in results_per_category}) + return results + + +def instances_to_coco_json(instances, img_id): + """ + Dump an "Instances" object to a COCO-format json that's used for evaluation. + + Args: + instances (Instances): + img_id (int): the image id + + Returns: + list[dict]: list of json annotations in COCO format. + """ + num_instance = len(instances) + if num_instance == 0: + return [] + + boxes = instances.pred_boxes.tensor.numpy() + boxes = BoxMode.convert(boxes, BoxMode.XYXY_ABS, BoxMode.XYWH_ABS) + boxes = boxes.tolist() + scores = instances.scores.tolist() + classes = instances.pred_classes.tolist() + + has_mask = instances.has("pred_masks") + if has_mask: + # use RLE to encode the masks, because they are too large and takes memory + # since this evaluator stores outputs of the entire dataset + rles = [ + mask_util.encode(np.array(mask[:, :, None], order="F", dtype="uint8"))[0] + for mask in instances.pred_masks + ] + for rle in rles: + # "counts" is an array encoded by mask_util as a byte-stream. Python3's + # json writer which always produces strings cannot serialize a bytestream + # unless you decode it. Thankfully, utf-8 works out (which is also what + # the pycocotools/_mask.pyx does). + rle["counts"] = rle["counts"].decode("utf-8") + + has_keypoints = instances.has("pred_keypoints") + if has_keypoints: + keypoints = instances.pred_keypoints + + results = [] + for k in range(num_instance): + result = { + "image_id": img_id, + "category_id": classes[k], + "bbox": boxes[k], + "score": scores[k], + } + if has_mask: + result["segmentation"] = rles[k] + if has_keypoints: + # In COCO annotations, + # keypoints coordinates are pixel indices. + # However our predictions are floating point coordinates. + # Therefore we subtract 0.5 to be consistent with the annotation format. + # This is the inverse of data loading logic in `datasets/coco.py`. + keypoints[k][:, :2] -= 0.5 + result["keypoints"] = keypoints[k].flatten().tolist() + results.append(result) + return results + + +# inspired from Detectron: +# https://github.com/facebookresearch/Detectron/blob/a6a835f5b8208c45d0dce217ce9bbda915f44df7/detectron/datasets/json_dataset_evaluator.py#L255 # noqa +def _evaluate_box_proposals(dataset_predictions, coco_api, thresholds=None, area="all", limit=None): + """ + Evaluate detection proposal recall metrics. This function is a much + faster alternative to the official COCO API recall evaluation code. However, + it produces slightly different results. + """ + # Record max overlap value for each gt box + # Return vector of overlap values + areas = { + "all": 0, + "small": 1, + "medium": 2, + "large": 3, + "96-128": 4, + "128-256": 5, + "256-512": 6, + "512-inf": 7, + } + area_ranges = [ + [0**2, 1e5**2], # all + [0**2, 32**2], # small + [32**2, 96**2], # medium + [96**2, 1e5**2], # large + [96**2, 128**2], # 96-128 + [128**2, 256**2], # 128-256 + [256**2, 512**2], # 256-512 + [512**2, 1e5**2], + ] # 512-inf + assert area in areas, "Unknown area range: {}".format(area) + area_range = area_ranges[areas[area]] + gt_overlaps = [] + num_pos = 0 + + for prediction_dict in dataset_predictions: + predictions = prediction_dict["proposals"] + + # sort predictions in descending order + # TODO maybe remove this and make it explicit in the documentation + inds = predictions.objectness_logits.sort(descending=True)[1] + predictions = predictions[inds] + + ann_ids = coco_api.getAnnIds(imgIds=prediction_dict["image_id"]) + anno = coco_api.loadAnns(ann_ids) + gt_boxes = [ + BoxMode.convert(obj["bbox"], BoxMode.XYWH_ABS, BoxMode.XYXY_ABS) + for obj in anno + if obj["iscrowd"] == 0 + ] + gt_boxes = torch.as_tensor(gt_boxes).reshape(-1, 4) # guard against no boxes + gt_boxes = Boxes(gt_boxes) + gt_areas = torch.as_tensor([obj["area"] for obj in anno if obj["iscrowd"] == 0]) + + if len(gt_boxes) == 0 or len(predictions) == 0: + continue + + valid_gt_inds = (gt_areas >= area_range[0]) & (gt_areas <= area_range[1]) + gt_boxes = gt_boxes[valid_gt_inds] + + num_pos += len(gt_boxes) + + if len(gt_boxes) == 0: + continue + + if limit is not None and len(predictions) > limit: + predictions = predictions[:limit] + + overlaps = pairwise_iou(predictions.proposal_boxes, gt_boxes) + + _gt_overlaps = torch.zeros(len(gt_boxes)) + for j in range(min(len(predictions), len(gt_boxes))): + # find which proposal box maximally covers each gt box + # and get the iou amount of coverage for each gt box + max_overlaps, argmax_overlaps = overlaps.max(dim=0) + + # find which gt box is 'best' covered (i.e. 'best' = most iou) + gt_ovr, gt_ind = max_overlaps.max(dim=0) + assert gt_ovr >= 0 + # find the proposal box that covers the best covered gt box + box_ind = argmax_overlaps[gt_ind] + # record the iou coverage of this gt box + _gt_overlaps[j] = overlaps[box_ind, gt_ind] + assert _gt_overlaps[j] == gt_ovr + # mark the proposal box and the gt box as used + overlaps[box_ind, :] = -1 + overlaps[:, gt_ind] = -1 + + # append recorded iou coverage level + gt_overlaps.append(_gt_overlaps) + gt_overlaps = ( + torch.cat(gt_overlaps, dim=0) if len(gt_overlaps) else torch.zeros(0, dtype=torch.float32) + ) + gt_overlaps, _ = torch.sort(gt_overlaps) + + if thresholds is None: + step = 0.05 + thresholds = torch.arange(0.5, 0.95 + 1e-5, step, dtype=torch.float32) + recalls = torch.zeros_like(thresholds) + # compute recall for each iou threshold + for i, t in enumerate(thresholds): + recalls[i] = (gt_overlaps >= t).float().sum() / float(num_pos) + # ar = 2 * np.trapz(recalls, thresholds) + ar = recalls.mean() + return { + "ar": ar, + "recalls": recalls, + "thresholds": thresholds, + "gt_overlaps": gt_overlaps, + "num_pos": num_pos, + } + + +def _evaluate_predictions_on_coco( + coco_gt, + coco_results, + iou_type, + kpt_oks_sigmas=None, + use_fast_impl=True, + img_ids=None, + max_dets_per_image=None, +): + """ + Evaluate the coco results using COCOEval API. + """ + assert len(coco_results) > 0 + + if iou_type == "segm": + coco_results = copy.deepcopy(coco_results) + # When evaluating mask AP, if the results contain bbox, cocoapi will + # use the box area as the area of the instance, instead of the mask area. + # This leads to a different definition of small/medium/large. + # We remove the bbox field to let mask AP use mask area. + for c in coco_results: + c.pop("bbox", None) + + coco_dt = coco_gt.loadRes(coco_results) + coco_eval = (COCOeval_opt if use_fast_impl else COCOeval)(coco_gt, coco_dt, iou_type) + # For COCO, the default max_dets_per_image is [1, 10, 100]. + if max_dets_per_image is None: + max_dets_per_image = [1, 10, 100] # Default from COCOEval + else: + assert ( + len(max_dets_per_image) >= 3 + ), "COCOeval requires maxDets (and max_dets_per_image) to have length at least 3" + # In the case that user supplies a custom input for max_dets_per_image, + # apply COCOevalMaxDets to evaluate AP with the custom input. + if max_dets_per_image[2] != 100: + coco_eval = COCOevalMaxDets(coco_gt, coco_dt, iou_type) + if iou_type != "keypoints": + coco_eval.params.maxDets = max_dets_per_image + + if img_ids is not None: + coco_eval.params.imgIds = img_ids + + if iou_type == "keypoints": + # Use the COCO default keypoint OKS sigmas unless overrides are specified + if kpt_oks_sigmas: + assert hasattr(coco_eval.params, "kpt_oks_sigmas"), "pycocotools is too old!" + coco_eval.params.kpt_oks_sigmas = np.array(kpt_oks_sigmas) + # COCOAPI requires every detection and every gt to have keypoints, so + # we just take the first entry from both + num_keypoints_dt = len(coco_results[0]["keypoints"]) // 3 + num_keypoints_gt = len(next(iter(coco_gt.anns.values()))["keypoints"]) // 3 + num_keypoints_oks = len(coco_eval.params.kpt_oks_sigmas) + assert num_keypoints_oks == num_keypoints_dt == num_keypoints_gt, ( + f"[COCOEvaluator] Prediction contain {num_keypoints_dt} keypoints. " + f"Ground truth contains {num_keypoints_gt} keypoints. " + f"The length of cfg.TEST.KEYPOINT_OKS_SIGMAS is {num_keypoints_oks}. " + "They have to agree with each other. For meaning of OKS, please refer to " + "http://cocodataset.org/#keypoints-eval." + ) + + coco_eval.evaluate() + coco_eval.accumulate() + coco_eval.summarize() + + return coco_eval + + +class COCOevalMaxDets(COCOeval): + """ + Modified version of COCOeval for evaluating AP with a custom + maxDets (by default for COCO, maxDets is 100) + """ + + def summarize(self): + """ + Compute and display summary metrics for evaluation results given + a custom value for max_dets_per_image + """ + + def _summarize(ap=1, iouThr=None, areaRng="all", maxDets=100): + p = self.params + iStr = " {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}" + titleStr = "Average Precision" if ap == 1 else "Average Recall" + typeStr = "(AP)" if ap == 1 else "(AR)" + iouStr = ( + "{:0.2f}:{:0.2f}".format(p.iouThrs[0], p.iouThrs[-1]) + if iouThr is None + else "{:0.2f}".format(iouThr) + ) + + aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng] + mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets] + if ap == 1: + # dimension of precision: [TxRxKxAxM] + s = self.eval["precision"] + # IoU + if iouThr is not None: + t = np.where(iouThr == p.iouThrs)[0] + s = s[t] + s = s[:, :, :, aind, mind] + else: + # dimension of recall: [TxKxAxM] + s = self.eval["recall"] + if iouThr is not None: + t = np.where(iouThr == p.iouThrs)[0] + s = s[t] + s = s[:, :, aind, mind] + if len(s[s > -1]) == 0: + mean_s = -1 + else: + mean_s = np.mean(s[s > -1]) + print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s)) + return mean_s + + def _summarizeDets(): + stats = np.zeros((12,)) + # Evaluate AP using the custom limit on maximum detections per image + stats[0] = _summarize(1, maxDets=self.params.maxDets[2]) + stats[1] = _summarize(1, iouThr=0.5, maxDets=self.params.maxDets[2]) + stats[2] = _summarize(1, iouThr=0.75, maxDets=self.params.maxDets[2]) + stats[3] = _summarize(1, areaRng="small", maxDets=self.params.maxDets[2]) + stats[4] = _summarize(1, areaRng="medium", maxDets=self.params.maxDets[2]) + stats[5] = _summarize(1, areaRng="large", maxDets=self.params.maxDets[2]) + stats[6] = _summarize(0, maxDets=self.params.maxDets[0]) + stats[7] = _summarize(0, maxDets=self.params.maxDets[1]) + stats[8] = _summarize(0, maxDets=self.params.maxDets[2]) + stats[9] = _summarize(0, areaRng="small", maxDets=self.params.maxDets[2]) + stats[10] = _summarize(0, areaRng="medium", maxDets=self.params.maxDets[2]) + stats[11] = _summarize(0, areaRng="large", maxDets=self.params.maxDets[2]) + return stats + + def _summarizeKps(): + stats = np.zeros((10,)) + stats[0] = _summarize(1, maxDets=20) + stats[1] = _summarize(1, maxDets=20, iouThr=0.5) + stats[2] = _summarize(1, maxDets=20, iouThr=0.75) + stats[3] = _summarize(1, maxDets=20, areaRng="medium") + stats[4] = _summarize(1, maxDets=20, areaRng="large") + stats[5] = _summarize(0, maxDets=20) + stats[6] = _summarize(0, maxDets=20, iouThr=0.5) + stats[7] = _summarize(0, maxDets=20, iouThr=0.75) + stats[8] = _summarize(0, maxDets=20, areaRng="medium") + stats[9] = _summarize(0, maxDets=20, areaRng="large") + return stats + + if not self.eval: + raise Exception("Please run accumulate() first") + iouType = self.params.iouType + if iouType == "segm" or iouType == "bbox": + summarize = _summarizeDets + elif iouType == "keypoints": + summarize = _summarizeKps + self.stats = summarize() + + def __str__(self): + self.summarize() \ No newline at end of file diff --git a/annotator/oneformer/oneformer/evaluation/evaluator.py b/annotator/oneformer/oneformer/evaluation/evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..2c85d90eaa5236773a901a68c43c28d42bfd47ec --- /dev/null +++ b/annotator/oneformer/oneformer/evaluation/evaluator.py @@ -0,0 +1,228 @@ +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/detectron2/blob/main/detectron2/evaluation/evaluator.py +# Modified by Jitesh Jain (https://github.com/praeclarumjj3) +# ------------------------------------------------------------------------------ + +import datetime +import logging +import time +from collections import OrderedDict, abc +from contextlib import ExitStack, contextmanager +from typing import List, Union +import torch +from torch import nn + +from annotator.oneformer.detectron2.utils.comm import get_world_size, is_main_process +from annotator.oneformer.detectron2.utils.logger import log_every_n_seconds + + +class DatasetEvaluator: + """ + Base class for a dataset evaluator. + + The function :func:`inference_on_dataset` runs the model over + all samples in the dataset, and have a DatasetEvaluator to process the inputs/outputs. + + This class will accumulate information of the inputs/outputs (by :meth:`process`), + and produce evaluation results in the end (by :meth:`evaluate`). + """ + + def reset(self): + """ + Preparation for a new round of evaluation. + Should be called before starting a round of evaluation. + """ + pass + + def process(self, inputs, outputs): + """ + Process the pair of inputs and outputs. + If they contain batches, the pairs can be consumed one-by-one using `zip`: + + .. code-block:: python + + for input_, output in zip(inputs, outputs): + # do evaluation on single input/output pair + ... + + Args: + inputs (list): the inputs that's used to call the model. + outputs (list): the return value of `model(inputs)` + """ + pass + + def evaluate(self): + """ + Evaluate/summarize the performance, after processing all input/output pairs. + + Returns: + dict: + A new evaluator class can return a dict of arbitrary format + as long as the user can process the results. + In our train_net.py, we expect the following format: + + * key: the name of the task (e.g., bbox) + * value: a dict of {metric name: score}, e.g.: {"AP50": 80} + """ + pass + + +class DatasetEvaluators(DatasetEvaluator): + """ + Wrapper class to combine multiple :class:`DatasetEvaluator` instances. + + This class dispatches every evaluation call to + all of its :class:`DatasetEvaluator`. + """ + + def __init__(self, evaluators): + """ + Args: + evaluators (list): the evaluators to combine. + """ + super().__init__() + self._evaluators = evaluators + + def reset(self): + for evaluator in self._evaluators: + evaluator.reset() + + def process(self, inputs, outputs): + for evaluator in self._evaluators: + evaluator.process(inputs, outputs) + + def evaluate(self): + results = OrderedDict() + for evaluator in self._evaluators: + result = evaluator.evaluate() + if is_main_process() and result is not None: + for k, v in result.items(): + assert ( + k not in results + ), "Different evaluators produce results with the same key {}".format(k) + results[k] = v + return results + + +def inference_on_dataset( + model, data_loader, evaluator: Union[DatasetEvaluator, List[DatasetEvaluator], None] +): + """ + Run model on the data_loader and evaluate the metrics with evaluator. + Also benchmark the inference speed of `model.__call__` accurately. + The model will be used in eval mode. + + Args: + model (callable): a callable which takes an object from + `data_loader` and returns some outputs. + + If it's an nn.Module, it will be temporarily set to `eval` mode. + If you wish to evaluate a model in `training` mode instead, you can + wrap the given model and override its behavior of `.eval()` and `.train()`. + data_loader: an iterable object with a length. + The elements it generates will be the inputs to the model. + evaluator: the evaluator(s) to run. Use `None` if you only want to benchmark, + but don't want to do any evaluation. + + Returns: + The return value of `evaluator.evaluate()` + """ + num_devices = get_world_size() + logger = logging.getLogger(__name__) + logger.info("Start inference on {} batches".format(len(data_loader))) + + total = len(data_loader) # inference data loader must have a fixed length + if evaluator is None: + # create a no-op evaluator + evaluator = DatasetEvaluators([]) + if isinstance(evaluator, abc.MutableSequence): + evaluator = DatasetEvaluators(evaluator) + evaluator.reset() + + num_warmup = min(5, total - 1) + start_time = time.perf_counter() + total_data_time = 0 + total_compute_time = 0 + total_eval_time = 0 + with ExitStack() as stack: + if isinstance(model, nn.Module): + stack.enter_context(inference_context(model)) + stack.enter_context(torch.no_grad()) + + start_data_time = time.perf_counter() + for idx, inputs in enumerate(data_loader): + total_data_time += time.perf_counter() - start_data_time + if idx == num_warmup: + start_time = time.perf_counter() + total_data_time = 0 + total_compute_time = 0 + total_eval_time = 0 + + start_compute_time = time.perf_counter() + outputs = model(inputs) + if torch.cuda.is_available(): + torch.cuda.synchronize() + total_compute_time += time.perf_counter() - start_compute_time + + start_eval_time = time.perf_counter() + evaluator.process(inputs, outputs) + total_eval_time += time.perf_counter() - start_eval_time + + iters_after_start = idx + 1 - num_warmup * int(idx >= num_warmup) + data_seconds_per_iter = total_data_time / iters_after_start + compute_seconds_per_iter = total_compute_time / iters_after_start + eval_seconds_per_iter = total_eval_time / iters_after_start + total_seconds_per_iter = (time.perf_counter() - start_time) / iters_after_start + if idx >= num_warmup * 2 or compute_seconds_per_iter > 5: + eta = datetime.timedelta(seconds=int(total_seconds_per_iter * (total - idx - 1))) + log_every_n_seconds( + logging.INFO, + ( + f"Inference done {idx + 1}/{total}. " + f"Dataloading: {data_seconds_per_iter:.4f} s/iter. " + f"Inference: {compute_seconds_per_iter:.4f} s/iter. " + f"Eval: {eval_seconds_per_iter:.4f} s/iter. " + f"Total: {total_seconds_per_iter:.4f} s/iter. " + f"ETA={eta}" + ), + n=5, + ) + start_data_time = time.perf_counter() + + # Measure the time only for this worker (before the synchronization barrier) + total_time = time.perf_counter() - start_time + total_time_str = str(datetime.timedelta(seconds=total_time)) + # NOTE this format is parsed by grep + logger.info( + "Total inference time: {} ({:.6f} s / iter per device, on {} devices)".format( + total_time_str, total_time / (total - num_warmup), num_devices + ) + ) + total_compute_time_str = str(datetime.timedelta(seconds=int(total_compute_time))) + logger.info( + "Total inference pure compute time: {} ({:.6f} s / iter per device, on {} devices)".format( + total_compute_time_str, total_compute_time / (total - num_warmup), num_devices + ) + ) + + results = evaluator.evaluate() + # An evaluator may return None when not in main process. + # Replace it by an empty dict instead to make it easier for downstream code to handle + if results is None: + results = {} + return results + + +@contextmanager +def inference_context(model): + """ + A context where the model is temporarily changed to eval mode, + and restored to previous mode afterwards. + + Args: + model: a torch Module + """ + training_mode = model.training + model.eval() + yield + model.train(training_mode) diff --git a/annotator/oneformer/oneformer/evaluation/instance_evaluation.py b/annotator/oneformer/oneformer/evaluation/instance_evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..10cd092a482c1608c1a20f79a94e79a6153fc48a --- /dev/null +++ b/annotator/oneformer/oneformer/evaluation/instance_evaluation.py @@ -0,0 +1,110 @@ +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/Mask2Former/blob/main/mask2former/evaluation/instance_evaluation.py +# ------------------------------------------------------------------------------ + +import contextlib +import copy +import io +import itertools +import json +import logging +import numpy as np +import os +import pickle +from collections import OrderedDict +import pycocotools.mask as mask_util +import torch +from pycocotools.coco import COCO +from pycocotools.cocoeval import COCOeval +from tabulate import tabulate + +import annotator.oneformer.detectron2.utils.comm as comm +from annotator.oneformer.detectron2.config import CfgNode +from annotator.oneformer.detectron2.data import MetadataCatalog +from annotator.oneformer.detectron2.data.datasets.coco import convert_to_coco_json +from annotator.oneformer.detectron2.evaluation.coco_evaluation import COCOEvaluator, _evaluate_predictions_on_coco +from annotator.oneformer.detectron2.evaluation.fast_eval_api import COCOeval_opt +from annotator.oneformer.detectron2.structures import Boxes, BoxMode, pairwise_iou +from annotator.oneformer.detectron2.utils.file_io import PathManager +from annotator.oneformer.detectron2.utils.logger import create_small_table + + +# modified from COCOEvaluator for instance segmetnat +class InstanceSegEvaluator(COCOEvaluator): + """ + Evaluate AR for object proposals, AP for instance detection/segmentation, AP + for keypoint detection outputs using COCO's metrics. + See http://cocodataset.org/#detection-eval and + http://cocodataset.org/#keypoints-eval to understand its metrics. + The metrics range from 0 to 100 (instead of 0 to 1), where a -1 or NaN means + the metric cannot be computed (e.g. due to no predictions made). + + In addition to COCO, this evaluator is able to support any bounding box detection, + instance segmentation, or keypoint detection dataset. + """ + + def _eval_predictions(self, predictions, img_ids=None): + """ + Evaluate predictions. Fill self._results with the metrics of the tasks. + """ + self._logger.info("Preparing results for COCO format ...") + coco_results = list(itertools.chain(*[x["instances"] for x in predictions])) + tasks = self._tasks or self._tasks_from_predictions(coco_results) + + # unmap the category ids for COCO + if hasattr(self._metadata, "thing_dataset_id_to_contiguous_id"): + dataset_id_to_contiguous_id = self._metadata.thing_dataset_id_to_contiguous_id + # all_contiguous_ids = list(dataset_id_to_contiguous_id.values()) + # num_classes = len(all_contiguous_ids) + # assert min(all_contiguous_ids) == 0 and max(all_contiguous_ids) == num_classes - 1 + + reverse_id_mapping = {v: k for k, v in dataset_id_to_contiguous_id.items()} + for result in coco_results: + category_id = result["category_id"] + # assert category_id < num_classes, ( + # f"A prediction has class={category_id}, " + # f"but the dataset only has {num_classes} classes and " + # f"predicted class id should be in [0, {num_classes - 1}]." + # ) + assert category_id in reverse_id_mapping, ( + f"A prediction has class={category_id}, " + f"but the dataset only has class ids in {dataset_id_to_contiguous_id}." + ) + result["category_id"] = reverse_id_mapping[category_id] + + if self._output_dir: + file_path = os.path.join(self._output_dir, "coco_instances_results.json") + self._logger.info("Saving results to {}".format(file_path)) + with PathManager.open(file_path, "w") as f: + f.write(json.dumps(coco_results)) + f.flush() + + if not self._do_evaluation: + self._logger.info("Annotations are not available for evaluation.") + return + + self._logger.info( + "Evaluating predictions with {} COCO API...".format( + "unofficial" if self._use_fast_impl else "official" + ) + ) + for task in sorted(tasks): + assert task in {"bbox", "segm", "keypoints"}, f"Got unknown task: {task}!" + coco_eval = ( + _evaluate_predictions_on_coco( + self._coco_api, + coco_results, + task, + kpt_oks_sigmas=self._kpt_oks_sigmas, + use_fast_impl=self._use_fast_impl, + img_ids=img_ids, + max_dets_per_image=self._max_dets_per_image, + ) + if len(coco_results) > 0 + else None # cocoapi does not handle empty results very well + ) + + res = self._derive_coco_results( + coco_eval, task, class_names=self._metadata.get("thing_classes") + ) + self._results[task] = res diff --git a/annotator/oneformer/oneformer/modeling/__init__.py b/annotator/oneformer/oneformer/modeling/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4e1338369a958062d6ca4a122435b2be6ad27315 --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/__init__.py @@ -0,0 +1,5 @@ +from .backbone.swin import D2SwinTransformer +from .backbone.dinat import D2DiNAT +from .pixel_decoder.fpn import BasePixelDecoder +from .pixel_decoder.msdeformattn import MSDeformAttnPixelDecoder +from .meta_arch.oneformer_head import OneFormerHead diff --git a/annotator/oneformer/oneformer/modeling/backbone/__init__.py b/annotator/oneformer/oneformer/modeling/backbone/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9020c2df23e2af280b7bb168b996ae9eaf312eb8 --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/backbone/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Facebook, Inc. and its affiliates. diff --git a/annotator/oneformer/oneformer/modeling/backbone/dinat.py b/annotator/oneformer/oneformer/modeling/backbone/dinat.py new file mode 100644 index 0000000000000000000000000000000000000000..17027574c7acc49aa6c452cb546736737ff1c752 --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/backbone/dinat.py @@ -0,0 +1,324 @@ +# -------------------------------------------------------- +# Neighborhood Attention Transformer +# Licensed under The MIT License +# Written by Ali Hassani +# -------------------------------------------------------- + +# Modified by Jitesh Jain + +import torch +import torch.nn as nn +from timm.models.layers import DropPath +from annotator.oneformer.detectron2.modeling import BACKBONE_REGISTRY, Backbone, ShapeSpec + +class NeighborhoodAttention(nn.Module): + """ + Neighborhood Attention 2D Module + """ + + def __init__( + self, + dim, + num_heads, + kernel_size, + dilation=1, + bias=True, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + ): + super().__init__() + + + def forward(self, x): + + return x + + def extra_repr(self) -> str: + return ( + f"head_dim={self.head_dim}, num_heads={self.num_heads}, " + + f"kernel_size={self.kernel_size}, dilation={self.dilation}, " + + f"rel_pos_bias={self.rpb is not None}" + ) + +class ConvTokenizer(nn.Module): + def __init__(self, in_chans=3, embed_dim=96, norm_layer=None): + super().__init__() + self.proj = nn.Sequential( + nn.Conv2d(in_chans, embed_dim // 2, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)), + nn.Conv2d(embed_dim // 2, embed_dim, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1)), + ) + if norm_layer is not None: + self.norm = norm_layer(embed_dim) + else: + self.norm = None + + def forward(self, x): + x = self.proj(x).permute(0, 2, 3, 1) + if self.norm is not None: + x = self.norm(x) + return x + + +class ConvDownsampler(nn.Module): + def __init__(self, dim, norm_layer=nn.LayerNorm): + super().__init__() + self.reduction = nn.Conv2d(dim, 2 * dim, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False) + self.norm = norm_layer(2 * dim) + + def forward(self, x): + x = self.reduction(x.permute(0, 3, 1, 2)).permute(0, 2, 3, 1) + x = self.norm(x) + return x + + +class Mlp(nn.Module): + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class NATLayer(nn.Module): + def __init__(self, dim, num_heads, kernel_size=7, dilation=None, + mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., drop_path=0., + act_layer=nn.GELU, norm_layer=nn.LayerNorm, layer_scale=None): + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.mlp_ratio = mlp_ratio + + self.norm1 = norm_layer(dim) + self.attn = NeighborhoodAttention( + dim, kernel_size=kernel_size, dilation=dilation, num_heads=num_heads, + qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop) + + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + self.mlp = Mlp(in_features=dim, hidden_features=int(dim * mlp_ratio), act_layer=act_layer, drop=drop) + self.layer_scale = False + if layer_scale is not None and type(layer_scale) in [int, float]: + self.layer_scale = True + self.gamma1 = nn.Parameter(layer_scale * torch.ones(dim), requires_grad=True) + self.gamma2 = nn.Parameter(layer_scale * torch.ones(dim), requires_grad=True) + + def forward(self, x): + if not self.layer_scale: + shortcut = x + x = self.norm1(x) + x = self.attn(x) + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + return x + shortcut = x + x = self.norm1(x) + x = self.attn(x) + x = shortcut + self.drop_path(self.gamma1 * x) + x = x + self.drop_path(self.gamma2 * self.mlp(self.norm2(x))) + return x + + + +class NATBlock(nn.Module): + def __init__(self, dim, depth, num_heads, kernel_size, dilations=None, + downsample=True, + mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., + drop_path=0., norm_layer=nn.LayerNorm, layer_scale=None): + super().__init__() + self.dim = dim + self.depth = depth + + self.blocks = nn.ModuleList([ + NATLayer(dim=dim, + num_heads=num_heads, + kernel_size=kernel_size, + dilation=None if dilations is None else dilations[i], + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop, attn_drop=attn_drop, + drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer, + layer_scale=layer_scale) + for i in range(depth)]) + + self.downsample = None if not downsample else ConvDownsampler(dim=dim, norm_layer=norm_layer) + + def forward(self, x): + for blk in self.blocks: + x = blk(x) + if self.downsample is None: + return x, x + return self.downsample(x), x + + +class DiNAT(nn.Module): + def __init__(self, + embed_dim, + mlp_ratio, + depths, + num_heads, + drop_path_rate=0.2, + in_chans=3, + kernel_size=7, + dilations=None, + out_indices=(0, 1, 2, 3), + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + norm_layer=nn.LayerNorm, + frozen_stages=-1, + layer_scale=None, + **kwargs): + super().__init__() + self.num_levels = len(depths) + self.embed_dim = embed_dim + self.num_features = [int(embed_dim * 2 ** i) for i in range(self.num_levels)] + self.mlp_ratio = mlp_ratio + + self.patch_embed = ConvTokenizer(in_chans=in_chans, embed_dim=embed_dim, norm_layer=norm_layer) + + self.pos_drop = nn.Dropout(p=drop_rate) + + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))] + self.levels = nn.ModuleList() + for i in range(self.num_levels): + level = NATBlock(dim=int(embed_dim * 2 ** i), + depth=depths[i], + num_heads=num_heads[i], + kernel_size=kernel_size, + dilations=None if dilations is None else dilations[i], + mlp_ratio=self.mlp_ratio, + qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, + drop_path=dpr[sum(depths[:i]):sum(depths[:i + 1])], + norm_layer=norm_layer, + downsample=(i < self.num_levels - 1), + layer_scale=layer_scale) + self.levels.append(level) + + # add a norm layer for each output + self.out_indices = out_indices + for i_layer in self.out_indices: + layer = norm_layer(self.num_features[i_layer]) + layer_name = f'norm{i_layer}' + self.add_module(layer_name, layer) + + self.frozen_stages = frozen_stages + + def _freeze_stages(self): + if self.frozen_stages >= 0: + self.patch_embed.eval() + for param in self.patch_embed.parameters(): + param.requires_grad = False + + if self.frozen_stages >= 2: + for i in range(0, self.frozen_stages - 1): + m = self.network[i] + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def train(self, mode=True): + super(DiNAT, self).train(mode) + self._freeze_stages() + + def forward_embeddings(self, x): + x = self.patch_embed(x) + return x + + def forward_tokens(self, x): + outs = {} + for idx, level in enumerate(self.levels): + x, xo = level(x) + if idx in self.out_indices: + norm_layer = getattr(self, f'norm{idx}') + x_out = norm_layer(xo) + outs["res{}".format(idx + 2)] = x_out.permute(0, 3, 1, 2).contiguous() + return outs + + def forward(self, x): + x = self.forward_embeddings(x) + return self.forward_tokens(x) + + +@BACKBONE_REGISTRY.register() +class D2DiNAT(DiNAT, Backbone): + def __init__(self, cfg, input_shape): + + embed_dim = cfg.MODEL.DiNAT.EMBED_DIM + mlp_ratio = cfg.MODEL.DiNAT.MLP_RATIO + depths = cfg.MODEL.DiNAT.DEPTHS + num_heads = cfg.MODEL.DiNAT.NUM_HEADS + drop_path_rate = cfg.MODEL.DiNAT.DROP_PATH_RATE + kernel_size = cfg.MODEL.DiNAT.KERNEL_SIZE + out_indices = cfg.MODEL.DiNAT.OUT_INDICES + dilations = cfg.MODEL.DiNAT.DILATIONS + + super().__init__( + embed_dim=embed_dim, + mlp_ratio=mlp_ratio, + depths=depths, + num_heads=num_heads, + drop_path_rate=drop_path_rate, + kernel_size=kernel_size, + out_indices=out_indices, + dilations=dilations, + ) + + self._out_features = cfg.MODEL.DiNAT.OUT_FEATURES + + self._out_feature_strides = { + "res2": 4, + "res3": 8, + "res4": 16, + "res5": 32, + } + self._out_feature_channels = { + "res2": self.num_features[0], + "res3": self.num_features[1], + "res4": self.num_features[2], + "res5": self.num_features[3], + } + + def forward(self, x): + """ + Args: + x: Tensor of shape (N,C,H,W). H, W must be a multiple of ``self.size_divisibility``. + Returns: + dict[str->Tensor]: names and the corresponding features + """ + assert ( + x.dim() == 4 + ), f"DiNAT takes an input of shape (N, C, H, W). Got {x.shape} instead!" + outputs = {} + y = super().forward(x) + for k in y.keys(): + if k in self._out_features: + outputs[k] = y[k] + return outputs + + def output_shape(self): + return { + name: ShapeSpec( + channels=self._out_feature_channels[name], stride=self._out_feature_strides[name] + ) + for name in self._out_features + } + + @property + def size_divisibility(self): + return 32 diff --git a/annotator/oneformer/oneformer/modeling/backbone/swin.py b/annotator/oneformer/oneformer/modeling/backbone/swin.py new file mode 100644 index 0000000000000000000000000000000000000000..2380cde59570e5d5b8fb2536d0961f8e27a07fd4 --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/backbone/swin.py @@ -0,0 +1,771 @@ +# -------------------------------------------------------- +# Swin Transformer +# Copyright (c) 2021 Microsoft +# Licensed under The MIT License [see LICENSE for details] +# Written by Ze Liu, Yutong Lin, Yixuan Wei +# -------------------------------------------------------- + +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/Mask2Former +# ------------------------------------------------------------------------------ + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ + +from annotator.oneformer.detectron2.modeling import BACKBONE_REGISTRY, Backbone, ShapeSpec + + +class Mlp(nn.Module): + """Multilayer perceptron.""" + + def __init__( + self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.0 + ): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +def window_partition(x, window_size): + """ + Args: + x: (B, H, W, C) + window_size (int): window size + Returns: + windows: (num_windows*B, window_size, window_size, C) + """ + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + return windows + + +def window_reverse(windows, window_size, H, W): + """ + Args: + windows: (num_windows*B, window_size, window_size, C) + window_size (int): Window size + H (int): Height of image + W (int): Width of image + Returns: + x: (B, H, W, C) + """ + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +class WindowAttention(nn.Module): + """Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__( + self, + dim, + window_size, + num_heads, + qkv_bias=True, + qk_scale=None, + attn_drop=0.0, + proj_drop=0.0, + ): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim ** -0.5 + + # define a parameter table of relative position bias + self.relative_position_bias_table = nn.Parameter( + torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads) + ) # 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(self.window_size[0]) + coords_w = torch.arange(self.window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w])) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer("relative_position_index", relative_position_index) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + trunc_normal_(self.relative_position_bias_table, std=0.02) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, mask=None): + """Forward function. + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None + """ + B_, N, C = x.shape + qkv = ( + self.qkv(x) + .reshape(B_, N, 3, self.num_heads, C // self.num_heads) + .permute(2, 0, 3, 1, 4) + ) + q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = q @ k.transpose(-2, -1) + + relative_position_bias = self.relative_position_bias_table[ + self.relative_position_index.view(-1) + ].view( + self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1 + ) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute( + 2, 0, 1 + ).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class SwinTransformerBlock(nn.Module): + """Swin Transformer Block. + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__( + self, + dim, + num_heads, + window_size=7, + shift_size=0, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop=0.0, + attn_drop=0.0, + drop_path=0.0, + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + ): + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size" + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention( + dim, + window_size=to_2tuple(self.window_size), + num_heads=num_heads, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + attn_drop=attn_drop, + proj_drop=drop, + ) + + self.drop_path = DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop + ) + + self.H = None + self.W = None + + def forward(self, x, mask_matrix): + """Forward function. + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + mask_matrix: Attention mask for cyclic shift. + """ + B, L, C = x.shape + H, W = self.H, self.W + assert L == H * W, "input feature has wrong size" + + shortcut = x + x = self.norm1(x) + x = x.view(B, H, W, C) + + # pad feature maps to multiples of window size + pad_l = pad_t = 0 + pad_r = (self.window_size - W % self.window_size) % self.window_size + pad_b = (self.window_size - H % self.window_size) % self.window_size + x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b)) + _, Hp, Wp, _ = x.shape + + # cyclic shift + if self.shift_size > 0: + shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) + attn_mask = mask_matrix + else: + shifted_x = x + attn_mask = None + + # partition windows + x_windows = window_partition( + shifted_x, self.window_size + ) # nW*B, window_size, window_size, C + x_windows = x_windows.view( + -1, self.window_size * self.window_size, C + ) # nW*B, window_size*window_size, C + + # W-MSA/SW-MSA + attn_windows = self.attn(x_windows, mask=attn_mask) # nW*B, window_size*window_size, C + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C) + shifted_x = window_reverse(attn_windows, self.window_size, Hp, Wp) # B H' W' C + + # reverse cyclic shift + if self.shift_size > 0: + x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2)) + else: + x = shifted_x + + if pad_r > 0 or pad_b > 0: + x = x[:, :H, :W, :].contiguous() + + x = x.view(B, H * W, C) + + # FFN + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + return x + + +class PatchMerging(nn.Module): + """Patch Merging Layer + Args: + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, dim, norm_layer=nn.LayerNorm): + super().__init__() + self.dim = dim + self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False) + self.norm = norm_layer(4 * dim) + + def forward(self, x, H, W): + """Forward function. + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + """ + B, L, C = x.shape + assert L == H * W, "input feature has wrong size" + + x = x.view(B, H, W, C) + + # padding + pad_input = (H % 2 == 1) or (W % 2 == 1) + if pad_input: + x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2)) + + x0 = x[:, 0::2, 0::2, :] # B H/2 W/2 C + x1 = x[:, 1::2, 0::2, :] # B H/2 W/2 C + x2 = x[:, 0::2, 1::2, :] # B H/2 W/2 C + x3 = x[:, 1::2, 1::2, :] # B H/2 W/2 C + x = torch.cat([x0, x1, x2, x3], -1) # B H/2 W/2 4*C + x = x.view(B, -1, 4 * C) # B H/2*W/2 4*C + + x = self.norm(x) + x = self.reduction(x) + + return x + + +class BasicLayer(nn.Module): + """A basic Swin Transformer layer for one stage. + Args: + dim (int): Number of feature channels + depth (int): Depths of this stage. + num_heads (int): Number of attention head. + window_size (int): Local window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__( + self, + dim, + depth, + num_heads, + window_size=7, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop=0.0, + attn_drop=0.0, + drop_path=0.0, + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False, + ): + super().__init__() + self.window_size = window_size + self.shift_size = window_size // 2 + self.depth = depth + self.use_checkpoint = use_checkpoint + + # build blocks + self.blocks = nn.ModuleList( + [ + SwinTransformerBlock( + dim=dim, + num_heads=num_heads, + window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer, + ) + for i in range(depth) + ] + ) + + # patch merging layer + if downsample is not None: + self.downsample = downsample(dim=dim, norm_layer=norm_layer) + else: + self.downsample = None + + def forward(self, x, H, W): + """Forward function. + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + """ + + # calculate attention mask for SW-MSA + Hp = int(np.ceil(H / self.window_size)) * self.window_size + Wp = int(np.ceil(W / self.window_size)) * self.window_size + img_mask = torch.zeros((1, Hp, Wp, 1), device=x.device) # 1 Hp Wp 1 + h_slices = ( + slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None), + ) + w_slices = ( + slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None), + ) + cnt = 0 + for h in h_slices: + for w in w_slices: + img_mask[:, h, w, :] = cnt + cnt += 1 + + mask_windows = window_partition( + img_mask, self.window_size + ) # nW, window_size, window_size, 1 + mask_windows = mask_windows.view(-1, self.window_size * self.window_size) + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill( + attn_mask == 0, float(0.0) + ) + + for blk in self.blocks: + blk.H, blk.W = H, W + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x, attn_mask) + else: + x = blk(x, attn_mask) + if self.downsample is not None: + x_down = self.downsample(x, H, W) + Wh, Ww = (H + 1) // 2, (W + 1) // 2 + return x, H, W, x_down, Wh, Ww + else: + return x, H, W, x, H, W + + +class PatchEmbed(nn.Module): + """Image to Patch Embedding + Args: + patch_size (int): Patch token size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__(self, patch_size=4, in_chans=3, embed_dim=96, norm_layer=None): + super().__init__() + patch_size = to_2tuple(patch_size) + self.patch_size = patch_size + + self.in_chans = in_chans + self.embed_dim = embed_dim + + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + if norm_layer is not None: + self.norm = norm_layer(embed_dim) + else: + self.norm = None + + def forward(self, x): + """Forward function.""" + # padding + _, _, H, W = x.size() + if W % self.patch_size[1] != 0: + x = F.pad(x, (0, self.patch_size[1] - W % self.patch_size[1])) + if H % self.patch_size[0] != 0: + x = F.pad(x, (0, 0, 0, self.patch_size[0] - H % self.patch_size[0])) + + x = self.proj(x) # B C Wh Ww + if self.norm is not None: + Wh, Ww = x.size(2), x.size(3) + x = x.flatten(2).transpose(1, 2) + x = self.norm(x) + x = x.transpose(1, 2).view(-1, self.embed_dim, Wh, Ww) + + return x + + +class SwinTransformer(nn.Module): + """Swin Transformer backbone. + A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows` - + https://arxiv.org/pdf/2103.14030 + Args: + pretrain_img_size (int): Input image size for training the pretrained model, + used in absolute postion embedding. Default 224. + patch_size (int | tuple(int)): Patch size. Default: 4. + in_chans (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + depths (tuple[int]): Depths of each Swin Transformer stage. + num_heads (tuple[int]): Number of attention head of each stage. + window_size (int): Window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. + drop_rate (float): Dropout rate. + attn_drop_rate (float): Attention dropout rate. Default: 0. + drop_path_rate (float): Stochastic depth rate. Default: 0.2. + norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm. + ape (bool): If True, add absolute position embedding to the patch embedding. Default: False. + patch_norm (bool): If True, add normalization after patch embedding. Default: True. + out_indices (Sequence[int]): Output from which stages. + frozen_stages (int): Stages to be frozen (stop grad and set eval mode). + -1 means not freezing any parameters. + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__( + self, + pretrain_img_size=224, + patch_size=4, + in_chans=3, + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_size=7, + mlp_ratio=4.0, + qkv_bias=True, + qk_scale=None, + drop_rate=0.0, + attn_drop_rate=0.0, + drop_path_rate=0.2, + norm_layer=nn.LayerNorm, + ape=False, + patch_norm=True, + out_indices=(0, 1, 2, 3), + frozen_stages=-1, + use_checkpoint=False, + ): + super().__init__() + + self.pretrain_img_size = pretrain_img_size + self.num_layers = len(depths) + self.embed_dim = embed_dim + self.ape = ape + self.patch_norm = patch_norm + self.out_indices = out_indices + self.frozen_stages = frozen_stages + + # split image into non-overlapping patches + self.patch_embed = PatchEmbed( + patch_size=patch_size, + in_chans=in_chans, + embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None, + ) + + # absolute position embedding + if self.ape: + pretrain_img_size = to_2tuple(pretrain_img_size) + patch_size = to_2tuple(patch_size) + patches_resolution = [ + pretrain_img_size[0] // patch_size[0], + pretrain_img_size[1] // patch_size[1], + ] + + self.absolute_pos_embed = nn.Parameter( + torch.zeros(1, embed_dim, patches_resolution[0], patches_resolution[1]) + ) + trunc_normal_(self.absolute_pos_embed, std=0.02) + + self.pos_drop = nn.Dropout(p=drop_rate) + + # stochastic depth + dpr = [ + x.item() for x in torch.linspace(0, drop_path_rate, sum(depths)) + ] # stochastic depth decay rule + + # build layers + self.layers = nn.ModuleList() + for i_layer in range(self.num_layers): + layer = BasicLayer( + dim=int(embed_dim * 2 ** i_layer), + depth=depths[i_layer], + num_heads=num_heads[i_layer], + window_size=window_size, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[sum(depths[:i_layer]) : sum(depths[: i_layer + 1])], + norm_layer=norm_layer, + downsample=PatchMerging if (i_layer < self.num_layers - 1) else None, + use_checkpoint=use_checkpoint, + ) + self.layers.append(layer) + + num_features = [int(embed_dim * 2 ** i) for i in range(self.num_layers)] + self.num_features = num_features + + # add a norm layer for each output + for i_layer in out_indices: + layer = norm_layer(num_features[i_layer]) + layer_name = f"norm{i_layer}" + self.add_module(layer_name, layer) + + self._freeze_stages() + + def _freeze_stages(self): + if self.frozen_stages >= 0: + self.patch_embed.eval() + for param in self.patch_embed.parameters(): + param.requires_grad = False + + if self.frozen_stages >= 1 and self.ape: + self.absolute_pos_embed.requires_grad = False + + if self.frozen_stages >= 2: + self.pos_drop.eval() + for i in range(0, self.frozen_stages - 1): + m = self.layers[i] + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + + def _init_weights(m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + def forward(self, x): + """Forward function.""" + x = self.patch_embed(x) + + Wh, Ww = x.size(2), x.size(3) + if self.ape: + # interpolate the position embedding to the corresponding size + absolute_pos_embed = F.interpolate( + self.absolute_pos_embed, size=(Wh, Ww), mode="bicubic" + ) + x = (x + absolute_pos_embed).flatten(2).transpose(1, 2) # B Wh*Ww C + else: + x = x.flatten(2).transpose(1, 2) + x = self.pos_drop(x) + + outs = {} + for i in range(self.num_layers): + layer = self.layers[i] + x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww) + + if i in self.out_indices: + norm_layer = getattr(self, f"norm{i}") + x_out = norm_layer(x_out) + + out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous() + outs["res{}".format(i + 2)] = out + + return outs + + def train(self, mode=True): + """Convert the model into training mode while keep layers freezed.""" + super(SwinTransformer, self).train(mode) + self._freeze_stages() + + +@BACKBONE_REGISTRY.register() +class D2SwinTransformer(SwinTransformer, Backbone): + def __init__(self, cfg, input_shape): + + pretrain_img_size = cfg.MODEL.SWIN.PRETRAIN_IMG_SIZE + patch_size = cfg.MODEL.SWIN.PATCH_SIZE + in_chans = 3 + embed_dim = cfg.MODEL.SWIN.EMBED_DIM + depths = cfg.MODEL.SWIN.DEPTHS + num_heads = cfg.MODEL.SWIN.NUM_HEADS + window_size = cfg.MODEL.SWIN.WINDOW_SIZE + mlp_ratio = cfg.MODEL.SWIN.MLP_RATIO + qkv_bias = cfg.MODEL.SWIN.QKV_BIAS + qk_scale = cfg.MODEL.SWIN.QK_SCALE + drop_rate = cfg.MODEL.SWIN.DROP_RATE + attn_drop_rate = cfg.MODEL.SWIN.ATTN_DROP_RATE + drop_path_rate = cfg.MODEL.SWIN.DROP_PATH_RATE + norm_layer = nn.LayerNorm + ape = cfg.MODEL.SWIN.APE + patch_norm = cfg.MODEL.SWIN.PATCH_NORM + use_checkpoint = cfg.MODEL.SWIN.USE_CHECKPOINT + + super().__init__( + pretrain_img_size, + patch_size, + in_chans, + embed_dim, + depths, + num_heads, + window_size, + mlp_ratio, + qkv_bias, + qk_scale, + drop_rate, + attn_drop_rate, + drop_path_rate, + norm_layer, + ape, + patch_norm, + use_checkpoint=use_checkpoint, + ) + + self._out_features = cfg.MODEL.SWIN.OUT_FEATURES + + self._out_feature_strides = { + "res2": 4, + "res3": 8, + "res4": 16, + "res5": 32, + } + self._out_feature_channels = { + "res2": self.num_features[0], + "res3": self.num_features[1], + "res4": self.num_features[2], + "res5": self.num_features[3], + } + + def forward(self, x): + """ + Args: + x: Tensor of shape (N,C,H,W). H, W must be a multiple of ``self.size_divisibility``. + Returns: + dict[str->Tensor]: names and the corresponding features + """ + assert ( + x.dim() == 4 + ), f"SwinTransformer takes an input of shape (N, C, H, W). Got {x.shape} instead!" + outputs = {} + y = super().forward(x) + for k in y.keys(): + if k in self._out_features: + outputs[k] = y[k] + return outputs + + def output_shape(self): + return { + name: ShapeSpec( + channels=self._out_feature_channels[name], stride=self._out_feature_strides[name] + ) + for name in self._out_features + } + + @property + def size_divisibility(self): + return 32 diff --git a/annotator/oneformer/oneformer/modeling/matcher.py b/annotator/oneformer/oneformer/modeling/matcher.py new file mode 100644 index 0000000000000000000000000000000000000000..4dba337a0f99ccd394931f52b063c8fb575bafbd --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/matcher.py @@ -0,0 +1,212 @@ +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/Mask2Former/blob/main/mask2former/modeling/matcher.py +# Modified by Jitesh Jain (https://github.com/praeclarumjj3) +# ------------------------------------------------------------------------------ + +""" +Modules to compute the matching cost and solve the corresponding LSAP. +""" +import torch +import torch.nn.functional as F +from scipy.optimize import linear_sum_assignment +from torch import nn +from torch.cuda.amp import autocast +import numpy as np + +# from annotator.oneformer.detectron2.projects.point_rend.point_features import point_sample + + +def linear_sum_assignment_with_nan(cost_matrix): + cost_matrix = np.asarray(cost_matrix) + nan = np.isnan(cost_matrix).any() + nan_all = np.isnan(cost_matrix).all() + empty = cost_matrix.size == 0 + + if not empty: + if nan_all: + print('Matrix contains all NaN values!') + elif nan: + print('Matrix contains NaN values!') + + if nan_all: + cost_matrix = np.empty(shape=(0, 0)) + elif nan: + cost_matrix[np.isnan(cost_matrix)] = 100 + + return linear_sum_assignment(cost_matrix) + +def batch_dice_loss(inputs: torch.Tensor, targets: torch.Tensor): + """ + Compute the DICE loss, similar to generalized IOU for masks + Args: + inputs: A float tensor of arbitrary shape. + The predictions for each example. + targets: A float tensor with the same shape as inputs. Stores the binary + classification label for each element in inputs + (0 for the negative class and 1 for the positive class). + """ + inputs = inputs.sigmoid() + inputs = inputs.flatten(1) + numerator = 2 * torch.einsum("nc,mc->nm", inputs, targets) + denominator = inputs.sum(-1)[:, None] + targets.sum(-1)[None, :] + loss = 1 - (numerator + 1) / (denominator + 1) + return loss + + +batch_dice_loss_jit = torch.jit.script( + batch_dice_loss +) # type: torch.jit.ScriptModule + + +def batch_sigmoid_ce_loss(inputs: torch.Tensor, targets: torch.Tensor): + """ + Args: + inputs: A float tensor of arbitrary shape. + The predictions for each example. + targets: A float tensor with the same shape as inputs. Stores the binary + classification label for each element in inputs + (0 for the negative class and 1 for the positive class). + Returns: + Loss tensor + """ + hw = inputs.shape[1] + + pos = F.binary_cross_entropy_with_logits( + inputs, torch.ones_like(inputs), reduction="none" + ) + neg = F.binary_cross_entropy_with_logits( + inputs, torch.zeros_like(inputs), reduction="none" + ) + + loss = torch.einsum("nc,mc->nm", pos, targets) + torch.einsum( + "nc,mc->nm", neg, (1 - targets) + ) + + return loss / hw + + +batch_sigmoid_ce_loss_jit = torch.jit.script( + batch_sigmoid_ce_loss +) # type: torch.jit.ScriptModule + + +class HungarianMatcher(nn.Module): + """This class computes an assignment between the targets and the predictions of the network + + For efficiency reasons, the targets don't include the no_object. Because of this, in general, + there are more predictions than targets. In this case, we do a 1-to-1 matching of the best predictions, + while the others are un-matched (and thus treated as non-objects). + """ + + def __init__(self, cost_class: float = 1, cost_mask: float = 1, + cost_dice: float = 1, num_points: int = 0): + """Creates the matcher + + Params: + cost_class: This is the relative weight of the classification error in the matching cost + cost_mask: This is the relative weight of the focal loss of the binary mask in the matching cost + cost_dice: This is the relative weight of the dice loss of the binary mask in the matching cost + """ + super().__init__() + self.cost_class = cost_class + self.cost_mask = cost_mask + self.cost_dice = cost_dice + + assert cost_class != 0 or cost_mask != 0 or cost_dice != 0, "all costs cant be 0" + + self.num_points = num_points + + @torch.no_grad() + def memory_efficient_forward(self, outputs, targets): + """More memory-friendly matching""" + bs, num_queries = outputs["pred_logits"].shape[:2] + + indices = [] + + # Iterate through batch size + for b in range(bs): + out_prob = outputs["pred_logits"][b].softmax(-1) # [num_queries, num_classes] + tgt_ids = targets[b]["labels"] + + # Compute the classification cost. Contrary to the loss, we don't use the NLL, + # but approximate it in 1 - proba[target class]. + # The 1 is a constant that doesn't change the matching, it can be ommitted. + cost_class = -out_prob[:, tgt_ids] + + out_mask = outputs["pred_masks"][b] # [num_queries, H_pred, W_pred] + # gt masks are already padded when preparing target + tgt_mask = targets[b]["masks"].to(out_mask) + + out_mask = out_mask[:, None] + tgt_mask = tgt_mask[:, None] + # all masks share the same set of points for efficient matching! + point_coords = torch.rand(1, self.num_points, 2, device=out_mask.device) + # get gt labels + tgt_mask = point_sample( + tgt_mask, + point_coords.repeat(tgt_mask.shape[0], 1, 1), + align_corners=False, + ).squeeze(1) + + out_mask = point_sample( + out_mask, + point_coords.repeat(out_mask.shape[0], 1, 1), + align_corners=False, + ).squeeze(1) + + with autocast(enabled=False): + out_mask = out_mask.float() + tgt_mask = tgt_mask.float() + # Compute the focal loss between masks + cost_mask = batch_sigmoid_ce_loss_jit(out_mask, tgt_mask) + # Compute the dice loss betwen masks + cost_dice = batch_dice_loss(out_mask, tgt_mask) + + # Final cost matrix + C = ( + self.cost_mask * cost_mask + + self.cost_class * cost_class + + self.cost_dice * cost_dice + ) + C = C.reshape(num_queries, -1).cpu() + + indices.append(linear_sum_assignment_with_nan(C)) + + return [ + (torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) + for i, j in indices + ] + + @torch.no_grad() + def forward(self, outputs, targets): + """Performs the matching + + Params: + outputs: This is a dict that contains at least these entries: + "pred_logits": Tensor of dim [batch_size, num_queries, num_classes] with the classification logits + "pred_masks": Tensor of dim [batch_size, num_queries, H_pred, W_pred] with the predicted masks + + targets: This is a list of targets (len(targets) = batch_size), where each target is a dict containing: + "labels": Tensor of dim [num_target_boxes] (where num_target_boxes is the number of ground-truth + objects in the target) containing the class labels + "masks": Tensor of dim [num_target_boxes, H_gt, W_gt] containing the target masks + + Returns: + A list of size batch_size, containing tuples of (index_i, index_j) where: + - index_i is the indices of the selected predictions (in order) + - index_j is the indices of the corresponding selected targets (in order) + For each batch element, it holds: + len(index_i) = len(index_j) = min(num_queries, num_target_boxes) + """ + + return self.memory_efficient_forward(outputs, targets) + + def __repr__(self, _repr_indent=4): + head = "Matcher " + self.__class__.__name__ + body = [ + "cost_class: {}".format(self.cost_class), + "cost_mask: {}".format(self.cost_mask), + "cost_dice: {}".format(self.cost_dice), + ] + lines = [head] + [" " * _repr_indent + line for line in body] + return "\n".join(lines) diff --git a/annotator/oneformer/oneformer/modeling/meta_arch/__init__.py b/annotator/oneformer/oneformer/modeling/meta_arch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/meta_arch/__init__.py @@ -0,0 +1 @@ + diff --git a/annotator/oneformer/oneformer/modeling/meta_arch/oneformer_head.py b/annotator/oneformer/oneformer/modeling/meta_arch/oneformer_head.py new file mode 100644 index 0000000000000000000000000000000000000000..f8f8eb11b95838d2b61de5fa249a318877182c01 --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/meta_arch/oneformer_head.py @@ -0,0 +1,135 @@ +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/Mask2Former/blob/main/mask2former/modeling/meta_arch/mask_former_head.py +# Modified by Jitesh Jain (https://github.com/praeclarumjj3) +# ------------------------------------------------------------------------------ + +import logging +from copy import deepcopy +from typing import Callable, Dict, List, Optional, Tuple, Union + +import fvcore.nn.weight_init as weight_init +from torch import nn +from torch.nn import functional as F + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.layers import Conv2d, ShapeSpec, get_norm +from annotator.oneformer.detectron2.modeling import SEM_SEG_HEADS_REGISTRY +from ..pixel_decoder.fpn import build_pixel_decoder +from ..transformer_decoder.oneformer_transformer_decoder import build_transformer_decoder + +@SEM_SEG_HEADS_REGISTRY.register() +class OneFormerHead(nn.Module): + + _version = 2 + + def _load_from_state_dict( + self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs + ): + version = local_metadata.get("version", None) + if version is None or version < 2: + # Do not warn if train from scratch + scratch = True + logger = logging.getLogger(__name__) + for k in list(state_dict.keys()): + newk = k + if "sem_seg_head" in k and not k.startswith(prefix + "predictor"): + newk = k.replace(prefix, prefix + "pixel_decoder.") + # logger.debug(f"{k} ==> {newk}") + if newk != k: + state_dict[newk] = state_dict[k] + del state_dict[k] + scratch = False + + if not scratch: + logger.warning( + f"Weight format of {self.__class__.__name__} have changed! " + "Please upgrade your models. Applying automatic conversion now ..." + ) + + @configurable + def __init__( + self, + input_shape: Dict[str, ShapeSpec], + *, + num_classes: int, + pixel_decoder: nn.Module, + loss_weight: float = 1.0, + ignore_value: int = -1, + # extra parameters + transformer_predictor: nn.Module, + transformer_in_feature: str, + ): + """ + NOTE: this interface is experimental. + Args: + input_shape: shapes (channels and stride) of the input features + num_classes: number of classes to predict + pixel_decoder: the pixel decoder module + loss_weight: loss weight + ignore_value: category id to be ignored during training. + transformer_predictor: the transformer decoder that makes prediction + transformer_in_feature: input feature name to the transformer_predictor + """ + super().__init__() + input_shape = sorted(input_shape.items(), key=lambda x: x[1].stride) + self.in_features = [k for k, v in input_shape] + feature_strides = [v.stride for k, v in input_shape] + feature_channels = [v.channels for k, v in input_shape] + + self.ignore_value = ignore_value + self.common_stride = 4 + self.loss_weight = loss_weight + + self.pixel_decoder = pixel_decoder + self.predictor = transformer_predictor + self.transformer_in_feature = transformer_in_feature + + self.num_classes = num_classes + + @classmethod + def from_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): + # figure out in_channels to transformer predictor + if cfg.MODEL.ONE_FORMER.TRANSFORMER_IN_FEATURE == "transformer_encoder": + transformer_predictor_in_channels = cfg.MODEL.SEM_SEG_HEAD.CONVS_DIM + elif cfg.MODEL.ONE_FORMER.TRANSFORMER_IN_FEATURE == "pixel_embedding": + transformer_predictor_in_channels = cfg.MODEL.SEM_SEG_HEAD.MASK_DIM + elif cfg.MODEL.ONE_FORMER.TRANSFORMER_IN_FEATURE == "multi_scale_pixel_decoder": + transformer_predictor_in_channels = cfg.MODEL.SEM_SEG_HEAD.CONVS_DIM + else: + transformer_predictor_in_channels = input_shape[cfg.MODEL.ONE_FORMER.TRANSFORMER_IN_FEATURE].channels + + return { + "input_shape": { + k: v for k, v in input_shape.items() if k in cfg.MODEL.SEM_SEG_HEAD.IN_FEATURES + }, + "ignore_value": cfg.MODEL.SEM_SEG_HEAD.IGNORE_VALUE, + "num_classes": cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES, + "pixel_decoder": build_pixel_decoder(cfg, input_shape), + "loss_weight": cfg.MODEL.SEM_SEG_HEAD.LOSS_WEIGHT, + "transformer_in_feature": cfg.MODEL.ONE_FORMER.TRANSFORMER_IN_FEATURE, + "transformer_predictor": build_transformer_decoder( + cfg, + transformer_predictor_in_channels, + mask_classification=True, + ), + } + + def forward(self, features, tasks, mask=None): + return self.layers(features, tasks, mask) + + def layers(self, features, tasks, mask=None): + mask_features, transformer_encoder_features, multi_scale_features, _, _ = self.pixel_decoder.forward_features(features) + + if self.transformer_in_feature == "multi_scale_pixel_decoder": + predictions = self.predictor(multi_scale_features, mask_features, tasks, mask) + else: + if self.transformer_in_feature == "transformer_encoder": + assert ( + transformer_encoder_features is not None + ), "Please use the TransformerEncoderPixelDecoder." + predictions = self.predictor(transformer_encoder_features, mask_features, mask) + elif self.transformer_in_feature == "pixel_embedding": + predictions = self.predictor(mask_features, mask_features, mask) + else: + predictions = self.predictor(features[self.transformer_in_feature], mask_features, mask) + return predictions diff --git a/annotator/oneformer/oneformer/modeling/pixel_decoder/__init__.py b/annotator/oneformer/oneformer/modeling/pixel_decoder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9020c2df23e2af280b7bb168b996ae9eaf312eb8 --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/pixel_decoder/__init__.py @@ -0,0 +1 @@ +# Copyright (c) Facebook, Inc. and its affiliates. diff --git a/annotator/oneformer/oneformer/modeling/pixel_decoder/fpn.py b/annotator/oneformer/oneformer/modeling/pixel_decoder/fpn.py new file mode 100644 index 0000000000000000000000000000000000000000..0f724775b5e237cd01f2b369950a04b38059f9bf --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/pixel_decoder/fpn.py @@ -0,0 +1,312 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import numpy as np +from typing import Callable, Dict, List, Optional, Tuple, Union + +import fvcore.nn.weight_init as weight_init +import torch +from torch import nn +from torch.nn import functional as F +from torch.nn.init import xavier_uniform_, constant_, uniform_, normal_ +from torch.cuda.amp import autocast + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.layers import Conv2d, DeformConv, ShapeSpec, get_norm +from annotator.oneformer.detectron2.modeling import SEM_SEG_HEADS_REGISTRY + +from ..transformer_decoder.position_encoding import PositionEmbeddingSine +from ..transformer_decoder.transformer import TransformerEncoder, TransformerEncoderLayer, _get_clones, _get_activation_fn + + +def build_pixel_decoder(cfg, input_shape): + """ + Build a pixel decoder from `cfg.MODEL.MASK_FORMER.PIXEL_DECODER_NAME`. + """ + name = cfg.MODEL.SEM_SEG_HEAD.PIXEL_DECODER_NAME + model = SEM_SEG_HEADS_REGISTRY.get(name)(cfg, input_shape) + forward_features = getattr(model, "forward_features", None) + if not callable(forward_features): + raise ValueError( + "Only SEM_SEG_HEADS with forward_features method can be used as pixel decoder. " + f"Please implement forward_features for {name} to only return mask features." + ) + return model + + +# This is a modified FPN decoder. +@SEM_SEG_HEADS_REGISTRY.register() +class BasePixelDecoder(nn.Module): + @configurable + def __init__( + self, + input_shape: Dict[str, ShapeSpec], + *, + conv_dim: int, + mask_dim: int, + norm: Optional[Union[str, Callable]] = None, + ): + """ + NOTE: this interface is experimental. + Args: + input_shape: shapes (channels and stride) of the input features + conv_dims: number of output channels for the intermediate conv layers. + mask_dim: number of output channels for the final conv layer. + norm (str or callable): normalization for all conv layers + """ + super().__init__() + + input_shape = sorted(input_shape.items(), key=lambda x: x[1].stride) + self.in_features = [k for k, v in input_shape] # starting from "res2" to "res5" + feature_channels = [v.channels for k, v in input_shape] + + lateral_convs = [] + output_convs = [] + + use_bias = norm == "" + for idx, in_channels in enumerate(feature_channels): + if idx == len(self.in_features) - 1: + output_norm = get_norm(norm, conv_dim) + output_conv = Conv2d( + in_channels, + conv_dim, + kernel_size=3, + stride=1, + padding=1, + bias=use_bias, + norm=output_norm, + activation=F.relu, + ) + weight_init.c2_xavier_fill(output_conv) + self.add_module("layer_{}".format(idx + 1), output_conv) + + lateral_convs.append(None) + output_convs.append(output_conv) + else: + lateral_norm = get_norm(norm, conv_dim) + output_norm = get_norm(norm, conv_dim) + + lateral_conv = Conv2d( + in_channels, conv_dim, kernel_size=1, bias=use_bias, norm=lateral_norm + ) + output_conv = Conv2d( + conv_dim, + conv_dim, + kernel_size=3, + stride=1, + padding=1, + bias=use_bias, + norm=output_norm, + activation=F.relu, + ) + weight_init.c2_xavier_fill(lateral_conv) + weight_init.c2_xavier_fill(output_conv) + self.add_module("adapter_{}".format(idx + 1), lateral_conv) + self.add_module("layer_{}".format(idx + 1), output_conv) + + lateral_convs.append(lateral_conv) + output_convs.append(output_conv) + # Place convs into top-down order (from low to high resolution) + # to make the top-down computation in forward clearer. + self.lateral_convs = lateral_convs[::-1] + self.output_convs = output_convs[::-1] + + self.mask_dim = mask_dim + self.mask_features = Conv2d( + conv_dim, + mask_dim, + kernel_size=3, + stride=1, + padding=1, + ) + weight_init.c2_xavier_fill(self.mask_features) + + self.oneformer_num_feature_levels = 3 # always use 3 scales + + @classmethod + def from_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): + ret = {} + ret["input_shape"] = { + k: v for k, v in input_shape.items() if k in cfg.MODEL.SEM_SEG_HEAD.IN_FEATURES + } + ret["conv_dim"] = cfg.MODEL.SEM_SEG_HEAD.CONVS_DIM + ret["mask_dim"] = cfg.MODEL.SEM_SEG_HEAD.MASK_DIM + ret["norm"] = cfg.MODEL.SEM_SEG_HEAD.NORM + return ret + + def forward_features(self, features): + multi_scale_features = [] + num_cur_levels = 0 + # Reverse feature maps into top-down order (from low to high resolution) + for idx, f in enumerate(self.in_features[::-1]): + x = features[f] + lateral_conv = self.lateral_convs[idx] + output_conv = self.output_convs[idx] + if lateral_conv is None: + y = output_conv(x) + else: + cur_fpn = lateral_conv(x) + # Following FPN implementation, we use nearest upsampling here + y = cur_fpn + F.interpolate(y, size=cur_fpn.shape[-2:], mode="nearest") + y = output_conv(y) + if num_cur_levels < self.oneformer_num_feature_levels: + multi_scale_features.append(y) + num_cur_levels += 1 + return self.mask_features(y), None, multi_scale_features + + def forward(self, features, targets=None): + logger = logging.getLogger(__name__) + logger.warning("Calling forward() may cause unpredicted behavior of PixelDecoder module.") + return self.forward_features(features) + + +class TransformerEncoderOnly(nn.Module): + def __init__( + self, + d_model=512, + nhead=8, + num_encoder_layers=6, + dim_feedforward=2048, + dropout=0.1, + activation="relu", + normalize_before=False, + ): + super().__init__() + + encoder_layer = TransformerEncoderLayer( + d_model, nhead, dim_feedforward, dropout, activation, normalize_before + ) + encoder_norm = nn.LayerNorm(d_model) if normalize_before else None + self.encoder = TransformerEncoder(encoder_layer, num_encoder_layers, encoder_norm) + + self._reset_parameters() + + self.d_model = d_model + self.nhead = nhead + + def _reset_parameters(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + + def forward(self, src, mask, pos_embed): + # flatten NxCxHxW to HWxNxC + bs, c, h, w = src.shape + src = src.flatten(2).permute(2, 0, 1) + pos_embed = pos_embed.flatten(2).permute(2, 0, 1) + if mask is not None: + mask = mask.flatten(1) + + memory = self.encoder(src, src_key_padding_mask=mask, pos=pos_embed) + return memory.permute(1, 2, 0).view(bs, c, h, w) + + +# This is a modified FPN decoder with extra Transformer encoder that processes the lowest-resolution feature map. +@SEM_SEG_HEADS_REGISTRY.register() +class TransformerEncoderPixelDecoder(BasePixelDecoder): + @configurable + def __init__( + self, + input_shape: Dict[str, ShapeSpec], + *, + transformer_dropout: float, + transformer_nheads: int, + transformer_dim_feedforward: int, + transformer_enc_layers: int, + transformer_pre_norm: bool, + conv_dim: int, + mask_dim: int, + norm: Optional[Union[str, Callable]] = None, + ): + """ + NOTE: this interface is experimental. + Args: + input_shape: shapes (channels and stride) of the input features + transformer_dropout: dropout probability in transformer + transformer_nheads: number of heads in transformer + transformer_dim_feedforward: dimension of feedforward network + transformer_enc_layers: number of transformer encoder layers + transformer_pre_norm: whether to use pre-layernorm or not + conv_dims: number of output channels for the intermediate conv layers. + mask_dim: number of output channels for the final conv layer. + norm (str or callable): normalization for all conv layers + """ + super().__init__(input_shape, conv_dim=conv_dim, mask_dim=mask_dim, norm=norm) + + input_shape = sorted(input_shape.items(), key=lambda x: x[1].stride) + self.in_features = [k for k, v in input_shape] # starting from "res2" to "res5" + feature_strides = [v.stride for k, v in input_shape] + feature_channels = [v.channels for k, v in input_shape] + + in_channels = feature_channels[len(self.in_features) - 1] + self.input_proj = Conv2d(in_channels, conv_dim, kernel_size=1) + weight_init.c2_xavier_fill(self.input_proj) + self.transformer = TransformerEncoderOnly( + d_model=conv_dim, + dropout=transformer_dropout, + nhead=transformer_nheads, + dim_feedforward=transformer_dim_feedforward, + num_encoder_layers=transformer_enc_layers, + normalize_before=transformer_pre_norm, + ) + N_steps = conv_dim // 2 + self.pe_layer = PositionEmbeddingSine(N_steps, normalize=True) + + # update layer + use_bias = norm == "" + output_norm = get_norm(norm, conv_dim) + output_conv = Conv2d( + conv_dim, + conv_dim, + kernel_size=3, + stride=1, + padding=1, + bias=use_bias, + norm=output_norm, + activation=F.relu, + ) + weight_init.c2_xavier_fill(output_conv) + delattr(self, "layer_{}".format(len(self.in_features))) + self.add_module("layer_{}".format(len(self.in_features)), output_conv) + self.output_convs[0] = output_conv + + @classmethod + def from_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): + ret = super().from_config(cfg, input_shape) + ret["transformer_dropout"] = cfg.MODEL.MASK_FORMER.DROPOUT + ret["transformer_nheads"] = cfg.MODEL.MASK_FORMER.NHEADS + ret["transformer_dim_feedforward"] = cfg.MODEL.MASK_FORMER.DIM_FEEDFORWARD + ret[ + "transformer_enc_layers" + ] = cfg.MODEL.SEM_SEG_HEAD.TRANSFORMER_ENC_LAYERS # a separate config + ret["transformer_pre_norm"] = cfg.MODEL.MASK_FORMER.PRE_NORM + return ret + + def forward_features(self, features): + multi_scale_features = [] + num_cur_levels = 0 + # Reverse feature maps into top-down order (from low to high resolution) + for idx, f in enumerate(self.in_features[::-1]): + x = features[f] + lateral_conv = self.lateral_convs[idx] + output_conv = self.output_convs[idx] + if lateral_conv is None: + transformer = self.input_proj(x) + pos = self.pe_layer(x) + transformer = self.transformer(transformer, None, pos) + y = output_conv(transformer) + # save intermediate feature as input to Transformer decoder + transformer_encoder_features = transformer + else: + cur_fpn = lateral_conv(x) + # Following FPN implementation, we use nearest upsampling here + y = cur_fpn + F.interpolate(y, size=cur_fpn.shape[-2:], mode="nearest") + y = output_conv(y) + if num_cur_levels < self.oneformer_num_feature_levels: + multi_scale_features.append(y) + num_cur_levels += 1 + return self.mask_features(y), transformer_encoder_features, multi_scale_features + + def forward(self, features, targets=None): + logger = logging.getLogger(__name__) + logger.warning("Calling forward() may cause unpredicted behavior of PixelDecoder module.") + return self.forward_features(features) diff --git a/annotator/oneformer/oneformer/modeling/pixel_decoder/msdeformattn.py b/annotator/oneformer/oneformer/modeling/pixel_decoder/msdeformattn.py new file mode 100644 index 0000000000000000000000000000000000000000..007051d713fd89a622154f5e0edc9902627cca14 --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/pixel_decoder/msdeformattn.py @@ -0,0 +1,358 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +import logging +import numpy as np +from typing import Callable, Dict, List, Optional, Tuple, Union + +import fvcore.nn.weight_init as weight_init +import torch +from torch import nn +from torch.nn import functional as F +from torch.nn.init import xavier_uniform_, constant_, uniform_, normal_ +from torch.cuda.amp import autocast + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.layers import Conv2d, ShapeSpec, get_norm +from annotator.oneformer.detectron2.modeling import SEM_SEG_HEADS_REGISTRY + +from ..transformer_decoder.position_encoding import PositionEmbeddingSine +from ..transformer_decoder.transformer import _get_clones, _get_activation_fn +from .ops.modules import MSDeformAttn + + +# MSDeformAttn Transformer encoder in deformable detr +class MSDeformAttnTransformerEncoderOnly(nn.Module): + def __init__(self, d_model=256, nhead=8, + num_encoder_layers=6, dim_feedforward=1024, dropout=0.1, + activation="relu", + num_feature_levels=4, enc_n_points=4, + ): + super().__init__() + + self.d_model = d_model + self.nhead = nhead + + encoder_layer = MSDeformAttnTransformerEncoderLayer(d_model, dim_feedforward, + dropout, activation, + num_feature_levels, nhead, enc_n_points) + self.encoder = MSDeformAttnTransformerEncoder(encoder_layer, num_encoder_layers) + + self.level_embed = nn.Parameter(torch.Tensor(num_feature_levels, d_model)) + + self._reset_parameters() + + def _reset_parameters(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + for m in self.modules(): + if isinstance(m, MSDeformAttn): + m._reset_parameters() + normal_(self.level_embed) + + def get_valid_ratio(self, mask): + _, H, W = mask.shape + valid_H = torch.sum(~mask[:, :, 0], 1) + valid_W = torch.sum(~mask[:, 0, :], 1) + valid_ratio_h = valid_H.float() / H + valid_ratio_w = valid_W.float() / W + valid_ratio = torch.stack([valid_ratio_w, valid_ratio_h], -1) + return valid_ratio + + def forward(self, srcs, pos_embeds): + masks = [torch.zeros((x.size(0), x.size(2), x.size(3)), device=x.device, dtype=torch.bool) for x in srcs] + # prepare input for encoder + src_flatten = [] + mask_flatten = [] + lvl_pos_embed_flatten = [] + spatial_shapes = [] + for lvl, (src, mask, pos_embed) in enumerate(zip(srcs, masks, pos_embeds)): + bs, c, h, w = src.shape + spatial_shape = (h, w) + spatial_shapes.append(spatial_shape) + src = src.flatten(2).transpose(1, 2) + mask = mask.flatten(1) + pos_embed = pos_embed.flatten(2).transpose(1, 2) + lvl_pos_embed = pos_embed + self.level_embed[lvl].view(1, 1, -1) + lvl_pos_embed_flatten.append(lvl_pos_embed) + src_flatten.append(src) + mask_flatten.append(mask) + src_flatten = torch.cat(src_flatten, 1) + mask_flatten = torch.cat(mask_flatten, 1) + lvl_pos_embed_flatten = torch.cat(lvl_pos_embed_flatten, 1) + spatial_shapes = torch.as_tensor(spatial_shapes, dtype=torch.long, device=src_flatten.device) + level_start_index = torch.cat((spatial_shapes.new_zeros((1, )), spatial_shapes.prod(1).cumsum(0)[:-1])) + valid_ratios = torch.stack([self.get_valid_ratio(m) for m in masks], 1) + + # encoder + memory = self.encoder(src_flatten, spatial_shapes, level_start_index, valid_ratios, lvl_pos_embed_flatten, mask_flatten) + + return memory, spatial_shapes, level_start_index, valid_ratios + + +class MSDeformAttnTransformerEncoderLayer(nn.Module): + def __init__(self, + d_model=256, d_ffn=1024, + dropout=0.1, activation="relu", + n_levels=4, n_heads=8, n_points=4): + super().__init__() + + # self attention + self.self_attn = MSDeformAttn(d_model, n_levels, n_heads, n_points) + self.dropout1 = nn.Dropout(dropout) + self.norm1 = nn.LayerNorm(d_model) + + # ffn + self.linear1 = nn.Linear(d_model, d_ffn) + self.activation = _get_activation_fn(activation) + self.dropout2 = nn.Dropout(dropout) + self.linear2 = nn.Linear(d_ffn, d_model) + self.dropout3 = nn.Dropout(dropout) + self.norm2 = nn.LayerNorm(d_model) + + @staticmethod + def with_pos_embed(tensor, pos): + return tensor if pos is None else tensor + pos + + def forward_ffn(self, src): + src2 = self.linear2(self.dropout2(self.activation(self.linear1(src)))) + src = src + self.dropout3(src2) + src = self.norm2(src) + return src + + def forward(self, src, pos, reference_points, spatial_shapes, level_start_index, padding_mask=None): + # self attention + src2 = self.self_attn(self.with_pos_embed(src, pos), reference_points, src, spatial_shapes, level_start_index, padding_mask) + src = src + self.dropout1(src2) + src = self.norm1(src) + + # ffn + src = self.forward_ffn(src) + + return src + + +class MSDeformAttnTransformerEncoder(nn.Module): + def __init__(self, encoder_layer, num_layers): + super().__init__() + self.layers = _get_clones(encoder_layer, num_layers) + self.num_layers = num_layers + + @staticmethod + def get_reference_points(spatial_shapes, valid_ratios, device): + reference_points_list = [] + for lvl, (H_, W_) in enumerate(spatial_shapes): + + ref_y, ref_x = torch.meshgrid(torch.linspace(0.5, H_ - 0.5, H_, dtype=torch.float32, device=device), + torch.linspace(0.5, W_ - 0.5, W_, dtype=torch.float32, device=device)) + ref_y = ref_y.reshape(-1)[None] / (valid_ratios[:, None, lvl, 1] * H_) + ref_x = ref_x.reshape(-1)[None] / (valid_ratios[:, None, lvl, 0] * W_) + ref = torch.stack((ref_x, ref_y), -1) + reference_points_list.append(ref) + reference_points = torch.cat(reference_points_list, 1) + reference_points = reference_points[:, :, None] * valid_ratios[:, None] + return reference_points + + def forward(self, src, spatial_shapes, level_start_index, valid_ratios, pos=None, padding_mask=None): + output = src + reference_points = self.get_reference_points(spatial_shapes, valid_ratios, device=src.device) + for _, layer in enumerate(self.layers): + output = layer(output, pos, reference_points, spatial_shapes, level_start_index, padding_mask) + + return output + + +@SEM_SEG_HEADS_REGISTRY.register() +class MSDeformAttnPixelDecoder(nn.Module): + @configurable + def __init__( + self, + input_shape: Dict[str, ShapeSpec], + *, + transformer_dropout: float, + transformer_nheads: int, + transformer_dim_feedforward: int, + transformer_enc_layers: int, + conv_dim: int, + mask_dim: int, + norm: Optional[Union[str, Callable]] = None, + # deformable transformer encoder args + transformer_in_features: List[str], + common_stride: int, + ): + """ + NOTE: this interface is experimental. + Args: + input_shape: shapes (channels and stride) of the input features + transformer_dropout: dropout probability in transformer + transformer_nheads: number of heads in transformer + transformer_dim_feedforward: dimension of feedforward network + transformer_enc_layers: number of transformer encoder layers + conv_dims: number of output channels for the intermediate conv layers. + mask_dim: number of output channels for the final conv layer. + norm (str or callable): normalization for all conv layers + """ + super().__init__() + transformer_input_shape = { + k: v for k, v in input_shape.items() if k in transformer_in_features + } + + # this is the input shape of pixel decoder + input_shape = sorted(input_shape.items(), key=lambda x: x[1].stride) + self.in_features = [k for k, v in input_shape] # starting from "res2" to "res5" + self.feature_strides = [v.stride for k, v in input_shape] + self.feature_channels = [v.channels for k, v in input_shape] + + # this is the input shape of transformer encoder (could use less features than pixel decoder + transformer_input_shape = sorted(transformer_input_shape.items(), key=lambda x: x[1].stride) + self.transformer_in_features = [k for k, v in transformer_input_shape] # starting from "res2" to "res5" + transformer_in_channels = [v.channels for k, v in transformer_input_shape] + self.transformer_feature_strides = [v.stride for k, v in transformer_input_shape] # to decide extra FPN layers + + self.transformer_num_feature_levels = len(self.transformer_in_features) + if self.transformer_num_feature_levels > 1: + input_proj_list = [] + # from low resolution to high resolution (res5 -> res2) + for in_channels in transformer_in_channels[::-1]: + input_proj_list.append(nn.Sequential( + nn.Conv2d(in_channels, conv_dim, kernel_size=1), + nn.GroupNorm(32, conv_dim), + )) + self.input_proj = nn.ModuleList(input_proj_list) + else: + self.input_proj = nn.ModuleList([ + nn.Sequential( + nn.Conv2d(transformer_in_channels[-1], conv_dim, kernel_size=1), + nn.GroupNorm(32, conv_dim), + )]) + + for proj in self.input_proj: + nn.init.xavier_uniform_(proj[0].weight, gain=1) + nn.init.constant_(proj[0].bias, 0) + + self.transformer = MSDeformAttnTransformerEncoderOnly( + d_model=conv_dim, + dropout=transformer_dropout, + nhead=transformer_nheads, + dim_feedforward=transformer_dim_feedforward, + num_encoder_layers=transformer_enc_layers, + num_feature_levels=self.transformer_num_feature_levels, + ) + N_steps = conv_dim // 2 + self.pe_layer = PositionEmbeddingSine(N_steps, normalize=True) + + self.mask_dim = mask_dim + # use 1x1 conv instead + self.mask_features = Conv2d( + conv_dim, + mask_dim, + kernel_size=1, + stride=1, + padding=0, + ) + weight_init.c2_xavier_fill(self.mask_features) + + self.oneformer_num_feature_levels = 3 # always use 3 scales + self.common_stride = common_stride + + # extra fpn levels + stride = min(self.transformer_feature_strides) + self.num_fpn_levels = int(np.log2(stride) - np.log2(self.common_stride)) + + lateral_convs = [] + output_convs = [] + + use_bias = norm == "" + for idx, in_channels in enumerate(self.feature_channels[:self.num_fpn_levels]): + lateral_norm = get_norm(norm, conv_dim) + output_norm = get_norm(norm, conv_dim) + + lateral_conv = Conv2d( + in_channels, conv_dim, kernel_size=1, bias=use_bias, norm=lateral_norm + ) + output_conv = Conv2d( + conv_dim, + conv_dim, + kernel_size=3, + stride=1, + padding=1, + bias=use_bias, + norm=output_norm, + activation=F.relu, + ) + weight_init.c2_xavier_fill(lateral_conv) + weight_init.c2_xavier_fill(output_conv) + self.add_module("adapter_{}".format(idx + 1), lateral_conv) + self.add_module("layer_{}".format(idx + 1), output_conv) + + lateral_convs.append(lateral_conv) + output_convs.append(output_conv) + # Place convs into top-down order (from low to high resolution) + # to make the top-down computation in forward clearer. + self.lateral_convs = lateral_convs[::-1] + self.output_convs = output_convs[::-1] + + @classmethod + def from_config(cls, cfg, input_shape: Dict[str, ShapeSpec]): + ret = {} + ret["input_shape"] = { + k: v for k, v in input_shape.items() if k in cfg.MODEL.SEM_SEG_HEAD.IN_FEATURES + } + ret["conv_dim"] = cfg.MODEL.SEM_SEG_HEAD.CONVS_DIM + ret["mask_dim"] = cfg.MODEL.SEM_SEG_HEAD.MASK_DIM + ret["norm"] = cfg.MODEL.SEM_SEG_HEAD.NORM + ret["transformer_dropout"] = cfg.MODEL.ONE_FORMER.DROPOUT + ret["transformer_nheads"] = cfg.MODEL.ONE_FORMER.NHEADS + # ret["transformer_dim_feedforward"] = cfg.MODEL.ONE_FORMER.DIM_FEEDFORWARD + ret["transformer_dim_feedforward"] = 1024 # use 1024 for deformable transformer encoder + ret[ + "transformer_enc_layers" + ] = cfg.MODEL.SEM_SEG_HEAD.TRANSFORMER_ENC_LAYERS # a separate config + ret["transformer_in_features"] = cfg.MODEL.SEM_SEG_HEAD.DEFORMABLE_TRANSFORMER_ENCODER_IN_FEATURES + ret["common_stride"] = cfg.MODEL.SEM_SEG_HEAD.COMMON_STRIDE + return ret + + @autocast(enabled=False) + def forward_features(self, features): + srcs = [] + pos = [] + # Reverse feature maps into top-down order (from low to high resolution) + for idx, f in enumerate(self.transformer_in_features[::-1]): + x = features[f].float() # deformable detr does not support half precision + srcs.append(self.input_proj[idx](x)) + pos.append(self.pe_layer(x)) + + y, spatial_shapes, level_start_index, valid_ratios = self.transformer(srcs, pos) + bs = y.shape[0] + + split_size_or_sections = [None] * self.transformer_num_feature_levels + for i in range(self.transformer_num_feature_levels): + if i < self.transformer_num_feature_levels - 1: + split_size_or_sections[i] = level_start_index[i + 1] - level_start_index[i] + else: + split_size_or_sections[i] = y.shape[1] - level_start_index[i] + y = torch.split(y, split_size_or_sections, dim=1) + + out = [] + multi_scale_features = [] + num_cur_levels = 0 + for i, z in enumerate(y): + out.append(z.transpose(1, 2).view(bs, -1, spatial_shapes[i][0], spatial_shapes[i][1])) + + # append `out` with extra FPN levels + # Reverse feature maps into top-down order (from low to high resolution) + for idx, f in enumerate(self.in_features[:self.num_fpn_levels][::-1]): + x = features[f].float() + lateral_conv = self.lateral_convs[idx] + output_conv = self.output_convs[idx] + cur_fpn = lateral_conv(x) + # Following FPN implementation, we use nearest upsampling here + y = cur_fpn + F.interpolate(out[-1], size=cur_fpn.shape[-2:], mode="bilinear", align_corners=False) + y = output_conv(y) + out.append(y) + + for o in out: + if num_cur_levels < self.oneformer_num_feature_levels: + multi_scale_features.append(o) + num_cur_levels += 1 + + return self.mask_features(out[-1]), out[0], multi_scale_features, spatial_shapes, level_start_index diff --git a/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/functions/__init__.py b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/functions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2b06b5ac538b63bdb9a6c82e4635b95bb5491d5b --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/functions/__init__.py @@ -0,0 +1,13 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +# Copyright (c) Facebook, Inc. and its affiliates. +# Modified by Bowen Cheng from https://github.com/fundamentalvision/Deformable-DETR + +from .ms_deform_attn_func import MSDeformAttnFunction + diff --git a/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/functions/ms_deform_attn_func.py b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/functions/ms_deform_attn_func.py new file mode 100644 index 0000000000000000000000000000000000000000..e074eb69819151add821a8ff9ed215ed9b874070 --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/functions/ms_deform_attn_func.py @@ -0,0 +1,77 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +# Copyright (c) Facebook, Inc. and its affiliates. +# Modified by Bowen Cheng from https://github.com/fundamentalvision/Deformable-DETR + + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import torch +import torch.nn.functional as F +from torch.autograd import Function +from torch.autograd.function import once_differentiable + +# if torch.cuda.is_available(): +# try: +# import MultiScaleDeformableAttention as MSDA +# except ModuleNotFoundError as e: +# info_string = ( +# "\n\nPlease compile MultiScaleDeformableAttention CUDA op with the following commands:\n" +# "\t`cd oneformer/modeling/pixel_decoder/ops`\n" +# "\t`sh make.sh`\n" +# ) +# raise ModuleNotFoundError(info_string) +# else: +# MultiScaleDeformableAttention = None + + + +class MSDeformAttnFunction(Function): + @staticmethod + def forward(ctx, value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, im2col_step): + # ctx.im2col_step = im2col_step + output = ms_deform_attn_core_pytorch( + value, value_spatial_shapes, sampling_locations, attention_weights) + # ctx.save_for_backward(value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights) + return output + + # @staticmethod + # @once_differentiable + # def backward(ctx, grad_output): + # value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights = ctx.saved_tensors + # grad_value, grad_sampling_loc, grad_attn_weight = \ + # MSDA.ms_deform_attn_backward( + # value, value_spatial_shapes, value_level_start_index, sampling_locations, attention_weights, grad_output, ctx.im2col_step) + # + # return grad_value, None, None, grad_sampling_loc, grad_attn_weight, None + + +def ms_deform_attn_core_pytorch(value, value_spatial_shapes, sampling_locations, attention_weights): + # for debug and test only, + # need to use cuda version instead + N_, S_, M_, D_ = value.shape + _, Lq_, M_, L_, P_, _ = sampling_locations.shape + value_list = value.split([H_ * W_ for H_, W_ in value_spatial_shapes], dim=1) + sampling_grids = 2 * sampling_locations - 1 + sampling_value_list = [] + for lid_, (H_, W_) in enumerate(value_spatial_shapes): + # N_, H_*W_, M_, D_ -> N_, H_*W_, M_*D_ -> N_, M_*D_, H_*W_ -> N_*M_, D_, H_, W_ + value_l_ = value_list[lid_].flatten(2).transpose(1, 2).reshape(N_*M_, D_, H_, W_) + # N_, Lq_, M_, P_, 2 -> N_, M_, Lq_, P_, 2 -> N_*M_, Lq_, P_, 2 + sampling_grid_l_ = sampling_grids[:, :, :, lid_].transpose(1, 2).flatten(0, 1) + # N_*M_, D_, Lq_, P_ + sampling_value_l_ = F.grid_sample(value_l_, sampling_grid_l_, + mode='bilinear', padding_mode='zeros', align_corners=False) + sampling_value_list.append(sampling_value_l_) + # (N_, Lq_, M_, L_, P_) -> (N_, M_, Lq_, L_, P_) -> (N_, M_, 1, Lq_, L_*P_) + attention_weights = attention_weights.transpose(1, 2).reshape(N_*M_, 1, Lq_, L_*P_) + output = (torch.stack(sampling_value_list, dim=-2).flatten(-2) * attention_weights).sum(-1).view(N_, M_*D_, Lq_) + return output.transpose(1, 2).contiguous() diff --git a/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/make.sh b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/make.sh new file mode 100644 index 0000000000000000000000000000000000000000..ca5c0b469da786c847ba04d437bb31ee0fc938da --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/make.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +# Copyright (c) Facebook, Inc. and its affiliates. +# Modified by Bowen Cheng from https://github.com/fundamentalvision/Deformable-DETR + +FORCE_CUDA=1 python setup.py build install diff --git a/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/modules/__init__.py b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6fdbf03359958f3d67ab00f879bf6b61a6c8f06a --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/modules/__init__.py @@ -0,0 +1,12 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +# Copyright (c) Facebook, Inc. and its affiliates. +# Modified by Bowen Cheng from https://github.com/fundamentalvision/Deformable-DETR + +from .ms_deform_attn import MSDeformAttn diff --git a/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/modules/ms_deform_attn.py b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/modules/ms_deform_attn.py new file mode 100644 index 0000000000000000000000000000000000000000..5bc471d2da550c839a3446a6041e40d338425129 --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/modules/ms_deform_attn.py @@ -0,0 +1,120 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +# Copyright (c) Facebook, Inc. and its affiliates. +# Modified by Bowen Cheng from https://github.com/fundamentalvision/Deformable-DETR + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import warnings +import math + +import torch +from torch import nn +import torch.nn.functional as F +from torch.nn.init import xavier_uniform_, constant_ + +MSDeformAttnFunction = None +from ..functions.ms_deform_attn_func import ms_deform_attn_core_pytorch + + +def _is_power_of_2(n): + if (not isinstance(n, int)) or (n < 0): + raise ValueError("invalid input for _is_power_of_2: {} (type: {})".format(n, type(n))) + return (n & (n-1) == 0) and n != 0 + + +class MSDeformAttn(nn.Module): + def __init__(self, d_model=256, n_levels=4, n_heads=8, n_points=4): + """ + Multi-Scale Deformable Attention Module + :param d_model hidden dimension + :param n_levels number of feature levels + :param n_heads number of attention heads + :param n_points number of sampling points per attention head per feature level + """ + super().__init__() + if d_model % n_heads != 0: + raise ValueError('d_model must be divisible by n_heads, but got {} and {}'.format(d_model, n_heads)) + _d_per_head = d_model // n_heads + # you'd better set _d_per_head to a power of 2 which is more efficient in our CUDA implementation + if not _is_power_of_2(_d_per_head): + warnings.warn("You'd better set d_model in MSDeformAttn to make the dimension of each attention head a power of 2 " + "which is more efficient in our CUDA implementation.") + + self.im2col_step = 128 + + self.d_model = d_model + self.n_levels = n_levels + self.n_heads = n_heads + self.n_points = n_points + + self.sampling_offsets = nn.Linear(d_model, n_heads * n_levels * n_points * 2) + self.attention_weights = nn.Linear(d_model, n_heads * n_levels * n_points) + self.value_proj = nn.Linear(d_model, d_model) + self.output_proj = nn.Linear(d_model, d_model) + + self._reset_parameters() + + def _reset_parameters(self): + constant_(self.sampling_offsets.weight.data, 0.) + thetas = torch.arange(self.n_heads, dtype=torch.float32) * (2.0 * math.pi / self.n_heads) + grid_init = torch.stack([thetas.cos(), thetas.sin()], -1) + grid_init = (grid_init / grid_init.abs().max(-1, keepdim=True)[0]).view(self.n_heads, 1, 1, 2).repeat(1, self.n_levels, self.n_points, 1) + for i in range(self.n_points): + grid_init[:, :, i, :] *= i + 1 + with torch.no_grad(): + self.sampling_offsets.bias = nn.Parameter(grid_init.view(-1)) + constant_(self.attention_weights.weight.data, 0.) + constant_(self.attention_weights.bias.data, 0.) + xavier_uniform_(self.value_proj.weight.data) + constant_(self.value_proj.bias.data, 0.) + xavier_uniform_(self.output_proj.weight.data) + constant_(self.output_proj.bias.data, 0.) + + def forward(self, query, reference_points, input_flatten, input_spatial_shapes, input_level_start_index, input_padding_mask=None): + """ + :param query (N, Length_{query}, C) + :param reference_points (N, Length_{query}, n_levels, 2), range in [0, 1], top-left (0,0), bottom-right (1, 1), including padding area + or (N, Length_{query}, n_levels, 4), add additional (w, h) to form reference boxes + :param input_flatten (N, \sum_{l=0}^{L-1} H_l \cdot W_l, C) + :param input_spatial_shapes (n_levels, 2), [(H_0, W_0), (H_1, W_1), ..., (H_{L-1}, W_{L-1})] + :param input_level_start_index (n_levels, ), [0, H_0*W_0, H_0*W_0+H_1*W_1, H_0*W_0+H_1*W_1+H_2*W_2, ..., H_0*W_0+H_1*W_1+...+H_{L-1}*W_{L-1}] + :param input_padding_mask (N, \sum_{l=0}^{L-1} H_l \cdot W_l), True for padding elements, False for non-padding elements + :return output (N, Length_{query}, C) + """ + N, Len_q, _ = query.shape + N, Len_in, _ = input_flatten.shape + assert (input_spatial_shapes[:, 0] * input_spatial_shapes[:, 1]).sum() == Len_in + + value = self.value_proj(input_flatten) + if input_padding_mask is not None: + value = value.masked_fill(input_padding_mask[..., None], float(0)) + value = value.view(N, Len_in, self.n_heads, self.d_model // self.n_heads) + sampling_offsets = self.sampling_offsets(query).view(N, Len_q, self.n_heads, self.n_levels, self.n_points, 2) + attention_weights = self.attention_weights(query).view(N, Len_q, self.n_heads, self.n_levels * self.n_points) + attention_weights = F.softmax(attention_weights, -1).view(N, Len_q, self.n_heads, self.n_levels, self.n_points) + # N, Len_q, n_heads, n_levels, n_points, 2 + if reference_points.shape[-1] == 2: + offset_normalizer = torch.stack([input_spatial_shapes[..., 1], input_spatial_shapes[..., 0]], -1) + sampling_locations = reference_points[:, :, None, :, None, :] \ + + sampling_offsets / offset_normalizer[None, None, None, :, None, :] + elif reference_points.shape[-1] == 4: + sampling_locations = reference_points[:, :, None, :, None, :2] \ + + sampling_offsets / self.n_points * reference_points[:, :, None, :, None, 2:] * 0.5 + else: + raise ValueError( + 'Last dim of reference_points must be 2 or 4, but get {} instead.'.format(reference_points.shape[-1])) + # try: + output = ms_deform_attn_core_pytorch(value, input_spatial_shapes, sampling_locations, attention_weights) + # # For FLOPs calculation only + # output = ms_deform_attn_core_pytorch(value, input_spatial_shapes, sampling_locations, attention_weights) + output = self.output_proj(output) + return output \ No newline at end of file diff --git a/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/setup.py b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..3b57ad313ac8f9b6586892142da8ba943e516cec --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/setup.py @@ -0,0 +1,78 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +# Copyright (c) Facebook, Inc. and its affiliates. +# Modified by Bowen Cheng from https://github.com/fundamentalvision/Deformable-DETR + +import os +import glob + +import torch + +from torch.utils.cpp_extension import CUDA_HOME +from torch.utils.cpp_extension import CppExtension +from torch.utils.cpp_extension import CUDAExtension + +from setuptools import find_packages +from setuptools import setup + +requirements = ["torch", "torchvision"] + +def get_extensions(): + this_dir = os.path.dirname(os.path.abspath(__file__)) + extensions_dir = os.path.join(this_dir, "src") + + main_file = glob.glob(os.path.join(extensions_dir, "*.cpp")) + source_cpu = glob.glob(os.path.join(extensions_dir, "cpu", "*.cpp")) + source_cuda = glob.glob(os.path.join(extensions_dir, "cuda", "*.cu")) + + sources = main_file + source_cpu + extension = CppExtension + extra_compile_args = {"cxx": []} + define_macros = [] + + # Force cuda since torch ask for a device, not if cuda is in fact available. + if (os.environ.get('FORCE_CUDA') or torch.cuda.is_available()) and CUDA_HOME is not None: + extension = CUDAExtension + sources += source_cuda + define_macros += [("WITH_CUDA", None)] + extra_compile_args["nvcc"] = [ + "-DCUDA_HAS_FP16=1", + "-D__CUDA_NO_HALF_OPERATORS__", + "-D__CUDA_NO_HALF_CONVERSIONS__", + "-D__CUDA_NO_HALF2_OPERATORS__", + ] + else: + if CUDA_HOME is None: + raise NotImplementedError('CUDA_HOME is None. Please set environment variable CUDA_HOME.') + else: + raise NotImplementedError('No CUDA runtime is found. Please set FORCE_CUDA=1 or test it by running torch.cuda.is_available().') + + sources = [os.path.join(extensions_dir, s) for s in sources] + include_dirs = [extensions_dir] + ext_modules = [ + extension( + "MultiScaleDeformableAttention", + sources, + include_dirs=include_dirs, + define_macros=define_macros, + extra_compile_args=extra_compile_args, + ) + ] + return ext_modules + +setup( + name="MultiScaleDeformableAttention", + version="1.0", + author="Weijie Su", + url="https://github.com/fundamentalvision/Deformable-DETR", + description="PyTorch Wrapper for CUDA Functions of Multi-Scale Deformable Attention", + packages=find_packages(exclude=("configs", "tests",)), + ext_modules=get_extensions(), + cmdclass={"build_ext": torch.utils.cpp_extension.BuildExtension}, +) diff --git a/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/cpu/ms_deform_attn_cpu.cpp b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/cpu/ms_deform_attn_cpu.cpp new file mode 100644 index 0000000000000000000000000000000000000000..48757e2b0156b2c1513b615d2a17e5aee5172ae7 --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/cpu/ms_deform_attn_cpu.cpp @@ -0,0 +1,46 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +/*! +* Copyright (c) Facebook, Inc. and its affiliates. +* Modified by Bowen Cheng from https://github.com/fundamentalvision/Deformable-DETR +*/ + +#include + +#include +#include + + +at::Tensor +ms_deform_attn_cpu_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + +std::vector +ms_deform_attn_cpu_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step) +{ + AT_ERROR("Not implement on cpu"); +} + diff --git a/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/cpu/ms_deform_attn_cpu.h b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/cpu/ms_deform_attn_cpu.h new file mode 100644 index 0000000000000000000000000000000000000000..51bb27e9ee828f967e8aa854c2d55574040c6d7e --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/cpu/ms_deform_attn_cpu.h @@ -0,0 +1,38 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +/*! +* Copyright (c) Facebook, Inc. and its affiliates. +* Modified by Bowen Cheng from https://github.com/fundamentalvision/Deformable-DETR +*/ + +#pragma once +#include + +at::Tensor +ms_deform_attn_cpu_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step); + +std::vector +ms_deform_attn_cpu_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step); + + diff --git a/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/cuda/ms_deform_attn_cuda.cu b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/cuda/ms_deform_attn_cuda.cu new file mode 100644 index 0000000000000000000000000000000000000000..0c465dab3d636dfd6a44523c63f148b6e15084d9 --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/cuda/ms_deform_attn_cuda.cu @@ -0,0 +1,158 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +/*! +* Copyright (c) Facebook, Inc. and its affiliates. +* Modified by Bowen Cheng from https://github.com/fundamentalvision/Deformable-DETR +*/ + +#include +#include "cuda/ms_deform_im2col_cuda.cuh" + +#include +#include +#include +#include + + +at::Tensor ms_deform_attn_cuda_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step) +{ + AT_ASSERTM(value.is_contiguous(), "value tensor has to be contiguous"); + AT_ASSERTM(spatial_shapes.is_contiguous(), "spatial_shapes tensor has to be contiguous"); + AT_ASSERTM(level_start_index.is_contiguous(), "level_start_index tensor has to be contiguous"); + AT_ASSERTM(sampling_loc.is_contiguous(), "sampling_loc tensor has to be contiguous"); + AT_ASSERTM(attn_weight.is_contiguous(), "attn_weight tensor has to be contiguous"); + + AT_ASSERTM(value.type().is_cuda(), "value must be a CUDA tensor"); + AT_ASSERTM(spatial_shapes.type().is_cuda(), "spatial_shapes must be a CUDA tensor"); + AT_ASSERTM(level_start_index.type().is_cuda(), "level_start_index must be a CUDA tensor"); + AT_ASSERTM(sampling_loc.type().is_cuda(), "sampling_loc must be a CUDA tensor"); + AT_ASSERTM(attn_weight.type().is_cuda(), "attn_weight must be a CUDA tensor"); + + const int batch = value.size(0); + const int spatial_size = value.size(1); + const int num_heads = value.size(2); + const int channels = value.size(3); + + const int num_levels = spatial_shapes.size(0); + + const int num_query = sampling_loc.size(1); + const int num_point = sampling_loc.size(4); + + const int im2col_step_ = std::min(batch, im2col_step); + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + + auto output = at::zeros({batch, num_query, num_heads, channels}, value.options()); + + const int batch_n = im2col_step_; + auto output_n = output.view({batch/im2col_step_, batch_n, num_query, num_heads, channels}); + auto per_value_size = spatial_size * num_heads * channels; + auto per_sample_loc_size = num_query * num_heads * num_levels * num_point * 2; + auto per_attn_weight_size = num_query * num_heads * num_levels * num_point; + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto columns = output_n.select(0, n); + AT_DISPATCH_FLOATING_TYPES(value.type(), "ms_deform_attn_forward_cuda", ([&] { + ms_deformable_im2col_cuda(at::cuda::getCurrentCUDAStream(), + value.data() + n * im2col_step_ * per_value_size, + spatial_shapes.data(), + level_start_index.data(), + sampling_loc.data() + n * im2col_step_ * per_sample_loc_size, + attn_weight.data() + n * im2col_step_ * per_attn_weight_size, + batch_n, spatial_size, num_heads, channels, num_levels, num_query, num_point, + columns.data()); + + })); + } + + output = output.view({batch, num_query, num_heads*channels}); + + return output; +} + + +std::vector ms_deform_attn_cuda_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step) +{ + + AT_ASSERTM(value.is_contiguous(), "value tensor has to be contiguous"); + AT_ASSERTM(spatial_shapes.is_contiguous(), "spatial_shapes tensor has to be contiguous"); + AT_ASSERTM(level_start_index.is_contiguous(), "level_start_index tensor has to be contiguous"); + AT_ASSERTM(sampling_loc.is_contiguous(), "sampling_loc tensor has to be contiguous"); + AT_ASSERTM(attn_weight.is_contiguous(), "attn_weight tensor has to be contiguous"); + AT_ASSERTM(grad_output.is_contiguous(), "grad_output tensor has to be contiguous"); + + AT_ASSERTM(value.type().is_cuda(), "value must be a CUDA tensor"); + AT_ASSERTM(spatial_shapes.type().is_cuda(), "spatial_shapes must be a CUDA tensor"); + AT_ASSERTM(level_start_index.type().is_cuda(), "level_start_index must be a CUDA tensor"); + AT_ASSERTM(sampling_loc.type().is_cuda(), "sampling_loc must be a CUDA tensor"); + AT_ASSERTM(attn_weight.type().is_cuda(), "attn_weight must be a CUDA tensor"); + AT_ASSERTM(grad_output.type().is_cuda(), "grad_output must be a CUDA tensor"); + + const int batch = value.size(0); + const int spatial_size = value.size(1); + const int num_heads = value.size(2); + const int channels = value.size(3); + + const int num_levels = spatial_shapes.size(0); + + const int num_query = sampling_loc.size(1); + const int num_point = sampling_loc.size(4); + + const int im2col_step_ = std::min(batch, im2col_step); + + AT_ASSERTM(batch % im2col_step_ == 0, "batch(%d) must divide im2col_step(%d)", batch, im2col_step_); + + auto grad_value = at::zeros_like(value); + auto grad_sampling_loc = at::zeros_like(sampling_loc); + auto grad_attn_weight = at::zeros_like(attn_weight); + + const int batch_n = im2col_step_; + auto per_value_size = spatial_size * num_heads * channels; + auto per_sample_loc_size = num_query * num_heads * num_levels * num_point * 2; + auto per_attn_weight_size = num_query * num_heads * num_levels * num_point; + auto grad_output_n = grad_output.view({batch/im2col_step_, batch_n, num_query, num_heads, channels}); + + for (int n = 0; n < batch/im2col_step_; ++n) + { + auto grad_output_g = grad_output_n.select(0, n); + AT_DISPATCH_FLOATING_TYPES(value.type(), "ms_deform_attn_backward_cuda", ([&] { + ms_deformable_col2im_cuda(at::cuda::getCurrentCUDAStream(), + grad_output_g.data(), + value.data() + n * im2col_step_ * per_value_size, + spatial_shapes.data(), + level_start_index.data(), + sampling_loc.data() + n * im2col_step_ * per_sample_loc_size, + attn_weight.data() + n * im2col_step_ * per_attn_weight_size, + batch_n, spatial_size, num_heads, channels, num_levels, num_query, num_point, + grad_value.data() + n * im2col_step_ * per_value_size, + grad_sampling_loc.data() + n * im2col_step_ * per_sample_loc_size, + grad_attn_weight.data() + n * im2col_step_ * per_attn_weight_size); + + })); + } + + return { + grad_value, grad_sampling_loc, grad_attn_weight + }; +} \ No newline at end of file diff --git a/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/cuda/ms_deform_attn_cuda.h b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/cuda/ms_deform_attn_cuda.h new file mode 100644 index 0000000000000000000000000000000000000000..4f0658e8668a11f0e7d71deff9adac71884f2e87 --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/cuda/ms_deform_attn_cuda.h @@ -0,0 +1,35 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +/*! +* Copyright (c) Facebook, Inc. and its affiliates. +* Modified by Bowen Cheng from https://github.com/fundamentalvision/Deformable-DETR +*/ + +#pragma once +#include + +at::Tensor ms_deform_attn_cuda_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step); + +std::vector ms_deform_attn_cuda_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step); + diff --git a/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/cuda/ms_deform_im2col_cuda.cuh b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/cuda/ms_deform_im2col_cuda.cuh new file mode 100644 index 0000000000000000000000000000000000000000..c04e0d4ab97d25c1756fcd8d08dd1e5a6d280b7c --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/cuda/ms_deform_im2col_cuda.cuh @@ -0,0 +1,1332 @@ +/*! +************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************** +* Modified from DCN (https://github.com/msracver/Deformable-ConvNets) +* Copyright (c) 2018 Microsoft +************************************************************************** +*/ + +/*! +* Copyright (c) Facebook, Inc. and its affiliates. +* Modified by Bowen Cheng from https://github.com/fundamentalvision/Deformable-DETR +*/ + +#include +#include +#include + +#include +#include + +#include + +#define CUDA_KERNEL_LOOP(i, n) \ + for (int i = blockIdx.x * blockDim.x + threadIdx.x; \ + i < (n); \ + i += blockDim.x * gridDim.x) + +const int CUDA_NUM_THREADS = 1024; +inline int GET_BLOCKS(const int N, const int num_threads) +{ + return (N + num_threads - 1) / num_threads; +} + + +template +__device__ scalar_t ms_deform_attn_im2col_bilinear(const scalar_t* &bottom_data, + const int &height, const int &width, const int &nheads, const int &channels, + const scalar_t &h, const scalar_t &w, const int &m, const int &c) +{ + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const scalar_t lh = h - h_low; + const scalar_t lw = w - w_low; + const scalar_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * channels + c; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + } + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + } + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + } + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + } + + const scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + + const scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + return val; +} + + +template +__device__ void ms_deform_attn_col2im_bilinear(const scalar_t* &bottom_data, + const int &height, const int &width, const int &nheads, const int &channels, + const scalar_t &h, const scalar_t &w, const int &m, const int &c, + const scalar_t &top_grad, + const scalar_t &attn_weight, + scalar_t* &grad_value, + scalar_t* grad_sampling_loc, + scalar_t* grad_attn_weight) +{ + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const scalar_t lh = h - h_low; + const scalar_t lw = w - w_low; + const scalar_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * channels + c; + + const scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + const scalar_t top_grad_value = top_grad * attn_weight; + scalar_t grad_h_weight = 0, grad_w_weight = 0; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + grad_h_weight -= hw * v1; + grad_w_weight -= hh * v1; + atomicAdd(grad_value+ptr1, w1*top_grad_value); + } + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + grad_h_weight -= lw * v2; + grad_w_weight += hh * v2; + atomicAdd(grad_value+ptr2, w2*top_grad_value); + } + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + grad_h_weight += hw * v3; + grad_w_weight -= lh * v3; + atomicAdd(grad_value+ptr3, w3*top_grad_value); + } + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + grad_h_weight += lw * v4; + grad_w_weight += lh * v4; + atomicAdd(grad_value+ptr4, w4*top_grad_value); + } + + const scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + *grad_attn_weight = top_grad * val; + *grad_sampling_loc = width * grad_w_weight * top_grad_value; + *(grad_sampling_loc + 1) = height * grad_h_weight * top_grad_value; +} + + +template +__device__ void ms_deform_attn_col2im_bilinear_gm(const scalar_t* &bottom_data, + const int &height, const int &width, const int &nheads, const int &channels, + const scalar_t &h, const scalar_t &w, const int &m, const int &c, + const scalar_t &top_grad, + const scalar_t &attn_weight, + scalar_t* &grad_value, + scalar_t* grad_sampling_loc, + scalar_t* grad_attn_weight) +{ + const int h_low = floor(h); + const int w_low = floor(w); + const int h_high = h_low + 1; + const int w_high = w_low + 1; + + const scalar_t lh = h - h_low; + const scalar_t lw = w - w_low; + const scalar_t hh = 1 - lh, hw = 1 - lw; + + const int w_stride = nheads * channels; + const int h_stride = width * w_stride; + const int h_low_ptr_offset = h_low * h_stride; + const int h_high_ptr_offset = h_low_ptr_offset + h_stride; + const int w_low_ptr_offset = w_low * w_stride; + const int w_high_ptr_offset = w_low_ptr_offset + w_stride; + const int base_ptr = m * channels + c; + + const scalar_t w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw; + const scalar_t top_grad_value = top_grad * attn_weight; + scalar_t grad_h_weight = 0, grad_w_weight = 0; + + scalar_t v1 = 0; + if (h_low >= 0 && w_low >= 0) + { + const int ptr1 = h_low_ptr_offset + w_low_ptr_offset + base_ptr; + v1 = bottom_data[ptr1]; + grad_h_weight -= hw * v1; + grad_w_weight -= hh * v1; + atomicAdd(grad_value+ptr1, w1*top_grad_value); + } + scalar_t v2 = 0; + if (h_low >= 0 && w_high <= width - 1) + { + const int ptr2 = h_low_ptr_offset + w_high_ptr_offset + base_ptr; + v2 = bottom_data[ptr2]; + grad_h_weight -= lw * v2; + grad_w_weight += hh * v2; + atomicAdd(grad_value+ptr2, w2*top_grad_value); + } + scalar_t v3 = 0; + if (h_high <= height - 1 && w_low >= 0) + { + const int ptr3 = h_high_ptr_offset + w_low_ptr_offset + base_ptr; + v3 = bottom_data[ptr3]; + grad_h_weight += hw * v3; + grad_w_weight -= lh * v3; + atomicAdd(grad_value+ptr3, w3*top_grad_value); + } + scalar_t v4 = 0; + if (h_high <= height - 1 && w_high <= width - 1) + { + const int ptr4 = h_high_ptr_offset + w_high_ptr_offset + base_ptr; + v4 = bottom_data[ptr4]; + grad_h_weight += lw * v4; + grad_w_weight += lh * v4; + atomicAdd(grad_value+ptr4, w4*top_grad_value); + } + + const scalar_t val = (w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4); + atomicAdd(grad_attn_weight, top_grad * val); + atomicAdd(grad_sampling_loc, width * grad_w_weight * top_grad_value); + atomicAdd(grad_sampling_loc + 1, height * grad_h_weight * top_grad_value); +} + + +template +__global__ void ms_deformable_im2col_gpu_kernel(const int n, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *data_col) +{ + CUDA_KERNEL_LOOP(index, n) + { + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + scalar_t *data_col_ptr = data_col + index; + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + scalar_t col = 0; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const scalar_t *data_value_ptr = data_value + (data_value_ptr_init_offset + level_start_id * qid_stride); + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + col += ms_deform_attn_im2col_bilinear(data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col) * weight; + } + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + } + } + *data_col_ptr = col; + } +} + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + __shared__ scalar_t cache_grad_sampling_loc[blockSize * 2]; + __shared__ scalar_t cache_grad_attn_weight[blockSize]; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + if (tid == 0) + { + scalar_t _grad_w=cache_grad_sampling_loc[0], _grad_h=cache_grad_sampling_loc[1], _grad_a=cache_grad_attn_weight[0]; + int sid=2; + for (unsigned int tid = 1; tid < blockSize; ++tid) + { + _grad_w += cache_grad_sampling_loc[sid]; + _grad_h += cache_grad_sampling_loc[sid + 1]; + _grad_a += cache_grad_attn_weight[tid]; + sid += 2; + } + + + *grad_sampling_loc = _grad_w; + *(grad_sampling_loc + 1) = _grad_h; + *grad_attn_weight = _grad_a; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + __shared__ scalar_t cache_grad_sampling_loc[blockSize * 2]; + __shared__ scalar_t cache_grad_attn_weight[blockSize]; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s=blockSize/2; s>0; s>>=1) + { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + s]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1]; + } + __syncthreads(); + } + + if (tid == 0) + { + *grad_sampling_loc = cache_grad_sampling_loc[0]; + *(grad_sampling_loc + 1) = cache_grad_sampling_loc[1]; + *grad_attn_weight = cache_grad_attn_weight[0]; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_reduce_v1(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + extern __shared__ int _s[]; + scalar_t* cache_grad_sampling_loc = (scalar_t*)_s; + scalar_t* cache_grad_attn_weight = cache_grad_sampling_loc + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + if (tid == 0) + { + scalar_t _grad_w=cache_grad_sampling_loc[0], _grad_h=cache_grad_sampling_loc[1], _grad_a=cache_grad_attn_weight[0]; + int sid=2; + for (unsigned int tid = 1; tid < blockDim.x; ++tid) + { + _grad_w += cache_grad_sampling_loc[sid]; + _grad_h += cache_grad_sampling_loc[sid + 1]; + _grad_a += cache_grad_attn_weight[tid]; + sid += 2; + } + + + *grad_sampling_loc = _grad_w; + *(grad_sampling_loc + 1) = _grad_h; + *grad_attn_weight = _grad_a; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_reduce_v2(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + extern __shared__ int _s[]; + scalar_t* cache_grad_sampling_loc = (scalar_t*)_s; + scalar_t* cache_grad_attn_weight = cache_grad_sampling_loc + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s=blockDim.x/2, spre=blockDim.x; s>0; s>>=1, spre>>=1) + { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + s]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1]; + if (tid + (s << 1) < spre) + { + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + (s << 1)]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2 + (s << 1)]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1 + (s << 1)]; + } + } + __syncthreads(); + } + + if (tid == 0) + { + *grad_sampling_loc = cache_grad_sampling_loc[0]; + *(grad_sampling_loc + 1) = cache_grad_sampling_loc[1]; + *grad_attn_weight = cache_grad_attn_weight[0]; + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + +template +__global__ void ms_deformable_col2im_gpu_kernel_shm_reduce_v2_multi_blocks(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + extern __shared__ int _s[]; + scalar_t* cache_grad_sampling_loc = (scalar_t*)_s; + scalar_t* cache_grad_attn_weight = cache_grad_sampling_loc + 2 * blockDim.x; + unsigned int tid = threadIdx.x; + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + *(cache_grad_sampling_loc+(threadIdx.x << 1)) = 0; + *(cache_grad_sampling_loc+((threadIdx.x << 1) + 1)) = 0; + *(cache_grad_attn_weight+threadIdx.x)=0; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + cache_grad_sampling_loc+(threadIdx.x << 1), cache_grad_attn_weight+threadIdx.x); + } + + __syncthreads(); + + for (unsigned int s=blockDim.x/2, spre=blockDim.x; s>0; s>>=1, spre>>=1) + { + if (tid < s) { + const unsigned int xid1 = tid << 1; + const unsigned int xid2 = (tid + s) << 1; + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + s]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1]; + if (tid + (s << 1) < spre) + { + cache_grad_attn_weight[tid] += cache_grad_attn_weight[tid + (s << 1)]; + cache_grad_sampling_loc[xid1] += cache_grad_sampling_loc[xid2 + (s << 1)]; + cache_grad_sampling_loc[xid1 + 1] += cache_grad_sampling_loc[xid2 + 1 + (s << 1)]; + } + } + __syncthreads(); + } + + if (tid == 0) + { + atomicAdd(grad_sampling_loc, cache_grad_sampling_loc[0]); + atomicAdd(grad_sampling_loc + 1, cache_grad_sampling_loc[1]); + atomicAdd(grad_attn_weight, cache_grad_attn_weight[0]); + } + __syncthreads(); + + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +__global__ void ms_deformable_col2im_gpu_kernel_gm(const int n, + const scalar_t *grad_col, + const scalar_t *data_value, + const int64_t *data_spatial_shapes, + const int64_t *data_level_start_index, + const scalar_t *data_sampling_loc, + const scalar_t *data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t *grad_value, + scalar_t *grad_sampling_loc, + scalar_t *grad_attn_weight) +{ + CUDA_KERNEL_LOOP(index, n) + { + int _temp = index; + const int c_col = _temp % channels; + _temp /= channels; + const int sampling_index = _temp; + const int m_col = _temp % num_heads; + _temp /= num_heads; + const int q_col = _temp % num_query; + _temp /= num_query; + const int b_col = _temp; + + const scalar_t top_grad = grad_col[index]; + + int data_weight_ptr = sampling_index * num_levels * num_point; + int data_loc_w_ptr = data_weight_ptr << 1; + const int grad_sampling_ptr = data_weight_ptr; + grad_sampling_loc += grad_sampling_ptr << 1; + grad_attn_weight += grad_sampling_ptr; + const int grad_weight_stride = 1; + const int grad_loc_stride = 2; + const int qid_stride = num_heads * channels; + const int data_value_ptr_init_offset = b_col * spatial_size * qid_stride; + + for (int l_col=0; l_col < num_levels; ++l_col) + { + const int level_start_id = data_level_start_index[l_col]; + const int spatial_h_ptr = l_col << 1; + const int spatial_h = data_spatial_shapes[spatial_h_ptr]; + const int spatial_w = data_spatial_shapes[spatial_h_ptr + 1]; + const int value_ptr_offset = data_value_ptr_init_offset + level_start_id * qid_stride; + const scalar_t *data_value_ptr = data_value + value_ptr_offset; + scalar_t *grad_value_ptr = grad_value + value_ptr_offset; + + for (int p_col=0; p_col < num_point; ++p_col) + { + const scalar_t loc_w = data_sampling_loc[data_loc_w_ptr]; + const scalar_t loc_h = data_sampling_loc[data_loc_w_ptr + 1]; + const scalar_t weight = data_attn_weight[data_weight_ptr]; + + const scalar_t h_im = loc_h * spatial_h - 0.5; + const scalar_t w_im = loc_w * spatial_w - 0.5; + if (h_im > -1 && w_im > -1 && h_im < spatial_h && w_im < spatial_w) + { + ms_deform_attn_col2im_bilinear_gm( + data_value_ptr, spatial_h, spatial_w, num_heads, channels, h_im, w_im, m_col, c_col, + top_grad, weight, grad_value_ptr, + grad_sampling_loc, grad_attn_weight); + } + data_weight_ptr += 1; + data_loc_w_ptr += 2; + grad_attn_weight += grad_weight_stride; + grad_sampling_loc += grad_loc_stride; + } + } + } +} + + +template +void ms_deformable_im2col_cuda(cudaStream_t stream, + const scalar_t* data_value, + const int64_t* data_spatial_shapes, + const int64_t* data_level_start_index, + const scalar_t* data_sampling_loc, + const scalar_t* data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t* data_col) +{ + const int num_kernels = batch_size * num_query * num_heads * channels; + const int num_actual_kernels = batch_size * num_query * num_heads * channels; + const int num_threads = CUDA_NUM_THREADS; + ms_deformable_im2col_gpu_kernel + <<>>( + num_kernels, data_value, data_spatial_shapes, data_level_start_index, data_sampling_loc, data_attn_weight, + batch_size, spatial_size, num_heads, channels, num_levels, num_query, num_point, data_col); + + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in ms_deformable_im2col_cuda: %s\n", cudaGetErrorString(err)); + } + +} + +template +void ms_deformable_col2im_cuda(cudaStream_t stream, + const scalar_t* grad_col, + const scalar_t* data_value, + const int64_t * data_spatial_shapes, + const int64_t * data_level_start_index, + const scalar_t * data_sampling_loc, + const scalar_t * data_attn_weight, + const int batch_size, + const int spatial_size, + const int num_heads, + const int channels, + const int num_levels, + const int num_query, + const int num_point, + scalar_t* grad_value, + scalar_t* grad_sampling_loc, + scalar_t* grad_attn_weight) +{ + const int num_threads = (channels > CUDA_NUM_THREADS)?CUDA_NUM_THREADS:channels; + const int num_kernels = batch_size * num_query * num_heads * channels; + const int num_actual_kernels = batch_size * num_query * num_heads * channels; + if (channels > 1024) + { + if ((channels & 1023) == 0) + { + ms_deformable_col2im_gpu_kernel_shm_reduce_v2_multi_blocks + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + else + { + ms_deformable_col2im_gpu_kernel_gm + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + } + else{ + switch(channels) + { + case 1: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 2: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 4: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 8: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 16: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 32: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 64: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 128: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 256: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 512: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + case 1024: + ms_deformable_col2im_gpu_kernel_shm_blocksize_aware_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + break; + default: + if (channels < 64) + { + ms_deformable_col2im_gpu_kernel_shm_reduce_v1 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + else + { + ms_deformable_col2im_gpu_kernel_shm_reduce_v2 + <<>>( + num_kernels, + grad_col, + data_value, + data_spatial_shapes, + data_level_start_index, + data_sampling_loc, + data_attn_weight, + batch_size, + spatial_size, + num_heads, + channels, + num_levels, + num_query, + num_point, + grad_value, + grad_sampling_loc, + grad_attn_weight); + } + } + } + cudaError_t err = cudaGetLastError(); + if (err != cudaSuccess) + { + printf("error in ms_deformable_col2im_cuda: %s\n", cudaGetErrorString(err)); + } + +} \ No newline at end of file diff --git a/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/ms_deform_attn.h b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/ms_deform_attn.h new file mode 100644 index 0000000000000000000000000000000000000000..2f80a1b294c55b37d13bb3558ff7aeadba3b37de --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/ms_deform_attn.h @@ -0,0 +1,67 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +/*! +* Copyright (c) Facebook, Inc. and its affiliates. +* Modified by Bowen Cheng from https://github.com/fundamentalvision/Deformable-DETR +*/ + +#pragma once + +#include "cpu/ms_deform_attn_cpu.h" + +#ifdef WITH_CUDA +#include "cuda/ms_deform_attn_cuda.h" +#endif + + +at::Tensor +ms_deform_attn_forward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const int im2col_step) +{ + if (value.type().is_cuda()) + { +#ifdef WITH_CUDA + return ms_deform_attn_cuda_forward( + value, spatial_shapes, level_start_index, sampling_loc, attn_weight, im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + +std::vector +ms_deform_attn_backward( + const at::Tensor &value, + const at::Tensor &spatial_shapes, + const at::Tensor &level_start_index, + const at::Tensor &sampling_loc, + const at::Tensor &attn_weight, + const at::Tensor &grad_output, + const int im2col_step) +{ + if (value.type().is_cuda()) + { +#ifdef WITH_CUDA + return ms_deform_attn_cuda_backward( + value, spatial_shapes, level_start_index, sampling_loc, attn_weight, grad_output, im2col_step); +#else + AT_ERROR("Not compiled with GPU support"); +#endif + } + AT_ERROR("Not implemented on the CPU"); +} + diff --git a/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/vision.cpp b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/vision.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4a08821e0121a77556aa7a263ec8ebfa928b13b6 --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/src/vision.cpp @@ -0,0 +1,21 @@ +/*! +************************************************************************************************** +* Deformable DETR +* Copyright (c) 2020 SenseTime. All Rights Reserved. +* Licensed under the Apache License, Version 2.0 [see LICENSE for details] +************************************************************************************************** +* Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +************************************************************************************************** +*/ + +/*! +* Copyright (c) Facebook, Inc. and its affiliates. +* Modified by Bowen Cheng from https://github.com/fundamentalvision/Deformable-DETR +*/ + +#include "ms_deform_attn.h" + +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { + m.def("ms_deform_attn_forward", &ms_deform_attn_forward, "ms_deform_attn_forward"); + m.def("ms_deform_attn_backward", &ms_deform_attn_backward, "ms_deform_attn_backward"); +} diff --git a/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/test.py b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/test.py new file mode 100644 index 0000000000000000000000000000000000000000..6e1b545459f6fd3235767e721eb5a1090ae14bef --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/pixel_decoder/ops/test.py @@ -0,0 +1,92 @@ +# ------------------------------------------------------------------------------------------------ +# Deformable DETR +# Copyright (c) 2020 SenseTime. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------------------------------ +# Modified from https://github.com/chengdazhi/Deformable-Convolution-V2-PyTorch/tree/pytorch_1.0.0 +# ------------------------------------------------------------------------------------------------ + +# Copyright (c) Facebook, Inc. and its affiliates. +# Modified by Bowen Cheng from https://github.com/fundamentalvision/Deformable-DETR + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import division + +import time +import torch +import torch.nn as nn +from torch.autograd import gradcheck + +from functions.ms_deform_attn_func import MSDeformAttnFunction, ms_deform_attn_core_pytorch + + +N, M, D = 1, 2, 2 +Lq, L, P = 2, 2, 2 +shapes = torch.as_tensor([(6, 4), (3, 2)], dtype=torch.long).cuda() +level_start_index = torch.cat((shapes.new_zeros((1, )), shapes.prod(1).cumsum(0)[:-1])) +S = sum([(H*W).item() for H, W in shapes]) + + +torch.manual_seed(3) + + +@torch.no_grad() +def check_forward_equal_with_pytorch_double(): + value = torch.rand(N, S, M, D).cuda() * 0.01 + sampling_locations = torch.rand(N, Lq, M, L, P, 2).cuda() + attention_weights = torch.rand(N, Lq, M, L, P).cuda() + 1e-5 + attention_weights /= attention_weights.sum(-1, keepdim=True).sum(-2, keepdim=True) + im2col_step = 2 + output_pytorch = ms_deform_attn_core_pytorch(value.double(), shapes, sampling_locations.double(), attention_weights.double()).detach().cpu() + output_cuda = MSDeformAttnFunction.apply(value.double(), shapes, level_start_index, sampling_locations.double(), attention_weights.double(), im2col_step).detach().cpu() + fwdok = torch.allclose(output_cuda, output_pytorch) + max_abs_err = (output_cuda - output_pytorch).abs().max() + max_rel_err = ((output_cuda - output_pytorch).abs() / output_pytorch.abs()).max() + + print(f'* {fwdok} check_forward_equal_with_pytorch_double: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + +@torch.no_grad() +def check_forward_equal_with_pytorch_float(): + value = torch.rand(N, S, M, D).cuda() * 0.01 + sampling_locations = torch.rand(N, Lq, M, L, P, 2).cuda() + attention_weights = torch.rand(N, Lq, M, L, P).cuda() + 1e-5 + attention_weights /= attention_weights.sum(-1, keepdim=True).sum(-2, keepdim=True) + im2col_step = 2 + output_pytorch = ms_deform_attn_core_pytorch(value, shapes, sampling_locations, attention_weights).detach().cpu() + output_cuda = MSDeformAttnFunction.apply(value, shapes, level_start_index, sampling_locations, attention_weights, im2col_step).detach().cpu() + fwdok = torch.allclose(output_cuda, output_pytorch, rtol=1e-2, atol=1e-3) + max_abs_err = (output_cuda - output_pytorch).abs().max() + max_rel_err = ((output_cuda - output_pytorch).abs() / output_pytorch.abs()).max() + + print(f'* {fwdok} check_forward_equal_with_pytorch_float: max_abs_err {max_abs_err:.2e} max_rel_err {max_rel_err:.2e}') + + +def check_gradient_numerical(channels=4, grad_value=True, grad_sampling_loc=True, grad_attn_weight=True): + + value = torch.rand(N, S, M, channels).cuda() * 0.01 + sampling_locations = torch.rand(N, Lq, M, L, P, 2).cuda() + attention_weights = torch.rand(N, Lq, M, L, P).cuda() + 1e-5 + attention_weights /= attention_weights.sum(-1, keepdim=True).sum(-2, keepdim=True) + im2col_step = 2 + func = MSDeformAttnFunction.apply + + value.requires_grad = grad_value + sampling_locations.requires_grad = grad_sampling_loc + attention_weights.requires_grad = grad_attn_weight + + gradok = gradcheck(func, (value.double(), shapes, level_start_index, sampling_locations.double(), attention_weights.double(), im2col_step)) + + print(f'* {gradok} check_gradient_numerical(D={channels})') + + +if __name__ == '__main__': + check_forward_equal_with_pytorch_double() + check_forward_equal_with_pytorch_float() + + for channels in [30, 32, 64, 71, 1025, 2048, 3096]: + check_gradient_numerical(channels, True, True, True) + + + diff --git a/annotator/oneformer/oneformer/modeling/transformer_decoder/__init__.py b/annotator/oneformer/oneformer/modeling/transformer_decoder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b84bd4ecb48f134ccc218c4d5f02c50f7033bcd9 --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/transformer_decoder/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .oneformer_transformer_decoder import ContrastiveMultiScaleMaskedTransformerDecoder \ No newline at end of file diff --git a/annotator/oneformer/oneformer/modeling/transformer_decoder/oneformer_transformer_decoder.py b/annotator/oneformer/oneformer/modeling/transformer_decoder/oneformer_transformer_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..2887c7718f864f5c64f245c7eee307c04835c41f --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/transformer_decoder/oneformer_transformer_decoder.py @@ -0,0 +1,528 @@ +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/Mask2Former/blob/main/mask2former/modeling/transformer_decoder/mask2former_transformer_decoder.py +# Modified by Jitesh Jain (https://github.com/praeclarumjj3) +# ------------------------------------------------------------------------------ + +import logging +import fvcore.nn.weight_init as weight_init +from typing import Optional +import torch +from torch import nn, Tensor +from torch.nn import functional as F + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.layers import Conv2d + +from .position_encoding import PositionEmbeddingSine +from .transformer import Transformer + +from annotator.oneformer.detectron2.utils.registry import Registry + + +TRANSFORMER_DECODER_REGISTRY = Registry("TRANSFORMER_MODULE") +TRANSFORMER_DECODER_REGISTRY.__doc__ = """ +Registry for transformer module in OneFormer. +""" + + +def build_transformer_decoder(cfg, in_channels, mask_classification=True): + """ + Build a instance embedding branch from `cfg.MODEL.INS_EMBED_HEAD.NAME`. + """ + name = cfg.MODEL.ONE_FORMER.TRANSFORMER_DECODER_NAME + return TRANSFORMER_DECODER_REGISTRY.get(name)(cfg, in_channels, mask_classification) + + +class SelfAttentionLayer(nn.Module): + + def __init__(self, d_model, nhead, dropout=0.0, + activation="relu", normalize_before=False): + super().__init__() + self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) + + self.norm = nn.LayerNorm(d_model) + self.dropout = nn.Dropout(dropout) + + self.activation = _get_activation_fn(activation) + self.normalize_before = normalize_before + + self._reset_parameters() + + def _reset_parameters(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + + def with_pos_embed(self, tensor, pos: Optional[Tensor]): + return tensor if pos is None else tensor + pos + + def forward_post(self, tgt, + tgt_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + q = k = self.with_pos_embed(tgt, query_pos) + tgt2 = self.self_attn(q, k, value=tgt, attn_mask=tgt_mask, + key_padding_mask=tgt_key_padding_mask)[0] + tgt = tgt + self.dropout(tgt2) + tgt = self.norm(tgt) + + return tgt + + def forward_pre(self, tgt, + tgt_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + tgt2 = self.norm(tgt) + q = k = self.with_pos_embed(tgt2, query_pos) + tgt2 = self.self_attn(q, k, value=tgt2, attn_mask=tgt_mask, + key_padding_mask=tgt_key_padding_mask)[0] + tgt = tgt + self.dropout(tgt2) + + return tgt + + def forward(self, tgt, + tgt_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + if self.normalize_before: + return self.forward_pre(tgt, tgt_mask, + tgt_key_padding_mask, query_pos) + return self.forward_post(tgt, tgt_mask, + tgt_key_padding_mask, query_pos) + + +class CrossAttentionLayer(nn.Module): + + def __init__(self, d_model, nhead, dropout=0.0, + activation="relu", normalize_before=False): + super().__init__() + self.multihead_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) + + self.norm = nn.LayerNorm(d_model) + self.dropout = nn.Dropout(dropout) + + self.activation = _get_activation_fn(activation) + self.normalize_before = normalize_before + + self._reset_parameters() + + def _reset_parameters(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + + def with_pos_embed(self, tensor, pos: Optional[Tensor]): + return tensor if pos is None else tensor + pos + + def forward_post(self, tgt, memory, + memory_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + tgt2 = self.multihead_attn(query=self.with_pos_embed(tgt, query_pos), + key=self.with_pos_embed(memory, pos), + value=memory, attn_mask=memory_mask, + key_padding_mask=memory_key_padding_mask)[0] + tgt = tgt + self.dropout(tgt2) + tgt = self.norm(tgt) + + return tgt + + def forward_pre(self, tgt, memory, + memory_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + tgt2 = self.norm(tgt) + tgt2 = self.multihead_attn(query=self.with_pos_embed(tgt2, query_pos), + key=self.with_pos_embed(memory, pos), + value=memory, attn_mask=memory_mask, + key_padding_mask=memory_key_padding_mask)[0] + tgt = tgt + self.dropout(tgt2) + + return tgt + + def forward(self, tgt, memory, + memory_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None): + if self.normalize_before: + return self.forward_pre(tgt, memory, memory_mask, + memory_key_padding_mask, pos, query_pos) + return self.forward_post(tgt, memory, memory_mask, + memory_key_padding_mask, pos, query_pos) + + +class FFNLayer(nn.Module): + + def __init__(self, d_model, dim_feedforward=2048, dropout=0.0, + activation="relu", normalize_before=False): + super().__init__() + # Implementation of Feedforward model + self.linear1 = nn.Linear(d_model, dim_feedforward) + self.dropout = nn.Dropout(dropout) + self.linear2 = nn.Linear(dim_feedforward, d_model) + + self.norm = nn.LayerNorm(d_model) + + self.activation = _get_activation_fn(activation) + self.normalize_before = normalize_before + + self._reset_parameters() + + def _reset_parameters(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + + def with_pos_embed(self, tensor, pos: Optional[Tensor]): + return tensor if pos is None else tensor + pos + + def forward_post(self, tgt): + tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt)))) + tgt = tgt + self.dropout(tgt2) + tgt = self.norm(tgt) + return tgt + + def forward_pre(self, tgt): + tgt2 = self.norm(tgt) + tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt2)))) + tgt = tgt + self.dropout(tgt2) + return tgt + + def forward(self, tgt): + if self.normalize_before: + return self.forward_pre(tgt) + return self.forward_post(tgt) + + +def _get_activation_fn(activation): + """Return an activation function given a string""" + if activation == "relu": + return F.relu + if activation == "gelu": + return F.gelu + if activation == "glu": + return F.glu + raise RuntimeError(F"activation should be relu/gelu, not {activation}.") + + +class MLP(nn.Module): + """ Very simple multi-layer perceptron (also called FFN)""" + + def __init__(self, input_dim, hidden_dim, output_dim, num_layers): + super().__init__() + self.num_layers = num_layers + h = [hidden_dim] * (num_layers - 1) + self.layers = nn.ModuleList(nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim])) + + def forward(self, x): + for i, layer in enumerate(self.layers): + x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x) + return x + + +@TRANSFORMER_DECODER_REGISTRY.register() +class ContrastiveMultiScaleMaskedTransformerDecoder(nn.Module): + + _version = 2 + + def _load_from_state_dict( + self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs + ): + version = local_metadata.get("version", None) + if version is None or version < 2: + # Do not warn if train from scratch + scratch = True + logger = logging.getLogger(__name__) + for k in list(state_dict.keys()): + newk = k + if "static_query" in k: + newk = k.replace("static_query", "query_feat") + if newk != k: + state_dict[newk] = state_dict[k] + del state_dict[k] + scratch = False + + if not scratch: + logger.warning( + f"Weight format of {self.__class__.__name__} have changed! " + "Please upgrade your models. Applying automatic conversion now ..." + ) + + @configurable + def __init__( + self, + in_channels, + mask_classification=True, + *, + num_classes: int, + hidden_dim: int, + num_queries: int, + nheads: int, + dropout: float, + dim_feedforward: int, + enc_layers: int, + is_train: bool, + dec_layers: int, + class_dec_layers: int, + pre_norm: bool, + mask_dim: int, + enforce_input_project: bool, + use_task_norm: bool, + ): + """ + NOTE: this interface is experimental. + Args: + in_channels: channels of the input features + mask_classification: whether to add mask classifier or not + num_classes: number of classes + hidden_dim: Transformer feature dimension + num_queries: number of queries + nheads: number of heads + dim_feedforward: feature dimension in feedforward network + enc_layers: number of Transformer encoder layers + dec_layers: number of Transformer decoder layers + pre_norm: whether to use pre-LayerNorm or not + mask_dim: mask feature dimension + enforce_input_project: add input project 1x1 conv even if input + channels and hidden dim is identical + """ + super().__init__() + + assert mask_classification, "Only support mask classification model" + self.mask_classification = mask_classification + self.is_train = is_train + self.use_task_norm = use_task_norm + + # positional encoding + N_steps = hidden_dim // 2 + self.pe_layer = PositionEmbeddingSine(N_steps, normalize=True) + + self.class_transformer = Transformer( + d_model=hidden_dim, + dropout=dropout, + nhead=nheads, + dim_feedforward=dim_feedforward, + num_encoder_layers=enc_layers, + num_decoder_layers=class_dec_layers, + normalize_before=pre_norm, + return_intermediate_dec=False, + ) + + # define Transformer decoder here + self.num_heads = nheads + self.num_layers = dec_layers + self.transformer_self_attention_layers = nn.ModuleList() + self.transformer_cross_attention_layers = nn.ModuleList() + self.transformer_ffn_layers = nn.ModuleList() + + for _ in range(self.num_layers): + self.transformer_self_attention_layers.append( + SelfAttentionLayer( + d_model=hidden_dim, + nhead=nheads, + dropout=0.0, + normalize_before=pre_norm, + ) + ) + + self.transformer_cross_attention_layers.append( + CrossAttentionLayer( + d_model=hidden_dim, + nhead=nheads, + dropout=0.0, + normalize_before=pre_norm, + ) + ) + + self.transformer_ffn_layers.append( + FFNLayer( + d_model=hidden_dim, + dim_feedforward=dim_feedforward, + dropout=0.0, + normalize_before=pre_norm, + ) + ) + + self.decoder_norm = nn.LayerNorm(hidden_dim) + + self.num_queries = num_queries + # learnable query p.e. + self.query_embed = nn.Embedding(num_queries, hidden_dim) + + # level embedding (we always use 3 scales) + self.num_feature_levels = 3 + self.level_embed = nn.Embedding(self.num_feature_levels, hidden_dim) + self.input_proj = nn.ModuleList() + for _ in range(self.num_feature_levels): + if in_channels != hidden_dim or enforce_input_project: + self.input_proj.append(Conv2d(in_channels, hidden_dim, kernel_size=1)) + weight_init.c2_xavier_fill(self.input_proj[-1]) + else: + self.input_proj.append(nn.Sequential()) + + self.class_input_proj = Conv2d(in_channels, hidden_dim, kernel_size=1) + weight_init.c2_xavier_fill(self.class_input_proj) + + # output FFNs + if self.mask_classification: + self.class_embed = nn.Linear(hidden_dim, num_classes + 1) + self.mask_embed = MLP(hidden_dim, hidden_dim, mask_dim, 3) + + @classmethod + def from_config(cls, cfg, in_channels, mask_classification): + ret = {} + ret["in_channels"] = in_channels + ret["mask_classification"] = mask_classification + + ret["num_classes"] = cfg.MODEL.SEM_SEG_HEAD.NUM_CLASSES + ret["hidden_dim"] = cfg.MODEL.ONE_FORMER.HIDDEN_DIM + ret["num_queries"] = cfg.MODEL.ONE_FORMER.NUM_OBJECT_QUERIES + # Transformer parameters: + ret["nheads"] = cfg.MODEL.ONE_FORMER.NHEADS + ret["dim_feedforward"] = cfg.MODEL.ONE_FORMER.DIM_FEEDFORWARD + + # NOTE: because we add learnable query features which requires supervision, + # we add minus 1 to decoder layers to be consistent with our loss + # implementation: that is, number of auxiliary losses is always + # equal to number of decoder layers. With learnable query features, the number of + # auxiliary losses equals number of decoders plus 1. + assert cfg.MODEL.ONE_FORMER.DEC_LAYERS >= 1 + ret["dec_layers"] = cfg.MODEL.ONE_FORMER.DEC_LAYERS - 1 + ret["class_dec_layers"] = cfg.MODEL.ONE_FORMER.CLASS_DEC_LAYERS + ret["enc_layers"] = cfg.MODEL.ONE_FORMER.ENC_LAYERS + ret["dropout"] = cfg.MODEL.ONE_FORMER.DROPOUT + ret["pre_norm"] = cfg.MODEL.ONE_FORMER.PRE_NORM + ret["enforce_input_project"] = cfg.MODEL.ONE_FORMER.ENFORCE_INPUT_PROJ + ret["is_train"] = cfg.MODEL.IS_TRAIN + ret["mask_dim"] = cfg.MODEL.SEM_SEG_HEAD.MASK_DIM + ret["use_task_norm"] = cfg.MODEL.ONE_FORMER.USE_TASK_NORM + + return ret + + def forward(self, x, mask_features, tasks, mask = None): + # x is a list of multi-scale feature + assert len(x) == self.num_feature_levels + src = [] + pos = [] + size_list = [] + + # disable mask, it does not affect performance + del mask + + for i in range(self.num_feature_levels): + size_list.append(x[i].shape[-2:]) + pos.append(self.pe_layer(x[i], None).flatten(2)) + src.append(self.input_proj[i](x[i]).flatten(2) + self.level_embed.weight[i][None, :, None]) + + # flatten NxCxHxW to HWxNxC + pos[-1] = pos[-1].permute(2, 0, 1) + src[-1] = src[-1].permute(2, 0, 1) + + _, bs, _ = src[0].shape + + # QxNxC + query_embed = self.query_embed.weight.unsqueeze(1).repeat(1, bs, 1) + tasks = tasks.unsqueeze(0) + if self.use_task_norm: + tasks = self.decoder_norm(tasks) + + feats = self.pe_layer(mask_features, None) + + out_t, _ = self.class_transformer(feats, None, + self.query_embed.weight[:-1], + self.class_input_proj(mask_features), + tasks if self.use_task_norm else None) + out_t = out_t[0].permute(1, 0, 2) + + out = torch.cat([out_t, tasks], dim=0) + + output = out.clone() + + predictions_class = [] + predictions_mask = [] + + # prediction heads on learnable query features + outputs_class, outputs_mask, attn_mask = self.forward_prediction_heads(output, mask_features, attn_mask_target_size=size_list[0], i=0) + predictions_class.append(outputs_class) + predictions_mask.append(outputs_mask) + + for i in range(self.num_layers): + level_index = i % self.num_feature_levels + attn_mask[torch.where(attn_mask.sum(-1) == attn_mask.shape[-1])] = False + # attention: cross-attention first + output = self.transformer_cross_attention_layers[i]( + output, src[level_index], + memory_mask=attn_mask, + memory_key_padding_mask=None, # here we do not apply masking on padded region + pos=pos[level_index], query_pos=query_embed + ) + + output = self.transformer_self_attention_layers[i]( + output, tgt_mask=None, + tgt_key_padding_mask=None, + query_pos=query_embed + ) + + # FFN + output = self.transformer_ffn_layers[i]( + output + ) + + outputs_class, outputs_mask, attn_mask = self.forward_prediction_heads(output, mask_features, attn_mask_target_size=size_list[(i + 1) % self.num_feature_levels], i=i+1) + predictions_class.append(outputs_class) + predictions_mask.append(outputs_mask) + + assert len(predictions_class) == self.num_layers + 1 + if self.is_train: + query_class = out.permute(1, 0, 2) + else: + query_class = None + out = { + 'contrastive_logits': query_class, + 'pred_logits': predictions_class[-1], + 'pred_masks': predictions_mask[-1], + 'aux_outputs': self._set_aux_loss( + predictions_class if self.mask_classification else None, + predictions_mask, + ) + } + + return out + + def forward_prediction_heads(self, output, mask_features, attn_mask_target_size, i): + decoder_output = self.decoder_norm(output) + decoder_output = decoder_output.transpose(0, 1) + outputs_class = self.class_embed(decoder_output) + mask_embed = self.mask_embed(decoder_output) + outputs_mask = torch.einsum("bqc,bchw->bqhw", mask_embed, mask_features) + + # NOTE: prediction is of higher-resolution + # [B, Q, H, W] -> [B, Q, H*W] -> [B, h, Q, H*W] -> [B*h, Q, HW] + attn_mask = F.interpolate(outputs_mask, size=attn_mask_target_size, mode="bilinear", align_corners=False) + + # save_attn_masks(attn_mask.sigmoid() < 0.5, fname=f'demo/maps/{i}_pre_bool') + + # must use bool type + # If a BoolTensor is provided, positions with ``True`` are not allowed to attend while ``False`` values will be unchanged. + attn_mask = (attn_mask.sigmoid().flatten(2).unsqueeze(1).repeat(1, self.num_heads, 1, 1).flatten(0, 1) < 0.5).bool() + attn_mask = attn_mask.detach() + + return outputs_class, outputs_mask, attn_mask + + @torch.jit.unused + def _set_aux_loss(self, outputs_class, outputs_seg_masks): + # this is a workaround to make torchscript happy, as torchscript + # doesn't support dictionary with non-homogeneous values, such + # as a dict having both a Tensor and a list. + if self.mask_classification: + aux_list = [ + {"pred_logits": a, "pred_masks": b} + for a, b in zip(outputs_class[:-1], outputs_seg_masks[:-1]) + ] + else: + aux_list = [{"pred_masks": b} for b, in outputs_seg_masks[:-1]] + + return aux_list \ No newline at end of file diff --git a/annotator/oneformer/oneformer/modeling/transformer_decoder/position_encoding.py b/annotator/oneformer/oneformer/modeling/transformer_decoder/position_encoding.py new file mode 100644 index 0000000000000000000000000000000000000000..051984d9ea6e04e834f6fae3daf7d8317c2f0819 --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/transformer_decoder/position_encoding.py @@ -0,0 +1,67 @@ +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/Mask2Former/blob/main/mask2former/modeling/transformer_decoder/position_encoding.py +# Modified by Jitesh Jain (https://github.com/praeclarumjj3) +# ------------------------------------------------------------------------------ + +""" +Various positional encodings for the transformer. +""" +import math + +import torch +from torch import nn + + +class PositionEmbeddingSine(nn.Module): + """ + This is a more standard version of the position embedding, very similar to the one + used by the Attention is all you need paper, generalized to work on images. + """ + + def __init__(self, num_pos_feats=64, temperature=10000, normalize=False, scale=None): + super().__init__() + self.num_pos_feats = num_pos_feats + self.temperature = temperature + self.normalize = normalize + if scale is not None and normalize is False: + raise ValueError("normalize should be True if scale is passed") + if scale is None: + scale = 2 * math.pi + self.scale = scale + + def forward(self, x, mask=None): + if mask is None: + mask = torch.zeros((x.size(0), x.size(2), x.size(3)), device=x.device, dtype=torch.bool) + not_mask = ~mask + y_embed = not_mask.cumsum(1, dtype=torch.float32) + x_embed = not_mask.cumsum(2, dtype=torch.float32) + if self.normalize: + eps = 1e-6 + y_embed = y_embed / (y_embed[:, -1:, :] + eps) * self.scale + x_embed = x_embed / (x_embed[:, :, -1:] + eps) * self.scale + + dim_t = torch.arange(self.num_pos_feats, dtype=torch.float32, device=x.device) + dim_t = self.temperature ** (2 * (dim_t // 2) / self.num_pos_feats) + + pos_x = x_embed[:, :, :, None] / dim_t + pos_y = y_embed[:, :, :, None] / dim_t + pos_x = torch.stack( + (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4 + ).flatten(3) + pos_y = torch.stack( + (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4 + ).flatten(3) + pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) + return pos + + def __repr__(self, _repr_indent=4): + head = "Positional encoding " + self.__class__.__name__ + body = [ + "num_pos_feats: {}".format(self.num_pos_feats), + "temperature: {}".format(self.temperature), + "normalize: {}".format(self.normalize), + "scale: {}".format(self.scale), + ] + # _repr_indent = 4 + lines = [head] + [" " * _repr_indent + line for line in body] + return "\n".join(lines) diff --git a/annotator/oneformer/oneformer/modeling/transformer_decoder/text_transformer.py b/annotator/oneformer/oneformer/modeling/transformer_decoder/text_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..d0b7292018ecfbf4111c0da9c90444d0e1e41cb6 --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/transformer_decoder/text_transformer.py @@ -0,0 +1,257 @@ +# ------------------------------------------------------------------------- +# MIT License +# +# Copyright (c) 2021 OpenAI +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# +# ------------------------------------------------------------------------- + +import torch +import torch.utils.checkpoint as checkpoint +from torch import nn +from collections import OrderedDict +from timm.models.layers import trunc_normal_ + +class Attention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0.): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + # NOTE scale factor was wrong in my original version, can set manually to be compat with prev weights + self.scale = qk_scale or head_dim ** -0.5 + + self.q_proj = nn.Linear(dim, dim, bias=qkv_bias) + self.k_proj = nn.Linear(dim, dim, bias=qkv_bias) + self.v_proj = nn.Linear(dim, dim, bias=qkv_bias) + + + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, q, k, v): + B, N, C = q.shape + assert k.shape == v.shape + B, M, C = k.shape + q = self.q_proj(q).reshape(B, N, self.num_heads, C // self.num_heads) + k = self.k_proj(k).reshape(B, M, self.num_heads, C // self.num_heads) + v = self.v_proj(v).reshape(B, M, self.num_heads, C // self.num_heads) + + attn = torch.einsum('bnkc,bmkc->bknm', q, k) * self.scale + + attn = attn.softmax(dim=-1) + + x = torch.einsum('bknm,bmkc->bnkc', attn, v).reshape(B, N, C) + + x = self.proj(x) + x = self.proj_drop(x) + return x + +class TransformerDecoderLayer(nn.Module): + def __init__( + self, + d_model, + nhead, + dropout=0.1, + ): + super().__init__() + self.self_attn = Attention(d_model, nhead, proj_drop=dropout) + self.cross_attn = Attention(d_model, nhead, proj_drop=dropout) + + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.norm3 = nn.LayerNorm(d_model) + self.dropout = nn.Dropout(dropout) + + self.mlp = nn.Sequential( + nn.Linear(d_model, d_model * 4), + nn.GELU(), + nn.Dropout(dropout), + nn.Linear(d_model * 4, d_model) + ) + + def forward(self, x, mem): + q = k = v = self.norm1(x) + x = x + self.self_attn(q, k, v) + q = self.norm2(x) + x = x + self.cross_attn(q, mem, mem) + x = x + self.dropout(self.mlp(self.norm3(x))) + return x + + +class ContextDecoder(nn.Module): + def __init__(self, + transformer_width=256, + transformer_heads=4, + transformer_layers=6, + visual_dim=1024, + dropout=0.1, + **kwargs): + super().__init__() + + self.memory_proj = nn.Sequential( + nn.LayerNorm(visual_dim), + nn.Linear(visual_dim, transformer_width), + nn.LayerNorm(transformer_width), + ) + + self.text_proj = nn.Sequential( + nn.LayerNorm(visual_dim), + nn.Linear(visual_dim, transformer_width), + ) + + self.decoder = nn.ModuleList([ + TransformerDecoderLayer(transformer_width, transformer_heads, dropout) for _ in range(transformer_layers) + ]) + + self.out_proj = nn.Sequential( + nn.LayerNorm(transformer_width), + nn.Linear(transformer_width, visual_dim) + ) + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + + def forward(self, text, visual): + B, N, C = visual.shape + visual = self.memory_proj(visual) + x = self.text_proj(text) + + for layer in self.decoder: + x = layer(x, visual) + + return self.out_proj(x) + + +class QuickGELU(nn.Module): + + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + + def __init__(self, d_model: int, n_head: int, attn_mask: torch.Tensor = None): + super().__init__() + + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = nn.LayerNorm(d_model) + self.mlp = nn.Sequential( + OrderedDict([('c_fc', nn.Linear(d_model, d_model * 4)), ('gelu', QuickGELU()), + ('c_proj', nn.Linear(d_model * 4, d_model))])) + self.ln_2 = nn.LayerNorm(d_model) + self.attn_mask = attn_mask + + def attention(self, x: torch.Tensor, key_padding_mask: torch.Tensor): + self.attn_mask = self.attn_mask.to(dtype=x.dtype, device=x.device) if self.attn_mask is not None else None + return self.attn(x, x, x, need_weights=False, attn_mask=self.attn_mask, key_padding_mask=key_padding_mask)[0] + + def forward(self, x: torch.Tensor, key_padding_mask=None): + x = x + self.attention(self.ln_1(x), key_padding_mask=key_padding_mask) + x = x + self.mlp(self.ln_2(x)) + return x + +class Transformer(nn.Module): + + def __init__(self, width: int, layers: int, heads: int, attn_mask: torch.Tensor = None, use_checkpoint=False): + super().__init__() + self.width = width + self.layers = layers + self.resblocks = nn.Sequential(*[ResidualAttentionBlock(width, heads, attn_mask) for _ in range(layers)]) + proj_std = (self.width**-0.5) * ((2 * self.layers)**-0.5) + attn_std = self.width**-0.5 + fc_std = (2 * self.width)**-0.5 + for block in self.resblocks: + nn.init.normal_(block.attn.in_proj_weight, std=attn_std) + nn.init.normal_(block.attn.out_proj.weight, std=proj_std) + nn.init.normal_(block.mlp.c_fc.weight, std=fc_std) + nn.init.normal_(block.mlp.c_proj.weight, std=proj_std) + + self.use_checkpoint = use_checkpoint + + def forward(self, x: torch.Tensor): + for resblock in self.resblocks: + if self.use_checkpoint: + x = checkpoint.checkpoint(resblock, x) + else: + x = resblock(x) + return x + + +class TextTransformer(nn.Module): + + def __init__( + self, + context_length: int, + width: int, + layers: int, + vocab_size, + use_checkpoint=False, + ): + + super().__init__() + heads = width // 64 + self.context_length = context_length + self.width = width + self.transformer = Transformer( + width=width, + layers=layers, + heads=heads, + attn_mask=self.build_attention_mask(), + use_checkpoint=use_checkpoint) + + self.positional_embedding = nn.Parameter(torch.empty(self.context_length, width)) + self.ln_final = nn.LayerNorm(width) + self.token_embedding = nn.Embedding(vocab_size, width) + nn.init.normal_(self.token_embedding.weight, std=0.02) + + # initialization + nn.init.normal_(self.positional_embedding, std=0.01) + + def build_attention_mask(self): + # lazily create causal attention mask, with full attention between the vision tokens + # pytorch uses additive attention mask; fill with -inf + mask = torch.empty(self.context_length, self.context_length) + mask.fill_(float('-inf')) + mask.triu_(1) # zero out the lower diagonal + return mask + + def forward(self, text): + x = self.token_embedding(text) + x = x + self.positional_embedding + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + x = self.ln_final(x) + + # x.shape = [batch_size, n_ctx, transformer.width] + # take features from the eot embedding (eot_token is the highest number in each sequence) + x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] + + return x \ No newline at end of file diff --git a/annotator/oneformer/oneformer/modeling/transformer_decoder/transformer.py b/annotator/oneformer/oneformer/modeling/transformer_decoder/transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..cd07525673b9b1165e1fdd0c9990a8f29c84f199 --- /dev/null +++ b/annotator/oneformer/oneformer/modeling/transformer_decoder/transformer.py @@ -0,0 +1,376 @@ +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/Mask2Former/blob/main/mask2former/modeling/transformer_decoder/transformer.py +# Modified by Jitesh Jain (https://github.com/praeclarumjj3) +# ------------------------------------------------------------------------------ + +""" +Transformer class. + +Copy-paste from torch.nn.Transformer with modifications: + * positional encodings are passed in MHattention + * extra LN at the end of encoder is removed + * decoder returns a stack of activations from all decoding layers +""" +import copy +from typing import List, Optional + +import torch +import torch.nn.functional as F +from torch import Tensor, nn + + +class Transformer(nn.Module): + def __init__( + self, + d_model=512, + nhead=8, + num_encoder_layers=6, + num_decoder_layers=6, + dim_feedforward=2048, + dropout=0.1, + activation="relu", + normalize_before=False, + return_intermediate_dec=False, + ): + super().__init__() + + encoder_layer = TransformerEncoderLayer( + d_model, nhead, dim_feedforward, dropout, activation, normalize_before + ) + encoder_norm = nn.LayerNorm(d_model) if normalize_before else None + self.encoder = TransformerEncoder(encoder_layer, num_encoder_layers, encoder_norm) + + decoder_layer = TransformerDecoderLayer( + d_model, nhead, dim_feedforward, dropout, activation, normalize_before + ) + decoder_norm = nn.LayerNorm(d_model) + self.decoder = TransformerDecoder( + decoder_layer, + num_decoder_layers, + decoder_norm, + return_intermediate=return_intermediate_dec, + ) + + self._reset_parameters() + + self.d_model = d_model + self.nhead = nhead + + def _reset_parameters(self): + for p in self.parameters(): + if p.dim() > 1: + nn.init.xavier_uniform_(p) + + def forward(self, src, mask, query_embed, pos_embed, task_token=None): + # flatten NxCxHxW to HWxNxC + bs, c, h, w = src.shape + src = src.flatten(2).permute(2, 0, 1) + pos_embed = pos_embed.flatten(2).permute(2, 0, 1) + query_embed = query_embed.unsqueeze(1).repeat(1, bs, 1) + if mask is not None: + mask = mask.flatten(1) + + if task_token is None: + tgt = torch.zeros_like(query_embed) + else: + tgt = task_token.repeat(query_embed.shape[0], 1, 1) + + memory = self.encoder(src, src_key_padding_mask=mask, pos=pos_embed) + hs = self.decoder( + tgt, memory, memory_key_padding_mask=mask, pos=pos_embed, query_pos=query_embed + ) + return hs.transpose(1, 2), memory.permute(1, 2, 0).view(bs, c, h, w) + + +class TransformerEncoder(nn.Module): + def __init__(self, encoder_layer, num_layers, norm=None): + super().__init__() + self.layers = _get_clones(encoder_layer, num_layers) + self.num_layers = num_layers + self.norm = norm + + def forward( + self, + src, + mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + ): + output = src + + for layer in self.layers: + output = layer( + output, src_mask=mask, src_key_padding_mask=src_key_padding_mask, pos=pos + ) + + if self.norm is not None: + output = self.norm(output) + + return output + + +class TransformerDecoder(nn.Module): + def __init__(self, decoder_layer, num_layers, norm=None, return_intermediate=False): + super().__init__() + self.layers = _get_clones(decoder_layer, num_layers) + self.num_layers = num_layers + self.norm = norm + self.return_intermediate = return_intermediate + + def forward( + self, + tgt, + memory, + tgt_mask: Optional[Tensor] = None, + memory_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None, + ): + output = tgt + + intermediate = [] + + for layer in self.layers: + output = layer( + output, + memory, + tgt_mask=tgt_mask, + memory_mask=memory_mask, + tgt_key_padding_mask=tgt_key_padding_mask, + memory_key_padding_mask=memory_key_padding_mask, + pos=pos, + query_pos=query_pos, + ) + if self.return_intermediate: + intermediate.append(self.norm(output)) + + if self.norm is not None: + output = self.norm(output) + if self.return_intermediate: + intermediate.pop() + intermediate.append(output) + + if self.return_intermediate: + return torch.stack(intermediate) + + return output.unsqueeze(0) + + +class TransformerEncoderLayer(nn.Module): + def __init__( + self, + d_model, + nhead, + dim_feedforward=2048, + dropout=0.1, + activation="relu", + normalize_before=False, + ): + super().__init__() + self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) + # Implementation of Feedforward model + self.linear1 = nn.Linear(d_model, dim_feedforward) + self.dropout = nn.Dropout(dropout) + self.linear2 = nn.Linear(dim_feedforward, d_model) + + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + + self.activation = _get_activation_fn(activation) + self.normalize_before = normalize_before + + def with_pos_embed(self, tensor, pos: Optional[Tensor]): + return tensor if pos is None else tensor + pos + + def forward_post( + self, + src, + src_mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + ): + q = k = self.with_pos_embed(src, pos) + src2 = self.self_attn( + q, k, value=src, attn_mask=src_mask, key_padding_mask=src_key_padding_mask + )[0] + src = src + self.dropout1(src2) + src = self.norm1(src) + src2 = self.linear2(self.dropout(self.activation(self.linear1(src)))) + src = src + self.dropout2(src2) + src = self.norm2(src) + return src + + def forward_pre( + self, + src, + src_mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + ): + src2 = self.norm1(src) + q = k = self.with_pos_embed(src2, pos) + src2 = self.self_attn( + q, k, value=src2, attn_mask=src_mask, key_padding_mask=src_key_padding_mask + )[0] + src = src + self.dropout1(src2) + src2 = self.norm2(src) + src2 = self.linear2(self.dropout(self.activation(self.linear1(src2)))) + src = src + self.dropout2(src2) + return src + + def forward( + self, + src, + src_mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + ): + if self.normalize_before: + return self.forward_pre(src, src_mask, src_key_padding_mask, pos) + return self.forward_post(src, src_mask, src_key_padding_mask, pos) + + +class TransformerDecoderLayer(nn.Module): + def __init__( + self, + d_model, + nhead, + dim_feedforward=2048, + dropout=0.1, + activation="relu", + normalize_before=False, + ): + super().__init__() + self.self_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) + self.multihead_attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout) + # Implementation of Feedforward model + self.linear1 = nn.Linear(d_model, dim_feedforward) + self.dropout = nn.Dropout(dropout) + self.linear2 = nn.Linear(dim_feedforward, d_model) + + self.norm1 = nn.LayerNorm(d_model) + self.norm2 = nn.LayerNorm(d_model) + self.norm3 = nn.LayerNorm(d_model) + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + self.dropout3 = nn.Dropout(dropout) + + self.activation = _get_activation_fn(activation) + self.normalize_before = normalize_before + + def with_pos_embed(self, tensor, pos: Optional[Tensor]): + return tensor if pos is None else tensor + pos + + def forward_post( + self, + tgt, + memory, + tgt_mask: Optional[Tensor] = None, + memory_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None, + ): + q = k = self.with_pos_embed(tgt, query_pos) + tgt2 = self.self_attn( + q, k, value=tgt, attn_mask=tgt_mask, key_padding_mask=tgt_key_padding_mask + )[0] + tgt = tgt + self.dropout1(tgt2) + tgt = self.norm1(tgt) + tgt2 = self.multihead_attn( + query=self.with_pos_embed(tgt, query_pos), + key=self.with_pos_embed(memory, pos), + value=memory, + attn_mask=memory_mask, + key_padding_mask=memory_key_padding_mask, + )[0] + tgt = tgt + self.dropout2(tgt2) + tgt = self.norm2(tgt) + tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt)))) + tgt = tgt + self.dropout3(tgt2) + tgt = self.norm3(tgt) + return tgt + + def forward_pre( + self, + tgt, + memory, + tgt_mask: Optional[Tensor] = None, + memory_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None, + ): + tgt2 = self.norm1(tgt) + q = k = self.with_pos_embed(tgt2, query_pos) + tgt2 = self.self_attn( + q, k, value=tgt2, attn_mask=tgt_mask, key_padding_mask=tgt_key_padding_mask + )[0] + tgt = tgt + self.dropout1(tgt2) + tgt2 = self.norm2(tgt) + tgt2 = self.multihead_attn( + query=self.with_pos_embed(tgt2, query_pos), + key=self.with_pos_embed(memory, pos), + value=memory, + attn_mask=memory_mask, + key_padding_mask=memory_key_padding_mask, + )[0] + tgt = tgt + self.dropout2(tgt2) + tgt2 = self.norm3(tgt) + tgt2 = self.linear2(self.dropout(self.activation(self.linear1(tgt2)))) + tgt = tgt + self.dropout3(tgt2) + return tgt + + def forward( + self, + tgt, + memory, + tgt_mask: Optional[Tensor] = None, + memory_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + pos: Optional[Tensor] = None, + query_pos: Optional[Tensor] = None, + ): + if self.normalize_before: + return self.forward_pre( + tgt, + memory, + tgt_mask, + memory_mask, + tgt_key_padding_mask, + memory_key_padding_mask, + pos, + query_pos, + ) + return self.forward_post( + tgt, + memory, + tgt_mask, + memory_mask, + tgt_key_padding_mask, + memory_key_padding_mask, + pos, + query_pos, + ) + + +def _get_clones(module, N): + return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) + + +def _get_activation_fn(activation): + """Return an activation function given a string""" + if activation == "relu": + return F.relu + if activation == "gelu": + return F.gelu + if activation == "glu": + return F.glu + raise RuntimeError(f"activation should be relu/gelu, not {activation}.") diff --git a/annotator/oneformer/oneformer/oneformer_model.py b/annotator/oneformer/oneformer/oneformer_model.py new file mode 100644 index 0000000000000000000000000000000000000000..8bb18a85a8ecdfa6a7bef912bd6eb038e79e5251 --- /dev/null +++ b/annotator/oneformer/oneformer/oneformer_model.py @@ -0,0 +1,470 @@ +# ------------------------------------------------------------------------------ +# Reference: https://github.com/facebookresearch/Mask2Former/blob/main/mask2former/maskformer_model.py +# Modified by Jitesh Jain (https://github.com/praeclarumjj3) +# ------------------------------------------------------------------------------ + +from typing import Tuple + +import torch +from torch import nn +from torch.nn import functional as F + +from annotator.oneformer.detectron2.config import configurable +from annotator.oneformer.detectron2.data import MetadataCatalog +from annotator.oneformer.detectron2.modeling import META_ARCH_REGISTRY, build_backbone, build_sem_seg_head +from annotator.oneformer.detectron2.modeling.backbone import Backbone +from annotator.oneformer.detectron2.modeling.postprocessing import sem_seg_postprocess +from annotator.oneformer.detectron2.structures import Boxes, ImageList, Instances, BitMasks +from annotator.oneformer.detectron2.utils.memory import retry_if_cuda_oom + +from .modeling.matcher import HungarianMatcher +from einops import rearrange +from .modeling.transformer_decoder.text_transformer import TextTransformer +from .modeling.transformer_decoder.oneformer_transformer_decoder import MLP +from annotator.oneformer.oneformer.data.tokenizer import SimpleTokenizer, Tokenize + +@META_ARCH_REGISTRY.register() +class OneFormer(nn.Module): + """ + Main class for mask classification semantic segmentation architectures. + """ + + @configurable + def __init__( + self, + *, + backbone: Backbone, + sem_seg_head: nn.Module, + task_mlp: nn.Module, + text_encoder: nn.Module, + text_projector: nn.Module, + prompt_ctx: nn.Embedding, + num_queries: int, + object_mask_threshold: float, + overlap_threshold: float, + metadata, + size_divisibility: int, + sem_seg_postprocess_before_inference: bool, + pixel_mean: Tuple[float], + pixel_std: Tuple[float], + # inference + semantic_on: bool, + panoptic_on: bool, + instance_on: bool, + detection_on: bool, + test_topk_per_image: int, + task_seq_len: int, + max_seq_len: int, + is_demo: bool, + ): + """ + Args: + backbone: a backbone module, must follow detectron2's backbone interface + sem_seg_head: a module that predicts semantic segmentation from backbone features + criterion: a module that defines the loss + num_queries: int, number of queries + object_mask_threshold: float, threshold to filter query based on classification score + for panoptic segmentation inference + overlap_threshold: overlap threshold used in general inference for panoptic segmentation + metadata: dataset meta, get `thing` and `stuff` category names for panoptic + segmentation inference + size_divisibility: Some backbones require the input height and width to be divisible by a + specific integer. We can use this to override such requirement. + sem_seg_postprocess_before_inference: whether to resize the prediction back + to original input size before semantic segmentation inference or after. + For high-resolution dataset like Mapillary, resizing predictions before + inference will cause OOM error. + pixel_mean, pixel_std: list or tuple with #channels element, representing + the per-channel mean and std to be used to normalize the input image + semantic_on: bool, whether to output semantic segmentation prediction + instance_on: bool, whether to output instance segmentation prediction + panoptic_on: bool, whether to output panoptic segmentation prediction + test_topk_per_image: int, instance segmentation parameter, keep topk instances per image + """ + super().__init__() + self.backbone = backbone + self.sem_seg_head = sem_seg_head + self.task_mlp = task_mlp + self.text_encoder = text_encoder + self.text_projector = text_projector + self.prompt_ctx = prompt_ctx + self.num_queries = num_queries + self.overlap_threshold = overlap_threshold + self.object_mask_threshold = object_mask_threshold + self.metadata = metadata + if size_divisibility < 0: + # use backbone size_divisibility if not set + size_divisibility = self.backbone.size_divisibility + self.size_divisibility = size_divisibility + self.sem_seg_postprocess_before_inference = sem_seg_postprocess_before_inference + self.register_buffer("pixel_mean", torch.Tensor(pixel_mean).view(-1, 1, 1), False) + self.register_buffer("pixel_std", torch.Tensor(pixel_std).view(-1, 1, 1), False) + + # additional args + self.semantic_on = semantic_on + self.instance_on = instance_on + self.panoptic_on = panoptic_on + self.detection_on = detection_on + self.test_topk_per_image = test_topk_per_image + + self.text_tokenizer = Tokenize(SimpleTokenizer(), max_seq_len=max_seq_len) + self.task_tokenizer = Tokenize(SimpleTokenizer(), max_seq_len=task_seq_len) + self.is_demo = is_demo + + self.thing_indices = [k for k in self.metadata.thing_dataset_id_to_contiguous_id.keys()] + + if not self.semantic_on: + assert self.sem_seg_postprocess_before_inference + + @classmethod + def from_config(cls, cfg): + backbone = build_backbone(cfg) + sem_seg_head = build_sem_seg_head(cfg, backbone.output_shape()) + + if cfg.MODEL.IS_TRAIN: + text_encoder = TextTransformer(context_length=cfg.MODEL.TEXT_ENCODER.CONTEXT_LENGTH, + width=cfg.MODEL.TEXT_ENCODER.WIDTH, + layers=cfg.MODEL.TEXT_ENCODER.NUM_LAYERS, + vocab_size=cfg.MODEL.TEXT_ENCODER.VOCAB_SIZE) + text_projector = MLP(text_encoder.width, cfg.MODEL.ONE_FORMER.HIDDEN_DIM, + cfg.MODEL.ONE_FORMER.HIDDEN_DIM, cfg.MODEL.TEXT_ENCODER.PROJ_NUM_LAYERS) + if cfg.MODEL.TEXT_ENCODER.N_CTX > 0: + prompt_ctx = nn.Embedding(cfg.MODEL.TEXT_ENCODER.N_CTX, cfg.MODEL.TEXT_ENCODER.WIDTH) + else: + prompt_ctx = None + else: + text_encoder = None + text_projector = None + prompt_ctx = None + + task_mlp = MLP(cfg.INPUT.TASK_SEQ_LEN, cfg.MODEL.ONE_FORMER.HIDDEN_DIM, + cfg.MODEL.ONE_FORMER.HIDDEN_DIM, 2) + + # Loss parameters: + deep_supervision = cfg.MODEL.ONE_FORMER.DEEP_SUPERVISION + no_object_weight = cfg.MODEL.ONE_FORMER.NO_OBJECT_WEIGHT + + # loss weights + class_weight = cfg.MODEL.ONE_FORMER.CLASS_WEIGHT + dice_weight = cfg.MODEL.ONE_FORMER.DICE_WEIGHT + mask_weight = cfg.MODEL.ONE_FORMER.MASK_WEIGHT + contrastive_weight = cfg.MODEL.ONE_FORMER.CONTRASTIVE_WEIGHT + + # building criterion + matcher = HungarianMatcher( + cost_class=class_weight, + cost_mask=mask_weight, + cost_dice=dice_weight, + num_points=cfg.MODEL.ONE_FORMER.TRAIN_NUM_POINTS, + ) + + weight_dict = {"loss_ce": class_weight, "loss_mask": mask_weight, + "loss_dice": dice_weight, "loss_contrastive": contrastive_weight} + + + if deep_supervision: + dec_layers = cfg.MODEL.ONE_FORMER.DEC_LAYERS + aux_weight_dict = {} + for i in range(dec_layers - 1): + aux_weight_dict.update({k + f"_{i}": v for k, v in weight_dict.items()}) + weight_dict.update(aux_weight_dict) + + losses = ["labels", "masks", "contrastive"] + + return { + "backbone": backbone, + "sem_seg_head": sem_seg_head, + "task_mlp": task_mlp, + "prompt_ctx": prompt_ctx, + "text_encoder": text_encoder, + "text_projector": text_projector, + "num_queries": cfg.MODEL.ONE_FORMER.NUM_OBJECT_QUERIES, + "object_mask_threshold": cfg.MODEL.TEST.OBJECT_MASK_THRESHOLD, + "overlap_threshold": cfg.MODEL.TEST.OVERLAP_THRESHOLD, + "metadata": MetadataCatalog.get(cfg.DATASETS.TRAIN[0]), + "size_divisibility": cfg.MODEL.ONE_FORMER.SIZE_DIVISIBILITY, + "sem_seg_postprocess_before_inference": ( + cfg.MODEL.TEST.SEM_SEG_POSTPROCESSING_BEFORE_INFERENCE + or cfg.MODEL.TEST.PANOPTIC_ON + or cfg.MODEL.TEST.INSTANCE_ON + ), + "pixel_mean": cfg.MODEL.PIXEL_MEAN, + "pixel_std": cfg.MODEL.PIXEL_STD, + # inference + "semantic_on": cfg.MODEL.TEST.SEMANTIC_ON, + "instance_on": cfg.MODEL.TEST.INSTANCE_ON, + "panoptic_on": cfg.MODEL.TEST.PANOPTIC_ON, + "detection_on": cfg.MODEL.TEST.DETECTION_ON, + "test_topk_per_image": cfg.TEST.DETECTIONS_PER_IMAGE, + "task_seq_len": cfg.INPUT.TASK_SEQ_LEN, + "max_seq_len": cfg.INPUT.MAX_SEQ_LEN, + "is_demo": cfg.MODEL.IS_DEMO, + } + + @property + def device(self): + return self.pixel_mean.device + + def encode_text(self, text): + assert text.ndim in [2, 3], text.ndim + b = text.shape[0] + squeeze_dim = False + num_text = 1 + if text.ndim == 3: + num_text = text.shape[1] + text = rearrange(text, 'b n l -> (b n) l', n=num_text) + squeeze_dim = True + + # [B, C] + x = self.text_encoder(text) + + text_x = self.text_projector(x) + + if squeeze_dim: + text_x = rearrange(text_x, '(b n) c -> b n c', n=num_text) + if self.prompt_ctx is not None: + text_ctx = self.prompt_ctx.weight.unsqueeze(0).repeat(text_x.shape[0], 1, 1) + text_x = torch.cat([text_x, text_ctx], dim=1) + + return {"texts": text_x} + + def forward(self, batched_inputs): + """ + Args: + batched_inputs: a list, batched outputs of :class:`DatasetMapper`. + Each item in the list contains the inputs for one image. + For now, each item in the list is a dict that contains: + * "image": Tensor, image in (C, H, W) format. + * "instances": per-region ground truth + * Other information that's included in the original dicts, such as: + "height", "width" (int): the output resolution of the model (may be different + from input resolution), used in inference. + Returns: + list[dict]: + each dict has the results for one image. The dict contains the following keys: + * "sem_seg": + A Tensor that represents the + per-pixel segmentation prediced by the head. + The prediction has shape KxHxW that represents the logits of + each class for each pixel. + * "panoptic_seg": + A tuple that represent panoptic output + panoptic_seg (Tensor): of shape (height, width) where the values are ids for each segment. + segments_info (list[dict]): Describe each segment in `panoptic_seg`. + Each dict contains keys "id", "category_id", "isthing". + """ + images = [x["image"].to(self.device) for x in batched_inputs] + images = [(x - self.pixel_mean) / self.pixel_std for x in images] + images = ImageList.from_tensors(images, self.size_divisibility) + + tasks = torch.cat([self.task_tokenizer(x["task"]).to(self.device).unsqueeze(0) for x in batched_inputs], dim=0) + tasks = self.task_mlp(tasks.float()) + + features = self.backbone(images.tensor) + outputs = self.sem_seg_head(features, tasks) + + if self.training: + texts = torch.cat([self.text_tokenizer(x["text"]).to(self.device).unsqueeze(0) for x in batched_inputs], dim=0) + texts_x = self.encode_text(texts) + + outputs = {**outputs, **texts_x} + + # mask classification target + if "instances" in batched_inputs[0]: + gt_instances = [x["instances"].to(self.device) for x in batched_inputs] + targets = self.prepare_targets(gt_instances, images) + else: + targets = None + + # bipartite matching-based loss + losses = self.criterion(outputs, targets) + + for k in list(losses.keys()): + if k in self.criterion.weight_dict: + losses[k] *= self.criterion.weight_dict[k] + else: + # remove this loss if not specified in `weight_dict` + losses.pop(k) + return losses + else: + mask_cls_results = outputs["pred_logits"] + mask_pred_results = outputs["pred_masks"] + # upsample masks + mask_pred_results = F.interpolate( + mask_pred_results, + size=(images.tensor.shape[-2], images.tensor.shape[-1]), + mode="bilinear", + align_corners=False, + ) + + del outputs + + processed_results = [] + for i, data in enumerate(zip( + mask_cls_results, mask_pred_results, batched_inputs, images.image_sizes + )): + mask_cls_result, mask_pred_result, input_per_image, image_size = data + height = input_per_image.get("height", image_size[0]) + width = input_per_image.get("width", image_size[1]) + processed_results.append({}) + + if self.sem_seg_postprocess_before_inference: + mask_pred_result = retry_if_cuda_oom(sem_seg_postprocess)( + mask_pred_result, image_size, height, width + ) + mask_cls_result = mask_cls_result.to(mask_pred_result) + + # semantic segmentation inference + if self.semantic_on: + r = retry_if_cuda_oom(self.semantic_inference)(mask_cls_result, mask_pred_result) + if not self.sem_seg_postprocess_before_inference: + r = retry_if_cuda_oom(sem_seg_postprocess)(r, image_size, height, width) + processed_results[-1]["sem_seg"] = r + + # panoptic segmentation inference + if self.panoptic_on: + panoptic_r = retry_if_cuda_oom(self.panoptic_inference)(mask_cls_result, mask_pred_result) + processed_results[-1]["panoptic_seg"] = panoptic_r + + # instance segmentation inference + if self.instance_on: + instance_r = retry_if_cuda_oom(self.instance_inference)(mask_cls_result, mask_pred_result) + processed_results[-1]["instances"] = instance_r + + if self.detection_on: + bbox_r = retry_if_cuda_oom(self.instance_inference)(mask_cls_result, mask_pred_result) + processed_results[-1]["box_instances"] = bbox_r + + return processed_results + + def prepare_targets(self, targets, images): + h_pad, w_pad = images.tensor.shape[-2:] + new_targets = [] + for targets_per_image in targets: + # pad gt + gt_masks = targets_per_image.gt_masks + padded_masks = torch.zeros((gt_masks.shape[0], h_pad, w_pad), dtype=gt_masks.dtype, device=gt_masks.device) + padded_masks[:, : gt_masks.shape[1], : gt_masks.shape[2]] = gt_masks + new_targets.append( + { + "labels": targets_per_image.gt_classes, + "masks": padded_masks, + } + ) + return new_targets + + def semantic_inference(self, mask_cls, mask_pred): + mask_cls = F.softmax(mask_cls, dim=-1)[..., :-1] + mask_pred = mask_pred.sigmoid() + semseg = torch.einsum("qc,qhw->chw", mask_cls, mask_pred) + return semseg + + def panoptic_inference(self, mask_cls, mask_pred): + scores, labels = F.softmax(mask_cls, dim=-1).max(-1) + mask_pred = mask_pred.sigmoid() + + keep = labels.ne(self.sem_seg_head.num_classes) & (scores > self.object_mask_threshold) + cur_scores = scores[keep] + cur_classes = labels[keep] + cur_masks = mask_pred[keep] + cur_mask_cls = mask_cls[keep] + cur_mask_cls = cur_mask_cls[:, :-1] + + cur_prob_masks = cur_scores.view(-1, 1, 1) * cur_masks + + h, w = cur_masks.shape[-2:] + panoptic_seg = torch.zeros((h, w), dtype=torch.int32, device=cur_masks.device) + segments_info = [] + + current_segment_id = 0 + + if cur_masks.shape[0] == 0: + # We didn't detect any mask :( + return panoptic_seg, segments_info + else: + # take argmax + cur_mask_ids = cur_prob_masks.argmax(0) + stuff_memory_list = {} + for k in range(cur_classes.shape[0]): + pred_class = cur_classes[k].item() + isthing = pred_class in self.metadata.thing_dataset_id_to_contiguous_id.values() + mask_area = (cur_mask_ids == k).sum().item() + original_area = (cur_masks[k] >= 0.5).sum().item() + mask = (cur_mask_ids == k) & (cur_masks[k] >= 0.5) + + if mask_area > 0 and original_area > 0 and mask.sum().item() > 0: + if mask_area / original_area < self.overlap_threshold: + continue + + # merge stuff regions + if not isthing: + if int(pred_class) in stuff_memory_list.keys(): + panoptic_seg[mask] = stuff_memory_list[int(pred_class)] + continue + else: + stuff_memory_list[int(pred_class)] = current_segment_id + 1 + + current_segment_id += 1 + panoptic_seg[mask] = current_segment_id + + segments_info.append( + { + "id": current_segment_id, + "isthing": bool(isthing), + "category_id": int(pred_class), + } + ) + + return panoptic_seg, segments_info + + def instance_inference(self, mask_cls, mask_pred): + # mask_pred is already processed to have the same shape as original input + image_size = mask_pred.shape[-2:] + + # [Q, K] + scores = F.softmax(mask_cls, dim=-1)[:, :-1] + labels = torch.arange(self.sem_seg_head.num_classes, device=self.device).unsqueeze(0).repeat(self.num_queries, 1).flatten(0, 1) + + # scores_per_image, topk_indices = scores.flatten(0, 1).topk(self.num_queries, sorted=False) + scores_per_image, topk_indices = scores.flatten(0, 1).topk(self.test_topk_per_image, sorted=False) + labels_per_image = labels[topk_indices] + + topk_indices = topk_indices // self.sem_seg_head.num_classes + # mask_pred = mask_pred.unsqueeze(1).repeat(1, self.sem_seg_head.num_classes, 1).flatten(0, 1) + mask_pred = mask_pred[topk_indices] + + # Only consider scores with confidence over [self.object_mask_threshold] for demo + if self.is_demo: + keep = scores_per_image > self.object_mask_threshold + scores_per_image = scores_per_image[keep] + labels_per_image = labels_per_image[keep] + mask_pred = mask_pred[keep] + + # if this is panoptic segmentation, we only keep the "thing" classes + if self.panoptic_on: + keep = torch.zeros_like(scores_per_image).bool() + for i, lab in enumerate(labels_per_image): + keep[i] = lab in self.metadata.thing_dataset_id_to_contiguous_id.values() + + scores_per_image = scores_per_image[keep] + labels_per_image = labels_per_image[keep] + mask_pred = mask_pred[keep] + + if 'ade20k' in self.metadata.name: + for i in range(labels_per_image.shape[0]): + labels_per_image[i] = self.thing_indices.index(labels_per_image[i].item()) + + result = Instances(image_size) + # mask (before sigmoid) + result.pred_masks = (mask_pred > 0).float() + if self.detection_on: + # Uncomment the following to get boxes from masks (this is slow) + result.pred_boxes = BitMasks(mask_pred > 0).get_bounding_boxes() + else: + result.pred_boxes = Boxes(torch.zeros(mask_pred.size(0), 4)) + + # calculate average mask prob + mask_scores_per_image = (mask_pred.sigmoid().flatten(1) * result.pred_masks.flatten(1)).sum(1) / (result.pred_masks.flatten(1).sum(1) + 1e-6) + result.scores = scores_per_image * mask_scores_per_image + result.pred_classes = labels_per_image + return result \ No newline at end of file diff --git a/annotator/oneformer/oneformer/utils/__init__.py b/annotator/oneformer/oneformer/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..130d3011b032f91df1a9cf965625e54922f6c81b --- /dev/null +++ b/annotator/oneformer/oneformer/utils/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +from .events import setup_wandb, WandbWriter \ No newline at end of file diff --git a/annotator/oneformer/oneformer/utils/box_ops.py b/annotator/oneformer/oneformer/utils/box_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..a2b62ad99ed1fc35cdb10a9e11acdeb0ff1abcc4 --- /dev/null +++ b/annotator/oneformer/oneformer/utils/box_ops.py @@ -0,0 +1,133 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Utilities for bounding box manipulation and GIoU. +""" +import torch, os +from torchvision.ops.boxes import box_area + + +def box_cxcywh_to_xyxy(x): + x_c, y_c, w, h = x.unbind(-1) + b = [(x_c - 0.5 * w), (y_c - 0.5 * h), + (x_c + 0.5 * w), (y_c + 0.5 * h)] + return torch.stack(b, dim=-1) + + +def box_xyxy_to_cxcywh(x): + x0, y0, x1, y1 = x.unbind(-1) + b = [(x0 + x1) / 2, (y0 + y1) / 2, + (x1 - x0), (y1 - y0)] + return torch.stack(b, dim=-1) + + +# modified from torchvision to also return the union +def box_iou(boxes1, boxes2): + area1 = box_area(boxes1) + area2 = box_area(boxes2) + + # import ipdb; ipdb.set_trace() + lt = torch.max(boxes1[:, None, :2], boxes2[:, :2]) # [N,M,2] + rb = torch.min(boxes1[:, None, 2:], boxes2[:, 2:]) # [N,M,2] + + wh = (rb - lt).clamp(min=0) # [N,M,2] + inter = wh[:, :, 0] * wh[:, :, 1] # [N,M] + + union = area1[:, None] + area2 - inter + + iou = inter / (union + 1e-6) + return iou, union + + +def generalized_box_iou(boxes1, boxes2): + """ + Generalized IoU from https://giou.stanford.edu/ + The boxes should be in [x0, y0, x1, y1] format + Returns a [N, M] pairwise matrix, where N = len(boxes1) + and M = len(boxes2) + """ + # degenerate boxes gives inf / nan results + # so do an early check + assert (boxes1[:, 2:] >= boxes1[:, :2]).all() + assert (boxes2[:, 2:] >= boxes2[:, :2]).all() + # except: + # import ipdb; ipdb.set_trace() + iou, union = box_iou(boxes1, boxes2) + + lt = torch.min(boxes1[:, None, :2], boxes2[:, :2]) + rb = torch.max(boxes1[:, None, 2:], boxes2[:, 2:]) + + wh = (rb - lt).clamp(min=0) # [N,M,2] + area = wh[:, :, 0] * wh[:, :, 1] + + return iou - (area - union) / (area + 1e-6) + + + +# modified from torchvision to also return the union +def box_iou_pairwise(boxes1, boxes2): + area1 = box_area(boxes1) + area2 = box_area(boxes2) + + lt = torch.max(boxes1[:, :2], boxes2[:, :2]) # [N,2] + rb = torch.min(boxes1[:, 2:], boxes2[:, 2:]) # [N,2] + + wh = (rb - lt).clamp(min=0) # [N,2] + inter = wh[:, 0] * wh[:, 1] # [N] + + union = area1 + area2 - inter + + iou = inter / union + return iou, union + + +def generalized_box_iou_pairwise(boxes1, boxes2): + """ + Generalized IoU from https://giou.stanford.edu/ + Input: + - boxes1, boxes2: N,4 + Output: + - giou: N, 4 + """ + # degenerate boxes gives inf / nan results + # so do an early check + assert (boxes1[:, 2:] >= boxes1[:, :2]).all() + assert (boxes2[:, 2:] >= boxes2[:, :2]).all() + assert boxes1.shape == boxes2.shape + iou, union = box_iou_pairwise(boxes1, boxes2) # N, 4 + + lt = torch.min(boxes1[:, :2], boxes2[:, :2]) + rb = torch.max(boxes1[:, 2:], boxes2[:, 2:]) + + wh = (rb - lt).clamp(min=0) # [N,2] + area = wh[:, 0] * wh[:, 1] + + return iou - (area - union) / area + +def masks_to_boxes(masks): + """Compute the bounding boxes around the provided masks + The masks should be in format [N, H, W] where N is the number of masks, (H, W) are the spatial dimensions. + Returns a [N, 4] tensors, with the boxes in xyxy format + """ + if masks.numel() == 0: + return torch.zeros((0, 4), device=masks.device) + + h, w = masks.shape[-2:] + + y = torch.arange(0, h, dtype=torch.float) + x = torch.arange(0, w, dtype=torch.float) + y, x = torch.meshgrid(y, x) + + x_mask = (masks * x.unsqueeze(0)) + x_max = x_mask.flatten(1).max(-1)[0] + x_min = x_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + y_mask = (masks * y.unsqueeze(0)) + y_max = y_mask.flatten(1).max(-1)[0] + y_min = y_mask.masked_fill(~(masks.bool()), 1e8).flatten(1).min(-1)[0] + + return torch.stack([x_min, y_min, x_max, y_max], 1) + +if __name__ == '__main__': + x = torch.rand(5, 4) + y = torch.rand(3, 4) + iou, union = box_iou(x, y) \ No newline at end of file diff --git a/annotator/oneformer/oneformer/utils/events.py b/annotator/oneformer/oneformer/utils/events.py new file mode 100644 index 0000000000000000000000000000000000000000..3c81e8f65ce8cc4428dd2f24d1ff72a424362d5d --- /dev/null +++ b/annotator/oneformer/oneformer/utils/events.py @@ -0,0 +1,120 @@ +import os +import wandb +from annotator.oneformer.detectron2.utils import comm +from annotator.oneformer.detectron2.utils.events import EventWriter, get_event_storage + + +def setup_wandb(cfg, args): + if comm.is_main_process(): + init_args = { + k.lower(): v + for k, v in cfg.WANDB.items() + if isinstance(k, str) and k not in ["config"] + } + # only include most related part to avoid too big table + # TODO: add configurable params to select which part of `cfg` should be saved in config + if "config_exclude_keys" in init_args: + init_args["config"] = cfg + init_args["config"]["cfg_file"] = args.config_file + else: + init_args["config"] = { + "model": cfg.MODEL, + "solver": cfg.SOLVER, + "cfg_file": args.config_file, + } + if ("name" not in init_args) or (init_args["name"] is None): + init_args["name"] = os.path.basename(args.config_file) + else: + init_args["name"] = init_args["name"] + '_' + os.path.basename(args.config_file) + wandb.init(**init_args) + + +class BaseRule(object): + def __call__(self, target): + return target + + +class IsIn(BaseRule): + def __init__(self, keyword: str): + self.keyword = keyword + + def __call__(self, target): + return self.keyword in target + + +class Prefix(BaseRule): + def __init__(self, keyword: str): + self.keyword = keyword + + def __call__(self, target): + return "/".join([self.keyword, target]) + + +class WandbWriter(EventWriter): + """ + Write all scalars to a tensorboard file. + """ + + def __init__(self): + """ + Args: + log_dir (str): the directory to save the output events + kwargs: other arguments passed to `torch.utils.tensorboard.SummaryWriter(...)` + """ + self._last_write = -1 + self._group_rules = [ + (IsIn("/"), BaseRule()), + (IsIn("loss"), Prefix("train")), + ] + + def write(self): + + storage = get_event_storage() + + def _group_name(scalar_name): + for (rule, op) in self._group_rules: + if rule(scalar_name): + return op(scalar_name) + return scalar_name + + stats = { + _group_name(name): scalars[0] + for name, scalars in storage.latest().items() + if scalars[1] > self._last_write + } + if len(stats) > 0: + self._last_write = max([v[1] for k, v in storage.latest().items()]) + + # storage.put_{image,histogram} is only meant to be used by + # tensorboard writer. So we access its internal fields directly from here. + if len(storage._vis_data) >= 1: + stats["image"] = [ + wandb.Image(img, caption=img_name) + for img_name, img, step_num in storage._vis_data + ] + # Storage stores all image data and rely on this writer to clear them. + # As a result it assumes only one writer will use its image data. + # An alternative design is to let storage store limited recent + # data (e.g. only the most recent image) that all writers can access. + # In that case a writer may not see all image data if its period is long. + storage.clear_images() + + if len(storage._histograms) >= 1: + + def create_bar(tag, bucket_limits, bucket_counts, **kwargs): + data = [ + [label, val] for (label, val) in zip(bucket_limits, bucket_counts) + ] + table = wandb.Table(data=data, columns=["label", "value"]) + return wandb.plot.bar(table, "label", "value", title=tag) + + stats["hist"] = [create_bar(**params) for params in storage._histograms] + + storage.clear_histograms() + + if len(stats) == 0: + return + wandb.log(stats, step=storage.iter) + + def close(self): + wandb.finish() \ No newline at end of file diff --git a/annotator/oneformer/oneformer/utils/misc.py b/annotator/oneformer/oneformer/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..f2bca7733278c3a4b1f145bd7e5da23683b74961 --- /dev/null +++ b/annotator/oneformer/oneformer/utils/misc.py @@ -0,0 +1,197 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# Modified by Bowen Cheng from https://github.com/facebookresearch/detr/blob/master/util/misc.py +""" +Misc functions, including distributed helpers. + +Mostly copy-paste from torchvision references. +""" +from typing import List, Optional + +import torch +import torch.distributed as dist +import torchvision +from torch import Tensor +import warnings +import torch.nn.functional as F +import math + +def inverse_sigmoid(x, eps=1e-3): + x = x.clamp(min=0, max=1) + x1 = x.clamp(min=eps) + x2 = (1 - x).clamp(min=eps) + return torch.log(x1/x2) + +def _no_grad_trunc_normal_(tensor, mean, std, a, b): + # Cut & paste from PyTorch official master until it's in a few official releases - RW + # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf + def norm_cdf(x): + # Computes standard normal cumulative distribution function + return (1. + math.erf(x / math.sqrt(2.))) / 2. + + if (mean < a - 2 * std) or (mean > b + 2 * std): + warnings.warn("mean is more than 2 std from [a, b] in nn.init.trunc_normal_. " + "The distribution of values may be incorrect.", + stacklevel=2) + + with torch.no_grad(): + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + l = norm_cdf((a - mean) / std) + u = norm_cdf((b - mean) / std) + + # Uniformly fill tensor with values from [l, u], then translate to + # [2l-1, 2u-1]. + tensor.uniform_(2 * l - 1, 2 * u - 1) + + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() + + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.)) + tensor.add_(mean) + + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor + +def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.): + # type: (Tensor, float, float, float, float) -> Tensor + r"""Fills the input Tensor with values drawn from a truncated + normal distribution. The values are effectively drawn from the + normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)` + with values outside :math:`[a, b]` redrawn until they are within + the bounds. The method used for generating the random values works + best when :math:`a \leq \text{mean} \leq b`. + Args: + tensor: an n-dimensional `torch.Tensor` + mean: the mean of the normal distribution + std: the standard deviation of the normal distribution + a: the minimum cutoff value + b: the maximum cutoff value + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.trunc_normal_(w) + """ + return _no_grad_trunc_normal_(tensor, mean, std, a, b) + +def resize(input, + size=None, + scale_factor=None, + mode='nearest', + align_corners=None, + warning=True): + if warning: + if size is not None and align_corners: + input_h, input_w = tuple(int(x) for x in input.shape[2:]) + output_h, output_w = tuple(int(x) for x in size) + if output_h > input_h or output_w > output_h: + if ((output_h > 1 and output_w > 1 and input_h > 1 + and input_w > 1) and (output_h - 1) % (input_h - 1) + and (output_w - 1) % (input_w - 1)): + warnings.warn( + f'When align_corners={align_corners}, ' + 'the output would more aligned if ' + f'input size {(input_h, input_w)} is `x+1` and ' + f'out size {(output_h, output_w)} is `nx+1`') + if isinstance(size, torch.Size): + size = tuple(int(x) for x in size) + return F.interpolate(input, size, scale_factor, mode, align_corners) + +def _max_by_axis(the_list): + # type: (List[List[int]]) -> List[int] + maxes = the_list[0] + for sublist in the_list[1:]: + for index, item in enumerate(sublist): + maxes[index] = max(maxes[index], item) + return maxes + + +class NestedTensor(object): + def __init__(self, tensors, mask: Optional[Tensor]): + self.tensors = tensors + self.mask = mask + + def to(self, device): + # type: (Device) -> NestedTensor # noqa + cast_tensor = self.tensors.to(device) + mask = self.mask + if mask is not None: + assert mask is not None + cast_mask = mask.to(device) + else: + cast_mask = None + return NestedTensor(cast_tensor, cast_mask) + + def decompose(self): + return self.tensors, self.mask + + def __repr__(self): + return str(self.tensors) + + +def nested_tensor_from_tensor_list(tensor_list: List[Tensor]): + # TODO make this more general + if tensor_list[0].ndim == 3: + if torchvision._is_tracing(): + # nested_tensor_from_tensor_list() does not export well to ONNX + # call _onnx_nested_tensor_from_tensor_list() instead + return _onnx_nested_tensor_from_tensor_list(tensor_list) + + # TODO make it support different-sized images + max_size = _max_by_axis([list(img.shape) for img in tensor_list]) + # min_size = tuple(min(s) for s in zip(*[img.shape for img in tensor_list])) + batch_shape = [len(tensor_list)] + max_size + b, c, h, w = batch_shape + dtype = tensor_list[0].dtype + device = tensor_list[0].device + tensor = torch.zeros(batch_shape, dtype=dtype, device=device) + mask = torch.ones((b, h, w), dtype=torch.bool, device=device) + for img, pad_img, m in zip(tensor_list, tensor, mask): + pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + m[: img.shape[1], : img.shape[2]] = False + else: + raise ValueError("not supported") + return NestedTensor(tensor, mask) + + +# _onnx_nested_tensor_from_tensor_list() is an implementation of +# nested_tensor_from_tensor_list() that is supported by ONNX tracing. +@torch.jit.unused +def _onnx_nested_tensor_from_tensor_list(tensor_list: List[Tensor]) -> NestedTensor: + max_size = [] + for i in range(tensor_list[0].dim()): + max_size_i = torch.max( + torch.stack([img.shape[i] for img in tensor_list]).to(torch.float32) + ).to(torch.int64) + max_size.append(max_size_i) + max_size = tuple(max_size) + + # work around for + # pad_img[: img.shape[0], : img.shape[1], : img.shape[2]].copy_(img) + # m[: img.shape[1], :img.shape[2]] = False + # which is not yet supported in onnx + padded_imgs = [] + padded_masks = [] + for img in tensor_list: + padding = [(s1 - s2) for s1, s2 in zip(max_size, tuple(img.shape))] + padded_img = torch.nn.functional.pad(img, (0, padding[2], 0, padding[1], 0, padding[0])) + padded_imgs.append(padded_img) + + m = torch.zeros_like(img[0], dtype=torch.int, device=img.device) + padded_mask = torch.nn.functional.pad(m, (0, padding[2], 0, padding[1]), "constant", 1) + padded_masks.append(padded_mask.to(torch.bool)) + + tensor = torch.stack(padded_imgs) + mask = torch.stack(padded_masks) + + return NestedTensor(tensor, mask=mask) + + +def is_dist_avail_and_initialized(): + if not dist.is_available(): + return False + if not dist.is_initialized(): + return False + return True diff --git a/annotator/oneformer/oneformer/utils/pos_embed.py b/annotator/oneformer/oneformer/utils/pos_embed.py new file mode 100644 index 0000000000000000000000000000000000000000..aa11d60db65fa98c140e7d75bdf985ff7ece8f18 --- /dev/null +++ b/annotator/oneformer/oneformer/utils/pos_embed.py @@ -0,0 +1,122 @@ +# -------------------------------------------------------- +# Position embedding utils +# -------------------------------------------------------- + +from typing import Tuple + +import numpy as np +import torch + + +# -------------------------------------------------------- +# 2D sine-cosine position embedding +# References: +# Transformer: https://github.com/tensorflow/models/blob/master/official/nlp/transformer/model_utils.py +# MoCo v3: https://github.com/facebookresearch/moco-v3 +# -------------------------------------------------------- +def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False): + """ + grid_size: int of the grid height and width + return: + pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token) + """ + grid_h = np.arange(grid_size, dtype=np.float32) + grid_w = np.arange(grid_size, dtype=np.float32) + grid = np.meshgrid(grid_w, grid_h) # here w goes first + grid = np.stack(grid, axis=0) + + grid = grid.reshape([2, 1, grid_size, grid_size]) + pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid) + if cls_token: + pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0) + return pos_embed + + +def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): + assert embed_dim % 2 == 0 + + # use half of dimensions to encode grid_h + emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2) + emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2) + + emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D) + return emb + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + """ + embed_dim: output dimension for each position + pos: a list of positions to be encoded: size (M,) + out: (M, D) + """ + assert embed_dim % 2 == 0 + omega = np.arange(embed_dim // 2, dtype=np.float) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000 ** omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product + + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) + return emb + + +# -------------------------------------------------------- +# Interpolate position embeddings for high-resolution +# References: +# DeiT: https://github.com/facebookresearch/deit +# -------------------------------------------------------- +def interpolate_pos_embed(model, checkpoint_model, pos_embed_key): + if pos_embed_key in checkpoint_model: + pos_embed_checkpoint = checkpoint_model[pos_embed_key] + embedding_size = pos_embed_checkpoint.shape[-1] + num_patches = model.num_patches + if pos_embed_key.startswith("decoder"): + num_extra_tokens = model.decoder_pos_embed.shape[-2] - num_patches + else: + num_extra_tokens = model.pos_embed.shape[-2] - num_patches + # height (== width) for the checkpoint position embedding + orig_size = int((pos_embed_checkpoint.shape[-2] - num_extra_tokens) ** 0.5) + # height (== width) for the new position embedding + new_size = int(num_patches ** 0.5) + # class_token and dist_token are kept unchanged + if orig_size != new_size: + print( + "Position interpolate from %dx%d to %dx%d" + % (orig_size, orig_size, new_size, new_size) + ) + extra_tokens = pos_embed_checkpoint[:, :num_extra_tokens] + # only the position tokens are interpolated + pos_tokens = pos_embed_checkpoint[:, num_extra_tokens:] + pos_tokens = pos_tokens.reshape( + -1, orig_size, orig_size, embedding_size + ).permute(0, 3, 1, 2) + pos_tokens = torch.nn.functional.interpolate( + pos_tokens, + size=(new_size, new_size), + mode="bicubic", + align_corners=False, + ) + pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2) + new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1) + checkpoint_model[pos_embed_key] = new_pos_embed + + +def interpolate_pos_embed_online( + pos_embed, orig_size: Tuple[int], new_size: Tuple[int], num_extra_tokens: int +): + extra_tokens = pos_embed[:, :num_extra_tokens] + pos_tokens = pos_embed[:, num_extra_tokens:] + embedding_size = pos_tokens.shape[-1] + pos_tokens = pos_tokens.reshape( + -1, orig_size[0], orig_size[1], embedding_size + ).permute(0, 3, 1, 2) + pos_tokens = torch.nn.functional.interpolate( + pos_tokens, size=new_size, mode="bicubic", align_corners=False, + ) + pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(1, 2) + new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=1) + return new_pos_embed diff --git a/annotator/oneformer/pycocotools/__init__.py b/annotator/oneformer/pycocotools/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3f7d85bba884ea8f83fc6ab2a1e6ade80d98d4d9 --- /dev/null +++ b/annotator/oneformer/pycocotools/__init__.py @@ -0,0 +1 @@ +__author__ = 'tylin' diff --git a/annotator/oneformer/pycocotools/coco.py b/annotator/oneformer/pycocotools/coco.py new file mode 100644 index 0000000000000000000000000000000000000000..1ecb6c1b00325b3073c67dd5081bd514f568ece5 --- /dev/null +++ b/annotator/oneformer/pycocotools/coco.py @@ -0,0 +1,444 @@ +__author__ = 'tylin' +__version__ = '2.0' +# Interface for accessing the Microsoft COCO dataset. + +# Microsoft COCO is a large image dataset designed for object detection, +# segmentation, and caption generation. annotator.oneformer.pycocotools is a Python API that +# assists in loading, parsing and visualizing the annotations in COCO. +# Please visit http://mscoco.org/ for more information on COCO, including +# for the data, paper, and tutorials. The exact format of the annotations +# is also described on the COCO website. For example usage of the annotator.oneformer.pycocotools +# please see annotator.oneformer.pycocotools_demo.ipynb. In addition to this API, please download both +# the COCO images and annotations in order to run the demo. + +# An alternative to using the API is to load the annotations directly +# into Python dictionary +# Using the API provides additional utility functions. Note that this API +# supports both *instance* and *caption* annotations. In the case of +# captions not all functions are defined (e.g. categories are undefined). + +# The following API functions are defined: +# COCO - COCO api class that loads COCO annotation file and prepare data structures. +# decodeMask - Decode binary mask M encoded via run-length encoding. +# encodeMask - Encode binary mask M using run-length encoding. +# getAnnIds - Get ann ids that satisfy given filter conditions. +# getCatIds - Get cat ids that satisfy given filter conditions. +# getImgIds - Get img ids that satisfy given filter conditions. +# loadAnns - Load anns with the specified ids. +# loadCats - Load cats with the specified ids. +# loadImgs - Load imgs with the specified ids. +# annToMask - Convert segmentation in an annotation to binary mask. +# showAnns - Display the specified annotations. +# loadRes - Load algorithm results and create API for accessing them. +# download - Download COCO images from mscoco.org server. +# Throughout the API "ann"=annotation, "cat"=category, and "img"=image. +# Help on each functions can be accessed by: "help COCO>function". + +# See also COCO>decodeMask, +# COCO>encodeMask, COCO>getAnnIds, COCO>getCatIds, +# COCO>getImgIds, COCO>loadAnns, COCO>loadCats, +# COCO>loadImgs, COCO>annToMask, COCO>showAnns + +# Microsoft COCO Toolbox. version 2.0 +# Data, paper, and tutorials available at: http://mscoco.org/ +# Code written by Piotr Dollar and Tsung-Yi Lin, 2014. +# Licensed under the Simplified BSD License [see bsd.txt] + +import json +import time +import numpy as np +import copy +import itertools +from . import mask as maskUtils +import os +from collections import defaultdict +import sys +PYTHON_VERSION = sys.version_info[0] +if PYTHON_VERSION == 2: + from urllib import urlretrieve +elif PYTHON_VERSION == 3: + from urllib.request import urlretrieve + + +def _isArrayLike(obj): + return hasattr(obj, '__iter__') and hasattr(obj, '__len__') + + +class COCO: + def __init__(self, annotation_file=None): + """ + Constructor of Microsoft COCO helper class for reading and visualizing annotations. + :param annotation_file (str): location of annotation file + :param image_folder (str): location to the folder that hosts images. + :return: + """ + # load dataset + self.dataset,self.anns,self.cats,self.imgs = dict(),dict(),dict(),dict() + self.imgToAnns, self.catToImgs = defaultdict(list), defaultdict(list) + if not annotation_file == None: + print('loading annotations into memory...') + tic = time.time() + with open(annotation_file, 'r') as f: + dataset = json.load(f) + assert type(dataset)==dict, 'annotation file format {} not supported'.format(type(dataset)) + print('Done (t={:0.2f}s)'.format(time.time()- tic)) + self.dataset = dataset + self.createIndex() + + def createIndex(self): + # create index + print('creating index...') + anns, cats, imgs = {}, {}, {} + imgToAnns,catToImgs = defaultdict(list),defaultdict(list) + if 'annotations' in self.dataset: + for ann in self.dataset['annotations']: + imgToAnns[ann['image_id']].append(ann) + anns[ann['id']] = ann + + if 'images' in self.dataset: + for img in self.dataset['images']: + imgs[img['id']] = img + + if 'categories' in self.dataset: + for cat in self.dataset['categories']: + cats[cat['id']] = cat + + if 'annotations' in self.dataset and 'categories' in self.dataset: + for ann in self.dataset['annotations']: + catToImgs[ann['category_id']].append(ann['image_id']) + + print('index created!') + + # create class members + self.anns = anns + self.imgToAnns = imgToAnns + self.catToImgs = catToImgs + self.imgs = imgs + self.cats = cats + + def info(self): + """ + Print information about the annotation file. + :return: + """ + for key, value in self.dataset['info'].items(): + print('{}: {}'.format(key, value)) + + def getAnnIds(self, imgIds=[], catIds=[], areaRng=[], iscrowd=None): + """ + Get ann ids that satisfy given filter conditions. default skips that filter + :param imgIds (int array) : get anns for given imgs + catIds (int array) : get anns for given cats + areaRng (float array) : get anns for given area range (e.g. [0 inf]) + iscrowd (boolean) : get anns for given crowd label (False or True) + :return: ids (int array) : integer array of ann ids + """ + imgIds = imgIds if _isArrayLike(imgIds) else [imgIds] + catIds = catIds if _isArrayLike(catIds) else [catIds] + + if len(imgIds) == len(catIds) == len(areaRng) == 0: + anns = self.dataset['annotations'] + else: + if not len(imgIds) == 0: + lists = [self.imgToAnns[imgId] for imgId in imgIds if imgId in self.imgToAnns] + anns = list(itertools.chain.from_iterable(lists)) + else: + anns = self.dataset['annotations'] + anns = anns if len(catIds) == 0 else [ann for ann in anns if ann['category_id'] in catIds] + anns = anns if len(areaRng) == 0 else [ann for ann in anns if ann['area'] > areaRng[0] and ann['area'] < areaRng[1]] + if not iscrowd == None: + ids = [ann['id'] for ann in anns if ann['iscrowd'] == iscrowd] + else: + ids = [ann['id'] for ann in anns] + return ids + + def getCatIds(self, catNms=[], supNms=[], catIds=[]): + """ + filtering parameters. default skips that filter. + :param catNms (str array) : get cats for given cat names + :param supNms (str array) : get cats for given supercategory names + :param catIds (int array) : get cats for given cat ids + :return: ids (int array) : integer array of cat ids + """ + catNms = catNms if _isArrayLike(catNms) else [catNms] + supNms = supNms if _isArrayLike(supNms) else [supNms] + catIds = catIds if _isArrayLike(catIds) else [catIds] + + if len(catNms) == len(supNms) == len(catIds) == 0: + cats = self.dataset['categories'] + else: + cats = self.dataset['categories'] + cats = cats if len(catNms) == 0 else [cat for cat in cats if cat['name'] in catNms] + cats = cats if len(supNms) == 0 else [cat for cat in cats if cat['supercategory'] in supNms] + cats = cats if len(catIds) == 0 else [cat for cat in cats if cat['id'] in catIds] + ids = [cat['id'] for cat in cats] + return ids + + def getImgIds(self, imgIds=[], catIds=[]): + ''' + Get img ids that satisfy given filter conditions. + :param imgIds (int array) : get imgs for given ids + :param catIds (int array) : get imgs with all given cats + :return: ids (int array) : integer array of img ids + ''' + imgIds = imgIds if _isArrayLike(imgIds) else [imgIds] + catIds = catIds if _isArrayLike(catIds) else [catIds] + + if len(imgIds) == len(catIds) == 0: + ids = self.imgs.keys() + else: + ids = set(imgIds) + for i, catId in enumerate(catIds): + if i == 0 and len(ids) == 0: + ids = set(self.catToImgs[catId]) + else: + ids &= set(self.catToImgs[catId]) + return list(ids) + + def loadAnns(self, ids=[]): + """ + Load anns with the specified ids. + :param ids (int array) : integer ids specifying anns + :return: anns (object array) : loaded ann objects + """ + if _isArrayLike(ids): + return [self.anns[id] for id in ids] + elif type(ids) == int: + return [self.anns[ids]] + + def loadCats(self, ids=[]): + """ + Load cats with the specified ids. + :param ids (int array) : integer ids specifying cats + :return: cats (object array) : loaded cat objects + """ + if _isArrayLike(ids): + return [self.cats[id] for id in ids] + elif type(ids) == int: + return [self.cats[ids]] + + def loadImgs(self, ids=[]): + """ + Load anns with the specified ids. + :param ids (int array) : integer ids specifying img + :return: imgs (object array) : loaded img objects + """ + if _isArrayLike(ids): + return [self.imgs[id] for id in ids] + elif type(ids) == int: + return [self.imgs[ids]] + + def showAnns(self, anns, draw_bbox=False): + """ + Display the specified annotations. + :param anns (array of object): annotations to display + :return: None + """ + if len(anns) == 0: + return 0 + if 'segmentation' in anns[0] or 'keypoints' in anns[0]: + datasetType = 'instances' + elif 'caption' in anns[0]: + datasetType = 'captions' + else: + raise Exception('datasetType not supported') + if datasetType == 'instances': + import matplotlib.pyplot as plt + from matplotlib.collections import PatchCollection + from matplotlib.patches import Polygon + + ax = plt.gca() + ax.set_autoscale_on(False) + polygons = [] + color = [] + for ann in anns: + c = (np.random.random((1, 3))*0.6+0.4).tolist()[0] + if 'segmentation' in ann: + if type(ann['segmentation']) == list: + # polygon + for seg in ann['segmentation']: + poly = np.array(seg).reshape((int(len(seg)/2), 2)) + polygons.append(Polygon(poly)) + color.append(c) + else: + # mask + t = self.imgs[ann['image_id']] + if type(ann['segmentation']['counts']) == list: + rle = maskUtils.frPyObjects([ann['segmentation']], t['height'], t['width']) + else: + rle = [ann['segmentation']] + m = maskUtils.decode(rle) + img = np.ones( (m.shape[0], m.shape[1], 3) ) + if ann['iscrowd'] == 1: + color_mask = np.array([2.0,166.0,101.0])/255 + if ann['iscrowd'] == 0: + color_mask = np.random.random((1, 3)).tolist()[0] + for i in range(3): + img[:,:,i] = color_mask[i] + ax.imshow(np.dstack( (img, m*0.5) )) + if 'keypoints' in ann and type(ann['keypoints']) == list: + # turn skeleton into zero-based index + sks = np.array(self.loadCats(ann['category_id'])[0]['skeleton'])-1 + kp = np.array(ann['keypoints']) + x = kp[0::3] + y = kp[1::3] + v = kp[2::3] + for sk in sks: + if np.all(v[sk]>0): + plt.plot(x[sk],y[sk], linewidth=3, color=c) + plt.plot(x[v>0], y[v>0],'o',markersize=8, markerfacecolor=c, markeredgecolor='k',markeredgewidth=2) + plt.plot(x[v>1], y[v>1],'o',markersize=8, markerfacecolor=c, markeredgecolor=c, markeredgewidth=2) + + if draw_bbox: + [bbox_x, bbox_y, bbox_w, bbox_h] = ann['bbox'] + poly = [[bbox_x, bbox_y], [bbox_x, bbox_y+bbox_h], [bbox_x+bbox_w, bbox_y+bbox_h], [bbox_x+bbox_w, bbox_y]] + np_poly = np.array(poly).reshape((4,2)) + polygons.append(Polygon(np_poly)) + color.append(c) + + p = PatchCollection(polygons, facecolor=color, linewidths=0, alpha=0.4) + ax.add_collection(p) + p = PatchCollection(polygons, facecolor='none', edgecolors=color, linewidths=2) + ax.add_collection(p) + elif datasetType == 'captions': + for ann in anns: + print(ann['caption']) + + def loadRes(self, resFile): + """ + Load result file and return a result api object. + :param resFile (str) : file name of result file + :return: res (obj) : result api object + """ + res = COCO() + res.dataset['images'] = [img for img in self.dataset['images']] + + print('Loading and preparing results...') + tic = time.time() + if type(resFile) == str or (PYTHON_VERSION == 2 and type(resFile) == unicode): + with open(resFile) as f: + anns = json.load(f) + elif type(resFile) == np.ndarray: + anns = self.loadNumpyAnnotations(resFile) + else: + anns = resFile + assert type(anns) == list, 'results in not an array of objects' + annsImgIds = [ann['image_id'] for ann in anns] + assert set(annsImgIds) == (set(annsImgIds) & set(self.getImgIds())), \ + 'Results do not correspond to current coco set' + if 'caption' in anns[0]: + imgIds = set([img['id'] for img in res.dataset['images']]) & set([ann['image_id'] for ann in anns]) + res.dataset['images'] = [img for img in res.dataset['images'] if img['id'] in imgIds] + for id, ann in enumerate(anns): + ann['id'] = id+1 + elif 'bbox' in anns[0] and not anns[0]['bbox'] == []: + res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) + for id, ann in enumerate(anns): + bb = ann['bbox'] + x1, x2, y1, y2 = [bb[0], bb[0]+bb[2], bb[1], bb[1]+bb[3]] + if not 'segmentation' in ann: + ann['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]] + ann['area'] = bb[2]*bb[3] + ann['id'] = id+1 + ann['iscrowd'] = 0 + elif 'segmentation' in anns[0]: + res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) + for id, ann in enumerate(anns): + # now only support compressed RLE format as segmentation results + ann['area'] = maskUtils.area(ann['segmentation']) + if not 'bbox' in ann: + ann['bbox'] = maskUtils.toBbox(ann['segmentation']) + ann['id'] = id+1 + ann['iscrowd'] = 0 + elif 'keypoints' in anns[0]: + res.dataset['categories'] = copy.deepcopy(self.dataset['categories']) + for id, ann in enumerate(anns): + s = ann['keypoints'] + x = s[0::3] + y = s[1::3] + x0,x1,y0,y1 = np.min(x), np.max(x), np.min(y), np.max(y) + ann['area'] = (x1-x0)*(y1-y0) + ann['id'] = id + 1 + ann['bbox'] = [x0,y0,x1-x0,y1-y0] + print('DONE (t={:0.2f}s)'.format(time.time()- tic)) + + res.dataset['annotations'] = anns + res.createIndex() + return res + + def download(self, tarDir = None, imgIds = [] ): + ''' + Download COCO images from mscoco.org server. + :param tarDir (str): COCO results directory name + imgIds (list): images to be downloaded + :return: + ''' + if tarDir is None: + print('Please specify target directory') + return -1 + if len(imgIds) == 0: + imgs = self.imgs.values() + else: + imgs = self.loadImgs(imgIds) + N = len(imgs) + if not os.path.exists(tarDir): + os.makedirs(tarDir) + for i, img in enumerate(imgs): + tic = time.time() + fname = os.path.join(tarDir, img['file_name']) + if not os.path.exists(fname): + urlretrieve(img['coco_url'], fname) + print('downloaded {}/{} images (t={:0.1f}s)'.format(i, N, time.time()- tic)) + + def loadNumpyAnnotations(self, data): + """ + Convert result data from a numpy array [Nx7] where each row contains {imageID,x1,y1,w,h,score,class} + :param data (numpy.ndarray) + :return: annotations (python nested list) + """ + print('Converting ndarray to lists...') + assert(type(data) == np.ndarray) + print(data.shape) + assert(data.shape[1] == 7) + N = data.shape[0] + ann = [] + for i in range(N): + if i % 1000000 == 0: + print('{}/{}'.format(i,N)) + ann += [{ + 'image_id' : int(data[i, 0]), + 'bbox' : [ data[i, 1], data[i, 2], data[i, 3], data[i, 4] ], + 'score' : data[i, 5], + 'category_id': int(data[i, 6]), + }] + return ann + + def annToRLE(self, ann): + """ + Convert annotation which can be polygons, uncompressed RLE to RLE. + :return: binary mask (numpy 2D array) + """ + t = self.imgs[ann['image_id']] + h, w = t['height'], t['width'] + segm = ann['segmentation'] + if type(segm) == list: + # polygon -- a single object might consist of multiple parts + # we merge all parts into one mask rle code + rles = maskUtils.frPyObjects(segm, h, w) + rle = maskUtils.merge(rles) + elif type(segm['counts']) == list: + # uncompressed RLE + rle = maskUtils.frPyObjects(segm, h, w) + else: + # rle + rle = ann['segmentation'] + return rle + + def annToMask(self, ann): + """ + Convert annotation which can be polygons, uncompressed RLE, or RLE to binary mask. + :return: binary mask (numpy 2D array) + """ + rle = self.annToRLE(ann) + m = maskUtils.decode(rle) + return m diff --git a/annotator/oneformer/pycocotools/cocoeval.py b/annotator/oneformer/pycocotools/cocoeval.py new file mode 100644 index 0000000000000000000000000000000000000000..89c251e1652a0cfc7e8ff1bbb1024a801ed2ebe7 --- /dev/null +++ b/annotator/oneformer/pycocotools/cocoeval.py @@ -0,0 +1,534 @@ +__author__ = 'tsungyi' + +import numpy as np +import datetime +import time +from collections import defaultdict +from . import mask as maskUtils +import copy + +class COCOeval: + # Interface for evaluating detection on the Microsoft COCO dataset. + # + # The usage for CocoEval is as follows: + # cocoGt=..., cocoDt=... # load dataset and results + # E = CocoEval(cocoGt,cocoDt); # initialize CocoEval object + # E.params.recThrs = ...; # set parameters as desired + # E.evaluate(); # run per image evaluation + # E.accumulate(); # accumulate per image results + # E.summarize(); # display summary metrics of results + # For example usage see evalDemo.m and http://mscoco.org/. + # + # The evaluation parameters are as follows (defaults in brackets): + # imgIds - [all] N img ids to use for evaluation + # catIds - [all] K cat ids to use for evaluation + # iouThrs - [.5:.05:.95] T=10 IoU thresholds for evaluation + # recThrs - [0:.01:1] R=101 recall thresholds for evaluation + # areaRng - [...] A=4 object area ranges for evaluation + # maxDets - [1 10 100] M=3 thresholds on max detections per image + # iouType - ['segm'] set iouType to 'segm', 'bbox' or 'keypoints' + # iouType replaced the now DEPRECATED useSegm parameter. + # useCats - [1] if true use category labels for evaluation + # Note: if useCats=0 category labels are ignored as in proposal scoring. + # Note: multiple areaRngs [Ax2] and maxDets [Mx1] can be specified. + # + # evaluate(): evaluates detections on every image and every category and + # concats the results into the "evalImgs" with fields: + # dtIds - [1xD] id for each of the D detections (dt) + # gtIds - [1xG] id for each of the G ground truths (gt) + # dtMatches - [TxD] matching gt id at each IoU or 0 + # gtMatches - [TxG] matching dt id at each IoU or 0 + # dtScores - [1xD] confidence of each dt + # gtIgnore - [1xG] ignore flag for each gt + # dtIgnore - [TxD] ignore flag for each dt at each IoU + # + # accumulate(): accumulates the per-image, per-category evaluation + # results in "evalImgs" into the dictionary "eval" with fields: + # params - parameters used for evaluation + # date - date evaluation was performed + # counts - [T,R,K,A,M] parameter dimensions (see above) + # precision - [TxRxKxAxM] precision for every evaluation setting + # recall - [TxKxAxM] max recall for every evaluation setting + # Note: precision and recall==-1 for settings with no gt objects. + # + # See also coco, mask, pycocoDemo, pycocoEvalDemo + # + # Microsoft COCO Toolbox. version 2.0 + # Data, paper, and tutorials available at: http://mscoco.org/ + # Code written by Piotr Dollar and Tsung-Yi Lin, 2015. + # Licensed under the Simplified BSD License [see coco/license.txt] + def __init__(self, cocoGt=None, cocoDt=None, iouType='segm'): + ''' + Initialize CocoEval using coco APIs for gt and dt + :param cocoGt: coco object with ground truth annotations + :param cocoDt: coco object with detection results + :return: None + ''' + if not iouType: + print('iouType not specified. use default iouType segm') + self.cocoGt = cocoGt # ground truth COCO API + self.cocoDt = cocoDt # detections COCO API + self.evalImgs = defaultdict(list) # per-image per-category evaluation results [KxAxI] elements + self.eval = {} # accumulated evaluation results + self._gts = defaultdict(list) # gt for evaluation + self._dts = defaultdict(list) # dt for evaluation + self.params = Params(iouType=iouType) # parameters + self._paramsEval = {} # parameters for evaluation + self.stats = [] # result summarization + self.ious = {} # ious between all gts and dts + if not cocoGt is None: + self.params.imgIds = sorted(cocoGt.getImgIds()) + self.params.catIds = sorted(cocoGt.getCatIds()) + + + def _prepare(self): + ''' + Prepare ._gts and ._dts for evaluation based on params + :return: None + ''' + def _toMask(anns, coco): + # modify ann['segmentation'] by reference + for ann in anns: + rle = coco.annToRLE(ann) + ann['segmentation'] = rle + p = self.params + if p.useCats: + gts=self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds, catIds=p.catIds)) + dts=self.cocoDt.loadAnns(self.cocoDt.getAnnIds(imgIds=p.imgIds, catIds=p.catIds)) + else: + gts=self.cocoGt.loadAnns(self.cocoGt.getAnnIds(imgIds=p.imgIds)) + dts=self.cocoDt.loadAnns(self.cocoDt.getAnnIds(imgIds=p.imgIds)) + + # convert ground truth to mask if iouType == 'segm' + if p.iouType == 'segm': + _toMask(gts, self.cocoGt) + _toMask(dts, self.cocoDt) + # set ignore flag + for gt in gts: + gt['ignore'] = gt['ignore'] if 'ignore' in gt else 0 + gt['ignore'] = 'iscrowd' in gt and gt['iscrowd'] + if p.iouType == 'keypoints': + gt['ignore'] = (gt['num_keypoints'] == 0) or gt['ignore'] + self._gts = defaultdict(list) # gt for evaluation + self._dts = defaultdict(list) # dt for evaluation + for gt in gts: + self._gts[gt['image_id'], gt['category_id']].append(gt) + for dt in dts: + self._dts[dt['image_id'], dt['category_id']].append(dt) + self.evalImgs = defaultdict(list) # per-image per-category evaluation results + self.eval = {} # accumulated evaluation results + + def evaluate(self): + ''' + Run per image evaluation on given images and store results (a list of dict) in self.evalImgs + :return: None + ''' + tic = time.time() + print('Running per image evaluation...') + p = self.params + # add backward compatibility if useSegm is specified in params + if not p.useSegm is None: + p.iouType = 'segm' if p.useSegm == 1 else 'bbox' + print('useSegm (deprecated) is not None. Running {} evaluation'.format(p.iouType)) + print('Evaluate annotation type *{}*'.format(p.iouType)) + p.imgIds = list(np.unique(p.imgIds)) + if p.useCats: + p.catIds = list(np.unique(p.catIds)) + p.maxDets = sorted(p.maxDets) + self.params=p + + self._prepare() + # loop through images, area range, max detection number + catIds = p.catIds if p.useCats else [-1] + + if p.iouType == 'segm' or p.iouType == 'bbox': + computeIoU = self.computeIoU + elif p.iouType == 'keypoints': + computeIoU = self.computeOks + self.ious = {(imgId, catId): computeIoU(imgId, catId) \ + for imgId in p.imgIds + for catId in catIds} + + evaluateImg = self.evaluateImg + maxDet = p.maxDets[-1] + self.evalImgs = [evaluateImg(imgId, catId, areaRng, maxDet) + for catId in catIds + for areaRng in p.areaRng + for imgId in p.imgIds + ] + self._paramsEval = copy.deepcopy(self.params) + toc = time.time() + print('DONE (t={:0.2f}s).'.format(toc-tic)) + + def computeIoU(self, imgId, catId): + p = self.params + if p.useCats: + gt = self._gts[imgId,catId] + dt = self._dts[imgId,catId] + else: + gt = [_ for cId in p.catIds for _ in self._gts[imgId,cId]] + dt = [_ for cId in p.catIds for _ in self._dts[imgId,cId]] + if len(gt) == 0 and len(dt) ==0: + return [] + inds = np.argsort([-d['score'] for d in dt], kind='mergesort') + dt = [dt[i] for i in inds] + if len(dt) > p.maxDets[-1]: + dt=dt[0:p.maxDets[-1]] + + if p.iouType == 'segm': + g = [g['segmentation'] for g in gt] + d = [d['segmentation'] for d in dt] + elif p.iouType == 'bbox': + g = [g['bbox'] for g in gt] + d = [d['bbox'] for d in dt] + else: + raise Exception('unknown iouType for iou computation') + + # compute iou between each dt and gt region + iscrowd = [int(o['iscrowd']) for o in gt] + ious = maskUtils.iou(d,g,iscrowd) + return ious + + def computeOks(self, imgId, catId): + p = self.params + # dimention here should be Nxm + gts = self._gts[imgId, catId] + dts = self._dts[imgId, catId] + inds = np.argsort([-d['score'] for d in dts], kind='mergesort') + dts = [dts[i] for i in inds] + if len(dts) > p.maxDets[-1]: + dts = dts[0:p.maxDets[-1]] + # if len(gts) == 0 and len(dts) == 0: + if len(gts) == 0 or len(dts) == 0: + return [] + ious = np.zeros((len(dts), len(gts))) + sigmas = p.kpt_oks_sigmas + vars = (sigmas * 2)**2 + k = len(sigmas) + # compute oks between each detection and ground truth object + for j, gt in enumerate(gts): + # create bounds for ignore regions(double the gt bbox) + g = np.array(gt['keypoints']) + xg = g[0::3]; yg = g[1::3]; vg = g[2::3] + k1 = np.count_nonzero(vg > 0) + bb = gt['bbox'] + x0 = bb[0] - bb[2]; x1 = bb[0] + bb[2] * 2 + y0 = bb[1] - bb[3]; y1 = bb[1] + bb[3] * 2 + for i, dt in enumerate(dts): + d = np.array(dt['keypoints']) + xd = d[0::3]; yd = d[1::3] + if k1>0: + # measure the per-keypoint distance if keypoints visible + dx = xd - xg + dy = yd - yg + else: + # measure minimum distance to keypoints in (x0,y0) & (x1,y1) + z = np.zeros((k)) + dx = np.max((z, x0-xd),axis=0)+np.max((z, xd-x1),axis=0) + dy = np.max((z, y0-yd),axis=0)+np.max((z, yd-y1),axis=0) + e = (dx**2 + dy**2) / vars / (gt['area']+np.spacing(1)) / 2 + if k1 > 0: + e=e[vg > 0] + ious[i, j] = np.sum(np.exp(-e)) / e.shape[0] + return ious + + def evaluateImg(self, imgId, catId, aRng, maxDet): + ''' + perform evaluation for single category and image + :return: dict (single image results) + ''' + p = self.params + if p.useCats: + gt = self._gts[imgId,catId] + dt = self._dts[imgId,catId] + else: + gt = [_ for cId in p.catIds for _ in self._gts[imgId,cId]] + dt = [_ for cId in p.catIds for _ in self._dts[imgId,cId]] + if len(gt) == 0 and len(dt) ==0: + return None + + for g in gt: + if g['ignore'] or (g['area']aRng[1]): + g['_ignore'] = 1 + else: + g['_ignore'] = 0 + + # sort dt highest score first, sort gt ignore last + gtind = np.argsort([g['_ignore'] for g in gt], kind='mergesort') + gt = [gt[i] for i in gtind] + dtind = np.argsort([-d['score'] for d in dt], kind='mergesort') + dt = [dt[i] for i in dtind[0:maxDet]] + iscrowd = [int(o['iscrowd']) for o in gt] + # load computed ious + ious = self.ious[imgId, catId][:, gtind] if len(self.ious[imgId, catId]) > 0 else self.ious[imgId, catId] + + T = len(p.iouThrs) + G = len(gt) + D = len(dt) + gtm = np.zeros((T,G)) + dtm = np.zeros((T,D)) + gtIg = np.array([g['_ignore'] for g in gt]) + dtIg = np.zeros((T,D)) + if not len(ious)==0: + for tind, t in enumerate(p.iouThrs): + for dind, d in enumerate(dt): + # information about best match so far (m=-1 -> unmatched) + iou = min([t,1-1e-10]) + m = -1 + for gind, g in enumerate(gt): + # if this gt already matched, and not a crowd, continue + if gtm[tind,gind]>0 and not iscrowd[gind]: + continue + # if dt matched to reg gt, and on ignore gt, stop + if m>-1 and gtIg[m]==0 and gtIg[gind]==1: + break + # continue to next gt unless better match made + if ious[dind,gind] < iou: + continue + # if match successful and best so far, store appropriately + iou=ious[dind,gind] + m=gind + # if match made store id of match for both dt and gt + if m ==-1: + continue + dtIg[tind,dind] = gtIg[m] + dtm[tind,dind] = gt[m]['id'] + gtm[tind,m] = d['id'] + # set unmatched detections outside of area range to ignore + a = np.array([d['area']aRng[1] for d in dt]).reshape((1, len(dt))) + dtIg = np.logical_or(dtIg, np.logical_and(dtm==0, np.repeat(a,T,0))) + # store results for given image and category + return { + 'image_id': imgId, + 'category_id': catId, + 'aRng': aRng, + 'maxDet': maxDet, + 'dtIds': [d['id'] for d in dt], + 'gtIds': [g['id'] for g in gt], + 'dtMatches': dtm, + 'gtMatches': gtm, + 'dtScores': [d['score'] for d in dt], + 'gtIgnore': gtIg, + 'dtIgnore': dtIg, + } + + def accumulate(self, p = None): + ''' + Accumulate per image evaluation results and store the result in self.eval + :param p: input params for evaluation + :return: None + ''' + print('Accumulating evaluation results...') + tic = time.time() + if not self.evalImgs: + print('Please run evaluate() first') + # allows input customized parameters + if p is None: + p = self.params + p.catIds = p.catIds if p.useCats == 1 else [-1] + T = len(p.iouThrs) + R = len(p.recThrs) + K = len(p.catIds) if p.useCats else 1 + A = len(p.areaRng) + M = len(p.maxDets) + precision = -np.ones((T,R,K,A,M)) # -1 for the precision of absent categories + recall = -np.ones((T,K,A,M)) + scores = -np.ones((T,R,K,A,M)) + + # create dictionary for future indexing + _pe = self._paramsEval + catIds = _pe.catIds if _pe.useCats else [-1] + setK = set(catIds) + setA = set(map(tuple, _pe.areaRng)) + setM = set(_pe.maxDets) + setI = set(_pe.imgIds) + # get inds to evaluate + k_list = [n for n, k in enumerate(p.catIds) if k in setK] + m_list = [m for n, m in enumerate(p.maxDets) if m in setM] + a_list = [n for n, a in enumerate(map(lambda x: tuple(x), p.areaRng)) if a in setA] + i_list = [n for n, i in enumerate(p.imgIds) if i in setI] + I0 = len(_pe.imgIds) + A0 = len(_pe.areaRng) + # retrieve E at each category, area range, and max number of detections + for k, k0 in enumerate(k_list): + Nk = k0*A0*I0 + for a, a0 in enumerate(a_list): + Na = a0*I0 + for m, maxDet in enumerate(m_list): + E = [self.evalImgs[Nk + Na + i] for i in i_list] + E = [e for e in E if not e is None] + if len(E) == 0: + continue + dtScores = np.concatenate([e['dtScores'][0:maxDet] for e in E]) + + # different sorting method generates slightly different results. + # mergesort is used to be consistent as Matlab implementation. + inds = np.argsort(-dtScores, kind='mergesort') + dtScoresSorted = dtScores[inds] + + dtm = np.concatenate([e['dtMatches'][:,0:maxDet] for e in E], axis=1)[:,inds] + dtIg = np.concatenate([e['dtIgnore'][:,0:maxDet] for e in E], axis=1)[:,inds] + gtIg = np.concatenate([e['gtIgnore'] for e in E]) + npig = np.count_nonzero(gtIg==0 ) + if npig == 0: + continue + tps = np.logical_and( dtm, np.logical_not(dtIg) ) + fps = np.logical_and(np.logical_not(dtm), np.logical_not(dtIg) ) + + tp_sum = np.cumsum(tps, axis=1).astype(dtype=float) + fp_sum = np.cumsum(fps, axis=1).astype(dtype=float) + for t, (tp, fp) in enumerate(zip(tp_sum, fp_sum)): + tp = np.array(tp) + fp = np.array(fp) + nd = len(tp) + rc = tp / npig + pr = tp / (fp+tp+np.spacing(1)) + q = np.zeros((R,)) + ss = np.zeros((R,)) + + if nd: + recall[t,k,a,m] = rc[-1] + else: + recall[t,k,a,m] = 0 + + # numpy is slow without cython optimization for accessing elements + # use python array gets significant speed improvement + pr = pr.tolist(); q = q.tolist() + + for i in range(nd-1, 0, -1): + if pr[i] > pr[i-1]: + pr[i-1] = pr[i] + + inds = np.searchsorted(rc, p.recThrs, side='left') + try: + for ri, pi in enumerate(inds): + q[ri] = pr[pi] + ss[ri] = dtScoresSorted[pi] + except: + pass + precision[t,:,k,a,m] = np.array(q) + scores[t,:,k,a,m] = np.array(ss) + self.eval = { + 'params': p, + 'counts': [T, R, K, A, M], + 'date': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), + 'precision': precision, + 'recall': recall, + 'scores': scores, + } + toc = time.time() + print('DONE (t={:0.2f}s).'.format( toc-tic)) + + def summarize(self): + ''' + Compute and display summary metrics for evaluation results. + Note this functin can *only* be applied on the default parameter setting + ''' + def _summarize( ap=1, iouThr=None, areaRng='all', maxDets=100 ): + p = self.params + iStr = ' {:<18} {} @[ IoU={:<9} | area={:>6s} | maxDets={:>3d} ] = {:0.3f}' + titleStr = 'Average Precision' if ap == 1 else 'Average Recall' + typeStr = '(AP)' if ap==1 else '(AR)' + iouStr = '{:0.2f}:{:0.2f}'.format(p.iouThrs[0], p.iouThrs[-1]) \ + if iouThr is None else '{:0.2f}'.format(iouThr) + + aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng] + mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets] + if ap == 1: + # dimension of precision: [TxRxKxAxM] + s = self.eval['precision'] + # IoU + if iouThr is not None: + t = np.where(iouThr == p.iouThrs)[0] + s = s[t] + s = s[:,:,:,aind,mind] + else: + # dimension of recall: [TxKxAxM] + s = self.eval['recall'] + if iouThr is not None: + t = np.where(iouThr == p.iouThrs)[0] + s = s[t] + s = s[:,:,aind,mind] + if len(s[s>-1])==0: + mean_s = -1 + else: + mean_s = np.mean(s[s>-1]) + print(iStr.format(titleStr, typeStr, iouStr, areaRng, maxDets, mean_s)) + return mean_s + def _summarizeDets(): + stats = np.zeros((12,)) + stats[0] = _summarize(1) + stats[1] = _summarize(1, iouThr=.5, maxDets=self.params.maxDets[2]) + stats[2] = _summarize(1, iouThr=.75, maxDets=self.params.maxDets[2]) + stats[3] = _summarize(1, areaRng='small', maxDets=self.params.maxDets[2]) + stats[4] = _summarize(1, areaRng='medium', maxDets=self.params.maxDets[2]) + stats[5] = _summarize(1, areaRng='large', maxDets=self.params.maxDets[2]) + stats[6] = _summarize(0, maxDets=self.params.maxDets[0]) + stats[7] = _summarize(0, maxDets=self.params.maxDets[1]) + stats[8] = _summarize(0, maxDets=self.params.maxDets[2]) + stats[9] = _summarize(0, areaRng='small', maxDets=self.params.maxDets[2]) + stats[10] = _summarize(0, areaRng='medium', maxDets=self.params.maxDets[2]) + stats[11] = _summarize(0, areaRng='large', maxDets=self.params.maxDets[2]) + return stats + def _summarizeKps(): + stats = np.zeros((10,)) + stats[0] = _summarize(1, maxDets=20) + stats[1] = _summarize(1, maxDets=20, iouThr=.5) + stats[2] = _summarize(1, maxDets=20, iouThr=.75) + stats[3] = _summarize(1, maxDets=20, areaRng='medium') + stats[4] = _summarize(1, maxDets=20, areaRng='large') + stats[5] = _summarize(0, maxDets=20) + stats[6] = _summarize(0, maxDets=20, iouThr=.5) + stats[7] = _summarize(0, maxDets=20, iouThr=.75) + stats[8] = _summarize(0, maxDets=20, areaRng='medium') + stats[9] = _summarize(0, maxDets=20, areaRng='large') + return stats + if not self.eval: + raise Exception('Please run accumulate() first') + iouType = self.params.iouType + if iouType == 'segm' or iouType == 'bbox': + summarize = _summarizeDets + elif iouType == 'keypoints': + summarize = _summarizeKps + self.stats = summarize() + + def __str__(self): + self.summarize() + +class Params: + ''' + Params for coco evaluation api + ''' + def setDetParams(self): + self.imgIds = [] + self.catIds = [] + # np.arange causes trouble. the data point on arange is slightly larger than the true value + self.iouThrs = np.linspace(.5, 0.95, int(np.round((0.95 - .5) / .05)) + 1, endpoint=True) + self.recThrs = np.linspace(.0, 1.00, int(np.round((1.00 - .0) / .01)) + 1, endpoint=True) + self.maxDets = [1, 10, 100] + self.areaRng = [[0 ** 2, 1e5 ** 2], [0 ** 2, 32 ** 2], [32 ** 2, 96 ** 2], [96 ** 2, 1e5 ** 2]] + self.areaRngLbl = ['all', 'small', 'medium', 'large'] + self.useCats = 1 + + def setKpParams(self): + self.imgIds = [] + self.catIds = [] + # np.arange causes trouble. the data point on arange is slightly larger than the true value + self.iouThrs = np.linspace(.5, 0.95, int(np.round((0.95 - .5) / .05)) + 1, endpoint=True) + self.recThrs = np.linspace(.0, 1.00, int(np.round((1.00 - .0) / .01)) + 1, endpoint=True) + self.maxDets = [20] + self.areaRng = [[0 ** 2, 1e5 ** 2], [32 ** 2, 96 ** 2], [96 ** 2, 1e5 ** 2]] + self.areaRngLbl = ['all', 'medium', 'large'] + self.useCats = 1 + self.kpt_oks_sigmas = np.array([.26, .25, .25, .35, .35, .79, .79, .72, .72, .62,.62, 1.07, 1.07, .87, .87, .89, .89])/10.0 + + def __init__(self, iouType='segm'): + if iouType == 'segm' or iouType == 'bbox': + self.setDetParams() + elif iouType == 'keypoints': + self.setKpParams() + else: + raise Exception('iouType not supported') + self.iouType = iouType + # useSegm is deprecated + self.useSegm = None diff --git a/annotator/oneformer/pycocotools/mask.py b/annotator/oneformer/pycocotools/mask.py new file mode 100644 index 0000000000000000000000000000000000000000..85a5643aadd5c3c5f02609aa918c38d6da14a929 --- /dev/null +++ b/annotator/oneformer/pycocotools/mask.py @@ -0,0 +1,107 @@ +__author__ = 'tsungyi' + +# import annotator.oneformer.pycocotools._mask as _mask + +# Interface for manipulating masks stored in RLE format. +# +# RLE is a simple yet efficient format for storing binary masks. RLE +# first divides a vector (or vectorized image) into a series of piecewise +# constant regions and then for each piece simply stores the length of +# that piece. For example, given M=[0 0 1 1 1 0 1] the RLE counts would +# be [2 3 1 1], or for M=[1 1 1 1 1 1 0] the counts would be [0 6 1] +# (note that the odd counts are always the numbers of zeros). Instead of +# storing the counts directly, additional compression is achieved with a +# variable bitrate representation based on a common scheme called LEB128. +# +# Compression is greatest given large piecewise constant regions. +# Specifically, the size of the RLE is proportional to the number of +# *boundaries* in M (or for an image the number of boundaries in the y +# direction). Assuming fairly simple shapes, the RLE representation is +# O(sqrt(n)) where n is number of pixels in the object. Hence space usage +# is substantially lower, especially for large simple objects (large n). +# +# Many common operations on masks can be computed directly using the RLE +# (without need for decoding). This includes computations such as area, +# union, intersection, etc. All of these operations are linear in the +# size of the RLE, in other words they are O(sqrt(n)) where n is the area +# of the object. Computing these operations on the original mask is O(n). +# Thus, using the RLE can result in substantial computational savings. +# +# The following API functions are defined: +# encode - Encode binary masks using RLE. +# decode - Decode binary masks encoded via RLE. +# merge - Compute union or intersection of encoded masks. +# iou - Compute intersection over union between masks. +# area - Compute area of encoded masks. +# toBbox - Get bounding boxes surrounding encoded masks. +# frPyObjects - Convert polygon, bbox, and uncompressed RLE to encoded RLE mask. +# +# Usage: +# Rs = encode( masks ) +# masks = decode( Rs ) +# R = merge( Rs, intersect=false ) +# o = iou( dt, gt, iscrowd ) +# a = area( Rs ) +# bbs = toBbox( Rs ) +# Rs = frPyObjects( [pyObjects], h, w ) +# +# In the API the following formats are used: +# Rs - [dict] Run-length encoding of binary masks +# R - dict Run-length encoding of binary mask +# masks - [hxwxn] Binary mask(s) (must have type np.ndarray(dtype=uint8) in column-major order) +# iscrowd - [nx1] list of np.ndarray. 1 indicates corresponding gt image has crowd region to ignore +# bbs - [nx4] Bounding box(es) stored as [x y w h] +# poly - Polygon stored as [[x1 y1 x2 y2...],[x1 y1 ...],...] (2D list) +# dt,gt - May be either bounding boxes or encoded masks +# Both poly and bbs are 0-indexed (bbox=[0 0 1 1] encloses first pixel). +# +# Finally, a note about the intersection over union (iou) computation. +# The standard iou of a ground truth (gt) and detected (dt) object is +# iou(gt,dt) = area(intersect(gt,dt)) / area(union(gt,dt)) +# For "crowd" regions, we use a modified criteria. If a gt object is +# marked as "iscrowd", we allow a dt to match any subregion of the gt. +# Choosing gt' in the crowd gt that best matches the dt can be done using +# gt'=intersect(dt,gt). Since by definition union(gt',dt)=dt, computing +# iou(gt,dt,iscrowd) = iou(gt',dt) = area(intersect(gt,dt)) / area(dt) +# For crowd gt regions we use this modified criteria above for the iou. +# +# To compile run "python setup.py build_ext --inplace" +# Please do not contact us for help with compiling. +# +# Microsoft COCO Toolbox. version 2.0 +# Data, paper, and tutorials available at: http://mscoco.org/ +# Code written by Piotr Dollar and Tsung-Yi Lin, 2015. +# Licensed under the Simplified BSD License [see coco/license.txt] + +# iou = _mask.iou +# merge = _mask.merge +# frPyObjects = _mask.frPyObjects + +def encode(bimask): + pass + # if len(bimask.shape) == 3: + # return _mask.encode(bimask) + # elif len(bimask.shape) == 2: + # h, w = bimask.shape + # return _mask.encode(bimask.reshape((h, w, 1), order='F'))[0] + +def decode(rleObjs): + pass + # if type(rleObjs) == list: + # return _mask.decode(rleObjs) + # else: + # return _mask.decode([rleObjs])[:,:,0] + +def area(rleObjs): + pass + # if type(rleObjs) == list: + # return _mask.area(rleObjs) + # else: + # return _mask.area([rleObjs])[0] + +def toBbox(rleObjs): + pass + # if type(rleObjs) == list: + # return _mask.toBbox(rleObjs) + # else: + # return _mask.toBbox([rleObjs])[0] \ No newline at end of file diff --git a/annotator/openpose/LICENSE b/annotator/openpose/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..6f60b76d35fa1012809985780964a5068adce4fd --- /dev/null +++ b/annotator/openpose/LICENSE @@ -0,0 +1,108 @@ +OPENPOSE: MULTIPERSON KEYPOINT DETECTION +SOFTWARE LICENSE AGREEMENT +ACADEMIC OR NON-PROFIT ORGANIZATION NONCOMMERCIAL RESEARCH USE ONLY + +BY USING OR DOWNLOADING THE SOFTWARE, YOU ARE AGREEING TO THE TERMS OF THIS LICENSE AGREEMENT. IF YOU DO NOT AGREE WITH THESE TERMS, YOU MAY NOT USE OR DOWNLOAD THE SOFTWARE. + +This is a license agreement ("Agreement") between your academic institution or non-profit organization or self (called "Licensee" or "You" in this Agreement) and Carnegie Mellon University (called "Licensor" in this Agreement). All rights not specifically granted to you in this Agreement are reserved for Licensor. + +RESERVATION OF OWNERSHIP AND GRANT OF LICENSE: +Licensor retains exclusive ownership of any copy of the Software (as defined below) licensed under this Agreement and hereby grants to Licensee a personal, non-exclusive, +non-transferable license to use the Software for noncommercial research purposes, without the right to sublicense, pursuant to the terms and conditions of this Agreement. As used in this Agreement, the term "Software" means (i) the actual copy of all or any portion of code for program routines made accessible to Licensee by Licensor pursuant to this Agreement, inclusive of backups, updates, and/or merged copies permitted hereunder or subsequently supplied by Licensor, including all or any file structures, programming instructions, user interfaces and screen formats and sequences as well as any and all documentation and instructions related to it, and (ii) all or any derivatives and/or modifications created or made by You to any of the items specified in (i). + +CONFIDENTIALITY: Licensee acknowledges that the Software is proprietary to Licensor, and as such, Licensee agrees to receive all such materials in confidence and use the Software only in accordance with the terms of this Agreement. Licensee agrees to use reasonable effort to protect the Software from unauthorized use, reproduction, distribution, or publication. + +COPYRIGHT: The Software is owned by Licensor and is protected by United +States copyright laws and applicable international treaties and/or conventions. + +PERMITTED USES: The Software may be used for your own noncommercial internal research purposes. You understand and agree that Licensor is not obligated to implement any suggestions and/or feedback you might provide regarding the Software, but to the extent Licensor does so, you are not entitled to any compensation related thereto. + +DERIVATIVES: You may create derivatives of or make modifications to the Software, however, You agree that all and any such derivatives and modifications will be owned by Licensor and become a part of the Software licensed to You under this Agreement. You may only use such derivatives and modifications for your own noncommercial internal research purposes, and you may not otherwise use, distribute or copy such derivatives and modifications in violation of this Agreement. + +BACKUPS: If Licensee is an organization, it may make that number of copies of the Software necessary for internal noncommercial use at a single site within its organization provided that all information appearing in or on the original labels, including the copyright and trademark notices are copied onto the labels of the copies. + +USES NOT PERMITTED: You may not distribute, copy or use the Software except as explicitly permitted herein. Licensee has not been granted any trademark license as part of this Agreement and may not use the name or mark “OpenPose", "Carnegie Mellon" or any renditions thereof without the prior written permission of Licensor. + +You may not sell, rent, lease, sublicense, lend, time-share or transfer, in whole or in part, or provide third parties access to prior or present versions (or any parts thereof) of the Software. + +ASSIGNMENT: You may not assign this Agreement or your rights hereunder without the prior written consent of Licensor. Any attempted assignment without such consent shall be null and void. + +TERM: The term of the license granted by this Agreement is from Licensee's acceptance of this Agreement by downloading the Software or by using the Software until terminated as provided below. + +The Agreement automatically terminates without notice if you fail to comply with any provision of this Agreement. Licensee may terminate this Agreement by ceasing using the Software. Upon any termination of this Agreement, Licensee will delete any and all copies of the Software. You agree that all provisions which operate to protect the proprietary rights of Licensor shall remain in force should breach occur and that the obligation of confidentiality described in this Agreement is binding in perpetuity and, as such, survives the term of the Agreement. + +FEE: Provided Licensee abides completely by the terms and conditions of this Agreement, there is no fee due to Licensor for Licensee's use of the Software in accordance with this Agreement. + +DISCLAIMER OF WARRANTIES: THE SOFTWARE IS PROVIDED "AS-IS" WITHOUT WARRANTY OF ANY KIND INCLUDING ANY WARRANTIES OF PERFORMANCE OR MERCHANTABILITY OR FITNESS FOR A PARTICULAR USE OR PURPOSE OR OF NON-INFRINGEMENT. LICENSEE BEARS ALL RISK RELATING TO QUALITY AND PERFORMANCE OF THE SOFTWARE AND RELATED MATERIALS. + +SUPPORT AND MAINTENANCE: No Software support or training by the Licensor is provided as part of this Agreement. + +EXCLUSIVE REMEDY AND LIMITATION OF LIABILITY: To the maximum extent permitted under applicable law, Licensor shall not be liable for direct, indirect, special, incidental, or consequential damages or lost profits related to Licensee's use of and/or inability to use the Software, even if Licensor is advised of the possibility of such damage. + +EXPORT REGULATION: Licensee agrees to comply with any and all applicable +U.S. export control laws, regulations, and/or other laws related to embargoes and sanction programs administered by the Office of Foreign Assets Control. + +SEVERABILITY: If any provision(s) of this Agreement shall be held to be invalid, illegal, or unenforceable by a court or other tribunal of competent jurisdiction, the validity, legality and enforceability of the remaining provisions shall not in any way be affected or impaired thereby. + +NO IMPLIED WAIVERS: No failure or delay by Licensor in enforcing any right or remedy under this Agreement shall be construed as a waiver of any future or other exercise of such right or remedy by Licensor. + +GOVERNING LAW: This Agreement shall be construed and enforced in accordance with the laws of the Commonwealth of Pennsylvania without reference to conflict of laws principles. You consent to the personal jurisdiction of the courts of this County and waive their rights to venue outside of Allegheny County, Pennsylvania. + +ENTIRE AGREEMENT AND AMENDMENTS: This Agreement constitutes the sole and entire agreement between Licensee and Licensor as to the matter set forth herein and supersedes any previous agreements, understandings, and arrangements between the parties relating hereto. + + + +************************************************************************ + +THIRD-PARTY SOFTWARE NOTICES AND INFORMATION + +This project incorporates material from the project(s) listed below (collectively, "Third Party Code"). This Third Party Code is licensed to you under their original license terms set forth below. We reserves all other rights not expressly granted, whether by implication, estoppel or otherwise. + +1. Caffe, version 1.0.0, (https://github.com/BVLC/caffe/) + +COPYRIGHT + +All contributions by the University of California: +Copyright (c) 2014-2017 The Regents of the University of California (Regents) +All rights reserved. + +All other contributions: +Copyright (c) 2014-2017, the respective contributors +All rights reserved. + +Caffe uses a shared copyright model: each contributor holds copyright over +their contributions to Caffe. The project versioning records all such +contribution and copyright details. If a contributor wants to further mark +their specific copyright on a particular contribution, they should indicate +their copyright solely in the commit message of the change when it is +committed. + +LICENSE + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +CONTRIBUTION AGREEMENT + +By contributing to the BVLC/caffe repository through pull-request, comment, +or otherwise, the contributor releases their content to the +license and copyright terms herein. + +************END OF THIRD-PARTY SOFTWARE NOTICES AND INFORMATION********** \ No newline at end of file diff --git a/annotator/openpose/__init__.py b/annotator/openpose/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..102434701a14621a66149fbabcf224b1bb726a6c --- /dev/null +++ b/annotator/openpose/__init__.py @@ -0,0 +1,100 @@ +# Openpose +# Original from CMU https://github.com/CMU-Perceptual-Computing-Lab/openpose +# 2nd Edited by https://github.com/Hzzone/pytorch-openpose +# 3rd Edited by ControlNet +# 4th Edited by ControlNet (added face and correct hands) + +import os +os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE" + +import torch +import numpy as np +from . import util +from .body import Body +from .hand import Hand +from .face import Face +from annotator.util import annotator_ckpts_path + + +body_model_path = "https://huggingface.co./lllyasviel/Annotators/resolve/main/body_pose_model.pth" +hand_model_path = "https://huggingface.co./lllyasviel/Annotators/resolve/main/hand_pose_model.pth" +face_model_path = "https://huggingface.co./lllyasviel/Annotators/resolve/main/facenet.pth" + + +def draw_pose(pose, H, W, draw_body=True, draw_hand=True, draw_face=True): + bodies = pose['bodies'] + faces = pose['faces'] + hands = pose['hands'] + candidate = bodies['candidate'] + subset = bodies['subset'] + canvas = np.zeros(shape=(H, W, 3), dtype=np.uint8) + + if draw_body: + canvas = util.draw_bodypose(canvas, candidate, subset) + + if draw_hand: + canvas = util.draw_handpose(canvas, hands) + + if draw_face: + canvas = util.draw_facepose(canvas, faces) + + return canvas + + +class OpenposeDetector: + def __init__(self): + body_modelpath = os.path.join(annotator_ckpts_path, "body_pose_model.pth") + hand_modelpath = os.path.join(annotator_ckpts_path, "hand_pose_model.pth") + face_modelpath = os.path.join(annotator_ckpts_path, "facenet.pth") + + if not os.path.exists(body_modelpath): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(body_model_path, model_dir=annotator_ckpts_path) + + if not os.path.exists(hand_modelpath): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(hand_model_path, model_dir=annotator_ckpts_path) + + if not os.path.exists(face_modelpath): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(face_model_path, model_dir=annotator_ckpts_path) + + self.body_estimation = Body(body_modelpath) + self.hand_estimation = Hand(hand_modelpath) + self.face_estimation = Face(face_modelpath) + + def __call__(self, oriImg, hand_and_face=False, return_is_index=False): + oriImg = oriImg[:, :, ::-1].copy() + H, W, C = oriImg.shape + with torch.no_grad(): + candidate, subset = self.body_estimation(oriImg) + hands = [] + faces = [] + if hand_and_face: + # Hand + hands_list = util.handDetect(candidate, subset, oriImg) + for x, y, w, is_left in hands_list: + peaks = self.hand_estimation(oriImg[y:y+w, x:x+w, :]).astype(np.float32) + if peaks.ndim == 2 and peaks.shape[1] == 2: + peaks[:, 0] = np.where(peaks[:, 0] < 1e-6, -1, peaks[:, 0] + x) / float(W) + peaks[:, 1] = np.where(peaks[:, 1] < 1e-6, -1, peaks[:, 1] + y) / float(H) + hands.append(peaks.tolist()) + # Face + faces_list = util.faceDetect(candidate, subset, oriImg) + for x, y, w in faces_list: + heatmaps = self.face_estimation(oriImg[y:y+w, x:x+w, :]) + peaks = self.face_estimation.compute_peaks_from_heatmaps(heatmaps).astype(np.float32) + if peaks.ndim == 2 and peaks.shape[1] == 2: + peaks[:, 0] = np.where(peaks[:, 0] < 1e-6, -1, peaks[:, 0] + x) / float(W) + peaks[:, 1] = np.where(peaks[:, 1] < 1e-6, -1, peaks[:, 1] + y) / float(H) + faces.append(peaks.tolist()) + if candidate.ndim == 2 and candidate.shape[1] == 4: + candidate = candidate[:, :2] + candidate[:, 0] /= float(W) + candidate[:, 1] /= float(H) + bodies = dict(candidate=candidate.tolist(), subset=subset.tolist()) + pose = dict(bodies=bodies, hands=hands, faces=faces) + if return_is_index: + return pose + else: + return draw_pose(pose, H, W) diff --git a/annotator/openpose/body.py b/annotator/openpose/body.py new file mode 100644 index 0000000000000000000000000000000000000000..025783a90f2075ab90067dedf18b375ffb52b32e --- /dev/null +++ b/annotator/openpose/body.py @@ -0,0 +1,219 @@ +import cv2 +import numpy as np +import math +import time +from scipy.ndimage.filters import gaussian_filter +import matplotlib.pyplot as plt +import matplotlib +import torch +from torchvision import transforms + +from . import util +from .model import bodypose_model + +class Body(object): + def __init__(self, model_path): + self.model = bodypose_model() + if torch.cuda.is_available(): + self.model = self.model.cuda() + print('cuda') + model_dict = util.transfer(self.model, torch.load(model_path)) + self.model.load_state_dict(model_dict) + self.model.eval() + + def __call__(self, oriImg): + # scale_search = [0.5, 1.0, 1.5, 2.0] + scale_search = [0.5] + boxsize = 368 + stride = 8 + padValue = 128 + thre1 = 0.1 + thre2 = 0.05 + multiplier = [x * boxsize / oriImg.shape[0] for x in scale_search] + heatmap_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 19)) + paf_avg = np.zeros((oriImg.shape[0], oriImg.shape[1], 38)) + + for m in range(len(multiplier)): + scale = multiplier[m] + imageToTest = util.smart_resize_k(oriImg, fx=scale, fy=scale) + imageToTest_padded, pad = util.padRightDownCorner(imageToTest, stride, padValue) + im = np.transpose(np.float32(imageToTest_padded[:, :, :, np.newaxis]), (3, 2, 0, 1)) / 256 - 0.5 + im = np.ascontiguousarray(im) + + data = torch.from_numpy(im).float() + if torch.cuda.is_available(): + data = data.cuda() + # data = data.permute([2, 0, 1]).unsqueeze(0).float() + with torch.no_grad(): + Mconv7_stage6_L1, Mconv7_stage6_L2 = self.model(data) + Mconv7_stage6_L1 = Mconv7_stage6_L1.cpu().numpy() + Mconv7_stage6_L2 = Mconv7_stage6_L2.cpu().numpy() + + # extract outputs, resize, and remove padding + # heatmap = np.transpose(np.squeeze(net.blobs[output_blobs.keys()[1]].data), (1, 2, 0)) # output 1 is heatmaps + heatmap = np.transpose(np.squeeze(Mconv7_stage6_L2), (1, 2, 0)) # output 1 is heatmaps + heatmap = util.smart_resize_k(heatmap, fx=stride, fy=stride) + heatmap = heatmap[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :] + heatmap = util.smart_resize(heatmap, (oriImg.shape[0], oriImg.shape[1])) + + # paf = np.transpose(np.squeeze(net.blobs[output_blobs.keys()[0]].data), (1, 2, 0)) # output 0 is PAFs + paf = np.transpose(np.squeeze(Mconv7_stage6_L1), (1, 2, 0)) # output 0 is PAFs + paf = util.smart_resize_k(paf, fx=stride, fy=stride) + paf = paf[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :] + paf = util.smart_resize(paf, (oriImg.shape[0], oriImg.shape[1])) + + heatmap_avg += heatmap_avg + heatmap / len(multiplier) + paf_avg += + paf / len(multiplier) + + all_peaks = [] + peak_counter = 0 + + for part in range(18): + map_ori = heatmap_avg[:, :, part] + one_heatmap = gaussian_filter(map_ori, sigma=3) + + map_left = np.zeros(one_heatmap.shape) + map_left[1:, :] = one_heatmap[:-1, :] + map_right = np.zeros(one_heatmap.shape) + map_right[:-1, :] = one_heatmap[1:, :] + map_up = np.zeros(one_heatmap.shape) + map_up[:, 1:] = one_heatmap[:, :-1] + map_down = np.zeros(one_heatmap.shape) + map_down[:, :-1] = one_heatmap[:, 1:] + + peaks_binary = np.logical_and.reduce( + (one_heatmap >= map_left, one_heatmap >= map_right, one_heatmap >= map_up, one_heatmap >= map_down, one_heatmap > thre1)) + peaks = list(zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0])) # note reverse + peaks_with_score = [x + (map_ori[x[1], x[0]],) for x in peaks] + peak_id = range(peak_counter, peak_counter + len(peaks)) + peaks_with_score_and_id = [peaks_with_score[i] + (peak_id[i],) for i in range(len(peak_id))] + + all_peaks.append(peaks_with_score_and_id) + peak_counter += len(peaks) + + # find connection in the specified sequence, center 29 is in the position 15 + limbSeq = [[2, 3], [2, 6], [3, 4], [4, 5], [6, 7], [7, 8], [2, 9], [9, 10], \ + [10, 11], [2, 12], [12, 13], [13, 14], [2, 1], [1, 15], [15, 17], \ + [1, 16], [16, 18], [3, 17], [6, 18]] + # the middle joints heatmap correpondence + mapIdx = [[31, 32], [39, 40], [33, 34], [35, 36], [41, 42], [43, 44], [19, 20], [21, 22], \ + [23, 24], [25, 26], [27, 28], [29, 30], [47, 48], [49, 50], [53, 54], [51, 52], \ + [55, 56], [37, 38], [45, 46]] + + connection_all = [] + special_k = [] + mid_num = 10 + + for k in range(len(mapIdx)): + score_mid = paf_avg[:, :, [x - 19 for x in mapIdx[k]]] + candA = all_peaks[limbSeq[k][0] - 1] + candB = all_peaks[limbSeq[k][1] - 1] + nA = len(candA) + nB = len(candB) + indexA, indexB = limbSeq[k] + if (nA != 0 and nB != 0): + connection_candidate = [] + for i in range(nA): + for j in range(nB): + vec = np.subtract(candB[j][:2], candA[i][:2]) + norm = math.sqrt(vec[0] * vec[0] + vec[1] * vec[1]) + norm = max(0.001, norm) + vec = np.divide(vec, norm) + + startend = list(zip(np.linspace(candA[i][0], candB[j][0], num=mid_num), \ + np.linspace(candA[i][1], candB[j][1], num=mid_num))) + + vec_x = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 0] \ + for I in range(len(startend))]) + vec_y = np.array([score_mid[int(round(startend[I][1])), int(round(startend[I][0])), 1] \ + for I in range(len(startend))]) + + score_midpts = np.multiply(vec_x, vec[0]) + np.multiply(vec_y, vec[1]) + score_with_dist_prior = sum(score_midpts) / len(score_midpts) + min( + 0.5 * oriImg.shape[0] / norm - 1, 0) + criterion1 = len(np.nonzero(score_midpts > thre2)[0]) > 0.8 * len(score_midpts) + criterion2 = score_with_dist_prior > 0 + if criterion1 and criterion2: + connection_candidate.append( + [i, j, score_with_dist_prior, score_with_dist_prior + candA[i][2] + candB[j][2]]) + + connection_candidate = sorted(connection_candidate, key=lambda x: x[2], reverse=True) + connection = np.zeros((0, 5)) + for c in range(len(connection_candidate)): + i, j, s = connection_candidate[c][0:3] + if (i not in connection[:, 3] and j not in connection[:, 4]): + connection = np.vstack([connection, [candA[i][3], candB[j][3], s, i, j]]) + if (len(connection) >= min(nA, nB)): + break + + connection_all.append(connection) + else: + special_k.append(k) + connection_all.append([]) + + # last number in each row is the total parts number of that person + # the second last number in each row is the score of the overall configuration + subset = -1 * np.ones((0, 20)) + candidate = np.array([item for sublist in all_peaks for item in sublist]) + + for k in range(len(mapIdx)): + if k not in special_k: + partAs = connection_all[k][:, 0] + partBs = connection_all[k][:, 1] + indexA, indexB = np.array(limbSeq[k]) - 1 + + for i in range(len(connection_all[k])): # = 1:size(temp,1) + found = 0 + subset_idx = [-1, -1] + for j in range(len(subset)): # 1:size(subset,1): + if subset[j][indexA] == partAs[i] or subset[j][indexB] == partBs[i]: + subset_idx[found] = j + found += 1 + + if found == 1: + j = subset_idx[0] + if subset[j][indexB] != partBs[i]: + subset[j][indexB] = partBs[i] + subset[j][-1] += 1 + subset[j][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2] + elif found == 2: # if found 2 and disjoint, merge them + j1, j2 = subset_idx + membership = ((subset[j1] >= 0).astype(int) + (subset[j2] >= 0).astype(int))[:-2] + if len(np.nonzero(membership == 2)[0]) == 0: # merge + subset[j1][:-2] += (subset[j2][:-2] + 1) + subset[j1][-2:] += subset[j2][-2:] + subset[j1][-2] += connection_all[k][i][2] + subset = np.delete(subset, j2, 0) + else: # as like found == 1 + subset[j1][indexB] = partBs[i] + subset[j1][-1] += 1 + subset[j1][-2] += candidate[partBs[i].astype(int), 2] + connection_all[k][i][2] + + # if find no partA in the subset, create a new subset + elif not found and k < 17: + row = -1 * np.ones(20) + row[indexA] = partAs[i] + row[indexB] = partBs[i] + row[-1] = 2 + row[-2] = sum(candidate[connection_all[k][i, :2].astype(int), 2]) + connection_all[k][i][2] + subset = np.vstack([subset, row]) + # delete some rows of subset which has few parts occur + deleteIdx = [] + for i in range(len(subset)): + if subset[i][-1] < 4 or subset[i][-2] / subset[i][-1] < 0.4: + deleteIdx.append(i) + subset = np.delete(subset, deleteIdx, axis=0) + + # subset: n*20 array, 0-17 is the index in candidate, 18 is the total score, 19 is the total parts + # candidate: x, y, score, id + return candidate, subset + +if __name__ == "__main__": + body_estimation = Body('../model/body_pose_model.pth') + + test_image = '../images/ski.jpg' + oriImg = cv2.imread(test_image) # B,G,R order + candidate, subset = body_estimation(oriImg) + canvas = util.draw_bodypose(oriImg, candidate, subset) + plt.imshow(canvas[:, :, [2, 1, 0]]) + plt.show() diff --git a/annotator/openpose/face.py b/annotator/openpose/face.py new file mode 100644 index 0000000000000000000000000000000000000000..24969e620186a5183da0cb9fae6ca8db62fe57fa --- /dev/null +++ b/annotator/openpose/face.py @@ -0,0 +1,363 @@ +import logging +import numpy as np +from torchvision.transforms import ToTensor, ToPILImage +import torch +import torch.nn.functional as F +import cv2 + +from . import util +from torch.nn import Conv2d, Module, ReLU, MaxPool2d, init + + +class FaceNet(Module): + """Model the cascading heatmaps. """ + def __init__(self): + super(FaceNet, self).__init__() + # cnn to make feature map + self.relu = ReLU() + self.max_pooling_2d = MaxPool2d(kernel_size=2, stride=2) + self.conv1_1 = Conv2d(in_channels=3, out_channels=64, + kernel_size=3, stride=1, padding=1) + self.conv1_2 = Conv2d( + in_channels=64, out_channels=64, kernel_size=3, stride=1, + padding=1) + self.conv2_1 = Conv2d( + in_channels=64, out_channels=128, kernel_size=3, stride=1, + padding=1) + self.conv2_2 = Conv2d( + in_channels=128, out_channels=128, kernel_size=3, stride=1, + padding=1) + self.conv3_1 = Conv2d( + in_channels=128, out_channels=256, kernel_size=3, stride=1, + padding=1) + self.conv3_2 = Conv2d( + in_channels=256, out_channels=256, kernel_size=3, stride=1, + padding=1) + self.conv3_3 = Conv2d( + in_channels=256, out_channels=256, kernel_size=3, stride=1, + padding=1) + self.conv3_4 = Conv2d( + in_channels=256, out_channels=256, kernel_size=3, stride=1, + padding=1) + self.conv4_1 = Conv2d( + in_channels=256, out_channels=512, kernel_size=3, stride=1, + padding=1) + self.conv4_2 = Conv2d( + in_channels=512, out_channels=512, kernel_size=3, stride=1, + padding=1) + self.conv4_3 = Conv2d( + in_channels=512, out_channels=512, kernel_size=3, stride=1, + padding=1) + self.conv4_4 = Conv2d( + in_channels=512, out_channels=512, kernel_size=3, stride=1, + padding=1) + self.conv5_1 = Conv2d( + in_channels=512, out_channels=512, kernel_size=3, stride=1, + padding=1) + self.conv5_2 = Conv2d( + in_channels=512, out_channels=512, kernel_size=3, stride=1, + padding=1) + self.conv5_3_CPM = Conv2d( + in_channels=512, out_channels=128, kernel_size=3, stride=1, + padding=1) + + # stage1 + self.conv6_1_CPM = Conv2d( + in_channels=128, out_channels=512, kernel_size=1, stride=1, + padding=0) + self.conv6_2_CPM = Conv2d( + in_channels=512, out_channels=71, kernel_size=1, stride=1, + padding=0) + + # stage2 + self.Mconv1_stage2 = Conv2d( + in_channels=199, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv2_stage2 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv3_stage2 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv4_stage2 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv5_stage2 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv6_stage2 = Conv2d( + in_channels=128, out_channels=128, kernel_size=1, stride=1, + padding=0) + self.Mconv7_stage2 = Conv2d( + in_channels=128, out_channels=71, kernel_size=1, stride=1, + padding=0) + + # stage3 + self.Mconv1_stage3 = Conv2d( + in_channels=199, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv2_stage3 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv3_stage3 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv4_stage3 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv5_stage3 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv6_stage3 = Conv2d( + in_channels=128, out_channels=128, kernel_size=1, stride=1, + padding=0) + self.Mconv7_stage3 = Conv2d( + in_channels=128, out_channels=71, kernel_size=1, stride=1, + padding=0) + + # stage4 + self.Mconv1_stage4 = Conv2d( + in_channels=199, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv2_stage4 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv3_stage4 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv4_stage4 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv5_stage4 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv6_stage4 = Conv2d( + in_channels=128, out_channels=128, kernel_size=1, stride=1, + padding=0) + self.Mconv7_stage4 = Conv2d( + in_channels=128, out_channels=71, kernel_size=1, stride=1, + padding=0) + + # stage5 + self.Mconv1_stage5 = Conv2d( + in_channels=199, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv2_stage5 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv3_stage5 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv4_stage5 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv5_stage5 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv6_stage5 = Conv2d( + in_channels=128, out_channels=128, kernel_size=1, stride=1, + padding=0) + self.Mconv7_stage5 = Conv2d( + in_channels=128, out_channels=71, kernel_size=1, stride=1, + padding=0) + + # stage6 + self.Mconv1_stage6 = Conv2d( + in_channels=199, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv2_stage6 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv3_stage6 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv4_stage6 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv5_stage6 = Conv2d( + in_channels=128, out_channels=128, kernel_size=7, stride=1, + padding=3) + self.Mconv6_stage6 = Conv2d( + in_channels=128, out_channels=128, kernel_size=1, stride=1, + padding=0) + self.Mconv7_stage6 = Conv2d( + in_channels=128, out_channels=71, kernel_size=1, stride=1, + padding=0) + + for m in self.modules(): + if isinstance(m, Conv2d): + init.constant_(m.bias, 0) + + def forward(self, x): + """Return a list of heatmaps.""" + heatmaps = [] + + h = self.relu(self.conv1_1(x)) + h = self.relu(self.conv1_2(h)) + h = self.max_pooling_2d(h) + h = self.relu(self.conv2_1(h)) + h = self.relu(self.conv2_2(h)) + h = self.max_pooling_2d(h) + h = self.relu(self.conv3_1(h)) + h = self.relu(self.conv3_2(h)) + h = self.relu(self.conv3_3(h)) + h = self.relu(self.conv3_4(h)) + h = self.max_pooling_2d(h) + h = self.relu(self.conv4_1(h)) + h = self.relu(self.conv4_2(h)) + h = self.relu(self.conv4_3(h)) + h = self.relu(self.conv4_4(h)) + h = self.relu(self.conv5_1(h)) + h = self.relu(self.conv5_2(h)) + h = self.relu(self.conv5_3_CPM(h)) + feature_map = h + + # stage1 + h = self.relu(self.conv6_1_CPM(h)) + h = self.conv6_2_CPM(h) + heatmaps.append(h) + + # stage2 + h = torch.cat([h, feature_map], dim=1) # channel concat + h = self.relu(self.Mconv1_stage2(h)) + h = self.relu(self.Mconv2_stage2(h)) + h = self.relu(self.Mconv3_stage2(h)) + h = self.relu(self.Mconv4_stage2(h)) + h = self.relu(self.Mconv5_stage2(h)) + h = self.relu(self.Mconv6_stage2(h)) + h = self.Mconv7_stage2(h) + heatmaps.append(h) + + # stage3 + h = torch.cat([h, feature_map], dim=1) # channel concat + h = self.relu(self.Mconv1_stage3(h)) + h = self.relu(self.Mconv2_stage3(h)) + h = self.relu(self.Mconv3_stage3(h)) + h = self.relu(self.Mconv4_stage3(h)) + h = self.relu(self.Mconv5_stage3(h)) + h = self.relu(self.Mconv6_stage3(h)) + h = self.Mconv7_stage3(h) + heatmaps.append(h) + + # stage4 + h = torch.cat([h, feature_map], dim=1) # channel concat + h = self.relu(self.Mconv1_stage4(h)) + h = self.relu(self.Mconv2_stage4(h)) + h = self.relu(self.Mconv3_stage4(h)) + h = self.relu(self.Mconv4_stage4(h)) + h = self.relu(self.Mconv5_stage4(h)) + h = self.relu(self.Mconv6_stage4(h)) + h = self.Mconv7_stage4(h) + heatmaps.append(h) + + # stage5 + h = torch.cat([h, feature_map], dim=1) # channel concat + h = self.relu(self.Mconv1_stage5(h)) + h = self.relu(self.Mconv2_stage5(h)) + h = self.relu(self.Mconv3_stage5(h)) + h = self.relu(self.Mconv4_stage5(h)) + h = self.relu(self.Mconv5_stage5(h)) + h = self.relu(self.Mconv6_stage5(h)) + h = self.Mconv7_stage5(h) + heatmaps.append(h) + + # stage6 + h = torch.cat([h, feature_map], dim=1) # channel concat + h = self.relu(self.Mconv1_stage6(h)) + h = self.relu(self.Mconv2_stage6(h)) + h = self.relu(self.Mconv3_stage6(h)) + h = self.relu(self.Mconv4_stage6(h)) + h = self.relu(self.Mconv5_stage6(h)) + h = self.relu(self.Mconv6_stage6(h)) + h = self.Mconv7_stage6(h) + heatmaps.append(h) + + return heatmaps + + +LOG = logging.getLogger(__name__) +TOTEN = ToTensor() +TOPIL = ToPILImage() + + +params = { + 'gaussian_sigma': 2.5, + 'inference_img_size': 736, # 368, 736, 1312 + 'heatmap_peak_thresh': 0.1, + 'crop_scale': 1.5, + 'line_indices': [ + [0, 1], [1, 2], [2, 3], [3, 4], [4, 5], [5, 6], + [6, 7], [7, 8], [8, 9], [9, 10], [10, 11], [11, 12], [12, 13], + [13, 14], [14, 15], [15, 16], + [17, 18], [18, 19], [19, 20], [20, 21], + [22, 23], [23, 24], [24, 25], [25, 26], + [27, 28], [28, 29], [29, 30], + [31, 32], [32, 33], [33, 34], [34, 35], + [36, 37], [37, 38], [38, 39], [39, 40], [40, 41], [41, 36], + [42, 43], [43, 44], [44, 45], [45, 46], [46, 47], [47, 42], + [48, 49], [49, 50], [50, 51], [51, 52], [52, 53], [53, 54], + [54, 55], [55, 56], [56, 57], [57, 58], [58, 59], [59, 48], + [60, 61], [61, 62], [62, 63], [63, 64], [64, 65], [65, 66], + [66, 67], [67, 60] + ], +} + + +class Face(object): + """ + The OpenPose face landmark detector model. + + Args: + inference_size: set the size of the inference image size, suggested: + 368, 736, 1312, default 736 + gaussian_sigma: blur the heatmaps, default 2.5 + heatmap_peak_thresh: return landmark if over threshold, default 0.1 + + """ + def __init__(self, face_model_path, + inference_size=None, + gaussian_sigma=None, + heatmap_peak_thresh=None): + self.inference_size = inference_size or params["inference_img_size"] + self.sigma = gaussian_sigma or params['gaussian_sigma'] + self.threshold = heatmap_peak_thresh or params["heatmap_peak_thresh"] + self.model = FaceNet() + self.model.load_state_dict(torch.load(face_model_path)) + if torch.cuda.is_available(): + self.model = self.model.cuda() + print('cuda') + self.model.eval() + + def __call__(self, face_img): + H, W, C = face_img.shape + + w_size = 384 + x_data = torch.from_numpy(util.smart_resize(face_img, (w_size, w_size))).permute([2, 0, 1]) / 256.0 - 0.5 + + if torch.cuda.is_available(): + x_data = x_data.cuda() + + with torch.no_grad(): + hs = self.model(x_data[None, ...]) + heatmaps = F.interpolate( + hs[-1], + (H, W), + mode='bilinear', align_corners=True).cpu().numpy()[0] + return heatmaps + + def compute_peaks_from_heatmaps(self, heatmaps): + all_peaks = [] + for part in range(heatmaps.shape[0]): + map_ori = heatmaps[part].copy() + binary = np.ascontiguousarray(map_ori > 0.05, dtype=np.uint8) + + if np.sum(binary) == 0: + continue + + positions = np.where(binary > 0.5) + intensities = map_ori[positions] + mi = np.argmax(intensities) + y, x = positions[0][mi], positions[1][mi] + all_peaks.append([x, y]) + + return np.array(all_peaks) diff --git a/annotator/openpose/hand.py b/annotator/openpose/hand.py new file mode 100644 index 0000000000000000000000000000000000000000..0b268d3c9e3077b6155df01fe7b7e5aae377dbe4 --- /dev/null +++ b/annotator/openpose/hand.py @@ -0,0 +1,93 @@ +import cv2 +import json +import numpy as np +import math +import time +from scipy.ndimage.filters import gaussian_filter +import matplotlib.pyplot as plt +import matplotlib +import torch +from skimage.measure import label + +from .model import handpose_model +from . import util + +class Hand(object): + def __init__(self, model_path): + self.model = handpose_model() + if torch.cuda.is_available(): + self.model = self.model.cuda() + print('cuda') + model_dict = util.transfer(self.model, torch.load(model_path)) + self.model.load_state_dict(model_dict) + self.model.eval() + + def __call__(self, oriImgRaw): + scale_search = [0.5, 1.0, 1.5, 2.0] + # scale_search = [0.5] + boxsize = 368 + stride = 8 + padValue = 128 + thre = 0.05 + multiplier = [x * boxsize for x in scale_search] + + wsize = 128 + heatmap_avg = np.zeros((wsize, wsize, 22)) + + Hr, Wr, Cr = oriImgRaw.shape + + oriImg = cv2.GaussianBlur(oriImgRaw, (0, 0), 0.8) + + for m in range(len(multiplier)): + scale = multiplier[m] + imageToTest = util.smart_resize(oriImg, (scale, scale)) + + imageToTest_padded, pad = util.padRightDownCorner(imageToTest, stride, padValue) + im = np.transpose(np.float32(imageToTest_padded[:, :, :, np.newaxis]), (3, 2, 0, 1)) / 256 - 0.5 + im = np.ascontiguousarray(im) + + data = torch.from_numpy(im).float() + if torch.cuda.is_available(): + data = data.cuda() + + with torch.no_grad(): + output = self.model(data).cpu().numpy() + + # extract outputs, resize, and remove padding + heatmap = np.transpose(np.squeeze(output), (1, 2, 0)) # output 1 is heatmaps + heatmap = util.smart_resize_k(heatmap, fx=stride, fy=stride) + heatmap = heatmap[:imageToTest_padded.shape[0] - pad[2], :imageToTest_padded.shape[1] - pad[3], :] + heatmap = util.smart_resize(heatmap, (wsize, wsize)) + + heatmap_avg += heatmap / len(multiplier) + + all_peaks = [] + for part in range(21): + map_ori = heatmap_avg[:, :, part] + one_heatmap = gaussian_filter(map_ori, sigma=3) + binary = np.ascontiguousarray(one_heatmap > thre, dtype=np.uint8) + + if np.sum(binary) == 0: + all_peaks.append([0, 0]) + continue + label_img, label_numbers = label(binary, return_num=True, connectivity=binary.ndim) + max_index = np.argmax([np.sum(map_ori[label_img == i]) for i in range(1, label_numbers + 1)]) + 1 + label_img[label_img != max_index] = 0 + map_ori[label_img == 0] = 0 + + y, x = util.npmax(map_ori) + y = int(float(y) * float(Hr) / float(wsize)) + x = int(float(x) * float(Wr) / float(wsize)) + all_peaks.append([x, y]) + return np.array(all_peaks) + +if __name__ == "__main__": + hand_estimation = Hand('../model/hand_pose_model.pth') + + # test_image = '../images/hand.jpg' + test_image = '../images/hand.jpg' + oriImg = cv2.imread(test_image) # B,G,R order + peaks = hand_estimation(oriImg) + canvas = util.draw_handpose(oriImg, peaks, True) + cv2.imshow('', canvas) + cv2.waitKey(0) \ No newline at end of file diff --git a/annotator/openpose/model.py b/annotator/openpose/model.py new file mode 100644 index 0000000000000000000000000000000000000000..5dfc80de827a17beccb9b0f3f7588545be78c9de --- /dev/null +++ b/annotator/openpose/model.py @@ -0,0 +1,219 @@ +import torch +from collections import OrderedDict + +import torch +import torch.nn as nn + +def make_layers(block, no_relu_layers): + layers = [] + for layer_name, v in block.items(): + if 'pool' in layer_name: + layer = nn.MaxPool2d(kernel_size=v[0], stride=v[1], + padding=v[2]) + layers.append((layer_name, layer)) + else: + conv2d = nn.Conv2d(in_channels=v[0], out_channels=v[1], + kernel_size=v[2], stride=v[3], + padding=v[4]) + layers.append((layer_name, conv2d)) + if layer_name not in no_relu_layers: + layers.append(('relu_'+layer_name, nn.ReLU(inplace=True))) + + return nn.Sequential(OrderedDict(layers)) + +class bodypose_model(nn.Module): + def __init__(self): + super(bodypose_model, self).__init__() + + # these layers have no relu layer + no_relu_layers = ['conv5_5_CPM_L1', 'conv5_5_CPM_L2', 'Mconv7_stage2_L1',\ + 'Mconv7_stage2_L2', 'Mconv7_stage3_L1', 'Mconv7_stage3_L2',\ + 'Mconv7_stage4_L1', 'Mconv7_stage4_L2', 'Mconv7_stage5_L1',\ + 'Mconv7_stage5_L2', 'Mconv7_stage6_L1', 'Mconv7_stage6_L1'] + blocks = {} + block0 = OrderedDict([ + ('conv1_1', [3, 64, 3, 1, 1]), + ('conv1_2', [64, 64, 3, 1, 1]), + ('pool1_stage1', [2, 2, 0]), + ('conv2_1', [64, 128, 3, 1, 1]), + ('conv2_2', [128, 128, 3, 1, 1]), + ('pool2_stage1', [2, 2, 0]), + ('conv3_1', [128, 256, 3, 1, 1]), + ('conv3_2', [256, 256, 3, 1, 1]), + ('conv3_3', [256, 256, 3, 1, 1]), + ('conv3_4', [256, 256, 3, 1, 1]), + ('pool3_stage1', [2, 2, 0]), + ('conv4_1', [256, 512, 3, 1, 1]), + ('conv4_2', [512, 512, 3, 1, 1]), + ('conv4_3_CPM', [512, 256, 3, 1, 1]), + ('conv4_4_CPM', [256, 128, 3, 1, 1]) + ]) + + + # Stage 1 + block1_1 = OrderedDict([ + ('conv5_1_CPM_L1', [128, 128, 3, 1, 1]), + ('conv5_2_CPM_L1', [128, 128, 3, 1, 1]), + ('conv5_3_CPM_L1', [128, 128, 3, 1, 1]), + ('conv5_4_CPM_L1', [128, 512, 1, 1, 0]), + ('conv5_5_CPM_L1', [512, 38, 1, 1, 0]) + ]) + + block1_2 = OrderedDict([ + ('conv5_1_CPM_L2', [128, 128, 3, 1, 1]), + ('conv5_2_CPM_L2', [128, 128, 3, 1, 1]), + ('conv5_3_CPM_L2', [128, 128, 3, 1, 1]), + ('conv5_4_CPM_L2', [128, 512, 1, 1, 0]), + ('conv5_5_CPM_L2', [512, 19, 1, 1, 0]) + ]) + blocks['block1_1'] = block1_1 + blocks['block1_2'] = block1_2 + + self.model0 = make_layers(block0, no_relu_layers) + + # Stages 2 - 6 + for i in range(2, 7): + blocks['block%d_1' % i] = OrderedDict([ + ('Mconv1_stage%d_L1' % i, [185, 128, 7, 1, 3]), + ('Mconv2_stage%d_L1' % i, [128, 128, 7, 1, 3]), + ('Mconv3_stage%d_L1' % i, [128, 128, 7, 1, 3]), + ('Mconv4_stage%d_L1' % i, [128, 128, 7, 1, 3]), + ('Mconv5_stage%d_L1' % i, [128, 128, 7, 1, 3]), + ('Mconv6_stage%d_L1' % i, [128, 128, 1, 1, 0]), + ('Mconv7_stage%d_L1' % i, [128, 38, 1, 1, 0]) + ]) + + blocks['block%d_2' % i] = OrderedDict([ + ('Mconv1_stage%d_L2' % i, [185, 128, 7, 1, 3]), + ('Mconv2_stage%d_L2' % i, [128, 128, 7, 1, 3]), + ('Mconv3_stage%d_L2' % i, [128, 128, 7, 1, 3]), + ('Mconv4_stage%d_L2' % i, [128, 128, 7, 1, 3]), + ('Mconv5_stage%d_L2' % i, [128, 128, 7, 1, 3]), + ('Mconv6_stage%d_L2' % i, [128, 128, 1, 1, 0]), + ('Mconv7_stage%d_L2' % i, [128, 19, 1, 1, 0]) + ]) + + for k in blocks.keys(): + blocks[k] = make_layers(blocks[k], no_relu_layers) + + self.model1_1 = blocks['block1_1'] + self.model2_1 = blocks['block2_1'] + self.model3_1 = blocks['block3_1'] + self.model4_1 = blocks['block4_1'] + self.model5_1 = blocks['block5_1'] + self.model6_1 = blocks['block6_1'] + + self.model1_2 = blocks['block1_2'] + self.model2_2 = blocks['block2_2'] + self.model3_2 = blocks['block3_2'] + self.model4_2 = blocks['block4_2'] + self.model5_2 = blocks['block5_2'] + self.model6_2 = blocks['block6_2'] + + + def forward(self, x): + + out1 = self.model0(x) + + out1_1 = self.model1_1(out1) + out1_2 = self.model1_2(out1) + out2 = torch.cat([out1_1, out1_2, out1], 1) + + out2_1 = self.model2_1(out2) + out2_2 = self.model2_2(out2) + out3 = torch.cat([out2_1, out2_2, out1], 1) + + out3_1 = self.model3_1(out3) + out3_2 = self.model3_2(out3) + out4 = torch.cat([out3_1, out3_2, out1], 1) + + out4_1 = self.model4_1(out4) + out4_2 = self.model4_2(out4) + out5 = torch.cat([out4_1, out4_2, out1], 1) + + out5_1 = self.model5_1(out5) + out5_2 = self.model5_2(out5) + out6 = torch.cat([out5_1, out5_2, out1], 1) + + out6_1 = self.model6_1(out6) + out6_2 = self.model6_2(out6) + + return out6_1, out6_2 + +class handpose_model(nn.Module): + def __init__(self): + super(handpose_model, self).__init__() + + # these layers have no relu layer + no_relu_layers = ['conv6_2_CPM', 'Mconv7_stage2', 'Mconv7_stage3',\ + 'Mconv7_stage4', 'Mconv7_stage5', 'Mconv7_stage6'] + # stage 1 + block1_0 = OrderedDict([ + ('conv1_1', [3, 64, 3, 1, 1]), + ('conv1_2', [64, 64, 3, 1, 1]), + ('pool1_stage1', [2, 2, 0]), + ('conv2_1', [64, 128, 3, 1, 1]), + ('conv2_2', [128, 128, 3, 1, 1]), + ('pool2_stage1', [2, 2, 0]), + ('conv3_1', [128, 256, 3, 1, 1]), + ('conv3_2', [256, 256, 3, 1, 1]), + ('conv3_3', [256, 256, 3, 1, 1]), + ('conv3_4', [256, 256, 3, 1, 1]), + ('pool3_stage1', [2, 2, 0]), + ('conv4_1', [256, 512, 3, 1, 1]), + ('conv4_2', [512, 512, 3, 1, 1]), + ('conv4_3', [512, 512, 3, 1, 1]), + ('conv4_4', [512, 512, 3, 1, 1]), + ('conv5_1', [512, 512, 3, 1, 1]), + ('conv5_2', [512, 512, 3, 1, 1]), + ('conv5_3_CPM', [512, 128, 3, 1, 1]) + ]) + + block1_1 = OrderedDict([ + ('conv6_1_CPM', [128, 512, 1, 1, 0]), + ('conv6_2_CPM', [512, 22, 1, 1, 0]) + ]) + + blocks = {} + blocks['block1_0'] = block1_0 + blocks['block1_1'] = block1_1 + + # stage 2-6 + for i in range(2, 7): + blocks['block%d' % i] = OrderedDict([ + ('Mconv1_stage%d' % i, [150, 128, 7, 1, 3]), + ('Mconv2_stage%d' % i, [128, 128, 7, 1, 3]), + ('Mconv3_stage%d' % i, [128, 128, 7, 1, 3]), + ('Mconv4_stage%d' % i, [128, 128, 7, 1, 3]), + ('Mconv5_stage%d' % i, [128, 128, 7, 1, 3]), + ('Mconv6_stage%d' % i, [128, 128, 1, 1, 0]), + ('Mconv7_stage%d' % i, [128, 22, 1, 1, 0]) + ]) + + for k in blocks.keys(): + blocks[k] = make_layers(blocks[k], no_relu_layers) + + self.model1_0 = blocks['block1_0'] + self.model1_1 = blocks['block1_1'] + self.model2 = blocks['block2'] + self.model3 = blocks['block3'] + self.model4 = blocks['block4'] + self.model5 = blocks['block5'] + self.model6 = blocks['block6'] + + def forward(self, x): + out1_0 = self.model1_0(x) + out1_1 = self.model1_1(out1_0) + concat_stage2 = torch.cat([out1_1, out1_0], 1) + out_stage2 = self.model2(concat_stage2) + concat_stage3 = torch.cat([out_stage2, out1_0], 1) + out_stage3 = self.model3(concat_stage3) + concat_stage4 = torch.cat([out_stage3, out1_0], 1) + out_stage4 = self.model4(concat_stage4) + concat_stage5 = torch.cat([out_stage4, out1_0], 1) + out_stage5 = self.model5(concat_stage5) + concat_stage6 = torch.cat([out_stage5, out1_0], 1) + out_stage6 = self.model6(concat_stage6) + return out_stage6 + + diff --git a/annotator/openpose/util.py b/annotator/openpose/util.py new file mode 100644 index 0000000000000000000000000000000000000000..73d7d0153b38d143eb8090e07a9784a274b619ed --- /dev/null +++ b/annotator/openpose/util.py @@ -0,0 +1,297 @@ +import math +import numpy as np +import matplotlib +import cv2 + + +eps = 0.01 + + +def smart_resize(x, s): + Ht, Wt = s + if x.ndim == 2: + Ho, Wo = x.shape + Co = 1 + else: + Ho, Wo, Co = x.shape + if Co == 3 or Co == 1: + k = float(Ht + Wt) / float(Ho + Wo) + return cv2.resize(x, (int(Wt), int(Ht)), interpolation=cv2.INTER_AREA if k < 1 else cv2.INTER_LANCZOS4) + else: + return np.stack([smart_resize(x[:, :, i], s) for i in range(Co)], axis=2) + + +def smart_resize_k(x, fx, fy): + if x.ndim == 2: + Ho, Wo = x.shape + Co = 1 + else: + Ho, Wo, Co = x.shape + Ht, Wt = Ho * fy, Wo * fx + if Co == 3 or Co == 1: + k = float(Ht + Wt) / float(Ho + Wo) + return cv2.resize(x, (int(Wt), int(Ht)), interpolation=cv2.INTER_AREA if k < 1 else cv2.INTER_LANCZOS4) + else: + return np.stack([smart_resize_k(x[:, :, i], fx, fy) for i in range(Co)], axis=2) + + +def padRightDownCorner(img, stride, padValue): + h = img.shape[0] + w = img.shape[1] + + pad = 4 * [None] + pad[0] = 0 # up + pad[1] = 0 # left + pad[2] = 0 if (h % stride == 0) else stride - (h % stride) # down + pad[3] = 0 if (w % stride == 0) else stride - (w % stride) # right + + img_padded = img + pad_up = np.tile(img_padded[0:1, :, :]*0 + padValue, (pad[0], 1, 1)) + img_padded = np.concatenate((pad_up, img_padded), axis=0) + pad_left = np.tile(img_padded[:, 0:1, :]*0 + padValue, (1, pad[1], 1)) + img_padded = np.concatenate((pad_left, img_padded), axis=1) + pad_down = np.tile(img_padded[-2:-1, :, :]*0 + padValue, (pad[2], 1, 1)) + img_padded = np.concatenate((img_padded, pad_down), axis=0) + pad_right = np.tile(img_padded[:, -2:-1, :]*0 + padValue, (1, pad[3], 1)) + img_padded = np.concatenate((img_padded, pad_right), axis=1) + + return img_padded, pad + + +def transfer(model, model_weights): + transfered_model_weights = {} + for weights_name in model.state_dict().keys(): + transfered_model_weights[weights_name] = model_weights['.'.join(weights_name.split('.')[1:])] + return transfered_model_weights + + +def draw_bodypose(canvas, candidate, subset): + H, W, C = canvas.shape + candidate = np.array(candidate) + subset = np.array(subset) + + stickwidth = 4 + + limbSeq = [[2, 3], [2, 6], [3, 4], [4, 5], [6, 7], [7, 8], [2, 9], [9, 10], \ + [10, 11], [2, 12], [12, 13], [13, 14], [2, 1], [1, 15], [15, 17], \ + [1, 16], [16, 18], [3, 17], [6, 18]] + + colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], \ + [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], \ + [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]] + + for i in range(17): + for n in range(len(subset)): + index = subset[n][np.array(limbSeq[i]) - 1] + if -1 in index: + continue + Y = candidate[index.astype(int), 0] * float(W) + X = candidate[index.astype(int), 1] * float(H) + mX = np.mean(X) + mY = np.mean(Y) + length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5 + angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1])) + polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 360, 1) + cv2.fillConvexPoly(canvas, polygon, colors[i]) + + canvas = (canvas * 0.6).astype(np.uint8) + + for i in range(18): + for n in range(len(subset)): + index = int(subset[n][i]) + if index == -1: + continue + x, y = candidate[index][0:2] + x = int(x * W) + y = int(y * H) + cv2.circle(canvas, (int(x), int(y)), 4, colors[i], thickness=-1) + + return canvas + + +def draw_handpose(canvas, all_hand_peaks): + H, W, C = canvas.shape + + edges = [[0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [6, 7], [7, 8], [0, 9], [9, 10], \ + [10, 11], [11, 12], [0, 13], [13, 14], [14, 15], [15, 16], [0, 17], [17, 18], [18, 19], [19, 20]] + + for peaks in all_hand_peaks: + peaks = np.array(peaks) + + for ie, e in enumerate(edges): + x1, y1 = peaks[e[0]] + x2, y2 = peaks[e[1]] + x1 = int(x1 * W) + y1 = int(y1 * H) + x2 = int(x2 * W) + y2 = int(y2 * H) + if x1 > eps and y1 > eps and x2 > eps and y2 > eps: + cv2.line(canvas, (x1, y1), (x2, y2), matplotlib.colors.hsv_to_rgb([ie / float(len(edges)), 1.0, 1.0]) * 255, thickness=2) + + for i, keyponit in enumerate(peaks): + x, y = keyponit + x = int(x * W) + y = int(y * H) + if x > eps and y > eps: + cv2.circle(canvas, (x, y), 4, (0, 0, 255), thickness=-1) + return canvas + + +def draw_facepose(canvas, all_lmks): + H, W, C = canvas.shape + for lmks in all_lmks: + lmks = np.array(lmks) + for lmk in lmks: + x, y = lmk + x = int(x * W) + y = int(y * H) + if x > eps and y > eps: + cv2.circle(canvas, (x, y), 3, (255, 255, 255), thickness=-1) + return canvas + + +# detect hand according to body pose keypoints +# please refer to https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/src/openpose/hand/handDetector.cpp +def handDetect(candidate, subset, oriImg): + # right hand: wrist 4, elbow 3, shoulder 2 + # left hand: wrist 7, elbow 6, shoulder 5 + ratioWristElbow = 0.33 + detect_result = [] + image_height, image_width = oriImg.shape[0:2] + for person in subset.astype(int): + # if any of three not detected + has_left = np.sum(person[[5, 6, 7]] == -1) == 0 + has_right = np.sum(person[[2, 3, 4]] == -1) == 0 + if not (has_left or has_right): + continue + hands = [] + #left hand + if has_left: + left_shoulder_index, left_elbow_index, left_wrist_index = person[[5, 6, 7]] + x1, y1 = candidate[left_shoulder_index][:2] + x2, y2 = candidate[left_elbow_index][:2] + x3, y3 = candidate[left_wrist_index][:2] + hands.append([x1, y1, x2, y2, x3, y3, True]) + # right hand + if has_right: + right_shoulder_index, right_elbow_index, right_wrist_index = person[[2, 3, 4]] + x1, y1 = candidate[right_shoulder_index][:2] + x2, y2 = candidate[right_elbow_index][:2] + x3, y3 = candidate[right_wrist_index][:2] + hands.append([x1, y1, x2, y2, x3, y3, False]) + + for x1, y1, x2, y2, x3, y3, is_left in hands: + # pos_hand = pos_wrist + ratio * (pos_wrist - pos_elbox) = (1 + ratio) * pos_wrist - ratio * pos_elbox + # handRectangle.x = posePtr[wrist*3] + ratioWristElbow * (posePtr[wrist*3] - posePtr[elbow*3]); + # handRectangle.y = posePtr[wrist*3+1] + ratioWristElbow * (posePtr[wrist*3+1] - posePtr[elbow*3+1]); + # const auto distanceWristElbow = getDistance(poseKeypoints, person, wrist, elbow); + # const auto distanceElbowShoulder = getDistance(poseKeypoints, person, elbow, shoulder); + # handRectangle.width = 1.5f * fastMax(distanceWristElbow, 0.9f * distanceElbowShoulder); + x = x3 + ratioWristElbow * (x3 - x2) + y = y3 + ratioWristElbow * (y3 - y2) + distanceWristElbow = math.sqrt((x3 - x2) ** 2 + (y3 - y2) ** 2) + distanceElbowShoulder = math.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2) + width = 1.5 * max(distanceWristElbow, 0.9 * distanceElbowShoulder) + # x-y refers to the center --> offset to topLeft point + # handRectangle.x -= handRectangle.width / 2.f; + # handRectangle.y -= handRectangle.height / 2.f; + x -= width / 2 + y -= width / 2 # width = height + # overflow the image + if x < 0: x = 0 + if y < 0: y = 0 + width1 = width + width2 = width + if x + width > image_width: width1 = image_width - x + if y + width > image_height: width2 = image_height - y + width = min(width1, width2) + # the max hand box value is 20 pixels + if width >= 20: + detect_result.append([int(x), int(y), int(width), is_left]) + + ''' + return value: [[x, y, w, True if left hand else False]]. + width=height since the network require squared input. + x, y is the coordinate of top left + ''' + return detect_result + + +# Written by Lvmin +def faceDetect(candidate, subset, oriImg): + # left right eye ear 14 15 16 17 + detect_result = [] + image_height, image_width = oriImg.shape[0:2] + for person in subset.astype(int): + has_head = person[0] > -1 + if not has_head: + continue + + has_left_eye = person[14] > -1 + has_right_eye = person[15] > -1 + has_left_ear = person[16] > -1 + has_right_ear = person[17] > -1 + + if not (has_left_eye or has_right_eye or has_left_ear or has_right_ear): + continue + + head, left_eye, right_eye, left_ear, right_ear = person[[0, 14, 15, 16, 17]] + + width = 0.0 + x0, y0 = candidate[head][:2] + + if has_left_eye: + x1, y1 = candidate[left_eye][:2] + d = max(abs(x0 - x1), abs(y0 - y1)) + width = max(width, d * 3.0) + + if has_right_eye: + x1, y1 = candidate[right_eye][:2] + d = max(abs(x0 - x1), abs(y0 - y1)) + width = max(width, d * 3.0) + + if has_left_ear: + x1, y1 = candidate[left_ear][:2] + d = max(abs(x0 - x1), abs(y0 - y1)) + width = max(width, d * 1.5) + + if has_right_ear: + x1, y1 = candidate[right_ear][:2] + d = max(abs(x0 - x1), abs(y0 - y1)) + width = max(width, d * 1.5) + + x, y = x0, y0 + + x -= width + y -= width + + if x < 0: + x = 0 + + if y < 0: + y = 0 + + width1 = width * 2 + width2 = width * 2 + + if x + width > image_width: + width1 = image_width - x + + if y + width > image_height: + width2 = image_height - y + + width = min(width1, width2) + + if width >= 20: + detect_result.append([int(x), int(y), int(width)]) + + return detect_result + + +# get max index of 2d array +def npmax(array): + arrayindex = array.argmax(1) + arrayvalue = array.max(1) + i = arrayvalue.argmax() + j = arrayindex[i] + return i, j diff --git a/annotator/pidinet/LICENSE b/annotator/pidinet/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..913b6cf92c19d37b6ee4f7bc99c65f655e7f840c --- /dev/null +++ b/annotator/pidinet/LICENSE @@ -0,0 +1,21 @@ +It is just for research purpose, and commercial use should be contacted with authors first. + +Copyright (c) 2021 Zhuo Su + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/annotator/pidinet/__init__.py b/annotator/pidinet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4f3e3d9038c068f69c56a1fbb6e51b6b11faa0fd --- /dev/null +++ b/annotator/pidinet/__init__.py @@ -0,0 +1,39 @@ +# Pidinet +# https://github.com/hellozhuo/pidinet + +import os +import torch +import numpy as np +from einops import rearrange +from annotator.pidinet.model import pidinet +from annotator.util import annotator_ckpts_path, safe_step + + +class PidiNetDetector: + def __init__(self): + remote_model_path = "https://huggingface.co./lllyasviel/Annotators/resolve/main/table5_pidinet.pth" + modelpath = os.path.join(annotator_ckpts_path, "table5_pidinet.pth") + if not os.path.exists(modelpath): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(remote_model_path, model_dir=annotator_ckpts_path) + self.netNetwork = pidinet() +# self.netNetwork.load_state_dict({k.replace('module.', ''): v for k, v in torch.load(modelpath)['state_dict'].items()}) + self.netNetwork.load_state_dict({k.replace('module.', ''): v for k, v in torch.load(modelpath, map_location=torch.device('cpu'))['state_dict'].items()}) +# self.netNetwork = self.netNetwork.cuda() + self.netNetwork = self.netNetwork.cpu() + self.netNetwork.eval() + + def __call__(self, input_image, safe=False): + assert input_image.ndim == 3 + input_image = input_image[:, :, ::-1].copy() + with torch.no_grad(): +# image_pidi = torch.from_numpy(input_image).float().cuda() + image_pidi = torch.from_numpy(input_image).float().cpu() + image_pidi = image_pidi / 255.0 + image_pidi = rearrange(image_pidi, 'h w c -> 1 c h w') + edge = self.netNetwork(image_pidi)[-1] + edge = edge.cpu().numpy() + if safe: + edge = safe_step(edge) + edge = (edge * 255.0).clip(0, 255).astype(np.uint8) + return edge[0][0] diff --git a/annotator/pidinet/model.py b/annotator/pidinet/model.py new file mode 100644 index 0000000000000000000000000000000000000000..d7dc6b8eccd945640e1aa8c35916df79a2fa69f9 --- /dev/null +++ b/annotator/pidinet/model.py @@ -0,0 +1,654 @@ +""" +Author: Zhuo Su, Wenzhe Liu +Date: Feb 18, 2021 +""" + +import math + +import cv2 +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from basicsr.utils import img2tensor + +nets = { + 'baseline': { + 'layer0': 'cv', + 'layer1': 'cv', + 'layer2': 'cv', + 'layer3': 'cv', + 'layer4': 'cv', + 'layer5': 'cv', + 'layer6': 'cv', + 'layer7': 'cv', + 'layer8': 'cv', + 'layer9': 'cv', + 'layer10': 'cv', + 'layer11': 'cv', + 'layer12': 'cv', + 'layer13': 'cv', + 'layer14': 'cv', + 'layer15': 'cv', + }, + 'c-v15': { + 'layer0': 'cd', + 'layer1': 'cv', + 'layer2': 'cv', + 'layer3': 'cv', + 'layer4': 'cv', + 'layer5': 'cv', + 'layer6': 'cv', + 'layer7': 'cv', + 'layer8': 'cv', + 'layer9': 'cv', + 'layer10': 'cv', + 'layer11': 'cv', + 'layer12': 'cv', + 'layer13': 'cv', + 'layer14': 'cv', + 'layer15': 'cv', + }, + 'a-v15': { + 'layer0': 'ad', + 'layer1': 'cv', + 'layer2': 'cv', + 'layer3': 'cv', + 'layer4': 'cv', + 'layer5': 'cv', + 'layer6': 'cv', + 'layer7': 'cv', + 'layer8': 'cv', + 'layer9': 'cv', + 'layer10': 'cv', + 'layer11': 'cv', + 'layer12': 'cv', + 'layer13': 'cv', + 'layer14': 'cv', + 'layer15': 'cv', + }, + 'r-v15': { + 'layer0': 'rd', + 'layer1': 'cv', + 'layer2': 'cv', + 'layer3': 'cv', + 'layer4': 'cv', + 'layer5': 'cv', + 'layer6': 'cv', + 'layer7': 'cv', + 'layer8': 'cv', + 'layer9': 'cv', + 'layer10': 'cv', + 'layer11': 'cv', + 'layer12': 'cv', + 'layer13': 'cv', + 'layer14': 'cv', + 'layer15': 'cv', + }, + 'cvvv4': { + 'layer0': 'cd', + 'layer1': 'cv', + 'layer2': 'cv', + 'layer3': 'cv', + 'layer4': 'cd', + 'layer5': 'cv', + 'layer6': 'cv', + 'layer7': 'cv', + 'layer8': 'cd', + 'layer9': 'cv', + 'layer10': 'cv', + 'layer11': 'cv', + 'layer12': 'cd', + 'layer13': 'cv', + 'layer14': 'cv', + 'layer15': 'cv', + }, + 'avvv4': { + 'layer0': 'ad', + 'layer1': 'cv', + 'layer2': 'cv', + 'layer3': 'cv', + 'layer4': 'ad', + 'layer5': 'cv', + 'layer6': 'cv', + 'layer7': 'cv', + 'layer8': 'ad', + 'layer9': 'cv', + 'layer10': 'cv', + 'layer11': 'cv', + 'layer12': 'ad', + 'layer13': 'cv', + 'layer14': 'cv', + 'layer15': 'cv', + }, + 'rvvv4': { + 'layer0': 'rd', + 'layer1': 'cv', + 'layer2': 'cv', + 'layer3': 'cv', + 'layer4': 'rd', + 'layer5': 'cv', + 'layer6': 'cv', + 'layer7': 'cv', + 'layer8': 'rd', + 'layer9': 'cv', + 'layer10': 'cv', + 'layer11': 'cv', + 'layer12': 'rd', + 'layer13': 'cv', + 'layer14': 'cv', + 'layer15': 'cv', + }, + 'cccv4': { + 'layer0': 'cd', + 'layer1': 'cd', + 'layer2': 'cd', + 'layer3': 'cv', + 'layer4': 'cd', + 'layer5': 'cd', + 'layer6': 'cd', + 'layer7': 'cv', + 'layer8': 'cd', + 'layer9': 'cd', + 'layer10': 'cd', + 'layer11': 'cv', + 'layer12': 'cd', + 'layer13': 'cd', + 'layer14': 'cd', + 'layer15': 'cv', + }, + 'aaav4': { + 'layer0': 'ad', + 'layer1': 'ad', + 'layer2': 'ad', + 'layer3': 'cv', + 'layer4': 'ad', + 'layer5': 'ad', + 'layer6': 'ad', + 'layer7': 'cv', + 'layer8': 'ad', + 'layer9': 'ad', + 'layer10': 'ad', + 'layer11': 'cv', + 'layer12': 'ad', + 'layer13': 'ad', + 'layer14': 'ad', + 'layer15': 'cv', + }, + 'rrrv4': { + 'layer0': 'rd', + 'layer1': 'rd', + 'layer2': 'rd', + 'layer3': 'cv', + 'layer4': 'rd', + 'layer5': 'rd', + 'layer6': 'rd', + 'layer7': 'cv', + 'layer8': 'rd', + 'layer9': 'rd', + 'layer10': 'rd', + 'layer11': 'cv', + 'layer12': 'rd', + 'layer13': 'rd', + 'layer14': 'rd', + 'layer15': 'cv', + }, + 'c16': { + 'layer0': 'cd', + 'layer1': 'cd', + 'layer2': 'cd', + 'layer3': 'cd', + 'layer4': 'cd', + 'layer5': 'cd', + 'layer6': 'cd', + 'layer7': 'cd', + 'layer8': 'cd', + 'layer9': 'cd', + 'layer10': 'cd', + 'layer11': 'cd', + 'layer12': 'cd', + 'layer13': 'cd', + 'layer14': 'cd', + 'layer15': 'cd', + }, + 'a16': { + 'layer0': 'ad', + 'layer1': 'ad', + 'layer2': 'ad', + 'layer3': 'ad', + 'layer4': 'ad', + 'layer5': 'ad', + 'layer6': 'ad', + 'layer7': 'ad', + 'layer8': 'ad', + 'layer9': 'ad', + 'layer10': 'ad', + 'layer11': 'ad', + 'layer12': 'ad', + 'layer13': 'ad', + 'layer14': 'ad', + 'layer15': 'ad', + }, + 'r16': { + 'layer0': 'rd', + 'layer1': 'rd', + 'layer2': 'rd', + 'layer3': 'rd', + 'layer4': 'rd', + 'layer5': 'rd', + 'layer6': 'rd', + 'layer7': 'rd', + 'layer8': 'rd', + 'layer9': 'rd', + 'layer10': 'rd', + 'layer11': 'rd', + 'layer12': 'rd', + 'layer13': 'rd', + 'layer14': 'rd', + 'layer15': 'rd', + }, + 'carv4': { + 'layer0': 'cd', + 'layer1': 'ad', + 'layer2': 'rd', + 'layer3': 'cv', + 'layer4': 'cd', + 'layer5': 'ad', + 'layer6': 'rd', + 'layer7': 'cv', + 'layer8': 'cd', + 'layer9': 'ad', + 'layer10': 'rd', + 'layer11': 'cv', + 'layer12': 'cd', + 'layer13': 'ad', + 'layer14': 'rd', + 'layer15': 'cv', + }, + } + +def createConvFunc(op_type): + assert op_type in ['cv', 'cd', 'ad', 'rd'], 'unknown op type: %s' % str(op_type) + if op_type == 'cv': + return F.conv2d + + if op_type == 'cd': + def func(x, weights, bias=None, stride=1, padding=0, dilation=1, groups=1): + assert dilation in [1, 2], 'dilation for cd_conv should be in 1 or 2' + assert weights.size(2) == 3 and weights.size(3) == 3, 'kernel size for cd_conv should be 3x3' + assert padding == dilation, 'padding for cd_conv set wrong' + + weights_c = weights.sum(dim=[2, 3], keepdim=True) + yc = F.conv2d(x, weights_c, stride=stride, padding=0, groups=groups) + y = F.conv2d(x, weights, bias, stride=stride, padding=padding, dilation=dilation, groups=groups) + return y - yc + return func + elif op_type == 'ad': + def func(x, weights, bias=None, stride=1, padding=0, dilation=1, groups=1): + assert dilation in [1, 2], 'dilation for ad_conv should be in 1 or 2' + assert weights.size(2) == 3 and weights.size(3) == 3, 'kernel size for ad_conv should be 3x3' + assert padding == dilation, 'padding for ad_conv set wrong' + + shape = weights.shape + weights = weights.view(shape[0], shape[1], -1) + weights_conv = (weights - weights[:, :, [3, 0, 1, 6, 4, 2, 7, 8, 5]]).view(shape) # clock-wise + y = F.conv2d(x, weights_conv, bias, stride=stride, padding=padding, dilation=dilation, groups=groups) + return y + return func + elif op_type == 'rd': + def func(x, weights, bias=None, stride=1, padding=0, dilation=1, groups=1): + assert dilation in [1, 2], 'dilation for rd_conv should be in 1 or 2' + assert weights.size(2) == 3 and weights.size(3) == 3, 'kernel size for rd_conv should be 3x3' + padding = 2 * dilation + + shape = weights.shape + if weights.is_cuda: + buffer = torch.cuda.FloatTensor(shape[0], shape[1], 5 * 5).fill_(0) + else: + buffer = torch.zeros(shape[0], shape[1], 5 * 5) + weights = weights.view(shape[0], shape[1], -1) + buffer[:, :, [0, 2, 4, 10, 14, 20, 22, 24]] = weights[:, :, 1:] + buffer[:, :, [6, 7, 8, 11, 13, 16, 17, 18]] = -weights[:, :, 1:] + buffer[:, :, 12] = 0 + buffer = buffer.view(shape[0], shape[1], 5, 5) + y = F.conv2d(x, buffer, bias, stride=stride, padding=padding, dilation=dilation, groups=groups) + return y + return func + else: + print('impossible to be here unless you force that') + return None + +class Conv2d(nn.Module): + def __init__(self, pdc, in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, bias=False): + super(Conv2d, self).__init__() + if in_channels % groups != 0: + raise ValueError('in_channels must be divisible by groups') + if out_channels % groups != 0: + raise ValueError('out_channels must be divisible by groups') + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = kernel_size + self.stride = stride + self.padding = padding + self.dilation = dilation + self.groups = groups + self.weight = nn.Parameter(torch.Tensor(out_channels, in_channels // groups, kernel_size, kernel_size)) + if bias: + self.bias = nn.Parameter(torch.Tensor(out_channels)) + else: + self.register_parameter('bias', None) + self.reset_parameters() + self.pdc = pdc + + def reset_parameters(self): + nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + if self.bias is not None: + fan_in, _ = nn.init._calculate_fan_in_and_fan_out(self.weight) + bound = 1 / math.sqrt(fan_in) + nn.init.uniform_(self.bias, -bound, bound) + + def forward(self, input): + + return self.pdc(input, self.weight, self.bias, self.stride, self.padding, self.dilation, self.groups) + +class CSAM(nn.Module): + """ + Compact Spatial Attention Module + """ + def __init__(self, channels): + super(CSAM, self).__init__() + + mid_channels = 4 + self.relu1 = nn.ReLU() + self.conv1 = nn.Conv2d(channels, mid_channels, kernel_size=1, padding=0) + self.conv2 = nn.Conv2d(mid_channels, 1, kernel_size=3, padding=1, bias=False) + self.sigmoid = nn.Sigmoid() + nn.init.constant_(self.conv1.bias, 0) + + def forward(self, x): + y = self.relu1(x) + y = self.conv1(y) + y = self.conv2(y) + y = self.sigmoid(y) + + return x * y + +class CDCM(nn.Module): + """ + Compact Dilation Convolution based Module + """ + def __init__(self, in_channels, out_channels): + super(CDCM, self).__init__() + + self.relu1 = nn.ReLU() + self.conv1 = nn.Conv2d(in_channels, out_channels, kernel_size=1, padding=0) + self.conv2_1 = nn.Conv2d(out_channels, out_channels, kernel_size=3, dilation=5, padding=5, bias=False) + self.conv2_2 = nn.Conv2d(out_channels, out_channels, kernel_size=3, dilation=7, padding=7, bias=False) + self.conv2_3 = nn.Conv2d(out_channels, out_channels, kernel_size=3, dilation=9, padding=9, bias=False) + self.conv2_4 = nn.Conv2d(out_channels, out_channels, kernel_size=3, dilation=11, padding=11, bias=False) + nn.init.constant_(self.conv1.bias, 0) + + def forward(self, x): + x = self.relu1(x) + x = self.conv1(x) + x1 = self.conv2_1(x) + x2 = self.conv2_2(x) + x3 = self.conv2_3(x) + x4 = self.conv2_4(x) + return x1 + x2 + x3 + x4 + + +class MapReduce(nn.Module): + """ + Reduce feature maps into a single edge map + """ + def __init__(self, channels): + super(MapReduce, self).__init__() + self.conv = nn.Conv2d(channels, 1, kernel_size=1, padding=0) + nn.init.constant_(self.conv.bias, 0) + + def forward(self, x): + return self.conv(x) + + +class PDCBlock(nn.Module): + def __init__(self, pdc, inplane, ouplane, stride=1): + super(PDCBlock, self).__init__() + self.stride=stride + + self.stride=stride + if self.stride > 1: + self.pool = nn.MaxPool2d(kernel_size=2, stride=2) + self.shortcut = nn.Conv2d(inplane, ouplane, kernel_size=1, padding=0) + self.conv1 = Conv2d(pdc, inplane, inplane, kernel_size=3, padding=1, groups=inplane, bias=False) + self.relu2 = nn.ReLU() + self.conv2 = nn.Conv2d(inplane, ouplane, kernel_size=1, padding=0, bias=False) + + def forward(self, x): + if self.stride > 1: + x = self.pool(x) + y = self.conv1(x) + y = self.relu2(y) + y = self.conv2(y) + if self.stride > 1: + x = self.shortcut(x) + y = y + x + return y + +class PDCBlock_converted(nn.Module): + """ + CPDC, APDC can be converted to vanilla 3x3 convolution + RPDC can be converted to vanilla 5x5 convolution + """ + def __init__(self, pdc, inplane, ouplane, stride=1): + super(PDCBlock_converted, self).__init__() + self.stride=stride + + if self.stride > 1: + self.pool = nn.MaxPool2d(kernel_size=2, stride=2) + self.shortcut = nn.Conv2d(inplane, ouplane, kernel_size=1, padding=0) + if pdc == 'rd': + self.conv1 = nn.Conv2d(inplane, inplane, kernel_size=5, padding=2, groups=inplane, bias=False) + else: + self.conv1 = nn.Conv2d(inplane, inplane, kernel_size=3, padding=1, groups=inplane, bias=False) + self.relu2 = nn.ReLU() + self.conv2 = nn.Conv2d(inplane, ouplane, kernel_size=1, padding=0, bias=False) + + def forward(self, x): + if self.stride > 1: + x = self.pool(x) + y = self.conv1(x) + y = self.relu2(y) + y = self.conv2(y) + if self.stride > 1: + x = self.shortcut(x) + y = y + x + return y + +class PiDiNet(nn.Module): + def __init__(self, inplane, pdcs, dil=None, sa=False, convert=False): + super(PiDiNet, self).__init__() + self.sa = sa + if dil is not None: + assert isinstance(dil, int), 'dil should be an int' + self.dil = dil + + self.fuseplanes = [] + + self.inplane = inplane + if convert: + if pdcs[0] == 'rd': + init_kernel_size = 5 + init_padding = 2 + else: + init_kernel_size = 3 + init_padding = 1 + self.init_block = nn.Conv2d(3, self.inplane, + kernel_size=init_kernel_size, padding=init_padding, bias=False) + block_class = PDCBlock_converted + else: + self.init_block = Conv2d(pdcs[0], 3, self.inplane, kernel_size=3, padding=1) + block_class = PDCBlock + + self.block1_1 = block_class(pdcs[1], self.inplane, self.inplane) + self.block1_2 = block_class(pdcs[2], self.inplane, self.inplane) + self.block1_3 = block_class(pdcs[3], self.inplane, self.inplane) + self.fuseplanes.append(self.inplane) # C + + inplane = self.inplane + self.inplane = self.inplane * 2 + self.block2_1 = block_class(pdcs[4], inplane, self.inplane, stride=2) + self.block2_2 = block_class(pdcs[5], self.inplane, self.inplane) + self.block2_3 = block_class(pdcs[6], self.inplane, self.inplane) + self.block2_4 = block_class(pdcs[7], self.inplane, self.inplane) + self.fuseplanes.append(self.inplane) # 2C + + inplane = self.inplane + self.inplane = self.inplane * 2 + self.block3_1 = block_class(pdcs[8], inplane, self.inplane, stride=2) + self.block3_2 = block_class(pdcs[9], self.inplane, self.inplane) + self.block3_3 = block_class(pdcs[10], self.inplane, self.inplane) + self.block3_4 = block_class(pdcs[11], self.inplane, self.inplane) + self.fuseplanes.append(self.inplane) # 4C + + self.block4_1 = block_class(pdcs[12], self.inplane, self.inplane, stride=2) + self.block4_2 = block_class(pdcs[13], self.inplane, self.inplane) + self.block4_3 = block_class(pdcs[14], self.inplane, self.inplane) + self.block4_4 = block_class(pdcs[15], self.inplane, self.inplane) + self.fuseplanes.append(self.inplane) # 4C + + self.conv_reduces = nn.ModuleList() + if self.sa and self.dil is not None: + self.attentions = nn.ModuleList() + self.dilations = nn.ModuleList() + for i in range(4): + self.dilations.append(CDCM(self.fuseplanes[i], self.dil)) + self.attentions.append(CSAM(self.dil)) + self.conv_reduces.append(MapReduce(self.dil)) + elif self.sa: + self.attentions = nn.ModuleList() + for i in range(4): + self.attentions.append(CSAM(self.fuseplanes[i])) + self.conv_reduces.append(MapReduce(self.fuseplanes[i])) + elif self.dil is not None: + self.dilations = nn.ModuleList() + for i in range(4): + self.dilations.append(CDCM(self.fuseplanes[i], self.dil)) + self.conv_reduces.append(MapReduce(self.dil)) + else: + for i in range(4): + self.conv_reduces.append(MapReduce(self.fuseplanes[i])) + + self.classifier = nn.Conv2d(4, 1, kernel_size=1) # has bias + nn.init.constant_(self.classifier.weight, 0.25) + nn.init.constant_(self.classifier.bias, 0) + + # print('initialization done') + + def get_weights(self): + conv_weights = [] + bn_weights = [] + relu_weights = [] + for pname, p in self.named_parameters(): + if 'bn' in pname: + bn_weights.append(p) + elif 'relu' in pname: + relu_weights.append(p) + else: + conv_weights.append(p) + + return conv_weights, bn_weights, relu_weights + + def forward(self, x): + H, W = x.size()[2:] + + x = self.init_block(x) + + x1 = self.block1_1(x) + x1 = self.block1_2(x1) + x1 = self.block1_3(x1) + + x2 = self.block2_1(x1) + x2 = self.block2_2(x2) + x2 = self.block2_3(x2) + x2 = self.block2_4(x2) + + x3 = self.block3_1(x2) + x3 = self.block3_2(x3) + x3 = self.block3_3(x3) + x3 = self.block3_4(x3) + + x4 = self.block4_1(x3) + x4 = self.block4_2(x4) + x4 = self.block4_3(x4) + x4 = self.block4_4(x4) + + x_fuses = [] + if self.sa and self.dil is not None: + for i, xi in enumerate([x1, x2, x3, x4]): + x_fuses.append(self.attentions[i](self.dilations[i](xi))) + elif self.sa: + for i, xi in enumerate([x1, x2, x3, x4]): + x_fuses.append(self.attentions[i](xi)) + elif self.dil is not None: + for i, xi in enumerate([x1, x2, x3, x4]): + x_fuses.append(self.dilations[i](xi)) + else: + x_fuses = [x1, x2, x3, x4] + + e1 = self.conv_reduces[0](x_fuses[0]) + e1 = F.interpolate(e1, (H, W), mode="bilinear", align_corners=False) + + e2 = self.conv_reduces[1](x_fuses[1]) + e2 = F.interpolate(e2, (H, W), mode="bilinear", align_corners=False) + + e3 = self.conv_reduces[2](x_fuses[2]) + e3 = F.interpolate(e3, (H, W), mode="bilinear", align_corners=False) + + e4 = self.conv_reduces[3](x_fuses[3]) + e4 = F.interpolate(e4, (H, W), mode="bilinear", align_corners=False) + + outputs = [e1, e2, e3, e4] + + output = self.classifier(torch.cat(outputs, dim=1)) + #if not self.training: + # return torch.sigmoid(output) + + outputs.append(output) + outputs = [torch.sigmoid(r) for r in outputs] + return outputs + +def config_model(model): + model_options = list(nets.keys()) + assert model in model_options, \ + 'unrecognized model, please choose from %s' % str(model_options) + + # print(str(nets[model])) + + pdcs = [] + for i in range(16): + layer_name = 'layer%d' % i + op = nets[model][layer_name] + pdcs.append(createConvFunc(op)) + + return pdcs + +def pidinet(): + pdcs = config_model('carv4') + dil = 24 #if args.dil else None + return PiDiNet(60, pdcs, dil=dil, sa=True) + + +if __name__ == '__main__': + model = pidinet() +# ckp = torch.load('table5_pidinet.pth')['state_dict'] + ckp = torch.load('table5_pidinet.pth', map_location=torch.device('cpu'))['state_dict'] + model.load_state_dict({k.replace('module.',''):v for k, v in ckp.items()}) + im = cv2.imread('examples/test_my/cat_v4.png') + im = img2tensor(im).unsqueeze(0)/255. + res = model(im)[-1] + res = res>0.5 + res = res.float() + res = (res[0,0].cpu().data.numpy()*255.).astype(np.uint8) + print(res.shape) + cv2.imwrite('edge.png', res) \ No newline at end of file diff --git a/annotator/shuffle/__init__.py b/annotator/shuffle/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1acd4e2648f25eee96a397084ace2c9a078a2e8e --- /dev/null +++ b/annotator/shuffle/__init__.py @@ -0,0 +1,74 @@ +import random + +import cv2 +import numpy as np +from annotator.util import make_noise_disk, img2mask + + +class ContentShuffleDetector: + def __call__(self, img, h=None, w=None, f=None): + H, W, C = img.shape + if h is None: + h = H + if w is None: + w = W + if f is None: + f = 256 + x = make_noise_disk(h, w, 1, f) * float(W - 1) + y = make_noise_disk(h, w, 1, f) * float(H - 1) + flow = np.concatenate([x, y], axis=2).astype(np.float32) + return cv2.remap(img, flow, None, cv2.INTER_LINEAR) + + +class ColorShuffleDetector: + def __call__(self, img): + H, W, C = img.shape + F = random.randint(64, 384) + A = make_noise_disk(H, W, 3, F) + B = make_noise_disk(H, W, 3, F) + C = (A + B) / 2.0 + A = (C + (A - C) * 3.0).clip(0, 1) + B = (C + (B - C) * 3.0).clip(0, 1) + L = img.astype(np.float32) / 255.0 + Y = A * L + B * (1 - L) + Y -= np.min(Y, axis=(0, 1), keepdims=True) + Y /= np.maximum(np.max(Y, axis=(0, 1), keepdims=True), 1e-5) + Y *= 255.0 + return Y.clip(0, 255).astype(np.uint8) + + +class GrayDetector: + def __call__(self, img): + eps = 1e-5 + X = img.astype(np.float32) + r, g, b = X[:, :, 0], X[:, :, 1], X[:, :, 2] + kr, kg, kb = [random.random() + eps for _ in range(3)] + ks = kr + kg + kb + kr /= ks + kg /= ks + kb /= ks + Y = r * kr + g * kg + b * kb + Y = np.stack([Y] * 3, axis=2) + return Y.clip(0, 255).astype(np.uint8) + + +class DownSampleDetector: + def __call__(self, img, level=3, k=16.0): + h = img.astype(np.float32) + for _ in range(level): + h += np.random.normal(loc=0.0, scale=k, size=h.shape) + h = cv2.pyrDown(h) + for _ in range(level): + h = cv2.pyrUp(h) + h += np.random.normal(loc=0.0, scale=k, size=h.shape) + return h.clip(0, 255).astype(np.uint8) + + +class Image2MaskShuffleDetector: + def __init__(self, resolution=(640, 512)): + self.H, self.W = resolution + + def __call__(self, img): + m = img2mask(img, self.H, self.W) + m *= 255.0 + return m.clip(0, 255).astype(np.uint8) diff --git a/annotator/uniformer/LICENSE b/annotator/uniformer/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..c38dc639e6e238fbf59608f80b3a6ff1928ac429 --- /dev/null +++ b/annotator/uniformer/LICENSE @@ -0,0 +1,203 @@ +Copyright 2022 SenseTime X-Lab. All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2022 SenseTime X-Lab. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/annotator/uniformer/__init__.py b/annotator/uniformer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..79e88fd41d5a98bab1017d0341a769682eb1887f --- /dev/null +++ b/annotator/uniformer/__init__.py @@ -0,0 +1,27 @@ +# Uniformer +# From https://github.com/Sense-X/UniFormer +# # Apache-2.0 license + +import os + +from annotator.uniformer.mmseg.apis import init_segmentor, inference_segmentor, show_result_pyplot +from annotator.uniformer.mmseg.core.evaluation import get_palette +from annotator.util import annotator_ckpts_path + + +checkpoint_file = "https://huggingface.co./lllyasviel/Annotators/resolve/main/upernet_global_small.pth" + + +class UniformerDetector: + def __init__(self): + modelpath = os.path.join(annotator_ckpts_path, "upernet_global_small.pth") + if not os.path.exists(modelpath): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(checkpoint_file, model_dir=annotator_ckpts_path) + config_file = os.path.join(os.path.dirname(annotator_ckpts_path), "uniformer", "exp", "upernet_global_small", "config.py") + self.model = init_segmentor(config_file, modelpath).cuda() + + def __call__(self, img): + result = inference_segmentor(self.model, img) + res_img = show_result_pyplot(self.model, img, result, get_palette('ade'), opacity=1) + return res_img diff --git a/annotator/uniformer/configs/_base_/datasets/ade20k.py b/annotator/uniformer/configs/_base_/datasets/ade20k.py new file mode 100644 index 0000000000000000000000000000000000000000..efc8b4bb20c981f3db6df7eb52b3dc0744c94cc0 --- /dev/null +++ b/annotator/uniformer/configs/_base_/datasets/ade20k.py @@ -0,0 +1,54 @@ +# dataset settings +dataset_type = 'ADE20KDataset' +data_root = 'data/ade/ADEChallengeData2016' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (512, 512) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', reduce_zero_label=True), + dict(type='Resize', img_scale=(2048, 512), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2048, 512), + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/training', + ann_dir='annotations/training', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline)) diff --git a/annotator/uniformer/configs/_base_/datasets/chase_db1.py b/annotator/uniformer/configs/_base_/datasets/chase_db1.py new file mode 100644 index 0000000000000000000000000000000000000000..298594ea925f87f22b37094a2ec50e370aec96a0 --- /dev/null +++ b/annotator/uniformer/configs/_base_/datasets/chase_db1.py @@ -0,0 +1,59 @@ +# dataset settings +dataset_type = 'ChaseDB1Dataset' +data_root = 'data/CHASE_DB1' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +img_scale = (960, 999) +crop_size = (128, 128) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='Resize', img_scale=img_scale, ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']) +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=img_scale, + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']) + ]) +] + +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type='RepeatDataset', + times=40000, + dataset=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/training', + ann_dir='annotations/training', + pipeline=train_pipeline)), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline)) diff --git a/annotator/uniformer/configs/_base_/datasets/cityscapes.py b/annotator/uniformer/configs/_base_/datasets/cityscapes.py new file mode 100644 index 0000000000000000000000000000000000000000..f21867c63e1835f6fceb61f066e802fd8fd2a735 --- /dev/null +++ b/annotator/uniformer/configs/_base_/datasets/cityscapes.py @@ -0,0 +1,54 @@ +# dataset settings +dataset_type = 'CityscapesDataset' +data_root = 'data/cityscapes/' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (512, 1024) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='Resize', img_scale=(2048, 1024), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2048, 1024), + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=2, + workers_per_gpu=2, + train=dict( + type=dataset_type, + data_root=data_root, + img_dir='leftImg8bit/train', + ann_dir='gtFine/train', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='leftImg8bit/val', + ann_dir='gtFine/val', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='leftImg8bit/val', + ann_dir='gtFine/val', + pipeline=test_pipeline)) diff --git a/annotator/uniformer/configs/_base_/datasets/cityscapes_769x769.py b/annotator/uniformer/configs/_base_/datasets/cityscapes_769x769.py new file mode 100644 index 0000000000000000000000000000000000000000..336c7b254fe392b4703039fec86a83acdbd2e1a5 --- /dev/null +++ b/annotator/uniformer/configs/_base_/datasets/cityscapes_769x769.py @@ -0,0 +1,35 @@ +_base_ = './cityscapes.py' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (769, 769) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='Resize', img_scale=(2049, 1025), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2049, 1025), + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + train=dict(pipeline=train_pipeline), + val=dict(pipeline=test_pipeline), + test=dict(pipeline=test_pipeline)) diff --git a/annotator/uniformer/configs/_base_/datasets/drive.py b/annotator/uniformer/configs/_base_/datasets/drive.py new file mode 100644 index 0000000000000000000000000000000000000000..06e8ff606e0d2a4514ec8b7d2c6c436a32efcbf4 --- /dev/null +++ b/annotator/uniformer/configs/_base_/datasets/drive.py @@ -0,0 +1,59 @@ +# dataset settings +dataset_type = 'DRIVEDataset' +data_root = 'data/DRIVE' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +img_scale = (584, 565) +crop_size = (64, 64) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='Resize', img_scale=img_scale, ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']) +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=img_scale, + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']) + ]) +] + +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type='RepeatDataset', + times=40000, + dataset=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/training', + ann_dir='annotations/training', + pipeline=train_pipeline)), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline)) diff --git a/annotator/uniformer/configs/_base_/datasets/hrf.py b/annotator/uniformer/configs/_base_/datasets/hrf.py new file mode 100644 index 0000000000000000000000000000000000000000..242d790eb1b83e75cf6b7eaa7a35c674099311ad --- /dev/null +++ b/annotator/uniformer/configs/_base_/datasets/hrf.py @@ -0,0 +1,59 @@ +# dataset settings +dataset_type = 'HRFDataset' +data_root = 'data/HRF' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +img_scale = (2336, 3504) +crop_size = (256, 256) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='Resize', img_scale=img_scale, ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']) +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=img_scale, + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']) + ]) +] + +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type='RepeatDataset', + times=40000, + dataset=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/training', + ann_dir='annotations/training', + pipeline=train_pipeline)), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline)) diff --git a/annotator/uniformer/configs/_base_/datasets/pascal_context.py b/annotator/uniformer/configs/_base_/datasets/pascal_context.py new file mode 100644 index 0000000000000000000000000000000000000000..ff65bad1b86d7e3a5980bb5b9fc55798dc8df5f4 --- /dev/null +++ b/annotator/uniformer/configs/_base_/datasets/pascal_context.py @@ -0,0 +1,60 @@ +# dataset settings +dataset_type = 'PascalContextDataset' +data_root = 'data/VOCdevkit/VOC2010/' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) + +img_scale = (520, 520) +crop_size = (480, 480) + +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='Resize', img_scale=img_scale, ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=img_scale, + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type=dataset_type, + data_root=data_root, + img_dir='JPEGImages', + ann_dir='SegmentationClassContext', + split='ImageSets/SegmentationContext/train.txt', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='JPEGImages', + ann_dir='SegmentationClassContext', + split='ImageSets/SegmentationContext/val.txt', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='JPEGImages', + ann_dir='SegmentationClassContext', + split='ImageSets/SegmentationContext/val.txt', + pipeline=test_pipeline)) diff --git a/annotator/uniformer/configs/_base_/datasets/pascal_context_59.py b/annotator/uniformer/configs/_base_/datasets/pascal_context_59.py new file mode 100644 index 0000000000000000000000000000000000000000..37585abab89834b95cd5bdd993b994fca1db65f6 --- /dev/null +++ b/annotator/uniformer/configs/_base_/datasets/pascal_context_59.py @@ -0,0 +1,60 @@ +# dataset settings +dataset_type = 'PascalContextDataset59' +data_root = 'data/VOCdevkit/VOC2010/' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) + +img_scale = (520, 520) +crop_size = (480, 480) + +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations', reduce_zero_label=True), + dict(type='Resize', img_scale=img_scale, ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=img_scale, + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type=dataset_type, + data_root=data_root, + img_dir='JPEGImages', + ann_dir='SegmentationClassContext', + split='ImageSets/SegmentationContext/train.txt', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='JPEGImages', + ann_dir='SegmentationClassContext', + split='ImageSets/SegmentationContext/val.txt', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='JPEGImages', + ann_dir='SegmentationClassContext', + split='ImageSets/SegmentationContext/val.txt', + pipeline=test_pipeline)) diff --git a/annotator/uniformer/configs/_base_/datasets/pascal_voc12.py b/annotator/uniformer/configs/_base_/datasets/pascal_voc12.py new file mode 100644 index 0000000000000000000000000000000000000000..ba1d42d0c5781f56dc177d860d856bb34adce555 --- /dev/null +++ b/annotator/uniformer/configs/_base_/datasets/pascal_voc12.py @@ -0,0 +1,57 @@ +# dataset settings +dataset_type = 'PascalVOCDataset' +data_root = 'data/VOCdevkit/VOC2012' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +crop_size = (512, 512) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='Resize', img_scale=(2048, 512), ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']), +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=(2048, 512), + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ]) +] +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type=dataset_type, + data_root=data_root, + img_dir='JPEGImages', + ann_dir='SegmentationClass', + split='ImageSets/Segmentation/train.txt', + pipeline=train_pipeline), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='JPEGImages', + ann_dir='SegmentationClass', + split='ImageSets/Segmentation/val.txt', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='JPEGImages', + ann_dir='SegmentationClass', + split='ImageSets/Segmentation/val.txt', + pipeline=test_pipeline)) diff --git a/annotator/uniformer/configs/_base_/datasets/pascal_voc12_aug.py b/annotator/uniformer/configs/_base_/datasets/pascal_voc12_aug.py new file mode 100644 index 0000000000000000000000000000000000000000..3f23b6717d53ad29f02dd15046802a2631a5076b --- /dev/null +++ b/annotator/uniformer/configs/_base_/datasets/pascal_voc12_aug.py @@ -0,0 +1,9 @@ +_base_ = './pascal_voc12.py' +# dataset settings +data = dict( + train=dict( + ann_dir=['SegmentationClass', 'SegmentationClassAug'], + split=[ + 'ImageSets/Segmentation/train.txt', + 'ImageSets/Segmentation/aug.txt' + ])) diff --git a/annotator/uniformer/configs/_base_/datasets/stare.py b/annotator/uniformer/configs/_base_/datasets/stare.py new file mode 100644 index 0000000000000000000000000000000000000000..3f71b25488cc11a6b4d582ac52b5a24e1ad1cf8e --- /dev/null +++ b/annotator/uniformer/configs/_base_/datasets/stare.py @@ -0,0 +1,59 @@ +# dataset settings +dataset_type = 'STAREDataset' +data_root = 'data/STARE' +img_norm_cfg = dict( + mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True) +img_scale = (605, 700) +crop_size = (128, 128) +train_pipeline = [ + dict(type='LoadImageFromFile'), + dict(type='LoadAnnotations'), + dict(type='Resize', img_scale=img_scale, ratio_range=(0.5, 2.0)), + dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75), + dict(type='RandomFlip', prob=0.5), + dict(type='PhotoMetricDistortion'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size=crop_size, pad_val=0, seg_pad_val=255), + dict(type='DefaultFormatBundle'), + dict(type='Collect', keys=['img', 'gt_semantic_seg']) +] +test_pipeline = [ + dict(type='LoadImageFromFile'), + dict( + type='MultiScaleFlipAug', + img_scale=img_scale, + # img_ratios=[0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0], + flip=False, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']) + ]) +] + +data = dict( + samples_per_gpu=4, + workers_per_gpu=4, + train=dict( + type='RepeatDataset', + times=40000, + dataset=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/training', + ann_dir='annotations/training', + pipeline=train_pipeline)), + val=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline), + test=dict( + type=dataset_type, + data_root=data_root, + img_dir='images/validation', + ann_dir='annotations/validation', + pipeline=test_pipeline)) diff --git a/annotator/uniformer/configs/_base_/default_runtime.py b/annotator/uniformer/configs/_base_/default_runtime.py new file mode 100644 index 0000000000000000000000000000000000000000..b564cc4e7e7d9a67dacaaddecb100e4d8f5c005b --- /dev/null +++ b/annotator/uniformer/configs/_base_/default_runtime.py @@ -0,0 +1,14 @@ +# yapf:disable +log_config = dict( + interval=50, + hooks=[ + dict(type='TextLoggerHook', by_epoch=False), + # dict(type='TensorboardLoggerHook') + ]) +# yapf:enable +dist_params = dict(backend='nccl') +log_level = 'INFO' +load_from = None +resume_from = None +workflow = [('train', 1)] +cudnn_benchmark = True diff --git a/annotator/uniformer/configs/_base_/models/ann_r50-d8.py b/annotator/uniformer/configs/_base_/models/ann_r50-d8.py new file mode 100644 index 0000000000000000000000000000000000000000..a2cb653827e44e6015b3b83bc578003e614a6aa1 --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/ann_r50-d8.py @@ -0,0 +1,46 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 2, 4), + strides=(1, 2, 1, 1), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + decode_head=dict( + type='ANNHead', + in_channels=[1024, 2048], + in_index=[2, 3], + channels=512, + project_channels=256, + query_scales=(1, ), + key_pool_scales=(1, 3, 6, 8), + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=1024, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/apcnet_r50-d8.py b/annotator/uniformer/configs/_base_/models/apcnet_r50-d8.py new file mode 100644 index 0000000000000000000000000000000000000000..c8f5316cbcf3896ba9de7ca2c801eba512f01d5e --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/apcnet_r50-d8.py @@ -0,0 +1,44 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 2, 4), + strides=(1, 2, 1, 1), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + decode_head=dict( + type='APCHead', + in_channels=2048, + in_index=3, + channels=512, + pool_scales=(1, 2, 3, 6), + dropout_ratio=0.1, + num_classes=19, + norm_cfg=dict(type='SyncBN', requires_grad=True), + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=1024, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/ccnet_r50-d8.py b/annotator/uniformer/configs/_base_/models/ccnet_r50-d8.py new file mode 100644 index 0000000000000000000000000000000000000000..794148f576b9e215c3c6963e73dffe98204b7717 --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/ccnet_r50-d8.py @@ -0,0 +1,44 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 2, 4), + strides=(1, 2, 1, 1), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + decode_head=dict( + type='CCHead', + in_channels=2048, + in_index=3, + channels=512, + recurrence=2, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=1024, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/cgnet.py b/annotator/uniformer/configs/_base_/models/cgnet.py new file mode 100644 index 0000000000000000000000000000000000000000..eff8d9458c877c5db894957e0b1b4597e40da6ab --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/cgnet.py @@ -0,0 +1,35 @@ +# model settings +norm_cfg = dict(type='SyncBN', eps=1e-03, requires_grad=True) +model = dict( + type='EncoderDecoder', + backbone=dict( + type='CGNet', + norm_cfg=norm_cfg, + in_channels=3, + num_channels=(32, 64, 128), + num_blocks=(3, 21), + dilations=(2, 4), + reductions=(8, 16)), + decode_head=dict( + type='FCNHead', + in_channels=256, + in_index=2, + channels=256, + num_convs=0, + concat_input=False, + dropout_ratio=0, + num_classes=19, + norm_cfg=norm_cfg, + loss_decode=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0, + class_weight=[ + 2.5959933, 6.7415504, 3.5354059, 9.8663225, 9.690899, 9.369352, + 10.289121, 9.953208, 4.3097677, 9.490387, 7.674431, 9.396905, + 10.347791, 6.3927646, 10.226669, 10.241062, 10.280587, + 10.396974, 10.055647 + ])), + # model training and testing settings + train_cfg=dict(sampler=None), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/danet_r50-d8.py b/annotator/uniformer/configs/_base_/models/danet_r50-d8.py new file mode 100644 index 0000000000000000000000000000000000000000..2c934939fac48525f22ad86f489a041dd7db7d09 --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/danet_r50-d8.py @@ -0,0 +1,44 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 2, 4), + strides=(1, 2, 1, 1), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + decode_head=dict( + type='DAHead', + in_channels=2048, + in_index=3, + channels=512, + pam_channels=64, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=1024, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/deeplabv3_r50-d8.py b/annotator/uniformer/configs/_base_/models/deeplabv3_r50-d8.py new file mode 100644 index 0000000000000000000000000000000000000000..d7a43bee01422ad4795dd27874e0cd4bb6cbfecf --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/deeplabv3_r50-d8.py @@ -0,0 +1,44 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 2, 4), + strides=(1, 2, 1, 1), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + decode_head=dict( + type='ASPPHead', + in_channels=2048, + in_index=3, + channels=512, + dilations=(1, 12, 24, 36), + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=1024, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/deeplabv3_unet_s5-d16.py b/annotator/uniformer/configs/_base_/models/deeplabv3_unet_s5-d16.py new file mode 100644 index 0000000000000000000000000000000000000000..0cd262999d8b2cb8e14a5c32190ae73f479d8e81 --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/deeplabv3_unet_s5-d16.py @@ -0,0 +1,50 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained=None, + backbone=dict( + type='UNet', + in_channels=3, + base_channels=64, + num_stages=5, + strides=(1, 1, 1, 1, 1), + enc_num_convs=(2, 2, 2, 2, 2), + dec_num_convs=(2, 2, 2, 2), + downsamples=(True, True, True, True), + enc_dilations=(1, 1, 1, 1, 1), + dec_dilations=(1, 1, 1, 1), + with_cp=False, + conv_cfg=None, + norm_cfg=norm_cfg, + act_cfg=dict(type='ReLU'), + upsample_cfg=dict(type='InterpConv'), + norm_eval=False), + decode_head=dict( + type='ASPPHead', + in_channels=64, + in_index=4, + channels=16, + dilations=(1, 12, 24, 36), + dropout_ratio=0.1, + num_classes=2, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=128, + in_index=3, + channels=64, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=2, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='slide', crop_size=256, stride=170)) diff --git a/annotator/uniformer/configs/_base_/models/deeplabv3plus_r50-d8.py b/annotator/uniformer/configs/_base_/models/deeplabv3plus_r50-d8.py new file mode 100644 index 0000000000000000000000000000000000000000..050e39e091d816df9028d23aa3ecf9db74e441e1 --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/deeplabv3plus_r50-d8.py @@ -0,0 +1,46 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 2, 4), + strides=(1, 2, 1, 1), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + decode_head=dict( + type='DepthwiseSeparableASPPHead', + in_channels=2048, + in_index=3, + channels=512, + dilations=(1, 12, 24, 36), + c1_in_channels=256, + c1_channels=48, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=1024, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/dmnet_r50-d8.py b/annotator/uniformer/configs/_base_/models/dmnet_r50-d8.py new file mode 100644 index 0000000000000000000000000000000000000000..d22ba52640bebd805b3b8d07025e276dfb023759 --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/dmnet_r50-d8.py @@ -0,0 +1,44 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 2, 4), + strides=(1, 2, 1, 1), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + decode_head=dict( + type='DMHead', + in_channels=2048, + in_index=3, + channels=512, + filter_sizes=(1, 3, 5, 7), + dropout_ratio=0.1, + num_classes=19, + norm_cfg=dict(type='SyncBN', requires_grad=True), + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=1024, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/dnl_r50-d8.py b/annotator/uniformer/configs/_base_/models/dnl_r50-d8.py new file mode 100644 index 0000000000000000000000000000000000000000..edb4c174c51e34c103737ba39bfc48bf831e561d --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/dnl_r50-d8.py @@ -0,0 +1,46 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 2, 4), + strides=(1, 2, 1, 1), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + decode_head=dict( + type='DNLHead', + in_channels=2048, + in_index=3, + channels=512, + dropout_ratio=0.1, + reduction=2, + use_scale=True, + mode='embedded_gaussian', + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=1024, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/emanet_r50-d8.py b/annotator/uniformer/configs/_base_/models/emanet_r50-d8.py new file mode 100644 index 0000000000000000000000000000000000000000..26adcd430926de0862204a71d345f2543167f27b --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/emanet_r50-d8.py @@ -0,0 +1,47 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 2, 4), + strides=(1, 2, 1, 1), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + decode_head=dict( + type='EMAHead', + in_channels=2048, + in_index=3, + channels=256, + ema_channels=512, + num_bases=64, + num_stages=3, + momentum=0.1, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=1024, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/encnet_r50-d8.py b/annotator/uniformer/configs/_base_/models/encnet_r50-d8.py new file mode 100644 index 0000000000000000000000000000000000000000..be777123a886503172a95fe0719e956a147bbd68 --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/encnet_r50-d8.py @@ -0,0 +1,48 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 2, 4), + strides=(1, 2, 1, 1), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + decode_head=dict( + type='EncHead', + in_channels=[512, 1024, 2048], + in_index=(1, 2, 3), + channels=512, + num_codes=32, + use_se_loss=True, + add_lateral=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), + loss_se_decode=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=0.2)), + auxiliary_head=dict( + type='FCNHead', + in_channels=1024, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/fast_scnn.py b/annotator/uniformer/configs/_base_/models/fast_scnn.py new file mode 100644 index 0000000000000000000000000000000000000000..32fdeb659355a5ce5ef2cc7c2f30742703811cdf --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/fast_scnn.py @@ -0,0 +1,57 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True, momentum=0.01) +model = dict( + type='EncoderDecoder', + backbone=dict( + type='FastSCNN', + downsample_dw_channels=(32, 48), + global_in_channels=64, + global_block_channels=(64, 96, 128), + global_block_strides=(2, 2, 1), + global_out_channels=128, + higher_in_channels=64, + lower_in_channels=128, + fusion_out_channels=128, + out_indices=(0, 1, 2), + norm_cfg=norm_cfg, + align_corners=False), + decode_head=dict( + type='DepthwiseSeparableFCNHead', + in_channels=128, + channels=128, + concat_input=False, + num_classes=19, + in_index=-1, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=0.4)), + auxiliary_head=[ + dict( + type='FCNHead', + in_channels=128, + channels=32, + num_convs=1, + num_classes=19, + in_index=-2, + norm_cfg=norm_cfg, + concat_input=False, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=0.4)), + dict( + type='FCNHead', + in_channels=64, + channels=32, + num_convs=1, + num_classes=19, + in_index=-3, + norm_cfg=norm_cfg, + concat_input=False, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=True, loss_weight=0.4)), + ], + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/fcn_hr18.py b/annotator/uniformer/configs/_base_/models/fcn_hr18.py new file mode 100644 index 0000000000000000000000000000000000000000..c3e299bc89ada56ca14bbffcbdb08a586b8ed9e9 --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/fcn_hr18.py @@ -0,0 +1,52 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained='open-mmlab://msra/hrnetv2_w18', + backbone=dict( + type='HRNet', + norm_cfg=norm_cfg, + norm_eval=False, + extra=dict( + stage1=dict( + num_modules=1, + num_branches=1, + block='BOTTLENECK', + num_blocks=(4, ), + num_channels=(64, )), + stage2=dict( + num_modules=1, + num_branches=2, + block='BASIC', + num_blocks=(4, 4), + num_channels=(18, 36)), + stage3=dict( + num_modules=4, + num_branches=3, + block='BASIC', + num_blocks=(4, 4, 4), + num_channels=(18, 36, 72)), + stage4=dict( + num_modules=3, + num_branches=4, + block='BASIC', + num_blocks=(4, 4, 4, 4), + num_channels=(18, 36, 72, 144)))), + decode_head=dict( + type='FCNHead', + in_channels=[18, 36, 72, 144], + in_index=(0, 1, 2, 3), + channels=sum([18, 36, 72, 144]), + input_transform='resize_concat', + kernel_size=1, + num_convs=1, + concat_input=False, + dropout_ratio=-1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/fcn_r50-d8.py b/annotator/uniformer/configs/_base_/models/fcn_r50-d8.py new file mode 100644 index 0000000000000000000000000000000000000000..5e98f6cc918b6146fc6d613c6918e825ef1355c3 --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/fcn_r50-d8.py @@ -0,0 +1,45 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 2, 4), + strides=(1, 2, 1, 1), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + decode_head=dict( + type='FCNHead', + in_channels=2048, + in_index=3, + channels=512, + num_convs=2, + concat_input=True, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=1024, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/fcn_unet_s5-d16.py b/annotator/uniformer/configs/_base_/models/fcn_unet_s5-d16.py new file mode 100644 index 0000000000000000000000000000000000000000..a33e7972877f902d0e7d18401ca675e3e4e60a18 --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/fcn_unet_s5-d16.py @@ -0,0 +1,51 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained=None, + backbone=dict( + type='UNet', + in_channels=3, + base_channels=64, + num_stages=5, + strides=(1, 1, 1, 1, 1), + enc_num_convs=(2, 2, 2, 2, 2), + dec_num_convs=(2, 2, 2, 2), + downsamples=(True, True, True, True), + enc_dilations=(1, 1, 1, 1, 1), + dec_dilations=(1, 1, 1, 1), + with_cp=False, + conv_cfg=None, + norm_cfg=norm_cfg, + act_cfg=dict(type='ReLU'), + upsample_cfg=dict(type='InterpConv'), + norm_eval=False), + decode_head=dict( + type='FCNHead', + in_channels=64, + in_index=4, + channels=64, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=2, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=128, + in_index=3, + channels=64, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=2, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='slide', crop_size=256, stride=170)) diff --git a/annotator/uniformer/configs/_base_/models/fpn_r50.py b/annotator/uniformer/configs/_base_/models/fpn_r50.py new file mode 100644 index 0000000000000000000000000000000000000000..86ab327db92e44c14822d65f1c9277cb007f17c1 --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/fpn_r50.py @@ -0,0 +1,36 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 1, 1), + strides=(1, 2, 2, 2), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + neck=dict( + type='FPN', + in_channels=[256, 512, 1024, 2048], + out_channels=256, + num_outs=4), + decode_head=dict( + type='FPNHead', + in_channels=[256, 256, 256, 256], + in_index=[0, 1, 2, 3], + feature_strides=[4, 8, 16, 32], + channels=128, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/fpn_uniformer.py b/annotator/uniformer/configs/_base_/models/fpn_uniformer.py new file mode 100644 index 0000000000000000000000000000000000000000..8aae98c5991055bfcc08e82ccdc09f8b1d9f8a8d --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/fpn_uniformer.py @@ -0,0 +1,35 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + backbone=dict( + type='UniFormer', + embed_dim=[64, 128, 320, 512], + layers=[3, 4, 8, 3], + head_dim=64, + mlp_ratio=4., + qkv_bias=True, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0.1), + neck=dict( + type='FPN', + in_channels=[64, 128, 320, 512], + out_channels=256, + num_outs=4), + decode_head=dict( + type='FPNHead', + in_channels=[256, 256, 256, 256], + in_index=[0, 1, 2, 3], + feature_strides=[4, 8, 16, 32], + channels=128, + dropout_ratio=0.1, + num_classes=150, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole') +) diff --git a/annotator/uniformer/configs/_base_/models/gcnet_r50-d8.py b/annotator/uniformer/configs/_base_/models/gcnet_r50-d8.py new file mode 100644 index 0000000000000000000000000000000000000000..3d2ad69f5c22adfe79d5fdabf920217628987166 --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/gcnet_r50-d8.py @@ -0,0 +1,46 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 2, 4), + strides=(1, 2, 1, 1), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + decode_head=dict( + type='GCHead', + in_channels=2048, + in_index=3, + channels=512, + ratio=1 / 4., + pooling_type='att', + fusion_types=('channel_add', ), + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=1024, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/lraspp_m-v3-d8.py b/annotator/uniformer/configs/_base_/models/lraspp_m-v3-d8.py new file mode 100644 index 0000000000000000000000000000000000000000..93258242a90695cc94a7c6bd41562d6a75988771 --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/lraspp_m-v3-d8.py @@ -0,0 +1,25 @@ +# model settings +norm_cfg = dict(type='SyncBN', eps=0.001, requires_grad=True) +model = dict( + type='EncoderDecoder', + backbone=dict( + type='MobileNetV3', + arch='large', + out_indices=(1, 3, 16), + norm_cfg=norm_cfg), + decode_head=dict( + type='LRASPPHead', + in_channels=(16, 24, 960), + in_index=(0, 1, 2), + channels=128, + input_transform='multiple_select', + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + act_cfg=dict(type='ReLU'), + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/nonlocal_r50-d8.py b/annotator/uniformer/configs/_base_/models/nonlocal_r50-d8.py new file mode 100644 index 0000000000000000000000000000000000000000..5674a39854cafd1f2e363bac99c58ccae62f24da --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/nonlocal_r50-d8.py @@ -0,0 +1,46 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 2, 4), + strides=(1, 2, 1, 1), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + decode_head=dict( + type='NLHead', + in_channels=2048, + in_index=3, + channels=512, + dropout_ratio=0.1, + reduction=2, + use_scale=True, + mode='embedded_gaussian', + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=1024, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/ocrnet_hr18.py b/annotator/uniformer/configs/_base_/models/ocrnet_hr18.py new file mode 100644 index 0000000000000000000000000000000000000000..c60f62a7cdf3f5c5096a7a7e725e8268fddcb057 --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/ocrnet_hr18.py @@ -0,0 +1,68 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='CascadeEncoderDecoder', + num_stages=2, + pretrained='open-mmlab://msra/hrnetv2_w18', + backbone=dict( + type='HRNet', + norm_cfg=norm_cfg, + norm_eval=False, + extra=dict( + stage1=dict( + num_modules=1, + num_branches=1, + block='BOTTLENECK', + num_blocks=(4, ), + num_channels=(64, )), + stage2=dict( + num_modules=1, + num_branches=2, + block='BASIC', + num_blocks=(4, 4), + num_channels=(18, 36)), + stage3=dict( + num_modules=4, + num_branches=3, + block='BASIC', + num_blocks=(4, 4, 4), + num_channels=(18, 36, 72)), + stage4=dict( + num_modules=3, + num_branches=4, + block='BASIC', + num_blocks=(4, 4, 4, 4), + num_channels=(18, 36, 72, 144)))), + decode_head=[ + dict( + type='FCNHead', + in_channels=[18, 36, 72, 144], + channels=sum([18, 36, 72, 144]), + in_index=(0, 1, 2, 3), + input_transform='resize_concat', + kernel_size=1, + num_convs=1, + concat_input=False, + dropout_ratio=-1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + dict( + type='OCRHead', + in_channels=[18, 36, 72, 144], + in_index=(0, 1, 2, 3), + input_transform='resize_concat', + channels=512, + ocr_channels=256, + dropout_ratio=-1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + ], + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/ocrnet_r50-d8.py b/annotator/uniformer/configs/_base_/models/ocrnet_r50-d8.py new file mode 100644 index 0000000000000000000000000000000000000000..615aa3ff703942b6c22b2d6e9642504dd3e41ebd --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/ocrnet_r50-d8.py @@ -0,0 +1,47 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='CascadeEncoderDecoder', + num_stages=2, + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 2, 4), + strides=(1, 2, 1, 1), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + decode_head=[ + dict( + type='FCNHead', + in_channels=1024, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + dict( + type='OCRHead', + in_channels=2048, + in_index=3, + channels=512, + ocr_channels=256, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)) + ], + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/pointrend_r50.py b/annotator/uniformer/configs/_base_/models/pointrend_r50.py new file mode 100644 index 0000000000000000000000000000000000000000..9d323dbf9466d41e0800aa57ef84045f3d874bdf --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/pointrend_r50.py @@ -0,0 +1,56 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='CascadeEncoderDecoder', + num_stages=2, + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 1, 1), + strides=(1, 2, 2, 2), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + neck=dict( + type='FPN', + in_channels=[256, 512, 1024, 2048], + out_channels=256, + num_outs=4), + decode_head=[ + dict( + type='FPNHead', + in_channels=[256, 256, 256, 256], + in_index=[0, 1, 2, 3], + feature_strides=[4, 8, 16, 32], + channels=128, + dropout_ratio=-1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + dict( + type='PointHead', + in_channels=[256], + in_index=[0], + channels=256, + num_fcs=3, + coarse_pred_each_layer=True, + dropout_ratio=-1, + num_classes=19, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)) + ], + # model training and testing settings + train_cfg=dict( + num_points=2048, oversample_ratio=3, importance_sample_ratio=0.75), + test_cfg=dict( + mode='whole', + subdivision_steps=2, + subdivision_num_points=8196, + scale_factor=2)) diff --git a/annotator/uniformer/configs/_base_/models/psanet_r50-d8.py b/annotator/uniformer/configs/_base_/models/psanet_r50-d8.py new file mode 100644 index 0000000000000000000000000000000000000000..689513fa9d2a40f14bf0ae4ae61f38f0dcc1b3da --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/psanet_r50-d8.py @@ -0,0 +1,49 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 2, 4), + strides=(1, 2, 1, 1), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + decode_head=dict( + type='PSAHead', + in_channels=2048, + in_index=3, + channels=512, + mask_size=(97, 97), + psa_type='bi-direction', + compact=False, + shrink_factor=2, + normalization_factor=1.0, + psa_softmax=True, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=1024, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/pspnet_r50-d8.py b/annotator/uniformer/configs/_base_/models/pspnet_r50-d8.py new file mode 100644 index 0000000000000000000000000000000000000000..f451e08ad2eb0732dcb806b1851eb978d4acf136 --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/pspnet_r50-d8.py @@ -0,0 +1,44 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 2, 4), + strides=(1, 2, 1, 1), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + decode_head=dict( + type='PSPHead', + in_channels=2048, + in_index=3, + channels=512, + pool_scales=(1, 2, 3, 6), + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=1024, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/pspnet_unet_s5-d16.py b/annotator/uniformer/configs/_base_/models/pspnet_unet_s5-d16.py new file mode 100644 index 0000000000000000000000000000000000000000..fcff9ec4f41fad158344ecd77313dc14564f3682 --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/pspnet_unet_s5-d16.py @@ -0,0 +1,50 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained=None, + backbone=dict( + type='UNet', + in_channels=3, + base_channels=64, + num_stages=5, + strides=(1, 1, 1, 1, 1), + enc_num_convs=(2, 2, 2, 2, 2), + dec_num_convs=(2, 2, 2, 2), + downsamples=(True, True, True, True), + enc_dilations=(1, 1, 1, 1, 1), + dec_dilations=(1, 1, 1, 1), + with_cp=False, + conv_cfg=None, + norm_cfg=norm_cfg, + act_cfg=dict(type='ReLU'), + upsample_cfg=dict(type='InterpConv'), + norm_eval=False), + decode_head=dict( + type='PSPHead', + in_channels=64, + in_index=4, + channels=16, + pool_scales=(1, 2, 3, 6), + dropout_ratio=0.1, + num_classes=2, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=128, + in_index=3, + channels=64, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=2, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='slide', crop_size=256, stride=170)) diff --git a/annotator/uniformer/configs/_base_/models/upernet_r50.py b/annotator/uniformer/configs/_base_/models/upernet_r50.py new file mode 100644 index 0000000000000000000000000000000000000000..10974962fdd7136031fd06de1700f497d355ceaa --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/upernet_r50.py @@ -0,0 +1,44 @@ +# model settings +norm_cfg = dict(type='SyncBN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained='open-mmlab://resnet50_v1c', + backbone=dict( + type='ResNetV1c', + depth=50, + num_stages=4, + out_indices=(0, 1, 2, 3), + dilations=(1, 1, 1, 1), + strides=(1, 2, 2, 2), + norm_cfg=norm_cfg, + norm_eval=False, + style='pytorch', + contract_dilation=True), + decode_head=dict( + type='UPerHead', + in_channels=[256, 512, 1024, 2048], + in_index=[0, 1, 2, 3], + pool_scales=(1, 2, 3, 6), + channels=512, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=1024, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) diff --git a/annotator/uniformer/configs/_base_/models/upernet_uniformer.py b/annotator/uniformer/configs/_base_/models/upernet_uniformer.py new file mode 100644 index 0000000000000000000000000000000000000000..41aa4db809dc6e2c508e98051f61807d07477903 --- /dev/null +++ b/annotator/uniformer/configs/_base_/models/upernet_uniformer.py @@ -0,0 +1,43 @@ +# model settings +norm_cfg = dict(type='BN', requires_grad=True) +model = dict( + type='EncoderDecoder', + pretrained=None, + backbone=dict( + type='UniFormer', + embed_dim=[64, 128, 320, 512], + layers=[3, 4, 8, 3], + head_dim=64, + mlp_ratio=4., + qkv_bias=True, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0.1), + decode_head=dict( + type='UPerHead', + in_channels=[64, 128, 320, 512], + in_index=[0, 1, 2, 3], + pool_scales=(1, 2, 3, 6), + channels=512, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0)), + auxiliary_head=dict( + type='FCNHead', + in_channels=320, + in_index=2, + channels=256, + num_convs=1, + concat_input=False, + dropout_ratio=0.1, + num_classes=19, + norm_cfg=norm_cfg, + align_corners=False, + loss_decode=dict( + type='CrossEntropyLoss', use_sigmoid=False, loss_weight=0.4)), + # model training and testing settings + train_cfg=dict(), + test_cfg=dict(mode='whole')) \ No newline at end of file diff --git a/annotator/uniformer/configs/_base_/schedules/schedule_160k.py b/annotator/uniformer/configs/_base_/schedules/schedule_160k.py new file mode 100644 index 0000000000000000000000000000000000000000..52603890b10f25faf8eec9f9e5a4468fae09b811 --- /dev/null +++ b/annotator/uniformer/configs/_base_/schedules/schedule_160k.py @@ -0,0 +1,9 @@ +# optimizer +optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005) +optimizer_config = dict() +# learning policy +lr_config = dict(policy='poly', power=0.9, min_lr=1e-4, by_epoch=False) +# runtime settings +runner = dict(type='IterBasedRunner', max_iters=160000) +checkpoint_config = dict(by_epoch=False, interval=16000) +evaluation = dict(interval=16000, metric='mIoU') diff --git a/annotator/uniformer/configs/_base_/schedules/schedule_20k.py b/annotator/uniformer/configs/_base_/schedules/schedule_20k.py new file mode 100644 index 0000000000000000000000000000000000000000..bf780a1b6f6521833c6a5859675147824efa599d --- /dev/null +++ b/annotator/uniformer/configs/_base_/schedules/schedule_20k.py @@ -0,0 +1,9 @@ +# optimizer +optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005) +optimizer_config = dict() +# learning policy +lr_config = dict(policy='poly', power=0.9, min_lr=1e-4, by_epoch=False) +# runtime settings +runner = dict(type='IterBasedRunner', max_iters=20000) +checkpoint_config = dict(by_epoch=False, interval=2000) +evaluation = dict(interval=2000, metric='mIoU') diff --git a/annotator/uniformer/configs/_base_/schedules/schedule_40k.py b/annotator/uniformer/configs/_base_/schedules/schedule_40k.py new file mode 100644 index 0000000000000000000000000000000000000000..cdbf841abcb26eed87bf76ab816aff4bae0630ee --- /dev/null +++ b/annotator/uniformer/configs/_base_/schedules/schedule_40k.py @@ -0,0 +1,9 @@ +# optimizer +optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005) +optimizer_config = dict() +# learning policy +lr_config = dict(policy='poly', power=0.9, min_lr=1e-4, by_epoch=False) +# runtime settings +runner = dict(type='IterBasedRunner', max_iters=40000) +checkpoint_config = dict(by_epoch=False, interval=4000) +evaluation = dict(interval=4000, metric='mIoU') diff --git a/annotator/uniformer/configs/_base_/schedules/schedule_80k.py b/annotator/uniformer/configs/_base_/schedules/schedule_80k.py new file mode 100644 index 0000000000000000000000000000000000000000..c190cee6bdc7922b688ea75dc8f152fa15c24617 --- /dev/null +++ b/annotator/uniformer/configs/_base_/schedules/schedule_80k.py @@ -0,0 +1,9 @@ +# optimizer +optimizer = dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0005) +optimizer_config = dict() +# learning policy +lr_config = dict(policy='poly', power=0.9, min_lr=1e-4, by_epoch=False) +# runtime settings +runner = dict(type='IterBasedRunner', max_iters=80000) +checkpoint_config = dict(by_epoch=False, interval=8000) +evaluation = dict(interval=8000, metric='mIoU') diff --git a/annotator/uniformer/exp/upernet_global_small/config.py b/annotator/uniformer/exp/upernet_global_small/config.py new file mode 100644 index 0000000000000000000000000000000000000000..01db96bf9b0be531aa0eaf62fee51543712f8670 --- /dev/null +++ b/annotator/uniformer/exp/upernet_global_small/config.py @@ -0,0 +1,38 @@ +_base_ = [ + '../../configs/_base_/models/upernet_uniformer.py', + '../../configs/_base_/datasets/ade20k.py', + '../../configs/_base_/default_runtime.py', + '../../configs/_base_/schedules/schedule_160k.py' +] +model = dict( + backbone=dict( + type='UniFormer', + embed_dim=[64, 128, 320, 512], + layers=[3, 4, 8, 3], + head_dim=64, + drop_path_rate=0.25, + windows=False, + hybrid=False + ), + decode_head=dict( + in_channels=[64, 128, 320, 512], + num_classes=150 + ), + auxiliary_head=dict( + in_channels=320, + num_classes=150 + )) + +# AdamW optimizer, no weight decay for position embedding & layer norm in backbone +optimizer = dict(_delete_=True, type='AdamW', lr=0.00006, betas=(0.9, 0.999), weight_decay=0.01, + paramwise_cfg=dict(custom_keys={'absolute_pos_embed': dict(decay_mult=0.), + 'relative_position_bias_table': dict(decay_mult=0.), + 'norm': dict(decay_mult=0.)})) + +lr_config = dict(_delete_=True, policy='poly', + warmup='linear', + warmup_iters=1500, + warmup_ratio=1e-6, + power=1.0, min_lr=0.0, by_epoch=False) + +data=dict(samples_per_gpu=2) \ No newline at end of file diff --git a/annotator/uniformer/exp/upernet_global_small/run.sh b/annotator/uniformer/exp/upernet_global_small/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..9fb22edfa7a32624ea08a63fe7d720c40db3b696 --- /dev/null +++ b/annotator/uniformer/exp/upernet_global_small/run.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +work_path=$(dirname $0) +PYTHONPATH="$(dirname $0)/../../":$PYTHONPATH \ +python -m torch.distributed.launch --nproc_per_node=8 \ + tools/train.py ${work_path}/config.py \ + --launcher pytorch \ + --options model.backbone.pretrained_path='your_model_path/uniformer_small_in1k.pth' \ + --work-dir ${work_path}/ckpt \ + 2>&1 | tee -a ${work_path}/log.txt diff --git a/annotator/uniformer/exp/upernet_global_small/test.sh b/annotator/uniformer/exp/upernet_global_small/test.sh new file mode 100644 index 0000000000000000000000000000000000000000..d9a85e7a0d3b7c96b060f473d41254b37a382fcb --- /dev/null +++ b/annotator/uniformer/exp/upernet_global_small/test.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +work_path=$(dirname $0) +PYTHONPATH="$(dirname $0)/../../":$PYTHONPATH \ +python -m torch.distributed.launch --nproc_per_node=8 \ + tools/test.py ${work_path}/test_config_h32.py \ + ${work_path}/ckpt/latest.pth \ + --launcher pytorch \ + --eval mIoU \ + 2>&1 | tee -a ${work_path}/log.txt diff --git a/annotator/uniformer/exp/upernet_global_small/test_config_g.py b/annotator/uniformer/exp/upernet_global_small/test_config_g.py new file mode 100644 index 0000000000000000000000000000000000000000..e43737a98a3b174a9f2fe059c06d511144686459 --- /dev/null +++ b/annotator/uniformer/exp/upernet_global_small/test_config_g.py @@ -0,0 +1,38 @@ +_base_ = [ + '../../configs/_base_/models/upernet_uniformer.py', + '../../configs/_base_/datasets/ade20k.py', + '../../configs/_base_/default_runtime.py', + '../../configs/_base_/schedules/schedule_160k.py' +] +model = dict( + backbone=dict( + type='UniFormer', + embed_dim=[64, 128, 320, 512], + layers=[3, 4, 8, 3], + head_dim=64, + drop_path_rate=0.25, + windows=False, + hybrid=False, + ), + decode_head=dict( + in_channels=[64, 128, 320, 512], + num_classes=150 + ), + auxiliary_head=dict( + in_channels=320, + num_classes=150 + )) + +# AdamW optimizer, no weight decay for position embedding & layer norm in backbone +optimizer = dict(_delete_=True, type='AdamW', lr=0.00006, betas=(0.9, 0.999), weight_decay=0.01, + paramwise_cfg=dict(custom_keys={'absolute_pos_embed': dict(decay_mult=0.), + 'relative_position_bias_table': dict(decay_mult=0.), + 'norm': dict(decay_mult=0.)})) + +lr_config = dict(_delete_=True, policy='poly', + warmup='linear', + warmup_iters=1500, + warmup_ratio=1e-6, + power=1.0, min_lr=0.0, by_epoch=False) + +data=dict(samples_per_gpu=2) \ No newline at end of file diff --git a/annotator/uniformer/exp/upernet_global_small/test_config_h32.py b/annotator/uniformer/exp/upernet_global_small/test_config_h32.py new file mode 100644 index 0000000000000000000000000000000000000000..a31e3874f76f9f7b089ac8834d85df2441af9b0e --- /dev/null +++ b/annotator/uniformer/exp/upernet_global_small/test_config_h32.py @@ -0,0 +1,39 @@ +_base_ = [ + '../../configs/_base_/models/upernet_uniformer.py', + '../../configs/_base_/datasets/ade20k.py', + '../../configs/_base_/default_runtime.py', + '../../configs/_base_/schedules/schedule_160k.py' +] +model = dict( + backbone=dict( + type='UniFormer', + embed_dim=[64, 128, 320, 512], + layers=[3, 4, 8, 3], + head_dim=64, + drop_path_rate=0.25, + windows=False, + hybrid=True, + window_size=32 + ), + decode_head=dict( + in_channels=[64, 128, 320, 512], + num_classes=150 + ), + auxiliary_head=dict( + in_channels=320, + num_classes=150 + )) + +# AdamW optimizer, no weight decay for position embedding & layer norm in backbone +optimizer = dict(_delete_=True, type='AdamW', lr=0.00006, betas=(0.9, 0.999), weight_decay=0.01, + paramwise_cfg=dict(custom_keys={'absolute_pos_embed': dict(decay_mult=0.), + 'relative_position_bias_table': dict(decay_mult=0.), + 'norm': dict(decay_mult=0.)})) + +lr_config = dict(_delete_=True, policy='poly', + warmup='linear', + warmup_iters=1500, + warmup_ratio=1e-6, + power=1.0, min_lr=0.0, by_epoch=False) + +data=dict(samples_per_gpu=2) \ No newline at end of file diff --git a/annotator/uniformer/exp/upernet_global_small/test_config_w32.py b/annotator/uniformer/exp/upernet_global_small/test_config_w32.py new file mode 100644 index 0000000000000000000000000000000000000000..3d9e06f029e46c14cb9ddb39319cabe86fef9b44 --- /dev/null +++ b/annotator/uniformer/exp/upernet_global_small/test_config_w32.py @@ -0,0 +1,39 @@ +_base_ = [ + '../../configs/_base_/models/upernet_uniformer.py', + '../../configs/_base_/datasets/ade20k.py', + '../../configs/_base_/default_runtime.py', + '../../configs/_base_/schedules/schedule_160k.py' +] +model = dict( + backbone=dict( + type='UniFormer', + embed_dim=[64, 128, 320, 512], + layers=[3, 4, 8, 3], + head_dim=64, + drop_path_rate=0.25, + windows=True, + hybrid=False, + window_size=32 + ), + decode_head=dict( + in_channels=[64, 128, 320, 512], + num_classes=150 + ), + auxiliary_head=dict( + in_channels=320, + num_classes=150 + )) + +# AdamW optimizer, no weight decay for position embedding & layer norm in backbone +optimizer = dict(_delete_=True, type='AdamW', lr=0.00006, betas=(0.9, 0.999), weight_decay=0.01, + paramwise_cfg=dict(custom_keys={'absolute_pos_embed': dict(decay_mult=0.), + 'relative_position_bias_table': dict(decay_mult=0.), + 'norm': dict(decay_mult=0.)})) + +lr_config = dict(_delete_=True, policy='poly', + warmup='linear', + warmup_iters=1500, + warmup_ratio=1e-6, + power=1.0, min_lr=0.0, by_epoch=False) + +data=dict(samples_per_gpu=2) \ No newline at end of file diff --git a/annotator/uniformer/inference.py b/annotator/uniformer/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..667d3e7ed5761bbe742226eb82f85bd952ca13bd --- /dev/null +++ b/annotator/uniformer/inference.py @@ -0,0 +1,144 @@ + +import torch + +try: + import mmcv as mmcv + from mmcv.parallel import collate, scatter + from mmcv.runner import load_checkpoint + from mmseg.datasets.pipelines import Compose + from mmseg.models import build_segmentor +except ImportError: + import annotator.mmpkg.mmcv as mmcv + from annotator.mmpkg.mmcv.parallel import collate, scatter + from annotator.mmpkg.mmcv.runner import load_checkpoint + from annotator.mmpkg.mmseg.datasets.pipelines import Compose + from annotator.mmpkg.mmseg.models import build_segmentor + +def init_segmentor(config, checkpoint=None, device='cuda:0'): + """Initialize a segmentor from config file. + + Args: + config (str or :obj:`mmcv.Config`): Config file path or the config + object. + checkpoint (str, optional): Checkpoint path. If left as None, the model + will not load any weights. + device (str, optional) CPU/CUDA device option. Default 'cuda:0'. + Use 'cpu' for loading model on CPU. + Returns: + nn.Module: The constructed segmentor. + """ + if isinstance(config, str): + config = mmcv.Config.fromfile(config) + elif not isinstance(config, mmcv.Config): + raise TypeError('config must be a filename or Config object, ' + 'but got {}'.format(type(config))) + config.model.pretrained = None + config.model.train_cfg = None + model = build_segmentor(config.model, test_cfg=config.get('test_cfg')) + if checkpoint is not None: + checkpoint = load_checkpoint(model, checkpoint, map_location='cpu') + model.CLASSES = checkpoint['meta']['CLASSES'] + model.PALETTE = checkpoint['meta']['PALETTE'] + model.cfg = config # save the config in the model for convenience + model.to(device) + model.eval() + return model + + +class LoadImage: + """A simple pipeline to load image.""" + + def __call__(self, results): + """Call function to load images into results. + + Args: + results (dict): A result dict contains the file name + of the image to be read. + + Returns: + dict: ``results`` will be returned containing loaded image. + """ + + if isinstance(results['img'], str): + results['filename'] = results['img'] + results['ori_filename'] = results['img'] + else: + results['filename'] = None + results['ori_filename'] = None + img = mmcv.imread(results['img']) + results['img'] = img + results['img_shape'] = img.shape + results['ori_shape'] = img.shape + return results + + +def inference_segmentor(model, img): + """Inference image(s) with the segmentor. + + Args: + model (nn.Module): The loaded segmentor. + imgs (str/ndarray or list[str/ndarray]): Either image files or loaded + images. + + Returns: + (list[Tensor]): The segmentation result. + """ + cfg = model.cfg + device = next(model.parameters()).device # model device + # build the data pipeline + test_pipeline = [LoadImage()] + cfg.data.test.pipeline[1:] + test_pipeline = Compose(test_pipeline) + # prepare data + data = dict(img=img) + data = test_pipeline(data) + data = collate([data], samples_per_gpu=1) + if next(model.parameters()).is_cuda: + # scatter to specified GPU + data = scatter(data, [device])[0] + else: + data['img_metas'] = [i.data[0] for i in data['img_metas']] + + data['img'] = [x.to(device) for x in data['img']] + + # forward the model + with torch.no_grad(): + result = model(return_loss=False, rescale=True, **data) + return result + + +def show_result_pyplot(model, + img, + result, + palette=None, + fig_size=(15, 10), + opacity=0.5, + title='', + block=True): + """Visualize the segmentation results on the image. + + Args: + model (nn.Module): The loaded segmentor. + img (str or np.ndarray): Image filename or loaded image. + result (list): The segmentation result. + palette (list[list[int]]] | None): The palette of segmentation + map. If None is given, random palette will be generated. + Default: None + fig_size (tuple): Figure size of the pyplot figure. + opacity(float): Opacity of painted segmentation map. + Default 0.5. + Must be in (0, 1] range. + title (str): The title of pyplot figure. + Default is ''. + block (bool): Whether to block the pyplot figure. + Default is True. + """ + if hasattr(model, 'module'): + model = model.module + img = model.show_result( + img, result, palette=palette, show=False, opacity=opacity) + # plt.figure(figsize=fig_size) + # plt.imshow(mmcv.bgr2rgb(img)) + # plt.title(title) + # plt.tight_layout() + # plt.show(block=block) + return mmcv.bgr2rgb(img) diff --git a/annotator/uniformer/mmcv/__init__.py b/annotator/uniformer/mmcv/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..210a2989138380559f23045b568d0fbbeb918c03 --- /dev/null +++ b/annotator/uniformer/mmcv/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# flake8: noqa +from .arraymisc import * +from .fileio import * +from .image import * +from .utils import * +from .version import * +from .video import * +from .visualization import * + +# The following modules are not imported to this level, so mmcv may be used +# without PyTorch. +# - runner +# - parallel +# - op diff --git a/annotator/uniformer/mmcv/arraymisc/__init__.py b/annotator/uniformer/mmcv/arraymisc/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4b4700d6139ae3d604ff6e542468cce4200c020c --- /dev/null +++ b/annotator/uniformer/mmcv/arraymisc/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .quantization import dequantize, quantize + +__all__ = ['quantize', 'dequantize'] diff --git a/annotator/uniformer/mmcv/arraymisc/quantization.py b/annotator/uniformer/mmcv/arraymisc/quantization.py new file mode 100644 index 0000000000000000000000000000000000000000..8e47a3545780cf071a1ef8195efb0b7b662c8186 --- /dev/null +++ b/annotator/uniformer/mmcv/arraymisc/quantization.py @@ -0,0 +1,55 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numpy as np + + +def quantize(arr, min_val, max_val, levels, dtype=np.int64): + """Quantize an array of (-inf, inf) to [0, levels-1]. + + Args: + arr (ndarray): Input array. + min_val (scalar): Minimum value to be clipped. + max_val (scalar): Maximum value to be clipped. + levels (int): Quantization levels. + dtype (np.type): The type of the quantized array. + + Returns: + tuple: Quantized array. + """ + if not (isinstance(levels, int) and levels > 1): + raise ValueError( + f'levels must be a positive integer, but got {levels}') + if min_val >= max_val: + raise ValueError( + f'min_val ({min_val}) must be smaller than max_val ({max_val})') + + arr = np.clip(arr, min_val, max_val) - min_val + quantized_arr = np.minimum( + np.floor(levels * arr / (max_val - min_val)).astype(dtype), levels - 1) + + return quantized_arr + + +def dequantize(arr, min_val, max_val, levels, dtype=np.float64): + """Dequantize an array. + + Args: + arr (ndarray): Input array. + min_val (scalar): Minimum value to be clipped. + max_val (scalar): Maximum value to be clipped. + levels (int): Quantization levels. + dtype (np.type): The type of the dequantized array. + + Returns: + tuple: Dequantized array. + """ + if not (isinstance(levels, int) and levels > 1): + raise ValueError( + f'levels must be a positive integer, but got {levels}') + if min_val >= max_val: + raise ValueError( + f'min_val ({min_val}) must be smaller than max_val ({max_val})') + + dequantized_arr = (arr + 0.5).astype(dtype) * (max_val - + min_val) / levels + min_val + + return dequantized_arr diff --git a/annotator/uniformer/mmcv/cnn/__init__.py b/annotator/uniformer/mmcv/cnn/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7246c897430f0cc7ce12719ad8608824fc734446 --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/__init__.py @@ -0,0 +1,41 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .alexnet import AlexNet +# yapf: disable +from .bricks import (ACTIVATION_LAYERS, CONV_LAYERS, NORM_LAYERS, + PADDING_LAYERS, PLUGIN_LAYERS, UPSAMPLE_LAYERS, + ContextBlock, Conv2d, Conv3d, ConvAWS2d, ConvModule, + ConvTranspose2d, ConvTranspose3d, ConvWS2d, + DepthwiseSeparableConvModule, GeneralizedAttention, + HSigmoid, HSwish, Linear, MaxPool2d, MaxPool3d, + NonLocal1d, NonLocal2d, NonLocal3d, Scale, Swish, + build_activation_layer, build_conv_layer, + build_norm_layer, build_padding_layer, build_plugin_layer, + build_upsample_layer, conv_ws_2d, is_norm) +from .builder import MODELS, build_model_from_cfg +# yapf: enable +from .resnet import ResNet, make_res_layer +from .utils import (INITIALIZERS, Caffe2XavierInit, ConstantInit, KaimingInit, + NormalInit, PretrainedInit, TruncNormalInit, UniformInit, + XavierInit, bias_init_with_prob, caffe2_xavier_init, + constant_init, fuse_conv_bn, get_model_complexity_info, + initialize, kaiming_init, normal_init, trunc_normal_init, + uniform_init, xavier_init) +from .vgg import VGG, make_vgg_layer + +__all__ = [ + 'AlexNet', 'VGG', 'make_vgg_layer', 'ResNet', 'make_res_layer', + 'constant_init', 'xavier_init', 'normal_init', 'trunc_normal_init', + 'uniform_init', 'kaiming_init', 'caffe2_xavier_init', + 'bias_init_with_prob', 'ConvModule', 'build_activation_layer', + 'build_conv_layer', 'build_norm_layer', 'build_padding_layer', + 'build_upsample_layer', 'build_plugin_layer', 'is_norm', 'NonLocal1d', + 'NonLocal2d', 'NonLocal3d', 'ContextBlock', 'HSigmoid', 'Swish', 'HSwish', + 'GeneralizedAttention', 'ACTIVATION_LAYERS', 'CONV_LAYERS', 'NORM_LAYERS', + 'PADDING_LAYERS', 'UPSAMPLE_LAYERS', 'PLUGIN_LAYERS', 'Scale', + 'get_model_complexity_info', 'conv_ws_2d', 'ConvAWS2d', 'ConvWS2d', + 'fuse_conv_bn', 'DepthwiseSeparableConvModule', 'Linear', 'Conv2d', + 'ConvTranspose2d', 'MaxPool2d', 'ConvTranspose3d', 'MaxPool3d', 'Conv3d', + 'initialize', 'INITIALIZERS', 'ConstantInit', 'XavierInit', 'NormalInit', + 'TruncNormalInit', 'UniformInit', 'KaimingInit', 'PretrainedInit', + 'Caffe2XavierInit', 'MODELS', 'build_model_from_cfg' +] diff --git a/annotator/uniformer/mmcv/cnn/alexnet.py b/annotator/uniformer/mmcv/cnn/alexnet.py new file mode 100644 index 0000000000000000000000000000000000000000..89e36b8c7851f895d9ae7f07149f0e707456aab0 --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/alexnet.py @@ -0,0 +1,61 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import logging + +import torch.nn as nn + + +class AlexNet(nn.Module): + """AlexNet backbone. + + Args: + num_classes (int): number of classes for classification. + """ + + def __init__(self, num_classes=-1): + super(AlexNet, self).__init__() + self.num_classes = num_classes + self.features = nn.Sequential( + nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=2), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=3, stride=2), + nn.Conv2d(64, 192, kernel_size=5, padding=2), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=3, stride=2), + nn.Conv2d(192, 384, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.Conv2d(384, 256, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.Conv2d(256, 256, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.MaxPool2d(kernel_size=3, stride=2), + ) + if self.num_classes > 0: + self.classifier = nn.Sequential( + nn.Dropout(), + nn.Linear(256 * 6 * 6, 4096), + nn.ReLU(inplace=True), + nn.Dropout(), + nn.Linear(4096, 4096), + nn.ReLU(inplace=True), + nn.Linear(4096, num_classes), + ) + + def init_weights(self, pretrained=None): + if isinstance(pretrained, str): + logger = logging.getLogger() + from ..runner import load_checkpoint + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + # use default initializer + pass + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + + x = self.features(x) + if self.num_classes > 0: + x = x.view(x.size(0), 256 * 6 * 6) + x = self.classifier(x) + + return x diff --git a/annotator/uniformer/mmcv/cnn/bricks/__init__.py b/annotator/uniformer/mmcv/cnn/bricks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0f33124ed23fc6f27119a37bcb5ab004d3572be0 --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/__init__.py @@ -0,0 +1,35 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .activation import build_activation_layer +from .context_block import ContextBlock +from .conv import build_conv_layer +from .conv2d_adaptive_padding import Conv2dAdaptivePadding +from .conv_module import ConvModule +from .conv_ws import ConvAWS2d, ConvWS2d, conv_ws_2d +from .depthwise_separable_conv_module import DepthwiseSeparableConvModule +from .drop import Dropout, DropPath +from .generalized_attention import GeneralizedAttention +from .hsigmoid import HSigmoid +from .hswish import HSwish +from .non_local import NonLocal1d, NonLocal2d, NonLocal3d +from .norm import build_norm_layer, is_norm +from .padding import build_padding_layer +from .plugin import build_plugin_layer +from .registry import (ACTIVATION_LAYERS, CONV_LAYERS, NORM_LAYERS, + PADDING_LAYERS, PLUGIN_LAYERS, UPSAMPLE_LAYERS) +from .scale import Scale +from .swish import Swish +from .upsample import build_upsample_layer +from .wrappers import (Conv2d, Conv3d, ConvTranspose2d, ConvTranspose3d, + Linear, MaxPool2d, MaxPool3d) + +__all__ = [ + 'ConvModule', 'build_activation_layer', 'build_conv_layer', + 'build_norm_layer', 'build_padding_layer', 'build_upsample_layer', + 'build_plugin_layer', 'is_norm', 'HSigmoid', 'HSwish', 'NonLocal1d', + 'NonLocal2d', 'NonLocal3d', 'ContextBlock', 'GeneralizedAttention', + 'ACTIVATION_LAYERS', 'CONV_LAYERS', 'NORM_LAYERS', 'PADDING_LAYERS', + 'UPSAMPLE_LAYERS', 'PLUGIN_LAYERS', 'Scale', 'ConvAWS2d', 'ConvWS2d', + 'conv_ws_2d', 'DepthwiseSeparableConvModule', 'Swish', 'Linear', + 'Conv2dAdaptivePadding', 'Conv2d', 'ConvTranspose2d', 'MaxPool2d', + 'ConvTranspose3d', 'MaxPool3d', 'Conv3d', 'Dropout', 'DropPath' +] diff --git a/annotator/uniformer/mmcv/cnn/bricks/activation.py b/annotator/uniformer/mmcv/cnn/bricks/activation.py new file mode 100644 index 0000000000000000000000000000000000000000..cab2712287d5ef7be2f079dcb54a94b96394eab5 --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/activation.py @@ -0,0 +1,92 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from annotator.uniformer.mmcv.utils import TORCH_VERSION, build_from_cfg, digit_version +from .registry import ACTIVATION_LAYERS + +for module in [ + nn.ReLU, nn.LeakyReLU, nn.PReLU, nn.RReLU, nn.ReLU6, nn.ELU, + nn.Sigmoid, nn.Tanh +]: + ACTIVATION_LAYERS.register_module(module=module) + + +@ACTIVATION_LAYERS.register_module(name='Clip') +@ACTIVATION_LAYERS.register_module() +class Clamp(nn.Module): + """Clamp activation layer. + + This activation function is to clamp the feature map value within + :math:`[min, max]`. More details can be found in ``torch.clamp()``. + + Args: + min (Number | optional): Lower-bound of the range to be clamped to. + Default to -1. + max (Number | optional): Upper-bound of the range to be clamped to. + Default to 1. + """ + + def __init__(self, min=-1., max=1.): + super(Clamp, self).__init__() + self.min = min + self.max = max + + def forward(self, x): + """Forward function. + + Args: + x (torch.Tensor): The input tensor. + + Returns: + torch.Tensor: Clamped tensor. + """ + return torch.clamp(x, min=self.min, max=self.max) + + +class GELU(nn.Module): + r"""Applies the Gaussian Error Linear Units function: + + .. math:: + \text{GELU}(x) = x * \Phi(x) + where :math:`\Phi(x)` is the Cumulative Distribution Function for + Gaussian Distribution. + + Shape: + - Input: :math:`(N, *)` where `*` means, any number of additional + dimensions + - Output: :math:`(N, *)`, same shape as the input + + .. image:: scripts/activation_images/GELU.png + + Examples:: + + >>> m = nn.GELU() + >>> input = torch.randn(2) + >>> output = m(input) + """ + + def forward(self, input): + return F.gelu(input) + + +if (TORCH_VERSION == 'parrots' + or digit_version(TORCH_VERSION) < digit_version('1.4')): + ACTIVATION_LAYERS.register_module(module=GELU) +else: + ACTIVATION_LAYERS.register_module(module=nn.GELU) + + +def build_activation_layer(cfg): + """Build activation layer. + + Args: + cfg (dict): The activation layer config, which should contain: + - type (str): Layer type. + - layer args: Args needed to instantiate an activation layer. + + Returns: + nn.Module: Created activation layer. + """ + return build_from_cfg(cfg, ACTIVATION_LAYERS) diff --git a/annotator/uniformer/mmcv/cnn/bricks/context_block.py b/annotator/uniformer/mmcv/cnn/bricks/context_block.py new file mode 100644 index 0000000000000000000000000000000000000000..d60fdb904c749ce3b251510dff3cc63cea70d42e --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/context_block.py @@ -0,0 +1,125 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch import nn + +from ..utils import constant_init, kaiming_init +from .registry import PLUGIN_LAYERS + + +def last_zero_init(m): + if isinstance(m, nn.Sequential): + constant_init(m[-1], val=0) + else: + constant_init(m, val=0) + + +@PLUGIN_LAYERS.register_module() +class ContextBlock(nn.Module): + """ContextBlock module in GCNet. + + See 'GCNet: Non-local Networks Meet Squeeze-Excitation Networks and Beyond' + (https://arxiv.org/abs/1904.11492) for details. + + Args: + in_channels (int): Channels of the input feature map. + ratio (float): Ratio of channels of transform bottleneck + pooling_type (str): Pooling method for context modeling. + Options are 'att' and 'avg', stand for attention pooling and + average pooling respectively. Default: 'att'. + fusion_types (Sequence[str]): Fusion method for feature fusion, + Options are 'channels_add', 'channel_mul', stand for channelwise + addition and multiplication respectively. Default: ('channel_add',) + """ + + _abbr_ = 'context_block' + + def __init__(self, + in_channels, + ratio, + pooling_type='att', + fusion_types=('channel_add', )): + super(ContextBlock, self).__init__() + assert pooling_type in ['avg', 'att'] + assert isinstance(fusion_types, (list, tuple)) + valid_fusion_types = ['channel_add', 'channel_mul'] + assert all([f in valid_fusion_types for f in fusion_types]) + assert len(fusion_types) > 0, 'at least one fusion should be used' + self.in_channels = in_channels + self.ratio = ratio + self.planes = int(in_channels * ratio) + self.pooling_type = pooling_type + self.fusion_types = fusion_types + if pooling_type == 'att': + self.conv_mask = nn.Conv2d(in_channels, 1, kernel_size=1) + self.softmax = nn.Softmax(dim=2) + else: + self.avg_pool = nn.AdaptiveAvgPool2d(1) + if 'channel_add' in fusion_types: + self.channel_add_conv = nn.Sequential( + nn.Conv2d(self.in_channels, self.planes, kernel_size=1), + nn.LayerNorm([self.planes, 1, 1]), + nn.ReLU(inplace=True), # yapf: disable + nn.Conv2d(self.planes, self.in_channels, kernel_size=1)) + else: + self.channel_add_conv = None + if 'channel_mul' in fusion_types: + self.channel_mul_conv = nn.Sequential( + nn.Conv2d(self.in_channels, self.planes, kernel_size=1), + nn.LayerNorm([self.planes, 1, 1]), + nn.ReLU(inplace=True), # yapf: disable + nn.Conv2d(self.planes, self.in_channels, kernel_size=1)) + else: + self.channel_mul_conv = None + self.reset_parameters() + + def reset_parameters(self): + if self.pooling_type == 'att': + kaiming_init(self.conv_mask, mode='fan_in') + self.conv_mask.inited = True + + if self.channel_add_conv is not None: + last_zero_init(self.channel_add_conv) + if self.channel_mul_conv is not None: + last_zero_init(self.channel_mul_conv) + + def spatial_pool(self, x): + batch, channel, height, width = x.size() + if self.pooling_type == 'att': + input_x = x + # [N, C, H * W] + input_x = input_x.view(batch, channel, height * width) + # [N, 1, C, H * W] + input_x = input_x.unsqueeze(1) + # [N, 1, H, W] + context_mask = self.conv_mask(x) + # [N, 1, H * W] + context_mask = context_mask.view(batch, 1, height * width) + # [N, 1, H * W] + context_mask = self.softmax(context_mask) + # [N, 1, H * W, 1] + context_mask = context_mask.unsqueeze(-1) + # [N, 1, C, 1] + context = torch.matmul(input_x, context_mask) + # [N, C, 1, 1] + context = context.view(batch, channel, 1, 1) + else: + # [N, C, 1, 1] + context = self.avg_pool(x) + + return context + + def forward(self, x): + # [N, C, 1, 1] + context = self.spatial_pool(x) + + out = x + if self.channel_mul_conv is not None: + # [N, C, 1, 1] + channel_mul_term = torch.sigmoid(self.channel_mul_conv(context)) + out = out * channel_mul_term + if self.channel_add_conv is not None: + # [N, C, 1, 1] + channel_add_term = self.channel_add_conv(context) + out = out + channel_add_term + + return out diff --git a/annotator/uniformer/mmcv/cnn/bricks/conv.py b/annotator/uniformer/mmcv/cnn/bricks/conv.py new file mode 100644 index 0000000000000000000000000000000000000000..cf54491997a48ac3e7fadc4183ab7bf3e831024c --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/conv.py @@ -0,0 +1,44 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from torch import nn + +from .registry import CONV_LAYERS + +CONV_LAYERS.register_module('Conv1d', module=nn.Conv1d) +CONV_LAYERS.register_module('Conv2d', module=nn.Conv2d) +CONV_LAYERS.register_module('Conv3d', module=nn.Conv3d) +CONV_LAYERS.register_module('Conv', module=nn.Conv2d) + + +def build_conv_layer(cfg, *args, **kwargs): + """Build convolution layer. + + Args: + cfg (None or dict): The conv layer config, which should contain: + - type (str): Layer type. + - layer args: Args needed to instantiate an conv layer. + args (argument list): Arguments passed to the `__init__` + method of the corresponding conv layer. + kwargs (keyword arguments): Keyword arguments passed to the `__init__` + method of the corresponding conv layer. + + Returns: + nn.Module: Created conv layer. + """ + if cfg is None: + cfg_ = dict(type='Conv2d') + else: + if not isinstance(cfg, dict): + raise TypeError('cfg must be a dict') + if 'type' not in cfg: + raise KeyError('the cfg dict must contain the key "type"') + cfg_ = cfg.copy() + + layer_type = cfg_.pop('type') + if layer_type not in CONV_LAYERS: + raise KeyError(f'Unrecognized norm type {layer_type}') + else: + conv_layer = CONV_LAYERS.get(layer_type) + + layer = conv_layer(*args, **kwargs, **cfg_) + + return layer diff --git a/annotator/uniformer/mmcv/cnn/bricks/conv2d_adaptive_padding.py b/annotator/uniformer/mmcv/cnn/bricks/conv2d_adaptive_padding.py new file mode 100644 index 0000000000000000000000000000000000000000..b45e758ac6cf8dfb0382d072fe09125bc7e9b888 --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/conv2d_adaptive_padding.py @@ -0,0 +1,62 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import math + +from torch import nn +from torch.nn import functional as F + +from .registry import CONV_LAYERS + + +@CONV_LAYERS.register_module() +class Conv2dAdaptivePadding(nn.Conv2d): + """Implementation of 2D convolution in tensorflow with `padding` as "same", + which applies padding to input (if needed) so that input image gets fully + covered by filter and stride you specified. For stride 1, this will ensure + that output image size is same as input. For stride of 2, output dimensions + will be half, for example. + + Args: + in_channels (int): Number of channels in the input image + out_channels (int): Number of channels produced by the convolution + kernel_size (int or tuple): Size of the convolving kernel + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of + the input. Default: 0 + dilation (int or tuple, optional): Spacing between kernel elements. + Default: 1 + groups (int, optional): Number of blocked connections from input + channels to output channels. Default: 1 + bias (bool, optional): If ``True``, adds a learnable bias to the + output. Default: ``True`` + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + bias=True): + super().__init__(in_channels, out_channels, kernel_size, stride, 0, + dilation, groups, bias) + + def forward(self, x): + img_h, img_w = x.size()[-2:] + kernel_h, kernel_w = self.weight.size()[-2:] + stride_h, stride_w = self.stride + output_h = math.ceil(img_h / stride_h) + output_w = math.ceil(img_w / stride_w) + pad_h = ( + max((output_h - 1) * self.stride[0] + + (kernel_h - 1) * self.dilation[0] + 1 - img_h, 0)) + pad_w = ( + max((output_w - 1) * self.stride[1] + + (kernel_w - 1) * self.dilation[1] + 1 - img_w, 0)) + if pad_h > 0 or pad_w > 0: + x = F.pad(x, [ + pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2 + ]) + return F.conv2d(x, self.weight, self.bias, self.stride, self.padding, + self.dilation, self.groups) diff --git a/annotator/uniformer/mmcv/cnn/bricks/conv_module.py b/annotator/uniformer/mmcv/cnn/bricks/conv_module.py new file mode 100644 index 0000000000000000000000000000000000000000..e60e7e62245071c77b652093fddebff3948d7c3e --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/conv_module.py @@ -0,0 +1,206 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import warnings + +import torch.nn as nn + +from annotator.uniformer.mmcv.utils import _BatchNorm, _InstanceNorm +from ..utils import constant_init, kaiming_init +from .activation import build_activation_layer +from .conv import build_conv_layer +from .norm import build_norm_layer +from .padding import build_padding_layer +from .registry import PLUGIN_LAYERS + + +@PLUGIN_LAYERS.register_module() +class ConvModule(nn.Module): + """A conv block that bundles conv/norm/activation layers. + + This block simplifies the usage of convolution layers, which are commonly + used with a norm layer (e.g., BatchNorm) and activation layer (e.g., ReLU). + It is based upon three build methods: `build_conv_layer()`, + `build_norm_layer()` and `build_activation_layer()`. + + Besides, we add some additional features in this module. + 1. Automatically set `bias` of the conv layer. + 2. Spectral norm is supported. + 3. More padding modes are supported. Before PyTorch 1.5, nn.Conv2d only + supports zero and circular padding, and we add "reflect" padding mode. + + Args: + in_channels (int): Number of channels in the input feature map. + Same as that in ``nn._ConvNd``. + out_channels (int): Number of channels produced by the convolution. + Same as that in ``nn._ConvNd``. + kernel_size (int | tuple[int]): Size of the convolving kernel. + Same as that in ``nn._ConvNd``. + stride (int | tuple[int]): Stride of the convolution. + Same as that in ``nn._ConvNd``. + padding (int | tuple[int]): Zero-padding added to both sides of + the input. Same as that in ``nn._ConvNd``. + dilation (int | tuple[int]): Spacing between kernel elements. + Same as that in ``nn._ConvNd``. + groups (int): Number of blocked connections from input channels to + output channels. Same as that in ``nn._ConvNd``. + bias (bool | str): If specified as `auto`, it will be decided by the + norm_cfg. Bias will be set as True if `norm_cfg` is None, otherwise + False. Default: "auto". + conv_cfg (dict): Config dict for convolution layer. Default: None, + which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. Default: None. + act_cfg (dict): Config dict for activation layer. + Default: dict(type='ReLU'). + inplace (bool): Whether to use inplace mode for activation. + Default: True. + with_spectral_norm (bool): Whether use spectral norm in conv module. + Default: False. + padding_mode (str): If the `padding_mode` has not been supported by + current `Conv2d` in PyTorch, we will use our own padding layer + instead. Currently, we support ['zeros', 'circular'] with official + implementation and ['reflect'] with our own implementation. + Default: 'zeros'. + order (tuple[str]): The order of conv/norm/activation layers. It is a + sequence of "conv", "norm" and "act". Common examples are + ("conv", "norm", "act") and ("act", "conv", "norm"). + Default: ('conv', 'norm', 'act'). + """ + + _abbr_ = 'conv_block' + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + bias='auto', + conv_cfg=None, + norm_cfg=None, + act_cfg=dict(type='ReLU'), + inplace=True, + with_spectral_norm=False, + padding_mode='zeros', + order=('conv', 'norm', 'act')): + super(ConvModule, self).__init__() + assert conv_cfg is None or isinstance(conv_cfg, dict) + assert norm_cfg is None or isinstance(norm_cfg, dict) + assert act_cfg is None or isinstance(act_cfg, dict) + official_padding_mode = ['zeros', 'circular'] + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.inplace = inplace + self.with_spectral_norm = with_spectral_norm + self.with_explicit_padding = padding_mode not in official_padding_mode + self.order = order + assert isinstance(self.order, tuple) and len(self.order) == 3 + assert set(order) == set(['conv', 'norm', 'act']) + + self.with_norm = norm_cfg is not None + self.with_activation = act_cfg is not None + # if the conv layer is before a norm layer, bias is unnecessary. + if bias == 'auto': + bias = not self.with_norm + self.with_bias = bias + + if self.with_explicit_padding: + pad_cfg = dict(type=padding_mode) + self.padding_layer = build_padding_layer(pad_cfg, padding) + + # reset padding to 0 for conv module + conv_padding = 0 if self.with_explicit_padding else padding + # build convolution layer + self.conv = build_conv_layer( + conv_cfg, + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=conv_padding, + dilation=dilation, + groups=groups, + bias=bias) + # export the attributes of self.conv to a higher level for convenience + self.in_channels = self.conv.in_channels + self.out_channels = self.conv.out_channels + self.kernel_size = self.conv.kernel_size + self.stride = self.conv.stride + self.padding = padding + self.dilation = self.conv.dilation + self.transposed = self.conv.transposed + self.output_padding = self.conv.output_padding + self.groups = self.conv.groups + + if self.with_spectral_norm: + self.conv = nn.utils.spectral_norm(self.conv) + + # build normalization layers + if self.with_norm: + # norm layer is after conv layer + if order.index('norm') > order.index('conv'): + norm_channels = out_channels + else: + norm_channels = in_channels + self.norm_name, norm = build_norm_layer(norm_cfg, norm_channels) + self.add_module(self.norm_name, norm) + if self.with_bias: + if isinstance(norm, (_BatchNorm, _InstanceNorm)): + warnings.warn( + 'Unnecessary conv bias before batch/instance norm') + else: + self.norm_name = None + + # build activation layer + if self.with_activation: + act_cfg_ = act_cfg.copy() + # nn.Tanh has no 'inplace' argument + if act_cfg_['type'] not in [ + 'Tanh', 'PReLU', 'Sigmoid', 'HSigmoid', 'Swish' + ]: + act_cfg_.setdefault('inplace', inplace) + self.activate = build_activation_layer(act_cfg_) + + # Use msra init by default + self.init_weights() + + @property + def norm(self): + if self.norm_name: + return getattr(self, self.norm_name) + else: + return None + + def init_weights(self): + # 1. It is mainly for customized conv layers with their own + # initialization manners by calling their own ``init_weights()``, + # and we do not want ConvModule to override the initialization. + # 2. For customized conv layers without their own initialization + # manners (that is, they don't have their own ``init_weights()``) + # and PyTorch's conv layers, they will be initialized by + # this method with default ``kaiming_init``. + # Note: For PyTorch's conv layers, they will be overwritten by our + # initialization implementation using default ``kaiming_init``. + if not hasattr(self.conv, 'init_weights'): + if self.with_activation and self.act_cfg['type'] == 'LeakyReLU': + nonlinearity = 'leaky_relu' + a = self.act_cfg.get('negative_slope', 0.01) + else: + nonlinearity = 'relu' + a = 0 + kaiming_init(self.conv, a=a, nonlinearity=nonlinearity) + if self.with_norm: + constant_init(self.norm, 1, bias=0) + + def forward(self, x, activate=True, norm=True): + for layer in self.order: + if layer == 'conv': + if self.with_explicit_padding: + x = self.padding_layer(x) + x = self.conv(x) + elif layer == 'norm' and norm and self.with_norm: + x = self.norm(x) + elif layer == 'act' and activate and self.with_activation: + x = self.activate(x) + return x diff --git a/annotator/uniformer/mmcv/cnn/bricks/conv_ws.py b/annotator/uniformer/mmcv/cnn/bricks/conv_ws.py new file mode 100644 index 0000000000000000000000000000000000000000..a3941e27874993418b3b5708d5a7485f175ff9c8 --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/conv_ws.py @@ -0,0 +1,148 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from .registry import CONV_LAYERS + + +def conv_ws_2d(input, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1, + eps=1e-5): + c_in = weight.size(0) + weight_flat = weight.view(c_in, -1) + mean = weight_flat.mean(dim=1, keepdim=True).view(c_in, 1, 1, 1) + std = weight_flat.std(dim=1, keepdim=True).view(c_in, 1, 1, 1) + weight = (weight - mean) / (std + eps) + return F.conv2d(input, weight, bias, stride, padding, dilation, groups) + + +@CONV_LAYERS.register_module('ConvWS') +class ConvWS2d(nn.Conv2d): + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + bias=True, + eps=1e-5): + super(ConvWS2d, self).__init__( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + bias=bias) + self.eps = eps + + def forward(self, x): + return conv_ws_2d(x, self.weight, self.bias, self.stride, self.padding, + self.dilation, self.groups, self.eps) + + +@CONV_LAYERS.register_module(name='ConvAWS') +class ConvAWS2d(nn.Conv2d): + """AWS (Adaptive Weight Standardization) + + This is a variant of Weight Standardization + (https://arxiv.org/pdf/1903.10520.pdf) + It is used in DetectoRS to avoid NaN + (https://arxiv.org/pdf/2006.02334.pdf) + + Args: + in_channels (int): Number of channels in the input image + out_channels (int): Number of channels produced by the convolution + kernel_size (int or tuple): Size of the conv kernel + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of + the input. Default: 0 + dilation (int or tuple, optional): Spacing between kernel elements. + Default: 1 + groups (int, optional): Number of blocked connections from input + channels to output channels. Default: 1 + bias (bool, optional): If set True, adds a learnable bias to the + output. Default: True + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + bias=True): + super().__init__( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + bias=bias) + self.register_buffer('weight_gamma', + torch.ones(self.out_channels, 1, 1, 1)) + self.register_buffer('weight_beta', + torch.zeros(self.out_channels, 1, 1, 1)) + + def _get_weight(self, weight): + weight_flat = weight.view(weight.size(0), -1) + mean = weight_flat.mean(dim=1).view(-1, 1, 1, 1) + std = torch.sqrt(weight_flat.var(dim=1) + 1e-5).view(-1, 1, 1, 1) + weight = (weight - mean) / std + weight = self.weight_gamma * weight + self.weight_beta + return weight + + def forward(self, x): + weight = self._get_weight(self.weight) + return F.conv2d(x, weight, self.bias, self.stride, self.padding, + self.dilation, self.groups) + + def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs): + """Override default load function. + + AWS overrides the function _load_from_state_dict to recover + weight_gamma and weight_beta if they are missing. If weight_gamma and + weight_beta are found in the checkpoint, this function will return + after super()._load_from_state_dict. Otherwise, it will compute the + mean and std of the pretrained weights and store them in weight_beta + and weight_gamma. + """ + + self.weight_gamma.data.fill_(-1) + local_missing_keys = [] + super()._load_from_state_dict(state_dict, prefix, local_metadata, + strict, local_missing_keys, + unexpected_keys, error_msgs) + if self.weight_gamma.data.mean() > 0: + for k in local_missing_keys: + missing_keys.append(k) + return + weight = self.weight.data + weight_flat = weight.view(weight.size(0), -1) + mean = weight_flat.mean(dim=1).view(-1, 1, 1, 1) + std = torch.sqrt(weight_flat.var(dim=1) + 1e-5).view(-1, 1, 1, 1) + self.weight_beta.data.copy_(mean) + self.weight_gamma.data.copy_(std) + missing_gamma_beta = [ + k for k in local_missing_keys + if k.endswith('weight_gamma') or k.endswith('weight_beta') + ] + for k in missing_gamma_beta: + local_missing_keys.remove(k) + for k in local_missing_keys: + missing_keys.append(k) diff --git a/annotator/uniformer/mmcv/cnn/bricks/depthwise_separable_conv_module.py b/annotator/uniformer/mmcv/cnn/bricks/depthwise_separable_conv_module.py new file mode 100644 index 0000000000000000000000000000000000000000..722d5d8d71f75486e2db3008907c4eadfca41d63 --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/depthwise_separable_conv_module.py @@ -0,0 +1,96 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch.nn as nn + +from .conv_module import ConvModule + + +class DepthwiseSeparableConvModule(nn.Module): + """Depthwise separable convolution module. + + See https://arxiv.org/pdf/1704.04861.pdf for details. + + This module can replace a ConvModule with the conv block replaced by two + conv block: depthwise conv block and pointwise conv block. The depthwise + conv block contains depthwise-conv/norm/activation layers. The pointwise + conv block contains pointwise-conv/norm/activation layers. It should be + noted that there will be norm/activation layer in the depthwise conv block + if `norm_cfg` and `act_cfg` are specified. + + Args: + in_channels (int): Number of channels in the input feature map. + Same as that in ``nn._ConvNd``. + out_channels (int): Number of channels produced by the convolution. + Same as that in ``nn._ConvNd``. + kernel_size (int | tuple[int]): Size of the convolving kernel. + Same as that in ``nn._ConvNd``. + stride (int | tuple[int]): Stride of the convolution. + Same as that in ``nn._ConvNd``. Default: 1. + padding (int | tuple[int]): Zero-padding added to both sides of + the input. Same as that in ``nn._ConvNd``. Default: 0. + dilation (int | tuple[int]): Spacing between kernel elements. + Same as that in ``nn._ConvNd``. Default: 1. + norm_cfg (dict): Default norm config for both depthwise ConvModule and + pointwise ConvModule. Default: None. + act_cfg (dict): Default activation config for both depthwise ConvModule + and pointwise ConvModule. Default: dict(type='ReLU'). + dw_norm_cfg (dict): Norm config of depthwise ConvModule. If it is + 'default', it will be the same as `norm_cfg`. Default: 'default'. + dw_act_cfg (dict): Activation config of depthwise ConvModule. If it is + 'default', it will be the same as `act_cfg`. Default: 'default'. + pw_norm_cfg (dict): Norm config of pointwise ConvModule. If it is + 'default', it will be the same as `norm_cfg`. Default: 'default'. + pw_act_cfg (dict): Activation config of pointwise ConvModule. If it is + 'default', it will be the same as `act_cfg`. Default: 'default'. + kwargs (optional): Other shared arguments for depthwise and pointwise + ConvModule. See ConvModule for ref. + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + norm_cfg=None, + act_cfg=dict(type='ReLU'), + dw_norm_cfg='default', + dw_act_cfg='default', + pw_norm_cfg='default', + pw_act_cfg='default', + **kwargs): + super(DepthwiseSeparableConvModule, self).__init__() + assert 'groups' not in kwargs, 'groups should not be specified' + + # if norm/activation config of depthwise/pointwise ConvModule is not + # specified, use default config. + dw_norm_cfg = dw_norm_cfg if dw_norm_cfg != 'default' else norm_cfg + dw_act_cfg = dw_act_cfg if dw_act_cfg != 'default' else act_cfg + pw_norm_cfg = pw_norm_cfg if pw_norm_cfg != 'default' else norm_cfg + pw_act_cfg = pw_act_cfg if pw_act_cfg != 'default' else act_cfg + + # depthwise convolution + self.depthwise_conv = ConvModule( + in_channels, + in_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=in_channels, + norm_cfg=dw_norm_cfg, + act_cfg=dw_act_cfg, + **kwargs) + + self.pointwise_conv = ConvModule( + in_channels, + out_channels, + 1, + norm_cfg=pw_norm_cfg, + act_cfg=pw_act_cfg, + **kwargs) + + def forward(self, x): + x = self.depthwise_conv(x) + x = self.pointwise_conv(x) + return x diff --git a/annotator/uniformer/mmcv/cnn/bricks/drop.py b/annotator/uniformer/mmcv/cnn/bricks/drop.py new file mode 100644 index 0000000000000000000000000000000000000000..b7b4fccd457a0d51fb10c789df3c8537fe7b67c1 --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/drop.py @@ -0,0 +1,65 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn + +from annotator.uniformer.mmcv import build_from_cfg +from .registry import DROPOUT_LAYERS + + +def drop_path(x, drop_prob=0., training=False): + """Drop paths (Stochastic Depth) per sample (when applied in main path of + residual blocks). + + We follow the implementation + https://github.com/rwightman/pytorch-image-models/blob/a2727c1bf78ba0d7b5727f5f95e37fb7f8866b1f/timm/models/layers/drop.py # noqa: E501 + """ + if drop_prob == 0. or not training: + return x + keep_prob = 1 - drop_prob + # handle tensors with different dimensions, not just 4D tensors. + shape = (x.shape[0], ) + (1, ) * (x.ndim - 1) + random_tensor = keep_prob + torch.rand( + shape, dtype=x.dtype, device=x.device) + output = x.div(keep_prob) * random_tensor.floor() + return output + + +@DROPOUT_LAYERS.register_module() +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of + residual blocks). + + We follow the implementation + https://github.com/rwightman/pytorch-image-models/blob/a2727c1bf78ba0d7b5727f5f95e37fb7f8866b1f/timm/models/layers/drop.py # noqa: E501 + + Args: + drop_prob (float): Probability of the path to be zeroed. Default: 0.1 + """ + + def __init__(self, drop_prob=0.1): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) + + +@DROPOUT_LAYERS.register_module() +class Dropout(nn.Dropout): + """A wrapper for ``torch.nn.Dropout``, We rename the ``p`` of + ``torch.nn.Dropout`` to ``drop_prob`` so as to be consistent with + ``DropPath`` + + Args: + drop_prob (float): Probability of the elements to be + zeroed. Default: 0.5. + inplace (bool): Do the operation inplace or not. Default: False. + """ + + def __init__(self, drop_prob=0.5, inplace=False): + super().__init__(p=drop_prob, inplace=inplace) + + +def build_dropout(cfg, default_args=None): + """Builder for drop out layers.""" + return build_from_cfg(cfg, DROPOUT_LAYERS, default_args) diff --git a/annotator/uniformer/mmcv/cnn/bricks/generalized_attention.py b/annotator/uniformer/mmcv/cnn/bricks/generalized_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..988d9adf2f289ef223bd1c680a5ae1d3387f0269 --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/generalized_attention.py @@ -0,0 +1,412 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import math + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..utils import kaiming_init +from .registry import PLUGIN_LAYERS + + +@PLUGIN_LAYERS.register_module() +class GeneralizedAttention(nn.Module): + """GeneralizedAttention module. + + See 'An Empirical Study of Spatial Attention Mechanisms in Deep Networks' + (https://arxiv.org/abs/1711.07971) for details. + + Args: + in_channels (int): Channels of the input feature map. + spatial_range (int): The spatial range. -1 indicates no spatial range + constraint. Default: -1. + num_heads (int): The head number of empirical_attention module. + Default: 9. + position_embedding_dim (int): The position embedding dimension. + Default: -1. + position_magnitude (int): A multiplier acting on coord difference. + Default: 1. + kv_stride (int): The feature stride acting on key/value feature map. + Default: 2. + q_stride (int): The feature stride acting on query feature map. + Default: 1. + attention_type (str): A binary indicator string for indicating which + items in generalized empirical_attention module are used. + Default: '1111'. + + - '1000' indicates 'query and key content' (appr - appr) item, + - '0100' indicates 'query content and relative position' + (appr - position) item, + - '0010' indicates 'key content only' (bias - appr) item, + - '0001' indicates 'relative position only' (bias - position) item. + """ + + _abbr_ = 'gen_attention_block' + + def __init__(self, + in_channels, + spatial_range=-1, + num_heads=9, + position_embedding_dim=-1, + position_magnitude=1, + kv_stride=2, + q_stride=1, + attention_type='1111'): + + super(GeneralizedAttention, self).__init__() + + # hard range means local range for non-local operation + self.position_embedding_dim = ( + position_embedding_dim + if position_embedding_dim > 0 else in_channels) + + self.position_magnitude = position_magnitude + self.num_heads = num_heads + self.in_channels = in_channels + self.spatial_range = spatial_range + self.kv_stride = kv_stride + self.q_stride = q_stride + self.attention_type = [bool(int(_)) for _ in attention_type] + self.qk_embed_dim = in_channels // num_heads + out_c = self.qk_embed_dim * num_heads + + if self.attention_type[0] or self.attention_type[1]: + self.query_conv = nn.Conv2d( + in_channels=in_channels, + out_channels=out_c, + kernel_size=1, + bias=False) + self.query_conv.kaiming_init = True + + if self.attention_type[0] or self.attention_type[2]: + self.key_conv = nn.Conv2d( + in_channels=in_channels, + out_channels=out_c, + kernel_size=1, + bias=False) + self.key_conv.kaiming_init = True + + self.v_dim = in_channels // num_heads + self.value_conv = nn.Conv2d( + in_channels=in_channels, + out_channels=self.v_dim * num_heads, + kernel_size=1, + bias=False) + self.value_conv.kaiming_init = True + + if self.attention_type[1] or self.attention_type[3]: + self.appr_geom_fc_x = nn.Linear( + self.position_embedding_dim // 2, out_c, bias=False) + self.appr_geom_fc_x.kaiming_init = True + + self.appr_geom_fc_y = nn.Linear( + self.position_embedding_dim // 2, out_c, bias=False) + self.appr_geom_fc_y.kaiming_init = True + + if self.attention_type[2]: + stdv = 1.0 / math.sqrt(self.qk_embed_dim * 2) + appr_bias_value = -2 * stdv * torch.rand(out_c) + stdv + self.appr_bias = nn.Parameter(appr_bias_value) + + if self.attention_type[3]: + stdv = 1.0 / math.sqrt(self.qk_embed_dim * 2) + geom_bias_value = -2 * stdv * torch.rand(out_c) + stdv + self.geom_bias = nn.Parameter(geom_bias_value) + + self.proj_conv = nn.Conv2d( + in_channels=self.v_dim * num_heads, + out_channels=in_channels, + kernel_size=1, + bias=True) + self.proj_conv.kaiming_init = True + self.gamma = nn.Parameter(torch.zeros(1)) + + if self.spatial_range >= 0: + # only works when non local is after 3*3 conv + if in_channels == 256: + max_len = 84 + elif in_channels == 512: + max_len = 42 + + max_len_kv = int((max_len - 1.0) / self.kv_stride + 1) + local_constraint_map = np.ones( + (max_len, max_len, max_len_kv, max_len_kv), dtype=np.int) + for iy in range(max_len): + for ix in range(max_len): + local_constraint_map[ + iy, ix, + max((iy - self.spatial_range) // + self.kv_stride, 0):min((iy + self.spatial_range + + 1) // self.kv_stride + + 1, max_len), + max((ix - self.spatial_range) // + self.kv_stride, 0):min((ix + self.spatial_range + + 1) // self.kv_stride + + 1, max_len)] = 0 + + self.local_constraint_map = nn.Parameter( + torch.from_numpy(local_constraint_map).byte(), + requires_grad=False) + + if self.q_stride > 1: + self.q_downsample = nn.AvgPool2d( + kernel_size=1, stride=self.q_stride) + else: + self.q_downsample = None + + if self.kv_stride > 1: + self.kv_downsample = nn.AvgPool2d( + kernel_size=1, stride=self.kv_stride) + else: + self.kv_downsample = None + + self.init_weights() + + def get_position_embedding(self, + h, + w, + h_kv, + w_kv, + q_stride, + kv_stride, + device, + dtype, + feat_dim, + wave_length=1000): + # the default type of Tensor is float32, leading to type mismatch + # in fp16 mode. Cast it to support fp16 mode. + h_idxs = torch.linspace(0, h - 1, h).to(device=device, dtype=dtype) + h_idxs = h_idxs.view((h, 1)) * q_stride + + w_idxs = torch.linspace(0, w - 1, w).to(device=device, dtype=dtype) + w_idxs = w_idxs.view((w, 1)) * q_stride + + h_kv_idxs = torch.linspace(0, h_kv - 1, h_kv).to( + device=device, dtype=dtype) + h_kv_idxs = h_kv_idxs.view((h_kv, 1)) * kv_stride + + w_kv_idxs = torch.linspace(0, w_kv - 1, w_kv).to( + device=device, dtype=dtype) + w_kv_idxs = w_kv_idxs.view((w_kv, 1)) * kv_stride + + # (h, h_kv, 1) + h_diff = h_idxs.unsqueeze(1) - h_kv_idxs.unsqueeze(0) + h_diff *= self.position_magnitude + + # (w, w_kv, 1) + w_diff = w_idxs.unsqueeze(1) - w_kv_idxs.unsqueeze(0) + w_diff *= self.position_magnitude + + feat_range = torch.arange(0, feat_dim / 4).to( + device=device, dtype=dtype) + + dim_mat = torch.Tensor([wave_length]).to(device=device, dtype=dtype) + dim_mat = dim_mat**((4. / feat_dim) * feat_range) + dim_mat = dim_mat.view((1, 1, -1)) + + embedding_x = torch.cat( + ((w_diff / dim_mat).sin(), (w_diff / dim_mat).cos()), dim=2) + + embedding_y = torch.cat( + ((h_diff / dim_mat).sin(), (h_diff / dim_mat).cos()), dim=2) + + return embedding_x, embedding_y + + def forward(self, x_input): + num_heads = self.num_heads + + # use empirical_attention + if self.q_downsample is not None: + x_q = self.q_downsample(x_input) + else: + x_q = x_input + n, _, h, w = x_q.shape + + if self.kv_downsample is not None: + x_kv = self.kv_downsample(x_input) + else: + x_kv = x_input + _, _, h_kv, w_kv = x_kv.shape + + if self.attention_type[0] or self.attention_type[1]: + proj_query = self.query_conv(x_q).view( + (n, num_heads, self.qk_embed_dim, h * w)) + proj_query = proj_query.permute(0, 1, 3, 2) + + if self.attention_type[0] or self.attention_type[2]: + proj_key = self.key_conv(x_kv).view( + (n, num_heads, self.qk_embed_dim, h_kv * w_kv)) + + if self.attention_type[1] or self.attention_type[3]: + position_embed_x, position_embed_y = self.get_position_embedding( + h, w, h_kv, w_kv, self.q_stride, self.kv_stride, + x_input.device, x_input.dtype, self.position_embedding_dim) + # (n, num_heads, w, w_kv, dim) + position_feat_x = self.appr_geom_fc_x(position_embed_x).\ + view(1, w, w_kv, num_heads, self.qk_embed_dim).\ + permute(0, 3, 1, 2, 4).\ + repeat(n, 1, 1, 1, 1) + + # (n, num_heads, h, h_kv, dim) + position_feat_y = self.appr_geom_fc_y(position_embed_y).\ + view(1, h, h_kv, num_heads, self.qk_embed_dim).\ + permute(0, 3, 1, 2, 4).\ + repeat(n, 1, 1, 1, 1) + + position_feat_x /= math.sqrt(2) + position_feat_y /= math.sqrt(2) + + # accelerate for saliency only + if (np.sum(self.attention_type) == 1) and self.attention_type[2]: + appr_bias = self.appr_bias.\ + view(1, num_heads, 1, self.qk_embed_dim).\ + repeat(n, 1, 1, 1) + + energy = torch.matmul(appr_bias, proj_key).\ + view(n, num_heads, 1, h_kv * w_kv) + + h = 1 + w = 1 + else: + # (n, num_heads, h*w, h_kv*w_kv), query before key, 540mb for + if not self.attention_type[0]: + energy = torch.zeros( + n, + num_heads, + h, + w, + h_kv, + w_kv, + dtype=x_input.dtype, + device=x_input.device) + + # attention_type[0]: appr - appr + # attention_type[1]: appr - position + # attention_type[2]: bias - appr + # attention_type[3]: bias - position + if self.attention_type[0] or self.attention_type[2]: + if self.attention_type[0] and self.attention_type[2]: + appr_bias = self.appr_bias.\ + view(1, num_heads, 1, self.qk_embed_dim) + energy = torch.matmul(proj_query + appr_bias, proj_key).\ + view(n, num_heads, h, w, h_kv, w_kv) + + elif self.attention_type[0]: + energy = torch.matmul(proj_query, proj_key).\ + view(n, num_heads, h, w, h_kv, w_kv) + + elif self.attention_type[2]: + appr_bias = self.appr_bias.\ + view(1, num_heads, 1, self.qk_embed_dim).\ + repeat(n, 1, 1, 1) + + energy += torch.matmul(appr_bias, proj_key).\ + view(n, num_heads, 1, 1, h_kv, w_kv) + + if self.attention_type[1] or self.attention_type[3]: + if self.attention_type[1] and self.attention_type[3]: + geom_bias = self.geom_bias.\ + view(1, num_heads, 1, self.qk_embed_dim) + + proj_query_reshape = (proj_query + geom_bias).\ + view(n, num_heads, h, w, self.qk_embed_dim) + + energy_x = torch.matmul( + proj_query_reshape.permute(0, 1, 3, 2, 4), + position_feat_x.permute(0, 1, 2, 4, 3)) + energy_x = energy_x.\ + permute(0, 1, 3, 2, 4).unsqueeze(4) + + energy_y = torch.matmul( + proj_query_reshape, + position_feat_y.permute(0, 1, 2, 4, 3)) + energy_y = energy_y.unsqueeze(5) + + energy += energy_x + energy_y + + elif self.attention_type[1]: + proj_query_reshape = proj_query.\ + view(n, num_heads, h, w, self.qk_embed_dim) + proj_query_reshape = proj_query_reshape.\ + permute(0, 1, 3, 2, 4) + position_feat_x_reshape = position_feat_x.\ + permute(0, 1, 2, 4, 3) + position_feat_y_reshape = position_feat_y.\ + permute(0, 1, 2, 4, 3) + + energy_x = torch.matmul(proj_query_reshape, + position_feat_x_reshape) + energy_x = energy_x.permute(0, 1, 3, 2, 4).unsqueeze(4) + + energy_y = torch.matmul(proj_query_reshape, + position_feat_y_reshape) + energy_y = energy_y.unsqueeze(5) + + energy += energy_x + energy_y + + elif self.attention_type[3]: + geom_bias = self.geom_bias.\ + view(1, num_heads, self.qk_embed_dim, 1).\ + repeat(n, 1, 1, 1) + + position_feat_x_reshape = position_feat_x.\ + view(n, num_heads, w*w_kv, self.qk_embed_dim) + + position_feat_y_reshape = position_feat_y.\ + view(n, num_heads, h * h_kv, self.qk_embed_dim) + + energy_x = torch.matmul(position_feat_x_reshape, geom_bias) + energy_x = energy_x.view(n, num_heads, 1, w, 1, w_kv) + + energy_y = torch.matmul(position_feat_y_reshape, geom_bias) + energy_y = energy_y.view(n, num_heads, h, 1, h_kv, 1) + + energy += energy_x + energy_y + + energy = energy.view(n, num_heads, h * w, h_kv * w_kv) + + if self.spatial_range >= 0: + cur_local_constraint_map = \ + self.local_constraint_map[:h, :w, :h_kv, :w_kv].\ + contiguous().\ + view(1, 1, h*w, h_kv*w_kv) + + energy = energy.masked_fill_(cur_local_constraint_map, + float('-inf')) + + attention = F.softmax(energy, 3) + + proj_value = self.value_conv(x_kv) + proj_value_reshape = proj_value.\ + view((n, num_heads, self.v_dim, h_kv * w_kv)).\ + permute(0, 1, 3, 2) + + out = torch.matmul(attention, proj_value_reshape).\ + permute(0, 1, 3, 2).\ + contiguous().\ + view(n, self.v_dim * self.num_heads, h, w) + + out = self.proj_conv(out) + + # output is downsampled, upsample back to input size + if self.q_downsample is not None: + out = F.interpolate( + out, + size=x_input.shape[2:], + mode='bilinear', + align_corners=False) + + out = self.gamma * out + x_input + return out + + def init_weights(self): + for m in self.modules(): + if hasattr(m, 'kaiming_init') and m.kaiming_init: + kaiming_init( + m, + mode='fan_in', + nonlinearity='leaky_relu', + bias=0, + distribution='uniform', + a=1) diff --git a/annotator/uniformer/mmcv/cnn/bricks/hsigmoid.py b/annotator/uniformer/mmcv/cnn/bricks/hsigmoid.py new file mode 100644 index 0000000000000000000000000000000000000000..30b1a3d6580cf0360710426fbea1f05acdf07b4b --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/hsigmoid.py @@ -0,0 +1,34 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch.nn as nn + +from .registry import ACTIVATION_LAYERS + + +@ACTIVATION_LAYERS.register_module() +class HSigmoid(nn.Module): + """Hard Sigmoid Module. Apply the hard sigmoid function: + Hsigmoid(x) = min(max((x + bias) / divisor, min_value), max_value) + Default: Hsigmoid(x) = min(max((x + 1) / 2, 0), 1) + + Args: + bias (float): Bias of the input feature map. Default: 1.0. + divisor (float): Divisor of the input feature map. Default: 2.0. + min_value (float): Lower bound value. Default: 0.0. + max_value (float): Upper bound value. Default: 1.0. + + Returns: + Tensor: The output tensor. + """ + + def __init__(self, bias=1.0, divisor=2.0, min_value=0.0, max_value=1.0): + super(HSigmoid, self).__init__() + self.bias = bias + self.divisor = divisor + assert self.divisor != 0 + self.min_value = min_value + self.max_value = max_value + + def forward(self, x): + x = (x + self.bias) / self.divisor + + return x.clamp_(self.min_value, self.max_value) diff --git a/annotator/uniformer/mmcv/cnn/bricks/hswish.py b/annotator/uniformer/mmcv/cnn/bricks/hswish.py new file mode 100644 index 0000000000000000000000000000000000000000..7e0c090ff037c99ee6c5c84c4592e87beae02208 --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/hswish.py @@ -0,0 +1,29 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch.nn as nn + +from .registry import ACTIVATION_LAYERS + + +@ACTIVATION_LAYERS.register_module() +class HSwish(nn.Module): + """Hard Swish Module. + + This module applies the hard swish function: + + .. math:: + Hswish(x) = x * ReLU6(x + 3) / 6 + + Args: + inplace (bool): can optionally do the operation in-place. + Default: False. + + Returns: + Tensor: The output tensor. + """ + + def __init__(self, inplace=False): + super(HSwish, self).__init__() + self.act = nn.ReLU6(inplace) + + def forward(self, x): + return x * self.act(x + 3) / 6 diff --git a/annotator/uniformer/mmcv/cnn/bricks/non_local.py b/annotator/uniformer/mmcv/cnn/bricks/non_local.py new file mode 100644 index 0000000000000000000000000000000000000000..92d00155ef275c1201ea66bba30470a1785cc5d7 --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/non_local.py @@ -0,0 +1,306 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from abc import ABCMeta + +import torch +import torch.nn as nn + +from ..utils import constant_init, normal_init +from .conv_module import ConvModule +from .registry import PLUGIN_LAYERS + + +class _NonLocalNd(nn.Module, metaclass=ABCMeta): + """Basic Non-local module. + + This module is proposed in + "Non-local Neural Networks" + Paper reference: https://arxiv.org/abs/1711.07971 + Code reference: https://github.com/AlexHex7/Non-local_pytorch + + Args: + in_channels (int): Channels of the input feature map. + reduction (int): Channel reduction ratio. Default: 2. + use_scale (bool): Whether to scale pairwise_weight by + `1/sqrt(inter_channels)` when the mode is `embedded_gaussian`. + Default: True. + conv_cfg (None | dict): The config dict for convolution layers. + If not specified, it will use `nn.Conv2d` for convolution layers. + Default: None. + norm_cfg (None | dict): The config dict for normalization layers. + Default: None. (This parameter is only applicable to conv_out.) + mode (str): Options are `gaussian`, `concatenation`, + `embedded_gaussian` and `dot_product`. Default: embedded_gaussian. + """ + + def __init__(self, + in_channels, + reduction=2, + use_scale=True, + conv_cfg=None, + norm_cfg=None, + mode='embedded_gaussian', + **kwargs): + super(_NonLocalNd, self).__init__() + self.in_channels = in_channels + self.reduction = reduction + self.use_scale = use_scale + self.inter_channels = max(in_channels // reduction, 1) + self.mode = mode + + if mode not in [ + 'gaussian', 'embedded_gaussian', 'dot_product', 'concatenation' + ]: + raise ValueError("Mode should be in 'gaussian', 'concatenation', " + f"'embedded_gaussian' or 'dot_product', but got " + f'{mode} instead.') + + # g, theta, phi are defaulted as `nn.ConvNd`. + # Here we use ConvModule for potential usage. + self.g = ConvModule( + self.in_channels, + self.inter_channels, + kernel_size=1, + conv_cfg=conv_cfg, + act_cfg=None) + self.conv_out = ConvModule( + self.inter_channels, + self.in_channels, + kernel_size=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=None) + + if self.mode != 'gaussian': + self.theta = ConvModule( + self.in_channels, + self.inter_channels, + kernel_size=1, + conv_cfg=conv_cfg, + act_cfg=None) + self.phi = ConvModule( + self.in_channels, + self.inter_channels, + kernel_size=1, + conv_cfg=conv_cfg, + act_cfg=None) + + if self.mode == 'concatenation': + self.concat_project = ConvModule( + self.inter_channels * 2, + 1, + kernel_size=1, + stride=1, + padding=0, + bias=False, + act_cfg=dict(type='ReLU')) + + self.init_weights(**kwargs) + + def init_weights(self, std=0.01, zeros_init=True): + if self.mode != 'gaussian': + for m in [self.g, self.theta, self.phi]: + normal_init(m.conv, std=std) + else: + normal_init(self.g.conv, std=std) + if zeros_init: + if self.conv_out.norm_cfg is None: + constant_init(self.conv_out.conv, 0) + else: + constant_init(self.conv_out.norm, 0) + else: + if self.conv_out.norm_cfg is None: + normal_init(self.conv_out.conv, std=std) + else: + normal_init(self.conv_out.norm, std=std) + + def gaussian(self, theta_x, phi_x): + # NonLocal1d pairwise_weight: [N, H, H] + # NonLocal2d pairwise_weight: [N, HxW, HxW] + # NonLocal3d pairwise_weight: [N, TxHxW, TxHxW] + pairwise_weight = torch.matmul(theta_x, phi_x) + pairwise_weight = pairwise_weight.softmax(dim=-1) + return pairwise_weight + + def embedded_gaussian(self, theta_x, phi_x): + # NonLocal1d pairwise_weight: [N, H, H] + # NonLocal2d pairwise_weight: [N, HxW, HxW] + # NonLocal3d pairwise_weight: [N, TxHxW, TxHxW] + pairwise_weight = torch.matmul(theta_x, phi_x) + if self.use_scale: + # theta_x.shape[-1] is `self.inter_channels` + pairwise_weight /= theta_x.shape[-1]**0.5 + pairwise_weight = pairwise_weight.softmax(dim=-1) + return pairwise_weight + + def dot_product(self, theta_x, phi_x): + # NonLocal1d pairwise_weight: [N, H, H] + # NonLocal2d pairwise_weight: [N, HxW, HxW] + # NonLocal3d pairwise_weight: [N, TxHxW, TxHxW] + pairwise_weight = torch.matmul(theta_x, phi_x) + pairwise_weight /= pairwise_weight.shape[-1] + return pairwise_weight + + def concatenation(self, theta_x, phi_x): + # NonLocal1d pairwise_weight: [N, H, H] + # NonLocal2d pairwise_weight: [N, HxW, HxW] + # NonLocal3d pairwise_weight: [N, TxHxW, TxHxW] + h = theta_x.size(2) + w = phi_x.size(3) + theta_x = theta_x.repeat(1, 1, 1, w) + phi_x = phi_x.repeat(1, 1, h, 1) + + concat_feature = torch.cat([theta_x, phi_x], dim=1) + pairwise_weight = self.concat_project(concat_feature) + n, _, h, w = pairwise_weight.size() + pairwise_weight = pairwise_weight.view(n, h, w) + pairwise_weight /= pairwise_weight.shape[-1] + + return pairwise_weight + + def forward(self, x): + # Assume `reduction = 1`, then `inter_channels = C` + # or `inter_channels = C` when `mode="gaussian"` + + # NonLocal1d x: [N, C, H] + # NonLocal2d x: [N, C, H, W] + # NonLocal3d x: [N, C, T, H, W] + n = x.size(0) + + # NonLocal1d g_x: [N, H, C] + # NonLocal2d g_x: [N, HxW, C] + # NonLocal3d g_x: [N, TxHxW, C] + g_x = self.g(x).view(n, self.inter_channels, -1) + g_x = g_x.permute(0, 2, 1) + + # NonLocal1d theta_x: [N, H, C], phi_x: [N, C, H] + # NonLocal2d theta_x: [N, HxW, C], phi_x: [N, C, HxW] + # NonLocal3d theta_x: [N, TxHxW, C], phi_x: [N, C, TxHxW] + if self.mode == 'gaussian': + theta_x = x.view(n, self.in_channels, -1) + theta_x = theta_x.permute(0, 2, 1) + if self.sub_sample: + phi_x = self.phi(x).view(n, self.in_channels, -1) + else: + phi_x = x.view(n, self.in_channels, -1) + elif self.mode == 'concatenation': + theta_x = self.theta(x).view(n, self.inter_channels, -1, 1) + phi_x = self.phi(x).view(n, self.inter_channels, 1, -1) + else: + theta_x = self.theta(x).view(n, self.inter_channels, -1) + theta_x = theta_x.permute(0, 2, 1) + phi_x = self.phi(x).view(n, self.inter_channels, -1) + + pairwise_func = getattr(self, self.mode) + # NonLocal1d pairwise_weight: [N, H, H] + # NonLocal2d pairwise_weight: [N, HxW, HxW] + # NonLocal3d pairwise_weight: [N, TxHxW, TxHxW] + pairwise_weight = pairwise_func(theta_x, phi_x) + + # NonLocal1d y: [N, H, C] + # NonLocal2d y: [N, HxW, C] + # NonLocal3d y: [N, TxHxW, C] + y = torch.matmul(pairwise_weight, g_x) + # NonLocal1d y: [N, C, H] + # NonLocal2d y: [N, C, H, W] + # NonLocal3d y: [N, C, T, H, W] + y = y.permute(0, 2, 1).contiguous().reshape(n, self.inter_channels, + *x.size()[2:]) + + output = x + self.conv_out(y) + + return output + + +class NonLocal1d(_NonLocalNd): + """1D Non-local module. + + Args: + in_channels (int): Same as `NonLocalND`. + sub_sample (bool): Whether to apply max pooling after pairwise + function (Note that the `sub_sample` is applied on spatial only). + Default: False. + conv_cfg (None | dict): Same as `NonLocalND`. + Default: dict(type='Conv1d'). + """ + + def __init__(self, + in_channels, + sub_sample=False, + conv_cfg=dict(type='Conv1d'), + **kwargs): + super(NonLocal1d, self).__init__( + in_channels, conv_cfg=conv_cfg, **kwargs) + + self.sub_sample = sub_sample + + if sub_sample: + max_pool_layer = nn.MaxPool1d(kernel_size=2) + self.g = nn.Sequential(self.g, max_pool_layer) + if self.mode != 'gaussian': + self.phi = nn.Sequential(self.phi, max_pool_layer) + else: + self.phi = max_pool_layer + + +@PLUGIN_LAYERS.register_module() +class NonLocal2d(_NonLocalNd): + """2D Non-local module. + + Args: + in_channels (int): Same as `NonLocalND`. + sub_sample (bool): Whether to apply max pooling after pairwise + function (Note that the `sub_sample` is applied on spatial only). + Default: False. + conv_cfg (None | dict): Same as `NonLocalND`. + Default: dict(type='Conv2d'). + """ + + _abbr_ = 'nonlocal_block' + + def __init__(self, + in_channels, + sub_sample=False, + conv_cfg=dict(type='Conv2d'), + **kwargs): + super(NonLocal2d, self).__init__( + in_channels, conv_cfg=conv_cfg, **kwargs) + + self.sub_sample = sub_sample + + if sub_sample: + max_pool_layer = nn.MaxPool2d(kernel_size=(2, 2)) + self.g = nn.Sequential(self.g, max_pool_layer) + if self.mode != 'gaussian': + self.phi = nn.Sequential(self.phi, max_pool_layer) + else: + self.phi = max_pool_layer + + +class NonLocal3d(_NonLocalNd): + """3D Non-local module. + + Args: + in_channels (int): Same as `NonLocalND`. + sub_sample (bool): Whether to apply max pooling after pairwise + function (Note that the `sub_sample` is applied on spatial only). + Default: False. + conv_cfg (None | dict): Same as `NonLocalND`. + Default: dict(type='Conv3d'). + """ + + def __init__(self, + in_channels, + sub_sample=False, + conv_cfg=dict(type='Conv3d'), + **kwargs): + super(NonLocal3d, self).__init__( + in_channels, conv_cfg=conv_cfg, **kwargs) + self.sub_sample = sub_sample + + if sub_sample: + max_pool_layer = nn.MaxPool3d(kernel_size=(1, 2, 2)) + self.g = nn.Sequential(self.g, max_pool_layer) + if self.mode != 'gaussian': + self.phi = nn.Sequential(self.phi, max_pool_layer) + else: + self.phi = max_pool_layer diff --git a/annotator/uniformer/mmcv/cnn/bricks/norm.py b/annotator/uniformer/mmcv/cnn/bricks/norm.py new file mode 100644 index 0000000000000000000000000000000000000000..408f4b42731b19a3beeef68b6a5e610d0bbc18b3 --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/norm.py @@ -0,0 +1,144 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import inspect + +import torch.nn as nn + +from annotator.uniformer.mmcv.utils import is_tuple_of +from annotator.uniformer.mmcv.utils.parrots_wrapper import SyncBatchNorm, _BatchNorm, _InstanceNorm +from .registry import NORM_LAYERS + +NORM_LAYERS.register_module('BN', module=nn.BatchNorm2d) +NORM_LAYERS.register_module('BN1d', module=nn.BatchNorm1d) +NORM_LAYERS.register_module('BN2d', module=nn.BatchNorm2d) +NORM_LAYERS.register_module('BN3d', module=nn.BatchNorm3d) +NORM_LAYERS.register_module('SyncBN', module=SyncBatchNorm) +NORM_LAYERS.register_module('GN', module=nn.GroupNorm) +NORM_LAYERS.register_module('LN', module=nn.LayerNorm) +NORM_LAYERS.register_module('IN', module=nn.InstanceNorm2d) +NORM_LAYERS.register_module('IN1d', module=nn.InstanceNorm1d) +NORM_LAYERS.register_module('IN2d', module=nn.InstanceNorm2d) +NORM_LAYERS.register_module('IN3d', module=nn.InstanceNorm3d) + + +def infer_abbr(class_type): + """Infer abbreviation from the class name. + + When we build a norm layer with `build_norm_layer()`, we want to preserve + the norm type in variable names, e.g, self.bn1, self.gn. This method will + infer the abbreviation to map class types to abbreviations. + + Rule 1: If the class has the property "_abbr_", return the property. + Rule 2: If the parent class is _BatchNorm, GroupNorm, LayerNorm or + InstanceNorm, the abbreviation of this layer will be "bn", "gn", "ln" and + "in" respectively. + Rule 3: If the class name contains "batch", "group", "layer" or "instance", + the abbreviation of this layer will be "bn", "gn", "ln" and "in" + respectively. + Rule 4: Otherwise, the abbreviation falls back to "norm". + + Args: + class_type (type): The norm layer type. + + Returns: + str: The inferred abbreviation. + """ + if not inspect.isclass(class_type): + raise TypeError( + f'class_type must be a type, but got {type(class_type)}') + if hasattr(class_type, '_abbr_'): + return class_type._abbr_ + if issubclass(class_type, _InstanceNorm): # IN is a subclass of BN + return 'in' + elif issubclass(class_type, _BatchNorm): + return 'bn' + elif issubclass(class_type, nn.GroupNorm): + return 'gn' + elif issubclass(class_type, nn.LayerNorm): + return 'ln' + else: + class_name = class_type.__name__.lower() + if 'batch' in class_name: + return 'bn' + elif 'group' in class_name: + return 'gn' + elif 'layer' in class_name: + return 'ln' + elif 'instance' in class_name: + return 'in' + else: + return 'norm_layer' + + +def build_norm_layer(cfg, num_features, postfix=''): + """Build normalization layer. + + Args: + cfg (dict): The norm layer config, which should contain: + + - type (str): Layer type. + - layer args: Args needed to instantiate a norm layer. + - requires_grad (bool, optional): Whether stop gradient updates. + num_features (int): Number of input channels. + postfix (int | str): The postfix to be appended into norm abbreviation + to create named layer. + + Returns: + (str, nn.Module): The first element is the layer name consisting of + abbreviation and postfix, e.g., bn1, gn. The second element is the + created norm layer. + """ + if not isinstance(cfg, dict): + raise TypeError('cfg must be a dict') + if 'type' not in cfg: + raise KeyError('the cfg dict must contain the key "type"') + cfg_ = cfg.copy() + + layer_type = cfg_.pop('type') + if layer_type not in NORM_LAYERS: + raise KeyError(f'Unrecognized norm type {layer_type}') + + norm_layer = NORM_LAYERS.get(layer_type) + abbr = infer_abbr(norm_layer) + + assert isinstance(postfix, (int, str)) + name = abbr + str(postfix) + + requires_grad = cfg_.pop('requires_grad', True) + cfg_.setdefault('eps', 1e-5) + if layer_type != 'GN': + layer = norm_layer(num_features, **cfg_) + if layer_type == 'SyncBN' and hasattr(layer, '_specify_ddp_gpu_num'): + layer._specify_ddp_gpu_num(1) + else: + assert 'num_groups' in cfg_ + layer = norm_layer(num_channels=num_features, **cfg_) + + for param in layer.parameters(): + param.requires_grad = requires_grad + + return name, layer + + +def is_norm(layer, exclude=None): + """Check if a layer is a normalization layer. + + Args: + layer (nn.Module): The layer to be checked. + exclude (type | tuple[type]): Types to be excluded. + + Returns: + bool: Whether the layer is a norm layer. + """ + if exclude is not None: + if not isinstance(exclude, tuple): + exclude = (exclude, ) + if not is_tuple_of(exclude, type): + raise TypeError( + f'"exclude" must be either None or type or a tuple of types, ' + f'but got {type(exclude)}: {exclude}') + + if exclude and isinstance(layer, exclude): + return False + + all_norm_bases = (_BatchNorm, _InstanceNorm, nn.GroupNorm, nn.LayerNorm) + return isinstance(layer, all_norm_bases) diff --git a/annotator/uniformer/mmcv/cnn/bricks/padding.py b/annotator/uniformer/mmcv/cnn/bricks/padding.py new file mode 100644 index 0000000000000000000000000000000000000000..e4ac6b28a1789bd551c613a7d3e7b622433ac7ec --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/padding.py @@ -0,0 +1,36 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch.nn as nn + +from .registry import PADDING_LAYERS + +PADDING_LAYERS.register_module('zero', module=nn.ZeroPad2d) +PADDING_LAYERS.register_module('reflect', module=nn.ReflectionPad2d) +PADDING_LAYERS.register_module('replicate', module=nn.ReplicationPad2d) + + +def build_padding_layer(cfg, *args, **kwargs): + """Build padding layer. + + Args: + cfg (None or dict): The padding layer config, which should contain: + - type (str): Layer type. + - layer args: Args needed to instantiate a padding layer. + + Returns: + nn.Module: Created padding layer. + """ + if not isinstance(cfg, dict): + raise TypeError('cfg must be a dict') + if 'type' not in cfg: + raise KeyError('the cfg dict must contain the key "type"') + + cfg_ = cfg.copy() + padding_type = cfg_.pop('type') + if padding_type not in PADDING_LAYERS: + raise KeyError(f'Unrecognized padding type {padding_type}.') + else: + padding_layer = PADDING_LAYERS.get(padding_type) + + layer = padding_layer(*args, **kwargs, **cfg_) + + return layer diff --git a/annotator/uniformer/mmcv/cnn/bricks/plugin.py b/annotator/uniformer/mmcv/cnn/bricks/plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..07c010d4053174dd41107aa654ea67e82b46a25c --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/plugin.py @@ -0,0 +1,88 @@ +import inspect +import platform + +from .registry import PLUGIN_LAYERS + +if platform.system() == 'Windows': + import regex as re +else: + import re + + +def infer_abbr(class_type): + """Infer abbreviation from the class name. + + This method will infer the abbreviation to map class types to + abbreviations. + + Rule 1: If the class has the property "abbr", return the property. + Rule 2: Otherwise, the abbreviation falls back to snake case of class + name, e.g. the abbreviation of ``FancyBlock`` will be ``fancy_block``. + + Args: + class_type (type): The norm layer type. + + Returns: + str: The inferred abbreviation. + """ + + def camel2snack(word): + """Convert camel case word into snack case. + + Modified from `inflection lib + `_. + + Example:: + + >>> camel2snack("FancyBlock") + 'fancy_block' + """ + + word = re.sub(r'([A-Z]+)([A-Z][a-z])', r'\1_\2', word) + word = re.sub(r'([a-z\d])([A-Z])', r'\1_\2', word) + word = word.replace('-', '_') + return word.lower() + + if not inspect.isclass(class_type): + raise TypeError( + f'class_type must be a type, but got {type(class_type)}') + if hasattr(class_type, '_abbr_'): + return class_type._abbr_ + else: + return camel2snack(class_type.__name__) + + +def build_plugin_layer(cfg, postfix='', **kwargs): + """Build plugin layer. + + Args: + cfg (None or dict): cfg should contain: + type (str): identify plugin layer type. + layer args: args needed to instantiate a plugin layer. + postfix (int, str): appended into norm abbreviation to + create named layer. Default: ''. + + Returns: + tuple[str, nn.Module]: + name (str): abbreviation + postfix + layer (nn.Module): created plugin layer + """ + if not isinstance(cfg, dict): + raise TypeError('cfg must be a dict') + if 'type' not in cfg: + raise KeyError('the cfg dict must contain the key "type"') + cfg_ = cfg.copy() + + layer_type = cfg_.pop('type') + if layer_type not in PLUGIN_LAYERS: + raise KeyError(f'Unrecognized plugin type {layer_type}') + + plugin_layer = PLUGIN_LAYERS.get(layer_type) + abbr = infer_abbr(plugin_layer) + + assert isinstance(postfix, (int, str)) + name = abbr + str(postfix) + + layer = plugin_layer(**kwargs, **cfg_) + + return name, layer diff --git a/annotator/uniformer/mmcv/cnn/bricks/registry.py b/annotator/uniformer/mmcv/cnn/bricks/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..39eabc58db4b5954478a2ac1ab91cea5e45ab055 --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/registry.py @@ -0,0 +1,16 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from annotator.uniformer.mmcv.utils import Registry + +CONV_LAYERS = Registry('conv layer') +NORM_LAYERS = Registry('norm layer') +ACTIVATION_LAYERS = Registry('activation layer') +PADDING_LAYERS = Registry('padding layer') +UPSAMPLE_LAYERS = Registry('upsample layer') +PLUGIN_LAYERS = Registry('plugin layer') + +DROPOUT_LAYERS = Registry('drop out layers') +POSITIONAL_ENCODING = Registry('position encoding') +ATTENTION = Registry('attention') +FEEDFORWARD_NETWORK = Registry('feed-forward Network') +TRANSFORMER_LAYER = Registry('transformerLayer') +TRANSFORMER_LAYER_SEQUENCE = Registry('transformer-layers sequence') diff --git a/annotator/uniformer/mmcv/cnn/bricks/scale.py b/annotator/uniformer/mmcv/cnn/bricks/scale.py new file mode 100644 index 0000000000000000000000000000000000000000..c905fffcc8bf998d18d94f927591963c428025e2 --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/scale.py @@ -0,0 +1,21 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn + + +class Scale(nn.Module): + """A learnable scale parameter. + + This layer scales the input by a learnable factor. It multiplies a + learnable scale parameter of shape (1,) with input of any shape. + + Args: + scale (float): Initial value of scale factor. Default: 1.0 + """ + + def __init__(self, scale=1.0): + super(Scale, self).__init__() + self.scale = nn.Parameter(torch.tensor(scale, dtype=torch.float)) + + def forward(self, x): + return x * self.scale diff --git a/annotator/uniformer/mmcv/cnn/bricks/swish.py b/annotator/uniformer/mmcv/cnn/bricks/swish.py new file mode 100644 index 0000000000000000000000000000000000000000..e2ca8ed7b749413f011ae54aac0cab27e6f0b51f --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/swish.py @@ -0,0 +1,25 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn + +from .registry import ACTIVATION_LAYERS + + +@ACTIVATION_LAYERS.register_module() +class Swish(nn.Module): + """Swish Module. + + This module applies the swish function: + + .. math:: + Swish(x) = x * Sigmoid(x) + + Returns: + Tensor: The output tensor. + """ + + def __init__(self): + super(Swish, self).__init__() + + def forward(self, x): + return x * torch.sigmoid(x) diff --git a/annotator/uniformer/mmcv/cnn/bricks/transformer.py b/annotator/uniformer/mmcv/cnn/bricks/transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..e61ae0dd941a7be00b3e41a3de833ec50470a45f --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/transformer.py @@ -0,0 +1,595 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy +import warnings + +import torch +import torch.nn as nn + +from annotator.uniformer.mmcv import ConfigDict, deprecated_api_warning +from annotator.uniformer.mmcv.cnn import Linear, build_activation_layer, build_norm_layer +from annotator.uniformer.mmcv.runner.base_module import BaseModule, ModuleList, Sequential +from annotator.uniformer.mmcv.utils import build_from_cfg +from .drop import build_dropout +from .registry import (ATTENTION, FEEDFORWARD_NETWORK, POSITIONAL_ENCODING, + TRANSFORMER_LAYER, TRANSFORMER_LAYER_SEQUENCE) + +# Avoid BC-breaking of importing MultiScaleDeformableAttention from this file +try: + from annotator.uniformer.mmcv.ops.multi_scale_deform_attn import MultiScaleDeformableAttention # noqa F401 + warnings.warn( + ImportWarning( + '``MultiScaleDeformableAttention`` has been moved to ' + '``mmcv.ops.multi_scale_deform_attn``, please change original path ' # noqa E501 + '``from annotator.uniformer.mmcv.cnn.bricks.transformer import MultiScaleDeformableAttention`` ' # noqa E501 + 'to ``from annotator.uniformer.mmcv.ops.multi_scale_deform_attn import MultiScaleDeformableAttention`` ' # noqa E501 + )) + +except ImportError: + warnings.warn('Fail to import ``MultiScaleDeformableAttention`` from ' + '``mmcv.ops.multi_scale_deform_attn``, ' + 'You should install ``mmcv-full`` if you need this module. ') + + +def build_positional_encoding(cfg, default_args=None): + """Builder for Position Encoding.""" + return build_from_cfg(cfg, POSITIONAL_ENCODING, default_args) + + +def build_attention(cfg, default_args=None): + """Builder for attention.""" + return build_from_cfg(cfg, ATTENTION, default_args) + + +def build_feedforward_network(cfg, default_args=None): + """Builder for feed-forward network (FFN).""" + return build_from_cfg(cfg, FEEDFORWARD_NETWORK, default_args) + + +def build_transformer_layer(cfg, default_args=None): + """Builder for transformer layer.""" + return build_from_cfg(cfg, TRANSFORMER_LAYER, default_args) + + +def build_transformer_layer_sequence(cfg, default_args=None): + """Builder for transformer encoder and transformer decoder.""" + return build_from_cfg(cfg, TRANSFORMER_LAYER_SEQUENCE, default_args) + + +@ATTENTION.register_module() +class MultiheadAttention(BaseModule): + """A wrapper for ``torch.nn.MultiheadAttention``. + + This module implements MultiheadAttention with identity connection, + and positional encoding is also passed as input. + + Args: + embed_dims (int): The embedding dimension. + num_heads (int): Parallel attention heads. + attn_drop (float): A Dropout layer on attn_output_weights. + Default: 0.0. + proj_drop (float): A Dropout layer after `nn.MultiheadAttention`. + Default: 0.0. + dropout_layer (obj:`ConfigDict`): The dropout_layer used + when adding the shortcut. + init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. + Default: None. + batch_first (bool): When it is True, Key, Query and Value are shape of + (batch, n, embed_dim), otherwise (n, batch, embed_dim). + Default to False. + """ + + def __init__(self, + embed_dims, + num_heads, + attn_drop=0., + proj_drop=0., + dropout_layer=dict(type='Dropout', drop_prob=0.), + init_cfg=None, + batch_first=False, + **kwargs): + super(MultiheadAttention, self).__init__(init_cfg) + if 'dropout' in kwargs: + warnings.warn('The arguments `dropout` in MultiheadAttention ' + 'has been deprecated, now you can separately ' + 'set `attn_drop`(float), proj_drop(float), ' + 'and `dropout_layer`(dict) ') + attn_drop = kwargs['dropout'] + dropout_layer['drop_prob'] = kwargs.pop('dropout') + + self.embed_dims = embed_dims + self.num_heads = num_heads + self.batch_first = batch_first + + self.attn = nn.MultiheadAttention(embed_dims, num_heads, attn_drop, + **kwargs) + + self.proj_drop = nn.Dropout(proj_drop) + self.dropout_layer = build_dropout( + dropout_layer) if dropout_layer else nn.Identity() + + @deprecated_api_warning({'residual': 'identity'}, + cls_name='MultiheadAttention') + def forward(self, + query, + key=None, + value=None, + identity=None, + query_pos=None, + key_pos=None, + attn_mask=None, + key_padding_mask=None, + **kwargs): + """Forward function for `MultiheadAttention`. + + **kwargs allow passing a more general data flow when combining + with other operations in `transformerlayer`. + + Args: + query (Tensor): The input query with shape [num_queries, bs, + embed_dims] if self.batch_first is False, else + [bs, num_queries embed_dims]. + key (Tensor): The key tensor with shape [num_keys, bs, + embed_dims] if self.batch_first is False, else + [bs, num_keys, embed_dims] . + If None, the ``query`` will be used. Defaults to None. + value (Tensor): The value tensor with same shape as `key`. + Same in `nn.MultiheadAttention.forward`. Defaults to None. + If None, the `key` will be used. + identity (Tensor): This tensor, with the same shape as x, + will be used for the identity link. + If None, `x` will be used. Defaults to None. + query_pos (Tensor): The positional encoding for query, with + the same shape as `x`. If not None, it will + be added to `x` before forward function. Defaults to None. + key_pos (Tensor): The positional encoding for `key`, with the + same shape as `key`. Defaults to None. If not None, it will + be added to `key` before forward function. If None, and + `query_pos` has the same shape as `key`, then `query_pos` + will be used for `key_pos`. Defaults to None. + attn_mask (Tensor): ByteTensor mask with shape [num_queries, + num_keys]. Same in `nn.MultiheadAttention.forward`. + Defaults to None. + key_padding_mask (Tensor): ByteTensor with shape [bs, num_keys]. + Defaults to None. + + Returns: + Tensor: forwarded results with shape + [num_queries, bs, embed_dims] + if self.batch_first is False, else + [bs, num_queries embed_dims]. + """ + + if key is None: + key = query + if value is None: + value = key + if identity is None: + identity = query + if key_pos is None: + if query_pos is not None: + # use query_pos if key_pos is not available + if query_pos.shape == key.shape: + key_pos = query_pos + else: + warnings.warn(f'position encoding of key is' + f'missing in {self.__class__.__name__}.') + if query_pos is not None: + query = query + query_pos + if key_pos is not None: + key = key + key_pos + + # Because the dataflow('key', 'query', 'value') of + # ``torch.nn.MultiheadAttention`` is (num_query, batch, + # embed_dims), We should adjust the shape of dataflow from + # batch_first (batch, num_query, embed_dims) to num_query_first + # (num_query ,batch, embed_dims), and recover ``attn_output`` + # from num_query_first to batch_first. + if self.batch_first: + query = query.transpose(0, 1) + key = key.transpose(0, 1) + value = value.transpose(0, 1) + + out = self.attn( + query=query, + key=key, + value=value, + attn_mask=attn_mask, + key_padding_mask=key_padding_mask)[0] + + if self.batch_first: + out = out.transpose(0, 1) + + return identity + self.dropout_layer(self.proj_drop(out)) + + +@FEEDFORWARD_NETWORK.register_module() +class FFN(BaseModule): + """Implements feed-forward networks (FFNs) with identity connection. + + Args: + embed_dims (int): The feature dimension. Same as + `MultiheadAttention`. Defaults: 256. + feedforward_channels (int): The hidden dimension of FFNs. + Defaults: 1024. + num_fcs (int, optional): The number of fully-connected layers in + FFNs. Default: 2. + act_cfg (dict, optional): The activation config for FFNs. + Default: dict(type='ReLU') + ffn_drop (float, optional): Probability of an element to be + zeroed in FFN. Default 0.0. + add_identity (bool, optional): Whether to add the + identity connection. Default: `True`. + dropout_layer (obj:`ConfigDict`): The dropout_layer used + when adding the shortcut. + init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. + Default: None. + """ + + @deprecated_api_warning( + { + 'dropout': 'ffn_drop', + 'add_residual': 'add_identity' + }, + cls_name='FFN') + def __init__(self, + embed_dims=256, + feedforward_channels=1024, + num_fcs=2, + act_cfg=dict(type='ReLU', inplace=True), + ffn_drop=0., + dropout_layer=None, + add_identity=True, + init_cfg=None, + **kwargs): + super(FFN, self).__init__(init_cfg) + assert num_fcs >= 2, 'num_fcs should be no less ' \ + f'than 2. got {num_fcs}.' + self.embed_dims = embed_dims + self.feedforward_channels = feedforward_channels + self.num_fcs = num_fcs + self.act_cfg = act_cfg + self.activate = build_activation_layer(act_cfg) + + layers = [] + in_channels = embed_dims + for _ in range(num_fcs - 1): + layers.append( + Sequential( + Linear(in_channels, feedforward_channels), self.activate, + nn.Dropout(ffn_drop))) + in_channels = feedforward_channels + layers.append(Linear(feedforward_channels, embed_dims)) + layers.append(nn.Dropout(ffn_drop)) + self.layers = Sequential(*layers) + self.dropout_layer = build_dropout( + dropout_layer) if dropout_layer else torch.nn.Identity() + self.add_identity = add_identity + + @deprecated_api_warning({'residual': 'identity'}, cls_name='FFN') + def forward(self, x, identity=None): + """Forward function for `FFN`. + + The function would add x to the output tensor if residue is None. + """ + out = self.layers(x) + if not self.add_identity: + return self.dropout_layer(out) + if identity is None: + identity = x + return identity + self.dropout_layer(out) + + +@TRANSFORMER_LAYER.register_module() +class BaseTransformerLayer(BaseModule): + """Base `TransformerLayer` for vision transformer. + + It can be built from `mmcv.ConfigDict` and support more flexible + customization, for example, using any number of `FFN or LN ` and + use different kinds of `attention` by specifying a list of `ConfigDict` + named `attn_cfgs`. It is worth mentioning that it supports `prenorm` + when you specifying `norm` as the first element of `operation_order`. + More details about the `prenorm`: `On Layer Normalization in the + Transformer Architecture `_ . + + Args: + attn_cfgs (list[`mmcv.ConfigDict`] | obj:`mmcv.ConfigDict` | None )): + Configs for `self_attention` or `cross_attention` modules, + The order of the configs in the list should be consistent with + corresponding attentions in operation_order. + If it is a dict, all of the attention modules in operation_order + will be built with this config. Default: None. + ffn_cfgs (list[`mmcv.ConfigDict`] | obj:`mmcv.ConfigDict` | None )): + Configs for FFN, The order of the configs in the list should be + consistent with corresponding ffn in operation_order. + If it is a dict, all of the attention modules in operation_order + will be built with this config. + operation_order (tuple[str]): The execution order of operation + in transformer. Such as ('self_attn', 'norm', 'ffn', 'norm'). + Support `prenorm` when you specifying first element as `norm`. + Default:None. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='LN'). + init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. + Default: None. + batch_first (bool): Key, Query and Value are shape + of (batch, n, embed_dim) + or (n, batch, embed_dim). Default to False. + """ + + def __init__(self, + attn_cfgs=None, + ffn_cfgs=dict( + type='FFN', + embed_dims=256, + feedforward_channels=1024, + num_fcs=2, + ffn_drop=0., + act_cfg=dict(type='ReLU', inplace=True), + ), + operation_order=None, + norm_cfg=dict(type='LN'), + init_cfg=None, + batch_first=False, + **kwargs): + + deprecated_args = dict( + feedforward_channels='feedforward_channels', + ffn_dropout='ffn_drop', + ffn_num_fcs='num_fcs') + for ori_name, new_name in deprecated_args.items(): + if ori_name in kwargs: + warnings.warn( + f'The arguments `{ori_name}` in BaseTransformerLayer ' + f'has been deprecated, now you should set `{new_name}` ' + f'and other FFN related arguments ' + f'to a dict named `ffn_cfgs`. ') + ffn_cfgs[new_name] = kwargs[ori_name] + + super(BaseTransformerLayer, self).__init__(init_cfg) + + self.batch_first = batch_first + + assert set(operation_order) & set( + ['self_attn', 'norm', 'ffn', 'cross_attn']) == \ + set(operation_order), f'The operation_order of' \ + f' {self.__class__.__name__} should ' \ + f'contains all four operation type ' \ + f"{['self_attn', 'norm', 'ffn', 'cross_attn']}" + + num_attn = operation_order.count('self_attn') + operation_order.count( + 'cross_attn') + if isinstance(attn_cfgs, dict): + attn_cfgs = [copy.deepcopy(attn_cfgs) for _ in range(num_attn)] + else: + assert num_attn == len(attn_cfgs), f'The length ' \ + f'of attn_cfg {num_attn} is ' \ + f'not consistent with the number of attention' \ + f'in operation_order {operation_order}.' + + self.num_attn = num_attn + self.operation_order = operation_order + self.norm_cfg = norm_cfg + self.pre_norm = operation_order[0] == 'norm' + self.attentions = ModuleList() + + index = 0 + for operation_name in operation_order: + if operation_name in ['self_attn', 'cross_attn']: + if 'batch_first' in attn_cfgs[index]: + assert self.batch_first == attn_cfgs[index]['batch_first'] + else: + attn_cfgs[index]['batch_first'] = self.batch_first + attention = build_attention(attn_cfgs[index]) + # Some custom attentions used as `self_attn` + # or `cross_attn` can have different behavior. + attention.operation_name = operation_name + self.attentions.append(attention) + index += 1 + + self.embed_dims = self.attentions[0].embed_dims + + self.ffns = ModuleList() + num_ffns = operation_order.count('ffn') + if isinstance(ffn_cfgs, dict): + ffn_cfgs = ConfigDict(ffn_cfgs) + if isinstance(ffn_cfgs, dict): + ffn_cfgs = [copy.deepcopy(ffn_cfgs) for _ in range(num_ffns)] + assert len(ffn_cfgs) == num_ffns + for ffn_index in range(num_ffns): + if 'embed_dims' not in ffn_cfgs[ffn_index]: + ffn_cfgs['embed_dims'] = self.embed_dims + else: + assert ffn_cfgs[ffn_index]['embed_dims'] == self.embed_dims + self.ffns.append( + build_feedforward_network(ffn_cfgs[ffn_index], + dict(type='FFN'))) + + self.norms = ModuleList() + num_norms = operation_order.count('norm') + for _ in range(num_norms): + self.norms.append(build_norm_layer(norm_cfg, self.embed_dims)[1]) + + def forward(self, + query, + key=None, + value=None, + query_pos=None, + key_pos=None, + attn_masks=None, + query_key_padding_mask=None, + key_padding_mask=None, + **kwargs): + """Forward function for `TransformerDecoderLayer`. + + **kwargs contains some specific arguments of attentions. + + Args: + query (Tensor): The input query with shape + [num_queries, bs, embed_dims] if + self.batch_first is False, else + [bs, num_queries embed_dims]. + key (Tensor): The key tensor with shape [num_keys, bs, + embed_dims] if self.batch_first is False, else + [bs, num_keys, embed_dims] . + value (Tensor): The value tensor with same shape as `key`. + query_pos (Tensor): The positional encoding for `query`. + Default: None. + key_pos (Tensor): The positional encoding for `key`. + Default: None. + attn_masks (List[Tensor] | None): 2D Tensor used in + calculation of corresponding attention. The length of + it should equal to the number of `attention` in + `operation_order`. Default: None. + query_key_padding_mask (Tensor): ByteTensor for `query`, with + shape [bs, num_queries]. Only used in `self_attn` layer. + Defaults to None. + key_padding_mask (Tensor): ByteTensor for `query`, with + shape [bs, num_keys]. Default: None. + + Returns: + Tensor: forwarded results with shape [num_queries, bs, embed_dims]. + """ + + norm_index = 0 + attn_index = 0 + ffn_index = 0 + identity = query + if attn_masks is None: + attn_masks = [None for _ in range(self.num_attn)] + elif isinstance(attn_masks, torch.Tensor): + attn_masks = [ + copy.deepcopy(attn_masks) for _ in range(self.num_attn) + ] + warnings.warn(f'Use same attn_mask in all attentions in ' + f'{self.__class__.__name__} ') + else: + assert len(attn_masks) == self.num_attn, f'The length of ' \ + f'attn_masks {len(attn_masks)} must be equal ' \ + f'to the number of attention in ' \ + f'operation_order {self.num_attn}' + + for layer in self.operation_order: + if layer == 'self_attn': + temp_key = temp_value = query + query = self.attentions[attn_index]( + query, + temp_key, + temp_value, + identity if self.pre_norm else None, + query_pos=query_pos, + key_pos=query_pos, + attn_mask=attn_masks[attn_index], + key_padding_mask=query_key_padding_mask, + **kwargs) + attn_index += 1 + identity = query + + elif layer == 'norm': + query = self.norms[norm_index](query) + norm_index += 1 + + elif layer == 'cross_attn': + query = self.attentions[attn_index]( + query, + key, + value, + identity if self.pre_norm else None, + query_pos=query_pos, + key_pos=key_pos, + attn_mask=attn_masks[attn_index], + key_padding_mask=key_padding_mask, + **kwargs) + attn_index += 1 + identity = query + + elif layer == 'ffn': + query = self.ffns[ffn_index]( + query, identity if self.pre_norm else None) + ffn_index += 1 + + return query + + +@TRANSFORMER_LAYER_SEQUENCE.register_module() +class TransformerLayerSequence(BaseModule): + """Base class for TransformerEncoder and TransformerDecoder in vision + transformer. + + As base-class of Encoder and Decoder in vision transformer. + Support customization such as specifying different kind + of `transformer_layer` in `transformer_coder`. + + Args: + transformerlayer (list[obj:`mmcv.ConfigDict`] | + obj:`mmcv.ConfigDict`): Config of transformerlayer + in TransformerCoder. If it is obj:`mmcv.ConfigDict`, + it would be repeated `num_layer` times to a + list[`mmcv.ConfigDict`]. Default: None. + num_layers (int): The number of `TransformerLayer`. Default: None. + init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. + Default: None. + """ + + def __init__(self, transformerlayers=None, num_layers=None, init_cfg=None): + super(TransformerLayerSequence, self).__init__(init_cfg) + if isinstance(transformerlayers, dict): + transformerlayers = [ + copy.deepcopy(transformerlayers) for _ in range(num_layers) + ] + else: + assert isinstance(transformerlayers, list) and \ + len(transformerlayers) == num_layers + self.num_layers = num_layers + self.layers = ModuleList() + for i in range(num_layers): + self.layers.append(build_transformer_layer(transformerlayers[i])) + self.embed_dims = self.layers[0].embed_dims + self.pre_norm = self.layers[0].pre_norm + + def forward(self, + query, + key, + value, + query_pos=None, + key_pos=None, + attn_masks=None, + query_key_padding_mask=None, + key_padding_mask=None, + **kwargs): + """Forward function for `TransformerCoder`. + + Args: + query (Tensor): Input query with shape + `(num_queries, bs, embed_dims)`. + key (Tensor): The key tensor with shape + `(num_keys, bs, embed_dims)`. + value (Tensor): The value tensor with shape + `(num_keys, bs, embed_dims)`. + query_pos (Tensor): The positional encoding for `query`. + Default: None. + key_pos (Tensor): The positional encoding for `key`. + Default: None. + attn_masks (List[Tensor], optional): Each element is 2D Tensor + which is used in calculation of corresponding attention in + operation_order. Default: None. + query_key_padding_mask (Tensor): ByteTensor for `query`, with + shape [bs, num_queries]. Only used in self-attention + Default: None. + key_padding_mask (Tensor): ByteTensor for `query`, with + shape [bs, num_keys]. Default: None. + + Returns: + Tensor: results with shape [num_queries, bs, embed_dims]. + """ + for layer in self.layers: + query = layer( + query, + key, + value, + query_pos=query_pos, + key_pos=key_pos, + attn_masks=attn_masks, + query_key_padding_mask=query_key_padding_mask, + key_padding_mask=key_padding_mask, + **kwargs) + return query diff --git a/annotator/uniformer/mmcv/cnn/bricks/upsample.py b/annotator/uniformer/mmcv/cnn/bricks/upsample.py new file mode 100644 index 0000000000000000000000000000000000000000..a1a353767d0ce8518f0d7289bed10dba0178ed12 --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/upsample.py @@ -0,0 +1,84 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch.nn as nn +import torch.nn.functional as F + +from ..utils import xavier_init +from .registry import UPSAMPLE_LAYERS + +UPSAMPLE_LAYERS.register_module('nearest', module=nn.Upsample) +UPSAMPLE_LAYERS.register_module('bilinear', module=nn.Upsample) + + +@UPSAMPLE_LAYERS.register_module(name='pixel_shuffle') +class PixelShufflePack(nn.Module): + """Pixel Shuffle upsample layer. + + This module packs `F.pixel_shuffle()` and a nn.Conv2d module together to + achieve a simple upsampling with pixel shuffle. + + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + scale_factor (int): Upsample ratio. + upsample_kernel (int): Kernel size of the conv layer to expand the + channels. + """ + + def __init__(self, in_channels, out_channels, scale_factor, + upsample_kernel): + super(PixelShufflePack, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.scale_factor = scale_factor + self.upsample_kernel = upsample_kernel + self.upsample_conv = nn.Conv2d( + self.in_channels, + self.out_channels * scale_factor * scale_factor, + self.upsample_kernel, + padding=(self.upsample_kernel - 1) // 2) + self.init_weights() + + def init_weights(self): + xavier_init(self.upsample_conv, distribution='uniform') + + def forward(self, x): + x = self.upsample_conv(x) + x = F.pixel_shuffle(x, self.scale_factor) + return x + + +def build_upsample_layer(cfg, *args, **kwargs): + """Build upsample layer. + + Args: + cfg (dict): The upsample layer config, which should contain: + + - type (str): Layer type. + - scale_factor (int): Upsample ratio, which is not applicable to + deconv. + - layer args: Args needed to instantiate a upsample layer. + args (argument list): Arguments passed to the ``__init__`` + method of the corresponding conv layer. + kwargs (keyword arguments): Keyword arguments passed to the + ``__init__`` method of the corresponding conv layer. + + Returns: + nn.Module: Created upsample layer. + """ + if not isinstance(cfg, dict): + raise TypeError(f'cfg must be a dict, but got {type(cfg)}') + if 'type' not in cfg: + raise KeyError( + f'the cfg dict must contain the key "type", but got {cfg}') + cfg_ = cfg.copy() + + layer_type = cfg_.pop('type') + if layer_type not in UPSAMPLE_LAYERS: + raise KeyError(f'Unrecognized upsample type {layer_type}') + else: + upsample = UPSAMPLE_LAYERS.get(layer_type) + + if upsample is nn.Upsample: + cfg_['mode'] = layer_type + layer = upsample(*args, **kwargs, **cfg_) + return layer diff --git a/annotator/uniformer/mmcv/cnn/bricks/wrappers.py b/annotator/uniformer/mmcv/cnn/bricks/wrappers.py new file mode 100644 index 0000000000000000000000000000000000000000..8aebf67bf52355a513f21756ee74fe510902d075 --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/bricks/wrappers.py @@ -0,0 +1,180 @@ +# Copyright (c) OpenMMLab. All rights reserved. +r"""Modified from https://github.com/facebookresearch/detectron2/blob/master/detectron2/layers/wrappers.py # noqa: E501 + +Wrap some nn modules to support empty tensor input. Currently, these wrappers +are mainly used in mask heads like fcn_mask_head and maskiou_heads since mask +heads are trained on only positive RoIs. +""" +import math + +import torch +import torch.nn as nn +from torch.nn.modules.utils import _pair, _triple + +from .registry import CONV_LAYERS, UPSAMPLE_LAYERS + +if torch.__version__ == 'parrots': + TORCH_VERSION = torch.__version__ +else: + # torch.__version__ could be 1.3.1+cu92, we only need the first two + # for comparison + TORCH_VERSION = tuple(int(x) for x in torch.__version__.split('.')[:2]) + + +def obsolete_torch_version(torch_version, version_threshold): + return torch_version == 'parrots' or torch_version <= version_threshold + + +class NewEmptyTensorOp(torch.autograd.Function): + + @staticmethod + def forward(ctx, x, new_shape): + ctx.shape = x.shape + return x.new_empty(new_shape) + + @staticmethod + def backward(ctx, grad): + shape = ctx.shape + return NewEmptyTensorOp.apply(grad, shape), None + + +@CONV_LAYERS.register_module('Conv', force=True) +class Conv2d(nn.Conv2d): + + def forward(self, x): + if x.numel() == 0 and obsolete_torch_version(TORCH_VERSION, (1, 4)): + out_shape = [x.shape[0], self.out_channels] + for i, k, p, s, d in zip(x.shape[-2:], self.kernel_size, + self.padding, self.stride, self.dilation): + o = (i + 2 * p - (d * (k - 1) + 1)) // s + 1 + out_shape.append(o) + empty = NewEmptyTensorOp.apply(x, out_shape) + if self.training: + # produce dummy gradient to avoid DDP warning. + dummy = sum(x.view(-1)[0] for x in self.parameters()) * 0.0 + return empty + dummy + else: + return empty + + return super().forward(x) + + +@CONV_LAYERS.register_module('Conv3d', force=True) +class Conv3d(nn.Conv3d): + + def forward(self, x): + if x.numel() == 0 and obsolete_torch_version(TORCH_VERSION, (1, 4)): + out_shape = [x.shape[0], self.out_channels] + for i, k, p, s, d in zip(x.shape[-3:], self.kernel_size, + self.padding, self.stride, self.dilation): + o = (i + 2 * p - (d * (k - 1) + 1)) // s + 1 + out_shape.append(o) + empty = NewEmptyTensorOp.apply(x, out_shape) + if self.training: + # produce dummy gradient to avoid DDP warning. + dummy = sum(x.view(-1)[0] for x in self.parameters()) * 0.0 + return empty + dummy + else: + return empty + + return super().forward(x) + + +@CONV_LAYERS.register_module() +@CONV_LAYERS.register_module('deconv') +@UPSAMPLE_LAYERS.register_module('deconv', force=True) +class ConvTranspose2d(nn.ConvTranspose2d): + + def forward(self, x): + if x.numel() == 0 and obsolete_torch_version(TORCH_VERSION, (1, 4)): + out_shape = [x.shape[0], self.out_channels] + for i, k, p, s, d, op in zip(x.shape[-2:], self.kernel_size, + self.padding, self.stride, + self.dilation, self.output_padding): + out_shape.append((i - 1) * s - 2 * p + (d * (k - 1) + 1) + op) + empty = NewEmptyTensorOp.apply(x, out_shape) + if self.training: + # produce dummy gradient to avoid DDP warning. + dummy = sum(x.view(-1)[0] for x in self.parameters()) * 0.0 + return empty + dummy + else: + return empty + + return super().forward(x) + + +@CONV_LAYERS.register_module() +@CONV_LAYERS.register_module('deconv3d') +@UPSAMPLE_LAYERS.register_module('deconv3d', force=True) +class ConvTranspose3d(nn.ConvTranspose3d): + + def forward(self, x): + if x.numel() == 0 and obsolete_torch_version(TORCH_VERSION, (1, 4)): + out_shape = [x.shape[0], self.out_channels] + for i, k, p, s, d, op in zip(x.shape[-3:], self.kernel_size, + self.padding, self.stride, + self.dilation, self.output_padding): + out_shape.append((i - 1) * s - 2 * p + (d * (k - 1) + 1) + op) + empty = NewEmptyTensorOp.apply(x, out_shape) + if self.training: + # produce dummy gradient to avoid DDP warning. + dummy = sum(x.view(-1)[0] for x in self.parameters()) * 0.0 + return empty + dummy + else: + return empty + + return super().forward(x) + + +class MaxPool2d(nn.MaxPool2d): + + def forward(self, x): + # PyTorch 1.9 does not support empty tensor inference yet + if x.numel() == 0 and obsolete_torch_version(TORCH_VERSION, (1, 9)): + out_shape = list(x.shape[:2]) + for i, k, p, s, d in zip(x.shape[-2:], _pair(self.kernel_size), + _pair(self.padding), _pair(self.stride), + _pair(self.dilation)): + o = (i + 2 * p - (d * (k - 1) + 1)) / s + 1 + o = math.ceil(o) if self.ceil_mode else math.floor(o) + out_shape.append(o) + empty = NewEmptyTensorOp.apply(x, out_shape) + return empty + + return super().forward(x) + + +class MaxPool3d(nn.MaxPool3d): + + def forward(self, x): + # PyTorch 1.9 does not support empty tensor inference yet + if x.numel() == 0 and obsolete_torch_version(TORCH_VERSION, (1, 9)): + out_shape = list(x.shape[:2]) + for i, k, p, s, d in zip(x.shape[-3:], _triple(self.kernel_size), + _triple(self.padding), + _triple(self.stride), + _triple(self.dilation)): + o = (i + 2 * p - (d * (k - 1) + 1)) / s + 1 + o = math.ceil(o) if self.ceil_mode else math.floor(o) + out_shape.append(o) + empty = NewEmptyTensorOp.apply(x, out_shape) + return empty + + return super().forward(x) + + +class Linear(torch.nn.Linear): + + def forward(self, x): + # empty tensor forward of Linear layer is supported in Pytorch 1.6 + if x.numel() == 0 and obsolete_torch_version(TORCH_VERSION, (1, 5)): + out_shape = [x.shape[0], self.out_features] + empty = NewEmptyTensorOp.apply(x, out_shape) + if self.training: + # produce dummy gradient to avoid DDP warning. + dummy = sum(x.view(-1)[0] for x in self.parameters()) * 0.0 + return empty + dummy + else: + return empty + + return super().forward(x) diff --git a/annotator/uniformer/mmcv/cnn/builder.py b/annotator/uniformer/mmcv/cnn/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..7567316c566bd3aca6d8f65a84b00e9e890948a7 --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/builder.py @@ -0,0 +1,30 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from ..runner import Sequential +from ..utils import Registry, build_from_cfg + + +def build_model_from_cfg(cfg, registry, default_args=None): + """Build a PyTorch model from config dict(s). Different from + ``build_from_cfg``, if cfg is a list, a ``nn.Sequential`` will be built. + + Args: + cfg (dict, list[dict]): The config of modules, is is either a config + dict or a list of config dicts. If cfg is a list, a + the built modules will be wrapped with ``nn.Sequential``. + registry (:obj:`Registry`): A registry the module belongs to. + default_args (dict, optional): Default arguments to build the module. + Defaults to None. + + Returns: + nn.Module: A built nn module. + """ + if isinstance(cfg, list): + modules = [ + build_from_cfg(cfg_, registry, default_args) for cfg_ in cfg + ] + return Sequential(*modules) + else: + return build_from_cfg(cfg, registry, default_args) + + +MODELS = Registry('model', build_func=build_model_from_cfg) diff --git a/annotator/uniformer/mmcv/cnn/resnet.py b/annotator/uniformer/mmcv/cnn/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..1cb3ac057ee2d52c46fc94685b5d4e698aad8d5f --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/resnet.py @@ -0,0 +1,316 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import logging + +import torch.nn as nn +import torch.utils.checkpoint as cp + +from .utils import constant_init, kaiming_init + + +def conv3x3(in_planes, out_planes, stride=1, dilation=1): + """3x3 convolution with padding.""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + stride=stride, + padding=dilation, + dilation=dilation, + bias=False) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, + inplanes, + planes, + stride=1, + dilation=1, + downsample=None, + style='pytorch', + with_cp=False): + super(BasicBlock, self).__init__() + assert style in ['pytorch', 'caffe'] + self.conv1 = conv3x3(inplanes, planes, stride, dilation) + self.bn1 = nn.BatchNorm2d(planes) + self.relu = nn.ReLU(inplace=True) + self.conv2 = conv3x3(planes, planes) + self.bn2 = nn.BatchNorm2d(planes) + self.downsample = downsample + self.stride = stride + self.dilation = dilation + assert not with_cp + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, + inplanes, + planes, + stride=1, + dilation=1, + downsample=None, + style='pytorch', + with_cp=False): + """Bottleneck block. + + If style is "pytorch", the stride-two layer is the 3x3 conv layer, if + it is "caffe", the stride-two layer is the first 1x1 conv layer. + """ + super(Bottleneck, self).__init__() + assert style in ['pytorch', 'caffe'] + if style == 'pytorch': + conv1_stride = 1 + conv2_stride = stride + else: + conv1_stride = stride + conv2_stride = 1 + self.conv1 = nn.Conv2d( + inplanes, planes, kernel_size=1, stride=conv1_stride, bias=False) + self.conv2 = nn.Conv2d( + planes, + planes, + kernel_size=3, + stride=conv2_stride, + padding=dilation, + dilation=dilation, + bias=False) + + self.bn1 = nn.BatchNorm2d(planes) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d( + planes, planes * self.expansion, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + self.dilation = dilation + self.with_cp = with_cp + + def forward(self, x): + + def _inner_forward(x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.downsample is not None: + residual = self.downsample(x) + + out += residual + + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + out = self.relu(out) + + return out + + +def make_res_layer(block, + inplanes, + planes, + blocks, + stride=1, + dilation=1, + style='pytorch', + with_cp=False): + downsample = None + if stride != 1 or inplanes != planes * block.expansion: + downsample = nn.Sequential( + nn.Conv2d( + inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + nn.BatchNorm2d(planes * block.expansion), + ) + + layers = [] + layers.append( + block( + inplanes, + planes, + stride, + dilation, + downsample, + style=style, + with_cp=with_cp)) + inplanes = planes * block.expansion + for _ in range(1, blocks): + layers.append( + block(inplanes, planes, 1, dilation, style=style, with_cp=with_cp)) + + return nn.Sequential(*layers) + + +class ResNet(nn.Module): + """ResNet backbone. + + Args: + depth (int): Depth of resnet, from {18, 34, 50, 101, 152}. + num_stages (int): Resnet stages, normally 4. + strides (Sequence[int]): Strides of the first block of each stage. + dilations (Sequence[int]): Dilation of each stage. + out_indices (Sequence[int]): Output from which stages. + style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two + layer is the 3x3 conv layer, otherwise the stride-two layer is + the first 1x1 conv layer. + frozen_stages (int): Stages to be frozen (all param fixed). -1 means + not freezing any parameters. + bn_eval (bool): Whether to set BN layers as eval mode, namely, freeze + running stats (mean and var). + bn_frozen (bool): Whether to freeze weight and bias of BN layers. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. + """ + + arch_settings = { + 18: (BasicBlock, (2, 2, 2, 2)), + 34: (BasicBlock, (3, 4, 6, 3)), + 50: (Bottleneck, (3, 4, 6, 3)), + 101: (Bottleneck, (3, 4, 23, 3)), + 152: (Bottleneck, (3, 8, 36, 3)) + } + + def __init__(self, + depth, + num_stages=4, + strides=(1, 2, 2, 2), + dilations=(1, 1, 1, 1), + out_indices=(0, 1, 2, 3), + style='pytorch', + frozen_stages=-1, + bn_eval=True, + bn_frozen=False, + with_cp=False): + super(ResNet, self).__init__() + if depth not in self.arch_settings: + raise KeyError(f'invalid depth {depth} for resnet') + assert num_stages >= 1 and num_stages <= 4 + block, stage_blocks = self.arch_settings[depth] + stage_blocks = stage_blocks[:num_stages] + assert len(strides) == len(dilations) == num_stages + assert max(out_indices) < num_stages + + self.out_indices = out_indices + self.style = style + self.frozen_stages = frozen_stages + self.bn_eval = bn_eval + self.bn_frozen = bn_frozen + self.with_cp = with_cp + + self.inplanes = 64 + self.conv1 = nn.Conv2d( + 3, 64, kernel_size=7, stride=2, padding=3, bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + self.res_layers = [] + for i, num_blocks in enumerate(stage_blocks): + stride = strides[i] + dilation = dilations[i] + planes = 64 * 2**i + res_layer = make_res_layer( + block, + self.inplanes, + planes, + num_blocks, + stride=stride, + dilation=dilation, + style=self.style, + with_cp=with_cp) + self.inplanes = planes * block.expansion + layer_name = f'layer{i + 1}' + self.add_module(layer_name, res_layer) + self.res_layers.append(layer_name) + + self.feat_dim = block.expansion * 64 * 2**(len(stage_blocks) - 1) + + def init_weights(self, pretrained=None): + if isinstance(pretrained, str): + logger = logging.getLogger() + from ..runner import load_checkpoint + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, nn.BatchNorm2d): + constant_init(m, 1) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + x = self.maxpool(x) + outs = [] + for i, layer_name in enumerate(self.res_layers): + res_layer = getattr(self, layer_name) + x = res_layer(x) + if i in self.out_indices: + outs.append(x) + if len(outs) == 1: + return outs[0] + else: + return tuple(outs) + + def train(self, mode=True): + super(ResNet, self).train(mode) + if self.bn_eval: + for m in self.modules(): + if isinstance(m, nn.BatchNorm2d): + m.eval() + if self.bn_frozen: + for params in m.parameters(): + params.requires_grad = False + if mode and self.frozen_stages >= 0: + for param in self.conv1.parameters(): + param.requires_grad = False + for param in self.bn1.parameters(): + param.requires_grad = False + self.bn1.eval() + self.bn1.weight.requires_grad = False + self.bn1.bias.requires_grad = False + for i in range(1, self.frozen_stages + 1): + mod = getattr(self, f'layer{i}') + mod.eval() + for param in mod.parameters(): + param.requires_grad = False diff --git a/annotator/uniformer/mmcv/cnn/utils/__init__.py b/annotator/uniformer/mmcv/cnn/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a263e31c1e3977712827ca229bbc04910b4e928e --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/utils/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .flops_counter import get_model_complexity_info +from .fuse_conv_bn import fuse_conv_bn +from .sync_bn import revert_sync_batchnorm +from .weight_init import (INITIALIZERS, Caffe2XavierInit, ConstantInit, + KaimingInit, NormalInit, PretrainedInit, + TruncNormalInit, UniformInit, XavierInit, + bias_init_with_prob, caffe2_xavier_init, + constant_init, initialize, kaiming_init, normal_init, + trunc_normal_init, uniform_init, xavier_init) + +__all__ = [ + 'get_model_complexity_info', 'bias_init_with_prob', 'caffe2_xavier_init', + 'constant_init', 'kaiming_init', 'normal_init', 'trunc_normal_init', + 'uniform_init', 'xavier_init', 'fuse_conv_bn', 'initialize', + 'INITIALIZERS', 'ConstantInit', 'XavierInit', 'NormalInit', + 'TruncNormalInit', 'UniformInit', 'KaimingInit', 'PretrainedInit', + 'Caffe2XavierInit', 'revert_sync_batchnorm' +] diff --git a/annotator/uniformer/mmcv/cnn/utils/flops_counter.py b/annotator/uniformer/mmcv/cnn/utils/flops_counter.py new file mode 100644 index 0000000000000000000000000000000000000000..d10af5feca7f4b8c0ba359b7b1c826f754e048be --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/utils/flops_counter.py @@ -0,0 +1,599 @@ +# Modified from flops-counter.pytorch by Vladislav Sovrasov +# original repo: https://github.com/sovrasov/flops-counter.pytorch + +# MIT License + +# Copyright (c) 2018 Vladislav Sovrasov + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import sys +from functools import partial + +import numpy as np +import torch +import torch.nn as nn + +import annotator.uniformer.mmcv as mmcv + + +def get_model_complexity_info(model, + input_shape, + print_per_layer_stat=True, + as_strings=True, + input_constructor=None, + flush=False, + ost=sys.stdout): + """Get complexity information of a model. + + This method can calculate FLOPs and parameter counts of a model with + corresponding input shape. It can also print complexity information for + each layer in a model. + + Supported layers are listed as below: + - Convolutions: ``nn.Conv1d``, ``nn.Conv2d``, ``nn.Conv3d``. + - Activations: ``nn.ReLU``, ``nn.PReLU``, ``nn.ELU``, ``nn.LeakyReLU``, + ``nn.ReLU6``. + - Poolings: ``nn.MaxPool1d``, ``nn.MaxPool2d``, ``nn.MaxPool3d``, + ``nn.AvgPool1d``, ``nn.AvgPool2d``, ``nn.AvgPool3d``, + ``nn.AdaptiveMaxPool1d``, ``nn.AdaptiveMaxPool2d``, + ``nn.AdaptiveMaxPool3d``, ``nn.AdaptiveAvgPool1d``, + ``nn.AdaptiveAvgPool2d``, ``nn.AdaptiveAvgPool3d``. + - BatchNorms: ``nn.BatchNorm1d``, ``nn.BatchNorm2d``, + ``nn.BatchNorm3d``, ``nn.GroupNorm``, ``nn.InstanceNorm1d``, + ``InstanceNorm2d``, ``InstanceNorm3d``, ``nn.LayerNorm``. + - Linear: ``nn.Linear``. + - Deconvolution: ``nn.ConvTranspose2d``. + - Upsample: ``nn.Upsample``. + + Args: + model (nn.Module): The model for complexity calculation. + input_shape (tuple): Input shape used for calculation. + print_per_layer_stat (bool): Whether to print complexity information + for each layer in a model. Default: True. + as_strings (bool): Output FLOPs and params counts in a string form. + Default: True. + input_constructor (None | callable): If specified, it takes a callable + method that generates input. otherwise, it will generate a random + tensor with input shape to calculate FLOPs. Default: None. + flush (bool): same as that in :func:`print`. Default: False. + ost (stream): same as ``file`` param in :func:`print`. + Default: sys.stdout. + + Returns: + tuple[float | str]: If ``as_strings`` is set to True, it will return + FLOPs and parameter counts in a string format. otherwise, it will + return those in a float number format. + """ + assert type(input_shape) is tuple + assert len(input_shape) >= 1 + assert isinstance(model, nn.Module) + flops_model = add_flops_counting_methods(model) + flops_model.eval() + flops_model.start_flops_count() + if input_constructor: + input = input_constructor(input_shape) + _ = flops_model(**input) + else: + try: + batch = torch.ones(()).new_empty( + (1, *input_shape), + dtype=next(flops_model.parameters()).dtype, + device=next(flops_model.parameters()).device) + except StopIteration: + # Avoid StopIteration for models which have no parameters, + # like `nn.Relu()`, `nn.AvgPool2d`, etc. + batch = torch.ones(()).new_empty((1, *input_shape)) + + _ = flops_model(batch) + + flops_count, params_count = flops_model.compute_average_flops_cost() + if print_per_layer_stat: + print_model_with_flops( + flops_model, flops_count, params_count, ost=ost, flush=flush) + flops_model.stop_flops_count() + + if as_strings: + return flops_to_string(flops_count), params_to_string(params_count) + + return flops_count, params_count + + +def flops_to_string(flops, units='GFLOPs', precision=2): + """Convert FLOPs number into a string. + + Note that Here we take a multiply-add counts as one FLOP. + + Args: + flops (float): FLOPs number to be converted. + units (str | None): Converted FLOPs units. Options are None, 'GFLOPs', + 'MFLOPs', 'KFLOPs', 'FLOPs'. If set to None, it will automatically + choose the most suitable unit for FLOPs. Default: 'GFLOPs'. + precision (int): Digit number after the decimal point. Default: 2. + + Returns: + str: The converted FLOPs number with units. + + Examples: + >>> flops_to_string(1e9) + '1.0 GFLOPs' + >>> flops_to_string(2e5, 'MFLOPs') + '0.2 MFLOPs' + >>> flops_to_string(3e-9, None) + '3e-09 FLOPs' + """ + if units is None: + if flops // 10**9 > 0: + return str(round(flops / 10.**9, precision)) + ' GFLOPs' + elif flops // 10**6 > 0: + return str(round(flops / 10.**6, precision)) + ' MFLOPs' + elif flops // 10**3 > 0: + return str(round(flops / 10.**3, precision)) + ' KFLOPs' + else: + return str(flops) + ' FLOPs' + else: + if units == 'GFLOPs': + return str(round(flops / 10.**9, precision)) + ' ' + units + elif units == 'MFLOPs': + return str(round(flops / 10.**6, precision)) + ' ' + units + elif units == 'KFLOPs': + return str(round(flops / 10.**3, precision)) + ' ' + units + else: + return str(flops) + ' FLOPs' + + +def params_to_string(num_params, units=None, precision=2): + """Convert parameter number into a string. + + Args: + num_params (float): Parameter number to be converted. + units (str | None): Converted FLOPs units. Options are None, 'M', + 'K' and ''. If set to None, it will automatically choose the most + suitable unit for Parameter number. Default: None. + precision (int): Digit number after the decimal point. Default: 2. + + Returns: + str: The converted parameter number with units. + + Examples: + >>> params_to_string(1e9) + '1000.0 M' + >>> params_to_string(2e5) + '200.0 k' + >>> params_to_string(3e-9) + '3e-09' + """ + if units is None: + if num_params // 10**6 > 0: + return str(round(num_params / 10**6, precision)) + ' M' + elif num_params // 10**3: + return str(round(num_params / 10**3, precision)) + ' k' + else: + return str(num_params) + else: + if units == 'M': + return str(round(num_params / 10.**6, precision)) + ' ' + units + elif units == 'K': + return str(round(num_params / 10.**3, precision)) + ' ' + units + else: + return str(num_params) + + +def print_model_with_flops(model, + total_flops, + total_params, + units='GFLOPs', + precision=3, + ost=sys.stdout, + flush=False): + """Print a model with FLOPs for each layer. + + Args: + model (nn.Module): The model to be printed. + total_flops (float): Total FLOPs of the model. + total_params (float): Total parameter counts of the model. + units (str | None): Converted FLOPs units. Default: 'GFLOPs'. + precision (int): Digit number after the decimal point. Default: 3. + ost (stream): same as `file` param in :func:`print`. + Default: sys.stdout. + flush (bool): same as that in :func:`print`. Default: False. + + Example: + >>> class ExampleModel(nn.Module): + + >>> def __init__(self): + >>> super().__init__() + >>> self.conv1 = nn.Conv2d(3, 8, 3) + >>> self.conv2 = nn.Conv2d(8, 256, 3) + >>> self.conv3 = nn.Conv2d(256, 8, 3) + >>> self.avg_pool = nn.AdaptiveAvgPool2d((1, 1)) + >>> self.flatten = nn.Flatten() + >>> self.fc = nn.Linear(8, 1) + + >>> def forward(self, x): + >>> x = self.conv1(x) + >>> x = self.conv2(x) + >>> x = self.conv3(x) + >>> x = self.avg_pool(x) + >>> x = self.flatten(x) + >>> x = self.fc(x) + >>> return x + + >>> model = ExampleModel() + >>> x = (3, 16, 16) + to print the complexity information state for each layer, you can use + >>> get_model_complexity_info(model, x) + or directly use + >>> print_model_with_flops(model, 4579784.0, 37361) + ExampleModel( + 0.037 M, 100.000% Params, 0.005 GFLOPs, 100.000% FLOPs, + (conv1): Conv2d(0.0 M, 0.600% Params, 0.0 GFLOPs, 0.959% FLOPs, 3, 8, kernel_size=(3, 3), stride=(1, 1)) # noqa: E501 + (conv2): Conv2d(0.019 M, 50.020% Params, 0.003 GFLOPs, 58.760% FLOPs, 8, 256, kernel_size=(3, 3), stride=(1, 1)) + (conv3): Conv2d(0.018 M, 49.356% Params, 0.002 GFLOPs, 40.264% FLOPs, 256, 8, kernel_size=(3, 3), stride=(1, 1)) + (avg_pool): AdaptiveAvgPool2d(0.0 M, 0.000% Params, 0.0 GFLOPs, 0.017% FLOPs, output_size=(1, 1)) + (flatten): Flatten(0.0 M, 0.000% Params, 0.0 GFLOPs, 0.000% FLOPs, ) + (fc): Linear(0.0 M, 0.024% Params, 0.0 GFLOPs, 0.000% FLOPs, in_features=8, out_features=1, bias=True) + ) + """ + + def accumulate_params(self): + if is_supported_instance(self): + return self.__params__ + else: + sum = 0 + for m in self.children(): + sum += m.accumulate_params() + return sum + + def accumulate_flops(self): + if is_supported_instance(self): + return self.__flops__ / model.__batch_counter__ + else: + sum = 0 + for m in self.children(): + sum += m.accumulate_flops() + return sum + + def flops_repr(self): + accumulated_num_params = self.accumulate_params() + accumulated_flops_cost = self.accumulate_flops() + return ', '.join([ + params_to_string( + accumulated_num_params, units='M', precision=precision), + '{:.3%} Params'.format(accumulated_num_params / total_params), + flops_to_string( + accumulated_flops_cost, units=units, precision=precision), + '{:.3%} FLOPs'.format(accumulated_flops_cost / total_flops), + self.original_extra_repr() + ]) + + def add_extra_repr(m): + m.accumulate_flops = accumulate_flops.__get__(m) + m.accumulate_params = accumulate_params.__get__(m) + flops_extra_repr = flops_repr.__get__(m) + if m.extra_repr != flops_extra_repr: + m.original_extra_repr = m.extra_repr + m.extra_repr = flops_extra_repr + assert m.extra_repr != m.original_extra_repr + + def del_extra_repr(m): + if hasattr(m, 'original_extra_repr'): + m.extra_repr = m.original_extra_repr + del m.original_extra_repr + if hasattr(m, 'accumulate_flops'): + del m.accumulate_flops + + model.apply(add_extra_repr) + print(model, file=ost, flush=flush) + model.apply(del_extra_repr) + + +def get_model_parameters_number(model): + """Calculate parameter number of a model. + + Args: + model (nn.module): The model for parameter number calculation. + + Returns: + float: Parameter number of the model. + """ + num_params = sum(p.numel() for p in model.parameters() if p.requires_grad) + return num_params + + +def add_flops_counting_methods(net_main_module): + # adding additional methods to the existing module object, + # this is done this way so that each function has access to self object + net_main_module.start_flops_count = start_flops_count.__get__( + net_main_module) + net_main_module.stop_flops_count = stop_flops_count.__get__( + net_main_module) + net_main_module.reset_flops_count = reset_flops_count.__get__( + net_main_module) + net_main_module.compute_average_flops_cost = compute_average_flops_cost.__get__( # noqa: E501 + net_main_module) + + net_main_module.reset_flops_count() + + return net_main_module + + +def compute_average_flops_cost(self): + """Compute average FLOPs cost. + + A method to compute average FLOPs cost, which will be available after + `add_flops_counting_methods()` is called on a desired net object. + + Returns: + float: Current mean flops consumption per image. + """ + batches_count = self.__batch_counter__ + flops_sum = 0 + for module in self.modules(): + if is_supported_instance(module): + flops_sum += module.__flops__ + params_sum = get_model_parameters_number(self) + return flops_sum / batches_count, params_sum + + +def start_flops_count(self): + """Activate the computation of mean flops consumption per image. + + A method to activate the computation of mean flops consumption per image. + which will be available after ``add_flops_counting_methods()`` is called on + a desired net object. It should be called before running the network. + """ + add_batch_counter_hook_function(self) + + def add_flops_counter_hook_function(module): + if is_supported_instance(module): + if hasattr(module, '__flops_handle__'): + return + + else: + handle = module.register_forward_hook( + get_modules_mapping()[type(module)]) + + module.__flops_handle__ = handle + + self.apply(partial(add_flops_counter_hook_function)) + + +def stop_flops_count(self): + """Stop computing the mean flops consumption per image. + + A method to stop computing the mean flops consumption per image, which will + be available after ``add_flops_counting_methods()`` is called on a desired + net object. It can be called to pause the computation whenever. + """ + remove_batch_counter_hook_function(self) + self.apply(remove_flops_counter_hook_function) + + +def reset_flops_count(self): + """Reset statistics computed so far. + + A method to Reset computed statistics, which will be available after + `add_flops_counting_methods()` is called on a desired net object. + """ + add_batch_counter_variables_or_reset(self) + self.apply(add_flops_counter_variable_or_reset) + + +# ---- Internal functions +def empty_flops_counter_hook(module, input, output): + module.__flops__ += 0 + + +def upsample_flops_counter_hook(module, input, output): + output_size = output[0] + batch_size = output_size.shape[0] + output_elements_count = batch_size + for val in output_size.shape[1:]: + output_elements_count *= val + module.__flops__ += int(output_elements_count) + + +def relu_flops_counter_hook(module, input, output): + active_elements_count = output.numel() + module.__flops__ += int(active_elements_count) + + +def linear_flops_counter_hook(module, input, output): + input = input[0] + output_last_dim = output.shape[ + -1] # pytorch checks dimensions, so here we don't care much + module.__flops__ += int(np.prod(input.shape) * output_last_dim) + + +def pool_flops_counter_hook(module, input, output): + input = input[0] + module.__flops__ += int(np.prod(input.shape)) + + +def norm_flops_counter_hook(module, input, output): + input = input[0] + + batch_flops = np.prod(input.shape) + if (getattr(module, 'affine', False) + or getattr(module, 'elementwise_affine', False)): + batch_flops *= 2 + module.__flops__ += int(batch_flops) + + +def deconv_flops_counter_hook(conv_module, input, output): + # Can have multiple inputs, getting the first one + input = input[0] + + batch_size = input.shape[0] + input_height, input_width = input.shape[2:] + + kernel_height, kernel_width = conv_module.kernel_size + in_channels = conv_module.in_channels + out_channels = conv_module.out_channels + groups = conv_module.groups + + filters_per_channel = out_channels // groups + conv_per_position_flops = ( + kernel_height * kernel_width * in_channels * filters_per_channel) + + active_elements_count = batch_size * input_height * input_width + overall_conv_flops = conv_per_position_flops * active_elements_count + bias_flops = 0 + if conv_module.bias is not None: + output_height, output_width = output.shape[2:] + bias_flops = out_channels * batch_size * output_height * output_height + overall_flops = overall_conv_flops + bias_flops + + conv_module.__flops__ += int(overall_flops) + + +def conv_flops_counter_hook(conv_module, input, output): + # Can have multiple inputs, getting the first one + input = input[0] + + batch_size = input.shape[0] + output_dims = list(output.shape[2:]) + + kernel_dims = list(conv_module.kernel_size) + in_channels = conv_module.in_channels + out_channels = conv_module.out_channels + groups = conv_module.groups + + filters_per_channel = out_channels // groups + conv_per_position_flops = int( + np.prod(kernel_dims)) * in_channels * filters_per_channel + + active_elements_count = batch_size * int(np.prod(output_dims)) + + overall_conv_flops = conv_per_position_flops * active_elements_count + + bias_flops = 0 + + if conv_module.bias is not None: + + bias_flops = out_channels * active_elements_count + + overall_flops = overall_conv_flops + bias_flops + + conv_module.__flops__ += int(overall_flops) + + +def batch_counter_hook(module, input, output): + batch_size = 1 + if len(input) > 0: + # Can have multiple inputs, getting the first one + input = input[0] + batch_size = len(input) + else: + pass + print('Warning! No positional inputs found for a module, ' + 'assuming batch size is 1.') + module.__batch_counter__ += batch_size + + +def add_batch_counter_variables_or_reset(module): + + module.__batch_counter__ = 0 + + +def add_batch_counter_hook_function(module): + if hasattr(module, '__batch_counter_handle__'): + return + + handle = module.register_forward_hook(batch_counter_hook) + module.__batch_counter_handle__ = handle + + +def remove_batch_counter_hook_function(module): + if hasattr(module, '__batch_counter_handle__'): + module.__batch_counter_handle__.remove() + del module.__batch_counter_handle__ + + +def add_flops_counter_variable_or_reset(module): + if is_supported_instance(module): + if hasattr(module, '__flops__') or hasattr(module, '__params__'): + print('Warning: variables __flops__ or __params__ are already ' + 'defined for the module' + type(module).__name__ + + ' ptflops can affect your code!') + module.__flops__ = 0 + module.__params__ = get_model_parameters_number(module) + + +def is_supported_instance(module): + if type(module) in get_modules_mapping(): + return True + return False + + +def remove_flops_counter_hook_function(module): + if is_supported_instance(module): + if hasattr(module, '__flops_handle__'): + module.__flops_handle__.remove() + del module.__flops_handle__ + + +def get_modules_mapping(): + return { + # convolutions + nn.Conv1d: conv_flops_counter_hook, + nn.Conv2d: conv_flops_counter_hook, + mmcv.cnn.bricks.Conv2d: conv_flops_counter_hook, + nn.Conv3d: conv_flops_counter_hook, + mmcv.cnn.bricks.Conv3d: conv_flops_counter_hook, + # activations + nn.ReLU: relu_flops_counter_hook, + nn.PReLU: relu_flops_counter_hook, + nn.ELU: relu_flops_counter_hook, + nn.LeakyReLU: relu_flops_counter_hook, + nn.ReLU6: relu_flops_counter_hook, + # poolings + nn.MaxPool1d: pool_flops_counter_hook, + nn.AvgPool1d: pool_flops_counter_hook, + nn.AvgPool2d: pool_flops_counter_hook, + nn.MaxPool2d: pool_flops_counter_hook, + mmcv.cnn.bricks.MaxPool2d: pool_flops_counter_hook, + nn.MaxPool3d: pool_flops_counter_hook, + mmcv.cnn.bricks.MaxPool3d: pool_flops_counter_hook, + nn.AvgPool3d: pool_flops_counter_hook, + nn.AdaptiveMaxPool1d: pool_flops_counter_hook, + nn.AdaptiveAvgPool1d: pool_flops_counter_hook, + nn.AdaptiveMaxPool2d: pool_flops_counter_hook, + nn.AdaptiveAvgPool2d: pool_flops_counter_hook, + nn.AdaptiveMaxPool3d: pool_flops_counter_hook, + nn.AdaptiveAvgPool3d: pool_flops_counter_hook, + # normalizations + nn.BatchNorm1d: norm_flops_counter_hook, + nn.BatchNorm2d: norm_flops_counter_hook, + nn.BatchNorm3d: norm_flops_counter_hook, + nn.GroupNorm: norm_flops_counter_hook, + nn.InstanceNorm1d: norm_flops_counter_hook, + nn.InstanceNorm2d: norm_flops_counter_hook, + nn.InstanceNorm3d: norm_flops_counter_hook, + nn.LayerNorm: norm_flops_counter_hook, + # FC + nn.Linear: linear_flops_counter_hook, + mmcv.cnn.bricks.Linear: linear_flops_counter_hook, + # Upscale + nn.Upsample: upsample_flops_counter_hook, + # Deconvolution + nn.ConvTranspose2d: deconv_flops_counter_hook, + mmcv.cnn.bricks.ConvTranspose2d: deconv_flops_counter_hook, + } diff --git a/annotator/uniformer/mmcv/cnn/utils/fuse_conv_bn.py b/annotator/uniformer/mmcv/cnn/utils/fuse_conv_bn.py new file mode 100644 index 0000000000000000000000000000000000000000..cb7076f80bf37f7931185bf0293ffcc1ce19c8ef --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/utils/fuse_conv_bn.py @@ -0,0 +1,59 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn + + +def _fuse_conv_bn(conv, bn): + """Fuse conv and bn into one module. + + Args: + conv (nn.Module): Conv to be fused. + bn (nn.Module): BN to be fused. + + Returns: + nn.Module: Fused module. + """ + conv_w = conv.weight + conv_b = conv.bias if conv.bias is not None else torch.zeros_like( + bn.running_mean) + + factor = bn.weight / torch.sqrt(bn.running_var + bn.eps) + conv.weight = nn.Parameter(conv_w * + factor.reshape([conv.out_channels, 1, 1, 1])) + conv.bias = nn.Parameter((conv_b - bn.running_mean) * factor + bn.bias) + return conv + + +def fuse_conv_bn(module): + """Recursively fuse conv and bn in a module. + + During inference, the functionary of batch norm layers is turned off + but only the mean and var alone channels are used, which exposes the + chance to fuse it with the preceding conv layers to save computations and + simplify network structures. + + Args: + module (nn.Module): Module to be fused. + + Returns: + nn.Module: Fused module. + """ + last_conv = None + last_conv_name = None + + for name, child in module.named_children(): + if isinstance(child, + (nn.modules.batchnorm._BatchNorm, nn.SyncBatchNorm)): + if last_conv is None: # only fuse BN that is after Conv + continue + fused_conv = _fuse_conv_bn(last_conv, child) + module._modules[last_conv_name] = fused_conv + # To reduce changes, set BN as Identity instead of deleting it. + module._modules[name] = nn.Identity() + last_conv = None + elif isinstance(child, nn.Conv2d): + last_conv = child + last_conv_name = name + else: + fuse_conv_bn(child) + return module diff --git a/annotator/uniformer/mmcv/cnn/utils/sync_bn.py b/annotator/uniformer/mmcv/cnn/utils/sync_bn.py new file mode 100644 index 0000000000000000000000000000000000000000..f78f39181d75bb85c53e8c7c8eaf45690e9f0bee --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/utils/sync_bn.py @@ -0,0 +1,59 @@ +import torch + +import annotator.uniformer.mmcv as mmcv + + +class _BatchNormXd(torch.nn.modules.batchnorm._BatchNorm): + """A general BatchNorm layer without input dimension check. + + Reproduced from @kapily's work: + (https://github.com/pytorch/pytorch/issues/41081#issuecomment-783961547) + The only difference between BatchNorm1d, BatchNorm2d, BatchNorm3d, etc + is `_check_input_dim` that is designed for tensor sanity checks. + The check has been bypassed in this class for the convenience of converting + SyncBatchNorm. + """ + + def _check_input_dim(self, input): + return + + +def revert_sync_batchnorm(module): + """Helper function to convert all `SyncBatchNorm` (SyncBN) and + `mmcv.ops.sync_bn.SyncBatchNorm`(MMSyncBN) layers in the model to + `BatchNormXd` layers. + + Adapted from @kapily's work: + (https://github.com/pytorch/pytorch/issues/41081#issuecomment-783961547) + + Args: + module (nn.Module): The module containing `SyncBatchNorm` layers. + + Returns: + module_output: The converted module with `BatchNormXd` layers. + """ + module_output = module + module_checklist = [torch.nn.modules.batchnorm.SyncBatchNorm] + if hasattr(mmcv, 'ops'): + module_checklist.append(mmcv.ops.SyncBatchNorm) + if isinstance(module, tuple(module_checklist)): + module_output = _BatchNormXd(module.num_features, module.eps, + module.momentum, module.affine, + module.track_running_stats) + if module.affine: + # no_grad() may not be needed here but + # just to be consistent with `convert_sync_batchnorm()` + with torch.no_grad(): + module_output.weight = module.weight + module_output.bias = module.bias + module_output.running_mean = module.running_mean + module_output.running_var = module.running_var + module_output.num_batches_tracked = module.num_batches_tracked + module_output.training = module.training + # qconfig exists in quantized models + if hasattr(module, 'qconfig'): + module_output.qconfig = module.qconfig + for name, child in module.named_children(): + module_output.add_module(name, revert_sync_batchnorm(child)) + del module + return module_output diff --git a/annotator/uniformer/mmcv/cnn/utils/weight_init.py b/annotator/uniformer/mmcv/cnn/utils/weight_init.py new file mode 100644 index 0000000000000000000000000000000000000000..287a1d0bffe26e023029d48634d9b761deda7ba4 --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/utils/weight_init.py @@ -0,0 +1,684 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy +import math +import warnings + +import numpy as np +import torch +import torch.nn as nn +from torch import Tensor + +from annotator.uniformer.mmcv.utils import Registry, build_from_cfg, get_logger, print_log + +INITIALIZERS = Registry('initializer') + + +def update_init_info(module, init_info): + """Update the `_params_init_info` in the module if the value of parameters + are changed. + + Args: + module (obj:`nn.Module`): The module of PyTorch with a user-defined + attribute `_params_init_info` which records the initialization + information. + init_info (str): The string that describes the initialization. + """ + assert hasattr( + module, + '_params_init_info'), f'Can not find `_params_init_info` in {module}' + for name, param in module.named_parameters(): + + assert param in module._params_init_info, ( + f'Find a new :obj:`Parameter` ' + f'named `{name}` during executing the ' + f'`init_weights` of ' + f'`{module.__class__.__name__}`. ' + f'Please do not add or ' + f'replace parameters during executing ' + f'the `init_weights`. ') + + # The parameter has been changed during executing the + # `init_weights` of module + mean_value = param.data.mean() + if module._params_init_info[param]['tmp_mean_value'] != mean_value: + module._params_init_info[param]['init_info'] = init_info + module._params_init_info[param]['tmp_mean_value'] = mean_value + + +def constant_init(module, val, bias=0): + if hasattr(module, 'weight') and module.weight is not None: + nn.init.constant_(module.weight, val) + if hasattr(module, 'bias') and module.bias is not None: + nn.init.constant_(module.bias, bias) + + +def xavier_init(module, gain=1, bias=0, distribution='normal'): + assert distribution in ['uniform', 'normal'] + if hasattr(module, 'weight') and module.weight is not None: + if distribution == 'uniform': + nn.init.xavier_uniform_(module.weight, gain=gain) + else: + nn.init.xavier_normal_(module.weight, gain=gain) + if hasattr(module, 'bias') and module.bias is not None: + nn.init.constant_(module.bias, bias) + + +def normal_init(module, mean=0, std=1, bias=0): + if hasattr(module, 'weight') and module.weight is not None: + nn.init.normal_(module.weight, mean, std) + if hasattr(module, 'bias') and module.bias is not None: + nn.init.constant_(module.bias, bias) + + +def trunc_normal_init(module: nn.Module, + mean: float = 0, + std: float = 1, + a: float = -2, + b: float = 2, + bias: float = 0) -> None: + if hasattr(module, 'weight') and module.weight is not None: + trunc_normal_(module.weight, mean, std, a, b) # type: ignore + if hasattr(module, 'bias') and module.bias is not None: + nn.init.constant_(module.bias, bias) # type: ignore + + +def uniform_init(module, a=0, b=1, bias=0): + if hasattr(module, 'weight') and module.weight is not None: + nn.init.uniform_(module.weight, a, b) + if hasattr(module, 'bias') and module.bias is not None: + nn.init.constant_(module.bias, bias) + + +def kaiming_init(module, + a=0, + mode='fan_out', + nonlinearity='relu', + bias=0, + distribution='normal'): + assert distribution in ['uniform', 'normal'] + if hasattr(module, 'weight') and module.weight is not None: + if distribution == 'uniform': + nn.init.kaiming_uniform_( + module.weight, a=a, mode=mode, nonlinearity=nonlinearity) + else: + nn.init.kaiming_normal_( + module.weight, a=a, mode=mode, nonlinearity=nonlinearity) + if hasattr(module, 'bias') and module.bias is not None: + nn.init.constant_(module.bias, bias) + + +def caffe2_xavier_init(module, bias=0): + # `XavierFill` in Caffe2 corresponds to `kaiming_uniform_` in PyTorch + # Acknowledgment to FAIR's internal code + kaiming_init( + module, + a=1, + mode='fan_in', + nonlinearity='leaky_relu', + bias=bias, + distribution='uniform') + + +def bias_init_with_prob(prior_prob): + """initialize conv/fc bias value according to a given probability value.""" + bias_init = float(-np.log((1 - prior_prob) / prior_prob)) + return bias_init + + +def _get_bases_name(m): + return [b.__name__ for b in m.__class__.__bases__] + + +class BaseInit(object): + + def __init__(self, *, bias=0, bias_prob=None, layer=None): + self.wholemodule = False + if not isinstance(bias, (int, float)): + raise TypeError(f'bias must be a number, but got a {type(bias)}') + + if bias_prob is not None: + if not isinstance(bias_prob, float): + raise TypeError(f'bias_prob type must be float, \ + but got {type(bias_prob)}') + + if layer is not None: + if not isinstance(layer, (str, list)): + raise TypeError(f'layer must be a str or a list of str, \ + but got a {type(layer)}') + else: + layer = [] + + if bias_prob is not None: + self.bias = bias_init_with_prob(bias_prob) + else: + self.bias = bias + self.layer = [layer] if isinstance(layer, str) else layer + + def _get_init_info(self): + info = f'{self.__class__.__name__}, bias={self.bias}' + return info + + +@INITIALIZERS.register_module(name='Constant') +class ConstantInit(BaseInit): + """Initialize module parameters with constant values. + + Args: + val (int | float): the value to fill the weights in the module with + bias (int | float): the value to fill the bias. Defaults to 0. + bias_prob (float, optional): the probability for bias initialization. + Defaults to None. + layer (str | list[str], optional): the layer will be initialized. + Defaults to None. + """ + + def __init__(self, val, **kwargs): + super().__init__(**kwargs) + self.val = val + + def __call__(self, module): + + def init(m): + if self.wholemodule: + constant_init(m, self.val, self.bias) + else: + layername = m.__class__.__name__ + basesname = _get_bases_name(m) + if len(set(self.layer) & set([layername] + basesname)): + constant_init(m, self.val, self.bias) + + module.apply(init) + if hasattr(module, '_params_init_info'): + update_init_info(module, init_info=self._get_init_info()) + + def _get_init_info(self): + info = f'{self.__class__.__name__}: val={self.val}, bias={self.bias}' + return info + + +@INITIALIZERS.register_module(name='Xavier') +class XavierInit(BaseInit): + r"""Initialize module parameters with values according to the method + described in `Understanding the difficulty of training deep feedforward + neural networks - Glorot, X. & Bengio, Y. (2010). + `_ + + Args: + gain (int | float): an optional scaling factor. Defaults to 1. + bias (int | float): the value to fill the bias. Defaults to 0. + bias_prob (float, optional): the probability for bias initialization. + Defaults to None. + distribution (str): distribution either be ``'normal'`` + or ``'uniform'``. Defaults to ``'normal'``. + layer (str | list[str], optional): the layer will be initialized. + Defaults to None. + """ + + def __init__(self, gain=1, distribution='normal', **kwargs): + super().__init__(**kwargs) + self.gain = gain + self.distribution = distribution + + def __call__(self, module): + + def init(m): + if self.wholemodule: + xavier_init(m, self.gain, self.bias, self.distribution) + else: + layername = m.__class__.__name__ + basesname = _get_bases_name(m) + if len(set(self.layer) & set([layername] + basesname)): + xavier_init(m, self.gain, self.bias, self.distribution) + + module.apply(init) + if hasattr(module, '_params_init_info'): + update_init_info(module, init_info=self._get_init_info()) + + def _get_init_info(self): + info = f'{self.__class__.__name__}: gain={self.gain}, ' \ + f'distribution={self.distribution}, bias={self.bias}' + return info + + +@INITIALIZERS.register_module(name='Normal') +class NormalInit(BaseInit): + r"""Initialize module parameters with the values drawn from the normal + distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)`. + + Args: + mean (int | float):the mean of the normal distribution. Defaults to 0. + std (int | float): the standard deviation of the normal distribution. + Defaults to 1. + bias (int | float): the value to fill the bias. Defaults to 0. + bias_prob (float, optional): the probability for bias initialization. + Defaults to None. + layer (str | list[str], optional): the layer will be initialized. + Defaults to None. + + """ + + def __init__(self, mean=0, std=1, **kwargs): + super().__init__(**kwargs) + self.mean = mean + self.std = std + + def __call__(self, module): + + def init(m): + if self.wholemodule: + normal_init(m, self.mean, self.std, self.bias) + else: + layername = m.__class__.__name__ + basesname = _get_bases_name(m) + if len(set(self.layer) & set([layername] + basesname)): + normal_init(m, self.mean, self.std, self.bias) + + module.apply(init) + if hasattr(module, '_params_init_info'): + update_init_info(module, init_info=self._get_init_info()) + + def _get_init_info(self): + info = f'{self.__class__.__name__}: mean={self.mean},' \ + f' std={self.std}, bias={self.bias}' + return info + + +@INITIALIZERS.register_module(name='TruncNormal') +class TruncNormalInit(BaseInit): + r"""Initialize module parameters with the values drawn from the normal + distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)` with values + outside :math:`[a, b]`. + + Args: + mean (float): the mean of the normal distribution. Defaults to 0. + std (float): the standard deviation of the normal distribution. + Defaults to 1. + a (float): The minimum cutoff value. + b ( float): The maximum cutoff value. + bias (float): the value to fill the bias. Defaults to 0. + bias_prob (float, optional): the probability for bias initialization. + Defaults to None. + layer (str | list[str], optional): the layer will be initialized. + Defaults to None. + + """ + + def __init__(self, + mean: float = 0, + std: float = 1, + a: float = -2, + b: float = 2, + **kwargs) -> None: + super().__init__(**kwargs) + self.mean = mean + self.std = std + self.a = a + self.b = b + + def __call__(self, module: nn.Module) -> None: + + def init(m): + if self.wholemodule: + trunc_normal_init(m, self.mean, self.std, self.a, self.b, + self.bias) + else: + layername = m.__class__.__name__ + basesname = _get_bases_name(m) + if len(set(self.layer) & set([layername] + basesname)): + trunc_normal_init(m, self.mean, self.std, self.a, self.b, + self.bias) + + module.apply(init) + if hasattr(module, '_params_init_info'): + update_init_info(module, init_info=self._get_init_info()) + + def _get_init_info(self): + info = f'{self.__class__.__name__}: a={self.a}, b={self.b},' \ + f' mean={self.mean}, std={self.std}, bias={self.bias}' + return info + + +@INITIALIZERS.register_module(name='Uniform') +class UniformInit(BaseInit): + r"""Initialize module parameters with values drawn from the uniform + distribution :math:`\mathcal{U}(a, b)`. + + Args: + a (int | float): the lower bound of the uniform distribution. + Defaults to 0. + b (int | float): the upper bound of the uniform distribution. + Defaults to 1. + bias (int | float): the value to fill the bias. Defaults to 0. + bias_prob (float, optional): the probability for bias initialization. + Defaults to None. + layer (str | list[str], optional): the layer will be initialized. + Defaults to None. + """ + + def __init__(self, a=0, b=1, **kwargs): + super().__init__(**kwargs) + self.a = a + self.b = b + + def __call__(self, module): + + def init(m): + if self.wholemodule: + uniform_init(m, self.a, self.b, self.bias) + else: + layername = m.__class__.__name__ + basesname = _get_bases_name(m) + if len(set(self.layer) & set([layername] + basesname)): + uniform_init(m, self.a, self.b, self.bias) + + module.apply(init) + if hasattr(module, '_params_init_info'): + update_init_info(module, init_info=self._get_init_info()) + + def _get_init_info(self): + info = f'{self.__class__.__name__}: a={self.a},' \ + f' b={self.b}, bias={self.bias}' + return info + + +@INITIALIZERS.register_module(name='Kaiming') +class KaimingInit(BaseInit): + r"""Initialize module parameters with the values according to the method + described in `Delving deep into rectifiers: Surpassing human-level + performance on ImageNet classification - He, K. et al. (2015). + `_ + + Args: + a (int | float): the negative slope of the rectifier used after this + layer (only used with ``'leaky_relu'``). Defaults to 0. + mode (str): either ``'fan_in'`` or ``'fan_out'``. Choosing + ``'fan_in'`` preserves the magnitude of the variance of the weights + in the forward pass. Choosing ``'fan_out'`` preserves the + magnitudes in the backwards pass. Defaults to ``'fan_out'``. + nonlinearity (str): the non-linear function (`nn.functional` name), + recommended to use only with ``'relu'`` or ``'leaky_relu'`` . + Defaults to 'relu'. + bias (int | float): the value to fill the bias. Defaults to 0. + bias_prob (float, optional): the probability for bias initialization. + Defaults to None. + distribution (str): distribution either be ``'normal'`` or + ``'uniform'``. Defaults to ``'normal'``. + layer (str | list[str], optional): the layer will be initialized. + Defaults to None. + """ + + def __init__(self, + a=0, + mode='fan_out', + nonlinearity='relu', + distribution='normal', + **kwargs): + super().__init__(**kwargs) + self.a = a + self.mode = mode + self.nonlinearity = nonlinearity + self.distribution = distribution + + def __call__(self, module): + + def init(m): + if self.wholemodule: + kaiming_init(m, self.a, self.mode, self.nonlinearity, + self.bias, self.distribution) + else: + layername = m.__class__.__name__ + basesname = _get_bases_name(m) + if len(set(self.layer) & set([layername] + basesname)): + kaiming_init(m, self.a, self.mode, self.nonlinearity, + self.bias, self.distribution) + + module.apply(init) + if hasattr(module, '_params_init_info'): + update_init_info(module, init_info=self._get_init_info()) + + def _get_init_info(self): + info = f'{self.__class__.__name__}: a={self.a}, mode={self.mode}, ' \ + f'nonlinearity={self.nonlinearity}, ' \ + f'distribution ={self.distribution}, bias={self.bias}' + return info + + +@INITIALIZERS.register_module(name='Caffe2Xavier') +class Caffe2XavierInit(KaimingInit): + # `XavierFill` in Caffe2 corresponds to `kaiming_uniform_` in PyTorch + # Acknowledgment to FAIR's internal code + def __init__(self, **kwargs): + super().__init__( + a=1, + mode='fan_in', + nonlinearity='leaky_relu', + distribution='uniform', + **kwargs) + + def __call__(self, module): + super().__call__(module) + + +@INITIALIZERS.register_module(name='Pretrained') +class PretrainedInit(object): + """Initialize module by loading a pretrained model. + + Args: + checkpoint (str): the checkpoint file of the pretrained model should + be load. + prefix (str, optional): the prefix of a sub-module in the pretrained + model. it is for loading a part of the pretrained model to + initialize. For example, if we would like to only load the + backbone of a detector model, we can set ``prefix='backbone.'``. + Defaults to None. + map_location (str): map tensors into proper locations. + """ + + def __init__(self, checkpoint, prefix=None, map_location=None): + self.checkpoint = checkpoint + self.prefix = prefix + self.map_location = map_location + + def __call__(self, module): + from annotator.uniformer.mmcv.runner import (_load_checkpoint_with_prefix, load_checkpoint, + load_state_dict) + logger = get_logger('mmcv') + if self.prefix is None: + print_log(f'load model from: {self.checkpoint}', logger=logger) + load_checkpoint( + module, + self.checkpoint, + map_location=self.map_location, + strict=False, + logger=logger) + else: + print_log( + f'load {self.prefix} in model from: {self.checkpoint}', + logger=logger) + state_dict = _load_checkpoint_with_prefix( + self.prefix, self.checkpoint, map_location=self.map_location) + load_state_dict(module, state_dict, strict=False, logger=logger) + + if hasattr(module, '_params_init_info'): + update_init_info(module, init_info=self._get_init_info()) + + def _get_init_info(self): + info = f'{self.__class__.__name__}: load from {self.checkpoint}' + return info + + +def _initialize(module, cfg, wholemodule=False): + func = build_from_cfg(cfg, INITIALIZERS) + # wholemodule flag is for override mode, there is no layer key in override + # and initializer will give init values for the whole module with the name + # in override. + func.wholemodule = wholemodule + func(module) + + +def _initialize_override(module, override, cfg): + if not isinstance(override, (dict, list)): + raise TypeError(f'override must be a dict or a list of dict, \ + but got {type(override)}') + + override = [override] if isinstance(override, dict) else override + + for override_ in override: + + cp_override = copy.deepcopy(override_) + name = cp_override.pop('name', None) + if name is None: + raise ValueError('`override` must contain the key "name",' + f'but got {cp_override}') + # if override only has name key, it means use args in init_cfg + if not cp_override: + cp_override.update(cfg) + # if override has name key and other args except type key, it will + # raise error + elif 'type' not in cp_override.keys(): + raise ValueError( + f'`override` need "type" key, but got {cp_override}') + + if hasattr(module, name): + _initialize(getattr(module, name), cp_override, wholemodule=True) + else: + raise RuntimeError(f'module did not have attribute {name}, ' + f'but init_cfg is {cp_override}.') + + +def initialize(module, init_cfg): + """Initialize a module. + + Args: + module (``torch.nn.Module``): the module will be initialized. + init_cfg (dict | list[dict]): initialization configuration dict to + define initializer. OpenMMLab has implemented 6 initializers + including ``Constant``, ``Xavier``, ``Normal``, ``Uniform``, + ``Kaiming``, and ``Pretrained``. + Example: + >>> module = nn.Linear(2, 3, bias=True) + >>> init_cfg = dict(type='Constant', layer='Linear', val =1 , bias =2) + >>> initialize(module, init_cfg) + + >>> module = nn.Sequential(nn.Conv1d(3, 1, 3), nn.Linear(1,2)) + >>> # define key ``'layer'`` for initializing layer with different + >>> # configuration + >>> init_cfg = [dict(type='Constant', layer='Conv1d', val=1), + dict(type='Constant', layer='Linear', val=2)] + >>> initialize(module, init_cfg) + + >>> # define key``'override'`` to initialize some specific part in + >>> # module + >>> class FooNet(nn.Module): + >>> def __init__(self): + >>> super().__init__() + >>> self.feat = nn.Conv2d(3, 16, 3) + >>> self.reg = nn.Conv2d(16, 10, 3) + >>> self.cls = nn.Conv2d(16, 5, 3) + >>> model = FooNet() + >>> init_cfg = dict(type='Constant', val=1, bias=2, layer='Conv2d', + >>> override=dict(type='Constant', name='reg', val=3, bias=4)) + >>> initialize(model, init_cfg) + + >>> model = ResNet(depth=50) + >>> # Initialize weights with the pretrained model. + >>> init_cfg = dict(type='Pretrained', + checkpoint='torchvision://resnet50') + >>> initialize(model, init_cfg) + + >>> # Initialize weights of a sub-module with the specific part of + >>> # a pretrained model by using "prefix". + >>> url = 'http://download.openmmlab.com/mmdetection/v2.0/retinanet/'\ + >>> 'retinanet_r50_fpn_1x_coco/'\ + >>> 'retinanet_r50_fpn_1x_coco_20200130-c2398f9e.pth' + >>> init_cfg = dict(type='Pretrained', + checkpoint=url, prefix='backbone.') + """ + if not isinstance(init_cfg, (dict, list)): + raise TypeError(f'init_cfg must be a dict or a list of dict, \ + but got {type(init_cfg)}') + + if isinstance(init_cfg, dict): + init_cfg = [init_cfg] + + for cfg in init_cfg: + # should deeply copy the original config because cfg may be used by + # other modules, e.g., one init_cfg shared by multiple bottleneck + # blocks, the expected cfg will be changed after pop and will change + # the initialization behavior of other modules + cp_cfg = copy.deepcopy(cfg) + override = cp_cfg.pop('override', None) + _initialize(module, cp_cfg) + + if override is not None: + cp_cfg.pop('layer', None) + _initialize_override(module, override, cp_cfg) + else: + # All attributes in module have same initialization. + pass + + +def _no_grad_trunc_normal_(tensor: Tensor, mean: float, std: float, a: float, + b: float) -> Tensor: + # Method based on + # https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf + # Modified from + # https://github.com/pytorch/pytorch/blob/master/torch/nn/init.py + def norm_cdf(x): + # Computes standard normal cumulative distribution function + return (1. + math.erf(x / math.sqrt(2.))) / 2. + + if (mean < a - 2 * std) or (mean > b + 2 * std): + warnings.warn( + 'mean is more than 2 std from [a, b] in nn.init.trunc_normal_. ' + 'The distribution of values may be incorrect.', + stacklevel=2) + + with torch.no_grad(): + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + lower = norm_cdf((a - mean) / std) + upper = norm_cdf((b - mean) / std) + + # Uniformly fill tensor with values from [lower, upper], then translate + # to [2lower-1, 2upper-1]. + tensor.uniform_(2 * lower - 1, 2 * upper - 1) + + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() + + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.)) + tensor.add_(mean) + + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor + + +def trunc_normal_(tensor: Tensor, + mean: float = 0., + std: float = 1., + a: float = -2., + b: float = 2.) -> Tensor: + r"""Fills the input Tensor with values drawn from a truncated + normal distribution. The values are effectively drawn from the + normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)` + with values outside :math:`[a, b]` redrawn until they are within + the bounds. The method used for generating the random values works + best when :math:`a \leq \text{mean} \leq b`. + + Modified from + https://github.com/pytorch/pytorch/blob/master/torch/nn/init.py + + Args: + tensor (``torch.Tensor``): an n-dimensional `torch.Tensor`. + mean (float): the mean of the normal distribution. + std (float): the standard deviation of the normal distribution. + a (float): the minimum cutoff value. + b (float): the maximum cutoff value. + """ + return _no_grad_trunc_normal_(tensor, mean, std, a, b) diff --git a/annotator/uniformer/mmcv/cnn/vgg.py b/annotator/uniformer/mmcv/cnn/vgg.py new file mode 100644 index 0000000000000000000000000000000000000000..8778b649561a45a9652b1a15a26c2d171e58f3e1 --- /dev/null +++ b/annotator/uniformer/mmcv/cnn/vgg.py @@ -0,0 +1,175 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import logging + +import torch.nn as nn + +from .utils import constant_init, kaiming_init, normal_init + + +def conv3x3(in_planes, out_planes, dilation=1): + """3x3 convolution with padding.""" + return nn.Conv2d( + in_planes, + out_planes, + kernel_size=3, + padding=dilation, + dilation=dilation) + + +def make_vgg_layer(inplanes, + planes, + num_blocks, + dilation=1, + with_bn=False, + ceil_mode=False): + layers = [] + for _ in range(num_blocks): + layers.append(conv3x3(inplanes, planes, dilation)) + if with_bn: + layers.append(nn.BatchNorm2d(planes)) + layers.append(nn.ReLU(inplace=True)) + inplanes = planes + layers.append(nn.MaxPool2d(kernel_size=2, stride=2, ceil_mode=ceil_mode)) + + return layers + + +class VGG(nn.Module): + """VGG backbone. + + Args: + depth (int): Depth of vgg, from {11, 13, 16, 19}. + with_bn (bool): Use BatchNorm or not. + num_classes (int): number of classes for classification. + num_stages (int): VGG stages, normally 5. + dilations (Sequence[int]): Dilation of each stage. + out_indices (Sequence[int]): Output from which stages. + frozen_stages (int): Stages to be frozen (all param fixed). -1 means + not freezing any parameters. + bn_eval (bool): Whether to set BN layers as eval mode, namely, freeze + running stats (mean and var). + bn_frozen (bool): Whether to freeze weight and bias of BN layers. + """ + + arch_settings = { + 11: (1, 1, 2, 2, 2), + 13: (2, 2, 2, 2, 2), + 16: (2, 2, 3, 3, 3), + 19: (2, 2, 4, 4, 4) + } + + def __init__(self, + depth, + with_bn=False, + num_classes=-1, + num_stages=5, + dilations=(1, 1, 1, 1, 1), + out_indices=(0, 1, 2, 3, 4), + frozen_stages=-1, + bn_eval=True, + bn_frozen=False, + ceil_mode=False, + with_last_pool=True): + super(VGG, self).__init__() + if depth not in self.arch_settings: + raise KeyError(f'invalid depth {depth} for vgg') + assert num_stages >= 1 and num_stages <= 5 + stage_blocks = self.arch_settings[depth] + self.stage_blocks = stage_blocks[:num_stages] + assert len(dilations) == num_stages + assert max(out_indices) <= num_stages + + self.num_classes = num_classes + self.out_indices = out_indices + self.frozen_stages = frozen_stages + self.bn_eval = bn_eval + self.bn_frozen = bn_frozen + + self.inplanes = 3 + start_idx = 0 + vgg_layers = [] + self.range_sub_modules = [] + for i, num_blocks in enumerate(self.stage_blocks): + num_modules = num_blocks * (2 + with_bn) + 1 + end_idx = start_idx + num_modules + dilation = dilations[i] + planes = 64 * 2**i if i < 4 else 512 + vgg_layer = make_vgg_layer( + self.inplanes, + planes, + num_blocks, + dilation=dilation, + with_bn=with_bn, + ceil_mode=ceil_mode) + vgg_layers.extend(vgg_layer) + self.inplanes = planes + self.range_sub_modules.append([start_idx, end_idx]) + start_idx = end_idx + if not with_last_pool: + vgg_layers.pop(-1) + self.range_sub_modules[-1][1] -= 1 + self.module_name = 'features' + self.add_module(self.module_name, nn.Sequential(*vgg_layers)) + + if self.num_classes > 0: + self.classifier = nn.Sequential( + nn.Linear(512 * 7 * 7, 4096), + nn.ReLU(True), + nn.Dropout(), + nn.Linear(4096, 4096), + nn.ReLU(True), + nn.Dropout(), + nn.Linear(4096, num_classes), + ) + + def init_weights(self, pretrained=None): + if isinstance(pretrained, str): + logger = logging.getLogger() + from ..runner import load_checkpoint + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, nn.BatchNorm2d): + constant_init(m, 1) + elif isinstance(m, nn.Linear): + normal_init(m, std=0.01) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + outs = [] + vgg_layers = getattr(self, self.module_name) + for i in range(len(self.stage_blocks)): + for j in range(*self.range_sub_modules[i]): + vgg_layer = vgg_layers[j] + x = vgg_layer(x) + if i in self.out_indices: + outs.append(x) + if self.num_classes > 0: + x = x.view(x.size(0), -1) + x = self.classifier(x) + outs.append(x) + if len(outs) == 1: + return outs[0] + else: + return tuple(outs) + + def train(self, mode=True): + super(VGG, self).train(mode) + if self.bn_eval: + for m in self.modules(): + if isinstance(m, nn.BatchNorm2d): + m.eval() + if self.bn_frozen: + for params in m.parameters(): + params.requires_grad = False + vgg_layers = getattr(self, self.module_name) + if mode and self.frozen_stages >= 0: + for i in range(self.frozen_stages): + for j in range(*self.range_sub_modules[i]): + mod = vgg_layers[j] + mod.eval() + for param in mod.parameters(): + param.requires_grad = False diff --git a/annotator/uniformer/mmcv/engine/__init__.py b/annotator/uniformer/mmcv/engine/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3193b7f664e19ce2458d81c836597fa22e4bb082 --- /dev/null +++ b/annotator/uniformer/mmcv/engine/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .test import (collect_results_cpu, collect_results_gpu, multi_gpu_test, + single_gpu_test) + +__all__ = [ + 'collect_results_cpu', 'collect_results_gpu', 'multi_gpu_test', + 'single_gpu_test' +] diff --git a/annotator/uniformer/mmcv/engine/test.py b/annotator/uniformer/mmcv/engine/test.py new file mode 100644 index 0000000000000000000000000000000000000000..8dbeef271db634ec2dadfda3bc0b5ef9c7a677ff --- /dev/null +++ b/annotator/uniformer/mmcv/engine/test.py @@ -0,0 +1,202 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os.path as osp +import pickle +import shutil +import tempfile +import time + +import torch +import torch.distributed as dist + +import annotator.uniformer.mmcv as mmcv +from annotator.uniformer.mmcv.runner import get_dist_info + + +def single_gpu_test(model, data_loader): + """Test model with a single gpu. + + This method tests model with a single gpu and displays test progress bar. + + Args: + model (nn.Module): Model to be tested. + data_loader (nn.Dataloader): Pytorch data loader. + + Returns: + list: The prediction results. + """ + model.eval() + results = [] + dataset = data_loader.dataset + prog_bar = mmcv.ProgressBar(len(dataset)) + for data in data_loader: + with torch.no_grad(): + result = model(return_loss=False, **data) + results.extend(result) + + # Assume result has the same length of batch_size + # refer to https://github.com/open-mmlab/mmcv/issues/985 + batch_size = len(result) + for _ in range(batch_size): + prog_bar.update() + return results + + +def multi_gpu_test(model, data_loader, tmpdir=None, gpu_collect=False): + """Test model with multiple gpus. + + This method tests model with multiple gpus and collects the results + under two different modes: gpu and cpu modes. By setting + ``gpu_collect=True``, it encodes results to gpu tensors and use gpu + communication for results collection. On cpu mode it saves the results on + different gpus to ``tmpdir`` and collects them by the rank 0 worker. + + Args: + model (nn.Module): Model to be tested. + data_loader (nn.Dataloader): Pytorch data loader. + tmpdir (str): Path of directory to save the temporary results from + different gpus under cpu mode. + gpu_collect (bool): Option to use either gpu or cpu to collect results. + + Returns: + list: The prediction results. + """ + model.eval() + results = [] + dataset = data_loader.dataset + rank, world_size = get_dist_info() + if rank == 0: + prog_bar = mmcv.ProgressBar(len(dataset)) + time.sleep(2) # This line can prevent deadlock problem in some cases. + for i, data in enumerate(data_loader): + with torch.no_grad(): + result = model(return_loss=False, **data) + results.extend(result) + + if rank == 0: + batch_size = len(result) + batch_size_all = batch_size * world_size + if batch_size_all + prog_bar.completed > len(dataset): + batch_size_all = len(dataset) - prog_bar.completed + for _ in range(batch_size_all): + prog_bar.update() + + # collect results from all ranks + if gpu_collect: + results = collect_results_gpu(results, len(dataset)) + else: + results = collect_results_cpu(results, len(dataset), tmpdir) + return results + + +def collect_results_cpu(result_part, size, tmpdir=None): + """Collect results under cpu mode. + + On cpu mode, this function will save the results on different gpus to + ``tmpdir`` and collect them by the rank 0 worker. + + Args: + result_part (list): Result list containing result parts + to be collected. + size (int): Size of the results, commonly equal to length of + the results. + tmpdir (str | None): temporal directory for collected results to + store. If set to None, it will create a random temporal directory + for it. + + Returns: + list: The collected results. + """ + rank, world_size = get_dist_info() + # create a tmp dir if it is not specified + if tmpdir is None: + MAX_LEN = 512 + # 32 is whitespace + dir_tensor = torch.full((MAX_LEN, ), + 32, + dtype=torch.uint8, + device='cuda') + if rank == 0: + mmcv.mkdir_or_exist('.dist_test') + tmpdir = tempfile.mkdtemp(dir='.dist_test') + tmpdir = torch.tensor( + bytearray(tmpdir.encode()), dtype=torch.uint8, device='cuda') + dir_tensor[:len(tmpdir)] = tmpdir + dist.broadcast(dir_tensor, 0) + tmpdir = dir_tensor.cpu().numpy().tobytes().decode().rstrip() + else: + mmcv.mkdir_or_exist(tmpdir) + # dump the part result to the dir + mmcv.dump(result_part, osp.join(tmpdir, f'part_{rank}.pkl')) + dist.barrier() + # collect all parts + if rank != 0: + return None + else: + # load results of all parts from tmp dir + part_list = [] + for i in range(world_size): + part_file = osp.join(tmpdir, f'part_{i}.pkl') + part_result = mmcv.load(part_file) + # When data is severely insufficient, an empty part_result + # on a certain gpu could makes the overall outputs empty. + if part_result: + part_list.append(part_result) + # sort the results + ordered_results = [] + for res in zip(*part_list): + ordered_results.extend(list(res)) + # the dataloader may pad some samples + ordered_results = ordered_results[:size] + # remove tmp dir + shutil.rmtree(tmpdir) + return ordered_results + + +def collect_results_gpu(result_part, size): + """Collect results under gpu mode. + + On gpu mode, this function will encode results to gpu tensors and use gpu + communication for results collection. + + Args: + result_part (list): Result list containing result parts + to be collected. + size (int): Size of the results, commonly equal to length of + the results. + + Returns: + list: The collected results. + """ + rank, world_size = get_dist_info() + # dump result part to tensor with pickle + part_tensor = torch.tensor( + bytearray(pickle.dumps(result_part)), dtype=torch.uint8, device='cuda') + # gather all result part tensor shape + shape_tensor = torch.tensor(part_tensor.shape, device='cuda') + shape_list = [shape_tensor.clone() for _ in range(world_size)] + dist.all_gather(shape_list, shape_tensor) + # padding result part tensor to max length + shape_max = torch.tensor(shape_list).max() + part_send = torch.zeros(shape_max, dtype=torch.uint8, device='cuda') + part_send[:shape_tensor[0]] = part_tensor + part_recv_list = [ + part_tensor.new_zeros(shape_max) for _ in range(world_size) + ] + # gather all result part + dist.all_gather(part_recv_list, part_send) + + if rank == 0: + part_list = [] + for recv, shape in zip(part_recv_list, shape_list): + part_result = pickle.loads(recv[:shape[0]].cpu().numpy().tobytes()) + # When data is severely insufficient, an empty part_result + # on a certain gpu could makes the overall outputs empty. + if part_result: + part_list.append(part_result) + # sort the results + ordered_results = [] + for res in zip(*part_list): + ordered_results.extend(list(res)) + # the dataloader may pad some samples + ordered_results = ordered_results[:size] + return ordered_results diff --git a/annotator/uniformer/mmcv/fileio/__init__.py b/annotator/uniformer/mmcv/fileio/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2051b85f7e59bff7bdbaa131849ce8cd31f059a4 --- /dev/null +++ b/annotator/uniformer/mmcv/fileio/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .file_client import BaseStorageBackend, FileClient +from .handlers import BaseFileHandler, JsonHandler, PickleHandler, YamlHandler +from .io import dump, load, register_handler +from .parse import dict_from_file, list_from_file + +__all__ = [ + 'BaseStorageBackend', 'FileClient', 'load', 'dump', 'register_handler', + 'BaseFileHandler', 'JsonHandler', 'PickleHandler', 'YamlHandler', + 'list_from_file', 'dict_from_file' +] diff --git a/annotator/uniformer/mmcv/fileio/file_client.py b/annotator/uniformer/mmcv/fileio/file_client.py new file mode 100644 index 0000000000000000000000000000000000000000..950f0c1aeab14b8e308a7455ccd64a95b5d98add --- /dev/null +++ b/annotator/uniformer/mmcv/fileio/file_client.py @@ -0,0 +1,1148 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import inspect +import os +import os.path as osp +import re +import tempfile +import warnings +from abc import ABCMeta, abstractmethod +from contextlib import contextmanager +from pathlib import Path +from typing import Iterable, Iterator, Optional, Tuple, Union +from urllib.request import urlopen + +import annotator.uniformer.mmcv as mmcv +from annotator.uniformer.mmcv.utils.misc import has_method +from annotator.uniformer.mmcv.utils.path import is_filepath + + +class BaseStorageBackend(metaclass=ABCMeta): + """Abstract class of storage backends. + + All backends need to implement two apis: ``get()`` and ``get_text()``. + ``get()`` reads the file as a byte stream and ``get_text()`` reads the file + as texts. + """ + + # a flag to indicate whether the backend can create a symlink for a file + _allow_symlink = False + + @property + def name(self): + return self.__class__.__name__ + + @property + def allow_symlink(self): + return self._allow_symlink + + @abstractmethod + def get(self, filepath): + pass + + @abstractmethod + def get_text(self, filepath): + pass + + +class CephBackend(BaseStorageBackend): + """Ceph storage backend (for internal use). + + Args: + path_mapping (dict|None): path mapping dict from local path to Petrel + path. When ``path_mapping={'src': 'dst'}``, ``src`` in ``filepath`` + will be replaced by ``dst``. Default: None. + + .. warning:: + :class:`mmcv.fileio.file_client.CephBackend` will be deprecated, + please use :class:`mmcv.fileio.file_client.PetrelBackend` instead. + """ + + def __init__(self, path_mapping=None): + try: + import ceph + except ImportError: + raise ImportError('Please install ceph to enable CephBackend.') + + warnings.warn( + 'CephBackend will be deprecated, please use PetrelBackend instead') + self._client = ceph.S3Client() + assert isinstance(path_mapping, dict) or path_mapping is None + self.path_mapping = path_mapping + + def get(self, filepath): + filepath = str(filepath) + if self.path_mapping is not None: + for k, v in self.path_mapping.items(): + filepath = filepath.replace(k, v) + value = self._client.Get(filepath) + value_buf = memoryview(value) + return value_buf + + def get_text(self, filepath, encoding=None): + raise NotImplementedError + + +class PetrelBackend(BaseStorageBackend): + """Petrel storage backend (for internal use). + + PetrelBackend supports reading and writing data to multiple clusters. + If the file path contains the cluster name, PetrelBackend will read data + from specified cluster or write data to it. Otherwise, PetrelBackend will + access the default cluster. + + Args: + path_mapping (dict, optional): Path mapping dict from local path to + Petrel path. When ``path_mapping={'src': 'dst'}``, ``src`` in + ``filepath`` will be replaced by ``dst``. Default: None. + enable_mc (bool, optional): Whether to enable memcached support. + Default: True. + + Examples: + >>> filepath1 = 's3://path/of/file' + >>> filepath2 = 'cluster-name:s3://path/of/file' + >>> client = PetrelBackend() + >>> client.get(filepath1) # get data from default cluster + >>> client.get(filepath2) # get data from 'cluster-name' cluster + """ + + def __init__(self, + path_mapping: Optional[dict] = None, + enable_mc: bool = True): + try: + from petrel_client import client + except ImportError: + raise ImportError('Please install petrel_client to enable ' + 'PetrelBackend.') + + self._client = client.Client(enable_mc=enable_mc) + assert isinstance(path_mapping, dict) or path_mapping is None + self.path_mapping = path_mapping + + def _map_path(self, filepath: Union[str, Path]) -> str: + """Map ``filepath`` to a string path whose prefix will be replaced by + :attr:`self.path_mapping`. + + Args: + filepath (str): Path to be mapped. + """ + filepath = str(filepath) + if self.path_mapping is not None: + for k, v in self.path_mapping.items(): + filepath = filepath.replace(k, v) + return filepath + + def _format_path(self, filepath: str) -> str: + """Convert a ``filepath`` to standard format of petrel oss. + + If the ``filepath`` is concatenated by ``os.path.join``, in a Windows + environment, the ``filepath`` will be the format of + 's3://bucket_name\\image.jpg'. By invoking :meth:`_format_path`, the + above ``filepath`` will be converted to 's3://bucket_name/image.jpg'. + + Args: + filepath (str): Path to be formatted. + """ + return re.sub(r'\\+', '/', filepath) + + def get(self, filepath: Union[str, Path]) -> memoryview: + """Read data from a given ``filepath`` with 'rb' mode. + + Args: + filepath (str or Path): Path to read data. + + Returns: + memoryview: A memory view of expected bytes object to avoid + copying. The memoryview object can be converted to bytes by + ``value_buf.tobytes()``. + """ + filepath = self._map_path(filepath) + filepath = self._format_path(filepath) + value = self._client.Get(filepath) + value_buf = memoryview(value) + return value_buf + + def get_text(self, + filepath: Union[str, Path], + encoding: str = 'utf-8') -> str: + """Read data from a given ``filepath`` with 'r' mode. + + Args: + filepath (str or Path): Path to read data. + encoding (str): The encoding format used to open the ``filepath``. + Default: 'utf-8'. + + Returns: + str: Expected text reading from ``filepath``. + """ + return str(self.get(filepath), encoding=encoding) + + def put(self, obj: bytes, filepath: Union[str, Path]) -> None: + """Save data to a given ``filepath``. + + Args: + obj (bytes): Data to be saved. + filepath (str or Path): Path to write data. + """ + filepath = self._map_path(filepath) + filepath = self._format_path(filepath) + self._client.put(filepath, obj) + + def put_text(self, + obj: str, + filepath: Union[str, Path], + encoding: str = 'utf-8') -> None: + """Save data to a given ``filepath``. + + Args: + obj (str): Data to be written. + filepath (str or Path): Path to write data. + encoding (str): The encoding format used to encode the ``obj``. + Default: 'utf-8'. + """ + self.put(bytes(obj, encoding=encoding), filepath) + + def remove(self, filepath: Union[str, Path]) -> None: + """Remove a file. + + Args: + filepath (str or Path): Path to be removed. + """ + if not has_method(self._client, 'delete'): + raise NotImplementedError( + ('Current version of Petrel Python SDK has not supported ' + 'the `delete` method, please use a higher version or dev' + ' branch instead.')) + + filepath = self._map_path(filepath) + filepath = self._format_path(filepath) + self._client.delete(filepath) + + def exists(self, filepath: Union[str, Path]) -> bool: + """Check whether a file path exists. + + Args: + filepath (str or Path): Path to be checked whether exists. + + Returns: + bool: Return ``True`` if ``filepath`` exists, ``False`` otherwise. + """ + if not (has_method(self._client, 'contains') + and has_method(self._client, 'isdir')): + raise NotImplementedError( + ('Current version of Petrel Python SDK has not supported ' + 'the `contains` and `isdir` methods, please use a higher' + 'version or dev branch instead.')) + + filepath = self._map_path(filepath) + filepath = self._format_path(filepath) + return self._client.contains(filepath) or self._client.isdir(filepath) + + def isdir(self, filepath: Union[str, Path]) -> bool: + """Check whether a file path is a directory. + + Args: + filepath (str or Path): Path to be checked whether it is a + directory. + + Returns: + bool: Return ``True`` if ``filepath`` points to a directory, + ``False`` otherwise. + """ + if not has_method(self._client, 'isdir'): + raise NotImplementedError( + ('Current version of Petrel Python SDK has not supported ' + 'the `isdir` method, please use a higher version or dev' + ' branch instead.')) + + filepath = self._map_path(filepath) + filepath = self._format_path(filepath) + return self._client.isdir(filepath) + + def isfile(self, filepath: Union[str, Path]) -> bool: + """Check whether a file path is a file. + + Args: + filepath (str or Path): Path to be checked whether it is a file. + + Returns: + bool: Return ``True`` if ``filepath`` points to a file, ``False`` + otherwise. + """ + if not has_method(self._client, 'contains'): + raise NotImplementedError( + ('Current version of Petrel Python SDK has not supported ' + 'the `contains` method, please use a higher version or ' + 'dev branch instead.')) + + filepath = self._map_path(filepath) + filepath = self._format_path(filepath) + return self._client.contains(filepath) + + def join_path(self, filepath: Union[str, Path], + *filepaths: Union[str, Path]) -> str: + """Concatenate all file paths. + + Args: + filepath (str or Path): Path to be concatenated. + + Returns: + str: The result after concatenation. + """ + filepath = self._format_path(self._map_path(filepath)) + if filepath.endswith('/'): + filepath = filepath[:-1] + formatted_paths = [filepath] + for path in filepaths: + formatted_paths.append(self._format_path(self._map_path(path))) + return '/'.join(formatted_paths) + + @contextmanager + def get_local_path(self, filepath: Union[str, Path]) -> Iterable[str]: + """Download a file from ``filepath`` and return a temporary path. + + ``get_local_path`` is decorated by :meth:`contxtlib.contextmanager`. It + can be called with ``with`` statement, and when exists from the + ``with`` statement, the temporary path will be released. + + Args: + filepath (str | Path): Download a file from ``filepath``. + + Examples: + >>> client = PetrelBackend() + >>> # After existing from the ``with`` clause, + >>> # the path will be removed + >>> with client.get_local_path('s3://path/of/your/file') as path: + ... # do something here + + Yields: + Iterable[str]: Only yield one temporary path. + """ + filepath = self._map_path(filepath) + filepath = self._format_path(filepath) + assert self.isfile(filepath) + try: + f = tempfile.NamedTemporaryFile(delete=False) + f.write(self.get(filepath)) + f.close() + yield f.name + finally: + os.remove(f.name) + + def list_dir_or_file(self, + dir_path: Union[str, Path], + list_dir: bool = True, + list_file: bool = True, + suffix: Optional[Union[str, Tuple[str]]] = None, + recursive: bool = False) -> Iterator[str]: + """Scan a directory to find the interested directories or files in + arbitrary order. + + Note: + Petrel has no concept of directories but it simulates the directory + hierarchy in the filesystem through public prefixes. In addition, + if the returned path ends with '/', it means the path is a public + prefix which is a logical directory. + + Note: + :meth:`list_dir_or_file` returns the path relative to ``dir_path``. + In addition, the returned path of directory will not contains the + suffix '/' which is consistent with other backends. + + Args: + dir_path (str | Path): Path of the directory. + list_dir (bool): List the directories. Default: True. + list_file (bool): List the path of files. Default: True. + suffix (str or tuple[str], optional): File suffix + that we are interested in. Default: None. + recursive (bool): If set to True, recursively scan the + directory. Default: False. + + Yields: + Iterable[str]: A relative path to ``dir_path``. + """ + if not has_method(self._client, 'list'): + raise NotImplementedError( + ('Current version of Petrel Python SDK has not supported ' + 'the `list` method, please use a higher version or dev' + ' branch instead.')) + + dir_path = self._map_path(dir_path) + dir_path = self._format_path(dir_path) + if list_dir and suffix is not None: + raise TypeError( + '`list_dir` should be False when `suffix` is not None') + + if (suffix is not None) and not isinstance(suffix, (str, tuple)): + raise TypeError('`suffix` must be a string or tuple of strings') + + # Petrel's simulated directory hierarchy assumes that directory paths + # should end with `/` + if not dir_path.endswith('/'): + dir_path += '/' + + root = dir_path + + def _list_dir_or_file(dir_path, list_dir, list_file, suffix, + recursive): + for path in self._client.list(dir_path): + # the `self.isdir` is not used here to determine whether path + # is a directory, because `self.isdir` relies on + # `self._client.list` + if path.endswith('/'): # a directory path + next_dir_path = self.join_path(dir_path, path) + if list_dir: + # get the relative path and exclude the last + # character '/' + rel_dir = next_dir_path[len(root):-1] + yield rel_dir + if recursive: + yield from _list_dir_or_file(next_dir_path, list_dir, + list_file, suffix, + recursive) + else: # a file path + absolute_path = self.join_path(dir_path, path) + rel_path = absolute_path[len(root):] + if (suffix is None + or rel_path.endswith(suffix)) and list_file: + yield rel_path + + return _list_dir_or_file(dir_path, list_dir, list_file, suffix, + recursive) + + +class MemcachedBackend(BaseStorageBackend): + """Memcached storage backend. + + Attributes: + server_list_cfg (str): Config file for memcached server list. + client_cfg (str): Config file for memcached client. + sys_path (str | None): Additional path to be appended to `sys.path`. + Default: None. + """ + + def __init__(self, server_list_cfg, client_cfg, sys_path=None): + if sys_path is not None: + import sys + sys.path.append(sys_path) + try: + import mc + except ImportError: + raise ImportError( + 'Please install memcached to enable MemcachedBackend.') + + self.server_list_cfg = server_list_cfg + self.client_cfg = client_cfg + self._client = mc.MemcachedClient.GetInstance(self.server_list_cfg, + self.client_cfg) + # mc.pyvector servers as a point which points to a memory cache + self._mc_buffer = mc.pyvector() + + def get(self, filepath): + filepath = str(filepath) + import mc + self._client.Get(filepath, self._mc_buffer) + value_buf = mc.ConvertBuffer(self._mc_buffer) + return value_buf + + def get_text(self, filepath, encoding=None): + raise NotImplementedError + + +class LmdbBackend(BaseStorageBackend): + """Lmdb storage backend. + + Args: + db_path (str): Lmdb database path. + readonly (bool, optional): Lmdb environment parameter. If True, + disallow any write operations. Default: True. + lock (bool, optional): Lmdb environment parameter. If False, when + concurrent access occurs, do not lock the database. Default: False. + readahead (bool, optional): Lmdb environment parameter. If False, + disable the OS filesystem readahead mechanism, which may improve + random read performance when a database is larger than RAM. + Default: False. + + Attributes: + db_path (str): Lmdb database path. + """ + + def __init__(self, + db_path, + readonly=True, + lock=False, + readahead=False, + **kwargs): + try: + import lmdb + except ImportError: + raise ImportError('Please install lmdb to enable LmdbBackend.') + + self.db_path = str(db_path) + self._client = lmdb.open( + self.db_path, + readonly=readonly, + lock=lock, + readahead=readahead, + **kwargs) + + def get(self, filepath): + """Get values according to the filepath. + + Args: + filepath (str | obj:`Path`): Here, filepath is the lmdb key. + """ + filepath = str(filepath) + with self._client.begin(write=False) as txn: + value_buf = txn.get(filepath.encode('ascii')) + return value_buf + + def get_text(self, filepath, encoding=None): + raise NotImplementedError + + +class HardDiskBackend(BaseStorageBackend): + """Raw hard disks storage backend.""" + + _allow_symlink = True + + def get(self, filepath: Union[str, Path]) -> bytes: + """Read data from a given ``filepath`` with 'rb' mode. + + Args: + filepath (str or Path): Path to read data. + + Returns: + bytes: Expected bytes object. + """ + with open(filepath, 'rb') as f: + value_buf = f.read() + return value_buf + + def get_text(self, + filepath: Union[str, Path], + encoding: str = 'utf-8') -> str: + """Read data from a given ``filepath`` with 'r' mode. + + Args: + filepath (str or Path): Path to read data. + encoding (str): The encoding format used to open the ``filepath``. + Default: 'utf-8'. + + Returns: + str: Expected text reading from ``filepath``. + """ + with open(filepath, 'r', encoding=encoding) as f: + value_buf = f.read() + return value_buf + + def put(self, obj: bytes, filepath: Union[str, Path]) -> None: + """Write data to a given ``filepath`` with 'wb' mode. + + Note: + ``put`` will create a directory if the directory of ``filepath`` + does not exist. + + Args: + obj (bytes): Data to be written. + filepath (str or Path): Path to write data. + """ + mmcv.mkdir_or_exist(osp.dirname(filepath)) + with open(filepath, 'wb') as f: + f.write(obj) + + def put_text(self, + obj: str, + filepath: Union[str, Path], + encoding: str = 'utf-8') -> None: + """Write data to a given ``filepath`` with 'w' mode. + + Note: + ``put_text`` will create a directory if the directory of + ``filepath`` does not exist. + + Args: + obj (str): Data to be written. + filepath (str or Path): Path to write data. + encoding (str): The encoding format used to open the ``filepath``. + Default: 'utf-8'. + """ + mmcv.mkdir_or_exist(osp.dirname(filepath)) + with open(filepath, 'w', encoding=encoding) as f: + f.write(obj) + + def remove(self, filepath: Union[str, Path]) -> None: + """Remove a file. + + Args: + filepath (str or Path): Path to be removed. + """ + os.remove(filepath) + + def exists(self, filepath: Union[str, Path]) -> bool: + """Check whether a file path exists. + + Args: + filepath (str or Path): Path to be checked whether exists. + + Returns: + bool: Return ``True`` if ``filepath`` exists, ``False`` otherwise. + """ + return osp.exists(filepath) + + def isdir(self, filepath: Union[str, Path]) -> bool: + """Check whether a file path is a directory. + + Args: + filepath (str or Path): Path to be checked whether it is a + directory. + + Returns: + bool: Return ``True`` if ``filepath`` points to a directory, + ``False`` otherwise. + """ + return osp.isdir(filepath) + + def isfile(self, filepath: Union[str, Path]) -> bool: + """Check whether a file path is a file. + + Args: + filepath (str or Path): Path to be checked whether it is a file. + + Returns: + bool: Return ``True`` if ``filepath`` points to a file, ``False`` + otherwise. + """ + return osp.isfile(filepath) + + def join_path(self, filepath: Union[str, Path], + *filepaths: Union[str, Path]) -> str: + """Concatenate all file paths. + + Join one or more filepath components intelligently. The return value + is the concatenation of filepath and any members of *filepaths. + + Args: + filepath (str or Path): Path to be concatenated. + + Returns: + str: The result of concatenation. + """ + return osp.join(filepath, *filepaths) + + @contextmanager + def get_local_path( + self, filepath: Union[str, Path]) -> Iterable[Union[str, Path]]: + """Only for unified API and do nothing.""" + yield filepath + + def list_dir_or_file(self, + dir_path: Union[str, Path], + list_dir: bool = True, + list_file: bool = True, + suffix: Optional[Union[str, Tuple[str]]] = None, + recursive: bool = False) -> Iterator[str]: + """Scan a directory to find the interested directories or files in + arbitrary order. + + Note: + :meth:`list_dir_or_file` returns the path relative to ``dir_path``. + + Args: + dir_path (str | Path): Path of the directory. + list_dir (bool): List the directories. Default: True. + list_file (bool): List the path of files. Default: True. + suffix (str or tuple[str], optional): File suffix + that we are interested in. Default: None. + recursive (bool): If set to True, recursively scan the + directory. Default: False. + + Yields: + Iterable[str]: A relative path to ``dir_path``. + """ + if list_dir and suffix is not None: + raise TypeError('`suffix` should be None when `list_dir` is True') + + if (suffix is not None) and not isinstance(suffix, (str, tuple)): + raise TypeError('`suffix` must be a string or tuple of strings') + + root = dir_path + + def _list_dir_or_file(dir_path, list_dir, list_file, suffix, + recursive): + for entry in os.scandir(dir_path): + if not entry.name.startswith('.') and entry.is_file(): + rel_path = osp.relpath(entry.path, root) + if (suffix is None + or rel_path.endswith(suffix)) and list_file: + yield rel_path + elif osp.isdir(entry.path): + if list_dir: + rel_dir = osp.relpath(entry.path, root) + yield rel_dir + if recursive: + yield from _list_dir_or_file(entry.path, list_dir, + list_file, suffix, + recursive) + + return _list_dir_or_file(dir_path, list_dir, list_file, suffix, + recursive) + + +class HTTPBackend(BaseStorageBackend): + """HTTP and HTTPS storage bachend.""" + + def get(self, filepath): + value_buf = urlopen(filepath).read() + return value_buf + + def get_text(self, filepath, encoding='utf-8'): + value_buf = urlopen(filepath).read() + return value_buf.decode(encoding) + + @contextmanager + def get_local_path(self, filepath: str) -> Iterable[str]: + """Download a file from ``filepath``. + + ``get_local_path`` is decorated by :meth:`contxtlib.contextmanager`. It + can be called with ``with`` statement, and when exists from the + ``with`` statement, the temporary path will be released. + + Args: + filepath (str): Download a file from ``filepath``. + + Examples: + >>> client = HTTPBackend() + >>> # After existing from the ``with`` clause, + >>> # the path will be removed + >>> with client.get_local_path('http://path/of/your/file') as path: + ... # do something here + """ + try: + f = tempfile.NamedTemporaryFile(delete=False) + f.write(self.get(filepath)) + f.close() + yield f.name + finally: + os.remove(f.name) + + +class FileClient: + """A general file client to access files in different backends. + + The client loads a file or text in a specified backend from its path + and returns it as a binary or text file. There are two ways to choose a + backend, the name of backend and the prefix of path. Although both of them + can be used to choose a storage backend, ``backend`` has a higher priority + that is if they are all set, the storage backend will be chosen by the + backend argument. If they are all `None`, the disk backend will be chosen. + Note that It can also register other backend accessor with a given name, + prefixes, and backend class. In addition, We use the singleton pattern to + avoid repeated object creation. If the arguments are the same, the same + object will be returned. + + Args: + backend (str, optional): The storage backend type. Options are "disk", + "ceph", "memcached", "lmdb", "http" and "petrel". Default: None. + prefix (str, optional): The prefix of the registered storage backend. + Options are "s3", "http", "https". Default: None. + + Examples: + >>> # only set backend + >>> file_client = FileClient(backend='petrel') + >>> # only set prefix + >>> file_client = FileClient(prefix='s3') + >>> # set both backend and prefix but use backend to choose client + >>> file_client = FileClient(backend='petrel', prefix='s3') + >>> # if the arguments are the same, the same object is returned + >>> file_client1 = FileClient(backend='petrel') + >>> file_client1 is file_client + True + + Attributes: + client (:obj:`BaseStorageBackend`): The backend object. + """ + + _backends = { + 'disk': HardDiskBackend, + 'ceph': CephBackend, + 'memcached': MemcachedBackend, + 'lmdb': LmdbBackend, + 'petrel': PetrelBackend, + 'http': HTTPBackend, + } + # This collection is used to record the overridden backends, and when a + # backend appears in the collection, the singleton pattern is disabled for + # that backend, because if the singleton pattern is used, then the object + # returned will be the backend before overwriting + _overridden_backends = set() + _prefix_to_backends = { + 's3': PetrelBackend, + 'http': HTTPBackend, + 'https': HTTPBackend, + } + _overridden_prefixes = set() + + _instances = {} + + def __new__(cls, backend=None, prefix=None, **kwargs): + if backend is None and prefix is None: + backend = 'disk' + if backend is not None and backend not in cls._backends: + raise ValueError( + f'Backend {backend} is not supported. Currently supported ones' + f' are {list(cls._backends.keys())}') + if prefix is not None and prefix not in cls._prefix_to_backends: + raise ValueError( + f'prefix {prefix} is not supported. Currently supported ones ' + f'are {list(cls._prefix_to_backends.keys())}') + + # concatenate the arguments to a unique key for determining whether + # objects with the same arguments were created + arg_key = f'{backend}:{prefix}' + for key, value in kwargs.items(): + arg_key += f':{key}:{value}' + + # if a backend was overridden, it will create a new object + if (arg_key in cls._instances + and backend not in cls._overridden_backends + and prefix not in cls._overridden_prefixes): + _instance = cls._instances[arg_key] + else: + # create a new object and put it to _instance + _instance = super().__new__(cls) + if backend is not None: + _instance.client = cls._backends[backend](**kwargs) + else: + _instance.client = cls._prefix_to_backends[prefix](**kwargs) + + cls._instances[arg_key] = _instance + + return _instance + + @property + def name(self): + return self.client.name + + @property + def allow_symlink(self): + return self.client.allow_symlink + + @staticmethod + def parse_uri_prefix(uri: Union[str, Path]) -> Optional[str]: + """Parse the prefix of a uri. + + Args: + uri (str | Path): Uri to be parsed that contains the file prefix. + + Examples: + >>> FileClient.parse_uri_prefix('s3://path/of/your/file') + 's3' + + Returns: + str | None: Return the prefix of uri if the uri contains '://' + else ``None``. + """ + assert is_filepath(uri) + uri = str(uri) + if '://' not in uri: + return None + else: + prefix, _ = uri.split('://') + # In the case of PetrelBackend, the prefix may contains the cluster + # name like clusterName:s3 + if ':' in prefix: + _, prefix = prefix.split(':') + return prefix + + @classmethod + def infer_client(cls, + file_client_args: Optional[dict] = None, + uri: Optional[Union[str, Path]] = None) -> 'FileClient': + """Infer a suitable file client based on the URI and arguments. + + Args: + file_client_args (dict, optional): Arguments to instantiate a + FileClient. Default: None. + uri (str | Path, optional): Uri to be parsed that contains the file + prefix. Default: None. + + Examples: + >>> uri = 's3://path/of/your/file' + >>> file_client = FileClient.infer_client(uri=uri) + >>> file_client_args = {'backend': 'petrel'} + >>> file_client = FileClient.infer_client(file_client_args) + + Returns: + FileClient: Instantiated FileClient object. + """ + assert file_client_args is not None or uri is not None + if file_client_args is None: + file_prefix = cls.parse_uri_prefix(uri) # type: ignore + return cls(prefix=file_prefix) + else: + return cls(**file_client_args) + + @classmethod + def _register_backend(cls, name, backend, force=False, prefixes=None): + if not isinstance(name, str): + raise TypeError('the backend name should be a string, ' + f'but got {type(name)}') + if not inspect.isclass(backend): + raise TypeError( + f'backend should be a class but got {type(backend)}') + if not issubclass(backend, BaseStorageBackend): + raise TypeError( + f'backend {backend} is not a subclass of BaseStorageBackend') + if not force and name in cls._backends: + raise KeyError( + f'{name} is already registered as a storage backend, ' + 'add "force=True" if you want to override it') + + if name in cls._backends and force: + cls._overridden_backends.add(name) + cls._backends[name] = backend + + if prefixes is not None: + if isinstance(prefixes, str): + prefixes = [prefixes] + else: + assert isinstance(prefixes, (list, tuple)) + for prefix in prefixes: + if prefix not in cls._prefix_to_backends: + cls._prefix_to_backends[prefix] = backend + elif (prefix in cls._prefix_to_backends) and force: + cls._overridden_prefixes.add(prefix) + cls._prefix_to_backends[prefix] = backend + else: + raise KeyError( + f'{prefix} is already registered as a storage backend,' + ' add "force=True" if you want to override it') + + @classmethod + def register_backend(cls, name, backend=None, force=False, prefixes=None): + """Register a backend to FileClient. + + This method can be used as a normal class method or a decorator. + + .. code-block:: python + + class NewBackend(BaseStorageBackend): + + def get(self, filepath): + return filepath + + def get_text(self, filepath): + return filepath + + FileClient.register_backend('new', NewBackend) + + or + + .. code-block:: python + + @FileClient.register_backend('new') + class NewBackend(BaseStorageBackend): + + def get(self, filepath): + return filepath + + def get_text(self, filepath): + return filepath + + Args: + name (str): The name of the registered backend. + backend (class, optional): The backend class to be registered, + which must be a subclass of :class:`BaseStorageBackend`. + When this method is used as a decorator, backend is None. + Defaults to None. + force (bool, optional): Whether to override the backend if the name + has already been registered. Defaults to False. + prefixes (str or list[str] or tuple[str], optional): The prefixes + of the registered storage backend. Default: None. + `New in version 1.3.15.` + """ + if backend is not None: + cls._register_backend( + name, backend, force=force, prefixes=prefixes) + return + + def _register(backend_cls): + cls._register_backend( + name, backend_cls, force=force, prefixes=prefixes) + return backend_cls + + return _register + + def get(self, filepath: Union[str, Path]) -> Union[bytes, memoryview]: + """Read data from a given ``filepath`` with 'rb' mode. + + Note: + There are two types of return values for ``get``, one is ``bytes`` + and the other is ``memoryview``. The advantage of using memoryview + is that you can avoid copying, and if you want to convert it to + ``bytes``, you can use ``.tobytes()``. + + Args: + filepath (str or Path): Path to read data. + + Returns: + bytes | memoryview: Expected bytes object or a memory view of the + bytes object. + """ + return self.client.get(filepath) + + def get_text(self, filepath: Union[str, Path], encoding='utf-8') -> str: + """Read data from a given ``filepath`` with 'r' mode. + + Args: + filepath (str or Path): Path to read data. + encoding (str): The encoding format used to open the ``filepath``. + Default: 'utf-8'. + + Returns: + str: Expected text reading from ``filepath``. + """ + return self.client.get_text(filepath, encoding) + + def put(self, obj: bytes, filepath: Union[str, Path]) -> None: + """Write data to a given ``filepath`` with 'wb' mode. + + Note: + ``put`` should create a directory if the directory of ``filepath`` + does not exist. + + Args: + obj (bytes): Data to be written. + filepath (str or Path): Path to write data. + """ + self.client.put(obj, filepath) + + def put_text(self, obj: str, filepath: Union[str, Path]) -> None: + """Write data to a given ``filepath`` with 'w' mode. + + Note: + ``put_text`` should create a directory if the directory of + ``filepath`` does not exist. + + Args: + obj (str): Data to be written. + filepath (str or Path): Path to write data. + encoding (str, optional): The encoding format used to open the + `filepath`. Default: 'utf-8'. + """ + self.client.put_text(obj, filepath) + + def remove(self, filepath: Union[str, Path]) -> None: + """Remove a file. + + Args: + filepath (str, Path): Path to be removed. + """ + self.client.remove(filepath) + + def exists(self, filepath: Union[str, Path]) -> bool: + """Check whether a file path exists. + + Args: + filepath (str or Path): Path to be checked whether exists. + + Returns: + bool: Return ``True`` if ``filepath`` exists, ``False`` otherwise. + """ + return self.client.exists(filepath) + + def isdir(self, filepath: Union[str, Path]) -> bool: + """Check whether a file path is a directory. + + Args: + filepath (str or Path): Path to be checked whether it is a + directory. + + Returns: + bool: Return ``True`` if ``filepath`` points to a directory, + ``False`` otherwise. + """ + return self.client.isdir(filepath) + + def isfile(self, filepath: Union[str, Path]) -> bool: + """Check whether a file path is a file. + + Args: + filepath (str or Path): Path to be checked whether it is a file. + + Returns: + bool: Return ``True`` if ``filepath`` points to a file, ``False`` + otherwise. + """ + return self.client.isfile(filepath) + + def join_path(self, filepath: Union[str, Path], + *filepaths: Union[str, Path]) -> str: + """Concatenate all file paths. + + Join one or more filepath components intelligently. The return value + is the concatenation of filepath and any members of *filepaths. + + Args: + filepath (str or Path): Path to be concatenated. + + Returns: + str: The result of concatenation. + """ + return self.client.join_path(filepath, *filepaths) + + @contextmanager + def get_local_path(self, filepath: Union[str, Path]) -> Iterable[str]: + """Download data from ``filepath`` and write the data to local path. + + ``get_local_path`` is decorated by :meth:`contxtlib.contextmanager`. It + can be called with ``with`` statement, and when exists from the + ``with`` statement, the temporary path will be released. + + Note: + If the ``filepath`` is a local path, just return itself. + + .. warning:: + ``get_local_path`` is an experimental interface that may change in + the future. + + Args: + filepath (str or Path): Path to be read data. + + Examples: + >>> file_client = FileClient(prefix='s3') + >>> with file_client.get_local_path('s3://bucket/abc.jpg') as path: + ... # do something here + + Yields: + Iterable[str]: Only yield one path. + """ + with self.client.get_local_path(str(filepath)) as local_path: + yield local_path + + def list_dir_or_file(self, + dir_path: Union[str, Path], + list_dir: bool = True, + list_file: bool = True, + suffix: Optional[Union[str, Tuple[str]]] = None, + recursive: bool = False) -> Iterator[str]: + """Scan a directory to find the interested directories or files in + arbitrary order. + + Note: + :meth:`list_dir_or_file` returns the path relative to ``dir_path``. + + Args: + dir_path (str | Path): Path of the directory. + list_dir (bool): List the directories. Default: True. + list_file (bool): List the path of files. Default: True. + suffix (str or tuple[str], optional): File suffix + that we are interested in. Default: None. + recursive (bool): If set to True, recursively scan the + directory. Default: False. + + Yields: + Iterable[str]: A relative path to ``dir_path``. + """ + yield from self.client.list_dir_or_file(dir_path, list_dir, list_file, + suffix, recursive) diff --git a/annotator/uniformer/mmcv/fileio/handlers/__init__.py b/annotator/uniformer/mmcv/fileio/handlers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..aa24d91972837b8756b225f4879bac20436eb72a --- /dev/null +++ b/annotator/uniformer/mmcv/fileio/handlers/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .base import BaseFileHandler +from .json_handler import JsonHandler +from .pickle_handler import PickleHandler +from .yaml_handler import YamlHandler + +__all__ = ['BaseFileHandler', 'JsonHandler', 'PickleHandler', 'YamlHandler'] diff --git a/annotator/uniformer/mmcv/fileio/handlers/base.py b/annotator/uniformer/mmcv/fileio/handlers/base.py new file mode 100644 index 0000000000000000000000000000000000000000..288878bc57282fbb2f12b32290152ca8e9d3cab0 --- /dev/null +++ b/annotator/uniformer/mmcv/fileio/handlers/base.py @@ -0,0 +1,30 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from abc import ABCMeta, abstractmethod + + +class BaseFileHandler(metaclass=ABCMeta): + # `str_like` is a flag to indicate whether the type of file object is + # str-like object or bytes-like object. Pickle only processes bytes-like + # objects but json only processes str-like object. If it is str-like + # object, `StringIO` will be used to process the buffer. + str_like = True + + @abstractmethod + def load_from_fileobj(self, file, **kwargs): + pass + + @abstractmethod + def dump_to_fileobj(self, obj, file, **kwargs): + pass + + @abstractmethod + def dump_to_str(self, obj, **kwargs): + pass + + def load_from_path(self, filepath, mode='r', **kwargs): + with open(filepath, mode) as f: + return self.load_from_fileobj(f, **kwargs) + + def dump_to_path(self, obj, filepath, mode='w', **kwargs): + with open(filepath, mode) as f: + self.dump_to_fileobj(obj, f, **kwargs) diff --git a/annotator/uniformer/mmcv/fileio/handlers/json_handler.py b/annotator/uniformer/mmcv/fileio/handlers/json_handler.py new file mode 100644 index 0000000000000000000000000000000000000000..18d4f15f74139d20adff18b20be5529c592a66b6 --- /dev/null +++ b/annotator/uniformer/mmcv/fileio/handlers/json_handler.py @@ -0,0 +1,36 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import json + +import numpy as np + +from .base import BaseFileHandler + + +def set_default(obj): + """Set default json values for non-serializable values. + + It helps convert ``set``, ``range`` and ``np.ndarray`` data types to list. + It also converts ``np.generic`` (including ``np.int32``, ``np.float32``, + etc.) into plain numbers of plain python built-in types. + """ + if isinstance(obj, (set, range)): + return list(obj) + elif isinstance(obj, np.ndarray): + return obj.tolist() + elif isinstance(obj, np.generic): + return obj.item() + raise TypeError(f'{type(obj)} is unsupported for json dump') + + +class JsonHandler(BaseFileHandler): + + def load_from_fileobj(self, file): + return json.load(file) + + def dump_to_fileobj(self, obj, file, **kwargs): + kwargs.setdefault('default', set_default) + json.dump(obj, file, **kwargs) + + def dump_to_str(self, obj, **kwargs): + kwargs.setdefault('default', set_default) + return json.dumps(obj, **kwargs) diff --git a/annotator/uniformer/mmcv/fileio/handlers/pickle_handler.py b/annotator/uniformer/mmcv/fileio/handlers/pickle_handler.py new file mode 100644 index 0000000000000000000000000000000000000000..b37c79bed4ef9fd8913715e62dbe3fc5cafdc3aa --- /dev/null +++ b/annotator/uniformer/mmcv/fileio/handlers/pickle_handler.py @@ -0,0 +1,28 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import pickle + +from .base import BaseFileHandler + + +class PickleHandler(BaseFileHandler): + + str_like = False + + def load_from_fileobj(self, file, **kwargs): + return pickle.load(file, **kwargs) + + def load_from_path(self, filepath, **kwargs): + return super(PickleHandler, self).load_from_path( + filepath, mode='rb', **kwargs) + + def dump_to_str(self, obj, **kwargs): + kwargs.setdefault('protocol', 2) + return pickle.dumps(obj, **kwargs) + + def dump_to_fileobj(self, obj, file, **kwargs): + kwargs.setdefault('protocol', 2) + pickle.dump(obj, file, **kwargs) + + def dump_to_path(self, obj, filepath, **kwargs): + super(PickleHandler, self).dump_to_path( + obj, filepath, mode='wb', **kwargs) diff --git a/annotator/uniformer/mmcv/fileio/handlers/yaml_handler.py b/annotator/uniformer/mmcv/fileio/handlers/yaml_handler.py new file mode 100644 index 0000000000000000000000000000000000000000..c5aa2eea1e8c76f8baf753d1c8c959dee665e543 --- /dev/null +++ b/annotator/uniformer/mmcv/fileio/handlers/yaml_handler.py @@ -0,0 +1,24 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import yaml + +try: + from yaml import CLoader as Loader, CDumper as Dumper +except ImportError: + from yaml import Loader, Dumper + +from .base import BaseFileHandler # isort:skip + + +class YamlHandler(BaseFileHandler): + + def load_from_fileobj(self, file, **kwargs): + kwargs.setdefault('Loader', Loader) + return yaml.load(file, **kwargs) + + def dump_to_fileobj(self, obj, file, **kwargs): + kwargs.setdefault('Dumper', Dumper) + yaml.dump(obj, file, **kwargs) + + def dump_to_str(self, obj, **kwargs): + kwargs.setdefault('Dumper', Dumper) + return yaml.dump(obj, **kwargs) diff --git a/annotator/uniformer/mmcv/fileio/io.py b/annotator/uniformer/mmcv/fileio/io.py new file mode 100644 index 0000000000000000000000000000000000000000..aaefde58aa3ea5b58f86249ce7e1c40c186eb8dd --- /dev/null +++ b/annotator/uniformer/mmcv/fileio/io.py @@ -0,0 +1,151 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from io import BytesIO, StringIO +from pathlib import Path + +from ..utils import is_list_of, is_str +from .file_client import FileClient +from .handlers import BaseFileHandler, JsonHandler, PickleHandler, YamlHandler + +file_handlers = { + 'json': JsonHandler(), + 'yaml': YamlHandler(), + 'yml': YamlHandler(), + 'pickle': PickleHandler(), + 'pkl': PickleHandler() +} + + +def load(file, file_format=None, file_client_args=None, **kwargs): + """Load data from json/yaml/pickle files. + + This method provides a unified api for loading data from serialized files. + + Note: + In v1.3.16 and later, ``load`` supports loading data from serialized + files those can be storaged in different backends. + + Args: + file (str or :obj:`Path` or file-like object): Filename or a file-like + object. + file_format (str, optional): If not specified, the file format will be + inferred from the file extension, otherwise use the specified one. + Currently supported formats include "json", "yaml/yml" and + "pickle/pkl". + file_client_args (dict, optional): Arguments to instantiate a + FileClient. See :class:`mmcv.fileio.FileClient` for details. + Default: None. + + Examples: + >>> load('/path/of/your/file') # file is storaged in disk + >>> load('https://path/of/your/file') # file is storaged in Internet + >>> load('s3://path/of/your/file') # file is storaged in petrel + + Returns: + The content from the file. + """ + if isinstance(file, Path): + file = str(file) + if file_format is None and is_str(file): + file_format = file.split('.')[-1] + if file_format not in file_handlers: + raise TypeError(f'Unsupported format: {file_format}') + + handler = file_handlers[file_format] + if is_str(file): + file_client = FileClient.infer_client(file_client_args, file) + if handler.str_like: + with StringIO(file_client.get_text(file)) as f: + obj = handler.load_from_fileobj(f, **kwargs) + else: + with BytesIO(file_client.get(file)) as f: + obj = handler.load_from_fileobj(f, **kwargs) + elif hasattr(file, 'read'): + obj = handler.load_from_fileobj(file, **kwargs) + else: + raise TypeError('"file" must be a filepath str or a file-object') + return obj + + +def dump(obj, file=None, file_format=None, file_client_args=None, **kwargs): + """Dump data to json/yaml/pickle strings or files. + + This method provides a unified api for dumping data as strings or to files, + and also supports custom arguments for each file format. + + Note: + In v1.3.16 and later, ``dump`` supports dumping data as strings or to + files which is saved to different backends. + + Args: + obj (any): The python object to be dumped. + file (str or :obj:`Path` or file-like object, optional): If not + specified, then the object is dumped to a str, otherwise to a file + specified by the filename or file-like object. + file_format (str, optional): Same as :func:`load`. + file_client_args (dict, optional): Arguments to instantiate a + FileClient. See :class:`mmcv.fileio.FileClient` for details. + Default: None. + + Examples: + >>> dump('hello world', '/path/of/your/file') # disk + >>> dump('hello world', 's3://path/of/your/file') # ceph or petrel + + Returns: + bool: True for success, False otherwise. + """ + if isinstance(file, Path): + file = str(file) + if file_format is None: + if is_str(file): + file_format = file.split('.')[-1] + elif file is None: + raise ValueError( + 'file_format must be specified since file is None') + if file_format not in file_handlers: + raise TypeError(f'Unsupported format: {file_format}') + + handler = file_handlers[file_format] + if file is None: + return handler.dump_to_str(obj, **kwargs) + elif is_str(file): + file_client = FileClient.infer_client(file_client_args, file) + if handler.str_like: + with StringIO() as f: + handler.dump_to_fileobj(obj, f, **kwargs) + file_client.put_text(f.getvalue(), file) + else: + with BytesIO() as f: + handler.dump_to_fileobj(obj, f, **kwargs) + file_client.put(f.getvalue(), file) + elif hasattr(file, 'write'): + handler.dump_to_fileobj(obj, file, **kwargs) + else: + raise TypeError('"file" must be a filename str or a file-object') + + +def _register_handler(handler, file_formats): + """Register a handler for some file extensions. + + Args: + handler (:obj:`BaseFileHandler`): Handler to be registered. + file_formats (str or list[str]): File formats to be handled by this + handler. + """ + if not isinstance(handler, BaseFileHandler): + raise TypeError( + f'handler must be a child of BaseFileHandler, not {type(handler)}') + if isinstance(file_formats, str): + file_formats = [file_formats] + if not is_list_of(file_formats, str): + raise TypeError('file_formats must be a str or a list of str') + for ext in file_formats: + file_handlers[ext] = handler + + +def register_handler(file_formats, **kwargs): + + def wrap(cls): + _register_handler(cls(**kwargs), file_formats) + return cls + + return wrap diff --git a/annotator/uniformer/mmcv/fileio/parse.py b/annotator/uniformer/mmcv/fileio/parse.py new file mode 100644 index 0000000000000000000000000000000000000000..f60f0d611b8d75692221d0edd7dc993b0a6445c9 --- /dev/null +++ b/annotator/uniformer/mmcv/fileio/parse.py @@ -0,0 +1,97 @@ +# Copyright (c) OpenMMLab. All rights reserved. + +from io import StringIO + +from .file_client import FileClient + + +def list_from_file(filename, + prefix='', + offset=0, + max_num=0, + encoding='utf-8', + file_client_args=None): + """Load a text file and parse the content as a list of strings. + + Note: + In v1.3.16 and later, ``list_from_file`` supports loading a text file + which can be storaged in different backends and parsing the content as + a list for strings. + + Args: + filename (str): Filename. + prefix (str): The prefix to be inserted to the beginning of each item. + offset (int): The offset of lines. + max_num (int): The maximum number of lines to be read, + zeros and negatives mean no limitation. + encoding (str): Encoding used to open the file. Default utf-8. + file_client_args (dict, optional): Arguments to instantiate a + FileClient. See :class:`mmcv.fileio.FileClient` for details. + Default: None. + + Examples: + >>> list_from_file('/path/of/your/file') # disk + ['hello', 'world'] + >>> list_from_file('s3://path/of/your/file') # ceph or petrel + ['hello', 'world'] + + Returns: + list[str]: A list of strings. + """ + cnt = 0 + item_list = [] + file_client = FileClient.infer_client(file_client_args, filename) + with StringIO(file_client.get_text(filename, encoding)) as f: + for _ in range(offset): + f.readline() + for line in f: + if 0 < max_num <= cnt: + break + item_list.append(prefix + line.rstrip('\n\r')) + cnt += 1 + return item_list + + +def dict_from_file(filename, + key_type=str, + encoding='utf-8', + file_client_args=None): + """Load a text file and parse the content as a dict. + + Each line of the text file will be two or more columns split by + whitespaces or tabs. The first column will be parsed as dict keys, and + the following columns will be parsed as dict values. + + Note: + In v1.3.16 and later, ``dict_from_file`` supports loading a text file + which can be storaged in different backends and parsing the content as + a dict. + + Args: + filename(str): Filename. + key_type(type): Type of the dict keys. str is user by default and + type conversion will be performed if specified. + encoding (str): Encoding used to open the file. Default utf-8. + file_client_args (dict, optional): Arguments to instantiate a + FileClient. See :class:`mmcv.fileio.FileClient` for details. + Default: None. + + Examples: + >>> dict_from_file('/path/of/your/file') # disk + {'key1': 'value1', 'key2': 'value2'} + >>> dict_from_file('s3://path/of/your/file') # ceph or petrel + {'key1': 'value1', 'key2': 'value2'} + + Returns: + dict: The parsed contents. + """ + mapping = {} + file_client = FileClient.infer_client(file_client_args, filename) + with StringIO(file_client.get_text(filename, encoding)) as f: + for line in f: + items = line.rstrip('\n').split() + assert len(items) >= 2 + key = key_type(items[0]) + val = items[1:] if len(items) > 2 else items[1] + mapping[key] = val + return mapping diff --git a/annotator/uniformer/mmcv/image/__init__.py b/annotator/uniformer/mmcv/image/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d0051d609d3de4e7562e3fe638335c66617c4d91 --- /dev/null +++ b/annotator/uniformer/mmcv/image/__init__.py @@ -0,0 +1,28 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .colorspace import (bgr2gray, bgr2hls, bgr2hsv, bgr2rgb, bgr2ycbcr, + gray2bgr, gray2rgb, hls2bgr, hsv2bgr, imconvert, + rgb2bgr, rgb2gray, rgb2ycbcr, ycbcr2bgr, ycbcr2rgb) +from .geometric import (cutout, imcrop, imflip, imflip_, impad, + impad_to_multiple, imrescale, imresize, imresize_like, + imresize_to_multiple, imrotate, imshear, imtranslate, + rescale_size) +from .io import imfrombytes, imread, imwrite, supported_backends, use_backend +from .misc import tensor2imgs +from .photometric import (adjust_brightness, adjust_color, adjust_contrast, + adjust_lighting, adjust_sharpness, auto_contrast, + clahe, imdenormalize, imequalize, iminvert, + imnormalize, imnormalize_, lut_transform, posterize, + solarize) + +__all__ = [ + 'bgr2gray', 'bgr2hls', 'bgr2hsv', 'bgr2rgb', 'gray2bgr', 'gray2rgb', + 'hls2bgr', 'hsv2bgr', 'imconvert', 'rgb2bgr', 'rgb2gray', 'imrescale', + 'imresize', 'imresize_like', 'imresize_to_multiple', 'rescale_size', + 'imcrop', 'imflip', 'imflip_', 'impad', 'impad_to_multiple', 'imrotate', + 'imfrombytes', 'imread', 'imwrite', 'supported_backends', 'use_backend', + 'imdenormalize', 'imnormalize', 'imnormalize_', 'iminvert', 'posterize', + 'solarize', 'rgb2ycbcr', 'bgr2ycbcr', 'ycbcr2rgb', 'ycbcr2bgr', + 'tensor2imgs', 'imshear', 'imtranslate', 'adjust_color', 'imequalize', + 'adjust_brightness', 'adjust_contrast', 'lut_transform', 'clahe', + 'adjust_sharpness', 'auto_contrast', 'cutout', 'adjust_lighting' +] diff --git a/annotator/uniformer/mmcv/image/colorspace.py b/annotator/uniformer/mmcv/image/colorspace.py new file mode 100644 index 0000000000000000000000000000000000000000..814533952fdfda23d67cb6a3073692d8c1156add --- /dev/null +++ b/annotator/uniformer/mmcv/image/colorspace.py @@ -0,0 +1,306 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import cv2 +import numpy as np + + +def imconvert(img, src, dst): + """Convert an image from the src colorspace to dst colorspace. + + Args: + img (ndarray): The input image. + src (str): The source colorspace, e.g., 'rgb', 'hsv'. + dst (str): The destination colorspace, e.g., 'rgb', 'hsv'. + + Returns: + ndarray: The converted image. + """ + code = getattr(cv2, f'COLOR_{src.upper()}2{dst.upper()}') + out_img = cv2.cvtColor(img, code) + return out_img + + +def bgr2gray(img, keepdim=False): + """Convert a BGR image to grayscale image. + + Args: + img (ndarray): The input image. + keepdim (bool): If False (by default), then return the grayscale image + with 2 dims, otherwise 3 dims. + + Returns: + ndarray: The converted grayscale image. + """ + out_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + if keepdim: + out_img = out_img[..., None] + return out_img + + +def rgb2gray(img, keepdim=False): + """Convert a RGB image to grayscale image. + + Args: + img (ndarray): The input image. + keepdim (bool): If False (by default), then return the grayscale image + with 2 dims, otherwise 3 dims. + + Returns: + ndarray: The converted grayscale image. + """ + out_img = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) + if keepdim: + out_img = out_img[..., None] + return out_img + + +def gray2bgr(img): + """Convert a grayscale image to BGR image. + + Args: + img (ndarray): The input image. + + Returns: + ndarray: The converted BGR image. + """ + img = img[..., None] if img.ndim == 2 else img + out_img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) + return out_img + + +def gray2rgb(img): + """Convert a grayscale image to RGB image. + + Args: + img (ndarray): The input image. + + Returns: + ndarray: The converted RGB image. + """ + img = img[..., None] if img.ndim == 2 else img + out_img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB) + return out_img + + +def _convert_input_type_range(img): + """Convert the type and range of the input image. + + It converts the input image to np.float32 type and range of [0, 1]. + It is mainly used for pre-processing the input image in colorspace + conversion functions such as rgb2ycbcr and ycbcr2rgb. + + Args: + img (ndarray): The input image. It accepts: + 1. np.uint8 type with range [0, 255]; + 2. np.float32 type with range [0, 1]. + + Returns: + (ndarray): The converted image with type of np.float32 and range of + [0, 1]. + """ + img_type = img.dtype + img = img.astype(np.float32) + if img_type == np.float32: + pass + elif img_type == np.uint8: + img /= 255. + else: + raise TypeError('The img type should be np.float32 or np.uint8, ' + f'but got {img_type}') + return img + + +def _convert_output_type_range(img, dst_type): + """Convert the type and range of the image according to dst_type. + + It converts the image to desired type and range. If `dst_type` is np.uint8, + images will be converted to np.uint8 type with range [0, 255]. If + `dst_type` is np.float32, it converts the image to np.float32 type with + range [0, 1]. + It is mainly used for post-processing images in colorspace conversion + functions such as rgb2ycbcr and ycbcr2rgb. + + Args: + img (ndarray): The image to be converted with np.float32 type and + range [0, 255]. + dst_type (np.uint8 | np.float32): If dst_type is np.uint8, it + converts the image to np.uint8 type with range [0, 255]. If + dst_type is np.float32, it converts the image to np.float32 type + with range [0, 1]. + + Returns: + (ndarray): The converted image with desired type and range. + """ + if dst_type not in (np.uint8, np.float32): + raise TypeError('The dst_type should be np.float32 or np.uint8, ' + f'but got {dst_type}') + if dst_type == np.uint8: + img = img.round() + else: + img /= 255. + return img.astype(dst_type) + + +def rgb2ycbcr(img, y_only=False): + """Convert a RGB image to YCbCr image. + + This function produces the same results as Matlab's `rgb2ycbcr` function. + It implements the ITU-R BT.601 conversion for standard-definition + television. See more details in + https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion. + + It differs from a similar function in cv2.cvtColor: `RGB <-> YCrCb`. + In OpenCV, it implements a JPEG conversion. See more details in + https://en.wikipedia.org/wiki/YCbCr#JPEG_conversion. + + Args: + img (ndarray): The input image. It accepts: + 1. np.uint8 type with range [0, 255]; + 2. np.float32 type with range [0, 1]. + y_only (bool): Whether to only return Y channel. Default: False. + + Returns: + ndarray: The converted YCbCr image. The output image has the same type + and range as input image. + """ + img_type = img.dtype + img = _convert_input_type_range(img) + if y_only: + out_img = np.dot(img, [65.481, 128.553, 24.966]) + 16.0 + else: + out_img = np.matmul( + img, [[65.481, -37.797, 112.0], [128.553, -74.203, -93.786], + [24.966, 112.0, -18.214]]) + [16, 128, 128] + out_img = _convert_output_type_range(out_img, img_type) + return out_img + + +def bgr2ycbcr(img, y_only=False): + """Convert a BGR image to YCbCr image. + + The bgr version of rgb2ycbcr. + It implements the ITU-R BT.601 conversion for standard-definition + television. See more details in + https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion. + + It differs from a similar function in cv2.cvtColor: `BGR <-> YCrCb`. + In OpenCV, it implements a JPEG conversion. See more details in + https://en.wikipedia.org/wiki/YCbCr#JPEG_conversion. + + Args: + img (ndarray): The input image. It accepts: + 1. np.uint8 type with range [0, 255]; + 2. np.float32 type with range [0, 1]. + y_only (bool): Whether to only return Y channel. Default: False. + + Returns: + ndarray: The converted YCbCr image. The output image has the same type + and range as input image. + """ + img_type = img.dtype + img = _convert_input_type_range(img) + if y_only: + out_img = np.dot(img, [24.966, 128.553, 65.481]) + 16.0 + else: + out_img = np.matmul( + img, [[24.966, 112.0, -18.214], [128.553, -74.203, -93.786], + [65.481, -37.797, 112.0]]) + [16, 128, 128] + out_img = _convert_output_type_range(out_img, img_type) + return out_img + + +def ycbcr2rgb(img): + """Convert a YCbCr image to RGB image. + + This function produces the same results as Matlab's ycbcr2rgb function. + It implements the ITU-R BT.601 conversion for standard-definition + television. See more details in + https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion. + + It differs from a similar function in cv2.cvtColor: `YCrCb <-> RGB`. + In OpenCV, it implements a JPEG conversion. See more details in + https://en.wikipedia.org/wiki/YCbCr#JPEG_conversion. + + Args: + img (ndarray): The input image. It accepts: + 1. np.uint8 type with range [0, 255]; + 2. np.float32 type with range [0, 1]. + + Returns: + ndarray: The converted RGB image. The output image has the same type + and range as input image. + """ + img_type = img.dtype + img = _convert_input_type_range(img) * 255 + out_img = np.matmul(img, [[0.00456621, 0.00456621, 0.00456621], + [0, -0.00153632, 0.00791071], + [0.00625893, -0.00318811, 0]]) * 255.0 + [ + -222.921, 135.576, -276.836 + ] + out_img = _convert_output_type_range(out_img, img_type) + return out_img + + +def ycbcr2bgr(img): + """Convert a YCbCr image to BGR image. + + The bgr version of ycbcr2rgb. + It implements the ITU-R BT.601 conversion for standard-definition + television. See more details in + https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion. + + It differs from a similar function in cv2.cvtColor: `YCrCb <-> BGR`. + In OpenCV, it implements a JPEG conversion. See more details in + https://en.wikipedia.org/wiki/YCbCr#JPEG_conversion. + + Args: + img (ndarray): The input image. It accepts: + 1. np.uint8 type with range [0, 255]; + 2. np.float32 type with range [0, 1]. + + Returns: + ndarray: The converted BGR image. The output image has the same type + and range as input image. + """ + img_type = img.dtype + img = _convert_input_type_range(img) * 255 + out_img = np.matmul(img, [[0.00456621, 0.00456621, 0.00456621], + [0.00791071, -0.00153632, 0], + [0, -0.00318811, 0.00625893]]) * 255.0 + [ + -276.836, 135.576, -222.921 + ] + out_img = _convert_output_type_range(out_img, img_type) + return out_img + + +def convert_color_factory(src, dst): + + code = getattr(cv2, f'COLOR_{src.upper()}2{dst.upper()}') + + def convert_color(img): + out_img = cv2.cvtColor(img, code) + return out_img + + convert_color.__doc__ = f"""Convert a {src.upper()} image to {dst.upper()} + image. + + Args: + img (ndarray or str): The input image. + + Returns: + ndarray: The converted {dst.upper()} image. + """ + + return convert_color + + +bgr2rgb = convert_color_factory('bgr', 'rgb') + +rgb2bgr = convert_color_factory('rgb', 'bgr') + +bgr2hsv = convert_color_factory('bgr', 'hsv') + +hsv2bgr = convert_color_factory('hsv', 'bgr') + +bgr2hls = convert_color_factory('bgr', 'hls') + +hls2bgr = convert_color_factory('hls', 'bgr') diff --git a/annotator/uniformer/mmcv/image/geometric.py b/annotator/uniformer/mmcv/image/geometric.py new file mode 100644 index 0000000000000000000000000000000000000000..cf97c201cb4e43796c911919d03fb26a07ed817d --- /dev/null +++ b/annotator/uniformer/mmcv/image/geometric.py @@ -0,0 +1,728 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numbers + +import cv2 +import numpy as np + +from ..utils import to_2tuple +from .io import imread_backend + +try: + from PIL import Image +except ImportError: + Image = None + + +def _scale_size(size, scale): + """Rescale a size by a ratio. + + Args: + size (tuple[int]): (w, h). + scale (float | tuple(float)): Scaling factor. + + Returns: + tuple[int]: scaled size. + """ + if isinstance(scale, (float, int)): + scale = (scale, scale) + w, h = size + return int(w * float(scale[0]) + 0.5), int(h * float(scale[1]) + 0.5) + + +cv2_interp_codes = { + 'nearest': cv2.INTER_NEAREST, + 'bilinear': cv2.INTER_LINEAR, + 'bicubic': cv2.INTER_CUBIC, + 'area': cv2.INTER_AREA, + 'lanczos': cv2.INTER_LANCZOS4 +} + +if Image is not None: + pillow_interp_codes = { + 'nearest': Image.NEAREST, + 'bilinear': Image.BILINEAR, + 'bicubic': Image.BICUBIC, + 'box': Image.BOX, + 'lanczos': Image.LANCZOS, + 'hamming': Image.HAMMING + } + + +def imresize(img, + size, + return_scale=False, + interpolation='bilinear', + out=None, + backend=None): + """Resize image to a given size. + + Args: + img (ndarray): The input image. + size (tuple[int]): Target size (w, h). + return_scale (bool): Whether to return `w_scale` and `h_scale`. + interpolation (str): Interpolation method, accepted values are + "nearest", "bilinear", "bicubic", "area", "lanczos" for 'cv2' + backend, "nearest", "bilinear" for 'pillow' backend. + out (ndarray): The output destination. + backend (str | None): The image resize backend type. Options are `cv2`, + `pillow`, `None`. If backend is None, the global imread_backend + specified by ``mmcv.use_backend()`` will be used. Default: None. + + Returns: + tuple | ndarray: (`resized_img`, `w_scale`, `h_scale`) or + `resized_img`. + """ + h, w = img.shape[:2] + if backend is None: + backend = imread_backend + if backend not in ['cv2', 'pillow']: + raise ValueError(f'backend: {backend} is not supported for resize.' + f"Supported backends are 'cv2', 'pillow'") + + if backend == 'pillow': + assert img.dtype == np.uint8, 'Pillow backend only support uint8 type' + pil_image = Image.fromarray(img) + pil_image = pil_image.resize(size, pillow_interp_codes[interpolation]) + resized_img = np.array(pil_image) + else: + resized_img = cv2.resize( + img, size, dst=out, interpolation=cv2_interp_codes[interpolation]) + if not return_scale: + return resized_img + else: + w_scale = size[0] / w + h_scale = size[1] / h + return resized_img, w_scale, h_scale + + +def imresize_to_multiple(img, + divisor, + size=None, + scale_factor=None, + keep_ratio=False, + return_scale=False, + interpolation='bilinear', + out=None, + backend=None): + """Resize image according to a given size or scale factor and then rounds + up the the resized or rescaled image size to the nearest value that can be + divided by the divisor. + + Args: + img (ndarray): The input image. + divisor (int | tuple): Resized image size will be a multiple of + divisor. If divisor is a tuple, divisor should be + (w_divisor, h_divisor). + size (None | int | tuple[int]): Target size (w, h). Default: None. + scale_factor (None | float | tuple[float]): Multiplier for spatial + size. Should match input size if it is a tuple and the 2D style is + (w_scale_factor, h_scale_factor). Default: None. + keep_ratio (bool): Whether to keep the aspect ratio when resizing the + image. Default: False. + return_scale (bool): Whether to return `w_scale` and `h_scale`. + interpolation (str): Interpolation method, accepted values are + "nearest", "bilinear", "bicubic", "area", "lanczos" for 'cv2' + backend, "nearest", "bilinear" for 'pillow' backend. + out (ndarray): The output destination. + backend (str | None): The image resize backend type. Options are `cv2`, + `pillow`, `None`. If backend is None, the global imread_backend + specified by ``mmcv.use_backend()`` will be used. Default: None. + + Returns: + tuple | ndarray: (`resized_img`, `w_scale`, `h_scale`) or + `resized_img`. + """ + h, w = img.shape[:2] + if size is not None and scale_factor is not None: + raise ValueError('only one of size or scale_factor should be defined') + elif size is None and scale_factor is None: + raise ValueError('one of size or scale_factor should be defined') + elif size is not None: + size = to_2tuple(size) + if keep_ratio: + size = rescale_size((w, h), size, return_scale=False) + else: + size = _scale_size((w, h), scale_factor) + + divisor = to_2tuple(divisor) + size = tuple([int(np.ceil(s / d)) * d for s, d in zip(size, divisor)]) + resized_img, w_scale, h_scale = imresize( + img, + size, + return_scale=True, + interpolation=interpolation, + out=out, + backend=backend) + if return_scale: + return resized_img, w_scale, h_scale + else: + return resized_img + + +def imresize_like(img, + dst_img, + return_scale=False, + interpolation='bilinear', + backend=None): + """Resize image to the same size of a given image. + + Args: + img (ndarray): The input image. + dst_img (ndarray): The target image. + return_scale (bool): Whether to return `w_scale` and `h_scale`. + interpolation (str): Same as :func:`resize`. + backend (str | None): Same as :func:`resize`. + + Returns: + tuple or ndarray: (`resized_img`, `w_scale`, `h_scale`) or + `resized_img`. + """ + h, w = dst_img.shape[:2] + return imresize(img, (w, h), return_scale, interpolation, backend=backend) + + +def rescale_size(old_size, scale, return_scale=False): + """Calculate the new size to be rescaled to. + + Args: + old_size (tuple[int]): The old size (w, h) of image. + scale (float | tuple[int]): The scaling factor or maximum size. + If it is a float number, then the image will be rescaled by this + factor, else if it is a tuple of 2 integers, then the image will + be rescaled as large as possible within the scale. + return_scale (bool): Whether to return the scaling factor besides the + rescaled image size. + + Returns: + tuple[int]: The new rescaled image size. + """ + w, h = old_size + if isinstance(scale, (float, int)): + if scale <= 0: + raise ValueError(f'Invalid scale {scale}, must be positive.') + scale_factor = scale + elif isinstance(scale, tuple): + max_long_edge = max(scale) + max_short_edge = min(scale) + scale_factor = min(max_long_edge / max(h, w), + max_short_edge / min(h, w)) + else: + raise TypeError( + f'Scale must be a number or tuple of int, but got {type(scale)}') + + new_size = _scale_size((w, h), scale_factor) + + if return_scale: + return new_size, scale_factor + else: + return new_size + + +def imrescale(img, + scale, + return_scale=False, + interpolation='bilinear', + backend=None): + """Resize image while keeping the aspect ratio. + + Args: + img (ndarray): The input image. + scale (float | tuple[int]): The scaling factor or maximum size. + If it is a float number, then the image will be rescaled by this + factor, else if it is a tuple of 2 integers, then the image will + be rescaled as large as possible within the scale. + return_scale (bool): Whether to return the scaling factor besides the + rescaled image. + interpolation (str): Same as :func:`resize`. + backend (str | None): Same as :func:`resize`. + + Returns: + ndarray: The rescaled image. + """ + h, w = img.shape[:2] + new_size, scale_factor = rescale_size((w, h), scale, return_scale=True) + rescaled_img = imresize( + img, new_size, interpolation=interpolation, backend=backend) + if return_scale: + return rescaled_img, scale_factor + else: + return rescaled_img + + +def imflip(img, direction='horizontal'): + """Flip an image horizontally or vertically. + + Args: + img (ndarray): Image to be flipped. + direction (str): The flip direction, either "horizontal" or + "vertical" or "diagonal". + + Returns: + ndarray: The flipped image. + """ + assert direction in ['horizontal', 'vertical', 'diagonal'] + if direction == 'horizontal': + return np.flip(img, axis=1) + elif direction == 'vertical': + return np.flip(img, axis=0) + else: + return np.flip(img, axis=(0, 1)) + + +def imflip_(img, direction='horizontal'): + """Inplace flip an image horizontally or vertically. + + Args: + img (ndarray): Image to be flipped. + direction (str): The flip direction, either "horizontal" or + "vertical" or "diagonal". + + Returns: + ndarray: The flipped image (inplace). + """ + assert direction in ['horizontal', 'vertical', 'diagonal'] + if direction == 'horizontal': + return cv2.flip(img, 1, img) + elif direction == 'vertical': + return cv2.flip(img, 0, img) + else: + return cv2.flip(img, -1, img) + + +def imrotate(img, + angle, + center=None, + scale=1.0, + border_value=0, + interpolation='bilinear', + auto_bound=False): + """Rotate an image. + + Args: + img (ndarray): Image to be rotated. + angle (float): Rotation angle in degrees, positive values mean + clockwise rotation. + center (tuple[float], optional): Center point (w, h) of the rotation in + the source image. If not specified, the center of the image will be + used. + scale (float): Isotropic scale factor. + border_value (int): Border value. + interpolation (str): Same as :func:`resize`. + auto_bound (bool): Whether to adjust the image size to cover the whole + rotated image. + + Returns: + ndarray: The rotated image. + """ + if center is not None and auto_bound: + raise ValueError('`auto_bound` conflicts with `center`') + h, w = img.shape[:2] + if center is None: + center = ((w - 1) * 0.5, (h - 1) * 0.5) + assert isinstance(center, tuple) + + matrix = cv2.getRotationMatrix2D(center, -angle, scale) + if auto_bound: + cos = np.abs(matrix[0, 0]) + sin = np.abs(matrix[0, 1]) + new_w = h * sin + w * cos + new_h = h * cos + w * sin + matrix[0, 2] += (new_w - w) * 0.5 + matrix[1, 2] += (new_h - h) * 0.5 + w = int(np.round(new_w)) + h = int(np.round(new_h)) + rotated = cv2.warpAffine( + img, + matrix, (w, h), + flags=cv2_interp_codes[interpolation], + borderValue=border_value) + return rotated + + +def bbox_clip(bboxes, img_shape): + """Clip bboxes to fit the image shape. + + Args: + bboxes (ndarray): Shape (..., 4*k) + img_shape (tuple[int]): (height, width) of the image. + + Returns: + ndarray: Clipped bboxes. + """ + assert bboxes.shape[-1] % 4 == 0 + cmin = np.empty(bboxes.shape[-1], dtype=bboxes.dtype) + cmin[0::2] = img_shape[1] - 1 + cmin[1::2] = img_shape[0] - 1 + clipped_bboxes = np.maximum(np.minimum(bboxes, cmin), 0) + return clipped_bboxes + + +def bbox_scaling(bboxes, scale, clip_shape=None): + """Scaling bboxes w.r.t the box center. + + Args: + bboxes (ndarray): Shape(..., 4). + scale (float): Scaling factor. + clip_shape (tuple[int], optional): If specified, bboxes that exceed the + boundary will be clipped according to the given shape (h, w). + + Returns: + ndarray: Scaled bboxes. + """ + if float(scale) == 1.0: + scaled_bboxes = bboxes.copy() + else: + w = bboxes[..., 2] - bboxes[..., 0] + 1 + h = bboxes[..., 3] - bboxes[..., 1] + 1 + dw = (w * (scale - 1)) * 0.5 + dh = (h * (scale - 1)) * 0.5 + scaled_bboxes = bboxes + np.stack((-dw, -dh, dw, dh), axis=-1) + if clip_shape is not None: + return bbox_clip(scaled_bboxes, clip_shape) + else: + return scaled_bboxes + + +def imcrop(img, bboxes, scale=1.0, pad_fill=None): + """Crop image patches. + + 3 steps: scale the bboxes -> clip bboxes -> crop and pad. + + Args: + img (ndarray): Image to be cropped. + bboxes (ndarray): Shape (k, 4) or (4, ), location of cropped bboxes. + scale (float, optional): Scale ratio of bboxes, the default value + 1.0 means no padding. + pad_fill (Number | list[Number]): Value to be filled for padding. + Default: None, which means no padding. + + Returns: + list[ndarray] | ndarray: The cropped image patches. + """ + chn = 1 if img.ndim == 2 else img.shape[2] + if pad_fill is not None: + if isinstance(pad_fill, (int, float)): + pad_fill = [pad_fill for _ in range(chn)] + assert len(pad_fill) == chn + + _bboxes = bboxes[None, ...] if bboxes.ndim == 1 else bboxes + scaled_bboxes = bbox_scaling(_bboxes, scale).astype(np.int32) + clipped_bbox = bbox_clip(scaled_bboxes, img.shape) + + patches = [] + for i in range(clipped_bbox.shape[0]): + x1, y1, x2, y2 = tuple(clipped_bbox[i, :]) + if pad_fill is None: + patch = img[y1:y2 + 1, x1:x2 + 1, ...] + else: + _x1, _y1, _x2, _y2 = tuple(scaled_bboxes[i, :]) + if chn == 1: + patch_shape = (_y2 - _y1 + 1, _x2 - _x1 + 1) + else: + patch_shape = (_y2 - _y1 + 1, _x2 - _x1 + 1, chn) + patch = np.array( + pad_fill, dtype=img.dtype) * np.ones( + patch_shape, dtype=img.dtype) + x_start = 0 if _x1 >= 0 else -_x1 + y_start = 0 if _y1 >= 0 else -_y1 + w = x2 - x1 + 1 + h = y2 - y1 + 1 + patch[y_start:y_start + h, x_start:x_start + w, + ...] = img[y1:y1 + h, x1:x1 + w, ...] + patches.append(patch) + + if bboxes.ndim == 1: + return patches[0] + else: + return patches + + +def impad(img, + *, + shape=None, + padding=None, + pad_val=0, + padding_mode='constant'): + """Pad the given image to a certain shape or pad on all sides with + specified padding mode and padding value. + + Args: + img (ndarray): Image to be padded. + shape (tuple[int]): Expected padding shape (h, w). Default: None. + padding (int or tuple[int]): Padding on each border. If a single int is + provided this is used to pad all borders. If tuple of length 2 is + provided this is the padding on left/right and top/bottom + respectively. If a tuple of length 4 is provided this is the + padding for the left, top, right and bottom borders respectively. + Default: None. Note that `shape` and `padding` can not be both + set. + pad_val (Number | Sequence[Number]): Values to be filled in padding + areas when padding_mode is 'constant'. Default: 0. + padding_mode (str): Type of padding. Should be: constant, edge, + reflect or symmetric. Default: constant. + + - constant: pads with a constant value, this value is specified + with pad_val. + - edge: pads with the last value at the edge of the image. + - reflect: pads with reflection of image without repeating the + last value on the edge. For example, padding [1, 2, 3, 4] + with 2 elements on both sides in reflect mode will result + in [3, 2, 1, 2, 3, 4, 3, 2]. + - symmetric: pads with reflection of image repeating the last + value on the edge. For example, padding [1, 2, 3, 4] with + 2 elements on both sides in symmetric mode will result in + [2, 1, 1, 2, 3, 4, 4, 3] + + Returns: + ndarray: The padded image. + """ + + assert (shape is not None) ^ (padding is not None) + if shape is not None: + padding = (0, 0, shape[1] - img.shape[1], shape[0] - img.shape[0]) + + # check pad_val + if isinstance(pad_val, tuple): + assert len(pad_val) == img.shape[-1] + elif not isinstance(pad_val, numbers.Number): + raise TypeError('pad_val must be a int or a tuple. ' + f'But received {type(pad_val)}') + + # check padding + if isinstance(padding, tuple) and len(padding) in [2, 4]: + if len(padding) == 2: + padding = (padding[0], padding[1], padding[0], padding[1]) + elif isinstance(padding, numbers.Number): + padding = (padding, padding, padding, padding) + else: + raise ValueError('Padding must be a int or a 2, or 4 element tuple.' + f'But received {padding}') + + # check padding mode + assert padding_mode in ['constant', 'edge', 'reflect', 'symmetric'] + + border_type = { + 'constant': cv2.BORDER_CONSTANT, + 'edge': cv2.BORDER_REPLICATE, + 'reflect': cv2.BORDER_REFLECT_101, + 'symmetric': cv2.BORDER_REFLECT + } + img = cv2.copyMakeBorder( + img, + padding[1], + padding[3], + padding[0], + padding[2], + border_type[padding_mode], + value=pad_val) + + return img + + +def impad_to_multiple(img, divisor, pad_val=0): + """Pad an image to ensure each edge to be multiple to some number. + + Args: + img (ndarray): Image to be padded. + divisor (int): Padded image edges will be multiple to divisor. + pad_val (Number | Sequence[Number]): Same as :func:`impad`. + + Returns: + ndarray: The padded image. + """ + pad_h = int(np.ceil(img.shape[0] / divisor)) * divisor + pad_w = int(np.ceil(img.shape[1] / divisor)) * divisor + return impad(img, shape=(pad_h, pad_w), pad_val=pad_val) + + +def cutout(img, shape, pad_val=0): + """Randomly cut out a rectangle from the original img. + + Args: + img (ndarray): Image to be cutout. + shape (int | tuple[int]): Expected cutout shape (h, w). If given as a + int, the value will be used for both h and w. + pad_val (int | float | tuple[int | float]): Values to be filled in the + cut area. Defaults to 0. + + Returns: + ndarray: The cutout image. + """ + + channels = 1 if img.ndim == 2 else img.shape[2] + if isinstance(shape, int): + cut_h, cut_w = shape, shape + else: + assert isinstance(shape, tuple) and len(shape) == 2, \ + f'shape must be a int or a tuple with length 2, but got type ' \ + f'{type(shape)} instead.' + cut_h, cut_w = shape + if isinstance(pad_val, (int, float)): + pad_val = tuple([pad_val] * channels) + elif isinstance(pad_val, tuple): + assert len(pad_val) == channels, \ + 'Expected the num of elements in tuple equals the channels' \ + 'of input image. Found {} vs {}'.format( + len(pad_val), channels) + else: + raise TypeError(f'Invalid type {type(pad_val)} for `pad_val`') + + img_h, img_w = img.shape[:2] + y0 = np.random.uniform(img_h) + x0 = np.random.uniform(img_w) + + y1 = int(max(0, y0 - cut_h / 2.)) + x1 = int(max(0, x0 - cut_w / 2.)) + y2 = min(img_h, y1 + cut_h) + x2 = min(img_w, x1 + cut_w) + + if img.ndim == 2: + patch_shape = (y2 - y1, x2 - x1) + else: + patch_shape = (y2 - y1, x2 - x1, channels) + + img_cutout = img.copy() + patch = np.array( + pad_val, dtype=img.dtype) * np.ones( + patch_shape, dtype=img.dtype) + img_cutout[y1:y2, x1:x2, ...] = patch + + return img_cutout + + +def _get_shear_matrix(magnitude, direction='horizontal'): + """Generate the shear matrix for transformation. + + Args: + magnitude (int | float): The magnitude used for shear. + direction (str): The flip direction, either "horizontal" + or "vertical". + + Returns: + ndarray: The shear matrix with dtype float32. + """ + if direction == 'horizontal': + shear_matrix = np.float32([[1, magnitude, 0], [0, 1, 0]]) + elif direction == 'vertical': + shear_matrix = np.float32([[1, 0, 0], [magnitude, 1, 0]]) + return shear_matrix + + +def imshear(img, + magnitude, + direction='horizontal', + border_value=0, + interpolation='bilinear'): + """Shear an image. + + Args: + img (ndarray): Image to be sheared with format (h, w) + or (h, w, c). + magnitude (int | float): The magnitude used for shear. + direction (str): The flip direction, either "horizontal" + or "vertical". + border_value (int | tuple[int]): Value used in case of a + constant border. + interpolation (str): Same as :func:`resize`. + + Returns: + ndarray: The sheared image. + """ + assert direction in ['horizontal', + 'vertical'], f'Invalid direction: {direction}' + height, width = img.shape[:2] + if img.ndim == 2: + channels = 1 + elif img.ndim == 3: + channels = img.shape[-1] + if isinstance(border_value, int): + border_value = tuple([border_value] * channels) + elif isinstance(border_value, tuple): + assert len(border_value) == channels, \ + 'Expected the num of elements in tuple equals the channels' \ + 'of input image. Found {} vs {}'.format( + len(border_value), channels) + else: + raise ValueError( + f'Invalid type {type(border_value)} for `border_value`') + shear_matrix = _get_shear_matrix(magnitude, direction) + sheared = cv2.warpAffine( + img, + shear_matrix, + (width, height), + # Note case when the number elements in `border_value` + # greater than 3 (e.g. shearing masks whose channels large + # than 3) will raise TypeError in `cv2.warpAffine`. + # Here simply slice the first 3 values in `border_value`. + borderValue=border_value[:3], + flags=cv2_interp_codes[interpolation]) + return sheared + + +def _get_translate_matrix(offset, direction='horizontal'): + """Generate the translate matrix. + + Args: + offset (int | float): The offset used for translate. + direction (str): The translate direction, either + "horizontal" or "vertical". + + Returns: + ndarray: The translate matrix with dtype float32. + """ + if direction == 'horizontal': + translate_matrix = np.float32([[1, 0, offset], [0, 1, 0]]) + elif direction == 'vertical': + translate_matrix = np.float32([[1, 0, 0], [0, 1, offset]]) + return translate_matrix + + +def imtranslate(img, + offset, + direction='horizontal', + border_value=0, + interpolation='bilinear'): + """Translate an image. + + Args: + img (ndarray): Image to be translated with format + (h, w) or (h, w, c). + offset (int | float): The offset used for translate. + direction (str): The translate direction, either "horizontal" + or "vertical". + border_value (int | tuple[int]): Value used in case of a + constant border. + interpolation (str): Same as :func:`resize`. + + Returns: + ndarray: The translated image. + """ + assert direction in ['horizontal', + 'vertical'], f'Invalid direction: {direction}' + height, width = img.shape[:2] + if img.ndim == 2: + channels = 1 + elif img.ndim == 3: + channels = img.shape[-1] + if isinstance(border_value, int): + border_value = tuple([border_value] * channels) + elif isinstance(border_value, tuple): + assert len(border_value) == channels, \ + 'Expected the num of elements in tuple equals the channels' \ + 'of input image. Found {} vs {}'.format( + len(border_value), channels) + else: + raise ValueError( + f'Invalid type {type(border_value)} for `border_value`.') + translate_matrix = _get_translate_matrix(offset, direction) + translated = cv2.warpAffine( + img, + translate_matrix, + (width, height), + # Note case when the number elements in `border_value` + # greater than 3 (e.g. translating masks whose channels + # large than 3) will raise TypeError in `cv2.warpAffine`. + # Here simply slice the first 3 values in `border_value`. + borderValue=border_value[:3], + flags=cv2_interp_codes[interpolation]) + return translated diff --git a/annotator/uniformer/mmcv/image/io.py b/annotator/uniformer/mmcv/image/io.py new file mode 100644 index 0000000000000000000000000000000000000000..d3fa2e8cc06b1a7b0b69de6406980b15d61a1e5d --- /dev/null +++ b/annotator/uniformer/mmcv/image/io.py @@ -0,0 +1,258 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import io +import os.path as osp +from pathlib import Path + +import cv2 +import numpy as np +from cv2 import (IMREAD_COLOR, IMREAD_GRAYSCALE, IMREAD_IGNORE_ORIENTATION, + IMREAD_UNCHANGED) + +from annotator.uniformer.mmcv.utils import check_file_exist, is_str, mkdir_or_exist + +try: + from turbojpeg import TJCS_RGB, TJPF_BGR, TJPF_GRAY, TurboJPEG +except ImportError: + TJCS_RGB = TJPF_GRAY = TJPF_BGR = TurboJPEG = None + +try: + from PIL import Image, ImageOps +except ImportError: + Image = None + +try: + import tifffile +except ImportError: + tifffile = None + +jpeg = None +supported_backends = ['cv2', 'turbojpeg', 'pillow', 'tifffile'] + +imread_flags = { + 'color': IMREAD_COLOR, + 'grayscale': IMREAD_GRAYSCALE, + 'unchanged': IMREAD_UNCHANGED, + 'color_ignore_orientation': IMREAD_IGNORE_ORIENTATION | IMREAD_COLOR, + 'grayscale_ignore_orientation': + IMREAD_IGNORE_ORIENTATION | IMREAD_GRAYSCALE +} + +imread_backend = 'cv2' + + +def use_backend(backend): + """Select a backend for image decoding. + + Args: + backend (str): The image decoding backend type. Options are `cv2`, + `pillow`, `turbojpeg` (see https://github.com/lilohuang/PyTurboJPEG) + and `tifffile`. `turbojpeg` is faster but it only supports `.jpeg` + file format. + """ + assert backend in supported_backends + global imread_backend + imread_backend = backend + if imread_backend == 'turbojpeg': + if TurboJPEG is None: + raise ImportError('`PyTurboJPEG` is not installed') + global jpeg + if jpeg is None: + jpeg = TurboJPEG() + elif imread_backend == 'pillow': + if Image is None: + raise ImportError('`Pillow` is not installed') + elif imread_backend == 'tifffile': + if tifffile is None: + raise ImportError('`tifffile` is not installed') + + +def _jpegflag(flag='color', channel_order='bgr'): + channel_order = channel_order.lower() + if channel_order not in ['rgb', 'bgr']: + raise ValueError('channel order must be either "rgb" or "bgr"') + + if flag == 'color': + if channel_order == 'bgr': + return TJPF_BGR + elif channel_order == 'rgb': + return TJCS_RGB + elif flag == 'grayscale': + return TJPF_GRAY + else: + raise ValueError('flag must be "color" or "grayscale"') + + +def _pillow2array(img, flag='color', channel_order='bgr'): + """Convert a pillow image to numpy array. + + Args: + img (:obj:`PIL.Image.Image`): The image loaded using PIL + flag (str): Flags specifying the color type of a loaded image, + candidates are 'color', 'grayscale' and 'unchanged'. + Default to 'color'. + channel_order (str): The channel order of the output image array, + candidates are 'bgr' and 'rgb'. Default to 'bgr'. + + Returns: + np.ndarray: The converted numpy array + """ + channel_order = channel_order.lower() + if channel_order not in ['rgb', 'bgr']: + raise ValueError('channel order must be either "rgb" or "bgr"') + + if flag == 'unchanged': + array = np.array(img) + if array.ndim >= 3 and array.shape[2] >= 3: # color image + array[:, :, :3] = array[:, :, (2, 1, 0)] # RGB to BGR + else: + # Handle exif orientation tag + if flag in ['color', 'grayscale']: + img = ImageOps.exif_transpose(img) + # If the image mode is not 'RGB', convert it to 'RGB' first. + if img.mode != 'RGB': + if img.mode != 'LA': + # Most formats except 'LA' can be directly converted to RGB + img = img.convert('RGB') + else: + # When the mode is 'LA', the default conversion will fill in + # the canvas with black, which sometimes shadows black objects + # in the foreground. + # + # Therefore, a random color (124, 117, 104) is used for canvas + img_rgba = img.convert('RGBA') + img = Image.new('RGB', img_rgba.size, (124, 117, 104)) + img.paste(img_rgba, mask=img_rgba.split()[3]) # 3 is alpha + if flag in ['color', 'color_ignore_orientation']: + array = np.array(img) + if channel_order != 'rgb': + array = array[:, :, ::-1] # RGB to BGR + elif flag in ['grayscale', 'grayscale_ignore_orientation']: + img = img.convert('L') + array = np.array(img) + else: + raise ValueError( + 'flag must be "color", "grayscale", "unchanged", ' + f'"color_ignore_orientation" or "grayscale_ignore_orientation"' + f' but got {flag}') + return array + + +def imread(img_or_path, flag='color', channel_order='bgr', backend=None): + """Read an image. + + Args: + img_or_path (ndarray or str or Path): Either a numpy array or str or + pathlib.Path. If it is a numpy array (loaded image), then + it will be returned as is. + flag (str): Flags specifying the color type of a loaded image, + candidates are `color`, `grayscale`, `unchanged`, + `color_ignore_orientation` and `grayscale_ignore_orientation`. + By default, `cv2` and `pillow` backend would rotate the image + according to its EXIF info unless called with `unchanged` or + `*_ignore_orientation` flags. `turbojpeg` and `tifffile` backend + always ignore image's EXIF info regardless of the flag. + The `turbojpeg` backend only supports `color` and `grayscale`. + channel_order (str): Order of channel, candidates are `bgr` and `rgb`. + backend (str | None): The image decoding backend type. Options are + `cv2`, `pillow`, `turbojpeg`, `tifffile`, `None`. + If backend is None, the global imread_backend specified by + ``mmcv.use_backend()`` will be used. Default: None. + + Returns: + ndarray: Loaded image array. + """ + + if backend is None: + backend = imread_backend + if backend not in supported_backends: + raise ValueError(f'backend: {backend} is not supported. Supported ' + "backends are 'cv2', 'turbojpeg', 'pillow'") + if isinstance(img_or_path, Path): + img_or_path = str(img_or_path) + + if isinstance(img_or_path, np.ndarray): + return img_or_path + elif is_str(img_or_path): + check_file_exist(img_or_path, + f'img file does not exist: {img_or_path}') + if backend == 'turbojpeg': + with open(img_or_path, 'rb') as in_file: + img = jpeg.decode(in_file.read(), + _jpegflag(flag, channel_order)) + if img.shape[-1] == 1: + img = img[:, :, 0] + return img + elif backend == 'pillow': + img = Image.open(img_or_path) + img = _pillow2array(img, flag, channel_order) + return img + elif backend == 'tifffile': + img = tifffile.imread(img_or_path) + return img + else: + flag = imread_flags[flag] if is_str(flag) else flag + img = cv2.imread(img_or_path, flag) + if flag == IMREAD_COLOR and channel_order == 'rgb': + cv2.cvtColor(img, cv2.COLOR_BGR2RGB, img) + return img + else: + raise TypeError('"img" must be a numpy array or a str or ' + 'a pathlib.Path object') + + +def imfrombytes(content, flag='color', channel_order='bgr', backend=None): + """Read an image from bytes. + + Args: + content (bytes): Image bytes got from files or other streams. + flag (str): Same as :func:`imread`. + backend (str | None): The image decoding backend type. Options are + `cv2`, `pillow`, `turbojpeg`, `None`. If backend is None, the + global imread_backend specified by ``mmcv.use_backend()`` will be + used. Default: None. + + Returns: + ndarray: Loaded image array. + """ + + if backend is None: + backend = imread_backend + if backend not in supported_backends: + raise ValueError(f'backend: {backend} is not supported. Supported ' + "backends are 'cv2', 'turbojpeg', 'pillow'") + if backend == 'turbojpeg': + img = jpeg.decode(content, _jpegflag(flag, channel_order)) + if img.shape[-1] == 1: + img = img[:, :, 0] + return img + elif backend == 'pillow': + buff = io.BytesIO(content) + img = Image.open(buff) + img = _pillow2array(img, flag, channel_order) + return img + else: + img_np = np.frombuffer(content, np.uint8) + flag = imread_flags[flag] if is_str(flag) else flag + img = cv2.imdecode(img_np, flag) + if flag == IMREAD_COLOR and channel_order == 'rgb': + cv2.cvtColor(img, cv2.COLOR_BGR2RGB, img) + return img + + +def imwrite(img, file_path, params=None, auto_mkdir=True): + """Write image to file. + + Args: + img (ndarray): Image array to be written. + file_path (str): Image file path. + params (None or list): Same as opencv :func:`imwrite` interface. + auto_mkdir (bool): If the parent folder of `file_path` does not exist, + whether to create it automatically. + + Returns: + bool: Successful or not. + """ + if auto_mkdir: + dir_name = osp.abspath(osp.dirname(file_path)) + mkdir_or_exist(dir_name) + return cv2.imwrite(file_path, img, params) diff --git a/annotator/uniformer/mmcv/image/misc.py b/annotator/uniformer/mmcv/image/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..3e61f05e3b05e4c7b40de4eb6c8eb100e6da41d0 --- /dev/null +++ b/annotator/uniformer/mmcv/image/misc.py @@ -0,0 +1,44 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numpy as np + +import annotator.uniformer.mmcv as mmcv + +try: + import torch +except ImportError: + torch = None + + +def tensor2imgs(tensor, mean=(0, 0, 0), std=(1, 1, 1), to_rgb=True): + """Convert tensor to 3-channel images. + + Args: + tensor (torch.Tensor): Tensor that contains multiple images, shape ( + N, C, H, W). + mean (tuple[float], optional): Mean of images. Defaults to (0, 0, 0). + std (tuple[float], optional): Standard deviation of images. + Defaults to (1, 1, 1). + to_rgb (bool, optional): Whether the tensor was converted to RGB + format in the first place. If so, convert it back to BGR. + Defaults to True. + + Returns: + list[np.ndarray]: A list that contains multiple images. + """ + + if torch is None: + raise RuntimeError('pytorch is not installed') + assert torch.is_tensor(tensor) and tensor.ndim == 4 + assert len(mean) == 3 + assert len(std) == 3 + + num_imgs = tensor.size(0) + mean = np.array(mean, dtype=np.float32) + std = np.array(std, dtype=np.float32) + imgs = [] + for img_id in range(num_imgs): + img = tensor[img_id, ...].cpu().numpy().transpose(1, 2, 0) + img = mmcv.imdenormalize( + img, mean, std, to_bgr=to_rgb).astype(np.uint8) + imgs.append(np.ascontiguousarray(img)) + return imgs diff --git a/annotator/uniformer/mmcv/image/photometric.py b/annotator/uniformer/mmcv/image/photometric.py new file mode 100644 index 0000000000000000000000000000000000000000..5085d012019c0cbf56f66f421a378278c1a058ae --- /dev/null +++ b/annotator/uniformer/mmcv/image/photometric.py @@ -0,0 +1,428 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import cv2 +import numpy as np + +from ..utils import is_tuple_of +from .colorspace import bgr2gray, gray2bgr + + +def imnormalize(img, mean, std, to_rgb=True): + """Normalize an image with mean and std. + + Args: + img (ndarray): Image to be normalized. + mean (ndarray): The mean to be used for normalize. + std (ndarray): The std to be used for normalize. + to_rgb (bool): Whether to convert to rgb. + + Returns: + ndarray: The normalized image. + """ + img = img.copy().astype(np.float32) + return imnormalize_(img, mean, std, to_rgb) + + +def imnormalize_(img, mean, std, to_rgb=True): + """Inplace normalize an image with mean and std. + + Args: + img (ndarray): Image to be normalized. + mean (ndarray): The mean to be used for normalize. + std (ndarray): The std to be used for normalize. + to_rgb (bool): Whether to convert to rgb. + + Returns: + ndarray: The normalized image. + """ + # cv2 inplace normalization does not accept uint8 + assert img.dtype != np.uint8 + mean = np.float64(mean.reshape(1, -1)) + stdinv = 1 / np.float64(std.reshape(1, -1)) + if to_rgb: + cv2.cvtColor(img, cv2.COLOR_BGR2RGB, img) # inplace + cv2.subtract(img, mean, img) # inplace + cv2.multiply(img, stdinv, img) # inplace + return img + + +def imdenormalize(img, mean, std, to_bgr=True): + assert img.dtype != np.uint8 + mean = mean.reshape(1, -1).astype(np.float64) + std = std.reshape(1, -1).astype(np.float64) + img = cv2.multiply(img, std) # make a copy + cv2.add(img, mean, img) # inplace + if to_bgr: + cv2.cvtColor(img, cv2.COLOR_RGB2BGR, img) # inplace + return img + + +def iminvert(img): + """Invert (negate) an image. + + Args: + img (ndarray): Image to be inverted. + + Returns: + ndarray: The inverted image. + """ + return np.full_like(img, 255) - img + + +def solarize(img, thr=128): + """Solarize an image (invert all pixel values above a threshold) + + Args: + img (ndarray): Image to be solarized. + thr (int): Threshold for solarizing (0 - 255). + + Returns: + ndarray: The solarized image. + """ + img = np.where(img < thr, img, 255 - img) + return img + + +def posterize(img, bits): + """Posterize an image (reduce the number of bits for each color channel) + + Args: + img (ndarray): Image to be posterized. + bits (int): Number of bits (1 to 8) to use for posterizing. + + Returns: + ndarray: The posterized image. + """ + shift = 8 - bits + img = np.left_shift(np.right_shift(img, shift), shift) + return img + + +def adjust_color(img, alpha=1, beta=None, gamma=0): + r"""It blends the source image and its gray image: + + .. math:: + output = img * alpha + gray\_img * beta + gamma + + Args: + img (ndarray): The input source image. + alpha (int | float): Weight for the source image. Default 1. + beta (int | float): Weight for the converted gray image. + If None, it's assigned the value (1 - `alpha`). + gamma (int | float): Scalar added to each sum. + Same as :func:`cv2.addWeighted`. Default 0. + + Returns: + ndarray: Colored image which has the same size and dtype as input. + """ + gray_img = bgr2gray(img) + gray_img = np.tile(gray_img[..., None], [1, 1, 3]) + if beta is None: + beta = 1 - alpha + colored_img = cv2.addWeighted(img, alpha, gray_img, beta, gamma) + if not colored_img.dtype == np.uint8: + # Note when the dtype of `img` is not the default `np.uint8` + # (e.g. np.float32), the value in `colored_img` got from cv2 + # is not guaranteed to be in range [0, 255], so here clip + # is needed. + colored_img = np.clip(colored_img, 0, 255) + return colored_img + + +def imequalize(img): + """Equalize the image histogram. + + This function applies a non-linear mapping to the input image, + in order to create a uniform distribution of grayscale values + in the output image. + + Args: + img (ndarray): Image to be equalized. + + Returns: + ndarray: The equalized image. + """ + + def _scale_channel(im, c): + """Scale the data in the corresponding channel.""" + im = im[:, :, c] + # Compute the histogram of the image channel. + histo = np.histogram(im, 256, (0, 255))[0] + # For computing the step, filter out the nonzeros. + nonzero_histo = histo[histo > 0] + step = (np.sum(nonzero_histo) - nonzero_histo[-1]) // 255 + if not step: + lut = np.array(range(256)) + else: + # Compute the cumulative sum, shifted by step // 2 + # and then normalized by step. + lut = (np.cumsum(histo) + (step // 2)) // step + # Shift lut, prepending with 0. + lut = np.concatenate([[0], lut[:-1]], 0) + # handle potential integer overflow + lut[lut > 255] = 255 + # If step is zero, return the original image. + # Otherwise, index from lut. + return np.where(np.equal(step, 0), im, lut[im]) + + # Scales each channel independently and then stacks + # the result. + s1 = _scale_channel(img, 0) + s2 = _scale_channel(img, 1) + s3 = _scale_channel(img, 2) + equalized_img = np.stack([s1, s2, s3], axis=-1) + return equalized_img.astype(img.dtype) + + +def adjust_brightness(img, factor=1.): + """Adjust image brightness. + + This function controls the brightness of an image. An + enhancement factor of 0.0 gives a black image. + A factor of 1.0 gives the original image. This function + blends the source image and the degenerated black image: + + .. math:: + output = img * factor + degenerated * (1 - factor) + + Args: + img (ndarray): Image to be brightened. + factor (float): A value controls the enhancement. + Factor 1.0 returns the original image, lower + factors mean less color (brightness, contrast, + etc), and higher values more. Default 1. + + Returns: + ndarray: The brightened image. + """ + degenerated = np.zeros_like(img) + # Note manually convert the dtype to np.float32, to + # achieve as close results as PIL.ImageEnhance.Brightness. + # Set beta=1-factor, and gamma=0 + brightened_img = cv2.addWeighted( + img.astype(np.float32), factor, degenerated.astype(np.float32), + 1 - factor, 0) + brightened_img = np.clip(brightened_img, 0, 255) + return brightened_img.astype(img.dtype) + + +def adjust_contrast(img, factor=1.): + """Adjust image contrast. + + This function controls the contrast of an image. An + enhancement factor of 0.0 gives a solid grey + image. A factor of 1.0 gives the original image. It + blends the source image and the degenerated mean image: + + .. math:: + output = img * factor + degenerated * (1 - factor) + + Args: + img (ndarray): Image to be contrasted. BGR order. + factor (float): Same as :func:`mmcv.adjust_brightness`. + + Returns: + ndarray: The contrasted image. + """ + gray_img = bgr2gray(img) + hist = np.histogram(gray_img, 256, (0, 255))[0] + mean = round(np.sum(gray_img) / np.sum(hist)) + degenerated = (np.ones_like(img[..., 0]) * mean).astype(img.dtype) + degenerated = gray2bgr(degenerated) + contrasted_img = cv2.addWeighted( + img.astype(np.float32), factor, degenerated.astype(np.float32), + 1 - factor, 0) + contrasted_img = np.clip(contrasted_img, 0, 255) + return contrasted_img.astype(img.dtype) + + +def auto_contrast(img, cutoff=0): + """Auto adjust image contrast. + + This function maximize (normalize) image contrast by first removing cutoff + percent of the lightest and darkest pixels from the histogram and remapping + the image so that the darkest pixel becomes black (0), and the lightest + becomes white (255). + + Args: + img (ndarray): Image to be contrasted. BGR order. + cutoff (int | float | tuple): The cutoff percent of the lightest and + darkest pixels to be removed. If given as tuple, it shall be + (low, high). Otherwise, the single value will be used for both. + Defaults to 0. + + Returns: + ndarray: The contrasted image. + """ + + def _auto_contrast_channel(im, c, cutoff): + im = im[:, :, c] + # Compute the histogram of the image channel. + histo = np.histogram(im, 256, (0, 255))[0] + # Remove cut-off percent pixels from histo + histo_sum = np.cumsum(histo) + cut_low = histo_sum[-1] * cutoff[0] // 100 + cut_high = histo_sum[-1] - histo_sum[-1] * cutoff[1] // 100 + histo_sum = np.clip(histo_sum, cut_low, cut_high) - cut_low + histo = np.concatenate([[histo_sum[0]], np.diff(histo_sum)], 0) + + # Compute mapping + low, high = np.nonzero(histo)[0][0], np.nonzero(histo)[0][-1] + # If all the values have been cut off, return the origin img + if low >= high: + return im + scale = 255.0 / (high - low) + offset = -low * scale + lut = np.array(range(256)) + lut = lut * scale + offset + lut = np.clip(lut, 0, 255) + return lut[im] + + if isinstance(cutoff, (int, float)): + cutoff = (cutoff, cutoff) + else: + assert isinstance(cutoff, tuple), 'cutoff must be of type int, ' \ + f'float or tuple, but got {type(cutoff)} instead.' + # Auto adjusts contrast for each channel independently and then stacks + # the result. + s1 = _auto_contrast_channel(img, 0, cutoff) + s2 = _auto_contrast_channel(img, 1, cutoff) + s3 = _auto_contrast_channel(img, 2, cutoff) + contrasted_img = np.stack([s1, s2, s3], axis=-1) + return contrasted_img.astype(img.dtype) + + +def adjust_sharpness(img, factor=1., kernel=None): + """Adjust image sharpness. + + This function controls the sharpness of an image. An + enhancement factor of 0.0 gives a blurred image. A + factor of 1.0 gives the original image. And a factor + of 2.0 gives a sharpened image. It blends the source + image and the degenerated mean image: + + .. math:: + output = img * factor + degenerated * (1 - factor) + + Args: + img (ndarray): Image to be sharpened. BGR order. + factor (float): Same as :func:`mmcv.adjust_brightness`. + kernel (np.ndarray, optional): Filter kernel to be applied on the img + to obtain the degenerated img. Defaults to None. + + Note: + No value sanity check is enforced on the kernel set by users. So with + an inappropriate kernel, the ``adjust_sharpness`` may fail to perform + the function its name indicates but end up performing whatever + transform determined by the kernel. + + Returns: + ndarray: The sharpened image. + """ + + if kernel is None: + # adopted from PIL.ImageFilter.SMOOTH + kernel = np.array([[1., 1., 1.], [1., 5., 1.], [1., 1., 1.]]) / 13 + assert isinstance(kernel, np.ndarray), \ + f'kernel must be of type np.ndarray, but got {type(kernel)} instead.' + assert kernel.ndim == 2, \ + f'kernel must have a dimension of 2, but got {kernel.ndim} instead.' + + degenerated = cv2.filter2D(img, -1, kernel) + sharpened_img = cv2.addWeighted( + img.astype(np.float32), factor, degenerated.astype(np.float32), + 1 - factor, 0) + sharpened_img = np.clip(sharpened_img, 0, 255) + return sharpened_img.astype(img.dtype) + + +def adjust_lighting(img, eigval, eigvec, alphastd=0.1, to_rgb=True): + """AlexNet-style PCA jitter. + + This data augmentation is proposed in `ImageNet Classification with Deep + Convolutional Neural Networks + `_. + + Args: + img (ndarray): Image to be adjusted lighting. BGR order. + eigval (ndarray): the eigenvalue of the convariance matrix of pixel + values, respectively. + eigvec (ndarray): the eigenvector of the convariance matrix of pixel + values, respectively. + alphastd (float): The standard deviation for distribution of alpha. + Defaults to 0.1 + to_rgb (bool): Whether to convert img to rgb. + + Returns: + ndarray: The adjusted image. + """ + assert isinstance(eigval, np.ndarray) and isinstance(eigvec, np.ndarray), \ + f'eigval and eigvec should both be of type np.ndarray, got ' \ + f'{type(eigval)} and {type(eigvec)} instead.' + + assert eigval.ndim == 1 and eigvec.ndim == 2 + assert eigvec.shape == (3, eigval.shape[0]) + n_eigval = eigval.shape[0] + assert isinstance(alphastd, float), 'alphastd should be of type float, ' \ + f'got {type(alphastd)} instead.' + + img = img.copy().astype(np.float32) + if to_rgb: + cv2.cvtColor(img, cv2.COLOR_BGR2RGB, img) # inplace + + alpha = np.random.normal(0, alphastd, n_eigval) + alter = eigvec \ + * np.broadcast_to(alpha.reshape(1, n_eigval), (3, n_eigval)) \ + * np.broadcast_to(eigval.reshape(1, n_eigval), (3, n_eigval)) + alter = np.broadcast_to(alter.sum(axis=1).reshape(1, 1, 3), img.shape) + img_adjusted = img + alter + return img_adjusted + + +def lut_transform(img, lut_table): + """Transform array by look-up table. + + The function lut_transform fills the output array with values from the + look-up table. Indices of the entries are taken from the input array. + + Args: + img (ndarray): Image to be transformed. + lut_table (ndarray): look-up table of 256 elements; in case of + multi-channel input array, the table should either have a single + channel (in this case the same table is used for all channels) or + the same number of channels as in the input array. + + Returns: + ndarray: The transformed image. + """ + assert isinstance(img, np.ndarray) + assert 0 <= np.min(img) and np.max(img) <= 255 + assert isinstance(lut_table, np.ndarray) + assert lut_table.shape == (256, ) + + return cv2.LUT(np.array(img, dtype=np.uint8), lut_table) + + +def clahe(img, clip_limit=40.0, tile_grid_size=(8, 8)): + """Use CLAHE method to process the image. + + See `ZUIDERVELD,K. Contrast Limited Adaptive Histogram Equalization[J]. + Graphics Gems, 1994:474-485.` for more information. + + Args: + img (ndarray): Image to be processed. + clip_limit (float): Threshold for contrast limiting. Default: 40.0. + tile_grid_size (tuple[int]): Size of grid for histogram equalization. + Input image will be divided into equally sized rectangular tiles. + It defines the number of tiles in row and column. Default: (8, 8). + + Returns: + ndarray: The processed image. + """ + assert isinstance(img, np.ndarray) + assert img.ndim == 2 + assert isinstance(clip_limit, (float, int)) + assert is_tuple_of(tile_grid_size, int) + assert len(tile_grid_size) == 2 + + clahe = cv2.createCLAHE(clip_limit, tile_grid_size) + return clahe.apply(np.array(img, dtype=np.uint8)) diff --git a/annotator/uniformer/mmcv/model_zoo/deprecated.json b/annotator/uniformer/mmcv/model_zoo/deprecated.json new file mode 100644 index 0000000000000000000000000000000000000000..25cf6f28caecc22a77e3136fefa6b8dfc0e6cb5b --- /dev/null +++ b/annotator/uniformer/mmcv/model_zoo/deprecated.json @@ -0,0 +1,6 @@ +{ + "resnet50_caffe": "detectron/resnet50_caffe", + "resnet50_caffe_bgr": "detectron2/resnet50_caffe_bgr", + "resnet101_caffe": "detectron/resnet101_caffe", + "resnet101_caffe_bgr": "detectron2/resnet101_caffe_bgr" +} diff --git a/annotator/uniformer/mmcv/model_zoo/mmcls.json b/annotator/uniformer/mmcv/model_zoo/mmcls.json new file mode 100644 index 0000000000000000000000000000000000000000..bdb311d9fe6d9f317290feedc9e37236c6cf6e8f --- /dev/null +++ b/annotator/uniformer/mmcv/model_zoo/mmcls.json @@ -0,0 +1,31 @@ +{ + "vgg11": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg11_batch256_imagenet_20210208-4271cd6c.pth", + "vgg13": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg13_batch256_imagenet_20210208-4d1d6080.pth", + "vgg16": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg16_batch256_imagenet_20210208-db26f1a5.pth", + "vgg19": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg19_batch256_imagenet_20210208-e6920e4a.pth", + "vgg11_bn": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg11_bn_batch256_imagenet_20210207-f244902c.pth", + "vgg13_bn": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg13_bn_batch256_imagenet_20210207-1a8b7864.pth", + "vgg16_bn": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg16_bn_batch256_imagenet_20210208-7e55cd29.pth", + "vgg19_bn": "https://download.openmmlab.com/mmclassification/v0/vgg/vgg19_bn_batch256_imagenet_20210208-da620c4f.pth", + "resnet18": "https://download.openmmlab.com/mmclassification/v0/resnet/resnet18_batch256_imagenet_20200708-34ab8f90.pth", + "resnet34": "https://download.openmmlab.com/mmclassification/v0/resnet/resnet34_batch256_imagenet_20200708-32ffb4f7.pth", + "resnet50": "https://download.openmmlab.com/mmclassification/v0/resnet/resnet50_batch256_imagenet_20200708-cfb998bf.pth", + "resnet101": "https://download.openmmlab.com/mmclassification/v0/resnet/resnet101_batch256_imagenet_20200708-753f3608.pth", + "resnet152": "https://download.openmmlab.com/mmclassification/v0/resnet/resnet152_batch256_imagenet_20200708-ec25b1f9.pth", + "resnet50_v1d": "https://download.openmmlab.com/mmclassification/v0/resnet/resnetv1d50_batch256_imagenet_20200708-1ad0ce94.pth", + "resnet101_v1d": "https://download.openmmlab.com/mmclassification/v0/resnet/resnetv1d101_batch256_imagenet_20200708-9cb302ef.pth", + "resnet152_v1d": "https://download.openmmlab.com/mmclassification/v0/resnet/resnetv1d152_batch256_imagenet_20200708-e79cb6a2.pth", + "resnext50_32x4d": "https://download.openmmlab.com/mmclassification/v0/resnext/resnext50_32x4d_b32x8_imagenet_20210429-56066e27.pth", + "resnext101_32x4d": "https://download.openmmlab.com/mmclassification/v0/resnext/resnext101_32x4d_b32x8_imagenet_20210506-e0fa3dd5.pth", + "resnext101_32x8d": "https://download.openmmlab.com/mmclassification/v0/resnext/resnext101_32x8d_b32x8_imagenet_20210506-23a247d5.pth", + "resnext152_32x4d": "https://download.openmmlab.com/mmclassification/v0/resnext/resnext152_32x4d_b32x8_imagenet_20210524-927787be.pth", + "se-resnet50": "https://download.openmmlab.com/mmclassification/v0/se-resnet/se-resnet50_batch256_imagenet_20200804-ae206104.pth", + "se-resnet101": "https://download.openmmlab.com/mmclassification/v0/se-resnet/se-resnet101_batch256_imagenet_20200804-ba5b51d4.pth", + "resnest50": "https://download.openmmlab.com/mmclassification/v0/resnest/resnest50_imagenet_converted-1ebf0afe.pth", + "resnest101": "https://download.openmmlab.com/mmclassification/v0/resnest/resnest101_imagenet_converted-032caa52.pth", + "resnest200": "https://download.openmmlab.com/mmclassification/v0/resnest/resnest200_imagenet_converted-581a60f2.pth", + "resnest269": "https://download.openmmlab.com/mmclassification/v0/resnest/resnest269_imagenet_converted-59930960.pth", + "shufflenet_v1": "https://download.openmmlab.com/mmclassification/v0/shufflenet_v1/shufflenet_v1_batch1024_imagenet_20200804-5d6cec73.pth", + "shufflenet_v2": "https://download.openmmlab.com/mmclassification/v0/shufflenet_v2/shufflenet_v2_batch1024_imagenet_20200812-5bf4721e.pth", + "mobilenet_v2": "https://download.openmmlab.com/mmclassification/v0/mobilenet_v2/mobilenet_v2_batch256_imagenet_20200708-3b2dc3af.pth" +} diff --git a/annotator/uniformer/mmcv/model_zoo/open_mmlab.json b/annotator/uniformer/mmcv/model_zoo/open_mmlab.json new file mode 100644 index 0000000000000000000000000000000000000000..8311db4feef92faa0841c697d75efbee8430c3a0 --- /dev/null +++ b/annotator/uniformer/mmcv/model_zoo/open_mmlab.json @@ -0,0 +1,50 @@ +{ + "vgg16_caffe": "https://download.openmmlab.com/pretrain/third_party/vgg16_caffe-292e1171.pth", + "detectron/resnet50_caffe": "https://download.openmmlab.com/pretrain/third_party/resnet50_caffe-788b5fa3.pth", + "detectron2/resnet50_caffe": "https://download.openmmlab.com/pretrain/third_party/resnet50_msra-5891d200.pth", + "detectron/resnet101_caffe": "https://download.openmmlab.com/pretrain/third_party/resnet101_caffe-3ad79236.pth", + "detectron2/resnet101_caffe": "https://download.openmmlab.com/pretrain/third_party/resnet101_msra-6cc46731.pth", + "detectron2/resnext101_32x8d": "https://download.openmmlab.com/pretrain/third_party/resnext101_32x8d-1516f1aa.pth", + "resnext50_32x4d": "https://download.openmmlab.com/pretrain/third_party/resnext50-32x4d-0ab1a123.pth", + "resnext101_32x4d": "https://download.openmmlab.com/pretrain/third_party/resnext101_32x4d-a5af3160.pth", + "resnext101_64x4d": "https://download.openmmlab.com/pretrain/third_party/resnext101_64x4d-ee2c6f71.pth", + "contrib/resnet50_gn": "https://download.openmmlab.com/pretrain/third_party/resnet50_gn_thangvubk-ad1730dd.pth", + "detectron/resnet50_gn": "https://download.openmmlab.com/pretrain/third_party/resnet50_gn-9186a21c.pth", + "detectron/resnet101_gn": "https://download.openmmlab.com/pretrain/third_party/resnet101_gn-cac0ab98.pth", + "jhu/resnet50_gn_ws": "https://download.openmmlab.com/pretrain/third_party/resnet50_gn_ws-15beedd8.pth", + "jhu/resnet101_gn_ws": "https://download.openmmlab.com/pretrain/third_party/resnet101_gn_ws-3e3c308c.pth", + "jhu/resnext50_32x4d_gn_ws": "https://download.openmmlab.com/pretrain/third_party/resnext50_32x4d_gn_ws-0d87ac85.pth", + "jhu/resnext101_32x4d_gn_ws": "https://download.openmmlab.com/pretrain/third_party/resnext101_32x4d_gn_ws-34ac1a9e.pth", + "jhu/resnext50_32x4d_gn": "https://download.openmmlab.com/pretrain/third_party/resnext50_32x4d_gn-c7e8b754.pth", + "jhu/resnext101_32x4d_gn": "https://download.openmmlab.com/pretrain/third_party/resnext101_32x4d_gn-ac3bb84e.pth", + "msra/hrnetv2_w18_small": "https://download.openmmlab.com/pretrain/third_party/hrnetv2_w18_small-b5a04e21.pth", + "msra/hrnetv2_w18": "https://download.openmmlab.com/pretrain/third_party/hrnetv2_w18-00eb2006.pth", + "msra/hrnetv2_w32": "https://download.openmmlab.com/pretrain/third_party/hrnetv2_w32-dc9eeb4f.pth", + "msra/hrnetv2_w40": "https://download.openmmlab.com/pretrain/third_party/hrnetv2_w40-ed0b031c.pth", + "msra/hrnetv2_w48": "https://download.openmmlab.com/pretrain/third_party/hrnetv2_w48-d2186c55.pth", + "bninception_caffe": "https://download.openmmlab.com/pretrain/third_party/bn_inception_caffe-ed2e8665.pth", + "kin400/i3d_r50_f32s2_k400": "https://download.openmmlab.com/pretrain/third_party/i3d_r50_f32s2_k400-2c57e077.pth", + "kin400/nl3d_r50_f32s2_k400": "https://download.openmmlab.com/pretrain/third_party/nl3d_r50_f32s2_k400-fa7e7caa.pth", + "res2net101_v1d_26w_4s": "https://download.openmmlab.com/pretrain/third_party/res2net101_v1d_26w_4s_mmdetv2-f0a600f9.pth", + "regnetx_400mf": "https://download.openmmlab.com/pretrain/third_party/regnetx_400mf-a5b10d96.pth", + "regnetx_800mf": "https://download.openmmlab.com/pretrain/third_party/regnetx_800mf-1f4be4c7.pth", + "regnetx_1.6gf": "https://download.openmmlab.com/pretrain/third_party/regnetx_1.6gf-5791c176.pth", + "regnetx_3.2gf": "https://download.openmmlab.com/pretrain/third_party/regnetx_3.2gf-c2599b0f.pth", + "regnetx_4.0gf": "https://download.openmmlab.com/pretrain/third_party/regnetx_4.0gf-a88f671e.pth", + "regnetx_6.4gf": "https://download.openmmlab.com/pretrain/third_party/regnetx_6.4gf-006af45d.pth", + "regnetx_8.0gf": "https://download.openmmlab.com/pretrain/third_party/regnetx_8.0gf-3c68abe7.pth", + "regnetx_12gf": "https://download.openmmlab.com/pretrain/third_party/regnetx_12gf-4c2a3350.pth", + "resnet18_v1c": "https://download.openmmlab.com/pretrain/third_party/resnet18_v1c-b5776b93.pth", + "resnet50_v1c": "https://download.openmmlab.com/pretrain/third_party/resnet50_v1c-2cccc1ad.pth", + "resnet101_v1c": "https://download.openmmlab.com/pretrain/third_party/resnet101_v1c-e67eebb6.pth", + "mmedit/vgg16": "https://download.openmmlab.com/mmediting/third_party/vgg_state_dict.pth", + "mmedit/res34_en_nomixup": "https://download.openmmlab.com/mmediting/third_party/model_best_resnet34_En_nomixup.pth", + "mmedit/mobilenet_v2": "https://download.openmmlab.com/mmediting/third_party/mobilenet_v2.pth", + "contrib/mobilenet_v3_large": "https://download.openmmlab.com/pretrain/third_party/mobilenet_v3_large-bc2c3fd3.pth", + "contrib/mobilenet_v3_small": "https://download.openmmlab.com/pretrain/third_party/mobilenet_v3_small-47085aa1.pth", + "resnest50": "https://download.openmmlab.com/pretrain/third_party/resnest50_d2-7497a55b.pth", + "resnest101": "https://download.openmmlab.com/pretrain/third_party/resnest101_d2-f3b931b2.pth", + "resnest200": "https://download.openmmlab.com/pretrain/third_party/resnest200_d2-ca88e41f.pth", + "darknet53": "https://download.openmmlab.com/pretrain/third_party/darknet53-a628ea1b.pth", + "mmdet/mobilenet_v2": "https://download.openmmlab.com/mmdetection/v2.0/third_party/mobilenet_v2_batch256_imagenet-ff34753d.pth" +} diff --git a/annotator/uniformer/mmcv/ops/__init__.py b/annotator/uniformer/mmcv/ops/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..999e090a458ee148ceca0649f1e3806a40e909bd --- /dev/null +++ b/annotator/uniformer/mmcv/ops/__init__.py @@ -0,0 +1,81 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .assign_score_withk import assign_score_withk +from .ball_query import ball_query +from .bbox import bbox_overlaps +from .border_align import BorderAlign, border_align +from .box_iou_rotated import box_iou_rotated +from .carafe import CARAFE, CARAFENaive, CARAFEPack, carafe, carafe_naive +from .cc_attention import CrissCrossAttention +from .contour_expand import contour_expand +from .corner_pool import CornerPool +from .correlation import Correlation +from .deform_conv import DeformConv2d, DeformConv2dPack, deform_conv2d +from .deform_roi_pool import (DeformRoIPool, DeformRoIPoolPack, + ModulatedDeformRoIPoolPack, deform_roi_pool) +from .deprecated_wrappers import Conv2d_deprecated as Conv2d +from .deprecated_wrappers import ConvTranspose2d_deprecated as ConvTranspose2d +from .deprecated_wrappers import Linear_deprecated as Linear +from .deprecated_wrappers import MaxPool2d_deprecated as MaxPool2d +from .focal_loss import (SigmoidFocalLoss, SoftmaxFocalLoss, + sigmoid_focal_loss, softmax_focal_loss) +from .furthest_point_sample import (furthest_point_sample, + furthest_point_sample_with_dist) +from .fused_bias_leakyrelu import FusedBiasLeakyReLU, fused_bias_leakyrelu +from .gather_points import gather_points +from .group_points import GroupAll, QueryAndGroup, grouping_operation +from .info import (get_compiler_version, get_compiling_cuda_version, + get_onnxruntime_op_path) +from .iou3d import boxes_iou_bev, nms_bev, nms_normal_bev +from .knn import knn +from .masked_conv import MaskedConv2d, masked_conv2d +from .modulated_deform_conv import (ModulatedDeformConv2d, + ModulatedDeformConv2dPack, + modulated_deform_conv2d) +from .multi_scale_deform_attn import MultiScaleDeformableAttention +from .nms import batched_nms, nms, nms_match, nms_rotated, soft_nms +from .pixel_group import pixel_group +from .point_sample import (SimpleRoIAlign, point_sample, + rel_roi_point_to_rel_img_point) +from .points_in_boxes import (points_in_boxes_all, points_in_boxes_cpu, + points_in_boxes_part) +from .points_sampler import PointsSampler +from .psa_mask import PSAMask +from .roi_align import RoIAlign, roi_align +from .roi_align_rotated import RoIAlignRotated, roi_align_rotated +from .roi_pool import RoIPool, roi_pool +from .roiaware_pool3d import RoIAwarePool3d +from .roipoint_pool3d import RoIPointPool3d +from .saconv import SAConv2d +from .scatter_points import DynamicScatter, dynamic_scatter +from .sync_bn import SyncBatchNorm +from .three_interpolate import three_interpolate +from .three_nn import three_nn +from .tin_shift import TINShift, tin_shift +from .upfirdn2d import upfirdn2d +from .voxelize import Voxelization, voxelization + +__all__ = [ + 'bbox_overlaps', 'CARAFE', 'CARAFENaive', 'CARAFEPack', 'carafe', + 'carafe_naive', 'CornerPool', 'DeformConv2d', 'DeformConv2dPack', + 'deform_conv2d', 'DeformRoIPool', 'DeformRoIPoolPack', + 'ModulatedDeformRoIPoolPack', 'deform_roi_pool', 'SigmoidFocalLoss', + 'SoftmaxFocalLoss', 'sigmoid_focal_loss', 'softmax_focal_loss', + 'get_compiler_version', 'get_compiling_cuda_version', + 'get_onnxruntime_op_path', 'MaskedConv2d', 'masked_conv2d', + 'ModulatedDeformConv2d', 'ModulatedDeformConv2dPack', + 'modulated_deform_conv2d', 'batched_nms', 'nms', 'soft_nms', 'nms_match', + 'RoIAlign', 'roi_align', 'RoIPool', 'roi_pool', 'SyncBatchNorm', 'Conv2d', + 'ConvTranspose2d', 'Linear', 'MaxPool2d', 'CrissCrossAttention', 'PSAMask', + 'point_sample', 'rel_roi_point_to_rel_img_point', 'SimpleRoIAlign', + 'SAConv2d', 'TINShift', 'tin_shift', 'assign_score_withk', + 'box_iou_rotated', 'RoIPointPool3d', 'nms_rotated', 'knn', 'ball_query', + 'upfirdn2d', 'FusedBiasLeakyReLU', 'fused_bias_leakyrelu', + 'RoIAlignRotated', 'roi_align_rotated', 'pixel_group', 'QueryAndGroup', + 'GroupAll', 'grouping_operation', 'contour_expand', 'three_nn', + 'three_interpolate', 'MultiScaleDeformableAttention', 'BorderAlign', + 'border_align', 'gather_points', 'furthest_point_sample', + 'furthest_point_sample_with_dist', 'PointsSampler', 'Correlation', + 'boxes_iou_bev', 'nms_bev', 'nms_normal_bev', 'Voxelization', + 'voxelization', 'dynamic_scatter', 'DynamicScatter', 'RoIAwarePool3d', + 'points_in_boxes_part', 'points_in_boxes_cpu', 'points_in_boxes_all' +] diff --git a/annotator/uniformer/mmcv/ops/assign_score_withk.py b/annotator/uniformer/mmcv/ops/assign_score_withk.py new file mode 100644 index 0000000000000000000000000000000000000000..4906adaa2cffd1b46912fbe7d4f87ef2f9fa0012 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/assign_score_withk.py @@ -0,0 +1,123 @@ +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['assign_score_withk_forward', 'assign_score_withk_backward']) + + +class AssignScoreWithK(Function): + r"""Perform weighted sum to generate output features according to scores. + Modified from `PAConv `_. + + This is a memory-efficient CUDA implementation of assign_scores operation, + which first transform all point features with weight bank, then assemble + neighbor features with ``knn_idx`` and perform weighted sum of ``scores``. + + See the `paper `_ appendix Sec. D for + more detailed descriptions. + + Note: + This implementation assumes using ``neighbor`` kernel input, which is + (point_features - center_features, point_features). + See https://github.com/CVMI-Lab/PAConv/blob/main/scene_seg/model/ + pointnet2/paconv.py#L128 for more details. + """ + + @staticmethod + def forward(ctx, + scores, + point_features, + center_features, + knn_idx, + aggregate='sum'): + """ + Args: + scores (torch.Tensor): (B, npoint, K, M), predicted scores to + aggregate weight matrices in the weight bank. + ``npoint`` is the number of sampled centers. + ``K`` is the number of queried neighbors. + ``M`` is the number of weight matrices in the weight bank. + point_features (torch.Tensor): (B, N, M, out_dim) + Pre-computed point features to be aggregated. + center_features (torch.Tensor): (B, N, M, out_dim) + Pre-computed center features to be aggregated. + knn_idx (torch.Tensor): (B, npoint, K), index of sampled kNN. + We assume the first idx in each row is the idx of the center. + aggregate (str, optional): Aggregation method. + Can be 'sum', 'avg' or 'max'. Defaults: 'sum'. + + Returns: + torch.Tensor: (B, out_dim, npoint, K), the aggregated features. + """ + agg = {'sum': 0, 'avg': 1, 'max': 2} + + B, N, M, out_dim = point_features.size() + _, npoint, K, _ = scores.size() + + output = point_features.new_zeros((B, out_dim, npoint, K)) + ext_module.assign_score_withk_forward( + point_features.contiguous(), + center_features.contiguous(), + scores.contiguous(), + knn_idx.contiguous(), + output, + B=B, + N0=N, + N1=npoint, + M=M, + K=K, + O=out_dim, + aggregate=agg[aggregate]) + + ctx.save_for_backward(output, point_features, center_features, scores, + knn_idx) + ctx.agg = agg[aggregate] + + return output + + @staticmethod + def backward(ctx, grad_out): + """ + Args: + grad_out (torch.Tensor): (B, out_dim, npoint, K) + + Returns: + grad_scores (torch.Tensor): (B, npoint, K, M) + grad_point_features (torch.Tensor): (B, N, M, out_dim) + grad_center_features (torch.Tensor): (B, N, M, out_dim) + """ + _, point_features, center_features, scores, knn_idx = ctx.saved_tensors + + agg = ctx.agg + + B, N, M, out_dim = point_features.size() + _, npoint, K, _ = scores.size() + + grad_point_features = point_features.new_zeros(point_features.shape) + grad_center_features = center_features.new_zeros(center_features.shape) + grad_scores = scores.new_zeros(scores.shape) + + ext_module.assign_score_withk_backward( + grad_out.contiguous(), + point_features.contiguous(), + center_features.contiguous(), + scores.contiguous(), + knn_idx.contiguous(), + grad_point_features, + grad_center_features, + grad_scores, + B=B, + N0=N, + N1=npoint, + M=M, + K=K, + O=out_dim, + aggregate=agg) + + return grad_scores, grad_point_features, \ + grad_center_features, None, None + + +assign_score_withk = AssignScoreWithK.apply diff --git a/annotator/uniformer/mmcv/ops/ball_query.py b/annotator/uniformer/mmcv/ops/ball_query.py new file mode 100644 index 0000000000000000000000000000000000000000..d0466847c6e5c1239e359a0397568413ebc1504a --- /dev/null +++ b/annotator/uniformer/mmcv/ops/ball_query.py @@ -0,0 +1,55 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', ['ball_query_forward']) + + +class BallQuery(Function): + """Find nearby points in spherical space.""" + + @staticmethod + def forward(ctx, min_radius: float, max_radius: float, sample_num: int, + xyz: torch.Tensor, center_xyz: torch.Tensor) -> torch.Tensor: + """ + Args: + min_radius (float): minimum radius of the balls. + max_radius (float): maximum radius of the balls. + sample_num (int): maximum number of features in the balls. + xyz (Tensor): (B, N, 3) xyz coordinates of the features. + center_xyz (Tensor): (B, npoint, 3) centers of the ball query. + + Returns: + Tensor: (B, npoint, nsample) tensor with the indices of + the features that form the query balls. + """ + assert center_xyz.is_contiguous() + assert xyz.is_contiguous() + assert min_radius < max_radius + + B, N, _ = xyz.size() + npoint = center_xyz.size(1) + idx = xyz.new_zeros(B, npoint, sample_num, dtype=torch.int) + + ext_module.ball_query_forward( + center_xyz, + xyz, + idx, + b=B, + n=N, + m=npoint, + min_radius=min_radius, + max_radius=max_radius, + nsample=sample_num) + if torch.__version__ != 'parrots': + ctx.mark_non_differentiable(idx) + return idx + + @staticmethod + def backward(ctx, a=None): + return None, None, None, None + + +ball_query = BallQuery.apply diff --git a/annotator/uniformer/mmcv/ops/bbox.py b/annotator/uniformer/mmcv/ops/bbox.py new file mode 100644 index 0000000000000000000000000000000000000000..0c4d58b6c91f652933974f519acd3403a833e906 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/bbox.py @@ -0,0 +1,72 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', ['bbox_overlaps']) + + +def bbox_overlaps(bboxes1, bboxes2, mode='iou', aligned=False, offset=0): + """Calculate overlap between two set of bboxes. + + If ``aligned`` is ``False``, then calculate the ious between each bbox + of bboxes1 and bboxes2, otherwise the ious between each aligned pair of + bboxes1 and bboxes2. + + Args: + bboxes1 (Tensor): shape (m, 4) in format or empty. + bboxes2 (Tensor): shape (n, 4) in format or empty. + If aligned is ``True``, then m and n must be equal. + mode (str): "iou" (intersection over union) or iof (intersection over + foreground). + + Returns: + ious(Tensor): shape (m, n) if aligned == False else shape (m, 1) + + Example: + >>> bboxes1 = torch.FloatTensor([ + >>> [0, 0, 10, 10], + >>> [10, 10, 20, 20], + >>> [32, 32, 38, 42], + >>> ]) + >>> bboxes2 = torch.FloatTensor([ + >>> [0, 0, 10, 20], + >>> [0, 10, 10, 19], + >>> [10, 10, 20, 20], + >>> ]) + >>> bbox_overlaps(bboxes1, bboxes2) + tensor([[0.5000, 0.0000, 0.0000], + [0.0000, 0.0000, 1.0000], + [0.0000, 0.0000, 0.0000]]) + + Example: + >>> empty = torch.FloatTensor([]) + >>> nonempty = torch.FloatTensor([ + >>> [0, 0, 10, 9], + >>> ]) + >>> assert tuple(bbox_overlaps(empty, nonempty).shape) == (0, 1) + >>> assert tuple(bbox_overlaps(nonempty, empty).shape) == (1, 0) + >>> assert tuple(bbox_overlaps(empty, empty).shape) == (0, 0) + """ + + mode_dict = {'iou': 0, 'iof': 1} + assert mode in mode_dict.keys() + mode_flag = mode_dict[mode] + # Either the boxes are empty or the length of boxes' last dimension is 4 + assert (bboxes1.size(-1) == 4 or bboxes1.size(0) == 0) + assert (bboxes2.size(-1) == 4 or bboxes2.size(0) == 0) + assert offset == 1 or offset == 0 + + rows = bboxes1.size(0) + cols = bboxes2.size(0) + if aligned: + assert rows == cols + + if rows * cols == 0: + return bboxes1.new(rows, 1) if aligned else bboxes1.new(rows, cols) + + if aligned: + ious = bboxes1.new_zeros(rows) + else: + ious = bboxes1.new_zeros((rows, cols)) + ext_module.bbox_overlaps( + bboxes1, bboxes2, ious, mode=mode_flag, aligned=aligned, offset=offset) + return ious diff --git a/annotator/uniformer/mmcv/ops/border_align.py b/annotator/uniformer/mmcv/ops/border_align.py new file mode 100644 index 0000000000000000000000000000000000000000..ff305be328e9b0a15e1bbb5e6b41beb940f55c81 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/border_align.py @@ -0,0 +1,109 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# modified from +# https://github.com/Megvii-BaseDetection/cvpods/blob/master/cvpods/layers/border_align.py + +import torch +import torch.nn as nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['border_align_forward', 'border_align_backward']) + + +class BorderAlignFunction(Function): + + @staticmethod + def symbolic(g, input, boxes, pool_size): + return g.op( + 'mmcv::MMCVBorderAlign', input, boxes, pool_size_i=pool_size) + + @staticmethod + def forward(ctx, input, boxes, pool_size): + ctx.pool_size = pool_size + ctx.input_shape = input.size() + + assert boxes.ndim == 3, 'boxes must be with shape [B, H*W, 4]' + assert boxes.size(2) == 4, \ + 'the last dimension of boxes must be (x1, y1, x2, y2)' + assert input.size(1) % 4 == 0, \ + 'the channel for input feature must be divisible by factor 4' + + # [B, C//4, H*W, 4] + output_shape = (input.size(0), input.size(1) // 4, boxes.size(1), 4) + output = input.new_zeros(output_shape) + # `argmax_idx` only used for backward + argmax_idx = input.new_zeros(output_shape).to(torch.int) + + ext_module.border_align_forward( + input, boxes, output, argmax_idx, pool_size=ctx.pool_size) + + ctx.save_for_backward(boxes, argmax_idx) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + boxes, argmax_idx = ctx.saved_tensors + grad_input = grad_output.new_zeros(ctx.input_shape) + # complex head architecture may cause grad_output uncontiguous + grad_output = grad_output.contiguous() + ext_module.border_align_backward( + grad_output, + boxes, + argmax_idx, + grad_input, + pool_size=ctx.pool_size) + return grad_input, None, None + + +border_align = BorderAlignFunction.apply + + +class BorderAlign(nn.Module): + r"""Border align pooling layer. + + Applies border_align over the input feature based on predicted bboxes. + The details were described in the paper + `BorderDet: Border Feature for Dense Object Detection + `_. + + For each border line (e.g. top, left, bottom or right) of each box, + border_align does the following: + 1. uniformly samples `pool_size`+1 positions on this line, involving \ + the start and end points. + 2. the corresponding features on these points are computed by \ + bilinear interpolation. + 3. max pooling over all the `pool_size`+1 positions are used for \ + computing pooled feature. + + Args: + pool_size (int): number of positions sampled over the boxes' borders + (e.g. top, bottom, left, right). + + """ + + def __init__(self, pool_size): + super(BorderAlign, self).__init__() + self.pool_size = pool_size + + def forward(self, input, boxes): + """ + Args: + input: Features with shape [N,4C,H,W]. Channels ranged in [0,C), + [C,2C), [2C,3C), [3C,4C) represent the top, left, bottom, + right features respectively. + boxes: Boxes with shape [N,H*W,4]. Coordinate format (x1,y1,x2,y2). + + Returns: + Tensor: Pooled features with shape [N,C,H*W,4]. The order is + (top,left,bottom,right) for the last dimension. + """ + return border_align(input, boxes, self.pool_size) + + def __repr__(self): + s = self.__class__.__name__ + s += f'(pool_size={self.pool_size})' + return s diff --git a/annotator/uniformer/mmcv/ops/box_iou_rotated.py b/annotator/uniformer/mmcv/ops/box_iou_rotated.py new file mode 100644 index 0000000000000000000000000000000000000000..2d78015e9c2a9e7a52859b4e18f84a9aa63481a0 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/box_iou_rotated.py @@ -0,0 +1,45 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', ['box_iou_rotated']) + + +def box_iou_rotated(bboxes1, bboxes2, mode='iou', aligned=False): + """Return intersection-over-union (Jaccard index) of boxes. + + Both sets of boxes are expected to be in + (x_center, y_center, width, height, angle) format. + + If ``aligned`` is ``False``, then calculate the ious between each bbox + of bboxes1 and bboxes2, otherwise the ious between each aligned pair of + bboxes1 and bboxes2. + + Arguments: + boxes1 (Tensor): rotated bboxes 1. \ + It has shape (N, 5), indicating (x, y, w, h, theta) for each row. + Note that theta is in radian. + boxes2 (Tensor): rotated bboxes 2. \ + It has shape (M, 5), indicating (x, y, w, h, theta) for each row. + Note that theta is in radian. + mode (str): "iou" (intersection over union) or iof (intersection over + foreground). + + Returns: + ious(Tensor): shape (N, M) if aligned == False else shape (N,) + """ + assert mode in ['iou', 'iof'] + mode_dict = {'iou': 0, 'iof': 1} + mode_flag = mode_dict[mode] + rows = bboxes1.size(0) + cols = bboxes2.size(0) + if aligned: + ious = bboxes1.new_zeros(rows) + else: + ious = bboxes1.new_zeros((rows * cols)) + bboxes1 = bboxes1.contiguous() + bboxes2 = bboxes2.contiguous() + ext_module.box_iou_rotated( + bboxes1, bboxes2, ious, mode_flag=mode_flag, aligned=aligned) + if not aligned: + ious = ious.view(rows, cols) + return ious diff --git a/annotator/uniformer/mmcv/ops/carafe.py b/annotator/uniformer/mmcv/ops/carafe.py new file mode 100644 index 0000000000000000000000000000000000000000..5154cb3abfccfbbe0a1b2daa67018dbf80aaf6d2 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/carafe.py @@ -0,0 +1,287 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd import Function +from torch.nn.modules.module import Module + +from ..cnn import UPSAMPLE_LAYERS, normal_init, xavier_init +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', [ + 'carafe_naive_forward', 'carafe_naive_backward', 'carafe_forward', + 'carafe_backward' +]) + + +class CARAFENaiveFunction(Function): + + @staticmethod + def symbolic(g, features, masks, kernel_size, group_size, scale_factor): + return g.op( + 'mmcv::MMCVCARAFENaive', + features, + masks, + kernel_size_i=kernel_size, + group_size_i=group_size, + scale_factor_f=scale_factor) + + @staticmethod + def forward(ctx, features, masks, kernel_size, group_size, scale_factor): + assert scale_factor >= 1 + assert masks.size(1) == kernel_size * kernel_size * group_size + assert masks.size(-1) == features.size(-1) * scale_factor + assert masks.size(-2) == features.size(-2) * scale_factor + assert features.size(1) % group_size == 0 + assert (kernel_size - 1) % 2 == 0 and kernel_size >= 1 + ctx.kernel_size = kernel_size + ctx.group_size = group_size + ctx.scale_factor = scale_factor + ctx.feature_size = features.size() + ctx.mask_size = masks.size() + + n, c, h, w = features.size() + output = features.new_zeros((n, c, h * scale_factor, w * scale_factor)) + ext_module.carafe_naive_forward( + features, + masks, + output, + kernel_size=kernel_size, + group_size=group_size, + scale_factor=scale_factor) + + if features.requires_grad or masks.requires_grad: + ctx.save_for_backward(features, masks) + return output + + @staticmethod + def backward(ctx, grad_output): + assert grad_output.is_cuda + + features, masks = ctx.saved_tensors + kernel_size = ctx.kernel_size + group_size = ctx.group_size + scale_factor = ctx.scale_factor + + grad_input = torch.zeros_like(features) + grad_masks = torch.zeros_like(masks) + ext_module.carafe_naive_backward( + grad_output.contiguous(), + features, + masks, + grad_input, + grad_masks, + kernel_size=kernel_size, + group_size=group_size, + scale_factor=scale_factor) + + return grad_input, grad_masks, None, None, None + + +carafe_naive = CARAFENaiveFunction.apply + + +class CARAFENaive(Module): + + def __init__(self, kernel_size, group_size, scale_factor): + super(CARAFENaive, self).__init__() + + assert isinstance(kernel_size, int) and isinstance( + group_size, int) and isinstance(scale_factor, int) + self.kernel_size = kernel_size + self.group_size = group_size + self.scale_factor = scale_factor + + def forward(self, features, masks): + return carafe_naive(features, masks, self.kernel_size, self.group_size, + self.scale_factor) + + +class CARAFEFunction(Function): + + @staticmethod + def symbolic(g, features, masks, kernel_size, group_size, scale_factor): + return g.op( + 'mmcv::MMCVCARAFE', + features, + masks, + kernel_size_i=kernel_size, + group_size_i=group_size, + scale_factor_f=scale_factor) + + @staticmethod + def forward(ctx, features, masks, kernel_size, group_size, scale_factor): + assert scale_factor >= 1 + assert masks.size(1) == kernel_size * kernel_size * group_size + assert masks.size(-1) == features.size(-1) * scale_factor + assert masks.size(-2) == features.size(-2) * scale_factor + assert features.size(1) % group_size == 0 + assert (kernel_size - 1) % 2 == 0 and kernel_size >= 1 + ctx.kernel_size = kernel_size + ctx.group_size = group_size + ctx.scale_factor = scale_factor + ctx.feature_size = features.size() + ctx.mask_size = masks.size() + + n, c, h, w = features.size() + output = features.new_zeros((n, c, h * scale_factor, w * scale_factor)) + routput = features.new_zeros(output.size(), requires_grad=False) + rfeatures = features.new_zeros(features.size(), requires_grad=False) + rmasks = masks.new_zeros(masks.size(), requires_grad=False) + ext_module.carafe_forward( + features, + masks, + rfeatures, + routput, + rmasks, + output, + kernel_size=kernel_size, + group_size=group_size, + scale_factor=scale_factor) + + if features.requires_grad or masks.requires_grad: + ctx.save_for_backward(features, masks, rfeatures) + return output + + @staticmethod + def backward(ctx, grad_output): + assert grad_output.is_cuda + + features, masks, rfeatures = ctx.saved_tensors + kernel_size = ctx.kernel_size + group_size = ctx.group_size + scale_factor = ctx.scale_factor + + rgrad_output = torch.zeros_like(grad_output, requires_grad=False) + rgrad_input_hs = torch.zeros_like(grad_output, requires_grad=False) + rgrad_input = torch.zeros_like(features, requires_grad=False) + rgrad_masks = torch.zeros_like(masks, requires_grad=False) + grad_input = torch.zeros_like(features, requires_grad=False) + grad_masks = torch.zeros_like(masks, requires_grad=False) + ext_module.carafe_backward( + grad_output.contiguous(), + rfeatures, + masks, + rgrad_output, + rgrad_input_hs, + rgrad_input, + rgrad_masks, + grad_input, + grad_masks, + kernel_size=kernel_size, + group_size=group_size, + scale_factor=scale_factor) + return grad_input, grad_masks, None, None, None + + +carafe = CARAFEFunction.apply + + +class CARAFE(Module): + """ CARAFE: Content-Aware ReAssembly of FEatures + + Please refer to https://arxiv.org/abs/1905.02188 for more details. + + Args: + kernel_size (int): reassemble kernel size + group_size (int): reassemble group size + scale_factor (int): upsample ratio + + Returns: + upsampled feature map + """ + + def __init__(self, kernel_size, group_size, scale_factor): + super(CARAFE, self).__init__() + + assert isinstance(kernel_size, int) and isinstance( + group_size, int) and isinstance(scale_factor, int) + self.kernel_size = kernel_size + self.group_size = group_size + self.scale_factor = scale_factor + + def forward(self, features, masks): + return carafe(features, masks, self.kernel_size, self.group_size, + self.scale_factor) + + +@UPSAMPLE_LAYERS.register_module(name='carafe') +class CARAFEPack(nn.Module): + """A unified package of CARAFE upsampler that contains: 1) channel + compressor 2) content encoder 3) CARAFE op. + + Official implementation of ICCV 2019 paper + CARAFE: Content-Aware ReAssembly of FEatures + Please refer to https://arxiv.org/abs/1905.02188 for more details. + + Args: + channels (int): input feature channels + scale_factor (int): upsample ratio + up_kernel (int): kernel size of CARAFE op + up_group (int): group size of CARAFE op + encoder_kernel (int): kernel size of content encoder + encoder_dilation (int): dilation of content encoder + compressed_channels (int): output channels of channels compressor + + Returns: + upsampled feature map + """ + + def __init__(self, + channels, + scale_factor, + up_kernel=5, + up_group=1, + encoder_kernel=3, + encoder_dilation=1, + compressed_channels=64): + super(CARAFEPack, self).__init__() + self.channels = channels + self.scale_factor = scale_factor + self.up_kernel = up_kernel + self.up_group = up_group + self.encoder_kernel = encoder_kernel + self.encoder_dilation = encoder_dilation + self.compressed_channels = compressed_channels + self.channel_compressor = nn.Conv2d(channels, self.compressed_channels, + 1) + self.content_encoder = nn.Conv2d( + self.compressed_channels, + self.up_kernel * self.up_kernel * self.up_group * + self.scale_factor * self.scale_factor, + self.encoder_kernel, + padding=int((self.encoder_kernel - 1) * self.encoder_dilation / 2), + dilation=self.encoder_dilation, + groups=1) + self.init_weights() + + def init_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + xavier_init(m, distribution='uniform') + normal_init(self.content_encoder, std=0.001) + + def kernel_normalizer(self, mask): + mask = F.pixel_shuffle(mask, self.scale_factor) + n, mask_c, h, w = mask.size() + # use float division explicitly, + # to void inconsistency while exporting to onnx + mask_channel = int(mask_c / float(self.up_kernel**2)) + mask = mask.view(n, mask_channel, -1, h, w) + + mask = F.softmax(mask, dim=2, dtype=mask.dtype) + mask = mask.view(n, mask_c, h, w).contiguous() + + return mask + + def feature_reassemble(self, x, mask): + x = carafe(x, mask, self.up_kernel, self.up_group, self.scale_factor) + return x + + def forward(self, x): + compressed_x = self.channel_compressor(x) + mask = self.content_encoder(compressed_x) + mask = self.kernel_normalizer(mask) + + x = self.feature_reassemble(x, mask) + return x diff --git a/annotator/uniformer/mmcv/ops/cc_attention.py b/annotator/uniformer/mmcv/ops/cc_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..9207aa95e6730bd9b3362dee612059a5f0ce1c5e --- /dev/null +++ b/annotator/uniformer/mmcv/ops/cc_attention.py @@ -0,0 +1,83 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from annotator.uniformer.mmcv.cnn import PLUGIN_LAYERS, Scale + + +def NEG_INF_DIAG(n, device): + """Returns a diagonal matrix of size [n, n]. + + The diagonal are all "-inf". This is for avoiding calculating the + overlapped element in the Criss-Cross twice. + """ + return torch.diag(torch.tensor(float('-inf')).to(device).repeat(n), 0) + + +@PLUGIN_LAYERS.register_module() +class CrissCrossAttention(nn.Module): + """Criss-Cross Attention Module. + + .. note:: + Before v1.3.13, we use a CUDA op. Since v1.3.13, we switch + to a pure PyTorch and equivalent implementation. For more + details, please refer to https://github.com/open-mmlab/mmcv/pull/1201. + + Speed comparison for one forward pass + + - Input size: [2,512,97,97] + - Device: 1 NVIDIA GeForce RTX 2080 Ti + + +-----------------------+---------------+------------+---------------+ + | |PyTorch version|CUDA version|Relative speed | + +=======================+===============+============+===============+ + |with torch.no_grad() |0.00554402 s |0.0299619 s |5.4x | + +-----------------------+---------------+------------+---------------+ + |no with torch.no_grad()|0.00562803 s |0.0301349 s |5.4x | + +-----------------------+---------------+------------+---------------+ + + Args: + in_channels (int): Channels of the input feature map. + """ + + def __init__(self, in_channels): + super().__init__() + self.query_conv = nn.Conv2d(in_channels, in_channels // 8, 1) + self.key_conv = nn.Conv2d(in_channels, in_channels // 8, 1) + self.value_conv = nn.Conv2d(in_channels, in_channels, 1) + self.gamma = Scale(0.) + self.in_channels = in_channels + + def forward(self, x): + """forward function of Criss-Cross Attention. + + Args: + x (Tensor): Input feature. \ + shape (batch_size, in_channels, height, width) + Returns: + Tensor: Output of the layer, with shape of \ + (batch_size, in_channels, height, width) + """ + B, C, H, W = x.size() + query = self.query_conv(x) + key = self.key_conv(x) + value = self.value_conv(x) + energy_H = torch.einsum('bchw,bciw->bwhi', query, key) + NEG_INF_DIAG( + H, query.device) + energy_H = energy_H.transpose(1, 2) + energy_W = torch.einsum('bchw,bchj->bhwj', query, key) + attn = F.softmax( + torch.cat([energy_H, energy_W], dim=-1), dim=-1) # [B,H,W,(H+W)] + out = torch.einsum('bciw,bhwi->bchw', value, attn[..., :H]) + out += torch.einsum('bchj,bhwj->bchw', value, attn[..., H:]) + + out = self.gamma(out) + x + out = out.contiguous() + + return out + + def __repr__(self): + s = self.__class__.__name__ + s += f'(in_channels={self.in_channels})' + return s diff --git a/annotator/uniformer/mmcv/ops/contour_expand.py b/annotator/uniformer/mmcv/ops/contour_expand.py new file mode 100644 index 0000000000000000000000000000000000000000..ea1111e1768b5f27e118bf7dbc0d9c70a7afd6d7 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/contour_expand.py @@ -0,0 +1,49 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numpy as np +import torch + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', ['contour_expand']) + + +def contour_expand(kernel_mask, internal_kernel_label, min_kernel_area, + kernel_num): + """Expand kernel contours so that foreground pixels are assigned into + instances. + + Arguments: + kernel_mask (np.array or Tensor): The instance kernel mask with + size hxw. + internal_kernel_label (np.array or Tensor): The instance internal + kernel label with size hxw. + min_kernel_area (int): The minimum kernel area. + kernel_num (int): The instance kernel number. + + Returns: + label (list): The instance index map with size hxw. + """ + assert isinstance(kernel_mask, (torch.Tensor, np.ndarray)) + assert isinstance(internal_kernel_label, (torch.Tensor, np.ndarray)) + assert isinstance(min_kernel_area, int) + assert isinstance(kernel_num, int) + + if isinstance(kernel_mask, np.ndarray): + kernel_mask = torch.from_numpy(kernel_mask) + if isinstance(internal_kernel_label, np.ndarray): + internal_kernel_label = torch.from_numpy(internal_kernel_label) + + if torch.__version__ == 'parrots': + if kernel_mask.shape[0] == 0 or internal_kernel_label.shape[0] == 0: + label = [] + else: + label = ext_module.contour_expand( + kernel_mask, + internal_kernel_label, + min_kernel_area=min_kernel_area, + kernel_num=kernel_num) + label = label.tolist() + else: + label = ext_module.contour_expand(kernel_mask, internal_kernel_label, + min_kernel_area, kernel_num) + return label diff --git a/annotator/uniformer/mmcv/ops/corner_pool.py b/annotator/uniformer/mmcv/ops/corner_pool.py new file mode 100644 index 0000000000000000000000000000000000000000..a33d798b43d405e4c86bee4cd6389be21ca9c637 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/corner_pool.py @@ -0,0 +1,161 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch import nn +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', [ + 'top_pool_forward', 'top_pool_backward', 'bottom_pool_forward', + 'bottom_pool_backward', 'left_pool_forward', 'left_pool_backward', + 'right_pool_forward', 'right_pool_backward' +]) + +_mode_dict = {'top': 0, 'bottom': 1, 'left': 2, 'right': 3} + + +class TopPoolFunction(Function): + + @staticmethod + def symbolic(g, input): + output = g.op( + 'mmcv::MMCVCornerPool', input, mode_i=int(_mode_dict['top'])) + return output + + @staticmethod + def forward(ctx, input): + output = ext_module.top_pool_forward(input) + ctx.save_for_backward(input) + return output + + @staticmethod + def backward(ctx, grad_output): + input, = ctx.saved_tensors + output = ext_module.top_pool_backward(input, grad_output) + return output + + +class BottomPoolFunction(Function): + + @staticmethod + def symbolic(g, input): + output = g.op( + 'mmcv::MMCVCornerPool', input, mode_i=int(_mode_dict['bottom'])) + return output + + @staticmethod + def forward(ctx, input): + output = ext_module.bottom_pool_forward(input) + ctx.save_for_backward(input) + return output + + @staticmethod + def backward(ctx, grad_output): + input, = ctx.saved_tensors + output = ext_module.bottom_pool_backward(input, grad_output) + return output + + +class LeftPoolFunction(Function): + + @staticmethod + def symbolic(g, input): + output = g.op( + 'mmcv::MMCVCornerPool', input, mode_i=int(_mode_dict['left'])) + return output + + @staticmethod + def forward(ctx, input): + output = ext_module.left_pool_forward(input) + ctx.save_for_backward(input) + return output + + @staticmethod + def backward(ctx, grad_output): + input, = ctx.saved_tensors + output = ext_module.left_pool_backward(input, grad_output) + return output + + +class RightPoolFunction(Function): + + @staticmethod + def symbolic(g, input): + output = g.op( + 'mmcv::MMCVCornerPool', input, mode_i=int(_mode_dict['right'])) + return output + + @staticmethod + def forward(ctx, input): + output = ext_module.right_pool_forward(input) + ctx.save_for_backward(input) + return output + + @staticmethod + def backward(ctx, grad_output): + input, = ctx.saved_tensors + output = ext_module.right_pool_backward(input, grad_output) + return output + + +class CornerPool(nn.Module): + """Corner Pooling. + + Corner Pooling is a new type of pooling layer that helps a + convolutional network better localize corners of bounding boxes. + + Please refer to https://arxiv.org/abs/1808.01244 for more details. + Code is modified from https://github.com/princeton-vl/CornerNet-Lite. + + Args: + mode(str): Pooling orientation for the pooling layer + + - 'bottom': Bottom Pooling + - 'left': Left Pooling + - 'right': Right Pooling + - 'top': Top Pooling + + Returns: + Feature map after pooling. + """ + + pool_functions = { + 'bottom': BottomPoolFunction, + 'left': LeftPoolFunction, + 'right': RightPoolFunction, + 'top': TopPoolFunction, + } + + cummax_dim_flip = { + 'bottom': (2, False), + 'left': (3, True), + 'right': (3, False), + 'top': (2, True), + } + + def __init__(self, mode): + super(CornerPool, self).__init__() + assert mode in self.pool_functions + self.mode = mode + self.corner_pool = self.pool_functions[mode] + + def forward(self, x): + if torch.__version__ != 'parrots' and torch.__version__ >= '1.5.0': + if torch.onnx.is_in_onnx_export(): + assert torch.__version__ >= '1.7.0', \ + 'When `cummax` serves as an intermediate component whose '\ + 'outputs is used as inputs for another modules, it\'s '\ + 'expected that pytorch version must be >= 1.7.0, '\ + 'otherwise Error appears like: `RuntimeError: tuple '\ + 'appears in op that does not forward tuples, unsupported '\ + 'kind: prim::PythonOp`.' + + dim, flip = self.cummax_dim_flip[self.mode] + if flip: + x = x.flip(dim) + pool_tensor, _ = torch.cummax(x, dim=dim) + if flip: + pool_tensor = pool_tensor.flip(dim) + return pool_tensor + else: + return self.corner_pool.apply(x) diff --git a/annotator/uniformer/mmcv/ops/correlation.py b/annotator/uniformer/mmcv/ops/correlation.py new file mode 100644 index 0000000000000000000000000000000000000000..3d0b79c301b29915dfaf4d2b1846c59be73127d3 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/correlation.py @@ -0,0 +1,196 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch import Tensor, nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.utils import _pair + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['correlation_forward', 'correlation_backward']) + + +class CorrelationFunction(Function): + + @staticmethod + def forward(ctx, + input1, + input2, + kernel_size=1, + max_displacement=1, + stride=1, + padding=1, + dilation=1, + dilation_patch=1): + + ctx.save_for_backward(input1, input2) + + kH, kW = ctx.kernel_size = _pair(kernel_size) + patch_size = max_displacement * 2 + 1 + ctx.patch_size = patch_size + dH, dW = ctx.stride = _pair(stride) + padH, padW = ctx.padding = _pair(padding) + dilationH, dilationW = ctx.dilation = _pair(dilation) + dilation_patchH, dilation_patchW = ctx.dilation_patch = _pair( + dilation_patch) + + output_size = CorrelationFunction._output_size(ctx, input1) + + output = input1.new_zeros(output_size) + + ext_module.correlation_forward( + input1, + input2, + output, + kH=kH, + kW=kW, + patchH=patch_size, + patchW=patch_size, + padH=padH, + padW=padW, + dilationH=dilationH, + dilationW=dilationW, + dilation_patchH=dilation_patchH, + dilation_patchW=dilation_patchW, + dH=dH, + dW=dW) + + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + input1, input2 = ctx.saved_tensors + + kH, kW = ctx.kernel_size + patch_size = ctx.patch_size + padH, padW = ctx.padding + dilationH, dilationW = ctx.dilation + dilation_patchH, dilation_patchW = ctx.dilation_patch + dH, dW = ctx.stride + grad_input1 = torch.zeros_like(input1) + grad_input2 = torch.zeros_like(input2) + + ext_module.correlation_backward( + grad_output, + input1, + input2, + grad_input1, + grad_input2, + kH=kH, + kW=kW, + patchH=patch_size, + patchW=patch_size, + padH=padH, + padW=padW, + dilationH=dilationH, + dilationW=dilationW, + dilation_patchH=dilation_patchH, + dilation_patchW=dilation_patchW, + dH=dH, + dW=dW) + return grad_input1, grad_input2, None, None, None, None, None, None + + @staticmethod + def _output_size(ctx, input1): + iH, iW = input1.size(2), input1.size(3) + batch_size = input1.size(0) + kH, kW = ctx.kernel_size + patch_size = ctx.patch_size + dH, dW = ctx.stride + padH, padW = ctx.padding + dilationH, dilationW = ctx.dilation + dilatedKH = (kH - 1) * dilationH + 1 + dilatedKW = (kW - 1) * dilationW + 1 + + oH = int((iH + 2 * padH - dilatedKH) / dH + 1) + oW = int((iW + 2 * padW - dilatedKW) / dW + 1) + + output_size = (batch_size, patch_size, patch_size, oH, oW) + return output_size + + +class Correlation(nn.Module): + r"""Correlation operator + + This correlation operator works for optical flow correlation computation. + + There are two batched tensors with shape :math:`(N, C, H, W)`, + and the correlation output's shape is :math:`(N, max\_displacement \times + 2 + 1, max\_displacement * 2 + 1, H_{out}, W_{out})` + + where + + .. math:: + H_{out} = \left\lfloor\frac{H_{in} + 2 \times padding - + dilation \times (kernel\_size - 1) - 1} + {stride} + 1\right\rfloor + + .. math:: + W_{out} = \left\lfloor\frac{W_{in} + 2 \times padding - dilation + \times (kernel\_size - 1) - 1} + {stride} + 1\right\rfloor + + the correlation item :math:`(N_i, dy, dx)` is formed by taking the sliding + window convolution between input1 and shifted input2, + + .. math:: + Corr(N_i, dx, dy) = + \sum_{c=0}^{C-1} + input1(N_i, c) \star + \mathcal{S}(input2(N_i, c), dy, dx) + + where :math:`\star` is the valid 2d sliding window convolution operator, + and :math:`\mathcal{S}` means shifting the input features (auto-complete + zero marginal), and :math:`dx, dy` are shifting distance, :math:`dx, dy \in + [-max\_displacement \times dilation\_patch, max\_displacement \times + dilation\_patch]`. + + Args: + kernel_size (int): The size of sliding window i.e. local neighborhood + representing the center points and involved in correlation + computation. Defaults to 1. + max_displacement (int): The radius for computing correlation volume, + but the actual working space can be dilated by dilation_patch. + Defaults to 1. + stride (int): The stride of the sliding blocks in the input spatial + dimensions. Defaults to 1. + padding (int): Zero padding added to all four sides of the input1. + Defaults to 0. + dilation (int): The spacing of local neighborhood that will involved + in correlation. Defaults to 1. + dilation_patch (int): The spacing between position need to compute + correlation. Defaults to 1. + """ + + def __init__(self, + kernel_size: int = 1, + max_displacement: int = 1, + stride: int = 1, + padding: int = 0, + dilation: int = 1, + dilation_patch: int = 1) -> None: + super().__init__() + self.kernel_size = kernel_size + self.max_displacement = max_displacement + self.stride = stride + self.padding = padding + self.dilation = dilation + self.dilation_patch = dilation_patch + + def forward(self, input1: Tensor, input2: Tensor) -> Tensor: + return CorrelationFunction.apply(input1, input2, self.kernel_size, + self.max_displacement, self.stride, + self.padding, self.dilation, + self.dilation_patch) + + def __repr__(self) -> str: + s = self.__class__.__name__ + s += f'(kernel_size={self.kernel_size}, ' + s += f'max_displacement={self.max_displacement}, ' + s += f'stride={self.stride}, ' + s += f'padding={self.padding}, ' + s += f'dilation={self.dilation}, ' + s += f'dilation_patch={self.dilation_patch})' + return s diff --git a/annotator/uniformer/mmcv/ops/deform_conv.py b/annotator/uniformer/mmcv/ops/deform_conv.py new file mode 100644 index 0000000000000000000000000000000000000000..a3f8c75ee774823eea334e3b3732af6a18f55038 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/deform_conv.py @@ -0,0 +1,405 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from typing import Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch import Tensor +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.utils import _pair, _single + +from annotator.uniformer.mmcv.utils import deprecated_api_warning +from ..cnn import CONV_LAYERS +from ..utils import ext_loader, print_log + +ext_module = ext_loader.load_ext('_ext', [ + 'deform_conv_forward', 'deform_conv_backward_input', + 'deform_conv_backward_parameters' +]) + + +class DeformConv2dFunction(Function): + + @staticmethod + def symbolic(g, + input, + offset, + weight, + stride, + padding, + dilation, + groups, + deform_groups, + bias=False, + im2col_step=32): + return g.op( + 'mmcv::MMCVDeformConv2d', + input, + offset, + weight, + stride_i=stride, + padding_i=padding, + dilation_i=dilation, + groups_i=groups, + deform_groups_i=deform_groups, + bias_i=bias, + im2col_step_i=im2col_step) + + @staticmethod + def forward(ctx, + input, + offset, + weight, + stride=1, + padding=0, + dilation=1, + groups=1, + deform_groups=1, + bias=False, + im2col_step=32): + if input is not None and input.dim() != 4: + raise ValueError( + f'Expected 4D tensor as input, got {input.dim()}D tensor \ + instead.') + assert bias is False, 'Only support bias is False.' + ctx.stride = _pair(stride) + ctx.padding = _pair(padding) + ctx.dilation = _pair(dilation) + ctx.groups = groups + ctx.deform_groups = deform_groups + ctx.im2col_step = im2col_step + + # When pytorch version >= 1.6.0, amp is adopted for fp16 mode; + # amp won't cast the type of model (float32), but "offset" is cast + # to float16 by nn.Conv2d automatically, leading to the type + # mismatch with input (when it is float32) or weight. + # The flag for whether to use fp16 or amp is the type of "offset", + # we cast weight and input to temporarily support fp16 and amp + # whatever the pytorch version is. + input = input.type_as(offset) + weight = weight.type_as(input) + ctx.save_for_backward(input, offset, weight) + + output = input.new_empty( + DeformConv2dFunction._output_size(ctx, input, weight)) + + ctx.bufs_ = [input.new_empty(0), input.new_empty(0)] # columns, ones + + cur_im2col_step = min(ctx.im2col_step, input.size(0)) + assert (input.size(0) % + cur_im2col_step) == 0, 'im2col step must divide batchsize' + ext_module.deform_conv_forward( + input, + weight, + offset, + output, + ctx.bufs_[0], + ctx.bufs_[1], + kW=weight.size(3), + kH=weight.size(2), + dW=ctx.stride[1], + dH=ctx.stride[0], + padW=ctx.padding[1], + padH=ctx.padding[0], + dilationW=ctx.dilation[1], + dilationH=ctx.dilation[0], + group=ctx.groups, + deformable_group=ctx.deform_groups, + im2col_step=cur_im2col_step) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + input, offset, weight = ctx.saved_tensors + + grad_input = grad_offset = grad_weight = None + + cur_im2col_step = min(ctx.im2col_step, input.size(0)) + assert (input.size(0) % cur_im2col_step + ) == 0, 'batch size must be divisible by im2col_step' + + grad_output = grad_output.contiguous() + if ctx.needs_input_grad[0] or ctx.needs_input_grad[1]: + grad_input = torch.zeros_like(input) + grad_offset = torch.zeros_like(offset) + ext_module.deform_conv_backward_input( + input, + offset, + grad_output, + grad_input, + grad_offset, + weight, + ctx.bufs_[0], + kW=weight.size(3), + kH=weight.size(2), + dW=ctx.stride[1], + dH=ctx.stride[0], + padW=ctx.padding[1], + padH=ctx.padding[0], + dilationW=ctx.dilation[1], + dilationH=ctx.dilation[0], + group=ctx.groups, + deformable_group=ctx.deform_groups, + im2col_step=cur_im2col_step) + + if ctx.needs_input_grad[2]: + grad_weight = torch.zeros_like(weight) + ext_module.deform_conv_backward_parameters( + input, + offset, + grad_output, + grad_weight, + ctx.bufs_[0], + ctx.bufs_[1], + kW=weight.size(3), + kH=weight.size(2), + dW=ctx.stride[1], + dH=ctx.stride[0], + padW=ctx.padding[1], + padH=ctx.padding[0], + dilationW=ctx.dilation[1], + dilationH=ctx.dilation[0], + group=ctx.groups, + deformable_group=ctx.deform_groups, + scale=1, + im2col_step=cur_im2col_step) + + return grad_input, grad_offset, grad_weight, \ + None, None, None, None, None, None, None + + @staticmethod + def _output_size(ctx, input, weight): + channels = weight.size(0) + output_size = (input.size(0), channels) + for d in range(input.dim() - 2): + in_size = input.size(d + 2) + pad = ctx.padding[d] + kernel = ctx.dilation[d] * (weight.size(d + 2) - 1) + 1 + stride_ = ctx.stride[d] + output_size += ((in_size + (2 * pad) - kernel) // stride_ + 1, ) + if not all(map(lambda s: s > 0, output_size)): + raise ValueError( + 'convolution input is too small (output would be ' + + 'x'.join(map(str, output_size)) + ')') + return output_size + + +deform_conv2d = DeformConv2dFunction.apply + + +class DeformConv2d(nn.Module): + r"""Deformable 2D convolution. + + Applies a deformable 2D convolution over an input signal composed of + several input planes. DeformConv2d was described in the paper + `Deformable Convolutional Networks + `_ + + Note: + The argument ``im2col_step`` was added in version 1.3.17, which means + number of samples processed by the ``im2col_cuda_kernel`` per call. + It enables users to define ``batch_size`` and ``im2col_step`` more + flexibly and solved `issue mmcv#1440 + `_. + + Args: + in_channels (int): Number of channels in the input image. + out_channels (int): Number of channels produced by the convolution. + kernel_size(int, tuple): Size of the convolving kernel. + stride(int, tuple): Stride of the convolution. Default: 1. + padding (int or tuple): Zero-padding added to both sides of the input. + Default: 0. + dilation (int or tuple): Spacing between kernel elements. Default: 1. + groups (int): Number of blocked connections from input. + channels to output channels. Default: 1. + deform_groups (int): Number of deformable group partitions. + bias (bool): If True, adds a learnable bias to the output. + Default: False. + im2col_step (int): Number of samples processed by im2col_cuda_kernel + per call. It will work when ``batch_size`` > ``im2col_step``, but + ``batch_size`` must be divisible by ``im2col_step``. Default: 32. + `New in version 1.3.17.` + """ + + @deprecated_api_warning({'deformable_groups': 'deform_groups'}, + cls_name='DeformConv2d') + def __init__(self, + in_channels: int, + out_channels: int, + kernel_size: Union[int, Tuple[int, ...]], + stride: Union[int, Tuple[int, ...]] = 1, + padding: Union[int, Tuple[int, ...]] = 0, + dilation: Union[int, Tuple[int, ...]] = 1, + groups: int = 1, + deform_groups: int = 1, + bias: bool = False, + im2col_step: int = 32) -> None: + super(DeformConv2d, self).__init__() + + assert not bias, \ + f'bias={bias} is not supported in DeformConv2d.' + assert in_channels % groups == 0, \ + f'in_channels {in_channels} cannot be divisible by groups {groups}' + assert out_channels % groups == 0, \ + f'out_channels {out_channels} cannot be divisible by groups \ + {groups}' + + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = _pair(stride) + self.padding = _pair(padding) + self.dilation = _pair(dilation) + self.groups = groups + self.deform_groups = deform_groups + self.im2col_step = im2col_step + # enable compatibility with nn.Conv2d + self.transposed = False + self.output_padding = _single(0) + + # only weight, no bias + self.weight = nn.Parameter( + torch.Tensor(out_channels, in_channels // self.groups, + *self.kernel_size)) + + self.reset_parameters() + + def reset_parameters(self): + # switch the initialization of `self.weight` to the standard kaiming + # method described in `Delving deep into rectifiers: Surpassing + # human-level performance on ImageNet classification` - He, K. et al. + # (2015), using a uniform distribution + nn.init.kaiming_uniform_(self.weight, nonlinearity='relu') + + def forward(self, x: Tensor, offset: Tensor) -> Tensor: + """Deformable Convolutional forward function. + + Args: + x (Tensor): Input feature, shape (B, C_in, H_in, W_in) + offset (Tensor): Offset for deformable convolution, shape + (B, deform_groups*kernel_size[0]*kernel_size[1]*2, + H_out, W_out), H_out, W_out are equal to the output's. + + An offset is like `[y0, x0, y1, x1, y2, x2, ..., y8, x8]`. + The spatial arrangement is like: + + .. code:: text + + (x0, y0) (x1, y1) (x2, y2) + (x3, y3) (x4, y4) (x5, y5) + (x6, y6) (x7, y7) (x8, y8) + + Returns: + Tensor: Output of the layer. + """ + # To fix an assert error in deform_conv_cuda.cpp:128 + # input image is smaller than kernel + input_pad = (x.size(2) < self.kernel_size[0]) or (x.size(3) < + self.kernel_size[1]) + if input_pad: + pad_h = max(self.kernel_size[0] - x.size(2), 0) + pad_w = max(self.kernel_size[1] - x.size(3), 0) + x = F.pad(x, (0, pad_w, 0, pad_h), 'constant', 0).contiguous() + offset = F.pad(offset, (0, pad_w, 0, pad_h), 'constant', 0) + offset = offset.contiguous() + out = deform_conv2d(x, offset, self.weight, self.stride, self.padding, + self.dilation, self.groups, self.deform_groups, + False, self.im2col_step) + if input_pad: + out = out[:, :, :out.size(2) - pad_h, :out.size(3) - + pad_w].contiguous() + return out + + def __repr__(self): + s = self.__class__.__name__ + s += f'(in_channels={self.in_channels},\n' + s += f'out_channels={self.out_channels},\n' + s += f'kernel_size={self.kernel_size},\n' + s += f'stride={self.stride},\n' + s += f'padding={self.padding},\n' + s += f'dilation={self.dilation},\n' + s += f'groups={self.groups},\n' + s += f'deform_groups={self.deform_groups},\n' + # bias is not supported in DeformConv2d. + s += 'bias=False)' + return s + + +@CONV_LAYERS.register_module('DCN') +class DeformConv2dPack(DeformConv2d): + """A Deformable Conv Encapsulation that acts as normal Conv layers. + + The offset tensor is like `[y0, x0, y1, x1, y2, x2, ..., y8, x8]`. + The spatial arrangement is like: + + .. code:: text + + (x0, y0) (x1, y1) (x2, y2) + (x3, y3) (x4, y4) (x5, y5) + (x6, y6) (x7, y7) (x8, y8) + + Args: + in_channels (int): Same as nn.Conv2d. + out_channels (int): Same as nn.Conv2d. + kernel_size (int or tuple[int]): Same as nn.Conv2d. + stride (int or tuple[int]): Same as nn.Conv2d. + padding (int or tuple[int]): Same as nn.Conv2d. + dilation (int or tuple[int]): Same as nn.Conv2d. + groups (int): Same as nn.Conv2d. + bias (bool or str): If specified as `auto`, it will be decided by the + norm_cfg. Bias will be set as True if norm_cfg is None, otherwise + False. + """ + + _version = 2 + + def __init__(self, *args, **kwargs): + super(DeformConv2dPack, self).__init__(*args, **kwargs) + self.conv_offset = nn.Conv2d( + self.in_channels, + self.deform_groups * 2 * self.kernel_size[0] * self.kernel_size[1], + kernel_size=self.kernel_size, + stride=_pair(self.stride), + padding=_pair(self.padding), + dilation=_pair(self.dilation), + bias=True) + self.init_offset() + + def init_offset(self): + self.conv_offset.weight.data.zero_() + self.conv_offset.bias.data.zero_() + + def forward(self, x): + offset = self.conv_offset(x) + return deform_conv2d(x, offset, self.weight, self.stride, self.padding, + self.dilation, self.groups, self.deform_groups, + False, self.im2col_step) + + def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs): + version = local_metadata.get('version', None) + + if version is None or version < 2: + # the key is different in early versions + # In version < 2, DeformConvPack loads previous benchmark models. + if (prefix + 'conv_offset.weight' not in state_dict + and prefix[:-1] + '_offset.weight' in state_dict): + state_dict[prefix + 'conv_offset.weight'] = state_dict.pop( + prefix[:-1] + '_offset.weight') + if (prefix + 'conv_offset.bias' not in state_dict + and prefix[:-1] + '_offset.bias' in state_dict): + state_dict[prefix + + 'conv_offset.bias'] = state_dict.pop(prefix[:-1] + + '_offset.bias') + + if version is not None and version > 1: + print_log( + f'DeformConv2dPack {prefix.rstrip(".")} is upgraded to ' + 'version 2.', + logger='root') + + super()._load_from_state_dict(state_dict, prefix, local_metadata, + strict, missing_keys, unexpected_keys, + error_msgs) diff --git a/annotator/uniformer/mmcv/ops/deform_roi_pool.py b/annotator/uniformer/mmcv/ops/deform_roi_pool.py new file mode 100644 index 0000000000000000000000000000000000000000..cc245ba91fee252226ba22e76bb94a35db9a629b --- /dev/null +++ b/annotator/uniformer/mmcv/ops/deform_roi_pool.py @@ -0,0 +1,204 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from torch import nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.utils import _pair + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['deform_roi_pool_forward', 'deform_roi_pool_backward']) + + +class DeformRoIPoolFunction(Function): + + @staticmethod + def symbolic(g, input, rois, offset, output_size, spatial_scale, + sampling_ratio, gamma): + return g.op( + 'mmcv::MMCVDeformRoIPool', + input, + rois, + offset, + pooled_height_i=output_size[0], + pooled_width_i=output_size[1], + spatial_scale_f=spatial_scale, + sampling_ratio_f=sampling_ratio, + gamma_f=gamma) + + @staticmethod + def forward(ctx, + input, + rois, + offset, + output_size, + spatial_scale=1.0, + sampling_ratio=0, + gamma=0.1): + if offset is None: + offset = input.new_zeros(0) + ctx.output_size = _pair(output_size) + ctx.spatial_scale = float(spatial_scale) + ctx.sampling_ratio = int(sampling_ratio) + ctx.gamma = float(gamma) + + assert rois.size(1) == 5, 'RoI must be (idx, x1, y1, x2, y2)!' + + output_shape = (rois.size(0), input.size(1), ctx.output_size[0], + ctx.output_size[1]) + output = input.new_zeros(output_shape) + + ext_module.deform_roi_pool_forward( + input, + rois, + offset, + output, + pooled_height=ctx.output_size[0], + pooled_width=ctx.output_size[1], + spatial_scale=ctx.spatial_scale, + sampling_ratio=ctx.sampling_ratio, + gamma=ctx.gamma) + + ctx.save_for_backward(input, rois, offset) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + input, rois, offset = ctx.saved_tensors + grad_input = grad_output.new_zeros(input.shape) + grad_offset = grad_output.new_zeros(offset.shape) + + ext_module.deform_roi_pool_backward( + grad_output, + input, + rois, + offset, + grad_input, + grad_offset, + pooled_height=ctx.output_size[0], + pooled_width=ctx.output_size[1], + spatial_scale=ctx.spatial_scale, + sampling_ratio=ctx.sampling_ratio, + gamma=ctx.gamma) + if grad_offset.numel() == 0: + grad_offset = None + return grad_input, None, grad_offset, None, None, None, None + + +deform_roi_pool = DeformRoIPoolFunction.apply + + +class DeformRoIPool(nn.Module): + + def __init__(self, + output_size, + spatial_scale=1.0, + sampling_ratio=0, + gamma=0.1): + super(DeformRoIPool, self).__init__() + self.output_size = _pair(output_size) + self.spatial_scale = float(spatial_scale) + self.sampling_ratio = int(sampling_ratio) + self.gamma = float(gamma) + + def forward(self, input, rois, offset=None): + return deform_roi_pool(input, rois, offset, self.output_size, + self.spatial_scale, self.sampling_ratio, + self.gamma) + + +class DeformRoIPoolPack(DeformRoIPool): + + def __init__(self, + output_size, + output_channels, + deform_fc_channels=1024, + spatial_scale=1.0, + sampling_ratio=0, + gamma=0.1): + super(DeformRoIPoolPack, self).__init__(output_size, spatial_scale, + sampling_ratio, gamma) + + self.output_channels = output_channels + self.deform_fc_channels = deform_fc_channels + + self.offset_fc = nn.Sequential( + nn.Linear( + self.output_size[0] * self.output_size[1] * + self.output_channels, self.deform_fc_channels), + nn.ReLU(inplace=True), + nn.Linear(self.deform_fc_channels, self.deform_fc_channels), + nn.ReLU(inplace=True), + nn.Linear(self.deform_fc_channels, + self.output_size[0] * self.output_size[1] * 2)) + self.offset_fc[-1].weight.data.zero_() + self.offset_fc[-1].bias.data.zero_() + + def forward(self, input, rois): + assert input.size(1) == self.output_channels + x = deform_roi_pool(input, rois, None, self.output_size, + self.spatial_scale, self.sampling_ratio, + self.gamma) + rois_num = rois.size(0) + offset = self.offset_fc(x.view(rois_num, -1)) + offset = offset.view(rois_num, 2, self.output_size[0], + self.output_size[1]) + return deform_roi_pool(input, rois, offset, self.output_size, + self.spatial_scale, self.sampling_ratio, + self.gamma) + + +class ModulatedDeformRoIPoolPack(DeformRoIPool): + + def __init__(self, + output_size, + output_channels, + deform_fc_channels=1024, + spatial_scale=1.0, + sampling_ratio=0, + gamma=0.1): + super(ModulatedDeformRoIPoolPack, + self).__init__(output_size, spatial_scale, sampling_ratio, gamma) + + self.output_channels = output_channels + self.deform_fc_channels = deform_fc_channels + + self.offset_fc = nn.Sequential( + nn.Linear( + self.output_size[0] * self.output_size[1] * + self.output_channels, self.deform_fc_channels), + nn.ReLU(inplace=True), + nn.Linear(self.deform_fc_channels, self.deform_fc_channels), + nn.ReLU(inplace=True), + nn.Linear(self.deform_fc_channels, + self.output_size[0] * self.output_size[1] * 2)) + self.offset_fc[-1].weight.data.zero_() + self.offset_fc[-1].bias.data.zero_() + + self.mask_fc = nn.Sequential( + nn.Linear( + self.output_size[0] * self.output_size[1] * + self.output_channels, self.deform_fc_channels), + nn.ReLU(inplace=True), + nn.Linear(self.deform_fc_channels, + self.output_size[0] * self.output_size[1] * 1), + nn.Sigmoid()) + self.mask_fc[2].weight.data.zero_() + self.mask_fc[2].bias.data.zero_() + + def forward(self, input, rois): + assert input.size(1) == self.output_channels + x = deform_roi_pool(input, rois, None, self.output_size, + self.spatial_scale, self.sampling_ratio, + self.gamma) + rois_num = rois.size(0) + offset = self.offset_fc(x.view(rois_num, -1)) + offset = offset.view(rois_num, 2, self.output_size[0], + self.output_size[1]) + mask = self.mask_fc(x.view(rois_num, -1)) + mask = mask.view(rois_num, 1, self.output_size[0], self.output_size[1]) + d = deform_roi_pool(input, rois, offset, self.output_size, + self.spatial_scale, self.sampling_ratio, + self.gamma) + return d * mask diff --git a/annotator/uniformer/mmcv/ops/deprecated_wrappers.py b/annotator/uniformer/mmcv/ops/deprecated_wrappers.py new file mode 100644 index 0000000000000000000000000000000000000000..a2e593df9ee57637038683d7a1efaa347b2b69e7 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/deprecated_wrappers.py @@ -0,0 +1,43 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# This file is for backward compatibility. +# Module wrappers for empty tensor have been moved to mmcv.cnn.bricks. +import warnings + +from ..cnn.bricks.wrappers import Conv2d, ConvTranspose2d, Linear, MaxPool2d + + +class Conv2d_deprecated(Conv2d): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + warnings.warn( + 'Importing Conv2d wrapper from "mmcv.ops" will be deprecated in' + ' the future. Please import them from "mmcv.cnn" instead') + + +class ConvTranspose2d_deprecated(ConvTranspose2d): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + warnings.warn( + 'Importing ConvTranspose2d wrapper from "mmcv.ops" will be ' + 'deprecated in the future. Please import them from "mmcv.cnn" ' + 'instead') + + +class MaxPool2d_deprecated(MaxPool2d): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + warnings.warn( + 'Importing MaxPool2d wrapper from "mmcv.ops" will be deprecated in' + ' the future. Please import them from "mmcv.cnn" instead') + + +class Linear_deprecated(Linear): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + warnings.warn( + 'Importing Linear wrapper from "mmcv.ops" will be deprecated in' + ' the future. Please import them from "mmcv.cnn" instead') diff --git a/annotator/uniformer/mmcv/ops/focal_loss.py b/annotator/uniformer/mmcv/ops/focal_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..763bc93bd2575c49ca8ccf20996bbd92d1e0d1a4 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/focal_loss.py @@ -0,0 +1,212 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', [ + 'sigmoid_focal_loss_forward', 'sigmoid_focal_loss_backward', + 'softmax_focal_loss_forward', 'softmax_focal_loss_backward' +]) + + +class SigmoidFocalLossFunction(Function): + + @staticmethod + def symbolic(g, input, target, gamma, alpha, weight, reduction): + return g.op( + 'mmcv::MMCVSigmoidFocalLoss', + input, + target, + gamma_f=gamma, + alpha_f=alpha, + weight_f=weight, + reduction_s=reduction) + + @staticmethod + def forward(ctx, + input, + target, + gamma=2.0, + alpha=0.25, + weight=None, + reduction='mean'): + + assert isinstance(target, (torch.LongTensor, torch.cuda.LongTensor)) + assert input.dim() == 2 + assert target.dim() == 1 + assert input.size(0) == target.size(0) + if weight is None: + weight = input.new_empty(0) + else: + assert weight.dim() == 1 + assert input.size(1) == weight.size(0) + ctx.reduction_dict = {'none': 0, 'mean': 1, 'sum': 2} + assert reduction in ctx.reduction_dict.keys() + + ctx.gamma = float(gamma) + ctx.alpha = float(alpha) + ctx.reduction = ctx.reduction_dict[reduction] + + output = input.new_zeros(input.size()) + + ext_module.sigmoid_focal_loss_forward( + input, target, weight, output, gamma=ctx.gamma, alpha=ctx.alpha) + if ctx.reduction == ctx.reduction_dict['mean']: + output = output.sum() / input.size(0) + elif ctx.reduction == ctx.reduction_dict['sum']: + output = output.sum() + ctx.save_for_backward(input, target, weight) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + input, target, weight = ctx.saved_tensors + + grad_input = input.new_zeros(input.size()) + + ext_module.sigmoid_focal_loss_backward( + input, + target, + weight, + grad_input, + gamma=ctx.gamma, + alpha=ctx.alpha) + + grad_input *= grad_output + if ctx.reduction == ctx.reduction_dict['mean']: + grad_input /= input.size(0) + return grad_input, None, None, None, None, None + + +sigmoid_focal_loss = SigmoidFocalLossFunction.apply + + +class SigmoidFocalLoss(nn.Module): + + def __init__(self, gamma, alpha, weight=None, reduction='mean'): + super(SigmoidFocalLoss, self).__init__() + self.gamma = gamma + self.alpha = alpha + self.register_buffer('weight', weight) + self.reduction = reduction + + def forward(self, input, target): + return sigmoid_focal_loss(input, target, self.gamma, self.alpha, + self.weight, self.reduction) + + def __repr__(self): + s = self.__class__.__name__ + s += f'(gamma={self.gamma}, ' + s += f'alpha={self.alpha}, ' + s += f'reduction={self.reduction})' + return s + + +class SoftmaxFocalLossFunction(Function): + + @staticmethod + def symbolic(g, input, target, gamma, alpha, weight, reduction): + return g.op( + 'mmcv::MMCVSoftmaxFocalLoss', + input, + target, + gamma_f=gamma, + alpha_f=alpha, + weight_f=weight, + reduction_s=reduction) + + @staticmethod + def forward(ctx, + input, + target, + gamma=2.0, + alpha=0.25, + weight=None, + reduction='mean'): + + assert isinstance(target, (torch.LongTensor, torch.cuda.LongTensor)) + assert input.dim() == 2 + assert target.dim() == 1 + assert input.size(0) == target.size(0) + if weight is None: + weight = input.new_empty(0) + else: + assert weight.dim() == 1 + assert input.size(1) == weight.size(0) + ctx.reduction_dict = {'none': 0, 'mean': 1, 'sum': 2} + assert reduction in ctx.reduction_dict.keys() + + ctx.gamma = float(gamma) + ctx.alpha = float(alpha) + ctx.reduction = ctx.reduction_dict[reduction] + + channel_stats, _ = torch.max(input, dim=1) + input_softmax = input - channel_stats.unsqueeze(1).expand_as(input) + input_softmax.exp_() + + channel_stats = input_softmax.sum(dim=1) + input_softmax /= channel_stats.unsqueeze(1).expand_as(input) + + output = input.new_zeros(input.size(0)) + ext_module.softmax_focal_loss_forward( + input_softmax, + target, + weight, + output, + gamma=ctx.gamma, + alpha=ctx.alpha) + + if ctx.reduction == ctx.reduction_dict['mean']: + output = output.sum() / input.size(0) + elif ctx.reduction == ctx.reduction_dict['sum']: + output = output.sum() + ctx.save_for_backward(input_softmax, target, weight) + return output + + @staticmethod + def backward(ctx, grad_output): + input_softmax, target, weight = ctx.saved_tensors + buff = input_softmax.new_zeros(input_softmax.size(0)) + grad_input = input_softmax.new_zeros(input_softmax.size()) + + ext_module.softmax_focal_loss_backward( + input_softmax, + target, + weight, + buff, + grad_input, + gamma=ctx.gamma, + alpha=ctx.alpha) + + grad_input *= grad_output + if ctx.reduction == ctx.reduction_dict['mean']: + grad_input /= input_softmax.size(0) + return grad_input, None, None, None, None, None + + +softmax_focal_loss = SoftmaxFocalLossFunction.apply + + +class SoftmaxFocalLoss(nn.Module): + + def __init__(self, gamma, alpha, weight=None, reduction='mean'): + super(SoftmaxFocalLoss, self).__init__() + self.gamma = gamma + self.alpha = alpha + self.register_buffer('weight', weight) + self.reduction = reduction + + def forward(self, input, target): + return softmax_focal_loss(input, target, self.gamma, self.alpha, + self.weight, self.reduction) + + def __repr__(self): + s = self.__class__.__name__ + s += f'(gamma={self.gamma}, ' + s += f'alpha={self.alpha}, ' + s += f'reduction={self.reduction})' + return s diff --git a/annotator/uniformer/mmcv/ops/furthest_point_sample.py b/annotator/uniformer/mmcv/ops/furthest_point_sample.py new file mode 100644 index 0000000000000000000000000000000000000000..374b7a878f1972c183941af28ba1df216ac1a60f --- /dev/null +++ b/annotator/uniformer/mmcv/ops/furthest_point_sample.py @@ -0,0 +1,83 @@ +import torch +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', [ + 'furthest_point_sampling_forward', + 'furthest_point_sampling_with_dist_forward' +]) + + +class FurthestPointSampling(Function): + """Uses iterative furthest point sampling to select a set of features whose + corresponding points have the furthest distance.""" + + @staticmethod + def forward(ctx, points_xyz: torch.Tensor, + num_points: int) -> torch.Tensor: + """ + Args: + points_xyz (Tensor): (B, N, 3) where N > num_points. + num_points (int): Number of points in the sampled set. + + Returns: + Tensor: (B, num_points) indices of the sampled points. + """ + assert points_xyz.is_contiguous() + + B, N = points_xyz.size()[:2] + output = torch.cuda.IntTensor(B, num_points) + temp = torch.cuda.FloatTensor(B, N).fill_(1e10) + + ext_module.furthest_point_sampling_forward( + points_xyz, + temp, + output, + b=B, + n=N, + m=num_points, + ) + if torch.__version__ != 'parrots': + ctx.mark_non_differentiable(output) + return output + + @staticmethod + def backward(xyz, a=None): + return None, None + + +class FurthestPointSamplingWithDist(Function): + """Uses iterative furthest point sampling to select a set of features whose + corresponding points have the furthest distance.""" + + @staticmethod + def forward(ctx, points_dist: torch.Tensor, + num_points: int) -> torch.Tensor: + """ + Args: + points_dist (Tensor): (B, N, N) Distance between each point pair. + num_points (int): Number of points in the sampled set. + + Returns: + Tensor: (B, num_points) indices of the sampled points. + """ + assert points_dist.is_contiguous() + + B, N, _ = points_dist.size() + output = points_dist.new_zeros([B, num_points], dtype=torch.int32) + temp = points_dist.new_zeros([B, N]).fill_(1e10) + + ext_module.furthest_point_sampling_with_dist_forward( + points_dist, temp, output, b=B, n=N, m=num_points) + if torch.__version__ != 'parrots': + ctx.mark_non_differentiable(output) + return output + + @staticmethod + def backward(xyz, a=None): + return None, None + + +furthest_point_sample = FurthestPointSampling.apply +furthest_point_sample_with_dist = FurthestPointSamplingWithDist.apply diff --git a/annotator/uniformer/mmcv/ops/fused_bias_leakyrelu.py b/annotator/uniformer/mmcv/ops/fused_bias_leakyrelu.py new file mode 100644 index 0000000000000000000000000000000000000000..6d12508469c6c8fa1884debece44c58d158cb6fa --- /dev/null +++ b/annotator/uniformer/mmcv/ops/fused_bias_leakyrelu.py @@ -0,0 +1,268 @@ +# modified from https://github.com/rosinality/stylegan2-pytorch/blob/master/op/fused_act.py # noqa:E501 + +# Copyright (c) 2021, NVIDIA Corporation. All rights reserved. +# NVIDIA Source Code License for StyleGAN2 with Adaptive Discriminator +# Augmentation (ADA) +# ======================================================================= + +# 1. Definitions + +# "Licensor" means any person or entity that distributes its Work. + +# "Software" means the original work of authorship made available under +# this License. + +# "Work" means the Software and any additions to or derivative works of +# the Software that are made available under this License. + +# The terms "reproduce," "reproduction," "derivative works," and +# "distribution" have the meaning as provided under U.S. copyright law; +# provided, however, that for the purposes of this License, derivative +# works shall not include works that remain separable from, or merely +# link (or bind by name) to the interfaces of, the Work. + +# Works, including the Software, are "made available" under this License +# by including in or with the Work either (a) a copyright notice +# referencing the applicability of this License to the Work, or (b) a +# copy of this License. + +# 2. License Grants + +# 2.1 Copyright Grant. Subject to the terms and conditions of this +# License, each Licensor grants to you a perpetual, worldwide, +# non-exclusive, royalty-free, copyright license to reproduce, +# prepare derivative works of, publicly display, publicly perform, +# sublicense and distribute its Work and any resulting derivative +# works in any form. + +# 3. Limitations + +# 3.1 Redistribution. You may reproduce or distribute the Work only +# if (a) you do so under this License, (b) you include a complete +# copy of this License with your distribution, and (c) you retain +# without modification any copyright, patent, trademark, or +# attribution notices that are present in the Work. + +# 3.2 Derivative Works. You may specify that additional or different +# terms apply to the use, reproduction, and distribution of your +# derivative works of the Work ("Your Terms") only if (a) Your Terms +# provide that the use limitation in Section 3.3 applies to your +# derivative works, and (b) you identify the specific derivative +# works that are subject to Your Terms. Notwithstanding Your Terms, +# this License (including the redistribution requirements in Section +# 3.1) will continue to apply to the Work itself. + +# 3.3 Use Limitation. The Work and any derivative works thereof only +# may be used or intended for use non-commercially. Notwithstanding +# the foregoing, NVIDIA and its affiliates may use the Work and any +# derivative works commercially. As used herein, "non-commercially" +# means for research or evaluation purposes only. + +# 3.4 Patent Claims. If you bring or threaten to bring a patent claim +# against any Licensor (including any claim, cross-claim or +# counterclaim in a lawsuit) to enforce any patents that you allege +# are infringed by any Work, then your rights under this License from +# such Licensor (including the grant in Section 2.1) will terminate +# immediately. + +# 3.5 Trademarks. This License does not grant any rights to use any +# Licensor’s or its affiliates’ names, logos, or trademarks, except +# as necessary to reproduce the notices described in this License. + +# 3.6 Termination. If you violate any term of this License, then your +# rights under this License (including the grant in Section 2.1) will +# terminate immediately. + +# 4. Disclaimer of Warranty. + +# THE WORK IS PROVIDED "AS IS" WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WARRANTIES OR CONDITIONS OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR +# NON-INFRINGEMENT. YOU BEAR THE RISK OF UNDERTAKING ANY ACTIVITIES UNDER +# THIS LICENSE. + +# 5. Limitation of Liability. + +# EXCEPT AS PROHIBITED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL +# THEORY, WHETHER IN TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE +# SHALL ANY LICENSOR BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, +# INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF +# OR RELATED TO THIS LICENSE, THE USE OR INABILITY TO USE THE WORK +# (INCLUDING BUT NOT LIMITED TO LOSS OF GOODWILL, BUSINESS INTERRUPTION, +# LOST PROFITS OR DATA, COMPUTER FAILURE OR MALFUNCTION, OR ANY OTHER +# COMMERCIAL DAMAGES OR LOSSES), EVEN IF THE LICENSOR HAS BEEN ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGES. + +# ======================================================================= + +import torch +import torch.nn.functional as F +from torch import nn +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', ['fused_bias_leakyrelu']) + + +class FusedBiasLeakyReLUFunctionBackward(Function): + """Calculate second order deviation. + + This function is to compute the second order deviation for the fused leaky + relu operation. + """ + + @staticmethod + def forward(ctx, grad_output, out, negative_slope, scale): + ctx.save_for_backward(out) + ctx.negative_slope = negative_slope + ctx.scale = scale + + empty = grad_output.new_empty(0) + + grad_input = ext_module.fused_bias_leakyrelu( + grad_output, + empty, + out, + act=3, + grad=1, + alpha=negative_slope, + scale=scale) + + dim = [0] + + if grad_input.ndim > 2: + dim += list(range(2, grad_input.ndim)) + + grad_bias = grad_input.sum(dim).detach() + + return grad_input, grad_bias + + @staticmethod + def backward(ctx, gradgrad_input, gradgrad_bias): + out, = ctx.saved_tensors + + # The second order deviation, in fact, contains two parts, while the + # the first part is zero. Thus, we direct consider the second part + # which is similar with the first order deviation in implementation. + gradgrad_out = ext_module.fused_bias_leakyrelu( + gradgrad_input, + gradgrad_bias.to(out.dtype), + out, + act=3, + grad=1, + alpha=ctx.negative_slope, + scale=ctx.scale) + + return gradgrad_out, None, None, None + + +class FusedBiasLeakyReLUFunction(Function): + + @staticmethod + def forward(ctx, input, bias, negative_slope, scale): + empty = input.new_empty(0) + + out = ext_module.fused_bias_leakyrelu( + input, + bias, + empty, + act=3, + grad=0, + alpha=negative_slope, + scale=scale) + ctx.save_for_backward(out) + ctx.negative_slope = negative_slope + ctx.scale = scale + + return out + + @staticmethod + def backward(ctx, grad_output): + out, = ctx.saved_tensors + + grad_input, grad_bias = FusedBiasLeakyReLUFunctionBackward.apply( + grad_output, out, ctx.negative_slope, ctx.scale) + + return grad_input, grad_bias, None, None + + +class FusedBiasLeakyReLU(nn.Module): + """Fused bias leaky ReLU. + + This function is introduced in the StyleGAN2: + http://arxiv.org/abs/1912.04958 + + The bias term comes from the convolution operation. In addition, to keep + the variance of the feature map or gradients unchanged, they also adopt a + scale similarly with Kaiming initialization. However, since the + :math:`1+{alpha}^2` : is too small, we can just ignore it. Therefore, the + final scale is just :math:`\sqrt{2}`:. Of course, you may change it with # noqa: W605, E501 + your own scale. + + TODO: Implement the CPU version. + + Args: + channel (int): The channel number of the feature map. + negative_slope (float, optional): Same as nn.LeakyRelu. + Defaults to 0.2. + scale (float, optional): A scalar to adjust the variance of the feature + map. Defaults to 2**0.5. + """ + + def __init__(self, num_channels, negative_slope=0.2, scale=2**0.5): + super(FusedBiasLeakyReLU, self).__init__() + + self.bias = nn.Parameter(torch.zeros(num_channels)) + self.negative_slope = negative_slope + self.scale = scale + + def forward(self, input): + return fused_bias_leakyrelu(input, self.bias, self.negative_slope, + self.scale) + + +def fused_bias_leakyrelu(input, bias, negative_slope=0.2, scale=2**0.5): + """Fused bias leaky ReLU function. + + This function is introduced in the StyleGAN2: + http://arxiv.org/abs/1912.04958 + + The bias term comes from the convolution operation. In addition, to keep + the variance of the feature map or gradients unchanged, they also adopt a + scale similarly with Kaiming initialization. However, since the + :math:`1+{alpha}^2` : is too small, we can just ignore it. Therefore, the + final scale is just :math:`\sqrt{2}`:. Of course, you may change it with # noqa: W605, E501 + your own scale. + + Args: + input (torch.Tensor): Input feature map. + bias (nn.Parameter): The bias from convolution operation. + negative_slope (float, optional): Same as nn.LeakyRelu. + Defaults to 0.2. + scale (float, optional): A scalar to adjust the variance of the feature + map. Defaults to 2**0.5. + + Returns: + torch.Tensor: Feature map after non-linear activation. + """ + + if not input.is_cuda: + return bias_leakyrelu_ref(input, bias, negative_slope, scale) + + return FusedBiasLeakyReLUFunction.apply(input, bias.to(input.dtype), + negative_slope, scale) + + +def bias_leakyrelu_ref(x, bias, negative_slope=0.2, scale=2**0.5): + + if bias is not None: + assert bias.ndim == 1 + assert bias.shape[0] == x.shape[1] + x = x + bias.reshape([-1 if i == 1 else 1 for i in range(x.ndim)]) + + x = F.leaky_relu(x, negative_slope) + if scale != 1: + x = x * scale + + return x diff --git a/annotator/uniformer/mmcv/ops/gather_points.py b/annotator/uniformer/mmcv/ops/gather_points.py new file mode 100644 index 0000000000000000000000000000000000000000..f52f1677d8ea0facafc56a3672d37adb44677ff3 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/gather_points.py @@ -0,0 +1,57 @@ +import torch +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['gather_points_forward', 'gather_points_backward']) + + +class GatherPoints(Function): + """Gather points with given index.""" + + @staticmethod + def forward(ctx, features: torch.Tensor, + indices: torch.Tensor) -> torch.Tensor: + """ + Args: + features (Tensor): (B, C, N) features to gather. + indices (Tensor): (B, M) where M is the number of points. + + Returns: + Tensor: (B, C, M) where M is the number of points. + """ + assert features.is_contiguous() + assert indices.is_contiguous() + + B, npoint = indices.size() + _, C, N = features.size() + output = torch.cuda.FloatTensor(B, C, npoint) + + ext_module.gather_points_forward( + features, indices, output, b=B, c=C, n=N, npoints=npoint) + + ctx.for_backwards = (indices, C, N) + if torch.__version__ != 'parrots': + ctx.mark_non_differentiable(indices) + return output + + @staticmethod + def backward(ctx, grad_out): + idx, C, N = ctx.for_backwards + B, npoint = idx.size() + + grad_features = torch.cuda.FloatTensor(B, C, N).zero_() + grad_out_data = grad_out.data.contiguous() + ext_module.gather_points_backward( + grad_out_data, + idx, + grad_features.data, + b=B, + c=C, + n=N, + npoints=npoint) + return grad_features, None + + +gather_points = GatherPoints.apply diff --git a/annotator/uniformer/mmcv/ops/group_points.py b/annotator/uniformer/mmcv/ops/group_points.py new file mode 100644 index 0000000000000000000000000000000000000000..6c3ec9d758ebe4e1c2205882af4be154008253a5 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/group_points.py @@ -0,0 +1,224 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from typing import Tuple + +import torch +from torch import nn as nn +from torch.autograd import Function + +from ..utils import ext_loader +from .ball_query import ball_query +from .knn import knn + +ext_module = ext_loader.load_ext( + '_ext', ['group_points_forward', 'group_points_backward']) + + +class QueryAndGroup(nn.Module): + """Groups points with a ball query of radius. + + Args: + max_radius (float): The maximum radius of the balls. + If None is given, we will use kNN sampling instead of ball query. + sample_num (int): Maximum number of features to gather in the ball. + min_radius (float, optional): The minimum radius of the balls. + Default: 0. + use_xyz (bool, optional): Whether to use xyz. + Default: True. + return_grouped_xyz (bool, optional): Whether to return grouped xyz. + Default: False. + normalize_xyz (bool, optional): Whether to normalize xyz. + Default: False. + uniform_sample (bool, optional): Whether to sample uniformly. + Default: False + return_unique_cnt (bool, optional): Whether to return the count of + unique samples. Default: False. + return_grouped_idx (bool, optional): Whether to return grouped idx. + Default: False. + """ + + def __init__(self, + max_radius, + sample_num, + min_radius=0, + use_xyz=True, + return_grouped_xyz=False, + normalize_xyz=False, + uniform_sample=False, + return_unique_cnt=False, + return_grouped_idx=False): + super().__init__() + self.max_radius = max_radius + self.min_radius = min_radius + self.sample_num = sample_num + self.use_xyz = use_xyz + self.return_grouped_xyz = return_grouped_xyz + self.normalize_xyz = normalize_xyz + self.uniform_sample = uniform_sample + self.return_unique_cnt = return_unique_cnt + self.return_grouped_idx = return_grouped_idx + if self.return_unique_cnt: + assert self.uniform_sample, \ + 'uniform_sample should be True when ' \ + 'returning the count of unique samples' + if self.max_radius is None: + assert not self.normalize_xyz, \ + 'can not normalize grouped xyz when max_radius is None' + + def forward(self, points_xyz, center_xyz, features=None): + """ + Args: + points_xyz (Tensor): (B, N, 3) xyz coordinates of the features. + center_xyz (Tensor): (B, npoint, 3) coordinates of the centriods. + features (Tensor): (B, C, N) Descriptors of the features. + + Returns: + Tensor: (B, 3 + C, npoint, sample_num) Grouped feature. + """ + # if self.max_radius is None, we will perform kNN instead of ball query + # idx is of shape [B, npoint, sample_num] + if self.max_radius is None: + idx = knn(self.sample_num, points_xyz, center_xyz, False) + idx = idx.transpose(1, 2).contiguous() + else: + idx = ball_query(self.min_radius, self.max_radius, self.sample_num, + points_xyz, center_xyz) + + if self.uniform_sample: + unique_cnt = torch.zeros((idx.shape[0], idx.shape[1])) + for i_batch in range(idx.shape[0]): + for i_region in range(idx.shape[1]): + unique_ind = torch.unique(idx[i_batch, i_region, :]) + num_unique = unique_ind.shape[0] + unique_cnt[i_batch, i_region] = num_unique + sample_ind = torch.randint( + 0, + num_unique, (self.sample_num - num_unique, ), + dtype=torch.long) + all_ind = torch.cat((unique_ind, unique_ind[sample_ind])) + idx[i_batch, i_region, :] = all_ind + + xyz_trans = points_xyz.transpose(1, 2).contiguous() + # (B, 3, npoint, sample_num) + grouped_xyz = grouping_operation(xyz_trans, idx) + grouped_xyz_diff = grouped_xyz - \ + center_xyz.transpose(1, 2).unsqueeze(-1) # relative offsets + if self.normalize_xyz: + grouped_xyz_diff /= self.max_radius + + if features is not None: + grouped_features = grouping_operation(features, idx) + if self.use_xyz: + # (B, C + 3, npoint, sample_num) + new_features = torch.cat([grouped_xyz_diff, grouped_features], + dim=1) + else: + new_features = grouped_features + else: + assert (self.use_xyz + ), 'Cannot have not features and not use xyz as a feature!' + new_features = grouped_xyz_diff + + ret = [new_features] + if self.return_grouped_xyz: + ret.append(grouped_xyz) + if self.return_unique_cnt: + ret.append(unique_cnt) + if self.return_grouped_idx: + ret.append(idx) + if len(ret) == 1: + return ret[0] + else: + return tuple(ret) + + +class GroupAll(nn.Module): + """Group xyz with feature. + + Args: + use_xyz (bool): Whether to use xyz. + """ + + def __init__(self, use_xyz: bool = True): + super().__init__() + self.use_xyz = use_xyz + + def forward(self, + xyz: torch.Tensor, + new_xyz: torch.Tensor, + features: torch.Tensor = None): + """ + Args: + xyz (Tensor): (B, N, 3) xyz coordinates of the features. + new_xyz (Tensor): new xyz coordinates of the features. + features (Tensor): (B, C, N) features to group. + + Returns: + Tensor: (B, C + 3, 1, N) Grouped feature. + """ + grouped_xyz = xyz.transpose(1, 2).unsqueeze(2) + if features is not None: + grouped_features = features.unsqueeze(2) + if self.use_xyz: + # (B, 3 + C, 1, N) + new_features = torch.cat([grouped_xyz, grouped_features], + dim=1) + else: + new_features = grouped_features + else: + new_features = grouped_xyz + + return new_features + + +class GroupingOperation(Function): + """Group feature with given index.""" + + @staticmethod + def forward(ctx, features: torch.Tensor, + indices: torch.Tensor) -> torch.Tensor: + """ + Args: + features (Tensor): (B, C, N) tensor of features to group. + indices (Tensor): (B, npoint, nsample) the indices of + features to group with. + + Returns: + Tensor: (B, C, npoint, nsample) Grouped features. + """ + features = features.contiguous() + indices = indices.contiguous() + + B, nfeatures, nsample = indices.size() + _, C, N = features.size() + output = torch.cuda.FloatTensor(B, C, nfeatures, nsample) + + ext_module.group_points_forward(B, C, N, nfeatures, nsample, features, + indices, output) + + ctx.for_backwards = (indices, N) + return output + + @staticmethod + def backward(ctx, + grad_out: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Args: + grad_out (Tensor): (B, C, npoint, nsample) tensor of the gradients + of the output from forward. + + Returns: + Tensor: (B, C, N) gradient of the features. + """ + idx, N = ctx.for_backwards + + B, C, npoint, nsample = grad_out.size() + grad_features = torch.cuda.FloatTensor(B, C, N).zero_() + + grad_out_data = grad_out.data.contiguous() + ext_module.group_points_backward(B, C, N, npoint, nsample, + grad_out_data, idx, + grad_features.data) + return grad_features, None + + +grouping_operation = GroupingOperation.apply diff --git a/annotator/uniformer/mmcv/ops/info.py b/annotator/uniformer/mmcv/ops/info.py new file mode 100644 index 0000000000000000000000000000000000000000..29f2e5598ae2bb5866ccd15a7d3b4de33c0cd14d --- /dev/null +++ b/annotator/uniformer/mmcv/ops/info.py @@ -0,0 +1,36 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import glob +import os + +import torch + +if torch.__version__ == 'parrots': + import parrots + + def get_compiler_version(): + return 'GCC ' + parrots.version.compiler + + def get_compiling_cuda_version(): + return parrots.version.cuda +else: + from ..utils import ext_loader + ext_module = ext_loader.load_ext( + '_ext', ['get_compiler_version', 'get_compiling_cuda_version']) + + def get_compiler_version(): + return ext_module.get_compiler_version() + + def get_compiling_cuda_version(): + return ext_module.get_compiling_cuda_version() + + +def get_onnxruntime_op_path(): + wildcard = os.path.join( + os.path.abspath(os.path.dirname(os.path.dirname(__file__))), + '_ext_ort.*.so') + + paths = glob.glob(wildcard) + if len(paths) > 0: + return paths[0] + else: + return '' diff --git a/annotator/uniformer/mmcv/ops/iou3d.py b/annotator/uniformer/mmcv/ops/iou3d.py new file mode 100644 index 0000000000000000000000000000000000000000..6fc71979190323f44c09f8b7e1761cf49cd2d76b --- /dev/null +++ b/annotator/uniformer/mmcv/ops/iou3d.py @@ -0,0 +1,85 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', [ + 'iou3d_boxes_iou_bev_forward', 'iou3d_nms_forward', + 'iou3d_nms_normal_forward' +]) + + +def boxes_iou_bev(boxes_a, boxes_b): + """Calculate boxes IoU in the Bird's Eye View. + + Args: + boxes_a (torch.Tensor): Input boxes a with shape (M, 5). + boxes_b (torch.Tensor): Input boxes b with shape (N, 5). + + Returns: + ans_iou (torch.Tensor): IoU result with shape (M, N). + """ + ans_iou = boxes_a.new_zeros( + torch.Size((boxes_a.shape[0], boxes_b.shape[0]))) + + ext_module.iou3d_boxes_iou_bev_forward(boxes_a.contiguous(), + boxes_b.contiguous(), ans_iou) + + return ans_iou + + +def nms_bev(boxes, scores, thresh, pre_max_size=None, post_max_size=None): + """NMS function GPU implementation (for BEV boxes). The overlap of two + boxes for IoU calculation is defined as the exact overlapping area of the + two boxes. In this function, one can also set ``pre_max_size`` and + ``post_max_size``. + + Args: + boxes (torch.Tensor): Input boxes with the shape of [N, 5] + ([x1, y1, x2, y2, ry]). + scores (torch.Tensor): Scores of boxes with the shape of [N]. + thresh (float): Overlap threshold of NMS. + pre_max_size (int, optional): Max size of boxes before NMS. + Default: None. + post_max_size (int, optional): Max size of boxes after NMS. + Default: None. + + Returns: + torch.Tensor: Indexes after NMS. + """ + assert boxes.size(1) == 5, 'Input boxes shape should be [N, 5]' + order = scores.sort(0, descending=True)[1] + + if pre_max_size is not None: + order = order[:pre_max_size] + boxes = boxes[order].contiguous() + + keep = torch.zeros(boxes.size(0), dtype=torch.long) + num_out = ext_module.iou3d_nms_forward(boxes, keep, thresh) + keep = order[keep[:num_out].cuda(boxes.device)].contiguous() + if post_max_size is not None: + keep = keep[:post_max_size] + return keep + + +def nms_normal_bev(boxes, scores, thresh): + """Normal NMS function GPU implementation (for BEV boxes). The overlap of + two boxes for IoU calculation is defined as the exact overlapping area of + the two boxes WITH their yaw angle set to 0. + + Args: + boxes (torch.Tensor): Input boxes with shape (N, 5). + scores (torch.Tensor): Scores of predicted boxes with shape (N). + thresh (float): Overlap threshold of NMS. + + Returns: + torch.Tensor: Remaining indices with scores in descending order. + """ + assert boxes.shape[1] == 5, 'Input boxes shape should be [N, 5]' + order = scores.sort(0, descending=True)[1] + + boxes = boxes[order].contiguous() + + keep = torch.zeros(boxes.size(0), dtype=torch.long) + num_out = ext_module.iou3d_nms_normal_forward(boxes, keep, thresh) + return order[keep[:num_out].cuda(boxes.device)].contiguous() diff --git a/annotator/uniformer/mmcv/ops/knn.py b/annotator/uniformer/mmcv/ops/knn.py new file mode 100644 index 0000000000000000000000000000000000000000..f335785036669fc19239825b0aae6dde3f73bf92 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/knn.py @@ -0,0 +1,77 @@ +import torch +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', ['knn_forward']) + + +class KNN(Function): + r"""KNN (CUDA) based on heap data structure. + Modified from `PAConv `_. + + Find k-nearest points. + """ + + @staticmethod + def forward(ctx, + k: int, + xyz: torch.Tensor, + center_xyz: torch.Tensor = None, + transposed: bool = False) -> torch.Tensor: + """ + Args: + k (int): number of nearest neighbors. + xyz (Tensor): (B, N, 3) if transposed == False, else (B, 3, N). + xyz coordinates of the features. + center_xyz (Tensor, optional): (B, npoint, 3) if transposed == + False, else (B, 3, npoint). centers of the knn query. + Default: None. + transposed (bool, optional): whether the input tensors are + transposed. Should not explicitly use this keyword when + calling knn (=KNN.apply), just add the fourth param. + Default: False. + + Returns: + Tensor: (B, k, npoint) tensor with the indices of + the features that form k-nearest neighbours. + """ + assert (k > 0) & (k < 100), 'k should be in range(0, 100)' + + if center_xyz is None: + center_xyz = xyz + + if transposed: + xyz = xyz.transpose(2, 1).contiguous() + center_xyz = center_xyz.transpose(2, 1).contiguous() + + assert xyz.is_contiguous() # [B, N, 3] + assert center_xyz.is_contiguous() # [B, npoint, 3] + + center_xyz_device = center_xyz.get_device() + assert center_xyz_device == xyz.get_device(), \ + 'center_xyz and xyz should be put on the same device' + if torch.cuda.current_device() != center_xyz_device: + torch.cuda.set_device(center_xyz_device) + + B, npoint, _ = center_xyz.shape + N = xyz.shape[1] + + idx = center_xyz.new_zeros((B, npoint, k)).int() + dist2 = center_xyz.new_zeros((B, npoint, k)).float() + + ext_module.knn_forward( + xyz, center_xyz, idx, dist2, b=B, n=N, m=npoint, nsample=k) + # idx shape to [B, k, npoint] + idx = idx.transpose(2, 1).contiguous() + if torch.__version__ != 'parrots': + ctx.mark_non_differentiable(idx) + return idx + + @staticmethod + def backward(ctx, a=None): + return None, None, None + + +knn = KNN.apply diff --git a/annotator/uniformer/mmcv/ops/masked_conv.py b/annotator/uniformer/mmcv/ops/masked_conv.py new file mode 100644 index 0000000000000000000000000000000000000000..cd514cc204c1d571ea5dc7e74b038c0f477a008b --- /dev/null +++ b/annotator/uniformer/mmcv/ops/masked_conv.py @@ -0,0 +1,111 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import math + +import torch +import torch.nn as nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.utils import _pair + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['masked_im2col_forward', 'masked_col2im_forward']) + + +class MaskedConv2dFunction(Function): + + @staticmethod + def symbolic(g, features, mask, weight, bias, padding, stride): + return g.op( + 'mmcv::MMCVMaskedConv2d', + features, + mask, + weight, + bias, + padding_i=padding, + stride_i=stride) + + @staticmethod + def forward(ctx, features, mask, weight, bias, padding=0, stride=1): + assert mask.dim() == 3 and mask.size(0) == 1 + assert features.dim() == 4 and features.size(0) == 1 + assert features.size()[2:] == mask.size()[1:] + pad_h, pad_w = _pair(padding) + stride_h, stride_w = _pair(stride) + if stride_h != 1 or stride_w != 1: + raise ValueError( + 'Stride could not only be 1 in masked_conv2d currently.') + out_channel, in_channel, kernel_h, kernel_w = weight.size() + + batch_size = features.size(0) + out_h = int( + math.floor((features.size(2) + 2 * pad_h - + (kernel_h - 1) - 1) / stride_h + 1)) + out_w = int( + math.floor((features.size(3) + 2 * pad_w - + (kernel_h - 1) - 1) / stride_w + 1)) + mask_inds = torch.nonzero(mask[0] > 0, as_tuple=False) + output = features.new_zeros(batch_size, out_channel, out_h, out_w) + if mask_inds.numel() > 0: + mask_h_idx = mask_inds[:, 0].contiguous() + mask_w_idx = mask_inds[:, 1].contiguous() + data_col = features.new_zeros(in_channel * kernel_h * kernel_w, + mask_inds.size(0)) + ext_module.masked_im2col_forward( + features, + mask_h_idx, + mask_w_idx, + data_col, + kernel_h=kernel_h, + kernel_w=kernel_w, + pad_h=pad_h, + pad_w=pad_w) + + masked_output = torch.addmm(1, bias[:, None], 1, + weight.view(out_channel, -1), data_col) + ext_module.masked_col2im_forward( + masked_output, + mask_h_idx, + mask_w_idx, + output, + height=out_h, + width=out_w, + channels=out_channel) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + return (None, ) * 5 + + +masked_conv2d = MaskedConv2dFunction.apply + + +class MaskedConv2d(nn.Conv2d): + """A MaskedConv2d which inherits the official Conv2d. + + The masked forward doesn't implement the backward function and only + supports the stride parameter to be 1 currently. + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + bias=True): + super(MaskedConv2d, + self).__init__(in_channels, out_channels, kernel_size, stride, + padding, dilation, groups, bias) + + def forward(self, input, mask=None): + if mask is None: # fallback to the normal Conv2d + return super(MaskedConv2d, self).forward(input) + else: + return masked_conv2d(input, mask, self.weight, self.bias, + self.padding) diff --git a/annotator/uniformer/mmcv/ops/merge_cells.py b/annotator/uniformer/mmcv/ops/merge_cells.py new file mode 100644 index 0000000000000000000000000000000000000000..48ca8cc0a8aca8432835bd760c0403a3c35b34cf --- /dev/null +++ b/annotator/uniformer/mmcv/ops/merge_cells.py @@ -0,0 +1,149 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from abc import abstractmethod + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..cnn import ConvModule + + +class BaseMergeCell(nn.Module): + """The basic class for cells used in NAS-FPN and NAS-FCOS. + + BaseMergeCell takes 2 inputs. After applying convolution + on them, they are resized to the target size. Then, + they go through binary_op, which depends on the type of cell. + If with_out_conv is True, the result of output will go through + another convolution layer. + + Args: + in_channels (int): number of input channels in out_conv layer. + out_channels (int): number of output channels in out_conv layer. + with_out_conv (bool): Whether to use out_conv layer + out_conv_cfg (dict): Config dict for convolution layer, which should + contain "groups", "kernel_size", "padding", "bias" to build + out_conv layer. + out_norm_cfg (dict): Config dict for normalization layer in out_conv. + out_conv_order (tuple): The order of conv/norm/activation layers in + out_conv. + with_input1_conv (bool): Whether to use convolution on input1. + with_input2_conv (bool): Whether to use convolution on input2. + input_conv_cfg (dict): Config dict for building input1_conv layer and + input2_conv layer, which is expected to contain the type of + convolution. + Default: None, which means using conv2d. + input_norm_cfg (dict): Config dict for normalization layer in + input1_conv and input2_conv layer. Default: None. + upsample_mode (str): Interpolation method used to resize the output + of input1_conv and input2_conv to target size. Currently, we + support ['nearest', 'bilinear']. Default: 'nearest'. + """ + + def __init__(self, + fused_channels=256, + out_channels=256, + with_out_conv=True, + out_conv_cfg=dict( + groups=1, kernel_size=3, padding=1, bias=True), + out_norm_cfg=None, + out_conv_order=('act', 'conv', 'norm'), + with_input1_conv=False, + with_input2_conv=False, + input_conv_cfg=None, + input_norm_cfg=None, + upsample_mode='nearest'): + super(BaseMergeCell, self).__init__() + assert upsample_mode in ['nearest', 'bilinear'] + self.with_out_conv = with_out_conv + self.with_input1_conv = with_input1_conv + self.with_input2_conv = with_input2_conv + self.upsample_mode = upsample_mode + + if self.with_out_conv: + self.out_conv = ConvModule( + fused_channels, + out_channels, + **out_conv_cfg, + norm_cfg=out_norm_cfg, + order=out_conv_order) + + self.input1_conv = self._build_input_conv( + out_channels, input_conv_cfg, + input_norm_cfg) if with_input1_conv else nn.Sequential() + self.input2_conv = self._build_input_conv( + out_channels, input_conv_cfg, + input_norm_cfg) if with_input2_conv else nn.Sequential() + + def _build_input_conv(self, channel, conv_cfg, norm_cfg): + return ConvModule( + channel, + channel, + 3, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + bias=True) + + @abstractmethod + def _binary_op(self, x1, x2): + pass + + def _resize(self, x, size): + if x.shape[-2:] == size: + return x + elif x.shape[-2:] < size: + return F.interpolate(x, size=size, mode=self.upsample_mode) + else: + assert x.shape[-2] % size[-2] == 0 and x.shape[-1] % size[-1] == 0 + kernel_size = x.shape[-1] // size[-1] + x = F.max_pool2d(x, kernel_size=kernel_size, stride=kernel_size) + return x + + def forward(self, x1, x2, out_size=None): + assert x1.shape[:2] == x2.shape[:2] + assert out_size is None or len(out_size) == 2 + if out_size is None: # resize to larger one + out_size = max(x1.size()[2:], x2.size()[2:]) + + x1 = self.input1_conv(x1) + x2 = self.input2_conv(x2) + + x1 = self._resize(x1, out_size) + x2 = self._resize(x2, out_size) + + x = self._binary_op(x1, x2) + if self.with_out_conv: + x = self.out_conv(x) + return x + + +class SumCell(BaseMergeCell): + + def __init__(self, in_channels, out_channels, **kwargs): + super(SumCell, self).__init__(in_channels, out_channels, **kwargs) + + def _binary_op(self, x1, x2): + return x1 + x2 + + +class ConcatCell(BaseMergeCell): + + def __init__(self, in_channels, out_channels, **kwargs): + super(ConcatCell, self).__init__(in_channels * 2, out_channels, + **kwargs) + + def _binary_op(self, x1, x2): + ret = torch.cat([x1, x2], dim=1) + return ret + + +class GlobalPoolingCell(BaseMergeCell): + + def __init__(self, in_channels=None, out_channels=None, **kwargs): + super().__init__(in_channels, out_channels, **kwargs) + self.global_pool = nn.AdaptiveAvgPool2d((1, 1)) + + def _binary_op(self, x1, x2): + x2_att = self.global_pool(x2).sigmoid() + return x2 + x2_att * x1 diff --git a/annotator/uniformer/mmcv/ops/modulated_deform_conv.py b/annotator/uniformer/mmcv/ops/modulated_deform_conv.py new file mode 100644 index 0000000000000000000000000000000000000000..75559579cf053abcc99538606cbb88c723faf783 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/modulated_deform_conv.py @@ -0,0 +1,282 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import math + +import torch +import torch.nn as nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.utils import _pair, _single + +from annotator.uniformer.mmcv.utils import deprecated_api_warning +from ..cnn import CONV_LAYERS +from ..utils import ext_loader, print_log + +ext_module = ext_loader.load_ext( + '_ext', + ['modulated_deform_conv_forward', 'modulated_deform_conv_backward']) + + +class ModulatedDeformConv2dFunction(Function): + + @staticmethod + def symbolic(g, input, offset, mask, weight, bias, stride, padding, + dilation, groups, deform_groups): + input_tensors = [input, offset, mask, weight] + if bias is not None: + input_tensors.append(bias) + return g.op( + 'mmcv::MMCVModulatedDeformConv2d', + *input_tensors, + stride_i=stride, + padding_i=padding, + dilation_i=dilation, + groups_i=groups, + deform_groups_i=deform_groups) + + @staticmethod + def forward(ctx, + input, + offset, + mask, + weight, + bias=None, + stride=1, + padding=0, + dilation=1, + groups=1, + deform_groups=1): + if input is not None and input.dim() != 4: + raise ValueError( + f'Expected 4D tensor as input, got {input.dim()}D tensor \ + instead.') + ctx.stride = _pair(stride) + ctx.padding = _pair(padding) + ctx.dilation = _pair(dilation) + ctx.groups = groups + ctx.deform_groups = deform_groups + ctx.with_bias = bias is not None + if not ctx.with_bias: + bias = input.new_empty(0) # fake tensor + # When pytorch version >= 1.6.0, amp is adopted for fp16 mode; + # amp won't cast the type of model (float32), but "offset" is cast + # to float16 by nn.Conv2d automatically, leading to the type + # mismatch with input (when it is float32) or weight. + # The flag for whether to use fp16 or amp is the type of "offset", + # we cast weight and input to temporarily support fp16 and amp + # whatever the pytorch version is. + input = input.type_as(offset) + weight = weight.type_as(input) + ctx.save_for_backward(input, offset, mask, weight, bias) + output = input.new_empty( + ModulatedDeformConv2dFunction._output_size(ctx, input, weight)) + ctx._bufs = [input.new_empty(0), input.new_empty(0)] + ext_module.modulated_deform_conv_forward( + input, + weight, + bias, + ctx._bufs[0], + offset, + mask, + output, + ctx._bufs[1], + kernel_h=weight.size(2), + kernel_w=weight.size(3), + stride_h=ctx.stride[0], + stride_w=ctx.stride[1], + pad_h=ctx.padding[0], + pad_w=ctx.padding[1], + dilation_h=ctx.dilation[0], + dilation_w=ctx.dilation[1], + group=ctx.groups, + deformable_group=ctx.deform_groups, + with_bias=ctx.with_bias) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + input, offset, mask, weight, bias = ctx.saved_tensors + grad_input = torch.zeros_like(input) + grad_offset = torch.zeros_like(offset) + grad_mask = torch.zeros_like(mask) + grad_weight = torch.zeros_like(weight) + grad_bias = torch.zeros_like(bias) + grad_output = grad_output.contiguous() + ext_module.modulated_deform_conv_backward( + input, + weight, + bias, + ctx._bufs[0], + offset, + mask, + ctx._bufs[1], + grad_input, + grad_weight, + grad_bias, + grad_offset, + grad_mask, + grad_output, + kernel_h=weight.size(2), + kernel_w=weight.size(3), + stride_h=ctx.stride[0], + stride_w=ctx.stride[1], + pad_h=ctx.padding[0], + pad_w=ctx.padding[1], + dilation_h=ctx.dilation[0], + dilation_w=ctx.dilation[1], + group=ctx.groups, + deformable_group=ctx.deform_groups, + with_bias=ctx.with_bias) + if not ctx.with_bias: + grad_bias = None + + return (grad_input, grad_offset, grad_mask, grad_weight, grad_bias, + None, None, None, None, None) + + @staticmethod + def _output_size(ctx, input, weight): + channels = weight.size(0) + output_size = (input.size(0), channels) + for d in range(input.dim() - 2): + in_size = input.size(d + 2) + pad = ctx.padding[d] + kernel = ctx.dilation[d] * (weight.size(d + 2) - 1) + 1 + stride_ = ctx.stride[d] + output_size += ((in_size + (2 * pad) - kernel) // stride_ + 1, ) + if not all(map(lambda s: s > 0, output_size)): + raise ValueError( + 'convolution input is too small (output would be ' + + 'x'.join(map(str, output_size)) + ')') + return output_size + + +modulated_deform_conv2d = ModulatedDeformConv2dFunction.apply + + +class ModulatedDeformConv2d(nn.Module): + + @deprecated_api_warning({'deformable_groups': 'deform_groups'}, + cls_name='ModulatedDeformConv2d') + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + deform_groups=1, + bias=True): + super(ModulatedDeformConv2d, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = _pair(kernel_size) + self.stride = _pair(stride) + self.padding = _pair(padding) + self.dilation = _pair(dilation) + self.groups = groups + self.deform_groups = deform_groups + # enable compatibility with nn.Conv2d + self.transposed = False + self.output_padding = _single(0) + + self.weight = nn.Parameter( + torch.Tensor(out_channels, in_channels // groups, + *self.kernel_size)) + if bias: + self.bias = nn.Parameter(torch.Tensor(out_channels)) + else: + self.register_parameter('bias', None) + self.init_weights() + + def init_weights(self): + n = self.in_channels + for k in self.kernel_size: + n *= k + stdv = 1. / math.sqrt(n) + self.weight.data.uniform_(-stdv, stdv) + if self.bias is not None: + self.bias.data.zero_() + + def forward(self, x, offset, mask): + return modulated_deform_conv2d(x, offset, mask, self.weight, self.bias, + self.stride, self.padding, + self.dilation, self.groups, + self.deform_groups) + + +@CONV_LAYERS.register_module('DCNv2') +class ModulatedDeformConv2dPack(ModulatedDeformConv2d): + """A ModulatedDeformable Conv Encapsulation that acts as normal Conv + layers. + + Args: + in_channels (int): Same as nn.Conv2d. + out_channels (int): Same as nn.Conv2d. + kernel_size (int or tuple[int]): Same as nn.Conv2d. + stride (int): Same as nn.Conv2d, while tuple is not supported. + padding (int): Same as nn.Conv2d, while tuple is not supported. + dilation (int): Same as nn.Conv2d, while tuple is not supported. + groups (int): Same as nn.Conv2d. + bias (bool or str): If specified as `auto`, it will be decided by the + norm_cfg. Bias will be set as True if norm_cfg is None, otherwise + False. + """ + + _version = 2 + + def __init__(self, *args, **kwargs): + super(ModulatedDeformConv2dPack, self).__init__(*args, **kwargs) + self.conv_offset = nn.Conv2d( + self.in_channels, + self.deform_groups * 3 * self.kernel_size[0] * self.kernel_size[1], + kernel_size=self.kernel_size, + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + bias=True) + self.init_weights() + + def init_weights(self): + super(ModulatedDeformConv2dPack, self).init_weights() + if hasattr(self, 'conv_offset'): + self.conv_offset.weight.data.zero_() + self.conv_offset.bias.data.zero_() + + def forward(self, x): + out = self.conv_offset(x) + o1, o2, mask = torch.chunk(out, 3, dim=1) + offset = torch.cat((o1, o2), dim=1) + mask = torch.sigmoid(mask) + return modulated_deform_conv2d(x, offset, mask, self.weight, self.bias, + self.stride, self.padding, + self.dilation, self.groups, + self.deform_groups) + + def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, + missing_keys, unexpected_keys, error_msgs): + version = local_metadata.get('version', None) + + if version is None or version < 2: + # the key is different in early versions + # In version < 2, ModulatedDeformConvPack + # loads previous benchmark models. + if (prefix + 'conv_offset.weight' not in state_dict + and prefix[:-1] + '_offset.weight' in state_dict): + state_dict[prefix + 'conv_offset.weight'] = state_dict.pop( + prefix[:-1] + '_offset.weight') + if (prefix + 'conv_offset.bias' not in state_dict + and prefix[:-1] + '_offset.bias' in state_dict): + state_dict[prefix + + 'conv_offset.bias'] = state_dict.pop(prefix[:-1] + + '_offset.bias') + + if version is not None and version > 1: + print_log( + f'ModulatedDeformConvPack {prefix.rstrip(".")} is upgraded to ' + 'version 2.', + logger='root') + + super()._load_from_state_dict(state_dict, prefix, local_metadata, + strict, missing_keys, unexpected_keys, + error_msgs) diff --git a/annotator/uniformer/mmcv/ops/multi_scale_deform_attn.py b/annotator/uniformer/mmcv/ops/multi_scale_deform_attn.py new file mode 100644 index 0000000000000000000000000000000000000000..c52dda18b41705705b47dd0e995b124048c16fba --- /dev/null +++ b/annotator/uniformer/mmcv/ops/multi_scale_deform_attn.py @@ -0,0 +1,358 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import math +import warnings + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.autograd.function import Function, once_differentiable + +from annotator.uniformer.mmcv import deprecated_api_warning +from annotator.uniformer.mmcv.cnn import constant_init, xavier_init +from annotator.uniformer.mmcv.cnn.bricks.registry import ATTENTION +from annotator.uniformer.mmcv.runner import BaseModule +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['ms_deform_attn_backward', 'ms_deform_attn_forward']) + + +class MultiScaleDeformableAttnFunction(Function): + + @staticmethod + def forward(ctx, value, value_spatial_shapes, value_level_start_index, + sampling_locations, attention_weights, im2col_step): + """GPU version of multi-scale deformable attention. + + Args: + value (Tensor): The value has shape + (bs, num_keys, mum_heads, embed_dims//num_heads) + value_spatial_shapes (Tensor): Spatial shape of + each feature map, has shape (num_levels, 2), + last dimension 2 represent (h, w) + sampling_locations (Tensor): The location of sampling points, + has shape + (bs ,num_queries, num_heads, num_levels, num_points, 2), + the last dimension 2 represent (x, y). + attention_weights (Tensor): The weight of sampling points used + when calculate the attention, has shape + (bs ,num_queries, num_heads, num_levels, num_points), + im2col_step (Tensor): The step used in image to column. + + Returns: + Tensor: has shape (bs, num_queries, embed_dims) + """ + + ctx.im2col_step = im2col_step + output = ext_module.ms_deform_attn_forward( + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + im2col_step=ctx.im2col_step) + ctx.save_for_backward(value, value_spatial_shapes, + value_level_start_index, sampling_locations, + attention_weights) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + """GPU version of backward function. + + Args: + grad_output (Tensor): Gradient + of output tensor of forward. + + Returns: + Tuple[Tensor]: Gradient + of input tensors in forward. + """ + value, value_spatial_shapes, value_level_start_index,\ + sampling_locations, attention_weights = ctx.saved_tensors + grad_value = torch.zeros_like(value) + grad_sampling_loc = torch.zeros_like(sampling_locations) + grad_attn_weight = torch.zeros_like(attention_weights) + + ext_module.ms_deform_attn_backward( + value, + value_spatial_shapes, + value_level_start_index, + sampling_locations, + attention_weights, + grad_output.contiguous(), + grad_value, + grad_sampling_loc, + grad_attn_weight, + im2col_step=ctx.im2col_step) + + return grad_value, None, None, \ + grad_sampling_loc, grad_attn_weight, None + + +def multi_scale_deformable_attn_pytorch(value, value_spatial_shapes, + sampling_locations, attention_weights): + """CPU version of multi-scale deformable attention. + + Args: + value (Tensor): The value has shape + (bs, num_keys, mum_heads, embed_dims//num_heads) + value_spatial_shapes (Tensor): Spatial shape of + each feature map, has shape (num_levels, 2), + last dimension 2 represent (h, w) + sampling_locations (Tensor): The location of sampling points, + has shape + (bs ,num_queries, num_heads, num_levels, num_points, 2), + the last dimension 2 represent (x, y). + attention_weights (Tensor): The weight of sampling points used + when calculate the attention, has shape + (bs ,num_queries, num_heads, num_levels, num_points), + + Returns: + Tensor: has shape (bs, num_queries, embed_dims) + """ + + bs, _, num_heads, embed_dims = value.shape + _, num_queries, num_heads, num_levels, num_points, _ =\ + sampling_locations.shape + value_list = value.split([H_ * W_ for H_, W_ in value_spatial_shapes], + dim=1) + sampling_grids = 2 * sampling_locations - 1 + sampling_value_list = [] + for level, (H_, W_) in enumerate(value_spatial_shapes): + # bs, H_*W_, num_heads, embed_dims -> + # bs, H_*W_, num_heads*embed_dims -> + # bs, num_heads*embed_dims, H_*W_ -> + # bs*num_heads, embed_dims, H_, W_ + value_l_ = value_list[level].flatten(2).transpose(1, 2).reshape( + bs * num_heads, embed_dims, H_, W_) + # bs, num_queries, num_heads, num_points, 2 -> + # bs, num_heads, num_queries, num_points, 2 -> + # bs*num_heads, num_queries, num_points, 2 + sampling_grid_l_ = sampling_grids[:, :, :, + level].transpose(1, 2).flatten(0, 1) + # bs*num_heads, embed_dims, num_queries, num_points + sampling_value_l_ = F.grid_sample( + value_l_, + sampling_grid_l_, + mode='bilinear', + padding_mode='zeros', + align_corners=False) + sampling_value_list.append(sampling_value_l_) + # (bs, num_queries, num_heads, num_levels, num_points) -> + # (bs, num_heads, num_queries, num_levels, num_points) -> + # (bs, num_heads, 1, num_queries, num_levels*num_points) + attention_weights = attention_weights.transpose(1, 2).reshape( + bs * num_heads, 1, num_queries, num_levels * num_points) + output = (torch.stack(sampling_value_list, dim=-2).flatten(-2) * + attention_weights).sum(-1).view(bs, num_heads * embed_dims, + num_queries) + return output.transpose(1, 2).contiguous() + + +@ATTENTION.register_module() +class MultiScaleDeformableAttention(BaseModule): + """An attention module used in Deformable-Detr. + + `Deformable DETR: Deformable Transformers for End-to-End Object Detection. + `_. + + Args: + embed_dims (int): The embedding dimension of Attention. + Default: 256. + num_heads (int): Parallel attention heads. Default: 64. + num_levels (int): The number of feature map used in + Attention. Default: 4. + num_points (int): The number of sampling points for + each query in each head. Default: 4. + im2col_step (int): The step used in image_to_column. + Default: 64. + dropout (float): A Dropout layer on `inp_identity`. + Default: 0.1. + batch_first (bool): Key, Query and Value are shape of + (batch, n, embed_dim) + or (n, batch, embed_dim). Default to False. + norm_cfg (dict): Config dict for normalization layer. + Default: None. + init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. + Default: None. + """ + + def __init__(self, + embed_dims=256, + num_heads=8, + num_levels=4, + num_points=4, + im2col_step=64, + dropout=0.1, + batch_first=False, + norm_cfg=None, + init_cfg=None): + super().__init__(init_cfg) + if embed_dims % num_heads != 0: + raise ValueError(f'embed_dims must be divisible by num_heads, ' + f'but got {embed_dims} and {num_heads}') + dim_per_head = embed_dims // num_heads + self.norm_cfg = norm_cfg + self.dropout = nn.Dropout(dropout) + self.batch_first = batch_first + + # you'd better set dim_per_head to a power of 2 + # which is more efficient in the CUDA implementation + def _is_power_of_2(n): + if (not isinstance(n, int)) or (n < 0): + raise ValueError( + 'invalid input for _is_power_of_2: {} (type: {})'.format( + n, type(n))) + return (n & (n - 1) == 0) and n != 0 + + if not _is_power_of_2(dim_per_head): + warnings.warn( + "You'd better set embed_dims in " + 'MultiScaleDeformAttention to make ' + 'the dimension of each attention head a power of 2 ' + 'which is more efficient in our CUDA implementation.') + + self.im2col_step = im2col_step + self.embed_dims = embed_dims + self.num_levels = num_levels + self.num_heads = num_heads + self.num_points = num_points + self.sampling_offsets = nn.Linear( + embed_dims, num_heads * num_levels * num_points * 2) + self.attention_weights = nn.Linear(embed_dims, + num_heads * num_levels * num_points) + self.value_proj = nn.Linear(embed_dims, embed_dims) + self.output_proj = nn.Linear(embed_dims, embed_dims) + self.init_weights() + + def init_weights(self): + """Default initialization for Parameters of Module.""" + constant_init(self.sampling_offsets, 0.) + thetas = torch.arange( + self.num_heads, + dtype=torch.float32) * (2.0 * math.pi / self.num_heads) + grid_init = torch.stack([thetas.cos(), thetas.sin()], -1) + grid_init = (grid_init / + grid_init.abs().max(-1, keepdim=True)[0]).view( + self.num_heads, 1, 1, + 2).repeat(1, self.num_levels, self.num_points, 1) + for i in range(self.num_points): + grid_init[:, :, i, :] *= i + 1 + + self.sampling_offsets.bias.data = grid_init.view(-1) + constant_init(self.attention_weights, val=0., bias=0.) + xavier_init(self.value_proj, distribution='uniform', bias=0.) + xavier_init(self.output_proj, distribution='uniform', bias=0.) + self._is_init = True + + @deprecated_api_warning({'residual': 'identity'}, + cls_name='MultiScaleDeformableAttention') + def forward(self, + query, + key=None, + value=None, + identity=None, + query_pos=None, + key_padding_mask=None, + reference_points=None, + spatial_shapes=None, + level_start_index=None, + **kwargs): + """Forward Function of MultiScaleDeformAttention. + + Args: + query (Tensor): Query of Transformer with shape + (num_query, bs, embed_dims). + key (Tensor): The key tensor with shape + `(num_key, bs, embed_dims)`. + value (Tensor): The value tensor with shape + `(num_key, bs, embed_dims)`. + identity (Tensor): The tensor used for addition, with the + same shape as `query`. Default None. If None, + `query` will be used. + query_pos (Tensor): The positional encoding for `query`. + Default: None. + key_pos (Tensor): The positional encoding for `key`. Default + None. + reference_points (Tensor): The normalized reference + points with shape (bs, num_query, num_levels, 2), + all elements is range in [0, 1], top-left (0,0), + bottom-right (1, 1), including padding area. + or (N, Length_{query}, num_levels, 4), add + additional two dimensions is (w, h) to + form reference boxes. + key_padding_mask (Tensor): ByteTensor for `query`, with + shape [bs, num_key]. + spatial_shapes (Tensor): Spatial shape of features in + different levels. With shape (num_levels, 2), + last dimension represents (h, w). + level_start_index (Tensor): The start index of each level. + A tensor has shape ``(num_levels, )`` and can be represented + as [0, h_0*w_0, h_0*w_0+h_1*w_1, ...]. + + Returns: + Tensor: forwarded results with shape [num_query, bs, embed_dims]. + """ + + if value is None: + value = query + + if identity is None: + identity = query + if query_pos is not None: + query = query + query_pos + if not self.batch_first: + # change to (bs, num_query ,embed_dims) + query = query.permute(1, 0, 2) + value = value.permute(1, 0, 2) + + bs, num_query, _ = query.shape + bs, num_value, _ = value.shape + assert (spatial_shapes[:, 0] * spatial_shapes[:, 1]).sum() == num_value + + value = self.value_proj(value) + if key_padding_mask is not None: + value = value.masked_fill(key_padding_mask[..., None], 0.0) + value = value.view(bs, num_value, self.num_heads, -1) + sampling_offsets = self.sampling_offsets(query).view( + bs, num_query, self.num_heads, self.num_levels, self.num_points, 2) + attention_weights = self.attention_weights(query).view( + bs, num_query, self.num_heads, self.num_levels * self.num_points) + attention_weights = attention_weights.softmax(-1) + + attention_weights = attention_weights.view(bs, num_query, + self.num_heads, + self.num_levels, + self.num_points) + if reference_points.shape[-1] == 2: + offset_normalizer = torch.stack( + [spatial_shapes[..., 1], spatial_shapes[..., 0]], -1) + sampling_locations = reference_points[:, :, None, :, None, :] \ + + sampling_offsets \ + / offset_normalizer[None, None, None, :, None, :] + elif reference_points.shape[-1] == 4: + sampling_locations = reference_points[:, :, None, :, None, :2] \ + + sampling_offsets / self.num_points \ + * reference_points[:, :, None, :, None, 2:] \ + * 0.5 + else: + raise ValueError( + f'Last dim of reference_points must be' + f' 2 or 4, but get {reference_points.shape[-1]} instead.') + if torch.cuda.is_available() and value.is_cuda: + output = MultiScaleDeformableAttnFunction.apply( + value, spatial_shapes, level_start_index, sampling_locations, + attention_weights, self.im2col_step) + else: + output = multi_scale_deformable_attn_pytorch( + value, spatial_shapes, sampling_locations, attention_weights) + + output = self.output_proj(output) + + if not self.batch_first: + # (num_query, bs ,embed_dims) + output = output.permute(1, 0, 2) + + return self.dropout(output) + identity diff --git a/annotator/uniformer/mmcv/ops/nms.py b/annotator/uniformer/mmcv/ops/nms.py new file mode 100644 index 0000000000000000000000000000000000000000..6d9634281f486ab284091786886854c451368052 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/nms.py @@ -0,0 +1,417 @@ +import os + +import numpy as np +import torch + +from annotator.uniformer.mmcv.utils import deprecated_api_warning +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['nms', 'softnms', 'nms_match', 'nms_rotated']) + + +# This function is modified from: https://github.com/pytorch/vision/ +class NMSop(torch.autograd.Function): + + @staticmethod + def forward(ctx, bboxes, scores, iou_threshold, offset, score_threshold, + max_num): + is_filtering_by_score = score_threshold > 0 + if is_filtering_by_score: + valid_mask = scores > score_threshold + bboxes, scores = bboxes[valid_mask], scores[valid_mask] + valid_inds = torch.nonzero( + valid_mask, as_tuple=False).squeeze(dim=1) + + inds = ext_module.nms( + bboxes, scores, iou_threshold=float(iou_threshold), offset=offset) + + if max_num > 0: + inds = inds[:max_num] + if is_filtering_by_score: + inds = valid_inds[inds] + return inds + + @staticmethod + def symbolic(g, bboxes, scores, iou_threshold, offset, score_threshold, + max_num): + from ..onnx import is_custom_op_loaded + has_custom_op = is_custom_op_loaded() + # TensorRT nms plugin is aligned with original nms in ONNXRuntime + is_trt_backend = os.environ.get('ONNX_BACKEND') == 'MMCVTensorRT' + if has_custom_op and (not is_trt_backend): + return g.op( + 'mmcv::NonMaxSuppression', + bboxes, + scores, + iou_threshold_f=float(iou_threshold), + offset_i=int(offset)) + else: + from torch.onnx.symbolic_opset9 import select, squeeze, unsqueeze + from ..onnx.onnx_utils.symbolic_helper import _size_helper + + boxes = unsqueeze(g, bboxes, 0) + scores = unsqueeze(g, unsqueeze(g, scores, 0), 0) + + if max_num > 0: + max_num = g.op( + 'Constant', + value_t=torch.tensor(max_num, dtype=torch.long)) + else: + dim = g.op('Constant', value_t=torch.tensor(0)) + max_num = _size_helper(g, bboxes, dim) + max_output_per_class = max_num + iou_threshold = g.op( + 'Constant', + value_t=torch.tensor([iou_threshold], dtype=torch.float)) + score_threshold = g.op( + 'Constant', + value_t=torch.tensor([score_threshold], dtype=torch.float)) + nms_out = g.op('NonMaxSuppression', boxes, scores, + max_output_per_class, iou_threshold, + score_threshold) + return squeeze( + g, + select( + g, nms_out, 1, + g.op( + 'Constant', + value_t=torch.tensor([2], dtype=torch.long))), 1) + + +class SoftNMSop(torch.autograd.Function): + + @staticmethod + def forward(ctx, boxes, scores, iou_threshold, sigma, min_score, method, + offset): + dets = boxes.new_empty((boxes.size(0), 5), device='cpu') + inds = ext_module.softnms( + boxes.cpu(), + scores.cpu(), + dets.cpu(), + iou_threshold=float(iou_threshold), + sigma=float(sigma), + min_score=float(min_score), + method=int(method), + offset=int(offset)) + return dets, inds + + @staticmethod + def symbolic(g, boxes, scores, iou_threshold, sigma, min_score, method, + offset): + from packaging import version + assert version.parse(torch.__version__) >= version.parse('1.7.0') + nms_out = g.op( + 'mmcv::SoftNonMaxSuppression', + boxes, + scores, + iou_threshold_f=float(iou_threshold), + sigma_f=float(sigma), + min_score_f=float(min_score), + method_i=int(method), + offset_i=int(offset), + outputs=2) + return nms_out + + +@deprecated_api_warning({'iou_thr': 'iou_threshold'}) +def nms(boxes, scores, iou_threshold, offset=0, score_threshold=0, max_num=-1): + """Dispatch to either CPU or GPU NMS implementations. + + The input can be either torch tensor or numpy array. GPU NMS will be used + if the input is gpu tensor, otherwise CPU NMS + will be used. The returned type will always be the same as inputs. + + Arguments: + boxes (torch.Tensor or np.ndarray): boxes in shape (N, 4). + scores (torch.Tensor or np.ndarray): scores in shape (N, ). + iou_threshold (float): IoU threshold for NMS. + offset (int, 0 or 1): boxes' width or height is (x2 - x1 + offset). + score_threshold (float): score threshold for NMS. + max_num (int): maximum number of boxes after NMS. + + Returns: + tuple: kept dets(boxes and scores) and indice, which is always the \ + same data type as the input. + + Example: + >>> boxes = np.array([[49.1, 32.4, 51.0, 35.9], + >>> [49.3, 32.9, 51.0, 35.3], + >>> [49.2, 31.8, 51.0, 35.4], + >>> [35.1, 11.5, 39.1, 15.7], + >>> [35.6, 11.8, 39.3, 14.2], + >>> [35.3, 11.5, 39.9, 14.5], + >>> [35.2, 11.7, 39.7, 15.7]], dtype=np.float32) + >>> scores = np.array([0.9, 0.9, 0.5, 0.5, 0.5, 0.4, 0.3],\ + dtype=np.float32) + >>> iou_threshold = 0.6 + >>> dets, inds = nms(boxes, scores, iou_threshold) + >>> assert len(inds) == len(dets) == 3 + """ + assert isinstance(boxes, (torch.Tensor, np.ndarray)) + assert isinstance(scores, (torch.Tensor, np.ndarray)) + is_numpy = False + if isinstance(boxes, np.ndarray): + is_numpy = True + boxes = torch.from_numpy(boxes) + if isinstance(scores, np.ndarray): + scores = torch.from_numpy(scores) + assert boxes.size(1) == 4 + assert boxes.size(0) == scores.size(0) + assert offset in (0, 1) + + if torch.__version__ == 'parrots': + indata_list = [boxes, scores] + indata_dict = { + 'iou_threshold': float(iou_threshold), + 'offset': int(offset) + } + inds = ext_module.nms(*indata_list, **indata_dict) + else: + inds = NMSop.apply(boxes, scores, iou_threshold, offset, + score_threshold, max_num) + dets = torch.cat((boxes[inds], scores[inds].reshape(-1, 1)), dim=1) + if is_numpy: + dets = dets.cpu().numpy() + inds = inds.cpu().numpy() + return dets, inds + + +@deprecated_api_warning({'iou_thr': 'iou_threshold'}) +def soft_nms(boxes, + scores, + iou_threshold=0.3, + sigma=0.5, + min_score=1e-3, + method='linear', + offset=0): + """Dispatch to only CPU Soft NMS implementations. + + The input can be either a torch tensor or numpy array. + The returned type will always be the same as inputs. + + Arguments: + boxes (torch.Tensor or np.ndarray): boxes in shape (N, 4). + scores (torch.Tensor or np.ndarray): scores in shape (N, ). + iou_threshold (float): IoU threshold for NMS. + sigma (float): hyperparameter for gaussian method + min_score (float): score filter threshold + method (str): either 'linear' or 'gaussian' + offset (int, 0 or 1): boxes' width or height is (x2 - x1 + offset). + + Returns: + tuple: kept dets(boxes and scores) and indice, which is always the \ + same data type as the input. + + Example: + >>> boxes = np.array([[4., 3., 5., 3.], + >>> [4., 3., 5., 4.], + >>> [3., 1., 3., 1.], + >>> [3., 1., 3., 1.], + >>> [3., 1., 3., 1.], + >>> [3., 1., 3., 1.]], dtype=np.float32) + >>> scores = np.array([0.9, 0.9, 0.5, 0.5, 0.4, 0.0], dtype=np.float32) + >>> iou_threshold = 0.6 + >>> dets, inds = soft_nms(boxes, scores, iou_threshold, sigma=0.5) + >>> assert len(inds) == len(dets) == 5 + """ + + assert isinstance(boxes, (torch.Tensor, np.ndarray)) + assert isinstance(scores, (torch.Tensor, np.ndarray)) + is_numpy = False + if isinstance(boxes, np.ndarray): + is_numpy = True + boxes = torch.from_numpy(boxes) + if isinstance(scores, np.ndarray): + scores = torch.from_numpy(scores) + assert boxes.size(1) == 4 + assert boxes.size(0) == scores.size(0) + assert offset in (0, 1) + method_dict = {'naive': 0, 'linear': 1, 'gaussian': 2} + assert method in method_dict.keys() + + if torch.__version__ == 'parrots': + dets = boxes.new_empty((boxes.size(0), 5), device='cpu') + indata_list = [boxes.cpu(), scores.cpu(), dets.cpu()] + indata_dict = { + 'iou_threshold': float(iou_threshold), + 'sigma': float(sigma), + 'min_score': min_score, + 'method': method_dict[method], + 'offset': int(offset) + } + inds = ext_module.softnms(*indata_list, **indata_dict) + else: + dets, inds = SoftNMSop.apply(boxes.cpu(), scores.cpu(), + float(iou_threshold), float(sigma), + float(min_score), method_dict[method], + int(offset)) + + dets = dets[:inds.size(0)] + + if is_numpy: + dets = dets.cpu().numpy() + inds = inds.cpu().numpy() + return dets, inds + else: + return dets.to(device=boxes.device), inds.to(device=boxes.device) + + +def batched_nms(boxes, scores, idxs, nms_cfg, class_agnostic=False): + """Performs non-maximum suppression in a batched fashion. + + Modified from https://github.com/pytorch/vision/blob + /505cd6957711af790211896d32b40291bea1bc21/torchvision/ops/boxes.py#L39. + In order to perform NMS independently per class, we add an offset to all + the boxes. The offset is dependent only on the class idx, and is large + enough so that boxes from different classes do not overlap. + + Arguments: + boxes (torch.Tensor): boxes in shape (N, 4). + scores (torch.Tensor): scores in shape (N, ). + idxs (torch.Tensor): each index value correspond to a bbox cluster, + and NMS will not be applied between elements of different idxs, + shape (N, ). + nms_cfg (dict): specify nms type and other parameters like iou_thr. + Possible keys includes the following. + + - iou_thr (float): IoU threshold used for NMS. + - split_thr (float): threshold number of boxes. In some cases the + number of boxes is large (e.g., 200k). To avoid OOM during + training, the users could set `split_thr` to a small value. + If the number of boxes is greater than the threshold, it will + perform NMS on each group of boxes separately and sequentially. + Defaults to 10000. + class_agnostic (bool): if true, nms is class agnostic, + i.e. IoU thresholding happens over all boxes, + regardless of the predicted class. + + Returns: + tuple: kept dets and indice. + """ + nms_cfg_ = nms_cfg.copy() + class_agnostic = nms_cfg_.pop('class_agnostic', class_agnostic) + if class_agnostic: + boxes_for_nms = boxes + else: + max_coordinate = boxes.max() + offsets = idxs.to(boxes) * (max_coordinate + torch.tensor(1).to(boxes)) + boxes_for_nms = boxes + offsets[:, None] + + nms_type = nms_cfg_.pop('type', 'nms') + nms_op = eval(nms_type) + + split_thr = nms_cfg_.pop('split_thr', 10000) + # Won't split to multiple nms nodes when exporting to onnx + if boxes_for_nms.shape[0] < split_thr or torch.onnx.is_in_onnx_export(): + dets, keep = nms_op(boxes_for_nms, scores, **nms_cfg_) + boxes = boxes[keep] + # -1 indexing works abnormal in TensorRT + # This assumes `dets` has 5 dimensions where + # the last dimension is score. + # TODO: more elegant way to handle the dimension issue. + # Some type of nms would reweight the score, such as SoftNMS + scores = dets[:, 4] + else: + max_num = nms_cfg_.pop('max_num', -1) + total_mask = scores.new_zeros(scores.size(), dtype=torch.bool) + # Some type of nms would reweight the score, such as SoftNMS + scores_after_nms = scores.new_zeros(scores.size()) + for id in torch.unique(idxs): + mask = (idxs == id).nonzero(as_tuple=False).view(-1) + dets, keep = nms_op(boxes_for_nms[mask], scores[mask], **nms_cfg_) + total_mask[mask[keep]] = True + scores_after_nms[mask[keep]] = dets[:, -1] + keep = total_mask.nonzero(as_tuple=False).view(-1) + + scores, inds = scores_after_nms[keep].sort(descending=True) + keep = keep[inds] + boxes = boxes[keep] + + if max_num > 0: + keep = keep[:max_num] + boxes = boxes[:max_num] + scores = scores[:max_num] + + return torch.cat([boxes, scores[:, None]], -1), keep + + +def nms_match(dets, iou_threshold): + """Matched dets into different groups by NMS. + + NMS match is Similar to NMS but when a bbox is suppressed, nms match will + record the indice of suppressed bbox and form a group with the indice of + kept bbox. In each group, indice is sorted as score order. + + Arguments: + dets (torch.Tensor | np.ndarray): Det boxes with scores, shape (N, 5). + iou_thr (float): IoU thresh for NMS. + + Returns: + List[torch.Tensor | np.ndarray]: The outer list corresponds different + matched group, the inner Tensor corresponds the indices for a group + in score order. + """ + if dets.shape[0] == 0: + matched = [] + else: + assert dets.shape[-1] == 5, 'inputs dets.shape should be (N, 5), ' \ + f'but get {dets.shape}' + if isinstance(dets, torch.Tensor): + dets_t = dets.detach().cpu() + else: + dets_t = torch.from_numpy(dets) + indata_list = [dets_t] + indata_dict = {'iou_threshold': float(iou_threshold)} + matched = ext_module.nms_match(*indata_list, **indata_dict) + if torch.__version__ == 'parrots': + matched = matched.tolist() + + if isinstance(dets, torch.Tensor): + return [dets.new_tensor(m, dtype=torch.long) for m in matched] + else: + return [np.array(m, dtype=np.int) for m in matched] + + +def nms_rotated(dets, scores, iou_threshold, labels=None): + """Performs non-maximum suppression (NMS) on the rotated boxes according to + their intersection-over-union (IoU). + + Rotated NMS iteratively removes lower scoring rotated boxes which have an + IoU greater than iou_threshold with another (higher scoring) rotated box. + + Args: + boxes (Tensor): Rotated boxes in shape (N, 5). They are expected to \ + be in (x_ctr, y_ctr, width, height, angle_radian) format. + scores (Tensor): scores in shape (N, ). + iou_threshold (float): IoU thresh for NMS. + labels (Tensor): boxes' label in shape (N,). + + Returns: + tuple: kept dets(boxes and scores) and indice, which is always the \ + same data type as the input. + """ + if dets.shape[0] == 0: + return dets, None + multi_label = labels is not None + if multi_label: + dets_wl = torch.cat((dets, labels.unsqueeze(1)), 1) + else: + dets_wl = dets + _, order = scores.sort(0, descending=True) + dets_sorted = dets_wl.index_select(0, order) + + if torch.__version__ == 'parrots': + keep_inds = ext_module.nms_rotated( + dets_wl, + scores, + order, + dets_sorted, + iou_threshold=iou_threshold, + multi_label=multi_label) + else: + keep_inds = ext_module.nms_rotated(dets_wl, scores, order, dets_sorted, + iou_threshold, multi_label) + dets = torch.cat((dets[keep_inds], scores[keep_inds].reshape(-1, 1)), + dim=1) + return dets, keep_inds diff --git a/annotator/uniformer/mmcv/ops/pixel_group.py b/annotator/uniformer/mmcv/ops/pixel_group.py new file mode 100644 index 0000000000000000000000000000000000000000..2143c75f835a467c802fc3c37ecd3ac0f85bcda4 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/pixel_group.py @@ -0,0 +1,75 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numpy as np +import torch + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', ['pixel_group']) + + +def pixel_group(score, mask, embedding, kernel_label, kernel_contour, + kernel_region_num, distance_threshold): + """Group pixels into text instances, which is widely used text detection + methods. + + Arguments: + score (np.array or Tensor): The foreground score with size hxw. + mask (np.array or Tensor): The foreground mask with size hxw. + embedding (np.array or Tensor): The embedding with size hxwxc to + distinguish instances. + kernel_label (np.array or Tensor): The instance kernel index with + size hxw. + kernel_contour (np.array or Tensor): The kernel contour with size hxw. + kernel_region_num (int): The instance kernel region number. + distance_threshold (float): The embedding distance threshold between + kernel and pixel in one instance. + + Returns: + pixel_assignment (List[List[float]]): The instance coordinate list. + Each element consists of averaged confidence, pixel number, and + coordinates (x_i, y_i for all pixels) in order. + """ + assert isinstance(score, (torch.Tensor, np.ndarray)) + assert isinstance(mask, (torch.Tensor, np.ndarray)) + assert isinstance(embedding, (torch.Tensor, np.ndarray)) + assert isinstance(kernel_label, (torch.Tensor, np.ndarray)) + assert isinstance(kernel_contour, (torch.Tensor, np.ndarray)) + assert isinstance(kernel_region_num, int) + assert isinstance(distance_threshold, float) + + if isinstance(score, np.ndarray): + score = torch.from_numpy(score) + if isinstance(mask, np.ndarray): + mask = torch.from_numpy(mask) + if isinstance(embedding, np.ndarray): + embedding = torch.from_numpy(embedding) + if isinstance(kernel_label, np.ndarray): + kernel_label = torch.from_numpy(kernel_label) + if isinstance(kernel_contour, np.ndarray): + kernel_contour = torch.from_numpy(kernel_contour) + + if torch.__version__ == 'parrots': + label = ext_module.pixel_group( + score, + mask, + embedding, + kernel_label, + kernel_contour, + kernel_region_num=kernel_region_num, + distance_threshold=distance_threshold) + label = label.tolist() + label = label[0] + list_index = kernel_region_num + pixel_assignment = [] + for x in range(kernel_region_num): + pixel_assignment.append( + np.array( + label[list_index:list_index + int(label[x])], + dtype=np.float)) + list_index = list_index + int(label[x]) + else: + pixel_assignment = ext_module.pixel_group(score, mask, embedding, + kernel_label, kernel_contour, + kernel_region_num, + distance_threshold) + return pixel_assignment diff --git a/annotator/uniformer/mmcv/ops/point_sample.py b/annotator/uniformer/mmcv/ops/point_sample.py new file mode 100644 index 0000000000000000000000000000000000000000..267f4b3c56630acd85f9bdc630b7be09abab0aba --- /dev/null +++ b/annotator/uniformer/mmcv/ops/point_sample.py @@ -0,0 +1,336 @@ +# Modified from https://github.com/facebookresearch/detectron2/tree/master/projects/PointRend # noqa + +from os import path as osp + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.modules.utils import _pair +from torch.onnx.operators import shape_as_tensor + + +def bilinear_grid_sample(im, grid, align_corners=False): + """Given an input and a flow-field grid, computes the output using input + values and pixel locations from grid. Supported only bilinear interpolation + method to sample the input pixels. + + Args: + im (torch.Tensor): Input feature map, shape (N, C, H, W) + grid (torch.Tensor): Point coordinates, shape (N, Hg, Wg, 2) + align_corners {bool}: If set to True, the extrema (-1 and 1) are + considered as referring to the center points of the input’s + corner pixels. If set to False, they are instead considered as + referring to the corner points of the input’s corner pixels, + making the sampling more resolution agnostic. + Returns: + torch.Tensor: A tensor with sampled points, shape (N, C, Hg, Wg) + """ + n, c, h, w = im.shape + gn, gh, gw, _ = grid.shape + assert n == gn + + x = grid[:, :, :, 0] + y = grid[:, :, :, 1] + + if align_corners: + x = ((x + 1) / 2) * (w - 1) + y = ((y + 1) / 2) * (h - 1) + else: + x = ((x + 1) * w - 1) / 2 + y = ((y + 1) * h - 1) / 2 + + x = x.view(n, -1) + y = y.view(n, -1) + + x0 = torch.floor(x).long() + y0 = torch.floor(y).long() + x1 = x0 + 1 + y1 = y0 + 1 + + wa = ((x1 - x) * (y1 - y)).unsqueeze(1) + wb = ((x1 - x) * (y - y0)).unsqueeze(1) + wc = ((x - x0) * (y1 - y)).unsqueeze(1) + wd = ((x - x0) * (y - y0)).unsqueeze(1) + + # Apply default for grid_sample function zero padding + im_padded = F.pad(im, pad=[1, 1, 1, 1], mode='constant', value=0) + padded_h = h + 2 + padded_w = w + 2 + # save points positions after padding + x0, x1, y0, y1 = x0 + 1, x1 + 1, y0 + 1, y1 + 1 + + # Clip coordinates to padded image size + x0 = torch.where(x0 < 0, torch.tensor(0), x0) + x0 = torch.where(x0 > padded_w - 1, torch.tensor(padded_w - 1), x0) + x1 = torch.where(x1 < 0, torch.tensor(0), x1) + x1 = torch.where(x1 > padded_w - 1, torch.tensor(padded_w - 1), x1) + y0 = torch.where(y0 < 0, torch.tensor(0), y0) + y0 = torch.where(y0 > padded_h - 1, torch.tensor(padded_h - 1), y0) + y1 = torch.where(y1 < 0, torch.tensor(0), y1) + y1 = torch.where(y1 > padded_h - 1, torch.tensor(padded_h - 1), y1) + + im_padded = im_padded.view(n, c, -1) + + x0_y0 = (x0 + y0 * padded_w).unsqueeze(1).expand(-1, c, -1) + x0_y1 = (x0 + y1 * padded_w).unsqueeze(1).expand(-1, c, -1) + x1_y0 = (x1 + y0 * padded_w).unsqueeze(1).expand(-1, c, -1) + x1_y1 = (x1 + y1 * padded_w).unsqueeze(1).expand(-1, c, -1) + + Ia = torch.gather(im_padded, 2, x0_y0) + Ib = torch.gather(im_padded, 2, x0_y1) + Ic = torch.gather(im_padded, 2, x1_y0) + Id = torch.gather(im_padded, 2, x1_y1) + + return (Ia * wa + Ib * wb + Ic * wc + Id * wd).reshape(n, c, gh, gw) + + +def is_in_onnx_export_without_custom_ops(): + from annotator.uniformer.mmcv.ops import get_onnxruntime_op_path + ort_custom_op_path = get_onnxruntime_op_path() + return torch.onnx.is_in_onnx_export( + ) and not osp.exists(ort_custom_op_path) + + +def normalize(grid): + """Normalize input grid from [-1, 1] to [0, 1] + Args: + grid (Tensor): The grid to be normalize, range [-1, 1]. + Returns: + Tensor: Normalized grid, range [0, 1]. + """ + + return (grid + 1.0) / 2.0 + + +def denormalize(grid): + """Denormalize input grid from range [0, 1] to [-1, 1] + Args: + grid (Tensor): The grid to be denormalize, range [0, 1]. + Returns: + Tensor: Denormalized grid, range [-1, 1]. + """ + + return grid * 2.0 - 1.0 + + +def generate_grid(num_grid, size, device): + """Generate regular square grid of points in [0, 1] x [0, 1] coordinate + space. + + Args: + num_grid (int): The number of grids to sample, one for each region. + size (tuple(int, int)): The side size of the regular grid. + device (torch.device): Desired device of returned tensor. + + Returns: + (torch.Tensor): A tensor of shape (num_grid, size[0]*size[1], 2) that + contains coordinates for the regular grids. + """ + + affine_trans = torch.tensor([[[1., 0., 0.], [0., 1., 0.]]], device=device) + grid = F.affine_grid( + affine_trans, torch.Size((1, 1, *size)), align_corners=False) + grid = normalize(grid) + return grid.view(1, -1, 2).expand(num_grid, -1, -1) + + +def rel_roi_point_to_abs_img_point(rois, rel_roi_points): + """Convert roi based relative point coordinates to image based absolute + point coordinates. + + Args: + rois (Tensor): RoIs or BBoxes, shape (N, 4) or (N, 5) + rel_roi_points (Tensor): Point coordinates inside RoI, relative to + RoI, location, range (0, 1), shape (N, P, 2) + Returns: + Tensor: Image based absolute point coordinates, shape (N, P, 2) + """ + + with torch.no_grad(): + assert rel_roi_points.size(0) == rois.size(0) + assert rois.dim() == 2 + assert rel_roi_points.dim() == 3 + assert rel_roi_points.size(2) == 2 + # remove batch idx + if rois.size(1) == 5: + rois = rois[:, 1:] + abs_img_points = rel_roi_points.clone() + # To avoid an error during exporting to onnx use independent + # variables instead inplace computation + xs = abs_img_points[:, :, 0] * (rois[:, None, 2] - rois[:, None, 0]) + ys = abs_img_points[:, :, 1] * (rois[:, None, 3] - rois[:, None, 1]) + xs += rois[:, None, 0] + ys += rois[:, None, 1] + abs_img_points = torch.stack([xs, ys], dim=2) + return abs_img_points + + +def get_shape_from_feature_map(x): + """Get spatial resolution of input feature map considering exporting to + onnx mode. + + Args: + x (torch.Tensor): Input tensor, shape (N, C, H, W) + Returns: + torch.Tensor: Spatial resolution (width, height), shape (1, 1, 2) + """ + if torch.onnx.is_in_onnx_export(): + img_shape = shape_as_tensor(x)[2:].flip(0).view(1, 1, 2).to( + x.device).float() + else: + img_shape = torch.tensor(x.shape[2:]).flip(0).view(1, 1, 2).to( + x.device).float() + return img_shape + + +def abs_img_point_to_rel_img_point(abs_img_points, img, spatial_scale=1.): + """Convert image based absolute point coordinates to image based relative + coordinates for sampling. + + Args: + abs_img_points (Tensor): Image based absolute point coordinates, + shape (N, P, 2) + img (tuple/Tensor): (height, width) of image or feature map. + spatial_scale (float): Scale points by this factor. Default: 1. + + Returns: + Tensor: Image based relative point coordinates for sampling, + shape (N, P, 2) + """ + + assert (isinstance(img, tuple) and len(img) == 2) or \ + (isinstance(img, torch.Tensor) and len(img.shape) == 4) + + if isinstance(img, tuple): + h, w = img + scale = torch.tensor([w, h], + dtype=torch.float, + device=abs_img_points.device) + scale = scale.view(1, 1, 2) + else: + scale = get_shape_from_feature_map(img) + + return abs_img_points / scale * spatial_scale + + +def rel_roi_point_to_rel_img_point(rois, + rel_roi_points, + img, + spatial_scale=1.): + """Convert roi based relative point coordinates to image based absolute + point coordinates. + + Args: + rois (Tensor): RoIs or BBoxes, shape (N, 4) or (N, 5) + rel_roi_points (Tensor): Point coordinates inside RoI, relative to + RoI, location, range (0, 1), shape (N, P, 2) + img (tuple/Tensor): (height, width) of image or feature map. + spatial_scale (float): Scale points by this factor. Default: 1. + + Returns: + Tensor: Image based relative point coordinates for sampling, + shape (N, P, 2) + """ + + abs_img_point = rel_roi_point_to_abs_img_point(rois, rel_roi_points) + rel_img_point = abs_img_point_to_rel_img_point(abs_img_point, img, + spatial_scale) + + return rel_img_point + + +def point_sample(input, points, align_corners=False, **kwargs): + """A wrapper around :func:`grid_sample` to support 3D point_coords tensors + Unlike :func:`torch.nn.functional.grid_sample` it assumes point_coords to + lie inside ``[0, 1] x [0, 1]`` square. + + Args: + input (Tensor): Feature map, shape (N, C, H, W). + points (Tensor): Image based absolute point coordinates (normalized), + range [0, 1] x [0, 1], shape (N, P, 2) or (N, Hgrid, Wgrid, 2). + align_corners (bool): Whether align_corners. Default: False + + Returns: + Tensor: Features of `point` on `input`, shape (N, C, P) or + (N, C, Hgrid, Wgrid). + """ + + add_dim = False + if points.dim() == 3: + add_dim = True + points = points.unsqueeze(2) + if is_in_onnx_export_without_custom_ops(): + # If custom ops for onnx runtime not compiled use python + # implementation of grid_sample function to make onnx graph + # with supported nodes + output = bilinear_grid_sample( + input, denormalize(points), align_corners=align_corners) + else: + output = F.grid_sample( + input, denormalize(points), align_corners=align_corners, **kwargs) + if add_dim: + output = output.squeeze(3) + return output + + +class SimpleRoIAlign(nn.Module): + + def __init__(self, output_size, spatial_scale, aligned=True): + """Simple RoI align in PointRend, faster than standard RoIAlign. + + Args: + output_size (tuple[int]): h, w + spatial_scale (float): scale the input boxes by this number + aligned (bool): if False, use the legacy implementation in + MMDetection, align_corners=True will be used in F.grid_sample. + If True, align the results more perfectly. + """ + + super(SimpleRoIAlign, self).__init__() + self.output_size = _pair(output_size) + self.spatial_scale = float(spatial_scale) + # to be consistent with other RoI ops + self.use_torchvision = False + self.aligned = aligned + + def forward(self, features, rois): + num_imgs = features.size(0) + num_rois = rois.size(0) + rel_roi_points = generate_grid( + num_rois, self.output_size, device=rois.device) + + if torch.onnx.is_in_onnx_export(): + rel_img_points = rel_roi_point_to_rel_img_point( + rois, rel_roi_points, features, self.spatial_scale) + rel_img_points = rel_img_points.reshape(num_imgs, -1, + *rel_img_points.shape[1:]) + point_feats = point_sample( + features, rel_img_points, align_corners=not self.aligned) + point_feats = point_feats.transpose(1, 2) + else: + point_feats = [] + for batch_ind in range(num_imgs): + # unravel batch dim + feat = features[batch_ind].unsqueeze(0) + inds = (rois[:, 0].long() == batch_ind) + if inds.any(): + rel_img_points = rel_roi_point_to_rel_img_point( + rois[inds], rel_roi_points[inds], feat, + self.spatial_scale).unsqueeze(0) + point_feat = point_sample( + feat, rel_img_points, align_corners=not self.aligned) + point_feat = point_feat.squeeze(0).transpose(0, 1) + point_feats.append(point_feat) + + point_feats = torch.cat(point_feats, dim=0) + + channels = features.size(1) + roi_feats = point_feats.reshape(num_rois, channels, *self.output_size) + + return roi_feats + + def __repr__(self): + format_str = self.__class__.__name__ + format_str += '(output_size={}, spatial_scale={}'.format( + self.output_size, self.spatial_scale) + return format_str diff --git a/annotator/uniformer/mmcv/ops/points_in_boxes.py b/annotator/uniformer/mmcv/ops/points_in_boxes.py new file mode 100644 index 0000000000000000000000000000000000000000..4003173a53052161dbcd687a2fa1d755642fdab8 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/points_in_boxes.py @@ -0,0 +1,133 @@ +import torch + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', [ + 'points_in_boxes_part_forward', 'points_in_boxes_cpu_forward', + 'points_in_boxes_all_forward' +]) + + +def points_in_boxes_part(points, boxes): + """Find the box in which each point is (CUDA). + + Args: + points (torch.Tensor): [B, M, 3], [x, y, z] in LiDAR/DEPTH coordinate + boxes (torch.Tensor): [B, T, 7], + num_valid_boxes <= T, [x, y, z, x_size, y_size, z_size, rz] in + LiDAR/DEPTH coordinate, (x, y, z) is the bottom center + + Returns: + box_idxs_of_pts (torch.Tensor): (B, M), default background = -1 + """ + assert points.shape[0] == boxes.shape[0], \ + 'Points and boxes should have the same batch size, ' \ + f'but got {points.shape[0]} and {boxes.shape[0]}' + assert boxes.shape[2] == 7, \ + 'boxes dimension should be 7, ' \ + f'but got unexpected shape {boxes.shape[2]}' + assert points.shape[2] == 3, \ + 'points dimension should be 3, ' \ + f'but got unexpected shape {points.shape[2]}' + batch_size, num_points, _ = points.shape + + box_idxs_of_pts = points.new_zeros((batch_size, num_points), + dtype=torch.int).fill_(-1) + + # If manually put the tensor 'points' or 'boxes' on a device + # which is not the current device, some temporary variables + # will be created on the current device in the cuda op, + # and the output will be incorrect. + # Therefore, we force the current device to be the same + # as the device of the tensors if it was not. + # Please refer to https://github.com/open-mmlab/mmdetection3d/issues/305 + # for the incorrect output before the fix. + points_device = points.get_device() + assert points_device == boxes.get_device(), \ + 'Points and boxes should be put on the same device' + if torch.cuda.current_device() != points_device: + torch.cuda.set_device(points_device) + + ext_module.points_in_boxes_part_forward(boxes.contiguous(), + points.contiguous(), + box_idxs_of_pts) + + return box_idxs_of_pts + + +def points_in_boxes_cpu(points, boxes): + """Find all boxes in which each point is (CPU). The CPU version of + :meth:`points_in_boxes_all`. + + Args: + points (torch.Tensor): [B, M, 3], [x, y, z] in + LiDAR/DEPTH coordinate + boxes (torch.Tensor): [B, T, 7], + num_valid_boxes <= T, [x, y, z, x_size, y_size, z_size, rz], + (x, y, z) is the bottom center. + + Returns: + box_idxs_of_pts (torch.Tensor): (B, M, T), default background = 0. + """ + assert points.shape[0] == boxes.shape[0], \ + 'Points and boxes should have the same batch size, ' \ + f'but got {points.shape[0]} and {boxes.shape[0]}' + assert boxes.shape[2] == 7, \ + 'boxes dimension should be 7, ' \ + f'but got unexpected shape {boxes.shape[2]}' + assert points.shape[2] == 3, \ + 'points dimension should be 3, ' \ + f'but got unexpected shape {points.shape[2]}' + batch_size, num_points, _ = points.shape + num_boxes = boxes.shape[1] + + point_indices = points.new_zeros((batch_size, num_boxes, num_points), + dtype=torch.int) + for b in range(batch_size): + ext_module.points_in_boxes_cpu_forward(boxes[b].float().contiguous(), + points[b].float().contiguous(), + point_indices[b]) + point_indices = point_indices.transpose(1, 2) + + return point_indices + + +def points_in_boxes_all(points, boxes): + """Find all boxes in which each point is (CUDA). + + Args: + points (torch.Tensor): [B, M, 3], [x, y, z] in LiDAR/DEPTH coordinate + boxes (torch.Tensor): [B, T, 7], + num_valid_boxes <= T, [x, y, z, x_size, y_size, z_size, rz], + (x, y, z) is the bottom center. + + Returns: + box_idxs_of_pts (torch.Tensor): (B, M, T), default background = 0. + """ + assert boxes.shape[0] == points.shape[0], \ + 'Points and boxes should have the same batch size, ' \ + f'but got {boxes.shape[0]} and {boxes.shape[0]}' + assert boxes.shape[2] == 7, \ + 'boxes dimension should be 7, ' \ + f'but got unexpected shape {boxes.shape[2]}' + assert points.shape[2] == 3, \ + 'points dimension should be 3, ' \ + f'but got unexpected shape {points.shape[2]}' + batch_size, num_points, _ = points.shape + num_boxes = boxes.shape[1] + + box_idxs_of_pts = points.new_zeros((batch_size, num_points, num_boxes), + dtype=torch.int).fill_(0) + + # Same reason as line 25-32 + points_device = points.get_device() + assert points_device == boxes.get_device(), \ + 'Points and boxes should be put on the same device' + if torch.cuda.current_device() != points_device: + torch.cuda.set_device(points_device) + + ext_module.points_in_boxes_all_forward(boxes.contiguous(), + points.contiguous(), + box_idxs_of_pts) + + return box_idxs_of_pts diff --git a/annotator/uniformer/mmcv/ops/points_sampler.py b/annotator/uniformer/mmcv/ops/points_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..a802a74fd6c3610d9ae178e6201f47423eca7ad1 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/points_sampler.py @@ -0,0 +1,177 @@ +from typing import List + +import torch +from torch import nn as nn + +from annotator.uniformer.mmcv.runner import force_fp32 +from .furthest_point_sample import (furthest_point_sample, + furthest_point_sample_with_dist) + + +def calc_square_dist(point_feat_a, point_feat_b, norm=True): + """Calculating square distance between a and b. + + Args: + point_feat_a (Tensor): (B, N, C) Feature vector of each point. + point_feat_b (Tensor): (B, M, C) Feature vector of each point. + norm (Bool, optional): Whether to normalize the distance. + Default: True. + + Returns: + Tensor: (B, N, M) Distance between each pair points. + """ + num_channel = point_feat_a.shape[-1] + # [bs, n, 1] + a_square = torch.sum(point_feat_a.unsqueeze(dim=2).pow(2), dim=-1) + # [bs, 1, m] + b_square = torch.sum(point_feat_b.unsqueeze(dim=1).pow(2), dim=-1) + + corr_matrix = torch.matmul(point_feat_a, point_feat_b.transpose(1, 2)) + + dist = a_square + b_square - 2 * corr_matrix + if norm: + dist = torch.sqrt(dist) / num_channel + return dist + + +def get_sampler_cls(sampler_type): + """Get the type and mode of points sampler. + + Args: + sampler_type (str): The type of points sampler. + The valid value are "D-FPS", "F-FPS", or "FS". + + Returns: + class: Points sampler type. + """ + sampler_mappings = { + 'D-FPS': DFPSSampler, + 'F-FPS': FFPSSampler, + 'FS': FSSampler, + } + try: + return sampler_mappings[sampler_type] + except KeyError: + raise KeyError( + f'Supported `sampler_type` are {sampler_mappings.keys()}, but got \ + {sampler_type}') + + +class PointsSampler(nn.Module): + """Points sampling. + + Args: + num_point (list[int]): Number of sample points. + fps_mod_list (list[str], optional): Type of FPS method, valid mod + ['F-FPS', 'D-FPS', 'FS'], Default: ['D-FPS']. + F-FPS: using feature distances for FPS. + D-FPS: using Euclidean distances of points for FPS. + FS: using F-FPS and D-FPS simultaneously. + fps_sample_range_list (list[int], optional): + Range of points to apply FPS. Default: [-1]. + """ + + def __init__(self, + num_point: List[int], + fps_mod_list: List[str] = ['D-FPS'], + fps_sample_range_list: List[int] = [-1]): + super().__init__() + # FPS would be applied to different fps_mod in the list, + # so the length of the num_point should be equal to + # fps_mod_list and fps_sample_range_list. + assert len(num_point) == len(fps_mod_list) == len( + fps_sample_range_list) + self.num_point = num_point + self.fps_sample_range_list = fps_sample_range_list + self.samplers = nn.ModuleList() + for fps_mod in fps_mod_list: + self.samplers.append(get_sampler_cls(fps_mod)()) + self.fp16_enabled = False + + @force_fp32() + def forward(self, points_xyz, features): + """ + Args: + points_xyz (Tensor): (B, N, 3) xyz coordinates of the features. + features (Tensor): (B, C, N) Descriptors of the features. + + Returns: + Tensor: (B, npoint, sample_num) Indices of sampled points. + """ + indices = [] + last_fps_end_index = 0 + + for fps_sample_range, sampler, npoint in zip( + self.fps_sample_range_list, self.samplers, self.num_point): + assert fps_sample_range < points_xyz.shape[1] + + if fps_sample_range == -1: + sample_points_xyz = points_xyz[:, last_fps_end_index:] + if features is not None: + sample_features = features[:, :, last_fps_end_index:] + else: + sample_features = None + else: + sample_points_xyz = \ + points_xyz[:, last_fps_end_index:fps_sample_range] + if features is not None: + sample_features = features[:, :, last_fps_end_index: + fps_sample_range] + else: + sample_features = None + + fps_idx = sampler(sample_points_xyz.contiguous(), sample_features, + npoint) + + indices.append(fps_idx + last_fps_end_index) + last_fps_end_index += fps_sample_range + indices = torch.cat(indices, dim=1) + + return indices + + +class DFPSSampler(nn.Module): + """Using Euclidean distances of points for FPS.""" + + def __init__(self): + super().__init__() + + def forward(self, points, features, npoint): + """Sampling points with D-FPS.""" + fps_idx = furthest_point_sample(points.contiguous(), npoint) + return fps_idx + + +class FFPSSampler(nn.Module): + """Using feature distances for FPS.""" + + def __init__(self): + super().__init__() + + def forward(self, points, features, npoint): + """Sampling points with F-FPS.""" + assert features is not None, \ + 'feature input to FFPS_Sampler should not be None' + features_for_fps = torch.cat([points, features.transpose(1, 2)], dim=2) + features_dist = calc_square_dist( + features_for_fps, features_for_fps, norm=False) + fps_idx = furthest_point_sample_with_dist(features_dist, npoint) + return fps_idx + + +class FSSampler(nn.Module): + """Using F-FPS and D-FPS simultaneously.""" + + def __init__(self): + super().__init__() + + def forward(self, points, features, npoint): + """Sampling points with FS_Sampling.""" + assert features is not None, \ + 'feature input to FS_Sampler should not be None' + ffps_sampler = FFPSSampler() + dfps_sampler = DFPSSampler() + fps_idx_ffps = ffps_sampler(points, features, npoint) + fps_idx_dfps = dfps_sampler(points, features, npoint) + fps_idx = torch.cat([fps_idx_ffps, fps_idx_dfps], dim=1) + return fps_idx diff --git a/annotator/uniformer/mmcv/ops/psa_mask.py b/annotator/uniformer/mmcv/ops/psa_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..cdf14e62b50e8d4dd6856c94333c703bcc4c9ab6 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/psa_mask.py @@ -0,0 +1,92 @@ +# Modified from https://github.com/hszhao/semseg/blob/master/lib/psa +from torch import nn +from torch.autograd import Function +from torch.nn.modules.utils import _pair + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', + ['psamask_forward', 'psamask_backward']) + + +class PSAMaskFunction(Function): + + @staticmethod + def symbolic(g, input, psa_type, mask_size): + return g.op( + 'mmcv::MMCVPSAMask', + input, + psa_type_i=psa_type, + mask_size_i=mask_size) + + @staticmethod + def forward(ctx, input, psa_type, mask_size): + ctx.psa_type = psa_type + ctx.mask_size = _pair(mask_size) + ctx.save_for_backward(input) + + h_mask, w_mask = ctx.mask_size + batch_size, channels, h_feature, w_feature = input.size() + assert channels == h_mask * w_mask + output = input.new_zeros( + (batch_size, h_feature * w_feature, h_feature, w_feature)) + + ext_module.psamask_forward( + input, + output, + psa_type=psa_type, + num_=batch_size, + h_feature=h_feature, + w_feature=w_feature, + h_mask=h_mask, + w_mask=w_mask, + half_h_mask=(h_mask - 1) // 2, + half_w_mask=(w_mask - 1) // 2) + return output + + @staticmethod + def backward(ctx, grad_output): + input = ctx.saved_tensors[0] + psa_type = ctx.psa_type + h_mask, w_mask = ctx.mask_size + batch_size, channels, h_feature, w_feature = input.size() + grad_input = grad_output.new_zeros( + (batch_size, channels, h_feature, w_feature)) + ext_module.psamask_backward( + grad_output, + grad_input, + psa_type=psa_type, + num_=batch_size, + h_feature=h_feature, + w_feature=w_feature, + h_mask=h_mask, + w_mask=w_mask, + half_h_mask=(h_mask - 1) // 2, + half_w_mask=(w_mask - 1) // 2) + return grad_input, None, None, None + + +psa_mask = PSAMaskFunction.apply + + +class PSAMask(nn.Module): + + def __init__(self, psa_type, mask_size=None): + super(PSAMask, self).__init__() + assert psa_type in ['collect', 'distribute'] + if psa_type == 'collect': + psa_type_enum = 0 + else: + psa_type_enum = 1 + self.psa_type_enum = psa_type_enum + self.mask_size = mask_size + self.psa_type = psa_type + + def forward(self, input): + return psa_mask(input, self.psa_type_enum, self.mask_size) + + def __repr__(self): + s = self.__class__.__name__ + s += f'(psa_type={self.psa_type}, ' + s += f'mask_size={self.mask_size})' + return s diff --git a/annotator/uniformer/mmcv/ops/roi_align.py b/annotator/uniformer/mmcv/ops/roi_align.py new file mode 100644 index 0000000000000000000000000000000000000000..0755aefc66e67233ceae0f4b77948301c443e9fb --- /dev/null +++ b/annotator/uniformer/mmcv/ops/roi_align.py @@ -0,0 +1,223 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.utils import _pair + +from ..utils import deprecated_api_warning, ext_loader + +ext_module = ext_loader.load_ext('_ext', + ['roi_align_forward', 'roi_align_backward']) + + +class RoIAlignFunction(Function): + + @staticmethod + def symbolic(g, input, rois, output_size, spatial_scale, sampling_ratio, + pool_mode, aligned): + from ..onnx import is_custom_op_loaded + has_custom_op = is_custom_op_loaded() + if has_custom_op: + return g.op( + 'mmcv::MMCVRoiAlign', + input, + rois, + output_height_i=output_size[0], + output_width_i=output_size[1], + spatial_scale_f=spatial_scale, + sampling_ratio_i=sampling_ratio, + mode_s=pool_mode, + aligned_i=aligned) + else: + from torch.onnx.symbolic_opset9 import sub, squeeze + from torch.onnx.symbolic_helper import _slice_helper + from torch.onnx import TensorProtoDataType + # batch_indices = rois[:, 0].long() + batch_indices = _slice_helper( + g, rois, axes=[1], starts=[0], ends=[1]) + batch_indices = squeeze(g, batch_indices, 1) + batch_indices = g.op( + 'Cast', batch_indices, to_i=TensorProtoDataType.INT64) + # rois = rois[:, 1:] + rois = _slice_helper(g, rois, axes=[1], starts=[1], ends=[5]) + if aligned: + # rois -= 0.5/spatial_scale + aligned_offset = g.op( + 'Constant', + value_t=torch.tensor([0.5 / spatial_scale], + dtype=torch.float32)) + rois = sub(g, rois, aligned_offset) + # roi align + return g.op( + 'RoiAlign', + input, + rois, + batch_indices, + output_height_i=output_size[0], + output_width_i=output_size[1], + spatial_scale_f=spatial_scale, + sampling_ratio_i=max(0, sampling_ratio), + mode_s=pool_mode) + + @staticmethod + def forward(ctx, + input, + rois, + output_size, + spatial_scale=1.0, + sampling_ratio=0, + pool_mode='avg', + aligned=True): + ctx.output_size = _pair(output_size) + ctx.spatial_scale = spatial_scale + ctx.sampling_ratio = sampling_ratio + assert pool_mode in ('max', 'avg') + ctx.pool_mode = 0 if pool_mode == 'max' else 1 + ctx.aligned = aligned + ctx.input_shape = input.size() + + assert rois.size(1) == 5, 'RoI must be (idx, x1, y1, x2, y2)!' + + output_shape = (rois.size(0), input.size(1), ctx.output_size[0], + ctx.output_size[1]) + output = input.new_zeros(output_shape) + if ctx.pool_mode == 0: + argmax_y = input.new_zeros(output_shape) + argmax_x = input.new_zeros(output_shape) + else: + argmax_y = input.new_zeros(0) + argmax_x = input.new_zeros(0) + + ext_module.roi_align_forward( + input, + rois, + output, + argmax_y, + argmax_x, + aligned_height=ctx.output_size[0], + aligned_width=ctx.output_size[1], + spatial_scale=ctx.spatial_scale, + sampling_ratio=ctx.sampling_ratio, + pool_mode=ctx.pool_mode, + aligned=ctx.aligned) + + ctx.save_for_backward(rois, argmax_y, argmax_x) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + rois, argmax_y, argmax_x = ctx.saved_tensors + grad_input = grad_output.new_zeros(ctx.input_shape) + # complex head architecture may cause grad_output uncontiguous. + grad_output = grad_output.contiguous() + ext_module.roi_align_backward( + grad_output, + rois, + argmax_y, + argmax_x, + grad_input, + aligned_height=ctx.output_size[0], + aligned_width=ctx.output_size[1], + spatial_scale=ctx.spatial_scale, + sampling_ratio=ctx.sampling_ratio, + pool_mode=ctx.pool_mode, + aligned=ctx.aligned) + return grad_input, None, None, None, None, None, None + + +roi_align = RoIAlignFunction.apply + + +class RoIAlign(nn.Module): + """RoI align pooling layer. + + Args: + output_size (tuple): h, w + spatial_scale (float): scale the input boxes by this number + sampling_ratio (int): number of inputs samples to take for each + output sample. 0 to take samples densely for current models. + pool_mode (str, 'avg' or 'max'): pooling mode in each bin. + aligned (bool): if False, use the legacy implementation in + MMDetection. If True, align the results more perfectly. + use_torchvision (bool): whether to use roi_align from torchvision. + + Note: + The implementation of RoIAlign when aligned=True is modified from + https://github.com/facebookresearch/detectron2/ + + The meaning of aligned=True: + + Given a continuous coordinate c, its two neighboring pixel + indices (in our pixel model) are computed by floor(c - 0.5) and + ceil(c - 0.5). For example, c=1.3 has pixel neighbors with discrete + indices [0] and [1] (which are sampled from the underlying signal + at continuous coordinates 0.5 and 1.5). But the original roi_align + (aligned=False) does not subtract the 0.5 when computing + neighboring pixel indices and therefore it uses pixels with a + slightly incorrect alignment (relative to our pixel model) when + performing bilinear interpolation. + + With `aligned=True`, + we first appropriately scale the ROI and then shift it by -0.5 + prior to calling roi_align. This produces the correct neighbors; + + The difference does not make a difference to the model's + performance if ROIAlign is used together with conv layers. + """ + + @deprecated_api_warning( + { + 'out_size': 'output_size', + 'sample_num': 'sampling_ratio' + }, + cls_name='RoIAlign') + def __init__(self, + output_size, + spatial_scale=1.0, + sampling_ratio=0, + pool_mode='avg', + aligned=True, + use_torchvision=False): + super(RoIAlign, self).__init__() + + self.output_size = _pair(output_size) + self.spatial_scale = float(spatial_scale) + self.sampling_ratio = int(sampling_ratio) + self.pool_mode = pool_mode + self.aligned = aligned + self.use_torchvision = use_torchvision + + def forward(self, input, rois): + """ + Args: + input: NCHW images + rois: Bx5 boxes. First column is the index into N.\ + The other 4 columns are xyxy. + """ + if self.use_torchvision: + from torchvision.ops import roi_align as tv_roi_align + if 'aligned' in tv_roi_align.__code__.co_varnames: + return tv_roi_align(input, rois, self.output_size, + self.spatial_scale, self.sampling_ratio, + self.aligned) + else: + if self.aligned: + rois -= rois.new_tensor([0.] + + [0.5 / self.spatial_scale] * 4) + return tv_roi_align(input, rois, self.output_size, + self.spatial_scale, self.sampling_ratio) + else: + return roi_align(input, rois, self.output_size, self.spatial_scale, + self.sampling_ratio, self.pool_mode, self.aligned) + + def __repr__(self): + s = self.__class__.__name__ + s += f'(output_size={self.output_size}, ' + s += f'spatial_scale={self.spatial_scale}, ' + s += f'sampling_ratio={self.sampling_ratio}, ' + s += f'pool_mode={self.pool_mode}, ' + s += f'aligned={self.aligned}, ' + s += f'use_torchvision={self.use_torchvision})' + return s diff --git a/annotator/uniformer/mmcv/ops/roi_align_rotated.py b/annotator/uniformer/mmcv/ops/roi_align_rotated.py new file mode 100644 index 0000000000000000000000000000000000000000..0ce4961a3555d4da8bc3e32f1f7d5ad50036587d --- /dev/null +++ b/annotator/uniformer/mmcv/ops/roi_align_rotated.py @@ -0,0 +1,177 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch.nn as nn +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['roi_align_rotated_forward', 'roi_align_rotated_backward']) + + +class RoIAlignRotatedFunction(Function): + + @staticmethod + def symbolic(g, features, rois, out_size, spatial_scale, sample_num, + aligned, clockwise): + if isinstance(out_size, int): + out_h = out_size + out_w = out_size + elif isinstance(out_size, tuple): + assert len(out_size) == 2 + assert isinstance(out_size[0], int) + assert isinstance(out_size[1], int) + out_h, out_w = out_size + else: + raise TypeError( + '"out_size" must be an integer or tuple of integers') + return g.op( + 'mmcv::MMCVRoIAlignRotated', + features, + rois, + output_height_i=out_h, + output_width_i=out_h, + spatial_scale_f=spatial_scale, + sampling_ratio_i=sample_num, + aligned_i=aligned, + clockwise_i=clockwise) + + @staticmethod + def forward(ctx, + features, + rois, + out_size, + spatial_scale, + sample_num=0, + aligned=True, + clockwise=False): + if isinstance(out_size, int): + out_h = out_size + out_w = out_size + elif isinstance(out_size, tuple): + assert len(out_size) == 2 + assert isinstance(out_size[0], int) + assert isinstance(out_size[1], int) + out_h, out_w = out_size + else: + raise TypeError( + '"out_size" must be an integer or tuple of integers') + ctx.spatial_scale = spatial_scale + ctx.sample_num = sample_num + ctx.aligned = aligned + ctx.clockwise = clockwise + ctx.save_for_backward(rois) + ctx.feature_size = features.size() + + batch_size, num_channels, data_height, data_width = features.size() + num_rois = rois.size(0) + + output = features.new_zeros(num_rois, num_channels, out_h, out_w) + ext_module.roi_align_rotated_forward( + features, + rois, + output, + pooled_height=out_h, + pooled_width=out_w, + spatial_scale=spatial_scale, + sample_num=sample_num, + aligned=aligned, + clockwise=clockwise) + return output + + @staticmethod + def backward(ctx, grad_output): + feature_size = ctx.feature_size + spatial_scale = ctx.spatial_scale + aligned = ctx.aligned + clockwise = ctx.clockwise + sample_num = ctx.sample_num + rois = ctx.saved_tensors[0] + assert feature_size is not None + batch_size, num_channels, data_height, data_width = feature_size + + out_w = grad_output.size(3) + out_h = grad_output.size(2) + + grad_input = grad_rois = None + + if ctx.needs_input_grad[0]: + grad_input = rois.new_zeros(batch_size, num_channels, data_height, + data_width) + ext_module.roi_align_rotated_backward( + grad_output.contiguous(), + rois, + grad_input, + pooled_height=out_h, + pooled_width=out_w, + spatial_scale=spatial_scale, + sample_num=sample_num, + aligned=aligned, + clockwise=clockwise) + return grad_input, grad_rois, None, None, None, None, None + + +roi_align_rotated = RoIAlignRotatedFunction.apply + + +class RoIAlignRotated(nn.Module): + """RoI align pooling layer for rotated proposals. + + It accepts a feature map of shape (N, C, H, W) and rois with shape + (n, 6) with each roi decoded as (batch_index, center_x, center_y, + w, h, angle). The angle is in radian. + + Args: + out_size (tuple): h, w + spatial_scale (float): scale the input boxes by this number + sample_num (int): number of inputs samples to take for each + output sample. 0 to take samples densely for current models. + aligned (bool): if False, use the legacy implementation in + MMDetection. If True, align the results more perfectly. + Default: True. + clockwise (bool): If True, the angle in each proposal follows a + clockwise fashion in image space, otherwise, the angle is + counterclockwise. Default: False. + + Note: + The implementation of RoIAlign when aligned=True is modified from + https://github.com/facebookresearch/detectron2/ + + The meaning of aligned=True: + + Given a continuous coordinate c, its two neighboring pixel + indices (in our pixel model) are computed by floor(c - 0.5) and + ceil(c - 0.5). For example, c=1.3 has pixel neighbors with discrete + indices [0] and [1] (which are sampled from the underlying signal + at continuous coordinates 0.5 and 1.5). But the original roi_align + (aligned=False) does not subtract the 0.5 when computing + neighboring pixel indices and therefore it uses pixels with a + slightly incorrect alignment (relative to our pixel model) when + performing bilinear interpolation. + + With `aligned=True`, + we first appropriately scale the ROI and then shift it by -0.5 + prior to calling roi_align. This produces the correct neighbors; + + The difference does not make a difference to the model's + performance if ROIAlign is used together with conv layers. + """ + + def __init__(self, + out_size, + spatial_scale, + sample_num=0, + aligned=True, + clockwise=False): + super(RoIAlignRotated, self).__init__() + + self.out_size = out_size + self.spatial_scale = float(spatial_scale) + self.sample_num = int(sample_num) + self.aligned = aligned + self.clockwise = clockwise + + def forward(self, features, rois): + return RoIAlignRotatedFunction.apply(features, rois, self.out_size, + self.spatial_scale, + self.sample_num, self.aligned, + self.clockwise) diff --git a/annotator/uniformer/mmcv/ops/roi_pool.py b/annotator/uniformer/mmcv/ops/roi_pool.py new file mode 100644 index 0000000000000000000000000000000000000000..d339d8f2941eabc1cbe181a9c6c5ab5ff4ff4e5f --- /dev/null +++ b/annotator/uniformer/mmcv/ops/roi_pool.py @@ -0,0 +1,86 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.utils import _pair + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', + ['roi_pool_forward', 'roi_pool_backward']) + + +class RoIPoolFunction(Function): + + @staticmethod + def symbolic(g, input, rois, output_size, spatial_scale): + return g.op( + 'MaxRoiPool', + input, + rois, + pooled_shape_i=output_size, + spatial_scale_f=spatial_scale) + + @staticmethod + def forward(ctx, input, rois, output_size, spatial_scale=1.0): + ctx.output_size = _pair(output_size) + ctx.spatial_scale = spatial_scale + ctx.input_shape = input.size() + + assert rois.size(1) == 5, 'RoI must be (idx, x1, y1, x2, y2)!' + + output_shape = (rois.size(0), input.size(1), ctx.output_size[0], + ctx.output_size[1]) + output = input.new_zeros(output_shape) + argmax = input.new_zeros(output_shape, dtype=torch.int) + + ext_module.roi_pool_forward( + input, + rois, + output, + argmax, + pooled_height=ctx.output_size[0], + pooled_width=ctx.output_size[1], + spatial_scale=ctx.spatial_scale) + + ctx.save_for_backward(rois, argmax) + return output + + @staticmethod + @once_differentiable + def backward(ctx, grad_output): + rois, argmax = ctx.saved_tensors + grad_input = grad_output.new_zeros(ctx.input_shape) + + ext_module.roi_pool_backward( + grad_output, + rois, + argmax, + grad_input, + pooled_height=ctx.output_size[0], + pooled_width=ctx.output_size[1], + spatial_scale=ctx.spatial_scale) + + return grad_input, None, None, None + + +roi_pool = RoIPoolFunction.apply + + +class RoIPool(nn.Module): + + def __init__(self, output_size, spatial_scale=1.0): + super(RoIPool, self).__init__() + + self.output_size = _pair(output_size) + self.spatial_scale = float(spatial_scale) + + def forward(self, input, rois): + return roi_pool(input, rois, self.output_size, self.spatial_scale) + + def __repr__(self): + s = self.__class__.__name__ + s += f'(output_size={self.output_size}, ' + s += f'spatial_scale={self.spatial_scale})' + return s diff --git a/annotator/uniformer/mmcv/ops/roiaware_pool3d.py b/annotator/uniformer/mmcv/ops/roiaware_pool3d.py new file mode 100644 index 0000000000000000000000000000000000000000..291b0e5a9b692492c7d7e495ea639c46042e2f18 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/roiaware_pool3d.py @@ -0,0 +1,114 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch import nn as nn +from torch.autograd import Function + +import annotator.uniformer.mmcv as mmcv +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['roiaware_pool3d_forward', 'roiaware_pool3d_backward']) + + +class RoIAwarePool3d(nn.Module): + """Encode the geometry-specific features of each 3D proposal. + + Please refer to `PartA2 `_ for more + details. + + Args: + out_size (int or tuple): The size of output features. n or + [n1, n2, n3]. + max_pts_per_voxel (int, optional): The maximum number of points per + voxel. Default: 128. + mode (str, optional): Pooling method of RoIAware, 'max' or 'avg'. + Default: 'max'. + """ + + def __init__(self, out_size, max_pts_per_voxel=128, mode='max'): + super().__init__() + + self.out_size = out_size + self.max_pts_per_voxel = max_pts_per_voxel + assert mode in ['max', 'avg'] + pool_mapping = {'max': 0, 'avg': 1} + self.mode = pool_mapping[mode] + + def forward(self, rois, pts, pts_feature): + """ + Args: + rois (torch.Tensor): [N, 7], in LiDAR coordinate, + (x, y, z) is the bottom center of rois. + pts (torch.Tensor): [npoints, 3], coordinates of input points. + pts_feature (torch.Tensor): [npoints, C], features of input points. + + Returns: + pooled_features (torch.Tensor): [N, out_x, out_y, out_z, C] + """ + + return RoIAwarePool3dFunction.apply(rois, pts, pts_feature, + self.out_size, + self.max_pts_per_voxel, self.mode) + + +class RoIAwarePool3dFunction(Function): + + @staticmethod + def forward(ctx, rois, pts, pts_feature, out_size, max_pts_per_voxel, + mode): + """ + Args: + rois (torch.Tensor): [N, 7], in LiDAR coordinate, + (x, y, z) is the bottom center of rois. + pts (torch.Tensor): [npoints, 3], coordinates of input points. + pts_feature (torch.Tensor): [npoints, C], features of input points. + out_size (int or tuple): The size of output features. n or + [n1, n2, n3]. + max_pts_per_voxel (int): The maximum number of points per voxel. + Default: 128. + mode (int): Pooling method of RoIAware, 0 (max pool) or 1 (average + pool). + + Returns: + pooled_features (torch.Tensor): [N, out_x, out_y, out_z, C], output + pooled features. + """ + + if isinstance(out_size, int): + out_x = out_y = out_z = out_size + else: + assert len(out_size) == 3 + assert mmcv.is_tuple_of(out_size, int) + out_x, out_y, out_z = out_size + + num_rois = rois.shape[0] + num_channels = pts_feature.shape[-1] + num_pts = pts.shape[0] + + pooled_features = pts_feature.new_zeros( + (num_rois, out_x, out_y, out_z, num_channels)) + argmax = pts_feature.new_zeros( + (num_rois, out_x, out_y, out_z, num_channels), dtype=torch.int) + pts_idx_of_voxels = pts_feature.new_zeros( + (num_rois, out_x, out_y, out_z, max_pts_per_voxel), + dtype=torch.int) + + ext_module.roiaware_pool3d_forward(rois, pts, pts_feature, argmax, + pts_idx_of_voxels, pooled_features, + mode) + + ctx.roiaware_pool3d_for_backward = (pts_idx_of_voxels, argmax, mode, + num_pts, num_channels) + return pooled_features + + @staticmethod + def backward(ctx, grad_out): + ret = ctx.roiaware_pool3d_for_backward + pts_idx_of_voxels, argmax, mode, num_pts, num_channels = ret + + grad_in = grad_out.new_zeros((num_pts, num_channels)) + ext_module.roiaware_pool3d_backward(pts_idx_of_voxels, argmax, + grad_out.contiguous(), grad_in, + mode) + + return None, None, grad_in, None, None, None diff --git a/annotator/uniformer/mmcv/ops/roipoint_pool3d.py b/annotator/uniformer/mmcv/ops/roipoint_pool3d.py new file mode 100644 index 0000000000000000000000000000000000000000..0a21412c0728431c04b84245bc2e3109eea9aefc --- /dev/null +++ b/annotator/uniformer/mmcv/ops/roipoint_pool3d.py @@ -0,0 +1,77 @@ +from torch import nn as nn +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', ['roipoint_pool3d_forward']) + + +class RoIPointPool3d(nn.Module): + """Encode the geometry-specific features of each 3D proposal. + + Please refer to `Paper of PartA2 `_ + for more details. + + Args: + num_sampled_points (int, optional): Number of samples in each roi. + Default: 512. + """ + + def __init__(self, num_sampled_points=512): + super().__init__() + self.num_sampled_points = num_sampled_points + + def forward(self, points, point_features, boxes3d): + """ + Args: + points (torch.Tensor): Input points whose shape is (B, N, C). + point_features (torch.Tensor): Features of input points whose shape + is (B, N, C). + boxes3d (B, M, 7), Input bounding boxes whose shape is (B, M, 7). + + Returns: + pooled_features (torch.Tensor): The output pooled features whose + shape is (B, M, 512, 3 + C). + pooled_empty_flag (torch.Tensor): Empty flag whose shape is (B, M). + """ + return RoIPointPool3dFunction.apply(points, point_features, boxes3d, + self.num_sampled_points) + + +class RoIPointPool3dFunction(Function): + + @staticmethod + def forward(ctx, points, point_features, boxes3d, num_sampled_points=512): + """ + Args: + points (torch.Tensor): Input points whose shape is (B, N, C). + point_features (torch.Tensor): Features of input points whose shape + is (B, N, C). + boxes3d (B, M, 7), Input bounding boxes whose shape is (B, M, 7). + num_sampled_points (int, optional): The num of sampled points. + Default: 512. + + Returns: + pooled_features (torch.Tensor): The output pooled features whose + shape is (B, M, 512, 3 + C). + pooled_empty_flag (torch.Tensor): Empty flag whose shape is (B, M). + """ + assert len(points.shape) == 3 and points.shape[2] == 3 + batch_size, boxes_num, feature_len = points.shape[0], boxes3d.shape[ + 1], point_features.shape[2] + pooled_boxes3d = boxes3d.view(batch_size, -1, 7) + pooled_features = point_features.new_zeros( + (batch_size, boxes_num, num_sampled_points, 3 + feature_len)) + pooled_empty_flag = point_features.new_zeros( + (batch_size, boxes_num)).int() + + ext_module.roipoint_pool3d_forward(points.contiguous(), + pooled_boxes3d.contiguous(), + point_features.contiguous(), + pooled_features, pooled_empty_flag) + + return pooled_features, pooled_empty_flag + + @staticmethod + def backward(ctx, grad_out): + raise NotImplementedError diff --git a/annotator/uniformer/mmcv/ops/saconv.py b/annotator/uniformer/mmcv/ops/saconv.py new file mode 100644 index 0000000000000000000000000000000000000000..b4ee3978e097fca422805db4e31ae481006d7971 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/saconv.py @@ -0,0 +1,145 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F + +from annotator.uniformer.mmcv.cnn import CONV_LAYERS, ConvAWS2d, constant_init +from annotator.uniformer.mmcv.ops.deform_conv import deform_conv2d +from annotator.uniformer.mmcv.utils import TORCH_VERSION, digit_version + + +@CONV_LAYERS.register_module(name='SAC') +class SAConv2d(ConvAWS2d): + """SAC (Switchable Atrous Convolution) + + This is an implementation of SAC in DetectoRS + (https://arxiv.org/pdf/2006.02334.pdf). + + Args: + in_channels (int): Number of channels in the input image + out_channels (int): Number of channels produced by the convolution + kernel_size (int or tuple): Size of the convolving kernel + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of + the input. Default: 0 + padding_mode (string, optional): ``'zeros'``, ``'reflect'``, + ``'replicate'`` or ``'circular'``. Default: ``'zeros'`` + dilation (int or tuple, optional): Spacing between kernel elements. + Default: 1 + groups (int, optional): Number of blocked connections from input + channels to output channels. Default: 1 + bias (bool, optional): If ``True``, adds a learnable bias to the + output. Default: ``True`` + use_deform: If ``True``, replace convolution with deformable + convolution. Default: ``False``. + """ + + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + bias=True, + use_deform=False): + super().__init__( + in_channels, + out_channels, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups, + bias=bias) + self.use_deform = use_deform + self.switch = nn.Conv2d( + self.in_channels, 1, kernel_size=1, stride=stride, bias=True) + self.weight_diff = nn.Parameter(torch.Tensor(self.weight.size())) + self.pre_context = nn.Conv2d( + self.in_channels, self.in_channels, kernel_size=1, bias=True) + self.post_context = nn.Conv2d( + self.out_channels, self.out_channels, kernel_size=1, bias=True) + if self.use_deform: + self.offset_s = nn.Conv2d( + self.in_channels, + 18, + kernel_size=3, + padding=1, + stride=stride, + bias=True) + self.offset_l = nn.Conv2d( + self.in_channels, + 18, + kernel_size=3, + padding=1, + stride=stride, + bias=True) + self.init_weights() + + def init_weights(self): + constant_init(self.switch, 0, bias=1) + self.weight_diff.data.zero_() + constant_init(self.pre_context, 0) + constant_init(self.post_context, 0) + if self.use_deform: + constant_init(self.offset_s, 0) + constant_init(self.offset_l, 0) + + def forward(self, x): + # pre-context + avg_x = F.adaptive_avg_pool2d(x, output_size=1) + avg_x = self.pre_context(avg_x) + avg_x = avg_x.expand_as(x) + x = x + avg_x + # switch + avg_x = F.pad(x, pad=(2, 2, 2, 2), mode='reflect') + avg_x = F.avg_pool2d(avg_x, kernel_size=5, stride=1, padding=0) + switch = self.switch(avg_x) + # sac + weight = self._get_weight(self.weight) + zero_bias = torch.zeros( + self.out_channels, device=weight.device, dtype=weight.dtype) + + if self.use_deform: + offset = self.offset_s(avg_x) + out_s = deform_conv2d(x, offset, weight, self.stride, self.padding, + self.dilation, self.groups, 1) + else: + if (TORCH_VERSION == 'parrots' + or digit_version(TORCH_VERSION) < digit_version('1.5.0')): + out_s = super().conv2d_forward(x, weight) + elif digit_version(TORCH_VERSION) >= digit_version('1.8.0'): + # bias is a required argument of _conv_forward in torch 1.8.0 + out_s = super()._conv_forward(x, weight, zero_bias) + else: + out_s = super()._conv_forward(x, weight) + ori_p = self.padding + ori_d = self.dilation + self.padding = tuple(3 * p for p in self.padding) + self.dilation = tuple(3 * d for d in self.dilation) + weight = weight + self.weight_diff + if self.use_deform: + offset = self.offset_l(avg_x) + out_l = deform_conv2d(x, offset, weight, self.stride, self.padding, + self.dilation, self.groups, 1) + else: + if (TORCH_VERSION == 'parrots' + or digit_version(TORCH_VERSION) < digit_version('1.5.0')): + out_l = super().conv2d_forward(x, weight) + elif digit_version(TORCH_VERSION) >= digit_version('1.8.0'): + # bias is a required argument of _conv_forward in torch 1.8.0 + out_l = super()._conv_forward(x, weight, zero_bias) + else: + out_l = super()._conv_forward(x, weight) + + out = switch * out_s + (1 - switch) * out_l + self.padding = ori_p + self.dilation = ori_d + # post-context + avg_x = F.adaptive_avg_pool2d(out, output_size=1) + avg_x = self.post_context(avg_x) + avg_x = avg_x.expand_as(out) + out = out + avg_x + return out diff --git a/annotator/uniformer/mmcv/ops/scatter_points.py b/annotator/uniformer/mmcv/ops/scatter_points.py new file mode 100644 index 0000000000000000000000000000000000000000..2b8aa4169e9f6ca4a6f845ce17d6d1e4db416bb8 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/scatter_points.py @@ -0,0 +1,135 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch import nn +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', + ['dynamic_point_to_voxel_forward', 'dynamic_point_to_voxel_backward']) + + +class _DynamicScatter(Function): + + @staticmethod + def forward(ctx, feats, coors, reduce_type='max'): + """convert kitti points(N, >=3) to voxels. + + Args: + feats (torch.Tensor): [N, C]. Points features to be reduced + into voxels. + coors (torch.Tensor): [N, ndim]. Corresponding voxel coordinates + (specifically multi-dim voxel index) of each points. + reduce_type (str, optional): Reduce op. support 'max', 'sum' and + 'mean'. Default: 'max'. + + Returns: + voxel_feats (torch.Tensor): [M, C]. Reduced features, input + features that shares the same voxel coordinates are reduced to + one row. + voxel_coors (torch.Tensor): [M, ndim]. Voxel coordinates. + """ + results = ext_module.dynamic_point_to_voxel_forward( + feats, coors, reduce_type) + (voxel_feats, voxel_coors, point2voxel_map, + voxel_points_count) = results + ctx.reduce_type = reduce_type + ctx.save_for_backward(feats, voxel_feats, point2voxel_map, + voxel_points_count) + ctx.mark_non_differentiable(voxel_coors) + return voxel_feats, voxel_coors + + @staticmethod + def backward(ctx, grad_voxel_feats, grad_voxel_coors=None): + (feats, voxel_feats, point2voxel_map, + voxel_points_count) = ctx.saved_tensors + grad_feats = torch.zeros_like(feats) + # TODO: whether to use index put or use cuda_backward + # To use index put, need point to voxel index + ext_module.dynamic_point_to_voxel_backward( + grad_feats, grad_voxel_feats.contiguous(), feats, voxel_feats, + point2voxel_map, voxel_points_count, ctx.reduce_type) + return grad_feats, None, None + + +dynamic_scatter = _DynamicScatter.apply + + +class DynamicScatter(nn.Module): + """Scatters points into voxels, used in the voxel encoder with dynamic + voxelization. + + Note: + The CPU and GPU implementation get the same output, but have numerical + difference after summation and division (e.g., 5e-7). + + Args: + voxel_size (list): list [x, y, z] size of three dimension. + point_cloud_range (list): The coordinate range of points, [x_min, + y_min, z_min, x_max, y_max, z_max]. + average_points (bool): whether to use avg pooling to scatter points + into voxel. + """ + + def __init__(self, voxel_size, point_cloud_range, average_points: bool): + super().__init__() + + self.voxel_size = voxel_size + self.point_cloud_range = point_cloud_range + self.average_points = average_points + + def forward_single(self, points, coors): + """Scatters points into voxels. + + Args: + points (torch.Tensor): Points to be reduced into voxels. + coors (torch.Tensor): Corresponding voxel coordinates (specifically + multi-dim voxel index) of each points. + + Returns: + voxel_feats (torch.Tensor): Reduced features, input features that + shares the same voxel coordinates are reduced to one row. + voxel_coors (torch.Tensor): Voxel coordinates. + """ + reduce = 'mean' if self.average_points else 'max' + return dynamic_scatter(points.contiguous(), coors.contiguous(), reduce) + + def forward(self, points, coors): + """Scatters points/features into voxels. + + Args: + points (torch.Tensor): Points to be reduced into voxels. + coors (torch.Tensor): Corresponding voxel coordinates (specifically + multi-dim voxel index) of each points. + + Returns: + voxel_feats (torch.Tensor): Reduced features, input features that + shares the same voxel coordinates are reduced to one row. + voxel_coors (torch.Tensor): Voxel coordinates. + """ + if coors.size(-1) == 3: + return self.forward_single(points, coors) + else: + batch_size = coors[-1, 0] + 1 + voxels, voxel_coors = [], [] + for i in range(batch_size): + inds = torch.where(coors[:, 0] == i) + voxel, voxel_coor = self.forward_single( + points[inds], coors[inds][:, 1:]) + coor_pad = nn.functional.pad( + voxel_coor, (1, 0), mode='constant', value=i) + voxel_coors.append(coor_pad) + voxels.append(voxel) + features = torch.cat(voxels, dim=0) + feature_coors = torch.cat(voxel_coors, dim=0) + + return features, feature_coors + + def __repr__(self): + s = self.__class__.__name__ + '(' + s += 'voxel_size=' + str(self.voxel_size) + s += ', point_cloud_range=' + str(self.point_cloud_range) + s += ', average_points=' + str(self.average_points) + s += ')' + return s diff --git a/annotator/uniformer/mmcv/ops/sync_bn.py b/annotator/uniformer/mmcv/ops/sync_bn.py new file mode 100644 index 0000000000000000000000000000000000000000..c9b016fcbe860989c56cd1040034bcfa60e146d2 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/sync_bn.py @@ -0,0 +1,279 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.distributed as dist +import torch.nn.functional as F +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.nn.modules.module import Module +from torch.nn.parameter import Parameter + +from annotator.uniformer.mmcv.cnn import NORM_LAYERS +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', [ + 'sync_bn_forward_mean', 'sync_bn_forward_var', 'sync_bn_forward_output', + 'sync_bn_backward_param', 'sync_bn_backward_data' +]) + + +class SyncBatchNormFunction(Function): + + @staticmethod + def symbolic(g, input, running_mean, running_var, weight, bias, momentum, + eps, group, group_size, stats_mode): + return g.op( + 'mmcv::MMCVSyncBatchNorm', + input, + running_mean, + running_var, + weight, + bias, + momentum_f=momentum, + eps_f=eps, + group_i=group, + group_size_i=group_size, + stats_mode=stats_mode) + + @staticmethod + def forward(self, input, running_mean, running_var, weight, bias, momentum, + eps, group, group_size, stats_mode): + self.momentum = momentum + self.eps = eps + self.group = group + self.group_size = group_size + self.stats_mode = stats_mode + + assert isinstance( + input, (torch.HalfTensor, torch.FloatTensor, + torch.cuda.HalfTensor, torch.cuda.FloatTensor)), \ + f'only support Half or Float Tensor, but {input.type()}' + output = torch.zeros_like(input) + input3d = input.flatten(start_dim=2) + output3d = output.view_as(input3d) + num_channels = input3d.size(1) + + # ensure mean/var/norm/std are initialized as zeros + # ``torch.empty()`` does not guarantee that + mean = torch.zeros( + num_channels, dtype=torch.float, device=input3d.device) + var = torch.zeros( + num_channels, dtype=torch.float, device=input3d.device) + norm = torch.zeros_like( + input3d, dtype=torch.float, device=input3d.device) + std = torch.zeros( + num_channels, dtype=torch.float, device=input3d.device) + + batch_size = input3d.size(0) + if batch_size > 0: + ext_module.sync_bn_forward_mean(input3d, mean) + batch_flag = torch.ones([1], device=mean.device, dtype=mean.dtype) + else: + # skip updating mean and leave it as zeros when the input is empty + batch_flag = torch.zeros([1], device=mean.device, dtype=mean.dtype) + + # synchronize mean and the batch flag + vec = torch.cat([mean, batch_flag]) + if self.stats_mode == 'N': + vec *= batch_size + if self.group_size > 1: + dist.all_reduce(vec, group=self.group) + total_batch = vec[-1].detach() + mean = vec[:num_channels] + + if self.stats_mode == 'default': + mean = mean / self.group_size + elif self.stats_mode == 'N': + mean = mean / total_batch.clamp(min=1) + else: + raise NotImplementedError + + # leave var as zeros when the input is empty + if batch_size > 0: + ext_module.sync_bn_forward_var(input3d, mean, var) + + if self.stats_mode == 'N': + var *= batch_size + if self.group_size > 1: + dist.all_reduce(var, group=self.group) + + if self.stats_mode == 'default': + var /= self.group_size + elif self.stats_mode == 'N': + var /= total_batch.clamp(min=1) + else: + raise NotImplementedError + + # if the total batch size over all the ranks is zero, + # we should not update the statistics in the current batch + update_flag = total_batch.clamp(max=1) + momentum = update_flag * self.momentum + ext_module.sync_bn_forward_output( + input3d, + mean, + var, + weight, + bias, + running_mean, + running_var, + norm, + std, + output3d, + eps=self.eps, + momentum=momentum, + group_size=self.group_size) + self.save_for_backward(norm, std, weight) + return output + + @staticmethod + @once_differentiable + def backward(self, grad_output): + norm, std, weight = self.saved_tensors + grad_weight = torch.zeros_like(weight) + grad_bias = torch.zeros_like(weight) + grad_input = torch.zeros_like(grad_output) + grad_output3d = grad_output.flatten(start_dim=2) + grad_input3d = grad_input.view_as(grad_output3d) + + batch_size = grad_input3d.size(0) + if batch_size > 0: + ext_module.sync_bn_backward_param(grad_output3d, norm, grad_weight, + grad_bias) + + # all reduce + if self.group_size > 1: + dist.all_reduce(grad_weight, group=self.group) + dist.all_reduce(grad_bias, group=self.group) + grad_weight /= self.group_size + grad_bias /= self.group_size + + if batch_size > 0: + ext_module.sync_bn_backward_data(grad_output3d, weight, + grad_weight, grad_bias, norm, std, + grad_input3d) + + return grad_input, None, None, grad_weight, grad_bias, \ + None, None, None, None, None + + +@NORM_LAYERS.register_module(name='MMSyncBN') +class SyncBatchNorm(Module): + """Synchronized Batch Normalization. + + Args: + num_features (int): number of features/chennels in input tensor + eps (float, optional): a value added to the denominator for numerical + stability. Defaults to 1e-5. + momentum (float, optional): the value used for the running_mean and + running_var computation. Defaults to 0.1. + affine (bool, optional): whether to use learnable affine parameters. + Defaults to True. + track_running_stats (bool, optional): whether to track the running + mean and variance during training. When set to False, this + module does not track such statistics, and initializes statistics + buffers ``running_mean`` and ``running_var`` as ``None``. When + these buffers are ``None``, this module always uses batch + statistics in both training and eval modes. Defaults to True. + group (int, optional): synchronization of stats happen within + each process group individually. By default it is synchronization + across the whole world. Defaults to None. + stats_mode (str, optional): The statistical mode. Available options + includes ``'default'`` and ``'N'``. Defaults to 'default'. + When ``stats_mode=='default'``, it computes the overall statistics + using those from each worker with equal weight, i.e., the + statistics are synchronized and simply divied by ``group``. This + mode will produce inaccurate statistics when empty tensors occur. + When ``stats_mode=='N'``, it compute the overall statistics using + the total number of batches in each worker ignoring the number of + group, i.e., the statistics are synchronized and then divied by + the total batch ``N``. This mode is beneficial when empty tensors + occur during training, as it average the total mean by the real + number of batch. + """ + + def __init__(self, + num_features, + eps=1e-5, + momentum=0.1, + affine=True, + track_running_stats=True, + group=None, + stats_mode='default'): + super(SyncBatchNorm, self).__init__() + self.num_features = num_features + self.eps = eps + self.momentum = momentum + self.affine = affine + self.track_running_stats = track_running_stats + group = dist.group.WORLD if group is None else group + self.group = group + self.group_size = dist.get_world_size(group) + assert stats_mode in ['default', 'N'], \ + f'"stats_mode" only accepts "default" and "N", got "{stats_mode}"' + self.stats_mode = stats_mode + if self.affine: + self.weight = Parameter(torch.Tensor(num_features)) + self.bias = Parameter(torch.Tensor(num_features)) + else: + self.register_parameter('weight', None) + self.register_parameter('bias', None) + if self.track_running_stats: + self.register_buffer('running_mean', torch.zeros(num_features)) + self.register_buffer('running_var', torch.ones(num_features)) + self.register_buffer('num_batches_tracked', + torch.tensor(0, dtype=torch.long)) + else: + self.register_buffer('running_mean', None) + self.register_buffer('running_var', None) + self.register_buffer('num_batches_tracked', None) + self.reset_parameters() + + def reset_running_stats(self): + if self.track_running_stats: + self.running_mean.zero_() + self.running_var.fill_(1) + self.num_batches_tracked.zero_() + + def reset_parameters(self): + self.reset_running_stats() + if self.affine: + self.weight.data.uniform_() # pytorch use ones_() + self.bias.data.zero_() + + def forward(self, input): + if input.dim() < 2: + raise ValueError( + f'expected at least 2D input, got {input.dim()}D input') + if self.momentum is None: + exponential_average_factor = 0.0 + else: + exponential_average_factor = self.momentum + + if self.training and self.track_running_stats: + if self.num_batches_tracked is not None: + self.num_batches_tracked += 1 + if self.momentum is None: # use cumulative moving average + exponential_average_factor = 1.0 / float( + self.num_batches_tracked) + else: # use exponential moving average + exponential_average_factor = self.momentum + + if self.training or not self.track_running_stats: + return SyncBatchNormFunction.apply( + input, self.running_mean, self.running_var, self.weight, + self.bias, exponential_average_factor, self.eps, self.group, + self.group_size, self.stats_mode) + else: + return F.batch_norm(input, self.running_mean, self.running_var, + self.weight, self.bias, False, + exponential_average_factor, self.eps) + + def __repr__(self): + s = self.__class__.__name__ + s += f'({self.num_features}, ' + s += f'eps={self.eps}, ' + s += f'momentum={self.momentum}, ' + s += f'affine={self.affine}, ' + s += f'track_running_stats={self.track_running_stats}, ' + s += f'group_size={self.group_size},' + s += f'stats_mode={self.stats_mode})' + return s diff --git a/annotator/uniformer/mmcv/ops/three_interpolate.py b/annotator/uniformer/mmcv/ops/three_interpolate.py new file mode 100644 index 0000000000000000000000000000000000000000..203f47f05d58087e034fb3cd8cd6a09233947b4a --- /dev/null +++ b/annotator/uniformer/mmcv/ops/three_interpolate.py @@ -0,0 +1,68 @@ +from typing import Tuple + +import torch +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['three_interpolate_forward', 'three_interpolate_backward']) + + +class ThreeInterpolate(Function): + """Performs weighted linear interpolation on 3 features. + + Please refer to `Paper of PointNet++ `_ + for more details. + """ + + @staticmethod + def forward(ctx, features: torch.Tensor, indices: torch.Tensor, + weight: torch.Tensor) -> torch.Tensor: + """ + Args: + features (Tensor): (B, C, M) Features descriptors to be + interpolated + indices (Tensor): (B, n, 3) index three nearest neighbors + of the target features in features + weight (Tensor): (B, n, 3) weights of interpolation + + Returns: + Tensor: (B, C, N) tensor of the interpolated features + """ + assert features.is_contiguous() + assert indices.is_contiguous() + assert weight.is_contiguous() + + B, c, m = features.size() + n = indices.size(1) + ctx.three_interpolate_for_backward = (indices, weight, m) + output = torch.cuda.FloatTensor(B, c, n) + + ext_module.three_interpolate_forward( + features, indices, weight, output, b=B, c=c, m=m, n=n) + return output + + @staticmethod + def backward( + ctx, grad_out: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Args: + grad_out (Tensor): (B, C, N) tensor with gradients of outputs + + Returns: + Tensor: (B, C, M) tensor with gradients of features + """ + idx, weight, m = ctx.three_interpolate_for_backward + B, c, n = grad_out.size() + + grad_features = torch.cuda.FloatTensor(B, c, m).zero_() + grad_out_data = grad_out.data.contiguous() + + ext_module.three_interpolate_backward( + grad_out_data, idx, weight, grad_features.data, b=B, c=c, n=n, m=m) + return grad_features, None, None + + +three_interpolate = ThreeInterpolate.apply diff --git a/annotator/uniformer/mmcv/ops/three_nn.py b/annotator/uniformer/mmcv/ops/three_nn.py new file mode 100644 index 0000000000000000000000000000000000000000..2b01047a129989cd5545a0a86f23a487f4a13ce1 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/three_nn.py @@ -0,0 +1,51 @@ +from typing import Tuple + +import torch +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', ['three_nn_forward']) + + +class ThreeNN(Function): + """Find the top-3 nearest neighbors of the target set from the source set. + + Please refer to `Paper of PointNet++ `_ + for more details. + """ + + @staticmethod + def forward(ctx, target: torch.Tensor, + source: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Args: + target (Tensor): shape (B, N, 3), points set that needs to + find the nearest neighbors. + source (Tensor): shape (B, M, 3), points set that is used + to find the nearest neighbors of points in target set. + + Returns: + Tensor: shape (B, N, 3), L2 distance of each point in target + set to their corresponding nearest neighbors. + """ + target = target.contiguous() + source = source.contiguous() + + B, N, _ = target.size() + m = source.size(1) + dist2 = torch.cuda.FloatTensor(B, N, 3) + idx = torch.cuda.IntTensor(B, N, 3) + + ext_module.three_nn_forward(target, source, dist2, idx, b=B, n=N, m=m) + if torch.__version__ != 'parrots': + ctx.mark_non_differentiable(idx) + + return torch.sqrt(dist2), idx + + @staticmethod + def backward(ctx, a=None, b=None): + return None, None + + +three_nn = ThreeNN.apply diff --git a/annotator/uniformer/mmcv/ops/tin_shift.py b/annotator/uniformer/mmcv/ops/tin_shift.py new file mode 100644 index 0000000000000000000000000000000000000000..472c9fcfe45a124e819b7ed5653e585f94a8811e --- /dev/null +++ b/annotator/uniformer/mmcv/ops/tin_shift.py @@ -0,0 +1,68 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# Code reference from "Temporal Interlacing Network" +# https://github.com/deepcs233/TIN/blob/master/cuda_shift/rtc_wrap.py +# Hao Shao, Shengju Qian, Yu Liu +# shaoh19@mails.tsinghua.edu.cn, sjqian@cse.cuhk.edu.hk, yuliu@ee.cuhk.edu.hk + +import torch +import torch.nn as nn +from torch.autograd import Function + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext('_ext', + ['tin_shift_forward', 'tin_shift_backward']) + + +class TINShiftFunction(Function): + + @staticmethod + def forward(ctx, input, shift): + C = input.size(2) + num_segments = shift.size(1) + if C // num_segments <= 0 or C % num_segments != 0: + raise ValueError('C should be a multiple of num_segments, ' + f'but got C={C} and num_segments={num_segments}.') + + ctx.save_for_backward(shift) + + out = torch.zeros_like(input) + ext_module.tin_shift_forward(input, shift, out) + + return out + + @staticmethod + def backward(ctx, grad_output): + + shift = ctx.saved_tensors[0] + data_grad_input = grad_output.new(*grad_output.size()).zero_() + shift_grad_input = shift.new(*shift.size()).zero_() + ext_module.tin_shift_backward(grad_output, shift, data_grad_input) + + return data_grad_input, shift_grad_input + + +tin_shift = TINShiftFunction.apply + + +class TINShift(nn.Module): + """Temporal Interlace Shift. + + Temporal Interlace shift is a differentiable temporal-wise frame shifting + which is proposed in "Temporal Interlacing Network" + + Please refer to https://arxiv.org/abs/2001.06499 for more details. + Code is modified from https://github.com/mit-han-lab/temporal-shift-module + """ + + def forward(self, input, shift): + """Perform temporal interlace shift. + + Args: + input (Tensor): Feature map with shape [N, num_segments, C, H * W]. + shift (Tensor): Shift tensor with shape [N, num_segments]. + + Returns: + Feature map after temporal interlace shift. + """ + return tin_shift(input, shift) diff --git a/annotator/uniformer/mmcv/ops/upfirdn2d.py b/annotator/uniformer/mmcv/ops/upfirdn2d.py new file mode 100644 index 0000000000000000000000000000000000000000..c8bb2c3c949eed38a6465ed369fa881538dca010 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/upfirdn2d.py @@ -0,0 +1,330 @@ +# modified from https://github.com/rosinality/stylegan2-pytorch/blob/master/op/upfirdn2d.py # noqa:E501 + +# Copyright (c) 2021, NVIDIA Corporation. All rights reserved. +# NVIDIA Source Code License for StyleGAN2 with Adaptive Discriminator +# Augmentation (ADA) +# ======================================================================= + +# 1. Definitions + +# "Licensor" means any person or entity that distributes its Work. + +# "Software" means the original work of authorship made available under +# this License. + +# "Work" means the Software and any additions to or derivative works of +# the Software that are made available under this License. + +# The terms "reproduce," "reproduction," "derivative works," and +# "distribution" have the meaning as provided under U.S. copyright law; +# provided, however, that for the purposes of this License, derivative +# works shall not include works that remain separable from, or merely +# link (or bind by name) to the interfaces of, the Work. + +# Works, including the Software, are "made available" under this License +# by including in or with the Work either (a) a copyright notice +# referencing the applicability of this License to the Work, or (b) a +# copy of this License. + +# 2. License Grants + +# 2.1 Copyright Grant. Subject to the terms and conditions of this +# License, each Licensor grants to you a perpetual, worldwide, +# non-exclusive, royalty-free, copyright license to reproduce, +# prepare derivative works of, publicly display, publicly perform, +# sublicense and distribute its Work and any resulting derivative +# works in any form. + +# 3. Limitations + +# 3.1 Redistribution. You may reproduce or distribute the Work only +# if (a) you do so under this License, (b) you include a complete +# copy of this License with your distribution, and (c) you retain +# without modification any copyright, patent, trademark, or +# attribution notices that are present in the Work. + +# 3.2 Derivative Works. You may specify that additional or different +# terms apply to the use, reproduction, and distribution of your +# derivative works of the Work ("Your Terms") only if (a) Your Terms +# provide that the use limitation in Section 3.3 applies to your +# derivative works, and (b) you identify the specific derivative +# works that are subject to Your Terms. Notwithstanding Your Terms, +# this License (including the redistribution requirements in Section +# 3.1) will continue to apply to the Work itself. + +# 3.3 Use Limitation. The Work and any derivative works thereof only +# may be used or intended for use non-commercially. Notwithstanding +# the foregoing, NVIDIA and its affiliates may use the Work and any +# derivative works commercially. As used herein, "non-commercially" +# means for research or evaluation purposes only. + +# 3.4 Patent Claims. If you bring or threaten to bring a patent claim +# against any Licensor (including any claim, cross-claim or +# counterclaim in a lawsuit) to enforce any patents that you allege +# are infringed by any Work, then your rights under this License from +# such Licensor (including the grant in Section 2.1) will terminate +# immediately. + +# 3.5 Trademarks. This License does not grant any rights to use any +# Licensor’s or its affiliates’ names, logos, or trademarks, except +# as necessary to reproduce the notices described in this License. + +# 3.6 Termination. If you violate any term of this License, then your +# rights under this License (including the grant in Section 2.1) will +# terminate immediately. + +# 4. Disclaimer of Warranty. + +# THE WORK IS PROVIDED "AS IS" WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WARRANTIES OR CONDITIONS OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE OR +# NON-INFRINGEMENT. YOU BEAR THE RISK OF UNDERTAKING ANY ACTIVITIES UNDER +# THIS LICENSE. + +# 5. Limitation of Liability. + +# EXCEPT AS PROHIBITED BY APPLICABLE LAW, IN NO EVENT AND UNDER NO LEGAL +# THEORY, WHETHER IN TORT (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE +# SHALL ANY LICENSOR BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY DIRECT, +# INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF +# OR RELATED TO THIS LICENSE, THE USE OR INABILITY TO USE THE WORK +# (INCLUDING BUT NOT LIMITED TO LOSS OF GOODWILL, BUSINESS INTERRUPTION, +# LOST PROFITS OR DATA, COMPUTER FAILURE OR MALFUNCTION, OR ANY OTHER +# COMMERCIAL DAMAGES OR LOSSES), EVEN IF THE LICENSOR HAS BEEN ADVISED OF +# THE POSSIBILITY OF SUCH DAMAGES. + +# ======================================================================= + +import torch +from torch.autograd import Function +from torch.nn import functional as F + +from annotator.uniformer.mmcv.utils import to_2tuple +from ..utils import ext_loader + +upfirdn2d_ext = ext_loader.load_ext('_ext', ['upfirdn2d']) + + +class UpFirDn2dBackward(Function): + + @staticmethod + def forward(ctx, grad_output, kernel, grad_kernel, up, down, pad, g_pad, + in_size, out_size): + + up_x, up_y = up + down_x, down_y = down + g_pad_x0, g_pad_x1, g_pad_y0, g_pad_y1 = g_pad + + grad_output = grad_output.reshape(-1, out_size[0], out_size[1], 1) + + grad_input = upfirdn2d_ext.upfirdn2d( + grad_output, + grad_kernel, + up_x=down_x, + up_y=down_y, + down_x=up_x, + down_y=up_y, + pad_x0=g_pad_x0, + pad_x1=g_pad_x1, + pad_y0=g_pad_y0, + pad_y1=g_pad_y1) + grad_input = grad_input.view(in_size[0], in_size[1], in_size[2], + in_size[3]) + + ctx.save_for_backward(kernel) + + pad_x0, pad_x1, pad_y0, pad_y1 = pad + + ctx.up_x = up_x + ctx.up_y = up_y + ctx.down_x = down_x + ctx.down_y = down_y + ctx.pad_x0 = pad_x0 + ctx.pad_x1 = pad_x1 + ctx.pad_y0 = pad_y0 + ctx.pad_y1 = pad_y1 + ctx.in_size = in_size + ctx.out_size = out_size + + return grad_input + + @staticmethod + def backward(ctx, gradgrad_input): + kernel, = ctx.saved_tensors + + gradgrad_input = gradgrad_input.reshape(-1, ctx.in_size[2], + ctx.in_size[3], 1) + + gradgrad_out = upfirdn2d_ext.upfirdn2d( + gradgrad_input, + kernel, + up_x=ctx.up_x, + up_y=ctx.up_y, + down_x=ctx.down_x, + down_y=ctx.down_y, + pad_x0=ctx.pad_x0, + pad_x1=ctx.pad_x1, + pad_y0=ctx.pad_y0, + pad_y1=ctx.pad_y1) + # gradgrad_out = gradgrad_out.view(ctx.in_size[0], ctx.out_size[0], + # ctx.out_size[1], ctx.in_size[3]) + gradgrad_out = gradgrad_out.view(ctx.in_size[0], ctx.in_size[1], + ctx.out_size[0], ctx.out_size[1]) + + return gradgrad_out, None, None, None, None, None, None, None, None + + +class UpFirDn2d(Function): + + @staticmethod + def forward(ctx, input, kernel, up, down, pad): + up_x, up_y = up + down_x, down_y = down + pad_x0, pad_x1, pad_y0, pad_y1 = pad + + kernel_h, kernel_w = kernel.shape + batch, channel, in_h, in_w = input.shape + ctx.in_size = input.shape + + input = input.reshape(-1, in_h, in_w, 1) + + ctx.save_for_backward(kernel, torch.flip(kernel, [0, 1])) + + out_h = (in_h * up_y + pad_y0 + pad_y1 - kernel_h) // down_y + 1 + out_w = (in_w * up_x + pad_x0 + pad_x1 - kernel_w) // down_x + 1 + ctx.out_size = (out_h, out_w) + + ctx.up = (up_x, up_y) + ctx.down = (down_x, down_y) + ctx.pad = (pad_x0, pad_x1, pad_y0, pad_y1) + + g_pad_x0 = kernel_w - pad_x0 - 1 + g_pad_y0 = kernel_h - pad_y0 - 1 + g_pad_x1 = in_w * up_x - out_w * down_x + pad_x0 - up_x + 1 + g_pad_y1 = in_h * up_y - out_h * down_y + pad_y0 - up_y + 1 + + ctx.g_pad = (g_pad_x0, g_pad_x1, g_pad_y0, g_pad_y1) + + out = upfirdn2d_ext.upfirdn2d( + input, + kernel, + up_x=up_x, + up_y=up_y, + down_x=down_x, + down_y=down_y, + pad_x0=pad_x0, + pad_x1=pad_x1, + pad_y0=pad_y0, + pad_y1=pad_y1) + # out = out.view(major, out_h, out_w, minor) + out = out.view(-1, channel, out_h, out_w) + + return out + + @staticmethod + def backward(ctx, grad_output): + kernel, grad_kernel = ctx.saved_tensors + + grad_input = UpFirDn2dBackward.apply( + grad_output, + kernel, + grad_kernel, + ctx.up, + ctx.down, + ctx.pad, + ctx.g_pad, + ctx.in_size, + ctx.out_size, + ) + + return grad_input, None, None, None, None + + +def upfirdn2d(input, kernel, up=1, down=1, pad=(0, 0)): + """UpFRIDn for 2d features. + + UpFIRDn is short for upsample, apply FIR filter and downsample. More + details can be found in: + https://www.mathworks.com/help/signal/ref/upfirdn.html + + Args: + input (Tensor): Tensor with shape of (n, c, h, w). + kernel (Tensor): Filter kernel. + up (int | tuple[int], optional): Upsampling factor. If given a number, + we will use this factor for the both height and width side. + Defaults to 1. + down (int | tuple[int], optional): Downsampling factor. If given a + number, we will use this factor for the both height and width side. + Defaults to 1. + pad (tuple[int], optional): Padding for tensors, (x_pad, y_pad) or + (x_pad_0, x_pad_1, y_pad_0, y_pad_1). Defaults to (0, 0). + + Returns: + Tensor: Tensor after UpFIRDn. + """ + if input.device.type == 'cpu': + if len(pad) == 2: + pad = (pad[0], pad[1], pad[0], pad[1]) + + up = to_2tuple(up) + + down = to_2tuple(down) + + out = upfirdn2d_native(input, kernel, up[0], up[1], down[0], down[1], + pad[0], pad[1], pad[2], pad[3]) + else: + _up = to_2tuple(up) + + _down = to_2tuple(down) + + if len(pad) == 4: + _pad = pad + elif len(pad) == 2: + _pad = (pad[0], pad[1], pad[0], pad[1]) + + out = UpFirDn2d.apply(input, kernel, _up, _down, _pad) + + return out + + +def upfirdn2d_native(input, kernel, up_x, up_y, down_x, down_y, pad_x0, pad_x1, + pad_y0, pad_y1): + _, channel, in_h, in_w = input.shape + input = input.reshape(-1, in_h, in_w, 1) + + _, in_h, in_w, minor = input.shape + kernel_h, kernel_w = kernel.shape + + out = input.view(-1, in_h, 1, in_w, 1, minor) + out = F.pad(out, [0, 0, 0, up_x - 1, 0, 0, 0, up_y - 1]) + out = out.view(-1, in_h * up_y, in_w * up_x, minor) + + out = F.pad( + out, + [0, 0, + max(pad_x0, 0), + max(pad_x1, 0), + max(pad_y0, 0), + max(pad_y1, 0)]) + out = out[:, + max(-pad_y0, 0):out.shape[1] - max(-pad_y1, 0), + max(-pad_x0, 0):out.shape[2] - max(-pad_x1, 0), :, ] + + out = out.permute(0, 3, 1, 2) + out = out.reshape( + [-1, 1, in_h * up_y + pad_y0 + pad_y1, in_w * up_x + pad_x0 + pad_x1]) + w = torch.flip(kernel, [0, 1]).view(1, 1, kernel_h, kernel_w) + out = F.conv2d(out, w) + out = out.reshape( + -1, + minor, + in_h * up_y + pad_y0 + pad_y1 - kernel_h + 1, + in_w * up_x + pad_x0 + pad_x1 - kernel_w + 1, + ) + out = out.permute(0, 2, 3, 1) + out = out[:, ::down_y, ::down_x, :] + + out_h = (in_h * up_y + pad_y0 + pad_y1 - kernel_h) // down_y + 1 + out_w = (in_w * up_x + pad_x0 + pad_x1 - kernel_w) // down_x + 1 + + return out.view(-1, channel, out_h, out_w) diff --git a/annotator/uniformer/mmcv/ops/voxelize.py b/annotator/uniformer/mmcv/ops/voxelize.py new file mode 100644 index 0000000000000000000000000000000000000000..ca3226a4fbcbfe58490fa2ea8e1c16b531214121 --- /dev/null +++ b/annotator/uniformer/mmcv/ops/voxelize.py @@ -0,0 +1,132 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch import nn +from torch.autograd import Function +from torch.nn.modules.utils import _pair + +from ..utils import ext_loader + +ext_module = ext_loader.load_ext( + '_ext', ['dynamic_voxelize_forward', 'hard_voxelize_forward']) + + +class _Voxelization(Function): + + @staticmethod + def forward(ctx, + points, + voxel_size, + coors_range, + max_points=35, + max_voxels=20000): + """Convert kitti points(N, >=3) to voxels. + + Args: + points (torch.Tensor): [N, ndim]. Points[:, :3] contain xyz points + and points[:, 3:] contain other information like reflectivity. + voxel_size (tuple or float): The size of voxel with the shape of + [3]. + coors_range (tuple or float): The coordinate range of voxel with + the shape of [6]. + max_points (int, optional): maximum points contained in a voxel. if + max_points=-1, it means using dynamic_voxelize. Default: 35. + max_voxels (int, optional): maximum voxels this function create. + for second, 20000 is a good choice. Users should shuffle points + before call this function because max_voxels may drop points. + Default: 20000. + + Returns: + voxels_out (torch.Tensor): Output voxels with the shape of [M, + max_points, ndim]. Only contain points and returned when + max_points != -1. + coors_out (torch.Tensor): Output coordinates with the shape of + [M, 3]. + num_points_per_voxel_out (torch.Tensor): Num points per voxel with + the shape of [M]. Only returned when max_points != -1. + """ + if max_points == -1 or max_voxels == -1: + coors = points.new_zeros(size=(points.size(0), 3), dtype=torch.int) + ext_module.dynamic_voxelize_forward(points, coors, voxel_size, + coors_range, 3) + return coors + else: + voxels = points.new_zeros( + size=(max_voxels, max_points, points.size(1))) + coors = points.new_zeros(size=(max_voxels, 3), dtype=torch.int) + num_points_per_voxel = points.new_zeros( + size=(max_voxels, ), dtype=torch.int) + voxel_num = ext_module.hard_voxelize_forward( + points, voxels, coors, num_points_per_voxel, voxel_size, + coors_range, max_points, max_voxels, 3) + # select the valid voxels + voxels_out = voxels[:voxel_num] + coors_out = coors[:voxel_num] + num_points_per_voxel_out = num_points_per_voxel[:voxel_num] + return voxels_out, coors_out, num_points_per_voxel_out + + +voxelization = _Voxelization.apply + + +class Voxelization(nn.Module): + """Convert kitti points(N, >=3) to voxels. + + Please refer to `PVCNN `_ for more + details. + + Args: + voxel_size (tuple or float): The size of voxel with the shape of [3]. + point_cloud_range (tuple or float): The coordinate range of voxel with + the shape of [6]. + max_num_points (int): maximum points contained in a voxel. if + max_points=-1, it means using dynamic_voxelize. + max_voxels (int, optional): maximum voxels this function create. + for second, 20000 is a good choice. Users should shuffle points + before call this function because max_voxels may drop points. + Default: 20000. + """ + + def __init__(self, + voxel_size, + point_cloud_range, + max_num_points, + max_voxels=20000): + super().__init__() + + self.voxel_size = voxel_size + self.point_cloud_range = point_cloud_range + self.max_num_points = max_num_points + if isinstance(max_voxels, tuple): + self.max_voxels = max_voxels + else: + self.max_voxels = _pair(max_voxels) + + point_cloud_range = torch.tensor( + point_cloud_range, dtype=torch.float32) + voxel_size = torch.tensor(voxel_size, dtype=torch.float32) + grid_size = (point_cloud_range[3:] - + point_cloud_range[:3]) / voxel_size + grid_size = torch.round(grid_size).long() + input_feat_shape = grid_size[:2] + self.grid_size = grid_size + # the origin shape is as [x-len, y-len, z-len] + # [w, h, d] -> [d, h, w] + self.pcd_shape = [*input_feat_shape, 1][::-1] + + def forward(self, input): + if self.training: + max_voxels = self.max_voxels[0] + else: + max_voxels = self.max_voxels[1] + + return voxelization(input, self.voxel_size, self.point_cloud_range, + self.max_num_points, max_voxels) + + def __repr__(self): + s = self.__class__.__name__ + '(' + s += 'voxel_size=' + str(self.voxel_size) + s += ', point_cloud_range=' + str(self.point_cloud_range) + s += ', max_num_points=' + str(self.max_num_points) + s += ', max_voxels=' + str(self.max_voxels) + s += ')' + return s diff --git a/annotator/uniformer/mmcv/parallel/__init__.py b/annotator/uniformer/mmcv/parallel/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2ed2c17ad357742e423beeaf4d35db03fe9af469 --- /dev/null +++ b/annotator/uniformer/mmcv/parallel/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .collate import collate +from .data_container import DataContainer +from .data_parallel import MMDataParallel +from .distributed import MMDistributedDataParallel +from .registry import MODULE_WRAPPERS +from .scatter_gather import scatter, scatter_kwargs +from .utils import is_module_wrapper + +__all__ = [ + 'collate', 'DataContainer', 'MMDataParallel', 'MMDistributedDataParallel', + 'scatter', 'scatter_kwargs', 'is_module_wrapper', 'MODULE_WRAPPERS' +] diff --git a/annotator/uniformer/mmcv/parallel/_functions.py b/annotator/uniformer/mmcv/parallel/_functions.py new file mode 100644 index 0000000000000000000000000000000000000000..9b5a8a44483ab991411d07122b22a1d027e4be8e --- /dev/null +++ b/annotator/uniformer/mmcv/parallel/_functions.py @@ -0,0 +1,79 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch.nn.parallel._functions import _get_stream + + +def scatter(input, devices, streams=None): + """Scatters tensor across multiple GPUs.""" + if streams is None: + streams = [None] * len(devices) + + if isinstance(input, list): + chunk_size = (len(input) - 1) // len(devices) + 1 + outputs = [ + scatter(input[i], [devices[i // chunk_size]], + [streams[i // chunk_size]]) for i in range(len(input)) + ] + return outputs + elif isinstance(input, torch.Tensor): + output = input.contiguous() + # TODO: copy to a pinned buffer first (if copying from CPU) + stream = streams[0] if output.numel() > 0 else None + if devices != [-1]: + with torch.cuda.device(devices[0]), torch.cuda.stream(stream): + output = output.cuda(devices[0], non_blocking=True) + else: + # unsqueeze the first dimension thus the tensor's shape is the + # same as those scattered with GPU. + output = output.unsqueeze(0) + return output + else: + raise Exception(f'Unknown type {type(input)}.') + + +def synchronize_stream(output, devices, streams): + if isinstance(output, list): + chunk_size = len(output) // len(devices) + for i in range(len(devices)): + for j in range(chunk_size): + synchronize_stream(output[i * chunk_size + j], [devices[i]], + [streams[i]]) + elif isinstance(output, torch.Tensor): + if output.numel() != 0: + with torch.cuda.device(devices[0]): + main_stream = torch.cuda.current_stream() + main_stream.wait_stream(streams[0]) + output.record_stream(main_stream) + else: + raise Exception(f'Unknown type {type(output)}.') + + +def get_input_device(input): + if isinstance(input, list): + for item in input: + input_device = get_input_device(item) + if input_device != -1: + return input_device + return -1 + elif isinstance(input, torch.Tensor): + return input.get_device() if input.is_cuda else -1 + else: + raise Exception(f'Unknown type {type(input)}.') + + +class Scatter: + + @staticmethod + def forward(target_gpus, input): + input_device = get_input_device(input) + streams = None + if input_device == -1 and target_gpus != [-1]: + # Perform CPU to GPU copies in a background stream + streams = [_get_stream(device) for device in target_gpus] + + outputs = scatter(input, target_gpus, streams) + # Synchronize with the copy stream + if streams is not None: + synchronize_stream(outputs, target_gpus, streams) + + return tuple(outputs) diff --git a/annotator/uniformer/mmcv/parallel/collate.py b/annotator/uniformer/mmcv/parallel/collate.py new file mode 100644 index 0000000000000000000000000000000000000000..ad749197df21b0d74297548be5f66a696adebf7f --- /dev/null +++ b/annotator/uniformer/mmcv/parallel/collate.py @@ -0,0 +1,84 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from collections.abc import Mapping, Sequence + +import torch +import torch.nn.functional as F +from torch.utils.data.dataloader import default_collate + +from .data_container import DataContainer + + +def collate(batch, samples_per_gpu=1): + """Puts each data field into a tensor/DataContainer with outer dimension + batch size. + + Extend default_collate to add support for + :type:`~mmcv.parallel.DataContainer`. There are 3 cases. + + 1. cpu_only = True, e.g., meta data + 2. cpu_only = False, stack = True, e.g., images tensors + 3. cpu_only = False, stack = False, e.g., gt bboxes + """ + + if not isinstance(batch, Sequence): + raise TypeError(f'{batch.dtype} is not supported.') + + if isinstance(batch[0], DataContainer): + stacked = [] + if batch[0].cpu_only: + for i in range(0, len(batch), samples_per_gpu): + stacked.append( + [sample.data for sample in batch[i:i + samples_per_gpu]]) + return DataContainer( + stacked, batch[0].stack, batch[0].padding_value, cpu_only=True) + elif batch[0].stack: + for i in range(0, len(batch), samples_per_gpu): + assert isinstance(batch[i].data, torch.Tensor) + + if batch[i].pad_dims is not None: + ndim = batch[i].dim() + assert ndim > batch[i].pad_dims + max_shape = [0 for _ in range(batch[i].pad_dims)] + for dim in range(1, batch[i].pad_dims + 1): + max_shape[dim - 1] = batch[i].size(-dim) + for sample in batch[i:i + samples_per_gpu]: + for dim in range(0, ndim - batch[i].pad_dims): + assert batch[i].size(dim) == sample.size(dim) + for dim in range(1, batch[i].pad_dims + 1): + max_shape[dim - 1] = max(max_shape[dim - 1], + sample.size(-dim)) + padded_samples = [] + for sample in batch[i:i + samples_per_gpu]: + pad = [0 for _ in range(batch[i].pad_dims * 2)] + for dim in range(1, batch[i].pad_dims + 1): + pad[2 * dim - + 1] = max_shape[dim - 1] - sample.size(-dim) + padded_samples.append( + F.pad( + sample.data, pad, value=sample.padding_value)) + stacked.append(default_collate(padded_samples)) + elif batch[i].pad_dims is None: + stacked.append( + default_collate([ + sample.data + for sample in batch[i:i + samples_per_gpu] + ])) + else: + raise ValueError( + 'pad_dims should be either None or integers (1-3)') + + else: + for i in range(0, len(batch), samples_per_gpu): + stacked.append( + [sample.data for sample in batch[i:i + samples_per_gpu]]) + return DataContainer(stacked, batch[0].stack, batch[0].padding_value) + elif isinstance(batch[0], Sequence): + transposed = zip(*batch) + return [collate(samples, samples_per_gpu) for samples in transposed] + elif isinstance(batch[0], Mapping): + return { + key: collate([d[key] for d in batch], samples_per_gpu) + for key in batch[0] + } + else: + return default_collate(batch) diff --git a/annotator/uniformer/mmcv/parallel/data_container.py b/annotator/uniformer/mmcv/parallel/data_container.py new file mode 100644 index 0000000000000000000000000000000000000000..cedb0d32a51a1f575a622b38de2cee3ab4757821 --- /dev/null +++ b/annotator/uniformer/mmcv/parallel/data_container.py @@ -0,0 +1,89 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import functools + +import torch + + +def assert_tensor_type(func): + + @functools.wraps(func) + def wrapper(*args, **kwargs): + if not isinstance(args[0].data, torch.Tensor): + raise AttributeError( + f'{args[0].__class__.__name__} has no attribute ' + f'{func.__name__} for type {args[0].datatype}') + return func(*args, **kwargs) + + return wrapper + + +class DataContainer: + """A container for any type of objects. + + Typically tensors will be stacked in the collate function and sliced along + some dimension in the scatter function. This behavior has some limitations. + 1. All tensors have to be the same size. + 2. Types are limited (numpy array or Tensor). + + We design `DataContainer` and `MMDataParallel` to overcome these + limitations. The behavior can be either of the following. + + - copy to GPU, pad all tensors to the same size and stack them + - copy to GPU without stacking + - leave the objects as is and pass it to the model + - pad_dims specifies the number of last few dimensions to do padding + """ + + def __init__(self, + data, + stack=False, + padding_value=0, + cpu_only=False, + pad_dims=2): + self._data = data + self._cpu_only = cpu_only + self._stack = stack + self._padding_value = padding_value + assert pad_dims in [None, 1, 2, 3] + self._pad_dims = pad_dims + + def __repr__(self): + return f'{self.__class__.__name__}({repr(self.data)})' + + def __len__(self): + return len(self._data) + + @property + def data(self): + return self._data + + @property + def datatype(self): + if isinstance(self.data, torch.Tensor): + return self.data.type() + else: + return type(self.data) + + @property + def cpu_only(self): + return self._cpu_only + + @property + def stack(self): + return self._stack + + @property + def padding_value(self): + return self._padding_value + + @property + def pad_dims(self): + return self._pad_dims + + @assert_tensor_type + def size(self, *args, **kwargs): + return self.data.size(*args, **kwargs) + + @assert_tensor_type + def dim(self): + return self.data.dim() diff --git a/annotator/uniformer/mmcv/parallel/data_parallel.py b/annotator/uniformer/mmcv/parallel/data_parallel.py new file mode 100644 index 0000000000000000000000000000000000000000..79b5f69b654cf647dc7ae9174223781ab5c607d2 --- /dev/null +++ b/annotator/uniformer/mmcv/parallel/data_parallel.py @@ -0,0 +1,89 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from itertools import chain + +from torch.nn.parallel import DataParallel + +from .scatter_gather import scatter_kwargs + + +class MMDataParallel(DataParallel): + """The DataParallel module that supports DataContainer. + + MMDataParallel has two main differences with PyTorch DataParallel: + + - It supports a custom type :class:`DataContainer` which allows more + flexible control of input data during both GPU and CPU inference. + - It implement two more APIs ``train_step()`` and ``val_step()``. + + Args: + module (:class:`nn.Module`): Module to be encapsulated. + device_ids (list[int]): Device IDS of modules to be scattered to. + Defaults to None when GPU is not available. + output_device (str | int): Device ID for output. Defaults to None. + dim (int): Dimension used to scatter the data. Defaults to 0. + """ + + def __init__(self, *args, dim=0, **kwargs): + super(MMDataParallel, self).__init__(*args, dim=dim, **kwargs) + self.dim = dim + + def forward(self, *inputs, **kwargs): + """Override the original forward function. + + The main difference lies in the CPU inference where the data in + :class:`DataContainers` will still be gathered. + """ + if not self.device_ids: + # We add the following line thus the module could gather and + # convert data containers as those in GPU inference + inputs, kwargs = self.scatter(inputs, kwargs, [-1]) + return self.module(*inputs[0], **kwargs[0]) + else: + return super().forward(*inputs, **kwargs) + + def scatter(self, inputs, kwargs, device_ids): + return scatter_kwargs(inputs, kwargs, device_ids, dim=self.dim) + + def train_step(self, *inputs, **kwargs): + if not self.device_ids: + # We add the following line thus the module could gather and + # convert data containers as those in GPU inference + inputs, kwargs = self.scatter(inputs, kwargs, [-1]) + return self.module.train_step(*inputs[0], **kwargs[0]) + + assert len(self.device_ids) == 1, \ + ('MMDataParallel only supports single GPU training, if you need to' + ' train with multiple GPUs, please use MMDistributedDataParallel' + 'instead.') + + for t in chain(self.module.parameters(), self.module.buffers()): + if t.device != self.src_device_obj: + raise RuntimeError( + 'module must have its parameters and buffers ' + f'on device {self.src_device_obj} (device_ids[0]) but ' + f'found one of them on device: {t.device}') + + inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids) + return self.module.train_step(*inputs[0], **kwargs[0]) + + def val_step(self, *inputs, **kwargs): + if not self.device_ids: + # We add the following line thus the module could gather and + # convert data containers as those in GPU inference + inputs, kwargs = self.scatter(inputs, kwargs, [-1]) + return self.module.val_step(*inputs[0], **kwargs[0]) + + assert len(self.device_ids) == 1, \ + ('MMDataParallel only supports single GPU training, if you need to' + ' train with multiple GPUs, please use MMDistributedDataParallel' + ' instead.') + + for t in chain(self.module.parameters(), self.module.buffers()): + if t.device != self.src_device_obj: + raise RuntimeError( + 'module must have its parameters and buffers ' + f'on device {self.src_device_obj} (device_ids[0]) but ' + f'found one of them on device: {t.device}') + + inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids) + return self.module.val_step(*inputs[0], **kwargs[0]) diff --git a/annotator/uniformer/mmcv/parallel/distributed.py b/annotator/uniformer/mmcv/parallel/distributed.py new file mode 100644 index 0000000000000000000000000000000000000000..1e4c27903db58a54d37ea1ed9ec0104098b486f2 --- /dev/null +++ b/annotator/uniformer/mmcv/parallel/distributed.py @@ -0,0 +1,112 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch.nn.parallel.distributed import (DistributedDataParallel, + _find_tensors) + +from annotator.uniformer.mmcv import print_log +from annotator.uniformer.mmcv.utils import TORCH_VERSION, digit_version +from .scatter_gather import scatter_kwargs + + +class MMDistributedDataParallel(DistributedDataParallel): + """The DDP module that supports DataContainer. + + MMDDP has two main differences with PyTorch DDP: + + - It supports a custom type :class:`DataContainer` which allows more + flexible control of input data. + - It implement two APIs ``train_step()`` and ``val_step()``. + """ + + def to_kwargs(self, inputs, kwargs, device_id): + # Use `self.to_kwargs` instead of `self.scatter` in pytorch1.8 + # to move all tensors to device_id + return scatter_kwargs(inputs, kwargs, [device_id], dim=self.dim) + + def scatter(self, inputs, kwargs, device_ids): + return scatter_kwargs(inputs, kwargs, device_ids, dim=self.dim) + + def train_step(self, *inputs, **kwargs): + """train_step() API for module wrapped by DistributedDataParallel. + + This method is basically the same as + ``DistributedDataParallel.forward()``, while replacing + ``self.module.forward()`` with ``self.module.train_step()``. + It is compatible with PyTorch 1.1 - 1.5. + """ + + # In PyTorch >= 1.7, ``reducer._rebuild_buckets()`` is moved from the + # end of backward to the beginning of forward. + if ('parrots' not in TORCH_VERSION + and digit_version(TORCH_VERSION) >= digit_version('1.7') + and self.reducer._rebuild_buckets()): + print_log( + 'Reducer buckets have been rebuilt in this iteration.', + logger='mmcv') + + if getattr(self, 'require_forward_param_sync', True): + self._sync_params() + if self.device_ids: + inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids) + if len(self.device_ids) == 1: + output = self.module.train_step(*inputs[0], **kwargs[0]) + else: + outputs = self.parallel_apply( + self._module_copies[:len(inputs)], inputs, kwargs) + output = self.gather(outputs, self.output_device) + else: + output = self.module.train_step(*inputs, **kwargs) + + if torch.is_grad_enabled() and getattr( + self, 'require_backward_grad_sync', True): + if self.find_unused_parameters: + self.reducer.prepare_for_backward(list(_find_tensors(output))) + else: + self.reducer.prepare_for_backward([]) + else: + if ('parrots' not in TORCH_VERSION + and digit_version(TORCH_VERSION) > digit_version('1.2')): + self.require_forward_param_sync = False + return output + + def val_step(self, *inputs, **kwargs): + """val_step() API for module wrapped by DistributedDataParallel. + + This method is basically the same as + ``DistributedDataParallel.forward()``, while replacing + ``self.module.forward()`` with ``self.module.val_step()``. + It is compatible with PyTorch 1.1 - 1.5. + """ + # In PyTorch >= 1.7, ``reducer._rebuild_buckets()`` is moved from the + # end of backward to the beginning of forward. + if ('parrots' not in TORCH_VERSION + and digit_version(TORCH_VERSION) >= digit_version('1.7') + and self.reducer._rebuild_buckets()): + print_log( + 'Reducer buckets have been rebuilt in this iteration.', + logger='mmcv') + + if getattr(self, 'require_forward_param_sync', True): + self._sync_params() + if self.device_ids: + inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids) + if len(self.device_ids) == 1: + output = self.module.val_step(*inputs[0], **kwargs[0]) + else: + outputs = self.parallel_apply( + self._module_copies[:len(inputs)], inputs, kwargs) + output = self.gather(outputs, self.output_device) + else: + output = self.module.val_step(*inputs, **kwargs) + + if torch.is_grad_enabled() and getattr( + self, 'require_backward_grad_sync', True): + if self.find_unused_parameters: + self.reducer.prepare_for_backward(list(_find_tensors(output))) + else: + self.reducer.prepare_for_backward([]) + else: + if ('parrots' not in TORCH_VERSION + and digit_version(TORCH_VERSION) > digit_version('1.2')): + self.require_forward_param_sync = False + return output diff --git a/annotator/uniformer/mmcv/parallel/distributed_deprecated.py b/annotator/uniformer/mmcv/parallel/distributed_deprecated.py new file mode 100644 index 0000000000000000000000000000000000000000..676937a2085d4da20fa87923041a200fca6214eb --- /dev/null +++ b/annotator/uniformer/mmcv/parallel/distributed_deprecated.py @@ -0,0 +1,70 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +import torch.distributed as dist +import torch.nn as nn +from torch._utils import (_flatten_dense_tensors, _take_tensors, + _unflatten_dense_tensors) + +from annotator.uniformer.mmcv.utils import TORCH_VERSION, digit_version +from .registry import MODULE_WRAPPERS +from .scatter_gather import scatter_kwargs + + +@MODULE_WRAPPERS.register_module() +class MMDistributedDataParallel(nn.Module): + + def __init__(self, + module, + dim=0, + broadcast_buffers=True, + bucket_cap_mb=25): + super(MMDistributedDataParallel, self).__init__() + self.module = module + self.dim = dim + self.broadcast_buffers = broadcast_buffers + + self.broadcast_bucket_size = bucket_cap_mb * 1024 * 1024 + self._sync_params() + + def _dist_broadcast_coalesced(self, tensors, buffer_size): + for tensors in _take_tensors(tensors, buffer_size): + flat_tensors = _flatten_dense_tensors(tensors) + dist.broadcast(flat_tensors, 0) + for tensor, synced in zip( + tensors, _unflatten_dense_tensors(flat_tensors, tensors)): + tensor.copy_(synced) + + def _sync_params(self): + module_states = list(self.module.state_dict().values()) + if len(module_states) > 0: + self._dist_broadcast_coalesced(module_states, + self.broadcast_bucket_size) + if self.broadcast_buffers: + if (TORCH_VERSION != 'parrots' + and digit_version(TORCH_VERSION) < digit_version('1.0')): + buffers = [b.data for b in self.module._all_buffers()] + else: + buffers = [b.data for b in self.module.buffers()] + if len(buffers) > 0: + self._dist_broadcast_coalesced(buffers, + self.broadcast_bucket_size) + + def scatter(self, inputs, kwargs, device_ids): + return scatter_kwargs(inputs, kwargs, device_ids, dim=self.dim) + + def forward(self, *inputs, **kwargs): + inputs, kwargs = self.scatter(inputs, kwargs, + [torch.cuda.current_device()]) + return self.module(*inputs[0], **kwargs[0]) + + def train_step(self, *inputs, **kwargs): + inputs, kwargs = self.scatter(inputs, kwargs, + [torch.cuda.current_device()]) + output = self.module.train_step(*inputs[0], **kwargs[0]) + return output + + def val_step(self, *inputs, **kwargs): + inputs, kwargs = self.scatter(inputs, kwargs, + [torch.cuda.current_device()]) + output = self.module.val_step(*inputs[0], **kwargs[0]) + return output diff --git a/annotator/uniformer/mmcv/parallel/registry.py b/annotator/uniformer/mmcv/parallel/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..a204a07fba10e614223f090d1a57cf9c4d74d4a1 --- /dev/null +++ b/annotator/uniformer/mmcv/parallel/registry.py @@ -0,0 +1,8 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from torch.nn.parallel import DataParallel, DistributedDataParallel + +from annotator.uniformer.mmcv.utils import Registry + +MODULE_WRAPPERS = Registry('module wrapper') +MODULE_WRAPPERS.register_module(module=DataParallel) +MODULE_WRAPPERS.register_module(module=DistributedDataParallel) diff --git a/annotator/uniformer/mmcv/parallel/scatter_gather.py b/annotator/uniformer/mmcv/parallel/scatter_gather.py new file mode 100644 index 0000000000000000000000000000000000000000..900ff88566f8f14830590459dc4fd16d4b382e47 --- /dev/null +++ b/annotator/uniformer/mmcv/parallel/scatter_gather.py @@ -0,0 +1,59 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch +from torch.nn.parallel._functions import Scatter as OrigScatter + +from ._functions import Scatter +from .data_container import DataContainer + + +def scatter(inputs, target_gpus, dim=0): + """Scatter inputs to target gpus. + + The only difference from original :func:`scatter` is to add support for + :type:`~mmcv.parallel.DataContainer`. + """ + + def scatter_map(obj): + if isinstance(obj, torch.Tensor): + if target_gpus != [-1]: + return OrigScatter.apply(target_gpus, None, dim, obj) + else: + # for CPU inference we use self-implemented scatter + return Scatter.forward(target_gpus, obj) + if isinstance(obj, DataContainer): + if obj.cpu_only: + return obj.data + else: + return Scatter.forward(target_gpus, obj.data) + if isinstance(obj, tuple) and len(obj) > 0: + return list(zip(*map(scatter_map, obj))) + if isinstance(obj, list) and len(obj) > 0: + out = list(map(list, zip(*map(scatter_map, obj)))) + return out + if isinstance(obj, dict) and len(obj) > 0: + out = list(map(type(obj), zip(*map(scatter_map, obj.items())))) + return out + return [obj for targets in target_gpus] + + # After scatter_map is called, a scatter_map cell will exist. This cell + # has a reference to the actual function scatter_map, which has references + # to a closure that has a reference to the scatter_map cell (because the + # fn is recursive). To avoid this reference cycle, we set the function to + # None, clearing the cell + try: + return scatter_map(inputs) + finally: + scatter_map = None + + +def scatter_kwargs(inputs, kwargs, target_gpus, dim=0): + """Scatter with support for kwargs dictionary.""" + inputs = scatter(inputs, target_gpus, dim) if inputs else [] + kwargs = scatter(kwargs, target_gpus, dim) if kwargs else [] + if len(inputs) < len(kwargs): + inputs.extend([() for _ in range(len(kwargs) - len(inputs))]) + elif len(kwargs) < len(inputs): + kwargs.extend([{} for _ in range(len(inputs) - len(kwargs))]) + inputs = tuple(inputs) + kwargs = tuple(kwargs) + return inputs, kwargs diff --git a/annotator/uniformer/mmcv/parallel/utils.py b/annotator/uniformer/mmcv/parallel/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0f5712cb42c38a2e8563bf563efb6681383cab9b --- /dev/null +++ b/annotator/uniformer/mmcv/parallel/utils.py @@ -0,0 +1,20 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .registry import MODULE_WRAPPERS + + +def is_module_wrapper(module): + """Check if a module is a module wrapper. + + The following 3 modules in MMCV (and their subclasses) are regarded as + module wrappers: DataParallel, DistributedDataParallel, + MMDistributedDataParallel (the deprecated version). You may add you own + module wrapper by registering it to mmcv.parallel.MODULE_WRAPPERS. + + Args: + module (nn.Module): The module to be checked. + + Returns: + bool: True if the input module is a module wrapper. + """ + module_wrappers = tuple(MODULE_WRAPPERS.module_dict.values()) + return isinstance(module, module_wrappers) diff --git a/annotator/uniformer/mmcv/runner/__init__.py b/annotator/uniformer/mmcv/runner/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..52e4b48d383a84a055dcd7f6236f6e8e58eab924 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/__init__.py @@ -0,0 +1,47 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .base_module import BaseModule, ModuleList, Sequential +from .base_runner import BaseRunner +from .builder import RUNNERS, build_runner +from .checkpoint import (CheckpointLoader, _load_checkpoint, + _load_checkpoint_with_prefix, load_checkpoint, + load_state_dict, save_checkpoint, weights_to_cpu) +from .default_constructor import DefaultRunnerConstructor +from .dist_utils import (allreduce_grads, allreduce_params, get_dist_info, + init_dist, master_only) +from .epoch_based_runner import EpochBasedRunner, Runner +from .fp16_utils import LossScaler, auto_fp16, force_fp32, wrap_fp16_model +from .hooks import (HOOKS, CheckpointHook, ClosureHook, DistEvalHook, + DistSamplerSeedHook, DvcliveLoggerHook, EMAHook, EvalHook, + Fp16OptimizerHook, GradientCumulativeFp16OptimizerHook, + GradientCumulativeOptimizerHook, Hook, IterTimerHook, + LoggerHook, LrUpdaterHook, MlflowLoggerHook, + NeptuneLoggerHook, OptimizerHook, PaviLoggerHook, + SyncBuffersHook, TensorboardLoggerHook, TextLoggerHook, + WandbLoggerHook) +from .iter_based_runner import IterBasedRunner, IterLoader +from .log_buffer import LogBuffer +from .optimizer import (OPTIMIZER_BUILDERS, OPTIMIZERS, + DefaultOptimizerConstructor, build_optimizer, + build_optimizer_constructor) +from .priority import Priority, get_priority +from .utils import get_host_info, get_time_str, obj_from_dict, set_random_seed + +__all__ = [ + 'BaseRunner', 'Runner', 'EpochBasedRunner', 'IterBasedRunner', 'LogBuffer', + 'HOOKS', 'Hook', 'CheckpointHook', 'ClosureHook', 'LrUpdaterHook', + 'OptimizerHook', 'IterTimerHook', 'DistSamplerSeedHook', 'LoggerHook', + 'PaviLoggerHook', 'TextLoggerHook', 'TensorboardLoggerHook', + 'NeptuneLoggerHook', 'WandbLoggerHook', 'MlflowLoggerHook', + 'DvcliveLoggerHook', '_load_checkpoint', 'load_state_dict', + 'load_checkpoint', 'weights_to_cpu', 'save_checkpoint', 'Priority', + 'get_priority', 'get_host_info', 'get_time_str', 'obj_from_dict', + 'init_dist', 'get_dist_info', 'master_only', 'OPTIMIZER_BUILDERS', + 'OPTIMIZERS', 'DefaultOptimizerConstructor', 'build_optimizer', + 'build_optimizer_constructor', 'IterLoader', 'set_random_seed', + 'auto_fp16', 'force_fp32', 'wrap_fp16_model', 'Fp16OptimizerHook', + 'SyncBuffersHook', 'EMAHook', 'build_runner', 'RUNNERS', 'allreduce_grads', + 'allreduce_params', 'LossScaler', 'CheckpointLoader', 'BaseModule', + '_load_checkpoint_with_prefix', 'EvalHook', 'DistEvalHook', 'Sequential', + 'ModuleList', 'GradientCumulativeOptimizerHook', + 'GradientCumulativeFp16OptimizerHook', 'DefaultRunnerConstructor' +] diff --git a/annotator/uniformer/mmcv/runner/base_module.py b/annotator/uniformer/mmcv/runner/base_module.py new file mode 100644 index 0000000000000000000000000000000000000000..617fad9bb89f10a9a0911d962dfb3bc8f3a3628c --- /dev/null +++ b/annotator/uniformer/mmcv/runner/base_module.py @@ -0,0 +1,195 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy +import warnings +from abc import ABCMeta +from collections import defaultdict +from logging import FileHandler + +import torch.nn as nn + +from annotator.uniformer.mmcv.runner.dist_utils import master_only +from annotator.uniformer.mmcv.utils.logging import get_logger, logger_initialized, print_log + + +class BaseModule(nn.Module, metaclass=ABCMeta): + """Base module for all modules in openmmlab. + + ``BaseModule`` is a wrapper of ``torch.nn.Module`` with additional + functionality of parameter initialization. Compared with + ``torch.nn.Module``, ``BaseModule`` mainly adds three attributes. + + - ``init_cfg``: the config to control the initialization. + - ``init_weights``: The function of parameter + initialization and recording initialization + information. + - ``_params_init_info``: Used to track the parameter + initialization information. This attribute only + exists during executing the ``init_weights``. + + Args: + init_cfg (dict, optional): Initialization config dict. + """ + + def __init__(self, init_cfg=None): + """Initialize BaseModule, inherited from `torch.nn.Module`""" + + # NOTE init_cfg can be defined in different levels, but init_cfg + # in low levels has a higher priority. + + super(BaseModule, self).__init__() + # define default value of init_cfg instead of hard code + # in init_weights() function + self._is_init = False + + self.init_cfg = copy.deepcopy(init_cfg) + + # Backward compatibility in derived classes + # if pretrained is not None: + # warnings.warn('DeprecationWarning: pretrained is a deprecated \ + # key, please consider using init_cfg') + # self.init_cfg = dict(type='Pretrained', checkpoint=pretrained) + + @property + def is_init(self): + return self._is_init + + def init_weights(self): + """Initialize the weights.""" + + is_top_level_module = False + # check if it is top-level module + if not hasattr(self, '_params_init_info'): + # The `_params_init_info` is used to record the initialization + # information of the parameters + # the key should be the obj:`nn.Parameter` of model and the value + # should be a dict containing + # - init_info (str): The string that describes the initialization. + # - tmp_mean_value (FloatTensor): The mean of the parameter, + # which indicates whether the parameter has been modified. + # this attribute would be deleted after all parameters + # is initialized. + self._params_init_info = defaultdict(dict) + is_top_level_module = True + + # Initialize the `_params_init_info`, + # When detecting the `tmp_mean_value` of + # the corresponding parameter is changed, update related + # initialization information + for name, param in self.named_parameters(): + self._params_init_info[param][ + 'init_info'] = f'The value is the same before and ' \ + f'after calling `init_weights` ' \ + f'of {self.__class__.__name__} ' + self._params_init_info[param][ + 'tmp_mean_value'] = param.data.mean() + + # pass `params_init_info` to all submodules + # All submodules share the same `params_init_info`, + # so it will be updated when parameters are + # modified at any level of the model. + for sub_module in self.modules(): + sub_module._params_init_info = self._params_init_info + + # Get the initialized logger, if not exist, + # create a logger named `mmcv` + logger_names = list(logger_initialized.keys()) + logger_name = logger_names[0] if logger_names else 'mmcv' + + from ..cnn import initialize + from ..cnn.utils.weight_init import update_init_info + module_name = self.__class__.__name__ + if not self._is_init: + if self.init_cfg: + print_log( + f'initialize {module_name} with init_cfg {self.init_cfg}', + logger=logger_name) + initialize(self, self.init_cfg) + if isinstance(self.init_cfg, dict): + # prevent the parameters of + # the pre-trained model + # from being overwritten by + # the `init_weights` + if self.init_cfg['type'] == 'Pretrained': + return + + for m in self.children(): + if hasattr(m, 'init_weights'): + m.init_weights() + # users may overload the `init_weights` + update_init_info( + m, + init_info=f'Initialized by ' + f'user-defined `init_weights`' + f' in {m.__class__.__name__} ') + + self._is_init = True + else: + warnings.warn(f'init_weights of {self.__class__.__name__} has ' + f'been called more than once.') + + if is_top_level_module: + self._dump_init_info(logger_name) + + for sub_module in self.modules(): + del sub_module._params_init_info + + @master_only + def _dump_init_info(self, logger_name): + """Dump the initialization information to a file named + `initialization.log.json` in workdir. + + Args: + logger_name (str): The name of logger. + """ + + logger = get_logger(logger_name) + + with_file_handler = False + # dump the information to the logger file if there is a `FileHandler` + for handler in logger.handlers: + if isinstance(handler, FileHandler): + handler.stream.write( + 'Name of parameter - Initialization information\n') + for name, param in self.named_parameters(): + handler.stream.write( + f'\n{name} - {param.shape}: ' + f"\n{self._params_init_info[param]['init_info']} \n") + handler.stream.flush() + with_file_handler = True + if not with_file_handler: + for name, param in self.named_parameters(): + print_log( + f'\n{name} - {param.shape}: ' + f"\n{self._params_init_info[param]['init_info']} \n ", + logger=logger_name) + + def __repr__(self): + s = super().__repr__() + if self.init_cfg: + s += f'\ninit_cfg={self.init_cfg}' + return s + + +class Sequential(BaseModule, nn.Sequential): + """Sequential module in openmmlab. + + Args: + init_cfg (dict, optional): Initialization config dict. + """ + + def __init__(self, *args, init_cfg=None): + BaseModule.__init__(self, init_cfg) + nn.Sequential.__init__(self, *args) + + +class ModuleList(BaseModule, nn.ModuleList): + """ModuleList in openmmlab. + + Args: + modules (iterable, optional): an iterable of modules to add. + init_cfg (dict, optional): Initialization config dict. + """ + + def __init__(self, modules=None, init_cfg=None): + BaseModule.__init__(self, init_cfg) + nn.ModuleList.__init__(self, modules) diff --git a/annotator/uniformer/mmcv/runner/base_runner.py b/annotator/uniformer/mmcv/runner/base_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..4928db0a73b56fe0218a4bf66ec4ffa082d31ccc --- /dev/null +++ b/annotator/uniformer/mmcv/runner/base_runner.py @@ -0,0 +1,542 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy +import logging +import os.path as osp +import warnings +from abc import ABCMeta, abstractmethod + +import torch +from torch.optim import Optimizer + +import annotator.uniformer.mmcv as mmcv +from ..parallel import is_module_wrapper +from .checkpoint import load_checkpoint +from .dist_utils import get_dist_info +from .hooks import HOOKS, Hook +from .log_buffer import LogBuffer +from .priority import Priority, get_priority +from .utils import get_time_str + + +class BaseRunner(metaclass=ABCMeta): + """The base class of Runner, a training helper for PyTorch. + + All subclasses should implement the following APIs: + + - ``run()`` + - ``train()`` + - ``val()`` + - ``save_checkpoint()`` + + Args: + model (:obj:`torch.nn.Module`): The model to be run. + batch_processor (callable): A callable method that process a data + batch. The interface of this method should be + `batch_processor(model, data, train_mode) -> dict` + optimizer (dict or :obj:`torch.optim.Optimizer`): It can be either an + optimizer (in most cases) or a dict of optimizers (in models that + requires more than one optimizer, e.g., GAN). + work_dir (str, optional): The working directory to save checkpoints + and logs. Defaults to None. + logger (:obj:`logging.Logger`): Logger used during training. + Defaults to None. (The default value is just for backward + compatibility) + meta (dict | None): A dict records some import information such as + environment info and seed, which will be logged in logger hook. + Defaults to None. + max_epochs (int, optional): Total training epochs. + max_iters (int, optional): Total training iterations. + """ + + def __init__(self, + model, + batch_processor=None, + optimizer=None, + work_dir=None, + logger=None, + meta=None, + max_iters=None, + max_epochs=None): + if batch_processor is not None: + if not callable(batch_processor): + raise TypeError('batch_processor must be callable, ' + f'but got {type(batch_processor)}') + warnings.warn('batch_processor is deprecated, please implement ' + 'train_step() and val_step() in the model instead.') + # raise an error is `batch_processor` is not None and + # `model.train_step()` exists. + if is_module_wrapper(model): + _model = model.module + else: + _model = model + if hasattr(_model, 'train_step') or hasattr(_model, 'val_step'): + raise RuntimeError( + 'batch_processor and model.train_step()/model.val_step() ' + 'cannot be both available.') + else: + assert hasattr(model, 'train_step') + + # check the type of `optimizer` + if isinstance(optimizer, dict): + for name, optim in optimizer.items(): + if not isinstance(optim, Optimizer): + raise TypeError( + f'optimizer must be a dict of torch.optim.Optimizers, ' + f'but optimizer["{name}"] is a {type(optim)}') + elif not isinstance(optimizer, Optimizer) and optimizer is not None: + raise TypeError( + f'optimizer must be a torch.optim.Optimizer object ' + f'or dict or None, but got {type(optimizer)}') + + # check the type of `logger` + if not isinstance(logger, logging.Logger): + raise TypeError(f'logger must be a logging.Logger object, ' + f'but got {type(logger)}') + + # check the type of `meta` + if meta is not None and not isinstance(meta, dict): + raise TypeError( + f'meta must be a dict or None, but got {type(meta)}') + + self.model = model + self.batch_processor = batch_processor + self.optimizer = optimizer + self.logger = logger + self.meta = meta + # create work_dir + if mmcv.is_str(work_dir): + self.work_dir = osp.abspath(work_dir) + mmcv.mkdir_or_exist(self.work_dir) + elif work_dir is None: + self.work_dir = None + else: + raise TypeError('"work_dir" must be a str or None') + + # get model name from the model class + if hasattr(self.model, 'module'): + self._model_name = self.model.module.__class__.__name__ + else: + self._model_name = self.model.__class__.__name__ + + self._rank, self._world_size = get_dist_info() + self.timestamp = get_time_str() + self.mode = None + self._hooks = [] + self._epoch = 0 + self._iter = 0 + self._inner_iter = 0 + + if max_epochs is not None and max_iters is not None: + raise ValueError( + 'Only one of `max_epochs` or `max_iters` can be set.') + + self._max_epochs = max_epochs + self._max_iters = max_iters + # TODO: Redesign LogBuffer, it is not flexible and elegant enough + self.log_buffer = LogBuffer() + + @property + def model_name(self): + """str: Name of the model, usually the module class name.""" + return self._model_name + + @property + def rank(self): + """int: Rank of current process. (distributed training)""" + return self._rank + + @property + def world_size(self): + """int: Number of processes participating in the job. + (distributed training)""" + return self._world_size + + @property + def hooks(self): + """list[:obj:`Hook`]: A list of registered hooks.""" + return self._hooks + + @property + def epoch(self): + """int: Current epoch.""" + return self._epoch + + @property + def iter(self): + """int: Current iteration.""" + return self._iter + + @property + def inner_iter(self): + """int: Iteration in an epoch.""" + return self._inner_iter + + @property + def max_epochs(self): + """int: Maximum training epochs.""" + return self._max_epochs + + @property + def max_iters(self): + """int: Maximum training iterations.""" + return self._max_iters + + @abstractmethod + def train(self): + pass + + @abstractmethod + def val(self): + pass + + @abstractmethod + def run(self, data_loaders, workflow, **kwargs): + pass + + @abstractmethod + def save_checkpoint(self, + out_dir, + filename_tmpl, + save_optimizer=True, + meta=None, + create_symlink=True): + pass + + def current_lr(self): + """Get current learning rates. + + Returns: + list[float] | dict[str, list[float]]: Current learning rates of all + param groups. If the runner has a dict of optimizers, this + method will return a dict. + """ + if isinstance(self.optimizer, torch.optim.Optimizer): + lr = [group['lr'] for group in self.optimizer.param_groups] + elif isinstance(self.optimizer, dict): + lr = dict() + for name, optim in self.optimizer.items(): + lr[name] = [group['lr'] for group in optim.param_groups] + else: + raise RuntimeError( + 'lr is not applicable because optimizer does not exist.') + return lr + + def current_momentum(self): + """Get current momentums. + + Returns: + list[float] | dict[str, list[float]]: Current momentums of all + param groups. If the runner has a dict of optimizers, this + method will return a dict. + """ + + def _get_momentum(optimizer): + momentums = [] + for group in optimizer.param_groups: + if 'momentum' in group.keys(): + momentums.append(group['momentum']) + elif 'betas' in group.keys(): + momentums.append(group['betas'][0]) + else: + momentums.append(0) + return momentums + + if self.optimizer is None: + raise RuntimeError( + 'momentum is not applicable because optimizer does not exist.') + elif isinstance(self.optimizer, torch.optim.Optimizer): + momentums = _get_momentum(self.optimizer) + elif isinstance(self.optimizer, dict): + momentums = dict() + for name, optim in self.optimizer.items(): + momentums[name] = _get_momentum(optim) + return momentums + + def register_hook(self, hook, priority='NORMAL'): + """Register a hook into the hook list. + + The hook will be inserted into a priority queue, with the specified + priority (See :class:`Priority` for details of priorities). + For hooks with the same priority, they will be triggered in the same + order as they are registered. + + Args: + hook (:obj:`Hook`): The hook to be registered. + priority (int or str or :obj:`Priority`): Hook priority. + Lower value means higher priority. + """ + assert isinstance(hook, Hook) + if hasattr(hook, 'priority'): + raise ValueError('"priority" is a reserved attribute for hooks') + priority = get_priority(priority) + hook.priority = priority + # insert the hook to a sorted list + inserted = False + for i in range(len(self._hooks) - 1, -1, -1): + if priority >= self._hooks[i].priority: + self._hooks.insert(i + 1, hook) + inserted = True + break + if not inserted: + self._hooks.insert(0, hook) + + def register_hook_from_cfg(self, hook_cfg): + """Register a hook from its cfg. + + Args: + hook_cfg (dict): Hook config. It should have at least keys 'type' + and 'priority' indicating its type and priority. + + Notes: + The specific hook class to register should not use 'type' and + 'priority' arguments during initialization. + """ + hook_cfg = hook_cfg.copy() + priority = hook_cfg.pop('priority', 'NORMAL') + hook = mmcv.build_from_cfg(hook_cfg, HOOKS) + self.register_hook(hook, priority=priority) + + def call_hook(self, fn_name): + """Call all hooks. + + Args: + fn_name (str): The function name in each hook to be called, such as + "before_train_epoch". + """ + for hook in self._hooks: + getattr(hook, fn_name)(self) + + def get_hook_info(self): + # Get hooks info in each stage + stage_hook_map = {stage: [] for stage in Hook.stages} + for hook in self.hooks: + try: + priority = Priority(hook.priority).name + except ValueError: + priority = hook.priority + classname = hook.__class__.__name__ + hook_info = f'({priority:<12}) {classname:<35}' + for trigger_stage in hook.get_triggered_stages(): + stage_hook_map[trigger_stage].append(hook_info) + + stage_hook_infos = [] + for stage in Hook.stages: + hook_infos = stage_hook_map[stage] + if len(hook_infos) > 0: + info = f'{stage}:\n' + info += '\n'.join(hook_infos) + info += '\n -------------------- ' + stage_hook_infos.append(info) + return '\n'.join(stage_hook_infos) + + def load_checkpoint(self, + filename, + map_location='cpu', + strict=False, + revise_keys=[(r'^module.', '')]): + return load_checkpoint( + self.model, + filename, + map_location, + strict, + self.logger, + revise_keys=revise_keys) + + def resume(self, + checkpoint, + resume_optimizer=True, + map_location='default'): + if map_location == 'default': + if torch.cuda.is_available(): + device_id = torch.cuda.current_device() + checkpoint = self.load_checkpoint( + checkpoint, + map_location=lambda storage, loc: storage.cuda(device_id)) + else: + checkpoint = self.load_checkpoint(checkpoint) + else: + checkpoint = self.load_checkpoint( + checkpoint, map_location=map_location) + + self._epoch = checkpoint['meta']['epoch'] + self._iter = checkpoint['meta']['iter'] + if self.meta is None: + self.meta = {} + self.meta.setdefault('hook_msgs', {}) + # load `last_ckpt`, `best_score`, `best_ckpt`, etc. for hook messages + self.meta['hook_msgs'].update(checkpoint['meta'].get('hook_msgs', {})) + + # Re-calculate the number of iterations when resuming + # models with different number of GPUs + if 'config' in checkpoint['meta']: + config = mmcv.Config.fromstring( + checkpoint['meta']['config'], file_format='.py') + previous_gpu_ids = config.get('gpu_ids', None) + if previous_gpu_ids and len(previous_gpu_ids) > 0 and len( + previous_gpu_ids) != self.world_size: + self._iter = int(self._iter * len(previous_gpu_ids) / + self.world_size) + self.logger.info('the iteration number is changed due to ' + 'change of GPU number') + + # resume meta information meta + self.meta = checkpoint['meta'] + + if 'optimizer' in checkpoint and resume_optimizer: + if isinstance(self.optimizer, Optimizer): + self.optimizer.load_state_dict(checkpoint['optimizer']) + elif isinstance(self.optimizer, dict): + for k in self.optimizer.keys(): + self.optimizer[k].load_state_dict( + checkpoint['optimizer'][k]) + else: + raise TypeError( + 'Optimizer should be dict or torch.optim.Optimizer ' + f'but got {type(self.optimizer)}') + + self.logger.info('resumed epoch %d, iter %d', self.epoch, self.iter) + + def register_lr_hook(self, lr_config): + if lr_config is None: + return + elif isinstance(lr_config, dict): + assert 'policy' in lr_config + policy_type = lr_config.pop('policy') + # If the type of policy is all in lower case, e.g., 'cyclic', + # then its first letter will be capitalized, e.g., to be 'Cyclic'. + # This is for the convenient usage of Lr updater. + # Since this is not applicable for ` + # CosineAnnealingLrUpdater`, + # the string will not be changed if it contains capital letters. + if policy_type == policy_type.lower(): + policy_type = policy_type.title() + hook_type = policy_type + 'LrUpdaterHook' + lr_config['type'] = hook_type + hook = mmcv.build_from_cfg(lr_config, HOOKS) + else: + hook = lr_config + self.register_hook(hook, priority='VERY_HIGH') + + def register_momentum_hook(self, momentum_config): + if momentum_config is None: + return + if isinstance(momentum_config, dict): + assert 'policy' in momentum_config + policy_type = momentum_config.pop('policy') + # If the type of policy is all in lower case, e.g., 'cyclic', + # then its first letter will be capitalized, e.g., to be 'Cyclic'. + # This is for the convenient usage of momentum updater. + # Since this is not applicable for + # `CosineAnnealingMomentumUpdater`, + # the string will not be changed if it contains capital letters. + if policy_type == policy_type.lower(): + policy_type = policy_type.title() + hook_type = policy_type + 'MomentumUpdaterHook' + momentum_config['type'] = hook_type + hook = mmcv.build_from_cfg(momentum_config, HOOKS) + else: + hook = momentum_config + self.register_hook(hook, priority='HIGH') + + def register_optimizer_hook(self, optimizer_config): + if optimizer_config is None: + return + if isinstance(optimizer_config, dict): + optimizer_config.setdefault('type', 'OptimizerHook') + hook = mmcv.build_from_cfg(optimizer_config, HOOKS) + else: + hook = optimizer_config + self.register_hook(hook, priority='ABOVE_NORMAL') + + def register_checkpoint_hook(self, checkpoint_config): + if checkpoint_config is None: + return + if isinstance(checkpoint_config, dict): + checkpoint_config.setdefault('type', 'CheckpointHook') + hook = mmcv.build_from_cfg(checkpoint_config, HOOKS) + else: + hook = checkpoint_config + self.register_hook(hook, priority='NORMAL') + + def register_logger_hooks(self, log_config): + if log_config is None: + return + log_interval = log_config['interval'] + for info in log_config['hooks']: + logger_hook = mmcv.build_from_cfg( + info, HOOKS, default_args=dict(interval=log_interval)) + self.register_hook(logger_hook, priority='VERY_LOW') + + def register_timer_hook(self, timer_config): + if timer_config is None: + return + if isinstance(timer_config, dict): + timer_config_ = copy.deepcopy(timer_config) + hook = mmcv.build_from_cfg(timer_config_, HOOKS) + else: + hook = timer_config + self.register_hook(hook, priority='LOW') + + def register_custom_hooks(self, custom_config): + if custom_config is None: + return + + if not isinstance(custom_config, list): + custom_config = [custom_config] + + for item in custom_config: + if isinstance(item, dict): + self.register_hook_from_cfg(item) + else: + self.register_hook(item, priority='NORMAL') + + def register_profiler_hook(self, profiler_config): + if profiler_config is None: + return + if isinstance(profiler_config, dict): + profiler_config.setdefault('type', 'ProfilerHook') + hook = mmcv.build_from_cfg(profiler_config, HOOKS) + else: + hook = profiler_config + self.register_hook(hook) + + def register_training_hooks(self, + lr_config, + optimizer_config=None, + checkpoint_config=None, + log_config=None, + momentum_config=None, + timer_config=dict(type='IterTimerHook'), + custom_hooks_config=None): + """Register default and custom hooks for training. + + Default and custom hooks include: + + +----------------------+-------------------------+ + | Hooks | Priority | + +======================+=========================+ + | LrUpdaterHook | VERY_HIGH (10) | + +----------------------+-------------------------+ + | MomentumUpdaterHook | HIGH (30) | + +----------------------+-------------------------+ + | OptimizerStepperHook | ABOVE_NORMAL (40) | + +----------------------+-------------------------+ + | CheckpointSaverHook | NORMAL (50) | + +----------------------+-------------------------+ + | IterTimerHook | LOW (70) | + +----------------------+-------------------------+ + | LoggerHook(s) | VERY_LOW (90) | + +----------------------+-------------------------+ + | CustomHook(s) | defaults to NORMAL (50) | + +----------------------+-------------------------+ + + If custom hooks have same priority with default hooks, custom hooks + will be triggered after default hooks. + """ + self.register_lr_hook(lr_config) + self.register_momentum_hook(momentum_config) + self.register_optimizer_hook(optimizer_config) + self.register_checkpoint_hook(checkpoint_config) + self.register_timer_hook(timer_config) + self.register_logger_hooks(log_config) + self.register_custom_hooks(custom_hooks_config) diff --git a/annotator/uniformer/mmcv/runner/builder.py b/annotator/uniformer/mmcv/runner/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..77c96ba0b2f30ead9da23f293c5dc84dd3e4a74f --- /dev/null +++ b/annotator/uniformer/mmcv/runner/builder.py @@ -0,0 +1,24 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy + +from ..utils import Registry + +RUNNERS = Registry('runner') +RUNNER_BUILDERS = Registry('runner builder') + + +def build_runner_constructor(cfg): + return RUNNER_BUILDERS.build(cfg) + + +def build_runner(cfg, default_args=None): + runner_cfg = copy.deepcopy(cfg) + constructor_type = runner_cfg.pop('constructor', + 'DefaultRunnerConstructor') + runner_constructor = build_runner_constructor( + dict( + type=constructor_type, + runner_cfg=runner_cfg, + default_args=default_args)) + runner = runner_constructor() + return runner diff --git a/annotator/uniformer/mmcv/runner/checkpoint.py b/annotator/uniformer/mmcv/runner/checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..b29ca320679164432f446adad893e33fb2b4b29e --- /dev/null +++ b/annotator/uniformer/mmcv/runner/checkpoint.py @@ -0,0 +1,707 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import io +import os +import os.path as osp +import pkgutil +import re +import time +import warnings +from collections import OrderedDict +from importlib import import_module +from tempfile import TemporaryDirectory + +import torch +import torchvision +from torch.optim import Optimizer +from torch.utils import model_zoo + +import annotator.uniformer.mmcv as mmcv +from ..fileio import FileClient +from ..fileio import load as load_file +from ..parallel import is_module_wrapper +from ..utils import mkdir_or_exist +from .dist_utils import get_dist_info + +ENV_MMCV_HOME = 'MMCV_HOME' +ENV_XDG_CACHE_HOME = 'XDG_CACHE_HOME' +DEFAULT_CACHE_DIR = '~/.cache' + + +def _get_mmcv_home(): + mmcv_home = os.path.expanduser( + os.getenv( + ENV_MMCV_HOME, + os.path.join( + os.getenv(ENV_XDG_CACHE_HOME, DEFAULT_CACHE_DIR), 'mmcv'))) + + mkdir_or_exist(mmcv_home) + return mmcv_home + + +def load_state_dict(module, state_dict, strict=False, logger=None): + """Load state_dict to a module. + + This method is modified from :meth:`torch.nn.Module.load_state_dict`. + Default value for ``strict`` is set to ``False`` and the message for + param mismatch will be shown even if strict is False. + + Args: + module (Module): Module that receives the state_dict. + state_dict (OrderedDict): Weights. + strict (bool): whether to strictly enforce that the keys + in :attr:`state_dict` match the keys returned by this module's + :meth:`~torch.nn.Module.state_dict` function. Default: ``False``. + logger (:obj:`logging.Logger`, optional): Logger to log the error + message. If not specified, print function will be used. + """ + unexpected_keys = [] + all_missing_keys = [] + err_msg = [] + + metadata = getattr(state_dict, '_metadata', None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + # use _load_from_state_dict to enable checkpoint version control + def load(module, prefix=''): + # recursively check parallel module in case that the model has a + # complicated structure, e.g., nn.Module(nn.Module(DDP)) + if is_module_wrapper(module): + module = module.module + local_metadata = {} if metadata is None else metadata.get( + prefix[:-1], {}) + module._load_from_state_dict(state_dict, prefix, local_metadata, True, + all_missing_keys, unexpected_keys, + err_msg) + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + '.') + + load(module) + load = None # break load->load reference cycle + + # ignore "num_batches_tracked" of BN layers + missing_keys = [ + key for key in all_missing_keys if 'num_batches_tracked' not in key + ] + + if unexpected_keys: + err_msg.append('unexpected key in source ' + f'state_dict: {", ".join(unexpected_keys)}\n') + if missing_keys: + err_msg.append( + f'missing keys in source state_dict: {", ".join(missing_keys)}\n') + + rank, _ = get_dist_info() + if len(err_msg) > 0 and rank == 0: + err_msg.insert( + 0, 'The model and loaded state dict do not match exactly\n') + err_msg = '\n'.join(err_msg) + if strict: + raise RuntimeError(err_msg) + elif logger is not None: + logger.warning(err_msg) + else: + print(err_msg) + + +def get_torchvision_models(): + model_urls = dict() + for _, name, ispkg in pkgutil.walk_packages(torchvision.models.__path__): + if ispkg: + continue + _zoo = import_module(f'torchvision.models.{name}') + if hasattr(_zoo, 'model_urls'): + _urls = getattr(_zoo, 'model_urls') + model_urls.update(_urls) + return model_urls + + +def get_external_models(): + mmcv_home = _get_mmcv_home() + default_json_path = osp.join(mmcv.__path__[0], 'model_zoo/open_mmlab.json') + default_urls = load_file(default_json_path) + assert isinstance(default_urls, dict) + external_json_path = osp.join(mmcv_home, 'open_mmlab.json') + if osp.exists(external_json_path): + external_urls = load_file(external_json_path) + assert isinstance(external_urls, dict) + default_urls.update(external_urls) + + return default_urls + + +def get_mmcls_models(): + mmcls_json_path = osp.join(mmcv.__path__[0], 'model_zoo/mmcls.json') + mmcls_urls = load_file(mmcls_json_path) + + return mmcls_urls + + +def get_deprecated_model_names(): + deprecate_json_path = osp.join(mmcv.__path__[0], + 'model_zoo/deprecated.json') + deprecate_urls = load_file(deprecate_json_path) + assert isinstance(deprecate_urls, dict) + + return deprecate_urls + + +def _process_mmcls_checkpoint(checkpoint): + state_dict = checkpoint['state_dict'] + new_state_dict = OrderedDict() + for k, v in state_dict.items(): + if k.startswith('backbone.'): + new_state_dict[k[9:]] = v + new_checkpoint = dict(state_dict=new_state_dict) + + return new_checkpoint + + +class CheckpointLoader: + """A general checkpoint loader to manage all schemes.""" + + _schemes = {} + + @classmethod + def _register_scheme(cls, prefixes, loader, force=False): + if isinstance(prefixes, str): + prefixes = [prefixes] + else: + assert isinstance(prefixes, (list, tuple)) + for prefix in prefixes: + if (prefix not in cls._schemes) or force: + cls._schemes[prefix] = loader + else: + raise KeyError( + f'{prefix} is already registered as a loader backend, ' + 'add "force=True" if you want to override it') + # sort, longer prefixes take priority + cls._schemes = OrderedDict( + sorted(cls._schemes.items(), key=lambda t: t[0], reverse=True)) + + @classmethod + def register_scheme(cls, prefixes, loader=None, force=False): + """Register a loader to CheckpointLoader. + + This method can be used as a normal class method or a decorator. + + Args: + prefixes (str or list[str] or tuple[str]): + The prefix of the registered loader. + loader (function, optional): The loader function to be registered. + When this method is used as a decorator, loader is None. + Defaults to None. + force (bool, optional): Whether to override the loader + if the prefix has already been registered. Defaults to False. + """ + + if loader is not None: + cls._register_scheme(prefixes, loader, force=force) + return + + def _register(loader_cls): + cls._register_scheme(prefixes, loader_cls, force=force) + return loader_cls + + return _register + + @classmethod + def _get_checkpoint_loader(cls, path): + """Finds a loader that supports the given path. Falls back to the local + loader if no other loader is found. + + Args: + path (str): checkpoint path + + Returns: + loader (function): checkpoint loader + """ + + for p in cls._schemes: + if path.startswith(p): + return cls._schemes[p] + + @classmethod + def load_checkpoint(cls, filename, map_location=None, logger=None): + """load checkpoint through URL scheme path. + + Args: + filename (str): checkpoint file name with given prefix + map_location (str, optional): Same as :func:`torch.load`. + Default: None + logger (:mod:`logging.Logger`, optional): The logger for message. + Default: None + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + + checkpoint_loader = cls._get_checkpoint_loader(filename) + class_name = checkpoint_loader.__name__ + mmcv.print_log( + f'load checkpoint from {class_name[10:]} path: {filename}', logger) + return checkpoint_loader(filename, map_location) + + +@CheckpointLoader.register_scheme(prefixes='') +def load_from_local(filename, map_location): + """load checkpoint by local file path. + + Args: + filename (str): local checkpoint file path + map_location (str, optional): Same as :func:`torch.load`. + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + + if not osp.isfile(filename): + raise IOError(f'{filename} is not a checkpoint file') + checkpoint = torch.load(filename, map_location=map_location) + return checkpoint + + +@CheckpointLoader.register_scheme(prefixes=('http://', 'https://')) +def load_from_http(filename, map_location=None, model_dir=None): + """load checkpoint through HTTP or HTTPS scheme path. In distributed + setting, this function only download checkpoint at local rank 0. + + Args: + filename (str): checkpoint file path with modelzoo or + torchvision prefix + map_location (str, optional): Same as :func:`torch.load`. + model_dir (string, optional): directory in which to save the object, + Default: None + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + rank, world_size = get_dist_info() + rank = int(os.environ.get('LOCAL_RANK', rank)) + if rank == 0: + checkpoint = model_zoo.load_url( + filename, model_dir=model_dir, map_location=map_location) + if world_size > 1: + torch.distributed.barrier() + if rank > 0: + checkpoint = model_zoo.load_url( + filename, model_dir=model_dir, map_location=map_location) + return checkpoint + + +@CheckpointLoader.register_scheme(prefixes='pavi://') +def load_from_pavi(filename, map_location=None): + """load checkpoint through the file path prefixed with pavi. In distributed + setting, this function download ckpt at all ranks to different temporary + directories. + + Args: + filename (str): checkpoint file path with pavi prefix + map_location (str, optional): Same as :func:`torch.load`. + Default: None + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + assert filename.startswith('pavi://'), \ + f'Expected filename startswith `pavi://`, but get {filename}' + model_path = filename[7:] + + try: + from pavi import modelcloud + except ImportError: + raise ImportError( + 'Please install pavi to load checkpoint from modelcloud.') + + model = modelcloud.get(model_path) + with TemporaryDirectory() as tmp_dir: + downloaded_file = osp.join(tmp_dir, model.name) + model.download(downloaded_file) + checkpoint = torch.load(downloaded_file, map_location=map_location) + return checkpoint + + +@CheckpointLoader.register_scheme(prefixes='s3://') +def load_from_ceph(filename, map_location=None, backend='petrel'): + """load checkpoint through the file path prefixed with s3. In distributed + setting, this function download ckpt at all ranks to different temporary + directories. + + Args: + filename (str): checkpoint file path with s3 prefix + map_location (str, optional): Same as :func:`torch.load`. + backend (str, optional): The storage backend type. Options are 'ceph', + 'petrel'. Default: 'petrel'. + + .. warning:: + :class:`mmcv.fileio.file_client.CephBackend` will be deprecated, + please use :class:`mmcv.fileio.file_client.PetrelBackend` instead. + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + allowed_backends = ['ceph', 'petrel'] + if backend not in allowed_backends: + raise ValueError(f'Load from Backend {backend} is not supported.') + + if backend == 'ceph': + warnings.warn( + 'CephBackend will be deprecated, please use PetrelBackend instead') + + # CephClient and PetrelBackend have the same prefix 's3://' and the latter + # will be chosen as default. If PetrelBackend can not be instantiated + # successfully, the CephClient will be chosen. + try: + file_client = FileClient(backend=backend) + except ImportError: + allowed_backends.remove(backend) + file_client = FileClient(backend=allowed_backends[0]) + + with io.BytesIO(file_client.get(filename)) as buffer: + checkpoint = torch.load(buffer, map_location=map_location) + return checkpoint + + +@CheckpointLoader.register_scheme(prefixes=('modelzoo://', 'torchvision://')) +def load_from_torchvision(filename, map_location=None): + """load checkpoint through the file path prefixed with modelzoo or + torchvision. + + Args: + filename (str): checkpoint file path with modelzoo or + torchvision prefix + map_location (str, optional): Same as :func:`torch.load`. + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + model_urls = get_torchvision_models() + if filename.startswith('modelzoo://'): + warnings.warn('The URL scheme of "modelzoo://" is deprecated, please ' + 'use "torchvision://" instead') + model_name = filename[11:] + else: + model_name = filename[14:] + return load_from_http(model_urls[model_name], map_location=map_location) + + +@CheckpointLoader.register_scheme(prefixes=('open-mmlab://', 'openmmlab://')) +def load_from_openmmlab(filename, map_location=None): + """load checkpoint through the file path prefixed with open-mmlab or + openmmlab. + + Args: + filename (str): checkpoint file path with open-mmlab or + openmmlab prefix + map_location (str, optional): Same as :func:`torch.load`. + Default: None + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + + model_urls = get_external_models() + prefix_str = 'open-mmlab://' + if filename.startswith(prefix_str): + model_name = filename[13:] + else: + model_name = filename[12:] + prefix_str = 'openmmlab://' + + deprecated_urls = get_deprecated_model_names() + if model_name in deprecated_urls: + warnings.warn(f'{prefix_str}{model_name} is deprecated in favor ' + f'of {prefix_str}{deprecated_urls[model_name]}') + model_name = deprecated_urls[model_name] + model_url = model_urls[model_name] + # check if is url + if model_url.startswith(('http://', 'https://')): + checkpoint = load_from_http(model_url, map_location=map_location) + else: + filename = osp.join(_get_mmcv_home(), model_url) + if not osp.isfile(filename): + raise IOError(f'{filename} is not a checkpoint file') + checkpoint = torch.load(filename, map_location=map_location) + return checkpoint + + +@CheckpointLoader.register_scheme(prefixes='mmcls://') +def load_from_mmcls(filename, map_location=None): + """load checkpoint through the file path prefixed with mmcls. + + Args: + filename (str): checkpoint file path with mmcls prefix + map_location (str, optional): Same as :func:`torch.load`. + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + + model_urls = get_mmcls_models() + model_name = filename[8:] + checkpoint = load_from_http( + model_urls[model_name], map_location=map_location) + checkpoint = _process_mmcls_checkpoint(checkpoint) + return checkpoint + + +def _load_checkpoint(filename, map_location=None, logger=None): + """Load checkpoint from somewhere (modelzoo, file, url). + + Args: + filename (str): Accept local filepath, URL, ``torchvision://xxx``, + ``open-mmlab://xxx``. Please refer to ``docs/model_zoo.md`` for + details. + map_location (str, optional): Same as :func:`torch.load`. + Default: None. + logger (:mod:`logging.Logger`, optional): The logger for error message. + Default: None + + Returns: + dict or OrderedDict: The loaded checkpoint. It can be either an + OrderedDict storing model weights or a dict containing other + information, which depends on the checkpoint. + """ + return CheckpointLoader.load_checkpoint(filename, map_location, logger) + + +def _load_checkpoint_with_prefix(prefix, filename, map_location=None): + """Load partial pretrained model with specific prefix. + + Args: + prefix (str): The prefix of sub-module. + filename (str): Accept local filepath, URL, ``torchvision://xxx``, + ``open-mmlab://xxx``. Please refer to ``docs/model_zoo.md`` for + details. + map_location (str | None): Same as :func:`torch.load`. Default: None. + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + + checkpoint = _load_checkpoint(filename, map_location=map_location) + + if 'state_dict' in checkpoint: + state_dict = checkpoint['state_dict'] + else: + state_dict = checkpoint + if not prefix.endswith('.'): + prefix += '.' + prefix_len = len(prefix) + + state_dict = { + k[prefix_len:]: v + for k, v in state_dict.items() if k.startswith(prefix) + } + + assert state_dict, f'{prefix} is not in the pretrained model' + return state_dict + + +def load_checkpoint(model, + filename, + map_location=None, + strict=False, + logger=None, + revise_keys=[(r'^module\.', '')]): + """Load checkpoint from a file or URI. + + Args: + model (Module): Module to load checkpoint. + filename (str): Accept local filepath, URL, ``torchvision://xxx``, + ``open-mmlab://xxx``. Please refer to ``docs/model_zoo.md`` for + details. + map_location (str): Same as :func:`torch.load`. + strict (bool): Whether to allow different params for the model and + checkpoint. + logger (:mod:`logging.Logger` or None): The logger for error message. + revise_keys (list): A list of customized keywords to modify the + state_dict in checkpoint. Each item is a (pattern, replacement) + pair of the regular expression operations. Default: strip + the prefix 'module.' by [(r'^module\\.', '')]. + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + checkpoint = _load_checkpoint(filename, map_location, logger) + # OrderedDict is a subclass of dict + if not isinstance(checkpoint, dict): + raise RuntimeError( + f'No state_dict found in checkpoint file {filename}') + # get state_dict from checkpoint + if 'state_dict' in checkpoint: + state_dict = checkpoint['state_dict'] + else: + state_dict = checkpoint + + # strip prefix of state_dict + metadata = getattr(state_dict, '_metadata', OrderedDict()) + for p, r in revise_keys: + state_dict = OrderedDict( + {re.sub(p, r, k): v + for k, v in state_dict.items()}) + # Keep metadata in state_dict + state_dict._metadata = metadata + + # load state_dict + load_state_dict(model, state_dict, strict, logger) + return checkpoint + + +def weights_to_cpu(state_dict): + """Copy a model state_dict to cpu. + + Args: + state_dict (OrderedDict): Model weights on GPU. + + Returns: + OrderedDict: Model weights on GPU. + """ + state_dict_cpu = OrderedDict() + for key, val in state_dict.items(): + state_dict_cpu[key] = val.cpu() + # Keep metadata in state_dict + state_dict_cpu._metadata = getattr(state_dict, '_metadata', OrderedDict()) + return state_dict_cpu + + +def _save_to_state_dict(module, destination, prefix, keep_vars): + """Saves module state to `destination` dictionary. + + This method is modified from :meth:`torch.nn.Module._save_to_state_dict`. + + Args: + module (nn.Module): The module to generate state_dict. + destination (dict): A dict where state will be stored. + prefix (str): The prefix for parameters and buffers used in this + module. + """ + for name, param in module._parameters.items(): + if param is not None: + destination[prefix + name] = param if keep_vars else param.detach() + for name, buf in module._buffers.items(): + # remove check of _non_persistent_buffers_set to allow nn.BatchNorm2d + if buf is not None: + destination[prefix + name] = buf if keep_vars else buf.detach() + + +def get_state_dict(module, destination=None, prefix='', keep_vars=False): + """Returns a dictionary containing a whole state of the module. + + Both parameters and persistent buffers (e.g. running averages) are + included. Keys are corresponding parameter and buffer names. + + This method is modified from :meth:`torch.nn.Module.state_dict` to + recursively check parallel module in case that the model has a complicated + structure, e.g., nn.Module(nn.Module(DDP)). + + Args: + module (nn.Module): The module to generate state_dict. + destination (OrderedDict): Returned dict for the state of the + module. + prefix (str): Prefix of the key. + keep_vars (bool): Whether to keep the variable property of the + parameters. Default: False. + + Returns: + dict: A dictionary containing a whole state of the module. + """ + # recursively check parallel module in case that the model has a + # complicated structure, e.g., nn.Module(nn.Module(DDP)) + if is_module_wrapper(module): + module = module.module + + # below is the same as torch.nn.Module.state_dict() + if destination is None: + destination = OrderedDict() + destination._metadata = OrderedDict() + destination._metadata[prefix[:-1]] = local_metadata = dict( + version=module._version) + _save_to_state_dict(module, destination, prefix, keep_vars) + for name, child in module._modules.items(): + if child is not None: + get_state_dict( + child, destination, prefix + name + '.', keep_vars=keep_vars) + for hook in module._state_dict_hooks.values(): + hook_result = hook(module, destination, prefix, local_metadata) + if hook_result is not None: + destination = hook_result + return destination + + +def save_checkpoint(model, + filename, + optimizer=None, + meta=None, + file_client_args=None): + """Save checkpoint to file. + + The checkpoint will have 3 fields: ``meta``, ``state_dict`` and + ``optimizer``. By default ``meta`` will contain version and time info. + + Args: + model (Module): Module whose params are to be saved. + filename (str): Checkpoint filename. + optimizer (:obj:`Optimizer`, optional): Optimizer to be saved. + meta (dict, optional): Metadata to be saved in checkpoint. + file_client_args (dict, optional): Arguments to instantiate a + FileClient. See :class:`mmcv.fileio.FileClient` for details. + Default: None. + `New in version 1.3.16.` + """ + if meta is None: + meta = {} + elif not isinstance(meta, dict): + raise TypeError(f'meta must be a dict or None, but got {type(meta)}') + meta.update(mmcv_version=mmcv.__version__, time=time.asctime()) + + if is_module_wrapper(model): + model = model.module + + if hasattr(model, 'CLASSES') and model.CLASSES is not None: + # save class name to the meta + meta.update(CLASSES=model.CLASSES) + + checkpoint = { + 'meta': meta, + 'state_dict': weights_to_cpu(get_state_dict(model)) + } + # save optimizer state dict in the checkpoint + if isinstance(optimizer, Optimizer): + checkpoint['optimizer'] = optimizer.state_dict() + elif isinstance(optimizer, dict): + checkpoint['optimizer'] = {} + for name, optim in optimizer.items(): + checkpoint['optimizer'][name] = optim.state_dict() + + if filename.startswith('pavi://'): + if file_client_args is not None: + raise ValueError( + 'file_client_args should be "None" if filename starts with' + f'"pavi://", but got {file_client_args}') + try: + from pavi import modelcloud + from pavi import exception + except ImportError: + raise ImportError( + 'Please install pavi to load checkpoint from modelcloud.') + model_path = filename[7:] + root = modelcloud.Folder() + model_dir, model_name = osp.split(model_path) + try: + model = modelcloud.get(model_dir) + except exception.NodeNotFoundError: + model = root.create_training_model(model_dir) + with TemporaryDirectory() as tmp_dir: + checkpoint_file = osp.join(tmp_dir, model_name) + with open(checkpoint_file, 'wb') as f: + torch.save(checkpoint, f) + f.flush() + model.create_file(checkpoint_file, name=model_name) + else: + file_client = FileClient.infer_client(file_client_args, filename) + with io.BytesIO() as f: + torch.save(checkpoint, f) + file_client.put(f.getvalue(), filename) diff --git a/annotator/uniformer/mmcv/runner/default_constructor.py b/annotator/uniformer/mmcv/runner/default_constructor.py new file mode 100644 index 0000000000000000000000000000000000000000..3f1f5b44168768dfda3947393a63a6cf9cf50b41 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/default_constructor.py @@ -0,0 +1,44 @@ +from .builder import RUNNER_BUILDERS, RUNNERS + + +@RUNNER_BUILDERS.register_module() +class DefaultRunnerConstructor: + """Default constructor for runners. + + Custom existing `Runner` like `EpocBasedRunner` though `RunnerConstructor`. + For example, We can inject some new properties and functions for `Runner`. + + Example: + >>> from annotator.uniformer.mmcv.runner import RUNNER_BUILDERS, build_runner + >>> # Define a new RunnerReconstructor + >>> @RUNNER_BUILDERS.register_module() + >>> class MyRunnerConstructor: + ... def __init__(self, runner_cfg, default_args=None): + ... if not isinstance(runner_cfg, dict): + ... raise TypeError('runner_cfg should be a dict', + ... f'but got {type(runner_cfg)}') + ... self.runner_cfg = runner_cfg + ... self.default_args = default_args + ... + ... def __call__(self): + ... runner = RUNNERS.build(self.runner_cfg, + ... default_args=self.default_args) + ... # Add new properties for existing runner + ... runner.my_name = 'my_runner' + ... runner.my_function = lambda self: print(self.my_name) + ... ... + >>> # build your runner + >>> runner_cfg = dict(type='EpochBasedRunner', max_epochs=40, + ... constructor='MyRunnerConstructor') + >>> runner = build_runner(runner_cfg) + """ + + def __init__(self, runner_cfg, default_args=None): + if not isinstance(runner_cfg, dict): + raise TypeError('runner_cfg should be a dict', + f'but got {type(runner_cfg)}') + self.runner_cfg = runner_cfg + self.default_args = default_args + + def __call__(self): + return RUNNERS.build(self.runner_cfg, default_args=self.default_args) diff --git a/annotator/uniformer/mmcv/runner/dist_utils.py b/annotator/uniformer/mmcv/runner/dist_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d3a1ef3fda5ceeb31bf15a73779da1b1903ab0fe --- /dev/null +++ b/annotator/uniformer/mmcv/runner/dist_utils.py @@ -0,0 +1,164 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import functools +import os +import subprocess +from collections import OrderedDict + +import torch +import torch.multiprocessing as mp +from torch import distributed as dist +from torch._utils import (_flatten_dense_tensors, _take_tensors, + _unflatten_dense_tensors) + + +def init_dist(launcher, backend='nccl', **kwargs): + if mp.get_start_method(allow_none=True) is None: + mp.set_start_method('spawn') + if launcher == 'pytorch': + _init_dist_pytorch(backend, **kwargs) + elif launcher == 'mpi': + _init_dist_mpi(backend, **kwargs) + elif launcher == 'slurm': + _init_dist_slurm(backend, **kwargs) + else: + raise ValueError(f'Invalid launcher type: {launcher}') + + +def _init_dist_pytorch(backend, **kwargs): + # TODO: use local_rank instead of rank % num_gpus + rank = int(os.environ['RANK']) + num_gpus = torch.cuda.device_count() + torch.cuda.set_device(rank % num_gpus) + dist.init_process_group(backend=backend, **kwargs) + + +def _init_dist_mpi(backend, **kwargs): + # TODO: use local_rank instead of rank % num_gpus + rank = int(os.environ['OMPI_COMM_WORLD_RANK']) + num_gpus = torch.cuda.device_count() + torch.cuda.set_device(rank % num_gpus) + dist.init_process_group(backend=backend, **kwargs) + + +def _init_dist_slurm(backend, port=None): + """Initialize slurm distributed training environment. + + If argument ``port`` is not specified, then the master port will be system + environment variable ``MASTER_PORT``. If ``MASTER_PORT`` is not in system + environment variable, then a default port ``29500`` will be used. + + Args: + backend (str): Backend of torch.distributed. + port (int, optional): Master port. Defaults to None. + """ + proc_id = int(os.environ['SLURM_PROCID']) + ntasks = int(os.environ['SLURM_NTASKS']) + node_list = os.environ['SLURM_NODELIST'] + num_gpus = torch.cuda.device_count() + torch.cuda.set_device(proc_id % num_gpus) + addr = subprocess.getoutput( + f'scontrol show hostname {node_list} | head -n1') + # specify master port + if port is not None: + os.environ['MASTER_PORT'] = str(port) + elif 'MASTER_PORT' in os.environ: + pass # use MASTER_PORT in the environment variable + else: + # 29500 is torch.distributed default port + os.environ['MASTER_PORT'] = '29500' + # use MASTER_ADDR in the environment variable if it already exists + if 'MASTER_ADDR' not in os.environ: + os.environ['MASTER_ADDR'] = addr + os.environ['WORLD_SIZE'] = str(ntasks) + os.environ['LOCAL_RANK'] = str(proc_id % num_gpus) + os.environ['RANK'] = str(proc_id) + dist.init_process_group(backend=backend) + + +def get_dist_info(): + if dist.is_available() and dist.is_initialized(): + rank = dist.get_rank() + world_size = dist.get_world_size() + else: + rank = 0 + world_size = 1 + return rank, world_size + + +def master_only(func): + + @functools.wraps(func) + def wrapper(*args, **kwargs): + rank, _ = get_dist_info() + if rank == 0: + return func(*args, **kwargs) + + return wrapper + + +def allreduce_params(params, coalesce=True, bucket_size_mb=-1): + """Allreduce parameters. + + Args: + params (list[torch.Parameters]): List of parameters or buffers of a + model. + coalesce (bool, optional): Whether allreduce parameters as a whole. + Defaults to True. + bucket_size_mb (int, optional): Size of bucket, the unit is MB. + Defaults to -1. + """ + _, world_size = get_dist_info() + if world_size == 1: + return + params = [param.data for param in params] + if coalesce: + _allreduce_coalesced(params, world_size, bucket_size_mb) + else: + for tensor in params: + dist.all_reduce(tensor.div_(world_size)) + + +def allreduce_grads(params, coalesce=True, bucket_size_mb=-1): + """Allreduce gradients. + + Args: + params (list[torch.Parameters]): List of parameters of a model + coalesce (bool, optional): Whether allreduce parameters as a whole. + Defaults to True. + bucket_size_mb (int, optional): Size of bucket, the unit is MB. + Defaults to -1. + """ + grads = [ + param.grad.data for param in params + if param.requires_grad and param.grad is not None + ] + _, world_size = get_dist_info() + if world_size == 1: + return + if coalesce: + _allreduce_coalesced(grads, world_size, bucket_size_mb) + else: + for tensor in grads: + dist.all_reduce(tensor.div_(world_size)) + + +def _allreduce_coalesced(tensors, world_size, bucket_size_mb=-1): + if bucket_size_mb > 0: + bucket_size_bytes = bucket_size_mb * 1024 * 1024 + buckets = _take_tensors(tensors, bucket_size_bytes) + else: + buckets = OrderedDict() + for tensor in tensors: + tp = tensor.type() + if tp not in buckets: + buckets[tp] = [] + buckets[tp].append(tensor) + buckets = buckets.values() + + for bucket in buckets: + flat_tensors = _flatten_dense_tensors(bucket) + dist.all_reduce(flat_tensors) + flat_tensors.div_(world_size) + for tensor, synced in zip( + bucket, _unflatten_dense_tensors(flat_tensors, bucket)): + tensor.copy_(synced) diff --git a/annotator/uniformer/mmcv/runner/epoch_based_runner.py b/annotator/uniformer/mmcv/runner/epoch_based_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..766a9ce6afdf09cd11b1b15005f5132583011348 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/epoch_based_runner.py @@ -0,0 +1,187 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os.path as osp +import platform +import shutil +import time +import warnings + +import torch + +import annotator.uniformer.mmcv as mmcv +from .base_runner import BaseRunner +from .builder import RUNNERS +from .checkpoint import save_checkpoint +from .utils import get_host_info + + +@RUNNERS.register_module() +class EpochBasedRunner(BaseRunner): + """Epoch-based Runner. + + This runner train models epoch by epoch. + """ + + def run_iter(self, data_batch, train_mode, **kwargs): + if self.batch_processor is not None: + outputs = self.batch_processor( + self.model, data_batch, train_mode=train_mode, **kwargs) + elif train_mode: + outputs = self.model.train_step(data_batch, self.optimizer, + **kwargs) + else: + outputs = self.model.val_step(data_batch, self.optimizer, **kwargs) + if not isinstance(outputs, dict): + raise TypeError('"batch_processor()" or "model.train_step()"' + 'and "model.val_step()" must return a dict') + if 'log_vars' in outputs: + self.log_buffer.update(outputs['log_vars'], outputs['num_samples']) + self.outputs = outputs + + def train(self, data_loader, **kwargs): + self.model.train() + self.mode = 'train' + self.data_loader = data_loader + self._max_iters = self._max_epochs * len(self.data_loader) + self.call_hook('before_train_epoch') + time.sleep(2) # Prevent possible deadlock during epoch transition + for i, data_batch in enumerate(self.data_loader): + self._inner_iter = i + self.call_hook('before_train_iter') + self.run_iter(data_batch, train_mode=True, **kwargs) + self.call_hook('after_train_iter') + self._iter += 1 + + self.call_hook('after_train_epoch') + self._epoch += 1 + + @torch.no_grad() + def val(self, data_loader, **kwargs): + self.model.eval() + self.mode = 'val' + self.data_loader = data_loader + self.call_hook('before_val_epoch') + time.sleep(2) # Prevent possible deadlock during epoch transition + for i, data_batch in enumerate(self.data_loader): + self._inner_iter = i + self.call_hook('before_val_iter') + self.run_iter(data_batch, train_mode=False) + self.call_hook('after_val_iter') + + self.call_hook('after_val_epoch') + + def run(self, data_loaders, workflow, max_epochs=None, **kwargs): + """Start running. + + Args: + data_loaders (list[:obj:`DataLoader`]): Dataloaders for training + and validation. + workflow (list[tuple]): A list of (phase, epochs) to specify the + running order and epochs. E.g, [('train', 2), ('val', 1)] means + running 2 epochs for training and 1 epoch for validation, + iteratively. + """ + assert isinstance(data_loaders, list) + assert mmcv.is_list_of(workflow, tuple) + assert len(data_loaders) == len(workflow) + if max_epochs is not None: + warnings.warn( + 'setting max_epochs in run is deprecated, ' + 'please set max_epochs in runner_config', DeprecationWarning) + self._max_epochs = max_epochs + + assert self._max_epochs is not None, ( + 'max_epochs must be specified during instantiation') + + for i, flow in enumerate(workflow): + mode, epochs = flow + if mode == 'train': + self._max_iters = self._max_epochs * len(data_loaders[i]) + break + + work_dir = self.work_dir if self.work_dir is not None else 'NONE' + self.logger.info('Start running, host: %s, work_dir: %s', + get_host_info(), work_dir) + self.logger.info('Hooks will be executed in the following order:\n%s', + self.get_hook_info()) + self.logger.info('workflow: %s, max: %d epochs', workflow, + self._max_epochs) + self.call_hook('before_run') + + while self.epoch < self._max_epochs: + for i, flow in enumerate(workflow): + mode, epochs = flow + if isinstance(mode, str): # self.train() + if not hasattr(self, mode): + raise ValueError( + f'runner has no method named "{mode}" to run an ' + 'epoch') + epoch_runner = getattr(self, mode) + else: + raise TypeError( + 'mode in workflow must be a str, but got {}'.format( + type(mode))) + + for _ in range(epochs): + if mode == 'train' and self.epoch >= self._max_epochs: + break + epoch_runner(data_loaders[i], **kwargs) + + time.sleep(1) # wait for some hooks like loggers to finish + self.call_hook('after_run') + + def save_checkpoint(self, + out_dir, + filename_tmpl='epoch_{}.pth', + save_optimizer=True, + meta=None, + create_symlink=True): + """Save the checkpoint. + + Args: + out_dir (str): The directory that checkpoints are saved. + filename_tmpl (str, optional): The checkpoint filename template, + which contains a placeholder for the epoch number. + Defaults to 'epoch_{}.pth'. + save_optimizer (bool, optional): Whether to save the optimizer to + the checkpoint. Defaults to True. + meta (dict, optional): The meta information to be saved in the + checkpoint. Defaults to None. + create_symlink (bool, optional): Whether to create a symlink + "latest.pth" to point to the latest checkpoint. + Defaults to True. + """ + if meta is None: + meta = {} + elif not isinstance(meta, dict): + raise TypeError( + f'meta should be a dict or None, but got {type(meta)}') + if self.meta is not None: + meta.update(self.meta) + # Note: meta.update(self.meta) should be done before + # meta.update(epoch=self.epoch + 1, iter=self.iter) otherwise + # there will be problems with resumed checkpoints. + # More details in https://github.com/open-mmlab/mmcv/pull/1108 + meta.update(epoch=self.epoch + 1, iter=self.iter) + + filename = filename_tmpl.format(self.epoch + 1) + filepath = osp.join(out_dir, filename) + optimizer = self.optimizer if save_optimizer else None + save_checkpoint(self.model, filepath, optimizer=optimizer, meta=meta) + # in some environments, `os.symlink` is not supported, you may need to + # set `create_symlink` to False + if create_symlink: + dst_file = osp.join(out_dir, 'latest.pth') + if platform.system() != 'Windows': + mmcv.symlink(filename, dst_file) + else: + shutil.copy(filepath, dst_file) + + +@RUNNERS.register_module() +class Runner(EpochBasedRunner): + """Deprecated name of EpochBasedRunner.""" + + def __init__(self, *args, **kwargs): + warnings.warn( + 'Runner was deprecated, please use EpochBasedRunner instead') + super().__init__(*args, **kwargs) diff --git a/annotator/uniformer/mmcv/runner/fp16_utils.py b/annotator/uniformer/mmcv/runner/fp16_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1981011d6859192e3e663e29d13500d56ba47f6c --- /dev/null +++ b/annotator/uniformer/mmcv/runner/fp16_utils.py @@ -0,0 +1,410 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import functools +import warnings +from collections import abc +from inspect import getfullargspec + +import numpy as np +import torch +import torch.nn as nn + +from annotator.uniformer.mmcv.utils import TORCH_VERSION, digit_version +from .dist_utils import allreduce_grads as _allreduce_grads + +try: + # If PyTorch version >= 1.6.0, torch.cuda.amp.autocast would be imported + # and used; otherwise, auto fp16 will adopt mmcv's implementation. + # Note that when PyTorch >= 1.6.0, we still cast tensor types to fp16 + # manually, so the behavior may not be consistent with real amp. + from torch.cuda.amp import autocast +except ImportError: + pass + + +def cast_tensor_type(inputs, src_type, dst_type): + """Recursively convert Tensor in inputs from src_type to dst_type. + + Args: + inputs: Inputs that to be casted. + src_type (torch.dtype): Source type.. + dst_type (torch.dtype): Destination type. + + Returns: + The same type with inputs, but all contained Tensors have been cast. + """ + if isinstance(inputs, nn.Module): + return inputs + elif isinstance(inputs, torch.Tensor): + return inputs.to(dst_type) + elif isinstance(inputs, str): + return inputs + elif isinstance(inputs, np.ndarray): + return inputs + elif isinstance(inputs, abc.Mapping): + return type(inputs)({ + k: cast_tensor_type(v, src_type, dst_type) + for k, v in inputs.items() + }) + elif isinstance(inputs, abc.Iterable): + return type(inputs)( + cast_tensor_type(item, src_type, dst_type) for item in inputs) + else: + return inputs + + +def auto_fp16(apply_to=None, out_fp32=False): + """Decorator to enable fp16 training automatically. + + This decorator is useful when you write custom modules and want to support + mixed precision training. If inputs arguments are fp32 tensors, they will + be converted to fp16 automatically. Arguments other than fp32 tensors are + ignored. If you are using PyTorch >= 1.6, torch.cuda.amp is used as the + backend, otherwise, original mmcv implementation will be adopted. + + Args: + apply_to (Iterable, optional): The argument names to be converted. + `None` indicates all arguments. + out_fp32 (bool): Whether to convert the output back to fp32. + + Example: + + >>> import torch.nn as nn + >>> class MyModule1(nn.Module): + >>> + >>> # Convert x and y to fp16 + >>> @auto_fp16() + >>> def forward(self, x, y): + >>> pass + + >>> import torch.nn as nn + >>> class MyModule2(nn.Module): + >>> + >>> # convert pred to fp16 + >>> @auto_fp16(apply_to=('pred', )) + >>> def do_something(self, pred, others): + >>> pass + """ + + def auto_fp16_wrapper(old_func): + + @functools.wraps(old_func) + def new_func(*args, **kwargs): + # check if the module has set the attribute `fp16_enabled`, if not, + # just fallback to the original method. + if not isinstance(args[0], torch.nn.Module): + raise TypeError('@auto_fp16 can only be used to decorate the ' + 'method of nn.Module') + if not (hasattr(args[0], 'fp16_enabled') and args[0].fp16_enabled): + return old_func(*args, **kwargs) + + # get the arg spec of the decorated method + args_info = getfullargspec(old_func) + # get the argument names to be casted + args_to_cast = args_info.args if apply_to is None else apply_to + # convert the args that need to be processed + new_args = [] + # NOTE: default args are not taken into consideration + if args: + arg_names = args_info.args[:len(args)] + for i, arg_name in enumerate(arg_names): + if arg_name in args_to_cast: + new_args.append( + cast_tensor_type(args[i], torch.float, torch.half)) + else: + new_args.append(args[i]) + # convert the kwargs that need to be processed + new_kwargs = {} + if kwargs: + for arg_name, arg_value in kwargs.items(): + if arg_name in args_to_cast: + new_kwargs[arg_name] = cast_tensor_type( + arg_value, torch.float, torch.half) + else: + new_kwargs[arg_name] = arg_value + # apply converted arguments to the decorated method + if (TORCH_VERSION != 'parrots' and + digit_version(TORCH_VERSION) >= digit_version('1.6.0')): + with autocast(enabled=True): + output = old_func(*new_args, **new_kwargs) + else: + output = old_func(*new_args, **new_kwargs) + # cast the results back to fp32 if necessary + if out_fp32: + output = cast_tensor_type(output, torch.half, torch.float) + return output + + return new_func + + return auto_fp16_wrapper + + +def force_fp32(apply_to=None, out_fp16=False): + """Decorator to convert input arguments to fp32 in force. + + This decorator is useful when you write custom modules and want to support + mixed precision training. If there are some inputs that must be processed + in fp32 mode, then this decorator can handle it. If inputs arguments are + fp16 tensors, they will be converted to fp32 automatically. Arguments other + than fp16 tensors are ignored. If you are using PyTorch >= 1.6, + torch.cuda.amp is used as the backend, otherwise, original mmcv + implementation will be adopted. + + Args: + apply_to (Iterable, optional): The argument names to be converted. + `None` indicates all arguments. + out_fp16 (bool): Whether to convert the output back to fp16. + + Example: + + >>> import torch.nn as nn + >>> class MyModule1(nn.Module): + >>> + >>> # Convert x and y to fp32 + >>> @force_fp32() + >>> def loss(self, x, y): + >>> pass + + >>> import torch.nn as nn + >>> class MyModule2(nn.Module): + >>> + >>> # convert pred to fp32 + >>> @force_fp32(apply_to=('pred', )) + >>> def post_process(self, pred, others): + >>> pass + """ + + def force_fp32_wrapper(old_func): + + @functools.wraps(old_func) + def new_func(*args, **kwargs): + # check if the module has set the attribute `fp16_enabled`, if not, + # just fallback to the original method. + if not isinstance(args[0], torch.nn.Module): + raise TypeError('@force_fp32 can only be used to decorate the ' + 'method of nn.Module') + if not (hasattr(args[0], 'fp16_enabled') and args[0].fp16_enabled): + return old_func(*args, **kwargs) + # get the arg spec of the decorated method + args_info = getfullargspec(old_func) + # get the argument names to be casted + args_to_cast = args_info.args if apply_to is None else apply_to + # convert the args that need to be processed + new_args = [] + if args: + arg_names = args_info.args[:len(args)] + for i, arg_name in enumerate(arg_names): + if arg_name in args_to_cast: + new_args.append( + cast_tensor_type(args[i], torch.half, torch.float)) + else: + new_args.append(args[i]) + # convert the kwargs that need to be processed + new_kwargs = dict() + if kwargs: + for arg_name, arg_value in kwargs.items(): + if arg_name in args_to_cast: + new_kwargs[arg_name] = cast_tensor_type( + arg_value, torch.half, torch.float) + else: + new_kwargs[arg_name] = arg_value + # apply converted arguments to the decorated method + if (TORCH_VERSION != 'parrots' and + digit_version(TORCH_VERSION) >= digit_version('1.6.0')): + with autocast(enabled=False): + output = old_func(*new_args, **new_kwargs) + else: + output = old_func(*new_args, **new_kwargs) + # cast the results back to fp32 if necessary + if out_fp16: + output = cast_tensor_type(output, torch.float, torch.half) + return output + + return new_func + + return force_fp32_wrapper + + +def allreduce_grads(params, coalesce=True, bucket_size_mb=-1): + warnings.warning( + '"mmcv.runner.fp16_utils.allreduce_grads" is deprecated, and will be ' + 'removed in v2.8. Please switch to "mmcv.runner.allreduce_grads') + _allreduce_grads(params, coalesce=coalesce, bucket_size_mb=bucket_size_mb) + + +def wrap_fp16_model(model): + """Wrap the FP32 model to FP16. + + If you are using PyTorch >= 1.6, torch.cuda.amp is used as the + backend, otherwise, original mmcv implementation will be adopted. + + For PyTorch >= 1.6, this function will + 1. Set fp16 flag inside the model to True. + + Otherwise: + 1. Convert FP32 model to FP16. + 2. Remain some necessary layers to be FP32, e.g., normalization layers. + 3. Set `fp16_enabled` flag inside the model to True. + + Args: + model (nn.Module): Model in FP32. + """ + if (TORCH_VERSION == 'parrots' + or digit_version(TORCH_VERSION) < digit_version('1.6.0')): + # convert model to fp16 + model.half() + # patch the normalization layers to make it work in fp32 mode + patch_norm_fp32(model) + # set `fp16_enabled` flag + for m in model.modules(): + if hasattr(m, 'fp16_enabled'): + m.fp16_enabled = True + + +def patch_norm_fp32(module): + """Recursively convert normalization layers from FP16 to FP32. + + Args: + module (nn.Module): The modules to be converted in FP16. + + Returns: + nn.Module: The converted module, the normalization layers have been + converted to FP32. + """ + if isinstance(module, (nn.modules.batchnorm._BatchNorm, nn.GroupNorm)): + module.float() + if isinstance(module, nn.GroupNorm) or torch.__version__ < '1.3': + module.forward = patch_forward_method(module.forward, torch.half, + torch.float) + for child in module.children(): + patch_norm_fp32(child) + return module + + +def patch_forward_method(func, src_type, dst_type, convert_output=True): + """Patch the forward method of a module. + + Args: + func (callable): The original forward method. + src_type (torch.dtype): Type of input arguments to be converted from. + dst_type (torch.dtype): Type of input arguments to be converted to. + convert_output (bool): Whether to convert the output back to src_type. + + Returns: + callable: The patched forward method. + """ + + def new_forward(*args, **kwargs): + output = func(*cast_tensor_type(args, src_type, dst_type), + **cast_tensor_type(kwargs, src_type, dst_type)) + if convert_output: + output = cast_tensor_type(output, dst_type, src_type) + return output + + return new_forward + + +class LossScaler: + """Class that manages loss scaling in mixed precision training which + supports both dynamic or static mode. + + The implementation refers to + https://github.com/NVIDIA/apex/blob/master/apex/fp16_utils/loss_scaler.py. + Indirectly, by supplying ``mode='dynamic'`` for dynamic loss scaling. + It's important to understand how :class:`LossScaler` operates. + Loss scaling is designed to combat the problem of underflowing + gradients encountered at long times when training fp16 networks. + Dynamic loss scaling begins by attempting a very high loss + scale. Ironically, this may result in OVERflowing gradients. + If overflowing gradients are encountered, :class:`FP16_Optimizer` then + skips the update step for this particular iteration/minibatch, + and :class:`LossScaler` adjusts the loss scale to a lower value. + If a certain number of iterations occur without overflowing gradients + detected,:class:`LossScaler` increases the loss scale once more. + In this way :class:`LossScaler` attempts to "ride the edge" of always + using the highest loss scale possible without incurring overflow. + + Args: + init_scale (float): Initial loss scale value, default: 2**32. + scale_factor (float): Factor used when adjusting the loss scale. + Default: 2. + mode (str): Loss scaling mode. 'dynamic' or 'static' + scale_window (int): Number of consecutive iterations without an + overflow to wait before increasing the loss scale. Default: 1000. + """ + + def __init__(self, + init_scale=2**32, + mode='dynamic', + scale_factor=2., + scale_window=1000): + self.cur_scale = init_scale + self.cur_iter = 0 + assert mode in ('dynamic', + 'static'), 'mode can only be dynamic or static' + self.mode = mode + self.last_overflow_iter = -1 + self.scale_factor = scale_factor + self.scale_window = scale_window + + def has_overflow(self, params): + """Check if params contain overflow.""" + if self.mode != 'dynamic': + return False + for p in params: + if p.grad is not None and LossScaler._has_inf_or_nan(p.grad.data): + return True + return False + + def _has_inf_or_nan(x): + """Check if params contain NaN.""" + try: + cpu_sum = float(x.float().sum()) + except RuntimeError as instance: + if 'value cannot be converted' not in instance.args[0]: + raise + return True + else: + if cpu_sum == float('inf') or cpu_sum == -float('inf') \ + or cpu_sum != cpu_sum: + return True + return False + + def update_scale(self, overflow): + """update the current loss scale value when overflow happens.""" + if self.mode != 'dynamic': + return + if overflow: + self.cur_scale = max(self.cur_scale / self.scale_factor, 1) + self.last_overflow_iter = self.cur_iter + else: + if (self.cur_iter - self.last_overflow_iter) % \ + self.scale_window == 0: + self.cur_scale *= self.scale_factor + self.cur_iter += 1 + + def state_dict(self): + """Returns the state of the scaler as a :class:`dict`.""" + return dict( + cur_scale=self.cur_scale, + cur_iter=self.cur_iter, + mode=self.mode, + last_overflow_iter=self.last_overflow_iter, + scale_factor=self.scale_factor, + scale_window=self.scale_window) + + def load_state_dict(self, state_dict): + """Loads the loss_scaler state dict. + + Args: + state_dict (dict): scaler state. + """ + self.cur_scale = state_dict['cur_scale'] + self.cur_iter = state_dict['cur_iter'] + self.mode = state_dict['mode'] + self.last_overflow_iter = state_dict['last_overflow_iter'] + self.scale_factor = state_dict['scale_factor'] + self.scale_window = state_dict['scale_window'] + + @property + def loss_scale(self): + return self.cur_scale diff --git a/annotator/uniformer/mmcv/runner/hooks/__init__.py b/annotator/uniformer/mmcv/runner/hooks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..915af28cefab14a14c1188ed861161080fd138a3 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/__init__.py @@ -0,0 +1,29 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .checkpoint import CheckpointHook +from .closure import ClosureHook +from .ema import EMAHook +from .evaluation import DistEvalHook, EvalHook +from .hook import HOOKS, Hook +from .iter_timer import IterTimerHook +from .logger import (DvcliveLoggerHook, LoggerHook, MlflowLoggerHook, + NeptuneLoggerHook, PaviLoggerHook, TensorboardLoggerHook, + TextLoggerHook, WandbLoggerHook) +from .lr_updater import LrUpdaterHook +from .memory import EmptyCacheHook +from .momentum_updater import MomentumUpdaterHook +from .optimizer import (Fp16OptimizerHook, GradientCumulativeFp16OptimizerHook, + GradientCumulativeOptimizerHook, OptimizerHook) +from .profiler import ProfilerHook +from .sampler_seed import DistSamplerSeedHook +from .sync_buffer import SyncBuffersHook + +__all__ = [ + 'HOOKS', 'Hook', 'CheckpointHook', 'ClosureHook', 'LrUpdaterHook', + 'OptimizerHook', 'Fp16OptimizerHook', 'IterTimerHook', + 'DistSamplerSeedHook', 'EmptyCacheHook', 'LoggerHook', 'MlflowLoggerHook', + 'PaviLoggerHook', 'TextLoggerHook', 'TensorboardLoggerHook', + 'NeptuneLoggerHook', 'WandbLoggerHook', 'DvcliveLoggerHook', + 'MomentumUpdaterHook', 'SyncBuffersHook', 'EMAHook', 'EvalHook', + 'DistEvalHook', 'ProfilerHook', 'GradientCumulativeOptimizerHook', + 'GradientCumulativeFp16OptimizerHook' +] diff --git a/annotator/uniformer/mmcv/runner/hooks/checkpoint.py b/annotator/uniformer/mmcv/runner/hooks/checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..6af3fae43ac4b35532641a81eb13557edfc7dfba --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/checkpoint.py @@ -0,0 +1,167 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os.path as osp +import warnings + +from annotator.uniformer.mmcv.fileio import FileClient +from ..dist_utils import allreduce_params, master_only +from .hook import HOOKS, Hook + + +@HOOKS.register_module() +class CheckpointHook(Hook): + """Save checkpoints periodically. + + Args: + interval (int): The saving period. If ``by_epoch=True``, interval + indicates epochs, otherwise it indicates iterations. + Default: -1, which means "never". + by_epoch (bool): Saving checkpoints by epoch or by iteration. + Default: True. + save_optimizer (bool): Whether to save optimizer state_dict in the + checkpoint. It is usually used for resuming experiments. + Default: True. + out_dir (str, optional): The root directory to save checkpoints. If not + specified, ``runner.work_dir`` will be used by default. If + specified, the ``out_dir`` will be the concatenation of ``out_dir`` + and the last level directory of ``runner.work_dir``. + `Changed in version 1.3.16.` + max_keep_ckpts (int, optional): The maximum checkpoints to keep. + In some cases we want only the latest few checkpoints and would + like to delete old ones to save the disk space. + Default: -1, which means unlimited. + save_last (bool, optional): Whether to force the last checkpoint to be + saved regardless of interval. Default: True. + sync_buffer (bool, optional): Whether to synchronize buffers in + different gpus. Default: False. + file_client_args (dict, optional): Arguments to instantiate a + FileClient. See :class:`mmcv.fileio.FileClient` for details. + Default: None. + `New in version 1.3.16.` + + .. warning:: + Before v1.3.16, the ``out_dir`` argument indicates the path where the + checkpoint is stored. However, since v1.3.16, ``out_dir`` indicates the + root directory and the final path to save checkpoint is the + concatenation of ``out_dir`` and the last level directory of + ``runner.work_dir``. Suppose the value of ``out_dir`` is "/path/of/A" + and the value of ``runner.work_dir`` is "/path/of/B", then the final + path will be "/path/of/A/B". + """ + + def __init__(self, + interval=-1, + by_epoch=True, + save_optimizer=True, + out_dir=None, + max_keep_ckpts=-1, + save_last=True, + sync_buffer=False, + file_client_args=None, + **kwargs): + self.interval = interval + self.by_epoch = by_epoch + self.save_optimizer = save_optimizer + self.out_dir = out_dir + self.max_keep_ckpts = max_keep_ckpts + self.save_last = save_last + self.args = kwargs + self.sync_buffer = sync_buffer + self.file_client_args = file_client_args + + def before_run(self, runner): + if not self.out_dir: + self.out_dir = runner.work_dir + + self.file_client = FileClient.infer_client(self.file_client_args, + self.out_dir) + + # if `self.out_dir` is not equal to `runner.work_dir`, it means that + # `self.out_dir` is set so the final `self.out_dir` is the + # concatenation of `self.out_dir` and the last level directory of + # `runner.work_dir` + if self.out_dir != runner.work_dir: + basename = osp.basename(runner.work_dir.rstrip(osp.sep)) + self.out_dir = self.file_client.join_path(self.out_dir, basename) + + runner.logger.info((f'Checkpoints will be saved to {self.out_dir} by ' + f'{self.file_client.name}.')) + + # disable the create_symlink option because some file backends do not + # allow to create a symlink + if 'create_symlink' in self.args: + if self.args[ + 'create_symlink'] and not self.file_client.allow_symlink: + self.args['create_symlink'] = False + warnings.warn( + ('create_symlink is set as True by the user but is changed' + 'to be False because creating symbolic link is not ' + f'allowed in {self.file_client.name}')) + else: + self.args['create_symlink'] = self.file_client.allow_symlink + + def after_train_epoch(self, runner): + if not self.by_epoch: + return + + # save checkpoint for following cases: + # 1. every ``self.interval`` epochs + # 2. reach the last epoch of training + if self.every_n_epochs( + runner, self.interval) or (self.save_last + and self.is_last_epoch(runner)): + runner.logger.info( + f'Saving checkpoint at {runner.epoch + 1} epochs') + if self.sync_buffer: + allreduce_params(runner.model.buffers()) + self._save_checkpoint(runner) + + @master_only + def _save_checkpoint(self, runner): + """Save the current checkpoint and delete unwanted checkpoint.""" + runner.save_checkpoint( + self.out_dir, save_optimizer=self.save_optimizer, **self.args) + if runner.meta is not None: + if self.by_epoch: + cur_ckpt_filename = self.args.get( + 'filename_tmpl', 'epoch_{}.pth').format(runner.epoch + 1) + else: + cur_ckpt_filename = self.args.get( + 'filename_tmpl', 'iter_{}.pth').format(runner.iter + 1) + runner.meta.setdefault('hook_msgs', dict()) + runner.meta['hook_msgs']['last_ckpt'] = self.file_client.join_path( + self.out_dir, cur_ckpt_filename) + # remove other checkpoints + if self.max_keep_ckpts > 0: + if self.by_epoch: + name = 'epoch_{}.pth' + current_ckpt = runner.epoch + 1 + else: + name = 'iter_{}.pth' + current_ckpt = runner.iter + 1 + redundant_ckpts = range( + current_ckpt - self.max_keep_ckpts * self.interval, 0, + -self.interval) + filename_tmpl = self.args.get('filename_tmpl', name) + for _step in redundant_ckpts: + ckpt_path = self.file_client.join_path( + self.out_dir, filename_tmpl.format(_step)) + if self.file_client.isfile(ckpt_path): + self.file_client.remove(ckpt_path) + else: + break + + def after_train_iter(self, runner): + if self.by_epoch: + return + + # save checkpoint for following cases: + # 1. every ``self.interval`` iterations + # 2. reach the last iteration of training + if self.every_n_iters( + runner, self.interval) or (self.save_last + and self.is_last_iter(runner)): + runner.logger.info( + f'Saving checkpoint at {runner.iter + 1} iterations') + if self.sync_buffer: + allreduce_params(runner.model.buffers()) + self._save_checkpoint(runner) diff --git a/annotator/uniformer/mmcv/runner/hooks/closure.py b/annotator/uniformer/mmcv/runner/hooks/closure.py new file mode 100644 index 0000000000000000000000000000000000000000..b955f81f425be4ac3e6bb3f4aac653887989e872 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/closure.py @@ -0,0 +1,11 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .hook import HOOKS, Hook + + +@HOOKS.register_module() +class ClosureHook(Hook): + + def __init__(self, fn_name, fn): + assert hasattr(self, fn_name) + assert callable(fn) + setattr(self, fn_name, fn) diff --git a/annotator/uniformer/mmcv/runner/hooks/ema.py b/annotator/uniformer/mmcv/runner/hooks/ema.py new file mode 100644 index 0000000000000000000000000000000000000000..15c7e68088f019802a59e7ae41cc1fe0c7f28f96 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/ema.py @@ -0,0 +1,89 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from ...parallel import is_module_wrapper +from ..hooks.hook import HOOKS, Hook + + +@HOOKS.register_module() +class EMAHook(Hook): + r"""Exponential Moving Average Hook. + + Use Exponential Moving Average on all parameters of model in training + process. All parameters have a ema backup, which update by the formula + as below. EMAHook takes priority over EvalHook and CheckpointSaverHook. + + .. math:: + + \text{Xema\_{t+1}} = (1 - \text{momentum}) \times + \text{Xema\_{t}} + \text{momentum} \times X_t + + Args: + momentum (float): The momentum used for updating ema parameter. + Defaults to 0.0002. + interval (int): Update ema parameter every interval iteration. + Defaults to 1. + warm_up (int): During first warm_up steps, we may use smaller momentum + to update ema parameters more slowly. Defaults to 100. + resume_from (str): The checkpoint path. Defaults to None. + """ + + def __init__(self, + momentum=0.0002, + interval=1, + warm_up=100, + resume_from=None): + assert isinstance(interval, int) and interval > 0 + self.warm_up = warm_up + self.interval = interval + assert momentum > 0 and momentum < 1 + self.momentum = momentum**interval + self.checkpoint = resume_from + + def before_run(self, runner): + """To resume model with it's ema parameters more friendly. + + Register ema parameter as ``named_buffer`` to model + """ + model = runner.model + if is_module_wrapper(model): + model = model.module + self.param_ema_buffer = {} + self.model_parameters = dict(model.named_parameters(recurse=True)) + for name, value in self.model_parameters.items(): + # "." is not allowed in module's buffer name + buffer_name = f"ema_{name.replace('.', '_')}" + self.param_ema_buffer[name] = buffer_name + model.register_buffer(buffer_name, value.data.clone()) + self.model_buffers = dict(model.named_buffers(recurse=True)) + if self.checkpoint is not None: + runner.resume(self.checkpoint) + + def after_train_iter(self, runner): + """Update ema parameter every self.interval iterations.""" + curr_step = runner.iter + # We warm up the momentum considering the instability at beginning + momentum = min(self.momentum, + (1 + curr_step) / (self.warm_up + curr_step)) + if curr_step % self.interval != 0: + return + for name, parameter in self.model_parameters.items(): + buffer_name = self.param_ema_buffer[name] + buffer_parameter = self.model_buffers[buffer_name] + buffer_parameter.mul_(1 - momentum).add_(momentum, parameter.data) + + def after_train_epoch(self, runner): + """We load parameter values from ema backup to model before the + EvalHook.""" + self._swap_ema_parameters() + + def before_train_epoch(self, runner): + """We recover model's parameter from ema backup after last epoch's + EvalHook.""" + self._swap_ema_parameters() + + def _swap_ema_parameters(self): + """Swap the parameter of model with parameter in ema_buffer.""" + for name, value in self.model_parameters.items(): + temp = value.data.clone() + ema_buffer = self.model_buffers[self.param_ema_buffer[name]] + value.data.copy_(ema_buffer.data) + ema_buffer.data.copy_(temp) diff --git a/annotator/uniformer/mmcv/runner/hooks/evaluation.py b/annotator/uniformer/mmcv/runner/hooks/evaluation.py new file mode 100644 index 0000000000000000000000000000000000000000..4d00999ce5665c53bded8de9e084943eee2d230d --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/evaluation.py @@ -0,0 +1,509 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os.path as osp +import warnings +from math import inf + +import torch.distributed as dist +from torch.nn.modules.batchnorm import _BatchNorm +from torch.utils.data import DataLoader + +from annotator.uniformer.mmcv.fileio import FileClient +from annotator.uniformer.mmcv.utils import is_seq_of +from .hook import Hook +from .logger import LoggerHook + + +class EvalHook(Hook): + """Non-Distributed evaluation hook. + + This hook will regularly perform evaluation in a given interval when + performing in non-distributed environment. + + Args: + dataloader (DataLoader): A PyTorch dataloader, whose dataset has + implemented ``evaluate`` function. + start (int | None, optional): Evaluation starting epoch. It enables + evaluation before the training starts if ``start`` <= the resuming + epoch. If None, whether to evaluate is merely decided by + ``interval``. Default: None. + interval (int): Evaluation interval. Default: 1. + by_epoch (bool): Determine perform evaluation by epoch or by iteration. + If set to True, it will perform by epoch. Otherwise, by iteration. + Default: True. + save_best (str, optional): If a metric is specified, it would measure + the best checkpoint during evaluation. The information about best + checkpoint would be saved in ``runner.meta['hook_msgs']`` to keep + best score value and best checkpoint path, which will be also + loaded when resume checkpoint. Options are the evaluation metrics + on the test dataset. e.g., ``bbox_mAP``, ``segm_mAP`` for bbox + detection and instance segmentation. ``AR@100`` for proposal + recall. If ``save_best`` is ``auto``, the first key of the returned + ``OrderedDict`` result will be used. Default: None. + rule (str | None, optional): Comparison rule for best score. If set to + None, it will infer a reasonable rule. Keys such as 'acc', 'top' + .etc will be inferred by 'greater' rule. Keys contain 'loss' will + be inferred by 'less' rule. Options are 'greater', 'less', None. + Default: None. + test_fn (callable, optional): test a model with samples from a + dataloader, and return the test results. If ``None``, the default + test function ``mmcv.engine.single_gpu_test`` will be used. + (default: ``None``) + greater_keys (List[str] | None, optional): Metric keys that will be + inferred by 'greater' comparison rule. If ``None``, + _default_greater_keys will be used. (default: ``None``) + less_keys (List[str] | None, optional): Metric keys that will be + inferred by 'less' comparison rule. If ``None``, _default_less_keys + will be used. (default: ``None``) + out_dir (str, optional): The root directory to save checkpoints. If not + specified, `runner.work_dir` will be used by default. If specified, + the `out_dir` will be the concatenation of `out_dir` and the last + level directory of `runner.work_dir`. + `New in version 1.3.16.` + file_client_args (dict): Arguments to instantiate a FileClient. + See :class:`mmcv.fileio.FileClient` for details. Default: None. + `New in version 1.3.16.` + **eval_kwargs: Evaluation arguments fed into the evaluate function of + the dataset. + + Notes: + If new arguments are added for EvalHook, tools/test.py, + tools/eval_metric.py may be affected. + """ + + # Since the key for determine greater or less is related to the downstream + # tasks, downstream repos may need to overwrite the following inner + # variable accordingly. + + rule_map = {'greater': lambda x, y: x > y, 'less': lambda x, y: x < y} + init_value_map = {'greater': -inf, 'less': inf} + _default_greater_keys = [ + 'acc', 'top', 'AR@', 'auc', 'precision', 'mAP', 'mDice', 'mIoU', + 'mAcc', 'aAcc' + ] + _default_less_keys = ['loss'] + + def __init__(self, + dataloader, + start=None, + interval=1, + by_epoch=True, + save_best=None, + rule=None, + test_fn=None, + greater_keys=None, + less_keys=None, + out_dir=None, + file_client_args=None, + **eval_kwargs): + if not isinstance(dataloader, DataLoader): + raise TypeError(f'dataloader must be a pytorch DataLoader, ' + f'but got {type(dataloader)}') + + if interval <= 0: + raise ValueError(f'interval must be a positive number, ' + f'but got {interval}') + + assert isinstance(by_epoch, bool), '``by_epoch`` should be a boolean' + + if start is not None and start < 0: + raise ValueError(f'The evaluation start epoch {start} is smaller ' + f'than 0') + + self.dataloader = dataloader + self.interval = interval + self.start = start + self.by_epoch = by_epoch + + assert isinstance(save_best, str) or save_best is None, \ + '""save_best"" should be a str or None ' \ + f'rather than {type(save_best)}' + self.save_best = save_best + self.eval_kwargs = eval_kwargs + self.initial_flag = True + + if test_fn is None: + from annotator.uniformer.mmcv.engine import single_gpu_test + self.test_fn = single_gpu_test + else: + self.test_fn = test_fn + + if greater_keys is None: + self.greater_keys = self._default_greater_keys + else: + if not isinstance(greater_keys, (list, tuple)): + greater_keys = (greater_keys, ) + assert is_seq_of(greater_keys, str) + self.greater_keys = greater_keys + + if less_keys is None: + self.less_keys = self._default_less_keys + else: + if not isinstance(less_keys, (list, tuple)): + less_keys = (less_keys, ) + assert is_seq_of(less_keys, str) + self.less_keys = less_keys + + if self.save_best is not None: + self.best_ckpt_path = None + self._init_rule(rule, self.save_best) + + self.out_dir = out_dir + self.file_client_args = file_client_args + + def _init_rule(self, rule, key_indicator): + """Initialize rule, key_indicator, comparison_func, and best score. + + Here is the rule to determine which rule is used for key indicator + when the rule is not specific (note that the key indicator matching + is case-insensitive): + 1. If the key indicator is in ``self.greater_keys``, the rule will be + specified as 'greater'. + 2. Or if the key indicator is in ``self.less_keys``, the rule will be + specified as 'less'. + 3. Or if the key indicator is equal to the substring in any one item + in ``self.greater_keys``, the rule will be specified as 'greater'. + 4. Or if the key indicator is equal to the substring in any one item + in ``self.less_keys``, the rule will be specified as 'less'. + + Args: + rule (str | None): Comparison rule for best score. + key_indicator (str | None): Key indicator to determine the + comparison rule. + """ + if rule not in self.rule_map and rule is not None: + raise KeyError(f'rule must be greater, less or None, ' + f'but got {rule}.') + + if rule is None: + if key_indicator != 'auto': + # `_lc` here means we use the lower case of keys for + # case-insensitive matching + key_indicator_lc = key_indicator.lower() + greater_keys = [key.lower() for key in self.greater_keys] + less_keys = [key.lower() for key in self.less_keys] + + if key_indicator_lc in greater_keys: + rule = 'greater' + elif key_indicator_lc in less_keys: + rule = 'less' + elif any(key in key_indicator_lc for key in greater_keys): + rule = 'greater' + elif any(key in key_indicator_lc for key in less_keys): + rule = 'less' + else: + raise ValueError(f'Cannot infer the rule for key ' + f'{key_indicator}, thus a specific rule ' + f'must be specified.') + self.rule = rule + self.key_indicator = key_indicator + if self.rule is not None: + self.compare_func = self.rule_map[self.rule] + + def before_run(self, runner): + if not self.out_dir: + self.out_dir = runner.work_dir + + self.file_client = FileClient.infer_client(self.file_client_args, + self.out_dir) + + # if `self.out_dir` is not equal to `runner.work_dir`, it means that + # `self.out_dir` is set so the final `self.out_dir` is the + # concatenation of `self.out_dir` and the last level directory of + # `runner.work_dir` + if self.out_dir != runner.work_dir: + basename = osp.basename(runner.work_dir.rstrip(osp.sep)) + self.out_dir = self.file_client.join_path(self.out_dir, basename) + runner.logger.info( + (f'The best checkpoint will be saved to {self.out_dir} by ' + f'{self.file_client.name}')) + + if self.save_best is not None: + if runner.meta is None: + warnings.warn('runner.meta is None. Creating an empty one.') + runner.meta = dict() + runner.meta.setdefault('hook_msgs', dict()) + self.best_ckpt_path = runner.meta['hook_msgs'].get( + 'best_ckpt', None) + + def before_train_iter(self, runner): + """Evaluate the model only at the start of training by iteration.""" + if self.by_epoch or not self.initial_flag: + return + if self.start is not None and runner.iter >= self.start: + self.after_train_iter(runner) + self.initial_flag = False + + def before_train_epoch(self, runner): + """Evaluate the model only at the start of training by epoch.""" + if not (self.by_epoch and self.initial_flag): + return + if self.start is not None and runner.epoch >= self.start: + self.after_train_epoch(runner) + self.initial_flag = False + + def after_train_iter(self, runner): + """Called after every training iter to evaluate the results.""" + if not self.by_epoch and self._should_evaluate(runner): + # Because the priority of EvalHook is higher than LoggerHook, the + # training log and the evaluating log are mixed. Therefore, + # we need to dump the training log and clear it before evaluating + # log is generated. In addition, this problem will only appear in + # `IterBasedRunner` whose `self.by_epoch` is False, because + # `EpochBasedRunner` whose `self.by_epoch` is True calls + # `_do_evaluate` in `after_train_epoch` stage, and at this stage + # the training log has been printed, so it will not cause any + # problem. more details at + # https://github.com/open-mmlab/mmsegmentation/issues/694 + for hook in runner._hooks: + if isinstance(hook, LoggerHook): + hook.after_train_iter(runner) + runner.log_buffer.clear() + + self._do_evaluate(runner) + + def after_train_epoch(self, runner): + """Called after every training epoch to evaluate the results.""" + if self.by_epoch and self._should_evaluate(runner): + self._do_evaluate(runner) + + def _do_evaluate(self, runner): + """perform evaluation and save ckpt.""" + results = self.test_fn(runner.model, self.dataloader) + runner.log_buffer.output['eval_iter_num'] = len(self.dataloader) + key_score = self.evaluate(runner, results) + # the key_score may be `None` so it needs to skip the action to save + # the best checkpoint + if self.save_best and key_score: + self._save_ckpt(runner, key_score) + + def _should_evaluate(self, runner): + """Judge whether to perform evaluation. + + Here is the rule to judge whether to perform evaluation: + 1. It will not perform evaluation during the epoch/iteration interval, + which is determined by ``self.interval``. + 2. It will not perform evaluation if the start time is larger than + current time. + 3. It will not perform evaluation when current time is larger than + the start time but during epoch/iteration interval. + + Returns: + bool: The flag indicating whether to perform evaluation. + """ + if self.by_epoch: + current = runner.epoch + check_time = self.every_n_epochs + else: + current = runner.iter + check_time = self.every_n_iters + + if self.start is None: + if not check_time(runner, self.interval): + # No evaluation during the interval. + return False + elif (current + 1) < self.start: + # No evaluation if start is larger than the current time. + return False + else: + # Evaluation only at epochs/iters 3, 5, 7... + # if start==3 and interval==2 + if (current + 1 - self.start) % self.interval: + return False + return True + + def _save_ckpt(self, runner, key_score): + """Save the best checkpoint. + + It will compare the score according to the compare function, write + related information (best score, best checkpoint path) and save the + best checkpoint into ``work_dir``. + """ + if self.by_epoch: + current = f'epoch_{runner.epoch + 1}' + cur_type, cur_time = 'epoch', runner.epoch + 1 + else: + current = f'iter_{runner.iter + 1}' + cur_type, cur_time = 'iter', runner.iter + 1 + + best_score = runner.meta['hook_msgs'].get( + 'best_score', self.init_value_map[self.rule]) + if self.compare_func(key_score, best_score): + best_score = key_score + runner.meta['hook_msgs']['best_score'] = best_score + + if self.best_ckpt_path and self.file_client.isfile( + self.best_ckpt_path): + self.file_client.remove(self.best_ckpt_path) + runner.logger.info( + (f'The previous best checkpoint {self.best_ckpt_path} was ' + 'removed')) + + best_ckpt_name = f'best_{self.key_indicator}_{current}.pth' + self.best_ckpt_path = self.file_client.join_path( + self.out_dir, best_ckpt_name) + runner.meta['hook_msgs']['best_ckpt'] = self.best_ckpt_path + + runner.save_checkpoint( + self.out_dir, best_ckpt_name, create_symlink=False) + runner.logger.info( + f'Now best checkpoint is saved as {best_ckpt_name}.') + runner.logger.info( + f'Best {self.key_indicator} is {best_score:0.4f} ' + f'at {cur_time} {cur_type}.') + + def evaluate(self, runner, results): + """Evaluate the results. + + Args: + runner (:obj:`mmcv.Runner`): The underlined training runner. + results (list): Output results. + """ + eval_res = self.dataloader.dataset.evaluate( + results, logger=runner.logger, **self.eval_kwargs) + + for name, val in eval_res.items(): + runner.log_buffer.output[name] = val + runner.log_buffer.ready = True + + if self.save_best is not None: + # If the performance of model is pool, the `eval_res` may be an + # empty dict and it will raise exception when `self.save_best` is + # not None. More details at + # https://github.com/open-mmlab/mmdetection/issues/6265. + if not eval_res: + warnings.warn( + 'Since `eval_res` is an empty dict, the behavior to save ' + 'the best checkpoint will be skipped in this evaluation.') + return None + + if self.key_indicator == 'auto': + # infer from eval_results + self._init_rule(self.rule, list(eval_res.keys())[0]) + return eval_res[self.key_indicator] + + return None + + +class DistEvalHook(EvalHook): + """Distributed evaluation hook. + + This hook will regularly perform evaluation in a given interval when + performing in distributed environment. + + Args: + dataloader (DataLoader): A PyTorch dataloader, whose dataset has + implemented ``evaluate`` function. + start (int | None, optional): Evaluation starting epoch. It enables + evaluation before the training starts if ``start`` <= the resuming + epoch. If None, whether to evaluate is merely decided by + ``interval``. Default: None. + interval (int): Evaluation interval. Default: 1. + by_epoch (bool): Determine perform evaluation by epoch or by iteration. + If set to True, it will perform by epoch. Otherwise, by iteration. + default: True. + save_best (str, optional): If a metric is specified, it would measure + the best checkpoint during evaluation. The information about best + checkpoint would be saved in ``runner.meta['hook_msgs']`` to keep + best score value and best checkpoint path, which will be also + loaded when resume checkpoint. Options are the evaluation metrics + on the test dataset. e.g., ``bbox_mAP``, ``segm_mAP`` for bbox + detection and instance segmentation. ``AR@100`` for proposal + recall. If ``save_best`` is ``auto``, the first key of the returned + ``OrderedDict`` result will be used. Default: None. + rule (str | None, optional): Comparison rule for best score. If set to + None, it will infer a reasonable rule. Keys such as 'acc', 'top' + .etc will be inferred by 'greater' rule. Keys contain 'loss' will + be inferred by 'less' rule. Options are 'greater', 'less', None. + Default: None. + test_fn (callable, optional): test a model with samples from a + dataloader in a multi-gpu manner, and return the test results. If + ``None``, the default test function ``mmcv.engine.multi_gpu_test`` + will be used. (default: ``None``) + tmpdir (str | None): Temporary directory to save the results of all + processes. Default: None. + gpu_collect (bool): Whether to use gpu or cpu to collect results. + Default: False. + broadcast_bn_buffer (bool): Whether to broadcast the + buffer(running_mean and running_var) of rank 0 to other rank + before evaluation. Default: True. + out_dir (str, optional): The root directory to save checkpoints. If not + specified, `runner.work_dir` will be used by default. If specified, + the `out_dir` will be the concatenation of `out_dir` and the last + level directory of `runner.work_dir`. + file_client_args (dict): Arguments to instantiate a FileClient. + See :class:`mmcv.fileio.FileClient` for details. Default: None. + **eval_kwargs: Evaluation arguments fed into the evaluate function of + the dataset. + """ + + def __init__(self, + dataloader, + start=None, + interval=1, + by_epoch=True, + save_best=None, + rule=None, + test_fn=None, + greater_keys=None, + less_keys=None, + broadcast_bn_buffer=True, + tmpdir=None, + gpu_collect=False, + out_dir=None, + file_client_args=None, + **eval_kwargs): + + if test_fn is None: + from annotator.uniformer.mmcv.engine import multi_gpu_test + test_fn = multi_gpu_test + + super().__init__( + dataloader, + start=start, + interval=interval, + by_epoch=by_epoch, + save_best=save_best, + rule=rule, + test_fn=test_fn, + greater_keys=greater_keys, + less_keys=less_keys, + out_dir=out_dir, + file_client_args=file_client_args, + **eval_kwargs) + + self.broadcast_bn_buffer = broadcast_bn_buffer + self.tmpdir = tmpdir + self.gpu_collect = gpu_collect + + def _do_evaluate(self, runner): + """perform evaluation and save ckpt.""" + # Synchronization of BatchNorm's buffer (running_mean + # and running_var) is not supported in the DDP of pytorch, + # which may cause the inconsistent performance of models in + # different ranks, so we broadcast BatchNorm's buffers + # of rank 0 to other ranks to avoid this. + if self.broadcast_bn_buffer: + model = runner.model + for name, module in model.named_modules(): + if isinstance(module, + _BatchNorm) and module.track_running_stats: + dist.broadcast(module.running_var, 0) + dist.broadcast(module.running_mean, 0) + + tmpdir = self.tmpdir + if tmpdir is None: + tmpdir = osp.join(runner.work_dir, '.eval_hook') + + results = self.test_fn( + runner.model, + self.dataloader, + tmpdir=tmpdir, + gpu_collect=self.gpu_collect) + if runner.rank == 0: + print('\n') + runner.log_buffer.output['eval_iter_num'] = len(self.dataloader) + key_score = self.evaluate(runner, results) + # the key_score may be `None` so it needs to skip the action to + # save the best checkpoint + if self.save_best and key_score: + self._save_ckpt(runner, key_score) diff --git a/annotator/uniformer/mmcv/runner/hooks/hook.py b/annotator/uniformer/mmcv/runner/hooks/hook.py new file mode 100644 index 0000000000000000000000000000000000000000..b8855c107727ecf85b917c890fc8b7f6359238a4 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/hook.py @@ -0,0 +1,92 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from annotator.uniformer.mmcv.utils import Registry, is_method_overridden + +HOOKS = Registry('hook') + + +class Hook: + stages = ('before_run', 'before_train_epoch', 'before_train_iter', + 'after_train_iter', 'after_train_epoch', 'before_val_epoch', + 'before_val_iter', 'after_val_iter', 'after_val_epoch', + 'after_run') + + def before_run(self, runner): + pass + + def after_run(self, runner): + pass + + def before_epoch(self, runner): + pass + + def after_epoch(self, runner): + pass + + def before_iter(self, runner): + pass + + def after_iter(self, runner): + pass + + def before_train_epoch(self, runner): + self.before_epoch(runner) + + def before_val_epoch(self, runner): + self.before_epoch(runner) + + def after_train_epoch(self, runner): + self.after_epoch(runner) + + def after_val_epoch(self, runner): + self.after_epoch(runner) + + def before_train_iter(self, runner): + self.before_iter(runner) + + def before_val_iter(self, runner): + self.before_iter(runner) + + def after_train_iter(self, runner): + self.after_iter(runner) + + def after_val_iter(self, runner): + self.after_iter(runner) + + def every_n_epochs(self, runner, n): + return (runner.epoch + 1) % n == 0 if n > 0 else False + + def every_n_inner_iters(self, runner, n): + return (runner.inner_iter + 1) % n == 0 if n > 0 else False + + def every_n_iters(self, runner, n): + return (runner.iter + 1) % n == 0 if n > 0 else False + + def end_of_epoch(self, runner): + return runner.inner_iter + 1 == len(runner.data_loader) + + def is_last_epoch(self, runner): + return runner.epoch + 1 == runner._max_epochs + + def is_last_iter(self, runner): + return runner.iter + 1 == runner._max_iters + + def get_triggered_stages(self): + trigger_stages = set() + for stage in Hook.stages: + if is_method_overridden(stage, Hook, self): + trigger_stages.add(stage) + + # some methods will be triggered in multi stages + # use this dict to map method to stages. + method_stages_map = { + 'before_epoch': ['before_train_epoch', 'before_val_epoch'], + 'after_epoch': ['after_train_epoch', 'after_val_epoch'], + 'before_iter': ['before_train_iter', 'before_val_iter'], + 'after_iter': ['after_train_iter', 'after_val_iter'], + } + + for method, map_stages in method_stages_map.items(): + if is_method_overridden(method, Hook, self): + trigger_stages.update(map_stages) + + return [stage for stage in Hook.stages if stage in trigger_stages] diff --git a/annotator/uniformer/mmcv/runner/hooks/iter_timer.py b/annotator/uniformer/mmcv/runner/hooks/iter_timer.py new file mode 100644 index 0000000000000000000000000000000000000000..cfd5002fe85ffc6992155ac01003878064a1d9be --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/iter_timer.py @@ -0,0 +1,18 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import time + +from .hook import HOOKS, Hook + + +@HOOKS.register_module() +class IterTimerHook(Hook): + + def before_epoch(self, runner): + self.t = time.time() + + def before_iter(self, runner): + runner.log_buffer.update({'data_time': time.time() - self.t}) + + def after_iter(self, runner): + runner.log_buffer.update({'time': time.time() - self.t}) + self.t = time.time() diff --git a/annotator/uniformer/mmcv/runner/hooks/logger/__init__.py b/annotator/uniformer/mmcv/runner/hooks/logger/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a0b6b345640a895368ac8a647afef6f24333d90e --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/logger/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .base import LoggerHook +from .dvclive import DvcliveLoggerHook +from .mlflow import MlflowLoggerHook +from .neptune import NeptuneLoggerHook +from .pavi import PaviLoggerHook +from .tensorboard import TensorboardLoggerHook +from .text import TextLoggerHook +from .wandb import WandbLoggerHook + +__all__ = [ + 'LoggerHook', 'MlflowLoggerHook', 'PaviLoggerHook', + 'TensorboardLoggerHook', 'TextLoggerHook', 'WandbLoggerHook', + 'NeptuneLoggerHook', 'DvcliveLoggerHook' +] diff --git a/annotator/uniformer/mmcv/runner/hooks/logger/base.py b/annotator/uniformer/mmcv/runner/hooks/logger/base.py new file mode 100644 index 0000000000000000000000000000000000000000..f845256729458ced821762a1b8ef881e17ff9955 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/logger/base.py @@ -0,0 +1,166 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numbers +from abc import ABCMeta, abstractmethod + +import numpy as np +import torch + +from ..hook import Hook + + +class LoggerHook(Hook): + """Base class for logger hooks. + + Args: + interval (int): Logging interval (every k iterations). + ignore_last (bool): Ignore the log of last iterations in each epoch + if less than `interval`. + reset_flag (bool): Whether to clear the output buffer after logging. + by_epoch (bool): Whether EpochBasedRunner is used. + """ + + __metaclass__ = ABCMeta + + def __init__(self, + interval=10, + ignore_last=True, + reset_flag=False, + by_epoch=True): + self.interval = interval + self.ignore_last = ignore_last + self.reset_flag = reset_flag + self.by_epoch = by_epoch + + @abstractmethod + def log(self, runner): + pass + + @staticmethod + def is_scalar(val, include_np=True, include_torch=True): + """Tell the input variable is a scalar or not. + + Args: + val: Input variable. + include_np (bool): Whether include 0-d np.ndarray as a scalar. + include_torch (bool): Whether include 0-d torch.Tensor as a scalar. + + Returns: + bool: True or False. + """ + if isinstance(val, numbers.Number): + return True + elif include_np and isinstance(val, np.ndarray) and val.ndim == 0: + return True + elif include_torch and isinstance(val, torch.Tensor) and len(val) == 1: + return True + else: + return False + + def get_mode(self, runner): + if runner.mode == 'train': + if 'time' in runner.log_buffer.output: + mode = 'train' + else: + mode = 'val' + elif runner.mode == 'val': + mode = 'val' + else: + raise ValueError(f"runner mode should be 'train' or 'val', " + f'but got {runner.mode}') + return mode + + def get_epoch(self, runner): + if runner.mode == 'train': + epoch = runner.epoch + 1 + elif runner.mode == 'val': + # normal val mode + # runner.epoch += 1 has been done before val workflow + epoch = runner.epoch + else: + raise ValueError(f"runner mode should be 'train' or 'val', " + f'but got {runner.mode}') + return epoch + + def get_iter(self, runner, inner_iter=False): + """Get the current training iteration step.""" + if self.by_epoch and inner_iter: + current_iter = runner.inner_iter + 1 + else: + current_iter = runner.iter + 1 + return current_iter + + def get_lr_tags(self, runner): + tags = {} + lrs = runner.current_lr() + if isinstance(lrs, dict): + for name, value in lrs.items(): + tags[f'learning_rate/{name}'] = value[0] + else: + tags['learning_rate'] = lrs[0] + return tags + + def get_momentum_tags(self, runner): + tags = {} + momentums = runner.current_momentum() + if isinstance(momentums, dict): + for name, value in momentums.items(): + tags[f'momentum/{name}'] = value[0] + else: + tags['momentum'] = momentums[0] + return tags + + def get_loggable_tags(self, + runner, + allow_scalar=True, + allow_text=False, + add_mode=True, + tags_to_skip=('time', 'data_time')): + tags = {} + for var, val in runner.log_buffer.output.items(): + if var in tags_to_skip: + continue + if self.is_scalar(val) and not allow_scalar: + continue + if isinstance(val, str) and not allow_text: + continue + if add_mode: + var = f'{self.get_mode(runner)}/{var}' + tags[var] = val + tags.update(self.get_lr_tags(runner)) + tags.update(self.get_momentum_tags(runner)) + return tags + + def before_run(self, runner): + for hook in runner.hooks[::-1]: + if isinstance(hook, LoggerHook): + hook.reset_flag = True + break + + def before_epoch(self, runner): + runner.log_buffer.clear() # clear logs of last epoch + + def after_train_iter(self, runner): + if self.by_epoch and self.every_n_inner_iters(runner, self.interval): + runner.log_buffer.average(self.interval) + elif not self.by_epoch and self.every_n_iters(runner, self.interval): + runner.log_buffer.average(self.interval) + elif self.end_of_epoch(runner) and not self.ignore_last: + # not precise but more stable + runner.log_buffer.average(self.interval) + + if runner.log_buffer.ready: + self.log(runner) + if self.reset_flag: + runner.log_buffer.clear_output() + + def after_train_epoch(self, runner): + if runner.log_buffer.ready: + self.log(runner) + if self.reset_flag: + runner.log_buffer.clear_output() + + def after_val_epoch(self, runner): + runner.log_buffer.average() + self.log(runner) + if self.reset_flag: + runner.log_buffer.clear_output() diff --git a/annotator/uniformer/mmcv/runner/hooks/logger/dvclive.py b/annotator/uniformer/mmcv/runner/hooks/logger/dvclive.py new file mode 100644 index 0000000000000000000000000000000000000000..687cdc58c0336c92b1e4f9a410ba67ebaab2bc7a --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/logger/dvclive.py @@ -0,0 +1,58 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from ...dist_utils import master_only +from ..hook import HOOKS +from .base import LoggerHook + + +@HOOKS.register_module() +class DvcliveLoggerHook(LoggerHook): + """Class to log metrics with dvclive. + + It requires `dvclive`_ to be installed. + + Args: + path (str): Directory where dvclive will write TSV log files. + interval (int): Logging interval (every k iterations). + Default 10. + ignore_last (bool): Ignore the log of last iterations in each epoch + if less than `interval`. + Default: True. + reset_flag (bool): Whether to clear the output buffer after logging. + Default: True. + by_epoch (bool): Whether EpochBasedRunner is used. + Default: True. + + .. _dvclive: + https://dvc.org/doc/dvclive + """ + + def __init__(self, + path, + interval=10, + ignore_last=True, + reset_flag=True, + by_epoch=True): + + super(DvcliveLoggerHook, self).__init__(interval, ignore_last, + reset_flag, by_epoch) + self.path = path + self.import_dvclive() + + def import_dvclive(self): + try: + import dvclive + except ImportError: + raise ImportError( + 'Please run "pip install dvclive" to install dvclive') + self.dvclive = dvclive + + @master_only + def before_run(self, runner): + self.dvclive.init(self.path) + + @master_only + def log(self, runner): + tags = self.get_loggable_tags(runner) + if tags: + for k, v in tags.items(): + self.dvclive.log(k, v, step=self.get_iter(runner)) diff --git a/annotator/uniformer/mmcv/runner/hooks/logger/mlflow.py b/annotator/uniformer/mmcv/runner/hooks/logger/mlflow.py new file mode 100644 index 0000000000000000000000000000000000000000..f9a72592be47b534ce22573775fd5a7e8e86d72d --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/logger/mlflow.py @@ -0,0 +1,78 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from ...dist_utils import master_only +from ..hook import HOOKS +from .base import LoggerHook + + +@HOOKS.register_module() +class MlflowLoggerHook(LoggerHook): + + def __init__(self, + exp_name=None, + tags=None, + log_model=True, + interval=10, + ignore_last=True, + reset_flag=False, + by_epoch=True): + """Class to log metrics and (optionally) a trained model to MLflow. + + It requires `MLflow`_ to be installed. + + Args: + exp_name (str, optional): Name of the experiment to be used. + Default None. + If not None, set the active experiment. + If experiment does not exist, an experiment with provided name + will be created. + tags (dict of str: str, optional): Tags for the current run. + Default None. + If not None, set tags for the current run. + log_model (bool, optional): Whether to log an MLflow artifact. + Default True. + If True, log runner.model as an MLflow artifact + for the current run. + interval (int): Logging interval (every k iterations). + ignore_last (bool): Ignore the log of last iterations in each epoch + if less than `interval`. + reset_flag (bool): Whether to clear the output buffer after logging + by_epoch (bool): Whether EpochBasedRunner is used. + + .. _MLflow: + https://www.mlflow.org/docs/latest/index.html + """ + super(MlflowLoggerHook, self).__init__(interval, ignore_last, + reset_flag, by_epoch) + self.import_mlflow() + self.exp_name = exp_name + self.tags = tags + self.log_model = log_model + + def import_mlflow(self): + try: + import mlflow + import mlflow.pytorch as mlflow_pytorch + except ImportError: + raise ImportError( + 'Please run "pip install mlflow" to install mlflow') + self.mlflow = mlflow + self.mlflow_pytorch = mlflow_pytorch + + @master_only + def before_run(self, runner): + super(MlflowLoggerHook, self).before_run(runner) + if self.exp_name is not None: + self.mlflow.set_experiment(self.exp_name) + if self.tags is not None: + self.mlflow.set_tags(self.tags) + + @master_only + def log(self, runner): + tags = self.get_loggable_tags(runner) + if tags: + self.mlflow.log_metrics(tags, step=self.get_iter(runner)) + + @master_only + def after_run(self, runner): + if self.log_model: + self.mlflow_pytorch.log_model(runner.model, 'models') diff --git a/annotator/uniformer/mmcv/runner/hooks/logger/neptune.py b/annotator/uniformer/mmcv/runner/hooks/logger/neptune.py new file mode 100644 index 0000000000000000000000000000000000000000..7a38772b0c93a8608f32c6357b8616e77c139dc9 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/logger/neptune.py @@ -0,0 +1,82 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from ...dist_utils import master_only +from ..hook import HOOKS +from .base import LoggerHook + + +@HOOKS.register_module() +class NeptuneLoggerHook(LoggerHook): + """Class to log metrics to NeptuneAI. + + It requires `neptune-client` to be installed. + + Args: + init_kwargs (dict): a dict contains the initialization keys as below: + - project (str): Name of a project in a form of + namespace/project_name. If None, the value of + NEPTUNE_PROJECT environment variable will be taken. + - api_token (str): User’s API token. + If None, the value of NEPTUNE_API_TOKEN environment + variable will be taken. Note: It is strongly recommended + to use NEPTUNE_API_TOKEN environment variable rather than + placing your API token in plain text in your source code. + - name (str, optional, default is 'Untitled'): Editable name of + the run. Name is displayed in the run's Details and in + Runs table as a column. + Check https://docs.neptune.ai/api-reference/neptune#init for + more init arguments. + interval (int): Logging interval (every k iterations). + ignore_last (bool): Ignore the log of last iterations in each epoch + if less than `interval`. + reset_flag (bool): Whether to clear the output buffer after logging + by_epoch (bool): Whether EpochBasedRunner is used. + + .. _NeptuneAI: + https://docs.neptune.ai/you-should-know/logging-metadata + """ + + def __init__(self, + init_kwargs=None, + interval=10, + ignore_last=True, + reset_flag=True, + with_step=True, + by_epoch=True): + + super(NeptuneLoggerHook, self).__init__(interval, ignore_last, + reset_flag, by_epoch) + self.import_neptune() + self.init_kwargs = init_kwargs + self.with_step = with_step + + def import_neptune(self): + try: + import neptune.new as neptune + except ImportError: + raise ImportError( + 'Please run "pip install neptune-client" to install neptune') + self.neptune = neptune + self.run = None + + @master_only + def before_run(self, runner): + if self.init_kwargs: + self.run = self.neptune.init(**self.init_kwargs) + else: + self.run = self.neptune.init() + + @master_only + def log(self, runner): + tags = self.get_loggable_tags(runner) + if tags: + for tag_name, tag_value in tags.items(): + if self.with_step: + self.run[tag_name].log( + tag_value, step=self.get_iter(runner)) + else: + tags['global_step'] = self.get_iter(runner) + self.run[tag_name].log(tags) + + @master_only + def after_run(self, runner): + self.run.stop() diff --git a/annotator/uniformer/mmcv/runner/hooks/logger/pavi.py b/annotator/uniformer/mmcv/runner/hooks/logger/pavi.py new file mode 100644 index 0000000000000000000000000000000000000000..1dcf146d8163aff1363e9764999b0a74d674a595 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/logger/pavi.py @@ -0,0 +1,117 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import json +import os +import os.path as osp + +import torch +import yaml + +import annotator.uniformer.mmcv as mmcv +from ....parallel.utils import is_module_wrapper +from ...dist_utils import master_only +from ..hook import HOOKS +from .base import LoggerHook + + +@HOOKS.register_module() +class PaviLoggerHook(LoggerHook): + + def __init__(self, + init_kwargs=None, + add_graph=False, + add_last_ckpt=False, + interval=10, + ignore_last=True, + reset_flag=False, + by_epoch=True, + img_key='img_info'): + super(PaviLoggerHook, self).__init__(interval, ignore_last, reset_flag, + by_epoch) + self.init_kwargs = init_kwargs + self.add_graph = add_graph + self.add_last_ckpt = add_last_ckpt + self.img_key = img_key + + @master_only + def before_run(self, runner): + super(PaviLoggerHook, self).before_run(runner) + try: + from pavi import SummaryWriter + except ImportError: + raise ImportError('Please run "pip install pavi" to install pavi.') + + self.run_name = runner.work_dir.split('/')[-1] + + if not self.init_kwargs: + self.init_kwargs = dict() + self.init_kwargs['name'] = self.run_name + self.init_kwargs['model'] = runner._model_name + if runner.meta is not None: + if 'config_dict' in runner.meta: + config_dict = runner.meta['config_dict'] + assert isinstance( + config_dict, + dict), ('meta["config_dict"] has to be of a dict, ' + f'but got {type(config_dict)}') + elif 'config_file' in runner.meta: + config_file = runner.meta['config_file'] + config_dict = dict(mmcv.Config.fromfile(config_file)) + else: + config_dict = None + if config_dict is not None: + # 'max_.*iter' is parsed in pavi sdk as the maximum iterations + # to properly set up the progress bar. + config_dict = config_dict.copy() + config_dict.setdefault('max_iter', runner.max_iters) + # non-serializable values are first converted in + # mmcv.dump to json + config_dict = json.loads( + mmcv.dump(config_dict, file_format='json')) + session_text = yaml.dump(config_dict) + self.init_kwargs['session_text'] = session_text + self.writer = SummaryWriter(**self.init_kwargs) + + def get_step(self, runner): + """Get the total training step/epoch.""" + if self.get_mode(runner) == 'val' and self.by_epoch: + return self.get_epoch(runner) + else: + return self.get_iter(runner) + + @master_only + def log(self, runner): + tags = self.get_loggable_tags(runner, add_mode=False) + if tags: + self.writer.add_scalars( + self.get_mode(runner), tags, self.get_step(runner)) + + @master_only + def after_run(self, runner): + if self.add_last_ckpt: + ckpt_path = osp.join(runner.work_dir, 'latest.pth') + if osp.islink(ckpt_path): + ckpt_path = osp.join(runner.work_dir, os.readlink(ckpt_path)) + + if osp.isfile(ckpt_path): + # runner.epoch += 1 has been done before `after_run`. + iteration = runner.epoch if self.by_epoch else runner.iter + return self.writer.add_snapshot_file( + tag=self.run_name, + snapshot_file_path=ckpt_path, + iteration=iteration) + + # flush the buffer and send a task ending signal to Pavi + self.writer.close() + + @master_only + def before_epoch(self, runner): + if runner.epoch == 0 and self.add_graph: + if is_module_wrapper(runner.model): + _model = runner.model.module + else: + _model = runner.model + device = next(_model.parameters()).device + data = next(iter(runner.data_loader)) + image = data[self.img_key][0:1].to(device) + with torch.no_grad(): + self.writer.add_graph(_model, image) diff --git a/annotator/uniformer/mmcv/runner/hooks/logger/tensorboard.py b/annotator/uniformer/mmcv/runner/hooks/logger/tensorboard.py new file mode 100644 index 0000000000000000000000000000000000000000..4dd5011dc08def6c09eef86d3ce5b124c9fc5372 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/logger/tensorboard.py @@ -0,0 +1,57 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os.path as osp + +from annotator.uniformer.mmcv.utils import TORCH_VERSION, digit_version +from ...dist_utils import master_only +from ..hook import HOOKS +from .base import LoggerHook + + +@HOOKS.register_module() +class TensorboardLoggerHook(LoggerHook): + + def __init__(self, + log_dir=None, + interval=10, + ignore_last=True, + reset_flag=False, + by_epoch=True): + super(TensorboardLoggerHook, self).__init__(interval, ignore_last, + reset_flag, by_epoch) + self.log_dir = log_dir + + @master_only + def before_run(self, runner): + super(TensorboardLoggerHook, self).before_run(runner) + if (TORCH_VERSION == 'parrots' + or digit_version(TORCH_VERSION) < digit_version('1.1')): + try: + from tensorboardX import SummaryWriter + except ImportError: + raise ImportError('Please install tensorboardX to use ' + 'TensorboardLoggerHook.') + else: + try: + from torch.utils.tensorboard import SummaryWriter + except ImportError: + raise ImportError( + 'Please run "pip install future tensorboard" to install ' + 'the dependencies to use torch.utils.tensorboard ' + '(applicable to PyTorch 1.1 or higher)') + + if self.log_dir is None: + self.log_dir = osp.join(runner.work_dir, 'tf_logs') + self.writer = SummaryWriter(self.log_dir) + + @master_only + def log(self, runner): + tags = self.get_loggable_tags(runner, allow_text=True) + for tag, val in tags.items(): + if isinstance(val, str): + self.writer.add_text(tag, val, self.get_iter(runner)) + else: + self.writer.add_scalar(tag, val, self.get_iter(runner)) + + @master_only + def after_run(self, runner): + self.writer.close() diff --git a/annotator/uniformer/mmcv/runner/hooks/logger/text.py b/annotator/uniformer/mmcv/runner/hooks/logger/text.py new file mode 100644 index 0000000000000000000000000000000000000000..87b1a3eca9595a130121526f8b4c29915387ab35 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/logger/text.py @@ -0,0 +1,256 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import datetime +import os +import os.path as osp +from collections import OrderedDict + +import torch +import torch.distributed as dist + +import annotator.uniformer.mmcv as mmcv +from annotator.uniformer.mmcv.fileio.file_client import FileClient +from annotator.uniformer.mmcv.utils import is_tuple_of, scandir +from ..hook import HOOKS +from .base import LoggerHook + + +@HOOKS.register_module() +class TextLoggerHook(LoggerHook): + """Logger hook in text. + + In this logger hook, the information will be printed on terminal and + saved in json file. + + Args: + by_epoch (bool, optional): Whether EpochBasedRunner is used. + Default: True. + interval (int, optional): Logging interval (every k iterations). + Default: 10. + ignore_last (bool, optional): Ignore the log of last iterations in each + epoch if less than :attr:`interval`. Default: True. + reset_flag (bool, optional): Whether to clear the output buffer after + logging. Default: False. + interval_exp_name (int, optional): Logging interval for experiment + name. This feature is to help users conveniently get the experiment + information from screen or log file. Default: 1000. + out_dir (str, optional): Logs are saved in ``runner.work_dir`` default. + If ``out_dir`` is specified, logs will be copied to a new directory + which is the concatenation of ``out_dir`` and the last level + directory of ``runner.work_dir``. Default: None. + `New in version 1.3.16.` + out_suffix (str or tuple[str], optional): Those filenames ending with + ``out_suffix`` will be copied to ``out_dir``. + Default: ('.log.json', '.log', '.py'). + `New in version 1.3.16.` + keep_local (bool, optional): Whether to keep local log when + :attr:`out_dir` is specified. If False, the local log will be + removed. Default: True. + `New in version 1.3.16.` + file_client_args (dict, optional): Arguments to instantiate a + FileClient. See :class:`mmcv.fileio.FileClient` for details. + Default: None. + `New in version 1.3.16.` + """ + + def __init__(self, + by_epoch=True, + interval=10, + ignore_last=True, + reset_flag=False, + interval_exp_name=1000, + out_dir=None, + out_suffix=('.log.json', '.log', '.py'), + keep_local=True, + file_client_args=None): + super(TextLoggerHook, self).__init__(interval, ignore_last, reset_flag, + by_epoch) + self.by_epoch = by_epoch + self.time_sec_tot = 0 + self.interval_exp_name = interval_exp_name + + if out_dir is None and file_client_args is not None: + raise ValueError( + 'file_client_args should be "None" when `out_dir` is not' + 'specified.') + self.out_dir = out_dir + + if not (out_dir is None or isinstance(out_dir, str) + or is_tuple_of(out_dir, str)): + raise TypeError('out_dir should be "None" or string or tuple of ' + 'string, but got {out_dir}') + self.out_suffix = out_suffix + + self.keep_local = keep_local + self.file_client_args = file_client_args + if self.out_dir is not None: + self.file_client = FileClient.infer_client(file_client_args, + self.out_dir) + + def before_run(self, runner): + super(TextLoggerHook, self).before_run(runner) + + if self.out_dir is not None: + self.file_client = FileClient.infer_client(self.file_client_args, + self.out_dir) + # The final `self.out_dir` is the concatenation of `self.out_dir` + # and the last level directory of `runner.work_dir` + basename = osp.basename(runner.work_dir.rstrip(osp.sep)) + self.out_dir = self.file_client.join_path(self.out_dir, basename) + runner.logger.info( + (f'Text logs will be saved to {self.out_dir} by ' + f'{self.file_client.name} after the training process.')) + + self.start_iter = runner.iter + self.json_log_path = osp.join(runner.work_dir, + f'{runner.timestamp}.log.json') + if runner.meta is not None: + self._dump_log(runner.meta, runner) + + def _get_max_memory(self, runner): + device = getattr(runner.model, 'output_device', None) + mem = torch.cuda.max_memory_allocated(device=device) + mem_mb = torch.tensor([mem / (1024 * 1024)], + dtype=torch.int, + device=device) + if runner.world_size > 1: + dist.reduce(mem_mb, 0, op=dist.ReduceOp.MAX) + return mem_mb.item() + + def _log_info(self, log_dict, runner): + # print exp name for users to distinguish experiments + # at every ``interval_exp_name`` iterations and the end of each epoch + if runner.meta is not None and 'exp_name' in runner.meta: + if (self.every_n_iters(runner, self.interval_exp_name)) or ( + self.by_epoch and self.end_of_epoch(runner)): + exp_info = f'Exp name: {runner.meta["exp_name"]}' + runner.logger.info(exp_info) + + if log_dict['mode'] == 'train': + if isinstance(log_dict['lr'], dict): + lr_str = [] + for k, val in log_dict['lr'].items(): + lr_str.append(f'lr_{k}: {val:.3e}') + lr_str = ' '.join(lr_str) + else: + lr_str = f'lr: {log_dict["lr"]:.3e}' + + # by epoch: Epoch [4][100/1000] + # by iter: Iter [100/100000] + if self.by_epoch: + log_str = f'Epoch [{log_dict["epoch"]}]' \ + f'[{log_dict["iter"]}/{len(runner.data_loader)}]\t' + else: + log_str = f'Iter [{log_dict["iter"]}/{runner.max_iters}]\t' + log_str += f'{lr_str}, ' + + if 'time' in log_dict.keys(): + self.time_sec_tot += (log_dict['time'] * self.interval) + time_sec_avg = self.time_sec_tot / ( + runner.iter - self.start_iter + 1) + eta_sec = time_sec_avg * (runner.max_iters - runner.iter - 1) + eta_str = str(datetime.timedelta(seconds=int(eta_sec))) + log_str += f'eta: {eta_str}, ' + log_str += f'time: {log_dict["time"]:.3f}, ' \ + f'data_time: {log_dict["data_time"]:.3f}, ' + # statistic memory + if torch.cuda.is_available(): + log_str += f'memory: {log_dict["memory"]}, ' + else: + # val/test time + # here 1000 is the length of the val dataloader + # by epoch: Epoch[val] [4][1000] + # by iter: Iter[val] [1000] + if self.by_epoch: + log_str = f'Epoch({log_dict["mode"]}) ' \ + f'[{log_dict["epoch"]}][{log_dict["iter"]}]\t' + else: + log_str = f'Iter({log_dict["mode"]}) [{log_dict["iter"]}]\t' + + log_items = [] + for name, val in log_dict.items(): + # TODO: resolve this hack + # these items have been in log_str + if name in [ + 'mode', 'Epoch', 'iter', 'lr', 'time', 'data_time', + 'memory', 'epoch' + ]: + continue + if isinstance(val, float): + val = f'{val:.4f}' + log_items.append(f'{name}: {val}') + log_str += ', '.join(log_items) + + runner.logger.info(log_str) + + def _dump_log(self, log_dict, runner): + # dump log in json format + json_log = OrderedDict() + for k, v in log_dict.items(): + json_log[k] = self._round_float(v) + # only append log at last line + if runner.rank == 0: + with open(self.json_log_path, 'a+') as f: + mmcv.dump(json_log, f, file_format='json') + f.write('\n') + + def _round_float(self, items): + if isinstance(items, list): + return [self._round_float(item) for item in items] + elif isinstance(items, float): + return round(items, 5) + else: + return items + + def log(self, runner): + if 'eval_iter_num' in runner.log_buffer.output: + # this doesn't modify runner.iter and is regardless of by_epoch + cur_iter = runner.log_buffer.output.pop('eval_iter_num') + else: + cur_iter = self.get_iter(runner, inner_iter=True) + + log_dict = OrderedDict( + mode=self.get_mode(runner), + epoch=self.get_epoch(runner), + iter=cur_iter) + + # only record lr of the first param group + cur_lr = runner.current_lr() + if isinstance(cur_lr, list): + log_dict['lr'] = cur_lr[0] + else: + assert isinstance(cur_lr, dict) + log_dict['lr'] = {} + for k, lr_ in cur_lr.items(): + assert isinstance(lr_, list) + log_dict['lr'].update({k: lr_[0]}) + + if 'time' in runner.log_buffer.output: + # statistic memory + if torch.cuda.is_available(): + log_dict['memory'] = self._get_max_memory(runner) + + log_dict = dict(log_dict, **runner.log_buffer.output) + + self._log_info(log_dict, runner) + self._dump_log(log_dict, runner) + return log_dict + + def after_run(self, runner): + # copy or upload logs to self.out_dir + if self.out_dir is not None: + for filename in scandir(runner.work_dir, self.out_suffix, True): + local_filepath = osp.join(runner.work_dir, filename) + out_filepath = self.file_client.join_path( + self.out_dir, filename) + with open(local_filepath, 'r') as f: + self.file_client.put_text(f.read(), out_filepath) + + runner.logger.info( + (f'The file {local_filepath} has been uploaded to ' + f'{out_filepath}.')) + + if not self.keep_local: + os.remove(local_filepath) + runner.logger.info( + (f'{local_filepath} was removed due to the ' + '`self.keep_local=False`')) diff --git a/annotator/uniformer/mmcv/runner/hooks/logger/wandb.py b/annotator/uniformer/mmcv/runner/hooks/logger/wandb.py new file mode 100644 index 0000000000000000000000000000000000000000..9f6808462eb79ab2b04806a5d9f0d3dd079b5ea9 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/logger/wandb.py @@ -0,0 +1,56 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from ...dist_utils import master_only +from ..hook import HOOKS +from .base import LoggerHook + + +@HOOKS.register_module() +class WandbLoggerHook(LoggerHook): + + def __init__(self, + init_kwargs=None, + interval=10, + ignore_last=True, + reset_flag=False, + commit=True, + by_epoch=True, + with_step=True): + super(WandbLoggerHook, self).__init__(interval, ignore_last, + reset_flag, by_epoch) + self.import_wandb() + self.init_kwargs = init_kwargs + self.commit = commit + self.with_step = with_step + + def import_wandb(self): + try: + import wandb + except ImportError: + raise ImportError( + 'Please run "pip install wandb" to install wandb') + self.wandb = wandb + + @master_only + def before_run(self, runner): + super(WandbLoggerHook, self).before_run(runner) + if self.wandb is None: + self.import_wandb() + if self.init_kwargs: + self.wandb.init(**self.init_kwargs) + else: + self.wandb.init() + + @master_only + def log(self, runner): + tags = self.get_loggable_tags(runner) + if tags: + if self.with_step: + self.wandb.log( + tags, step=self.get_iter(runner), commit=self.commit) + else: + tags['global_step'] = self.get_iter(runner) + self.wandb.log(tags, commit=self.commit) + + @master_only + def after_run(self, runner): + self.wandb.join() diff --git a/annotator/uniformer/mmcv/runner/hooks/lr_updater.py b/annotator/uniformer/mmcv/runner/hooks/lr_updater.py new file mode 100644 index 0000000000000000000000000000000000000000..6365908ddf6070086de2ffc0afada46ed2f32256 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/lr_updater.py @@ -0,0 +1,670 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import numbers +from math import cos, pi + +import annotator.uniformer.mmcv as mmcv +from .hook import HOOKS, Hook + + +class LrUpdaterHook(Hook): + """LR Scheduler in MMCV. + + Args: + by_epoch (bool): LR changes epoch by epoch + warmup (string): Type of warmup used. It can be None(use no warmup), + 'constant', 'linear' or 'exp' + warmup_iters (int): The number of iterations or epochs that warmup + lasts + warmup_ratio (float): LR used at the beginning of warmup equals to + warmup_ratio * initial_lr + warmup_by_epoch (bool): When warmup_by_epoch == True, warmup_iters + means the number of epochs that warmup lasts, otherwise means the + number of iteration that warmup lasts + """ + + def __init__(self, + by_epoch=True, + warmup=None, + warmup_iters=0, + warmup_ratio=0.1, + warmup_by_epoch=False): + # validate the "warmup" argument + if warmup is not None: + if warmup not in ['constant', 'linear', 'exp']: + raise ValueError( + f'"{warmup}" is not a supported type for warming up, valid' + ' types are "constant" and "linear"') + if warmup is not None: + assert warmup_iters > 0, \ + '"warmup_iters" must be a positive integer' + assert 0 < warmup_ratio <= 1.0, \ + '"warmup_ratio" must be in range (0,1]' + + self.by_epoch = by_epoch + self.warmup = warmup + self.warmup_iters = warmup_iters + self.warmup_ratio = warmup_ratio + self.warmup_by_epoch = warmup_by_epoch + + if self.warmup_by_epoch: + self.warmup_epochs = self.warmup_iters + self.warmup_iters = None + else: + self.warmup_epochs = None + + self.base_lr = [] # initial lr for all param groups + self.regular_lr = [] # expected lr if no warming up is performed + + def _set_lr(self, runner, lr_groups): + if isinstance(runner.optimizer, dict): + for k, optim in runner.optimizer.items(): + for param_group, lr in zip(optim.param_groups, lr_groups[k]): + param_group['lr'] = lr + else: + for param_group, lr in zip(runner.optimizer.param_groups, + lr_groups): + param_group['lr'] = lr + + def get_lr(self, runner, base_lr): + raise NotImplementedError + + def get_regular_lr(self, runner): + if isinstance(runner.optimizer, dict): + lr_groups = {} + for k in runner.optimizer.keys(): + _lr_group = [ + self.get_lr(runner, _base_lr) + for _base_lr in self.base_lr[k] + ] + lr_groups.update({k: _lr_group}) + + return lr_groups + else: + return [self.get_lr(runner, _base_lr) for _base_lr in self.base_lr] + + def get_warmup_lr(self, cur_iters): + + def _get_warmup_lr(cur_iters, regular_lr): + if self.warmup == 'constant': + warmup_lr = [_lr * self.warmup_ratio for _lr in regular_lr] + elif self.warmup == 'linear': + k = (1 - cur_iters / self.warmup_iters) * (1 - + self.warmup_ratio) + warmup_lr = [_lr * (1 - k) for _lr in regular_lr] + elif self.warmup == 'exp': + k = self.warmup_ratio**(1 - cur_iters / self.warmup_iters) + warmup_lr = [_lr * k for _lr in regular_lr] + return warmup_lr + + if isinstance(self.regular_lr, dict): + lr_groups = {} + for key, regular_lr in self.regular_lr.items(): + lr_groups[key] = _get_warmup_lr(cur_iters, regular_lr) + return lr_groups + else: + return _get_warmup_lr(cur_iters, self.regular_lr) + + def before_run(self, runner): + # NOTE: when resuming from a checkpoint, if 'initial_lr' is not saved, + # it will be set according to the optimizer params + if isinstance(runner.optimizer, dict): + self.base_lr = {} + for k, optim in runner.optimizer.items(): + for group in optim.param_groups: + group.setdefault('initial_lr', group['lr']) + _base_lr = [ + group['initial_lr'] for group in optim.param_groups + ] + self.base_lr.update({k: _base_lr}) + else: + for group in runner.optimizer.param_groups: + group.setdefault('initial_lr', group['lr']) + self.base_lr = [ + group['initial_lr'] for group in runner.optimizer.param_groups + ] + + def before_train_epoch(self, runner): + if self.warmup_iters is None: + epoch_len = len(runner.data_loader) + self.warmup_iters = self.warmup_epochs * epoch_len + + if not self.by_epoch: + return + + self.regular_lr = self.get_regular_lr(runner) + self._set_lr(runner, self.regular_lr) + + def before_train_iter(self, runner): + cur_iter = runner.iter + if not self.by_epoch: + self.regular_lr = self.get_regular_lr(runner) + if self.warmup is None or cur_iter >= self.warmup_iters: + self._set_lr(runner, self.regular_lr) + else: + warmup_lr = self.get_warmup_lr(cur_iter) + self._set_lr(runner, warmup_lr) + elif self.by_epoch: + if self.warmup is None or cur_iter > self.warmup_iters: + return + elif cur_iter == self.warmup_iters: + self._set_lr(runner, self.regular_lr) + else: + warmup_lr = self.get_warmup_lr(cur_iter) + self._set_lr(runner, warmup_lr) + + +@HOOKS.register_module() +class FixedLrUpdaterHook(LrUpdaterHook): + + def __init__(self, **kwargs): + super(FixedLrUpdaterHook, self).__init__(**kwargs) + + def get_lr(self, runner, base_lr): + return base_lr + + +@HOOKS.register_module() +class StepLrUpdaterHook(LrUpdaterHook): + """Step LR scheduler with min_lr clipping. + + Args: + step (int | list[int]): Step to decay the LR. If an int value is given, + regard it as the decay interval. If a list is given, decay LR at + these steps. + gamma (float, optional): Decay LR ratio. Default: 0.1. + min_lr (float, optional): Minimum LR value to keep. If LR after decay + is lower than `min_lr`, it will be clipped to this value. If None + is given, we don't perform lr clipping. Default: None. + """ + + def __init__(self, step, gamma=0.1, min_lr=None, **kwargs): + if isinstance(step, list): + assert mmcv.is_list_of(step, int) + assert all([s > 0 for s in step]) + elif isinstance(step, int): + assert step > 0 + else: + raise TypeError('"step" must be a list or integer') + self.step = step + self.gamma = gamma + self.min_lr = min_lr + super(StepLrUpdaterHook, self).__init__(**kwargs) + + def get_lr(self, runner, base_lr): + progress = runner.epoch if self.by_epoch else runner.iter + + # calculate exponential term + if isinstance(self.step, int): + exp = progress // self.step + else: + exp = len(self.step) + for i, s in enumerate(self.step): + if progress < s: + exp = i + break + + lr = base_lr * (self.gamma**exp) + if self.min_lr is not None: + # clip to a minimum value + lr = max(lr, self.min_lr) + return lr + + +@HOOKS.register_module() +class ExpLrUpdaterHook(LrUpdaterHook): + + def __init__(self, gamma, **kwargs): + self.gamma = gamma + super(ExpLrUpdaterHook, self).__init__(**kwargs) + + def get_lr(self, runner, base_lr): + progress = runner.epoch if self.by_epoch else runner.iter + return base_lr * self.gamma**progress + + +@HOOKS.register_module() +class PolyLrUpdaterHook(LrUpdaterHook): + + def __init__(self, power=1., min_lr=0., **kwargs): + self.power = power + self.min_lr = min_lr + super(PolyLrUpdaterHook, self).__init__(**kwargs) + + def get_lr(self, runner, base_lr): + if self.by_epoch: + progress = runner.epoch + max_progress = runner.max_epochs + else: + progress = runner.iter + max_progress = runner.max_iters + coeff = (1 - progress / max_progress)**self.power + return (base_lr - self.min_lr) * coeff + self.min_lr + + +@HOOKS.register_module() +class InvLrUpdaterHook(LrUpdaterHook): + + def __init__(self, gamma, power=1., **kwargs): + self.gamma = gamma + self.power = power + super(InvLrUpdaterHook, self).__init__(**kwargs) + + def get_lr(self, runner, base_lr): + progress = runner.epoch if self.by_epoch else runner.iter + return base_lr * (1 + self.gamma * progress)**(-self.power) + + +@HOOKS.register_module() +class CosineAnnealingLrUpdaterHook(LrUpdaterHook): + + def __init__(self, min_lr=None, min_lr_ratio=None, **kwargs): + assert (min_lr is None) ^ (min_lr_ratio is None) + self.min_lr = min_lr + self.min_lr_ratio = min_lr_ratio + super(CosineAnnealingLrUpdaterHook, self).__init__(**kwargs) + + def get_lr(self, runner, base_lr): + if self.by_epoch: + progress = runner.epoch + max_progress = runner.max_epochs + else: + progress = runner.iter + max_progress = runner.max_iters + + if self.min_lr_ratio is not None: + target_lr = base_lr * self.min_lr_ratio + else: + target_lr = self.min_lr + return annealing_cos(base_lr, target_lr, progress / max_progress) + + +@HOOKS.register_module() +class FlatCosineAnnealingLrUpdaterHook(LrUpdaterHook): + """Flat + Cosine lr schedule. + + Modified from https://github.com/fastai/fastai/blob/master/fastai/callback/schedule.py#L128 # noqa: E501 + + Args: + start_percent (float): When to start annealing the learning rate + after the percentage of the total training steps. + The value should be in range [0, 1). + Default: 0.75 + min_lr (float, optional): The minimum lr. Default: None. + min_lr_ratio (float, optional): The ratio of minimum lr to the base lr. + Either `min_lr` or `min_lr_ratio` should be specified. + Default: None. + """ + + def __init__(self, + start_percent=0.75, + min_lr=None, + min_lr_ratio=None, + **kwargs): + assert (min_lr is None) ^ (min_lr_ratio is None) + if start_percent < 0 or start_percent > 1 or not isinstance( + start_percent, float): + raise ValueError( + 'expected float between 0 and 1 start_percent, but ' + f'got {start_percent}') + self.start_percent = start_percent + self.min_lr = min_lr + self.min_lr_ratio = min_lr_ratio + super(FlatCosineAnnealingLrUpdaterHook, self).__init__(**kwargs) + + def get_lr(self, runner, base_lr): + if self.by_epoch: + start = round(runner.max_epochs * self.start_percent) + progress = runner.epoch - start + max_progress = runner.max_epochs - start + else: + start = round(runner.max_iters * self.start_percent) + progress = runner.iter - start + max_progress = runner.max_iters - start + + if self.min_lr_ratio is not None: + target_lr = base_lr * self.min_lr_ratio + else: + target_lr = self.min_lr + + if progress < 0: + return base_lr + else: + return annealing_cos(base_lr, target_lr, progress / max_progress) + + +@HOOKS.register_module() +class CosineRestartLrUpdaterHook(LrUpdaterHook): + """Cosine annealing with restarts learning rate scheme. + + Args: + periods (list[int]): Periods for each cosine anneling cycle. + restart_weights (list[float], optional): Restart weights at each + restart iteration. Default: [1]. + min_lr (float, optional): The minimum lr. Default: None. + min_lr_ratio (float, optional): The ratio of minimum lr to the base lr. + Either `min_lr` or `min_lr_ratio` should be specified. + Default: None. + """ + + def __init__(self, + periods, + restart_weights=[1], + min_lr=None, + min_lr_ratio=None, + **kwargs): + assert (min_lr is None) ^ (min_lr_ratio is None) + self.periods = periods + self.min_lr = min_lr + self.min_lr_ratio = min_lr_ratio + self.restart_weights = restart_weights + assert (len(self.periods) == len(self.restart_weights) + ), 'periods and restart_weights should have the same length.' + super(CosineRestartLrUpdaterHook, self).__init__(**kwargs) + + self.cumulative_periods = [ + sum(self.periods[0:i + 1]) for i in range(0, len(self.periods)) + ] + + def get_lr(self, runner, base_lr): + if self.by_epoch: + progress = runner.epoch + else: + progress = runner.iter + + if self.min_lr_ratio is not None: + target_lr = base_lr * self.min_lr_ratio + else: + target_lr = self.min_lr + + idx = get_position_from_periods(progress, self.cumulative_periods) + current_weight = self.restart_weights[idx] + nearest_restart = 0 if idx == 0 else self.cumulative_periods[idx - 1] + current_periods = self.periods[idx] + + alpha = min((progress - nearest_restart) / current_periods, 1) + return annealing_cos(base_lr, target_lr, alpha, current_weight) + + +def get_position_from_periods(iteration, cumulative_periods): + """Get the position from a period list. + + It will return the index of the right-closest number in the period list. + For example, the cumulative_periods = [100, 200, 300, 400], + if iteration == 50, return 0; + if iteration == 210, return 2; + if iteration == 300, return 3. + + Args: + iteration (int): Current iteration. + cumulative_periods (list[int]): Cumulative period list. + + Returns: + int: The position of the right-closest number in the period list. + """ + for i, period in enumerate(cumulative_periods): + if iteration < period: + return i + raise ValueError(f'Current iteration {iteration} exceeds ' + f'cumulative_periods {cumulative_periods}') + + +@HOOKS.register_module() +class CyclicLrUpdaterHook(LrUpdaterHook): + """Cyclic LR Scheduler. + + Implement the cyclical learning rate policy (CLR) described in + https://arxiv.org/pdf/1506.01186.pdf + + Different from the original paper, we use cosine annealing rather than + triangular policy inside a cycle. This improves the performance in the + 3D detection area. + + Args: + by_epoch (bool): Whether to update LR by epoch. + target_ratio (tuple[float]): Relative ratio of the highest LR and the + lowest LR to the initial LR. + cyclic_times (int): Number of cycles during training + step_ratio_up (float): The ratio of the increasing process of LR in + the total cycle. + anneal_strategy (str): {'cos', 'linear'} + Specifies the annealing strategy: 'cos' for cosine annealing, + 'linear' for linear annealing. Default: 'cos'. + """ + + def __init__(self, + by_epoch=False, + target_ratio=(10, 1e-4), + cyclic_times=1, + step_ratio_up=0.4, + anneal_strategy='cos', + **kwargs): + if isinstance(target_ratio, float): + target_ratio = (target_ratio, target_ratio / 1e5) + elif isinstance(target_ratio, tuple): + target_ratio = (target_ratio[0], target_ratio[0] / 1e5) \ + if len(target_ratio) == 1 else target_ratio + else: + raise ValueError('target_ratio should be either float ' + f'or tuple, got {type(target_ratio)}') + + assert len(target_ratio) == 2, \ + '"target_ratio" must be list or tuple of two floats' + assert 0 <= step_ratio_up < 1.0, \ + '"step_ratio_up" must be in range [0,1)' + + self.target_ratio = target_ratio + self.cyclic_times = cyclic_times + self.step_ratio_up = step_ratio_up + self.lr_phases = [] # init lr_phases + # validate anneal_strategy + if anneal_strategy not in ['cos', 'linear']: + raise ValueError('anneal_strategy must be one of "cos" or ' + f'"linear", instead got {anneal_strategy}') + elif anneal_strategy == 'cos': + self.anneal_func = annealing_cos + elif anneal_strategy == 'linear': + self.anneal_func = annealing_linear + + assert not by_epoch, \ + 'currently only support "by_epoch" = False' + super(CyclicLrUpdaterHook, self).__init__(by_epoch, **kwargs) + + def before_run(self, runner): + super(CyclicLrUpdaterHook, self).before_run(runner) + # initiate lr_phases + # total lr_phases are separated as up and down + max_iter_per_phase = runner.max_iters // self.cyclic_times + iter_up_phase = int(self.step_ratio_up * max_iter_per_phase) + self.lr_phases.append( + [0, iter_up_phase, max_iter_per_phase, 1, self.target_ratio[0]]) + self.lr_phases.append([ + iter_up_phase, max_iter_per_phase, max_iter_per_phase, + self.target_ratio[0], self.target_ratio[1] + ]) + + def get_lr(self, runner, base_lr): + curr_iter = runner.iter + for (start_iter, end_iter, max_iter_per_phase, start_ratio, + end_ratio) in self.lr_phases: + curr_iter %= max_iter_per_phase + if start_iter <= curr_iter < end_iter: + progress = curr_iter - start_iter + return self.anneal_func(base_lr * start_ratio, + base_lr * end_ratio, + progress / (end_iter - start_iter)) + + +@HOOKS.register_module() +class OneCycleLrUpdaterHook(LrUpdaterHook): + """One Cycle LR Scheduler. + + The 1cycle learning rate policy changes the learning rate after every + batch. The one cycle learning rate policy is described in + https://arxiv.org/pdf/1708.07120.pdf + + Args: + max_lr (float or list): Upper learning rate boundaries in the cycle + for each parameter group. + total_steps (int, optional): The total number of steps in the cycle. + Note that if a value is not provided here, it will be the max_iter + of runner. Default: None. + pct_start (float): The percentage of the cycle (in number of steps) + spent increasing the learning rate. + Default: 0.3 + anneal_strategy (str): {'cos', 'linear'} + Specifies the annealing strategy: 'cos' for cosine annealing, + 'linear' for linear annealing. + Default: 'cos' + div_factor (float): Determines the initial learning rate via + initial_lr = max_lr/div_factor + Default: 25 + final_div_factor (float): Determines the minimum learning rate via + min_lr = initial_lr/final_div_factor + Default: 1e4 + three_phase (bool): If three_phase is True, use a third phase of the + schedule to annihilate the learning rate according to + final_div_factor instead of modifying the second phase (the first + two phases will be symmetrical about the step indicated by + pct_start). + Default: False + """ + + def __init__(self, + max_lr, + total_steps=None, + pct_start=0.3, + anneal_strategy='cos', + div_factor=25, + final_div_factor=1e4, + three_phase=False, + **kwargs): + # validate by_epoch, currently only support by_epoch = False + if 'by_epoch' not in kwargs: + kwargs['by_epoch'] = False + else: + assert not kwargs['by_epoch'], \ + 'currently only support "by_epoch" = False' + if not isinstance(max_lr, (numbers.Number, list, dict)): + raise ValueError('the type of max_lr must be the one of list or ' + f'dict, but got {type(max_lr)}') + self._max_lr = max_lr + if total_steps is not None: + if not isinstance(total_steps, int): + raise ValueError('the type of total_steps must be int, but' + f'got {type(total_steps)}') + self.total_steps = total_steps + # validate pct_start + if pct_start < 0 or pct_start > 1 or not isinstance(pct_start, float): + raise ValueError('expected float between 0 and 1 pct_start, but ' + f'got {pct_start}') + self.pct_start = pct_start + # validate anneal_strategy + if anneal_strategy not in ['cos', 'linear']: + raise ValueError('anneal_strategy must be one of "cos" or ' + f'"linear", instead got {anneal_strategy}') + elif anneal_strategy == 'cos': + self.anneal_func = annealing_cos + elif anneal_strategy == 'linear': + self.anneal_func = annealing_linear + self.div_factor = div_factor + self.final_div_factor = final_div_factor + self.three_phase = three_phase + self.lr_phases = [] # init lr_phases + super(OneCycleLrUpdaterHook, self).__init__(**kwargs) + + def before_run(self, runner): + if hasattr(self, 'total_steps'): + total_steps = self.total_steps + else: + total_steps = runner.max_iters + if total_steps < runner.max_iters: + raise ValueError( + 'The total steps must be greater than or equal to max ' + f'iterations {runner.max_iters} of runner, but total steps ' + f'is {total_steps}.') + + if isinstance(runner.optimizer, dict): + self.base_lr = {} + for k, optim in runner.optimizer.items(): + _max_lr = format_param(k, optim, self._max_lr) + self.base_lr[k] = [lr / self.div_factor for lr in _max_lr] + for group, lr in zip(optim.param_groups, self.base_lr[k]): + group.setdefault('initial_lr', lr) + else: + k = type(runner.optimizer).__name__ + _max_lr = format_param(k, runner.optimizer, self._max_lr) + self.base_lr = [lr / self.div_factor for lr in _max_lr] + for group, lr in zip(runner.optimizer.param_groups, self.base_lr): + group.setdefault('initial_lr', lr) + + if self.three_phase: + self.lr_phases.append( + [float(self.pct_start * total_steps) - 1, 1, self.div_factor]) + self.lr_phases.append([ + float(2 * self.pct_start * total_steps) - 2, self.div_factor, 1 + ]) + self.lr_phases.append( + [total_steps - 1, 1, 1 / self.final_div_factor]) + else: + self.lr_phases.append( + [float(self.pct_start * total_steps) - 1, 1, self.div_factor]) + self.lr_phases.append( + [total_steps - 1, self.div_factor, 1 / self.final_div_factor]) + + def get_lr(self, runner, base_lr): + curr_iter = runner.iter + start_iter = 0 + for i, (end_iter, start_lr, end_lr) in enumerate(self.lr_phases): + if curr_iter <= end_iter: + pct = (curr_iter - start_iter) / (end_iter - start_iter) + lr = self.anneal_func(base_lr * start_lr, base_lr * end_lr, + pct) + break + start_iter = end_iter + return lr + + +def annealing_cos(start, end, factor, weight=1): + """Calculate annealing cos learning rate. + + Cosine anneal from `weight * start + (1 - weight) * end` to `end` as + percentage goes from 0.0 to 1.0. + + Args: + start (float): The starting learning rate of the cosine annealing. + end (float): The ending learing rate of the cosine annealing. + factor (float): The coefficient of `pi` when calculating the current + percentage. Range from 0.0 to 1.0. + weight (float, optional): The combination factor of `start` and `end` + when calculating the actual starting learning rate. Default to 1. + """ + cos_out = cos(pi * factor) + 1 + return end + 0.5 * weight * (start - end) * cos_out + + +def annealing_linear(start, end, factor): + """Calculate annealing linear learning rate. + + Linear anneal from `start` to `end` as percentage goes from 0.0 to 1.0. + + Args: + start (float): The starting learning rate of the linear annealing. + end (float): The ending learing rate of the linear annealing. + factor (float): The coefficient of `pi` when calculating the current + percentage. Range from 0.0 to 1.0. + """ + return start + (end - start) * factor + + +def format_param(name, optim, param): + if isinstance(param, numbers.Number): + return [param] * len(optim.param_groups) + elif isinstance(param, (list, tuple)): # multi param groups + if len(param) != len(optim.param_groups): + raise ValueError(f'expected {len(optim.param_groups)} ' + f'values for {name}, got {len(param)}') + return param + else: # multi optimizers + if name not in param: + raise KeyError(f'{name} is not found in {param.keys()}') + return param[name] diff --git a/annotator/uniformer/mmcv/runner/hooks/memory.py b/annotator/uniformer/mmcv/runner/hooks/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..70cf9a838fb314e3bd3c07aadbc00921a81e83ed --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/memory.py @@ -0,0 +1,25 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import torch + +from .hook import HOOKS, Hook + + +@HOOKS.register_module() +class EmptyCacheHook(Hook): + + def __init__(self, before_epoch=False, after_epoch=True, after_iter=False): + self._before_epoch = before_epoch + self._after_epoch = after_epoch + self._after_iter = after_iter + + def after_iter(self, runner): + if self._after_iter: + torch.cuda.empty_cache() + + def before_epoch(self, runner): + if self._before_epoch: + torch.cuda.empty_cache() + + def after_epoch(self, runner): + if self._after_epoch: + torch.cuda.empty_cache() diff --git a/annotator/uniformer/mmcv/runner/hooks/momentum_updater.py b/annotator/uniformer/mmcv/runner/hooks/momentum_updater.py new file mode 100644 index 0000000000000000000000000000000000000000..60437756ceedf06055ec349df69a25465738d3f0 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/momentum_updater.py @@ -0,0 +1,493 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import annotator.uniformer.mmcv as mmcv +from .hook import HOOKS, Hook +from .lr_updater import annealing_cos, annealing_linear, format_param + + +class MomentumUpdaterHook(Hook): + + def __init__(self, + by_epoch=True, + warmup=None, + warmup_iters=0, + warmup_ratio=0.9): + # validate the "warmup" argument + if warmup is not None: + if warmup not in ['constant', 'linear', 'exp']: + raise ValueError( + f'"{warmup}" is not a supported type for warming up, valid' + ' types are "constant" and "linear"') + if warmup is not None: + assert warmup_iters > 0, \ + '"warmup_iters" must be a positive integer' + assert 0 < warmup_ratio <= 1.0, \ + '"warmup_momentum" must be in range (0,1]' + + self.by_epoch = by_epoch + self.warmup = warmup + self.warmup_iters = warmup_iters + self.warmup_ratio = warmup_ratio + + self.base_momentum = [] # initial momentum for all param groups + self.regular_momentum = [ + ] # expected momentum if no warming up is performed + + def _set_momentum(self, runner, momentum_groups): + if isinstance(runner.optimizer, dict): + for k, optim in runner.optimizer.items(): + for param_group, mom in zip(optim.param_groups, + momentum_groups[k]): + if 'momentum' in param_group.keys(): + param_group['momentum'] = mom + elif 'betas' in param_group.keys(): + param_group['betas'] = (mom, param_group['betas'][1]) + else: + for param_group, mom in zip(runner.optimizer.param_groups, + momentum_groups): + if 'momentum' in param_group.keys(): + param_group['momentum'] = mom + elif 'betas' in param_group.keys(): + param_group['betas'] = (mom, param_group['betas'][1]) + + def get_momentum(self, runner, base_momentum): + raise NotImplementedError + + def get_regular_momentum(self, runner): + if isinstance(runner.optimizer, dict): + momentum_groups = {} + for k in runner.optimizer.keys(): + _momentum_group = [ + self.get_momentum(runner, _base_momentum) + for _base_momentum in self.base_momentum[k] + ] + momentum_groups.update({k: _momentum_group}) + return momentum_groups + else: + return [ + self.get_momentum(runner, _base_momentum) + for _base_momentum in self.base_momentum + ] + + def get_warmup_momentum(self, cur_iters): + + def _get_warmup_momentum(cur_iters, regular_momentum): + if self.warmup == 'constant': + warmup_momentum = [ + _momentum / self.warmup_ratio + for _momentum in self.regular_momentum + ] + elif self.warmup == 'linear': + k = (1 - cur_iters / self.warmup_iters) * (1 - + self.warmup_ratio) + warmup_momentum = [ + _momentum / (1 - k) for _momentum in self.regular_mom + ] + elif self.warmup == 'exp': + k = self.warmup_ratio**(1 - cur_iters / self.warmup_iters) + warmup_momentum = [ + _momentum / k for _momentum in self.regular_mom + ] + return warmup_momentum + + if isinstance(self.regular_momentum, dict): + momentum_groups = {} + for key, regular_momentum in self.regular_momentum.items(): + momentum_groups[key] = _get_warmup_momentum( + cur_iters, regular_momentum) + return momentum_groups + else: + return _get_warmup_momentum(cur_iters, self.regular_momentum) + + def before_run(self, runner): + # NOTE: when resuming from a checkpoint, + # if 'initial_momentum' is not saved, + # it will be set according to the optimizer params + if isinstance(runner.optimizer, dict): + self.base_momentum = {} + for k, optim in runner.optimizer.items(): + for group in optim.param_groups: + if 'momentum' in group.keys(): + group.setdefault('initial_momentum', group['momentum']) + else: + group.setdefault('initial_momentum', group['betas'][0]) + _base_momentum = [ + group['initial_momentum'] for group in optim.param_groups + ] + self.base_momentum.update({k: _base_momentum}) + else: + for group in runner.optimizer.param_groups: + if 'momentum' in group.keys(): + group.setdefault('initial_momentum', group['momentum']) + else: + group.setdefault('initial_momentum', group['betas'][0]) + self.base_momentum = [ + group['initial_momentum'] + for group in runner.optimizer.param_groups + ] + + def before_train_epoch(self, runner): + if not self.by_epoch: + return + self.regular_mom = self.get_regular_momentum(runner) + self._set_momentum(runner, self.regular_mom) + + def before_train_iter(self, runner): + cur_iter = runner.iter + if not self.by_epoch: + self.regular_mom = self.get_regular_momentum(runner) + if self.warmup is None or cur_iter >= self.warmup_iters: + self._set_momentum(runner, self.regular_mom) + else: + warmup_momentum = self.get_warmup_momentum(cur_iter) + self._set_momentum(runner, warmup_momentum) + elif self.by_epoch: + if self.warmup is None or cur_iter > self.warmup_iters: + return + elif cur_iter == self.warmup_iters: + self._set_momentum(runner, self.regular_mom) + else: + warmup_momentum = self.get_warmup_momentum(cur_iter) + self._set_momentum(runner, warmup_momentum) + + +@HOOKS.register_module() +class StepMomentumUpdaterHook(MomentumUpdaterHook): + """Step momentum scheduler with min value clipping. + + Args: + step (int | list[int]): Step to decay the momentum. If an int value is + given, regard it as the decay interval. If a list is given, decay + momentum at these steps. + gamma (float, optional): Decay momentum ratio. Default: 0.5. + min_momentum (float, optional): Minimum momentum value to keep. If + momentum after decay is lower than this value, it will be clipped + accordingly. If None is given, we don't perform lr clipping. + Default: None. + """ + + def __init__(self, step, gamma=0.5, min_momentum=None, **kwargs): + if isinstance(step, list): + assert mmcv.is_list_of(step, int) + assert all([s > 0 for s in step]) + elif isinstance(step, int): + assert step > 0 + else: + raise TypeError('"step" must be a list or integer') + self.step = step + self.gamma = gamma + self.min_momentum = min_momentum + super(StepMomentumUpdaterHook, self).__init__(**kwargs) + + def get_momentum(self, runner, base_momentum): + progress = runner.epoch if self.by_epoch else runner.iter + + # calculate exponential term + if isinstance(self.step, int): + exp = progress // self.step + else: + exp = len(self.step) + for i, s in enumerate(self.step): + if progress < s: + exp = i + break + + momentum = base_momentum * (self.gamma**exp) + if self.min_momentum is not None: + # clip to a minimum value + momentum = max(momentum, self.min_momentum) + return momentum + + +@HOOKS.register_module() +class CosineAnnealingMomentumUpdaterHook(MomentumUpdaterHook): + + def __init__(self, min_momentum=None, min_momentum_ratio=None, **kwargs): + assert (min_momentum is None) ^ (min_momentum_ratio is None) + self.min_momentum = min_momentum + self.min_momentum_ratio = min_momentum_ratio + super(CosineAnnealingMomentumUpdaterHook, self).__init__(**kwargs) + + def get_momentum(self, runner, base_momentum): + if self.by_epoch: + progress = runner.epoch + max_progress = runner.max_epochs + else: + progress = runner.iter + max_progress = runner.max_iters + if self.min_momentum_ratio is not None: + target_momentum = base_momentum * self.min_momentum_ratio + else: + target_momentum = self.min_momentum + return annealing_cos(base_momentum, target_momentum, + progress / max_progress) + + +@HOOKS.register_module() +class CyclicMomentumUpdaterHook(MomentumUpdaterHook): + """Cyclic momentum Scheduler. + + Implement the cyclical momentum scheduler policy described in + https://arxiv.org/pdf/1708.07120.pdf + + This momentum scheduler usually used together with the CyclicLRUpdater + to improve the performance in the 3D detection area. + + Attributes: + target_ratio (tuple[float]): Relative ratio of the lowest momentum and + the highest momentum to the initial momentum. + cyclic_times (int): Number of cycles during training + step_ratio_up (float): The ratio of the increasing process of momentum + in the total cycle. + by_epoch (bool): Whether to update momentum by epoch. + """ + + def __init__(self, + by_epoch=False, + target_ratio=(0.85 / 0.95, 1), + cyclic_times=1, + step_ratio_up=0.4, + **kwargs): + if isinstance(target_ratio, float): + target_ratio = (target_ratio, target_ratio / 1e5) + elif isinstance(target_ratio, tuple): + target_ratio = (target_ratio[0], target_ratio[0] / 1e5) \ + if len(target_ratio) == 1 else target_ratio + else: + raise ValueError('target_ratio should be either float ' + f'or tuple, got {type(target_ratio)}') + + assert len(target_ratio) == 2, \ + '"target_ratio" must be list or tuple of two floats' + assert 0 <= step_ratio_up < 1.0, \ + '"step_ratio_up" must be in range [0,1)' + + self.target_ratio = target_ratio + self.cyclic_times = cyclic_times + self.step_ratio_up = step_ratio_up + self.momentum_phases = [] # init momentum_phases + # currently only support by_epoch=False + assert not by_epoch, \ + 'currently only support "by_epoch" = False' + super(CyclicMomentumUpdaterHook, self).__init__(by_epoch, **kwargs) + + def before_run(self, runner): + super(CyclicMomentumUpdaterHook, self).before_run(runner) + # initiate momentum_phases + # total momentum_phases are separated as up and down + max_iter_per_phase = runner.max_iters // self.cyclic_times + iter_up_phase = int(self.step_ratio_up * max_iter_per_phase) + self.momentum_phases.append( + [0, iter_up_phase, max_iter_per_phase, 1, self.target_ratio[0]]) + self.momentum_phases.append([ + iter_up_phase, max_iter_per_phase, max_iter_per_phase, + self.target_ratio[0], self.target_ratio[1] + ]) + + def get_momentum(self, runner, base_momentum): + curr_iter = runner.iter + for (start_iter, end_iter, max_iter_per_phase, start_ratio, + end_ratio) in self.momentum_phases: + curr_iter %= max_iter_per_phase + if start_iter <= curr_iter < end_iter: + progress = curr_iter - start_iter + return annealing_cos(base_momentum * start_ratio, + base_momentum * end_ratio, + progress / (end_iter - start_iter)) + + +@HOOKS.register_module() +class OneCycleMomentumUpdaterHook(MomentumUpdaterHook): + """OneCycle momentum Scheduler. + + This momentum scheduler usually used together with the OneCycleLrUpdater + to improve the performance. + + Args: + base_momentum (float or list): Lower momentum boundaries in the cycle + for each parameter group. Note that momentum is cycled inversely + to learning rate; at the peak of a cycle, momentum is + 'base_momentum' and learning rate is 'max_lr'. + Default: 0.85 + max_momentum (float or list): Upper momentum boundaries in the cycle + for each parameter group. Functionally, + it defines the cycle amplitude (max_momentum - base_momentum). + Note that momentum is cycled inversely + to learning rate; at the start of a cycle, momentum is + 'max_momentum' and learning rate is 'base_lr' + Default: 0.95 + pct_start (float): The percentage of the cycle (in number of steps) + spent increasing the learning rate. + Default: 0.3 + anneal_strategy (str): {'cos', 'linear'} + Specifies the annealing strategy: 'cos' for cosine annealing, + 'linear' for linear annealing. + Default: 'cos' + three_phase (bool): If three_phase is True, use a third phase of the + schedule to annihilate the learning rate according to + final_div_factor instead of modifying the second phase (the first + two phases will be symmetrical about the step indicated by + pct_start). + Default: False + """ + + def __init__(self, + base_momentum=0.85, + max_momentum=0.95, + pct_start=0.3, + anneal_strategy='cos', + three_phase=False, + **kwargs): + # validate by_epoch, currently only support by_epoch=False + if 'by_epoch' not in kwargs: + kwargs['by_epoch'] = False + else: + assert not kwargs['by_epoch'], \ + 'currently only support "by_epoch" = False' + if not isinstance(base_momentum, (float, list, dict)): + raise ValueError('base_momentum must be the type among of float,' + 'list or dict.') + self._base_momentum = base_momentum + if not isinstance(max_momentum, (float, list, dict)): + raise ValueError('max_momentum must be the type among of float,' + 'list or dict.') + self._max_momentum = max_momentum + # validate pct_start + if pct_start < 0 or pct_start > 1 or not isinstance(pct_start, float): + raise ValueError('Expected float between 0 and 1 pct_start, but ' + f'got {pct_start}') + self.pct_start = pct_start + # validate anneal_strategy + if anneal_strategy not in ['cos', 'linear']: + raise ValueError('anneal_strategy must by one of "cos" or ' + f'"linear", instead got {anneal_strategy}') + elif anneal_strategy == 'cos': + self.anneal_func = annealing_cos + elif anneal_strategy == 'linear': + self.anneal_func = annealing_linear + self.three_phase = three_phase + self.momentum_phases = [] # init momentum_phases + super(OneCycleMomentumUpdaterHook, self).__init__(**kwargs) + + def before_run(self, runner): + if isinstance(runner.optimizer, dict): + for k, optim in runner.optimizer.items(): + if ('momentum' not in optim.defaults + and 'betas' not in optim.defaults): + raise ValueError('optimizer must support momentum with' + 'option enabled') + self.use_beta1 = 'betas' in optim.defaults + _base_momentum = format_param(k, optim, self._base_momentum) + _max_momentum = format_param(k, optim, self._max_momentum) + for group, b_momentum, m_momentum in zip( + optim.param_groups, _base_momentum, _max_momentum): + if self.use_beta1: + _, beta2 = group['betas'] + group['betas'] = (m_momentum, beta2) + else: + group['momentum'] = m_momentum + group['base_momentum'] = b_momentum + group['max_momentum'] = m_momentum + else: + optim = runner.optimizer + if ('momentum' not in optim.defaults + and 'betas' not in optim.defaults): + raise ValueError('optimizer must support momentum with' + 'option enabled') + self.use_beta1 = 'betas' in optim.defaults + k = type(optim).__name__ + _base_momentum = format_param(k, optim, self._base_momentum) + _max_momentum = format_param(k, optim, self._max_momentum) + for group, b_momentum, m_momentum in zip(optim.param_groups, + _base_momentum, + _max_momentum): + if self.use_beta1: + _, beta2 = group['betas'] + group['betas'] = (m_momentum, beta2) + else: + group['momentum'] = m_momentum + group['base_momentum'] = b_momentum + group['max_momentum'] = m_momentum + + if self.three_phase: + self.momentum_phases.append({ + 'end_iter': + float(self.pct_start * runner.max_iters) - 1, + 'start_momentum': + 'max_momentum', + 'end_momentum': + 'base_momentum' + }) + self.momentum_phases.append({ + 'end_iter': + float(2 * self.pct_start * runner.max_iters) - 2, + 'start_momentum': + 'base_momentum', + 'end_momentum': + 'max_momentum' + }) + self.momentum_phases.append({ + 'end_iter': runner.max_iters - 1, + 'start_momentum': 'max_momentum', + 'end_momentum': 'max_momentum' + }) + else: + self.momentum_phases.append({ + 'end_iter': + float(self.pct_start * runner.max_iters) - 1, + 'start_momentum': + 'max_momentum', + 'end_momentum': + 'base_momentum' + }) + self.momentum_phases.append({ + 'end_iter': runner.max_iters - 1, + 'start_momentum': 'base_momentum', + 'end_momentum': 'max_momentum' + }) + + def _set_momentum(self, runner, momentum_groups): + if isinstance(runner.optimizer, dict): + for k, optim in runner.optimizer.items(): + for param_group, mom in zip(optim.param_groups, + momentum_groups[k]): + if 'momentum' in param_group.keys(): + param_group['momentum'] = mom + elif 'betas' in param_group.keys(): + param_group['betas'] = (mom, param_group['betas'][1]) + else: + for param_group, mom in zip(runner.optimizer.param_groups, + momentum_groups): + if 'momentum' in param_group.keys(): + param_group['momentum'] = mom + elif 'betas' in param_group.keys(): + param_group['betas'] = (mom, param_group['betas'][1]) + + def get_momentum(self, runner, param_group): + curr_iter = runner.iter + start_iter = 0 + for i, phase in enumerate(self.momentum_phases): + end_iter = phase['end_iter'] + if curr_iter <= end_iter or i == len(self.momentum_phases) - 1: + pct = (curr_iter - start_iter) / (end_iter - start_iter) + momentum = self.anneal_func( + param_group[phase['start_momentum']], + param_group[phase['end_momentum']], pct) + break + start_iter = end_iter + return momentum + + def get_regular_momentum(self, runner): + if isinstance(runner.optimizer, dict): + momentum_groups = {} + for k, optim in runner.optimizer.items(): + _momentum_group = [ + self.get_momentum(runner, param_group) + for param_group in optim.param_groups + ] + momentum_groups.update({k: _momentum_group}) + return momentum_groups + else: + momentum_groups = [] + for param_group in runner.optimizer.param_groups: + momentum_groups.append(self.get_momentum(runner, param_group)) + return momentum_groups diff --git a/annotator/uniformer/mmcv/runner/hooks/optimizer.py b/annotator/uniformer/mmcv/runner/hooks/optimizer.py new file mode 100644 index 0000000000000000000000000000000000000000..4ef3e9ff8f9c6926e32bdf027612267b64ed80df --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/optimizer.py @@ -0,0 +1,508 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy +from collections import defaultdict +from itertools import chain + +from torch.nn.utils import clip_grad + +from annotator.uniformer.mmcv.utils import TORCH_VERSION, _BatchNorm, digit_version +from ..dist_utils import allreduce_grads +from ..fp16_utils import LossScaler, wrap_fp16_model +from .hook import HOOKS, Hook + +try: + # If PyTorch version >= 1.6.0, torch.cuda.amp.GradScaler would be imported + # and used; otherwise, auto fp16 will adopt mmcv's implementation. + from torch.cuda.amp import GradScaler +except ImportError: + pass + + +@HOOKS.register_module() +class OptimizerHook(Hook): + + def __init__(self, grad_clip=None): + self.grad_clip = grad_clip + + def clip_grads(self, params): + params = list( + filter(lambda p: p.requires_grad and p.grad is not None, params)) + if len(params) > 0: + return clip_grad.clip_grad_norm_(params, **self.grad_clip) + + def after_train_iter(self, runner): + runner.optimizer.zero_grad() + runner.outputs['loss'].backward() + if self.grad_clip is not None: + grad_norm = self.clip_grads(runner.model.parameters()) + if grad_norm is not None: + # Add grad norm to the logger + runner.log_buffer.update({'grad_norm': float(grad_norm)}, + runner.outputs['num_samples']) + runner.optimizer.step() + + +@HOOKS.register_module() +class GradientCumulativeOptimizerHook(OptimizerHook): + """Optimizer Hook implements multi-iters gradient cumulating. + + Args: + cumulative_iters (int, optional): Num of gradient cumulative iters. + The optimizer will step every `cumulative_iters` iters. + Defaults to 1. + + Examples: + >>> # Use cumulative_iters to simulate a large batch size + >>> # It is helpful when the hardware cannot handle a large batch size. + >>> loader = DataLoader(data, batch_size=64) + >>> optim_hook = GradientCumulativeOptimizerHook(cumulative_iters=4) + >>> # almost equals to + >>> loader = DataLoader(data, batch_size=256) + >>> optim_hook = OptimizerHook() + """ + + def __init__(self, cumulative_iters=1, **kwargs): + super(GradientCumulativeOptimizerHook, self).__init__(**kwargs) + + assert isinstance(cumulative_iters, int) and cumulative_iters > 0, \ + f'cumulative_iters only accepts positive int, but got ' \ + f'{type(cumulative_iters)} instead.' + + self.cumulative_iters = cumulative_iters + self.divisible_iters = 0 + self.remainder_iters = 0 + self.initialized = False + + def has_batch_norm(self, module): + if isinstance(module, _BatchNorm): + return True + for m in module.children(): + if self.has_batch_norm(m): + return True + return False + + def _init(self, runner): + if runner.iter % self.cumulative_iters != 0: + runner.logger.warning( + 'Resume iter number is not divisible by cumulative_iters in ' + 'GradientCumulativeOptimizerHook, which means the gradient of ' + 'some iters is lost and the result may be influenced slightly.' + ) + + if self.has_batch_norm(runner.model) and self.cumulative_iters > 1: + runner.logger.warning( + 'GradientCumulativeOptimizerHook may slightly decrease ' + 'performance if the model has BatchNorm layers.') + + residual_iters = runner.max_iters - runner.iter + + self.divisible_iters = ( + residual_iters // self.cumulative_iters * self.cumulative_iters) + self.remainder_iters = residual_iters - self.divisible_iters + + self.initialized = True + + def after_train_iter(self, runner): + if not self.initialized: + self._init(runner) + + if runner.iter < self.divisible_iters: + loss_factor = self.cumulative_iters + else: + loss_factor = self.remainder_iters + loss = runner.outputs['loss'] + loss = loss / loss_factor + loss.backward() + + if (self.every_n_iters(runner, self.cumulative_iters) + or self.is_last_iter(runner)): + + if self.grad_clip is not None: + grad_norm = self.clip_grads(runner.model.parameters()) + if grad_norm is not None: + # Add grad norm to the logger + runner.log_buffer.update({'grad_norm': float(grad_norm)}, + runner.outputs['num_samples']) + runner.optimizer.step() + runner.optimizer.zero_grad() + + +if (TORCH_VERSION != 'parrots' + and digit_version(TORCH_VERSION) >= digit_version('1.6.0')): + + @HOOKS.register_module() + class Fp16OptimizerHook(OptimizerHook): + """FP16 optimizer hook (using PyTorch's implementation). + + If you are using PyTorch >= 1.6, torch.cuda.amp is used as the backend, + to take care of the optimization procedure. + + Args: + loss_scale (float | str | dict): Scale factor configuration. + If loss_scale is a float, static loss scaling will be used with + the specified scale. If loss_scale is a string, it must be + 'dynamic', then dynamic loss scaling will be used. + It can also be a dict containing arguments of GradScalar. + Defaults to 512. For Pytorch >= 1.6, mmcv uses official + implementation of GradScaler. If you use a dict version of + loss_scale to create GradScaler, please refer to: + https://pytorch.org/docs/stable/amp.html#torch.cuda.amp.GradScaler + for the parameters. + + Examples: + >>> loss_scale = dict( + ... init_scale=65536.0, + ... growth_factor=2.0, + ... backoff_factor=0.5, + ... growth_interval=2000 + ... ) + >>> optimizer_hook = Fp16OptimizerHook(loss_scale=loss_scale) + """ + + def __init__(self, + grad_clip=None, + coalesce=True, + bucket_size_mb=-1, + loss_scale=512., + distributed=True): + self.grad_clip = grad_clip + self.coalesce = coalesce + self.bucket_size_mb = bucket_size_mb + self.distributed = distributed + self._scale_update_param = None + if loss_scale == 'dynamic': + self.loss_scaler = GradScaler() + elif isinstance(loss_scale, float): + self._scale_update_param = loss_scale + self.loss_scaler = GradScaler(init_scale=loss_scale) + elif isinstance(loss_scale, dict): + self.loss_scaler = GradScaler(**loss_scale) + else: + raise ValueError('loss_scale must be of type float, dict, or ' + f'"dynamic", got {loss_scale}') + + def before_run(self, runner): + """Preparing steps before Mixed Precision Training.""" + # wrap model mode to fp16 + wrap_fp16_model(runner.model) + # resume from state dict + if 'fp16' in runner.meta and 'loss_scaler' in runner.meta['fp16']: + scaler_state_dict = runner.meta['fp16']['loss_scaler'] + self.loss_scaler.load_state_dict(scaler_state_dict) + + def copy_grads_to_fp32(self, fp16_net, fp32_weights): + """Copy gradients from fp16 model to fp32 weight copy.""" + for fp32_param, fp16_param in zip(fp32_weights, + fp16_net.parameters()): + if fp16_param.grad is not None: + if fp32_param.grad is None: + fp32_param.grad = fp32_param.data.new( + fp32_param.size()) + fp32_param.grad.copy_(fp16_param.grad) + + def copy_params_to_fp16(self, fp16_net, fp32_weights): + """Copy updated params from fp32 weight copy to fp16 model.""" + for fp16_param, fp32_param in zip(fp16_net.parameters(), + fp32_weights): + fp16_param.data.copy_(fp32_param.data) + + def after_train_iter(self, runner): + """Backward optimization steps for Mixed Precision Training. For + dynamic loss scaling, please refer to + https://pytorch.org/docs/stable/amp.html#torch.cuda.amp.GradScaler. + + 1. Scale the loss by a scale factor. + 2. Backward the loss to obtain the gradients. + 3. Unscale the optimizer’s gradient tensors. + 4. Call optimizer.step() and update scale factor. + 5. Save loss_scaler state_dict for resume purpose. + """ + # clear grads of last iteration + runner.model.zero_grad() + runner.optimizer.zero_grad() + + self.loss_scaler.scale(runner.outputs['loss']).backward() + self.loss_scaler.unscale_(runner.optimizer) + # grad clip + if self.grad_clip is not None: + grad_norm = self.clip_grads(runner.model.parameters()) + if grad_norm is not None: + # Add grad norm to the logger + runner.log_buffer.update({'grad_norm': float(grad_norm)}, + runner.outputs['num_samples']) + # backward and update scaler + self.loss_scaler.step(runner.optimizer) + self.loss_scaler.update(self._scale_update_param) + + # save state_dict of loss_scaler + runner.meta.setdefault( + 'fp16', {})['loss_scaler'] = self.loss_scaler.state_dict() + + @HOOKS.register_module() + class GradientCumulativeFp16OptimizerHook(GradientCumulativeOptimizerHook, + Fp16OptimizerHook): + """Fp16 optimizer Hook (using PyTorch's implementation) implements + multi-iters gradient cumulating. + + If you are using PyTorch >= 1.6, torch.cuda.amp is used as the backend, + to take care of the optimization procedure. + """ + + def __init__(self, *args, **kwargs): + super(GradientCumulativeFp16OptimizerHook, + self).__init__(*args, **kwargs) + + def after_train_iter(self, runner): + if not self.initialized: + self._init(runner) + + if runner.iter < self.divisible_iters: + loss_factor = self.cumulative_iters + else: + loss_factor = self.remainder_iters + loss = runner.outputs['loss'] + loss = loss / loss_factor + + self.loss_scaler.scale(loss).backward() + + if (self.every_n_iters(runner, self.cumulative_iters) + or self.is_last_iter(runner)): + + # copy fp16 grads in the model to fp32 params in the optimizer + self.loss_scaler.unscale_(runner.optimizer) + + if self.grad_clip is not None: + grad_norm = self.clip_grads(runner.model.parameters()) + if grad_norm is not None: + # Add grad norm to the logger + runner.log_buffer.update( + {'grad_norm': float(grad_norm)}, + runner.outputs['num_samples']) + + # backward and update scaler + self.loss_scaler.step(runner.optimizer) + self.loss_scaler.update(self._scale_update_param) + + # save state_dict of loss_scaler + runner.meta.setdefault( + 'fp16', {})['loss_scaler'] = self.loss_scaler.state_dict() + + # clear grads + runner.model.zero_grad() + runner.optimizer.zero_grad() + +else: + + @HOOKS.register_module() + class Fp16OptimizerHook(OptimizerHook): + """FP16 optimizer hook (mmcv's implementation). + + The steps of fp16 optimizer is as follows. + 1. Scale the loss value. + 2. BP in the fp16 model. + 2. Copy gradients from fp16 model to fp32 weights. + 3. Update fp32 weights. + 4. Copy updated parameters from fp32 weights to fp16 model. + + Refer to https://arxiv.org/abs/1710.03740 for more details. + + Args: + loss_scale (float | str | dict): Scale factor configuration. + If loss_scale is a float, static loss scaling will be used with + the specified scale. If loss_scale is a string, it must be + 'dynamic', then dynamic loss scaling will be used. + It can also be a dict containing arguments of LossScaler. + Defaults to 512. + """ + + def __init__(self, + grad_clip=None, + coalesce=True, + bucket_size_mb=-1, + loss_scale=512., + distributed=True): + self.grad_clip = grad_clip + self.coalesce = coalesce + self.bucket_size_mb = bucket_size_mb + self.distributed = distributed + if loss_scale == 'dynamic': + self.loss_scaler = LossScaler(mode='dynamic') + elif isinstance(loss_scale, float): + self.loss_scaler = LossScaler( + init_scale=loss_scale, mode='static') + elif isinstance(loss_scale, dict): + self.loss_scaler = LossScaler(**loss_scale) + else: + raise ValueError('loss_scale must be of type float, dict, or ' + f'"dynamic", got {loss_scale}') + + def before_run(self, runner): + """Preparing steps before Mixed Precision Training. + + 1. Make a master copy of fp32 weights for optimization. + 2. Convert the main model from fp32 to fp16. + """ + # keep a copy of fp32 weights + old_groups = runner.optimizer.param_groups + runner.optimizer.param_groups = copy.deepcopy( + runner.optimizer.param_groups) + state = defaultdict(dict) + p_map = { + old_p: p + for old_p, p in zip( + chain(*(g['params'] for g in old_groups)), + chain(*(g['params'] + for g in runner.optimizer.param_groups))) + } + for k, v in runner.optimizer.state.items(): + state[p_map[k]] = v + runner.optimizer.state = state + # convert model to fp16 + wrap_fp16_model(runner.model) + # resume from state dict + if 'fp16' in runner.meta and 'loss_scaler' in runner.meta['fp16']: + scaler_state_dict = runner.meta['fp16']['loss_scaler'] + self.loss_scaler.load_state_dict(scaler_state_dict) + + def copy_grads_to_fp32(self, fp16_net, fp32_weights): + """Copy gradients from fp16 model to fp32 weight copy.""" + for fp32_param, fp16_param in zip(fp32_weights, + fp16_net.parameters()): + if fp16_param.grad is not None: + if fp32_param.grad is None: + fp32_param.grad = fp32_param.data.new( + fp32_param.size()) + fp32_param.grad.copy_(fp16_param.grad) + + def copy_params_to_fp16(self, fp16_net, fp32_weights): + """Copy updated params from fp32 weight copy to fp16 model.""" + for fp16_param, fp32_param in zip(fp16_net.parameters(), + fp32_weights): + fp16_param.data.copy_(fp32_param.data) + + def after_train_iter(self, runner): + """Backward optimization steps for Mixed Precision Training. For + dynamic loss scaling, please refer `loss_scalar.py` + + 1. Scale the loss by a scale factor. + 2. Backward the loss to obtain the gradients (fp16). + 3. Copy gradients from the model to the fp32 weight copy. + 4. Scale the gradients back and update the fp32 weight copy. + 5. Copy back the params from fp32 weight copy to the fp16 model. + 6. Save loss_scaler state_dict for resume purpose. + """ + # clear grads of last iteration + runner.model.zero_grad() + runner.optimizer.zero_grad() + # scale the loss value + scaled_loss = runner.outputs['loss'] * self.loss_scaler.loss_scale + scaled_loss.backward() + # copy fp16 grads in the model to fp32 params in the optimizer + + fp32_weights = [] + for param_group in runner.optimizer.param_groups: + fp32_weights += param_group['params'] + self.copy_grads_to_fp32(runner.model, fp32_weights) + # allreduce grads + if self.distributed: + allreduce_grads(fp32_weights, self.coalesce, + self.bucket_size_mb) + + has_overflow = self.loss_scaler.has_overflow(fp32_weights) + # if has overflow, skip this iteration + if not has_overflow: + # scale the gradients back + for param in fp32_weights: + if param.grad is not None: + param.grad.div_(self.loss_scaler.loss_scale) + if self.grad_clip is not None: + grad_norm = self.clip_grads(fp32_weights) + if grad_norm is not None: + # Add grad norm to the logger + runner.log_buffer.update( + {'grad_norm': float(grad_norm)}, + runner.outputs['num_samples']) + # update fp32 params + runner.optimizer.step() + # copy fp32 params to the fp16 model + self.copy_params_to_fp16(runner.model, fp32_weights) + self.loss_scaler.update_scale(has_overflow) + if has_overflow: + runner.logger.warning('Check overflow, downscale loss scale ' + f'to {self.loss_scaler.cur_scale}') + + # save state_dict of loss_scaler + runner.meta.setdefault( + 'fp16', {})['loss_scaler'] = self.loss_scaler.state_dict() + + @HOOKS.register_module() + class GradientCumulativeFp16OptimizerHook(GradientCumulativeOptimizerHook, + Fp16OptimizerHook): + """Fp16 optimizer Hook (using mmcv implementation) implements multi- + iters gradient cumulating.""" + + def __init__(self, *args, **kwargs): + super(GradientCumulativeFp16OptimizerHook, + self).__init__(*args, **kwargs) + + def after_train_iter(self, runner): + if not self.initialized: + self._init(runner) + + if runner.iter < self.divisible_iters: + loss_factor = self.cumulative_iters + else: + loss_factor = self.remainder_iters + + loss = runner.outputs['loss'] + loss = loss / loss_factor + + # scale the loss value + scaled_loss = loss * self.loss_scaler.loss_scale + scaled_loss.backward() + + if (self.every_n_iters(runner, self.cumulative_iters) + or self.is_last_iter(runner)): + + # copy fp16 grads in the model to fp32 params in the optimizer + fp32_weights = [] + for param_group in runner.optimizer.param_groups: + fp32_weights += param_group['params'] + self.copy_grads_to_fp32(runner.model, fp32_weights) + # allreduce grads + if self.distributed: + allreduce_grads(fp32_weights, self.coalesce, + self.bucket_size_mb) + + has_overflow = self.loss_scaler.has_overflow(fp32_weights) + # if has overflow, skip this iteration + if not has_overflow: + # scale the gradients back + for param in fp32_weights: + if param.grad is not None: + param.grad.div_(self.loss_scaler.loss_scale) + if self.grad_clip is not None: + grad_norm = self.clip_grads(fp32_weights) + if grad_norm is not None: + # Add grad norm to the logger + runner.log_buffer.update( + {'grad_norm': float(grad_norm)}, + runner.outputs['num_samples']) + # update fp32 params + runner.optimizer.step() + # copy fp32 params to the fp16 model + self.copy_params_to_fp16(runner.model, fp32_weights) + else: + runner.logger.warning( + 'Check overflow, downscale loss scale ' + f'to {self.loss_scaler.cur_scale}') + + self.loss_scaler.update_scale(has_overflow) + + # save state_dict of loss_scaler + runner.meta.setdefault( + 'fp16', {})['loss_scaler'] = self.loss_scaler.state_dict() + + # clear grads + runner.model.zero_grad() + runner.optimizer.zero_grad() diff --git a/annotator/uniformer/mmcv/runner/hooks/profiler.py b/annotator/uniformer/mmcv/runner/hooks/profiler.py new file mode 100644 index 0000000000000000000000000000000000000000..b70236997eec59c2209ef351ae38863b4112d0ec --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/profiler.py @@ -0,0 +1,180 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import warnings +from typing import Callable, List, Optional, Union + +import torch + +from ..dist_utils import master_only +from .hook import HOOKS, Hook + + +@HOOKS.register_module() +class ProfilerHook(Hook): + """Profiler to analyze performance during training. + + PyTorch Profiler is a tool that allows the collection of the performance + metrics during the training. More details on Profiler can be found at + https://pytorch.org/docs/1.8.1/profiler.html#torch.profiler.profile + + Args: + by_epoch (bool): Profile performance by epoch or by iteration. + Default: True. + profile_iters (int): Number of iterations for profiling. + If ``by_epoch=True``, profile_iters indicates that they are the + first profile_iters epochs at the beginning of the + training, otherwise it indicates the first profile_iters + iterations. Default: 1. + activities (list[str]): List of activity groups (CPU, CUDA) to use in + profiling. Default: ['cpu', 'cuda']. + schedule (dict, optional): Config of generating the callable schedule. + if schedule is None, profiler will not add step markers into the + trace and table view. Default: None. + on_trace_ready (callable, dict): Either a handler or a dict of generate + handler. Default: None. + record_shapes (bool): Save information about operator's input shapes. + Default: False. + profile_memory (bool): Track tensor memory allocation/deallocation. + Default: False. + with_stack (bool): Record source information (file and line number) + for the ops. Default: False. + with_flops (bool): Use formula to estimate the FLOPS of specific + operators (matrix multiplication and 2D convolution). + Default: False. + json_trace_path (str, optional): Exports the collected trace in Chrome + JSON format. Default: None. + + Example: + >>> runner = ... # instantiate a Runner + >>> # tensorboard trace + >>> trace_config = dict(type='tb_trace', dir_name='work_dir') + >>> profiler_config = dict(on_trace_ready=trace_config) + >>> runner.register_profiler_hook(profiler_config) + >>> runner.run(data_loaders=[trainloader], workflow=[('train', 1)]) + """ + + def __init__(self, + by_epoch: bool = True, + profile_iters: int = 1, + activities: List[str] = ['cpu', 'cuda'], + schedule: Optional[dict] = None, + on_trace_ready: Optional[Union[Callable, dict]] = None, + record_shapes: bool = False, + profile_memory: bool = False, + with_stack: bool = False, + with_flops: bool = False, + json_trace_path: Optional[str] = None) -> None: + try: + from torch import profiler # torch version >= 1.8.1 + except ImportError: + raise ImportError('profiler is the new feature of torch1.8.1, ' + f'but your version is {torch.__version__}') + + assert isinstance(by_epoch, bool), '``by_epoch`` should be a boolean.' + self.by_epoch = by_epoch + + if profile_iters < 1: + raise ValueError('profile_iters should be greater than 0, but got ' + f'{profile_iters}') + self.profile_iters = profile_iters + + if not isinstance(activities, list): + raise ValueError( + f'activities should be list, but got {type(activities)}') + self.activities = [] + for activity in activities: + activity = activity.lower() + if activity == 'cpu': + self.activities.append(profiler.ProfilerActivity.CPU) + elif activity == 'cuda': + self.activities.append(profiler.ProfilerActivity.CUDA) + else: + raise ValueError( + f'activity should be "cpu" or "cuda", but got {activity}') + + if schedule is not None: + self.schedule = profiler.schedule(**schedule) + else: + self.schedule = None + + self.on_trace_ready = on_trace_ready + self.record_shapes = record_shapes + self.profile_memory = profile_memory + self.with_stack = with_stack + self.with_flops = with_flops + self.json_trace_path = json_trace_path + + @master_only + def before_run(self, runner): + if self.by_epoch and runner.max_epochs < self.profile_iters: + raise ValueError('self.profile_iters should not be greater than ' + f'{runner.max_epochs}') + + if not self.by_epoch and runner.max_iters < self.profile_iters: + raise ValueError('self.profile_iters should not be greater than ' + f'{runner.max_iters}') + + if callable(self.on_trace_ready): # handler + _on_trace_ready = self.on_trace_ready + elif isinstance(self.on_trace_ready, dict): # config of handler + trace_cfg = self.on_trace_ready.copy() + trace_type = trace_cfg.pop('type') # log_trace handler + if trace_type == 'log_trace': + + def _log_handler(prof): + print(prof.key_averages().table(**trace_cfg)) + + _on_trace_ready = _log_handler + elif trace_type == 'tb_trace': # tensorboard_trace handler + try: + import torch_tb_profiler # noqa: F401 + except ImportError: + raise ImportError('please run "pip install ' + 'torch-tb-profiler" to install ' + 'torch_tb_profiler') + _on_trace_ready = torch.profiler.tensorboard_trace_handler( + **trace_cfg) + else: + raise ValueError('trace_type should be "log_trace" or ' + f'"tb_trace", but got {trace_type}') + elif self.on_trace_ready is None: + _on_trace_ready = None # type: ignore + else: + raise ValueError('on_trace_ready should be handler, dict or None, ' + f'but got {type(self.on_trace_ready)}') + + if runner.max_epochs > 1: + warnings.warn(f'profiler will profile {runner.max_epochs} epochs ' + 'instead of 1 epoch. Since profiler will slow down ' + 'the training, it is recommended to train 1 epoch ' + 'with ProfilerHook and adjust your setting according' + ' to the profiler summary. During normal training ' + '(epoch > 1), you may disable the ProfilerHook.') + + self.profiler = torch.profiler.profile( + activities=self.activities, + schedule=self.schedule, + on_trace_ready=_on_trace_ready, + record_shapes=self.record_shapes, + profile_memory=self.profile_memory, + with_stack=self.with_stack, + with_flops=self.with_flops) + + self.profiler.__enter__() + runner.logger.info('profiler is profiling...') + + @master_only + def after_train_epoch(self, runner): + if self.by_epoch and runner.epoch == self.profile_iters - 1: + runner.logger.info('profiler may take a few minutes...') + self.profiler.__exit__(None, None, None) + if self.json_trace_path is not None: + self.profiler.export_chrome_trace(self.json_trace_path) + + @master_only + def after_train_iter(self, runner): + self.profiler.step() + if not self.by_epoch and runner.iter == self.profile_iters - 1: + runner.logger.info('profiler may take a few minutes...') + self.profiler.__exit__(None, None, None) + if self.json_trace_path is not None: + self.profiler.export_chrome_trace(self.json_trace_path) diff --git a/annotator/uniformer/mmcv/runner/hooks/sampler_seed.py b/annotator/uniformer/mmcv/runner/hooks/sampler_seed.py new file mode 100644 index 0000000000000000000000000000000000000000..ee0dc6bdd8df5775857028aaed5444c0f59caf80 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/sampler_seed.py @@ -0,0 +1,20 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .hook import HOOKS, Hook + + +@HOOKS.register_module() +class DistSamplerSeedHook(Hook): + """Data-loading sampler for distributed training. + + When distributed training, it is only useful in conjunction with + :obj:`EpochBasedRunner`, while :obj:`IterBasedRunner` achieves the same + purpose with :obj:`IterLoader`. + """ + + def before_epoch(self, runner): + if hasattr(runner.data_loader.sampler, 'set_epoch'): + # in case the data loader uses `SequentialSampler` in Pytorch + runner.data_loader.sampler.set_epoch(runner.epoch) + elif hasattr(runner.data_loader.batch_sampler.sampler, 'set_epoch'): + # batch sampler in pytorch warps the sampler as its attributes. + runner.data_loader.batch_sampler.sampler.set_epoch(runner.epoch) diff --git a/annotator/uniformer/mmcv/runner/hooks/sync_buffer.py b/annotator/uniformer/mmcv/runner/hooks/sync_buffer.py new file mode 100644 index 0000000000000000000000000000000000000000..6376b7ff894280cb2782243b25e8973650591577 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/hooks/sync_buffer.py @@ -0,0 +1,22 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from ..dist_utils import allreduce_params +from .hook import HOOKS, Hook + + +@HOOKS.register_module() +class SyncBuffersHook(Hook): + """Synchronize model buffers such as running_mean and running_var in BN at + the end of each epoch. + + Args: + distributed (bool): Whether distributed training is used. It is + effective only for distributed training. Defaults to True. + """ + + def __init__(self, distributed=True): + self.distributed = distributed + + def after_epoch(self, runner): + """All-reduce model buffers at the end of each epoch.""" + if self.distributed: + allreduce_params(runner.model.buffers()) diff --git a/annotator/uniformer/mmcv/runner/iter_based_runner.py b/annotator/uniformer/mmcv/runner/iter_based_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..1df4de8c0285669dec9b014dfd1f3dd1600f0831 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/iter_based_runner.py @@ -0,0 +1,273 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os.path as osp +import platform +import shutil +import time +import warnings + +import torch +from torch.optim import Optimizer + +import annotator.uniformer.mmcv as mmcv +from .base_runner import BaseRunner +from .builder import RUNNERS +from .checkpoint import save_checkpoint +from .hooks import IterTimerHook +from .utils import get_host_info + + +class IterLoader: + + def __init__(self, dataloader): + self._dataloader = dataloader + self.iter_loader = iter(self._dataloader) + self._epoch = 0 + + @property + def epoch(self): + return self._epoch + + def __next__(self): + try: + data = next(self.iter_loader) + except StopIteration: + self._epoch += 1 + if hasattr(self._dataloader.sampler, 'set_epoch'): + self._dataloader.sampler.set_epoch(self._epoch) + time.sleep(2) # Prevent possible deadlock during epoch transition + self.iter_loader = iter(self._dataloader) + data = next(self.iter_loader) + + return data + + def __len__(self): + return len(self._dataloader) + + +@RUNNERS.register_module() +class IterBasedRunner(BaseRunner): + """Iteration-based Runner. + + This runner train models iteration by iteration. + """ + + def train(self, data_loader, **kwargs): + self.model.train() + self.mode = 'train' + self.data_loader = data_loader + self._epoch = data_loader.epoch + data_batch = next(data_loader) + self.call_hook('before_train_iter') + outputs = self.model.train_step(data_batch, self.optimizer, **kwargs) + if not isinstance(outputs, dict): + raise TypeError('model.train_step() must return a dict') + if 'log_vars' in outputs: + self.log_buffer.update(outputs['log_vars'], outputs['num_samples']) + self.outputs = outputs + self.call_hook('after_train_iter') + self._inner_iter += 1 + self._iter += 1 + + @torch.no_grad() + def val(self, data_loader, **kwargs): + self.model.eval() + self.mode = 'val' + self.data_loader = data_loader + data_batch = next(data_loader) + self.call_hook('before_val_iter') + outputs = self.model.val_step(data_batch, **kwargs) + if not isinstance(outputs, dict): + raise TypeError('model.val_step() must return a dict') + if 'log_vars' in outputs: + self.log_buffer.update(outputs['log_vars'], outputs['num_samples']) + self.outputs = outputs + self.call_hook('after_val_iter') + self._inner_iter += 1 + + def run(self, data_loaders, workflow, max_iters=None, **kwargs): + """Start running. + + Args: + data_loaders (list[:obj:`DataLoader`]): Dataloaders for training + and validation. + workflow (list[tuple]): A list of (phase, iters) to specify the + running order and iterations. E.g, [('train', 10000), + ('val', 1000)] means running 10000 iterations for training and + 1000 iterations for validation, iteratively. + """ + assert isinstance(data_loaders, list) + assert mmcv.is_list_of(workflow, tuple) + assert len(data_loaders) == len(workflow) + if max_iters is not None: + warnings.warn( + 'setting max_iters in run is deprecated, ' + 'please set max_iters in runner_config', DeprecationWarning) + self._max_iters = max_iters + assert self._max_iters is not None, ( + 'max_iters must be specified during instantiation') + + work_dir = self.work_dir if self.work_dir is not None else 'NONE' + self.logger.info('Start running, host: %s, work_dir: %s', + get_host_info(), work_dir) + self.logger.info('Hooks will be executed in the following order:\n%s', + self.get_hook_info()) + self.logger.info('workflow: %s, max: %d iters', workflow, + self._max_iters) + self.call_hook('before_run') + + iter_loaders = [IterLoader(x) for x in data_loaders] + + self.call_hook('before_epoch') + + while self.iter < self._max_iters: + for i, flow in enumerate(workflow): + self._inner_iter = 0 + mode, iters = flow + if not isinstance(mode, str) or not hasattr(self, mode): + raise ValueError( + 'runner has no method named "{}" to run a workflow'. + format(mode)) + iter_runner = getattr(self, mode) + for _ in range(iters): + if mode == 'train' and self.iter >= self._max_iters: + break + iter_runner(iter_loaders[i], **kwargs) + + time.sleep(1) # wait for some hooks like loggers to finish + self.call_hook('after_epoch') + self.call_hook('after_run') + + def resume(self, + checkpoint, + resume_optimizer=True, + map_location='default'): + """Resume model from checkpoint. + + Args: + checkpoint (str): Checkpoint to resume from. + resume_optimizer (bool, optional): Whether resume the optimizer(s) + if the checkpoint file includes optimizer(s). Default to True. + map_location (str, optional): Same as :func:`torch.load`. + Default to 'default'. + """ + if map_location == 'default': + device_id = torch.cuda.current_device() + checkpoint = self.load_checkpoint( + checkpoint, + map_location=lambda storage, loc: storage.cuda(device_id)) + else: + checkpoint = self.load_checkpoint( + checkpoint, map_location=map_location) + + self._epoch = checkpoint['meta']['epoch'] + self._iter = checkpoint['meta']['iter'] + self._inner_iter = checkpoint['meta']['iter'] + if 'optimizer' in checkpoint and resume_optimizer: + if isinstance(self.optimizer, Optimizer): + self.optimizer.load_state_dict(checkpoint['optimizer']) + elif isinstance(self.optimizer, dict): + for k in self.optimizer.keys(): + self.optimizer[k].load_state_dict( + checkpoint['optimizer'][k]) + else: + raise TypeError( + 'Optimizer should be dict or torch.optim.Optimizer ' + f'but got {type(self.optimizer)}') + + self.logger.info(f'resumed from epoch: {self.epoch}, iter {self.iter}') + + def save_checkpoint(self, + out_dir, + filename_tmpl='iter_{}.pth', + meta=None, + save_optimizer=True, + create_symlink=True): + """Save checkpoint to file. + + Args: + out_dir (str): Directory to save checkpoint files. + filename_tmpl (str, optional): Checkpoint file template. + Defaults to 'iter_{}.pth'. + meta (dict, optional): Metadata to be saved in checkpoint. + Defaults to None. + save_optimizer (bool, optional): Whether save optimizer. + Defaults to True. + create_symlink (bool, optional): Whether create symlink to the + latest checkpoint file. Defaults to True. + """ + if meta is None: + meta = {} + elif not isinstance(meta, dict): + raise TypeError( + f'meta should be a dict or None, but got {type(meta)}') + if self.meta is not None: + meta.update(self.meta) + # Note: meta.update(self.meta) should be done before + # meta.update(epoch=self.epoch + 1, iter=self.iter) otherwise + # there will be problems with resumed checkpoints. + # More details in https://github.com/open-mmlab/mmcv/pull/1108 + meta.update(epoch=self.epoch + 1, iter=self.iter) + + filename = filename_tmpl.format(self.iter + 1) + filepath = osp.join(out_dir, filename) + optimizer = self.optimizer if save_optimizer else None + save_checkpoint(self.model, filepath, optimizer=optimizer, meta=meta) + # in some environments, `os.symlink` is not supported, you may need to + # set `create_symlink` to False + if create_symlink: + dst_file = osp.join(out_dir, 'latest.pth') + if platform.system() != 'Windows': + mmcv.symlink(filename, dst_file) + else: + shutil.copy(filepath, dst_file) + + def register_training_hooks(self, + lr_config, + optimizer_config=None, + checkpoint_config=None, + log_config=None, + momentum_config=None, + custom_hooks_config=None): + """Register default hooks for iter-based training. + + Checkpoint hook, optimizer stepper hook and logger hooks will be set to + `by_epoch=False` by default. + + Default hooks include: + + +----------------------+-------------------------+ + | Hooks | Priority | + +======================+=========================+ + | LrUpdaterHook | VERY_HIGH (10) | + +----------------------+-------------------------+ + | MomentumUpdaterHook | HIGH (30) | + +----------------------+-------------------------+ + | OptimizerStepperHook | ABOVE_NORMAL (40) | + +----------------------+-------------------------+ + | CheckpointSaverHook | NORMAL (50) | + +----------------------+-------------------------+ + | IterTimerHook | LOW (70) | + +----------------------+-------------------------+ + | LoggerHook(s) | VERY_LOW (90) | + +----------------------+-------------------------+ + | CustomHook(s) | defaults to NORMAL (50) | + +----------------------+-------------------------+ + + If custom hooks have same priority with default hooks, custom hooks + will be triggered after default hooks. + """ + if checkpoint_config is not None: + checkpoint_config.setdefault('by_epoch', False) + if lr_config is not None: + lr_config.setdefault('by_epoch', False) + if log_config is not None: + for info in log_config['hooks']: + info.setdefault('by_epoch', False) + super(IterBasedRunner, self).register_training_hooks( + lr_config=lr_config, + momentum_config=momentum_config, + optimizer_config=optimizer_config, + checkpoint_config=checkpoint_config, + log_config=log_config, + timer_config=IterTimerHook(), + custom_hooks_config=custom_hooks_config) diff --git a/annotator/uniformer/mmcv/runner/log_buffer.py b/annotator/uniformer/mmcv/runner/log_buffer.py new file mode 100644 index 0000000000000000000000000000000000000000..d949e2941c5400088c7cd8a1dc893d8b233ae785 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/log_buffer.py @@ -0,0 +1,41 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from collections import OrderedDict + +import numpy as np + + +class LogBuffer: + + def __init__(self): + self.val_history = OrderedDict() + self.n_history = OrderedDict() + self.output = OrderedDict() + self.ready = False + + def clear(self): + self.val_history.clear() + self.n_history.clear() + self.clear_output() + + def clear_output(self): + self.output.clear() + self.ready = False + + def update(self, vars, count=1): + assert isinstance(vars, dict) + for key, var in vars.items(): + if key not in self.val_history: + self.val_history[key] = [] + self.n_history[key] = [] + self.val_history[key].append(var) + self.n_history[key].append(count) + + def average(self, n=0): + """Average latest n values or all values.""" + assert n >= 0 + for key in self.val_history: + values = np.array(self.val_history[key][-n:]) + nums = np.array(self.n_history[key][-n:]) + avg = np.sum(values * nums) / np.sum(nums) + self.output[key] = avg + self.ready = True diff --git a/annotator/uniformer/mmcv/runner/optimizer/__init__.py b/annotator/uniformer/mmcv/runner/optimizer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..53c34d0470992cbc374f29681fdd00dc0e57968d --- /dev/null +++ b/annotator/uniformer/mmcv/runner/optimizer/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .builder import (OPTIMIZER_BUILDERS, OPTIMIZERS, build_optimizer, + build_optimizer_constructor) +from .default_constructor import DefaultOptimizerConstructor + +__all__ = [ + 'OPTIMIZER_BUILDERS', 'OPTIMIZERS', 'DefaultOptimizerConstructor', + 'build_optimizer', 'build_optimizer_constructor' +] diff --git a/annotator/uniformer/mmcv/runner/optimizer/builder.py b/annotator/uniformer/mmcv/runner/optimizer/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..f9234eed8f1f186d9d8dfda34562157ee39bdb3a --- /dev/null +++ b/annotator/uniformer/mmcv/runner/optimizer/builder.py @@ -0,0 +1,44 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import copy +import inspect + +import torch + +from ...utils import Registry, build_from_cfg + +OPTIMIZERS = Registry('optimizer') +OPTIMIZER_BUILDERS = Registry('optimizer builder') + + +def register_torch_optimizers(): + torch_optimizers = [] + for module_name in dir(torch.optim): + if module_name.startswith('__'): + continue + _optim = getattr(torch.optim, module_name) + if inspect.isclass(_optim) and issubclass(_optim, + torch.optim.Optimizer): + OPTIMIZERS.register_module()(_optim) + torch_optimizers.append(module_name) + return torch_optimizers + + +TORCH_OPTIMIZERS = register_torch_optimizers() + + +def build_optimizer_constructor(cfg): + return build_from_cfg(cfg, OPTIMIZER_BUILDERS) + + +def build_optimizer(model, cfg): + optimizer_cfg = copy.deepcopy(cfg) + constructor_type = optimizer_cfg.pop('constructor', + 'DefaultOptimizerConstructor') + paramwise_cfg = optimizer_cfg.pop('paramwise_cfg', None) + optim_constructor = build_optimizer_constructor( + dict( + type=constructor_type, + optimizer_cfg=optimizer_cfg, + paramwise_cfg=paramwise_cfg)) + optimizer = optim_constructor(model) + return optimizer diff --git a/annotator/uniformer/mmcv/runner/optimizer/default_constructor.py b/annotator/uniformer/mmcv/runner/optimizer/default_constructor.py new file mode 100644 index 0000000000000000000000000000000000000000..2c0da3503b75441738efe38d70352b55a210a34a --- /dev/null +++ b/annotator/uniformer/mmcv/runner/optimizer/default_constructor.py @@ -0,0 +1,249 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import warnings + +import torch +from torch.nn import GroupNorm, LayerNorm + +from annotator.uniformer.mmcv.utils import _BatchNorm, _InstanceNorm, build_from_cfg, is_list_of +from annotator.uniformer.mmcv.utils.ext_loader import check_ops_exist +from .builder import OPTIMIZER_BUILDERS, OPTIMIZERS + + +@OPTIMIZER_BUILDERS.register_module() +class DefaultOptimizerConstructor: + """Default constructor for optimizers. + + By default each parameter share the same optimizer settings, and we + provide an argument ``paramwise_cfg`` to specify parameter-wise settings. + It is a dict and may contain the following fields: + + - ``custom_keys`` (dict): Specified parameters-wise settings by keys. If + one of the keys in ``custom_keys`` is a substring of the name of one + parameter, then the setting of the parameter will be specified by + ``custom_keys[key]`` and other setting like ``bias_lr_mult`` etc. will + be ignored. It should be noted that the aforementioned ``key`` is the + longest key that is a substring of the name of the parameter. If there + are multiple matched keys with the same length, then the key with lower + alphabet order will be chosen. + ``custom_keys[key]`` should be a dict and may contain fields ``lr_mult`` + and ``decay_mult``. See Example 2 below. + - ``bias_lr_mult`` (float): It will be multiplied to the learning + rate for all bias parameters (except for those in normalization + layers and offset layers of DCN). + - ``bias_decay_mult`` (float): It will be multiplied to the weight + decay for all bias parameters (except for those in + normalization layers, depthwise conv layers, offset layers of DCN). + - ``norm_decay_mult`` (float): It will be multiplied to the weight + decay for all weight and bias parameters of normalization + layers. + - ``dwconv_decay_mult`` (float): It will be multiplied to the weight + decay for all weight and bias parameters of depthwise conv + layers. + - ``dcn_offset_lr_mult`` (float): It will be multiplied to the learning + rate for parameters of offset layer in the deformable convs + of a model. + - ``bypass_duplicate`` (bool): If true, the duplicate parameters + would not be added into optimizer. Default: False. + + Note: + 1. If the option ``dcn_offset_lr_mult`` is used, the constructor will + override the effect of ``bias_lr_mult`` in the bias of offset + layer. So be careful when using both ``bias_lr_mult`` and + ``dcn_offset_lr_mult``. If you wish to apply both of them to the + offset layer in deformable convs, set ``dcn_offset_lr_mult`` + to the original ``dcn_offset_lr_mult`` * ``bias_lr_mult``. + 2. If the option ``dcn_offset_lr_mult`` is used, the constructor will + apply it to all the DCN layers in the model. So be careful when + the model contains multiple DCN layers in places other than + backbone. + + Args: + model (:obj:`nn.Module`): The model with parameters to be optimized. + optimizer_cfg (dict): The config dict of the optimizer. + Positional fields are + + - `type`: class name of the optimizer. + + Optional fields are + + - any arguments of the corresponding optimizer type, e.g., + lr, weight_decay, momentum, etc. + paramwise_cfg (dict, optional): Parameter-wise options. + + Example 1: + >>> model = torch.nn.modules.Conv1d(1, 1, 1) + >>> optimizer_cfg = dict(type='SGD', lr=0.01, momentum=0.9, + >>> weight_decay=0.0001) + >>> paramwise_cfg = dict(norm_decay_mult=0.) + >>> optim_builder = DefaultOptimizerConstructor( + >>> optimizer_cfg, paramwise_cfg) + >>> optimizer = optim_builder(model) + + Example 2: + >>> # assume model have attribute model.backbone and model.cls_head + >>> optimizer_cfg = dict(type='SGD', lr=0.01, weight_decay=0.95) + >>> paramwise_cfg = dict(custom_keys={ + '.backbone': dict(lr_mult=0.1, decay_mult=0.9)}) + >>> optim_builder = DefaultOptimizerConstructor( + >>> optimizer_cfg, paramwise_cfg) + >>> optimizer = optim_builder(model) + >>> # Then the `lr` and `weight_decay` for model.backbone is + >>> # (0.01 * 0.1, 0.95 * 0.9). `lr` and `weight_decay` for + >>> # model.cls_head is (0.01, 0.95). + """ + + def __init__(self, optimizer_cfg, paramwise_cfg=None): + if not isinstance(optimizer_cfg, dict): + raise TypeError('optimizer_cfg should be a dict', + f'but got {type(optimizer_cfg)}') + self.optimizer_cfg = optimizer_cfg + self.paramwise_cfg = {} if paramwise_cfg is None else paramwise_cfg + self.base_lr = optimizer_cfg.get('lr', None) + self.base_wd = optimizer_cfg.get('weight_decay', None) + self._validate_cfg() + + def _validate_cfg(self): + if not isinstance(self.paramwise_cfg, dict): + raise TypeError('paramwise_cfg should be None or a dict, ' + f'but got {type(self.paramwise_cfg)}') + + if 'custom_keys' in self.paramwise_cfg: + if not isinstance(self.paramwise_cfg['custom_keys'], dict): + raise TypeError( + 'If specified, custom_keys must be a dict, ' + f'but got {type(self.paramwise_cfg["custom_keys"])}') + if self.base_wd is None: + for key in self.paramwise_cfg['custom_keys']: + if 'decay_mult' in self.paramwise_cfg['custom_keys'][key]: + raise ValueError('base_wd should not be None') + + # get base lr and weight decay + # weight_decay must be explicitly specified if mult is specified + if ('bias_decay_mult' in self.paramwise_cfg + or 'norm_decay_mult' in self.paramwise_cfg + or 'dwconv_decay_mult' in self.paramwise_cfg): + if self.base_wd is None: + raise ValueError('base_wd should not be None') + + def _is_in(self, param_group, param_group_list): + assert is_list_of(param_group_list, dict) + param = set(param_group['params']) + param_set = set() + for group in param_group_list: + param_set.update(set(group['params'])) + + return not param.isdisjoint(param_set) + + def add_params(self, params, module, prefix='', is_dcn_module=None): + """Add all parameters of module to the params list. + + The parameters of the given module will be added to the list of param + groups, with specific rules defined by paramwise_cfg. + + Args: + params (list[dict]): A list of param groups, it will be modified + in place. + module (nn.Module): The module to be added. + prefix (str): The prefix of the module + is_dcn_module (int|float|None): If the current module is a + submodule of DCN, `is_dcn_module` will be passed to + control conv_offset layer's learning rate. Defaults to None. + """ + # get param-wise options + custom_keys = self.paramwise_cfg.get('custom_keys', {}) + # first sort with alphabet order and then sort with reversed len of str + sorted_keys = sorted(sorted(custom_keys.keys()), key=len, reverse=True) + + bias_lr_mult = self.paramwise_cfg.get('bias_lr_mult', 1.) + bias_decay_mult = self.paramwise_cfg.get('bias_decay_mult', 1.) + norm_decay_mult = self.paramwise_cfg.get('norm_decay_mult', 1.) + dwconv_decay_mult = self.paramwise_cfg.get('dwconv_decay_mult', 1.) + bypass_duplicate = self.paramwise_cfg.get('bypass_duplicate', False) + dcn_offset_lr_mult = self.paramwise_cfg.get('dcn_offset_lr_mult', 1.) + + # special rules for norm layers and depth-wise conv layers + is_norm = isinstance(module, + (_BatchNorm, _InstanceNorm, GroupNorm, LayerNorm)) + is_dwconv = ( + isinstance(module, torch.nn.Conv2d) + and module.in_channels == module.groups) + + for name, param in module.named_parameters(recurse=False): + param_group = {'params': [param]} + if not param.requires_grad: + params.append(param_group) + continue + if bypass_duplicate and self._is_in(param_group, params): + warnings.warn(f'{prefix} is duplicate. It is skipped since ' + f'bypass_duplicate={bypass_duplicate}') + continue + # if the parameter match one of the custom keys, ignore other rules + is_custom = False + for key in sorted_keys: + if key in f'{prefix}.{name}': + is_custom = True + lr_mult = custom_keys[key].get('lr_mult', 1.) + param_group['lr'] = self.base_lr * lr_mult + if self.base_wd is not None: + decay_mult = custom_keys[key].get('decay_mult', 1.) + param_group['weight_decay'] = self.base_wd * decay_mult + break + + if not is_custom: + # bias_lr_mult affects all bias parameters + # except for norm.bias dcn.conv_offset.bias + if name == 'bias' and not (is_norm or is_dcn_module): + param_group['lr'] = self.base_lr * bias_lr_mult + + if (prefix.find('conv_offset') != -1 and is_dcn_module + and isinstance(module, torch.nn.Conv2d)): + # deal with both dcn_offset's bias & weight + param_group['lr'] = self.base_lr * dcn_offset_lr_mult + + # apply weight decay policies + if self.base_wd is not None: + # norm decay + if is_norm: + param_group[ + 'weight_decay'] = self.base_wd * norm_decay_mult + # depth-wise conv + elif is_dwconv: + param_group[ + 'weight_decay'] = self.base_wd * dwconv_decay_mult + # bias lr and decay + elif name == 'bias' and not is_dcn_module: + # TODO: current bias_decay_mult will have affect on DCN + param_group[ + 'weight_decay'] = self.base_wd * bias_decay_mult + params.append(param_group) + + if check_ops_exist(): + from annotator.uniformer.mmcv.ops import DeformConv2d, ModulatedDeformConv2d + is_dcn_module = isinstance(module, + (DeformConv2d, ModulatedDeformConv2d)) + else: + is_dcn_module = False + for child_name, child_mod in module.named_children(): + child_prefix = f'{prefix}.{child_name}' if prefix else child_name + self.add_params( + params, + child_mod, + prefix=child_prefix, + is_dcn_module=is_dcn_module) + + def __call__(self, model): + if hasattr(model, 'module'): + model = model.module + + optimizer_cfg = self.optimizer_cfg.copy() + # if no paramwise option is specified, just use the global setting + if not self.paramwise_cfg: + optimizer_cfg['params'] = model.parameters() + return build_from_cfg(optimizer_cfg, OPTIMIZERS) + + # set param-wise lr and weight decay recursively + params = [] + self.add_params(params, model) + optimizer_cfg['params'] = params + + return build_from_cfg(optimizer_cfg, OPTIMIZERS) diff --git a/annotator/uniformer/mmcv/runner/priority.py b/annotator/uniformer/mmcv/runner/priority.py new file mode 100644 index 0000000000000000000000000000000000000000..64cc4e3a05f8d5b89ab6eb32461e6e80f1d62e67 --- /dev/null +++ b/annotator/uniformer/mmcv/runner/priority.py @@ -0,0 +1,60 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from enum import Enum + + +class Priority(Enum): + """Hook priority levels. + + +--------------+------------+ + | Level | Value | + +==============+============+ + | HIGHEST | 0 | + +--------------+------------+ + | VERY_HIGH | 10 | + +--------------+------------+ + | HIGH | 30 | + +--------------+------------+ + | ABOVE_NORMAL | 40 | + +--------------+------------+ + | NORMAL | 50 | + +--------------+------------+ + | BELOW_NORMAL | 60 | + +--------------+------------+ + | LOW | 70 | + +--------------+------------+ + | VERY_LOW | 90 | + +--------------+------------+ + | LOWEST | 100 | + +--------------+------------+ + """ + + HIGHEST = 0 + VERY_HIGH = 10 + HIGH = 30 + ABOVE_NORMAL = 40 + NORMAL = 50 + BELOW_NORMAL = 60 + LOW = 70 + VERY_LOW = 90 + LOWEST = 100 + + +def get_priority(priority): + """Get priority value. + + Args: + priority (int or str or :obj:`Priority`): Priority. + + Returns: + int: The priority value. + """ + if isinstance(priority, int): + if priority < 0 or priority > 100: + raise ValueError('priority must be between 0 and 100') + return priority + elif isinstance(priority, Priority): + return priority.value + elif isinstance(priority, str): + return Priority[priority.upper()].value + else: + raise TypeError('priority must be an integer or Priority enum value') diff --git a/annotator/uniformer/mmcv/runner/utils.py b/annotator/uniformer/mmcv/runner/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c5befb8e56ece50b5fecfd007b26f8a29124c0bd --- /dev/null +++ b/annotator/uniformer/mmcv/runner/utils.py @@ -0,0 +1,93 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os +import random +import sys +import time +import warnings +from getpass import getuser +from socket import gethostname + +import numpy as np +import torch + +import annotator.uniformer.mmcv as mmcv + + +def get_host_info(): + """Get hostname and username. + + Return empty string if exception raised, e.g. ``getpass.getuser()`` will + lead to error in docker container + """ + host = '' + try: + host = f'{getuser()}@{gethostname()}' + except Exception as e: + warnings.warn(f'Host or user not found: {str(e)}') + finally: + return host + + +def get_time_str(): + return time.strftime('%Y%m%d_%H%M%S', time.localtime()) + + +def obj_from_dict(info, parent=None, default_args=None): + """Initialize an object from dict. + + The dict must contain the key "type", which indicates the object type, it + can be either a string or type, such as "list" or ``list``. Remaining + fields are treated as the arguments for constructing the object. + + Args: + info (dict): Object types and arguments. + parent (:class:`module`): Module which may containing expected object + classes. + default_args (dict, optional): Default arguments for initializing the + object. + + Returns: + any type: Object built from the dict. + """ + assert isinstance(info, dict) and 'type' in info + assert isinstance(default_args, dict) or default_args is None + args = info.copy() + obj_type = args.pop('type') + if mmcv.is_str(obj_type): + if parent is not None: + obj_type = getattr(parent, obj_type) + else: + obj_type = sys.modules[obj_type] + elif not isinstance(obj_type, type): + raise TypeError('type must be a str or valid type, but ' + f'got {type(obj_type)}') + if default_args is not None: + for name, value in default_args.items(): + args.setdefault(name, value) + return obj_type(**args) + + +def set_random_seed(seed, deterministic=False, use_rank_shift=False): + """Set random seed. + + Args: + seed (int): Seed to be used. + deterministic (bool): Whether to set the deterministic option for + CUDNN backend, i.e., set `torch.backends.cudnn.deterministic` + to True and `torch.backends.cudnn.benchmark` to False. + Default: False. + rank_shift (bool): Whether to add rank number to the random seed to + have different random seed in different threads. Default: False. + """ + if use_rank_shift: + rank, _ = mmcv.runner.get_dist_info() + seed += rank + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + os.environ['PYTHONHASHSEED'] = str(seed) + if deterministic: + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False diff --git a/annotator/uniformer/mmcv/utils/__init__.py b/annotator/uniformer/mmcv/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..378a0068432a371af364de9d73785901c0f83383 --- /dev/null +++ b/annotator/uniformer/mmcv/utils/__init__.py @@ -0,0 +1,69 @@ +# flake8: noqa +# Copyright (c) OpenMMLab. All rights reserved. +from .config import Config, ConfigDict, DictAction +from .misc import (check_prerequisites, concat_list, deprecated_api_warning, + has_method, import_modules_from_strings, is_list_of, + is_method_overridden, is_seq_of, is_str, is_tuple_of, + iter_cast, list_cast, requires_executable, requires_package, + slice_list, to_1tuple, to_2tuple, to_3tuple, to_4tuple, + to_ntuple, tuple_cast) +from .path import (check_file_exist, fopen, is_filepath, mkdir_or_exist, + scandir, symlink) +from .progressbar import (ProgressBar, track_iter_progress, + track_parallel_progress, track_progress) +from .testing import (assert_attrs_equal, assert_dict_contains_subset, + assert_dict_has_keys, assert_is_norm_layer, + assert_keys_equal, assert_params_all_zeros, + check_python_script) +from .timer import Timer, TimerError, check_time +from .version_utils import digit_version, get_git_hash + +try: + import torch +except ImportError: + __all__ = [ + 'Config', 'ConfigDict', 'DictAction', 'is_str', 'iter_cast', + 'list_cast', 'tuple_cast', 'is_seq_of', 'is_list_of', 'is_tuple_of', + 'slice_list', 'concat_list', 'check_prerequisites', 'requires_package', + 'requires_executable', 'is_filepath', 'fopen', 'check_file_exist', + 'mkdir_or_exist', 'symlink', 'scandir', 'ProgressBar', + 'track_progress', 'track_iter_progress', 'track_parallel_progress', + 'Timer', 'TimerError', 'check_time', 'deprecated_api_warning', + 'digit_version', 'get_git_hash', 'import_modules_from_strings', + 'assert_dict_contains_subset', 'assert_attrs_equal', + 'assert_dict_has_keys', 'assert_keys_equal', 'check_python_script', + 'to_1tuple', 'to_2tuple', 'to_3tuple', 'to_4tuple', 'to_ntuple', + 'is_method_overridden', 'has_method' + ] +else: + from .env import collect_env + from .logging import get_logger, print_log + from .parrots_jit import jit, skip_no_elena + from .parrots_wrapper import ( + TORCH_VERSION, BuildExtension, CppExtension, CUDAExtension, DataLoader, + PoolDataLoader, SyncBatchNorm, _AdaptiveAvgPoolNd, _AdaptiveMaxPoolNd, + _AvgPoolNd, _BatchNorm, _ConvNd, _ConvTransposeMixin, _InstanceNorm, + _MaxPoolNd, get_build_config, is_rocm_pytorch, _get_cuda_home) + from .registry import Registry, build_from_cfg + from .trace import is_jit_tracing + __all__ = [ + 'Config', 'ConfigDict', 'DictAction', 'collect_env', 'get_logger', + 'print_log', 'is_str', 'iter_cast', 'list_cast', 'tuple_cast', + 'is_seq_of', 'is_list_of', 'is_tuple_of', 'slice_list', 'concat_list', + 'check_prerequisites', 'requires_package', 'requires_executable', + 'is_filepath', 'fopen', 'check_file_exist', 'mkdir_or_exist', + 'symlink', 'scandir', 'ProgressBar', 'track_progress', + 'track_iter_progress', 'track_parallel_progress', 'Registry', + 'build_from_cfg', 'Timer', 'TimerError', 'check_time', 'SyncBatchNorm', + '_AdaptiveAvgPoolNd', '_AdaptiveMaxPoolNd', '_AvgPoolNd', '_BatchNorm', + '_ConvNd', '_ConvTransposeMixin', '_InstanceNorm', '_MaxPoolNd', + 'get_build_config', 'BuildExtension', 'CppExtension', 'CUDAExtension', + 'DataLoader', 'PoolDataLoader', 'TORCH_VERSION', + 'deprecated_api_warning', 'digit_version', 'get_git_hash', + 'import_modules_from_strings', 'jit', 'skip_no_elena', + 'assert_dict_contains_subset', 'assert_attrs_equal', + 'assert_dict_has_keys', 'assert_keys_equal', 'assert_is_norm_layer', + 'assert_params_all_zeros', 'check_python_script', + 'is_method_overridden', 'is_jit_tracing', 'is_rocm_pytorch', + '_get_cuda_home', 'has_method' + ] diff --git a/annotator/uniformer/mmcv/utils/config.py b/annotator/uniformer/mmcv/utils/config.py new file mode 100644 index 0000000000000000000000000000000000000000..17149353aefac6d737c67bb2f35a3a6cd2147b0a --- /dev/null +++ b/annotator/uniformer/mmcv/utils/config.py @@ -0,0 +1,688 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import ast +import copy +import os +import os.path as osp +import platform +import shutil +import sys +import tempfile +import uuid +import warnings +from argparse import Action, ArgumentParser +from collections import abc +from importlib import import_module + +from addict import Dict +from yapf.yapflib.yapf_api import FormatCode + +from .misc import import_modules_from_strings +from .path import check_file_exist + +if platform.system() == 'Windows': + import regex as re +else: + import re + +BASE_KEY = '_base_' +DELETE_KEY = '_delete_' +DEPRECATION_KEY = '_deprecation_' +RESERVED_KEYS = ['filename', 'text', 'pretty_text'] + + +class ConfigDict(Dict): + + def __missing__(self, name): + raise KeyError(name) + + def __getattr__(self, name): + try: + value = super(ConfigDict, self).__getattr__(name) + except KeyError: + ex = AttributeError(f"'{self.__class__.__name__}' object has no " + f"attribute '{name}'") + except Exception as e: + ex = e + else: + return value + raise ex + + +def add_args(parser, cfg, prefix=''): + for k, v in cfg.items(): + if isinstance(v, str): + parser.add_argument('--' + prefix + k) + elif isinstance(v, int): + parser.add_argument('--' + prefix + k, type=int) + elif isinstance(v, float): + parser.add_argument('--' + prefix + k, type=float) + elif isinstance(v, bool): + parser.add_argument('--' + prefix + k, action='store_true') + elif isinstance(v, dict): + add_args(parser, v, prefix + k + '.') + elif isinstance(v, abc.Iterable): + parser.add_argument('--' + prefix + k, type=type(v[0]), nargs='+') + else: + print(f'cannot parse key {prefix + k} of type {type(v)}') + return parser + + +class Config: + """A facility for config and config files. + + It supports common file formats as configs: python/json/yaml. The interface + is the same as a dict object and also allows access config values as + attributes. + + Example: + >>> cfg = Config(dict(a=1, b=dict(b1=[0, 1]))) + >>> cfg.a + 1 + >>> cfg.b + {'b1': [0, 1]} + >>> cfg.b.b1 + [0, 1] + >>> cfg = Config.fromfile('tests/data/config/a.py') + >>> cfg.filename + "/home/kchen/projects/mmcv/tests/data/config/a.py" + >>> cfg.item4 + 'test' + >>> cfg + "Config [path: /home/kchen/projects/mmcv/tests/data/config/a.py]: " + "{'item1': [1, 2], 'item2': {'a': 0}, 'item3': True, 'item4': 'test'}" + """ + + @staticmethod + def _validate_py_syntax(filename): + with open(filename, 'r', encoding='utf-8') as f: + # Setting encoding explicitly to resolve coding issue on windows + content = f.read() + try: + ast.parse(content) + except SyntaxError as e: + raise SyntaxError('There are syntax errors in config ' + f'file {filename}: {e}') + + @staticmethod + def _substitute_predefined_vars(filename, temp_config_name): + file_dirname = osp.dirname(filename) + file_basename = osp.basename(filename) + file_basename_no_extension = osp.splitext(file_basename)[0] + file_extname = osp.splitext(filename)[1] + support_templates = dict( + fileDirname=file_dirname, + fileBasename=file_basename, + fileBasenameNoExtension=file_basename_no_extension, + fileExtname=file_extname) + with open(filename, 'r', encoding='utf-8') as f: + # Setting encoding explicitly to resolve coding issue on windows + config_file = f.read() + for key, value in support_templates.items(): + regexp = r'\{\{\s*' + str(key) + r'\s*\}\}' + value = value.replace('\\', '/') + config_file = re.sub(regexp, value, config_file) + with open(temp_config_name, 'w', encoding='utf-8') as tmp_config_file: + tmp_config_file.write(config_file) + + @staticmethod + def _pre_substitute_base_vars(filename, temp_config_name): + """Substitute base variable placehoders to string, so that parsing + would work.""" + with open(filename, 'r', encoding='utf-8') as f: + # Setting encoding explicitly to resolve coding issue on windows + config_file = f.read() + base_var_dict = {} + regexp = r'\{\{\s*' + BASE_KEY + r'\.([\w\.]+)\s*\}\}' + base_vars = set(re.findall(regexp, config_file)) + for base_var in base_vars: + randstr = f'_{base_var}_{uuid.uuid4().hex.lower()[:6]}' + base_var_dict[randstr] = base_var + regexp = r'\{\{\s*' + BASE_KEY + r'\.' + base_var + r'\s*\}\}' + config_file = re.sub(regexp, f'"{randstr}"', config_file) + with open(temp_config_name, 'w', encoding='utf-8') as tmp_config_file: + tmp_config_file.write(config_file) + return base_var_dict + + @staticmethod + def _substitute_base_vars(cfg, base_var_dict, base_cfg): + """Substitute variable strings to their actual values.""" + cfg = copy.deepcopy(cfg) + + if isinstance(cfg, dict): + for k, v in cfg.items(): + if isinstance(v, str) and v in base_var_dict: + new_v = base_cfg + for new_k in base_var_dict[v].split('.'): + new_v = new_v[new_k] + cfg[k] = new_v + elif isinstance(v, (list, tuple, dict)): + cfg[k] = Config._substitute_base_vars( + v, base_var_dict, base_cfg) + elif isinstance(cfg, tuple): + cfg = tuple( + Config._substitute_base_vars(c, base_var_dict, base_cfg) + for c in cfg) + elif isinstance(cfg, list): + cfg = [ + Config._substitute_base_vars(c, base_var_dict, base_cfg) + for c in cfg + ] + elif isinstance(cfg, str) and cfg in base_var_dict: + new_v = base_cfg + for new_k in base_var_dict[cfg].split('.'): + new_v = new_v[new_k] + cfg = new_v + + return cfg + + @staticmethod + def _file2dict(filename, use_predefined_variables=True): + filename = osp.abspath(osp.expanduser(filename)) + check_file_exist(filename) + fileExtname = osp.splitext(filename)[1] + if fileExtname not in ['.py', '.json', '.yaml', '.yml']: + raise IOError('Only py/yml/yaml/json type are supported now!') + + with tempfile.TemporaryDirectory() as temp_config_dir: + temp_config_file = tempfile.NamedTemporaryFile( + dir=temp_config_dir, suffix=fileExtname) + if platform.system() == 'Windows': + temp_config_file.close() + temp_config_name = osp.basename(temp_config_file.name) + # Substitute predefined variables + if use_predefined_variables: + Config._substitute_predefined_vars(filename, + temp_config_file.name) + else: + shutil.copyfile(filename, temp_config_file.name) + # Substitute base variables from placeholders to strings + base_var_dict = Config._pre_substitute_base_vars( + temp_config_file.name, temp_config_file.name) + + if filename.endswith('.py'): + temp_module_name = osp.splitext(temp_config_name)[0] + sys.path.insert(0, temp_config_dir) + Config._validate_py_syntax(filename) + mod = import_module(temp_module_name) + sys.path.pop(0) + cfg_dict = { + name: value + for name, value in mod.__dict__.items() + if not name.startswith('__') + } + # delete imported module + del sys.modules[temp_module_name] + elif filename.endswith(('.yml', '.yaml', '.json')): + import annotator.uniformer.mmcv as mmcv + cfg_dict = mmcv.load(temp_config_file.name) + # close temp file + temp_config_file.close() + + # check deprecation information + if DEPRECATION_KEY in cfg_dict: + deprecation_info = cfg_dict.pop(DEPRECATION_KEY) + warning_msg = f'The config file {filename} will be deprecated ' \ + 'in the future.' + if 'expected' in deprecation_info: + warning_msg += f' Please use {deprecation_info["expected"]} ' \ + 'instead.' + if 'reference' in deprecation_info: + warning_msg += ' More information can be found at ' \ + f'{deprecation_info["reference"]}' + warnings.warn(warning_msg) + + cfg_text = filename + '\n' + with open(filename, 'r', encoding='utf-8') as f: + # Setting encoding explicitly to resolve coding issue on windows + cfg_text += f.read() + + if BASE_KEY in cfg_dict: + cfg_dir = osp.dirname(filename) + base_filename = cfg_dict.pop(BASE_KEY) + base_filename = base_filename if isinstance( + base_filename, list) else [base_filename] + + cfg_dict_list = list() + cfg_text_list = list() + for f in base_filename: + _cfg_dict, _cfg_text = Config._file2dict(osp.join(cfg_dir, f)) + cfg_dict_list.append(_cfg_dict) + cfg_text_list.append(_cfg_text) + + base_cfg_dict = dict() + for c in cfg_dict_list: + duplicate_keys = base_cfg_dict.keys() & c.keys() + if len(duplicate_keys) > 0: + raise KeyError('Duplicate key is not allowed among bases. ' + f'Duplicate keys: {duplicate_keys}') + base_cfg_dict.update(c) + + # Substitute base variables from strings to their actual values + cfg_dict = Config._substitute_base_vars(cfg_dict, base_var_dict, + base_cfg_dict) + + base_cfg_dict = Config._merge_a_into_b(cfg_dict, base_cfg_dict) + cfg_dict = base_cfg_dict + + # merge cfg_text + cfg_text_list.append(cfg_text) + cfg_text = '\n'.join(cfg_text_list) + + return cfg_dict, cfg_text + + @staticmethod + def _merge_a_into_b(a, b, allow_list_keys=False): + """merge dict ``a`` into dict ``b`` (non-inplace). + + Values in ``a`` will overwrite ``b``. ``b`` is copied first to avoid + in-place modifications. + + Args: + a (dict): The source dict to be merged into ``b``. + b (dict): The origin dict to be fetch keys from ``a``. + allow_list_keys (bool): If True, int string keys (e.g. '0', '1') + are allowed in source ``a`` and will replace the element of the + corresponding index in b if b is a list. Default: False. + + Returns: + dict: The modified dict of ``b`` using ``a``. + + Examples: + # Normally merge a into b. + >>> Config._merge_a_into_b( + ... dict(obj=dict(a=2)), dict(obj=dict(a=1))) + {'obj': {'a': 2}} + + # Delete b first and merge a into b. + >>> Config._merge_a_into_b( + ... dict(obj=dict(_delete_=True, a=2)), dict(obj=dict(a=1))) + {'obj': {'a': 2}} + + # b is a list + >>> Config._merge_a_into_b( + ... {'0': dict(a=2)}, [dict(a=1), dict(b=2)], True) + [{'a': 2}, {'b': 2}] + """ + b = b.copy() + for k, v in a.items(): + if allow_list_keys and k.isdigit() and isinstance(b, list): + k = int(k) + if len(b) <= k: + raise KeyError(f'Index {k} exceeds the length of list {b}') + b[k] = Config._merge_a_into_b(v, b[k], allow_list_keys) + elif isinstance(v, + dict) and k in b and not v.pop(DELETE_KEY, False): + allowed_types = (dict, list) if allow_list_keys else dict + if not isinstance(b[k], allowed_types): + raise TypeError( + f'{k}={v} in child config cannot inherit from base ' + f'because {k} is a dict in the child config but is of ' + f'type {type(b[k])} in base config. You may set ' + f'`{DELETE_KEY}=True` to ignore the base config') + b[k] = Config._merge_a_into_b(v, b[k], allow_list_keys) + else: + b[k] = v + return b + + @staticmethod + def fromfile(filename, + use_predefined_variables=True, + import_custom_modules=True): + cfg_dict, cfg_text = Config._file2dict(filename, + use_predefined_variables) + if import_custom_modules and cfg_dict.get('custom_imports', None): + import_modules_from_strings(**cfg_dict['custom_imports']) + return Config(cfg_dict, cfg_text=cfg_text, filename=filename) + + @staticmethod + def fromstring(cfg_str, file_format): + """Generate config from config str. + + Args: + cfg_str (str): Config str. + file_format (str): Config file format corresponding to the + config str. Only py/yml/yaml/json type are supported now! + + Returns: + obj:`Config`: Config obj. + """ + if file_format not in ['.py', '.json', '.yaml', '.yml']: + raise IOError('Only py/yml/yaml/json type are supported now!') + if file_format != '.py' and 'dict(' in cfg_str: + # check if users specify a wrong suffix for python + warnings.warn( + 'Please check "file_format", the file format may be .py') + with tempfile.NamedTemporaryFile( + 'w', encoding='utf-8', suffix=file_format, + delete=False) as temp_file: + temp_file.write(cfg_str) + # on windows, previous implementation cause error + # see PR 1077 for details + cfg = Config.fromfile(temp_file.name) + os.remove(temp_file.name) + return cfg + + @staticmethod + def auto_argparser(description=None): + """Generate argparser from config file automatically (experimental)""" + partial_parser = ArgumentParser(description=description) + partial_parser.add_argument('config', help='config file path') + cfg_file = partial_parser.parse_known_args()[0].config + cfg = Config.fromfile(cfg_file) + parser = ArgumentParser(description=description) + parser.add_argument('config', help='config file path') + add_args(parser, cfg) + return parser, cfg + + def __init__(self, cfg_dict=None, cfg_text=None, filename=None): + if cfg_dict is None: + cfg_dict = dict() + elif not isinstance(cfg_dict, dict): + raise TypeError('cfg_dict must be a dict, but ' + f'got {type(cfg_dict)}') + for key in cfg_dict: + if key in RESERVED_KEYS: + raise KeyError(f'{key} is reserved for config file') + + super(Config, self).__setattr__('_cfg_dict', ConfigDict(cfg_dict)) + super(Config, self).__setattr__('_filename', filename) + if cfg_text: + text = cfg_text + elif filename: + with open(filename, 'r') as f: + text = f.read() + else: + text = '' + super(Config, self).__setattr__('_text', text) + + @property + def filename(self): + return self._filename + + @property + def text(self): + return self._text + + @property + def pretty_text(self): + + indent = 4 + + def _indent(s_, num_spaces): + s = s_.split('\n') + if len(s) == 1: + return s_ + first = s.pop(0) + s = [(num_spaces * ' ') + line for line in s] + s = '\n'.join(s) + s = first + '\n' + s + return s + + def _format_basic_types(k, v, use_mapping=False): + if isinstance(v, str): + v_str = f"'{v}'" + else: + v_str = str(v) + + if use_mapping: + k_str = f"'{k}'" if isinstance(k, str) else str(k) + attr_str = f'{k_str}: {v_str}' + else: + attr_str = f'{str(k)}={v_str}' + attr_str = _indent(attr_str, indent) + + return attr_str + + def _format_list(k, v, use_mapping=False): + # check if all items in the list are dict + if all(isinstance(_, dict) for _ in v): + v_str = '[\n' + v_str += '\n'.join( + f'dict({_indent(_format_dict(v_), indent)}),' + for v_ in v).rstrip(',') + if use_mapping: + k_str = f"'{k}'" if isinstance(k, str) else str(k) + attr_str = f'{k_str}: {v_str}' + else: + attr_str = f'{str(k)}={v_str}' + attr_str = _indent(attr_str, indent) + ']' + else: + attr_str = _format_basic_types(k, v, use_mapping) + return attr_str + + def _contain_invalid_identifier(dict_str): + contain_invalid_identifier = False + for key_name in dict_str: + contain_invalid_identifier |= \ + (not str(key_name).isidentifier()) + return contain_invalid_identifier + + def _format_dict(input_dict, outest_level=False): + r = '' + s = [] + + use_mapping = _contain_invalid_identifier(input_dict) + if use_mapping: + r += '{' + for idx, (k, v) in enumerate(input_dict.items()): + is_last = idx >= len(input_dict) - 1 + end = '' if outest_level or is_last else ',' + if isinstance(v, dict): + v_str = '\n' + _format_dict(v) + if use_mapping: + k_str = f"'{k}'" if isinstance(k, str) else str(k) + attr_str = f'{k_str}: dict({v_str}' + else: + attr_str = f'{str(k)}=dict({v_str}' + attr_str = _indent(attr_str, indent) + ')' + end + elif isinstance(v, list): + attr_str = _format_list(k, v, use_mapping) + end + else: + attr_str = _format_basic_types(k, v, use_mapping) + end + + s.append(attr_str) + r += '\n'.join(s) + if use_mapping: + r += '}' + return r + + cfg_dict = self._cfg_dict.to_dict() + text = _format_dict(cfg_dict, outest_level=True) + # copied from setup.cfg + yapf_style = dict( + based_on_style='pep8', + blank_line_before_nested_class_or_def=True, + split_before_expression_after_opening_paren=True) + text, _ = FormatCode(text, style_config=yapf_style, verify=True) + + return text + + def __repr__(self): + return f'Config (path: {self.filename}): {self._cfg_dict.__repr__()}' + + def __len__(self): + return len(self._cfg_dict) + + def __getattr__(self, name): + return getattr(self._cfg_dict, name) + + def __getitem__(self, name): + return self._cfg_dict.__getitem__(name) + + def __setattr__(self, name, value): + if isinstance(value, dict): + value = ConfigDict(value) + self._cfg_dict.__setattr__(name, value) + + def __setitem__(self, name, value): + if isinstance(value, dict): + value = ConfigDict(value) + self._cfg_dict.__setitem__(name, value) + + def __iter__(self): + return iter(self._cfg_dict) + + def __getstate__(self): + return (self._cfg_dict, self._filename, self._text) + + def __setstate__(self, state): + _cfg_dict, _filename, _text = state + super(Config, self).__setattr__('_cfg_dict', _cfg_dict) + super(Config, self).__setattr__('_filename', _filename) + super(Config, self).__setattr__('_text', _text) + + def dump(self, file=None): + cfg_dict = super(Config, self).__getattribute__('_cfg_dict').to_dict() + if self.filename.endswith('.py'): + if file is None: + return self.pretty_text + else: + with open(file, 'w', encoding='utf-8') as f: + f.write(self.pretty_text) + else: + import annotator.uniformer.mmcv as mmcv + if file is None: + file_format = self.filename.split('.')[-1] + return mmcv.dump(cfg_dict, file_format=file_format) + else: + mmcv.dump(cfg_dict, file) + + def merge_from_dict(self, options, allow_list_keys=True): + """Merge list into cfg_dict. + + Merge the dict parsed by MultipleKVAction into this cfg. + + Examples: + >>> options = {'model.backbone.depth': 50, + ... 'model.backbone.with_cp':True} + >>> cfg = Config(dict(model=dict(backbone=dict(type='ResNet')))) + >>> cfg.merge_from_dict(options) + >>> cfg_dict = super(Config, self).__getattribute__('_cfg_dict') + >>> assert cfg_dict == dict( + ... model=dict(backbone=dict(depth=50, with_cp=True))) + + # Merge list element + >>> cfg = Config(dict(pipeline=[ + ... dict(type='LoadImage'), dict(type='LoadAnnotations')])) + >>> options = dict(pipeline={'0': dict(type='SelfLoadImage')}) + >>> cfg.merge_from_dict(options, allow_list_keys=True) + >>> cfg_dict = super(Config, self).__getattribute__('_cfg_dict') + >>> assert cfg_dict == dict(pipeline=[ + ... dict(type='SelfLoadImage'), dict(type='LoadAnnotations')]) + + Args: + options (dict): dict of configs to merge from. + allow_list_keys (bool): If True, int string keys (e.g. '0', '1') + are allowed in ``options`` and will replace the element of the + corresponding index in the config if the config is a list. + Default: True. + """ + option_cfg_dict = {} + for full_key, v in options.items(): + d = option_cfg_dict + key_list = full_key.split('.') + for subkey in key_list[:-1]: + d.setdefault(subkey, ConfigDict()) + d = d[subkey] + subkey = key_list[-1] + d[subkey] = v + + cfg_dict = super(Config, self).__getattribute__('_cfg_dict') + super(Config, self).__setattr__( + '_cfg_dict', + Config._merge_a_into_b( + option_cfg_dict, cfg_dict, allow_list_keys=allow_list_keys)) + + +class DictAction(Action): + """ + argparse action to split an argument into KEY=VALUE form + on the first = and append to a dictionary. List options can + be passed as comma separated values, i.e 'KEY=V1,V2,V3', or with explicit + brackets, i.e. 'KEY=[V1,V2,V3]'. It also support nested brackets to build + list/tuple values. e.g. 'KEY=[(V1,V2),(V3,V4)]' + """ + + @staticmethod + def _parse_int_float_bool(val): + try: + return int(val) + except ValueError: + pass + try: + return float(val) + except ValueError: + pass + if val.lower() in ['true', 'false']: + return True if val.lower() == 'true' else False + return val + + @staticmethod + def _parse_iterable(val): + """Parse iterable values in the string. + + All elements inside '()' or '[]' are treated as iterable values. + + Args: + val (str): Value string. + + Returns: + list | tuple: The expanded list or tuple from the string. + + Examples: + >>> DictAction._parse_iterable('1,2,3') + [1, 2, 3] + >>> DictAction._parse_iterable('[a, b, c]') + ['a', 'b', 'c'] + >>> DictAction._parse_iterable('[(1, 2, 3), [a, b], c]') + [(1, 2, 3), ['a', 'b'], 'c'] + """ + + def find_next_comma(string): + """Find the position of next comma in the string. + + If no ',' is found in the string, return the string length. All + chars inside '()' and '[]' are treated as one element and thus ',' + inside these brackets are ignored. + """ + assert (string.count('(') == string.count(')')) and ( + string.count('[') == string.count(']')), \ + f'Imbalanced brackets exist in {string}' + end = len(string) + for idx, char in enumerate(string): + pre = string[:idx] + # The string before this ',' is balanced + if ((char == ',') and (pre.count('(') == pre.count(')')) + and (pre.count('[') == pre.count(']'))): + end = idx + break + return end + + # Strip ' and " characters and replace whitespace. + val = val.strip('\'\"').replace(' ', '') + is_tuple = False + if val.startswith('(') and val.endswith(')'): + is_tuple = True + val = val[1:-1] + elif val.startswith('[') and val.endswith(']'): + val = val[1:-1] + elif ',' not in val: + # val is a single value + return DictAction._parse_int_float_bool(val) + + values = [] + while len(val) > 0: + comma_idx = find_next_comma(val) + element = DictAction._parse_iterable(val[:comma_idx]) + values.append(element) + val = val[comma_idx + 1:] + if is_tuple: + values = tuple(values) + return values + + def __call__(self, parser, namespace, values, option_string=None): + options = {} + for kv in values: + key, val = kv.split('=', maxsplit=1) + options[key] = self._parse_iterable(val) + setattr(namespace, self.dest, options) diff --git a/annotator/uniformer/mmcv/utils/env.py b/annotator/uniformer/mmcv/utils/env.py new file mode 100644 index 0000000000000000000000000000000000000000..e3f0d92529e193e6d8339419bcd9bed7901a7769 --- /dev/null +++ b/annotator/uniformer/mmcv/utils/env.py @@ -0,0 +1,95 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""This file holding some environment constant for sharing by other files.""" + +import os.path as osp +import subprocess +import sys +from collections import defaultdict + +import cv2 +import torch + +import annotator.uniformer.mmcv as mmcv +from .parrots_wrapper import get_build_config + + +def collect_env(): + """Collect the information of the running environments. + + Returns: + dict: The environment information. The following fields are contained. + + - sys.platform: The variable of ``sys.platform``. + - Python: Python version. + - CUDA available: Bool, indicating if CUDA is available. + - GPU devices: Device type of each GPU. + - CUDA_HOME (optional): The env var ``CUDA_HOME``. + - NVCC (optional): NVCC version. + - GCC: GCC version, "n/a" if GCC is not installed. + - PyTorch: PyTorch version. + - PyTorch compiling details: The output of \ + ``torch.__config__.show()``. + - TorchVision (optional): TorchVision version. + - OpenCV: OpenCV version. + - MMCV: MMCV version. + - MMCV Compiler: The GCC version for compiling MMCV ops. + - MMCV CUDA Compiler: The CUDA version for compiling MMCV ops. + """ + env_info = {} + env_info['sys.platform'] = sys.platform + env_info['Python'] = sys.version.replace('\n', '') + + cuda_available = torch.cuda.is_available() + env_info['CUDA available'] = cuda_available + + if cuda_available: + devices = defaultdict(list) + for k in range(torch.cuda.device_count()): + devices[torch.cuda.get_device_name(k)].append(str(k)) + for name, device_ids in devices.items(): + env_info['GPU ' + ','.join(device_ids)] = name + + from annotator.uniformer.mmcv.utils.parrots_wrapper import _get_cuda_home + CUDA_HOME = _get_cuda_home() + env_info['CUDA_HOME'] = CUDA_HOME + + if CUDA_HOME is not None and osp.isdir(CUDA_HOME): + try: + nvcc = osp.join(CUDA_HOME, 'bin/nvcc') + nvcc = subprocess.check_output( + f'"{nvcc}" -V | tail -n1', shell=True) + nvcc = nvcc.decode('utf-8').strip() + except subprocess.SubprocessError: + nvcc = 'Not Available' + env_info['NVCC'] = nvcc + + try: + gcc = subprocess.check_output('gcc --version | head -n1', shell=True) + gcc = gcc.decode('utf-8').strip() + env_info['GCC'] = gcc + except subprocess.CalledProcessError: # gcc is unavailable + env_info['GCC'] = 'n/a' + + env_info['PyTorch'] = torch.__version__ + env_info['PyTorch compiling details'] = get_build_config() + + try: + import torchvision + env_info['TorchVision'] = torchvision.__version__ + except ModuleNotFoundError: + pass + + env_info['OpenCV'] = cv2.__version__ + + env_info['MMCV'] = mmcv.__version__ + + try: + from annotator.uniformer.mmcv.ops import get_compiler_version, get_compiling_cuda_version + except ModuleNotFoundError: + env_info['MMCV Compiler'] = 'n/a' + env_info['MMCV CUDA Compiler'] = 'n/a' + else: + env_info['MMCV Compiler'] = get_compiler_version() + env_info['MMCV CUDA Compiler'] = get_compiling_cuda_version() + + return env_info diff --git a/annotator/uniformer/mmcv/utils/ext_loader.py b/annotator/uniformer/mmcv/utils/ext_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..08132d2c1b9a1c28880e4bab4d4fa1ba39d9d083 --- /dev/null +++ b/annotator/uniformer/mmcv/utils/ext_loader.py @@ -0,0 +1,71 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import importlib +import os +import pkgutil +import warnings +from collections import namedtuple + +import torch + +if torch.__version__ != 'parrots': + + def load_ext(name, funcs): + ext = importlib.import_module('mmcv.' + name) + for fun in funcs: + assert hasattr(ext, fun), f'{fun} miss in module {name}' + return ext +else: + from parrots import extension + from parrots.base import ParrotsException + + has_return_value_ops = [ + 'nms', + 'softnms', + 'nms_match', + 'nms_rotated', + 'top_pool_forward', + 'top_pool_backward', + 'bottom_pool_forward', + 'bottom_pool_backward', + 'left_pool_forward', + 'left_pool_backward', + 'right_pool_forward', + 'right_pool_backward', + 'fused_bias_leakyrelu', + 'upfirdn2d', + 'ms_deform_attn_forward', + 'pixel_group', + 'contour_expand', + ] + + def get_fake_func(name, e): + + def fake_func(*args, **kwargs): + warnings.warn(f'{name} is not supported in parrots now') + raise e + + return fake_func + + def load_ext(name, funcs): + ExtModule = namedtuple('ExtModule', funcs) + ext_list = [] + lib_root = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + for fun in funcs: + try: + ext_fun = extension.load(fun, name, lib_dir=lib_root) + except ParrotsException as e: + if 'No element registered' not in e.message: + warnings.warn(e.message) + ext_fun = get_fake_func(fun, e) + ext_list.append(ext_fun) + else: + if fun in has_return_value_ops: + ext_list.append(ext_fun.op) + else: + ext_list.append(ext_fun.op_) + return ExtModule(*ext_list) + + +def check_ops_exist(): + ext_loader = pkgutil.find_loader('mmcv._ext') + return ext_loader is not None diff --git a/annotator/uniformer/mmcv/utils/logging.py b/annotator/uniformer/mmcv/utils/logging.py new file mode 100644 index 0000000000000000000000000000000000000000..4aa0e04bb9b3ab2a4bfbc4def50404ccbac2c6e6 --- /dev/null +++ b/annotator/uniformer/mmcv/utils/logging.py @@ -0,0 +1,110 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import logging + +import torch.distributed as dist + +logger_initialized = {} + + +def get_logger(name, log_file=None, log_level=logging.INFO, file_mode='w'): + """Initialize and get a logger by name. + + If the logger has not been initialized, this method will initialize the + logger by adding one or two handlers, otherwise the initialized logger will + be directly returned. During initialization, a StreamHandler will always be + added. If `log_file` is specified and the process rank is 0, a FileHandler + will also be added. + + Args: + name (str): Logger name. + log_file (str | None): The log filename. If specified, a FileHandler + will be added to the logger. + log_level (int): The logger level. Note that only the process of + rank 0 is affected, and other processes will set the level to + "Error" thus be silent most of the time. + file_mode (str): The file mode used in opening log file. + Defaults to 'w'. + + Returns: + logging.Logger: The expected logger. + """ + logger = logging.getLogger(name) + if name in logger_initialized: + return logger + # handle hierarchical names + # e.g., logger "a" is initialized, then logger "a.b" will skip the + # initialization since it is a child of "a". + for logger_name in logger_initialized: + if name.startswith(logger_name): + return logger + + # handle duplicate logs to the console + # Starting in 1.8.0, PyTorch DDP attaches a StreamHandler (NOTSET) + # to the root logger. As logger.propagate is True by default, this root + # level handler causes logging messages from rank>0 processes to + # unexpectedly show up on the console, creating much unwanted clutter. + # To fix this issue, we set the root logger's StreamHandler, if any, to log + # at the ERROR level. + for handler in logger.root.handlers: + if type(handler) is logging.StreamHandler: + handler.setLevel(logging.ERROR) + + stream_handler = logging.StreamHandler() + handlers = [stream_handler] + + if dist.is_available() and dist.is_initialized(): + rank = dist.get_rank() + else: + rank = 0 + + # only rank 0 will add a FileHandler + if rank == 0 and log_file is not None: + # Here, the default behaviour of the official logger is 'a'. Thus, we + # provide an interface to change the file mode to the default + # behaviour. + file_handler = logging.FileHandler(log_file, file_mode) + handlers.append(file_handler) + + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + for handler in handlers: + handler.setFormatter(formatter) + handler.setLevel(log_level) + logger.addHandler(handler) + + if rank == 0: + logger.setLevel(log_level) + else: + logger.setLevel(logging.ERROR) + + logger_initialized[name] = True + + return logger + + +def print_log(msg, logger=None, level=logging.INFO): + """Print a log message. + + Args: + msg (str): The message to be logged. + logger (logging.Logger | str | None): The logger to be used. + Some special loggers are: + - "silent": no message will be printed. + - other str: the logger obtained with `get_root_logger(logger)`. + - None: The `print()` method will be used to print log messages. + level (int): Logging level. Only available when `logger` is a Logger + object or "root". + """ + if logger is None: + print(msg) + elif isinstance(logger, logging.Logger): + logger.log(level, msg) + elif logger == 'silent': + pass + elif isinstance(logger, str): + _logger = get_logger(logger) + _logger.log(level, msg) + else: + raise TypeError( + 'logger should be either a logging.Logger object, str, ' + f'"silent" or None, but got {type(logger)}') diff --git a/annotator/uniformer/mmcv/utils/misc.py b/annotator/uniformer/mmcv/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..2c58d0d7fee9fe3d4519270ad8c1e998d0d8a18c --- /dev/null +++ b/annotator/uniformer/mmcv/utils/misc.py @@ -0,0 +1,377 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import collections.abc +import functools +import itertools +import subprocess +import warnings +from collections import abc +from importlib import import_module +from inspect import getfullargspec +from itertools import repeat + + +# From PyTorch internals +def _ntuple(n): + + def parse(x): + if isinstance(x, collections.abc.Iterable): + return x + return tuple(repeat(x, n)) + + return parse + + +to_1tuple = _ntuple(1) +to_2tuple = _ntuple(2) +to_3tuple = _ntuple(3) +to_4tuple = _ntuple(4) +to_ntuple = _ntuple + + +def is_str(x): + """Whether the input is an string instance. + + Note: This method is deprecated since python 2 is no longer supported. + """ + return isinstance(x, str) + + +def import_modules_from_strings(imports, allow_failed_imports=False): + """Import modules from the given list of strings. + + Args: + imports (list | str | None): The given module names to be imported. + allow_failed_imports (bool): If True, the failed imports will return + None. Otherwise, an ImportError is raise. Default: False. + + Returns: + list[module] | module | None: The imported modules. + + Examples: + >>> osp, sys = import_modules_from_strings( + ... ['os.path', 'sys']) + >>> import os.path as osp_ + >>> import sys as sys_ + >>> assert osp == osp_ + >>> assert sys == sys_ + """ + if not imports: + return + single_import = False + if isinstance(imports, str): + single_import = True + imports = [imports] + if not isinstance(imports, list): + raise TypeError( + f'custom_imports must be a list but got type {type(imports)}') + imported = [] + for imp in imports: + if not isinstance(imp, str): + raise TypeError( + f'{imp} is of type {type(imp)} and cannot be imported.') + try: + imported_tmp = import_module(imp) + except ImportError: + if allow_failed_imports: + warnings.warn(f'{imp} failed to import and is ignored.', + UserWarning) + imported_tmp = None + else: + raise ImportError + imported.append(imported_tmp) + if single_import: + imported = imported[0] + return imported + + +def iter_cast(inputs, dst_type, return_type=None): + """Cast elements of an iterable object into some type. + + Args: + inputs (Iterable): The input object. + dst_type (type): Destination type. + return_type (type, optional): If specified, the output object will be + converted to this type, otherwise an iterator. + + Returns: + iterator or specified type: The converted object. + """ + if not isinstance(inputs, abc.Iterable): + raise TypeError('inputs must be an iterable object') + if not isinstance(dst_type, type): + raise TypeError('"dst_type" must be a valid type') + + out_iterable = map(dst_type, inputs) + + if return_type is None: + return out_iterable + else: + return return_type(out_iterable) + + +def list_cast(inputs, dst_type): + """Cast elements of an iterable object into a list of some type. + + A partial method of :func:`iter_cast`. + """ + return iter_cast(inputs, dst_type, return_type=list) + + +def tuple_cast(inputs, dst_type): + """Cast elements of an iterable object into a tuple of some type. + + A partial method of :func:`iter_cast`. + """ + return iter_cast(inputs, dst_type, return_type=tuple) + + +def is_seq_of(seq, expected_type, seq_type=None): + """Check whether it is a sequence of some type. + + Args: + seq (Sequence): The sequence to be checked. + expected_type (type): Expected type of sequence items. + seq_type (type, optional): Expected sequence type. + + Returns: + bool: Whether the sequence is valid. + """ + if seq_type is None: + exp_seq_type = abc.Sequence + else: + assert isinstance(seq_type, type) + exp_seq_type = seq_type + if not isinstance(seq, exp_seq_type): + return False + for item in seq: + if not isinstance(item, expected_type): + return False + return True + + +def is_list_of(seq, expected_type): + """Check whether it is a list of some type. + + A partial method of :func:`is_seq_of`. + """ + return is_seq_of(seq, expected_type, seq_type=list) + + +def is_tuple_of(seq, expected_type): + """Check whether it is a tuple of some type. + + A partial method of :func:`is_seq_of`. + """ + return is_seq_of(seq, expected_type, seq_type=tuple) + + +def slice_list(in_list, lens): + """Slice a list into several sub lists by a list of given length. + + Args: + in_list (list): The list to be sliced. + lens(int or list): The expected length of each out list. + + Returns: + list: A list of sliced list. + """ + if isinstance(lens, int): + assert len(in_list) % lens == 0 + lens = [lens] * int(len(in_list) / lens) + if not isinstance(lens, list): + raise TypeError('"indices" must be an integer or a list of integers') + elif sum(lens) != len(in_list): + raise ValueError('sum of lens and list length does not ' + f'match: {sum(lens)} != {len(in_list)}') + out_list = [] + idx = 0 + for i in range(len(lens)): + out_list.append(in_list[idx:idx + lens[i]]) + idx += lens[i] + return out_list + + +def concat_list(in_list): + """Concatenate a list of list into a single list. + + Args: + in_list (list): The list of list to be merged. + + Returns: + list: The concatenated flat list. + """ + return list(itertools.chain(*in_list)) + + +def check_prerequisites( + prerequisites, + checker, + msg_tmpl='Prerequisites "{}" are required in method "{}" but not ' + 'found, please install them first.'): # yapf: disable + """A decorator factory to check if prerequisites are satisfied. + + Args: + prerequisites (str of list[str]): Prerequisites to be checked. + checker (callable): The checker method that returns True if a + prerequisite is meet, False otherwise. + msg_tmpl (str): The message template with two variables. + + Returns: + decorator: A specific decorator. + """ + + def wrap(func): + + @functools.wraps(func) + def wrapped_func(*args, **kwargs): + requirements = [prerequisites] if isinstance( + prerequisites, str) else prerequisites + missing = [] + for item in requirements: + if not checker(item): + missing.append(item) + if missing: + print(msg_tmpl.format(', '.join(missing), func.__name__)) + raise RuntimeError('Prerequisites not meet.') + else: + return func(*args, **kwargs) + + return wrapped_func + + return wrap + + +def _check_py_package(package): + try: + import_module(package) + except ImportError: + return False + else: + return True + + +def _check_executable(cmd): + if subprocess.call(f'which {cmd}', shell=True) != 0: + return False + else: + return True + + +def requires_package(prerequisites): + """A decorator to check if some python packages are installed. + + Example: + >>> @requires_package('numpy') + >>> func(arg1, args): + >>> return numpy.zeros(1) + array([0.]) + >>> @requires_package(['numpy', 'non_package']) + >>> func(arg1, args): + >>> return numpy.zeros(1) + ImportError + """ + return check_prerequisites(prerequisites, checker=_check_py_package) + + +def requires_executable(prerequisites): + """A decorator to check if some executable files are installed. + + Example: + >>> @requires_executable('ffmpeg') + >>> func(arg1, args): + >>> print(1) + 1 + """ + return check_prerequisites(prerequisites, checker=_check_executable) + + +def deprecated_api_warning(name_dict, cls_name=None): + """A decorator to check if some arguments are deprecate and try to replace + deprecate src_arg_name to dst_arg_name. + + Args: + name_dict(dict): + key (str): Deprecate argument names. + val (str): Expected argument names. + + Returns: + func: New function. + """ + + def api_warning_wrapper(old_func): + + @functools.wraps(old_func) + def new_func(*args, **kwargs): + # get the arg spec of the decorated method + args_info = getfullargspec(old_func) + # get name of the function + func_name = old_func.__name__ + if cls_name is not None: + func_name = f'{cls_name}.{func_name}' + if args: + arg_names = args_info.args[:len(args)] + for src_arg_name, dst_arg_name in name_dict.items(): + if src_arg_name in arg_names: + warnings.warn( + f'"{src_arg_name}" is deprecated in ' + f'`{func_name}`, please use "{dst_arg_name}" ' + 'instead') + arg_names[arg_names.index(src_arg_name)] = dst_arg_name + if kwargs: + for src_arg_name, dst_arg_name in name_dict.items(): + if src_arg_name in kwargs: + + assert dst_arg_name not in kwargs, ( + f'The expected behavior is to replace ' + f'the deprecated key `{src_arg_name}` to ' + f'new key `{dst_arg_name}`, but got them ' + f'in the arguments at the same time, which ' + f'is confusing. `{src_arg_name} will be ' + f'deprecated in the future, please ' + f'use `{dst_arg_name}` instead.') + + warnings.warn( + f'"{src_arg_name}" is deprecated in ' + f'`{func_name}`, please use "{dst_arg_name}" ' + 'instead') + kwargs[dst_arg_name] = kwargs.pop(src_arg_name) + + # apply converted arguments to the decorated method + output = old_func(*args, **kwargs) + return output + + return new_func + + return api_warning_wrapper + + +def is_method_overridden(method, base_class, derived_class): + """Check if a method of base class is overridden in derived class. + + Args: + method (str): the method name to check. + base_class (type): the class of the base class. + derived_class (type | Any): the class or instance of the derived class. + """ + assert isinstance(base_class, type), \ + "base_class doesn't accept instance, Please pass class instead." + + if not isinstance(derived_class, type): + derived_class = derived_class.__class__ + + base_method = getattr(base_class, method) + derived_method = getattr(derived_class, method) + return derived_method != base_method + + +def has_method(obj: object, method: str) -> bool: + """Check whether the object has a method. + + Args: + method (str): The method name to check. + obj (object): The object to check. + + Returns: + bool: True if the object has the method else False. + """ + return hasattr(obj, method) and callable(getattr(obj, method)) diff --git a/annotator/uniformer/mmcv/utils/parrots_jit.py b/annotator/uniformer/mmcv/utils/parrots_jit.py new file mode 100644 index 0000000000000000000000000000000000000000..61873f6dbb9b10ed972c90aa8faa321e3cb3249e --- /dev/null +++ b/annotator/uniformer/mmcv/utils/parrots_jit.py @@ -0,0 +1,41 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os + +from .parrots_wrapper import TORCH_VERSION + +parrots_jit_option = os.getenv('PARROTS_JIT_OPTION') + +if TORCH_VERSION == 'parrots' and parrots_jit_option == 'ON': + from parrots.jit import pat as jit +else: + + def jit(func=None, + check_input=None, + full_shape=True, + derivate=False, + coderize=False, + optimize=False): + + def wrapper(func): + + def wrapper_inner(*args, **kargs): + return func(*args, **kargs) + + return wrapper_inner + + if func is None: + return wrapper + else: + return func + + +if TORCH_VERSION == 'parrots': + from parrots.utils.tester import skip_no_elena +else: + + def skip_no_elena(func): + + def wrapper(*args, **kargs): + return func(*args, **kargs) + + return wrapper diff --git a/annotator/uniformer/mmcv/utils/parrots_wrapper.py b/annotator/uniformer/mmcv/utils/parrots_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..93c97640d4b9ed088ca82cfe03e6efebfcfa9dbf --- /dev/null +++ b/annotator/uniformer/mmcv/utils/parrots_wrapper.py @@ -0,0 +1,107 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from functools import partial + +import torch + +TORCH_VERSION = torch.__version__ + + +def is_rocm_pytorch() -> bool: + is_rocm = False + if TORCH_VERSION != 'parrots': + try: + from torch.utils.cpp_extension import ROCM_HOME + is_rocm = True if ((torch.version.hip is not None) and + (ROCM_HOME is not None)) else False + except ImportError: + pass + return is_rocm + + +def _get_cuda_home(): + if TORCH_VERSION == 'parrots': + from parrots.utils.build_extension import CUDA_HOME + else: + if is_rocm_pytorch(): + from torch.utils.cpp_extension import ROCM_HOME + CUDA_HOME = ROCM_HOME + else: + from torch.utils.cpp_extension import CUDA_HOME + return CUDA_HOME + + +def get_build_config(): + if TORCH_VERSION == 'parrots': + from parrots.config import get_build_info + return get_build_info() + else: + return torch.__config__.show() + + +def _get_conv(): + if TORCH_VERSION == 'parrots': + from parrots.nn.modules.conv import _ConvNd, _ConvTransposeMixin + else: + from torch.nn.modules.conv import _ConvNd, _ConvTransposeMixin + return _ConvNd, _ConvTransposeMixin + + +def _get_dataloader(): + if TORCH_VERSION == 'parrots': + from torch.utils.data import DataLoader, PoolDataLoader + else: + from torch.utils.data import DataLoader + PoolDataLoader = DataLoader + return DataLoader, PoolDataLoader + + +def _get_extension(): + if TORCH_VERSION == 'parrots': + from parrots.utils.build_extension import BuildExtension, Extension + CppExtension = partial(Extension, cuda=False) + CUDAExtension = partial(Extension, cuda=True) + else: + from torch.utils.cpp_extension import (BuildExtension, CppExtension, + CUDAExtension) + return BuildExtension, CppExtension, CUDAExtension + + +def _get_pool(): + if TORCH_VERSION == 'parrots': + from parrots.nn.modules.pool import (_AdaptiveAvgPoolNd, + _AdaptiveMaxPoolNd, _AvgPoolNd, + _MaxPoolNd) + else: + from torch.nn.modules.pooling import (_AdaptiveAvgPoolNd, + _AdaptiveMaxPoolNd, _AvgPoolNd, + _MaxPoolNd) + return _AdaptiveAvgPoolNd, _AdaptiveMaxPoolNd, _AvgPoolNd, _MaxPoolNd + + +def _get_norm(): + if TORCH_VERSION == 'parrots': + from parrots.nn.modules.batchnorm import _BatchNorm, _InstanceNorm + SyncBatchNorm_ = torch.nn.SyncBatchNorm2d + else: + from torch.nn.modules.instancenorm import _InstanceNorm + from torch.nn.modules.batchnorm import _BatchNorm + SyncBatchNorm_ = torch.nn.SyncBatchNorm + return _BatchNorm, _InstanceNorm, SyncBatchNorm_ + + +_ConvNd, _ConvTransposeMixin = _get_conv() +DataLoader, PoolDataLoader = _get_dataloader() +BuildExtension, CppExtension, CUDAExtension = _get_extension() +_BatchNorm, _InstanceNorm, SyncBatchNorm_ = _get_norm() +_AdaptiveAvgPoolNd, _AdaptiveMaxPoolNd, _AvgPoolNd, _MaxPoolNd = _get_pool() + + +class SyncBatchNorm(SyncBatchNorm_): + + def _check_input_dim(self, input): + if TORCH_VERSION == 'parrots': + if input.dim() < 2: + raise ValueError( + f'expected at least 2D input (got {input.dim()}D input)') + else: + super()._check_input_dim(input) diff --git a/annotator/uniformer/mmcv/utils/path.py b/annotator/uniformer/mmcv/utils/path.py new file mode 100644 index 0000000000000000000000000000000000000000..7dab4b3041413b1432b0f434b8b14783097d33c6 --- /dev/null +++ b/annotator/uniformer/mmcv/utils/path.py @@ -0,0 +1,101 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os +import os.path as osp +from pathlib import Path + +from .misc import is_str + + +def is_filepath(x): + return is_str(x) or isinstance(x, Path) + + +def fopen(filepath, *args, **kwargs): + if is_str(filepath): + return open(filepath, *args, **kwargs) + elif isinstance(filepath, Path): + return filepath.open(*args, **kwargs) + raise ValueError('`filepath` should be a string or a Path') + + +def check_file_exist(filename, msg_tmpl='file "{}" does not exist'): + if not osp.isfile(filename): + raise FileNotFoundError(msg_tmpl.format(filename)) + + +def mkdir_or_exist(dir_name, mode=0o777): + if dir_name == '': + return + dir_name = osp.expanduser(dir_name) + os.makedirs(dir_name, mode=mode, exist_ok=True) + + +def symlink(src, dst, overwrite=True, **kwargs): + if os.path.lexists(dst) and overwrite: + os.remove(dst) + os.symlink(src, dst, **kwargs) + + +def scandir(dir_path, suffix=None, recursive=False, case_sensitive=True): + """Scan a directory to find the interested files. + + Args: + dir_path (str | obj:`Path`): Path of the directory. + suffix (str | tuple(str), optional): File suffix that we are + interested in. Default: None. + recursive (bool, optional): If set to True, recursively scan the + directory. Default: False. + case_sensitive (bool, optional) : If set to False, ignore the case of + suffix. Default: True. + + Returns: + A generator for all the interested files with relative paths. + """ + if isinstance(dir_path, (str, Path)): + dir_path = str(dir_path) + else: + raise TypeError('"dir_path" must be a string or Path object') + + if (suffix is not None) and not isinstance(suffix, (str, tuple)): + raise TypeError('"suffix" must be a string or tuple of strings') + + if suffix is not None and not case_sensitive: + suffix = suffix.lower() if isinstance(suffix, str) else tuple( + item.lower() for item in suffix) + + root = dir_path + + def _scandir(dir_path, suffix, recursive, case_sensitive): + for entry in os.scandir(dir_path): + if not entry.name.startswith('.') and entry.is_file(): + rel_path = osp.relpath(entry.path, root) + _rel_path = rel_path if case_sensitive else rel_path.lower() + if suffix is None or _rel_path.endswith(suffix): + yield rel_path + elif recursive and os.path.isdir(entry.path): + # scan recursively if entry.path is a directory + yield from _scandir(entry.path, suffix, recursive, + case_sensitive) + + return _scandir(dir_path, suffix, recursive, case_sensitive) + + +def find_vcs_root(path, markers=('.git', )): + """Finds the root directory (including itself) of specified markers. + + Args: + path (str): Path of directory or file. + markers (list[str], optional): List of file or directory names. + + Returns: + The directory contained one of the markers or None if not found. + """ + if osp.isfile(path): + path = osp.dirname(path) + + prev, cur = None, osp.abspath(osp.expanduser(path)) + while cur != prev: + if any(osp.exists(osp.join(cur, marker)) for marker in markers): + return cur + prev, cur = cur, osp.split(cur)[0] + return None diff --git a/annotator/uniformer/mmcv/utils/progressbar.py b/annotator/uniformer/mmcv/utils/progressbar.py new file mode 100644 index 0000000000000000000000000000000000000000..0062f670dd94fa9da559ab26ef85517dcf5211c7 --- /dev/null +++ b/annotator/uniformer/mmcv/utils/progressbar.py @@ -0,0 +1,208 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import sys +from collections.abc import Iterable +from multiprocessing import Pool +from shutil import get_terminal_size + +from .timer import Timer + + +class ProgressBar: + """A progress bar which can print the progress.""" + + def __init__(self, task_num=0, bar_width=50, start=True, file=sys.stdout): + self.task_num = task_num + self.bar_width = bar_width + self.completed = 0 + self.file = file + if start: + self.start() + + @property + def terminal_width(self): + width, _ = get_terminal_size() + return width + + def start(self): + if self.task_num > 0: + self.file.write(f'[{" " * self.bar_width}] 0/{self.task_num}, ' + 'elapsed: 0s, ETA:') + else: + self.file.write('completed: 0, elapsed: 0s') + self.file.flush() + self.timer = Timer() + + def update(self, num_tasks=1): + assert num_tasks > 0 + self.completed += num_tasks + elapsed = self.timer.since_start() + if elapsed > 0: + fps = self.completed / elapsed + else: + fps = float('inf') + if self.task_num > 0: + percentage = self.completed / float(self.task_num) + eta = int(elapsed * (1 - percentage) / percentage + 0.5) + msg = f'\r[{{}}] {self.completed}/{self.task_num}, ' \ + f'{fps:.1f} task/s, elapsed: {int(elapsed + 0.5)}s, ' \ + f'ETA: {eta:5}s' + + bar_width = min(self.bar_width, + int(self.terminal_width - len(msg)) + 2, + int(self.terminal_width * 0.6)) + bar_width = max(2, bar_width) + mark_width = int(bar_width * percentage) + bar_chars = '>' * mark_width + ' ' * (bar_width - mark_width) + self.file.write(msg.format(bar_chars)) + else: + self.file.write( + f'completed: {self.completed}, elapsed: {int(elapsed + 0.5)}s,' + f' {fps:.1f} tasks/s') + self.file.flush() + + +def track_progress(func, tasks, bar_width=50, file=sys.stdout, **kwargs): + """Track the progress of tasks execution with a progress bar. + + Tasks are done with a simple for-loop. + + Args: + func (callable): The function to be applied to each task. + tasks (list or tuple[Iterable, int]): A list of tasks or + (tasks, total num). + bar_width (int): Width of progress bar. + + Returns: + list: The task results. + """ + if isinstance(tasks, tuple): + assert len(tasks) == 2 + assert isinstance(tasks[0], Iterable) + assert isinstance(tasks[1], int) + task_num = tasks[1] + tasks = tasks[0] + elif isinstance(tasks, Iterable): + task_num = len(tasks) + else: + raise TypeError( + '"tasks" must be an iterable object or a (iterator, int) tuple') + prog_bar = ProgressBar(task_num, bar_width, file=file) + results = [] + for task in tasks: + results.append(func(task, **kwargs)) + prog_bar.update() + prog_bar.file.write('\n') + return results + + +def init_pool(process_num, initializer=None, initargs=None): + if initializer is None: + return Pool(process_num) + elif initargs is None: + return Pool(process_num, initializer) + else: + if not isinstance(initargs, tuple): + raise TypeError('"initargs" must be a tuple') + return Pool(process_num, initializer, initargs) + + +def track_parallel_progress(func, + tasks, + nproc, + initializer=None, + initargs=None, + bar_width=50, + chunksize=1, + skip_first=False, + keep_order=True, + file=sys.stdout): + """Track the progress of parallel task execution with a progress bar. + + The built-in :mod:`multiprocessing` module is used for process pools and + tasks are done with :func:`Pool.map` or :func:`Pool.imap_unordered`. + + Args: + func (callable): The function to be applied to each task. + tasks (list or tuple[Iterable, int]): A list of tasks or + (tasks, total num). + nproc (int): Process (worker) number. + initializer (None or callable): Refer to :class:`multiprocessing.Pool` + for details. + initargs (None or tuple): Refer to :class:`multiprocessing.Pool` for + details. + chunksize (int): Refer to :class:`multiprocessing.Pool` for details. + bar_width (int): Width of progress bar. + skip_first (bool): Whether to skip the first sample for each worker + when estimating fps, since the initialization step may takes + longer. + keep_order (bool): If True, :func:`Pool.imap` is used, otherwise + :func:`Pool.imap_unordered` is used. + + Returns: + list: The task results. + """ + if isinstance(tasks, tuple): + assert len(tasks) == 2 + assert isinstance(tasks[0], Iterable) + assert isinstance(tasks[1], int) + task_num = tasks[1] + tasks = tasks[0] + elif isinstance(tasks, Iterable): + task_num = len(tasks) + else: + raise TypeError( + '"tasks" must be an iterable object or a (iterator, int) tuple') + pool = init_pool(nproc, initializer, initargs) + start = not skip_first + task_num -= nproc * chunksize * int(skip_first) + prog_bar = ProgressBar(task_num, bar_width, start, file=file) + results = [] + if keep_order: + gen = pool.imap(func, tasks, chunksize) + else: + gen = pool.imap_unordered(func, tasks, chunksize) + for result in gen: + results.append(result) + if skip_first: + if len(results) < nproc * chunksize: + continue + elif len(results) == nproc * chunksize: + prog_bar.start() + continue + prog_bar.update() + prog_bar.file.write('\n') + pool.close() + pool.join() + return results + + +def track_iter_progress(tasks, bar_width=50, file=sys.stdout): + """Track the progress of tasks iteration or enumeration with a progress + bar. + + Tasks are yielded with a simple for-loop. + + Args: + tasks (list or tuple[Iterable, int]): A list of tasks or + (tasks, total num). + bar_width (int): Width of progress bar. + + Yields: + list: The task results. + """ + if isinstance(tasks, tuple): + assert len(tasks) == 2 + assert isinstance(tasks[0], Iterable) + assert isinstance(tasks[1], int) + task_num = tasks[1] + tasks = tasks[0] + elif isinstance(tasks, Iterable): + task_num = len(tasks) + else: + raise TypeError( + '"tasks" must be an iterable object or a (iterator, int) tuple') + prog_bar = ProgressBar(task_num, bar_width, file=file) + for task in tasks: + yield task + prog_bar.update() + prog_bar.file.write('\n') diff --git a/annotator/uniformer/mmcv/utils/registry.py b/annotator/uniformer/mmcv/utils/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..fa9df39bc9f3d8d568361e7250ab35468f2b74e0 --- /dev/null +++ b/annotator/uniformer/mmcv/utils/registry.py @@ -0,0 +1,315 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import inspect +import warnings +from functools import partial + +from .misc import is_seq_of + + +def build_from_cfg(cfg, registry, default_args=None): + """Build a module from config dict. + + Args: + cfg (dict): Config dict. It should at least contain the key "type". + registry (:obj:`Registry`): The registry to search the type from. + default_args (dict, optional): Default initialization arguments. + + Returns: + object: The constructed object. + """ + if not isinstance(cfg, dict): + raise TypeError(f'cfg must be a dict, but got {type(cfg)}') + if 'type' not in cfg: + if default_args is None or 'type' not in default_args: + raise KeyError( + '`cfg` or `default_args` must contain the key "type", ' + f'but got {cfg}\n{default_args}') + if not isinstance(registry, Registry): + raise TypeError('registry must be an mmcv.Registry object, ' + f'but got {type(registry)}') + if not (isinstance(default_args, dict) or default_args is None): + raise TypeError('default_args must be a dict or None, ' + f'but got {type(default_args)}') + + args = cfg.copy() + + if default_args is not None: + for name, value in default_args.items(): + args.setdefault(name, value) + + obj_type = args.pop('type') + if isinstance(obj_type, str): + obj_cls = registry.get(obj_type) + if obj_cls is None: + raise KeyError( + f'{obj_type} is not in the {registry.name} registry') + elif inspect.isclass(obj_type): + obj_cls = obj_type + else: + raise TypeError( + f'type must be a str or valid type, but got {type(obj_type)}') + try: + return obj_cls(**args) + except Exception as e: + # Normal TypeError does not print class name. + raise type(e)(f'{obj_cls.__name__}: {e}') + + +class Registry: + """A registry to map strings to classes. + + Registered object could be built from registry. + Example: + >>> MODELS = Registry('models') + >>> @MODELS.register_module() + >>> class ResNet: + >>> pass + >>> resnet = MODELS.build(dict(type='ResNet')) + + Please refer to + https://mmcv.readthedocs.io/en/latest/understand_mmcv/registry.html for + advanced usage. + + Args: + name (str): Registry name. + build_func(func, optional): Build function to construct instance from + Registry, func:`build_from_cfg` is used if neither ``parent`` or + ``build_func`` is specified. If ``parent`` is specified and + ``build_func`` is not given, ``build_func`` will be inherited + from ``parent``. Default: None. + parent (Registry, optional): Parent registry. The class registered in + children registry could be built from parent. Default: None. + scope (str, optional): The scope of registry. It is the key to search + for children registry. If not specified, scope will be the name of + the package where class is defined, e.g. mmdet, mmcls, mmseg. + Default: None. + """ + + def __init__(self, name, build_func=None, parent=None, scope=None): + self._name = name + self._module_dict = dict() + self._children = dict() + self._scope = self.infer_scope() if scope is None else scope + + # self.build_func will be set with the following priority: + # 1. build_func + # 2. parent.build_func + # 3. build_from_cfg + if build_func is None: + if parent is not None: + self.build_func = parent.build_func + else: + self.build_func = build_from_cfg + else: + self.build_func = build_func + if parent is not None: + assert isinstance(parent, Registry) + parent._add_children(self) + self.parent = parent + else: + self.parent = None + + def __len__(self): + return len(self._module_dict) + + def __contains__(self, key): + return self.get(key) is not None + + def __repr__(self): + format_str = self.__class__.__name__ + \ + f'(name={self._name}, ' \ + f'items={self._module_dict})' + return format_str + + @staticmethod + def infer_scope(): + """Infer the scope of registry. + + The name of the package where registry is defined will be returned. + + Example: + # in mmdet/models/backbone/resnet.py + >>> MODELS = Registry('models') + >>> @MODELS.register_module() + >>> class ResNet: + >>> pass + The scope of ``ResNet`` will be ``mmdet``. + + + Returns: + scope (str): The inferred scope name. + """ + # inspect.stack() trace where this function is called, the index-2 + # indicates the frame where `infer_scope()` is called + filename = inspect.getmodule(inspect.stack()[2][0]).__name__ + split_filename = filename.split('.') + return split_filename[0] + + @staticmethod + def split_scope_key(key): + """Split scope and key. + + The first scope will be split from key. + + Examples: + >>> Registry.split_scope_key('mmdet.ResNet') + 'mmdet', 'ResNet' + >>> Registry.split_scope_key('ResNet') + None, 'ResNet' + + Return: + scope (str, None): The first scope. + key (str): The remaining key. + """ + split_index = key.find('.') + if split_index != -1: + return key[:split_index], key[split_index + 1:] + else: + return None, key + + @property + def name(self): + return self._name + + @property + def scope(self): + return self._scope + + @property + def module_dict(self): + return self._module_dict + + @property + def children(self): + return self._children + + def get(self, key): + """Get the registry record. + + Args: + key (str): The class name in string format. + + Returns: + class: The corresponding class. + """ + scope, real_key = self.split_scope_key(key) + if scope is None or scope == self._scope: + # get from self + if real_key in self._module_dict: + return self._module_dict[real_key] + else: + # get from self._children + if scope in self._children: + return self._children[scope].get(real_key) + else: + # goto root + parent = self.parent + while parent.parent is not None: + parent = parent.parent + return parent.get(key) + + def build(self, *args, **kwargs): + return self.build_func(*args, **kwargs, registry=self) + + def _add_children(self, registry): + """Add children for a registry. + + The ``registry`` will be added as children based on its scope. + The parent registry could build objects from children registry. + + Example: + >>> models = Registry('models') + >>> mmdet_models = Registry('models', parent=models) + >>> @mmdet_models.register_module() + >>> class ResNet: + >>> pass + >>> resnet = models.build(dict(type='mmdet.ResNet')) + """ + + assert isinstance(registry, Registry) + assert registry.scope is not None + assert registry.scope not in self.children, \ + f'scope {registry.scope} exists in {self.name} registry' + self.children[registry.scope] = registry + + def _register_module(self, module_class, module_name=None, force=False): + if not inspect.isclass(module_class): + raise TypeError('module must be a class, ' + f'but got {type(module_class)}') + + if module_name is None: + module_name = module_class.__name__ + if isinstance(module_name, str): + module_name = [module_name] + for name in module_name: + if not force and name in self._module_dict: + raise KeyError(f'{name} is already registered ' + f'in {self.name}') + self._module_dict[name] = module_class + + def deprecated_register_module(self, cls=None, force=False): + warnings.warn( + 'The old API of register_module(module, force=False) ' + 'is deprecated and will be removed, please use the new API ' + 'register_module(name=None, force=False, module=None) instead.') + if cls is None: + return partial(self.deprecated_register_module, force=force) + self._register_module(cls, force=force) + return cls + + def register_module(self, name=None, force=False, module=None): + """Register a module. + + A record will be added to `self._module_dict`, whose key is the class + name or the specified name, and value is the class itself. + It can be used as a decorator or a normal function. + + Example: + >>> backbones = Registry('backbone') + >>> @backbones.register_module() + >>> class ResNet: + >>> pass + + >>> backbones = Registry('backbone') + >>> @backbones.register_module(name='mnet') + >>> class MobileNet: + >>> pass + + >>> backbones = Registry('backbone') + >>> class ResNet: + >>> pass + >>> backbones.register_module(ResNet) + + Args: + name (str | None): The module name to be registered. If not + specified, the class name will be used. + force (bool, optional): Whether to override an existing class with + the same name. Default: False. + module (type): Module class to be registered. + """ + if not isinstance(force, bool): + raise TypeError(f'force must be a boolean, but got {type(force)}') + # NOTE: This is a walkaround to be compatible with the old api, + # while it may introduce unexpected bugs. + if isinstance(name, type): + return self.deprecated_register_module(name, force=force) + + # raise the error ahead of time + if not (name is None or isinstance(name, str) or is_seq_of(name, str)): + raise TypeError( + 'name must be either of None, an instance of str or a sequence' + f' of str, but got {type(name)}') + + # use it as a normal method: x.register_module(module=SomeClass) + if module is not None: + self._register_module( + module_class=module, module_name=name, force=force) + return module + + # use it as a decorator: @x.register_module() + def _register(cls): + self._register_module( + module_class=cls, module_name=name, force=force) + return cls + + return _register diff --git a/annotator/uniformer/mmcv/utils/testing.py b/annotator/uniformer/mmcv/utils/testing.py new file mode 100644 index 0000000000000000000000000000000000000000..a27f936da8ec14bac18562ede0a79d476d82f797 --- /dev/null +++ b/annotator/uniformer/mmcv/utils/testing.py @@ -0,0 +1,140 @@ +# Copyright (c) Open-MMLab. +import sys +from collections.abc import Iterable +from runpy import run_path +from shlex import split +from typing import Any, Dict, List +from unittest.mock import patch + + +def check_python_script(cmd): + """Run the python cmd script with `__main__`. The difference between + `os.system` is that, this function exectues code in the current process, so + that it can be tracked by coverage tools. Currently it supports two forms: + + - ./tests/data/scripts/hello.py zz + - python tests/data/scripts/hello.py zz + """ + args = split(cmd) + if args[0] == 'python': + args = args[1:] + with patch.object(sys, 'argv', args): + run_path(args[0], run_name='__main__') + + +def _any(judge_result): + """Since built-in ``any`` works only when the element of iterable is not + iterable, implement the function.""" + if not isinstance(judge_result, Iterable): + return judge_result + + try: + for element in judge_result: + if _any(element): + return True + except TypeError: + # Maybe encounter the case: torch.tensor(True) | torch.tensor(False) + if judge_result: + return True + return False + + +def assert_dict_contains_subset(dict_obj: Dict[Any, Any], + expected_subset: Dict[Any, Any]) -> bool: + """Check if the dict_obj contains the expected_subset. + + Args: + dict_obj (Dict[Any, Any]): Dict object to be checked. + expected_subset (Dict[Any, Any]): Subset expected to be contained in + dict_obj. + + Returns: + bool: Whether the dict_obj contains the expected_subset. + """ + + for key, value in expected_subset.items(): + if key not in dict_obj.keys() or _any(dict_obj[key] != value): + return False + return True + + +def assert_attrs_equal(obj: Any, expected_attrs: Dict[str, Any]) -> bool: + """Check if attribute of class object is correct. + + Args: + obj (object): Class object to be checked. + expected_attrs (Dict[str, Any]): Dict of the expected attrs. + + Returns: + bool: Whether the attribute of class object is correct. + """ + for attr, value in expected_attrs.items(): + if not hasattr(obj, attr) or _any(getattr(obj, attr) != value): + return False + return True + + +def assert_dict_has_keys(obj: Dict[str, Any], + expected_keys: List[str]) -> bool: + """Check if the obj has all the expected_keys. + + Args: + obj (Dict[str, Any]): Object to be checked. + expected_keys (List[str]): Keys expected to contained in the keys of + the obj. + + Returns: + bool: Whether the obj has the expected keys. + """ + return set(expected_keys).issubset(set(obj.keys())) + + +def assert_keys_equal(result_keys: List[str], target_keys: List[str]) -> bool: + """Check if target_keys is equal to result_keys. + + Args: + result_keys (List[str]): Result keys to be checked. + target_keys (List[str]): Target keys to be checked. + + Returns: + bool: Whether target_keys is equal to result_keys. + """ + return set(result_keys) == set(target_keys) + + +def assert_is_norm_layer(module) -> bool: + """Check if the module is a norm layer. + + Args: + module (nn.Module): The module to be checked. + + Returns: + bool: Whether the module is a norm layer. + """ + from .parrots_wrapper import _BatchNorm, _InstanceNorm + from torch.nn import GroupNorm, LayerNorm + norm_layer_candidates = (_BatchNorm, _InstanceNorm, GroupNorm, LayerNorm) + return isinstance(module, norm_layer_candidates) + + +def assert_params_all_zeros(module) -> bool: + """Check if the parameters of the module is all zeros. + + Args: + module (nn.Module): The module to be checked. + + Returns: + bool: Whether the parameters of the module is all zeros. + """ + weight_data = module.weight.data + is_weight_zero = weight_data.allclose( + weight_data.new_zeros(weight_data.size())) + + if hasattr(module, 'bias') and module.bias is not None: + bias_data = module.bias.data + is_bias_zero = bias_data.allclose( + bias_data.new_zeros(bias_data.size())) + else: + is_bias_zero = True + + return is_weight_zero and is_bias_zero diff --git a/annotator/uniformer/mmcv/utils/timer.py b/annotator/uniformer/mmcv/utils/timer.py new file mode 100644 index 0000000000000000000000000000000000000000..e3db7d497d8b374e18b5297e0a1d6eb186fd8cba --- /dev/null +++ b/annotator/uniformer/mmcv/utils/timer.py @@ -0,0 +1,118 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from time import time + + +class TimerError(Exception): + + def __init__(self, message): + self.message = message + super(TimerError, self).__init__(message) + + +class Timer: + """A flexible Timer class. + + :Example: + + >>> import time + >>> import annotator.uniformer.mmcv as mmcv + >>> with mmcv.Timer(): + >>> # simulate a code block that will run for 1s + >>> time.sleep(1) + 1.000 + >>> with mmcv.Timer(print_tmpl='it takes {:.1f} seconds'): + >>> # simulate a code block that will run for 1s + >>> time.sleep(1) + it takes 1.0 seconds + >>> timer = mmcv.Timer() + >>> time.sleep(0.5) + >>> print(timer.since_start()) + 0.500 + >>> time.sleep(0.5) + >>> print(timer.since_last_check()) + 0.500 + >>> print(timer.since_start()) + 1.000 + """ + + def __init__(self, start=True, print_tmpl=None): + self._is_running = False + self.print_tmpl = print_tmpl if print_tmpl else '{:.3f}' + if start: + self.start() + + @property + def is_running(self): + """bool: indicate whether the timer is running""" + return self._is_running + + def __enter__(self): + self.start() + return self + + def __exit__(self, type, value, traceback): + print(self.print_tmpl.format(self.since_last_check())) + self._is_running = False + + def start(self): + """Start the timer.""" + if not self._is_running: + self._t_start = time() + self._is_running = True + self._t_last = time() + + def since_start(self): + """Total time since the timer is started. + + Returns (float): Time in seconds. + """ + if not self._is_running: + raise TimerError('timer is not running') + self._t_last = time() + return self._t_last - self._t_start + + def since_last_check(self): + """Time since the last checking. + + Either :func:`since_start` or :func:`since_last_check` is a checking + operation. + + Returns (float): Time in seconds. + """ + if not self._is_running: + raise TimerError('timer is not running') + dur = time() - self._t_last + self._t_last = time() + return dur + + +_g_timers = {} # global timers + + +def check_time(timer_id): + """Add check points in a single line. + + This method is suitable for running a task on a list of items. A timer will + be registered when the method is called for the first time. + + :Example: + + >>> import time + >>> import annotator.uniformer.mmcv as mmcv + >>> for i in range(1, 6): + >>> # simulate a code block + >>> time.sleep(i) + >>> mmcv.check_time('task1') + 2.000 + 3.000 + 4.000 + 5.000 + + Args: + timer_id (str): Timer identifier. + """ + if timer_id not in _g_timers: + _g_timers[timer_id] = Timer() + return 0 + else: + return _g_timers[timer_id].since_last_check() diff --git a/annotator/uniformer/mmcv/utils/trace.py b/annotator/uniformer/mmcv/utils/trace.py new file mode 100644 index 0000000000000000000000000000000000000000..5ca99dc3eda05ef980d9a4249b50deca8273b6cc --- /dev/null +++ b/annotator/uniformer/mmcv/utils/trace.py @@ -0,0 +1,23 @@ +import warnings + +import torch + +from annotator.uniformer.mmcv.utils import digit_version + + +def is_jit_tracing() -> bool: + if (torch.__version__ != 'parrots' + and digit_version(torch.__version__) >= digit_version('1.6.0')): + on_trace = torch.jit.is_tracing() + # In PyTorch 1.6, torch.jit.is_tracing has a bug. + # Refers to https://github.com/pytorch/pytorch/issues/42448 + if isinstance(on_trace, bool): + return on_trace + else: + return torch._C._is_tracing() + else: + warnings.warn( + 'torch.jit.is_tracing is only supported after v1.6.0. ' + 'Therefore is_tracing returns False automatically. Please ' + 'set on_trace manually if you are using trace.', UserWarning) + return False diff --git a/annotator/uniformer/mmcv/utils/version_utils.py b/annotator/uniformer/mmcv/utils/version_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..963c45a2e8a86a88413ab6c18c22481fb9831985 --- /dev/null +++ b/annotator/uniformer/mmcv/utils/version_utils.py @@ -0,0 +1,90 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os +import subprocess +import warnings + +from packaging.version import parse + + +def digit_version(version_str: str, length: int = 4): + """Convert a version string into a tuple of integers. + + This method is usually used for comparing two versions. For pre-release + versions: alpha < beta < rc. + + Args: + version_str (str): The version string. + length (int): The maximum number of version levels. Default: 4. + + Returns: + tuple[int]: The version info in digits (integers). + """ + assert 'parrots' not in version_str + version = parse(version_str) + assert version.release, f'failed to parse version {version_str}' + release = list(version.release) + release = release[:length] + if len(release) < length: + release = release + [0] * (length - len(release)) + if version.is_prerelease: + mapping = {'a': -3, 'b': -2, 'rc': -1} + val = -4 + # version.pre can be None + if version.pre: + if version.pre[0] not in mapping: + warnings.warn(f'unknown prerelease version {version.pre[0]}, ' + 'version checking may go wrong') + else: + val = mapping[version.pre[0]] + release.extend([val, version.pre[-1]]) + else: + release.extend([val, 0]) + + elif version.is_postrelease: + release.extend([1, version.post]) + else: + release.extend([0, 0]) + return tuple(release) + + +def _minimal_ext_cmd(cmd): + # construct minimal environment + env = {} + for k in ['SYSTEMROOT', 'PATH', 'HOME']: + v = os.environ.get(k) + if v is not None: + env[k] = v + # LANGUAGE is used on win32 + env['LANGUAGE'] = 'C' + env['LANG'] = 'C' + env['LC_ALL'] = 'C' + out = subprocess.Popen( + cmd, stdout=subprocess.PIPE, env=env).communicate()[0] + return out + + +def get_git_hash(fallback='unknown', digits=None): + """Get the git hash of the current repo. + + Args: + fallback (str, optional): The fallback string when git hash is + unavailable. Defaults to 'unknown'. + digits (int, optional): kept digits of the hash. Defaults to None, + meaning all digits are kept. + + Returns: + str: Git commit hash. + """ + + if digits is not None and not isinstance(digits, int): + raise TypeError('digits must be None or an integer') + + try: + out = _minimal_ext_cmd(['git', 'rev-parse', 'HEAD']) + sha = out.strip().decode('ascii') + if digits is not None: + sha = sha[:digits] + except OSError: + sha = fallback + + return sha diff --git a/annotator/uniformer/mmcv/version.py b/annotator/uniformer/mmcv/version.py new file mode 100644 index 0000000000000000000000000000000000000000..1cce4e50bd692d4002e3cac3c545a3fb2efe95d0 --- /dev/null +++ b/annotator/uniformer/mmcv/version.py @@ -0,0 +1,35 @@ +# Copyright (c) OpenMMLab. All rights reserved. +__version__ = '1.3.17' + + +def parse_version_info(version_str: str, length: int = 4) -> tuple: + """Parse a version string into a tuple. + + Args: + version_str (str): The version string. + length (int): The maximum number of version levels. Default: 4. + + Returns: + tuple[int | str]: The version info, e.g., "1.3.0" is parsed into + (1, 3, 0, 0, 0, 0), and "2.0.0rc1" is parsed into + (2, 0, 0, 0, 'rc', 1) (when length is set to 4). + """ + from packaging.version import parse + version = parse(version_str) + assert version.release, f'failed to parse version {version_str}' + release = list(version.release) + release = release[:length] + if len(release) < length: + release = release + [0] * (length - len(release)) + if version.is_prerelease: + release.extend(list(version.pre)) + elif version.is_postrelease: + release.extend(list(version.post)) + else: + release.extend([0, 0]) + return tuple(release) + + +version_info = tuple(int(x) for x in __version__.split('.')[:3]) + +__all__ = ['__version__', 'version_info', 'parse_version_info'] diff --git a/annotator/uniformer/mmcv/video/__init__.py b/annotator/uniformer/mmcv/video/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..73199b01dec52820dc6ca0139903536344d5a1eb --- /dev/null +++ b/annotator/uniformer/mmcv/video/__init__.py @@ -0,0 +1,11 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .io import Cache, VideoReader, frames2video +from .optflow import (dequantize_flow, flow_from_bytes, flow_warp, flowread, + flowwrite, quantize_flow, sparse_flow_from_bytes) +from .processing import concat_video, convert_video, cut_video, resize_video + +__all__ = [ + 'Cache', 'VideoReader', 'frames2video', 'convert_video', 'resize_video', + 'cut_video', 'concat_video', 'flowread', 'flowwrite', 'quantize_flow', + 'dequantize_flow', 'flow_warp', 'flow_from_bytes', 'sparse_flow_from_bytes' +] diff --git a/annotator/uniformer/mmcv/video/io.py b/annotator/uniformer/mmcv/video/io.py new file mode 100644 index 0000000000000000000000000000000000000000..9879154227f640c262853b92c219461c6f67ee8e --- /dev/null +++ b/annotator/uniformer/mmcv/video/io.py @@ -0,0 +1,318 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os.path as osp +from collections import OrderedDict + +import cv2 +from cv2 import (CAP_PROP_FOURCC, CAP_PROP_FPS, CAP_PROP_FRAME_COUNT, + CAP_PROP_FRAME_HEIGHT, CAP_PROP_FRAME_WIDTH, + CAP_PROP_POS_FRAMES, VideoWriter_fourcc) + +from annotator.uniformer.mmcv.utils import (check_file_exist, mkdir_or_exist, scandir, + track_progress) + + +class Cache: + + def __init__(self, capacity): + self._cache = OrderedDict() + self._capacity = int(capacity) + if capacity <= 0: + raise ValueError('capacity must be a positive integer') + + @property + def capacity(self): + return self._capacity + + @property + def size(self): + return len(self._cache) + + def put(self, key, val): + if key in self._cache: + return + if len(self._cache) >= self.capacity: + self._cache.popitem(last=False) + self._cache[key] = val + + def get(self, key, default=None): + val = self._cache[key] if key in self._cache else default + return val + + +class VideoReader: + """Video class with similar usage to a list object. + + This video warpper class provides convenient apis to access frames. + There exists an issue of OpenCV's VideoCapture class that jumping to a + certain frame may be inaccurate. It is fixed in this class by checking + the position after jumping each time. + Cache is used when decoding videos. So if the same frame is visited for + the second time, there is no need to decode again if it is stored in the + cache. + + :Example: + + >>> import annotator.uniformer.mmcv as mmcv + >>> v = mmcv.VideoReader('sample.mp4') + >>> len(v) # get the total frame number with `len()` + 120 + >>> for img in v: # v is iterable + >>> mmcv.imshow(img) + >>> v[5] # get the 6th frame + """ + + def __init__(self, filename, cache_capacity=10): + # Check whether the video path is a url + if not filename.startswith(('https://', 'http://')): + check_file_exist(filename, 'Video file not found: ' + filename) + self._vcap = cv2.VideoCapture(filename) + assert cache_capacity > 0 + self._cache = Cache(cache_capacity) + self._position = 0 + # get basic info + self._width = int(self._vcap.get(CAP_PROP_FRAME_WIDTH)) + self._height = int(self._vcap.get(CAP_PROP_FRAME_HEIGHT)) + self._fps = self._vcap.get(CAP_PROP_FPS) + self._frame_cnt = int(self._vcap.get(CAP_PROP_FRAME_COUNT)) + self._fourcc = self._vcap.get(CAP_PROP_FOURCC) + + @property + def vcap(self): + """:obj:`cv2.VideoCapture`: The raw VideoCapture object.""" + return self._vcap + + @property + def opened(self): + """bool: Indicate whether the video is opened.""" + return self._vcap.isOpened() + + @property + def width(self): + """int: Width of video frames.""" + return self._width + + @property + def height(self): + """int: Height of video frames.""" + return self._height + + @property + def resolution(self): + """tuple: Video resolution (width, height).""" + return (self._width, self._height) + + @property + def fps(self): + """float: FPS of the video.""" + return self._fps + + @property + def frame_cnt(self): + """int: Total frames of the video.""" + return self._frame_cnt + + @property + def fourcc(self): + """str: "Four character code" of the video.""" + return self._fourcc + + @property + def position(self): + """int: Current cursor position, indicating frame decoded.""" + return self._position + + def _get_real_position(self): + return int(round(self._vcap.get(CAP_PROP_POS_FRAMES))) + + def _set_real_position(self, frame_id): + self._vcap.set(CAP_PROP_POS_FRAMES, frame_id) + pos = self._get_real_position() + for _ in range(frame_id - pos): + self._vcap.read() + self._position = frame_id + + def read(self): + """Read the next frame. + + If the next frame have been decoded before and in the cache, then + return it directly, otherwise decode, cache and return it. + + Returns: + ndarray or None: Return the frame if successful, otherwise None. + """ + # pos = self._position + if self._cache: + img = self._cache.get(self._position) + if img is not None: + ret = True + else: + if self._position != self._get_real_position(): + self._set_real_position(self._position) + ret, img = self._vcap.read() + if ret: + self._cache.put(self._position, img) + else: + ret, img = self._vcap.read() + if ret: + self._position += 1 + return img + + def get_frame(self, frame_id): + """Get frame by index. + + Args: + frame_id (int): Index of the expected frame, 0-based. + + Returns: + ndarray or None: Return the frame if successful, otherwise None. + """ + if frame_id < 0 or frame_id >= self._frame_cnt: + raise IndexError( + f'"frame_id" must be between 0 and {self._frame_cnt - 1}') + if frame_id == self._position: + return self.read() + if self._cache: + img = self._cache.get(frame_id) + if img is not None: + self._position = frame_id + 1 + return img + self._set_real_position(frame_id) + ret, img = self._vcap.read() + if ret: + if self._cache: + self._cache.put(self._position, img) + self._position += 1 + return img + + def current_frame(self): + """Get the current frame (frame that is just visited). + + Returns: + ndarray or None: If the video is fresh, return None, otherwise + return the frame. + """ + if self._position == 0: + return None + return self._cache.get(self._position - 1) + + def cvt2frames(self, + frame_dir, + file_start=0, + filename_tmpl='{:06d}.jpg', + start=0, + max_num=0, + show_progress=True): + """Convert a video to frame images. + + Args: + frame_dir (str): Output directory to store all the frame images. + file_start (int): Filenames will start from the specified number. + filename_tmpl (str): Filename template with the index as the + placeholder. + start (int): The starting frame index. + max_num (int): Maximum number of frames to be written. + show_progress (bool): Whether to show a progress bar. + """ + mkdir_or_exist(frame_dir) + if max_num == 0: + task_num = self.frame_cnt - start + else: + task_num = min(self.frame_cnt - start, max_num) + if task_num <= 0: + raise ValueError('start must be less than total frame number') + if start > 0: + self._set_real_position(start) + + def write_frame(file_idx): + img = self.read() + if img is None: + return + filename = osp.join(frame_dir, filename_tmpl.format(file_idx)) + cv2.imwrite(filename, img) + + if show_progress: + track_progress(write_frame, range(file_start, + file_start + task_num)) + else: + for i in range(task_num): + write_frame(file_start + i) + + def __len__(self): + return self.frame_cnt + + def __getitem__(self, index): + if isinstance(index, slice): + return [ + self.get_frame(i) + for i in range(*index.indices(self.frame_cnt)) + ] + # support negative indexing + if index < 0: + index += self.frame_cnt + if index < 0: + raise IndexError('index out of range') + return self.get_frame(index) + + def __iter__(self): + self._set_real_position(0) + return self + + def __next__(self): + img = self.read() + if img is not None: + return img + else: + raise StopIteration + + next = __next__ + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self._vcap.release() + + +def frames2video(frame_dir, + video_file, + fps=30, + fourcc='XVID', + filename_tmpl='{:06d}.jpg', + start=0, + end=0, + show_progress=True): + """Read the frame images from a directory and join them as a video. + + Args: + frame_dir (str): The directory containing video frames. + video_file (str): Output filename. + fps (float): FPS of the output video. + fourcc (str): Fourcc of the output video, this should be compatible + with the output file type. + filename_tmpl (str): Filename template with the index as the variable. + start (int): Starting frame index. + end (int): Ending frame index. + show_progress (bool): Whether to show a progress bar. + """ + if end == 0: + ext = filename_tmpl.split('.')[-1] + end = len([name for name in scandir(frame_dir, ext)]) + first_file = osp.join(frame_dir, filename_tmpl.format(start)) + check_file_exist(first_file, 'The start frame not found: ' + first_file) + img = cv2.imread(first_file) + height, width = img.shape[:2] + resolution = (width, height) + vwriter = cv2.VideoWriter(video_file, VideoWriter_fourcc(*fourcc), fps, + resolution) + + def write_frame(file_idx): + filename = osp.join(frame_dir, filename_tmpl.format(file_idx)) + img = cv2.imread(filename) + vwriter.write(img) + + if show_progress: + track_progress(write_frame, range(start, end)) + else: + for i in range(start, end): + write_frame(i) + vwriter.release() diff --git a/annotator/uniformer/mmcv/video/optflow.py b/annotator/uniformer/mmcv/video/optflow.py new file mode 100644 index 0000000000000000000000000000000000000000..84160f8d6ef9fceb5a2f89e7481593109fc1905d --- /dev/null +++ b/annotator/uniformer/mmcv/video/optflow.py @@ -0,0 +1,254 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import warnings + +import cv2 +import numpy as np + +from annotator.uniformer.mmcv.arraymisc import dequantize, quantize +from annotator.uniformer.mmcv.image import imread, imwrite +from annotator.uniformer.mmcv.utils import is_str + + +def flowread(flow_or_path, quantize=False, concat_axis=0, *args, **kwargs): + """Read an optical flow map. + + Args: + flow_or_path (ndarray or str): A flow map or filepath. + quantize (bool): whether to read quantized pair, if set to True, + remaining args will be passed to :func:`dequantize_flow`. + concat_axis (int): The axis that dx and dy are concatenated, + can be either 0 or 1. Ignored if quantize is False. + + Returns: + ndarray: Optical flow represented as a (h, w, 2) numpy array + """ + if isinstance(flow_or_path, np.ndarray): + if (flow_or_path.ndim != 3) or (flow_or_path.shape[-1] != 2): + raise ValueError(f'Invalid flow with shape {flow_or_path.shape}') + return flow_or_path + elif not is_str(flow_or_path): + raise TypeError(f'"flow_or_path" must be a filename or numpy array, ' + f'not {type(flow_or_path)}') + + if not quantize: + with open(flow_or_path, 'rb') as f: + try: + header = f.read(4).decode('utf-8') + except Exception: + raise IOError(f'Invalid flow file: {flow_or_path}') + else: + if header != 'PIEH': + raise IOError(f'Invalid flow file: {flow_or_path}, ' + 'header does not contain PIEH') + + w = np.fromfile(f, np.int32, 1).squeeze() + h = np.fromfile(f, np.int32, 1).squeeze() + flow = np.fromfile(f, np.float32, w * h * 2).reshape((h, w, 2)) + else: + assert concat_axis in [0, 1] + cat_flow = imread(flow_or_path, flag='unchanged') + if cat_flow.ndim != 2: + raise IOError( + f'{flow_or_path} is not a valid quantized flow file, ' + f'its dimension is {cat_flow.ndim}.') + assert cat_flow.shape[concat_axis] % 2 == 0 + dx, dy = np.split(cat_flow, 2, axis=concat_axis) + flow = dequantize_flow(dx, dy, *args, **kwargs) + + return flow.astype(np.float32) + + +def flowwrite(flow, filename, quantize=False, concat_axis=0, *args, **kwargs): + """Write optical flow to file. + + If the flow is not quantized, it will be saved as a .flo file losslessly, + otherwise a jpeg image which is lossy but of much smaller size. (dx and dy + will be concatenated horizontally into a single image if quantize is True.) + + Args: + flow (ndarray): (h, w, 2) array of optical flow. + filename (str): Output filepath. + quantize (bool): Whether to quantize the flow and save it to 2 jpeg + images. If set to True, remaining args will be passed to + :func:`quantize_flow`. + concat_axis (int): The axis that dx and dy are concatenated, + can be either 0 or 1. Ignored if quantize is False. + """ + if not quantize: + with open(filename, 'wb') as f: + f.write('PIEH'.encode('utf-8')) + np.array([flow.shape[1], flow.shape[0]], dtype=np.int32).tofile(f) + flow = flow.astype(np.float32) + flow.tofile(f) + f.flush() + else: + assert concat_axis in [0, 1] + dx, dy = quantize_flow(flow, *args, **kwargs) + dxdy = np.concatenate((dx, dy), axis=concat_axis) + imwrite(dxdy, filename) + + +def quantize_flow(flow, max_val=0.02, norm=True): + """Quantize flow to [0, 255]. + + After this step, the size of flow will be much smaller, and can be + dumped as jpeg images. + + Args: + flow (ndarray): (h, w, 2) array of optical flow. + max_val (float): Maximum value of flow, values beyond + [-max_val, max_val] will be truncated. + norm (bool): Whether to divide flow values by image width/height. + + Returns: + tuple[ndarray]: Quantized dx and dy. + """ + h, w, _ = flow.shape + dx = flow[..., 0] + dy = flow[..., 1] + if norm: + dx = dx / w # avoid inplace operations + dy = dy / h + # use 255 levels instead of 256 to make sure 0 is 0 after dequantization. + flow_comps = [ + quantize(d, -max_val, max_val, 255, np.uint8) for d in [dx, dy] + ] + return tuple(flow_comps) + + +def dequantize_flow(dx, dy, max_val=0.02, denorm=True): + """Recover from quantized flow. + + Args: + dx (ndarray): Quantized dx. + dy (ndarray): Quantized dy. + max_val (float): Maximum value used when quantizing. + denorm (bool): Whether to multiply flow values with width/height. + + Returns: + ndarray: Dequantized flow. + """ + assert dx.shape == dy.shape + assert dx.ndim == 2 or (dx.ndim == 3 and dx.shape[-1] == 1) + + dx, dy = [dequantize(d, -max_val, max_val, 255) for d in [dx, dy]] + + if denorm: + dx *= dx.shape[1] + dy *= dx.shape[0] + flow = np.dstack((dx, dy)) + return flow + + +def flow_warp(img, flow, filling_value=0, interpolate_mode='nearest'): + """Use flow to warp img. + + Args: + img (ndarray, float or uint8): Image to be warped. + flow (ndarray, float): Optical Flow. + filling_value (int): The missing pixels will be set with filling_value. + interpolate_mode (str): bilinear -> Bilinear Interpolation; + nearest -> Nearest Neighbor. + + Returns: + ndarray: Warped image with the same shape of img + """ + warnings.warn('This function is just for prototyping and cannot ' + 'guarantee the computational efficiency.') + assert flow.ndim == 3, 'Flow must be in 3D arrays.' + height = flow.shape[0] + width = flow.shape[1] + channels = img.shape[2] + + output = np.ones( + (height, width, channels), dtype=img.dtype) * filling_value + + grid = np.indices((height, width)).swapaxes(0, 1).swapaxes(1, 2) + dx = grid[:, :, 0] + flow[:, :, 1] + dy = grid[:, :, 1] + flow[:, :, 0] + sx = np.floor(dx).astype(int) + sy = np.floor(dy).astype(int) + valid = (sx >= 0) & (sx < height - 1) & (sy >= 0) & (sy < width - 1) + + if interpolate_mode == 'nearest': + output[valid, :] = img[dx[valid].round().astype(int), + dy[valid].round().astype(int), :] + elif interpolate_mode == 'bilinear': + # dirty walkround for integer positions + eps_ = 1e-6 + dx, dy = dx + eps_, dy + eps_ + left_top_ = img[np.floor(dx[valid]).astype(int), + np.floor(dy[valid]).astype(int), :] * ( + np.ceil(dx[valid]) - dx[valid])[:, None] * ( + np.ceil(dy[valid]) - dy[valid])[:, None] + left_down_ = img[np.ceil(dx[valid]).astype(int), + np.floor(dy[valid]).astype(int), :] * ( + dx[valid] - np.floor(dx[valid]))[:, None] * ( + np.ceil(dy[valid]) - dy[valid])[:, None] + right_top_ = img[np.floor(dx[valid]).astype(int), + np.ceil(dy[valid]).astype(int), :] * ( + np.ceil(dx[valid]) - dx[valid])[:, None] * ( + dy[valid] - np.floor(dy[valid]))[:, None] + right_down_ = img[np.ceil(dx[valid]).astype(int), + np.ceil(dy[valid]).astype(int), :] * ( + dx[valid] - np.floor(dx[valid]))[:, None] * ( + dy[valid] - np.floor(dy[valid]))[:, None] + output[valid, :] = left_top_ + left_down_ + right_top_ + right_down_ + else: + raise NotImplementedError( + 'We only support interpolation modes of nearest and bilinear, ' + f'but got {interpolate_mode}.') + return output.astype(img.dtype) + + +def flow_from_bytes(content): + """Read dense optical flow from bytes. + + .. note:: + This load optical flow function works for FlyingChairs, FlyingThings3D, + Sintel, FlyingChairsOcc datasets, but cannot load the data from + ChairsSDHom. + + Args: + content (bytes): Optical flow bytes got from files or other streams. + + Returns: + ndarray: Loaded optical flow with the shape (H, W, 2). + """ + + # header in first 4 bytes + header = content[:4] + if header.decode('utf-8') != 'PIEH': + raise Exception('Flow file header does not contain PIEH') + # width in second 4 bytes + width = np.frombuffer(content[4:], np.int32, 1).squeeze() + # height in third 4 bytes + height = np.frombuffer(content[8:], np.int32, 1).squeeze() + # after first 12 bytes, all bytes are flow + flow = np.frombuffer(content[12:], np.float32, width * height * 2).reshape( + (height, width, 2)) + + return flow + + +def sparse_flow_from_bytes(content): + """Read the optical flow in KITTI datasets from bytes. + + This function is modified from RAFT load the `KITTI datasets + `_. + + Args: + content (bytes): Optical flow bytes got from files or other streams. + + Returns: + Tuple(ndarray, ndarray): Loaded optical flow with the shape (H, W, 2) + and flow valid mask with the shape (H, W). + """ # nopa + + content = np.frombuffer(content, np.uint8) + flow = cv2.imdecode(content, cv2.IMREAD_ANYDEPTH | cv2.IMREAD_COLOR) + flow = flow[:, :, ::-1].astype(np.float32) + # flow shape (H, W, 2) valid shape (H, W) + flow, valid = flow[:, :, :2], flow[:, :, 2] + flow = (flow - 2**15) / 64.0 + return flow, valid diff --git a/annotator/uniformer/mmcv/video/processing.py b/annotator/uniformer/mmcv/video/processing.py new file mode 100644 index 0000000000000000000000000000000000000000..3d90b96e0823d5f116755e7f498d25d17017224a --- /dev/null +++ b/annotator/uniformer/mmcv/video/processing.py @@ -0,0 +1,160 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os +import os.path as osp +import subprocess +import tempfile + +from annotator.uniformer.mmcv.utils import requires_executable + + +@requires_executable('ffmpeg') +def convert_video(in_file, + out_file, + print_cmd=False, + pre_options='', + **kwargs): + """Convert a video with ffmpeg. + + This provides a general api to ffmpeg, the executed command is:: + + `ffmpeg -y -i ` + + Options(kwargs) are mapped to ffmpeg commands with the following rules: + + - key=val: "-key val" + - key=True: "-key" + - key=False: "" + + Args: + in_file (str): Input video filename. + out_file (str): Output video filename. + pre_options (str): Options appears before "-i ". + print_cmd (bool): Whether to print the final ffmpeg command. + """ + options = [] + for k, v in kwargs.items(): + if isinstance(v, bool): + if v: + options.append(f'-{k}') + elif k == 'log_level': + assert v in [ + 'quiet', 'panic', 'fatal', 'error', 'warning', 'info', + 'verbose', 'debug', 'trace' + ] + options.append(f'-loglevel {v}') + else: + options.append(f'-{k} {v}') + cmd = f'ffmpeg -y {pre_options} -i {in_file} {" ".join(options)} ' \ + f'{out_file}' + if print_cmd: + print(cmd) + subprocess.call(cmd, shell=True) + + +@requires_executable('ffmpeg') +def resize_video(in_file, + out_file, + size=None, + ratio=None, + keep_ar=False, + log_level='info', + print_cmd=False): + """Resize a video. + + Args: + in_file (str): Input video filename. + out_file (str): Output video filename. + size (tuple): Expected size (w, h), eg, (320, 240) or (320, -1). + ratio (tuple or float): Expected resize ratio, (2, 0.5) means + (w*2, h*0.5). + keep_ar (bool): Whether to keep original aspect ratio. + log_level (str): Logging level of ffmpeg. + print_cmd (bool): Whether to print the final ffmpeg command. + """ + if size is None and ratio is None: + raise ValueError('expected size or ratio must be specified') + if size is not None and ratio is not None: + raise ValueError('size and ratio cannot be specified at the same time') + options = {'log_level': log_level} + if size: + if not keep_ar: + options['vf'] = f'scale={size[0]}:{size[1]}' + else: + options['vf'] = f'scale=w={size[0]}:h={size[1]}:' \ + 'force_original_aspect_ratio=decrease' + else: + if not isinstance(ratio, tuple): + ratio = (ratio, ratio) + options['vf'] = f'scale="trunc(iw*{ratio[0]}):trunc(ih*{ratio[1]})"' + convert_video(in_file, out_file, print_cmd, **options) + + +@requires_executable('ffmpeg') +def cut_video(in_file, + out_file, + start=None, + end=None, + vcodec=None, + acodec=None, + log_level='info', + print_cmd=False): + """Cut a clip from a video. + + Args: + in_file (str): Input video filename. + out_file (str): Output video filename. + start (None or float): Start time (in seconds). + end (None or float): End time (in seconds). + vcodec (None or str): Output video codec, None for unchanged. + acodec (None or str): Output audio codec, None for unchanged. + log_level (str): Logging level of ffmpeg. + print_cmd (bool): Whether to print the final ffmpeg command. + """ + options = {'log_level': log_level} + if vcodec is None: + options['vcodec'] = 'copy' + if acodec is None: + options['acodec'] = 'copy' + if start: + options['ss'] = start + else: + start = 0 + if end: + options['t'] = end - start + convert_video(in_file, out_file, print_cmd, **options) + + +@requires_executable('ffmpeg') +def concat_video(video_list, + out_file, + vcodec=None, + acodec=None, + log_level='info', + print_cmd=False): + """Concatenate multiple videos into a single one. + + Args: + video_list (list): A list of video filenames + out_file (str): Output video filename + vcodec (None or str): Output video codec, None for unchanged + acodec (None or str): Output audio codec, None for unchanged + log_level (str): Logging level of ffmpeg. + print_cmd (bool): Whether to print the final ffmpeg command. + """ + tmp_filehandler, tmp_filename = tempfile.mkstemp(suffix='.txt', text=True) + with open(tmp_filename, 'w') as f: + for filename in video_list: + f.write(f'file {osp.abspath(filename)}\n') + options = {'log_level': log_level} + if vcodec is None: + options['vcodec'] = 'copy' + if acodec is None: + options['acodec'] = 'copy' + convert_video( + tmp_filename, + out_file, + print_cmd, + pre_options='-f concat -safe 0', + **options) + os.close(tmp_filehandler) + os.remove(tmp_filename) diff --git a/annotator/uniformer/mmcv/visualization/__init__.py b/annotator/uniformer/mmcv/visualization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..835df136bdcf69348281d22914d41aa84cdf92b1 --- /dev/null +++ b/annotator/uniformer/mmcv/visualization/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .color import Color, color_val +from .image import imshow, imshow_bboxes, imshow_det_bboxes +from .optflow import flow2rgb, flowshow, make_color_wheel + +__all__ = [ + 'Color', 'color_val', 'imshow', 'imshow_bboxes', 'imshow_det_bboxes', + 'flowshow', 'flow2rgb', 'make_color_wheel' +] diff --git a/annotator/uniformer/mmcv/visualization/color.py b/annotator/uniformer/mmcv/visualization/color.py new file mode 100644 index 0000000000000000000000000000000000000000..9041e0e6b7581c3356795d6a3c5e84667c88f025 --- /dev/null +++ b/annotator/uniformer/mmcv/visualization/color.py @@ -0,0 +1,51 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from enum import Enum + +import numpy as np + +from annotator.uniformer.mmcv.utils import is_str + + +class Color(Enum): + """An enum that defines common colors. + + Contains red, green, blue, cyan, yellow, magenta, white and black. + """ + red = (0, 0, 255) + green = (0, 255, 0) + blue = (255, 0, 0) + cyan = (255, 255, 0) + yellow = (0, 255, 255) + magenta = (255, 0, 255) + white = (255, 255, 255) + black = (0, 0, 0) + + +def color_val(color): + """Convert various input to color tuples. + + Args: + color (:obj:`Color`/str/tuple/int/ndarray): Color inputs + + Returns: + tuple[int]: A tuple of 3 integers indicating BGR channels. + """ + if is_str(color): + return Color[color].value + elif isinstance(color, Color): + return color.value + elif isinstance(color, tuple): + assert len(color) == 3 + for channel in color: + assert 0 <= channel <= 255 + return color + elif isinstance(color, int): + assert 0 <= color <= 255 + return color, color, color + elif isinstance(color, np.ndarray): + assert color.ndim == 1 and color.size == 3 + assert np.all((color >= 0) & (color <= 255)) + color = color.astype(np.uint8) + return tuple(color) + else: + raise TypeError(f'Invalid type for color: {type(color)}') diff --git a/annotator/uniformer/mmcv/visualization/image.py b/annotator/uniformer/mmcv/visualization/image.py new file mode 100644 index 0000000000000000000000000000000000000000..61a56c75b67f593c298408462c63c0468be8e276 --- /dev/null +++ b/annotator/uniformer/mmcv/visualization/image.py @@ -0,0 +1,152 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import cv2 +import numpy as np + +from annotator.uniformer.mmcv.image import imread, imwrite +from .color import color_val + + +def imshow(img, win_name='', wait_time=0): + """Show an image. + + Args: + img (str or ndarray): The image to be displayed. + win_name (str): The window name. + wait_time (int): Value of waitKey param. + """ + cv2.imshow(win_name, imread(img)) + if wait_time == 0: # prevent from hanging if windows was closed + while True: + ret = cv2.waitKey(1) + + closed = cv2.getWindowProperty(win_name, cv2.WND_PROP_VISIBLE) < 1 + # if user closed window or if some key pressed + if closed or ret != -1: + break + else: + ret = cv2.waitKey(wait_time) + + +def imshow_bboxes(img, + bboxes, + colors='green', + top_k=-1, + thickness=1, + show=True, + win_name='', + wait_time=0, + out_file=None): + """Draw bboxes on an image. + + Args: + img (str or ndarray): The image to be displayed. + bboxes (list or ndarray): A list of ndarray of shape (k, 4). + colors (list[str or tuple or Color]): A list of colors. + top_k (int): Plot the first k bboxes only if set positive. + thickness (int): Thickness of lines. + show (bool): Whether to show the image. + win_name (str): The window name. + wait_time (int): Value of waitKey param. + out_file (str, optional): The filename to write the image. + + Returns: + ndarray: The image with bboxes drawn on it. + """ + img = imread(img) + img = np.ascontiguousarray(img) + + if isinstance(bboxes, np.ndarray): + bboxes = [bboxes] + if not isinstance(colors, list): + colors = [colors for _ in range(len(bboxes))] + colors = [color_val(c) for c in colors] + assert len(bboxes) == len(colors) + + for i, _bboxes in enumerate(bboxes): + _bboxes = _bboxes.astype(np.int32) + if top_k <= 0: + _top_k = _bboxes.shape[0] + else: + _top_k = min(top_k, _bboxes.shape[0]) + for j in range(_top_k): + left_top = (_bboxes[j, 0], _bboxes[j, 1]) + right_bottom = (_bboxes[j, 2], _bboxes[j, 3]) + cv2.rectangle( + img, left_top, right_bottom, colors[i], thickness=thickness) + + if show: + imshow(img, win_name, wait_time) + if out_file is not None: + imwrite(img, out_file) + return img + + +def imshow_det_bboxes(img, + bboxes, + labels, + class_names=None, + score_thr=0, + bbox_color='green', + text_color='green', + thickness=1, + font_scale=0.5, + show=True, + win_name='', + wait_time=0, + out_file=None): + """Draw bboxes and class labels (with scores) on an image. + + Args: + img (str or ndarray): The image to be displayed. + bboxes (ndarray): Bounding boxes (with scores), shaped (n, 4) or + (n, 5). + labels (ndarray): Labels of bboxes. + class_names (list[str]): Names of each classes. + score_thr (float): Minimum score of bboxes to be shown. + bbox_color (str or tuple or :obj:`Color`): Color of bbox lines. + text_color (str or tuple or :obj:`Color`): Color of texts. + thickness (int): Thickness of lines. + font_scale (float): Font scales of texts. + show (bool): Whether to show the image. + win_name (str): The window name. + wait_time (int): Value of waitKey param. + out_file (str or None): The filename to write the image. + + Returns: + ndarray: The image with bboxes drawn on it. + """ + assert bboxes.ndim == 2 + assert labels.ndim == 1 + assert bboxes.shape[0] == labels.shape[0] + assert bboxes.shape[1] == 4 or bboxes.shape[1] == 5 + img = imread(img) + img = np.ascontiguousarray(img) + + if score_thr > 0: + assert bboxes.shape[1] == 5 + scores = bboxes[:, -1] + inds = scores > score_thr + bboxes = bboxes[inds, :] + labels = labels[inds] + + bbox_color = color_val(bbox_color) + text_color = color_val(text_color) + + for bbox, label in zip(bboxes, labels): + bbox_int = bbox.astype(np.int32) + left_top = (bbox_int[0], bbox_int[1]) + right_bottom = (bbox_int[2], bbox_int[3]) + cv2.rectangle( + img, left_top, right_bottom, bbox_color, thickness=thickness) + label_text = class_names[ + label] if class_names is not None else f'cls {label}' + if len(bbox) > 4: + label_text += f'|{bbox[-1]:.02f}' + cv2.putText(img, label_text, (bbox_int[0], bbox_int[1] - 2), + cv2.FONT_HERSHEY_COMPLEX, font_scale, text_color) + + if show: + imshow(img, win_name, wait_time) + if out_file is not None: + imwrite(img, out_file) + return img diff --git a/annotator/uniformer/mmcv/visualization/optflow.py b/annotator/uniformer/mmcv/visualization/optflow.py new file mode 100644 index 0000000000000000000000000000000000000000..c3870c700f7c946177ee5d536ce3f6c814a77ce7 --- /dev/null +++ b/annotator/uniformer/mmcv/visualization/optflow.py @@ -0,0 +1,112 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from __future__ import division + +import numpy as np + +from annotator.uniformer.mmcv.image import rgb2bgr +from annotator.uniformer.mmcv.video import flowread +from .image import imshow + + +def flowshow(flow, win_name='', wait_time=0): + """Show optical flow. + + Args: + flow (ndarray or str): The optical flow to be displayed. + win_name (str): The window name. + wait_time (int): Value of waitKey param. + """ + flow = flowread(flow) + flow_img = flow2rgb(flow) + imshow(rgb2bgr(flow_img), win_name, wait_time) + + +def flow2rgb(flow, color_wheel=None, unknown_thr=1e6): + """Convert flow map to RGB image. + + Args: + flow (ndarray): Array of optical flow. + color_wheel (ndarray or None): Color wheel used to map flow field to + RGB colorspace. Default color wheel will be used if not specified. + unknown_thr (str): Values above this threshold will be marked as + unknown and thus ignored. + + Returns: + ndarray: RGB image that can be visualized. + """ + assert flow.ndim == 3 and flow.shape[-1] == 2 + if color_wheel is None: + color_wheel = make_color_wheel() + assert color_wheel.ndim == 2 and color_wheel.shape[1] == 3 + num_bins = color_wheel.shape[0] + + dx = flow[:, :, 0].copy() + dy = flow[:, :, 1].copy() + + ignore_inds = ( + np.isnan(dx) | np.isnan(dy) | (np.abs(dx) > unknown_thr) | + (np.abs(dy) > unknown_thr)) + dx[ignore_inds] = 0 + dy[ignore_inds] = 0 + + rad = np.sqrt(dx**2 + dy**2) + if np.any(rad > np.finfo(float).eps): + max_rad = np.max(rad) + dx /= max_rad + dy /= max_rad + + rad = np.sqrt(dx**2 + dy**2) + angle = np.arctan2(-dy, -dx) / np.pi + + bin_real = (angle + 1) / 2 * (num_bins - 1) + bin_left = np.floor(bin_real).astype(int) + bin_right = (bin_left + 1) % num_bins + w = (bin_real - bin_left.astype(np.float32))[..., None] + flow_img = (1 - + w) * color_wheel[bin_left, :] + w * color_wheel[bin_right, :] + small_ind = rad <= 1 + flow_img[small_ind] = 1 - rad[small_ind, None] * (1 - flow_img[small_ind]) + flow_img[np.logical_not(small_ind)] *= 0.75 + + flow_img[ignore_inds, :] = 0 + + return flow_img + + +def make_color_wheel(bins=None): + """Build a color wheel. + + Args: + bins(list or tuple, optional): Specify the number of bins for each + color range, corresponding to six ranges: red -> yellow, + yellow -> green, green -> cyan, cyan -> blue, blue -> magenta, + magenta -> red. [15, 6, 4, 11, 13, 6] is used for default + (see Middlebury). + + Returns: + ndarray: Color wheel of shape (total_bins, 3). + """ + if bins is None: + bins = [15, 6, 4, 11, 13, 6] + assert len(bins) == 6 + + RY, YG, GC, CB, BM, MR = tuple(bins) + + ry = [1, np.arange(RY) / RY, 0] + yg = [1 - np.arange(YG) / YG, 1, 0] + gc = [0, 1, np.arange(GC) / GC] + cb = [0, 1 - np.arange(CB) / CB, 1] + bm = [np.arange(BM) / BM, 0, 1] + mr = [1, 0, 1 - np.arange(MR) / MR] + + num_bins = RY + YG + GC + CB + BM + MR + + color_wheel = np.zeros((3, num_bins), dtype=np.float32) + + col = 0 + for i, color in enumerate([ry, yg, gc, cb, bm, mr]): + for j in range(3): + color_wheel[j, col:col + bins[i]] = color[j] + col += bins[i] + + return color_wheel.T diff --git a/annotator/uniformer/mmcv_custom/__init__.py b/annotator/uniformer/mmcv_custom/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4b958738b9fd93bfcec239c550df1d9a44b8c536 --- /dev/null +++ b/annotator/uniformer/mmcv_custom/__init__.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- + +from .checkpoint import load_checkpoint + +__all__ = ['load_checkpoint'] \ No newline at end of file diff --git a/annotator/uniformer/mmcv_custom/checkpoint.py b/annotator/uniformer/mmcv_custom/checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..19b87fef0a52d31babcdb3edb8f3089b6420173f --- /dev/null +++ b/annotator/uniformer/mmcv_custom/checkpoint.py @@ -0,0 +1,500 @@ +# Copyright (c) Open-MMLab. All rights reserved. +import io +import os +import os.path as osp +import pkgutil +import time +import warnings +from collections import OrderedDict +from importlib import import_module +from tempfile import TemporaryDirectory + +import torch +import torchvision +from torch.optim import Optimizer +from torch.utils import model_zoo +from torch.nn import functional as F + +import annotator.uniformer.mmcv as mmcv +from annotator.uniformer.mmcv.fileio import FileClient +from annotator.uniformer.mmcv.fileio import load as load_file +from annotator.uniformer.mmcv.parallel import is_module_wrapper +from annotator.uniformer.mmcv.utils import mkdir_or_exist +from annotator.uniformer.mmcv.runner import get_dist_info + +ENV_MMCV_HOME = 'MMCV_HOME' +ENV_XDG_CACHE_HOME = 'XDG_CACHE_HOME' +DEFAULT_CACHE_DIR = '~/.cache' + + +def _get_mmcv_home(): + mmcv_home = os.path.expanduser( + os.getenv( + ENV_MMCV_HOME, + os.path.join( + os.getenv(ENV_XDG_CACHE_HOME, DEFAULT_CACHE_DIR), 'mmcv'))) + + mkdir_or_exist(mmcv_home) + return mmcv_home + + +def load_state_dict(module, state_dict, strict=False, logger=None): + """Load state_dict to a module. + + This method is modified from :meth:`torch.nn.Module.load_state_dict`. + Default value for ``strict`` is set to ``False`` and the message for + param mismatch will be shown even if strict is False. + + Args: + module (Module): Module that receives the state_dict. + state_dict (OrderedDict): Weights. + strict (bool): whether to strictly enforce that the keys + in :attr:`state_dict` match the keys returned by this module's + :meth:`~torch.nn.Module.state_dict` function. Default: ``False``. + logger (:obj:`logging.Logger`, optional): Logger to log the error + message. If not specified, print function will be used. + """ + unexpected_keys = [] + all_missing_keys = [] + err_msg = [] + + metadata = getattr(state_dict, '_metadata', None) + state_dict = state_dict.copy() + if metadata is not None: + state_dict._metadata = metadata + + # use _load_from_state_dict to enable checkpoint version control + def load(module, prefix=''): + # recursively check parallel module in case that the model has a + # complicated structure, e.g., nn.Module(nn.Module(DDP)) + if is_module_wrapper(module): + module = module.module + local_metadata = {} if metadata is None else metadata.get( + prefix[:-1], {}) + module._load_from_state_dict(state_dict, prefix, local_metadata, True, + all_missing_keys, unexpected_keys, + err_msg) + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + '.') + + load(module) + load = None # break load->load reference cycle + + # ignore "num_batches_tracked" of BN layers + missing_keys = [ + key for key in all_missing_keys if 'num_batches_tracked' not in key + ] + + if unexpected_keys: + err_msg.append('unexpected key in source ' + f'state_dict: {", ".join(unexpected_keys)}\n') + if missing_keys: + err_msg.append( + f'missing keys in source state_dict: {", ".join(missing_keys)}\n') + + rank, _ = get_dist_info() + if len(err_msg) > 0 and rank == 0: + err_msg.insert( + 0, 'The model and loaded state dict do not match exactly\n') + err_msg = '\n'.join(err_msg) + if strict: + raise RuntimeError(err_msg) + elif logger is not None: + logger.warning(err_msg) + else: + print(err_msg) + + +def load_url_dist(url, model_dir=None): + """In distributed setting, this function only download checkpoint at local + rank 0.""" + rank, world_size = get_dist_info() + rank = int(os.environ.get('LOCAL_RANK', rank)) + if rank == 0: + checkpoint = model_zoo.load_url(url, model_dir=model_dir) + if world_size > 1: + torch.distributed.barrier() + if rank > 0: + checkpoint = model_zoo.load_url(url, model_dir=model_dir) + return checkpoint + + +def load_pavimodel_dist(model_path, map_location=None): + """In distributed setting, this function only download checkpoint at local + rank 0.""" + try: + from pavi import modelcloud + except ImportError: + raise ImportError( + 'Please install pavi to load checkpoint from modelcloud.') + rank, world_size = get_dist_info() + rank = int(os.environ.get('LOCAL_RANK', rank)) + if rank == 0: + model = modelcloud.get(model_path) + with TemporaryDirectory() as tmp_dir: + downloaded_file = osp.join(tmp_dir, model.name) + model.download(downloaded_file) + checkpoint = torch.load(downloaded_file, map_location=map_location) + if world_size > 1: + torch.distributed.barrier() + if rank > 0: + model = modelcloud.get(model_path) + with TemporaryDirectory() as tmp_dir: + downloaded_file = osp.join(tmp_dir, model.name) + model.download(downloaded_file) + checkpoint = torch.load( + downloaded_file, map_location=map_location) + return checkpoint + + +def load_fileclient_dist(filename, backend, map_location): + """In distributed setting, this function only download checkpoint at local + rank 0.""" + rank, world_size = get_dist_info() + rank = int(os.environ.get('LOCAL_RANK', rank)) + allowed_backends = ['ceph'] + if backend not in allowed_backends: + raise ValueError(f'Load from Backend {backend} is not supported.') + if rank == 0: + fileclient = FileClient(backend=backend) + buffer = io.BytesIO(fileclient.get(filename)) + checkpoint = torch.load(buffer, map_location=map_location) + if world_size > 1: + torch.distributed.barrier() + if rank > 0: + fileclient = FileClient(backend=backend) + buffer = io.BytesIO(fileclient.get(filename)) + checkpoint = torch.load(buffer, map_location=map_location) + return checkpoint + + +def get_torchvision_models(): + model_urls = dict() + for _, name, ispkg in pkgutil.walk_packages(torchvision.models.__path__): + if ispkg: + continue + _zoo = import_module(f'torchvision.models.{name}') + if hasattr(_zoo, 'model_urls'): + _urls = getattr(_zoo, 'model_urls') + model_urls.update(_urls) + return model_urls + + +def get_external_models(): + mmcv_home = _get_mmcv_home() + default_json_path = osp.join(mmcv.__path__[0], 'model_zoo/open_mmlab.json') + default_urls = load_file(default_json_path) + assert isinstance(default_urls, dict) + external_json_path = osp.join(mmcv_home, 'open_mmlab.json') + if osp.exists(external_json_path): + external_urls = load_file(external_json_path) + assert isinstance(external_urls, dict) + default_urls.update(external_urls) + + return default_urls + + +def get_mmcls_models(): + mmcls_json_path = osp.join(mmcv.__path__[0], 'model_zoo/mmcls.json') + mmcls_urls = load_file(mmcls_json_path) + + return mmcls_urls + + +def get_deprecated_model_names(): + deprecate_json_path = osp.join(mmcv.__path__[0], + 'model_zoo/deprecated.json') + deprecate_urls = load_file(deprecate_json_path) + assert isinstance(deprecate_urls, dict) + + return deprecate_urls + + +def _process_mmcls_checkpoint(checkpoint): + state_dict = checkpoint['state_dict'] + new_state_dict = OrderedDict() + for k, v in state_dict.items(): + if k.startswith('backbone.'): + new_state_dict[k[9:]] = v + new_checkpoint = dict(state_dict=new_state_dict) + + return new_checkpoint + + +def _load_checkpoint(filename, map_location=None): + """Load checkpoint from somewhere (modelzoo, file, url). + + Args: + filename (str): Accept local filepath, URL, ``torchvision://xxx``, + ``open-mmlab://xxx``. Please refer to ``docs/model_zoo.md`` for + details. + map_location (str | None): Same as :func:`torch.load`. Default: None. + + Returns: + dict | OrderedDict: The loaded checkpoint. It can be either an + OrderedDict storing model weights or a dict containing other + information, which depends on the checkpoint. + """ + if filename.startswith('modelzoo://'): + warnings.warn('The URL scheme of "modelzoo://" is deprecated, please ' + 'use "torchvision://" instead') + model_urls = get_torchvision_models() + model_name = filename[11:] + checkpoint = load_url_dist(model_urls[model_name]) + elif filename.startswith('torchvision://'): + model_urls = get_torchvision_models() + model_name = filename[14:] + checkpoint = load_url_dist(model_urls[model_name]) + elif filename.startswith('open-mmlab://'): + model_urls = get_external_models() + model_name = filename[13:] + deprecated_urls = get_deprecated_model_names() + if model_name in deprecated_urls: + warnings.warn(f'open-mmlab://{model_name} is deprecated in favor ' + f'of open-mmlab://{deprecated_urls[model_name]}') + model_name = deprecated_urls[model_name] + model_url = model_urls[model_name] + # check if is url + if model_url.startswith(('http://', 'https://')): + checkpoint = load_url_dist(model_url) + else: + filename = osp.join(_get_mmcv_home(), model_url) + if not osp.isfile(filename): + raise IOError(f'{filename} is not a checkpoint file') + checkpoint = torch.load(filename, map_location=map_location) + elif filename.startswith('mmcls://'): + model_urls = get_mmcls_models() + model_name = filename[8:] + checkpoint = load_url_dist(model_urls[model_name]) + checkpoint = _process_mmcls_checkpoint(checkpoint) + elif filename.startswith(('http://', 'https://')): + checkpoint = load_url_dist(filename) + elif filename.startswith('pavi://'): + model_path = filename[7:] + checkpoint = load_pavimodel_dist(model_path, map_location=map_location) + elif filename.startswith('s3://'): + checkpoint = load_fileclient_dist( + filename, backend='ceph', map_location=map_location) + else: + if not osp.isfile(filename): + raise IOError(f'{filename} is not a checkpoint file') + checkpoint = torch.load(filename, map_location=map_location) + return checkpoint + + +def load_checkpoint(model, + filename, + map_location='cpu', + strict=False, + logger=None): + """Load checkpoint from a file or URI. + + Args: + model (Module): Module to load checkpoint. + filename (str): Accept local filepath, URL, ``torchvision://xxx``, + ``open-mmlab://xxx``. Please refer to ``docs/model_zoo.md`` for + details. + map_location (str): Same as :func:`torch.load`. + strict (bool): Whether to allow different params for the model and + checkpoint. + logger (:mod:`logging.Logger` or None): The logger for error message. + + Returns: + dict or OrderedDict: The loaded checkpoint. + """ + checkpoint = _load_checkpoint(filename, map_location) + # OrderedDict is a subclass of dict + if not isinstance(checkpoint, dict): + raise RuntimeError( + f'No state_dict found in checkpoint file {filename}') + # get state_dict from checkpoint + if 'state_dict' in checkpoint: + state_dict = checkpoint['state_dict'] + elif 'model' in checkpoint: + state_dict = checkpoint['model'] + else: + state_dict = checkpoint + # strip prefix of state_dict + if list(state_dict.keys())[0].startswith('module.'): + state_dict = {k[7:]: v for k, v in state_dict.items()} + + # for MoBY, load model of online branch + if sorted(list(state_dict.keys()))[0].startswith('encoder'): + state_dict = {k.replace('encoder.', ''): v for k, v in state_dict.items() if k.startswith('encoder.')} + + # reshape absolute position embedding + if state_dict.get('absolute_pos_embed') is not None: + absolute_pos_embed = state_dict['absolute_pos_embed'] + N1, L, C1 = absolute_pos_embed.size() + N2, C2, H, W = model.absolute_pos_embed.size() + if N1 != N2 or C1 != C2 or L != H*W: + logger.warning("Error in loading absolute_pos_embed, pass") + else: + state_dict['absolute_pos_embed'] = absolute_pos_embed.view(N2, H, W, C2).permute(0, 3, 1, 2) + + # interpolate position bias table if needed + relative_position_bias_table_keys = [k for k in state_dict.keys() if "relative_position_bias_table" in k] + for table_key in relative_position_bias_table_keys: + table_pretrained = state_dict[table_key] + table_current = model.state_dict()[table_key] + L1, nH1 = table_pretrained.size() + L2, nH2 = table_current.size() + if nH1 != nH2: + logger.warning(f"Error in loading {table_key}, pass") + else: + if L1 != L2: + S1 = int(L1 ** 0.5) + S2 = int(L2 ** 0.5) + table_pretrained_resized = F.interpolate( + table_pretrained.permute(1, 0).view(1, nH1, S1, S1), + size=(S2, S2), mode='bicubic') + state_dict[table_key] = table_pretrained_resized.view(nH2, L2).permute(1, 0) + + # load state_dict + load_state_dict(model, state_dict, strict, logger) + return checkpoint + + +def weights_to_cpu(state_dict): + """Copy a model state_dict to cpu. + + Args: + state_dict (OrderedDict): Model weights on GPU. + + Returns: + OrderedDict: Model weights on GPU. + """ + state_dict_cpu = OrderedDict() + for key, val in state_dict.items(): + state_dict_cpu[key] = val.cpu() + return state_dict_cpu + + +def _save_to_state_dict(module, destination, prefix, keep_vars): + """Saves module state to `destination` dictionary. + + This method is modified from :meth:`torch.nn.Module._save_to_state_dict`. + + Args: + module (nn.Module): The module to generate state_dict. + destination (dict): A dict where state will be stored. + prefix (str): The prefix for parameters and buffers used in this + module. + """ + for name, param in module._parameters.items(): + if param is not None: + destination[prefix + name] = param if keep_vars else param.detach() + for name, buf in module._buffers.items(): + # remove check of _non_persistent_buffers_set to allow nn.BatchNorm2d + if buf is not None: + destination[prefix + name] = buf if keep_vars else buf.detach() + + +def get_state_dict(module, destination=None, prefix='', keep_vars=False): + """Returns a dictionary containing a whole state of the module. + + Both parameters and persistent buffers (e.g. running averages) are + included. Keys are corresponding parameter and buffer names. + + This method is modified from :meth:`torch.nn.Module.state_dict` to + recursively check parallel module in case that the model has a complicated + structure, e.g., nn.Module(nn.Module(DDP)). + + Args: + module (nn.Module): The module to generate state_dict. + destination (OrderedDict): Returned dict for the state of the + module. + prefix (str): Prefix of the key. + keep_vars (bool): Whether to keep the variable property of the + parameters. Default: False. + + Returns: + dict: A dictionary containing a whole state of the module. + """ + # recursively check parallel module in case that the model has a + # complicated structure, e.g., nn.Module(nn.Module(DDP)) + if is_module_wrapper(module): + module = module.module + + # below is the same as torch.nn.Module.state_dict() + if destination is None: + destination = OrderedDict() + destination._metadata = OrderedDict() + destination._metadata[prefix[:-1]] = local_metadata = dict( + version=module._version) + _save_to_state_dict(module, destination, prefix, keep_vars) + for name, child in module._modules.items(): + if child is not None: + get_state_dict( + child, destination, prefix + name + '.', keep_vars=keep_vars) + for hook in module._state_dict_hooks.values(): + hook_result = hook(module, destination, prefix, local_metadata) + if hook_result is not None: + destination = hook_result + return destination + + +def save_checkpoint(model, filename, optimizer=None, meta=None): + """Save checkpoint to file. + + The checkpoint will have 3 fields: ``meta``, ``state_dict`` and + ``optimizer``. By default ``meta`` will contain version and time info. + + Args: + model (Module): Module whose params are to be saved. + filename (str): Checkpoint filename. + optimizer (:obj:`Optimizer`, optional): Optimizer to be saved. + meta (dict, optional): Metadata to be saved in checkpoint. + """ + if meta is None: + meta = {} + elif not isinstance(meta, dict): + raise TypeError(f'meta must be a dict or None, but got {type(meta)}') + meta.update(mmcv_version=mmcv.__version__, time=time.asctime()) + + if is_module_wrapper(model): + model = model.module + + if hasattr(model, 'CLASSES') and model.CLASSES is not None: + # save class name to the meta + meta.update(CLASSES=model.CLASSES) + + checkpoint = { + 'meta': meta, + 'state_dict': weights_to_cpu(get_state_dict(model)) + } + # save optimizer state dict in the checkpoint + if isinstance(optimizer, Optimizer): + checkpoint['optimizer'] = optimizer.state_dict() + elif isinstance(optimizer, dict): + checkpoint['optimizer'] = {} + for name, optim in optimizer.items(): + checkpoint['optimizer'][name] = optim.state_dict() + + if filename.startswith('pavi://'): + try: + from pavi import modelcloud + from pavi.exception import NodeNotFoundError + except ImportError: + raise ImportError( + 'Please install pavi to load checkpoint from modelcloud.') + model_path = filename[7:] + root = modelcloud.Folder() + model_dir, model_name = osp.split(model_path) + try: + model = modelcloud.get(model_dir) + except NodeNotFoundError: + model = root.create_training_model(model_dir) + with TemporaryDirectory() as tmp_dir: + checkpoint_file = osp.join(tmp_dir, model_name) + with open(checkpoint_file, 'wb') as f: + torch.save(checkpoint, f) + f.flush() + model.create_file(checkpoint_file, name=model_name) + else: + mmcv.mkdir_or_exist(osp.dirname(filename)) + # immediately flush buffer + with open(filename, 'wb') as f: + torch.save(checkpoint, f) + f.flush() \ No newline at end of file diff --git a/annotator/uniformer/mmseg/apis/__init__.py b/annotator/uniformer/mmseg/apis/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..170724be38de42daf2bc1a1910e181d68818f165 --- /dev/null +++ b/annotator/uniformer/mmseg/apis/__init__.py @@ -0,0 +1,9 @@ +from .inference import inference_segmentor, init_segmentor, show_result_pyplot +from .test import multi_gpu_test, single_gpu_test +from .train import get_root_logger, set_random_seed, train_segmentor + +__all__ = [ + 'get_root_logger', 'set_random_seed', 'train_segmentor', 'init_segmentor', + 'inference_segmentor', 'multi_gpu_test', 'single_gpu_test', + 'show_result_pyplot' +] diff --git a/annotator/uniformer/mmseg/apis/inference.py b/annotator/uniformer/mmseg/apis/inference.py new file mode 100644 index 0000000000000000000000000000000000000000..90bc1c0c68525734bd6793f07c15fe97d3c8342c --- /dev/null +++ b/annotator/uniformer/mmseg/apis/inference.py @@ -0,0 +1,136 @@ +import matplotlib.pyplot as plt +import annotator.uniformer.mmcv as mmcv +import torch +from annotator.uniformer.mmcv.parallel import collate, scatter +from annotator.uniformer.mmcv.runner import load_checkpoint + +from annotator.uniformer.mmseg.datasets.pipelines import Compose +from annotator.uniformer.mmseg.models import build_segmentor + + +def init_segmentor(config, checkpoint=None, device='cuda:0'): + """Initialize a segmentor from config file. + + Args: + config (str or :obj:`mmcv.Config`): Config file path or the config + object. + checkpoint (str, optional): Checkpoint path. If left as None, the model + will not load any weights. + device (str, optional) CPU/CUDA device option. Default 'cuda:0'. + Use 'cpu' for loading model on CPU. + Returns: + nn.Module: The constructed segmentor. + """ + if isinstance(config, str): + config = mmcv.Config.fromfile(config) + elif not isinstance(config, mmcv.Config): + raise TypeError('config must be a filename or Config object, ' + 'but got {}'.format(type(config))) + config.model.pretrained = None + config.model.train_cfg = None + model = build_segmentor(config.model, test_cfg=config.get('test_cfg')) + if checkpoint is not None: + checkpoint = load_checkpoint(model, checkpoint, map_location='cpu') + model.CLASSES = checkpoint['meta']['CLASSES'] + model.PALETTE = checkpoint['meta']['PALETTE'] + model.cfg = config # save the config in the model for convenience + model.to(device) + model.eval() + return model + + +class LoadImage: + """A simple pipeline to load image.""" + + def __call__(self, results): + """Call function to load images into results. + + Args: + results (dict): A result dict contains the file name + of the image to be read. + + Returns: + dict: ``results`` will be returned containing loaded image. + """ + + if isinstance(results['img'], str): + results['filename'] = results['img'] + results['ori_filename'] = results['img'] + else: + results['filename'] = None + results['ori_filename'] = None + img = mmcv.imread(results['img']) + results['img'] = img + results['img_shape'] = img.shape + results['ori_shape'] = img.shape + return results + + +def inference_segmentor(model, img): + """Inference image(s) with the segmentor. + + Args: + model (nn.Module): The loaded segmentor. + imgs (str/ndarray or list[str/ndarray]): Either image files or loaded + images. + + Returns: + (list[Tensor]): The segmentation result. + """ + cfg = model.cfg + device = next(model.parameters()).device # model device + # build the data pipeline + test_pipeline = [LoadImage()] + cfg.data.test.pipeline[1:] + test_pipeline = Compose(test_pipeline) + # prepare data + data = dict(img=img) + data = test_pipeline(data) + data = collate([data], samples_per_gpu=1) + if next(model.parameters()).is_cuda: + # scatter to specified GPU + data = scatter(data, [device])[0] + else: + data['img_metas'] = [i.data[0] for i in data['img_metas']] + + # forward the model + with torch.no_grad(): + result = model(return_loss=False, rescale=True, **data) + return result + + +def show_result_pyplot(model, + img, + result, + palette=None, + fig_size=(15, 10), + opacity=0.5, + title='', + block=True): + """Visualize the segmentation results on the image. + + Args: + model (nn.Module): The loaded segmentor. + img (str or np.ndarray): Image filename or loaded image. + result (list): The segmentation result. + palette (list[list[int]]] | None): The palette of segmentation + map. If None is given, random palette will be generated. + Default: None + fig_size (tuple): Figure size of the pyplot figure. + opacity(float): Opacity of painted segmentation map. + Default 0.5. + Must be in (0, 1] range. + title (str): The title of pyplot figure. + Default is ''. + block (bool): Whether to block the pyplot figure. + Default is True. + """ + if hasattr(model, 'module'): + model = model.module + img = model.show_result( + img, result, palette=palette, show=False, opacity=opacity) + # plt.figure(figsize=fig_size) + # plt.imshow(mmcv.bgr2rgb(img)) + # plt.title(title) + # plt.tight_layout() + # plt.show(block=block) + return mmcv.bgr2rgb(img) diff --git a/annotator/uniformer/mmseg/apis/test.py b/annotator/uniformer/mmseg/apis/test.py new file mode 100644 index 0000000000000000000000000000000000000000..e574eb7da04f09a59cf99ff953c36468ae87a326 --- /dev/null +++ b/annotator/uniformer/mmseg/apis/test.py @@ -0,0 +1,238 @@ +import os.path as osp +import pickle +import shutil +import tempfile + +import annotator.uniformer.mmcv as mmcv +import numpy as np +import torch +import torch.distributed as dist +from annotator.uniformer.mmcv.image import tensor2imgs +from annotator.uniformer.mmcv.runner import get_dist_info + + +def np2tmp(array, temp_file_name=None): + """Save ndarray to local numpy file. + + Args: + array (ndarray): Ndarray to save. + temp_file_name (str): Numpy file name. If 'temp_file_name=None', this + function will generate a file name with tempfile.NamedTemporaryFile + to save ndarray. Default: None. + + Returns: + str: The numpy file name. + """ + + if temp_file_name is None: + temp_file_name = tempfile.NamedTemporaryFile( + suffix='.npy', delete=False).name + np.save(temp_file_name, array) + return temp_file_name + + +def single_gpu_test(model, + data_loader, + show=False, + out_dir=None, + efficient_test=False, + opacity=0.5): + """Test with single GPU. + + Args: + model (nn.Module): Model to be tested. + data_loader (utils.data.Dataloader): Pytorch data loader. + show (bool): Whether show results during inference. Default: False. + out_dir (str, optional): If specified, the results will be dumped into + the directory to save output results. + efficient_test (bool): Whether save the results as local numpy files to + save CPU memory during evaluation. Default: False. + opacity(float): Opacity of painted segmentation map. + Default 0.5. + Must be in (0, 1] range. + Returns: + list: The prediction results. + """ + + model.eval() + results = [] + dataset = data_loader.dataset + prog_bar = mmcv.ProgressBar(len(dataset)) + for i, data in enumerate(data_loader): + with torch.no_grad(): + result = model(return_loss=False, **data) + + if show or out_dir: + img_tensor = data['img'][0] + img_metas = data['img_metas'][0].data[0] + imgs = tensor2imgs(img_tensor, **img_metas[0]['img_norm_cfg']) + assert len(imgs) == len(img_metas) + + for img, img_meta in zip(imgs, img_metas): + h, w, _ = img_meta['img_shape'] + img_show = img[:h, :w, :] + + ori_h, ori_w = img_meta['ori_shape'][:-1] + img_show = mmcv.imresize(img_show, (ori_w, ori_h)) + + if out_dir: + out_file = osp.join(out_dir, img_meta['ori_filename']) + else: + out_file = None + + model.module.show_result( + img_show, + result, + palette=dataset.PALETTE, + show=show, + out_file=out_file, + opacity=opacity) + + if isinstance(result, list): + if efficient_test: + result = [np2tmp(_) for _ in result] + results.extend(result) + else: + if efficient_test: + result = np2tmp(result) + results.append(result) + + batch_size = len(result) + for _ in range(batch_size): + prog_bar.update() + return results + + +def multi_gpu_test(model, + data_loader, + tmpdir=None, + gpu_collect=False, + efficient_test=False): + """Test model with multiple gpus. + + This method tests model with multiple gpus and collects the results + under two different modes: gpu and cpu modes. By setting 'gpu_collect=True' + it encodes results to gpu tensors and use gpu communication for results + collection. On cpu mode it saves the results on different gpus to 'tmpdir' + and collects them by the rank 0 worker. + + Args: + model (nn.Module): Model to be tested. + data_loader (utils.data.Dataloader): Pytorch data loader. + tmpdir (str): Path of directory to save the temporary results from + different gpus under cpu mode. + gpu_collect (bool): Option to use either gpu or cpu to collect results. + efficient_test (bool): Whether save the results as local numpy files to + save CPU memory during evaluation. Default: False. + + Returns: + list: The prediction results. + """ + + model.eval() + results = [] + dataset = data_loader.dataset + rank, world_size = get_dist_info() + if rank == 0: + prog_bar = mmcv.ProgressBar(len(dataset)) + for i, data in enumerate(data_loader): + with torch.no_grad(): + result = model(return_loss=False, rescale=True, **data) + + if isinstance(result, list): + if efficient_test: + result = [np2tmp(_) for _ in result] + results.extend(result) + else: + if efficient_test: + result = np2tmp(result) + results.append(result) + + if rank == 0: + batch_size = data['img'][0].size(0) + for _ in range(batch_size * world_size): + prog_bar.update() + + # collect results from all ranks + if gpu_collect: + results = collect_results_gpu(results, len(dataset)) + else: + results = collect_results_cpu(results, len(dataset), tmpdir) + return results + + +def collect_results_cpu(result_part, size, tmpdir=None): + """Collect results with CPU.""" + rank, world_size = get_dist_info() + # create a tmp dir if it is not specified + if tmpdir is None: + MAX_LEN = 512 + # 32 is whitespace + dir_tensor = torch.full((MAX_LEN, ), + 32, + dtype=torch.uint8, + device='cuda') + if rank == 0: + tmpdir = tempfile.mkdtemp() + tmpdir = torch.tensor( + bytearray(tmpdir.encode()), dtype=torch.uint8, device='cuda') + dir_tensor[:len(tmpdir)] = tmpdir + dist.broadcast(dir_tensor, 0) + tmpdir = dir_tensor.cpu().numpy().tobytes().decode().rstrip() + else: + mmcv.mkdir_or_exist(tmpdir) + # dump the part result to the dir + mmcv.dump(result_part, osp.join(tmpdir, 'part_{}.pkl'.format(rank))) + dist.barrier() + # collect all parts + if rank != 0: + return None + else: + # load results of all parts from tmp dir + part_list = [] + for i in range(world_size): + part_file = osp.join(tmpdir, 'part_{}.pkl'.format(i)) + part_list.append(mmcv.load(part_file)) + # sort the results + ordered_results = [] + for res in zip(*part_list): + ordered_results.extend(list(res)) + # the dataloader may pad some samples + ordered_results = ordered_results[:size] + # remove tmp dir + shutil.rmtree(tmpdir) + return ordered_results + + +def collect_results_gpu(result_part, size): + """Collect results with GPU.""" + rank, world_size = get_dist_info() + # dump result part to tensor with pickle + part_tensor = torch.tensor( + bytearray(pickle.dumps(result_part)), dtype=torch.uint8, device='cuda') + # gather all result part tensor shape + shape_tensor = torch.tensor(part_tensor.shape, device='cuda') + shape_list = [shape_tensor.clone() for _ in range(world_size)] + dist.all_gather(shape_list, shape_tensor) + # padding result part tensor to max length + shape_max = torch.tensor(shape_list).max() + part_send = torch.zeros(shape_max, dtype=torch.uint8, device='cuda') + part_send[:shape_tensor[0]] = part_tensor + part_recv_list = [ + part_tensor.new_zeros(shape_max) for _ in range(world_size) + ] + # gather all result part + dist.all_gather(part_recv_list, part_send) + + if rank == 0: + part_list = [] + for recv, shape in zip(part_recv_list, shape_list): + part_list.append( + pickle.loads(recv[:shape[0]].cpu().numpy().tobytes())) + # sort the results + ordered_results = [] + for res in zip(*part_list): + ordered_results.extend(list(res)) + # the dataloader may pad some samples + ordered_results = ordered_results[:size] + return ordered_results diff --git a/annotator/uniformer/mmseg/apis/train.py b/annotator/uniformer/mmseg/apis/train.py new file mode 100644 index 0000000000000000000000000000000000000000..63f319a919ff023931a6a663e668f27dd1a07a2e --- /dev/null +++ b/annotator/uniformer/mmseg/apis/train.py @@ -0,0 +1,116 @@ +import random +import warnings + +import numpy as np +import torch +from annotator.uniformer.mmcv.parallel import MMDataParallel, MMDistributedDataParallel +from annotator.uniformer.mmcv.runner import build_optimizer, build_runner + +from annotator.uniformer.mmseg.core import DistEvalHook, EvalHook +from annotator.uniformer.mmseg.datasets import build_dataloader, build_dataset +from annotator.uniformer.mmseg.utils import get_root_logger + + +def set_random_seed(seed, deterministic=False): + """Set random seed. + + Args: + seed (int): Seed to be used. + deterministic (bool): Whether to set the deterministic option for + CUDNN backend, i.e., set `torch.backends.cudnn.deterministic` + to True and `torch.backends.cudnn.benchmark` to False. + Default: False. + """ + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + if deterministic: + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + +def train_segmentor(model, + dataset, + cfg, + distributed=False, + validate=False, + timestamp=None, + meta=None): + """Launch segmentor training.""" + logger = get_root_logger(cfg.log_level) + + # prepare data loaders + dataset = dataset if isinstance(dataset, (list, tuple)) else [dataset] + data_loaders = [ + build_dataloader( + ds, + cfg.data.samples_per_gpu, + cfg.data.workers_per_gpu, + # cfg.gpus will be ignored if distributed + len(cfg.gpu_ids), + dist=distributed, + seed=cfg.seed, + drop_last=True) for ds in dataset + ] + + # put model on gpus + if distributed: + find_unused_parameters = cfg.get('find_unused_parameters', False) + # Sets the `find_unused_parameters` parameter in + # torch.nn.parallel.DistributedDataParallel + model = MMDistributedDataParallel( + model.cuda(), + device_ids=[torch.cuda.current_device()], + broadcast_buffers=False, + find_unused_parameters=find_unused_parameters) + else: + model = MMDataParallel( + model.cuda(cfg.gpu_ids[0]), device_ids=cfg.gpu_ids) + + # build runner + optimizer = build_optimizer(model, cfg.optimizer) + + if cfg.get('runner') is None: + cfg.runner = {'type': 'IterBasedRunner', 'max_iters': cfg.total_iters} + warnings.warn( + 'config is now expected to have a `runner` section, ' + 'please set `runner` in your config.', UserWarning) + + runner = build_runner( + cfg.runner, + default_args=dict( + model=model, + batch_processor=None, + optimizer=optimizer, + work_dir=cfg.work_dir, + logger=logger, + meta=meta)) + + # register hooks + runner.register_training_hooks(cfg.lr_config, cfg.optimizer_config, + cfg.checkpoint_config, cfg.log_config, + cfg.get('momentum_config', None)) + + # an ugly walkaround to make the .log and .log.json filenames the same + runner.timestamp = timestamp + + # register eval hooks + if validate: + val_dataset = build_dataset(cfg.data.val, dict(test_mode=True)) + val_dataloader = build_dataloader( + val_dataset, + samples_per_gpu=1, + workers_per_gpu=cfg.data.workers_per_gpu, + dist=distributed, + shuffle=False) + eval_cfg = cfg.get('evaluation', {}) + eval_cfg['by_epoch'] = cfg.runner['type'] != 'IterBasedRunner' + eval_hook = DistEvalHook if distributed else EvalHook + runner.register_hook(eval_hook(val_dataloader, **eval_cfg), priority='LOW') + + if cfg.resume_from: + runner.resume(cfg.resume_from) + elif cfg.load_from: + runner.load_checkpoint(cfg.load_from) + runner.run(data_loaders, cfg.workflow) diff --git a/annotator/uniformer/mmseg/core/__init__.py b/annotator/uniformer/mmseg/core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..965605587211b7bf0bd6bc3acdbb33dd49cab023 --- /dev/null +++ b/annotator/uniformer/mmseg/core/__init__.py @@ -0,0 +1,3 @@ +from .evaluation import * # noqa: F401, F403 +from .seg import * # noqa: F401, F403 +from .utils import * # noqa: F401, F403 diff --git a/annotator/uniformer/mmseg/core/evaluation/__init__.py b/annotator/uniformer/mmseg/core/evaluation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f7cc4b23413a0639e9de00eeb0bf600632d2c6cd --- /dev/null +++ b/annotator/uniformer/mmseg/core/evaluation/__init__.py @@ -0,0 +1,8 @@ +from .class_names import get_classes, get_palette +from .eval_hooks import DistEvalHook, EvalHook +from .metrics import eval_metrics, mean_dice, mean_fscore, mean_iou + +__all__ = [ + 'EvalHook', 'DistEvalHook', 'mean_dice', 'mean_iou', 'mean_fscore', + 'eval_metrics', 'get_classes', 'get_palette' +] diff --git a/annotator/uniformer/mmseg/core/evaluation/class_names.py b/annotator/uniformer/mmseg/core/evaluation/class_names.py new file mode 100644 index 0000000000000000000000000000000000000000..ffae816cf980ce4b03e491cc0c4298cb823797e6 --- /dev/null +++ b/annotator/uniformer/mmseg/core/evaluation/class_names.py @@ -0,0 +1,152 @@ +import annotator.uniformer.mmcv as mmcv + + +def cityscapes_classes(): + """Cityscapes class names for external use.""" + return [ + 'road', 'sidewalk', 'building', 'wall', 'fence', 'pole', + 'traffic light', 'traffic sign', 'vegetation', 'terrain', 'sky', + 'person', 'rider', 'car', 'truck', 'bus', 'train', 'motorcycle', + 'bicycle' + ] + + +def ade_classes(): + """ADE20K class names for external use.""" + return [ + 'wall', 'building', 'sky', 'floor', 'tree', 'ceiling', 'road', 'bed ', + 'windowpane', 'grass', 'cabinet', 'sidewalk', 'person', 'earth', + 'door', 'table', 'mountain', 'plant', 'curtain', 'chair', 'car', + 'water', 'painting', 'sofa', 'shelf', 'house', 'sea', 'mirror', 'rug', + 'field', 'armchair', 'seat', 'fence', 'desk', 'rock', 'wardrobe', + 'lamp', 'bathtub', 'railing', 'cushion', 'base', 'box', 'column', + 'signboard', 'chest of drawers', 'counter', 'sand', 'sink', + 'skyscraper', 'fireplace', 'refrigerator', 'grandstand', 'path', + 'stairs', 'runway', 'case', 'pool table', 'pillow', 'screen door', + 'stairway', 'river', 'bridge', 'bookcase', 'blind', 'coffee table', + 'toilet', 'flower', 'book', 'hill', 'bench', 'countertop', 'stove', + 'palm', 'kitchen island', 'computer', 'swivel chair', 'boat', 'bar', + 'arcade machine', 'hovel', 'bus', 'towel', 'light', 'truck', 'tower', + 'chandelier', 'awning', 'streetlight', 'booth', 'television receiver', + 'airplane', 'dirt track', 'apparel', 'pole', 'land', 'bannister', + 'escalator', 'ottoman', 'bottle', 'buffet', 'poster', 'stage', 'van', + 'ship', 'fountain', 'conveyer belt', 'canopy', 'washer', 'plaything', + 'swimming pool', 'stool', 'barrel', 'basket', 'waterfall', 'tent', + 'bag', 'minibike', 'cradle', 'oven', 'ball', 'food', 'step', 'tank', + 'trade name', 'microwave', 'pot', 'animal', 'bicycle', 'lake', + 'dishwasher', 'screen', 'blanket', 'sculpture', 'hood', 'sconce', + 'vase', 'traffic light', 'tray', 'ashcan', 'fan', 'pier', 'crt screen', + 'plate', 'monitor', 'bulletin board', 'shower', 'radiator', 'glass', + 'clock', 'flag' + ] + + +def voc_classes(): + """Pascal VOC class names for external use.""" + return [ + 'background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', + 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', + 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', + 'tvmonitor' + ] + + +def cityscapes_palette(): + """Cityscapes palette for external use.""" + return [[128, 64, 128], [244, 35, 232], [70, 70, 70], [102, 102, 156], + [190, 153, 153], [153, 153, 153], [250, 170, 30], [220, 220, 0], + [107, 142, 35], [152, 251, 152], [70, 130, 180], [220, 20, 60], + [255, 0, 0], [0, 0, 142], [0, 0, 70], [0, 60, 100], [0, 80, 100], + [0, 0, 230], [119, 11, 32]] + + +def ade_palette(): + """ADE20K palette for external use.""" + return [[120, 120, 120], [180, 120, 120], [6, 230, 230], [80, 50, 50], + [4, 200, 3], [120, 120, 80], [140, 140, 140], [204, 5, 255], + [230, 230, 230], [4, 250, 7], [224, 5, 255], [235, 255, 7], + [150, 5, 61], [120, 120, 70], [8, 255, 51], [255, 6, 82], + [143, 255, 140], [204, 255, 4], [255, 51, 7], [204, 70, 3], + [0, 102, 200], [61, 230, 250], [255, 6, 51], [11, 102, 255], + [255, 7, 71], [255, 9, 224], [9, 7, 230], [220, 220, 220], + [255, 9, 92], [112, 9, 255], [8, 255, 214], [7, 255, 224], + [255, 184, 6], [10, 255, 71], [255, 41, 10], [7, 255, 255], + [224, 255, 8], [102, 8, 255], [255, 61, 6], [255, 194, 7], + [255, 122, 8], [0, 255, 20], [255, 8, 41], [255, 5, 153], + [6, 51, 255], [235, 12, 255], [160, 150, 20], [0, 163, 255], + [140, 140, 140], [250, 10, 15], [20, 255, 0], [31, 255, 0], + [255, 31, 0], [255, 224, 0], [153, 255, 0], [0, 0, 255], + [255, 71, 0], [0, 235, 255], [0, 173, 255], [31, 0, 255], + [11, 200, 200], [255, 82, 0], [0, 255, 245], [0, 61, 255], + [0, 255, 112], [0, 255, 133], [255, 0, 0], [255, 163, 0], + [255, 102, 0], [194, 255, 0], [0, 143, 255], [51, 255, 0], + [0, 82, 255], [0, 255, 41], [0, 255, 173], [10, 0, 255], + [173, 255, 0], [0, 255, 153], [255, 92, 0], [255, 0, 255], + [255, 0, 245], [255, 0, 102], [255, 173, 0], [255, 0, 20], + [255, 184, 184], [0, 31, 255], [0, 255, 61], [0, 71, 255], + [255, 0, 204], [0, 255, 194], [0, 255, 82], [0, 10, 255], + [0, 112, 255], [51, 0, 255], [0, 194, 255], [0, 122, 255], + [0, 255, 163], [255, 153, 0], [0, 255, 10], [255, 112, 0], + [143, 255, 0], [82, 0, 255], [163, 255, 0], [255, 235, 0], + [8, 184, 170], [133, 0, 255], [0, 255, 92], [184, 0, 255], + [255, 0, 31], [0, 184, 255], [0, 214, 255], [255, 0, 112], + [92, 255, 0], [0, 224, 255], [112, 224, 255], [70, 184, 160], + [163, 0, 255], [153, 0, 255], [71, 255, 0], [255, 0, 163], + [255, 204, 0], [255, 0, 143], [0, 255, 235], [133, 255, 0], + [255, 0, 235], [245, 0, 255], [255, 0, 122], [255, 245, 0], + [10, 190, 212], [214, 255, 0], [0, 204, 255], [20, 0, 255], + [255, 255, 0], [0, 153, 255], [0, 41, 255], [0, 255, 204], + [41, 0, 255], [41, 255, 0], [173, 0, 255], [0, 245, 255], + [71, 0, 255], [122, 0, 255], [0, 255, 184], [0, 92, 255], + [184, 255, 0], [0, 133, 255], [255, 214, 0], [25, 194, 194], + [102, 255, 0], [92, 0, 255]] + + +def voc_palette(): + """Pascal VOC palette for external use.""" + return [[0, 0, 0], [128, 0, 0], [0, 128, 0], [128, 128, 0], [0, 0, 128], + [128, 0, 128], [0, 128, 128], [128, 128, 128], [64, 0, 0], + [192, 0, 0], [64, 128, 0], [192, 128, 0], [64, 0, 128], + [192, 0, 128], [64, 128, 128], [192, 128, 128], [0, 64, 0], + [128, 64, 0], [0, 192, 0], [128, 192, 0], [0, 64, 128]] + + +dataset_aliases = { + 'cityscapes': ['cityscapes'], + 'ade': ['ade', 'ade20k'], + 'voc': ['voc', 'pascal_voc', 'voc12', 'voc12aug'] +} + + +def get_classes(dataset): + """Get class names of a dataset.""" + alias2name = {} + for name, aliases in dataset_aliases.items(): + for alias in aliases: + alias2name[alias] = name + + if mmcv.is_str(dataset): + if dataset in alias2name: + labels = eval(alias2name[dataset] + '_classes()') + else: + raise ValueError(f'Unrecognized dataset: {dataset}') + else: + raise TypeError(f'dataset must a str, but got {type(dataset)}') + return labels + + +def get_palette(dataset): + """Get class palette (RGB) of a dataset.""" + alias2name = {} + for name, aliases in dataset_aliases.items(): + for alias in aliases: + alias2name[alias] = name + + if mmcv.is_str(dataset): + if dataset in alias2name: + labels = eval(alias2name[dataset] + '_palette()') + else: + raise ValueError(f'Unrecognized dataset: {dataset}') + else: + raise TypeError(f'dataset must a str, but got {type(dataset)}') + return labels diff --git a/annotator/uniformer/mmseg/core/evaluation/eval_hooks.py b/annotator/uniformer/mmseg/core/evaluation/eval_hooks.py new file mode 100644 index 0000000000000000000000000000000000000000..6fc100c8f96e817a6ed2666f7c9f762af2463b48 --- /dev/null +++ b/annotator/uniformer/mmseg/core/evaluation/eval_hooks.py @@ -0,0 +1,109 @@ +import os.path as osp + +from annotator.uniformer.mmcv.runner import DistEvalHook as _DistEvalHook +from annotator.uniformer.mmcv.runner import EvalHook as _EvalHook + + +class EvalHook(_EvalHook): + """Single GPU EvalHook, with efficient test support. + + Args: + by_epoch (bool): Determine perform evaluation by epoch or by iteration. + If set to True, it will perform by epoch. Otherwise, by iteration. + Default: False. + efficient_test (bool): Whether save the results as local numpy files to + save CPU memory during evaluation. Default: False. + Returns: + list: The prediction results. + """ + + greater_keys = ['mIoU', 'mAcc', 'aAcc'] + + def __init__(self, *args, by_epoch=False, efficient_test=False, **kwargs): + super().__init__(*args, by_epoch=by_epoch, **kwargs) + self.efficient_test = efficient_test + + def after_train_iter(self, runner): + """After train epoch hook. + + Override default ``single_gpu_test``. + """ + if self.by_epoch or not self.every_n_iters(runner, self.interval): + return + from annotator.uniformer.mmseg.apis import single_gpu_test + runner.log_buffer.clear() + results = single_gpu_test( + runner.model, + self.dataloader, + show=False, + efficient_test=self.efficient_test) + self.evaluate(runner, results) + + def after_train_epoch(self, runner): + """After train epoch hook. + + Override default ``single_gpu_test``. + """ + if not self.by_epoch or not self.every_n_epochs(runner, self.interval): + return + from annotator.uniformer.mmseg.apis import single_gpu_test + runner.log_buffer.clear() + results = single_gpu_test(runner.model, self.dataloader, show=False) + self.evaluate(runner, results) + + +class DistEvalHook(_DistEvalHook): + """Distributed EvalHook, with efficient test support. + + Args: + by_epoch (bool): Determine perform evaluation by epoch or by iteration. + If set to True, it will perform by epoch. Otherwise, by iteration. + Default: False. + efficient_test (bool): Whether save the results as local numpy files to + save CPU memory during evaluation. Default: False. + Returns: + list: The prediction results. + """ + + greater_keys = ['mIoU', 'mAcc', 'aAcc'] + + def __init__(self, *args, by_epoch=False, efficient_test=False, **kwargs): + super().__init__(*args, by_epoch=by_epoch, **kwargs) + self.efficient_test = efficient_test + + def after_train_iter(self, runner): + """After train epoch hook. + + Override default ``multi_gpu_test``. + """ + if self.by_epoch or not self.every_n_iters(runner, self.interval): + return + from annotator.uniformer.mmseg.apis import multi_gpu_test + runner.log_buffer.clear() + results = multi_gpu_test( + runner.model, + self.dataloader, + tmpdir=osp.join(runner.work_dir, '.eval_hook'), + gpu_collect=self.gpu_collect, + efficient_test=self.efficient_test) + if runner.rank == 0: + print('\n') + self.evaluate(runner, results) + + def after_train_epoch(self, runner): + """After train epoch hook. + + Override default ``multi_gpu_test``. + """ + if not self.by_epoch or not self.every_n_epochs(runner, self.interval): + return + from annotator.uniformer.mmseg.apis import multi_gpu_test + runner.log_buffer.clear() + results = multi_gpu_test( + runner.model, + self.dataloader, + tmpdir=osp.join(runner.work_dir, '.eval_hook'), + gpu_collect=self.gpu_collect) + if runner.rank == 0: + print('\n') + self.evaluate(runner, results) diff --git a/annotator/uniformer/mmseg/core/evaluation/metrics.py b/annotator/uniformer/mmseg/core/evaluation/metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..16c7dd47cadd53cf1caaa194e28a343f2aacc599 --- /dev/null +++ b/annotator/uniformer/mmseg/core/evaluation/metrics.py @@ -0,0 +1,326 @@ +from collections import OrderedDict + +import annotator.uniformer.mmcv as mmcv +import numpy as np +import torch + + +def f_score(precision, recall, beta=1): + """calcuate the f-score value. + + Args: + precision (float | torch.Tensor): The precision value. + recall (float | torch.Tensor): The recall value. + beta (int): Determines the weight of recall in the combined score. + Default: False. + + Returns: + [torch.tensor]: The f-score value. + """ + score = (1 + beta**2) * (precision * recall) / ( + (beta**2 * precision) + recall) + return score + + +def intersect_and_union(pred_label, + label, + num_classes, + ignore_index, + label_map=dict(), + reduce_zero_label=False): + """Calculate intersection and Union. + + Args: + pred_label (ndarray | str): Prediction segmentation map + or predict result filename. + label (ndarray | str): Ground truth segmentation map + or label filename. + num_classes (int): Number of categories. + ignore_index (int): Index that will be ignored in evaluation. + label_map (dict): Mapping old labels to new labels. The parameter will + work only when label is str. Default: dict(). + reduce_zero_label (bool): Wether ignore zero label. The parameter will + work only when label is str. Default: False. + + Returns: + torch.Tensor: The intersection of prediction and ground truth + histogram on all classes. + torch.Tensor: The union of prediction and ground truth histogram on + all classes. + torch.Tensor: The prediction histogram on all classes. + torch.Tensor: The ground truth histogram on all classes. + """ + + if isinstance(pred_label, str): + pred_label = torch.from_numpy(np.load(pred_label)) + else: + pred_label = torch.from_numpy((pred_label)) + + if isinstance(label, str): + label = torch.from_numpy( + mmcv.imread(label, flag='unchanged', backend='pillow')) + else: + label = torch.from_numpy(label) + + if label_map is not None: + for old_id, new_id in label_map.items(): + label[label == old_id] = new_id + if reduce_zero_label: + label[label == 0] = 255 + label = label - 1 + label[label == 254] = 255 + + mask = (label != ignore_index) + pred_label = pred_label[mask] + label = label[mask] + + intersect = pred_label[pred_label == label] + area_intersect = torch.histc( + intersect.float(), bins=(num_classes), min=0, max=num_classes - 1) + area_pred_label = torch.histc( + pred_label.float(), bins=(num_classes), min=0, max=num_classes - 1) + area_label = torch.histc( + label.float(), bins=(num_classes), min=0, max=num_classes - 1) + area_union = area_pred_label + area_label - area_intersect + return area_intersect, area_union, area_pred_label, area_label + + +def total_intersect_and_union(results, + gt_seg_maps, + num_classes, + ignore_index, + label_map=dict(), + reduce_zero_label=False): + """Calculate Total Intersection and Union. + + Args: + results (list[ndarray] | list[str]): List of prediction segmentation + maps or list of prediction result filenames. + gt_seg_maps (list[ndarray] | list[str]): list of ground truth + segmentation maps or list of label filenames. + num_classes (int): Number of categories. + ignore_index (int): Index that will be ignored in evaluation. + label_map (dict): Mapping old labels to new labels. Default: dict(). + reduce_zero_label (bool): Wether ignore zero label. Default: False. + + Returns: + ndarray: The intersection of prediction and ground truth histogram + on all classes. + ndarray: The union of prediction and ground truth histogram on all + classes. + ndarray: The prediction histogram on all classes. + ndarray: The ground truth histogram on all classes. + """ + num_imgs = len(results) + assert len(gt_seg_maps) == num_imgs + total_area_intersect = torch.zeros((num_classes, ), dtype=torch.float64) + total_area_union = torch.zeros((num_classes, ), dtype=torch.float64) + total_area_pred_label = torch.zeros((num_classes, ), dtype=torch.float64) + total_area_label = torch.zeros((num_classes, ), dtype=torch.float64) + for i in range(num_imgs): + area_intersect, area_union, area_pred_label, area_label = \ + intersect_and_union( + results[i], gt_seg_maps[i], num_classes, ignore_index, + label_map, reduce_zero_label) + total_area_intersect += area_intersect + total_area_union += area_union + total_area_pred_label += area_pred_label + total_area_label += area_label + return total_area_intersect, total_area_union, total_area_pred_label, \ + total_area_label + + +def mean_iou(results, + gt_seg_maps, + num_classes, + ignore_index, + nan_to_num=None, + label_map=dict(), + reduce_zero_label=False): + """Calculate Mean Intersection and Union (mIoU) + + Args: + results (list[ndarray] | list[str]): List of prediction segmentation + maps or list of prediction result filenames. + gt_seg_maps (list[ndarray] | list[str]): list of ground truth + segmentation maps or list of label filenames. + num_classes (int): Number of categories. + ignore_index (int): Index that will be ignored in evaluation. + nan_to_num (int, optional): If specified, NaN values will be replaced + by the numbers defined by the user. Default: None. + label_map (dict): Mapping old labels to new labels. Default: dict(). + reduce_zero_label (bool): Wether ignore zero label. Default: False. + + Returns: + dict[str, float | ndarray]: + float: Overall accuracy on all images. + ndarray: Per category accuracy, shape (num_classes, ). + ndarray: Per category IoU, shape (num_classes, ). + """ + iou_result = eval_metrics( + results=results, + gt_seg_maps=gt_seg_maps, + num_classes=num_classes, + ignore_index=ignore_index, + metrics=['mIoU'], + nan_to_num=nan_to_num, + label_map=label_map, + reduce_zero_label=reduce_zero_label) + return iou_result + + +def mean_dice(results, + gt_seg_maps, + num_classes, + ignore_index, + nan_to_num=None, + label_map=dict(), + reduce_zero_label=False): + """Calculate Mean Dice (mDice) + + Args: + results (list[ndarray] | list[str]): List of prediction segmentation + maps or list of prediction result filenames. + gt_seg_maps (list[ndarray] | list[str]): list of ground truth + segmentation maps or list of label filenames. + num_classes (int): Number of categories. + ignore_index (int): Index that will be ignored in evaluation. + nan_to_num (int, optional): If specified, NaN values will be replaced + by the numbers defined by the user. Default: None. + label_map (dict): Mapping old labels to new labels. Default: dict(). + reduce_zero_label (bool): Wether ignore zero label. Default: False. + + Returns: + dict[str, float | ndarray]: Default metrics. + float: Overall accuracy on all images. + ndarray: Per category accuracy, shape (num_classes, ). + ndarray: Per category dice, shape (num_classes, ). + """ + + dice_result = eval_metrics( + results=results, + gt_seg_maps=gt_seg_maps, + num_classes=num_classes, + ignore_index=ignore_index, + metrics=['mDice'], + nan_to_num=nan_to_num, + label_map=label_map, + reduce_zero_label=reduce_zero_label) + return dice_result + + +def mean_fscore(results, + gt_seg_maps, + num_classes, + ignore_index, + nan_to_num=None, + label_map=dict(), + reduce_zero_label=False, + beta=1): + """Calculate Mean Intersection and Union (mIoU) + + Args: + results (list[ndarray] | list[str]): List of prediction segmentation + maps or list of prediction result filenames. + gt_seg_maps (list[ndarray] | list[str]): list of ground truth + segmentation maps or list of label filenames. + num_classes (int): Number of categories. + ignore_index (int): Index that will be ignored in evaluation. + nan_to_num (int, optional): If specified, NaN values will be replaced + by the numbers defined by the user. Default: None. + label_map (dict): Mapping old labels to new labels. Default: dict(). + reduce_zero_label (bool): Wether ignore zero label. Default: False. + beta (int): Determines the weight of recall in the combined score. + Default: False. + + + Returns: + dict[str, float | ndarray]: Default metrics. + float: Overall accuracy on all images. + ndarray: Per category recall, shape (num_classes, ). + ndarray: Per category precision, shape (num_classes, ). + ndarray: Per category f-score, shape (num_classes, ). + """ + fscore_result = eval_metrics( + results=results, + gt_seg_maps=gt_seg_maps, + num_classes=num_classes, + ignore_index=ignore_index, + metrics=['mFscore'], + nan_to_num=nan_to_num, + label_map=label_map, + reduce_zero_label=reduce_zero_label, + beta=beta) + return fscore_result + + +def eval_metrics(results, + gt_seg_maps, + num_classes, + ignore_index, + metrics=['mIoU'], + nan_to_num=None, + label_map=dict(), + reduce_zero_label=False, + beta=1): + """Calculate evaluation metrics + Args: + results (list[ndarray] | list[str]): List of prediction segmentation + maps or list of prediction result filenames. + gt_seg_maps (list[ndarray] | list[str]): list of ground truth + segmentation maps or list of label filenames. + num_classes (int): Number of categories. + ignore_index (int): Index that will be ignored in evaluation. + metrics (list[str] | str): Metrics to be evaluated, 'mIoU' and 'mDice'. + nan_to_num (int, optional): If specified, NaN values will be replaced + by the numbers defined by the user. Default: None. + label_map (dict): Mapping old labels to new labels. Default: dict(). + reduce_zero_label (bool): Wether ignore zero label. Default: False. + Returns: + float: Overall accuracy on all images. + ndarray: Per category accuracy, shape (num_classes, ). + ndarray: Per category evaluation metrics, shape (num_classes, ). + """ + if isinstance(metrics, str): + metrics = [metrics] + allowed_metrics = ['mIoU', 'mDice', 'mFscore'] + if not set(metrics).issubset(set(allowed_metrics)): + raise KeyError('metrics {} is not supported'.format(metrics)) + + total_area_intersect, total_area_union, total_area_pred_label, \ + total_area_label = total_intersect_and_union( + results, gt_seg_maps, num_classes, ignore_index, label_map, + reduce_zero_label) + all_acc = total_area_intersect.sum() / total_area_label.sum() + ret_metrics = OrderedDict({'aAcc': all_acc}) + for metric in metrics: + if metric == 'mIoU': + iou = total_area_intersect / total_area_union + acc = total_area_intersect / total_area_label + ret_metrics['IoU'] = iou + ret_metrics['Acc'] = acc + elif metric == 'mDice': + dice = 2 * total_area_intersect / ( + total_area_pred_label + total_area_label) + acc = total_area_intersect / total_area_label + ret_metrics['Dice'] = dice + ret_metrics['Acc'] = acc + elif metric == 'mFscore': + precision = total_area_intersect / total_area_pred_label + recall = total_area_intersect / total_area_label + f_value = torch.tensor( + [f_score(x[0], x[1], beta) for x in zip(precision, recall)]) + ret_metrics['Fscore'] = f_value + ret_metrics['Precision'] = precision + ret_metrics['Recall'] = recall + + ret_metrics = { + metric: value.numpy() + for metric, value in ret_metrics.items() + } + if nan_to_num is not None: + ret_metrics = OrderedDict({ + metric: np.nan_to_num(metric_value, nan=nan_to_num) + for metric, metric_value in ret_metrics.items() + }) + return ret_metrics diff --git a/annotator/uniformer/mmseg/core/seg/__init__.py b/annotator/uniformer/mmseg/core/seg/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..93bc129b685e4a3efca2cc891729981b2865900d --- /dev/null +++ b/annotator/uniformer/mmseg/core/seg/__init__.py @@ -0,0 +1,4 @@ +from .builder import build_pixel_sampler +from .sampler import BasePixelSampler, OHEMPixelSampler + +__all__ = ['build_pixel_sampler', 'BasePixelSampler', 'OHEMPixelSampler'] diff --git a/annotator/uniformer/mmseg/core/seg/builder.py b/annotator/uniformer/mmseg/core/seg/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..db61f03d4abb2072f2532ce4429c0842495e015b --- /dev/null +++ b/annotator/uniformer/mmseg/core/seg/builder.py @@ -0,0 +1,8 @@ +from annotator.uniformer.mmcv.utils import Registry, build_from_cfg + +PIXEL_SAMPLERS = Registry('pixel sampler') + + +def build_pixel_sampler(cfg, **default_args): + """Build pixel sampler for segmentation map.""" + return build_from_cfg(cfg, PIXEL_SAMPLERS, default_args) diff --git a/annotator/uniformer/mmseg/core/seg/sampler/__init__.py b/annotator/uniformer/mmseg/core/seg/sampler/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..332b242c03d1c5e80d4577df442a9a037b1816e1 --- /dev/null +++ b/annotator/uniformer/mmseg/core/seg/sampler/__init__.py @@ -0,0 +1,4 @@ +from .base_pixel_sampler import BasePixelSampler +from .ohem_pixel_sampler import OHEMPixelSampler + +__all__ = ['BasePixelSampler', 'OHEMPixelSampler'] diff --git a/annotator/uniformer/mmseg/core/seg/sampler/base_pixel_sampler.py b/annotator/uniformer/mmseg/core/seg/sampler/base_pixel_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..b75b1566c9f18169cee51d4b55d75e0357b69c57 --- /dev/null +++ b/annotator/uniformer/mmseg/core/seg/sampler/base_pixel_sampler.py @@ -0,0 +1,12 @@ +from abc import ABCMeta, abstractmethod + + +class BasePixelSampler(metaclass=ABCMeta): + """Base class of pixel sampler.""" + + def __init__(self, **kwargs): + pass + + @abstractmethod + def sample(self, seg_logit, seg_label): + """Placeholder for sample function.""" diff --git a/annotator/uniformer/mmseg/core/seg/sampler/ohem_pixel_sampler.py b/annotator/uniformer/mmseg/core/seg/sampler/ohem_pixel_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..88bb10d44026ba9f21756eaea9e550841cd59b9f --- /dev/null +++ b/annotator/uniformer/mmseg/core/seg/sampler/ohem_pixel_sampler.py @@ -0,0 +1,76 @@ +import torch +import torch.nn.functional as F + +from ..builder import PIXEL_SAMPLERS +from .base_pixel_sampler import BasePixelSampler + + +@PIXEL_SAMPLERS.register_module() +class OHEMPixelSampler(BasePixelSampler): + """Online Hard Example Mining Sampler for segmentation. + + Args: + context (nn.Module): The context of sampler, subclass of + :obj:`BaseDecodeHead`. + thresh (float, optional): The threshold for hard example selection. + Below which, are prediction with low confidence. If not + specified, the hard examples will be pixels of top ``min_kept`` + loss. Default: None. + min_kept (int, optional): The minimum number of predictions to keep. + Default: 100000. + """ + + def __init__(self, context, thresh=None, min_kept=100000): + super(OHEMPixelSampler, self).__init__() + self.context = context + assert min_kept > 1 + self.thresh = thresh + self.min_kept = min_kept + + def sample(self, seg_logit, seg_label): + """Sample pixels that have high loss or with low prediction confidence. + + Args: + seg_logit (torch.Tensor): segmentation logits, shape (N, C, H, W) + seg_label (torch.Tensor): segmentation label, shape (N, 1, H, W) + + Returns: + torch.Tensor: segmentation weight, shape (N, H, W) + """ + with torch.no_grad(): + assert seg_logit.shape[2:] == seg_label.shape[2:] + assert seg_label.shape[1] == 1 + seg_label = seg_label.squeeze(1).long() + batch_kept = self.min_kept * seg_label.size(0) + valid_mask = seg_label != self.context.ignore_index + seg_weight = seg_logit.new_zeros(size=seg_label.size()) + valid_seg_weight = seg_weight[valid_mask] + if self.thresh is not None: + seg_prob = F.softmax(seg_logit, dim=1) + + tmp_seg_label = seg_label.clone().unsqueeze(1) + tmp_seg_label[tmp_seg_label == self.context.ignore_index] = 0 + seg_prob = seg_prob.gather(1, tmp_seg_label).squeeze(1) + sort_prob, sort_indices = seg_prob[valid_mask].sort() + + if sort_prob.numel() > 0: + min_threshold = sort_prob[min(batch_kept, + sort_prob.numel() - 1)] + else: + min_threshold = 0.0 + threshold = max(min_threshold, self.thresh) + valid_seg_weight[seg_prob[valid_mask] < threshold] = 1. + else: + losses = self.context.loss_decode( + seg_logit, + seg_label, + weight=None, + ignore_index=self.context.ignore_index, + reduction_override='none') + # faster than topk according to https://github.com/pytorch/pytorch/issues/22812 # noqa + _, sort_indices = losses[valid_mask].sort(descending=True) + valid_seg_weight[sort_indices[:batch_kept]] = 1. + + seg_weight[valid_mask] = valid_seg_weight + + return seg_weight diff --git a/annotator/uniformer/mmseg/core/utils/__init__.py b/annotator/uniformer/mmseg/core/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f2678b321c295bcceaef945111ac3524be19d6e4 --- /dev/null +++ b/annotator/uniformer/mmseg/core/utils/__init__.py @@ -0,0 +1,3 @@ +from .misc import add_prefix + +__all__ = ['add_prefix'] diff --git a/annotator/uniformer/mmseg/core/utils/misc.py b/annotator/uniformer/mmseg/core/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..eb862a82bd47c8624db3dd5c6fb6ad8a03b62466 --- /dev/null +++ b/annotator/uniformer/mmseg/core/utils/misc.py @@ -0,0 +1,17 @@ +def add_prefix(inputs, prefix): + """Add prefix for dict. + + Args: + inputs (dict): The input dict with str keys. + prefix (str): The prefix to add. + + Returns: + + dict: The dict with keys updated with ``prefix``. + """ + + outputs = dict() + for name, value in inputs.items(): + outputs[f'{prefix}.{name}'] = value + + return outputs diff --git a/annotator/uniformer/mmseg/datasets/__init__.py b/annotator/uniformer/mmseg/datasets/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ebeaef4a28ef655e43578552a8aef6b77f13a636 --- /dev/null +++ b/annotator/uniformer/mmseg/datasets/__init__.py @@ -0,0 +1,19 @@ +from .ade import ADE20KDataset +from .builder import DATASETS, PIPELINES, build_dataloader, build_dataset +from .chase_db1 import ChaseDB1Dataset +from .cityscapes import CityscapesDataset +from .custom import CustomDataset +from .dataset_wrappers import ConcatDataset, RepeatDataset +from .drive import DRIVEDataset +from .hrf import HRFDataset +from .pascal_context import PascalContextDataset, PascalContextDataset59 +from .stare import STAREDataset +from .voc import PascalVOCDataset + +__all__ = [ + 'CustomDataset', 'build_dataloader', 'ConcatDataset', 'RepeatDataset', + 'DATASETS', 'build_dataset', 'PIPELINES', 'CityscapesDataset', + 'PascalVOCDataset', 'ADE20KDataset', 'PascalContextDataset', + 'PascalContextDataset59', 'ChaseDB1Dataset', 'DRIVEDataset', 'HRFDataset', + 'STAREDataset' +] diff --git a/annotator/uniformer/mmseg/datasets/ade.py b/annotator/uniformer/mmseg/datasets/ade.py new file mode 100644 index 0000000000000000000000000000000000000000..5913e43775ed4920b6934c855eb5a37c54218ebf --- /dev/null +++ b/annotator/uniformer/mmseg/datasets/ade.py @@ -0,0 +1,84 @@ +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class ADE20KDataset(CustomDataset): + """ADE20K dataset. + + In segmentation map annotation for ADE20K, 0 stands for background, which + is not included in 150 categories. ``reduce_zero_label`` is fixed to True. + The ``img_suffix`` is fixed to '.jpg' and ``seg_map_suffix`` is fixed to + '.png'. + """ + CLASSES = ( + 'wall', 'building', 'sky', 'floor', 'tree', 'ceiling', 'road', 'bed ', + 'windowpane', 'grass', 'cabinet', 'sidewalk', 'person', 'earth', + 'door', 'table', 'mountain', 'plant', 'curtain', 'chair', 'car', + 'water', 'painting', 'sofa', 'shelf', 'house', 'sea', 'mirror', 'rug', + 'field', 'armchair', 'seat', 'fence', 'desk', 'rock', 'wardrobe', + 'lamp', 'bathtub', 'railing', 'cushion', 'base', 'box', 'column', + 'signboard', 'chest of drawers', 'counter', 'sand', 'sink', + 'skyscraper', 'fireplace', 'refrigerator', 'grandstand', 'path', + 'stairs', 'runway', 'case', 'pool table', 'pillow', 'screen door', + 'stairway', 'river', 'bridge', 'bookcase', 'blind', 'coffee table', + 'toilet', 'flower', 'book', 'hill', 'bench', 'countertop', 'stove', + 'palm', 'kitchen island', 'computer', 'swivel chair', 'boat', 'bar', + 'arcade machine', 'hovel', 'bus', 'towel', 'light', 'truck', 'tower', + 'chandelier', 'awning', 'streetlight', 'booth', 'television receiver', + 'airplane', 'dirt track', 'apparel', 'pole', 'land', 'bannister', + 'escalator', 'ottoman', 'bottle', 'buffet', 'poster', 'stage', 'van', + 'ship', 'fountain', 'conveyer belt', 'canopy', 'washer', 'plaything', + 'swimming pool', 'stool', 'barrel', 'basket', 'waterfall', 'tent', + 'bag', 'minibike', 'cradle', 'oven', 'ball', 'food', 'step', 'tank', + 'trade name', 'microwave', 'pot', 'animal', 'bicycle', 'lake', + 'dishwasher', 'screen', 'blanket', 'sculpture', 'hood', 'sconce', + 'vase', 'traffic light', 'tray', 'ashcan', 'fan', 'pier', 'crt screen', + 'plate', 'monitor', 'bulletin board', 'shower', 'radiator', 'glass', + 'clock', 'flag') + + PALETTE = [[120, 120, 120], [180, 120, 120], [6, 230, 230], [80, 50, 50], + [4, 200, 3], [120, 120, 80], [140, 140, 140], [204, 5, 255], + [230, 230, 230], [4, 250, 7], [224, 5, 255], [235, 255, 7], + [150, 5, 61], [120, 120, 70], [8, 255, 51], [255, 6, 82], + [143, 255, 140], [204, 255, 4], [255, 51, 7], [204, 70, 3], + [0, 102, 200], [61, 230, 250], [255, 6, 51], [11, 102, 255], + [255, 7, 71], [255, 9, 224], [9, 7, 230], [220, 220, 220], + [255, 9, 92], [112, 9, 255], [8, 255, 214], [7, 255, 224], + [255, 184, 6], [10, 255, 71], [255, 41, 10], [7, 255, 255], + [224, 255, 8], [102, 8, 255], [255, 61, 6], [255, 194, 7], + [255, 122, 8], [0, 255, 20], [255, 8, 41], [255, 5, 153], + [6, 51, 255], [235, 12, 255], [160, 150, 20], [0, 163, 255], + [140, 140, 140], [250, 10, 15], [20, 255, 0], [31, 255, 0], + [255, 31, 0], [255, 224, 0], [153, 255, 0], [0, 0, 255], + [255, 71, 0], [0, 235, 255], [0, 173, 255], [31, 0, 255], + [11, 200, 200], [255, 82, 0], [0, 255, 245], [0, 61, 255], + [0, 255, 112], [0, 255, 133], [255, 0, 0], [255, 163, 0], + [255, 102, 0], [194, 255, 0], [0, 143, 255], [51, 255, 0], + [0, 82, 255], [0, 255, 41], [0, 255, 173], [10, 0, 255], + [173, 255, 0], [0, 255, 153], [255, 92, 0], [255, 0, 255], + [255, 0, 245], [255, 0, 102], [255, 173, 0], [255, 0, 20], + [255, 184, 184], [0, 31, 255], [0, 255, 61], [0, 71, 255], + [255, 0, 204], [0, 255, 194], [0, 255, 82], [0, 10, 255], + [0, 112, 255], [51, 0, 255], [0, 194, 255], [0, 122, 255], + [0, 255, 163], [255, 153, 0], [0, 255, 10], [255, 112, 0], + [143, 255, 0], [82, 0, 255], [163, 255, 0], [255, 235, 0], + [8, 184, 170], [133, 0, 255], [0, 255, 92], [184, 0, 255], + [255, 0, 31], [0, 184, 255], [0, 214, 255], [255, 0, 112], + [92, 255, 0], [0, 224, 255], [112, 224, 255], [70, 184, 160], + [163, 0, 255], [153, 0, 255], [71, 255, 0], [255, 0, 163], + [255, 204, 0], [255, 0, 143], [0, 255, 235], [133, 255, 0], + [255, 0, 235], [245, 0, 255], [255, 0, 122], [255, 245, 0], + [10, 190, 212], [214, 255, 0], [0, 204, 255], [20, 0, 255], + [255, 255, 0], [0, 153, 255], [0, 41, 255], [0, 255, 204], + [41, 0, 255], [41, 255, 0], [173, 0, 255], [0, 245, 255], + [71, 0, 255], [122, 0, 255], [0, 255, 184], [0, 92, 255], + [184, 255, 0], [0, 133, 255], [255, 214, 0], [25, 194, 194], + [102, 255, 0], [92, 0, 255]] + + def __init__(self, **kwargs): + super(ADE20KDataset, self).__init__( + img_suffix='.jpg', + seg_map_suffix='.png', + reduce_zero_label=True, + **kwargs) diff --git a/annotator/uniformer/mmseg/datasets/builder.py b/annotator/uniformer/mmseg/datasets/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..0798b14cd8b39fc58d8f2a4930f1e079b5bf8b55 --- /dev/null +++ b/annotator/uniformer/mmseg/datasets/builder.py @@ -0,0 +1,169 @@ +import copy +import platform +import random +from functools import partial + +import numpy as np +from annotator.uniformer.mmcv.parallel import collate +from annotator.uniformer.mmcv.runner import get_dist_info +from annotator.uniformer.mmcv.utils import Registry, build_from_cfg +from annotator.uniformer.mmcv.utils.parrots_wrapper import DataLoader, PoolDataLoader +from torch.utils.data import DistributedSampler + +if platform.system() != 'Windows': + # https://github.com/pytorch/pytorch/issues/973 + import resource + rlimit = resource.getrlimit(resource.RLIMIT_NOFILE) + hard_limit = rlimit[1] + soft_limit = min(4096, hard_limit) + resource.setrlimit(resource.RLIMIT_NOFILE, (soft_limit, hard_limit)) + +DATASETS = Registry('dataset') +PIPELINES = Registry('pipeline') + + +def _concat_dataset(cfg, default_args=None): + """Build :obj:`ConcatDataset by.""" + from .dataset_wrappers import ConcatDataset + img_dir = cfg['img_dir'] + ann_dir = cfg.get('ann_dir', None) + split = cfg.get('split', None) + num_img_dir = len(img_dir) if isinstance(img_dir, (list, tuple)) else 1 + if ann_dir is not None: + num_ann_dir = len(ann_dir) if isinstance(ann_dir, (list, tuple)) else 1 + else: + num_ann_dir = 0 + if split is not None: + num_split = len(split) if isinstance(split, (list, tuple)) else 1 + else: + num_split = 0 + if num_img_dir > 1: + assert num_img_dir == num_ann_dir or num_ann_dir == 0 + assert num_img_dir == num_split or num_split == 0 + else: + assert num_split == num_ann_dir or num_ann_dir <= 1 + num_dset = max(num_split, num_img_dir) + + datasets = [] + for i in range(num_dset): + data_cfg = copy.deepcopy(cfg) + if isinstance(img_dir, (list, tuple)): + data_cfg['img_dir'] = img_dir[i] + if isinstance(ann_dir, (list, tuple)): + data_cfg['ann_dir'] = ann_dir[i] + if isinstance(split, (list, tuple)): + data_cfg['split'] = split[i] + datasets.append(build_dataset(data_cfg, default_args)) + + return ConcatDataset(datasets) + + +def build_dataset(cfg, default_args=None): + """Build datasets.""" + from .dataset_wrappers import ConcatDataset, RepeatDataset + if isinstance(cfg, (list, tuple)): + dataset = ConcatDataset([build_dataset(c, default_args) for c in cfg]) + elif cfg['type'] == 'RepeatDataset': + dataset = RepeatDataset( + build_dataset(cfg['dataset'], default_args), cfg['times']) + elif isinstance(cfg.get('img_dir'), (list, tuple)) or isinstance( + cfg.get('split', None), (list, tuple)): + dataset = _concat_dataset(cfg, default_args) + else: + dataset = build_from_cfg(cfg, DATASETS, default_args) + + return dataset + + +def build_dataloader(dataset, + samples_per_gpu, + workers_per_gpu, + num_gpus=1, + dist=True, + shuffle=True, + seed=None, + drop_last=False, + pin_memory=True, + dataloader_type='PoolDataLoader', + **kwargs): + """Build PyTorch DataLoader. + + In distributed training, each GPU/process has a dataloader. + In non-distributed training, there is only one dataloader for all GPUs. + + Args: + dataset (Dataset): A PyTorch dataset. + samples_per_gpu (int): Number of training samples on each GPU, i.e., + batch size of each GPU. + workers_per_gpu (int): How many subprocesses to use for data loading + for each GPU. + num_gpus (int): Number of GPUs. Only used in non-distributed training. + dist (bool): Distributed training/test or not. Default: True. + shuffle (bool): Whether to shuffle the data at every epoch. + Default: True. + seed (int | None): Seed to be used. Default: None. + drop_last (bool): Whether to drop the last incomplete batch in epoch. + Default: False + pin_memory (bool): Whether to use pin_memory in DataLoader. + Default: True + dataloader_type (str): Type of dataloader. Default: 'PoolDataLoader' + kwargs: any keyword argument to be used to initialize DataLoader + + Returns: + DataLoader: A PyTorch dataloader. + """ + rank, world_size = get_dist_info() + if dist: + sampler = DistributedSampler( + dataset, world_size, rank, shuffle=shuffle) + shuffle = False + batch_size = samples_per_gpu + num_workers = workers_per_gpu + else: + sampler = None + batch_size = num_gpus * samples_per_gpu + num_workers = num_gpus * workers_per_gpu + + init_fn = partial( + worker_init_fn, num_workers=num_workers, rank=rank, + seed=seed) if seed is not None else None + + assert dataloader_type in ( + 'DataLoader', + 'PoolDataLoader'), f'unsupported dataloader {dataloader_type}' + + if dataloader_type == 'PoolDataLoader': + dataloader = PoolDataLoader + elif dataloader_type == 'DataLoader': + dataloader = DataLoader + + data_loader = dataloader( + dataset, + batch_size=batch_size, + sampler=sampler, + num_workers=num_workers, + collate_fn=partial(collate, samples_per_gpu=samples_per_gpu), + pin_memory=pin_memory, + shuffle=shuffle, + worker_init_fn=init_fn, + drop_last=drop_last, + **kwargs) + + return data_loader + + +def worker_init_fn(worker_id, num_workers, rank, seed): + """Worker init func for dataloader. + + The seed of each worker equals to num_worker * rank + worker_id + user_seed + + Args: + worker_id (int): Worker id. + num_workers (int): Number of workers. + rank (int): The rank of current process. + seed (int): The random seed to use. + """ + + worker_seed = num_workers * rank + worker_id + seed + np.random.seed(worker_seed) + random.seed(worker_seed) diff --git a/annotator/uniformer/mmseg/datasets/chase_db1.py b/annotator/uniformer/mmseg/datasets/chase_db1.py new file mode 100644 index 0000000000000000000000000000000000000000..8bc29bea14704a4407f83474610cbc3bef32c708 --- /dev/null +++ b/annotator/uniformer/mmseg/datasets/chase_db1.py @@ -0,0 +1,27 @@ +import os.path as osp + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class ChaseDB1Dataset(CustomDataset): + """Chase_db1 dataset. + + In segmentation map annotation for Chase_db1, 0 stands for background, + which is included in 2 categories. ``reduce_zero_label`` is fixed to False. + The ``img_suffix`` is fixed to '.png' and ``seg_map_suffix`` is fixed to + '_1stHO.png'. + """ + + CLASSES = ('background', 'vessel') + + PALETTE = [[120, 120, 120], [6, 230, 230]] + + def __init__(self, **kwargs): + super(ChaseDB1Dataset, self).__init__( + img_suffix='.png', + seg_map_suffix='_1stHO.png', + reduce_zero_label=False, + **kwargs) + assert osp.exists(self.img_dir) diff --git a/annotator/uniformer/mmseg/datasets/cityscapes.py b/annotator/uniformer/mmseg/datasets/cityscapes.py new file mode 100644 index 0000000000000000000000000000000000000000..81e47a914a1aa2e5458e18669d65ffb742f46fc6 --- /dev/null +++ b/annotator/uniformer/mmseg/datasets/cityscapes.py @@ -0,0 +1,217 @@ +import os.path as osp +import tempfile + +import annotator.uniformer.mmcv as mmcv +import numpy as np +from annotator.uniformer.mmcv.utils import print_log +from PIL import Image + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class CityscapesDataset(CustomDataset): + """Cityscapes dataset. + + The ``img_suffix`` is fixed to '_leftImg8bit.png' and ``seg_map_suffix`` is + fixed to '_gtFine_labelTrainIds.png' for Cityscapes dataset. + """ + + CLASSES = ('road', 'sidewalk', 'building', 'wall', 'fence', 'pole', + 'traffic light', 'traffic sign', 'vegetation', 'terrain', 'sky', + 'person', 'rider', 'car', 'truck', 'bus', 'train', 'motorcycle', + 'bicycle') + + PALETTE = [[128, 64, 128], [244, 35, 232], [70, 70, 70], [102, 102, 156], + [190, 153, 153], [153, 153, 153], [250, 170, 30], [220, 220, 0], + [107, 142, 35], [152, 251, 152], [70, 130, 180], [220, 20, 60], + [255, 0, 0], [0, 0, 142], [0, 0, 70], [0, 60, 100], + [0, 80, 100], [0, 0, 230], [119, 11, 32]] + + def __init__(self, **kwargs): + super(CityscapesDataset, self).__init__( + img_suffix='_leftImg8bit.png', + seg_map_suffix='_gtFine_labelTrainIds.png', + **kwargs) + + @staticmethod + def _convert_to_label_id(result): + """Convert trainId to id for cityscapes.""" + if isinstance(result, str): + result = np.load(result) + import cityscapesscripts.helpers.labels as CSLabels + result_copy = result.copy() + for trainId, label in CSLabels.trainId2label.items(): + result_copy[result == trainId] = label.id + + return result_copy + + def results2img(self, results, imgfile_prefix, to_label_id): + """Write the segmentation results to images. + + Args: + results (list[list | tuple | ndarray]): Testing results of the + dataset. + imgfile_prefix (str): The filename prefix of the png files. + If the prefix is "somepath/xxx", + the png files will be named "somepath/xxx.png". + to_label_id (bool): whether convert output to label_id for + submission + + Returns: + list[str: str]: result txt files which contains corresponding + semantic segmentation images. + """ + mmcv.mkdir_or_exist(imgfile_prefix) + result_files = [] + prog_bar = mmcv.ProgressBar(len(self)) + for idx in range(len(self)): + result = results[idx] + if to_label_id: + result = self._convert_to_label_id(result) + filename = self.img_infos[idx]['filename'] + basename = osp.splitext(osp.basename(filename))[0] + + png_filename = osp.join(imgfile_prefix, f'{basename}.png') + + output = Image.fromarray(result.astype(np.uint8)).convert('P') + import cityscapesscripts.helpers.labels as CSLabels + palette = np.zeros((len(CSLabels.id2label), 3), dtype=np.uint8) + for label_id, label in CSLabels.id2label.items(): + palette[label_id] = label.color + + output.putpalette(palette) + output.save(png_filename) + result_files.append(png_filename) + prog_bar.update() + + return result_files + + def format_results(self, results, imgfile_prefix=None, to_label_id=True): + """Format the results into dir (standard format for Cityscapes + evaluation). + + Args: + results (list): Testing results of the dataset. + imgfile_prefix (str | None): The prefix of images files. It + includes the file path and the prefix of filename, e.g., + "a/b/prefix". If not specified, a temp file will be created. + Default: None. + to_label_id (bool): whether convert output to label_id for + submission. Default: False + + Returns: + tuple: (result_files, tmp_dir), result_files is a list containing + the image paths, tmp_dir is the temporal directory created + for saving json/png files when img_prefix is not specified. + """ + + assert isinstance(results, list), 'results must be a list' + assert len(results) == len(self), ( + 'The length of results is not equal to the dataset len: ' + f'{len(results)} != {len(self)}') + + if imgfile_prefix is None: + tmp_dir = tempfile.TemporaryDirectory() + imgfile_prefix = tmp_dir.name + else: + tmp_dir = None + result_files = self.results2img(results, imgfile_prefix, to_label_id) + + return result_files, tmp_dir + + def evaluate(self, + results, + metric='mIoU', + logger=None, + imgfile_prefix=None, + efficient_test=False): + """Evaluation in Cityscapes/default protocol. + + Args: + results (list): Testing results of the dataset. + metric (str | list[str]): Metrics to be evaluated. + logger (logging.Logger | None | str): Logger used for printing + related information during evaluation. Default: None. + imgfile_prefix (str | None): The prefix of output image file, + for cityscapes evaluation only. It includes the file path and + the prefix of filename, e.g., "a/b/prefix". + If results are evaluated with cityscapes protocol, it would be + the prefix of output png files. The output files would be + png images under folder "a/b/prefix/xxx.png", where "xxx" is + the image name of cityscapes. If not specified, a temp file + will be created for evaluation. + Default: None. + + Returns: + dict[str, float]: Cityscapes/default metrics. + """ + + eval_results = dict() + metrics = metric.copy() if isinstance(metric, list) else [metric] + if 'cityscapes' in metrics: + eval_results.update( + self._evaluate_cityscapes(results, logger, imgfile_prefix)) + metrics.remove('cityscapes') + if len(metrics) > 0: + eval_results.update( + super(CityscapesDataset, + self).evaluate(results, metrics, logger, efficient_test)) + + return eval_results + + def _evaluate_cityscapes(self, results, logger, imgfile_prefix): + """Evaluation in Cityscapes protocol. + + Args: + results (list): Testing results of the dataset. + logger (logging.Logger | str | None): Logger used for printing + related information during evaluation. Default: None. + imgfile_prefix (str | None): The prefix of output image file + + Returns: + dict[str: float]: Cityscapes evaluation results. + """ + try: + import cityscapesscripts.evaluation.evalPixelLevelSemanticLabeling as CSEval # noqa + except ImportError: + raise ImportError('Please run "pip install cityscapesscripts" to ' + 'install cityscapesscripts first.') + msg = 'Evaluating in Cityscapes style' + if logger is None: + msg = '\n' + msg + print_log(msg, logger=logger) + + result_files, tmp_dir = self.format_results(results, imgfile_prefix) + + if tmp_dir is None: + result_dir = imgfile_prefix + else: + result_dir = tmp_dir.name + + eval_results = dict() + print_log(f'Evaluating results under {result_dir} ...', logger=logger) + + CSEval.args.evalInstLevelScore = True + CSEval.args.predictionPath = osp.abspath(result_dir) + CSEval.args.evalPixelAccuracy = True + CSEval.args.JSONOutput = False + + seg_map_list = [] + pred_list = [] + + # when evaluating with official cityscapesscripts, + # **_gtFine_labelIds.png is used + for seg_map in mmcv.scandir( + self.ann_dir, 'gtFine_labelIds.png', recursive=True): + seg_map_list.append(osp.join(self.ann_dir, seg_map)) + pred_list.append(CSEval.getPrediction(CSEval.args, seg_map)) + + eval_results.update( + CSEval.evaluateImgLists(pred_list, seg_map_list, CSEval.args)) + + if tmp_dir is not None: + tmp_dir.cleanup() + + return eval_results diff --git a/annotator/uniformer/mmseg/datasets/custom.py b/annotator/uniformer/mmseg/datasets/custom.py new file mode 100644 index 0000000000000000000000000000000000000000..d8eb2a709cc7a3a68fc6a1e3a1ad98faef4c5b7b --- /dev/null +++ b/annotator/uniformer/mmseg/datasets/custom.py @@ -0,0 +1,400 @@ +import os +import os.path as osp +from collections import OrderedDict +from functools import reduce + +import annotator.uniformer.mmcv as mmcv +import numpy as np +from annotator.uniformer.mmcv.utils import print_log +from prettytable import PrettyTable +from torch.utils.data import Dataset + +from annotator.uniformer.mmseg.core import eval_metrics +from annotator.uniformer.mmseg.utils import get_root_logger +from .builder import DATASETS +from .pipelines import Compose + + +@DATASETS.register_module() +class CustomDataset(Dataset): + """Custom dataset for semantic segmentation. An example of file structure + is as followed. + + .. code-block:: none + + ├── data + │ ├── my_dataset + │ │ ├── img_dir + │ │ │ ├── train + │ │ │ │ ├── xxx{img_suffix} + │ │ │ │ ├── yyy{img_suffix} + │ │ │ │ ├── zzz{img_suffix} + │ │ │ ├── val + │ │ ├── ann_dir + │ │ │ ├── train + │ │ │ │ ├── xxx{seg_map_suffix} + │ │ │ │ ├── yyy{seg_map_suffix} + │ │ │ │ ├── zzz{seg_map_suffix} + │ │ │ ├── val + + The img/gt_semantic_seg pair of CustomDataset should be of the same + except suffix. A valid img/gt_semantic_seg filename pair should be like + ``xxx{img_suffix}`` and ``xxx{seg_map_suffix}`` (extension is also included + in the suffix). If split is given, then ``xxx`` is specified in txt file. + Otherwise, all files in ``img_dir/``and ``ann_dir`` will be loaded. + Please refer to ``docs/tutorials/new_dataset.md`` for more details. + + + Args: + pipeline (list[dict]): Processing pipeline + img_dir (str): Path to image directory + img_suffix (str): Suffix of images. Default: '.jpg' + ann_dir (str, optional): Path to annotation directory. Default: None + seg_map_suffix (str): Suffix of segmentation maps. Default: '.png' + split (str, optional): Split txt file. If split is specified, only + file with suffix in the splits will be loaded. Otherwise, all + images in img_dir/ann_dir will be loaded. Default: None + data_root (str, optional): Data root for img_dir/ann_dir. Default: + None. + test_mode (bool): If test_mode=True, gt wouldn't be loaded. + ignore_index (int): The label index to be ignored. Default: 255 + reduce_zero_label (bool): Whether to mark label zero as ignored. + Default: False + classes (str | Sequence[str], optional): Specify classes to load. + If is None, ``cls.CLASSES`` will be used. Default: None. + palette (Sequence[Sequence[int]]] | np.ndarray | None): + The palette of segmentation map. If None is given, and + self.PALETTE is None, random palette will be generated. + Default: None + """ + + CLASSES = None + + PALETTE = None + + def __init__(self, + pipeline, + img_dir, + img_suffix='.jpg', + ann_dir=None, + seg_map_suffix='.png', + split=None, + data_root=None, + test_mode=False, + ignore_index=255, + reduce_zero_label=False, + classes=None, + palette=None): + self.pipeline = Compose(pipeline) + self.img_dir = img_dir + self.img_suffix = img_suffix + self.ann_dir = ann_dir + self.seg_map_suffix = seg_map_suffix + self.split = split + self.data_root = data_root + self.test_mode = test_mode + self.ignore_index = ignore_index + self.reduce_zero_label = reduce_zero_label + self.label_map = None + self.CLASSES, self.PALETTE = self.get_classes_and_palette( + classes, palette) + + # join paths if data_root is specified + if self.data_root is not None: + if not osp.isabs(self.img_dir): + self.img_dir = osp.join(self.data_root, self.img_dir) + if not (self.ann_dir is None or osp.isabs(self.ann_dir)): + self.ann_dir = osp.join(self.data_root, self.ann_dir) + if not (self.split is None or osp.isabs(self.split)): + self.split = osp.join(self.data_root, self.split) + + # load annotations + self.img_infos = self.load_annotations(self.img_dir, self.img_suffix, + self.ann_dir, + self.seg_map_suffix, self.split) + + def __len__(self): + """Total number of samples of data.""" + return len(self.img_infos) + + def load_annotations(self, img_dir, img_suffix, ann_dir, seg_map_suffix, + split): + """Load annotation from directory. + + Args: + img_dir (str): Path to image directory + img_suffix (str): Suffix of images. + ann_dir (str|None): Path to annotation directory. + seg_map_suffix (str|None): Suffix of segmentation maps. + split (str|None): Split txt file. If split is specified, only file + with suffix in the splits will be loaded. Otherwise, all images + in img_dir/ann_dir will be loaded. Default: None + + Returns: + list[dict]: All image info of dataset. + """ + + img_infos = [] + if split is not None: + with open(split) as f: + for line in f: + img_name = line.strip() + img_info = dict(filename=img_name + img_suffix) + if ann_dir is not None: + seg_map = img_name + seg_map_suffix + img_info['ann'] = dict(seg_map=seg_map) + img_infos.append(img_info) + else: + for img in mmcv.scandir(img_dir, img_suffix, recursive=True): + img_info = dict(filename=img) + if ann_dir is not None: + seg_map = img.replace(img_suffix, seg_map_suffix) + img_info['ann'] = dict(seg_map=seg_map) + img_infos.append(img_info) + + print_log(f'Loaded {len(img_infos)} images', logger=get_root_logger()) + return img_infos + + def get_ann_info(self, idx): + """Get annotation by index. + + Args: + idx (int): Index of data. + + Returns: + dict: Annotation info of specified index. + """ + + return self.img_infos[idx]['ann'] + + def pre_pipeline(self, results): + """Prepare results dict for pipeline.""" + results['seg_fields'] = [] + results['img_prefix'] = self.img_dir + results['seg_prefix'] = self.ann_dir + if self.custom_classes: + results['label_map'] = self.label_map + + def __getitem__(self, idx): + """Get training/test data after pipeline. + + Args: + idx (int): Index of data. + + Returns: + dict: Training/test data (with annotation if `test_mode` is set + False). + """ + + if self.test_mode: + return self.prepare_test_img(idx) + else: + return self.prepare_train_img(idx) + + def prepare_train_img(self, idx): + """Get training data and annotations after pipeline. + + Args: + idx (int): Index of data. + + Returns: + dict: Training data and annotation after pipeline with new keys + introduced by pipeline. + """ + + img_info = self.img_infos[idx] + ann_info = self.get_ann_info(idx) + results = dict(img_info=img_info, ann_info=ann_info) + self.pre_pipeline(results) + return self.pipeline(results) + + def prepare_test_img(self, idx): + """Get testing data after pipeline. + + Args: + idx (int): Index of data. + + Returns: + dict: Testing data after pipeline with new keys introduced by + pipeline. + """ + + img_info = self.img_infos[idx] + results = dict(img_info=img_info) + self.pre_pipeline(results) + return self.pipeline(results) + + def format_results(self, results, **kwargs): + """Place holder to format result to dataset specific output.""" + + def get_gt_seg_maps(self, efficient_test=False): + """Get ground truth segmentation maps for evaluation.""" + gt_seg_maps = [] + for img_info in self.img_infos: + seg_map = osp.join(self.ann_dir, img_info['ann']['seg_map']) + if efficient_test: + gt_seg_map = seg_map + else: + gt_seg_map = mmcv.imread( + seg_map, flag='unchanged', backend='pillow') + gt_seg_maps.append(gt_seg_map) + return gt_seg_maps + + def get_classes_and_palette(self, classes=None, palette=None): + """Get class names of current dataset. + + Args: + classes (Sequence[str] | str | None): If classes is None, use + default CLASSES defined by builtin dataset. If classes is a + string, take it as a file name. The file contains the name of + classes where each line contains one class name. If classes is + a tuple or list, override the CLASSES defined by the dataset. + palette (Sequence[Sequence[int]]] | np.ndarray | None): + The palette of segmentation map. If None is given, random + palette will be generated. Default: None + """ + if classes is None: + self.custom_classes = False + return self.CLASSES, self.PALETTE + + self.custom_classes = True + if isinstance(classes, str): + # take it as a file path + class_names = mmcv.list_from_file(classes) + elif isinstance(classes, (tuple, list)): + class_names = classes + else: + raise ValueError(f'Unsupported type {type(classes)} of classes.') + + if self.CLASSES: + if not set(classes).issubset(self.CLASSES): + raise ValueError('classes is not a subset of CLASSES.') + + # dictionary, its keys are the old label ids and its values + # are the new label ids. + # used for changing pixel labels in load_annotations. + self.label_map = {} + for i, c in enumerate(self.CLASSES): + if c not in class_names: + self.label_map[i] = -1 + else: + self.label_map[i] = classes.index(c) + + palette = self.get_palette_for_custom_classes(class_names, palette) + + return class_names, palette + + def get_palette_for_custom_classes(self, class_names, palette=None): + + if self.label_map is not None: + # return subset of palette + palette = [] + for old_id, new_id in sorted( + self.label_map.items(), key=lambda x: x[1]): + if new_id != -1: + palette.append(self.PALETTE[old_id]) + palette = type(self.PALETTE)(palette) + + elif palette is None: + if self.PALETTE is None: + palette = np.random.randint(0, 255, size=(len(class_names), 3)) + else: + palette = self.PALETTE + + return palette + + def evaluate(self, + results, + metric='mIoU', + logger=None, + efficient_test=False, + **kwargs): + """Evaluate the dataset. + + Args: + results (list): Testing results of the dataset. + metric (str | list[str]): Metrics to be evaluated. 'mIoU', + 'mDice' and 'mFscore' are supported. + logger (logging.Logger | None | str): Logger used for printing + related information during evaluation. Default: None. + + Returns: + dict[str, float]: Default metrics. + """ + + if isinstance(metric, str): + metric = [metric] + allowed_metrics = ['mIoU', 'mDice', 'mFscore'] + if not set(metric).issubset(set(allowed_metrics)): + raise KeyError('metric {} is not supported'.format(metric)) + eval_results = {} + gt_seg_maps = self.get_gt_seg_maps(efficient_test) + if self.CLASSES is None: + num_classes = len( + reduce(np.union1d, [np.unique(_) for _ in gt_seg_maps])) + else: + num_classes = len(self.CLASSES) + ret_metrics = eval_metrics( + results, + gt_seg_maps, + num_classes, + self.ignore_index, + metric, + label_map=self.label_map, + reduce_zero_label=self.reduce_zero_label) + + if self.CLASSES is None: + class_names = tuple(range(num_classes)) + else: + class_names = self.CLASSES + + # summary table + ret_metrics_summary = OrderedDict({ + ret_metric: np.round(np.nanmean(ret_metric_value) * 100, 2) + for ret_metric, ret_metric_value in ret_metrics.items() + }) + + # each class table + ret_metrics.pop('aAcc', None) + ret_metrics_class = OrderedDict({ + ret_metric: np.round(ret_metric_value * 100, 2) + for ret_metric, ret_metric_value in ret_metrics.items() + }) + ret_metrics_class.update({'Class': class_names}) + ret_metrics_class.move_to_end('Class', last=False) + + # for logger + class_table_data = PrettyTable() + for key, val in ret_metrics_class.items(): + class_table_data.add_column(key, val) + + summary_table_data = PrettyTable() + for key, val in ret_metrics_summary.items(): + if key == 'aAcc': + summary_table_data.add_column(key, [val]) + else: + summary_table_data.add_column('m' + key, [val]) + + print_log('per class results:', logger) + print_log('\n' + class_table_data.get_string(), logger=logger) + print_log('Summary:', logger) + print_log('\n' + summary_table_data.get_string(), logger=logger) + + # each metric dict + for key, value in ret_metrics_summary.items(): + if key == 'aAcc': + eval_results[key] = value / 100.0 + else: + eval_results['m' + key] = value / 100.0 + + ret_metrics_class.pop('Class', None) + for key, value in ret_metrics_class.items(): + eval_results.update({ + key + '.' + str(name): value[idx] / 100.0 + for idx, name in enumerate(class_names) + }) + + if mmcv.is_list_of(results, str): + for file_name in results: + os.remove(file_name) + return eval_results diff --git a/annotator/uniformer/mmseg/datasets/dataset_wrappers.py b/annotator/uniformer/mmseg/datasets/dataset_wrappers.py new file mode 100644 index 0000000000000000000000000000000000000000..d6a5e957ec3b44465432617cf6e8f0b86a8a5efa --- /dev/null +++ b/annotator/uniformer/mmseg/datasets/dataset_wrappers.py @@ -0,0 +1,50 @@ +from torch.utils.data.dataset import ConcatDataset as _ConcatDataset + +from .builder import DATASETS + + +@DATASETS.register_module() +class ConcatDataset(_ConcatDataset): + """A wrapper of concatenated dataset. + + Same as :obj:`torch.utils.data.dataset.ConcatDataset`, but + concat the group flag for image aspect ratio. + + Args: + datasets (list[:obj:`Dataset`]): A list of datasets. + """ + + def __init__(self, datasets): + super(ConcatDataset, self).__init__(datasets) + self.CLASSES = datasets[0].CLASSES + self.PALETTE = datasets[0].PALETTE + + +@DATASETS.register_module() +class RepeatDataset(object): + """A wrapper of repeated dataset. + + The length of repeated dataset will be `times` larger than the original + dataset. This is useful when the data loading time is long but the dataset + is small. Using RepeatDataset can reduce the data loading time between + epochs. + + Args: + dataset (:obj:`Dataset`): The dataset to be repeated. + times (int): Repeat times. + """ + + def __init__(self, dataset, times): + self.dataset = dataset + self.times = times + self.CLASSES = dataset.CLASSES + self.PALETTE = dataset.PALETTE + self._ori_len = len(self.dataset) + + def __getitem__(self, idx): + """Get item from original dataset.""" + return self.dataset[idx % self._ori_len] + + def __len__(self): + """The length is multiplied by ``times``""" + return self.times * self._ori_len diff --git a/annotator/uniformer/mmseg/datasets/drive.py b/annotator/uniformer/mmseg/datasets/drive.py new file mode 100644 index 0000000000000000000000000000000000000000..3cbfda8ae74bdf26c5aef197ff2866a7c7ad0cfd --- /dev/null +++ b/annotator/uniformer/mmseg/datasets/drive.py @@ -0,0 +1,27 @@ +import os.path as osp + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class DRIVEDataset(CustomDataset): + """DRIVE dataset. + + In segmentation map annotation for DRIVE, 0 stands for background, which is + included in 2 categories. ``reduce_zero_label`` is fixed to False. The + ``img_suffix`` is fixed to '.png' and ``seg_map_suffix`` is fixed to + '_manual1.png'. + """ + + CLASSES = ('background', 'vessel') + + PALETTE = [[120, 120, 120], [6, 230, 230]] + + def __init__(self, **kwargs): + super(DRIVEDataset, self).__init__( + img_suffix='.png', + seg_map_suffix='_manual1.png', + reduce_zero_label=False, + **kwargs) + assert osp.exists(self.img_dir) diff --git a/annotator/uniformer/mmseg/datasets/hrf.py b/annotator/uniformer/mmseg/datasets/hrf.py new file mode 100644 index 0000000000000000000000000000000000000000..923203b51377f9344277fc561803d7a78bd2c684 --- /dev/null +++ b/annotator/uniformer/mmseg/datasets/hrf.py @@ -0,0 +1,27 @@ +import os.path as osp + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class HRFDataset(CustomDataset): + """HRF dataset. + + In segmentation map annotation for HRF, 0 stands for background, which is + included in 2 categories. ``reduce_zero_label`` is fixed to False. The + ``img_suffix`` is fixed to '.png' and ``seg_map_suffix`` is fixed to + '.png'. + """ + + CLASSES = ('background', 'vessel') + + PALETTE = [[120, 120, 120], [6, 230, 230]] + + def __init__(self, **kwargs): + super(HRFDataset, self).__init__( + img_suffix='.png', + seg_map_suffix='.png', + reduce_zero_label=False, + **kwargs) + assert osp.exists(self.img_dir) diff --git a/annotator/uniformer/mmseg/datasets/pascal_context.py b/annotator/uniformer/mmseg/datasets/pascal_context.py new file mode 100644 index 0000000000000000000000000000000000000000..541a63c66a13fb16fd52921e755715ad8d078fdd --- /dev/null +++ b/annotator/uniformer/mmseg/datasets/pascal_context.py @@ -0,0 +1,103 @@ +import os.path as osp + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class PascalContextDataset(CustomDataset): + """PascalContext dataset. + + In segmentation map annotation for PascalContext, 0 stands for background, + which is included in 60 categories. ``reduce_zero_label`` is fixed to + False. The ``img_suffix`` is fixed to '.jpg' and ``seg_map_suffix`` is + fixed to '.png'. + + Args: + split (str): Split txt file for PascalContext. + """ + + CLASSES = ('background', 'aeroplane', 'bag', 'bed', 'bedclothes', 'bench', + 'bicycle', 'bird', 'boat', 'book', 'bottle', 'building', 'bus', + 'cabinet', 'car', 'cat', 'ceiling', 'chair', 'cloth', + 'computer', 'cow', 'cup', 'curtain', 'dog', 'door', 'fence', + 'floor', 'flower', 'food', 'grass', 'ground', 'horse', + 'keyboard', 'light', 'motorbike', 'mountain', 'mouse', 'person', + 'plate', 'platform', 'pottedplant', 'road', 'rock', 'sheep', + 'shelves', 'sidewalk', 'sign', 'sky', 'snow', 'sofa', 'table', + 'track', 'train', 'tree', 'truck', 'tvmonitor', 'wall', 'water', + 'window', 'wood') + + PALETTE = [[120, 120, 120], [180, 120, 120], [6, 230, 230], [80, 50, 50], + [4, 200, 3], [120, 120, 80], [140, 140, 140], [204, 5, 255], + [230, 230, 230], [4, 250, 7], [224, 5, 255], [235, 255, 7], + [150, 5, 61], [120, 120, 70], [8, 255, 51], [255, 6, 82], + [143, 255, 140], [204, 255, 4], [255, 51, 7], [204, 70, 3], + [0, 102, 200], [61, 230, 250], [255, 6, 51], [11, 102, 255], + [255, 7, 71], [255, 9, 224], [9, 7, 230], [220, 220, 220], + [255, 9, 92], [112, 9, 255], [8, 255, 214], [7, 255, 224], + [255, 184, 6], [10, 255, 71], [255, 41, 10], [7, 255, 255], + [224, 255, 8], [102, 8, 255], [255, 61, 6], [255, 194, 7], + [255, 122, 8], [0, 255, 20], [255, 8, 41], [255, 5, 153], + [6, 51, 255], [235, 12, 255], [160, 150, 20], [0, 163, 255], + [140, 140, 140], [250, 10, 15], [20, 255, 0], [31, 255, 0], + [255, 31, 0], [255, 224, 0], [153, 255, 0], [0, 0, 255], + [255, 71, 0], [0, 235, 255], [0, 173, 255], [31, 0, 255]] + + def __init__(self, split, **kwargs): + super(PascalContextDataset, self).__init__( + img_suffix='.jpg', + seg_map_suffix='.png', + split=split, + reduce_zero_label=False, + **kwargs) + assert osp.exists(self.img_dir) and self.split is not None + + +@DATASETS.register_module() +class PascalContextDataset59(CustomDataset): + """PascalContext dataset. + + In segmentation map annotation for PascalContext, 0 stands for background, + which is included in 60 categories. ``reduce_zero_label`` is fixed to + False. The ``img_suffix`` is fixed to '.jpg' and ``seg_map_suffix`` is + fixed to '.png'. + + Args: + split (str): Split txt file for PascalContext. + """ + + CLASSES = ('aeroplane', 'bag', 'bed', 'bedclothes', 'bench', 'bicycle', + 'bird', 'boat', 'book', 'bottle', 'building', 'bus', 'cabinet', + 'car', 'cat', 'ceiling', 'chair', 'cloth', 'computer', 'cow', + 'cup', 'curtain', 'dog', 'door', 'fence', 'floor', 'flower', + 'food', 'grass', 'ground', 'horse', 'keyboard', 'light', + 'motorbike', 'mountain', 'mouse', 'person', 'plate', 'platform', + 'pottedplant', 'road', 'rock', 'sheep', 'shelves', 'sidewalk', + 'sign', 'sky', 'snow', 'sofa', 'table', 'track', 'train', + 'tree', 'truck', 'tvmonitor', 'wall', 'water', 'window', 'wood') + + PALETTE = [[180, 120, 120], [6, 230, 230], [80, 50, 50], [4, 200, 3], + [120, 120, 80], [140, 140, 140], [204, 5, 255], [230, 230, 230], + [4, 250, 7], [224, 5, 255], [235, 255, 7], [150, 5, 61], + [120, 120, 70], [8, 255, 51], [255, 6, 82], [143, 255, 140], + [204, 255, 4], [255, 51, 7], [204, 70, 3], [0, 102, 200], + [61, 230, 250], [255, 6, 51], [11, 102, 255], [255, 7, 71], + [255, 9, 224], [9, 7, 230], [220, 220, 220], [255, 9, 92], + [112, 9, 255], [8, 255, 214], [7, 255, 224], [255, 184, 6], + [10, 255, 71], [255, 41, 10], [7, 255, 255], [224, 255, 8], + [102, 8, 255], [255, 61, 6], [255, 194, 7], [255, 122, 8], + [0, 255, 20], [255, 8, 41], [255, 5, 153], [6, 51, 255], + [235, 12, 255], [160, 150, 20], [0, 163, 255], [140, 140, 140], + [250, 10, 15], [20, 255, 0], [31, 255, 0], [255, 31, 0], + [255, 224, 0], [153, 255, 0], [0, 0, 255], [255, 71, 0], + [0, 235, 255], [0, 173, 255], [31, 0, 255]] + + def __init__(self, split, **kwargs): + super(PascalContextDataset59, self).__init__( + img_suffix='.jpg', + seg_map_suffix='.png', + split=split, + reduce_zero_label=True, + **kwargs) + assert osp.exists(self.img_dir) and self.split is not None diff --git a/annotator/uniformer/mmseg/datasets/pipelines/__init__.py b/annotator/uniformer/mmseg/datasets/pipelines/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b9046b07bb4ddea7a707a392b42e72db7c9df67 --- /dev/null +++ b/annotator/uniformer/mmseg/datasets/pipelines/__init__.py @@ -0,0 +1,16 @@ +from .compose import Compose +from .formating import (Collect, ImageToTensor, ToDataContainer, ToTensor, + Transpose, to_tensor) +from .loading import LoadAnnotations, LoadImageFromFile +from .test_time_aug import MultiScaleFlipAug +from .transforms import (CLAHE, AdjustGamma, Normalize, Pad, + PhotoMetricDistortion, RandomCrop, RandomFlip, + RandomRotate, Rerange, Resize, RGB2Gray, SegRescale) + +__all__ = [ + 'Compose', 'to_tensor', 'ToTensor', 'ImageToTensor', 'ToDataContainer', + 'Transpose', 'Collect', 'LoadAnnotations', 'LoadImageFromFile', + 'MultiScaleFlipAug', 'Resize', 'RandomFlip', 'Pad', 'RandomCrop', + 'Normalize', 'SegRescale', 'PhotoMetricDistortion', 'RandomRotate', + 'AdjustGamma', 'CLAHE', 'Rerange', 'RGB2Gray' +] diff --git a/annotator/uniformer/mmseg/datasets/pipelines/compose.py b/annotator/uniformer/mmseg/datasets/pipelines/compose.py new file mode 100644 index 0000000000000000000000000000000000000000..cbfcbb925c6d4ebf849328b9f94ef6fc24359bf5 --- /dev/null +++ b/annotator/uniformer/mmseg/datasets/pipelines/compose.py @@ -0,0 +1,51 @@ +import collections + +from annotator.uniformer.mmcv.utils import build_from_cfg + +from ..builder import PIPELINES + + +@PIPELINES.register_module() +class Compose(object): + """Compose multiple transforms sequentially. + + Args: + transforms (Sequence[dict | callable]): Sequence of transform object or + config dict to be composed. + """ + + def __init__(self, transforms): + assert isinstance(transforms, collections.abc.Sequence) + self.transforms = [] + for transform in transforms: + if isinstance(transform, dict): + transform = build_from_cfg(transform, PIPELINES) + self.transforms.append(transform) + elif callable(transform): + self.transforms.append(transform) + else: + raise TypeError('transform must be callable or a dict') + + def __call__(self, data): + """Call function to apply transforms sequentially. + + Args: + data (dict): A result dict contains the data to transform. + + Returns: + dict: Transformed data. + """ + + for t in self.transforms: + data = t(data) + if data is None: + return None + return data + + def __repr__(self): + format_string = self.__class__.__name__ + '(' + for t in self.transforms: + format_string += '\n' + format_string += f' {t}' + format_string += '\n)' + return format_string diff --git a/annotator/uniformer/mmseg/datasets/pipelines/formating.py b/annotator/uniformer/mmseg/datasets/pipelines/formating.py new file mode 100644 index 0000000000000000000000000000000000000000..97db85f4f9db39fb86ba77ead7d1a8407d810adb --- /dev/null +++ b/annotator/uniformer/mmseg/datasets/pipelines/formating.py @@ -0,0 +1,288 @@ +from collections.abc import Sequence + +import annotator.uniformer.mmcv as mmcv +import numpy as np +import torch +from annotator.uniformer.mmcv.parallel import DataContainer as DC + +from ..builder import PIPELINES + + +def to_tensor(data): + """Convert objects of various python types to :obj:`torch.Tensor`. + + Supported types are: :class:`numpy.ndarray`, :class:`torch.Tensor`, + :class:`Sequence`, :class:`int` and :class:`float`. + + Args: + data (torch.Tensor | numpy.ndarray | Sequence | int | float): Data to + be converted. + """ + + if isinstance(data, torch.Tensor): + return data + elif isinstance(data, np.ndarray): + return torch.from_numpy(data) + elif isinstance(data, Sequence) and not mmcv.is_str(data): + return torch.tensor(data) + elif isinstance(data, int): + return torch.LongTensor([data]) + elif isinstance(data, float): + return torch.FloatTensor([data]) + else: + raise TypeError(f'type {type(data)} cannot be converted to tensor.') + + +@PIPELINES.register_module() +class ToTensor(object): + """Convert some results to :obj:`torch.Tensor` by given keys. + + Args: + keys (Sequence[str]): Keys that need to be converted to Tensor. + """ + + def __init__(self, keys): + self.keys = keys + + def __call__(self, results): + """Call function to convert data in results to :obj:`torch.Tensor`. + + Args: + results (dict): Result dict contains the data to convert. + + Returns: + dict: The result dict contains the data converted + to :obj:`torch.Tensor`. + """ + + for key in self.keys: + results[key] = to_tensor(results[key]) + return results + + def __repr__(self): + return self.__class__.__name__ + f'(keys={self.keys})' + + +@PIPELINES.register_module() +class ImageToTensor(object): + """Convert image to :obj:`torch.Tensor` by given keys. + + The dimension order of input image is (H, W, C). The pipeline will convert + it to (C, H, W). If only 2 dimension (H, W) is given, the output would be + (1, H, W). + + Args: + keys (Sequence[str]): Key of images to be converted to Tensor. + """ + + def __init__(self, keys): + self.keys = keys + + def __call__(self, results): + """Call function to convert image in results to :obj:`torch.Tensor` and + transpose the channel order. + + Args: + results (dict): Result dict contains the image data to convert. + + Returns: + dict: The result dict contains the image converted + to :obj:`torch.Tensor` and transposed to (C, H, W) order. + """ + + for key in self.keys: + img = results[key] + if len(img.shape) < 3: + img = np.expand_dims(img, -1) + results[key] = to_tensor(img.transpose(2, 0, 1)) + return results + + def __repr__(self): + return self.__class__.__name__ + f'(keys={self.keys})' + + +@PIPELINES.register_module() +class Transpose(object): + """Transpose some results by given keys. + + Args: + keys (Sequence[str]): Keys of results to be transposed. + order (Sequence[int]): Order of transpose. + """ + + def __init__(self, keys, order): + self.keys = keys + self.order = order + + def __call__(self, results): + """Call function to convert image in results to :obj:`torch.Tensor` and + transpose the channel order. + + Args: + results (dict): Result dict contains the image data to convert. + + Returns: + dict: The result dict contains the image converted + to :obj:`torch.Tensor` and transposed to (C, H, W) order. + """ + + for key in self.keys: + results[key] = results[key].transpose(self.order) + return results + + def __repr__(self): + return self.__class__.__name__ + \ + f'(keys={self.keys}, order={self.order})' + + +@PIPELINES.register_module() +class ToDataContainer(object): + """Convert results to :obj:`mmcv.DataContainer` by given fields. + + Args: + fields (Sequence[dict]): Each field is a dict like + ``dict(key='xxx', **kwargs)``. The ``key`` in result will + be converted to :obj:`mmcv.DataContainer` with ``**kwargs``. + Default: ``(dict(key='img', stack=True), + dict(key='gt_semantic_seg'))``. + """ + + def __init__(self, + fields=(dict(key='img', + stack=True), dict(key='gt_semantic_seg'))): + self.fields = fields + + def __call__(self, results): + """Call function to convert data in results to + :obj:`mmcv.DataContainer`. + + Args: + results (dict): Result dict contains the data to convert. + + Returns: + dict: The result dict contains the data converted to + :obj:`mmcv.DataContainer`. + """ + + for field in self.fields: + field = field.copy() + key = field.pop('key') + results[key] = DC(results[key], **field) + return results + + def __repr__(self): + return self.__class__.__name__ + f'(fields={self.fields})' + + +@PIPELINES.register_module() +class DefaultFormatBundle(object): + """Default formatting bundle. + + It simplifies the pipeline of formatting common fields, including "img" + and "gt_semantic_seg". These fields are formatted as follows. + + - img: (1)transpose, (2)to tensor, (3)to DataContainer (stack=True) + - gt_semantic_seg: (1)unsqueeze dim-0 (2)to tensor, + (3)to DataContainer (stack=True) + """ + + def __call__(self, results): + """Call function to transform and format common fields in results. + + Args: + results (dict): Result dict contains the data to convert. + + Returns: + dict: The result dict contains the data that is formatted with + default bundle. + """ + + if 'img' in results: + img = results['img'] + if len(img.shape) < 3: + img = np.expand_dims(img, -1) + img = np.ascontiguousarray(img.transpose(2, 0, 1)) + results['img'] = DC(to_tensor(img), stack=True) + if 'gt_semantic_seg' in results: + # convert to long + results['gt_semantic_seg'] = DC( + to_tensor(results['gt_semantic_seg'][None, + ...].astype(np.int64)), + stack=True) + return results + + def __repr__(self): + return self.__class__.__name__ + + +@PIPELINES.register_module() +class Collect(object): + """Collect data from the loader relevant to the specific task. + + This is usually the last stage of the data loader pipeline. Typically keys + is set to some subset of "img", "gt_semantic_seg". + + The "img_meta" item is always populated. The contents of the "img_meta" + dictionary depends on "meta_keys". By default this includes: + + - "img_shape": shape of the image input to the network as a tuple + (h, w, c). Note that images may be zero padded on the bottom/right + if the batch tensor is larger than this shape. + + - "scale_factor": a float indicating the preprocessing scale + + - "flip": a boolean indicating if image flip transform was used + + - "filename": path to the image file + + - "ori_shape": original shape of the image as a tuple (h, w, c) + + - "pad_shape": image shape after padding + + - "img_norm_cfg": a dict of normalization information: + - mean - per channel mean subtraction + - std - per channel std divisor + - to_rgb - bool indicating if bgr was converted to rgb + + Args: + keys (Sequence[str]): Keys of results to be collected in ``data``. + meta_keys (Sequence[str], optional): Meta keys to be converted to + ``mmcv.DataContainer`` and collected in ``data[img_metas]``. + Default: ``('filename', 'ori_filename', 'ori_shape', 'img_shape', + 'pad_shape', 'scale_factor', 'flip', 'flip_direction', + 'img_norm_cfg')`` + """ + + def __init__(self, + keys, + meta_keys=('filename', 'ori_filename', 'ori_shape', + 'img_shape', 'pad_shape', 'scale_factor', 'flip', + 'flip_direction', 'img_norm_cfg')): + self.keys = keys + self.meta_keys = meta_keys + + def __call__(self, results): + """Call function to collect keys in results. The keys in ``meta_keys`` + will be converted to :obj:mmcv.DataContainer. + + Args: + results (dict): Result dict contains the data to collect. + + Returns: + dict: The result dict contains the following keys + - keys in``self.keys`` + - ``img_metas`` + """ + + data = {} + img_meta = {} + for key in self.meta_keys: + img_meta[key] = results[key] + data['img_metas'] = DC(img_meta, cpu_only=True) + for key in self.keys: + data[key] = results[key] + return data + + def __repr__(self): + return self.__class__.__name__ + \ + f'(keys={self.keys}, meta_keys={self.meta_keys})' diff --git a/annotator/uniformer/mmseg/datasets/pipelines/loading.py b/annotator/uniformer/mmseg/datasets/pipelines/loading.py new file mode 100644 index 0000000000000000000000000000000000000000..d3692ae91f19b9c7ccf6023168788ff42c9e93e3 --- /dev/null +++ b/annotator/uniformer/mmseg/datasets/pipelines/loading.py @@ -0,0 +1,153 @@ +import os.path as osp + +import annotator.uniformer.mmcv as mmcv +import numpy as np + +from ..builder import PIPELINES + + +@PIPELINES.register_module() +class LoadImageFromFile(object): + """Load an image from file. + + Required keys are "img_prefix" and "img_info" (a dict that must contain the + key "filename"). Added or updated keys are "filename", "img", "img_shape", + "ori_shape" (same as `img_shape`), "pad_shape" (same as `img_shape`), + "scale_factor" (1.0) and "img_norm_cfg" (means=0 and stds=1). + + Args: + to_float32 (bool): Whether to convert the loaded image to a float32 + numpy array. If set to False, the loaded image is an uint8 array. + Defaults to False. + color_type (str): The flag argument for :func:`mmcv.imfrombytes`. + Defaults to 'color'. + file_client_args (dict): Arguments to instantiate a FileClient. + See :class:`mmcv.fileio.FileClient` for details. + Defaults to ``dict(backend='disk')``. + imdecode_backend (str): Backend for :func:`mmcv.imdecode`. Default: + 'cv2' + """ + + def __init__(self, + to_float32=False, + color_type='color', + file_client_args=dict(backend='disk'), + imdecode_backend='cv2'): + self.to_float32 = to_float32 + self.color_type = color_type + self.file_client_args = file_client_args.copy() + self.file_client = None + self.imdecode_backend = imdecode_backend + + def __call__(self, results): + """Call functions to load image and get image meta information. + + Args: + results (dict): Result dict from :obj:`mmseg.CustomDataset`. + + Returns: + dict: The dict contains loaded image and meta information. + """ + + if self.file_client is None: + self.file_client = mmcv.FileClient(**self.file_client_args) + + if results.get('img_prefix') is not None: + filename = osp.join(results['img_prefix'], + results['img_info']['filename']) + else: + filename = results['img_info']['filename'] + img_bytes = self.file_client.get(filename) + img = mmcv.imfrombytes( + img_bytes, flag=self.color_type, backend=self.imdecode_backend) + if self.to_float32: + img = img.astype(np.float32) + + results['filename'] = filename + results['ori_filename'] = results['img_info']['filename'] + results['img'] = img + results['img_shape'] = img.shape + results['ori_shape'] = img.shape + # Set initial values for default meta_keys + results['pad_shape'] = img.shape + results['scale_factor'] = 1.0 + num_channels = 1 if len(img.shape) < 3 else img.shape[2] + results['img_norm_cfg'] = dict( + mean=np.zeros(num_channels, dtype=np.float32), + std=np.ones(num_channels, dtype=np.float32), + to_rgb=False) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(to_float32={self.to_float32},' + repr_str += f"color_type='{self.color_type}'," + repr_str += f"imdecode_backend='{self.imdecode_backend}')" + return repr_str + + +@PIPELINES.register_module() +class LoadAnnotations(object): + """Load annotations for semantic segmentation. + + Args: + reduce_zero_label (bool): Whether reduce all label value by 1. + Usually used for datasets where 0 is background label. + Default: False. + file_client_args (dict): Arguments to instantiate a FileClient. + See :class:`mmcv.fileio.FileClient` for details. + Defaults to ``dict(backend='disk')``. + imdecode_backend (str): Backend for :func:`mmcv.imdecode`. Default: + 'pillow' + """ + + def __init__(self, + reduce_zero_label=False, + file_client_args=dict(backend='disk'), + imdecode_backend='pillow'): + self.reduce_zero_label = reduce_zero_label + self.file_client_args = file_client_args.copy() + self.file_client = None + self.imdecode_backend = imdecode_backend + + def __call__(self, results): + """Call function to load multiple types annotations. + + Args: + results (dict): Result dict from :obj:`mmseg.CustomDataset`. + + Returns: + dict: The dict contains loaded semantic segmentation annotations. + """ + + if self.file_client is None: + self.file_client = mmcv.FileClient(**self.file_client_args) + + if results.get('seg_prefix', None) is not None: + filename = osp.join(results['seg_prefix'], + results['ann_info']['seg_map']) + else: + filename = results['ann_info']['seg_map'] + img_bytes = self.file_client.get(filename) + gt_semantic_seg = mmcv.imfrombytes( + img_bytes, flag='unchanged', + backend=self.imdecode_backend).squeeze().astype(np.uint8) + # modify if custom classes + if results.get('label_map', None) is not None: + for old_id, new_id in results['label_map'].items(): + gt_semantic_seg[gt_semantic_seg == old_id] = new_id + # reduce zero_label + if self.reduce_zero_label: + # avoid using underflow conversion + gt_semantic_seg[gt_semantic_seg == 0] = 255 + gt_semantic_seg = gt_semantic_seg - 1 + gt_semantic_seg[gt_semantic_seg == 254] = 255 + results['gt_semantic_seg'] = gt_semantic_seg + results['seg_fields'].append('gt_semantic_seg') + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(reduce_zero_label={self.reduce_zero_label},' + repr_str += f"imdecode_backend='{self.imdecode_backend}')" + return repr_str diff --git a/annotator/uniformer/mmseg/datasets/pipelines/test_time_aug.py b/annotator/uniformer/mmseg/datasets/pipelines/test_time_aug.py new file mode 100644 index 0000000000000000000000000000000000000000..6a1611a04d9d927223c9afbe5bf68af04d62937a --- /dev/null +++ b/annotator/uniformer/mmseg/datasets/pipelines/test_time_aug.py @@ -0,0 +1,133 @@ +import warnings + +import annotator.uniformer.mmcv as mmcv + +from ..builder import PIPELINES +from .compose import Compose + + +@PIPELINES.register_module() +class MultiScaleFlipAug(object): + """Test-time augmentation with multiple scales and flipping. + + An example configuration is as followed: + + .. code-block:: + + img_scale=(2048, 1024), + img_ratios=[0.5, 1.0], + flip=True, + transforms=[ + dict(type='Resize', keep_ratio=True), + dict(type='RandomFlip'), + dict(type='Normalize', **img_norm_cfg), + dict(type='Pad', size_divisor=32), + dict(type='ImageToTensor', keys=['img']), + dict(type='Collect', keys=['img']), + ] + + After MultiScaleFLipAug with above configuration, the results are wrapped + into lists of the same length as followed: + + .. code-block:: + + dict( + img=[...], + img_shape=[...], + scale=[(1024, 512), (1024, 512), (2048, 1024), (2048, 1024)] + flip=[False, True, False, True] + ... + ) + + Args: + transforms (list[dict]): Transforms to apply in each augmentation. + img_scale (None | tuple | list[tuple]): Images scales for resizing. + img_ratios (float | list[float]): Image ratios for resizing + flip (bool): Whether apply flip augmentation. Default: False. + flip_direction (str | list[str]): Flip augmentation directions, + options are "horizontal" and "vertical". If flip_direction is list, + multiple flip augmentations will be applied. + It has no effect when flip == False. Default: "horizontal". + """ + + def __init__(self, + transforms, + img_scale, + img_ratios=None, + flip=False, + flip_direction='horizontal'): + self.transforms = Compose(transforms) + if img_ratios is not None: + img_ratios = img_ratios if isinstance(img_ratios, + list) else [img_ratios] + assert mmcv.is_list_of(img_ratios, float) + if img_scale is None: + # mode 1: given img_scale=None and a range of image ratio + self.img_scale = None + assert mmcv.is_list_of(img_ratios, float) + elif isinstance(img_scale, tuple) and mmcv.is_list_of( + img_ratios, float): + assert len(img_scale) == 2 + # mode 2: given a scale and a range of image ratio + self.img_scale = [(int(img_scale[0] * ratio), + int(img_scale[1] * ratio)) + for ratio in img_ratios] + else: + # mode 3: given multiple scales + self.img_scale = img_scale if isinstance(img_scale, + list) else [img_scale] + assert mmcv.is_list_of(self.img_scale, tuple) or self.img_scale is None + self.flip = flip + self.img_ratios = img_ratios + self.flip_direction = flip_direction if isinstance( + flip_direction, list) else [flip_direction] + assert mmcv.is_list_of(self.flip_direction, str) + if not self.flip and self.flip_direction != ['horizontal']: + warnings.warn( + 'flip_direction has no effect when flip is set to False') + if (self.flip + and not any([t['type'] == 'RandomFlip' for t in transforms])): + warnings.warn( + 'flip has no effect when RandomFlip is not in transforms') + + def __call__(self, results): + """Call function to apply test time augment transforms on results. + + Args: + results (dict): Result dict contains the data to transform. + + Returns: + dict[str: list]: The augmented data, where each value is wrapped + into a list. + """ + + aug_data = [] + if self.img_scale is None and mmcv.is_list_of(self.img_ratios, float): + h, w = results['img'].shape[:2] + img_scale = [(int(w * ratio), int(h * ratio)) + for ratio in self.img_ratios] + else: + img_scale = self.img_scale + flip_aug = [False, True] if self.flip else [False] + for scale in img_scale: + for flip in flip_aug: + for direction in self.flip_direction: + _results = results.copy() + _results['scale'] = scale + _results['flip'] = flip + _results['flip_direction'] = direction + data = self.transforms(_results) + aug_data.append(data) + # list of dict to dict of list + aug_data_dict = {key: [] for key in aug_data[0]} + for data in aug_data: + for key, val in data.items(): + aug_data_dict[key].append(val) + return aug_data_dict + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(transforms={self.transforms}, ' + repr_str += f'img_scale={self.img_scale}, flip={self.flip})' + repr_str += f'flip_direction={self.flip_direction}' + return repr_str diff --git a/annotator/uniformer/mmseg/datasets/pipelines/transforms.py b/annotator/uniformer/mmseg/datasets/pipelines/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..94e869b252ef6d8b43604add2bbc02f034614bfb --- /dev/null +++ b/annotator/uniformer/mmseg/datasets/pipelines/transforms.py @@ -0,0 +1,889 @@ +import annotator.uniformer.mmcv as mmcv +import numpy as np +from annotator.uniformer.mmcv.utils import deprecated_api_warning, is_tuple_of +from numpy import random + +from ..builder import PIPELINES + + +@PIPELINES.register_module() +class Resize(object): + """Resize images & seg. + + This transform resizes the input image to some scale. If the input dict + contains the key "scale", then the scale in the input dict is used, + otherwise the specified scale in the init method is used. + + ``img_scale`` can be None, a tuple (single-scale) or a list of tuple + (multi-scale). There are 4 multiscale modes: + + - ``ratio_range is not None``: + 1. When img_scale is None, img_scale is the shape of image in results + (img_scale = results['img'].shape[:2]) and the image is resized based + on the original size. (mode 1) + 2. When img_scale is a tuple (single-scale), randomly sample a ratio from + the ratio range and multiply it with the image scale. (mode 2) + + - ``ratio_range is None and multiscale_mode == "range"``: randomly sample a + scale from the a range. (mode 3) + + - ``ratio_range is None and multiscale_mode == "value"``: randomly sample a + scale from multiple scales. (mode 4) + + Args: + img_scale (tuple or list[tuple]): Images scales for resizing. + multiscale_mode (str): Either "range" or "value". + ratio_range (tuple[float]): (min_ratio, max_ratio) + keep_ratio (bool): Whether to keep the aspect ratio when resizing the + image. + """ + + def __init__(self, + img_scale=None, + multiscale_mode='range', + ratio_range=None, + keep_ratio=True): + if img_scale is None: + self.img_scale = None + else: + if isinstance(img_scale, list): + self.img_scale = img_scale + else: + self.img_scale = [img_scale] + assert mmcv.is_list_of(self.img_scale, tuple) + + if ratio_range is not None: + # mode 1: given img_scale=None and a range of image ratio + # mode 2: given a scale and a range of image ratio + assert self.img_scale is None or len(self.img_scale) == 1 + else: + # mode 3 and 4: given multiple scales or a range of scales + assert multiscale_mode in ['value', 'range'] + + self.multiscale_mode = multiscale_mode + self.ratio_range = ratio_range + self.keep_ratio = keep_ratio + + @staticmethod + def random_select(img_scales): + """Randomly select an img_scale from given candidates. + + Args: + img_scales (list[tuple]): Images scales for selection. + + Returns: + (tuple, int): Returns a tuple ``(img_scale, scale_dix)``, + where ``img_scale`` is the selected image scale and + ``scale_idx`` is the selected index in the given candidates. + """ + + assert mmcv.is_list_of(img_scales, tuple) + scale_idx = np.random.randint(len(img_scales)) + img_scale = img_scales[scale_idx] + return img_scale, scale_idx + + @staticmethod + def random_sample(img_scales): + """Randomly sample an img_scale when ``multiscale_mode=='range'``. + + Args: + img_scales (list[tuple]): Images scale range for sampling. + There must be two tuples in img_scales, which specify the lower + and upper bound of image scales. + + Returns: + (tuple, None): Returns a tuple ``(img_scale, None)``, where + ``img_scale`` is sampled scale and None is just a placeholder + to be consistent with :func:`random_select`. + """ + + assert mmcv.is_list_of(img_scales, tuple) and len(img_scales) == 2 + img_scale_long = [max(s) for s in img_scales] + img_scale_short = [min(s) for s in img_scales] + long_edge = np.random.randint( + min(img_scale_long), + max(img_scale_long) + 1) + short_edge = np.random.randint( + min(img_scale_short), + max(img_scale_short) + 1) + img_scale = (long_edge, short_edge) + return img_scale, None + + @staticmethod + def random_sample_ratio(img_scale, ratio_range): + """Randomly sample an img_scale when ``ratio_range`` is specified. + + A ratio will be randomly sampled from the range specified by + ``ratio_range``. Then it would be multiplied with ``img_scale`` to + generate sampled scale. + + Args: + img_scale (tuple): Images scale base to multiply with ratio. + ratio_range (tuple[float]): The minimum and maximum ratio to scale + the ``img_scale``. + + Returns: + (tuple, None): Returns a tuple ``(scale, None)``, where + ``scale`` is sampled ratio multiplied with ``img_scale`` and + None is just a placeholder to be consistent with + :func:`random_select`. + """ + + assert isinstance(img_scale, tuple) and len(img_scale) == 2 + min_ratio, max_ratio = ratio_range + assert min_ratio <= max_ratio + ratio = np.random.random_sample() * (max_ratio - min_ratio) + min_ratio + scale = int(img_scale[0] * ratio), int(img_scale[1] * ratio) + return scale, None + + def _random_scale(self, results): + """Randomly sample an img_scale according to ``ratio_range`` and + ``multiscale_mode``. + + If ``ratio_range`` is specified, a ratio will be sampled and be + multiplied with ``img_scale``. + If multiple scales are specified by ``img_scale``, a scale will be + sampled according to ``multiscale_mode``. + Otherwise, single scale will be used. + + Args: + results (dict): Result dict from :obj:`dataset`. + + Returns: + dict: Two new keys 'scale` and 'scale_idx` are added into + ``results``, which would be used by subsequent pipelines. + """ + + if self.ratio_range is not None: + if self.img_scale is None: + h, w = results['img'].shape[:2] + scale, scale_idx = self.random_sample_ratio((w, h), + self.ratio_range) + else: + scale, scale_idx = self.random_sample_ratio( + self.img_scale[0], self.ratio_range) + elif len(self.img_scale) == 1: + scale, scale_idx = self.img_scale[0], 0 + elif self.multiscale_mode == 'range': + scale, scale_idx = self.random_sample(self.img_scale) + elif self.multiscale_mode == 'value': + scale, scale_idx = self.random_select(self.img_scale) + else: + raise NotImplementedError + + results['scale'] = scale + results['scale_idx'] = scale_idx + + def _resize_img(self, results): + """Resize images with ``results['scale']``.""" + if self.keep_ratio: + img, scale_factor = mmcv.imrescale( + results['img'], results['scale'], return_scale=True) + # the w_scale and h_scale has minor difference + # a real fix should be done in the mmcv.imrescale in the future + new_h, new_w = img.shape[:2] + h, w = results['img'].shape[:2] + w_scale = new_w / w + h_scale = new_h / h + else: + img, w_scale, h_scale = mmcv.imresize( + results['img'], results['scale'], return_scale=True) + scale_factor = np.array([w_scale, h_scale, w_scale, h_scale], + dtype=np.float32) + results['img'] = img + results['img_shape'] = img.shape + results['pad_shape'] = img.shape # in case that there is no padding + results['scale_factor'] = scale_factor + results['keep_ratio'] = self.keep_ratio + + def _resize_seg(self, results): + """Resize semantic segmentation map with ``results['scale']``.""" + for key in results.get('seg_fields', []): + if self.keep_ratio: + gt_seg = mmcv.imrescale( + results[key], results['scale'], interpolation='nearest') + else: + gt_seg = mmcv.imresize( + results[key], results['scale'], interpolation='nearest') + results[key] = gt_seg + + def __call__(self, results): + """Call function to resize images, bounding boxes, masks, semantic + segmentation map. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Resized results, 'img_shape', 'pad_shape', 'scale_factor', + 'keep_ratio' keys are added into result dict. + """ + + if 'scale' not in results: + self._random_scale(results) + self._resize_img(results) + self._resize_seg(results) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += (f'(img_scale={self.img_scale}, ' + f'multiscale_mode={self.multiscale_mode}, ' + f'ratio_range={self.ratio_range}, ' + f'keep_ratio={self.keep_ratio})') + return repr_str + + +@PIPELINES.register_module() +class RandomFlip(object): + """Flip the image & seg. + + If the input dict contains the key "flip", then the flag will be used, + otherwise it will be randomly decided by a ratio specified in the init + method. + + Args: + prob (float, optional): The flipping probability. Default: None. + direction(str, optional): The flipping direction. Options are + 'horizontal' and 'vertical'. Default: 'horizontal'. + """ + + @deprecated_api_warning({'flip_ratio': 'prob'}, cls_name='RandomFlip') + def __init__(self, prob=None, direction='horizontal'): + self.prob = prob + self.direction = direction + if prob is not None: + assert prob >= 0 and prob <= 1 + assert direction in ['horizontal', 'vertical'] + + def __call__(self, results): + """Call function to flip bounding boxes, masks, semantic segmentation + maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Flipped results, 'flip', 'flip_direction' keys are added into + result dict. + """ + + if 'flip' not in results: + flip = True if np.random.rand() < self.prob else False + results['flip'] = flip + if 'flip_direction' not in results: + results['flip_direction'] = self.direction + if results['flip']: + # flip image + results['img'] = mmcv.imflip( + results['img'], direction=results['flip_direction']) + + # flip segs + for key in results.get('seg_fields', []): + # use copy() to make numpy stride positive + results[key] = mmcv.imflip( + results[key], direction=results['flip_direction']).copy() + return results + + def __repr__(self): + return self.__class__.__name__ + f'(prob={self.prob})' + + +@PIPELINES.register_module() +class Pad(object): + """Pad the image & mask. + + There are two padding modes: (1) pad to a fixed size and (2) pad to the + minimum size that is divisible by some number. + Added keys are "pad_shape", "pad_fixed_size", "pad_size_divisor", + + Args: + size (tuple, optional): Fixed padding size. + size_divisor (int, optional): The divisor of padded size. + pad_val (float, optional): Padding value. Default: 0. + seg_pad_val (float, optional): Padding value of segmentation map. + Default: 255. + """ + + def __init__(self, + size=None, + size_divisor=None, + pad_val=0, + seg_pad_val=255): + self.size = size + self.size_divisor = size_divisor + self.pad_val = pad_val + self.seg_pad_val = seg_pad_val + # only one of size and size_divisor should be valid + assert size is not None or size_divisor is not None + assert size is None or size_divisor is None + + def _pad_img(self, results): + """Pad images according to ``self.size``.""" + if self.size is not None: + padded_img = mmcv.impad( + results['img'], shape=self.size, pad_val=self.pad_val) + elif self.size_divisor is not None: + padded_img = mmcv.impad_to_multiple( + results['img'], self.size_divisor, pad_val=self.pad_val) + results['img'] = padded_img + results['pad_shape'] = padded_img.shape + results['pad_fixed_size'] = self.size + results['pad_size_divisor'] = self.size_divisor + + def _pad_seg(self, results): + """Pad masks according to ``results['pad_shape']``.""" + for key in results.get('seg_fields', []): + results[key] = mmcv.impad( + results[key], + shape=results['pad_shape'][:2], + pad_val=self.seg_pad_val) + + def __call__(self, results): + """Call function to pad images, masks, semantic segmentation maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Updated result dict. + """ + + self._pad_img(results) + self._pad_seg(results) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(size={self.size}, size_divisor={self.size_divisor}, ' \ + f'pad_val={self.pad_val})' + return repr_str + + +@PIPELINES.register_module() +class Normalize(object): + """Normalize the image. + + Added key is "img_norm_cfg". + + Args: + mean (sequence): Mean values of 3 channels. + std (sequence): Std values of 3 channels. + to_rgb (bool): Whether to convert the image from BGR to RGB, + default is true. + """ + + def __init__(self, mean, std, to_rgb=True): + self.mean = np.array(mean, dtype=np.float32) + self.std = np.array(std, dtype=np.float32) + self.to_rgb = to_rgb + + def __call__(self, results): + """Call function to normalize images. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Normalized results, 'img_norm_cfg' key is added into + result dict. + """ + + results['img'] = mmcv.imnormalize(results['img'], self.mean, self.std, + self.to_rgb) + results['img_norm_cfg'] = dict( + mean=self.mean, std=self.std, to_rgb=self.to_rgb) + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(mean={self.mean}, std={self.std}, to_rgb=' \ + f'{self.to_rgb})' + return repr_str + + +@PIPELINES.register_module() +class Rerange(object): + """Rerange the image pixel value. + + Args: + min_value (float or int): Minimum value of the reranged image. + Default: 0. + max_value (float or int): Maximum value of the reranged image. + Default: 255. + """ + + def __init__(self, min_value=0, max_value=255): + assert isinstance(min_value, float) or isinstance(min_value, int) + assert isinstance(max_value, float) or isinstance(max_value, int) + assert min_value < max_value + self.min_value = min_value + self.max_value = max_value + + def __call__(self, results): + """Call function to rerange images. + + Args: + results (dict): Result dict from loading pipeline. + Returns: + dict: Reranged results. + """ + + img = results['img'] + img_min_value = np.min(img) + img_max_value = np.max(img) + + assert img_min_value < img_max_value + # rerange to [0, 1] + img = (img - img_min_value) / (img_max_value - img_min_value) + # rerange to [min_value, max_value] + img = img * (self.max_value - self.min_value) + self.min_value + results['img'] = img + + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(min_value={self.min_value}, max_value={self.max_value})' + return repr_str + + +@PIPELINES.register_module() +class CLAHE(object): + """Use CLAHE method to process the image. + + See `ZUIDERVELD,K. Contrast Limited Adaptive Histogram Equalization[J]. + Graphics Gems, 1994:474-485.` for more information. + + Args: + clip_limit (float): Threshold for contrast limiting. Default: 40.0. + tile_grid_size (tuple[int]): Size of grid for histogram equalization. + Input image will be divided into equally sized rectangular tiles. + It defines the number of tiles in row and column. Default: (8, 8). + """ + + def __init__(self, clip_limit=40.0, tile_grid_size=(8, 8)): + assert isinstance(clip_limit, (float, int)) + self.clip_limit = clip_limit + assert is_tuple_of(tile_grid_size, int) + assert len(tile_grid_size) == 2 + self.tile_grid_size = tile_grid_size + + def __call__(self, results): + """Call function to Use CLAHE method process images. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Processed results. + """ + + for i in range(results['img'].shape[2]): + results['img'][:, :, i] = mmcv.clahe( + np.array(results['img'][:, :, i], dtype=np.uint8), + self.clip_limit, self.tile_grid_size) + + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(clip_limit={self.clip_limit}, '\ + f'tile_grid_size={self.tile_grid_size})' + return repr_str + + +@PIPELINES.register_module() +class RandomCrop(object): + """Random crop the image & seg. + + Args: + crop_size (tuple): Expected size after cropping, (h, w). + cat_max_ratio (float): The maximum ratio that single category could + occupy. + """ + + def __init__(self, crop_size, cat_max_ratio=1., ignore_index=255): + assert crop_size[0] > 0 and crop_size[1] > 0 + self.crop_size = crop_size + self.cat_max_ratio = cat_max_ratio + self.ignore_index = ignore_index + + def get_crop_bbox(self, img): + """Randomly get a crop bounding box.""" + margin_h = max(img.shape[0] - self.crop_size[0], 0) + margin_w = max(img.shape[1] - self.crop_size[1], 0) + offset_h = np.random.randint(0, margin_h + 1) + offset_w = np.random.randint(0, margin_w + 1) + crop_y1, crop_y2 = offset_h, offset_h + self.crop_size[0] + crop_x1, crop_x2 = offset_w, offset_w + self.crop_size[1] + + return crop_y1, crop_y2, crop_x1, crop_x2 + + def crop(self, img, crop_bbox): + """Crop from ``img``""" + crop_y1, crop_y2, crop_x1, crop_x2 = crop_bbox + img = img[crop_y1:crop_y2, crop_x1:crop_x2, ...] + return img + + def __call__(self, results): + """Call function to randomly crop images, semantic segmentation maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Randomly cropped results, 'img_shape' key in result dict is + updated according to crop size. + """ + + img = results['img'] + crop_bbox = self.get_crop_bbox(img) + if self.cat_max_ratio < 1.: + # Repeat 10 times + for _ in range(10): + seg_temp = self.crop(results['gt_semantic_seg'], crop_bbox) + labels, cnt = np.unique(seg_temp, return_counts=True) + cnt = cnt[labels != self.ignore_index] + if len(cnt) > 1 and np.max(cnt) / np.sum( + cnt) < self.cat_max_ratio: + break + crop_bbox = self.get_crop_bbox(img) + + # crop the image + img = self.crop(img, crop_bbox) + img_shape = img.shape + results['img'] = img + results['img_shape'] = img_shape + + # crop semantic seg + for key in results.get('seg_fields', []): + results[key] = self.crop(results[key], crop_bbox) + + return results + + def __repr__(self): + return self.__class__.__name__ + f'(crop_size={self.crop_size})' + + +@PIPELINES.register_module() +class RandomRotate(object): + """Rotate the image & seg. + + Args: + prob (float): The rotation probability. + degree (float, tuple[float]): Range of degrees to select from. If + degree is a number instead of tuple like (min, max), + the range of degree will be (``-degree``, ``+degree``) + pad_val (float, optional): Padding value of image. Default: 0. + seg_pad_val (float, optional): Padding value of segmentation map. + Default: 255. + center (tuple[float], optional): Center point (w, h) of the rotation in + the source image. If not specified, the center of the image will be + used. Default: None. + auto_bound (bool): Whether to adjust the image size to cover the whole + rotated image. Default: False + """ + + def __init__(self, + prob, + degree, + pad_val=0, + seg_pad_val=255, + center=None, + auto_bound=False): + self.prob = prob + assert prob >= 0 and prob <= 1 + if isinstance(degree, (float, int)): + assert degree > 0, f'degree {degree} should be positive' + self.degree = (-degree, degree) + else: + self.degree = degree + assert len(self.degree) == 2, f'degree {self.degree} should be a ' \ + f'tuple of (min, max)' + self.pal_val = pad_val + self.seg_pad_val = seg_pad_val + self.center = center + self.auto_bound = auto_bound + + def __call__(self, results): + """Call function to rotate image, semantic segmentation maps. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Rotated results. + """ + + rotate = True if np.random.rand() < self.prob else False + degree = np.random.uniform(min(*self.degree), max(*self.degree)) + if rotate: + # rotate image + results['img'] = mmcv.imrotate( + results['img'], + angle=degree, + border_value=self.pal_val, + center=self.center, + auto_bound=self.auto_bound) + + # rotate segs + for key in results.get('seg_fields', []): + results[key] = mmcv.imrotate( + results[key], + angle=degree, + border_value=self.seg_pad_val, + center=self.center, + auto_bound=self.auto_bound, + interpolation='nearest') + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(prob={self.prob}, ' \ + f'degree={self.degree}, ' \ + f'pad_val={self.pal_val}, ' \ + f'seg_pad_val={self.seg_pad_val}, ' \ + f'center={self.center}, ' \ + f'auto_bound={self.auto_bound})' + return repr_str + + +@PIPELINES.register_module() +class RGB2Gray(object): + """Convert RGB image to grayscale image. + + This transform calculate the weighted mean of input image channels with + ``weights`` and then expand the channels to ``out_channels``. When + ``out_channels`` is None, the number of output channels is the same as + input channels. + + Args: + out_channels (int): Expected number of output channels after + transforming. Default: None. + weights (tuple[float]): The weights to calculate the weighted mean. + Default: (0.299, 0.587, 0.114). + """ + + def __init__(self, out_channels=None, weights=(0.299, 0.587, 0.114)): + assert out_channels is None or out_channels > 0 + self.out_channels = out_channels + assert isinstance(weights, tuple) + for item in weights: + assert isinstance(item, (float, int)) + self.weights = weights + + def __call__(self, results): + """Call function to convert RGB image to grayscale image. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Result dict with grayscale image. + """ + img = results['img'] + assert len(img.shape) == 3 + assert img.shape[2] == len(self.weights) + weights = np.array(self.weights).reshape((1, 1, -1)) + img = (img * weights).sum(2, keepdims=True) + if self.out_channels is None: + img = img.repeat(weights.shape[2], axis=2) + else: + img = img.repeat(self.out_channels, axis=2) + + results['img'] = img + results['img_shape'] = img.shape + + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(out_channels={self.out_channels}, ' \ + f'weights={self.weights})' + return repr_str + + +@PIPELINES.register_module() +class AdjustGamma(object): + """Using gamma correction to process the image. + + Args: + gamma (float or int): Gamma value used in gamma correction. + Default: 1.0. + """ + + def __init__(self, gamma=1.0): + assert isinstance(gamma, float) or isinstance(gamma, int) + assert gamma > 0 + self.gamma = gamma + inv_gamma = 1.0 / gamma + self.table = np.array([(i / 255.0)**inv_gamma * 255 + for i in np.arange(256)]).astype('uint8') + + def __call__(self, results): + """Call function to process the image with gamma correction. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Processed results. + """ + + results['img'] = mmcv.lut_transform( + np.array(results['img'], dtype=np.uint8), self.table) + + return results + + def __repr__(self): + return self.__class__.__name__ + f'(gamma={self.gamma})' + + +@PIPELINES.register_module() +class SegRescale(object): + """Rescale semantic segmentation maps. + + Args: + scale_factor (float): The scale factor of the final output. + """ + + def __init__(self, scale_factor=1): + self.scale_factor = scale_factor + + def __call__(self, results): + """Call function to scale the semantic segmentation map. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Result dict with semantic segmentation map scaled. + """ + for key in results.get('seg_fields', []): + if self.scale_factor != 1: + results[key] = mmcv.imrescale( + results[key], self.scale_factor, interpolation='nearest') + return results + + def __repr__(self): + return self.__class__.__name__ + f'(scale_factor={self.scale_factor})' + + +@PIPELINES.register_module() +class PhotoMetricDistortion(object): + """Apply photometric distortion to image sequentially, every transformation + is applied with a probability of 0.5. The position of random contrast is in + second or second to last. + + 1. random brightness + 2. random contrast (mode 0) + 3. convert color from BGR to HSV + 4. random saturation + 5. random hue + 6. convert color from HSV to BGR + 7. random contrast (mode 1) + + Args: + brightness_delta (int): delta of brightness. + contrast_range (tuple): range of contrast. + saturation_range (tuple): range of saturation. + hue_delta (int): delta of hue. + """ + + def __init__(self, + brightness_delta=32, + contrast_range=(0.5, 1.5), + saturation_range=(0.5, 1.5), + hue_delta=18): + self.brightness_delta = brightness_delta + self.contrast_lower, self.contrast_upper = contrast_range + self.saturation_lower, self.saturation_upper = saturation_range + self.hue_delta = hue_delta + + def convert(self, img, alpha=1, beta=0): + """Multiple with alpha and add beat with clip.""" + img = img.astype(np.float32) * alpha + beta + img = np.clip(img, 0, 255) + return img.astype(np.uint8) + + def brightness(self, img): + """Brightness distortion.""" + if random.randint(2): + return self.convert( + img, + beta=random.uniform(-self.brightness_delta, + self.brightness_delta)) + return img + + def contrast(self, img): + """Contrast distortion.""" + if random.randint(2): + return self.convert( + img, + alpha=random.uniform(self.contrast_lower, self.contrast_upper)) + return img + + def saturation(self, img): + """Saturation distortion.""" + if random.randint(2): + img = mmcv.bgr2hsv(img) + img[:, :, 1] = self.convert( + img[:, :, 1], + alpha=random.uniform(self.saturation_lower, + self.saturation_upper)) + img = mmcv.hsv2bgr(img) + return img + + def hue(self, img): + """Hue distortion.""" + if random.randint(2): + img = mmcv.bgr2hsv(img) + img[:, :, + 0] = (img[:, :, 0].astype(int) + + random.randint(-self.hue_delta, self.hue_delta)) % 180 + img = mmcv.hsv2bgr(img) + return img + + def __call__(self, results): + """Call function to perform photometric distortion on images. + + Args: + results (dict): Result dict from loading pipeline. + + Returns: + dict: Result dict with images distorted. + """ + + img = results['img'] + # random brightness + img = self.brightness(img) + + # mode == 0 --> do random contrast first + # mode == 1 --> do random contrast last + mode = random.randint(2) + if mode == 1: + img = self.contrast(img) + + # random saturation + img = self.saturation(img) + + # random hue + img = self.hue(img) + + # random contrast + if mode == 0: + img = self.contrast(img) + + results['img'] = img + return results + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += (f'(brightness_delta={self.brightness_delta}, ' + f'contrast_range=({self.contrast_lower}, ' + f'{self.contrast_upper}), ' + f'saturation_range=({self.saturation_lower}, ' + f'{self.saturation_upper}), ' + f'hue_delta={self.hue_delta})') + return repr_str diff --git a/annotator/uniformer/mmseg/datasets/stare.py b/annotator/uniformer/mmseg/datasets/stare.py new file mode 100644 index 0000000000000000000000000000000000000000..cbd14e0920e7f6a73baff1432e5a32ccfdb0dfae --- /dev/null +++ b/annotator/uniformer/mmseg/datasets/stare.py @@ -0,0 +1,27 @@ +import os.path as osp + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class STAREDataset(CustomDataset): + """STARE dataset. + + In segmentation map annotation for STARE, 0 stands for background, which is + included in 2 categories. ``reduce_zero_label`` is fixed to False. The + ``img_suffix`` is fixed to '.png' and ``seg_map_suffix`` is fixed to + '.ah.png'. + """ + + CLASSES = ('background', 'vessel') + + PALETTE = [[120, 120, 120], [6, 230, 230]] + + def __init__(self, **kwargs): + super(STAREDataset, self).__init__( + img_suffix='.png', + seg_map_suffix='.ah.png', + reduce_zero_label=False, + **kwargs) + assert osp.exists(self.img_dir) diff --git a/annotator/uniformer/mmseg/datasets/voc.py b/annotator/uniformer/mmseg/datasets/voc.py new file mode 100644 index 0000000000000000000000000000000000000000..a8855203b14ee0dc4da9099a2945d4aedcffbcd6 --- /dev/null +++ b/annotator/uniformer/mmseg/datasets/voc.py @@ -0,0 +1,29 @@ +import os.path as osp + +from .builder import DATASETS +from .custom import CustomDataset + + +@DATASETS.register_module() +class PascalVOCDataset(CustomDataset): + """Pascal VOC dataset. + + Args: + split (str): Split txt file for Pascal VOC. + """ + + CLASSES = ('background', 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', + 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', + 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', + 'train', 'tvmonitor') + + PALETTE = [[0, 0, 0], [128, 0, 0], [0, 128, 0], [128, 128, 0], [0, 0, 128], + [128, 0, 128], [0, 128, 128], [128, 128, 128], [64, 0, 0], + [192, 0, 0], [64, 128, 0], [192, 128, 0], [64, 0, 128], + [192, 0, 128], [64, 128, 128], [192, 128, 128], [0, 64, 0], + [128, 64, 0], [0, 192, 0], [128, 192, 0], [0, 64, 128]] + + def __init__(self, split, **kwargs): + super(PascalVOCDataset, self).__init__( + img_suffix='.jpg', seg_map_suffix='.png', split=split, **kwargs) + assert osp.exists(self.img_dir) and self.split is not None diff --git a/annotator/uniformer/mmseg/models/__init__.py b/annotator/uniformer/mmseg/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3cf93f8bec9cf0cef0a3bd76ca3ca92eb188f535 --- /dev/null +++ b/annotator/uniformer/mmseg/models/__init__.py @@ -0,0 +1,12 @@ +from .backbones import * # noqa: F401,F403 +from .builder import (BACKBONES, HEADS, LOSSES, SEGMENTORS, build_backbone, + build_head, build_loss, build_segmentor) +from .decode_heads import * # noqa: F401,F403 +from .losses import * # noqa: F401,F403 +from .necks import * # noqa: F401,F403 +from .segmentors import * # noqa: F401,F403 + +__all__ = [ + 'BACKBONES', 'HEADS', 'LOSSES', 'SEGMENTORS', 'build_backbone', + 'build_head', 'build_loss', 'build_segmentor' +] diff --git a/annotator/uniformer/mmseg/models/backbones/__init__.py b/annotator/uniformer/mmseg/models/backbones/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8339983905fb5d20bae42ba6f76fea75d278b1aa --- /dev/null +++ b/annotator/uniformer/mmseg/models/backbones/__init__.py @@ -0,0 +1,17 @@ +from .cgnet import CGNet +# from .fast_scnn import FastSCNN +from .hrnet import HRNet +from .mobilenet_v2 import MobileNetV2 +from .mobilenet_v3 import MobileNetV3 +from .resnest import ResNeSt +from .resnet import ResNet, ResNetV1c, ResNetV1d +from .resnext import ResNeXt +from .unet import UNet +from .vit import VisionTransformer +from .uniformer import UniFormer + +__all__ = [ + 'ResNet', 'ResNetV1c', 'ResNetV1d', 'ResNeXt', 'HRNet', + 'ResNeSt', 'MobileNetV2', 'UNet', 'CGNet', 'MobileNetV3', + 'VisionTransformer', 'UniFormer' +] diff --git a/annotator/uniformer/mmseg/models/backbones/cgnet.py b/annotator/uniformer/mmseg/models/backbones/cgnet.py new file mode 100644 index 0000000000000000000000000000000000000000..f8bca442c8f18179f217e40c298fb5ef39df77c4 --- /dev/null +++ b/annotator/uniformer/mmseg/models/backbones/cgnet.py @@ -0,0 +1,367 @@ +import torch +import torch.nn as nn +import torch.utils.checkpoint as cp +from annotator.uniformer.mmcv.cnn import (ConvModule, build_conv_layer, build_norm_layer, + constant_init, kaiming_init) +from annotator.uniformer.mmcv.runner import load_checkpoint +from annotator.uniformer.mmcv.utils.parrots_wrapper import _BatchNorm + +from annotator.uniformer.mmseg.utils import get_root_logger +from ..builder import BACKBONES + + +class GlobalContextExtractor(nn.Module): + """Global Context Extractor for CGNet. + + This class is employed to refine the joint feature of both local feature + and surrounding context. + + Args: + channel (int): Number of input feature channels. + reduction (int): Reductions for global context extractor. Default: 16. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + """ + + def __init__(self, channel, reduction=16, with_cp=False): + super(GlobalContextExtractor, self).__init__() + self.channel = channel + self.reduction = reduction + assert reduction >= 1 and channel >= reduction + self.with_cp = with_cp + self.avg_pool = nn.AdaptiveAvgPool2d(1) + self.fc = nn.Sequential( + nn.Linear(channel, channel // reduction), nn.ReLU(inplace=True), + nn.Linear(channel // reduction, channel), nn.Sigmoid()) + + def forward(self, x): + + def _inner_forward(x): + num_batch, num_channel = x.size()[:2] + y = self.avg_pool(x).view(num_batch, num_channel) + y = self.fc(y).view(num_batch, num_channel, 1, 1) + return x * y + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + return out + + +class ContextGuidedBlock(nn.Module): + """Context Guided Block for CGNet. + + This class consists of four components: local feature extractor, + surrounding feature extractor, joint feature extractor and global + context extractor. + + Args: + in_channels (int): Number of input feature channels. + out_channels (int): Number of output feature channels. + dilation (int): Dilation rate for surrounding context extractor. + Default: 2. + reduction (int): Reduction for global context extractor. Default: 16. + skip_connect (bool): Add input to output or not. Default: True. + downsample (bool): Downsample the input to 1/2 or not. Default: False. + conv_cfg (dict): Config dict for convolution layer. + Default: None, which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN', requires_grad=True). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='PReLU'). + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + """ + + def __init__(self, + in_channels, + out_channels, + dilation=2, + reduction=16, + skip_connect=True, + downsample=False, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + act_cfg=dict(type='PReLU'), + with_cp=False): + super(ContextGuidedBlock, self).__init__() + self.with_cp = with_cp + self.downsample = downsample + + channels = out_channels if downsample else out_channels // 2 + if 'type' in act_cfg and act_cfg['type'] == 'PReLU': + act_cfg['num_parameters'] = channels + kernel_size = 3 if downsample else 1 + stride = 2 if downsample else 1 + padding = (kernel_size - 1) // 2 + + self.conv1x1 = ConvModule( + in_channels, + channels, + kernel_size, + stride, + padding, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + + self.f_loc = build_conv_layer( + conv_cfg, + channels, + channels, + kernel_size=3, + padding=1, + groups=channels, + bias=False) + self.f_sur = build_conv_layer( + conv_cfg, + channels, + channels, + kernel_size=3, + padding=dilation, + groups=channels, + dilation=dilation, + bias=False) + + self.bn = build_norm_layer(norm_cfg, 2 * channels)[1] + self.activate = nn.PReLU(2 * channels) + + if downsample: + self.bottleneck = build_conv_layer( + conv_cfg, + 2 * channels, + out_channels, + kernel_size=1, + bias=False) + + self.skip_connect = skip_connect and not downsample + self.f_glo = GlobalContextExtractor(out_channels, reduction, with_cp) + + def forward(self, x): + + def _inner_forward(x): + out = self.conv1x1(x) + loc = self.f_loc(out) + sur = self.f_sur(out) + + joi_feat = torch.cat([loc, sur], 1) # the joint feature + joi_feat = self.bn(joi_feat) + joi_feat = self.activate(joi_feat) + if self.downsample: + joi_feat = self.bottleneck(joi_feat) # channel = out_channels + # f_glo is employed to refine the joint feature + out = self.f_glo(joi_feat) + + if self.skip_connect: + return x + out + else: + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + return out + + +class InputInjection(nn.Module): + """Downsampling module for CGNet.""" + + def __init__(self, num_downsampling): + super(InputInjection, self).__init__() + self.pool = nn.ModuleList() + for i in range(num_downsampling): + self.pool.append(nn.AvgPool2d(3, stride=2, padding=1)) + + def forward(self, x): + for pool in self.pool: + x = pool(x) + return x + + +@BACKBONES.register_module() +class CGNet(nn.Module): + """CGNet backbone. + + A Light-weight Context Guided Network for Semantic Segmentation + arXiv: https://arxiv.org/abs/1811.08201 + + Args: + in_channels (int): Number of input image channels. Normally 3. + num_channels (tuple[int]): Numbers of feature channels at each stages. + Default: (32, 64, 128). + num_blocks (tuple[int]): Numbers of CG blocks at stage 1 and stage 2. + Default: (3, 21). + dilations (tuple[int]): Dilation rate for surrounding context + extractors at stage 1 and stage 2. Default: (2, 4). + reductions (tuple[int]): Reductions for global context extractors at + stage 1 and stage 2. Default: (8, 16). + conv_cfg (dict): Config dict for convolution layer. + Default: None, which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN', requires_grad=True). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='PReLU'). + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. Default: False. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + """ + + def __init__(self, + in_channels=3, + num_channels=(32, 64, 128), + num_blocks=(3, 21), + dilations=(2, 4), + reductions=(8, 16), + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + act_cfg=dict(type='PReLU'), + norm_eval=False, + with_cp=False): + + super(CGNet, self).__init__() + self.in_channels = in_channels + self.num_channels = num_channels + assert isinstance(self.num_channels, tuple) and len( + self.num_channels) == 3 + self.num_blocks = num_blocks + assert isinstance(self.num_blocks, tuple) and len(self.num_blocks) == 2 + self.dilations = dilations + assert isinstance(self.dilations, tuple) and len(self.dilations) == 2 + self.reductions = reductions + assert isinstance(self.reductions, tuple) and len(self.reductions) == 2 + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + if 'type' in self.act_cfg and self.act_cfg['type'] == 'PReLU': + self.act_cfg['num_parameters'] = num_channels[0] + self.norm_eval = norm_eval + self.with_cp = with_cp + + cur_channels = in_channels + self.stem = nn.ModuleList() + for i in range(3): + self.stem.append( + ConvModule( + cur_channels, + num_channels[0], + 3, + 2 if i == 0 else 1, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + cur_channels = num_channels[0] + + self.inject_2x = InputInjection(1) # down-sample for Input, factor=2 + self.inject_4x = InputInjection(2) # down-sample for Input, factor=4 + + cur_channels += in_channels + self.norm_prelu_0 = nn.Sequential( + build_norm_layer(norm_cfg, cur_channels)[1], + nn.PReLU(cur_channels)) + + # stage 1 + self.level1 = nn.ModuleList() + for i in range(num_blocks[0]): + self.level1.append( + ContextGuidedBlock( + cur_channels if i == 0 else num_channels[1], + num_channels[1], + dilations[0], + reductions[0], + downsample=(i == 0), + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + with_cp=with_cp)) # CG block + + cur_channels = 2 * num_channels[1] + in_channels + self.norm_prelu_1 = nn.Sequential( + build_norm_layer(norm_cfg, cur_channels)[1], + nn.PReLU(cur_channels)) + + # stage 2 + self.level2 = nn.ModuleList() + for i in range(num_blocks[1]): + self.level2.append( + ContextGuidedBlock( + cur_channels if i == 0 else num_channels[2], + num_channels[2], + dilations[1], + reductions[1], + downsample=(i == 0), + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + with_cp=with_cp)) # CG block + + cur_channels = 2 * num_channels[2] + self.norm_prelu_2 = nn.Sequential( + build_norm_layer(norm_cfg, cur_channels)[1], + nn.PReLU(cur_channels)) + + def forward(self, x): + output = [] + + # stage 0 + inp_2x = self.inject_2x(x) + inp_4x = self.inject_4x(x) + for layer in self.stem: + x = layer(x) + x = self.norm_prelu_0(torch.cat([x, inp_2x], 1)) + output.append(x) + + # stage 1 + for i, layer in enumerate(self.level1): + x = layer(x) + if i == 0: + down1 = x + x = self.norm_prelu_1(torch.cat([x, down1, inp_4x], 1)) + output.append(x) + + # stage 2 + for i, layer in enumerate(self.level2): + x = layer(x) + if i == 0: + down2 = x + x = self.norm_prelu_2(torch.cat([down2, x], 1)) + output.append(x) + + return output + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, (nn.Conv2d, nn.Linear)): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + elif isinstance(m, nn.PReLU): + constant_init(m, 0) + else: + raise TypeError('pretrained must be a str or None') + + def train(self, mode=True): + """Convert the model into training mode will keeping the normalization + layer freezed.""" + super(CGNet, self).train(mode) + if mode and self.norm_eval: + for m in self.modules(): + # trick: eval have effect on BatchNorm only + if isinstance(m, _BatchNorm): + m.eval() diff --git a/annotator/uniformer/mmseg/models/backbones/fast_scnn.py b/annotator/uniformer/mmseg/models/backbones/fast_scnn.py new file mode 100644 index 0000000000000000000000000000000000000000..38c2350177cbc2066f45add568d30eb6041f74f3 --- /dev/null +++ b/annotator/uniformer/mmseg/models/backbones/fast_scnn.py @@ -0,0 +1,375 @@ +import torch +import torch.nn as nn +from annotator.uniformer.mmcv.cnn import (ConvModule, DepthwiseSeparableConvModule, constant_init, + kaiming_init) +from torch.nn.modules.batchnorm import _BatchNorm + +from annotator.uniformer.mmseg.models.decode_heads.psp_head import PPM +from annotator.uniformer.mmseg.ops import resize +from ..builder import BACKBONES +from ..utils.inverted_residual import InvertedResidual + + +class LearningToDownsample(nn.Module): + """Learning to downsample module. + + Args: + in_channels (int): Number of input channels. + dw_channels (tuple[int]): Number of output channels of the first and + the second depthwise conv (dwconv) layers. + out_channels (int): Number of output channels of the whole + 'learning to downsample' module. + conv_cfg (dict | None): Config of conv layers. Default: None + norm_cfg (dict | None): Config of norm layers. Default: + dict(type='BN') + act_cfg (dict): Config of activation layers. Default: + dict(type='ReLU') + """ + + def __init__(self, + in_channels, + dw_channels, + out_channels, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU')): + super(LearningToDownsample, self).__init__() + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + dw_channels1 = dw_channels[0] + dw_channels2 = dw_channels[1] + + self.conv = ConvModule( + in_channels, + dw_channels1, + 3, + stride=2, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.dsconv1 = DepthwiseSeparableConvModule( + dw_channels1, + dw_channels2, + kernel_size=3, + stride=2, + padding=1, + norm_cfg=self.norm_cfg) + self.dsconv2 = DepthwiseSeparableConvModule( + dw_channels2, + out_channels, + kernel_size=3, + stride=2, + padding=1, + norm_cfg=self.norm_cfg) + + def forward(self, x): + x = self.conv(x) + x = self.dsconv1(x) + x = self.dsconv2(x) + return x + + +class GlobalFeatureExtractor(nn.Module): + """Global feature extractor module. + + Args: + in_channels (int): Number of input channels of the GFE module. + Default: 64 + block_channels (tuple[int]): Tuple of ints. Each int specifies the + number of output channels of each Inverted Residual module. + Default: (64, 96, 128) + out_channels(int): Number of output channels of the GFE module. + Default: 128 + expand_ratio (int): Adjusts number of channels of the hidden layer + in InvertedResidual by this amount. + Default: 6 + num_blocks (tuple[int]): Tuple of ints. Each int specifies the + number of times each Inverted Residual module is repeated. + The repeated Inverted Residual modules are called a 'group'. + Default: (3, 3, 3) + strides (tuple[int]): Tuple of ints. Each int specifies + the downsampling factor of each 'group'. + Default: (2, 2, 1) + pool_scales (tuple[int]): Tuple of ints. Each int specifies + the parameter required in 'global average pooling' within PPM. + Default: (1, 2, 3, 6) + conv_cfg (dict | None): Config of conv layers. Default: None + norm_cfg (dict | None): Config of norm layers. Default: + dict(type='BN') + act_cfg (dict): Config of activation layers. Default: + dict(type='ReLU') + align_corners (bool): align_corners argument of F.interpolate. + Default: False + """ + + def __init__(self, + in_channels=64, + block_channels=(64, 96, 128), + out_channels=128, + expand_ratio=6, + num_blocks=(3, 3, 3), + strides=(2, 2, 1), + pool_scales=(1, 2, 3, 6), + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + align_corners=False): + super(GlobalFeatureExtractor, self).__init__() + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + assert len(block_channels) == len(num_blocks) == 3 + self.bottleneck1 = self._make_layer(in_channels, block_channels[0], + num_blocks[0], strides[0], + expand_ratio) + self.bottleneck2 = self._make_layer(block_channels[0], + block_channels[1], num_blocks[1], + strides[1], expand_ratio) + self.bottleneck3 = self._make_layer(block_channels[1], + block_channels[2], num_blocks[2], + strides[2], expand_ratio) + self.ppm = PPM( + pool_scales, + block_channels[2], + block_channels[2] // 4, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + align_corners=align_corners) + self.out = ConvModule( + block_channels[2] * 2, + out_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def _make_layer(self, + in_channels, + out_channels, + blocks, + stride=1, + expand_ratio=6): + layers = [ + InvertedResidual( + in_channels, + out_channels, + stride, + expand_ratio, + norm_cfg=self.norm_cfg) + ] + for i in range(1, blocks): + layers.append( + InvertedResidual( + out_channels, + out_channels, + 1, + expand_ratio, + norm_cfg=self.norm_cfg)) + return nn.Sequential(*layers) + + def forward(self, x): + x = self.bottleneck1(x) + x = self.bottleneck2(x) + x = self.bottleneck3(x) + x = torch.cat([x, *self.ppm(x)], dim=1) + x = self.out(x) + return x + + +class FeatureFusionModule(nn.Module): + """Feature fusion module. + + Args: + higher_in_channels (int): Number of input channels of the + higher-resolution branch. + lower_in_channels (int): Number of input channels of the + lower-resolution branch. + out_channels (int): Number of output channels. + conv_cfg (dict | None): Config of conv layers. Default: None + norm_cfg (dict | None): Config of norm layers. Default: + dict(type='BN') + act_cfg (dict): Config of activation layers. Default: + dict(type='ReLU') + align_corners (bool): align_corners argument of F.interpolate. + Default: False + """ + + def __init__(self, + higher_in_channels, + lower_in_channels, + out_channels, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + align_corners=False): + super(FeatureFusionModule, self).__init__() + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.align_corners = align_corners + self.dwconv = ConvModule( + lower_in_channels, + out_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.conv_lower_res = ConvModule( + out_channels, + out_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=None) + self.conv_higher_res = ConvModule( + higher_in_channels, + out_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=None) + self.relu = nn.ReLU(True) + + def forward(self, higher_res_feature, lower_res_feature): + lower_res_feature = resize( + lower_res_feature, + size=higher_res_feature.size()[2:], + mode='bilinear', + align_corners=self.align_corners) + lower_res_feature = self.dwconv(lower_res_feature) + lower_res_feature = self.conv_lower_res(lower_res_feature) + + higher_res_feature = self.conv_higher_res(higher_res_feature) + out = higher_res_feature + lower_res_feature + return self.relu(out) + + +@BACKBONES.register_module() +class FastSCNN(nn.Module): + """Fast-SCNN Backbone. + + Args: + in_channels (int): Number of input image channels. Default: 3. + downsample_dw_channels (tuple[int]): Number of output channels after + the first conv layer & the second conv layer in + Learning-To-Downsample (LTD) module. + Default: (32, 48). + global_in_channels (int): Number of input channels of + Global Feature Extractor(GFE). + Equal to number of output channels of LTD. + Default: 64. + global_block_channels (tuple[int]): Tuple of integers that describe + the output channels for each of the MobileNet-v2 bottleneck + residual blocks in GFE. + Default: (64, 96, 128). + global_block_strides (tuple[int]): Tuple of integers + that describe the strides (downsampling factors) for each of the + MobileNet-v2 bottleneck residual blocks in GFE. + Default: (2, 2, 1). + global_out_channels (int): Number of output channels of GFE. + Default: 128. + higher_in_channels (int): Number of input channels of the higher + resolution branch in FFM. + Equal to global_in_channels. + Default: 64. + lower_in_channels (int): Number of input channels of the lower + resolution branch in FFM. + Equal to global_out_channels. + Default: 128. + fusion_out_channels (int): Number of output channels of FFM. + Default: 128. + out_indices (tuple): Tuple of indices of list + [higher_res_features, lower_res_features, fusion_output]. + Often set to (0,1,2) to enable aux. heads. + Default: (0, 1, 2). + conv_cfg (dict | None): Config of conv layers. Default: None + norm_cfg (dict | None): Config of norm layers. Default: + dict(type='BN') + act_cfg (dict): Config of activation layers. Default: + dict(type='ReLU') + align_corners (bool): align_corners argument of F.interpolate. + Default: False + """ + + def __init__(self, + in_channels=3, + downsample_dw_channels=(32, 48), + global_in_channels=64, + global_block_channels=(64, 96, 128), + global_block_strides=(2, 2, 1), + global_out_channels=128, + higher_in_channels=64, + lower_in_channels=128, + fusion_out_channels=128, + out_indices=(0, 1, 2), + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + align_corners=False): + + super(FastSCNN, self).__init__() + if global_in_channels != higher_in_channels: + raise AssertionError('Global Input Channels must be the same \ + with Higher Input Channels!') + elif global_out_channels != lower_in_channels: + raise AssertionError('Global Output Channels must be the same \ + with Lower Input Channels!') + + self.in_channels = in_channels + self.downsample_dw_channels1 = downsample_dw_channels[0] + self.downsample_dw_channels2 = downsample_dw_channels[1] + self.global_in_channels = global_in_channels + self.global_block_channels = global_block_channels + self.global_block_strides = global_block_strides + self.global_out_channels = global_out_channels + self.higher_in_channels = higher_in_channels + self.lower_in_channels = lower_in_channels + self.fusion_out_channels = fusion_out_channels + self.out_indices = out_indices + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.align_corners = align_corners + self.learning_to_downsample = LearningToDownsample( + in_channels, + downsample_dw_channels, + global_in_channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.global_feature_extractor = GlobalFeatureExtractor( + global_in_channels, + global_block_channels, + global_out_channels, + strides=self.global_block_strides, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + align_corners=self.align_corners) + self.feature_fusion = FeatureFusionModule( + higher_in_channels, + lower_in_channels, + fusion_out_channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + align_corners=self.align_corners) + + def init_weights(self, pretrained=None): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + + def forward(self, x): + higher_res_features = self.learning_to_downsample(x) + lower_res_features = self.global_feature_extractor(higher_res_features) + fusion_output = self.feature_fusion(higher_res_features, + lower_res_features) + + outs = [higher_res_features, lower_res_features, fusion_output] + outs = [outs[i] for i in self.out_indices] + return tuple(outs) diff --git a/annotator/uniformer/mmseg/models/backbones/hrnet.py b/annotator/uniformer/mmseg/models/backbones/hrnet.py new file mode 100644 index 0000000000000000000000000000000000000000..331ebf3ccb8597b3f507670753789073fc3c946d --- /dev/null +++ b/annotator/uniformer/mmseg/models/backbones/hrnet.py @@ -0,0 +1,555 @@ +import torch.nn as nn +from annotator.uniformer.mmcv.cnn import (build_conv_layer, build_norm_layer, constant_init, + kaiming_init) +from annotator.uniformer.mmcv.runner import load_checkpoint +from annotator.uniformer.mmcv.utils.parrots_wrapper import _BatchNorm + +from annotator.uniformer.mmseg.ops import Upsample, resize +from annotator.uniformer.mmseg.utils import get_root_logger +from ..builder import BACKBONES +from .resnet import BasicBlock, Bottleneck + + +class HRModule(nn.Module): + """High-Resolution Module for HRNet. + + In this module, every branch has 4 BasicBlocks/Bottlenecks. Fusion/Exchange + is in this module. + """ + + def __init__(self, + num_branches, + blocks, + num_blocks, + in_channels, + num_channels, + multiscale_output=True, + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True)): + super(HRModule, self).__init__() + self._check_branches(num_branches, num_blocks, in_channels, + num_channels) + + self.in_channels = in_channels + self.num_branches = num_branches + + self.multiscale_output = multiscale_output + self.norm_cfg = norm_cfg + self.conv_cfg = conv_cfg + self.with_cp = with_cp + self.branches = self._make_branches(num_branches, blocks, num_blocks, + num_channels) + self.fuse_layers = self._make_fuse_layers() + self.relu = nn.ReLU(inplace=False) + + def _check_branches(self, num_branches, num_blocks, in_channels, + num_channels): + """Check branches configuration.""" + if num_branches != len(num_blocks): + error_msg = f'NUM_BRANCHES({num_branches}) <> NUM_BLOCKS(' \ + f'{len(num_blocks)})' + raise ValueError(error_msg) + + if num_branches != len(num_channels): + error_msg = f'NUM_BRANCHES({num_branches}) <> NUM_CHANNELS(' \ + f'{len(num_channels)})' + raise ValueError(error_msg) + + if num_branches != len(in_channels): + error_msg = f'NUM_BRANCHES({num_branches}) <> NUM_INCHANNELS(' \ + f'{len(in_channels)})' + raise ValueError(error_msg) + + def _make_one_branch(self, + branch_index, + block, + num_blocks, + num_channels, + stride=1): + """Build one branch.""" + downsample = None + if stride != 1 or \ + self.in_channels[branch_index] != \ + num_channels[branch_index] * block.expansion: + downsample = nn.Sequential( + build_conv_layer( + self.conv_cfg, + self.in_channels[branch_index], + num_channels[branch_index] * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + build_norm_layer(self.norm_cfg, num_channels[branch_index] * + block.expansion)[1]) + + layers = [] + layers.append( + block( + self.in_channels[branch_index], + num_channels[branch_index], + stride, + downsample=downsample, + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + self.in_channels[branch_index] = \ + num_channels[branch_index] * block.expansion + for i in range(1, num_blocks[branch_index]): + layers.append( + block( + self.in_channels[branch_index], + num_channels[branch_index], + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + + return nn.Sequential(*layers) + + def _make_branches(self, num_branches, block, num_blocks, num_channels): + """Build multiple branch.""" + branches = [] + + for i in range(num_branches): + branches.append( + self._make_one_branch(i, block, num_blocks, num_channels)) + + return nn.ModuleList(branches) + + def _make_fuse_layers(self): + """Build fuse layer.""" + if self.num_branches == 1: + return None + + num_branches = self.num_branches + in_channels = self.in_channels + fuse_layers = [] + num_out_branches = num_branches if self.multiscale_output else 1 + for i in range(num_out_branches): + fuse_layer = [] + for j in range(num_branches): + if j > i: + fuse_layer.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels[j], + in_channels[i], + kernel_size=1, + stride=1, + padding=0, + bias=False), + build_norm_layer(self.norm_cfg, in_channels[i])[1], + # we set align_corners=False for HRNet + Upsample( + scale_factor=2**(j - i), + mode='bilinear', + align_corners=False))) + elif j == i: + fuse_layer.append(None) + else: + conv_downsamples = [] + for k in range(i - j): + if k == i - j - 1: + conv_downsamples.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels[j], + in_channels[i], + kernel_size=3, + stride=2, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, + in_channels[i])[1])) + else: + conv_downsamples.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels[j], + in_channels[j], + kernel_size=3, + stride=2, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, + in_channels[j])[1], + nn.ReLU(inplace=False))) + fuse_layer.append(nn.Sequential(*conv_downsamples)) + fuse_layers.append(nn.ModuleList(fuse_layer)) + + return nn.ModuleList(fuse_layers) + + def forward(self, x): + """Forward function.""" + if self.num_branches == 1: + return [self.branches[0](x[0])] + + for i in range(self.num_branches): + x[i] = self.branches[i](x[i]) + + x_fuse = [] + for i in range(len(self.fuse_layers)): + y = 0 + for j in range(self.num_branches): + if i == j: + y += x[j] + elif j > i: + y = y + resize( + self.fuse_layers[i][j](x[j]), + size=x[i].shape[2:], + mode='bilinear', + align_corners=False) + else: + y += self.fuse_layers[i][j](x[j]) + x_fuse.append(self.relu(y)) + return x_fuse + + +@BACKBONES.register_module() +class HRNet(nn.Module): + """HRNet backbone. + + High-Resolution Representations for Labeling Pixels and Regions + arXiv: https://arxiv.org/abs/1904.04514 + + Args: + extra (dict): detailed configuration for each stage of HRNet. + in_channels (int): Number of input image channels. Normally 3. + conv_cfg (dict): dictionary to construct and config conv layer. + norm_cfg (dict): dictionary to construct and config norm layer. + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. + zero_init_residual (bool): whether to use zero init for last norm layer + in resblocks to let them behave as identity. + + Example: + >>> from annotator.uniformer.mmseg.models import HRNet + >>> import torch + >>> extra = dict( + >>> stage1=dict( + >>> num_modules=1, + >>> num_branches=1, + >>> block='BOTTLENECK', + >>> num_blocks=(4, ), + >>> num_channels=(64, )), + >>> stage2=dict( + >>> num_modules=1, + >>> num_branches=2, + >>> block='BASIC', + >>> num_blocks=(4, 4), + >>> num_channels=(32, 64)), + >>> stage3=dict( + >>> num_modules=4, + >>> num_branches=3, + >>> block='BASIC', + >>> num_blocks=(4, 4, 4), + >>> num_channels=(32, 64, 128)), + >>> stage4=dict( + >>> num_modules=3, + >>> num_branches=4, + >>> block='BASIC', + >>> num_blocks=(4, 4, 4, 4), + >>> num_channels=(32, 64, 128, 256))) + >>> self = HRNet(extra, in_channels=1) + >>> self.eval() + >>> inputs = torch.rand(1, 1, 32, 32) + >>> level_outputs = self.forward(inputs) + >>> for level_out in level_outputs: + ... print(tuple(level_out.shape)) + (1, 32, 8, 8) + (1, 64, 4, 4) + (1, 128, 2, 2) + (1, 256, 1, 1) + """ + + blocks_dict = {'BASIC': BasicBlock, 'BOTTLENECK': Bottleneck} + + def __init__(self, + extra, + in_channels=3, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + norm_eval=False, + with_cp=False, + zero_init_residual=False): + super(HRNet, self).__init__() + self.extra = extra + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.norm_eval = norm_eval + self.with_cp = with_cp + self.zero_init_residual = zero_init_residual + + # stem net + self.norm1_name, norm1 = build_norm_layer(self.norm_cfg, 64, postfix=1) + self.norm2_name, norm2 = build_norm_layer(self.norm_cfg, 64, postfix=2) + + self.conv1 = build_conv_layer( + self.conv_cfg, + in_channels, + 64, + kernel_size=3, + stride=2, + padding=1, + bias=False) + + self.add_module(self.norm1_name, norm1) + self.conv2 = build_conv_layer( + self.conv_cfg, + 64, + 64, + kernel_size=3, + stride=2, + padding=1, + bias=False) + + self.add_module(self.norm2_name, norm2) + self.relu = nn.ReLU(inplace=True) + + # stage 1 + self.stage1_cfg = self.extra['stage1'] + num_channels = self.stage1_cfg['num_channels'][0] + block_type = self.stage1_cfg['block'] + num_blocks = self.stage1_cfg['num_blocks'][0] + + block = self.blocks_dict[block_type] + stage1_out_channels = num_channels * block.expansion + self.layer1 = self._make_layer(block, 64, num_channels, num_blocks) + + # stage 2 + self.stage2_cfg = self.extra['stage2'] + num_channels = self.stage2_cfg['num_channels'] + block_type = self.stage2_cfg['block'] + + block = self.blocks_dict[block_type] + num_channels = [channel * block.expansion for channel in num_channels] + self.transition1 = self._make_transition_layer([stage1_out_channels], + num_channels) + self.stage2, pre_stage_channels = self._make_stage( + self.stage2_cfg, num_channels) + + # stage 3 + self.stage3_cfg = self.extra['stage3'] + num_channels = self.stage3_cfg['num_channels'] + block_type = self.stage3_cfg['block'] + + block = self.blocks_dict[block_type] + num_channels = [channel * block.expansion for channel in num_channels] + self.transition2 = self._make_transition_layer(pre_stage_channels, + num_channels) + self.stage3, pre_stage_channels = self._make_stage( + self.stage3_cfg, num_channels) + + # stage 4 + self.stage4_cfg = self.extra['stage4'] + num_channels = self.stage4_cfg['num_channels'] + block_type = self.stage4_cfg['block'] + + block = self.blocks_dict[block_type] + num_channels = [channel * block.expansion for channel in num_channels] + self.transition3 = self._make_transition_layer(pre_stage_channels, + num_channels) + self.stage4, pre_stage_channels = self._make_stage( + self.stage4_cfg, num_channels) + + @property + def norm1(self): + """nn.Module: the normalization layer named "norm1" """ + return getattr(self, self.norm1_name) + + @property + def norm2(self): + """nn.Module: the normalization layer named "norm2" """ + return getattr(self, self.norm2_name) + + def _make_transition_layer(self, num_channels_pre_layer, + num_channels_cur_layer): + """Make transition layer.""" + num_branches_cur = len(num_channels_cur_layer) + num_branches_pre = len(num_channels_pre_layer) + + transition_layers = [] + for i in range(num_branches_cur): + if i < num_branches_pre: + if num_channels_cur_layer[i] != num_channels_pre_layer[i]: + transition_layers.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + num_channels_pre_layer[i], + num_channels_cur_layer[i], + kernel_size=3, + stride=1, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, + num_channels_cur_layer[i])[1], + nn.ReLU(inplace=True))) + else: + transition_layers.append(None) + else: + conv_downsamples = [] + for j in range(i + 1 - num_branches_pre): + in_channels = num_channels_pre_layer[-1] + out_channels = num_channels_cur_layer[i] \ + if j == i - num_branches_pre else in_channels + conv_downsamples.append( + nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels, + out_channels, + kernel_size=3, + stride=2, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, out_channels)[1], + nn.ReLU(inplace=True))) + transition_layers.append(nn.Sequential(*conv_downsamples)) + + return nn.ModuleList(transition_layers) + + def _make_layer(self, block, inplanes, planes, blocks, stride=1): + """Make each layer.""" + downsample = None + if stride != 1 or inplanes != planes * block.expansion: + downsample = nn.Sequential( + build_conv_layer( + self.conv_cfg, + inplanes, + planes * block.expansion, + kernel_size=1, + stride=stride, + bias=False), + build_norm_layer(self.norm_cfg, planes * block.expansion)[1]) + + layers = [] + layers.append( + block( + inplanes, + planes, + stride, + downsample=downsample, + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + inplanes = planes * block.expansion + for i in range(1, blocks): + layers.append( + block( + inplanes, + planes, + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + + return nn.Sequential(*layers) + + def _make_stage(self, layer_config, in_channels, multiscale_output=True): + """Make each stage.""" + num_modules = layer_config['num_modules'] + num_branches = layer_config['num_branches'] + num_blocks = layer_config['num_blocks'] + num_channels = layer_config['num_channels'] + block = self.blocks_dict[layer_config['block']] + + hr_modules = [] + for i in range(num_modules): + # multi_scale_output is only used for the last module + if not multiscale_output and i == num_modules - 1: + reset_multiscale_output = False + else: + reset_multiscale_output = True + + hr_modules.append( + HRModule( + num_branches, + block, + num_blocks, + in_channels, + num_channels, + reset_multiscale_output, + with_cp=self.with_cp, + norm_cfg=self.norm_cfg, + conv_cfg=self.conv_cfg)) + + return nn.Sequential(*hr_modules), in_channels + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + + if self.zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + constant_init(m.norm3, 0) + elif isinstance(m, BasicBlock): + constant_init(m.norm2, 0) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + """Forward function.""" + + x = self.conv1(x) + x = self.norm1(x) + x = self.relu(x) + x = self.conv2(x) + x = self.norm2(x) + x = self.relu(x) + x = self.layer1(x) + + x_list = [] + for i in range(self.stage2_cfg['num_branches']): + if self.transition1[i] is not None: + x_list.append(self.transition1[i](x)) + else: + x_list.append(x) + y_list = self.stage2(x_list) + + x_list = [] + for i in range(self.stage3_cfg['num_branches']): + if self.transition2[i] is not None: + x_list.append(self.transition2[i](y_list[-1])) + else: + x_list.append(y_list[i]) + y_list = self.stage3(x_list) + + x_list = [] + for i in range(self.stage4_cfg['num_branches']): + if self.transition3[i] is not None: + x_list.append(self.transition3[i](y_list[-1])) + else: + x_list.append(y_list[i]) + y_list = self.stage4(x_list) + + return y_list + + def train(self, mode=True): + """Convert the model into training mode will keeping the normalization + layer freezed.""" + super(HRNet, self).train(mode) + if mode and self.norm_eval: + for m in self.modules(): + # trick: eval have effect on BatchNorm only + if isinstance(m, _BatchNorm): + m.eval() diff --git a/annotator/uniformer/mmseg/models/backbones/mobilenet_v2.py b/annotator/uniformer/mmseg/models/backbones/mobilenet_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..ab6b3791692a0d1b5da3601875711710b7bd01ba --- /dev/null +++ b/annotator/uniformer/mmseg/models/backbones/mobilenet_v2.py @@ -0,0 +1,180 @@ +import logging + +import torch.nn as nn +from annotator.uniformer.mmcv.cnn import ConvModule, constant_init, kaiming_init +from annotator.uniformer.mmcv.runner import load_checkpoint +from torch.nn.modules.batchnorm import _BatchNorm + +from ..builder import BACKBONES +from ..utils import InvertedResidual, make_divisible + + +@BACKBONES.register_module() +class MobileNetV2(nn.Module): + """MobileNetV2 backbone. + + Args: + widen_factor (float): Width multiplier, multiply number of + channels in each layer by this amount. Default: 1.0. + strides (Sequence[int], optional): Strides of the first block of each + layer. If not specified, default config in ``arch_setting`` will + be used. + dilations (Sequence[int]): Dilation of each layer. + out_indices (None or Sequence[int]): Output from which stages. + Default: (7, ). + frozen_stages (int): Stages to be frozen (all param fixed). + Default: -1, which means not freezing any parameters. + conv_cfg (dict): Config dict for convolution layer. + Default: None, which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='ReLU6'). + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. Default: False. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + """ + + # Parameters to build layers. 3 parameters are needed to construct a + # layer, from left to right: expand_ratio, channel, num_blocks. + arch_settings = [[1, 16, 1], [6, 24, 2], [6, 32, 3], [6, 64, 4], + [6, 96, 3], [6, 160, 3], [6, 320, 1]] + + def __init__(self, + widen_factor=1., + strides=(1, 2, 2, 2, 1, 2, 1), + dilations=(1, 1, 1, 1, 1, 1, 1), + out_indices=(1, 2, 4, 6), + frozen_stages=-1, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU6'), + norm_eval=False, + with_cp=False): + super(MobileNetV2, self).__init__() + self.widen_factor = widen_factor + self.strides = strides + self.dilations = dilations + assert len(strides) == len(dilations) == len(self.arch_settings) + self.out_indices = out_indices + for index in out_indices: + if index not in range(0, 7): + raise ValueError('the item in out_indices must in ' + f'range(0, 8). But received {index}') + + if frozen_stages not in range(-1, 7): + raise ValueError('frozen_stages must be in range(-1, 7). ' + f'But received {frozen_stages}') + self.out_indices = out_indices + self.frozen_stages = frozen_stages + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.norm_eval = norm_eval + self.with_cp = with_cp + + self.in_channels = make_divisible(32 * widen_factor, 8) + + self.conv1 = ConvModule( + in_channels=3, + out_channels=self.in_channels, + kernel_size=3, + stride=2, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + self.layers = [] + + for i, layer_cfg in enumerate(self.arch_settings): + expand_ratio, channel, num_blocks = layer_cfg + stride = self.strides[i] + dilation = self.dilations[i] + out_channels = make_divisible(channel * widen_factor, 8) + inverted_res_layer = self.make_layer( + out_channels=out_channels, + num_blocks=num_blocks, + stride=stride, + dilation=dilation, + expand_ratio=expand_ratio) + layer_name = f'layer{i + 1}' + self.add_module(layer_name, inverted_res_layer) + self.layers.append(layer_name) + + def make_layer(self, out_channels, num_blocks, stride, dilation, + expand_ratio): + """Stack InvertedResidual blocks to build a layer for MobileNetV2. + + Args: + out_channels (int): out_channels of block. + num_blocks (int): Number of blocks. + stride (int): Stride of the first block. + dilation (int): Dilation of the first block. + expand_ratio (int): Expand the number of channels of the + hidden layer in InvertedResidual by this ratio. + """ + layers = [] + for i in range(num_blocks): + layers.append( + InvertedResidual( + self.in_channels, + out_channels, + stride if i == 0 else 1, + expand_ratio=expand_ratio, + dilation=dilation if i == 0 else 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + with_cp=self.with_cp)) + self.in_channels = out_channels + + return nn.Sequential(*layers) + + def init_weights(self, pretrained=None): + if isinstance(pretrained, str): + logger = logging.getLogger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + x = self.conv1(x) + + outs = [] + for i, layer_name in enumerate(self.layers): + layer = getattr(self, layer_name) + x = layer(x) + if i in self.out_indices: + outs.append(x) + + if len(outs) == 1: + return outs[0] + else: + return tuple(outs) + + def _freeze_stages(self): + if self.frozen_stages >= 0: + for param in self.conv1.parameters(): + param.requires_grad = False + for i in range(1, self.frozen_stages + 1): + layer = getattr(self, f'layer{i}') + layer.eval() + for param in layer.parameters(): + param.requires_grad = False + + def train(self, mode=True): + super(MobileNetV2, self).train(mode) + self._freeze_stages() + if mode and self.norm_eval: + for m in self.modules(): + if isinstance(m, _BatchNorm): + m.eval() diff --git a/annotator/uniformer/mmseg/models/backbones/mobilenet_v3.py b/annotator/uniformer/mmseg/models/backbones/mobilenet_v3.py new file mode 100644 index 0000000000000000000000000000000000000000..16817400b4102899794fe64c9644713a4e54e2f9 --- /dev/null +++ b/annotator/uniformer/mmseg/models/backbones/mobilenet_v3.py @@ -0,0 +1,255 @@ +import logging + +import annotator.uniformer.mmcv as mmcv +import torch.nn as nn +from annotator.uniformer.mmcv.cnn import ConvModule, constant_init, kaiming_init +from annotator.uniformer.mmcv.cnn.bricks import Conv2dAdaptivePadding +from annotator.uniformer.mmcv.runner import load_checkpoint +from torch.nn.modules.batchnorm import _BatchNorm + +from ..builder import BACKBONES +from ..utils import InvertedResidualV3 as InvertedResidual + + +@BACKBONES.register_module() +class MobileNetV3(nn.Module): + """MobileNetV3 backbone. + + This backbone is the improved implementation of `Searching for MobileNetV3 + `_. + + Args: + arch (str): Architecture of mobilnetv3, from {'small', 'large'}. + Default: 'small'. + conv_cfg (dict): Config dict for convolution layer. + Default: None, which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN'). + out_indices (tuple[int]): Output from which layer. + Default: (0, 1, 12). + frozen_stages (int): Stages to be frozen (all param fixed). + Default: -1, which means not freezing any parameters. + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. Default: False. + with_cp (bool): Use checkpoint or not. Using checkpoint will save + some memory while slowing down the training speed. + Default: False. + """ + # Parameters to build each block: + # [kernel size, mid channels, out channels, with_se, act type, stride] + arch_settings = { + 'small': [[3, 16, 16, True, 'ReLU', 2], # block0 layer1 os=4 + [3, 72, 24, False, 'ReLU', 2], # block1 layer2 os=8 + [3, 88, 24, False, 'ReLU', 1], + [5, 96, 40, True, 'HSwish', 2], # block2 layer4 os=16 + [5, 240, 40, True, 'HSwish', 1], + [5, 240, 40, True, 'HSwish', 1], + [5, 120, 48, True, 'HSwish', 1], # block3 layer7 os=16 + [5, 144, 48, True, 'HSwish', 1], + [5, 288, 96, True, 'HSwish', 2], # block4 layer9 os=32 + [5, 576, 96, True, 'HSwish', 1], + [5, 576, 96, True, 'HSwish', 1]], + 'large': [[3, 16, 16, False, 'ReLU', 1], # block0 layer1 os=2 + [3, 64, 24, False, 'ReLU', 2], # block1 layer2 os=4 + [3, 72, 24, False, 'ReLU', 1], + [5, 72, 40, True, 'ReLU', 2], # block2 layer4 os=8 + [5, 120, 40, True, 'ReLU', 1], + [5, 120, 40, True, 'ReLU', 1], + [3, 240, 80, False, 'HSwish', 2], # block3 layer7 os=16 + [3, 200, 80, False, 'HSwish', 1], + [3, 184, 80, False, 'HSwish', 1], + [3, 184, 80, False, 'HSwish', 1], + [3, 480, 112, True, 'HSwish', 1], # block4 layer11 os=16 + [3, 672, 112, True, 'HSwish', 1], + [5, 672, 160, True, 'HSwish', 2], # block5 layer13 os=32 + [5, 960, 160, True, 'HSwish', 1], + [5, 960, 160, True, 'HSwish', 1]] + } # yapf: disable + + def __init__(self, + arch='small', + conv_cfg=None, + norm_cfg=dict(type='BN'), + out_indices=(0, 1, 12), + frozen_stages=-1, + reduction_factor=1, + norm_eval=False, + with_cp=False): + super(MobileNetV3, self).__init__() + assert arch in self.arch_settings + assert isinstance(reduction_factor, int) and reduction_factor > 0 + assert mmcv.is_tuple_of(out_indices, int) + for index in out_indices: + if index not in range(0, len(self.arch_settings[arch]) + 2): + raise ValueError( + 'the item in out_indices must in ' + f'range(0, {len(self.arch_settings[arch])+2}). ' + f'But received {index}') + + if frozen_stages not in range(-1, len(self.arch_settings[arch]) + 2): + raise ValueError('frozen_stages must be in range(-1, ' + f'{len(self.arch_settings[arch])+2}). ' + f'But received {frozen_stages}') + self.arch = arch + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.out_indices = out_indices + self.frozen_stages = frozen_stages + self.reduction_factor = reduction_factor + self.norm_eval = norm_eval + self.with_cp = with_cp + self.layers = self._make_layer() + + def _make_layer(self): + layers = [] + + # build the first layer (layer0) + in_channels = 16 + layer = ConvModule( + in_channels=3, + out_channels=in_channels, + kernel_size=3, + stride=2, + padding=1, + conv_cfg=dict(type='Conv2dAdaptivePadding'), + norm_cfg=self.norm_cfg, + act_cfg=dict(type='HSwish')) + self.add_module('layer0', layer) + layers.append('layer0') + + layer_setting = self.arch_settings[self.arch] + for i, params in enumerate(layer_setting): + (kernel_size, mid_channels, out_channels, with_se, act, + stride) = params + + if self.arch == 'large' and i >= 12 or self.arch == 'small' and \ + i >= 8: + mid_channels = mid_channels // self.reduction_factor + out_channels = out_channels // self.reduction_factor + + if with_se: + se_cfg = dict( + channels=mid_channels, + ratio=4, + act_cfg=(dict(type='ReLU'), + dict(type='HSigmoid', bias=3.0, divisor=6.0))) + else: + se_cfg = None + + layer = InvertedResidual( + in_channels=in_channels, + out_channels=out_channels, + mid_channels=mid_channels, + kernel_size=kernel_size, + stride=stride, + se_cfg=se_cfg, + with_expand_conv=(in_channels != mid_channels), + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=dict(type=act), + with_cp=self.with_cp) + in_channels = out_channels + layer_name = 'layer{}'.format(i + 1) + self.add_module(layer_name, layer) + layers.append(layer_name) + + # build the last layer + # block5 layer12 os=32 for small model + # block6 layer16 os=32 for large model + layer = ConvModule( + in_channels=in_channels, + out_channels=576 if self.arch == 'small' else 960, + kernel_size=1, + stride=1, + dilation=4, + padding=0, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=dict(type='HSwish')) + layer_name = 'layer{}'.format(len(layer_setting) + 1) + self.add_module(layer_name, layer) + layers.append(layer_name) + + # next, convert backbone MobileNetV3 to a semantic segmentation version + if self.arch == 'small': + self.layer4.depthwise_conv.conv.stride = (1, 1) + self.layer9.depthwise_conv.conv.stride = (1, 1) + for i in range(4, len(layers)): + layer = getattr(self, layers[i]) + if isinstance(layer, InvertedResidual): + modified_module = layer.depthwise_conv.conv + else: + modified_module = layer.conv + + if i < 9: + modified_module.dilation = (2, 2) + pad = 2 + else: + modified_module.dilation = (4, 4) + pad = 4 + + if not isinstance(modified_module, Conv2dAdaptivePadding): + # Adjust padding + pad *= (modified_module.kernel_size[0] - 1) // 2 + modified_module.padding = (pad, pad) + else: + self.layer7.depthwise_conv.conv.stride = (1, 1) + self.layer13.depthwise_conv.conv.stride = (1, 1) + for i in range(7, len(layers)): + layer = getattr(self, layers[i]) + if isinstance(layer, InvertedResidual): + modified_module = layer.depthwise_conv.conv + else: + modified_module = layer.conv + + if i < 13: + modified_module.dilation = (2, 2) + pad = 2 + else: + modified_module.dilation = (4, 4) + pad = 4 + + if not isinstance(modified_module, Conv2dAdaptivePadding): + # Adjust padding + pad *= (modified_module.kernel_size[0] - 1) // 2 + modified_module.padding = (pad, pad) + + return layers + + def init_weights(self, pretrained=None): + if isinstance(pretrained, str): + logger = logging.getLogger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, nn.BatchNorm2d): + constant_init(m, 1) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + outs = [] + for i, layer_name in enumerate(self.layers): + layer = getattr(self, layer_name) + x = layer(x) + if i in self.out_indices: + outs.append(x) + return outs + + def _freeze_stages(self): + for i in range(self.frozen_stages + 1): + layer = getattr(self, f'layer{i}') + layer.eval() + for param in layer.parameters(): + param.requires_grad = False + + def train(self, mode=True): + super(MobileNetV3, self).train(mode) + self._freeze_stages() + if mode and self.norm_eval: + for m in self.modules(): + if isinstance(m, _BatchNorm): + m.eval() diff --git a/annotator/uniformer/mmseg/models/backbones/resnest.py b/annotator/uniformer/mmseg/models/backbones/resnest.py new file mode 100644 index 0000000000000000000000000000000000000000..b45a837f395230029e9d4194ff9f7f2f8f7067b0 --- /dev/null +++ b/annotator/uniformer/mmseg/models/backbones/resnest.py @@ -0,0 +1,314 @@ +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as cp +from annotator.uniformer.mmcv.cnn import build_conv_layer, build_norm_layer + +from ..builder import BACKBONES +from ..utils import ResLayer +from .resnet import Bottleneck as _Bottleneck +from .resnet import ResNetV1d + + +class RSoftmax(nn.Module): + """Radix Softmax module in ``SplitAttentionConv2d``. + + Args: + radix (int): Radix of input. + groups (int): Groups of input. + """ + + def __init__(self, radix, groups): + super().__init__() + self.radix = radix + self.groups = groups + + def forward(self, x): + batch = x.size(0) + if self.radix > 1: + x = x.view(batch, self.groups, self.radix, -1).transpose(1, 2) + x = F.softmax(x, dim=1) + x = x.reshape(batch, -1) + else: + x = torch.sigmoid(x) + return x + + +class SplitAttentionConv2d(nn.Module): + """Split-Attention Conv2d in ResNeSt. + + Args: + in_channels (int): Same as nn.Conv2d. + out_channels (int): Same as nn.Conv2d. + kernel_size (int | tuple[int]): Same as nn.Conv2d. + stride (int | tuple[int]): Same as nn.Conv2d. + padding (int | tuple[int]): Same as nn.Conv2d. + dilation (int | tuple[int]): Same as nn.Conv2d. + groups (int): Same as nn.Conv2d. + radix (int): Radix of SpltAtConv2d. Default: 2 + reduction_factor (int): Reduction factor of inter_channels. Default: 4. + conv_cfg (dict): Config dict for convolution layer. Default: None, + which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. Default: None. + dcn (dict): Config dict for DCN. Default: None. + """ + + def __init__(self, + in_channels, + channels, + kernel_size, + stride=1, + padding=0, + dilation=1, + groups=1, + radix=2, + reduction_factor=4, + conv_cfg=None, + norm_cfg=dict(type='BN'), + dcn=None): + super(SplitAttentionConv2d, self).__init__() + inter_channels = max(in_channels * radix // reduction_factor, 32) + self.radix = radix + self.groups = groups + self.channels = channels + self.with_dcn = dcn is not None + self.dcn = dcn + fallback_on_stride = False + if self.with_dcn: + fallback_on_stride = self.dcn.pop('fallback_on_stride', False) + if self.with_dcn and not fallback_on_stride: + assert conv_cfg is None, 'conv_cfg must be None for DCN' + conv_cfg = dcn + self.conv = build_conv_layer( + conv_cfg, + in_channels, + channels * radix, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + groups=groups * radix, + bias=False) + self.norm0_name, norm0 = build_norm_layer( + norm_cfg, channels * radix, postfix=0) + self.add_module(self.norm0_name, norm0) + self.relu = nn.ReLU(inplace=True) + self.fc1 = build_conv_layer( + None, channels, inter_channels, 1, groups=self.groups) + self.norm1_name, norm1 = build_norm_layer( + norm_cfg, inter_channels, postfix=1) + self.add_module(self.norm1_name, norm1) + self.fc2 = build_conv_layer( + None, inter_channels, channels * radix, 1, groups=self.groups) + self.rsoftmax = RSoftmax(radix, groups) + + @property + def norm0(self): + """nn.Module: the normalization layer named "norm0" """ + return getattr(self, self.norm0_name) + + @property + def norm1(self): + """nn.Module: the normalization layer named "norm1" """ + return getattr(self, self.norm1_name) + + def forward(self, x): + x = self.conv(x) + x = self.norm0(x) + x = self.relu(x) + + batch, rchannel = x.shape[:2] + batch = x.size(0) + if self.radix > 1: + splits = x.view(batch, self.radix, -1, *x.shape[2:]) + gap = splits.sum(dim=1) + else: + gap = x + gap = F.adaptive_avg_pool2d(gap, 1) + gap = self.fc1(gap) + + gap = self.norm1(gap) + gap = self.relu(gap) + + atten = self.fc2(gap) + atten = self.rsoftmax(atten).view(batch, -1, 1, 1) + + if self.radix > 1: + attens = atten.view(batch, self.radix, -1, *atten.shape[2:]) + out = torch.sum(attens * splits, dim=1) + else: + out = atten * x + return out.contiguous() + + +class Bottleneck(_Bottleneck): + """Bottleneck block for ResNeSt. + + Args: + inplane (int): Input planes of this block. + planes (int): Middle planes of this block. + groups (int): Groups of conv2. + width_per_group (int): Width per group of conv2. 64x4d indicates + ``groups=64, width_per_group=4`` and 32x8d indicates + ``groups=32, width_per_group=8``. + radix (int): Radix of SpltAtConv2d. Default: 2 + reduction_factor (int): Reduction factor of inter_channels in + SplitAttentionConv2d. Default: 4. + avg_down_stride (bool): Whether to use average pool for stride in + Bottleneck. Default: True. + kwargs (dict): Key word arguments for base class. + """ + expansion = 4 + + def __init__(self, + inplanes, + planes, + groups=1, + base_width=4, + base_channels=64, + radix=2, + reduction_factor=4, + avg_down_stride=True, + **kwargs): + """Bottleneck block for ResNeSt.""" + super(Bottleneck, self).__init__(inplanes, planes, **kwargs) + + if groups == 1: + width = self.planes + else: + width = math.floor(self.planes * + (base_width / base_channels)) * groups + + self.avg_down_stride = avg_down_stride and self.conv2_stride > 1 + + self.norm1_name, norm1 = build_norm_layer( + self.norm_cfg, width, postfix=1) + self.norm3_name, norm3 = build_norm_layer( + self.norm_cfg, self.planes * self.expansion, postfix=3) + + self.conv1 = build_conv_layer( + self.conv_cfg, + self.inplanes, + width, + kernel_size=1, + stride=self.conv1_stride, + bias=False) + self.add_module(self.norm1_name, norm1) + self.with_modulated_dcn = False + self.conv2 = SplitAttentionConv2d( + width, + width, + kernel_size=3, + stride=1 if self.avg_down_stride else self.conv2_stride, + padding=self.dilation, + dilation=self.dilation, + groups=groups, + radix=radix, + reduction_factor=reduction_factor, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + dcn=self.dcn) + delattr(self, self.norm2_name) + + if self.avg_down_stride: + self.avd_layer = nn.AvgPool2d(3, self.conv2_stride, padding=1) + + self.conv3 = build_conv_layer( + self.conv_cfg, + width, + self.planes * self.expansion, + kernel_size=1, + bias=False) + self.add_module(self.norm3_name, norm3) + + def forward(self, x): + + def _inner_forward(x): + identity = x + + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv1_plugin_names) + + out = self.conv2(out) + + if self.avg_down_stride: + out = self.avd_layer(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv2_plugin_names) + + out = self.conv3(out) + out = self.norm3(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv3_plugin_names) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + out = self.relu(out) + + return out + + +@BACKBONES.register_module() +class ResNeSt(ResNetV1d): + """ResNeSt backbone. + + Args: + groups (int): Number of groups of Bottleneck. Default: 1 + base_width (int): Base width of Bottleneck. Default: 4 + radix (int): Radix of SpltAtConv2d. Default: 2 + reduction_factor (int): Reduction factor of inter_channels in + SplitAttentionConv2d. Default: 4. + avg_down_stride (bool): Whether to use average pool for stride in + Bottleneck. Default: True. + kwargs (dict): Keyword arguments for ResNet. + """ + + arch_settings = { + 50: (Bottleneck, (3, 4, 6, 3)), + 101: (Bottleneck, (3, 4, 23, 3)), + 152: (Bottleneck, (3, 8, 36, 3)), + 200: (Bottleneck, (3, 24, 36, 3)) + } + + def __init__(self, + groups=1, + base_width=4, + radix=2, + reduction_factor=4, + avg_down_stride=True, + **kwargs): + self.groups = groups + self.base_width = base_width + self.radix = radix + self.reduction_factor = reduction_factor + self.avg_down_stride = avg_down_stride + super(ResNeSt, self).__init__(**kwargs) + + def make_res_layer(self, **kwargs): + """Pack all blocks in a stage into a ``ResLayer``.""" + return ResLayer( + groups=self.groups, + base_width=self.base_width, + base_channels=self.base_channels, + radix=self.radix, + reduction_factor=self.reduction_factor, + avg_down_stride=self.avg_down_stride, + **kwargs) diff --git a/annotator/uniformer/mmseg/models/backbones/resnet.py b/annotator/uniformer/mmseg/models/backbones/resnet.py new file mode 100644 index 0000000000000000000000000000000000000000..4e52bf048d28ecb069db4728e5f05ad85ac53198 --- /dev/null +++ b/annotator/uniformer/mmseg/models/backbones/resnet.py @@ -0,0 +1,688 @@ +import torch.nn as nn +import torch.utils.checkpoint as cp +from annotator.uniformer.mmcv.cnn import (build_conv_layer, build_norm_layer, build_plugin_layer, + constant_init, kaiming_init) +from annotator.uniformer.mmcv.runner import load_checkpoint +from annotator.uniformer.mmcv.utils.parrots_wrapper import _BatchNorm + +from annotator.uniformer.mmseg.utils import get_root_logger +from ..builder import BACKBONES +from ..utils import ResLayer + + +class BasicBlock(nn.Module): + """Basic block for ResNet.""" + + expansion = 1 + + def __init__(self, + inplanes, + planes, + stride=1, + dilation=1, + downsample=None, + style='pytorch', + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + dcn=None, + plugins=None): + super(BasicBlock, self).__init__() + assert dcn is None, 'Not implemented yet.' + assert plugins is None, 'Not implemented yet.' + + self.norm1_name, norm1 = build_norm_layer(norm_cfg, planes, postfix=1) + self.norm2_name, norm2 = build_norm_layer(norm_cfg, planes, postfix=2) + + self.conv1 = build_conv_layer( + conv_cfg, + inplanes, + planes, + 3, + stride=stride, + padding=dilation, + dilation=dilation, + bias=False) + self.add_module(self.norm1_name, norm1) + self.conv2 = build_conv_layer( + conv_cfg, planes, planes, 3, padding=1, bias=False) + self.add_module(self.norm2_name, norm2) + + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + self.dilation = dilation + self.with_cp = with_cp + + @property + def norm1(self): + """nn.Module: normalization layer after the first convolution layer""" + return getattr(self, self.norm1_name) + + @property + def norm2(self): + """nn.Module: normalization layer after the second convolution layer""" + return getattr(self, self.norm2_name) + + def forward(self, x): + """Forward function.""" + + def _inner_forward(x): + identity = x + + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.norm2(out) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + """Bottleneck block for ResNet. + + If style is "pytorch", the stride-two layer is the 3x3 conv layer, if it is + "caffe", the stride-two layer is the first 1x1 conv layer. + """ + + expansion = 4 + + def __init__(self, + inplanes, + planes, + stride=1, + dilation=1, + downsample=None, + style='pytorch', + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + dcn=None, + plugins=None): + super(Bottleneck, self).__init__() + assert style in ['pytorch', 'caffe'] + assert dcn is None or isinstance(dcn, dict) + assert plugins is None or isinstance(plugins, list) + if plugins is not None: + allowed_position = ['after_conv1', 'after_conv2', 'after_conv3'] + assert all(p['position'] in allowed_position for p in plugins) + + self.inplanes = inplanes + self.planes = planes + self.stride = stride + self.dilation = dilation + self.style = style + self.with_cp = with_cp + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.dcn = dcn + self.with_dcn = dcn is not None + self.plugins = plugins + self.with_plugins = plugins is not None + + if self.with_plugins: + # collect plugins for conv1/conv2/conv3 + self.after_conv1_plugins = [ + plugin['cfg'] for plugin in plugins + if plugin['position'] == 'after_conv1' + ] + self.after_conv2_plugins = [ + plugin['cfg'] for plugin in plugins + if plugin['position'] == 'after_conv2' + ] + self.after_conv3_plugins = [ + plugin['cfg'] for plugin in plugins + if plugin['position'] == 'after_conv3' + ] + + if self.style == 'pytorch': + self.conv1_stride = 1 + self.conv2_stride = stride + else: + self.conv1_stride = stride + self.conv2_stride = 1 + + self.norm1_name, norm1 = build_norm_layer(norm_cfg, planes, postfix=1) + self.norm2_name, norm2 = build_norm_layer(norm_cfg, planes, postfix=2) + self.norm3_name, norm3 = build_norm_layer( + norm_cfg, planes * self.expansion, postfix=3) + + self.conv1 = build_conv_layer( + conv_cfg, + inplanes, + planes, + kernel_size=1, + stride=self.conv1_stride, + bias=False) + self.add_module(self.norm1_name, norm1) + fallback_on_stride = False + if self.with_dcn: + fallback_on_stride = dcn.pop('fallback_on_stride', False) + if not self.with_dcn or fallback_on_stride: + self.conv2 = build_conv_layer( + conv_cfg, + planes, + planes, + kernel_size=3, + stride=self.conv2_stride, + padding=dilation, + dilation=dilation, + bias=False) + else: + assert self.conv_cfg is None, 'conv_cfg must be None for DCN' + self.conv2 = build_conv_layer( + dcn, + planes, + planes, + kernel_size=3, + stride=self.conv2_stride, + padding=dilation, + dilation=dilation, + bias=False) + + self.add_module(self.norm2_name, norm2) + self.conv3 = build_conv_layer( + conv_cfg, + planes, + planes * self.expansion, + kernel_size=1, + bias=False) + self.add_module(self.norm3_name, norm3) + + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + + if self.with_plugins: + self.after_conv1_plugin_names = self.make_block_plugins( + planes, self.after_conv1_plugins) + self.after_conv2_plugin_names = self.make_block_plugins( + planes, self.after_conv2_plugins) + self.after_conv3_plugin_names = self.make_block_plugins( + planes * self.expansion, self.after_conv3_plugins) + + def make_block_plugins(self, in_channels, plugins): + """make plugins for block. + + Args: + in_channels (int): Input channels of plugin. + plugins (list[dict]): List of plugins cfg to build. + + Returns: + list[str]: List of the names of plugin. + """ + assert isinstance(plugins, list) + plugin_names = [] + for plugin in plugins: + plugin = plugin.copy() + name, layer = build_plugin_layer( + plugin, + in_channels=in_channels, + postfix=plugin.pop('postfix', '')) + assert not hasattr(self, name), f'duplicate plugin {name}' + self.add_module(name, layer) + plugin_names.append(name) + return plugin_names + + def forward_plugin(self, x, plugin_names): + """Forward function for plugins.""" + out = x + for name in plugin_names: + out = getattr(self, name)(x) + return out + + @property + def norm1(self): + """nn.Module: normalization layer after the first convolution layer""" + return getattr(self, self.norm1_name) + + @property + def norm2(self): + """nn.Module: normalization layer after the second convolution layer""" + return getattr(self, self.norm2_name) + + @property + def norm3(self): + """nn.Module: normalization layer after the third convolution layer""" + return getattr(self, self.norm3_name) + + def forward(self, x): + """Forward function.""" + + def _inner_forward(x): + identity = x + + out = self.conv1(x) + out = self.norm1(out) + out = self.relu(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv1_plugin_names) + + out = self.conv2(out) + out = self.norm2(out) + out = self.relu(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv2_plugin_names) + + out = self.conv3(out) + out = self.norm3(out) + + if self.with_plugins: + out = self.forward_plugin(out, self.after_conv3_plugin_names) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + out = self.relu(out) + + return out + + +@BACKBONES.register_module() +class ResNet(nn.Module): + """ResNet backbone. + + Args: + depth (int): Depth of resnet, from {18, 34, 50, 101, 152}. + in_channels (int): Number of input image channels. Default" 3. + stem_channels (int): Number of stem channels. Default: 64. + base_channels (int): Number of base channels of res layer. Default: 64. + num_stages (int): Resnet stages, normally 4. + strides (Sequence[int]): Strides of the first block of each stage. + dilations (Sequence[int]): Dilation of each stage. + out_indices (Sequence[int]): Output from which stages. + style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two + layer is the 3x3 conv layer, otherwise the stride-two layer is + the first 1x1 conv layer. + deep_stem (bool): Replace 7x7 conv in input stem with 3 3x3 conv + avg_down (bool): Use AvgPool instead of stride conv when + downsampling in the bottleneck. + frozen_stages (int): Stages to be frozen (stop grad and set eval mode). + -1 means not freezing any parameters. + norm_cfg (dict): Dictionary to construct and config norm layer. + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. + plugins (list[dict]): List of plugins for stages, each dict contains: + + - cfg (dict, required): Cfg dict to build plugin. + + - position (str, required): Position inside block to insert plugin, + options: 'after_conv1', 'after_conv2', 'after_conv3'. + + - stages (tuple[bool], optional): Stages to apply plugin, length + should be same as 'num_stages' + multi_grid (Sequence[int]|None): Multi grid dilation rates of last + stage. Default: None + contract_dilation (bool): Whether contract first dilation of each layer + Default: False + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. + zero_init_residual (bool): Whether to use zero init for last norm layer + in resblocks to let them behave as identity. + + Example: + >>> from annotator.uniformer.mmseg.models import ResNet + >>> import torch + >>> self = ResNet(depth=18) + >>> self.eval() + >>> inputs = torch.rand(1, 3, 32, 32) + >>> level_outputs = self.forward(inputs) + >>> for level_out in level_outputs: + ... print(tuple(level_out.shape)) + (1, 64, 8, 8) + (1, 128, 4, 4) + (1, 256, 2, 2) + (1, 512, 1, 1) + """ + + arch_settings = { + 18: (BasicBlock, (2, 2, 2, 2)), + 34: (BasicBlock, (3, 4, 6, 3)), + 50: (Bottleneck, (3, 4, 6, 3)), + 101: (Bottleneck, (3, 4, 23, 3)), + 152: (Bottleneck, (3, 8, 36, 3)) + } + + def __init__(self, + depth, + in_channels=3, + stem_channels=64, + base_channels=64, + num_stages=4, + strides=(1, 2, 2, 2), + dilations=(1, 1, 1, 1), + out_indices=(0, 1, 2, 3), + style='pytorch', + deep_stem=False, + avg_down=False, + frozen_stages=-1, + conv_cfg=None, + norm_cfg=dict(type='BN', requires_grad=True), + norm_eval=False, + dcn=None, + stage_with_dcn=(False, False, False, False), + plugins=None, + multi_grid=None, + contract_dilation=False, + with_cp=False, + zero_init_residual=True): + super(ResNet, self).__init__() + if depth not in self.arch_settings: + raise KeyError(f'invalid depth {depth} for resnet') + self.depth = depth + self.stem_channels = stem_channels + self.base_channels = base_channels + self.num_stages = num_stages + assert num_stages >= 1 and num_stages <= 4 + self.strides = strides + self.dilations = dilations + assert len(strides) == len(dilations) == num_stages + self.out_indices = out_indices + assert max(out_indices) < num_stages + self.style = style + self.deep_stem = deep_stem + self.avg_down = avg_down + self.frozen_stages = frozen_stages + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.with_cp = with_cp + self.norm_eval = norm_eval + self.dcn = dcn + self.stage_with_dcn = stage_with_dcn + if dcn is not None: + assert len(stage_with_dcn) == num_stages + self.plugins = plugins + self.multi_grid = multi_grid + self.contract_dilation = contract_dilation + self.zero_init_residual = zero_init_residual + self.block, stage_blocks = self.arch_settings[depth] + self.stage_blocks = stage_blocks[:num_stages] + self.inplanes = stem_channels + + self._make_stem_layer(in_channels, stem_channels) + + self.res_layers = [] + for i, num_blocks in enumerate(self.stage_blocks): + stride = strides[i] + dilation = dilations[i] + dcn = self.dcn if self.stage_with_dcn[i] else None + if plugins is not None: + stage_plugins = self.make_stage_plugins(plugins, i) + else: + stage_plugins = None + # multi grid is applied to last layer only + stage_multi_grid = multi_grid if i == len( + self.stage_blocks) - 1 else None + planes = base_channels * 2**i + res_layer = self.make_res_layer( + block=self.block, + inplanes=self.inplanes, + planes=planes, + num_blocks=num_blocks, + stride=stride, + dilation=dilation, + style=self.style, + avg_down=self.avg_down, + with_cp=with_cp, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + dcn=dcn, + plugins=stage_plugins, + multi_grid=stage_multi_grid, + contract_dilation=contract_dilation) + self.inplanes = planes * self.block.expansion + layer_name = f'layer{i+1}' + self.add_module(layer_name, res_layer) + self.res_layers.append(layer_name) + + self._freeze_stages() + + self.feat_dim = self.block.expansion * base_channels * 2**( + len(self.stage_blocks) - 1) + + def make_stage_plugins(self, plugins, stage_idx): + """make plugins for ResNet 'stage_idx'th stage . + + Currently we support to insert 'context_block', + 'empirical_attention_block', 'nonlocal_block' into the backbone like + ResNet/ResNeXt. They could be inserted after conv1/conv2/conv3 of + Bottleneck. + + An example of plugins format could be : + >>> plugins=[ + ... dict(cfg=dict(type='xxx', arg1='xxx'), + ... stages=(False, True, True, True), + ... position='after_conv2'), + ... dict(cfg=dict(type='yyy'), + ... stages=(True, True, True, True), + ... position='after_conv3'), + ... dict(cfg=dict(type='zzz', postfix='1'), + ... stages=(True, True, True, True), + ... position='after_conv3'), + ... dict(cfg=dict(type='zzz', postfix='2'), + ... stages=(True, True, True, True), + ... position='after_conv3') + ... ] + >>> self = ResNet(depth=18) + >>> stage_plugins = self.make_stage_plugins(plugins, 0) + >>> assert len(stage_plugins) == 3 + + Suppose 'stage_idx=0', the structure of blocks in the stage would be: + conv1-> conv2->conv3->yyy->zzz1->zzz2 + Suppose 'stage_idx=1', the structure of blocks in the stage would be: + conv1-> conv2->xxx->conv3->yyy->zzz1->zzz2 + + If stages is missing, the plugin would be applied to all stages. + + Args: + plugins (list[dict]): List of plugins cfg to build. The postfix is + required if multiple same type plugins are inserted. + stage_idx (int): Index of stage to build + + Returns: + list[dict]: Plugins for current stage + """ + stage_plugins = [] + for plugin in plugins: + plugin = plugin.copy() + stages = plugin.pop('stages', None) + assert stages is None or len(stages) == self.num_stages + # whether to insert plugin into current stage + if stages is None or stages[stage_idx]: + stage_plugins.append(plugin) + + return stage_plugins + + def make_res_layer(self, **kwargs): + """Pack all blocks in a stage into a ``ResLayer``.""" + return ResLayer(**kwargs) + + @property + def norm1(self): + """nn.Module: the normalization layer named "norm1" """ + return getattr(self, self.norm1_name) + + def _make_stem_layer(self, in_channels, stem_channels): + """Make stem layer for ResNet.""" + if self.deep_stem: + self.stem = nn.Sequential( + build_conv_layer( + self.conv_cfg, + in_channels, + stem_channels // 2, + kernel_size=3, + stride=2, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, stem_channels // 2)[1], + nn.ReLU(inplace=True), + build_conv_layer( + self.conv_cfg, + stem_channels // 2, + stem_channels // 2, + kernel_size=3, + stride=1, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, stem_channels // 2)[1], + nn.ReLU(inplace=True), + build_conv_layer( + self.conv_cfg, + stem_channels // 2, + stem_channels, + kernel_size=3, + stride=1, + padding=1, + bias=False), + build_norm_layer(self.norm_cfg, stem_channels)[1], + nn.ReLU(inplace=True)) + else: + self.conv1 = build_conv_layer( + self.conv_cfg, + in_channels, + stem_channels, + kernel_size=7, + stride=2, + padding=3, + bias=False) + self.norm1_name, norm1 = build_norm_layer( + self.norm_cfg, stem_channels, postfix=1) + self.add_module(self.norm1_name, norm1) + self.relu = nn.ReLU(inplace=True) + self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) + + def _freeze_stages(self): + """Freeze stages param and norm stats.""" + if self.frozen_stages >= 0: + if self.deep_stem: + self.stem.eval() + for param in self.stem.parameters(): + param.requires_grad = False + else: + self.norm1.eval() + for m in [self.conv1, self.norm1]: + for param in m.parameters(): + param.requires_grad = False + + for i in range(1, self.frozen_stages + 1): + m = getattr(self, f'layer{i}') + m.eval() + for param in m.parameters(): + param.requires_grad = False + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + + if self.dcn is not None: + for m in self.modules(): + if isinstance(m, Bottleneck) and hasattr( + m, 'conv2_offset'): + constant_init(m.conv2_offset, 0) + + if self.zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + constant_init(m.norm3, 0) + elif isinstance(m, BasicBlock): + constant_init(m.norm2, 0) + else: + raise TypeError('pretrained must be a str or None') + + def forward(self, x): + """Forward function.""" + if self.deep_stem: + x = self.stem(x) + else: + x = self.conv1(x) + x = self.norm1(x) + x = self.relu(x) + x = self.maxpool(x) + outs = [] + for i, layer_name in enumerate(self.res_layers): + res_layer = getattr(self, layer_name) + x = res_layer(x) + if i in self.out_indices: + outs.append(x) + return tuple(outs) + + def train(self, mode=True): + """Convert the model into training mode while keep normalization layer + freezed.""" + super(ResNet, self).train(mode) + self._freeze_stages() + if mode and self.norm_eval: + for m in self.modules(): + # trick: eval have effect on BatchNorm only + if isinstance(m, _BatchNorm): + m.eval() + + +@BACKBONES.register_module() +class ResNetV1c(ResNet): + """ResNetV1c variant described in [1]_. + + Compared with default ResNet(ResNetV1b), ResNetV1c replaces the 7x7 conv + in the input stem with three 3x3 convs. + + References: + .. [1] https://arxiv.org/pdf/1812.01187.pdf + """ + + def __init__(self, **kwargs): + super(ResNetV1c, self).__init__( + deep_stem=True, avg_down=False, **kwargs) + + +@BACKBONES.register_module() +class ResNetV1d(ResNet): + """ResNetV1d variant described in [1]_. + + Compared with default ResNet(ResNetV1b), ResNetV1d replaces the 7x7 conv in + the input stem with three 3x3 convs. And in the downsampling block, a 2x2 + avg_pool with stride 2 is added before conv, whose stride is changed to 1. + """ + + def __init__(self, **kwargs): + super(ResNetV1d, self).__init__( + deep_stem=True, avg_down=True, **kwargs) diff --git a/annotator/uniformer/mmseg/models/backbones/resnext.py b/annotator/uniformer/mmseg/models/backbones/resnext.py new file mode 100644 index 0000000000000000000000000000000000000000..962249ad6fd9b50960ad6426f7ce3cac6ed8c5bc --- /dev/null +++ b/annotator/uniformer/mmseg/models/backbones/resnext.py @@ -0,0 +1,145 @@ +import math + +from annotator.uniformer.mmcv.cnn import build_conv_layer, build_norm_layer + +from ..builder import BACKBONES +from ..utils import ResLayer +from .resnet import Bottleneck as _Bottleneck +from .resnet import ResNet + + +class Bottleneck(_Bottleneck): + """Bottleneck block for ResNeXt. + + If style is "pytorch", the stride-two layer is the 3x3 conv layer, if it is + "caffe", the stride-two layer is the first 1x1 conv layer. + """ + + def __init__(self, + inplanes, + planes, + groups=1, + base_width=4, + base_channels=64, + **kwargs): + super(Bottleneck, self).__init__(inplanes, planes, **kwargs) + + if groups == 1: + width = self.planes + else: + width = math.floor(self.planes * + (base_width / base_channels)) * groups + + self.norm1_name, norm1 = build_norm_layer( + self.norm_cfg, width, postfix=1) + self.norm2_name, norm2 = build_norm_layer( + self.norm_cfg, width, postfix=2) + self.norm3_name, norm3 = build_norm_layer( + self.norm_cfg, self.planes * self.expansion, postfix=3) + + self.conv1 = build_conv_layer( + self.conv_cfg, + self.inplanes, + width, + kernel_size=1, + stride=self.conv1_stride, + bias=False) + self.add_module(self.norm1_name, norm1) + fallback_on_stride = False + self.with_modulated_dcn = False + if self.with_dcn: + fallback_on_stride = self.dcn.pop('fallback_on_stride', False) + if not self.with_dcn or fallback_on_stride: + self.conv2 = build_conv_layer( + self.conv_cfg, + width, + width, + kernel_size=3, + stride=self.conv2_stride, + padding=self.dilation, + dilation=self.dilation, + groups=groups, + bias=False) + else: + assert self.conv_cfg is None, 'conv_cfg must be None for DCN' + self.conv2 = build_conv_layer( + self.dcn, + width, + width, + kernel_size=3, + stride=self.conv2_stride, + padding=self.dilation, + dilation=self.dilation, + groups=groups, + bias=False) + + self.add_module(self.norm2_name, norm2) + self.conv3 = build_conv_layer( + self.conv_cfg, + width, + self.planes * self.expansion, + kernel_size=1, + bias=False) + self.add_module(self.norm3_name, norm3) + + +@BACKBONES.register_module() +class ResNeXt(ResNet): + """ResNeXt backbone. + + Args: + depth (int): Depth of resnet, from {18, 34, 50, 101, 152}. + in_channels (int): Number of input image channels. Normally 3. + num_stages (int): Resnet stages, normally 4. + groups (int): Group of resnext. + base_width (int): Base width of resnext. + strides (Sequence[int]): Strides of the first block of each stage. + dilations (Sequence[int]): Dilation of each stage. + out_indices (Sequence[int]): Output from which stages. + style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two + layer is the 3x3 conv layer, otherwise the stride-two layer is + the first 1x1 conv layer. + frozen_stages (int): Stages to be frozen (all param fixed). -1 means + not freezing any parameters. + norm_cfg (dict): dictionary to construct and config norm layer. + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. + zero_init_residual (bool): whether to use zero init for last norm layer + in resblocks to let them behave as identity. + + Example: + >>> from annotator.uniformer.mmseg.models import ResNeXt + >>> import torch + >>> self = ResNeXt(depth=50) + >>> self.eval() + >>> inputs = torch.rand(1, 3, 32, 32) + >>> level_outputs = self.forward(inputs) + >>> for level_out in level_outputs: + ... print(tuple(level_out.shape)) + (1, 256, 8, 8) + (1, 512, 4, 4) + (1, 1024, 2, 2) + (1, 2048, 1, 1) + """ + + arch_settings = { + 50: (Bottleneck, (3, 4, 6, 3)), + 101: (Bottleneck, (3, 4, 23, 3)), + 152: (Bottleneck, (3, 8, 36, 3)) + } + + def __init__(self, groups=1, base_width=4, **kwargs): + self.groups = groups + self.base_width = base_width + super(ResNeXt, self).__init__(**kwargs) + + def make_res_layer(self, **kwargs): + """Pack all blocks in a stage into a ``ResLayer``""" + return ResLayer( + groups=self.groups, + base_width=self.base_width, + base_channels=self.base_channels, + **kwargs) diff --git a/annotator/uniformer/mmseg/models/backbones/unet.py b/annotator/uniformer/mmseg/models/backbones/unet.py new file mode 100644 index 0000000000000000000000000000000000000000..82caa16a94c195c192a2a920fb7bc7e60f0f3ce3 --- /dev/null +++ b/annotator/uniformer/mmseg/models/backbones/unet.py @@ -0,0 +1,429 @@ +import torch.nn as nn +import torch.utils.checkpoint as cp +from annotator.uniformer.mmcv.cnn import (UPSAMPLE_LAYERS, ConvModule, build_activation_layer, + build_norm_layer, constant_init, kaiming_init) +from annotator.uniformer.mmcv.runner import load_checkpoint +from annotator.uniformer.mmcv.utils.parrots_wrapper import _BatchNorm + +from annotator.uniformer.mmseg.utils import get_root_logger +from ..builder import BACKBONES +from ..utils import UpConvBlock + + +class BasicConvBlock(nn.Module): + """Basic convolutional block for UNet. + + This module consists of several plain convolutional layers. + + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + num_convs (int): Number of convolutional layers. Default: 2. + stride (int): Whether use stride convolution to downsample + the input feature map. If stride=2, it only uses stride convolution + in the first convolutional layer to downsample the input feature + map. Options are 1 or 2. Default: 1. + dilation (int): Whether use dilated convolution to expand the + receptive field. Set dilation rate of each convolutional layer and + the dilation rate of the first convolutional layer is always 1. + Default: 1. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + conv_cfg (dict | None): Config dict for convolution layer. + Default: None. + norm_cfg (dict | None): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict | None): Config dict for activation layer in ConvModule. + Default: dict(type='ReLU'). + dcn (bool): Use deformable convolution in convolutional layer or not. + Default: None. + plugins (dict): plugins for convolutional layers. Default: None. + """ + + def __init__(self, + in_channels, + out_channels, + num_convs=2, + stride=1, + dilation=1, + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + dcn=None, + plugins=None): + super(BasicConvBlock, self).__init__() + assert dcn is None, 'Not implemented yet.' + assert plugins is None, 'Not implemented yet.' + + self.with_cp = with_cp + convs = [] + for i in range(num_convs): + convs.append( + ConvModule( + in_channels=in_channels if i == 0 else out_channels, + out_channels=out_channels, + kernel_size=3, + stride=stride if i == 0 else 1, + dilation=1 if i == 0 else dilation, + padding=1 if i == 0 else dilation, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + + self.convs = nn.Sequential(*convs) + + def forward(self, x): + """Forward function.""" + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(self.convs, x) + else: + out = self.convs(x) + return out + + +@UPSAMPLE_LAYERS.register_module() +class DeconvModule(nn.Module): + """Deconvolution upsample module in decoder for UNet (2X upsample). + + This module uses deconvolution to upsample feature map in the decoder + of UNet. + + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + norm_cfg (dict | None): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict | None): Config dict for activation layer in ConvModule. + Default: dict(type='ReLU'). + kernel_size (int): Kernel size of the convolutional layer. Default: 4. + """ + + def __init__(self, + in_channels, + out_channels, + with_cp=False, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + *, + kernel_size=4, + scale_factor=2): + super(DeconvModule, self).__init__() + + assert (kernel_size - scale_factor >= 0) and\ + (kernel_size - scale_factor) % 2 == 0,\ + f'kernel_size should be greater than or equal to scale_factor '\ + f'and (kernel_size - scale_factor) should be even numbers, '\ + f'while the kernel size is {kernel_size} and scale_factor is '\ + f'{scale_factor}.' + + stride = scale_factor + padding = (kernel_size - scale_factor) // 2 + self.with_cp = with_cp + deconv = nn.ConvTranspose2d( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding) + + norm_name, norm = build_norm_layer(norm_cfg, out_channels) + activate = build_activation_layer(act_cfg) + self.deconv_upsamping = nn.Sequential(deconv, norm, activate) + + def forward(self, x): + """Forward function.""" + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(self.deconv_upsamping, x) + else: + out = self.deconv_upsamping(x) + return out + + +@UPSAMPLE_LAYERS.register_module() +class InterpConv(nn.Module): + """Interpolation upsample module in decoder for UNet. + + This module uses interpolation to upsample feature map in the decoder + of UNet. It consists of one interpolation upsample layer and one + convolutional layer. It can be one interpolation upsample layer followed + by one convolutional layer (conv_first=False) or one convolutional layer + followed by one interpolation upsample layer (conv_first=True). + + Args: + in_channels (int): Number of input channels. + out_channels (int): Number of output channels. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + norm_cfg (dict | None): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict | None): Config dict for activation layer in ConvModule. + Default: dict(type='ReLU'). + conv_cfg (dict | None): Config dict for convolution layer. + Default: None. + conv_first (bool): Whether convolutional layer or interpolation + upsample layer first. Default: False. It means interpolation + upsample layer followed by one convolutional layer. + kernel_size (int): Kernel size of the convolutional layer. Default: 1. + stride (int): Stride of the convolutional layer. Default: 1. + padding (int): Padding of the convolutional layer. Default: 1. + upsample_cfg (dict): Interpolation config of the upsample layer. + Default: dict( + scale_factor=2, mode='bilinear', align_corners=False). + """ + + def __init__(self, + in_channels, + out_channels, + with_cp=False, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + *, + conv_cfg=None, + conv_first=False, + kernel_size=1, + stride=1, + padding=0, + upsample_cfg=dict( + scale_factor=2, mode='bilinear', align_corners=False)): + super(InterpConv, self).__init__() + + self.with_cp = with_cp + conv = ConvModule( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + upsample = nn.Upsample(**upsample_cfg) + if conv_first: + self.interp_upsample = nn.Sequential(conv, upsample) + else: + self.interp_upsample = nn.Sequential(upsample, conv) + + def forward(self, x): + """Forward function.""" + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(self.interp_upsample, x) + else: + out = self.interp_upsample(x) + return out + + +@BACKBONES.register_module() +class UNet(nn.Module): + """UNet backbone. + U-Net: Convolutional Networks for Biomedical Image Segmentation. + https://arxiv.org/pdf/1505.04597.pdf + + Args: + in_channels (int): Number of input image channels. Default" 3. + base_channels (int): Number of base channels of each stage. + The output channels of the first stage. Default: 64. + num_stages (int): Number of stages in encoder, normally 5. Default: 5. + strides (Sequence[int 1 | 2]): Strides of each stage in encoder. + len(strides) is equal to num_stages. Normally the stride of the + first stage in encoder is 1. If strides[i]=2, it uses stride + convolution to downsample in the correspondence encoder stage. + Default: (1, 1, 1, 1, 1). + enc_num_convs (Sequence[int]): Number of convolutional layers in the + convolution block of the correspondence encoder stage. + Default: (2, 2, 2, 2, 2). + dec_num_convs (Sequence[int]): Number of convolutional layers in the + convolution block of the correspondence decoder stage. + Default: (2, 2, 2, 2). + downsamples (Sequence[int]): Whether use MaxPool to downsample the + feature map after the first stage of encoder + (stages: [1, num_stages)). If the correspondence encoder stage use + stride convolution (strides[i]=2), it will never use MaxPool to + downsample, even downsamples[i-1]=True. + Default: (True, True, True, True). + enc_dilations (Sequence[int]): Dilation rate of each stage in encoder. + Default: (1, 1, 1, 1, 1). + dec_dilations (Sequence[int]): Dilation rate of each stage in decoder. + Default: (1, 1, 1, 1). + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + conv_cfg (dict | None): Config dict for convolution layer. + Default: None. + norm_cfg (dict | None): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict | None): Config dict for activation layer in ConvModule. + Default: dict(type='ReLU'). + upsample_cfg (dict): The upsample config of the upsample module in + decoder. Default: dict(type='InterpConv'). + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. Default: False. + dcn (bool): Use deformable convolution in convolutional layer or not. + Default: None. + plugins (dict): plugins for convolutional layers. Default: None. + + Notice: + The input image size should be divisible by the whole downsample rate + of the encoder. More detail of the whole downsample rate can be found + in UNet._check_input_divisible. + + """ + + def __init__(self, + in_channels=3, + base_channels=64, + num_stages=5, + strides=(1, 1, 1, 1, 1), + enc_num_convs=(2, 2, 2, 2, 2), + dec_num_convs=(2, 2, 2, 2), + downsamples=(True, True, True, True), + enc_dilations=(1, 1, 1, 1, 1), + dec_dilations=(1, 1, 1, 1), + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + upsample_cfg=dict(type='InterpConv'), + norm_eval=False, + dcn=None, + plugins=None): + super(UNet, self).__init__() + assert dcn is None, 'Not implemented yet.' + assert plugins is None, 'Not implemented yet.' + assert len(strides) == num_stages, \ + 'The length of strides should be equal to num_stages, '\ + f'while the strides is {strides}, the length of '\ + f'strides is {len(strides)}, and the num_stages is '\ + f'{num_stages}.' + assert len(enc_num_convs) == num_stages, \ + 'The length of enc_num_convs should be equal to num_stages, '\ + f'while the enc_num_convs is {enc_num_convs}, the length of '\ + f'enc_num_convs is {len(enc_num_convs)}, and the num_stages is '\ + f'{num_stages}.' + assert len(dec_num_convs) == (num_stages-1), \ + 'The length of dec_num_convs should be equal to (num_stages-1), '\ + f'while the dec_num_convs is {dec_num_convs}, the length of '\ + f'dec_num_convs is {len(dec_num_convs)}, and the num_stages is '\ + f'{num_stages}.' + assert len(downsamples) == (num_stages-1), \ + 'The length of downsamples should be equal to (num_stages-1), '\ + f'while the downsamples is {downsamples}, the length of '\ + f'downsamples is {len(downsamples)}, and the num_stages is '\ + f'{num_stages}.' + assert len(enc_dilations) == num_stages, \ + 'The length of enc_dilations should be equal to num_stages, '\ + f'while the enc_dilations is {enc_dilations}, the length of '\ + f'enc_dilations is {len(enc_dilations)}, and the num_stages is '\ + f'{num_stages}.' + assert len(dec_dilations) == (num_stages-1), \ + 'The length of dec_dilations should be equal to (num_stages-1), '\ + f'while the dec_dilations is {dec_dilations}, the length of '\ + f'dec_dilations is {len(dec_dilations)}, and the num_stages is '\ + f'{num_stages}.' + self.num_stages = num_stages + self.strides = strides + self.downsamples = downsamples + self.norm_eval = norm_eval + self.base_channels = base_channels + + self.encoder = nn.ModuleList() + self.decoder = nn.ModuleList() + + for i in range(num_stages): + enc_conv_block = [] + if i != 0: + if strides[i] == 1 and downsamples[i - 1]: + enc_conv_block.append(nn.MaxPool2d(kernel_size=2)) + upsample = (strides[i] != 1 or downsamples[i - 1]) + self.decoder.append( + UpConvBlock( + conv_block=BasicConvBlock, + in_channels=base_channels * 2**i, + skip_channels=base_channels * 2**(i - 1), + out_channels=base_channels * 2**(i - 1), + num_convs=dec_num_convs[i - 1], + stride=1, + dilation=dec_dilations[i - 1], + with_cp=with_cp, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + upsample_cfg=upsample_cfg if upsample else None, + dcn=None, + plugins=None)) + + enc_conv_block.append( + BasicConvBlock( + in_channels=in_channels, + out_channels=base_channels * 2**i, + num_convs=enc_num_convs[i], + stride=strides[i], + dilation=enc_dilations[i], + with_cp=with_cp, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + dcn=None, + plugins=None)) + self.encoder.append((nn.Sequential(*enc_conv_block))) + in_channels = base_channels * 2**i + + def forward(self, x): + self._check_input_divisible(x) + enc_outs = [] + for enc in self.encoder: + x = enc(x) + enc_outs.append(x) + dec_outs = [x] + for i in reversed(range(len(self.decoder))): + x = self.decoder[i](enc_outs[i], x) + dec_outs.append(x) + + return dec_outs + + def train(self, mode=True): + """Convert the model into training mode while keep normalization layer + freezed.""" + super(UNet, self).train(mode) + if mode and self.norm_eval: + for m in self.modules(): + # trick: eval have effect on BatchNorm only + if isinstance(m, _BatchNorm): + m.eval() + + def _check_input_divisible(self, x): + h, w = x.shape[-2:] + whole_downsample_rate = 1 + for i in range(1, self.num_stages): + if self.strides[i] == 2 or self.downsamples[i - 1]: + whole_downsample_rate *= 2 + assert (h % whole_downsample_rate == 0) \ + and (w % whole_downsample_rate == 0),\ + f'The input image size {(h, w)} should be divisible by the whole '\ + f'downsample rate {whole_downsample_rate}, when num_stages is '\ + f'{self.num_stages}, strides is {self.strides}, and downsamples '\ + f'is {self.downsamples}.' + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, strict=False, logger=logger) + elif pretrained is None: + for m in self.modules(): + if isinstance(m, nn.Conv2d): + kaiming_init(m) + elif isinstance(m, (_BatchNorm, nn.GroupNorm)): + constant_init(m, 1) + else: + raise TypeError('pretrained must be a str or None') diff --git a/annotator/uniformer/mmseg/models/backbones/uniformer.py b/annotator/uniformer/mmseg/models/backbones/uniformer.py new file mode 100644 index 0000000000000000000000000000000000000000..0c4bb88e4c928540cca9ab609988b916520f5b7a --- /dev/null +++ b/annotator/uniformer/mmseg/models/backbones/uniformer.py @@ -0,0 +1,422 @@ +# -------------------------------------------------------- +# UniFormer +# Copyright (c) 2022 SenseTime X-Lab +# Licensed under The MIT License [see LICENSE for details] +# Written by Kunchang Li +# -------------------------------------------------------- + +from collections import OrderedDict +import math + +from functools import partial +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +import numpy as np +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ + +from annotator.uniformer.mmcv_custom import load_checkpoint +from annotator.uniformer.mmseg.utils import get_root_logger +from ..builder import BACKBONES + + +class Mlp(nn.Module): + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class CMlp(nn.Module): + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Conv2d(in_features, hidden_features, 1) + self.act = act_layer() + self.fc2 = nn.Conv2d(hidden_features, out_features, 1) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class CBlock(nn.Module): + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm): + super().__init__() + self.pos_embed = nn.Conv2d(dim, dim, 3, padding=1, groups=dim) + self.norm1 = nn.BatchNorm2d(dim) + self.conv1 = nn.Conv2d(dim, dim, 1) + self.conv2 = nn.Conv2d(dim, dim, 1) + self.attn = nn.Conv2d(dim, dim, 5, padding=2, groups=dim) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = nn.BatchNorm2d(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = CMlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + def forward(self, x): + x = x + self.pos_embed(x) + x = x + self.drop_path(self.conv2(self.attn(self.conv1(self.norm1(x))))) + x = x + self.drop_path(self.mlp(self.norm2(x))) + return x + + +class Attention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0.): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + # NOTE scale factor was wrong in my original version, can set manually to be compat with prev weights + self.scale = qk_scale or head_dim ** -0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x): + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class SABlock(nn.Module): + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm): + super().__init__() + self.pos_embed = nn.Conv2d(dim, dim, 3, padding=1, groups=dim) + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, + num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, + attn_drop=attn_drop, proj_drop=drop) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + def forward(self, x): + x = x + self.pos_embed(x) + B, N, H, W = x.shape + x = x.flatten(2).transpose(1, 2) + x = x + self.drop_path(self.attn(self.norm1(x))) + x = x + self.drop_path(self.mlp(self.norm2(x))) + x = x.transpose(1, 2).reshape(B, N, H, W) + return x + + +def window_partition(x, window_size): + """ + Args: + x: (B, H, W, C) + window_size (int): window size + Returns: + windows: (num_windows*B, window_size, window_size, C) + """ + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + return windows + + +def window_reverse(windows, window_size, H, W): + """ + Args: + windows: (num_windows*B, window_size, window_size, C) + window_size (int): Window size + H (int): Height of image + W (int): Width of image + Returns: + x: (B, H, W, C) + """ + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +class SABlock_Windows(nn.Module): + def __init__(self, dim, num_heads, window_size=14, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm): + super().__init__() + self.window_size=window_size + self.pos_embed = nn.Conv2d(dim, dim, 3, padding=1, groups=dim) + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, + num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, + attn_drop=attn_drop, proj_drop=drop) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + def forward(self, x): + x = x + self.pos_embed(x) + x = x.permute(0, 2, 3, 1) + B, H, W, C = x.shape + shortcut = x + x = self.norm1(x) + + pad_l = pad_t = 0 + pad_r = (self.window_size - W % self.window_size) % self.window_size + pad_b = (self.window_size - H % self.window_size) % self.window_size + x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b)) + _, Hp, Wp, _ = x.shape + + x_windows = window_partition(x, self.window_size) # nW*B, window_size, window_size, C + x_windows = x_windows.view(-1, self.window_size * self.window_size, C) # nW*B, window_size*window_size, C + + # W-MSA/SW-MSA + attn_windows = self.attn(x_windows) # nW*B, window_size*window_size, C + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C) + x = window_reverse(attn_windows, self.window_size, Hp, Wp) # B H' W' C + + # reverse cyclic shift + if pad_r > 0 or pad_b > 0: + x = x[:, :H, :W, :].contiguous() + + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + x = x.permute(0, 3, 1, 2).reshape(B, C, H, W) + return x + + +class PatchEmbed(nn.Module): + """ Image to Patch Embedding + """ + def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + num_patches = (img_size[1] // patch_size[1]) * (img_size[0] // patch_size[0]) + self.img_size = img_size + self.patch_size = patch_size + self.num_patches = num_patches + self.norm = nn.LayerNorm(embed_dim) + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + + def forward(self, x): + B, _, H, W = x.shape + x = self.proj(x) + B, _, H, W = x.shape + x = x.flatten(2).transpose(1, 2) + x = self.norm(x) + x = x.reshape(B, H, W, -1).permute(0, 3, 1, 2).contiguous() + return x + + +@BACKBONES.register_module() +class UniFormer(nn.Module): + """ Vision Transformer + A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale` - + https://arxiv.org/abs/2010.11929 + """ + def __init__(self, layers=[3, 4, 8, 3], img_size=224, in_chans=3, num_classes=80, embed_dim=[64, 128, 320, 512], + head_dim=64, mlp_ratio=4., qkv_bias=True, qk_scale=None, representation_size=None, + drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=partial(nn.LayerNorm, eps=1e-6), + pretrained_path=None, use_checkpoint=False, checkpoint_num=[0, 0, 0, 0], + windows=False, hybrid=False, window_size=14): + """ + Args: + layer (list): number of block in each layer + img_size (int, tuple): input image size + in_chans (int): number of input channels + num_classes (int): number of classes for classification head + embed_dim (int): embedding dimension + head_dim (int): dimension of attention heads + mlp_ratio (int): ratio of mlp hidden dim to embedding dim + qkv_bias (bool): enable bias for qkv if True + qk_scale (float): override default qk scale of head_dim ** -0.5 if set + representation_size (Optional[int]): enable and set representation layer (pre-logits) to this value if set + drop_rate (float): dropout rate + attn_drop_rate (float): attention dropout rate + drop_path_rate (float): stochastic depth rate + norm_layer (nn.Module): normalization layer + pretrained_path (str): path of pretrained model + use_checkpoint (bool): whether use checkpoint + checkpoint_num (list): index for using checkpoint in every stage + windows (bool): whether use window MHRA + hybrid (bool): whether use hybrid MHRA + window_size (int): size of window (>14) + """ + super().__init__() + self.num_classes = num_classes + self.use_checkpoint = use_checkpoint + self.checkpoint_num = checkpoint_num + self.windows = windows + print(f'Use Checkpoint: {self.use_checkpoint}') + print(f'Checkpoint Number: {self.checkpoint_num}') + self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models + norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) + + self.patch_embed1 = PatchEmbed( + img_size=img_size, patch_size=4, in_chans=in_chans, embed_dim=embed_dim[0]) + self.patch_embed2 = PatchEmbed( + img_size=img_size // 4, patch_size=2, in_chans=embed_dim[0], embed_dim=embed_dim[1]) + self.patch_embed3 = PatchEmbed( + img_size=img_size // 8, patch_size=2, in_chans=embed_dim[1], embed_dim=embed_dim[2]) + self.patch_embed4 = PatchEmbed( + img_size=img_size // 16, patch_size=2, in_chans=embed_dim[2], embed_dim=embed_dim[3]) + + self.pos_drop = nn.Dropout(p=drop_rate) + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(layers))] # stochastic depth decay rule + num_heads = [dim // head_dim for dim in embed_dim] + self.blocks1 = nn.ModuleList([ + CBlock( + dim=embed_dim[0], num_heads=num_heads[0], mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer) + for i in range(layers[0])]) + self.norm1=norm_layer(embed_dim[0]) + self.blocks2 = nn.ModuleList([ + CBlock( + dim=embed_dim[1], num_heads=num_heads[1], mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i+layers[0]], norm_layer=norm_layer) + for i in range(layers[1])]) + self.norm2 = norm_layer(embed_dim[1]) + if self.windows: + print('Use local window for all blocks in stage3') + self.blocks3 = nn.ModuleList([ + SABlock_Windows( + dim=embed_dim[2], num_heads=num_heads[2], window_size=window_size, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i+layers[0]+layers[1]], norm_layer=norm_layer) + for i in range(layers[2])]) + elif hybrid: + print('Use hybrid window for blocks in stage3') + block3 = [] + for i in range(layers[2]): + if (i + 1) % 4 == 0: + block3.append(SABlock( + dim=embed_dim[2], num_heads=num_heads[2], mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i+layers[0]+layers[1]], norm_layer=norm_layer)) + else: + block3.append(SABlock_Windows( + dim=embed_dim[2], num_heads=num_heads[2], window_size=window_size, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i+layers[0]+layers[1]], norm_layer=norm_layer)) + self.blocks3 = nn.ModuleList(block3) + else: + print('Use global window for all blocks in stage3') + self.blocks3 = nn.ModuleList([ + SABlock( + dim=embed_dim[2], num_heads=num_heads[2], mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i+layers[0]+layers[1]], norm_layer=norm_layer) + for i in range(layers[2])]) + self.norm3 = norm_layer(embed_dim[2]) + self.blocks4 = nn.ModuleList([ + SABlock( + dim=embed_dim[3], num_heads=num_heads[3], mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i+layers[0]+layers[1]+layers[2]], norm_layer=norm_layer) + for i in range(layers[3])]) + self.norm4 = norm_layer(embed_dim[3]) + + # Representation layer + if representation_size: + self.num_features = representation_size + self.pre_logits = nn.Sequential(OrderedDict([ + ('fc', nn.Linear(embed_dim, representation_size)), + ('act', nn.Tanh()) + ])) + else: + self.pre_logits = nn.Identity() + + self.apply(self._init_weights) + self.init_weights(pretrained=pretrained_path) + + def init_weights(self, pretrained): + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, map_location='cpu', strict=False, logger=logger) + print(f'Load pretrained model from {pretrained}') + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + @torch.jit.ignore + def no_weight_decay(self): + return {'pos_embed', 'cls_token'} + + def get_classifier(self): + return self.head + + def reset_classifier(self, num_classes, global_pool=''): + self.num_classes = num_classes + self.head = nn.Linear(self.embed_dim, num_classes) if num_classes > 0 else nn.Identity() + + def forward_features(self, x): + out = [] + x = self.patch_embed1(x) + x = self.pos_drop(x) + for i, blk in enumerate(self.blocks1): + if self.use_checkpoint and i < self.checkpoint_num[0]: + x = checkpoint.checkpoint(blk, x) + else: + x = blk(x) + x_out = self.norm1(x.permute(0, 2, 3, 1)) + out.append(x_out.permute(0, 3, 1, 2).contiguous()) + x = self.patch_embed2(x) + for i, blk in enumerate(self.blocks2): + if self.use_checkpoint and i < self.checkpoint_num[1]: + x = checkpoint.checkpoint(blk, x) + else: + x = blk(x) + x_out = self.norm2(x.permute(0, 2, 3, 1)) + out.append(x_out.permute(0, 3, 1, 2).contiguous()) + x = self.patch_embed3(x) + for i, blk in enumerate(self.blocks3): + if self.use_checkpoint and i < self.checkpoint_num[2]: + x = checkpoint.checkpoint(blk, x) + else: + x = blk(x) + x_out = self.norm3(x.permute(0, 2, 3, 1)) + out.append(x_out.permute(0, 3, 1, 2).contiguous()) + x = self.patch_embed4(x) + for i, blk in enumerate(self.blocks4): + if self.use_checkpoint and i < self.checkpoint_num[3]: + x = checkpoint.checkpoint(blk, x) + else: + x = blk(x) + x_out = self.norm4(x.permute(0, 2, 3, 1)) + out.append(x_out.permute(0, 3, 1, 2).contiguous()) + return tuple(out) + + def forward(self, x): + x = self.forward_features(x) + return x diff --git a/annotator/uniformer/mmseg/models/backbones/vit.py b/annotator/uniformer/mmseg/models/backbones/vit.py new file mode 100644 index 0000000000000000000000000000000000000000..59e4479650690e08cbc4cab9427aefda47c2116d --- /dev/null +++ b/annotator/uniformer/mmseg/models/backbones/vit.py @@ -0,0 +1,459 @@ +"""Modified from https://github.com/rwightman/pytorch-image- +models/blob/master/timm/models/vision_transformer.py.""" + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as cp +from annotator.uniformer.mmcv.cnn import (Conv2d, Linear, build_activation_layer, build_norm_layer, + constant_init, kaiming_init, normal_init) +from annotator.uniformer.mmcv.runner import _load_checkpoint +from annotator.uniformer.mmcv.utils.parrots_wrapper import _BatchNorm + +from annotator.uniformer.mmseg.utils import get_root_logger +from ..builder import BACKBONES +from ..utils import DropPath, trunc_normal_ + + +class Mlp(nn.Module): + """MLP layer for Encoder block. + + Args: + in_features(int): Input dimension for the first fully + connected layer. + hidden_features(int): Output dimension for the first fully + connected layer. + out_features(int): Output dementsion for the second fully + connected layer. + act_cfg(dict): Config dict for activation layer. + Default: dict(type='GELU'). + drop(float): Drop rate for the dropout layer. Dropout rate has + to be between 0 and 1. Default: 0. + """ + + def __init__(self, + in_features, + hidden_features=None, + out_features=None, + act_cfg=dict(type='GELU'), + drop=0.): + super(Mlp, self).__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = Linear(in_features, hidden_features) + self.act = build_activation_layer(act_cfg) + self.fc2 = Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class Attention(nn.Module): + """Attention layer for Encoder block. + + Args: + dim (int): Dimension for the input vector. + num_heads (int): Number of parallel attention heads. + qkv_bias (bool): Enable bias for qkv if True. Default: False. + qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. + attn_drop (float): Drop rate for attention output weights. + Default: 0. + proj_drop (float): Drop rate for output weights. Default: 0. + """ + + def __init__(self, + dim, + num_heads=8, + qkv_bias=False, + qk_scale=None, + attn_drop=0., + proj_drop=0.): + super(Attention, self).__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim**-0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x): + b, n, c = x.shape + qkv = self.qkv(x).reshape(b, n, 3, self.num_heads, + c // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(b, n, c) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class Block(nn.Module): + """Implements encoder block with residual connection. + + Args: + dim (int): The feature dimension. + num_heads (int): Number of parallel attention heads. + mlp_ratio (int): Ratio of mlp hidden dim to embedding dim. + qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. + drop (float): Drop rate for mlp output weights. Default: 0. + attn_drop (float): Drop rate for attention output weights. + Default: 0. + proj_drop (float): Drop rate for attn layer output weights. + Default: 0. + drop_path (float): Drop rate for paths of model. + Default: 0. + act_cfg (dict): Config dict for activation layer. + Default: dict(type='GELU'). + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='LN', requires_grad=True). + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + """ + + def __init__(self, + dim, + num_heads, + mlp_ratio=4, + qkv_bias=False, + qk_scale=None, + drop=0., + attn_drop=0., + proj_drop=0., + drop_path=0., + act_cfg=dict(type='GELU'), + norm_cfg=dict(type='LN', eps=1e-6), + with_cp=False): + super(Block, self).__init__() + self.with_cp = with_cp + _, self.norm1 = build_norm_layer(norm_cfg, dim) + self.attn = Attention(dim, num_heads, qkv_bias, qk_scale, attn_drop, + proj_drop) + self.drop_path = DropPath( + drop_path) if drop_path > 0. else nn.Identity() + _, self.norm2 = build_norm_layer(norm_cfg, dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + act_cfg=act_cfg, + drop=drop) + + def forward(self, x): + + def _inner_forward(x): + out = x + self.drop_path(self.attn(self.norm1(x))) + out = out + self.drop_path(self.mlp(self.norm2(out))) + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + return out + + +class PatchEmbed(nn.Module): + """Image to Patch Embedding. + + Args: + img_size (int | tuple): Input image size. + default: 224. + patch_size (int): Width and height for a patch. + default: 16. + in_channels (int): Input channels for images. Default: 3. + embed_dim (int): The embedding dimension. Default: 768. + """ + + def __init__(self, + img_size=224, + patch_size=16, + in_channels=3, + embed_dim=768): + super(PatchEmbed, self).__init__() + if isinstance(img_size, int): + self.img_size = (img_size, img_size) + elif isinstance(img_size, tuple): + self.img_size = img_size + else: + raise TypeError('img_size must be type of int or tuple') + h, w = self.img_size + self.patch_size = (patch_size, patch_size) + self.num_patches = (h // patch_size) * (w // patch_size) + self.proj = Conv2d( + in_channels, embed_dim, kernel_size=patch_size, stride=patch_size) + + def forward(self, x): + return self.proj(x).flatten(2).transpose(1, 2) + + +@BACKBONES.register_module() +class VisionTransformer(nn.Module): + """Vision transformer backbone. + + A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for + Image Recognition at Scale` - https://arxiv.org/abs/2010.11929 + + Args: + img_size (tuple): input image size. Default: (224, 224). + patch_size (int, tuple): patch size. Default: 16. + in_channels (int): number of input channels. Default: 3. + embed_dim (int): embedding dimension. Default: 768. + depth (int): depth of transformer. Default: 12. + num_heads (int): number of attention heads. Default: 12. + mlp_ratio (int): ratio of mlp hidden dim to embedding dim. + Default: 4. + out_indices (list | tuple | int): Output from which stages. + Default: -1. + qkv_bias (bool): enable bias for qkv if True. Default: True. + qk_scale (float): override default qk scale of head_dim ** -0.5 if set. + drop_rate (float): dropout rate. Default: 0. + attn_drop_rate (float): attention dropout rate. Default: 0. + drop_path_rate (float): Rate of DropPath. Default: 0. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='LN', eps=1e-6, requires_grad=True). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='GELU'). + norm_eval (bool): Whether to set norm layers to eval mode, namely, + freeze running stats (mean and var). Note: Effect on Batch Norm + and its variants only. Default: False. + final_norm (bool): Whether to add a additional layer to normalize + final feature map. Default: False. + interpolate_mode (str): Select the interpolate mode for position + embeding vector resize. Default: bicubic. + with_cls_token (bool): If concatenating class token into image tokens + as transformer input. Default: True. + with_cp (bool): Use checkpoint or not. Using checkpoint + will save some memory while slowing down the training speed. + Default: False. + """ + + def __init__(self, + img_size=(224, 224), + patch_size=16, + in_channels=3, + embed_dim=768, + depth=12, + num_heads=12, + mlp_ratio=4, + out_indices=11, + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0., + norm_cfg=dict(type='LN', eps=1e-6, requires_grad=True), + act_cfg=dict(type='GELU'), + norm_eval=False, + final_norm=False, + with_cls_token=True, + interpolate_mode='bicubic', + with_cp=False): + super(VisionTransformer, self).__init__() + self.img_size = img_size + self.patch_size = patch_size + self.features = self.embed_dim = embed_dim + self.patch_embed = PatchEmbed( + img_size=img_size, + patch_size=patch_size, + in_channels=in_channels, + embed_dim=embed_dim) + + self.with_cls_token = with_cls_token + self.cls_token = nn.Parameter(torch.zeros(1, 1, self.embed_dim)) + self.pos_embed = nn.Parameter( + torch.zeros(1, self.patch_embed.num_patches + 1, embed_dim)) + self.pos_drop = nn.Dropout(p=drop_rate) + + if isinstance(out_indices, int): + self.out_indices = [out_indices] + elif isinstance(out_indices, list) or isinstance(out_indices, tuple): + self.out_indices = out_indices + else: + raise TypeError('out_indices must be type of int, list or tuple') + + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth) + ] # stochastic depth decay rule + self.blocks = nn.ModuleList([ + Block( + dim=embed_dim, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=dpr[i], + attn_drop=attn_drop_rate, + act_cfg=act_cfg, + norm_cfg=norm_cfg, + with_cp=with_cp) for i in range(depth) + ]) + + self.interpolate_mode = interpolate_mode + self.final_norm = final_norm + if final_norm: + _, self.norm = build_norm_layer(norm_cfg, embed_dim) + + self.norm_eval = norm_eval + self.with_cp = with_cp + + def init_weights(self, pretrained=None): + if isinstance(pretrained, str): + logger = get_root_logger() + checkpoint = _load_checkpoint(pretrained, logger=logger) + if 'state_dict' in checkpoint: + state_dict = checkpoint['state_dict'] + else: + state_dict = checkpoint + + if 'pos_embed' in state_dict.keys(): + if self.pos_embed.shape != state_dict['pos_embed'].shape: + logger.info(msg=f'Resize the pos_embed shape from \ +{state_dict["pos_embed"].shape} to {self.pos_embed.shape}') + h, w = self.img_size + pos_size = int( + math.sqrt(state_dict['pos_embed'].shape[1] - 1)) + state_dict['pos_embed'] = self.resize_pos_embed( + state_dict['pos_embed'], (h, w), (pos_size, pos_size), + self.patch_size, self.interpolate_mode) + + self.load_state_dict(state_dict, False) + + elif pretrained is None: + # We only implement the 'jax_impl' initialization implemented at + # https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py#L353 # noqa: E501 + trunc_normal_(self.pos_embed, std=.02) + trunc_normal_(self.cls_token, std=.02) + for n, m in self.named_modules(): + if isinstance(m, Linear): + trunc_normal_(m.weight, std=.02) + if m.bias is not None: + if 'mlp' in n: + normal_init(m.bias, std=1e-6) + else: + constant_init(m.bias, 0) + elif isinstance(m, Conv2d): + kaiming_init(m.weight, mode='fan_in') + if m.bias is not None: + constant_init(m.bias, 0) + elif isinstance(m, (_BatchNorm, nn.GroupNorm, nn.LayerNorm)): + constant_init(m.bias, 0) + constant_init(m.weight, 1.0) + else: + raise TypeError('pretrained must be a str or None') + + def _pos_embeding(self, img, patched_img, pos_embed): + """Positiong embeding method. + + Resize the pos_embed, if the input image size doesn't match + the training size. + Args: + img (torch.Tensor): The inference image tensor, the shape + must be [B, C, H, W]. + patched_img (torch.Tensor): The patched image, it should be + shape of [B, L1, C]. + pos_embed (torch.Tensor): The pos_embed weighs, it should be + shape of [B, L2, c]. + Return: + torch.Tensor: The pos encoded image feature. + """ + assert patched_img.ndim == 3 and pos_embed.ndim == 3, \ + 'the shapes of patched_img and pos_embed must be [B, L, C]' + x_len, pos_len = patched_img.shape[1], pos_embed.shape[1] + if x_len != pos_len: + if pos_len == (self.img_size[0] // self.patch_size) * ( + self.img_size[1] // self.patch_size) + 1: + pos_h = self.img_size[0] // self.patch_size + pos_w = self.img_size[1] // self.patch_size + else: + raise ValueError( + 'Unexpected shape of pos_embed, got {}.'.format( + pos_embed.shape)) + pos_embed = self.resize_pos_embed(pos_embed, img.shape[2:], + (pos_h, pos_w), self.patch_size, + self.interpolate_mode) + return self.pos_drop(patched_img + pos_embed) + + @staticmethod + def resize_pos_embed(pos_embed, input_shpae, pos_shape, patch_size, mode): + """Resize pos_embed weights. + + Resize pos_embed using bicubic interpolate method. + Args: + pos_embed (torch.Tensor): pos_embed weights. + input_shpae (tuple): Tuple for (input_h, intput_w). + pos_shape (tuple): Tuple for (pos_h, pos_w). + patch_size (int): Patch size. + Return: + torch.Tensor: The resized pos_embed of shape [B, L_new, C] + """ + assert pos_embed.ndim == 3, 'shape of pos_embed must be [B, L, C]' + input_h, input_w = input_shpae + pos_h, pos_w = pos_shape + cls_token_weight = pos_embed[:, 0] + pos_embed_weight = pos_embed[:, (-1 * pos_h * pos_w):] + pos_embed_weight = pos_embed_weight.reshape( + 1, pos_h, pos_w, pos_embed.shape[2]).permute(0, 3, 1, 2) + pos_embed_weight = F.interpolate( + pos_embed_weight, + size=[input_h // patch_size, input_w // patch_size], + align_corners=False, + mode=mode) + cls_token_weight = cls_token_weight.unsqueeze(1) + pos_embed_weight = torch.flatten(pos_embed_weight, 2).transpose(1, 2) + pos_embed = torch.cat((cls_token_weight, pos_embed_weight), dim=1) + return pos_embed + + def forward(self, inputs): + B = inputs.shape[0] + + x = self.patch_embed(inputs) + + cls_tokens = self.cls_token.expand(B, -1, -1) + x = torch.cat((cls_tokens, x), dim=1) + x = self._pos_embeding(inputs, x, self.pos_embed) + + if not self.with_cls_token: + # Remove class token for transformer input + x = x[:, 1:] + + outs = [] + for i, blk in enumerate(self.blocks): + x = blk(x) + if i == len(self.blocks) - 1: + if self.final_norm: + x = self.norm(x) + if i in self.out_indices: + if self.with_cls_token: + # Remove class token and reshape token for decoder head + out = x[:, 1:] + else: + out = x + B, _, C = out.shape + out = out.reshape(B, inputs.shape[2] // self.patch_size, + inputs.shape[3] // self.patch_size, + C).permute(0, 3, 1, 2) + outs.append(out) + + return tuple(outs) + + def train(self, mode=True): + super(VisionTransformer, self).train(mode) + if mode and self.norm_eval: + for m in self.modules(): + if isinstance(m, nn.LayerNorm): + m.eval() diff --git a/annotator/uniformer/mmseg/models/builder.py b/annotator/uniformer/mmseg/models/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..1f5b971252bfc971c3ffbaa27746d69b1d3ea9fd --- /dev/null +++ b/annotator/uniformer/mmseg/models/builder.py @@ -0,0 +1,46 @@ +import warnings + +from annotator.uniformer.mmcv.cnn import MODELS as MMCV_MODELS +from annotator.uniformer.mmcv.utils import Registry + +MODELS = Registry('models', parent=MMCV_MODELS) + +BACKBONES = MODELS +NECKS = MODELS +HEADS = MODELS +LOSSES = MODELS +SEGMENTORS = MODELS + + +def build_backbone(cfg): + """Build backbone.""" + return BACKBONES.build(cfg) + + +def build_neck(cfg): + """Build neck.""" + return NECKS.build(cfg) + + +def build_head(cfg): + """Build head.""" + return HEADS.build(cfg) + + +def build_loss(cfg): + """Build loss.""" + return LOSSES.build(cfg) + + +def build_segmentor(cfg, train_cfg=None, test_cfg=None): + """Build segmentor.""" + if train_cfg is not None or test_cfg is not None: + warnings.warn( + 'train_cfg and test_cfg is deprecated, ' + 'please specify them in model', UserWarning) + assert cfg.get('train_cfg') is None or train_cfg is None, \ + 'train_cfg specified in both outer field and model field ' + assert cfg.get('test_cfg') is None or test_cfg is None, \ + 'test_cfg specified in both outer field and model field ' + return SEGMENTORS.build( + cfg, default_args=dict(train_cfg=train_cfg, test_cfg=test_cfg)) diff --git a/annotator/uniformer/mmseg/models/decode_heads/__init__.py b/annotator/uniformer/mmseg/models/decode_heads/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ac66d3cfe0ea04af45c0f3594bf135841c3812e3 --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/__init__.py @@ -0,0 +1,28 @@ +from .ann_head import ANNHead +from .apc_head import APCHead +from .aspp_head import ASPPHead +from .cc_head import CCHead +from .da_head import DAHead +from .dm_head import DMHead +from .dnl_head import DNLHead +from .ema_head import EMAHead +from .enc_head import EncHead +from .fcn_head import FCNHead +from .fpn_head import FPNHead +from .gc_head import GCHead +from .lraspp_head import LRASPPHead +from .nl_head import NLHead +from .ocr_head import OCRHead +# from .point_head import PointHead +from .psa_head import PSAHead +from .psp_head import PSPHead +from .sep_aspp_head import DepthwiseSeparableASPPHead +from .sep_fcn_head import DepthwiseSeparableFCNHead +from .uper_head import UPerHead + +__all__ = [ + 'FCNHead', 'PSPHead', 'ASPPHead', 'PSAHead', 'NLHead', 'GCHead', 'CCHead', + 'UPerHead', 'DepthwiseSeparableASPPHead', 'ANNHead', 'DAHead', 'OCRHead', + 'EncHead', 'DepthwiseSeparableFCNHead', 'FPNHead', 'EMAHead', 'DNLHead', + 'APCHead', 'DMHead', 'LRASPPHead' +] diff --git a/annotator/uniformer/mmseg/models/decode_heads/ann_head.py b/annotator/uniformer/mmseg/models/decode_heads/ann_head.py new file mode 100644 index 0000000000000000000000000000000000000000..30aaacc2cafc568d3de71d1477b4de0dc0fea9d3 --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/ann_head.py @@ -0,0 +1,245 @@ +import torch +import torch.nn as nn +from annotator.uniformer.mmcv.cnn import ConvModule + +from ..builder import HEADS +from ..utils import SelfAttentionBlock as _SelfAttentionBlock +from .decode_head import BaseDecodeHead + + +class PPMConcat(nn.ModuleList): + """Pyramid Pooling Module that only concat the features of each layer. + + Args: + pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module. + """ + + def __init__(self, pool_scales=(1, 3, 6, 8)): + super(PPMConcat, self).__init__( + [nn.AdaptiveAvgPool2d(pool_scale) for pool_scale in pool_scales]) + + def forward(self, feats): + """Forward function.""" + ppm_outs = [] + for ppm in self: + ppm_out = ppm(feats) + ppm_outs.append(ppm_out.view(*feats.shape[:2], -1)) + concat_outs = torch.cat(ppm_outs, dim=2) + return concat_outs + + +class SelfAttentionBlock(_SelfAttentionBlock): + """Make a ANN used SelfAttentionBlock. + + Args: + low_in_channels (int): Input channels of lower level feature, + which is the key feature for self-attention. + high_in_channels (int): Input channels of higher level feature, + which is the query feature for self-attention. + channels (int): Output channels of key/query transform. + out_channels (int): Output channels. + share_key_query (bool): Whether share projection weight between key + and query projection. + query_scale (int): The scale of query feature map. + key_pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module of key feature. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict|None): Config of activation layers. + """ + + def __init__(self, low_in_channels, high_in_channels, channels, + out_channels, share_key_query, query_scale, key_pool_scales, + conv_cfg, norm_cfg, act_cfg): + key_psp = PPMConcat(key_pool_scales) + if query_scale > 1: + query_downsample = nn.MaxPool2d(kernel_size=query_scale) + else: + query_downsample = None + super(SelfAttentionBlock, self).__init__( + key_in_channels=low_in_channels, + query_in_channels=high_in_channels, + channels=channels, + out_channels=out_channels, + share_key_query=share_key_query, + query_downsample=query_downsample, + key_downsample=key_psp, + key_query_num_convs=1, + key_query_norm=True, + value_out_num_convs=1, + value_out_norm=False, + matmul_norm=True, + with_out=True, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + + +class AFNB(nn.Module): + """Asymmetric Fusion Non-local Block(AFNB) + + Args: + low_in_channels (int): Input channels of lower level feature, + which is the key feature for self-attention. + high_in_channels (int): Input channels of higher level feature, + which is the query feature for self-attention. + channels (int): Output channels of key/query transform. + out_channels (int): Output channels. + and query projection. + query_scales (tuple[int]): The scales of query feature map. + Default: (1,) + key_pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module of key feature. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict|None): Config of activation layers. + """ + + def __init__(self, low_in_channels, high_in_channels, channels, + out_channels, query_scales, key_pool_scales, conv_cfg, + norm_cfg, act_cfg): + super(AFNB, self).__init__() + self.stages = nn.ModuleList() + for query_scale in query_scales: + self.stages.append( + SelfAttentionBlock( + low_in_channels=low_in_channels, + high_in_channels=high_in_channels, + channels=channels, + out_channels=out_channels, + share_key_query=False, + query_scale=query_scale, + key_pool_scales=key_pool_scales, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + self.bottleneck = ConvModule( + out_channels + high_in_channels, + out_channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=None) + + def forward(self, low_feats, high_feats): + """Forward function.""" + priors = [stage(high_feats, low_feats) for stage in self.stages] + context = torch.stack(priors, dim=0).sum(dim=0) + output = self.bottleneck(torch.cat([context, high_feats], 1)) + return output + + +class APNB(nn.Module): + """Asymmetric Pyramid Non-local Block (APNB) + + Args: + in_channels (int): Input channels of key/query feature, + which is the key feature for self-attention. + channels (int): Output channels of key/query transform. + out_channels (int): Output channels. + query_scales (tuple[int]): The scales of query feature map. + Default: (1,) + key_pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module of key feature. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict|None): Config of activation layers. + """ + + def __init__(self, in_channels, channels, out_channels, query_scales, + key_pool_scales, conv_cfg, norm_cfg, act_cfg): + super(APNB, self).__init__() + self.stages = nn.ModuleList() + for query_scale in query_scales: + self.stages.append( + SelfAttentionBlock( + low_in_channels=in_channels, + high_in_channels=in_channels, + channels=channels, + out_channels=out_channels, + share_key_query=True, + query_scale=query_scale, + key_pool_scales=key_pool_scales, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + self.bottleneck = ConvModule( + 2 * in_channels, + out_channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + + def forward(self, feats): + """Forward function.""" + priors = [stage(feats, feats) for stage in self.stages] + context = torch.stack(priors, dim=0).sum(dim=0) + output = self.bottleneck(torch.cat([context, feats], 1)) + return output + + +@HEADS.register_module() +class ANNHead(BaseDecodeHead): + """Asymmetric Non-local Neural Networks for Semantic Segmentation. + + This head is the implementation of `ANNNet + `_. + + Args: + project_channels (int): Projection channels for Nonlocal. + query_scales (tuple[int]): The scales of query feature map. + Default: (1,) + key_pool_scales (tuple[int]): The pooling scales of key feature map. + Default: (1, 3, 6, 8). + """ + + def __init__(self, + project_channels, + query_scales=(1, ), + key_pool_scales=(1, 3, 6, 8), + **kwargs): + super(ANNHead, self).__init__( + input_transform='multiple_select', **kwargs) + assert len(self.in_channels) == 2 + low_in_channels, high_in_channels = self.in_channels + self.project_channels = project_channels + self.fusion = AFNB( + low_in_channels=low_in_channels, + high_in_channels=high_in_channels, + out_channels=high_in_channels, + channels=project_channels, + query_scales=query_scales, + key_pool_scales=key_pool_scales, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.bottleneck = ConvModule( + high_in_channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.context = APNB( + in_channels=self.channels, + out_channels=self.channels, + channels=project_channels, + query_scales=query_scales, + key_pool_scales=key_pool_scales, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + low_feats, high_feats = self._transform_inputs(inputs) + output = self.fusion(low_feats, high_feats) + output = self.dropout(output) + output = self.bottleneck(output) + output = self.context(output) + output = self.cls_seg(output) + + return output diff --git a/annotator/uniformer/mmseg/models/decode_heads/apc_head.py b/annotator/uniformer/mmseg/models/decode_heads/apc_head.py new file mode 100644 index 0000000000000000000000000000000000000000..c7038bdbe0edf2a1f184b6899486d2d190dda076 --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/apc_head.py @@ -0,0 +1,158 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from annotator.uniformer.mmcv.cnn import ConvModule + +from annotator.uniformer.mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +class ACM(nn.Module): + """Adaptive Context Module used in APCNet. + + Args: + pool_scale (int): Pooling scale used in Adaptive Context + Module to extract region features. + fusion (bool): Add one conv to fuse residual feature. + in_channels (int): Input channels. + channels (int): Channels after modules, before conv_seg. + conv_cfg (dict | None): Config of conv layers. + norm_cfg (dict | None): Config of norm layers. + act_cfg (dict): Config of activation layers. + """ + + def __init__(self, pool_scale, fusion, in_channels, channels, conv_cfg, + norm_cfg, act_cfg): + super(ACM, self).__init__() + self.pool_scale = pool_scale + self.fusion = fusion + self.in_channels = in_channels + self.channels = channels + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.pooled_redu_conv = ConvModule( + self.in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + self.input_redu_conv = ConvModule( + self.in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + self.global_info = ConvModule( + self.channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + self.gla = nn.Conv2d(self.channels, self.pool_scale**2, 1, 1, 0) + + self.residual_conv = ConvModule( + self.channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + if self.fusion: + self.fusion_conv = ConvModule( + self.channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, x): + """Forward function.""" + pooled_x = F.adaptive_avg_pool2d(x, self.pool_scale) + # [batch_size, channels, h, w] + x = self.input_redu_conv(x) + # [batch_size, channels, pool_scale, pool_scale] + pooled_x = self.pooled_redu_conv(pooled_x) + batch_size = x.size(0) + # [batch_size, pool_scale * pool_scale, channels] + pooled_x = pooled_x.view(batch_size, self.channels, + -1).permute(0, 2, 1).contiguous() + # [batch_size, h * w, pool_scale * pool_scale] + affinity_matrix = self.gla(x + resize( + self.global_info(F.adaptive_avg_pool2d(x, 1)), size=x.shape[2:]) + ).permute(0, 2, 3, 1).reshape( + batch_size, -1, self.pool_scale**2) + affinity_matrix = F.sigmoid(affinity_matrix) + # [batch_size, h * w, channels] + z_out = torch.matmul(affinity_matrix, pooled_x) + # [batch_size, channels, h * w] + z_out = z_out.permute(0, 2, 1).contiguous() + # [batch_size, channels, h, w] + z_out = z_out.view(batch_size, self.channels, x.size(2), x.size(3)) + z_out = self.residual_conv(z_out) + z_out = F.relu(z_out + x) + if self.fusion: + z_out = self.fusion_conv(z_out) + + return z_out + + +@HEADS.register_module() +class APCHead(BaseDecodeHead): + """Adaptive Pyramid Context Network for Semantic Segmentation. + + This head is the implementation of + `APCNet `_. + + Args: + pool_scales (tuple[int]): Pooling scales used in Adaptive Context + Module. Default: (1, 2, 3, 6). + fusion (bool): Add one conv to fuse residual feature. + """ + + def __init__(self, pool_scales=(1, 2, 3, 6), fusion=True, **kwargs): + super(APCHead, self).__init__(**kwargs) + assert isinstance(pool_scales, (list, tuple)) + self.pool_scales = pool_scales + self.fusion = fusion + acm_modules = [] + for pool_scale in self.pool_scales: + acm_modules.append( + ACM(pool_scale, + self.fusion, + self.in_channels, + self.channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + self.acm_modules = nn.ModuleList(acm_modules) + self.bottleneck = ConvModule( + self.in_channels + len(pool_scales) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + acm_outs = [x] + for acm_module in self.acm_modules: + acm_outs.append(acm_module(x)) + acm_outs = torch.cat(acm_outs, dim=1) + output = self.bottleneck(acm_outs) + output = self.cls_seg(output) + return output diff --git a/annotator/uniformer/mmseg/models/decode_heads/aspp_head.py b/annotator/uniformer/mmseg/models/decode_heads/aspp_head.py new file mode 100644 index 0000000000000000000000000000000000000000..aa914b5bb25124d1ff199553d96713d6a80484c0 --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/aspp_head.py @@ -0,0 +1,107 @@ +import torch +import torch.nn as nn +from annotator.uniformer.mmcv.cnn import ConvModule + +from annotator.uniformer.mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +class ASPPModule(nn.ModuleList): + """Atrous Spatial Pyramid Pooling (ASPP) Module. + + Args: + dilations (tuple[int]): Dilation rate of each layer. + in_channels (int): Input channels. + channels (int): Channels after modules, before conv_seg. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict): Config of activation layers. + """ + + def __init__(self, dilations, in_channels, channels, conv_cfg, norm_cfg, + act_cfg): + super(ASPPModule, self).__init__() + self.dilations = dilations + self.in_channels = in_channels + self.channels = channels + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + for dilation in dilations: + self.append( + ConvModule( + self.in_channels, + self.channels, + 1 if dilation == 1 else 3, + dilation=dilation, + padding=0 if dilation == 1 else dilation, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + + def forward(self, x): + """Forward function.""" + aspp_outs = [] + for aspp_module in self: + aspp_outs.append(aspp_module(x)) + + return aspp_outs + + +@HEADS.register_module() +class ASPPHead(BaseDecodeHead): + """Rethinking Atrous Convolution for Semantic Image Segmentation. + + This head is the implementation of `DeepLabV3 + `_. + + Args: + dilations (tuple[int]): Dilation rates for ASPP module. + Default: (1, 6, 12, 18). + """ + + def __init__(self, dilations=(1, 6, 12, 18), **kwargs): + super(ASPPHead, self).__init__(**kwargs) + assert isinstance(dilations, (list, tuple)) + self.dilations = dilations + self.image_pool = nn.Sequential( + nn.AdaptiveAvgPool2d(1), + ConvModule( + self.in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + self.aspp_modules = ASPPModule( + dilations, + self.in_channels, + self.channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.bottleneck = ConvModule( + (len(dilations) + 1) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + aspp_outs = [ + resize( + self.image_pool(x), + size=x.size()[2:], + mode='bilinear', + align_corners=self.align_corners) + ] + aspp_outs.extend(self.aspp_modules(x)) + aspp_outs = torch.cat(aspp_outs, dim=1) + output = self.bottleneck(aspp_outs) + output = self.cls_seg(output) + return output diff --git a/annotator/uniformer/mmseg/models/decode_heads/cascade_decode_head.py b/annotator/uniformer/mmseg/models/decode_heads/cascade_decode_head.py new file mode 100644 index 0000000000000000000000000000000000000000..d02122ca0e68743b1bf7a893afae96042f23838c --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/cascade_decode_head.py @@ -0,0 +1,57 @@ +from abc import ABCMeta, abstractmethod + +from .decode_head import BaseDecodeHead + + +class BaseCascadeDecodeHead(BaseDecodeHead, metaclass=ABCMeta): + """Base class for cascade decode head used in + :class:`CascadeEncoderDecoder.""" + + def __init__(self, *args, **kwargs): + super(BaseCascadeDecodeHead, self).__init__(*args, **kwargs) + + @abstractmethod + def forward(self, inputs, prev_output): + """Placeholder of forward function.""" + pass + + def forward_train(self, inputs, prev_output, img_metas, gt_semantic_seg, + train_cfg): + """Forward function for training. + Args: + inputs (list[Tensor]): List of multi-level img features. + prev_output (Tensor): The output of previous decode head. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + gt_semantic_seg (Tensor): Semantic segmentation masks + used if the architecture supports semantic segmentation task. + train_cfg (dict): The training config. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + seg_logits = self.forward(inputs, prev_output) + losses = self.losses(seg_logits, gt_semantic_seg) + + return losses + + def forward_test(self, inputs, prev_output, img_metas, test_cfg): + """Forward function for testing. + + Args: + inputs (list[Tensor]): List of multi-level img features. + prev_output (Tensor): The output of previous decode head. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + test_cfg (dict): The testing config. + + Returns: + Tensor: Output segmentation map. + """ + return self.forward(inputs, prev_output) diff --git a/annotator/uniformer/mmseg/models/decode_heads/cc_head.py b/annotator/uniformer/mmseg/models/decode_heads/cc_head.py new file mode 100644 index 0000000000000000000000000000000000000000..5b9abb4e747f92657f4220b29788539340986c00 --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/cc_head.py @@ -0,0 +1,42 @@ +import torch + +from ..builder import HEADS +from .fcn_head import FCNHead + +try: + from annotator.uniformer.mmcv.ops import CrissCrossAttention +except ModuleNotFoundError: + CrissCrossAttention = None + + +@HEADS.register_module() +class CCHead(FCNHead): + """CCNet: Criss-Cross Attention for Semantic Segmentation. + + This head is the implementation of `CCNet + `_. + + Args: + recurrence (int): Number of recurrence of Criss Cross Attention + module. Default: 2. + """ + + def __init__(self, recurrence=2, **kwargs): + if CrissCrossAttention is None: + raise RuntimeError('Please install mmcv-full for ' + 'CrissCrossAttention ops') + super(CCHead, self).__init__(num_convs=2, **kwargs) + self.recurrence = recurrence + self.cca = CrissCrossAttention(self.channels) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + output = self.convs[0](x) + for _ in range(self.recurrence): + output = self.cca(output) + output = self.convs[1](output) + if self.concat_input: + output = self.conv_cat(torch.cat([x, output], dim=1)) + output = self.cls_seg(output) + return output diff --git a/annotator/uniformer/mmseg/models/decode_heads/da_head.py b/annotator/uniformer/mmseg/models/decode_heads/da_head.py new file mode 100644 index 0000000000000000000000000000000000000000..5cd49fcfdc7c0a70f9485cc71843dcf3e0cb1774 --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/da_head.py @@ -0,0 +1,178 @@ +import torch +import torch.nn.functional as F +from annotator.uniformer.mmcv.cnn import ConvModule, Scale +from torch import nn + +from annotator.uniformer.mmseg.core import add_prefix +from ..builder import HEADS +from ..utils import SelfAttentionBlock as _SelfAttentionBlock +from .decode_head import BaseDecodeHead + + +class PAM(_SelfAttentionBlock): + """Position Attention Module (PAM) + + Args: + in_channels (int): Input channels of key/query feature. + channels (int): Output channels of key/query transform. + """ + + def __init__(self, in_channels, channels): + super(PAM, self).__init__( + key_in_channels=in_channels, + query_in_channels=in_channels, + channels=channels, + out_channels=in_channels, + share_key_query=False, + query_downsample=None, + key_downsample=None, + key_query_num_convs=1, + key_query_norm=False, + value_out_num_convs=1, + value_out_norm=False, + matmul_norm=False, + with_out=False, + conv_cfg=None, + norm_cfg=None, + act_cfg=None) + + self.gamma = Scale(0) + + def forward(self, x): + """Forward function.""" + out = super(PAM, self).forward(x, x) + + out = self.gamma(out) + x + return out + + +class CAM(nn.Module): + """Channel Attention Module (CAM)""" + + def __init__(self): + super(CAM, self).__init__() + self.gamma = Scale(0) + + def forward(self, x): + """Forward function.""" + batch_size, channels, height, width = x.size() + proj_query = x.view(batch_size, channels, -1) + proj_key = x.view(batch_size, channels, -1).permute(0, 2, 1) + energy = torch.bmm(proj_query, proj_key) + energy_new = torch.max( + energy, -1, keepdim=True)[0].expand_as(energy) - energy + attention = F.softmax(energy_new, dim=-1) + proj_value = x.view(batch_size, channels, -1) + + out = torch.bmm(attention, proj_value) + out = out.view(batch_size, channels, height, width) + + out = self.gamma(out) + x + return out + + +@HEADS.register_module() +class DAHead(BaseDecodeHead): + """Dual Attention Network for Scene Segmentation. + + This head is the implementation of `DANet + `_. + + Args: + pam_channels (int): The channels of Position Attention Module(PAM). + """ + + def __init__(self, pam_channels, **kwargs): + super(DAHead, self).__init__(**kwargs) + self.pam_channels = pam_channels + self.pam_in_conv = ConvModule( + self.in_channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.pam = PAM(self.channels, pam_channels) + self.pam_out_conv = ConvModule( + self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.pam_conv_seg = nn.Conv2d( + self.channels, self.num_classes, kernel_size=1) + + self.cam_in_conv = ConvModule( + self.in_channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.cam = CAM() + self.cam_out_conv = ConvModule( + self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.cam_conv_seg = nn.Conv2d( + self.channels, self.num_classes, kernel_size=1) + + def pam_cls_seg(self, feat): + """PAM feature classification.""" + if self.dropout is not None: + feat = self.dropout(feat) + output = self.pam_conv_seg(feat) + return output + + def cam_cls_seg(self, feat): + """CAM feature classification.""" + if self.dropout is not None: + feat = self.dropout(feat) + output = self.cam_conv_seg(feat) + return output + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + pam_feat = self.pam_in_conv(x) + pam_feat = self.pam(pam_feat) + pam_feat = self.pam_out_conv(pam_feat) + pam_out = self.pam_cls_seg(pam_feat) + + cam_feat = self.cam_in_conv(x) + cam_feat = self.cam(cam_feat) + cam_feat = self.cam_out_conv(cam_feat) + cam_out = self.cam_cls_seg(cam_feat) + + feat_sum = pam_feat + cam_feat + pam_cam_out = self.cls_seg(feat_sum) + + return pam_cam_out, pam_out, cam_out + + def forward_test(self, inputs, img_metas, test_cfg): + """Forward function for testing, only ``pam_cam`` is used.""" + return self.forward(inputs)[0] + + def losses(self, seg_logit, seg_label): + """Compute ``pam_cam``, ``pam``, ``cam`` loss.""" + pam_cam_seg_logit, pam_seg_logit, cam_seg_logit = seg_logit + loss = dict() + loss.update( + add_prefix( + super(DAHead, self).losses(pam_cam_seg_logit, seg_label), + 'pam_cam')) + loss.update( + add_prefix( + super(DAHead, self).losses(pam_seg_logit, seg_label), 'pam')) + loss.update( + add_prefix( + super(DAHead, self).losses(cam_seg_logit, seg_label), 'cam')) + return loss diff --git a/annotator/uniformer/mmseg/models/decode_heads/decode_head.py b/annotator/uniformer/mmseg/models/decode_heads/decode_head.py new file mode 100644 index 0000000000000000000000000000000000000000..88a661b8f6fec5d4c031d3d85e80777ee63951a6 --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/decode_head.py @@ -0,0 +1,234 @@ +from abc import ABCMeta, abstractmethod + +import torch +import torch.nn as nn +from annotator.uniformer.mmcv.cnn import normal_init +from annotator.uniformer.mmcv.runner import auto_fp16, force_fp32 + +from annotator.uniformer.mmseg.core import build_pixel_sampler +from annotator.uniformer.mmseg.ops import resize +from ..builder import build_loss +from ..losses import accuracy + + +class BaseDecodeHead(nn.Module, metaclass=ABCMeta): + """Base class for BaseDecodeHead. + + Args: + in_channels (int|Sequence[int]): Input channels. + channels (int): Channels after modules, before conv_seg. + num_classes (int): Number of classes. + dropout_ratio (float): Ratio of dropout layer. Default: 0.1. + conv_cfg (dict|None): Config of conv layers. Default: None. + norm_cfg (dict|None): Config of norm layers. Default: None. + act_cfg (dict): Config of activation layers. + Default: dict(type='ReLU') + in_index (int|Sequence[int]): Input feature index. Default: -1 + input_transform (str|None): Transformation type of input features. + Options: 'resize_concat', 'multiple_select', None. + 'resize_concat': Multiple feature maps will be resize to the + same size as first one and than concat together. + Usually used in FCN head of HRNet. + 'multiple_select': Multiple feature maps will be bundle into + a list and passed into decode head. + None: Only one select feature map is allowed. + Default: None. + loss_decode (dict): Config of decode loss. + Default: dict(type='CrossEntropyLoss'). + ignore_index (int | None): The label index to be ignored. When using + masked BCE loss, ignore_index should be set to None. Default: 255 + sampler (dict|None): The config of segmentation map sampler. + Default: None. + align_corners (bool): align_corners argument of F.interpolate. + Default: False. + """ + + def __init__(self, + in_channels, + channels, + *, + num_classes, + dropout_ratio=0.1, + conv_cfg=None, + norm_cfg=None, + act_cfg=dict(type='ReLU'), + in_index=-1, + input_transform=None, + loss_decode=dict( + type='CrossEntropyLoss', + use_sigmoid=False, + loss_weight=1.0), + ignore_index=255, + sampler=None, + align_corners=False): + super(BaseDecodeHead, self).__init__() + self._init_inputs(in_channels, in_index, input_transform) + self.channels = channels + self.num_classes = num_classes + self.dropout_ratio = dropout_ratio + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.in_index = in_index + self.loss_decode = build_loss(loss_decode) + self.ignore_index = ignore_index + self.align_corners = align_corners + if sampler is not None: + self.sampler = build_pixel_sampler(sampler, context=self) + else: + self.sampler = None + + self.conv_seg = nn.Conv2d(channels, num_classes, kernel_size=1) + if dropout_ratio > 0: + self.dropout = nn.Dropout2d(dropout_ratio) + else: + self.dropout = None + self.fp16_enabled = False + + def extra_repr(self): + """Extra repr.""" + s = f'input_transform={self.input_transform}, ' \ + f'ignore_index={self.ignore_index}, ' \ + f'align_corners={self.align_corners}' + return s + + def _init_inputs(self, in_channels, in_index, input_transform): + """Check and initialize input transforms. + + The in_channels, in_index and input_transform must match. + Specifically, when input_transform is None, only single feature map + will be selected. So in_channels and in_index must be of type int. + When input_transform + + Args: + in_channels (int|Sequence[int]): Input channels. + in_index (int|Sequence[int]): Input feature index. + input_transform (str|None): Transformation type of input features. + Options: 'resize_concat', 'multiple_select', None. + 'resize_concat': Multiple feature maps will be resize to the + same size as first one and than concat together. + Usually used in FCN head of HRNet. + 'multiple_select': Multiple feature maps will be bundle into + a list and passed into decode head. + None: Only one select feature map is allowed. + """ + + if input_transform is not None: + assert input_transform in ['resize_concat', 'multiple_select'] + self.input_transform = input_transform + self.in_index = in_index + if input_transform is not None: + assert isinstance(in_channels, (list, tuple)) + assert isinstance(in_index, (list, tuple)) + assert len(in_channels) == len(in_index) + if input_transform == 'resize_concat': + self.in_channels = sum(in_channels) + else: + self.in_channels = in_channels + else: + assert isinstance(in_channels, int) + assert isinstance(in_index, int) + self.in_channels = in_channels + + def init_weights(self): + """Initialize weights of classification layer.""" + normal_init(self.conv_seg, mean=0, std=0.01) + + def _transform_inputs(self, inputs): + """Transform inputs for decoder. + + Args: + inputs (list[Tensor]): List of multi-level img features. + + Returns: + Tensor: The transformed inputs + """ + + if self.input_transform == 'resize_concat': + inputs = [inputs[i] for i in self.in_index] + upsampled_inputs = [ + resize( + input=x, + size=inputs[0].shape[2:], + mode='bilinear', + align_corners=self.align_corners) for x in inputs + ] + inputs = torch.cat(upsampled_inputs, dim=1) + elif self.input_transform == 'multiple_select': + inputs = [inputs[i] for i in self.in_index] + else: + inputs = inputs[self.in_index] + + return inputs + + @auto_fp16() + @abstractmethod + def forward(self, inputs): + """Placeholder of forward function.""" + pass + + def forward_train(self, inputs, img_metas, gt_semantic_seg, train_cfg): + """Forward function for training. + Args: + inputs (list[Tensor]): List of multi-level img features. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + gt_semantic_seg (Tensor): Semantic segmentation masks + used if the architecture supports semantic segmentation task. + train_cfg (dict): The training config. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + seg_logits = self.forward(inputs) + losses = self.losses(seg_logits, gt_semantic_seg) + return losses + + def forward_test(self, inputs, img_metas, test_cfg): + """Forward function for testing. + + Args: + inputs (list[Tensor]): List of multi-level img features. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + test_cfg (dict): The testing config. + + Returns: + Tensor: Output segmentation map. + """ + return self.forward(inputs) + + def cls_seg(self, feat): + """Classify each pixel.""" + if self.dropout is not None: + feat = self.dropout(feat) + output = self.conv_seg(feat) + return output + + @force_fp32(apply_to=('seg_logit', )) + def losses(self, seg_logit, seg_label): + """Compute segmentation loss.""" + loss = dict() + seg_logit = resize( + input=seg_logit, + size=seg_label.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + if self.sampler is not None: + seg_weight = self.sampler.sample(seg_logit, seg_label) + else: + seg_weight = None + seg_label = seg_label.squeeze(1) + loss['loss_seg'] = self.loss_decode( + seg_logit, + seg_label, + weight=seg_weight, + ignore_index=self.ignore_index) + loss['acc_seg'] = accuracy(seg_logit, seg_label) + return loss diff --git a/annotator/uniformer/mmseg/models/decode_heads/dm_head.py b/annotator/uniformer/mmseg/models/decode_heads/dm_head.py new file mode 100644 index 0000000000000000000000000000000000000000..19c963923126b53ce22f60813540a35badf24b3d --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/dm_head.py @@ -0,0 +1,140 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from annotator.uniformer.mmcv.cnn import ConvModule, build_activation_layer, build_norm_layer + +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +class DCM(nn.Module): + """Dynamic Convolutional Module used in DMNet. + + Args: + filter_size (int): The filter size of generated convolution kernel + used in Dynamic Convolutional Module. + fusion (bool): Add one conv to fuse DCM output feature. + in_channels (int): Input channels. + channels (int): Channels after modules, before conv_seg. + conv_cfg (dict | None): Config of conv layers. + norm_cfg (dict | None): Config of norm layers. + act_cfg (dict): Config of activation layers. + """ + + def __init__(self, filter_size, fusion, in_channels, channels, conv_cfg, + norm_cfg, act_cfg): + super(DCM, self).__init__() + self.filter_size = filter_size + self.fusion = fusion + self.in_channels = in_channels + self.channels = channels + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.filter_gen_conv = nn.Conv2d(self.in_channels, self.channels, 1, 1, + 0) + + self.input_redu_conv = ConvModule( + self.in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + if self.norm_cfg is not None: + self.norm = build_norm_layer(self.norm_cfg, self.channels)[1] + else: + self.norm = None + self.activate = build_activation_layer(self.act_cfg) + + if self.fusion: + self.fusion_conv = ConvModule( + self.channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, x): + """Forward function.""" + generated_filter = self.filter_gen_conv( + F.adaptive_avg_pool2d(x, self.filter_size)) + x = self.input_redu_conv(x) + b, c, h, w = x.shape + # [1, b * c, h, w], c = self.channels + x = x.view(1, b * c, h, w) + # [b * c, 1, filter_size, filter_size] + generated_filter = generated_filter.view(b * c, 1, self.filter_size, + self.filter_size) + pad = (self.filter_size - 1) // 2 + if (self.filter_size - 1) % 2 == 0: + p2d = (pad, pad, pad, pad) + else: + p2d = (pad + 1, pad, pad + 1, pad) + x = F.pad(input=x, pad=p2d, mode='constant', value=0) + # [1, b * c, h, w] + output = F.conv2d(input=x, weight=generated_filter, groups=b * c) + # [b, c, h, w] + output = output.view(b, c, h, w) + if self.norm is not None: + output = self.norm(output) + output = self.activate(output) + + if self.fusion: + output = self.fusion_conv(output) + + return output + + +@HEADS.register_module() +class DMHead(BaseDecodeHead): + """Dynamic Multi-scale Filters for Semantic Segmentation. + + This head is the implementation of + `DMNet `_. + + Args: + filter_sizes (tuple[int]): The size of generated convolutional filters + used in Dynamic Convolutional Module. Default: (1, 3, 5, 7). + fusion (bool): Add one conv to fuse DCM output feature. + """ + + def __init__(self, filter_sizes=(1, 3, 5, 7), fusion=False, **kwargs): + super(DMHead, self).__init__(**kwargs) + assert isinstance(filter_sizes, (list, tuple)) + self.filter_sizes = filter_sizes + self.fusion = fusion + dcm_modules = [] + for filter_size in self.filter_sizes: + dcm_modules.append( + DCM(filter_size, + self.fusion, + self.in_channels, + self.channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + self.dcm_modules = nn.ModuleList(dcm_modules) + self.bottleneck = ConvModule( + self.in_channels + len(filter_sizes) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + dcm_outs = [x] + for dcm_module in self.dcm_modules: + dcm_outs.append(dcm_module(x)) + dcm_outs = torch.cat(dcm_outs, dim=1) + output = self.bottleneck(dcm_outs) + output = self.cls_seg(output) + return output diff --git a/annotator/uniformer/mmseg/models/decode_heads/dnl_head.py b/annotator/uniformer/mmseg/models/decode_heads/dnl_head.py new file mode 100644 index 0000000000000000000000000000000000000000..333280c5947066fd3c7ebcfe302a0e7ad65480d5 --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/dnl_head.py @@ -0,0 +1,131 @@ +import torch +from annotator.uniformer.mmcv.cnn import NonLocal2d +from torch import nn + +from ..builder import HEADS +from .fcn_head import FCNHead + + +class DisentangledNonLocal2d(NonLocal2d): + """Disentangled Non-Local Blocks. + + Args: + temperature (float): Temperature to adjust attention. Default: 0.05 + """ + + def __init__(self, *arg, temperature, **kwargs): + super().__init__(*arg, **kwargs) + self.temperature = temperature + self.conv_mask = nn.Conv2d(self.in_channels, 1, kernel_size=1) + + def embedded_gaussian(self, theta_x, phi_x): + """Embedded gaussian with temperature.""" + + # NonLocal2d pairwise_weight: [N, HxW, HxW] + pairwise_weight = torch.matmul(theta_x, phi_x) + if self.use_scale: + # theta_x.shape[-1] is `self.inter_channels` + pairwise_weight /= theta_x.shape[-1]**0.5 + pairwise_weight /= self.temperature + pairwise_weight = pairwise_weight.softmax(dim=-1) + return pairwise_weight + + def forward(self, x): + # x: [N, C, H, W] + n = x.size(0) + + # g_x: [N, HxW, C] + g_x = self.g(x).view(n, self.inter_channels, -1) + g_x = g_x.permute(0, 2, 1) + + # theta_x: [N, HxW, C], phi_x: [N, C, HxW] + if self.mode == 'gaussian': + theta_x = x.view(n, self.in_channels, -1) + theta_x = theta_x.permute(0, 2, 1) + if self.sub_sample: + phi_x = self.phi(x).view(n, self.in_channels, -1) + else: + phi_x = x.view(n, self.in_channels, -1) + elif self.mode == 'concatenation': + theta_x = self.theta(x).view(n, self.inter_channels, -1, 1) + phi_x = self.phi(x).view(n, self.inter_channels, 1, -1) + else: + theta_x = self.theta(x).view(n, self.inter_channels, -1) + theta_x = theta_x.permute(0, 2, 1) + phi_x = self.phi(x).view(n, self.inter_channels, -1) + + # subtract mean + theta_x -= theta_x.mean(dim=-2, keepdim=True) + phi_x -= phi_x.mean(dim=-1, keepdim=True) + + pairwise_func = getattr(self, self.mode) + # pairwise_weight: [N, HxW, HxW] + pairwise_weight = pairwise_func(theta_x, phi_x) + + # y: [N, HxW, C] + y = torch.matmul(pairwise_weight, g_x) + # y: [N, C, H, W] + y = y.permute(0, 2, 1).contiguous().reshape(n, self.inter_channels, + *x.size()[2:]) + + # unary_mask: [N, 1, HxW] + unary_mask = self.conv_mask(x) + unary_mask = unary_mask.view(n, 1, -1) + unary_mask = unary_mask.softmax(dim=-1) + # unary_x: [N, 1, C] + unary_x = torch.matmul(unary_mask, g_x) + # unary_x: [N, C, 1, 1] + unary_x = unary_x.permute(0, 2, 1).contiguous().reshape( + n, self.inter_channels, 1, 1) + + output = x + self.conv_out(y + unary_x) + + return output + + +@HEADS.register_module() +class DNLHead(FCNHead): + """Disentangled Non-Local Neural Networks. + + This head is the implementation of `DNLNet + `_. + + Args: + reduction (int): Reduction factor of projection transform. Default: 2. + use_scale (bool): Whether to scale pairwise_weight by + sqrt(1/inter_channels). Default: False. + mode (str): The nonlocal mode. Options are 'embedded_gaussian', + 'dot_product'. Default: 'embedded_gaussian.'. + temperature (float): Temperature to adjust attention. Default: 0.05 + """ + + def __init__(self, + reduction=2, + use_scale=True, + mode='embedded_gaussian', + temperature=0.05, + **kwargs): + super(DNLHead, self).__init__(num_convs=2, **kwargs) + self.reduction = reduction + self.use_scale = use_scale + self.mode = mode + self.temperature = temperature + self.dnl_block = DisentangledNonLocal2d( + in_channels=self.channels, + reduction=self.reduction, + use_scale=self.use_scale, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + mode=self.mode, + temperature=self.temperature) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + output = self.convs[0](x) + output = self.dnl_block(output) + output = self.convs[1](output) + if self.concat_input: + output = self.conv_cat(torch.cat([x, output], dim=1)) + output = self.cls_seg(output) + return output diff --git a/annotator/uniformer/mmseg/models/decode_heads/ema_head.py b/annotator/uniformer/mmseg/models/decode_heads/ema_head.py new file mode 100644 index 0000000000000000000000000000000000000000..12267cb40569d2b5a4a2955a6dc2671377ff5e0a --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/ema_head.py @@ -0,0 +1,168 @@ +import math + +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.nn.functional as F +from annotator.uniformer.mmcv.cnn import ConvModule + +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +def reduce_mean(tensor): + """Reduce mean when distributed training.""" + if not (dist.is_available() and dist.is_initialized()): + return tensor + tensor = tensor.clone() + dist.all_reduce(tensor.div_(dist.get_world_size()), op=dist.ReduceOp.SUM) + return tensor + + +class EMAModule(nn.Module): + """Expectation Maximization Attention Module used in EMANet. + + Args: + channels (int): Channels of the whole module. + num_bases (int): Number of bases. + num_stages (int): Number of the EM iterations. + """ + + def __init__(self, channels, num_bases, num_stages, momentum): + super(EMAModule, self).__init__() + assert num_stages >= 1, 'num_stages must be at least 1!' + self.num_bases = num_bases + self.num_stages = num_stages + self.momentum = momentum + + bases = torch.zeros(1, channels, self.num_bases) + bases.normal_(0, math.sqrt(2. / self.num_bases)) + # [1, channels, num_bases] + bases = F.normalize(bases, dim=1, p=2) + self.register_buffer('bases', bases) + + def forward(self, feats): + """Forward function.""" + batch_size, channels, height, width = feats.size() + # [batch_size, channels, height*width] + feats = feats.view(batch_size, channels, height * width) + # [batch_size, channels, num_bases] + bases = self.bases.repeat(batch_size, 1, 1) + + with torch.no_grad(): + for i in range(self.num_stages): + # [batch_size, height*width, num_bases] + attention = torch.einsum('bcn,bck->bnk', feats, bases) + attention = F.softmax(attention, dim=2) + # l1 norm + attention_normed = F.normalize(attention, dim=1, p=1) + # [batch_size, channels, num_bases] + bases = torch.einsum('bcn,bnk->bck', feats, attention_normed) + # l2 norm + bases = F.normalize(bases, dim=1, p=2) + + feats_recon = torch.einsum('bck,bnk->bcn', bases, attention) + feats_recon = feats_recon.view(batch_size, channels, height, width) + + if self.training: + bases = bases.mean(dim=0, keepdim=True) + bases = reduce_mean(bases) + # l2 norm + bases = F.normalize(bases, dim=1, p=2) + self.bases = (1 - + self.momentum) * self.bases + self.momentum * bases + + return feats_recon + + +@HEADS.register_module() +class EMAHead(BaseDecodeHead): + """Expectation Maximization Attention Networks for Semantic Segmentation. + + This head is the implementation of `EMANet + `_. + + Args: + ema_channels (int): EMA module channels + num_bases (int): Number of bases. + num_stages (int): Number of the EM iterations. + concat_input (bool): Whether concat the input and output of convs + before classification layer. Default: True + momentum (float): Momentum to update the base. Default: 0.1. + """ + + def __init__(self, + ema_channels, + num_bases, + num_stages, + concat_input=True, + momentum=0.1, + **kwargs): + super(EMAHead, self).__init__(**kwargs) + self.ema_channels = ema_channels + self.num_bases = num_bases + self.num_stages = num_stages + self.concat_input = concat_input + self.momentum = momentum + self.ema_module = EMAModule(self.ema_channels, self.num_bases, + self.num_stages, self.momentum) + + self.ema_in_conv = ConvModule( + self.in_channels, + self.ema_channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + # project (0, inf) -> (-inf, inf) + self.ema_mid_conv = ConvModule( + self.ema_channels, + self.ema_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=None, + act_cfg=None) + for param in self.ema_mid_conv.parameters(): + param.requires_grad = False + + self.ema_out_conv = ConvModule( + self.ema_channels, + self.ema_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=None) + self.bottleneck = ConvModule( + self.ema_channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + if self.concat_input: + self.conv_cat = ConvModule( + self.in_channels + self.channels, + self.channels, + kernel_size=3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + feats = self.ema_in_conv(x) + identity = feats + feats = self.ema_mid_conv(feats) + recon = self.ema_module(feats) + recon = F.relu(recon, inplace=True) + recon = self.ema_out_conv(recon) + output = F.relu(identity + recon, inplace=True) + output = self.bottleneck(output) + if self.concat_input: + output = self.conv_cat(torch.cat([x, output], dim=1)) + output = self.cls_seg(output) + return output diff --git a/annotator/uniformer/mmseg/models/decode_heads/enc_head.py b/annotator/uniformer/mmseg/models/decode_heads/enc_head.py new file mode 100644 index 0000000000000000000000000000000000000000..da57af617e05d41761628fd2d6d232655b32d905 --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/enc_head.py @@ -0,0 +1,187 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from annotator.uniformer.mmcv.cnn import ConvModule, build_norm_layer + +from annotator.uniformer.mmseg.ops import Encoding, resize +from ..builder import HEADS, build_loss +from .decode_head import BaseDecodeHead + + +class EncModule(nn.Module): + """Encoding Module used in EncNet. + + Args: + in_channels (int): Input channels. + num_codes (int): Number of code words. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict): Config of activation layers. + """ + + def __init__(self, in_channels, num_codes, conv_cfg, norm_cfg, act_cfg): + super(EncModule, self).__init__() + self.encoding_project = ConvModule( + in_channels, + in_channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + # TODO: resolve this hack + # change to 1d + if norm_cfg is not None: + encoding_norm_cfg = norm_cfg.copy() + if encoding_norm_cfg['type'] in ['BN', 'IN']: + encoding_norm_cfg['type'] += '1d' + else: + encoding_norm_cfg['type'] = encoding_norm_cfg['type'].replace( + '2d', '1d') + else: + # fallback to BN1d + encoding_norm_cfg = dict(type='BN1d') + self.encoding = nn.Sequential( + Encoding(channels=in_channels, num_codes=num_codes), + build_norm_layer(encoding_norm_cfg, num_codes)[1], + nn.ReLU(inplace=True)) + self.fc = nn.Sequential( + nn.Linear(in_channels, in_channels), nn.Sigmoid()) + + def forward(self, x): + """Forward function.""" + encoding_projection = self.encoding_project(x) + encoding_feat = self.encoding(encoding_projection).mean(dim=1) + batch_size, channels, _, _ = x.size() + gamma = self.fc(encoding_feat) + y = gamma.view(batch_size, channels, 1, 1) + output = F.relu_(x + x * y) + return encoding_feat, output + + +@HEADS.register_module() +class EncHead(BaseDecodeHead): + """Context Encoding for Semantic Segmentation. + + This head is the implementation of `EncNet + `_. + + Args: + num_codes (int): Number of code words. Default: 32. + use_se_loss (bool): Whether use Semantic Encoding Loss (SE-loss) to + regularize the training. Default: True. + add_lateral (bool): Whether use lateral connection to fuse features. + Default: False. + loss_se_decode (dict): Config of decode loss. + Default: dict(type='CrossEntropyLoss', use_sigmoid=True). + """ + + def __init__(self, + num_codes=32, + use_se_loss=True, + add_lateral=False, + loss_se_decode=dict( + type='CrossEntropyLoss', + use_sigmoid=True, + loss_weight=0.2), + **kwargs): + super(EncHead, self).__init__( + input_transform='multiple_select', **kwargs) + self.use_se_loss = use_se_loss + self.add_lateral = add_lateral + self.num_codes = num_codes + self.bottleneck = ConvModule( + self.in_channels[-1], + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + if add_lateral: + self.lateral_convs = nn.ModuleList() + for in_channels in self.in_channels[:-1]: # skip the last one + self.lateral_convs.append( + ConvModule( + in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + self.fusion = ConvModule( + len(self.in_channels) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.enc_module = EncModule( + self.channels, + num_codes=num_codes, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + if self.use_se_loss: + self.loss_se_decode = build_loss(loss_se_decode) + self.se_layer = nn.Linear(self.channels, self.num_classes) + + def forward(self, inputs): + """Forward function.""" + inputs = self._transform_inputs(inputs) + feat = self.bottleneck(inputs[-1]) + if self.add_lateral: + laterals = [ + resize( + lateral_conv(inputs[i]), + size=feat.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + feat = self.fusion(torch.cat([feat, *laterals], 1)) + encode_feat, output = self.enc_module(feat) + output = self.cls_seg(output) + if self.use_se_loss: + se_output = self.se_layer(encode_feat) + return output, se_output + else: + return output + + def forward_test(self, inputs, img_metas, test_cfg): + """Forward function for testing, ignore se_loss.""" + if self.use_se_loss: + return self.forward(inputs)[0] + else: + return self.forward(inputs) + + @staticmethod + def _convert_to_onehot_labels(seg_label, num_classes): + """Convert segmentation label to onehot. + + Args: + seg_label (Tensor): Segmentation label of shape (N, H, W). + num_classes (int): Number of classes. + + Returns: + Tensor: Onehot labels of shape (N, num_classes). + """ + + batch_size = seg_label.size(0) + onehot_labels = seg_label.new_zeros((batch_size, num_classes)) + for i in range(batch_size): + hist = seg_label[i].float().histc( + bins=num_classes, min=0, max=num_classes - 1) + onehot_labels[i] = hist > 0 + return onehot_labels + + def losses(self, seg_logit, seg_label): + """Compute segmentation and semantic encoding loss.""" + seg_logit, se_seg_logit = seg_logit + loss = dict() + loss.update(super(EncHead, self).losses(seg_logit, seg_label)) + se_loss = self.loss_se_decode( + se_seg_logit, + self._convert_to_onehot_labels(seg_label, self.num_classes)) + loss['loss_se'] = se_loss + return loss diff --git a/annotator/uniformer/mmseg/models/decode_heads/fcn_head.py b/annotator/uniformer/mmseg/models/decode_heads/fcn_head.py new file mode 100644 index 0000000000000000000000000000000000000000..edb32c283fa4baada6b4a0bf3f7540c3580c3468 --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/fcn_head.py @@ -0,0 +1,81 @@ +import torch +import torch.nn as nn +from annotator.uniformer.mmcv.cnn import ConvModule + +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +@HEADS.register_module() +class FCNHead(BaseDecodeHead): + """Fully Convolution Networks for Semantic Segmentation. + + This head is implemented of `FCNNet `_. + + Args: + num_convs (int): Number of convs in the head. Default: 2. + kernel_size (int): The kernel size for convs in the head. Default: 3. + concat_input (bool): Whether concat the input and output of convs + before classification layer. + dilation (int): The dilation rate for convs in the head. Default: 1. + """ + + def __init__(self, + num_convs=2, + kernel_size=3, + concat_input=True, + dilation=1, + **kwargs): + assert num_convs >= 0 and dilation > 0 and isinstance(dilation, int) + self.num_convs = num_convs + self.concat_input = concat_input + self.kernel_size = kernel_size + super(FCNHead, self).__init__(**kwargs) + if num_convs == 0: + assert self.in_channels == self.channels + + conv_padding = (kernel_size // 2) * dilation + convs = [] + convs.append( + ConvModule( + self.in_channels, + self.channels, + kernel_size=kernel_size, + padding=conv_padding, + dilation=dilation, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + for i in range(num_convs - 1): + convs.append( + ConvModule( + self.channels, + self.channels, + kernel_size=kernel_size, + padding=conv_padding, + dilation=dilation, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + if num_convs == 0: + self.convs = nn.Identity() + else: + self.convs = nn.Sequential(*convs) + if self.concat_input: + self.conv_cat = ConvModule( + self.in_channels + self.channels, + self.channels, + kernel_size=kernel_size, + padding=kernel_size // 2, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + output = self.convs(x) + if self.concat_input: + output = self.conv_cat(torch.cat([x, output], dim=1)) + output = self.cls_seg(output) + return output diff --git a/annotator/uniformer/mmseg/models/decode_heads/fpn_head.py b/annotator/uniformer/mmseg/models/decode_heads/fpn_head.py new file mode 100644 index 0000000000000000000000000000000000000000..1241c55b0813d1ecdddf1e66e7c5031fbf78ed50 --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/fpn_head.py @@ -0,0 +1,68 @@ +import numpy as np +import torch.nn as nn +from annotator.uniformer.mmcv.cnn import ConvModule + +from annotator.uniformer.mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +@HEADS.register_module() +class FPNHead(BaseDecodeHead): + """Panoptic Feature Pyramid Networks. + + This head is the implementation of `Semantic FPN + `_. + + Args: + feature_strides (tuple[int]): The strides for input feature maps. + stack_lateral. All strides suppose to be power of 2. The first + one is of largest resolution. + """ + + def __init__(self, feature_strides, **kwargs): + super(FPNHead, self).__init__( + input_transform='multiple_select', **kwargs) + assert len(feature_strides) == len(self.in_channels) + assert min(feature_strides) == feature_strides[0] + self.feature_strides = feature_strides + + self.scale_heads = nn.ModuleList() + for i in range(len(feature_strides)): + head_length = max( + 1, + int(np.log2(feature_strides[i]) - np.log2(feature_strides[0]))) + scale_head = [] + for k in range(head_length): + scale_head.append( + ConvModule( + self.in_channels[i] if k == 0 else self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + if feature_strides[i] != feature_strides[0]: + scale_head.append( + nn.Upsample( + scale_factor=2, + mode='bilinear', + align_corners=self.align_corners)) + self.scale_heads.append(nn.Sequential(*scale_head)) + + def forward(self, inputs): + + x = self._transform_inputs(inputs) + + output = self.scale_heads[0](x[0]) + for i in range(1, len(self.feature_strides)): + # non inplace + output = output + resize( + self.scale_heads[i](x[i]), + size=output.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + + output = self.cls_seg(output) + return output diff --git a/annotator/uniformer/mmseg/models/decode_heads/gc_head.py b/annotator/uniformer/mmseg/models/decode_heads/gc_head.py new file mode 100644 index 0000000000000000000000000000000000000000..70741245af975800840709911bd18d72247e3e04 --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/gc_head.py @@ -0,0 +1,47 @@ +import torch +from annotator.uniformer.mmcv.cnn import ContextBlock + +from ..builder import HEADS +from .fcn_head import FCNHead + + +@HEADS.register_module() +class GCHead(FCNHead): + """GCNet: Non-local Networks Meet Squeeze-Excitation Networks and Beyond. + + This head is the implementation of `GCNet + `_. + + Args: + ratio (float): Multiplier of channels ratio. Default: 1/4. + pooling_type (str): The pooling type of context aggregation. + Options are 'att', 'avg'. Default: 'avg'. + fusion_types (tuple[str]): The fusion type for feature fusion. + Options are 'channel_add', 'channel_mul'. Default: ('channel_add',) + """ + + def __init__(self, + ratio=1 / 4., + pooling_type='att', + fusion_types=('channel_add', ), + **kwargs): + super(GCHead, self).__init__(num_convs=2, **kwargs) + self.ratio = ratio + self.pooling_type = pooling_type + self.fusion_types = fusion_types + self.gc_block = ContextBlock( + in_channels=self.channels, + ratio=self.ratio, + pooling_type=self.pooling_type, + fusion_types=self.fusion_types) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + output = self.convs[0](x) + output = self.gc_block(output) + output = self.convs[1](output) + if self.concat_input: + output = self.conv_cat(torch.cat([x, output], dim=1)) + output = self.cls_seg(output) + return output diff --git a/annotator/uniformer/mmseg/models/decode_heads/lraspp_head.py b/annotator/uniformer/mmseg/models/decode_heads/lraspp_head.py new file mode 100644 index 0000000000000000000000000000000000000000..69bf320934d787aaa11984a0c4effe9ad8015b22 --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/lraspp_head.py @@ -0,0 +1,90 @@ +import torch +import torch.nn as nn +from annotator.uniformer.mmcv import is_tuple_of +from annotator.uniformer.mmcv.cnn import ConvModule + +from annotator.uniformer.mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +@HEADS.register_module() +class LRASPPHead(BaseDecodeHead): + """Lite R-ASPP (LRASPP) head is proposed in Searching for MobileNetV3. + + This head is the improved implementation of `Searching for MobileNetV3 + `_. + + Args: + branch_channels (tuple[int]): The number of output channels in every + each branch. Default: (32, 64). + """ + + def __init__(self, branch_channels=(32, 64), **kwargs): + super(LRASPPHead, self).__init__(**kwargs) + if self.input_transform != 'multiple_select': + raise ValueError('in Lite R-ASPP (LRASPP) head, input_transform ' + f'must be \'multiple_select\'. But received ' + f'\'{self.input_transform}\'') + assert is_tuple_of(branch_channels, int) + assert len(branch_channels) == len(self.in_channels) - 1 + self.branch_channels = branch_channels + + self.convs = nn.Sequential() + self.conv_ups = nn.Sequential() + for i in range(len(branch_channels)): + self.convs.add_module( + f'conv{i}', + nn.Conv2d( + self.in_channels[i], branch_channels[i], 1, bias=False)) + self.conv_ups.add_module( + f'conv_up{i}', + ConvModule( + self.channels + branch_channels[i], + self.channels, + 1, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + bias=False)) + + self.conv_up_input = nn.Conv2d(self.channels, self.channels, 1) + + self.aspp_conv = ConvModule( + self.in_channels[-1], + self.channels, + 1, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + bias=False) + self.image_pool = nn.Sequential( + nn.AvgPool2d(kernel_size=49, stride=(16, 20)), + ConvModule( + self.in_channels[2], + self.channels, + 1, + act_cfg=dict(type='Sigmoid'), + bias=False)) + + def forward(self, inputs): + """Forward function.""" + inputs = self._transform_inputs(inputs) + + x = inputs[-1] + + x = self.aspp_conv(x) * resize( + self.image_pool(x), + size=x.size()[2:], + mode='bilinear', + align_corners=self.align_corners) + x = self.conv_up_input(x) + + for i in range(len(self.branch_channels) - 1, -1, -1): + x = resize( + x, + size=inputs[i].size()[2:], + mode='bilinear', + align_corners=self.align_corners) + x = torch.cat([x, self.convs[i](inputs[i])], 1) + x = self.conv_ups[i](x) + + return self.cls_seg(x) diff --git a/annotator/uniformer/mmseg/models/decode_heads/nl_head.py b/annotator/uniformer/mmseg/models/decode_heads/nl_head.py new file mode 100644 index 0000000000000000000000000000000000000000..3eee424199e6aa363b564e2a3340a070db04db86 --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/nl_head.py @@ -0,0 +1,49 @@ +import torch +from annotator.uniformer.mmcv.cnn import NonLocal2d + +from ..builder import HEADS +from .fcn_head import FCNHead + + +@HEADS.register_module() +class NLHead(FCNHead): + """Non-local Neural Networks. + + This head is the implementation of `NLNet + `_. + + Args: + reduction (int): Reduction factor of projection transform. Default: 2. + use_scale (bool): Whether to scale pairwise_weight by + sqrt(1/inter_channels). Default: True. + mode (str): The nonlocal mode. Options are 'embedded_gaussian', + 'dot_product'. Default: 'embedded_gaussian.'. + """ + + def __init__(self, + reduction=2, + use_scale=True, + mode='embedded_gaussian', + **kwargs): + super(NLHead, self).__init__(num_convs=2, **kwargs) + self.reduction = reduction + self.use_scale = use_scale + self.mode = mode + self.nl_block = NonLocal2d( + in_channels=self.channels, + reduction=self.reduction, + use_scale=self.use_scale, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + mode=self.mode) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + output = self.convs[0](x) + output = self.nl_block(output) + output = self.convs[1](output) + if self.concat_input: + output = self.conv_cat(torch.cat([x, output], dim=1)) + output = self.cls_seg(output) + return output diff --git a/annotator/uniformer/mmseg/models/decode_heads/ocr_head.py b/annotator/uniformer/mmseg/models/decode_heads/ocr_head.py new file mode 100644 index 0000000000000000000000000000000000000000..715852e94e81dc46623972748285d2d19237a341 --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/ocr_head.py @@ -0,0 +1,127 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from annotator.uniformer.mmcv.cnn import ConvModule + +from annotator.uniformer.mmseg.ops import resize +from ..builder import HEADS +from ..utils import SelfAttentionBlock as _SelfAttentionBlock +from .cascade_decode_head import BaseCascadeDecodeHead + + +class SpatialGatherModule(nn.Module): + """Aggregate the context features according to the initial predicted + probability distribution. + + Employ the soft-weighted method to aggregate the context. + """ + + def __init__(self, scale): + super(SpatialGatherModule, self).__init__() + self.scale = scale + + def forward(self, feats, probs): + """Forward function.""" + batch_size, num_classes, height, width = probs.size() + channels = feats.size(1) + probs = probs.view(batch_size, num_classes, -1) + feats = feats.view(batch_size, channels, -1) + # [batch_size, height*width, num_classes] + feats = feats.permute(0, 2, 1) + # [batch_size, channels, height*width] + probs = F.softmax(self.scale * probs, dim=2) + # [batch_size, channels, num_classes] + ocr_context = torch.matmul(probs, feats) + ocr_context = ocr_context.permute(0, 2, 1).contiguous().unsqueeze(3) + return ocr_context + + +class ObjectAttentionBlock(_SelfAttentionBlock): + """Make a OCR used SelfAttentionBlock.""" + + def __init__(self, in_channels, channels, scale, conv_cfg, norm_cfg, + act_cfg): + if scale > 1: + query_downsample = nn.MaxPool2d(kernel_size=scale) + else: + query_downsample = None + super(ObjectAttentionBlock, self).__init__( + key_in_channels=in_channels, + query_in_channels=in_channels, + channels=channels, + out_channels=in_channels, + share_key_query=False, + query_downsample=query_downsample, + key_downsample=None, + key_query_num_convs=2, + key_query_norm=True, + value_out_num_convs=1, + value_out_norm=True, + matmul_norm=True, + with_out=True, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + self.bottleneck = ConvModule( + in_channels * 2, + in_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, query_feats, key_feats): + """Forward function.""" + context = super(ObjectAttentionBlock, + self).forward(query_feats, key_feats) + output = self.bottleneck(torch.cat([context, query_feats], dim=1)) + if self.query_downsample is not None: + output = resize(query_feats) + + return output + + +@HEADS.register_module() +class OCRHead(BaseCascadeDecodeHead): + """Object-Contextual Representations for Semantic Segmentation. + + This head is the implementation of `OCRNet + `_. + + Args: + ocr_channels (int): The intermediate channels of OCR block. + scale (int): The scale of probability map in SpatialGatherModule in + Default: 1. + """ + + def __init__(self, ocr_channels, scale=1, **kwargs): + super(OCRHead, self).__init__(**kwargs) + self.ocr_channels = ocr_channels + self.scale = scale + self.object_context_block = ObjectAttentionBlock( + self.channels, + self.ocr_channels, + self.scale, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.spatial_gather_module = SpatialGatherModule(self.scale) + + self.bottleneck = ConvModule( + self.in_channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs, prev_output): + """Forward function.""" + x = self._transform_inputs(inputs) + feats = self.bottleneck(x) + context = self.spatial_gather_module(feats, prev_output) + object_context = self.object_context_block(feats, context) + output = self.cls_seg(object_context) + + return output diff --git a/annotator/uniformer/mmseg/models/decode_heads/point_head.py b/annotator/uniformer/mmseg/models/decode_heads/point_head.py new file mode 100644 index 0000000000000000000000000000000000000000..3342aa28bb8d264b2c3d01cbf5098d145943c193 --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/point_head.py @@ -0,0 +1,349 @@ +# Modified from https://github.com/facebookresearch/detectron2/tree/master/projects/PointRend/point_head/point_head.py # noqa + +import torch +import torch.nn as nn +from annotator.uniformer.mmcv.cnn import ConvModule, normal_init +from annotator.uniformer.mmcv.ops import point_sample + +from annotator.uniformer.mmseg.models.builder import HEADS +from annotator.uniformer.mmseg.ops import resize +from ..losses import accuracy +from .cascade_decode_head import BaseCascadeDecodeHead + + +def calculate_uncertainty(seg_logits): + """Estimate uncertainty based on seg logits. + + For each location of the prediction ``seg_logits`` we estimate + uncertainty as the difference between top first and top second + predicted logits. + + Args: + seg_logits (Tensor): Semantic segmentation logits, + shape (batch_size, num_classes, height, width). + + Returns: + scores (Tensor): T uncertainty scores with the most uncertain + locations having the highest uncertainty score, shape ( + batch_size, 1, height, width) + """ + top2_scores = torch.topk(seg_logits, k=2, dim=1)[0] + return (top2_scores[:, 1] - top2_scores[:, 0]).unsqueeze(1) + + +@HEADS.register_module() +class PointHead(BaseCascadeDecodeHead): + """A mask point head use in PointRend. + + ``PointHead`` use shared multi-layer perceptron (equivalent to + nn.Conv1d) to predict the logit of input points. The fine-grained feature + and coarse feature will be concatenate together for predication. + + Args: + num_fcs (int): Number of fc layers in the head. Default: 3. + in_channels (int): Number of input channels. Default: 256. + fc_channels (int): Number of fc channels. Default: 256. + num_classes (int): Number of classes for logits. Default: 80. + class_agnostic (bool): Whether use class agnostic classification. + If so, the output channels of logits will be 1. Default: False. + coarse_pred_each_layer (bool): Whether concatenate coarse feature with + the output of each fc layer. Default: True. + conv_cfg (dict|None): Dictionary to construct and config conv layer. + Default: dict(type='Conv1d')) + norm_cfg (dict|None): Dictionary to construct and config norm layer. + Default: None. + loss_point (dict): Dictionary to construct and config loss layer of + point head. Default: dict(type='CrossEntropyLoss', use_mask=True, + loss_weight=1.0). + """ + + def __init__(self, + num_fcs=3, + coarse_pred_each_layer=True, + conv_cfg=dict(type='Conv1d'), + norm_cfg=None, + act_cfg=dict(type='ReLU', inplace=False), + **kwargs): + super(PointHead, self).__init__( + input_transform='multiple_select', + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + **kwargs) + + self.num_fcs = num_fcs + self.coarse_pred_each_layer = coarse_pred_each_layer + + fc_in_channels = sum(self.in_channels) + self.num_classes + fc_channels = self.channels + self.fcs = nn.ModuleList() + for k in range(num_fcs): + fc = ConvModule( + fc_in_channels, + fc_channels, + kernel_size=1, + stride=1, + padding=0, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + self.fcs.append(fc) + fc_in_channels = fc_channels + fc_in_channels += self.num_classes if self.coarse_pred_each_layer \ + else 0 + self.fc_seg = nn.Conv1d( + fc_in_channels, + self.num_classes, + kernel_size=1, + stride=1, + padding=0) + if self.dropout_ratio > 0: + self.dropout = nn.Dropout(self.dropout_ratio) + delattr(self, 'conv_seg') + + def init_weights(self): + """Initialize weights of classification layer.""" + normal_init(self.fc_seg, std=0.001) + + def cls_seg(self, feat): + """Classify each pixel with fc.""" + if self.dropout is not None: + feat = self.dropout(feat) + output = self.fc_seg(feat) + return output + + def forward(self, fine_grained_point_feats, coarse_point_feats): + x = torch.cat([fine_grained_point_feats, coarse_point_feats], dim=1) + for fc in self.fcs: + x = fc(x) + if self.coarse_pred_each_layer: + x = torch.cat((x, coarse_point_feats), dim=1) + return self.cls_seg(x) + + def _get_fine_grained_point_feats(self, x, points): + """Sample from fine grained features. + + Args: + x (list[Tensor]): Feature pyramid from by neck or backbone. + points (Tensor): Point coordinates, shape (batch_size, + num_points, 2). + + Returns: + fine_grained_feats (Tensor): Sampled fine grained feature, + shape (batch_size, sum(channels of x), num_points). + """ + + fine_grained_feats_list = [ + point_sample(_, points, align_corners=self.align_corners) + for _ in x + ] + if len(fine_grained_feats_list) > 1: + fine_grained_feats = torch.cat(fine_grained_feats_list, dim=1) + else: + fine_grained_feats = fine_grained_feats_list[0] + + return fine_grained_feats + + def _get_coarse_point_feats(self, prev_output, points): + """Sample from fine grained features. + + Args: + prev_output (list[Tensor]): Prediction of previous decode head. + points (Tensor): Point coordinates, shape (batch_size, + num_points, 2). + + Returns: + coarse_feats (Tensor): Sampled coarse feature, shape (batch_size, + num_classes, num_points). + """ + + coarse_feats = point_sample( + prev_output, points, align_corners=self.align_corners) + + return coarse_feats + + def forward_train(self, inputs, prev_output, img_metas, gt_semantic_seg, + train_cfg): + """Forward function for training. + Args: + inputs (list[Tensor]): List of multi-level img features. + prev_output (Tensor): The output of previous decode head. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + gt_semantic_seg (Tensor): Semantic segmentation masks + used if the architecture supports semantic segmentation task. + train_cfg (dict): The training config. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + x = self._transform_inputs(inputs) + with torch.no_grad(): + points = self.get_points_train( + prev_output, calculate_uncertainty, cfg=train_cfg) + fine_grained_point_feats = self._get_fine_grained_point_feats( + x, points) + coarse_point_feats = self._get_coarse_point_feats(prev_output, points) + point_logits = self.forward(fine_grained_point_feats, + coarse_point_feats) + point_label = point_sample( + gt_semantic_seg.float(), + points, + mode='nearest', + align_corners=self.align_corners) + point_label = point_label.squeeze(1).long() + + losses = self.losses(point_logits, point_label) + + return losses + + def forward_test(self, inputs, prev_output, img_metas, test_cfg): + """Forward function for testing. + + Args: + inputs (list[Tensor]): List of multi-level img features. + prev_output (Tensor): The output of previous decode head. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + test_cfg (dict): The testing config. + + Returns: + Tensor: Output segmentation map. + """ + + x = self._transform_inputs(inputs) + refined_seg_logits = prev_output.clone() + for _ in range(test_cfg.subdivision_steps): + refined_seg_logits = resize( + refined_seg_logits, + scale_factor=test_cfg.scale_factor, + mode='bilinear', + align_corners=self.align_corners) + batch_size, channels, height, width = refined_seg_logits.shape + point_indices, points = self.get_points_test( + refined_seg_logits, calculate_uncertainty, cfg=test_cfg) + fine_grained_point_feats = self._get_fine_grained_point_feats( + x, points) + coarse_point_feats = self._get_coarse_point_feats( + prev_output, points) + point_logits = self.forward(fine_grained_point_feats, + coarse_point_feats) + + point_indices = point_indices.unsqueeze(1).expand(-1, channels, -1) + refined_seg_logits = refined_seg_logits.reshape( + batch_size, channels, height * width) + refined_seg_logits = refined_seg_logits.scatter_( + 2, point_indices, point_logits) + refined_seg_logits = refined_seg_logits.view( + batch_size, channels, height, width) + + return refined_seg_logits + + def losses(self, point_logits, point_label): + """Compute segmentation loss.""" + loss = dict() + loss['loss_point'] = self.loss_decode( + point_logits, point_label, ignore_index=self.ignore_index) + loss['acc_point'] = accuracy(point_logits, point_label) + return loss + + def get_points_train(self, seg_logits, uncertainty_func, cfg): + """Sample points for training. + + Sample points in [0, 1] x [0, 1] coordinate space based on their + uncertainty. The uncertainties are calculated for each point using + 'uncertainty_func' function that takes point's logit prediction as + input. + + Args: + seg_logits (Tensor): Semantic segmentation logits, shape ( + batch_size, num_classes, height, width). + uncertainty_func (func): uncertainty calculation function. + cfg (dict): Training config of point head. + + Returns: + point_coords (Tensor): A tensor of shape (batch_size, num_points, + 2) that contains the coordinates of ``num_points`` sampled + points. + """ + num_points = cfg.num_points + oversample_ratio = cfg.oversample_ratio + importance_sample_ratio = cfg.importance_sample_ratio + assert oversample_ratio >= 1 + assert 0 <= importance_sample_ratio <= 1 + batch_size = seg_logits.shape[0] + num_sampled = int(num_points * oversample_ratio) + point_coords = torch.rand( + batch_size, num_sampled, 2, device=seg_logits.device) + point_logits = point_sample(seg_logits, point_coords) + # It is crucial to calculate uncertainty based on the sampled + # prediction value for the points. Calculating uncertainties of the + # coarse predictions first and sampling them for points leads to + # incorrect results. To illustrate this: assume uncertainty func( + # logits)=-abs(logits), a sampled point between two coarse + # predictions with -1 and 1 logits has 0 logits, and therefore 0 + # uncertainty value. However, if we calculate uncertainties for the + # coarse predictions first, both will have -1 uncertainty, + # and sampled point will get -1 uncertainty. + point_uncertainties = uncertainty_func(point_logits) + num_uncertain_points = int(importance_sample_ratio * num_points) + num_random_points = num_points - num_uncertain_points + idx = torch.topk( + point_uncertainties[:, 0, :], k=num_uncertain_points, dim=1)[1] + shift = num_sampled * torch.arange( + batch_size, dtype=torch.long, device=seg_logits.device) + idx += shift[:, None] + point_coords = point_coords.view(-1, 2)[idx.view(-1), :].view( + batch_size, num_uncertain_points, 2) + if num_random_points > 0: + rand_point_coords = torch.rand( + batch_size, num_random_points, 2, device=seg_logits.device) + point_coords = torch.cat((point_coords, rand_point_coords), dim=1) + return point_coords + + def get_points_test(self, seg_logits, uncertainty_func, cfg): + """Sample points for testing. + + Find ``num_points`` most uncertain points from ``uncertainty_map``. + + Args: + seg_logits (Tensor): A tensor of shape (batch_size, num_classes, + height, width) for class-specific or class-agnostic prediction. + uncertainty_func (func): uncertainty calculation function. + cfg (dict): Testing config of point head. + + Returns: + point_indices (Tensor): A tensor of shape (batch_size, num_points) + that contains indices from [0, height x width) of the most + uncertain points. + point_coords (Tensor): A tensor of shape (batch_size, num_points, + 2) that contains [0, 1] x [0, 1] normalized coordinates of the + most uncertain points from the ``height x width`` grid . + """ + + num_points = cfg.subdivision_num_points + uncertainty_map = uncertainty_func(seg_logits) + batch_size, _, height, width = uncertainty_map.shape + h_step = 1.0 / height + w_step = 1.0 / width + + uncertainty_map = uncertainty_map.view(batch_size, height * width) + num_points = min(height * width, num_points) + point_indices = uncertainty_map.topk(num_points, dim=1)[1] + point_coords = torch.zeros( + batch_size, + num_points, + 2, + dtype=torch.float, + device=seg_logits.device) + point_coords[:, :, 0] = w_step / 2.0 + (point_indices % + width).float() * w_step + point_coords[:, :, 1] = h_step / 2.0 + (point_indices // + width).float() * h_step + return point_indices, point_coords diff --git a/annotator/uniformer/mmseg/models/decode_heads/psa_head.py b/annotator/uniformer/mmseg/models/decode_heads/psa_head.py new file mode 100644 index 0000000000000000000000000000000000000000..480dbd1a081262e45bf87e32c4a339ac8f8b4ffb --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/psa_head.py @@ -0,0 +1,196 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from annotator.uniformer.mmcv.cnn import ConvModule + +from annotator.uniformer.mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead + +try: + from annotator.uniformer.mmcv.ops import PSAMask +except ModuleNotFoundError: + PSAMask = None + + +@HEADS.register_module() +class PSAHead(BaseDecodeHead): + """Point-wise Spatial Attention Network for Scene Parsing. + + This head is the implementation of `PSANet + `_. + + Args: + mask_size (tuple[int]): The PSA mask size. It usually equals input + size. + psa_type (str): The type of psa module. Options are 'collect', + 'distribute', 'bi-direction'. Default: 'bi-direction' + compact (bool): Whether use compact map for 'collect' mode. + Default: True. + shrink_factor (int): The downsample factors of psa mask. Default: 2. + normalization_factor (float): The normalize factor of attention. + psa_softmax (bool): Whether use softmax for attention. + """ + + def __init__(self, + mask_size, + psa_type='bi-direction', + compact=False, + shrink_factor=2, + normalization_factor=1.0, + psa_softmax=True, + **kwargs): + if PSAMask is None: + raise RuntimeError('Please install mmcv-full for PSAMask ops') + super(PSAHead, self).__init__(**kwargs) + assert psa_type in ['collect', 'distribute', 'bi-direction'] + self.psa_type = psa_type + self.compact = compact + self.shrink_factor = shrink_factor + self.mask_size = mask_size + mask_h, mask_w = mask_size + self.psa_softmax = psa_softmax + if normalization_factor is None: + normalization_factor = mask_h * mask_w + self.normalization_factor = normalization_factor + + self.reduce = ConvModule( + self.in_channels, + self.channels, + kernel_size=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.attention = nn.Sequential( + ConvModule( + self.channels, + self.channels, + kernel_size=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg), + nn.Conv2d( + self.channels, mask_h * mask_w, kernel_size=1, bias=False)) + if psa_type == 'bi-direction': + self.reduce_p = ConvModule( + self.in_channels, + self.channels, + kernel_size=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.attention_p = nn.Sequential( + ConvModule( + self.channels, + self.channels, + kernel_size=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg), + nn.Conv2d( + self.channels, mask_h * mask_w, kernel_size=1, bias=False)) + self.psamask_collect = PSAMask('collect', mask_size) + self.psamask_distribute = PSAMask('distribute', mask_size) + else: + self.psamask = PSAMask(psa_type, mask_size) + self.proj = ConvModule( + self.channels * (2 if psa_type == 'bi-direction' else 1), + self.in_channels, + kernel_size=1, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + self.bottleneck = ConvModule( + self.in_channels * 2, + self.channels, + kernel_size=3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + identity = x + align_corners = self.align_corners + if self.psa_type in ['collect', 'distribute']: + out = self.reduce(x) + n, c, h, w = out.size() + if self.shrink_factor != 1: + if h % self.shrink_factor and w % self.shrink_factor: + h = (h - 1) // self.shrink_factor + 1 + w = (w - 1) // self.shrink_factor + 1 + align_corners = True + else: + h = h // self.shrink_factor + w = w // self.shrink_factor + align_corners = False + out = resize( + out, + size=(h, w), + mode='bilinear', + align_corners=align_corners) + y = self.attention(out) + if self.compact: + if self.psa_type == 'collect': + y = y.view(n, h * w, + h * w).transpose(1, 2).view(n, h * w, h, w) + else: + y = self.psamask(y) + if self.psa_softmax: + y = F.softmax(y, dim=1) + out = torch.bmm( + out.view(n, c, h * w), y.view(n, h * w, h * w)).view( + n, c, h, w) * (1.0 / self.normalization_factor) + else: + x_col = self.reduce(x) + x_dis = self.reduce_p(x) + n, c, h, w = x_col.size() + if self.shrink_factor != 1: + if h % self.shrink_factor and w % self.shrink_factor: + h = (h - 1) // self.shrink_factor + 1 + w = (w - 1) // self.shrink_factor + 1 + align_corners = True + else: + h = h // self.shrink_factor + w = w // self.shrink_factor + align_corners = False + x_col = resize( + x_col, + size=(h, w), + mode='bilinear', + align_corners=align_corners) + x_dis = resize( + x_dis, + size=(h, w), + mode='bilinear', + align_corners=align_corners) + y_col = self.attention(x_col) + y_dis = self.attention_p(x_dis) + if self.compact: + y_dis = y_dis.view(n, h * w, + h * w).transpose(1, 2).view(n, h * w, h, w) + else: + y_col = self.psamask_collect(y_col) + y_dis = self.psamask_distribute(y_dis) + if self.psa_softmax: + y_col = F.softmax(y_col, dim=1) + y_dis = F.softmax(y_dis, dim=1) + x_col = torch.bmm( + x_col.view(n, c, h * w), y_col.view(n, h * w, h * w)).view( + n, c, h, w) * (1.0 / self.normalization_factor) + x_dis = torch.bmm( + x_dis.view(n, c, h * w), y_dis.view(n, h * w, h * w)).view( + n, c, h, w) * (1.0 / self.normalization_factor) + out = torch.cat([x_col, x_dis], 1) + out = self.proj(out) + out = resize( + out, + size=identity.shape[2:], + mode='bilinear', + align_corners=align_corners) + out = self.bottleneck(torch.cat((identity, out), dim=1)) + out = self.cls_seg(out) + return out diff --git a/annotator/uniformer/mmseg/models/decode_heads/psp_head.py b/annotator/uniformer/mmseg/models/decode_heads/psp_head.py new file mode 100644 index 0000000000000000000000000000000000000000..b5f1e71c70c3a20f4007c263ec471a87bb214a48 --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/psp_head.py @@ -0,0 +1,101 @@ +import torch +import torch.nn as nn +from annotator.uniformer.mmcv.cnn import ConvModule + +from annotator.uniformer.mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead + + +class PPM(nn.ModuleList): + """Pooling Pyramid Module used in PSPNet. + + Args: + pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module. + in_channels (int): Input channels. + channels (int): Channels after modules, before conv_seg. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict): Config of activation layers. + align_corners (bool): align_corners argument of F.interpolate. + """ + + def __init__(self, pool_scales, in_channels, channels, conv_cfg, norm_cfg, + act_cfg, align_corners): + super(PPM, self).__init__() + self.pool_scales = pool_scales + self.align_corners = align_corners + self.in_channels = in_channels + self.channels = channels + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + for pool_scale in pool_scales: + self.append( + nn.Sequential( + nn.AdaptiveAvgPool2d(pool_scale), + ConvModule( + self.in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg))) + + def forward(self, x): + """Forward function.""" + ppm_outs = [] + for ppm in self: + ppm_out = ppm(x) + upsampled_ppm_out = resize( + ppm_out, + size=x.size()[2:], + mode='bilinear', + align_corners=self.align_corners) + ppm_outs.append(upsampled_ppm_out) + return ppm_outs + + +@HEADS.register_module() +class PSPHead(BaseDecodeHead): + """Pyramid Scene Parsing Network. + + This head is the implementation of + `PSPNet `_. + + Args: + pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module. Default: (1, 2, 3, 6). + """ + + def __init__(self, pool_scales=(1, 2, 3, 6), **kwargs): + super(PSPHead, self).__init__(**kwargs) + assert isinstance(pool_scales, (list, tuple)) + self.pool_scales = pool_scales + self.psp_modules = PPM( + self.pool_scales, + self.in_channels, + self.channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + align_corners=self.align_corners) + self.bottleneck = ConvModule( + self.in_channels + len(pool_scales) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + psp_outs = [x] + psp_outs.extend(self.psp_modules(x)) + psp_outs = torch.cat(psp_outs, dim=1) + output = self.bottleneck(psp_outs) + output = self.cls_seg(output) + return output diff --git a/annotator/uniformer/mmseg/models/decode_heads/sep_aspp_head.py b/annotator/uniformer/mmseg/models/decode_heads/sep_aspp_head.py new file mode 100644 index 0000000000000000000000000000000000000000..3339a7ac56e77dfc638e9bffb557d4699148686b --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/sep_aspp_head.py @@ -0,0 +1,101 @@ +import torch +import torch.nn as nn +from annotator.uniformer.mmcv.cnn import ConvModule, DepthwiseSeparableConvModule + +from annotator.uniformer.mmseg.ops import resize +from ..builder import HEADS +from .aspp_head import ASPPHead, ASPPModule + + +class DepthwiseSeparableASPPModule(ASPPModule): + """Atrous Spatial Pyramid Pooling (ASPP) Module with depthwise separable + conv.""" + + def __init__(self, **kwargs): + super(DepthwiseSeparableASPPModule, self).__init__(**kwargs) + for i, dilation in enumerate(self.dilations): + if dilation > 1: + self[i] = DepthwiseSeparableConvModule( + self.in_channels, + self.channels, + 3, + dilation=dilation, + padding=dilation, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + +@HEADS.register_module() +class DepthwiseSeparableASPPHead(ASPPHead): + """Encoder-Decoder with Atrous Separable Convolution for Semantic Image + Segmentation. + + This head is the implementation of `DeepLabV3+ + `_. + + Args: + c1_in_channels (int): The input channels of c1 decoder. If is 0, + the no decoder will be used. + c1_channels (int): The intermediate channels of c1 decoder. + """ + + def __init__(self, c1_in_channels, c1_channels, **kwargs): + super(DepthwiseSeparableASPPHead, self).__init__(**kwargs) + assert c1_in_channels >= 0 + self.aspp_modules = DepthwiseSeparableASPPModule( + dilations=self.dilations, + in_channels=self.in_channels, + channels=self.channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + if c1_in_channels > 0: + self.c1_bottleneck = ConvModule( + c1_in_channels, + c1_channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + else: + self.c1_bottleneck = None + self.sep_bottleneck = nn.Sequential( + DepthwiseSeparableConvModule( + self.channels + c1_channels, + self.channels, + 3, + padding=1, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg), + DepthwiseSeparableConvModule( + self.channels, + self.channels, + 3, + padding=1, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg)) + + def forward(self, inputs): + """Forward function.""" + x = self._transform_inputs(inputs) + aspp_outs = [ + resize( + self.image_pool(x), + size=x.size()[2:], + mode='bilinear', + align_corners=self.align_corners) + ] + aspp_outs.extend(self.aspp_modules(x)) + aspp_outs = torch.cat(aspp_outs, dim=1) + output = self.bottleneck(aspp_outs) + if self.c1_bottleneck is not None: + c1_output = self.c1_bottleneck(inputs[0]) + output = resize( + input=output, + size=c1_output.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + output = torch.cat([output, c1_output], dim=1) + output = self.sep_bottleneck(output) + output = self.cls_seg(output) + return output diff --git a/annotator/uniformer/mmseg/models/decode_heads/sep_fcn_head.py b/annotator/uniformer/mmseg/models/decode_heads/sep_fcn_head.py new file mode 100644 index 0000000000000000000000000000000000000000..a0986143fa4f2bd36f5271354fe5f843f35b9e6f --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/sep_fcn_head.py @@ -0,0 +1,51 @@ +from annotator.uniformer.mmcv.cnn import DepthwiseSeparableConvModule + +from ..builder import HEADS +from .fcn_head import FCNHead + + +@HEADS.register_module() +class DepthwiseSeparableFCNHead(FCNHead): + """Depthwise-Separable Fully Convolutional Network for Semantic + Segmentation. + + This head is implemented according to Fast-SCNN paper. + Args: + in_channels(int): Number of output channels of FFM. + channels(int): Number of middle-stage channels in the decode head. + concat_input(bool): Whether to concatenate original decode input into + the result of several consecutive convolution layers. + Default: True. + num_classes(int): Used to determine the dimension of + final prediction tensor. + in_index(int): Correspond with 'out_indices' in FastSCNN backbone. + norm_cfg (dict | None): Config of norm layers. + align_corners (bool): align_corners argument of F.interpolate. + Default: False. + loss_decode(dict): Config of loss type and some + relevant additional options. + """ + + def __init__(self, **kwargs): + super(DepthwiseSeparableFCNHead, self).__init__(**kwargs) + self.convs[0] = DepthwiseSeparableConvModule( + self.in_channels, + self.channels, + kernel_size=self.kernel_size, + padding=self.kernel_size // 2, + norm_cfg=self.norm_cfg) + for i in range(1, self.num_convs): + self.convs[i] = DepthwiseSeparableConvModule( + self.channels, + self.channels, + kernel_size=self.kernel_size, + padding=self.kernel_size // 2, + norm_cfg=self.norm_cfg) + + if self.concat_input: + self.conv_cat = DepthwiseSeparableConvModule( + self.in_channels + self.channels, + self.channels, + kernel_size=self.kernel_size, + padding=self.kernel_size // 2, + norm_cfg=self.norm_cfg) diff --git a/annotator/uniformer/mmseg/models/decode_heads/uper_head.py b/annotator/uniformer/mmseg/models/decode_heads/uper_head.py new file mode 100644 index 0000000000000000000000000000000000000000..9e1301b706b0d83ed714bbdee8ee24693f150455 --- /dev/null +++ b/annotator/uniformer/mmseg/models/decode_heads/uper_head.py @@ -0,0 +1,126 @@ +import torch +import torch.nn as nn +from annotator.uniformer.mmcv.cnn import ConvModule + +from annotator.uniformer.mmseg.ops import resize +from ..builder import HEADS +from .decode_head import BaseDecodeHead +from .psp_head import PPM + + +@HEADS.register_module() +class UPerHead(BaseDecodeHead): + """Unified Perceptual Parsing for Scene Understanding. + + This head is the implementation of `UPerNet + `_. + + Args: + pool_scales (tuple[int]): Pooling scales used in Pooling Pyramid + Module applied on the last feature. Default: (1, 2, 3, 6). + """ + + def __init__(self, pool_scales=(1, 2, 3, 6), **kwargs): + super(UPerHead, self).__init__( + input_transform='multiple_select', **kwargs) + # PSP Module + self.psp_modules = PPM( + pool_scales, + self.in_channels[-1], + self.channels, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + align_corners=self.align_corners) + self.bottleneck = ConvModule( + self.in_channels[-1] + len(pool_scales) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + # FPN Module + self.lateral_convs = nn.ModuleList() + self.fpn_convs = nn.ModuleList() + for in_channels in self.in_channels[:-1]: # skip the top layer + l_conv = ConvModule( + in_channels, + self.channels, + 1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + inplace=False) + fpn_conv = ConvModule( + self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg, + inplace=False) + self.lateral_convs.append(l_conv) + self.fpn_convs.append(fpn_conv) + + self.fpn_bottleneck = ConvModule( + len(self.in_channels) * self.channels, + self.channels, + 3, + padding=1, + conv_cfg=self.conv_cfg, + norm_cfg=self.norm_cfg, + act_cfg=self.act_cfg) + + def psp_forward(self, inputs): + """Forward function of PSP module.""" + x = inputs[-1] + psp_outs = [x] + psp_outs.extend(self.psp_modules(x)) + psp_outs = torch.cat(psp_outs, dim=1) + output = self.bottleneck(psp_outs) + + return output + + def forward(self, inputs): + """Forward function.""" + + inputs = self._transform_inputs(inputs) + + # build laterals + laterals = [ + lateral_conv(inputs[i]) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + + laterals.append(self.psp_forward(inputs)) + + # build top-down path + used_backbone_levels = len(laterals) + for i in range(used_backbone_levels - 1, 0, -1): + prev_shape = laterals[i - 1].shape[2:] + laterals[i - 1] += resize( + laterals[i], + size=prev_shape, + mode='bilinear', + align_corners=self.align_corners) + + # build outputs + fpn_outs = [ + self.fpn_convs[i](laterals[i]) + for i in range(used_backbone_levels - 1) + ] + # append psp feature + fpn_outs.append(laterals[-1]) + + for i in range(used_backbone_levels - 1, 0, -1): + fpn_outs[i] = resize( + fpn_outs[i], + size=fpn_outs[0].shape[2:], + mode='bilinear', + align_corners=self.align_corners) + fpn_outs = torch.cat(fpn_outs, dim=1) + output = self.fpn_bottleneck(fpn_outs) + output = self.cls_seg(output) + return output diff --git a/annotator/uniformer/mmseg/models/losses/__init__.py b/annotator/uniformer/mmseg/models/losses/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..beca72045694273d63465bac2f27dbc6672271db --- /dev/null +++ b/annotator/uniformer/mmseg/models/losses/__init__.py @@ -0,0 +1,12 @@ +from .accuracy import Accuracy, accuracy +from .cross_entropy_loss import (CrossEntropyLoss, binary_cross_entropy, + cross_entropy, mask_cross_entropy) +from .dice_loss import DiceLoss +from .lovasz_loss import LovaszLoss +from .utils import reduce_loss, weight_reduce_loss, weighted_loss + +__all__ = [ + 'accuracy', 'Accuracy', 'cross_entropy', 'binary_cross_entropy', + 'mask_cross_entropy', 'CrossEntropyLoss', 'reduce_loss', + 'weight_reduce_loss', 'weighted_loss', 'LovaszLoss', 'DiceLoss' +] diff --git a/annotator/uniformer/mmseg/models/losses/accuracy.py b/annotator/uniformer/mmseg/models/losses/accuracy.py new file mode 100644 index 0000000000000000000000000000000000000000..c0fd2e7e74a0f721c4a814c09d6e453e5956bb38 --- /dev/null +++ b/annotator/uniformer/mmseg/models/losses/accuracy.py @@ -0,0 +1,78 @@ +import torch.nn as nn + + +def accuracy(pred, target, topk=1, thresh=None): + """Calculate accuracy according to the prediction and target. + + Args: + pred (torch.Tensor): The model prediction, shape (N, num_class, ...) + target (torch.Tensor): The target of each prediction, shape (N, , ...) + topk (int | tuple[int], optional): If the predictions in ``topk`` + matches the target, the predictions will be regarded as + correct ones. Defaults to 1. + thresh (float, optional): If not None, predictions with scores under + this threshold are considered incorrect. Default to None. + + Returns: + float | tuple[float]: If the input ``topk`` is a single integer, + the function will return a single float as accuracy. If + ``topk`` is a tuple containing multiple integers, the + function will return a tuple containing accuracies of + each ``topk`` number. + """ + assert isinstance(topk, (int, tuple)) + if isinstance(topk, int): + topk = (topk, ) + return_single = True + else: + return_single = False + + maxk = max(topk) + if pred.size(0) == 0: + accu = [pred.new_tensor(0.) for i in range(len(topk))] + return accu[0] if return_single else accu + assert pred.ndim == target.ndim + 1 + assert pred.size(0) == target.size(0) + assert maxk <= pred.size(1), \ + f'maxk {maxk} exceeds pred dimension {pred.size(1)}' + pred_value, pred_label = pred.topk(maxk, dim=1) + # transpose to shape (maxk, N, ...) + pred_label = pred_label.transpose(0, 1) + correct = pred_label.eq(target.unsqueeze(0).expand_as(pred_label)) + if thresh is not None: + # Only prediction values larger than thresh are counted as correct + correct = correct & (pred_value > thresh).t() + res = [] + for k in topk: + correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True) + res.append(correct_k.mul_(100.0 / target.numel())) + return res[0] if return_single else res + + +class Accuracy(nn.Module): + """Accuracy calculation module.""" + + def __init__(self, topk=(1, ), thresh=None): + """Module to calculate the accuracy. + + Args: + topk (tuple, optional): The criterion used to calculate the + accuracy. Defaults to (1,). + thresh (float, optional): If not None, predictions with scores + under this threshold are considered incorrect. Default to None. + """ + super().__init__() + self.topk = topk + self.thresh = thresh + + def forward(self, pred, target): + """Forward function to calculate accuracy. + + Args: + pred (torch.Tensor): Prediction of models. + target (torch.Tensor): Target for each prediction. + + Returns: + tuple[float]: The accuracies under different topk criterions. + """ + return accuracy(pred, target, self.topk, self.thresh) diff --git a/annotator/uniformer/mmseg/models/losses/cross_entropy_loss.py b/annotator/uniformer/mmseg/models/losses/cross_entropy_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..42c0790c98616bb69621deed55547fc04c7392ef --- /dev/null +++ b/annotator/uniformer/mmseg/models/losses/cross_entropy_loss.py @@ -0,0 +1,198 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..builder import LOSSES +from .utils import get_class_weight, weight_reduce_loss + + +def cross_entropy(pred, + label, + weight=None, + class_weight=None, + reduction='mean', + avg_factor=None, + ignore_index=-100): + """The wrapper function for :func:`F.cross_entropy`""" + # class_weight is a manual rescaling weight given to each class. + # If given, has to be a Tensor of size C element-wise losses + loss = F.cross_entropy( + pred, + label, + weight=class_weight, + reduction='none', + ignore_index=ignore_index) + + # apply weights and do the reduction + if weight is not None: + weight = weight.float() + loss = weight_reduce_loss( + loss, weight=weight, reduction=reduction, avg_factor=avg_factor) + + return loss + + +def _expand_onehot_labels(labels, label_weights, target_shape, ignore_index): + """Expand onehot labels to match the size of prediction.""" + bin_labels = labels.new_zeros(target_shape) + valid_mask = (labels >= 0) & (labels != ignore_index) + inds = torch.nonzero(valid_mask, as_tuple=True) + + if inds[0].numel() > 0: + if labels.dim() == 3: + bin_labels[inds[0], labels[valid_mask], inds[1], inds[2]] = 1 + else: + bin_labels[inds[0], labels[valid_mask]] = 1 + + valid_mask = valid_mask.unsqueeze(1).expand(target_shape).float() + if label_weights is None: + bin_label_weights = valid_mask + else: + bin_label_weights = label_weights.unsqueeze(1).expand(target_shape) + bin_label_weights *= valid_mask + + return bin_labels, bin_label_weights + + +def binary_cross_entropy(pred, + label, + weight=None, + reduction='mean', + avg_factor=None, + class_weight=None, + ignore_index=255): + """Calculate the binary CrossEntropy loss. + + Args: + pred (torch.Tensor): The prediction with shape (N, 1). + label (torch.Tensor): The learning label of the prediction. + weight (torch.Tensor, optional): Sample-wise loss weight. + reduction (str, optional): The method used to reduce the loss. + Options are "none", "mean" and "sum". + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + class_weight (list[float], optional): The weight for each class. + ignore_index (int | None): The label index to be ignored. Default: 255 + + Returns: + torch.Tensor: The calculated loss + """ + if pred.dim() != label.dim(): + assert (pred.dim() == 2 and label.dim() == 1) or ( + pred.dim() == 4 and label.dim() == 3), \ + 'Only pred shape [N, C], label shape [N] or pred shape [N, C, ' \ + 'H, W], label shape [N, H, W] are supported' + label, weight = _expand_onehot_labels(label, weight, pred.shape, + ignore_index) + + # weighted element-wise losses + if weight is not None: + weight = weight.float() + loss = F.binary_cross_entropy_with_logits( + pred, label.float(), pos_weight=class_weight, reduction='none') + # do the reduction for the weighted loss + loss = weight_reduce_loss( + loss, weight, reduction=reduction, avg_factor=avg_factor) + + return loss + + +def mask_cross_entropy(pred, + target, + label, + reduction='mean', + avg_factor=None, + class_weight=None, + ignore_index=None): + """Calculate the CrossEntropy loss for masks. + + Args: + pred (torch.Tensor): The prediction with shape (N, C), C is the number + of classes. + target (torch.Tensor): The learning label of the prediction. + label (torch.Tensor): ``label`` indicates the class label of the mask' + corresponding object. This will be used to select the mask in the + of the class which the object belongs to when the mask prediction + if not class-agnostic. + reduction (str, optional): The method used to reduce the loss. + Options are "none", "mean" and "sum". + avg_factor (int, optional): Average factor that is used to average + the loss. Defaults to None. + class_weight (list[float], optional): The weight for each class. + ignore_index (None): Placeholder, to be consistent with other loss. + Default: None. + + Returns: + torch.Tensor: The calculated loss + """ + assert ignore_index is None, 'BCE loss does not support ignore_index' + # TODO: handle these two reserved arguments + assert reduction == 'mean' and avg_factor is None + num_rois = pred.size()[0] + inds = torch.arange(0, num_rois, dtype=torch.long, device=pred.device) + pred_slice = pred[inds, label].squeeze(1) + return F.binary_cross_entropy_with_logits( + pred_slice, target, weight=class_weight, reduction='mean')[None] + + +@LOSSES.register_module() +class CrossEntropyLoss(nn.Module): + """CrossEntropyLoss. + + Args: + use_sigmoid (bool, optional): Whether the prediction uses sigmoid + of softmax. Defaults to False. + use_mask (bool, optional): Whether to use mask cross entropy loss. + Defaults to False. + reduction (str, optional): . Defaults to 'mean'. + Options are "none", "mean" and "sum". + class_weight (list[float] | str, optional): Weight of each class. If in + str format, read them from a file. Defaults to None. + loss_weight (float, optional): Weight of the loss. Defaults to 1.0. + """ + + def __init__(self, + use_sigmoid=False, + use_mask=False, + reduction='mean', + class_weight=None, + loss_weight=1.0): + super(CrossEntropyLoss, self).__init__() + assert (use_sigmoid is False) or (use_mask is False) + self.use_sigmoid = use_sigmoid + self.use_mask = use_mask + self.reduction = reduction + self.loss_weight = loss_weight + self.class_weight = get_class_weight(class_weight) + + if self.use_sigmoid: + self.cls_criterion = binary_cross_entropy + elif self.use_mask: + self.cls_criterion = mask_cross_entropy + else: + self.cls_criterion = cross_entropy + + def forward(self, + cls_score, + label, + weight=None, + avg_factor=None, + reduction_override=None, + **kwargs): + """Forward function.""" + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + if self.class_weight is not None: + class_weight = cls_score.new_tensor(self.class_weight) + else: + class_weight = None + loss_cls = self.loss_weight * self.cls_criterion( + cls_score, + label, + weight, + class_weight=class_weight, + reduction=reduction, + avg_factor=avg_factor, + **kwargs) + return loss_cls diff --git a/annotator/uniformer/mmseg/models/losses/dice_loss.py b/annotator/uniformer/mmseg/models/losses/dice_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..27a77b962d7d8b3079c7d6cd9db52280c6fb4970 --- /dev/null +++ b/annotator/uniformer/mmseg/models/losses/dice_loss.py @@ -0,0 +1,119 @@ +"""Modified from https://github.com/LikeLy-Journey/SegmenTron/blob/master/ +segmentron/solver/loss.py (Apache-2.0 License)""" +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..builder import LOSSES +from .utils import get_class_weight, weighted_loss + + +@weighted_loss +def dice_loss(pred, + target, + valid_mask, + smooth=1, + exponent=2, + class_weight=None, + ignore_index=255): + assert pred.shape[0] == target.shape[0] + total_loss = 0 + num_classes = pred.shape[1] + for i in range(num_classes): + if i != ignore_index: + dice_loss = binary_dice_loss( + pred[:, i], + target[..., i], + valid_mask=valid_mask, + smooth=smooth, + exponent=exponent) + if class_weight is not None: + dice_loss *= class_weight[i] + total_loss += dice_loss + return total_loss / num_classes + + +@weighted_loss +def binary_dice_loss(pred, target, valid_mask, smooth=1, exponent=2, **kwards): + assert pred.shape[0] == target.shape[0] + pred = pred.reshape(pred.shape[0], -1) + target = target.reshape(target.shape[0], -1) + valid_mask = valid_mask.reshape(valid_mask.shape[0], -1) + + num = torch.sum(torch.mul(pred, target) * valid_mask, dim=1) * 2 + smooth + den = torch.sum(pred.pow(exponent) + target.pow(exponent), dim=1) + smooth + + return 1 - num / den + + +@LOSSES.register_module() +class DiceLoss(nn.Module): + """DiceLoss. + + This loss is proposed in `V-Net: Fully Convolutional Neural Networks for + Volumetric Medical Image Segmentation `_. + + Args: + loss_type (str, optional): Binary or multi-class loss. + Default: 'multi_class'. Options are "binary" and "multi_class". + smooth (float): A float number to smooth loss, and avoid NaN error. + Default: 1 + exponent (float): An float number to calculate denominator + value: \\sum{x^exponent} + \\sum{y^exponent}. Default: 2. + reduction (str, optional): The method used to reduce the loss. Options + are "none", "mean" and "sum". This parameter only works when + per_image is True. Default: 'mean'. + class_weight (list[float] | str, optional): Weight of each class. If in + str format, read them from a file. Defaults to None. + loss_weight (float, optional): Weight of the loss. Default to 1.0. + ignore_index (int | None): The label index to be ignored. Default: 255. + """ + + def __init__(self, + smooth=1, + exponent=2, + reduction='mean', + class_weight=None, + loss_weight=1.0, + ignore_index=255, + **kwards): + super(DiceLoss, self).__init__() + self.smooth = smooth + self.exponent = exponent + self.reduction = reduction + self.class_weight = get_class_weight(class_weight) + self.loss_weight = loss_weight + self.ignore_index = ignore_index + + def forward(self, + pred, + target, + avg_factor=None, + reduction_override=None, + **kwards): + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + if self.class_weight is not None: + class_weight = pred.new_tensor(self.class_weight) + else: + class_weight = None + + pred = F.softmax(pred, dim=1) + num_classes = pred.shape[1] + one_hot_target = F.one_hot( + torch.clamp(target.long(), 0, num_classes - 1), + num_classes=num_classes) + valid_mask = (target != self.ignore_index).long() + + loss = self.loss_weight * dice_loss( + pred, + one_hot_target, + valid_mask=valid_mask, + reduction=reduction, + avg_factor=avg_factor, + smooth=self.smooth, + exponent=self.exponent, + class_weight=class_weight, + ignore_index=self.ignore_index) + return loss diff --git a/annotator/uniformer/mmseg/models/losses/lovasz_loss.py b/annotator/uniformer/mmseg/models/losses/lovasz_loss.py new file mode 100644 index 0000000000000000000000000000000000000000..6badb67f6d987b59fb07aa97caaaf89896e27a8d --- /dev/null +++ b/annotator/uniformer/mmseg/models/losses/lovasz_loss.py @@ -0,0 +1,303 @@ +"""Modified from https://github.com/bermanmaxim/LovaszSoftmax/blob/master/pytor +ch/lovasz_losses.py Lovasz-Softmax and Jaccard hinge loss in PyTorch Maxim +Berman 2018 ESAT-PSI KU Leuven (MIT License)""" + +import annotator.uniformer.mmcv as mmcv +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..builder import LOSSES +from .utils import get_class_weight, weight_reduce_loss + + +def lovasz_grad(gt_sorted): + """Computes gradient of the Lovasz extension w.r.t sorted errors. + + See Alg. 1 in paper. + """ + p = len(gt_sorted) + gts = gt_sorted.sum() + intersection = gts - gt_sorted.float().cumsum(0) + union = gts + (1 - gt_sorted).float().cumsum(0) + jaccard = 1. - intersection / union + if p > 1: # cover 1-pixel case + jaccard[1:p] = jaccard[1:p] - jaccard[0:-1] + return jaccard + + +def flatten_binary_logits(logits, labels, ignore_index=None): + """Flattens predictions in the batch (binary case) Remove labels equal to + 'ignore_index'.""" + logits = logits.view(-1) + labels = labels.view(-1) + if ignore_index is None: + return logits, labels + valid = (labels != ignore_index) + vlogits = logits[valid] + vlabels = labels[valid] + return vlogits, vlabels + + +def flatten_probs(probs, labels, ignore_index=None): + """Flattens predictions in the batch.""" + if probs.dim() == 3: + # assumes output of a sigmoid layer + B, H, W = probs.size() + probs = probs.view(B, 1, H, W) + B, C, H, W = probs.size() + probs = probs.permute(0, 2, 3, 1).contiguous().view(-1, C) # B*H*W, C=P,C + labels = labels.view(-1) + if ignore_index is None: + return probs, labels + valid = (labels != ignore_index) + vprobs = probs[valid.nonzero().squeeze()] + vlabels = labels[valid] + return vprobs, vlabels + + +def lovasz_hinge_flat(logits, labels): + """Binary Lovasz hinge loss. + + Args: + logits (torch.Tensor): [P], logits at each prediction + (between -infty and +infty). + labels (torch.Tensor): [P], binary ground truth labels (0 or 1). + + Returns: + torch.Tensor: The calculated loss. + """ + if len(labels) == 0: + # only void pixels, the gradients should be 0 + return logits.sum() * 0. + signs = 2. * labels.float() - 1. + errors = (1. - logits * signs) + errors_sorted, perm = torch.sort(errors, dim=0, descending=True) + perm = perm.data + gt_sorted = labels[perm] + grad = lovasz_grad(gt_sorted) + loss = torch.dot(F.relu(errors_sorted), grad) + return loss + + +def lovasz_hinge(logits, + labels, + classes='present', + per_image=False, + class_weight=None, + reduction='mean', + avg_factor=None, + ignore_index=255): + """Binary Lovasz hinge loss. + + Args: + logits (torch.Tensor): [B, H, W], logits at each pixel + (between -infty and +infty). + labels (torch.Tensor): [B, H, W], binary ground truth masks (0 or 1). + classes (str | list[int], optional): Placeholder, to be consistent with + other loss. Default: None. + per_image (bool, optional): If per_image is True, compute the loss per + image instead of per batch. Default: False. + class_weight (list[float], optional): Placeholder, to be consistent + with other loss. Default: None. + reduction (str, optional): The method used to reduce the loss. Options + are "none", "mean" and "sum". This parameter only works when + per_image is True. Default: 'mean'. + avg_factor (int, optional): Average factor that is used to average + the loss. This parameter only works when per_image is True. + Default: None. + ignore_index (int | None): The label index to be ignored. Default: 255. + + Returns: + torch.Tensor: The calculated loss. + """ + if per_image: + loss = [ + lovasz_hinge_flat(*flatten_binary_logits( + logit.unsqueeze(0), label.unsqueeze(0), ignore_index)) + for logit, label in zip(logits, labels) + ] + loss = weight_reduce_loss( + torch.stack(loss), None, reduction, avg_factor) + else: + loss = lovasz_hinge_flat( + *flatten_binary_logits(logits, labels, ignore_index)) + return loss + + +def lovasz_softmax_flat(probs, labels, classes='present', class_weight=None): + """Multi-class Lovasz-Softmax loss. + + Args: + probs (torch.Tensor): [P, C], class probabilities at each prediction + (between 0 and 1). + labels (torch.Tensor): [P], ground truth labels (between 0 and C - 1). + classes (str | list[int], optional): Classes chosen to calculate loss. + 'all' for all classes, 'present' for classes present in labels, or + a list of classes to average. Default: 'present'. + class_weight (list[float], optional): The weight for each class. + Default: None. + + Returns: + torch.Tensor: The calculated loss. + """ + if probs.numel() == 0: + # only void pixels, the gradients should be 0 + return probs * 0. + C = probs.size(1) + losses = [] + class_to_sum = list(range(C)) if classes in ['all', 'present'] else classes + for c in class_to_sum: + fg = (labels == c).float() # foreground for class c + if (classes == 'present' and fg.sum() == 0): + continue + if C == 1: + if len(classes) > 1: + raise ValueError('Sigmoid output possible only with 1 class') + class_pred = probs[:, 0] + else: + class_pred = probs[:, c] + errors = (fg - class_pred).abs() + errors_sorted, perm = torch.sort(errors, 0, descending=True) + perm = perm.data + fg_sorted = fg[perm] + loss = torch.dot(errors_sorted, lovasz_grad(fg_sorted)) + if class_weight is not None: + loss *= class_weight[c] + losses.append(loss) + return torch.stack(losses).mean() + + +def lovasz_softmax(probs, + labels, + classes='present', + per_image=False, + class_weight=None, + reduction='mean', + avg_factor=None, + ignore_index=255): + """Multi-class Lovasz-Softmax loss. + + Args: + probs (torch.Tensor): [B, C, H, W], class probabilities at each + prediction (between 0 and 1). + labels (torch.Tensor): [B, H, W], ground truth labels (between 0 and + C - 1). + classes (str | list[int], optional): Classes chosen to calculate loss. + 'all' for all classes, 'present' for classes present in labels, or + a list of classes to average. Default: 'present'. + per_image (bool, optional): If per_image is True, compute the loss per + image instead of per batch. Default: False. + class_weight (list[float], optional): The weight for each class. + Default: None. + reduction (str, optional): The method used to reduce the loss. Options + are "none", "mean" and "sum". This parameter only works when + per_image is True. Default: 'mean'. + avg_factor (int, optional): Average factor that is used to average + the loss. This parameter only works when per_image is True. + Default: None. + ignore_index (int | None): The label index to be ignored. Default: 255. + + Returns: + torch.Tensor: The calculated loss. + """ + + if per_image: + loss = [ + lovasz_softmax_flat( + *flatten_probs( + prob.unsqueeze(0), label.unsqueeze(0), ignore_index), + classes=classes, + class_weight=class_weight) + for prob, label in zip(probs, labels) + ] + loss = weight_reduce_loss( + torch.stack(loss), None, reduction, avg_factor) + else: + loss = lovasz_softmax_flat( + *flatten_probs(probs, labels, ignore_index), + classes=classes, + class_weight=class_weight) + return loss + + +@LOSSES.register_module() +class LovaszLoss(nn.Module): + """LovaszLoss. + + This loss is proposed in `The Lovasz-Softmax loss: A tractable surrogate + for the optimization of the intersection-over-union measure in neural + networks `_. + + Args: + loss_type (str, optional): Binary or multi-class loss. + Default: 'multi_class'. Options are "binary" and "multi_class". + classes (str | list[int], optional): Classes chosen to calculate loss. + 'all' for all classes, 'present' for classes present in labels, or + a list of classes to average. Default: 'present'. + per_image (bool, optional): If per_image is True, compute the loss per + image instead of per batch. Default: False. + reduction (str, optional): The method used to reduce the loss. Options + are "none", "mean" and "sum". This parameter only works when + per_image is True. Default: 'mean'. + class_weight (list[float] | str, optional): Weight of each class. If in + str format, read them from a file. Defaults to None. + loss_weight (float, optional): Weight of the loss. Defaults to 1.0. + """ + + def __init__(self, + loss_type='multi_class', + classes='present', + per_image=False, + reduction='mean', + class_weight=None, + loss_weight=1.0): + super(LovaszLoss, self).__init__() + assert loss_type in ('binary', 'multi_class'), "loss_type should be \ + 'binary' or 'multi_class'." + + if loss_type == 'binary': + self.cls_criterion = lovasz_hinge + else: + self.cls_criterion = lovasz_softmax + assert classes in ('all', 'present') or mmcv.is_list_of(classes, int) + if not per_image: + assert reduction == 'none', "reduction should be 'none' when \ + per_image is False." + + self.classes = classes + self.per_image = per_image + self.reduction = reduction + self.loss_weight = loss_weight + self.class_weight = get_class_weight(class_weight) + + def forward(self, + cls_score, + label, + weight=None, + avg_factor=None, + reduction_override=None, + **kwargs): + """Forward function.""" + assert reduction_override in (None, 'none', 'mean', 'sum') + reduction = ( + reduction_override if reduction_override else self.reduction) + if self.class_weight is not None: + class_weight = cls_score.new_tensor(self.class_weight) + else: + class_weight = None + + # if multi-class loss, transform logits to probs + if self.cls_criterion == lovasz_softmax: + cls_score = F.softmax(cls_score, dim=1) + + loss_cls = self.loss_weight * self.cls_criterion( + cls_score, + label, + self.classes, + self.per_image, + class_weight=class_weight, + reduction=reduction, + avg_factor=avg_factor, + **kwargs) + return loss_cls diff --git a/annotator/uniformer/mmseg/models/losses/utils.py b/annotator/uniformer/mmseg/models/losses/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..85aec9f3045240c3de96a928324ae8f5c3aebe8b --- /dev/null +++ b/annotator/uniformer/mmseg/models/losses/utils.py @@ -0,0 +1,121 @@ +import functools + +import annotator.uniformer.mmcv as mmcv +import numpy as np +import torch.nn.functional as F + + +def get_class_weight(class_weight): + """Get class weight for loss function. + + Args: + class_weight (list[float] | str | None): If class_weight is a str, + take it as a file name and read from it. + """ + if isinstance(class_weight, str): + # take it as a file path + if class_weight.endswith('.npy'): + class_weight = np.load(class_weight) + else: + # pkl, json or yaml + class_weight = mmcv.load(class_weight) + + return class_weight + + +def reduce_loss(loss, reduction): + """Reduce loss as specified. + + Args: + loss (Tensor): Elementwise loss tensor. + reduction (str): Options are "none", "mean" and "sum". + + Return: + Tensor: Reduced loss tensor. + """ + reduction_enum = F._Reduction.get_enum(reduction) + # none: 0, elementwise_mean:1, sum: 2 + if reduction_enum == 0: + return loss + elif reduction_enum == 1: + return loss.mean() + elif reduction_enum == 2: + return loss.sum() + + +def weight_reduce_loss(loss, weight=None, reduction='mean', avg_factor=None): + """Apply element-wise weight and reduce loss. + + Args: + loss (Tensor): Element-wise loss. + weight (Tensor): Element-wise weights. + reduction (str): Same as built-in losses of PyTorch. + avg_factor (float): Avarage factor when computing the mean of losses. + + Returns: + Tensor: Processed loss values. + """ + # if weight is specified, apply element-wise weight + if weight is not None: + assert weight.dim() == loss.dim() + if weight.dim() > 1: + assert weight.size(1) == 1 or weight.size(1) == loss.size(1) + loss = loss * weight + + # if avg_factor is not specified, just reduce the loss + if avg_factor is None: + loss = reduce_loss(loss, reduction) + else: + # if reduction is mean, then average the loss by avg_factor + if reduction == 'mean': + loss = loss.sum() / avg_factor + # if reduction is 'none', then do nothing, otherwise raise an error + elif reduction != 'none': + raise ValueError('avg_factor can not be used with reduction="sum"') + return loss + + +def weighted_loss(loss_func): + """Create a weighted version of a given loss function. + + To use this decorator, the loss function must have the signature like + `loss_func(pred, target, **kwargs)`. The function only needs to compute + element-wise loss without any reduction. This decorator will add weight + and reduction arguments to the function. The decorated function will have + the signature like `loss_func(pred, target, weight=None, reduction='mean', + avg_factor=None, **kwargs)`. + + :Example: + + >>> import torch + >>> @weighted_loss + >>> def l1_loss(pred, target): + >>> return (pred - target).abs() + + >>> pred = torch.Tensor([0, 2, 3]) + >>> target = torch.Tensor([1, 1, 1]) + >>> weight = torch.Tensor([1, 0, 1]) + + >>> l1_loss(pred, target) + tensor(1.3333) + >>> l1_loss(pred, target, weight) + tensor(1.) + >>> l1_loss(pred, target, reduction='none') + tensor([1., 1., 2.]) + >>> l1_loss(pred, target, weight, avg_factor=2) + tensor(1.5000) + """ + + @functools.wraps(loss_func) + def wrapper(pred, + target, + weight=None, + reduction='mean', + avg_factor=None, + **kwargs): + # get element-wise loss + loss = loss_func(pred, target, **kwargs) + loss = weight_reduce_loss(loss, weight, reduction, avg_factor) + return loss + + return wrapper diff --git a/annotator/uniformer/mmseg/models/necks/__init__.py b/annotator/uniformer/mmseg/models/necks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9b9d3d5b3fe80247642d962edd6fb787537d01d6 --- /dev/null +++ b/annotator/uniformer/mmseg/models/necks/__init__.py @@ -0,0 +1,4 @@ +from .fpn import FPN +from .multilevel_neck import MultiLevelNeck + +__all__ = ['FPN', 'MultiLevelNeck'] diff --git a/annotator/uniformer/mmseg/models/necks/fpn.py b/annotator/uniformer/mmseg/models/necks/fpn.py new file mode 100644 index 0000000000000000000000000000000000000000..a53b2a69500f8c2edb835abc3ff0ccc2173d1fb1 --- /dev/null +++ b/annotator/uniformer/mmseg/models/necks/fpn.py @@ -0,0 +1,212 @@ +import torch.nn as nn +import torch.nn.functional as F +from annotator.uniformer.mmcv.cnn import ConvModule, xavier_init + +from ..builder import NECKS + + +@NECKS.register_module() +class FPN(nn.Module): + """Feature Pyramid Network. + + This is an implementation of - Feature Pyramid Networks for Object + Detection (https://arxiv.org/abs/1612.03144) + + Args: + in_channels (List[int]): Number of input channels per scale. + out_channels (int): Number of output channels (used at each scale) + num_outs (int): Number of output scales. + start_level (int): Index of the start input backbone level used to + build the feature pyramid. Default: 0. + end_level (int): Index of the end input backbone level (exclusive) to + build the feature pyramid. Default: -1, which means the last level. + add_extra_convs (bool | str): If bool, it decides whether to add conv + layers on top of the original feature maps. Default to False. + If True, its actual mode is specified by `extra_convs_on_inputs`. + If str, it specifies the source feature map of the extra convs. + Only the following options are allowed + + - 'on_input': Last feat map of neck inputs (i.e. backbone feature). + - 'on_lateral': Last feature map after lateral convs. + - 'on_output': The last output feature map after fpn convs. + extra_convs_on_inputs (bool, deprecated): Whether to apply extra convs + on the original feature from the backbone. If True, + it is equivalent to `add_extra_convs='on_input'`. If False, it is + equivalent to set `add_extra_convs='on_output'`. Default to True. + relu_before_extra_convs (bool): Whether to apply relu before the extra + conv. Default: False. + no_norm_on_lateral (bool): Whether to apply norm on lateral. + Default: False. + conv_cfg (dict): Config dict for convolution layer. Default: None. + norm_cfg (dict): Config dict for normalization layer. Default: None. + act_cfg (str): Config dict for activation layer in ConvModule. + Default: None. + upsample_cfg (dict): Config dict for interpolate layer. + Default: `dict(mode='nearest')` + + Example: + >>> import torch + >>> in_channels = [2, 3, 5, 7] + >>> scales = [340, 170, 84, 43] + >>> inputs = [torch.rand(1, c, s, s) + ... for c, s in zip(in_channels, scales)] + >>> self = FPN(in_channels, 11, len(in_channels)).eval() + >>> outputs = self.forward(inputs) + >>> for i in range(len(outputs)): + ... print(f'outputs[{i}].shape = {outputs[i].shape}') + outputs[0].shape = torch.Size([1, 11, 340, 340]) + outputs[1].shape = torch.Size([1, 11, 170, 170]) + outputs[2].shape = torch.Size([1, 11, 84, 84]) + outputs[3].shape = torch.Size([1, 11, 43, 43]) + """ + + def __init__(self, + in_channels, + out_channels, + num_outs, + start_level=0, + end_level=-1, + add_extra_convs=False, + extra_convs_on_inputs=False, + relu_before_extra_convs=False, + no_norm_on_lateral=False, + conv_cfg=None, + norm_cfg=None, + act_cfg=None, + upsample_cfg=dict(mode='nearest')): + super(FPN, self).__init__() + assert isinstance(in_channels, list) + self.in_channels = in_channels + self.out_channels = out_channels + self.num_ins = len(in_channels) + self.num_outs = num_outs + self.relu_before_extra_convs = relu_before_extra_convs + self.no_norm_on_lateral = no_norm_on_lateral + self.fp16_enabled = False + self.upsample_cfg = upsample_cfg.copy() + + if end_level == -1: + self.backbone_end_level = self.num_ins + assert num_outs >= self.num_ins - start_level + else: + # if end_level < inputs, no extra level is allowed + self.backbone_end_level = end_level + assert end_level <= len(in_channels) + assert num_outs == end_level - start_level + self.start_level = start_level + self.end_level = end_level + self.add_extra_convs = add_extra_convs + assert isinstance(add_extra_convs, (str, bool)) + if isinstance(add_extra_convs, str): + # Extra_convs_source choices: 'on_input', 'on_lateral', 'on_output' + assert add_extra_convs in ('on_input', 'on_lateral', 'on_output') + elif add_extra_convs: # True + if extra_convs_on_inputs: + # For compatibility with previous release + # TODO: deprecate `extra_convs_on_inputs` + self.add_extra_convs = 'on_input' + else: + self.add_extra_convs = 'on_output' + + self.lateral_convs = nn.ModuleList() + self.fpn_convs = nn.ModuleList() + + for i in range(self.start_level, self.backbone_end_level): + l_conv = ConvModule( + in_channels[i], + out_channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg if not self.no_norm_on_lateral else None, + act_cfg=act_cfg, + inplace=False) + fpn_conv = ConvModule( + out_channels, + out_channels, + 3, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + inplace=False) + + self.lateral_convs.append(l_conv) + self.fpn_convs.append(fpn_conv) + + # add extra conv layers (e.g., RetinaNet) + extra_levels = num_outs - self.backbone_end_level + self.start_level + if self.add_extra_convs and extra_levels >= 1: + for i in range(extra_levels): + if i == 0 and self.add_extra_convs == 'on_input': + in_channels = self.in_channels[self.backbone_end_level - 1] + else: + in_channels = out_channels + extra_fpn_conv = ConvModule( + in_channels, + out_channels, + 3, + stride=2, + padding=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + inplace=False) + self.fpn_convs.append(extra_fpn_conv) + + # default init_weights for conv(msra) and norm in ConvModule + def init_weights(self): + for m in self.modules(): + if isinstance(m, nn.Conv2d): + xavier_init(m, distribution='uniform') + + def forward(self, inputs): + assert len(inputs) == len(self.in_channels) + + # build laterals + laterals = [ + lateral_conv(inputs[i + self.start_level]) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + + # build top-down path + used_backbone_levels = len(laterals) + for i in range(used_backbone_levels - 1, 0, -1): + # In some cases, fixing `scale factor` (e.g. 2) is preferred, but + # it cannot co-exist with `size` in `F.interpolate`. + if 'scale_factor' in self.upsample_cfg: + laterals[i - 1] += F.interpolate(laterals[i], + **self.upsample_cfg) + else: + prev_shape = laterals[i - 1].shape[2:] + laterals[i - 1] += F.interpolate( + laterals[i], size=prev_shape, **self.upsample_cfg) + + # build outputs + # part 1: from original levels + outs = [ + self.fpn_convs[i](laterals[i]) for i in range(used_backbone_levels) + ] + # part 2: add extra levels + if self.num_outs > len(outs): + # use max pool to get more levels on top of outputs + # (e.g., Faster R-CNN, Mask R-CNN) + if not self.add_extra_convs: + for i in range(self.num_outs - used_backbone_levels): + outs.append(F.max_pool2d(outs[-1], 1, stride=2)) + # add conv layers on top of original feature maps (RetinaNet) + else: + if self.add_extra_convs == 'on_input': + extra_source = inputs[self.backbone_end_level - 1] + elif self.add_extra_convs == 'on_lateral': + extra_source = laterals[-1] + elif self.add_extra_convs == 'on_output': + extra_source = outs[-1] + else: + raise NotImplementedError + outs.append(self.fpn_convs[used_backbone_levels](extra_source)) + for i in range(used_backbone_levels + 1, self.num_outs): + if self.relu_before_extra_convs: + outs.append(self.fpn_convs[i](F.relu(outs[-1]))) + else: + outs.append(self.fpn_convs[i](outs[-1])) + return tuple(outs) diff --git a/annotator/uniformer/mmseg/models/necks/multilevel_neck.py b/annotator/uniformer/mmseg/models/necks/multilevel_neck.py new file mode 100644 index 0000000000000000000000000000000000000000..766144d8136326a1fab5906a153a0c0df69b6b60 --- /dev/null +++ b/annotator/uniformer/mmseg/models/necks/multilevel_neck.py @@ -0,0 +1,70 @@ +import torch.nn as nn +import torch.nn.functional as F +from annotator.uniformer.mmcv.cnn import ConvModule + +from ..builder import NECKS + + +@NECKS.register_module() +class MultiLevelNeck(nn.Module): + """MultiLevelNeck. + + A neck structure connect vit backbone and decoder_heads. + Args: + in_channels (List[int]): Number of input channels per scale. + out_channels (int): Number of output channels (used at each scale). + scales (List[int]): Scale factors for each input feature map. + norm_cfg (dict): Config dict for normalization layer. Default: None. + act_cfg (dict): Config dict for activation layer in ConvModule. + Default: None. + """ + + def __init__(self, + in_channels, + out_channels, + scales=[0.5, 1, 2, 4], + norm_cfg=None, + act_cfg=None): + super(MultiLevelNeck, self).__init__() + assert isinstance(in_channels, list) + self.in_channels = in_channels + self.out_channels = out_channels + self.scales = scales + self.num_outs = len(scales) + self.lateral_convs = nn.ModuleList() + self.convs = nn.ModuleList() + for in_channel in in_channels: + self.lateral_convs.append( + ConvModule( + in_channel, + out_channels, + kernel_size=1, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + for _ in range(self.num_outs): + self.convs.append( + ConvModule( + out_channels, + out_channels, + kernel_size=3, + padding=1, + stride=1, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + + def forward(self, inputs): + assert len(inputs) == len(self.in_channels) + print(inputs[0].shape) + inputs = [ + lateral_conv(inputs[i]) + for i, lateral_conv in enumerate(self.lateral_convs) + ] + # for len(inputs) not equal to self.num_outs + if len(inputs) == 1: + inputs = [inputs[0] for _ in range(self.num_outs)] + outs = [] + for i in range(self.num_outs): + x_resize = F.interpolate( + inputs[i], scale_factor=self.scales[i], mode='bilinear') + outs.append(self.convs[i](x_resize)) + return tuple(outs) diff --git a/annotator/uniformer/mmseg/models/segmentors/__init__.py b/annotator/uniformer/mmseg/models/segmentors/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..dca2f09405330743c476e190896bee39c45498ea --- /dev/null +++ b/annotator/uniformer/mmseg/models/segmentors/__init__.py @@ -0,0 +1,5 @@ +from .base import BaseSegmentor +from .cascade_encoder_decoder import CascadeEncoderDecoder +from .encoder_decoder import EncoderDecoder + +__all__ = ['BaseSegmentor', 'EncoderDecoder', 'CascadeEncoderDecoder'] diff --git a/annotator/uniformer/mmseg/models/segmentors/base.py b/annotator/uniformer/mmseg/models/segmentors/base.py new file mode 100644 index 0000000000000000000000000000000000000000..172fc63b736c4f13be1cd909433bc260760a1eaa --- /dev/null +++ b/annotator/uniformer/mmseg/models/segmentors/base.py @@ -0,0 +1,273 @@ +import logging +import warnings +from abc import ABCMeta, abstractmethod +from collections import OrderedDict + +import annotator.uniformer.mmcv as mmcv +import numpy as np +import torch +import torch.distributed as dist +import torch.nn as nn +from annotator.uniformer.mmcv.runner import auto_fp16 + + +class BaseSegmentor(nn.Module): + """Base class for segmentors.""" + + __metaclass__ = ABCMeta + + def __init__(self): + super(BaseSegmentor, self).__init__() + self.fp16_enabled = False + + @property + def with_neck(self): + """bool: whether the segmentor has neck""" + return hasattr(self, 'neck') and self.neck is not None + + @property + def with_auxiliary_head(self): + """bool: whether the segmentor has auxiliary head""" + return hasattr(self, + 'auxiliary_head') and self.auxiliary_head is not None + + @property + def with_decode_head(self): + """bool: whether the segmentor has decode head""" + return hasattr(self, 'decode_head') and self.decode_head is not None + + @abstractmethod + def extract_feat(self, imgs): + """Placeholder for extract features from images.""" + pass + + @abstractmethod + def encode_decode(self, img, img_metas): + """Placeholder for encode images with backbone and decode into a + semantic segmentation map of the same size as input.""" + pass + + @abstractmethod + def forward_train(self, imgs, img_metas, **kwargs): + """Placeholder for Forward function for training.""" + pass + + @abstractmethod + def simple_test(self, img, img_meta, **kwargs): + """Placeholder for single image test.""" + pass + + @abstractmethod + def aug_test(self, imgs, img_metas, **kwargs): + """Placeholder for augmentation test.""" + pass + + def init_weights(self, pretrained=None): + """Initialize the weights in segmentor. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + if pretrained is not None: + logger = logging.getLogger() + logger.info(f'load model from: {pretrained}') + + def forward_test(self, imgs, img_metas, **kwargs): + """ + Args: + imgs (List[Tensor]): the outer list indicates test-time + augmentations and inner Tensor should have a shape NxCxHxW, + which contains all images in the batch. + img_metas (List[List[dict]]): the outer list indicates test-time + augs (multiscale, flip, etc.) and the inner list indicates + images in a batch. + """ + for var, name in [(imgs, 'imgs'), (img_metas, 'img_metas')]: + if not isinstance(var, list): + raise TypeError(f'{name} must be a list, but got ' + f'{type(var)}') + + num_augs = len(imgs) + if num_augs != len(img_metas): + raise ValueError(f'num of augmentations ({len(imgs)}) != ' + f'num of image meta ({len(img_metas)})') + # all images in the same aug batch all of the same ori_shape and pad + # shape + for img_meta in img_metas: + ori_shapes = [_['ori_shape'] for _ in img_meta] + assert all(shape == ori_shapes[0] for shape in ori_shapes) + img_shapes = [_['img_shape'] for _ in img_meta] + assert all(shape == img_shapes[0] for shape in img_shapes) + pad_shapes = [_['pad_shape'] for _ in img_meta] + assert all(shape == pad_shapes[0] for shape in pad_shapes) + + if num_augs == 1: + return self.simple_test(imgs[0], img_metas[0], **kwargs) + else: + return self.aug_test(imgs, img_metas, **kwargs) + + @auto_fp16(apply_to=('img', )) + def forward(self, img, img_metas, return_loss=True, **kwargs): + """Calls either :func:`forward_train` or :func:`forward_test` depending + on whether ``return_loss`` is ``True``. + + Note this setting will change the expected inputs. When + ``return_loss=True``, img and img_meta are single-nested (i.e. Tensor + and List[dict]), and when ``resturn_loss=False``, img and img_meta + should be double nested (i.e. List[Tensor], List[List[dict]]), with + the outer list indicating test time augmentations. + """ + if return_loss: + return self.forward_train(img, img_metas, **kwargs) + else: + return self.forward_test(img, img_metas, **kwargs) + + def train_step(self, data_batch, optimizer, **kwargs): + """The iteration step during training. + + This method defines an iteration step during training, except for the + back propagation and optimizer updating, which are done in an optimizer + hook. Note that in some complicated cases or models, the whole process + including back propagation and optimizer updating is also defined in + this method, such as GAN. + + Args: + data (dict): The output of dataloader. + optimizer (:obj:`torch.optim.Optimizer` | dict): The optimizer of + runner is passed to ``train_step()``. This argument is unused + and reserved. + + Returns: + dict: It should contain at least 3 keys: ``loss``, ``log_vars``, + ``num_samples``. + ``loss`` is a tensor for back propagation, which can be a + weighted sum of multiple losses. + ``log_vars`` contains all the variables to be sent to the + logger. + ``num_samples`` indicates the batch size (when the model is + DDP, it means the batch size on each GPU), which is used for + averaging the logs. + """ + losses = self(**data_batch) + loss, log_vars = self._parse_losses(losses) + + outputs = dict( + loss=loss, + log_vars=log_vars, + num_samples=len(data_batch['img_metas'])) + + return outputs + + def val_step(self, data_batch, **kwargs): + """The iteration step during validation. + + This method shares the same signature as :func:`train_step`, but used + during val epochs. Note that the evaluation after training epochs is + not implemented with this method, but an evaluation hook. + """ + output = self(**data_batch, **kwargs) + return output + + @staticmethod + def _parse_losses(losses): + """Parse the raw outputs (losses) of the network. + + Args: + losses (dict): Raw output of the network, which usually contain + losses and other necessary information. + + Returns: + tuple[Tensor, dict]: (loss, log_vars), loss is the loss tensor + which may be a weighted sum of all losses, log_vars contains + all the variables to be sent to the logger. + """ + log_vars = OrderedDict() + for loss_name, loss_value in losses.items(): + if isinstance(loss_value, torch.Tensor): + log_vars[loss_name] = loss_value.mean() + elif isinstance(loss_value, list): + log_vars[loss_name] = sum(_loss.mean() for _loss in loss_value) + else: + raise TypeError( + f'{loss_name} is not a tensor or list of tensors') + + loss = sum(_value for _key, _value in log_vars.items() + if 'loss' in _key) + + log_vars['loss'] = loss + for loss_name, loss_value in log_vars.items(): + # reduce loss when distributed training + if dist.is_available() and dist.is_initialized(): + loss_value = loss_value.data.clone() + dist.all_reduce(loss_value.div_(dist.get_world_size())) + log_vars[loss_name] = loss_value.item() + + return loss, log_vars + + def show_result(self, + img, + result, + palette=None, + win_name='', + show=False, + wait_time=0, + out_file=None, + opacity=0.5): + """Draw `result` over `img`. + + Args: + img (str or Tensor): The image to be displayed. + result (Tensor): The semantic segmentation results to draw over + `img`. + palette (list[list[int]]] | np.ndarray | None): The palette of + segmentation map. If None is given, random palette will be + generated. Default: None + win_name (str): The window name. + wait_time (int): Value of waitKey param. + Default: 0. + show (bool): Whether to show the image. + Default: False. + out_file (str or None): The filename to write the image. + Default: None. + opacity(float): Opacity of painted segmentation map. + Default 0.5. + Must be in (0, 1] range. + Returns: + img (Tensor): Only if not `show` or `out_file` + """ + img = mmcv.imread(img) + img = img.copy() + seg = result[0] + if palette is None: + if self.PALETTE is None: + palette = np.random.randint( + 0, 255, size=(len(self.CLASSES), 3)) + else: + palette = self.PALETTE + palette = np.array(palette) + assert palette.shape[0] == len(self.CLASSES) + assert palette.shape[1] == 3 + assert len(palette.shape) == 2 + assert 0 < opacity <= 1.0 + color_seg = np.zeros((seg.shape[0], seg.shape[1], 3), dtype=np.uint8) + for label, color in enumerate(palette): + color_seg[seg == label, :] = color + # convert to BGR + color_seg = color_seg[..., ::-1] + + img = img * (1 - opacity) + color_seg * opacity + img = img.astype(np.uint8) + # if out_file specified, do not show image in window + if out_file is not None: + show = False + + if show: + mmcv.imshow(img, win_name, wait_time) + if out_file is not None: + mmcv.imwrite(img, out_file) + + if not (show or out_file): + warnings.warn('show==False and out_file is not specified, only ' + 'result image will be returned') + return img diff --git a/annotator/uniformer/mmseg/models/segmentors/cascade_encoder_decoder.py b/annotator/uniformer/mmseg/models/segmentors/cascade_encoder_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..873957d8d6468147c994493d92ff5c1b15bfb703 --- /dev/null +++ b/annotator/uniformer/mmseg/models/segmentors/cascade_encoder_decoder.py @@ -0,0 +1,98 @@ +from torch import nn + +from annotator.uniformer.mmseg.core import add_prefix +from annotator.uniformer.mmseg.ops import resize +from .. import builder +from ..builder import SEGMENTORS +from .encoder_decoder import EncoderDecoder + + +@SEGMENTORS.register_module() +class CascadeEncoderDecoder(EncoderDecoder): + """Cascade Encoder Decoder segmentors. + + CascadeEncoderDecoder almost the same as EncoderDecoder, while decoders of + CascadeEncoderDecoder are cascaded. The output of previous decoder_head + will be the input of next decoder_head. + """ + + def __init__(self, + num_stages, + backbone, + decode_head, + neck=None, + auxiliary_head=None, + train_cfg=None, + test_cfg=None, + pretrained=None): + self.num_stages = num_stages + super(CascadeEncoderDecoder, self).__init__( + backbone=backbone, + decode_head=decode_head, + neck=neck, + auxiliary_head=auxiliary_head, + train_cfg=train_cfg, + test_cfg=test_cfg, + pretrained=pretrained) + + def _init_decode_head(self, decode_head): + """Initialize ``decode_head``""" + assert isinstance(decode_head, list) + assert len(decode_head) == self.num_stages + self.decode_head = nn.ModuleList() + for i in range(self.num_stages): + self.decode_head.append(builder.build_head(decode_head[i])) + self.align_corners = self.decode_head[-1].align_corners + self.num_classes = self.decode_head[-1].num_classes + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone and heads. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + self.backbone.init_weights(pretrained=pretrained) + for i in range(self.num_stages): + self.decode_head[i].init_weights() + if self.with_auxiliary_head: + if isinstance(self.auxiliary_head, nn.ModuleList): + for aux_head in self.auxiliary_head: + aux_head.init_weights() + else: + self.auxiliary_head.init_weights() + + def encode_decode(self, img, img_metas): + """Encode images with backbone and decode into a semantic segmentation + map of the same size as input.""" + x = self.extract_feat(img) + out = self.decode_head[0].forward_test(x, img_metas, self.test_cfg) + for i in range(1, self.num_stages): + out = self.decode_head[i].forward_test(x, out, img_metas, + self.test_cfg) + out = resize( + input=out, + size=img.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + return out + + def _decode_head_forward_train(self, x, img_metas, gt_semantic_seg): + """Run forward function and calculate loss for decode head in + training.""" + losses = dict() + + loss_decode = self.decode_head[0].forward_train( + x, img_metas, gt_semantic_seg, self.train_cfg) + + losses.update(add_prefix(loss_decode, 'decode_0')) + + for i in range(1, self.num_stages): + # forward test again, maybe unnecessary for most methods. + prev_outputs = self.decode_head[i - 1].forward_test( + x, img_metas, self.test_cfg) + loss_decode = self.decode_head[i].forward_train( + x, prev_outputs, img_metas, gt_semantic_seg, self.train_cfg) + losses.update(add_prefix(loss_decode, f'decode_{i}')) + + return losses diff --git a/annotator/uniformer/mmseg/models/segmentors/encoder_decoder.py b/annotator/uniformer/mmseg/models/segmentors/encoder_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..98392ac04c4c44a7f4e7b1c0808266875877dd1f --- /dev/null +++ b/annotator/uniformer/mmseg/models/segmentors/encoder_decoder.py @@ -0,0 +1,298 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +from annotator.uniformer.mmseg.core import add_prefix +from annotator.uniformer.mmseg.ops import resize +from .. import builder +from ..builder import SEGMENTORS +from .base import BaseSegmentor + + +@SEGMENTORS.register_module() +class EncoderDecoder(BaseSegmentor): + """Encoder Decoder segmentors. + + EncoderDecoder typically consists of backbone, decode_head, auxiliary_head. + Note that auxiliary_head is only used for deep supervision during training, + which could be dumped during inference. + """ + + def __init__(self, + backbone, + decode_head, + neck=None, + auxiliary_head=None, + train_cfg=None, + test_cfg=None, + pretrained=None): + super(EncoderDecoder, self).__init__() + self.backbone = builder.build_backbone(backbone) + if neck is not None: + self.neck = builder.build_neck(neck) + self._init_decode_head(decode_head) + self._init_auxiliary_head(auxiliary_head) + + self.train_cfg = train_cfg + self.test_cfg = test_cfg + + self.init_weights(pretrained=pretrained) + + assert self.with_decode_head + + def _init_decode_head(self, decode_head): + """Initialize ``decode_head``""" + self.decode_head = builder.build_head(decode_head) + self.align_corners = self.decode_head.align_corners + self.num_classes = self.decode_head.num_classes + + def _init_auxiliary_head(self, auxiliary_head): + """Initialize ``auxiliary_head``""" + if auxiliary_head is not None: + if isinstance(auxiliary_head, list): + self.auxiliary_head = nn.ModuleList() + for head_cfg in auxiliary_head: + self.auxiliary_head.append(builder.build_head(head_cfg)) + else: + self.auxiliary_head = builder.build_head(auxiliary_head) + + def init_weights(self, pretrained=None): + """Initialize the weights in backbone and heads. + + Args: + pretrained (str, optional): Path to pre-trained weights. + Defaults to None. + """ + + super(EncoderDecoder, self).init_weights(pretrained) + self.backbone.init_weights(pretrained=pretrained) + self.decode_head.init_weights() + if self.with_auxiliary_head: + if isinstance(self.auxiliary_head, nn.ModuleList): + for aux_head in self.auxiliary_head: + aux_head.init_weights() + else: + self.auxiliary_head.init_weights() + + def extract_feat(self, img): + """Extract features from images.""" + x = self.backbone(img) + if self.with_neck: + x = self.neck(x) + return x + + def encode_decode(self, img, img_metas): + """Encode images with backbone and decode into a semantic segmentation + map of the same size as input.""" + x = self.extract_feat(img) + out = self._decode_head_forward_test(x, img_metas) + out = resize( + input=out, + size=img.shape[2:], + mode='bilinear', + align_corners=self.align_corners) + return out + + def _decode_head_forward_train(self, x, img_metas, gt_semantic_seg): + """Run forward function and calculate loss for decode head in + training.""" + losses = dict() + loss_decode = self.decode_head.forward_train(x, img_metas, + gt_semantic_seg, + self.train_cfg) + + losses.update(add_prefix(loss_decode, 'decode')) + return losses + + def _decode_head_forward_test(self, x, img_metas): + """Run forward function and calculate loss for decode head in + inference.""" + seg_logits = self.decode_head.forward_test(x, img_metas, self.test_cfg) + return seg_logits + + def _auxiliary_head_forward_train(self, x, img_metas, gt_semantic_seg): + """Run forward function and calculate loss for auxiliary head in + training.""" + losses = dict() + if isinstance(self.auxiliary_head, nn.ModuleList): + for idx, aux_head in enumerate(self.auxiliary_head): + loss_aux = aux_head.forward_train(x, img_metas, + gt_semantic_seg, + self.train_cfg) + losses.update(add_prefix(loss_aux, f'aux_{idx}')) + else: + loss_aux = self.auxiliary_head.forward_train( + x, img_metas, gt_semantic_seg, self.train_cfg) + losses.update(add_prefix(loss_aux, 'aux')) + + return losses + + def forward_dummy(self, img): + """Dummy forward function.""" + seg_logit = self.encode_decode(img, None) + + return seg_logit + + def forward_train(self, img, img_metas, gt_semantic_seg): + """Forward function for training. + + Args: + img (Tensor): Input images. + img_metas (list[dict]): List of image info dict where each dict + has: 'img_shape', 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + gt_semantic_seg (Tensor): Semantic segmentation masks + used if the architecture supports semantic segmentation task. + + Returns: + dict[str, Tensor]: a dictionary of loss components + """ + + x = self.extract_feat(img) + + losses = dict() + + loss_decode = self._decode_head_forward_train(x, img_metas, + gt_semantic_seg) + losses.update(loss_decode) + + if self.with_auxiliary_head: + loss_aux = self._auxiliary_head_forward_train( + x, img_metas, gt_semantic_seg) + losses.update(loss_aux) + + return losses + + # TODO refactor + def slide_inference(self, img, img_meta, rescale): + """Inference by sliding-window with overlap. + + If h_crop > h_img or w_crop > w_img, the small patch will be used to + decode without padding. + """ + + h_stride, w_stride = self.test_cfg.stride + h_crop, w_crop = self.test_cfg.crop_size + batch_size, _, h_img, w_img = img.size() + num_classes = self.num_classes + h_grids = max(h_img - h_crop + h_stride - 1, 0) // h_stride + 1 + w_grids = max(w_img - w_crop + w_stride - 1, 0) // w_stride + 1 + preds = img.new_zeros((batch_size, num_classes, h_img, w_img)) + count_mat = img.new_zeros((batch_size, 1, h_img, w_img)) + for h_idx in range(h_grids): + for w_idx in range(w_grids): + y1 = h_idx * h_stride + x1 = w_idx * w_stride + y2 = min(y1 + h_crop, h_img) + x2 = min(x1 + w_crop, w_img) + y1 = max(y2 - h_crop, 0) + x1 = max(x2 - w_crop, 0) + crop_img = img[:, :, y1:y2, x1:x2] + crop_seg_logit = self.encode_decode(crop_img, img_meta) + preds += F.pad(crop_seg_logit, + (int(x1), int(preds.shape[3] - x2), int(y1), + int(preds.shape[2] - y2))) + + count_mat[:, :, y1:y2, x1:x2] += 1 + assert (count_mat == 0).sum() == 0 + if torch.onnx.is_in_onnx_export(): + # cast count_mat to constant while exporting to ONNX + count_mat = torch.from_numpy( + count_mat.cpu().detach().numpy()).to(device=img.device) + preds = preds / count_mat + if rescale: + preds = resize( + preds, + size=img_meta[0]['ori_shape'][:2], + mode='bilinear', + align_corners=self.align_corners, + warning=False) + return preds + + def whole_inference(self, img, img_meta, rescale): + """Inference with full image.""" + + seg_logit = self.encode_decode(img, img_meta) + if rescale: + # support dynamic shape for onnx + if torch.onnx.is_in_onnx_export(): + size = img.shape[2:] + else: + size = img_meta[0]['ori_shape'][:2] + seg_logit = resize( + seg_logit, + size=size, + mode='bilinear', + align_corners=self.align_corners, + warning=False) + + return seg_logit + + def inference(self, img, img_meta, rescale): + """Inference with slide/whole style. + + Args: + img (Tensor): The input image of shape (N, 3, H, W). + img_meta (dict): Image info dict where each dict has: 'img_shape', + 'scale_factor', 'flip', and may also contain + 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. + For details on the values of these keys see + `mmseg/datasets/pipelines/formatting.py:Collect`. + rescale (bool): Whether rescale back to original shape. + + Returns: + Tensor: The output segmentation map. + """ + + assert self.test_cfg.mode in ['slide', 'whole'] + ori_shape = img_meta[0]['ori_shape'] + assert all(_['ori_shape'] == ori_shape for _ in img_meta) + if self.test_cfg.mode == 'slide': + seg_logit = self.slide_inference(img, img_meta, rescale) + else: + seg_logit = self.whole_inference(img, img_meta, rescale) + output = F.softmax(seg_logit, dim=1) + flip = img_meta[0]['flip'] + if flip: + flip_direction = img_meta[0]['flip_direction'] + assert flip_direction in ['horizontal', 'vertical'] + if flip_direction == 'horizontal': + output = output.flip(dims=(3, )) + elif flip_direction == 'vertical': + output = output.flip(dims=(2, )) + + return output + + def simple_test(self, img, img_meta, rescale=True): + """Simple test with single image.""" + seg_logit = self.inference(img, img_meta, rescale) + seg_pred = seg_logit.argmax(dim=1) + if torch.onnx.is_in_onnx_export(): + # our inference backend only support 4D output + seg_pred = seg_pred.unsqueeze(0) + return seg_pred + seg_pred = seg_pred.cpu().numpy() + # unravel batch dim + seg_pred = list(seg_pred) + return seg_pred + + def aug_test(self, imgs, img_metas, rescale=True): + """Test with augmentations. + + Only rescale=True is supported. + """ + # aug_test rescale all imgs back to ori_shape for now + assert rescale + # to save memory, we get augmented seg logit inplace + seg_logit = self.inference(imgs[0], img_metas[0], rescale) + for i in range(1, len(imgs)): + cur_seg_logit = self.inference(imgs[i], img_metas[i], rescale) + seg_logit += cur_seg_logit + seg_logit /= len(imgs) + seg_pred = seg_logit.argmax(dim=1) + seg_pred = seg_pred.cpu().numpy() + # unravel batch dim + seg_pred = list(seg_pred) + return seg_pred diff --git a/annotator/uniformer/mmseg/models/utils/__init__.py b/annotator/uniformer/mmseg/models/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3d3bdd349b9f2ae499a2fcb2ac1d2e3c77befebe --- /dev/null +++ b/annotator/uniformer/mmseg/models/utils/__init__.py @@ -0,0 +1,13 @@ +from .drop import DropPath +from .inverted_residual import InvertedResidual, InvertedResidualV3 +from .make_divisible import make_divisible +from .res_layer import ResLayer +from .se_layer import SELayer +from .self_attention_block import SelfAttentionBlock +from .up_conv_block import UpConvBlock +from .weight_init import trunc_normal_ + +__all__ = [ + 'ResLayer', 'SelfAttentionBlock', 'make_divisible', 'InvertedResidual', + 'UpConvBlock', 'InvertedResidualV3', 'SELayer', 'DropPath', 'trunc_normal_' +] diff --git a/annotator/uniformer/mmseg/models/utils/drop.py b/annotator/uniformer/mmseg/models/utils/drop.py new file mode 100644 index 0000000000000000000000000000000000000000..4520b0ff407d2a95a864086bdbca0065f222aa63 --- /dev/null +++ b/annotator/uniformer/mmseg/models/utils/drop.py @@ -0,0 +1,31 @@ +"""Modified from https://github.com/rwightman/pytorch-image- +models/blob/master/timm/models/layers/drop.py.""" + +import torch +from torch import nn + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of + residual blocks). + + Args: + drop_prob (float): Drop rate for paths of model. Dropout rate has + to be between 0 and 1. Default: 0. + """ + + def __init__(self, drop_prob=0.): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + self.keep_prob = 1 - drop_prob + + def forward(self, x): + if self.drop_prob == 0. or not self.training: + return x + shape = (x.shape[0], ) + (1, ) * ( + x.ndim - 1) # work with diff dim tensors, not just 2D ConvNets + random_tensor = self.keep_prob + torch.rand( + shape, dtype=x.dtype, device=x.device) + random_tensor.floor_() # binarize + output = x.div(self.keep_prob) * random_tensor + return output diff --git a/annotator/uniformer/mmseg/models/utils/inverted_residual.py b/annotator/uniformer/mmseg/models/utils/inverted_residual.py new file mode 100644 index 0000000000000000000000000000000000000000..53b8fcd41f71d814738f1ac3f5acd3c3d701bf96 --- /dev/null +++ b/annotator/uniformer/mmseg/models/utils/inverted_residual.py @@ -0,0 +1,208 @@ +from annotator.uniformer.mmcv.cnn import ConvModule +from torch import nn +from torch.utils import checkpoint as cp + +from .se_layer import SELayer + + +class InvertedResidual(nn.Module): + """InvertedResidual block for MobileNetV2. + + Args: + in_channels (int): The input channels of the InvertedResidual block. + out_channels (int): The output channels of the InvertedResidual block. + stride (int): Stride of the middle (first) 3x3 convolution. + expand_ratio (int): Adjusts number of channels of the hidden layer + in InvertedResidual by this amount. + dilation (int): Dilation rate of depthwise conv. Default: 1 + conv_cfg (dict): Config dict for convolution layer. + Default: None, which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='ReLU6'). + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + + Returns: + Tensor: The output tensor. + """ + + def __init__(self, + in_channels, + out_channels, + stride, + expand_ratio, + dilation=1, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU6'), + with_cp=False): + super(InvertedResidual, self).__init__() + self.stride = stride + assert stride in [1, 2], f'stride must in [1, 2]. ' \ + f'But received {stride}.' + self.with_cp = with_cp + self.use_res_connect = self.stride == 1 and in_channels == out_channels + hidden_dim = int(round(in_channels * expand_ratio)) + + layers = [] + if expand_ratio != 1: + layers.append( + ConvModule( + in_channels=in_channels, + out_channels=hidden_dim, + kernel_size=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + layers.extend([ + ConvModule( + in_channels=hidden_dim, + out_channels=hidden_dim, + kernel_size=3, + stride=stride, + padding=dilation, + dilation=dilation, + groups=hidden_dim, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg), + ConvModule( + in_channels=hidden_dim, + out_channels=out_channels, + kernel_size=1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=None) + ]) + self.conv = nn.Sequential(*layers) + + def forward(self, x): + + def _inner_forward(x): + if self.use_res_connect: + return x + self.conv(x) + else: + return self.conv(x) + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + return out + + +class InvertedResidualV3(nn.Module): + """Inverted Residual Block for MobileNetV3. + + Args: + in_channels (int): The input channels of this Module. + out_channels (int): The output channels of this Module. + mid_channels (int): The input channels of the depthwise convolution. + kernel_size (int): The kernel size of the depthwise convolution. + Default: 3. + stride (int): The stride of the depthwise convolution. Default: 1. + se_cfg (dict): Config dict for se layer. Default: None, which means no + se layer. + with_expand_conv (bool): Use expand conv or not. If set False, + mid_channels must be the same with in_channels. Default: True. + conv_cfg (dict): Config dict for convolution layer. Default: None, + which means using conv2d. + norm_cfg (dict): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict): Config dict for activation layer. + Default: dict(type='ReLU'). + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + + Returns: + Tensor: The output tensor. + """ + + def __init__(self, + in_channels, + out_channels, + mid_channels, + kernel_size=3, + stride=1, + se_cfg=None, + with_expand_conv=True, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + with_cp=False): + super(InvertedResidualV3, self).__init__() + self.with_res_shortcut = (stride == 1 and in_channels == out_channels) + assert stride in [1, 2] + self.with_cp = with_cp + self.with_se = se_cfg is not None + self.with_expand_conv = with_expand_conv + + if self.with_se: + assert isinstance(se_cfg, dict) + if not self.with_expand_conv: + assert mid_channels == in_channels + + if self.with_expand_conv: + self.expand_conv = ConvModule( + in_channels=in_channels, + out_channels=mid_channels, + kernel_size=1, + stride=1, + padding=0, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + self.depthwise_conv = ConvModule( + in_channels=mid_channels, + out_channels=mid_channels, + kernel_size=kernel_size, + stride=stride, + padding=kernel_size // 2, + groups=mid_channels, + conv_cfg=dict( + type='Conv2dAdaptivePadding') if stride == 2 else conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + + if self.with_se: + self.se = SELayer(**se_cfg) + + self.linear_conv = ConvModule( + in_channels=mid_channels, + out_channels=out_channels, + kernel_size=1, + stride=1, + padding=0, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=None) + + def forward(self, x): + + def _inner_forward(x): + out = x + + if self.with_expand_conv: + out = self.expand_conv(out) + + out = self.depthwise_conv(out) + + if self.with_se: + out = self.se(out) + + out = self.linear_conv(out) + + if self.with_res_shortcut: + return x + out + else: + return out + + if self.with_cp and x.requires_grad: + out = cp.checkpoint(_inner_forward, x) + else: + out = _inner_forward(x) + + return out diff --git a/annotator/uniformer/mmseg/models/utils/make_divisible.py b/annotator/uniformer/mmseg/models/utils/make_divisible.py new file mode 100644 index 0000000000000000000000000000000000000000..75ad756052529f52fe83bb95dd1f0ecfc9a13078 --- /dev/null +++ b/annotator/uniformer/mmseg/models/utils/make_divisible.py @@ -0,0 +1,27 @@ +def make_divisible(value, divisor, min_value=None, min_ratio=0.9): + """Make divisible function. + + This function rounds the channel number to the nearest value that can be + divisible by the divisor. It is taken from the original tf repo. It ensures + that all layers have a channel number that is divisible by divisor. It can + be seen here: https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py # noqa + + Args: + value (int): The original channel number. + divisor (int): The divisor to fully divide the channel number. + min_value (int): The minimum value of the output channel. + Default: None, means that the minimum value equal to the divisor. + min_ratio (float): The minimum ratio of the rounded channel number to + the original channel number. Default: 0.9. + + Returns: + int: The modified output channel number. + """ + + if min_value is None: + min_value = divisor + new_value = max(min_value, int(value + divisor / 2) // divisor * divisor) + # Make sure that round down does not go down by more than (1-min_ratio). + if new_value < min_ratio * value: + new_value += divisor + return new_value diff --git a/annotator/uniformer/mmseg/models/utils/res_layer.py b/annotator/uniformer/mmseg/models/utils/res_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..b2c07b47007e92e4c3945b989e79f9d50306f5fe --- /dev/null +++ b/annotator/uniformer/mmseg/models/utils/res_layer.py @@ -0,0 +1,94 @@ +from annotator.uniformer.mmcv.cnn import build_conv_layer, build_norm_layer +from torch import nn as nn + + +class ResLayer(nn.Sequential): + """ResLayer to build ResNet style backbone. + + Args: + block (nn.Module): block used to build ResLayer. + inplanes (int): inplanes of block. + planes (int): planes of block. + num_blocks (int): number of blocks. + stride (int): stride of the first block. Default: 1 + avg_down (bool): Use AvgPool instead of stride conv when + downsampling in the bottleneck. Default: False + conv_cfg (dict): dictionary to construct and config conv layer. + Default: None + norm_cfg (dict): dictionary to construct and config norm layer. + Default: dict(type='BN') + multi_grid (int | None): Multi grid dilation rates of last + stage. Default: None + contract_dilation (bool): Whether contract first dilation of each layer + Default: False + """ + + def __init__(self, + block, + inplanes, + planes, + num_blocks, + stride=1, + dilation=1, + avg_down=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + multi_grid=None, + contract_dilation=False, + **kwargs): + self.block = block + + downsample = None + if stride != 1 or inplanes != planes * block.expansion: + downsample = [] + conv_stride = stride + if avg_down: + conv_stride = 1 + downsample.append( + nn.AvgPool2d( + kernel_size=stride, + stride=stride, + ceil_mode=True, + count_include_pad=False)) + downsample.extend([ + build_conv_layer( + conv_cfg, + inplanes, + planes * block.expansion, + kernel_size=1, + stride=conv_stride, + bias=False), + build_norm_layer(norm_cfg, planes * block.expansion)[1] + ]) + downsample = nn.Sequential(*downsample) + + layers = [] + if multi_grid is None: + if dilation > 1 and contract_dilation: + first_dilation = dilation // 2 + else: + first_dilation = dilation + else: + first_dilation = multi_grid[0] + layers.append( + block( + inplanes=inplanes, + planes=planes, + stride=stride, + dilation=first_dilation, + downsample=downsample, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + **kwargs)) + inplanes = planes * block.expansion + for i in range(1, num_blocks): + layers.append( + block( + inplanes=inplanes, + planes=planes, + stride=1, + dilation=dilation if multi_grid is None else multi_grid[i], + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + **kwargs)) + super(ResLayer, self).__init__(*layers) diff --git a/annotator/uniformer/mmseg/models/utils/se_layer.py b/annotator/uniformer/mmseg/models/utils/se_layer.py new file mode 100644 index 0000000000000000000000000000000000000000..083bd7d1ccee909c900c7aed2cc928bf14727f3e --- /dev/null +++ b/annotator/uniformer/mmseg/models/utils/se_layer.py @@ -0,0 +1,57 @@ +import annotator.uniformer.mmcv as mmcv +import torch.nn as nn +from annotator.uniformer.mmcv.cnn import ConvModule + +from .make_divisible import make_divisible + + +class SELayer(nn.Module): + """Squeeze-and-Excitation Module. + + Args: + channels (int): The input (and output) channels of the SE layer. + ratio (int): Squeeze ratio in SELayer, the intermediate channel will be + ``int(channels/ratio)``. Default: 16. + conv_cfg (None or dict): Config dict for convolution layer. + Default: None, which means using conv2d. + act_cfg (dict or Sequence[dict]): Config dict for activation layer. + If act_cfg is a dict, two activation layers will be configured + by this dict. If act_cfg is a sequence of dicts, the first + activation layer will be configured by the first dict and the + second activation layer will be configured by the second dict. + Default: (dict(type='ReLU'), dict(type='HSigmoid', bias=3.0, + divisor=6.0)). + """ + + def __init__(self, + channels, + ratio=16, + conv_cfg=None, + act_cfg=(dict(type='ReLU'), + dict(type='HSigmoid', bias=3.0, divisor=6.0))): + super(SELayer, self).__init__() + if isinstance(act_cfg, dict): + act_cfg = (act_cfg, act_cfg) + assert len(act_cfg) == 2 + assert mmcv.is_tuple_of(act_cfg, dict) + self.global_avgpool = nn.AdaptiveAvgPool2d(1) + self.conv1 = ConvModule( + in_channels=channels, + out_channels=make_divisible(channels // ratio, 8), + kernel_size=1, + stride=1, + conv_cfg=conv_cfg, + act_cfg=act_cfg[0]) + self.conv2 = ConvModule( + in_channels=make_divisible(channels // ratio, 8), + out_channels=channels, + kernel_size=1, + stride=1, + conv_cfg=conv_cfg, + act_cfg=act_cfg[1]) + + def forward(self, x): + out = self.global_avgpool(x) + out = self.conv1(out) + out = self.conv2(out) + return x * out diff --git a/annotator/uniformer/mmseg/models/utils/self_attention_block.py b/annotator/uniformer/mmseg/models/utils/self_attention_block.py new file mode 100644 index 0000000000000000000000000000000000000000..440c7b73ee4706fde555595926d63a18d7574acc --- /dev/null +++ b/annotator/uniformer/mmseg/models/utils/self_attention_block.py @@ -0,0 +1,159 @@ +import torch +from annotator.uniformer.mmcv.cnn import ConvModule, constant_init +from torch import nn as nn +from torch.nn import functional as F + + +class SelfAttentionBlock(nn.Module): + """General self-attention block/non-local block. + + Please refer to https://arxiv.org/abs/1706.03762 for details about key, + query and value. + + Args: + key_in_channels (int): Input channels of key feature. + query_in_channels (int): Input channels of query feature. + channels (int): Output channels of key/query transform. + out_channels (int): Output channels. + share_key_query (bool): Whether share projection weight between key + and query projection. + query_downsample (nn.Module): Query downsample module. + key_downsample (nn.Module): Key downsample module. + key_query_num_convs (int): Number of convs for key/query projection. + value_num_convs (int): Number of convs for value projection. + matmul_norm (bool): Whether normalize attention map with sqrt of + channels + with_out (bool): Whether use out projection. + conv_cfg (dict|None): Config of conv layers. + norm_cfg (dict|None): Config of norm layers. + act_cfg (dict|None): Config of activation layers. + """ + + def __init__(self, key_in_channels, query_in_channels, channels, + out_channels, share_key_query, query_downsample, + key_downsample, key_query_num_convs, value_out_num_convs, + key_query_norm, value_out_norm, matmul_norm, with_out, + conv_cfg, norm_cfg, act_cfg): + super(SelfAttentionBlock, self).__init__() + if share_key_query: + assert key_in_channels == query_in_channels + self.key_in_channels = key_in_channels + self.query_in_channels = query_in_channels + self.out_channels = out_channels + self.channels = channels + self.share_key_query = share_key_query + self.conv_cfg = conv_cfg + self.norm_cfg = norm_cfg + self.act_cfg = act_cfg + self.key_project = self.build_project( + key_in_channels, + channels, + num_convs=key_query_num_convs, + use_conv_module=key_query_norm, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + if share_key_query: + self.query_project = self.key_project + else: + self.query_project = self.build_project( + query_in_channels, + channels, + num_convs=key_query_num_convs, + use_conv_module=key_query_norm, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + self.value_project = self.build_project( + key_in_channels, + channels if with_out else out_channels, + num_convs=value_out_num_convs, + use_conv_module=value_out_norm, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + if with_out: + self.out_project = self.build_project( + channels, + out_channels, + num_convs=value_out_num_convs, + use_conv_module=value_out_norm, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + else: + self.out_project = None + + self.query_downsample = query_downsample + self.key_downsample = key_downsample + self.matmul_norm = matmul_norm + + self.init_weights() + + def init_weights(self): + """Initialize weight of later layer.""" + if self.out_project is not None: + if not isinstance(self.out_project, ConvModule): + constant_init(self.out_project, 0) + + def build_project(self, in_channels, channels, num_convs, use_conv_module, + conv_cfg, norm_cfg, act_cfg): + """Build projection layer for key/query/value/out.""" + if use_conv_module: + convs = [ + ConvModule( + in_channels, + channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + ] + for _ in range(num_convs - 1): + convs.append( + ConvModule( + channels, + channels, + 1, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg)) + else: + convs = [nn.Conv2d(in_channels, channels, 1)] + for _ in range(num_convs - 1): + convs.append(nn.Conv2d(channels, channels, 1)) + if len(convs) > 1: + convs = nn.Sequential(*convs) + else: + convs = convs[0] + return convs + + def forward(self, query_feats, key_feats): + """Forward function.""" + batch_size = query_feats.size(0) + query = self.query_project(query_feats) + if self.query_downsample is not None: + query = self.query_downsample(query) + query = query.reshape(*query.shape[:2], -1) + query = query.permute(0, 2, 1).contiguous() + + key = self.key_project(key_feats) + value = self.value_project(key_feats) + if self.key_downsample is not None: + key = self.key_downsample(key) + value = self.key_downsample(value) + key = key.reshape(*key.shape[:2], -1) + value = value.reshape(*value.shape[:2], -1) + value = value.permute(0, 2, 1).contiguous() + + sim_map = torch.matmul(query, key) + if self.matmul_norm: + sim_map = (self.channels**-.5) * sim_map + sim_map = F.softmax(sim_map, dim=-1) + + context = torch.matmul(sim_map, value) + context = context.permute(0, 2, 1).contiguous() + context = context.reshape(batch_size, -1, *query_feats.shape[2:]) + if self.out_project is not None: + context = self.out_project(context) + return context diff --git a/annotator/uniformer/mmseg/models/utils/up_conv_block.py b/annotator/uniformer/mmseg/models/utils/up_conv_block.py new file mode 100644 index 0000000000000000000000000000000000000000..378469da76cb7bff6a639e7877b3c275d50490fb --- /dev/null +++ b/annotator/uniformer/mmseg/models/utils/up_conv_block.py @@ -0,0 +1,101 @@ +import torch +import torch.nn as nn +from annotator.uniformer.mmcv.cnn import ConvModule, build_upsample_layer + + +class UpConvBlock(nn.Module): + """Upsample convolution block in decoder for UNet. + + This upsample convolution block consists of one upsample module + followed by one convolution block. The upsample module expands the + high-level low-resolution feature map and the convolution block fuses + the upsampled high-level low-resolution feature map and the low-level + high-resolution feature map from encoder. + + Args: + conv_block (nn.Sequential): Sequential of convolutional layers. + in_channels (int): Number of input channels of the high-level + skip_channels (int): Number of input channels of the low-level + high-resolution feature map from encoder. + out_channels (int): Number of output channels. + num_convs (int): Number of convolutional layers in the conv_block. + Default: 2. + stride (int): Stride of convolutional layer in conv_block. Default: 1. + dilation (int): Dilation rate of convolutional layer in conv_block. + Default: 1. + with_cp (bool): Use checkpoint or not. Using checkpoint will save some + memory while slowing down the training speed. Default: False. + conv_cfg (dict | None): Config dict for convolution layer. + Default: None. + norm_cfg (dict | None): Config dict for normalization layer. + Default: dict(type='BN'). + act_cfg (dict | None): Config dict for activation layer in ConvModule. + Default: dict(type='ReLU'). + upsample_cfg (dict): The upsample config of the upsample module in + decoder. Default: dict(type='InterpConv'). If the size of + high-level feature map is the same as that of skip feature map + (low-level feature map from encoder), it does not need upsample the + high-level feature map and the upsample_cfg is None. + dcn (bool): Use deformable convolution in convolutional layer or not. + Default: None. + plugins (dict): plugins for convolutional layers. Default: None. + """ + + def __init__(self, + conv_block, + in_channels, + skip_channels, + out_channels, + num_convs=2, + stride=1, + dilation=1, + with_cp=False, + conv_cfg=None, + norm_cfg=dict(type='BN'), + act_cfg=dict(type='ReLU'), + upsample_cfg=dict(type='InterpConv'), + dcn=None, + plugins=None): + super(UpConvBlock, self).__init__() + assert dcn is None, 'Not implemented yet.' + assert plugins is None, 'Not implemented yet.' + + self.conv_block = conv_block( + in_channels=2 * skip_channels, + out_channels=out_channels, + num_convs=num_convs, + stride=stride, + dilation=dilation, + with_cp=with_cp, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg, + dcn=None, + plugins=None) + if upsample_cfg is not None: + self.upsample = build_upsample_layer( + cfg=upsample_cfg, + in_channels=in_channels, + out_channels=skip_channels, + with_cp=with_cp, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + else: + self.upsample = ConvModule( + in_channels, + skip_channels, + kernel_size=1, + stride=1, + padding=0, + conv_cfg=conv_cfg, + norm_cfg=norm_cfg, + act_cfg=act_cfg) + + def forward(self, skip, x): + """Forward function.""" + + x = self.upsample(x) + out = torch.cat([skip, x], dim=1) + out = self.conv_block(out) + + return out diff --git a/annotator/uniformer/mmseg/models/utils/weight_init.py b/annotator/uniformer/mmseg/models/utils/weight_init.py new file mode 100644 index 0000000000000000000000000000000000000000..38141ba3d61f64ddfc0a31574b4648cbad96d7dd --- /dev/null +++ b/annotator/uniformer/mmseg/models/utils/weight_init.py @@ -0,0 +1,62 @@ +"""Modified from https://github.com/rwightman/pytorch-image- +models/blob/master/timm/models/layers/drop.py.""" + +import math +import warnings + +import torch + + +def _no_grad_trunc_normal_(tensor, mean, std, a, b): + """Reference: https://people.sc.fsu.edu/~jburkardt/presentations + /truncated_normal.pdf""" + + def norm_cdf(x): + # Computes standard normal cumulative distribution function + return (1. + math.erf(x / math.sqrt(2.))) / 2. + + if (mean < a - 2 * std) or (mean > b + 2 * std): + warnings.warn( + 'mean is more than 2 std from [a, b] in nn.init.trunc_normal_. ' + 'The distribution of values may be incorrect.', + stacklevel=2) + + with torch.no_grad(): + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + lower_bound = norm_cdf((a - mean) / std) + upper_bound = norm_cdf((b - mean) / std) + + # Uniformly fill tensor with values from [l, u], then translate to + # [2l-1, 2u-1]. + tensor.uniform_(2 * lower_bound - 1, 2 * upper_bound - 1) + + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() + + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.)) + tensor.add_(mean) + + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor + + +def trunc_normal_(tensor, mean=0., std=1., a=-2., b=2.): + r"""Fills the input Tensor with values drawn from a truncated + normal distribution. The values are effectively drawn from the + normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)` + with values outside :math:`[a, b]` redrawn until they are within + the bounds. The method used for generating the random values works + best when :math:`a \leq \text{mean} \leq b`. + Args: + tensor (``torch.Tensor``): an n-dimensional `torch.Tensor` + mean (float): the mean of the normal distribution + std (float): the standard deviation of the normal distribution + a (float): the minimum cutoff value + b (float): the maximum cutoff value + """ + return _no_grad_trunc_normal_(tensor, mean, std, a, b) diff --git a/annotator/uniformer/mmseg/ops/__init__.py b/annotator/uniformer/mmseg/ops/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bec51c75b9363a9a19e9fb5c35f4e7dbd6f7751c --- /dev/null +++ b/annotator/uniformer/mmseg/ops/__init__.py @@ -0,0 +1,4 @@ +from .encoding import Encoding +from .wrappers import Upsample, resize + +__all__ = ['Upsample', 'resize', 'Encoding'] diff --git a/annotator/uniformer/mmseg/ops/encoding.py b/annotator/uniformer/mmseg/ops/encoding.py new file mode 100644 index 0000000000000000000000000000000000000000..7eb3629a6426550b8e4c537ee1ff4341893e489e --- /dev/null +++ b/annotator/uniformer/mmseg/ops/encoding.py @@ -0,0 +1,74 @@ +import torch +from torch import nn +from torch.nn import functional as F + + +class Encoding(nn.Module): + """Encoding Layer: a learnable residual encoder. + + Input is of shape (batch_size, channels, height, width). + Output is of shape (batch_size, num_codes, channels). + + Args: + channels: dimension of the features or feature channels + num_codes: number of code words + """ + + def __init__(self, channels, num_codes): + super(Encoding, self).__init__() + # init codewords and smoothing factor + self.channels, self.num_codes = channels, num_codes + std = 1. / ((num_codes * channels)**0.5) + # [num_codes, channels] + self.codewords = nn.Parameter( + torch.empty(num_codes, channels, + dtype=torch.float).uniform_(-std, std), + requires_grad=True) + # [num_codes] + self.scale = nn.Parameter( + torch.empty(num_codes, dtype=torch.float).uniform_(-1, 0), + requires_grad=True) + + @staticmethod + def scaled_l2(x, codewords, scale): + num_codes, channels = codewords.size() + batch_size = x.size(0) + reshaped_scale = scale.view((1, 1, num_codes)) + expanded_x = x.unsqueeze(2).expand( + (batch_size, x.size(1), num_codes, channels)) + reshaped_codewords = codewords.view((1, 1, num_codes, channels)) + + scaled_l2_norm = reshaped_scale * ( + expanded_x - reshaped_codewords).pow(2).sum(dim=3) + return scaled_l2_norm + + @staticmethod + def aggregate(assignment_weights, x, codewords): + num_codes, channels = codewords.size() + reshaped_codewords = codewords.view((1, 1, num_codes, channels)) + batch_size = x.size(0) + + expanded_x = x.unsqueeze(2).expand( + (batch_size, x.size(1), num_codes, channels)) + encoded_feat = (assignment_weights.unsqueeze(3) * + (expanded_x - reshaped_codewords)).sum(dim=1) + return encoded_feat + + def forward(self, x): + assert x.dim() == 4 and x.size(1) == self.channels + # [batch_size, channels, height, width] + batch_size = x.size(0) + # [batch_size, height x width, channels] + x = x.view(batch_size, self.channels, -1).transpose(1, 2).contiguous() + # assignment_weights: [batch_size, channels, num_codes] + assignment_weights = F.softmax( + self.scaled_l2(x, self.codewords, self.scale), dim=2) + # aggregate + encoded_feat = self.aggregate(assignment_weights, x, self.codewords) + return encoded_feat + + def __repr__(self): + repr_str = self.__class__.__name__ + repr_str += f'(Nx{self.channels}xHxW =>Nx{self.num_codes}' \ + f'x{self.channels})' + return repr_str diff --git a/annotator/uniformer/mmseg/ops/wrappers.py b/annotator/uniformer/mmseg/ops/wrappers.py new file mode 100644 index 0000000000000000000000000000000000000000..0ed9a0cb8d7c0e0ec2748dd89c652756653cac78 --- /dev/null +++ b/annotator/uniformer/mmseg/ops/wrappers.py @@ -0,0 +1,50 @@ +import warnings + +import torch.nn as nn +import torch.nn.functional as F + + +def resize(input, + size=None, + scale_factor=None, + mode='nearest', + align_corners=None, + warning=True): + if warning: + if size is not None and align_corners: + input_h, input_w = tuple(int(x) for x in input.shape[2:]) + output_h, output_w = tuple(int(x) for x in size) + if output_h > input_h or output_w > output_h: + if ((output_h > 1 and output_w > 1 and input_h > 1 + and input_w > 1) and (output_h - 1) % (input_h - 1) + and (output_w - 1) % (input_w - 1)): + warnings.warn( + f'When align_corners={align_corners}, ' + 'the output would more aligned if ' + f'input size {(input_h, input_w)} is `x+1` and ' + f'out size {(output_h, output_w)} is `nx+1`') + return F.interpolate(input, size, scale_factor, mode, align_corners) + + +class Upsample(nn.Module): + + def __init__(self, + size=None, + scale_factor=None, + mode='nearest', + align_corners=None): + super(Upsample, self).__init__() + self.size = size + if isinstance(scale_factor, tuple): + self.scale_factor = tuple(float(factor) for factor in scale_factor) + else: + self.scale_factor = float(scale_factor) if scale_factor else None + self.mode = mode + self.align_corners = align_corners + + def forward(self, x): + if not self.size: + size = [int(t * self.scale_factor) for t in x.shape[-2:]] + else: + size = self.size + return resize(x, size, None, self.mode, self.align_corners) diff --git a/annotator/uniformer/mmseg/utils/__init__.py b/annotator/uniformer/mmseg/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ac489e2dbbc0e6fa87f5088b4edcc20f8cadc1a6 --- /dev/null +++ b/annotator/uniformer/mmseg/utils/__init__.py @@ -0,0 +1,4 @@ +from .collect_env import collect_env +from .logger import get_root_logger + +__all__ = ['get_root_logger', 'collect_env'] diff --git a/annotator/uniformer/mmseg/utils/collect_env.py b/annotator/uniformer/mmseg/utils/collect_env.py new file mode 100644 index 0000000000000000000000000000000000000000..65c2134ddbee9655161237dd0894d38c768c2624 --- /dev/null +++ b/annotator/uniformer/mmseg/utils/collect_env.py @@ -0,0 +1,17 @@ +from annotator.uniformer.mmcv.utils import collect_env as collect_base_env +from annotator.uniformer.mmcv.utils import get_git_hash + +import annotator.uniformer.mmseg as mmseg + + +def collect_env(): + """Collect the information of the running environments.""" + env_info = collect_base_env() + env_info['MMSegmentation'] = f'{mmseg.__version__}+{get_git_hash()[:7]}' + + return env_info + + +if __name__ == '__main__': + for name, val in collect_env().items(): + print('{}: {}'.format(name, val)) diff --git a/annotator/uniformer/mmseg/utils/logger.py b/annotator/uniformer/mmseg/utils/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..4149d9eda3dfef07490352d22ac40c42460315e4 --- /dev/null +++ b/annotator/uniformer/mmseg/utils/logger.py @@ -0,0 +1,27 @@ +import logging + +from annotator.uniformer.mmcv.utils import get_logger + + +def get_root_logger(log_file=None, log_level=logging.INFO): + """Get the root logger. + + The logger will be initialized if it has not been initialized. By default a + StreamHandler will be added. If `log_file` is specified, a FileHandler will + also be added. The name of the root logger is the top-level package name, + e.g., "mmseg". + + Args: + log_file (str | None): The log filename. If specified, a FileHandler + will be added to the root logger. + log_level (int): The root logger level. Note that only the process of + rank 0 is affected, while other processes will set the level to + "Error" and be silent most of the time. + + Returns: + logging.Logger: The root logger. + """ + + logger = get_logger(name='mmseg', log_file=log_file, log_level=log_level) + + return logger diff --git a/annotator/uniformer/uniformer.py b/annotator/uniformer/uniformer.py new file mode 100644 index 0000000000000000000000000000000000000000..f5726fbe63888e0d7a85563308ffd2ab526fed32 --- /dev/null +++ b/annotator/uniformer/uniformer.py @@ -0,0 +1,426 @@ +# -------------------------------------------------------- +# UniFormer +# Copyright (c) 2022 SenseTime X-Lab +# Licensed under The MIT License [see LICENSE for details] +# Written by Kunchang Li +# -------------------------------------------------------- + + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint + +from functools import partial +from collections import OrderedDict +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ + +try: + from mmseg.utils import get_root_logger + from mmseg.models.builder import BACKBONES +except ImportError: + from annotator.mmpkg.mmseg.utils import get_root_logger + from annotator.mmpkg.mmseg.models.builder import BACKBONES + +from annotator.uniformer.mmcv_custom import load_checkpoint + + +class Mlp(nn.Module): + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class CMlp(nn.Module): + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Conv2d(in_features, hidden_features, 1) + self.act = act_layer() + self.fc2 = nn.Conv2d(hidden_features, out_features, 1) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class CBlock(nn.Module): + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm): + super().__init__() + self.pos_embed = nn.Conv2d(dim, dim, 3, padding=1, groups=dim) + self.norm1 = nn.BatchNorm2d(dim) + self.conv1 = nn.Conv2d(dim, dim, 1) + self.conv2 = nn.Conv2d(dim, dim, 1) + self.attn = nn.Conv2d(dim, dim, 5, padding=2, groups=dim) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = nn.BatchNorm2d(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = CMlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + def forward(self, x): + x = x + self.pos_embed(x) + x = x + self.drop_path(self.conv2(self.attn(self.conv1(self.norm1(x))))) + x = x + self.drop_path(self.mlp(self.norm2(x))) + return x + + +class Attention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0.): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + # NOTE scale factor was wrong in my original version, can set manually to be compat with prev weights + self.scale = qk_scale or head_dim ** -0.5 + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + def forward(self, x): + B, N, C = x.shape + qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class SABlock(nn.Module): + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm): + super().__init__() + self.pos_embed = nn.Conv2d(dim, dim, 3, padding=1, groups=dim) + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, + num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, + attn_drop=attn_drop, proj_drop=drop) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + def forward(self, x): + x = x + self.pos_embed(x) + B, N, H, W = x.shape + x = x.flatten(2).transpose(1, 2) + x = x + self.drop_path(self.attn(self.norm1(x))) + x = x + self.drop_path(self.mlp(self.norm2(x))) + x = x.transpose(1, 2).reshape(B, N, H, W) + return x + + +def window_partition(x, window_size): + """ + Args: + x: (B, H, W, C) + window_size (int): window size + Returns: + windows: (num_windows*B, window_size, window_size, C) + """ + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + return windows + + +def window_reverse(windows, window_size, H, W): + """ + Args: + windows: (num_windows*B, window_size, window_size, C) + window_size (int): Window size + H (int): Height of image + W (int): Width of image + Returns: + x: (B, H, W, C) + """ + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +class SABlock_Windows(nn.Module): + def __init__(self, dim, num_heads, window_size=14, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm): + super().__init__() + self.window_size=window_size + self.pos_embed = nn.Conv2d(dim, dim, 3, padding=1, groups=dim) + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, + num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, + attn_drop=attn_drop, proj_drop=drop) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + def forward(self, x): + x = x + self.pos_embed(x) + x = x.permute(0, 2, 3, 1) + B, H, W, C = x.shape + shortcut = x + x = self.norm1(x) + + pad_l = pad_t = 0 + pad_r = (self.window_size - W % self.window_size) % self.window_size + pad_b = (self.window_size - H % self.window_size) % self.window_size + x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b)) + _, Hp, Wp, _ = x.shape + + x_windows = window_partition(x, self.window_size) # nW*B, window_size, window_size, C + x_windows = x_windows.view(-1, self.window_size * self.window_size, C) # nW*B, window_size*window_size, C + + # W-MSA/SW-MSA + attn_windows = self.attn(x_windows) # nW*B, window_size*window_size, C + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C) + x = window_reverse(attn_windows, self.window_size, Hp, Wp) # B H' W' C + + # reverse cyclic shift + if pad_r > 0 or pad_b > 0: + x = x[:, :H, :W, :].contiguous() + + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + x = x.permute(0, 3, 1, 2).reshape(B, C, H, W) + return x + + +class PatchEmbed(nn.Module): + """ Image to Patch Embedding + """ + def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + num_patches = (img_size[1] // patch_size[1]) * (img_size[0] // patch_size[0]) + self.img_size = img_size + self.patch_size = patch_size + self.num_patches = num_patches + self.norm = nn.LayerNorm(embed_dim) + self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + + def forward(self, x): + B, _, H, W = x.shape + x = self.proj(x) + B, _, H, W = x.shape + x = x.flatten(2).transpose(1, 2) + x = self.norm(x) + x = x.reshape(B, H, W, -1).permute(0, 3, 1, 2).contiguous() + return x + + +@BACKBONES.register_module() +class UniFormer(nn.Module): + """ Vision Transformer + A PyTorch impl of : `An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale` - + https://arxiv.org/abs/2010.11929 + """ + def __init__(self, layers=[3, 4, 8, 3], img_size=224, in_chans=3, num_classes=80, embed_dim=[64, 128, 320, 512], + head_dim=64, mlp_ratio=4., qkv_bias=True, qk_scale=None, representation_size=None, + drop_rate=0., attn_drop_rate=0., drop_path_rate=0., norm_layer=partial(nn.LayerNorm, eps=1e-6), + pretrained_path=None, use_checkpoint=False, checkpoint_num=[0, 0, 0, 0], + windows=False, hybrid=False, window_size=14): + """ + Args: + layer (list): number of block in each layer + img_size (int, tuple): input image size + in_chans (int): number of input channels + num_classes (int): number of classes for classification head + embed_dim (int): embedding dimension + head_dim (int): dimension of attention heads + mlp_ratio (int): ratio of mlp hidden dim to embedding dim + qkv_bias (bool): enable bias for qkv if True + qk_scale (float): override default qk scale of head_dim ** -0.5 if set + representation_size (Optional[int]): enable and set representation layer (pre-logits) to this value if set + drop_rate (float): dropout rate + attn_drop_rate (float): attention dropout rate + drop_path_rate (float): stochastic depth rate + norm_layer (nn.Module): normalization layer + pretrained_path (str): path of pretrained model + use_checkpoint (bool): whether use checkpoint + checkpoint_num (list): index for using checkpoint in every stage + windows (bool): whether use window MHRA + hybrid (bool): whether use hybrid MHRA + window_size (int): size of window (>14) + """ + super().__init__() + self.num_classes = num_classes + self.use_checkpoint = use_checkpoint + self.checkpoint_num = checkpoint_num + self.windows = windows + print(f'Use Checkpoint: {self.use_checkpoint}') + print(f'Checkpoint Number: {self.checkpoint_num}') + self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models + norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6) + + self.patch_embed1 = PatchEmbed( + img_size=img_size, patch_size=4, in_chans=in_chans, embed_dim=embed_dim[0]) + self.patch_embed2 = PatchEmbed( + img_size=img_size // 4, patch_size=2, in_chans=embed_dim[0], embed_dim=embed_dim[1]) + self.patch_embed3 = PatchEmbed( + img_size=img_size // 8, patch_size=2, in_chans=embed_dim[1], embed_dim=embed_dim[2]) + self.patch_embed4 = PatchEmbed( + img_size=img_size // 16, patch_size=2, in_chans=embed_dim[2], embed_dim=embed_dim[3]) + + self.pos_drop = nn.Dropout(p=drop_rate) + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(layers))] # stochastic depth decay rule + num_heads = [dim // head_dim for dim in embed_dim] + self.blocks1 = nn.ModuleList([ + CBlock( + dim=embed_dim[0], num_heads=num_heads[0], mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i], norm_layer=norm_layer) + for i in range(layers[0])]) + self.norm1=norm_layer(embed_dim[0]) + self.blocks2 = nn.ModuleList([ + CBlock( + dim=embed_dim[1], num_heads=num_heads[1], mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i+layers[0]], norm_layer=norm_layer) + for i in range(layers[1])]) + self.norm2 = norm_layer(embed_dim[1]) + if self.windows: + print('Use local window for all blocks in stage3') + self.blocks3 = nn.ModuleList([ + SABlock_Windows( + dim=embed_dim[2], num_heads=num_heads[2], window_size=window_size, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i+layers[0]+layers[1]], norm_layer=norm_layer) + for i in range(layers[2])]) + elif hybrid: + print('Use hybrid window for blocks in stage3') + block3 = [] + for i in range(layers[2]): + if (i + 1) % 4 == 0: + block3.append(SABlock( + dim=embed_dim[2], num_heads=num_heads[2], mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i+layers[0]+layers[1]], norm_layer=norm_layer)) + else: + block3.append(SABlock_Windows( + dim=embed_dim[2], num_heads=num_heads[2], window_size=window_size, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i+layers[0]+layers[1]], norm_layer=norm_layer)) + self.blocks3 = nn.ModuleList(block3) + else: + print('Use global window for all blocks in stage3') + self.blocks3 = nn.ModuleList([ + SABlock( + dim=embed_dim[2], num_heads=num_heads[2], mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i+layers[0]+layers[1]], norm_layer=norm_layer) + for i in range(layers[2])]) + self.norm3 = norm_layer(embed_dim[2]) + self.blocks4 = nn.ModuleList([ + SABlock( + dim=embed_dim[3], num_heads=num_heads[3], mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[i+layers[0]+layers[1]+layers[2]], norm_layer=norm_layer) + for i in range(layers[3])]) + self.norm4 = norm_layer(embed_dim[3]) + + # Representation layer + if representation_size: + self.num_features = representation_size + self.pre_logits = nn.Sequential(OrderedDict([ + ('fc', nn.Linear(embed_dim, representation_size)), + ('act', nn.Tanh()) + ])) + else: + self.pre_logits = nn.Identity() + + self.apply(self._init_weights) + self.init_weights(pretrained=pretrained_path) + + def init_weights(self, pretrained): + if isinstance(pretrained, str): + logger = get_root_logger() + load_checkpoint(self, pretrained, map_location='cpu', strict=False, logger=logger) + print(f'Load pretrained model from {pretrained}') + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + @torch.jit.ignore + def no_weight_decay(self): + return {'pos_embed', 'cls_token'} + + def get_classifier(self): + return self.head + + def reset_classifier(self, num_classes, global_pool=''): + self.num_classes = num_classes + self.head = nn.Linear(self.embed_dim, num_classes) if num_classes > 0 else nn.Identity() + + def forward_features(self, x): + out = [] + x = self.patch_embed1(x) + x = self.pos_drop(x) + for i, blk in enumerate(self.blocks1): + if self.use_checkpoint and i < self.checkpoint_num[0]: + x = checkpoint.checkpoint(blk, x) + else: + x = blk(x) + x_out = self.norm1(x.permute(0, 2, 3, 1)) + out.append(x_out.permute(0, 3, 1, 2).contiguous()) + x = self.patch_embed2(x) + for i, blk in enumerate(self.blocks2): + if self.use_checkpoint and i < self.checkpoint_num[1]: + x = checkpoint.checkpoint(blk, x) + else: + x = blk(x) + x_out = self.norm2(x.permute(0, 2, 3, 1)) + out.append(x_out.permute(0, 3, 1, 2).contiguous()) + x = self.patch_embed3(x) + for i, blk in enumerate(self.blocks3): + if self.use_checkpoint and i < self.checkpoint_num[2]: + x = checkpoint.checkpoint(blk, x) + else: + x = blk(x) + x_out = self.norm3(x.permute(0, 2, 3, 1)) + out.append(x_out.permute(0, 3, 1, 2).contiguous()) + x = self.patch_embed4(x) + for i, blk in enumerate(self.blocks4): + if self.use_checkpoint and i < self.checkpoint_num[3]: + x = checkpoint.checkpoint(blk, x) + else: + x = blk(x) + x_out = self.norm4(x.permute(0, 2, 3, 1)) + out.append(x_out.permute(0, 3, 1, 2).contiguous()) + return tuple(out) + + def forward(self, x): + x = self.forward_features(x) + return x diff --git a/annotator/uniformer/upernet_global_small.py b/annotator/uniformer/upernet_global_small.py new file mode 100644 index 0000000000000000000000000000000000000000..16b14768b80035b52a9a975af67c23c1c7693265 --- /dev/null +++ b/annotator/uniformer/upernet_global_small.py @@ -0,0 +1,44 @@ +_base_ = [ + 'configs/_base_/models/upernet_uniformer.py', + 'configs/_base_/datasets/ade20k.py', + 'configs/_base_/default_runtime.py', + 'configs/_base_/schedules/schedule_160k.py' +] + +custom_imports = dict( + imports=['annotator.uniformer.uniformer'], + allow_failed_imports=False +) + +model = dict( + backbone=dict( + type='UniFormer', + embed_dim=[64, 128, 320, 512], + layers=[3, 4, 8, 3], + head_dim=64, + drop_path_rate=0.25, + windows=False, + hybrid=False + ), + decode_head=dict( + in_channels=[64, 128, 320, 512], + num_classes=150 + ), + auxiliary_head=dict( + in_channels=320, + num_classes=150 + )) + +# AdamW optimizer, no weight decay for position embedding & layer norm in backbone +optimizer = dict(_delete_=True, type='AdamW', lr=0.00006, betas=(0.9, 0.999), weight_decay=0.01, + paramwise_cfg=dict(custom_keys={'absolute_pos_embed': dict(decay_mult=0.), + 'relative_position_bias_table': dict(decay_mult=0.), + 'norm': dict(decay_mult=0.)})) + +lr_config = dict(_delete_=True, policy='poly', + warmup='linear', + warmup_iters=1500, + warmup_ratio=1e-6, + power=1.0, min_lr=0.0, by_epoch=False) + +data=dict(samples_per_gpu=2) \ No newline at end of file diff --git a/annotator/util.py b/annotator/util.py new file mode 100644 index 0000000000000000000000000000000000000000..e0b217ef9adf92dd5b1fe0debcfb07d0f241a4cb --- /dev/null +++ b/annotator/util.py @@ -0,0 +1,98 @@ +import random + +import numpy as np +import cv2 +import os + + +annotator_ckpts_path = os.path.join(os.path.dirname(__file__), 'ckpts') + + +def HWC3(x): + assert x.dtype == np.uint8 + if x.ndim == 2: + x = x[:, :, None] + assert x.ndim == 3 + H, W, C = x.shape + assert C == 1 or C == 3 or C == 4 + if C == 3: + return x + if C == 1: + return np.concatenate([x, x, x], axis=2) + if C == 4: + color = x[:, :, 0:3].astype(np.float32) + alpha = x[:, :, 3:4].astype(np.float32) / 255.0 + y = color * alpha + 255.0 * (1.0 - alpha) + y = y.clip(0, 255).astype(np.uint8) + return y + + +def resize_image(input_image, resolution): + H, W, C = input_image.shape + H = float(H) + W = float(W) + k = float(resolution) / min(H, W) + H *= k + W *= k + H = int(np.round(H / 64.0)) * 64 + W = int(np.round(W / 64.0)) * 64 + img = cv2.resize(input_image, (W, H), interpolation=cv2.INTER_LANCZOS4 if k > 1 else cv2.INTER_AREA) + return img + + +def nms(x, t, s): + x = cv2.GaussianBlur(x.astype(np.float32), (0, 0), s) + + f1 = np.array([[0, 0, 0], [1, 1, 1], [0, 0, 0]], dtype=np.uint8) + f2 = np.array([[0, 1, 0], [0, 1, 0], [0, 1, 0]], dtype=np.uint8) + f3 = np.array([[1, 0, 0], [0, 1, 0], [0, 0, 1]], dtype=np.uint8) + f4 = np.array([[0, 0, 1], [0, 1, 0], [1, 0, 0]], dtype=np.uint8) + + y = np.zeros_like(x) + + for f in [f1, f2, f3, f4]: + np.putmask(y, cv2.dilate(x, kernel=f) == x, x) + + z = np.zeros_like(y, dtype=np.uint8) + z[y > t] = 255 + return z + + +def make_noise_disk(H, W, C, F): + noise = np.random.uniform(low=0, high=1, size=((H // F) + 2, (W // F) + 2, C)) + noise = cv2.resize(noise, (W + 2 * F, H + 2 * F), interpolation=cv2.INTER_CUBIC) + noise = noise[F: F + H, F: F + W] + noise -= np.min(noise) + noise /= np.max(noise) + if C == 1: + noise = noise[:, :, None] + return noise + + +def min_max_norm(x): + x -= np.min(x) + x /= np.maximum(np.max(x), 1e-5) + return x + + +def safe_step(x, step=2): + y = x.astype(np.float32) * float(step + 1) + y = y.astype(np.int32).astype(np.float32) / float(step) + return y + + +def img2mask(img, H, W, low=10, high=90): + assert img.ndim == 3 or img.ndim == 2 + assert img.dtype == np.uint8 + + if img.ndim == 3: + y = img[:, :, random.randrange(0, img.shape[2])] + else: + y = img + + y = cv2.resize(y, (W, H), interpolation=cv2.INTER_CUBIC) + + if random.uniform(0, 1) < 0.5: + y = 255 - y + + return y < np.percentile(y, random.randrange(low, high)) diff --git a/annotator/zoe/LICENSE b/annotator/zoe/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..7a1e90d007836c327846ce8e5151013b115042ab --- /dev/null +++ b/annotator/zoe/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Intelligent Systems Lab Org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/annotator/zoe/__init__.py b/annotator/zoe/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..2c2e699fb2e2f86833a0baa61f0aed8369850277 --- /dev/null +++ b/annotator/zoe/__init__.py @@ -0,0 +1,52 @@ +# ZoeDepth +# https://github.com/isl-org/ZoeDepth + +import os +import cv2 +import numpy as np +import torch + +from einops import rearrange +from .zoedepth.models.zoedepth.zoedepth_v1 import ZoeDepth +from .zoedepth.utils.config import get_config +from annotator.util import annotator_ckpts_path + + +class ZoeDetector: + def __init__(self): + remote_model_path = "https://huggingface.co./lllyasviel/Annotators/resolve/main/ZoeD_M12_N.pt" + modelpath = os.path.join(annotator_ckpts_path, "ZoeD_M12_N.pt") + if not os.path.exists(modelpath): + from basicsr.utils.download_util import load_file_from_url + load_file_from_url(remote_model_path, model_dir=annotator_ckpts_path) + conf = get_config("zoedepth", "infer") + model = ZoeDepth.build_from_config(conf) + model.load_state_dict(torch.load(modelpath, map_location=torch.device('cpu'))['model']) +# model = model.cuda() +# model.device = 'cuda' + model = model.cpu() + model.device = 'cpu' + model.eval() + self.model = model + + def __call__(self, input_image): + assert input_image.ndim == 3 + image_depth = input_image + with torch.no_grad(): +# image_depth = torch.from_numpy(image_depth).float().cuda() + image_depth = torch.from_numpy(image_depth).float().cpu() + image_depth = image_depth / 255.0 + image_depth = rearrange(image_depth, 'h w c -> 1 c h w') + depth = self.model.infer(image_depth) + + depth = depth[0, 0].cpu().numpy() + + vmin = np.percentile(depth, 2) + vmax = np.percentile(depth, 85) + + depth -= vmin + depth /= vmax - vmin + depth = 1.0 - depth + depth_image = (depth * 255.0).clip(0, 255).astype(np.uint8) + + return depth_image diff --git a/annotator/zoe/zoedepth/data/__init__.py b/annotator/zoe/zoedepth/data/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5f2668792389157609abb2a0846fb620e7d67eb9 --- /dev/null +++ b/annotator/zoe/zoedepth/data/__init__.py @@ -0,0 +1,24 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + diff --git a/annotator/zoe/zoedepth/data/data_mono.py b/annotator/zoe/zoedepth/data/data_mono.py new file mode 100644 index 0000000000000000000000000000000000000000..80a8486f239a35331df553f490e213f9bf71e735 --- /dev/null +++ b/annotator/zoe/zoedepth/data/data_mono.py @@ -0,0 +1,573 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +# This file is partly inspired from BTS (https://github.com/cleinc/bts/blob/master/pytorch/bts_dataloader.py); author: Jin Han Lee + +import itertools +import os +import random + +import numpy as np +import cv2 +import torch +import torch.nn as nn +import torch.utils.data.distributed +from zoedepth.utils.easydict import EasyDict as edict +from PIL import Image, ImageOps +from torch.utils.data import DataLoader, Dataset +from torchvision import transforms + +from zoedepth.utils.config import change_dataset + +from .ddad import get_ddad_loader +from .diml_indoor_test import get_diml_indoor_loader +from .diml_outdoor_test import get_diml_outdoor_loader +from .diode import get_diode_loader +from .hypersim import get_hypersim_loader +from .ibims import get_ibims_loader +from .sun_rgbd_loader import get_sunrgbd_loader +from .vkitti import get_vkitti_loader +from .vkitti2 import get_vkitti2_loader + +from .preprocess import CropParams, get_white_border, get_black_border + + +def _is_pil_image(img): + return isinstance(img, Image.Image) + + +def _is_numpy_image(img): + return isinstance(img, np.ndarray) and (img.ndim in {2, 3}) + + +def preprocessing_transforms(mode, **kwargs): + return transforms.Compose([ + ToTensor(mode=mode, **kwargs) + ]) + + +class DepthDataLoader(object): + def __init__(self, config, mode, device='cpu', transform=None, **kwargs): + """ + Data loader for depth datasets + + Args: + config (dict): Config dictionary. Refer to utils/config.py + mode (str): "train" or "online_eval" + device (str, optional): Device to load the data on. Defaults to 'cpu'. + transform (torchvision.transforms, optional): Transform to apply to the data. Defaults to None. + """ + + self.config = config + + if config.dataset == 'ibims': + self.data = get_ibims_loader(config, batch_size=1, num_workers=1) + return + + if config.dataset == 'sunrgbd': + self.data = get_sunrgbd_loader( + data_dir_root=config.sunrgbd_root, batch_size=1, num_workers=1) + return + + if config.dataset == 'diml_indoor': + self.data = get_diml_indoor_loader( + data_dir_root=config.diml_indoor_root, batch_size=1, num_workers=1) + return + + if config.dataset == 'diml_outdoor': + self.data = get_diml_outdoor_loader( + data_dir_root=config.diml_outdoor_root, batch_size=1, num_workers=1) + return + + if "diode" in config.dataset: + self.data = get_diode_loader( + config[config.dataset+"_root"], batch_size=1, num_workers=1) + return + + if config.dataset == 'hypersim_test': + self.data = get_hypersim_loader( + config.hypersim_test_root, batch_size=1, num_workers=1) + return + + if config.dataset == 'vkitti': + self.data = get_vkitti_loader( + config.vkitti_root, batch_size=1, num_workers=1) + return + + if config.dataset == 'vkitti2': + self.data = get_vkitti2_loader( + config.vkitti2_root, batch_size=1, num_workers=1) + return + + if config.dataset == 'ddad': + self.data = get_ddad_loader(config.ddad_root, resize_shape=( + 352, 1216), batch_size=1, num_workers=1) + return + + img_size = self.config.get("img_size", None) + img_size = img_size if self.config.get( + "do_input_resize", False) else None + + if transform is None: + transform = preprocessing_transforms(mode, size=img_size) + + if mode == 'train': + + Dataset = DataLoadPreprocess + self.training_samples = Dataset( + config, mode, transform=transform, device=device) + + if config.distributed: + self.train_sampler = torch.utils.data.distributed.DistributedSampler( + self.training_samples) + else: + self.train_sampler = None + + self.data = DataLoader(self.training_samples, + batch_size=config.batch_size, + shuffle=(self.train_sampler is None), + num_workers=config.workers, + pin_memory=True, + persistent_workers=True, + # prefetch_factor=2, + sampler=self.train_sampler) + + elif mode == 'online_eval': + self.testing_samples = DataLoadPreprocess( + config, mode, transform=transform) + if config.distributed: # redundant. here only for readability and to be more explicit + # Give whole test set to all processes (and report evaluation only on one) regardless + self.eval_sampler = None + else: + self.eval_sampler = None + self.data = DataLoader(self.testing_samples, 1, + shuffle=kwargs.get("shuffle_test", False), + num_workers=1, + pin_memory=False, + sampler=self.eval_sampler) + + elif mode == 'test': + self.testing_samples = DataLoadPreprocess( + config, mode, transform=transform) + self.data = DataLoader(self.testing_samples, + 1, shuffle=False, num_workers=1) + + else: + print( + 'mode should be one of \'train, test, online_eval\'. Got {}'.format(mode)) + + +def repetitive_roundrobin(*iterables): + """ + cycles through iterables but sample wise + first yield first sample from first iterable then first sample from second iterable and so on + then second sample from first iterable then second sample from second iterable and so on + + If one iterable is shorter than the others, it is repeated until all iterables are exhausted + repetitive_roundrobin('ABC', 'D', 'EF') --> A D E B D F C D E + """ + # Repetitive roundrobin + iterables_ = [iter(it) for it in iterables] + exhausted = [False] * len(iterables) + while not all(exhausted): + for i, it in enumerate(iterables_): + try: + yield next(it) + except StopIteration: + exhausted[i] = True + iterables_[i] = itertools.cycle(iterables[i]) + # First elements may get repeated if one iterable is shorter than the others + yield next(iterables_[i]) + + +class RepetitiveRoundRobinDataLoader(object): + def __init__(self, *dataloaders): + self.dataloaders = dataloaders + + def __iter__(self): + return repetitive_roundrobin(*self.dataloaders) + + def __len__(self): + # First samples get repeated, thats why the plus one + return len(self.dataloaders) * (max(len(dl) for dl in self.dataloaders) + 1) + + +class MixedNYUKITTI(object): + def __init__(self, config, mode, device='cpu', **kwargs): + config = edict(config) + config.workers = config.workers // 2 + self.config = config + nyu_conf = change_dataset(edict(config), 'nyu') + kitti_conf = change_dataset(edict(config), 'kitti') + + # make nyu default for testing + self.config = config = nyu_conf + img_size = self.config.get("img_size", None) + img_size = img_size if self.config.get( + "do_input_resize", False) else None + if mode == 'train': + nyu_loader = DepthDataLoader( + nyu_conf, mode, device=device, transform=preprocessing_transforms(mode, size=img_size)).data + kitti_loader = DepthDataLoader( + kitti_conf, mode, device=device, transform=preprocessing_transforms(mode, size=img_size)).data + # It has been changed to repetitive roundrobin + self.data = RepetitiveRoundRobinDataLoader( + nyu_loader, kitti_loader) + else: + self.data = DepthDataLoader(nyu_conf, mode, device=device).data + + +def remove_leading_slash(s): + if s[0] == '/' or s[0] == '\\': + return s[1:] + return s + + +class CachedReader: + def __init__(self, shared_dict=None): + if shared_dict: + self._cache = shared_dict + else: + self._cache = {} + + def open(self, fpath): + im = self._cache.get(fpath, None) + if im is None: + im = self._cache[fpath] = Image.open(fpath) + return im + + +class ImReader: + def __init__(self): + pass + + # @cache + def open(self, fpath): + return Image.open(fpath) + + +class DataLoadPreprocess(Dataset): + def __init__(self, config, mode, transform=None, is_for_online_eval=False, **kwargs): + self.config = config + if mode == 'online_eval': + with open(config.filenames_file_eval, 'r') as f: + self.filenames = f.readlines() + else: + with open(config.filenames_file, 'r') as f: + self.filenames = f.readlines() + + self.mode = mode + self.transform = transform + self.to_tensor = ToTensor(mode) + self.is_for_online_eval = is_for_online_eval + if config.use_shared_dict: + self.reader = CachedReader(config.shared_dict) + else: + self.reader = ImReader() + + def postprocess(self, sample): + return sample + + def __getitem__(self, idx): + sample_path = self.filenames[idx] + focal = float(sample_path.split()[2]) + sample = {} + + if self.mode == 'train': + if self.config.dataset == 'kitti' and self.config.use_right and random.random() > 0.5: + image_path = os.path.join( + self.config.data_path, remove_leading_slash(sample_path.split()[3])) + depth_path = os.path.join( + self.config.gt_path, remove_leading_slash(sample_path.split()[4])) + else: + image_path = os.path.join( + self.config.data_path, remove_leading_slash(sample_path.split()[0])) + depth_path = os.path.join( + self.config.gt_path, remove_leading_slash(sample_path.split()[1])) + + image = self.reader.open(image_path) + depth_gt = self.reader.open(depth_path) + w, h = image.size + + if self.config.do_kb_crop: + height = image.height + width = image.width + top_margin = int(height - 352) + left_margin = int((width - 1216) / 2) + depth_gt = depth_gt.crop( + (left_margin, top_margin, left_margin + 1216, top_margin + 352)) + image = image.crop( + (left_margin, top_margin, left_margin + 1216, top_margin + 352)) + + # Avoid blank boundaries due to pixel registration? + # Train images have white border. Test images have black border. + if self.config.dataset == 'nyu' and self.config.avoid_boundary: + # print("Avoiding Blank Boundaries!") + # We just crop and pad again with reflect padding to original size + # original_size = image.size + crop_params = get_white_border(np.array(image, dtype=np.uint8)) + image = image.crop((crop_params.left, crop_params.top, crop_params.right, crop_params.bottom)) + depth_gt = depth_gt.crop((crop_params.left, crop_params.top, crop_params.right, crop_params.bottom)) + + # Use reflect padding to fill the blank + image = np.array(image) + image = np.pad(image, ((crop_params.top, h - crop_params.bottom), (crop_params.left, w - crop_params.right), (0, 0)), mode='reflect') + image = Image.fromarray(image) + + depth_gt = np.array(depth_gt) + depth_gt = np.pad(depth_gt, ((crop_params.top, h - crop_params.bottom), (crop_params.left, w - crop_params.right)), 'constant', constant_values=0) + depth_gt = Image.fromarray(depth_gt) + + + if self.config.do_random_rotate and (self.config.aug): + random_angle = (random.random() - 0.5) * 2 * self.config.degree + image = self.rotate_image(image, random_angle) + depth_gt = self.rotate_image( + depth_gt, random_angle, flag=Image.NEAREST) + + image = np.asarray(image, dtype=np.float32) / 255.0 + depth_gt = np.asarray(depth_gt, dtype=np.float32) + depth_gt = np.expand_dims(depth_gt, axis=2) + + if self.config.dataset == 'nyu': + depth_gt = depth_gt / 1000.0 + else: + depth_gt = depth_gt / 256.0 + + if self.config.aug and (self.config.random_crop): + image, depth_gt = self.random_crop( + image, depth_gt, self.config.input_height, self.config.input_width) + + if self.config.aug and self.config.random_translate: + # print("Random Translation!") + image, depth_gt = self.random_translate(image, depth_gt, self.config.max_translation) + + image, depth_gt = self.train_preprocess(image, depth_gt) + mask = np.logical_and(depth_gt > self.config.min_depth, + depth_gt < self.config.max_depth).squeeze()[None, ...] + sample = {'image': image, 'depth': depth_gt, 'focal': focal, + 'mask': mask, **sample} + + else: + if self.mode == 'online_eval': + data_path = self.config.data_path_eval + else: + data_path = self.config.data_path + + image_path = os.path.join( + data_path, remove_leading_slash(sample_path.split()[0])) + image = np.asarray(self.reader.open(image_path), + dtype=np.float32) / 255.0 + + if self.mode == 'online_eval': + gt_path = self.config.gt_path_eval + depth_path = os.path.join( + gt_path, remove_leading_slash(sample_path.split()[1])) + has_valid_depth = False + try: + depth_gt = self.reader.open(depth_path) + has_valid_depth = True + except IOError: + depth_gt = False + # print('Missing gt for {}'.format(image_path)) + + if has_valid_depth: + depth_gt = np.asarray(depth_gt, dtype=np.float32) + depth_gt = np.expand_dims(depth_gt, axis=2) + if self.config.dataset == 'nyu': + depth_gt = depth_gt / 1000.0 + else: + depth_gt = depth_gt / 256.0 + + mask = np.logical_and( + depth_gt >= self.config.min_depth, depth_gt <= self.config.max_depth).squeeze()[None, ...] + else: + mask = False + + if self.config.do_kb_crop: + height = image.shape[0] + width = image.shape[1] + top_margin = int(height - 352) + left_margin = int((width - 1216) / 2) + image = image[top_margin:top_margin + 352, + left_margin:left_margin + 1216, :] + if self.mode == 'online_eval' and has_valid_depth: + depth_gt = depth_gt[top_margin:top_margin + + 352, left_margin:left_margin + 1216, :] + + if self.mode == 'online_eval': + sample = {'image': image, 'depth': depth_gt, 'focal': focal, 'has_valid_depth': has_valid_depth, + 'image_path': sample_path.split()[0], 'depth_path': sample_path.split()[1], + 'mask': mask} + else: + sample = {'image': image, 'focal': focal} + + if (self.mode == 'train') or ('has_valid_depth' in sample and sample['has_valid_depth']): + mask = np.logical_and(depth_gt > self.config.min_depth, + depth_gt < self.config.max_depth).squeeze()[None, ...] + sample['mask'] = mask + + if self.transform: + sample = self.transform(sample) + + sample = self.postprocess(sample) + sample['dataset'] = self.config.dataset + sample = {**sample, 'image_path': sample_path.split()[0], 'depth_path': sample_path.split()[1]} + + return sample + + def rotate_image(self, image, angle, flag=Image.BILINEAR): + result = image.rotate(angle, resample=flag) + return result + + def random_crop(self, img, depth, height, width): + assert img.shape[0] >= height + assert img.shape[1] >= width + assert img.shape[0] == depth.shape[0] + assert img.shape[1] == depth.shape[1] + x = random.randint(0, img.shape[1] - width) + y = random.randint(0, img.shape[0] - height) + img = img[y:y + height, x:x + width, :] + depth = depth[y:y + height, x:x + width, :] + + return img, depth + + def random_translate(self, img, depth, max_t=20): + assert img.shape[0] == depth.shape[0] + assert img.shape[1] == depth.shape[1] + p = self.config.translate_prob + do_translate = random.random() + if do_translate > p: + return img, depth + x = random.randint(-max_t, max_t) + y = random.randint(-max_t, max_t) + M = np.float32([[1, 0, x], [0, 1, y]]) + # print(img.shape, depth.shape) + img = cv2.warpAffine(img, M, (img.shape[1], img.shape[0])) + depth = cv2.warpAffine(depth, M, (depth.shape[1], depth.shape[0])) + depth = depth.squeeze()[..., None] # add channel dim back. Affine warp removes it + # print("after", img.shape, depth.shape) + return img, depth + + def train_preprocess(self, image, depth_gt): + if self.config.aug: + # Random flipping + do_flip = random.random() + if do_flip > 0.5: + image = (image[:, ::-1, :]).copy() + depth_gt = (depth_gt[:, ::-1, :]).copy() + + # Random gamma, brightness, color augmentation + do_augment = random.random() + if do_augment > 0.5: + image = self.augment_image(image) + + return image, depth_gt + + def augment_image(self, image): + # gamma augmentation + gamma = random.uniform(0.9, 1.1) + image_aug = image ** gamma + + # brightness augmentation + if self.config.dataset == 'nyu': + brightness = random.uniform(0.75, 1.25) + else: + brightness = random.uniform(0.9, 1.1) + image_aug = image_aug * brightness + + # color augmentation + colors = np.random.uniform(0.9, 1.1, size=3) + white = np.ones((image.shape[0], image.shape[1])) + color_image = np.stack([white * colors[i] for i in range(3)], axis=2) + image_aug *= color_image + image_aug = np.clip(image_aug, 0, 1) + + return image_aug + + def __len__(self): + return len(self.filenames) + + +class ToTensor(object): + def __init__(self, mode, do_normalize=False, size=None): + self.mode = mode + self.normalize = transforms.Normalize( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) if do_normalize else nn.Identity() + self.size = size + if size is not None: + self.resize = transforms.Resize(size=size) + else: + self.resize = nn.Identity() + + def __call__(self, sample): + image, focal = sample['image'], sample['focal'] + image = self.to_tensor(image) + image = self.normalize(image) + image = self.resize(image) + + if self.mode == 'test': + return {'image': image, 'focal': focal} + + depth = sample['depth'] + if self.mode == 'train': + depth = self.to_tensor(depth) + return {**sample, 'image': image, 'depth': depth, 'focal': focal} + else: + has_valid_depth = sample['has_valid_depth'] + image = self.resize(image) + return {**sample, 'image': image, 'depth': depth, 'focal': focal, 'has_valid_depth': has_valid_depth, + 'image_path': sample['image_path'], 'depth_path': sample['depth_path']} + + def to_tensor(self, pic): + if not (_is_pil_image(pic) or _is_numpy_image(pic)): + raise TypeError( + 'pic should be PIL Image or ndarray. Got {}'.format(type(pic))) + + if isinstance(pic, np.ndarray): + img = torch.from_numpy(pic.transpose((2, 0, 1))) + return img + + # handle PIL Image + if pic.mode == 'I': + img = torch.from_numpy(np.array(pic, np.int32, copy=False)) + elif pic.mode == 'I;16': + img = torch.from_numpy(np.array(pic, np.int16, copy=False)) + else: + img = torch.ByteTensor( + torch.ByteStorage.from_buffer(pic.tobytes())) + # PIL image mode: 1, L, P, I, F, RGB, YCbCr, RGBA, CMYK + if pic.mode == 'YCbCr': + nchannel = 3 + elif pic.mode == 'I;16': + nchannel = 1 + else: + nchannel = len(pic.mode) + img = img.view(pic.size[1], pic.size[0], nchannel) + + img = img.transpose(0, 1).transpose(0, 2).contiguous() + if isinstance(img, torch.ByteTensor): + return img.float() + else: + return img diff --git a/annotator/zoe/zoedepth/data/ddad.py b/annotator/zoe/zoedepth/data/ddad.py new file mode 100644 index 0000000000000000000000000000000000000000..4bd0492bdec767685d3a21992b4a26e62d002d97 --- /dev/null +++ b/annotator/zoe/zoedepth/data/ddad.py @@ -0,0 +1,117 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import os + +import numpy as np +import torch +from PIL import Image +from torch.utils.data import DataLoader, Dataset +from torchvision import transforms + + +class ToTensor(object): + def __init__(self, resize_shape): + # self.normalize = transforms.Normalize( + # mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + self.normalize = lambda x : x + self.resize = transforms.Resize(resize_shape) + + def __call__(self, sample): + image, depth = sample['image'], sample['depth'] + image = self.to_tensor(image) + image = self.normalize(image) + depth = self.to_tensor(depth) + + image = self.resize(image) + + return {'image': image, 'depth': depth, 'dataset': "ddad"} + + def to_tensor(self, pic): + + if isinstance(pic, np.ndarray): + img = torch.from_numpy(pic.transpose((2, 0, 1))) + return img + + # # handle PIL Image + if pic.mode == 'I': + img = torch.from_numpy(np.array(pic, np.int32, copy=False)) + elif pic.mode == 'I;16': + img = torch.from_numpy(np.array(pic, np.int16, copy=False)) + else: + img = torch.ByteTensor( + torch.ByteStorage.from_buffer(pic.tobytes())) + # PIL image mode: 1, L, P, I, F, RGB, YCbCr, RGBA, CMYK + if pic.mode == 'YCbCr': + nchannel = 3 + elif pic.mode == 'I;16': + nchannel = 1 + else: + nchannel = len(pic.mode) + img = img.view(pic.size[1], pic.size[0], nchannel) + + img = img.transpose(0, 1).transpose(0, 2).contiguous() + + if isinstance(img, torch.ByteTensor): + return img.float() + else: + return img + + +class DDAD(Dataset): + def __init__(self, data_dir_root, resize_shape): + import glob + + # image paths are of the form /{outleft, depthmap}/*.png + self.image_files = glob.glob(os.path.join(data_dir_root, '*.png')) + self.depth_files = [r.replace("_rgb.png", "_depth.npy") + for r in self.image_files] + self.transform = ToTensor(resize_shape) + + def __getitem__(self, idx): + + image_path = self.image_files[idx] + depth_path = self.depth_files[idx] + + image = np.asarray(Image.open(image_path), dtype=np.float32) / 255.0 + depth = np.load(depth_path) # meters + + # depth[depth > 8] = -1 + depth = depth[..., None] + + sample = dict(image=image, depth=depth) + sample = self.transform(sample) + + if idx == 0: + print(sample["image"].shape) + + return sample + + def __len__(self): + return len(self.image_files) + + +def get_ddad_loader(data_dir_root, resize_shape, batch_size=1, **kwargs): + dataset = DDAD(data_dir_root, resize_shape) + return DataLoader(dataset, batch_size, **kwargs) diff --git a/annotator/zoe/zoedepth/data/diml_indoor_test.py b/annotator/zoe/zoedepth/data/diml_indoor_test.py new file mode 100644 index 0000000000000000000000000000000000000000..f720ad9aefaee78ef4ec363dfef0f82ace850a6d --- /dev/null +++ b/annotator/zoe/zoedepth/data/diml_indoor_test.py @@ -0,0 +1,125 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import os + +import numpy as np +import torch +from PIL import Image +from torch.utils.data import DataLoader, Dataset +from torchvision import transforms + + +class ToTensor(object): + def __init__(self): + # self.normalize = transforms.Normalize( + # mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + self.normalize = lambda x : x + self.resize = transforms.Resize((480, 640)) + + def __call__(self, sample): + image, depth = sample['image'], sample['depth'] + image = self.to_tensor(image) + image = self.normalize(image) + depth = self.to_tensor(depth) + + image = self.resize(image) + + return {'image': image, 'depth': depth, 'dataset': "diml_indoor"} + + def to_tensor(self, pic): + + if isinstance(pic, np.ndarray): + img = torch.from_numpy(pic.transpose((2, 0, 1))) + return img + + # # handle PIL Image + if pic.mode == 'I': + img = torch.from_numpy(np.array(pic, np.int32, copy=False)) + elif pic.mode == 'I;16': + img = torch.from_numpy(np.array(pic, np.int16, copy=False)) + else: + img = torch.ByteTensor( + torch.ByteStorage.from_buffer(pic.tobytes())) + # PIL image mode: 1, L, P, I, F, RGB, YCbCr, RGBA, CMYK + if pic.mode == 'YCbCr': + nchannel = 3 + elif pic.mode == 'I;16': + nchannel = 1 + else: + nchannel = len(pic.mode) + img = img.view(pic.size[1], pic.size[0], nchannel) + + img = img.transpose(0, 1).transpose(0, 2).contiguous() + if isinstance(img, torch.ByteTensor): + return img.float() + else: + return img + + +class DIML_Indoor(Dataset): + def __init__(self, data_dir_root): + import glob + + # image paths are of the form /{HR, LR}//{color, depth_filled}/*.png + self.image_files = glob.glob(os.path.join( + data_dir_root, "LR", '*', 'color', '*.png')) + self.depth_files = [r.replace("color", "depth_filled").replace( + "_c.png", "_depth_filled.png") for r in self.image_files] + self.transform = ToTensor() + + def __getitem__(self, idx): + image_path = self.image_files[idx] + depth_path = self.depth_files[idx] + + image = np.asarray(Image.open(image_path), dtype=np.float32) / 255.0 + depth = np.asarray(Image.open(depth_path), + dtype='uint16') / 1000.0 # mm to meters + + # print(np.shape(image)) + # print(np.shape(depth)) + + # depth[depth > 8] = -1 + depth = depth[..., None] + + sample = dict(image=image, depth=depth) + + # return sample + sample = self.transform(sample) + + if idx == 0: + print(sample["image"].shape) + + return sample + + def __len__(self): + return len(self.image_files) + + +def get_diml_indoor_loader(data_dir_root, batch_size=1, **kwargs): + dataset = DIML_Indoor(data_dir_root) + return DataLoader(dataset, batch_size, **kwargs) + +# get_diml_indoor_loader(data_dir_root="datasets/diml/indoor/test/HR") +# get_diml_indoor_loader(data_dir_root="datasets/diml/indoor/test/LR") diff --git a/annotator/zoe/zoedepth/data/diml_outdoor_test.py b/annotator/zoe/zoedepth/data/diml_outdoor_test.py new file mode 100644 index 0000000000000000000000000000000000000000..8670b48f5febafb819dac22848ad79ccb5dd5ae4 --- /dev/null +++ b/annotator/zoe/zoedepth/data/diml_outdoor_test.py @@ -0,0 +1,114 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import os + +import numpy as np +import torch +from PIL import Image +from torch.utils.data import DataLoader, Dataset +from torchvision import transforms + + +class ToTensor(object): + def __init__(self): + # self.normalize = transforms.Normalize( + # mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + self.normalize = lambda x : x + + def __call__(self, sample): + image, depth = sample['image'], sample['depth'] + image = self.to_tensor(image) + image = self.normalize(image) + depth = self.to_tensor(depth) + + return {'image': image, 'depth': depth, 'dataset': "diml_outdoor"} + + def to_tensor(self, pic): + + if isinstance(pic, np.ndarray): + img = torch.from_numpy(pic.transpose((2, 0, 1))) + return img + + # # handle PIL Image + if pic.mode == 'I': + img = torch.from_numpy(np.array(pic, np.int32, copy=False)) + elif pic.mode == 'I;16': + img = torch.from_numpy(np.array(pic, np.int16, copy=False)) + else: + img = torch.ByteTensor( + torch.ByteStorage.from_buffer(pic.tobytes())) + # PIL image mode: 1, L, P, I, F, RGB, YCbCr, RGBA, CMYK + if pic.mode == 'YCbCr': + nchannel = 3 + elif pic.mode == 'I;16': + nchannel = 1 + else: + nchannel = len(pic.mode) + img = img.view(pic.size[1], pic.size[0], nchannel) + + img = img.transpose(0, 1).transpose(0, 2).contiguous() + if isinstance(img, torch.ByteTensor): + return img.float() + else: + return img + + +class DIML_Outdoor(Dataset): + def __init__(self, data_dir_root): + import glob + + # image paths are of the form /{outleft, depthmap}/*.png + self.image_files = glob.glob(os.path.join( + data_dir_root, "*", 'outleft', '*.png')) + self.depth_files = [r.replace("outleft", "depthmap") + for r in self.image_files] + self.transform = ToTensor() + + def __getitem__(self, idx): + image_path = self.image_files[idx] + depth_path = self.depth_files[idx] + + image = np.asarray(Image.open(image_path), dtype=np.float32) / 255.0 + depth = np.asarray(Image.open(depth_path), + dtype='uint16') / 1000.0 # mm to meters + + # depth[depth > 8] = -1 + depth = depth[..., None] + + sample = dict(image=image, depth=depth, dataset="diml_outdoor") + + # return sample + return self.transform(sample) + + def __len__(self): + return len(self.image_files) + + +def get_diml_outdoor_loader(data_dir_root, batch_size=1, **kwargs): + dataset = DIML_Outdoor(data_dir_root) + return DataLoader(dataset, batch_size, **kwargs) + +# get_diml_outdoor_loader(data_dir_root="datasets/diml/outdoor/test/HR") +# get_diml_outdoor_loader(data_dir_root="datasets/diml/outdoor/test/LR") diff --git a/annotator/zoe/zoedepth/data/diode.py b/annotator/zoe/zoedepth/data/diode.py new file mode 100644 index 0000000000000000000000000000000000000000..1510c87116b8f70ce2e1428873a8e4da042bee23 --- /dev/null +++ b/annotator/zoe/zoedepth/data/diode.py @@ -0,0 +1,125 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import os + +import numpy as np +import torch +from PIL import Image +from torch.utils.data import DataLoader, Dataset +from torchvision import transforms + + +class ToTensor(object): + def __init__(self): + # self.normalize = transforms.Normalize( + # mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + self.normalize = lambda x : x + self.resize = transforms.Resize(480) + + def __call__(self, sample): + image, depth = sample['image'], sample['depth'] + image = self.to_tensor(image) + image = self.normalize(image) + depth = self.to_tensor(depth) + + image = self.resize(image) + + return {'image': image, 'depth': depth, 'dataset': "diode"} + + def to_tensor(self, pic): + + if isinstance(pic, np.ndarray): + img = torch.from_numpy(pic.transpose((2, 0, 1))) + return img + + # # handle PIL Image + if pic.mode == 'I': + img = torch.from_numpy(np.array(pic, np.int32, copy=False)) + elif pic.mode == 'I;16': + img = torch.from_numpy(np.array(pic, np.int16, copy=False)) + else: + img = torch.ByteTensor( + torch.ByteStorage.from_buffer(pic.tobytes())) + # PIL image mode: 1, L, P, I, F, RGB, YCbCr, RGBA, CMYK + if pic.mode == 'YCbCr': + nchannel = 3 + elif pic.mode == 'I;16': + nchannel = 1 + else: + nchannel = len(pic.mode) + img = img.view(pic.size[1], pic.size[0], nchannel) + + img = img.transpose(0, 1).transpose(0, 2).contiguous() + + if isinstance(img, torch.ByteTensor): + return img.float() + else: + return img + + +class DIODE(Dataset): + def __init__(self, data_dir_root): + import glob + + # image paths are of the form /scene_#/scan_#/*.png + self.image_files = glob.glob( + os.path.join(data_dir_root, '*', '*', '*.png')) + self.depth_files = [r.replace(".png", "_depth.npy") + for r in self.image_files] + self.depth_mask_files = [ + r.replace(".png", "_depth_mask.npy") for r in self.image_files] + self.transform = ToTensor() + + def __getitem__(self, idx): + image_path = self.image_files[idx] + depth_path = self.depth_files[idx] + depth_mask_path = self.depth_mask_files[idx] + + image = np.asarray(Image.open(image_path), dtype=np.float32) / 255.0 + depth = np.load(depth_path) # in meters + valid = np.load(depth_mask_path) # binary + + # depth[depth > 8] = -1 + # depth = depth[..., None] + + sample = dict(image=image, depth=depth, valid=valid) + + # return sample + sample = self.transform(sample) + + if idx == 0: + print(sample["image"].shape) + + return sample + + def __len__(self): + return len(self.image_files) + + +def get_diode_loader(data_dir_root, batch_size=1, **kwargs): + dataset = DIODE(data_dir_root) + return DataLoader(dataset, batch_size, **kwargs) + +# get_diode_loader(data_dir_root="datasets/diode/val/outdoor") diff --git a/annotator/zoe/zoedepth/data/hypersim.py b/annotator/zoe/zoedepth/data/hypersim.py new file mode 100644 index 0000000000000000000000000000000000000000..4334198971830200f72ea2910d03f4c7d6a43334 --- /dev/null +++ b/annotator/zoe/zoedepth/data/hypersim.py @@ -0,0 +1,138 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import glob +import os + +import h5py +import numpy as np +import torch +from PIL import Image +from torch.utils.data import DataLoader, Dataset +from torchvision import transforms + + +def hypersim_distance_to_depth(npyDistance): + intWidth, intHeight, fltFocal = 1024, 768, 886.81 + + npyImageplaneX = np.linspace((-0.5 * intWidth) + 0.5, (0.5 * intWidth) - 0.5, intWidth).reshape( + 1, intWidth).repeat(intHeight, 0).astype(np.float32)[:, :, None] + npyImageplaneY = np.linspace((-0.5 * intHeight) + 0.5, (0.5 * intHeight) - 0.5, + intHeight).reshape(intHeight, 1).repeat(intWidth, 1).astype(np.float32)[:, :, None] + npyImageplaneZ = np.full([intHeight, intWidth, 1], fltFocal, np.float32) + npyImageplane = np.concatenate( + [npyImageplaneX, npyImageplaneY, npyImageplaneZ], 2) + + npyDepth = npyDistance / np.linalg.norm(npyImageplane, 2, 2) * fltFocal + return npyDepth + + +class ToTensor(object): + def __init__(self): + # self.normalize = transforms.Normalize( + # mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + self.normalize = lambda x: x + self.resize = transforms.Resize((480, 640)) + + def __call__(self, sample): + image, depth = sample['image'], sample['depth'] + image = self.to_tensor(image) + image = self.normalize(image) + depth = self.to_tensor(depth) + + image = self.resize(image) + + return {'image': image, 'depth': depth, 'dataset': "hypersim"} + + def to_tensor(self, pic): + + if isinstance(pic, np.ndarray): + img = torch.from_numpy(pic.transpose((2, 0, 1))) + return img + + # # handle PIL Image + if pic.mode == 'I': + img = torch.from_numpy(np.array(pic, np.int32, copy=False)) + elif pic.mode == 'I;16': + img = torch.from_numpy(np.array(pic, np.int16, copy=False)) + else: + img = torch.ByteTensor( + torch.ByteStorage.from_buffer(pic.tobytes())) + # PIL image mode: 1, L, P, I, F, RGB, YCbCr, RGBA, CMYK + if pic.mode == 'YCbCr': + nchannel = 3 + elif pic.mode == 'I;16': + nchannel = 1 + else: + nchannel = len(pic.mode) + img = img.view(pic.size[1], pic.size[0], nchannel) + + img = img.transpose(0, 1).transpose(0, 2).contiguous() + if isinstance(img, torch.ByteTensor): + return img.float() + else: + return img + + +class HyperSim(Dataset): + def __init__(self, data_dir_root): + # image paths are of the form //images/scene_cam_#_final_preview/*.tonemap.jpg + # depth paths are of the form //images/scene_cam_#_final_preview/*.depth_meters.hdf5 + self.image_files = glob.glob(os.path.join( + data_dir_root, '*', 'images', 'scene_cam_*_final_preview', '*.tonemap.jpg')) + self.depth_files = [r.replace("_final_preview", "_geometry_hdf5").replace( + ".tonemap.jpg", ".depth_meters.hdf5") for r in self.image_files] + self.transform = ToTensor() + + def __getitem__(self, idx): + image_path = self.image_files[idx] + depth_path = self.depth_files[idx] + + image = np.asarray(Image.open(image_path), dtype=np.float32) / 255.0 + + # depth from hdf5 + depth_fd = h5py.File(depth_path, "r") + # in meters (Euclidean distance) + distance_meters = np.array(depth_fd['dataset']) + depth = hypersim_distance_to_depth( + distance_meters) # in meters (planar depth) + + # depth[depth > 8] = -1 + depth = depth[..., None] + + sample = dict(image=image, depth=depth) + sample = self.transform(sample) + + if idx == 0: + print(sample["image"].shape) + + return sample + + def __len__(self): + return len(self.image_files) + + +def get_hypersim_loader(data_dir_root, batch_size=1, **kwargs): + dataset = HyperSim(data_dir_root) + return DataLoader(dataset, batch_size, **kwargs) diff --git a/annotator/zoe/zoedepth/data/ibims.py b/annotator/zoe/zoedepth/data/ibims.py new file mode 100644 index 0000000000000000000000000000000000000000..b66abfabcf4cfc617d4a60ec818780c3548d9920 --- /dev/null +++ b/annotator/zoe/zoedepth/data/ibims.py @@ -0,0 +1,81 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import os + +import numpy as np +import torch +from PIL import Image +from torch.utils.data import DataLoader, Dataset +from torchvision import transforms as T + + +class iBims(Dataset): + def __init__(self, config): + root_folder = config.ibims_root + with open(os.path.join(root_folder, "imagelist.txt"), 'r') as f: + imglist = f.read().split() + + samples = [] + for basename in imglist: + img_path = os.path.join(root_folder, 'rgb', basename + ".png") + depth_path = os.path.join(root_folder, 'depth', basename + ".png") + valid_mask_path = os.path.join( + root_folder, 'mask_invalid', basename+".png") + transp_mask_path = os.path.join( + root_folder, 'mask_transp', basename+".png") + + samples.append( + (img_path, depth_path, valid_mask_path, transp_mask_path)) + + self.samples = samples + # self.normalize = T.Normalize( + # mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + self.normalize = lambda x : x + + def __getitem__(self, idx): + img_path, depth_path, valid_mask_path, transp_mask_path = self.samples[idx] + + img = np.asarray(Image.open(img_path), dtype=np.float32) / 255.0 + depth = np.asarray(Image.open(depth_path), + dtype=np.uint16).astype('float')*50.0/65535 + + mask_valid = np.asarray(Image.open(valid_mask_path)) + mask_transp = np.asarray(Image.open(transp_mask_path)) + + # depth = depth * mask_valid * mask_transp + depth = np.where(mask_valid * mask_transp, depth, -1) + + img = torch.from_numpy(img).permute(2, 0, 1) + img = self.normalize(img) + depth = torch.from_numpy(depth).unsqueeze(0) + return dict(image=img, depth=depth, image_path=img_path, depth_path=depth_path, dataset='ibims') + + def __len__(self): + return len(self.samples) + + +def get_ibims_loader(config, batch_size=1, **kwargs): + dataloader = DataLoader(iBims(config), batch_size=batch_size, **kwargs) + return dataloader diff --git a/annotator/zoe/zoedepth/data/preprocess.py b/annotator/zoe/zoedepth/data/preprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..e08cc309dc823ae6efd7cda8db9eb37130dc5499 --- /dev/null +++ b/annotator/zoe/zoedepth/data/preprocess.py @@ -0,0 +1,154 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import numpy as np +from dataclasses import dataclass +from typing import Tuple, List + +# dataclass to store the crop parameters +@dataclass +class CropParams: + top: int + bottom: int + left: int + right: int + + + +def get_border_params(rgb_image, tolerance=0.1, cut_off=20, value=0, level_diff_threshold=5, channel_axis=-1, min_border=5) -> CropParams: + gray_image = np.mean(rgb_image, axis=channel_axis) + h, w = gray_image.shape + + + def num_value_pixels(arr): + return np.sum(np.abs(arr - value) < level_diff_threshold) + + def is_above_tolerance(arr, total_pixels): + return (num_value_pixels(arr) / total_pixels) > tolerance + + # Crop top border until number of value pixels become below tolerance + top = min_border + while is_above_tolerance(gray_image[top, :], w) and top < h-1: + top += 1 + if top > cut_off: + break + + # Crop bottom border until number of value pixels become below tolerance + bottom = h - min_border + while is_above_tolerance(gray_image[bottom, :], w) and bottom > 0: + bottom -= 1 + if h - bottom > cut_off: + break + + # Crop left border until number of value pixels become below tolerance + left = min_border + while is_above_tolerance(gray_image[:, left], h) and left < w-1: + left += 1 + if left > cut_off: + break + + # Crop right border until number of value pixels become below tolerance + right = w - min_border + while is_above_tolerance(gray_image[:, right], h) and right > 0: + right -= 1 + if w - right > cut_off: + break + + + return CropParams(top, bottom, left, right) + + +def get_white_border(rgb_image, value=255, **kwargs) -> CropParams: + """Crops the white border of the RGB. + + Args: + rgb: RGB image, shape (H, W, 3). + Returns: + Crop parameters. + """ + if value == 255: + # assert range of values in rgb image is [0, 255] + assert np.max(rgb_image) <= 255 and np.min(rgb_image) >= 0, "RGB image values are not in range [0, 255]." + assert rgb_image.max() > 1, "RGB image values are not in range [0, 255]." + elif value == 1: + # assert range of values in rgb image is [0, 1] + assert np.max(rgb_image) <= 1 and np.min(rgb_image) >= 0, "RGB image values are not in range [0, 1]." + + return get_border_params(rgb_image, value=value, **kwargs) + +def get_black_border(rgb_image, **kwargs) -> CropParams: + """Crops the black border of the RGB. + + Args: + rgb: RGB image, shape (H, W, 3). + + Returns: + Crop parameters. + """ + + return get_border_params(rgb_image, value=0, **kwargs) + +def crop_image(image: np.ndarray, crop_params: CropParams) -> np.ndarray: + """Crops the image according to the crop parameters. + + Args: + image: RGB or depth image, shape (H, W, 3) or (H, W). + crop_params: Crop parameters. + + Returns: + Cropped image. + """ + return image[crop_params.top:crop_params.bottom, crop_params.left:crop_params.right] + +def crop_images(*images: np.ndarray, crop_params: CropParams) -> Tuple[np.ndarray]: + """Crops the images according to the crop parameters. + + Args: + images: RGB or depth images, shape (H, W, 3) or (H, W). + crop_params: Crop parameters. + + Returns: + Cropped images. + """ + return tuple(crop_image(image, crop_params) for image in images) + +def crop_black_or_white_border(rgb_image, *other_images: np.ndarray, tolerance=0.1, cut_off=20, level_diff_threshold=5) -> Tuple[np.ndarray]: + """Crops the white and black border of the RGB and depth images. + + Args: + rgb: RGB image, shape (H, W, 3). This image is used to determine the border. + other_images: The other images to crop according to the border of the RGB image. + Returns: + Cropped RGB and other images. + """ + # crop black border + crop_params = get_black_border(rgb_image, tolerance=tolerance, cut_off=cut_off, level_diff_threshold=level_diff_threshold) + cropped_images = crop_images(rgb_image, *other_images, crop_params=crop_params) + + # crop white border + crop_params = get_white_border(cropped_images[0], tolerance=tolerance, cut_off=cut_off, level_diff_threshold=level_diff_threshold) + cropped_images = crop_images(*cropped_images, crop_params=crop_params) + + return cropped_images + \ No newline at end of file diff --git a/annotator/zoe/zoedepth/data/sun_rgbd_loader.py b/annotator/zoe/zoedepth/data/sun_rgbd_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..9e2bdb9aefe68ca4439f41eff3bba722c49fb976 --- /dev/null +++ b/annotator/zoe/zoedepth/data/sun_rgbd_loader.py @@ -0,0 +1,106 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import os + +import numpy as np +import torch +from PIL import Image +from torch.utils.data import DataLoader, Dataset +from torchvision import transforms + + +class ToTensor(object): + def __init__(self): + # self.normalize = transforms.Normalize( + # mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + self.normalize = lambda x : x + + def __call__(self, sample): + image, depth = sample['image'], sample['depth'] + image = self.to_tensor(image) + image = self.normalize(image) + depth = self.to_tensor(depth) + + return {'image': image, 'depth': depth, 'dataset': "sunrgbd"} + + def to_tensor(self, pic): + + if isinstance(pic, np.ndarray): + img = torch.from_numpy(pic.transpose((2, 0, 1))) + return img + + # # handle PIL Image + if pic.mode == 'I': + img = torch.from_numpy(np.array(pic, np.int32, copy=False)) + elif pic.mode == 'I;16': + img = torch.from_numpy(np.array(pic, np.int16, copy=False)) + else: + img = torch.ByteTensor( + torch.ByteStorage.from_buffer(pic.tobytes())) + # PIL image mode: 1, L, P, I, F, RGB, YCbCr, RGBA, CMYK + if pic.mode == 'YCbCr': + nchannel = 3 + elif pic.mode == 'I;16': + nchannel = 1 + else: + nchannel = len(pic.mode) + img = img.view(pic.size[1], pic.size[0], nchannel) + + img = img.transpose(0, 1).transpose(0, 2).contiguous() + if isinstance(img, torch.ByteTensor): + return img.float() + else: + return img + + +class SunRGBD(Dataset): + def __init__(self, data_dir_root): + # test_file_dirs = loadmat(train_test_file)['alltest'].squeeze() + # all_test = [t[0].replace("/n/fs/sun3d/data/", "") for t in test_file_dirs] + # self.all_test = [os.path.join(data_dir_root, t) for t in all_test] + import glob + self.image_files = glob.glob( + os.path.join(data_dir_root, 'rgb', 'rgb', '*')) + self.depth_files = [ + r.replace("rgb/rgb", "gt/gt").replace("jpg", "png") for r in self.image_files] + self.transform = ToTensor() + + def __getitem__(self, idx): + image_path = self.image_files[idx] + depth_path = self.depth_files[idx] + + image = np.asarray(Image.open(image_path), dtype=np.float32) / 255.0 + depth = np.asarray(Image.open(depth_path), dtype='uint16') / 1000.0 + depth[depth > 8] = -1 + depth = depth[..., None] + return self.transform(dict(image=image, depth=depth)) + + def __len__(self): + return len(self.image_files) + + +def get_sunrgbd_loader(data_dir_root, batch_size=1, **kwargs): + dataset = SunRGBD(data_dir_root) + return DataLoader(dataset, batch_size, **kwargs) diff --git a/annotator/zoe/zoedepth/data/transforms.py b/annotator/zoe/zoedepth/data/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..374416dff24fb4fd55598f3946d6d6b091ddefc9 --- /dev/null +++ b/annotator/zoe/zoedepth/data/transforms.py @@ -0,0 +1,481 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import math +import random + +import cv2 +import numpy as np + + +class RandomFliplr(object): + """Horizontal flip of the sample with given probability. + """ + + def __init__(self, probability=0.5): + """Init. + + Args: + probability (float, optional): Flip probability. Defaults to 0.5. + """ + self.__probability = probability + + def __call__(self, sample): + prob = random.random() + + if prob < self.__probability: + for k, v in sample.items(): + if len(v.shape) >= 2: + sample[k] = np.fliplr(v).copy() + + return sample + + +def apply_min_size(sample, size, image_interpolation_method=cv2.INTER_AREA): + """Rezise the sample to ensure the given size. Keeps aspect ratio. + + Args: + sample (dict): sample + size (tuple): image size + + Returns: + tuple: new size + """ + shape = list(sample["disparity"].shape) + + if shape[0] >= size[0] and shape[1] >= size[1]: + return sample + + scale = [0, 0] + scale[0] = size[0] / shape[0] + scale[1] = size[1] / shape[1] + + scale = max(scale) + + shape[0] = math.ceil(scale * shape[0]) + shape[1] = math.ceil(scale * shape[1]) + + # resize + sample["image"] = cv2.resize( + sample["image"], tuple(shape[::-1]), interpolation=image_interpolation_method + ) + + sample["disparity"] = cv2.resize( + sample["disparity"], tuple(shape[::-1]), interpolation=cv2.INTER_NEAREST + ) + sample["mask"] = cv2.resize( + sample["mask"].astype(np.float32), + tuple(shape[::-1]), + interpolation=cv2.INTER_NEAREST, + ) + sample["mask"] = sample["mask"].astype(bool) + + return tuple(shape) + + +class RandomCrop(object): + """Get a random crop of the sample with the given size (width, height). + """ + + def __init__( + self, + width, + height, + resize_if_needed=False, + image_interpolation_method=cv2.INTER_AREA, + ): + """Init. + + Args: + width (int): output width + height (int): output height + resize_if_needed (bool, optional): If True, sample might be upsampled to ensure + that a crop of size (width, height) is possbile. Defaults to False. + """ + self.__size = (height, width) + self.__resize_if_needed = resize_if_needed + self.__image_interpolation_method = image_interpolation_method + + def __call__(self, sample): + + shape = sample["disparity"].shape + + if self.__size[0] > shape[0] or self.__size[1] > shape[1]: + if self.__resize_if_needed: + shape = apply_min_size( + sample, self.__size, self.__image_interpolation_method + ) + else: + raise Exception( + "Output size {} bigger than input size {}.".format( + self.__size, shape + ) + ) + + offset = ( + np.random.randint(shape[0] - self.__size[0] + 1), + np.random.randint(shape[1] - self.__size[1] + 1), + ) + + for k, v in sample.items(): + if k == "code" or k == "basis": + continue + + if len(sample[k].shape) >= 2: + sample[k] = v[ + offset[0]: offset[0] + self.__size[0], + offset[1]: offset[1] + self.__size[1], + ] + + return sample + + +class Resize(object): + """Resize sample to given size (width, height). + """ + + def __init__( + self, + width, + height, + resize_target=True, + keep_aspect_ratio=False, + ensure_multiple_of=1, + resize_method="lower_bound", + image_interpolation_method=cv2.INTER_AREA, + letter_box=False, + ): + """Init. + + Args: + width (int): desired output width + height (int): desired output height + resize_target (bool, optional): + True: Resize the full sample (image, mask, target). + False: Resize image only. + Defaults to True. + keep_aspect_ratio (bool, optional): + True: Keep the aspect ratio of the input sample. + Output sample might not have the given width and height, and + resize behaviour depends on the parameter 'resize_method'. + Defaults to False. + ensure_multiple_of (int, optional): + Output width and height is constrained to be multiple of this parameter. + Defaults to 1. + resize_method (str, optional): + "lower_bound": Output will be at least as large as the given size. + "upper_bound": Output will be at max as large as the given size. (Output size might be smaller than given size.) + "minimal": Scale as least as possible. (Output size might be smaller than given size.) + Defaults to "lower_bound". + """ + self.__width = width + self.__height = height + + self.__resize_target = resize_target + self.__keep_aspect_ratio = keep_aspect_ratio + self.__multiple_of = ensure_multiple_of + self.__resize_method = resize_method + self.__image_interpolation_method = image_interpolation_method + self.__letter_box = letter_box + + def constrain_to_multiple_of(self, x, min_val=0, max_val=None): + y = (np.round(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if max_val is not None and y > max_val: + y = (np.floor(x / self.__multiple_of) + * self.__multiple_of).astype(int) + + if y < min_val: + y = (np.ceil(x / self.__multiple_of) + * self.__multiple_of).astype(int) + + return y + + def get_size(self, width, height): + # determine new height and width + scale_height = self.__height / height + scale_width = self.__width / width + + if self.__keep_aspect_ratio: + if self.__resize_method == "lower_bound": + # scale such that output size is lower bound + if scale_width > scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "upper_bound": + # scale such that output size is upper bound + if scale_width < scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "minimal": + # scale as least as possbile + if abs(1 - scale_width) < abs(1 - scale_height): + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + else: + raise ValueError( + f"resize_method {self.__resize_method} not implemented" + ) + + if self.__resize_method == "lower_bound": + new_height = self.constrain_to_multiple_of( + scale_height * height, min_val=self.__height + ) + new_width = self.constrain_to_multiple_of( + scale_width * width, min_val=self.__width + ) + elif self.__resize_method == "upper_bound": + new_height = self.constrain_to_multiple_of( + scale_height * height, max_val=self.__height + ) + new_width = self.constrain_to_multiple_of( + scale_width * width, max_val=self.__width + ) + elif self.__resize_method == "minimal": + new_height = self.constrain_to_multiple_of(scale_height * height) + new_width = self.constrain_to_multiple_of(scale_width * width) + else: + raise ValueError( + f"resize_method {self.__resize_method} not implemented") + + return (new_width, new_height) + + def make_letter_box(self, sample): + top = bottom = (self.__height - sample.shape[0]) // 2 + left = right = (self.__width - sample.shape[1]) // 2 + sample = cv2.copyMakeBorder( + sample, top, bottom, left, right, cv2.BORDER_CONSTANT, None, 0) + return sample + + def __call__(self, sample): + width, height = self.get_size( + sample["image"].shape[1], sample["image"].shape[0] + ) + + # resize sample + sample["image"] = cv2.resize( + sample["image"], + (width, height), + interpolation=self.__image_interpolation_method, + ) + + if self.__letter_box: + sample["image"] = self.make_letter_box(sample["image"]) + + if self.__resize_target: + if "disparity" in sample: + sample["disparity"] = cv2.resize( + sample["disparity"], + (width, height), + interpolation=cv2.INTER_NEAREST, + ) + + if self.__letter_box: + sample["disparity"] = self.make_letter_box( + sample["disparity"]) + + if "depth" in sample: + sample["depth"] = cv2.resize( + sample["depth"], (width, + height), interpolation=cv2.INTER_NEAREST + ) + + if self.__letter_box: + sample["depth"] = self.make_letter_box(sample["depth"]) + + sample["mask"] = cv2.resize( + sample["mask"].astype(np.float32), + (width, height), + interpolation=cv2.INTER_NEAREST, + ) + + if self.__letter_box: + sample["mask"] = self.make_letter_box(sample["mask"]) + + sample["mask"] = sample["mask"].astype(bool) + + return sample + + +class ResizeFixed(object): + def __init__(self, size): + self.__size = size + + def __call__(self, sample): + sample["image"] = cv2.resize( + sample["image"], self.__size[::-1], interpolation=cv2.INTER_LINEAR + ) + + sample["disparity"] = cv2.resize( + sample["disparity"], self.__size[::- + 1], interpolation=cv2.INTER_NEAREST + ) + + sample["mask"] = cv2.resize( + sample["mask"].astype(np.float32), + self.__size[::-1], + interpolation=cv2.INTER_NEAREST, + ) + sample["mask"] = sample["mask"].astype(bool) + + return sample + + +class Rescale(object): + """Rescale target values to the interval [0, max_val]. + If input is constant, values are set to max_val / 2. + """ + + def __init__(self, max_val=1.0, use_mask=True): + """Init. + + Args: + max_val (float, optional): Max output value. Defaults to 1.0. + use_mask (bool, optional): Only operate on valid pixels (mask == True). Defaults to True. + """ + self.__max_val = max_val + self.__use_mask = use_mask + + def __call__(self, sample): + disp = sample["disparity"] + + if self.__use_mask: + mask = sample["mask"] + else: + mask = np.ones_like(disp, dtype=np.bool) + + if np.sum(mask) == 0: + return sample + + min_val = np.min(disp[mask]) + max_val = np.max(disp[mask]) + + if max_val > min_val: + sample["disparity"][mask] = ( + (disp[mask] - min_val) / (max_val - min_val) * self.__max_val + ) + else: + sample["disparity"][mask] = np.ones_like( + disp[mask]) * self.__max_val / 2.0 + + return sample + + +# mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] +class NormalizeImage(object): + """Normlize image by given mean and std. + """ + + def __init__(self, mean, std): + self.__mean = mean + self.__std = std + + def __call__(self, sample): + sample["image"] = (sample["image"] - self.__mean) / self.__std + + return sample + + +class DepthToDisparity(object): + """Convert depth to disparity. Removes depth from sample. + """ + + def __init__(self, eps=1e-4): + self.__eps = eps + + def __call__(self, sample): + assert "depth" in sample + + sample["mask"][sample["depth"] < self.__eps] = False + + sample["disparity"] = np.zeros_like(sample["depth"]) + sample["disparity"][sample["depth"] >= self.__eps] = ( + 1.0 / sample["depth"][sample["depth"] >= self.__eps] + ) + + del sample["depth"] + + return sample + + +class DisparityToDepth(object): + """Convert disparity to depth. Removes disparity from sample. + """ + + def __init__(self, eps=1e-4): + self.__eps = eps + + def __call__(self, sample): + assert "disparity" in sample + + disp = np.abs(sample["disparity"]) + sample["mask"][disp < self.__eps] = False + + # print(sample["disparity"]) + # print(sample["mask"].sum()) + # exit() + + sample["depth"] = np.zeros_like(disp) + sample["depth"][disp >= self.__eps] = ( + 1.0 / disp[disp >= self.__eps] + ) + + del sample["disparity"] + + return sample + + +class PrepareForNet(object): + """Prepare sample for usage as network input. + """ + + def __init__(self): + pass + + def __call__(self, sample): + image = np.transpose(sample["image"], (2, 0, 1)) + sample["image"] = np.ascontiguousarray(image).astype(np.float32) + + if "mask" in sample: + sample["mask"] = sample["mask"].astype(np.float32) + sample["mask"] = np.ascontiguousarray(sample["mask"]) + + if "disparity" in sample: + disparity = sample["disparity"].astype(np.float32) + sample["disparity"] = np.ascontiguousarray(disparity) + + if "depth" in sample: + depth = sample["depth"].astype(np.float32) + sample["depth"] = np.ascontiguousarray(depth) + + return sample diff --git a/annotator/zoe/zoedepth/data/vkitti.py b/annotator/zoe/zoedepth/data/vkitti.py new file mode 100644 index 0000000000000000000000000000000000000000..72a2e5a8346f6e630ede0e28d6959725af8d7e72 --- /dev/null +++ b/annotator/zoe/zoedepth/data/vkitti.py @@ -0,0 +1,151 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import torch +from torch.utils.data import Dataset, DataLoader +from torchvision import transforms +import os + +from PIL import Image +import numpy as np +import cv2 + + +class ToTensor(object): + def __init__(self): + self.normalize = transforms.Normalize( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + # self.resize = transforms.Resize((375, 1242)) + + def __call__(self, sample): + image, depth = sample['image'], sample['depth'] + + image = self.to_tensor(image) + image = self.normalize(image) + depth = self.to_tensor(depth) + + # image = self.resize(image) + + return {'image': image, 'depth': depth, 'dataset': "vkitti"} + + def to_tensor(self, pic): + + if isinstance(pic, np.ndarray): + img = torch.from_numpy(pic.transpose((2, 0, 1))) + return img + + # # handle PIL Image + if pic.mode == 'I': + img = torch.from_numpy(np.array(pic, np.int32, copy=False)) + elif pic.mode == 'I;16': + img = torch.from_numpy(np.array(pic, np.int16, copy=False)) + else: + img = torch.ByteTensor( + torch.ByteStorage.from_buffer(pic.tobytes())) + # PIL image mode: 1, L, P, I, F, RGB, YCbCr, RGBA, CMYK + if pic.mode == 'YCbCr': + nchannel = 3 + elif pic.mode == 'I;16': + nchannel = 1 + else: + nchannel = len(pic.mode) + img = img.view(pic.size[1], pic.size[0], nchannel) + + img = img.transpose(0, 1).transpose(0, 2).contiguous() + if isinstance(img, torch.ByteTensor): + return img.float() + else: + return img + + +class VKITTI(Dataset): + def __init__(self, data_dir_root, do_kb_crop=True): + import glob + # image paths are of the form /{HR, LR}//{color, depth_filled}/*.png + self.image_files = glob.glob(os.path.join( + data_dir_root, "test_color", '*.png')) + self.depth_files = [r.replace("test_color", "test_depth") + for r in self.image_files] + self.do_kb_crop = True + self.transform = ToTensor() + + def __getitem__(self, idx): + image_path = self.image_files[idx] + depth_path = self.depth_files[idx] + + image = Image.open(image_path) + depth = Image.open(depth_path) + depth = cv2.imread(depth_path, cv2.IMREAD_ANYCOLOR | + cv2.IMREAD_ANYDEPTH) + print("dpeth min max", depth.min(), depth.max()) + + # print(np.shape(image)) + # print(np.shape(depth)) + + # depth[depth > 8] = -1 + + if self.do_kb_crop and False: + height = image.height + width = image.width + top_margin = int(height - 352) + left_margin = int((width - 1216) / 2) + depth = depth.crop( + (left_margin, top_margin, left_margin + 1216, top_margin + 352)) + image = image.crop( + (left_margin, top_margin, left_margin + 1216, top_margin + 352)) + # uv = uv[:, top_margin:top_margin + 352, left_margin:left_margin + 1216] + + image = np.asarray(image, dtype=np.float32) / 255.0 + # depth = np.asarray(depth, dtype=np.uint16) /1. + depth = depth[..., None] + sample = dict(image=image, depth=depth) + + # return sample + sample = self.transform(sample) + + if idx == 0: + print(sample["image"].shape) + + return sample + + def __len__(self): + return len(self.image_files) + + +def get_vkitti_loader(data_dir_root, batch_size=1, **kwargs): + dataset = VKITTI(data_dir_root) + return DataLoader(dataset, batch_size, **kwargs) + + +if __name__ == "__main__": + loader = get_vkitti_loader( + data_dir_root="/home/bhatsf/shortcuts/datasets/vkitti_test") + print("Total files", len(loader.dataset)) + for i, sample in enumerate(loader): + print(sample["image"].shape) + print(sample["depth"].shape) + print(sample["dataset"]) + print(sample['depth'].min(), sample['depth'].max()) + if i > 5: + break diff --git a/annotator/zoe/zoedepth/data/vkitti2.py b/annotator/zoe/zoedepth/data/vkitti2.py new file mode 100644 index 0000000000000000000000000000000000000000..9bcfb0414b7f3f21859f30ae34bd71689516a3e7 --- /dev/null +++ b/annotator/zoe/zoedepth/data/vkitti2.py @@ -0,0 +1,187 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import os + +import cv2 +import numpy as np +import torch +from PIL import Image +from torch.utils.data import DataLoader, Dataset +from torchvision import transforms + + +class ToTensor(object): + def __init__(self): + # self.normalize = transforms.Normalize( + # mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) + self.normalize = lambda x: x + # self.resize = transforms.Resize((375, 1242)) + + def __call__(self, sample): + image, depth = sample['image'], sample['depth'] + + image = self.to_tensor(image) + image = self.normalize(image) + depth = self.to_tensor(depth) + + # image = self.resize(image) + + return {'image': image, 'depth': depth, 'dataset': "vkitti"} + + def to_tensor(self, pic): + + if isinstance(pic, np.ndarray): + img = torch.from_numpy(pic.transpose((2, 0, 1))) + return img + + # # handle PIL Image + if pic.mode == 'I': + img = torch.from_numpy(np.array(pic, np.int32, copy=False)) + elif pic.mode == 'I;16': + img = torch.from_numpy(np.array(pic, np.int16, copy=False)) + else: + img = torch.ByteTensor( + torch.ByteStorage.from_buffer(pic.tobytes())) + # PIL image mode: 1, L, P, I, F, RGB, YCbCr, RGBA, CMYK + if pic.mode == 'YCbCr': + nchannel = 3 + elif pic.mode == 'I;16': + nchannel = 1 + else: + nchannel = len(pic.mode) + img = img.view(pic.size[1], pic.size[0], nchannel) + + img = img.transpose(0, 1).transpose(0, 2).contiguous() + if isinstance(img, torch.ByteTensor): + return img.float() + else: + return img + + +class VKITTI2(Dataset): + def __init__(self, data_dir_root, do_kb_crop=True, split="test"): + import glob + + # image paths are of the form /rgb///frames//Camera<0,1>/rgb_{}.jpg + self.image_files = glob.glob(os.path.join( + data_dir_root, "rgb", "**", "frames", "rgb", "Camera_0", '*.jpg'), recursive=True) + self.depth_files = [r.replace("/rgb/", "/depth/").replace( + "rgb_", "depth_").replace(".jpg", ".png") for r in self.image_files] + self.do_kb_crop = True + self.transform = ToTensor() + + # If train test split is not created, then create one. + # Split is such that 8% of the frames from each scene are used for testing. + if not os.path.exists(os.path.join(data_dir_root, "train.txt")): + import random + scenes = set([os.path.basename(os.path.dirname( + os.path.dirname(os.path.dirname(f)))) for f in self.image_files]) + train_files = [] + test_files = [] + for scene in scenes: + scene_files = [f for f in self.image_files if os.path.basename( + os.path.dirname(os.path.dirname(os.path.dirname(f)))) == scene] + random.shuffle(scene_files) + train_files.extend(scene_files[:int(len(scene_files) * 0.92)]) + test_files.extend(scene_files[int(len(scene_files) * 0.92):]) + with open(os.path.join(data_dir_root, "train.txt"), "w") as f: + f.write("\n".join(train_files)) + with open(os.path.join(data_dir_root, "test.txt"), "w") as f: + f.write("\n".join(test_files)) + + if split == "train": + with open(os.path.join(data_dir_root, "train.txt"), "r") as f: + self.image_files = f.read().splitlines() + self.depth_files = [r.replace("/rgb/", "/depth/").replace( + "rgb_", "depth_").replace(".jpg", ".png") for r in self.image_files] + elif split == "test": + with open(os.path.join(data_dir_root, "test.txt"), "r") as f: + self.image_files = f.read().splitlines() + self.depth_files = [r.replace("/rgb/", "/depth/").replace( + "rgb_", "depth_").replace(".jpg", ".png") for r in self.image_files] + + def __getitem__(self, idx): + image_path = self.image_files[idx] + depth_path = self.depth_files[idx] + + image = Image.open(image_path) + # depth = Image.open(depth_path) + depth = cv2.imread(depth_path, cv2.IMREAD_ANYCOLOR | + cv2.IMREAD_ANYDEPTH) / 100.0 # cm to m + depth = Image.fromarray(depth) + # print("dpeth min max", depth.min(), depth.max()) + + # print(np.shape(image)) + # print(np.shape(depth)) + + if self.do_kb_crop: + if idx == 0: + print("Using KB input crop") + height = image.height + width = image.width + top_margin = int(height - 352) + left_margin = int((width - 1216) / 2) + depth = depth.crop( + (left_margin, top_margin, left_margin + 1216, top_margin + 352)) + image = image.crop( + (left_margin, top_margin, left_margin + 1216, top_margin + 352)) + # uv = uv[:, top_margin:top_margin + 352, left_margin:left_margin + 1216] + + image = np.asarray(image, dtype=np.float32) / 255.0 + # depth = np.asarray(depth, dtype=np.uint16) /1. + depth = np.asarray(depth, dtype=np.float32) / 1. + depth[depth > 80] = -1 + + depth = depth[..., None] + sample = dict(image=image, depth=depth) + + # return sample + sample = self.transform(sample) + + if idx == 0: + print(sample["image"].shape) + + return sample + + def __len__(self): + return len(self.image_files) + + +def get_vkitti2_loader(data_dir_root, batch_size=1, **kwargs): + dataset = VKITTI2(data_dir_root) + return DataLoader(dataset, batch_size, **kwargs) + + +if __name__ == "__main__": + loader = get_vkitti2_loader( + data_dir_root="/home/bhatsf/shortcuts/datasets/vkitti2") + print("Total files", len(loader.dataset)) + for i, sample in enumerate(loader): + print(sample["image"].shape) + print(sample["depth"].shape) + print(sample["dataset"]) + print(sample['depth'].min(), sample['depth'].max()) + if i > 5: + break diff --git a/annotator/zoe/zoedepth/models/__init__.py b/annotator/zoe/zoedepth/models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5f2668792389157609abb2a0846fb620e7d67eb9 --- /dev/null +++ b/annotator/zoe/zoedepth/models/__init__.py @@ -0,0 +1,24 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + diff --git a/annotator/zoe/zoedepth/models/base_models/__init__.py b/annotator/zoe/zoedepth/models/base_models/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5f2668792389157609abb2a0846fb620e7d67eb9 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/__init__.py @@ -0,0 +1,24 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + diff --git a/annotator/zoe/zoedepth/models/base_models/midas.py b/annotator/zoe/zoedepth/models/base_models/midas.py new file mode 100644 index 0000000000000000000000000000000000000000..ee660bc93d44c28efe8d8c674e715ea2ecb4c183 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas.py @@ -0,0 +1,379 @@ +# MIT License +import os + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import torch +import torch.nn as nn +import numpy as np +from torchvision.transforms import Normalize + + +def denormalize(x): + """Reverses the imagenet normalization applied to the input. + + Args: + x (torch.Tensor - shape(N,3,H,W)): input tensor + + Returns: + torch.Tensor - shape(N,3,H,W): Denormalized input + """ + mean = torch.Tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1).to(x.device) + std = torch.Tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1).to(x.device) + return x * std + mean + +def get_activation(name, bank): + def hook(model, input, output): + bank[name] = output + return hook + + +class Resize(object): + """Resize sample to given size (width, height). + """ + + def __init__( + self, + width, + height, + resize_target=True, + keep_aspect_ratio=False, + ensure_multiple_of=1, + resize_method="lower_bound", + ): + """Init. + Args: + width (int): desired output width + height (int): desired output height + resize_target (bool, optional): + True: Resize the full sample (image, mask, target). + False: Resize image only. + Defaults to True. + keep_aspect_ratio (bool, optional): + True: Keep the aspect ratio of the input sample. + Output sample might not have the given width and height, and + resize behaviour depends on the parameter 'resize_method'. + Defaults to False. + ensure_multiple_of (int, optional): + Output width and height is constrained to be multiple of this parameter. + Defaults to 1. + resize_method (str, optional): + "lower_bound": Output will be at least as large as the given size. + "upper_bound": Output will be at max as large as the given size. (Output size might be smaller than given size.) + "minimal": Scale as least as possible. (Output size might be smaller than given size.) + Defaults to "lower_bound". + """ + print("Params passed to Resize transform:") + print("\twidth: ", width) + print("\theight: ", height) + print("\tresize_target: ", resize_target) + print("\tkeep_aspect_ratio: ", keep_aspect_ratio) + print("\tensure_multiple_of: ", ensure_multiple_of) + print("\tresize_method: ", resize_method) + + self.__width = width + self.__height = height + + self.__keep_aspect_ratio = keep_aspect_ratio + self.__multiple_of = ensure_multiple_of + self.__resize_method = resize_method + + def constrain_to_multiple_of(self, x, min_val=0, max_val=None): + y = (np.round(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if max_val is not None and y > max_val: + y = (np.floor(x / self.__multiple_of) + * self.__multiple_of).astype(int) + + if y < min_val: + y = (np.ceil(x / self.__multiple_of) + * self.__multiple_of).astype(int) + + return y + + def get_size(self, width, height): + # determine new height and width + scale_height = self.__height / height + scale_width = self.__width / width + + if self.__keep_aspect_ratio: + if self.__resize_method == "lower_bound": + # scale such that output size is lower bound + if scale_width > scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "upper_bound": + # scale such that output size is upper bound + if scale_width < scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "minimal": + # scale as least as possbile + if abs(1 - scale_width) < abs(1 - scale_height): + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + else: + raise ValueError( + f"resize_method {self.__resize_method} not implemented" + ) + + if self.__resize_method == "lower_bound": + new_height = self.constrain_to_multiple_of( + scale_height * height, min_val=self.__height + ) + new_width = self.constrain_to_multiple_of( + scale_width * width, min_val=self.__width + ) + elif self.__resize_method == "upper_bound": + new_height = self.constrain_to_multiple_of( + scale_height * height, max_val=self.__height + ) + new_width = self.constrain_to_multiple_of( + scale_width * width, max_val=self.__width + ) + elif self.__resize_method == "minimal": + new_height = self.constrain_to_multiple_of(scale_height * height) + new_width = self.constrain_to_multiple_of(scale_width * width) + else: + raise ValueError( + f"resize_method {self.__resize_method} not implemented") + + return (new_width, new_height) + + def __call__(self, x): + width, height = self.get_size(*x.shape[-2:][::-1]) + return nn.functional.interpolate(x, (height, width), mode='bilinear', align_corners=True) + +class PrepForMidas(object): + def __init__(self, resize_mode="minimal", keep_aspect_ratio=True, img_size=384, do_resize=True): + if isinstance(img_size, int): + img_size = (img_size, img_size) + net_h, net_w = img_size + self.normalization = Normalize( + mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + self.resizer = Resize(net_w, net_h, keep_aspect_ratio=keep_aspect_ratio, ensure_multiple_of=32, resize_method=resize_mode) \ + if do_resize else nn.Identity() + + def __call__(self, x): + return self.normalization(self.resizer(x)) + + +class MidasCore(nn.Module): + def __init__(self, midas, trainable=False, fetch_features=True, layer_names=('out_conv', 'l4_rn', 'r4', 'r3', 'r2', 'r1'), freeze_bn=False, keep_aspect_ratio=True, + img_size=384, **kwargs): + """Midas Base model used for multi-scale feature extraction. + + Args: + midas (torch.nn.Module): Midas model. + trainable (bool, optional): Train midas model. Defaults to False. + fetch_features (bool, optional): Extract multi-scale features. Defaults to True. + layer_names (tuple, optional): Layers used for feature extraction. Order = (head output features, last layer features, ...decoder features). Defaults to ('out_conv', 'l4_rn', 'r4', 'r3', 'r2', 'r1'). + freeze_bn (bool, optional): Freeze BatchNorm. Generally results in better finetuning performance. Defaults to False. + keep_aspect_ratio (bool, optional): Keep the aspect ratio of input images while resizing. Defaults to True. + img_size (int, tuple, optional): Input resolution. Defaults to 384. + """ + super().__init__() + self.core = midas + self.output_channels = None + self.core_out = {} + self.trainable = trainable + self.fetch_features = fetch_features + # midas.scratch.output_conv = nn.Identity() + self.handles = [] + # self.layer_names = ['out_conv','l4_rn', 'r4', 'r3', 'r2', 'r1'] + self.layer_names = layer_names + + self.set_trainable(trainable) + self.set_fetch_features(fetch_features) + + self.prep = PrepForMidas(keep_aspect_ratio=keep_aspect_ratio, + img_size=img_size, do_resize=kwargs.get('do_resize', True)) + + if freeze_bn: + self.freeze_bn() + + def set_trainable(self, trainable): + self.trainable = trainable + if trainable: + self.unfreeze() + else: + self.freeze() + return self + + def set_fetch_features(self, fetch_features): + self.fetch_features = fetch_features + if fetch_features: + if len(self.handles) == 0: + self.attach_hooks(self.core) + else: + self.remove_hooks() + return self + + def freeze(self): + for p in self.parameters(): + p.requires_grad = False + self.trainable = False + return self + + def unfreeze(self): + for p in self.parameters(): + p.requires_grad = True + self.trainable = True + return self + + def freeze_bn(self): + for m in self.modules(): + if isinstance(m, nn.BatchNorm2d): + m.eval() + return self + + def forward(self, x, denorm=False, return_rel_depth=False): + with torch.no_grad(): + if denorm: + x = denormalize(x) + x = self.prep(x) + # print("Shape after prep: ", x.shape) + + with torch.set_grad_enabled(self.trainable): + + # print("Input size to Midascore", x.shape) + rel_depth = self.core(x) + # print("Output from midas shape", rel_depth.shape) + if not self.fetch_features: + return rel_depth + out = [self.core_out[k] for k in self.layer_names] + + if return_rel_depth: + return rel_depth, out + return out + + def get_rel_pos_params(self): + for name, p in self.core.pretrained.named_parameters(): + if "relative_position" in name: + yield p + + def get_enc_params_except_rel_pos(self): + for name, p in self.core.pretrained.named_parameters(): + if "relative_position" not in name: + yield p + + def freeze_encoder(self, freeze_rel_pos=False): + if freeze_rel_pos: + for p in self.core.pretrained.parameters(): + p.requires_grad = False + else: + for p in self.get_enc_params_except_rel_pos(): + p.requires_grad = False + return self + + def attach_hooks(self, midas): + if len(self.handles) > 0: + self.remove_hooks() + if "out_conv" in self.layer_names: + self.handles.append(list(midas.scratch.output_conv.children())[ + 3].register_forward_hook(get_activation("out_conv", self.core_out))) + if "r4" in self.layer_names: + self.handles.append(midas.scratch.refinenet4.register_forward_hook( + get_activation("r4", self.core_out))) + if "r3" in self.layer_names: + self.handles.append(midas.scratch.refinenet3.register_forward_hook( + get_activation("r3", self.core_out))) + if "r2" in self.layer_names: + self.handles.append(midas.scratch.refinenet2.register_forward_hook( + get_activation("r2", self.core_out))) + if "r1" in self.layer_names: + self.handles.append(midas.scratch.refinenet1.register_forward_hook( + get_activation("r1", self.core_out))) + if "l4_rn" in self.layer_names: + self.handles.append(midas.scratch.layer4_rn.register_forward_hook( + get_activation("l4_rn", self.core_out))) + + return self + + def remove_hooks(self): + for h in self.handles: + h.remove() + return self + + def __del__(self): + self.remove_hooks() + + def set_output_channels(self, model_type): + self.output_channels = MIDAS_SETTINGS[model_type] + + @staticmethod + def build(midas_model_type="DPT_BEiT_L_384", train_midas=False, use_pretrained_midas=True, fetch_features=False, freeze_bn=True, force_keep_ar=False, force_reload=False, **kwargs): + if midas_model_type not in MIDAS_SETTINGS: + raise ValueError( + f"Invalid model type: {midas_model_type}. Must be one of {list(MIDAS_SETTINGS.keys())}") + if "img_size" in kwargs: + kwargs = MidasCore.parse_img_size(kwargs) + img_size = kwargs.pop("img_size", [384, 384]) + print("img_size", img_size) + midas_path = os.path.join(os.path.dirname(__file__), 'midas_repo') + midas = torch.hub.load(midas_path, midas_model_type, + pretrained=use_pretrained_midas, force_reload=force_reload, source='local') + kwargs.update({'keep_aspect_ratio': force_keep_ar}) + midas_core = MidasCore(midas, trainable=train_midas, fetch_features=fetch_features, + freeze_bn=freeze_bn, img_size=img_size, **kwargs) + midas_core.set_output_channels(midas_model_type) + return midas_core + + @staticmethod + def build_from_config(config): + return MidasCore.build(**config) + + @staticmethod + def parse_img_size(config): + assert 'img_size' in config + if isinstance(config['img_size'], str): + assert "," in config['img_size'], "img_size should be a string with comma separated img_size=H,W" + config['img_size'] = list(map(int, config['img_size'].split(","))) + assert len( + config['img_size']) == 2, "img_size should be a string with comma separated img_size=H,W" + elif isinstance(config['img_size'], int): + config['img_size'] = [config['img_size'], config['img_size']] + else: + assert isinstance(config['img_size'], list) and len( + config['img_size']) == 2, "img_size should be a list of H,W" + return config + + +nchannels2models = { + tuple([256]*5): ["DPT_BEiT_L_384", "DPT_BEiT_L_512", "DPT_BEiT_B_384", "DPT_SwinV2_L_384", "DPT_SwinV2_B_384", "DPT_SwinV2_T_256", "DPT_Large", "DPT_Hybrid"], + (512, 256, 128, 64, 64): ["MiDaS_small"] +} + +# Model name to number of output channels +MIDAS_SETTINGS = {m: k for k, v in nchannels2models.items() + for m in v + } diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/Dockerfile b/annotator/zoe/zoedepth/models/base_models/midas_repo/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..466bc94ba3128ea9cbe4bde82bd2fd1fc9daa8af --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/Dockerfile @@ -0,0 +1,29 @@ +# enables cuda support in docker +FROM nvidia/cuda:10.2-cudnn7-runtime-ubuntu18.04 + +# install python 3.6, pip and requirements for opencv-python +# (see https://github.com/NVIDIA/nvidia-docker/issues/864) +RUN apt-get update && apt-get -y install \ + python3 \ + python3-pip \ + libsm6 \ + libxext6 \ + libxrender-dev \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# install python dependencies +RUN pip3 install --upgrade pip +RUN pip3 install torch~=1.8 torchvision opencv-python-headless~=3.4 timm + +# copy inference code +WORKDIR /opt/MiDaS +COPY ./midas ./midas +COPY ./*.py ./ + +# download model weights so the docker image can be used offline +RUN cd weights && {curl -OL https://github.com/isl-org/MiDaS/releases/download/v3/dpt_hybrid_384.pt; cd -; } +RUN python3 run.py --model_type dpt_hybrid; exit 0 + +# entrypoint (dont forget to mount input and output directories) +CMD python3 run.py --model_type dpt_hybrid diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/LICENSE b/annotator/zoe/zoedepth/models/base_models/midas_repo/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..277b5c11be103f028a8d10985139f1da10c2f08e --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Intel ISL (Intel Intelligent Systems Lab) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/README.md b/annotator/zoe/zoedepth/models/base_models/midas_repo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9568ea71c755b6938ee5482ba9f09be722e75943 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/README.md @@ -0,0 +1,259 @@ +## Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer + +This repository contains code to compute depth from a single image. It accompanies our [paper](https://arxiv.org/abs/1907.01341v3): + +>Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer +René Ranftl, Katrin Lasinger, David Hafner, Konrad Schindler, Vladlen Koltun + + +and our [preprint](https://arxiv.org/abs/2103.13413): + +> Vision Transformers for Dense Prediction +> René Ranftl, Alexey Bochkovskiy, Vladlen Koltun + + +MiDaS was trained on up to 12 datasets (ReDWeb, DIML, Movies, MegaDepth, WSVD, TartanAir, HRWSI, ApolloScape, BlendedMVS, IRS, KITTI, NYU Depth V2) with +multi-objective optimization. +The original model that was trained on 5 datasets (`MIX 5` in the paper) can be found [here](https://github.com/isl-org/MiDaS/releases/tag/v2). +The figure below shows an overview of the different MiDaS models; the bubble size scales with number of parameters. + +![](figures/Improvement_vs_FPS.png) + +### Setup + +1) Pick one or more models and download the corresponding weights to the `weights` folder: + +MiDaS 3.1 +- For highest quality: [dpt_beit_large_512](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_beit_large_512.pt) +- For moderately less quality, but better speed-performance trade-off: [dpt_swin2_large_384](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin2_large_384.pt) +- For embedded devices: [dpt_swin2_tiny_256](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin2_tiny_256.pt), [dpt_levit_224](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_levit_224.pt) +- For inference on Intel CPUs, OpenVINO may be used for the small legacy model: openvino_midas_v21_small [.xml](https://github.com/isl-org/MiDaS/releases/download/v3_1/openvino_midas_v21_small_256.xml), [.bin](https://github.com/isl-org/MiDaS/releases/download/v3_1/openvino_midas_v21_small_256.bin) + +MiDaS 3.0: Legacy transformer models [dpt_large_384](https://github.com/isl-org/MiDaS/releases/download/v3/dpt_large_384.pt) and [dpt_hybrid_384](https://github.com/isl-org/MiDaS/releases/download/v3/dpt_hybrid_384.pt) + +MiDaS 2.1: Legacy convolutional models [midas_v21_384](https://github.com/isl-org/MiDaS/releases/download/v2_1/midas_v21_384.pt) and [midas_v21_small_256](https://github.com/isl-org/MiDaS/releases/download/v2_1/midas_v21_small_256.pt) + +1) Set up dependencies: + + ```shell + conda env create -f environment.yaml + conda activate midas-py310 + ``` + +#### optional + +For the Next-ViT model, execute + +```shell +git submodule add https://github.com/isl-org/Next-ViT midas/external/next_vit +``` + +For the OpenVINO model, install + +```shell +pip install openvino +``` + +### Usage + +1) Place one or more input images in the folder `input`. + +2) Run the model with + + ```shell + python run.py --model_type --input_path input --output_path output + ``` + where `````` is chosen from [dpt_beit_large_512](#model_type), [dpt_beit_large_384](#model_type), + [dpt_beit_base_384](#model_type), [dpt_swin2_large_384](#model_type), [dpt_swin2_base_384](#model_type), + [dpt_swin2_tiny_256](#model_type), [dpt_swin_large_384](#model_type), [dpt_next_vit_large_384](#model_type), + [dpt_levit_224](#model_type), [dpt_large_384](#model_type), [dpt_hybrid_384](#model_type), + [midas_v21_384](#model_type), [midas_v21_small_256](#model_type), [openvino_midas_v21_small_256](#model_type). + +3) The resulting depth maps are written to the `output` folder. + +#### optional + +1) By default, the inference resizes the height of input images to the size of a model to fit into the encoder. This + size is given by the numbers in the model names of the [accuracy table](#accuracy). Some models do not only support a single + inference height but a range of different heights. Feel free to explore different heights by appending the extra + command line argument `--height`. Unsupported height values will throw an error. Note that using this argument may + decrease the model accuracy. +2) By default, the inference keeps the aspect ratio of input images when feeding them into the encoder if this is + supported by a model (all models except for Swin, Swin2, LeViT). In order to resize to a square resolution, + disregarding the aspect ratio while preserving the height, use the command line argument `--square`. + +#### via Camera + + If you want the input images to be grabbed from the camera and shown in a window, leave the input and output paths + away and choose a model type as shown above: + + ```shell + python run.py --model_type --side + ``` + + The argument `--side` is optional and causes both the input RGB image and the output depth map to be shown + side-by-side for comparison. + +#### via Docker + +1) Make sure you have installed Docker and the + [NVIDIA Docker runtime](https://github.com/NVIDIA/nvidia-docker/wiki/Installation-\(Native-GPU-Support\)). + +2) Build the Docker image: + + ```shell + docker build -t midas . + ``` + +3) Run inference: + + ```shell + docker run --rm --gpus all -v $PWD/input:/opt/MiDaS/input -v $PWD/output:/opt/MiDaS/output -v $PWD/weights:/opt/MiDaS/weights midas + ``` + + This command passes through all of your NVIDIA GPUs to the container, mounts the + `input` and `output` directories and then runs the inference. + +#### via PyTorch Hub + +The pretrained model is also available on [PyTorch Hub](https://pytorch.org/hub/intelisl_midas_v2/) + +#### via TensorFlow or ONNX + +See [README](https://github.com/isl-org/MiDaS/tree/master/tf) in the `tf` subdirectory. + +Currently only supports MiDaS v2.1. + + +#### via Mobile (iOS / Android) + +See [README](https://github.com/isl-org/MiDaS/tree/master/mobile) in the `mobile` subdirectory. + +#### via ROS1 (Robot Operating System) + +See [README](https://github.com/isl-org/MiDaS/tree/master/ros) in the `ros` subdirectory. + +Currently only supports MiDaS v2.1. DPT-based models to be added. + + +### Accuracy + +We provide a **zero-shot error** $\epsilon_d$ which is evaluated for 6 different datasets +(see [paper](https://arxiv.org/abs/1907.01341v3)). **Lower error values are better**. +$\color{green}{\textsf{Overall model quality is represented by the improvement}}$ ([Imp.](#improvement)) with respect to +MiDaS 3.0 DPTL-384. The models are grouped by the height used for inference, whereas the square training resolution is given by +the numbers in the model names. The table also shows the **number of parameters** (in millions) and the +**frames per second** for inference at the training resolution (for GPU RTX 3090): + +| MiDaS Model | DIW
WHDR | Eth3d
AbsRel | Sintel
AbsRel | TUM
δ1 | KITTI
δ1 | NYUv2
δ1 | $\color{green}{\textsf{Imp.}}$
% | Par.
M | FPS
  | +|-----------------------------------------------------------------------------------------------------------------------|-------------------------:|-----------------------------:|------------------------------:|-------------------------:|-------------------------:|-------------------------:|-------------------------------------------------:|----------------------:|--------------------------:| +| **Inference height 512** | | | | | | | | | | +| [v3.1 BEiTL-512](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_beit_large_512.pt) | 0.1137 | 0.0659 | 0.2366 | **6.13** | 11.56* | **1.86*** | $\color{green}{\textsf{19}}$ | **345** | **5.7** | +| [v3.1 BEiTL-512](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_beit_large_512.pt)$\tiny{\square}$ | **0.1121** | **0.0614** | **0.2090** | 6.46 | **5.00*** | 1.90* | $\color{green}{\textsf{34}}$ | **345** | **5.7** | +| | | | | | | | | | | +| **Inference height 384** | | | | | | | | | | +| [v3.1 BEiTL-512](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_beit_large_512.pt) | 0.1245 | 0.0681 | **0.2176** | **6.13** | 6.28* | **2.16*** | $\color{green}{\textsf{28}}$ | 345 | 12 | +| [v3.1 Swin2L-384](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin2_large_384.pt)$\tiny{\square}$ | 0.1106 | 0.0732 | 0.2442 | 8.87 | **5.84*** | 2.92* | $\color{green}{\textsf{22}}$ | 213 | 41 | +| [v3.1 Swin2B-384](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin2_base_384.pt)$\tiny{\square}$ | 0.1095 | 0.0790 | 0.2404 | 8.93 | 5.97* | 3.28* | $\color{green}{\textsf{22}}$ | 102 | 39 | +| [v3.1 SwinL-384](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin_large_384.pt)$\tiny{\square}$ | 0.1126 | 0.0853 | 0.2428 | 8.74 | 6.60* | 3.34* | $\color{green}{\textsf{17}}$ | 213 | 49 | +| [v3.1 BEiTL-384](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_beit_large_384.pt) | 0.1239 | **0.0667** | 0.2545 | 7.17 | 9.84* | 2.21* | $\color{green}{\textsf{17}}$ | 344 | 13 | +| [v3.1 Next-ViTL-384](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_next_vit_large_384.pt) | **0.1031** | 0.0954 | 0.2295 | 9.21 | 6.89* | 3.47* | $\color{green}{\textsf{16}}$ | **72** | 30 | +| [v3.1 BEiTB-384](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_beit_base_384.pt) | 0.1159 | 0.0967 | 0.2901 | 9.88 | 26.60* | 3.91* | $\color{green}{\textsf{-31}}$ | 112 | 31 | +| [v3.0 DPTL-384](https://github.com/isl-org/MiDaS/releases/download/v3/dpt_large_384.pt) | 0.1082 | 0.0888 | 0.2697 | 9.97 | 8.46 | 8.32 | $\color{green}{\textsf{0}}$ | 344 | **61** | +| [v3.0 DPTH-384](https://github.com/isl-org/MiDaS/releases/download/v3/dpt_hybrid_384.pt) | 0.1106 | 0.0934 | 0.2741 | 10.89 | 11.56 | 8.69 | $\color{green}{\textsf{-10}}$ | 123 | 50 | +| [v2.1 Large384](https://github.com/isl-org/MiDaS/releases/download/v2_1/midas_v21_384.pt) | 0.1295 | 0.1155 | 0.3285 | 12.51 | 16.08 | 8.71 | $\color{green}{\textsf{-32}}$ | 105 | 47 | +| | | | | | | | | | | +| **Inference height 256** | | | | | | | | | | +| [v3.1 Swin2T-256](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin2_tiny_256.pt)$\tiny{\square}$ | **0.1211** | **0.1106** | **0.2868** | **13.43** | **10.13*** | **5.55*** | $\color{green}{\textsf{-11}}$ | 42 | 64 | +| [v2.1 Small256](https://github.com/isl-org/MiDaS/releases/download/v2_1/midas_v21_small_256.pt) | 0.1344 | 0.1344 | 0.3370 | 14.53 | 29.27 | 13.43 | $\color{green}{\textsf{-76}}$ | **21** | **90** | +| | | | | | | | | | | +| **Inference height 224** | | | | | | | | | | +| [v3.1 LeViT224](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_levit_224.pt)$\tiny{\square}$ | **0.1314** | **0.1206** | **0.3148** | **18.21** | **15.27*** | **8.64*** | $\color{green}{\textsf{-40}}$ | **51** | **73** | + +* No zero-shot error, because models are also trained on KITTI and NYU Depth V2\ +$\square$ Validation performed at **square resolution**, either because the transformer encoder backbone of a model +does not support non-square resolutions (Swin, Swin2, LeViT) or for comparison with these models. All other +validations keep the aspect ratio. A difference in resolution limits the comparability of the zero-shot error and the +improvement, because these quantities are averages over the pixels of an image and do not take into account the +advantage of more details due to a higher resolution.\ +Best values per column and same validation height in bold + +#### Improvement + +The improvement in the above table is defined as the relative zero-shot error with respect to MiDaS v3.0 +DPTL-384 and averaging over the datasets. So, if $\epsilon_d$ is the zero-shot error for dataset $d$, then +the $\color{green}{\textsf{improvement}}$ is given by $100(1-(1/6)\sum_d\epsilon_d/\epsilon_{d,\rm{DPT_{L-384}}})$%. + +Note that the improvements of 10% for MiDaS v2.0 → v2.1 and 21% for MiDaS v2.1 → v3.0 are not visible from the +improvement column (Imp.) in the table but would require an evaluation with respect to MiDaS v2.1 Large384 +and v2.0 Large384 respectively instead of v3.0 DPTL-384. + +### Depth map comparison + +Zoom in for better visibility +![](figures/Comparison.png) + +### Speed on Camera Feed + +Test configuration +- Windows 10 +- 11th Gen Intel Core i7-1185G7 3.00GHz +- 16GB RAM +- Camera resolution 640x480 +- openvino_midas_v21_small_256 + +Speed: 22 FPS + +### Changelog + +* [Dec 2022] Released MiDaS v3.1: + - New models based on 5 different types of transformers ([BEiT](https://arxiv.org/pdf/2106.08254.pdf), [Swin2](https://arxiv.org/pdf/2111.09883.pdf), [Swin](https://arxiv.org/pdf/2103.14030.pdf), [Next-ViT](https://arxiv.org/pdf/2207.05501.pdf), [LeViT](https://arxiv.org/pdf/2104.01136.pdf)) + - Training datasets extended from 10 to 12, including also KITTI and NYU Depth V2 using [BTS](https://github.com/cleinc/bts) split + - Best model, BEiTLarge 512, with resolution 512x512, is on average about [28% more accurate](#Accuracy) than MiDaS v3.0 + - Integrated live depth estimation from camera feed +* [Sep 2021] Integrated to [Huggingface Spaces](https://huggingface.co./spaces) with [Gradio](https://github.com/gradio-app/gradio). See [Gradio Web Demo](https://huggingface.co./spaces/akhaliq/DPT-Large). +* [Apr 2021] Released MiDaS v3.0: + - New models based on [Dense Prediction Transformers](https://arxiv.org/abs/2103.13413) are on average [21% more accurate](#Accuracy) than MiDaS v2.1 + - Additional models can be found [here](https://github.com/isl-org/DPT) +* [Nov 2020] Released MiDaS v2.1: + - New model that was trained on 10 datasets and is on average about [10% more accurate](#Accuracy) than [MiDaS v2.0](https://github.com/isl-org/MiDaS/releases/tag/v2) + - New light-weight model that achieves [real-time performance](https://github.com/isl-org/MiDaS/tree/master/mobile) on mobile platforms. + - Sample applications for [iOS](https://github.com/isl-org/MiDaS/tree/master/mobile/ios) and [Android](https://github.com/isl-org/MiDaS/tree/master/mobile/android) + - [ROS package](https://github.com/isl-org/MiDaS/tree/master/ros) for easy deployment on robots +* [Jul 2020] Added TensorFlow and ONNX code. Added [online demo](http://35.202.76.57/). +* [Dec 2019] Released new version of MiDaS - the new model is significantly more accurate and robust +* [Jul 2019] Initial release of MiDaS ([Link](https://github.com/isl-org/MiDaS/releases/tag/v1)) + +### Citation + +Please cite our paper if you use this code or any of the models: +``` +@ARTICLE {Ranftl2022, + author = "Ren\'{e} Ranftl and Katrin Lasinger and David Hafner and Konrad Schindler and Vladlen Koltun", + title = "Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-Shot Cross-Dataset Transfer", + journal = "IEEE Transactions on Pattern Analysis and Machine Intelligence", + year = "2022", + volume = "44", + number = "3" +} +``` + +If you use a DPT-based model, please also cite: + +``` +@article{Ranftl2021, + author = {Ren\'{e} Ranftl and Alexey Bochkovskiy and Vladlen Koltun}, + title = {Vision Transformers for Dense Prediction}, + journal = {ICCV}, + year = {2021}, +} +``` + +### Acknowledgements + +Our work builds on and uses code from [timm](https://github.com/rwightman/pytorch-image-models) and [Next-ViT](https://github.com/bytedance/Next-ViT). +We'd like to thank the authors for making these libraries available. + +### License + +MIT License diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/environment.yaml b/annotator/zoe/zoedepth/models/base_models/midas_repo/environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9abe5693b9e0de56b7d20728f4d0e6333c5822d --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/environment.yaml @@ -0,0 +1,16 @@ +name: midas-py310 +channels: + - pytorch + - defaults +dependencies: + - nvidia::cudatoolkit=11.7 + - python=3.10.8 + - pytorch::pytorch=1.13.0 + - torchvision=0.14.0 + - pip=22.3.1 + - numpy=1.23.4 + - pip: + - opencv-python==4.6.0.66 + - imutils==0.5.4 + - timm==0.6.12 + - einops==0.6.0 \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/hubconf.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/hubconf.py new file mode 100644 index 0000000000000000000000000000000000000000..0d638be5151c4e305daff0c47d1ea3fc8066377d --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/hubconf.py @@ -0,0 +1,435 @@ +dependencies = ["torch"] + +import torch + +from midas.dpt_depth import DPTDepthModel +from midas.midas_net import MidasNet +from midas.midas_net_custom import MidasNet_small + +def DPT_BEiT_L_512(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT_BEiT_L_512 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="beitl16_512", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_beit_large_512.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_BEiT_L_384(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT_BEiT_L_384 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="beitl16_384", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_beit_large_384.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_BEiT_B_384(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT_BEiT_B_384 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="beitb16_384", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_beit_base_384.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_SwinV2_L_384(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT_SwinV2_L_384 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="swin2l24_384", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin2_large_384.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_SwinV2_B_384(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT_SwinV2_B_384 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="swin2b24_384", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin2_base_384.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_SwinV2_T_256(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT_SwinV2_T_256 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="swin2t16_256", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin2_tiny_256.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_Swin_L_384(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT_Swin_L_384 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="swinl12_384", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin_large_384.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_Next_ViT_L_384(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT_Next_ViT_L_384 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="next_vit_large_6m", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_next_vit_large_384.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_LeViT_224(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT_LeViT_224 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="levit_384", + non_negative=True, + head_features_1=64, + head_features_2=8, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_levit_224.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_Large(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT-Large model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="vitl16_384", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3/dpt_large_384.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_Hybrid(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT-Hybrid model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="vitb_rn50_384", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3/dpt_hybrid_384.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def MiDaS(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS v2.1 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = MidasNet() + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v2_1/midas_v21_384.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def MiDaS_small(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS v2.1 small model for monocular depth estimation on resource-constrained devices + pretrained (bool): load pretrained weights into model + """ + + model = MidasNet_small(None, features=64, backbone="efficientnet_lite3", exportable=True, non_negative=True, blocks={'expand': True}) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v2_1/midas_v21_small_256.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + + +def transforms(): + import cv2 + from torchvision.transforms import Compose + from midas.transforms import Resize, NormalizeImage, PrepareForNet + from midas import transforms + + transforms.default_transform = Compose( + [ + lambda img: {"image": img / 255.0}, + Resize( + 384, + 384, + resize_target=None, + keep_aspect_ratio=True, + ensure_multiple_of=32, + resize_method="upper_bound", + image_interpolation_method=cv2.INTER_CUBIC, + ), + NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + PrepareForNet(), + lambda sample: torch.from_numpy(sample["image"]).unsqueeze(0), + ] + ) + + transforms.small_transform = Compose( + [ + lambda img: {"image": img / 255.0}, + Resize( + 256, + 256, + resize_target=None, + keep_aspect_ratio=True, + ensure_multiple_of=32, + resize_method="upper_bound", + image_interpolation_method=cv2.INTER_CUBIC, + ), + NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + PrepareForNet(), + lambda sample: torch.from_numpy(sample["image"]).unsqueeze(0), + ] + ) + + transforms.dpt_transform = Compose( + [ + lambda img: {"image": img / 255.0}, + Resize( + 384, + 384, + resize_target=None, + keep_aspect_ratio=True, + ensure_multiple_of=32, + resize_method="minimal", + image_interpolation_method=cv2.INTER_CUBIC, + ), + NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), + PrepareForNet(), + lambda sample: torch.from_numpy(sample["image"]).unsqueeze(0), + ] + ) + + transforms.beit512_transform = Compose( + [ + lambda img: {"image": img / 255.0}, + Resize( + 512, + 512, + resize_target=None, + keep_aspect_ratio=True, + ensure_multiple_of=32, + resize_method="minimal", + image_interpolation_method=cv2.INTER_CUBIC, + ), + NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), + PrepareForNet(), + lambda sample: torch.from_numpy(sample["image"]).unsqueeze(0), + ] + ) + + transforms.swin384_transform = Compose( + [ + lambda img: {"image": img / 255.0}, + Resize( + 384, + 384, + resize_target=None, + keep_aspect_ratio=False, + ensure_multiple_of=32, + resize_method="minimal", + image_interpolation_method=cv2.INTER_CUBIC, + ), + NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), + PrepareForNet(), + lambda sample: torch.from_numpy(sample["image"]).unsqueeze(0), + ] + ) + + transforms.swin256_transform = Compose( + [ + lambda img: {"image": img / 255.0}, + Resize( + 256, + 256, + resize_target=None, + keep_aspect_ratio=False, + ensure_multiple_of=32, + resize_method="minimal", + image_interpolation_method=cv2.INTER_CUBIC, + ), + NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), + PrepareForNet(), + lambda sample: torch.from_numpy(sample["image"]).unsqueeze(0), + ] + ) + + transforms.levit_transform = Compose( + [ + lambda img: {"image": img / 255.0}, + Resize( + 224, + 224, + resize_target=None, + keep_aspect_ratio=False, + ensure_multiple_of=32, + resize_method="minimal", + image_interpolation_method=cv2.INTER_CUBIC, + ), + NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), + PrepareForNet(), + lambda sample: torch.from_numpy(sample["image"]).unsqueeze(0), + ] + ) + + return transforms diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/beit.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/beit.py new file mode 100644 index 0000000000000000000000000000000000000000..7a24e02cd2b979844bf638b46ac60949ee9ce691 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/beit.py @@ -0,0 +1,196 @@ +import timm +import torch +import types + +import numpy as np +import torch.nn.functional as F + +from .utils import forward_adapted_unflatten, make_backbone_default +from timm.models.beit import gen_relative_position_index +from torch.utils.checkpoint import checkpoint +from typing import Optional + + +def forward_beit(pretrained, x): + return forward_adapted_unflatten(pretrained, x, "forward_features") + + +def patch_embed_forward(self, x): + """ + Modification of timm.models.layers.patch_embed.py: PatchEmbed.forward to support arbitrary window sizes. + """ + x = self.proj(x) + if self.flatten: + x = x.flatten(2).transpose(1, 2) + x = self.norm(x) + return x + + +def _get_rel_pos_bias(self, window_size): + """ + Modification of timm.models.beit.py: Attention._get_rel_pos_bias to support arbitrary window sizes. + """ + old_height = 2 * self.window_size[0] - 1 + old_width = 2 * self.window_size[1] - 1 + + new_height = 2 * window_size[0] - 1 + new_width = 2 * window_size[1] - 1 + + old_relative_position_bias_table = self.relative_position_bias_table + + old_num_relative_distance = self.num_relative_distance + new_num_relative_distance = new_height * new_width + 3 + + old_sub_table = old_relative_position_bias_table[:old_num_relative_distance - 3] + + old_sub_table = old_sub_table.reshape(1, old_width, old_height, -1).permute(0, 3, 1, 2) + new_sub_table = F.interpolate(old_sub_table, size=(new_height, new_width), mode="bilinear") + new_sub_table = new_sub_table.permute(0, 2, 3, 1).reshape(new_num_relative_distance - 3, -1) + + new_relative_position_bias_table = torch.cat( + [new_sub_table, old_relative_position_bias_table[old_num_relative_distance - 3:]]) + + key = str(window_size[1]) + "," + str(window_size[0]) + if key not in self.relative_position_indices.keys(): + self.relative_position_indices[key] = gen_relative_position_index(window_size) + + relative_position_bias = new_relative_position_bias_table[ + self.relative_position_indices[key].view(-1)].view( + window_size[0] * window_size[1] + 1, + window_size[0] * window_size[1] + 1, -1) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + return relative_position_bias.unsqueeze(0) + + +def attention_forward(self, x, resolution, shared_rel_pos_bias: Optional[torch.Tensor] = None): + """ + Modification of timm.models.beit.py: Attention.forward to support arbitrary window sizes. + """ + B, N, C = x.shape + + qkv_bias = torch.cat((self.q_bias, self.k_bias, self.v_bias)) if self.q_bias is not None else None + qkv = F.linear(input=x, weight=self.qkv.weight, bias=qkv_bias) + qkv = qkv.reshape(B, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + q, k, v = qkv.unbind(0) # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) + + if self.relative_position_bias_table is not None: + window_size = tuple(np.array(resolution) // 16) + attn = attn + self._get_rel_pos_bias(window_size) + if shared_rel_pos_bias is not None: + attn = attn + shared_rel_pos_bias + + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, -1) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +def block_forward(self, x, resolution, shared_rel_pos_bias: Optional[torch.Tensor] = None): + """ + Modification of timm.models.beit.py: Block.forward to support arbitrary window sizes. + """ + if self.gamma_1 is None: + x = x + self.drop_path(self.attn(self.norm1(x), resolution, shared_rel_pos_bias=shared_rel_pos_bias)) + x = x + self.drop_path(self.mlp(self.norm2(x))) + else: + x = x + self.drop_path(self.gamma_1 * self.attn(self.norm1(x), resolution, + shared_rel_pos_bias=shared_rel_pos_bias)) + x = x + self.drop_path(self.gamma_2 * self.mlp(self.norm2(x))) + return x + + +def beit_forward_features(self, x): + """ + Modification of timm.models.beit.py: Beit.forward_features to support arbitrary window sizes. + """ + resolution = x.shape[2:] + + x = self.patch_embed(x) + x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1) + if self.pos_embed is not None: + x = x + self.pos_embed + x = self.pos_drop(x) + + rel_pos_bias = self.rel_pos_bias() if self.rel_pos_bias is not None else None + for blk in self.blocks: + if self.grad_checkpointing and not torch.jit.is_scripting(): + x = checkpoint(blk, x, shared_rel_pos_bias=rel_pos_bias) + else: + x = blk(x, resolution, shared_rel_pos_bias=rel_pos_bias) + x = self.norm(x) + return x + + +def _make_beit_backbone( + model, + features=[96, 192, 384, 768], + size=[384, 384], + hooks=[0, 4, 8, 11], + vit_features=768, + use_readout="ignore", + start_index=1, + start_index_readout=1, +): + backbone = make_backbone_default(model, features, size, hooks, vit_features, use_readout, start_index, + start_index_readout) + + backbone.model.patch_embed.forward = types.MethodType(patch_embed_forward, backbone.model.patch_embed) + backbone.model.forward_features = types.MethodType(beit_forward_features, backbone.model) + + for block in backbone.model.blocks: + attn = block.attn + attn._get_rel_pos_bias = types.MethodType(_get_rel_pos_bias, attn) + attn.forward = types.MethodType(attention_forward, attn) + attn.relative_position_indices = {} + + block.forward = types.MethodType(block_forward, block) + + return backbone + + +def _make_pretrained_beitl16_512(pretrained, use_readout="ignore", hooks=None): + model = timm.create_model("beit_large_patch16_512", pretrained=pretrained) + + hooks = [5, 11, 17, 23] if hooks is None else hooks + + features = [256, 512, 1024, 1024] + + return _make_beit_backbone( + model, + features=features, + size=[512, 512], + hooks=hooks, + vit_features=1024, + use_readout=use_readout, + ) + + +def _make_pretrained_beitl16_384(pretrained, use_readout="ignore", hooks=None): + model = timm.create_model("beit_large_patch16_384", pretrained=pretrained) + + hooks = [5, 11, 17, 23] if hooks is None else hooks + return _make_beit_backbone( + model, + features=[256, 512, 1024, 1024], + hooks=hooks, + vit_features=1024, + use_readout=use_readout, + ) + + +def _make_pretrained_beitb16_384(pretrained, use_readout="ignore", hooks=None): + model = timm.create_model("beit_base_patch16_384", pretrained=pretrained) + + hooks = [2, 5, 8, 11] if hooks is None else hooks + return _make_beit_backbone( + model, + features=[96, 192, 384, 768], + hooks=hooks, + use_readout=use_readout, + ) diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/levit.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/levit.py new file mode 100644 index 0000000000000000000000000000000000000000..6d023a98702a0451806d26f33f8bccf931814f10 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/levit.py @@ -0,0 +1,106 @@ +import timm +import torch +import torch.nn as nn +import numpy as np + +from .utils import activations, get_activation, Transpose + + +def forward_levit(pretrained, x): + pretrained.model.forward_features(x) + + layer_1 = pretrained.activations["1"] + layer_2 = pretrained.activations["2"] + layer_3 = pretrained.activations["3"] + + layer_1 = pretrained.act_postprocess1(layer_1) + layer_2 = pretrained.act_postprocess2(layer_2) + layer_3 = pretrained.act_postprocess3(layer_3) + + return layer_1, layer_2, layer_3 + + +def _make_levit_backbone( + model, + hooks=[3, 11, 21], + patch_grid=[14, 14] +): + pretrained = nn.Module() + + pretrained.model = model + pretrained.model.blocks[hooks[0]].register_forward_hook(get_activation("1")) + pretrained.model.blocks[hooks[1]].register_forward_hook(get_activation("2")) + pretrained.model.blocks[hooks[2]].register_forward_hook(get_activation("3")) + + pretrained.activations = activations + + patch_grid_size = np.array(patch_grid, dtype=int) + + pretrained.act_postprocess1 = nn.Sequential( + Transpose(1, 2), + nn.Unflatten(2, torch.Size(patch_grid_size.tolist())) + ) + pretrained.act_postprocess2 = nn.Sequential( + Transpose(1, 2), + nn.Unflatten(2, torch.Size((np.ceil(patch_grid_size / 2).astype(int)).tolist())) + ) + pretrained.act_postprocess3 = nn.Sequential( + Transpose(1, 2), + nn.Unflatten(2, torch.Size((np.ceil(patch_grid_size / 4).astype(int)).tolist())) + ) + + return pretrained + + +class ConvTransposeNorm(nn.Sequential): + """ + Modification of + https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/levit.py: ConvNorm + such that ConvTranspose2d is used instead of Conv2d. + """ + + def __init__( + self, in_chs, out_chs, kernel_size=1, stride=1, pad=0, dilation=1, + groups=1, bn_weight_init=1): + super().__init__() + self.add_module('c', + nn.ConvTranspose2d(in_chs, out_chs, kernel_size, stride, pad, dilation, groups, bias=False)) + self.add_module('bn', nn.BatchNorm2d(out_chs)) + + nn.init.constant_(self.bn.weight, bn_weight_init) + + @torch.no_grad() + def fuse(self): + c, bn = self._modules.values() + w = bn.weight / (bn.running_var + bn.eps) ** 0.5 + w = c.weight * w[:, None, None, None] + b = bn.bias - bn.running_mean * bn.weight / (bn.running_var + bn.eps) ** 0.5 + m = nn.ConvTranspose2d( + w.size(1), w.size(0), w.shape[2:], stride=self.c.stride, + padding=self.c.padding, dilation=self.c.dilation, groups=self.c.groups) + m.weight.data.copy_(w) + m.bias.data.copy_(b) + return m + + +def stem_b4_transpose(in_chs, out_chs, activation): + """ + Modification of + https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/levit.py: stem_b16 + such that ConvTranspose2d is used instead of Conv2d and stem is also reduced to the half. + """ + return nn.Sequential( + ConvTransposeNorm(in_chs, out_chs, 3, 2, 1), + activation(), + ConvTransposeNorm(out_chs, out_chs // 2, 3, 2, 1), + activation()) + + +def _make_pretrained_levit_384(pretrained, hooks=None): + model = timm.create_model("levit_384", pretrained=pretrained) + + hooks = [3, 11, 21] if hooks == None else hooks + return _make_levit_backbone( + model, + hooks=hooks + ) diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/next_vit.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/next_vit.py new file mode 100644 index 0000000000000000000000000000000000000000..8afdd8b743b5ab023a359dc3b721e601b1a40d11 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/next_vit.py @@ -0,0 +1,39 @@ +import timm + +import torch.nn as nn + +from pathlib import Path +from .utils import activations, forward_default, get_activation + +from ..external.next_vit.classification.nextvit import * + + +def forward_next_vit(pretrained, x): + return forward_default(pretrained, x, "forward") + + +def _make_next_vit_backbone( + model, + hooks=[2, 6, 36, 39], +): + pretrained = nn.Module() + + pretrained.model = model + pretrained.model.features[hooks[0]].register_forward_hook(get_activation("1")) + pretrained.model.features[hooks[1]].register_forward_hook(get_activation("2")) + pretrained.model.features[hooks[2]].register_forward_hook(get_activation("3")) + pretrained.model.features[hooks[3]].register_forward_hook(get_activation("4")) + + pretrained.activations = activations + + return pretrained + + +def _make_pretrained_next_vit_large_6m(hooks=None): + model = timm.create_model("nextvit_large") + + hooks = [2, 6, 36, 39] if hooks == None else hooks + return _make_next_vit_backbone( + model, + hooks=hooks, + ) diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/swin.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/swin.py new file mode 100644 index 0000000000000000000000000000000000000000..f8c71367e3e78b087f80b2ab3e2f495a9c372f1a --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/swin.py @@ -0,0 +1,13 @@ +import timm + +from .swin_common import _make_swin_backbone + + +def _make_pretrained_swinl12_384(pretrained, hooks=None): + model = timm.create_model("swin_large_patch4_window12_384", pretrained=pretrained) + + hooks = [1, 1, 17, 1] if hooks == None else hooks + return _make_swin_backbone( + model, + hooks=hooks + ) diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/swin2.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/swin2.py new file mode 100644 index 0000000000000000000000000000000000000000..ce4c8f1d6fc1807a207dc6b9a261c6f7b14a87a3 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/swin2.py @@ -0,0 +1,34 @@ +import timm + +from .swin_common import _make_swin_backbone + + +def _make_pretrained_swin2l24_384(pretrained, hooks=None): + model = timm.create_model("swinv2_large_window12to24_192to384_22kft1k", pretrained=pretrained) + + hooks = [1, 1, 17, 1] if hooks == None else hooks + return _make_swin_backbone( + model, + hooks=hooks + ) + + +def _make_pretrained_swin2b24_384(pretrained, hooks=None): + model = timm.create_model("swinv2_base_window12to24_192to384_22kft1k", pretrained=pretrained) + + hooks = [1, 1, 17, 1] if hooks == None else hooks + return _make_swin_backbone( + model, + hooks=hooks + ) + + +def _make_pretrained_swin2t16_256(pretrained, hooks=None): + model = timm.create_model("swinv2_tiny_window16_256", pretrained=pretrained) + + hooks = [1, 1, 5, 1] if hooks == None else hooks + return _make_swin_backbone( + model, + hooks=hooks, + patch_grid=[64, 64] + ) diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/swin_common.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/swin_common.py new file mode 100644 index 0000000000000000000000000000000000000000..94d63d408f18511179d90b3ac6f697385d1e556d --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/swin_common.py @@ -0,0 +1,52 @@ +import torch + +import torch.nn as nn +import numpy as np + +from .utils import activations, forward_default, get_activation, Transpose + + +def forward_swin(pretrained, x): + return forward_default(pretrained, x) + + +def _make_swin_backbone( + model, + hooks=[1, 1, 17, 1], + patch_grid=[96, 96] +): + pretrained = nn.Module() + + pretrained.model = model + pretrained.model.layers[0].blocks[hooks[0]].register_forward_hook(get_activation("1")) + pretrained.model.layers[1].blocks[hooks[1]].register_forward_hook(get_activation("2")) + pretrained.model.layers[2].blocks[hooks[2]].register_forward_hook(get_activation("3")) + pretrained.model.layers[3].blocks[hooks[3]].register_forward_hook(get_activation("4")) + + pretrained.activations = activations + + if hasattr(model, "patch_grid"): + used_patch_grid = model.patch_grid + else: + used_patch_grid = patch_grid + + patch_grid_size = np.array(used_patch_grid, dtype=int) + + pretrained.act_postprocess1 = nn.Sequential( + Transpose(1, 2), + nn.Unflatten(2, torch.Size(patch_grid_size.tolist())) + ) + pretrained.act_postprocess2 = nn.Sequential( + Transpose(1, 2), + nn.Unflatten(2, torch.Size((patch_grid_size // 2).tolist())) + ) + pretrained.act_postprocess3 = nn.Sequential( + Transpose(1, 2), + nn.Unflatten(2, torch.Size((patch_grid_size // 4).tolist())) + ) + pretrained.act_postprocess4 = nn.Sequential( + Transpose(1, 2), + nn.Unflatten(2, torch.Size((patch_grid_size // 8).tolist())) + ) + + return pretrained diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/utils.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0558899dddcfccec5f01a764d4f21738eb612149 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/utils.py @@ -0,0 +1,249 @@ +import torch + +import torch.nn as nn + + +class Slice(nn.Module): + def __init__(self, start_index=1): + super(Slice, self).__init__() + self.start_index = start_index + + def forward(self, x): + return x[:, self.start_index:] + + +class AddReadout(nn.Module): + def __init__(self, start_index=1): + super(AddReadout, self).__init__() + self.start_index = start_index + + def forward(self, x): + if self.start_index == 2: + readout = (x[:, 0] + x[:, 1]) / 2 + else: + readout = x[:, 0] + return x[:, self.start_index:] + readout.unsqueeze(1) + + +class ProjectReadout(nn.Module): + def __init__(self, in_features, start_index=1): + super(ProjectReadout, self).__init__() + self.start_index = start_index + + self.project = nn.Sequential(nn.Linear(2 * in_features, in_features), nn.GELU()) + + def forward(self, x): + readout = x[:, 0].unsqueeze(1).expand_as(x[:, self.start_index:]) + features = torch.cat((x[:, self.start_index:], readout), -1) + + return self.project(features) + + +class Transpose(nn.Module): + def __init__(self, dim0, dim1): + super(Transpose, self).__init__() + self.dim0 = dim0 + self.dim1 = dim1 + + def forward(self, x): + x = x.transpose(self.dim0, self.dim1) + return x + + +activations = {} + + +def get_activation(name): + def hook(model, input, output): + activations[name] = output + + return hook + + +def forward_default(pretrained, x, function_name="forward_features"): + exec(f"pretrained.model.{function_name}(x)") + + layer_1 = pretrained.activations["1"] + layer_2 = pretrained.activations["2"] + layer_3 = pretrained.activations["3"] + layer_4 = pretrained.activations["4"] + + if hasattr(pretrained, "act_postprocess1"): + layer_1 = pretrained.act_postprocess1(layer_1) + if hasattr(pretrained, "act_postprocess2"): + layer_2 = pretrained.act_postprocess2(layer_2) + if hasattr(pretrained, "act_postprocess3"): + layer_3 = pretrained.act_postprocess3(layer_3) + if hasattr(pretrained, "act_postprocess4"): + layer_4 = pretrained.act_postprocess4(layer_4) + + return layer_1, layer_2, layer_3, layer_4 + + +def forward_adapted_unflatten(pretrained, x, function_name="forward_features"): + b, c, h, w = x.shape + + exec(f"glob = pretrained.model.{function_name}(x)") + + layer_1 = pretrained.activations["1"] + layer_2 = pretrained.activations["2"] + layer_3 = pretrained.activations["3"] + layer_4 = pretrained.activations["4"] + + layer_1 = pretrained.act_postprocess1[0:2](layer_1) + layer_2 = pretrained.act_postprocess2[0:2](layer_2) + layer_3 = pretrained.act_postprocess3[0:2](layer_3) + layer_4 = pretrained.act_postprocess4[0:2](layer_4) + + unflatten = nn.Sequential( + nn.Unflatten( + 2, + torch.Size( + [ + h // pretrained.model.patch_size[1], + w // pretrained.model.patch_size[0], + ] + ), + ) + ) + + if layer_1.ndim == 3: + layer_1 = unflatten(layer_1) + if layer_2.ndim == 3: + layer_2 = unflatten(layer_2) + if layer_3.ndim == 3: + layer_3 = unflatten(layer_3) + if layer_4.ndim == 3: + layer_4 = unflatten(layer_4) + + layer_1 = pretrained.act_postprocess1[3: len(pretrained.act_postprocess1)](layer_1) + layer_2 = pretrained.act_postprocess2[3: len(pretrained.act_postprocess2)](layer_2) + layer_3 = pretrained.act_postprocess3[3: len(pretrained.act_postprocess3)](layer_3) + layer_4 = pretrained.act_postprocess4[3: len(pretrained.act_postprocess4)](layer_4) + + return layer_1, layer_2, layer_3, layer_4 + + +def get_readout_oper(vit_features, features, use_readout, start_index=1): + if use_readout == "ignore": + readout_oper = [Slice(start_index)] * len(features) + elif use_readout == "add": + readout_oper = [AddReadout(start_index)] * len(features) + elif use_readout == "project": + readout_oper = [ + ProjectReadout(vit_features, start_index) for out_feat in features + ] + else: + assert ( + False + ), "wrong operation for readout token, use_readout can be 'ignore', 'add', or 'project'" + + return readout_oper + + +def make_backbone_default( + model, + features=[96, 192, 384, 768], + size=[384, 384], + hooks=[2, 5, 8, 11], + vit_features=768, + use_readout="ignore", + start_index=1, + start_index_readout=1, +): + pretrained = nn.Module() + + pretrained.model = model + pretrained.model.blocks[hooks[0]].register_forward_hook(get_activation("1")) + pretrained.model.blocks[hooks[1]].register_forward_hook(get_activation("2")) + pretrained.model.blocks[hooks[2]].register_forward_hook(get_activation("3")) + pretrained.model.blocks[hooks[3]].register_forward_hook(get_activation("4")) + + pretrained.activations = activations + + readout_oper = get_readout_oper(vit_features, features, use_readout, start_index_readout) + + # 32, 48, 136, 384 + pretrained.act_postprocess1 = nn.Sequential( + readout_oper[0], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[0], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[0], + out_channels=features[0], + kernel_size=4, + stride=4, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess2 = nn.Sequential( + readout_oper[1], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[1], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[1], + out_channels=features[1], + kernel_size=2, + stride=2, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess3 = nn.Sequential( + readout_oper[2], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[2], + kernel_size=1, + stride=1, + padding=0, + ), + ) + + pretrained.act_postprocess4 = nn.Sequential( + readout_oper[3], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[3], + kernel_size=1, + stride=1, + padding=0, + ), + nn.Conv2d( + in_channels=features[3], + out_channels=features[3], + kernel_size=3, + stride=2, + padding=1, + ), + ) + + pretrained.model.start_index = start_index + pretrained.model.patch_size = [16, 16] + + return pretrained diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/vit.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/vit.py new file mode 100644 index 0000000000000000000000000000000000000000..413f9693bd4548342280e329c9128c1a52cea920 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/vit.py @@ -0,0 +1,221 @@ +import torch +import torch.nn as nn +import timm +import types +import math +import torch.nn.functional as F + +from .utils import (activations, forward_adapted_unflatten, get_activation, get_readout_oper, + make_backbone_default, Transpose) + + +def forward_vit(pretrained, x): + return forward_adapted_unflatten(pretrained, x, "forward_flex") + + +def _resize_pos_embed(self, posemb, gs_h, gs_w): + posemb_tok, posemb_grid = ( + posemb[:, : self.start_index], + posemb[0, self.start_index:], + ) + + gs_old = int(math.sqrt(len(posemb_grid))) + + posemb_grid = posemb_grid.reshape(1, gs_old, gs_old, -1).permute(0, 3, 1, 2) + posemb_grid = F.interpolate(posemb_grid, size=(gs_h, gs_w), mode="bilinear") + posemb_grid = posemb_grid.permute(0, 2, 3, 1).reshape(1, gs_h * gs_w, -1) + + posemb = torch.cat([posemb_tok, posemb_grid], dim=1) + + return posemb + + +def forward_flex(self, x): + b, c, h, w = x.shape + + pos_embed = self._resize_pos_embed( + self.pos_embed, h // self.patch_size[1], w // self.patch_size[0] + ) + + B = x.shape[0] + + if hasattr(self.patch_embed, "backbone"): + x = self.patch_embed.backbone(x) + if isinstance(x, (list, tuple)): + x = x[-1] # last feature if backbone outputs list/tuple of features + + x = self.patch_embed.proj(x).flatten(2).transpose(1, 2) + + if getattr(self, "dist_token", None) is not None: + cls_tokens = self.cls_token.expand( + B, -1, -1 + ) # stole cls_tokens impl from Phil Wang, thanks + dist_token = self.dist_token.expand(B, -1, -1) + x = torch.cat((cls_tokens, dist_token, x), dim=1) + else: + if self.no_embed_class: + x = x + pos_embed + cls_tokens = self.cls_token.expand( + B, -1, -1 + ) # stole cls_tokens impl from Phil Wang, thanks + x = torch.cat((cls_tokens, x), dim=1) + + if not self.no_embed_class: + x = x + pos_embed + x = self.pos_drop(x) + + for blk in self.blocks: + x = blk(x) + + x = self.norm(x) + + return x + + +def _make_vit_b16_backbone( + model, + features=[96, 192, 384, 768], + size=[384, 384], + hooks=[2, 5, 8, 11], + vit_features=768, + use_readout="ignore", + start_index=1, + start_index_readout=1, +): + pretrained = make_backbone_default(model, features, size, hooks, vit_features, use_readout, start_index, + start_index_readout) + + # We inject this function into the VisionTransformer instances so that + # we can use it with interpolated position embeddings without modifying the library source. + pretrained.model.forward_flex = types.MethodType(forward_flex, pretrained.model) + pretrained.model._resize_pos_embed = types.MethodType( + _resize_pos_embed, pretrained.model + ) + + return pretrained + + +def _make_pretrained_vitl16_384(pretrained, use_readout="ignore", hooks=None): + model = timm.create_model("vit_large_patch16_384", pretrained=pretrained) + + hooks = [5, 11, 17, 23] if hooks == None else hooks + return _make_vit_b16_backbone( + model, + features=[256, 512, 1024, 1024], + hooks=hooks, + vit_features=1024, + use_readout=use_readout, + ) + + +def _make_pretrained_vitb16_384(pretrained, use_readout="ignore", hooks=None): + model = timm.create_model("vit_base_patch16_384", pretrained=pretrained) + + hooks = [2, 5, 8, 11] if hooks == None else hooks + return _make_vit_b16_backbone( + model, features=[96, 192, 384, 768], hooks=hooks, use_readout=use_readout + ) + + +def _make_vit_b_rn50_backbone( + model, + features=[256, 512, 768, 768], + size=[384, 384], + hooks=[0, 1, 8, 11], + vit_features=768, + patch_size=[16, 16], + number_stages=2, + use_vit_only=False, + use_readout="ignore", + start_index=1, +): + pretrained = nn.Module() + + pretrained.model = model + + used_number_stages = 0 if use_vit_only else number_stages + for s in range(used_number_stages): + pretrained.model.patch_embed.backbone.stages[s].register_forward_hook( + get_activation(str(s + 1)) + ) + for s in range(used_number_stages, 4): + pretrained.model.blocks[hooks[s]].register_forward_hook(get_activation(str(s + 1))) + + pretrained.activations = activations + + readout_oper = get_readout_oper(vit_features, features, use_readout, start_index) + + for s in range(used_number_stages): + value = nn.Sequential(nn.Identity(), nn.Identity(), nn.Identity()) + exec(f"pretrained.act_postprocess{s + 1}=value") + for s in range(used_number_stages, 4): + if s < number_stages: + final_layer = nn.ConvTranspose2d( + in_channels=features[s], + out_channels=features[s], + kernel_size=4 // (2 ** s), + stride=4 // (2 ** s), + padding=0, + bias=True, + dilation=1, + groups=1, + ) + elif s > number_stages: + final_layer = nn.Conv2d( + in_channels=features[3], + out_channels=features[3], + kernel_size=3, + stride=2, + padding=1, + ) + else: + final_layer = None + + layers = [ + readout_oper[s], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[s], + kernel_size=1, + stride=1, + padding=0, + ), + ] + if final_layer is not None: + layers.append(final_layer) + + value = nn.Sequential(*layers) + exec(f"pretrained.act_postprocess{s + 1}=value") + + pretrained.model.start_index = start_index + pretrained.model.patch_size = patch_size + + # We inject this function into the VisionTransformer instances so that + # we can use it with interpolated position embeddings without modifying the library source. + pretrained.model.forward_flex = types.MethodType(forward_flex, pretrained.model) + + # We inject this function into the VisionTransformer instances so that + # we can use it with interpolated position embeddings without modifying the library source. + pretrained.model._resize_pos_embed = types.MethodType( + _resize_pos_embed, pretrained.model + ) + + return pretrained + + +def _make_pretrained_vitb_rn50_384( + pretrained, use_readout="ignore", hooks=None, use_vit_only=False +): + model = timm.create_model("vit_base_resnet50_384", pretrained=pretrained) + + hooks = [0, 1, 8, 11] if hooks == None else hooks + return _make_vit_b_rn50_backbone( + model, + features=[256, 512, 768, 768], + size=[384, 384], + hooks=hooks, + use_vit_only=use_vit_only, + use_readout=use_readout, + ) diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/base_model.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/base_model.py new file mode 100644 index 0000000000000000000000000000000000000000..5cf430239b47ec5ec07531263f26f5c24a2311cd --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/base_model.py @@ -0,0 +1,16 @@ +import torch + + +class BaseModel(torch.nn.Module): + def load(self, path): + """Load model from file. + + Args: + path (str): file path + """ + parameters = torch.load(path, map_location=torch.device('cpu')) + + if "optimizer" in parameters: + parameters = parameters["model"] + + self.load_state_dict(parameters) diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/blocks.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..6d87a00680bb6ed9a6d7c3043ea30a1e90361794 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/blocks.py @@ -0,0 +1,439 @@ +import torch +import torch.nn as nn + +from .backbones.beit import ( + _make_pretrained_beitl16_512, + _make_pretrained_beitl16_384, + _make_pretrained_beitb16_384, + forward_beit, +) +from .backbones.swin_common import ( + forward_swin, +) +from .backbones.swin2 import ( + _make_pretrained_swin2l24_384, + _make_pretrained_swin2b24_384, + _make_pretrained_swin2t16_256, +) +from .backbones.swin import ( + _make_pretrained_swinl12_384, +) +from .backbones.levit import ( + _make_pretrained_levit_384, + forward_levit, +) +from .backbones.vit import ( + _make_pretrained_vitb_rn50_384, + _make_pretrained_vitl16_384, + _make_pretrained_vitb16_384, + forward_vit, +) + +def _make_encoder(backbone, features, use_pretrained, groups=1, expand=False, exportable=True, hooks=None, + use_vit_only=False, use_readout="ignore", in_features=[96, 256, 512, 1024]): + if backbone == "beitl16_512": + pretrained = _make_pretrained_beitl16_512( + use_pretrained, hooks=hooks, use_readout=use_readout + ) + scratch = _make_scratch( + [256, 512, 1024, 1024], features, groups=groups, expand=expand + ) # BEiT_512-L (backbone) + elif backbone == "beitl16_384": + pretrained = _make_pretrained_beitl16_384( + use_pretrained, hooks=hooks, use_readout=use_readout + ) + scratch = _make_scratch( + [256, 512, 1024, 1024], features, groups=groups, expand=expand + ) # BEiT_384-L (backbone) + elif backbone == "beitb16_384": + pretrained = _make_pretrained_beitb16_384( + use_pretrained, hooks=hooks, use_readout=use_readout + ) + scratch = _make_scratch( + [96, 192, 384, 768], features, groups=groups, expand=expand + ) # BEiT_384-B (backbone) + elif backbone == "swin2l24_384": + pretrained = _make_pretrained_swin2l24_384( + use_pretrained, hooks=hooks + ) + scratch = _make_scratch( + [192, 384, 768, 1536], features, groups=groups, expand=expand + ) # Swin2-L/12to24 (backbone) + elif backbone == "swin2b24_384": + pretrained = _make_pretrained_swin2b24_384( + use_pretrained, hooks=hooks + ) + scratch = _make_scratch( + [128, 256, 512, 1024], features, groups=groups, expand=expand + ) # Swin2-B/12to24 (backbone) + elif backbone == "swin2t16_256": + pretrained = _make_pretrained_swin2t16_256( + use_pretrained, hooks=hooks + ) + scratch = _make_scratch( + [96, 192, 384, 768], features, groups=groups, expand=expand + ) # Swin2-T/16 (backbone) + elif backbone == "swinl12_384": + pretrained = _make_pretrained_swinl12_384( + use_pretrained, hooks=hooks + ) + scratch = _make_scratch( + [192, 384, 768, 1536], features, groups=groups, expand=expand + ) # Swin-L/12 (backbone) + elif backbone == "next_vit_large_6m": + from .backbones.next_vit import _make_pretrained_next_vit_large_6m + pretrained = _make_pretrained_next_vit_large_6m(hooks=hooks) + scratch = _make_scratch( + in_features, features, groups=groups, expand=expand + ) # Next-ViT-L on ImageNet-1K-6M (backbone) + elif backbone == "levit_384": + pretrained = _make_pretrained_levit_384( + use_pretrained, hooks=hooks + ) + scratch = _make_scratch( + [384, 512, 768], features, groups=groups, expand=expand + ) # LeViT 384 (backbone) + elif backbone == "vitl16_384": + pretrained = _make_pretrained_vitl16_384( + use_pretrained, hooks=hooks, use_readout=use_readout + ) + scratch = _make_scratch( + [256, 512, 1024, 1024], features, groups=groups, expand=expand + ) # ViT-L/16 - 85.0% Top1 (backbone) + elif backbone == "vitb_rn50_384": + pretrained = _make_pretrained_vitb_rn50_384( + use_pretrained, + hooks=hooks, + use_vit_only=use_vit_only, + use_readout=use_readout, + ) + scratch = _make_scratch( + [256, 512, 768, 768], features, groups=groups, expand=expand + ) # ViT-H/16 - 85.0% Top1 (backbone) + elif backbone == "vitb16_384": + pretrained = _make_pretrained_vitb16_384( + use_pretrained, hooks=hooks, use_readout=use_readout + ) + scratch = _make_scratch( + [96, 192, 384, 768], features, groups=groups, expand=expand + ) # ViT-B/16 - 84.6% Top1 (backbone) + elif backbone == "resnext101_wsl": + pretrained = _make_pretrained_resnext101_wsl(use_pretrained) + scratch = _make_scratch([256, 512, 1024, 2048], features, groups=groups, expand=expand) # efficientnet_lite3 + elif backbone == "efficientnet_lite3": + pretrained = _make_pretrained_efficientnet_lite3(use_pretrained, exportable=exportable) + scratch = _make_scratch([32, 48, 136, 384], features, groups=groups, expand=expand) # efficientnet_lite3 + else: + print(f"Backbone '{backbone}' not implemented") + assert False + + return pretrained, scratch + + +def _make_scratch(in_shape, out_shape, groups=1, expand=False): + scratch = nn.Module() + + out_shape1 = out_shape + out_shape2 = out_shape + out_shape3 = out_shape + if len(in_shape) >= 4: + out_shape4 = out_shape + + if expand: + out_shape1 = out_shape + out_shape2 = out_shape*2 + out_shape3 = out_shape*4 + if len(in_shape) >= 4: + out_shape4 = out_shape*8 + + scratch.layer1_rn = nn.Conv2d( + in_shape[0], out_shape1, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + scratch.layer2_rn = nn.Conv2d( + in_shape[1], out_shape2, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + scratch.layer3_rn = nn.Conv2d( + in_shape[2], out_shape3, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + if len(in_shape) >= 4: + scratch.layer4_rn = nn.Conv2d( + in_shape[3], out_shape4, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + + return scratch + + +def _make_pretrained_efficientnet_lite3(use_pretrained, exportable=False): + efficientnet = torch.hub.load( + "rwightman/gen-efficientnet-pytorch", + "tf_efficientnet_lite3", + pretrained=use_pretrained, + exportable=exportable + ) + return _make_efficientnet_backbone(efficientnet) + + +def _make_efficientnet_backbone(effnet): + pretrained = nn.Module() + + pretrained.layer1 = nn.Sequential( + effnet.conv_stem, effnet.bn1, effnet.act1, *effnet.blocks[0:2] + ) + pretrained.layer2 = nn.Sequential(*effnet.blocks[2:3]) + pretrained.layer3 = nn.Sequential(*effnet.blocks[3:5]) + pretrained.layer4 = nn.Sequential(*effnet.blocks[5:9]) + + return pretrained + + +def _make_resnet_backbone(resnet): + pretrained = nn.Module() + pretrained.layer1 = nn.Sequential( + resnet.conv1, resnet.bn1, resnet.relu, resnet.maxpool, resnet.layer1 + ) + + pretrained.layer2 = resnet.layer2 + pretrained.layer3 = resnet.layer3 + pretrained.layer4 = resnet.layer4 + + return pretrained + + +def _make_pretrained_resnext101_wsl(use_pretrained): + resnet = torch.hub.load("facebookresearch/WSL-Images", "resnext101_32x8d_wsl") + return _make_resnet_backbone(resnet) + + + +class Interpolate(nn.Module): + """Interpolation module. + """ + + def __init__(self, scale_factor, mode, align_corners=False): + """Init. + + Args: + scale_factor (float): scaling + mode (str): interpolation mode + """ + super(Interpolate, self).__init__() + + self.interp = nn.functional.interpolate + self.scale_factor = scale_factor + self.mode = mode + self.align_corners = align_corners + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: interpolated data + """ + + x = self.interp( + x, scale_factor=self.scale_factor, mode=self.mode, align_corners=self.align_corners + ) + + return x + + +class ResidualConvUnit(nn.Module): + """Residual convolution module. + """ + + def __init__(self, features): + """Init. + + Args: + features (int): number of features + """ + super().__init__() + + self.conv1 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True + ) + + self.conv2 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True + ) + + self.relu = nn.ReLU(inplace=True) + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: output + """ + out = self.relu(x) + out = self.conv1(out) + out = self.relu(out) + out = self.conv2(out) + + return out + x + + +class FeatureFusionBlock(nn.Module): + """Feature fusion block. + """ + + def __init__(self, features): + """Init. + + Args: + features (int): number of features + """ + super(FeatureFusionBlock, self).__init__() + + self.resConfUnit1 = ResidualConvUnit(features) + self.resConfUnit2 = ResidualConvUnit(features) + + def forward(self, *xs): + """Forward pass. + + Returns: + tensor: output + """ + output = xs[0] + + if len(xs) == 2: + output += self.resConfUnit1(xs[1]) + + output = self.resConfUnit2(output) + + output = nn.functional.interpolate( + output, scale_factor=2, mode="bilinear", align_corners=True + ) + + return output + + + + +class ResidualConvUnit_custom(nn.Module): + """Residual convolution module. + """ + + def __init__(self, features, activation, bn): + """Init. + + Args: + features (int): number of features + """ + super().__init__() + + self.bn = bn + + self.groups=1 + + self.conv1 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups + ) + + self.conv2 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups + ) + + if self.bn==True: + self.bn1 = nn.BatchNorm2d(features) + self.bn2 = nn.BatchNorm2d(features) + + self.activation = activation + + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: output + """ + + out = self.activation(x) + out = self.conv1(out) + if self.bn==True: + out = self.bn1(out) + + out = self.activation(out) + out = self.conv2(out) + if self.bn==True: + out = self.bn2(out) + + if self.groups > 1: + out = self.conv_merge(out) + + return self.skip_add.add(out, x) + + # return out + x + + +class FeatureFusionBlock_custom(nn.Module): + """Feature fusion block. + """ + + def __init__(self, features, activation, deconv=False, bn=False, expand=False, align_corners=True, size=None): + """Init. + + Args: + features (int): number of features + """ + super(FeatureFusionBlock_custom, self).__init__() + + self.deconv = deconv + self.align_corners = align_corners + + self.groups=1 + + self.expand = expand + out_features = features + if self.expand==True: + out_features = features//2 + + self.out_conv = nn.Conv2d(features, out_features, kernel_size=1, stride=1, padding=0, bias=True, groups=1) + + self.resConfUnit1 = ResidualConvUnit_custom(features, activation, bn) + self.resConfUnit2 = ResidualConvUnit_custom(features, activation, bn) + + self.skip_add = nn.quantized.FloatFunctional() + + self.size=size + + def forward(self, *xs, size=None): + """Forward pass. + + Returns: + tensor: output + """ + output = xs[0] + + if len(xs) == 2: + res = self.resConfUnit1(xs[1]) + output = self.skip_add.add(output, res) + # output += res + + output = self.resConfUnit2(output) + + if (size is None) and (self.size is None): + modifier = {"scale_factor": 2} + elif size is None: + modifier = {"size": self.size} + else: + modifier = {"size": size} + + output = nn.functional.interpolate( + output, **modifier, mode="bilinear", align_corners=self.align_corners + ) + + output = self.out_conv(output) + + return output + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/dpt_depth.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/dpt_depth.py new file mode 100644 index 0000000000000000000000000000000000000000..3129d09cb43a7c79b23916236991fabbedb78f55 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/dpt_depth.py @@ -0,0 +1,166 @@ +import torch +import torch.nn as nn + +from .base_model import BaseModel +from .blocks import ( + FeatureFusionBlock_custom, + Interpolate, + _make_encoder, + forward_beit, + forward_swin, + forward_levit, + forward_vit, +) +from .backbones.levit import stem_b4_transpose +from timm.models.layers import get_act_layer + + +def _make_fusion_block(features, use_bn, size = None): + return FeatureFusionBlock_custom( + features, + nn.ReLU(False), + deconv=False, + bn=use_bn, + expand=False, + align_corners=True, + size=size, + ) + + +class DPT(BaseModel): + def __init__( + self, + head, + features=256, + backbone="vitb_rn50_384", + readout="project", + channels_last=False, + use_bn=False, + **kwargs + ): + + super(DPT, self).__init__() + + self.channels_last = channels_last + + # For the Swin, Swin 2, LeViT and Next-ViT Transformers, the hierarchical architectures prevent setting the + # hooks freely. Instead, the hooks have to be chosen according to the ranges specified in the comments. + hooks = { + "beitl16_512": [5, 11, 17, 23], + "beitl16_384": [5, 11, 17, 23], + "beitb16_384": [2, 5, 8, 11], + "swin2l24_384": [1, 1, 17, 1], # Allowed ranges: [0, 1], [0, 1], [ 0, 17], [ 0, 1] + "swin2b24_384": [1, 1, 17, 1], # [0, 1], [0, 1], [ 0, 17], [ 0, 1] + "swin2t16_256": [1, 1, 5, 1], # [0, 1], [0, 1], [ 0, 5], [ 0, 1] + "swinl12_384": [1, 1, 17, 1], # [0, 1], [0, 1], [ 0, 17], [ 0, 1] + "next_vit_large_6m": [2, 6, 36, 39], # [0, 2], [3, 6], [ 7, 36], [37, 39] + "levit_384": [3, 11, 21], # [0, 3], [6, 11], [14, 21] + "vitb_rn50_384": [0, 1, 8, 11], + "vitb16_384": [2, 5, 8, 11], + "vitl16_384": [5, 11, 17, 23], + }[backbone] + + if "next_vit" in backbone: + in_features = { + "next_vit_large_6m": [96, 256, 512, 1024], + }[backbone] + else: + in_features = None + + # Instantiate backbone and reassemble blocks + self.pretrained, self.scratch = _make_encoder( + backbone, + features, + False, # Set to true of you want to train from scratch, uses ImageNet weights + groups=1, + expand=False, + exportable=False, + hooks=hooks, + use_readout=readout, + in_features=in_features, + ) + + self.number_layers = len(hooks) if hooks is not None else 4 + size_refinenet3 = None + self.scratch.stem_transpose = None + + if "beit" in backbone: + self.forward_transformer = forward_beit + elif "swin" in backbone: + self.forward_transformer = forward_swin + elif "next_vit" in backbone: + from .backbones.next_vit import forward_next_vit + self.forward_transformer = forward_next_vit + elif "levit" in backbone: + self.forward_transformer = forward_levit + size_refinenet3 = 7 + self.scratch.stem_transpose = stem_b4_transpose(256, 128, get_act_layer("hard_swish")) + else: + self.forward_transformer = forward_vit + + self.scratch.refinenet1 = _make_fusion_block(features, use_bn) + self.scratch.refinenet2 = _make_fusion_block(features, use_bn) + self.scratch.refinenet3 = _make_fusion_block(features, use_bn, size_refinenet3) + if self.number_layers >= 4: + self.scratch.refinenet4 = _make_fusion_block(features, use_bn) + + self.scratch.output_conv = head + + + def forward(self, x): + if self.channels_last == True: + x.contiguous(memory_format=torch.channels_last) + + layers = self.forward_transformer(self.pretrained, x) + if self.number_layers == 3: + layer_1, layer_2, layer_3 = layers + else: + layer_1, layer_2, layer_3, layer_4 = layers + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + if self.number_layers >= 4: + layer_4_rn = self.scratch.layer4_rn(layer_4) + + if self.number_layers == 3: + path_3 = self.scratch.refinenet3(layer_3_rn, size=layer_2_rn.shape[2:]) + else: + path_4 = self.scratch.refinenet4(layer_4_rn, size=layer_3_rn.shape[2:]) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn, size=layer_2_rn.shape[2:]) + path_2 = self.scratch.refinenet2(path_3, layer_2_rn, size=layer_1_rn.shape[2:]) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + if self.scratch.stem_transpose is not None: + path_1 = self.scratch.stem_transpose(path_1) + + out = self.scratch.output_conv(path_1) + + return out + + +class DPTDepthModel(DPT): + def __init__(self, path=None, non_negative=True, **kwargs): + features = kwargs["features"] if "features" in kwargs else 256 + head_features_1 = kwargs["head_features_1"] if "head_features_1" in kwargs else features + head_features_2 = kwargs["head_features_2"] if "head_features_2" in kwargs else 32 + kwargs.pop("head_features_1", None) + kwargs.pop("head_features_2", None) + + head = nn.Sequential( + nn.Conv2d(head_features_1, head_features_1 // 2, kernel_size=3, stride=1, padding=1), + Interpolate(scale_factor=2, mode="bilinear", align_corners=True), + nn.Conv2d(head_features_1 // 2, head_features_2, kernel_size=3, stride=1, padding=1), + nn.ReLU(True), + nn.Conv2d(head_features_2, 1, kernel_size=1, stride=1, padding=0), + nn.ReLU(True) if non_negative else nn.Identity(), + nn.Identity(), + ) + + super().__init__(head, **kwargs) + + if path is not None: + self.load(path) + + def forward(self, x): + return super().forward(x).squeeze(dim=1) diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/midas_net.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/midas_net.py new file mode 100644 index 0000000000000000000000000000000000000000..8a954977800b0a0f48807e80fa63041910e33c1f --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/midas_net.py @@ -0,0 +1,76 @@ +"""MidashNet: Network for monocular depth estimation trained by mixing several datasets. +This file contains code that is adapted from +https://github.com/thomasjpfan/pytorch_refinenet/blob/master/pytorch_refinenet/refinenet/refinenet_4cascade.py +""" +import torch +import torch.nn as nn + +from .base_model import BaseModel +from .blocks import FeatureFusionBlock, Interpolate, _make_encoder + + +class MidasNet(BaseModel): + """Network for monocular depth estimation. + """ + + def __init__(self, path=None, features=256, non_negative=True): + """Init. + + Args: + path (str, optional): Path to saved model. Defaults to None. + features (int, optional): Number of features. Defaults to 256. + backbone (str, optional): Backbone network for encoder. Defaults to resnet50 + """ + print("Loading weights: ", path) + + super(MidasNet, self).__init__() + + use_pretrained = False if path is None else True + + self.pretrained, self.scratch = _make_encoder(backbone="resnext101_wsl", features=features, use_pretrained=use_pretrained) + + self.scratch.refinenet4 = FeatureFusionBlock(features) + self.scratch.refinenet3 = FeatureFusionBlock(features) + self.scratch.refinenet2 = FeatureFusionBlock(features) + self.scratch.refinenet1 = FeatureFusionBlock(features) + + self.scratch.output_conv = nn.Sequential( + nn.Conv2d(features, 128, kernel_size=3, stride=1, padding=1), + Interpolate(scale_factor=2, mode="bilinear"), + nn.Conv2d(128, 32, kernel_size=3, stride=1, padding=1), + nn.ReLU(True), + nn.Conv2d(32, 1, kernel_size=1, stride=1, padding=0), + nn.ReLU(True) if non_negative else nn.Identity(), + ) + + if path: + self.load(path) + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input data (image) + + Returns: + tensor: depth + """ + + layer_1 = self.pretrained.layer1(x) + layer_2 = self.pretrained.layer2(layer_1) + layer_3 = self.pretrained.layer3(layer_2) + layer_4 = self.pretrained.layer4(layer_3) + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + layer_4_rn = self.scratch.layer4_rn(layer_4) + + path_4 = self.scratch.refinenet4(layer_4_rn) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn) + path_2 = self.scratch.refinenet2(path_3, layer_2_rn) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + out = self.scratch.output_conv(path_1) + + return torch.squeeze(out, dim=1) diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/midas_net_custom.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/midas_net_custom.py new file mode 100644 index 0000000000000000000000000000000000000000..50e4acb5e53d5fabefe3dde16ab49c33c2b7797c --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/midas_net_custom.py @@ -0,0 +1,128 @@ +"""MidashNet: Network for monocular depth estimation trained by mixing several datasets. +This file contains code that is adapted from +https://github.com/thomasjpfan/pytorch_refinenet/blob/master/pytorch_refinenet/refinenet/refinenet_4cascade.py +""" +import torch +import torch.nn as nn + +from .base_model import BaseModel +from .blocks import FeatureFusionBlock, FeatureFusionBlock_custom, Interpolate, _make_encoder + + +class MidasNet_small(BaseModel): + """Network for monocular depth estimation. + """ + + def __init__(self, path=None, features=64, backbone="efficientnet_lite3", non_negative=True, exportable=True, channels_last=False, align_corners=True, + blocks={'expand': True}): + """Init. + + Args: + path (str, optional): Path to saved model. Defaults to None. + features (int, optional): Number of features. Defaults to 256. + backbone (str, optional): Backbone network for encoder. Defaults to resnet50 + """ + print("Loading weights: ", path) + + super(MidasNet_small, self).__init__() + + use_pretrained = False if path else True + + self.channels_last = channels_last + self.blocks = blocks + self.backbone = backbone + + self.groups = 1 + + features1=features + features2=features + features3=features + features4=features + self.expand = False + if "expand" in self.blocks and self.blocks['expand'] == True: + self.expand = True + features1=features + features2=features*2 + features3=features*4 + features4=features*8 + + self.pretrained, self.scratch = _make_encoder(self.backbone, features, use_pretrained, groups=self.groups, expand=self.expand, exportable=exportable) + + self.scratch.activation = nn.ReLU(False) + + self.scratch.refinenet4 = FeatureFusionBlock_custom(features4, self.scratch.activation, deconv=False, bn=False, expand=self.expand, align_corners=align_corners) + self.scratch.refinenet3 = FeatureFusionBlock_custom(features3, self.scratch.activation, deconv=False, bn=False, expand=self.expand, align_corners=align_corners) + self.scratch.refinenet2 = FeatureFusionBlock_custom(features2, self.scratch.activation, deconv=False, bn=False, expand=self.expand, align_corners=align_corners) + self.scratch.refinenet1 = FeatureFusionBlock_custom(features1, self.scratch.activation, deconv=False, bn=False, align_corners=align_corners) + + + self.scratch.output_conv = nn.Sequential( + nn.Conv2d(features, features//2, kernel_size=3, stride=1, padding=1, groups=self.groups), + Interpolate(scale_factor=2, mode="bilinear"), + nn.Conv2d(features//2, 32, kernel_size=3, stride=1, padding=1), + self.scratch.activation, + nn.Conv2d(32, 1, kernel_size=1, stride=1, padding=0), + nn.ReLU(True) if non_negative else nn.Identity(), + nn.Identity(), + ) + + if path: + self.load(path) + + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input data (image) + + Returns: + tensor: depth + """ + if self.channels_last==True: + print("self.channels_last = ", self.channels_last) + x.contiguous(memory_format=torch.channels_last) + + + layer_1 = self.pretrained.layer1(x) + layer_2 = self.pretrained.layer2(layer_1) + layer_3 = self.pretrained.layer3(layer_2) + layer_4 = self.pretrained.layer4(layer_3) + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + layer_4_rn = self.scratch.layer4_rn(layer_4) + + + path_4 = self.scratch.refinenet4(layer_4_rn) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn) + path_2 = self.scratch.refinenet2(path_3, layer_2_rn) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + out = self.scratch.output_conv(path_1) + + return torch.squeeze(out, dim=1) + + + +def fuse_model(m): + prev_previous_type = nn.Identity() + prev_previous_name = '' + previous_type = nn.Identity() + previous_name = '' + for name, module in m.named_modules(): + if prev_previous_type == nn.Conv2d and previous_type == nn.BatchNorm2d and type(module) == nn.ReLU: + # print("FUSED ", prev_previous_name, previous_name, name) + torch.quantization.fuse_modules(m, [prev_previous_name, previous_name, name], inplace=True) + elif prev_previous_type == nn.Conv2d and previous_type == nn.BatchNorm2d: + # print("FUSED ", prev_previous_name, previous_name) + torch.quantization.fuse_modules(m, [prev_previous_name, previous_name], inplace=True) + # elif previous_type == nn.Conv2d and type(module) == nn.ReLU: + # print("FUSED ", previous_name, name) + # torch.quantization.fuse_modules(m, [previous_name, name], inplace=True) + + prev_previous_type = previous_type + prev_previous_name = previous_name + previous_type = type(module) + previous_name = name \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/model_loader.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/model_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..94241f0e668cfcad914a3b64f79852bb4fd719ab --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/model_loader.py @@ -0,0 +1,243 @@ +import cv2 +import torch + +from midas.dpt_depth import DPTDepthModel +from midas.midas_net import MidasNet +from midas.midas_net_custom import MidasNet_small +from midas.transforms import Resize, NormalizeImage, PrepareForNet + +from torchvision.transforms import Compose + +default_models = { + "dpt_beit_large_512": "weights/dpt_beit_large_512.pt", + "dpt_beit_large_384": "weights/dpt_beit_large_384.pt", + "dpt_beit_base_384": "weights/dpt_beit_base_384.pt", + "dpt_swin2_large_384": "weights/dpt_swin2_large_384.pt", + "dpt_swin2_base_384": "weights/dpt_swin2_base_384.pt", + "dpt_swin2_tiny_256": "weights/dpt_swin2_tiny_256.pt", + "dpt_swin_large_384": "weights/dpt_swin_large_384.pt", + "dpt_next_vit_large_384": "weights/dpt_next_vit_large_384.pt", + "dpt_levit_224": "weights/dpt_levit_224.pt", + "dpt_large_384": "weights/dpt_large_384.pt", + "dpt_hybrid_384": "weights/dpt_hybrid_384.pt", + "midas_v21_384": "weights/midas_v21_384.pt", + "midas_v21_small_256": "weights/midas_v21_small_256.pt", + "openvino_midas_v21_small_256": "weights/openvino_midas_v21_small_256.xml", +} + + +def load_model(device, model_path, model_type="dpt_large_384", optimize=True, height=None, square=False): +#def load_model(device, model_path, model_type="dpt_large_384", optimize=False, height=None, square=False): + """Load the specified network. + + Args: + device (device): the torch device used + model_path (str): path to saved model + model_type (str): the type of the model to be loaded + optimize (bool): optimize the model to half-integer on CUDA? + height (int): inference encoder image height + square (bool): resize to a square resolution? + + Returns: + The loaded network, the transform which prepares images as input to the network and the dimensions of the + network input + """ + if "openvino" in model_type: + from openvino.runtime import Core + + keep_aspect_ratio = not square + + if model_type == "dpt_beit_large_512": + model = DPTDepthModel( + path=model_path, + backbone="beitl16_512", + non_negative=True, + ) + net_w, net_h = 512, 512 + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_beit_large_384": + model = DPTDepthModel( + path=model_path, + backbone="beitl16_384", + non_negative=True, + ) + net_w, net_h = 384, 384 + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_beit_base_384": + model = DPTDepthModel( + path=model_path, + backbone="beitb16_384", + non_negative=True, + ) + net_w, net_h = 384, 384 + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_swin2_large_384": + model = DPTDepthModel( + path=model_path, + backbone="swin2l24_384", + non_negative=True, + ) + net_w, net_h = 384, 384 + keep_aspect_ratio = False + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_swin2_base_384": + model = DPTDepthModel( + path=model_path, + backbone="swin2b24_384", + non_negative=True, + ) + net_w, net_h = 384, 384 + keep_aspect_ratio = False + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_swin2_tiny_256": + model = DPTDepthModel( + path=model_path, + backbone="swin2t16_256", + non_negative=True, + ) + net_w, net_h = 256, 256 + keep_aspect_ratio = False + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_swin_large_384": + model = DPTDepthModel( + path=model_path, + backbone="swinl12_384", + non_negative=True, + ) + net_w, net_h = 384, 384 + keep_aspect_ratio = False + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_next_vit_large_384": + model = DPTDepthModel( + path=model_path, + backbone="next_vit_large_6m", + non_negative=True, + ) + net_w, net_h = 384, 384 + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + # We change the notation from dpt_levit_224 (MiDaS notation) to levit_384 (timm notation) here, where the 224 refers + # to the resolution 224x224 used by LeViT and 384 is the first entry of the embed_dim, see _cfg and model_cfgs of + # https://github.com/rwightman/pytorch-image-models/blob/main/timm/models/levit.py + # (commit id: 927f031293a30afb940fff0bee34b85d9c059b0e) + elif model_type == "dpt_levit_224": + model = DPTDepthModel( + path=model_path, + backbone="levit_384", + non_negative=True, + head_features_1=64, + head_features_2=8, + ) + net_w, net_h = 224, 224 + keep_aspect_ratio = False + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_large_384": + model = DPTDepthModel( + path=model_path, + backbone="vitl16_384", + non_negative=True, + ) + net_w, net_h = 384, 384 + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_hybrid_384": + model = DPTDepthModel( + path=model_path, + backbone="vitb_rn50_384", + non_negative=True, + ) + net_w, net_h = 384, 384 + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "midas_v21_384": + model = MidasNet(model_path, non_negative=True) + net_w, net_h = 384, 384 + resize_mode = "upper_bound" + normalization = NormalizeImage( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] + ) + + elif model_type == "midas_v21_small_256": + model = MidasNet_small(model_path, features=64, backbone="efficientnet_lite3", exportable=True, + non_negative=True, blocks={'expand': True}) + net_w, net_h = 256, 256 + resize_mode = "upper_bound" + normalization = NormalizeImage( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] + ) + + elif model_type == "openvino_midas_v21_small_256": + ie = Core() + uncompiled_model = ie.read_model(model=model_path) + model = ie.compile_model(uncompiled_model, "CPU") + net_w, net_h = 256, 256 + resize_mode = "upper_bound" + normalization = NormalizeImage( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] + ) + + else: + print(f"model_type '{model_type}' not implemented, use: --model_type large") + assert False + + if not "openvino" in model_type: + print("Model loaded, number of parameters = {:.0f}M".format(sum(p.numel() for p in model.parameters()) / 1e6)) + else: + print("Model loaded, optimized with OpenVINO") + + if "openvino" in model_type: + keep_aspect_ratio = False + + if height is not None: + net_w, net_h = height, height + + transform = Compose( + [ + Resize( + net_w, + net_h, + resize_target=None, + keep_aspect_ratio=keep_aspect_ratio, + ensure_multiple_of=32, + resize_method=resize_mode, + image_interpolation_method=cv2.INTER_CUBIC, + ), + normalization, + PrepareForNet(), + ] + ) + + if not "openvino" in model_type: + model.eval() + + if optimize and (device == torch.device("cuda")): + if not "openvino" in model_type: + model = model.to(memory_format=torch.channels_last) + model = model.half() + else: + print("Error: OpenVINO models are already optimized. No optimization to half-float possible.") + exit() + + if not "openvino" in model_type: + model.to(device) + + return model, transform, net_w, net_h diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/transforms.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..350cbc11662633ad7f8968eb10be2e7de6e384e9 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/transforms.py @@ -0,0 +1,234 @@ +import numpy as np +import cv2 +import math + + +def apply_min_size(sample, size, image_interpolation_method=cv2.INTER_AREA): + """Rezise the sample to ensure the given size. Keeps aspect ratio. + + Args: + sample (dict): sample + size (tuple): image size + + Returns: + tuple: new size + """ + shape = list(sample["disparity"].shape) + + if shape[0] >= size[0] and shape[1] >= size[1]: + return sample + + scale = [0, 0] + scale[0] = size[0] / shape[0] + scale[1] = size[1] / shape[1] + + scale = max(scale) + + shape[0] = math.ceil(scale * shape[0]) + shape[1] = math.ceil(scale * shape[1]) + + # resize + sample["image"] = cv2.resize( + sample["image"], tuple(shape[::-1]), interpolation=image_interpolation_method + ) + + sample["disparity"] = cv2.resize( + sample["disparity"], tuple(shape[::-1]), interpolation=cv2.INTER_NEAREST + ) + sample["mask"] = cv2.resize( + sample["mask"].astype(np.float32), + tuple(shape[::-1]), + interpolation=cv2.INTER_NEAREST, + ) + sample["mask"] = sample["mask"].astype(bool) + + return tuple(shape) + + +class Resize(object): + """Resize sample to given size (width, height). + """ + + def __init__( + self, + width, + height, + resize_target=True, + keep_aspect_ratio=False, + ensure_multiple_of=1, + resize_method="lower_bound", + image_interpolation_method=cv2.INTER_AREA, + ): + """Init. + + Args: + width (int): desired output width + height (int): desired output height + resize_target (bool, optional): + True: Resize the full sample (image, mask, target). + False: Resize image only. + Defaults to True. + keep_aspect_ratio (bool, optional): + True: Keep the aspect ratio of the input sample. + Output sample might not have the given width and height, and + resize behaviour depends on the parameter 'resize_method'. + Defaults to False. + ensure_multiple_of (int, optional): + Output width and height is constrained to be multiple of this parameter. + Defaults to 1. + resize_method (str, optional): + "lower_bound": Output will be at least as large as the given size. + "upper_bound": Output will be at max as large as the given size. (Output size might be smaller than given size.) + "minimal": Scale as least as possible. (Output size might be smaller than given size.) + Defaults to "lower_bound". + """ + self.__width = width + self.__height = height + + self.__resize_target = resize_target + self.__keep_aspect_ratio = keep_aspect_ratio + self.__multiple_of = ensure_multiple_of + self.__resize_method = resize_method + self.__image_interpolation_method = image_interpolation_method + + def constrain_to_multiple_of(self, x, min_val=0, max_val=None): + y = (np.round(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if max_val is not None and y > max_val: + y = (np.floor(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if y < min_val: + y = (np.ceil(x / self.__multiple_of) * self.__multiple_of).astype(int) + + return y + + def get_size(self, width, height): + # determine new height and width + scale_height = self.__height / height + scale_width = self.__width / width + + if self.__keep_aspect_ratio: + if self.__resize_method == "lower_bound": + # scale such that output size is lower bound + if scale_width > scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "upper_bound": + # scale such that output size is upper bound + if scale_width < scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "minimal": + # scale as least as possbile + if abs(1 - scale_width) < abs(1 - scale_height): + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + else: + raise ValueError( + f"resize_method {self.__resize_method} not implemented" + ) + + if self.__resize_method == "lower_bound": + new_height = self.constrain_to_multiple_of( + scale_height * height, min_val=self.__height + ) + new_width = self.constrain_to_multiple_of( + scale_width * width, min_val=self.__width + ) + elif self.__resize_method == "upper_bound": + new_height = self.constrain_to_multiple_of( + scale_height * height, max_val=self.__height + ) + new_width = self.constrain_to_multiple_of( + scale_width * width, max_val=self.__width + ) + elif self.__resize_method == "minimal": + new_height = self.constrain_to_multiple_of(scale_height * height) + new_width = self.constrain_to_multiple_of(scale_width * width) + else: + raise ValueError(f"resize_method {self.__resize_method} not implemented") + + return (new_width, new_height) + + def __call__(self, sample): + width, height = self.get_size( + sample["image"].shape[1], sample["image"].shape[0] + ) + + # resize sample + sample["image"] = cv2.resize( + sample["image"], + (width, height), + interpolation=self.__image_interpolation_method, + ) + + if self.__resize_target: + if "disparity" in sample: + sample["disparity"] = cv2.resize( + sample["disparity"], + (width, height), + interpolation=cv2.INTER_NEAREST, + ) + + if "depth" in sample: + sample["depth"] = cv2.resize( + sample["depth"], (width, height), interpolation=cv2.INTER_NEAREST + ) + + sample["mask"] = cv2.resize( + sample["mask"].astype(np.float32), + (width, height), + interpolation=cv2.INTER_NEAREST, + ) + sample["mask"] = sample["mask"].astype(bool) + + return sample + + +class NormalizeImage(object): + """Normlize image by given mean and std. + """ + + def __init__(self, mean, std): + self.__mean = mean + self.__std = std + + def __call__(self, sample): + sample["image"] = (sample["image"] - self.__mean) / self.__std + + return sample + + +class PrepareForNet(object): + """Prepare sample for usage as network input. + """ + + def __init__(self): + pass + + def __call__(self, sample): + image = np.transpose(sample["image"], (2, 0, 1)) + sample["image"] = np.ascontiguousarray(image).astype(np.float32) + + if "mask" in sample: + sample["mask"] = sample["mask"].astype(np.float32) + sample["mask"] = np.ascontiguousarray(sample["mask"]) + + if "disparity" in sample: + disparity = sample["disparity"].astype(np.float32) + sample["disparity"] = np.ascontiguousarray(disparity) + + if "depth" in sample: + depth = sample["depth"].astype(np.float32) + sample["depth"] = np.ascontiguousarray(depth) + + return sample diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/README.md b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/README.md new file mode 100644 index 0000000000000000000000000000000000000000..45c18f7f0bfe40c0db373e8a94716867705f5827 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/README.md @@ -0,0 +1,70 @@ +## Mobile version of MiDaS for iOS / Android - Monocular Depth Estimation + +### Accuracy + +* Old small model - ResNet50 default-decoder 384x384 +* New small model - EfficientNet-Lite3 small-decoder 256x256 + +**Zero-shot error** (the lower - the better): + +| Model | DIW WHDR | Eth3d AbsRel | Sintel AbsRel | Kitti δ>1.25 | NyuDepthV2 δ>1.25 | TUM δ>1.25 | +|---|---|---|---|---|---|---| +| Old small model 384x384 | **0.1248** | 0.1550 | **0.3300** | **21.81** | 15.73 | 17.00 | +| New small model 256x256 | 0.1344 | **0.1344** | 0.3370 | 29.27 | **13.43** | **14.53** | +| Relative improvement, % | -8 % | **+13 %** | -2 % | -34 % | **+15 %** | **+15 %** | + +None of Train/Valid/Test subsets of datasets (DIW, Eth3d, Sintel, Kitti, NyuDepthV2, TUM) were not involved in Training or Fine Tuning. + +### Inference speed (FPS) on iOS / Android + +**Frames Per Second** (the higher - the better): + +| Model | iPhone CPU | iPhone GPU | iPhone NPU | OnePlus8 CPU | OnePlus8 GPU | OnePlus8 NNAPI | +|---|---|---|---|---|---|---| +| Old small model 384x384 | 0.6 | N/A | N/A | 0.45 | 0.50 | 0.50 | +| New small model 256x256 | 8 | 22 | **30** | 6 | **22** | 4 | +| SpeedUp, X times | **12.8x** | - | - | **13.2x** | **44x** | **8x** | + +N/A - run-time error (no data available) + + +#### Models: + +* Old small model - ResNet50 default-decoder 1x384x384x3, batch=1 FP32 (converters: Pytorch -> ONNX - [onnx_tf](https://github.com/onnx/onnx-tensorflow) -> (saved model) PB -> TFlite) + + (Trained on datasets: RedWeb, MegaDepth, WSVD, 3D Movies, DIML indoor) + +* New small model - EfficientNet-Lite3 small-decoder 1x256x256x3, batch=1 FP32 (custom converter: Pytorch -> TFlite) + + (Trained on datasets: RedWeb, MegaDepth, WSVD, 3D Movies, DIML indoor, HRWSI, IRS, TartanAir, BlendedMVS, ApolloScape) + +#### Frameworks for training and conversions: +``` +pip install torch==1.6.0 torchvision==0.7.0 +pip install tf-nightly-gpu==2.5.0.dev20201031 tensorflow-addons==0.11.2 numpy==1.18.0 +git clone --depth 1 --branch v1.6.0 https://github.com/onnx/onnx-tensorflow +``` + +#### SoC - OS - Library: + +* iPhone 11 (A13 Bionic) - iOS 13.7 - TensorFlowLiteSwift 0.0.1-nightly +* OnePlus 8 (Snapdragon 865) - Andoird 10 - org.tensorflow:tensorflow-lite-task-vision:0.0.0-nightly + + +### Citation + +This repository contains code to compute depth from a single image. It accompanies our [paper](https://arxiv.org/abs/1907.01341v3): + +>Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer +René Ranftl, Katrin Lasinger, David Hafner, Konrad Schindler, Vladlen Koltun + +Please cite our paper if you use this code or any of the models: +``` +@article{Ranftl2020, + author = {Ren\'{e} Ranftl and Katrin Lasinger and David Hafner and Konrad Schindler and Vladlen Koltun}, + title = {Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer}, + journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (TPAMI)}, + year = {2020}, +} +``` + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/EXPLORE_THE_CODE.md b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/EXPLORE_THE_CODE.md new file mode 100644 index 0000000000000000000000000000000000000000..72014bdfa2cd701a6453debbc8e53fcc15c0a5dc --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/EXPLORE_THE_CODE.md @@ -0,0 +1,414 @@ +# TensorFlow Lite Android image classification example + +This document walks through the code of a simple Android mobile application that +demonstrates +[image classification](https://www.tensorflow.org/lite/models/image_classification/overview) +using the device camera. + +## Explore the code + +We're now going to walk through the most important parts of the sample code. + +### Get camera input + +This mobile application gets the camera input using the functions defined in the +file +[`CameraActivity.java`](https://github.com/tensorflow/examples/tree/master/lite/examples/image_classification/android/app/src/main/java/org/tensorflow/lite/examples/classification/CameraActivity.java). +This file depends on +[`AndroidManifest.xml`](https://github.com/tensorflow/examples/tree/master/lite/examples/image_classification/android/app/src/main/AndroidManifest.xml) +to set the camera orientation. + +`CameraActivity` also contains code to capture user preferences from the UI and +make them available to other classes via convenience methods. + +```java +model = Model.valueOf(modelSpinner.getSelectedItem().toString().toUpperCase()); +device = Device.valueOf(deviceSpinner.getSelectedItem().toString()); +numThreads = Integer.parseInt(threadsTextView.getText().toString().trim()); +``` + +### Classifier + +This Image Classification Android reference app demonstrates two implementation +solutions, +[`lib_task_api`](https://github.com/tensorflow/examples/tree/master/lite/examples/image_classification/android/lib_task_api) +that leverages the out-of-box API from the +[TensorFlow Lite Task Library](https://www.tensorflow.org/lite/inference_with_metadata/task_library/image_classifier), +and +[`lib_support`](https://github.com/tensorflow/examples/tree/master/lite/examples/image_classification/android/lib_support) +that creates the custom inference pipleline using the +[TensorFlow Lite Support Library](https://www.tensorflow.org/lite/inference_with_metadata/lite_support). + +Both solutions implement the file `Classifier.java` (see +[the one in lib_task_api](https://github.com/tensorflow/examples/tree/master/lite/examples/image_classification/android/lib_task_api/src/main/java/org/tensorflow/lite/examples/classification/tflite/Classifier.java) +and +[the one in lib_support](https://github.com/tensorflow/examples/tree/master/lite/examples/image_classification/android/lib_support/src/main/java/org/tensorflow/lite/examples/classification/tflite/Classifier.java)) +that contains most of the complex logic for processing the camera input and +running inference. + +Two subclasses of the `Classifier` exist, as in `ClassifierFloatMobileNet.java` +and `ClassifierQuantizedMobileNet.java`, which contain settings for both +floating point and +[quantized](https://www.tensorflow.org/lite/performance/post_training_quantization) +models. + +The `Classifier` class implements a static method, `create`, which is used to +instantiate the appropriate subclass based on the supplied model type (quantized +vs floating point). + +#### Using the TensorFlow Lite Task Library + +Inference can be done using just a few lines of code with the +[`ImageClassifier`](https://www.tensorflow.org/lite/inference_with_metadata/task_library/image_classifier) +in the TensorFlow Lite Task Library. + +##### Load model and create ImageClassifier + +`ImageClassifier` expects a model populated with the +[model metadata](https://www.tensorflow.org/lite/convert/metadata) and the label +file. See the +[model compatibility requirements](https://www.tensorflow.org/lite/inference_with_metadata/task_library/image_classifier#model_compatibility_requirements) +for more details. + +`ImageClassifierOptions` allows manipulation on various inference options, such +as setting the maximum number of top scored results to return using +`setMaxResults(MAX_RESULTS)`, and setting the score threshold using +`setScoreThreshold(scoreThreshold)`. + +```java +// Create the ImageClassifier instance. +ImageClassifierOptions options = + ImageClassifierOptions.builder().setMaxResults(MAX_RESULTS).build(); +imageClassifier = ImageClassifier.createFromFileAndOptions(activity, + getModelPath(), options); +``` + +`ImageClassifier` currently does not support configuring delegates and +multithread, but those are on our roadmap. Please stay tuned! + +##### Run inference + +`ImageClassifier` contains builtin logic to preprocess the input image, such as +rotating and resizing an image. Processing options can be configured through +`ImageProcessingOptions`. In the following example, input images are rotated to +the up-right angle and cropped to the center as the model expects a square input +(`224x224`). See the +[Java doc of `ImageClassifier`](https://github.com/tensorflow/tflite-support/blob/195b574f0aa9856c618b3f1ad87bd185cddeb657/tensorflow_lite_support/java/src/java/org/tensorflow/lite/task/core/vision/ImageProcessingOptions.java#L22) +for more details about how the underlying image processing is performed. + +```java +TensorImage inputImage = TensorImage.fromBitmap(bitmap); +int width = bitmap.getWidth(); +int height = bitmap.getHeight(); +int cropSize = min(width, height); +ImageProcessingOptions imageOptions = + ImageProcessingOptions.builder() + .setOrientation(getOrientation(sensorOrientation)) + // Set the ROI to the center of the image. + .setRoi( + new Rect( + /*left=*/ (width - cropSize) / 2, + /*top=*/ (height - cropSize) / 2, + /*right=*/ (width + cropSize) / 2, + /*bottom=*/ (height + cropSize) / 2)) + .build(); + +List results = imageClassifier.classify(inputImage, + imageOptions); +``` + +The output of `ImageClassifier` is a list of `Classifications` instance, where +each `Classifications` element is a single head classification result. All the +demo models are single head models, therefore, `results` only contains one +`Classifications` object. Use `Classifications.getCategories()` to get a list of +top-k categories as specified with `MAX_RESULTS`. Each `Category` object +contains the srting label and the score of that category. + +To match the implementation of +[`lib_support`](https://github.com/tensorflow/examples/tree/master/lite/examples/image_classification/android/lib_support), +`results` is converted into `List` in the method, +`getRecognitions`. + +#### Using the TensorFlow Lite Support Library + +##### Load model and create interpreter + +To perform inference, we need to load a model file and instantiate an +`Interpreter`. This happens in the constructor of the `Classifier` class, along +with loading the list of class labels. Information about the device type and +number of threads is used to configure the `Interpreter` via the +`Interpreter.Options` instance passed into its constructor. Note that if a GPU, +DSP (Digital Signal Processor) or NPU (Neural Processing Unit) is available, a +[`Delegate`](https://www.tensorflow.org/lite/performance/delegates) can be used +to take full advantage of these hardware. + +Please note that there are performance edge cases and developers are adviced to +test with a representative set of devices prior to production. + +```java +protected Classifier(Activity activity, Device device, int numThreads) throws + IOException { + tfliteModel = FileUtil.loadMappedFile(activity, getModelPath()); + switch (device) { + case NNAPI: + nnApiDelegate = new NnApiDelegate(); + tfliteOptions.addDelegate(nnApiDelegate); + break; + case GPU: + gpuDelegate = new GpuDelegate(); + tfliteOptions.addDelegate(gpuDelegate); + break; + case CPU: + break; + } + tfliteOptions.setNumThreads(numThreads); + tflite = new Interpreter(tfliteModel, tfliteOptions); + labels = FileUtil.loadLabels(activity, getLabelPath()); +... +``` + +For Android devices, we recommend pre-loading and memory mapping the model file +to offer faster load times and reduce the dirty pages in memory. The method +`FileUtil.loadMappedFile` does this, returning a `MappedByteBuffer` containing +the model. + +The `MappedByteBuffer` is passed into the `Interpreter` constructor, along with +an `Interpreter.Options` object. This object can be used to configure the +interpreter, for example by setting the number of threads (`.setNumThreads(1)`) +or enabling [NNAPI](https://developer.android.com/ndk/guides/neuralnetworks) +(`.addDelegate(nnApiDelegate)`). + +##### Pre-process bitmap image + +Next in the `Classifier` constructor, we take the input camera bitmap image, +convert it to a `TensorImage` format for efficient processing and pre-process +it. The steps are shown in the private 'loadImage' method: + +```java +/** Loads input image, and applys preprocessing. */ +private TensorImage loadImage(final Bitmap bitmap, int sensorOrientation) { + // Loads bitmap into a TensorImage. + image.load(bitmap); + + // Creates processor for the TensorImage. + int cropSize = Math.min(bitmap.getWidth(), bitmap.getHeight()); + int numRoration = sensorOrientation / 90; + ImageProcessor imageProcessor = + new ImageProcessor.Builder() + .add(new ResizeWithCropOrPadOp(cropSize, cropSize)) + .add(new ResizeOp(imageSizeX, imageSizeY, ResizeMethod.BILINEAR)) + .add(new Rot90Op(numRoration)) + .add(getPreprocessNormalizeOp()) + .build(); + return imageProcessor.process(inputImageBuffer); +} +``` + +The pre-processing is largely the same for quantized and float models with one +exception: Normalization. + +In `ClassifierFloatMobileNet`, the normalization parameters are defined as: + +```java +private static final float IMAGE_MEAN = 127.5f; +private static final float IMAGE_STD = 127.5f; +``` + +In `ClassifierQuantizedMobileNet`, normalization is not required. Thus the +nomalization parameters are defined as: + +```java +private static final float IMAGE_MEAN = 0.0f; +private static final float IMAGE_STD = 1.0f; +``` + +##### Allocate output object + +Initiate the output `TensorBuffer` for the output of the model. + +```java +/** Output probability TensorBuffer. */ +private final TensorBuffer outputProbabilityBuffer; + +//... +// Get the array size for the output buffer from the TensorFlow Lite model file +int probabilityTensorIndex = 0; +int[] probabilityShape = + tflite.getOutputTensor(probabilityTensorIndex).shape(); // {1, 1001} +DataType probabilityDataType = + tflite.getOutputTensor(probabilityTensorIndex).dataType(); + +// Creates the output tensor and its processor. +outputProbabilityBuffer = + TensorBuffer.createFixedSize(probabilityShape, probabilityDataType); + +// Creates the post processor for the output probability. +probabilityProcessor = + new TensorProcessor.Builder().add(getPostprocessNormalizeOp()).build(); +``` + +For quantized models, we need to de-quantize the prediction with the NormalizeOp +(as they are all essentially linear transformation). For float model, +de-quantize is not required. But to uniform the API, de-quantize is added to +float model too. Mean and std are set to 0.0f and 1.0f, respectively. To be more +specific, + +In `ClassifierQuantizedMobileNet`, the normalized parameters are defined as: + +```java +private static final float PROBABILITY_MEAN = 0.0f; +private static final float PROBABILITY_STD = 255.0f; +``` + +In `ClassifierFloatMobileNet`, the normalized parameters are defined as: + +```java +private static final float PROBABILITY_MEAN = 0.0f; +private static final float PROBABILITY_STD = 1.0f; +``` + +##### Run inference + +Inference is performed using the following in `Classifier` class: + +```java +tflite.run(inputImageBuffer.getBuffer(), + outputProbabilityBuffer.getBuffer().rewind()); +``` + +##### Recognize image + +Rather than call `run` directly, the method `recognizeImage` is used. It accepts +a bitmap and sensor orientation, runs inference, and returns a sorted `List` of +`Recognition` instances, each corresponding to a label. The method will return a +number of results bounded by `MAX_RESULTS`, which is 3 by default. + +`Recognition` is a simple class that contains information about a specific +recognition result, including its `title` and `confidence`. Using the +post-processing normalization method specified, the confidence is converted to +between 0 and 1 of a given class being represented by the image. + +```java +/** Gets the label to probability map. */ +Map labeledProbability = + new TensorLabel(labels, + probabilityProcessor.process(outputProbabilityBuffer)) + .getMapWithFloatValue(); +``` + +A `PriorityQueue` is used for sorting. + +```java +/** Gets the top-k results. */ +private static List getTopKProbability( + Map labelProb) { + // Find the best classifications. + PriorityQueue pq = + new PriorityQueue<>( + MAX_RESULTS, + new Comparator() { + @Override + public int compare(Recognition lhs, Recognition rhs) { + // Intentionally reversed to put high confidence at the head of + // the queue. + return Float.compare(rhs.getConfidence(), lhs.getConfidence()); + } + }); + + for (Map.Entry entry : labelProb.entrySet()) { + pq.add(new Recognition("" + entry.getKey(), entry.getKey(), + entry.getValue(), null)); + } + + final ArrayList recognitions = new ArrayList<>(); + int recognitionsSize = Math.min(pq.size(), MAX_RESULTS); + for (int i = 0; i < recognitionsSize; ++i) { + recognitions.add(pq.poll()); + } + return recognitions; +} +``` + +### Display results + +The classifier is invoked and inference results are displayed by the +`processImage()` function in +[`ClassifierActivity.java`](https://github.com/tensorflow/examples/tree/master/lite/examples/image_classification/android/app/src/main/java/org/tensorflow/lite/examples/classification/ClassifierActivity.java). + +`ClassifierActivity` is a subclass of `CameraActivity` that contains method +implementations that render the camera image, run classification, and display +the results. The method `processImage()` runs classification on a background +thread as fast as possible, rendering information on the UI thread to avoid +blocking inference and creating latency. + +```java +@Override +protected void processImage() { + rgbFrameBitmap.setPixels(getRgbBytes(), 0, previewWidth, 0, 0, previewWidth, + previewHeight); + final int imageSizeX = classifier.getImageSizeX(); + final int imageSizeY = classifier.getImageSizeY(); + + runInBackground( + new Runnable() { + @Override + public void run() { + if (classifier != null) { + final long startTime = SystemClock.uptimeMillis(); + final List results = + classifier.recognizeImage(rgbFrameBitmap, sensorOrientation); + lastProcessingTimeMs = SystemClock.uptimeMillis() - startTime; + LOGGER.v("Detect: %s", results); + + runOnUiThread( + new Runnable() { + @Override + public void run() { + showResultsInBottomSheet(results); + showFrameInfo(previewWidth + "x" + previewHeight); + showCropInfo(imageSizeX + "x" + imageSizeY); + showCameraResolution(imageSizeX + "x" + imageSizeY); + showRotationInfo(String.valueOf(sensorOrientation)); + showInference(lastProcessingTimeMs + "ms"); + } + }); + } + readyForNextImage(); + } + }); +} +``` + +Another important role of `ClassifierActivity` is to determine user preferences +(by interrogating `CameraActivity`), and instantiate the appropriately +configured `Classifier` subclass. This happens when the video feed begins (via +`onPreviewSizeChosen()`) and when options are changed in the UI (via +`onInferenceConfigurationChanged()`). + +```java +private void recreateClassifier(Model model, Device device, int numThreads) { + if (classifier != null) { + LOGGER.d("Closing classifier."); + classifier.close(); + classifier = null; + } + if (device == Device.GPU && model == Model.QUANTIZED) { + LOGGER.d("Not creating classifier: GPU doesn't support quantized models."); + runOnUiThread( + () -> { + Toast.makeText(this, "GPU does not yet supported quantized models.", + Toast.LENGTH_LONG) + .show(); + }); + return; + } + try { + LOGGER.d( + "Creating classifier (model=%s, device=%s, numThreads=%d)", model, + device, numThreads); + classifier = Classifier.create(this, model, device, numThreads); + } catch (IOException e) { + LOGGER.e(e, "Failed to create classifier."); + } +} +``` diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/LICENSE b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..6606ec028d1c629986e7019fe3564f5b4bfe425d --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Alexey + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/README.md b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/README.md new file mode 100644 index 0000000000000000000000000000000000000000..faf415eb27ccc1a62357718d1e0a9b8c746de4e8 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/README.md @@ -0,0 +1,21 @@ +# MiDaS on Android smartphone by using TensorFlow-lite (TFLite) + + +* Either use Android Studio for compilation. + +* Or use ready to install apk-file: + * Or use URL: https://i.diawi.com/CVb8a9 + * Or use QR-code: + +Scan QR-code or open URL -> Press `Install application` -> Press `Download` and wait for download -> Open -> Install -> Open -> Press: Allow MiDaS to take photo and video from the camera While using the APP + +![CVb8a9](https://user-images.githubusercontent.com/4096485/97727213-38552500-1ae1-11eb-8b76-4ea11216f76d.png) + +---- + +To use another model, you should convert it to `model_opt.tflite` and place it to the directory: `models\src\main\assets` + + +---- + +Original repository: https://github.com/isl-org/MiDaS diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/build.gradle b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/build.gradle new file mode 100644 index 0000000000000000000000000000000000000000..94e9886a55c7d54f71b424bb246c849dd6bd795d --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/build.gradle @@ -0,0 +1,56 @@ +apply plugin: 'com.android.application' + +android { + compileSdkVersion 28 + defaultConfig { + applicationId "org.tensorflow.lite.examples.classification" + minSdkVersion 21 + targetSdkVersion 28 + versionCode 1 + versionName "1.0" + + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + } + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' + } + } + aaptOptions { + noCompress "tflite" + } + compileOptions { + sourceCompatibility = '1.8' + targetCompatibility = '1.8' + } + lintOptions { + abortOnError false + } + flavorDimensions "tfliteInference" + productFlavors { + // The TFLite inference is built using the TFLite Support library. + support { + dimension "tfliteInference" + } + // The TFLite inference is built using the TFLite Task library. + taskApi { + dimension "tfliteInference" + } + } + +} + +dependencies { + implementation fileTree(dir: 'libs', include: ['*.jar']) + supportImplementation project(":lib_support") + taskApiImplementation project(":lib_task_api") + implementation 'androidx.appcompat:appcompat:1.0.0' + implementation 'androidx.coordinatorlayout:coordinatorlayout:1.0.0' + implementation 'com.google.android.material:material:1.0.0' + + androidTestImplementation 'androidx.test.ext:junit:1.1.1' + androidTestImplementation 'com.google.truth:truth:1.0.1' + androidTestImplementation 'androidx.test:runner:1.2.0' + androidTestImplementation 'androidx.test:rules:1.1.0' +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/proguard-rules.pro b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/proguard-rules.pro new file mode 100644 index 0000000000000000000000000000000000000000..f1b424510da51fd82143bc74a0a801ae5a1e2fcd --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/androidTest/assets/fox-mobilenet_v1_1.0_224_support.txt b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/androidTest/assets/fox-mobilenet_v1_1.0_224_support.txt new file mode 100644 index 0000000000000000000000000000000000000000..bdfad31f9b3e694817025d8b8f2ca0b40aa436bb --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/androidTest/assets/fox-mobilenet_v1_1.0_224_support.txt @@ -0,0 +1,3 @@ +red_fox 0.79403335 +kit_fox 0.16753247 +grey_fox 0.03619214 diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/androidTest/assets/fox-mobilenet_v1_1.0_224_task_api.txt b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/androidTest/assets/fox-mobilenet_v1_1.0_224_task_api.txt new file mode 100644 index 0000000000000000000000000000000000000000..3668ce54df0d1e57e31c58281d6085b83928f991 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/androidTest/assets/fox-mobilenet_v1_1.0_224_task_api.txt @@ -0,0 +1,3 @@ +red_fox 0.85 +kit_fox 0.13 +grey_fox 0.02 diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/androidTest/java/AndroidManifest.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/androidTest/java/AndroidManifest.xml new file mode 100644 index 0000000000000000000000000000000000000000..3653d8799092492ebbb16c7c956eb50e3d404aa4 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/androidTest/java/AndroidManifest.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/androidTest/java/org/tensorflow/lite/examples/classification/ClassifierTest.java b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/androidTest/java/org/tensorflow/lite/examples/classification/ClassifierTest.java new file mode 100644 index 0000000000000000000000000000000000000000..0194132890aae659c2a70d33106306ed665b22e8 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/androidTest/java/org/tensorflow/lite/examples/classification/ClassifierTest.java @@ -0,0 +1,121 @@ +/* + * Copyright 2019 The TensorFlow Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.tensorflow.lite.examples.classification; + +import static com.google.common.truth.Truth.assertThat; + +import android.content.res.AssetManager; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.util.Log; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.platform.app.InstrumentationRegistry; +import androidx.test.rule.ActivityTestRule; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Scanner; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.tensorflow.lite.examples.classification.tflite.Classifier; +import org.tensorflow.lite.examples.classification.tflite.Classifier.Device; +import org.tensorflow.lite.examples.classification.tflite.Classifier.Model; +import org.tensorflow.lite.examples.classification.tflite.Classifier.Recognition; + +/** Golden test for Image Classification Reference app. */ +@RunWith(AndroidJUnit4.class) +public class ClassifierTest { + + @Rule + public ActivityTestRule rule = + new ActivityTestRule<>(ClassifierActivity.class); + + private static final String[] INPUTS = {"fox.jpg"}; + private static final String[] GOLDEN_OUTPUTS_SUPPORT = {"fox-mobilenet_v1_1.0_224_support.txt"}; + private static final String[] GOLDEN_OUTPUTS_TASK = {"fox-mobilenet_v1_1.0_224_task_api.txt"}; + + @Test + public void classificationResultsShouldNotChange() throws IOException { + ClassifierActivity activity = rule.getActivity(); + Classifier classifier = Classifier.create(activity, Model.FLOAT_MOBILENET, Device.CPU, 1); + for (int i = 0; i < INPUTS.length; i++) { + String imageFileName = INPUTS[i]; + String goldenOutputFileName; + // TODO(b/169379396): investigate the impact of the resize algorithm on accuracy. + // This is a temporary workaround to set different golden rest results as the preprocessing + // of lib_support and lib_task_api are different. Will merge them once the above TODO is + // resolved. + if (Classifier.TAG.equals("ClassifierWithSupport")) { + goldenOutputFileName = GOLDEN_OUTPUTS_SUPPORT[i]; + } else { + goldenOutputFileName = GOLDEN_OUTPUTS_TASK[i]; + } + Bitmap input = loadImage(imageFileName); + List goldenOutput = loadRecognitions(goldenOutputFileName); + + List result = classifier.recognizeImage(input, 0); + Iterator goldenOutputIterator = goldenOutput.iterator(); + + for (Recognition actual : result) { + Assert.assertTrue(goldenOutputIterator.hasNext()); + Recognition expected = goldenOutputIterator.next(); + assertThat(actual.getTitle()).isEqualTo(expected.getTitle()); + assertThat(actual.getConfidence()).isWithin(0.01f).of(expected.getConfidence()); + } + } + } + + private static Bitmap loadImage(String fileName) { + AssetManager assetManager = + InstrumentationRegistry.getInstrumentation().getContext().getAssets(); + InputStream inputStream = null; + try { + inputStream = assetManager.open(fileName); + } catch (IOException e) { + Log.e("Test", "Cannot load image from assets"); + } + return BitmapFactory.decodeStream(inputStream); + } + + private static List loadRecognitions(String fileName) { + AssetManager assetManager = + InstrumentationRegistry.getInstrumentation().getContext().getAssets(); + InputStream inputStream = null; + try { + inputStream = assetManager.open(fileName); + } catch (IOException e) { + Log.e("Test", "Cannot load probability results from assets"); + } + Scanner scanner = new Scanner(inputStream); + List result = new ArrayList<>(); + while (scanner.hasNext()) { + String category = scanner.next(); + category = category.replace('_', ' '); + if (!scanner.hasNextFloat()) { + break; + } + float probability = scanner.nextFloat(); + Recognition recognition = new Recognition(null, category, probability, null); + result.add(recognition); + } + return result; + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/AndroidManifest.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/AndroidManifest.xml new file mode 100644 index 0000000000000000000000000000000000000000..7a414d5176a117262dce56c2220e6b71791287de --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/AndroidManifest.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/CameraActivity.java b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/CameraActivity.java new file mode 100644 index 0000000000000000000000000000000000000000..d1eb26c862c04bf573ecc4eb127e7460f0b100fc --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/CameraActivity.java @@ -0,0 +1,717 @@ +/* + * Copyright 2019 The TensorFlow Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.tensorflow.lite.examples.classification; + +import android.Manifest; +import android.app.Fragment; +import android.content.Context; +import android.content.pm.PackageManager; +import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.graphics.Color; +import android.graphics.Paint; +import android.graphics.RectF; +import android.hardware.Camera; +import android.hardware.camera2.CameraAccessException; +import android.hardware.camera2.CameraCharacteristics; +import android.hardware.camera2.CameraManager; +import android.hardware.camera2.params.StreamConfigurationMap; +import android.media.Image; +import android.media.Image.Plane; +import android.media.ImageReader; +import android.media.ImageReader.OnImageAvailableListener; +import android.os.Build; +import android.os.Bundle; +import android.os.Handler; +import android.os.HandlerThread; +import android.os.Trace; +import androidx.annotation.NonNull; +import androidx.annotation.UiThread; +import androidx.appcompat.app.AppCompatActivity; +import android.util.Size; +import android.view.Surface; +import android.view.TextureView; +import android.view.View; +import android.view.ViewTreeObserver; +import android.view.WindowManager; +import android.widget.AdapterView; +import android.widget.ImageView; +import android.widget.LinearLayout; +import android.widget.Spinner; +import android.widget.TextView; +import android.widget.Toast; +import com.google.android.material.bottomsheet.BottomSheetBehavior; +import java.nio.ByteBuffer; +import java.util.List; +import org.tensorflow.lite.examples.classification.env.ImageUtils; +import org.tensorflow.lite.examples.classification.env.Logger; +import org.tensorflow.lite.examples.classification.tflite.Classifier.Device; +import org.tensorflow.lite.examples.classification.tflite.Classifier.Model; +import org.tensorflow.lite.examples.classification.tflite.Classifier.Recognition; + +public abstract class CameraActivity extends AppCompatActivity + implements OnImageAvailableListener, + Camera.PreviewCallback, + View.OnClickListener, + AdapterView.OnItemSelectedListener { + private static final Logger LOGGER = new Logger(); + + private static final int PERMISSIONS_REQUEST = 1; + + private static final String PERMISSION_CAMERA = Manifest.permission.CAMERA; + protected int previewWidth = 0; + protected int previewHeight = 0; + private Handler handler; + private HandlerThread handlerThread; + private boolean useCamera2API; + private boolean isProcessingFrame = false; + private byte[][] yuvBytes = new byte[3][]; + private int[] rgbBytes = null; + private int yRowStride; + private Runnable postInferenceCallback; + private Runnable imageConverter; + private LinearLayout bottomSheetLayout; + private LinearLayout gestureLayout; + private BottomSheetBehavior sheetBehavior; + protected TextView recognitionTextView, + recognition1TextView, + recognition2TextView, + recognitionValueTextView, + recognition1ValueTextView, + recognition2ValueTextView; + protected TextView frameValueTextView, + cropValueTextView, + cameraResolutionTextView, + rotationTextView, + inferenceTimeTextView; + protected ImageView bottomSheetArrowImageView; + private ImageView plusImageView, minusImageView; + private Spinner modelSpinner; + private Spinner deviceSpinner; + private TextView threadsTextView; + + //private Model model = Model.QUANTIZED_EFFICIENTNET; + //private Device device = Device.CPU; + private Model model = Model.FLOAT_EFFICIENTNET; + private Device device = Device.GPU; + private int numThreads = -1; + + @Override + protected void onCreate(final Bundle savedInstanceState) { + LOGGER.d("onCreate " + this); + super.onCreate(null); + getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); + + setContentView(R.layout.tfe_ic_activity_camera); + + if (hasPermission()) { + setFragment(); + } else { + requestPermission(); + } + + threadsTextView = findViewById(R.id.threads); + plusImageView = findViewById(R.id.plus); + minusImageView = findViewById(R.id.minus); + modelSpinner = findViewById(R.id.model_spinner); + deviceSpinner = findViewById(R.id.device_spinner); + bottomSheetLayout = findViewById(R.id.bottom_sheet_layout); + gestureLayout = findViewById(R.id.gesture_layout); + sheetBehavior = BottomSheetBehavior.from(bottomSheetLayout); + bottomSheetArrowImageView = findViewById(R.id.bottom_sheet_arrow); + + ViewTreeObserver vto = gestureLayout.getViewTreeObserver(); + vto.addOnGlobalLayoutListener( + new ViewTreeObserver.OnGlobalLayoutListener() { + @Override + public void onGlobalLayout() { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.JELLY_BEAN) { + gestureLayout.getViewTreeObserver().removeGlobalOnLayoutListener(this); + } else { + gestureLayout.getViewTreeObserver().removeOnGlobalLayoutListener(this); + } + // int width = bottomSheetLayout.getMeasuredWidth(); + int height = gestureLayout.getMeasuredHeight(); + + sheetBehavior.setPeekHeight(height); + } + }); + sheetBehavior.setHideable(false); + + sheetBehavior.setBottomSheetCallback( + new BottomSheetBehavior.BottomSheetCallback() { + @Override + public void onStateChanged(@NonNull View bottomSheet, int newState) { + switch (newState) { + case BottomSheetBehavior.STATE_HIDDEN: + break; + case BottomSheetBehavior.STATE_EXPANDED: + { + bottomSheetArrowImageView.setImageResource(R.drawable.icn_chevron_down); + } + break; + case BottomSheetBehavior.STATE_COLLAPSED: + { + bottomSheetArrowImageView.setImageResource(R.drawable.icn_chevron_up); + } + break; + case BottomSheetBehavior.STATE_DRAGGING: + break; + case BottomSheetBehavior.STATE_SETTLING: + bottomSheetArrowImageView.setImageResource(R.drawable.icn_chevron_up); + break; + } + } + + @Override + public void onSlide(@NonNull View bottomSheet, float slideOffset) {} + }); + + recognitionTextView = findViewById(R.id.detected_item); + recognitionValueTextView = findViewById(R.id.detected_item_value); + recognition1TextView = findViewById(R.id.detected_item1); + recognition1ValueTextView = findViewById(R.id.detected_item1_value); + recognition2TextView = findViewById(R.id.detected_item2); + recognition2ValueTextView = findViewById(R.id.detected_item2_value); + + frameValueTextView = findViewById(R.id.frame_info); + cropValueTextView = findViewById(R.id.crop_info); + cameraResolutionTextView = findViewById(R.id.view_info); + rotationTextView = findViewById(R.id.rotation_info); + inferenceTimeTextView = findViewById(R.id.inference_info); + + modelSpinner.setOnItemSelectedListener(this); + deviceSpinner.setOnItemSelectedListener(this); + + plusImageView.setOnClickListener(this); + minusImageView.setOnClickListener(this); + + model = Model.valueOf(modelSpinner.getSelectedItem().toString().toUpperCase()); + device = Device.valueOf(deviceSpinner.getSelectedItem().toString()); + numThreads = Integer.parseInt(threadsTextView.getText().toString().trim()); + } + + protected int[] getRgbBytes() { + imageConverter.run(); + return rgbBytes; + } + + protected int getLuminanceStride() { + return yRowStride; + } + + protected byte[] getLuminance() { + return yuvBytes[0]; + } + + /** Callback for android.hardware.Camera API */ + @Override + public void onPreviewFrame(final byte[] bytes, final Camera camera) { + if (isProcessingFrame) { + LOGGER.w("Dropping frame!"); + return; + } + + try { + // Initialize the storage bitmaps once when the resolution is known. + if (rgbBytes == null) { + Camera.Size previewSize = camera.getParameters().getPreviewSize(); + previewHeight = previewSize.height; + previewWidth = previewSize.width; + rgbBytes = new int[previewWidth * previewHeight]; + onPreviewSizeChosen(new Size(previewSize.width, previewSize.height), 90); + } + } catch (final Exception e) { + LOGGER.e(e, "Exception!"); + return; + } + + isProcessingFrame = true; + yuvBytes[0] = bytes; + yRowStride = previewWidth; + + imageConverter = + new Runnable() { + @Override + public void run() { + ImageUtils.convertYUV420SPToARGB8888(bytes, previewWidth, previewHeight, rgbBytes); + } + }; + + postInferenceCallback = + new Runnable() { + @Override + public void run() { + camera.addCallbackBuffer(bytes); + isProcessingFrame = false; + } + }; + processImage(); + } + + /** Callback for Camera2 API */ + @Override + public void onImageAvailable(final ImageReader reader) { + // We need wait until we have some size from onPreviewSizeChosen + if (previewWidth == 0 || previewHeight == 0) { + return; + } + if (rgbBytes == null) { + rgbBytes = new int[previewWidth * previewHeight]; + } + try { + final Image image = reader.acquireLatestImage(); + + if (image == null) { + return; + } + + if (isProcessingFrame) { + image.close(); + return; + } + isProcessingFrame = true; + Trace.beginSection("imageAvailable"); + final Plane[] planes = image.getPlanes(); + fillBytes(planes, yuvBytes); + yRowStride = planes[0].getRowStride(); + final int uvRowStride = planes[1].getRowStride(); + final int uvPixelStride = planes[1].getPixelStride(); + + imageConverter = + new Runnable() { + @Override + public void run() { + ImageUtils.convertYUV420ToARGB8888( + yuvBytes[0], + yuvBytes[1], + yuvBytes[2], + previewWidth, + previewHeight, + yRowStride, + uvRowStride, + uvPixelStride, + rgbBytes); + } + }; + + postInferenceCallback = + new Runnable() { + @Override + public void run() { + image.close(); + isProcessingFrame = false; + } + }; + + processImage(); + } catch (final Exception e) { + LOGGER.e(e, "Exception!"); + Trace.endSection(); + return; + } + Trace.endSection(); + } + + @Override + public synchronized void onStart() { + LOGGER.d("onStart " + this); + super.onStart(); + } + + @Override + public synchronized void onResume() { + LOGGER.d("onResume " + this); + super.onResume(); + + handlerThread = new HandlerThread("inference"); + handlerThread.start(); + handler = new Handler(handlerThread.getLooper()); + } + + @Override + public synchronized void onPause() { + LOGGER.d("onPause " + this); + + handlerThread.quitSafely(); + try { + handlerThread.join(); + handlerThread = null; + handler = null; + } catch (final InterruptedException e) { + LOGGER.e(e, "Exception!"); + } + + super.onPause(); + } + + @Override + public synchronized void onStop() { + LOGGER.d("onStop " + this); + super.onStop(); + } + + @Override + public synchronized void onDestroy() { + LOGGER.d("onDestroy " + this); + super.onDestroy(); + } + + protected synchronized void runInBackground(final Runnable r) { + if (handler != null) { + handler.post(r); + } + } + + @Override + public void onRequestPermissionsResult( + final int requestCode, final String[] permissions, final int[] grantResults) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults); + if (requestCode == PERMISSIONS_REQUEST) { + if (allPermissionsGranted(grantResults)) { + setFragment(); + } else { + requestPermission(); + } + } + } + + private static boolean allPermissionsGranted(final int[] grantResults) { + for (int result : grantResults) { + if (result != PackageManager.PERMISSION_GRANTED) { + return false; + } + } + return true; + } + + private boolean hasPermission() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + return checkSelfPermission(PERMISSION_CAMERA) == PackageManager.PERMISSION_GRANTED; + } else { + return true; + } + } + + private void requestPermission() { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { + if (shouldShowRequestPermissionRationale(PERMISSION_CAMERA)) { + Toast.makeText( + CameraActivity.this, + "Camera permission is required for this demo", + Toast.LENGTH_LONG) + .show(); + } + requestPermissions(new String[] {PERMISSION_CAMERA}, PERMISSIONS_REQUEST); + } + } + + // Returns true if the device supports the required hardware level, or better. + private boolean isHardwareLevelSupported( + CameraCharacteristics characteristics, int requiredLevel) { + int deviceLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL); + if (deviceLevel == CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY) { + return requiredLevel == deviceLevel; + } + // deviceLevel is not LEGACY, can use numerical sort + return requiredLevel <= deviceLevel; + } + + private String chooseCamera() { + final CameraManager manager = (CameraManager) getSystemService(Context.CAMERA_SERVICE); + try { + for (final String cameraId : manager.getCameraIdList()) { + final CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraId); + + // We don't use a front facing camera in this sample. + final Integer facing = characteristics.get(CameraCharacteristics.LENS_FACING); + if (facing != null && facing == CameraCharacteristics.LENS_FACING_FRONT) { + continue; + } + + final StreamConfigurationMap map = + characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); + + if (map == null) { + continue; + } + + // Fallback to camera1 API for internal cameras that don't have full support. + // This should help with legacy situations where using the camera2 API causes + // distorted or otherwise broken previews. + useCamera2API = + (facing == CameraCharacteristics.LENS_FACING_EXTERNAL) + || isHardwareLevelSupported( + characteristics, CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_FULL); + LOGGER.i("Camera API lv2?: %s", useCamera2API); + return cameraId; + } + } catch (CameraAccessException e) { + LOGGER.e(e, "Not allowed to access camera"); + } + + return null; + } + + protected void setFragment() { + String cameraId = chooseCamera(); + + Fragment fragment; + if (useCamera2API) { + CameraConnectionFragment camera2Fragment = + CameraConnectionFragment.newInstance( + new CameraConnectionFragment.ConnectionCallback() { + @Override + public void onPreviewSizeChosen(final Size size, final int rotation) { + previewHeight = size.getHeight(); + previewWidth = size.getWidth(); + CameraActivity.this.onPreviewSizeChosen(size, rotation); + } + }, + this, + getLayoutId(), + getDesiredPreviewFrameSize()); + + camera2Fragment.setCamera(cameraId); + fragment = camera2Fragment; + } else { + fragment = + new LegacyCameraConnectionFragment(this, getLayoutId(), getDesiredPreviewFrameSize()); + } + + getFragmentManager().beginTransaction().replace(R.id.container, fragment).commit(); + } + + protected void fillBytes(final Plane[] planes, final byte[][] yuvBytes) { + // Because of the variable row stride it's not possible to know in + // advance the actual necessary dimensions of the yuv planes. + for (int i = 0; i < planes.length; ++i) { + final ByteBuffer buffer = planes[i].getBuffer(); + if (yuvBytes[i] == null) { + LOGGER.d("Initializing buffer %d at size %d", i, buffer.capacity()); + yuvBytes[i] = new byte[buffer.capacity()]; + } + buffer.get(yuvBytes[i]); + } + } + + protected void readyForNextImage() { + if (postInferenceCallback != null) { + postInferenceCallback.run(); + } + } + + protected int getScreenOrientation() { + switch (getWindowManager().getDefaultDisplay().getRotation()) { + case Surface.ROTATION_270: + return 270; + case Surface.ROTATION_180: + return 180; + case Surface.ROTATION_90: + return 90; + default: + return 0; + } + } + + @UiThread + protected void showResultsInTexture(float[] img_array, int imageSizeX, int imageSizeY) { + float maxval = Float.NEGATIVE_INFINITY; + float minval = Float.POSITIVE_INFINITY; + for (float cur : img_array) { + maxval = Math.max(maxval, cur); + minval = Math.min(minval, cur); + } + float multiplier = 0; + if ((maxval - minval) > 0) multiplier = 255 / (maxval - minval); + + int[] img_normalized = new int[img_array.length]; + for (int i = 0; i < img_array.length; ++i) { + float val = (float) (multiplier * (img_array[i] - minval)); + img_normalized[i] = (int) val; + } + + + + TextureView textureView = findViewById(R.id.textureView3); + //AutoFitTextureView textureView = (AutoFitTextureView) findViewById(R.id.texture); + + if(textureView.isAvailable()) { + int width = imageSizeX; + int height = imageSizeY; + + Canvas canvas = textureView.lockCanvas(); + canvas.drawColor(Color.BLUE); + Paint paint = new Paint(); + paint.setStyle(Paint.Style.FILL); + paint.setARGB(255, 150, 150, 150); + + int canvas_size = Math.min(canvas.getWidth(), canvas.getHeight()); + + Bitmap bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.RGB_565); + + for (int ii = 0; ii < width; ii++) //pass the screen pixels in 2 directions + { + for (int jj = 0; jj < height; jj++) { + //int val = img_normalized[ii + jj * width]; + int index = (width - ii - 1) + (height - jj - 1) * width; + if(index < img_array.length) { + int val = img_normalized[index]; + bitmap.setPixel(ii, jj, Color.rgb(val, val, val)); + } + } + } + + canvas.drawBitmap(bitmap, null, new RectF(0, 0, canvas_size, canvas_size), null); + + textureView.unlockCanvasAndPost(canvas); + + } + + } + + protected void showResultsInBottomSheet(List results) { + if (results != null && results.size() >= 3) { + Recognition recognition = results.get(0); + if (recognition != null) { + if (recognition.getTitle() != null) recognitionTextView.setText(recognition.getTitle()); + if (recognition.getConfidence() != null) + recognitionValueTextView.setText( + String.format("%.2f", (100 * recognition.getConfidence())) + "%"); + } + + Recognition recognition1 = results.get(1); + if (recognition1 != null) { + if (recognition1.getTitle() != null) recognition1TextView.setText(recognition1.getTitle()); + if (recognition1.getConfidence() != null) + recognition1ValueTextView.setText( + String.format("%.2f", (100 * recognition1.getConfidence())) + "%"); + } + + Recognition recognition2 = results.get(2); + if (recognition2 != null) { + if (recognition2.getTitle() != null) recognition2TextView.setText(recognition2.getTitle()); + if (recognition2.getConfidence() != null) + recognition2ValueTextView.setText( + String.format("%.2f", (100 * recognition2.getConfidence())) + "%"); + } + } + } + + protected void showFrameInfo(String frameInfo) { + frameValueTextView.setText(frameInfo); + } + + protected void showCropInfo(String cropInfo) { + cropValueTextView.setText(cropInfo); + } + + protected void showCameraResolution(String cameraInfo) { + cameraResolutionTextView.setText(cameraInfo); + } + + protected void showRotationInfo(String rotation) { + rotationTextView.setText(rotation); + } + + protected void showInference(String inferenceTime) { + inferenceTimeTextView.setText(inferenceTime); + } + + protected Model getModel() { + return model; + } + + private void setModel(Model model) { + if (this.model != model) { + LOGGER.d("Updating model: " + model); + this.model = model; + onInferenceConfigurationChanged(); + } + } + + protected Device getDevice() { + return device; + } + + private void setDevice(Device device) { + if (this.device != device) { + LOGGER.d("Updating device: " + device); + this.device = device; + final boolean threadsEnabled = device == Device.CPU; + plusImageView.setEnabled(threadsEnabled); + minusImageView.setEnabled(threadsEnabled); + threadsTextView.setText(threadsEnabled ? String.valueOf(numThreads) : "N/A"); + onInferenceConfigurationChanged(); + } + } + + protected int getNumThreads() { + return numThreads; + } + + private void setNumThreads(int numThreads) { + if (this.numThreads != numThreads) { + LOGGER.d("Updating numThreads: " + numThreads); + this.numThreads = numThreads; + onInferenceConfigurationChanged(); + } + } + + protected abstract void processImage(); + + protected abstract void onPreviewSizeChosen(final Size size, final int rotation); + + protected abstract int getLayoutId(); + + protected abstract Size getDesiredPreviewFrameSize(); + + protected abstract void onInferenceConfigurationChanged(); + + @Override + public void onClick(View v) { + if (v.getId() == R.id.plus) { + String threads = threadsTextView.getText().toString().trim(); + int numThreads = Integer.parseInt(threads); + if (numThreads >= 9) return; + setNumThreads(++numThreads); + threadsTextView.setText(String.valueOf(numThreads)); + } else if (v.getId() == R.id.minus) { + String threads = threadsTextView.getText().toString().trim(); + int numThreads = Integer.parseInt(threads); + if (numThreads == 1) { + return; + } + setNumThreads(--numThreads); + threadsTextView.setText(String.valueOf(numThreads)); + } + } + + @Override + public void onItemSelected(AdapterView parent, View view, int pos, long id) { + if (parent == modelSpinner) { + setModel(Model.valueOf(parent.getItemAtPosition(pos).toString().toUpperCase())); + } else if (parent == deviceSpinner) { + setDevice(Device.valueOf(parent.getItemAtPosition(pos).toString())); + } + } + + @Override + public void onNothingSelected(AdapterView parent) { + // Do nothing. + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/CameraConnectionFragment.java b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/CameraConnectionFragment.java new file mode 100644 index 0000000000000000000000000000000000000000..13e5c0dc341a86b1ddd66c4b562e0bf767641b42 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/CameraConnectionFragment.java @@ -0,0 +1,575 @@ +/* + * Copyright 2019 The TensorFlow Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.tensorflow.lite.examples.classification; + +import android.annotation.SuppressLint; +import android.app.Activity; +import android.app.AlertDialog; +import android.app.Dialog; +import android.app.DialogFragment; +import android.app.Fragment; +import android.content.Context; +import android.content.DialogInterface; +import android.content.res.Configuration; +import android.graphics.ImageFormat; +import android.graphics.Matrix; +import android.graphics.RectF; +import android.graphics.SurfaceTexture; +import android.hardware.camera2.CameraAccessException; +import android.hardware.camera2.CameraCaptureSession; +import android.hardware.camera2.CameraCharacteristics; +import android.hardware.camera2.CameraDevice; +import android.hardware.camera2.CameraManager; +import android.hardware.camera2.CaptureRequest; +import android.hardware.camera2.CaptureResult; +import android.hardware.camera2.TotalCaptureResult; +import android.hardware.camera2.params.StreamConfigurationMap; +import android.media.ImageReader; +import android.media.ImageReader.OnImageAvailableListener; +import android.os.Bundle; +import android.os.Handler; +import android.os.HandlerThread; +import android.text.TextUtils; +import android.util.Size; +import android.util.SparseIntArray; +import android.view.LayoutInflater; +import android.view.Surface; +import android.view.TextureView; +import android.view.View; +import android.view.ViewGroup; +import android.widget.Toast; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; +import org.tensorflow.lite.examples.classification.customview.AutoFitTextureView; +import org.tensorflow.lite.examples.classification.env.Logger; + +/** + * Camera Connection Fragment that captures images from camera. + * + *

Instantiated by newInstance.

+ */ +@SuppressWarnings("FragmentNotInstantiable") +public class CameraConnectionFragment extends Fragment { + private static final Logger LOGGER = new Logger(); + + /** + * The camera preview size will be chosen to be the smallest frame by pixel size capable of + * containing a DESIRED_SIZE x DESIRED_SIZE square. + */ + private static final int MINIMUM_PREVIEW_SIZE = 320; + + /** Conversion from screen rotation to JPEG orientation. */ + private static final SparseIntArray ORIENTATIONS = new SparseIntArray(); + + private static final String FRAGMENT_DIALOG = "dialog"; + + static { + ORIENTATIONS.append(Surface.ROTATION_0, 90); + ORIENTATIONS.append(Surface.ROTATION_90, 0); + ORIENTATIONS.append(Surface.ROTATION_180, 270); + ORIENTATIONS.append(Surface.ROTATION_270, 180); + } + + /** A {@link Semaphore} to prevent the app from exiting before closing the camera. */ + private final Semaphore cameraOpenCloseLock = new Semaphore(1); + /** A {@link OnImageAvailableListener} to receive frames as they are available. */ + private final OnImageAvailableListener imageListener; + /** The input size in pixels desired by TensorFlow (width and height of a square bitmap). */ + private final Size inputSize; + /** The layout identifier to inflate for this Fragment. */ + private final int layout; + + private final ConnectionCallback cameraConnectionCallback; + private final CameraCaptureSession.CaptureCallback captureCallback = + new CameraCaptureSession.CaptureCallback() { + @Override + public void onCaptureProgressed( + final CameraCaptureSession session, + final CaptureRequest request, + final CaptureResult partialResult) {} + + @Override + public void onCaptureCompleted( + final CameraCaptureSession session, + final CaptureRequest request, + final TotalCaptureResult result) {} + }; + /** ID of the current {@link CameraDevice}. */ + private String cameraId; + /** An {@link AutoFitTextureView} for camera preview. */ + private AutoFitTextureView textureView; + /** A {@link CameraCaptureSession } for camera preview. */ + private CameraCaptureSession captureSession; + /** A reference to the opened {@link CameraDevice}. */ + private CameraDevice cameraDevice; + /** The rotation in degrees of the camera sensor from the display. */ + private Integer sensorOrientation; + /** The {@link Size} of camera preview. */ + private Size previewSize; + /** An additional thread for running tasks that shouldn't block the UI. */ + private HandlerThread backgroundThread; + /** A {@link Handler} for running tasks in the background. */ + private Handler backgroundHandler; + /** + * {@link TextureView.SurfaceTextureListener} handles several lifecycle events on a {@link + * TextureView}. + */ + private final TextureView.SurfaceTextureListener surfaceTextureListener = + new TextureView.SurfaceTextureListener() { + @Override + public void onSurfaceTextureAvailable( + final SurfaceTexture texture, final int width, final int height) { + openCamera(width, height); + } + + @Override + public void onSurfaceTextureSizeChanged( + final SurfaceTexture texture, final int width, final int height) { + configureTransform(width, height); + } + + @Override + public boolean onSurfaceTextureDestroyed(final SurfaceTexture texture) { + return true; + } + + @Override + public void onSurfaceTextureUpdated(final SurfaceTexture texture) {} + }; + /** An {@link ImageReader} that handles preview frame capture. */ + private ImageReader previewReader; + /** {@link CaptureRequest.Builder} for the camera preview */ + private CaptureRequest.Builder previewRequestBuilder; + /** {@link CaptureRequest} generated by {@link #previewRequestBuilder} */ + private CaptureRequest previewRequest; + /** {@link CameraDevice.StateCallback} is called when {@link CameraDevice} changes its state. */ + private final CameraDevice.StateCallback stateCallback = + new CameraDevice.StateCallback() { + @Override + public void onOpened(final CameraDevice cd) { + // This method is called when the camera is opened. We start camera preview here. + cameraOpenCloseLock.release(); + cameraDevice = cd; + createCameraPreviewSession(); + } + + @Override + public void onDisconnected(final CameraDevice cd) { + cameraOpenCloseLock.release(); + cd.close(); + cameraDevice = null; + } + + @Override + public void onError(final CameraDevice cd, final int error) { + cameraOpenCloseLock.release(); + cd.close(); + cameraDevice = null; + final Activity activity = getActivity(); + if (null != activity) { + activity.finish(); + } + } + }; + + @SuppressLint("ValidFragment") + private CameraConnectionFragment( + final ConnectionCallback connectionCallback, + final OnImageAvailableListener imageListener, + final int layout, + final Size inputSize) { + this.cameraConnectionCallback = connectionCallback; + this.imageListener = imageListener; + this.layout = layout; + this.inputSize = inputSize; + } + + /** + * Given {@code choices} of {@code Size}s supported by a camera, chooses the smallest one whose + * width and height are at least as large as the minimum of both, or an exact match if possible. + * + * @param choices The list of sizes that the camera supports for the intended output class + * @param width The minimum desired width + * @param height The minimum desired height + * @return The optimal {@code Size}, or an arbitrary one if none were big enough + */ + protected static Size chooseOptimalSize(final Size[] choices, final int width, final int height) { + final int minSize = Math.max(Math.min(width, height), MINIMUM_PREVIEW_SIZE); + final Size desiredSize = new Size(width, height); + + // Collect the supported resolutions that are at least as big as the preview Surface + boolean exactSizeFound = false; + final List bigEnough = new ArrayList(); + final List tooSmall = new ArrayList(); + for (final Size option : choices) { + if (option.equals(desiredSize)) { + // Set the size but don't return yet so that remaining sizes will still be logged. + exactSizeFound = true; + } + + if (option.getHeight() >= minSize && option.getWidth() >= minSize) { + bigEnough.add(option); + } else { + tooSmall.add(option); + } + } + + LOGGER.i("Desired size: " + desiredSize + ", min size: " + minSize + "x" + minSize); + LOGGER.i("Valid preview sizes: [" + TextUtils.join(", ", bigEnough) + "]"); + LOGGER.i("Rejected preview sizes: [" + TextUtils.join(", ", tooSmall) + "]"); + + if (exactSizeFound) { + LOGGER.i("Exact size match found."); + return desiredSize; + } + + // Pick the smallest of those, assuming we found any + if (bigEnough.size() > 0) { + final Size chosenSize = Collections.min(bigEnough, new CompareSizesByArea()); + LOGGER.i("Chosen size: " + chosenSize.getWidth() + "x" + chosenSize.getHeight()); + return chosenSize; + } else { + LOGGER.e("Couldn't find any suitable preview size"); + return choices[0]; + } + } + + public static CameraConnectionFragment newInstance( + final ConnectionCallback callback, + final OnImageAvailableListener imageListener, + final int layout, + final Size inputSize) { + return new CameraConnectionFragment(callback, imageListener, layout, inputSize); + } + + /** + * Shows a {@link Toast} on the UI thread. + * + * @param text The message to show + */ + private void showToast(final String text) { + final Activity activity = getActivity(); + if (activity != null) { + activity.runOnUiThread( + new Runnable() { + @Override + public void run() { + Toast.makeText(activity, text, Toast.LENGTH_SHORT).show(); + } + }); + } + } + + @Override + public View onCreateView( + final LayoutInflater inflater, final ViewGroup container, final Bundle savedInstanceState) { + return inflater.inflate(layout, container, false); + } + + @Override + public void onViewCreated(final View view, final Bundle savedInstanceState) { + textureView = (AutoFitTextureView) view.findViewById(R.id.texture); + } + + @Override + public void onActivityCreated(final Bundle savedInstanceState) { + super.onActivityCreated(savedInstanceState); + } + + @Override + public void onResume() { + super.onResume(); + startBackgroundThread(); + + // When the screen is turned off and turned back on, the SurfaceTexture is already + // available, and "onSurfaceTextureAvailable" will not be called. In that case, we can open + // a camera and start preview from here (otherwise, we wait until the surface is ready in + // the SurfaceTextureListener). + if (textureView.isAvailable()) { + openCamera(textureView.getWidth(), textureView.getHeight()); + } else { + textureView.setSurfaceTextureListener(surfaceTextureListener); + } + } + + @Override + public void onPause() { + closeCamera(); + stopBackgroundThread(); + super.onPause(); + } + + public void setCamera(String cameraId) { + this.cameraId = cameraId; + } + + /** Sets up member variables related to camera. */ + private void setUpCameraOutputs() { + final Activity activity = getActivity(); + final CameraManager manager = (CameraManager) activity.getSystemService(Context.CAMERA_SERVICE); + try { + final CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraId); + + final StreamConfigurationMap map = + characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); + + sensorOrientation = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION); + + // Danger, W.R.! Attempting to use too large a preview size could exceed the camera + // bus' bandwidth limitation, resulting in gorgeous previews but the storage of + // garbage capture data. + previewSize = + chooseOptimalSize( + map.getOutputSizes(SurfaceTexture.class), + inputSize.getWidth(), + inputSize.getHeight()); + + // We fit the aspect ratio of TextureView to the size of preview we picked. + final int orientation = getResources().getConfiguration().orientation; + if (orientation == Configuration.ORIENTATION_LANDSCAPE) { + textureView.setAspectRatio(previewSize.getWidth(), previewSize.getHeight()); + } else { + textureView.setAspectRatio(previewSize.getHeight(), previewSize.getWidth()); + } + } catch (final CameraAccessException e) { + LOGGER.e(e, "Exception!"); + } catch (final NullPointerException e) { + // Currently an NPE is thrown when the Camera2API is used but not supported on the + // device this code runs. + ErrorDialog.newInstance(getString(R.string.tfe_ic_camera_error)) + .show(getChildFragmentManager(), FRAGMENT_DIALOG); + throw new IllegalStateException(getString(R.string.tfe_ic_camera_error)); + } + + cameraConnectionCallback.onPreviewSizeChosen(previewSize, sensorOrientation); + } + + /** Opens the camera specified by {@link CameraConnectionFragment#cameraId}. */ + private void openCamera(final int width, final int height) { + setUpCameraOutputs(); + configureTransform(width, height); + final Activity activity = getActivity(); + final CameraManager manager = (CameraManager) activity.getSystemService(Context.CAMERA_SERVICE); + try { + if (!cameraOpenCloseLock.tryAcquire(2500, TimeUnit.MILLISECONDS)) { + throw new RuntimeException("Time out waiting to lock camera opening."); + } + manager.openCamera(cameraId, stateCallback, backgroundHandler); + } catch (final CameraAccessException e) { + LOGGER.e(e, "Exception!"); + } catch (final InterruptedException e) { + throw new RuntimeException("Interrupted while trying to lock camera opening.", e); + } + } + + /** Closes the current {@link CameraDevice}. */ + private void closeCamera() { + try { + cameraOpenCloseLock.acquire(); + if (null != captureSession) { + captureSession.close(); + captureSession = null; + } + if (null != cameraDevice) { + cameraDevice.close(); + cameraDevice = null; + } + if (null != previewReader) { + previewReader.close(); + previewReader = null; + } + } catch (final InterruptedException e) { + throw new RuntimeException("Interrupted while trying to lock camera closing.", e); + } finally { + cameraOpenCloseLock.release(); + } + } + + /** Starts a background thread and its {@link Handler}. */ + private void startBackgroundThread() { + backgroundThread = new HandlerThread("ImageListener"); + backgroundThread.start(); + backgroundHandler = new Handler(backgroundThread.getLooper()); + } + + /** Stops the background thread and its {@link Handler}. */ + private void stopBackgroundThread() { + backgroundThread.quitSafely(); + try { + backgroundThread.join(); + backgroundThread = null; + backgroundHandler = null; + } catch (final InterruptedException e) { + LOGGER.e(e, "Exception!"); + } + } + + /** Creates a new {@link CameraCaptureSession} for camera preview. */ + private void createCameraPreviewSession() { + try { + final SurfaceTexture texture = textureView.getSurfaceTexture(); + assert texture != null; + + // We configure the size of default buffer to be the size of camera preview we want. + texture.setDefaultBufferSize(previewSize.getWidth(), previewSize.getHeight()); + + // This is the output Surface we need to start preview. + final Surface surface = new Surface(texture); + + // We set up a CaptureRequest.Builder with the output Surface. + previewRequestBuilder = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW); + previewRequestBuilder.addTarget(surface); + + LOGGER.i("Opening camera preview: " + previewSize.getWidth() + "x" + previewSize.getHeight()); + + // Create the reader for the preview frames. + previewReader = + ImageReader.newInstance( + previewSize.getWidth(), previewSize.getHeight(), ImageFormat.YUV_420_888, 2); + + previewReader.setOnImageAvailableListener(imageListener, backgroundHandler); + previewRequestBuilder.addTarget(previewReader.getSurface()); + + // Here, we create a CameraCaptureSession for camera preview. + cameraDevice.createCaptureSession( + Arrays.asList(surface, previewReader.getSurface()), + new CameraCaptureSession.StateCallback() { + + @Override + public void onConfigured(final CameraCaptureSession cameraCaptureSession) { + // The camera is already closed + if (null == cameraDevice) { + return; + } + + // When the session is ready, we start displaying the preview. + captureSession = cameraCaptureSession; + try { + // Auto focus should be continuous for camera preview. + previewRequestBuilder.set( + CaptureRequest.CONTROL_AF_MODE, + CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE); + // Flash is automatically enabled when necessary. + previewRequestBuilder.set( + CaptureRequest.CONTROL_AE_MODE, CaptureRequest.CONTROL_AE_MODE_ON_AUTO_FLASH); + + // Finally, we start displaying the camera preview. + previewRequest = previewRequestBuilder.build(); + captureSession.setRepeatingRequest( + previewRequest, captureCallback, backgroundHandler); + } catch (final CameraAccessException e) { + LOGGER.e(e, "Exception!"); + } + } + + @Override + public void onConfigureFailed(final CameraCaptureSession cameraCaptureSession) { + showToast("Failed"); + } + }, + null); + } catch (final CameraAccessException e) { + LOGGER.e(e, "Exception!"); + } + } + + /** + * Configures the necessary {@link Matrix} transformation to `mTextureView`. This method should be + * called after the camera preview size is determined in setUpCameraOutputs and also the size of + * `mTextureView` is fixed. + * + * @param viewWidth The width of `mTextureView` + * @param viewHeight The height of `mTextureView` + */ + private void configureTransform(final int viewWidth, final int viewHeight) { + final Activity activity = getActivity(); + if (null == textureView || null == previewSize || null == activity) { + return; + } + final int rotation = activity.getWindowManager().getDefaultDisplay().getRotation(); + final Matrix matrix = new Matrix(); + final RectF viewRect = new RectF(0, 0, viewWidth, viewHeight); + final RectF bufferRect = new RectF(0, 0, previewSize.getHeight(), previewSize.getWidth()); + final float centerX = viewRect.centerX(); + final float centerY = viewRect.centerY(); + if (Surface.ROTATION_90 == rotation || Surface.ROTATION_270 == rotation) { + bufferRect.offset(centerX - bufferRect.centerX(), centerY - bufferRect.centerY()); + matrix.setRectToRect(viewRect, bufferRect, Matrix.ScaleToFit.FILL); + final float scale = + Math.max( + (float) viewHeight / previewSize.getHeight(), + (float) viewWidth / previewSize.getWidth()); + matrix.postScale(scale, scale, centerX, centerY); + matrix.postRotate(90 * (rotation - 2), centerX, centerY); + } else if (Surface.ROTATION_180 == rotation) { + matrix.postRotate(180, centerX, centerY); + } + textureView.setTransform(matrix); + } + + /** + * Callback for Activities to use to initialize their data once the selected preview size is + * known. + */ + public interface ConnectionCallback { + void onPreviewSizeChosen(Size size, int cameraRotation); + } + + /** Compares two {@code Size}s based on their areas. */ + static class CompareSizesByArea implements Comparator { + @Override + public int compare(final Size lhs, final Size rhs) { + // We cast here to ensure the multiplications won't overflow + return Long.signum( + (long) lhs.getWidth() * lhs.getHeight() - (long) rhs.getWidth() * rhs.getHeight()); + } + } + + /** Shows an error message dialog. */ + public static class ErrorDialog extends DialogFragment { + private static final String ARG_MESSAGE = "message"; + + public static ErrorDialog newInstance(final String message) { + final ErrorDialog dialog = new ErrorDialog(); + final Bundle args = new Bundle(); + args.putString(ARG_MESSAGE, message); + dialog.setArguments(args); + return dialog; + } + + @Override + public Dialog onCreateDialog(final Bundle savedInstanceState) { + final Activity activity = getActivity(); + return new AlertDialog.Builder(activity) + .setMessage(getArguments().getString(ARG_MESSAGE)) + .setPositiveButton( + android.R.string.ok, + new DialogInterface.OnClickListener() { + @Override + public void onClick(final DialogInterface dialogInterface, final int i) { + activity.finish(); + } + }) + .create(); + } + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/ClassifierActivity.java b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/ClassifierActivity.java new file mode 100644 index 0000000000000000000000000000000000000000..24b5d72fdb42d47e5d2c87e3f944b71105748c1b --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/ClassifierActivity.java @@ -0,0 +1,238 @@ +/* + * Copyright 2019 The TensorFlow Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.tensorflow.lite.examples.classification; + +import android.graphics.Bitmap; +import android.graphics.Bitmap.Config; +import android.graphics.Typeface; +import android.media.ImageReader.OnImageAvailableListener; +import android.os.SystemClock; +import android.util.Size; +import android.util.TypedValue; +import android.view.TextureView; +import android.view.ViewStub; +import android.widget.TextView; +import android.widget.Toast; +import java.io.IOException; +import java.util.List; +import java.util.ArrayList; + +import org.tensorflow.lite.examples.classification.customview.AutoFitTextureView; +import org.tensorflow.lite.examples.classification.env.BorderedText; +import org.tensorflow.lite.examples.classification.env.Logger; +import org.tensorflow.lite.examples.classification.tflite.Classifier; +import org.tensorflow.lite.examples.classification.tflite.Classifier.Device; +import org.tensorflow.lite.examples.classification.tflite.Classifier.Model; + +import android.widget.ImageView; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.Canvas; +import android.graphics.Color; +import android.graphics.Paint; +import android.graphics.Rect; +import android.graphics.RectF; +import android.graphics.PixelFormat; +import java.nio.ByteBuffer; + +public class ClassifierActivity extends CameraActivity implements OnImageAvailableListener { + private static final Logger LOGGER = new Logger(); + private static final Size DESIRED_PREVIEW_SIZE = new Size(640, 480); + private static final float TEXT_SIZE_DIP = 10; + private Bitmap rgbFrameBitmap = null; + private long lastProcessingTimeMs; + private Integer sensorOrientation; + private Classifier classifier; + private BorderedText borderedText; + /** Input image size of the model along x axis. */ + private int imageSizeX; + /** Input image size of the model along y axis. */ + private int imageSizeY; + + @Override + protected int getLayoutId() { + return R.layout.tfe_ic_camera_connection_fragment; + } + + @Override + protected Size getDesiredPreviewFrameSize() { + return DESIRED_PREVIEW_SIZE; + } + + @Override + public void onPreviewSizeChosen(final Size size, final int rotation) { + final float textSizePx = + TypedValue.applyDimension( + TypedValue.COMPLEX_UNIT_DIP, TEXT_SIZE_DIP, getResources().getDisplayMetrics()); + borderedText = new BorderedText(textSizePx); + borderedText.setTypeface(Typeface.MONOSPACE); + + recreateClassifier(getModel(), getDevice(), getNumThreads()); + if (classifier == null) { + LOGGER.e("No classifier on preview!"); + return; + } + + previewWidth = size.getWidth(); + previewHeight = size.getHeight(); + + sensorOrientation = rotation - getScreenOrientation(); + LOGGER.i("Camera orientation relative to screen canvas: %d", sensorOrientation); + + LOGGER.i("Initializing at size %dx%d", previewWidth, previewHeight); + rgbFrameBitmap = Bitmap.createBitmap(previewWidth, previewHeight, Config.ARGB_8888); + } + + @Override + protected void processImage() { + rgbFrameBitmap.setPixels(getRgbBytes(), 0, previewWidth, 0, 0, previewWidth, previewHeight); + final int cropSize = Math.min(previewWidth, previewHeight); + + runInBackground( + new Runnable() { + @Override + public void run() { + if (classifier != null) { + final long startTime = SystemClock.uptimeMillis(); + //final List results = + // classifier.recognizeImage(rgbFrameBitmap, sensorOrientation); + final List results = new ArrayList<>(); + + float[] img_array = classifier.recognizeImage(rgbFrameBitmap, sensorOrientation); + + + /* + float maxval = Float.NEGATIVE_INFINITY; + float minval = Float.POSITIVE_INFINITY; + for (float cur : img_array) { + maxval = Math.max(maxval, cur); + minval = Math.min(minval, cur); + } + float multiplier = 0; + if ((maxval - minval) > 0) multiplier = 255 / (maxval - minval); + + int[] img_normalized = new int[img_array.length]; + for (int i = 0; i < img_array.length; ++i) { + float val = (float) (multiplier * (img_array[i] - minval)); + img_normalized[i] = (int) val; + } + + + + TextureView textureView = findViewById(R.id.textureView3); + //AutoFitTextureView textureView = (AutoFitTextureView) findViewById(R.id.texture); + + if(textureView.isAvailable()) { + int width = imageSizeX; + int height = imageSizeY; + + Canvas canvas = textureView.lockCanvas(); + canvas.drawColor(Color.BLUE); + Paint paint = new Paint(); + paint.setStyle(Paint.Style.FILL); + paint.setARGB(255, 150, 150, 150); + + int canvas_size = Math.min(canvas.getWidth(), canvas.getHeight()); + + Bitmap bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.RGB_565); + + for (int ii = 0; ii < width; ii++) //pass the screen pixels in 2 directions + { + for (int jj = 0; jj < height; jj++) { + //int val = img_normalized[ii + jj * width]; + int index = (width - ii - 1) + (height - jj - 1) * width; + if(index < img_array.length) { + int val = img_normalized[index]; + bitmap.setPixel(ii, jj, Color.rgb(val, val, val)); + } + } + } + + canvas.drawBitmap(bitmap, null, new RectF(0, 0, canvas_size, canvas_size), null); + + textureView.unlockCanvasAndPost(canvas); + + } + */ + + lastProcessingTimeMs = SystemClock.uptimeMillis() - startTime; + LOGGER.v("Detect: %s", results); + + runOnUiThread( + new Runnable() { + @Override + public void run() { + //showResultsInBottomSheet(results); + showResultsInTexture(img_array, imageSizeX, imageSizeY); + showFrameInfo(previewWidth + "x" + previewHeight); + showCropInfo(imageSizeX + "x" + imageSizeY); + showCameraResolution(cropSize + "x" + cropSize); + showRotationInfo(String.valueOf(sensorOrientation)); + showInference(lastProcessingTimeMs + "ms"); + } + }); + } + readyForNextImage(); + } + }); + } + + @Override + protected void onInferenceConfigurationChanged() { + if (rgbFrameBitmap == null) { + // Defer creation until we're getting camera frames. + return; + } + final Device device = getDevice(); + final Model model = getModel(); + final int numThreads = getNumThreads(); + runInBackground(() -> recreateClassifier(model, device, numThreads)); + } + + private void recreateClassifier(Model model, Device device, int numThreads) { + if (classifier != null) { + LOGGER.d("Closing classifier."); + classifier.close(); + classifier = null; + } + if (device == Device.GPU + && (model == Model.QUANTIZED_MOBILENET || model == Model.QUANTIZED_EFFICIENTNET)) { + LOGGER.d("Not creating classifier: GPU doesn't support quantized models."); + runOnUiThread( + () -> { + Toast.makeText(this, R.string.tfe_ic_gpu_quant_error, Toast.LENGTH_LONG).show(); + }); + return; + } + try { + LOGGER.d( + "Creating classifier (model=%s, device=%s, numThreads=%d)", model, device, numThreads); + classifier = Classifier.create(this, model, device, numThreads); + } catch (IOException | IllegalArgumentException e) { + LOGGER.e(e, "Failed to create classifier."); + runOnUiThread( + () -> { + Toast.makeText(this, e.getMessage(), Toast.LENGTH_LONG).show(); + }); + return; + } + + // Updates the input image size. + imageSizeX = classifier.getImageSizeX(); + imageSizeY = classifier.getImageSizeY(); + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/LegacyCameraConnectionFragment.java b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/LegacyCameraConnectionFragment.java new file mode 100644 index 0000000000000000000000000000000000000000..760fe90375450c7b1356603c83fb37a68548ca13 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/LegacyCameraConnectionFragment.java @@ -0,0 +1,203 @@ +package org.tensorflow.lite.examples.classification; + +/* + * Copyright 2019 The TensorFlow Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import android.annotation.SuppressLint; +import android.app.Fragment; +import android.graphics.SurfaceTexture; +import android.hardware.Camera; +import android.hardware.Camera.CameraInfo; +import android.os.Bundle; +import android.os.Handler; +import android.os.HandlerThread; +import android.util.Size; +import android.util.SparseIntArray; +import android.view.LayoutInflater; +import android.view.Surface; +import android.view.TextureView; +import android.view.View; +import android.view.ViewGroup; +import java.io.IOException; +import java.util.List; +import org.tensorflow.lite.examples.classification.customview.AutoFitTextureView; +import org.tensorflow.lite.examples.classification.env.ImageUtils; +import org.tensorflow.lite.examples.classification.env.Logger; + +public class LegacyCameraConnectionFragment extends Fragment { + private static final Logger LOGGER = new Logger(); + /** Conversion from screen rotation to JPEG orientation. */ + private static final SparseIntArray ORIENTATIONS = new SparseIntArray(); + + static { + ORIENTATIONS.append(Surface.ROTATION_0, 90); + ORIENTATIONS.append(Surface.ROTATION_90, 0); + ORIENTATIONS.append(Surface.ROTATION_180, 270); + ORIENTATIONS.append(Surface.ROTATION_270, 180); + } + + private Camera camera; + private Camera.PreviewCallback imageListener; + private Size desiredSize; + /** The layout identifier to inflate for this Fragment. */ + private int layout; + /** An {@link AutoFitTextureView} for camera preview. */ + private AutoFitTextureView textureView; + /** + * {@link TextureView.SurfaceTextureListener} handles several lifecycle events on a {@link + * TextureView}. + */ + private final TextureView.SurfaceTextureListener surfaceTextureListener = + new TextureView.SurfaceTextureListener() { + @Override + public void onSurfaceTextureAvailable( + final SurfaceTexture texture, final int width, final int height) { + + int index = getCameraId(); + camera = Camera.open(index); + + try { + Camera.Parameters parameters = camera.getParameters(); + List focusModes = parameters.getSupportedFocusModes(); + if (focusModes != null + && focusModes.contains(Camera.Parameters.FOCUS_MODE_CONTINUOUS_PICTURE)) { + parameters.setFocusMode(Camera.Parameters.FOCUS_MODE_CONTINUOUS_PICTURE); + } + List cameraSizes = parameters.getSupportedPreviewSizes(); + Size[] sizes = new Size[cameraSizes.size()]; + int i = 0; + for (Camera.Size size : cameraSizes) { + sizes[i++] = new Size(size.width, size.height); + } + Size previewSize = + CameraConnectionFragment.chooseOptimalSize( + sizes, desiredSize.getWidth(), desiredSize.getHeight()); + parameters.setPreviewSize(previewSize.getWidth(), previewSize.getHeight()); + camera.setDisplayOrientation(90); + camera.setParameters(parameters); + camera.setPreviewTexture(texture); + } catch (IOException exception) { + camera.release(); + } + + camera.setPreviewCallbackWithBuffer(imageListener); + Camera.Size s = camera.getParameters().getPreviewSize(); + camera.addCallbackBuffer(new byte[ImageUtils.getYUVByteSize(s.height, s.width)]); + + textureView.setAspectRatio(s.height, s.width); + + camera.startPreview(); + } + + @Override + public void onSurfaceTextureSizeChanged( + final SurfaceTexture texture, final int width, final int height) {} + + @Override + public boolean onSurfaceTextureDestroyed(final SurfaceTexture texture) { + return true; + } + + @Override + public void onSurfaceTextureUpdated(final SurfaceTexture texture) {} + }; + /** An additional thread for running tasks that shouldn't block the UI. */ + private HandlerThread backgroundThread; + + @SuppressLint("ValidFragment") + public LegacyCameraConnectionFragment( + final Camera.PreviewCallback imageListener, final int layout, final Size desiredSize) { + this.imageListener = imageListener; + this.layout = layout; + this.desiredSize = desiredSize; + } + + @Override + public View onCreateView( + final LayoutInflater inflater, final ViewGroup container, final Bundle savedInstanceState) { + return inflater.inflate(layout, container, false); + } + + @Override + public void onViewCreated(final View view, final Bundle savedInstanceState) { + textureView = (AutoFitTextureView) view.findViewById(R.id.texture); + } + + @Override + public void onActivityCreated(final Bundle savedInstanceState) { + super.onActivityCreated(savedInstanceState); + } + + @Override + public void onResume() { + super.onResume(); + startBackgroundThread(); + // When the screen is turned off and turned back on, the SurfaceTexture is already + // available, and "onSurfaceTextureAvailable" will not be called. In that case, we can open + // a camera and start preview from here (otherwise, we wait until the surface is ready in + // the SurfaceTextureListener). + + if (textureView.isAvailable()) { + if (camera != null) { + camera.startPreview(); + } + } else { + textureView.setSurfaceTextureListener(surfaceTextureListener); + } + } + + @Override + public void onPause() { + stopCamera(); + stopBackgroundThread(); + super.onPause(); + } + + /** Starts a background thread and its {@link Handler}. */ + private void startBackgroundThread() { + backgroundThread = new HandlerThread("CameraBackground"); + backgroundThread.start(); + } + + /** Stops the background thread and its {@link Handler}. */ + private void stopBackgroundThread() { + backgroundThread.quitSafely(); + try { + backgroundThread.join(); + backgroundThread = null; + } catch (final InterruptedException e) { + LOGGER.e(e, "Exception!"); + } + } + + protected void stopCamera() { + if (camera != null) { + camera.stopPreview(); + camera.setPreviewCallback(null); + camera.release(); + camera = null; + } + } + + private int getCameraId() { + CameraInfo ci = new CameraInfo(); + for (int i = 0; i < Camera.getNumberOfCameras(); i++) { + Camera.getCameraInfo(i, ci); + if (ci.facing == CameraInfo.CAMERA_FACING_BACK) return i; + } + return -1; // No camera found + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/customview/AutoFitTextureView.java b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/customview/AutoFitTextureView.java new file mode 100644 index 0000000000000000000000000000000000000000..62e99ae70c2a7c4c60a776e7490742c5339e85f3 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/customview/AutoFitTextureView.java @@ -0,0 +1,72 @@ +/* + * Copyright 2019 The TensorFlow Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.tensorflow.lite.examples.classification.customview; + +import android.content.Context; +import android.util.AttributeSet; +import android.view.TextureView; + +/** A {@link TextureView} that can be adjusted to a specified aspect ratio. */ +public class AutoFitTextureView extends TextureView { + private int ratioWidth = 0; + private int ratioHeight = 0; + + public AutoFitTextureView(final Context context) { + this(context, null); + } + + public AutoFitTextureView(final Context context, final AttributeSet attrs) { + this(context, attrs, 0); + } + + public AutoFitTextureView(final Context context, final AttributeSet attrs, final int defStyle) { + super(context, attrs, defStyle); + } + + /** + * Sets the aspect ratio for this view. The size of the view will be measured based on the ratio + * calculated from the parameters. Note that the actual sizes of parameters don't matter, that is, + * calling setAspectRatio(2, 3) and setAspectRatio(4, 6) make the same result. + * + * @param width Relative horizontal size + * @param height Relative vertical size + */ + public void setAspectRatio(final int width, final int height) { + if (width < 0 || height < 0) { + throw new IllegalArgumentException("Size cannot be negative."); + } + ratioWidth = width; + ratioHeight = height; + requestLayout(); + } + + @Override + protected void onMeasure(final int widthMeasureSpec, final int heightMeasureSpec) { + super.onMeasure(widthMeasureSpec, heightMeasureSpec); + final int width = MeasureSpec.getSize(widthMeasureSpec); + final int height = MeasureSpec.getSize(heightMeasureSpec); + if (0 == ratioWidth || 0 == ratioHeight) { + setMeasuredDimension(width, height); + } else { + if (width < height * ratioWidth / ratioHeight) { + setMeasuredDimension(width, width * ratioHeight / ratioWidth); + } else { + setMeasuredDimension(height * ratioWidth / ratioHeight, height); + } + } + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/customview/OverlayView.java b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/customview/OverlayView.java new file mode 100644 index 0000000000000000000000000000000000000000..dc302ac04f145c9a1673a2d7e630a94a05ab1b1a --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/customview/OverlayView.java @@ -0,0 +1,48 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.lite.examples.classification.customview; + +import android.content.Context; +import android.graphics.Canvas; +import android.util.AttributeSet; +import android.view.View; +import java.util.LinkedList; +import java.util.List; + +/** A simple View providing a render callback to other classes. */ +public class OverlayView extends View { + private final List callbacks = new LinkedList(); + + public OverlayView(final Context context, final AttributeSet attrs) { + super(context, attrs); + } + + public void addCallback(final DrawCallback callback) { + callbacks.add(callback); + } + + @Override + public synchronized void draw(final Canvas canvas) { + for (final DrawCallback callback : callbacks) { + callback.drawCallback(canvas); + } + } + + /** Interface defining the callback for client classes. */ + public interface DrawCallback { + public void drawCallback(final Canvas canvas); + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/customview/RecognitionScoreView.java b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/customview/RecognitionScoreView.java new file mode 100644 index 0000000000000000000000000000000000000000..2c57f603f12200079c888793cfa40d9b10dabde3 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/customview/RecognitionScoreView.java @@ -0,0 +1,67 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.lite.examples.classification.customview; + +import android.content.Context; +import android.graphics.Canvas; +import android.graphics.Paint; +import android.util.AttributeSet; +import android.util.TypedValue; +import android.view.View; +import java.util.List; +import org.tensorflow.lite.examples.classification.tflite.Classifier.Recognition; + +public class RecognitionScoreView extends View implements ResultsView { + private static final float TEXT_SIZE_DIP = 16; + private final float textSizePx; + private final Paint fgPaint; + private final Paint bgPaint; + private List results; + + public RecognitionScoreView(final Context context, final AttributeSet set) { + super(context, set); + + textSizePx = + TypedValue.applyDimension( + TypedValue.COMPLEX_UNIT_DIP, TEXT_SIZE_DIP, getResources().getDisplayMetrics()); + fgPaint = new Paint(); + fgPaint.setTextSize(textSizePx); + + bgPaint = new Paint(); + bgPaint.setColor(0xcc4285f4); + } + + @Override + public void setResults(final List results) { + this.results = results; + postInvalidate(); + } + + @Override + public void onDraw(final Canvas canvas) { + final int x = 10; + int y = (int) (fgPaint.getTextSize() * 1.5f); + + canvas.drawPaint(bgPaint); + + if (results != null) { + for (final Recognition recog : results) { + canvas.drawText(recog.getTitle() + ": " + recog.getConfidence(), x, y, fgPaint); + y += (int) (fgPaint.getTextSize() * 1.5f); + } + } + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/customview/ResultsView.java b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/customview/ResultsView.java new file mode 100644 index 0000000000000000000000000000000000000000..d055eb5f161a57fc439716efe6d49b7e45ef3fc7 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/java/org/tensorflow/lite/examples/classification/customview/ResultsView.java @@ -0,0 +1,23 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.lite.examples.classification.customview; + +import java.util.List; +import org.tensorflow.lite.examples.classification.tflite.Classifier.Recognition; + +public interface ResultsView { + public void setResults(final List results); +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml new file mode 100644 index 0000000000000000000000000000000000000000..b1517edf496ef5800b97d046b92012a9f94a34d0 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/drawable/bottom_sheet_bg.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/drawable/bottom_sheet_bg.xml new file mode 100644 index 0000000000000000000000000000000000000000..70f4b24e35039e6bfc35989bcbe570a4bdc2ae07 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/drawable/bottom_sheet_bg.xml @@ -0,0 +1,9 @@ + + + + + + \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/drawable/ic_baseline_add.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/drawable/ic_baseline_add.xml new file mode 100644 index 0000000000000000000000000000000000000000..757f4503314fb9e5837f68ac515f4487d9b5fc2c --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/drawable/ic_baseline_add.xml @@ -0,0 +1,9 @@ + + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/drawable/ic_baseline_remove.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/drawable/ic_baseline_remove.xml new file mode 100644 index 0000000000000000000000000000000000000000..a64b853e79137f0fd95f9d5fa6e0552cc255c7ae --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/drawable/ic_baseline_remove.xml @@ -0,0 +1,9 @@ + + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/drawable/ic_launcher_background.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/drawable/ic_launcher_background.xml new file mode 100644 index 0000000000000000000000000000000000000000..d5fccc538c179838bfdce779c26eebb4fa0b5ce9 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/drawable/ic_launcher_background.xml @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/drawable/rectangle.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/drawable/rectangle.xml new file mode 100644 index 0000000000000000000000000000000000000000..b8f5d3559c4e83072d5d73a3241d240aa68daccf --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/drawable/rectangle.xml @@ -0,0 +1,13 @@ + + + + + + \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/layout/tfe_ic_activity_camera.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/layout/tfe_ic_activity_camera.xml new file mode 100644 index 0000000000000000000000000000000000000000..f0e1dae7afa15cf4a832de708f345482a6dfeff6 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/layout/tfe_ic_activity_camera.xml @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/layout/tfe_ic_camera_connection_fragment.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/layout/tfe_ic_camera_connection_fragment.xml new file mode 100644 index 0000000000000000000000000000000000000000..97e5e7c6df25da48977f9064a888fd3735e4986f --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/layout/tfe_ic_camera_connection_fragment.xml @@ -0,0 +1,32 @@ + + + + + + + + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/layout/tfe_ic_layout_bottom_sheet.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/layout/tfe_ic_layout_bottom_sheet.xml new file mode 100644 index 0000000000000000000000000000000000000000..77a348af90e2ed995ff106cd209cbf304c6b9153 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/layout/tfe_ic_layout_bottom_sheet.xml @@ -0,0 +1,321 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml new file mode 100644 index 0000000000000000000000000000000000000000..0c2a915e91af65a077d2e01db4ca21acd42906f3 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml new file mode 100644 index 0000000000000000000000000000000000000000..0c2a915e91af65a077d2e01db4ca21acd42906f3 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/values/colors.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/values/colors.xml new file mode 100644 index 0000000000000000000000000000000000000000..ed82bafb536474c6a88c996b439a2781f31f3d3e --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/values/colors.xml @@ -0,0 +1,8 @@ + + + #ffa800 + #ff6f00 + #425066 + + #66000000 + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/values/dimens.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/values/dimens.xml new file mode 100644 index 0000000000000000000000000000000000000000..5d3609029ca66b612c88b4f395e4e2e3cfc1f0e6 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/values/dimens.xml @@ -0,0 +1,5 @@ + + + 15dp + 8dp + \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/values/strings.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/values/strings.xml new file mode 100644 index 0000000000000000000000000000000000000000..7d763d85efc49879c8d3c0641484f5f472bfaca0 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/values/strings.xml @@ -0,0 +1,21 @@ + + Midas + This device doesn\'t support Camera2 API. + GPU does not yet supported quantized models. + Model: + + Float_EfficientNet + + + + Device: + + GPU + CPU + NNAPI + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/values/styles.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/values/styles.xml new file mode 100644 index 0000000000000000000000000000000000000000..ad09a13ec6b2de8920a7441c9992f3cc0eedcfda --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/app/src/main/res/values/styles.xml @@ -0,0 +1,11 @@ + + + + + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/build.gradle b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/build.gradle new file mode 100644 index 0000000000000000000000000000000000000000..14492756847191ca3beff4c2e012d378c4e44be6 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/build.gradle @@ -0,0 +1,27 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. + +buildscript { + + repositories { + google() + jcenter() + } + dependencies { + classpath 'com.android.tools.build:gradle:4.0.0' + classpath 'de.undercouch:gradle-download-task:4.0.2' + // NOTE: Do not place your application dependencies here; they belong + // in the individual module build.gradle files + } +} + +allprojects { + repositories { + google() + jcenter() + } +} + +task clean(type: Delete) { + delete rootProject.buildDir +} + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/gradle.properties b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/gradle.properties new file mode 100644 index 0000000000000000000000000000000000000000..9592636c07d9d5e6f61c0cfce1311d3e1ffcf34d --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/gradle.properties @@ -0,0 +1,15 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx1536m +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true +android.useAndroidX=true +android.enableJetifier=true diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/gradle/wrapper/gradle-wrapper.jar b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..f3d88b1c2faf2fc91d853cd5d4242b5547257070 Binary files /dev/null and b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/gradle/wrapper/gradle-wrapper.jar differ diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/gradle/wrapper/gradle-wrapper.properties b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000000000000000000000000000000000000..1b16c34a71cf212ed0cfb883d14d1b8511903eb2 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-6.1.1-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/gradlew b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/gradlew new file mode 100644 index 0000000000000000000000000000000000000000..2fe81a7d95e4f9ad2c9b2a046707d36ceb3980b3 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/gradlew @@ -0,0 +1,183 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/gradlew.bat b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/gradlew.bat new file mode 100644 index 0000000000000000000000000000000000000000..9618d8d9607cd91a0efb866bcac4810064ba6fac --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/gradlew.bat @@ -0,0 +1,100 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/build.gradle b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/build.gradle new file mode 100644 index 0000000000000000000000000000000000000000..5d463975293264765a941795601cddb6cfc84f00 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/build.gradle @@ -0,0 +1,47 @@ +apply plugin: 'com.android.library' + +android { + compileSdkVersion 28 + buildToolsVersion "28.0.0" + + defaultConfig { + minSdkVersion 21 + targetSdkVersion 28 + versionCode 1 + versionName "1.0" + + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + + aaptOptions { + noCompress "tflite" + } + + lintOptions { + checkReleaseBuilds false + // Or, if you prefer, you can continue to check for errors in release builds, + // but continue the build even when errors are found: + abortOnError false + } +} + +dependencies { + implementation fileTree(dir: 'libs', include: ['*.jar']) + implementation project(":models") + implementation 'androidx.appcompat:appcompat:1.1.0' + + // Build off of nightly TensorFlow Lite + implementation('org.tensorflow:tensorflow-lite:0.0.0-nightly') { changing = true } + implementation('org.tensorflow:tensorflow-lite-gpu:0.0.0-nightly') { changing = true } + implementation('org.tensorflow:tensorflow-lite-support:0.0.0-nightly') { changing = true } + // Use local TensorFlow library + // implementation 'org.tensorflow:tensorflow-lite-local:0.0.0' +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/proguard-rules.pro b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/proguard-rules.pro new file mode 100644 index 0000000000000000000000000000000000000000..f1b424510da51fd82143bc74a0a801ae5a1e2fcd --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/src/main/AndroidManifest.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/src/main/AndroidManifest.xml new file mode 100644 index 0000000000000000000000000000000000000000..ebe3c56c60a9b67eec218d969aecfdf5311d7b49 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/src/main/AndroidManifest.xml @@ -0,0 +1,3 @@ + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/src/main/java/org/tensorflow/lite/examples/classification/tflite/Classifier.java b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/src/main/java/org/tensorflow/lite/examples/classification/tflite/Classifier.java new file mode 100644 index 0000000000000000000000000000000000000000..24ec573e7d184e7d64118a723d6645fd92d6e6d9 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/src/main/java/org/tensorflow/lite/examples/classification/tflite/Classifier.java @@ -0,0 +1,376 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.lite.examples.classification.tflite; + +import static java.lang.Math.min; + +import android.app.Activity; +import android.graphics.Bitmap; +import android.graphics.RectF; +import android.os.SystemClock; +import android.os.Trace; +import android.util.Log; +import android.view.TextureView; +import android.view.ViewStub; + +import java.io.IOException; +import java.nio.MappedByteBuffer; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; +import org.tensorflow.lite.DataType; +import org.tensorflow.lite.Interpreter; +import org.tensorflow.lite.examples.classification.tflite.Classifier.Device; +import org.tensorflow.lite.gpu.GpuDelegate; +import org.tensorflow.lite.nnapi.NnApiDelegate; +import org.tensorflow.lite.support.common.FileUtil; +import org.tensorflow.lite.support.common.TensorOperator; +import org.tensorflow.lite.support.common.TensorProcessor; +import org.tensorflow.lite.support.image.ImageProcessor; +import org.tensorflow.lite.support.image.TensorImage; +import org.tensorflow.lite.support.image.ops.ResizeOp; +import org.tensorflow.lite.support.image.ops.ResizeOp.ResizeMethod; +import org.tensorflow.lite.support.image.ops.ResizeWithCropOrPadOp; +import org.tensorflow.lite.support.image.ops.Rot90Op; +import org.tensorflow.lite.support.label.TensorLabel; +import org.tensorflow.lite.support.tensorbuffer.TensorBuffer; + +/** A classifier specialized to label images using TensorFlow Lite. */ +public abstract class Classifier { + public static final String TAG = "ClassifierWithSupport"; + + /** The model type used for classification. */ + public enum Model { + FLOAT_MOBILENET, + QUANTIZED_MOBILENET, + QUANTIZED_EFFICIENTNET, + FLOAT_EFFICIENTNET + } + + /** The runtime device type used for executing classification. */ + public enum Device { + CPU, + NNAPI, + GPU + } + + /** Number of results to show in the UI. */ + private static final int MAX_RESULTS = 3; + + /** The loaded TensorFlow Lite model. */ + + /** Image size along the x axis. */ + private final int imageSizeX; + + /** Image size along the y axis. */ + private final int imageSizeY; + + /** Optional GPU delegate for accleration. */ + private GpuDelegate gpuDelegate = null; + + /** Optional NNAPI delegate for accleration. */ + private NnApiDelegate nnApiDelegate = null; + + /** An instance of the driver class to run model inference with Tensorflow Lite. */ + protected Interpreter tflite; + + /** Options for configuring the Interpreter. */ + private final Interpreter.Options tfliteOptions = new Interpreter.Options(); + + /** Labels corresponding to the output of the vision model. */ + private final List labels; + + /** Input image TensorBuffer. */ + private TensorImage inputImageBuffer; + + /** Output probability TensorBuffer. */ + private final TensorBuffer outputProbabilityBuffer; + + /** Processer to apply post processing of the output probability. */ + private final TensorProcessor probabilityProcessor; + + /** + * Creates a classifier with the provided configuration. + * + * @param activity The current Activity. + * @param model The model to use for classification. + * @param device The device to use for classification. + * @param numThreads The number of threads to use for classification. + * @return A classifier with the desired configuration. + */ + public static Classifier create(Activity activity, Model model, Device device, int numThreads) + throws IOException { + if (model == Model.QUANTIZED_MOBILENET) { + return new ClassifierQuantizedMobileNet(activity, device, numThreads); + } else if (model == Model.FLOAT_MOBILENET) { + return new ClassifierFloatMobileNet(activity, device, numThreads); + } else if (model == Model.FLOAT_EFFICIENTNET) { + return new ClassifierFloatEfficientNet(activity, device, numThreads); + } else if (model == Model.QUANTIZED_EFFICIENTNET) { + return new ClassifierQuantizedEfficientNet(activity, device, numThreads); + } else { + throw new UnsupportedOperationException(); + } + } + + /** An immutable result returned by a Classifier describing what was recognized. */ + public static class Recognition { + /** + * A unique identifier for what has been recognized. Specific to the class, not the instance of + * the object. + */ + private final String id; + + /** Display name for the recognition. */ + private final String title; + + /** + * A sortable score for how good the recognition is relative to others. Higher should be better. + */ + private final Float confidence; + + /** Optional location within the source image for the location of the recognized object. */ + private RectF location; + + public Recognition( + final String id, final String title, final Float confidence, final RectF location) { + this.id = id; + this.title = title; + this.confidence = confidence; + this.location = location; + } + + public String getId() { + return id; + } + + public String getTitle() { + return title; + } + + public Float getConfidence() { + return confidence; + } + + public RectF getLocation() { + return new RectF(location); + } + + public void setLocation(RectF location) { + this.location = location; + } + + @Override + public String toString() { + String resultString = ""; + if (id != null) { + resultString += "[" + id + "] "; + } + + if (title != null) { + resultString += title + " "; + } + + if (confidence != null) { + resultString += String.format("(%.1f%%) ", confidence * 100.0f); + } + + if (location != null) { + resultString += location + " "; + } + + return resultString.trim(); + } + } + + /** Initializes a {@code Classifier}. */ + protected Classifier(Activity activity, Device device, int numThreads) throws IOException { + MappedByteBuffer tfliteModel = FileUtil.loadMappedFile(activity, getModelPath()); + switch (device) { + case NNAPI: + nnApiDelegate = new NnApiDelegate(); + tfliteOptions.addDelegate(nnApiDelegate); + break; + case GPU: + gpuDelegate = new GpuDelegate(); + tfliteOptions.addDelegate(gpuDelegate); + break; + case CPU: + break; + } + tfliteOptions.setNumThreads(numThreads); + tflite = new Interpreter(tfliteModel, tfliteOptions); + + // Loads labels out from the label file. + labels = FileUtil.loadLabels(activity, getLabelPath()); + + // Reads type and shape of input and output tensors, respectively. + int imageTensorIndex = 0; + int[] imageShape = tflite.getInputTensor(imageTensorIndex).shape(); // {1, height, width, 3} + if(imageShape[1] != imageShape[2]) { + imageSizeY = imageShape[2]; + imageSizeX = imageShape[3]; + } else { + imageSizeY = imageShape[1]; + imageSizeX = imageShape[2]; + } + DataType imageDataType = tflite.getInputTensor(imageTensorIndex).dataType(); + int probabilityTensorIndex = 0; + int[] probabilityShape = + tflite.getOutputTensor(probabilityTensorIndex).shape(); // {1, NUM_CLASSES} + DataType probabilityDataType = tflite.getOutputTensor(probabilityTensorIndex).dataType(); + + // Creates the input tensor. + inputImageBuffer = new TensorImage(imageDataType); + + // Creates the output tensor and its processor. + outputProbabilityBuffer = TensorBuffer.createFixedSize(probabilityShape, probabilityDataType); + + // Creates the post processor for the output probability. + probabilityProcessor = new TensorProcessor.Builder().add(getPostprocessNormalizeOp()).build(); + + Log.d(TAG, "Created a Tensorflow Lite Image Classifier."); + } + + /** Runs inference and returns the classification results. */ + //public List recognizeImage(final Bitmap bitmap, int sensorOrientation) { + public float[] recognizeImage(final Bitmap bitmap, int sensorOrientation) { + // Logs this method so that it can be analyzed with systrace. + Trace.beginSection("recognizeImage"); + + Trace.beginSection("loadImage"); + long startTimeForLoadImage = SystemClock.uptimeMillis(); + inputImageBuffer = loadImage(bitmap, sensorOrientation); + long endTimeForLoadImage = SystemClock.uptimeMillis(); + Trace.endSection(); + Log.v(TAG, "Timecost to load the image: " + (endTimeForLoadImage - startTimeForLoadImage)); + + // Runs the inference call. + Trace.beginSection("runInference"); + long startTimeForReference = SystemClock.uptimeMillis(); + tflite.run(inputImageBuffer.getBuffer(), outputProbabilityBuffer.getBuffer().rewind()); + long endTimeForReference = SystemClock.uptimeMillis(); + Trace.endSection(); + Log.v(TAG, "Timecost to run model inference: " + (endTimeForReference - startTimeForReference)); + + float[] img_array = outputProbabilityBuffer.getFloatArray(); + + // Gets the map of label and probability. + //Map labeledProbability = + // new TensorLabel(labels, probabilityProcessor.process(outputProbabilityBuffer)) + // .getMapWithFloatValue(); + Trace.endSection(); + + // Gets top-k results. + return img_array;//getTopKProbability(labeledProbability); + } + + /** Closes the interpreter and model to release resources. */ + public void close() { + if (tflite != null) { + tflite.close(); + tflite = null; + } + if (gpuDelegate != null) { + gpuDelegate.close(); + gpuDelegate = null; + } + if (nnApiDelegate != null) { + nnApiDelegate.close(); + nnApiDelegate = null; + } + } + + /** Get the image size along the x axis. */ + public int getImageSizeX() { + return imageSizeX; + } + + /** Get the image size along the y axis. */ + public int getImageSizeY() { + return imageSizeY; + } + + /** Loads input image, and applies preprocessing. */ + private TensorImage loadImage(final Bitmap bitmap, int sensorOrientation) { + // Loads bitmap into a TensorImage. + inputImageBuffer.load(bitmap); + + // Creates processor for the TensorImage. + int cropSize = min(bitmap.getWidth(), bitmap.getHeight()); + int numRotation = sensorOrientation / 90; + // TODO(b/143564309): Fuse ops inside ImageProcessor. + ImageProcessor imageProcessor = + new ImageProcessor.Builder() + .add(new ResizeWithCropOrPadOp(cropSize, cropSize)) + // TODO(b/169379396): investigate the impact of the resize algorithm on accuracy. + // To get the same inference results as lib_task_api, which is built on top of the Task + // Library, use ResizeMethod.BILINEAR. + .add(new ResizeOp(imageSizeX, imageSizeY, ResizeMethod.NEAREST_NEIGHBOR)) + //.add(new ResizeOp(224, 224, ResizeMethod.NEAREST_NEIGHBOR)) + .add(new Rot90Op(numRotation)) + .add(getPreprocessNormalizeOp()) + .build(); + return imageProcessor.process(inputImageBuffer); + } + + /** Gets the top-k results. */ + private static List getTopKProbability(Map labelProb) { + // Find the best classifications. + PriorityQueue pq = + new PriorityQueue<>( + MAX_RESULTS, + new Comparator() { + @Override + public int compare(Recognition lhs, Recognition rhs) { + // Intentionally reversed to put high confidence at the head of the queue. + return Float.compare(rhs.getConfidence(), lhs.getConfidence()); + } + }); + + for (Map.Entry entry : labelProb.entrySet()) { + pq.add(new Recognition("" + entry.getKey(), entry.getKey(), entry.getValue(), null)); + } + + final ArrayList recognitions = new ArrayList<>(); + int recognitionsSize = min(pq.size(), MAX_RESULTS); + for (int i = 0; i < recognitionsSize; ++i) { + recognitions.add(pq.poll()); + } + return recognitions; + } + + /** Gets the name of the model file stored in Assets. */ + protected abstract String getModelPath(); + + /** Gets the name of the label file stored in Assets. */ + protected abstract String getLabelPath(); + + /** Gets the TensorOperator to nomalize the input image in preprocessing. */ + protected abstract TensorOperator getPreprocessNormalizeOp(); + + /** + * Gets the TensorOperator to dequantize the output probability in post processing. + * + *

For quantized model, we need de-quantize the prediction with NormalizeOp (as they are all + * essentially linear transformation). For float model, de-quantize is not required. But to + * uniform the API, de-quantize is added to float model too. Mean and std are set to 0.0f and + * 1.0f, respectively. + */ + protected abstract TensorOperator getPostprocessNormalizeOp(); +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierFloatEfficientNet.java b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierFloatEfficientNet.java new file mode 100644 index 0000000000000000000000000000000000000000..14dd027b26baefaedd979a8ac37f0bf984210ed4 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierFloatEfficientNet.java @@ -0,0 +1,71 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.lite.examples.classification.tflite; + +import android.app.Activity; +import java.io.IOException; +import org.tensorflow.lite.examples.classification.tflite.Classifier.Device; +import org.tensorflow.lite.support.common.TensorOperator; +import org.tensorflow.lite.support.common.ops.NormalizeOp; + +/** This TensorFlowLite classifier works with the float EfficientNet model. */ +public class ClassifierFloatEfficientNet extends Classifier { + + private static final float IMAGE_MEAN = 115.0f; //127.0f; + private static final float IMAGE_STD = 58.0f; //128.0f; + + /** + * Float model does not need dequantization in the post-processing. Setting mean and std as 0.0f + * and 1.0f, repectively, to bypass the normalization. + */ + private static final float PROBABILITY_MEAN = 0.0f; + + private static final float PROBABILITY_STD = 1.0f; + + /** + * Initializes a {@code ClassifierFloatMobileNet}. + * + * @param activity + */ + public ClassifierFloatEfficientNet(Activity activity, Device device, int numThreads) + throws IOException { + super(activity, device, numThreads); + } + + @Override + protected String getModelPath() { + // you can download this file from + // see build.gradle for where to obtain this file. It should be auto + // downloaded into assets. + //return "efficientnet-lite0-fp32.tflite"; + return "model_opt.tflite"; + } + + @Override + protected String getLabelPath() { + return "labels_without_background.txt"; + } + + @Override + protected TensorOperator getPreprocessNormalizeOp() { + return new NormalizeOp(IMAGE_MEAN, IMAGE_STD); + } + + @Override + protected TensorOperator getPostprocessNormalizeOp() { + return new NormalizeOp(PROBABILITY_MEAN, PROBABILITY_STD); + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierFloatMobileNet.java b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierFloatMobileNet.java new file mode 100644 index 0000000000000000000000000000000000000000..40519de07cf5e887773250a4609a832b6060d684 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierFloatMobileNet.java @@ -0,0 +1,72 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.lite.examples.classification.tflite; + +import android.app.Activity; +import java.io.IOException; +import org.tensorflow.lite.examples.classification.tflite.Classifier.Device; +import org.tensorflow.lite.support.common.TensorOperator; +import org.tensorflow.lite.support.common.ops.NormalizeOp; + +/** This TensorFlowLite classifier works with the float MobileNet model. */ +public class ClassifierFloatMobileNet extends Classifier { + + /** Float MobileNet requires additional normalization of the used input. */ + private static final float IMAGE_MEAN = 127.5f; + + private static final float IMAGE_STD = 127.5f; + + /** + * Float model does not need dequantization in the post-processing. Setting mean and std as 0.0f + * and 1.0f, repectively, to bypass the normalization. + */ + private static final float PROBABILITY_MEAN = 0.0f; + + private static final float PROBABILITY_STD = 1.0f; + + /** + * Initializes a {@code ClassifierFloatMobileNet}. + * + * @param activity + */ + public ClassifierFloatMobileNet(Activity activity, Device device, int numThreads) + throws IOException { + super(activity, device, numThreads); + } + + @Override + protected String getModelPath() { + // you can download this file from + // see build.gradle for where to obtain this file. It should be auto + // downloaded into assets. + return "model_0.tflite"; + } + + @Override + protected String getLabelPath() { + return "labels.txt"; + } + + @Override + protected TensorOperator getPreprocessNormalizeOp() { + return new NormalizeOp(IMAGE_MEAN, IMAGE_STD); + } + + @Override + protected TensorOperator getPostprocessNormalizeOp() { + return new NormalizeOp(PROBABILITY_MEAN, PROBABILITY_STD); + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierQuantizedEfficientNet.java b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierQuantizedEfficientNet.java new file mode 100644 index 0000000000000000000000000000000000000000..d0d62f58d18333b6360ec30a4c85c9f1d38955ce --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierQuantizedEfficientNet.java @@ -0,0 +1,71 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.lite.examples.classification.tflite; + +import android.app.Activity; +import java.io.IOException; +import org.tensorflow.lite.support.common.TensorOperator; +import org.tensorflow.lite.support.common.ops.NormalizeOp; + +/** This TensorFlow Lite classifier works with the quantized EfficientNet model. */ +public class ClassifierQuantizedEfficientNet extends Classifier { + + /** + * The quantized model does not require normalization, thus set mean as 0.0f, and std as 1.0f to + * bypass the normalization. + */ + private static final float IMAGE_MEAN = 0.0f; + + private static final float IMAGE_STD = 1.0f; + + /** Quantized MobileNet requires additional dequantization to the output probability. */ + private static final float PROBABILITY_MEAN = 0.0f; + + private static final float PROBABILITY_STD = 255.0f; + + /** + * Initializes a {@code ClassifierQuantizedMobileNet}. + * + * @param activity + */ + public ClassifierQuantizedEfficientNet(Activity activity, Device device, int numThreads) + throws IOException { + super(activity, device, numThreads); + } + + @Override + protected String getModelPath() { + // you can download this file from + // see build.gradle for where to obtain this file. It should be auto + // downloaded into assets. + return "model_quant.tflite"; + } + + @Override + protected String getLabelPath() { + return "labels_without_background.txt"; + } + + @Override + protected TensorOperator getPreprocessNormalizeOp() { + return new NormalizeOp(IMAGE_MEAN, IMAGE_STD); + } + + @Override + protected TensorOperator getPostprocessNormalizeOp() { + return new NormalizeOp(PROBABILITY_MEAN, PROBABILITY_STD); + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierQuantizedMobileNet.java b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierQuantizedMobileNet.java new file mode 100644 index 0000000000000000000000000000000000000000..94b06e3df659005c287733a8a37672863fdadd71 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_support/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierQuantizedMobileNet.java @@ -0,0 +1,72 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.lite.examples.classification.tflite; + +import android.app.Activity; +import java.io.IOException; +import org.tensorflow.lite.examples.classification.tflite.Classifier.Device; +import org.tensorflow.lite.support.common.TensorOperator; +import org.tensorflow.lite.support.common.ops.NormalizeOp; + +/** This TensorFlow Lite classifier works with the quantized MobileNet model. */ +public class ClassifierQuantizedMobileNet extends Classifier { + + /** + * The quantized model does not require normalization, thus set mean as 0.0f, and std as 1.0f to + * bypass the normalization. + */ + private static final float IMAGE_MEAN = 0.0f; + + private static final float IMAGE_STD = 1.0f; + + /** Quantized MobileNet requires additional dequantization to the output probability. */ + private static final float PROBABILITY_MEAN = 0.0f; + + private static final float PROBABILITY_STD = 255.0f; + + /** + * Initializes a {@code ClassifierQuantizedMobileNet}. + * + * @param activity + */ + public ClassifierQuantizedMobileNet(Activity activity, Device device, int numThreads) + throws IOException { + super(activity, device, numThreads); + } + + @Override + protected String getModelPath() { + // you can download this file from + // see build.gradle for where to obtain this file. It should be auto + // downloaded into assets. + return "model_quant_0.tflite"; + } + + @Override + protected String getLabelPath() { + return "labels.txt"; + } + + @Override + protected TensorOperator getPreprocessNormalizeOp() { + return new NormalizeOp(IMAGE_MEAN, IMAGE_STD); + } + + @Override + protected TensorOperator getPostprocessNormalizeOp() { + return new NormalizeOp(PROBABILITY_MEAN, PROBABILITY_STD); + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/build.gradle b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/build.gradle new file mode 100644 index 0000000000000000000000000000000000000000..b5983986e3d56a77a41676b9195b0d0882b5fb96 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/build.gradle @@ -0,0 +1,47 @@ +apply plugin: 'com.android.library' + +android { + compileSdkVersion 28 + buildToolsVersion "28.0.0" + + defaultConfig { + minSdkVersion 21 + targetSdkVersion 28 + versionCode 1 + versionName "1.0" + + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + compileOptions { + sourceCompatibility = '1.8' + targetCompatibility = '1.8' + } + aaptOptions { + noCompress "tflite" + } + + lintOptions { + checkReleaseBuilds false + // Or, if you prefer, you can continue to check for errors in release builds, + // but continue the build even when errors are found: + abortOnError false + } +} + +dependencies { + implementation fileTree(dir: 'libs', include: ['*.jar']) + implementation project(":models") + implementation 'androidx.appcompat:appcompat:1.1.0' + + // Build off of nightly TensorFlow Lite Task Library + implementation('org.tensorflow:tensorflow-lite-task-vision:0.0.0-nightly') { changing = true } + implementation('org.tensorflow:tensorflow-lite-metadata:0.0.0-nightly') { changing = true } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/proguard-rules.pro b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/proguard-rules.pro new file mode 100644 index 0000000000000000000000000000000000000000..f1b424510da51fd82143bc74a0a801ae5a1e2fcd --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/src/main/AndroidManifest.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/src/main/AndroidManifest.xml new file mode 100644 index 0000000000000000000000000000000000000000..ebe3c56c60a9b67eec218d969aecfdf5311d7b49 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/src/main/AndroidManifest.xml @@ -0,0 +1,3 @@ + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/src/main/java/org/tensorflow/lite/examples/classification/tflite/Classifier.java b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/src/main/java/org/tensorflow/lite/examples/classification/tflite/Classifier.java new file mode 100644 index 0000000000000000000000000000000000000000..45da52a0d0dfa203255e0f2d44901ee0618e739f --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/src/main/java/org/tensorflow/lite/examples/classification/tflite/Classifier.java @@ -0,0 +1,278 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.lite.examples.classification.tflite; + +import static java.lang.Math.min; + +import android.app.Activity; +import android.graphics.Bitmap; +import android.graphics.Rect; +import android.graphics.RectF; +import android.os.SystemClock; +import android.os.Trace; +import android.util.Log; +import java.io.IOException; +import java.nio.MappedByteBuffer; +import java.util.ArrayList; +import java.util.List; +import org.tensorflow.lite.examples.classification.tflite.Classifier.Device; +import org.tensorflow.lite.support.common.FileUtil; +import org.tensorflow.lite.support.image.TensorImage; +import org.tensorflow.lite.support.label.Category; +import org.tensorflow.lite.support.metadata.MetadataExtractor; +import org.tensorflow.lite.task.core.vision.ImageProcessingOptions; +import org.tensorflow.lite.task.core.vision.ImageProcessingOptions.Orientation; +import org.tensorflow.lite.task.vision.classifier.Classifications; +import org.tensorflow.lite.task.vision.classifier.ImageClassifier; +import org.tensorflow.lite.task.vision.classifier.ImageClassifier.ImageClassifierOptions; + +/** A classifier specialized to label images using TensorFlow Lite. */ +public abstract class Classifier { + public static final String TAG = "ClassifierWithTaskApi"; + + /** The model type used for classification. */ + public enum Model { + FLOAT_MOBILENET, + QUANTIZED_MOBILENET, + FLOAT_EFFICIENTNET, + QUANTIZED_EFFICIENTNET + } + + /** The runtime device type used for executing classification. */ + public enum Device { + CPU, + NNAPI, + GPU + } + + /** Number of results to show in the UI. */ + private static final int MAX_RESULTS = 3; + + /** Image size along the x axis. */ + private final int imageSizeX; + + /** Image size along the y axis. */ + private final int imageSizeY; + /** An instance of the driver class to run model inference with Tensorflow Lite. */ + protected final ImageClassifier imageClassifier; + + /** + * Creates a classifier with the provided configuration. + * + * @param activity The current Activity. + * @param model The model to use for classification. + * @param device The device to use for classification. + * @param numThreads The number of threads to use for classification. + * @return A classifier with the desired configuration. + */ + public static Classifier create(Activity activity, Model model, Device device, int numThreads) + throws IOException { + if (model == Model.QUANTIZED_MOBILENET) { + return new ClassifierQuantizedMobileNet(activity, device, numThreads); + } else if (model == Model.FLOAT_MOBILENET) { + return new ClassifierFloatMobileNet(activity, device, numThreads); + } else if (model == Model.FLOAT_EFFICIENTNET) { + return new ClassifierFloatEfficientNet(activity, device, numThreads); + } else if (model == Model.QUANTIZED_EFFICIENTNET) { + return new ClassifierQuantizedEfficientNet(activity, device, numThreads); + } else { + throw new UnsupportedOperationException(); + } + } + + /** An immutable result returned by a Classifier describing what was recognized. */ + public static class Recognition { + /** + * A unique identifier for what has been recognized. Specific to the class, not the instance of + * the object. + */ + private final String id; + + /** Display name for the recognition. */ + private final String title; + + /** + * A sortable score for how good the recognition is relative to others. Higher should be better. + */ + private final Float confidence; + + /** Optional location within the source image for the location of the recognized object. */ + private RectF location; + + public Recognition( + final String id, final String title, final Float confidence, final RectF location) { + this.id = id; + this.title = title; + this.confidence = confidence; + this.location = location; + } + + public String getId() { + return id; + } + + public String getTitle() { + return title; + } + + public Float getConfidence() { + return confidence; + } + + public RectF getLocation() { + return new RectF(location); + } + + public void setLocation(RectF location) { + this.location = location; + } + + @Override + public String toString() { + String resultString = ""; + if (id != null) { + resultString += "[" + id + "] "; + } + + if (title != null) { + resultString += title + " "; + } + + if (confidence != null) { + resultString += String.format("(%.1f%%) ", confidence * 100.0f); + } + + if (location != null) { + resultString += location + " "; + } + + return resultString.trim(); + } + } + + /** Initializes a {@code Classifier}. */ + protected Classifier(Activity activity, Device device, int numThreads) throws IOException { + if (device != Device.CPU || numThreads != 1) { + throw new IllegalArgumentException( + "Manipulating the hardware accelerators and numbers of threads is not allowed in the Task" + + " library currently. Only CPU + single thread is allowed."); + } + + // Create the ImageClassifier instance. + ImageClassifierOptions options = + ImageClassifierOptions.builder().setMaxResults(MAX_RESULTS).build(); + imageClassifier = ImageClassifier.createFromFileAndOptions(activity, getModelPath(), options); + Log.d(TAG, "Created a Tensorflow Lite Image Classifier."); + + // Get the input image size information of the underlying tflite model. + MappedByteBuffer tfliteModel = FileUtil.loadMappedFile(activity, getModelPath()); + MetadataExtractor metadataExtractor = new MetadataExtractor(tfliteModel); + // Image shape is in the format of {1, height, width, 3}. + int[] imageShape = metadataExtractor.getInputTensorShape(/*inputIndex=*/ 0); + imageSizeY = imageShape[1]; + imageSizeX = imageShape[2]; + } + + /** Runs inference and returns the classification results. */ + public List recognizeImage(final Bitmap bitmap, int sensorOrientation) { + // Logs this method so that it can be analyzed with systrace. + Trace.beginSection("recognizeImage"); + + TensorImage inputImage = TensorImage.fromBitmap(bitmap); + int width = bitmap.getWidth(); + int height = bitmap.getHeight(); + int cropSize = min(width, height); + // TODO(b/169379396): investigate the impact of the resize algorithm on accuracy. + // Task Library resize the images using bilinear interpolation, which is slightly different from + // the nearest neighbor sampling algorithm used in lib_support. See + // https://github.com/tensorflow/examples/blob/0ef3d93e2af95d325c70ef3bcbbd6844d0631e07/lite/examples/image_classification/android/lib_support/src/main/java/org/tensorflow/lite/examples/classification/tflite/Classifier.java#L310. + ImageProcessingOptions imageOptions = + ImageProcessingOptions.builder() + .setOrientation(getOrientation(sensorOrientation)) + // Set the ROI to the center of the image. + .setRoi( + new Rect( + /*left=*/ (width - cropSize) / 2, + /*top=*/ (height - cropSize) / 2, + /*right=*/ (width + cropSize) / 2, + /*bottom=*/ (height + cropSize) / 2)) + .build(); + + // Runs the inference call. + Trace.beginSection("runInference"); + long startTimeForReference = SystemClock.uptimeMillis(); + List results = imageClassifier.classify(inputImage, imageOptions); + long endTimeForReference = SystemClock.uptimeMillis(); + Trace.endSection(); + Log.v(TAG, "Timecost to run model inference: " + (endTimeForReference - startTimeForReference)); + + Trace.endSection(); + + return getRecognitions(results); + } + + /** Closes the interpreter and model to release resources. */ + public void close() { + if (imageClassifier != null) { + imageClassifier.close(); + } + } + + /** Get the image size along the x axis. */ + public int getImageSizeX() { + return imageSizeX; + } + + /** Get the image size along the y axis. */ + public int getImageSizeY() { + return imageSizeY; + } + + /** + * Converts a list of {@link Classifications} objects into a list of {@link Recognition} objects + * to match the interface of other inference method, such as using the TFLite + * Support Library.. + */ + private static List getRecognitions(List classifications) { + + final ArrayList recognitions = new ArrayList<>(); + // All the demo models are single head models. Get the first Classifications in the results. + for (Category category : classifications.get(0).getCategories()) { + recognitions.add( + new Recognition( + "" + category.getLabel(), category.getLabel(), category.getScore(), null)); + } + return recognitions; + } + + /* Convert the camera orientation in degree into {@link ImageProcessingOptions#Orientation}.*/ + private static Orientation getOrientation(int cameraOrientation) { + switch (cameraOrientation / 90) { + case 3: + return Orientation.BOTTOM_LEFT; + case 2: + return Orientation.BOTTOM_RIGHT; + case 1: + return Orientation.TOP_RIGHT; + default: + return Orientation.TOP_LEFT; + } + } + + /** Gets the name of the model file stored in Assets. */ + protected abstract String getModelPath(); +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierFloatEfficientNet.java b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierFloatEfficientNet.java new file mode 100644 index 0000000000000000000000000000000000000000..250794cc12d0e603aa47502322dc646d50689848 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierFloatEfficientNet.java @@ -0,0 +1,45 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.lite.examples.classification.tflite; + +import android.app.Activity; +import java.io.IOException; +import org.tensorflow.lite.examples.classification.tflite.Classifier.Device; + +/** This TensorFlowLite classifier works with the float EfficientNet model. */ +public class ClassifierFloatEfficientNet extends Classifier { + + /** + * Initializes a {@code ClassifierFloatMobileNet}. + * + * @param device a {@link Device} object to configure the hardware accelerator + * @param numThreads the number of threads during the inference + * @throws IOException if the model is not loaded correctly + */ + public ClassifierFloatEfficientNet(Activity activity, Device device, int numThreads) + throws IOException { + super(activity, device, numThreads); + } + + @Override + protected String getModelPath() { + // you can download this file from + // see build.gradle for where to obtain this file. It should be auto + // downloaded into assets. + //return "efficientnet-lite0-fp32.tflite"; + return "model.tflite"; + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierFloatMobileNet.java b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierFloatMobileNet.java new file mode 100644 index 0000000000000000000000000000000000000000..0707de98de41395eaf3ddcfd74d6e36229a63760 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierFloatMobileNet.java @@ -0,0 +1,43 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.lite.examples.classification.tflite; + +import android.app.Activity; +import java.io.IOException; +import org.tensorflow.lite.examples.classification.tflite.Classifier.Device; + +/** This TensorFlowLite classifier works with the float MobileNet model. */ +public class ClassifierFloatMobileNet extends Classifier { + /** + * Initializes a {@code ClassifierFloatMobileNet}. + * + * @param device a {@link Device} object to configure the hardware accelerator + * @param numThreads the number of threads during the inference + * @throws IOException if the model is not loaded correctly + */ + public ClassifierFloatMobileNet(Activity activity, Device device, int numThreads) + throws IOException { + super(activity, device, numThreads); + } + + @Override + protected String getModelPath() { + // you can download this file from + // see build.gradle for where to obtain this file. It should be auto + // downloaded into assets. + return "mobilenet_v1_1.0_224.tflite"; + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierQuantizedEfficientNet.java b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierQuantizedEfficientNet.java new file mode 100644 index 0000000000000000000000000000000000000000..05ca4fa6c409d0274a396c9b26c3c39ca8a8194e --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierQuantizedEfficientNet.java @@ -0,0 +1,43 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.lite.examples.classification.tflite; + +import android.app.Activity; +import java.io.IOException; + +/** This TensorFlow Lite classifier works with the quantized EfficientNet model. */ +public class ClassifierQuantizedEfficientNet extends Classifier { + + /** + * Initializes a {@code ClassifierQuantizedMobileNet}. + * + * @param device a {@link Device} object to configure the hardware accelerator + * @param numThreads the number of threads during the inference + * @throws IOException if the model is not loaded correctly + */ + public ClassifierQuantizedEfficientNet(Activity activity, Device device, int numThreads) + throws IOException { + super(activity, device, numThreads); + } + + @Override + protected String getModelPath() { + // you can download this file from + // see build.gradle for where to obtain this file. It should be auto + // downloaded into assets. + return "efficientnet-lite0-int8.tflite"; + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierQuantizedMobileNet.java b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierQuantizedMobileNet.java new file mode 100644 index 0000000000000000000000000000000000000000..978b08eeaf52a23eede437d61045db08d1dff163 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/lib_task_api/src/main/java/org/tensorflow/lite/examples/classification/tflite/ClassifierQuantizedMobileNet.java @@ -0,0 +1,44 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.lite.examples.classification.tflite; + +import android.app.Activity; +import java.io.IOException; +import org.tensorflow.lite.examples.classification.tflite.Classifier.Device; + +/** This TensorFlow Lite classifier works with the quantized MobileNet model. */ +public class ClassifierQuantizedMobileNet extends Classifier { + + /** + * Initializes a {@code ClassifierQuantizedMobileNet}. + * + * @param device a {@link Device} object to configure the hardware accelerator + * @param numThreads the number of threads during the inference + * @throws IOException if the model is not loaded correctly + */ + public ClassifierQuantizedMobileNet(Activity activity, Device device, int numThreads) + throws IOException { + super(activity, device, numThreads); + } + + @Override + protected String getModelPath() { + // you can download this file from + // see build.gradle for where to obtain this file. It should be auto + // downloaded into assets. + return "mobilenet_v1_1.0_224_quant.tflite"; + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/build.gradle b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/build.gradle new file mode 100644 index 0000000000000000000000000000000000000000..8d825707af20cbbead6c4599f075599148e3511c --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/build.gradle @@ -0,0 +1,40 @@ +apply plugin: 'com.android.library' +apply plugin: 'de.undercouch.download' + +android { + compileSdkVersion 28 + buildToolsVersion "28.0.0" + + defaultConfig { + minSdkVersion 21 + targetSdkVersion 28 + versionCode 1 + versionName "1.0" + + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" + + } + + buildTypes { + release { + minifyEnabled false + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' + } + } + + aaptOptions { + noCompress "tflite" + } + + lintOptions { + checkReleaseBuilds false + // Or, if you prefer, you can continue to check for errors in release builds, + // but continue the build even when errors are found: + abortOnError false + } +} + +// Download default models; if you wish to use your own models then +// place them in the "assets" directory and comment out this line. +project.ext.ASSET_DIR = projectDir.toString() + '/src/main/assets' +apply from:'download.gradle' diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/download.gradle b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/download.gradle new file mode 100644 index 0000000000000000000000000000000000000000..ce76974a2c3bc6f8214461028e0dfa6ebc25d588 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/download.gradle @@ -0,0 +1,10 @@ +def modelFloatDownloadUrl = "https://github.com/isl-org/MiDaS/releases/download/v2_1/model_opt.tflite" +def modelFloatFile = "model_opt.tflite" + +task downloadModelFloat(type: Download) { + src "${modelFloatDownloadUrl}" + dest project.ext.ASSET_DIR + "/${modelFloatFile}" + overwrite false +} + +preBuild.dependsOn downloadModelFloat diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/proguard-rules.pro b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/proguard-rules.pro new file mode 100644 index 0000000000000000000000000000000000000000..f1b424510da51fd82143bc74a0a801ae5a1e2fcd --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/proguard-rules.pro @@ -0,0 +1,21 @@ +# Add project specific ProGuard rules here. +# You can control the set of applied configuration files using the +# proguardFiles setting in build.gradle. +# +# For more details, see +# http://developer.android.com/guide/developing/tools/proguard.html + +# If your project uses WebView with JS, uncomment the following +# and specify the fully qualified class name to the JavaScript interface +# class: +#-keepclassmembers class fqcn.of.javascript.interface.for.webview { +# public *; +#} + +# Uncomment this to preserve the line number information for +# debugging stack traces. +#-keepattributes SourceFile,LineNumberTable + +# If you keep the line number information, uncomment this to +# hide the original source file name. +#-renamesourcefileattribute SourceFile diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/src/main/AndroidManifest.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/src/main/AndroidManifest.xml new file mode 100644 index 0000000000000000000000000000000000000000..42951a56497c5f947efe4aea6a07462019fb152c --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/src/main/AndroidManifest.xml @@ -0,0 +1,3 @@ + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/src/main/assets/labels.txt b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/src/main/assets/labels.txt new file mode 100644 index 0000000000000000000000000000000000000000..fe811239d8e2989de19fecabb1ebb0c9dddac514 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/src/main/assets/labels.txt @@ -0,0 +1,1001 @@ +background +tench +goldfish +great white shark +tiger shark +hammerhead +electric ray +stingray +cock +hen +ostrich +brambling +goldfinch +house finch +junco +indigo bunting +robin +bulbul +jay +magpie +chickadee +water ouzel +kite +bald eagle +vulture +great grey owl +European fire salamander +common newt +eft +spotted salamander +axolotl +bullfrog +tree frog +tailed frog +loggerhead +leatherback turtle +mud turtle +terrapin +box turtle +banded gecko +common iguana +American chameleon +whiptail +agama +frilled lizard +alligator lizard +Gila monster +green lizard +African chameleon +Komodo dragon +African crocodile +American alligator +triceratops +thunder snake +ringneck snake +hognose snake +green snake +king snake +garter snake +water snake +vine snake +night snake +boa constrictor +rock python +Indian cobra +green mamba +sea snake +horned viper +diamondback +sidewinder +trilobite +harvestman +scorpion +black and gold garden spider +barn spider +garden spider +black widow +tarantula +wolf spider +tick +centipede +black grouse +ptarmigan +ruffed grouse +prairie chicken +peacock +quail +partridge +African grey +macaw +sulphur-crested cockatoo +lorikeet +coucal +bee eater +hornbill +hummingbird +jacamar +toucan +drake +red-breasted merganser +goose +black swan +tusker +echidna +platypus +wallaby +koala +wombat +jellyfish +sea anemone +brain coral +flatworm +nematode +conch +snail +slug +sea slug +chiton +chambered nautilus +Dungeness crab +rock crab +fiddler crab +king crab +American lobster +spiny lobster +crayfish +hermit crab +isopod +white stork +black stork +spoonbill +flamingo +little blue heron +American egret +bittern +crane +limpkin +European gallinule +American coot +bustard +ruddy turnstone +red-backed sandpiper +redshank +dowitcher +oystercatcher +pelican +king penguin +albatross +grey whale +killer whale +dugong +sea lion +Chihuahua +Japanese spaniel +Maltese dog +Pekinese +Shih-Tzu +Blenheim spaniel +papillon +toy terrier +Rhodesian ridgeback +Afghan hound +basset +beagle +bloodhound +bluetick +black-and-tan coonhound +Walker hound +English foxhound +redbone +borzoi +Irish wolfhound +Italian greyhound +whippet +Ibizan hound +Norwegian elkhound +otterhound +Saluki +Scottish deerhound +Weimaraner +Staffordshire bullterrier +American Staffordshire terrier +Bedlington terrier +Border terrier +Kerry blue terrier +Irish terrier +Norfolk terrier +Norwich terrier +Yorkshire terrier +wire-haired fox terrier +Lakeland terrier +Sealyham terrier +Airedale +cairn +Australian terrier +Dandie Dinmont +Boston bull +miniature schnauzer +giant schnauzer +standard schnauzer +Scotch terrier +Tibetan terrier +silky terrier +soft-coated wheaten terrier +West Highland white terrier +Lhasa +flat-coated retriever +curly-coated retriever +golden retriever +Labrador retriever +Chesapeake Bay retriever +German short-haired pointer +vizsla +English setter +Irish setter +Gordon setter +Brittany spaniel +clumber +English springer +Welsh springer spaniel +cocker spaniel +Sussex spaniel +Irish water spaniel +kuvasz +schipperke +groenendael +malinois +briard +kelpie +komondor +Old English sheepdog +Shetland sheepdog +collie +Border collie +Bouvier des Flandres +Rottweiler +German shepherd +Doberman +miniature pinscher +Greater Swiss Mountain dog +Bernese mountain dog +Appenzeller +EntleBucher +boxer +bull mastiff +Tibetan mastiff +French bulldog +Great Dane +Saint Bernard +Eskimo dog +malamute +Siberian husky +dalmatian +affenpinscher +basenji +pug +Leonberg +Newfoundland +Great Pyrenees +Samoyed +Pomeranian +chow +keeshond +Brabancon griffon +Pembroke +Cardigan +toy poodle +miniature poodle +standard poodle +Mexican hairless +timber wolf +white wolf +red wolf +coyote +dingo +dhole +African hunting dog +hyena +red fox +kit fox +Arctic fox +grey fox +tabby +tiger cat +Persian cat +Siamese cat +Egyptian cat +cougar +lynx +leopard +snow leopard +jaguar +lion +tiger +cheetah +brown bear +American black bear +ice bear +sloth bear +mongoose +meerkat +tiger beetle +ladybug +ground beetle +long-horned beetle +leaf beetle +dung beetle +rhinoceros beetle +weevil +fly +bee +ant +grasshopper +cricket +walking stick +cockroach +mantis +cicada +leafhopper +lacewing +dragonfly +damselfly +admiral +ringlet +monarch +cabbage butterfly +sulphur butterfly +lycaenid +starfish +sea urchin +sea cucumber +wood rabbit +hare +Angora +hamster +porcupine +fox squirrel +marmot +beaver +guinea pig +sorrel +zebra +hog +wild boar +warthog +hippopotamus +ox +water buffalo +bison +ram +bighorn +ibex +hartebeest +impala +gazelle +Arabian camel +llama +weasel +mink +polecat +black-footed ferret +otter +skunk +badger +armadillo +three-toed sloth +orangutan +gorilla +chimpanzee +gibbon +siamang +guenon +patas +baboon +macaque +langur +colobus +proboscis monkey +marmoset +capuchin +howler monkey +titi +spider monkey +squirrel monkey +Madagascar cat +indri +Indian elephant +African elephant +lesser panda +giant panda +barracouta +eel +coho +rock beauty +anemone fish +sturgeon +gar +lionfish +puffer +abacus +abaya +academic gown +accordion +acoustic guitar +aircraft carrier +airliner +airship +altar +ambulance +amphibian +analog clock +apiary +apron +ashcan +assault rifle +backpack +bakery +balance beam +balloon +ballpoint +Band Aid +banjo +bannister +barbell +barber chair +barbershop +barn +barometer +barrel +barrow +baseball +basketball +bassinet +bassoon +bathing cap +bath towel +bathtub +beach wagon +beacon +beaker +bearskin +beer bottle +beer glass +bell cote +bib +bicycle-built-for-two +bikini +binder +binoculars +birdhouse +boathouse +bobsled +bolo tie +bonnet +bookcase +bookshop +bottlecap +bow +bow tie +brass +brassiere +breakwater +breastplate +broom +bucket +buckle +bulletproof vest +bullet train +butcher shop +cab +caldron +candle +cannon +canoe +can opener +cardigan +car mirror +carousel +carpenter's kit +carton +car wheel +cash machine +cassette +cassette player +castle +catamaran +CD player +cello +cellular telephone +chain +chainlink fence +chain mail +chain saw +chest +chiffonier +chime +china cabinet +Christmas stocking +church +cinema +cleaver +cliff dwelling +cloak +clog +cocktail shaker +coffee mug +coffeepot +coil +combination lock +computer keyboard +confectionery +container ship +convertible +corkscrew +cornet +cowboy boot +cowboy hat +cradle +crane +crash helmet +crate +crib +Crock Pot +croquet ball +crutch +cuirass +dam +desk +desktop computer +dial telephone +diaper +digital clock +digital watch +dining table +dishrag +dishwasher +disk brake +dock +dogsled +dome +doormat +drilling platform +drum +drumstick +dumbbell +Dutch oven +electric fan +electric guitar +electric locomotive +entertainment center +envelope +espresso maker +face powder +feather boa +file +fireboat +fire engine +fire screen +flagpole +flute +folding chair +football helmet +forklift +fountain +fountain pen +four-poster +freight car +French horn +frying pan +fur coat +garbage truck +gasmask +gas pump +goblet +go-kart +golf ball +golfcart +gondola +gong +gown +grand piano +greenhouse +grille +grocery store +guillotine +hair slide +hair spray +half track +hammer +hamper +hand blower +hand-held computer +handkerchief +hard disc +harmonica +harp +harvester +hatchet +holster +home theater +honeycomb +hook +hoopskirt +horizontal bar +horse cart +hourglass +iPod +iron +jack-o'-lantern +jean +jeep +jersey +jigsaw puzzle +jinrikisha +joystick +kimono +knee pad +knot +lab coat +ladle +lampshade +laptop +lawn mower +lens cap +letter opener +library +lifeboat +lighter +limousine +liner +lipstick +Loafer +lotion +loudspeaker +loupe +lumbermill +magnetic compass +mailbag +mailbox +maillot +maillot +manhole cover +maraca +marimba +mask +matchstick +maypole +maze +measuring cup +medicine chest +megalith +microphone +microwave +military uniform +milk can +minibus +miniskirt +minivan +missile +mitten +mixing bowl +mobile home +Model T +modem +monastery +monitor +moped +mortar +mortarboard +mosque +mosquito net +motor scooter +mountain bike +mountain tent +mouse +mousetrap +moving van +muzzle +nail +neck brace +necklace +nipple +notebook +obelisk +oboe +ocarina +odometer +oil filter +organ +oscilloscope +overskirt +oxcart +oxygen mask +packet +paddle +paddlewheel +padlock +paintbrush +pajama +palace +panpipe +paper towel +parachute +parallel bars +park bench +parking meter +passenger car +patio +pay-phone +pedestal +pencil box +pencil sharpener +perfume +Petri dish +photocopier +pick +pickelhaube +picket fence +pickup +pier +piggy bank +pill bottle +pillow +ping-pong ball +pinwheel +pirate +pitcher +plane +planetarium +plastic bag +plate rack +plow +plunger +Polaroid camera +pole +police van +poncho +pool table +pop bottle +pot +potter's wheel +power drill +prayer rug +printer +prison +projectile +projector +puck +punching bag +purse +quill +quilt +racer +racket +radiator +radio +radio telescope +rain barrel +recreational vehicle +reel +reflex camera +refrigerator +remote control +restaurant +revolver +rifle +rocking chair +rotisserie +rubber eraser +rugby ball +rule +running shoe +safe +safety pin +saltshaker +sandal +sarong +sax +scabbard +scale +school bus +schooner +scoreboard +screen +screw +screwdriver +seat belt +sewing machine +shield +shoe shop +shoji +shopping basket +shopping cart +shovel +shower cap +shower curtain +ski +ski mask +sleeping bag +slide rule +sliding door +slot +snorkel +snowmobile +snowplow +soap dispenser +soccer ball +sock +solar dish +sombrero +soup bowl +space bar +space heater +space shuttle +spatula +speedboat +spider web +spindle +sports car +spotlight +stage +steam locomotive +steel arch bridge +steel drum +stethoscope +stole +stone wall +stopwatch +stove +strainer +streetcar +stretcher +studio couch +stupa +submarine +suit +sundial +sunglass +sunglasses +sunscreen +suspension bridge +swab +sweatshirt +swimming trunks +swing +switch +syringe +table lamp +tank +tape player +teapot +teddy +television +tennis ball +thatch +theater curtain +thimble +thresher +throne +tile roof +toaster +tobacco shop +toilet seat +torch +totem pole +tow truck +toyshop +tractor +trailer truck +tray +trench coat +tricycle +trimaran +tripod +triumphal arch +trolleybus +trombone +tub +turnstile +typewriter keyboard +umbrella +unicycle +upright +vacuum +vase +vault +velvet +vending machine +vestment +viaduct +violin +volleyball +waffle iron +wall clock +wallet +wardrobe +warplane +washbasin +washer +water bottle +water jug +water tower +whiskey jug +whistle +wig +window screen +window shade +Windsor tie +wine bottle +wing +wok +wooden spoon +wool +worm fence +wreck +yawl +yurt +web site +comic book +crossword puzzle +street sign +traffic light +book jacket +menu +plate +guacamole +consomme +hot pot +trifle +ice cream +ice lolly +French loaf +bagel +pretzel +cheeseburger +hotdog +mashed potato +head cabbage +broccoli +cauliflower +zucchini +spaghetti squash +acorn squash +butternut squash +cucumber +artichoke +bell pepper +cardoon +mushroom +Granny Smith +strawberry +orange +lemon +fig +pineapple +banana +jackfruit +custard apple +pomegranate +hay +carbonara +chocolate sauce +dough +meat loaf +pizza +potpie +burrito +red wine +espresso +cup +eggnog +alp +bubble +cliff +coral reef +geyser +lakeside +promontory +sandbar +seashore +valley +volcano +ballplayer +groom +scuba diver +rapeseed +daisy +yellow lady's slipper +corn +acorn +hip +buckeye +coral fungus +agaric +gyromitra +stinkhorn +earthstar +hen-of-the-woods +bolete +ear +toilet tissue diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/src/main/assets/labels_without_background.txt b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/src/main/assets/labels_without_background.txt new file mode 100644 index 0000000000000000000000000000000000000000..f40829ed0fc318c673860fae4be6c48529da116e --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/src/main/assets/labels_without_background.txt @@ -0,0 +1,1000 @@ +tench +goldfish +great white shark +tiger shark +hammerhead +electric ray +stingray +cock +hen +ostrich +brambling +goldfinch +house finch +junco +indigo bunting +robin +bulbul +jay +magpie +chickadee +water ouzel +kite +bald eagle +vulture +great grey owl +European fire salamander +common newt +eft +spotted salamander +axolotl +bullfrog +tree frog +tailed frog +loggerhead +leatherback turtle +mud turtle +terrapin +box turtle +banded gecko +common iguana +American chameleon +whiptail +agama +frilled lizard +alligator lizard +Gila monster +green lizard +African chameleon +Komodo dragon +African crocodile +American alligator +triceratops +thunder snake +ringneck snake +hognose snake +green snake +king snake +garter snake +water snake +vine snake +night snake +boa constrictor +rock python +Indian cobra +green mamba +sea snake +horned viper +diamondback +sidewinder +trilobite +harvestman +scorpion +black and gold garden spider +barn spider +garden spider +black widow +tarantula +wolf spider +tick +centipede +black grouse +ptarmigan +ruffed grouse +prairie chicken +peacock +quail +partridge +African grey +macaw +sulphur-crested cockatoo +lorikeet +coucal +bee eater +hornbill +hummingbird +jacamar +toucan +drake +red-breasted merganser +goose +black swan +tusker +echidna +platypus +wallaby +koala +wombat +jellyfish +sea anemone +brain coral +flatworm +nematode +conch +snail +slug +sea slug +chiton +chambered nautilus +Dungeness crab +rock crab +fiddler crab +king crab +American lobster +spiny lobster +crayfish +hermit crab +isopod +white stork +black stork +spoonbill +flamingo +little blue heron +American egret +bittern +crane +limpkin +European gallinule +American coot +bustard +ruddy turnstone +red-backed sandpiper +redshank +dowitcher +oystercatcher +pelican +king penguin +albatross +grey whale +killer whale +dugong +sea lion +Chihuahua +Japanese spaniel +Maltese dog +Pekinese +Shih-Tzu +Blenheim spaniel +papillon +toy terrier +Rhodesian ridgeback +Afghan hound +basset +beagle +bloodhound +bluetick +black-and-tan coonhound +Walker hound +English foxhound +redbone +borzoi +Irish wolfhound +Italian greyhound +whippet +Ibizan hound +Norwegian elkhound +otterhound +Saluki +Scottish deerhound +Weimaraner +Staffordshire bullterrier +American Staffordshire terrier +Bedlington terrier +Border terrier +Kerry blue terrier +Irish terrier +Norfolk terrier +Norwich terrier +Yorkshire terrier +wire-haired fox terrier +Lakeland terrier +Sealyham terrier +Airedale +cairn +Australian terrier +Dandie Dinmont +Boston bull +miniature schnauzer +giant schnauzer +standard schnauzer +Scotch terrier +Tibetan terrier +silky terrier +soft-coated wheaten terrier +West Highland white terrier +Lhasa +flat-coated retriever +curly-coated retriever +golden retriever +Labrador retriever +Chesapeake Bay retriever +German short-haired pointer +vizsla +English setter +Irish setter +Gordon setter +Brittany spaniel +clumber +English springer +Welsh springer spaniel +cocker spaniel +Sussex spaniel +Irish water spaniel +kuvasz +schipperke +groenendael +malinois +briard +kelpie +komondor +Old English sheepdog +Shetland sheepdog +collie +Border collie +Bouvier des Flandres +Rottweiler +German shepherd +Doberman +miniature pinscher +Greater Swiss Mountain dog +Bernese mountain dog +Appenzeller +EntleBucher +boxer +bull mastiff +Tibetan mastiff +French bulldog +Great Dane +Saint Bernard +Eskimo dog +malamute +Siberian husky +dalmatian +affenpinscher +basenji +pug +Leonberg +Newfoundland +Great Pyrenees +Samoyed +Pomeranian +chow +keeshond +Brabancon griffon +Pembroke +Cardigan +toy poodle +miniature poodle +standard poodle +Mexican hairless +timber wolf +white wolf +red wolf +coyote +dingo +dhole +African hunting dog +hyena +red fox +kit fox +Arctic fox +grey fox +tabby +tiger cat +Persian cat +Siamese cat +Egyptian cat +cougar +lynx +leopard +snow leopard +jaguar +lion +tiger +cheetah +brown bear +American black bear +ice bear +sloth bear +mongoose +meerkat +tiger beetle +ladybug +ground beetle +long-horned beetle +leaf beetle +dung beetle +rhinoceros beetle +weevil +fly +bee +ant +grasshopper +cricket +walking stick +cockroach +mantis +cicada +leafhopper +lacewing +dragonfly +damselfly +admiral +ringlet +monarch +cabbage butterfly +sulphur butterfly +lycaenid +starfish +sea urchin +sea cucumber +wood rabbit +hare +Angora +hamster +porcupine +fox squirrel +marmot +beaver +guinea pig +sorrel +zebra +hog +wild boar +warthog +hippopotamus +ox +water buffalo +bison +ram +bighorn +ibex +hartebeest +impala +gazelle +Arabian camel +llama +weasel +mink +polecat +black-footed ferret +otter +skunk +badger +armadillo +three-toed sloth +orangutan +gorilla +chimpanzee +gibbon +siamang +guenon +patas +baboon +macaque +langur +colobus +proboscis monkey +marmoset +capuchin +howler monkey +titi +spider monkey +squirrel monkey +Madagascar cat +indri +Indian elephant +African elephant +lesser panda +giant panda +barracouta +eel +coho +rock beauty +anemone fish +sturgeon +gar +lionfish +puffer +abacus +abaya +academic gown +accordion +acoustic guitar +aircraft carrier +airliner +airship +altar +ambulance +amphibian +analog clock +apiary +apron +ashcan +assault rifle +backpack +bakery +balance beam +balloon +ballpoint +Band Aid +banjo +bannister +barbell +barber chair +barbershop +barn +barometer +barrel +barrow +baseball +basketball +bassinet +bassoon +bathing cap +bath towel +bathtub +beach wagon +beacon +beaker +bearskin +beer bottle +beer glass +bell cote +bib +bicycle-built-for-two +bikini +binder +binoculars +birdhouse +boathouse +bobsled +bolo tie +bonnet +bookcase +bookshop +bottlecap +bow +bow tie +brass +brassiere +breakwater +breastplate +broom +bucket +buckle +bulletproof vest +bullet train +butcher shop +cab +caldron +candle +cannon +canoe +can opener +cardigan +car mirror +carousel +carpenter's kit +carton +car wheel +cash machine +cassette +cassette player +castle +catamaran +CD player +cello +cellular telephone +chain +chainlink fence +chain mail +chain saw +chest +chiffonier +chime +china cabinet +Christmas stocking +church +cinema +cleaver +cliff dwelling +cloak +clog +cocktail shaker +coffee mug +coffeepot +coil +combination lock +computer keyboard +confectionery +container ship +convertible +corkscrew +cornet +cowboy boot +cowboy hat +cradle +crane +crash helmet +crate +crib +Crock Pot +croquet ball +crutch +cuirass +dam +desk +desktop computer +dial telephone +diaper +digital clock +digital watch +dining table +dishrag +dishwasher +disk brake +dock +dogsled +dome +doormat +drilling platform +drum +drumstick +dumbbell +Dutch oven +electric fan +electric guitar +electric locomotive +entertainment center +envelope +espresso maker +face powder +feather boa +file +fireboat +fire engine +fire screen +flagpole +flute +folding chair +football helmet +forklift +fountain +fountain pen +four-poster +freight car +French horn +frying pan +fur coat +garbage truck +gasmask +gas pump +goblet +go-kart +golf ball +golfcart +gondola +gong +gown +grand piano +greenhouse +grille +grocery store +guillotine +hair slide +hair spray +half track +hammer +hamper +hand blower +hand-held computer +handkerchief +hard disc +harmonica +harp +harvester +hatchet +holster +home theater +honeycomb +hook +hoopskirt +horizontal bar +horse cart +hourglass +iPod +iron +jack-o'-lantern +jean +jeep +jersey +jigsaw puzzle +jinrikisha +joystick +kimono +knee pad +knot +lab coat +ladle +lampshade +laptop +lawn mower +lens cap +letter opener +library +lifeboat +lighter +limousine +liner +lipstick +Loafer +lotion +loudspeaker +loupe +lumbermill +magnetic compass +mailbag +mailbox +maillot +maillot +manhole cover +maraca +marimba +mask +matchstick +maypole +maze +measuring cup +medicine chest +megalith +microphone +microwave +military uniform +milk can +minibus +miniskirt +minivan +missile +mitten +mixing bowl +mobile home +Model T +modem +monastery +monitor +moped +mortar +mortarboard +mosque +mosquito net +motor scooter +mountain bike +mountain tent +mouse +mousetrap +moving van +muzzle +nail +neck brace +necklace +nipple +notebook +obelisk +oboe +ocarina +odometer +oil filter +organ +oscilloscope +overskirt +oxcart +oxygen mask +packet +paddle +paddlewheel +padlock +paintbrush +pajama +palace +panpipe +paper towel +parachute +parallel bars +park bench +parking meter +passenger car +patio +pay-phone +pedestal +pencil box +pencil sharpener +perfume +Petri dish +photocopier +pick +pickelhaube +picket fence +pickup +pier +piggy bank +pill bottle +pillow +ping-pong ball +pinwheel +pirate +pitcher +plane +planetarium +plastic bag +plate rack +plow +plunger +Polaroid camera +pole +police van +poncho +pool table +pop bottle +pot +potter's wheel +power drill +prayer rug +printer +prison +projectile +projector +puck +punching bag +purse +quill +quilt +racer +racket +radiator +radio +radio telescope +rain barrel +recreational vehicle +reel +reflex camera +refrigerator +remote control +restaurant +revolver +rifle +rocking chair +rotisserie +rubber eraser +rugby ball +rule +running shoe +safe +safety pin +saltshaker +sandal +sarong +sax +scabbard +scale +school bus +schooner +scoreboard +screen +screw +screwdriver +seat belt +sewing machine +shield +shoe shop +shoji +shopping basket +shopping cart +shovel +shower cap +shower curtain +ski +ski mask +sleeping bag +slide rule +sliding door +slot +snorkel +snowmobile +snowplow +soap dispenser +soccer ball +sock +solar dish +sombrero +soup bowl +space bar +space heater +space shuttle +spatula +speedboat +spider web +spindle +sports car +spotlight +stage +steam locomotive +steel arch bridge +steel drum +stethoscope +stole +stone wall +stopwatch +stove +strainer +streetcar +stretcher +studio couch +stupa +submarine +suit +sundial +sunglass +sunglasses +sunscreen +suspension bridge +swab +sweatshirt +swimming trunks +swing +switch +syringe +table lamp +tank +tape player +teapot +teddy +television +tennis ball +thatch +theater curtain +thimble +thresher +throne +tile roof +toaster +tobacco shop +toilet seat +torch +totem pole +tow truck +toyshop +tractor +trailer truck +tray +trench coat +tricycle +trimaran +tripod +triumphal arch +trolleybus +trombone +tub +turnstile +typewriter keyboard +umbrella +unicycle +upright +vacuum +vase +vault +velvet +vending machine +vestment +viaduct +violin +volleyball +waffle iron +wall clock +wallet +wardrobe +warplane +washbasin +washer +water bottle +water jug +water tower +whiskey jug +whistle +wig +window screen +window shade +Windsor tie +wine bottle +wing +wok +wooden spoon +wool +worm fence +wreck +yawl +yurt +web site +comic book +crossword puzzle +street sign +traffic light +book jacket +menu +plate +guacamole +consomme +hot pot +trifle +ice cream +ice lolly +French loaf +bagel +pretzel +cheeseburger +hotdog +mashed potato +head cabbage +broccoli +cauliflower +zucchini +spaghetti squash +acorn squash +butternut squash +cucumber +artichoke +bell pepper +cardoon +mushroom +Granny Smith +strawberry +orange +lemon +fig +pineapple +banana +jackfruit +custard apple +pomegranate +hay +carbonara +chocolate sauce +dough +meat loaf +pizza +potpie +burrito +red wine +espresso +cup +eggnog +alp +bubble +cliff +coral reef +geyser +lakeside +promontory +sandbar +seashore +valley +volcano +ballplayer +groom +scuba diver +rapeseed +daisy +yellow lady's slipper +corn +acorn +hip +buckeye +coral fungus +agaric +gyromitra +stinkhorn +earthstar +hen-of-the-woods +bolete +ear +toilet tissue diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/src/main/assets/run_tflite.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/src/main/assets/run_tflite.py new file mode 100644 index 0000000000000000000000000000000000000000..4b8ebe235758d3d0f3d357c51ed54d78ac7eea8e --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/models/src/main/assets/run_tflite.py @@ -0,0 +1,75 @@ +# Flex ops are included in the nightly build of the TensorFlow Python package. You can use TFLite models containing Flex ops by the same Python API as normal TFLite models. The nightly TensorFlow build can be installed with this command: +# Flex ops will be added to the TensorFlow Python package's and the tflite_runtime package from version 2.3 for Linux and 2.4 for other environments. +# https://www.tensorflow.org/lite/guide/ops_select#running_the_model + +# You must use: tf-nightly +# pip install tf-nightly + +import os +import glob +import cv2 +import numpy as np + +import tensorflow as tf + +width=256 +height=256 +model_name="model.tflite" +#model_name="model_quant.tflite" +image_name="dog.jpg" + +# input +img = cv2.imread(image_name) +img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) / 255.0 + +mean=[0.485, 0.456, 0.406] +std=[0.229, 0.224, 0.225] +img = (img - mean) / std + +img_resized = tf.image.resize(img, [width,height], method='bicubic', preserve_aspect_ratio=False) +#img_resized = tf.transpose(img_resized, [2, 0, 1]) +img_input = img_resized.numpy() +reshape_img = img_input.reshape(1,width,height,3) +tensor = tf.convert_to_tensor(reshape_img, dtype=tf.float32) + +# load model +print("Load model...") +interpreter = tf.lite.Interpreter(model_path=model_name) +print("Allocate tensor...") +interpreter.allocate_tensors() +print("Get input/output details...") +input_details = interpreter.get_input_details() +output_details = interpreter.get_output_details() +print("Get input shape...") +input_shape = input_details[0]['shape'] +print(input_shape) +print(input_details) +print(output_details) +#input_data = np.array(np.random.random_sample(input_shape), dtype=np.float32) +print("Set input tensor...") +interpreter.set_tensor(input_details[0]['index'], tensor) + +print("invoke()...") +interpreter.invoke() + +# The function `get_tensor()` returns a copy of the tensor data. +# Use `tensor()` in order to get a pointer to the tensor. +print("get output tensor...") +output = interpreter.get_tensor(output_details[0]['index']) +#output = np.squeeze(output) +output = output.reshape(width, height) +#print(output) +prediction = np.array(output) +print("reshape prediction...") +prediction = prediction.reshape(width, height) + +# output file +#prediction = cv2.resize(prediction, (img.shape[1], img.shape[0]), interpolation=cv2.INTER_CUBIC) +print(" Write image to: output.png") +depth_min = prediction.min() +depth_max = prediction.max() +img_out = (255 * (prediction - depth_min) / (depth_max - depth_min)).astype("uint8") +print("save output image...") +cv2.imwrite("output.png", img_out) + +print("finished") \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/settings.gradle b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/settings.gradle new file mode 100644 index 0000000000000000000000000000000000000000..e86d89d2483f92b7e778589011fad60fbba3a318 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/android/settings.gradle @@ -0,0 +1,2 @@ +rootProject.name = 'TFLite Image Classification Demo App' +include ':app', ':lib_support', ':lib_task_api', ':models' \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/LICENSE b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas.xcodeproj/project.pbxproj b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas.xcodeproj/project.pbxproj new file mode 100644 index 0000000000000000000000000000000000000000..4917371aa33a65fdfc66c02d914f05489c446430 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas.xcodeproj/project.pbxproj @@ -0,0 +1,538 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 50; + objects = { + +/* Begin PBXBuildFile section */ + 0CDA8C85042ADF65D0787629 /* Pods_Midas.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = A1CE41C09920CCEC31985547 /* Pods_Midas.framework */; }; + 8402440123D9834600704ABD /* README.md in Resources */ = {isa = PBXBuildFile; fileRef = 8402440023D9834600704ABD /* README.md */; }; + 840ECB20238BAA2300C7D88A /* InfoCell.swift in Sources */ = {isa = PBXBuildFile; fileRef = 840ECB1F238BAA2300C7D88A /* InfoCell.swift */; }; + 840EDCFD2341DDD30017ED42 /* Launch Screen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 840EDCFB2341DDD30017ED42 /* Launch Screen.storyboard */; }; + 840EDD022341DE380017ED42 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 840EDD002341DE380017ED42 /* Main.storyboard */; }; + 842DDB6E2372A82000F6BB94 /* OverlayView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 842DDB6D2372A82000F6BB94 /* OverlayView.swift */; }; + 846499C2235DAB0D009CBBC7 /* ModelDataHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 846499C1235DAB0D009CBBC7 /* ModelDataHandler.swift */; }; + 846BAF7623E7FE13006FC136 /* Constants.swift in Sources */ = {isa = PBXBuildFile; fileRef = 846BAF7523E7FE13006FC136 /* Constants.swift */; }; + 8474FEC92341D36E00377D34 /* PreviewView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8474FEC82341D36E00377D34 /* PreviewView.swift */; }; + 8474FECB2341D39800377D34 /* CameraFeedManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8474FECA2341D39800377D34 /* CameraFeedManager.swift */; }; + 84952CB5236186BE0052C104 /* CVPixelBufferExtension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84952CB4236186BE0052C104 /* CVPixelBufferExtension.swift */; }; + 84952CB92361874A0052C104 /* TFLiteExtension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84952CB82361874A0052C104 /* TFLiteExtension.swift */; }; + 84B67CEF2326338300A11A08 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84B67CEE2326338300A11A08 /* AppDelegate.swift */; }; + 84B67CF12326338300A11A08 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84B67CF02326338300A11A08 /* ViewController.swift */; }; + 84B67CF62326338400A11A08 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 84B67CF52326338400A11A08 /* Assets.xcassets */; }; + 84D6576D2387BB7E0048171E /* CGSizeExtension.swift in Sources */ = {isa = PBXBuildFile; fileRef = 84D6576C2387BB7E0048171E /* CGSizeExtension.swift */; }; + 84F232D5254C831E0011862E /* model_opt.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 84F232D4254C831E0011862E /* model_opt.tflite */; }; + 84FCF5922387BD7900663812 /* tfl_logo.png in Resources */ = {isa = PBXBuildFile; fileRef = 84FCF5912387BD7900663812 /* tfl_logo.png */; }; +/* End PBXBuildFile section */ + +/* Begin PBXFileReference section */ + 8402440023D9834600704ABD /* README.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = ""; }; + 840ECB1F238BAA2300C7D88A /* InfoCell.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InfoCell.swift; sourceTree = ""; }; + 840EDCFC2341DDD30017ED42 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = "Base.lproj/Launch Screen.storyboard"; sourceTree = ""; }; + 840EDD012341DE380017ED42 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = ""; }; + 842DDB6D2372A82000F6BB94 /* OverlayView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = OverlayView.swift; sourceTree = ""; }; + 846499C1235DAB0D009CBBC7 /* ModelDataHandler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ModelDataHandler.swift; sourceTree = ""; }; + 846BAF7523E7FE13006FC136 /* Constants.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Constants.swift; sourceTree = ""; }; + 8474FEC82341D36E00377D34 /* PreviewView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PreviewView.swift; sourceTree = ""; }; + 8474FECA2341D39800377D34 /* CameraFeedManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CameraFeedManager.swift; sourceTree = ""; }; + 84884291236FF0A30043FC4C /* download_models.sh */ = {isa = PBXFileReference; lastKnownFileType = text.script.sh; path = download_models.sh; sourceTree = ""; }; + 84952CB4236186BE0052C104 /* CVPixelBufferExtension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CVPixelBufferExtension.swift; sourceTree = ""; }; + 84952CB82361874A0052C104 /* TFLiteExtension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TFLiteExtension.swift; sourceTree = ""; }; + 84B67CEB2326338300A11A08 /* Midas.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Midas.app; sourceTree = BUILT_PRODUCTS_DIR; }; + 84B67CEE2326338300A11A08 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; + 84B67CF02326338300A11A08 /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = ""; }; + 84B67CF52326338400A11A08 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; + 84B67CFA2326338400A11A08 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + 84D6576C2387BB7E0048171E /* CGSizeExtension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CGSizeExtension.swift; sourceTree = ""; }; + 84F232D4254C831E0011862E /* model_opt.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = model_opt.tflite; sourceTree = ""; }; + 84FCF5912387BD7900663812 /* tfl_logo.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; name = tfl_logo.png; path = Assets.xcassets/tfl_logo.png; sourceTree = ""; }; + A1CE41C09920CCEC31985547 /* Pods_Midas.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Midas.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + D2BFF06D0AE9137D332447F3 /* Pods-Midas.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Midas.release.xcconfig"; path = "Target Support Files/Pods-Midas/Pods-Midas.release.xcconfig"; sourceTree = ""; }; + FCA88463911267B1001A596F /* Pods-Midas.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Midas.debug.xcconfig"; path = "Target Support Files/Pods-Midas/Pods-Midas.debug.xcconfig"; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + 84B67CE82326338300A11A08 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 0CDA8C85042ADF65D0787629 /* Pods_Midas.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 840ECB1E238BAA0D00C7D88A /* Cells */ = { + isa = PBXGroup; + children = ( + 840ECB1F238BAA2300C7D88A /* InfoCell.swift */, + ); + path = Cells; + sourceTree = ""; + }; + 842DDB6C2372A80E00F6BB94 /* Views */ = { + isa = PBXGroup; + children = ( + 842DDB6D2372A82000F6BB94 /* OverlayView.swift */, + ); + path = Views; + sourceTree = ""; + }; + 846499C0235DAAE7009CBBC7 /* ModelDataHandler */ = { + isa = PBXGroup; + children = ( + 846499C1235DAB0D009CBBC7 /* ModelDataHandler.swift */, + ); + path = ModelDataHandler; + sourceTree = ""; + }; + 8474FEC62341D2BE00377D34 /* ViewControllers */ = { + isa = PBXGroup; + children = ( + 84B67CF02326338300A11A08 /* ViewController.swift */, + ); + path = ViewControllers; + sourceTree = ""; + }; + 8474FEC72341D35800377D34 /* Camera Feed */ = { + isa = PBXGroup; + children = ( + 8474FEC82341D36E00377D34 /* PreviewView.swift */, + 8474FECA2341D39800377D34 /* CameraFeedManager.swift */, + ); + path = "Camera Feed"; + sourceTree = ""; + }; + 84884290236FF07F0043FC4C /* RunScripts */ = { + isa = PBXGroup; + children = ( + 84884291236FF0A30043FC4C /* download_models.sh */, + ); + path = RunScripts; + sourceTree = ""; + }; + 848842A22370180C0043FC4C /* Model */ = { + isa = PBXGroup; + children = ( + 84F232D4254C831E0011862E /* model_opt.tflite */, + ); + path = Model; + sourceTree = ""; + }; + 84952CB3236186A20052C104 /* Extensions */ = { + isa = PBXGroup; + children = ( + 84952CB4236186BE0052C104 /* CVPixelBufferExtension.swift */, + 84952CB82361874A0052C104 /* TFLiteExtension.swift */, + 84D6576C2387BB7E0048171E /* CGSizeExtension.swift */, + ); + path = Extensions; + sourceTree = ""; + }; + 84B67CE22326338300A11A08 = { + isa = PBXGroup; + children = ( + 8402440023D9834600704ABD /* README.md */, + 84884290236FF07F0043FC4C /* RunScripts */, + 84B67CED2326338300A11A08 /* Midas */, + 84B67CEC2326338300A11A08 /* Products */, + B4DFDCC28443B641BC36251D /* Pods */, + A3DA804B8D3F6891E3A02852 /* Frameworks */, + ); + sourceTree = ""; + }; + 84B67CEC2326338300A11A08 /* Products */ = { + isa = PBXGroup; + children = ( + 84B67CEB2326338300A11A08 /* Midas.app */, + ); + name = Products; + sourceTree = ""; + }; + 84B67CED2326338300A11A08 /* Midas */ = { + isa = PBXGroup; + children = ( + 840ECB1E238BAA0D00C7D88A /* Cells */, + 842DDB6C2372A80E00F6BB94 /* Views */, + 848842A22370180C0043FC4C /* Model */, + 84952CB3236186A20052C104 /* Extensions */, + 846499C0235DAAE7009CBBC7 /* ModelDataHandler */, + 8474FEC72341D35800377D34 /* Camera Feed */, + 8474FEC62341D2BE00377D34 /* ViewControllers */, + 84B67D002326339000A11A08 /* Storyboards */, + 84B67CEE2326338300A11A08 /* AppDelegate.swift */, + 846BAF7523E7FE13006FC136 /* Constants.swift */, + 84B67CF52326338400A11A08 /* Assets.xcassets */, + 84FCF5912387BD7900663812 /* tfl_logo.png */, + 84B67CFA2326338400A11A08 /* Info.plist */, + ); + path = Midas; + sourceTree = ""; + }; + 84B67D002326339000A11A08 /* Storyboards */ = { + isa = PBXGroup; + children = ( + 840EDCFB2341DDD30017ED42 /* Launch Screen.storyboard */, + 840EDD002341DE380017ED42 /* Main.storyboard */, + ); + path = Storyboards; + sourceTree = ""; + }; + A3DA804B8D3F6891E3A02852 /* Frameworks */ = { + isa = PBXGroup; + children = ( + A1CE41C09920CCEC31985547 /* Pods_Midas.framework */, + ); + name = Frameworks; + sourceTree = ""; + }; + B4DFDCC28443B641BC36251D /* Pods */ = { + isa = PBXGroup; + children = ( + FCA88463911267B1001A596F /* Pods-Midas.debug.xcconfig */, + D2BFF06D0AE9137D332447F3 /* Pods-Midas.release.xcconfig */, + ); + path = Pods; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 84B67CEA2326338300A11A08 /* Midas */ = { + isa = PBXNativeTarget; + buildConfigurationList = 84B67CFD2326338400A11A08 /* Build configuration list for PBXNativeTarget "Midas" */; + buildPhases = ( + 14067F3CF309C9DB723C9F6F /* [CP] Check Pods Manifest.lock */, + 84884298237010B90043FC4C /* Download TensorFlow Lite model */, + 84B67CE72326338300A11A08 /* Sources */, + 84B67CE82326338300A11A08 /* Frameworks */, + 84B67CE92326338300A11A08 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + name = Midas; + productName = Midas; + productReference = 84B67CEB2326338300A11A08 /* Midas.app */; + productType = "com.apple.product-type.application"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 84B67CE32326338300A11A08 /* Project object */ = { + isa = PBXProject; + attributes = { + LastSwiftUpdateCheck = 1030; + LastUpgradeCheck = 1030; + ORGANIZATIONNAME = tensorflow; + TargetAttributes = { + 84B67CEA2326338300A11A08 = { + CreatedOnToolsVersion = 10.3; + }; + }; + }; + buildConfigurationList = 84B67CE62326338300A11A08 /* Build configuration list for PBXProject "Midas" */; + compatibilityVersion = "Xcode 9.3"; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = 84B67CE22326338300A11A08; + productRefGroup = 84B67CEC2326338300A11A08 /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + 84B67CEA2326338300A11A08 /* Midas */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + 84B67CE92326338300A11A08 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 8402440123D9834600704ABD /* README.md in Resources */, + 84F232D5254C831E0011862E /* model_opt.tflite in Resources */, + 840EDD022341DE380017ED42 /* Main.storyboard in Resources */, + 840EDCFD2341DDD30017ED42 /* Launch Screen.storyboard in Resources */, + 84FCF5922387BD7900663812 /* tfl_logo.png in Resources */, + 84B67CF62326338400A11A08 /* Assets.xcassets in Resources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXShellScriptBuildPhase section */ + 14067F3CF309C9DB723C9F6F /* [CP] Check Pods Manifest.lock */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + "${PODS_PODFILE_DIR_PATH}/Podfile.lock", + "${PODS_ROOT}/Manifest.lock", + ); + name = "[CP] Check Pods Manifest.lock"; + outputFileListPaths = ( + ); + outputPaths = ( + "$(DERIVED_FILE_DIR)/Pods-Midas-checkManifestLockResult.txt", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; + showEnvVarsInLog = 0; + }; + 84884298237010B90043FC4C /* Download TensorFlow Lite model */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + ); + name = "Download TensorFlow Lite model"; + outputFileListPaths = ( + ); + outputPaths = ( + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/bash; + shellScript = "\"$SRCROOT/RunScripts/download_models.sh\"\n"; + }; +/* End PBXShellScriptBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + 84B67CE72326338300A11A08 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 842DDB6E2372A82000F6BB94 /* OverlayView.swift in Sources */, + 846BAF7623E7FE13006FC136 /* Constants.swift in Sources */, + 84952CB92361874A0052C104 /* TFLiteExtension.swift in Sources */, + 84D6576D2387BB7E0048171E /* CGSizeExtension.swift in Sources */, + 84B67CF12326338300A11A08 /* ViewController.swift in Sources */, + 84B67CEF2326338300A11A08 /* AppDelegate.swift in Sources */, + 8474FECB2341D39800377D34 /* CameraFeedManager.swift in Sources */, + 846499C2235DAB0D009CBBC7 /* ModelDataHandler.swift in Sources */, + 8474FEC92341D36E00377D34 /* PreviewView.swift in Sources */, + 84952CB5236186BE0052C104 /* CVPixelBufferExtension.swift in Sources */, + 840ECB20238BAA2300C7D88A /* InfoCell.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXVariantGroup section */ + 840EDCFB2341DDD30017ED42 /* Launch Screen.storyboard */ = { + isa = PBXVariantGroup; + children = ( + 840EDCFC2341DDD30017ED42 /* Base */, + ); + name = "Launch Screen.storyboard"; + sourceTree = ""; + }; + 840EDD002341DE380017ED42 /* Main.storyboard */ = { + isa = PBXVariantGroup; + children = ( + 840EDD012341DE380017ED42 /* Base */, + ); + name = Main.storyboard; + sourceTree = ""; + }; +/* End PBXVariantGroup section */ + +/* Begin XCBuildConfiguration section */ + 84B67CFB2326338400A11A08 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + CODE_SIGN_IDENTITY = "iPhone Developer"; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_STRICT_OBJC_MSGSEND = YES; + ENABLE_TESTABILITY = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_DYNAMIC_NO_PIC = NO; + GCC_NO_COMMON_BLOCKS = YES; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_PREPROCESSOR_DEFINITIONS = ( + "DEBUG=1", + "$(inherited)", + ); + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 12.4; + MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; + MTL_FAST_MATH = YES; + ONLY_ACTIVE_ARCH = YES; + SDKROOT = iphoneos; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = DEBUG; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + }; + name = Debug; + }; + 84B67CFC2326338400A11A08 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + CLANG_ANALYZER_NONNULL = YES; + CLANG_ANALYZER_NUMBER_OBJECT_CONVERSION = YES_AGGRESSIVE; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++14"; + CLANG_CXX_LIBRARY = "libc++"; + CLANG_ENABLE_MODULES = YES; + CLANG_ENABLE_OBJC_ARC = YES; + CLANG_ENABLE_OBJC_WEAK = YES; + CLANG_WARN_BLOCK_CAPTURE_AUTORELEASING = YES; + CLANG_WARN_BOOL_CONVERSION = YES; + CLANG_WARN_COMMA = YES; + CLANG_WARN_CONSTANT_CONVERSION = YES; + CLANG_WARN_DEPRECATED_OBJC_IMPLEMENTATIONS = YES; + CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; + CLANG_WARN_DOCUMENTATION_COMMENTS = YES; + CLANG_WARN_EMPTY_BODY = YES; + CLANG_WARN_ENUM_CONVERSION = YES; + CLANG_WARN_INFINITE_RECURSION = YES; + CLANG_WARN_INT_CONVERSION = YES; + CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES; + CLANG_WARN_OBJC_IMPLICIT_RETAIN_SELF = YES; + CLANG_WARN_OBJC_LITERAL_CONVERSION = YES; + CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; + CLANG_WARN_RANGE_LOOP_ANALYSIS = YES; + CLANG_WARN_STRICT_PROTOTYPES = YES; + CLANG_WARN_SUSPICIOUS_MOVE = YES; + CLANG_WARN_UNGUARDED_AVAILABILITY = YES_AGGRESSIVE; + CLANG_WARN_UNREACHABLE_CODE = YES; + CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; + CODE_SIGN_IDENTITY = "iPhone Developer"; + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + ENABLE_NS_ASSERTIONS = NO; + ENABLE_STRICT_OBJC_MSGSEND = YES; + GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_NO_COMMON_BLOCKS = YES; + GCC_WARN_64_TO_32_BIT_CONVERSION = YES; + GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; + GCC_WARN_UNDECLARED_SELECTOR = YES; + GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; + GCC_WARN_UNUSED_FUNCTION = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + IPHONEOS_DEPLOYMENT_TARGET = 12.4; + MTL_ENABLE_DEBUG_INFO = NO; + MTL_FAST_MATH = YES; + SDKROOT = iphoneos; + SWIFT_COMPILATION_MODE = wholemodule; + SWIFT_OPTIMIZATION_LEVEL = "-O"; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; + 84B67CFE2326338400A11A08 /* Debug */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = FCA88463911267B1001A596F /* Pods-Midas.debug.xcconfig */; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CODE_SIGN_IDENTITY = "iPhone Developer"; + CODE_SIGN_STYLE = Automatic; + DEVELOPMENT_TEAM = BV6M48J3RX; + INFOPLIST_FILE = Midas/Info.plist; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = "org.midas.midas-tflite-npu"; + PRODUCT_NAME = Midas; + PROVISIONING_PROFILE_SPECIFIER = ""; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + 84B67CFF2326338400A11A08 /* Release */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = D2BFF06D0AE9137D332447F3 /* Pods-Midas.release.xcconfig */; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + CODE_SIGN_IDENTITY = "iPhone Developer"; + CODE_SIGN_STYLE = Automatic; + DEVELOPMENT_TEAM = BV6M48J3RX; + INFOPLIST_FILE = Midas/Info.plist; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = "org.midas.midas-tflite-npu"; + PRODUCT_NAME = Midas; + PROVISIONING_PROFILE_SPECIFIER = ""; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 84B67CE62326338300A11A08 /* Build configuration list for PBXProject "Midas" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 84B67CFB2326338400A11A08 /* Debug */, + 84B67CFC2326338400A11A08 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 84B67CFD2326338400A11A08 /* Build configuration list for PBXNativeTarget "Midas" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 84B67CFE2326338400A11A08 /* Debug */, + 84B67CFF2326338400A11A08 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = 84B67CE32326338300A11A08 /* Project object */; +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000000000000000000000000000000000000..919434a6254f0e9651f402737811be6634a03e9c --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist new file mode 100644 index 0000000000000000000000000000000000000000..18d981003d68d0546c4804ac2ff47dd97c6e7921 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist @@ -0,0 +1,8 @@ + + + + + IDEDidComputeMac32BitWarning + + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas.xcodeproj/project.xcworkspace/xcuserdata/admin.xcuserdatad/UserInterfaceState.xcuserstate b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas.xcodeproj/project.xcworkspace/xcuserdata/admin.xcuserdatad/UserInterfaceState.xcuserstate new file mode 100644 index 0000000000000000000000000000000000000000..1d20756ee57b79e9f9f886453bdb7997ca2ee2d4 Binary files /dev/null and b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas.xcodeproj/project.xcworkspace/xcuserdata/admin.xcuserdatad/UserInterfaceState.xcuserstate differ diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas.xcodeproj/xcuserdata/admin.xcuserdatad/xcschemes/xcschememanagement.plist b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas.xcodeproj/xcuserdata/admin.xcuserdatad/xcschemes/xcschememanagement.plist new file mode 100644 index 0000000000000000000000000000000000000000..6093f6160eedfdfc20e96396247a7dbc9247cc55 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas.xcodeproj/xcuserdata/admin.xcuserdatad/xcschemes/xcschememanagement.plist @@ -0,0 +1,14 @@ + + + + + SchemeUserState + + PoseNet.xcscheme_^#shared#^_ + + orderHint + 3 + + + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/AppDelegate.swift b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/AppDelegate.swift new file mode 100644 index 0000000000000000000000000000000000000000..233f0291ab4f379067543bdad3cc198a2dc3ab0f --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/AppDelegate.swift @@ -0,0 +1,41 @@ +// Copyright 2019 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import UIKit + +@UIApplicationMain +class AppDelegate: UIResponder, UIApplicationDelegate { + + var window: UIWindow? + + func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool { + return true + } + + func applicationWillResignActive(_ application: UIApplication) { + } + + func applicationDidEnterBackground(_ application: UIApplication) { + } + + func applicationWillEnterForeground(_ application: UIApplication) { + } + + func applicationDidBecomeActive(_ application: UIApplication) { + } + + func applicationWillTerminate(_ application: UIApplication) { + } +} + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Assets.xcassets/AppIcon.appiconset/Contents.json b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 0000000000000000000000000000000000000000..65b74d7ef11fa59fafa829e681ac90906f3ac8b2 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1 @@ +{"images":[{"size":"60x60","expected-size":"180","filename":"180.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"iphone","scale":"3x"},{"size":"40x40","expected-size":"80","filename":"80.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"iphone","scale":"2x"},{"size":"40x40","expected-size":"120","filename":"120.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"iphone","scale":"3x"},{"size":"60x60","expected-size":"120","filename":"120.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"iphone","scale":"2x"},{"size":"57x57","expected-size":"57","filename":"57.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"iphone","scale":"1x"},{"size":"29x29","expected-size":"58","filename":"58.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"iphone","scale":"2x"},{"size":"29x29","expected-size":"29","filename":"29.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"iphone","scale":"1x"},{"size":"29x29","expected-size":"87","filename":"87.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"iphone","scale":"3x"},{"size":"57x57","expected-size":"114","filename":"114.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"iphone","scale":"2x"},{"size":"20x20","expected-size":"40","filename":"40.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"iphone","scale":"2x"},{"size":"20x20","expected-size":"60","filename":"60.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"iphone","scale":"3x"},{"size":"1024x1024","filename":"1024.png","expected-size":"1024","idiom":"ios-marketing","folder":"Assets.xcassets/AppIcon.appiconset/","scale":"1x"},{"size":"40x40","expected-size":"80","filename":"80.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"ipad","scale":"2x"},{"size":"72x72","expected-size":"72","filename":"72.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"ipad","scale":"1x"},{"size":"76x76","expected-size":"152","filename":"152.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"ipad","scale":"2x"},{"size":"50x50","expected-size":"100","filename":"100.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"ipad","scale":"2x"},{"size":"29x29","expected-size":"58","filename":"58.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"ipad","scale":"2x"},{"size":"76x76","expected-size":"76","filename":"76.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"ipad","scale":"1x"},{"size":"29x29","expected-size":"29","filename":"29.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"ipad","scale":"1x"},{"size":"50x50","expected-size":"50","filename":"50.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"ipad","scale":"1x"},{"size":"72x72","expected-size":"144","filename":"144.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"ipad","scale":"2x"},{"size":"40x40","expected-size":"40","filename":"40.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"ipad","scale":"1x"},{"size":"83.5x83.5","expected-size":"167","filename":"167.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"ipad","scale":"2x"},{"size":"20x20","expected-size":"20","filename":"20.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"ipad","scale":"1x"},{"size":"20x20","expected-size":"40","filename":"40.png","folder":"Assets.xcassets/AppIcon.appiconset/","idiom":"ipad","scale":"2x"}]} \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Assets.xcassets/Contents.json b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Assets.xcassets/Contents.json new file mode 100644 index 0000000000000000000000000000000000000000..da4a164c918651cdd1e11dca5cc62c333f097601 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Assets.xcassets/Contents.json @@ -0,0 +1,6 @@ +{ + "info" : { + "version" : 1, + "author" : "xcode" + } +} \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Camera Feed/CameraFeedManager.swift b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Camera Feed/CameraFeedManager.swift new file mode 100644 index 0000000000000000000000000000000000000000..48d65b88ee220e722fbad2570c8e879a431cd0f5 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Camera Feed/CameraFeedManager.swift @@ -0,0 +1,316 @@ +// Copyright 2019 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import AVFoundation +import UIKit +import os + +// MARK: - CameraFeedManagerDelegate Declaration +@objc protocol CameraFeedManagerDelegate: class { + /// This method delivers the pixel buffer of the current frame seen by the device's camera. + @objc optional func cameraFeedManager( + _ manager: CameraFeedManager, didOutput pixelBuffer: CVPixelBuffer + ) + + /// This method initimates that a session runtime error occured. + func cameraFeedManagerDidEncounterSessionRunTimeError(_ manager: CameraFeedManager) + + /// This method initimates that the session was interrupted. + func cameraFeedManager( + _ manager: CameraFeedManager, sessionWasInterrupted canResumeManually: Bool + ) + + /// This method initimates that the session interruption has ended. + func cameraFeedManagerDidEndSessionInterruption(_ manager: CameraFeedManager) + + /// This method initimates that there was an error in video configurtion. + func presentVideoConfigurationErrorAlert(_ manager: CameraFeedManager) + + /// This method initimates that the camera permissions have been denied. + func presentCameraPermissionsDeniedAlert(_ manager: CameraFeedManager) +} + +/// This enum holds the state of the camera initialization. +// MARK: - Camera Initialization State Enum +enum CameraConfiguration { + case success + case failed + case permissionDenied +} + +/// This class manages all camera related functionalities. +// MARK: - Camera Related Functionalies Manager +class CameraFeedManager: NSObject { + // MARK: Camera Related Instance Variables + private let session: AVCaptureSession = AVCaptureSession() + + private let previewView: PreviewView + private let sessionQueue = DispatchQueue(label: "sessionQueue") + private var cameraConfiguration: CameraConfiguration = .failed + private lazy var videoDataOutput = AVCaptureVideoDataOutput() + private var isSessionRunning = false + + // MARK: CameraFeedManagerDelegate + weak var delegate: CameraFeedManagerDelegate? + + // MARK: Initializer + init(previewView: PreviewView) { + self.previewView = previewView + super.init() + + // Initializes the session + session.sessionPreset = .high + self.previewView.session = session + self.previewView.previewLayer.connection?.videoOrientation = .portrait + self.previewView.previewLayer.videoGravity = .resizeAspectFill + self.attemptToConfigureSession() + } + + // MARK: Session Start and End methods + + /// This method starts an AVCaptureSession based on whether the camera configuration was successful. + func checkCameraConfigurationAndStartSession() { + sessionQueue.async { + switch self.cameraConfiguration { + case .success: + self.addObservers() + self.startSession() + case .failed: + DispatchQueue.main.async { + self.delegate?.presentVideoConfigurationErrorAlert(self) + } + case .permissionDenied: + DispatchQueue.main.async { + self.delegate?.presentCameraPermissionsDeniedAlert(self) + } + } + } + } + + /// This method stops a running an AVCaptureSession. + func stopSession() { + self.removeObservers() + sessionQueue.async { + if self.session.isRunning { + self.session.stopRunning() + self.isSessionRunning = self.session.isRunning + } + } + + } + + /// This method resumes an interrupted AVCaptureSession. + func resumeInterruptedSession(withCompletion completion: @escaping (Bool) -> Void) { + sessionQueue.async { + self.startSession() + + DispatchQueue.main.async { + completion(self.isSessionRunning) + } + } + } + + /// This method starts the AVCaptureSession + private func startSession() { + self.session.startRunning() + self.isSessionRunning = self.session.isRunning + } + + // MARK: Session Configuration Methods. + /// This method requests for camera permissions and handles the configuration of the session and stores the result of configuration. + private func attemptToConfigureSession() { + switch AVCaptureDevice.authorizationStatus(for: .video) { + case .authorized: + self.cameraConfiguration = .success + case .notDetermined: + self.sessionQueue.suspend() + self.requestCameraAccess(completion: { granted in + self.sessionQueue.resume() + }) + case .denied: + self.cameraConfiguration = .permissionDenied + default: + break + } + + self.sessionQueue.async { + self.configureSession() + } + } + + /// This method requests for camera permissions. + private func requestCameraAccess(completion: @escaping (Bool) -> Void) { + AVCaptureDevice.requestAccess(for: .video) { (granted) in + if !granted { + self.cameraConfiguration = .permissionDenied + } else { + self.cameraConfiguration = .success + } + completion(granted) + } + } + + /// This method handles all the steps to configure an AVCaptureSession. + private func configureSession() { + guard cameraConfiguration == .success else { + return + } + session.beginConfiguration() + + // Tries to add an AVCaptureDeviceInput. + guard addVideoDeviceInput() == true else { + self.session.commitConfiguration() + self.cameraConfiguration = .failed + return + } + + // Tries to add an AVCaptureVideoDataOutput. + guard addVideoDataOutput() else { + self.session.commitConfiguration() + self.cameraConfiguration = .failed + return + } + + session.commitConfiguration() + self.cameraConfiguration = .success + } + + /// This method tries to an AVCaptureDeviceInput to the current AVCaptureSession. + private func addVideoDeviceInput() -> Bool { + /// Tries to get the default back camera. + guard + let camera = AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: .back) + else { + fatalError("Cannot find camera") + } + + do { + let videoDeviceInput = try AVCaptureDeviceInput(device: camera) + if session.canAddInput(videoDeviceInput) { + session.addInput(videoDeviceInput) + return true + } else { + return false + } + } catch { + fatalError("Cannot create video device input") + } + } + + /// This method tries to an AVCaptureVideoDataOutput to the current AVCaptureSession. + private func addVideoDataOutput() -> Bool { + let sampleBufferQueue = DispatchQueue(label: "sampleBufferQueue") + videoDataOutput.setSampleBufferDelegate(self, queue: sampleBufferQueue) + videoDataOutput.alwaysDiscardsLateVideoFrames = true + videoDataOutput.videoSettings = [ + String(kCVPixelBufferPixelFormatTypeKey): kCMPixelFormat_32BGRA + ] + + if session.canAddOutput(videoDataOutput) { + session.addOutput(videoDataOutput) + videoDataOutput.connection(with: .video)?.videoOrientation = .portrait + return true + } + return false + } + + // MARK: Notification Observer Handling + private func addObservers() { + NotificationCenter.default.addObserver( + self, selector: #selector(CameraFeedManager.sessionRuntimeErrorOccured(notification:)), + name: NSNotification.Name.AVCaptureSessionRuntimeError, object: session) + NotificationCenter.default.addObserver( + self, selector: #selector(CameraFeedManager.sessionWasInterrupted(notification:)), + name: NSNotification.Name.AVCaptureSessionWasInterrupted, object: session) + NotificationCenter.default.addObserver( + self, selector: #selector(CameraFeedManager.sessionInterruptionEnded), + name: NSNotification.Name.AVCaptureSessionInterruptionEnded, object: session) + } + + private func removeObservers() { + NotificationCenter.default.removeObserver( + self, name: NSNotification.Name.AVCaptureSessionRuntimeError, object: session) + NotificationCenter.default.removeObserver( + self, name: NSNotification.Name.AVCaptureSessionWasInterrupted, object: session) + NotificationCenter.default.removeObserver( + self, name: NSNotification.Name.AVCaptureSessionInterruptionEnded, object: session) + } + + // MARK: Notification Observers + @objc func sessionWasInterrupted(notification: Notification) { + if let userInfoValue = notification.userInfo?[AVCaptureSessionInterruptionReasonKey] + as AnyObject?, + let reasonIntegerValue = userInfoValue.integerValue, + let reason = AVCaptureSession.InterruptionReason(rawValue: reasonIntegerValue) + { + os_log("Capture session was interrupted with reason: %s", type: .error, reason.rawValue) + + var canResumeManually = false + if reason == .videoDeviceInUseByAnotherClient { + canResumeManually = true + } else if reason == .videoDeviceNotAvailableWithMultipleForegroundApps { + canResumeManually = false + } + + delegate?.cameraFeedManager(self, sessionWasInterrupted: canResumeManually) + + } + } + + @objc func sessionInterruptionEnded(notification: Notification) { + delegate?.cameraFeedManagerDidEndSessionInterruption(self) + } + + @objc func sessionRuntimeErrorOccured(notification: Notification) { + guard let error = notification.userInfo?[AVCaptureSessionErrorKey] as? AVError else { + return + } + + os_log("Capture session runtime error: %s", type: .error, error.localizedDescription) + + if error.code == .mediaServicesWereReset { + sessionQueue.async { + if self.isSessionRunning { + self.startSession() + } else { + DispatchQueue.main.async { + self.delegate?.cameraFeedManagerDidEncounterSessionRunTimeError(self) + } + } + } + } else { + delegate?.cameraFeedManagerDidEncounterSessionRunTimeError(self) + } + } +} + +/// AVCaptureVideoDataOutputSampleBufferDelegate +extension CameraFeedManager: AVCaptureVideoDataOutputSampleBufferDelegate { + /// This method delegates the CVPixelBuffer of the frame seen by the camera currently. + func captureOutput( + _ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, + from connection: AVCaptureConnection + ) { + + // Converts the CMSampleBuffer to a CVPixelBuffer. + let pixelBuffer: CVPixelBuffer? = CMSampleBufferGetImageBuffer(sampleBuffer) + + guard let imagePixelBuffer = pixelBuffer else { + return + } + + // Delegates the pixel buffer to the ViewController. + delegate?.cameraFeedManager?(self, didOutput: imagePixelBuffer) + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Camera Feed/PreviewView.swift b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Camera Feed/PreviewView.swift new file mode 100644 index 0000000000000000000000000000000000000000..308c7ec54308af5c152ff6038670b26501a8e82c --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Camera Feed/PreviewView.swift @@ -0,0 +1,39 @@ +// Copyright 2019 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import UIKit +import AVFoundation + + /// The camera frame is displayed on this view. +class PreviewView: UIView { + var previewLayer: AVCaptureVideoPreviewLayer { + guard let layer = layer as? AVCaptureVideoPreviewLayer else { + fatalError("Layer expected is of type VideoPreviewLayer") + } + return layer + } + + var session: AVCaptureSession? { + get { + return previewLayer.session + } + set { + previewLayer.session = newValue + } + } + + override class var layerClass: AnyClass { + return AVCaptureVideoPreviewLayer.self + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Cells/InfoCell.swift b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Cells/InfoCell.swift new file mode 100644 index 0000000000000000000000000000000000000000..c6be64af5678541ec09fc367b03c80155876f0ba --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Cells/InfoCell.swift @@ -0,0 +1,21 @@ +// Copyright 2019 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import UIKit + +/// Table cell for inference result in bottom view. +class InfoCell: UITableViewCell { + @IBOutlet weak var fieldNameLabel: UILabel! + @IBOutlet weak var infoLabel: UILabel! +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Constants.swift b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Constants.swift new file mode 100644 index 0000000000000000000000000000000000000000..b0789ee58a1ea373d441f05333d8ce8914adadb7 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Constants.swift @@ -0,0 +1,25 @@ +// Copyright 2020 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================= + +enum Constants { + // MARK: - Constants related to the image processing + static let bgraPixel = (channels: 4, alphaComponent: 3, lastBgrComponent: 2) + static let rgbPixelChannels = 3 + static let maxRGBValue: Float32 = 255.0 + + // MARK: - Constants related to the model interperter + static let defaultThreadCount = 2 + static let defaultDelegate: Delegates = .CPU +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Extensions/CGSizeExtension.swift b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Extensions/CGSizeExtension.swift new file mode 100644 index 0000000000000000000000000000000000000000..031550ea0081963d18b5b83712854babaf7c0a34 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Extensions/CGSizeExtension.swift @@ -0,0 +1,45 @@ +// Copyright 2019 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================= + +import Accelerate +import Foundation + +extension CGSize { + /// Returns `CGAfineTransform` to resize `self` to fit in destination size, keeping aspect ratio + /// of `self`. `self` image is resized to be inscribe to destination size and located in center of + /// destination. + /// + /// - Parameter toFitIn: destination size to be filled. + /// - Returns: `CGAffineTransform` to transform `self` image to `dest` image. + func transformKeepAspect(toFitIn dest: CGSize) -> CGAffineTransform { + let sourceRatio = self.height / self.width + let destRatio = dest.height / dest.width + + // Calculates ratio `self` to `dest`. + var ratio: CGFloat + var x: CGFloat = 0 + var y: CGFloat = 0 + if sourceRatio > destRatio { + // Source size is taller than destination. Resized to fit in destination height, and find + // horizontal starting point to be centered. + ratio = dest.height / self.height + x = (dest.width - self.width * ratio) / 2 + } else { + ratio = dest.width / self.width + y = (dest.height - self.height * ratio) / 2 + } + return CGAffineTransform(a: ratio, b: 0, c: 0, d: ratio, tx: x, ty: y) + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Extensions/CVPixelBufferExtension.swift b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Extensions/CVPixelBufferExtension.swift new file mode 100644 index 0000000000000000000000000000000000000000..4899c76562a546c513736fbf4556629b08d2c929 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Extensions/CVPixelBufferExtension.swift @@ -0,0 +1,172 @@ +// Copyright 2019 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================= + +import Accelerate +import Foundation + +extension CVPixelBuffer { + var size: CGSize { + return CGSize(width: CVPixelBufferGetWidth(self), height: CVPixelBufferGetHeight(self)) + } + + /// Returns a new `CVPixelBuffer` created by taking the self area and resizing it to the + /// specified target size. Aspect ratios of source image and destination image are expected to be + /// same. + /// + /// - Parameters: + /// - from: Source area of image to be cropped and resized. + /// - to: Size to scale the image to(i.e. image size used while training the model). + /// - Returns: The cropped and resized image of itself. + func resize(from source: CGRect, to size: CGSize) -> CVPixelBuffer? { + let rect = CGRect(origin: CGPoint(x: 0, y: 0), size: self.size) + guard rect.contains(source) else { + os_log("Resizing Error: source area is out of index", type: .error) + return nil + } + guard rect.size.width / rect.size.height - source.size.width / source.size.height < 1e-5 + else { + os_log( + "Resizing Error: source image ratio and destination image ratio is different", + type: .error) + return nil + } + + let inputImageRowBytes = CVPixelBufferGetBytesPerRow(self) + let imageChannels = 4 + + CVPixelBufferLockBaseAddress(self, CVPixelBufferLockFlags(rawValue: 0)) + defer { CVPixelBufferUnlockBaseAddress(self, CVPixelBufferLockFlags(rawValue: 0)) } + + // Finds the address of the upper leftmost pixel of the source area. + guard + let inputBaseAddress = CVPixelBufferGetBaseAddress(self)?.advanced( + by: Int(source.minY) * inputImageRowBytes + Int(source.minX) * imageChannels) + else { + return nil + } + + // Crops given area as vImage Buffer. + var croppedImage = vImage_Buffer( + data: inputBaseAddress, height: UInt(source.height), width: UInt(source.width), + rowBytes: inputImageRowBytes) + + let resultRowBytes = Int(size.width) * imageChannels + guard let resultAddress = malloc(Int(size.height) * resultRowBytes) else { + return nil + } + + // Allocates a vacant vImage buffer for resized image. + var resizedImage = vImage_Buffer( + data: resultAddress, + height: UInt(size.height), width: UInt(size.width), + rowBytes: resultRowBytes + ) + + // Performs the scale operation on cropped image and stores it in result image buffer. + guard vImageScale_ARGB8888(&croppedImage, &resizedImage, nil, vImage_Flags(0)) == kvImageNoError + else { + return nil + } + + let releaseCallBack: CVPixelBufferReleaseBytesCallback = { mutablePointer, pointer in + if let pointer = pointer { + free(UnsafeMutableRawPointer(mutating: pointer)) + } + } + + var result: CVPixelBuffer? + + // Converts the thumbnail vImage buffer to CVPixelBuffer + let conversionStatus = CVPixelBufferCreateWithBytes( + nil, + Int(size.width), Int(size.height), + CVPixelBufferGetPixelFormatType(self), + resultAddress, + resultRowBytes, + releaseCallBack, + nil, + nil, + &result + ) + + guard conversionStatus == kCVReturnSuccess else { + free(resultAddress) + return nil + } + + return result + } + + /// Returns the RGB `Data` representation of the given image buffer. + /// + /// - Parameters: + /// - isModelQuantized: Whether the model is quantized (i.e. fixed point values rather than + /// floating point values). + /// - Returns: The RGB data representation of the image buffer or `nil` if the buffer could not be + /// converted. + func rgbData( + isModelQuantized: Bool + ) -> Data? { + CVPixelBufferLockBaseAddress(self, .readOnly) + defer { CVPixelBufferUnlockBaseAddress(self, .readOnly) } + guard let sourceData = CVPixelBufferGetBaseAddress(self) else { + return nil + } + + let width = CVPixelBufferGetWidth(self) + let height = CVPixelBufferGetHeight(self) + let sourceBytesPerRow = CVPixelBufferGetBytesPerRow(self) + let destinationBytesPerRow = Constants.rgbPixelChannels * width + + // Assign input image to `sourceBuffer` to convert it. + var sourceBuffer = vImage_Buffer( + data: sourceData, + height: vImagePixelCount(height), + width: vImagePixelCount(width), + rowBytes: sourceBytesPerRow) + + // Make `destinationBuffer` and `destinationData` for its data to be assigned. + guard let destinationData = malloc(height * destinationBytesPerRow) else { + os_log("Error: out of memory", type: .error) + return nil + } + defer { free(destinationData) } + var destinationBuffer = vImage_Buffer( + data: destinationData, + height: vImagePixelCount(height), + width: vImagePixelCount(width), + rowBytes: destinationBytesPerRow) + + // Convert image type. + switch CVPixelBufferGetPixelFormatType(self) { + case kCVPixelFormatType_32BGRA: + vImageConvert_BGRA8888toRGB888(&sourceBuffer, &destinationBuffer, UInt32(kvImageNoFlags)) + case kCVPixelFormatType_32ARGB: + vImageConvert_BGRA8888toRGB888(&sourceBuffer, &destinationBuffer, UInt32(kvImageNoFlags)) + default: + os_log("The type of this image is not supported.", type: .error) + return nil + } + + // Make `Data` with converted image. + let imageByteData = Data( + bytes: destinationBuffer.data, count: destinationBuffer.rowBytes * height) + + if isModelQuantized { return imageByteData } + + let imageBytes = [UInt8](imageByteData) + return Data(copyingBufferOf: imageBytes.map { Float($0) / Constants.maxRGBValue }) + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Extensions/TFLiteExtension.swift b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Extensions/TFLiteExtension.swift new file mode 100644 index 0000000000000000000000000000000000000000..63f7ced786e2b550391c77af534d1d3c431522c6 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Extensions/TFLiteExtension.swift @@ -0,0 +1,75 @@ +// Copyright 2019 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================= + +import Accelerate +import CoreImage +import Foundation +import TensorFlowLite + +// MARK: - Data +extension Data { + /// Creates a new buffer by copying the buffer pointer of the given array. + /// + /// - Warning: The given array's element type `T` must be trivial in that it can be copied bit + /// for bit with no indirection or reference-counting operations; otherwise, reinterpreting + /// data from the resulting buffer has undefined behavior. + /// - Parameter array: An array with elements of type `T`. + init(copyingBufferOf array: [T]) { + self = array.withUnsafeBufferPointer(Data.init) + } + + /// Convert a Data instance to Array representation. + func toArray(type: T.Type) -> [T] where T: AdditiveArithmetic { + var array = [T](repeating: T.zero, count: self.count / MemoryLayout.stride) + _ = array.withUnsafeMutableBytes { self.copyBytes(to: $0) } + return array + } +} + +// MARK: - Wrappers +/// Struct for handling multidimension `Data` in flat `Array`. +struct FlatArray { + private var array: [Element] + var dimensions: [Int] + + init(tensor: Tensor) { + dimensions = tensor.shape.dimensions + array = tensor.data.toArray(type: Element.self) + } + + private func flatIndex(_ index: [Int]) -> Int { + guard index.count == dimensions.count else { + fatalError("Invalid index: got \(index.count) index(es) for \(dimensions.count) index(es).") + } + + var result = 0 + for i in 0.. index[i] else { + fatalError("Invalid index: \(index[i]) is bigger than \(dimensions[i])") + } + result = dimensions[i] * result + index[i] + } + return result + } + + subscript(_ index: Int...) -> Element { + get { + return array[flatIndex(index)] + } + set(newValue) { + array[flatIndex(index)] = newValue + } + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Info.plist b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Info.plist new file mode 100644 index 0000000000000000000000000000000000000000..4330d9b33f31010549802febc6f6f2bc9fd9b950 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Info.plist @@ -0,0 +1,42 @@ + + + + + CFBundleDevelopmentRegion + $(DEVELOPMENT_LANGUAGE) + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + LSRequiresIPhoneOS + + NSCameraUsageDescription + This app will use camera to continuously estimate the depth map. + UILaunchStoryboardName + LaunchScreen + UIMainStoryboardFile + Main + UIRequiredDeviceCapabilities + + armv7 + + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/ModelDataHandler/ModelDataHandler.swift b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/ModelDataHandler/ModelDataHandler.swift new file mode 100644 index 0000000000000000000000000000000000000000..144cfe1fa3a65af5adcb572237f2bf9718e570ae --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/ModelDataHandler/ModelDataHandler.swift @@ -0,0 +1,464 @@ +// Copyright 2019 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Accelerate +import CoreImage +import Foundation +import TensorFlowLite +import UIKit + +/// This class handles all data preprocessing and makes calls to run inference on a given frame +/// by invoking the `Interpreter`. It then formats the inferences obtained. +class ModelDataHandler { + // MARK: - Private Properties + + /// TensorFlow Lite `Interpreter` object for performing inference on a given model. + private var interpreter: Interpreter + + /// TensorFlow lite `Tensor` of model input and output. + private var inputTensor: Tensor + + //private var heatsTensor: Tensor + //private var offsetsTensor: Tensor + private var outputTensor: Tensor + // MARK: - Initialization + + /// A failable initializer for `ModelDataHandler`. A new instance is created if the model is + /// successfully loaded from the app's main bundle. Default `threadCount` is 2. + init( + threadCount: Int = Constants.defaultThreadCount, + delegate: Delegates = Constants.defaultDelegate + ) throws { + // Construct the path to the model file. + guard + let modelPath = Bundle.main.path( + forResource: Model.file.name, + ofType: Model.file.extension + ) + else { + fatalError("Failed to load the model file with name: \(Model.file.name).") + } + + // Specify the options for the `Interpreter`. + var options = Interpreter.Options() + options.threadCount = threadCount + + // Specify the delegates for the `Interpreter`. + var delegates: [Delegate]? + switch delegate { + case .Metal: + delegates = [MetalDelegate()] + case .CoreML: + if let coreMLDelegate = CoreMLDelegate() { + delegates = [coreMLDelegate] + } else { + delegates = nil + } + default: + delegates = nil + } + + // Create the `Interpreter`. + interpreter = try Interpreter(modelPath: modelPath, options: options, delegates: delegates) + + // Initialize input and output `Tensor`s. + // Allocate memory for the model's input `Tensor`s. + try interpreter.allocateTensors() + + // Get allocated input and output `Tensor`s. + inputTensor = try interpreter.input(at: 0) + outputTensor = try interpreter.output(at: 0) + //heatsTensor = try interpreter.output(at: 0) + //offsetsTensor = try interpreter.output(at: 1) + + /* + // Check if input and output `Tensor`s are in the expected formats. + guard (inputTensor.dataType == .uInt8) == Model.isQuantized else { + fatalError("Unexpected Model: quantization is \(!Model.isQuantized)") + } + + guard inputTensor.shape.dimensions[0] == Model.input.batchSize, + inputTensor.shape.dimensions[1] == Model.input.height, + inputTensor.shape.dimensions[2] == Model.input.width, + inputTensor.shape.dimensions[3] == Model.input.channelSize + else { + fatalError("Unexpected Model: input shape") + } + + + guard heatsTensor.shape.dimensions[0] == Model.output.batchSize, + heatsTensor.shape.dimensions[1] == Model.output.height, + heatsTensor.shape.dimensions[2] == Model.output.width, + heatsTensor.shape.dimensions[3] == Model.output.keypointSize + else { + fatalError("Unexpected Model: heat tensor") + } + + guard offsetsTensor.shape.dimensions[0] == Model.output.batchSize, + offsetsTensor.shape.dimensions[1] == Model.output.height, + offsetsTensor.shape.dimensions[2] == Model.output.width, + offsetsTensor.shape.dimensions[3] == Model.output.offsetSize + else { + fatalError("Unexpected Model: offset tensor") + } + */ + + } + + /// Runs Midas model with given image with given source area to destination area. + /// + /// - Parameters: + /// - on: Input image to run the model. + /// - from: Range of input image to run the model. + /// - to: Size of view to render the result. + /// - Returns: Result of the inference and the times consumed in every steps. + func runMidas(on pixelbuffer: CVPixelBuffer, from source: CGRect, to dest: CGSize) + //-> (Result, Times)? + //-> (FlatArray, Times)? + -> ([Float], Int, Int, Times)? + { + // Start times of each process. + let preprocessingStartTime: Date + let inferenceStartTime: Date + let postprocessingStartTime: Date + + // Processing times in miliseconds. + let preprocessingTime: TimeInterval + let inferenceTime: TimeInterval + let postprocessingTime: TimeInterval + + preprocessingStartTime = Date() + guard let data = preprocess(of: pixelbuffer, from: source) else { + os_log("Preprocessing failed", type: .error) + return nil + } + preprocessingTime = Date().timeIntervalSince(preprocessingStartTime) * 1000 + + inferenceStartTime = Date() + inference(from: data) + inferenceTime = Date().timeIntervalSince(inferenceStartTime) * 1000 + + postprocessingStartTime = Date() + //guard let result = postprocess(to: dest) else { + // os_log("Postprocessing failed", type: .error) + // return nil + //} + postprocessingTime = Date().timeIntervalSince(postprocessingStartTime) * 1000 + + + let results: [Float] + switch outputTensor.dataType { + case .uInt8: + guard let quantization = outputTensor.quantizationParameters else { + print("No results returned because the quantization values for the output tensor are nil.") + return nil + } + let quantizedResults = [UInt8](outputTensor.data) + results = quantizedResults.map { + quantization.scale * Float(Int($0) - quantization.zeroPoint) + } + case .float32: + results = [Float32](unsafeData: outputTensor.data) ?? [] + default: + print("Output tensor data type \(outputTensor.dataType) is unsupported for this example app.") + return nil + } + + + let times = Times( + preprocessing: preprocessingTime, + inference: inferenceTime, + postprocessing: postprocessingTime) + + return (results, Model.input.width, Model.input.height, times) + } + + // MARK: - Private functions to run model + /// Preprocesses given rectangle image to be `Data` of disired size by croping and resizing it. + /// + /// - Parameters: + /// - of: Input image to crop and resize. + /// - from: Target area to be cropped and resized. + /// - Returns: The cropped and resized image. `nil` if it can not be processed. + private func preprocess(of pixelBuffer: CVPixelBuffer, from targetSquare: CGRect) -> Data? { + let sourcePixelFormat = CVPixelBufferGetPixelFormatType(pixelBuffer) + assert(sourcePixelFormat == kCVPixelFormatType_32BGRA) + + // Resize `targetSquare` of input image to `modelSize`. + let modelSize = CGSize(width: Model.input.width, height: Model.input.height) + guard let thumbnail = pixelBuffer.resize(from: targetSquare, to: modelSize) + else { + return nil + } + + // Remove the alpha component from the image buffer to get the initialized `Data`. + let byteCount = + Model.input.batchSize + * Model.input.height * Model.input.width + * Model.input.channelSize + guard + let inputData = thumbnail.rgbData( + isModelQuantized: Model.isQuantized + ) + else { + os_log("Failed to convert the image buffer to RGB data.", type: .error) + return nil + } + + return inputData + } + + + + /* + /// Postprocesses output `Tensor`s to `Result` with size of view to render the result. + /// + /// - Parameters: + /// - to: Size of view to be displaied. + /// - Returns: Postprocessed `Result`. `nil` if it can not be processed. + private func postprocess(to viewSize: CGSize) -> Result? { + // MARK: Formats output tensors + // Convert `Tensor` to `FlatArray`. As Midas is not quantized, convert them to Float type + // `FlatArray`. + let heats = FlatArray(tensor: heatsTensor) + let offsets = FlatArray(tensor: offsetsTensor) + + // MARK: Find position of each key point + // Finds the (row, col) locations of where the keypoints are most likely to be. The highest + // `heats[0, row, col, keypoint]` value, the more likely `keypoint` being located in (`row`, + // `col`). + let keypointPositions = (0.. (Int, Int) in + var maxValue = heats[0, 0, 0, keypoint] + var maxRow = 0 + var maxCol = 0 + for row in 0.. maxValue { + maxValue = heats[0, row, col, keypoint] + maxRow = row + maxCol = col + } + } + } + return (maxRow, maxCol) + } + + // MARK: Calculates total confidence score + // Calculates total confidence score of each key position. + let totalScoreSum = keypointPositions.enumerated().reduce(0.0) { accumulator, elem -> Float32 in + accumulator + sigmoid(heats[0, elem.element.0, elem.element.1, elem.offset]) + } + let totalScore = totalScoreSum / Float32(Model.output.keypointSize) + + // MARK: Calculate key point position on model input + // Calculates `KeyPoint` coordination model input image with `offsets` adjustment. + let coords = keypointPositions.enumerated().map { index, elem -> (y: Float32, x: Float32) in + let (y, x) = elem + let yCoord = + Float32(y) / Float32(Model.output.height - 1) * Float32(Model.input.height) + + offsets[0, y, x, index] + let xCoord = + Float32(x) / Float32(Model.output.width - 1) * Float32(Model.input.width) + + offsets[0, y, x, index + Model.output.keypointSize] + return (y: yCoord, x: xCoord) + } + + // MARK: Transform key point position and make lines + // Make `Result` from `keypointPosition'. Each point is adjusted to `ViewSize` to be drawn. + var result = Result(dots: [], lines: [], score: totalScore) + var bodyPartToDotMap = [BodyPart: CGPoint]() + for (index, part) in BodyPart.allCases.enumerated() { + let position = CGPoint( + x: CGFloat(coords[index].x) * viewSize.width / CGFloat(Model.input.width), + y: CGFloat(coords[index].y) * viewSize.height / CGFloat(Model.input.height) + ) + bodyPartToDotMap[part] = position + result.dots.append(position) + } + + do { + try result.lines = BodyPart.lines.map { map throws -> Line in + guard let from = bodyPartToDotMap[map.from] else { + throw PostprocessError.missingBodyPart(of: map.from) + } + guard let to = bodyPartToDotMap[map.to] else { + throw PostprocessError.missingBodyPart(of: map.to) + } + return Line(from: from, to: to) + } + } catch PostprocessError.missingBodyPart(let missingPart) { + os_log("Postprocessing error: %s is missing.", type: .error, missingPart.rawValue) + return nil + } catch { + os_log("Postprocessing error: %s", type: .error, error.localizedDescription) + return nil + } + + return result + } +*/ + + + + /// Run inference with given `Data` + /// + /// Parameter `from`: `Data` of input image to run model. + private func inference(from data: Data) { + // Copy the initialized `Data` to the input `Tensor`. + do { + try interpreter.copy(data, toInputAt: 0) + + // Run inference by invoking the `Interpreter`. + try interpreter.invoke() + + // Get the output `Tensor` to process the inference results. + outputTensor = try interpreter.output(at: 0) + //heatsTensor = try interpreter.output(at: 0) + //offsetsTensor = try interpreter.output(at: 1) + + + } catch let error { + os_log( + "Failed to invoke the interpreter with error: %s", type: .error, + error.localizedDescription) + return + } + } + + /// Returns value within [0,1]. + private func sigmoid(_ x: Float32) -> Float32 { + return (1.0 / (1.0 + exp(-x))) + } +} + +// MARK: - Data types for inference result +struct KeyPoint { + var bodyPart: BodyPart = BodyPart.NOSE + var position: CGPoint = CGPoint() + var score: Float = 0.0 +} + +struct Line { + let from: CGPoint + let to: CGPoint +} + +struct Times { + var preprocessing: Double + var inference: Double + var postprocessing: Double +} + +struct Result { + var dots: [CGPoint] + var lines: [Line] + var score: Float +} + +enum BodyPart: String, CaseIterable { + case NOSE = "nose" + case LEFT_EYE = "left eye" + case RIGHT_EYE = "right eye" + case LEFT_EAR = "left ear" + case RIGHT_EAR = "right ear" + case LEFT_SHOULDER = "left shoulder" + case RIGHT_SHOULDER = "right shoulder" + case LEFT_ELBOW = "left elbow" + case RIGHT_ELBOW = "right elbow" + case LEFT_WRIST = "left wrist" + case RIGHT_WRIST = "right wrist" + case LEFT_HIP = "left hip" + case RIGHT_HIP = "right hip" + case LEFT_KNEE = "left knee" + case RIGHT_KNEE = "right knee" + case LEFT_ANKLE = "left ankle" + case RIGHT_ANKLE = "right ankle" + + /// List of lines connecting each part. + static let lines = [ + (from: BodyPart.LEFT_WRIST, to: BodyPart.LEFT_ELBOW), + (from: BodyPart.LEFT_ELBOW, to: BodyPart.LEFT_SHOULDER), + (from: BodyPart.LEFT_SHOULDER, to: BodyPart.RIGHT_SHOULDER), + (from: BodyPart.RIGHT_SHOULDER, to: BodyPart.RIGHT_ELBOW), + (from: BodyPart.RIGHT_ELBOW, to: BodyPart.RIGHT_WRIST), + (from: BodyPart.LEFT_SHOULDER, to: BodyPart.LEFT_HIP), + (from: BodyPart.LEFT_HIP, to: BodyPart.RIGHT_HIP), + (from: BodyPart.RIGHT_HIP, to: BodyPart.RIGHT_SHOULDER), + (from: BodyPart.LEFT_HIP, to: BodyPart.LEFT_KNEE), + (from: BodyPart.LEFT_KNEE, to: BodyPart.LEFT_ANKLE), + (from: BodyPart.RIGHT_HIP, to: BodyPart.RIGHT_KNEE), + (from: BodyPart.RIGHT_KNEE, to: BodyPart.RIGHT_ANKLE), + ] +} + +// MARK: - Delegates Enum +enum Delegates: Int, CaseIterable { + case CPU + case Metal + case CoreML + + var description: String { + switch self { + case .CPU: + return "CPU" + case .Metal: + return "GPU" + case .CoreML: + return "NPU" + } + } +} + +// MARK: - Custom Errors +enum PostprocessError: Error { + case missingBodyPart(of: BodyPart) +} + +// MARK: - Information about the model file. +typealias FileInfo = (name: String, extension: String) + +enum Model { + static let file: FileInfo = ( + name: "model_opt", extension: "tflite" + ) + + static let input = (batchSize: 1, height: 256, width: 256, channelSize: 3) + static let output = (batchSize: 1, height: 256, width: 256, channelSize: 1) + static let isQuantized = false +} + + +extension Array { + /// Creates a new array from the bytes of the given unsafe data. + /// + /// - Warning: The array's `Element` type must be trivial in that it can be copied bit for bit + /// with no indirection or reference-counting operations; otherwise, copying the raw bytes in + /// the `unsafeData`'s buffer to a new array returns an unsafe copy. + /// - Note: Returns `nil` if `unsafeData.count` is not a multiple of + /// `MemoryLayout.stride`. + /// - Parameter unsafeData: The data containing the bytes to turn into an array. + init?(unsafeData: Data) { + guard unsafeData.count % MemoryLayout.stride == 0 else { return nil } + #if swift(>=5.0) + self = unsafeData.withUnsafeBytes { .init($0.bindMemory(to: Element.self)) } + #else + self = unsafeData.withUnsafeBytes { + .init(UnsafeBufferPointer( + start: $0, + count: unsafeData.count / MemoryLayout.stride + )) + } + #endif // swift(>=5.0) + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Storyboards/Base.lproj/Launch Screen.storyboard b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Storyboards/Base.lproj/Launch Screen.storyboard new file mode 100644 index 0000000000000000000000000000000000000000..a04c79f554777863bd0dc8287bfd60704ce28bf2 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Storyboards/Base.lproj/Launch Screen.storyboard @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Storyboards/Base.lproj/Main.storyboard b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Storyboards/Base.lproj/Main.storyboard new file mode 100644 index 0000000000000000000000000000000000000000..5f5623794bd35b9bb75efd7b7e249fd7357fdfbd --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Storyboards/Base.lproj/Main.storyboard @@ -0,0 +1,236 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/ViewControllers/ViewController.swift b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/ViewControllers/ViewController.swift new file mode 100644 index 0000000000000000000000000000000000000000..fbb51b5a303412c0bbd158d76d025cf88fee6f8f --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/ViewControllers/ViewController.swift @@ -0,0 +1,489 @@ +// Copyright 2019 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import AVFoundation +import UIKit +import os + + +public struct PixelData { + var a: UInt8 + var r: UInt8 + var g: UInt8 + var b: UInt8 +} + +extension UIImage { + convenience init?(pixels: [PixelData], width: Int, height: Int) { + guard width > 0 && height > 0, pixels.count == width * height else { return nil } + var data = pixels + guard let providerRef = CGDataProvider(data: Data(bytes: &data, count: data.count * MemoryLayout.size) as CFData) + else { return nil } + guard let cgim = CGImage( + width: width, + height: height, + bitsPerComponent: 8, + bitsPerPixel: 32, + bytesPerRow: width * MemoryLayout.size, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGBitmapInfo(rawValue: CGImageAlphaInfo.premultipliedFirst.rawValue), + provider: providerRef, + decode: nil, + shouldInterpolate: false, + intent: .defaultIntent) + else { return nil } + self.init(cgImage: cgim) + } +} + + +class ViewController: UIViewController { + // MARK: Storyboards Connections + @IBOutlet weak var previewView: PreviewView! + + //@IBOutlet weak var overlayView: OverlayView! + @IBOutlet weak var overlayView: UIImageView! + + private var imageView : UIImageView = UIImageView(frame:CGRect(x:0, y:0, width:400, height:400)) + + private var imageViewInitialized: Bool = false + + @IBOutlet weak var resumeButton: UIButton! + @IBOutlet weak var cameraUnavailableLabel: UILabel! + + @IBOutlet weak var tableView: UITableView! + + @IBOutlet weak var threadCountLabel: UILabel! + @IBOutlet weak var threadCountStepper: UIStepper! + + @IBOutlet weak var delegatesControl: UISegmentedControl! + + // MARK: ModelDataHandler traits + var threadCount: Int = Constants.defaultThreadCount + var delegate: Delegates = Constants.defaultDelegate + + // MARK: Result Variables + // Inferenced data to render. + private var inferencedData: InferencedData? + + // Minimum score to render the result. + private let minimumScore: Float = 0.5 + + private var avg_latency: Double = 0.0 + + // Relative location of `overlayView` to `previewView`. + private var overlayViewFrame: CGRect? + + private var previewViewFrame: CGRect? + + // MARK: Controllers that manage functionality + // Handles all the camera related functionality + private lazy var cameraCapture = CameraFeedManager(previewView: previewView) + + // Handles all data preprocessing and makes calls to run inference. + private var modelDataHandler: ModelDataHandler? + + // MARK: View Handling Methods + override func viewDidLoad() { + super.viewDidLoad() + + do { + modelDataHandler = try ModelDataHandler() + } catch let error { + fatalError(error.localizedDescription) + } + + cameraCapture.delegate = self + tableView.delegate = self + tableView.dataSource = self + + // MARK: UI Initialization + // Setup thread count stepper with white color. + // https://forums.developer.apple.com/thread/121495 + threadCountStepper.setDecrementImage( + threadCountStepper.decrementImage(for: .normal), for: .normal) + threadCountStepper.setIncrementImage( + threadCountStepper.incrementImage(for: .normal), for: .normal) + // Setup initial stepper value and its label. + threadCountStepper.value = Double(Constants.defaultThreadCount) + threadCountLabel.text = Constants.defaultThreadCount.description + + // Setup segmented controller's color. + delegatesControl.setTitleTextAttributes( + [NSAttributedString.Key.foregroundColor: UIColor.lightGray], + for: .normal) + delegatesControl.setTitleTextAttributes( + [NSAttributedString.Key.foregroundColor: UIColor.black], + for: .selected) + // Remove existing segments to initialize it with `Delegates` entries. + delegatesControl.removeAllSegments() + Delegates.allCases.forEach { delegate in + delegatesControl.insertSegment( + withTitle: delegate.description, + at: delegate.rawValue, + animated: false) + } + delegatesControl.selectedSegmentIndex = 0 + } + + override func viewWillAppear(_ animated: Bool) { + super.viewWillAppear(animated) + + cameraCapture.checkCameraConfigurationAndStartSession() + } + + override func viewWillDisappear(_ animated: Bool) { + cameraCapture.stopSession() + } + + override func viewDidLayoutSubviews() { + overlayViewFrame = overlayView.frame + previewViewFrame = previewView.frame + } + + // MARK: Button Actions + @IBAction func didChangeThreadCount(_ sender: UIStepper) { + let changedCount = Int(sender.value) + if threadCountLabel.text == changedCount.description { + return + } + + do { + modelDataHandler = try ModelDataHandler(threadCount: changedCount, delegate: delegate) + } catch let error { + fatalError(error.localizedDescription) + } + threadCount = changedCount + threadCountLabel.text = changedCount.description + os_log("Thread count is changed to: %d", threadCount) + } + + @IBAction func didChangeDelegate(_ sender: UISegmentedControl) { + guard let changedDelegate = Delegates(rawValue: delegatesControl.selectedSegmentIndex) else { + fatalError("Unexpected value from delegates segemented controller.") + } + do { + modelDataHandler = try ModelDataHandler(threadCount: threadCount, delegate: changedDelegate) + } catch let error { + fatalError(error.localizedDescription) + } + delegate = changedDelegate + os_log("Delegate is changed to: %s", delegate.description) + } + + @IBAction func didTapResumeButton(_ sender: Any) { + cameraCapture.resumeInterruptedSession { complete in + + if complete { + self.resumeButton.isHidden = true + self.cameraUnavailableLabel.isHidden = true + } else { + self.presentUnableToResumeSessionAlert() + } + } + } + + func presentUnableToResumeSessionAlert() { + let alert = UIAlertController( + title: "Unable to Resume Session", + message: "There was an error while attempting to resume session.", + preferredStyle: .alert + ) + alert.addAction(UIAlertAction(title: "OK", style: .default, handler: nil)) + + self.present(alert, animated: true) + } +} + +// MARK: - CameraFeedManagerDelegate Methods +extension ViewController: CameraFeedManagerDelegate { + func cameraFeedManager(_ manager: CameraFeedManager, didOutput pixelBuffer: CVPixelBuffer) { + runModel(on: pixelBuffer) + } + + // MARK: Session Handling Alerts + func cameraFeedManagerDidEncounterSessionRunTimeError(_ manager: CameraFeedManager) { + // Handles session run time error by updating the UI and providing a button if session can be + // manually resumed. + self.resumeButton.isHidden = false + } + + func cameraFeedManager( + _ manager: CameraFeedManager, sessionWasInterrupted canResumeManually: Bool + ) { + // Updates the UI when session is interupted. + if canResumeManually { + self.resumeButton.isHidden = false + } else { + self.cameraUnavailableLabel.isHidden = false + } + } + + func cameraFeedManagerDidEndSessionInterruption(_ manager: CameraFeedManager) { + // Updates UI once session interruption has ended. + self.cameraUnavailableLabel.isHidden = true + self.resumeButton.isHidden = true + } + + func presentVideoConfigurationErrorAlert(_ manager: CameraFeedManager) { + let alertController = UIAlertController( + title: "Confirguration Failed", message: "Configuration of camera has failed.", + preferredStyle: .alert) + let okAction = UIAlertAction(title: "OK", style: .cancel, handler: nil) + alertController.addAction(okAction) + + present(alertController, animated: true, completion: nil) + } + + func presentCameraPermissionsDeniedAlert(_ manager: CameraFeedManager) { + let alertController = UIAlertController( + title: "Camera Permissions Denied", + message: + "Camera permissions have been denied for this app. You can change this by going to Settings", + preferredStyle: .alert) + + let cancelAction = UIAlertAction(title: "Cancel", style: .cancel, handler: nil) + let settingsAction = UIAlertAction(title: "Settings", style: .default) { action in + if let url = URL.init(string: UIApplication.openSettingsURLString) { + UIApplication.shared.open(url, options: [:], completionHandler: nil) + } + } + + alertController.addAction(cancelAction) + alertController.addAction(settingsAction) + + present(alertController, animated: true, completion: nil) + } + + @objc func runModel(on pixelBuffer: CVPixelBuffer) { + guard let overlayViewFrame = overlayViewFrame, let previewViewFrame = previewViewFrame + else { + return + } + // To put `overlayView` area as model input, transform `overlayViewFrame` following transform + // from `previewView` to `pixelBuffer`. `previewView` area is transformed to fit in + // `pixelBuffer`, because `pixelBuffer` as a camera output is resized to fill `previewView`. + // https://developer.apple.com/documentation/avfoundation/avlayervideogravity/1385607-resizeaspectfill + let modelInputRange = overlayViewFrame.applying( + previewViewFrame.size.transformKeepAspect(toFitIn: pixelBuffer.size)) + + // Run Midas model. + guard + let (result, width, height, times) = self.modelDataHandler?.runMidas( + on: pixelBuffer, + from: modelInputRange, + to: overlayViewFrame.size) + else { + os_log("Cannot get inference result.", type: .error) + return + } + + if avg_latency == 0 { + avg_latency = times.inference + } else { + avg_latency = times.inference*0.1 + avg_latency*0.9 + } + + // Udpate `inferencedData` to render data in `tableView`. + inferencedData = InferencedData(score: Float(avg_latency), times: times) + + //let height = 256 + //let width = 256 + + let outputs = result + let outputs_size = width * height; + + var multiplier : Float = 1.0; + + let max_val : Float = outputs.max() ?? 0 + let min_val : Float = outputs.min() ?? 0 + + if((max_val - min_val) > 0) { + multiplier = 255 / (max_val - min_val); + } + + // Draw result. + DispatchQueue.main.async { + self.tableView.reloadData() + + var pixels: [PixelData] = .init(repeating: .init(a: 255, r: 0, g: 0, b: 0), count: width * height) + + for i in pixels.indices { + //if(i < 1000) + //{ + let val = UInt8((outputs[i] - min_val) * multiplier) + + pixels[i].r = val + pixels[i].g = val + pixels[i].b = val + //} + } + + + /* + pixels[i].a = 255 + pixels[i].r = .random(in: 0...255) + pixels[i].g = .random(in: 0...255) + pixels[i].b = .random(in: 0...255) + } + */ + + DispatchQueue.main.async { + let image = UIImage(pixels: pixels, width: width, height: height) + + self.imageView.image = image + + if (self.imageViewInitialized == false) { + self.imageViewInitialized = true + self.overlayView.addSubview(self.imageView) + self.overlayView.setNeedsDisplay() + } + } + + /* + let image = UIImage(pixels: pixels, width: width, height: height) + + var imageView : UIImageView + imageView = UIImageView(frame:CGRect(x:0, y:0, width:400, height:400)); + imageView.image = image + self.overlayView.addSubview(imageView) + self.overlayView.setNeedsDisplay() + */ + } + } +/* + func drawResult(of result: Result) { + self.overlayView.dots = result.dots + self.overlayView.lines = result.lines + self.overlayView.setNeedsDisplay() + } + + func clearResult() { + self.overlayView.clear() + self.overlayView.setNeedsDisplay() + } + */ + +} + + +// MARK: - TableViewDelegate, TableViewDataSource Methods +extension ViewController: UITableViewDelegate, UITableViewDataSource { + func numberOfSections(in tableView: UITableView) -> Int { + return InferenceSections.allCases.count + } + + func tableView(_ tableView: UITableView, numberOfRowsInSection section: Int) -> Int { + guard let section = InferenceSections(rawValue: section) else { + return 0 + } + + return section.subcaseCount + } + + func tableView(_ tableView: UITableView, cellForRowAt indexPath: IndexPath) -> UITableViewCell { + let cell = tableView.dequeueReusableCell(withIdentifier: "InfoCell") as! InfoCell + guard let section = InferenceSections(rawValue: indexPath.section) else { + return cell + } + guard let data = inferencedData else { return cell } + + var fieldName: String + var info: String + + switch section { + case .Score: + fieldName = section.description + info = String(format: "%.3f", data.score) + case .Time: + guard let row = ProcessingTimes(rawValue: indexPath.row) else { + return cell + } + var time: Double + switch row { + case .InferenceTime: + time = data.times.inference + } + fieldName = row.description + info = String(format: "%.2fms", time) + } + + cell.fieldNameLabel.text = fieldName + cell.infoLabel.text = info + + return cell + } + + func tableView(_ tableView: UITableView, heightForRowAt indexPath: IndexPath) -> CGFloat { + guard let section = InferenceSections(rawValue: indexPath.section) else { + return 0 + } + + var height = Traits.normalCellHeight + if indexPath.row == section.subcaseCount - 1 { + height = Traits.separatorCellHeight + Traits.bottomSpacing + } + return height + } + +} + +// MARK: - Private enums +/// UI coinstraint values +fileprivate enum Traits { + static let normalCellHeight: CGFloat = 35.0 + static let separatorCellHeight: CGFloat = 25.0 + static let bottomSpacing: CGFloat = 30.0 +} + +fileprivate struct InferencedData { + var score: Float + var times: Times +} + +/// Type of sections in Info Cell +fileprivate enum InferenceSections: Int, CaseIterable { + case Score + case Time + + var description: String { + switch self { + case .Score: + return "Average" + case .Time: + return "Processing Time" + } + } + + var subcaseCount: Int { + switch self { + case .Score: + return 1 + case .Time: + return ProcessingTimes.allCases.count + } + } +} + +/// Type of processing times in Time section in Info Cell +fileprivate enum ProcessingTimes: Int, CaseIterable { + case InferenceTime + + var description: String { + switch self { + case .InferenceTime: + return "Inference Time" + } + } +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Views/OverlayView.swift b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Views/OverlayView.swift new file mode 100644 index 0000000000000000000000000000000000000000..3b53910b57563b6a195fd53321fa2a24ebaf3d3f --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Midas/Views/OverlayView.swift @@ -0,0 +1,63 @@ +// Copyright 2019 The TensorFlow Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import UIKit + +/// UIView for rendering inference output. +class OverlayView: UIView { + + var dots = [CGPoint]() + var lines = [Line]() + + override func draw(_ rect: CGRect) { + for dot in dots { + drawDot(of: dot) + } + for line in lines { + drawLine(of: line) + } + } + + func drawDot(of dot: CGPoint) { + let dotRect = CGRect( + x: dot.x - Traits.dot.radius / 2, y: dot.y - Traits.dot.radius / 2, + width: Traits.dot.radius, height: Traits.dot.radius) + let dotPath = UIBezierPath(ovalIn: dotRect) + + Traits.dot.color.setFill() + dotPath.fill() + } + + func drawLine(of line: Line) { + let linePath = UIBezierPath() + linePath.move(to: CGPoint(x: line.from.x, y: line.from.y)) + linePath.addLine(to: CGPoint(x: line.to.x, y: line.to.y)) + linePath.close() + + linePath.lineWidth = Traits.line.width + Traits.line.color.setStroke() + + linePath.stroke() + } + + func clear() { + self.dots = [] + self.lines = [] + } +} + +private enum Traits { + static let dot = (radius: CGFloat(5), color: UIColor.orange) + static let line = (width: CGFloat(1.0), color: UIColor.orange) +} diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Podfile b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Podfile new file mode 100644 index 0000000000000000000000000000000000000000..5e9461fc96dbbe3c22ca6bbf2bfd7df3981b9462 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/Podfile @@ -0,0 +1,12 @@ +# Uncomment the next line to define a global platform for your project + platform :ios, '12.0' + +target 'Midas' do + # Comment the next line if you're not using Swift and don't want to use dynamic frameworks + use_frameworks! + + # Pods for Midas + pod 'TensorFlowLiteSwift', '~> 0.0.1-nightly' + pod 'TensorFlowLiteSwift/CoreML', '~> 0.0.1-nightly' + pod 'TensorFlowLiteSwift/Metal', '~> 0.0.1-nightly' +end diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/README.md b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7b8eb29feaa21e67814b035dbd5c5fb2c62a4151 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/README.md @@ -0,0 +1,105 @@ +# Tensorflow Lite MiDaS iOS Example + +### Requirements + +- XCode 11.0 or above +- iOS 12.0 or above, [iOS 14 breaks the NPU Delegate](https://github.com/tensorflow/tensorflow/issues/43339) +- TensorFlow 2.4.0, TensorFlowLiteSwift -> 0.0.1-nightly + +## Quick Start with a MiDaS Example + +MiDaS is a neural network to compute depth from a single image. It uses TensorFlowLiteSwift / C++ libraries on iOS. The code is written in Swift. + +Paper: https://arxiv.org/abs/1907.01341 + +> Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer +> René Ranftl, Katrin Lasinger, David Hafner, Konrad Schindler, Vladlen Koltun + +### Install TensorFlow + +Set default python version to python3: + +``` +echo 'export PATH=/usr/local/opt/python/libexec/bin:$PATH' >> ~/.zshenv +echo 'alias python=python3' >> ~/.zshenv +echo 'alias pip=pip3' >> ~/.zshenv +``` + +Install TensorFlow + +```shell +pip install tensorflow +``` + +### Install TensorFlowLiteSwift via Cocoapods + +Set required TensorFlowLiteSwift version in the file (`0.0.1-nightly` is recommended): https://github.com/isl-org/MiDaS/blob/master/mobile/ios/Podfile#L9 + +Install: brew, ruby, cocoapods + +``` +ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" +brew install mc rbenv ruby-build +sudo gem install cocoapods +``` + + +The TensorFlowLiteSwift library is available in [Cocoapods](https://cocoapods.org/), to integrate it to our project, we can run in the root directory of the project: + +```ruby +pod install +``` + +Now open the `Midas.xcworkspace` file in XCode, select your iPhone device (XCode->Product->Destination->iPhone) and launch it (cmd + R). If everything works well, you should see a real-time depth map from your camera. + +### Model + +The TensorFlow (TFlite) model `midas.tflite` is in the folder `/Midas/Model` + + +To use another model, you should convert it from TensorFlow saved-model to TFlite model (so that it can be deployed): + +```python +saved_model_export_dir = "./saved_model" +converter = tf.lite.TFLiteConverter.from_saved_model(saved_model_export_dir) +tflite_model = converter.convert() +open(model_tflite_name, "wb").write("model.tflite") +``` + +### Setup XCode + +* Open directory `.xcworkspace` from the XCode + +* Press on your ProjectName (left-top corner) -> change Bundle Identifier to `com.midas.tflite-npu` or something like this (it should be unique) + +* select your Developer Team (your should be signed-in by using your AppleID) + +* Connect your iPhone (if you want to run it on real device instead of simulator), select your iPhone device (XCode->Product->Destination->iPhone) + +* Click in the XCode: Product -> Run + +* On your iPhone device go to the: Settings -> General -> Device Management (or Profiles) -> Apple Development -> Trust Apple Development + +---- + +Original repository: https://github.com/isl-org/MiDaS + + +### Examples: + +| ![photo_2020-09-27_17-43-20](https://user-images.githubusercontent.com/4096485/94367804-9610de80-00e9-11eb-8a23-8b32a6f52d41.jpg) | ![photo_2020-09-27_17-49-22](https://user-images.githubusercontent.com/4096485/94367974-7201cd00-00ea-11eb-8e0a-68eb9ea10f63.jpg) | ![photo_2020-09-27_17-52-30](https://user-images.githubusercontent.com/4096485/94367976-729a6380-00ea-11eb-8ce0-39d3e26dd550.jpg) | ![photo_2020-09-27_17-43-21](https://user-images.githubusercontent.com/4096485/94367807-97420b80-00e9-11eb-9dcd-848ad9e89e03.jpg) | +|---|---|---|---| + +## LICENSE + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/RunScripts/download_models.sh b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/RunScripts/download_models.sh new file mode 100644 index 0000000000000000000000000000000000000000..d737b39d966278f5c6bc29802526ab86f8473de4 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/mobile/ios/RunScripts/download_models.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Download TF Lite model from the internet if it does not exist. + +TFLITE_MODEL="model_opt.tflite" +TFLITE_FILE="Midas/Model/${TFLITE_MODEL}" +MODEL_SRC="https://github.com/isl-org/MiDaS/releases/download/v2/${TFLITE_MODEL}" + +if test -f "${TFLITE_FILE}"; then + echo "INFO: TF Lite model already exists. Skip downloading and use the local model." +else + curl --create-dirs -o "${TFLITE_FILE}" -LJO "${MODEL_SRC}" + echo "INFO: Downloaded TensorFlow Lite model to ${TFLITE_FILE}." +fi + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/LICENSE b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..6606ec028d1c629986e7019fe3564f5b4bfe425d --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Alexey + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/README.md b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1d43c2606767798ee46b34292e0483197424ec23 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/README.md @@ -0,0 +1,131 @@ +# MiDaS for ROS1 by using LibTorch in C++ + +### Requirements + +- Ubuntu 17.10 / 18.04 / 20.04, Debian Stretch +- ROS Melodic for Ubuntu (17.10 / 18.04) / Debian Stretch, ROS Noetic for Ubuntu 20.04 +- C++11 +- LibTorch >= 1.6 + +## Quick Start with a MiDaS Example + +MiDaS is a neural network to compute depth from a single image. + +* input from `image_topic`: `sensor_msgs/Image` - `RGB8` image with any shape +* output to `midas_topic`: `sensor_msgs/Image` - `TYPE_32FC1` inverse relative depth maps in range [0 - 255] with original size and channels=1 + +### Install Dependecies + +* install ROS Melodic for Ubuntu 17.10 / 18.04: +```bash +wget https://raw.githubusercontent.com/isl-org/MiDaS/master/ros/additions/install_ros_melodic_ubuntu_17_18.sh +./install_ros_melodic_ubuntu_17_18.sh +``` + +or Noetic for Ubuntu 20.04: + +```bash +wget https://raw.githubusercontent.com/isl-org/MiDaS/master/ros/additions/install_ros_noetic_ubuntu_20.sh +./install_ros_noetic_ubuntu_20.sh +``` + + +* install LibTorch 1.7 with CUDA 11.0: + +On **Jetson (ARM)**: +```bash +wget https://nvidia.box.com/shared/static/wa34qwrwtk9njtyarwt5nvo6imenfy26.whl -O torch-1.7.0-cp36-cp36m-linux_aarch64.whl +sudo apt-get install python3-pip libopenblas-base libopenmpi-dev +pip3 install Cython +pip3 install numpy torch-1.7.0-cp36-cp36m-linux_aarch64.whl +``` +Or compile LibTorch from source: https://github.com/pytorch/pytorch#from-source + +On **Linux (x86_64)**: +```bash +cd ~/ +wget https://download.pytorch.org/libtorch/cu110/libtorch-cxx11-abi-shared-with-deps-1.7.0%2Bcu110.zip +unzip libtorch-cxx11-abi-shared-with-deps-1.7.0+cu110.zip +``` + +* create symlink for OpenCV: + +```bash +sudo ln -s /usr/include/opencv4 /usr/include/opencv +``` + +* download and install MiDaS: + +```bash +source ~/.bashrc +cd ~/ +mkdir catkin_ws +cd catkin_ws +git clone https://github.com/isl-org/MiDaS +mkdir src +cp -r MiDaS/ros/* src + +chmod +x src/additions/*.sh +chmod +x src/*.sh +chmod +x src/midas_cpp/scripts/*.py +cp src/additions/do_catkin_make.sh ./do_catkin_make.sh +./do_catkin_make.sh +./src/additions/downloads.sh +``` + +### Usage + +* run only `midas` node: `~/catkin_ws/src/launch_midas_cpp.sh` + +#### Test + +* Test - capture video and show result in the window: + * place any `test.mp4` video file to the directory `~/catkin_ws/src/` + * run `midas` node: `~/catkin_ws/src/launch_midas_cpp.sh` + * run test nodes in another terminal: `cd ~/catkin_ws/src && ./run_talker_listener_test.sh` and wait 30 seconds + + (to use Python 2, run command `sed -i 's/python3/python2/' ~/catkin_ws/src/midas_cpp/scripts/*.py` ) + +## Mobile version of MiDaS - Monocular Depth Estimation + +### Accuracy + +* MiDaS v2 small - ResNet50 default-decoder 384x384 +* MiDaS v2.1 small - EfficientNet-Lite3 small-decoder 256x256 + +**Zero-shot error** (the lower - the better): + +| Model | DIW WHDR | Eth3d AbsRel | Sintel AbsRel | Kitti δ>1.25 | NyuDepthV2 δ>1.25 | TUM δ>1.25 | +|---|---|---|---|---|---|---| +| MiDaS v2 small 384x384 | **0.1248** | 0.1550 | **0.3300** | **21.81** | 15.73 | 17.00 | +| MiDaS v2.1 small 256x256 | 0.1344 | **0.1344** | 0.3370 | 29.27 | **13.43** | **14.53** | +| Relative improvement, % | -8 % | **+13 %** | -2 % | -34 % | **+15 %** | **+15 %** | + +None of Train/Valid/Test subsets of datasets (DIW, Eth3d, Sintel, Kitti, NyuDepthV2, TUM) were not involved in Training or Fine Tuning. + +### Inference speed (FPS) on nVidia GPU + +Inference speed excluding pre and post processing, batch=1, **Frames Per Second** (the higher - the better): + +| Model | Jetson Nano, FPS | RTX 2080Ti, FPS | +|---|---|---| +| MiDaS v2 small 384x384 | 1.6 | 117 | +| MiDaS v2.1 small 256x256 | 8.1 | 232 | +| SpeedUp, X times | **5x** | **2x** | + +### Citation + +This repository contains code to compute depth from a single image. It accompanies our [paper](https://arxiv.org/abs/1907.01341v3): + +>Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer +René Ranftl, Katrin Lasinger, David Hafner, Konrad Schindler, Vladlen Koltun + +Please cite our paper if you use this code or any of the models: +``` +@article{Ranftl2020, + author = {Ren\'{e} Ranftl and Katrin Lasinger and David Hafner and Konrad Schindler and Vladlen Koltun}, + title = {Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer}, + journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (TPAMI)}, + year = {2020}, +} +``` diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/do_catkin_make.sh b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/do_catkin_make.sh new file mode 100644 index 0000000000000000000000000000000000000000..0d416fc00282aab146326bbba12a9274e1ba29b8 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/do_catkin_make.sh @@ -0,0 +1,5 @@ +mkdir src +catkin_make +source devel/setup.bash +echo $ROS_PACKAGE_PATH +chmod +x ./devel/setup.bash diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/downloads.sh b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/downloads.sh new file mode 100644 index 0000000000000000000000000000000000000000..9c967d4e2dc7997da26399a063b5a54ecc314eb1 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/downloads.sh @@ -0,0 +1,5 @@ +mkdir ~/.ros +wget https://github.com/isl-org/MiDaS/releases/download/v2_1/model-small-traced.pt +cp ./model-small-traced.pt ~/.ros/model-small-traced.pt + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/install_ros_melodic_ubuntu_17_18.sh b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/install_ros_melodic_ubuntu_17_18.sh new file mode 100644 index 0000000000000000000000000000000000000000..b868112631e9d9bc7bccb601407dfc857b8a99d5 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/install_ros_melodic_ubuntu_17_18.sh @@ -0,0 +1,34 @@ +#@title { display-mode: "code" } + +#from http://wiki.ros.org/indigo/Installation/Ubuntu + +#1.2 Setup sources.list +sudo sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list' + +# 1.3 Setup keys +sudo apt-key adv --keyserver 'hkp://keyserver.ubuntu.com:80' --recv-key C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654 +sudo apt-key adv --keyserver 'hkp://ha.pool.sks-keyservers.net:80' --recv-key 421C365BD9FF1F717815A3895523BAEEB01FA116 + +curl -sSL 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xC1CF6E31E6BADE8868B172B4F42ED6FBAB17C654' | sudo apt-key add - + +# 1.4 Installation +sudo apt-get update +sudo apt-get upgrade + +# Desktop-Full Install: +sudo apt-get install ros-melodic-desktop-full + +printf "\nsource /opt/ros/melodic/setup.bash\n" >> ~/.bashrc + +# 1.5 Initialize rosdep +sudo rosdep init +rosdep update + + +# 1.7 Getting rosinstall (python) +sudo apt-get install python-rosinstall +sudo apt-get install python-catkin-tools +sudo apt-get install python-rospy +sudo apt-get install python-rosdep +sudo apt-get install python-roscd +sudo apt-get install python-pip \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/install_ros_noetic_ubuntu_20.sh b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/install_ros_noetic_ubuntu_20.sh new file mode 100644 index 0000000000000000000000000000000000000000..d73ea1a3d92359819167d735a92d2a650b9bc245 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/install_ros_noetic_ubuntu_20.sh @@ -0,0 +1,33 @@ +#@title { display-mode: "code" } + +#from http://wiki.ros.org/indigo/Installation/Ubuntu + +#1.2 Setup sources.list +sudo sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list' + +# 1.3 Setup keys +sudo apt-key adv --keyserver 'hkp://keyserver.ubuntu.com:80' --recv-key C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654 + +curl -sSL 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xC1CF6E31E6BADE8868B172B4F42ED6FBAB17C654' | sudo apt-key add - + +# 1.4 Installation +sudo apt-get update +sudo apt-get upgrade + +# Desktop-Full Install: +sudo apt-get install ros-noetic-desktop-full + +printf "\nsource /opt/ros/noetic/setup.bash\n" >> ~/.bashrc + +# 1.5 Initialize rosdep +sudo rosdep init +rosdep update + + +# 1.7 Getting rosinstall (python) +sudo apt-get install python3-rosinstall +sudo apt-get install python3-catkin-tools +sudo apt-get install python3-rospy +sudo apt-get install python3-rosdep +sudo apt-get install python3-roscd +sudo apt-get install python3-pip \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/make_package_cpp.sh b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/make_package_cpp.sh new file mode 100644 index 0000000000000000000000000000000000000000..d0ef6073a9c9ce40744e1c81d557c1c68255b95e --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/make_package_cpp.sh @@ -0,0 +1,16 @@ +cd ~/catkin_ws/src +catkin_create_pkg midas_cpp std_msgs roscpp cv_bridge sensor_msgs image_transport +cd ~/catkin_ws +catkin_make + +chmod +x ~/catkin_ws/devel/setup.bash +printf "\nsource ~/catkin_ws/devel/setup.bash" >> ~/.bashrc +source ~/catkin_ws/devel/setup.bash + + +sudo rosdep init +rosdep update +#rospack depends1 midas_cpp +roscd midas_cpp +#cat package.xml +#rospack depends midas_cpp \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/launch_midas_cpp.sh b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/launch_midas_cpp.sh new file mode 100644 index 0000000000000000000000000000000000000000..5a0d1583fffdc49216c625dfd07af2ae3b01a7a0 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/launch_midas_cpp.sh @@ -0,0 +1,2 @@ +source ~/catkin_ws/devel/setup.bash +roslaunch midas_cpp midas_cpp.launch model_name:="model-small-traced.pt" input_topic:="image_topic" output_topic:="midas_topic" out_orig_size:="true" \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/CMakeLists.txt b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..885341691d217f9c4c8fcb1e4ff568d87788c7b8 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/CMakeLists.txt @@ -0,0 +1,189 @@ +cmake_minimum_required(VERSION 3.0.2) +project(midas_cpp) + +## Compile as C++11, supported in ROS Kinetic and newer +# add_compile_options(-std=c++11) + +## Find catkin macros and libraries +## if COMPONENTS list like find_package(catkin REQUIRED COMPONENTS xyz) +## is used, also find other catkin packages +find_package(catkin REQUIRED COMPONENTS + cv_bridge + image_transport + roscpp + rospy + sensor_msgs + std_msgs +) + +## System dependencies are found with CMake's conventions +# find_package(Boost REQUIRED COMPONENTS system) + +list(APPEND CMAKE_PREFIX_PATH "~/libtorch") +list(APPEND CMAKE_PREFIX_PATH "/usr/local/lib/python3.6/dist-packages/torch/lib") +list(APPEND CMAKE_PREFIX_PATH "/usr/local/lib/python2.7/dist-packages/torch/lib") + +if(NOT EXISTS "~/libtorch") + if (EXISTS "/usr/local/lib/python3.6/dist-packages/torch") + include_directories(/usr/local/include) + include_directories(/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include) + include_directories(/usr/local/lib/python3.6/dist-packages/torch/include) + + link_directories(/usr/local/lib) + link_directories(/usr/local/lib/python3.6/dist-packages/torch/lib) + + set(CMAKE_PREFIX_PATH /usr/local/lib/python3.6/dist-packages/torch) + set(Boost_USE_MULTITHREADED ON) + set(Torch_DIR /usr/local/lib/python3.6/dist-packages/torch) + + elseif (EXISTS "/usr/local/lib/python2.7/dist-packages/torch") + + include_directories(/usr/local/include) + include_directories(/usr/local/lib/python2.7/dist-packages/torch/include/torch/csrc/api/include) + include_directories(/usr/local/lib/python2.7/dist-packages/torch/include) + + link_directories(/usr/local/lib) + link_directories(/usr/local/lib/python2.7/dist-packages/torch/lib) + + set(CMAKE_PREFIX_PATH /usr/local/lib/python2.7/dist-packages/torch) + set(Boost_USE_MULTITHREADED ON) + set(Torch_DIR /usr/local/lib/python2.7/dist-packages/torch) + endif() +endif() + + + +find_package(Torch REQUIRED) +find_package(OpenCV REQUIRED) +include_directories( ${OpenCV_INCLUDE_DIRS} ) + +add_executable(midas_cpp src/main.cpp) +target_link_libraries(midas_cpp "${TORCH_LIBRARIES}" "${OpenCV_LIBS} ${catkin_LIBRARIES}") +set_property(TARGET midas_cpp PROPERTY CXX_STANDARD 14) + + + +################################### +## catkin specific configuration ## +################################### +## The catkin_package macro generates cmake config files for your package +## Declare things to be passed to dependent projects +## INCLUDE_DIRS: uncomment this if your package contains header files +## LIBRARIES: libraries you create in this project that dependent projects also need +## CATKIN_DEPENDS: catkin_packages dependent projects also need +## DEPENDS: system dependencies of this project that dependent projects also need +catkin_package( +# INCLUDE_DIRS include +# LIBRARIES midas_cpp +# CATKIN_DEPENDS cv_bridge image_transport roscpp sensor_msgs std_msgs +# DEPENDS system_lib +) + +########### +## Build ## +########### + +## Specify additional locations of header files +## Your package locations should be listed before other locations +include_directories( +# include + ${catkin_INCLUDE_DIRS} +) + +## Declare a C++ library +# add_library(${PROJECT_NAME} +# src/${PROJECT_NAME}/midas_cpp.cpp +# ) + +## Add cmake target dependencies of the library +## as an example, code may need to be generated before libraries +## either from message generation or dynamic reconfigure +# add_dependencies(${PROJECT_NAME} ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) + +## Declare a C++ executable +## With catkin_make all packages are built within a single CMake context +## The recommended prefix ensures that target names across packages don't collide +# add_executable(${PROJECT_NAME}_node src/midas_cpp_node.cpp) + +## Rename C++ executable without prefix +## The above recommended prefix causes long target names, the following renames the +## target back to the shorter version for ease of user use +## e.g. "rosrun someones_pkg node" instead of "rosrun someones_pkg someones_pkg_node" +# set_target_properties(${PROJECT_NAME}_node PROPERTIES OUTPUT_NAME node PREFIX "") + +## Add cmake target dependencies of the executable +## same as for the library above +# add_dependencies(${PROJECT_NAME}_node ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) + +## Specify libraries to link a library or executable target against +# target_link_libraries(${PROJECT_NAME}_node +# ${catkin_LIBRARIES} +# ) + +############# +## Install ## +############# + +# all install targets should use catkin DESTINATION variables +# See http://ros.org/doc/api/catkin/html/adv_user_guide/variables.html + +## Mark executable scripts (Python etc.) for installation +## in contrast to setup.py, you can choose the destination +# catkin_install_python(PROGRAMS +# scripts/my_python_script +# DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} +# ) + +## Mark executables for installation +## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_executables.html +# install(TARGETS ${PROJECT_NAME}_node +# RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} +# ) + +## Mark libraries for installation +## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_libraries.html +# install(TARGETS ${PROJECT_NAME} +# ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} +# LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} +# RUNTIME DESTINATION ${CATKIN_GLOBAL_BIN_DESTINATION} +# ) + +## Mark cpp header files for installation +# install(DIRECTORY include/${PROJECT_NAME}/ +# DESTINATION ${CATKIN_PACKAGE_INCLUDE_DESTINATION} +# FILES_MATCHING PATTERN "*.h" +# PATTERN ".svn" EXCLUDE +# ) + +## Mark other files for installation (e.g. launch and bag files, etc.) +# install(FILES +# # myfile1 +# # myfile2 +# DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION} +# ) + +############# +## Testing ## +############# + +## Add gtest based cpp test target and link libraries +# catkin_add_gtest(${PROJECT_NAME}-test test/test_midas_cpp.cpp) +# if(TARGET ${PROJECT_NAME}-test) +# target_link_libraries(${PROJECT_NAME}-test ${PROJECT_NAME}) +# endif() + +## Add folders to be run by python nosetests +# catkin_add_nosetests(test) + +install(TARGETS ${PROJECT_NAME} + ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} + LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} + RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} +) + +add_custom_command( + TARGET midas_cpp POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/midas_cpp + ${CMAKE_SOURCE_DIR}/midas_cpp +) \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/launch/midas_cpp.launch b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/launch/midas_cpp.launch new file mode 100644 index 0000000000000000000000000000000000000000..88e86f42f668e76ad4976ec6794a8cb0f20cac65 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/launch/midas_cpp.launch @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/launch/midas_talker_listener.launch b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/launch/midas_talker_listener.launch new file mode 100644 index 0000000000000000000000000000000000000000..8817a4f4933c56986fe0edc0886b2fded3d3406d --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/launch/midas_talker_listener.launch @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/package.xml b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/package.xml new file mode 100644 index 0000000000000000000000000000000000000000..9cac90eba75409bd170f73531c54c83c52ff047a --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/package.xml @@ -0,0 +1,77 @@ + + + midas_cpp + 0.1.0 + The midas_cpp package + + Alexey Bochkovskiy + MIT + https://github.com/isl-org/MiDaS/tree/master/ros + + + + + + + TODO + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + catkin + cv_bridge + image_transport + roscpp + rospy + sensor_msgs + std_msgs + cv_bridge + image_transport + roscpp + rospy + sensor_msgs + std_msgs + cv_bridge + image_transport + roscpp + rospy + sensor_msgs + std_msgs + + + + + + + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/scripts/listener.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/scripts/listener.py new file mode 100644 index 0000000000000000000000000000000000000000..6927ea7a83ac9309e5f883ee974a5dcfa8a2aa3b --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/scripts/listener.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +from __future__ import print_function + +import roslib +#roslib.load_manifest('my_package') +import sys +import rospy +import cv2 +import numpy as np +from std_msgs.msg import String +from sensor_msgs.msg import Image +from cv_bridge import CvBridge, CvBridgeError + +class video_show: + + def __init__(self): + self.show_output = rospy.get_param('~show_output', True) + self.save_output = rospy.get_param('~save_output', False) + self.output_video_file = rospy.get_param('~output_video_file','result.mp4') + # rospy.loginfo(f"Listener - params: show_output={self.show_output}, save_output={self.save_output}, output_video_file={self.output_video_file}") + + self.bridge = CvBridge() + self.image_sub = rospy.Subscriber("midas_topic", Image, self.callback) + + def callback(self, data): + try: + cv_image = self.bridge.imgmsg_to_cv2(data) + except CvBridgeError as e: + print(e) + return + + if cv_image.size == 0: + return + + rospy.loginfo("Listener: Received new frame") + cv_image = cv_image.astype("uint8") + + if self.show_output==True: + cv2.imshow("video_show", cv_image) + cv2.waitKey(10) + + if self.save_output==True: + if self.video_writer_init==False: + fourcc = cv2.VideoWriter_fourcc(*'XVID') + self.out = cv2.VideoWriter(self.output_video_file, fourcc, 25, (cv_image.shape[1], cv_image.shape[0])) + + self.out.write(cv_image) + + + +def main(args): + rospy.init_node('listener', anonymous=True) + ic = video_show() + try: + rospy.spin() + except KeyboardInterrupt: + print("Shutting down") + cv2.destroyAllWindows() + +if __name__ == '__main__': + main(sys.argv) \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/scripts/listener_original.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/scripts/listener_original.py new file mode 100644 index 0000000000000000000000000000000000000000..20e235f6958d644b89383752ab18e9e2275f55e5 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/scripts/listener_original.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +from __future__ import print_function + +import roslib +#roslib.load_manifest('my_package') +import sys +import rospy +import cv2 +import numpy as np +from std_msgs.msg import String +from sensor_msgs.msg import Image +from cv_bridge import CvBridge, CvBridgeError + +class video_show: + + def __init__(self): + self.show_output = rospy.get_param('~show_output', True) + self.save_output = rospy.get_param('~save_output', False) + self.output_video_file = rospy.get_param('~output_video_file','result.mp4') + # rospy.loginfo(f"Listener original - params: show_output={self.show_output}, save_output={self.save_output}, output_video_file={self.output_video_file}") + + self.bridge = CvBridge() + self.image_sub = rospy.Subscriber("image_topic", Image, self.callback) + + def callback(self, data): + try: + cv_image = self.bridge.imgmsg_to_cv2(data) + except CvBridgeError as e: + print(e) + return + + if cv_image.size == 0: + return + + rospy.loginfo("Listener_original: Received new frame") + cv_image = cv_image.astype("uint8") + + if self.show_output==True: + cv2.imshow("video_show_orig", cv_image) + cv2.waitKey(10) + + if self.save_output==True: + if self.video_writer_init==False: + fourcc = cv2.VideoWriter_fourcc(*'XVID') + self.out = cv2.VideoWriter(self.output_video_file, fourcc, 25, (cv_image.shape[1], cv_image.shape[0])) + + self.out.write(cv_image) + + + +def main(args): + rospy.init_node('listener_original', anonymous=True) + ic = video_show() + try: + rospy.spin() + except KeyboardInterrupt: + print("Shutting down") + cv2.destroyAllWindows() + +if __name__ == '__main__': + main(sys.argv) \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/scripts/talker.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/scripts/talker.py new file mode 100644 index 0000000000000000000000000000000000000000..8219cc8632484a2efd02984347c615efad6b78b2 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/scripts/talker.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + + +import roslib +#roslib.load_manifest('my_package') +import sys +import rospy +import cv2 +from std_msgs.msg import String +from sensor_msgs.msg import Image +from cv_bridge import CvBridge, CvBridgeError + + +def talker(): + rospy.init_node('talker', anonymous=True) + + use_camera = rospy.get_param('~use_camera', False) + input_video_file = rospy.get_param('~input_video_file','test.mp4') + # rospy.loginfo(f"Talker - params: use_camera={use_camera}, input_video_file={input_video_file}") + + # rospy.loginfo("Talker: Trying to open a video stream") + if use_camera == True: + cap = cv2.VideoCapture(0) + else: + cap = cv2.VideoCapture(input_video_file) + + pub = rospy.Publisher('image_topic', Image, queue_size=1) + rate = rospy.Rate(30) # 30hz + bridge = CvBridge() + + while not rospy.is_shutdown(): + ret, cv_image = cap.read() + if ret==False: + print("Talker: Video is over") + rospy.loginfo("Video is over") + return + + try: + image = bridge.cv2_to_imgmsg(cv_image, "bgr8") + except CvBridgeError as e: + rospy.logerr("Talker: cv2image conversion failed: ", e) + print(e) + continue + + rospy.loginfo("Talker: Publishing frame") + pub.publish(image) + rate.sleep() + +if __name__ == '__main__': + try: + talker() + except rospy.ROSInterruptException: + pass diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/src/main.cpp b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/src/main.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e4fc72c6955f66af71c9cb1fc7a7b1f643129685 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/src/main.cpp @@ -0,0 +1,285 @@ +#include +#include +#include +#include + +#include + +#include // One-stop header. + +#include +#include +#include +#include + +#include +#include + +// includes for OpenCV >= 3.x +#ifndef CV_VERSION_EPOCH +#include +#include +#include +#endif + +// OpenCV includes for OpenCV 2.x +#ifdef CV_VERSION_EPOCH +#include +#include +#include +#include +#endif + +static const std::string OPENCV_WINDOW = "Image window"; + +class Midas +{ + ros::NodeHandle nh_; + image_transport::ImageTransport it_; + image_transport::Subscriber image_sub_; + image_transport::Publisher image_pub_; + + torch::jit::script::Module module; + torch::Device device; + + auto ToTensor(cv::Mat img, bool show_output = false, bool unsqueeze = false, int unsqueeze_dim = 0) + { + //std::cout << "image shape: " << img.size() << std::endl; + at::Tensor tensor_image = torch::from_blob(img.data, { img.rows, img.cols, 3 }, at::kByte); + + if (unsqueeze) + { + tensor_image.unsqueeze_(unsqueeze_dim); + //std::cout << "tensors new shape: " << tensor_image.sizes() << std::endl; + } + + if (show_output) + { + std::cout << tensor_image.slice(2, 0, 1) << std::endl; + } + //std::cout << "tenor shape: " << tensor_image.sizes() << std::endl; + return tensor_image; + } + + auto ToInput(at::Tensor tensor_image) + { + // Create a vector of inputs. + return std::vector{tensor_image}; + } + + auto ToCvImage(at::Tensor tensor, int cv_type = CV_8UC3) + { + int width = tensor.sizes()[0]; + int height = tensor.sizes()[1]; + try + { + cv::Mat output_mat; + if (cv_type == CV_8UC4 || cv_type == CV_8UC3 || cv_type == CV_8UC2 || cv_type == CV_8UC1) { + cv::Mat cv_image(cv::Size{ height, width }, cv_type, tensor.data_ptr()); + output_mat = cv_image; + } + else if (cv_type == CV_32FC4 || cv_type == CV_32FC3 || cv_type == CV_32FC2 || cv_type == CV_32FC1) { + cv::Mat cv_image(cv::Size{ height, width }, cv_type, tensor.data_ptr()); + output_mat = cv_image; + } + else if (cv_type == CV_64FC4 || cv_type == CV_64FC3 || cv_type == CV_64FC2 || cv_type == CV_64FC1) { + cv::Mat cv_image(cv::Size{ height, width }, cv_type, tensor.data_ptr()); + output_mat = cv_image; + } + + //show_image(output_mat, "converted image from tensor"); + return output_mat.clone(); + } + catch (const c10::Error& e) + { + std::cout << "an error has occured : " << e.msg() << std::endl; + } + return cv::Mat(height, width, CV_8UC3); + } + + std::string input_topic, output_topic, model_name; + bool out_orig_size; + int net_width, net_height; + torch::NoGradGuard guard; + at::Tensor mean, std; + at::Tensor output, tensor; + +public: + Midas() + : nh_(), it_(nh_), device(torch::Device(torch::kCPU)) + { + ros::param::param("~input_topic", input_topic, "image_topic"); + ros::param::param("~output_topic", output_topic, "midas_topic"); + ros::param::param("~model_name", model_name, "model-small-traced.pt"); + ros::param::param("~out_orig_size", out_orig_size, true); + ros::param::param("~net_width", net_width, 256); + ros::param::param("~net_height", net_height, 256); + + std::cout << ", input_topic = " << input_topic << + ", output_topic = " << output_topic << + ", model_name = " << model_name << + ", out_orig_size = " << out_orig_size << + ", net_width = " << net_width << + ", net_height = " << net_height << + std::endl; + + // Subscrive to input video feed and publish output video feed + image_sub_ = it_.subscribe(input_topic, 1, &Midas::imageCb, this); + image_pub_ = it_.advertise(output_topic, 1); + + std::cout << "Try to load torchscript model \n"; + + try { + // Deserialize the ScriptModule from a file using torch::jit::load(). + module = torch::jit::load(model_name); + } + catch (const c10::Error& e) { + std::cerr << "error loading the model\n"; + exit(0); + } + + std::cout << "ok\n"; + + try { + module.eval(); + torch::jit::getProfilingMode() = false; + torch::jit::setGraphExecutorOptimize(true); + + mean = torch::tensor({ 0.485, 0.456, 0.406 }); + std = torch::tensor({ 0.229, 0.224, 0.225 }); + + if (torch::hasCUDA()) { + std::cout << "cuda is available" << std::endl; + at::globalContext().setBenchmarkCuDNN(true); + device = torch::Device(torch::kCUDA); + module.to(device); + mean = mean.to(device); + std = std.to(device); + } + } + catch (const c10::Error& e) + { + std::cerr << " module initialization: " << e.msg() << std::endl; + } + } + + ~Midas() + { + } + + void imageCb(const sensor_msgs::ImageConstPtr& msg) + { + cv_bridge::CvImagePtr cv_ptr; + try + { + // sensor_msgs::Image to cv::Mat + cv_ptr = cv_bridge::toCvCopy(msg, sensor_msgs::image_encodings::RGB8); + } + catch (cv_bridge::Exception& e) + { + ROS_ERROR("cv_bridge exception: %s", e.what()); + return; + } + + // pre-processing + auto tensor_cpu = ToTensor(cv_ptr->image); // OpenCV-image -> Libtorch-tensor + + try { + tensor = tensor_cpu.to(device); // move to device (CPU or GPU) + + tensor = tensor.toType(c10::kFloat); + tensor = tensor.permute({ 2, 0, 1 }); // HWC -> CHW + tensor = tensor.unsqueeze(0); + tensor = at::upsample_bilinear2d(tensor, { net_height, net_width }, true); // resize + tensor = tensor.squeeze(0); + tensor = tensor.permute({ 1, 2, 0 }); // CHW -> HWC + + tensor = tensor.div(255).sub(mean).div(std); // normalization + tensor = tensor.permute({ 2, 0, 1 }); // HWC -> CHW + tensor.unsqueeze_(0); // CHW -> NCHW + } + catch (const c10::Error& e) + { + std::cerr << " pre-processing exception: " << e.msg() << std::endl; + return; + } + + auto input_to_net = ToInput(tensor); // input to the network + + // inference + output; + try { + output = module.forward(input_to_net).toTensor(); // run inference + } + catch (const c10::Error& e) + { + std::cerr << " module.forward() exception: " << e.msg() << std::endl; + return; + } + + output = output.detach().to(torch::kF32); + + // move to CPU temporary + at::Tensor output_tmp = output; + output_tmp = output_tmp.to(torch::kCPU); + + // normalization + float min_val = std::numeric_limits::max(); + float max_val = std::numeric_limits::min(); + + for (int i = 0; i < net_width * net_height; ++i) { + float val = output_tmp.data_ptr()[i]; + if (min_val > val) min_val = val; + if (max_val < val) max_val = val; + } + float range_val = max_val - min_val; + + output = output.sub(min_val).div(range_val).mul(255.0F).clamp(0, 255).to(torch::kF32); // .to(torch::kU8); + + // resize to the original size if required + if (out_orig_size) { + try { + output = at::upsample_bilinear2d(output.unsqueeze(0), { cv_ptr->image.size().height, cv_ptr->image.size().width }, true); + output = output.squeeze(0); + } + catch (const c10::Error& e) + { + std::cout << " upsample_bilinear2d() exception: " << e.msg() << std::endl; + return; + } + } + output = output.permute({ 1, 2, 0 }).to(torch::kCPU); + + int cv_type = CV_32FC1; // CV_8UC1; + auto cv_img = ToCvImage(output, cv_type); + + sensor_msgs::Image img_msg; + + try { + // cv::Mat -> sensor_msgs::Image + std_msgs::Header header; // empty header + header.seq = 0; // user defined counter + header.stamp = ros::Time::now();// time + //cv_bridge::CvImage img_bridge = cv_bridge::CvImage(header, sensor_msgs::image_encodings::MONO8, cv_img); + cv_bridge::CvImage img_bridge = cv_bridge::CvImage(header, sensor_msgs::image_encodings::TYPE_32FC1, cv_img); + + img_bridge.toImageMsg(img_msg); // cv_bridge -> sensor_msgs::Image + } + catch (cv_bridge::Exception& e) + { + ROS_ERROR("cv_bridge exception: %s", e.what()); + return; + } + + // Output modified video stream + image_pub_.publish(img_msg); + } +}; + +int main(int argc, char** argv) +{ + ros::init(argc, argv, "midas", ros::init_options::AnonymousName); + Midas ic; + ros::spin(); + return 0; +} \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/run_talker_listener_test.sh b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/run_talker_listener_test.sh new file mode 100644 index 0000000000000000000000000000000000000000..a997c4261072d0d627598fe06a723fcc7522d347 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/run_talker_listener_test.sh @@ -0,0 +1,16 @@ +# place any test.mp4 file near with this file + +# roscore +# rosnode kill -a + +source ~/catkin_ws/devel/setup.bash + +roscore & +P1=$! +rosrun midas_cpp talker.py & +P2=$! +rosrun midas_cpp listener_original.py & +P3=$! +rosrun midas_cpp listener.py & +P4=$! +wait $P1 $P2 $P3 $P4 \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/run.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/run.py new file mode 100644 index 0000000000000000000000000000000000000000..53194ed1137ef2eaf63e8a83b51a5c8fde24aea4 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/run.py @@ -0,0 +1,278 @@ +"""Compute depth maps for images in the input folder. +""" +import os +import glob +import torch +import utils +import cv2 +import argparse +import time + +import numpy as np + +from imutils.video import VideoStream +from midas.model_loader import default_models, load_model + +first_execution = True +def process(device, model, model_type, image, input_size, target_size, optimize, use_camera): + """ + Run the inference and interpolate. + + Args: + device (torch.device): the torch device used + model: the model used for inference + model_type: the type of the model + image: the image fed into the neural network + input_size: the size (width, height) of the neural network input (for OpenVINO) + target_size: the size (width, height) the neural network output is interpolated to + optimize: optimize the model to half-floats on CUDA? + use_camera: is the camera used? + + Returns: + the prediction + """ + global first_execution + + if "openvino" in model_type: + if first_execution or not use_camera: + print(f" Input resized to {input_size[0]}x{input_size[1]} before entering the encoder") + first_execution = False + + sample = [np.reshape(image, (1, 3, *input_size))] + prediction = model(sample)[model.output(0)][0] + prediction = cv2.resize(prediction, dsize=target_size, + interpolation=cv2.INTER_CUBIC) + else: + sample = torch.from_numpy(image).to(device).unsqueeze(0) + + if optimize and device == torch.device("cuda"): + if first_execution: + print(" Optimization to half-floats activated. Use with caution, because models like Swin require\n" + " float precision to work properly and may yield non-finite depth values to some extent for\n" + " half-floats.") + sample = sample.to(memory_format=torch.channels_last) + sample = sample.half() + + if first_execution or not use_camera: + height, width = sample.shape[2:] + print(f" Input resized to {width}x{height} before entering the encoder") + first_execution = False + + prediction = model.forward(sample) + prediction = ( + torch.nn.functional.interpolate( + prediction.unsqueeze(1), + size=target_size[::-1], + mode="bicubic", + align_corners=False, + ) + .squeeze() + .cpu() + .numpy() + ) + + return prediction + + +def create_side_by_side(image, depth, grayscale): + """ + Take an RGB image and depth map and place them side by side. This includes a proper normalization of the depth map + for better visibility. + + Args: + image: the RGB image + depth: the depth map + grayscale: use a grayscale colormap? + + Returns: + the image and depth map place side by side + """ + depth_min = depth.min() + depth_max = depth.max() + normalized_depth = 255 * (depth - depth_min) / (depth_max - depth_min) + normalized_depth *= 3 + + right_side = np.repeat(np.expand_dims(normalized_depth, 2), 3, axis=2) / 3 + if not grayscale: + right_side = cv2.applyColorMap(np.uint8(right_side), cv2.COLORMAP_INFERNO) + + if image is None: + return right_side + else: + return np.concatenate((image, right_side), axis=1) + + +def run(input_path, output_path, model_path, model_type="dpt_beit_large_512", optimize=False, side=False, height=None, + square=False, grayscale=False): + """Run MonoDepthNN to compute depth maps. + + Args: + input_path (str): path to input folder + output_path (str): path to output folder + model_path (str): path to saved model + model_type (str): the model type + optimize (bool): optimize the model to half-floats on CUDA? + side (bool): RGB and depth side by side in output images? + height (int): inference encoder image height + square (bool): resize to a square resolution? + grayscale (bool): use a grayscale colormap? + """ + print("Initialize") + + # select device +# device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + device = torch.device("cpu") + print("Device: %s" % device) + + model, transform, net_w, net_h = load_model(device, model_path, model_type, optimize, height, square) + + # get input + if input_path is not None: + image_names = glob.glob(os.path.join(input_path, "*")) + num_images = len(image_names) + else: + print("No input path specified. Grabbing images from camera.") + + # create output folder + if output_path is not None: + os.makedirs(output_path, exist_ok=True) + + print("Start processing") + + if input_path is not None: + if output_path is None: + print("Warning: No output path specified. Images will be processed but not shown or stored anywhere.") + for index, image_name in enumerate(image_names): + + print(" Processing {} ({}/{})".format(image_name, index + 1, num_images)) + + # input + original_image_rgb = utils.read_image(image_name) # in [0, 1] + image = transform({"image": original_image_rgb})["image"] + + # compute + with torch.no_grad(): + prediction = process(device, model, model_type, image, (net_w, net_h), original_image_rgb.shape[1::-1], + optimize, False) + + # output + if output_path is not None: + filename = os.path.join( + output_path, os.path.splitext(os.path.basename(image_name))[0] + '-' + model_type + ) + if not side: + utils.write_depth(filename, prediction, grayscale, bits=2) + else: + original_image_bgr = np.flip(original_image_rgb, 2) + content = create_side_by_side(original_image_bgr*255, prediction, grayscale) + cv2.imwrite(filename + ".png", content) + utils.write_pfm(filename + ".pfm", prediction.astype(np.float32)) + + else: + with torch.no_grad(): + fps = 1 + video = VideoStream(0).start() + time_start = time.time() + frame_index = 0 + while True: + frame = video.read() + if frame is not None: + original_image_rgb = np.flip(frame, 2) # in [0, 255] (flip required to get RGB) + image = transform({"image": original_image_rgb/255})["image"] + + prediction = process(device, model, model_type, image, (net_w, net_h), + original_image_rgb.shape[1::-1], optimize, True) + + original_image_bgr = np.flip(original_image_rgb, 2) if side else None + content = create_side_by_side(original_image_bgr, prediction, grayscale) + cv2.imshow('MiDaS Depth Estimation - Press Escape to close window ', content/255) + + if output_path is not None: + filename = os.path.join(output_path, 'Camera' + '-' + model_type + '_' + str(frame_index)) + cv2.imwrite(filename + ".png", content) + + alpha = 0.1 + if time.time()-time_start > 0: + fps = (1 - alpha) * fps + alpha * 1 / (time.time()-time_start) # exponential moving average + time_start = time.time() + print(f"\rFPS: {round(fps,2)}", end="") + + if cv2.waitKey(1) == 27: # Escape key + break + + frame_index += 1 + print() + + print("Finished") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument('-i', '--input_path', + default=None, + help='Folder with input images (if no input path is specified, images are tried to be grabbed ' + 'from camera)' + ) + + parser.add_argument('-o', '--output_path', + default=None, + help='Folder for output images' + ) + + parser.add_argument('-m', '--model_weights', + default=None, + help='Path to the trained weights of model' + ) + + parser.add_argument('-t', '--model_type', + default='dpt_beit_large_512', + help='Model type: ' + 'dpt_beit_large_512, dpt_beit_large_384, dpt_beit_base_384, dpt_swin2_large_384, ' + 'dpt_swin2_base_384, dpt_swin2_tiny_256, dpt_swin_large_384, dpt_next_vit_large_384, ' + 'dpt_levit_224, dpt_large_384, dpt_hybrid_384, midas_v21_384, midas_v21_small_256 or ' + 'openvino_midas_v21_small_256' + ) + + parser.add_argument('-s', '--side', + action='store_true', + help='Output images contain RGB and depth images side by side' + ) + + parser.add_argument('--optimize', dest='optimize', action='store_true', help='Use half-float optimization') + parser.set_defaults(optimize=False) + + parser.add_argument('--height', + type=int, default=None, + help='Preferred height of images feed into the encoder during inference. Note that the ' + 'preferred height may differ from the actual height, because an alignment to multiples of ' + '32 takes place. Many models support only the height chosen during training, which is ' + 'used automatically if this parameter is not set.' + ) + parser.add_argument('--square', + action='store_true', + help='Option to resize images to a square resolution by changing their widths when images are ' + 'fed into the encoder during inference. If this parameter is not set, the aspect ratio of ' + 'images is tried to be preserved if supported by the model.' + ) + parser.add_argument('--grayscale', + action='store_true', + help='Use a grayscale colormap instead of the inferno one. Although the inferno colormap, ' + 'which is used by default, is better for visibility, it does not allow storing 16-bit ' + 'depth values in PNGs but only 8-bit ones due to the precision limitation of this ' + 'colormap.' + ) + + args = parser.parse_args() + + + if args.model_weights is None: + args.model_weights = default_models[args.model_type] + + # set torch options + torch.backends.cudnn.enabled = True + torch.backends.cudnn.benchmark = True + + # compute depth maps + run(args.input_path, args.output_path, args.model_weights, args.model_type, args.optimize, args.side, args.height, + args.square, args.grayscale) diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/README.md b/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/README.md new file mode 100644 index 0000000000000000000000000000000000000000..5b5fe0e63668eab45a55b140826cb3762862b17c --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/README.md @@ -0,0 +1,147 @@ +## Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer + +### TensorFlow inference using `.pb` and `.onnx` models + +1. [Run inference on TensorFlow-model by using TensorFlow](#run-inference-on-tensorflow-model-by-using-tensorFlow) + +2. [Run inference on ONNX-model by using TensorFlow](#run-inference-on-onnx-model-by-using-tensorflow) + +3. [Make ONNX model from downloaded Pytorch model file](#make-onnx-model-from-downloaded-pytorch-model-file) + + +### Run inference on TensorFlow-model by using TensorFlow + +1) Download the model weights [model-f6b98070.pb](https://github.com/isl-org/MiDaS/releases/download/v2_1/model-f6b98070.pb) +and [model-small.pb](https://github.com/isl-org/MiDaS/releases/download/v2_1/model-small.pb) and place the +file in the `/tf/` folder. + +2) Set up dependencies: + +```shell +# install OpenCV +pip install --upgrade pip +pip install opencv-python + +# install TensorFlow +pip install -I grpcio tensorflow==2.3.0 tensorflow-addons==0.11.2 numpy==1.18.0 +``` + +#### Usage + +1) Place one or more input images in the folder `tf/input`. + +2) Run the model: + + ```shell + python tf/run_pb.py + ``` + + Or run the small model: + + ```shell + python tf/run_pb.py --model_weights model-small.pb --model_type small + ``` + +3) The resulting inverse depth maps are written to the `tf/output` folder. + + +### Run inference on ONNX-model by using ONNX-Runtime + +1) Download the model weights [model-f6b98070.onnx](https://github.com/isl-org/MiDaS/releases/download/v2_1/model-f6b98070.onnx) +and [model-small.onnx](https://github.com/isl-org/MiDaS/releases/download/v2_1/model-small.onnx) and place the +file in the `/tf/` folder. + +2) Set up dependencies: + +```shell +# install OpenCV +pip install --upgrade pip +pip install opencv-python + +# install ONNX +pip install onnx==1.7.0 + +# install ONNX Runtime +pip install onnxruntime==1.5.2 +``` + +#### Usage + +1) Place one or more input images in the folder `tf/input`. + +2) Run the model: + + ```shell + python tf/run_onnx.py + ``` + + Or run the small model: + + ```shell + python tf/run_onnx.py --model_weights model-small.onnx --model_type small + ``` + +3) The resulting inverse depth maps are written to the `tf/output` folder. + + + +### Make ONNX model from downloaded Pytorch model file + +1) Download the model weights [model-f6b98070.pt](https://github.com/isl-org/MiDaS/releases/download/v2_1/model-f6b98070.pt) and place the +file in the root folder. + +2) Set up dependencies: + +```shell +# install OpenCV +pip install --upgrade pip +pip install opencv-python + +# install PyTorch TorchVision +pip install -I torch==1.7.0 torchvision==0.8.0 + +# install TensorFlow +pip install -I grpcio tensorflow==2.3.0 tensorflow-addons==0.11.2 numpy==1.18.0 + +# install ONNX +pip install onnx==1.7.0 + +# install ONNX-TensorFlow +git clone https://github.com/onnx/onnx-tensorflow.git +cd onnx-tensorflow +git checkout 095b51b88e35c4001d70f15f80f31014b592b81e +pip install -e . +``` + +#### Usage + +1) Run the converter: + + ```shell + python tf/make_onnx_model.py + ``` + +2) The resulting `model-f6b98070.onnx` file is written to the `/tf/` folder. + + +### Requirements + + The code was tested with Python 3.6.9, PyTorch 1.5.1, TensorFlow 2.2.0, TensorFlow-addons 0.8.3, ONNX 1.7.0, ONNX-TensorFlow (GitHub-master-17.07.2020) and OpenCV 4.3.0. + +### Citation + +Please cite our paper if you use this code or any of the models: +``` +@article{Ranftl2019, + author = {Ren\'{e} Ranftl and Katrin Lasinger and David Hafner and Konrad Schindler and Vladlen Koltun}, + title = {Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer}, + journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (TPAMI)}, + year = {2020}, +} +``` + +### License + +MIT License + + diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/make_onnx_model.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/make_onnx_model.py new file mode 100644 index 0000000000000000000000000000000000000000..d14b0e4e1d2ea70fa315fd7ca7dfd72440a19376 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/make_onnx_model.py @@ -0,0 +1,112 @@ +"""Compute depth maps for images in the input folder. +""" +import os +import ntpath +import glob +import torch +import utils +import cv2 +import numpy as np +from torchvision.transforms import Compose, Normalize +from torchvision import transforms + +from shutil import copyfile +import fileinput +import sys +sys.path.append(os.getcwd() + '/..') + +def modify_file(): + modify_filename = '../midas/blocks.py' + copyfile(modify_filename, modify_filename+'.bak') + + with open(modify_filename, 'r') as file : + filedata = file.read() + + filedata = filedata.replace('align_corners=True', 'align_corners=False') + filedata = filedata.replace('import torch.nn as nn', 'import torch.nn as nn\nimport torchvision.models as models') + filedata = filedata.replace('torch.hub.load("facebookresearch/WSL-Images", "resnext101_32x8d_wsl")', 'models.resnext101_32x8d()') + + with open(modify_filename, 'w') as file: + file.write(filedata) + +def restore_file(): + modify_filename = '../midas/blocks.py' + copyfile(modify_filename+'.bak', modify_filename) + +modify_file() + +from midas.midas_net import MidasNet +from midas.transforms import Resize, NormalizeImage, PrepareForNet + +restore_file() + + +class MidasNet_preprocessing(MidasNet): + """Network for monocular depth estimation. + """ + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input data (image) + + Returns: + tensor: depth + """ + + mean = torch.tensor([0.485, 0.456, 0.406]) + std = torch.tensor([0.229, 0.224, 0.225]) + x.sub_(mean[None, :, None, None]).div_(std[None, :, None, None]) + + return MidasNet.forward(self, x) + + +def run(model_path): + """Run MonoDepthNN to compute depth maps. + + Args: + model_path (str): path to saved model + """ + print("initialize") + + # select device + + # load network + #model = MidasNet(model_path, non_negative=True) + model = MidasNet_preprocessing(model_path, non_negative=True) + + model.eval() + + print("start processing") + + # input + img_input = np.zeros((3, 384, 384), np.float32) + + # compute + with torch.no_grad(): + sample = torch.from_numpy(img_input).unsqueeze(0) + prediction = model.forward(sample) + prediction = ( + torch.nn.functional.interpolate( + prediction.unsqueeze(1), + size=img_input.shape[:2], + mode="bicubic", + align_corners=False, + ) + .squeeze() + .cpu() + .numpy() + ) + + torch.onnx.export(model, sample, ntpath.basename(model_path).rsplit('.', 1)[0]+'.onnx', opset_version=9) + + print("finished") + + +if __name__ == "__main__": + # set paths + # MODEL_PATH = "model.pt" + MODEL_PATH = "../model-f6b98070.pt" + + # compute depth maps + run(MODEL_PATH) diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/run_onnx.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/run_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..8edc754721136c1eb8ef634aa033718114bae00a --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/run_onnx.py @@ -0,0 +1,119 @@ +"""Compute depth maps for images in the input folder. +""" +import os +import glob +import utils +import cv2 +import sys +import numpy as np +import argparse + +import onnx +import onnxruntime as rt + +from transforms import Resize, NormalizeImage, PrepareForNet + + +def run(input_path, output_path, model_path, model_type="large"): + """Run MonoDepthNN to compute depth maps. + + Args: + input_path (str): path to input folder + output_path (str): path to output folder + model_path (str): path to saved model + """ + print("initialize") + + # select device +# device = "CUDA:0" + device = "CPU" + print("device: %s" % device) + + # network resolution + if model_type == "large": + net_w, net_h = 384, 384 + elif model_type == "small": + net_w, net_h = 256, 256 + else: + print(f"model_type '{model_type}' not implemented, use: --model_type large") + assert False + + # load network + print("loading model...") + model = rt.InferenceSession(model_path) + input_name = model.get_inputs()[0].name + output_name = model.get_outputs()[0].name + + resize_image = Resize( + net_w, + net_h, + resize_target=None, + keep_aspect_ratio=False, + ensure_multiple_of=32, + resize_method="upper_bound", + image_interpolation_method=cv2.INTER_CUBIC, + ) + + def compose2(f1, f2): + return lambda x: f2(f1(x)) + + transform = compose2(resize_image, PrepareForNet()) + + # get input + img_names = glob.glob(os.path.join(input_path, "*")) + num_images = len(img_names) + + # create output folder + os.makedirs(output_path, exist_ok=True) + + print("start processing") + + for ind, img_name in enumerate(img_names): + + print(" processing {} ({}/{})".format(img_name, ind + 1, num_images)) + + # input + img = utils.read_image(img_name) + img_input = transform({"image": img})["image"] + + # compute + output = model.run([output_name], {input_name: img_input.reshape(1, 3, net_h, net_w).astype(np.float32)})[0] + prediction = np.array(output).reshape(net_h, net_w) + prediction = cv2.resize(prediction, (img.shape[1], img.shape[0]), interpolation=cv2.INTER_CUBIC) + + # output + filename = os.path.join( + output_path, os.path.splitext(os.path.basename(img_name))[0] + ) + utils.write_depth(filename, prediction, bits=2) + + print("finished") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument('-i', '--input_path', + default='input', + help='folder with input images' + ) + + parser.add_argument('-o', '--output_path', + default='output', + help='folder for output images' + ) + + parser.add_argument('-m', '--model_weights', + default='model-f6b98070.onnx', + help='path to the trained weights of model' + ) + + parser.add_argument('-t', '--model_type', + default='large', + help='model type: large or small' + ) + + args = parser.parse_args() + + # compute depth maps + run(args.input_path, args.output_path, args.model_weights, args.model_type) diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/run_pb.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/run_pb.py new file mode 100644 index 0000000000000000000000000000000000000000..e46254f7b37f72e7d87672d70fd4b2f393ad7658 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/run_pb.py @@ -0,0 +1,135 @@ +"""Compute depth maps for images in the input folder. +""" +import os +import glob +import utils +import cv2 +import argparse + +import tensorflow as tf + +from transforms import Resize, NormalizeImage, PrepareForNet + +def run(input_path, output_path, model_path, model_type="large"): + """Run MonoDepthNN to compute depth maps. + + Args: + input_path (str): path to input folder + output_path (str): path to output folder + model_path (str): path to saved model + """ + print("initialize") + + # the runtime initialization will not allocate all memory on the device to avoid out of GPU memory + gpus = tf.config.experimental.list_physical_devices('GPU') + if gpus: + try: + for gpu in gpus: + #tf.config.experimental.set_memory_growth(gpu, True) + tf.config.experimental.set_virtual_device_configuration(gpu, + [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=4000)]) + except RuntimeError as e: + print(e) + + # network resolution + if model_type == "large": + net_w, net_h = 384, 384 + elif model_type == "small": + net_w, net_h = 256, 256 + else: + print(f"model_type '{model_type}' not implemented, use: --model_type large") + assert False + + # load network + graph_def = tf.compat.v1.GraphDef() + with tf.io.gfile.GFile(model_path, 'rb') as f: + graph_def.ParseFromString(f.read()) + tf.import_graph_def(graph_def, name='') + + + model_operations = tf.compat.v1.get_default_graph().get_operations() + input_node = '0:0' + output_layer = model_operations[len(model_operations) - 1].name + ':0' + print("Last layer name: ", output_layer) + + resize_image = Resize( + net_w, + net_h, + resize_target=None, + keep_aspect_ratio=False, + ensure_multiple_of=32, + resize_method="upper_bound", + image_interpolation_method=cv2.INTER_CUBIC, + ) + + def compose2(f1, f2): + return lambda x: f2(f1(x)) + + transform = compose2(resize_image, PrepareForNet()) + + # get input + img_names = glob.glob(os.path.join(input_path, "*")) + num_images = len(img_names) + + # create output folder + os.makedirs(output_path, exist_ok=True) + + print("start processing") + + with tf.compat.v1.Session() as sess: + try: + # load images + for ind, img_name in enumerate(img_names): + + print(" processing {} ({}/{})".format(img_name, ind + 1, num_images)) + + # input + img = utils.read_image(img_name) + img_input = transform({"image": img})["image"] + + # compute + prob_tensor = sess.graph.get_tensor_by_name(output_layer) + prediction, = sess.run(prob_tensor, {input_node: [img_input] }) + prediction = prediction.reshape(net_h, net_w) + prediction = cv2.resize(prediction, (img.shape[1], img.shape[0]), interpolation=cv2.INTER_CUBIC) + + # output + filename = os.path.join( + output_path, os.path.splitext(os.path.basename(img_name))[0] + ) + utils.write_depth(filename, prediction, bits=2) + + except KeyError: + print ("Couldn't find input node: ' + input_node + ' or output layer: " + output_layer + ".") + exit(-1) + + print("finished") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument('-i', '--input_path', + default='input', + help='folder with input images' + ) + + parser.add_argument('-o', '--output_path', + default='output', + help='folder for output images' + ) + + parser.add_argument('-m', '--model_weights', + default='model-f6b98070.pb', + help='path to the trained weights of model' + ) + + parser.add_argument('-t', '--model_type', + default='large', + help='model type: large or small' + ) + + args = parser.parse_args() + + # compute depth maps + run(args.input_path, args.output_path, args.model_weights, args.model_type) diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/transforms.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..350cbc11662633ad7f8968eb10be2e7de6e384e9 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/transforms.py @@ -0,0 +1,234 @@ +import numpy as np +import cv2 +import math + + +def apply_min_size(sample, size, image_interpolation_method=cv2.INTER_AREA): + """Rezise the sample to ensure the given size. Keeps aspect ratio. + + Args: + sample (dict): sample + size (tuple): image size + + Returns: + tuple: new size + """ + shape = list(sample["disparity"].shape) + + if shape[0] >= size[0] and shape[1] >= size[1]: + return sample + + scale = [0, 0] + scale[0] = size[0] / shape[0] + scale[1] = size[1] / shape[1] + + scale = max(scale) + + shape[0] = math.ceil(scale * shape[0]) + shape[1] = math.ceil(scale * shape[1]) + + # resize + sample["image"] = cv2.resize( + sample["image"], tuple(shape[::-1]), interpolation=image_interpolation_method + ) + + sample["disparity"] = cv2.resize( + sample["disparity"], tuple(shape[::-1]), interpolation=cv2.INTER_NEAREST + ) + sample["mask"] = cv2.resize( + sample["mask"].astype(np.float32), + tuple(shape[::-1]), + interpolation=cv2.INTER_NEAREST, + ) + sample["mask"] = sample["mask"].astype(bool) + + return tuple(shape) + + +class Resize(object): + """Resize sample to given size (width, height). + """ + + def __init__( + self, + width, + height, + resize_target=True, + keep_aspect_ratio=False, + ensure_multiple_of=1, + resize_method="lower_bound", + image_interpolation_method=cv2.INTER_AREA, + ): + """Init. + + Args: + width (int): desired output width + height (int): desired output height + resize_target (bool, optional): + True: Resize the full sample (image, mask, target). + False: Resize image only. + Defaults to True. + keep_aspect_ratio (bool, optional): + True: Keep the aspect ratio of the input sample. + Output sample might not have the given width and height, and + resize behaviour depends on the parameter 'resize_method'. + Defaults to False. + ensure_multiple_of (int, optional): + Output width and height is constrained to be multiple of this parameter. + Defaults to 1. + resize_method (str, optional): + "lower_bound": Output will be at least as large as the given size. + "upper_bound": Output will be at max as large as the given size. (Output size might be smaller than given size.) + "minimal": Scale as least as possible. (Output size might be smaller than given size.) + Defaults to "lower_bound". + """ + self.__width = width + self.__height = height + + self.__resize_target = resize_target + self.__keep_aspect_ratio = keep_aspect_ratio + self.__multiple_of = ensure_multiple_of + self.__resize_method = resize_method + self.__image_interpolation_method = image_interpolation_method + + def constrain_to_multiple_of(self, x, min_val=0, max_val=None): + y = (np.round(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if max_val is not None and y > max_val: + y = (np.floor(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if y < min_val: + y = (np.ceil(x / self.__multiple_of) * self.__multiple_of).astype(int) + + return y + + def get_size(self, width, height): + # determine new height and width + scale_height = self.__height / height + scale_width = self.__width / width + + if self.__keep_aspect_ratio: + if self.__resize_method == "lower_bound": + # scale such that output size is lower bound + if scale_width > scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "upper_bound": + # scale such that output size is upper bound + if scale_width < scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "minimal": + # scale as least as possbile + if abs(1 - scale_width) < abs(1 - scale_height): + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + else: + raise ValueError( + f"resize_method {self.__resize_method} not implemented" + ) + + if self.__resize_method == "lower_bound": + new_height = self.constrain_to_multiple_of( + scale_height * height, min_val=self.__height + ) + new_width = self.constrain_to_multiple_of( + scale_width * width, min_val=self.__width + ) + elif self.__resize_method == "upper_bound": + new_height = self.constrain_to_multiple_of( + scale_height * height, max_val=self.__height + ) + new_width = self.constrain_to_multiple_of( + scale_width * width, max_val=self.__width + ) + elif self.__resize_method == "minimal": + new_height = self.constrain_to_multiple_of(scale_height * height) + new_width = self.constrain_to_multiple_of(scale_width * width) + else: + raise ValueError(f"resize_method {self.__resize_method} not implemented") + + return (new_width, new_height) + + def __call__(self, sample): + width, height = self.get_size( + sample["image"].shape[1], sample["image"].shape[0] + ) + + # resize sample + sample["image"] = cv2.resize( + sample["image"], + (width, height), + interpolation=self.__image_interpolation_method, + ) + + if self.__resize_target: + if "disparity" in sample: + sample["disparity"] = cv2.resize( + sample["disparity"], + (width, height), + interpolation=cv2.INTER_NEAREST, + ) + + if "depth" in sample: + sample["depth"] = cv2.resize( + sample["depth"], (width, height), interpolation=cv2.INTER_NEAREST + ) + + sample["mask"] = cv2.resize( + sample["mask"].astype(np.float32), + (width, height), + interpolation=cv2.INTER_NEAREST, + ) + sample["mask"] = sample["mask"].astype(bool) + + return sample + + +class NormalizeImage(object): + """Normlize image by given mean and std. + """ + + def __init__(self, mean, std): + self.__mean = mean + self.__std = std + + def __call__(self, sample): + sample["image"] = (sample["image"] - self.__mean) / self.__std + + return sample + + +class PrepareForNet(object): + """Prepare sample for usage as network input. + """ + + def __init__(self): + pass + + def __call__(self, sample): + image = np.transpose(sample["image"], (2, 0, 1)) + sample["image"] = np.ascontiguousarray(image).astype(np.float32) + + if "mask" in sample: + sample["mask"] = sample["mask"].astype(np.float32) + sample["mask"] = np.ascontiguousarray(sample["mask"]) + + if "disparity" in sample: + disparity = sample["disparity"].astype(np.float32) + sample["disparity"] = np.ascontiguousarray(disparity) + + if "depth" in sample: + depth = sample["depth"].astype(np.float32) + sample["depth"] = np.ascontiguousarray(depth) + + return sample diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/utils.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ff9a54bd55f5e31a90fad21242efbfda5a6cc1a7 --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/utils.py @@ -0,0 +1,82 @@ +import numpy as np +import sys +import cv2 + + +def write_pfm(path, image, scale=1): + """Write pfm file. + Args: + path (str): pathto file + image (array): data + scale (int, optional): Scale. Defaults to 1. + """ + + with open(path, "wb") as file: + color = None + + if image.dtype.name != "float32": + raise Exception("Image dtype must be float32.") + + image = np.flipud(image) + + if len(image.shape) == 3 and image.shape[2] == 3: # color image + color = True + elif ( + len(image.shape) == 2 or len(image.shape) == 3 and image.shape[2] == 1 + ): # greyscale + color = False + else: + raise Exception("Image must have H x W x 3, H x W x 1 or H x W dimensions.") + + file.write("PF\n" if color else "Pf\n".encode()) + file.write("%d %d\n".encode() % (image.shape[1], image.shape[0])) + + endian = image.dtype.byteorder + + if endian == "<" or endian == "=" and sys.byteorder == "little": + scale = -scale + + file.write("%f\n".encode() % scale) + + image.tofile(file) + +def read_image(path): + """Read image and output RGB image (0-1). + Args: + path (str): path to file + Returns: + array: RGB image (0-1) + """ + img = cv2.imread(path) + + if img.ndim == 2: + img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) + + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) / 255.0 + + return img + +def write_depth(path, depth, bits=1): + """Write depth map to pfm and png file. + Args: + path (str): filepath without extension + depth (array): depth + """ + write_pfm(path + ".pfm", depth.astype(np.float32)) + + depth_min = depth.min() + depth_max = depth.max() + + max_val = (2**(8*bits))-1 + + if depth_max - depth_min > np.finfo("float").eps: + out = max_val * (depth - depth_min) / (depth_max - depth_min) + else: + out = 0 + + if bits == 1: + cv2.imwrite(path + ".png", out.astype("uint8")) + elif bits == 2: + cv2.imwrite(path + ".png", out.astype("uint16")) + + return \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/base_models/midas_repo/utils.py b/annotator/zoe/zoedepth/models/base_models/midas_repo/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7a3976fd97dfe6a9dc7d4fa144be8fcb0b18b2db --- /dev/null +++ b/annotator/zoe/zoedepth/models/base_models/midas_repo/utils.py @@ -0,0 +1,199 @@ +"""Utils for monoDepth. +""" +import sys +import re +import numpy as np +import cv2 +import torch + + +def read_pfm(path): + """Read pfm file. + + Args: + path (str): path to file + + Returns: + tuple: (data, scale) + """ + with open(path, "rb") as file: + + color = None + width = None + height = None + scale = None + endian = None + + header = file.readline().rstrip() + if header.decode("ascii") == "PF": + color = True + elif header.decode("ascii") == "Pf": + color = False + else: + raise Exception("Not a PFM file: " + path) + + dim_match = re.match(r"^(\d+)\s(\d+)\s$", file.readline().decode("ascii")) + if dim_match: + width, height = list(map(int, dim_match.groups())) + else: + raise Exception("Malformed PFM header.") + + scale = float(file.readline().decode("ascii").rstrip()) + if scale < 0: + # little-endian + endian = "<" + scale = -scale + else: + # big-endian + endian = ">" + + data = np.fromfile(file, endian + "f") + shape = (height, width, 3) if color else (height, width) + + data = np.reshape(data, shape) + data = np.flipud(data) + + return data, scale + + +def write_pfm(path, image, scale=1): + """Write pfm file. + + Args: + path (str): pathto file + image (array): data + scale (int, optional): Scale. Defaults to 1. + """ + + with open(path, "wb") as file: + color = None + + if image.dtype.name != "float32": + raise Exception("Image dtype must be float32.") + + image = np.flipud(image) + + if len(image.shape) == 3 and image.shape[2] == 3: # color image + color = True + elif ( + len(image.shape) == 2 or len(image.shape) == 3 and image.shape[2] == 1 + ): # greyscale + color = False + else: + raise Exception("Image must have H x W x 3, H x W x 1 or H x W dimensions.") + + file.write("PF\n" if color else "Pf\n".encode()) + file.write("%d %d\n".encode() % (image.shape[1], image.shape[0])) + + endian = image.dtype.byteorder + + if endian == "<" or endian == "=" and sys.byteorder == "little": + scale = -scale + + file.write("%f\n".encode() % scale) + + image.tofile(file) + + +def read_image(path): + """Read image and output RGB image (0-1). + + Args: + path (str): path to file + + Returns: + array: RGB image (0-1) + """ + img = cv2.imread(path) + + if img.ndim == 2: + img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) + + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) / 255.0 + + return img + + +def resize_image(img): + """Resize image and make it fit for network. + + Args: + img (array): image + + Returns: + tensor: data ready for network + """ + height_orig = img.shape[0] + width_orig = img.shape[1] + + if width_orig > height_orig: + scale = width_orig / 384 + else: + scale = height_orig / 384 + + height = (np.ceil(height_orig / scale / 32) * 32).astype(int) + width = (np.ceil(width_orig / scale / 32) * 32).astype(int) + + img_resized = cv2.resize(img, (width, height), interpolation=cv2.INTER_AREA) + + img_resized = ( + torch.from_numpy(np.transpose(img_resized, (2, 0, 1))).contiguous().float() + ) + img_resized = img_resized.unsqueeze(0) + + return img_resized + + +def resize_depth(depth, width, height): + """Resize depth map and bring to CPU (numpy). + + Args: + depth (tensor): depth + width (int): image width + height (int): image height + + Returns: + array: processed depth + """ + depth = torch.squeeze(depth[0, :, :, :]).to("cpu") + + depth_resized = cv2.resize( + depth.numpy(), (width, height), interpolation=cv2.INTER_CUBIC + ) + + return depth_resized + +def write_depth(path, depth, grayscale, bits=1): + """Write depth map to png file. + + Args: + path (str): filepath without extension + depth (array): depth + grayscale (bool): use a grayscale colormap? + """ + if not grayscale: + bits = 1 + + if not np.isfinite(depth).all(): + depth=np.nan_to_num(depth, nan=0.0, posinf=0.0, neginf=0.0) + print("WARNING: Non-finite depth values present") + + depth_min = depth.min() + depth_max = depth.max() + + max_val = (2**(8*bits))-1 + + if depth_max - depth_min > np.finfo("float").eps: + out = max_val * (depth - depth_min) / (depth_max - depth_min) + else: + out = np.zeros(depth.shape, dtype=depth.dtype) + + if not grayscale: + out = cv2.applyColorMap(np.uint8(out), cv2.COLORMAP_INFERNO) + + if bits == 1: + cv2.imwrite(path + ".png", out.astype("uint8")) + elif bits == 2: + cv2.imwrite(path + ".png", out.astype("uint16")) + + return diff --git a/annotator/zoe/zoedepth/models/builder.py b/annotator/zoe/zoedepth/models/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..0818311b642561712a03a66655c638ce09a04cca --- /dev/null +++ b/annotator/zoe/zoedepth/models/builder.py @@ -0,0 +1,51 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +from importlib import import_module +from .depth_model import DepthModel + +def build_model(config) -> DepthModel: + """Builds a model from a config. The model is specified by the model name and version in the config. The model is then constructed using the build_from_config function of the model interface. + This function should be used to construct models for training and evaluation. + + Args: + config (dict): Config dict. Config is constructed in utils/config.py. Each model has its own config file(s) saved in its root model folder. + + Returns: + torch.nn.Module: Model corresponding to name and version as specified in config + """ + module_name = f"zoedepth.models.{config.model}" + try: + module = import_module(module_name) + except ModuleNotFoundError as e: + # print the original error message + print(e) + raise ValueError( + f"Model {config.model} not found. Refer above error for details.") from e + try: + get_version = getattr(module, "get_version") + except AttributeError as e: + raise ValueError( + f"Model {config.model} has no get_version function.") from e + return get_version(config.version_name).build_from_config(config) diff --git a/annotator/zoe/zoedepth/models/depth_model.py b/annotator/zoe/zoedepth/models/depth_model.py new file mode 100644 index 0000000000000000000000000000000000000000..fc421c108ea3928c9add62b4c190500d9bd4eda1 --- /dev/null +++ b/annotator/zoe/zoedepth/models/depth_model.py @@ -0,0 +1,152 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torchvision import transforms +import PIL.Image +from PIL import Image +from typing import Union + + +class DepthModel(nn.Module): + def __init__(self): + super().__init__() + self.device = 'cpu' + + def to(self, device) -> nn.Module: + self.device = device + return super().to(device) + + def forward(self, x, *args, **kwargs): + raise NotImplementedError + + def _infer(self, x: torch.Tensor): + """ + Inference interface for the model + Args: + x (torch.Tensor): input tensor of shape (b, c, h, w) + Returns: + torch.Tensor: output tensor of shape (b, 1, h, w) + """ + return self(x)['metric_depth'] + + def _infer_with_pad_aug(self, x: torch.Tensor, pad_input: bool=True, fh: float=3, fw: float=3, upsampling_mode: str='bicubic', padding_mode="reflect", **kwargs) -> torch.Tensor: + """ + Inference interface for the model with padding augmentation + Padding augmentation fixes the boundary artifacts in the output depth map. + Boundary artifacts are sometimes caused by the fact that the model is trained on NYU raw dataset which has a black or white border around the image. + This augmentation pads the input image and crops the prediction back to the original size / view. + + Note: This augmentation is not required for the models trained with 'avoid_boundary'=True. + Args: + x (torch.Tensor): input tensor of shape (b, c, h, w) + pad_input (bool, optional): whether to pad the input or not. Defaults to True. + fh (float, optional): height padding factor. The padding is calculated as sqrt(h/2) * fh. Defaults to 3. + fw (float, optional): width padding factor. The padding is calculated as sqrt(w/2) * fw. Defaults to 3. + upsampling_mode (str, optional): upsampling mode. Defaults to 'bicubic'. + padding_mode (str, optional): padding mode. Defaults to "reflect". + Returns: + torch.Tensor: output tensor of shape (b, 1, h, w) + """ + # assert x is nchw and c = 3 + assert x.dim() == 4, "x must be 4 dimensional, got {}".format(x.dim()) + assert x.shape[1] == 3, "x must have 3 channels, got {}".format(x.shape[1]) + + if pad_input: + assert fh > 0 or fw > 0, "atlease one of fh and fw must be greater than 0" + pad_h = int(np.sqrt(x.shape[2]/2) * fh) + pad_w = int(np.sqrt(x.shape[3]/2) * fw) + padding = [pad_w, pad_w] + if pad_h > 0: + padding += [pad_h, pad_h] + + x = F.pad(x, padding, mode=padding_mode, **kwargs) + out = self._infer(x) + if out.shape[-2:] != x.shape[-2:]: + out = F.interpolate(out, size=(x.shape[2], x.shape[3]), mode=upsampling_mode, align_corners=False) + if pad_input: + # crop to the original size, handling the case where pad_h and pad_w is 0 + if pad_h > 0: + out = out[:, :, pad_h:-pad_h,:] + if pad_w > 0: + out = out[:, :, :, pad_w:-pad_w] + return out + + def infer_with_flip_aug(self, x, pad_input: bool=True, **kwargs) -> torch.Tensor: + """ + Inference interface for the model with horizontal flip augmentation + Horizontal flip augmentation improves the accuracy of the model by averaging the output of the model with and without horizontal flip. + Args: + x (torch.Tensor): input tensor of shape (b, c, h, w) + pad_input (bool, optional): whether to use padding augmentation. Defaults to True. + Returns: + torch.Tensor: output tensor of shape (b, 1, h, w) + """ + # infer with horizontal flip and average + out = self._infer_with_pad_aug(x, pad_input=pad_input, **kwargs) + out_flip = self._infer_with_pad_aug(torch.flip(x, dims=[3]), pad_input=pad_input, **kwargs) + out = (out + torch.flip(out_flip, dims=[3])) / 2 + return out + + def infer(self, x, pad_input: bool=True, with_flip_aug: bool=True, **kwargs) -> torch.Tensor: + """ + Inference interface for the model + Args: + x (torch.Tensor): input tensor of shape (b, c, h, w) + pad_input (bool, optional): whether to use padding augmentation. Defaults to True. + with_flip_aug (bool, optional): whether to use horizontal flip augmentation. Defaults to True. + Returns: + torch.Tensor: output tensor of shape (b, 1, h, w) + """ + if with_flip_aug: + return self.infer_with_flip_aug(x, pad_input=pad_input, **kwargs) + else: + return self._infer_with_pad_aug(x, pad_input=pad_input, **kwargs) + + @torch.no_grad() + def infer_pil(self, pil_img, pad_input: bool=True, with_flip_aug: bool=True, output_type: str="numpy", **kwargs) -> Union[np.ndarray, PIL.Image.Image, torch.Tensor]: + """ + Inference interface for the model for PIL image + Args: + pil_img (PIL.Image.Image): input PIL image + pad_input (bool, optional): whether to use padding augmentation. Defaults to True. + with_flip_aug (bool, optional): whether to use horizontal flip augmentation. Defaults to True. + output_type (str, optional): output type. Supported values are 'numpy', 'pil' and 'tensor'. Defaults to "numpy". + """ + x = transforms.ToTensor()(pil_img).unsqueeze(0).to(self.device) + out_tensor = self.infer(x, pad_input=pad_input, with_flip_aug=with_flip_aug, **kwargs) + if output_type == "numpy": + return out_tensor.squeeze().cpu().numpy() + elif output_type == "pil": + # uint16 is required for depth pil image + out_16bit_numpy = (out_tensor.squeeze().cpu().numpy()*256).astype(np.uint16) + return Image.fromarray(out_16bit_numpy) + elif output_type == "tensor": + return out_tensor.squeeze().cpu() + else: + raise ValueError(f"output_type {output_type} not supported. Supported values are 'numpy', 'pil' and 'tensor'") + \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/layers/attractor.py b/annotator/zoe/zoedepth/models/layers/attractor.py new file mode 100644 index 0000000000000000000000000000000000000000..2a8efe645adea1d88a12e2ac5cc6bb2a251eef9d --- /dev/null +++ b/annotator/zoe/zoedepth/models/layers/attractor.py @@ -0,0 +1,208 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import torch +import torch.nn as nn + + +@torch.jit.script +def exp_attractor(dx, alpha: float = 300, gamma: int = 2): + """Exponential attractor: dc = exp(-alpha*|dx|^gamma) * dx , where dx = a - c, a = attractor point, c = bin center, dc = shift in bin centermmary for exp_attractor + + Args: + dx (torch.Tensor): The difference tensor dx = Ai - Cj, where Ai is the attractor point and Cj is the bin center. + alpha (float, optional): Proportional Attractor strength. Determines the absolute strength. Lower alpha = greater attraction. Defaults to 300. + gamma (int, optional): Exponential Attractor strength. Determines the "region of influence" and indirectly number of bin centers affected. Lower gamma = farther reach. Defaults to 2. + + Returns: + torch.Tensor : Delta shifts - dc; New bin centers = Old bin centers + dc + """ + return torch.exp(-alpha*(torch.abs(dx)**gamma)) * (dx) + + +@torch.jit.script +def inv_attractor(dx, alpha: float = 300, gamma: int = 2): + """Inverse attractor: dc = dx / (1 + alpha*dx^gamma), where dx = a - c, a = attractor point, c = bin center, dc = shift in bin center + This is the default one according to the accompanying paper. + + Args: + dx (torch.Tensor): The difference tensor dx = Ai - Cj, where Ai is the attractor point and Cj is the bin center. + alpha (float, optional): Proportional Attractor strength. Determines the absolute strength. Lower alpha = greater attraction. Defaults to 300. + gamma (int, optional): Exponential Attractor strength. Determines the "region of influence" and indirectly number of bin centers affected. Lower gamma = farther reach. Defaults to 2. + + Returns: + torch.Tensor: Delta shifts - dc; New bin centers = Old bin centers + dc + """ + return dx.div(1+alpha*dx.pow(gamma)) + + +class AttractorLayer(nn.Module): + def __init__(self, in_features, n_bins, n_attractors=16, mlp_dim=128, min_depth=1e-3, max_depth=10, + alpha=300, gamma=2, kind='sum', attractor_type='exp', memory_efficient=False): + """ + Attractor layer for bin centers. Bin centers are bounded on the interval (min_depth, max_depth) + """ + super().__init__() + + self.n_attractors = n_attractors + self.n_bins = n_bins + self.min_depth = min_depth + self.max_depth = max_depth + self.alpha = alpha + self.gamma = gamma + self.kind = kind + self.attractor_type = attractor_type + self.memory_efficient = memory_efficient + + self._net = nn.Sequential( + nn.Conv2d(in_features, mlp_dim, 1, 1, 0), + nn.ReLU(inplace=True), + nn.Conv2d(mlp_dim, n_attractors*2, 1, 1, 0), # x2 for linear norm + nn.ReLU(inplace=True) + ) + + def forward(self, x, b_prev, prev_b_embedding=None, interpolate=True, is_for_query=False): + """ + Args: + x (torch.Tensor) : feature block; shape - n, c, h, w + b_prev (torch.Tensor) : previous bin centers normed; shape - n, prev_nbins, h, w + + Returns: + tuple(torch.Tensor,torch.Tensor) : new bin centers normed and scaled; shape - n, nbins, h, w + """ + if prev_b_embedding is not None: + if interpolate: + prev_b_embedding = nn.functional.interpolate( + prev_b_embedding, x.shape[-2:], mode='bilinear', align_corners=True) + x = x + prev_b_embedding + + A = self._net(x) + eps = 1e-3 + A = A + eps + n, c, h, w = A.shape + A = A.view(n, self.n_attractors, 2, h, w) + A_normed = A / A.sum(dim=2, keepdim=True) # n, a, 2, h, w + A_normed = A[:, :, 0, ...] # n, na, h, w + + b_prev = nn.functional.interpolate( + b_prev, (h, w), mode='bilinear', align_corners=True) + b_centers = b_prev + + if self.attractor_type == 'exp': + dist = exp_attractor + else: + dist = inv_attractor + + if not self.memory_efficient: + func = {'mean': torch.mean, 'sum': torch.sum}[self.kind] + # .shape N, nbins, h, w + delta_c = func(dist(A_normed.unsqueeze( + 2) - b_centers.unsqueeze(1)), dim=1) + else: + delta_c = torch.zeros_like(b_centers, device=b_centers.device) + for i in range(self.n_attractors): + # .shape N, nbins, h, w + delta_c += dist(A_normed[:, i, ...].unsqueeze(1) - b_centers) + + if self.kind == 'mean': + delta_c = delta_c / self.n_attractors + + b_new_centers = b_centers + delta_c + B_centers = (self.max_depth - self.min_depth) * \ + b_new_centers + self.min_depth + B_centers, _ = torch.sort(B_centers, dim=1) + B_centers = torch.clip(B_centers, self.min_depth, self.max_depth) + return b_new_centers, B_centers + + +class AttractorLayerUnnormed(nn.Module): + def __init__(self, in_features, n_bins, n_attractors=16, mlp_dim=128, min_depth=1e-3, max_depth=10, + alpha=300, gamma=2, kind='sum', attractor_type='exp', memory_efficient=False): + """ + Attractor layer for bin centers. Bin centers are unbounded + """ + super().__init__() + + self.n_attractors = n_attractors + self.n_bins = n_bins + self.min_depth = min_depth + self.max_depth = max_depth + self.alpha = alpha + self.gamma = gamma + self.kind = kind + self.attractor_type = attractor_type + self.memory_efficient = memory_efficient + + self._net = nn.Sequential( + nn.Conv2d(in_features, mlp_dim, 1, 1, 0), + nn.ReLU(inplace=True), + nn.Conv2d(mlp_dim, n_attractors, 1, 1, 0), + nn.Softplus() + ) + + def forward(self, x, b_prev, prev_b_embedding=None, interpolate=True, is_for_query=False): + """ + Args: + x (torch.Tensor) : feature block; shape - n, c, h, w + b_prev (torch.Tensor) : previous bin centers normed; shape - n, prev_nbins, h, w + + Returns: + tuple(torch.Tensor,torch.Tensor) : new bin centers unbounded; shape - n, nbins, h, w. Two outputs just to keep the API consistent with the normed version + """ + if prev_b_embedding is not None: + if interpolate: + prev_b_embedding = nn.functional.interpolate( + prev_b_embedding, x.shape[-2:], mode='bilinear', align_corners=True) + x = x + prev_b_embedding + + A = self._net(x) + n, c, h, w = A.shape + + b_prev = nn.functional.interpolate( + b_prev, (h, w), mode='bilinear', align_corners=True) + b_centers = b_prev + + if self.attractor_type == 'exp': + dist = exp_attractor + else: + dist = inv_attractor + + if not self.memory_efficient: + func = {'mean': torch.mean, 'sum': torch.sum}[self.kind] + # .shape N, nbins, h, w + delta_c = func( + dist(A.unsqueeze(2) - b_centers.unsqueeze(1)), dim=1) + else: + delta_c = torch.zeros_like(b_centers, device=b_centers.device) + for i in range(self.n_attractors): + delta_c += dist(A[:, i, ...].unsqueeze(1) - + b_centers) # .shape N, nbins, h, w + + if self.kind == 'mean': + delta_c = delta_c / self.n_attractors + + b_new_centers = b_centers + delta_c + B_centers = b_new_centers + + return b_new_centers, B_centers diff --git a/annotator/zoe/zoedepth/models/layers/dist_layers.py b/annotator/zoe/zoedepth/models/layers/dist_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..3208405dfb78fdfc28d5765e5a6d5dbe31967a23 --- /dev/null +++ b/annotator/zoe/zoedepth/models/layers/dist_layers.py @@ -0,0 +1,121 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import torch +import torch.nn as nn + + +def log_binom(n, k, eps=1e-7): + """ log(nCk) using stirling approximation """ + n = n + eps + k = k + eps + return n * torch.log(n) - k * torch.log(k) - (n-k) * torch.log(n-k+eps) + + +class LogBinomial(nn.Module): + def __init__(self, n_classes=256, act=torch.softmax): + """Compute log binomial distribution for n_classes + + Args: + n_classes (int, optional): number of output classes. Defaults to 256. + """ + super().__init__() + self.K = n_classes + self.act = act + self.register_buffer('k_idx', torch.arange( + 0, n_classes).view(1, -1, 1, 1)) + self.register_buffer('K_minus_1', torch.Tensor( + [self.K-1]).view(1, -1, 1, 1)) + + def forward(self, x, t=1., eps=1e-4): + """Compute log binomial distribution for x + + Args: + x (torch.Tensor - NCHW): probabilities + t (float, torch.Tensor - NCHW, optional): Temperature of distribution. Defaults to 1.. + eps (float, optional): Small number for numerical stability. Defaults to 1e-4. + + Returns: + torch.Tensor -NCHW: log binomial distribution logbinomial(p;t) + """ + if x.ndim == 3: + x = x.unsqueeze(1) # make it nchw + + one_minus_x = torch.clamp(1 - x, eps, 1) + x = torch.clamp(x, eps, 1) + y = log_binom(self.K_minus_1, self.k_idx) + self.k_idx * \ + torch.log(x) + (self.K - 1 - self.k_idx) * torch.log(one_minus_x) + return self.act(y/t, dim=1) + + +class ConditionalLogBinomial(nn.Module): + def __init__(self, in_features, condition_dim, n_classes=256, bottleneck_factor=2, p_eps=1e-4, max_temp=50, min_temp=1e-7, act=torch.softmax): + """Conditional Log Binomial distribution + + Args: + in_features (int): number of input channels in main feature + condition_dim (int): number of input channels in condition feature + n_classes (int, optional): Number of classes. Defaults to 256. + bottleneck_factor (int, optional): Hidden dim factor. Defaults to 2. + p_eps (float, optional): small eps value. Defaults to 1e-4. + max_temp (float, optional): Maximum temperature of output distribution. Defaults to 50. + min_temp (float, optional): Minimum temperature of output distribution. Defaults to 1e-7. + """ + super().__init__() + self.p_eps = p_eps + self.max_temp = max_temp + self.min_temp = min_temp + self.log_binomial_transform = LogBinomial(n_classes, act=act) + bottleneck = (in_features + condition_dim) // bottleneck_factor + self.mlp = nn.Sequential( + nn.Conv2d(in_features + condition_dim, bottleneck, + kernel_size=1, stride=1, padding=0), + nn.GELU(), + # 2 for p linear norm, 2 for t linear norm + nn.Conv2d(bottleneck, 2+2, kernel_size=1, stride=1, padding=0), + nn.Softplus() + ) + + def forward(self, x, cond): + """Forward pass + + Args: + x (torch.Tensor - NCHW): Main feature + cond (torch.Tensor - NCHW): condition feature + + Returns: + torch.Tensor: Output log binomial distribution + """ + pt = self.mlp(torch.concat((x, cond), dim=1)) + p, t = pt[:, :2, ...], pt[:, 2:, ...] + + p = p + self.p_eps + p = p[:, 0, ...] / (p[:, 0, ...] + p[:, 1, ...]) + + t = t + self.p_eps + t = t[:, 0, ...] / (t[:, 0, ...] + t[:, 1, ...]) + t = t.unsqueeze(1) + t = (self.max_temp - self.min_temp) * t + self.min_temp + + return self.log_binomial_transform(p, t) diff --git a/annotator/zoe/zoedepth/models/layers/localbins_layers.py b/annotator/zoe/zoedepth/models/layers/localbins_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..f94481605c3e6958ce50e73b2eb31d9f0c07dc67 --- /dev/null +++ b/annotator/zoe/zoedepth/models/layers/localbins_layers.py @@ -0,0 +1,169 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import torch +import torch.nn as nn + + +class SeedBinRegressor(nn.Module): + def __init__(self, in_features, n_bins=16, mlp_dim=256, min_depth=1e-3, max_depth=10): + """Bin center regressor network. Bin centers are bounded on (min_depth, max_depth) interval. + + Args: + in_features (int): input channels + n_bins (int, optional): Number of bin centers. Defaults to 16. + mlp_dim (int, optional): Hidden dimension. Defaults to 256. + min_depth (float, optional): Min depth value. Defaults to 1e-3. + max_depth (float, optional): Max depth value. Defaults to 10. + """ + super().__init__() + self.version = "1_1" + self.min_depth = min_depth + self.max_depth = max_depth + + self._net = nn.Sequential( + nn.Conv2d(in_features, mlp_dim, 1, 1, 0), + nn.ReLU(inplace=True), + nn.Conv2d(mlp_dim, n_bins, 1, 1, 0), + nn.ReLU(inplace=True) + ) + + def forward(self, x): + """ + Returns tensor of bin_width vectors (centers). One vector b for every pixel + """ + B = self._net(x) + eps = 1e-3 + B = B + eps + B_widths_normed = B / B.sum(dim=1, keepdim=True) + B_widths = (self.max_depth - self.min_depth) * \ + B_widths_normed # .shape NCHW + # pad has the form (left, right, top, bottom, front, back) + B_widths = nn.functional.pad( + B_widths, (0, 0, 0, 0, 1, 0), mode='constant', value=self.min_depth) + B_edges = torch.cumsum(B_widths, dim=1) # .shape NCHW + + B_centers = 0.5 * (B_edges[:, :-1, ...] + B_edges[:, 1:, ...]) + return B_widths_normed, B_centers + + +class SeedBinRegressorUnnormed(nn.Module): + def __init__(self, in_features, n_bins=16, mlp_dim=256, min_depth=1e-3, max_depth=10): + """Bin center regressor network. Bin centers are unbounded + + Args: + in_features (int): input channels + n_bins (int, optional): Number of bin centers. Defaults to 16. + mlp_dim (int, optional): Hidden dimension. Defaults to 256. + min_depth (float, optional): Not used. (for compatibility with SeedBinRegressor) + max_depth (float, optional): Not used. (for compatibility with SeedBinRegressor) + """ + super().__init__() + self.version = "1_1" + self._net = nn.Sequential( + nn.Conv2d(in_features, mlp_dim, 1, 1, 0), + nn.ReLU(inplace=True), + nn.Conv2d(mlp_dim, n_bins, 1, 1, 0), + nn.Softplus() + ) + + def forward(self, x): + """ + Returns tensor of bin_width vectors (centers). One vector b for every pixel + """ + B_centers = self._net(x) + return B_centers, B_centers + + +class Projector(nn.Module): + def __init__(self, in_features, out_features, mlp_dim=128): + """Projector MLP + + Args: + in_features (int): input channels + out_features (int): output channels + mlp_dim (int, optional): hidden dimension. Defaults to 128. + """ + super().__init__() + + self._net = nn.Sequential( + nn.Conv2d(in_features, mlp_dim, 1, 1, 0), + nn.ReLU(inplace=True), + nn.Conv2d(mlp_dim, out_features, 1, 1, 0), + ) + + def forward(self, x): + return self._net(x) + + + +class LinearSplitter(nn.Module): + def __init__(self, in_features, prev_nbins, split_factor=2, mlp_dim=128, min_depth=1e-3, max_depth=10): + super().__init__() + + self.prev_nbins = prev_nbins + self.split_factor = split_factor + self.min_depth = min_depth + self.max_depth = max_depth + + self._net = nn.Sequential( + nn.Conv2d(in_features, mlp_dim, 1, 1, 0), + nn.GELU(), + nn.Conv2d(mlp_dim, prev_nbins * split_factor, 1, 1, 0), + nn.ReLU() + ) + + def forward(self, x, b_prev, prev_b_embedding=None, interpolate=True, is_for_query=False): + """ + x : feature block; shape - n, c, h, w + b_prev : previous bin widths normed; shape - n, prev_nbins, h, w + """ + if prev_b_embedding is not None: + if interpolate: + prev_b_embedding = nn.functional.interpolate(prev_b_embedding, x.shape[-2:], mode='bilinear', align_corners=True) + x = x + prev_b_embedding + S = self._net(x) + eps = 1e-3 + S = S + eps + n, c, h, w = S.shape + S = S.view(n, self.prev_nbins, self.split_factor, h, w) + S_normed = S / S.sum(dim=2, keepdim=True) # fractional splits + + b_prev = nn.functional.interpolate(b_prev, (h,w), mode='bilinear', align_corners=True) + + + b_prev = b_prev / b_prev.sum(dim=1, keepdim=True) # renormalize for gurantees + # print(b_prev.shape, S_normed.shape) + # if is_for_query:(1).expand(-1, b_prev.size(0)//n, -1, -1, -1, -1).flatten(0,1) # TODO ? can replace all this with a single torch.repeat? + b = b_prev.unsqueeze(2) * S_normed + b = b.flatten(1,2) # .shape n, prev_nbins * split_factor, h, w + + # calculate bin centers for loss calculation + B_widths = (self.max_depth - self.min_depth) * b # .shape N, nprev * splitfactor, H, W + # pad has the form (left, right, top, bottom, front, back) + B_widths = nn.functional.pad(B_widths, (0,0,0,0,1,0), mode='constant', value=self.min_depth) + B_edges = torch.cumsum(B_widths, dim=1) # .shape NCHW + + B_centers = 0.5 * (B_edges[:, :-1, ...] + B_edges[:,1:,...]) + return b, B_centers \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/layers/patch_transformer.py b/annotator/zoe/zoedepth/models/layers/patch_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..99d9e51a06b981bae45ce7dd64eaef19a4121991 --- /dev/null +++ b/annotator/zoe/zoedepth/models/layers/patch_transformer.py @@ -0,0 +1,91 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import torch +import torch.nn as nn + + +class PatchTransformerEncoder(nn.Module): + def __init__(self, in_channels, patch_size=10, embedding_dim=128, num_heads=4, use_class_token=False): + """ViT-like transformer block + + Args: + in_channels (int): Input channels + patch_size (int, optional): patch size. Defaults to 10. + embedding_dim (int, optional): Embedding dimension in transformer model. Defaults to 128. + num_heads (int, optional): number of attention heads. Defaults to 4. + use_class_token (bool, optional): Whether to use extra token at the start for global accumulation (called as "class token"). Defaults to False. + """ + super(PatchTransformerEncoder, self).__init__() + self.use_class_token = use_class_token + encoder_layers = nn.TransformerEncoderLayer( + embedding_dim, num_heads, dim_feedforward=1024) + self.transformer_encoder = nn.TransformerEncoder( + encoder_layers, num_layers=4) # takes shape S,N,E + + self.embedding_convPxP = nn.Conv2d(in_channels, embedding_dim, + kernel_size=patch_size, stride=patch_size, padding=0) + + def positional_encoding_1d(self, sequence_length, batch_size, embedding_dim, device='cpu'): + """Generate positional encodings + + Args: + sequence_length (int): Sequence length + embedding_dim (int): Embedding dimension + + Returns: + torch.Tensor SBE: Positional encodings + """ + position = torch.arange( + 0, sequence_length, dtype=torch.float32, device=device).unsqueeze(1) + index = torch.arange( + 0, embedding_dim, 2, dtype=torch.float32, device=device).unsqueeze(0) + div_term = torch.exp(index * (-torch.log(torch.tensor(10000.0, device=device)) / embedding_dim)) + pos_encoding = position * div_term + pos_encoding = torch.cat([torch.sin(pos_encoding), torch.cos(pos_encoding)], dim=1) + pos_encoding = pos_encoding.unsqueeze(1).repeat(1, batch_size, 1) + return pos_encoding + + + def forward(self, x): + """Forward pass + + Args: + x (torch.Tensor - NCHW): Input feature tensor + + Returns: + torch.Tensor - SNE: Transformer output embeddings. S - sequence length (=HW/patch_size^2), N - batch size, E - embedding dim + """ + embeddings = self.embedding_convPxP(x).flatten( + 2) # .shape = n,c,s = n, embedding_dim, s + if self.use_class_token: + # extra special token at start ? + embeddings = nn.functional.pad(embeddings, (1, 0)) + + # change to S,N,E format required by transformer + embeddings = embeddings.permute(2, 0, 1) + S, N, E = embeddings.shape + embeddings = embeddings + self.positional_encoding_1d(S, N, E, device=embeddings.device) + x = self.transformer_encoder(embeddings) # .shape = S, N, E + return x diff --git a/annotator/zoe/zoedepth/models/model_io.py b/annotator/zoe/zoedepth/models/model_io.py new file mode 100644 index 0000000000000000000000000000000000000000..78b6579631dd847ac76651238cb5a948b5a66286 --- /dev/null +++ b/annotator/zoe/zoedepth/models/model_io.py @@ -0,0 +1,92 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import torch + +def load_state_dict(model, state_dict): + """Load state_dict into model, handling DataParallel and DistributedDataParallel. Also checks for "model" key in state_dict. + + DataParallel prefixes state_dict keys with 'module.' when saving. + If the model is not a DataParallel model but the state_dict is, then prefixes are removed. + If the model is a DataParallel model but the state_dict is not, then prefixes are added. + """ + state_dict = state_dict.get('model', state_dict) + # if model is a DataParallel model, then state_dict keys are prefixed with 'module.' + + do_prefix = isinstance( + model, (torch.nn.DataParallel, torch.nn.parallel.DistributedDataParallel)) + state = {} + for k, v in state_dict.items(): + if k.startswith('module.') and not do_prefix: + k = k[7:] + + if not k.startswith('module.') and do_prefix: + k = 'module.' + k + + state[k] = v + + model.load_state_dict(state) + print("Loaded successfully") + return model + + +def load_wts(model, checkpoint_path): + ckpt = torch.load(checkpoint_path, map_location='cpu') + return load_state_dict(model, ckpt) + + +def load_state_dict_from_url(model, url, **kwargs): + state_dict = torch.hub.load_state_dict_from_url(url, map_location='cpu', **kwargs) + return load_state_dict(model, state_dict) + + +def load_state_from_resource(model, resource: str): + """Loads weights to the model from a given resource. A resource can be of following types: + 1. URL. Prefixed with "url::" + e.g. url::http(s)://url.resource.com/ckpt.pt + + 2. Local path. Prefixed with "local::" + e.g. local::/path/to/ckpt.pt + + + Args: + model (torch.nn.Module): Model + resource (str): resource string + + Returns: + torch.nn.Module: Model with loaded weights + """ + print(f"Using pretrained resource {resource}") + + if resource.startswith('url::'): + url = resource.split('url::')[1] + return load_state_dict_from_url(model, url, progress=True) + + elif resource.startswith('local::'): + path = resource.split('local::')[1] + return load_wts(model, path) + + else: + raise ValueError("Invalid resource type, only url:: and local:: are supported") + \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/zoedepth/__init__.py b/annotator/zoe/zoedepth/models/zoedepth/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cc33f737d238766559f0e3a8def3c0b568f23b7f --- /dev/null +++ b/annotator/zoe/zoedepth/models/zoedepth/__init__.py @@ -0,0 +1,31 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +from .zoedepth_v1 import ZoeDepth + +all_versions = { + "v1": ZoeDepth, +} + +get_version = lambda v : all_versions[v] \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/zoedepth/config_zoedepth.json b/annotator/zoe/zoedepth/models/zoedepth/config_zoedepth.json new file mode 100644 index 0000000000000000000000000000000000000000..3112ed78c89f00e1d13f5d6e5be87cd3216b6dc7 --- /dev/null +++ b/annotator/zoe/zoedepth/models/zoedepth/config_zoedepth.json @@ -0,0 +1,58 @@ +{ + "model": { + "name": "ZoeDepth", + "version_name": "v1", + "n_bins": 64, + "bin_embedding_dim": 128, + "bin_centers_type": "softplus", + "n_attractors":[16, 8, 4, 1], + "attractor_alpha": 1000, + "attractor_gamma": 2, + "attractor_kind" : "mean", + "attractor_type" : "inv", + "midas_model_type" : "DPT_BEiT_L_384", + "min_temp": 0.0212, + "max_temp": 50.0, + "output_distribution": "logbinomial", + "memory_efficient": true, + "inverse_midas": false, + "img_size": [384, 512] + }, + + "train": { + "train_midas": true, + "use_pretrained_midas": true, + "trainer": "zoedepth", + "epochs": 5, + "bs": 16, + "optim_kwargs": {"lr": 0.000161, "wd": 0.01}, + "sched_kwargs": {"div_factor": 1, "final_div_factor": 10000, "pct_start": 0.7, "three_phase":false, "cycle_momentum": true}, + "same_lr": false, + "w_si": 1, + "w_domain": 0.2, + "w_reg": 0, + "w_grad": 0, + "avoid_boundary": false, + "random_crop": false, + "input_width": 640, + "input_height": 480, + "midas_lr_factor": 1, + "encoder_lr_factor":10, + "pos_enc_lr_factor":10, + "freeze_midas_bn": true + + }, + + "infer":{ + "train_midas": false, + "use_pretrained_midas": false, + "pretrained_resource" : null, + "force_keep_ar": true + }, + + "eval":{ + "train_midas": false, + "use_pretrained_midas": false, + "pretrained_resource" : null + } +} \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/zoedepth/config_zoedepth_kitti.json b/annotator/zoe/zoedepth/models/zoedepth/config_zoedepth_kitti.json new file mode 100644 index 0000000000000000000000000000000000000000..b51802aa44b91c39e15aacaac4b5ab6bec884414 --- /dev/null +++ b/annotator/zoe/zoedepth/models/zoedepth/config_zoedepth_kitti.json @@ -0,0 +1,22 @@ +{ + "model": { + "bin_centers_type": "normed", + "img_size": [384, 768] + }, + + "train": { + }, + + "infer":{ + "train_midas": false, + "use_pretrained_midas": false, + "pretrained_resource" : "url::https://github.com/isl-org/ZoeDepth/releases/download/v1.0/ZoeD_M12_K.pt", + "force_keep_ar": true + }, + + "eval":{ + "train_midas": false, + "use_pretrained_midas": false, + "pretrained_resource" : "url::https://github.com/isl-org/ZoeDepth/releases/download/v1.0/ZoeD_M12_K.pt" + } +} \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/zoedepth/zoedepth_v1.py b/annotator/zoe/zoedepth/models/zoedepth/zoedepth_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..bc931b059d6165c84e8ff4f09d5c62d19930cee9 --- /dev/null +++ b/annotator/zoe/zoedepth/models/zoedepth/zoedepth_v1.py @@ -0,0 +1,250 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import itertools + +import torch +import torch.nn as nn +from ..depth_model import DepthModel +from ..base_models.midas import MidasCore +from ..layers.attractor import AttractorLayer, AttractorLayerUnnormed +from ..layers.dist_layers import ConditionalLogBinomial +from ..layers.localbins_layers import (Projector, SeedBinRegressor, + SeedBinRegressorUnnormed) +from ..model_io import load_state_from_resource + + +class ZoeDepth(DepthModel): + def __init__(self, core, n_bins=64, bin_centers_type="softplus", bin_embedding_dim=128, min_depth=1e-3, max_depth=10, + n_attractors=[16, 8, 4, 1], attractor_alpha=300, attractor_gamma=2, attractor_kind='sum', attractor_type='exp', min_temp=5, max_temp=50, train_midas=True, + midas_lr_factor=10, encoder_lr_factor=10, pos_enc_lr_factor=10, inverse_midas=False, **kwargs): + """ZoeDepth model. This is the version of ZoeDepth that has a single metric head + + Args: + core (models.base_models.midas.MidasCore): The base midas model that is used for extraction of "relative" features + n_bins (int, optional): Number of bin centers. Defaults to 64. + bin_centers_type (str, optional): "normed" or "softplus". Activation type used for bin centers. For "normed" bin centers, linear normalization trick is applied. This results in bounded bin centers. + For "softplus", softplus activation is used and thus are unbounded. Defaults to "softplus". + bin_embedding_dim (int, optional): bin embedding dimension. Defaults to 128. + min_depth (float, optional): Lower bound for normed bin centers. Defaults to 1e-3. + max_depth (float, optional): Upper bound for normed bin centers. Defaults to 10. + n_attractors (List[int], optional): Number of bin attractors at decoder layers. Defaults to [16, 8, 4, 1]. + attractor_alpha (int, optional): Proportional attractor strength. Refer to models.layers.attractor for more details. Defaults to 300. + attractor_gamma (int, optional): Exponential attractor strength. Refer to models.layers.attractor for more details. Defaults to 2. + attractor_kind (str, optional): Attraction aggregation "sum" or "mean". Defaults to 'sum'. + attractor_type (str, optional): Type of attractor to use; "inv" (Inverse attractor) or "exp" (Exponential attractor). Defaults to 'exp'. + min_temp (int, optional): Lower bound for temperature of output probability distribution. Defaults to 5. + max_temp (int, optional): Upper bound for temperature of output probability distribution. Defaults to 50. + train_midas (bool, optional): Whether to train "core", the base midas model. Defaults to True. + midas_lr_factor (int, optional): Learning rate reduction factor for base midas model except its encoder and positional encodings. Defaults to 10. + encoder_lr_factor (int, optional): Learning rate reduction factor for the encoder in midas model. Defaults to 10. + pos_enc_lr_factor (int, optional): Learning rate reduction factor for positional encodings in the base midas model. Defaults to 10. + """ + super().__init__() + + self.core = core + self.max_depth = max_depth + self.min_depth = min_depth + self.min_temp = min_temp + self.bin_centers_type = bin_centers_type + + self.midas_lr_factor = midas_lr_factor + self.encoder_lr_factor = encoder_lr_factor + self.pos_enc_lr_factor = pos_enc_lr_factor + self.train_midas = train_midas + self.inverse_midas = inverse_midas + + if self.encoder_lr_factor <= 0: + self.core.freeze_encoder( + freeze_rel_pos=self.pos_enc_lr_factor <= 0) + + N_MIDAS_OUT = 32 + btlnck_features = self.core.output_channels[0] + num_out_features = self.core.output_channels[1:] + + self.conv2 = nn.Conv2d(btlnck_features, btlnck_features, + kernel_size=1, stride=1, padding=0) # btlnck conv + + if bin_centers_type == "normed": + SeedBinRegressorLayer = SeedBinRegressor + Attractor = AttractorLayer + elif bin_centers_type == "softplus": + SeedBinRegressorLayer = SeedBinRegressorUnnormed + Attractor = AttractorLayerUnnormed + elif bin_centers_type == "hybrid1": + SeedBinRegressorLayer = SeedBinRegressor + Attractor = AttractorLayerUnnormed + elif bin_centers_type == "hybrid2": + SeedBinRegressorLayer = SeedBinRegressorUnnormed + Attractor = AttractorLayer + else: + raise ValueError( + "bin_centers_type should be one of 'normed', 'softplus', 'hybrid1', 'hybrid2'") + + self.seed_bin_regressor = SeedBinRegressorLayer( + btlnck_features, n_bins=n_bins, min_depth=min_depth, max_depth=max_depth) + self.seed_projector = Projector(btlnck_features, bin_embedding_dim) + self.projectors = nn.ModuleList([ + Projector(num_out, bin_embedding_dim) + for num_out in num_out_features + ]) + self.attractors = nn.ModuleList([ + Attractor(bin_embedding_dim, n_bins, n_attractors=n_attractors[i], min_depth=min_depth, max_depth=max_depth, + alpha=attractor_alpha, gamma=attractor_gamma, kind=attractor_kind, attractor_type=attractor_type) + for i in range(len(num_out_features)) + ]) + + last_in = N_MIDAS_OUT + 1 # +1 for relative depth + + # use log binomial instead of softmax + self.conditional_log_binomial = ConditionalLogBinomial( + last_in, bin_embedding_dim, n_classes=n_bins, min_temp=min_temp, max_temp=max_temp) + + def forward(self, x, return_final_centers=False, denorm=False, return_probs=False, **kwargs): + """ + Args: + x (torch.Tensor): Input image tensor of shape (B, C, H, W) + return_final_centers (bool, optional): Whether to return the final bin centers. Defaults to False. + denorm (bool, optional): Whether to denormalize the input image. This reverses ImageNet normalization as midas normalization is different. Defaults to False. + return_probs (bool, optional): Whether to return the output probability distribution. Defaults to False. + + Returns: + dict: Dictionary containing the following keys: + - rel_depth (torch.Tensor): Relative depth map of shape (B, H, W) + - metric_depth (torch.Tensor): Metric depth map of shape (B, 1, H, W) + - bin_centers (torch.Tensor): Bin centers of shape (B, n_bins). Present only if return_final_centers is True + - probs (torch.Tensor): Output probability distribution of shape (B, n_bins, H, W). Present only if return_probs is True + + """ + b, c, h, w = x.shape + # print("input shape ", x.shape) + self.orig_input_width = w + self.orig_input_height = h + rel_depth, out = self.core(x, denorm=denorm, return_rel_depth=True) + # print("output shapes", rel_depth.shape, out.shape) + + outconv_activation = out[0] + btlnck = out[1] + x_blocks = out[2:] + + x_d0 = self.conv2(btlnck) + x = x_d0 + _, seed_b_centers = self.seed_bin_regressor(x) + + if self.bin_centers_type == 'normed' or self.bin_centers_type == 'hybrid2': + b_prev = (seed_b_centers - self.min_depth) / \ + (self.max_depth - self.min_depth) + else: + b_prev = seed_b_centers + + prev_b_embedding = self.seed_projector(x) + + # unroll this loop for better performance + for projector, attractor, x in zip(self.projectors, self.attractors, x_blocks): + b_embedding = projector(x) + b, b_centers = attractor( + b_embedding, b_prev, prev_b_embedding, interpolate=True) + b_prev = b.clone() + prev_b_embedding = b_embedding.clone() + + last = outconv_activation + + if self.inverse_midas: + # invert depth followed by normalization + rel_depth = 1.0 / (rel_depth + 1e-6) + rel_depth = (rel_depth - rel_depth.min()) / \ + (rel_depth.max() - rel_depth.min()) + # concat rel depth with last. First interpolate rel depth to last size + rel_cond = rel_depth.unsqueeze(1) + rel_cond = nn.functional.interpolate( + rel_cond, size=last.shape[2:], mode='bilinear', align_corners=True) + last = torch.cat([last, rel_cond], dim=1) + + b_embedding = nn.functional.interpolate( + b_embedding, last.shape[-2:], mode='bilinear', align_corners=True) + x = self.conditional_log_binomial(last, b_embedding) + + # Now depth value is Sum px * cx , where cx are bin_centers from the last bin tensor + # print(x.shape, b_centers.shape) + b_centers = nn.functional.interpolate( + b_centers, x.shape[-2:], mode='bilinear', align_corners=True) + out = torch.sum(x * b_centers, dim=1, keepdim=True) + + # Structure output dict + output = dict(metric_depth=out) + if return_final_centers or return_probs: + output['bin_centers'] = b_centers + + if return_probs: + output['probs'] = x + + return output + + def get_lr_params(self, lr): + """ + Learning rate configuration for different layers of the model + Args: + lr (float) : Base learning rate + Returns: + list : list of parameters to optimize and their learning rates, in the format required by torch optimizers. + """ + param_conf = [] + if self.train_midas: + if self.encoder_lr_factor > 0: + param_conf.append({'params': self.core.get_enc_params_except_rel_pos( + ), 'lr': lr / self.encoder_lr_factor}) + + if self.pos_enc_lr_factor > 0: + param_conf.append( + {'params': self.core.get_rel_pos_params(), 'lr': lr / self.pos_enc_lr_factor}) + + midas_params = self.core.core.scratch.parameters() + midas_lr_factor = self.midas_lr_factor + param_conf.append( + {'params': midas_params, 'lr': lr / midas_lr_factor}) + + remaining_modules = [] + for name, child in self.named_children(): + if name != 'core': + remaining_modules.append(child) + remaining_params = itertools.chain( + *[child.parameters() for child in remaining_modules]) + + param_conf.append({'params': remaining_params, 'lr': lr}) + + return param_conf + + @staticmethod + def build(midas_model_type="DPT_BEiT_L_384", pretrained_resource=None, use_pretrained_midas=False, train_midas=False, freeze_midas_bn=True, **kwargs): + core = MidasCore.build(midas_model_type=midas_model_type, use_pretrained_midas=use_pretrained_midas, + train_midas=train_midas, fetch_features=True, freeze_bn=freeze_midas_bn, **kwargs) + model = ZoeDepth(core, **kwargs) + if pretrained_resource: + assert isinstance(pretrained_resource, str), "pretrained_resource must be a string" + model = load_state_from_resource(model, pretrained_resource) + return model + + @staticmethod + def build_from_config(config): + return ZoeDepth.build(**config) diff --git a/annotator/zoe/zoedepth/models/zoedepth_nk/__init__.py b/annotator/zoe/zoedepth/models/zoedepth_nk/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..513a278b939c10c010e3c0250ec73544d5663886 --- /dev/null +++ b/annotator/zoe/zoedepth/models/zoedepth_nk/__init__.py @@ -0,0 +1,31 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +from .zoedepth_nk_v1 import ZoeDepthNK + +all_versions = { + "v1": ZoeDepthNK, +} + +get_version = lambda v : all_versions[v] \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/zoedepth_nk/config_zoedepth_nk.json b/annotator/zoe/zoedepth/models/zoedepth_nk/config_zoedepth_nk.json new file mode 100644 index 0000000000000000000000000000000000000000..42bab2a3ad159a09599a5aba270c491021a3cf1a --- /dev/null +++ b/annotator/zoe/zoedepth/models/zoedepth_nk/config_zoedepth_nk.json @@ -0,0 +1,67 @@ +{ + "model": { + "name": "ZoeDepthNK", + "version_name": "v1", + "bin_conf" : [ + { + "name": "nyu", + "n_bins": 64, + "min_depth": 1e-3, + "max_depth": 10.0 + }, + { + "name": "kitti", + "n_bins": 64, + "min_depth": 1e-3, + "max_depth": 80.0 + } + ], + "bin_embedding_dim": 128, + "bin_centers_type": "softplus", + "n_attractors":[16, 8, 4, 1], + "attractor_alpha": 1000, + "attractor_gamma": 2, + "attractor_kind" : "mean", + "attractor_type" : "inv", + "min_temp": 0.0212, + "max_temp": 50.0, + "memory_efficient": true, + "midas_model_type" : "DPT_BEiT_L_384", + "img_size": [384, 512] + }, + + "train": { + "train_midas": true, + "use_pretrained_midas": true, + "trainer": "zoedepth_nk", + "epochs": 5, + "bs": 16, + "optim_kwargs": {"lr": 0.0002512, "wd": 0.01}, + "sched_kwargs": {"div_factor": 1, "final_div_factor": 10000, "pct_start": 0.7, "three_phase":false, "cycle_momentum": true}, + "same_lr": false, + "w_si": 1, + "w_domain": 100, + "avoid_boundary": false, + "random_crop": false, + "input_width": 640, + "input_height": 480, + "w_grad": 0, + "w_reg": 0, + "midas_lr_factor": 10, + "encoder_lr_factor":10, + "pos_enc_lr_factor":10 + }, + + "infer": { + "train_midas": false, + "pretrained_resource": "url::https://github.com/isl-org/ZoeDepth/releases/download/v1.0/ZoeD_M12_NK.pt", + "use_pretrained_midas": false, + "force_keep_ar": true + }, + + "eval": { + "train_midas": false, + "pretrained_resource": "url::https://github.com/isl-org/ZoeDepth/releases/download/v1.0/ZoeD_M12_NK.pt", + "use_pretrained_midas": false + } +} \ No newline at end of file diff --git a/annotator/zoe/zoedepth/models/zoedepth_nk/zoedepth_nk_v1.py b/annotator/zoe/zoedepth/models/zoedepth_nk/zoedepth_nk_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..7368ae8031188a9f946d9d3f29633c96e791e68e --- /dev/null +++ b/annotator/zoe/zoedepth/models/zoedepth_nk/zoedepth_nk_v1.py @@ -0,0 +1,333 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import itertools + +import torch +import torch.nn as nn + +from zoedepth.models.depth_model import DepthModel +from zoedepth.models.base_models.midas import MidasCore +from zoedepth.models.layers.attractor import AttractorLayer, AttractorLayerUnnormed +from zoedepth.models.layers.dist_layers import ConditionalLogBinomial +from zoedepth.models.layers.localbins_layers import (Projector, SeedBinRegressor, + SeedBinRegressorUnnormed) +from zoedepth.models.layers.patch_transformer import PatchTransformerEncoder +from zoedepth.models.model_io import load_state_from_resource + + +class ZoeDepthNK(DepthModel): + def __init__(self, core, bin_conf, bin_centers_type="softplus", bin_embedding_dim=128, + n_attractors=[16, 8, 4, 1], attractor_alpha=300, attractor_gamma=2, attractor_kind='sum', attractor_type='exp', + min_temp=5, max_temp=50, + memory_efficient=False, train_midas=True, + is_midas_pretrained=True, midas_lr_factor=1, encoder_lr_factor=10, pos_enc_lr_factor=10, inverse_midas=False, **kwargs): + """ZoeDepthNK model. This is the version of ZoeDepth that has two metric heads and uses a learned router to route to experts. + + Args: + core (models.base_models.midas.MidasCore): The base midas model that is used for extraction of "relative" features + + bin_conf (List[dict]): A list of dictionaries that contain the bin configuration for each metric head. Each dictionary should contain the following keys: + "name" (str, typically same as the dataset name), "n_bins" (int), "min_depth" (float), "max_depth" (float) + + The length of this list determines the number of metric heads. + bin_centers_type (str, optional): "normed" or "softplus". Activation type used for bin centers. For "normed" bin centers, linear normalization trick is applied. This results in bounded bin centers. + For "softplus", softplus activation is used and thus are unbounded. Defaults to "normed". + bin_embedding_dim (int, optional): bin embedding dimension. Defaults to 128. + + n_attractors (List[int], optional): Number of bin attractors at decoder layers. Defaults to [16, 8, 4, 1]. + attractor_alpha (int, optional): Proportional attractor strength. Refer to models.layers.attractor for more details. Defaults to 300. + attractor_gamma (int, optional): Exponential attractor strength. Refer to models.layers.attractor for more details. Defaults to 2. + attractor_kind (str, optional): Attraction aggregation "sum" or "mean". Defaults to 'sum'. + attractor_type (str, optional): Type of attractor to use; "inv" (Inverse attractor) or "exp" (Exponential attractor). Defaults to 'exp'. + + min_temp (int, optional): Lower bound for temperature of output probability distribution. Defaults to 5. + max_temp (int, optional): Upper bound for temperature of output probability distribution. Defaults to 50. + + memory_efficient (bool, optional): Whether to use memory efficient version of attractor layers. Memory efficient version is slower but is recommended incase of multiple metric heads in order save GPU memory. Defaults to False. + + train_midas (bool, optional): Whether to train "core", the base midas model. Defaults to True. + is_midas_pretrained (bool, optional): Is "core" pretrained? Defaults to True. + midas_lr_factor (int, optional): Learning rate reduction factor for base midas model except its encoder and positional encodings. Defaults to 10. + encoder_lr_factor (int, optional): Learning rate reduction factor for the encoder in midas model. Defaults to 10. + pos_enc_lr_factor (int, optional): Learning rate reduction factor for positional encodings in the base midas model. Defaults to 10. + + """ + + super().__init__() + + self.core = core + self.bin_conf = bin_conf + self.min_temp = min_temp + self.max_temp = max_temp + self.memory_efficient = memory_efficient + self.train_midas = train_midas + self.is_midas_pretrained = is_midas_pretrained + self.midas_lr_factor = midas_lr_factor + self.encoder_lr_factor = encoder_lr_factor + self.pos_enc_lr_factor = pos_enc_lr_factor + self.inverse_midas = inverse_midas + + N_MIDAS_OUT = 32 + btlnck_features = self.core.output_channels[0] + num_out_features = self.core.output_channels[1:] + # self.scales = [16, 8, 4, 2] # spatial scale factors + + self.conv2 = nn.Conv2d( + btlnck_features, btlnck_features, kernel_size=1, stride=1, padding=0) + + # Transformer classifier on the bottleneck + self.patch_transformer = PatchTransformerEncoder( + btlnck_features, 1, 128, use_class_token=True) + self.mlp_classifier = nn.Sequential( + nn.Linear(128, 128), + nn.ReLU(), + nn.Linear(128, 2) + ) + + if bin_centers_type == "normed": + SeedBinRegressorLayer = SeedBinRegressor + Attractor = AttractorLayer + elif bin_centers_type == "softplus": + SeedBinRegressorLayer = SeedBinRegressorUnnormed + Attractor = AttractorLayerUnnormed + elif bin_centers_type == "hybrid1": + SeedBinRegressorLayer = SeedBinRegressor + Attractor = AttractorLayerUnnormed + elif bin_centers_type == "hybrid2": + SeedBinRegressorLayer = SeedBinRegressorUnnormed + Attractor = AttractorLayer + else: + raise ValueError( + "bin_centers_type should be one of 'normed', 'softplus', 'hybrid1', 'hybrid2'") + self.bin_centers_type = bin_centers_type + # We have bins for each bin conf. + # Create a map (ModuleDict) of 'name' -> seed_bin_regressor + self.seed_bin_regressors = nn.ModuleDict( + {conf['name']: SeedBinRegressorLayer(btlnck_features, conf["n_bins"], mlp_dim=bin_embedding_dim//2, min_depth=conf["min_depth"], max_depth=conf["max_depth"]) + for conf in bin_conf} + ) + + self.seed_projector = Projector( + btlnck_features, bin_embedding_dim, mlp_dim=bin_embedding_dim//2) + self.projectors = nn.ModuleList([ + Projector(num_out, bin_embedding_dim, mlp_dim=bin_embedding_dim//2) + for num_out in num_out_features + ]) + + # Create a map (ModuleDict) of 'name' -> attractors (ModuleList) + self.attractors = nn.ModuleDict( + {conf['name']: nn.ModuleList([ + Attractor(bin_embedding_dim, n_attractors[i], + mlp_dim=bin_embedding_dim, alpha=attractor_alpha, + gamma=attractor_gamma, kind=attractor_kind, + attractor_type=attractor_type, memory_efficient=memory_efficient, + min_depth=conf["min_depth"], max_depth=conf["max_depth"]) + for i in range(len(n_attractors)) + ]) + for conf in bin_conf} + ) + + last_in = N_MIDAS_OUT + # conditional log binomial for each bin conf + self.conditional_log_binomial = nn.ModuleDict( + {conf['name']: ConditionalLogBinomial(last_in, bin_embedding_dim, conf['n_bins'], bottleneck_factor=4, min_temp=self.min_temp, max_temp=self.max_temp) + for conf in bin_conf} + ) + + def forward(self, x, return_final_centers=False, denorm=False, return_probs=False, **kwargs): + """ + Args: + x (torch.Tensor): Input image tensor of shape (B, C, H, W). Assumes all images are from the same domain. + return_final_centers (bool, optional): Whether to return the final centers of the attractors. Defaults to False. + denorm (bool, optional): Whether to denormalize the input image. Defaults to False. + return_probs (bool, optional): Whether to return the probabilities of the bins. Defaults to False. + + Returns: + dict: Dictionary of outputs with keys: + - "rel_depth": Relative depth map of shape (B, 1, H, W) + - "metric_depth": Metric depth map of shape (B, 1, H, W) + - "domain_logits": Domain logits of shape (B, 2) + - "bin_centers": Bin centers of shape (B, N, H, W). Present only if return_final_centers is True + - "probs": Bin probabilities of shape (B, N, H, W). Present only if return_probs is True + """ + b, c, h, w = x.shape + self.orig_input_width = w + self.orig_input_height = h + rel_depth, out = self.core(x, denorm=denorm, return_rel_depth=True) + + outconv_activation = out[0] + btlnck = out[1] + x_blocks = out[2:] + + x_d0 = self.conv2(btlnck) + x = x_d0 + + # Predict which path to take + embedding = self.patch_transformer(x)[0] # N, E + domain_logits = self.mlp_classifier(embedding) # N, 2 + domain_vote = torch.softmax(domain_logits.sum( + dim=0, keepdim=True), dim=-1) # 1, 2 + + # Get the path + bin_conf_name = ["nyu", "kitti"][torch.argmax( + domain_vote, dim=-1).squeeze().item()] + + try: + conf = [c for c in self.bin_conf if c.name == bin_conf_name][0] + except IndexError: + raise ValueError( + f"bin_conf_name {bin_conf_name} not found in bin_confs") + + min_depth = conf['min_depth'] + max_depth = conf['max_depth'] + + seed_bin_regressor = self.seed_bin_regressors[bin_conf_name] + _, seed_b_centers = seed_bin_regressor(x) + if self.bin_centers_type == 'normed' or self.bin_centers_type == 'hybrid2': + b_prev = (seed_b_centers - min_depth)/(max_depth - min_depth) + else: + b_prev = seed_b_centers + prev_b_embedding = self.seed_projector(x) + + attractors = self.attractors[bin_conf_name] + for projector, attractor, x in zip(self.projectors, attractors, x_blocks): + b_embedding = projector(x) + b, b_centers = attractor( + b_embedding, b_prev, prev_b_embedding, interpolate=True) + b_prev = b + prev_b_embedding = b_embedding + + last = outconv_activation + + b_centers = nn.functional.interpolate( + b_centers, last.shape[-2:], mode='bilinear', align_corners=True) + b_embedding = nn.functional.interpolate( + b_embedding, last.shape[-2:], mode='bilinear', align_corners=True) + + clb = self.conditional_log_binomial[bin_conf_name] + x = clb(last, b_embedding) + + # Now depth value is Sum px * cx , where cx are bin_centers from the last bin tensor + # print(x.shape, b_centers.shape) + # b_centers = nn.functional.interpolate(b_centers, x.shape[-2:], mode='bilinear', align_corners=True) + out = torch.sum(x * b_centers, dim=1, keepdim=True) + + output = dict(domain_logits=domain_logits, metric_depth=out) + if return_final_centers or return_probs: + output['bin_centers'] = b_centers + + if return_probs: + output['probs'] = x + return output + + def get_lr_params(self, lr): + """ + Learning rate configuration for different layers of the model + + Args: + lr (float) : Base learning rate + Returns: + list : list of parameters to optimize and their learning rates, in the format required by torch optimizers. + """ + param_conf = [] + if self.train_midas: + def get_rel_pos_params(): + for name, p in self.core.core.pretrained.named_parameters(): + if "relative_position" in name: + yield p + + def get_enc_params_except_rel_pos(): + for name, p in self.core.core.pretrained.named_parameters(): + if "relative_position" not in name: + yield p + + encoder_params = get_enc_params_except_rel_pos() + rel_pos_params = get_rel_pos_params() + midas_params = self.core.core.scratch.parameters() + midas_lr_factor = self.midas_lr_factor if self.is_midas_pretrained else 1.0 + param_conf.extend([ + {'params': encoder_params, 'lr': lr / self.encoder_lr_factor}, + {'params': rel_pos_params, 'lr': lr / self.pos_enc_lr_factor}, + {'params': midas_params, 'lr': lr / midas_lr_factor} + ]) + + remaining_modules = [] + for name, child in self.named_children(): + if name != 'core': + remaining_modules.append(child) + remaining_params = itertools.chain( + *[child.parameters() for child in remaining_modules]) + param_conf.append({'params': remaining_params, 'lr': lr}) + return param_conf + + def get_conf_parameters(self, conf_name): + """ + Returns parameters of all the ModuleDicts children that are exclusively used for the given bin configuration + """ + params = [] + for name, child in self.named_children(): + if isinstance(child, nn.ModuleDict): + for bin_conf_name, module in child.items(): + if bin_conf_name == conf_name: + params += list(module.parameters()) + return params + + def freeze_conf(self, conf_name): + """ + Freezes all the parameters of all the ModuleDicts children that are exclusively used for the given bin configuration + """ + for p in self.get_conf_parameters(conf_name): + p.requires_grad = False + + def unfreeze_conf(self, conf_name): + """ + Unfreezes all the parameters of all the ModuleDicts children that are exclusively used for the given bin configuration + """ + for p in self.get_conf_parameters(conf_name): + p.requires_grad = True + + def freeze_all_confs(self): + """ + Freezes all the parameters of all the ModuleDicts children + """ + for name, child in self.named_children(): + if isinstance(child, nn.ModuleDict): + for bin_conf_name, module in child.items(): + for p in module.parameters(): + p.requires_grad = False + + @staticmethod + def build(midas_model_type="DPT_BEiT_L_384", pretrained_resource=None, use_pretrained_midas=False, train_midas=False, freeze_midas_bn=True, **kwargs): + core = MidasCore.build(midas_model_type=midas_model_type, use_pretrained_midas=use_pretrained_midas, + train_midas=train_midas, fetch_features=True, freeze_bn=freeze_midas_bn, **kwargs) + model = ZoeDepthNK(core, **kwargs) + if pretrained_resource: + assert isinstance(pretrained_resource, str), "pretrained_resource must be a string" + model = load_state_from_resource(model, pretrained_resource) + return model + + @staticmethod + def build_from_config(config): + return ZoeDepthNK.build(**config) diff --git a/annotator/zoe/zoedepth/trainers/base_trainer.py b/annotator/zoe/zoedepth/trainers/base_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..982271270967c75ee9092b69140a7067dacc39ce --- /dev/null +++ b/annotator/zoe/zoedepth/trainers/base_trainer.py @@ -0,0 +1,327 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import os +import uuid +import warnings +from datetime import datetime as dt +from typing import Dict + +import matplotlib.pyplot as plt +import numpy as np +import torch +import torch.distributed as dist +import torch.nn as nn +import torch.optim as optim +import wandb +from tqdm import tqdm + +from zoedepth.utils.config import flatten +from zoedepth.utils.misc import RunningAverageDict, colorize, colors + + +def is_rank_zero(args): + return args.rank == 0 + + +class BaseTrainer: + def __init__(self, config, model, train_loader, test_loader=None, device=None): + """ Base Trainer class for training a model.""" + + self.config = config + self.metric_criterion = "abs_rel" + if device is None: + device = torch.device( + 'cuda') if torch.cuda.is_available() else torch.device('cpu') +# device = torch.device('cpu') + self.device = device + self.model = model + self.train_loader = train_loader + self.test_loader = test_loader + self.optimizer = self.init_optimizer() + self.scheduler = self.init_scheduler() + + def resize_to_target(self, prediction, target): + if prediction.shape[2:] != target.shape[-2:]: + prediction = nn.functional.interpolate( + prediction, size=target.shape[-2:], mode="bilinear", align_corners=True + ) + return prediction + + def load_ckpt(self, checkpoint_dir="./checkpoints", ckpt_type="best"): + import glob + import os + + from zoedepth.models.model_io import load_wts + + if hasattr(self.config, "checkpoint"): + checkpoint = self.config.checkpoint + elif hasattr(self.config, "ckpt_pattern"): + pattern = self.config.ckpt_pattern + matches = glob.glob(os.path.join( + checkpoint_dir, f"*{pattern}*{ckpt_type}*")) + if not (len(matches) > 0): + raise ValueError(f"No matches found for the pattern {pattern}") + checkpoint = matches[0] + else: + return + model = load_wts(self.model, checkpoint) + # TODO : Resuming training is not properly supported in this repo. Implement loading / saving of optimizer and scheduler to support it. + print("Loaded weights from {0}".format(checkpoint)) + warnings.warn( + "Resuming training is not properly supported in this repo. Implement loading / saving of optimizer and scheduler to support it.") + self.model = model + + def init_optimizer(self): + m = self.model.module if self.config.multigpu else self.model + + if self.config.same_lr: + print("Using same LR") + if hasattr(m, 'core'): + m.core.unfreeze() + params = self.model.parameters() + else: + print("Using diff LR") + if not hasattr(m, 'get_lr_params'): + raise NotImplementedError( + f"Model {m.__class__.__name__} does not implement get_lr_params. Please implement it or use the same LR for all parameters.") + + params = m.get_lr_params(self.config.lr) + + return optim.AdamW(params, lr=self.config.lr, weight_decay=self.config.wd) + + def init_scheduler(self): + lrs = [l['lr'] for l in self.optimizer.param_groups] + return optim.lr_scheduler.OneCycleLR(self.optimizer, lrs, epochs=self.config.epochs, steps_per_epoch=len(self.train_loader), + cycle_momentum=self.config.cycle_momentum, + base_momentum=0.85, max_momentum=0.95, div_factor=self.config.div_factor, final_div_factor=self.config.final_div_factor, pct_start=self.config.pct_start, three_phase=self.config.three_phase) + + def train_on_batch(self, batch, train_step): + raise NotImplementedError + + def validate_on_batch(self, batch, val_step): + raise NotImplementedError + + def raise_if_nan(self, losses): + for key, value in losses.items(): + if torch.isnan(value): + raise ValueError(f"{key} is NaN, Stopping training") + + @property + def iters_per_epoch(self): + return len(self.train_loader) + + @property + def total_iters(self): + return self.config.epochs * self.iters_per_epoch + + def should_early_stop(self): + if self.config.get('early_stop', False) and self.step > self.config.early_stop: + return True + + def train(self): + print(f"Training {self.config.name}") + if self.config.uid is None: + self.config.uid = str(uuid.uuid4()).split('-')[-1] + run_id = f"{dt.now().strftime('%d-%h_%H-%M')}-{self.config.uid}" + self.config.run_id = run_id + self.config.experiment_id = f"{self.config.name}{self.config.version_name}_{run_id}" + self.should_write = ((not self.config.distributed) + or self.config.rank == 0) + self.should_log = self.should_write # and logging + if self.should_log: + tags = self.config.tags.split( + ',') if self.config.tags != '' else None + wandb.init(project=self.config.project, name=self.config.experiment_id, config=flatten(self.config), dir=self.config.root, + tags=tags, notes=self.config.notes, settings=wandb.Settings(start_method="fork")) + + self.model.train() + self.step = 0 + best_loss = np.inf + validate_every = int(self.config.validate_every * self.iters_per_epoch) + + + if self.config.prefetch: + + for i, batch in tqdm(enumerate(self.train_loader), desc=f"Prefetching...", + total=self.iters_per_epoch) if is_rank_zero(self.config) else enumerate(self.train_loader): + pass + + losses = {} + def stringify_losses(L): return "; ".join(map( + lambda kv: f"{colors.fg.purple}{kv[0]}{colors.reset}: {round(kv[1].item(),3):.4e}", L.items())) + for epoch in range(self.config.epochs): + if self.should_early_stop(): + break + + self.epoch = epoch + ################################# Train loop ########################################################## + if self.should_log: + wandb.log({"Epoch": epoch}, step=self.step) + pbar = tqdm(enumerate(self.train_loader), desc=f"Epoch: {epoch + 1}/{self.config.epochs}. Loop: Train", + total=self.iters_per_epoch) if is_rank_zero(self.config) else enumerate(self.train_loader) + for i, batch in pbar: + if self.should_early_stop(): + print("Early stopping") + break + # print(f"Batch {self.step+1} on rank {self.config.rank}") + losses = self.train_on_batch(batch, i) + # print(f"trained batch {self.step+1} on rank {self.config.rank}") + + self.raise_if_nan(losses) + if is_rank_zero(self.config) and self.config.print_losses: + pbar.set_description( + f"Epoch: {epoch + 1}/{self.config.epochs}. Loop: Train. Losses: {stringify_losses(losses)}") + self.scheduler.step() + + if self.should_log and self.step % 50 == 0: + wandb.log({f"Train/{name}": loss.item() + for name, loss in losses.items()}, step=self.step) + + self.step += 1 + + ######################################################################################################## + + if self.test_loader: + if (self.step % validate_every) == 0: + self.model.eval() + if self.should_write: + self.save_checkpoint( + f"{self.config.experiment_id}_latest.pt") + + ################################# Validation loop ################################################## + # validate on the entire validation set in every process but save only from rank 0, I know, inefficient, but avoids divergence of processes + metrics, test_losses = self.validate() + # print("Validated: {}".format(metrics)) + if self.should_log: + wandb.log( + {f"Test/{name}": tloss for name, tloss in test_losses.items()}, step=self.step) + + wandb.log({f"Metrics/{k}": v for k, + v in metrics.items()}, step=self.step) + + if (metrics[self.metric_criterion] < best_loss) and self.should_write: + self.save_checkpoint( + f"{self.config.experiment_id}_best.pt") + best_loss = metrics[self.metric_criterion] + + self.model.train() + + if self.config.distributed: + dist.barrier() + # print(f"Validated: {metrics} on device {self.config.rank}") + + # print(f"Finished step {self.step} on device {self.config.rank}") + ################################################################################################# + + # Save / validate at the end + self.step += 1 # log as final point + self.model.eval() + self.save_checkpoint(f"{self.config.experiment_id}_latest.pt") + if self.test_loader: + + ################################# Validation loop ################################################## + metrics, test_losses = self.validate() + # print("Validated: {}".format(metrics)) + if self.should_log: + wandb.log({f"Test/{name}": tloss for name, + tloss in test_losses.items()}, step=self.step) + wandb.log({f"Metrics/{k}": v for k, + v in metrics.items()}, step=self.step) + + if (metrics[self.metric_criterion] < best_loss) and self.should_write: + self.save_checkpoint( + f"{self.config.experiment_id}_best.pt") + best_loss = metrics[self.metric_criterion] + + self.model.train() + + def validate(self): + with torch.no_grad(): + losses_avg = RunningAverageDict() + metrics_avg = RunningAverageDict() + for i, batch in tqdm(enumerate(self.test_loader), desc=f"Epoch: {self.epoch + 1}/{self.config.epochs}. Loop: Validation", total=len(self.test_loader), disable=not is_rank_zero(self.config)): + metrics, losses = self.validate_on_batch(batch, val_step=i) + + if losses: + losses_avg.update(losses) + if metrics: + metrics_avg.update(metrics) + + return metrics_avg.get_value(), losses_avg.get_value() + + def save_checkpoint(self, filename): + if not self.should_write: + return + root = self.config.save_dir + if not os.path.isdir(root): + os.makedirs(root) + + fpath = os.path.join(root, filename) + m = self.model.module if self.config.multigpu else self.model + torch.save( + { + "model": m.state_dict(), + "optimizer": None, # TODO : Change to self.optimizer.state_dict() if resume support is needed, currently None to reduce file size + "epoch": self.epoch + }, fpath) + + def log_images(self, rgb: Dict[str, list] = {}, depth: Dict[str, list] = {}, scalar_field: Dict[str, list] = {}, prefix="", scalar_cmap="jet", min_depth=None, max_depth=None): + if not self.should_log: + return + + if min_depth is None: + try: + min_depth = self.config.min_depth + max_depth = self.config.max_depth + except AttributeError: + min_depth = None + max_depth = None + + depth = {k: colorize(v, vmin=min_depth, vmax=max_depth) + for k, v in depth.items()} + scalar_field = {k: colorize( + v, vmin=None, vmax=None, cmap=scalar_cmap) for k, v in scalar_field.items()} + images = {**rgb, **depth, **scalar_field} + wimages = { + prefix+"Predictions": [wandb.Image(v, caption=k) for k, v in images.items()]} + wandb.log(wimages, step=self.step) + + def log_line_plot(self, data): + if not self.should_log: + return + + plt.plot(data) + plt.ylabel("Scale factors") + wandb.log({"Scale factors": wandb.Image(plt)}, step=self.step) + plt.close() + + def log_bar_plot(self, title, labels, values): + if not self.should_log: + return + + data = [[label, val] for (label, val) in zip(labels, values)] + table = wandb.Table(data=data, columns=["label", "value"]) + wandb.log({title: wandb.plot.bar(table, "label", + "value", title=title)}, step=self.step) diff --git a/annotator/zoe/zoedepth/trainers/builder.py b/annotator/zoe/zoedepth/trainers/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..a663541b08912ebedce21a68c7599ce4c06e85d0 --- /dev/null +++ b/annotator/zoe/zoedepth/trainers/builder.py @@ -0,0 +1,48 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +from importlib import import_module + + +def get_trainer(config): + """Builds and returns a trainer based on the config. + + Args: + config (dict): the config dict (typically constructed using utils.config.get_config) + config.trainer (str): the name of the trainer to use. The module named "{config.trainer}_trainer" must exist in trainers root module + + Raises: + ValueError: If the specified trainer does not exist under trainers/ folder + + Returns: + Trainer (inherited from zoedepth.trainers.BaseTrainer): The Trainer object + """ + assert "trainer" in config and config.trainer is not None and config.trainer != '', "Trainer not specified. Config: {0}".format( + config) + try: + Trainer = getattr(import_module( + f"zoedepth.trainers.{config.trainer}_trainer"), 'Trainer') + except ModuleNotFoundError as e: + raise ValueError(f"Trainer {config.trainer}_trainer not found.") from e + return Trainer diff --git a/annotator/zoe/zoedepth/trainers/loss.py b/annotator/zoe/zoedepth/trainers/loss.py new file mode 100644 index 0000000000000000000000000000000000000000..0c5a1c15cdf5628c1474c566fdc6e58159d7f5ab --- /dev/null +++ b/annotator/zoe/zoedepth/trainers/loss.py @@ -0,0 +1,316 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.cuda.amp as amp +import numpy as np + + +KEY_OUTPUT = 'metric_depth' + + +def extract_key(prediction, key): + if isinstance(prediction, dict): + return prediction[key] + return prediction + + +# Main loss function used for ZoeDepth. Copy/paste from AdaBins repo (https://github.com/shariqfarooq123/AdaBins/blob/0952d91e9e762be310bb4cd055cbfe2448c0ce20/loss.py#L7) +class SILogLoss(nn.Module): + """SILog loss (pixel-wise)""" + def __init__(self, beta=0.15): + super(SILogLoss, self).__init__() + self.name = 'SILog' + self.beta = beta + + def forward(self, input, target, mask=None, interpolate=True, return_interpolated=False): + input = extract_key(input, KEY_OUTPUT) + if input.shape[-1] != target.shape[-1] and interpolate: + input = nn.functional.interpolate( + input, target.shape[-2:], mode='bilinear', align_corners=True) + intr_input = input + else: + intr_input = input + + if target.ndim == 3: + target = target.unsqueeze(1) + + if mask is not None: + if mask.ndim == 3: + mask = mask.unsqueeze(1) + + input = input[mask] + target = target[mask] + + with amp.autocast(enabled=False): # amp causes NaNs in this loss function + alpha = 1e-7 + g = torch.log(input + alpha) - torch.log(target + alpha) + + # n, c, h, w = g.shape + # norm = 1/(h*w) + # Dg = norm * torch.sum(g**2) - (0.85/(norm**2)) * (torch.sum(g))**2 + + Dg = torch.var(g) + self.beta * torch.pow(torch.mean(g), 2) + + loss = 10 * torch.sqrt(Dg) + + if torch.isnan(loss): + print("Nan SILog loss") + print("input:", input.shape) + print("target:", target.shape) + print("G", torch.sum(torch.isnan(g))) + print("Input min max", torch.min(input), torch.max(input)) + print("Target min max", torch.min(target), torch.max(target)) + print("Dg", torch.isnan(Dg)) + print("loss", torch.isnan(loss)) + + if not return_interpolated: + return loss + + return loss, intr_input + + +def grad(x): + # x.shape : n, c, h, w + diff_x = x[..., 1:, 1:] - x[..., 1:, :-1] + diff_y = x[..., 1:, 1:] - x[..., :-1, 1:] + mag = diff_x**2 + diff_y**2 + # angle_ratio + angle = torch.atan(diff_y / (diff_x + 1e-10)) + return mag, angle + + +def grad_mask(mask): + return mask[..., 1:, 1:] & mask[..., 1:, :-1] & mask[..., :-1, 1:] + + +class GradL1Loss(nn.Module): + """Gradient loss""" + def __init__(self): + super(GradL1Loss, self).__init__() + self.name = 'GradL1' + + def forward(self, input, target, mask=None, interpolate=True, return_interpolated=False): + input = extract_key(input, KEY_OUTPUT) + if input.shape[-1] != target.shape[-1] and interpolate: + input = nn.functional.interpolate( + input, target.shape[-2:], mode='bilinear', align_corners=True) + intr_input = input + else: + intr_input = input + + grad_gt = grad(target) + grad_pred = grad(input) + mask_g = grad_mask(mask) + + loss = nn.functional.l1_loss(grad_pred[0][mask_g], grad_gt[0][mask_g]) + loss = loss + \ + nn.functional.l1_loss(grad_pred[1][mask_g], grad_gt[1][mask_g]) + if not return_interpolated: + return loss + return loss, intr_input + + +class OrdinalRegressionLoss(object): + + def __init__(self, ord_num, beta, discretization="SID"): + self.ord_num = ord_num + self.beta = beta + self.discretization = discretization + + def _create_ord_label(self, gt): + N,one, H, W = gt.shape + # print("gt shape:", gt.shape) + + ord_c0 = torch.ones(N, self.ord_num, H, W).to(gt.device) + if self.discretization == "SID": + label = self.ord_num * torch.log(gt) / np.log(self.beta) + else: + label = self.ord_num * (gt - 1.0) / (self.beta - 1.0) + label = label.long() + mask = torch.linspace(0, self.ord_num - 1, self.ord_num, requires_grad=False) \ + .view(1, self.ord_num, 1, 1).to(gt.device) + mask = mask.repeat(N, 1, H, W).contiguous().long() + mask = (mask > label) + ord_c0[mask] = 0 + ord_c1 = 1 - ord_c0 + # implementation according to the paper. + # ord_label = torch.ones(N, self.ord_num * 2, H, W).to(gt.device) + # ord_label[:, 0::2, :, :] = ord_c0 + # ord_label[:, 1::2, :, :] = ord_c1 + # reimplementation for fast speed. + ord_label = torch.cat((ord_c0, ord_c1), dim=1) + return ord_label, mask + + def __call__(self, prob, gt): + """ + :param prob: ordinal regression probability, N x 2*Ord Num x H x W, torch.Tensor + :param gt: depth ground truth, NXHxW, torch.Tensor + :return: loss: loss value, torch.float + """ + # N, C, H, W = prob.shape + valid_mask = gt > 0. + ord_label, mask = self._create_ord_label(gt) + # print("prob shape: {}, ord label shape: {}".format(prob.shape, ord_label.shape)) + entropy = -prob * ord_label + loss = torch.sum(entropy, dim=1)[valid_mask.squeeze(1)] + return loss.mean() + + +class DiscreteNLLLoss(nn.Module): + """Cross entropy loss""" + def __init__(self, min_depth=1e-3, max_depth=10, depth_bins=64): + super(DiscreteNLLLoss, self).__init__() + self.name = 'CrossEntropy' + self.ignore_index = -(depth_bins + 1) + # self._loss_func = nn.NLLLoss(ignore_index=self.ignore_index) + self._loss_func = nn.CrossEntropyLoss(ignore_index=self.ignore_index) + self.min_depth = min_depth + self.max_depth = max_depth + self.depth_bins = depth_bins + self.alpha = 1 + self.zeta = 1 - min_depth + self.beta = max_depth + self.zeta + + def quantize_depth(self, depth): + # depth : N1HW + # output : NCHW + + # Quantize depth log-uniformly on [1, self.beta] into self.depth_bins bins + depth = torch.log(depth / self.alpha) / np.log(self.beta / self.alpha) + depth = depth * (self.depth_bins - 1) + depth = torch.round(depth) + depth = depth.long() + return depth + + + + def _dequantize_depth(self, depth): + """ + Inverse of quantization + depth : NCHW -> N1HW + """ + # Get the center of the bin + + + + + def forward(self, input, target, mask=None, interpolate=True, return_interpolated=False): + input = extract_key(input, KEY_OUTPUT) + # assert torch.all(input <= 0), "Input should be negative" + + if input.shape[-1] != target.shape[-1] and interpolate: + input = nn.functional.interpolate( + input, target.shape[-2:], mode='bilinear', align_corners=True) + intr_input = input + else: + intr_input = input + + # assert torch.all(input)<=1) + if target.ndim == 3: + target = target.unsqueeze(1) + + target = self.quantize_depth(target) + if mask is not None: + if mask.ndim == 3: + mask = mask.unsqueeze(1) + + # Set the mask to ignore_index + mask = mask.long() + input = input * mask + (1 - mask) * self.ignore_index + target = target * mask + (1 - mask) * self.ignore_index + + + + input = input.flatten(2) # N, nbins, H*W + target = target.flatten(1) # N, H*W + loss = self._loss_func(input, target) + + if not return_interpolated: + return loss + return loss, intr_input + + + + +def compute_scale_and_shift(prediction, target, mask): + # system matrix: A = [[a_00, a_01], [a_10, a_11]] + a_00 = torch.sum(mask * prediction * prediction, (1, 2)) + a_01 = torch.sum(mask * prediction, (1, 2)) + a_11 = torch.sum(mask, (1, 2)) + + # right hand side: b = [b_0, b_1] + b_0 = torch.sum(mask * prediction * target, (1, 2)) + b_1 = torch.sum(mask * target, (1, 2)) + + # solution: x = A^-1 . b = [[a_11, -a_01], [-a_10, a_00]] / (a_00 * a_11 - a_01 * a_10) . b + x_0 = torch.zeros_like(b_0) + x_1 = torch.zeros_like(b_1) + + det = a_00 * a_11 - a_01 * a_01 + # A needs to be a positive definite matrix. + valid = det > 0 + + x_0[valid] = (a_11[valid] * b_0[valid] - a_01[valid] * b_1[valid]) / det[valid] + x_1[valid] = (-a_01[valid] * b_0[valid] + a_00[valid] * b_1[valid]) / det[valid] + + return x_0, x_1 +class ScaleAndShiftInvariantLoss(nn.Module): + def __init__(self): + super().__init__() + self.name = "SSILoss" + + def forward(self, prediction, target, mask, interpolate=True, return_interpolated=False): + + if prediction.shape[-1] != target.shape[-1] and interpolate: + prediction = nn.functional.interpolate(prediction, target.shape[-2:], mode='bilinear', align_corners=True) + intr_input = prediction + else: + intr_input = prediction + + + prediction, target, mask = prediction.squeeze(), target.squeeze(), mask.squeeze() + assert prediction.shape == target.shape, f"Shape mismatch: Expected same shape but got {prediction.shape} and {target.shape}." + + scale, shift = compute_scale_and_shift(prediction, target, mask) + + scaled_prediction = scale.view(-1, 1, 1) * prediction + shift.view(-1, 1, 1) + + loss = nn.functional.l1_loss(scaled_prediction[mask], target[mask]) + if not return_interpolated: + return loss + return loss, intr_input + + + + +if __name__ == '__main__': + # Tests for DiscreteNLLLoss + celoss = DiscreteNLLLoss() + print(celoss(torch.rand(4, 64, 26, 32)*10, torch.rand(4, 1, 26, 32)*10, )) + + d = torch.Tensor([6.59, 3.8, 10.0]) + print(celoss.dequantize_depth(celoss.quantize_depth(d))) diff --git a/annotator/zoe/zoedepth/trainers/zoedepth_nk_trainer.py b/annotator/zoe/zoedepth/trainers/zoedepth_nk_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..d528ae126f1c51b2f25fd31f94a39591ceb2f43a --- /dev/null +++ b/annotator/zoe/zoedepth/trainers/zoedepth_nk_trainer.py @@ -0,0 +1,143 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import torch +import torch.cuda.amp as amp +import torch.nn as nn + +from zoedepth.trainers.loss import GradL1Loss, SILogLoss +from zoedepth.utils.config import DATASETS_CONFIG +from zoedepth.utils.misc import compute_metrics + +from .base_trainer import BaseTrainer + + +class Trainer(BaseTrainer): + def __init__(self, config, model, train_loader, test_loader=None, device=None): + super().__init__(config, model, train_loader, + test_loader=test_loader, device=device) + self.device = device + self.silog_loss = SILogLoss() + self.grad_loss = GradL1Loss() + self.domain_classifier_loss = nn.CrossEntropyLoss() + + self.scaler = amp.GradScaler(enabled=self.config.use_amp) + + def train_on_batch(self, batch, train_step): + """ + Expects a batch of images and depth as input + batch["image"].shape : batch_size, c, h, w + batch["depth"].shape : batch_size, 1, h, w + + Assumes all images in a batch are from the same dataset + """ + + images, depths_gt = batch['image'].to( + self.device), batch['depth'].to(self.device) + # batch['dataset'] is a tensor strings all valued either 'nyu' or 'kitti'. labels nyu -> 0, kitti -> 1 + dataset = batch['dataset'][0] + # Convert to 0s or 1s + domain_labels = torch.Tensor([dataset == 'kitti' for _ in range( + images.size(0))]).to(torch.long).to(self.device) + + # m = self.model.module if self.config.multigpu else self.model + + b, c, h, w = images.size() + mask = batch["mask"].to(self.device).to(torch.bool) + + losses = {} + + with amp.autocast(enabled=self.config.use_amp): + output = self.model(images) + pred_depths = output['metric_depth'] + domain_logits = output['domain_logits'] + + l_si, pred = self.silog_loss( + pred_depths, depths_gt, mask=mask, interpolate=True, return_interpolated=True) + loss = self.config.w_si * l_si + losses[self.silog_loss.name] = l_si + + if self.config.w_grad > 0: + l_grad = self.grad_loss(pred, depths_gt, mask=mask) + loss = loss + self.config.w_grad * l_grad + losses[self.grad_loss.name] = l_grad + else: + l_grad = torch.Tensor([0]) + + if self.config.w_domain > 0: + l_domain = self.domain_classifier_loss( + domain_logits, domain_labels) + loss = loss + self.config.w_domain * l_domain + losses["DomainLoss"] = l_domain + else: + l_domain = torch.Tensor([0.]) + + self.scaler.scale(loss).backward() + + if self.config.clip_grad > 0: + self.scaler.unscale_(self.optimizer) + nn.utils.clip_grad_norm_( + self.model.parameters(), self.config.clip_grad) + + self.scaler.step(self.optimizer) + + if self.should_log and self.step > 1 and (self.step % int(self.config.log_images_every * self.iters_per_epoch)) == 0: + depths_gt[torch.logical_not(mask)] = -99 + self.log_images(rgb={"Input": images[0, ...]}, depth={"GT": depths_gt[0], "PredictedMono": pred[0]}, prefix="Train", + min_depth=DATASETS_CONFIG[dataset]['min_depth'], max_depth=DATASETS_CONFIG[dataset]['max_depth']) + + self.scaler.update() + self.optimizer.zero_grad(set_to_none=True) + + return losses + + def validate_on_batch(self, batch, val_step): + images = batch['image'].to(self.device) + depths_gt = batch['depth'].to(self.device) + dataset = batch['dataset'][0] + if 'has_valid_depth' in batch: + if not batch['has_valid_depth']: + return None, None + + depths_gt = depths_gt.squeeze().unsqueeze(0).unsqueeze(0) + with amp.autocast(enabled=self.config.use_amp): + m = self.model.module if self.config.multigpu else self.model + pred_depths = m(images)["metric_depth"] + pred_depths = pred_depths.squeeze().unsqueeze(0).unsqueeze(0) + + mask = torch.logical_and( + depths_gt > self.config.min_depth, depths_gt < self.config.max_depth) + with amp.autocast(enabled=self.config.use_amp): + l_depth = self.silog_loss( + pred_depths, depths_gt, mask=mask.to(torch.bool), interpolate=True) + + metrics = compute_metrics(depths_gt, pred_depths, **self.config) + losses = {f"{self.silog_loss.name}": l_depth.item()} + + if val_step == 1 and self.should_log: + depths_gt[torch.logical_not(mask)] = -99 + self.log_images(rgb={"Input": images[0]}, depth={"GT": depths_gt[0], "PredictedMono": pred_depths[0]}, prefix="Test", + min_depth=DATASETS_CONFIG[dataset]['min_depth'], max_depth=DATASETS_CONFIG[dataset]['max_depth']) + + return metrics, losses diff --git a/annotator/zoe/zoedepth/trainers/zoedepth_trainer.py b/annotator/zoe/zoedepth/trainers/zoedepth_trainer.py new file mode 100644 index 0000000000000000000000000000000000000000..3ac1c24c0512c1c1b191670a7c24abb4fca47ba1 --- /dev/null +++ b/annotator/zoe/zoedepth/trainers/zoedepth_trainer.py @@ -0,0 +1,177 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import torch +import torch.cuda.amp as amp +import torch.nn as nn + +from zoedepth.trainers.loss import GradL1Loss, SILogLoss +from zoedepth.utils.config import DATASETS_CONFIG +from zoedepth.utils.misc import compute_metrics +from zoedepth.data.preprocess import get_black_border + +from .base_trainer import BaseTrainer +from torchvision import transforms +from PIL import Image +import numpy as np + +class Trainer(BaseTrainer): + def __init__(self, config, model, train_loader, test_loader=None, device=None): + super().__init__(config, model, train_loader, + test_loader=test_loader, device=device) + self.device = device + self.silog_loss = SILogLoss() + self.grad_loss = GradL1Loss() + self.scaler = amp.GradScaler(enabled=self.config.use_amp) + + def train_on_batch(self, batch, train_step): + """ + Expects a batch of images and depth as input + batch["image"].shape : batch_size, c, h, w + batch["depth"].shape : batch_size, 1, h, w + """ + + images, depths_gt = batch['image'].to( + self.device), batch['depth'].to(self.device) + dataset = batch['dataset'][0] + + b, c, h, w = images.size() + mask = batch["mask"].to(self.device).to(torch.bool) + + losses = {} + + with amp.autocast(enabled=self.config.use_amp): + + output = self.model(images) + pred_depths = output['metric_depth'] + + l_si, pred = self.silog_loss( + pred_depths, depths_gt, mask=mask, interpolate=True, return_interpolated=True) + loss = self.config.w_si * l_si + losses[self.silog_loss.name] = l_si + + if self.config.w_grad > 0: + l_grad = self.grad_loss(pred, depths_gt, mask=mask) + loss = loss + self.config.w_grad * l_grad + losses[self.grad_loss.name] = l_grad + else: + l_grad = torch.Tensor([0]) + + self.scaler.scale(loss).backward() + + if self.config.clip_grad > 0: + self.scaler.unscale_(self.optimizer) + nn.utils.clip_grad_norm_( + self.model.parameters(), self.config.clip_grad) + + self.scaler.step(self.optimizer) + + if self.should_log and (self.step % int(self.config.log_images_every * self.iters_per_epoch)) == 0: + # -99 is treated as invalid depth in the log_images function and is colored grey. + depths_gt[torch.logical_not(mask)] = -99 + + self.log_images(rgb={"Input": images[0, ...]}, depth={"GT": depths_gt[0], "PredictedMono": pred[0]}, prefix="Train", + min_depth=DATASETS_CONFIG[dataset]['min_depth'], max_depth=DATASETS_CONFIG[dataset]['max_depth']) + + if self.config.get("log_rel", False): + self.log_images( + scalar_field={"RelPred": output["relative_depth"][0]}, prefix="TrainRel") + + self.scaler.update() + self.optimizer.zero_grad() + + return losses + + @torch.no_grad() + def eval_infer(self, x): + with amp.autocast(enabled=self.config.use_amp): + m = self.model.module if self.config.multigpu else self.model + pred_depths = m(x)['metric_depth'] + return pred_depths + + @torch.no_grad() + def crop_aware_infer(self, x): + # if we are not avoiding the black border, we can just use the normal inference + if not self.config.get("avoid_boundary", False): + return self.eval_infer(x) + + # otherwise, we need to crop the image to avoid the black border + # For now, this may be a bit slow due to converting to numpy and back + # We assume no normalization is done on the input image + + # get the black border + assert x.shape[0] == 1, "Only batch size 1 is supported for now" + x_pil = transforms.ToPILImage()(x[0].cpu()) + x_np = np.array(x_pil, dtype=np.uint8) + black_border_params = get_black_border(x_np) + top, bottom, left, right = black_border_params.top, black_border_params.bottom, black_border_params.left, black_border_params.right + x_np_cropped = x_np[top:bottom, left:right, :] + x_cropped = transforms.ToTensor()(Image.fromarray(x_np_cropped)) + + # run inference on the cropped image + pred_depths_cropped = self.eval_infer(x_cropped.unsqueeze(0).to(self.device)) + + # resize the prediction to x_np_cropped's size + pred_depths_cropped = nn.functional.interpolate( + pred_depths_cropped, size=(x_np_cropped.shape[0], x_np_cropped.shape[1]), mode="bilinear", align_corners=False) + + + # pad the prediction back to the original size + pred_depths = torch.zeros((1, 1, x_np.shape[0], x_np.shape[1]), device=pred_depths_cropped.device, dtype=pred_depths_cropped.dtype) + pred_depths[:, :, top:bottom, left:right] = pred_depths_cropped + + return pred_depths + + + + def validate_on_batch(self, batch, val_step): + images = batch['image'].to(self.device) + depths_gt = batch['depth'].to(self.device) + dataset = batch['dataset'][0] + mask = batch["mask"].to(self.device) + if 'has_valid_depth' in batch: + if not batch['has_valid_depth']: + return None, None + + depths_gt = depths_gt.squeeze().unsqueeze(0).unsqueeze(0) + mask = mask.squeeze().unsqueeze(0).unsqueeze(0) + if dataset == 'nyu': + pred_depths = self.crop_aware_infer(images) + else: + pred_depths = self.eval_infer(images) + pred_depths = pred_depths.squeeze().unsqueeze(0).unsqueeze(0) + + with amp.autocast(enabled=self.config.use_amp): + l_depth = self.silog_loss( + pred_depths, depths_gt, mask=mask.to(torch.bool), interpolate=True) + + metrics = compute_metrics(depths_gt, pred_depths, **self.config) + losses = {f"{self.silog_loss.name}": l_depth.item()} + + if val_step == 1 and self.should_log: + depths_gt[torch.logical_not(mask)] = -99 + self.log_images(rgb={"Input": images[0]}, depth={"GT": depths_gt[0], "PredictedMono": pred_depths[0]}, prefix="Test", + min_depth=DATASETS_CONFIG[dataset]['min_depth'], max_depth=DATASETS_CONFIG[dataset]['max_depth']) + + return metrics, losses diff --git a/annotator/zoe/zoedepth/utils/__init__.py b/annotator/zoe/zoedepth/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5f2668792389157609abb2a0846fb620e7d67eb9 --- /dev/null +++ b/annotator/zoe/zoedepth/utils/__init__.py @@ -0,0 +1,24 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + diff --git a/annotator/zoe/zoedepth/utils/arg_utils.py b/annotator/zoe/zoedepth/utils/arg_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8a3004ec3679c0a40fd8961253733fb4343ad545 --- /dev/null +++ b/annotator/zoe/zoedepth/utils/arg_utils.py @@ -0,0 +1,33 @@ + + +def infer_type(x): # hacky way to infer type from string args + if not isinstance(x, str): + return x + + try: + x = int(x) + return x + except ValueError: + pass + + try: + x = float(x) + return x + except ValueError: + pass + + return x + + +def parse_unknown(unknown_args): + clean = [] + for a in unknown_args: + if "=" in a: + k, v = a.split("=") + clean.extend([k, v]) + else: + clean.append(a) + + keys = clean[::2] + values = clean[1::2] + return {k.replace("--", ""): infer_type(v) for k, v in zip(keys, values)} diff --git a/annotator/zoe/zoedepth/utils/config.py b/annotator/zoe/zoedepth/utils/config.py new file mode 100644 index 0000000000000000000000000000000000000000..84996564663dadf0e720de2a68ef8c53106ed666 --- /dev/null +++ b/annotator/zoe/zoedepth/utils/config.py @@ -0,0 +1,437 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import json +import os + +from .easydict import EasyDict as edict +from .arg_utils import infer_type + +import pathlib +import platform + +ROOT = pathlib.Path(__file__).parent.parent.resolve() + +HOME_DIR = os.path.expanduser("~") + +COMMON_CONFIG = { + "save_dir": os.path.expanduser("~/shortcuts/monodepth3_checkpoints"), + "project": "ZoeDepth", + "tags": '', + "notes": "", + "gpu": None, + "root": ".", + "uid": None, + "print_losses": False +} + +DATASETS_CONFIG = { + "kitti": { + "dataset": "kitti", + "min_depth": 0.001, + "max_depth": 80, + "data_path": os.path.join(HOME_DIR, "shortcuts/datasets/kitti/raw"), + "gt_path": os.path.join(HOME_DIR, "shortcuts/datasets/kitti/gts"), + "filenames_file": "./train_test_inputs/kitti_eigen_train_files_with_gt.txt", + "input_height": 352, + "input_width": 1216, # 704 + "data_path_eval": os.path.join(HOME_DIR, "shortcuts/datasets/kitti/raw"), + "gt_path_eval": os.path.join(HOME_DIR, "shortcuts/datasets/kitti/gts"), + "filenames_file_eval": "./train_test_inputs/kitti_eigen_test_files_with_gt.txt", + + "min_depth_eval": 1e-3, + "max_depth_eval": 80, + + "do_random_rotate": True, + "degree": 1.0, + "do_kb_crop": True, + "garg_crop": True, + "eigen_crop": False, + "use_right": False + }, + "kitti_test": { + "dataset": "kitti", + "min_depth": 0.001, + "max_depth": 80, + "data_path": os.path.join(HOME_DIR, "shortcuts/datasets/kitti/raw"), + "gt_path": os.path.join(HOME_DIR, "shortcuts/datasets/kitti/gts"), + "filenames_file": "./train_test_inputs/kitti_eigen_train_files_with_gt.txt", + "input_height": 352, + "input_width": 1216, + "data_path_eval": os.path.join(HOME_DIR, "shortcuts/datasets/kitti/raw"), + "gt_path_eval": os.path.join(HOME_DIR, "shortcuts/datasets/kitti/gts"), + "filenames_file_eval": "./train_test_inputs/kitti_eigen_test_files_with_gt.txt", + + "min_depth_eval": 1e-3, + "max_depth_eval": 80, + + "do_random_rotate": False, + "degree": 1.0, + "do_kb_crop": True, + "garg_crop": True, + "eigen_crop": False, + "use_right": False + }, + "nyu": { + "dataset": "nyu", + "avoid_boundary": False, + "min_depth": 1e-3, # originally 0.1 + "max_depth": 10, + "data_path": os.path.join(HOME_DIR, "shortcuts/datasets/nyu_depth_v2/sync/"), + "gt_path": os.path.join(HOME_DIR, "shortcuts/datasets/nyu_depth_v2/sync/"), + "filenames_file": "./train_test_inputs/nyudepthv2_train_files_with_gt.txt", + "input_height": 480, + "input_width": 640, + "data_path_eval": os.path.join(HOME_DIR, "shortcuts/datasets/nyu_depth_v2/official_splits/test/"), + "gt_path_eval": os.path.join(HOME_DIR, "shortcuts/datasets/nyu_depth_v2/official_splits/test/"), + "filenames_file_eval": "./train_test_inputs/nyudepthv2_test_files_with_gt.txt", + "min_depth_eval": 1e-3, + "max_depth_eval": 10, + "min_depth_diff": -10, + "max_depth_diff": 10, + + "do_random_rotate": True, + "degree": 1.0, + "do_kb_crop": False, + "garg_crop": False, + "eigen_crop": True + }, + "ibims": { + "dataset": "ibims", + "ibims_root": os.path.join(HOME_DIR, "shortcuts/datasets/ibims/ibims1_core_raw/"), + "eigen_crop": True, + "garg_crop": False, + "do_kb_crop": False, + "min_depth_eval": 0, + "max_depth_eval": 10, + "min_depth": 1e-3, + "max_depth": 10 + }, + "sunrgbd": { + "dataset": "sunrgbd", + "sunrgbd_root": os.path.join(HOME_DIR, "shortcuts/datasets/SUNRGBD/test/"), + "eigen_crop": True, + "garg_crop": False, + "do_kb_crop": False, + "min_depth_eval": 0, + "max_depth_eval": 8, + "min_depth": 1e-3, + "max_depth": 10 + }, + "diml_indoor": { + "dataset": "diml_indoor", + "diml_indoor_root": os.path.join(HOME_DIR, "shortcuts/datasets/diml_indoor_test/"), + "eigen_crop": True, + "garg_crop": False, + "do_kb_crop": False, + "min_depth_eval": 0, + "max_depth_eval": 10, + "min_depth": 1e-3, + "max_depth": 10 + }, + "diml_outdoor": { + "dataset": "diml_outdoor", + "diml_outdoor_root": os.path.join(HOME_DIR, "shortcuts/datasets/diml_outdoor_test/"), + "eigen_crop": False, + "garg_crop": True, + "do_kb_crop": False, + "min_depth_eval": 2, + "max_depth_eval": 80, + "min_depth": 1e-3, + "max_depth": 80 + }, + "diode_indoor": { + "dataset": "diode_indoor", + "diode_indoor_root": os.path.join(HOME_DIR, "shortcuts/datasets/diode_indoor/"), + "eigen_crop": True, + "garg_crop": False, + "do_kb_crop": False, + "min_depth_eval": 1e-3, + "max_depth_eval": 10, + "min_depth": 1e-3, + "max_depth": 10 + }, + "diode_outdoor": { + "dataset": "diode_outdoor", + "diode_outdoor_root": os.path.join(HOME_DIR, "shortcuts/datasets/diode_outdoor/"), + "eigen_crop": False, + "garg_crop": True, + "do_kb_crop": False, + "min_depth_eval": 1e-3, + "max_depth_eval": 80, + "min_depth": 1e-3, + "max_depth": 80 + }, + "hypersim_test": { + "dataset": "hypersim_test", + "hypersim_test_root": os.path.join(HOME_DIR, "shortcuts/datasets/hypersim_test/"), + "eigen_crop": True, + "garg_crop": False, + "do_kb_crop": False, + "min_depth_eval": 1e-3, + "max_depth_eval": 80, + "min_depth": 1e-3, + "max_depth": 10 + }, + "vkitti": { + "dataset": "vkitti", + "vkitti_root": os.path.join(HOME_DIR, "shortcuts/datasets/vkitti_test/"), + "eigen_crop": False, + "garg_crop": True, + "do_kb_crop": True, + "min_depth_eval": 1e-3, + "max_depth_eval": 80, + "min_depth": 1e-3, + "max_depth": 80 + }, + "vkitti2": { + "dataset": "vkitti2", + "vkitti2_root": os.path.join(HOME_DIR, "shortcuts/datasets/vkitti2/"), + "eigen_crop": False, + "garg_crop": True, + "do_kb_crop": True, + "min_depth_eval": 1e-3, + "max_depth_eval": 80, + "min_depth": 1e-3, + "max_depth": 80, + }, + "ddad": { + "dataset": "ddad", + "ddad_root": os.path.join(HOME_DIR, "shortcuts/datasets/ddad/ddad_val/"), + "eigen_crop": False, + "garg_crop": True, + "do_kb_crop": True, + "min_depth_eval": 1e-3, + "max_depth_eval": 80, + "min_depth": 1e-3, + "max_depth": 80, + }, +} + +ALL_INDOOR = ["nyu", "ibims", "sunrgbd", "diode_indoor", "hypersim_test"] +ALL_OUTDOOR = ["kitti", "diml_outdoor", "diode_outdoor", "vkitti2", "ddad"] +ALL_EVAL_DATASETS = ALL_INDOOR + ALL_OUTDOOR + +COMMON_TRAINING_CONFIG = { + "dataset": "nyu", + "distributed": True, + "workers": 16, + "clip_grad": 0.1, + "use_shared_dict": False, + "shared_dict": None, + "use_amp": False, + + "aug": True, + "random_crop": False, + "random_translate": False, + "translate_prob": 0.2, + "max_translation": 100, + + "validate_every": 0.25, + "log_images_every": 0.1, + "prefetch": False, +} + + +def flatten(config, except_keys=('bin_conf')): + def recurse(inp): + if isinstance(inp, dict): + for key, value in inp.items(): + if key in except_keys: + yield (key, value) + if isinstance(value, dict): + yield from recurse(value) + else: + yield (key, value) + + return dict(list(recurse(config))) + + +def split_combined_args(kwargs): + """Splits the arguments that are combined with '__' into multiple arguments. + Combined arguments should have equal number of keys and values. + Keys are separated by '__' and Values are separated with ';'. + For example, '__n_bins__lr=256;0.001' + + Args: + kwargs (dict): key-value pairs of arguments where key-value is optionally combined according to the above format. + + Returns: + dict: Parsed dict with the combined arguments split into individual key-value pairs. + """ + new_kwargs = dict(kwargs) + for key, value in kwargs.items(): + if key.startswith("__"): + keys = key.split("__")[1:] + values = value.split(";") + assert len(keys) == len( + values), f"Combined arguments should have equal number of keys and values. Keys are separated by '__' and Values are separated with ';'. For example, '__n_bins__lr=256;0.001. Given (keys,values) is ({keys}, {values})" + for k, v in zip(keys, values): + new_kwargs[k] = v + return new_kwargs + + +def parse_list(config, key, dtype=int): + """Parse a list of values for the key if the value is a string. The values are separated by a comma. + Modifies the config in place. + """ + if key in config: + if isinstance(config[key], str): + config[key] = list(map(dtype, config[key].split(','))) + assert isinstance(config[key], list) and all([isinstance(e, dtype) for e in config[key]] + ), f"{key} should be a list of values dtype {dtype}. Given {config[key]} of type {type(config[key])} with values of type {[type(e) for e in config[key]]}." + + +def get_model_config(model_name, model_version=None): + """Find and parse the .json config file for the model. + + Args: + model_name (str): name of the model. The config file should be named config_{model_name}[_{model_version}].json under the models/{model_name} directory. + model_version (str, optional): Specific config version. If specified config_{model_name}_{model_version}.json is searched for and used. Otherwise config_{model_name}.json is used. Defaults to None. + + Returns: + easydict: the config dictionary for the model. + """ + config_fname = f"config_{model_name}_{model_version}.json" if model_version is not None else f"config_{model_name}.json" + config_file = os.path.join(ROOT, "models", model_name, config_fname) + if not os.path.exists(config_file): + return None + + with open(config_file, "r") as f: + config = edict(json.load(f)) + + # handle dictionary inheritance + # only training config is supported for inheritance + if "inherit" in config.train and config.train.inherit is not None: + inherit_config = get_model_config(config.train["inherit"]).train + for key, value in inherit_config.items(): + if key not in config.train: + config.train[key] = value + return edict(config) + + +def update_model_config(config, mode, model_name, model_version=None, strict=False): + model_config = get_model_config(model_name, model_version) + if model_config is not None: + config = {**config, ** + flatten({**model_config.model, **model_config[mode]})} + elif strict: + raise ValueError(f"Config file for model {model_name} not found.") + return config + + +def check_choices(name, value, choices): + # return # No checks in dev branch + if value not in choices: + raise ValueError(f"{name} {value} not in supported choices {choices}") + + +KEYS_TYPE_BOOL = ["use_amp", "distributed", "use_shared_dict", "same_lr", "aug", "three_phase", + "prefetch", "cycle_momentum"] # Casting is not necessary as their int casted values in config are 0 or 1 + + +def get_config(model_name, mode='train', dataset=None, **overwrite_kwargs): + """Main entry point to get the config for the model. + + Args: + model_name (str): name of the desired model. + mode (str, optional): "train" or "infer". Defaults to 'train'. + dataset (str, optional): If specified, the corresponding dataset configuration is loaded as well. Defaults to None. + + Keyword Args: key-value pairs of arguments to overwrite the default config. + + The order of precedence for overwriting the config is (Higher precedence first): + # 1. overwrite_kwargs + # 2. "config_version": Config file version if specified in overwrite_kwargs. The corresponding config loaded is config_{model_name}_{config_version}.json + # 3. "version_name": Default Model version specific config specified in overwrite_kwargs. The corresponding config loaded is config_{model_name}_{version_name}.json + # 4. common_config: Default config for all models specified in COMMON_CONFIG + + Returns: + easydict: The config dictionary for the model. + """ + + + check_choices("Model", model_name, ["zoedepth", "zoedepth_nk"]) + check_choices("Mode", mode, ["train", "infer", "eval"]) + if mode == "train": + check_choices("Dataset", dataset, ["nyu", "kitti", "mix", None]) + + config = flatten({**COMMON_CONFIG, **COMMON_TRAINING_CONFIG}) + config = update_model_config(config, mode, model_name) + + # update with model version specific config + version_name = overwrite_kwargs.get("version_name", config["version_name"]) + config = update_model_config(config, mode, model_name, version_name) + + # update with config version if specified + config_version = overwrite_kwargs.get("config_version", None) + if config_version is not None: + print("Overwriting config with config_version", config_version) + config = update_model_config(config, mode, model_name, config_version) + + # update with overwrite_kwargs + # Combined args are useful for hyperparameter search + overwrite_kwargs = split_combined_args(overwrite_kwargs) + config = {**config, **overwrite_kwargs} + + # Casting to bool # TODO: Not necessary. Remove and test + for key in KEYS_TYPE_BOOL: + if key in config: + config[key] = bool(config[key]) + + # Model specific post processing of config + parse_list(config, "n_attractors") + + # adjust n_bins for each bin configuration if bin_conf is given and n_bins is passed in overwrite_kwargs + if 'bin_conf' in config and 'n_bins' in overwrite_kwargs: + bin_conf = config['bin_conf'] # list of dicts + n_bins = overwrite_kwargs['n_bins'] + new_bin_conf = [] + for conf in bin_conf: + conf['n_bins'] = n_bins + new_bin_conf.append(conf) + config['bin_conf'] = new_bin_conf + + if mode == "train": + orig_dataset = dataset + if dataset == "mix": + dataset = 'nyu' # Use nyu as default for mix. Dataset config is changed accordingly while loading the dataloader + if dataset is not None: + config['project'] = f"MonoDepth3-{orig_dataset}" # Set project for wandb + + if dataset is not None: + config['dataset'] = dataset + config = {**DATASETS_CONFIG[dataset], **config} + + + config['model'] = model_name + typed_config = {k: infer_type(v) for k, v in config.items()} + # add hostname to config + config['hostname'] = platform.node() + return edict(typed_config) + + +def change_dataset(config, new_dataset): + config.update(DATASETS_CONFIG[new_dataset]) + return config diff --git a/annotator/zoe/zoedepth/utils/easydict/__init__.py b/annotator/zoe/zoedepth/utils/easydict/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..15928179b0182c6045d98bc0a7be1c6ca45f675e --- /dev/null +++ b/annotator/zoe/zoedepth/utils/easydict/__init__.py @@ -0,0 +1,158 @@ +""" +EasyDict +Copy/pasted from https://github.com/makinacorpus/easydict +Original author: Mathieu Leplatre +""" + +class EasyDict(dict): + """ + Get attributes + + >>> d = EasyDict({'foo':3}) + >>> d['foo'] + 3 + >>> d.foo + 3 + >>> d.bar + Traceback (most recent call last): + ... + AttributeError: 'EasyDict' object has no attribute 'bar' + + Works recursively + + >>> d = EasyDict({'foo':3, 'bar':{'x':1, 'y':2}}) + >>> isinstance(d.bar, dict) + True + >>> d.bar.x + 1 + + Bullet-proof + + >>> EasyDict({}) + {} + >>> EasyDict(d={}) + {} + >>> EasyDict(None) + {} + >>> d = {'a': 1} + >>> EasyDict(**d) + {'a': 1} + >>> EasyDict((('a', 1), ('b', 2))) + {'a': 1, 'b': 2} + + Set attributes + + >>> d = EasyDict() + >>> d.foo = 3 + >>> d.foo + 3 + >>> d.bar = {'prop': 'value'} + >>> d.bar.prop + 'value' + >>> d + {'foo': 3, 'bar': {'prop': 'value'}} + >>> d.bar.prop = 'newer' + >>> d.bar.prop + 'newer' + + + Values extraction + + >>> d = EasyDict({'foo':0, 'bar':[{'x':1, 'y':2}, {'x':3, 'y':4}]}) + >>> isinstance(d.bar, list) + True + >>> from operator import attrgetter + >>> list(map(attrgetter('x'), d.bar)) + [1, 3] + >>> list(map(attrgetter('y'), d.bar)) + [2, 4] + >>> d = EasyDict() + >>> list(d.keys()) + [] + >>> d = EasyDict(foo=3, bar=dict(x=1, y=2)) + >>> d.foo + 3 + >>> d.bar.x + 1 + + Still like a dict though + + >>> o = EasyDict({'clean':True}) + >>> list(o.items()) + [('clean', True)] + + And like a class + + >>> class Flower(EasyDict): + ... power = 1 + ... + >>> f = Flower() + >>> f.power + 1 + >>> f = Flower({'height': 12}) + >>> f.height + 12 + >>> f['power'] + 1 + >>> sorted(f.keys()) + ['height', 'power'] + + update and pop items + >>> d = EasyDict(a=1, b='2') + >>> e = EasyDict(c=3.0, a=9.0) + >>> d.update(e) + >>> d.c + 3.0 + >>> d['c'] + 3.0 + >>> d.get('c') + 3.0 + >>> d.update(a=4, b=4) + >>> d.b + 4 + >>> d.pop('a') + 4 + >>> d.a + Traceback (most recent call last): + ... + AttributeError: 'EasyDict' object has no attribute 'a' + """ + def __init__(self, d=None, **kwargs): + if d is None: + d = {} + else: + d = dict(d) + if kwargs: + d.update(**kwargs) + for k, v in d.items(): + setattr(self, k, v) + # Class attributes + for k in self.__class__.__dict__.keys(): + if not (k.startswith('__') and k.endswith('__')) and not k in ('update', 'pop'): + setattr(self, k, getattr(self, k)) + + def __setattr__(self, name, value): + if isinstance(value, (list, tuple)): + value = [self.__class__(x) + if isinstance(x, dict) else x for x in value] + elif isinstance(value, dict) and not isinstance(value, self.__class__): + value = self.__class__(value) + super(EasyDict, self).__setattr__(name, value) + super(EasyDict, self).__setitem__(name, value) + + __setitem__ = __setattr__ + + def update(self, e=None, **f): + d = e or dict() + d.update(f) + for k in d: + setattr(self, k, d[k]) + + def pop(self, k, d=None): + delattr(self, k) + return super(EasyDict, self).pop(k, d) + + +if __name__ == "__main__": + import doctest + doctest.testmod() \ No newline at end of file diff --git a/annotator/zoe/zoedepth/utils/geometry.py b/annotator/zoe/zoedepth/utils/geometry.py new file mode 100644 index 0000000000000000000000000000000000000000..e3da8c75b5a8e39b4b58a4dcd827b84d79b9115c --- /dev/null +++ b/annotator/zoe/zoedepth/utils/geometry.py @@ -0,0 +1,98 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import numpy as np + +def get_intrinsics(H,W): + """ + Intrinsics for a pinhole camera model. + Assume fov of 55 degrees and central principal point. + """ + f = 0.5 * W / np.tan(0.5 * 55 * np.pi / 180.0) + cx = 0.5 * W + cy = 0.5 * H + return np.array([[f, 0, cx], + [0, f, cy], + [0, 0, 1]]) + +def depth_to_points(depth, R=None, t=None): + + K = get_intrinsics(depth.shape[1], depth.shape[2]) + Kinv = np.linalg.inv(K) + if R is None: + R = np.eye(3) + if t is None: + t = np.zeros(3) + + # M converts from your coordinate to PyTorch3D's coordinate system + M = np.eye(3) + M[0, 0] = -1.0 + M[1, 1] = -1.0 + + height, width = depth.shape[1:3] + + x = np.arange(width) + y = np.arange(height) + coord = np.stack(np.meshgrid(x, y), -1) + coord = np.concatenate((coord, np.ones_like(coord)[:, :, [0]]), -1) # z=1 + coord = coord.astype(np.float32) + # coord = torch.as_tensor(coord, dtype=torch.float32, device=device) + coord = coord[None] # bs, h, w, 3 + + D = depth[:, :, :, None, None] + # print(D.shape, Kinv[None, None, None, ...].shape, coord[:, :, :, :, None].shape ) + pts3D_1 = D * Kinv[None, None, None, ...] @ coord[:, :, :, :, None] + # pts3D_1 live in your coordinate system. Convert them to Py3D's + pts3D_1 = M[None, None, None, ...] @ pts3D_1 + # from reference to targe tviewpoint + pts3D_2 = R[None, None, None, ...] @ pts3D_1 + t[None, None, None, :, None] + # pts3D_2 = pts3D_1 + # depth_2 = pts3D_2[:, :, :, 2, :] # b,1,h,w + return pts3D_2[:, :, :, :3, 0][0] + + +def create_triangles(h, w, mask=None): + """ + Reference: https://github.com/google-research/google-research/blob/e96197de06613f1b027d20328e06d69829fa5a89/infinite_nature/render_utils.py#L68 + Creates mesh triangle indices from a given pixel grid size. + This function is not and need not be differentiable as triangle indices are + fixed. + Args: + h: (int) denoting the height of the image. + w: (int) denoting the width of the image. + Returns: + triangles: 2D numpy array of indices (int) with shape (2(W-1)(H-1) x 3) + """ + x, y = np.meshgrid(range(w - 1), range(h - 1)) + tl = y * w + x + tr = y * w + x + 1 + bl = (y + 1) * w + x + br = (y + 1) * w + x + 1 + triangles = np.array([tl, bl, tr, br, tr, bl]) + triangles = np.transpose(triangles, (1, 2, 0)).reshape( + ((w - 1) * (h - 1) * 2, 3)) + if mask is not None: + mask = mask.reshape(-1) + triangles = triangles[mask[triangles].all(1)] + return triangles diff --git a/annotator/zoe/zoedepth/utils/misc.py b/annotator/zoe/zoedepth/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..4bbe403d3669829eecdf658458c76aa5e87e2b33 --- /dev/null +++ b/annotator/zoe/zoedepth/utils/misc.py @@ -0,0 +1,368 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +"""Miscellaneous utility functions.""" + +from scipy import ndimage + +import base64 +import math +import re +from io import BytesIO + +import matplotlib +import matplotlib.cm +import numpy as np +import requests +import torch +import torch.distributed as dist +import torch.nn +import torch.nn as nn +import torch.utils.data.distributed +from PIL import Image +from torchvision.transforms import ToTensor + + +class RunningAverage: + def __init__(self): + self.avg = 0 + self.count = 0 + + def append(self, value): + self.avg = (value + self.count * self.avg) / (self.count + 1) + self.count += 1 + + def get_value(self): + return self.avg + + +def denormalize(x): + """Reverses the imagenet normalization applied to the input. + + Args: + x (torch.Tensor - shape(N,3,H,W)): input tensor + + Returns: + torch.Tensor - shape(N,3,H,W): Denormalized input + """ + mean = torch.Tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1).to(x.device) + std = torch.Tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1).to(x.device) + return x * std + mean + + +class RunningAverageDict: + """A dictionary of running averages.""" + def __init__(self): + self._dict = None + + def update(self, new_dict): + if new_dict is None: + return + + if self._dict is None: + self._dict = dict() + for key, value in new_dict.items(): + self._dict[key] = RunningAverage() + + for key, value in new_dict.items(): + self._dict[key].append(value) + + def get_value(self): + if self._dict is None: + return None + return {key: value.get_value() for key, value in self._dict.items()} + + +def colorize(value, vmin=None, vmax=None, cmap='gray_r', invalid_val=-99, invalid_mask=None, background_color=(128, 128, 128, 255), gamma_corrected=False, value_transform=None): + """Converts a depth map to a color image. + + Args: + value (torch.Tensor, numpy.ndarry): Input depth map. Shape: (H, W) or (1, H, W) or (1, 1, H, W). All singular dimensions are squeezed + vmin (float, optional): vmin-valued entries are mapped to start color of cmap. If None, value.min() is used. Defaults to None. + vmax (float, optional): vmax-valued entries are mapped to end color of cmap. If None, value.max() is used. Defaults to None. + cmap (str, optional): matplotlib colormap to use. Defaults to 'magma_r'. + invalid_val (int, optional): Specifies value of invalid pixels that should be colored as 'background_color'. Defaults to -99. + invalid_mask (numpy.ndarray, optional): Boolean mask for invalid regions. Defaults to None. + background_color (tuple[int], optional): 4-tuple RGB color to give to invalid pixels. Defaults to (128, 128, 128, 255). + gamma_corrected (bool, optional): Apply gamma correction to colored image. Defaults to False. + value_transform (Callable, optional): Apply transform function to valid pixels before coloring. Defaults to None. + + Returns: + numpy.ndarray, dtype - uint8: Colored depth map. Shape: (H, W, 4) + """ + if isinstance(value, torch.Tensor): + value = value.detach().cpu().numpy() + + value = value.squeeze() + if invalid_mask is None: + invalid_mask = value == invalid_val + mask = np.logical_not(invalid_mask) + + # normalize + vmin = np.percentile(value[mask],2) if vmin is None else vmin + vmax = np.percentile(value[mask],85) if vmax is None else vmax + if vmin != vmax: + value = (value - vmin) / (vmax - vmin) # vmin..vmax + else: + # Avoid 0-division + value = value * 0. + + # squeeze last dim if it exists + # grey out the invalid values + + value[invalid_mask] = np.nan + cmapper = matplotlib.cm.get_cmap(cmap) + if value_transform: + value = value_transform(value) + # value = value / value.max() + value = cmapper(value, bytes=True) # (nxmx4) + + # img = value[:, :, :] + img = value[...] + img[invalid_mask] = background_color + + # return img.transpose((2, 0, 1)) + if gamma_corrected: + # gamma correction + img = img / 255 + img = np.power(img, 2.2) + img = img * 255 + img = img.astype(np.uint8) + return img + + +def count_parameters(model, include_all=False): + return sum(p.numel() for p in model.parameters() if p.requires_grad or include_all) + + +def compute_errors(gt, pred): + """Compute metrics for 'pred' compared to 'gt' + + Args: + gt (numpy.ndarray): Ground truth values + pred (numpy.ndarray): Predicted values + + gt.shape should be equal to pred.shape + + Returns: + dict: Dictionary containing the following metrics: + 'a1': Delta1 accuracy: Fraction of pixels that are within a scale factor of 1.25 + 'a2': Delta2 accuracy: Fraction of pixels that are within a scale factor of 1.25^2 + 'a3': Delta3 accuracy: Fraction of pixels that are within a scale factor of 1.25^3 + 'abs_rel': Absolute relative error + 'rmse': Root mean squared error + 'log_10': Absolute log10 error + 'sq_rel': Squared relative error + 'rmse_log': Root mean squared error on the log scale + 'silog': Scale invariant log error + """ + thresh = np.maximum((gt / pred), (pred / gt)) + a1 = (thresh < 1.25).mean() + a2 = (thresh < 1.25 ** 2).mean() + a3 = (thresh < 1.25 ** 3).mean() + + abs_rel = np.mean(np.abs(gt - pred) / gt) + sq_rel = np.mean(((gt - pred) ** 2) / gt) + + rmse = (gt - pred) ** 2 + rmse = np.sqrt(rmse.mean()) + + rmse_log = (np.log(gt) - np.log(pred)) ** 2 + rmse_log = np.sqrt(rmse_log.mean()) + + err = np.log(pred) - np.log(gt) + silog = np.sqrt(np.mean(err ** 2) - np.mean(err) ** 2) * 100 + + log_10 = (np.abs(np.log10(gt) - np.log10(pred))).mean() + return dict(a1=a1, a2=a2, a3=a3, abs_rel=abs_rel, rmse=rmse, log_10=log_10, rmse_log=rmse_log, + silog=silog, sq_rel=sq_rel) + + +def compute_metrics(gt, pred, interpolate=True, garg_crop=False, eigen_crop=True, dataset='nyu', min_depth_eval=0.1, max_depth_eval=10, **kwargs): + """Compute metrics of predicted depth maps. Applies cropping and masking as necessary or specified via arguments. Refer to compute_errors for more details on metrics. + """ + if 'config' in kwargs: + config = kwargs['config'] + garg_crop = config.garg_crop + eigen_crop = config.eigen_crop + min_depth_eval = config.min_depth_eval + max_depth_eval = config.max_depth_eval + + if gt.shape[-2:] != pred.shape[-2:] and interpolate: + pred = nn.functional.interpolate( + pred, gt.shape[-2:], mode='bilinear', align_corners=True) + + pred = pred.squeeze().cpu().numpy() + pred[pred < min_depth_eval] = min_depth_eval + pred[pred > max_depth_eval] = max_depth_eval + pred[np.isinf(pred)] = max_depth_eval + pred[np.isnan(pred)] = min_depth_eval + + gt_depth = gt.squeeze().cpu().numpy() + valid_mask = np.logical_and( + gt_depth > min_depth_eval, gt_depth < max_depth_eval) + + if garg_crop or eigen_crop: + gt_height, gt_width = gt_depth.shape + eval_mask = np.zeros(valid_mask.shape) + + if garg_crop: + eval_mask[int(0.40810811 * gt_height):int(0.99189189 * gt_height), + int(0.03594771 * gt_width):int(0.96405229 * gt_width)] = 1 + + elif eigen_crop: + # print("-"*10, " EIGEN CROP ", "-"*10) + if dataset == 'kitti': + eval_mask[int(0.3324324 * gt_height):int(0.91351351 * gt_height), + int(0.0359477 * gt_width):int(0.96405229 * gt_width)] = 1 + else: + # assert gt_depth.shape == (480, 640), "Error: Eigen crop is currently only valid for (480, 640) images" + eval_mask[45:471, 41:601] = 1 + else: + eval_mask = np.ones(valid_mask.shape) + valid_mask = np.logical_and(valid_mask, eval_mask) + return compute_errors(gt_depth[valid_mask], pred[valid_mask]) + + +#################################### Model uilts ################################################ + + +def parallelize(config, model, find_unused_parameters=True): + + if config.gpu is not None: + torch.cuda.set_device(config.gpu) + model = model.cuda(config.gpu) + + config.multigpu = False + if config.distributed: + # Use DDP + config.multigpu = True + config.rank = config.rank * config.ngpus_per_node + config.gpu + dist.init_process_group(backend=config.dist_backend, init_method=config.dist_url, + world_size=config.world_size, rank=config.rank) + config.batch_size = int(config.batch_size / config.ngpus_per_node) + # config.batch_size = 8 + config.workers = int( + (config.num_workers + config.ngpus_per_node - 1) / config.ngpus_per_node) + print("Device", config.gpu, "Rank", config.rank, "batch size", + config.batch_size, "Workers", config.workers) + torch.cuda.set_device(config.gpu) + model = nn.SyncBatchNorm.convert_sync_batchnorm(model) + model = model.cuda(config.gpu) + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[config.gpu], output_device=config.gpu, + find_unused_parameters=find_unused_parameters) + + elif config.gpu is None: + # Use DP + config.multigpu = True + model = model.cuda() + model = torch.nn.DataParallel(model) + + return model + + +################################################################################################# + + +##################################################################################################### + + +class colors: + '''Colors class: + Reset all colors with colors.reset + Two subclasses fg for foreground and bg for background. + Use as colors.subclass.colorname. + i.e. colors.fg.red or colors.bg.green + Also, the generic bold, disable, underline, reverse, strikethrough, + and invisible work with the main class + i.e. colors.bold + ''' + reset = '\033[0m' + bold = '\033[01m' + disable = '\033[02m' + underline = '\033[04m' + reverse = '\033[07m' + strikethrough = '\033[09m' + invisible = '\033[08m' + + class fg: + black = '\033[30m' + red = '\033[31m' + green = '\033[32m' + orange = '\033[33m' + blue = '\033[34m' + purple = '\033[35m' + cyan = '\033[36m' + lightgrey = '\033[37m' + darkgrey = '\033[90m' + lightred = '\033[91m' + lightgreen = '\033[92m' + yellow = '\033[93m' + lightblue = '\033[94m' + pink = '\033[95m' + lightcyan = '\033[96m' + + class bg: + black = '\033[40m' + red = '\033[41m' + green = '\033[42m' + orange = '\033[43m' + blue = '\033[44m' + purple = '\033[45m' + cyan = '\033[46m' + lightgrey = '\033[47m' + + +def printc(text, color): + print(f"{color}{text}{colors.reset}") + +############################################ + +def get_image_from_url(url): + response = requests.get(url) + img = Image.open(BytesIO(response.content)).convert("RGB") + return img + +def url_to_torch(url, size=(384, 384)): + img = get_image_from_url(url) + img = img.resize(size, Image.ANTIALIAS) + img = torch.from_numpy(np.asarray(img)).float() + img = img.permute(2, 0, 1) + img.div_(255) + return img + +def pil_to_batched_tensor(img): + return ToTensor()(img).unsqueeze(0) + +def save_raw_16bit(depth, fpath="raw.png"): + if isinstance(depth, torch.Tensor): + depth = depth.squeeze().cpu().numpy() + + assert isinstance(depth, np.ndarray), "Depth must be a torch tensor or numpy array" + assert depth.ndim == 2, "Depth must be 2D" + depth = depth * 256 # scale for 16-bit png + depth = depth.astype(np.uint16) + depth = Image.fromarray(depth) + depth.save(fpath) + print("Saved raw depth to", fpath) \ No newline at end of file diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..db85a6963cc6aac7cd97535eb74d1bbcb4a606fe --- /dev/null +++ b/app.py @@ -0,0 +1,383 @@ +import gradio as gr +import cv2 + +from annotator.util import resize_image, HWC3 + +DESCRIPTION = '# ControlNet v1.1 Annotators (that runs on cpu only)' +DESCRIPTION += '\n

If the source image is HEIC file, convert it to PNG or JPG.

' + + +model_canny = None + + +def canny(img, res, l, h): + img = resize_image(HWC3(img), res) + global model_canny + if model_canny is None: + from annotator.canny import CannyDetector + model_canny = CannyDetector() + result = model_canny(img, l, h) + return [result] + + +model_hed = None + + +def hed(img, res): + img = resize_image(HWC3(img), res) + global model_hed + if model_hed is None: + from annotator.hed import HEDdetector + model_hed = HEDdetector() + result = model_hed(img) + return [result] + + +model_pidi = None + + +def pidi(img, res): + img = resize_image(HWC3(img), res) + global model_pidi + if model_pidi is None: + from annotator.pidinet import PidiNetDetector + model_pidi = PidiNetDetector() + result = model_pidi(img) + return [result] + + +model_mlsd = None + + +def mlsd(img, res, thr_v, thr_d): + img = resize_image(HWC3(img), res) + global model_mlsd + if model_mlsd is None: + from annotator.mlsd import MLSDdetector + model_mlsd = MLSDdetector() + result = model_mlsd(img, thr_v, thr_d) + return [result] + + +model_midas = None + + +def midas(img, res): + img = resize_image(HWC3(img), res) + global model_midas + if model_midas is None: + from annotator.midas import MidasDetector + model_midas = MidasDetector() + result = model_midas(img) + return [result] + + +model_zoe = None + + +def zoe(img, res): + img = resize_image(HWC3(img), res) + global model_zoe + if model_zoe is None: + from annotator.zoe import ZoeDetector + model_zoe = ZoeDetector() + result = model_zoe(img) + return [result] + + +#model_normalbae = None + + +#def normalbae(img, res): +# img = resize_image(HWC3(img), res) +# global model_normalbae +# if model_normalbae is None: +# from annotator.normalbae import NormalBaeDetector +# model_normalbae = NormalBaeDetector() +# result = model_normalbae(img) +# return [result] + + +model_openpose = None + + +def openpose(img, res, hand_and_face): + img = resize_image(HWC3(img), res) + global model_openpose + if model_openpose is None: + from annotator.openpose import OpenposeDetector + model_openpose = OpenposeDetector() + result = model_openpose(img, hand_and_face) + return [result] + + +model_uniformer = None + + +#def uniformer(img, res): +# img = resize_image(HWC3(img), res) +# global model_uniformer +# if model_uniformer is None: +# from annotator.uniformer import UniformerDetector +# model_uniformer = UniformerDetector() +# result = model_uniformer(img) +# return [result] + + +model_lineart_anime = None + + +def lineart_anime(img, res): + img = resize_image(HWC3(img), res) + global model_lineart_anime + if model_lineart_anime is None: + from annotator.lineart_anime import LineartAnimeDetector + model_lineart_anime = LineartAnimeDetector() +# result = model_lineart_anime(img) + result = cv2.bitwise_not(model_lineart_anime(img)) + return [result] + + +model_lineart = None + + +def lineart(img, res, coarse=False): + img = resize_image(HWC3(img), res) + global model_lineart + if model_lineart is None: + from annotator.lineart import LineartDetector + model_lineart = LineartDetector() +# result = model_lineart(img, coarse) + result = cv2.bitwise_not(model_lineart(img, coarse)) + return [result] + + +model_oneformer_coco = None + + +def oneformer_coco(img, res): + img = resize_image(HWC3(img), res) + global model_oneformer_coco + if model_oneformer_coco is None: + from annotator.oneformer import OneformerCOCODetector + model_oneformer_coco = OneformerCOCODetector() + result = model_oneformer_coco(img) + return [result] + + +model_oneformer_ade20k = None + + +def oneformer_ade20k(img, res): + img = resize_image(HWC3(img), res) + global model_oneformer_ade20k + if model_oneformer_ade20k is None: + from annotator.oneformer import OneformerADE20kDetector + model_oneformer_ade20k = OneformerADE20kDetector() + result = model_oneformer_ade20k(img) + return [result] + + +model_content_shuffler = None + + +def content_shuffler(img, res): + img = resize_image(HWC3(img), res) + global model_content_shuffler + if model_content_shuffler is None: + from annotator.shuffle import ContentShuffleDetector + model_content_shuffler = ContentShuffleDetector() + result = model_content_shuffler(img) + return [result] + + +model_color_shuffler = None + + +def color_shuffler(img, res): + img = resize_image(HWC3(img), res) + global model_color_shuffler + if model_color_shuffler is None: + from annotator.shuffle import ColorShuffleDetector + model_color_shuffler = ColorShuffleDetector() + result = model_color_shuffler(img) + return [result] + + +block = gr.Blocks().queue() +with block: + gr.Markdown(DESCRIPTION) + with gr.Row(): + gr.Markdown("## Canny Edge") + with gr.Row(): + with gr.Column(): + input_image = gr.Image(source='upload', type="numpy") + low_threshold = gr.Slider(label="low_threshold", minimum=1, maximum=255, value=100, step=1) + high_threshold = gr.Slider(label="high_threshold", minimum=1, maximum=255, value=200, step=1) + resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64) + run_button = gr.Button(label="Run") + with gr.Column(): + gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto") + run_button.click(fn=canny, inputs=[input_image, resolution, low_threshold, high_threshold], outputs=[gallery]) + + with gr.Row(): + gr.Markdown("## HED Edge") + with gr.Row(): + with gr.Column(): + input_image = gr.Image(source='upload', type="numpy") + resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64) + run_button = gr.Button(label="Run") + with gr.Column(): + gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto") + run_button.click(fn=hed, inputs=[input_image, resolution], outputs=[gallery]) + + with gr.Row(): + gr.Markdown("## Pidi Edge") + with gr.Row(): + with gr.Column(): + input_image = gr.Image(source='upload', type="numpy") + resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64) + run_button = gr.Button(label="Run") + with gr.Column(): + gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto") + run_button.click(fn=pidi, inputs=[input_image, resolution], outputs=[gallery]) + + with gr.Row(): + gr.Markdown("## MLSD Edge") + with gr.Row(): + with gr.Column(): + input_image = gr.Image(source='upload', type="numpy") + value_threshold = gr.Slider(label="value_threshold", minimum=0.01, maximum=2.0, value=0.1, step=0.01) + distance_threshold = gr.Slider(label="distance_threshold", minimum=0.01, maximum=20.0, value=0.1, step=0.01) + resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=384, step=64) + run_button = gr.Button(label="Run") + with gr.Column(): + gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto") + run_button.click(fn=mlsd, inputs=[input_image, resolution, value_threshold, distance_threshold], outputs=[gallery]) + + with gr.Row(): + gr.Markdown("## MIDAS Depth") + with gr.Row(): + with gr.Column(): + input_image = gr.Image(source='upload', type="numpy") + resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=384, step=64) + run_button = gr.Button(label="Run") + with gr.Column(): + gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto") + run_button.click(fn=midas, inputs=[input_image, resolution], outputs=[gallery]) + + + with gr.Row(): + gr.Markdown("## Zoe Depth") + with gr.Row(): + with gr.Column(): + input_image = gr.Image(source='upload', type="numpy") + resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64) + run_button = gr.Button(label="Run") + with gr.Column(): + gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto") + run_button.click(fn=zoe, inputs=[input_image, resolution], outputs=[gallery]) + +# with gr.Row(): +# gr.Markdown("## Normal Bae") +# with gr.Row(): +# with gr.Column(): +# input_image = gr.Image(source='upload', type="numpy") +# resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64) +# run_button = gr.Button(label="Run") +# with gr.Column(): +# gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto") +# run_button.click(fn=normalbae, inputs=[input_image, resolution], outputs=[gallery]) + + with gr.Row(): + gr.Markdown("## Openpose") + with gr.Row(): + with gr.Column(): + input_image = gr.Image(source='upload', type="numpy") + hand_and_face = gr.Checkbox(label='Hand and Face', value=False) + resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64) + run_button = gr.Button(label="Run") + with gr.Column(): + gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto") + run_button.click(fn=openpose, inputs=[input_image, resolution, hand_and_face], outputs=[gallery]) + + with gr.Row(): + gr.Markdown("## Lineart Anime (The converted image is color-inverted.)") + with gr.Row(): + with gr.Column(): + input_image = gr.Image(source='upload', type="numpy") + resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64) + run_button = gr.Button(label="Run") + with gr.Column(): + gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto") + run_button.click(fn=lineart_anime, inputs=[input_image, resolution], outputs=[gallery]) + + with gr.Row(): + gr.Markdown("## Lineart (The converted image is color-inverted.)") + with gr.Row(): + with gr.Column(): + input_image = gr.Image(source='upload', type="numpy") + coarse = gr.Checkbox(label='Using coarse model', value=False) + resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64) + run_button = gr.Button(label="Run") + with gr.Column(): + gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto") + run_button.click(fn=lineart, inputs=[input_image, resolution, coarse], outputs=[gallery]) + +# with gr.Row(): +# gr.Markdown("## Uniformer Segmentation") +# with gr.Row(): +# with gr.Column(): +# input_image = gr.Image(source='upload', type="numpy") +# resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64) +# run_button = gr.Button(label="Run") +# with gr.Column(): +# gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto") +# run_button.click(fn=uniformer, inputs=[input_image, resolution], outputs=[gallery]) + + with gr.Row(): + gr.Markdown("## Oneformer COCO Segmentation") + with gr.Row(): + with gr.Column(): + input_image = gr.Image(source='upload', type="numpy") + resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64) + run_button = gr.Button(label="Run") + with gr.Column(): + gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto") + run_button.click(fn=oneformer_coco, inputs=[input_image, resolution], outputs=[gallery]) + + with gr.Row(): + gr.Markdown("## Oneformer ADE20K Segmentation") + with gr.Row(): + with gr.Column(): + input_image = gr.Image(source='upload', type="numpy") + resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=640, step=64) + run_button = gr.Button(label="Run") + with gr.Column(): + gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto") + run_button.click(fn=oneformer_ade20k, inputs=[input_image, resolution], outputs=[gallery]) + + with gr.Row(): + gr.Markdown("## Content Shuffle") + with gr.Row(): + with gr.Column(): + input_image = gr.Image(source='upload', type="numpy") + resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64) + run_button = gr.Button(label="Run") + with gr.Column(): + gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto") + run_button.click(fn=content_shuffler, inputs=[input_image, resolution], outputs=[gallery]) + + with gr.Row(): + gr.Markdown("## Color Shuffle") + with gr.Row(): + with gr.Column(): + input_image = gr.Image(source='upload', type="numpy") + resolution = gr.Slider(label="resolution", minimum=256, maximum=1024, value=512, step=64) + run_button = gr.Button(label="Run") + with gr.Column(): + gallery = gr.Gallery(label="Generated images", show_label=False).style(height="auto") + run_button.click(fn=color_shuffler, inputs=[input_image, resolution], outputs=[gallery]) + + +block.launch(server_name='0.0.0.0') diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..bff8857655956422fbff163ae89cf502223559fd --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +accelerate==0.20.3 +controlnet_aux==0.0.5 +einops==0.6.1 +#gradio==3.34.0 +opencv-python-headless==4.7.0.72 +torchvision==0.15.2 +basicsr +fvcore +omegaconf +pycocotools +ftfy +regex +timm==0.6.7